From d1e48baf90174784a31d4946d2d14e2d3047c858 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 27 Oct 2026 06:09:18 -0700 Subject: [PATCH 1/4] Import upstream v1.0.0, stripped Upstream commit: 47df4c0baeb8 Enterprise-only files removed or emptied: 68 Enterprise-only snippets removed: 2080 in 147 files Dangling module declarations removed: 533 Edits turning enterprise off: 12 Third-party code: 25 files, 14 not in THIRD-PARTY.md Renamed identifiers: 51 in 11 files Verification: clean The last upstream import. 1.0 has a new store layout that can't open a 0.16 store, renumbers the settings schema, ties the store's schema version to the edition, and moves calcard, smtp-proto, jmap-tools and sieve in-tree (AGPL-3.0-only OR LicenseRef-SEL; taken under the AGPL). The tests/ directory is gone upstream; main keeps its own. The build check is clean, and three files on the known list now compile: tests/src/directory/issuer.rs, tests/src/directory/mod.rs and tests/src/smtp/inbound/antispam.rs. The 14 new third-party files are the in-tree calcard, jmap-tools, smtp-proto and smtp-report code. --- .cargo/config.toml | 5 + .dockerignore | 1 - .github/workflows/ci.yml | 1 + .github/workflows/test.yml | 57 - .gitignore | 3 +- CHANGELOG.md | 55 + Cargo.lock | 2705 +- Cargo.toml | 56 +- Dockerfile | 2 +- Dockerfile.build | 20 +- Dockerfile.fdb | 4 +- LICENSES/Apache-2.0.txt | 201 + LICENSES/MIT.txt | 23 + LICENSES/Unlicense.txt | 24 + README.md | 56 +- SECURITY.md | 3 +- crates/calcard/Cargo.toml | 46 + crates/calcard/src/common/archive.rs | 88 + crates/calcard/src/common/blob.rs | 361 + crates/calcard/src/common/decode.rs | 101 + crates/calcard/src/common/elements.rs | 42 + crates/calcard/src/common/embedded.rs | 187 + crates/calcard/src/common/export.rs | 126 + crates/calcard/src/common/format.rs | 320 + crates/calcard/src/common/iana.rs | 101 + crates/calcard/src/common/jsprop.rs | 79 + crates/calcard/src/common/jsprop/ordered.rs | 174 + crates/calcard/src/common/jsprop/text.rs | 287 + crates/calcard/src/common/mod.rs | 294 + crates/calcard/src/common/parser.rs | 534 + crates/calcard/src/common/scan.rs | 108 + crates/calcard/src/common/stack.rs | 43 + crates/calcard/src/common/timezone.rs | 1052 + crates/calcard/src/common/tokenizer.rs | 784 + crates/calcard/src/common/types.rs | 467 + crates/calcard/src/common/tzdb.rs | 1238 + crates/calcard/src/common/writer.rs | 587 + crates/calcard/src/datecalc/error.rs | 117 + crates/calcard/src/datecalc/mod.rs | 20 + crates/calcard/src/datecalc/recurrence.rs | 2217 + crates/calcard/src/datecalc/rrule.rs | 152 + crates/calcard/src/datecalc/weekdate.rs | 171 + crates/calcard/src/icalendar/builder.rs | 760 + crates/calcard/src/icalendar/dates.rs | 861 + crates/calcard/src/icalendar/mod.rs | 978 + crates/calcard/src/icalendar/parser.rs | 1124 + crates/calcard/src/icalendar/rkyv_timezone.rs | 69 + crates/calcard/src/icalendar/rkyv_types.rs | 871 + crates/calcard/src/icalendar/rkyv_utils.rs | 478 + crates/calcard/src/icalendar/rkyv_writer.rs | 494 + crates/calcard/src/icalendar/timezone.rs | 700 + crates/calcard/src/icalendar/types.rs | 1512 + crates/calcard/src/icalendar/utils.rs | 583 + crates/calcard/src/icalendar/writer.rs | 524 + .../calcard/src/jscalendar/export/convert.rs | 3337 ++ crates/calcard/src/jscalendar/export/mod.rs | 211 + .../calcard/src/jscalendar/export/params.rs | 441 + crates/calcard/src/jscalendar/export/props.rs | 403 + crates/calcard/src/jscalendar/ext.rs | 317 + .../calcard/src/jscalendar/import/convert.rs | 2312 + crates/calcard/src/jscalendar/import/mod.rs | 249 + .../calcard/src/jscalendar/import/params.rs | 482 + crates/calcard/src/jscalendar/import/props.rs | 917 + crates/calcard/src/jscalendar/mod.rs | 508 + crates/calcard/src/jscalendar/overrides.rs | 693 + crates/calcard/src/jscalendar/parser.rs | 550 + crates/calcard/src/jscalendar/types.rs | 995 + .../calcard/src/jscontact/export/convert.rs | 2143 + crates/calcard/src/jscontact/export/entry.rs | 155 + crates/calcard/src/jscontact/export/mod.rs | 121 + crates/calcard/src/jscontact/export/params.rs | 365 + crates/calcard/src/jscontact/export/props.rs | 426 + .../calcard/src/jscontact/import/convert.rs | 1486 + crates/calcard/src/jscontact/import/entry.rs | 280 + crates/calcard/src/jscontact/import/mod.rs | 260 + crates/calcard/src/jscontact/import/params.rs | 410 + crates/calcard/src/jscontact/import/props.rs | 965 + crates/calcard/src/jscontact/mod.rs | 440 + crates/calcard/src/jscontact/parser.rs | 399 + crates/calcard/src/jscontact/types.rs | 602 + crates/calcard/src/lib.rs | 138 + crates/calcard/src/vcard/builder.rs | 348 + crates/calcard/src/vcard/media_type.rs | 327 + crates/calcard/src/vcard/mod.rs | 474 + crates/calcard/src/vcard/parser.rs | 901 + crates/calcard/src/vcard/rkyv_types.rs | 425 + crates/calcard/src/vcard/rkyv_utils.rs | 225 + crates/calcard/src/vcard/rkyv_writer.rs | 365 + crates/calcard/src/vcard/types.rs | 629 + crates/calcard/src/vcard/utils.rs | 430 + crates/calcard/src/vcard/writer.rs | 573 + crates/common/Cargo.toml | 84 +- crates/common/build.rs | 26 +- crates/common/src/auth/access_token.rs | 611 +- crates/common/src/auth/active_script.rs | 69 + crates/common/src/auth/authentication.rs | 819 +- crates/common/src/auth/credential.rs | 31 +- crates/common/src/auth/mod.rs | 145 +- crates/common/src/auth/oauth/cimd.rs | 412 + crates/common/src/auth/oauth/client.rs | 420 + crates/common/src/auth/oauth/client_id.rs | 159 +- crates/common/src/auth/oauth/config.rs | 237 +- crates/common/src/auth/oauth/crypto.rs | 1 + crates/common/src/auth/oauth/grant.rs | 462 + crates/common/src/auth/oauth/introspect.rs | 75 +- crates/common/src/auth/oauth/mod.rs | 15 +- crates/common/src/auth/oauth/oidc.rs | 26 +- crates/common/src/auth/oauth/registration.rs | 391 +- crates/common/src/auth/oauth/scope.rs | 544 + crates/common/src/auth/oauth/token.rs | 436 +- crates/common/src/auth/permissions.rs | 41 +- crates/common/src/auth/rate_limit.rs | 162 +- crates/common/src/auth/sasl.rs | 129 + crates/common/src/cache/directory.rs | 120 +- crates/common/src/cache/email.rs | 1924 + crates/common/src/cache/http_auth.rs | 182 + crates/common/src/cache/invalidate.rs | 416 +- crates/common/src/cache/large.rs | 714 + crates/common/src/cache/mod.rs | 167 +- crates/common/src/cache/principals.rs | 361 +- crates/common/src/cache/refresh.rs | 197 + crates/common/src/cache/reload.rs | 18 +- crates/common/src/cache/swap/blob.rs | 255 + crates/common/src/cache/swap/evict.rs | 94 + crates/common/src/cache/swap/file.rs | 157 + crates/common/src/cache/swap/frame.rs | 88 + crates/common/src/cache/swap/messages.rs | 328 + crates/common/src/cache/swap/mod.rs | 527 + crates/common/src/cache/swap/pointer.rs | 188 + crates/common/src/cache/swap/redis.rs | 244 + crates/common/src/cache/swap/resources.rs | 788 + crates/common/src/cache/swap/writer.rs | 611 + crates/common/src/config/groupware.rs | 132 +- crates/common/src/config/inner.rs | 189 +- .../src/config/mailstore/capabilities.rs | 120 +- crates/common/src/config/mailstore/email.rs | 70 +- crates/common/src/config/mailstore/imap.rs | 10 +- crates/common/src/config/mailstore/jmap.rs | 21 +- crates/common/src/config/mailstore/limits.rs | 199 + crates/common/src/config/mailstore/mod.rs | 1 + crates/common/src/config/mailstore/scripts.rs | 52 +- .../common/src/config/mailstore/spamfilter.rs | 369 +- crates/common/src/config/metadata.rs | 72 + crates/common/src/config/mod.rs | 182 +- crates/common/src/config/network.rs | 71 +- crates/common/src/config/resource.rs | 139 + crates/common/src/config/server/listener.rs | 23 +- crates/common/src/config/server/tls.rs | 25 +- crates/common/src/config/smtp/auth.rs | 267 +- crates/common/src/config/smtp/mod.rs | 41 +- crates/common/src/config/smtp/queue.rs | 58 +- crates/common/src/config/smtp/report.rs | 6 +- crates/common/src/config/smtp/resolver.rs | 108 +- crates/common/src/config/smtp/session.rs | 38 +- crates/common/src/config/storage.rs | 4 - crates/common/src/config/telemetry.rs | 214 +- crates/common/src/expr/capture.rs | 121 + crates/common/src/expr/eval.rs | 1179 +- crates/common/src/expr/functions/array.rs | 131 +- crates/common/src/expr/functions/asynch.rs | 579 +- crates/common/src/expr/functions/email.rs | 65 +- crates/common/src/expr/functions/misc.rs | 86 +- crates/common/src/expr/functions/mod.rs | 232 +- crates/common/src/expr/functions/text.rs | 553 +- crates/common/src/expr/guard.rs | 350 + crates/common/src/expr/if_block.rs | 204 +- crates/common/src/expr/kernels/case.rs | 48 + crates/common/src/expr/kernels/encode.rs | 27 + crates/common/src/expr/kernels/mod.rs | 17 + crates/common/src/expr/mod.rs | 167 +- crates/common/src/expr/parser.rs | 45 +- crates/common/src/expr/program.rs | 940 + crates/common/src/expr/tokenizer.rs | 435 +- crates/common/src/expr/variable.rs | 426 + crates/common/src/i18n.rs | 68 - crates/common/src/ipc.rs | 190 +- crates/common/src/ipc/subscribers.rs | 643 + crates/common/src/lib.rs | 409 +- crates/common/src/manager/app_client.rs | 411 + crates/common/src/manager/application.rs | 771 +- crates/common/src/manager/backup.rs | 105 +- crates/common/src/manager/boot.rs | 72 +- crates/common/src/manager/console.rs | 170 +- crates/common/src/manager/defaults.rs | 391 +- crates/common/src/manager/mod.rs | 26 +- crates/common/src/manager/restore.rs | 33 +- crates/common/src/manager/schema.rs | 120 + crates/common/src/network/acme/account.rs | 29 +- crates/common/src/network/acme/directory.rs | 64 +- crates/common/src/network/acme/http.rs | 163 +- crates/common/src/network/acme/jose.rs | 25 +- crates/common/src/network/acme/mod.rs | 53 +- crates/common/src/network/acme/order.rs | 194 +- crates/common/src/network/acme/persist.rs | 195 + crates/common/src/network/acme/renew.rs | 68 +- crates/common/src/network/acme/resolver.rs | 18 +- crates/common/src/network/asn.rs | 196 +- crates/common/src/network/asn/table.rs | 389 + .../src/network/autoconfig/autodiscover.rs | 238 +- .../network/autoconfig/legacy_autoconfig.rs | 178 +- crates/common/src/network/autoconfig/pacc.rs | 206 - crates/common/src/network/dkim.rs | 191 +- crates/common/src/network/dns/records.rs | 150 +- crates/common/src/network/dns/resolve.rs | 8 +- crates/common/src/network/dns/update.rs | 352 +- crates/common/src/network/forwarded.rs | 171 + crates/common/src/network/limiter.rs | 369 +- crates/common/src/network/listen.rs | 133 +- crates/common/src/network/memory.rs | 192 + crates/common/src/network/mod.rs | 216 +- crates/common/src/network/mta.rs | 124 +- crates/common/src/network/security.rs | 250 +- crates/common/src/network/tls.rs | 64 +- crates/common/src/network/webpush.rs | 296 +- crates/common/src/scripts/functions/array.rs | 47 +- crates/common/src/scripts/functions/email.rs | 16 +- crates/common/src/scripts/functions/header.rs | 79 +- crates/common/src/scripts/functions/image.rs | 110 +- crates/common/src/scripts/functions/misc.rs | 42 +- crates/common/src/scripts/functions/mod.rs | 82 +- crates/common/src/scripts/functions/sniff.rs | 110 + crates/common/src/scripts/functions/text.rs | 208 +- .../common/src/scripts/functions/unicode.rs | 12 +- crates/common/src/scripts/functions/url.rs | 6 +- crates/common/src/scripts/mod.rs | 25 +- crates/common/src/scripts/plugins/dns.rs | 18 +- crates/common/src/scripts/plugins/exec.rs | 9 +- crates/common/src/scripts/plugins/headers.rs | 29 +- crates/common/src/scripts/plugins/http.rs | 4 +- .../common/src/scripts/plugins/llm_prompt.rs | 8 +- crates/common/src/scripts/plugins/lookup.rs | 47 +- crates/common/src/scripts/plugins/mod.rs | 17 +- crates/common/src/scripts/plugins/query.rs | 15 +- crates/common/src/scripts/plugins/text.rs | 29 +- crates/common/src/sharing/acl.rs | 92 - crates/common/src/sharing/file.rs | 214 + crates/common/src/sharing/grants.rs | 92 + crates/common/src/sharing/mod.rs | 3 +- crates/common/src/sharing/notification.rs | 26 +- crates/common/src/sharing/resources.rs | 208 +- crates/common/src/storage/archive.rs | 322 +- crates/common/src/storage/batch.rs | 203 + crates/common/src/storage/blob.rs | 290 +- crates/common/src/storage/chunk.rs | 160 + crates/common/src/storage/dav.rs | 403 - crates/common/src/storage/dav/file.rs | 205 + crates/common/src/storage/dav/hierarchy.rs | 256 + crates/common/src/storage/dav/mod.rs | 283 + crates/common/src/storage/dav/paths.rs | 379 + crates/common/src/storage/dav/presence.rs | 133 + crates/common/src/storage/dav/resource.rs | 680 + crates/common/src/storage/dav/store.rs | 673 + crates/common/src/storage/encryption.rs | 262 +- crates/common/src/storage/index.rs | 784 - crates/common/src/storage/index/build.rs | 464 + crates/common/src/storage/index/mod.rs | 164 + crates/common/src/storage/index/object.rs | 246 + crates/common/src/storage/index/presence.rs | 80 + crates/common/src/storage/index/split.rs | 261 + crates/common/src/storage/index/value.rs | 223 + crates/common/src/storage/metadata/change.rs | 94 + crates/common/src/storage/metadata/cleanup.rs | 339 + .../common/src/storage/metadata/containers.rs | 98 + crates/common/src/storage/metadata/mod.rs | 445 + crates/common/src/storage/metadata/purge.rs | 321 + crates/common/src/storage/metadata/quota.rs | 84 + crates/common/src/storage/metadata/read.rs | 286 + crates/common/src/storage/metadata/stored.rs | 196 + crates/common/src/storage/metadata/viewer.rs | 402 + crates/common/src/storage/mod.rs | 70 +- crates/common/src/storage/quota.rs | 215 +- crates/common/src/storage/search.rs | 185 + crates/common/src/storage/state.rs | 121 +- crates/common/src/storage/transaction.rs | 236 +- crates/common/src/telemetry/metrics/mod.rs | 21 + crates/common/src/telemetry/metrics/otel.rs | 5 +- .../src/telemetry/metrics/prometheus.rs | 4 + crates/common/src/telemetry/mod.rs | 26 +- .../common/src/telemetry/tracers/journald.rs | 103 +- crates/common/src/telemetry/tracers/log.rs | 216 +- crates/common/src/telemetry/tracers/mod.rs | 101 +- crates/common/src/telemetry/tracers/otel.rs | 128 +- crates/common/src/telemetry/tracers/spans.rs | 124 + crates/common/src/telemetry/tracers/stdout.rs | 111 +- crates/common/src/telemetry/webhooks/mod.rs | 321 +- crates/coordinator/Cargo.toml | 21 +- crates/coordinator/src/backend/kafka/mod.rs | 35 +- .../coordinator/src/backend/kafka/pubsub.rs | 72 +- crates/coordinator/src/backend/nats/mod.rs | 21 +- crates/coordinator/src/backend/nats/pubsub.rs | 39 +- .../coordinator/src/backend/redis/pubsub.rs | 135 +- crates/coordinator/src/backend/zenoh/mod.rs | 3 +- .../coordinator/src/backend/zenoh/pubsub.rs | 89 +- crates/coordinator/src/bootstrap.rs | 21 +- crates/coordinator/src/dispatch.rs | 99 +- crates/coordinator/src/lib.rs | 45 +- crates/dav-proto/Cargo.toml | 24 +- .../dav-proto/resources/requests/acl-001.json | 30 - .../dav-proto/resources/requests/acl-001.xml | 15 - .../dav-proto/resources/requests/acl-002.json | 17 - .../dav-proto/resources/requests/acl-002.xml | 9 - .../dav-proto/resources/requests/acl-003.json | 17 - .../dav-proto/resources/requests/acl-003.xml | 15 - .../dav-proto/resources/requests/acl-004.json | 53 - .../dav-proto/resources/requests/acl-004.xml | 27 - .../resources/requests/lockinfo-001.json | 20 - .../resources/requests/lockinfo-001.xml | 8 - .../resources/requests/lockinfo-002.json | 20 - .../resources/requests/lockinfo-002.xml | 9 - .../resources/requests/mkcol-001.json | 41 - .../resources/requests/mkcol-001.xml | 15 - .../resources/requests/mkcol-002.json | 369 - .../resources/requests/mkcol-002.xml | 37 - .../resources/requests/mkcol-003.json | 29 - .../resources/requests/mkcol-003.xml | 14 - .../resources/requests/mkcol-004.json | 30 - .../resources/requests/mkcol-004.xml | 13 - .../requests/propertyupdate-001.json | 398 - .../resources/requests/propertyupdate-001.xml | 55 - .../requests/propertyupdate-002.json | 176 - .../resources/requests/propertyupdate-002.xml | 11 - .../resources/requests/propfind-001.json | 33 - .../resources/requests/propfind-001.xml | 9 - .../resources/requests/propfind-002.json | 3 - .../resources/requests/propfind-002.xml | 4 - .../resources/requests/propfind-003.json | 4 - .../resources/requests/propfind-003.xml | 4 - .../resources/requests/propfind-004.json | 11 - .../resources/requests/propfind-004.xml | 8 - .../resources/requests/propfind-005.json | 11 - .../resources/requests/propfind-005.xml | 5 - .../resources/requests/propfind-006.json | 17 - .../resources/requests/propfind-006.xml | 6 - .../resources/requests/propfind-007.json | 36 - .../resources/requests/propfind-007.xml | 10 - .../resources/requests/propfind-008.json | 29 - .../resources/requests/propfind-008.xml | 8 - .../resources/requests/propfind-009.json | 0 .../resources/requests/propfind-009.xml | 1 - .../resources/requests/propfind-010.json | 36 - .../resources/requests/propfind-010.xml | 1 - .../resources/requests/report-001.json | 128 - .../resources/requests/report-001.xml | 34 - .../resources/requests/report-002.json | 42 - .../resources/requests/report-002.xml | 17 - .../resources/requests/report-003.json | 42 - .../resources/requests/report-003.xml | 19 - .../resources/requests/report-004.json | 42 - .../resources/requests/report-004.xml | 18 - .../resources/requests/report-005.json | 46 - .../resources/requests/report-005.xml | 17 - .../resources/requests/report-006.json | 51 - .../resources/requests/report-006.xml | 17 - .../resources/requests/report-007.json | 72 - .../resources/requests/report-007.xml | 21 - .../resources/requests/report-008.json | 41 - .../resources/requests/report-008.xml | 14 - .../resources/requests/report-009.json | 64 - .../resources/requests/report-009.xml | 20 - .../resources/requests/report-010.json | 52 - .../resources/requests/report-010.xml | 17 - .../resources/requests/report-011.json | 33 - .../resources/requests/report-011.xml | 15 - .../resources/requests/report-012.json | 30 - .../resources/requests/report-012.xml | 11 - .../resources/requests/report-013.json | 30 - .../resources/requests/report-013.xml | 10 - .../resources/requests/report-014.json | 82 - .../resources/requests/report-014.xml | 22 - .../resources/requests/report-015.json | 105 - .../resources/requests/report-015.xml | 26 - .../resources/requests/report-016.json | 40 - .../resources/requests/report-016.xml | 17 - .../resources/requests/report-017.json | 63 - .../resources/requests/report-017.xml | 16 - .../resources/requests/report-018.json | 26 - .../resources/requests/report-018.xml | 9 - .../resources/requests/report-019.json | 17 - .../resources/requests/report-019.xml | 10 - .../resources/requests/report-020.json | 11 - .../resources/requests/report-020.xml | 6 - .../resources/requests/report-021.json | 14 - .../resources/requests/report-021.xml | 7 - .../resources/requests/report-022.json | 61 - .../resources/requests/report-022.xml | 22 - .../resources/requests/report-023.json | 3 - .../resources/requests/report-023.xml | 3 - .../resources/requests/report-024.json | 45 - .../resources/requests/report-024.xml | 9 - .../resources/requests/report-025.json | 83 - .../resources/requests/report-025.xml | 12 - crates/dav-proto/resources/responses/001.xml | 6 - crates/dav-proto/resources/responses/002.xml | 18 - crates/dav-proto/resources/responses/003.xml | 50 - crates/dav-proto/resources/responses/004.xml | 8 - crates/dav-proto/resources/responses/005.xml | 22 - crates/dav-proto/resources/responses/006.xml | 27 - crates/dav-proto/resources/responses/007.xml | 6 - crates/dav-proto/resources/responses/008.xml | 43 - crates/dav-proto/resources/responses/009.xml | 11 - crates/dav-proto/resources/responses/010.xml | 20 - crates/dav-proto/resources/responses/011.xml | 27 - crates/dav-proto/resources/responses/012.xml | 12 - crates/dav-proto/resources/responses/013.xml | 15 - crates/dav-proto/resources/responses/014.xml | 55 - crates/dav-proto/resources/responses/015.xml | 14 - crates/dav-proto/resources/responses/016.xml | 29 - crates/dav-proto/resources/responses/017.xml | 17 - crates/dav-proto/resources/responses/018.xml | 15 - crates/dav-proto/resources/responses/019.xml | 80 - crates/dav-proto/resources/responses/020.xml | 23 - crates/dav-proto/src/lib.rs | 13 +- crates/dav-proto/src/parser/header.rs | 542 +- crates/dav-proto/src/parser/mod.rs | 45 +- crates/dav-proto/src/parser/property.rs | 73 +- crates/dav-proto/src/parser/tokenizer.rs | 378 +- crates/dav-proto/src/parser/value.rs | 314 + crates/dav-proto/src/requests/acl.rs | 13 +- crates/dav-proto/src/requests/filter.rs | 489 + crates/dav-proto/src/requests/lockinfo.rs | 10 +- crates/dav-proto/src/requests/mkcol.rs | 20 +- crates/dav-proto/src/requests/mod.rs | 189 +- .../dav-proto/src/requests/propertyupdate.rs | 31 +- crates/dav-proto/src/requests/propfind.rs | 89 +- crates/dav-proto/src/requests/report.rs | 364 +- crates/dav-proto/src/responses/error.rs | 21 +- crates/dav-proto/src/responses/lock.rs | 24 +- crates/dav-proto/src/responses/mod.rs | 629 +- crates/dav-proto/src/responses/property.rs | 221 +- crates/dav-proto/src/responses/propstat.rs | 2 +- crates/dav-proto/src/schema/collation.rs | 210 + crates/dav-proto/src/schema/mod.rs | 160 +- crates/dav-proto/src/schema/property.rs | 132 +- crates/dav-proto/src/schema/request.rs | 338 +- crates/dav-proto/src/schema/response.rs | 154 +- crates/dav/Cargo.toml | 16 +- crates/dav/src/calendar/copy_move.rs | 1161 +- crates/dav/src/calendar/delete.rs | 172 +- crates/dav/src/calendar/filter.rs | 375 + crates/dav/src/calendar/freebusy.rs | 466 +- crates/dav/src/calendar/get.rs | 104 +- crates/dav/src/calendar/mkcol.rs | 95 +- crates/dav/src/calendar/mod.rs | 404 +- crates/dav/src/calendar/proppatch.rs | 389 +- crates/dav/src/calendar/query.rs | 812 +- crates/dav/src/calendar/scheduling.rs | 180 +- crates/dav/src/calendar/update.rs | 604 +- crates/dav/src/calendar/view.rs | 550 + crates/dav/src/card/copy_move.rs | 704 +- crates/dav/src/card/delete.rs | 98 +- crates/dav/src/card/filter.rs | 236 + crates/dav/src/card/get.rs | 71 +- crates/dav/src/card/mkcol.rs | 88 +- crates/dav/src/card/mod.rs | 67 +- crates/dav/src/card/proppatch.rs | 346 +- crates/dav/src/card/query.rs | 187 +- crates/dav/src/card/update.rs | 187 +- crates/dav/src/common/acl.rs | 263 +- crates/dav/src/common/dead/containers.rs | 262 + crates/dav/src/common/dead/mod.rs | 59 + crates/dav/src/common/dead/patch.rs | 352 + crates/dav/src/common/lock.rs | 456 +- crates/dav/src/common/mod.rs | 504 +- crates/dav/src/common/propfind.rs | 1825 - crates/dav/src/common/propfind/data.rs | 205 + crates/dav/src/common/propfind/dead.rs | 138 + crates/dav/src/common/propfind/get.rs | 656 + crates/dav/src/common/propfind/item.rs | 803 + crates/dav/src/common/propfind/load.rs | 250 + crates/dav/src/common/propfind/mod.rs | 857 + crates/dav/src/common/search.rs | 353 + crates/dav/src/common/uri.rs | 159 +- crates/dav/src/file/copy_move.rs | 846 +- crates/dav/src/file/delete.rs | 58 +- crates/dav/src/file/get.rs | 41 +- crates/dav/src/file/mkcol.rs | 112 +- crates/dav/src/file/mod.rs | 171 +- crates/dav/src/file/proppatch.rs | 239 +- crates/dav/src/file/update.rs | 182 +- crates/dav/src/lib.rs | 95 +- crates/dav/src/principal/propfind.rs | 38 +- crates/dav/src/principal/propsearch.rs | 98 +- crates/dav/src/request.rs | 245 +- crates/directory/Cargo.toml | 19 +- crates/directory/src/backend/ldap/config.rs | 67 +- crates/directory/src/backend/ldap/failover.rs | 123 + crates/directory/src/backend/ldap/lookup.rs | 277 +- crates/directory/src/backend/ldap/mod.rs | 67 +- crates/directory/src/backend/ldap/pool.rs | 149 +- crates/directory/src/backend/oidc/config.rs | 538 +- crates/directory/src/backend/oidc/jwks.rs | 304 + crates/directory/src/backend/oidc/lookup.rs | 702 +- crates/directory/src/backend/oidc/mod.rs | 282 +- crates/directory/src/backend/sql/lookup.rs | 150 +- crates/directory/src/core/dispatch.rs | 38 +- crates/directory/src/core/mod.rs | 1 + crates/directory/src/core/sasl.rs | 86 +- crates/directory/src/core/secret.rs | 599 +- crates/directory/src/core/verifier.rs | 105 + crates/email/Cargo.toml | 33 +- crates/email/src/cache/email.rs | 717 +- crates/email/src/cache/mailbox.rs | 484 +- crates/email/src/cache/mod.rs | 719 +- crates/email/src/identity/index.rs | 16 +- crates/email/src/lib.rs | 1 + crates/email/src/mailbox/destroy.rs | 213 +- crates/email/src/mailbox/index.rs | 57 +- crates/email/src/mailbox/lookup.rs | 326 + crates/email/src/mailbox/manage.rs | 91 +- crates/email/src/mailbox/mod.rs | 88 +- crates/email/src/mailbox/role.rs | 66 + crates/email/src/message/body.rs | 303 - crates/email/src/message/copy.rs | 303 +- crates/email/src/message/copy/sources.rs | 214 + crates/email/src/message/crypto.rs | 156 +- crates/email/src/message/delete.rs | 284 +- crates/email/src/message/delivery.rs | 47 +- crates/email/src/message/headers.rs | 344 +- crates/email/src/message/index/attachment.rs | 202 + crates/email/src/message/index/extractors.rs | 272 +- crates/email/src/message/index/metadata.rs | 256 +- crates/email/src/message/index/mod.rs | 160 +- crates/email/src/message/index/search.rs | 594 +- crates/email/src/message/index/subject.rs | 50 + crates/email/src/message/ingest.rs | 1003 +- crates/email/src/message/ingest/checks.rs | 137 + crates/email/src/message/ingest/group.rs | 163 + crates/email/src/message/ingest/recheck.rs | 262 + crates/email/src/message/ingest_metadata.rs | 51 + crates/email/src/message/jmap/body.rs | 229 + crates/email/src/message/jmap/estimate.rs | 221 + crates/email/src/message/jmap/header.rs | 239 + crates/email/src/message/jmap/mod.rs | 254 + crates/email/src/message/messagedata.rs | 801 + crates/email/src/message/metadata.rs | 1320 - crates/email/src/message/metadata/build.rs | 817 + .../email/src/message/metadata/header_id.rs | 255 + crates/email/src/message/metadata/mod.rs | 416 + crates/email/src/message/metadata/row.rs | 213 + crates/email/src/message/metadata/source.rs | 313 + crates/email/src/message/metadata/view.rs | 1049 + crates/email/src/message/mod.rs | 5 + crates/email/src/message/sortkeys.rs | 560 + crates/email/src/message/thread.rs | 113 + crates/email/src/presence.rs | 162 + crates/email/src/push/mod.rs | 5 + crates/email/src/sieve/delete.rs | 32 +- crates/email/src/sieve/index.rs | 28 +- crates/email/src/sieve/ingest.rs | 1368 +- crates/email/src/sieve/mod.rs | 118 +- crates/email/src/submission/index.rs | 23 +- crates/email/src/submission/mod.rs | 108 +- crates/groupware/Cargo.toml | 20 +- crates/groupware/src/cache/calcard.rs | 646 +- crates/groupware/src/cache/file.rs | 310 +- crates/groupware/src/cache/mod.rs | 937 +- crates/groupware/src/calendar/alarm.rs | 1178 +- crates/groupware/src/calendar/alerts.rs | 882 + crates/groupware/src/calendar/availability.rs | 423 + crates/groupware/src/calendar/color.rs | 236 + crates/groupware/src/calendar/compare.rs | 417 + crates/groupware/src/calendar/dates.rs | 234 +- crates/groupware/src/calendar/expand.rs | 760 +- crates/groupware/src/calendar/hidden.rs | 547 + crates/groupware/src/calendar/identity.rs | 468 + crates/groupware/src/calendar/index.rs | 581 +- .../groupware/src/calendar/instance_filter.rs | 338 + crates/groupware/src/calendar/itip.rs | 471 +- crates/groupware/src/calendar/mod.rs | 419 +- crates/groupware/src/calendar/notification.rs | 521 + crates/groupware/src/calendar/overlap.rs | 188 + crates/groupware/src/calendar/participants.rs | 639 + crates/groupware/src/calendar/privacy.rs | 285 + crates/groupware/src/calendar/rights.rs | 941 + crates/groupware/src/calendar/schedule.rs | 721 + crates/groupware/src/calendar/sequence.rs | 129 + crates/groupware/src/calendar/storage.rs | 1281 +- crates/groupware/src/calendar/user.rs | 1344 + crates/groupware/src/contact/index.rs | 315 +- crates/groupware/src/contact/mod.rs | 44 +- crates/groupware/src/contact/storage.rs | 361 +- crates/groupware/src/file/content.rs | 110 + crates/groupware/src/file/index.rs | 111 +- crates/groupware/src/file/mod.rs | 292 +- crates/groupware/src/file/storage.rs | 194 +- crates/groupware/src/file/symlink.rs | 84 + crates/groupware/src/lib.rs | 229 +- crates/groupware/src/metadata/mod.rs | 120 + crates/groupware/src/scheduling/attendee.rs | 161 +- .../groupware/src/scheduling/event_cancel.rs | 147 +- .../groupware/src/scheduling/event_create.rs | 29 +- .../groupware/src/scheduling/event_update.rs | 23 +- crates/groupware/src/scheduling/format.rs | 227 +- crates/groupware/src/scheduling/inbound.rs | 602 +- crates/groupware/src/scheduling/itip.rs | 92 +- crates/groupware/src/scheduling/mod.rs | 92 +- crates/groupware/src/scheduling/organizer.rs | 403 +- crates/groupware/src/scheduling/recipient.rs | 75 + .../src/scheduling/recipient/series.rs | 483 + .../scheduling/recipient/series/occurrence.rs | 287 + .../src/scheduling/recipient/split.rs | 886 + crates/groupware/src/scheduling/snapshot.rs | 355 +- crates/http-proto/Cargo.toml | 7 +- crates/http-proto/src/body.rs | 61 + crates/http-proto/src/context.rs | 55 +- crates/http-proto/src/lib.rs | 9 +- crates/http-proto/src/request.rs | 51 +- crates/http-proto/src/response.rs | 56 +- crates/http/Cargo.toml | 25 +- crates/http/src/api/diagnose.rs | 78 +- crates/http/src/api/mod.rs | 101 +- crates/http/src/auth/authenticate.rs | 83 +- crates/http/src/auth/oauth/auth.rs | 906 +- crates/http/src/auth/oauth/authorize.rs | 499 + crates/http/src/auth/oauth/logout.rs | 177 + crates/http/src/auth/oauth/mod.rs | 58 +- crates/http/src/auth/oauth/openid.rs | 69 +- crates/http/src/auth/oauth/registration.rs | 319 +- crates/http/src/auth/oauth/token.rs | 978 +- crates/http/src/auth/permissions.rs | 109 +- crates/http/src/form/mod.rs | 26 +- crates/http/src/lib.rs | 1 + crates/http/src/request.rs | 508 +- crates/http/src/stream.rs | 67 + crates/imap-proto/Cargo.toml | 12 +- crates/imap-proto/src/lib.rs | 83 +- crates/imap-proto/src/parser/acl.rs | 122 +- crates/imap-proto/src/parser/append.rs | 247 +- crates/imap-proto/src/parser/authenticate.rs | 71 +- crates/imap-proto/src/parser/copy_move.rs | 63 +- crates/imap-proto/src/parser/create.rs | 151 +- crates/imap-proto/src/parser/delete.rs | 54 +- crates/imap-proto/src/parser/enable.rs | 113 +- crates/imap-proto/src/parser/fetch.rs | 646 +- crates/imap-proto/src/parser/list.rs | 276 +- crates/imap-proto/src/parser/login.rs | 43 +- crates/imap-proto/src/parser/lsub.rs | 62 +- crates/imap-proto/src/parser/metadata.rs | 233 + crates/imap-proto/src/parser/mod.rs | 586 +- crates/imap-proto/src/parser/quota.rs | 62 +- crates/imap-proto/src/parser/rename.rs | 66 +- crates/imap-proto/src/parser/search.rs | 376 +- crates/imap-proto/src/parser/select.rs | 350 +- crates/imap-proto/src/parser/sort.rs | 171 +- crates/imap-proto/src/parser/status.rs | 104 +- crates/imap-proto/src/parser/store.rs | 154 +- crates/imap-proto/src/parser/subscribe.rs | 54 +- crates/imap-proto/src/parser/thread.rs | 72 +- crates/imap-proto/src/parser/uidbatches.rs | 84 +- crates/imap-proto/src/protocol/acl.rs | 207 +- crates/imap-proto/src/protocol/append.rs | 5 +- .../imap-proto/src/protocol/authenticate.rs | 15 +- crates/imap-proto/src/protocol/capability.rs | 73 +- crates/imap-proto/src/protocol/copy_move.rs | 5 +- crates/imap-proto/src/protocol/create.rs | 5 +- crates/imap-proto/src/protocol/delete.rs | 5 +- crates/imap-proto/src/protocol/enable.rs | 22 +- crates/imap-proto/src/protocol/expunge.rs | 158 +- crates/imap-proto/src/protocol/fetch.rs | 1370 +- crates/imap-proto/src/protocol/list.rs | 306 +- crates/imap-proto/src/protocol/login.rs | 7 +- crates/imap-proto/src/protocol/metadata.rs | 177 + crates/imap-proto/src/protocol/mod.rs | 768 +- crates/imap-proto/src/protocol/namespace.rs | 16 +- crates/imap-proto/src/protocol/quota.rs | 106 +- crates/imap-proto/src/protocol/rename.rs | 7 +- crates/imap-proto/src/protocol/search.rs | 113 +- crates/imap-proto/src/protocol/select.rs | 242 +- crates/imap-proto/src/protocol/status.rs | 56 +- crates/imap-proto/src/protocol/store.rs | 14 +- crates/imap-proto/src/protocol/subscribe.rs | 5 +- crates/imap-proto/src/protocol/thread.rs | 41 +- crates/imap-proto/src/protocol/uidbatches.rs | 46 +- crates/imap-proto/src/receiver.rs | 1242 +- crates/imap-proto/src/utf7.rs | 189 - crates/imap/Cargo.toml | 14 +- crates/imap/src/core/client.rs | 215 +- crates/imap/src/core/mailbox.rs | 1098 +- crates/imap/src/core/message.rs | 332 - crates/imap/src/core/mod.rs | 465 +- crates/imap/src/core/rights.rs | 65 + crates/imap/src/core/session.rs | 77 +- crates/imap/src/core/view.rs | 885 + crates/imap/src/lib.rs | 39 +- crates/imap/src/op/acl.rs | 419 +- crates/imap/src/op/append.rs | 274 +- crates/imap/src/op/authenticate.rs | 267 +- crates/imap/src/op/capability.rs | 39 +- crates/imap/src/op/close.rs | 42 +- crates/imap/src/op/copy_move.rs | 594 +- crates/imap/src/op/create.rs | 161 +- crates/imap/src/op/delete.rs | 52 +- crates/imap/src/op/enable.rs | 41 +- crates/imap/src/op/expunge.rs | 306 +- crates/imap/src/op/fetch.rs | 1279 - crates/imap/src/op/fetch/mod.rs | 844 + crates/imap/src/op/fetch/section.rs | 401 + crates/imap/src/op/fetch/source.rs | 189 + crates/imap/src/op/fetch/structure.rs | 737 + crates/imap/src/op/idle.rs | 477 +- crates/imap/src/op/list.rs | 285 +- crates/imap/src/op/login.rs | 1 + crates/imap/src/op/metadata/edit.rs | 110 + crates/imap/src/op/metadata/get.rs | 173 + crates/imap/src/op/metadata/list.rs | 142 + crates/imap/src/op/metadata/mod.rs | 231 + crates/imap/src/op/metadata/select.rs | 202 + crates/imap/src/op/metadata/set.rs | 491 + crates/imap/src/op/mod.rs | 37 +- crates/imap/src/op/namespace.rs | 23 +- crates/imap/src/op/noop.rs | 17 +- crates/imap/src/op/quota.rs | 43 +- crates/imap/src/op/rename.rs | 126 +- crates/imap/src/op/search.rs | 577 +- crates/imap/src/op/select.rs | 227 +- crates/imap/src/op/status.rs | 296 +- crates/imap/src/op/store.rs | 571 +- crates/imap/src/op/subscribe.rs | 101 +- crates/imap/src/op/thread.rs | 57 +- crates/imap/src/op/uidbatches.rs | 87 +- crates/jmap-proto/Cargo.toml | 18 +- crates/jmap-proto/src/error/method.rs | 24 + crates/jmap-proto/src/error/request.rs | 59 - crates/jmap-proto/src/error/set.rs | 14 +- crates/jmap-proto/src/method/availability.rs | 21 +- crates/jmap-proto/src/method/changes.rs | 6 +- crates/jmap-proto/src/method/copy.rs | 97 +- crates/jmap-proto/src/method/get.rs | 59 +- crates/jmap-proto/src/method/import.rs | 7 +- crates/jmap-proto/src/method/mod.rs | 4 +- crates/jmap-proto/src/method/query.rs | 9 +- crates/jmap-proto/src/method/set.rs | 60 +- crates/jmap-proto/src/method/upload.rs | 8 +- crates/jmap-proto/src/object/addressbook.rs | 225 +- crates/jmap-proto/src/object/blob.rs | 48 +- crates/jmap-proto/src/object/calendar.rs | 287 +- .../jmap-proto/src/object/calendar_event.rs | 133 +- .../src/object/calendar_event_notification.rs | 130 +- crates/jmap-proto/src/object/contact.rs | 75 +- crates/jmap-proto/src/object/email.rs | 456 +- .../jmap-proto/src/object/email_submission.rs | 200 +- crates/jmap-proto/src/object/file_node.rs | 329 +- crates/jmap-proto/src/object/identity.rs | 104 +- crates/jmap-proto/src/object/mailbox.rs | 232 +- crates/jmap-proto/src/object/metadata.rs | 504 + crates/jmap-proto/src/object/mod.rs | 10 + .../src/object/participant_identity.rs | 30 +- crates/jmap-proto/src/object/principal.rs | 112 +- .../src/object/push_subscription.rs | 109 +- crates/jmap-proto/src/object/quota.rs | 50 +- crates/jmap-proto/src/object/registry.rs | 8 +- .../jmap-proto/src/object/search_snippet.rs | 28 +- .../src/object/share_notification.rs | 64 +- crates/jmap-proto/src/object/sieve.rs | 156 +- crates/jmap-proto/src/object/thread.rs | 26 +- .../src/object/vacation_response.rs | 41 +- crates/jmap-proto/src/references/eval.rs | 85 +- crates/jmap-proto/src/references/jsptr.rs | 107 +- crates/jmap-proto/src/references/mod.rs | 923 +- crates/jmap-proto/src/references/resolve.rs | 42 +- crates/jmap-proto/src/request/capability.rs | 127 +- crates/jmap-proto/src/request/deserialize.rs | 83 +- crates/jmap-proto/src/request/method.rs | 35 +- crates/jmap-proto/src/request/mod.rs | 5 +- crates/jmap-proto/src/request/parser.rs | 1290 +- crates/jmap-proto/src/request/reference.rs | 10 + crates/jmap-proto/src/request/websocket.rs | 72 +- crates/jmap-proto/src/response/mod.rs | 13 +- crates/jmap-proto/src/response/serialize.rs | 5 +- crates/jmap-proto/src/types/date.rs | 139 +- crates/jmap-proto/src/types/state.rs | 114 +- crates/jmap-tools/Cargo.toml | 19 + crates/jmap-tools/src/json/de.rs | 320 + crates/jmap-tools/src/json/deserializer.rs | 355 + crates/jmap-tools/src/json/index.rs | 37 + crates/jmap-tools/src/json/key.rs | 290 + crates/jmap-tools/src/json/mod.rs | 17 + crates/jmap-tools/src/json/num.rs | 210 + crates/jmap-tools/src/json/object_vec.rs | 343 + crates/jmap-tools/src/json/parser.rs | 755 + crates/jmap-tools/src/json/rkyv.rs | 42 + crates/jmap-tools/src/json/ser.rs | 49 + crates/jmap-tools/src/json/value.rs | 599 + crates/jmap-tools/src/lib.rs | 17 + crates/jmap-tools/src/pointer/eval.rs | 319 + crates/jmap-tools/src/pointer/mod.rs | 221 + crates/jmap-tools/src/pointer/parser.rs | 235 + crates/jmap/Cargo.toml | 39 +- crates/jmap/src/addressbook/get.rs | 109 +- crates/jmap/src/addressbook/set.rs | 459 +- crates/jmap/src/api/acl.rs | 180 +- crates/jmap/src/api/auth.rs | 95 +- crates/jmap/src/api/budget.rs | 65 + crates/jmap/src/api/event_source.rs | 38 +- crates/jmap/src/api/metadata/apply.rs | 434 + crates/jmap/src/api/metadata/copy.rs | 61 + crates/jmap/src/api/metadata/filter.rs | 77 + crates/jmap/src/api/metadata/get.rs | 236 + crates/jmap/src/api/metadata/mod.rs | 173 + crates/jmap/src/api/metadata/object.rs | 286 + crates/jmap/src/api/metadata/patch.rs | 359 + crates/jmap/src/api/metadata/preload.rs | 88 + crates/jmap/src/api/metadata/prepared.rs | 66 + crates/jmap/src/api/metadata/query.rs | 307 + crates/jmap/src/api/metadata/reject.rs | 25 + crates/jmap/src/api/metadata/validate.rs | 187 + crates/jmap/src/api/metadata/violation.rs | 87 + crates/jmap/src/api/metadata/write.rs | 118 + crates/jmap/src/api/metadata/writer.rs | 301 + crates/jmap/src/api/mod.rs | 38 +- crates/jmap/src/api/parent_ref.rs | 142 + crates/jmap/src/api/pending_creates.rs | 43 + crates/jmap/src/api/query.rs | 104 +- crates/jmap/src/api/request.rs | 240 +- crates/jmap/src/api/session.rs | 210 +- crates/jmap/src/blob/copy.rs | 55 +- crates/jmap/src/blob/download.rs | 387 +- crates/jmap/src/blob/embedded.rs | 604 + crates/jmap/src/blob/get.rs | 197 +- crates/jmap/src/blob/mod.rs | 1 + crates/jmap/src/blob/upload.rs | 63 +- crates/jmap/src/calendar/get.rs | 196 +- crates/jmap/src/calendar/mod.rs | 18 +- crates/jmap/src/calendar/set.rs | 984 +- crates/jmap/src/calendar_event/copy.rs | 329 +- crates/jmap/src/calendar_event/get.rs | 1791 +- crates/jmap/src/calendar_event/hidden.rs | 93 + crates/jmap/src/calendar_event/mod.rs | 381 +- crates/jmap/src/calendar_event/parse.rs | 70 +- crates/jmap/src/calendar_event/privacy.rs | 173 + crates/jmap/src/calendar_event/query.rs | 1089 +- crates/jmap/src/calendar_event/server_set.rs | 345 + crates/jmap/src/calendar_event/set.rs | 2994 +- crates/jmap/src/calendar_event/user.rs | 44 + crates/jmap/src/calendar_event/validate.rs | 328 + .../src/calendar_event_notification/get.rs | 306 +- .../src/calendar_event_notification/mod.rs | 20 + .../src/calendar_event_notification/patch.rs | 105 + .../src/calendar_event_notification/query.rs | 137 +- .../src/calendar_event_notification/set.rs | 42 +- crates/jmap/src/changes/get.rs | 767 +- crates/jmap/src/changes/mod.rs | 2 + crates/jmap/src/changes/page.rs | 410 + crates/jmap/src/changes/query.rs | 58 +- crates/jmap/src/changes/state.rs | 63 +- crates/jmap/src/contact/copy.rs | 154 +- crates/jmap/src/contact/get.rs | 245 +- crates/jmap/src/contact/mod.rs | 68 +- crates/jmap/src/contact/query.rs | 262 +- crates/jmap/src/contact/set.rs | 469 +- crates/jmap/src/email/copy.rs | 255 +- crates/jmap/src/email/get.rs | 670 +- crates/jmap/src/email/grouped.rs | 136 + crates/jmap/src/email/import.rs | 234 +- crates/jmap/src/email/mod.rs | 17 + crates/jmap/src/email/parse.rs | 265 +- crates/jmap/src/email/query.rs | 1007 +- crates/jmap/src/email/query/filter.rs | 707 + crates/jmap/src/email/query/text.rs | 105 + crates/jmap/src/email/query/window.rs | 384 + crates/jmap/src/email/set.rs | 1205 +- crates/jmap/src/email/snippet.rs | 316 +- crates/jmap/src/file/copy.rs | 495 +- crates/jmap/src/file/get.rs | 631 +- crates/jmap/src/file/mod.rs | 113 + crates/jmap/src/file/node.rs | 575 + crates/jmap/src/file/query.rs | 821 +- crates/jmap/src/file/set.rs | 1895 +- crates/jmap/src/file/writer.rs | 1346 + crates/jmap/src/identity/get.rs | 149 +- crates/jmap/src/identity/mod.rs | 31 + crates/jmap/src/identity/set.rs | 57 +- crates/jmap/src/lib.rs | 1 + crates/jmap/src/mailbox/get.rs | 161 +- crates/jmap/src/mailbox/query.rs | 90 +- crates/jmap/src/mailbox/set.rs | 623 +- .../jmap/src/participant_identity/changes.rs | 87 + crates/jmap/src/participant_identity/get.rs | 40 +- crates/jmap/src/participant_identity/mod.rs | 1 + crates/jmap/src/participant_identity/set.rs | 270 +- crates/jmap/src/principal/availability.rs | 669 +- crates/jmap/src/principal/get.rs | 323 +- crates/jmap/src/principal/query.rs | 113 +- crates/jmap/src/push/get.rs | 91 +- crates/jmap/src/push/set.rs | 136 +- crates/jmap/src/quota/get.rs | 23 +- crates/jmap/src/registry/get.rs | 301 +- crates/jmap/src/registry/mapping/account.rs | 83 +- crates/jmap/src/registry/mapping/action.rs | 105 +- crates/jmap/src/registry/mapping/bootstrap.rs | 105 +- crates/jmap/src/registry/mapping/cluster.rs | 66 +- crates/jmap/src/registry/mapping/dkim.rs | 24 +- crates/jmap/src/registry/mapping/domain.rs | 186 +- .../jmap/src/registry/mapping/index_queue.rs | 483 + crates/jmap/src/registry/mapping/log.rs | 35 +- crates/jmap/src/registry/mapping/mod.rs | 5 +- .../jmap/src/registry/mapping/oauth_client.rs | 49 + crates/jmap/src/registry/mapping/principal.rs | 97 +- .../src/registry/mapping/queued_message.rs | 390 +- crates/jmap/src/registry/mapping/report.rs | 196 +- .../jmap/src/registry/mapping/spam_sample.rs | 91 +- crates/jmap/src/registry/mapping/task.rs | 453 +- crates/jmap/src/registry/mapping/tls.rs | 48 +- crates/jmap/src/registry/mod.rs | 18 +- crates/jmap/src/registry/query.rs | 134 +- crates/jmap/src/registry/set.rs | 979 +- crates/jmap/src/share_notification/get.rs | 46 +- crates/jmap/src/sieve/get.rs | 86 +- crates/jmap/src/sieve/query.rs | 37 +- crates/jmap/src/sieve/set.rs | 500 +- crates/jmap/src/submission/get.rs | 120 +- crates/jmap/src/submission/query.rs | 16 +- crates/jmap/src/submission/set.rs | 289 +- crates/jmap/src/thread/get.rs | 106 +- crates/jmap/src/vacation/get.rs | 4 +- crates/jmap/src/vacation/set.rs | 80 +- crates/jmap/src/websocket/stream.rs | 225 +- crates/jmap/src/websocket/upgrade.rs | 28 +- crates/main/Cargo.toml | 23 +- crates/main/src/main.rs | 43 +- crates/main/src/test_data.rs | 47 +- crates/managesieve/Cargo.toml | 11 +- crates/managesieve/src/core/client.rs | 83 +- crates/managesieve/src/core/mod.rs | 12 +- crates/managesieve/src/core/session.rs | 14 +- crates/managesieve/src/lib.rs | 131 - crates/managesieve/src/op/authenticate.rs | 215 +- crates/managesieve/src/op/capability.rs | 9 +- crates/managesieve/src/op/checkscript.rs | 3 +- crates/managesieve/src/op/deletescript.rs | 2 +- crates/managesieve/src/op/getscript.rs | 21 +- crates/managesieve/src/op/listscripts.rs | 4 +- crates/managesieve/src/op/putscript.rs | 88 +- crates/managesieve/src/op/renamescript.rs | 11 +- crates/managesieve/src/op/setactive.rs | 8 +- crates/migration/Cargo.toml | 24 - crates/migration/src/destroy.rs | 29 - crates/migration/src/lib.rs | 136 - crates/migration/src/v016.rs | 299 - crates/nlp/Cargo.toml | 15 +- crates/nlp/build.rs | 44 + crates/nlp/src/classifier/feature.rs | 50 +- crates/nlp/src/classifier/model.rs | 53 + crates/nlp/src/classifier/reservoir.rs | 15 +- crates/nlp/src/classifier/sgd.rs | 329 - crates/nlp/src/language/detect.rs | 268 +- crates/nlp/src/language/mod.rs | 81 +- crates/nlp/src/language/search_snippet.rs | 378 +- crates/nlp/src/language/stemmer.rs | 23 - crates/nlp/src/tokenizers/chinese.rs | 107 - crates/nlp/src/tokenizers/japanese.rs | 280 +- crates/nlp/src/tokenizers/mod.rs | 1 + crates/nlp/src/tokenizers/space.rs | 79 +- crates/nlp/src/tokenizers/stream.rs | 105 - crates/nlp/src/tokenizers/types.rs | 2600 +- crates/nlp/src/tokenizers/types/scan.rs | 218 + crates/nlp/src/tokenizers/unicode.rs | 31 + crates/nlp/src/tokenizers/word.rs | 273 +- crates/pop3/Cargo.toml | 11 +- crates/pop3/src/client.rs | 85 +- crates/pop3/src/lib.rs | 8 + crates/pop3/src/mailbox.rs | 43 +- crates/pop3/src/op/authenticate.rs | 211 +- crates/pop3/src/op/delete.rs | 3 +- crates/pop3/src/op/fetch.rs | 57 +- crates/pop3/src/op/list.rs | 10 +- crates/pop3/src/op/mod.rs | 7 +- crates/pop3/src/protocol/dot_stuffer.rs | 134 + crates/pop3/src/protocol/mod.rs | 1 + crates/pop3/src/protocol/request.rs | 132 - crates/pop3/src/protocol/response.rs | 372 +- crates/pop3/src/session.rs | 7 +- crates/registry/Cargo.toml | 15 +- crates/registry/src/jmap/mod.rs | 13 +- crates/registry/src/jmap/patch.rs | 17 - crates/registry/src/jmap/properties.rs | 26 +- crates/registry/src/jmap/ser.rs | 14 +- crates/registry/src/pickle.rs | 422 +- crates/registry/src/schema/enums.rs | 2302 +- crates/registry/src/schema/enums_impl.rs | 5220 ++- crates/registry/src/schema/mod.rs | 1 + crates/registry/src/schema/prelude.rs | 26 +- crates/registry/src/schema/properties.rs | 2202 +- crates/registry/src/schema/properties_impl.rs | 5803 ++- crates/registry/src/schema/structs.rs | 2336 +- crates/registry/src/schema/structs_impl.rs | 37256 +++++++++++----- crates/registry/src/types/datetime.rs | 37 +- crates/registry/src/types/error.rs | 15 +- crates/registry/src/types/float.rs | 5 +- crates/registry/src/types/id.rs | 8 +- crates/registry/src/types/index.rs | 23 + crates/registry/src/types/ipaddr.rs | 19 +- crates/registry/src/types/ipmask.rs | 54 +- crates/registry/src/types/list.rs | 13 +- crates/registry/src/types/map.rs | 30 +- crates/registry/src/types/mod.rs | 10 +- crates/registry/src/utils/archived_item.rs | 10 + crates/registry/src/utils/dkim.rs | 53 +- crates/registry/src/utils/mod.rs | 7 +- crates/registry/src/utils/report.rs | 335 +- crates/registry/src/utils/secret.rs | 1 + crates/registry/src/utils/task.rs | 134 +- crates/s3/Cargo.toml | 25 + crates/s3/src/client.rs | 465 + crates/s3/src/credentials.rs | 175 + crates/s3/src/delete.rs | 205 + crates/s3/src/endpoint.rs | 257 + crates/s3/src/error.rs | 267 + crates/s3/src/lib.rs | 20 + crates/s3/src/retry.rs | 99 + crates/s3/src/signer.rs | 258 + crates/scim-proto/Cargo.toml | 6 +- crates/scim/Cargo.toml | 13 +- crates/services/Cargo.toml | 24 +- crates/services/src/broadcast/backlog.rs | 447 + crates/services/src/broadcast/mod.rs | 323 +- crates/services/src/broadcast/publisher.rs | 211 +- crates/services/src/broadcast/subscriber.rs | 421 +- crates/services/src/index_queue/document.rs | 163 + crates/services/src/index_queue/file.rs | 212 + crates/services/src/index_queue/index.rs | 871 + crates/services/src/index_queue/manager.rs | 386 + crates/services/src/index_queue/mod.rs | 162 + crates/services/src/index_queue/reindex.rs | 201 + crates/services/src/lib.rs | 8 +- crates/services/src/state_manager/ece.rs | 45 +- .../services/src/state_manager/email_push.rs | 667 +- crates/services/src/state_manager/http.rs | 160 +- crates/services/src/state_manager/manager.rs | 138 +- crates/services/src/state_manager/mod.rs | 105 +- crates/services/src/state_manager/push.rs | 48 +- crates/services/src/task_manager/acme.rs | 113 - crates/services/src/task_manager/alarm.rs | 669 - .../src/task_manager/destroy_account.rs | 236 - crates/services/src/task_manager/imip.rs | 612 - crates/services/src/task_manager/index.rs | 594 - crates/services/src/task_manager/lock.rs | 101 +- crates/services/src/task_manager/manager.rs | 973 +- crates/services/src/task_manager/mod.rs | 224 +- crates/services/src/task_manager/outcome.rs | 382 + crates/services/src/task_manager/reconcile.rs | 77 + crates/services/src/task_manager/report.rs | 44 - crates/services/src/task_manager/scheduler.rs | 56 +- .../services/src/task_manager/tasks/acme.rs | 96 + .../services/src/task_manager/tasks/alarm.rs | 1020 + .../src/task_manager/tasks/destroy_account.rs | 287 + .../src/task_manager/{ => tasks}/dkim.rs | 345 +- .../src/task_manager/{ => tasks}/dns.rs | 82 +- .../src/task_manager/tasks/grantee_cleanup.rs | 208 + .../tasks/grantee_cleanup/cleanup.rs | 398 + .../tasks/grantee_cleanup/stage.rs | 404 + .../services/src/task_manager/tasks/imip.rs | 734 + .../task_manager/{ => tasks}/maintenance.rs | 286 +- .../task_manager/{ => tasks}/merge_threads.rs | 104 +- crates/services/src/task_manager/tasks/mod.rs | 42 + .../src/task_manager/tasks/rebuild_shares.rs | 236 + .../services/src/task_manager/tasks/report.rs | 59 + .../task_manager/{ => tasks}/restore_item.rs | 29 +- .../{ => tasks}/spam_classifier.rs | 140 +- crates/services/src/task_manager/worker.rs | 353 + crates/sieve/Cargo.toml | 37 + crates/sieve/src/bytecode/cursor.rs | 260 + crates/sieve/src/bytecode/emit.rs | 205 + crates/sieve/src/bytecode/header_id.rs | 171 + crates/sieve/src/bytecode/mod.rs | 104 + crates/sieve/src/bytecode/ops.rs | 244 + crates/sieve/src/bytecode/rec.rs | 128 + crates/sieve/src/bytecode/verify.rs | 224 + crates/sieve/src/compiler/emit.rs | 1132 + .../grammar/actions/action_convert.rs | 47 + .../grammar/actions/action_editheader.rs | 206 + .../grammar/actions/action_fileinto.rs | 105 + .../compiler/grammar/actions/action_flags.rs | 80 + .../grammar/actions/action_include.rs | 85 + .../compiler/grammar/actions/action_keep.rs | 45 + .../compiler/grammar/actions/action_mime.rs | 209 + .../compiler/grammar/actions/action_notify.rs | 179 + .../grammar/actions/action_redirect.rs | 289 + .../compiler/grammar/actions/action_reject.rs | 31 + .../grammar/actions/action_require.rs | 111 + .../compiler/grammar/actions/action_set.rs | 242 + .../grammar/actions/action_vacation.rs | 232 + .../sieve/src/compiler/grammar/actions/mod.rs | 19 + crates/sieve/src/compiler/grammar/expr/mod.rs | 104 + .../sieve/src/compiler/grammar/expr/parser.rs | 262 + .../src/compiler/grammar/expr/tokenizer.rs | 300 + .../sieve/src/compiler/grammar/instruction.rs | 1344 + crates/sieve/src/compiler/grammar/mod.rs | 921 + crates/sieve/src/compiler/grammar/test.rs | 706 + .../sieve/src/compiler/grammar/tests/mod.rs | 24 + .../compiler/grammar/tests/test_address.rs | 172 + .../src/compiler/grammar/tests/test_body.rs | 111 + .../src/compiler/grammar/tests/test_date.rs | 373 + .../compiler/grammar/tests/test_duplicate.rs | 109 + .../compiler/grammar/tests/test_envelope.rs | 233 + .../grammar/tests/test_environment.rs | 80 + .../src/compiler/grammar/tests/test_exists.rs | 78 + .../compiler/grammar/tests/test_extlists.rs | 30 + .../compiler/grammar/tests/test_hasflag.rs | 141 + .../src/compiler/grammar/tests/test_header.rs | 172 + .../src/compiler/grammar/tests/test_ihave.rs | 52 + .../compiler/grammar/tests/test_mailbox.rs | 227 + .../compiler/grammar/tests/test_mailboxid.rs | 30 + .../src/compiler/grammar/tests/test_notify.rs | 108 + .../src/compiler/grammar/tests/test_size.rs | 47 + .../compiler/grammar/tests/test_spamtest.rs | 154 + .../compiler/grammar/tests/test_specialuse.rs | 50 + .../src/compiler/grammar/tests/test_string.rs | 85 + crates/sieve/src/compiler/lexer/mod.rs | 79 + crates/sieve/src/compiler/lexer/string.rs | 761 + crates/sieve/src/compiler/lexer/tokenizer.rs | 646 + crates/sieve/src/compiler/lexer/word.rs | 411 + crates/sieve/src/compiler/mod.rs | 689 + crates/sieve/src/lib.rs | 239 + crates/sieve/src/regex.rs | 22 + .../src/runtime/actions/action_convert.rs | 144 + .../src/runtime/actions/action_editheader.rs | 172 + .../src/runtime/actions/action_fileinto.rs | 41 + .../sieve/src/runtime/actions/action_flags.rs | 191 + .../sieve/src/runtime/actions/action_keep.rs | 30 + .../sieve/src/runtime/actions/action_mime.rs | 355 + .../src/runtime/actions/action_notify.rs | 600 + .../src/runtime/actions/action_redirect.rs | 203 + .../sieve/src/runtime/actions/action_set.rs | 168 + .../src/runtime/actions/action_vacation.rs | 323 + crates/sieve/src/runtime/actions/mod.rs | 16 + crates/sieve/src/runtime/arena.rs | 108 + crates/sieve/src/runtime/context.rs | 1285 + crates/sieve/src/runtime/eval.rs | 763 + crates/sieve/src/runtime/expression.rs | 335 + crates/sieve/src/runtime/handler.rs | 204 + crates/sieve/src/runtime/message/body.rs | 144 + crates/sieve/src/runtime/message/edits.rs | 207 + crates/sieve/src/runtime/message/headers.rs | 315 + .../sieve/src/runtime/message}/mod.rs | 7 +- crates/sieve/src/runtime/message/parts.rs | 100 + crates/sieve/src/runtime/message/serialize.rs | 105 + crates/sieve/src/runtime/mod.rs | 583 + crates/sieve/src/runtime/platform.rs | 29 + crates/sieve/src/runtime/tests/comparator.rs | 243 + crates/sieve/src/runtime/tests/glob.rs | 610 + crates/sieve/src/runtime/tests/matching.rs | 232 + crates/sieve/src/runtime/tests/mime.rs | 84 + crates/sieve/src/runtime/tests/mod.rs | 157 + .../sieve/src/runtime/tests/test_address.rs | 250 + crates/sieve/src/runtime/tests/test_body.rs | 194 + crates/sieve/src/runtime/tests/test_date.rs | 255 + .../sieve/src/runtime/tests/test_duplicate.rs | 77 + .../sieve/src/runtime/tests/test_envelope.rs | 256 + crates/sieve/src/runtime/tests/test_exists.rs | 66 + .../sieve/src/runtime/tests/test_extlists.rs | 27 + .../sieve/src/runtime/tests/test_hasflag.rs | 110 + crates/sieve/src/runtime/tests/test_header.rs | 433 + .../sieve/src/runtime/tests/test_metadata.rs | 117 + crates/sieve/src/runtime/tests/test_notify.rs | 90 + crates/sieve/src/runtime/tests/test_size.rs | 20 + .../sieve/src/runtime/tests/test_spamtest.rs | 197 + crates/sieve/src/runtime/tests/test_string.rs | 128 + crates/sieve/src/runtime/variable.rs | 435 + crates/sieve/src/runtime/variables.rs | 130 + crates/sieve/src/sieve.rs | 428 + crates/smtp-proto/Cargo.toml | 20 + crates/smtp-proto/src/lib.rs | 464 + crates/smtp-proto/src/request/mod.rs | 121 + crates/smtp-proto/src/request/parser.rs | 1435 + crates/smtp-proto/src/request/receiver.rs | 327 + crates/smtp-proto/src/request/scan.rs | 131 + crates/smtp-proto/src/response/generate.rs | 1290 + crates/smtp-proto/src/response/mod.rs | 95 + crates/smtp-proto/src/response/parser.rs | 468 + crates/smtp-proto/src/tokens.rs | 46 + crates/smtp-report/Cargo.toml | 27 + crates/smtp-report/src/arf/builder.rs | 70 + crates/smtp-report/src/arf/generate.rs | 280 + crates/smtp-report/src/arf/mod.rs | 207 + crates/smtp-report/src/arf/parse.rs | 288 + crates/smtp-report/src/dmarc/builder.rs | 211 + crates/smtp-report/src/dmarc/generate.rs | 543 + crates/smtp-report/src/dmarc/mod.rs | 473 + crates/smtp-report/src/dmarc/parse.rs | 866 + crates/smtp-report/src/headers.rs | 120 + crates/smtp-report/src/lib.rs | 131 + crates/smtp-report/src/tlsrpt/generate.rs | 182 + crates/smtp-report/src/tlsrpt/mod.rs | 321 + crates/smtp-report/src/tlsrpt/parse.rs | 137 + crates/smtp-report/src/utf8.rs | 30 + crates/smtp/Cargo.toml | 37 +- crates/smtp/src/core/mod.rs | 46 +- crates/smtp/src/core/params.rs | 110 +- crates/smtp/src/core/throttle.rs | 126 +- crates/smtp/src/inbound/auth.rs | 201 +- crates/smtp/src/inbound/data.rs | 582 +- crates/smtp/src/inbound/dkim.rs | 669 +- crates/smtp/src/inbound/ehlo.rs | 135 +- crates/smtp/src/inbound/hooks/client.rs | 9 +- crates/smtp/src/inbound/hooks/message.rs | 62 +- crates/smtp/src/inbound/hooks/mod.rs | 12 +- crates/smtp/src/inbound/mail.rs | 98 +- crates/smtp/src/inbound/milter/client.rs | 10 +- crates/smtp/src/inbound/milter/message.rs | 58 +- crates/smtp/src/inbound/milter/protocol.rs | 242 - crates/smtp/src/inbound/rcpt.rs | 59 +- crates/smtp/src/inbound/session.rs | 255 +- crates/smtp/src/inbound/spam.rs | 9 +- crates/smtp/src/inbound/spawn.rs | 51 +- crates/smtp/src/inbound/vrfy.rs | 17 +- crates/smtp/src/outbound/client.rs | 474 +- crates/smtp/src/outbound/dane/dnssec.rs | 266 +- crates/smtp/src/outbound/dane/verify.rs | 27 +- crates/smtp/src/outbound/delivery.rs | 756 +- crates/smtp/src/outbound/error.rs | 7 +- crates/smtp/src/outbound/local.rs | 163 +- crates/smtp/src/outbound/lookup.rs | 43 +- crates/smtp/src/outbound/mod.rs | 149 +- crates/smtp/src/outbound/mta_sts/lookup.rs | 236 +- crates/smtp/src/outbound/mta_sts/mod.rs | 6 + crates/smtp/src/outbound/mta_sts/parse.rs | 25 - crates/smtp/src/outbound/session.rs | 302 +- crates/smtp/src/queue/dsn.rs | 35 +- crates/smtp/src/queue/manager.rs | 919 +- crates/smtp/src/queue/mod.rs | 150 +- crates/smtp/src/queue/quota.rs | 250 +- crates/smtp/src/queue/spool.rs | 1008 +- crates/smtp/src/queue/submission.rs | 136 + crates/smtp/src/queue/throttle.rs | 18 +- crates/smtp/src/queue/walk.rs | 183 + crates/smtp/src/reporting/aggregate/codec.rs | 296 + crates/smtp/src/reporting/aggregate/mod.rs | 684 + crates/smtp/src/reporting/aggregate/window.rs | 182 + crates/smtp/src/reporting/analysis.rs | 166 +- crates/smtp/src/reporting/dkim.rs | 103 +- crates/smtp/src/reporting/dmarc.rs | 719 +- crates/smtp/src/reporting/inbound.rs | 121 +- crates/smtp/src/reporting/index.rs | 128 +- crates/smtp/src/reporting/mod.rs | 45 + crates/smtp/src/reporting/scheduler.rs | 13 +- crates/smtp/src/reporting/send.rs | 10 +- crates/smtp/src/reporting/spf.rs | 118 +- crates/smtp/src/reporting/tls.rs | 667 +- crates/smtp/src/scripts/envelope.rs | 11 +- crates/smtp/src/scripts/event_loop.rs | 819 +- crates/smtp/src/scripts/exec.rs | 39 +- crates/smtp/src/scripts/mod.rs | 62 +- crates/spam-filter/Cargo.toml | 28 +- crates/spam-filter/src/analysis/date.rs | 9 +- crates/spam-filter/src/analysis/dmarc.rs | 6 +- crates/spam-filter/src/analysis/domain.rs | 373 +- crates/spam-filter/src/analysis/ehlo.rs | 10 +- crates/spam-filter/src/analysis/from.rs | 400 +- crates/spam-filter/src/analysis/headers.rs | 132 +- crates/spam-filter/src/analysis/html.rs | 96 +- crates/spam-filter/src/analysis/init.rs | 321 +- crates/spam-filter/src/analysis/ip.rs | 88 +- crates/spam-filter/src/analysis/messageid.rs | 63 +- crates/spam-filter/src/analysis/mime.rs | 483 +- crates/spam-filter/src/analysis/mod.rs | 157 +- crates/spam-filter/src/analysis/pyzor.rs | 8 +- crates/spam-filter/src/analysis/received.rs | 26 +- crates/spam-filter/src/analysis/recipient.rs | 142 +- crates/spam-filter/src/analysis/replyto.rs | 41 +- crates/spam-filter/src/analysis/rules.rs | 310 +- crates/spam-filter/src/analysis/score.rs | 449 +- crates/spam-filter/src/analysis/subject.rs | 137 +- crates/spam-filter/src/analysis/url.rs | 478 +- crates/spam-filter/src/lib.rs | 102 +- crates/spam-filter/src/modules/classifier.rs | 959 +- crates/spam-filter/src/modules/dnsbl.rs | 212 +- crates/spam-filter/src/modules/expression.rs | 719 +- crates/spam-filter/src/modules/html.rs | 1106 +- crates/spam-filter/src/modules/pyzor.rs | 828 +- crates/spam-filter/src/modules/sanitize.rs | 163 +- crates/spam-filter/src/texts.rs | 165 + crates/store/Cargo.toml | 52 +- crates/store/src/backend/azure/mod.rs | 432 +- crates/store/src/backend/azure/shared_key.rs | 171 + crates/store/src/backend/elastic/main.rs | 29 +- crates/store/src/backend/elastic/mod.rs | 19 +- crates/store/src/backend/elastic/search.rs | 180 +- crates/store/src/backend/ephemeral/blob.rs | 8 +- crates/store/src/backend/ephemeral/mod.rs | 3 +- crates/store/src/backend/ephemeral/read.rs | 43 +- crates/store/src/backend/ephemeral/write.rs | 153 +- .../src/backend/foundationdb/assembly.rs | 303 + crates/store/src/backend/foundationdb/blob.rs | 316 +- crates/store/src/backend/foundationdb/main.rs | 6 + crates/store/src/backend/foundationdb/mod.rs | 30 +- .../store/src/backend/foundationdb/preread.rs | 189 + crates/store/src/backend/foundationdb/read.rs | 460 +- .../store/src/backend/foundationdb/window.rs | 430 + .../store/src/backend/foundationdb/write.rs | 828 +- crates/store/src/backend/fs/mod.rs | 4 +- crates/store/src/backend/http/config.rs | 26 +- crates/store/src/backend/http/lookup.rs | 203 +- crates/store/src/backend/http/mod.rs | 4 +- crates/store/src/backend/local/entry.rs | 266 + crates/store/src/backend/local/maintenance.rs | 146 + crates/store/src/backend/local/mod.rs | 252 + crates/store/src/backend/meili/main.rs | 72 +- crates/store/src/backend/meili/search.rs | 159 +- crates/store/src/backend/mod.rs | 142 + crates/store/src/backend/mysql/blob.rs | 6 +- crates/store/src/backend/mysql/main.rs | 166 +- crates/store/src/backend/mysql/mod.rs | 30 +- crates/store/src/backend/mysql/pooled.rs | 64 + crates/store/src/backend/mysql/read.rs | 670 +- crates/store/src/backend/mysql/search.rs | 431 +- crates/store/src/backend/mysql/sql.rs | 133 + crates/store/src/backend/mysql/write.rs | 708 +- crates/store/src/backend/mysql/write/reads.rs | 269 + crates/store/src/backend/postgres/blob.rs | 12 +- crates/store/src/backend/postgres/main.rs | 165 +- crates/store/src/backend/postgres/mod.rs | 133 +- crates/store/src/backend/postgres/pool.rs | 128 + crates/store/src/backend/postgres/pooled.rs | 101 + crates/store/src/backend/postgres/read.rs | 649 +- crates/store/src/backend/postgres/search.rs | 821 +- crates/store/src/backend/postgres/sql.rs | 156 + crates/store/src/backend/postgres/write.rs | 1064 +- .../src/backend/postgres/write/flight.rs | 414 + .../store/src/backend/postgres/write/reads.rs | 402 + crates/store/src/backend/range_batch.rs | 64 + crates/store/src/backend/redis/lookup.rs | 405 +- crates/store/src/backend/redis/mod.rs | 54 +- crates/store/src/backend/redis/pool.rs | 47 +- crates/store/src/backend/rocksdb/blob.rs | 47 +- crates/store/src/backend/rocksdb/blob_gc.rs | 229 + crates/store/src/backend/rocksdb/cache.rs | 45 + crates/store/src/backend/rocksdb/main.rs | 242 +- crates/store/src/backend/rocksdb/mod.rs | 49 +- crates/store/src/backend/rocksdb/read.rs | 206 +- crates/store/src/backend/rocksdb/slots.rs | 133 + crates/store/src/backend/rocksdb/write.rs | 405 +- crates/store/src/backend/s3/mod.rs | 371 +- crates/store/src/backend/sqlite/blob.rs | 39 +- crates/store/src/backend/sqlite/lookup.rs | 22 +- crates/store/src/backend/sqlite/main.rs | 196 +- crates/store/src/backend/sqlite/mod.rs | 8 +- crates/store/src/backend/sqlite/pool.rs | 41 +- crates/store/src/backend/sqlite/read.rs | 198 +- crates/store/src/backend/sqlite/sql.rs | 92 + crates/store/src/backend/sqlite/write.rs | 388 +- crates/store/src/build/cluster.rs | 162 +- crates/store/src/build/data.rs | 66 +- crates/store/src/build/lookup.rs | 20 +- crates/store/src/build/memory.rs | 15 +- crates/store/src/build/mod.rs | 1 + crates/store/src/build/registry.rs | 35 +- crates/store/src/build/search.rs | 18 +- crates/store/src/build/swap.rs | 96 + crates/store/src/dispatch/blob.rs | 210 +- crates/store/src/dispatch/lookup.rs | 544 +- crates/store/src/dispatch/mod.rs | 135 + crates/store/src/dispatch/search.rs | 372 +- crates/store/src/dispatch/store.rs | 364 +- crates/store/src/lib.rs | 333 +- crates/store/src/query/acl.rs | 88 +- crates/store/src/query/log.rs | 576 +- crates/store/src/query/mod.rs | 21 +- crates/store/src/registry/bootstrap.rs | 113 +- crates/store/src/registry/get.rs | 103 +- crates/store/src/registry/mod.rs | 60 +- crates/store/src/registry/query.rs | 583 +- crates/store/src/registry/write.rs | 132 +- crates/store/src/search/bm_u32.rs | 278 - crates/store/src/search/bm_u64.rs | 205 - crates/store/src/search/codec.rs | 350 + crates/store/src/search/document.rs | 158 +- crates/store/src/search/fields.rs | 64 +- crates/store/src/search/index.rs | 616 +- crates/store/src/search/leaf.rs | 329 + crates/store/src/search/local.rs | 31 +- crates/store/src/search/mod.rs | 175 +- crates/store/src/search/query.rs | 1412 +- crates/store/src/search/split.rs | 602 +- crates/store/src/search/term.rs | 1070 +- crates/store/src/search/tokenize.rs | 75 + crates/store/src/write/batch.rs | 750 +- crates/store/src/write/bitpack.rs | 126 +- crates/store/src/write/blob.rs | 116 +- crates/store/src/write/compress.rs | 235 + crates/store/src/write/counter.rs | 353 + crates/store/src/write/key.rs | 1045 +- crates/store/src/write/lazybitmap.rs | 116 + crates/store/src/write/log.rs | 750 +- crates/store/src/write/metadata.rs | 184 + crates/store/src/write/mod.rs | 1624 +- crates/store/src/write/new_key.rs | 54 + crates/store/src/write/serialize.rs | 457 +- crates/text-extract/CREDITS.md | 507 + crates/text-extract/Cargo.toml | 19 + crates/text-extract/src/epub.rs | 219 + crates/text-extract/src/html.rs | 151 + crates/text-extract/src/lib.rs | 405 + crates/text-extract/src/odf.rs | 277 + crates/text-extract/src/ooxml/chart.rs | 196 + crates/text-extract/src/ooxml/comments.rs | 48 + crates/text-extract/src/ooxml/layout.rs | 61 + crates/text-extract/src/ooxml/mod.rs | 265 + .../text-extract/src/ooxml/number/decimal.rs | 285 + crates/text-extract/src/ooxml/number/mod.rs | 241 + .../text-extract/src/ooxml/number/render.rs | 294 + crates/text-extract/src/ooxml/pptx.rs | 82 + crates/text-extract/src/ooxml/rels.rs | 140 + crates/text-extract/src/ooxml/sheet.rs | 312 + crates/text-extract/src/ooxml/styles.rs | 139 + crates/text-extract/src/ooxml/text.rs | 252 + crates/text-extract/src/ooxml/xlsx.rs | 60 + crates/text-extract/src/output.rs | 161 + crates/text-extract/src/package.rs | 266 + crates/text-extract/src/pdf/content/form.rs | 149 + .../src/pdf/content/inline/markers.rs | 173 + .../src/pdf/content/inline/mod.rs | 181 + crates/text-extract/src/pdf/content/mod.rs | 334 + crates/text-extract/src/pdf/content/ops.rs | 213 + .../text-extract/src/pdf/content/resources.rs | 243 + crates/text-extract/src/pdf/content/state.rs | 213 + crates/text-extract/src/pdf/content/text.rs | 151 + crates/text-extract/src/pdf/crypt/aes.rs | 93 + crates/text-extract/src/pdf/crypt/derive.rs | 164 + crates/text-extract/src/pdf/crypt/key.rs | 46 + crates/text-extract/src/pdf/crypt/legacy.rs | 116 + crates/text-extract/src/pdf/crypt/mod.rs | 207 + crates/text-extract/src/pdf/crypt/modern.rs | 156 + crates/text-extract/src/pdf/crypt/rc4.rs | 46 + crates/text-extract/src/pdf/decode.rs | 203 + crates/text-extract/src/pdf/document/crypt.rs | 143 + crates/text-extract/src/pdf/document/fetch.rs | 160 + crates/text-extract/src/pdf/document/mod.rs | 310 + .../text-extract/src/pdf/document/recover.rs | 255 + .../text-extract/src/pdf/document/streams.rs | 67 + crates/text-extract/src/pdf/document/table.rs | 87 + crates/text-extract/src/pdf/filter/ascii85.rs | 67 + .../text-extract/src/pdf/filter/ascii_hex.rs | 44 + crates/text-extract/src/pdf/filter/flate.rs | 143 + crates/text-extract/src/pdf/filter/lzw.rs | 185 + crates/text-extract/src/pdf/filter/mod.rs | 136 + crates/text-extract/src/pdf/filter/output.rs | 93 + .../text-extract/src/pdf/filter/predictor.rs | 319 + .../text-extract/src/pdf/filter/run_length.rs | 38 + crates/text-extract/src/pdf/filter/stage.rs | 112 + crates/text-extract/src/pdf/font/cache.rs | 157 + crates/text-extract/src/pdf/font/cmap/mod.rs | 456 + crates/text-extract/src/pdf/font/code.rs | 127 + crates/text-extract/src/pdf/font/composite.rs | 401 + crates/text-extract/src/pdf/font/embedded.rs | 212 + crates/text-extract/src/pdf/font/mod.rs | 196 + .../text-extract/src/pdf/font/predefined.rs | 87 + .../text-extract/src/pdf/font/program/cff.rs | 390 + .../src/pdf/font/program/cff_data.rs | 52 + .../text-extract/src/pdf/font/program/cmap.rs | 395 + .../text-extract/src/pdf/font/program/mod.rs | 65 + .../text-extract/src/pdf/font/program/post.rs | 106 + .../src/pdf/font/program/post_data.rs | 24 + .../text-extract/src/pdf/font/program/read.rs | 82 + .../text-extract/src/pdf/font/program/sfnt.rs | 57 + .../src/pdf/font/program/truetype.rs | 164 + .../src/pdf/font/program/type1.rs | 222 + crates/text-extract/src/pdf/font/ranges.rs | 130 + .../src/pdf/font/simple/encoding.rs | 203 + .../text-extract/src/pdf/font/simple/mod.rs | 273 + .../src/pdf/font/simple/widths.rs | 127 + crates/text-extract/src/pdf/font/unicode.rs | 148 + crates/text-extract/src/pdf/font/widths.rs | 247 + crates/text-extract/src/pdf/forms.rs | 286 + crates/text-extract/src/pdf/layout/dedupe.rs | 171 + crates/text-extract/src/pdf/layout/line.rs | 329 + crates/text-extract/src/pdf/layout/mod.rs | 539 + crates/text-extract/src/pdf/layout/script.rs | 88 + crates/text-extract/src/pdf/layout/shift.rs | 223 + crates/text-extract/src/pdf/lexer/mod.rs | 234 + crates/text-extract/src/pdf/lexer/number.rs | 131 + crates/text-extract/src/pdf/mod.rs | 125 + .../text-extract/src/pdf/object/container.rs | 151 + crates/text-extract/src/pdf/object/mod.rs | 318 + crates/text-extract/src/pdf/object/string.rs | 187 + crates/text-extract/src/pdf/objstm.rs | 127 + crates/text-extract/src/pdf/pages.rs | 355 + crates/text-extract/src/pdf/repair.rs | 226 + crates/text-extract/src/pdf/seek.rs | 35 + crates/text-extract/src/pdf/source.rs | 292 + crates/text-extract/src/pdf/tables/cid.rs | 198 + .../text-extract/src/pdf/tables/cid_cns1.bin | Bin 0 -> 31281 bytes .../text-extract/src/pdf/tables/cid_data.rs | 24 + .../text-extract/src/pdf/tables/cid_gb1.bin | Bin 0 -> 19897 bytes .../src/pdf/tables/cid_japan1.bin | Bin 0 -> 30063 bytes .../src/pdf/tables/cid_korea1.bin | Bin 0 -> 12924 bytes .../text-extract/src/pdf/tables/cmap_data.rs | 795 + crates/text-extract/src/pdf/tables/cmaps.rs | 118 + .../text-extract/src/pdf/tables/encoding.rs | 82 + .../src/pdf/tables/encoding_data.rs | 242 + .../text-extract/src/pdf/tables/font_data.rs | 283 + crates/text-extract/src/pdf/tables/fonts.rs | 115 + .../text-extract/src/pdf/tables/glyph_data.rs | 391 + crates/text-extract/src/pdf/tables/glyphs.rs | 301 + crates/text-extract/src/pdf/tables/mod.rs | 29 + crates/text-extract/src/pdf/tables/names.rs | 42 + .../text-extract/src/pdf/tables/normalize.rs | 35 + .../src/pdf/tables/presentation_data.rs | 144 + crates/text-extract/src/pdf/text_string.rs | 69 + crates/text-extract/src/pdf/xref/mod.rs | 385 + crates/text-extract/src/pdf/xref/stream.rs | 115 + crates/text-extract/src/pdf/xref/table.rs | 117 + crates/text-extract/src/rtf/codepage.rs | 66 + crates/text-extract/src/rtf/mod.rs | 586 + crates/text-extract/src/rtf/words.rs | 133 + crates/text-extract/src/utf8.rs | 76 + crates/text-extract/src/xml/attr.rs | 127 + crates/text-extract/src/xml/mod.rs | 155 + crates/text-extract/src/xml/scan.rs | 403 + crates/text-extract/src/xml/stream.rs | 381 + crates/text-extract/src/zip.rs | 351 + crates/trc/Cargo.toml | 22 +- crates/trc/event-macro/Cargo.toml | 5 +- crates/trc/event-macro/src/lib.rs | 13 +- crates/trc/src/atomics/bitset.rs | 64 - crates/trc/src/atomics/histogram.rs | 40 +- crates/trc/src/event/conv.rs | 243 +- crates/trc/src/event/enums.rs | 1930 +- crates/trc/src/event/enums_impl.rs | 5589 +-- crates/trc/src/event/mod.rs | 221 +- crates/trc/src/ipc/bitset.rs | 4 + crates/trc/src/ipc/channel.rs | 91 +- crates/trc/src/ipc/collector.rs | 381 +- crates/trc/src/ipc/metrics.rs | 84 +- crates/trc/src/ipc/subscriber.rs | 72 +- crates/trc/src/lib.rs | 2 +- crates/trc/src/serializers/escape.rs | 88 + crates/trc/src/serializers/json.rs | 207 +- crates/trc/src/serializers/mod.rs | 9 + crates/trc/src/serializers/text.rs | 431 +- crates/trc/src/serializers/timestamp.rs | 49 + crates/types/Cargo.toml | 12 +- crates/types/src/acl.rs | 19 +- crates/types/src/blob.rs | 331 +- crates/types/src/blob_hash.rs | 8 +- crates/types/src/collation.rs | 45 + crates/types/src/collection.rs | 89 +- crates/types/src/dead_property.rs | 191 - crates/types/src/field.rs | 80 +- crates/types/src/id.rs | 97 +- crates/types/src/keyword.rs | 355 +- crates/types/src/lib.rs | 98 +- crates/types/src/media_type/catalogue.rs | 2928 ++ crates/types/src/media_type/mod.rs | 103 + crates/types/src/metadata/builder.rs | 342 + crates/types/src/metadata/codec.rs | 163 + crates/types/src/metadata/entries.rs | 145 + crates/types/src/metadata/json.rs | 561 + crates/types/src/metadata/limits.rs | 119 + crates/types/src/metadata/mod.rs | 110 + crates/types/src/metadata/registry.rs | 187 + crates/types/src/metadata/view.rs | 333 + crates/types/src/metadata/xml/mod.rs | 406 + crates/types/src/metadata/xml/validate.rs | 159 + crates/types/src/metadata/xml/view.rs | 224 + crates/types/src/metadata/xml/writer.rs | 349 + crates/types/src/special_use.rs | 76 +- crates/types/src/text.rs | 86 + crates/types/src/type_state.rs | 8 +- crates/utils/Cargo.toml | 30 +- crates/utils/proc-macros/Cargo.toml | 2 +- crates/utils/src/cache.rs | 586 +- crates/utils/src/chained_bytes.rs | 208 +- crates/utils/src/cheeky_hash.rs | 272 +- crates/utils/src/codec/base32_custom.rs | 221 - crates/utils/src/codec/leb128.rs | 189 - crates/utils/src/codec/leb128/kernel.rs | 129 + crates/utils/src/codec/leb128/mod.rs | 158 + crates/utils/src/codec/mod.rs | 1 - crates/utils/src/cron.rs | 165 +- crates/utils/src/glob.rs | 86 +- crates/utils/src/hash128.rs | 82 + crates/utils/src/http.rs | 26 +- crates/utils/src/lib.rs | 139 +- crates/utils/src/map/bitmap.rs | 21 +- crates/utils/src/map/vec_map.rs | 19 + crates/utils/src/memory.rs | 113 + crates/utils/src/outbound.rs | 367 + crates/utils/src/runtime.rs | 28 + crates/utils/src/snowflake.rs | 18 +- crates/utils/src/template.rs | 378 - crates/utils/src/text/case.rs | 81 + crates/utils/src/text/count.rs | 96 + crates/utils/src/text/encode.rs | 67 + crates/utils/src/text/ignore_case.rs | 240 + crates/utils/src/text/mod.rs | 27 + crates/utils/src/text/needle.rs | 47 + crates/utils/src/text/rank.rs | 22 + crates/utils/src/text/segments.rs | 87 + crates/utils/src/text/set.rs | 29 + crates/utils/src/text/split.rs | 93 + crates/utils/src/topological.rs | 37 - install.sh | 87 +- resources/html-templates/login.html | 724 +- resources/html-templates/login.html.min | 2 +- resources/html-templates/login.html.min.gz | Bin 5290 -> 6981 bytes resources/schema/schema.json.gz | Bin 150325 -> 171254 bytes resources/schema/schema.json.sha256 | 2 +- resources/schema/schema_oss.json.gz | Bin 0 -> 151878 bytes resources/schema/schema_oss.json.sha256 | 1 + resources/scripts/migrate_v016.py | 32 +- resources/scripts/minify_html.sh | 70 - resources/zstd/calendar-v1.dict | Bin 0 -> 114688 bytes resources/zstd/common-v1.dict | Bin 0 -> 65536 bytes resources/zstd/contact-v1.dict | Bin 0 -> 65536 bytes resources/zstd/email-v1.dict | Bin 0 -> 114179 bytes resources/zstd/sieve-v1.dict | Bin 0 -> 65536 bytes resources/zstd/term-v1.dict | Bin 0 -> 114688 bytes tests/Cargo.toml | 92 - tests/docker/INSTRUCTIONS.md | 220 - tests/docker/docker-compose.yml | 276 - tests/docker/keycloak/stalwart-realm.json | 271 - tests/docker/ldap/50-users.ldif | 71 - tests/docker/ldap/60-groups.ldif | 20 - tests/docker/pebble/Dockerfile.challtestsrv | 5 - tests/docker/pebble/pebble-config.json | 17 - tests/docker/powerdns/entrypoint.sh | 14 - tests/docker/powerdns/init-zone.sh | 42 - tests/docker/powerdns/pdns.conf | 3 - tests/docker/scim/Dockerfile | 9 - tests/docker/scim/driver.py | 457 - tests/docker/scripts/gen-certs.sh | 26 - tests/docker/scripts/init-fdb.sh | 18 - tests/docker/scripts/init-minio.sh | 12 - tests/resources/artwork_data.csv.gz | Bin 3051275 -> 0 bytes tests/resources/crypto/cert_mixed.pem | 35 - tests/resources/crypto/cert_pgp.der | Bin 2240 -> 0 bytes tests/resources/crypto/cert_pgp.pem | 53 - tests/resources/crypto/cert_smime.der | Bin 882 -> 0 bytes tests/resources/crypto/cert_smime.pem | 98 - tests/resources/crypto/is_encrypted.txt | 211 - tests/resources/imap-test/append | 43 - tests/resources/imap-test/append-binary | 39 - tests/resources/imap-test/atoms | 13 - .../resources/imap-test/broken/search-intdate | 75 - .../imap-test/broken/search-intdate.mbox | 80 - tests/resources/imap-test/catenate | 235 - .../resources/imap-test/catenate-multiappend | 130 - tests/resources/imap-test/close | 9 - tests/resources/imap-test/copy | 48 - tests/resources/imap-test/default.mbox | 42 - tests/resources/imap-test/esearch | 76 - tests/resources/imap-test/esearch-condstore | 38 - tests/resources/imap-test/esearch.mbox | 28 - tests/resources/imap-test/esort | 11 - tests/resources/imap-test/expunge | 62 - tests/resources/imap-test/expunge2 | 44 - tests/resources/imap-test/fetch | 54 - tests/resources/imap-test/fetch-binary-mime | 104 - .../imap-test/fetch-binary-mime-base64 | 67 - .../imap-test/fetch-binary-mime-base64.mbox | 49 - .../resources/imap-test/fetch-binary-mime-qp | 54 - .../imap-test/fetch-binary-mime-qp.mbox | 31 - .../imap-test/fetch-binary-mime.mbox | 38 - tests/resources/imap-test/fetch-body | 122 - .../imap-test/fetch-body-message-rfc822 | 73 - .../imap-test/fetch-body-message-rfc822-mime | 185 - .../fetch-body-message-rfc822-mime.mbox | 30 - .../imap-test/fetch-body-message-rfc822-x2 | 103 - .../fetch-body-message-rfc822-x2.mbox | 16 - .../imap-test/fetch-body-message-rfc822.mbox | 11 - tests/resources/imap-test/fetch-body-mime | 193 - .../resources/imap-test/fetch-body-mime.mbox | 38 - tests/resources/imap-test/fetch-body.mbox | 28 - tests/resources/imap-test/fetch-bodystructure | 4 - .../imap-test/fetch-bodystructure.mbox | 13 - tests/resources/imap-test/fetch-envelope | 10 - tests/resources/imap-test/fetch-envelope.mbox | 47 - tests/resources/imap-test/id | 4 - tests/resources/imap-test/list | 79 - tests/resources/imap-test/listext | 95 - tests/resources/imap-test/logout | 14 - tests/resources/imap-test/move | 47 - tests/resources/imap-test/multiappend | 30 - tests/resources/imap-test/mutf7 | 12 - tests/resources/imap-test/nil | 13 - tests/resources/imap-test/nil.mbox | 5 - tests/resources/imap-test/notify | 20 - tests/resources/imap-test/pipeline | 23 - tests/resources/imap-test/search-addresses | 71 - .../resources/imap-test/search-addresses.mbox | 50 - tests/resources/imap-test/search-body | 31 - tests/resources/imap-test/search-body.mbox | 21 - .../resources/imap-test/search-context-update | 35 - .../imap-test/search-context-update2 | 21 - .../imap-test/search-context-update3 | 31 - tests/resources/imap-test/search-date | 71 - tests/resources/imap-test/search-date.mbox | 80 - tests/resources/imap-test/search-flags | 95 - tests/resources/imap-test/search-header | 37 - tests/resources/imap-test/search-header.mbox | 10 - tests/resources/imap-test/search-partial | 54 - tests/resources/imap-test/search-partial.mbox | 19 - tests/resources/imap-test/search-sets | 62 - tests/resources/imap-test/search-size | 22 - tests/resources/imap-test/search-size.mbox | 21 - tests/resources/imap-test/select | 33 - tests/resources/imap-test/select.mbox | 10 - tests/resources/imap-test/sort-addresses | 23 - tests/resources/imap-test/sort-addresses.mbox | 21 - tests/resources/imap-test/sort-arrival | 11 - tests/resources/imap-test/sort-arrival.mbox | 20 - tests/resources/imap-test/sort-date | 11 - tests/resources/imap-test/sort-date.mbox | 37 - tests/resources/imap-test/sort-display-from | 8 - .../imap-test/sort-display-from.mbox | 24 - tests/resources/imap-test/sort-display-to | 8 - .../resources/imap-test/sort-display-to.mbox | 39 - tests/resources/imap-test/sort-partial | 19 - tests/resources/imap-test/sort-partial.mbox | 19 - tests/resources/imap-test/sort-size | 11 - tests/resources/imap-test/sort-size.mbox | 41 - tests/resources/imap-test/sort-subject | 11 - tests/resources/imap-test/sort-subject.mbox | 81 - tests/resources/imap-test/store | 33 - tests/resources/imap-test/subscribe | 49 - tests/resources/imap-test/thread | 11 - .../resources/imap-test/thread-orderedsubject | 5 - .../imap-test/thread-orderedsubject.mbox | 81 - .../imap-test/thread-orderedsubject2 | 5 - .../imap-test/thread-orderedsubject2.mbox | 44 - tests/resources/imap-test/thread.mbox | 16 - tests/resources/imap-test/thread2 | 5 - tests/resources/imap-test/thread2.mbox | 9 - tests/resources/imap-test/thread3 | 5 - tests/resources/imap-test/thread3.mbox | 11 - tests/resources/imap-test/thread4 | 5 - tests/resources/imap-test/thread4.mbox | 12 - tests/resources/imap-test/thread5 | 20 - tests/resources/imap-test/thread5.mbox | 29 - tests/resources/imap-test/thread6 | 5 - tests/resources/imap-test/thread6.mbox | 9 - tests/resources/imap-test/thread7 | 11 - tests/resources/imap-test/thread7.mbox | 22 - tests/resources/imap-test/thread8 | 20 - tests/resources/imap-test/thread8.mbox | 16 - tests/resources/imap-test/uidplus | 23 - tests/resources/imap-test/uidvalidity | 13 - tests/resources/imap-test/uidvalidity-rename | 19 - tests/resources/imap-test/urlauth | 26 - tests/resources/imap-test/urlauth-binary | 88 - tests/resources/imap-test/urlauth-binary.mbox | 41 - tests/resources/imap-test/urlauth2 | 27 - tests/resources/imap/000.imap | 726 - tests/resources/imap/000.txt | 73 - tests/resources/imap/001.imap | 354 - tests/resources/imap/001.txt | 41 - tests/resources/imap/002.imap | 736 - tests/resources/imap/002.txt | 73 - tests/resources/imap/003.imap | 254 - tests/resources/imap/003.txt | 25 - tests/resources/imap/004.imap | 489 - tests/resources/imap/004.txt | 36 - tests/resources/imap/005.imap | 216 - tests/resources/imap/005.txt | 25 - tests/resources/imap/006.imap | 233 - tests/resources/imap/006.txt | 23 - tests/resources/imap/007.imap | 1164 - tests/resources/imap/007.txt | 73 - tests/resources/imap/008.imap | 458 - tests/resources/imap/008.txt | 30 - tests/resources/imap/009.imap | 447 - tests/resources/imap/009.txt | 39 - tests/resources/imap/010.imap | 170 - tests/resources/imap/010.txt | 10 - tests/resources/imap/011.imap | 463 - tests/resources/imap/011.txt | 30 - tests/resources/imap/012.imap | 423 - tests/resources/imap/012.txt | 37 - tests/resources/imap/013.imap | 148 - tests/resources/imap/013.txt | 22 - tests/resources/imap/014.imap | 252 - tests/resources/imap/014.txt | 31 - tests/resources/imap/015.imap | 214 - tests/resources/imap/015.txt | 22 - tests/resources/itip/google_calendar.txt | 276 - tests/resources/itip/itip_incoming.txt | 138 - tests/resources/itip/put_validation.txt | 512 - .../itip/rfc5546_event_recurring.txt | 987 - tests/resources/itip/rfc5546_event_single.txt | 752 - tests/resources/itip/rfc5546_todo.txt | 501 - tests/resources/itip/rfc6638_recurring.txt | 439 - tests/resources/itip/rfc6638_single.txt | 218 - tests/resources/itip/vtimezone.txt | 264 - tests/resources/jmap/email_get/headers.eml | 98 - tests/resources/jmap/email_get/headers.json | 1309 - .../jmap/email_get/message_attachment.eml | 72 - .../jmap/email_get/message_attachment.json | 154 - .../jmap/email_get/multipart_alternative.eml | 22 - .../jmap/email_get/multipart_alternative.json | 153 - .../jmap/email_get/multipart_cid.eml | 156 - .../jmap/email_get/multipart_cid.json | 269 - .../jmap/email_get/multipart_mixed.eml | 72 - .../jmap/email_get/multipart_mixed.json | 272 - .../jmap/email_get/multipart_related.eml | 198 - .../jmap/email_get/multipart_related.json | 380 - tests/resources/jmap/email_get/rfc8621.eml | 72 - tests/resources/jmap/email_get/rfc8621.json | 468 - .../resources/jmap/email_get/single_part.eml | 30 - .../resources/jmap/email_get/single_part.json | 224 - .../jmap/email_get/text_attachments.eml | 52 - .../jmap/email_get/text_attachments.json | 270 - .../jmap/email_get/text_body_missing.eml | 43 - .../jmap/email_get/text_body_missing.json | 165 - .../email_get/text_body_missing_multipart.eml | 83 - .../text_body_missing_multipart.json | 176 - .../resources/jmap/email_parse/attachment.eml | 72 - .../jmap/email_parse/attachment.json | 222 - .../jmap/email_parse/attachment.part1 | 13 - .../jmap/email_parse/attachment.part2 | Bin 288 -> 0 bytes .../jmap/email_parse/attachment.part3 | 1 - .../jmap/email_parse/attachment_b64.eml | 30 - .../jmap/email_parse/attachment_b64.json | 150 - .../jmap/email_parse/attachment_b64.part1 | 1 - .../jmap/email_parse/attachment_b64.part2 | 8 - tests/resources/jmap/email_parse/headers.eml | 98 - tests/resources/jmap/email_parse/headers.json | 1275 - tests/resources/jmap/email_set/headers.eml | 53 - tests/resources/jmap/email_set/headers.jmap | 307 - tests/resources/jmap/email_set/headers.json | 186 - tests/resources/jmap/email_set/minimal.eml | 7 - tests/resources/jmap/email_set/minimal.jmap | 112 - tests/resources/jmap/email_set/minimal.json | 7 - tests/resources/jmap/email_set/mixed.eml | 53 - tests/resources/jmap/email_set/mixed.jmap | 326 - tests/resources/jmap/email_set/mixed.json | 70 - .../resources/jmap/email_set/nested_body.eml | 99 - .../resources/jmap/email_set/nested_body.jmap | 586 - .../resources/jmap/email_set/nested_body.json | 89 - tests/resources/jmap/email_set/rfc8621_1.eml | 11 - tests/resources/jmap/email_set/rfc8621_1.jmap | 167 - tests/resources/jmap/email_set/rfc8621_1.json | 29 - tests/resources/jmap/email_set/rfc8621_2.eml | 26 - tests/resources/jmap/email_set/rfc8621_2.jmap | 177 - tests/resources/jmap/email_set/rfc8621_2.json | 49 - tests/resources/jmap/email_snippet/html.eml | 36 - tests/resources/jmap/email_snippet/mixed.eml | 77 - .../resources/jmap/email_snippet/subpart.eml | 73 - .../jmap/email_snippet/text_plain.eml | 20 - .../jmap/email_snippet/text_plain_chinese.eml | 18 - .../jmap/sieve/test_discard_reject.sieve | 41 - tests/resources/jmap/sieve/test_include.sieve | 5 - .../jmap/sieve/test_include_case.sieve | 5 - .../jmap/sieve/test_include_global.sieve | 5 - .../jmap/sieve/test_include_this.sieve | 4 - tests/resources/jmap/sieve/test_mailbox.sieve | 81 - .../jmap/sieve/test_notify_fcc.sieve | 13 - .../jmap/sieve/test_redirect_enclose.sieve | 5 - .../resources/jmap/sieve/validate_error.sieve | 1 - tests/resources/jmap/sieve/validate_ok.sieve | 3 - tests/resources/ldap/ldap.cfg | 126 - tests/resources/ldap/run_glauth.sh | 3 - tests/resources/mailbox.gz | Bin 2136385 -> 0 bytes tests/resources/otel/docker-compose.yaml | 45 - .../resources/otel/otel-collector-config.yaml | 44 - tests/resources/otel/stalwart-config.toml | 7 - tests/resources/proxy-protocol/Docker.haproxy | 6 - tests/resources/proxy-protocol/haproxy.cfg | 18 - tests/resources/scripts/imap_import.py | 55 - tests/resources/scripts/imap_import_single.py | 27 - tests/resources/scripts/smtp_stress_test.py | 627 - tests/resources/scripts/stress_test.py | 490 - .../resources/scripts/stress_test_prepare.py | 244 - tests/resources/smtp/antispam/bounce.test | 99 - tests/resources/smtp/antispam/classifier.ham | 59 - tests/resources/smtp/antispam/classifier.spam | 50 - tests/resources/smtp/antispam/classifier.test | 22 - .../smtp/antispam/classifier_features.test | 2751 -- .../smtp/antispam/classifier_html.test | 695 - tests/resources/smtp/antispam/combined.test | 1111 - tests/resources/smtp/antispam/date.test | 23 - tests/resources/smtp/antispam/dmarc.test | 140 - tests/resources/smtp/antispam/from.test | 232 - tests/resources/smtp/antispam/headers.test | 90 - tests/resources/smtp/antispam/helo.test | 65 - tests/resources/smtp/antispam/html.test | 335 - tests/resources/smtp/antispam/ip.test | 17 - tests/resources/smtp/antispam/llm.test | 72 - tests/resources/smtp/antispam/messageid.test | 104 - tests/resources/smtp/antispam/mime.test | 566 - tests/resources/smtp/antispam/pyzor.test | 52 - tests/resources/smtp/antispam/rbl.test | 90 - tests/resources/smtp/antispam/received.test | 83 - tests/resources/smtp/antispam/recipient.test | 235 - tests/resources/smtp/antispam/replyto.test | 112 - tests/resources/smtp/antispam/spamtrap.test | 8 - tests/resources/smtp/antispam/subject.test | 138 - tests/resources/smtp/antispam/url.test | 124 - tests/resources/smtp/certs/tls_cert.pem | 29 - tests/resources/smtp/certs/tls_privatekey.pem | 52 - tests/resources/smtp/dane/dns.txt | 3 - tests/resources/smtp/dane/internet.nl.0.cert | Bin 1874 -> 0 bytes tests/resources/smtp/dane/internet.nl.1.cert | Bin 1559 -> 0 bytes .../resources/smtp/dane/mail.ietf.org.0.cert | Bin 1713 -> 0 bytes .../resources/smtp/dane/mail.ietf.org.1.cert | Bin 1284 -> 0 bytes .../resources/smtp/dane/mail.ietf.org.2.cert | Bin 1188 -> 0 bytes .../resources/smtp/dane/mail.ietf.org.3.cert | Bin 1043 -> 0 bytes tests/resources/smtp/dsn/delay.eml | 50 - tests/resources/smtp/dsn/failure.eml | 48 - tests/resources/smtp/dsn/mixed.eml | 69 - tests/resources/smtp/dsn/original.txt | 10 - tests/resources/smtp/dsn/success.eml | 47 - tests/resources/smtp/lists/test-list1.txt | 2 - tests/resources/smtp/lists/test-list2.txt | 3 - tests/resources/smtp/messages/arc.eml | 34 - tests/resources/smtp/messages/dkim.eml | 14 - tests/resources/smtp/messages/idn_from.eml | 7 - tests/resources/smtp/messages/invalid_arc.eml | 25 - .../resources/smtp/messages/invalid_dkim.eml | 14 - tests/resources/smtp/messages/loop.eml | 23 - tests/resources/smtp/messages/multipart.eml | 34 - tests/resources/smtp/messages/no_dkim.eml | 14 - tests/resources/smtp/messages/no_msgid.eml | 9 - tests/resources/smtp/milter/message.eml | 11 - tests/resources/smtp/milter/message.json | 149 - tests/resources/smtp/reports/arf1.eml | 44 - tests/resources/smtp/reports/arf2.eml | 55 - tests/resources/smtp/reports/arf3.eml | 58 - tests/resources/smtp/reports/arf4.eml | 73 - tests/resources/smtp/reports/arf5.eml | 87 - tests/resources/smtp/reports/dmarc1.eml | 66 - tests/resources/smtp/reports/dmarc2.eml | 68 - tests/resources/smtp/reports/dmarc3.eml | 52 - tests/resources/smtp/reports/dmarc4.eml | 126 - tests/resources/smtp/reports/dmarc5.eml | 54 - tests/resources/smtp/reports/tls1.eml | 41 - tests/resources/smtp/reports/tls2.eml | 64 - tests/resources/smtp/sieve/awl.sieve | 32 - tests/resources/smtp/sieve/awl_include.sieve | 20 - .../smtp/sieve/spamtest_include.sieve | 14 - .../resources/smtp/sieve/stage_connect.sieve | 5 - tests/resources/smtp/sieve/stage_data.sieve | 35 - tests/resources/smtp/sieve/stage_ehlo.sieve | 5 - tests/resources/smtp/sieve/stage_mail.sieve | 17 - tests/resources/smtp/sieve/stage_rcpt.sieve | 12 - tests/resources/tls_cert.pem | 29 - tests/resources/tls_privatekey.pem | 52 - tests/src/automation/acme.rs | 871 - tests/src/automation/dkim.rs | 1017 - tests/src/automation/dns.rs | 218 - tests/src/automation/mod.rs | 122 - tests/src/automation/rfc2136.rs | 179 - tests/src/cluster/broadcast.rs | 223 - tests/src/cluster/stress.rs | 447 - tests/src/directory/discovery.rs | 87 - tests/src/directory/integration.rs | 232 - tests/src/directory/issuer.rs | 115 - tests/src/directory/ldap.rs | 243 - tests/src/directory/mod.rs | 27 - tests/src/directory/sql.rs | 204 - tests/src/directory/synchronization.rs | 305 - tests/src/directory/unavailable.rs | 78 - tests/src/imap/acl.rs | 391 - tests/src/imap/antispam.rs | 201 - tests/src/imap/append.rs | 107 - tests/src/imap/basic.rs | 59 - tests/src/imap/body_structure.rs | 242 - tests/src/imap/condstore.rs | 280 - tests/src/imap/copy_move.rs | 226 - tests/src/imap/fetch.rs | 183 - tests/src/imap/idle.rs | 200 - tests/src/imap/mailbox.rs | 455 - tests/src/imap/managesieve.rs | 126 - tests/src/imap/messagelimit.rs | 144 - tests/src/imap/mod.rs | 323 - tests/src/imap/objectid.rs | 204 - tests/src/imap/pop.rs | 200 - tests/src/imap/search.rs | 198 - tests/src/imap/store.rs | 75 - tests/src/imap/thread.rs | 120 - tests/src/imap/uidbatches.rs | 116 - tests/src/imap/uidonly.rs | 144 - tests/src/jmap/calendar/acl.rs | 711 - tests/src/jmap/calendar/alarm.rs | 175 - tests/src/jmap/calendar/calendars.rs | 377 - tests/src/jmap/calendar/event.rs | 1194 - tests/src/jmap/calendar/identity.rs | 151 - tests/src/jmap/calendar/instance.rs | 991 - tests/src/jmap/calendar/mod.rs | 13 - tests/src/jmap/calendar/notification.rs | 582 - tests/src/jmap/compliance/binary.rs | 275 - tests/src/jmap/compliance/core.rs | 742 - tests/src/jmap/compliance/email/inspect.rs | 1147 - tests/src/jmap/compliance/email/mod.rs | 18 - tests/src/jmap/compliance/email/mutate.rs | 1671 - tests/src/jmap/compliance/email/query.rs | 1293 - tests/src/jmap/compliance/expected.rs | 317 - tests/src/jmap/compliance/identity.rs | 452 - tests/src/jmap/compliance/mailbox.rs | 1304 - tests/src/jmap/compliance/mod.rs | 1181 - tests/src/jmap/compliance/push.rs | 628 - tests/src/jmap/compliance/search_snippet.rs | 263 - tests/src/jmap/compliance/submission.rs | 569 - tests/src/jmap/compliance/thread.rs | 314 - tests/src/jmap/compliance/vacation.rs | 305 - tests/src/jmap/contacts/acl.rs | 674 - tests/src/jmap/contacts/addressbook.rs | 216 - tests/src/jmap/contacts/contact.rs | 1361 - tests/src/jmap/contacts/mod.rs | 9 - tests/src/jmap/core/blob.rs | 505 - tests/src/jmap/core/event_source.rs | 184 - tests/src/jmap/core/mod.rs | 10 - tests/src/jmap/core/push_subscription.rs | 817 - tests/src/jmap/core/websocket.rs | 149 - tests/src/jmap/files/acl.rs | 586 - tests/src/jmap/files/mod.rs | 8 - tests/src/jmap/files/node.rs | 729 - tests/src/jmap/mail/acl.rs | 755 - tests/src/jmap/mail/changes.rs | 336 - tests/src/jmap/mail/copy.rs | 102 - tests/src/jmap/mail/get.rs | 338 - tests/src/jmap/mail/mailbox.rs | 799 - tests/src/jmap/mail/mod.rs | 21 - tests/src/jmap/mail/parse.rs | 229 - tests/src/jmap/mail/query.rs | 908 - tests/src/jmap/mail/query_changes.rs | 337 - tests/src/jmap/mail/search_snippet.rs | 174 - tests/src/jmap/mail/set.rs | 423 - tests/src/jmap/mail/sieve_script.rs | 581 - tests/src/jmap/mail/submission.rs | 873 - tests/src/jmap/mail/thread_get.rs | 52 - tests/src/jmap/mail/thread_merge.rs | 864 - tests/src/jmap/mail/vacation_response.rs | 203 - tests/src/jmap/mod.rs | 287 - tests/src/jmap/principal/availability.rs | 266 - tests/src/jmap/principal/get.rs | 471 - tests/src/jmap/principal/mod.rs | 8 - tests/src/lib.rs | 60 - tests/src/scim/conformance.rs | 177 - tests/src/scim/oidc.rs | 332 - tests/src/scim/tenant.rs | 362 - tests/src/smtp/inbound/antispam.rs | 831 - tests/src/smtp/inbound/asn.rs | 108 - tests/src/smtp/inbound/auth.rs | 178 - tests/src/smtp/inbound/basic.rs | 88 - tests/src/smtp/inbound/data.rs | 391 - tests/src/smtp/inbound/dkim2.rs | 633 - tests/src/smtp/inbound/dmarc.rs | 483 - tests/src/smtp/inbound/ehlo.rs | 173 - tests/src/smtp/inbound/limits.rs | 92 - tests/src/smtp/inbound/mail.rs | 423 - tests/src/smtp/inbound/milter.rs | 1042 - tests/src/smtp/inbound/mod.rs | 412 - tests/src/smtp/inbound/rcpt.rs | 220 - tests/src/smtp/inbound/rewrite.rs | 166 - tests/src/smtp/inbound/scripts.rs | 478 - tests/src/smtp/inbound/sign.rs | 245 - tests/src/smtp/inbound/spam_rules.rs | 364 - tests/src/smtp/inbound/throttle.rs | 126 - tests/src/smtp/inbound/vrfy.rs | 119 - tests/src/smtp/lookup/expressions.rs | 139 - tests/src/smtp/lookup/mod.rs | 8 - tests/src/smtp/lookup/utils.rs | 358 - tests/src/smtp/management/mod.rs | 8 - tests/src/smtp/management/queue.rs | 625 - tests/src/smtp/management/report.rs | 257 - tests/src/smtp/mod.rs | 13 - tests/src/smtp/outbound/dane.rs | 1669 - tests/src/smtp/outbound/extensions.rs | 183 - tests/src/smtp/outbound/fallback_relay.rs | 139 - tests/src/smtp/outbound/ip_lookup.rs | 129 - tests/src/smtp/outbound/lmtp.rs | 296 - tests/src/smtp/outbound/mod.rs | 15 - tests/src/smtp/outbound/mta_sts.rs | 366 - tests/src/smtp/outbound/smtp.rs | 429 - tests/src/smtp/outbound/throttle.rs | 412 - tests/src/smtp/outbound/tls.rs | 161 - tests/src/smtp/queue/concurrent.rs | 217 - tests/src/smtp/queue/dsn.rs | 329 - tests/src/smtp/queue/manager.rs | 209 - tests/src/smtp/queue/mod.rs | 51 - tests/src/smtp/queue/retry.rs | 321 - tests/src/smtp/queue/virtualq.rs | 286 - tests/src/smtp/reporting/analyze.rs | 376 - tests/src/smtp/reporting/dmarc.rs | 189 - tests/src/smtp/reporting/mod.rs | 10 - tests/src/smtp/reporting/scheduler.rs | 189 - tests/src/smtp/reporting/tls.rs | 256 - tests/src/smtp/session.rs | 392 - tests/src/store/blob.rs | 533 - tests/src/store/import_export.rs | 328 - tests/src/store/lookup.rs | 300 - tests/src/store/mod.rs | 68 - tests/src/store/ops.rs | 801 - tests/src/store/query.rs | 811 - tests/src/store/registry.rs | 804 - tests/src/store/sql_timeout.rs | 385 - tests/src/system/antispam.rs | 824 - tests/src/system/authentication.rs | 431 - tests/src/system/authorization.rs | 391 - tests/src/system/crypto.rs | 289 - tests/src/system/delivery.rs | 813 - tests/src/system/directory.rs | 680 - tests/src/system/mod.rs | 76 - tests/src/system/oidc.rs | 1134 - tests/src/system/purge.rs | 255 - tests/src/system/quota.rs | 511 - tests/src/system/security.rs | 391 - tests/src/system/task.rs | 419 - tests/src/telemetry/alerts.rs | 115 - tests/src/telemetry/metrics.rs | 205 - tests/src/telemetry/mod.rs | 66 - tests/src/telemetry/tracing.rs | 171 - tests/src/telemetry/webhooks.rs | 244 - tests/src/utils/account.rs | 299 - tests/src/utils/cleanup.rs | 428 - tests/src/utils/containers.rs | 540 - tests/src/utils/dns.rs | 146 - tests/src/utils/http.rs | 189 - tests/src/utils/http_server.rs | 134 - tests/src/utils/imap.rs | 386 - tests/src/utils/jmap.rs | 945 - tests/src/utils/mod.rs | 22 - tests/src/utils/pop3.rs | 105 - tests/src/utils/registry.rs | 470 - tests/src/utils/server.rs | 622 - tests/src/utils/sieve.rs | 105 - tests/src/utils/smtp.rs | 192 - tests/src/utils/storage.rs | 267 - tests/src/utils/temp_dir.rs | 37 - tests/src/utils/webdav.rs | 1418 - tests/src/webdav/acl.rs | 438 - tests/src/webdav/basic.rs | 107 - tests/src/webdav/cal_alarm.rs | 132 - tests/src/webdav/cal_itip.rs | 451 - tests/src/webdav/cal_query.rs | 1200 - tests/src/webdav/cal_scheduling.rs | 1400 - tests/src/webdav/card_query.rs | 341 - tests/src/webdav/copy_move.rs | 904 - tests/src/webdav/lock.rs | 203 - tests/src/webdav/mkcol.rs | 293 - tests/src/webdav/mod.rs | 330 - tests/src/webdav/multiget.rs | 76 - tests/src/webdav/principals.rs | 515 - tests/src/webdav/prop.rs | 852 - tests/src/webdav/put_get.rs | 634 - tests/src/webdav/sync.rs | 292 - 2137 files changed, 322754 insertions(+), 226619 deletions(-) delete mode 100644 .github/workflows/test.yml create mode 100644 LICENSES/Apache-2.0.txt create mode 100644 LICENSES/MIT.txt create mode 100644 LICENSES/Unlicense.txt create mode 100644 crates/calcard/Cargo.toml create mode 100644 crates/calcard/src/common/archive.rs create mode 100644 crates/calcard/src/common/blob.rs create mode 100644 crates/calcard/src/common/decode.rs create mode 100644 crates/calcard/src/common/elements.rs create mode 100644 crates/calcard/src/common/embedded.rs create mode 100644 crates/calcard/src/common/export.rs create mode 100644 crates/calcard/src/common/format.rs create mode 100644 crates/calcard/src/common/iana.rs create mode 100644 crates/calcard/src/common/jsprop.rs create mode 100644 crates/calcard/src/common/jsprop/ordered.rs create mode 100644 crates/calcard/src/common/jsprop/text.rs create mode 100644 crates/calcard/src/common/mod.rs create mode 100644 crates/calcard/src/common/parser.rs create mode 100644 crates/calcard/src/common/scan.rs create mode 100644 crates/calcard/src/common/stack.rs create mode 100644 crates/calcard/src/common/timezone.rs create mode 100644 crates/calcard/src/common/tokenizer.rs create mode 100644 crates/calcard/src/common/types.rs create mode 100644 crates/calcard/src/common/tzdb.rs create mode 100644 crates/calcard/src/common/writer.rs create mode 100644 crates/calcard/src/datecalc/error.rs create mode 100644 crates/calcard/src/datecalc/mod.rs create mode 100644 crates/calcard/src/datecalc/recurrence.rs create mode 100644 crates/calcard/src/datecalc/rrule.rs create mode 100644 crates/calcard/src/datecalc/weekdate.rs create mode 100644 crates/calcard/src/icalendar/builder.rs create mode 100644 crates/calcard/src/icalendar/dates.rs create mode 100644 crates/calcard/src/icalendar/mod.rs create mode 100644 crates/calcard/src/icalendar/parser.rs create mode 100644 crates/calcard/src/icalendar/rkyv_timezone.rs create mode 100644 crates/calcard/src/icalendar/rkyv_types.rs create mode 100644 crates/calcard/src/icalendar/rkyv_utils.rs create mode 100644 crates/calcard/src/icalendar/rkyv_writer.rs create mode 100644 crates/calcard/src/icalendar/timezone.rs create mode 100644 crates/calcard/src/icalendar/types.rs create mode 100644 crates/calcard/src/icalendar/utils.rs create mode 100644 crates/calcard/src/icalendar/writer.rs create mode 100644 crates/calcard/src/jscalendar/export/convert.rs create mode 100644 crates/calcard/src/jscalendar/export/mod.rs create mode 100644 crates/calcard/src/jscalendar/export/params.rs create mode 100644 crates/calcard/src/jscalendar/export/props.rs create mode 100644 crates/calcard/src/jscalendar/ext.rs create mode 100644 crates/calcard/src/jscalendar/import/convert.rs create mode 100644 crates/calcard/src/jscalendar/import/mod.rs create mode 100644 crates/calcard/src/jscalendar/import/params.rs create mode 100644 crates/calcard/src/jscalendar/import/props.rs create mode 100644 crates/calcard/src/jscalendar/mod.rs create mode 100644 crates/calcard/src/jscalendar/overrides.rs create mode 100644 crates/calcard/src/jscalendar/parser.rs create mode 100644 crates/calcard/src/jscalendar/types.rs create mode 100644 crates/calcard/src/jscontact/export/convert.rs create mode 100644 crates/calcard/src/jscontact/export/entry.rs create mode 100644 crates/calcard/src/jscontact/export/mod.rs create mode 100644 crates/calcard/src/jscontact/export/params.rs create mode 100644 crates/calcard/src/jscontact/export/props.rs create mode 100644 crates/calcard/src/jscontact/import/convert.rs create mode 100644 crates/calcard/src/jscontact/import/entry.rs create mode 100644 crates/calcard/src/jscontact/import/mod.rs create mode 100644 crates/calcard/src/jscontact/import/params.rs create mode 100644 crates/calcard/src/jscontact/import/props.rs create mode 100644 crates/calcard/src/jscontact/mod.rs create mode 100644 crates/calcard/src/jscontact/parser.rs create mode 100644 crates/calcard/src/jscontact/types.rs create mode 100644 crates/calcard/src/lib.rs create mode 100644 crates/calcard/src/vcard/builder.rs create mode 100644 crates/calcard/src/vcard/media_type.rs create mode 100644 crates/calcard/src/vcard/mod.rs create mode 100644 crates/calcard/src/vcard/parser.rs create mode 100644 crates/calcard/src/vcard/rkyv_types.rs create mode 100644 crates/calcard/src/vcard/rkyv_utils.rs create mode 100644 crates/calcard/src/vcard/rkyv_writer.rs create mode 100644 crates/calcard/src/vcard/types.rs create mode 100644 crates/calcard/src/vcard/utils.rs create mode 100644 crates/calcard/src/vcard/writer.rs create mode 100644 crates/common/src/auth/active_script.rs create mode 100644 crates/common/src/auth/oauth/cimd.rs create mode 100644 crates/common/src/auth/oauth/client.rs create mode 100644 crates/common/src/auth/oauth/grant.rs create mode 100644 crates/common/src/auth/oauth/scope.rs create mode 100644 crates/common/src/auth/sasl.rs create mode 100644 crates/common/src/cache/email.rs create mode 100644 crates/common/src/cache/http_auth.rs create mode 100644 crates/common/src/cache/large.rs create mode 100644 crates/common/src/cache/refresh.rs create mode 100644 crates/common/src/cache/swap/blob.rs create mode 100644 crates/common/src/cache/swap/evict.rs create mode 100644 crates/common/src/cache/swap/file.rs create mode 100644 crates/common/src/cache/swap/frame.rs create mode 100644 crates/common/src/cache/swap/messages.rs create mode 100644 crates/common/src/cache/swap/mod.rs create mode 100644 crates/common/src/cache/swap/pointer.rs create mode 100644 crates/common/src/cache/swap/redis.rs create mode 100644 crates/common/src/cache/swap/resources.rs create mode 100644 crates/common/src/cache/swap/writer.rs create mode 100644 crates/common/src/config/mailstore/limits.rs create mode 100644 crates/common/src/config/metadata.rs create mode 100644 crates/common/src/config/resource.rs create mode 100644 crates/common/src/expr/capture.rs create mode 100644 crates/common/src/expr/guard.rs create mode 100644 crates/common/src/expr/kernels/case.rs create mode 100644 crates/common/src/expr/kernels/encode.rs create mode 100644 crates/common/src/expr/kernels/mod.rs create mode 100644 crates/common/src/expr/program.rs create mode 100644 crates/common/src/expr/variable.rs create mode 100644 crates/common/src/ipc/subscribers.rs create mode 100644 crates/common/src/manager/app_client.rs create mode 100644 crates/common/src/manager/schema.rs create mode 100644 crates/common/src/network/acme/persist.rs create mode 100644 crates/common/src/network/asn/table.rs create mode 100644 crates/common/src/network/forwarded.rs create mode 100644 crates/common/src/network/memory.rs create mode 100644 crates/common/src/scripts/functions/sniff.rs delete mode 100644 crates/common/src/sharing/acl.rs create mode 100644 crates/common/src/sharing/file.rs create mode 100644 crates/common/src/sharing/grants.rs create mode 100644 crates/common/src/storage/batch.rs create mode 100644 crates/common/src/storage/chunk.rs delete mode 100644 crates/common/src/storage/dav.rs create mode 100644 crates/common/src/storage/dav/file.rs create mode 100644 crates/common/src/storage/dav/hierarchy.rs create mode 100644 crates/common/src/storage/dav/mod.rs create mode 100644 crates/common/src/storage/dav/paths.rs create mode 100644 crates/common/src/storage/dav/presence.rs create mode 100644 crates/common/src/storage/dav/resource.rs create mode 100644 crates/common/src/storage/dav/store.rs delete mode 100644 crates/common/src/storage/index.rs create mode 100644 crates/common/src/storage/index/build.rs create mode 100644 crates/common/src/storage/index/mod.rs create mode 100644 crates/common/src/storage/index/object.rs create mode 100644 crates/common/src/storage/index/presence.rs create mode 100644 crates/common/src/storage/index/split.rs create mode 100644 crates/common/src/storage/index/value.rs create mode 100644 crates/common/src/storage/metadata/change.rs create mode 100644 crates/common/src/storage/metadata/cleanup.rs create mode 100644 crates/common/src/storage/metadata/containers.rs create mode 100644 crates/common/src/storage/metadata/mod.rs create mode 100644 crates/common/src/storage/metadata/purge.rs create mode 100644 crates/common/src/storage/metadata/quota.rs create mode 100644 crates/common/src/storage/metadata/read.rs create mode 100644 crates/common/src/storage/metadata/stored.rs create mode 100644 crates/common/src/storage/metadata/viewer.rs create mode 100644 crates/common/src/storage/search.rs create mode 100644 crates/common/src/telemetry/tracers/spans.rs delete mode 100644 crates/dav-proto/resources/requests/acl-001.json delete mode 100644 crates/dav-proto/resources/requests/acl-001.xml delete mode 100644 crates/dav-proto/resources/requests/acl-002.json delete mode 100644 crates/dav-proto/resources/requests/acl-002.xml delete mode 100644 crates/dav-proto/resources/requests/acl-003.json delete mode 100644 crates/dav-proto/resources/requests/acl-003.xml delete mode 100644 crates/dav-proto/resources/requests/acl-004.json delete mode 100644 crates/dav-proto/resources/requests/acl-004.xml delete mode 100644 crates/dav-proto/resources/requests/lockinfo-001.json delete mode 100644 crates/dav-proto/resources/requests/lockinfo-001.xml delete mode 100644 crates/dav-proto/resources/requests/lockinfo-002.json delete mode 100644 crates/dav-proto/resources/requests/lockinfo-002.xml delete mode 100644 crates/dav-proto/resources/requests/mkcol-001.json delete mode 100644 crates/dav-proto/resources/requests/mkcol-001.xml delete mode 100644 crates/dav-proto/resources/requests/mkcol-002.json delete mode 100644 crates/dav-proto/resources/requests/mkcol-002.xml delete mode 100644 crates/dav-proto/resources/requests/mkcol-003.json delete mode 100644 crates/dav-proto/resources/requests/mkcol-003.xml delete mode 100644 crates/dav-proto/resources/requests/mkcol-004.json delete mode 100644 crates/dav-proto/resources/requests/mkcol-004.xml delete mode 100644 crates/dav-proto/resources/requests/propertyupdate-001.json delete mode 100644 crates/dav-proto/resources/requests/propertyupdate-001.xml delete mode 100644 crates/dav-proto/resources/requests/propertyupdate-002.json delete mode 100644 crates/dav-proto/resources/requests/propertyupdate-002.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-001.json delete mode 100644 crates/dav-proto/resources/requests/propfind-001.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-002.json delete mode 100644 crates/dav-proto/resources/requests/propfind-002.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-003.json delete mode 100644 crates/dav-proto/resources/requests/propfind-003.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-004.json delete mode 100644 crates/dav-proto/resources/requests/propfind-004.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-005.json delete mode 100644 crates/dav-proto/resources/requests/propfind-005.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-006.json delete mode 100644 crates/dav-proto/resources/requests/propfind-006.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-007.json delete mode 100644 crates/dav-proto/resources/requests/propfind-007.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-008.json delete mode 100644 crates/dav-proto/resources/requests/propfind-008.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-009.json delete mode 100644 crates/dav-proto/resources/requests/propfind-009.xml delete mode 100644 crates/dav-proto/resources/requests/propfind-010.json delete mode 100644 crates/dav-proto/resources/requests/propfind-010.xml delete mode 100644 crates/dav-proto/resources/requests/report-001.json delete mode 100644 crates/dav-proto/resources/requests/report-001.xml delete mode 100644 crates/dav-proto/resources/requests/report-002.json delete mode 100644 crates/dav-proto/resources/requests/report-002.xml delete mode 100644 crates/dav-proto/resources/requests/report-003.json delete mode 100644 crates/dav-proto/resources/requests/report-003.xml delete mode 100644 crates/dav-proto/resources/requests/report-004.json delete mode 100644 crates/dav-proto/resources/requests/report-004.xml delete mode 100644 crates/dav-proto/resources/requests/report-005.json delete mode 100644 crates/dav-proto/resources/requests/report-005.xml delete mode 100644 crates/dav-proto/resources/requests/report-006.json delete mode 100644 crates/dav-proto/resources/requests/report-006.xml delete mode 100644 crates/dav-proto/resources/requests/report-007.json delete mode 100644 crates/dav-proto/resources/requests/report-007.xml delete mode 100644 crates/dav-proto/resources/requests/report-008.json delete mode 100644 crates/dav-proto/resources/requests/report-008.xml delete mode 100644 crates/dav-proto/resources/requests/report-009.json delete mode 100644 crates/dav-proto/resources/requests/report-009.xml delete mode 100644 crates/dav-proto/resources/requests/report-010.json delete mode 100644 crates/dav-proto/resources/requests/report-010.xml delete mode 100644 crates/dav-proto/resources/requests/report-011.json delete mode 100644 crates/dav-proto/resources/requests/report-011.xml delete mode 100644 crates/dav-proto/resources/requests/report-012.json delete mode 100644 crates/dav-proto/resources/requests/report-012.xml delete mode 100644 crates/dav-proto/resources/requests/report-013.json delete mode 100644 crates/dav-proto/resources/requests/report-013.xml delete mode 100644 crates/dav-proto/resources/requests/report-014.json delete mode 100644 crates/dav-proto/resources/requests/report-014.xml delete mode 100644 crates/dav-proto/resources/requests/report-015.json delete mode 100644 crates/dav-proto/resources/requests/report-015.xml delete mode 100644 crates/dav-proto/resources/requests/report-016.json delete mode 100644 crates/dav-proto/resources/requests/report-016.xml delete mode 100644 crates/dav-proto/resources/requests/report-017.json delete mode 100644 crates/dav-proto/resources/requests/report-017.xml delete mode 100644 crates/dav-proto/resources/requests/report-018.json delete mode 100644 crates/dav-proto/resources/requests/report-018.xml delete mode 100644 crates/dav-proto/resources/requests/report-019.json delete mode 100644 crates/dav-proto/resources/requests/report-019.xml delete mode 100644 crates/dav-proto/resources/requests/report-020.json delete mode 100644 crates/dav-proto/resources/requests/report-020.xml delete mode 100644 crates/dav-proto/resources/requests/report-021.json delete mode 100644 crates/dav-proto/resources/requests/report-021.xml delete mode 100644 crates/dav-proto/resources/requests/report-022.json delete mode 100644 crates/dav-proto/resources/requests/report-022.xml delete mode 100644 crates/dav-proto/resources/requests/report-023.json delete mode 100644 crates/dav-proto/resources/requests/report-023.xml delete mode 100644 crates/dav-proto/resources/requests/report-024.json delete mode 100644 crates/dav-proto/resources/requests/report-024.xml delete mode 100644 crates/dav-proto/resources/requests/report-025.json delete mode 100644 crates/dav-proto/resources/requests/report-025.xml delete mode 100644 crates/dav-proto/resources/responses/001.xml delete mode 100644 crates/dav-proto/resources/responses/002.xml delete mode 100644 crates/dav-proto/resources/responses/003.xml delete mode 100644 crates/dav-proto/resources/responses/004.xml delete mode 100644 crates/dav-proto/resources/responses/005.xml delete mode 100644 crates/dav-proto/resources/responses/006.xml delete mode 100644 crates/dav-proto/resources/responses/007.xml delete mode 100644 crates/dav-proto/resources/responses/008.xml delete mode 100644 crates/dav-proto/resources/responses/009.xml delete mode 100644 crates/dav-proto/resources/responses/010.xml delete mode 100644 crates/dav-proto/resources/responses/011.xml delete mode 100644 crates/dav-proto/resources/responses/012.xml delete mode 100644 crates/dav-proto/resources/responses/013.xml delete mode 100644 crates/dav-proto/resources/responses/014.xml delete mode 100644 crates/dav-proto/resources/responses/015.xml delete mode 100644 crates/dav-proto/resources/responses/016.xml delete mode 100644 crates/dav-proto/resources/responses/017.xml delete mode 100644 crates/dav-proto/resources/responses/018.xml delete mode 100644 crates/dav-proto/resources/responses/019.xml delete mode 100644 crates/dav-proto/resources/responses/020.xml create mode 100644 crates/dav-proto/src/parser/value.rs create mode 100644 crates/dav-proto/src/requests/filter.rs create mode 100644 crates/dav-proto/src/schema/collation.rs create mode 100644 crates/dav/src/calendar/filter.rs create mode 100644 crates/dav/src/calendar/view.rs create mode 100644 crates/dav/src/card/filter.rs create mode 100644 crates/dav/src/common/dead/containers.rs create mode 100644 crates/dav/src/common/dead/mod.rs create mode 100644 crates/dav/src/common/dead/patch.rs delete mode 100644 crates/dav/src/common/propfind.rs create mode 100644 crates/dav/src/common/propfind/data.rs create mode 100644 crates/dav/src/common/propfind/dead.rs create mode 100644 crates/dav/src/common/propfind/get.rs create mode 100644 crates/dav/src/common/propfind/item.rs create mode 100644 crates/dav/src/common/propfind/load.rs create mode 100644 crates/dav/src/common/propfind/mod.rs create mode 100644 crates/dav/src/common/search.rs create mode 100644 crates/directory/src/backend/ldap/failover.rs create mode 100644 crates/directory/src/backend/oidc/jwks.rs create mode 100644 crates/directory/src/core/verifier.rs create mode 100644 crates/email/src/mailbox/lookup.rs create mode 100644 crates/email/src/mailbox/role.rs create mode 100644 crates/email/src/message/copy/sources.rs create mode 100644 crates/email/src/message/index/attachment.rs create mode 100644 crates/email/src/message/index/subject.rs create mode 100644 crates/email/src/message/ingest/checks.rs create mode 100644 crates/email/src/message/ingest/group.rs create mode 100644 crates/email/src/message/ingest/recheck.rs create mode 100644 crates/email/src/message/ingest_metadata.rs create mode 100644 crates/email/src/message/jmap/body.rs create mode 100644 crates/email/src/message/jmap/estimate.rs create mode 100644 crates/email/src/message/jmap/header.rs create mode 100644 crates/email/src/message/jmap/mod.rs create mode 100644 crates/email/src/message/messagedata.rs delete mode 100644 crates/email/src/message/metadata.rs create mode 100644 crates/email/src/message/metadata/build.rs create mode 100644 crates/email/src/message/metadata/header_id.rs create mode 100644 crates/email/src/message/metadata/mod.rs create mode 100644 crates/email/src/message/metadata/row.rs create mode 100644 crates/email/src/message/metadata/source.rs create mode 100644 crates/email/src/message/metadata/view.rs create mode 100644 crates/email/src/message/sortkeys.rs create mode 100644 crates/email/src/message/thread.rs create mode 100644 crates/email/src/presence.rs create mode 100644 crates/groupware/src/calendar/alerts.rs create mode 100644 crates/groupware/src/calendar/availability.rs create mode 100644 crates/groupware/src/calendar/color.rs create mode 100644 crates/groupware/src/calendar/compare.rs create mode 100644 crates/groupware/src/calendar/hidden.rs create mode 100644 crates/groupware/src/calendar/identity.rs create mode 100644 crates/groupware/src/calendar/instance_filter.rs create mode 100644 crates/groupware/src/calendar/notification.rs create mode 100644 crates/groupware/src/calendar/overlap.rs create mode 100644 crates/groupware/src/calendar/participants.rs create mode 100644 crates/groupware/src/calendar/privacy.rs create mode 100644 crates/groupware/src/calendar/rights.rs create mode 100644 crates/groupware/src/calendar/schedule.rs create mode 100644 crates/groupware/src/calendar/sequence.rs create mode 100644 crates/groupware/src/calendar/user.rs create mode 100644 crates/groupware/src/file/content.rs create mode 100644 crates/groupware/src/file/symlink.rs create mode 100644 crates/groupware/src/metadata/mod.rs create mode 100644 crates/groupware/src/scheduling/recipient.rs create mode 100644 crates/groupware/src/scheduling/recipient/series.rs create mode 100644 crates/groupware/src/scheduling/recipient/series/occurrence.rs create mode 100644 crates/groupware/src/scheduling/recipient/split.rs create mode 100644 crates/http-proto/src/body.rs create mode 100644 crates/http/src/auth/oauth/authorize.rs create mode 100644 crates/http/src/auth/oauth/logout.rs create mode 100644 crates/http/src/stream.rs create mode 100644 crates/imap-proto/src/parser/metadata.rs create mode 100644 crates/imap-proto/src/protocol/metadata.rs delete mode 100644 crates/imap-proto/src/utf7.rs delete mode 100644 crates/imap/src/core/message.rs create mode 100644 crates/imap/src/core/rights.rs create mode 100644 crates/imap/src/core/view.rs delete mode 100644 crates/imap/src/op/fetch.rs create mode 100644 crates/imap/src/op/fetch/mod.rs create mode 100644 crates/imap/src/op/fetch/section.rs create mode 100644 crates/imap/src/op/fetch/source.rs create mode 100644 crates/imap/src/op/fetch/structure.rs create mode 100644 crates/imap/src/op/metadata/edit.rs create mode 100644 crates/imap/src/op/metadata/get.rs create mode 100644 crates/imap/src/op/metadata/list.rs create mode 100644 crates/imap/src/op/metadata/mod.rs create mode 100644 crates/imap/src/op/metadata/select.rs create mode 100644 crates/imap/src/op/metadata/set.rs create mode 100644 crates/jmap-proto/src/object/metadata.rs create mode 100644 crates/jmap-tools/Cargo.toml create mode 100644 crates/jmap-tools/src/json/de.rs create mode 100644 crates/jmap-tools/src/json/deserializer.rs create mode 100644 crates/jmap-tools/src/json/index.rs create mode 100644 crates/jmap-tools/src/json/key.rs create mode 100644 crates/jmap-tools/src/json/mod.rs create mode 100644 crates/jmap-tools/src/json/num.rs create mode 100644 crates/jmap-tools/src/json/object_vec.rs create mode 100644 crates/jmap-tools/src/json/parser.rs create mode 100644 crates/jmap-tools/src/json/rkyv.rs create mode 100644 crates/jmap-tools/src/json/ser.rs create mode 100644 crates/jmap-tools/src/json/value.rs create mode 100644 crates/jmap-tools/src/lib.rs create mode 100644 crates/jmap-tools/src/pointer/eval.rs create mode 100644 crates/jmap-tools/src/pointer/mod.rs create mode 100644 crates/jmap-tools/src/pointer/parser.rs create mode 100644 crates/jmap/src/api/budget.rs create mode 100644 crates/jmap/src/api/metadata/apply.rs create mode 100644 crates/jmap/src/api/metadata/copy.rs create mode 100644 crates/jmap/src/api/metadata/filter.rs create mode 100644 crates/jmap/src/api/metadata/get.rs create mode 100644 crates/jmap/src/api/metadata/mod.rs create mode 100644 crates/jmap/src/api/metadata/object.rs create mode 100644 crates/jmap/src/api/metadata/patch.rs create mode 100644 crates/jmap/src/api/metadata/preload.rs create mode 100644 crates/jmap/src/api/metadata/prepared.rs create mode 100644 crates/jmap/src/api/metadata/query.rs create mode 100644 crates/jmap/src/api/metadata/reject.rs create mode 100644 crates/jmap/src/api/metadata/validate.rs create mode 100644 crates/jmap/src/api/metadata/violation.rs create mode 100644 crates/jmap/src/api/metadata/write.rs create mode 100644 crates/jmap/src/api/metadata/writer.rs create mode 100644 crates/jmap/src/api/parent_ref.rs create mode 100644 crates/jmap/src/api/pending_creates.rs create mode 100644 crates/jmap/src/blob/embedded.rs create mode 100644 crates/jmap/src/calendar_event/hidden.rs create mode 100644 crates/jmap/src/calendar_event/privacy.rs create mode 100644 crates/jmap/src/calendar_event/server_set.rs create mode 100644 crates/jmap/src/calendar_event/user.rs create mode 100644 crates/jmap/src/calendar_event/validate.rs create mode 100644 crates/jmap/src/calendar_event_notification/patch.rs create mode 100644 crates/jmap/src/changes/page.rs create mode 100644 crates/jmap/src/email/grouped.rs create mode 100644 crates/jmap/src/email/query/filter.rs create mode 100644 crates/jmap/src/email/query/text.rs create mode 100644 crates/jmap/src/email/query/window.rs create mode 100644 crates/jmap/src/file/node.rs create mode 100644 crates/jmap/src/file/writer.rs create mode 100644 crates/jmap/src/participant_identity/changes.rs create mode 100644 crates/jmap/src/registry/mapping/index_queue.rs create mode 100644 crates/jmap/src/registry/mapping/oauth_client.rs delete mode 100644 crates/migration/Cargo.toml delete mode 100644 crates/migration/src/destroy.rs delete mode 100644 crates/migration/src/lib.rs delete mode 100644 crates/migration/src/v016.rs create mode 100644 crates/nlp/build.rs create mode 100644 crates/nlp/src/tokenizers/types/scan.rs create mode 100644 crates/nlp/src/tokenizers/unicode.rs create mode 100644 crates/pop3/src/protocol/dot_stuffer.rs create mode 100644 crates/s3/Cargo.toml create mode 100644 crates/s3/src/client.rs create mode 100644 crates/s3/src/credentials.rs create mode 100644 crates/s3/src/delete.rs create mode 100644 crates/s3/src/endpoint.rs create mode 100644 crates/s3/src/error.rs create mode 100644 crates/s3/src/lib.rs create mode 100644 crates/s3/src/retry.rs create mode 100644 crates/s3/src/signer.rs create mode 100644 crates/services/src/broadcast/backlog.rs create mode 100644 crates/services/src/index_queue/document.rs create mode 100644 crates/services/src/index_queue/file.rs create mode 100644 crates/services/src/index_queue/index.rs create mode 100644 crates/services/src/index_queue/manager.rs create mode 100644 crates/services/src/index_queue/mod.rs create mode 100644 crates/services/src/index_queue/reindex.rs delete mode 100644 crates/services/src/task_manager/acme.rs delete mode 100644 crates/services/src/task_manager/alarm.rs delete mode 100644 crates/services/src/task_manager/destroy_account.rs delete mode 100644 crates/services/src/task_manager/imip.rs delete mode 100644 crates/services/src/task_manager/index.rs create mode 100644 crates/services/src/task_manager/outcome.rs create mode 100644 crates/services/src/task_manager/reconcile.rs delete mode 100644 crates/services/src/task_manager/report.rs create mode 100644 crates/services/src/task_manager/tasks/acme.rs create mode 100644 crates/services/src/task_manager/tasks/alarm.rs create mode 100644 crates/services/src/task_manager/tasks/destroy_account.rs rename crates/services/src/task_manager/{ => tasks}/dkim.rs (66%) rename crates/services/src/task_manager/{ => tasks}/dns.rs (74%) create mode 100644 crates/services/src/task_manager/tasks/grantee_cleanup.rs create mode 100644 crates/services/src/task_manager/tasks/grantee_cleanup/cleanup.rs create mode 100644 crates/services/src/task_manager/tasks/grantee_cleanup/stage.rs create mode 100644 crates/services/src/task_manager/tasks/imip.rs rename crates/services/src/task_manager/{ => tasks}/maintenance.rs (66%) rename crates/services/src/task_manager/{ => tasks}/merge_threads.rs (74%) create mode 100644 crates/services/src/task_manager/tasks/mod.rs create mode 100644 crates/services/src/task_manager/tasks/rebuild_shares.rs create mode 100644 crates/services/src/task_manager/tasks/report.rs rename crates/services/src/task_manager/{ => tasks}/restore_item.rs (80%) rename crates/services/src/task_manager/{ => tasks}/spam_classifier.rs (78%) create mode 100644 crates/services/src/task_manager/worker.rs create mode 100644 crates/sieve/Cargo.toml create mode 100644 crates/sieve/src/bytecode/cursor.rs create mode 100644 crates/sieve/src/bytecode/emit.rs create mode 100644 crates/sieve/src/bytecode/header_id.rs create mode 100644 crates/sieve/src/bytecode/mod.rs create mode 100644 crates/sieve/src/bytecode/ops.rs create mode 100644 crates/sieve/src/bytecode/rec.rs create mode 100644 crates/sieve/src/bytecode/verify.rs create mode 100644 crates/sieve/src/compiler/emit.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_convert.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_editheader.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_fileinto.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_flags.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_include.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_keep.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_mime.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_notify.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_redirect.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_reject.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_require.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_set.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/action_vacation.rs create mode 100644 crates/sieve/src/compiler/grammar/actions/mod.rs create mode 100644 crates/sieve/src/compiler/grammar/expr/mod.rs create mode 100644 crates/sieve/src/compiler/grammar/expr/parser.rs create mode 100644 crates/sieve/src/compiler/grammar/expr/tokenizer.rs create mode 100644 crates/sieve/src/compiler/grammar/instruction.rs create mode 100644 crates/sieve/src/compiler/grammar/mod.rs create mode 100644 crates/sieve/src/compiler/grammar/test.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/mod.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_address.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_body.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_date.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_duplicate.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_envelope.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_environment.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_exists.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_extlists.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_hasflag.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_header.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_ihave.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_mailbox.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_mailboxid.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_notify.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_size.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_spamtest.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_specialuse.rs create mode 100644 crates/sieve/src/compiler/grammar/tests/test_string.rs create mode 100644 crates/sieve/src/compiler/lexer/mod.rs create mode 100644 crates/sieve/src/compiler/lexer/string.rs create mode 100644 crates/sieve/src/compiler/lexer/tokenizer.rs create mode 100644 crates/sieve/src/compiler/lexer/word.rs create mode 100644 crates/sieve/src/compiler/mod.rs create mode 100644 crates/sieve/src/lib.rs create mode 100644 crates/sieve/src/regex.rs create mode 100644 crates/sieve/src/runtime/actions/action_convert.rs create mode 100644 crates/sieve/src/runtime/actions/action_editheader.rs create mode 100644 crates/sieve/src/runtime/actions/action_fileinto.rs create mode 100644 crates/sieve/src/runtime/actions/action_flags.rs create mode 100644 crates/sieve/src/runtime/actions/action_keep.rs create mode 100644 crates/sieve/src/runtime/actions/action_mime.rs create mode 100644 crates/sieve/src/runtime/actions/action_notify.rs create mode 100644 crates/sieve/src/runtime/actions/action_redirect.rs create mode 100644 crates/sieve/src/runtime/actions/action_set.rs create mode 100644 crates/sieve/src/runtime/actions/action_vacation.rs create mode 100644 crates/sieve/src/runtime/actions/mod.rs create mode 100644 crates/sieve/src/runtime/arena.rs create mode 100644 crates/sieve/src/runtime/context.rs create mode 100644 crates/sieve/src/runtime/eval.rs create mode 100644 crates/sieve/src/runtime/expression.rs create mode 100644 crates/sieve/src/runtime/handler.rs create mode 100644 crates/sieve/src/runtime/message/body.rs create mode 100644 crates/sieve/src/runtime/message/edits.rs create mode 100644 crates/sieve/src/runtime/message/headers.rs rename {tests/src/cluster => crates/sieve/src/runtime/message}/mod.rs (54%) create mode 100644 crates/sieve/src/runtime/message/parts.rs create mode 100644 crates/sieve/src/runtime/message/serialize.rs create mode 100644 crates/sieve/src/runtime/mod.rs create mode 100644 crates/sieve/src/runtime/platform.rs create mode 100644 crates/sieve/src/runtime/tests/comparator.rs create mode 100644 crates/sieve/src/runtime/tests/glob.rs create mode 100644 crates/sieve/src/runtime/tests/matching.rs create mode 100644 crates/sieve/src/runtime/tests/mime.rs create mode 100644 crates/sieve/src/runtime/tests/mod.rs create mode 100644 crates/sieve/src/runtime/tests/test_address.rs create mode 100644 crates/sieve/src/runtime/tests/test_body.rs create mode 100644 crates/sieve/src/runtime/tests/test_date.rs create mode 100644 crates/sieve/src/runtime/tests/test_duplicate.rs create mode 100644 crates/sieve/src/runtime/tests/test_envelope.rs create mode 100644 crates/sieve/src/runtime/tests/test_exists.rs create mode 100644 crates/sieve/src/runtime/tests/test_extlists.rs create mode 100644 crates/sieve/src/runtime/tests/test_hasflag.rs create mode 100644 crates/sieve/src/runtime/tests/test_header.rs create mode 100644 crates/sieve/src/runtime/tests/test_metadata.rs create mode 100644 crates/sieve/src/runtime/tests/test_notify.rs create mode 100644 crates/sieve/src/runtime/tests/test_size.rs create mode 100644 crates/sieve/src/runtime/tests/test_spamtest.rs create mode 100644 crates/sieve/src/runtime/tests/test_string.rs create mode 100644 crates/sieve/src/runtime/variable.rs create mode 100644 crates/sieve/src/runtime/variables.rs create mode 100644 crates/sieve/src/sieve.rs create mode 100644 crates/smtp-proto/Cargo.toml create mode 100644 crates/smtp-proto/src/lib.rs create mode 100644 crates/smtp-proto/src/request/mod.rs create mode 100644 crates/smtp-proto/src/request/parser.rs create mode 100644 crates/smtp-proto/src/request/receiver.rs create mode 100644 crates/smtp-proto/src/request/scan.rs create mode 100644 crates/smtp-proto/src/response/generate.rs create mode 100644 crates/smtp-proto/src/response/mod.rs create mode 100644 crates/smtp-proto/src/response/parser.rs create mode 100644 crates/smtp-proto/src/tokens.rs create mode 100644 crates/smtp-report/Cargo.toml create mode 100644 crates/smtp-report/src/arf/builder.rs create mode 100644 crates/smtp-report/src/arf/generate.rs create mode 100644 crates/smtp-report/src/arf/mod.rs create mode 100644 crates/smtp-report/src/arf/parse.rs create mode 100644 crates/smtp-report/src/dmarc/builder.rs create mode 100644 crates/smtp-report/src/dmarc/generate.rs create mode 100644 crates/smtp-report/src/dmarc/mod.rs create mode 100644 crates/smtp-report/src/dmarc/parse.rs create mode 100644 crates/smtp-report/src/headers.rs create mode 100644 crates/smtp-report/src/lib.rs create mode 100644 crates/smtp-report/src/tlsrpt/generate.rs create mode 100644 crates/smtp-report/src/tlsrpt/mod.rs create mode 100644 crates/smtp-report/src/tlsrpt/parse.rs create mode 100644 crates/smtp-report/src/utf8.rs create mode 100644 crates/smtp/src/queue/submission.rs create mode 100644 crates/smtp/src/queue/walk.rs create mode 100644 crates/smtp/src/reporting/aggregate/codec.rs create mode 100644 crates/smtp/src/reporting/aggregate/mod.rs create mode 100644 crates/smtp/src/reporting/aggregate/window.rs create mode 100644 crates/spam-filter/src/texts.rs create mode 100644 crates/store/src/backend/azure/shared_key.rs create mode 100644 crates/store/src/backend/foundationdb/assembly.rs create mode 100644 crates/store/src/backend/foundationdb/preread.rs create mode 100644 crates/store/src/backend/foundationdb/window.rs create mode 100644 crates/store/src/backend/local/entry.rs create mode 100644 crates/store/src/backend/local/maintenance.rs create mode 100644 crates/store/src/backend/local/mod.rs create mode 100644 crates/store/src/backend/mysql/pooled.rs create mode 100644 crates/store/src/backend/mysql/sql.rs create mode 100644 crates/store/src/backend/mysql/write/reads.rs create mode 100644 crates/store/src/backend/postgres/pool.rs create mode 100644 crates/store/src/backend/postgres/pooled.rs create mode 100644 crates/store/src/backend/postgres/sql.rs create mode 100644 crates/store/src/backend/postgres/write/flight.rs create mode 100644 crates/store/src/backend/postgres/write/reads.rs create mode 100644 crates/store/src/backend/range_batch.rs create mode 100644 crates/store/src/backend/rocksdb/blob_gc.rs create mode 100644 crates/store/src/backend/rocksdb/cache.rs create mode 100644 crates/store/src/backend/rocksdb/slots.rs create mode 100644 crates/store/src/backend/sqlite/sql.rs create mode 100644 crates/store/src/build/swap.rs delete mode 100644 crates/store/src/search/bm_u32.rs delete mode 100644 crates/store/src/search/bm_u64.rs create mode 100644 crates/store/src/search/codec.rs create mode 100644 crates/store/src/search/leaf.rs create mode 100644 crates/store/src/search/tokenize.rs create mode 100644 crates/store/src/write/compress.rs create mode 100644 crates/store/src/write/counter.rs create mode 100644 crates/store/src/write/lazybitmap.rs create mode 100644 crates/store/src/write/metadata.rs create mode 100644 crates/store/src/write/new_key.rs create mode 100644 crates/text-extract/CREDITS.md create mode 100644 crates/text-extract/Cargo.toml create mode 100644 crates/text-extract/src/epub.rs create mode 100644 crates/text-extract/src/html.rs create mode 100644 crates/text-extract/src/lib.rs create mode 100644 crates/text-extract/src/odf.rs create mode 100644 crates/text-extract/src/ooxml/chart.rs create mode 100644 crates/text-extract/src/ooxml/comments.rs create mode 100644 crates/text-extract/src/ooxml/layout.rs create mode 100644 crates/text-extract/src/ooxml/mod.rs create mode 100644 crates/text-extract/src/ooxml/number/decimal.rs create mode 100644 crates/text-extract/src/ooxml/number/mod.rs create mode 100644 crates/text-extract/src/ooxml/number/render.rs create mode 100644 crates/text-extract/src/ooxml/pptx.rs create mode 100644 crates/text-extract/src/ooxml/rels.rs create mode 100644 crates/text-extract/src/ooxml/sheet.rs create mode 100644 crates/text-extract/src/ooxml/styles.rs create mode 100644 crates/text-extract/src/ooxml/text.rs create mode 100644 crates/text-extract/src/ooxml/xlsx.rs create mode 100644 crates/text-extract/src/output.rs create mode 100644 crates/text-extract/src/package.rs create mode 100644 crates/text-extract/src/pdf/content/form.rs create mode 100644 crates/text-extract/src/pdf/content/inline/markers.rs create mode 100644 crates/text-extract/src/pdf/content/inline/mod.rs create mode 100644 crates/text-extract/src/pdf/content/mod.rs create mode 100644 crates/text-extract/src/pdf/content/ops.rs create mode 100644 crates/text-extract/src/pdf/content/resources.rs create mode 100644 crates/text-extract/src/pdf/content/state.rs create mode 100644 crates/text-extract/src/pdf/content/text.rs create mode 100644 crates/text-extract/src/pdf/crypt/aes.rs create mode 100644 crates/text-extract/src/pdf/crypt/derive.rs create mode 100644 crates/text-extract/src/pdf/crypt/key.rs create mode 100644 crates/text-extract/src/pdf/crypt/legacy.rs create mode 100644 crates/text-extract/src/pdf/crypt/mod.rs create mode 100644 crates/text-extract/src/pdf/crypt/modern.rs create mode 100644 crates/text-extract/src/pdf/crypt/rc4.rs create mode 100644 crates/text-extract/src/pdf/decode.rs create mode 100644 crates/text-extract/src/pdf/document/crypt.rs create mode 100644 crates/text-extract/src/pdf/document/fetch.rs create mode 100644 crates/text-extract/src/pdf/document/mod.rs create mode 100644 crates/text-extract/src/pdf/document/recover.rs create mode 100644 crates/text-extract/src/pdf/document/streams.rs create mode 100644 crates/text-extract/src/pdf/document/table.rs create mode 100644 crates/text-extract/src/pdf/filter/ascii85.rs create mode 100644 crates/text-extract/src/pdf/filter/ascii_hex.rs create mode 100644 crates/text-extract/src/pdf/filter/flate.rs create mode 100644 crates/text-extract/src/pdf/filter/lzw.rs create mode 100644 crates/text-extract/src/pdf/filter/mod.rs create mode 100644 crates/text-extract/src/pdf/filter/output.rs create mode 100644 crates/text-extract/src/pdf/filter/predictor.rs create mode 100644 crates/text-extract/src/pdf/filter/run_length.rs create mode 100644 crates/text-extract/src/pdf/filter/stage.rs create mode 100644 crates/text-extract/src/pdf/font/cache.rs create mode 100644 crates/text-extract/src/pdf/font/cmap/mod.rs create mode 100644 crates/text-extract/src/pdf/font/code.rs create mode 100644 crates/text-extract/src/pdf/font/composite.rs create mode 100644 crates/text-extract/src/pdf/font/embedded.rs create mode 100644 crates/text-extract/src/pdf/font/mod.rs create mode 100644 crates/text-extract/src/pdf/font/predefined.rs create mode 100644 crates/text-extract/src/pdf/font/program/cff.rs create mode 100644 crates/text-extract/src/pdf/font/program/cff_data.rs create mode 100644 crates/text-extract/src/pdf/font/program/cmap.rs create mode 100644 crates/text-extract/src/pdf/font/program/mod.rs create mode 100644 crates/text-extract/src/pdf/font/program/post.rs create mode 100644 crates/text-extract/src/pdf/font/program/post_data.rs create mode 100644 crates/text-extract/src/pdf/font/program/read.rs create mode 100644 crates/text-extract/src/pdf/font/program/sfnt.rs create mode 100644 crates/text-extract/src/pdf/font/program/truetype.rs create mode 100644 crates/text-extract/src/pdf/font/program/type1.rs create mode 100644 crates/text-extract/src/pdf/font/ranges.rs create mode 100644 crates/text-extract/src/pdf/font/simple/encoding.rs create mode 100644 crates/text-extract/src/pdf/font/simple/mod.rs create mode 100644 crates/text-extract/src/pdf/font/simple/widths.rs create mode 100644 crates/text-extract/src/pdf/font/unicode.rs create mode 100644 crates/text-extract/src/pdf/font/widths.rs create mode 100644 crates/text-extract/src/pdf/forms.rs create mode 100644 crates/text-extract/src/pdf/layout/dedupe.rs create mode 100644 crates/text-extract/src/pdf/layout/line.rs create mode 100644 crates/text-extract/src/pdf/layout/mod.rs create mode 100644 crates/text-extract/src/pdf/layout/script.rs create mode 100644 crates/text-extract/src/pdf/layout/shift.rs create mode 100644 crates/text-extract/src/pdf/lexer/mod.rs create mode 100644 crates/text-extract/src/pdf/lexer/number.rs create mode 100644 crates/text-extract/src/pdf/mod.rs create mode 100644 crates/text-extract/src/pdf/object/container.rs create mode 100644 crates/text-extract/src/pdf/object/mod.rs create mode 100644 crates/text-extract/src/pdf/object/string.rs create mode 100644 crates/text-extract/src/pdf/objstm.rs create mode 100644 crates/text-extract/src/pdf/pages.rs create mode 100644 crates/text-extract/src/pdf/repair.rs create mode 100644 crates/text-extract/src/pdf/seek.rs create mode 100644 crates/text-extract/src/pdf/source.rs create mode 100644 crates/text-extract/src/pdf/tables/cid.rs create mode 100644 crates/text-extract/src/pdf/tables/cid_cns1.bin create mode 100644 crates/text-extract/src/pdf/tables/cid_data.rs create mode 100644 crates/text-extract/src/pdf/tables/cid_gb1.bin create mode 100644 crates/text-extract/src/pdf/tables/cid_japan1.bin create mode 100644 crates/text-extract/src/pdf/tables/cid_korea1.bin create mode 100644 crates/text-extract/src/pdf/tables/cmap_data.rs create mode 100644 crates/text-extract/src/pdf/tables/cmaps.rs create mode 100644 crates/text-extract/src/pdf/tables/encoding.rs create mode 100644 crates/text-extract/src/pdf/tables/encoding_data.rs create mode 100644 crates/text-extract/src/pdf/tables/font_data.rs create mode 100644 crates/text-extract/src/pdf/tables/fonts.rs create mode 100644 crates/text-extract/src/pdf/tables/glyph_data.rs create mode 100644 crates/text-extract/src/pdf/tables/glyphs.rs create mode 100644 crates/text-extract/src/pdf/tables/mod.rs create mode 100644 crates/text-extract/src/pdf/tables/names.rs create mode 100644 crates/text-extract/src/pdf/tables/normalize.rs create mode 100644 crates/text-extract/src/pdf/tables/presentation_data.rs create mode 100644 crates/text-extract/src/pdf/text_string.rs create mode 100644 crates/text-extract/src/pdf/xref/mod.rs create mode 100644 crates/text-extract/src/pdf/xref/stream.rs create mode 100644 crates/text-extract/src/pdf/xref/table.rs create mode 100644 crates/text-extract/src/rtf/codepage.rs create mode 100644 crates/text-extract/src/rtf/mod.rs create mode 100644 crates/text-extract/src/rtf/words.rs create mode 100644 crates/text-extract/src/utf8.rs create mode 100644 crates/text-extract/src/xml/attr.rs create mode 100644 crates/text-extract/src/xml/mod.rs create mode 100644 crates/text-extract/src/xml/scan.rs create mode 100644 crates/text-extract/src/xml/stream.rs create mode 100644 crates/text-extract/src/zip.rs create mode 100644 crates/trc/src/serializers/escape.rs create mode 100644 crates/trc/src/serializers/timestamp.rs create mode 100644 crates/types/src/collation.rs delete mode 100644 crates/types/src/dead_property.rs create mode 100644 crates/types/src/media_type/catalogue.rs create mode 100644 crates/types/src/media_type/mod.rs create mode 100644 crates/types/src/metadata/builder.rs create mode 100644 crates/types/src/metadata/codec.rs create mode 100644 crates/types/src/metadata/entries.rs create mode 100644 crates/types/src/metadata/json.rs create mode 100644 crates/types/src/metadata/limits.rs create mode 100644 crates/types/src/metadata/mod.rs create mode 100644 crates/types/src/metadata/registry.rs create mode 100644 crates/types/src/metadata/view.rs create mode 100644 crates/types/src/metadata/xml/mod.rs create mode 100644 crates/types/src/metadata/xml/validate.rs create mode 100644 crates/types/src/metadata/xml/view.rs create mode 100644 crates/types/src/metadata/xml/writer.rs create mode 100644 crates/types/src/text.rs delete mode 100644 crates/utils/src/codec/base32_custom.rs delete mode 100644 crates/utils/src/codec/leb128.rs create mode 100644 crates/utils/src/codec/leb128/kernel.rs create mode 100644 crates/utils/src/codec/leb128/mod.rs create mode 100644 crates/utils/src/hash128.rs create mode 100644 crates/utils/src/memory.rs create mode 100644 crates/utils/src/outbound.rs create mode 100644 crates/utils/src/runtime.rs create mode 100644 crates/utils/src/text/case.rs create mode 100644 crates/utils/src/text/count.rs create mode 100644 crates/utils/src/text/encode.rs create mode 100644 crates/utils/src/text/ignore_case.rs create mode 100644 crates/utils/src/text/mod.rs create mode 100644 crates/utils/src/text/needle.rs create mode 100644 crates/utils/src/text/rank.rs create mode 100644 crates/utils/src/text/segments.rs create mode 100644 crates/utils/src/text/set.rs create mode 100644 crates/utils/src/text/split.rs create mode 100644 resources/schema/schema_oss.json.gz create mode 100644 resources/schema/schema_oss.json.sha256 delete mode 100755 resources/scripts/minify_html.sh create mode 100644 resources/zstd/calendar-v1.dict create mode 100644 resources/zstd/common-v1.dict create mode 100644 resources/zstd/contact-v1.dict create mode 100644 resources/zstd/email-v1.dict create mode 100644 resources/zstd/sieve-v1.dict create mode 100644 resources/zstd/term-v1.dict delete mode 100644 tests/Cargo.toml delete mode 100644 tests/docker/INSTRUCTIONS.md delete mode 100644 tests/docker/docker-compose.yml delete mode 100644 tests/docker/keycloak/stalwart-realm.json delete mode 100644 tests/docker/ldap/50-users.ldif delete mode 100644 tests/docker/ldap/60-groups.ldif delete mode 100644 tests/docker/pebble/Dockerfile.challtestsrv delete mode 100644 tests/docker/pebble/pebble-config.json delete mode 100755 tests/docker/powerdns/entrypoint.sh delete mode 100755 tests/docker/powerdns/init-zone.sh delete mode 100644 tests/docker/powerdns/pdns.conf delete mode 100644 tests/docker/scim/Dockerfile delete mode 100644 tests/docker/scim/driver.py delete mode 100755 tests/docker/scripts/gen-certs.sh delete mode 100755 tests/docker/scripts/init-fdb.sh delete mode 100755 tests/docker/scripts/init-minio.sh delete mode 100644 tests/resources/artwork_data.csv.gz delete mode 100644 tests/resources/crypto/cert_mixed.pem delete mode 100644 tests/resources/crypto/cert_pgp.der delete mode 100644 tests/resources/crypto/cert_pgp.pem delete mode 100644 tests/resources/crypto/cert_smime.der delete mode 100644 tests/resources/crypto/cert_smime.pem delete mode 100644 tests/resources/crypto/is_encrypted.txt delete mode 100644 tests/resources/imap-test/append delete mode 100644 tests/resources/imap-test/append-binary delete mode 100644 tests/resources/imap-test/atoms delete mode 100644 tests/resources/imap-test/broken/search-intdate delete mode 100644 tests/resources/imap-test/broken/search-intdate.mbox delete mode 100644 tests/resources/imap-test/catenate delete mode 100644 tests/resources/imap-test/catenate-multiappend delete mode 100644 tests/resources/imap-test/close delete mode 100644 tests/resources/imap-test/copy delete mode 100644 tests/resources/imap-test/default.mbox delete mode 100644 tests/resources/imap-test/esearch delete mode 100644 tests/resources/imap-test/esearch-condstore delete mode 100644 tests/resources/imap-test/esearch.mbox delete mode 100644 tests/resources/imap-test/esort delete mode 100644 tests/resources/imap-test/expunge delete mode 100644 tests/resources/imap-test/expunge2 delete mode 100644 tests/resources/imap-test/fetch delete mode 100644 tests/resources/imap-test/fetch-binary-mime delete mode 100644 tests/resources/imap-test/fetch-binary-mime-base64 delete mode 100644 tests/resources/imap-test/fetch-binary-mime-base64.mbox delete mode 100644 tests/resources/imap-test/fetch-binary-mime-qp delete mode 100644 tests/resources/imap-test/fetch-binary-mime-qp.mbox delete mode 100644 tests/resources/imap-test/fetch-binary-mime.mbox delete mode 100644 tests/resources/imap-test/fetch-body delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822 delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822-mime delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822-mime.mbox delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822-x2 delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822-x2.mbox delete mode 100644 tests/resources/imap-test/fetch-body-message-rfc822.mbox delete mode 100644 tests/resources/imap-test/fetch-body-mime delete mode 100644 tests/resources/imap-test/fetch-body-mime.mbox delete mode 100644 tests/resources/imap-test/fetch-body.mbox delete mode 100644 tests/resources/imap-test/fetch-bodystructure delete mode 100644 tests/resources/imap-test/fetch-bodystructure.mbox delete mode 100644 tests/resources/imap-test/fetch-envelope delete mode 100644 tests/resources/imap-test/fetch-envelope.mbox delete mode 100644 tests/resources/imap-test/id delete mode 100644 tests/resources/imap-test/list delete mode 100644 tests/resources/imap-test/listext delete mode 100644 tests/resources/imap-test/logout delete mode 100644 tests/resources/imap-test/move delete mode 100644 tests/resources/imap-test/multiappend delete mode 100644 tests/resources/imap-test/mutf7 delete mode 100644 tests/resources/imap-test/nil delete mode 100644 tests/resources/imap-test/nil.mbox delete mode 100644 tests/resources/imap-test/notify delete mode 100644 tests/resources/imap-test/pipeline delete mode 100644 tests/resources/imap-test/search-addresses delete mode 100644 tests/resources/imap-test/search-addresses.mbox delete mode 100644 tests/resources/imap-test/search-body delete mode 100644 tests/resources/imap-test/search-body.mbox delete mode 100644 tests/resources/imap-test/search-context-update delete mode 100644 tests/resources/imap-test/search-context-update2 delete mode 100644 tests/resources/imap-test/search-context-update3 delete mode 100644 tests/resources/imap-test/search-date delete mode 100644 tests/resources/imap-test/search-date.mbox delete mode 100644 tests/resources/imap-test/search-flags delete mode 100644 tests/resources/imap-test/search-header delete mode 100644 tests/resources/imap-test/search-header.mbox delete mode 100644 tests/resources/imap-test/search-partial delete mode 100644 tests/resources/imap-test/search-partial.mbox delete mode 100644 tests/resources/imap-test/search-sets delete mode 100644 tests/resources/imap-test/search-size delete mode 100644 tests/resources/imap-test/search-size.mbox delete mode 100644 tests/resources/imap-test/select delete mode 100644 tests/resources/imap-test/select.mbox delete mode 100644 tests/resources/imap-test/sort-addresses delete mode 100644 tests/resources/imap-test/sort-addresses.mbox delete mode 100644 tests/resources/imap-test/sort-arrival delete mode 100644 tests/resources/imap-test/sort-arrival.mbox delete mode 100644 tests/resources/imap-test/sort-date delete mode 100644 tests/resources/imap-test/sort-date.mbox delete mode 100644 tests/resources/imap-test/sort-display-from delete mode 100644 tests/resources/imap-test/sort-display-from.mbox delete mode 100644 tests/resources/imap-test/sort-display-to delete mode 100644 tests/resources/imap-test/sort-display-to.mbox delete mode 100644 tests/resources/imap-test/sort-partial delete mode 100644 tests/resources/imap-test/sort-partial.mbox delete mode 100644 tests/resources/imap-test/sort-size delete mode 100644 tests/resources/imap-test/sort-size.mbox delete mode 100644 tests/resources/imap-test/sort-subject delete mode 100644 tests/resources/imap-test/sort-subject.mbox delete mode 100644 tests/resources/imap-test/store delete mode 100644 tests/resources/imap-test/subscribe delete mode 100644 tests/resources/imap-test/thread delete mode 100644 tests/resources/imap-test/thread-orderedsubject delete mode 100644 tests/resources/imap-test/thread-orderedsubject.mbox delete mode 100644 tests/resources/imap-test/thread-orderedsubject2 delete mode 100644 tests/resources/imap-test/thread-orderedsubject2.mbox delete mode 100644 tests/resources/imap-test/thread.mbox delete mode 100644 tests/resources/imap-test/thread2 delete mode 100644 tests/resources/imap-test/thread2.mbox delete mode 100644 tests/resources/imap-test/thread3 delete mode 100644 tests/resources/imap-test/thread3.mbox delete mode 100644 tests/resources/imap-test/thread4 delete mode 100644 tests/resources/imap-test/thread4.mbox delete mode 100644 tests/resources/imap-test/thread5 delete mode 100644 tests/resources/imap-test/thread5.mbox delete mode 100644 tests/resources/imap-test/thread6 delete mode 100644 tests/resources/imap-test/thread6.mbox delete mode 100644 tests/resources/imap-test/thread7 delete mode 100644 tests/resources/imap-test/thread7.mbox delete mode 100644 tests/resources/imap-test/thread8 delete mode 100644 tests/resources/imap-test/thread8.mbox delete mode 100644 tests/resources/imap-test/uidplus delete mode 100644 tests/resources/imap-test/uidvalidity delete mode 100644 tests/resources/imap-test/uidvalidity-rename delete mode 100644 tests/resources/imap-test/urlauth delete mode 100644 tests/resources/imap-test/urlauth-binary delete mode 100644 tests/resources/imap-test/urlauth-binary.mbox delete mode 100644 tests/resources/imap-test/urlauth2 delete mode 100644 tests/resources/imap/000.imap delete mode 100644 tests/resources/imap/000.txt delete mode 100644 tests/resources/imap/001.imap delete mode 100644 tests/resources/imap/001.txt delete mode 100644 tests/resources/imap/002.imap delete mode 100644 tests/resources/imap/002.txt delete mode 100644 tests/resources/imap/003.imap delete mode 100644 tests/resources/imap/003.txt delete mode 100644 tests/resources/imap/004.imap delete mode 100644 tests/resources/imap/004.txt delete mode 100644 tests/resources/imap/005.imap delete mode 100644 tests/resources/imap/005.txt delete mode 100644 tests/resources/imap/006.imap delete mode 100644 tests/resources/imap/006.txt delete mode 100644 tests/resources/imap/007.imap delete mode 100644 tests/resources/imap/007.txt delete mode 100644 tests/resources/imap/008.imap delete mode 100644 tests/resources/imap/008.txt delete mode 100644 tests/resources/imap/009.imap delete mode 100644 tests/resources/imap/009.txt delete mode 100644 tests/resources/imap/010.imap delete mode 100644 tests/resources/imap/010.txt delete mode 100644 tests/resources/imap/011.imap delete mode 100644 tests/resources/imap/011.txt delete mode 100644 tests/resources/imap/012.imap delete mode 100644 tests/resources/imap/012.txt delete mode 100644 tests/resources/imap/013.imap delete mode 100644 tests/resources/imap/013.txt delete mode 100644 tests/resources/imap/014.imap delete mode 100644 tests/resources/imap/014.txt delete mode 100644 tests/resources/imap/015.imap delete mode 100644 tests/resources/imap/015.txt delete mode 100644 tests/resources/itip/google_calendar.txt delete mode 100644 tests/resources/itip/itip_incoming.txt delete mode 100644 tests/resources/itip/put_validation.txt delete mode 100644 tests/resources/itip/rfc5546_event_recurring.txt delete mode 100644 tests/resources/itip/rfc5546_event_single.txt delete mode 100644 tests/resources/itip/rfc5546_todo.txt delete mode 100644 tests/resources/itip/rfc6638_recurring.txt delete mode 100644 tests/resources/itip/rfc6638_single.txt delete mode 100644 tests/resources/itip/vtimezone.txt delete mode 100644 tests/resources/jmap/email_get/headers.eml delete mode 100644 tests/resources/jmap/email_get/headers.json delete mode 100644 tests/resources/jmap/email_get/message_attachment.eml delete mode 100644 tests/resources/jmap/email_get/message_attachment.json delete mode 100644 tests/resources/jmap/email_get/multipart_alternative.eml delete mode 100644 tests/resources/jmap/email_get/multipart_alternative.json delete mode 100644 tests/resources/jmap/email_get/multipart_cid.eml delete mode 100644 tests/resources/jmap/email_get/multipart_cid.json delete mode 100644 tests/resources/jmap/email_get/multipart_mixed.eml delete mode 100644 tests/resources/jmap/email_get/multipart_mixed.json delete mode 100644 tests/resources/jmap/email_get/multipart_related.eml delete mode 100644 tests/resources/jmap/email_get/multipart_related.json delete mode 100644 tests/resources/jmap/email_get/rfc8621.eml delete mode 100644 tests/resources/jmap/email_get/rfc8621.json delete mode 100644 tests/resources/jmap/email_get/single_part.eml delete mode 100644 tests/resources/jmap/email_get/single_part.json delete mode 100644 tests/resources/jmap/email_get/text_attachments.eml delete mode 100644 tests/resources/jmap/email_get/text_attachments.json delete mode 100644 tests/resources/jmap/email_get/text_body_missing.eml delete mode 100644 tests/resources/jmap/email_get/text_body_missing.json delete mode 100644 tests/resources/jmap/email_get/text_body_missing_multipart.eml delete mode 100644 tests/resources/jmap/email_get/text_body_missing_multipart.json delete mode 100644 tests/resources/jmap/email_parse/attachment.eml delete mode 100644 tests/resources/jmap/email_parse/attachment.json delete mode 100644 tests/resources/jmap/email_parse/attachment.part1 delete mode 100644 tests/resources/jmap/email_parse/attachment.part2 delete mode 100644 tests/resources/jmap/email_parse/attachment.part3 delete mode 100644 tests/resources/jmap/email_parse/attachment_b64.eml delete mode 100644 tests/resources/jmap/email_parse/attachment_b64.json delete mode 100644 tests/resources/jmap/email_parse/attachment_b64.part1 delete mode 100644 tests/resources/jmap/email_parse/attachment_b64.part2 delete mode 100644 tests/resources/jmap/email_parse/headers.eml delete mode 100644 tests/resources/jmap/email_parse/headers.json delete mode 100644 tests/resources/jmap/email_set/headers.eml delete mode 100644 tests/resources/jmap/email_set/headers.jmap delete mode 100644 tests/resources/jmap/email_set/headers.json delete mode 100644 tests/resources/jmap/email_set/minimal.eml delete mode 100644 tests/resources/jmap/email_set/minimal.jmap delete mode 100644 tests/resources/jmap/email_set/minimal.json delete mode 100644 tests/resources/jmap/email_set/mixed.eml delete mode 100644 tests/resources/jmap/email_set/mixed.jmap delete mode 100644 tests/resources/jmap/email_set/mixed.json delete mode 100644 tests/resources/jmap/email_set/nested_body.eml delete mode 100644 tests/resources/jmap/email_set/nested_body.jmap delete mode 100644 tests/resources/jmap/email_set/nested_body.json delete mode 100644 tests/resources/jmap/email_set/rfc8621_1.eml delete mode 100644 tests/resources/jmap/email_set/rfc8621_1.jmap delete mode 100644 tests/resources/jmap/email_set/rfc8621_1.json delete mode 100644 tests/resources/jmap/email_set/rfc8621_2.eml delete mode 100644 tests/resources/jmap/email_set/rfc8621_2.jmap delete mode 100644 tests/resources/jmap/email_set/rfc8621_2.json delete mode 100644 tests/resources/jmap/email_snippet/html.eml delete mode 100644 tests/resources/jmap/email_snippet/mixed.eml delete mode 100644 tests/resources/jmap/email_snippet/subpart.eml delete mode 100644 tests/resources/jmap/email_snippet/text_plain.eml delete mode 100644 tests/resources/jmap/email_snippet/text_plain_chinese.eml delete mode 100644 tests/resources/jmap/sieve/test_discard_reject.sieve delete mode 100644 tests/resources/jmap/sieve/test_include.sieve delete mode 100644 tests/resources/jmap/sieve/test_include_case.sieve delete mode 100644 tests/resources/jmap/sieve/test_include_global.sieve delete mode 100644 tests/resources/jmap/sieve/test_include_this.sieve delete mode 100644 tests/resources/jmap/sieve/test_mailbox.sieve delete mode 100644 tests/resources/jmap/sieve/test_notify_fcc.sieve delete mode 100644 tests/resources/jmap/sieve/test_redirect_enclose.sieve delete mode 100644 tests/resources/jmap/sieve/validate_error.sieve delete mode 100644 tests/resources/jmap/sieve/validate_ok.sieve delete mode 100644 tests/resources/ldap/ldap.cfg delete mode 100644 tests/resources/ldap/run_glauth.sh delete mode 100644 tests/resources/mailbox.gz delete mode 100644 tests/resources/otel/docker-compose.yaml delete mode 100644 tests/resources/otel/otel-collector-config.yaml delete mode 100644 tests/resources/otel/stalwart-config.toml delete mode 100644 tests/resources/proxy-protocol/Docker.haproxy delete mode 100644 tests/resources/proxy-protocol/haproxy.cfg delete mode 100644 tests/resources/scripts/imap_import.py delete mode 100644 tests/resources/scripts/imap_import_single.py delete mode 100644 tests/resources/scripts/smtp_stress_test.py delete mode 100644 tests/resources/scripts/stress_test.py delete mode 100644 tests/resources/scripts/stress_test_prepare.py delete mode 100644 tests/resources/smtp/antispam/bounce.test delete mode 100644 tests/resources/smtp/antispam/classifier.ham delete mode 100644 tests/resources/smtp/antispam/classifier.spam delete mode 100644 tests/resources/smtp/antispam/classifier.test delete mode 100644 tests/resources/smtp/antispam/classifier_features.test delete mode 100644 tests/resources/smtp/antispam/classifier_html.test delete mode 100644 tests/resources/smtp/antispam/combined.test delete mode 100644 tests/resources/smtp/antispam/date.test delete mode 100644 tests/resources/smtp/antispam/dmarc.test delete mode 100644 tests/resources/smtp/antispam/from.test delete mode 100644 tests/resources/smtp/antispam/headers.test delete mode 100644 tests/resources/smtp/antispam/helo.test delete mode 100644 tests/resources/smtp/antispam/html.test delete mode 100644 tests/resources/smtp/antispam/ip.test delete mode 100644 tests/resources/smtp/antispam/llm.test delete mode 100644 tests/resources/smtp/antispam/messageid.test delete mode 100644 tests/resources/smtp/antispam/mime.test delete mode 100644 tests/resources/smtp/antispam/pyzor.test delete mode 100644 tests/resources/smtp/antispam/rbl.test delete mode 100644 tests/resources/smtp/antispam/received.test delete mode 100644 tests/resources/smtp/antispam/recipient.test delete mode 100644 tests/resources/smtp/antispam/replyto.test delete mode 100644 tests/resources/smtp/antispam/spamtrap.test delete mode 100644 tests/resources/smtp/antispam/subject.test delete mode 100644 tests/resources/smtp/antispam/url.test delete mode 100644 tests/resources/smtp/certs/tls_cert.pem delete mode 100644 tests/resources/smtp/certs/tls_privatekey.pem delete mode 100644 tests/resources/smtp/dane/dns.txt delete mode 100644 tests/resources/smtp/dane/internet.nl.0.cert delete mode 100644 tests/resources/smtp/dane/internet.nl.1.cert delete mode 100644 tests/resources/smtp/dane/mail.ietf.org.0.cert delete mode 100644 tests/resources/smtp/dane/mail.ietf.org.1.cert delete mode 100644 tests/resources/smtp/dane/mail.ietf.org.2.cert delete mode 100644 tests/resources/smtp/dane/mail.ietf.org.3.cert delete mode 100644 tests/resources/smtp/dsn/delay.eml delete mode 100644 tests/resources/smtp/dsn/failure.eml delete mode 100644 tests/resources/smtp/dsn/mixed.eml delete mode 100644 tests/resources/smtp/dsn/original.txt delete mode 100644 tests/resources/smtp/dsn/success.eml delete mode 100644 tests/resources/smtp/lists/test-list1.txt delete mode 100644 tests/resources/smtp/lists/test-list2.txt delete mode 100644 tests/resources/smtp/messages/arc.eml delete mode 100644 tests/resources/smtp/messages/dkim.eml delete mode 100644 tests/resources/smtp/messages/idn_from.eml delete mode 100644 tests/resources/smtp/messages/invalid_arc.eml delete mode 100644 tests/resources/smtp/messages/invalid_dkim.eml delete mode 100644 tests/resources/smtp/messages/loop.eml delete mode 100644 tests/resources/smtp/messages/multipart.eml delete mode 100644 tests/resources/smtp/messages/no_dkim.eml delete mode 100644 tests/resources/smtp/messages/no_msgid.eml delete mode 100644 tests/resources/smtp/milter/message.eml delete mode 100644 tests/resources/smtp/milter/message.json delete mode 100644 tests/resources/smtp/reports/arf1.eml delete mode 100644 tests/resources/smtp/reports/arf2.eml delete mode 100644 tests/resources/smtp/reports/arf3.eml delete mode 100644 tests/resources/smtp/reports/arf4.eml delete mode 100644 tests/resources/smtp/reports/arf5.eml delete mode 100644 tests/resources/smtp/reports/dmarc1.eml delete mode 100644 tests/resources/smtp/reports/dmarc2.eml delete mode 100644 tests/resources/smtp/reports/dmarc3.eml delete mode 100644 tests/resources/smtp/reports/dmarc4.eml delete mode 100644 tests/resources/smtp/reports/dmarc5.eml delete mode 100644 tests/resources/smtp/reports/tls1.eml delete mode 100644 tests/resources/smtp/reports/tls2.eml delete mode 100644 tests/resources/smtp/sieve/awl.sieve delete mode 100644 tests/resources/smtp/sieve/awl_include.sieve delete mode 100644 tests/resources/smtp/sieve/spamtest_include.sieve delete mode 100644 tests/resources/smtp/sieve/stage_connect.sieve delete mode 100644 tests/resources/smtp/sieve/stage_data.sieve delete mode 100644 tests/resources/smtp/sieve/stage_ehlo.sieve delete mode 100644 tests/resources/smtp/sieve/stage_mail.sieve delete mode 100644 tests/resources/smtp/sieve/stage_rcpt.sieve delete mode 100644 tests/resources/tls_cert.pem delete mode 100644 tests/resources/tls_privatekey.pem delete mode 100644 tests/src/automation/acme.rs delete mode 100644 tests/src/automation/dkim.rs delete mode 100644 tests/src/automation/dns.rs delete mode 100644 tests/src/automation/mod.rs delete mode 100644 tests/src/automation/rfc2136.rs delete mode 100644 tests/src/cluster/broadcast.rs delete mode 100644 tests/src/cluster/stress.rs delete mode 100644 tests/src/directory/discovery.rs delete mode 100644 tests/src/directory/integration.rs delete mode 100644 tests/src/directory/issuer.rs delete mode 100644 tests/src/directory/ldap.rs delete mode 100644 tests/src/directory/mod.rs delete mode 100644 tests/src/directory/sql.rs delete mode 100644 tests/src/directory/synchronization.rs delete mode 100644 tests/src/directory/unavailable.rs delete mode 100644 tests/src/imap/acl.rs delete mode 100644 tests/src/imap/antispam.rs delete mode 100644 tests/src/imap/append.rs delete mode 100644 tests/src/imap/basic.rs delete mode 100644 tests/src/imap/body_structure.rs delete mode 100644 tests/src/imap/condstore.rs delete mode 100644 tests/src/imap/copy_move.rs delete mode 100644 tests/src/imap/fetch.rs delete mode 100644 tests/src/imap/idle.rs delete mode 100644 tests/src/imap/mailbox.rs delete mode 100644 tests/src/imap/managesieve.rs delete mode 100644 tests/src/imap/messagelimit.rs delete mode 100644 tests/src/imap/mod.rs delete mode 100644 tests/src/imap/objectid.rs delete mode 100644 tests/src/imap/pop.rs delete mode 100644 tests/src/imap/search.rs delete mode 100644 tests/src/imap/store.rs delete mode 100644 tests/src/imap/thread.rs delete mode 100644 tests/src/imap/uidbatches.rs delete mode 100644 tests/src/imap/uidonly.rs delete mode 100644 tests/src/jmap/calendar/acl.rs delete mode 100644 tests/src/jmap/calendar/alarm.rs delete mode 100644 tests/src/jmap/calendar/calendars.rs delete mode 100644 tests/src/jmap/calendar/event.rs delete mode 100644 tests/src/jmap/calendar/identity.rs delete mode 100644 tests/src/jmap/calendar/instance.rs delete mode 100644 tests/src/jmap/calendar/mod.rs delete mode 100644 tests/src/jmap/calendar/notification.rs delete mode 100644 tests/src/jmap/compliance/binary.rs delete mode 100644 tests/src/jmap/compliance/core.rs delete mode 100644 tests/src/jmap/compliance/email/inspect.rs delete mode 100644 tests/src/jmap/compliance/email/mod.rs delete mode 100644 tests/src/jmap/compliance/email/mutate.rs delete mode 100644 tests/src/jmap/compliance/email/query.rs delete mode 100644 tests/src/jmap/compliance/expected.rs delete mode 100644 tests/src/jmap/compliance/identity.rs delete mode 100644 tests/src/jmap/compliance/mailbox.rs delete mode 100644 tests/src/jmap/compliance/mod.rs delete mode 100644 tests/src/jmap/compliance/push.rs delete mode 100644 tests/src/jmap/compliance/search_snippet.rs delete mode 100644 tests/src/jmap/compliance/submission.rs delete mode 100644 tests/src/jmap/compliance/thread.rs delete mode 100644 tests/src/jmap/compliance/vacation.rs delete mode 100644 tests/src/jmap/contacts/acl.rs delete mode 100644 tests/src/jmap/contacts/addressbook.rs delete mode 100644 tests/src/jmap/contacts/contact.rs delete mode 100644 tests/src/jmap/contacts/mod.rs delete mode 100644 tests/src/jmap/core/blob.rs delete mode 100644 tests/src/jmap/core/event_source.rs delete mode 100644 tests/src/jmap/core/mod.rs delete mode 100644 tests/src/jmap/core/push_subscription.rs delete mode 100644 tests/src/jmap/core/websocket.rs delete mode 100644 tests/src/jmap/files/acl.rs delete mode 100644 tests/src/jmap/files/mod.rs delete mode 100644 tests/src/jmap/files/node.rs delete mode 100644 tests/src/jmap/mail/acl.rs delete mode 100644 tests/src/jmap/mail/changes.rs delete mode 100644 tests/src/jmap/mail/copy.rs delete mode 100644 tests/src/jmap/mail/get.rs delete mode 100644 tests/src/jmap/mail/mailbox.rs delete mode 100644 tests/src/jmap/mail/mod.rs delete mode 100644 tests/src/jmap/mail/parse.rs delete mode 100644 tests/src/jmap/mail/query.rs delete mode 100644 tests/src/jmap/mail/query_changes.rs delete mode 100644 tests/src/jmap/mail/search_snippet.rs delete mode 100644 tests/src/jmap/mail/set.rs delete mode 100644 tests/src/jmap/mail/sieve_script.rs delete mode 100644 tests/src/jmap/mail/submission.rs delete mode 100644 tests/src/jmap/mail/thread_get.rs delete mode 100644 tests/src/jmap/mail/thread_merge.rs delete mode 100644 tests/src/jmap/mail/vacation_response.rs delete mode 100644 tests/src/jmap/mod.rs delete mode 100644 tests/src/jmap/principal/availability.rs delete mode 100644 tests/src/jmap/principal/get.rs delete mode 100644 tests/src/jmap/principal/mod.rs delete mode 100644 tests/src/lib.rs delete mode 100644 tests/src/scim/conformance.rs delete mode 100644 tests/src/scim/oidc.rs delete mode 100644 tests/src/scim/tenant.rs delete mode 100644 tests/src/smtp/inbound/antispam.rs delete mode 100644 tests/src/smtp/inbound/asn.rs delete mode 100644 tests/src/smtp/inbound/auth.rs delete mode 100644 tests/src/smtp/inbound/basic.rs delete mode 100644 tests/src/smtp/inbound/data.rs delete mode 100644 tests/src/smtp/inbound/dkim2.rs delete mode 100644 tests/src/smtp/inbound/dmarc.rs delete mode 100644 tests/src/smtp/inbound/ehlo.rs delete mode 100644 tests/src/smtp/inbound/limits.rs delete mode 100644 tests/src/smtp/inbound/mail.rs delete mode 100644 tests/src/smtp/inbound/milter.rs delete mode 100644 tests/src/smtp/inbound/mod.rs delete mode 100644 tests/src/smtp/inbound/rcpt.rs delete mode 100644 tests/src/smtp/inbound/rewrite.rs delete mode 100644 tests/src/smtp/inbound/scripts.rs delete mode 100644 tests/src/smtp/inbound/sign.rs delete mode 100644 tests/src/smtp/inbound/spam_rules.rs delete mode 100644 tests/src/smtp/inbound/throttle.rs delete mode 100644 tests/src/smtp/inbound/vrfy.rs delete mode 100644 tests/src/smtp/lookup/expressions.rs delete mode 100644 tests/src/smtp/lookup/mod.rs delete mode 100644 tests/src/smtp/lookup/utils.rs delete mode 100644 tests/src/smtp/management/mod.rs delete mode 100644 tests/src/smtp/management/queue.rs delete mode 100644 tests/src/smtp/management/report.rs delete mode 100644 tests/src/smtp/mod.rs delete mode 100644 tests/src/smtp/outbound/dane.rs delete mode 100644 tests/src/smtp/outbound/extensions.rs delete mode 100644 tests/src/smtp/outbound/fallback_relay.rs delete mode 100644 tests/src/smtp/outbound/ip_lookup.rs delete mode 100644 tests/src/smtp/outbound/lmtp.rs delete mode 100644 tests/src/smtp/outbound/mod.rs delete mode 100644 tests/src/smtp/outbound/mta_sts.rs delete mode 100644 tests/src/smtp/outbound/smtp.rs delete mode 100644 tests/src/smtp/outbound/throttle.rs delete mode 100644 tests/src/smtp/outbound/tls.rs delete mode 100644 tests/src/smtp/queue/concurrent.rs delete mode 100644 tests/src/smtp/queue/dsn.rs delete mode 100644 tests/src/smtp/queue/manager.rs delete mode 100644 tests/src/smtp/queue/mod.rs delete mode 100644 tests/src/smtp/queue/retry.rs delete mode 100644 tests/src/smtp/queue/virtualq.rs delete mode 100644 tests/src/smtp/reporting/analyze.rs delete mode 100644 tests/src/smtp/reporting/dmarc.rs delete mode 100644 tests/src/smtp/reporting/mod.rs delete mode 100644 tests/src/smtp/reporting/scheduler.rs delete mode 100644 tests/src/smtp/reporting/tls.rs delete mode 100644 tests/src/smtp/session.rs delete mode 100644 tests/src/store/blob.rs delete mode 100644 tests/src/store/import_export.rs delete mode 100644 tests/src/store/lookup.rs delete mode 100644 tests/src/store/mod.rs delete mode 100644 tests/src/store/ops.rs delete mode 100644 tests/src/store/query.rs delete mode 100644 tests/src/store/registry.rs delete mode 100644 tests/src/store/sql_timeout.rs delete mode 100644 tests/src/system/antispam.rs delete mode 100644 tests/src/system/authentication.rs delete mode 100644 tests/src/system/authorization.rs delete mode 100644 tests/src/system/crypto.rs delete mode 100644 tests/src/system/delivery.rs delete mode 100644 tests/src/system/directory.rs delete mode 100644 tests/src/system/mod.rs delete mode 100644 tests/src/system/oidc.rs delete mode 100644 tests/src/system/purge.rs delete mode 100644 tests/src/system/quota.rs delete mode 100644 tests/src/system/security.rs delete mode 100644 tests/src/system/task.rs delete mode 100644 tests/src/telemetry/alerts.rs delete mode 100644 tests/src/telemetry/metrics.rs delete mode 100644 tests/src/telemetry/mod.rs delete mode 100644 tests/src/telemetry/tracing.rs delete mode 100644 tests/src/telemetry/webhooks.rs delete mode 100644 tests/src/utils/account.rs delete mode 100644 tests/src/utils/cleanup.rs delete mode 100644 tests/src/utils/containers.rs delete mode 100644 tests/src/utils/dns.rs delete mode 100644 tests/src/utils/http.rs delete mode 100644 tests/src/utils/http_server.rs delete mode 100644 tests/src/utils/imap.rs delete mode 100644 tests/src/utils/jmap.rs delete mode 100644 tests/src/utils/mod.rs delete mode 100644 tests/src/utils/pop3.rs delete mode 100644 tests/src/utils/registry.rs delete mode 100644 tests/src/utils/server.rs delete mode 100644 tests/src/utils/sieve.rs delete mode 100644 tests/src/utils/smtp.rs delete mode 100644 tests/src/utils/storage.rs delete mode 100644 tests/src/utils/temp_dir.rs delete mode 100644 tests/src/utils/webdav.rs delete mode 100644 tests/src/webdav/acl.rs delete mode 100644 tests/src/webdav/basic.rs delete mode 100644 tests/src/webdav/cal_alarm.rs delete mode 100644 tests/src/webdav/cal_itip.rs delete mode 100644 tests/src/webdav/cal_query.rs delete mode 100644 tests/src/webdav/cal_scheduling.rs delete mode 100644 tests/src/webdav/card_query.rs delete mode 100644 tests/src/webdav/copy_move.rs delete mode 100644 tests/src/webdav/lock.rs delete mode 100644 tests/src/webdav/mkcol.rs delete mode 100644 tests/src/webdav/mod.rs delete mode 100644 tests/src/webdav/multiget.rs delete mode 100644 tests/src/webdav/principals.rs delete mode 100644 tests/src/webdav/prop.rs delete mode 100644 tests/src/webdav/put_get.rs delete mode 100644 tests/src/webdav/sync.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 059c0a65..a47616bb 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,7 @@ +[env] +RUST_MIN_STACK = "8388608" +CXXFLAGS_aarch64_unknown_linux_gnu = "-march=armv8-a+crc+crypto -DROCKSDB_AUXV_GETAUXVAL_PRESENT" +CXXFLAGS_aarch64_unknown_linux_musl = "-march=armv8-a+crc+crypto -DROCKSDB_AUXV_GETAUXVAL_PRESENT" + [target.'cfg(all(target_arch = "x86_64", not(target_env = "musl")))'] rustflags = ["-C", "target-cpu=x86-64-v2"] diff --git a/.dockerignore b/.dockerignore index 89819958..dec3a6e6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ // Allow what is needed !crates -!tests !resources !.cargo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30471e3c..803b7280 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,6 +270,7 @@ jobs: type=ref,event=branch,prefix=branch- type=edge,branch=main type=semver,pattern=v{{major}}.{{minor}} + type=semver,pattern=v{{major}} - name: Build Artifact id: bake diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index daadca36..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Test - -on: - workflow_dispatch: - -jobs: - style: - name: Check Style - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Check Style - run: cargo fmt --all --check - - test: - name: Test - needs: style - runs-on: ubuntu-latest - - env: - STORE: RocksDb - RUST_MIN_STACK: "16777216" - - steps: - - name: Checkout - uses: actions/checkout@v7 - - # External services (OpenLDAP, Keycloak, PostgreSQL, MySQL, Redis, NATS, - # MinIO, OpenSearch, Meilisearch) are provisioned on demand by the test - # suite via testcontainers using the Docker daemon available on the - # runner; see tests/src/utils/containers.rs. - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: JMAP Protocol Tests - run: cargo test -p jmap_proto -- --nocapture - - - name: IMAP Protocol Tests - run: cargo test -p imap_proto -- --nocapture - - - name: Full-text search Tests - run: cargo test -p store -- --nocapture - - - name: Directory Tests - run: cargo test -p tests directory -- --nocapture - - - name: SMTP Tests - run: cargo test -p tests smtp -- --nocapture - - - name: IMAP Tests - run: cargo test -p tests imap -- --nocapture - - - name: JMAP Tests - run: cargo test -p tests jmap -- --nocapture diff --git a/.gitignore b/.gitignore index 1a302cb3..844ba5e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/target +target/ *.failed *_failed run.sh @@ -7,4 +7,3 @@ run.sh !.gitattributes !.github !.cargo -CLAUDE.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d19d57a5..e64ed8a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,61 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.0] - 2026-10-26 + +This version includes **multiple breaking changes** and a new database layout, so existing v0.16.x deployments cannot be upgraded in place. Please read the [migration documentation](https://stalw.art/docs/migration/upgrade-0-16/) for more information on how to migrate from v0.16.x. + +## Added +- Full-text search: + - Text extraction and indexing of PDF, Microsoft Office (`docx`, `xlsx`, `pptx`), OpenDocument (`odt`, `ods`, `odp`), EPUB and RTF documents found in attachments and files. + - Full-text search of file storage contents. + - Index queue with per-partition status, concurrency control, retries and index rebuilds. +- Metadata: + - JMAP Object Metadata ([draft-ietf-jmap-metadata-04](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-metadata-04)) for e-mails, mailboxes, Sieve scripts, calendars, events, address books, contacts and files. + - IMAP `METADATA` extension ([RFC 5464](https://www.rfc-editor.org/rfc/rfc5464.html)). +- DKIM2: + - Post-quantum composite signatures using `ML-DSA-44` and `Ed25519` with `SHA-512`. + - Configurable `Message-Instance` hash algorithms (`SHA-256` and `SHA-512`). +- OAuth and OIDC: + - OAuth Client ID Metadata Documents ([draft-ietf-oauth-client-id-metadata-document-02](https://datatracker.ietf.org/doc/html/draft-ietf-oauth-client-id-metadata-document-02)). + - OAuth scopes (`mail`, `contacts`, `calendars` and `files`) per client and grant. + - Connected applications: users can list and revoke the applications authorized to access their account. + - First-party clients, post-logout redirect URIs and per-client grant types. + - External OIDC directories: audience matching, opaque token validation via UserInfo or token introspection and a policy for unscoped tokens. +- Caching: + - Redesigned cache subsystem with a dedicated tier for large accounts. + - Cache snapshots can be swapped to local disk, Redis or the blob store. +- Built-in local in-memory store for single-node deployments. +- Azure Blob Storage authentication using shared keys, SAS tokens, client secrets, managed identities or workload identities. +- Secondary LDAP servers for failover. +- SSRF protection for outbound HTTP requests (webhooks, OIDC discovery, AI and DNS providers). +- Configurable client IP source (`X-Forwarded-For` or `Forwarded`) and number of trusted proxy hops. +- Memory budget for buffered data. +- Configurable limits for e-mail headers, keywords and mailboxes per message, calendars, address books and spam filter text processing. +- TLS options for Kafka and NATS cluster coordinators. +- Automatic cleanup of per-user data after a share is revoked. +- AI (*Enterprise*): + - AI models from multiple providers: OpenAI-compatible endpoints (OpenAI, Azure, vLLM, llama.cpp), Anthropic, Google Gemini and Ollama, with per-model rate limits, concurrency caps, timeouts and reasoning levels. + - AI spam classifier with structured verdicts, model tiers with escalation, training from AI verdicts and optional web search. + +## Changed +- New database layout: the key layout was redesigned and the database schema version is now tied to the edition (AGPL-only or AGPL/SEL). Data must be migrated from v0.16.x using the migration tool. +- Settings schema: multiple objects, fields and enumeration values were renamed, restructured (AI models, OIDC directories, Azure Blob Storage, HTTP forwarded headers) or removed, and internal identifiers were renumbered. See [SCHEMA_CHANGES.md](https://github.com/stalwartlabs/stalwart-migrate/blob/main/SCHEMA_CHANGES.md) for the full list. +- Telemetry: New events and metrics were added and event identifiers were renumbered. See [SCHEMA_CHANGES.md](https://github.com/stalwartlabs/stalwart-migrate/blob/main/SCHEMA_CHANGES.md). +- OAuth: The [OAuth Profile for Open Public Clients](https://datatracker.ietf.org/doc/draft-ietf-mailmaint-oauth-public/) is now fully enforced. If you use an external OIDC provider, make sure it conforms to the draft; otherwise mail clients such as Apple Mail will stop working with it as they adopt the profile. Ask your provider to add support, or, at your own risk, relax the corresponding security checks. +- Updated IETF drafts: + - DKIM2 to [draft-ietf-dkim-dkim2-spec-07](https://datatracker.ietf.org/doc/html/draft-ietf-dkim-dkim2-spec-07). + - JMAP for Calendars to [draft-ietf-jmap-calendars-33](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-calendars-33). + - JMAP for File Storage to [draft-ietf-jmap-filenode-18](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-filenode-18). + - JSCalendar to [draft-ietf-calext-jscalendarbis-22](https://datatracker.ietf.org/doc/html/draft-ietf-calext-jscalendarbis-22) and [draft-ietf-calext-jscalendar-icalendar-28](https://datatracker.ietf.org/doc/html/draft-ietf-calext-jscalendar-icalendar-28). + - iCalendar tasks to [draft-ietf-calext-ical-tasks-17](https://datatracker.ietf.org/doc/html/draft-ietf-calext-ical-tasks-17). + - IMAP object identifiers to [draft-ietf-mailmaint-imap-objectid-bis-08](https://datatracker.ietf.org/doc/html/draft-ietf-mailmaint-imap-objectid-bis-08). +- Performance improvements in storage backends, expression evaluation, IMAP parsing, spam filter tokenization and search. +- Development: Test suites (unit, integration, conformance and stress) are now kept in a private repository, and new features are developed privately before release. The source code of every release remains public under the AGPL-3.0. [Learn why](https://stalw.art/blog/protecting-stalwart/) + +## Fixed +- Multiple bug and security fixes. + ## [0.16.27] - 2026-10-18 If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. diff --git a/Cargo.lock b/Cargo.lock index e00d5677..2798bf64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - [[package]] name = "adler2" version = "2.0.1" @@ -37,7 +31,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.3", + "inout 0.2.5", ] [[package]] @@ -151,9 +145,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" dependencies = [ "rustversion", ] @@ -227,7 +221,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.21", + "thiserror", "time", ] @@ -254,38 +248,11 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "astral-tokio-tar" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" -dependencies = [ - "futures-core", - "libc", - "portable-atomic", - "rustc-hash", - "rustix", - "tokio", - "tokio-stream", - "xattr", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - [[package]] name = "async-compression" -version = "0.4.50" +version = "0.4.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" dependencies = [ "compression-codecs", "compression-core", @@ -299,7 +266,7 @@ version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "event-listener 5.4.2", + "event-listener", "event-listener-strategy", "pin-project-lite", ] @@ -324,7 +291,7 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "thiserror 2.0.21", + "thiserror", "tokio", "tokio-rustls", "tokio-stream", @@ -384,45 +351,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "attohttpc" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" -dependencies = [ - "base64 0.22.1", - "http 1.5.0", - "log", - "rustls", - "serde", - "serde_json", - "url", - "webpki-roots 1.0.9", -] - [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "aws-creds" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "886acd98f2df6e988d5428e92cdf041792ffa87a494d2b5125982502ec726a9c" -dependencies = [ - "attohttpc", - "home", - "log", - "quick-xml 0.41.0", - "rust-ini", - "serde", - "thiserror 2.0.21", - "time", - "url", -] - [[package]] name = "aws-lc-rs" version = "1.18.2" @@ -447,141 +381,86 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "aws-region" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f38493193e45cb4788ee7d479ee94a0a56721ccd28de79bbd0227bb8c11fee3" -dependencies = [ - "thiserror 2.0.21", -] - -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http 1.5.0", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http 1.5.0", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "azure_core" -version = "0.21.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +checksum = "4e41cbd819986ba41904c207d8ffc4106f8f8352a548d773e9554906379bb2fb" dependencies = [ + "async-lock", "async-trait", - "base64 0.22.1", + "azure_core_macros", "bytes", - "dyn-clone", "futures", - "getrandom 0.2.17", - "hmac 0.12.1", - "http-types", - "once_cell", - "paste", "pin-project", - "quick-xml 0.31.0", - "rand 0.8.8", - "reqwest 0.12.28", "rustc_version", "serde", "serde_json", - "sha2 0.10.9", - "time", + "tokio", "tracing", - "url", - "uuid", + "typespec", + "typespec_client_core", ] [[package]] -name = "azure_storage" -version = "0.21.0" +name = "azure_core_macros" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "tracing", +] + +[[package]] +name = "azure_identity" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32edf96b356ca7c51d7590c4925cc36efc3947a5da4468e8e0b25c56ecbb3de5" dependencies = [ - "RustyXML", "async-lock", "async-trait", "azure_core", - "bytes", + "futures", + "pin-project", "serde", - "serde_derive", + "serde_json", "time", + "tokio", "tracing", "url", - "uuid", ] [[package]] -name = "azure_storage_blobs" -version = "0.21.0" +name = "azure_storage_blob" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +checksum = "17b10207ecf7d666df6940b50051f433b3cd5d2b9b1dd190613208d7a84e7eed" dependencies = [ - "RustyXML", + "async-stream", + "async-trait", "azure_core", - "azure_storage", - "azure_svc_blobstorage", + "azure_storage_common", "bytes", "futures", + "percent-encoding", + "pin-project", "serde", - "serde_derive", "serde_json", "time", - "tracing", - "url", - "uuid", + "tokio", ] [[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" +name = "azure_storage_common" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +checksum = "0af2e6aeb8d76b17fc998f453c320913f73787b944e3cc29509d19411fa0321d" dependencies = [ "azure_core", - "bytes", - "futures", - "log", - "once_cell", "serde", - "serde_json", "time", ] @@ -591,18 +470,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base32" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.22.1" @@ -661,22 +540,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "biscuit" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dbfddc8cc5cc927876d9f138e7ded44c6116c544d3cca2c3a2759e46ba916c7" -dependencies = [ - "chrono", - "data-encoding", - "num-bigint 0.4.8", - "num-traits", - "once_cell", - "ring", - "serde", - "serde_json", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -688,11 +551,11 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.9.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ - "bit-vec 0.9.1", + "bit-vec 0.10.1", ] [[package]] @@ -710,6 +573,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" + [[package]] name = "bitflags" version = "1.3.2" @@ -773,7 +642,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.4.2", "cpufeatures 0.3.1", ] @@ -844,79 +713,6 @@ dependencies = [ "cipher 0.4.4", ] -[[package]] -name = "bollard" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" -dependencies = [ - "async-stream", - "base64 0.22.1", - "bitflags 2.13.2", - "bollard-buildkit-proto", - "bollard-stubs", - "bytes", - "futures-core", - "futures-util", - "hex", - "home", - "http 1.5.0", - "http-body-util", - "hyper", - "hyper-named-pipe", - "hyper-rustls", - "hyper-util", - "hyperlocal", - "log", - "num", - "pin-project-lite", - "rand 0.10.3", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_derive", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.21", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "tonic", - "tower-service", - "url", - "winapi", -] - -[[package]] -name = "bollard-buildkit-proto" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c97450e79c7c565302dd92e86b08823b47550fcb4fc5ce910194d1b087a1a3" -dependencies = [ - "prost", - "prost-types", - "tonic", - "tonic-prost", -] - -[[package]] -name = "bollard-stubs" -version = "1.53.1-rc.29.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" -dependencies = [ - "base64 0.22.1", - "bollard-buildkit-proto", - "bytes", - "prost", - "serde", - "serde_json", - "serde_repr", - "time", -] - [[package]] name = "bs58" version = "0.5.1" @@ -1031,20 +827,22 @@ dependencies = [ [[package]] name = "calcard" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c601473ec15a875626bce73db1a1f0fc81e9c949ca25f1463b714947c0a70a1f" +version = "0.4.0" dependencies = [ "ahash", - "chrono", - "chrono-tz", + "encodify", "hashify", + "insta", + "jiff", "jmap-tools", - "mail-builder 1.0.0", "mail-parser", + "memchr", "rkyv", "serde", "serde_json", + "sha1 0.11.0", + "simdutf8", + "smallvec", "uuid", ] @@ -1106,9 +904,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "jobserver", @@ -1116,15 +914,6 @@ dependencies = [ "shlex 2.0.1", ] -[[package]] -name = "cedarwood" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" -dependencies = [ - "smallvec", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -1201,17 +990,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", -] - -[[package]] -name = "chrono-tz" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" -dependencies = [ - "chrono", - "phf 0.12.1", + "windows-link", ] [[package]] @@ -1242,7 +1021,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.3", + "inout 0.2.5", ] [[package]] @@ -1298,21 +1077,22 @@ dependencies = [ [[package]] name = "common" -version = "0.16.27" +version = "1.0.0" dependencies = [ "aes-gcm-siv", "ahash", "arc-swap", "aws-lc-rs", - "base64 0.23.1", "bincode", + "bumpalo", + "bytes", "calcard", - "chrono", "compact_str", "coordinator", "decancer", "directory", "dns-update", + "encodify", "futures", "hashify", "hickory-proto", @@ -1320,38 +1100,43 @@ dependencies = [ "hyper", "idna", "imagesize", - "infer 0.22.0", + "infer", + "itoa", + "jiff", "jmap_proto", "jsonwebtoken", "libc", "lz4_flex 0.14.0", "mail-auth", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", "md5", + "memchr", "nlp", "nohash-hasher", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", - "p256", - "p384", - "parking_lot", + "p256 0.14.0", + "p384 0.14.0", + "parking_lot 0.13.0", "pem 4.0.0", - "pkcs8", + "percent-encoding", + "pkcs8 0.11.0", "privdrop", "prometheus", "proxy-header", "psl", - "quick-xml 0.41.0", + "quick-xml 0.42.0", "quick_cache", "rasn", "rasn-pkix", "rcgen", "regex", + "regex-automata", "registry", - "reqwest 0.13.5", + "reqwest", "rkyv", "rsa", "rustls", @@ -1364,7 +1149,9 @@ dependencies = [ "sha1 0.11.0", "sha2 0.11.1", "sieve-rs", + "siphasher", "smtp-proto", + "smtp-report", "socket2 0.6.5", "store", "tinyvec", @@ -1378,6 +1165,7 @@ dependencies = [ "x509-parser", "xxhash-rust", "zip", + "zmij", "zxcvbn", ] @@ -1398,9 +1186,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" dependencies = [ "compression-core", "flate2", @@ -1409,17 +1197,19 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" [[package]] -name = "concurrent-queue" -version = "2.5.0" +name = "console" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ - "crossbeam-utils", + "encode_unicode", + "libc", + "windows-sys 0.61.2", ] [[package]] @@ -1434,31 +1224,11 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", -] - [[package]] name = "const_format" -version = "0.2.36" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +checksum = "a3decb4d6965448659923544a634a0571ba44cede4e11ef15a5a317126840454" dependencies = [ "const_format_proc_macros", "konst", @@ -1466,9 +1236,9 @@ dependencies = [ [[package]] name = "const_format_proc_macros" -version = "0.2.34" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +checksum = "941b570269a642506f08cb5b688c04bfa6cfd41a26f8bcdc49db8fd00c0ab659" dependencies = [ "proc-macro2", "quote", @@ -1481,11 +1251,21 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "constant_time_eq" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9" +dependencies = [ + "rustversion", +] + [[package]] name = "coordinator" -version = "0.16.27" +version = "1.0.0" dependencies = [ "async-nats", + "deadpool", "futures", "rdkafka", "redis", @@ -1493,6 +1273,7 @@ dependencies = [ "store", "tokio", "trc", + "utils", "zenoh", ] @@ -1551,6 +1332,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc16" version = "0.4.0" @@ -1559,9 +1350,9 @@ checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -1646,6 +1437,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726facc8b27e604391b232b970cdc19b5468c69f08b466b9734c759216c7415" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array 0.4.15", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -1678,27 +1485,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "csv" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", -] - -[[package]] -name = "csv-core" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" -dependencies = [ - "memchr", -] - [[package]] name = "ctr" version = "0.9.2" @@ -1719,11 +1505,12 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", + "subtle", ] [[package]] @@ -1736,7 +1523,7 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto", + "fiat-crypto 0.2.9", "rustc_version", "subtle", "zeroize", @@ -1759,8 +1546,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c0cf476284b03eb6c10e78787b21c7abb7d7d43cb2f02532ba6b831ed892fa" dependencies = [ - "crypto-bigint", - "elliptic-curve", + "crypto-bigint 0.5.5", + "elliptic-curve 0.13.8", "rand_core 0.6.4", "sha3 0.10.9", "signature 2.2.0", @@ -1885,10 +1672,9 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dav" -version = "0.16.27" +version = "1.0.0" dependencies = [ "calcard", - "chrono", "common", "compact_str", "dav-proto", @@ -1897,10 +1683,12 @@ dependencies = [ "hashify", "http_proto", "hyper", + "nlp", "percent-encoding", "registry", "rkyv", "store", + "tokio", "trc", "types", "utils", @@ -1908,15 +1696,15 @@ dependencies = [ [[package]] name = "dav-proto" -version = "0.16.27" +version = "1.0.0" dependencies = [ "calcard", - "chrono", "compact_str", "hashify", "hyper", "mail-parser", - "quick-xml 0.41.0", + "memchr", + "quick-xml 0.42.0", "serde", "serde_json", "trc", @@ -1967,11 +1755,10 @@ dependencies = [ [[package]] name = "decancer" -version = "3.3.3" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9244323129647178bf41ac861a2cdb9d9c81b9b09d3d0d1de9cd302b33b8a1d" +checksum = "453589e364ce786381e7bcbf0d659088ff7e4d3e77f948dd1cf861344acf7a86" dependencies = [ - "lazy_static", "regex", ] @@ -1983,9 +1770,9 @@ checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1993,23 +1780,23 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ - "thiserror 2.0.21", + "thiserror", ] [[package]] @@ -2019,7 +1806,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", "zeroize", ] @@ -2122,22 +1920,24 @@ dependencies = [ [[package]] name = "directory" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "argon2 0.6.0", - "base64 0.23.1", + "compact_str", "deadpool", + "encodify", + "form_urlencoded", + "hashify", "jsonwebtoken", "ldap3", - "mail-builder 1.0.0", - "mail-parser", "md5", "nohash-hasher", + "parking_lot 0.13.0", "pbkdf2", "pwhash", "registry", - "reqwest 0.13.5", + "reqwest", "scrypt", "serde", "serde_json", @@ -2182,15 +1982,6 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "dns-update" version = "0.6.0" @@ -2204,7 +1995,7 @@ dependencies = [ "hickory-proto", "jiff", "quick-xml 0.42.0", - "reqwest 0.13.5", + "reqwest", "rustls", "serde", "serde_json", @@ -2212,17 +2003,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "docker_credential" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" -dependencies = [ - "base64 0.22.1", - "serde", - "serde_json", -] - [[package]] name = "dsa" version = "0.6.3" @@ -2232,8 +2012,8 @@ dependencies = [ "digest 0.10.7", "num-bigint-dig", "num-traits", - "pkcs8", - "rfc6979", + "pkcs8 0.10.2", + "rfc6979 0.4.0", "sha2 0.10.9", "signature 2.2.0", "zeroize", @@ -2279,30 +2059,27 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", - "rfc6979", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", "signature 2.2.0", - "spki", + "spki 0.7.3", ] [[package]] -name = "ece" -version = "2.4.2" +name = "ecdsa" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2467bac73e5a36d75e16cab0fa8d40676f075db6afde7d78b35f033e1f66e37" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "base64 0.22.1", - "byteorder", - "hex", - "hkdf 0.12.4", - "lazy_static", - "once_cell", - "openssl", - "serde", - "sha2 0.10.9", - "thiserror 2.0.21", + "der 0.8.2", + "digest 0.11.3", + "elliptic-curve 0.14.3", + "rfc6979 0.6.0", + "signature 3.0.0", + "spki 0.8.1", + "zeroize", ] [[package]] @@ -2311,7 +2088,7 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "signature 2.2.0", ] @@ -2342,39 +2119,64 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", + "base16ct 0.2.0", "base64ct", - "crypto-bigint", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.1", "generic-array 0.14.7", - "group", + "group 0.13.0", "hkdf 0.12.4", - "pem-rfc7468", - "pkcs8", + "pem-rfc7468 0.7.0", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "serde_json", - "serdect", + "serdect 0.2.0", "subtle", "tap", "zeroize", ] +[[package]] +name = "elliptic-curve" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.8", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff 0.14.0", + "group 0.14.0", + "hkdf 0.13.0", + "hybrid-array 0.4.15", + "pem-rfc7468 1.0.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", + "subtle", + "zeroize", +] + [[package]] name = "email" -version = "0.16.27" +version = "1.0.0" dependencies = [ "aes 0.9.3", "aes-gcm 0.11.1", + "bytecount", "cbc 0.2.1", "chacha20poly1305", "common", + "compact_str", + "encodify", "groupware", "hashify", "jmap-tools", "jmap_proto", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", "nlp", "rand 0.8.8", @@ -2385,8 +2187,11 @@ dependencies = [ "rkyv", "rsa", "sequoia-openpgp", + "serde_json", "sieve-rs", "store", + "text-extract", + "tinyvec", "tokio", "trc", "types", @@ -2402,6 +2207,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encodify" version = "1.0.2" @@ -2443,20 +2254,10 @@ dependencies = [ ] [[package]] -name = "etcetera" -version = "0.11.0" +name = "evalexpr" +version = "13.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" -dependencies = [ - "cfg-if", - "windows-sys 0.61.2", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "25929004897f2bbab309121a60400d36992f6d911d09baa6c172f6cc55706601" [[package]] name = "event-listener" @@ -2474,13 +2275,13 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.2", + "event-listener", "pin-project-lite", ] [[package]] name = "event_macro" -version = "0.16.27" +version = "1.0.0" dependencies = [ "quote", "syn 3.0.9", @@ -2517,9 +2318,9 @@ dependencies = [ [[package]] name = "fancy-regex" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +checksum = "20ed7f62ce0ae0da1a8b93c3c7104ab7e0760a74491e097c701fff966e945390" dependencies = [ "bit-set 0.8.0", "regex-automata", @@ -2544,15 +2345,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -2560,14 +2352,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] -name = "ferroid" -version = "2.0.0" +name = "faststr" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" dependencies = [ - "portable-atomic", - "rand 0.10.3", - "web-time", + "bytes", + "rkyv", + "serde", + "simdutf8", ] [[package]] @@ -2581,6 +2374,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -2588,16 +2391,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] -name = "find-msvc-tools" -version = "0.1.14" +name = "fiat-crypto" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "fixed_decimal" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79c3c892f121fff406e5dd6b28c1b30096b95111c30701a899d4f2b18da6d1bd" +checksum = "2214fb00f669ee96bee5bb29dfb1ddabe058b2bc426817efc79afa03c9bc55ae" dependencies = [ "displaydoc", "smallvec", @@ -2652,21 +2461,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form-data" version = "0.6.0" @@ -2679,7 +2473,7 @@ dependencies = [ "memchr", "mime", "serde", - "thiserror 2.0.21", + "thiserror", "tracing", ] @@ -2814,21 +2608,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.34" @@ -2897,18 +2676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix", - "windows-link 0.2.1", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "windows-link", ] [[package]] @@ -3001,18 +2769,28 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "groupware" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "calcard", - "chrono", "common", "compact_str", "hashify", @@ -3021,6 +2799,8 @@ dependencies = [ "icu_plurals", "icu_provider", "indexmap 2.14.2", + "jiff", + "jmap-tools", "nlp", "percent-encoding", "registry", @@ -3028,6 +2808,8 @@ dependencies = [ "serde", "serde_json", "store", + "text-extract", + "tokio", "trc", "types", "utils", @@ -3096,19 +2878,20 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] [[package]] name = "hashify" -version = "0.2.9" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" +checksum = "66f5a65bf7824ada2de4a2cf9fa0981dabffc5392bb8550767c3c3d71ed52e4d" dependencies = [ - "indexmap 2.14.2", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -3159,12 +2942,12 @@ dependencies = [ "ipnet", "jni", "lru-cache", - "parking_lot", + "parking_lot 0.12.5", "rand 0.10.3", "rustls", "rustls-pki-types", "rustls-platform-verifier", - "thiserror 2.0.21", + "thiserror", "time", "tinyvec", "tokio", @@ -3189,7 +2972,7 @@ dependencies = [ "rand 0.10.3", "ring", "rustls-pki-types", - "thiserror 2.0.21", + "thiserror", "time", "tinyvec", "tracing", @@ -3211,13 +2994,13 @@ dependencies = [ "moka", "ndk-context", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "rand 0.10.3", "resolv-conf", "rustls", "smallvec", "system-configuration", - "thiserror 2.0.21", + "thiserror", "tokio", "tokio-rustls", "tracing", @@ -3286,20 +3069,20 @@ checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" dependencies = [ "cfg-if", "libc", - "windows-link 0.2.1", + "windows-link", ] [[package]] name = "http" -version = "0.16.27" +version = "1.0.0" dependencies = [ "async-stream", - "base64 0.23.1", - "chrono", "common", + "compact_str", "dav", "directory", "email", + "encodify", "flate2", "form-data", "groupware", @@ -3311,7 +3094,7 @@ dependencies = [ "jmap", "jmap_proto", "mail-auth", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", "mime", "percent-encoding", @@ -3363,29 +3146,9 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel", - "base64 0.13.1", - "futures-lite", - "infer 0.2.3", - "pin-project-lite", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url", -] - [[package]] name = "http_proto" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", "compact_str", @@ -3433,6 +3196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "ctutils", + "subtle", "typenum", "zeroize", ] @@ -3459,20 +3223,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-named-pipe" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" -dependencies = [ - "hex", - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "hyper-rustls" version = "0.27.10" @@ -3483,11 +3233,9 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.9", ] [[package]] @@ -3527,21 +3275,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hyperlocal" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" -dependencies = [ - "hex", - "http-body-util", - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -3554,7 +3287,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -3568,33 +3301,33 @@ dependencies = [ [[package]] name = "icu_calendar" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58655df2f728e46e4eee80bddebefacf52ec3e77cdb925014b47c5a5905eb55c" +checksum = "a68ffb801cc7acb8dcae62da7c692c47934698d2dd4f479ddee1eb442763dea1" dependencies = [ "calendrical_calculations", - "chrono", "displaydoc", "icu_calendar_data", "icu_locale_core", "icu_locale_fallback", "icu_provider", "ixdtf", + "jiff", "tinystr", "zerovec", ] [[package]] name = "icu_calendar_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc00caaa3fb3201ff7a18aa458e8c3ab042b9da154cfdf948bdde3936c31c36e" +checksum = "514da54e79efa36b5976232d5b3d1d8ee1b47b19ab658f1b1ebe39221696c2d8" [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -3606,11 +3339,10 @@ dependencies = [ [[package]] name = "icu_datetime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c44df99c4c5297b05653d1a1629f13bdc9fd4044acfbbaf30b15194a1eed4cd6" +checksum = "3c338aa7ebff734470e216a4196fa84b123a366cc786c445741b6f5647412c3c" dependencies = [ - "chrono", "displaydoc", "fixed_decimal", "icu_calendar", @@ -3622,6 +3354,7 @@ dependencies = [ "icu_plurals", "icu_provider", "icu_time", + "jiff", "potential_utf", "tinystr", "writeable", @@ -3630,15 +3363,15 @@ dependencies = [ [[package]] name = "icu_datetime_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb5555f6e4b42233f58564351ce16d73e9e6ebfce6bdcf29875d0cfc554d6a8" +checksum = "4bddb757c2df1635453c4e8cb2d4ba7ad026e904893f29699315826c0951bbd9" [[package]] name = "icu_decimal" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eb8f655bba2d0c0459e43a5b0e6cd3cd388109898fb3d85d048165e8b0abd08" +checksum = "1150ad5e272740670464934ee76facda43ef2e11f85c27b7ba76ccb33edf2d05" dependencies = [ "displaydoc", "fixed_decimal", @@ -3653,15 +3386,15 @@ dependencies = [ [[package]] name = "icu_decimal_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c887fc7d4c297cf3f9436864af9e3dba7f8be9415a6c2a7f392f3b1636298c" +checksum = "5595ab78370d48abd24a062f3db115d5108dd0b65634352036cae532aebb0df8" [[package]] name = "icu_locale" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "785f595c61ef57169a467eeed7b4b6936a66f6cacbb116a96ee86da983251bf4" +checksum = "fe8fa57bcddc15cb921582ee5259c5b77da39f946f188ca5213fad24d9241236" dependencies = [ "icu_collections", "icu_locale_core", @@ -3674,9 +3407,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -3688,15 +3421,15 @@ dependencies = [ [[package]] name = "icu_locale_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37d91460e362a5cf58907cd7ce871411775ee6294e49a5cc8e6cec16dfa75ea" +checksum = "a64014dac577a11d8d2cee48517c73c1984bd72ab11031e934269bd3cefb2ad9" [[package]] name = "icu_locale_fallback" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +checksum = "406e9ae5b95d8aa37db6293405931b766a6172d1848daee22d70eb44aa78a43c" dependencies = [ "icu_locale_core", "icu_locale_fallback_data", @@ -3708,15 +3441,15 @@ dependencies = [ [[package]] name = "icu_locale_fallback_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" +checksum = "3632ebbd0c5c3637ba1a40bdf8288f2cb9d2268e73441a69d5fc73d7cfbaa87c" [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -3728,15 +3461,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_pattern" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3db611446bd92c246b03e4aa0bafa71ddb194af431ddbb42b9d70e4bfce1a0" +checksum = "bc43ce0e76ee8cb0482dc6e2dbb9b76278e8038f45daabf2a2877577e2ceb18f" dependencies = [ "displaydoc", "either", @@ -3747,9 +3480,9 @@ dependencies = [ [[package]] name = "icu_plurals" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e475e6766ef87b1d3c1f97be6363995b0bfaeddbc789671615df598a97ff0593" +checksum = "2ef046647b445acf95ce0f8083fa292467d3df649959fa9a20f3f8abfd957c5d" dependencies = [ "fixed_decimal", "icu_locale_fallback", @@ -3760,15 +3493,15 @@ dependencies = [ [[package]] name = "icu_plurals_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1251aa39a95e1333888e499b1263e1c196447a28948acf5bdabd82c046f153bf" +checksum = "d43ff7df2c1a1d41588a6ddcba354d513d4a3e10eec22c06fe56d13c12a848ea" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -3781,15 +3514,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -3804,29 +3537,29 @@ dependencies = [ [[package]] name = "icu_time" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af37e1c3b9f3ab564b619fdf4b22de169f6381eb0f7048e3ca454290c438126b" +checksum = "17d45ee651f21433b1c6352311f2ad9b5253c26be0e25d9a0869cb8247b8cfde" dependencies = [ "calendrical_calculations", - "chrono", - "chrono-tz", "displaydoc", "icu_calendar", "icu_locale_core", "icu_provider", "icu_time_data", "ixdtf", + "jiff", "serde", + "writeable", "zerotrie", "zerovec", ] [[package]] name = "icu_time_data" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e21be527807cda7562cbfbdb04179f9cca46c12691d6a8073ffae6cfd7aeec" +checksum = "a6d7d711f44237fa9300406ab6e521148594056862343370b5df265a0d15c088" [[package]] name = "idea" @@ -3872,18 +3605,18 @@ checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" [[package]] name = "imap" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "common", "compact_str", "directory", "email", + "encodify", "imap_proto", "mail-parser", - "md5", "nlp", - "parking_lot", + "parking_lot 0.13.0", "rand 0.10.3", "registry", "store", @@ -3896,14 +3629,16 @@ dependencies = [ [[package]] name = "imap_proto" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", - "base64 0.23.1", - "chrono", "compact_str", + "encodify", "hashify", + "itoa", + "jiff", "mail-parser", + "smallvec", "trc", "types", "utils", @@ -3939,7 +3674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44fbb9c5ccb9a5b67b4afa2974c27e5507ea1bf6d22828cef418e4dfaeca51dd" dependencies = [ "libflate", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -3965,12 +3700,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "infer" version = "0.22.0" @@ -3992,21 +3721,24 @@ dependencies = [ [[package]] name = "inout" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c0d789294fb457b791f7797a04478704788f30b407e5d0b01deaa846572d9f" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" dependencies = [ "block-padding 0.4.2", "hybrid-array 0.4.15", ] [[package]] -name = "instant" -version = "0.1.13" +name = "insta" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "b67d3d2e287e4b86c10b3f3b641033d1f89b74bdb39d05f34952e2b9a6fe21cd" dependencies = [ - "cfg-if", + "console", + "once_cell", + "similar 2.7.0", + "tempfile", ] [[package]] @@ -4018,7 +3750,7 @@ dependencies = [ "socket2 0.6.5", "widestring", "windows-registry", - "windows-result 0.4.1", + "windows-result", "windows-sys 0.61.2", ] @@ -4058,6 +3790,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -4066,30 +3807,25 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "ixdtf" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3667095d64c3ecffc96463a21157b04bf3e252f6e8d5750b20c02e33c194e3" +checksum = "4754a1731458af0443709c9667302793d3f63cc90ec4ba4c8259f2b9515478e5" [[package]] name = "jieba-macros" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455f837e9d0255b68a712200db247c68fdad4941b72471b76bfa61c3b0c1f79f" -dependencies = [ - "phf_codegen", -] +checksum = "2987cf45b1bfdb4ff1f6a814b1aa2ad6361f4ff43bf353a9bd29f391a9568f70" [[package]] name = "jieba-rs" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a8bbb0f77ee810f0689a30b7cec56b875751ef4ec2e74fd995613dc52b3ae1" +checksum = "0a3a7d02ea31e6fc6f2fe10495dc7c5efcf381e1f21d7172f58ba792efa1c2fd" dependencies = [ "bytecount", - "cedarwood", "include-flate", "jieba-macros", - "phf 0.13.1", "rustc-hash", ] @@ -4102,12 +3838,15 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", + "jiff-tzdb", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", - "windows-link 0.2.1", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -4149,15 +3888,15 @@ dependencies = [ [[package]] name = "jmap" -version = "0.16.27" +version = "1.0.0" dependencies = [ "async-stream", - "base64 0.23.1", "calcard", - "chrono", "common", + "compact_str", "directory", "email", + "encodify", "futures-util", "groupware", "hashify", @@ -4165,16 +3904,18 @@ dependencies = [ "http_proto", "hyper", "hyper-util", + "jiff", "jmap-tools", "jmap_proto", "mail-auth", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", + "memchr", "nlp", - "p256", + "p256 0.14.0", "rand 0.10.3", "registry", - "reqwest 0.13.5", + "reqwest", "rkyv", "serde", "serde_json", @@ -4187,42 +3928,17 @@ dependencies = [ "spam-filter", "store", "tokio", - "tokio-tungstenite 0.30.0", + "tokio-tungstenite", "trc", - "tungstenite 0.30.0", + "tungstenite", "types", "utils", ] -[[package]] -name = "jmap-client" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b5bc66252cc8e779ef1238f40ab93971d54ad00b5d7afb5c1d447a9647ed62" -dependencies = [ - "ahash", - "async-stream", - "chrono", - "encodify", - "futures-util", - "maybe-async", - "parking_lot", - "reqwest 0.13.5", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "tokio", - "tokio-tungstenite 0.29.0", -] - [[package]] name = "jmap-tools" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bdcd368dcd58a5efaac10c142fb8cddb4d2949af1297b624df3ff0db4cbff54" +version = "0.1.9" dependencies = [ - "hashify", "rkyv", "serde", "serde_json", @@ -4230,11 +3946,12 @@ dependencies = [ [[package]] name = "jmap_proto" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "calcard", "compact_str", + "encodify", "hashify", "jmap-tools", "mail-parser", @@ -4242,6 +3959,8 @@ dependencies = [ "rkyv", "serde", "serde_json", + "simdutf8", + "sonic-rs", "trc", "types", "utils", @@ -4259,9 +3978,9 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.21", + "thiserror", "walkdir", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -4410,19 +4129,19 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lalrpop" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a80a963123205c7157323c99611bc4abb65dcbd62ef46dc4bac74a3941bc75" +checksum = "6d315a5ee87f679f2e7caead474b08ec8de60de01a08c942248d4e3c2754565d" dependencies = [ "ascii-canvas", - "bit-set 0.9.1", + "bit-set 0.11.1", "ena", - "itertools 0.14.0", + "itertools 0.15.0", "lalrpop-util", "petgraph", "regex", "regex-syntax", - "sha3 0.10.9", + "sha3 0.12.0", "string_cache", "term", "unicode-xid", @@ -4431,9 +4150,9 @@ dependencies = [ [[package]] name = "lalrpop-util" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884f3e747ed2dcee867cda1b0c31a048f9e20de2d916a248949319921a2e666e" +checksum = "364b49d1c98c07972e16fe1346463becc08bcd663da04e6e37ab1bea068dce6a" dependencies = [ "regex-automata", ] @@ -4473,7 +4192,7 @@ dependencies = [ "percent-encoding", "rustls", "rustls-native-certs", - "thiserror 2.0.21", + "thiserror", "tokio", "tokio-rustls", "tokio-stream", @@ -4490,9 +4209,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libflate" @@ -4525,7 +4244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -4560,9 +4279,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" dependencies = [ "cc", "pkg-config", @@ -4608,6 +4327,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d91b22da179613dcf4d6ec8349563f52a363f6849e03b00dd0a185ec4a59af" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -4668,83 +4396,74 @@ dependencies = [ [[package]] name = "lzma-rust2" -version = "0.16.5" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b" +checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa" dependencies = [ "sha2 0.11.1", ] [[package]] name = "mail-auth" -version = "0.13.3" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505122ba86e1f4adeb664196c1e787c3f29bb6e7c128e4a366d47d209911440" +checksum = "96848b9e5340bf531190f600e978310907e4ddfd7757868cb999f59b382d274b" dependencies = [ "aws-lc-rs", - "flate2", + "aws-lc-sys", + "encodify", "getrandom 0.2.17", "getrandom 0.4.3", "hashify", "hickory-resolver", "idna", - "mail-builder 1.0.0", "mail-parser", "memchr", - "quick-xml 0.42.0", "rand 0.10.3", - "rkyv", "rsa", "rustls-pki-types", "serde", "serde_json", - "similar", - "zip", + "simdutf8", + "similar 3.2.1", ] [[package]] name = "mail-builder" -version = "0.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c942e8a4b83f9351236c1e531ea9fa0237913d63c7fc36818430e0128a1ddf3" -dependencies = [ - "gethostname", -] - -[[package]] -name = "mail-builder" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccbc27cb02a58a311a404769a22f5abf63fe3173b290d80be7c534f9c7023151" +checksum = "a8bca634885db8e836b80700a73661de20ffd93cb5787729afc31575545e2aed" dependencies = [ + "encodify", "gethostname", ] [[package]] name = "mail-parser" -version = "0.11.9" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" +checksum = "908e05ffd2f281e4494cdb77552e6df2dc1476da22af811685b37773d415b2df" dependencies = [ + "encodify", "encoding_rs", "hashify", "memchr", - "rkyv", "serde", + "simdutf8", ] [[package]] name = "managesieve" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", "compact_str", "directory", "email", + "encodify", "imap", "imap_proto", "jmap_proto", - "mail-parser", "registry", "sieve-rs", "store", @@ -4777,12 +4496,6 @@ dependencies = [ "quote", ] -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - [[package]] name = "matchers" version = "0.2.0" @@ -4792,23 +4505,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - -[[package]] -name = "maybe-async" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "md-5" version = "0.9.1" @@ -4868,36 +4564,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c797b9d6bb23aab2fc369c65f871be49214f5c759af65bde26ffaaa2b646b492" -[[package]] -name = "migration" -version = "0.16.27" -dependencies = [ - "common", - "email", - "registry", - "spam-filter", - "store", - "tokio", - "trc", - "types", -] - [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4942,15 +4614,16 @@ dependencies = [ [[package]] name = "ml-kem" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" +checksum = "a9ba01ebcf90f6aff8e9b9f55bb69b19e6250e4ade5787d0040eaa27dd226238" dependencies = [ "hybrid-array 0.4.15", "kem", "module-lattice", "rand_core 0.10.1", - "sha3 0.11.0", + "sha3 0.12.0", + "shake", "zeroize", ] @@ -4976,7 +4649,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "parking_lot", + "parking_lot 0.12.5", "portable-atomic", "smallvec", "tagptr", @@ -5024,7 +4697,7 @@ dependencies = [ "quote", "syn 2.0.119", "termcolor", - "thiserror 2.0.21", + "thiserror", ] [[package]] @@ -5049,13 +4722,13 @@ dependencies = [ "rustls", "serde", "socket2 0.6.5", - "thiserror 2.0.21", + "thiserror", "tokio", "tokio-rustls", "tokio-util", "twox-hash 2.1.5", "url", - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", ] [[package]] @@ -5081,7 +4754,7 @@ dependencies = [ "serde_json", "sha1 0.10.7", "sha2 0.10.9", - "thiserror 2.0.21", + "thiserror", "uuid", ] @@ -5118,14 +4791,26 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nlp" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "hashify", "jieba-rs", - "maplit", + "memchr", "psl", "rand 0.10.3", "rkyv", @@ -5175,15 +4860,6 @@ dependencies = [ "serde", ] -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -5193,20 +4869,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint 0.4.8", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.8" @@ -5243,15 +4905,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.2" @@ -5277,17 +4930,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint 0.4.8", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -5339,16 +4981,6 @@ dependencies = [ "bitflags 2.13.2", ] -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - [[package]] name = "objc2-system-configuration" version = "0.3.2" @@ -5395,31 +5027,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openssl" -version = "0.10.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "foreign-types", - "libc", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "openssl-probe" version = "0.2.1" @@ -5447,7 +5054,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.21", + "thiserror", "tracing", ] @@ -5460,7 +5067,7 @@ dependencies = [ "bytes", "http 1.5.0", "opentelemetry", - "reqwest 0.13.5", + "reqwest", ] [[package]] @@ -5475,8 +5082,8 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.13.5", - "thiserror 2.0.21", + "reqwest", + "thiserror", "tokio", "tonic", "tonic-types", @@ -5511,7 +5118,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.21", + "thiserror", ] [[package]] @@ -5520,54 +5127,81 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "sha2 0.10.9", ] +[[package]] +name = "p256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.1", +] + [[package]] name = "p384" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "sha2 0.10.9", ] +[[package]] +name = "p384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "fiat-crypto 0.3.0", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.1", +] + [[package]] name = "p521" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", - "ecdsa", - "elliptic-curve", - "primeorder", + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "rand_core 0.6.4", "sha2 0.10.9", ] +[[package]] +name = "papaya" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2442474a9404698c42509b8967f437249dbc7b50493e83020333d3943ec0ae" +dependencies = [ + "equivalent", + "seize", +] + [[package]] name = "parking" version = "2.2.1" @@ -5580,8 +5214,18 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.14", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cf937ca4e1d0b7578784559063422fc39ae7c5a00f02471f2614d5486fd226" +dependencies = [ + "lock_api 0.5.0", + "parking_lot_core 0.10.0", ] [[package]] @@ -5592,34 +5236,21 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", - "windows-link 0.2.1", + "windows-link", ] [[package]] -name = "parse-display" -version = "0.11.0" +name = "parking_lot_core" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78deb158fb1d73b29efb4b7e9b9860b78059c670de06bd28df8d0b458ded0eb" +checksum = "9cfff060068cc18aa7815010fcaf80c68c7fb9e85fc7f5c0f9bf142c21d19826" dependencies = [ - "parse-display-derive", - "regex", - "regex-syntax", -] - -[[package]] -name = "parse-display-derive" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e95a50d1084dab562913062c4c34bb204b68fc6ec38a1395909ff5aaaf4f10a" -dependencies = [ - "proc-macro2", - "quote", - "regex", - "regex-syntax", - "structmeta", - "syn 2.0.119", + "libc", + "redox_syscall 0.9.5", + "smallvec", + "windows-link", ] [[package]] @@ -5690,6 +5321,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -5698,9 +5338,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b568374ba38b33a6c627141f891faf16902b08d2db26b8ede1bcb0a15b1919fa" +checksum = "fd47a460a9cbc3b3429c7e22bf0d76653e779319bae45a499ca010c9a990bb23" dependencies = [ "memchr", "psm", @@ -5710,9 +5350,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66e184b924cebaaff20ab2256ca52f12332d528a39aa76553b5d96f92aacf7f" +checksum = "203e46697708f7519ab1608393cf89239cafb46b8e6d08a8ffe359359b11ef44" dependencies = [ "pest", "pest_generator", @@ -5720,9 +5360,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a87478d267e4de54a626af9754f2f0f58e927aac6ed0575fe89bc05ad6851694" +checksum = "e84ae6340a458d3d6cfc73cf328cf3cfe90b2484cd0de25eb66aa954615ed4d5" dependencies = [ "pest", "pest_meta", @@ -5733,9 +5373,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f986f248b4241ac359b831f6139aaa34e03b08a37b6caf7e201a33f95c869e1" +checksum = "b57f92f32b3dbce6598fd8f5cdf1f06a6072570b8103fa21728fa9036923e710" dependencies = [ "pest", ] @@ -5763,15 +5403,6 @@ dependencies = [ "getrandom 0.4.3", ] -[[package]] -name = "phf" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" -dependencies = [ - "phf_shared 0.12.1", -] - [[package]] name = "phf" version = "0.13.1" @@ -5783,23 +5414,13 @@ dependencies = [ "serde", ] -[[package]] -name = "phf_codegen" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" -dependencies = [ - "phf_generator", - "phf_shared 0.13.1", -] - [[package]] name = "phf_generator" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ - "fastrand 2.5.0", + "fastrand", "phf_shared 0.13.1", ] @@ -5818,18 +5439,18 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ "siphasher", ] [[package]] name = "phf_shared" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" dependencies = [ "siphasher", ] @@ -5866,9 +5487,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -5877,8 +5498,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.1", ] [[package]] @@ -5954,13 +5585,16 @@ dependencies = [ [[package]] name = "pop3" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", + "compact_str", "directory", "email", + "encodify", "imap", - "mail-parser", + "itoa", + "memchr", "registry", "store", "tokio", @@ -6035,9 +5669,9 @@ checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppmd-rust" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196a7c80b9a7652aba7cc070827516c2abe4ccdf53d128e1944003cf5726cff1" +checksum = "cd4acc8d264ea385db64ccfdcdfd5909ee9eda0975666723d59f3bfec80b3611" [[package]] name = "ppv-lite86" @@ -6077,13 +5711,40 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint 0.7.8", + "crypto-common 0.2.2", + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "primeorder" version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve 0.14.3", + "once_cell", + "primefield", + "serdect 0.4.3", + "wnaf", ] [[package]] @@ -6093,7 +5754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "454e2cdcd17f5b868a2f600b14660c1af558b85b69540b15ddc6b0a890d5b7f0" dependencies = [ "libc", - "nix", + "nix 0.30.1", ] [[package]] @@ -6157,8 +5818,8 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", - "thiserror 2.0.21", + "parking_lot 0.12.5", + "thiserror", ] [[package]] @@ -6205,9 +5866,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.241" +version = "2.1.245" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7319b480e204d67e759cad7886178335d26324c7c8573c126cc9dfa7350d064" +checksum = "46cce9bc841c0eeb395275a775d11e3cc91b904e6ab370a9a42369e111b3038e" dependencies = [ "psl-types", ] @@ -6262,16 +5923,6 @@ dependencies = [ "sha2 0.9.9", ] -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -6301,7 +5952,7 @@ dependencies = [ "equivalent", "foldhash 0.2.0", "hashbrown 0.17.1", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -6317,8 +5968,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", - "thiserror 2.0.21", + "socket2 0.6.5", + "thiserror", "tokio", "tracing", "web-time", @@ -6343,7 +5994,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "slab", - "thiserror 2.0.21", + "thiserror", "tinyvec", "tracing", "web-time", @@ -6358,7 +6009,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] @@ -6391,7 +6042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot", + "parking_lot 0.12.5", "scheduled-thread-pool", ] @@ -6416,19 +6067,6 @@ dependencies = [ "ptr_meta", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.8" @@ -6461,16 +6099,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -6491,15 +6119,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -6524,15 +6143,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_pcg" version = "0.10.2" @@ -6544,9 +6154,9 @@ dependencies = [ [[package]] name = "rasn" -version = "0.28.15" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff8f77a3a7082ed9f3e878927f34b1d633293fce9363d8ba14ee1c24bd2e967" +checksum = "0c442ad572d02886fe3a24f770763cb39afb9b3dfe16e05fcdf133b3895e4a24" dependencies = [ "bitvec", "bytes", @@ -6566,9 +6176,9 @@ dependencies = [ [[package]] name = "rasn-cms" -version = "0.28.15" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d11eca85a3699d3e52795874704ddd6d2f03bda3585be3e6655a0d9a166e84e" +checksum = "e30155c100fb601e0e82c125214dfc7d3fc7c91a36a5001336ec01d36c52872d" dependencies = [ "rasn", "rasn-pkix", @@ -6576,9 +6186,9 @@ dependencies = [ [[package]] name = "rasn-derive" -version = "0.28.15" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe71aa61cfb0191e50b95ebd72d454e9eeabff8a35b6252b99f2e90b7ef4c6d" +checksum = "261556388c573dde262717e06ad2c9a9ed78405418f16f2789a0dd8004793fba" dependencies = [ "proc-macro2", "rasn-derive-impl", @@ -6587,9 +6197,9 @@ dependencies = [ [[package]] name = "rasn-derive-impl" -version = "0.28.15" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b7d207ad810df71b46ef90916dbb71fe366e0ad58a62e737fe6098525684c" +checksum = "8e17b9360b511cde533e75baa711dc652f6f613457afe02a66c748ccd6c31ef7" dependencies = [ "either", "itertools 0.13.0", @@ -6601,9 +6211,9 @@ dependencies = [ [[package]] name = "rasn-pkix" -version = "0.28.15" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773eb443e9c58d9d356c065bbfea2c04322277c44073b4e0716967d8ccd7ef58" +checksum = "6e0869fd5847184c93887f98eeb0aa2f70cc3f1eec1f0adf3ff8f0f867d60b4e" dependencies = [ "rasn", ] @@ -6671,14 +6281,15 @@ dependencies = [ "libc", "libz-sys", "num_enum", + "openssl-sys", "pkg-config", ] [[package]] name = "redis" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9" +checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5" dependencies = [ "arcstr", "async-lock", @@ -6715,6 +6326,15 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "redox_users" version = "0.5.3" @@ -6722,7 +6342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ "libredox", - "thiserror 2.0.21", + "thiserror", ] [[package]] @@ -6776,14 +6396,16 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "registry" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", + "compact_str", "hashify", "jmap-tools", "lz4_flex 0.14.0", "mail-auth", "serde", + "smtp-report", "tokio", "trc", "types", @@ -6799,48 +6421,6 @@ dependencies = [ "bytecheck", ] -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-core", - "futures-util", - "http 1.5.0", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams 0.4.2", - "web-sys", - "webpki-roots 1.0.9", -] - [[package]] name = "reqwest" version = "0.13.5" @@ -6860,7 +6440,6 @@ dependencies = [ "hyper-util", "js-sys", "log", - "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -6869,7 +6448,6 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -6880,7 +6458,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.5.0", + "wasm-streams", "web-sys", ] @@ -6900,6 +6478,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint 0.7.8", + "hmac 0.13.0", +] + [[package]] name = "ring" version = "0.17.14" @@ -6948,6 +6536,7 @@ dependencies = [ "rancor", "rend", "rkyv_derive", + "smallvec", "tinyvec", "uuid", ] @@ -7015,36 +6604,35 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2 0.10.9", "signature 2.2.0", - "spki", + "spki 0.7.3", "subtle", "zeroize", ] [[package]] name = "rsqlite-vfs" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +checksum = "353b74f9810d5b085b6c42c9d86a5f0fdb795553235cdd48753b32e3809d18a7" dependencies = [ - "hashbrown 0.16.1", - "thiserror 2.0.21", + "thiserror", ] [[package]] name = "rtrb" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9278fb35b3e730abe136e9b395b5b81b96d06b9f5478a50f0c8430a2237b22de" +checksum = "5974e397833d890f2f74963408e0d4a6cd8eda2f6a445ae06750291b5d7a997e" [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "9c026b4444579e8d754556c6c75d040d69c4a44283cdb648ab8a8d5965c2cfa8" dependencies = [ "bitflags 2.13.2", "fallible-iterator 0.3.0", @@ -7055,50 +6643,6 @@ dependencies = [ "sqlite-wasm-rs", ] -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rust-s3" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d4ed794cdfede6c59f81d485cae6bb63f47a7e35d190148d40754f783073c4" -dependencies = [ - "async-trait", - "aws-creds", - "aws-region", - "base64 0.22.1", - "bytes", - "cfg-if", - "futures-util", - "hex", - "hmac 0.12.1", - "http 1.5.0", - "log", - "maybe-async", - "md5", - "percent-encoding", - "quick-xml 0.41.0", - "reqwest 0.12.28", - "serde", - "serde_derive", - "serde_json", - "sha2 0.10.9", - "sysinfo", - "thiserror 2.0.21", - "time", - "tokio", - "tokio-stream", - "url", -] - [[package]] name = "rust-stemmers" version = "1.2.0" @@ -7222,9 +6766,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -7244,6 +6788,24 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s3" +version = "1.0.0" +dependencies = [ + "aws-lc-rs", + "bytes", + "crc-fast", + "encodify", + "hashify", + "jiff", + "md5", + "memchr", + "quick-xml 0.42.0", + "rand 0.10.3", + "reqwest", + "tokio", +] + [[package]] name = "salsa20" version = "0.11.0" @@ -7271,9 +6833,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -7284,7 +6846,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -7327,9 +6889,10 @@ dependencies = [ [[package]] name = "scim" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", + "compact_str", "directory", "http_proto", "hyper", @@ -7340,6 +6903,7 @@ dependencies = [ "scim-proto", "serde", "serde_json", + "services", "store", "trc", "types", @@ -7349,7 +6913,7 @@ dependencies = [ [[package]] name = "scim-proto" -version = "0.16.27" +version = "1.0.0" dependencies = [ "hashify", "serde", @@ -7390,11 +6954,25 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array 0.14.7", - "pkcs8", - "serdect", + "pkcs8 0.10.2", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.2", + "hybrid-array 0.4.15", "subtle", "zeroize", ] @@ -7432,6 +7010,16 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "semver" version = "1.0.28" @@ -7465,7 +7053,7 @@ dependencies = [ "dyn-clone", "eax", "ecb", - "ecdsa", + "ecdsa 0.16.9", "ed25519", "ed25519-dalek", "getrandom 0.2.17", @@ -7482,8 +7070,8 @@ dependencies = [ "num-bigint-dig", "num-integer", "ocb3", - "p256", - "p384", + "p256 0.13.2", + "p384 0.13.1", "p521", "rand 0.10.3", "regex", @@ -7494,7 +7082,7 @@ dependencies = [ "sha2 0.10.9", "sha3 0.10.9", "slh-dsa", - "thiserror 1.0.69", + "thiserror", "twofish", "typenum", "x25519-dalek", @@ -7558,8 +7146,6 @@ version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ - "foldhash 0.2.0", - "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -7567,17 +7153,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_qs" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" -dependencies = [ - "percent-encoding", - "serde", - "thiserror 1.0.69", -] - [[package]] name = "serde_repr" version = "0.1.21" @@ -7653,63 +7228,51 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "base16ct", + "base16ct 0.2.0", "serde", ] [[package]] -name = "serial_test" -version = "4.0.1" +name = "serdect" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6df5ed973ad8d834e09f824f9e9f449af6b9a3745f78dec7cc752770bd3bf11" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "futures-executor", - "futures-util", - "log", - "once_cell", - "parking_lot", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", + "base16ct 1.0.0", + "serde", ] [[package]] name = "services" -version = "0.16.27" +version = "1.0.0" dependencies = [ "aes-gcm 0.11.1", "aho-corasick", - "base64 0.23.1", "calcard", "common", + "compact_str", + "coordinator", "dns-update", "email", "groupware", "hkdf 0.13.0", "jmap-tools", "jmap_proto", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", "memory-stats", - "p256", + "nlp", + "p256 0.14.0", "psl", "registry", - "reqwest 0.13.5", + "reqwest", "serde_json", "sha2 0.11.1", "smtp", "smtp-proto", "spam-filter", "store", + "text-extract", "tokio", "trc", "types", @@ -7821,12 +7384,13 @@ dependencies = [ [[package]] name = "sha3" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ "digest 0.11.3", "keccak 0.2.2", + "sponge-cursor", ] [[package]] @@ -7872,18 +7436,21 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "sieve-rs" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd00a548fde57bd0c8e7c13ae65fc5fe30bd923ff8655c81e010f3f02a90997b" +version = "2.0.0" dependencies = [ "ahash", - "arc-swap", - "fancy-regex 0.19.2", + "bumpalo", + "encodify", + "evalexpr", + "fancy-regex 0.20.0", + "hashbrown 0.17.1", "hashify", - "mail-builder 0.5.0", + "mail-builder", "mail-parser", - "rkyv", + "memchr", "serde", + "serde_json", + "smallvec", ] [[package]] @@ -7913,6 +7480,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ "digest 0.11.3", + "rand_core 0.10.1", ] [[package]] @@ -7939,9 +7507,15 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" -version = "3.2.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f66ca1f7aca2474dc10c942eb22feffc897735f54cd1db90138c2fddb490987" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "similar" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f8bec09ae8fae0f4f265a4559a9334be3929d8504e9b536df0a41302bd1733" dependencies = [ "bstr", ] @@ -7954,7 +7528,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint 0.4.8", "num-traits", - "thiserror 2.0.21", + "thiserror", "time", ] @@ -7980,7 +7554,7 @@ dependencies = [ "digest 0.10.7", "hmac 0.12.1", "hybrid-array 0.3.1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2 0.10.9", "sha3 0.10.9", @@ -7991,29 +7565,33 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.3" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" +dependencies = [ + "serde", +] [[package]] name = "smtp" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", - "base64 0.23.1", "blake3", "common", "compact_str", "directory", "email", + "encodify", "hashify", "mail-auth", - "mail-builder 1.0.0", + "mail-builder", "mail-parser", - "parking_lot", + "memchr", + "parking_lot 0.13.0", "rand 0.10.3", "registry", - "reqwest 0.13.5", + "reqwest", "rkyv", "rustls", "rustls-pki-types", @@ -8023,6 +7601,7 @@ dependencies = [ "sha2 0.11.1", "sieve-rs", "smtp-proto", + "smtp-report", "spam-filter", "store", "tokio", @@ -8036,14 +7615,29 @@ dependencies = [ [[package]] name = "smtp-proto" version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142a5a642c6bd7ffd7e1b525ad6a6e9921ccef992dba4663974f8519209a00c1" dependencies = [ "memchr", "rkyv", "serde", ] +[[package]] +name = "smtp-report" +version = "1.0.0" +dependencies = [ + "flate2", + "hashify", + "mail-auth", + "mail-builder", + "mail-parser", + "memchr", + "quick-xml 0.42.0", + "serde", + "serde_json", + "simdutf8", + "zip", +] + [[package]] name = "snafu" version = "0.8.9" @@ -8085,9 +7679,48 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonic-number" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c350a05f295f922fc153a3af8b443c10db70904a9291a624ddb6058c6110d0" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756586caeeb21126a57c17d1ef2b9386e2613ba784aae0f27104f5e9982335ec" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + [[package]] name = "spam-filter" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", "compact_str", @@ -8095,17 +7728,20 @@ dependencies = [ "hashify", "hyper", "idna", - "infer 0.22.0", + "infer", "mail-auth", "mail-parser", + "memchr", "nlp", "psl", "registry", "rkyv", "serde", + "serde_json", "sha1 0.11.0", "smtp-proto", "store", + "sync_wrapper", "tokio", "trc", "types", @@ -8121,7 +7757,7 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ - "lock_api", + "lock_api 0.4.14", ] [[package]] @@ -8137,7 +7773,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -8148,14 +7794,12 @@ checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" [[package]] name = "sqlite-wasm-rs" -version = "0.5.5" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +checksum = "36bd8e29bf08189a005b2df1df99823aa0861cabe9044e5f31ed54ca292ee618" dependencies = [ "cc", - "js-sys", "rsqlite-vfs", - "wasm-bindgen", ] [[package]] @@ -8213,7 +7857,7 @@ dependencies = [ [[package]] name = "stalwart" -version = "0.16.27" +version = "1.0.0" dependencies = [ "common", "coordinator", @@ -8221,13 +7865,12 @@ dependencies = [ "directory", "email", "groupware", - "http 0.16.27", + "http 1.0.0", "http_proto", "imap", "jmap", "libc", "managesieve", - "migration", "pop3", "registry", "rustls", @@ -8252,47 +7895,53 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "arc-swap", "aws-lc-rs", "azure_core", - "azure_storage", - "azure_storage_blobs", + "azure_identity", + "azure_storage_blob", "bitpacking", "blake3", "bytes", "compact_str", "deadpool", "deadpool-postgres", + "encodify", "flate2", "foundationdb", "futures", "gethostname", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "libc", "lru-cache", "lz4_flex 0.14.0", "mysql_async", "nlp", + "nohash-hasher", "num_cpus", - "parking_lot", + "papaya", + "parking_lot 0.13.0", "r2d2", "radsort", "rand 0.10.3", "rayon", "redis", "registry", - "reqwest 0.12.28", - "reqwest 0.13.5", + "reqwest", "rkyv", "roaring", "rocksdb", "rusqlite", - "rust-s3", "rustls", "rustls-pki-types", + "s3", "serde", "serde_json", + "tinyvec", "tokio", "tokio-postgres", "tokio-rustls", @@ -8301,17 +7950,18 @@ dependencies = [ "utils", "x509-parser", "xxhash-rust", + "zstd 0.14.3", ] [[package]] name = "string_cache" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" dependencies = [ "new_debug_unreachable", - "parking_lot", - "phf_shared 0.13.1", + "parking_lot 0.12.5", + "phf_shared 0.14.0", "precomputed-hash", ] @@ -8332,29 +7982,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "structmeta" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" -dependencies = [ - "proc-macro2", - "quote", - "structmeta-derive", - "syn 2.0.119", -] - -[[package]] -name = "structmeta-derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "subtle" version = "2.6.1" @@ -8414,20 +8041,6 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "sysinfo" -version = "0.37.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - [[package]] name = "system-configuration" version = "0.8.0" @@ -8461,6 +8074,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "term" version = "1.2.1" @@ -8480,115 +8106,17 @@ dependencies = [ ] [[package]] -name = "testcontainers" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2bbe381afaaa58ea610c5fc3ffb2184063a32b3e358a179f0b4865dd59934a" +name = "text-extract" +version = "1.0.0" dependencies = [ - "astral-tokio-tar", - "async-trait", - "bollard", - "bytes", - "docker_credential", - "either", - "etcetera", - "ferroid", - "futures", - "http 1.5.0", - "itertools 0.14.0", - "log", - "memchr", - "parse-display", - "pin-project-lite", - "serde", - "serde_json", - "serde_with", - "thiserror 2.0.21", - "tokio", - "tokio-stream", - "tokio-util", - "url", -] - -[[package]] -name = "tests" -version = "0.16.27" -dependencies = [ - "ahash", "aws-lc-rs", - "base64 0.23.1", - "biscuit", - "bytes", - "calcard", - "chrono", - "common", - "coordinator", - "csv", - "dav", - "dav-proto", - "directory", - "dns-update", - "ece", - "email", + "encoding_rs", "flate2", - "form_urlencoded", - "futures", - "groupware", - "http 0.16.27", - "http_proto", - "hyper", - "hyper-util", - "imap", - "imap_proto", - "jmap", - "jmap-client", - "jmap-tools", - "jmap_proto", - "mail-auth", - "mail-builder 1.0.0", + "hashify", "mail-parser", - "managesieve", - "nlp", - "pop3", - "quick-xml 0.41.0", - "rayon", - "rcgen", - "registry", - "reqwest 0.13.5", - "rkyv", - "rust-s3", - "rustls", - "rustls-pki-types", - "scim", - "scim-proto", - "serde", - "serde_json", - "serial_test", - "services", - "sha2 0.11.1", - "sieve-rs", - "smtp", - "smtp-proto", - "spam-filter", - "store", - "testcontainers", - "tikv-jemallocator", - "time", - "tokio", - "tokio-rustls", - "trc", - "types", - "utils", - "x509-parser", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", + "md5", + "memchr", + "zip", ] [[package]] @@ -8597,18 +8125,7 @@ version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.21", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "thiserror-impl", ] [[package]] @@ -8682,20 +8199,11 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "serde_core", @@ -8716,7 +8224,7 @@ checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530" dependencies = [ "futures-util", "pin-project-lite", - "thiserror 2.0.21", + "thiserror", "tokio", "tokio-rustls", ] @@ -8734,14 +8242,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.3" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "socket2 0.6.5", @@ -8773,9 +8281,9 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", - "phf 0.13.1", + "phf", "pin-project-lite", "postgres-protocol", "postgres-types", @@ -8807,22 +8315,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" -dependencies = [ - "futures-util", - "log", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tungstenite 0.29.0", -] - [[package]] name = "tokio-tungstenite" version = "0.30.0" @@ -8832,7 +8324,7 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.30.0", + "tungstenite", ] [[package]] @@ -8908,10 +8400,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", - "axum", "base64 0.22.1", "bytes", - "h2", "http 1.5.0", "http-body", "http-body-util", @@ -8921,7 +8411,6 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", - "socket2 0.6.5", "sync_wrapper", "tokio", "tokio-rustls", @@ -8961,7 +8450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49d0f9b7aa5def86d3a8d8ba926fe8c1990924dd715d2b5fc54a4aa3788c11c1" dependencies = [ "base32", - "constant_time_eq", + "constant_time_eq 0.4.2", "hmac 0.13.0", "percent-encoding", "rand 0.10.3", @@ -9103,17 +8592,18 @@ dependencies = [ [[package]] name = "trc" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", - "base64 0.23.1", "compact_str", + "encodify", "event_macro", "hashify", + "itoa", "mail-auth", "mail-parser", - "parking_lot", - "reqwest 0.13.5", + "parking_lot 0.13.0", + "reqwest", "rkyv", "rtrb", "serde", @@ -9133,24 +8623,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb1626d07cb5c1bb2cf17d94c0be4852e8a7c02b041acec9a8c5bdda99f9d580" -[[package]] -name = "tungstenite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" -dependencies = [ - "bytes", - "data-encoding", - "http 1.5.0", - "httparse", - "log", - "rand 0.9.5", - "rustls", - "rustls-pki-types", - "sha1 0.10.7", - "thiserror 2.0.21", -] - [[package]] name = "tungstenite" version = "0.30.0" @@ -9164,7 +8636,7 @@ dependencies = [ "log", "rand 0.10.3", "sha1 0.11.0", - "thiserror 2.0.21", + "thiserror", ] [[package]] @@ -9194,9 +8666,9 @@ checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" [[package]] name = "typed-path" -version = "0.12.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" [[package]] name = "typeid" @@ -9212,18 +8684,73 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "types" -version = "0.16.27" +version = "1.0.0" dependencies = [ "blake3", "compact_str", + "encodify", "hashify", "jmap-tools", + "memchr", "rkyv", "serde", "trc", + "unicode-normalization", "utils", ] +[[package]] +name = "typespec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures", + "quick-xml 0.41.0", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0373af0f9d4f580b3a1a9d9639cedaabe015ed262b35bfbe13941bfb14fe1ea6" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.3", + "reqwest", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c608f4427943f8adb211abc95c87672b1b98847152783507d54e3246e502f60" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -9244,12 +8771,6 @@ dependencies = [ "spin 0.10.1", ] -[[package]] -name = "unicase" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" - [[package]] name = "unicode-bidi" version = "0.3.18" @@ -9270,9 +8791,9 @@ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" dependencies = [ "tinyvec", ] @@ -9370,7 +8891,6 @@ dependencies = [ "idna", "percent-encoding", "serde", - "serde_derive", ] [[package]] @@ -9381,44 +8901,49 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utils" -version = "0.16.27" +version = "1.0.0" dependencies = [ "ahash", "arcstr", "aws-lc-rs", - "base64 0.23.1", - "chrono", "compact_str", + "encodify", "farmhash", "form_urlencoded", "futures", + "hashify", "idna", + "jiff", + "libc", "mail-auth", + "memchr", "nohash-hasher", "privdrop", "psl", "quick_cache", - "reqwest 0.13.5", + "reqwest", "rkyv", "rustls", "rustls-pki-types", "rustls-platform-verifier", "serde", + "simdutf8", "tokio", "tokio-rustls", "trc", + "url", + "windows-sys 0.61.2", "xxhash-rust", ] [[package]] name = "uuid" -version = "1.28.0" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" dependencies = [ "getrandom 0.4.3", "js-sys", - "serde_core", "sha1_smol", "wasm-bindgen", ] @@ -9477,12 +9002,6 @@ version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" -[[package]] -name = "waker-fn" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" - [[package]] name = "walkdir" version = "2.5.0" @@ -9502,12 +9021,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -9597,19 +9110,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -9645,9 +9145,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] @@ -9658,14 +9158,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -9729,41 +9229,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core 0.61.2", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -9772,20 +9237,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -9810,46 +9264,21 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - [[package]] name = "windows-registry" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -9858,16 +9287,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -9876,7 +9296,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -9894,7 +9314,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -9913,15 +9333,6 @@ dependencies = [ "windows_x86_64_msvc", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -9985,6 +9396,18 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff 0.14.0", + "group 0.14.0", + "hybrid-array 0.4.15", + "primefield", +] + [[package]] name = "writeable" version = "0.6.4" @@ -10029,20 +9452,10 @@ dependencies = [ "oid-registry", "ring", "rusticata-macros", - "thiserror 2.0.21", + "thiserror", "time", ] -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "xml" version = "1.4.0" @@ -10082,9 +9495,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -10124,7 +9537,7 @@ dependencies = [ "nonempty-collections", "once_cell", "petgraph", - "phf 0.13.1", + "phf", "rand 0.8.8", "rustc_version", "serde", @@ -10293,7 +9706,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", "x509-parser", "zenoh-buffers", "zenoh-codec", @@ -10377,7 +9790,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tracing", - "webpki-roots 1.0.9", + "webpki-roots 1.0.11", "x509-parser", "zenoh-config", "zenoh-core", @@ -10487,7 +9900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d57ed2156b61df6e877b03fb13dd72227d3d286a7fa689120109bad289ddf12b" dependencies = [ "arc-swap", - "event-listener 5.4.2", + "event-listener", "futures", "tokio", "zenoh-buffers", @@ -10667,9 +10080,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "serde", "yoke", @@ -10679,9 +10092,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", @@ -10690,13 +10103,13 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704" dependencies = [ "aes 0.9.3", "bzip2", - "constant_time_eq", + "constant_time_eq 0.6.1", "crc32fast", "deflate64", "flate2", @@ -10705,14 +10118,16 @@ dependencies = [ "indexmap 2.14.2", "lzma-rust2", "memchr", + "nix 0.31.3", "pbkdf2", "ppmd-rust", "sha1 0.11.0", "time", "typed-path", + "windows-sys 0.61.2", "zeroize", "zopfli", - "zstd", + "zstd 0.14.3", ] [[package]] @@ -10745,7 +10160,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", ] [[package]] @@ -10758,10 +10182,19 @@ dependencies = [ ] [[package]] -name = "zstd-sys" -version = "2.1.1+zstd.1.5.7" +name = "zstd-safe" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index c1218df3..1ec30576 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,55 @@ members = [ "crates/utils", "crates/common", "crates/trc", - "crates/migration", - "tests", + "crates/text-extract", + "crates/smtp-proto", + "crates/smtp-report", + "crates/calcard", + "crates/sieve", + "crates/jmap-tools", + "crates/s3", ] +default-members = [ + "crates/main", + "crates/types", + "crates/http", + "crates/http-proto", + "crates/jmap", + "crates/jmap-proto", + "crates/email", + "crates/imap", + "crates/imap-proto", + "crates/smtp", + "crates/managesieve", + "crates/pop3", + "crates/dav-proto", + "crates/scim-proto", + "crates/scim", + "crates/dav", + "crates/groupware", + "crates/spam-filter", + "crates/nlp", + "crates/store", + "crates/coordinator", + "crates/directory", + "crates/registry", + "crates/services", + "crates/utils", + "crates/common", + "crates/trc", + "crates/text-extract", + "crates/smtp-proto", + "crates/smtp-report", + "crates/calcard", + "crates/sieve", + "crates/jmap-tools", +] + +[workspace.dependencies] +rkyv = { version = "0.8.18", features = ["little_endian", "unaligned"] } + +[workspace.lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(stalwart_tests)'] } [workspace.lints.clippy] result_unit_err = "allow" @@ -46,6 +92,12 @@ debug-assertions = true overflow-checks = false rpath = false +[profile.dev.package.argon2] +opt-level = 3 + +[profile.dev.package.blake2] +opt-level = 3 + [profile.release] opt-level = 3 debug = false diff --git a/Dockerfile b/Dockerfile index 11ead516..1f78166a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,6 @@ VOLUME ["/etc/stalwart", "/var/lib/stalwart"] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENV STALWART_HEALTHCHECK_URL=https://127.0.0.1:443/healthz/live HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ - CMD curl -fsSk -H "X-Forwarded-For: 127.0.0.1" "$STALWART_HEALTHCHECK_URL" || curl -fsS -H "X-Forwarded-For: 127.0.0.1" http://127.0.0.1:8080/healthz/live || exit 1 + CMD curl -fsSk "$STALWART_HEALTHCHECK_URL" || curl -fsS http://127.0.0.1:8080/healthz/live || exit 1 ENTRYPOINT ["/usr/local/bin/stalwart"] CMD ["--config", "/etc/stalwart/config.json"] diff --git a/Dockerfile.build b/Dockerfile.build index 2810a713..f5ff43b7 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -79,6 +79,22 @@ RUN \ echo "export ${BUILD_ENV}" >> /env-cargo; \ echo "Setting up ${BUILD_ENV}"; \ fi && \ + if [[ "${TARGET}" == arm*-*eabihf ]]; then \ + apt-get update && \ + apt-get install -yq --no-install-recommends cmake && \ + cargo binstall --no-confirm bindgen-cli@0.73.2 && \ + case "${TARGET}" in \ + *-gnueabihf) zig_target="arm-linux-gnueabihf.${GLIBC_VERSION}" ;; \ + *-musleabihf) zig_target="arm-linux-musleabihf" ;; \ + esac && \ + includes="$(zig cc -target "${zig_target}" -E -v -x c /dev/null -o /dev/null 2>&1 \ + | sed -n '/<...> search starts here/,/End of search/p' \ + | grep '^ ' | grep -v '/lib/include$' \ + | sed 's/^ */-isystem /' | tr '\n' ' ')" && \ + [ -n "${includes}" ] && \ + target_var="${TARGET//-/_}" && \ + echo "export BINDGEN_EXTRA_CLANG_ARGS_${target_var}='${includes}'" >> /env-cargo; \ + fi && \ if [[ "${TARGET}" == *gnu ]]; then \ base_arch="${TARGET%%-*}"; \ case "$base_arch" in \ @@ -182,7 +198,7 @@ VOLUME ["/etc/stalwart", "/var/lib/stalwart"] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENV STALWART_HEALTHCHECK_URL=https://127.0.0.1:443/healthz/live HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ - CMD curl -fsSk -H "X-Forwarded-For: 127.0.0.1" "$STALWART_HEALTHCHECK_URL" || curl -fsS -H "X-Forwarded-For: 127.0.0.1" http://127.0.0.1:8080/healthz/live || exit 1 + CMD curl -fsSk "$STALWART_HEALTHCHECK_URL" || curl -fsS http://127.0.0.1:8080/healthz/live || exit 1 ENTRYPOINT ["/usr/local/bin/stalwart"] CMD ["--config", "/etc/stalwart/config.json"] @@ -204,6 +220,6 @@ VOLUME ["/etc/stalwart", "/var/lib/stalwart"] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENV STALWART_HEALTHCHECK_URL=https://127.0.0.1:443/healthz/live HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ - CMD curl -fsSk -H "X-Forwarded-For: 127.0.0.1" "$STALWART_HEALTHCHECK_URL" || curl -fsS -H "X-Forwarded-For: 127.0.0.1" http://127.0.0.1:8080/healthz/live || exit 1 + CMD curl -fsSk "$STALWART_HEALTHCHECK_URL" || curl -fsS http://127.0.0.1:8080/healthz/live || exit 1 ENTRYPOINT ["/usr/local/bin/stalwart"] CMD ["--config", "/etc/stalwart/config.json"] diff --git a/Dockerfile.fdb b/Dockerfile.fdb index a78fc93d..1928f0bf 100644 --- a/Dockerfile.fdb +++ b/Dockerfile.fdb @@ -42,7 +42,6 @@ COPY Cargo.toml . COPY Cargo.lock . COPY crates/ crates/ COPY resources/ resources/ -COPY tests/ tests/ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder @@ -52,7 +51,6 @@ COPY Cargo.toml . COPY Cargo.lock . COPY crates/ crates/ COPY resources/ resources/ -COPY tests/ tests/ RUN cargo build -p stalwart --no-default-features --features "foundationdb s3 redis azure nats" --release FROM debian:trixie-slim AS runtime @@ -75,6 +73,6 @@ VOLUME ["/etc/stalwart", "/var/lib/stalwart"] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENV STALWART_HEALTHCHECK_URL=https://127.0.0.1:443/healthz/live HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ - CMD curl -fsSk -H "X-Forwarded-For: 127.0.0.1" "$STALWART_HEALTHCHECK_URL" || curl -fsS -H "X-Forwarded-For: 127.0.0.1" http://127.0.0.1:8080/healthz/live || exit 1 + CMD curl -fsSk "$STALWART_HEALTHCHECK_URL" || curl -fsS http://127.0.0.1:8080/healthz/live || exit 1 ENTRYPOINT ["/usr/local/bin/stalwart"] CMD ["--config", "/etc/stalwart/config.json"] diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 00000000..16fe87b0 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 00000000..31aa7938 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/LICENSES/Unlicense.txt b/LICENSES/Unlicense.txt new file mode 100644 index 00000000..68a49daa --- /dev/null +++ b/LICENSES/Unlicense.txt @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/README.md b/README.md index 1fe48426..cb85b274 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ Key features: - **Email** server with complete protocol support: - JMAP: * [JMAP for Mail](https://datatracker.ietf.org/doc/html/rfc8621) server. - * [JMAP for Sieve Scripts](https://www.ietf.org/archive/id/draft-ietf-jmap-sieve-22.html). + * [JMAP for Sieve Scripts](https://www.rfc-editor.org/rfc/rfc9661.html). + * [JMAP Object Metadata](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-metadata-04) for e-mails, mailboxes, calendars, contacts, files and Sieve scripts. * [WebSocket](https://datatracker.ietf.org/doc/html/rfc8887), [Blob Management](https://www.rfc-editor.org/rfc/rfc9404.html) and [Quotas](https://www.rfc-editor.org/rfc/rfc9425.html) extensions. - IMAP: * [IMAP4rev2](https://datatracker.ietf.org/doc/html/rfc9051) and [IMAP4rev1](https://datatracker.ietf.org/doc/html/rfc3501) server. @@ -47,7 +48,7 @@ Key features: - [POP3](https://datatracker.ietf.org/doc/html/rfc1939) server. - [STLS](https://datatracker.ietf.org/doc/html/rfc2595) and [SASL](https://datatracker.ietf.org/doc/html/rfc5034) support as well as other [extensions](https://datatracker.ietf.org/doc/html/rfc2449). - SMTP: - * SMTP server with built-in [DMARC](https://datatracker.ietf.org/doc/html/rfc7489), [DKIMv2](https://datatracker.ietf.org/doc/draft-ietf-dkim-dkim2-spec/), [DKIMv1](https://datatracker.ietf.org/doc/html/rfc6376), [SPF](https://datatracker.ietf.org/doc/html/rfc7208) and [ARC](https://datatracker.ietf.org/doc/html/rfc8617) support for message authentication. + * SMTP server with built-in [DMARC](https://datatracker.ietf.org/doc/html/rfc7489), [DKIMv2](https://datatracker.ietf.org/doc/draft-ietf-dkim-dkim2-spec/) (including post-quantum signatures), [DKIMv1](https://datatracker.ietf.org/doc/html/rfc6376), [SPF](https://datatracker.ietf.org/doc/html/rfc7208) and [ARC](https://datatracker.ietf.org/doc/html/rfc8617) support for message authentication. * Strong transport security through [DANE](https://datatracker.ietf.org/doc/html/rfc6698), [MTA-STS](https://datatracker.ietf.org/doc/html/rfc8461) and [SMTP TLS](https://datatracker.ietf.org/doc/html/rfc8460) reporting. * Automated DKIM key rotation and management. * Inbound throttling and filtering with granular configuration rules, sieve scripting, MTA hooks and milter integration. @@ -56,19 +57,19 @@ Key features: - **Collaboration** server: - Calendaring and scheduling: - [CalDAV](https://datatracker.ietf.org/doc/html/rfc4791) and [CalDAV Scheduling](https://datatracker.ietf.org/doc/html/rfc6638) support. - - [JMAP for Calendars](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-calendars-24) support. + - [JMAP for Calendars](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-calendars-32) support. - Contact management: - [CardDAV](https://datatracker.ietf.org/doc/html/rfc6352) support. - [JMAP for Contacts](https://datatracker.ietf.org/doc/html/rfc9610) support. - File storage: - [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) support. - - [JMAP for File Storage](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-filenode-03) support. + - [JMAP for File Storage](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-filenode-17) support. - Sharing with fine-grained access controls: - [WebDAV ACL](https://datatracker.ietf.org/doc/html/rfc3744) support. - [JMAP Sharing](https://datatracker.ietf.org/doc/html/rfc9670) support. - **Spam** and **Phishing** built-in filter: - Comprehensive set of filtering **rules** on par with popular solutions. - - LLM-driven spam filtering and message analysis. + - AI spam classifier with support for OpenAI-compatible, Anthropic, Google Gemini and Ollama models. - Statistical **spam classifier** with collaborative filtering, automatic training capabilities and address book integration. - DNS Blocklists (**DNSBLs**) checking of IP addresses, domains, and hashes. - Collaborative digest-based spam filtering with **Pyzor**. @@ -80,6 +81,7 @@ Key features: - **Flexible**: - Pluggable storage backends with **RocksDB**, **FoundationDB**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Azure** and **Redis** support. - Full-text search available in 17 languages using the built-in search engine or via **Meilisearch**, **ElasticSearch**, **OpenSearch**, **PostgreSQL** or **mySQL** backends. + - Text extraction and indexing of **PDF**, **Microsoft Office**, **OpenDocument**, **EPUB** and **RTF** documents. - Sieve scripting language with support for all [registered extensions](https://www.iana.org/assignments/sieve-extensions/sieve-extensions.xhtml). - Email aliases, mailing lists, subaddressing and catch-all addresses support. - Automated DNS management. @@ -90,6 +92,7 @@ Key features: - Encryption at rest with **S/MIME** or **OpenPGP**. - Automatic TLS certificate provisioning with [ACME](https://datatracker.ietf.org/doc/html/rfc8555) using `TLS-ALPN-01`, `DNS-01`, `DNS-PERSIST-01` or `HTTP-01` challenges. - Automated blocking of IP addresses that attack, abuse or scan the server for exploits. + - SSRF protection for outbound requests. - Rate limiting. - Security audited (read the [report](https://stalw.art/blog/security-audit)). - Memory safe (thanks to Rust). @@ -99,9 +102,11 @@ Key features: - Peer-to-peer cluster coordination or with **Kafka**, **Redpanda**, **NATS** or **Redis**. - **Kubernetes**, **Apache Mesos** and **Docker Swarm** support for automated scaling and container orchestration. - Read replicas, sharded blob storage and in-memory data stores for high performance and low latency. + - Tiered caching with snapshots that can be swapped to disk, Redis or the blob store. - **Authentication and Authorization**: - **OpenID Connect** authentication. - OAuth 2.0 authorization with [authorization code](https://www.rfc-editor.org/rfc/rfc8628) and [device authorization](https://www.rfc-editor.org/rfc/rfc8628) flows. + - OAuth scopes, connected applications management and [Client ID Metadata Documents](https://datatracker.ietf.org/doc/draft-ietf-oauth-client-id-metadata-document/). - **LDAP**, **OIDC**, **SQL** or built-in authentication backend support. - System for Cross-domain Identity Management ([SCIM](https://www.rfc-editor.org/info/rfc7643/)) v2 for automated provisioning. - Two-factor authentication with Time-based One-Time Passwords (`2FA-TOTP`) @@ -129,7 +134,7 @@ Key features: ## Presentation -**Want a deeper dive?** Need to explain to your boss why Stalwart is the perfect fit? Whether you're evaluating options, making a case to your team, or simply curious about how it all works under the hood, these slides walk you through the key features, architecture, and benefits of Stalwart. Browse the [slides](https://stalw.art/slides) to see what makes it stand out. +Whether you are evaluating options, presenting Stalwart to your team or your boss, or just curious about how it works under the hood, the [slides](https://stalw.art/slides) walk you through its key features, architecture and benefits. ## Get Started @@ -141,27 +146,40 @@ Install Stalwart on your server by following the instructions for your platform: All documentation is available at [stalw.art/docs](https://stalw.art/docs/install/get-started). -## Support +## Where do I get support? -If you are having problems running Stalwart, found a bug, or just have a question, please head to the [Stalwart Support Portal](https://support.stalw.art) at [support.stalw.art](https://support.stalw.art). -Additionally, you may purchase an [Enterprise License](https://stalw.art/enterprise) to obtain priority support from Stalwart Labs LLC, including response-time commitments and a private Priority Support area on the portal. +If you run into a problem, find a bug or have a question, head to the [Stalwart Support Portal](https://support.stalw.art). It is free for everyone, and our team answers. -## Contributing +Organizations that need more can purchase an [Enterprise license](https://stalw.art/enterprise) for priority support from Stalwart Labs LLC, with response-time commitments and a private Priority Support area on the portal. -We welcome contributions, but to keep the project maintainable there are a few things to know before opening a pull request. Because of the high volume of low-quality, AI-generated submissions, pull requests are limited to a list of vouched contributors; to be added, post at [support.stalw.art](https://support.stalw.art) describing the change you would like to submit, together with a link to the proposed change. At this stage only bug fixes and translations are accepted, and new features are not, unless they involve just a few lines of code. -For the full guidelines, please read [CONTRIBUTING.md](CONTRIBUTING.md). +## Why isn't everything AGPL? -## Roadmap +About 85% of Stalwart's features are free and open source under the AGPL-3.0. The rest make up the [Enterprise edition](https://stalw.art/enterprise/), and are mostly features that matter to large organizations and hosting providers running Stalwart at scale, such as multi-tenancy, SCIM provisioning, sharded storage and live telemetry. The full breakdown is on the [comparison page](https://stalw.art/compare/). -Stalwart has reached an exciting point in its journey, it’s now **feature complete**. All the core functionality and open standard email and collaboration protocols that we set out to support are in place. In other words, Stalwart already does everything you’d expect from a modern, standards-compliant mail and collaboration platform. +Stalwart is built by a full-time professional team, not as a side project, and open source sponsorship covers only a small fraction of what that costs. Enterprise subscriptions pay for the work that every Stalwart user benefits from: -The next major milestone is all about refinement: finalizing the database schema and focusing on performance optimizations to ensure everything runs as efficiently and reliably as possible. Once that’s done, we’ll be ready to roll out version **1.0**. +- The engineers who maintain and extend the Community edition, with a new release every week. +- Security fixes within 24 hours of a report, and an independent security audit every time a large part of Stalwart is rewritten. +- Free community support at [support.stalw.art](https://support.stalw.art). +- Stress tests and benchmarks on dozens of cloud servers at a time across many regions. +- Work on email standards at the IETF, where we are the authors of multiple Internet-Drafts in progress. -Of course, development doesn’t stop there. The community has contributed hundreds of great ideas for improvements and new features, everything from subtle usability tweaks to entirely new integrations. You can see the full list of proposals over on our [GitHub issues](https://github.com/stalwartlabs/stalwart/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc+label%3Aenhancement). If there’s something you’d like to see prioritized, just give it a thumbs up as we plan to implement enhancements based on the community’s votes. +The Community edition is complete and open source, and it will stay that way: -## Sponsorship +- Stalwart remains licensed under the AGPL-3.0. +- No existing open source feature will ever be moved to the Enterprise edition. +- Most new features go to the Community edition. -Your support is crucial in helping us continue to improve the project, add new features, and maintain the highest level of quality. By [becoming a sponsor](https://opencollective.com/stalwart), you help fund the development and future of Stalwart. As a thank-you, sponsors who contribute $5 per month or more will automatically receive a [Enterprise edition](https://stalw.art/enterprise/) license. And, sponsors who contribute $30 per month or more, also have access to [Premium Support](https://stalw.art/support) from Stalwart Labs. +If you find Stalwart useful, there are two ways to help keep it free. If your organization needs Enterprise features, [subscribe to the Enterprise edition](https://stalw.art/pricing/). Otherwise, you can [become a sponsor on OpenCollective](https://opencollective.com/stalwart): sponsors who contribute $5 per month or more automatically receive an Enterprise license, and those who contribute $30 per month or more also get access to [Premium Support](https://stalw.art/support) from Stalwart Labs. Thank you! + +## How can I contribute? + +Contributions are welcome, with a few rules that keep the project maintainable: + +- Because of the high volume of low-quality, AI-generated submissions, only vouched contributors can open pull requests. To be added to the list, post at [support.stalw.art](https://support.stalw.art) describing the change you would like to submit, with a link to it. +- For now, only bug fixes and translations are accepted. New features are not, unless they are just a few lines of code. + +The full guidelines are in [CONTRIBUTING.md](CONTRIBUTING.md). You can also help without writing code: report bugs, answer questions on the support portal, or tell others about Stalwart. ## Funding @@ -170,8 +188,6 @@ Part of the development of this project was funded through: - [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. - [NGI Zero Core](https://nlnet.nl/NGI0/), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101092990. -If you find the project useful you can help by [becoming a sponsor](https://opencollective.com/stalwart). Thank you! - ## License This project is dual-licensed under the **GNU Affero General Public License v3.0** (AGPL-3.0; as published by the Free Software Foundation) and the **Stalwart Enterprise License v2 (SELv2)**: diff --git a/SECURITY.md b/SECURITY.md index 99f41f3b..5198a5f5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,8 @@ We provide security updates for the following versions of Stalwart: | Version | Supported | End of Support | | ------- | ------------------ | -------------- | -| 0.16.x | :white_check_mark: | TBD | +| 1.0.x | :white_check_mark: | TBD | +| 0.16.x | :white_check_mark: | 2027-06-01 | | 0.15.x | :white_check_mark: | 2026-12-01 | | < 0.14 | :x: | Ended | diff --git a/crates/calcard/Cargo.toml b/crates/calcard/Cargo.toml new file mode 100644 index 00000000..3121b650 --- /dev/null +++ b/crates/calcard/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "calcard" +description = "iCalendar/JSCalendar and vCard/JSContact parsing, building and conversion library" +version = "0.4.0" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[dependencies] +jmap-tools = { path = "../jmap-tools", optional = true } +ahash = "0.8.12" +jiff = "0.2.38" +hashify = "0.3" +memchr = "2.8" +encodify = "1.0" +mail-parser = "1.0" +simdutf8 = "0.1.5" +smallvec = { version = "1.16", features = ["union"] } +uuid = { version = "1.29", default-features = false, optional = true } +sha1 = { version = "0.11", default-features = false, optional = true } +rkyv = { version = "0.8", optional = true, features = ["smallvec-1"] } +serde = { version = "1.0", features = ["derive"], optional = true } +serde_json = { version = "1.0", optional = true } + +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] +jiff = { version = "0.2.38", features = ["js"] } + +[features] +default = ["jmap", "tzdb-bundle"] +rkyv = ["dep:rkyv"] +serde = ["dep:serde", "smallvec/serde"] +jmap = ["jmap-tools", "uuid", "sha1", "serde", "serde_json"] +tzdb-bundle = ["jiff/tzdb-bundle-always"] + +[dev-dependencies] +smallvec = { version = "1.16", features = ["serde"] } +insta = "1.49" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +uuid = { version = "1.29", default-features = false, features = ["v5"] } + +[lib] +doctest = false + +[lints] +workspace = true diff --git a/crates/calcard/src/common/archive.rs b/crates/calcard/src/common/archive.rs new file mode 100644 index 00000000..5f0f25c0 --- /dev/null +++ b/crates/calcard/src/common/archive.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use rkyv::{ + Archive, Archived, Deserialize, Place, Serialize, + rancor::Fallible, + ser::{Allocator, Writer}, + vec::{ArchivedVec, VecResolver}, + with::{ArchiveWith, DeserializeWith, SerializeWith}, +}; +use smallvec::{Array, SmallVec}; + +pub struct Unboxed; + +impl ArchiveWith> for Unboxed { + type Archived = T::Archived; + type Resolver = T::Resolver; + + fn resolve_with(field: &Box, resolver: Self::Resolver, out: Place) { + T::resolve(field, resolver, out); + } +} + +impl, S: Fallible + ?Sized> SerializeWith, S> for Unboxed { + fn serialize_with(field: &Box, serializer: &mut S) -> Result { + T::serialize(field, serializer) + } +} + +impl DeserializeWith, D> for Unboxed +where + T: Archive, + T::Archived: Deserialize, + D: Fallible + ?Sized, +{ + fn deserialize_with(field: &T::Archived, deserializer: &mut D) -> Result, D::Error> { + field.deserialize(deserializer).map(Box::new) + } +} + +pub struct InlineVec; + +impl ArchiveWith> for InlineVec +where + A: Array, + A::Item: Archive, +{ + type Archived = ArchivedVec>; + type Resolver = VecResolver; + + fn resolve_with(field: &SmallVec, resolver: Self::Resolver, out: Place) { + ArchivedVec::resolve_from_slice(field.as_slice(), resolver, out); + } +} + +impl SerializeWith, S> for InlineVec +where + A: Array, + A::Item: Serialize, + S: Fallible + Allocator + Writer + ?Sized, +{ + fn serialize_with(field: &SmallVec, serializer: &mut S) -> Result { + ArchivedVec::serialize_from_slice(field.as_slice(), serializer) + } +} + +impl DeserializeWith>, SmallVec, D> for InlineVec +where + A: Array, + A::Item: Archive, + Archived: Deserialize, + D: Fallible + ?Sized, +{ + fn deserialize_with( + field: &ArchivedVec>, + deserializer: &mut D, + ) -> Result, D::Error> { + let mut items = SmallVec::with_capacity(field.len()); + for item in field.iter() { + items.push(item.deserialize(deserializer)?); + } + Ok(items) + } +} + diff --git a/crates/calcard/src/common/blob.rs b/crates/calcard/src/common/blob.rs new file mode 100644 index 00000000..7d769a65 --- /dev/null +++ b/crates/calcard/src/common/blob.rs @@ -0,0 +1,361 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::export::EmbeddedBudget; +use ahash::AHashMap; +use std::hash::Hash; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum BlobIdOutcome { + Generated(B), + Declined(Vec), + Failed, +} + +pub trait BlobIdGenerator { + fn blob_id(&mut self, data: Vec, content_type: Option<&str>) -> BlobIdOutcome; + + fn is_enabled(&self) -> bool { + true + } +} + +pub trait BlobResolver { + fn resolve(&mut self, blob_id: &B) -> Option>; + + fn is_enabled(&self) -> bool { + true + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub struct NoBlobIds; + +#[derive(Debug, Clone, Copy)] +pub struct BlobIdFn(pub F); + +#[derive(Debug, Clone, Copy)] +pub struct BlobResolverFn(pub F); + +impl BlobIdGenerator for NoBlobIds { + fn blob_id(&mut self, data: Vec, _: Option<&str>) -> BlobIdOutcome { + BlobIdOutcome::Declined(data) + } + + fn is_enabled(&self) -> bool { + false + } +} + +impl BlobResolver for NoBlobIds { + fn resolve(&mut self, _: &B) -> Option> { + None + } + + fn is_enabled(&self) -> bool { + false + } +} + +impl BlobIdGenerator for BlobIdFn +where + F: FnMut(&[u8]) -> Option, +{ + fn blob_id(&mut self, data: Vec, _: Option<&str>) -> BlobIdOutcome { + match (self.0)(&data) { + Some(blob_id) => BlobIdOutcome::Generated(blob_id), + None => BlobIdOutcome::Declined(data), + } + } +} + +impl BlobResolver for BlobResolverFn +where + F: FnMut(&B) -> Option>, +{ + fn resolve(&mut self, blob_id: &B) -> Option> { + (self.0)(blob_id) + } +} + +impl + ?Sized> BlobIdGenerator for &mut T { + fn blob_id(&mut self, data: Vec, content_type: Option<&str>) -> BlobIdOutcome { + (**self).blob_id(data, content_type) + } + + fn is_enabled(&self) -> bool { + (**self).is_enabled() + } +} + +impl + ?Sized> BlobResolver for &mut T { + fn resolve(&mut self, blob_id: &B) -> Option> { + (**self).resolve(blob_id) + } + + fn is_enabled(&self) -> bool { + (**self).is_enabled() + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BlobOptions { + handler: Option, + max_embedded_size: Option, +} + +impl Default for BlobOptions { + fn default() -> Self { + Self { + handler: None, + max_embedded_size: None, + } + } +} + +impl BlobOptions { + pub(crate) fn with_handler(self, handler: U) -> BlobOptions { + BlobOptions { + handler: Some(handler), + max_embedded_size: self.max_embedded_size, + } + } + + pub(crate) fn with_max_embedded_size(mut self, max_embedded_size: usize) -> Self { + self.max_embedded_size = Some(max_embedded_size); + self + } + + pub(crate) fn budget(&self) -> EmbeddedBudget { + EmbeddedBudget::new(self.max_embedded_size) + } + + pub(crate) fn blob_ids(&mut self) -> Option> + where + T: BlobIdGenerator, + { + self.handler + .as_mut() + .filter(|generator| BlobIdGenerator::::is_enabled(*generator)) + .map(|generator| BlobIds::new(generator as &mut dyn BlobIdGenerator)) + } + + pub(crate) fn resolved_blobs(&mut self) -> Option> + where + T: BlobResolver, + { + self.handler + .as_mut() + .filter(|resolver| BlobResolver::::is_enabled(*resolver)) + .map(|resolver| ResolvedBlobs::new(resolver as &mut dyn BlobResolver)) + } +} + +pub(crate) struct BlobIds<'a, B> { + generator: &'a mut dyn BlobIdGenerator, + expected_sizes: Option>, + generated: AHashMap<(usize, u64), Vec>>, + has_failed: bool, +} + +#[derive(Default)] +struct ExpectedSize { + total: usize, + pending: usize, +} + +struct GeneratedBinary { + blob_id: B, + data: Vec, +} + +pub(crate) struct GeneratedBlobId { + pub blob_id: B, + pub size: usize, +} + +impl<'a, B: Clone> BlobIds<'a, B> { + pub(crate) fn new(generator: &'a mut dyn BlobIdGenerator) -> Self { + Self { + generator, + expected_sizes: None, + generated: AHashMap::new(), + has_failed: false, + } + } + + pub(crate) fn has_failed(&self) -> bool { + self.has_failed + } + + pub(crate) fn expect_sizes(&mut self, sizes: impl IntoIterator) { + let expected_sizes = self.expected_sizes.get_or_insert_with(AHashMap::new); + for size in sizes { + let expected = expected_sizes.entry(size).or_default(); + expected.total += 1; + expected.pending += 1; + } + } + + pub(crate) fn blob_id( + &mut self, + data: Vec, + content_type: Option<&str>, + ) -> Result, Vec> { + if data.is_empty() { + return Err(data); + } + let size = data.len(); + let (is_repeated, expects_more) = match &mut self.expected_sizes { + Some(expected_sizes) => { + expected_sizes + .get_mut(&size) + .map_or((false, false), |expected| { + expected.pending = expected.pending.saturating_sub(1); + (expected.total > 1, expected.pending > 0) + }) + } + None => (true, true), + }; + if !is_repeated { + return self.generate(data, content_type); + } + + let key = (size, self.generated.hasher().hash_one(&data)); + if let Some(generated) = self + .generated + .get(&key) + .and_then(|generated| generated.iter().find(|generated| generated.data == data)) + { + return Ok(GeneratedBlobId { + blob_id: generated.blob_id.clone(), + size, + }); + } + if !expects_more { + return self.generate(data, content_type); + } + let blob_id = self.generate(data.clone(), content_type)?.blob_id; + self.generated + .entry(key) + .or_default() + .push(GeneratedBinary { + blob_id: blob_id.clone(), + data, + }); + Ok(GeneratedBlobId { blob_id, size }) + } + + fn generate( + &mut self, + data: Vec, + content_type: Option<&str>, + ) -> Result, Vec> { + let size = data.len(); + match self.generator.blob_id(data, content_type) { + BlobIdOutcome::Generated(blob_id) => Ok(GeneratedBlobId { blob_id, size }), + BlobIdOutcome::Declined(data) => Err(data), + BlobIdOutcome::Failed => { + self.has_failed = true; + Err(Vec::new()) + } + } + } +} + +pub(crate) struct ResolvedBlobs<'a, B> { + resolver: &'a mut dyn BlobResolver, + uses: AHashMap, + resolved: AHashMap>>, +} + +impl<'a, B: Clone + Eq + Hash> ResolvedBlobs<'a, B> { + pub(crate) fn new(resolver: &'a mut dyn BlobResolver) -> Self { + Self { + resolver, + uses: AHashMap::new(), + resolved: AHashMap::new(), + } + } + + pub(crate) fn add_uses<'x>(&mut self, blob_ids: impl Iterator) + where + B: 'x, + { + for blob_id in blob_ids { + *self.uses.entry(blob_id.clone()).or_default() += 1; + } + } + + pub(crate) fn resolve(&mut self, blob_id: &B) -> Option> { + let remaining = self.uses.get_mut(blob_id).map_or(0, |uses| { + *uses = uses.saturating_sub(1); + *uses + }); + let data = match self.resolved.remove(blob_id) { + Some(data) => data, + None => self + .resolver + .resolve(blob_id) + .filter(|data| !data.is_empty()), + }; + if remaining > 0 { + self.resolved.insert(blob_id.clone(), data.clone()); + } + data + } +} + +pub(crate) fn sniff_media_type(data: &[u8]) -> &'static str { + match data { + [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, ..] => "image/png", + [0xFF, 0xD8, 0xFF, ..] => "image/jpeg", + [b'G', b'I', b'F', b'8', b'7' | b'9', b'a', ..] => "image/gif", + [ + b'R', + b'I', + b'F', + b'F', + _, + _, + _, + _, + b'W', + b'E', + b'B', + b'P', + .., + ] => "image/webp", + [ + b'R', + b'I', + b'F', + b'F', + _, + _, + _, + _, + b'W', + b'A', + b'V', + b'E', + .., + ] => "audio/wav", + [b'O', b'g', b'g', b'S', ..] => "audio/ogg", + [b'I', b'D', b'3', ..] | [0xFF, 0xFB | 0xF3 | 0xF2, ..] => "audio/mpeg", + _ if is_svg(data) => "image/svg+xml", + _ => "application/octet-stream", + } +} + +fn is_svg(data: &[u8]) -> bool { + let head = data.get(..data.len().min(512)).unwrap_or_default(); + let text = head.trim_ascii_start(); + (text.starts_with(b" + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::{Encoding, tokenizer::StopChar}; +use encodify::{ + base64, + qp::{self, QuotedPrintable}, +}; +use mail_parser::Charset; +use std::borrow::Cow; + +const QUOTED_PRINTABLE_CHARSET: &str = "iso-8859-1"; +const QUOTED_PRINTABLE: QuotedPrintable = qp::BODY.strict(); + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Decoded { + Binary(Vec), + Text(String), + Undecodable, +} + +pub(crate) struct ValueDecoder<'x> { + pub(crate) encoding: Encoding, + pub(crate) charset: Option<&'x str>, + pub(crate) binary: bool, +} + +impl ValueDecoder<'_> { + #[inline(never)] + pub(crate) fn decode( + &self, + input: &[u8], + stop_char: StopChar, + payload: Option>, + ) -> Decoded { + let Some(bytes) = payload + .map(Cow::Owned) + .or_else(|| self.encoding.decode_value(input, stop_char)) + else { + return Decoded::Undecodable; + }; + if self.binary { + return Decoded::Binary(bytes.into_owned()); + } + let default_charset = match self.encoding { + Encoding::QuotedPrintable => Some(QUOTED_PRINTABLE_CHARSET), + Encoding::Base64 => None, + }; + match self + .charset + .or(default_charset) + .and_then(|charset| Charset::from_label(charset.as_bytes())) + .filter(|charset| *charset != Charset::Utf8) + { + Some(charset) => Decoded::Text(match bytes { + Cow::Borrowed(bytes) => charset.decode(bytes).into_owned(), + Cow::Owned(bytes) => charset.decode_owned(bytes), + }), + None => match String::from_utf8(bytes.into_owned()) { + Ok(text) => Decoded::Text(text), + Err(err) => Decoded::Binary(err.into_bytes()), + }, + } + } +} + +impl Encoding { + pub(crate) fn decode(self, input: &[u8]) -> Option> { + match self { + Encoding::Base64 => base64::LENIENT.decode(input).ok().map(Cow::Owned), + Encoding::QuotedPrintable => QUOTED_PRINTABLE.decode(input).ok(), + } + } + + fn decode_value(self, input: &[u8], stop_char: StopChar) -> Option> { + match (self, stop_char) { + (Encoding::QuotedPrintable, StopChar::Lf) | (Encoding::Base64, _) => self.decode(input), + (Encoding::QuotedPrintable, _) => Self::decode_quoted_printable_component(input), + } + } + + fn decode_quoted_printable_component(input: &[u8]) -> Option> { + let mut text = input; + while let [head @ .., b' ' | b'\t'] = text { + text = head; + } + match input.get(text.len()..) { + Some(blanks @ [_, ..]) => { + let mut bytes = Vec::new(); + QUOTED_PRINTABLE.decode_append(text, &mut bytes).ok()?; + bytes.extend_from_slice(blanks); + Some(Cow::Owned(bytes)) + } + _ => QUOTED_PRINTABLE.decode(input).ok(), + } + } +} + diff --git a/crates/calcard/src/common/elements.rs b/crates/calcard/src/common/elements.rs new file mode 100644 index 00000000..bd2e31d1 --- /dev/null +++ b/crates/calcard/src/common/elements.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_tools::{Element, Property, Value}; + +pub(crate) struct Elements<'a, 'x, P: Property, E: Element> { + stack: Vec<&'a Value<'x, P, E>>, +} + +impl<'a, 'x, P: Property, E: Element> Elements<'a, 'x, P, E> { + pub(crate) fn new(value: &'a Value<'x, P, E>) -> Self { + let mut stack = Vec::with_capacity(16); + stack.push(value); + Self { stack } + } + + pub(crate) fn from_values(values: impl Iterator>) -> Self { + let mut stack = Vec::with_capacity(16); + stack.extend(values); + Self { stack } + } +} + +impl<'a, 'x, P: Property, E: Element> Iterator for Elements<'a, 'x, P, E> { + type Item = &'a E; + + fn next(&mut self) -> Option { + while let Some(value) = self.stack.pop() { + match value { + Value::Element(element) => return Some(element), + Value::Array(items) => self.stack.extend(items.iter().rev()), + Value::Object(obj) => self.stack.extend(obj.as_vec().iter().rev().map(|(_, v)| v)), + Value::Null | Value::Bool(_) | Value::Number(_) | Value::Str(_) => {} + } + } + + None + } +} diff --git a/crates/calcard/src/common/embedded.rs b/crates/calcard/src/common/embedded.rs new file mode 100644 index 00000000..f8aeb761 --- /dev/null +++ b/crates/calcard/src/common/embedded.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashSet; + +use crate::{ + icalendar::{ICalendar, ICalendarProperty, ICalendarValue, Uri}, + vcard::{VCard, VCardProperty, VCardValue}, +}; + +pub(crate) fn distinct_size<'x>(binaries: impl Iterator) -> usize { + let mut distinct: AHashSet<&[u8]> = AHashSet::new(); + let mut size = 0usize; + + for data in binaries { + if distinct.insert(data) { + size = size.saturating_add(data.len()); + } + } + + size +} + +impl ICalendar { + pub fn blob_binaries(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| component.component_type.converts_links()) + .flat_map(|component| component.entries.iter()) + .filter(|entry| { + matches!( + entry.name, + ICalendarProperty::Attach | ICalendarProperty::Image + ) + }) + .filter_map(|entry| entry.values.first().and_then(ICalendarValue::binary_bytes)) + } + + pub fn embedded_binaries(&self) -> impl Iterator { + self.components + .iter() + .flat_map(|component| component.entries.iter()) + .flat_map(|entry| entry.values.iter()) + .filter_map(|value| match value { + ICalendarValue::Binary(data) => Some(data.as_slice()), + ICalendarValue::Uri(Uri::Data(data)) => Some(data.data.as_slice()), + _ => None, + }) + } + + pub fn embedded_size(&self) -> usize { + distinct_size(self.embedded_binaries()) + } +} + +impl VCard { + pub fn blob_binaries(&self) -> impl Iterator { + self.entries + .iter() + .filter(|entry| { + matches!( + entry.name, + VCardProperty::Photo | VCardProperty::Logo | VCardProperty::Sound + ) + }) + .filter_map(|entry| match entry.values.first() { + Some(VCardValue::Binary(data)) if !data.data.is_empty() => { + Some(data.data.as_slice()) + } + _ => None, + }) + } + + pub fn embedded_binaries(&self) -> impl Iterator { + self.entries + .iter() + .flat_map(|entry| entry.values.iter()) + .filter_map(|value| match value { + VCardValue::Binary(data) => Some(data.data.as_slice()), + _ => None, + }) + } + + pub fn embedded_size(&self) -> usize { + distinct_size(self.embedded_binaries()) + } +} + +#[cfg(feature = "rkyv")] +mod archived { + use super::distinct_size; + use crate::{ + icalendar::{ + ArchivedICalendar, ArchivedICalendarProperty, ArchivedICalendarValue, ArchivedUri, + }, + vcard::{ArchivedVCard, ArchivedVCardProperty, ArchivedVCardValue}, + }; + + impl ArchivedICalendar { + pub fn blob_binaries(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| component.component_type.converts_links()) + .flat_map(|component| component.entries.iter()) + .filter(|entry| { + matches!( + entry.name, + ArchivedICalendarProperty::Attach | ArchivedICalendarProperty::Image + ) + }) + .filter_map(|entry| { + entry + .values + .first() + .and_then(ArchivedICalendarValue::binary_bytes) + }) + } + + pub fn embedded_binaries(&self) -> impl Iterator { + self.components + .iter() + .flat_map(|component| component.entries.iter()) + .flat_map(|entry| entry.values.iter()) + .filter_map(|value| match value { + ArchivedICalendarValue::Binary(data) => Some(data.as_ref()), + ArchivedICalendarValue::Uri(ArchivedUri::Data(data)) => { + Some(data.data.as_ref()) + } + _ => None, + }) + } + + pub fn embedded_size(&self) -> usize { + distinct_size(self.embedded_binaries()) + } + } + + impl ArchivedICalendarValue { + pub(crate) fn binary_bytes(&self) -> Option<&[u8]> { + match self { + ArchivedICalendarValue::Binary(data) => Some(data.as_ref()), + ArchivedICalendarValue::Uri(ArchivedUri::Data(data)) => Some(data.data.as_ref()), + _ => None, + } + .filter(|data| !data.is_empty()) + } + } + + impl ArchivedVCard { + pub fn blob_binaries(&self) -> impl Iterator { + self.entries + .iter() + .filter(|entry| { + matches!( + entry.name, + ArchivedVCardProperty::Photo + | ArchivedVCardProperty::Logo + | ArchivedVCardProperty::Sound + ) + }) + .filter_map(|entry| match entry.values.first() { + Some(ArchivedVCardValue::Binary(data)) if !data.data.is_empty() => { + Some(data.data.as_ref()) + } + _ => None, + }) + } + + pub fn embedded_binaries(&self) -> impl Iterator { + self.entries + .iter() + .flat_map(|entry| entry.values.iter()) + .filter_map(|value| match value { + ArchivedVCardValue::Binary(data) => Some(data.data.as_ref()), + _ => None, + }) + } + + pub fn embedded_size(&self) -> usize { + distinct_size(self.embedded_binaries()) + } + } +} + diff --git a/crates/calcard/src/common/export.rs b/crates/calcard/src/common/export.rs new file mode 100644 index 00000000..1c902305 --- /dev/null +++ b/crates/calcard/src/common/export.rs @@ -0,0 +1,126 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashSet; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ExportError { + NotGroup, + NoComponents, + InvalidPatch { + recurrence_id: String, + pointer: String, + }, + EmbeddedSizeExceeded { + max: usize, + }, + UnresolvedBlob { + blob_id: String, + }, +} + +impl Display for ExportError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ExportError::NotGroup => { + f.write_str("the top-level object is not a JSCalendar Group or a JSContact Card") + } + ExportError::NoComponents => { + f.write_str("the object has no entries that convert to a component") + } + ExportError::InvalidPatch { + recurrence_id, + pointer, + } => write!( + f, + "the patch at recurrenceOverrides/{recurrence_id}/{pointer} does not apply" + ), + ExportError::EmbeddedSizeExceeded { max } => write!( + f, + "the embedded binaries exceed the maximum size of {max} bytes" + ), + ExportError::UnresolvedBlob { blob_id } => { + write!(f, "the blobId {blob_id} could not be resolved") + } + } + } +} + +impl std::error::Error for ExportError {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ImportError { + BlobIdFailed, +} + +impl Display for ImportError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ImportError::BlobIdFailed => { + f.write_str("the blob id generator failed to store a binary") + } + } + } +} + +impl std::error::Error for ImportError {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct RejectedPatch { + pub recurrence_id: String, + pub pointer: String, +} + +impl Display for RejectedPatch { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the patch at recurrenceOverrides/{}/{} does not apply", + self.recurrence_id, self.pointer + ) + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct EmbeddedBudget { + max: Option, + left: usize, + charged: AHashSet<(usize, u64)>, +} + +impl EmbeddedBudget { + pub(crate) fn new(max: Option) -> Self { + Self { + max, + left: max.unwrap_or(usize::MAX), + charged: AHashSet::new(), + } + } + + pub(crate) fn take(&mut self, data: &[u8]) -> Result<(), ExportError> { + let Some(max) = self.max else { + return Ok(()); + }; + let key = (data.len(), self.charged.hasher().hash_one(data)); + if !self.charged.insert(key) { + return Ok(()); + } + match self.left.checked_sub(data.len()) { + Some(left) => { + self.left = left; + Ok(()) + } + None => { + self.left = 0; + Err(ExportError::EmbeddedSizeExceeded { max }) + } + } + } +} diff --git a/crates/calcard/src/common/format.rs b/crates/calcard/src/common/format.rs new file mode 100644 index 00000000..d36ef0d7 --- /dev/null +++ b/crates/calcard/src/common/format.rs @@ -0,0 +1,320 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use simdutf8::basic::from_utf8; +use std::fmt::{self, Write}; + +const MAX_DIGITS: usize = 20; +const DIRECT_WRITE_LEN: usize = 256; + +trait CopyShort { + fn copy_short_from(&mut self, src: &[u8]); +} + +impl CopyShort for [u8] { + #[inline(always)] + fn copy_short_from(&mut self, src: &[u8]) { + match src.len() { + 0 => {} + len @ 1..=3 => { + for pos in [0, len / 2, len - 1] { + if let (Some(dst), Some(src)) = (self.get_mut(pos), src.get(pos)) { + *dst = *src; + } + } + } + 4..=7 => { + if let (Some(dst), Some(src)) = + (self.first_chunk_mut::<4>(), src.first_chunk::<4>()) + { + *dst = *src; + } + if let (Some(dst), Some(src)) = (self.last_chunk_mut::<4>(), src.last_chunk::<4>()) + { + *dst = *src; + } + } + 8..=16 => { + if let (Some(dst), Some(src)) = + (self.first_chunk_mut::<8>(), src.first_chunk::<8>()) + { + *dst = *src; + } + if let (Some(dst), Some(src)) = (self.last_chunk_mut::<8>(), src.last_chunk::<8>()) + { + *dst = *src; + } + } + 17..=32 => { + if let (Some(dst), Some(src)) = + (self.first_chunk_mut::<16>(), src.first_chunk::<16>()) + { + *dst = *src; + } + if let (Some(dst), Some(src)) = + (self.last_chunk_mut::<16>(), src.last_chunk::<16>()) + { + *dst = *src; + } + } + 33..=64 => { + if let (Some(dst), Some(src)) = + (self.first_chunk_mut::<32>(), src.first_chunk::<32>()) + { + *dst = *src; + } + if let (Some(dst), Some(src)) = + (self.last_chunk_mut::<32>(), src.last_chunk::<32>()) + { + *dst = *src; + } + } + 65..=128 => { + if let (Some(dst), Some(src)) = + (self.first_chunk_mut::<64>(), src.first_chunk::<64>()) + { + *dst = *src; + } + if let (Some(dst), Some(src)) = + (self.last_chunk_mut::<64>(), src.last_chunk::<64>()) + { + *dst = *src; + } + } + _ => self.copy_from_slice(src), + } + } +} + +pub(crate) trait AsciiPush { + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result; + + #[inline(always)] + fn push_ascii_str(&mut self, text: &'static str) -> fmt::Result { + self.push_ascii(text.as_bytes()) + } + + #[inline(always)] + fn push_byte(&mut self, ch: u8) -> fmt::Result { + self.push_ascii(&[ch]) + } + + #[inline(always)] + fn push_u64(&mut self, value: u64) -> fmt::Result { + if value < 10 { + self.push_digits::<1>(value) + } else { + self.push_digits::<2>(value) + } + } + + #[inline(always)] + fn push_i64(&mut self, value: i64) -> fmt::Result { + if value < 0 { + self.push_byte(b'-')?; + } + self.push_u64(value.unsigned_abs()) + } + + #[inline(always)] + fn push_2_digits(&mut self, value: u8) -> fmt::Result { + self.push_digits::<2>(value.into()) + } + + #[inline(always)] + fn push_4_digits(&mut self, value: u16) -> fmt::Result { + self.push_digits::<4>(value.into()) + } + + #[inline(always)] + fn push_digits(&mut self, value: u64) -> fmt::Result { + match u32::try_from(value) { + Ok(value) if value < 10u32.pow(WIDTH as u32) => { + let mut digits = [b'0'; WIDTH]; + for (place, digit) in digits.iter_mut().rev().enumerate() { + *digit += (value / 10u32.pow(place as u32) % 10) as u8; + } + self.push_ascii(&digits) + } + _ => self.push_padded(value, WIDTH), + } + } + + fn push_padded(&mut self, value: u64, width: usize) -> fmt::Result { + let mut digits = [b'0'; MAX_DIGITS]; + let mut rest = value; + let mut count = 0; + for slot in digits.iter_mut().rev() { + *slot = b'0' + (rest % 10) as u8; + rest /= 10; + count += 1; + if rest == 0 { + break; + } + } + let start = MAX_DIGITS - count.max(width).min(MAX_DIGITS); + self.push_ascii(digits.get(start..).unwrap_or_default()) + } + + fn push_list( + &mut self, + name: &'static str, + items: &[T], + mut push_item: impl FnMut(&mut Self, &T) -> fmt::Result, + ) -> fmt::Result + where + Self: Sized, + { + for (pos, item) in items.iter().enumerate() { + self.push_ascii_str(if pos == 0 { name } else { "," })?; + push_item(self, item)?; + } + Ok(()) + } + + fn push_duration(&mut self, duration: DurationParts) -> fmt::Result { + let DurationParts { + neg, + weeks, + days, + hours, + minutes, + seconds, + } = duration; + self.push_ascii(if neg { b"-P" } else { b"P" })?; + if weeks == 0 && days == 0 && hours == 0 && minutes == 0 && seconds == 0 { + return self.push_ascii(b"T0S"); + } + for (value, unit) in [(weeks, b'W'), (days, b'D')] { + if value != 0 { + self.push_u64(value.into())?; + self.push_byte(unit)?; + } + } + if hours != 0 || minutes != 0 || seconds != 0 { + self.push_byte(b'T')?; + for (value, unit) in [(hours, b'H'), (minutes, b'M'), (seconds, b'S')] { + if value != 0 { + self.push_u64(value.into())?; + self.push_byte(unit)?; + } + } + } + Ok(()) + } +} + +pub(crate) struct BufferedWriter<'x, W: Write + ?Sized, const N: usize> { + out: &'x mut W, + len: usize, + buf: [u8; N], +} + +impl<'x, W: Write + ?Sized, const N: usize> BufferedWriter<'x, W, N> { + #[inline(always)] + pub(crate) fn new(out: &'x mut W) -> Self { + Self { + out, + len: 0, + buf: [0; N], + } + } + + pub(crate) fn flush(&mut self) -> fmt::Result { + if self.len > 0 { + let text = self + .buf + .get(..self.len) + .and_then(|bytes| from_utf8(bytes).ok()) + .ok_or(fmt::Error)?; + self.out.write_str(text)?; + self.len = 0; + } + Ok(()) + } + + #[inline(always)] + fn push_bytes(&mut self, bytes: &[u8]) -> Option<()> { + if bytes.len() < DIRECT_WRITE_LEN + && let Some(slot) = self.buf.get_mut(self.len..self.len + bytes.len()) + { + slot.copy_short_from(bytes); + self.len += bytes.len(); + Some(()) + } else { + None + } + } + + #[inline(never)] + fn write_str_after_flush(&mut self, text: &str) -> fmt::Result { + self.flush()?; + match self.buf.get_mut(..text.len()) { + Some(slot) if text.len() < DIRECT_WRITE_LEN => { + slot.copy_from_slice(text.as_bytes()); + self.len = text.len(); + Ok(()) + } + _ => self.out.write_str(text), + } + } + + #[inline(never)] + fn push_ascii_after_flush(&mut self, bytes: &[u8]) -> fmt::Result { + self.write_str_after_flush(str::from_utf8(bytes).map_err(|_| fmt::Error)?) + } +} + +impl AsciiPush for BufferedWriter<'_, W, N> { + #[inline(always)] + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + match self.push_bytes(bytes) { + Some(()) => Ok(()), + None => self.push_ascii_after_flush(bytes), + } + } +} + +impl Write for BufferedWriter<'_, W, N> { + #[inline(always)] + fn write_str(&mut self, text: &str) -> fmt::Result { + match self.push_bytes(text.as_bytes()) { + Some(()) => Ok(()), + None => self.write_str_after_flush(text), + } + } +} + +impl AsciiPush for String { + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + self.push_str(str::from_utf8(bytes).map_err(|_| fmt::Error)?); + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct DurationParts { + pub neg: bool, + pub weeks: u32, + pub days: u32, + pub hours: u32, + pub minutes: u32, + pub seconds: u32, +} + +pub(crate) trait BufferedWrite: Write { + fn write_buffered( + &mut self, + push: impl FnOnce(&mut BufferedWriter<'_, Self, N>) -> fmt::Result, + ) -> fmt::Result { + let mut buf = BufferedWriter::new(self); + push(&mut buf)?; + buf.flush() + } +} + +impl BufferedWrite for W {} + diff --git a/crates/calcard/src/common/iana.rs b/crates/calcard/src/common/iana.rs new file mode 100644 index 00000000..ac624e74 --- /dev/null +++ b/crates/calcard/src/common/iana.rs @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +use crate::common::{IanaString, IanaType}; + +impl IanaType { + pub fn iana(&self) -> Option<&I> { + match self { + IanaType::Iana(iana) => Some(iana), + _ => None, + } + } + + pub fn other(&self) -> Option<&O> { + match self { + IanaType::Other(s) => Some(s), + _ => None, + } + } + + pub fn into_iana(self) -> Option { + match self { + IanaType::Iana(iana) => Some(iana), + _ => None, + } + } + + pub fn is_iana(&self) -> bool { + matches!(self, IanaType::Iana(_)) + } + + pub fn is_iana_and(&self, f: impl FnOnce(&I) -> bool) -> bool { + match &self { + IanaType::Iana(x) => f(x), + _ => false, + } + } +} + +impl AsRef for IanaType +where + I: IanaString, + O: AsRef, +{ + fn as_ref(&self) -> &str { + match self { + IanaType::Iana(iana) => iana.as_str(), + IanaType::Other(s) => s.as_ref(), + } + } +} + +impl IanaType +where + I: IanaString, +{ + pub fn into_string(self) -> Cow<'static, str> { + match self { + IanaType::Iana(iana) => Cow::Borrowed(iana.as_str()), + IanaType::Other(s) => Cow::Owned(s), + } + } +} + +impl From for IanaType +where + I: IanaString, +{ + fn from(iana: I) -> Self { + IanaType::Iana(iana) + } +} + +impl From for IanaType { + fn from(s: String) -> Self { + IanaType::Other(s) + } +} + +impl Default for IanaType +where + I: IanaString + + std::fmt::Debug + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + std::hash::Hash + + Default, + O: std::fmt::Debug + Clone + PartialEq + Eq + PartialOrd + Ord + std::hash::Hash, +{ + fn default() -> Self { + IanaType::Iana(I::default()) + } +} diff --git a/crates/calcard/src/common/jsprop.rs b/crates/calcard/src/common/jsprop.rs new file mode 100644 index 00000000..9050f516 --- /dev/null +++ b/crates/calcard/src/common/jsprop.rs @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Property, Value}; +use std::{io::Write, iter::repeat_n}; + +pub(crate) mod ordered; +pub(crate) mod text; + +const KEY_CAPACITY: usize = 16; + +macro_rules! property_names { + ($property:ident, $($name:literal => $variant:ident,)*) => { + pub(crate) fn parse_name(s: &str) -> Option { + hashify::fnc_map!(s.as_bytes(), + $($name => Some($property::$variant),)* + "metadata" => Some($property::Metadata), + "privateMetadata" => Some($property::PrivateMetadata), + _ => None, + ) + } + + pub(crate) fn parse_nested_name(s: &str) -> Option { + hashify::fnc_map!(s.as_bytes(), + $($name => Some($property::$variant),)* + _ => None, + ) + } + }; +} + +pub(crate) use property_names; + +pub(crate) trait JSPropPointer { + fn parse_jsprop_value>( + &self, + json: &str, + ) -> Option>; +} + +impl JSPropPointer

for JsonPointer

{ + fn parse_jsprop_value>( + &self, + json: &str, + ) -> Option> { + let items = self.as_slice(); + if items + .iter() + .any(|item| matches!(item, JsonPointerItem::Root | JsonPointerItem::Wildcard)) + { + return Value::parse_json(json).ok().map(Value::into_owned); + } + + let mut wrapped = Vec::with_capacity(json.len() + items.len() * KEY_CAPACITY); + for item in items { + wrapped.push(b'{'); + match item { + JsonPointerItem::Key(key) => serde_json::to_writer(&mut wrapped, key).ok()?, + JsonPointerItem::Number(number) => write!(wrapped, "\"{number}\"").ok()?, + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => { + return None; + } + } + wrapped.push(b':'); + } + wrapped.extend_from_slice(json.as_bytes()); + wrapped.extend(repeat_n(b'}', items.len())); + + let mut value = Value::::parse_json(std::str::from_utf8(&wrapped).ok()?).ok()?; + for _ in items { + let [(_, inner)] = <[_; 1]>::try_from(value.into_object()?.into_vec()).ok()?; + value = inner; + } + Some(value.into_owned()) + } +} diff --git a/crates/calcard/src/common/jsprop/ordered.rs b/crates/calcard/src/common/jsprop/ordered.rs new file mode 100644 index 00000000..a0ece9f0 --- /dev/null +++ b/crates/calcard/src/common/jsprop/ordered.rs @@ -0,0 +1,174 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::{AHashMap, RandomState}; +use std::{borrow::Borrow, hash::Hash, vec::IntoIter}; + +pub(crate) const INDEX_THRESHOLD: usize = 16; + +#[derive(Debug)] +pub(crate) struct OrderedMap { + entries: Vec<(K, V)>, + index: Option>, +} + +#[derive(Debug)] +struct HashIndex { + state: RandomState, + positions: AHashMap, +} + +impl Default for OrderedMap { + fn default() -> Self { + Self { + entries: Vec::new(), + index: None, + } + } +} + +impl OrderedMap { + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(key, value)| (key, value)) + } + + pub(crate) fn contains_key(&self, key: &Q) -> bool + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + self.position(key).is_some() + } + + pub(crate) fn get_mut(&mut self, key: &Q) -> Option<&mut V> + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let position = self.position(key)?; + self.entries.get_mut(position).map(|(_, value)| value) + } + + pub(crate) fn push(&mut self, key: K, value: V) { + if let Some(index) = &mut self.index { + index.insert(&key, self.entries.len()); + } + self.entries.push((key, value)); + if self.index.is_none() { + self.index = HashIndex::build(&self.entries); + } + } + + pub(crate) fn insert(&mut self, key: K, value: V) { + match self.get_mut(&key) { + Some(existing) => *existing = value, + None => self.push(key, value), + } + } + + pub(crate) fn insert_if_absent(&mut self, key: K, value: V) { + if !self.contains_key(&key) { + self.push(key, value); + } + } + + pub(crate) fn upsert( + &mut self, + key: K, + value: impl FnOnce() -> V, + update: impl FnOnce(&mut V), + ) { + match self.get_mut(&key) { + Some(existing) => update(existing), + None => { + let mut value = value(); + update(&mut value); + self.push(key, value); + } + } + } + + pub(crate) fn retain(&mut self, mut keep: impl FnMut(&K, &mut V) -> bool) { + let len = self.entries.len(); + self.entries.retain_mut(|(key, value)| keep(key, value)); + if self.entries.len() != len { + self.index = HashIndex::build(&self.entries); + } + } + + fn position(&self, key: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let Some(index) = &self.index else { + return self.scan(key); + }; + let position = index.get(key)?; + match self.entries.get(position) { + Some((item, _)) if item.borrow() == key => Some(position), + _ => self.scan(key), + } + } + + fn scan(&self, key: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + self.entries + .iter() + .position(|(item, _)| item.borrow() == key) + } +} + +impl IntoIterator for OrderedMap { + type Item = (K, V); + type IntoIter = IntoIter<(K, V)>; + + fn into_iter(self) -> Self::IntoIter { + self.entries.into_iter() + } +} + +impl HashIndex { + #[inline] + fn build(entries: &[(K, V)]) -> Option> { + (entries.len() > INDEX_THRESHOLD).then(|| Self::index(entries)) + } + + #[cold] + #[inline(never)] + fn index(entries: &[(K, V)]) -> Box { + let mut index = Box::new(HashIndex { + state: RandomState::new(), + positions: AHashMap::with_capacity(entries.len()), + }); + for (position, (key, _)) in entries.iter().enumerate() { + index.insert(key, position); + } + index + } + + fn insert(&mut self, key: &Q, position: usize) { + self.positions + .entry(self.state.hash_one(key)) + .or_insert(position); + } + + fn get(&self, key: &Q) -> Option { + self.positions.get(&self.state.hash_one(key)).copied() + } +} + diff --git a/crates/calcard/src/common/jsprop/text.rs b/crates/calcard/src/common/jsprop/text.rs new file mode 100644 index 00000000..5509dd77 --- /dev/null +++ b/crates/calcard/src/common/jsprop/text.rs @@ -0,0 +1,287 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::format::AsciiPush; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Property}; +use std::{ + borrow::Cow, + fmt::{self, Display, Write}, +}; + +const POINTER_CAPACITY: usize = 48; +const CONVERTED_KEYS_CAPACITY: usize = 4; + +pub(crate) trait PointerText: Write { + fn reserve_escapes(&mut self, _escapes: usize) {} + + fn push_pointer_token(&mut self, token: &str) -> fmt::Result { + let escapes = token + .bytes() + .filter(|byte| matches!(byte, b'~' | b'/')) + .count(); + if escapes == 0 { + return self.write_str(token); + } + self.reserve_escapes(escapes); + for piece in token.split_inclusive(['~', '/']) { + match piece.strip_suffix('~') { + Some(head) => { + self.write_str(head)?; + self.write_str("~0")?; + } + None => match piece.strip_suffix('/') { + Some(head) => { + self.write_str(head)?; + self.write_str("~1")?; + } + None => self.write_str(piece)?, + }, + } + } + Ok(()) + } +} + +pub(crate) trait PointerString: Sized { + fn from_pointer<'x>(parts: impl IntoIterator + Clone) -> Self; +} + +pub(crate) trait PointerTextExt { + fn to_text(&self) -> String; + fn text_eq(&self, text: &str) -> bool; +} + +pub(crate) trait DisplayEq { + fn display_eq(&self, text: &str) -> bool; +} + +pub(crate) trait IdReferenceText { + fn to_id_reference(&self) -> String; +} + +pub(crate) trait IntoAsciiLowercase { + fn into_ascii_lowercase(self) -> String; +} + +pub(crate) trait PointerProperty: Property { + fn into_pointer(self) -> Result, Self>; +} + +pub(crate) trait ConvertedKeys { + fn into_converted_keys(self) -> Vec>; +} + +pub(crate) struct Expect<'x>(&'x [u8]); + +pub(crate) struct AsciiString(String); + +impl PointerText for String { + fn reserve_escapes(&mut self, escapes: usize) { + self.reserve_exact(self.capacity() - self.len() + escapes); + } +} + +impl PointerString for String { + fn from_pointer<'x>(parts: impl IntoIterator + Clone) -> Self { + let len = parts + .clone() + .into_iter() + .map(|part| part.len() + 1) + .sum::(); + let mut text = String::with_capacity(len.saturating_sub(1)); + for (position, part) in parts.into_iter().enumerate() { + if position > 0 { + text.push('/'); + } + let _ = text.push_pointer_token(part); + } + text + } +} + +impl PointerTextExt

for JsonPointer

{ + fn to_text(&self) -> String { + let mut text = String::with_capacity(POINTER_CAPACITY); + let _ = write!(text, "{self}"); + text + } + + fn text_eq(&self, text: &str) -> bool { + if text.bytes().any(|byte| matches!(byte, b'/' | b'~')) { + return self.display_eq(text); + } + match self.as_slice() { + [] | [JsonPointerItem::Root] => text.is_empty(), + [JsonPointerItem::Wildcard] => text == "*", + [JsonPointerItem::Invalid(invalid)] => invalid == text, + [JsonPointerItem::Key(key)] => *key == text, + [JsonPointerItem::Number(number)] => number.display_eq(text), + _ => false, + } + } +} + +impl DisplayEq for T { + fn display_eq(&self, text: &str) -> bool { + let mut expect = Expect::new(text); + write!(expect, "{self}").is_ok() && expect.is_complete() + } +} + +impl IdReferenceText for str { + fn to_id_reference(&self) -> String { + let mut text = String::with_capacity(self.len() + 1); + text.push('#'); + text.push_str(self); + text + } +} + +impl IntoAsciiLowercase for Cow<'_, str> { + fn into_ascii_lowercase(self) -> String { + let mut text = self.into_owned(); + text.make_ascii_lowercase(); + text + } +} + +impl AsciiString { + pub(crate) fn with_capacity(capacity: usize) -> Self { + AsciiString(String::with_capacity(capacity)) + } + + pub(crate) fn into_string(self) -> String { + self.0 + } +} + +impl From for AsciiString { + fn from(text: String) -> Self { + AsciiString(text) + } +} + +impl AsciiPush for AsciiString { + #[inline(always)] + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + self.0.extend(bytes.iter().copied().map(char::from)); + Ok(()) + } +} + +impl<'x> Expect<'x> { + pub(crate) fn new(text: &'x str) -> Self { + Expect(text.as_bytes()) + } + + pub(crate) fn is_complete(&self) -> bool { + self.0.is_empty() + } +} + +impl AsciiPush for Expect<'_> { + #[inline(always)] + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + self.0 = self.0.strip_prefix(bytes).ok_or(fmt::Error)?; + Ok(()) + } +} + +impl Write for Expect<'_> { + #[inline(always)] + fn write_str(&mut self, text: &str) -> fmt::Result { + self.push_ascii(text.as_bytes()) + } +} + +impl ConvertedKeys

for Key<'_, P> { + fn into_converted_keys(self) -> Vec> { + match self { + Key::Property(property) => match property.into_pointer() { + Ok(pointer) => pointer.into_converted_keys(), + Err(property) => property.to_cow().as_ref().into_converted_keys(), + }, + Key::Borrowed(text) => text.into_converted_keys(), + Key::Owned(text) => text.as_str().into_converted_keys(), + } + } +} + +impl ConvertedKeys

for JsonPointer

{ + fn into_converted_keys(self) -> Vec> { + let mut keys = Vec::with_capacity(self.len()); + for item in self.into_iter() { + match item { + JsonPointerItem::Key(key) => { + match key.as_string_key() { + Some(text) if text.contains('/') => { + keys.extend(JsonPointer::

::parse(text).into_iter().filter_map( + |item| match item { + JsonPointerItem::Key(key) => Some(key), + JsonPointerItem::Number(number) => { + Some(Key::Owned(number.to_string())) + } + JsonPointerItem::Root + | JsonPointerItem::Wildcard + | JsonPointerItem::Invalid(_) => None, + }, + )); + } + _ => keys.push(key), + } + } + JsonPointerItem::Number(number) => keys.push(Key::Owned(number.to_string())), + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => { + } + } + } + keys + } +} + +impl ConvertedKeys

for &str { + fn into_converted_keys(self) -> Vec> { + if self.contains('~') { + return JsonPointer::

::parse(self).into_converted_keys(); + } + let mut keys: Vec> = Vec::with_capacity(CONVERTED_KEYS_CAPACITY); + let mut parent_is_key = false; + let first = usize::from(self.starts_with('/')); + let mut segments = self.split('/').enumerate(); + for (position, segment) in segments.by_ref() { + let parent = if parent_is_key { keys.last() } else { None }; + let (key, is_key) = match segment { + "" if position == 0 => continue, + "" => (Key::from(""), true), + "*" => { + parent_is_key = false; + continue; + } + _ => match P::try_parse(parent, segment) { + Some(property) => (Key::Property(property), true), + None => ( + Key::Owned(segment.to_string()), + !segment.bytes().all(|byte| byte.is_ascii_digit()) + || (segment != "0" && segment.starts_with('0')) + || segment.parse::().is_err(), + ), + }, + }; + let opaque = position == first + && key + .as_property() + .is_some_and(|property| property.is_opaque()); + keys.push(key); + parent_is_key = is_key; + if opaque { + break; + } + } + keys.extend(segments.map(|(_, segment)| Key::Owned(segment.to_string()))); + keys + } +} + diff --git a/crates/calcard/src/common/mod.rs b/crates/calcard/src/common/mod.rs new file mode 100644 index 00000000..6fb00f02 --- /dev/null +++ b/crates/calcard/src/common/mod.rs @@ -0,0 +1,294 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jiff::{civil, tz::Offset}; +use mail_parser::DateTime; + +#[cfg(feature = "rkyv")] +pub mod archive; +#[cfg(feature = "jmap")] +pub mod blob; +pub(crate) mod decode; +#[cfg(feature = "jmap")] +pub(crate) mod elements; +pub mod embedded; +#[cfg(feature = "jmap")] +pub mod export; +pub(crate) mod format; +pub mod iana; +#[cfg(feature = "jmap")] +pub(crate) mod jsprop; +pub mod parser; +pub(crate) mod scan; +pub(crate) mod stack; +pub mod timezone; +pub mod tokenizer; +pub mod types; +mod tzdb; +pub mod writer; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum IanaType { + Iana(I), + Other(O), +} + +pub trait IanaString { + fn as_str(&self) -> &'static str; +} + +pub trait IanaParse: Sized { + fn parse(value: &[u8]) -> Option; +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub struct PartialDateTime { + pub year: Option, + pub month: Option, + pub day: Option, + pub hour: Option, + pub minute: Option, + pub second: Option, + pub tz_hour: Option, + pub tz_minute: Option, + pub tz_minus: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum CalendarScale { + #[default] + Gregorian, + Chinese, + IslamicCivil, + Hebrew, + Ethiopic, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Encoding { + QuotedPrintable, + Base64, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub struct Data { + pub content_type: Option, + pub data: Vec, +} + +#[cfg(feature = "rkyv")] +impl PartialEq> for ArchivedData { + fn eq(&self, other: &Box) -> bool { + self == other.as_ref() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DateTimeResult { + pub date_time: civil::DateTime, + pub offset: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum LinkRelation { + About, + Acl, + Alternate, + Amphtml, + ApiCatalog, + Appendix, + AppleTouchIcon, + AppleTouchStartupImage, + Archives, + Author, + BlockedBy, + Bookmark, + C2paManifest, + Canonical, + Chapter, + CiteAs, + Collection, + CompressionDictionary, + Contents, + Convertedfrom, + Copyright, + CreateForm, + Current, + Deprecation, + Describedby, + Describes, + Disclosure, + DnsPrefetch, + Duplicate, + Edit, + EditForm, + EditMedia, + Enclosure, + External, + First, + Geofeed, + Glossary, + Help, + Hosts, + Hub, + IceServer, + Icon, + Index, + Intervalafter, + Intervalbefore, + Intervalcontains, + Intervaldisjoint, + Intervalduring, + Intervalequals, + Intervalfinishedby, + Intervalfinishes, + Intervalin, + Intervalmeets, + Intervalmetby, + Intervaloverlappedby, + Intervaloverlaps, + Intervalstartedby, + Intervalstarts, + Item, + Last, + LatestVersion, + License, + Linkset, + Lrdd, + Manifest, + MaskIcon, + Me, + MediaFeed, + Memento, + Micropub, + Modulepreload, + Monitor, + MonitorGroup, + Next, + NextArchive, + Nofollow, + Noopener, + Noreferrer, + Opener, + Openid2LocalId, + Openid2Provider, + Original, + P3pv1, + Payment, + Pingback, + Preconnect, + PredecessorVersion, + Prefetch, + Preload, + Prerender, + Prev, + Preview, + Previous, + PrevArchive, + PrivacyPolicy, + Profile, + Publication, + RdapActive, + RdapBottom, + RdapDown, + RdapTop, + RdapUp, + Related, + Restconf, + Replies, + Ruleinput, + Search, + Section, + Self_, + Service, + ServiceDesc, + ServiceDoc, + ServiceMeta, + SipTrunkingCapability, + Sponsored, + Start, + Status, + Stylesheet, + Subsection, + SuccessorVersion, + Sunset, + Tag, + TermsOfService, + Timegate, + Timemap, + Type, + Ugc, + Up, + VersionHistory, + Via, + Webmention, + WorkingCopy, + WorkingCopyOf, +} + +#[cfg(feature = "jmap")] +pub(crate) enum IdReference { + Value(T), + Reference(String), + Error, +} + +#[cfg(feature = "jmap")] +impl IdReference { + pub fn parse(s: &str) -> Self { + if let Some(reference) = s.strip_prefix('#') { + IdReference::Reference(reference.to_string()) + } else { + T::from_str(s) + .map(IdReference::Value) + .unwrap_or(IdReference::Error) + } + } +} diff --git a/crates/calcard/src/common/parser.rs b/crates/calcard/src/common/parser.rs new file mode 100644 index 00000000..99b56e0e --- /dev/null +++ b/crates/calcard/src/common/parser.rs @@ -0,0 +1,534 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Data, Encoding, PartialDateTime, tokenizer::Token}; +use crate::{ + Parser, + common::{IanaParse, IanaString, IanaType}, + icalendar::Uri, +}; +use encodify::hex; +use mail_parser::DateTime; +use std::{borrow::Cow, iter::Peekable, slice::Iter, str::FromStr}; + +impl<'x> Parser<'x> { + pub(crate) fn raw_token(&mut self) -> Option> { + let (first, last) = (self.token_buf.first()?, self.token_buf.last()?); + self.source.get(first.start..=last.end).map(unfold) + } + + pub(crate) fn buf_parse_one>>(&mut self) -> Option { + let result = self.token_buf.pop().map(T::from); + self.token_buf.clear(); + result + } +} + +impl Token<'_> { + pub(crate) fn into_uri_bytes(self) -> std::result::Result { + match Data::try_parse(self.text.as_ref()) { + Some(data) => Ok(data), + None => Err(self.into_string()), + } + } + + pub(crate) fn into_timestamp( + self, + require_time: bool, + ) -> std::result::Result { + match PartialDateTime::from_timestamp_bytes(&self.text, require_time) { + (dt, true) => Ok(dt), + (_, false) => Err(self.into_string()), + } + } + + pub(crate) fn into_offset(self) -> std::result::Result { + let mut dt = PartialDateTime::default(); + dt.parse_zone(&mut self.text.iter().peekable()); + if !dt.is_null() { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_float(self) -> std::result::Result { + if let Ok(float) = self.text.as_str().parse::() + && float.is_finite() + { + return Ok(float); + } + + Err(self.into_string()) + } + + pub(crate) fn into_integer(self) -> std::result::Result { + if let Ok(integer) = self.text.as_str().parse::() { + return Ok(integer); + } + + Err(self.into_string()) + } + + pub(crate) fn into_boolean(self) -> bool { + self.text.as_ref().eq_ignore_ascii_case(b"true") + } +} + +impl Data { + pub fn try_parse(text: &[u8]) -> Option { + const BASE64: &[u8] = b"base64"; + + let (scheme, text) = text.split_at_checked(5)?; + if !scheme.eq_ignore_ascii_case(b"data:") { + return None; + } + let mut parts = text.splitn(2, |ch| *ch == b','); + let header = parts.next()?; + let data = parts.next().filter(|data| !data.is_empty())?; + let header = header.strip_suffix(b"\\").unwrap_or(header); + let mut segments = header.rsplitn(2, |ch| *ch == b';'); + let (media_type, is_base64) = match (segments.next(), segments.next()) { + (Some(last), media_type) if last.eq_ignore_ascii_case(BASE64) => { + (media_type.unwrap_or_default(), true) + } + _ => (header, false), + }; + + Some(Data { + content_type: std::str::from_utf8(media_type) + .ok() + .filter(|media_type| !media_type.is_empty()) + .map(str::to_string), + data: if is_base64 { + Encoding::Base64.decode(data)?.into_owned() + } else { + hex::PERCENT.decode(data).ok()?.into_owned() + }, + }) + } +} + +impl PartialDateTime { + pub(crate) fn from_timestamp_bytes(bytes: &[u8], require_time: bool) -> (Self, bool) { + if let Some(dt) = Self::from_basic_format(bytes) { + let valid = !require_time || dt.has_time(); + (dt, valid) + } else { + let mut dt = Self::default(); + let valid = dt.parse_timestamp(&mut bytes.iter().peekable(), require_time); + (dt, valid) + } + } + + fn from_basic_format(bytes: &[u8]) -> Option { + let (date, time) = bytes.split_first_chunk::<8>()?; + let mut dt = Self::from_basic_date(date)?; + match time { + [] => {} + [b'T' | b't', h0, h1, m0, m1, s0, s1, zone @ ..] => { + dt.hour = Some(Self::two_digits(*h0, *h1)?); + dt.minute = Some(Self::two_digits(*m0, *m1)?); + dt.second = Some(Self::two_digits(*s0, *s1)?); + match zone { + [] => {} + [b'Z' | b'z'] => { + dt.tz_hour = Some(0); + dt.tz_minute = Some(0); + } + _ => return None, + } + } + _ => return None, + } + Some(dt) + } + + pub(crate) fn from_basic_date(&[y0, y1, y2, y3, m0, m1, d0, d1]: &[u8; 8]) -> Option { + Some(PartialDateTime { + year: Some( + u16::from(Self::two_digits(y0, y1)?) * 100 + u16::from(Self::two_digits(y2, y3)?), + ), + month: Some(Self::two_digits(m0, m1)?), + day: Some(Self::two_digits(d0, d1)?), + ..Default::default() + }) + } + + #[inline(always)] + fn two_digits(tens: u8, ones: u8) -> Option { + let (tens, ones) = (tens.wrapping_sub(b'0'), ones.wrapping_sub(b'0')); + (tens < 10 && ones < 10).then(|| tens * 10 + ones) + } + + pub fn parse_timestamp( + &mut self, + iter: &mut Peekable>, + require_time: bool, + ) -> bool { + let mut idx = 0; + for ch in iter { + match ch { + b'0'..=b'9' => { + let value = match idx { + 0..=3 => { + if let Some(value) = &mut self.year { + *value = + value.saturating_mul(10).saturating_add((ch - b'0') as u16); + } else { + self.year = Some((ch - b'0') as u16); + } + idx += 1; + continue; + } + 4..=5 => &mut self.month, + 6..=7 => &mut self.day, + 9..=10 => &mut self.hour, + 11..=12 => &mut self.minute, + 13..=14 => &mut self.second, + 16..=17 => &mut self.tz_hour, + 18..=19 => &mut self.tz_minute, + _ => return false, + }; + + if let Some(value) = value { + *value = value.saturating_mul(10).saturating_add(ch - b'0'); + } else { + *value = Some(ch - b'0'); + } + } + b'T' | b't' if idx == 8 => {} + b'+' if idx == 15 => {} + b'Z' | b'z' if idx == 15 => { + self.tz_hour = Some(0); + self.tz_minute = Some(0); + } + b'-' if idx == 15 => { + self.tz_minus = true; + } + b' ' | b'\t' | b'\r' | b'\n' => { + continue; + } + _ => break, + } + idx += 1; + } + + if self.hour.is_some() { + self.minute.get_or_insert(0); + self.second.get_or_insert(0); + } + + self.has_date() && (!require_time || self.has_time()) + } + + pub(crate) fn parse_zone(&mut self, iter: &mut Peekable>) -> bool { + self.tz_minus = match iter.peek() { + Some(b'-') => true, + Some(b'+') => false, + Some(b'Z') | Some(b'z') => { + self.tz_hour = Some(0); + self.tz_minute = Some(0); + iter.next(); + return true; + } + _ => return false, + }; + + iter.next(); + let mut idx = 0; + for ch in iter { + match ch { + b'0'..=b'9' => { + idx += 1; + let value = match idx { + 1 | 2 => &mut self.tz_hour, + 3 | 4 => &mut self.tz_minute, + _ => return false, + }; + + if let Some(value) = value { + *value = value.saturating_mul(10).saturating_add(ch - b'0'); + } else { + *value = Some(ch - b'0'); + } + } + _ => { + if !ch.is_ascii_whitespace() { + return false; + } + } + } + } + + self.tz_hour.is_some() + } + + pub fn is_null(&self) -> bool { + self.year.is_none() + && self.month.is_none() + && self.day.is_none() + && self.hour.is_none() + && self.minute.is_none() + && self.second.is_none() + && self.tz_hour.is_none() + && self.tz_minute.is_none() + } + + #[inline(always)] + pub fn has_date(&self) -> bool { + self.year.is_some() && self.month.is_some() && self.day.is_some() + } + + #[inline(always)] + pub fn has_time(&self) -> bool { + self.hour.is_some() && self.minute.is_some() + } + + #[inline(always)] + pub fn has_zone(&self) -> bool { + self.tz_hour.is_some() + } + + #[inline(always)] + pub fn has_date_and_time(&self) -> bool { + self.has_date() && self.has_time() + } + + fn to_datetime(&self) -> Option { + if self.has_date() && self.has_time() { + DateTime { + year: self.year.unwrap(), + month: self.month.unwrap(), + day: self.day.unwrap(), + hour: self.hour.unwrap(), + minute: self.minute.unwrap(), + second: self.second.unwrap_or_default(), + tz_before_gmt: self.tz_minus, + tz_hour: self.tz_hour.unwrap_or_default(), + tz_minute: self.tz_minute.unwrap_or_default(), + } + .into() + } else { + None + } + } + + pub fn to_rfc3339(&self) -> Option { + self.to_datetime().map(|dt| dt.to_rfc3339()) + } + + pub fn to_timestamp(&self) -> Option { + self.to_datetime().map(|dt| dt.to_timestamp()) + } +} + +pub(crate) fn parse_digits( + iter: &mut Peekable>, + target: &mut Option, + num: usize, + nullable: bool, +) -> bool { + let mut idx = 0; + while let Some(ch) = iter.peek() { + match ch { + b'0'..=b'9' => { + let ch = (*ch - b'0') as u16; + idx += 1; + iter.next(); + + if let Some(target) = target { + *target = target.saturating_mul(10).saturating_add(ch); + + if idx == num { + return true; + } + } else { + *target = Some(ch); + } + } + b'-' if nullable => { + idx += 1; + iter.next(); + if idx == num / 2 { + return true; + } + } + _ => { + if !ch.is_ascii_whitespace() { + return false; + } else { + iter.next(); + } + } + }; + } + + false +} + +pub(crate) fn parse_small_digits( + iter: &mut Peekable>, + target: &mut Option, + num: usize, + nullable: bool, +) -> bool { + let mut idx = 0; + while let Some(ch) = iter.peek() { + match ch { + b'0'..=b'9' => { + let ch = *ch - b'0'; + idx += 1; + iter.next(); + + if let Some(target) = target { + *target = target.saturating_mul(10).saturating_add(ch); + + if idx == num { + return true; + } + } else { + *target = Some(ch); + } + } + b'-' if nullable => { + idx += 1; + iter.next(); + if idx == num / 2 { + return true; + } + } + _ => { + if !ch.is_ascii_whitespace() { + return false; + } else { + iter.next(); + } + } + }; + } + + false +} + +pub(crate) fn unfold(text: &str) -> Cow<'_, str> { + if !text.as_bytes().contains(&b'\n') { + return Cow::Borrowed(text); + } + + let mut unfolded = String::with_capacity(text.len()); + + for (pos, chunk) in text.split('\n').enumerate() { + let chunk = if pos == 0 { + chunk + } else { + chunk.strip_prefix([' ', '\t']).unwrap_or(chunk) + }; + unfolded.push_str(chunk.strip_suffix('\r').unwrap_or(chunk)); + } + + Cow::Owned(unfolded) +} + +#[derive(Default)] +pub(crate) struct Timestamp(pub i64); + +impl FromStr for Timestamp { + type Err = (); + + fn from_str(s: &str) -> std::result::Result { + Timestamp::parse(s.as_bytes()).ok_or(()) + } +} + +impl IanaParse for Timestamp { + fn parse(value: &[u8]) -> Option { + PartialDateTime::from_timestamp_bytes(value, true) + .0 + .to_timestamp() + .map(Timestamp) + } +} + +pub(crate) struct Integer(pub i64); + +impl IanaParse for Integer { + fn parse(value: &[u8]) -> Option { + let mut num: i64 = 0; + let mut is_negative = false; + + for (pos, ch) in value.iter().enumerate() { + match ch { + b'-' if pos == 0 => { + is_negative = true; + } + b'+' if pos == 0 => {} + b'0'..=b'9' => { + num = num.saturating_mul(10).saturating_add((*ch - b'0') as i64); + } + _ => { + if !ch.is_ascii_whitespace() { + return None; + } + } + } + } + + Some(Integer(if is_negative { -num } else { num })) + } +} + +pub(crate) struct Boolean(pub bool); + +impl IanaParse for Boolean { + fn parse(value: &[u8]) -> Option { + if value.eq_ignore_ascii_case(b"true") { + Some(Boolean(true)) + } else if value.eq_ignore_ascii_case(b"false") { + Some(Boolean(false)) + } else { + None + } + } +} + +impl From> for IanaType +where + I: IanaParse, +{ + fn from(value: Token<'_>) -> Self { + I::parse(value.text.as_ref()) + .map(IanaType::Iana) + .unwrap_or_else(|| IanaType::Other(value.into_string())) + } +} + +impl From> for IanaType +where + I: IanaString + + IanaParse + + std::fmt::Debug + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + std::hash::Hash, +{ + fn from(value: Token<'_>) -> Self { + I::parse(value.text.as_ref()) + .map(IanaType::Iana) + .unwrap_or_else(|| { + IanaType::Other( + value + .into_uri_bytes() + .map(Uri::from) + .unwrap_or_else(Uri::Location), + ) + }) + } +} + diff --git a/crates/calcard/src/common/scan.rs b/crates/calcard/src/common/scan.rs new file mode 100644 index 00000000..1ca842d5 --- /dev/null +++ b/crates/calcard/src/common/scan.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::{memchr2, memchr3}; + +pub(crate) const EOL: u16 = 1; +pub(crate) const WHITESPACE: u16 = 1 << 1; +pub(crate) const BACKSLASH: u16 = 1 << 2; +pub(crate) const QUOTE: u16 = 1 << 3; +pub(crate) const COLON: u16 = 1 << 4; +pub(crate) const SEMICOLON: u16 = 1 << 5; +pub(crate) const COMMA: u16 = 1 << 6; +pub(crate) const EQUAL: u16 = 1 << 7; +pub(crate) const DOT: u16 = 1 << 8; +pub(crate) const CONTROL: u16 = 1 << 9; +pub(crate) const CARET: u16 = 1 << 10; + +const LINE_ESCAPES: u16 = EOL | BACKSLASH; + +static CLASS: [u16; 256] = class_table(); + +const fn class_table() -> [u16; 256] { + let mut table = [0u16; 256]; + let mut byte = 0; + while byte < table.len() { + table[byte] = match byte as u8 { + b'\r' | b'\n' => EOL, + b' ' | b'\t' => WHITESPACE, + b'\\' => BACKSLASH, + b'"' => QUOTE, + b':' => COLON, + b';' => SEMICOLON, + b',' => COMMA, + b'=' => EQUAL, + b'.' => DOT, + b'^' => CARET, + 0x00..=0x08 | 0x0B | 0x0C | 0x0E..=0x1F | 0x7F => CONTROL, + _ => 0, + }; + byte += 1; + } + table +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Mask(u16); + +impl Mask { + #[inline(always)] + pub(crate) const fn new(classes: u16) -> Self { + Mask(classes | EOL) + } + + #[inline(always)] + pub(crate) fn class_of(self, byte: u8) -> u16 { + CLASS[byte as usize] & self.0 + } + + #[inline(always)] + pub(crate) fn plain_len(self, bytes: &[u8]) -> usize { + match bytes.first() { + Some(&byte) if self.class_of(byte) == 0 => { + if self.0 & !LINE_ESCAPES != 0 { + self.class_scan(bytes) + } else if self.0 & BACKSLASH != 0 { + memchr3(b'\r', b'\n', b'\\', bytes).unwrap_or(bytes.len()) + } else { + memchr2(b'\r', b'\n', bytes).unwrap_or(bytes.len()) + } + } + _ => 0, + } + } + + #[inline(always)] + fn class_scan(self, bytes: &[u8]) -> usize { + let (chunks, tail) = bytes.as_chunks::<4>(); + for (chunk_idx, &[a, b, c, d]) in chunks.iter().enumerate() { + let (a, b, c, d) = ( + self.class_of(a), + self.class_of(b), + self.class_of(c), + self.class_of(d), + ); + if a | b | c | d != 0 { + return chunk_idx * 4 + + if a != 0 { + 0 + } else if b != 0 { + 1 + } else if c != 0 { + 2 + } else { + 3 + }; + } + } + chunks.len() * 4 + + tail + .iter() + .position(|&byte| self.class_of(byte) != 0) + .unwrap_or(tail.len()) + } +} + diff --git a/crates/calcard/src/common/stack.rs b/crates/calcard/src/common/stack.rs new file mode 100644 index 00000000..5c76c719 --- /dev/null +++ b/crates/calcard/src/common/stack.rs @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const INLINE_DEPTH: usize = 8; + +pub(crate) struct ComponentStack { + inline: [Option; INLINE_DEPTH], + len: usize, + spilled: Vec, +} + +impl ComponentStack { + pub(crate) fn new() -> Self { + Self { + inline: [const { None }; INLINE_DEPTH], + len: 0, + spilled: Vec::new(), + } + } + + pub(crate) fn push(&mut self, item: T) { + match self.inline.get_mut(self.len) { + Some(slot) => *slot = Some(item), + None => self.spilled.push(item), + } + self.len += 1; + } + + pub(crate) fn pop(&mut self) -> Option { + self.len = self.len.checked_sub(1)?; + match self.inline.get_mut(self.len) { + Some(slot) => slot.take(), + None => self.spilled.pop(), + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.len == 0 + } +} diff --git a/crates/calcard/src/common/timezone.rs b/crates/calcard/src/common/timezone.rs new file mode 100644 index 00000000..adb1d6eb --- /dev/null +++ b/crates/calcard/src/common/timezone.rs @@ -0,0 +1,1052 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{DateTimeResult, PartialDateTime, tzdb}; +use hashify::map; +use jiff::{ + SignedDuration, Timestamp, + civil::{Date, DateTime, Time}, + tz::{AmbiguousOffset, Offset, TimeZone, TimeZoneDatabase}, +}; +use std::{borrow::Cow, hash::Hash, str::FromStr, sync::OnceLock}; + +/// The value type of the proprietary time zone name tables. +type TzName = &'static str; + +pub(crate) const SECONDS_PER_DAY: i64 = 86_400; + +/// A reference to the time zone a calendar value is interpreted in. +/// +/// The three forms map onto the three date-time forms of RFC 5545 section +/// 3.3.5: a floating value carries no zone at all, a fixed value carries a UTC +/// offset, and a named value references the IANA time zone database. +/// +/// `Tz` is a small `Copy` value. Named zones are stored as an identifier into +/// a committed table (see [`Tz::as_id`]) and resolved to a `jiff` time zone +/// once per process, so passing a `Tz` around never touches a reference count. +#[derive(Clone, Copy, Default, Eq)] +pub enum Tz { + /// No time zone: the value means the same wall clock reading everywhere. + #[default] + Floating, + /// A fixed offset from UTC. + Fixed(Offset), + /// A named IANA zone, held as an identifier into the time zone table. + Iana(u16), +} + +/// Resolved time zones, kept for the lifetime of the process. +static RESOLVED: [OnceLock>; tzdb::len()] = + [const { OnceLock::new() }; tzdb::len()]; + +impl Tz { + /// The UTC time zone. + pub const UTC: Self = Self::Iana(tzdb::UTC_ID); + + /// Returns the time zone for an IANA name, if the database knows it and + /// the name has an identifier assigned. + pub fn iana(name: &str) -> Option { + let tz = Self::Iana(tzdb::name_to_id(name)?); + tz.time_zone().is_some().then_some(tz) + } + + /// Resolves a named zone, caching it for the lifetime of the process. + /// + /// Returns `None` for floating and fixed zones, and for a name the time + /// zone database does not know. + pub fn time_zone(&self) -> Option<&'static TimeZone> { + let Self::Iana(id) = *self else { + return None; + }; + RESOLVED + .get(usize::from(id))? + .get_or_init(|| { + let name = tzdb::id_to_name(id)?; + TimeZone::get(name) + .or_else(|_| TimeZoneDatabase::bundled().get(name)) + .ok() + .or_else(|| Self::builtin_time_zone(name)) + }) + .as_ref() + } + + fn builtin_time_zone(name: &str) -> Option { + let name = name.strip_prefix("Etc/").unwrap_or(name); + if matches!( + name, + "UTC" | "UCT" | "Universal" | "Zulu" | "GMT" | "GMT0" | "GMT+0" | "GMT-0" | "Greenwich" + ) { + return Some(TimeZone::UTC); + } + let hours = name.strip_prefix("GMT")?.parse::().ok()?; + Offset::from_hours(-hours).ok().map(TimeZone::fixed) + } + + /// Returns the UTC offset in effect at an instant. + fn offset_at(&self, timestamp: Timestamp) -> Offset { + match self { + Self::Floating => Offset::UTC, + Self::Fixed(offset) => *offset, + Self::Iana(_) => self + .time_zone() + .map_or(Offset::UTC, |tz| tz.to_offset(timestamp)), + } + } + + /// Interprets a wall clock reading in this time zone. + pub fn from_local(&self, local: DateTime) -> Option { + self.interpret_local(local).map(|(zoned, _)| zoned) + } + + pub fn from_local_later(&self, local: DateTime) -> Option { + match self + .time_zone() + .map(|tz| tz.to_ambiguous_timestamp(local).offset()) + { + Some(AmbiguousOffset::Fold { after, .. }) => Some(ZonedDateTime { + local, + offset: after, + tz: *self, + }), + _ => self.from_local(local), + } + } + + pub(crate) fn interpret_local(&self, local: DateTime) -> Option<(ZonedDateTime, bool)> { + let (offset, in_gap) = match self { + Self::Floating => (Offset::UTC, false), + Self::Fixed(offset) => (*offset, false), + Self::Iana(_) => match self.time_zone()?.to_ambiguous_timestamp(local).offset() { + AmbiguousOffset::Unambiguous { offset } => (offset, false), + AmbiguousOffset::Fold { before, .. } => (before, false), + AmbiguousOffset::Gap { before, .. } => (before, true), + }, + }; + Some(( + ZonedDateTime { + local, + offset, + tz: *self, + }, + in_gap, + )) + } + + /// Interprets an instant in this time zone. + pub fn from_timestamp(&self, timestamp: Timestamp) -> ZonedDateTime { + let offset = self.offset_at(timestamp); + ZonedDateTime { + local: offset.to_datetime(timestamp), + offset, + tz: *self, + } + } + + /// Interprets a wall clock reading, kept for call sites that predate + /// [`Tz::from_local`] handling gaps itself. + #[inline] + pub fn resolve_local_datetime(&self, local: DateTime) -> Option { + self.from_local(local) + } + + /// Returns the identifier for this time zone. + pub fn as_id(&self) -> u16 { + match self { + Self::Iana(id) => *id, + Self::Fixed(offset) => { + let minutes = offset.seconds() / 60; + debug_assert!((-1440..=1440).contains(&minutes)); + 0x8000 | ((minutes + 1440) as u16) + } + Self::Floating => 0x8000, + } + } + + /// Returns the time zone an identifier refers to. + pub fn from_id(id: u16) -> Option { + if id & 0x8000 == 0 { + tzdb::id_to_name(id).map(|_| Self::Iana(id)) + } else if id == 0x8000 { + Some(Self::Floating) + } else { + let minutes = i32::from(id & 0x7FFF) - 1440; + (-1440..=1440) + .contains(&minutes) + .then(|| Offset::from_seconds(minutes * 60).ok()) + .flatten() + .map(Self::Fixed) + } + } + + /// Returns the IANA name of this time zone, if it has one. + pub fn name(&self) -> Option> { + match self { + Self::Iana(id) if *id == tzdb::UTC_ID => Some(Cow::Borrowed("Etc/UTC")), + Self::Iana(id) => tzdb::id_to_name(*id).map(Cow::Borrowed), + Self::Fixed(offset) => { + let hour = offset.seconds() / 3600; + Some(if hour != 0 { + // `Etc/GMT` names invert the sign of the offset. + Cow::Owned(format!( + "Etc/GMT{}{}", + if hour < 0 { "+" } else { "-" }, + hour.abs() + )) + } else { + Cow::Borrowed("Etc/UTC") + }) + } + Self::Floating => None, + } + } + + #[inline] + pub fn to_resolved(&self) -> Option { + match self { + Self::Floating => None, + _ => Some(*self), + } + } + + #[inline] + pub fn is_floating(&self) -> bool { + matches!(self, Self::Floating) + } + + /// Returns whether this zone is UTC, under any of its names. + pub fn is_utc(&self) -> bool { + match self { + Self::Iana(_) => self + .name() + .is_some_and(|name| matches!(name.as_ref(), "Etc/UTC" | "Etc/GMT" | "UTC" | "GMT")), + Self::Fixed(offset) => offset.seconds() == 0, + Self::Floating => false, + } + } + + /// Returns whether this zone has the same offset all year round. + pub fn has_fixed_offset(&self) -> bool { + match self { + Self::Floating | Self::Fixed(_) => true, + Self::Iana(_) => self + .time_zone() + .is_some_and(|tz| tz.following(Timestamp::MIN).next().is_none()), + } + } + + /// Returns the offset from UTC, in seconds, in effect on a date. + pub fn offset_from_utc_date(&self, utc: Date) -> i32 { + match self { + Self::Floating => 0, + Self::Fixed(offset) => offset.seconds(), + Self::Iana(_) => Offset::UTC + .to_timestamp(utc.to_datetime(Time::midnight())) + .map_or(0, |timestamp| self.offset_at(timestamp).seconds()), + } + } + + /// Returns the current offset from UTC, split into the parts a serialized + /// value is written from. + pub fn offset_parts(&self) -> PartialDateTime { + let mut seconds = self.offset_at(Timestamp::now()).seconds(); + + let tz_minus = if seconds < 0 { + seconds = -seconds; + true + } else { + false + }; + + PartialDateTime { + tz_hour: Some((seconds / 3600) as u8), + tz_minute: Some(((seconds % 3600) / 60) as u8), + tz_minus, + ..Default::default() + } + } +} + +impl PartialEq for Tz { + fn eq(&self, other: &Self) -> bool { + self.as_id() == other.as_id() + } +} + +impl PartialOrd for Tz { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Tz { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.as_id().cmp(&other.as_id()) + } +} + +impl Hash for Tz { + fn hash(&self, state: &mut H) { + self.as_id().hash(state); + } +} + +impl std::fmt::Debug for Tz { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +impl std::fmt::Display for Tz { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Floating => f.write_str("Floating"), + Self::Fixed(offset) => write!(f, "{offset}"), + Self::Iana(id) => match tzdb::id_to_name(*id) { + Some(name) => f.write_str(name), + None => write!(f, "Unknown({id})"), + }, + } + } +} + +/// A wall clock reading together with the time zone it belongs to. +#[derive(Clone, Copy)] +pub struct ZonedDateTime { + local: DateTime, + offset: Offset, + tz: Tz, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NominalDuration { + civil: SignedDuration, + exact: SignedDuration, +} + +impl NominalDuration { + pub const DAY: Self = Self::new(1, 0); + + pub const fn new(days: i32, exact_seconds: i64) -> Self { + Self { + civil: SignedDuration::from_secs(days as i64 * SECONDS_PER_DAY), + exact: SignedDuration::from_secs(exact_seconds), + } + } + + pub fn between(from: ZonedDateTime, to: ZonedDateTime) -> Self { + Self { + civil: SignedDuration::from_secs(to.naive_timestamp() - from.naive_timestamp()), + exact: SignedDuration::ZERO, + } + } +} + +impl ZonedDateTime { + /// Returns the wall clock reading in this value's own time zone. + #[inline] + pub fn naive_local(&self) -> DateTime { + self.local + } + + /// Returns the time zone this value is interpreted in. + #[inline] + pub fn timezone(&self) -> Tz { + self.tz + } + + /// Returns the offset from UTC in effect at this value's instant. + #[inline] + pub fn offset(&self) -> Offset { + self.offset + } + + /// Returns the instant this value refers to. + #[inline] + pub fn to_timestamp(&self) -> Timestamp { + self.offset + .to_timestamp(self.local) + .unwrap_or(if self.local < DateTime::default() { + Timestamp::MIN + } else { + Timestamp::MAX + }) + } + + /// Returns the instant this value refers to, in whole seconds since the + /// Unix epoch. + #[inline] + pub fn timestamp(&self) -> i64 { + self.to_timestamp().as_second() + } + + /// Returns the wall clock reading as seconds since the Unix epoch, as + /// though it were UTC. + #[inline] + pub fn naive_timestamp(&self) -> i64 { + Offset::UTC + .to_timestamp(self.local) + .map_or(0, |timestamp| timestamp.as_second()) + } + + /// Returns the calendar date this value falls on in its own zone. + #[inline] + pub fn date(&self) -> Date { + self.local.date() + } + + /// Returns how long has elapsed between another value and this one. + pub fn signed_duration_since(&self, earlier: Self) -> SignedDuration { + SignedDuration::from_secs(self.timestamp() - earlier.timestamp()) + } + + /// Returns the number of whole days between another value's date and this + /// one's, counted in each value's own zone. + pub fn days_since(&self, earlier: Self) -> i32 { + earlier + .local + .date() + .until(self.local.date()) + .map_or(0, |span| span.get_days()) + } + + /// Returns this same instant, read in another time zone. + pub fn with_timezone(&self, tz: Tz) -> Self { + if tz == self.tz { + return *self; + } + match tz { + // A floating value keeps the wall clock reading and drops the + // instant, which is the only way to move into a floating zone. + Tz::Floating => Self { + local: self.local, + offset: Offset::UTC, + tz, + }, + _ => tz.from_timestamp(self.to_timestamp()), + } + } + + /// Returns this value moved by an exact duration. + pub fn checked_add(&self, duration: SignedDuration) -> Option { + let timestamp = self.to_timestamp().checked_add(duration).ok()?; + Some(self.tz.from_timestamp(timestamp)) + } + + /// Returns this value moved by a calendar span. + pub fn checked_add_nominal(&self, duration: NominalDuration) -> Option { + let moved = if duration.civil.is_zero() { + *self + } else { + self.tz + .from_local(self.local.checked_add(duration.civil).ok()?)? + }; + if duration.exact.is_zero() { + Some(moved) + } else { + moved.checked_add(duration.exact) + } + } +} + +impl PartialEq for ZonedDateTime { + /// Compares the instant, so the same moment read in two zones is equal. + fn eq(&self, other: &Self) -> bool { + self.to_timestamp() == other.to_timestamp() + } +} + +impl Eq for ZonedDateTime {} + +impl PartialOrd for ZonedDateTime { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ZonedDateTime { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.to_timestamp().cmp(&other.to_timestamp()) + } +} + +impl Hash for ZonedDateTime { + fn hash(&self, state: &mut H) { + self.to_timestamp().hash(state); + } +} + +impl std::fmt::Display for ZonedDateTime { + /// Writes the value in the form RFC 3339 defines, always spelling out the + /// offset. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.write_rfc3339(f, false) + } +} + +impl std::fmt::Debug for ZonedDateTime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self} [{}]", self.tz) + } +} + +impl ZonedDateTime { + /// Writes the value in the form RFC 3339 defines, abbreviating a zero + /// offset as `Z`. + fn write_rfc3339(&self, f: &mut std::fmt::Formatter<'_>, zulu: bool) -> std::fmt::Result { + let seconds = self.offset.seconds(); + write!( + f, + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + self.local.year(), + self.local.month(), + self.local.day(), + self.local.hour(), + self.local.minute(), + self.local.second(), + )?; + if zulu && seconds == 0 { + return f.write_str("Z"); + } + let (sign, seconds) = if seconds < 0 { + ('-', -seconds) + } else { + ('+', seconds) + }; + write!( + f, + "{sign}{:02}:{:02}", + seconds / 3600, + (seconds % 3600) / 60 + ) + } +} + +#[cfg(any(all(test, stalwart_tests), feature = "serde"))] +impl serde::Serialize for ZonedDateTime { + fn serialize(&self, serializer: S) -> Result { + /// Serializes with `Z` for a zero offset, the form RFC 3339 prefers + /// and the one a serialized expansion is read back from. + struct Zulu<'x>(&'x ZonedDateTime); + + impl std::fmt::Display for Zulu<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.write_rfc3339(f, true) + } + } + + serializer.collect_str(&Zulu(self)) + } +} + +impl PartialDateTime { + pub fn to_date_time_with_tz(&self, tz: Tz) -> Option { + self.to_date_time() + .and_then(|dt| dt.to_date_time_with_tz(tz)) + } +} + +impl DateTimeResult { + /// Returns the time zone the value carried itself, if it carried one. + pub fn tz(&self) -> Option { + self.offset.map(|offset| { + if offset.seconds() == 0 { + Tz::UTC + } else { + Tz::Fixed(offset) + } + }) + } + + /// Interprets the value, preferring the offset it carries itself over the + /// zone supplied by the caller. + pub fn to_date_time_with_tz(&self, tz: Tz) -> Option { + match self.offset { + Some(offset) if offset.seconds() == 0 => Tz::UTC.from_local(self.date_time), + Some(offset) => Tz::Fixed(offset).from_local(self.date_time), + None => tz.from_local(self.date_time), + } + } + + /// Interprets the value the same way as [`Self::to_date_time_with_tz`]. + #[inline] + pub fn resolve_with_tz(&self, tz: Tz) -> Option { + self.to_date_time_with_tz(tz) + } +} + +impl Tz { + pub fn from_ms_cdo_zone_id(id: &str) -> Option { + // Source https://learn.microsoft.com/en-us/previous-versions/office/developer/exchange-server-2007/aa563018(v=exchg.80) + map!(id.as_bytes(), &'static str, + "0" => "UTC", + "1" => "Europe/London", + "10" => "America/New_York", + "11" => "America/Chicago", + "12" => "America/Denver", + "13" => "America/Los_Angeles", + "14" => "America/Anchorage", + "15" => "Pacific/Honolulu", + "16" => "Pacific/Midway", + "17" => "Pacific/Auckland", + "18" => "Australia/Brisbane", + "19" => "Australia/Adelaide", + "2" => "Europe/Lisbon", + "20" => "Asia/Tokyo", + "21" => "Asia/Singapore", + "22" => "Asia/Bangkok", + "23" => "Asia/Calcutta", + "24" => "Asia/Muscat", + "25" => "Asia/Tehran", + "26" => "Asia/Baghdad", + "27" => "Asia/Jerusalem", + "28" => "America/St_Johns", + "29" => "Atlantic/Azores", + "3" => "Europe/Paris", + "30" => "America/Noronha", + "31" => "Africa/Casablanca", + "32" => "America/Argentina/Buenos_Aires", + "33" => "America/Caracas", + "34" => "America/Indiana/Indianapolis", + "35" => "America/Bogota", + "36" => "America/Edmonton", + "37" => "America/Mexico_City", + "38" => "America/Phoenix", + "39" => "Pacific/Kwajalein", + "4" => "Europe/Berlin", + "40" => "Pacific/Fiji", + "41" => "Asia/Magadan", + "42" => "Australia/Hobart", + "43" => "Pacific/Guam", + "44" => "Australia/Darwin", + "45" => "Asia/Shanghai", + "46" => "Asia/Almaty", + "47" => "Asia/Karachi", + "48" => "Asia/Kabul", + "49" => "Africa/Cairo", + "5" => "Europe/Bucharest", + "50" => "Africa/Harare", + "51" => "Europe/Moscow", + "53" => "Atlantic/Cape_Verde", + "54" => "Asia/Baku", + "55" => "America/Guatemala", + "56" => "Africa/Nairobi", + "58" => "Asia/Yekaterinburg", + "59" => "Europe/Helsinki", + "6" => "Europe/Prague", + "60" => "America/Godthab", + "61" => "Asia/Rangoon", + "62" => "Asia/Kathmandu", + "63" => "Asia/Irkutsk", + "64" => "Asia/Krasnoyarsk", + "65" => "America/Santiago", + "66" => "Asia/Colombo", + "67" => "Pacific/Tongatapu", + "68" => "Asia/Vladivostok", + "69" => "Africa/Luanda", + "7" => "Europe/Athens", + "70" => "Asia/Yakutsk", + "71" => "Asia/Dhaka", + "72" => "Asia/Seoul", + "73" => "Australia/Perth", + "74" => "Asia/Kuwait", + "75" => "Asia/Taipei", + "76" => "Australia/Sydney", + "8" => "America/Sao_Paulo", + "9" => "America/Halifax", + ) + .copied() + .and_then(Tz::iana) + } +} + +impl FromStr for Tz { + type Err = (); + + /* + Calconnect recommends ignoring VTIMEZONE and just "guess" the timezone + + See: https://standards.calconnect.org/cc/cc-r0602-2006.html + + */ + + fn from_str(s: &str) -> Result { + // First try the name as given + if let Some(tz) = Tz::iana(s) { + return Ok(tz); + } + + // Strip common prefixes + let mut s = s.trim(); + let mut zone_offset = None; + let mut retry_iana = false; + if let Some(part) = s.strip_prefix('(') { + if let Some((zone, name)) = part.split_once(')') { + s = name.trim_start(); + zone_offset = Some(zone.trim()); + } + } else if let Some(mut name) = s.strip_prefix('/') { + /* + The presence of the SOLIDUS character as a prefix, indicates that + this "TZID" represents a unique ID in a globally defined time zone + registry (when such registry is defined). + */ + if name.as_bytes().iter().filter(|&&c| c == b'/').count() > 2 { + // Extract zones such as '/softwarestudio.org/Olson_20011030_5/America/Chicago' + if let Some(new_name) = name.splitn(3, '/').nth(2) { + name = new_name.strip_prefix("SystemV/").unwrap_or(new_name); + } + } + retry_iana = true; + s = name; + } + + // Try again with the stripped name + if retry_iana && let Some(tz) = Tz::iana(s) { + return Ok(tz); + } + + // Map proprietary timezones to IANA names + let result = hashify::map!(s.as_bytes(), TzName, + "AUS Central Standard Time" => "Australia/Darwin", + "AUS Central" => "Australia/Darwin", + "AUS Eastern Standard Time" => "Australia/Sydney", + "AUS Eastern" => "Australia/Sydney", + "Abu Dhabi, Muscat" => "Asia/Muscat", + "Adelaide, Central Australia" => "Australia/Adelaide", + "Afghanistan Standard Time" => "Asia/Kabul", + "Afghanistan" => "Asia/Kabul", + "Alaska" => "America/Anchorage", + "Alaskan Standard Time" => "America/Anchorage", + "Alaskan" => "America/Anchorage", + "Aleutian Standard Time" => "America/Adak", + "Almaty, Novosibirsk, North Central Asia" => "Asia/Almaty", + "Altai Standard Time" => "Asia/Barnaul", + "Amsterdam, Berlin, Bern, Rom, Stockholm, Wien" => "Europe/Berlin", + "Amsterdam, Berlin, Bern, Rome, Stockholm, Vienna" => "Europe/Berlin", + "Arab Standard Time" => "Asia/Riyadh", + "Arab" => "Asia/Kuwait", + "Arab, Kuwait, Riyadh" => "Asia/Kuwait", + "Arabian Standard Time" => "Asia/Dubai", + "Arabian" => "Asia/Muscat", + "Arabic Standard Time" => "Asia/Baghdad", + "Arabic" => "Asia/Baghdad", + "Argentina Standard Time" => "America/Buenos_Aires", + "Argentina" => "America/Argentina/Buenos_Aires", + "Arizona" => "America/Phoenix", + "Armenian" => "Asia/Yerevan", + "Astana, Dhaka" => "Asia/Dhaka", + "Astrakhan Standard Time" => "Europe/Astrakhan", + "Athens, Istanbul, Minsk" => "Europe/Athens", + "Atlantic Standard Time" => "America/Halifax", + "Atlantic Time (Canada)" => "America/Halifax", + "Atlantic" => "America/Halifax", + "Auckland, Wellington" => "Pacific/Auckland", + "Aus Central W. Standard Time" => "Australia/Eucla", + "Azerbaijan Standard Time" => "Asia/Baku", + "Azerbijan" => "Asia/Baku", + "Azores Standard Time" => "Atlantic/Azores", + "Azores" => "Atlantic/Azores", + "Baghdad" => "Asia/Baghdad", + "Bahia Standard Time" => "America/Bahia", + "Baku, Tbilisi, Yerevan" => "Asia/Baku", + "Bangkok, Hanoi, Jakarta" => "Asia/Bangkok", + "Bangladesh Standard Time" => "Asia/Dhaka", + "Beijing, Chongqing, Hong Kong SAR, Urumqi" => "Asia/Shanghai", + "Belarus Standard Time" => "Europe/Minsk", + "Belgrade, Pozsony, Budapest, Ljubljana, Prague" => "Europe/Prague", + "Bogota, Lima, Quito" => "America/Bogota", + "Bougainville Standard Time" => "Pacific/Bougainville", + "Brasilia" => "America/Sao_Paulo", + "Brisbane, East Australia" => "Australia/Brisbane", + "Brussels, Copenhagen, Madrid, Paris" => "Europe/Paris", + "Bucharest" => "Europe/Bucharest", + "Buenos Aires" => "America/Argentina/Buenos_Aires", + "Cairo" => "Africa/Cairo", + "Canada Central Standard Time" => "America/Regina", + "Canada Central" => "America/Edmonton", + "Canberra, Melbourne, Sydney, Hobart" => "Australia/Sydney", + "Canberra, Melbourne, Sydney" => "Australia/Sydney", + "Cape Verde Is." => "Atlantic/Cape_Verde", + "Cape Verde Standard Time" => "Atlantic/Cape_Verde", + "Cape Verde" => "Atlantic/Cape_Verde", + "Caracas, La Paz" => "America/Caracas", + "Casablanca, Monrovia" => "Africa/Casablanca", + "Caucasus Standard Time" => "Asia/Yerevan", + "Caucasus" => "Asia/Yerevan", + "Cen. Australia Standard Time" => "Australia/Adelaide", + "Cen. Australia" => "Australia/Adelaide", + "Central America Standard Time" => "America/Guatemala", + "Central America" => "America/Guatemala", + "Central Asia Standard Time" => "Asia/Almaty", + "Central Asia" => "Asia/Dhaka", + "Central Brazilian Standard Time" => "America/Cuiaba", + "Central Brazilian" => "America/Manaus", + "Central Europe Standard Time" => "Europe/Budapest", + "Central Europe" => "Europe/Prague", + "Central European Standard Time" => "Europe/Warsaw", + "Central European" => "Europe/Sarajevo", + "Central Pacific Standard Time" => "Pacific/Guadalcanal", + "Central Pacific" => "Asia/Magadan", + "Central Standard Time (Mexico)" => "America/Mexico_City", + "Central Standard Time" => "America/Chicago", + "Central Time (US & Canada)" => "America/Chicago", + "Central Time (US and Canada)" => "America/Chicago", + "Central" => "America/Chicago", + "Chatham Islands Standard Time" => "Pacific/Chatham", + "China Standard Time" => "Asia/Shanghai", + "China" => "Asia/Shanghai", + "Cuba Standard Time" => "America/Havana", + "Darwin" => "Australia/Darwin", + "Dateline Standard Time" => "Etc/GMT+12", + "Dateline" => "Etc/GMT-12", + "E. Africa Standard Time" => "Africa/Nairobi", + "E. Africa" => "Africa/Nairobi", + "E. Australia Standard Time" => "Australia/Brisbane", + "E. Australia" => "Australia/Brisbane", + "E. Europe Standard Time" => "Europe/Chisinau", + "E. Europe" => "Europe/Minsk", + "E. South America Standard Time" => "America/Sao_Paulo", + "E. South America" => "America/Belem", + "East Africa, Nairobi" => "Africa/Nairobi", + "Easter Island Standard Time" => "Pacific/Easter", + "Eastern Standard Time (Mexico)" => "America/Cancun", + "Eastern Standard Time" => "America/New_York", + "Eastern Time (US & Canada)" => "America/New_York", + "Eastern Time (US and Canada)" => "America/New_York", + "Eastern" => "America/New_York", + "Egypt Standard Time" => "Africa/Cairo", + "Egypt" => "Africa/Cairo", + "Ekaterinburg Standard Time" => "Asia/Yekaterinburg", + "Ekaterinburg" => "Asia/Yekaterinburg", + "Eniwetok, Kwajalein, Dateline Time" => "Pacific/Kwajalein", + "FLE Standard Time" => "Europe/Kiev", + "FLE" => "Europe/Helsinki", + "Fiji Islands, Kamchatka, Marshall Is." => "Pacific/Fiji", + "Fiji Standard Time" => "Pacific/Fiji", + "Fiji" => "Pacific/Fiji", + "GMT Standard Time" => "Europe/London", + "GTB Standard Time" => "Europe/Bucharest", + "GTB" => "Europe/Athens", + "Georgian Standard Time" => "Asia/Tbilisi", + "Georgian" => "Asia/Tbilisi", + "Greenland Standard Time" => "America/Godthab", + "Greenland" => "America/Godthab", + "Greenwich Mean Time: Dublin, Edinburgh, Lisbon, London" => "Europe/Lisbon", + "Greenwich Mean Time; Dublin, Edinburgh, London" => "Europe/London", + "Greenwich Standard Time" => "Atlantic/Reykjavik", + "Greenwich" => "Atlantic/Reykjavik", + "Guam, Port Moresby" => "Pacific/Guam", + "Haiti Standard Time" => "America/Port-au-Prince", + "Harare, Pretoria" => "Africa/Harare", + "Hawaii" => "Pacific/Honolulu", + "Hawaiian Standard Time" => "Pacific/Honolulu", + "Hawaiian" => "Pacific/Honolulu", + "Helsinki, Riga, Tallinn" => "Europe/Helsinki", + "Hobart, Tasmania" => "Australia/Hobart", + "India Standard Time" => "Asia/Calcutta", + "India" => "Asia/Calcutta", + "Indiana (East)" => "America/Indiana/Indianapolis", + "Iran Standard Time" => "Asia/Tehran", + "Iran" => "Asia/Tehran", + "Irkutsk, Ulaan Bataar" => "Asia/Irkutsk", + "Islamabad, Karachi, Tashkent" => "Asia/Karachi", + "Israel Standard Time" => "Asia/Jerusalem", + "Israel" => "Asia/Jerusalem", + "Israel, Jerusalem Standard Time" => "Asia/Jerusalem", + "Jordan Standard Time" => "Asia/Amman", + "Jordan" => "Asia/Amman", + "Kabul" => "Asia/Kabul", + "Kaliningrad Standard Time" => "Europe/Kaliningrad", + "Kathmandu, Nepal" => "Asia/Kathmandu", + "Kolkata, Chennai, Mumbai, New Delhi, India Standard Time" => "Asia/Calcutta", + "Korea Standard Time" => "Asia/Seoul", + "Korea" => "Asia/Seoul", + "Krasnoyarsk" => "Asia/Krasnoyarsk", + "Kuala Lumpur, Singapore" => "Asia/Singapore", + "Libya Standard Time" => "Africa/Tripoli", + "Line Islands Standard Time" => "Pacific/Kiritimati", + "Lord Howe Standard Time" => "Australia/Lord_Howe", + "Magadan Standard Time" => "Asia/Magadan", + "Magadan, Solomon Is., New Caledonia" => "Asia/Magadan", + "Magallanes Standard Time" => "America/Punta_Arenas", + "Marquesas Standard Time" => "Pacific/Marquesas", + "Mauritius Standard Time" => "Indian/Mauritius", + "Mauritius" => "Indian/Mauritius", + "Mexico City, Tegucigalpa" => "America/Mexico_City", + "Mexico Standard Time 2" => "America/Chihuahua", + "Mexico" => "America/Mexico_City", + "Mid-Atlantic" => "America/Noronha", + "Middle East Standard Time" => "Asia/Beirut", + "Middle East" => "Asia/Beirut", + "Midway Island, Samoa" => "Pacific/Midway", + "Montevideo Standard Time" => "America/Montevideo", + "Montevideo" => "America/Montevideo", + "Morocco Standard Time" => "Africa/Casablanca", + "Morocco" => "Africa/Casablanca", + "Moscow, St. Petersburg, Volgograd" => "Europe/Moscow", + "Mountain Standard Time (Mexico)" => "America/Chihuahua", + "Mountain Standard Time" => "America/Denver", + "Mountain Time (US & Canada)" => "America/Denver", + "Mountain Time (US and Canada)" => "America/Denver", + "Mountain" => "America/Denver", + "Myanmar Standard Time" => "Asia/Rangoon", + "Myanmar" => "Asia/Rangoon", + "N. Central Asia Standard Time" => "Asia/Novosibirsk", + "N. Central Asia" => "Asia/Almaty", + "Namibia Standard Time" => "Africa/Windhoek", + "Namibia" => "Africa/Windhoek", + "Nepal Standard Time" => "Asia/Katmandu", + "Nepal" => "Asia/Kathmandu", + "New Zealand Standard Time" => "Pacific/Auckland", + "New Zealand" => "Pacific/Auckland", + "Newfoundland Standard Time" => "America/St_Johns", + "Newfoundland" => "America/St_Johns", + "Norfolk Standard Time" => "Pacific/Norfolk", + "North Asia East Standard Time" => "Asia/Irkutsk", + "North Asia East" => "Asia/Irkutsk", + "North Asia Standard Time" => "Asia/Krasnoyarsk", + "North Asia" => "Asia/Krasnoyarsk", + "North Korea Standard Time" => "Asia/Pyongyang", + "Omsk Standard Time" => "Asia/Omsk", + "Osaka, Sapporo, Tokyo" => "Asia/Tokyo", + "Japan" => "Asia/Tokyo", + "Pacific SA Standard Time" => "America/Santiago", + "Pacific SA" => "America/Santiago", + "Pacific Standard Time (Mexico)" => "America/Tijuana", + "Pacific Standard Time" => "America/Los_Angeles", + "Pacific Time (US & Canada)" => "America/Los_Angeles", + "Pacific Time (US & Canada); Tijuana" => "America/Los_Angeles", + "Pacific Time (US and Canada)" => "America/Los_Angeles", + "Pacific Time (US and Canada); Tijuana" => "America/Los_Angeles", + "Pacific" => "America/Los_Angeles", + "Pakistan Standard Time" => "Asia/Karachi", + "Pakistan" => "Asia/Karachi", + "Paraguay Standard Time" => "America/Asuncion", + "Paris, Madrid, Brussels, Copenhagen" => "Europe/Paris", + "Perth, Western Australia" => "Australia/Perth", + "Prague, Central Europe" => "Europe/Prague", + "Qyzylorda Standard Time" => "Asia/Qyzylorda", + "Rangoon" => "Asia/Rangoon", + "Romance Standard Time" => "Europe/Paris", + "Romance" => "Europe/Paris", + "Russia Time Zone 10" => "Asia/Srednekolymsk", + "Russia Time Zone 11" => "Asia/Kamchatka", + "Russia Time Zone 3" => "Europe/Samara", + "Russian Standard Time" => "Europe/Moscow", + "Russian" => "Europe/Moscow", + "SA Eastern Standard Time" => "America/Cayenne", + "SA Eastern" => "America/Belem", + "SA Pacific Standard Time" => "America/Bogota", + "SA Pacific" => "America/Bogota", + "SA Western Standard Time" => "America/La_Paz", + "SA Western" => "America/La_Paz", + "SE Asia Standard Time" => "Asia/Bangkok", + "SE Asia" => "Asia/Bangkok", + "Saint Pierre Standard Time" => "America/Miquelon", + "Sakhalin Standard Time" => "Asia/Sakhalin", + "Samoa Standard Time" => "Pacific/Apia", + "Samoa" => "Pacific/Apia", + "Santiago" => "America/Santiago", + "Sao Tome Standard Time" => "Africa/Sao_Tome", + "Sarajevo, Skopje, Sofija, Vilnius, Warsaw, Zagreb" => "Europe/Sarajevo", + "Saratov Standard Time" => "Europe/Saratov", + "Saskatchewan" => "America/Edmonton", + "Seoul, Korea Standard time" => "Asia/Seoul", + "Singapore Standard Time" => "Asia/Singapore", + "Singapore" => "Asia/Singapore", + "South Africa Standard Time" => "Africa/Johannesburg", + "South Africa" => "Africa/Harare", + "Sri Jayawardenepura, Sri Lanka" => "Asia/Colombo", + "Sri Lanka Standard Time" => "Asia/Colombo", + "Sri Lanka" => "Asia/Colombo", + "Sudan Standard Time" => "Africa/Khartoum", + "Syria Standard Time" => "Asia/Damascus", + "Taipei Standard Time" => "Asia/Taipei", + "Taipei" => "Asia/Taipei", + "Tasmania Standard Time" => "Australia/Hobart", + "Tasmania" => "Australia/Hobart", + "Tehran" => "Asia/Tehran", + "Tocantins Standard Time" => "America/Araguaina", + "Tokyo Standard Time" => "Asia/Tokyo", + "Tokyo" => "Asia/Tokyo", + "Tomsk Standard Time" => "Asia/Tomsk", + "Tonga Standard Time" => "Pacific/Tongatapu", + "Tonga" => "Pacific/Tongatapu", + "Transbaikal Standard Time" => "Asia/Chita", + "Turkey Standard Time" => "Europe/Istanbul", + "Turks And Caicos Standard Time" => "America/Grand_Turk", + "US Eastern Standard Time" => "America/Indianapolis", + "US Eastern" => "America/Indiana/Indianapolis", + "US Mountain Standard Time" => "America/Phoenix", + "US Mountain" => "America/Phoenix", + "UTC" => "Etc/GMT", + "UTC+12" => "Etc/GMT-12", + "UTC+13" => "Etc/GMT-13", + "UTC-02" => "Etc/GMT+2", + "UTC-08" => "Etc/GMT+8", + "UTC-09" => "Etc/GMT+9", + "UTC-11" => "Etc/GMT+11", + "Ulaanbaatar Standard Time" => "Asia/Ulaanbaatar", + "Universal Coordinated Time" => "UTC", + "Venezuela Standard Time" => "America/Caracas", + "Venezuela" => "America/Caracas", + "Vladivostok Standard Time" => "Asia/Vladivostok", + "Vladivostok" => "Asia/Vladivostok", + "Volgograd Standard Time" => "Europe/Volgograd", + "W. Australia Standard Time" => "Australia/Perth", + "W. Australia" => "Australia/Perth", + "W. Central Africa Standard Time" => "Africa/Lagos", + "W. Central Africa" => "Africa/Lagos", + "W. Europe Standard Time" => "Europe/Berlin", + "W. Europe" => "Europe/Amsterdam", + "W. Mongolia Standard Time" => "Asia/Hovd", + "West Asia Standard Time" => "Asia/Tashkent", + "West Asia" => "Asia/Tashkent", + "West Bank Standard Time" => "Asia/Hebron", + "West Central Africa" => "Africa/Luanda", + "West Pacific Standard Time" => "Pacific/Port_Moresby", + "West Pacific" => "Pacific/Guam", + "Yakutsk Standard Time" => "Asia/Yakutsk", + "Yakutsk" => "Asia/Yakutsk", + "Yukon Standard Time" => "America/Whitehorse", + "Nuku'alofa, Tonga" => "Pacific/Tongatapu", + ); + + if let Some(name) = result { + if let Some(tz) = Tz::iana(name) { + return Ok(tz); + } + } else if let Some(zone_offset) = zone_offset { + let (zone, sign, time) = if let Some((zone, part)) = zone_offset.split_once('+') { + (zone.trim(), '-', part.trim()) + } else if let Some((zone, part)) = zone_offset.split_once('-') { + (zone.trim(), '+', part.trim()) + } else { + return Err(()); + }; + if !zone.eq_ignore_ascii_case("UTC") && !zone.eq_ignore_ascii_case("GMT") { + return Err(()); + } + let mut zone = String::with_capacity(10); + zone.push_str("Etc/GMT"); + zone.push(sign); + + for (pos, ch) in time.chars().enumerate() { + if !ch.is_ascii_digit() { + break; + } else if ch != '0' || pos > 0 { + zone.push(ch); + } + } + + if let Some(tz) = Tz::iana(&zone) { + return Ok(tz); + } + } + Err(()) + } +} + diff --git a/crates/calcard/src/common/tokenizer.rs b/crates/calcard/src/common/tokenizer.rs new file mode 100644 index 00000000..b4ea7d65 --- /dev/null +++ b/crates/calcard/src/common/tokenizer.rs @@ -0,0 +1,784 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Parser, + common::{ + Encoding, IanaParse, + scan::{self, Mask}, + }, +}; +use encodify::base64; +use std::ops::{Deref, Range}; + +const SPARE_BYTES: usize = 64; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Token<'x> { + pub(crate) text: TokenText<'x>, + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) stop_char: StopChar, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum TokenText<'x> { + Borrowed(&'x str), + Owned(String), +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub(crate) enum StopChar { + Colon, + Semicolon, + Comma, + Equal, + Dot, + Lf, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Mode(u16); + +impl Mode { + pub(crate) const STOP_COLON: Mode = Mode(scan::COLON); + pub(crate) const STOP_SEMICOLON: Mode = Mode(scan::SEMICOLON); + pub(crate) const STOP_COMMA: Mode = Mode(scan::COMMA); + pub(crate) const STOP_EQUAL: Mode = Mode(scan::EQUAL); + pub(crate) const STOP_DOT: Mode = Mode(scan::DOT); + pub(crate) const UNFOLD_QP: Mode = Mode(1 << 11); + pub(crate) const UNFOLD_B64: Mode = Mode(1 << 12); + pub(crate) const UNQUOTE: Mode = Mode(scan::QUOTE); + pub(crate) const SKIP_WS: Mode = Mode(scan::WHITESPACE); + pub(crate) const STRIP_CTL: Mode = Mode(scan::CONTROL); + pub(crate) const UNESCAPE_CARET: Mode = Mode(scan::CARET); + pub(crate) const UNESCAPE_BACKSLASH: Mode = Mode(scan::BACKSLASH); + + pub(crate) const INITIAL: Mode = Mode( + Mode::STOP_COLON.0 + | Mode::STOP_SEMICOLON.0 + | Mode::STOP_COMMA.0 + | Mode::STOP_EQUAL.0 + | Mode::UNQUOTE.0 + | Mode::UNESCAPE_BACKSLASH.0, + ); + const IANA_TOKEN: Mode = Mode( + Mode::STOP_COLON.0 + | Mode::STOP_SEMICOLON.0 + | Mode::STOP_COMMA.0 + | Mode::STOP_EQUAL.0 + | Mode::UNQUOTE.0 + | Mode::SKIP_WS.0 + | Mode::UNESCAPE_BACKSLASH.0, + ); + const SINGLE_VALUE: Mode = Mode::UNESCAPE_BACKSLASH; + const MULTI_VALUE_COMMA: Mode = + Mode(Mode::STOP_COMMA.0 | Mode::SKIP_WS.0 | Mode::UNESCAPE_BACKSLASH.0); + const MULTI_VALUE_SEMICOLON: Mode = Mode(Mode::STOP_SEMICOLON.0 | Mode::UNESCAPE_BACKSLASH.0); + const MULTI_VALUE_SEMICOLON_AND_COMMA: Mode = + Mode(Mode::STOP_SEMICOLON.0 | Mode::STOP_COMMA.0 | Mode::UNESCAPE_BACKSLASH.0); + const PARAM_VALUE: Mode = Mode( + Mode::STOP_COLON.0 + | Mode::STOP_SEMICOLON.0 + | Mode::STOP_COMMA.0 + | Mode::UNQUOTE.0 + | Mode::SKIP_WS.0 + | Mode::STRIP_CTL.0 + | Mode::UNESCAPE_CARET.0 + | Mode::UNESCAPE_BACKSLASH.0, + ); + const RRULE_VALUE: Mode = Mode( + Mode::STOP_COLON.0 + | Mode::STOP_COMMA.0 + | Mode::STOP_EQUAL.0 + | Mode::STOP_SEMICOLON.0 + | Mode::SKIP_WS.0 + | Mode::UNESCAPE_BACKSLASH.0, + ); + + const PLAIN_VALUE: Mode = Mode(Mode::UNESCAPE_BACKSLASH.0 | Mode::UNFOLD_B64.0); + + const UNFOLD: u16 = Self::UNFOLD_QP.0 | Self::UNFOLD_B64.0; + const UNQUOTED_ONLY: u16 = + scan::WHITESPACE | scan::COLON | scan::SEMICOLON | scan::COMMA | scan::EQUAL | scan::DOT; + + #[inline(always)] + pub(crate) const fn contains(self, flag: Mode) -> bool { + self.0 & flag.0 == flag.0 + } + + #[inline(always)] + pub(crate) fn set(&mut self, flag: Mode, enabled: bool) { + if enabled { + self.0 |= flag.0; + } else { + self.0 &= !flag.0; + } + } + + #[inline(always)] + const fn unquoted_mask(self) -> Mask { + Mask::new(self.0 & !Self::UNFOLD) + } + + #[inline(always)] + const fn quoted_mask(self) -> Mask { + Mask::new(self.0 & !Self::UNFOLD & !Self::UNQUOTED_ONLY) + } +} + +impl<'x> Parser<'x> { + pub(crate) fn expect_iana_token(&mut self) { + self.mode = Mode::IANA_TOKEN; + } + + pub(crate) fn expect_single_value(&mut self) { + self.mode = Mode::SINGLE_VALUE; + } + + pub(crate) fn expect_multi_value_comma(&mut self) { + self.mode = Mode::MULTI_VALUE_COMMA; + } + + pub(crate) fn expect_multi_value_semicolon(&mut self) { + self.mode = Mode::MULTI_VALUE_SEMICOLON; + } + + pub(crate) fn expect_multi_value_semicolon_and_comma(&mut self) { + self.mode = Mode::MULTI_VALUE_SEMICOLON_AND_COMMA; + } + + pub(crate) fn expect_param_value(&mut self) { + self.mode = Mode::PARAM_VALUE; + } + + pub(crate) fn expect_rrule_value(&mut self) { + self.mode = Mode::RRULE_VALUE; + } + + fn caret_escape(&self, idx: usize) -> Option<(usize, char)> { + let mut bytes = self.input.get(idx + 1..)?.iter().zip(idx + 1..); + while let Some((ch, pos)) = bytes.next() { + match ch { + b'\r' if matches!(self.input.get(pos + 1), Some(b'\n')) => {} + b'\n' => { + bytes.next().filter(|(ch, _)| matches!(ch, b' ' | b'\t'))?; + } + b'^' => return Some((pos, '^')), + b'n' => return Some((pos, '\n')), + b'\'' => return Some((pos, '"')), + _ => return None, + } + } + None + } + + fn is_base64_continuation(&self, idx: usize) -> bool { + let mut has_data = false; + + for ch in self.input.get(idx + 1..).unwrap_or_default() { + match ch { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'+' | b'/' | b'=' => has_data = true, + b'\r' => {} + b'\n' => break, + _ => return false, + } + } + + has_data + } + + #[inline(always)] + fn borrowed(&self, start: usize, end: usize) -> TokenText<'x> { + TokenText::Borrowed(self.source.get(start..end).unwrap_or_default()) + } + + pub(crate) fn token(&mut self) -> Option> { + let input = self.input; + let start = self.pos; + let mask = self.mode.unquoted_mask(); + let rest = input.get(start..).unwrap_or_default(); + let run = mask.plain_len(rest); + let end = start + run; + let Some(&ch) = rest.get(run) else { + if run == 0 { + return None; + } + self.pos = end; + self.last_token_end = end - 1; + return Some(Token { + text: self.borrowed(start, end), + start, + end: end - 1, + stop_char: StopChar::Lf, + }); + }; + let (stop_char, next) = match mask.class_of(ch) { + scan::COLON => (StopChar::Colon, end + 1), + scan::SEMICOLON => (StopChar::Semicolon, end + 1), + scan::COMMA => (StopChar::Comma, end + 1), + scan::EQUAL => (StopChar::Equal, end + 1), + scan::DOT => (StopChar::Dot, end + 1), + scan::EOL => { + let lf = if ch == b'\n' { + end + } else if input.get(end + 1) == Some(&b'\n') { + end + 1 + } else { + return self.token_slow(start, run); + }; + if matches!(input.get(lf + 1), Some(b' ' | b'\t')) + || (run != 0 + && self.mode.contains(Mode::UNFOLD_QP) + && input.get(end - 1) == Some(&b'=')) + || self.mode.contains(Mode::UNFOLD_B64) + { + return self.token_slow(start, run); + } + (StopChar::Lf, lf + 1) + } + scan::QUOTE if run == 0 => return self.quoted_token(start), + _ => return self.token_slow(start, run), + }; + self.pos = next; + Some(if run != 0 { + self.last_token_end = end - 1; + Token { + text: self.borrowed(start, end), + start, + end: end - 1, + stop_char, + } + } else { + Token { + text: TokenText::Borrowed(""), + start: next - 1, + end: next - 1, + stop_char, + } + }) + } + + #[inline(always)] + fn quoted_token(&mut self, quote: usize) -> Option> { + let input = self.input; + let inner_start = quote + 1; + let inner = input.get(inner_start..).unwrap_or_default(); + let inner_len = self.mode.quoted_mask().plain_len(inner); + let close = inner_start + inner_len; + let stop_char = match input.get(close..close + 2) { + Some(&[b'"', stop]) => match self.mode.unquoted_mask().class_of(stop) { + scan::COLON => StopChar::Colon, + scan::SEMICOLON => StopChar::Semicolon, + scan::COMMA => StopChar::Comma, + scan::EQUAL => StopChar::Equal, + scan::DOT => StopChar::Dot, + _ => return self.token_slow(quote, 0), + }, + _ => return self.token_slow(quote, 0), + }; + self.pos = close + 2; + Some(if inner_len != 0 { + self.last_token_end = close - 1; + Token { + text: self.borrowed(inner_start, close), + start: inner_start, + end: close - 1, + stop_char, + } + } else { + Token { + text: TokenText::Borrowed(""), + start: close + 1, + end: close + 1, + stop_char, + } + }) + } + + #[inline(never)] + fn token_slow(&mut self, start: usize, run: usize) -> Option> { + let input = self.input; + let source = self.source; + let unquoted_mask = self.mode.unquoted_mask(); + let quoted_mask = self.mode.quoted_mask(); + let mut mask = unquoted_mask; + let mut pos = start + run; + let mut offset_start = if run != 0 { start } else { usize::MAX }; + let mut offset_end = if run != 0 { + start + run - 1 + } else { + usize::MAX + }; + let mut last_idx = 0; + let mut in_quote = false; + let mut buf = OwnedText::default(); + let mut skipped_ws = usize::MAX; + + let stop_char = 'outer: loop { + let Some(&ch) = input.get(pos) else { + if offset_start != usize::MAX { + break StopChar::Lf; + } + self.pos = pos; + return None; + }; + let idx = pos; + pos += 1; + last_idx = idx; + + 'special: { + let content = match mask.class_of(ch) { + scan::EOL => { + let idx = if ch == b'\n' { + idx + } else if input.get(pos) == Some(&b'\n') { + last_idx = pos; + pos += 1; + last_idx + } else { + break 'special; + }; + if self.mode.contains(Mode::UNFOLD_QP) + && buf.last().or_else(|| input.get(offset_end).copied()) == Some(b'=') + { + offset_end = idx; + + if !buf.is_empty() { + buf.push('\n'); + } + } else if let Some(b' ' | b'\t') = input.get(pos) { + pos += 1; + if buf.is_empty() && offset_start != usize::MAX { + buf.start(source.get(offset_start..=offset_end)); + } + + if skipped_ws != usize::MAX && !buf.is_empty() { + buf.restore_whitespace(input.get(skipped_ws..idx)); + } + skipped_ws = usize::MAX; + } else if self.mode.contains(Mode::UNFOLD_B64) + && self.is_base64_continuation(idx) + { + if buf.is_empty() && offset_start != usize::MAX { + buf.start(source.get(offset_start..=offset_end)); + } + skipped_ws = usize::MAX; + } else { + break 'outer StopChar::Lf; + } + break 'special; + } + scan::WHITESPACE => { + if !buf.last().is_some_and(|last| last != ch) { + if skipped_ws == usize::MAX { + skipped_ws = idx; + } + break 'special; + } + if ch == b'\t' { '\t' } else { ' ' } + } + scan::BACKSLASH => { + let mut next_ch = b'\\'; + let mut next_offset_end = idx; + while let Some(&escaped) = input.get(pos) { + let escaped_idx = pos; + pos += 1; + match escaped { + b'\t' | b'\r' => {} + b'\n' => { + if let Some(b' ' | b'\t') = input.get(pos) { + pos += 1; + if let Some(&folded) = input.get(pos) { + next_ch = folded; + next_offset_end = pos; + pos += 1; + break; + } + } else { + offset_end = escaped_idx - 1; + break 'outer StopChar::Lf; + } + } + _ => { + next_ch = escaped; + next_offset_end = escaped_idx; + break; + } + } + } + if offset_start != usize::MAX { + if buf.is_empty() { + buf.start(source.get(offset_start..=offset_end)); + } + + if skipped_ws != usize::MAX { + buf.restore_whitespace(input.get(skipped_ws..idx)); + } + } else { + offset_start = next_offset_end; + } + skipped_ws = usize::MAX; + match next_ch { + b'n' | b'N' => buf.push('\n'), + b'r' | b'R' => buf.push('\r'), + ascii if ascii.is_ascii() => buf.push(char::from(ascii)), + _ => { + if let Some(escaped) = source + .get(next_offset_end..) + .and_then(|rest| rest.chars().next()) + { + buf.push(escaped); + pos = next_offset_end + escaped.len_utf8(); + next_offset_end = pos - 1; + } + } + } + offset_end = next_offset_end; + break 'special; + } + scan::QUOTE => { + in_quote = !in_quote; + mask = if in_quote { quoted_mask } else { unquoted_mask }; + break 'special; + } + scan::COLON => break 'outer StopChar::Colon, + scan::SEMICOLON => break 'outer StopChar::Semicolon, + scan::COMMA => break 'outer StopChar::Comma, + scan::EQUAL => break 'outer StopChar::Equal, + scan::DOT => break 'outer StopChar::Dot, + scan::CONTROL => { + if buf.is_empty() && offset_start != usize::MAX { + buf.start(source.get(offset_start..=offset_end)); + } + break 'special; + } + scan::CARET => { + if let Some((escape_idx, decoded)) = self.caret_escape(idx) { + if offset_start != usize::MAX { + if buf.is_empty() { + buf.start(source.get(offset_start..=offset_end)); + } + if skipped_ws != usize::MAX { + buf.restore_whitespace(input.get(skipped_ws..idx)); + } + } else { + offset_start = escape_idx; + } + skipped_ws = usize::MAX; + buf.push(decoded); + offset_end = escape_idx; + pos = escape_idx + 1; + break 'special; + } + '^' + } + _ => { + pos = idx; + break 'special; + } + }; + + if offset_start == usize::MAX { + offset_start = idx; + } + offset_end = idx; + skipped_ws = usize::MAX; + + if !buf.is_empty() { + buf.push(content); + } + } + + let run = mask.plain_len(input.get(pos..).unwrap_or_default()); + if run != 0 { + if offset_start == usize::MAX { + offset_start = pos; + } + if !buf.is_empty() { + buf.push_str(source.get(pos..pos + run)); + } + pos += run; + offset_end = pos - 1; + skipped_ws = usize::MAX; + } + }; + + self.pos = pos; + + if offset_start != usize::MAX { + self.last_token_end = offset_end; + } + + let (text, start, end) = if !buf.is_empty() { + (buf.into_text(), offset_start, offset_end) + } else if offset_start != usize::MAX { + ( + TokenText::Borrowed(source.get(offset_start..=offset_end).unwrap_or_default()), + offset_start, + offset_end, + ) + } else { + (TokenText::Borrowed(""), last_idx, last_idx) + }; + Some(Token { + text, + start, + end, + stop_char, + }) + } + + #[inline] + pub(crate) fn token_until_lf(&mut self, last_stop_char: &mut StopChar) -> Option> { + if last_stop_char != &StopChar::Lf { + self.token() + .inspect(|token| *last_stop_char = token.stop_char) + } else { + None + } + } + + #[inline] + pub(crate) fn parse_value_until_lf( + &mut self, + separator: StopChar, + last_stop_char: &mut StopChar, + ) -> Option> + where + T: IanaParse + 'static, + { + if *last_stop_char != separator { + self.token_until_lf(last_stop_char) + .map(|token| T::parse(token.text.as_ref())) + } else { + None + } + } + + #[inline(always)] + pub(crate) fn value_token( + &mut self, + encoding: Option, + payload: &mut Option>, + ) -> Option> { + if encoding == Some(Encoding::Base64) { + self.base64_token(payload) + } else { + self.token() + } + } + + #[inline(never)] + fn base64_token(&mut self, payload: &mut Option>) -> Option> { + if let Some((token, next, bytes)) = self.folded_base64() { + self.pos = next; + self.last_token_end = token.end; + *payload = Some(bytes); + return Some(token); + } + self.token() + } + + fn folded_base64(&self) -> Option<(Token<'x>, usize, Vec)> { + if self.mode.0 & !Mode::PLAIN_VALUE.0 != 0 { + return None; + } + let start = self.pos; + let value = self.input.get(start..)?; + let mut bytes = Vec::new(); + let folded = base64::STANDARD.decode_folded(value, &mut bytes).ok()?; + if folded.next != folded.end + && self.mode.contains(Mode::UNFOLD_B64) + && self.is_base64_continuation(start + folded.next - 1) + { + return None; + } + let span = FoldedContent(value.get(..folded.end)?).span(); + if span.is_empty() { + return None; + } + if bytes.capacity() > 2 * bytes.len() + SPARE_BYTES { + bytes.shrink_to_fit(); + } + let (first, end) = (start + span.start, start + span.end); + Some(( + Token { + text: TokenText::Borrowed(self.source.get(first..end)?), + start: first, + end: end - 1, + stop_char: StopChar::Lf, + }, + start + folded.next, + bytes, + )) + } + + pub(crate) fn seek_lf(&mut self) -> bool { + loop { + match self.token() { + Some(Token { + stop_char: StopChar::Lf, + .. + }) => return true, + None => return false, + _ => {} + } + } + } + + pub(crate) fn seek_value_or_eol(&mut self) -> StopChar { + loop { + match self.token() { + Some(Token { + stop_char: StopChar::Colon, + .. + }) => return StopChar::Colon, + Some(Token { + stop_char: StopChar::Lf, + .. + }) + | None => return StopChar::Lf, + _ => {} + } + } + } + + pub(crate) fn seek_param_value_or_eol(&mut self) -> StopChar { + loop { + match self.token() { + Some(Token { + stop_char: stop_char @ (StopChar::Colon | StopChar::Semicolon | StopChar::Equal), + .. + }) => return stop_char, + Some(Token { + stop_char: StopChar::Lf, + .. + }) + | None => return StopChar::Lf, + _ => {} + } + } + } +} + +struct FoldedContent<'x>(&'x [u8]); + +impl FoldedContent<'_> { + fn span(&self) -> Range { + let mut rest = self.0; + loop { + rest = match rest { + [b'\r', b'\n', b' ' | b'\t', tail @ ..] | [b'\n', b' ' | b'\t', tail @ ..] => tail, + _ => break, + }; + } + let first = self.0.len() - rest.len(); + loop { + rest = match rest { + [head @ .., b'\r', b'\n', b' ' | b'\t'] => head, + [head @ .., b'\n', b' ' | b'\t'] => head, + _ => return first..first + rest.len(), + }; + } + } +} + +#[derive(Default)] +struct OwnedText(String); + +impl OwnedText { + #[inline(always)] + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline(always)] + fn last(&self) -> Option { + self.0.as_bytes().last().copied() + } + + #[inline(always)] + fn push(&mut self, ch: char) { + self.0.push(ch); + } + + #[inline(always)] + fn push_str(&mut self, run: Option<&str>) { + self.0.push_str(run.unwrap_or_default()); + } + + #[inline(always)] + fn start(&mut self, prefix: Option<&str>) { + self.push_str(prefix); + } + + #[inline(always)] + fn into_text<'x>(self) -> TokenText<'x> { + TokenText::Owned(self.0) + } + + #[inline(always)] + fn restore_whitespace(&mut self, skipped: Option<&[u8]>) { + self.0.extend( + skipped + .unwrap_or_default() + .iter() + .filter_map(|ch| match ch { + b' ' => Some(' '), + b'\t' => Some('\t'), + _ => None, + }), + ); + } +} + +impl TokenText<'_> { + #[inline(always)] + pub(crate) fn as_str(&self) -> &str { + match self { + TokenText::Borrowed(text) => text, + TokenText::Owned(text) => text, + } + } + + #[inline(always)] + pub(crate) fn into_string(self) -> String { + match self { + TokenText::Borrowed(text) => text.to_owned(), + TokenText::Owned(text) => text, + } + } +} + +impl Deref for TokenText<'_> { + type Target = [u8]; + + #[inline(always)] + fn deref(&self) -> &[u8] { + self.as_str().as_bytes() + } +} + +impl AsRef<[u8]> for TokenText<'_> { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + self + } +} + +impl From for TokenText<'_> { + fn from(text: String) -> Self { + TokenText::Owned(text) + } +} + +impl<'x> Token<'x> { + pub fn into_string(self) -> String { + self.text.into_string() + } +} + diff --git a/crates/calcard/src/common/types.rs b/crates/calcard/src/common/types.rs new file mode 100644 index 00000000..15978fb3 --- /dev/null +++ b/crates/calcard/src/common/types.rs @@ -0,0 +1,467 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use jiff::Timestamp; + +impl IanaString for CalendarScale { + fn as_str(&self) -> &'static str { + match self { + CalendarScale::Gregorian => "GREGORIAN", + CalendarScale::Chinese => "CHINESE", + CalendarScale::IslamicCivil => "ISLAMIC-CIVIL", + CalendarScale::Hebrew => "HEBREW", + CalendarScale::Ethiopic => "ETHIOPIC", + } + } +} + +#[cfg(feature = "jmap")] +impl CalendarScale { + pub fn as_js_str(&self) -> &'static str { + match self { + CalendarScale::Gregorian => "gregorian", + CalendarScale::Chinese => "chinese", + CalendarScale::IslamicCivil => "islamic-civil", + CalendarScale::Hebrew => "hebrew", + CalendarScale::Ethiopic => "ethiopic", + } + } +} + +#[cfg(feature = "rkyv")] +impl IanaString for ArchivedCalendarScale { + fn as_str(&self) -> &'static str { + match self { + ArchivedCalendarScale::Gregorian => "GREGORIAN", + ArchivedCalendarScale::Chinese => "CHINESE", + ArchivedCalendarScale::IslamicCivil => "ISLAMIC-CIVIL", + ArchivedCalendarScale::Hebrew => "HEBREW", + ArchivedCalendarScale::Ethiopic => "ETHIOPIC", + } + } +} + +impl IanaParse for CalendarScale { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "gregorian" => Some(CalendarScale::Gregorian), + "chinese" => Some(CalendarScale::Chinese), + "islamic-civil" => Some(CalendarScale::IslamicCivil), + "hebrew" => Some(CalendarScale::Hebrew), + "ethiopic" => Some(CalendarScale::Ethiopic), + _ => None, + ) + } +} + +impl Encoding { + pub fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, Encoding, + b"QUOTED-PRINTABLE" => Encoding::QuotedPrintable, + b"BASE64" => Encoding::Base64, + b"Q" => Encoding::QuotedPrintable, + b"B" => Encoding::Base64, + ) + .copied() + } +} + +impl PartialDateTime { + pub fn now() -> Self { + Self::from_utc_timestamp(Timestamp::now().as_second()) + } + + pub fn from_utc_timestamp(value: i64) -> Self { + let dt = DateTime::from_timestamp(value); + + PartialDateTime { + year: dt.year.into(), + month: dt.month.into(), + day: dt.day.into(), + hour: dt.hour.into(), + minute: dt.minute.into(), + second: dt.second.into(), + tz_hour: dt.tz_hour.into(), + tz_minute: dt.tz_minute.into(), + tz_minus: false, + } + } + + pub fn from_date_timestamp(value: i64) -> Self { + let dt = DateTime::from_timestamp(value); + + PartialDateTime { + year: dt.year.into(), + month: dt.month.into(), + day: dt.day.into(), + ..Default::default() + } + } + + pub fn from_naive_timestamp(value: i64) -> Self { + let dt = DateTime::from_timestamp(value); + + PartialDateTime { + year: dt.year.into(), + month: dt.month.into(), + day: dt.day.into(), + hour: dt.hour.into(), + minute: dt.minute.into(), + second: dt.second.into(), + ..Default::default() + } + } + + pub fn to_date_time(&self) -> Option { + let mut dt = DateTimeResult { + date_time: civil::DateTime::new( + i16::try_from(self.year?).ok()?, + i8::try_from(self.month?).ok()?, + i8::try_from(self.day?).ok()?, + i8::try_from(self.hour.unwrap_or(0)).ok()?, + i8::try_from(self.minute.unwrap_or(0)).ok()?, + i8::try_from(self.second.unwrap_or(0)).ok()?, + 0, + ) + .ok()?, + offset: None, + }; + if let Some(tz_hour) = self.tz_hour { + let secs = (tz_hour as i32 * 3600) + (self.tz_minute.unwrap_or(0) as i32 * 60); + if i64::from(secs) >= timezone::SECONDS_PER_DAY { + return None; + } + dt.offset = Some(Offset::from_seconds(if self.tz_minus { -secs } else { secs }).ok()?); + } + Some(dt) + } +} + +#[cfg(feature = "rkyv")] +impl From<&ArchivedPartialDateTime> for PartialDateTime { + fn from(date: &ArchivedPartialDateTime) -> Self { + PartialDateTime { + year: date.year.as_ref().map(|year| year.to_native()), + month: date.month.as_ref().copied(), + day: date.day.as_ref().copied(), + hour: date.hour.as_ref().copied(), + minute: date.minute.as_ref().copied(), + second: date.second.as_ref().copied(), + tz_hour: date.tz_hour.as_ref().copied(), + tz_minute: date.tz_minute.as_ref().copied(), + tz_minus: date.tz_minus, + } + } +} + +#[cfg(feature = "rkyv")] +impl ArchivedPartialDateTime { + pub fn to_date_time(&self) -> Option { + let mut dt = DateTimeResult { + date_time: civil::DateTime::new( + i16::try_from(self.year.as_ref()?.to_native()).ok()?, + i8::try_from(*self.month.as_ref()?).ok()?, + i8::try_from(*self.day.as_ref()?).ok()?, + i8::try_from(self.hour.unwrap_or(0)).ok()?, + i8::try_from(self.minute.unwrap_or(0)).ok()?, + i8::try_from(self.second.unwrap_or(0)).ok()?, + 0, + ) + .ok()?, + offset: None, + }; + if let Some(tz_hour) = self.tz_hour.as_ref() { + let secs = (*tz_hour as i32 * 3600) + (self.tz_minute.unwrap_or(0) as i32 * 60); + if i64::from(secs) >= timezone::SECONDS_PER_DAY { + return None; + } + dt.offset = Some(Offset::from_seconds(if self.tz_minus { -secs } else { secs }).ok()?); + } + Some(dt) + } +} + +impl IanaParse for LinkRelation { + fn parse(s: &[u8]) -> Option { + hashify::map_ignore_case!(s, LinkRelation, + "about" => LinkRelation::About, + "acl" => LinkRelation::Acl, + "alternate" => LinkRelation::Alternate, + "amphtml" => LinkRelation::Amphtml, + "api-catalog" => LinkRelation::ApiCatalog, + "appendix" => LinkRelation::Appendix, + "apple-touch-icon" => LinkRelation::AppleTouchIcon, + "apple-touch-startup-image" => LinkRelation::AppleTouchStartupImage, + "archives" => LinkRelation::Archives, + "author" => LinkRelation::Author, + "blocked-by" => LinkRelation::BlockedBy, + "bookmark" => LinkRelation::Bookmark, + "c2pa-manifest" => LinkRelation::C2paManifest, + "canonical" => LinkRelation::Canonical, + "chapter" => LinkRelation::Chapter, + "cite-as" => LinkRelation::CiteAs, + "collection" => LinkRelation::Collection, + "compression-dictionary" => LinkRelation::CompressionDictionary, + "contents" => LinkRelation::Contents, + "convertedfrom" => LinkRelation::Convertedfrom, + "copyright" => LinkRelation::Copyright, + "create-form" => LinkRelation::CreateForm, + "current" => LinkRelation::Current, + "deprecation" => LinkRelation::Deprecation, + "describedby" => LinkRelation::Describedby, + "describes" => LinkRelation::Describes, + "disclosure" => LinkRelation::Disclosure, + "dns-prefetch" => LinkRelation::DnsPrefetch, + "duplicate" => LinkRelation::Duplicate, + "edit" => LinkRelation::Edit, + "edit-form" => LinkRelation::EditForm, + "edit-media" => LinkRelation::EditMedia, + "enclosure" => LinkRelation::Enclosure, + "external" => LinkRelation::External, + "first" => LinkRelation::First, + "geofeed" => LinkRelation::Geofeed, + "glossary" => LinkRelation::Glossary, + "help" => LinkRelation::Help, + "hosts" => LinkRelation::Hosts, + "hub" => LinkRelation::Hub, + "ice-server" => LinkRelation::IceServer, + "icon" => LinkRelation::Icon, + "index" => LinkRelation::Index, + "intervalafter" => LinkRelation::Intervalafter, + "intervalbefore" => LinkRelation::Intervalbefore, + "intervalcontains" => LinkRelation::Intervalcontains, + "intervaldisjoint" => LinkRelation::Intervaldisjoint, + "intervalduring" => LinkRelation::Intervalduring, + "intervalequals" => LinkRelation::Intervalequals, + "intervalfinishedby" => LinkRelation::Intervalfinishedby, + "intervalfinishes" => LinkRelation::Intervalfinishes, + "intervalin" => LinkRelation::Intervalin, + "intervalmeets" => LinkRelation::Intervalmeets, + "intervalmetby" => LinkRelation::Intervalmetby, + "intervaloverlappedby" => LinkRelation::Intervaloverlappedby, + "intervaloverlaps" => LinkRelation::Intervaloverlaps, + "intervalstartedby" => LinkRelation::Intervalstartedby, + "intervalstarts" => LinkRelation::Intervalstarts, + "item" => LinkRelation::Item, + "last" => LinkRelation::Last, + "latest-version" => LinkRelation::LatestVersion, + "license" => LinkRelation::License, + "linkset" => LinkRelation::Linkset, + "lrdd" => LinkRelation::Lrdd, + "manifest" => LinkRelation::Manifest, + "mask-icon" => LinkRelation::MaskIcon, + "me" => LinkRelation::Me, + "media-feed" => LinkRelation::MediaFeed, + "memento" => LinkRelation::Memento, + "micropub" => LinkRelation::Micropub, + "modulepreload" => LinkRelation::Modulepreload, + "monitor" => LinkRelation::Monitor, + "monitor-group" => LinkRelation::MonitorGroup, + "next" => LinkRelation::Next, + "next-archive" => LinkRelation::NextArchive, + "nofollow" => LinkRelation::Nofollow, + "noopener" => LinkRelation::Noopener, + "noreferrer" => LinkRelation::Noreferrer, + "opener" => LinkRelation::Opener, + "openid2.local_id" => LinkRelation::Openid2LocalId, + "openid2.provider" => LinkRelation::Openid2Provider, + "original" => LinkRelation::Original, + "p3pv1" => LinkRelation::P3pv1, + "payment" => LinkRelation::Payment, + "pingback" => LinkRelation::Pingback, + "preconnect" => LinkRelation::Preconnect, + "predecessor-version" => LinkRelation::PredecessorVersion, + "prefetch" => LinkRelation::Prefetch, + "preload" => LinkRelation::Preload, + "prerender" => LinkRelation::Prerender, + "prev" => LinkRelation::Prev, + "preview" => LinkRelation::Preview, + "previous" => LinkRelation::Previous, + "prev-archive" => LinkRelation::PrevArchive, + "privacy-policy" => LinkRelation::PrivacyPolicy, + "profile" => LinkRelation::Profile, + "publication" => LinkRelation::Publication, + "rdap-active" => LinkRelation::RdapActive, + "rdap-bottom" => LinkRelation::RdapBottom, + "rdap-down" => LinkRelation::RdapDown, + "rdap-top" => LinkRelation::RdapTop, + "rdap-up" => LinkRelation::RdapUp, + "related" => LinkRelation::Related, + "restconf" => LinkRelation::Restconf, + "replies" => LinkRelation::Replies, + "ruleinput" => LinkRelation::Ruleinput, + "search" => LinkRelation::Search, + "section" => LinkRelation::Section, + "self" => LinkRelation::Self_, + "service" => LinkRelation::Service, + "service-desc" => LinkRelation::ServiceDesc, + "service-doc" => LinkRelation::ServiceDoc, + "service-meta" => LinkRelation::ServiceMeta, + "sip-trunking-capability" => LinkRelation::SipTrunkingCapability, + "sponsored" => LinkRelation::Sponsored, + "start" => LinkRelation::Start, + "status" => LinkRelation::Status, + "stylesheet" => LinkRelation::Stylesheet, + "subsection" => LinkRelation::Subsection, + "successor-version" => LinkRelation::SuccessorVersion, + "sunset" => LinkRelation::Sunset, + "tag" => LinkRelation::Tag, + "terms-of-service" => LinkRelation::TermsOfService, + "timegate" => LinkRelation::Timegate, + "timemap" => LinkRelation::Timemap, + "type" => LinkRelation::Type, + "ugc" => LinkRelation::Ugc, + "up" => LinkRelation::Up, + "version-history" => LinkRelation::VersionHistory, + "via" => LinkRelation::Via, + "webmention" => LinkRelation::Webmention, + "working-copy" => LinkRelation::WorkingCopy, + "working-copy-of" => LinkRelation::WorkingCopyOf, + ) + .copied() + } +} + +impl IanaString for LinkRelation { + fn as_str(&self) -> &'static str { + match self { + LinkRelation::About => "about", + LinkRelation::Acl => "acl", + LinkRelation::Alternate => "alternate", + LinkRelation::Amphtml => "amphtml", + LinkRelation::ApiCatalog => "api-catalog", + LinkRelation::Appendix => "appendix", + LinkRelation::AppleTouchIcon => "apple-touch-icon", + LinkRelation::AppleTouchStartupImage => "apple-touch-startup-image", + LinkRelation::Archives => "archives", + LinkRelation::Author => "author", + LinkRelation::BlockedBy => "blocked-by", + LinkRelation::Bookmark => "bookmark", + LinkRelation::C2paManifest => "c2pa-manifest", + LinkRelation::Canonical => "canonical", + LinkRelation::Chapter => "chapter", + LinkRelation::CiteAs => "cite-as", + LinkRelation::Collection => "collection", + LinkRelation::CompressionDictionary => "compression-dictionary", + LinkRelation::Contents => "contents", + LinkRelation::Convertedfrom => "convertedfrom", + LinkRelation::Copyright => "copyright", + LinkRelation::CreateForm => "create-form", + LinkRelation::Current => "current", + LinkRelation::Deprecation => "deprecation", + LinkRelation::Describedby => "describedby", + LinkRelation::Describes => "describes", + LinkRelation::Disclosure => "disclosure", + LinkRelation::DnsPrefetch => "dns-prefetch", + LinkRelation::Duplicate => "duplicate", + LinkRelation::Edit => "edit", + LinkRelation::EditForm => "edit-form", + LinkRelation::EditMedia => "edit-media", + LinkRelation::Enclosure => "enclosure", + LinkRelation::External => "external", + LinkRelation::First => "first", + LinkRelation::Geofeed => "geofeed", + LinkRelation::Glossary => "glossary", + LinkRelation::Help => "help", + LinkRelation::Hosts => "hosts", + LinkRelation::Hub => "hub", + LinkRelation::IceServer => "ice-server", + LinkRelation::Icon => "icon", + LinkRelation::Index => "index", + LinkRelation::Intervalafter => "intervalafter", + LinkRelation::Intervalbefore => "intervalbefore", + LinkRelation::Intervalcontains => "intervalcontains", + LinkRelation::Intervaldisjoint => "intervaldisjoint", + LinkRelation::Intervalduring => "intervalduring", + LinkRelation::Intervalequals => "intervalequals", + LinkRelation::Intervalfinishedby => "intervalfinishedby", + LinkRelation::Intervalfinishes => "intervalfinishes", + LinkRelation::Intervalin => "intervalin", + LinkRelation::Intervalmeets => "intervalmeets", + LinkRelation::Intervalmetby => "intervalmetby", + LinkRelation::Intervaloverlappedby => "intervaloverlappedby", + LinkRelation::Intervaloverlaps => "intervaloverlaps", + LinkRelation::Intervalstartedby => "intervalstartedby", + LinkRelation::Intervalstarts => "intervalstarts", + LinkRelation::Item => "item", + LinkRelation::Last => "last", + LinkRelation::LatestVersion => "latest-version", + LinkRelation::License => "license", + LinkRelation::Linkset => "linkset", + LinkRelation::Lrdd => "lrdd", + LinkRelation::Manifest => "manifest", + LinkRelation::MaskIcon => "mask-icon", + LinkRelation::Me => "me", + LinkRelation::MediaFeed => "media-feed", + LinkRelation::Memento => "memento", + LinkRelation::Micropub => "micropub", + LinkRelation::Modulepreload => "modulepreload", + LinkRelation::Monitor => "monitor", + LinkRelation::MonitorGroup => "monitor-group", + LinkRelation::Next => "next", + LinkRelation::NextArchive => "next-archive", + LinkRelation::Nofollow => "nofollow", + LinkRelation::Noopener => "noopener", + LinkRelation::Noreferrer => "noreferrer", + LinkRelation::Opener => "opener", + LinkRelation::Openid2LocalId => "openid2.local_id", + LinkRelation::Openid2Provider => "openid2.provider", + LinkRelation::Original => "original", + LinkRelation::P3pv1 => "p3pv1", + LinkRelation::Payment => "payment", + LinkRelation::Pingback => "pingback", + LinkRelation::Preconnect => "preconnect", + LinkRelation::PredecessorVersion => "predecessor-version", + LinkRelation::Prefetch => "prefetch", + LinkRelation::Preload => "preload", + LinkRelation::Prerender => "prerender", + LinkRelation::Prev => "prev", + LinkRelation::Preview => "preview", + LinkRelation::Previous => "previous", + LinkRelation::PrevArchive => "prev-archive", + LinkRelation::PrivacyPolicy => "privacy-policy", + LinkRelation::Profile => "profile", + LinkRelation::Publication => "publication", + LinkRelation::RdapActive => "rdap-active", + LinkRelation::RdapBottom => "rdap-bottom", + LinkRelation::RdapDown => "rdap-down", + LinkRelation::RdapTop => "rdap-top", + LinkRelation::RdapUp => "rdap-up", + LinkRelation::Related => "related", + LinkRelation::Restconf => "restconf", + LinkRelation::Replies => "replies", + LinkRelation::Ruleinput => "ruleinput", + LinkRelation::Search => "search", + LinkRelation::Section => "section", + LinkRelation::Self_ => "self", + LinkRelation::Service => "service", + LinkRelation::ServiceDesc => "service-desc", + LinkRelation::ServiceDoc => "service-doc", + LinkRelation::ServiceMeta => "service-meta", + LinkRelation::SipTrunkingCapability => "sip-trunking-capability", + LinkRelation::Sponsored => "sponsored", + LinkRelation::Start => "start", + LinkRelation::Status => "status", + LinkRelation::Stylesheet => "stylesheet", + LinkRelation::Subsection => "subsection", + LinkRelation::SuccessorVersion => "successor-version", + LinkRelation::Sunset => "sunset", + LinkRelation::Tag => "tag", + LinkRelation::TermsOfService => "terms-of-service", + LinkRelation::Timegate => "timegate", + LinkRelation::Timemap => "timemap", + LinkRelation::Type => "type", + LinkRelation::Ugc => "ugc", + LinkRelation::Up => "up", + LinkRelation::VersionHistory => "version-history", + LinkRelation::Via => "via", + LinkRelation::Webmention => "webmention", + LinkRelation::WorkingCopy => "working-copy", + LinkRelation::WorkingCopyOf => "working-copy-of", + } + } +} diff --git a/crates/calcard/src/common/tzdb.rs b/crates/calcard/src/common/tzdb.rs new file mode 100644 index 00000000..f2bcfb0d --- /dev/null +++ b/crates/calcard/src/common/tzdb.rs @@ -0,0 +1,1238 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Stable numeric identifiers for IANA time zone names. +//! +//! [`Tz::as_id`](super::Tz::as_id) and [`Tz::from_id`](super::Tz::from_id) +//! expose these identifiers as a compact storage format, so the table is +//! committed verbatim rather than derived at build time: entries are never +//! reordered or removed, only appended. It was generated from the time zone +//! database bundled with `jiff`. +//! +//! A name maps to an identifier through a perfect hash and an identifier maps +//! back to a name by indexing, so neither direction searches. + +/// Identifier of the `UTC` entry. +pub(super) const UTC_ID: u16 = 593; + +static TZ_NAMES: [&str; 599] = [ + "Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + "Africa/Algiers", + "Africa/Asmara", + "Africa/Asmera", + "Africa/Bamako", + "Africa/Bangui", + "Africa/Banjul", + "Africa/Bissau", + "Africa/Blantyre", + "Africa/Brazzaville", + "Africa/Bujumbura", + "Africa/Cairo", + "Africa/Casablanca", + "Africa/Ceuta", + "Africa/Conakry", + "Africa/Dakar", + "Africa/Dar_es_Salaam", + "Africa/Djibouti", + "Africa/Douala", + "Africa/El_Aaiun", + "Africa/Freetown", + "Africa/Gaborone", + "Africa/Harare", + "Africa/Johannesburg", + "Africa/Juba", + "Africa/Kampala", + "Africa/Khartoum", + "Africa/Kigali", + "Africa/Kinshasa", + "Africa/Lagos", + "Africa/Libreville", + "Africa/Lome", + "Africa/Luanda", + "Africa/Lubumbashi", + "Africa/Lusaka", + "Africa/Malabo", + "Africa/Maputo", + "Africa/Maseru", + "Africa/Mbabane", + "Africa/Mogadishu", + "Africa/Monrovia", + "Africa/Nairobi", + "Africa/Ndjamena", + "Africa/Niamey", + "Africa/Nouakchott", + "Africa/Ouagadougou", + "Africa/Porto-Novo", + "Africa/Sao_Tome", + "Africa/Timbuktu", + "Africa/Tripoli", + "Africa/Tunis", + "Africa/Windhoek", + "America/Adak", + "America/Anchorage", + "America/Anguilla", + "America/Antigua", + "America/Araguaina", + "America/Argentina/Buenos_Aires", + "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", + "America/Argentina/Cordoba", + "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", + "America/Argentina/Mendoza", + "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", + "America/Argentina/San_Juan", + "America/Argentina/San_Luis", + "America/Argentina/Tucuman", + "America/Argentina/Ushuaia", + "America/Aruba", + "America/Asuncion", + "America/Atikokan", + "America/Atka", + "America/Bahia", + "America/Bahia_Banderas", + "America/Barbados", + "America/Belem", + "America/Belize", + "America/Blanc-Sablon", + "America/Boa_Vista", + "America/Bogota", + "America/Boise", + "America/Buenos_Aires", + "America/Cambridge_Bay", + "America/Campo_Grande", + "America/Cancun", + "America/Caracas", + "America/Catamarca", + "America/Cayenne", + "America/Cayman", + "America/Chicago", + "America/Chihuahua", + "America/Ciudad_Juarez", + "America/Coral_Harbour", + "America/Cordoba", + "America/Costa_Rica", + "America/Coyhaique", + "America/Creston", + "America/Cuiaba", + "America/Curacao", + "America/Danmarkshavn", + "America/Dawson", + "America/Dawson_Creek", + "America/Denver", + "America/Detroit", + "America/Dominica", + "America/Edmonton", + "America/Eirunepe", + "America/El_Salvador", + "America/Ensenada", + "America/Fort_Nelson", + "America/Fort_Wayne", + "America/Fortaleza", + "America/Glace_Bay", + "America/Godthab", + "America/Goose_Bay", + "America/Grand_Turk", + "America/Grenada", + "America/Guadeloupe", + "America/Guatemala", + "America/Guayaquil", + "America/Guyana", + "America/Halifax", + "America/Havana", + "America/Hermosillo", + "America/Indiana/Indianapolis", + "America/Indiana/Knox", + "America/Indiana/Marengo", + "America/Indiana/Petersburg", + "America/Indiana/Tell_City", + "America/Indiana/Vevay", + "America/Indiana/Vincennes", + "America/Indiana/Winamac", + "America/Indianapolis", + "America/Inuvik", + "America/Iqaluit", + "America/Jamaica", + "America/Jujuy", + "America/Juneau", + "America/Kentucky/Louisville", + "America/Kentucky/Monticello", + "America/Knox_IN", + "America/Kralendijk", + "America/La_Paz", + "America/Lima", + "America/Los_Angeles", + "America/Louisville", + "America/Lower_Princes", + "America/Maceio", + "America/Managua", + "America/Manaus", + "America/Marigot", + "America/Martinique", + "America/Matamoros", + "America/Mazatlan", + "America/Mendoza", + "America/Menominee", + "America/Merida", + "America/Metlakatla", + "America/Mexico_City", + "America/Miquelon", + "America/Moncton", + "America/Monterrey", + "America/Montevideo", + "America/Montreal", + "America/Montserrat", + "America/Nassau", + "America/New_York", + "America/Nipigon", + "America/Nome", + "America/Noronha", + "America/North_Dakota/Beulah", + "America/North_Dakota/Center", + "America/North_Dakota/New_Salem", + "America/Nuuk", + "America/Ojinaga", + "America/Panama", + "America/Pangnirtung", + "America/Paramaribo", + "America/Phoenix", + "America/Port-au-Prince", + "America/Port_of_Spain", + "America/Porto_Acre", + "America/Porto_Velho", + "America/Puerto_Rico", + "America/Punta_Arenas", + "America/Rainy_River", + "America/Rankin_Inlet", + "America/Recife", + "America/Regina", + "America/Resolute", + "America/Rio_Branco", + "America/Rosario", + "America/Santa_Isabel", + "America/Santarem", + "America/Santiago", + "America/Santo_Domingo", + "America/Sao_Paulo", + "America/Scoresbysund", + "America/Shiprock", + "America/Sitka", + "America/St_Barthelemy", + "America/St_Johns", + "America/St_Kitts", + "America/St_Lucia", + "America/St_Thomas", + "America/St_Vincent", + "America/Swift_Current", + "America/Tegucigalpa", + "America/Thule", + "America/Thunder_Bay", + "America/Tijuana", + "America/Toronto", + "America/Tortola", + "America/Vancouver", + "America/Virgin", + "America/Whitehorse", + "America/Winnipeg", + "America/Yakutat", + "America/Yellowknife", + "Antarctica/Casey", + "Antarctica/Davis", + "Antarctica/DumontDUrville", + "Antarctica/Macquarie", + "Antarctica/Mawson", + "Antarctica/McMurdo", + "Antarctica/Palmer", + "Antarctica/Rothera", + "Antarctica/South_Pole", + "Antarctica/Syowa", + "Antarctica/Troll", + "Antarctica/Vostok", + "Arctic/Longyearbyen", + "Asia/Aden", + "Asia/Almaty", + "Asia/Amman", + "Asia/Anadyr", + "Asia/Aqtau", + "Asia/Aqtobe", + "Asia/Ashgabat", + "Asia/Ashkhabad", + "Asia/Atyrau", + "Asia/Baghdad", + "Asia/Bahrain", + "Asia/Baku", + "Asia/Bangkok", + "Asia/Barnaul", + "Asia/Beirut", + "Asia/Bishkek", + "Asia/Brunei", + "Asia/Calcutta", + "Asia/Chita", + "Asia/Choibalsan", + "Asia/Chongqing", + "Asia/Chungking", + "Asia/Colombo", + "Asia/Dacca", + "Asia/Damascus", + "Asia/Dhaka", + "Asia/Dili", + "Asia/Dubai", + "Asia/Dushanbe", + "Asia/Famagusta", + "Asia/Gaza", + "Asia/Harbin", + "Asia/Hebron", + "Asia/Ho_Chi_Minh", + "Asia/Hong_Kong", + "Asia/Hovd", + "Asia/Irkutsk", + "Asia/Istanbul", + "Asia/Jakarta", + "Asia/Jayapura", + "Asia/Jerusalem", + "Asia/Kabul", + "Asia/Kamchatka", + "Asia/Karachi", + "Asia/Kashgar", + "Asia/Kathmandu", + "Asia/Katmandu", + "Asia/Khandyga", + "Asia/Kolkata", + "Asia/Krasnoyarsk", + "Asia/Kuala_Lumpur", + "Asia/Kuching", + "Asia/Kuwait", + "Asia/Macao", + "Asia/Macau", + "Asia/Magadan", + "Asia/Makassar", + "Asia/Manila", + "Asia/Muscat", + "Asia/Nicosia", + "Asia/Novokuznetsk", + "Asia/Novosibirsk", + "Asia/Omsk", + "Asia/Oral", + "Asia/Phnom_Penh", + "Asia/Pontianak", + "Asia/Pyongyang", + "Asia/Qatar", + "Asia/Qostanay", + "Asia/Qyzylorda", + "Asia/Rangoon", + "Asia/Riyadh", + "Asia/Saigon", + "Asia/Sakhalin", + "Asia/Samarkand", + "Asia/Seoul", + "Asia/Shanghai", + "Asia/Singapore", + "Asia/Srednekolymsk", + "Asia/Taipei", + "Asia/Tashkent", + "Asia/Tbilisi", + "Asia/Tehran", + "Asia/Tel_Aviv", + "Asia/Thimbu", + "Asia/Thimphu", + "Asia/Tokyo", + "Asia/Tomsk", + "Asia/Ujung_Pandang", + "Asia/Ulaanbaatar", + "Asia/Ulan_Bator", + "Asia/Urumqi", + "Asia/Ust-Nera", + "Asia/Vientiane", + "Asia/Vladivostok", + "Asia/Yakutsk", + "Asia/Yangon", + "Asia/Yekaterinburg", + "Asia/Yerevan", + "Atlantic/Azores", + "Atlantic/Bermuda", + "Atlantic/Canary", + "Atlantic/Cape_Verde", + "Atlantic/Faeroe", + "Atlantic/Faroe", + "Atlantic/Jan_Mayen", + "Atlantic/Madeira", + "Atlantic/Reykjavik", + "Atlantic/South_Georgia", + "Atlantic/St_Helena", + "Atlantic/Stanley", + "Australia/ACT", + "Australia/Adelaide", + "Australia/Brisbane", + "Australia/Broken_Hill", + "Australia/Canberra", + "Australia/Currie", + "Australia/Darwin", + "Australia/Eucla", + "Australia/Hobart", + "Australia/LHI", + "Australia/Lindeman", + "Australia/Lord_Howe", + "Australia/Melbourne", + "Australia/NSW", + "Australia/North", + "Australia/Perth", + "Australia/Queensland", + "Australia/South", + "Australia/Sydney", + "Australia/Tasmania", + "Australia/Victoria", + "Australia/West", + "Australia/Yancowinna", + "Brazil/Acre", + "Brazil/DeNoronha", + "Brazil/East", + "Brazil/West", + "CET", + "CST6CDT", + "Canada/Atlantic", + "Canada/Central", + "Canada/Eastern", + "Canada/Mountain", + "Canada/Newfoundland", + "Canada/Pacific", + "Canada/Saskatchewan", + "Canada/Yukon", + "Chile/Continental", + "Chile/EasterIsland", + "Cuba", + "EET", + "EST", + "EST5EDT", + "Egypt", + "Eire", + "Etc/GMT", + "Etc/GMT+0", + "Etc/GMT+1", + "Etc/GMT+10", + "Etc/GMT+11", + "Etc/GMT+12", + "Etc/GMT+2", + "Etc/GMT+3", + "Etc/GMT+4", + "Etc/GMT+5", + "Etc/GMT+6", + "Etc/GMT+7", + "Etc/GMT+8", + "Etc/GMT+9", + "Etc/GMT-0", + "Etc/GMT-1", + "Etc/GMT-10", + "Etc/GMT-11", + "Etc/GMT-12", + "Etc/GMT-13", + "Etc/GMT-14", + "Etc/GMT-2", + "Etc/GMT-3", + "Etc/GMT-4", + "Etc/GMT-5", + "Etc/GMT-6", + "Etc/GMT-7", + "Etc/GMT-8", + "Etc/GMT-9", + "Etc/GMT0", + "Etc/Greenwich", + "Etc/UCT", + "Etc/UTC", + "Etc/Universal", + "Etc/Zulu", + "Europe/Amsterdam", + "Europe/Andorra", + "Europe/Astrakhan", + "Europe/Athens", + "Europe/Belfast", + "Europe/Belgrade", + "Europe/Berlin", + "Europe/Bratislava", + "Europe/Brussels", + "Europe/Bucharest", + "Europe/Budapest", + "Europe/Busingen", + "Europe/Chisinau", + "Europe/Copenhagen", + "Europe/Dublin", + "Europe/Gibraltar", + "Europe/Guernsey", + "Europe/Helsinki", + "Europe/Isle_of_Man", + "Europe/Istanbul", + "Europe/Jersey", + "Europe/Kaliningrad", + "Europe/Kiev", + "Europe/Kirov", + "Europe/Kyiv", + "Europe/Lisbon", + "Europe/Ljubljana", + "Europe/London", + "Europe/Luxembourg", + "Europe/Madrid", + "Europe/Malta", + "Europe/Mariehamn", + "Europe/Minsk", + "Europe/Monaco", + "Europe/Moscow", + "Europe/Nicosia", + "Europe/Oslo", + "Europe/Paris", + "Europe/Podgorica", + "Europe/Prague", + "Europe/Riga", + "Europe/Rome", + "Europe/Samara", + "Europe/San_Marino", + "Europe/Sarajevo", + "Europe/Saratov", + "Europe/Simferopol", + "Europe/Skopje", + "Europe/Sofia", + "Europe/Stockholm", + "Europe/Tallinn", + "Europe/Tirane", + "Europe/Tiraspol", + "Europe/Ulyanovsk", + "Europe/Uzhgorod", + "Europe/Vaduz", + "Europe/Vatican", + "Europe/Vienna", + "Europe/Vilnius", + "Europe/Volgograd", + "Europe/Warsaw", + "Europe/Zagreb", + "Europe/Zaporozhye", + "Europe/Zurich", + "Factory", + "GB", + "GB-Eire", + "GMT", + "GMT+0", + "GMT-0", + "GMT0", + "Greenwich", + "HST", + "Hongkong", + "Iceland", + "Indian/Antananarivo", + "Indian/Chagos", + "Indian/Christmas", + "Indian/Cocos", + "Indian/Comoro", + "Indian/Kerguelen", + "Indian/Mahe", + "Indian/Maldives", + "Indian/Mauritius", + "Indian/Mayotte", + "Indian/Reunion", + "Iran", + "Israel", + "Jamaica", + "Japan", + "Kwajalein", + "Libya", + "MET", + "MST", + "MST7MDT", + "Mexico/BajaNorte", + "Mexico/BajaSur", + "Mexico/General", + "NZ", + "NZ-CHAT", + "Navajo", + "PRC", + "PST8PDT", + "Pacific/Apia", + "Pacific/Auckland", + "Pacific/Bougainville", + "Pacific/Chatham", + "Pacific/Chuuk", + "Pacific/Easter", + "Pacific/Efate", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Fiji", + "Pacific/Funafuti", + "Pacific/Galapagos", + "Pacific/Gambier", + "Pacific/Guadalcanal", + "Pacific/Guam", + "Pacific/Honolulu", + "Pacific/Johnston", + "Pacific/Kanton", + "Pacific/Kiritimati", + "Pacific/Kosrae", + "Pacific/Kwajalein", + "Pacific/Majuro", + "Pacific/Marquesas", + "Pacific/Midway", + "Pacific/Nauru", + "Pacific/Niue", + "Pacific/Norfolk", + "Pacific/Noumea", + "Pacific/Pago_Pago", + "Pacific/Palau", + "Pacific/Pitcairn", + "Pacific/Pohnpei", + "Pacific/Ponape", + "Pacific/Port_Moresby", + "Pacific/Rarotonga", + "Pacific/Saipan", + "Pacific/Samoa", + "Pacific/Tahiti", + "Pacific/Tarawa", + "Pacific/Tongatapu", + "Pacific/Truk", + "Pacific/Wake", + "Pacific/Wallis", + "Pacific/Yap", + "Poland", + "Portugal", + "ROC", + "ROK", + "Singapore", + "Turkey", + "UCT", + "US/Alaska", + "US/Aleutian", + "US/Arizona", + "US/Central", + "US/East-Indiana", + "US/Eastern", + "US/Hawaii", + "US/Indiana-Starke", + "US/Michigan", + "US/Mountain", + "US/Pacific", + "US/Samoa", + "UTC", + "Universal", + "W-SU", + "WET", + "Zulu", + "posixrules", +]; + +/// Returns the identifier assigned to an IANA time zone name. +pub(super) fn name_to_id(name: &str) -> Option { + hashify::map!(name.as_bytes(), u16, + "Africa/Abidjan" => 0, + "Africa/Accra" => 1, + "Africa/Addis_Ababa" => 2, + "Africa/Algiers" => 3, + "Africa/Asmara" => 4, + "Africa/Asmera" => 5, + "Africa/Bamako" => 6, + "Africa/Bangui" => 7, + "Africa/Banjul" => 8, + "Africa/Bissau" => 9, + "Africa/Blantyre" => 10, + "Africa/Brazzaville" => 11, + "Africa/Bujumbura" => 12, + "Africa/Cairo" => 13, + "Africa/Casablanca" => 14, + "Africa/Ceuta" => 15, + "Africa/Conakry" => 16, + "Africa/Dakar" => 17, + "Africa/Dar_es_Salaam" => 18, + "Africa/Djibouti" => 19, + "Africa/Douala" => 20, + "Africa/El_Aaiun" => 21, + "Africa/Freetown" => 22, + "Africa/Gaborone" => 23, + "Africa/Harare" => 24, + "Africa/Johannesburg" => 25, + "Africa/Juba" => 26, + "Africa/Kampala" => 27, + "Africa/Khartoum" => 28, + "Africa/Kigali" => 29, + "Africa/Kinshasa" => 30, + "Africa/Lagos" => 31, + "Africa/Libreville" => 32, + "Africa/Lome" => 33, + "Africa/Luanda" => 34, + "Africa/Lubumbashi" => 35, + "Africa/Lusaka" => 36, + "Africa/Malabo" => 37, + "Africa/Maputo" => 38, + "Africa/Maseru" => 39, + "Africa/Mbabane" => 40, + "Africa/Mogadishu" => 41, + "Africa/Monrovia" => 42, + "Africa/Nairobi" => 43, + "Africa/Ndjamena" => 44, + "Africa/Niamey" => 45, + "Africa/Nouakchott" => 46, + "Africa/Ouagadougou" => 47, + "Africa/Porto-Novo" => 48, + "Africa/Sao_Tome" => 49, + "Africa/Timbuktu" => 50, + "Africa/Tripoli" => 51, + "Africa/Tunis" => 52, + "Africa/Windhoek" => 53, + "America/Adak" => 54, + "America/Anchorage" => 55, + "America/Anguilla" => 56, + "America/Antigua" => 57, + "America/Araguaina" => 58, + "America/Argentina/Buenos_Aires" => 59, + "America/Argentina/Catamarca" => 60, + "America/Argentina/ComodRivadavia" => 61, + "America/Argentina/Cordoba" => 62, + "America/Argentina/Jujuy" => 63, + "America/Argentina/La_Rioja" => 64, + "America/Argentina/Mendoza" => 65, + "America/Argentina/Rio_Gallegos" => 66, + "America/Argentina/Salta" => 67, + "America/Argentina/San_Juan" => 68, + "America/Argentina/San_Luis" => 69, + "America/Argentina/Tucuman" => 70, + "America/Argentina/Ushuaia" => 71, + "America/Aruba" => 72, + "America/Asuncion" => 73, + "America/Atikokan" => 74, + "America/Atka" => 75, + "America/Bahia" => 76, + "America/Bahia_Banderas" => 77, + "America/Barbados" => 78, + "America/Belem" => 79, + "America/Belize" => 80, + "America/Blanc-Sablon" => 81, + "America/Boa_Vista" => 82, + "America/Bogota" => 83, + "America/Boise" => 84, + "America/Buenos_Aires" => 85, + "America/Cambridge_Bay" => 86, + "America/Campo_Grande" => 87, + "America/Cancun" => 88, + "America/Caracas" => 89, + "America/Catamarca" => 90, + "America/Cayenne" => 91, + "America/Cayman" => 92, + "America/Chicago" => 93, + "America/Chihuahua" => 94, + "America/Ciudad_Juarez" => 95, + "America/Coral_Harbour" => 96, + "America/Cordoba" => 97, + "America/Costa_Rica" => 98, + "America/Coyhaique" => 99, + "America/Creston" => 100, + "America/Cuiaba" => 101, + "America/Curacao" => 102, + "America/Danmarkshavn" => 103, + "America/Dawson" => 104, + "America/Dawson_Creek" => 105, + "America/Denver" => 106, + "America/Detroit" => 107, + "America/Dominica" => 108, + "America/Edmonton" => 109, + "America/Eirunepe" => 110, + "America/El_Salvador" => 111, + "America/Ensenada" => 112, + "America/Fort_Nelson" => 113, + "America/Fort_Wayne" => 114, + "America/Fortaleza" => 115, + "America/Glace_Bay" => 116, + "America/Godthab" => 117, + "America/Goose_Bay" => 118, + "America/Grand_Turk" => 119, + "America/Grenada" => 120, + "America/Guadeloupe" => 121, + "America/Guatemala" => 122, + "America/Guayaquil" => 123, + "America/Guyana" => 124, + "America/Halifax" => 125, + "America/Havana" => 126, + "America/Hermosillo" => 127, + "America/Indiana/Indianapolis" => 128, + "America/Indiana/Knox" => 129, + "America/Indiana/Marengo" => 130, + "America/Indiana/Petersburg" => 131, + "America/Indiana/Tell_City" => 132, + "America/Indiana/Vevay" => 133, + "America/Indiana/Vincennes" => 134, + "America/Indiana/Winamac" => 135, + "America/Indianapolis" => 136, + "America/Inuvik" => 137, + "America/Iqaluit" => 138, + "America/Jamaica" => 139, + "America/Jujuy" => 140, + "America/Juneau" => 141, + "America/Kentucky/Louisville" => 142, + "America/Kentucky/Monticello" => 143, + "America/Knox_IN" => 144, + "America/Kralendijk" => 145, + "America/La_Paz" => 146, + "America/Lima" => 147, + "America/Los_Angeles" => 148, + "America/Louisville" => 149, + "America/Lower_Princes" => 150, + "America/Maceio" => 151, + "America/Managua" => 152, + "America/Manaus" => 153, + "America/Marigot" => 154, + "America/Martinique" => 155, + "America/Matamoros" => 156, + "America/Mazatlan" => 157, + "America/Mendoza" => 158, + "America/Menominee" => 159, + "America/Merida" => 160, + "America/Metlakatla" => 161, + "America/Mexico_City" => 162, + "America/Miquelon" => 163, + "America/Moncton" => 164, + "America/Monterrey" => 165, + "America/Montevideo" => 166, + "America/Montreal" => 167, + "America/Montserrat" => 168, + "America/Nassau" => 169, + "America/New_York" => 170, + "America/Nipigon" => 171, + "America/Nome" => 172, + "America/Noronha" => 173, + "America/North_Dakota/Beulah" => 174, + "America/North_Dakota/Center" => 175, + "America/North_Dakota/New_Salem" => 176, + "America/Nuuk" => 177, + "America/Ojinaga" => 178, + "America/Panama" => 179, + "America/Pangnirtung" => 180, + "America/Paramaribo" => 181, + "America/Phoenix" => 182, + "America/Port-au-Prince" => 183, + "America/Port_of_Spain" => 184, + "America/Porto_Acre" => 185, + "America/Porto_Velho" => 186, + "America/Puerto_Rico" => 187, + "America/Punta_Arenas" => 188, + "America/Rainy_River" => 189, + "America/Rankin_Inlet" => 190, + "America/Recife" => 191, + "America/Regina" => 192, + "America/Resolute" => 193, + "America/Rio_Branco" => 194, + "America/Rosario" => 195, + "America/Santa_Isabel" => 196, + "America/Santarem" => 197, + "America/Santiago" => 198, + "America/Santo_Domingo" => 199, + "America/Sao_Paulo" => 200, + "America/Scoresbysund" => 201, + "America/Shiprock" => 202, + "America/Sitka" => 203, + "America/St_Barthelemy" => 204, + "America/St_Johns" => 205, + "America/St_Kitts" => 206, + "America/St_Lucia" => 207, + "America/St_Thomas" => 208, + "America/St_Vincent" => 209, + "America/Swift_Current" => 210, + "America/Tegucigalpa" => 211, + "America/Thule" => 212, + "America/Thunder_Bay" => 213, + "America/Tijuana" => 214, + "America/Toronto" => 215, + "America/Tortola" => 216, + "America/Vancouver" => 217, + "America/Virgin" => 218, + "America/Whitehorse" => 219, + "America/Winnipeg" => 220, + "America/Yakutat" => 221, + "America/Yellowknife" => 222, + "Antarctica/Casey" => 223, + "Antarctica/Davis" => 224, + "Antarctica/DumontDUrville" => 225, + "Antarctica/Macquarie" => 226, + "Antarctica/Mawson" => 227, + "Antarctica/McMurdo" => 228, + "Antarctica/Palmer" => 229, + "Antarctica/Rothera" => 230, + "Antarctica/South_Pole" => 231, + "Antarctica/Syowa" => 232, + "Antarctica/Troll" => 233, + "Antarctica/Vostok" => 234, + "Arctic/Longyearbyen" => 235, + "Asia/Aden" => 236, + "Asia/Almaty" => 237, + "Asia/Amman" => 238, + "Asia/Anadyr" => 239, + "Asia/Aqtau" => 240, + "Asia/Aqtobe" => 241, + "Asia/Ashgabat" => 242, + "Asia/Ashkhabad" => 243, + "Asia/Atyrau" => 244, + "Asia/Baghdad" => 245, + "Asia/Bahrain" => 246, + "Asia/Baku" => 247, + "Asia/Bangkok" => 248, + "Asia/Barnaul" => 249, + "Asia/Beirut" => 250, + "Asia/Bishkek" => 251, + "Asia/Brunei" => 252, + "Asia/Calcutta" => 253, + "Asia/Chita" => 254, + "Asia/Choibalsan" => 255, + "Asia/Chongqing" => 256, + "Asia/Chungking" => 257, + "Asia/Colombo" => 258, + "Asia/Dacca" => 259, + "Asia/Damascus" => 260, + "Asia/Dhaka" => 261, + "Asia/Dili" => 262, + "Asia/Dubai" => 263, + "Asia/Dushanbe" => 264, + "Asia/Famagusta" => 265, + "Asia/Gaza" => 266, + "Asia/Harbin" => 267, + "Asia/Hebron" => 268, + "Asia/Ho_Chi_Minh" => 269, + "Asia/Hong_Kong" => 270, + "Asia/Hovd" => 271, + "Asia/Irkutsk" => 272, + "Asia/Istanbul" => 273, + "Asia/Jakarta" => 274, + "Asia/Jayapura" => 275, + "Asia/Jerusalem" => 276, + "Asia/Kabul" => 277, + "Asia/Kamchatka" => 278, + "Asia/Karachi" => 279, + "Asia/Kashgar" => 280, + "Asia/Kathmandu" => 281, + "Asia/Katmandu" => 282, + "Asia/Khandyga" => 283, + "Asia/Kolkata" => 284, + "Asia/Krasnoyarsk" => 285, + "Asia/Kuala_Lumpur" => 286, + "Asia/Kuching" => 287, + "Asia/Kuwait" => 288, + "Asia/Macao" => 289, + "Asia/Macau" => 290, + "Asia/Magadan" => 291, + "Asia/Makassar" => 292, + "Asia/Manila" => 293, + "Asia/Muscat" => 294, + "Asia/Nicosia" => 295, + "Asia/Novokuznetsk" => 296, + "Asia/Novosibirsk" => 297, + "Asia/Omsk" => 298, + "Asia/Oral" => 299, + "Asia/Phnom_Penh" => 300, + "Asia/Pontianak" => 301, + "Asia/Pyongyang" => 302, + "Asia/Qatar" => 303, + "Asia/Qostanay" => 304, + "Asia/Qyzylorda" => 305, + "Asia/Rangoon" => 306, + "Asia/Riyadh" => 307, + "Asia/Saigon" => 308, + "Asia/Sakhalin" => 309, + "Asia/Samarkand" => 310, + "Asia/Seoul" => 311, + "Asia/Shanghai" => 312, + "Asia/Singapore" => 313, + "Asia/Srednekolymsk" => 314, + "Asia/Taipei" => 315, + "Asia/Tashkent" => 316, + "Asia/Tbilisi" => 317, + "Asia/Tehran" => 318, + "Asia/Tel_Aviv" => 319, + "Asia/Thimbu" => 320, + "Asia/Thimphu" => 321, + "Asia/Tokyo" => 322, + "Asia/Tomsk" => 323, + "Asia/Ujung_Pandang" => 324, + "Asia/Ulaanbaatar" => 325, + "Asia/Ulan_Bator" => 326, + "Asia/Urumqi" => 327, + "Asia/Ust-Nera" => 328, + "Asia/Vientiane" => 329, + "Asia/Vladivostok" => 330, + "Asia/Yakutsk" => 331, + "Asia/Yangon" => 332, + "Asia/Yekaterinburg" => 333, + "Asia/Yerevan" => 334, + "Atlantic/Azores" => 335, + "Atlantic/Bermuda" => 336, + "Atlantic/Canary" => 337, + "Atlantic/Cape_Verde" => 338, + "Atlantic/Faeroe" => 339, + "Atlantic/Faroe" => 340, + "Atlantic/Jan_Mayen" => 341, + "Atlantic/Madeira" => 342, + "Atlantic/Reykjavik" => 343, + "Atlantic/South_Georgia" => 344, + "Atlantic/St_Helena" => 345, + "Atlantic/Stanley" => 346, + "Australia/ACT" => 347, + "Australia/Adelaide" => 348, + "Australia/Brisbane" => 349, + "Australia/Broken_Hill" => 350, + "Australia/Canberra" => 351, + "Australia/Currie" => 352, + "Australia/Darwin" => 353, + "Australia/Eucla" => 354, + "Australia/Hobart" => 355, + "Australia/LHI" => 356, + "Australia/Lindeman" => 357, + "Australia/Lord_Howe" => 358, + "Australia/Melbourne" => 359, + "Australia/NSW" => 360, + "Australia/North" => 361, + "Australia/Perth" => 362, + "Australia/Queensland" => 363, + "Australia/South" => 364, + "Australia/Sydney" => 365, + "Australia/Tasmania" => 366, + "Australia/Victoria" => 367, + "Australia/West" => 368, + "Australia/Yancowinna" => 369, + "Brazil/Acre" => 370, + "Brazil/DeNoronha" => 371, + "Brazil/East" => 372, + "Brazil/West" => 373, + "CET" => 374, + "CST6CDT" => 375, + "Canada/Atlantic" => 376, + "Canada/Central" => 377, + "Canada/Eastern" => 378, + "Canada/Mountain" => 379, + "Canada/Newfoundland" => 380, + "Canada/Pacific" => 381, + "Canada/Saskatchewan" => 382, + "Canada/Yukon" => 383, + "Chile/Continental" => 384, + "Chile/EasterIsland" => 385, + "Cuba" => 386, + "EET" => 387, + "EST" => 388, + "EST5EDT" => 389, + "Egypt" => 390, + "Eire" => 391, + "Etc/GMT" => 392, + "Etc/GMT+0" => 393, + "Etc/GMT+1" => 394, + "Etc/GMT+10" => 395, + "Etc/GMT+11" => 396, + "Etc/GMT+12" => 397, + "Etc/GMT+2" => 398, + "Etc/GMT+3" => 399, + "Etc/GMT+4" => 400, + "Etc/GMT+5" => 401, + "Etc/GMT+6" => 402, + "Etc/GMT+7" => 403, + "Etc/GMT+8" => 404, + "Etc/GMT+9" => 405, + "Etc/GMT-0" => 406, + "Etc/GMT-1" => 407, + "Etc/GMT-10" => 408, + "Etc/GMT-11" => 409, + "Etc/GMT-12" => 410, + "Etc/GMT-13" => 411, + "Etc/GMT-14" => 412, + "Etc/GMT-2" => 413, + "Etc/GMT-3" => 414, + "Etc/GMT-4" => 415, + "Etc/GMT-5" => 416, + "Etc/GMT-6" => 417, + "Etc/GMT-7" => 418, + "Etc/GMT-8" => 419, + "Etc/GMT-9" => 420, + "Etc/GMT0" => 421, + "Etc/Greenwich" => 422, + "Etc/UCT" => 423, + "Etc/UTC" => 424, + "Etc/Universal" => 425, + "Etc/Zulu" => 426, + "Europe/Amsterdam" => 427, + "Europe/Andorra" => 428, + "Europe/Astrakhan" => 429, + "Europe/Athens" => 430, + "Europe/Belfast" => 431, + "Europe/Belgrade" => 432, + "Europe/Berlin" => 433, + "Europe/Bratislava" => 434, + "Europe/Brussels" => 435, + "Europe/Bucharest" => 436, + "Europe/Budapest" => 437, + "Europe/Busingen" => 438, + "Europe/Chisinau" => 439, + "Europe/Copenhagen" => 440, + "Europe/Dublin" => 441, + "Europe/Gibraltar" => 442, + "Europe/Guernsey" => 443, + "Europe/Helsinki" => 444, + "Europe/Isle_of_Man" => 445, + "Europe/Istanbul" => 446, + "Europe/Jersey" => 447, + "Europe/Kaliningrad" => 448, + "Europe/Kiev" => 449, + "Europe/Kirov" => 450, + "Europe/Kyiv" => 451, + "Europe/Lisbon" => 452, + "Europe/Ljubljana" => 453, + "Europe/London" => 454, + "Europe/Luxembourg" => 455, + "Europe/Madrid" => 456, + "Europe/Malta" => 457, + "Europe/Mariehamn" => 458, + "Europe/Minsk" => 459, + "Europe/Monaco" => 460, + "Europe/Moscow" => 461, + "Europe/Nicosia" => 462, + "Europe/Oslo" => 463, + "Europe/Paris" => 464, + "Europe/Podgorica" => 465, + "Europe/Prague" => 466, + "Europe/Riga" => 467, + "Europe/Rome" => 468, + "Europe/Samara" => 469, + "Europe/San_Marino" => 470, + "Europe/Sarajevo" => 471, + "Europe/Saratov" => 472, + "Europe/Simferopol" => 473, + "Europe/Skopje" => 474, + "Europe/Sofia" => 475, + "Europe/Stockholm" => 476, + "Europe/Tallinn" => 477, + "Europe/Tirane" => 478, + "Europe/Tiraspol" => 479, + "Europe/Ulyanovsk" => 480, + "Europe/Uzhgorod" => 481, + "Europe/Vaduz" => 482, + "Europe/Vatican" => 483, + "Europe/Vienna" => 484, + "Europe/Vilnius" => 485, + "Europe/Volgograd" => 486, + "Europe/Warsaw" => 487, + "Europe/Zagreb" => 488, + "Europe/Zaporozhye" => 489, + "Europe/Zurich" => 490, + "Factory" => 491, + "GB" => 492, + "GB-Eire" => 493, + "GMT" => 494, + "GMT+0" => 495, + "GMT-0" => 496, + "GMT0" => 497, + "Greenwich" => 498, + "HST" => 499, + "Hongkong" => 500, + "Iceland" => 501, + "Indian/Antananarivo" => 502, + "Indian/Chagos" => 503, + "Indian/Christmas" => 504, + "Indian/Cocos" => 505, + "Indian/Comoro" => 506, + "Indian/Kerguelen" => 507, + "Indian/Mahe" => 508, + "Indian/Maldives" => 509, + "Indian/Mauritius" => 510, + "Indian/Mayotte" => 511, + "Indian/Reunion" => 512, + "Iran" => 513, + "Israel" => 514, + "Jamaica" => 515, + "Japan" => 516, + "Kwajalein" => 517, + "Libya" => 518, + "MET" => 519, + "MST" => 520, + "MST7MDT" => 521, + "Mexico/BajaNorte" => 522, + "Mexico/BajaSur" => 523, + "Mexico/General" => 524, + "NZ" => 525, + "NZ-CHAT" => 526, + "Navajo" => 527, + "PRC" => 528, + "PST8PDT" => 529, + "Pacific/Apia" => 530, + "Pacific/Auckland" => 531, + "Pacific/Bougainville" => 532, + "Pacific/Chatham" => 533, + "Pacific/Chuuk" => 534, + "Pacific/Easter" => 535, + "Pacific/Efate" => 536, + "Pacific/Enderbury" => 537, + "Pacific/Fakaofo" => 538, + "Pacific/Fiji" => 539, + "Pacific/Funafuti" => 540, + "Pacific/Galapagos" => 541, + "Pacific/Gambier" => 542, + "Pacific/Guadalcanal" => 543, + "Pacific/Guam" => 544, + "Pacific/Honolulu" => 545, + "Pacific/Johnston" => 546, + "Pacific/Kanton" => 547, + "Pacific/Kiritimati" => 548, + "Pacific/Kosrae" => 549, + "Pacific/Kwajalein" => 550, + "Pacific/Majuro" => 551, + "Pacific/Marquesas" => 552, + "Pacific/Midway" => 553, + "Pacific/Nauru" => 554, + "Pacific/Niue" => 555, + "Pacific/Norfolk" => 556, + "Pacific/Noumea" => 557, + "Pacific/Pago_Pago" => 558, + "Pacific/Palau" => 559, + "Pacific/Pitcairn" => 560, + "Pacific/Pohnpei" => 561, + "Pacific/Ponape" => 562, + "Pacific/Port_Moresby" => 563, + "Pacific/Rarotonga" => 564, + "Pacific/Saipan" => 565, + "Pacific/Samoa" => 566, + "Pacific/Tahiti" => 567, + "Pacific/Tarawa" => 568, + "Pacific/Tongatapu" => 569, + "Pacific/Truk" => 570, + "Pacific/Wake" => 571, + "Pacific/Wallis" => 572, + "Pacific/Yap" => 573, + "Poland" => 574, + "Portugal" => 575, + "ROC" => 576, + "ROK" => 577, + "Singapore" => 578, + "Turkey" => 579, + "UCT" => 580, + "US/Alaska" => 581, + "US/Aleutian" => 582, + "US/Arizona" => 583, + "US/Central" => 584, + "US/East-Indiana" => 585, + "US/Eastern" => 586, + "US/Hawaii" => 587, + "US/Indiana-Starke" => 588, + "US/Michigan" => 589, + "US/Mountain" => 590, + "US/Pacific" => 591, + "US/Samoa" => 592, + "UTC" => 593, + "Universal" => 594, + "W-SU" => 595, + "WET" => 596, + "Zulu" => 597, + "posixrules" => 598, + ) + .copied() +} + +/// Returns the IANA time zone name assigned to an identifier. +pub(super) fn id_to_name(id: u16) -> Option<&'static str> { + TZ_NAMES.get(usize::from(id)).copied() +} + +/// Returns the number of entries in the table. +pub(super) const fn len() -> usize { + TZ_NAMES.len() +} + diff --git a/crates/calcard/src/common/writer.rs b/crates/calcard/src/common/writer.rs new file mode 100644 index 00000000..66543574 --- /dev/null +++ b/crates/calcard/src/common/writer.rs @@ -0,0 +1,587 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + format::{AsciiPush, BufferedWrite}, + parser::Timestamp, +}; +use crate::vcard::Jscomp; +use encodify::{Fold, base64}; +use mail_parser::DateTime; +use std::fmt::{self, Display, Write}; + +pub(crate) const MAX_LINE_LEN: usize = 75; +const FOLD: &str = "\r\n "; + +pub(crate) const DOCUMENT_BUFFER: usize = 4096; +pub(crate) const ENTRY_BUFFER: usize = 512; + +const TEXT: u8 = 1; +const TEXT_SEMICOLON: u8 = 1 << 1; +const TEXT_COMMA: u8 = 1 << 2; +const URI: u8 = 1 << 3; +const PARAM: u8 = 1 << 4; +const PARAM_CARET: u8 = 1 << 5; +const JSCOMP: u8 = 1 << 6; +const QUOTE: u8 = 1 << 7; + +const fn byte_class(ch: u8) -> u8 { + let mut class = 0; + if matches!(ch, b'\r' | b'\n' | b'\\') { + class |= TEXT; + } + if ch == b';' { + class |= TEXT_SEMICOLON; + } + if ch == b',' { + class |= TEXT_COMMA; + } + if matches!(ch, b'\r' | b'\n') { + class |= URI; + } + if ch == b'\\' || !matches!(ch, 0x09 | 0x20 | 0x21 | 0x23..=0x7E | 0x80..=0xFF) { + class |= PARAM; + } + if ch == b'^' { + class |= PARAM_CARET; + } + if matches!(ch, b'\\' | b',' | b':' | b'=' | b';' | b'"' | b'\r' | b'\n') { + class |= JSCOMP; + } + if matches!(ch, b',' | b':' | b'=' | b' ' | b';' | b'"') { + class |= QUOTE; + } + class +} + +static BYTE_CLASS: [u8; 256] = { + let mut table = [0; 256]; + let mut ch = 0; + while ch < table.len() { + table[ch] = byte_class(ch as u8); + ch += 1; + } + table +}; + +const LANE_ONES: u64 = 0x0101_0101_0101_0101; +const LANE_HIGHS: u64 = 0x8080_8080_8080_8080; + +#[inline(always)] +const fn lanes_equal(word: u64, byte: u8) -> u64 { + let lanes = word ^ (LANE_ONES * byte as u64); + lanes.wrapping_sub(LANE_ONES) & !lanes & LANE_HIGHS +} + +#[inline(always)] +const fn lanes_below(word: u64, byte: u8) -> u64 { + word.wrapping_sub(LANE_ONES * byte as u64) & !word & LANE_HIGHS +} + +#[inline(always)] +const fn candidate_lanes(word: u64) -> u64 { + let mut lanes = 0; + if MASK & (TEXT | URI | JSCOMP) != 0 { + lanes |= lanes_equal(word, b'\r') | lanes_equal(word, b'\n'); + } + if MASK & (TEXT | PARAM | JSCOMP) != 0 { + lanes |= lanes_equal(word, b'\\'); + } + if MASK & (TEXT_SEMICOLON | JSCOMP | QUOTE) != 0 { + lanes |= lanes_equal(word, b';'); + } + if MASK & (TEXT_COMMA | JSCOMP | QUOTE) != 0 { + lanes |= lanes_equal(word, b','); + } + if MASK & (PARAM | JSCOMP | QUOTE) != 0 { + lanes |= lanes_equal(word, b'"'); + } + if MASK & (JSCOMP | QUOTE) != 0 { + lanes |= lanes_equal(word, b':') | lanes_equal(word, b'='); + } + if MASK & QUOTE != 0 { + lanes |= lanes_equal(word, b' '); + } + if MASK & PARAM != 0 { + lanes |= lanes_below(word, 0x20) | lanes_equal(word, 0x7F); + } + if MASK & PARAM_CARET != 0 { + lanes |= lanes_equal(word, b'^'); + } + lanes +} + +#[inline(always)] +fn find_class(bytes: &[u8]) -> Option { + let (words, tail) = bytes.as_chunks::<8>(); + let mut offset = 0; + for word in words { + let mut lanes = candidate_lanes::(u64::from_le_bytes(*word)); + while lanes != 0 { + let pos = offset + (lanes.trailing_zeros() / 8) as usize; + if MASK & PARAM == 0 + || bytes + .get(pos) + .is_some_and(|&ch| BYTE_CLASS[usize::from(ch)] & MASK != 0) + { + return Some(pos); + } + lanes &= lanes - 1; + } + offset += 8; + } + tail.iter() + .position(|&ch| BYTE_CLASS[usize::from(ch)] & MASK != 0) + .map(|pos| offset + pos) +} + +#[inline(always)] +fn split_at_class(text: &str) -> Option<(&str, u8, &str)> { + let pos = find_class::(text.as_bytes())?; + let (run, rest) = text.split_at_checked(pos)?; + let special = *rest.as_bytes().first()?; + Some((run, special, rest.get(1..)?)) +} + +pub(crate) trait LineWriter: Write + AsciiPush { + fn write_atomic(&mut self, text: &str) -> fmt::Result; + + fn write_base64(&mut self, data: &[u8]) -> fmt::Result; +} + +pub(crate) struct FoldingWriter<'x, W: Write + AsciiPush + ?Sized> { + out: &'x mut W, + line_len: usize, +} + +impl<'x, W: Write + AsciiPush + ?Sized> FoldingWriter<'x, W> { + pub(crate) fn new(out: &'x mut W) -> Self { + Self { out, line_len: 0 } + } + + pub(crate) fn end_line(&mut self) -> fmt::Result { + self.line_len = 0; + self.out.push_ascii(b"\r\n") + } + + pub(crate) fn write_boundary(&mut self, keyword: &str, name: &str) -> fmt::Result { + self.write_atomic(keyword)?; + self.write_atomic(name)?; + self.end_line() + } + + pub(crate) fn write_raw(&mut self, text: &str) -> fmt::Result { + self.out.write_str(text) + } + + #[inline(never)] + fn write_folded(&mut self, text: &str) -> fmt::Result { + let mut rest = text; + + loop { + let available = MAX_LINE_LEN.saturating_sub(self.line_len); + + if rest.len() <= available { + self.line_len += rest.len(); + return self.out.write_str(rest); + } + + match rest.split_at_checked(rest.floor_char_boundary(available)) { + Some((line, tail)) if !line.is_empty() => { + self.out.write_str(line)?; + self.line_len += line.len(); + rest = tail; + } + _ => { + self.out.push_ascii(FOLD.as_bytes())?; + self.line_len = 1; + } + } + } + } + + #[inline(never)] + fn push_ascii_folded(&mut self, bytes: &[u8]) -> fmt::Result { + let mut rest = bytes; + + while !rest.is_empty() { + let available = MAX_LINE_LEN.saturating_sub(self.line_len); + if available == 0 { + self.out.push_ascii(FOLD.as_bytes())?; + self.line_len = 1; + continue; + } + let (line, tail) = rest + .split_at_checked(available.min(rest.len())) + .unwrap_or((rest, &[])); + self.out.push_ascii(line)?; + self.line_len += line.len(); + rest = tail; + } + + Ok(()) + } +} + +impl AsciiPush for FoldingWriter<'_, W> { + #[inline(always)] + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + if bytes.len() <= MAX_LINE_LEN.saturating_sub(self.line_len) { + self.line_len += bytes.len(); + self.out.push_ascii(bytes) + } else { + self.push_ascii_folded(bytes) + } + } +} + +impl LineWriter for FoldingWriter<'_, W> { + #[inline(always)] + fn write_atomic(&mut self, text: &str) -> fmt::Result { + let len = text.len(); + if len > MAX_LINE_LEN - 1 { + return self.write_str(text); + } + if self.line_len + len > MAX_LINE_LEN { + self.out.push_ascii(FOLD.as_bytes())?; + self.line_len = 1; + } + self.line_len += len; + self.out.write_str(text) + } + + fn write_base64(&mut self, data: &[u8]) -> fmt::Result { + let mut result = Ok(()); + let out = &mut *self.out; + base64::STANDARD.encode_folded(data, &mut self.line_len, Fold::CONTENT_LINE, |piece| { + if result.is_ok() { + result = out.push_ascii(piece); + } + }); + result + } +} + +impl LineWriter for String { + fn write_atomic(&mut self, text: &str) -> fmt::Result { + self.push_str(text); + Ok(()) + } + + fn write_base64(&mut self, data: &[u8]) -> fmt::Result { + base64::STANDARD.encode_append(data, self); + Ok(()) + } +} + +impl Write for FoldingWriter<'_, W> { + #[inline] + fn write_str(&mut self, text: &str) -> fmt::Result { + if text.len() <= MAX_LINE_LEN.saturating_sub(self.line_len) { + self.line_len += text.len(); + self.out.write_str(text) + } else { + self.write_folded(text) + } + } + + fn write_char(&mut self, ch: char) -> fmt::Result { + let ch_len = ch.len_utf8(); + if self.line_len + ch_len > MAX_LINE_LEN { + self.out.push_ascii(FOLD.as_bytes())?; + self.line_len = 1; + } + self.line_len += ch_len; + self.out.write_char(ch) + } +} + +const INLINE_COMPONENTS: usize = 256; + +pub(crate) struct VisitedComponents { + inline: [u64; INLINE_COMPONENTS / 64], + spilled: Vec, +} + +impl VisitedComponents { + pub(crate) fn new(len: usize) -> Self { + Self { + inline: [0; INLINE_COMPONENTS / 64], + spilled: if len > INLINE_COMPONENTS { + vec![0; len.div_ceil(64)] + } else { + Vec::new() + }, + } + } + + #[inline] + pub(crate) fn insert(&mut self, id: usize) -> bool { + let words = if self.spilled.is_empty() { + self.inline.as_mut_slice() + } else { + self.spilled.as_mut_slice() + }; + match words.get_mut(id / 64) { + Some(word) => { + let bit = 1 << (id % 64); + let fresh = *word & bit == 0; + *word |= bit; + fresh + } + None => false, + } + } +} + +pub(crate) fn write_text( + out: &mut impl LineWriter, + value: &str, + escape_semicolon: bool, + escape_comma: bool, +) -> fmt::Result { + match (escape_semicolon, escape_comma) { + (true, true) => write_text_class::<{ TEXT | TEXT_SEMICOLON | TEXT_COMMA }>(out, value), + (true, false) => write_text_class::<{ TEXT | TEXT_SEMICOLON }>(out, value), + (false, true) => write_text_class::<{ TEXT | TEXT_COMMA }>(out, value), + (false, false) => write_text_class::(out, value), + } +} + +fn write_text_class(out: &mut impl LineWriter, value: &str) -> fmt::Result { + let mut rest = value; + while let Some((run, special, tail)) = split_at_class::(rest) { + if !run.is_empty() { + out.write_str(run)?; + } + out.write_atomic(match special { + b'\r' => "\\r", + b'\n' => "\\n", + b'\\' => "\\\\", + b';' => "\\;", + _ => "\\,", + })?; + rest = tail; + } + + if rest.is_empty() { + Ok(()) + } else { + out.write_str(rest) + } +} + +pub(crate) fn write_uri(out: &mut impl LineWriter, value: &str) -> fmt::Result { + let mut rest = value; + while let Some((run, special, tail)) = split_at_class::(rest) { + if !run.is_empty() { + out.write_str(run)?; + } + out.write_atomic(if special == b'\r' { "\\r" } else { "\\n" })?; + rest = tail; + } + + if rest.is_empty() { + Ok(()) + } else { + out.write_str(rest) + } +} + +pub(crate) trait NeedsQuotes { + fn needs_quotes(&self) -> bool; +} + +impl> NeedsQuotes for T { + fn needs_quotes(&self) -> bool { + find_class::(self.as_ref()).is_some() + } +} + +pub(crate) fn write_param_value( + out: &mut impl LineWriter, + value: &str, + caret_escape: bool, +) -> fmt::Result { + if value.needs_quotes() { + write_quoted_param_value(out, value, caret_escape) + } else { + write_param_text(out, value, caret_escape) + } +} + +pub(crate) fn write_quoted_param_value( + out: &mut impl LineWriter, + value: &str, + caret_escape: bool, +) -> fmt::Result { + out.write_atomic("\"")?; + write_param_text(out, value, caret_escape)?; + out.write_atomic("\"") +} + +pub(crate) fn write_param_text( + out: &mut impl LineWriter, + value: &str, + caret_escape: bool, +) -> fmt::Result { + if caret_escape { + write_param_text_class::<{ PARAM | PARAM_CARET }>(out, value) + } else { + write_param_text_class::(out, value) + } +} + +fn write_param_text_class(out: &mut impl LineWriter, value: &str) -> fmt::Result { + let caret_escape = MASK & PARAM_CARET != 0; + let mut rest = value; + while let Some((run, special, tail)) = split_at_class::(rest) { + if !run.is_empty() { + out.write_str(run)?; + } + let escaped = match special { + b'\r' if tail.starts_with('\n') => None, + b'\n' | b'\r' if caret_escape => Some("^n"), + b'\n' | b'\r' => Some("\\n"), + b'^' => Some("^^"), + b'"' if caret_escape => Some("^'"), + b'"' => Some("\\\""), + b'\\' => Some("\\\\"), + _ => None, + }; + if let Some(escaped) = escaped { + out.write_atomic(escaped)?; + } + rest = tail; + } + + if rest.is_empty() { + Ok(()) + } else { + out.write_str(rest) + } +} + +pub(crate) enum JscompRef<'x> { + Entry { position: u32, value: u32 }, + Separator(&'x str), +} + +impl<'x> From<&'x Jscomp> for JscompRef<'x> { + fn from(jscomp: &'x Jscomp) -> Self { + match jscomp { + Jscomp::Entry { position, value } => JscompRef::Entry { + position: *position, + value: *value, + }, + Jscomp::Separator(separator) => JscompRef::Separator(separator), + } + } +} + +impl JscompRef<'_> { + fn write_to(self, out: &mut impl LineWriter) -> fmt::Result { + match self { + JscompRef::Entry { position, value } => { + out.push_u64(position.into())?; + if value > 0 { + out.push_byte(b',')?; + out.push_u64(value.into())?; + } + Ok(()) + } + JscompRef::Separator("") => Ok(()), + JscompRef::Separator(separator) => { + out.write_atomic("s,")?; + let mut rest = separator; + while let Some((run, special, tail)) = split_at_class::(rest) { + if !run.is_empty() { + out.write_str(run)?; + } + let escaped = match special { + b'\\' => "\\\\", + b',' => "\\,", + b':' => "\\:", + b'=' => "\\=", + b';' => "\\;", + b'"' => "\\\"", + _ => "", + }; + if !escaped.is_empty() { + out.write_atomic(escaped)?; + } + rest = tail; + } + + if rest.is_empty() { + Ok(()) + } else { + out.write_str(rest) + } + } + } + } +} + +pub(crate) fn write_jscomps<'x, J: 'x>(out: &mut impl LineWriter, values: &'x [J]) -> fmt::Result +where + &'x J: Into>, +{ + for (pos, item) in values.iter().enumerate() { + if pos > 0 { + out.write_atomic(";")?; + } + item.into().write_to(out)?; + } + + Ok(()) +} + +impl Timestamp { + pub(crate) fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + let dt = DateTime::from_timestamp(self.0); + buf.push_4_digits(dt.year)?; + buf.push_2_digits(dt.month)?; + buf.push_2_digits(dt.day)?; + buf.push_byte(b'T')?; + buf.push_2_digits(dt.hour)?; + buf.push_2_digits(dt.minute)?; + buf.push_2_digits(dt.second)?; + buf.push_byte(b'Z') + } +} + +#[cfg(all(test, stalwart_tests))] +pub(crate) fn assert_fold_width(text: &str, context: &str) { + let mut lines = text.split("\r\n").peekable(); + let is_v21 = text.contains("\r\nVERSION:2.1\r\n"); + let mut ends_v21_base64 = false; + + while let Some(line) = lines.next() { + if !line.is_empty() && !line.starts_with(' ') { + ends_v21_base64 = is_v21 && line.contains(";ENCODING=BASE64"); + } + assert!( + line.len() <= MAX_LINE_LEN, + "physical line of {} octets exceeds the {MAX_LINE_LEN} octet fold width in {context}: {line:?}", + line.len() + ); + assert!( + !line.is_empty() || lines.peek().is_none() || std::mem::take(&mut ends_v21_base64), + "empty physical line in {context}: {text:?}" + ); + assert!( + line != " ", + "continuation line holding nothing but the fold character in {context}: {text:?}" + ); + } +} + +impl Display for Timestamp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::<32>(|buf| self.push_to(buf)) + } +} diff --git a/crates/calcard/src/datecalc/error.rs b/crates/calcard/src/datecalc/error.rs new file mode 100644 index 00000000..afc5a58d --- /dev/null +++ b/crates/calcard/src/datecalc/error.rs @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: 2021 Fredrik Meringdal, Ralph Bisschops + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::{ + common::{CalendarScale, IanaString}, + icalendar::ICalendarFrequency, +}; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub enum RRuleError { + ValidationError(ValidationError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub enum ValidationError { + BySetPosWithoutByRule, + InvalidFieldValue { + field: String, + value: String, + }, + InvalidFieldValueRange { + field: String, + value: String, + start_idx: String, + end_idx: String, + }, + InvalidFieldValueRangeWithFreq { + field: String, + value: String, + freq: ICalendarFrequency, + start_idx: String, + end_idx: String, + }, + InvalidByRuleAndFrequency { + by_rule: String, + freq: ICalendarFrequency, + }, + TooBigInterval(u16), + /// The rule is written in a calendar other than the Gregorian one. + /// + /// RFC 7529 scales place instances on entirely different dates, so a rule + /// naming one cannot be expanded by treating the scale as absent. + UnsupportedCalendarScale(CalendarScale), +} + +impl From for RRuleError { + fn from(err: ValidationError) -> Self { + Self::ValidationError(err) + } +} + +impl Display for RRuleError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + RRuleError::ValidationError(err) => write!(f, "{err}"), + } + } +} + +impl Display for ValidationError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ValidationError::BySetPosWithoutByRule => { + write!(f, "BYSETPOS without BYxxx rule is not allowed") + } + ValidationError::InvalidFieldValue { field, value } => { + write!(f, "Invalid value `{value}` for field `{field}`") + } + ValidationError::InvalidFieldValueRange { + field, + value, + start_idx, + end_idx, + } => write!( + f, + "Invalid value `{value}` for field `{field}`, start index `{start_idx}` and end index `{end_idx}`" + ), + ValidationError::InvalidFieldValueRangeWithFreq { + field, + value, + freq, + start_idx, + end_idx, + } => write!( + f, + "Invalid value `{value}` for field `{field}`, frequency `{}`, start index `{start_idx}` and end index `{end_idx}`", + freq.as_str() + ), + ValidationError::InvalidByRuleAndFrequency { by_rule, freq } => write!( + f, + "Invalid BY rule `{by_rule}` with frequency `{}`", + freq.as_str() + ), + ValidationError::TooBigInterval(interval) => write!( + f, + "Interval of {interval} is too big. The maximum interval is 32767." + ), + ValidationError::UnsupportedCalendarScale(rscale) => write!( + f, + "Recurrence rules in the {} calendar are not supported", + rscale.as_str() + ), + } + } +} diff --git a/crates/calcard/src/datecalc/mod.rs b/crates/calcard/src/datecalc/mod.rs new file mode 100644 index 00000000..979c1896 --- /dev/null +++ b/crates/calcard/src/datecalc/mod.rs @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Recurrence rule expansion. +//! +//! The engine in [`recurrence`] is a port of the one in +//! [`bttf`](https://github.com/BurntSushi/bttf) by Andrew Gallant; see that +//! module for what was changed. [`rrule`] adapts a parsed `RRULE` property +//! onto it. + +pub mod error; +pub mod recurrence; +pub mod rrule; +mod weekdate; + +pub use recurrence::{MAX_ITER_LOOP, MAX_UNPRODUCTIVE_WORK, RecurrenceIter}; +pub use rrule::RRule; diff --git a/crates/calcard/src/datecalc/recurrence.rs b/crates/calcard/src/datecalc/recurrence.rs new file mode 100644 index 00000000..d00b4c08 --- /dev/null +++ b/crates/calcard/src/datecalc/recurrence.rs @@ -0,0 +1,2217 @@ +/* + * SPDX-FileCopyrightText: 2025 Andrew Gallant (https://github.com/BurntSushi/bttf) + * + * SPDX-License-Identifier: Unlicense OR MIT + */ + +//! The RFC 5545 recurrence rule engine. +//! +//! Ported from [`bttf`](https://github.com/BurntSushi/bttf) by Andrew Gallant, +//! with these changes: +//! +//! * `anyhow` errors became [`ValidationError`], so the crate stays free of an +//! error reporting dependency and a bad rule is reported the way the rest of +//! calcard reports one. +//! * Instants are [`ZonedDateTime`] rather than [`jiff::Zoned`], because +//! RFC 5545 has floating values, which carry no zone, and because a +//! `ZonedDateTime` is `Copy` and so costs no reference counting per instance. +//! * Ambiguous wall clock readings follow RFC 5545 section 3.3.5 through +//! [`Tz::from_local`]; see [`RecurrenceSet::pop_civil`]. +//! * COUNT and an iteration budget were added, neither of which `bttf` needs. + +use super::error::ValidationError; +use crate::common::timezone::{Tz, ZonedDateTime}; +use crate::datecalc::weekdate::{WeekDate, first_of_week, last_of_week, weeks_in_year}; +use crate::icalendar::ICalendarFrequency; +use jiff::{ + SignedDuration, Span, Timestamp, ToSpan, + civil::{DateTime, Time, Weekday}, +}; +use std::{ + cmp::Ordering, + ops::{Range, RangeInclusive}, + sync::Arc, +}; + +/// Stops an expansion that keeps finding nothing. +/// +/// A rule such as `FREQ=SECONDLY;BYMONTH=2;BYMONTHDAY=31` matches no date at +/// all, so without a bound the iterator would step one second at a time up to +/// the year 9999. The budget counts attempts that yield no instance and is +/// reset whenever one is produced, so a long but productive series is never +/// cut short. +pub const MAX_ITER_LOOP: u32 = 100_000; + +pub const MAX_PERIOD_CANDIDATES: usize = 100_000; + +pub const MAX_EXPANSION_WORK: usize = 10_000_000; + +pub const MAX_UNPRODUCTIVE_WORK: usize = 1_000_000; + +const ALL_WEEKDAYS: u8 = 0b111_1111; + +/// The RFC 5545 recurrence rule implementation. +#[derive(Clone, Debug)] +pub struct RecurrenceRule { + inner: Arc, +} + +#[derive(Debug)] +struct RecurrenceRuleInner { + freq: ICalendarFrequency, + start: ZonedDateTime, + civil_start: DateTime, + until: Option, + count: Option, + interval: Span, + by_month: Box<[i8]>, + // can be negative + by_week: Box<[i8]>, + // can be negative + by_year_day: Box<[i16]>, + // can be negative + by_month_day: Box<[i8]>, + // can be negative + by_week_day: Box<[ByWeekday]>, + by_hour: Box<[i8]>, + by_minute: Box<[i8]>, + by_second: Box<[i8]>, + // can be negative + by_set_pos: Box<[i32]>, + week_start: Weekday, +} + +impl RecurrenceRule { + /// Returns a builder for constructing a `RecurrenceRule`. + /// + /// The frequency and the starting point are the only two things required + /// to create a rule. + pub fn builder(freq: ICalendarFrequency, start: ZonedDateTime) -> RecurrenceRuleBuilder { + RecurrenceRuleBuilder::new(freq, start) + } + + /// Returns an iterator over all datetimes in this recurrence rule. + /// + /// Note that the iterator may be "infinite," in the sense that it returns + /// datetimes all the way up to Jiff's supported maximum datetime. Callers + /// should therefore either specify an `RecurrenceRuleBuilder::until` rule + /// or call `take(N)` to limit the number of datetimes to `N`. + pub fn iter(&self) -> RecurrenceIter<'_> { + RecurrenceIter { + rule: self, + set: RecurrenceSet::new(), + cur: Some((0, self.inner.civil_start)), + remaining: self.inner.count, + budget: MAX_ITER_LOOP, + work: MAX_EXPANSION_WORK, + unproductive: usize::MAX, + period_work: 0, + exhausted: false, + gap_instants: Vec::new(), + } + } + + /// Returns the time zone that datetimes emitted by this rule should be in. + pub fn tz(&self) -> Tz { + self.inner.start.timezone() + } +} + +impl<'r> IntoIterator for &'r RecurrenceRule { + type IntoIter = RecurrenceIter<'r>; + type Item = ZonedDateTime; + + fn into_iter(self) -> RecurrenceIter<'r> { + self.iter() + } +} + +/// An expander for a single RFC 5545 recurrence rule and datetime. +#[derive(Clone, Debug)] +struct Expander<'a> { + /// The rule we are expanding. + rule: &'a RecurrenceRule, + /// The "current" datetime we are expanding. + /// + /// How this datetime is used depends on the frequency we are expanding + /// for. + cur: DateTime, +} + +impl<'a> Expander<'a> { + /// Expand into the set provided. + fn expand(&self, set: &mut RecurrenceSet) { + set.begin_period(self.cur); + match self.frequency() { + ICalendarFrequency::Yearly => self.yearly(set), + ICalendarFrequency::Monthly => self.monthly(set), + ICalendarFrequency::Weekly => self.weekly(set), + ICalendarFrequency::Daily => self.daily(set), + ICalendarFrequency::Hourly => self.hourly(set), + ICalendarFrequency::Minutely => self.minutely(set), + ICalendarFrequency::Secondly => self.secondly(set), + } + set.canonicalize(self.rule()); + + // When BYSETPOS is present, we need to convert all of our civil + // datetimes to zoned datetimes, check them with our start/until rules + // and put them into another set. Only when we do this can we process + // the BYSETPOS rule. In particular, BYSETPOS may contain negative + // values, which fundamentally requires knowing the length of the + // recurrence set. + // + // We could in theory do better here in the case where BYSETPOS is only + // positive values, but it seems like it's most commonly used with + // negative values. Moreover, BYSETPOS is probably rare by itself, + // so it's not clear that we need to care too much about this extra + // shuffling. + if self.has_by_set_pos() { + set.select_positions(self.rule); + } else { + let start = self.rule().civil_start; + set.seek(self.rule(), start.checked_sub(1.day()).unwrap_or(start)); + } + } + + /// Populate `set` with datetimes according to this rule at a YEARLY + /// frequency. + fn yearly(&self, set: &mut RecurrenceSet) { + set.insert(self.cur); + if self.has_by_week_day() { + if self.has_by_week() { + self.expand_by_week(set); + self.limit_by_month(set); + } else if self.has_by_month() { + self.expand_by_month(set); + self.expand_by_week_day_monthly(set); + } else { + self.expand_by_week_day_yearly(set); + } + self.limit_by_year_day(set); + self.limit_by_month_day(set); + } else if self.has_by_week() { + self.expand_by_week(set); + // Note that we use slightly different expansionary + // behavior here for `by_week` than for other things[1]. + // For example, at YEARLY frequency with BYMONTH={2,5}, + // you'll get two datetimes in February and May with the + // day and time of the starting point. You don't get every + // day in the specified months. But when `by_week` is + // present, you get every day in each week specified. + // + // This is... somewhat odd. But I guess makes sense for + // certain kinds of expressions. And it's also what + // Python's `dateutil` does. So we do the same. + // + // [1]: https://stackoverflow.com/questions/48064349 + self.limit_by_month(set); + self.limit_by_year_day(set); + self.limit_by_month_day(set); + } else if self.has_by_month() && (self.has_by_month_day() || !self.has_by_year_day()) { + self.expand_by_month(set); + self.expand_by_month_day(set); + self.limit_by_year_day(set); + } else if self.has_by_month() { + self.expand_by_year_day(set); + self.limit_by_month(set); + } else if self.has_by_month_day() { + set.expand(|dt| { + (1..=12).filter_map(move |month| dt.with().month(month).day(1).build().ok()) + }); + self.expand_by_month_day(set); + self.limit_by_year_day(set); + } else if self.has_by_year_day() { + self.expand_by_year_day(set); + } + self.expand_by_hour(set); + self.expand_by_minute(set); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a MONTHLY + /// frequency. + fn monthly(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNO and BYYEARDAY are not allowed here. + if !self.satisfies_by_month(self.cur) { + return; + } + + set.insert(self.cur); + if self.has_by_week_day() { + self.expand_by_week_day_monthly(set); + self.limit_by_month_day(set); + } else { + self.expand_by_month_day(set); + } + self.expand_by_hour(set); + self.expand_by_minute(set); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a WEEKLY + /// frequency. + fn weekly(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNO, BYYEARDAY and BYMONTHDAY are not allowed here. Also, + // BYDAY cannot have a numeric weekday. + set.insert(self.cur); + self.expand_by_week_day_weekly(set); + // BYMONTH limits each day the week expands to, not the day the week + // is anchored on. A week runs across a month boundary often enough + // that testing the anchor instead would both admit days outside the + // requested months and discard days inside them. + self.limit_by_month(set); + self.expand_by_hour(set); + self.expand_by_minute(set); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a DAILY + /// frequency. + fn daily(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNO and BYYEARDAY are not allowed here. Also, BYDAY + // cannot have a numeric weekday. I get the restriction on BYDAY, but + // it seems like BYWEEKNO and BYYEARDAY could be trivially supported + // in exactly the same way that BYMONTH and BYMONTHDAY are supported + // below. + // + // This is just so deliciously easy compared to YEARLY... + if !self.satisfies_by_month(self.cur) { + return; + } + if !self.satisfies_by_month_day(self.cur) { + return; + } + if !self.satisfies_by_week_day(self.cur) { + return; + } + set.insert(self.cur); + self.expand_by_hour(set); + self.expand_by_minute(set); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a HOURLY + /// frequency. + fn hourly(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNOis not allowed here. Also, BYDAY cannot have a numeric + // weekday. + if !self.satisfies_by_month(self.cur) { + return; + } + if !self.satisfies_by_year_day(self.cur) { + return; + } + if !self.satisfies_by_month_day(self.cur) { + return; + } + if !self.satisfies_by_week_day(self.cur) { + return; + } + if !self.satisfies_by_hour(self.cur) { + return; + } + set.insert(self.cur); + self.expand_by_minute(set); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a MINUTELY + /// frequency. + fn minutely(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNOis not allowed here. Also, BYDAY cannot have a numeric + // weekday. + if !self.satisfies_by_month(self.cur) { + return; + } + if !self.satisfies_by_year_day(self.cur) { + return; + } + if !self.satisfies_by_month_day(self.cur) { + return; + } + if !self.satisfies_by_week_day(self.cur) { + return; + } + if !self.satisfies_by_hour(self.cur) { + return; + } + if !self.satisfies_by_minute(self.cur) { + return; + } + set.insert(self.cur); + self.expand_by_second(set); + } + + /// Populate `set` with datetimes according to this rule at a SECONDLY + /// frequency. + fn secondly(&self, set: &mut RecurrenceSet) { + // N.B. BYWEEKNOis not allowed here. Also, BYDAY cannot have a numeric + // weekday. + if !self.satisfies_by_month(self.cur) { + return; + } + if !self.satisfies_by_year_day(self.cur) { + return; + } + if !self.satisfies_by_month_day(self.cur) { + return; + } + if !self.satisfies_by_week_day(self.cur) { + return; + } + if !self.satisfies_by_hour(self.cur) { + return; + } + if !self.satisfies_by_minute(self.cur) { + return; + } + if !self.satisfies_by_second(self.cur) { + return; + } + set.insert(self.cur); + } + + /// Returns true if there is at least one BYMONTH value. + fn has_by_month(&self) -> bool { + !self.rule().by_month.is_empty() + } + + /// Returns true if there is at least one BYWEEKNO value. + fn has_by_week(&self) -> bool { + !self.rule().by_week.is_empty() + } + + /// Returns true if there is at least one BYYEARDAY value. + fn has_by_year_day(&self) -> bool { + !self.rule().by_year_day.is_empty() + } + + /// Returns true if there is at least one BYMONTHDAY value. + fn has_by_month_day(&self) -> bool { + !self.rule().by_month_day.is_empty() + } + + /// Returns true if there is at least one BYDAY value. + fn has_by_week_day(&self) -> bool { + !self.rule().by_week_day.is_empty() + } + + /// Returns true if there is at least one BYHOUR value. + fn has_by_hour(&self) -> bool { + !self.rule().by_hour.is_empty() + } + + /// Returns true if there is at least one BYMINUTE value. + fn has_by_minute(&self) -> bool { + !self.rule().by_minute.is_empty() + } + + /// Returns true if there is at least one BYSECOND value. + fn has_by_second(&self) -> bool { + !self.rule().by_second.is_empty() + } + + /// Returns true if there is at least one BYSETPOS value. + fn has_by_set_pos(&self) -> bool { + !self.rule().by_set_pos.is_empty() + } + + /// Returns true if and only if the given datetime satisfies the + /// BYMONTH rule. + fn satisfies_by_month(&self, dt: DateTime) -> bool { + !self.has_by_month() || self.rule().by_month.contains(&dt.month()) + } + + /// Returns true if and only if the given datetime satisfies the + /// BYYEARDAY rule. + fn satisfies_by_year_day(&self, dt: DateTime) -> bool { + if !self.has_by_year_day() { + return true; + } + let positive = dt.day_of_year(); + // Minus 1 because -1 is the last day of the year, and the days of the + // year are 1-indexed. + let negative = positive - 1 - dt.days_in_year(); + self.rule().by_year_day.binary_search(&positive).is_ok() + || self.rule().by_year_day.binary_search(&negative).is_ok() + } + + /// Returns true if and only if the given datetime satisfies the + /// BYMONTHDAY rule. + fn satisfies_by_month_day(&self, dt: DateTime) -> bool { + if !self.has_by_month_day() { + return true; + } + let positive = dt.day(); + // Minus 1 because -1 is the last day of month, and the days of the + // month are 1-indexed. + let negative = positive - 1 - dt.days_in_month(); + self.rule().by_month_day.binary_search(&positive).is_ok() + || self.rule().by_month_day.binary_search(&negative).is_ok() + } + + /// Returns true only if the weekday for the given datetime is allowed by + /// this recurrence rule. + /// + /// # Panics + /// + /// This panics when this is a numbered weekday. In effect, this shouldn't + /// be used in contexts where BYDAY can contain numbered weekdays. That + /// limits this to YEARLY and MONTHLY frequencies (and in the YEARLY case, + /// only when BYWEEKNO is not used). + fn satisfies_by_week_day(&self, dt: DateTime) -> bool { + if !self.has_by_week_day() { + return true; + } + let wd = dt.weekday(); + self.rule().by_week_day.iter().any(|bywd| bywd.is_match(wd)) + } + + /// Returns true if and only if the given datetime satisfies the + /// BYHOUR rule. + fn satisfies_by_hour(&self, dt: DateTime) -> bool { + !self.has_by_hour() || self.rule().by_hour.contains(&dt.hour()) + } + + /// Returns true if and only if the given datetime satisfies the + /// BYMINUTE rule. + fn satisfies_by_minute(&self, dt: DateTime) -> bool { + !self.has_by_minute() || self.rule().by_minute.contains(&dt.minute()) + } + + /// Returns true if and only if the given datetime satisfies the + /// BYSECOND rule. + fn satisfies_by_second(&self, dt: DateTime) -> bool { + !self.has_by_second() || self.rule().by_second.contains(&dt.second()) + } + + /// Removes any element in the given set whose month is + /// inconsistent with the BYMONTH rule. + /// + /// If the BYMONTH rule is empty, then this is a no-op. + fn limit_by_month(&self, set: &mut RecurrenceSet) { + if !self.has_by_month() { + return; + } + set.retain(|dt| self.satisfies_by_month(*dt)); + } + + /// Removes any element in the given set whose day of the year is + /// inconsistent with the BYYEARDAY rule. + /// + /// If the BYYEARDAY rule is empty, then this is a no-op. + fn limit_by_year_day(&self, set: &mut RecurrenceSet) { + if !self.has_by_year_day() { + return; + } + set.retain(|dt| self.satisfies_by_year_day(*dt)); + } + + /// Removes any element in the given set whose day of the month is + /// inconsistent with the BYMONTHDAY rule. + /// + /// If the BYMONTHDAY rule is empty, then this is a no-op. + fn limit_by_month_day(&self, set: &mut RecurrenceSet) { + if !self.has_by_month_day() { + return; + } + set.retain(|dt| self.satisfies_by_month_day(*dt)); + } + + /// When the rule has a non-empty number of BYMONTH values, then this + /// expands every datetime in the given set with the corresponding + /// months. + fn expand_by_month(&self, set: &mut RecurrenceSet) { + if !self.has_by_month() { + return; + } + set.expand(|dt| self.iter_by_month(dt)) + } + + /// When the rule has a non-empty number of BYWEEKNO values, then this + /// expands every datetime in the given set with the corresponding + /// week numbers. + fn expand_by_week(&self, set: &mut RecurrenceSet) { + if !self.has_by_week() { + return; + } + set.expand(|dt| self.iter_by_week(dt)) + } + + /// When the rule has a non-empty number of BYYEARDAY values, then this + /// expands every datetime in the given set with the corresponding + /// year days. + fn expand_by_year_day(&self, set: &mut RecurrenceSet) { + if !self.has_by_year_day() { + return; + } + set.expand(|dt| self.iter_by_year_day(dt)) + } + + /// When the rule has a non-empty number of BYMONTHDAY values, then this + /// expands every datetime in the given set with the corresponding + /// month days. + fn expand_by_month_day(&self, set: &mut RecurrenceSet) { + if !self.has_by_month_day() { + return; + } + set.expand(|dt| self.iter_by_month_day(dt)) + } + + /// When the rule has a non-empty number of BYDAY values, then this + /// expands every datetime in the given set with the corresponding + /// BYDAY rules at a YEARLY frequency. + fn expand_by_week_day_yearly(&self, set: &mut RecurrenceSet) { + if !self.has_by_week_day() { + return; + } + set.expand(|dt| self.iter_by_week_day_yearly(dt)) + } + + /// When the rule has a non-empty number of BYDAY values, then this + /// expands every datetime in the given set with the corresponding + /// BYDAY rules at a MONTHLY frequency. + fn expand_by_week_day_monthly(&self, set: &mut RecurrenceSet) { + if !self.has_by_week_day() { + return; + } + set.expand(|dt| self.iter_by_week_day_monthly(dt)) + } + + /// When the rule has a non-empty number of BYDAY values, then this + /// expands every datetime in the given set with the corresponding + /// BYDAY rules at a WEEKLY frequency. + /// + /// # Panics + /// + /// When any `ByWeekday` is `Numbered`. RFC 5545 doesn't permit that + /// construction at anything other than YEARLY and MONTHLY frequency. + fn expand_by_week_day_weekly(&self, set: &mut RecurrenceSet) { + if !self.has_by_week_day() { + return; + } + set.expand(|dt| self.iter_by_week_day_weekly(dt)) + } + + /// When the rule has a non-empty number of BYHOUR values, then this + /// expands every datetime in the given set with the corresponding + /// hours. + fn expand_by_hour(&self, set: &mut RecurrenceSet) { + if self.has_by_hour() { + set.hours = Axis::Rule; + } + } + + /// When the rule has a non-empty number of BYMINUTE values, then this + /// expands every datetime in the given set with the corresponding + /// minutes. + fn expand_by_minute(&self, set: &mut RecurrenceSet) { + if self.has_by_minute() { + set.minutes = Axis::Rule; + } + } + + /// When the rule has a non-empty number of BYSECOND values, then this + /// expands every datetime in the given set with the corresponding + /// seconds. + fn expand_by_second(&self, set: &mut RecurrenceSet) { + if self.has_by_second() { + set.seconds = Axis::Rule; + } + } + + /// Returns an iterator over the BYMONTH values in this recurrence rule. + /// + /// The values returned are datetimes with each of the corresponding + /// months. The other parts of the datetime are copied from `dt`. + /// + /// If there are no month values, then this iterator does not yield + /// any items. + fn iter_by_month(&self, dt: DateTime) -> impl Iterator { + let day = self.rule().civil_start.day(); + // BYMONTH is always followed here by a rule that sets the day of the + // month, so when the starting point's day does not exist in a month + // the month itself must still be kept: `DTSTART` on October 31 with + // `BYMONTH=11;BYDAY=4SU` means the fourth Sunday of November, not + // nothing at all. Only the month is carried in that case. + let day_is_replaced = self.has_by_week_day() || self.has_by_month_day(); + self.rule() + .by_month + .iter() + .copied() + .filter_map(move |month| { + // This is subtle, but in the case where BYMONTHDAY is set, the + // starting point is Feb 29 and all other date modifying rules are + // empty, then the day here could have been constrained to Feb 28. + // In which case, we really should pull the day from the actual + // starting point. + // + // (I kinda wonder if we should do this more broadly, but it + // doesn't appear necessary.) + dt.with().month(month).day(day).build().ok().or_else(|| { + day_is_replaced + .then(|| dt.with().month(month).day(1).build().ok()) + .flatten() + }) + }) + } + + /// Returns an iterator over the BYWEEKNO values in this recurrence rule. + /// + /// The values returned are the days of the year of `dt` whose week number, + /// in the week year the day belongs to, matches a BYWEEKNO value, limited + /// to the BYDAY weekdays when there are any. The time is copied from `dt`. + /// + /// If there are no week number values, then this iterator does not yield + /// any items. + fn iter_by_week(&self, dt: DateTime) -> impl Iterator { + let rule = self.rule(); + let week_start = rule.week_start; + let year = dt.year(); + let time = dt.time(); + let offsets = match rule.by_week_day.iter().fold(0u8, |mask, weekday| { + mask | (1 << weekday.weekday().since(week_start)) + }) { + 0 => ALL_WEEKDAYS, + mask => mask, + }; + let first_offset = i64::from(offsets.trailing_zeros()); + let last_offset = i64::from(u8::BITS - 1 - offsets.leading_zeros()); + [year.saturating_sub(1), year, year.saturating_add(1)] + .into_iter() + .flat_map(move |week_year| { + let weeks_in_year = weeks_in_year(week_start, week_year); + rule.by_week.iter().filter_map(move |&week| { + let week = if week.is_negative() { + weeks_in_year.checked_add(week + 1)? + } else { + week + }; + if (week_year < year && week != weeks_in_year) + || (week_year > year && week != 1) + { + return None; + } + WeekDate::new(week_start, week_year, week, week_start)?.date() + }) + }) + .filter_map(move |first_day| match first_offset { + 0 => Some(first_day), + offset => first_day.checked_add(offset.days()).ok(), + }) + .flat_map(move |first_match| { + (first_offset..=last_offset) + .zip(std::iter::successors(Some(first_match), |day| { + day.tomorrow().ok() + })) + .filter(move |(offset, day)| offsets & (1 << offset) != 0 && day.year() == year) + .map(move |(_, day)| day.to_datetime(time)) + }) + } + + /// Returns an iterator over the BYYEARDAY values in this recurrence rule. + /// + /// The values returned are datetimes with each of the corresponding + /// days of the year. The other parts of the datetime are copied from `dt`. + /// + /// This handles any negative day of the year values according to the + /// number of days of the year in `dt`. + /// + /// If there are no day of the year values, then this iterator does not + /// yield any items. + fn iter_by_year_day(&self, dt: DateTime) -> impl Iterator { + let days_in_year = dt.days_in_year(); + self.rule() + .by_year_day + .iter() + .copied() + .filter_map(move |mut day| { + if day.is_negative() { + // Add 1 because -1 is the last day of the year, and the days + // of the year are 1-indexed. + day = days_in_year.checked_add(day + 1)?; + } + dt.with().day_of_year(day).build().ok() + }) + } + + /// Returns an iterator over the BYMONTHDAY values in this recurrence rule. + /// + /// The values returned are datetimes with each of the corresponding + /// month days. The other parts of the datetime are copied from `dt`. + /// + /// This handles any negative month day values according to the number of + /// days of the month in `dt`. + /// + /// If there are no month day values, then this iterator does not yield + /// any items. + fn iter_by_month_day(&self, dt: DateTime) -> impl Iterator { + let days_in_month = dt.days_in_month(); + self.rule() + .by_month_day + .iter() + .copied() + .filter_map(move |mut day| { + if day.is_negative() { + // Add 1 because -1 is the last day of month, and the days of + // the month are 1-indexed. + day = days_in_month.checked_add(day + 1)?; + } + dt.with().day(day).build().ok() + }) + } + + /// Returns an iterator over the BYDAY values in this recurrence rule at + /// a YEARLY frequency. + /// + /// The values returned are datetimes with each of the corresponding + /// weekdays. The other parts of the datetime are copied from `dt`. + /// + /// This handles any negative numbered weekday values. Negative values are + /// interpreted with respect to the end of the year. Where as positive + /// values are interpreted with respect to the beginning of the year. + /// (Where the year is taken from `dt`.) + /// + /// If there are no week day values, then this iterator does not yield + /// any items. + fn iter_by_week_day_yearly(&self, dt: DateTime) -> impl Iterator { + let start = dt.first_of_year(); + let end = dt.last_of_year(); + self.rule() + .by_week_day + .iter() + .copied() + .flat_map(move |weekday| weekday.iter_yearly(start, end)) + } + + /// Returns an iterator over the BYDAY values in this recurrence rule at + /// a MONTHLY frequency. + /// + /// The values returned are datetimes with each of the corresponding + /// weekdays. The other parts of the datetime are copied from `dt`. + /// + /// This handles any negative numbered weekday values. Negative values are + /// interpreted with respect to the end of the month. Where as positive + /// values are interpreted with respect to the beginning of the month. + /// (Where the month is taken from `dt`.) + /// + /// If there are no week day values, then this iterator does not yield + /// any items. + fn iter_by_week_day_monthly(&self, dt: DateTime) -> impl Iterator { + let start = dt.first_of_month(); + let end = dt.last_of_month(); + self.rule() + .by_week_day + .iter() + .copied() + .flat_map(move |weekday| weekday.iter_monthly(start, end)) + } + + /// Returns an iterator over the BYDAY values in this recurrence rule at + /// a WEEKLY frequency. + /// + /// The values returned are datetimes with each of the corresponding + /// weekdays. The other parts of the datetime are copied from `dt`. + /// + /// If there are no week day values, then this iterator does not yield + /// any items. + /// + /// # Panics + /// + /// When any `ByWeekday` is `Numbered`. RFC 5545 doesn't permit that + /// construction at anything other than YEARLY and MONTHLY frequency. + fn iter_by_week_day_weekly(&self, dt: DateTime) -> impl Iterator { + // I tried writing this as one iterator chain, but it was a complex + // mess. Since I already defined `Either` for other uses, I felt it + // was simpler to just use it here. + let Some(start) = first_of_week(self.rule().week_start, dt.date()) else { + return Either::Left(std::iter::empty()); + }; + let Some(end) = last_of_week(self.rule().week_start, dt.date()) else { + return Either::Left(std::iter::empty()); + }; + let Ok(start) = dt.with().date(start).build() else { + return Either::Left(std::iter::empty()); + }; + let Ok(end) = dt.with().date(end).build() else { + return Either::Left(std::iter::empty()); + }; + + Either::Right( + self.rule() + .by_week_day + .iter() + .copied() + .flat_map(move |weekday| weekday.iter_weekly(start, end)), + ) + } + + /// Returns the frequency at which we are generating datetimes. + /// + /// The frequency determines how we interpret the `by_*` data in our rule + /// to generate datetimes. + fn frequency(&self) -> ICalendarFrequency { + self.rule().freq + } + + /// Returns the "inner" rule associated with this expander. + /// + /// This is somewhat of a misnomer, since `self.rule` is the actual + /// rule, but this returns the "inner" rule. But the "inner" rule is + /// where all the goods are. + fn rule(&self) -> &RecurrenceRuleInner { + &self.rule.inner + } +} + +#[derive(Clone, Debug)] +pub struct RecurrenceIter<'r> { + /// The recurrence rule that we're generating zoned datetimes for. + rule: &'r RecurrenceRule, + /// The set of datetimes that we should drain and emit before + /// incrementing by our interval and refilling the set. + set: RecurrenceSet, + /// The interval index along with the current datetime. + /// + /// In order to get the next datetime, the interval index should be + /// incremented by one and then multiplied by the frequency `Span`. This + /// is done instead of just adding to the previous datetime to avoid + /// cases where we go from 2025-03-31 -> 2025-04-30 -> 2025-05-30 instead + /// of 2025-03-31 -> 2025-04-30 -> 2025-05-31. + /// + /// When this is `None`, iteration has ceased. + cur: Option<(i64, DateTime)>, + /// Instances left to emit under COUNT, if the rule sets one. + remaining: Option, + /// Attempts left before the expansion gives up; see [`MAX_ITER_LOOP`]. + budget: u32, + work: usize, + unproductive: usize, + period_work: usize, + /// Whether the budget ran out with the rule still unfinished. + exhausted: bool, + gap_instants: Vec, +} + +impl<'r> RecurrenceIter<'r> { + fn expand(&mut self) { + let Some((_, cur)) = self.cur else { return }; + Expander { + rule: self.rule, + cur, + } + .expand(&mut self.set); + } + + fn is_done(&self) -> bool { + self.cur.is_none() && self.set.is_empty() + } + + /// Returns whether iteration stopped at the budget rather than at the end + /// of the rule, so the caller can tell a truncated series from a finished + /// one. + pub fn is_exhausted(&self) -> bool { + self.exhausted + } + + pub fn with_unproductive_budget(mut self, units: usize) -> Self { + self.unproductive = units; + self + } + + pub fn unproductive_budget(&self) -> usize { + self.unproductive.saturating_sub(self.set.spent) + } + + fn period_start(&self, dt: DateTime) -> Option { + let rule = &self.rule.inner; + match rule.freq { + ICalendarFrequency::Yearly => Some(dt.first_of_year().start_of_day()), + ICalendarFrequency::Monthly => Some(dt.first_of_month().start_of_day()), + ICalendarFrequency::Weekly => { + first_of_week(rule.week_start, dt.date()).map(|date| date.to_datetime(Time::MIN)) + } + ICalendarFrequency::Daily => Some(dt.start_of_day()), + ICalendarFrequency::Hourly => dt.with().minute(0).second(0).build().ok(), + ICalendarFrequency::Minutely => dt.with().second(0).build().ok(), + ICalendarFrequency::Secondly => Some(dt), + } + } + + fn skip_ahead(&self, attempt: i64, next: DateTime) -> Option<(i64, DateTime)> { + let rule = &self.rule.inner; + if !matches!( + rule.freq, + ICalendarFrequency::Hourly + | ICalendarFrequency::Minutely + | ICalendarFrequency::Secondly + ) { + return None; + } + let expander = Expander { + rule: self.rule, + cur: next, + }; + let boundary = if !expander.satisfies_by_month(next) + || !expander.satisfies_by_year_day(next) + || !expander.satisfies_by_month_day(next) + || !expander.satisfies_by_week_day(next) + { + next.date().tomorrow().ok()?.to_datetime(Time::MIN) + } else if rule.freq != ICalendarFrequency::Hourly && !expander.satisfies_by_hour(next) { + next.with() + .minute(0) + .second(0) + .build() + .ok()? + .checked_add(1.hour()) + .ok()? + } else if rule.freq == ICalendarFrequency::Secondly && !expander.satisfies_by_minute(next) { + next.with() + .second(0) + .build() + .ok()? + .checked_add(1.minute()) + .ok()? + } else { + return None; + }; + let step = SignedDuration::try_from(rule.interval).ok()?.as_secs(); + let elapsed = rule.civil_start.duration_until(boundary).as_secs(); + let skipped = attempt.max(elapsed.checked_add(step - 1)? / step); + let anchor = rule + .civil_start + .checked_add(rule.interval.checked_mul(skipped).ok()?) + .ok()?; + Some((skipped, anchor)) + } + + fn increment(&self) -> Option<(i64, DateTime)> { + let (mut attempt, orig) = self.cur?; + let interval = self.rule.inner.interval; + loop { + attempt = attempt.checked_add(1)?; + let interval = interval.checked_mul(attempt).ok()?; + let mut next = self.rule.inner.civil_start.checked_add(interval).ok()?; + if let Some((skipped, anchor)) = self.skip_ahead(attempt, next) { + attempt = skipped; + next = anchor; + } + // Subtle, but this is not technically required for correctness, + // since we filter based on `until` when we pop from the + // recurrence set. The reason we do this is for cases where the + // recurrence set is always empty (or nearly always). Namely, this + // lets us "bound" our work even if we aren't visibly producing + // anything. + // + // It is somewhat unfortunate that in order to do this, we need + // to convert our next datetime to a physical instant. Perhaps + // we can avoid doing this when the previous size of the recurrence + // set is non-empty. Why? Because if the recurrence set is getting + // datetimes in it, then filtering can be applied when we pop from + // it and this optimization ends up not being necessary. + // + // I think one tricky thing here is that just because a recurrence + // set is empty after expansion for a single interval, that doesn't + // mean iteration can stop. Because a subsequent interval might + // generate a non-empty recurrence set. + if let Some(ref until) = self.rule.inner.until + && self + .rule + .tz() + .from_local(self.period_start(next).unwrap_or(next)) + .is_none_or(|start| &start > until) + { + return None; + } + // In cases where we add N years or N months, the day number can + // change. For example, in Jiff, 2024-02-29 + 1 year = 2025-02-28. + // But RFC 5545 wants us to treat it like 2025-02-29, which + // is invalid and thus ignore such things. So we assume the + // "constraining" behavior of Jiff (as Temporal calls it) occurs + // when the result's day number is not equal to the one we started + // with. + // + // When constraining behavior happens, we just continue on and try + // the next point by adding our interval again. We do this *not* + // by just continually adding our interval to an ever increasing + // datetime, which is the natural thing to do. The problem is that + // constraining behavior is infectious. Once you get, for example, + // 2025-02-28, adding 3 years will give you 2028-02-28 and so on. + // You'll never get back to a leap day. So instead, we track our + // number of attempts and multiply our `Span` as we go. + // + // This doesn't account for the other way to get invalid datetimes, + // which is to end up at a time that is invalid for the given time + // zone. We handle that when we pop datetimes from the set. + // + // This is pretty tricky, but there is an exception to skipping + // invalid datetimes here. When expansion rules are set that choose + // the date (instead of using it from the start date), then we DO + // need to consider the interval instead of skipping over it. This + // works because if a date-setting expansion rule is present, then + // the constrained datetime should not remain as part of the set, + // unless the rule would have otherwise selected it. + let r = &self.rule.inner; + match r.freq { + ICalendarFrequency::Yearly if next.day() != orig.day() => { + if r.by_month.is_empty() + && r.by_week.is_empty() + && r.by_year_day.is_empty() + && r.by_month_day.is_empty() + && r.by_week_day.is_empty() + { + continue; + } + } + ICalendarFrequency::Monthly + if next.day() != orig.day() + && r.by_week_day.is_empty() + && r.by_month_day.is_empty() => + { + continue; + } + _ => {} + } + return Some((attempt, next)); + } + } +} + +impl<'r> Iterator for RecurrenceIter<'r> { + type Item = ZonedDateTime; + + fn next(&mut self) -> Option { + if self.remaining == Some(0) { + return None; + } + while !self.is_done() { + if let Some((zdt, in_gap)) = self.set.pop(self.rule) { + let instant = zdt.to_timestamp(); + if in_gap { + self.gap_instants.push(instant); + } else if self.gap_instants.binary_search(&instant).is_ok() { + continue; + } else if self.gap_instants.last().is_some_and(|last| *last < instant) { + self.gap_instants.clear(); + } + // Producing an instance means the rule is making progress, so + // the budget starts over. + self.budget = MAX_ITER_LOOP; + self.unproductive += std::mem::take(&mut self.period_work); + self.remaining = self.remaining.map(|count| count - 1); + return Some(zdt); + } + let discarded = self.set.take_spent(); + self.work = self.work.saturating_sub(discarded); + self.unproductive = self.unproductive.saturating_sub(discarded); + let Some(budget) = self.budget.checked_sub(1).filter(|_| self.unproductive > 0) else { + self.exhausted = true; + return None; + }; + self.budget = budget; + self.expand(); + let spent = self.set.take_spent() + 1; + self.work = self.work.saturating_sub(spent); + self.period_work = spent.min(self.unproductive); + self.unproductive -= self.period_work; + if self.set.overflowed || self.work == 0 { + self.set.clear(); + self.cur = None; + self.exhausted = true; + return None; + } + self.cur = self.increment(); + } + None + } +} + +impl<'r> std::iter::FusedIterator for RecurrenceIter<'r> {} + +/// A builder for constructing a valid recurrence rule. +#[derive(Clone, Debug)] +pub struct RecurrenceRuleBuilder { + freq: ICalendarFrequency, + start: ZonedDateTime, + until: Option, + count: Option, + interval: i32, + by_month: Vec, + by_week: Vec, + by_year_day: Vec, + by_month_day: Vec, + by_week_day: Vec, + by_hour: Vec, + by_minute: Vec, + by_second: Vec, + by_set_pos: Vec, + week_start: Weekday, +} + +impl RecurrenceRuleBuilder { + fn new(freq: ICalendarFrequency, start: ZonedDateTime) -> RecurrenceRuleBuilder { + RecurrenceRuleBuilder { + freq, + start, + until: None, + count: None, + interval: 1, + by_month: vec![], + by_week: vec![], + by_year_day: vec![], + by_month_day: vec![], + by_week_day: vec![], + by_hour: vec![], + by_minute: vec![], + by_second: vec![], + by_set_pos: vec![], + week_start: Weekday::Monday, + } + } + + pub fn build(&self) -> Result { + fn sort_and_dedup(slice: &[T]) -> Box<[T]> { + let mut vec = slice.to_vec(); + vec.sort(); + vec.dedup(); + vec.into_boxed_slice() + } + + fn range( + field: &str, + value: T, + start_idx: &str, + end_idx: &str, + ) -> ValidationError { + ValidationError::InvalidFieldValueRange { + field: field.to_string(), + value: value.to_string(), + start_idx: start_idx.to_string(), + end_idx: end_idx.to_string(), + } + } + + if self.interval < 1 { + return Err(ValidationError::InvalidFieldValue { + field: "INTERVAL".to_string(), + value: self.interval.to_string(), + }); + } + for &v in self.by_month.iter() { + if !(1..=12).contains(&v) { + return Err(range("BYMONTH", v, "1", "12")); + } + } + for &v in self.by_week.iter() { + if !((-53..=-1).contains(&v) || (1..=53).contains(&v)) { + return Err(range("BYWEEKNO", v, "-53", "53")); + } + } + for &v in self.by_year_day.iter() { + if !((-366..=-1).contains(&v) || (1..=366).contains(&v)) { + return Err(range("BYYEARDAY", v, "-366", "366")); + } + } + for &v in self.by_month_day.iter() { + if !((-31..=-1).contains(&v) || (1..=31).contains(&v)) { + return Err(range("BYMONTHDAY", v, "-31", "31")); + } + } + for &v in self.by_week_day.iter() { + let nth = match v { + ByWeekday::Any(_) => continue, + ByWeekday::Numbered { nth, .. } => nth, + }; + // Firstly, numbered weekdays are only allowed for YEARLY or + // MONTHLY frequencies, and at YEARLY only when BYWEEKNO is unset. + if !matches!( + self.freq, + ICalendarFrequency::Yearly | ICalendarFrequency::Monthly + ) || (matches!(self.freq, ICalendarFrequency::Yearly) && !self.by_week.is_empty()) + { + return Err(ValidationError::InvalidByRuleAndFrequency { + by_rule: "BYDAY".to_string(), + freq: self.freq, + }); + } + // Secondly, check the bounds on `nth`. It's yearly when the + // frequency is yearly and BYMONTH isn't set. Otherwise, it's + // monthly. + let (lo, hi) = + if matches!(self.freq, ICalendarFrequency::Yearly) && self.by_month.is_empty() { + (53, "53") + } else { + (5, "5") + }; + if !((-lo..=-1).contains(&nth) || (1..=lo).contains(&nth)) { + return Err(ValidationError::InvalidFieldValueRangeWithFreq { + field: "BYDAY".to_string(), + value: v.to_string(), + freq: self.freq, + start_idx: format!("-{hi}"), + end_idx: hi.to_string(), + }); + } + } + for &v in self.by_hour.iter() { + if !(0..=23).contains(&v) { + return Err(range("BYHOUR", v, "0", "23")); + } + } + for &v in self.by_minute.iter() { + if !(0..=59).contains(&v) { + return Err(range("BYMINUTE", v, "0", "59")); + } + } + for &v in self.by_second.iter() { + // RFC 5545 technically allows a value of `60` here, presumably + // for leap seconds. Jiff doesn't support leap seconds outside of + // parsing, in which case, Jiff just clamps the value. Clamping + // doesn't really make sense here, so just reject it. This is also + // what `python-dateutil` does. + if !(0..=59).contains(&v) { + return Err(range("BYSECOND", v, "0", "59")); + } + } + for &v in self.by_set_pos.iter() { + if !((-366..=-1).contains(&v) || (1..=366).contains(&v)) { + return Err(range("BYSETPOS", v, "-366", "366")); + } + } + + // Some additional frequency-specific errors. + let by_rule_freq = |by_rule: &str| ValidationError::InvalidByRuleAndFrequency { + by_rule: by_rule.to_string(), + freq: self.freq, + }; + if !self.by_week.is_empty() && !matches!(self.freq, ICalendarFrequency::Yearly) { + return Err(by_rule_freq("BYWEEKNO")); + } + if !self.by_year_day.is_empty() + && matches!( + self.freq, + ICalendarFrequency::Monthly + | ICalendarFrequency::Weekly + | ICalendarFrequency::Daily + ) + { + return Err(by_rule_freq("BYYEARDAY")); + } + if !self.by_month_day.is_empty() && matches!(self.freq, ICalendarFrequency::Weekly) { + return Err(by_rule_freq("BYMONTHDAY")); + } + + // A BYSETPOS specific error is that, if it's given, then there MUST + // be another BY* rule. + if !self.by_set_pos.is_empty() + && self.by_month.is_empty() + && self.by_week.is_empty() + && self.by_year_day.is_empty() + && self.by_month_day.is_empty() + && self.by_week_day.is_empty() + && self.by_hour.is_empty() + && self.by_minute.is_empty() + && self.by_second.is_empty() + { + return Err(ValidationError::BySetPosWithoutByRule); + } + + let interval = freq_to_span(self.freq, self.interval) + .ok_or(ValidationError::TooBigInterval(self.interval as u16))?; + let inner = Arc::new(RecurrenceRuleInner { + freq: self.freq, + start: self.start, + civil_start: self.start.naive_local(), + until: self.until, + count: self.count, + interval, + by_month: sort_and_dedup(&self.by_month), + by_week: sort_and_dedup(&self.by_week), + by_year_day: sort_and_dedup(&self.by_year_day), + by_month_day: sort_and_dedup(&self.by_month_day), + by_week_day: sort_and_dedup(&self.by_week_day), + by_hour: sort_and_dedup(&self.by_hour), + by_minute: sort_and_dedup(&self.by_minute), + by_second: sort_and_dedup(&self.by_second), + by_set_pos: sort_and_dedup(&self.by_set_pos), + week_start: self.week_start, + }); + Ok(RecurrenceRule { inner }) + } + + pub fn until(&mut self, until: ZonedDateTime) -> &mut RecurrenceRuleBuilder { + self.until = Some(until); + self + } + + /// Limits the rule to a number of instances. + /// + /// RFC 5545 counts instances generated by the rule itself, so dates added + /// by RDATE do not consume the count and dates removed by EXDATE do. + pub fn count(&mut self, count: u32) -> &mut RecurrenceRuleBuilder { + self.count = Some(count); + self + } + + pub fn interval(&mut self, increment: i32) -> &mut RecurrenceRuleBuilder { + self.interval = increment; + self + } + + pub fn by_month(&mut self, months: I) -> &mut RecurrenceRuleBuilder { + self.by_month.extend(months.into_i8_iter()); + self + } + + pub fn by_week(&mut self, weeks: I) -> &mut RecurrenceRuleBuilder { + self.by_week.extend(weeks.into_i8_iter()); + self + } + + pub fn by_year_day(&mut self, days: I) -> &mut RecurrenceRuleBuilder { + self.by_year_day.extend(days.into_i16_iter()); + self + } + + pub fn by_month_day(&mut self, days: I) -> &mut RecurrenceRuleBuilder { + self.by_month_day.extend(days.into_i8_iter()); + self + } + + pub fn by_week_day( + &mut self, + week_days: I, + ) -> &mut RecurrenceRuleBuilder { + self.by_week_day.extend(week_days.into_by_weekday_iter()); + self + } + + pub fn by_hour(&mut self, hours: I) -> &mut RecurrenceRuleBuilder { + self.by_hour.extend(hours.into_i8_iter()); + self + } + + pub fn by_minute(&mut self, minutes: I) -> &mut RecurrenceRuleBuilder { + self.by_minute.extend(minutes.into_i8_iter()); + self + } + + pub fn by_second(&mut self, seconds: I) -> &mut RecurrenceRuleBuilder { + self.by_second.extend(seconds.into_i8_iter()); + self + } + + pub fn by_set_position(&mut self, positions: I) -> &mut RecurrenceRuleBuilder { + self.by_set_pos.extend(positions.into_i32_iter()); + self + } + + pub fn week_start(&mut self, weekday: Weekday) -> &mut RecurrenceRuleBuilder { + self.week_start = weekday; + self + } +} + +#[derive(Clone, Debug)] +struct RecurrenceSet { + civil: Vec, + /// A set of *zoned* datetimes that we use exactly like `set`, but only + /// when BYSETPOS has a non-zero number of values. + /// + /// Basically, when BYSETPOS is used, `set` is drained into `zoned_set`, + /// `zoned_set` is then filtered according to BYSETPOS and the iterator + /// pops from `zoned_set` instead of `set`. + /// + /// Otherwise, this is totally ignored and no allocation is done. + /// + /// Generally speaking, this isn't really managed by the methods on + /// `RecurrenceSet`. The abstraction somewhat sucks. If this needed to be + /// a public API, we'd probably use more type machinery. But in practice, + /// this is mostly treated as scratch space by `Expander`. + zoned: Vec<(ZonedDateTime, bool)>, + hours: Axis, + minutes: Axis, + seconds: Axis, + cursor: usize, + len: usize, + spent: usize, + overflowed: bool, +} + +#[derive(Clone, Copy, Debug)] +enum Axis { + Fixed(i8), + Rule, +} + +impl Axis { + fn values<'a>(&'a self, rule: &'a [i8]) -> &'a [i8] { + match self { + Axis::Fixed(value) => std::slice::from_ref(value), + Axis::Rule => rule, + } + } +} + +impl RecurrenceSet { + fn new() -> RecurrenceSet { + RecurrenceSet { + civil: vec![], + zoned: vec![], + hours: Axis::Fixed(0), + minutes: Axis::Fixed(0), + seconds: Axis::Fixed(0), + cursor: 0, + len: 0, + spent: 0, + overflowed: false, + } + } + + fn take_spent(&mut self) -> usize { + std::mem::take(&mut self.spent) + } + + fn is_empty(&self) -> bool { + self.cursor >= self.len && self.zoned.is_empty() + } + + fn begin_period(&mut self, cur: DateTime) { + self.civil.clear(); + self.zoned.clear(); + self.hours = Axis::Fixed(cur.hour()); + self.minutes = Axis::Fixed(cur.minute()); + self.seconds = Axis::Fixed(cur.second()); + self.cursor = 0; + self.len = 0; + } + + fn candidate(&self, rule: &RecurrenceRuleInner, index: usize) -> Option { + let hours = self.hours.values(&rule.by_hour); + let minutes = self.minutes.values(&rule.by_minute); + let seconds = self.seconds.values(&rule.by_second); + let per_hour = minutes.len() * seconds.len(); + let per_date = hours.len() * per_hour; + let date = self.civil.get(index.checked_div(per_date)?)?; + let time = index % per_date; + date.with() + .hour(*hours.get(time / per_hour)?) + .minute(*minutes.get(time / seconds.len() % minutes.len())?) + .second(*seconds.get(time % seconds.len())?) + .build() + .ok() + } + + fn seek(&mut self, rule: &RecurrenceRuleInner, start: DateTime) { + let (mut low, mut high) = (0, self.len); + while low < high { + let middle = low + (high - low) / 2; + if self + .candidate(rule, middle) + .is_some_and(|candidate| candidate < start) + { + low = middle + 1; + } else { + high = middle; + } + } + self.cursor = low; + } + + fn select_positions(&mut self, rule: &RecurrenceRule) { + self.spent += rule.inner.by_set_pos.len(); + let len = self.len; + let mut indices = rule + .inner + .by_set_pos + .iter() + .filter_map(|position| { + let offset = usize::try_from(position.unsigned_abs()).ok()?; + if *position > 0 { + offset.checked_sub(1).filter(|index| *index < len) + } else { + len.checked_sub(offset) + } + }) + .collect::>(); + indices.sort_unstable(); + indices.dedup(); + let tz = rule.tz(); + let selected = indices + .into_iter() + .filter_map(|index| tz.interpret_local(self.candidate(&rule.inner, index)?)) + .collect::>(); + self.zoned = selected; + self.cursor = self.len; + self.zoned.sort_by_key(|(zdt, _)| *zdt); + self.zoned.dedup_by_key(|(zdt, _)| *zdt); + self.zoned.reverse(); + } + + fn insert(&mut self, dt: DateTime) { + self.spent += 1; + self.civil.push(dt); + } + + fn clear(&mut self) { + self.civil.clear(); + self.zoned.clear(); + self.cursor = 0; + self.len = 0; + } + + fn retain(&mut self, predicate: impl FnMut(&mut DateTime) -> bool) { + self.civil.retain_mut(predicate); + } + + fn expand(&mut self, expand: E) + where + E: Fn(DateTime) -> I, + I: Iterator, + { + // We're going to replace every datetime in the set at this point + // with its expansion provided by the closure. So record how many + // datetimes we have now. At the end, we'll drain them in one swoop. + let len = self.civil.len(); + for i in 0..len { + self.civil.extend(expand(self.civil[i])); + if self.civil.len() > MAX_PERIOD_CANDIDATES { + self.overflowed = true; + self.civil.clear(); + return; + } + } + self.spent += self.civil.len() - len; + self.civil.drain(..len); + } + + fn canonicalize(&mut self, rule: &RecurrenceRuleInner) { + self.civil.sort(); + self.civil.dedup(); + let per_date = self.hours.values(&rule.by_hour).len() + * self.minutes.values(&rule.by_minute).len() + * self.seconds.values(&rule.by_second).len(); + match self.civil.len().checked_mul(per_date) { + Some(len) => self.len = len, + None => { + self.overflowed = true; + self.clear(); + } + } + } + + fn pop(&mut self, rule: &RecurrenceRule) -> Option<(ZonedDateTime, bool)> { + loop { + let (next, in_gap) = self.zoned.pop().or_else(|| self.pop_civil(rule))?; + // For simplicity of implementation, the recurrence rule + // generator may create datetimes before our starting point. Since + // this can generally only happen on the first "iteration," we + // don't worry about it too much and just filter them here. + // + // Ideally we would do this even earlier, and that might make the + // BYSETPOS implementation simpler. But the starting point is a + // zoned datetime and this is the earliest that we convert a + // datetime from our buffered set to a zoned datetime. + if next < rule.inner.start { + self.spent += 1; + continue; + } + // For similar reasons, we handle our "until" constraint here too. + if let Some(ref until) = rule.inner.until + && &next > until + { + // Since we always move forward in time, if we reach + // this point, we'll never be able to return any other + // datetime. So we can drain the rest of the set and + // bail. + self.clear(); + return None; + } + return Some((next, in_gap)); + } + } + + fn pop_civil(&mut self, rule: &RecurrenceRule) -> Option<(ZonedDateTime, bool)> { + let tz = rule.tz(); + while self.cursor < self.len { + let index = self.cursor; + self.cursor += 1; + let Some(dt) = self.candidate(&rule.inner, index) else { + continue; + }; + // RFC 5545 section 3.3.5 settles what an ambiguous wall clock + // reading means: a reading that occurs twice means its first + // occurrence, and a reading that does not occur is interpreted + // with the offset in effect before the gap, which moves it + // forward by the length of the gap. `Tz::from_local` applies + // exactly that rule, so an instance around a daylight saving + // transition is neither dropped nor duplicated. + let Some(next) = tz.interpret_local(dt) else { + continue; + }; + // N.B. We don't check if the civil datetime is before our starting + // point or after our `until` here because BYSETPOS wants those + // included in the recurrence set when resolving indices. They + // will get filtered out later in `pop()`. + return Some(next); + } + None + } +} + +/// A trait that permits flexibly specifying a sequence of `i8` integers. +/// +/// This trait is used for builder methods on `RecurrenceRuleBuilder`. It +/// permits callers to provide integers in a number of flexible ways: +/// +/// * A single integer: `5` +/// * An array of integers: `[1, 3, 5]`. +/// * A single range of integers: `5..8` or `5..=8`. +/// * An array of ranges of integers: `[5..=10, 15..=20]`. +/// +/// # Design +/// +/// The reason this trait _and_ `IntoI16Iter` exists is to make specifying a +/// sequence more ergonomic. In particular, an alternative design is: +/// +/// ```ignore +/// pub trait IntoIntegerIter { +/// type Integer; +/// fn into_integer_iter(self) -> impl Iterator; +/// } +/// ``` +/// +/// But since this would be implemented for both `i8` and `i16`, this +/// means that `builder.by_month(5)` cannot have the type of `5` inferred +/// unambiguously. +// N.B. If we end up copying this design into Jiff, we should make this +// trait sealed so that people downstream can't add trait impls and fuck up +// inference. +pub trait IntoI8Iter { + /// Creates an iterator over all integers in this sequence. + fn into_i8_iter(self) -> impl Iterator; +} + +/// A trait that permits flexibly specifying a sequence of `i16` integers. +/// +/// This trait is used for builder methods on `RecurrenceRuleBuilder`. It +/// permits callers to provide integers in a number of flexible ways: +/// +/// * A single integer: `5` +/// * An array of integers: `[1, 3, 5]`. +/// * A single range of integers: `5..8` or `5..=8`. +/// * An array of ranges of integers: `[5..=10, 15..=20]`. +/// +/// # Design +/// +/// The reason this trait _and_ `IntoI8Iter` exists is to make specifying a +/// sequence more ergonomic. In particular, an alternative design is: +/// +/// ```ignore +/// pub trait IntoIntegerIter { +/// type Integer; +/// fn into_integer_iter(self) -> impl Iterator; +/// } +/// ``` +/// +/// But since this would be implemented for both `i8` and `i16`, this +/// means that `builder.by_month(5)` cannot have the type of `5` inferred +/// unambiguously. +pub trait IntoI16Iter { + /// Creates an iterator over all integers in this sequence. + fn into_i16_iter(self) -> impl Iterator; +} + +/// A trait that permits flexibly specifying a sequence of `i32` integers. +/// +/// This trait is used for builder methods on `RecurrenceRuleBuilder`. It +/// permits callers to provide integers in a number of flexible ways: +/// +/// * A single integer: `5` +/// * An array of integers: `[1, 3, 5]`. +/// * A single range of integers: `5..8` or `5..=8`. +/// * An array of ranges of integers: `[5..=10, 15..=20]`. +/// +/// # Design +/// +/// The reason this trait, `IntoI8Iter` _and_ `IntoI16Iter` exists is to make +/// specifying a sequence more ergonomic. In particular, an alternative design +/// is: +/// +/// ```ignore +/// pub trait IntoIntegerIter { +/// type Integer; +/// fn into_integer_iter(self) -> impl Iterator; +/// } +/// ``` +/// +/// But since this would be implemented for `i8`, `i16` and `i32`, this +/// means that `builder.by_month(5)` cannot have the type of `5` inferred +/// unambiguously. +pub trait IntoI32Iter { + /// Creates an iterator over all integers in this sequence. + fn into_i32_iter(self) -> impl Iterator; +} + +impl IntoI8Iter for Vec { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI8Iter for &[i8] { + fn into_i8_iter(self) -> impl Iterator { + self.iter().copied() + } +} + +impl IntoI16Iter for &[i16] { + fn into_i16_iter(self) -> impl Iterator { + self.iter().copied() + } +} + +impl IntoI32Iter for &[i32] { + fn into_i32_iter(self) -> impl Iterator { + self.iter().copied() + } +} + +impl IntoByWeekdayIter for Vec { + fn into_by_weekday_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI8Iter for i8 { + fn into_i8_iter(self) -> impl Iterator { + std::iter::once(self) + } +} + +impl IntoI16Iter for i16 { + fn into_i16_iter(self) -> impl Iterator { + std::iter::once(self) + } +} + +impl IntoI32Iter for i32 { + fn into_i32_iter(self) -> impl Iterator { + std::iter::once(self) + } +} + +impl IntoI8Iter for Range { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI16Iter for Range { + fn into_i16_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI32Iter for Range { + fn into_i32_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI8Iter for RangeInclusive { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI16Iter for RangeInclusive { + fn into_i16_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI32Iter for RangeInclusive { + fn into_i32_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI8Iter for [i8; N] { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI16Iter for [i16; N] { + fn into_i16_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI32Iter for [i32; N] { + fn into_i32_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoI8Iter for [Range; N] { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +impl IntoI16Iter for [Range; N] { + fn into_i16_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +impl IntoI32Iter for [Range; N] { + fn into_i32_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +impl IntoI8Iter for [RangeInclusive; N] { + fn into_i8_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +impl IntoI16Iter for [RangeInclusive; N] { + fn into_i16_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +impl IntoI32Iter for [RangeInclusive; N] { + fn into_i32_iter(self) -> impl Iterator { + self.into_iter().flatten() + } +} + +/// A trait that permits flexibly specifying a sequence of weekdays. +/// +/// Each weekday can just mean "any" weekday (e.g., `Weekday::Saturday`), or +/// it can mean a numbered weekday. For example, when the frequency for a +/// recurrence rule is yearly, then `(3, Weekday::Saturday)` corresponds to the +/// third Saturday of the year. +/// +/// This trait is primarily used for the `RecurrenceRuleBuilder::by_week_day` +/// builder methods. It permits callers to provide weekdays in a number of +/// flexible ways: +/// +/// * Directly via `ByWeekday::Numbered { nth: 3, weekday: Weekday::Monday }`. +/// * As just any weekday via `Weekday::Monday`. +/// * As a range of weekdays via `Weekday::Monday..=Weekday::Wednesday`. +/// * As an array of weekdays via `[Weekday::Monday, Weekday::Friday]`. +/// * As an array of numbered weekdays via +/// `[(2, Weekday::Monday), (1, Weekday::Friday)]`. +pub trait IntoByWeekdayIter { + fn into_by_weekday_iter(self) -> impl Iterator; +} + +impl IntoByWeekdayIter for ByWeekday { + fn into_by_weekday_iter(self) -> impl Iterator { + std::iter::once(self) + } +} + +impl IntoByWeekdayIter for Weekday { + fn into_by_weekday_iter(self) -> impl Iterator { + std::iter::once(ByWeekday::Any(self)) + } +} + +impl IntoByWeekdayIter for (i8, Weekday) { + fn into_by_weekday_iter(self) -> impl Iterator { + let (nth, weekday) = self; + std::iter::once(ByWeekday::Numbered { nth, weekday }) + } +} + +impl IntoByWeekdayIter for RangeInclusive { + fn into_by_weekday_iter(self) -> impl Iterator { + let (start, end) = (*self.start(), *self.end()); + // OK because `Weekday::until` guarantees `0..=6`. + // And add `1` because this is an inclusive range. + let count = + 1 + usize::try_from(start.until(end)).expect("Weekday::until is always within 0..=6"); + start.cycle_forward().take(count).map(ByWeekday::Any) + } +} + +impl IntoByWeekdayIter for [ByWeekday; N] { + fn into_by_weekday_iter(self) -> impl Iterator { + self.into_iter() + } +} + +impl IntoByWeekdayIter for [Weekday; N] { + fn into_by_weekday_iter(self) -> impl Iterator { + self.into_iter().flat_map(|any| any.into_by_weekday_iter()) + } +} + +impl IntoByWeekdayIter for [(i8, Weekday); N] { + fn into_by_weekday_iter(self) -> impl Iterator { + self.into_iter() + .flat_map(|numbered| numbered.into_by_weekday_iter()) + } +} + +impl IntoByWeekdayIter for [RangeInclusive; N] { + fn into_by_weekday_iter(self) -> impl Iterator { + self.into_iter().flat_map(|any| any.into_by_weekday_iter()) + } +} + +/// Converts a frequency and interval into the span one step advances by. +/// +/// Returns `None` when the interval is too large for a span to hold. +fn freq_to_span(freq: ICalendarFrequency, interval: i32) -> Option { + let base = match freq { + ICalendarFrequency::Yearly => 1.year(), + ICalendarFrequency::Monthly => 1.month(), + ICalendarFrequency::Weekly => 1.week(), + ICalendarFrequency::Daily => 1.day(), + ICalendarFrequency::Hourly => 1.hour(), + ICalendarFrequency::Minutely => 1.minute(), + ICalendarFrequency::Secondly => 1.second(), + }; + base.checked_mul(i64::from(interval)).ok() +} + +/// A type describing "day of week" inputs. +/// +/// This implements `Ord` even though the actual order of weekdays cannot be +/// determined unless the _start_ of the week is known (which is commonly +/// either or Sunday or Monday, but RFC 5545 lets any day be the start). +/// However, we implement `Ord` to make it easy to sort and de-duplicate +/// collections containing a `ByWeekday`. We never actually rely on its +/// ordering for generating datetimes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ByWeekday { + Any(Weekday), + Numbered { nth: i8, weekday: Weekday }, +} + +impl ByWeekday { + fn weekday(&self) -> Weekday { + match *self { + ByWeekday::Any(weekday) | ByWeekday::Numbered { weekday, .. } => weekday, + } + } + + /// Returns true if and only if the given weekday matches this one. + /// + /// # Panics + /// + /// This panics when this is a numbered weekday. In effect, this shouldn't + /// be used in contexts where BYDAY can contain numbered weekdays. That + /// limits this to YEARLY and MONTHLY frequencies (and in the YEARLY case, + /// only when BYWEEKNO is not used). + fn is_match(&self, wd: Weekday) -> bool { + match *self { + ByWeekday::Any(weekday) => weekday == wd, + _ => unreachable!(), + } + } + + /// Return an iterator of weekdays, at yearly frequency, within the given + /// range of datetimes. + /// + /// Generally speaking, `start` should be the first day of a year and `end` + /// should be the last day of that same year. + /// + /// When this is a numbered weekday, then a positive number is interpreted + /// relative to the start and a negative number is interpreted relative to + /// the end. Either way, the iterator returned yields at most one element + /// (but may yield zero). + /// + /// When this is "any" weekday, then every date with that weekday between + /// `start` and `end` (inclusive) is returned. + fn iter_yearly( + &self, + start: DateTime, + end: DateTime, + ) -> impl Iterator + use<> { + match *self { + ByWeekday::Any(weekday) => Either::Left(iter_weekdays_between(weekday, start, end)), + ByWeekday::Numbered { nth, weekday } => { + let from = if nth < 0 { end } else { start }; + let nth = if from.weekday() != weekday { + nth + } else if nth.abs() == 1 { + return Either::Right(Some(from).into_iter()); + } else { + nth - nth.signum() + }; + Either::Right( + from.nth_weekday(i32::from(nth), weekday) + .ok() + .filter(|dt| (start..=end).contains(dt)) + .into_iter(), + ) + } + } + } + + /// Return an iterator of weekdays, at monthly frequency, within the given + /// range of datetimes. + /// + /// Generally speaking, `start` should be the first day of a month and + /// `end` should be the last day of that same month. + /// + /// When this is a numbered weekday, then a positive number is interpreted + /// relative to the start and a negative number is interpreted relative to + /// the end. Either way, the iterator returned yields at most one element + /// (but may yield zero). + /// + /// When this is "any" weekday, then every date with that weekday between + /// `start` and `end` (inclusive) is returned. + fn iter_monthly( + &self, + start: DateTime, + end: DateTime, + ) -> impl Iterator + use<> { + match *self { + ByWeekday::Any(weekday) => Either::Left(iter_weekdays_between(weekday, start, end)), + ByWeekday::Numbered { nth, weekday } => { + Either::Right(start.nth_weekday_of_month(nth, weekday).ok().into_iter()) + } + } + } + + /// Return an iterator of weekdays, at weekly frequency, within the given + /// range of datetimes. + /// + /// Generally speaking, `start` should be the first day of a week and + /// `end` should be the last day of that same week. + /// + /// When this is "any" weekday, then every date with that weekday between + /// `start` and `end` (inclusive) is returned. + /// + /// # Panics + /// + /// When this `ByWeekday` is `Numbered`. RFC 5545 doesn't permit that + /// construction at anything other than YEARLY and MONTHLY frequency. + fn iter_weekly( + &self, + start: DateTime, + end: DateTime, + ) -> impl Iterator + use<> { + match *self { + ByWeekday::Any(weekday) => iter_weekdays_between(weekday, start, end), + // Not allowed according to RFC 5545. This case should be prevented + // by recurrence rule construction. + ByWeekday::Numbered { .. } => unreachable!(), + } + } +} + +impl Ord for ByWeekday { + fn cmp(&self, rhs: &ByWeekday) -> Ordering { + match (*self, *rhs) { + (ByWeekday::Any(lhs), ByWeekday::Any(rhs)) => { + lhs.to_monday_one_offset().cmp(&rhs.to_monday_one_offset()) + } + ( + ByWeekday::Numbered { + nth: lhs_nth, + weekday: lhs_weekday, + }, + ByWeekday::Numbered { + nth: rhs_nth, + weekday: rhs_weekday, + }, + ) => { + let lhs = (lhs_nth, lhs_weekday.to_monday_one_offset()); + let rhs = (rhs_nth, rhs_weekday.to_monday_one_offset()); + lhs.cmp(&rhs) + } + (ByWeekday::Any(_), ByWeekday::Numbered { .. }) => Ordering::Less, + (ByWeekday::Numbered { .. }, ByWeekday::Any(_)) => Ordering::Greater, + } + } +} + +impl PartialOrd for ByWeekday { + fn partial_cmp(&self, rhs: &ByWeekday) -> Option { + Some(self.cmp(rhs)) + } +} + +impl std::fmt::Display for ByWeekday { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt_weekday(wd: Weekday, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match wd { + Weekday::Sunday => write!(f, "Sun"), + Weekday::Monday => write!(f, "Mon"), + Weekday::Tuesday => write!(f, "Tue"), + Weekday::Wednesday => write!(f, "Wed"), + Weekday::Thursday => write!(f, "Thu"), + Weekday::Friday => write!(f, "Fri"), + Weekday::Saturday => write!(f, "Sat"), + } + } + + match *self { + ByWeekday::Any(weekday) => fmt_weekday(weekday, f), + ByWeekday::Numbered { nth, weekday } => { + write!(f, "{nth}-")?; + fmt_weekday(weekday, f) + } + } + } +} + +/// A simple `Either` type for easy construction of `impl Iterator`. +/// +/// Specifically, this is useful when it's supremely annoying to write a +/// single iterator chain when it would be more naturally written using case +/// analysis. +enum Either { + Left(L), + Right(R), +} + +impl Iterator for Either +where + L: Iterator, + R: Iterator, +{ + type Item = I; + + fn next(&mut self) -> Option { + match *self { + Either::Left(ref mut it) => it.next(), + Either::Right(ref mut it) => it.next(), + } + } +} + +/// Returns an iterator for every weekday between `start` and `end` (inclusive). +fn iter_weekdays_between( + weekday: Weekday, + start: DateTime, + end: DateTime, +) -> impl Iterator { + (start.weekday() == weekday) + .then_some(start) + .into_iter() + .chain({ + let mut cur = start.nth_weekday(1, weekday).ok(); + std::iter::from_fn(move || { + let next = cur.take()?; + if next > end { + return None; + } + cur = next.nth_weekday(1, weekday).ok(); + Some(next) + }) + }) +} + diff --git a/crates/calcard/src/datecalc/rrule.rs b/crates/calcard/src/datecalc/rrule.rs new file mode 100644 index 00000000..b88bd994 --- /dev/null +++ b/crates/calcard/src/datecalc/rrule.rs @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! Turns a parsed `RRULE` property into an expandable recurrence rule. + +use super::{ + error::{RRuleError, ValidationError}, + recurrence::{ByWeekday, RecurrenceIter, RecurrenceRule}, +}; +use crate::{ + common::{ + CalendarScale, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::{ICalendarFrequency, ICalendarRecurrenceRule}, +}; +use jiff::{Unit, civil::Time}; + +const END_OF_DAY: Time = Time::constant(23, 59, 0, 0); + +/// A recurrence rule bound to the start of its series. +#[derive(Clone, Debug)] +pub struct RRule { + rule: RecurrenceRule, +} + +impl RRule { + /// Builds a rule from a parsed `RRULE` and the start of its series. + pub fn from_ical( + rule: &ICalendarRecurrenceRule, + dt_start: ZonedDateTime, + is_date: bool, + ) -> Result { + // A non-Gregorian calendar scale puts instances on entirely different + // dates, so expanding the rule as though the scale were absent would + // silently produce the wrong ones. + if let Some(rscale) = rule + .rscale + .as_ref() + .filter(|rscale| **rscale != CalendarScale::Gregorian) + { + return Err(ValidationError::UnsupportedCalendarScale(rscale.clone()).into()); + } + + let mut builder = RecurrenceRule::builder(rule.freq, dt_start); + + if let Some(interval) = rule.interval.filter(|interval| *interval > 0) { + builder.interval(i32::from(interval)); + } + if let Some(count) = rule.count { + builder.count(count); + } + if let Some(until) = rule.until.as_ref().and_then(|until| until.to_date_time()) { + let tz = dt_start.timezone(); + let until = if is_date && until.offset.is_some() { + let written = until.date_time; + let last_day = if written.time() >= END_OF_DAY { + Some(written.date()) + } else { + written.round(Unit::Day).ok().map(|day| day.date()) + }; + last_day.and_then(|day| tz.from_local(day.to_datetime(Time::MAX))) + } else { + until.to_date_time_with_tz(tz) + }; + if let Some(until) = until { + builder.until(until); + } + } + if let Some(wkst) = rule.wkst { + builder.week_start(wkst.into()); + } + + builder.by_month( + rule.bymonth + .iter() + .map(|month| month.month() as i8) + .collect::>(), + ); + let by_week: &[i8] = if rule.freq == ICalendarFrequency::Yearly { + &rule.byweekno + } else { + &[] + }; + let by_year_day: &[i16] = if matches!( + rule.freq, + ICalendarFrequency::Monthly | ICalendarFrequency::Weekly | ICalendarFrequency::Daily + ) { + &[] + } else { + &rule.byyearday + }; + let by_month_day: &[i8] = if rule.freq == ICalendarFrequency::Weekly { + &[] + } else { + &rule.bymonthday + }; + let numbered_weekdays = rule.freq == ICalendarFrequency::Monthly + || (rule.freq == ICalendarFrequency::Yearly && by_week.is_empty()); + + builder.by_week(by_week); + builder.by_year_day(by_year_day); + builder.by_month_day(by_month_day); + builder.by_week_day( + rule.byday + .iter() + .map(|day| match day.ordwk.filter(|_| numbered_weekdays) { + Some(nth) => ByWeekday::Numbered { + nth: i8::try_from(nth).unwrap_or(i8::MAX), + weekday: day.weekday.into(), + }, + None => ByWeekday::Any(day.weekday.into()), + }) + .collect::>(), + ); + builder.by_hour(rule.byhour.iter().map(|v| *v as i8).collect::>()); + builder.by_minute(rule.byminute.iter().map(|v| *v as i8).collect::>()); + builder.by_second(rule.bysecond.iter().map(|v| *v as i8).collect::>()); + let set_position_alone = rule.bymonth.is_empty() + && by_week.is_empty() + && by_year_day.is_empty() + && by_month_day.is_empty() + && rule.byday.is_empty() + && rule.byhour.is_empty() + && rule.byminute.is_empty() + && rule.bysecond.is_empty(); + if !set_position_alone { + builder.by_set_position(rule.bysetpos.as_slice()); + } else if !rule.bysetpos.is_empty() + && !rule.bysetpos.iter().any(|position| position.abs() == 1) + { + builder.count(0); + } + + Ok(RRule { + rule: builder.build()?, + }) + } + + /// Returns an iterator over the instances this rule generates. + pub fn iter(&self) -> RecurrenceIter<'_> { + self.rule.iter() + } + + /// Returns the time zone instances are generated in. + pub fn tz(&self) -> Tz { + self.rule.tz() + } +} diff --git a/crates/calcard/src/datecalc/weekdate.rs b/crates/calcard/src/datecalc/weekdate.rs new file mode 100644 index 00000000..2fb9c802 --- /dev/null +++ b/crates/calcard/src/datecalc/weekdate.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2025 Andrew Gallant (https://github.com/BurntSushi/bttf) + * + * SPDX-License-Identifier: Unlicense OR MIT + */ + +//! Week dates under an arbitrary week start. +//! +//! RFC 5545 lets a rule choose the day a week starts on (WKST), so BYWEEKNO +//! cannot use [`jiff::civil::ISOWeekDate`], which always starts on Monday. +//! +//! Ported from `bttf`, with the `anyhow` errors replaced by `Option`: every +//! failure here means a date outside the supported range, which a recurrence +//! rule skips rather than reports. + +use jiff::{ + ToSpan, + civil::{Date, Weekday}, +}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct WeekDate { + // The weekday on which this week date calendar starts weeks. This is the + // key difference from `jiff::civil::ISOWeekDate`, which always starts on + // Monday. + start: Weekday, + year: i16, + week: i8, + weekday: Weekday, +} + +impl WeekDate { + /// Create a new week date. + /// + /// `year` must be in the range `-9999..=9999`. `week` must be in the range + /// `1..=53`, although `53` is only valid for "long" years. + /// + /// `start` corresponds to how the week numbering scheme determines the + /// start of a week. + pub(crate) fn new(start: Weekday, year: i16, week: i8, weekday: Weekday) -> Option { + if !(1..=53).contains(&week) || (week == 53 && !is_long_year(start, year)) { + return None; + } + if year == -9999 || year == 9999 { + let start_of_year = week_start_of_year(start, year)?; + let mut days = i32::from(week - 1) * 7; + days += i32::from(weekday.since(start)); + if start_of_year.checked_add(days.days()).is_err() { + return None; + } + } + Some(WeekDate { + start, + year, + week, + weekday, + }) + } + + /// Returns a new week date for the given Gregorian date. + /// + /// The week date uses a week numbering scheme where the given weekday + /// is the first day in the week. That is, the first week date of a year + /// starts on the given weekday and is the first week whose majority of + /// days (>= 4) falls in the same Gregorian year. + #[cfg(all(test, stalwart_tests))] + pub(crate) fn from_date(start: Weekday, date: Date) -> Option { + let mut start_of_year = week_start_of_year(start, date.year())?; + if date < start_of_year { + start_of_year = week_start_of_year(start, date.year() - 1)?; + } else { + let next = date.year() + 1; + // This fails when `date` is in year 9999, which is Jiff's maximum. + // But by the same token, in that case, `date` will never be + // greater than or equal to the start of that year. So we can just + // ignore it. + if let Some(next_start_of_year) = week_start_of_year(start, next) + && date >= next_start_of_year + { + start_of_year = next_start_of_year; + } + } + + debug_assert!(date >= start_of_year); + let diff_days = start_of_year.until(date).ok()?.get_days(); + // +1 because weeks are one-indexed. `date` cannot be more than 53 + // weeks after `start_of_year`, so the conversion always fits. + let week = i8::try_from(diff_days / 7).ok()? + 1; + // The week date year is guaranteed to match the Gregorian year 4 + // days after the start of the first day of the week date year. + let year = start_of_year.checked_add(4.days()).ok()?.year(); + let weekday = date.weekday(); + Some(WeekDate { + start, + year, + week, + weekday, + }) + } + + /// Converts this week date to its corresponding Gregorian date. + pub(crate) fn date(self) -> Option { + let start_of_year = week_start_of_year(self.start, self.year)?; + let mut days = i32::from(self.week - 1) * 7; + days += i32::from(self.weekday.since(self.start)); + start_of_year.checked_add(days.days()).ok() + } +} + +pub(crate) fn weeks_in_year(start: Weekday, year: i16) -> i8 { + if is_long_year(start, year) { 53 } else { 52 } +} + +/// Returns the start of the week that the given date resides in. +/// +/// The starting point of the week is determined by `start`. +pub(crate) fn first_of_week(start: Weekday, date: Date) -> Option { + let wd = date.weekday(); + if start == wd { + Some(date) + } else { + date.nth_weekday(-1, start).ok() + } +} + +/// Returns the end of the week that the given date resides in. +/// +/// The starting point of the week is determined by `start`. +pub(crate) fn last_of_week(start: Weekday, date: Date) -> Option { + let last = start.wrapping_sub(1); + let wd = date.weekday(); + if last == wd { + Some(date) + } else { + date.nth_weekday(1, last).ok() + } +} + +/// Returns true if the given week year (with weeks starting on `start`) is a +/// "long" year or not. +/// +/// A "long" year is a year with 53 weeks. Otherwise, it's a "short" year with +/// 52 weeks. +fn is_long_year(start: Weekday, year: i16) -> bool { + // Inspired by: https://en.wikipedia.org/wiki/ISO_week_date#Weeks_per_year + let Ok(last) = Date::new(year, 12, 31) else { + return false; + }; + let weekday = last.weekday(); + weekday == start.wrapping_add(3) || (last.in_leap_year() && weekday == start.wrapping_add(4)) +} + +/// Returns the first date in the first week of the given year. +/// +/// The date returned is guaranteed to have a weekday equivalent to `start`. +fn week_start_of_year(start: Weekday, year: i16) -> Option { + // RFC 5545 says: + // + // > A week is defined as a seven day period, starting on the day of the + // > week defined to be the week start (see WKST). Week number one of the + // > calendar year is the first week that contains at least four (4) days + // > in that calendar year. + // + // Which means that Jan 4 *must* be in the first week of the year. + let date_in_first_week = Date::new(year, 1, 4).ok()?; + // Now find the number of days since the start of the week from a date that + // we know is in the first week of `year`. + let diff_from_start = date_in_first_week.weekday().since(start); + date_in_first_week.checked_sub(diff_from_start.days()).ok() +} + diff --git a/crates/calcard/src/icalendar/builder.rs b/crates/calcard/src/icalendar/builder.rs new file mode 100644 index 00000000..9dbc05a4 --- /dev/null +++ b/crates/calcard/src/icalendar/builder.rs @@ -0,0 +1,760 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + Data, IanaType, PartialDateTime, + parser::{Boolean, Integer}, + }, + icalendar::*, +}; +use ahash::{AHashMap, AHashSet}; +use smallvec::{SmallVec, smallvec}; + +impl ICalendar { + #[must_use] + pub fn remove_component_ids(&mut self, component_ids: &[u32]) -> bool { + let max_component_id = u32::try_from(self.components.len()).unwrap_or(u32::MAX); + let mut remove_component_ids: AHashSet = component_ids + .iter() + .copied() + .filter(|id| *id < max_component_id) + .collect(); + + let mut pending = remove_component_ids.iter().copied().collect::>(); + while let Some(component_id) = pending.pop() { + let Some(component) = self.components.get(component_id as usize) else { + continue; + }; + for child_id in &component.component_ids { + if *child_id < max_component_id && remove_component_ids.insert(*child_id) { + pending.push(*child_id); + } + } + } + + if remove_component_ids.is_empty() { + return true; + } + + let removes_all_calendar_components = self.has_calendar_components() + && self.calendar_root().is_some_and(|root| { + !root.component_ids.iter().any(|id| { + *id != 0 && *id < max_component_id && !remove_component_ids.contains(id) + }) + }); + if remove_component_ids.contains(&0) || removes_all_calendar_components { + return false; + } + + let id_mappings = (0..max_component_id) + .filter(|i| !remove_component_ids.contains(i)) + .zip(0u32..) + .collect::>(); + + let components = + std::mem::replace(&mut self.components, Vec::with_capacity(id_mappings.len())); + self.components.extend( + (0..max_component_id) + .zip(components) + .filter(|(component_id, _)| !remove_component_ids.contains(component_id)) + .map(|(_, mut component)| { + component.component_ids = component + .component_ids + .iter() + .filter_map(|id| id_mappings.get(id).copied()) + .collect(); + component + }), + ); + + true + } + + pub fn copy_timezones(&mut self, other: &ICalendar) { + if self.calendar_root().is_none() { + return; + } + + for component in other + .components + .iter() + .filter(|component| component.component_type == ICalendarComponentType::VTimezone) + { + let items = component + .component_ids + .iter() + .filter_map(|component_id| other.component_by_id(*component_id)); + let has_observances = items.clone().any(|item| { + matches!( + item.component_type, + ICalendarComponentType::Standard | ICalendarComponentType::Daylight + ) + }); + let item_count = items.clone().count(); + let Some((tz_component_id, first_item_id, last_item_id)) = + u32::try_from(self.components.len()) + .ok() + .filter(|_| has_observances) + .and_then(|tz_component_id| { + let first_item_id = tz_component_id.checked_add(1)?; + let last_item_id = + first_item_id.checked_add(u32::try_from(item_count).ok()?)?; + Some((tz_component_id, first_item_id, last_item_id)) + }) + else { + continue; + }; + + self.components.reserve(item_count + 1); + self.components.push(ICalendarComponent { + component_type: ICalendarComponentType::VTimezone, + entries: component.entries.clone(), + component_ids: (first_item_id..last_item_id).collect(), + }); + self.components.extend(items.map(|item| ICalendarComponent { + component_type: item.component_type.clone(), + entries: item.entries.clone(), + component_ids: vec![], + })); + + if let Some(root) = self.components.first_mut() { + let insert_at = root.component_ids.len().min(1); + root.component_ids.insert(insert_at, tz_component_id); + } + } + } +} + +impl ICalendarComponent { + pub fn new(component_type: ICalendarComponentType) -> Self { + Self { + component_type, + entries: Vec::new(), + component_ids: Vec::new(), + } + } + + pub fn add_dtstamp(&mut self, dt_stamp: PartialDateTime) { + self.entries.push(ICalendarEntry { + name: ICalendarProperty::Dtstamp, + params: vec![], + values: smallvec![ICalendarValue::PartialDateTime(dt_stamp)], + }); + } + + pub fn add_sequence(&mut self, sequence: i64) { + self.entries.push(ICalendarEntry { + name: ICalendarProperty::Sequence, + params: vec![], + values: smallvec![ICalendarValue::Integer(sequence)], + }); + } + + pub fn add_uid(&mut self, uid: &str) { + self.entries.push(ICalendarEntry { + name: ICalendarProperty::Uid, + params: vec![], + values: smallvec![ICalendarValue::Text(uid.to_string())], + }); + } + + pub fn add_property(&mut self, name: ICalendarProperty, value: impl Into) { + self.entries.push(ICalendarEntry { + name, + params: vec![], + values: smallvec![value.into()], + }); + } + + pub fn add_property_with_params( + &mut self, + name: ICalendarProperty, + params: impl IntoIterator, + value: impl Into, + ) { + self.entries.push(ICalendarEntry { + name, + params: params.into_iter().collect(), + values: smallvec![value.into()], + }); + } +} + +impl ICalendarEntry { + pub fn new(name: ICalendarProperty) -> Self { + Self { + name, + params: vec![], + values: SmallVec::new(), + } + } + + pub fn with_params(mut self, params: Vec) -> Self { + self.params = params; + self + } + + pub fn with_value(mut self, value: impl Into) -> Self { + self.values.push(value.into()); + self + } + + pub fn with_values(mut self, values: impl IntoIterator) -> Self { + self.values = values.into_iter().collect(); + self + } + + pub fn with_param(mut self, param: impl Into) -> Self { + self.params.push(param.into()); + self + } + + pub fn add_param(&mut self, param: impl Into) { + self.params.push(param.into()); + } + + pub fn with_param_opt(mut self, param: Option>) -> Self { + if let Some(param) = param { + self.params.push(param.into()); + } + self + } + + pub fn is_type(&self, typ: &ICalendarValueType) -> bool { + self.parameters(&ICalendarParameterName::Value) + .any(|p| matches!(p, ICalendarParameterValue::Value(v) if v == typ)) + } +} + +impl ICalendarParameter { + pub fn new(name: ICalendarParameterName, value: impl Into) -> Self { + ICalendarParameter { + name, + value: value.into(), + } + } + + pub fn altrep(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Altrep, + value: value.into(), + } + } + + pub fn cn(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Cn, + value: value.into(), + } + } + + pub fn cutype(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Cutype, + value: value.into(), + } + } + + pub fn delegated_from(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::DelegatedFrom, + value: value.into(), + } + } + + pub fn delegated_to(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::DelegatedTo, + value: value.into(), + } + } + + pub fn dir(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Dir, + value: value.into(), + } + } + + pub fn fmttype(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Fmttype, + value: value.into(), + } + } + + pub fn fbtype(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Fbtype, + value: value.into(), + } + } + + pub fn language(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Language, + value: value.into(), + } + } + + pub fn member(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Member, + value: value.into(), + } + } + + pub fn partstat(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Partstat, + value: value.into(), + } + } + + pub fn range(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Range, + value: value.into(), + } + } + + pub fn related(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Related, + value: value.into(), + } + } + + pub fn reltype(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Reltype, + value: value.into(), + } + } + + pub fn role(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Role, + value: value.into(), + } + } + + pub fn rsvp(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Rsvp, + value: value.into(), + } + } + + pub fn schedule_agent(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::ScheduleAgent, + value: value.into(), + } + } + + pub fn schedule_force_send(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::ScheduleForceSend, + value: value.into(), + } + } + + pub fn schedule_status(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::ScheduleStatus, + value: value.into(), + } + } + + pub fn sent_by(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::SentBy, + value: value.into(), + } + } + + pub fn tzid(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Tzid, + value: value.into(), + } + } + + pub fn value(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Value, + value: value.into(), + } + } + + pub fn display(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Display, + value: value.into(), + } + } + + pub fn email(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Email, + value: value.into(), + } + } + + pub fn feature(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Feature, + value: value.into(), + } + } + + pub fn label(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Label, + value: value.into(), + } + } + + pub fn size(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Size, + value: value.into(), + } + } + + pub fn filename(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Filename, + value: value.into(), + } + } + + pub fn managed_id(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::ManagedId, + value: value.into(), + } + } + + pub fn order(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Order, + value: value.into(), + } + } + + pub fn schema(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Schema, + value: value.into(), + } + } + + pub fn derived(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Derived, + value: value.into(), + } + } + + pub fn gap(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Gap, + value: value.into(), + } + } + + pub fn linkrel(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Linkrel, + value: value.into(), + } + } + + pub fn jsptr(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Jsptr, + value: value.into(), + } + } + + pub fn jsid(value: impl Into) -> Self { + ICalendarParameter { + name: ICalendarParameterName::Jsid, + value: value.into(), + } + } +} + +impl From for ICalendarParameterValue { + fn from(text: String) -> Self { + ICalendarParameterValue::Text(text) + } +} + +impl From for ICalendarParameterValue { + fn from(integer: u64) -> Self { + ICalendarParameterValue::Integer(integer) + } +} + +impl From for ICalendarParameterValue { + fn from(integer: Integer) -> Self { + ICalendarParameterValue::Integer(integer.0.unsigned_abs()) + } +} + +impl From for ICalendarParameterValue { + fn from(boolean: Boolean) -> Self { + ICalendarParameterValue::Bool(boolean.0) + } +} + +impl From for ICalendarParameterValue { + fn from(boolean: bool) -> Self { + ICalendarParameterValue::Bool(boolean) + } +} + +impl From for ICalendarParameterValue { + fn from(uri: Uri) -> Self { + ICalendarParameterValue::Uri(uri) + } +} + +impl From for ICalendarParameterValue { + fn from(cutype: ICalendarUserTypes) -> Self { + ICalendarParameterValue::Cutype(cutype) + } +} + +impl From for ICalendarParameterValue { + fn from(fbtype: ICalendarFreeBusyType) -> Self { + ICalendarParameterValue::Fbtype(fbtype) + } +} + +impl From for ICalendarParameterValue { + fn from(partstat: ICalendarParticipationStatus) -> Self { + ICalendarParameterValue::Partstat(partstat) + } +} + +impl From for ICalendarParameterValue { + fn from(related: ICalendarRelated) -> Self { + ICalendarParameterValue::Related(related) + } +} + +impl From for ICalendarParameterValue { + fn from(reltype: ICalendarRelationshipType) -> Self { + ICalendarParameterValue::Reltype(reltype) + } +} + +impl From for ICalendarParameterValue { + fn from(role: ICalendarParticipationRole) -> Self { + ICalendarParameterValue::Role(role) + } +} + +impl From for ICalendarParameterValue { + fn from(schedule_agent: ICalendarScheduleAgentValue) -> Self { + ICalendarParameterValue::ScheduleAgent(schedule_agent) + } +} + +impl From for ICalendarParameterValue { + fn from(schedule_force_send: ICalendarScheduleForceSendValue) -> Self { + ICalendarParameterValue::ScheduleForceSend(schedule_force_send) + } +} + +impl From for ICalendarParameterValue { + fn from(value: ICalendarValueType) -> Self { + ICalendarParameterValue::Value(value) + } +} + +impl From for ICalendarParameterValue { + fn from(display: ICalendarDisplayType) -> Self { + ICalendarParameterValue::Display(display) + } +} + +impl From for ICalendarParameterValue { + fn from(feature: ICalendarFeatureType) -> Self { + ICalendarParameterValue::Feature(feature) + } +} + +impl From for ICalendarParameterValue { + fn from(duration: ICalendarDuration) -> Self { + ICalendarParameterValue::Duration(duration) + } +} + +impl From for ICalendarParameterValue { + fn from(linkrel: LinkRelation) -> Self { + ICalendarParameterValue::Linkrel(linkrel) + } +} + +impl> From> for ICalendarParameterValue { + fn from(value: IanaType) -> Self { + match value { + IanaType::Iana(v) => v.into(), + IanaType::Other(s) => s.into(), + } + } +} + +impl From> for ICalendarValue { + fn from(value: Vec) -> Self { + ICalendarValue::Binary(value) + } +} + +impl From for ICalendarValue { + fn from(value: bool) -> Self { + ICalendarValue::Boolean(value) + } +} + +impl From for Uri { + fn from(value: Data) -> Self { + Uri::Data(Box::new(value)) + } +} + +impl From for ICalendarValue { + fn from(value: Uri) -> Self { + ICalendarValue::Uri(value) + } +} + +impl From for ICalendarValue { + fn from(value: PartialDateTime) -> Self { + ICalendarValue::PartialDateTime(value) + } +} + +impl From> for ICalendarValue { + fn from(value: Box) -> Self { + ICalendarValue::PartialDateTime(*value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarDuration) -> Self { + ICalendarValue::Duration(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarRecurrenceRule) -> Self { + ICalendarValue::RecurrenceRule(Box::new(value)) + } +} + +impl From> for ICalendarValue { + fn from(value: Box) -> Self { + ICalendarValue::RecurrenceRule(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarPeriod) -> Self { + ICalendarValue::Period(Box::new(value)) + } +} + +impl From for ICalendarValue { + fn from(value: f64) -> Self { + ICalendarValue::Float(value) + } +} + +impl From for ICalendarValue { + fn from(value: i64) -> Self { + ICalendarValue::Integer(value) + } +} + +impl From for ICalendarValue { + fn from(value: String) -> Self { + ICalendarValue::Text(value) + } +} + +impl From<&str> for ICalendarValue { + fn from(value: &str) -> Self { + ICalendarValue::Text(value.to_string()) + } +} + +impl From for ICalendarValue { + fn from(value: CalendarScale) -> Self { + ICalendarValue::CalendarScale(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarMethod) -> Self { + ICalendarValue::Method(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarClassification) -> Self { + ICalendarValue::Classification(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarStatus) -> Self { + ICalendarValue::Status(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarTransparency) -> Self { + ICalendarValue::Transparency(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarAction) -> Self { + ICalendarValue::Action(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarFreeBusyType) -> Self { + ICalendarValue::BusyType(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarParticipantType) -> Self { + ICalendarValue::ParticipantType(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarResourceType) -> Self { + ICalendarValue::ResourceType(value) + } +} + +impl From for ICalendarValue { + fn from(value: ICalendarProximityValue) -> Self { + ICalendarValue::Proximity(value) + } +} + diff --git a/crates/calcard/src/icalendar/dates.rs b/crates/calcard/src/icalendar/dates.rs new file mode 100644 index 00000000..e0f93fdb --- /dev/null +++ b/crates/calcard/src/icalendar/dates.rs @@ -0,0 +1,861 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarPeriod, ICalendarProperty, + ICalendarRecurrenceRule, ICalendarValue, timezone::TzResolver, +}; +use crate::{ + common::{ + DateTimeResult, + timezone::{NominalDuration, Tz, ZonedDateTime}, + }, + datecalc::{MAX_UNPRODUCTIVE_WORK, error::RRuleError, rrule::RRule}, + icalendar::ICalendarParameterName, +}; +use ahash::{AHashMap, AHashSet}; +use jiff::{SignedDuration, civil}; +use std::{ + collections::hash_map::Entry, + fmt::{Display, Formatter}, + hash::Hash, +}; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub struct CalendarExpand { + pub events: Vec, + pub errors: Vec, +} + +/// One instance of a calendar component, with its end already resolved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub struct CalendarEvent { + pub comp_id: u32, + pub start: ZonedDateTime, + pub end: ZonedDateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub struct CalendarError { + pub comp_id: u32, + pub error: CalendarErrorType, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize) +)] +pub enum CalendarErrorType { + MissingDtStart, + InvalidDtStart, + InvalidDtEnd, + InvalidDuration, + RRule(RRuleError), + ExpansionLimitReached, +} + +impl ICalendar { + pub fn expand_dates(&self, default_tz: impl Into, mut limit: usize) -> CalendarExpand { + let tz_resolver = self.build_tz_resolver().with_default(default_tz); + let mut expand = CalendarExpand::default(); + let mut recurrences = Vec::new(); + let mut overridden = OverriddenInstances::default(); + + for (comp_id, comp) in self.components.iter().enumerate() { + if comp.component_type.has_time_ranges() { + match comp.build_calendar_date(comp_id as u32, &tz_resolver) { + Ok(Some(mut event)) => { + if event.rid.is_some() { + expand + .events + .extend(event.rdates.drain(..).map(|rdate| rdate.event)); + overridden.insert(comp_id as u32, event); + } else if event.rrule.is_some() + || !event.rdates.is_empty() + || !event.exdates.is_empty() + { + recurrences.push((comp_id as u32, event)); + } else if let Some(cal_event) = event.event { + expand.events.push(cal_event); + } + } + Ok(None) => {} + Err(error) => { + expand.errors.push(CalendarError { + comp_id: comp_id as u32, + error, + }); + } + } + } + } + + // Expand recurrences + let mut unproductive_budget = MAX_UNPRODUCTIVE_WORK; + for (series_id, event) in &mut recurrences { + let mut comp_id = *series_id; + let length = event.length(); + let exdates = event + .exdates + .iter() + .filter_map(|(tz_id, dt)| { + dt.to_date_time_with_tz(tz_id.map_or(event.start_tz, |tz_id| { + tz_resolver.resolve_or_default(Some(tz_id)) + })) + }) + .collect::>(); + let start_instance = event + .rrule + .is_none() + .then(|| event.event.take()) + .flatten() + .map(|start| RecurrenceInstance { + event: start, + length, + }); + for instance in std::mem::take(&mut event.rdates) + .into_iter() + .chain(start_instance) + { + match overridden.take(event, &instance.event.start) { + Some((_, overridden_event)) => expand.events.extend( + overridden_event + .replacing(instance.event.start, instance.length) + .map(|(replacement, _)| replacement), + ), + None if !exdates.contains(&instance.event.start) => { + expand.events.push(instance.event) + } + None => {} + } + } + let Some(rrule) = event.rrule.take() else { + continue; + }; + let rrule = match RRule::from_ical(rrule, event.dt_start_zoned, event.is_date) { + Ok(rrule) => rrule, + Err(err) => { + expand.errors.push(CalendarError { + comp_id, + error: CalendarErrorType::RRule(err), + }); + continue; + } + }; + let mut override_offset = None; + let mut override_duration = None; + let mut instances = rrule.iter().with_unproductive_budget(unproductive_budget); + + while limit != 0 { + let Some(date) = instances.next() else { + break; + }; + limit -= 1; + if event.rdate_starts.contains(&date) { + continue; + } + match overridden.take(event, &date) { + Some((new_comp_id, overridden_event)) => { + if let Some((new_event, new_duration)) = + overridden_event.replacing(date, length) + { + if overridden_event.rid_this_and_future { + comp_id = new_comp_id; + override_offset = + Some(DefaultDuration::shift(date, new_event.start)); + override_duration = Some(new_duration); + } + expand.events.push(new_event); + } + } + None if !exdates.contains(&date) => { + let start = match override_offset { + Some(offset) => match offset.after(date) { + Some(start) => start, + None => continue, + }, + None => date, + }; + let duration = override_duration.unwrap_or(event.default_duration); + if let Some(end) = duration.after(start) { + expand.events.push(CalendarEvent { + start, + end, + comp_id, + }); + } + } + None => {} + } + } + + unproductive_budget = instances.unproductive_budget(); + if instances.is_exhausted() { + expand.errors.push(CalendarError { + comp_id, + error: CalendarErrorType::ExpansionLimitReached, + }); + } + } + + // Add missing overridden events (this should not occur unless the iCalendar is malformed) + expand.events.extend(overridden.into_events(&recurrences)); + + expand + } +} + +/// How long an instance of a component lasts. +#[derive(Debug, Clone, Copy)] +enum DefaultDuration { + Exact(SignedDuration), + Nominal(NominalDuration), +} + +impl DefaultDuration { + /// Returns the duration between the start and end of a series, as + /// RFC 5545 section 3.8.5.3 says every instance of it should last. + fn between(start: ZonedDateTime, end: ZonedDateTime, has_time: bool) -> Self { + if has_time { + Self::Exact(SignedDuration::from_secs( + end.timestamp() - start.timestamp(), + )) + } else { + Self::Nominal(NominalDuration::new(end.days_since(start), 0)) + } + } + + /// Returns the end of an instance that starts at `start`. + fn shift(from: ZonedDateTime, to: ZonedDateTime) -> Self { + if from.timezone() == to.timezone() { + Self::Nominal(NominalDuration::between(from, to)) + } else { + Self::Exact(to.signed_duration_since(from)) + } + } + + fn after(&self, start: ZonedDateTime) -> Option { + match self { + Self::Exact(duration) => start.checked_add(*duration), + Self::Nominal(duration) => start.checked_add_nominal(*duration), + } + } +} + +struct CalendarEventBuilder<'x> { + event: Option, + component_type: &'x ICalendarComponentType, + start_tz: Tz, + default_duration: DefaultDuration, + inherits_start: bool, + instance_end: InstanceEnd, + dt_start_zoned: ZonedDateTime, + is_date: bool, + rrule: Option<&'x ICalendarRecurrenceRule>, + uid: Option<&'x str>, + sequence: i64, + rdates: Vec, + rdate_starts: AHashSet, + exdates: Vec<(Option<&'x str>, DateTimeResult)>, + rid: Option, + rid_this_and_future: bool, +} + +#[derive(Debug, Clone, Copy)] +struct RecurrenceInstance { + event: CalendarEvent, + length: InstanceLength, +} + +#[derive(Debug, Clone, Copy)] +struct InstanceLength { + duration: DefaultDuration, + is_date: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum InstanceEnd { + Inherited, + Relative, + Fixed, +} + +impl InstanceEnd { + fn of(tz_id: Option<&str>, end: &DateTimeResult) -> Self { + if tz_id.is_some() || end.tz().is_some() { + InstanceEnd::Fixed + } else { + InstanceEnd::Relative + } + } +} + +impl CalendarEventBuilder<'_> { + fn length(&self) -> InstanceLength { + InstanceLength { + duration: self.default_duration, + is_date: self.is_date, + } + } + + fn anchor(&self, recurrence_id: ZonedDateTime) -> Option { + if recurrence_id.timezone().is_floating() { + self.start_tz.from_local(recurrence_id.naive_local()) + } else { + Some(recurrence_id.with_timezone(self.start_tz)) + } + } + + fn replacing( + &self, + start: ZonedDateTime, + replaced: InstanceLength, + ) -> Option<(CalendarEvent, DefaultDuration)> { + let event = self.event?; + if !self.inherits_start || self.is_date != replaced.is_date { + return Some((event, self.default_duration)); + } + let (end, duration) = match self.instance_end { + InstanceEnd::Fixed => ( + event.end, + DefaultDuration::Exact(event.end.signed_duration_since(start)), + ), + InstanceEnd::Relative => ( + self.default_duration.after(start).unwrap_or(event.end), + self.default_duration, + ), + InstanceEnd::Inherited => ( + replaced.duration.after(start).unwrap_or(event.end), + replaced.duration, + ), + }; + Some(( + CalendarEvent { + start, + end, + ..event + }, + duration, + )) + } +} + +type OverriddenInstance<'x> = (u32, CalendarEventBuilder<'x>); + +type OverrideKey<'x, T> = (Option<&'x str>, &'x ICalendarComponentType, T); + +#[derive(Default)] +struct OverriddenInstances<'x> { + zoned: AHashMap, OverriddenInstance<'x>>, + floating: AHashMap, OverriddenInstance<'x>>, +} + +impl<'x> OverriddenInstances<'x> { + fn insert(&mut self, comp_id: u32, instance: CalendarEventBuilder<'x>) { + match instance.rid { + Some(rid) if rid.timezone().is_floating() => { + Self::insert_latest( + &mut self.floating, + (instance.uid, instance.component_type, rid.naive_local()), + (comp_id, instance), + ); + } + Some(rid) => { + Self::insert_latest( + &mut self.zoned, + (instance.uid, instance.component_type, rid), + (comp_id, instance), + ); + } + None => {} + } + } + + fn insert_latest( + instances: &mut AHashMap>, + key: K, + instance: OverriddenInstance<'x>, + ) { + match instances.entry(key) { + Entry::Occupied(current) if current.get().1.sequence > instance.1.sequence => {} + Entry::Occupied(mut current) => { + current.insert(instance); + } + Entry::Vacant(slot) => { + slot.insert(instance); + } + } + } + + fn take( + &mut self, + series: &CalendarEventBuilder<'x>, + start: &ZonedDateTime, + ) -> Option> { + (!self.zoned.is_empty()) + .then(|| { + self.zoned + .remove(&(series.uid, series.component_type, *start)) + }) + .flatten() + .or_else(|| { + (!self.floating.is_empty()) + .then(|| { + self.floating.remove(&( + series.uid, + series.component_type, + start.naive_local(), + )) + }) + .flatten() + }) + } + + fn into_events<'y>( + self, + recurrences: &'y [(u32, CalendarEventBuilder<'x>)], + ) -> impl Iterator + use<'x, 'y> { + let series = if self.zoned.is_empty() && self.floating.is_empty() { + AHashMap::new() + } else { + recurrences + .iter() + .rev() + .map(|(_, series)| ((series.uid, series.component_type), series)) + .collect::>() + }; + self.zoned + .into_values() + .chain(self.floating.into_values()) + .filter_map(move |(_, instance)| { + series + .get(&(instance.uid, instance.component_type)) + .and_then(|series| { + instance.replacing(series.anchor(instance.rid?)?, series.length()) + }) + .map(|(replacement, _)| replacement) + .or(instance.event) + }) + } +} + +/// How an `RDATE` of PERIOD value type names the end of its instance. +enum PeriodEnd { + Time(DateTimeResult), + Duration(NominalDuration), +} + +impl ICalendarComponent { + fn build_calendar_date( + &self, + comp_id: u32, + tz_resolver: &TzResolver<&'_ str>, + ) -> Result>, CalendarErrorType> { + let mut dt_start = None; + let mut dt_start_tzid = None; + let mut dt_start_has_time = false; + let mut dt_end: Option = None; + let mut dt_end_tzid = None; + let mut todo_dates = vec![]; + let mut rid: Option = None; + let mut rid_tzid = None; + let mut rid_has_time = false; + let mut rid_this_and_future = false; + let mut duration = None; + let mut rrule = None; + let mut uid = None; + let mut sequence = 0; + let mut rdates = vec![]; + let mut rdates_periods = vec![]; + let mut exdates = vec![]; + + for entry in &self.entries { + match (&entry.name, entry.values.first()) { + (ICalendarProperty::Dtstart, Some(ICalendarValue::PartialDateTime(dt))) => { + dt_start = dt.to_date_time(); + dt_start_tzid = entry.tz_id(); + dt_start_has_time = dt.has_time(); + } + (ICalendarProperty::Dtend, Some(ICalendarValue::PartialDateTime(dt))) => { + if let Some(dt) = dt.to_date_time() { + dt_end = Some(dt); + dt_end_tzid = entry.tz_id(); + } + } + ( + ICalendarProperty::Due + | ICalendarProperty::Completed + | ICalendarProperty::Created, + Some(ICalendarValue::PartialDateTime(dt)), + ) if self.component_type == ICalendarComponentType::VTodo => { + todo_dates.push((&entry.name, dt.to_date_time(), entry.tz_id())); + } + (ICalendarProperty::RecurrenceId, Some(ICalendarValue::PartialDateTime(value))) => { + if let Some(dt) = value.to_date_time() { + for param in &entry.params { + match ¶m.name { + ICalendarParameterName::Tzid => { + rid_tzid = param.value.as_text(); + } + ICalendarParameterName::Range => { + rid_this_and_future = true; + } + _ => (), + } + } + + rid_has_time = value.has_time(); + rid = Some(dt); + } + } + (ICalendarProperty::Duration, Some(ICalendarValue::Duration(dur))) => { + duration = Some(dur); + } + (ICalendarProperty::Rrule, Some(ICalendarValue::RecurrenceRule(rule))) => { + rrule = Some(rule); + } + (ICalendarProperty::Uid, Some(ICalendarValue::Text(value))) => { + uid = Some(value.as_str()); + } + (ICalendarProperty::Sequence, Some(ICalendarValue::Integer(value))) => { + sequence = *value; + } + (ICalendarProperty::Rdate, _) => { + let tz_id = entry.tz_id(); + for value in &entry.values { + match value { + ICalendarValue::PartialDateTime(dt) => { + if let Some(dt) = dt.to_date_time() { + rdates.push((tz_id, dt)); + } + } + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, end } => { + if let (Some(start), Some(end)) = + (start.to_date_time(), end.to_date_time()) + { + rdates_periods.push((tz_id, start, PeriodEnd::Time(end))); + } + } + ICalendarPeriod::Duration { start, duration } => { + if let (Some(start), Some(duration)) = + (start.to_date_time(), duration.to_nominal()) + { + rdates_periods.push(( + tz_id, + start, + PeriodEnd::Duration(duration), + )); + } + } + }, + _ => (), + } + } + } + (ICalendarProperty::Exdate, _) => { + let tz_id = entry.tz_id(); + for value in &entry.values { + if let ICalendarValue::PartialDateTime(dt) = value + && let Some(dt) = dt.to_date_time() + { + exdates.push((tz_id, dt)); + } + } + } + _ => (), + } + } + + let has_start = dt_start.is_some(); + let dt_start = match dt_start { + Some(dt_start) => dt_start, + None => match &rid { + Some(rid) => { + dt_start_tzid = rid_tzid; + dt_start_has_time = rid_has_time; + rid.clone() + } + None => match self.component_type { + ICalendarComponentType::VEvent => { + return Err(CalendarErrorType::MissingDtStart); + } + ICalendarComponentType::VTodo => { + let mut due_idx = None; + let mut completed_idx = None; + let mut created_idx = None; + + for (idx, (prop, dt, _)) in todo_dates.iter().enumerate() { + if dt.is_some() { + match prop { + ICalendarProperty::Due => due_idx = Some(idx), + ICalendarProperty::Completed => completed_idx = Some(idx), + ICalendarProperty::Created => created_idx = Some(idx), + _ => (), + } + } + } + + match (due_idx, completed_idx, created_idx) { + (Some(due_idx), _, _) => { + let due = &mut todo_dates[due_idx]; + dt_start_tzid = due.2; + dt_start_has_time = true; + due.1.take().unwrap() + } + (_, Some(completed_idx), Some(created_idx)) => { + let completed = &mut todo_dates[completed_idx]; + dt_end = completed.1.take(); + dt_end_tzid = completed.2; + + let created = &mut todo_dates[created_idx]; + dt_start_tzid = created.2; + created.1.take().unwrap() + } + (_, Some(date_idx), _) | (_, _, Some(date_idx)) => { + let date = &mut todo_dates[date_idx]; + dt_start_tzid = date.2; + dt_start_has_time = true; + date.1.take().unwrap() + } + _ => { + return Ok(None); + } + } + } + _ => { + return Ok(None); + } + }, + }, + }; + let mut event = None; + let is_instance = rid.is_some(); + let start_tz = dt_start + .tz() + .unwrap_or_else(|| tz_resolver.resolve_or_default(dt_start_tzid)); + let value_tz = |tz_id: Option<&str>| { + tz_id.map_or(start_tz, |tz_id| { + tz_resolver.resolve_or_default(Some(tz_id)) + }) + }; + let dt_start_tz = dt_start + .to_date_time_with_tz(start_tz) + .ok_or(CalendarErrorType::InvalidDtStart)?; + let mut instance_end = InstanceEnd::Relative; + let default_duration = if let Some(dt_end) = dt_end { + let end = dt_end + .to_date_time_with_tz(tz_resolver.resolve_or_default(dt_end_tzid.or(dt_start_tzid))) + .ok_or(CalendarErrorType::InvalidDtEnd)?; + + if rrule.is_none() || is_instance { + event = Some(CalendarEvent { + start: dt_start_tz, + end, + comp_id, + }); + } + instance_end = InstanceEnd::of(dt_end_tzid, &dt_end); + DefaultDuration::between(dt_start_tz, end, dt_start_has_time) + } else if let Some(duration) = duration { + let duration = duration + .to_nominal() + .ok_or(CalendarErrorType::InvalidDuration)?; + let duration = DefaultDuration::Nominal(duration); + if rrule.is_none() || is_instance { + event = Some(CalendarEvent { + start: dt_start_tz, + end: duration + .after(dt_start_tz) + .ok_or(CalendarErrorType::InvalidDuration)?, + comp_id, + }); + } + duration + } else if let Some((due, due_tzid)) = todo_dates + .into_iter() + .filter_map(|(prop, dt, tz_id)| { + if prop == &ICalendarProperty::Due { + dt.map(|dt| (dt, tz_id)) + } else { + None + } + }) + .next() + { + let end = due + .to_date_time_with_tz(tz_resolver.resolve_or_default(due_tzid.or(dt_start_tzid))) + .ok_or(CalendarErrorType::InvalidDtEnd)?; + + if rrule.is_none() || is_instance { + event = Some(CalendarEvent { + start: dt_start_tz, + end, + comp_id, + }); + } + instance_end = InstanceEnd::of(due_tzid, &due); + DefaultDuration::between(dt_start_tz, end, dt_start_has_time) + } else { + /* + For cases where a "VEVENT" calendar component + specifies a "DTSTART" property with a DATE value type but no + "DTEND" nor "DURATION" property, the event's duration is taken to + be one day. For cases where a "VEVENT" calendar component + specifies a "DTSTART" property with a DATE-TIME value type but no + "DTEND" property, the event ends on the same calendar date and + time of day specified by the "DTSTART" property. + */ + + instance_end = InstanceEnd::Inherited; + let duration = + if dt_start_has_time || self.component_type == ICalendarComponentType::VTodo { + DefaultDuration::Exact(SignedDuration::ZERO) + } else { + DefaultDuration::Nominal(NominalDuration::DAY) + }; + if rrule.is_none() || is_instance { + event = Some(CalendarEvent { + start: dt_start_tz, + end: duration + .after(dt_start_tz) + .ok_or(CalendarErrorType::InvalidDuration)?, + comp_id, + }); + } + duration + }; + let rid = rid.and_then(|rid| { + rid.to_date_time_with_tz(rid_tzid.map_or(Tz::Floating, |tz_id| { + tz_resolver.resolve_or_default(Some(tz_id)) + })) + }); + + // Add rdates + let is_date = !dt_start_has_time; + let series_length = InstanceLength { + duration: default_duration, + is_date, + }; + let mut rdate_events = rdates + .into_iter() + .filter_map(|(tz_id, rdate)| { + let start = rdate.to_date_time_with_tz(value_tz(tz_id))?; + Some(RecurrenceInstance { + event: CalendarEvent { + start, + end: default_duration.after(start)?, + comp_id, + }, + length: series_length, + }) + }) + .chain( + rdates_periods + .into_iter() + .filter_map(|(tz_id, start, end)| { + let tz = value_tz(tz_id); + let start = start.to_date_time_with_tz(tz)?; + let (end, duration) = match end { + PeriodEnd::Time(end) => { + let end = end.to_date_time_with_tz(tz)?; + ( + end, + DefaultDuration::Exact(end.signed_duration_since(start)), + ) + } + PeriodEnd::Duration(duration) => { + let duration = DefaultDuration::Nominal(duration); + (duration.after(start)?, duration) + } + }; + Some(RecurrenceInstance { + event: CalendarEvent { + start, + end, + comp_id, + }, + length: InstanceLength { + duration, + is_date: false, + }, + }) + }), + ) + .collect::>(); + let mut rdate_starts = AHashSet::new(); + if !rdate_events.is_empty() { + rdate_events.retain(|rdate| rdate_starts.insert(rdate.event.start)); + if event + .as_ref() + .is_some_and(|event| rdate_starts.contains(&event.start)) + { + event = None; + } + } + + Ok(Some(CalendarEventBuilder { + event, + component_type: &self.component_type, + default_duration, + inherits_start: is_instance && !has_start, + instance_end, + dt_start_zoned: dt_start_tz, + is_date, + uid, + sequence, + rrule: rrule.map(|rule| &**rule), + rdates: rdate_events, + rdate_starts, + exdates, + start_tz, + rid, + rid_this_and_future, + })) + } +} + +impl CalendarEvent { + /// Returns the instant the instance starts and the instant it ends. + pub fn timestamps(&self) -> (i64, i64) { + (self.start.timestamp(), self.end.timestamp()) + } + + /// Returns the wall clock readings of the start and end, as seconds since + /// the Unix epoch, for callers that index instances by displayed time. + pub fn naive_timestamps(&self) -> (i64, i64) { + (self.start.naive_timestamp(), self.end.naive_timestamp()) + } +} + +impl Display for CalendarErrorType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + CalendarErrorType::MissingDtStart => write!(f, "Missing DTSTART property"), + CalendarErrorType::InvalidDtStart => write!(f, "Invalid DTSTART property"), + CalendarErrorType::InvalidDtEnd => write!(f, "Invalid DTEND property"), + CalendarErrorType::InvalidDuration => write!(f, "Invalid DURATION property"), + CalendarErrorType::RRule(err) => write!(f, "RRule error: {err}"), + CalendarErrorType::ExpansionLimitReached => { + write!(f, "Recurrence expansion gave up before the rule finished") + } + } + } +} + diff --git a/crates/calcard/src/icalendar/mod.rs b/crates/calcard/src/icalendar/mod.rs new file mode 100644 index 00000000..0f4f1c74 --- /dev/null +++ b/crates/calcard/src/icalendar/mod.rs @@ -0,0 +1,978 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[cfg(feature = "rkyv")] +use crate::common::archive::{InlineVec, Unboxed}; +use crate::common::{ + CalendarScale, Data, IanaParse, IanaString, IanaType, LinkRelation, PartialDateTime, +}; +use smallvec::SmallVec; +use std::hash::{Hash, Hasher}; + +pub mod builder; +pub mod dates; +pub mod parser; +pub mod timezone; +pub mod types; +pub mod utils; +pub mod writer; + +#[cfg(feature = "rkyv")] +pub mod rkyv_timezone; +#[cfg(feature = "rkyv")] +pub mod rkyv_types; +#[cfg(feature = "rkyv")] +pub mod rkyv_utils; +#[cfg(feature = "rkyv")] +pub mod rkyv_writer; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct ICalendar { + pub components: Vec, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct ICalendarComponent { + pub component_type: ICalendarComponentType, + pub entries: Vec, + pub component_ids: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct ICalendarEntry { + pub name: ICalendarProperty, + pub params: Vec, + #[cfg_attr(feature = "rkyv", rkyv(with = InlineVec))] + pub values: SmallVec<[ICalendarValue; 1]>, +} + +#[derive(Debug, Clone, PartialEq, PartialOrd)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub enum ICalendarValue { + Binary(Vec), + Boolean(bool), + Uri(Uri), + PartialDateTime(PartialDateTime), + Duration(ICalendarDuration), + RecurrenceRule(Box), + Period(#[cfg_attr(feature = "rkyv", rkyv(with = Unboxed))] Box), + Float(f64), + Integer(i64), + Text(String), + CalendarScale(CalendarScale), + Method(ICalendarMethod), + Classification(ICalendarClassification), + Status(ICalendarStatus), + Transparency(ICalendarTransparency), + Action(ICalendarAction), + BusyType(ICalendarFreeBusyType), + ParticipantType(ICalendarParticipantType), + ResourceType(ICalendarResourceType), + Proximity(ICalendarProximityValue), +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub struct ICalendarRecurrenceRule { + pub freq: ICalendarFrequency, + pub until: Option, + pub count: Option, + pub interval: Option, + pub bysecond: Vec, + pub byminute: Vec, + pub byhour: Vec, + pub byday: Vec, + pub bymonthday: Vec, + pub byyearday: Vec, + pub byweekno: Vec, + pub bymonth: Vec, + pub bysetpos: Vec, + pub wkst: Option, + pub rscale: Option, + pub skip: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +#[repr(u8)] +pub enum ICalendarSkip { + Omit, + Backward, + Forward, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub struct ICalendarDay { + pub ordwk: Option, + pub weekday: ICalendarWeekday, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +#[repr(transparent)] +pub struct ICalendarMonth(i8); + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +#[repr(u8)] +pub enum ICalendarFrequency { + Yearly = 0, + Monthly = 1, + Weekly = 2, + #[default] + Daily = 3, + Hourly = 4, + Minutely = 5, + Secondly = 6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarWeekday { + Sunday, + Monday, + Tuesday, + Wednesday, + Thursday, + Friday, + Saturday, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarPeriod { + Range { + start: PartialDateTime, + end: PartialDateTime, + }, + Duration { + start: PartialDateTime, + duration: ICalendarDuration, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarAction { + Audio, // [RFC5545, Section 3.8.6.1] + Display, // [RFC5545, Section 3.8.6.1] + Email, // [RFC5545, Section 3.8.6.1] + Procedure, // [RFC2445, Section 4.8.6.1] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarUserTypes { + Individual, // [RFC5545, Section 3.2.3] + Group, // [RFC5545, Section 3.2.3] + Resource, // [RFC5545, Section 3.2.3] + Room, // [RFC5545, Section 3.2.3] + Unknown, // [RFC5545, Section 3.2.3] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarClassification { + Public, // [RFC5545, Section 3.8.1.3] + Private, // [RFC5545, Section 3.8.1.3] + Confidential, // [RFC5545, Section 3.8.1.3] +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarComponentType { + Other(String), + #[default] + VCalendar, // [RFC5545, Section 3.4] + VEvent, // [RFC5545, Section 3.6.1] + VTodo, // [RFC5545, Section 3.6.2] + VJournal, // [RFC5545, Section 3.6.3] + VFreebusy, // [RFC5545, Section 3.6.4] + VTimezone, // [RFC5545, Section 3.6.5] + VAlarm, // [RFC5545, Section 3.6.6] + Standard, // [RFC5545, Section 3.6.5] + Daylight, // [RFC5545, Section 3.6.5] + VAvailability, // [RFC7953, Section 3.1] + Available, // [RFC7953, Section 3.1] + Participant, // [RFC9073, Section 7.1] + VLocation, // [RFC9073, Section 7.2] [RFC Errata 7381] + VResource, // [RFC9073, Section 7.3] + VStatus, // draft-ietf-calext-ical-tasks-17 +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarDisplayType { + Badge, // [RFC7986, Section 6.1] + Graphic, // [RFC7986, Section 6.1] + Fullsize, // [RFC7986, Section 6.1] + Thumbnail, // [RFC7986, Section 6.1] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarFeatureType { + Audio, // [RFC7986, Section 6.3] + Chat, // [RFC7986, Section 6.3] + Feed, // [RFC7986, Section 6.3] + Moderator, // [RFC7986, Section 6.3] + Phone, // [RFC7986, Section 6.3] + Screen, // [RFC7986, Section 6.3] + Video, // [RFC7986, Section 6.3] +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarFreeBusyType { + Free, // [RFC5545, Section 3.2.9] + Busy, // [RFC5545, Section 3.2.9] + BusyUnavailable, // [RFC5545, Section 3.2.9] + BusyTentative, // [RFC5545, Section 3.2.9] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarMethod { + Publish, // [RFC5546] + Request, // [RFC5546] + Reply, // [RFC5546] + Add, // [RFC5546] + Cancel, // [RFC5546] + Refresh, // [RFC5546] + Counter, // [RFC5546] + Declinecounter, // [RFC5546] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct ICalendarParameter { + pub name: ICalendarParameterName, + pub value: ICalendarParameterValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub enum ICalendarParameterValue { + Text(String), + Integer(u64), + Bool(bool), + Uri(Uri), + Cutype(ICalendarUserTypes), + Fbtype(ICalendarFreeBusyType), + Partstat(ICalendarParticipationStatus), + Related(ICalendarRelated), + Reltype(ICalendarRelationshipType), + Role(ICalendarParticipationRole), + ScheduleAgent(ICalendarScheduleAgentValue), + ScheduleForceSend(ICalendarScheduleForceSendValue), + Value(ICalendarValueType), + Display(ICalendarDisplayType), + Feature(ICalendarFeatureType), + Duration(ICalendarDuration), + Linkrel(LinkRelation), + Null, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(PartialEq)))] +pub enum ICalendarParameterName { + Other(String), + Altrep, // [RFC5545, Section 3.2.1] + Cn, // [RFC5545, Section 3.2.2] + Cutype, // [RFC5545, Section 3.2.3] + DelegatedFrom, // [RFC5545, Section 3.2.4] + DelegatedTo, // [RFC5545, Section 3.2.5] + Dir, // [RFC5545, Section 3.2.6] + Fmttype, // [RFC5545, Section 3.2.8] + Fbtype, // [RFC5545, Section 3.2.9] + Language, // [RFC5545, Section 3.2.10] + Member, // [RFC5545, Section 3.2.11] + Partstat, // [RFC5545, Section 3.2.12] + Range, // [RFC5545, Section 3.2.13] + Related, // [RFC5545, Section 3.2.14] + Reltype, // [RFC5545, Section 3.2.15] + Role, // [RFC5545, Section 3.2.16] + Rsvp, // [RFC5545, Section 3.2.17] + ScheduleAgent, // [RFC6638, Section 7.1] + ScheduleForceSend, // [RFC6638, Section 7.2] + ScheduleStatus, // [RFC6638, Section 7.3] + SentBy, // [RFC5545, Section 3.2.18] + Tzid, // [RFC5545, Section 3.2.19] + Value, // [RFC5545, Section 3.2.20] + Display, // [RFC7986, Section 6.1] + Email, // [RFC7986, Section 6.2] + Feature, // [RFC7986, Section 6.3] + Label, // [RFC7986, Section 6.4] + Size, // [RFC8607, Section 4.1] + Filename, // [RFC8607, Section 4.2] + ManagedId, // [RFC8607, Section 4.3] + Order, // [RFC9073, Section 5.1] + Schema, // [RFC9073, Section 5.2] + Derived, // [RFC9073, Section 5.3] + Gap, // [RFC9253, Section 6.2] + Linkrel, // [RFC9253, Section 6.1] + Jsptr, // draft-ietf-calext-jscalendar-icalendar + Jsid, // draft-ietf-calext-jscalendar-icalendar +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub struct ICalendarDuration { + pub neg: bool, + pub weeks: u32, + pub days: u32, + pub hours: u32, + pub minutes: u32, + pub seconds: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum Uri { + Data(#[cfg_attr(feature = "rkyv", rkyv(with = Unboxed))] Box), + Location(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarRelated { + Start, + End, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarParticipantType { + Active, // [RFC9073, Section 6.2] + Inactive, // [RFC9073, Section 6.2] + Sponsor, // [RFC9073, Section 6.2] + Contact, // [RFC9073, Section 6.2] + BookingContact, // [RFC9073, Section 6.2] + EmergencyContact, // [RFC9073, Section 6.2] + PublicityContact, // [RFC9073, Section 6.2] + PlannerContact, // [RFC9073, Section 6.2] + Performer, // [RFC9073, Section 6.2] + Speaker, // [RFC9073, Section 6.2] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarParticipationRole { + Chair, // [RFC5545, Section 3.2.16] + ReqParticipant, // [RFC5545, Section 3.2.16] + OptParticipant, // [RFC5545, Section 3.2.16] + NonParticipant, // [RFC5545, Section 3.2.16] + Owner, // draft-ietf-calext-icalendar-jscalendar-extensions +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarStatus { + Tentative, // [RFC5545, Section 3.8.1] + Confirmed, // [RFC5545, Section 3.8.1] + Cancelled, // [RFC5545, Section 3.8.1] + NeedsAction, // [RFC5545, Section 3.8.1] + Completed, // [RFC5545, Section 3.8.1] + InProcess, // [RFC5545, Section 3.8.1] + Draft, // [RFC5545, Section 3.8.1] + Final, // [RFC5545, Section 3.8.1] + Failed, // draft-ietf-calext-ical-tasks + Pending, // draft-ietf-calext-ical-tasks +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarParticipationStatus { + NeedsAction, // [RFC5545, Section 3.2.12] + Accepted, // [RFC5545, Section 3.2.12] + Declined, // [RFC5545, Section 3.2.12] + Tentative, // [RFC5545, Section 3.2.12] + Delegated, // [RFC5545, Section 3.2.12] + Completed, // [RFC5545, Section 3.2.12] + InProcess, // [RFC5545, Section 3.2.12] + Failed, // draft-ietf-calext-ical-tasks +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarProperty { + Begin, + End, + Other(String), + Calscale, // [RFC5545, Section 3.7.1] + Method, // [RFC5545, Section 3.7.2] + Prodid, // [RFC5545, Section 3.7.3] + Version, // [RFC5545, Section 3.7.4] + Attach, // [RFC5545, Section 3.8.1.1] + Categories, // [RFC5545, Section 3.8.1.2] [RFC7986, Section 5.6] + Class, // [RFC5545, Section 3.8.1.3] + Comment, // [RFC5545, Section 3.8.1.4] + Description, // [RFC5545, Section 3.8.1.5] [RFC7986, Section 5.2] + Geo, // [RFC5545, Section 3.8.1.6] + Location, // [RFC5545, Section 3.8.1.7] + PercentComplete, // [RFC5545, Section 3.8.1.8] + Priority, // [RFC5545, Section 3.8.1.9] + Resources, // [RFC5545, Section 3.8.1.10] + Status, // [RFC5545, Section 3.8.1.11] + Summary, // [RFC5545, Section 3.8.1.12] + Completed, // [RFC5545, Section 3.8.2.1] + Dtend, // [RFC5545, Section 3.8.2.2] + Due, // [RFC5545, Section 3.8.2.3] + Dtstart, // [RFC5545, Section 3.8.2.4] + Duration, // [RFC5545, Section 3.8.2.5] + Freebusy, // [RFC5545, Section 3.8.2.6] + Transp, // [RFC5545, Section 3.8.2.7] + Tzid, // [RFC5545, Section 3.8.3.1] + Tzname, // [RFC5545, Section 3.8.3.2] + Tzoffsetfrom, // [RFC5545, Section 3.8.3.3] + Tzoffsetto, // [RFC5545, Section 3.8.3.4] + Tzurl, // [RFC5545, Section 3.8.3.5] + Attendee, // [RFC5545, Section 3.8.4.1] + Contact, // [RFC5545, Section 3.8.4.2] + Organizer, // [RFC5545, Section 3.8.4.3] + RecurrenceId, // [RFC5545, Section 3.8.4.4] + RelatedTo, // [RFC5545, Section 3.8.4.5] [RFC9253, Section 9.1] + Url, // [RFC5545, Section 3.8.4.6] [RFC7986, Section 5.5] + Uid, // [RFC5545, Section 3.8.4.7] [RFC7986, Section 5.3] + Exdate, // [RFC5545, Section 3.8.5.1] + Exrule, // [RFC2445, Section 4.8.5.2] + Rdate, // [RFC5545, Section 3.8.5.2] + Rrule, // [RFC5545, Section 3.8.5.3] + Action, // [RFC5545, Section 3.8.6.1] + Repeat, // [RFC5545, Section 3.8.6.2] + Trigger, // [RFC5545, Section 3.8.6.3] + Created, // [RFC5545, Section 3.8.7.1] + Dtstamp, // [RFC5545, Section 3.8.7.2] + LastModified, // [RFC5545, Section 3.8.7.3] [RFC7986, Section 5.4] + Sequence, // [RFC5545, Section 3.8.7.4] + RequestStatus, // [RFC5545, Section 3.8.8.3] + Xml, // [RFC6321, Section 4.2] + Tzuntil, // [RFC7808, Section 7.1] + TzidAliasOf, // [RFC7808, Section 7.2] + Busytype, // [RFC7953, Section 3.2] + Name, // [RFC7986, Section 5.1] + RefreshInterval, // [RFC7986, Section 5.7] + Source, // [RFC7986, Section 5.8] + Color, // [RFC7986, Section 5.9] + Image, // [RFC7986, Section 5.10] + Conference, // [RFC7986, Section 5.11] + CalendarAddress, // [RFC9073, Section 6.4] + LocationType, // [RFC9073, Section 6.1] + ParticipantType, // [RFC9073, Section 6.2] + ResourceType, // [RFC9073, Section 6.3] + StructuredData, // [RFC9073, Section 6.6] + StyledDescription, // [RFC9073, Section 6.5] + Acknowledged, // [RFC9074, Section 6.1] + Proximity, // [RFC9074, Section 8.1] + Concept, // [RFC9253, Section 8.1] + Link, // [RFC9253, Section 8.2] + Refid, // [RFC9253, Section 8.3] + Coordinates, // draft-ietf-calext-icalendar-jscalendar-extensions + ShowWithoutTime, // draft-ietf-calext-icalendar-jscalendar-extensions + Jsid, // draft-ietf-calext-jscalendar-icalendar + Jsprop, // draft-ietf-calext-jscalendar-icalendar + EstimatedDuration, // draft-ietf-calext-ical-tasks + Reason, // draft-ietf-calext-ical-tasks + Substate, // draft-ietf-calext-ical-tasks + TaskMode, // draft-ietf-calext-ical-tasks +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarProximityValue { + Arrive, // [RFC9074, Section 8.1] + Depart, // [RFC9074, Section 8.1] + Connect, // [RFC9074, Section 8.1] + Disconnect, // [RFC9074, Section 8.1] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarRelationshipType { + Child, // [RFC5545, Section 3.2.15] + Parent, // [RFC5545, Section 3.2.15] + Sibling, // [RFC5545, Section 3.2.15] + Snooze, // [RFC9074, Section 7.1] + Concept, // [RFC9253, Section 5] + DependsOn, // [RFC9253, Section 5] + Finishtofinish, // [RFC9253, Section 4] + Finishtostart, // [RFC9253, Section 4] + First, // [RFC9253, Section 5] + Next, // [RFC9253, Section 5] + Refid, // [RFC9253, Section 5] + Starttofinish, // [RFC9253, Section 4] + Starttostart, // [RFC9253, Section 4] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarResourceType { + Projector, // [RFC9073, Section 6.3] + Room, // [RFC9073, Section 6.3] + RemoteConferenceAudio, // [RFC9073, Section 6.3] + RemoteConferenceVideo, // [RFC9073, Section 6.3] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarScheduleAgentValue { + Server, // [RFC6638, Section 7.1] + Client, // [RFC6638, Section 7.1] + None, // [RFC6638, Section 7.1] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarScheduleForceSendValue { + Request, // [RFC6638, Section 7.2] + Reply, // [RFC6638, Section 7.2] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarValueType { + Binary, // [RFC5545, Section 3.3.1] + Boolean, // [RFC5545, Section 3.3.2] + CalAddress, // [RFC5545, Section 3.3.3] + Date, // [RFC5545, Section 3.3.4] + DateTime, // [RFC5545, Section 3.3.5] + Duration, // [RFC5545, Section 3.3.6] + Float, // [RFC5545, Section 3.3.7] + Integer, // [RFC5545, Section 3.3.8] + Period, // [RFC5545, Section 3.3.9] + Recur, // [RFC5545, Section 3.3.10] + Text, // [RFC5545, Section 3.3.11] + Time, // [RFC5545, Section 3.3.12] + Unknown, // [RFC7265, Section 5] + Uri, // [RFC5545, Section 3.3.13] + UtcOffset, // [RFC5545, Section 3.3.14] + XmlReference, // [RFC9253, Section 7] + Uid, // [RFC9253, Section 7] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum ICalendarTransparency { + Opaque, + Transparent, +} + +pub(crate) const VALUE_SIZE: usize = 56; +pub(crate) const PARAMETER_VALUE_SIZE: usize = 48; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ValueSeparator { + None, + Comma, + Semicolon, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ValueType { + Ical(ICalendarValueType), + CalendarScale, + Method, + Classification, + Status, + Transparency, + Action, + BusyType, + ParticipantType, + ResourceType, + Proximity, +} diff --git a/crates/calcard/src/icalendar/parser.rs b/crates/calcard/src/icalendar/parser.rs new file mode 100644 index 00000000..4d13bbc7 --- /dev/null +++ b/crates/calcard/src/icalendar/parser.rs @@ -0,0 +1,1124 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::{ + Entry, Parser, StopChar, Token, + common::{ + CalendarScale, Encoding, PartialDateTime, + decode::{Decoded, ValueDecoder}, + parser::{Boolean, Integer, parse_digits, parse_small_digits}, + stack::ComponentStack, + tokenizer::Mode, + }, + icalendar::{ICalendarDay, ICalendarWeekday}, +}; +use smallvec::{SmallVec, smallvec}; +use std::{iter::Peekable, mem::take, slice::Iter}; + +const EMPTY_CALENDAR_LINE: &str = "BEGIN:VCALENDAR"; +const COMPONENT_CAPACITY: usize = 4; + +impl ICalendar { + fn into_entry(self, strict: bool) -> Entry { + if strict + && self + .calendar_root() + .is_some_and(|root| root.component_ids.is_empty()) + { + Entry::InvalidLine(EMPTY_CALENDAR_LINE.to_string()) + } else { + Entry::ICalendar(self) + } + } +} + +struct Params { + params: Vec, + stop_char: StopChar, + data_type: Option>, + charset: Option, + encoding: Option, +} + +impl Params { + fn decode_value( + &self, + token: &mut Token<'_>, + payload: Option>, + encoding: Encoding, + name: &ICalendarProperty, + ) -> Option> { + let binary = match &self.data_type { + Some(data_type) => data_type == &IanaType::Iana(ICalendarValueType::Binary), + None => { + encoding == Encoding::Base64 + && matches!(name, ICalendarProperty::Attach | ICalendarProperty::Image) + } + }; + let decoder = ValueDecoder { + encoding, + charset: self.charset.as_deref(), + binary, + }; + match decoder.decode(&token.text, token.stop_char, payload) { + Decoded::Text(text) => { + token.text = text.into(); + None + } + Decoded::Binary(bytes) => Some(bytes), + Decoded::Undecodable => None, + } + } + + fn value(&mut self, token: Token<'_>, default_type: &ValueType) -> ICalendarValue { + let value_type = match default_type { + ValueType::Ical(default_type) => self + .data_type + .as_ref() + .map(|v| v.iana().unwrap_or(&ICalendarValueType::Text)) + .unwrap_or(default_type), + ValueType::CalendarScale => { + return IanaType::from(token).map_or_text(ICalendarValue::CalendarScale); + } + ValueType::Method => return IanaType::from(token).map_or_text(ICalendarValue::Method), + ValueType::Classification => { + return IanaType::from(token).map_or_text(ICalendarValue::Classification); + } + ValueType::Status => return IanaType::from(token).map_or_text(ICalendarValue::Status), + ValueType::Transparency => { + return IanaType::from(token).map_or_text(ICalendarValue::Transparency); + } + ValueType::Action => return IanaType::from(token).map_or_text(ICalendarValue::Action), + ValueType::BusyType => { + return IanaType::from(token).map_or_text(ICalendarValue::BusyType); + } + ValueType::ParticipantType => { + return IanaType::from(token).map_or_text(ICalendarValue::ParticipantType); + } + ValueType::ResourceType => { + return IanaType::from(token).map_or_text(ICalendarValue::ResourceType); + } + ValueType::Proximity => { + return IanaType::from(token).map_or_text(ICalendarValue::Proximity); + } + }; + match value_type { + ICalendarValueType::Date => token + .into_ical_date() + .map(ICalendarValue::PartialDateTime) + .unwrap_or_else(ICalendarValue::Text), + ICalendarValueType::DateTime => match token.into_timestamp(false) { + Ok(timestamp) => { + if !timestamp.has_time() { + self.data_type = Some(IanaType::Iana(ICalendarValueType::Date)); + } + ICalendarValue::PartialDateTime(timestamp) + } + Err(other) => ICalendarValue::Text(other), + }, + ICalendarValueType::Time => token + .into_ical_time() + .map(ICalendarValue::PartialDateTime) + .unwrap_or_else(ICalendarValue::Text), + ICalendarValueType::UtcOffset => token + .into_offset() + .map(ICalendarValue::PartialDateTime) + .unwrap_or_else(ICalendarValue::Text), + ICalendarValueType::Boolean => ICalendarValue::Boolean(token.into_boolean()), + ICalendarValueType::Float => token + .into_float() + .map(ICalendarValue::Float) + .unwrap_or_else(ICalendarValue::Text), + ICalendarValueType::Integer => token + .into_integer() + .map(ICalendarValue::Integer) + .unwrap_or_else(ICalendarValue::Text), + ICalendarValueType::Uri | ICalendarValueType::CalAddress => token + .into_uri_bytes() + .map(|data| ICalendarValue::Uri(Uri::from(data))) + .unwrap_or_else(|uri| ICalendarValue::Uri(Uri::Location(uri))), + ICalendarValueType::Duration => match ICalendarDuration::parse(token.text.as_ref()) { + Some(duration) => ICalendarValue::Duration(duration), + None => ICalendarValue::Text(token.into_string()), + }, + ICalendarValueType::Period => match ICalendarPeriod::parse(token.text.as_ref()) { + Some(period) => ICalendarValue::Period(Box::new(period)), + None => ICalendarValue::Text(token.into_string()), + }, + ICalendarValueType::Text + | ICalendarValueType::Binary + | ICalendarValueType::Unknown + | ICalendarValueType::XmlReference + | ICalendarValueType::Uid + | ICalendarValueType::Recur => ICalendarValue::Text(token.into_string()), + } + } + + fn push_known(&mut self, name: &ICalendarParameterName, token: Token<'_>) { + let value = match name { + ICalendarParameterName::Altrep + | ICalendarParameterName::DelegatedFrom + | ICalendarParameterName::DelegatedTo + | ICalendarParameterName::Dir + | ICalendarParameterName::Member + | ICalendarParameterName::SentBy + | ICalendarParameterName::Schema => ICalendarParameterValue::Uri(Uri::from(token)), + ICalendarParameterName::Rsvp | ICalendarParameterName::Derived => { + IanaType::::from(token).into() + } + ICalendarParameterName::Range => { + if token.text.as_ref().eq_ignore_ascii_case(b"THISANDFUTURE") { + ICalendarParameterValue::Bool(true) + } else { + return; + } + } + ICalendarParameterName::Size | ICalendarParameterName::Order => { + IanaType::::from(token).into() + } + ICalendarParameterName::Gap => { + IanaType::::from(token).into() + } + ICalendarParameterName::Cutype => { + IanaType::::from(token).into() + } + ICalendarParameterName::Fbtype => { + IanaType::::from(token).into() + } + ICalendarParameterName::Partstat => { + IanaType::::from(token).into() + } + ICalendarParameterName::Related => { + IanaType::::from(token).into() + } + ICalendarParameterName::Reltype => { + IanaType::::from(token).into() + } + ICalendarParameterName::Role => { + IanaType::::from(token).into() + } + ICalendarParameterName::ScheduleAgent => { + IanaType::::from(token).into() + } + ICalendarParameterName::ScheduleForceSend => { + IanaType::::from(token).into() + } + ICalendarParameterName::Value => { + self.data_type = Some(token.into()); + return; + } + ICalendarParameterName::Display => { + IanaType::::from(token).into() + } + ICalendarParameterName::Feature => { + IanaType::::from(token).into() + } + ICalendarParameterName::Linkrel => IanaType::::from(token).into(), + _ => ICalendarParameterValue::Text(token.into_string()), + }; + self.params + .push(ICalendarParameter::new(name.clone(), value)); + } +} + +trait IntoValue { + fn map_or_text(self, iana: impl FnOnce(I) -> ICalendarValue) -> ICalendarValue; +} + +impl IntoValue for IanaType { + fn map_or_text(self, iana: impl FnOnce(I) -> ICalendarValue) -> ICalendarValue { + match self { + IanaType::Iana(value) => iana(value), + IanaType::Other(value) => ICalendarValue::Text(value), + } + } +} + +enum ParamKind { + Known(ICalendarParameterName), + Charset, + Encoding, + Other(ICalendarParameterName), + Pending, +} + +impl ParamKind { + fn classify(token: Token<'_>, params: &mut Params) -> Self { + let name = token.text.as_ref(); + if let Some(name) = ICalendarParameterName::try_parse(name) { + return ParamKind::Known(name); + } + match name.first() { + None => ParamKind::Pending, + Some(b'c' | b'C') if name.eq_ignore_ascii_case(b"charset") => ParamKind::Charset, + Some(b'e' | b'E') if name.eq_ignore_ascii_case(b"encoding") => ParamKind::Encoding, + _ if params.encoding.is_none() && name.eq_ignore_ascii_case(b"base64") => { + params.encoding = Some(Encoding::Base64); + ParamKind::Pending + } + _ => ParamKind::Other(ICalendarParameterName::Other(token.into_string())), + } + } +} + +#[derive(Default)] +struct ParamValues { + count: usize, + last_other: Option, +} + +impl ParamValues { + fn push(&mut self, kind: &ParamKind, token: Token<'_>, params: &mut Params) { + self.count += 1; + match kind { + ParamKind::Known(name) => params.push_known(name, token), + ParamKind::Charset => params.charset = Some(token.into_string()), + ParamKind::Encoding => params.encoding = Encoding::parse(token.text.as_ref()), + ParamKind::Other(name) => { + if let Some(value) = self.last_other.replace(token.into_string()) { + params.params.push(ICalendarParameter::new( + name.clone(), + ICalendarParameterValue::Text(value), + )); + } + } + ParamKind::Pending => {} + } + } + + fn finish(self, kind: ParamKind, params: &mut Params) { + match kind { + ParamKind::Known(name) if self.count == 0 => { + params + .params + .push(ICalendarParameter::new(name, ICalendarParameterValue::Null)); + } + ParamKind::Other(name) => params.params.push(ICalendarParameter::new( + name, + self.last_other + .map_or(ICalendarParameterValue::Null, ICalendarParameterValue::Text), + )), + _ => {} + } + } +} + +impl Parser<'_> { + pub fn icalendar(&mut self, component_type: ICalendarComponentType) -> Entry { + let mut components = Vec::with_capacity(COMPONENT_CAPACITY); + components.push(ICalendarComponent { + component_type, + ..Default::default() + }); + let mut parents = ComponentStack::new(); + let mut current = 0; + let mut next_component_id: u32 = 1; + + loop { + self.expect_iana_token(); + let Some(name) = self.token() else { + break; + }; + let property = ICalendarProperty::parse(name.text.as_ref()); + + let mut params = Params { + params: Vec::new(), + stop_char: name.stop_char, + data_type: None, + encoding: None, + charset: None, + }; + + match params.stop_char { + StopChar::Semicolon => { + params.params = Vec::with_capacity( + property + .as_ref() + .map_or(0, ICalendarProperty::parameter_capacity), + ); + self.ical_parameters(&mut params); + } + StopChar::Colon => {} + StopChar::Lf => { + if name.text.is_empty() || !self.strict { + continue; + } else { + return Entry::InvalidLine(name.into_string()); + } + } + StopChar::Comma | StopChar::Equal | StopChar::Dot => { + params.stop_char = self.seek_value_or_eol(); + } + } + + let name = match property { + Some(ICalendarProperty::Begin) => { + if params.stop_char == StopChar::Colon + && let Some(component_type) = self.component_type() + { + if let Some(parent) = components.get_mut(current) { + parent.component_ids.push(next_component_id); + } + components.push(ICalendarComponent { + component_type, + ..Default::default() + }); + parents.push(current); + current = next_component_id as usize; + let Some(id) = next_component_id.checked_add(1) else { + return Entry::TooManyComponents; + }; + next_component_id = id; + continue; + } + + if !self.strict { + continue; + } else { + return Entry::InvalidLine("BEGIN".to_string()); + } + } + Some(ICalendarProperty::End) => { + if params.stop_char == StopChar::Colon + && let Some(component_type) = self.component_type() + { + let open_type = components + .get(current) + .map(|component| &component.component_type); + if open_type == Some(&component_type) || !self.strict { + if let Some(parent) = parents.pop() { + current = parent; + continue; + } else { + break; + } + } else { + return Entry::UnexpectedComponentEnd { + expected: open_type.cloned().unwrap_or_default(), + found: component_type, + }; + } + } + + if !self.strict { + continue; + } else { + return Entry::InvalidLine("END".to_string()); + } + } + Some(name) => name, + None => { + if !name.text.is_empty() { + ICalendarProperty::Other(name.into_string()) + } else { + if params.stop_char != StopChar::Lf { + self.seek_lf(); + } + continue; + } + } + }; + + let entry = self.ical_entry(name, params); + if let Some(component) = components.get_mut(current) { + if component.entries.capacity() == 0 { + self.presize( + &mut component.entries, + component.component_type.entry_capacity(), + ); + } + component.entries.push(entry); + } + } + + if !parents.is_empty() && self.strict { + return Entry::UnterminatedComponent( + components + .get(current) + .map_or("", |component| component.component_type.as_str()) + .to_string() + .into(), + ); + } + + ICalendar { components }.into_entry(self.strict) + } + + fn component_type(&mut self) -> Option { + self.expect_single_value(); + self.token().map(|token| { + ICalendarComponentType::parse(token.text.as_ref()) + .unwrap_or_else(|| ICalendarComponentType::Other(token.into_string())) + }) + } + + fn ical_entry(&mut self, name: ICalendarProperty, mut params: Params) -> ICalendarEntry { + let mut entry = ICalendarEntry { + name, + params: take(&mut params.params), + values: SmallVec::new(), + }; + + if params.stop_char != StopChar::Lf { + let (default_type, multi_value) = entry.name.default_types(); + match multi_value { + ValueSeparator::None => { + self.expect_single_value(); + } + ValueSeparator::Comma => { + self.expect_multi_value_comma(); + } + ValueSeparator::Semicolon => { + self.expect_multi_value_semicolon(); + } + } + + match params.encoding { + Some(Encoding::Base64) if multi_value != ValueSeparator::None => { + self.expect_single_value(); + } + Some(Encoding::QuotedPrintable) => { + self.mode.set(Mode::UNFOLD_QP, true); + } + _ => {} + } + + self.mode.set( + Mode::UNESCAPE_BACKSLASH, + !matches!( + params + .data_type + .as_ref() + .map(|data_type| data_type.iana().unwrap_or(&ICalendarValueType::Text)) + .or(match &default_type { + ValueType::Ical(default_type) => Some(default_type), + _ => None, + }), + Some(ICalendarValueType::Uri | ICalendarValueType::CalAddress) + ), + ); + + if matches!( + (¶ms.data_type, &default_type), + (Some(IanaType::Iana(ICalendarValueType::Recur)), _) + | (None, ValueType::Ical(ICalendarValueType::Recur)) + ) { + entry.values = smallvec![match self.rrule() { + Ok(rrule) => ICalendarValue::RecurrenceRule(Box::new(rrule)), + Err(other) => ICalendarValue::Text(other), + }]; + } else { + let mut payload = None; + while let Some(mut token) = self.value_token(params.encoding, &mut payload) { + let eol = token.stop_char == StopChar::Lf; + + if token.text.is_empty() + && (matches!(multi_value, ValueSeparator::None) + || matches!(entry.name, ICalendarProperty::Other(_)) + && entry.values.is_empty()) + { + if eol { + break; + } else { + continue; + } + } + + let value = if let Some(encoding) = params.encoding + && let Some(binary) = + params.decode_value(&mut token, payload.take(), encoding, &entry.name) + { + params.data_type = Some(IanaType::Iana(ICalendarValueType::Binary)); + ICalendarValue::Binary(binary) + } else { + params.value(token, &default_type) + }; + entry.values.push(value); + + if eol { + break; + } + } + } + } + + if let Some(data_type) = params.data_type { + entry.params.push(ICalendarParameter { + name: ICalendarParameterName::Value, + value: match data_type { + IanaType::Iana(value) => ICalendarParameterValue::Value(value), + IanaType::Other(value) => ICalendarParameterValue::Text(value), + }, + }); + } + + entry + } + + fn ical_parameters(&mut self, params: &mut Params) { + while params.stop_char == StopChar::Semicolon { + self.expect_iana_token(); + let Some(token) = self.token() else { + params.stop_char = StopChar::Lf; + break; + }; + params.stop_char = token.stop_char; + let kind = ParamKind::classify(token, params); + let buffered = matches!(kind, ParamKind::Pending) || !self.token_buf.is_empty(); + let mut values = ParamValues::default(); + + if !matches!( + params.stop_char, + StopChar::Lf | StopChar::Colon | StopChar::Semicolon + ) { + if params.stop_char != StopChar::Equal { + params.stop_char = self.seek_param_value_or_eol(); + } + if params.stop_char == StopChar::Equal { + self.expect_param_value(); + while !matches!( + params.stop_char, + StopChar::Lf | StopChar::Colon | StopChar::Semicolon + ) { + match self.token() { + Some(token) => { + params.stop_char = token.stop_char; + if buffered { + self.token_buf.push(token); + } else { + values.push(&kind, token, params); + } + } + None => { + params.stop_char = StopChar::Lf; + break; + } + } + } + } + } + + if buffered { + self.buffered_param(kind, params); + } else { + values.finish(kind, params); + } + } + } + + fn buffered_param(&mut self, kind: ParamKind, params: &mut Params) { + match kind { + ParamKind::Known(name) => { + if self.token_buf.is_empty() { + params + .params + .push(ICalendarParameter::new(name, ICalendarParameterValue::Null)); + } else { + for token in self.token_buf.drain(..) { + params.push_known(&name, token); + } + } + } + ParamKind::Charset => { + for token in self.token_buf.drain(..) { + params.charset = token.into_string().into(); + } + } + ParamKind::Encoding => { + for token in self.token_buf.drain(..) { + params.encoding = Encoding::parse(token.text.as_ref()); + } + } + ParamKind::Other(name) => { + if self.token_buf.is_empty() { + params + .params + .push(ICalendarParameter::new(name, ICalendarParameterValue::Null)); + } else { + params.params.extend(self.token_buf.drain(..).map(|token| { + ICalendarParameter::new( + name.clone(), + ICalendarParameterValue::Text(token.into_string()), + ) + })); + } + } + ParamKind::Pending => {} + } + } + + pub(crate) fn rrule(&mut self) -> Result { + self.expect_rrule_value(); + + let mut last_stop_char = StopChar::Equal; + let mut is_valid = true; + let mut has_freq = false; + let mut rrule = ICalendarRecurrenceRule::default(); + + let mut token_start = usize::MAX; + + while let Some(mut token) = self.token_until_lf(&mut last_stop_char) { + if token_start == usize::MAX && !token.text.is_empty() { + token_start = token.start; + } + if !is_valid { + continue; + } + if token.text.is_empty() && token.stop_char != StopChar::Equal { + continue; + } + if token.stop_char != StopChar::Equal { + if !self.strict { + // Ignore unknown tokens + while let Some(token_) = self.token_until_lf(&mut last_stop_char) { + if token_.stop_char == StopChar::Equal { + token = token_; + break; + } + } + if token.stop_char != StopChar::Equal { + is_valid = false; + continue; + } + } else { + is_valid = false; + continue; + } + } + + hashify::fnc_map_ignore_case!(token.text.as_ref(), + b"FREQ" => { + while let Some(value) = self.parse_value_until_lf(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.freq = value; + has_freq = true; + } else if !self.strict { + is_valid = false; + } + } + }, + b"UNTIL" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.until = Some(value.0); + } else if !self.strict { + is_valid = false; + } + } + }, + b"COUNT" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + let count = value.0.unsigned_abs() as u32; + if count > 0 { + rrule.count = Some(count); + } + } else if !self.strict { + is_valid = false; + } + } + }, + b"INTERVAL" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + let interval = value.0.unsigned_abs() as u16; + if interval > 0 { + rrule.interval = Some(interval); + } + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYSECOND" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.bysecond.push(value.0.unsigned_abs() as u8); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYMINUTE" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.byminute.push(value.0.unsigned_abs() as u8); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYHOUR" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.byhour.push(value.0.unsigned_abs() as u8); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYDAY" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.byday.push(value); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYMONTHDAY" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.bymonthday.push(value.0 as i8); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYYEARDAY" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.byyearday.push(value.0 as i16); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYWEEKNO" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.byweekno.push(value.0 as i8); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYMONTH" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.bymonth.push(value); + } else if !self.strict { + is_valid = false; + } + } + }, + b"BYSETPOS" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.bysetpos.push(value.0 as i32); + } else if !self.strict { + is_valid = false; + } + } + }, + b"WKST" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.wkst = Some(value); + } else if !self.strict { + is_valid = false; + } + } + }, + b"RSCALE" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.rscale = Some(value); + } else if !self.strict { + is_valid = false; + } + } + }, + b"SKIP" => { + while let Some(value) = self.parse_value_until_lf::(StopChar::Semicolon, &mut last_stop_char) { if let Some(value)= value { + rrule.skip = Some(value); + } else if !self.strict { + is_valid = false; + } + } + }, + _ => { + if !self.strict { + // Ignore unknown tokens + while let Some(token) = self.token_until_lf(&mut last_stop_char) { if token.stop_char == StopChar::Semicolon{ + break; + } + } + } else { + is_valid = false; + } + } + ); + } + + if has_freq && is_valid { + Ok(rrule) + } else if token_start != usize::MAX && self.last_token_end >= token_start { + Err(self + .input + .get(token_start..=self.last_token_end) + .map(|slice| { + let mut buf = Vec::with_capacity(slice.len()); + let mut iter = slice.iter().peekable(); + + while let Some(ch) = iter.next() { + match ch { + b'\r' => {} + b'\n' => { + iter.next_if(|ch| matches!(ch, b' ' | b'\t')); + } + _ => buf.push(*ch), + } + } + + String::from_utf8(buf) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) + }) + .unwrap_or_default()) + } else { + Err("".to_string()) + } + } +} + +impl Token<'_> { + pub(crate) fn into_ical_date(self) -> std::result::Result { + if let Some(dt) = self + .text + .first_chunk::<8>() + .and_then(PartialDateTime::from_basic_date) + { + return Ok(dt); + } + let mut dt = PartialDateTime::default(); + if dt.parse_ical_date(&mut self.text.iter().peekable()) { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_ical_time(self) -> std::result::Result { + let mut dt = PartialDateTime::default(); + if dt.parse_ical_time(&mut self.text.iter().peekable()) { + Ok(dt) + } else { + Err(self.into_string()) + } + } +} + +impl PartialDateTime { + pub fn parse_ical_date(&mut self, iter: &mut Peekable>) -> bool { + parse_digits(iter, &mut self.year, 4, false) + && parse_small_digits(iter, &mut self.month, 2, false) + && parse_small_digits(iter, &mut self.day, 2, false) + } + + pub fn parse_ical_time(&mut self, iter: &mut Peekable>) -> bool { + if parse_small_digits(iter, &mut self.hour, 2, false) + && parse_small_digits(iter, &mut self.minute, 2, false) + && parse_small_digits(iter, &mut self.second, 2, false) + { + self.parse_zone(iter); + true + } else { + false + } + } +} + +struct ICalendarDateOrTime(PartialDateTime); + +impl IanaParse for ICalendarDateOrTime { + fn parse(value: &[u8]) -> Option { + match PartialDateTime::from_timestamp_bytes(value, false) { + (dt, true) => Some(ICalendarDateOrTime(dt)), + (_, false) => None, + } + } +} + +impl IanaParse for ICalendarPeriod { + fn parse(value: &[u8]) -> Option { + let mut iter = value.iter().peekable(); + let mut start = PartialDateTime::default(); + if start.parse_timestamp(&mut iter, true) { + if let Some(duration) = ICalendarDuration::try_parse(&mut iter) { + Some(ICalendarPeriod::Duration { start, duration }) + } else { + let mut end = PartialDateTime::default(); + if end.parse_timestamp(&mut iter, true) { + Some(ICalendarPeriod::Range { start, end }) + } else { + None + } + } + } else { + None + } + } +} + +impl IanaParse for ICalendarMonth { + fn parse(value: &[u8]) -> Option { + let mut num: i8 = 0; + let mut is_leap = false; + + for (pos, ch) in value.iter().enumerate() { + match ch { + b'L' | b'l' if pos > 0 => { + is_leap = true; + } + b'0'..=b'9' => { + num = num.saturating_mul(10).saturating_add((*ch - b'0') as i8); + } + _ => { + if !ch.is_ascii_whitespace() { + return None; + } + } + } + } + + Some(ICalendarMonth(if is_leap { -num } else { num })) + } +} + +impl ICalendarMonth { + pub fn new(month: u8, is_leap: bool) -> Self { + let month = i8::try_from(month).unwrap_or(i8::MAX); + ICalendarMonth(if is_leap { -month } else { month }) + } + + pub fn is_leap(&self) -> bool { + self.0 < 0 + } + + pub fn month(&self) -> u8 { + self.0.unsigned_abs() + } +} + +impl From for u8 { + fn from(value: ICalendarMonth) -> Self { + value.month() + } +} + +impl ICalendarDuration { + fn try_parse(iter: &mut Peekable>) -> Option { + let mut dur = ICalendarDuration::default(); + loop { + match iter.peek() { + Some(b'P' | b'p') => { + iter.next(); + break; + } + Some(b'+') => { + iter.next(); + } + Some(b'-') if !dur.neg => { + iter.next(); + dur.neg = true; + } + Some(b' ' | b'\t') => { + iter.next(); + } + _ => { + return None; + } + } + } + + let mut num: u32 = 0; + let mut saw_component = false; + for ch in iter { + match ch { + b'0'..=b'9' => { + num = num.saturating_mul(10).saturating_add((ch - b'0') as u32); + } + b'T' | b't' => {} + b'W' | b'w' => { + dur.weeks = num; + num = 0; + saw_component = true; + } + b'D' | b'd' => { + dur.days = num; + num = 0; + saw_component = true; + } + b'H' | b'h' => { + dur.hours = num; + num = 0; + saw_component = true; + } + b'M' | b'm' => { + dur.minutes = num; + num = 0; + saw_component = true; + } + b'S' | b's' => { + dur.seconds = num; + num = 0; + saw_component = true; + } + _ => { + if !ch.is_ascii_whitespace() { + return None; + } + } + } + } + + if saw_component { Some(dur) } else { None } + } +} + +impl IanaParse for ICalendarDuration { + fn parse(value: &[u8]) -> Option { + ICalendarDuration::try_parse(&mut value.iter().peekable()) + } +} + +impl From> for Uri { + fn from(token: Token<'_>) -> Self { + token + .into_uri_bytes() + .map(Uri::from) + .unwrap_or_else(Uri::Location) + } +} + +impl IanaParse for ICalendarDay { + fn parse(value: &[u8]) -> Option { + let mut iter = value.iter().enumerate(); + let mut is_negative = false; + let mut has_ordwk = false; + let mut ordwk: i16 = 0; + + loop { + let (pos, ch) = iter.next()?; + + match ch { + b'0'..=b'9' => { + ordwk = ordwk.saturating_mul(10).saturating_add((ch - b'0') as i16); + has_ordwk = true; + } + b'-' if pos == 0 => { + is_negative = true; + } + b'+' if pos == 0 => {} + b'A'..=b'Z' | b'a'..=b'z' => { + return ICalendarWeekday::parse(value.get(pos..).unwrap_or_default()).map( + |weekday| ICalendarDay { + ordwk: has_ordwk.then_some(if is_negative { -ordwk } else { ordwk }), + weekday, + }, + ); + } + _ => return None, + } + } + } +} + +impl Uri { + pub fn parse(value: impl Into) -> Self { + let uri = value.into(); + Data::try_parse(uri.as_bytes()) + .map(Uri::from) + .unwrap_or_else(|| Uri::Location(uri)) + } +} + diff --git a/crates/calcard/src/icalendar/rkyv_timezone.rs b/crates/calcard/src/icalendar/rkyv_timezone.rs new file mode 100644 index 00000000..80e5656f --- /dev/null +++ b/crates/calcard/src/icalendar/rkyv_timezone.rs @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarComponentType, + ArchivedICalendarEntry, ArchivedICalendarProperty, ArchivedICalendarValue, +}; +use crate::{common::timezone::Tz, icalendar::ICalendarParameterName}; +use std::str::FromStr; + +impl ArchivedICalendar { + pub fn timezones(&self) -> impl Iterator { + self.components.iter().filter(|comp| { + matches!( + comp.component_type, + ArchivedICalendarComponentType::VTimezone + ) + }) + } + + pub fn is_timezone(&self) -> bool { + self.timezones().count() == 1 + } +} + +impl ArchivedICalendarComponent { + pub fn timezone(&self) -> Option<(&str, Tz)> { + let mut tz_name = None; + let mut tz_id = None; + let mut tz_lic = None; + let mut tz_cdo_id = None; + + for entry in self.entries.iter() { + match (&entry.name, entry.values.first()) { + (ArchivedICalendarProperty::Tzid, Some(ArchivedICalendarValue::Text(id))) => { + tz_id = Tz::from_str(id.as_str()).ok(); + tz_name = Some(id.as_str()); + } + ( + ArchivedICalendarProperty::Other(value), + Some(ArchivedICalendarValue::Text(id)), + ) => { + hashify::fnc_map!(value.as_bytes(), + "X-LIC-LOCATION" => { + tz_lic = Tz::from_str(id.strip_prefix("SystemV/").unwrap_or(id.as_str())).ok(); + }, + "X-MICROSOFT-CDO-TZID" => { + tz_cdo_id = Tz::from_ms_cdo_zone_id(id); + }, + _ => {} + ); + } + _ => (), + } + } + + tz_name.zip(tz_id.or(tz_lic).or(tz_cdo_id)) + } +} + +impl ArchivedICalendarEntry { + pub fn tz_id(&self) -> Option<&str> { + self.parameters(&ICalendarParameterName::Tzid) + .find_map(|param| param.as_text()) + } +} diff --git a/crates/calcard/src/icalendar/rkyv_types.rs b/crates/calcard/src/icalendar/rkyv_types.rs new file mode 100644 index 00000000..38fe6c2b --- /dev/null +++ b/crates/calcard/src/icalendar/rkyv_types.rs @@ -0,0 +1,871 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::ArchivedLinkRelation; + +use super::*; + +impl ArchivedICalendarProperty { + // Returns the default value type and whether the property is multi-valued. + pub(crate) fn default_types(&self) -> (ArchivedValueType, ValueSeparator) { + match self { + ArchivedICalendarProperty::Calscale => { + (ArchivedValueType::CalendarScale, ValueSeparator::None) + } + ArchivedICalendarProperty::Method => (ArchivedValueType::Method, ValueSeparator::None), + ArchivedICalendarProperty::Prodid => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Version => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Attach => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Categories => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::Class => { + (ArchivedValueType::Classification, ValueSeparator::None) + } + ArchivedICalendarProperty::Comment => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Description => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Geo => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Float), + ValueSeparator::Semicolon, + ), + ArchivedICalendarProperty::Location => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::PercentComplete => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Integer), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Priority => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Integer), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Resources => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::Status => (ArchivedValueType::Status, ValueSeparator::None), + ArchivedICalendarProperty::Summary => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Completed => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Dtend => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Due => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Dtstart => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Duration => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Duration), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Freebusy => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Period), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::Transp => { + (ArchivedValueType::Transparency, ValueSeparator::None) + } + ArchivedICalendarProperty::Tzid => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Tzname => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Tzoffsetfrom => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::UtcOffset), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Tzoffsetto => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::UtcOffset), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Tzurl => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Attendee => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Contact => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Organizer => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::RecurrenceId => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::RelatedTo => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Url => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Uid => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Exdate => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::Exrule => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Recur), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Rdate => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::Rrule => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Recur), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Action => (ArchivedValueType::Action, ValueSeparator::None), + ArchivedICalendarProperty::Repeat => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Integer), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Trigger => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Duration), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Created => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Dtstamp => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::LastModified => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Sequence => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Integer), + ValueSeparator::None, + ), + ArchivedICalendarProperty::RequestStatus => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::Semicolon, + ), + ArchivedICalendarProperty::Xml => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Tzuntil => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::TzidAliasOf => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Busytype => { + (ArchivedValueType::BusyType, ValueSeparator::None) + } + ArchivedICalendarProperty::Name => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::RefreshInterval => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Duration), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Source => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Color => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Image => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Conference => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::CalendarAddress => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::LocationType => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::Comma, + ), + ArchivedICalendarProperty::ParticipantType => { + (ArchivedValueType::ParticipantType, ValueSeparator::None) + } + ArchivedICalendarProperty::ResourceType => { + (ArchivedValueType::ResourceType, ValueSeparator::None) + } + ArchivedICalendarProperty::StructuredData => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::StyledDescription => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Acknowledged => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::DateTime), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Proximity => { + (ArchivedValueType::Proximity, ValueSeparator::None) + } + ArchivedICalendarProperty::Concept => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Link => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Refid => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Begin => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::End => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Other(_) => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::Semicolon, + ), + ArchivedICalendarProperty::Coordinates => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::ShowWithoutTime => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Boolean), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Jsid | ArchivedICalendarProperty::Jsprop => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::EstimatedDuration => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Duration), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Reason => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Uri), + ValueSeparator::None, + ), + ArchivedICalendarProperty::Substate => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + ArchivedICalendarProperty::TaskMode => ( + ArchivedValueType::Ical(ArchivedICalendarValueType::Text), + ValueSeparator::None, + ), + } + } +} + +impl IanaString for ArchivedICalendarFrequency { + fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarFrequency::Secondly => "SECONDLY", + ArchivedICalendarFrequency::Minutely => "MINUTELY", + ArchivedICalendarFrequency::Hourly => "HOURLY", + ArchivedICalendarFrequency::Daily => "DAILY", + ArchivedICalendarFrequency::Weekly => "WEEKLY", + ArchivedICalendarFrequency::Monthly => "MONTHLY", + ArchivedICalendarFrequency::Yearly => "YEARLY", + } + } +} + +impl ArchivedICalendarSkip { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarSkip::Omit => "OMIT", + ArchivedICalendarSkip::Backward => "BACKWARD", + ArchivedICalendarSkip::Forward => "FORWARD", + } + } +} + +impl ArchivedICalendarWeekday { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarWeekday::Sunday => "SU", + ArchivedICalendarWeekday::Monday => "MO", + ArchivedICalendarWeekday::Tuesday => "TU", + ArchivedICalendarWeekday::Wednesday => "WE", + ArchivedICalendarWeekday::Thursday => "TH", + ArchivedICalendarWeekday::Friday => "FR", + ArchivedICalendarWeekday::Saturday => "SA", + } + } +} + +impl ArchivedICalendarAction { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarAction::Audio => "AUDIO", + ArchivedICalendarAction::Display => "DISPLAY", + ArchivedICalendarAction::Email => "EMAIL", + ArchivedICalendarAction::Procedure => "PROCEDURE", + } + } +} + +impl ArchivedICalendarUserTypes { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarUserTypes::Individual => "INDIVIDUAL", + ArchivedICalendarUserTypes::Group => "GROUP", + ArchivedICalendarUserTypes::Resource => "RESOURCE", + ArchivedICalendarUserTypes::Room => "ROOM", + ArchivedICalendarUserTypes::Unknown => "UNKNOWN", + } + } +} + +impl ArchivedICalendarClassification { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarClassification::Public => "PUBLIC", + ArchivedICalendarClassification::Private => "PRIVATE", + ArchivedICalendarClassification::Confidential => "CONFIDENTIAL", + } + } +} + +impl ArchivedICalendarComponentType { + pub fn as_str(&self) -> &str { + match self { + ArchivedICalendarComponentType::VCalendar => "VCALENDAR", + ArchivedICalendarComponentType::VEvent => "VEVENT", + ArchivedICalendarComponentType::VTodo => "VTODO", + ArchivedICalendarComponentType::VJournal => "VJOURNAL", + ArchivedICalendarComponentType::VFreebusy => "VFREEBUSY", + ArchivedICalendarComponentType::VTimezone => "VTIMEZONE", + ArchivedICalendarComponentType::VAlarm => "VALARM", + ArchivedICalendarComponentType::Standard => "STANDARD", + ArchivedICalendarComponentType::Daylight => "DAYLIGHT", + ArchivedICalendarComponentType::VAvailability => "VAVAILABILITY", + ArchivedICalendarComponentType::Available => "AVAILABLE", + ArchivedICalendarComponentType::Participant => "PARTICIPANT", + ArchivedICalendarComponentType::VLocation => "VLOCATION", + ArchivedICalendarComponentType::VResource => "VRESOURCE", + ArchivedICalendarComponentType::VStatus => "VSTATUS", + ArchivedICalendarComponentType::Other(name) => name.as_str(), + } + } +} + +impl ArchivedICalendarDisplayType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarDisplayType::Badge => "BADGE", + ArchivedICalendarDisplayType::Graphic => "GRAPHIC", + ArchivedICalendarDisplayType::Fullsize => "FULLSIZE", + ArchivedICalendarDisplayType::Thumbnail => "THUMBNAIL", + } + } +} + +impl ArchivedICalendarFeatureType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarFeatureType::Audio => "AUDIO", + ArchivedICalendarFeatureType::Chat => "CHAT", + ArchivedICalendarFeatureType::Feed => "FEED", + ArchivedICalendarFeatureType::Moderator => "MODERATOR", + ArchivedICalendarFeatureType::Phone => "PHONE", + ArchivedICalendarFeatureType::Screen => "SCREEN", + ArchivedICalendarFeatureType::Video => "VIDEO", + } + } +} + +impl ArchivedICalendarFreeBusyType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarFreeBusyType::Free => "FREE", + ArchivedICalendarFreeBusyType::Busy => "BUSY", + ArchivedICalendarFreeBusyType::BusyUnavailable => "BUSY-UNAVAILABLE", + ArchivedICalendarFreeBusyType::BusyTentative => "BUSY-TENTATIVE", + } + } +} + +impl ArchivedICalendarMethod { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarMethod::Publish => "PUBLISH", + ArchivedICalendarMethod::Request => "REQUEST", + ArchivedICalendarMethod::Reply => "REPLY", + ArchivedICalendarMethod::Add => "ADD", + ArchivedICalendarMethod::Cancel => "CANCEL", + ArchivedICalendarMethod::Refresh => "REFRESH", + ArchivedICalendarMethod::Counter => "COUNTER", + ArchivedICalendarMethod::Declinecounter => "DECLINECOUNTER", + } + } +} + +impl ArchivedICalendarParticipantType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarParticipantType::Active => "ACTIVE", + ArchivedICalendarParticipantType::Inactive => "INACTIVE", + ArchivedICalendarParticipantType::Sponsor => "SPONSOR", + ArchivedICalendarParticipantType::Contact => "CONTACT", + ArchivedICalendarParticipantType::BookingContact => "BOOKING-CONTACT", + ArchivedICalendarParticipantType::EmergencyContact => "EMERGENCY-CONTACT", + ArchivedICalendarParticipantType::PublicityContact => "PUBLICITY-CONTACT", + ArchivedICalendarParticipantType::PlannerContact => "PLANNER-CONTACT", + ArchivedICalendarParticipantType::Performer => "PERFORMER", + ArchivedICalendarParticipantType::Speaker => "SPEAKER", + } + } +} + +impl ArchivedICalendarParticipationRole { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarParticipationRole::Chair => "CHAIR", + ArchivedICalendarParticipationRole::ReqParticipant => "REQ-PARTICIPANT", + ArchivedICalendarParticipationRole::OptParticipant => "OPT-PARTICIPANT", + ArchivedICalendarParticipationRole::NonParticipant => "NON-PARTICIPANT", + ArchivedICalendarParticipationRole::Owner => "OWNER", + } + } +} + +impl ArchivedICalendarParticipationStatus { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarParticipationStatus::NeedsAction => "NEEDS-ACTION", + ArchivedICalendarParticipationStatus::Accepted => "ACCEPTED", + ArchivedICalendarParticipationStatus::Declined => "DECLINED", + ArchivedICalendarParticipationStatus::Tentative => "TENTATIVE", + ArchivedICalendarParticipationStatus::Delegated => "DELEGATED", + ArchivedICalendarParticipationStatus::Completed => "COMPLETED", + ArchivedICalendarParticipationStatus::InProcess => "IN-PROCESS", + ArchivedICalendarParticipationStatus::Failed => "FAILED", + } + } +} + +impl ArchivedICalendarStatus { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarStatus::Tentative => "TENTATIVE", + ArchivedICalendarStatus::Confirmed => "CONFIRMED", + ArchivedICalendarStatus::Cancelled => "CANCELLED", + ArchivedICalendarStatus::NeedsAction => "NEEDS-ACTION", + ArchivedICalendarStatus::Completed => "COMPLETED", + ArchivedICalendarStatus::InProcess => "IN-PROCESS", + ArchivedICalendarStatus::Draft => "DRAFT", + ArchivedICalendarStatus::Final => "FINAL", + ArchivedICalendarStatus::Failed => "FAILED", + ArchivedICalendarStatus::Pending => "PENDING", + } + } +} + +impl ArchivedICalendarProperty { + pub fn as_str(&self) -> &str { + match self { + ArchivedICalendarProperty::Calscale => "CALSCALE", + ArchivedICalendarProperty::Method => "METHOD", + ArchivedICalendarProperty::Prodid => "PRODID", + ArchivedICalendarProperty::Version => "VERSION", + ArchivedICalendarProperty::Attach => "ATTACH", + ArchivedICalendarProperty::Categories => "CATEGORIES", + ArchivedICalendarProperty::Class => "CLASS", + ArchivedICalendarProperty::Comment => "COMMENT", + ArchivedICalendarProperty::Description => "DESCRIPTION", + ArchivedICalendarProperty::Geo => "GEO", + ArchivedICalendarProperty::Location => "LOCATION", + ArchivedICalendarProperty::PercentComplete => "PERCENT-COMPLETE", + ArchivedICalendarProperty::Priority => "PRIORITY", + ArchivedICalendarProperty::Resources => "RESOURCES", + ArchivedICalendarProperty::Status => "STATUS", + ArchivedICalendarProperty::Summary => "SUMMARY", + ArchivedICalendarProperty::Completed => "COMPLETED", + ArchivedICalendarProperty::Dtend => "DTEND", + ArchivedICalendarProperty::Due => "DUE", + ArchivedICalendarProperty::Dtstart => "DTSTART", + ArchivedICalendarProperty::Duration => "DURATION", + ArchivedICalendarProperty::Freebusy => "FREEBUSY", + ArchivedICalendarProperty::Transp => "TRANSP", + ArchivedICalendarProperty::Tzid => "TZID", + ArchivedICalendarProperty::Tzname => "TZNAME", + ArchivedICalendarProperty::Tzoffsetfrom => "TZOFFSETFROM", + ArchivedICalendarProperty::Tzoffsetto => "TZOFFSETTO", + ArchivedICalendarProperty::Tzurl => "TZURL", + ArchivedICalendarProperty::Attendee => "ATTENDEE", + ArchivedICalendarProperty::Contact => "CONTACT", + ArchivedICalendarProperty::Organizer => "ORGANIZER", + ArchivedICalendarProperty::RecurrenceId => "RECURRENCE-ID", + ArchivedICalendarProperty::RelatedTo => "RELATED-TO", + ArchivedICalendarProperty::Url => "URL", + ArchivedICalendarProperty::Uid => "UID", + ArchivedICalendarProperty::Exdate => "EXDATE", + ArchivedICalendarProperty::Exrule => "EXRULE", + ArchivedICalendarProperty::Rdate => "RDATE", + ArchivedICalendarProperty::Rrule => "RRULE", + ArchivedICalendarProperty::Action => "ACTION", + ArchivedICalendarProperty::Repeat => "REPEAT", + ArchivedICalendarProperty::Trigger => "TRIGGER", + ArchivedICalendarProperty::Created => "CREATED", + ArchivedICalendarProperty::Dtstamp => "DTSTAMP", + ArchivedICalendarProperty::LastModified => "LAST-MODIFIED", + ArchivedICalendarProperty::Sequence => "SEQUENCE", + ArchivedICalendarProperty::RequestStatus => "REQUEST-STATUS", + ArchivedICalendarProperty::Xml => "XML", + ArchivedICalendarProperty::Tzuntil => "TZUNTIL", + ArchivedICalendarProperty::TzidAliasOf => "TZID-ALIAS-OF", + ArchivedICalendarProperty::Busytype => "BUSYTYPE", + ArchivedICalendarProperty::Name => "NAME", + ArchivedICalendarProperty::RefreshInterval => "REFRESH-INTERVAL", + ArchivedICalendarProperty::Source => "SOURCE", + ArchivedICalendarProperty::Color => "COLOR", + ArchivedICalendarProperty::Image => "IMAGE", + ArchivedICalendarProperty::Conference => "CONFERENCE", + ArchivedICalendarProperty::CalendarAddress => "CALENDAR-ADDRESS", + ArchivedICalendarProperty::LocationType => "LOCATION-TYPE", + ArchivedICalendarProperty::ParticipantType => "PARTICIPANT-TYPE", + ArchivedICalendarProperty::ResourceType => "RESOURCE-TYPE", + ArchivedICalendarProperty::StructuredData => "STRUCTURED-DATA", + ArchivedICalendarProperty::StyledDescription => "STYLED-DESCRIPTION", + ArchivedICalendarProperty::Acknowledged => "ACKNOWLEDGED", + ArchivedICalendarProperty::Proximity => "PROXIMITY", + ArchivedICalendarProperty::Concept => "CONCEPT", + ArchivedICalendarProperty::Link => "LINK", + ArchivedICalendarProperty::Refid => "REFID", + ArchivedICalendarProperty::Begin => "BEGIN", + ArchivedICalendarProperty::End => "END", + ArchivedICalendarProperty::Coordinates => "COORDINATES", + ArchivedICalendarProperty::ShowWithoutTime => "SHOW-WITHOUT-TIME", + ArchivedICalendarProperty::Jsid => "JSID", + ArchivedICalendarProperty::Jsprop => "JSPROP", + ArchivedICalendarProperty::EstimatedDuration => "ESTIMATED-DURATION", + ArchivedICalendarProperty::Reason => "REASON", + ArchivedICalendarProperty::Substate => "SUBSTATE", + ArchivedICalendarProperty::TaskMode => "TASK-MODE", + ArchivedICalendarProperty::Other(value) => value.as_str(), + } + } +} + +impl ArchivedICalendarProximityValue { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarProximityValue::Arrive => "ARRIVE", + ArchivedICalendarProximityValue::Depart => "DEPART", + ArchivedICalendarProximityValue::Connect => "CONNECT", + ArchivedICalendarProximityValue::Disconnect => "DISCONNECT", + } + } +} + +impl ArchivedICalendarRelationshipType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarRelationshipType::Child => "CHILD", + ArchivedICalendarRelationshipType::Parent => "PARENT", + ArchivedICalendarRelationshipType::Sibling => "SIBLING", + ArchivedICalendarRelationshipType::Snooze => "SNOOZE", + ArchivedICalendarRelationshipType::Concept => "CONCEPT", + ArchivedICalendarRelationshipType::DependsOn => "DEPENDS-ON", + ArchivedICalendarRelationshipType::Finishtofinish => "FINISHTOFINISH", + ArchivedICalendarRelationshipType::Finishtostart => "FINISHTOSTART", + ArchivedICalendarRelationshipType::First => "FIRST", + ArchivedICalendarRelationshipType::Next => "NEXT", + ArchivedICalendarRelationshipType::Refid => "REFID", + ArchivedICalendarRelationshipType::Starttofinish => "STARTTOFINISH", + ArchivedICalendarRelationshipType::Starttostart => "STARTTOSTART", + } + } +} + +impl ArchivedICalendarResourceType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarResourceType::Projector => "PROJECTOR", + ArchivedICalendarResourceType::Room => "ROOM", + ArchivedICalendarResourceType::RemoteConferenceAudio => "REMOTE-CONFERENCE-AUDIO", + ArchivedICalendarResourceType::RemoteConferenceVideo => "REMOTE-CONFERENCE-VIDEO", + } + } +} + +impl ArchivedICalendarScheduleAgentValue { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarScheduleAgentValue::Server => "SERVER", + ArchivedICalendarScheduleAgentValue::Client => "CLIENT", + ArchivedICalendarScheduleAgentValue::None => "NONE", + } + } +} + +impl ArchivedICalendarScheduleForceSendValue { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarScheduleForceSendValue::Request => "REQUEST", + ArchivedICalendarScheduleForceSendValue::Reply => "REPLY", + } + } +} + +impl ArchivedICalendarValueType { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarValueType::Binary => "BINARY", + ArchivedICalendarValueType::Boolean => "BOOLEAN", + ArchivedICalendarValueType::CalAddress => "CAL-ADDRESS", + ArchivedICalendarValueType::Date => "DATE", + ArchivedICalendarValueType::DateTime => "DATE-TIME", + ArchivedICalendarValueType::Duration => "DURATION", + ArchivedICalendarValueType::Float => "FLOAT", + ArchivedICalendarValueType::Integer => "INTEGER", + ArchivedICalendarValueType::Period => "PERIOD", + ArchivedICalendarValueType::Recur => "RECUR", + ArchivedICalendarValueType::Text => "TEXT", + ArchivedICalendarValueType::Time => "TIME", + ArchivedICalendarValueType::Unknown => "UNKNOWN", + ArchivedICalendarValueType::Uri => "URI", + ArchivedICalendarValueType::UtcOffset => "UTC-OFFSET", + ArchivedICalendarValueType::XmlReference => "XML-REFERENCE", + ArchivedICalendarValueType::Uid => "UID", + } + } +} + +impl ArchivedICalendarTransparency { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarTransparency::Opaque => "OPAQUE", + ArchivedICalendarTransparency::Transparent => "TRANSPARENT", + } + } +} + +impl ArchivedLinkRelation { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedLinkRelation::About => "about", + ArchivedLinkRelation::Acl => "acl", + ArchivedLinkRelation::Alternate => "alternate", + ArchivedLinkRelation::Amphtml => "amphtml", + ArchivedLinkRelation::ApiCatalog => "api-catalog", + ArchivedLinkRelation::Appendix => "appendix", + ArchivedLinkRelation::AppleTouchIcon => "apple-touch-icon", + ArchivedLinkRelation::AppleTouchStartupImage => "apple-touch-startup-image", + ArchivedLinkRelation::Archives => "archives", + ArchivedLinkRelation::Author => "author", + ArchivedLinkRelation::BlockedBy => "blocked-by", + ArchivedLinkRelation::Bookmark => "bookmark", + ArchivedLinkRelation::C2paManifest => "c2pa-manifest", + ArchivedLinkRelation::Canonical => "canonical", + ArchivedLinkRelation::Chapter => "chapter", + ArchivedLinkRelation::CiteAs => "cite-as", + ArchivedLinkRelation::Collection => "collection", + ArchivedLinkRelation::CompressionDictionary => "compression-dictionary", + ArchivedLinkRelation::Contents => "contents", + ArchivedLinkRelation::Convertedfrom => "convertedfrom", + ArchivedLinkRelation::Copyright => "copyright", + ArchivedLinkRelation::CreateForm => "create-form", + ArchivedLinkRelation::Current => "current", + ArchivedLinkRelation::Deprecation => "deprecation", + ArchivedLinkRelation::Describedby => "describedby", + ArchivedLinkRelation::Describes => "describes", + ArchivedLinkRelation::Disclosure => "disclosure", + ArchivedLinkRelation::DnsPrefetch => "dns-prefetch", + ArchivedLinkRelation::Duplicate => "duplicate", + ArchivedLinkRelation::Edit => "edit", + ArchivedLinkRelation::EditForm => "edit-form", + ArchivedLinkRelation::EditMedia => "edit-media", + ArchivedLinkRelation::Enclosure => "enclosure", + ArchivedLinkRelation::External => "external", + ArchivedLinkRelation::First => "first", + ArchivedLinkRelation::Geofeed => "geofeed", + ArchivedLinkRelation::Glossary => "glossary", + ArchivedLinkRelation::Help => "help", + ArchivedLinkRelation::Hosts => "hosts", + ArchivedLinkRelation::Hub => "hub", + ArchivedLinkRelation::IceServer => "ice-server", + ArchivedLinkRelation::Icon => "icon", + ArchivedLinkRelation::Index => "index", + ArchivedLinkRelation::Intervalafter => "intervalafter", + ArchivedLinkRelation::Intervalbefore => "intervalbefore", + ArchivedLinkRelation::Intervalcontains => "intervalcontains", + ArchivedLinkRelation::Intervaldisjoint => "intervaldisjoint", + ArchivedLinkRelation::Intervalduring => "intervalduring", + ArchivedLinkRelation::Intervalequals => "intervalequals", + ArchivedLinkRelation::Intervalfinishedby => "intervalfinishedby", + ArchivedLinkRelation::Intervalfinishes => "intervalfinishes", + ArchivedLinkRelation::Intervalin => "intervalin", + ArchivedLinkRelation::Intervalmeets => "intervalmeets", + ArchivedLinkRelation::Intervalmetby => "intervalmetby", + ArchivedLinkRelation::Intervaloverlappedby => "intervaloverlappedby", + ArchivedLinkRelation::Intervaloverlaps => "intervaloverlaps", + ArchivedLinkRelation::Intervalstartedby => "intervalstartedby", + ArchivedLinkRelation::Intervalstarts => "intervalstarts", + ArchivedLinkRelation::Item => "item", + ArchivedLinkRelation::Last => "last", + ArchivedLinkRelation::LatestVersion => "latest-version", + ArchivedLinkRelation::License => "license", + ArchivedLinkRelation::Linkset => "linkset", + ArchivedLinkRelation::Lrdd => "lrdd", + ArchivedLinkRelation::Manifest => "manifest", + ArchivedLinkRelation::MaskIcon => "mask-icon", + ArchivedLinkRelation::Me => "me", + ArchivedLinkRelation::MediaFeed => "media-feed", + ArchivedLinkRelation::Memento => "memento", + ArchivedLinkRelation::Micropub => "micropub", + ArchivedLinkRelation::Modulepreload => "modulepreload", + ArchivedLinkRelation::Monitor => "monitor", + ArchivedLinkRelation::MonitorGroup => "monitor-group", + ArchivedLinkRelation::Next => "next", + ArchivedLinkRelation::NextArchive => "next-archive", + ArchivedLinkRelation::Nofollow => "nofollow", + ArchivedLinkRelation::Noopener => "noopener", + ArchivedLinkRelation::Noreferrer => "noreferrer", + ArchivedLinkRelation::Opener => "opener", + ArchivedLinkRelation::Openid2LocalId => "openid2.local_id", + ArchivedLinkRelation::Openid2Provider => "openid2.provider", + ArchivedLinkRelation::Original => "original", + ArchivedLinkRelation::P3pv1 => "p3pv1", + ArchivedLinkRelation::Payment => "payment", + ArchivedLinkRelation::Pingback => "pingback", + ArchivedLinkRelation::Preconnect => "preconnect", + ArchivedLinkRelation::PredecessorVersion => "predecessor-version", + ArchivedLinkRelation::Prefetch => "prefetch", + ArchivedLinkRelation::Preload => "preload", + ArchivedLinkRelation::Prerender => "prerender", + ArchivedLinkRelation::Prev => "prev", + ArchivedLinkRelation::Preview => "preview", + ArchivedLinkRelation::Previous => "previous", + ArchivedLinkRelation::PrevArchive => "prev-archive", + ArchivedLinkRelation::PrivacyPolicy => "privacy-policy", + ArchivedLinkRelation::Profile => "profile", + ArchivedLinkRelation::Publication => "publication", + ArchivedLinkRelation::RdapActive => "rdap-active", + ArchivedLinkRelation::RdapBottom => "rdap-bottom", + ArchivedLinkRelation::RdapDown => "rdap-down", + ArchivedLinkRelation::RdapTop => "rdap-top", + ArchivedLinkRelation::RdapUp => "rdap-up", + ArchivedLinkRelation::Related => "related", + ArchivedLinkRelation::Restconf => "restconf", + ArchivedLinkRelation::Replies => "replies", + ArchivedLinkRelation::Ruleinput => "ruleinput", + ArchivedLinkRelation::Search => "search", + ArchivedLinkRelation::Section => "section", + ArchivedLinkRelation::Self_ => "self", + ArchivedLinkRelation::Service => "service", + ArchivedLinkRelation::ServiceDesc => "service-desc", + ArchivedLinkRelation::ServiceDoc => "service-doc", + ArchivedLinkRelation::ServiceMeta => "service-meta", + ArchivedLinkRelation::SipTrunkingCapability => "sip-trunking-capability", + ArchivedLinkRelation::Sponsored => "sponsored", + ArchivedLinkRelation::Start => "start", + ArchivedLinkRelation::Status => "status", + ArchivedLinkRelation::Stylesheet => "stylesheet", + ArchivedLinkRelation::Subsection => "subsection", + ArchivedLinkRelation::SuccessorVersion => "successor-version", + ArchivedLinkRelation::Sunset => "sunset", + ArchivedLinkRelation::Tag => "tag", + ArchivedLinkRelation::TermsOfService => "terms-of-service", + ArchivedLinkRelation::Timegate => "timegate", + ArchivedLinkRelation::Timemap => "timemap", + ArchivedLinkRelation::Type => "type", + ArchivedLinkRelation::Ugc => "ugc", + ArchivedLinkRelation::Up => "up", + ArchivedLinkRelation::VersionHistory => "version-history", + ArchivedLinkRelation::Via => "via", + ArchivedLinkRelation::Webmention => "webmention", + ArchivedLinkRelation::WorkingCopy => "working-copy", + ArchivedLinkRelation::WorkingCopyOf => "working-copy-of", + } + } +} + +impl ArchivedICalendarRelated { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedICalendarRelated::Start => "START", + ArchivedICalendarRelated::End => "END", + } + } +} + +#[derive(Debug)] +pub(crate) enum ArchivedValueType { + Ical(ArchivedICalendarValueType), + CalendarScale, + Method, + Classification, + Status, + Transparency, + Action, + BusyType, + ParticipantType, + ResourceType, + Proximity, +} +impl ArchivedValueType { + pub fn unwrap_ical(self) -> ArchivedICalendarValueType { + match self { + ArchivedValueType::Ical(v) => v, + _ => ArchivedICalendarValueType::Text, + } + } +} diff --git a/crates/calcard/src/icalendar/rkyv_utils.rs b/crates/calcard/src/icalendar/rkyv_utils.rs new file mode 100644 index 00000000..4639bf62 --- /dev/null +++ b/crates/calcard/src/icalendar/rkyv_utils.rs @@ -0,0 +1,478 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::common::{ + ArchivedPartialDateTime, + timezone::{NominalDuration, Tz, ZonedDateTime}, +}; + +impl ArchivedICalendar { + pub fn uids(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| { + component.component_type.is_calendar_object() + || component.component_type == ICalendarComponentType::VCalendar + }) + .filter_map(|component| component.uid()) + } + + pub fn size(&self) -> usize { + self.components + .iter() + .map(|component| component.size()) + .sum() + } +} + +impl ArchivedICalendarComponent { + pub fn uid(&self) -> Option<&str> { + self.property(&ICalendarProperty::Uid) + .and_then(|e| e.values.first()) + .and_then(|v| v.as_text()) + } + + pub fn property(&self, prop: &ICalendarProperty) -> Option<&ArchivedICalendarEntry> { + self.entries.iter().find(|entry| &entry.name == prop) + } + + pub fn properties<'x, 'y: 'x>( + &'x self, + prop: &'y ICalendarProperty, + ) -> impl Iterator + 'x { + self.entries.iter().filter(move |entry| &entry.name == prop) + } + + pub fn size(&self) -> usize { + self.entries.iter().map(|entry| entry.size()).sum() + } + + pub fn is_recurrent(&self) -> bool { + self.entries.iter().any(|entry| { + matches!( + entry.name, + ArchivedICalendarProperty::Rrule | ArchivedICalendarProperty::Rdate + ) + }) + } + + pub fn is_recurrence_override(&self) -> bool { + self.entries + .iter() + .any(|entry| matches!(entry.name, ArchivedICalendarProperty::RecurrenceId)) + } + + pub fn is_recurrent_or_override(&self) -> bool { + self.entries.iter().any(|entry| { + matches!( + entry.name, + ArchivedICalendarProperty::Rrule + | ArchivedICalendarProperty::Rdate + | ArchivedICalendarProperty::RecurrenceId + ) + }) + } + + pub fn status(&self) -> Option<&ArchivedICalendarStatus> { + self.entries + .iter() + .find_map(|entry| match (&entry.name, entry.values.first()) { + ( + ArchivedICalendarProperty::Status, + Some(ArchivedICalendarValue::Status(status)), + ) => Some(status), + _ => None, + }) + } + + pub fn transparency(&self) -> Option<&ArchivedICalendarTransparency> { + self.entries + .iter() + .find_map(|entry| match (&entry.name, entry.values.first()) { + ( + ArchivedICalendarProperty::Transp, + Some(ArchivedICalendarValue::Transparency(trans)), + ) => Some(trans), + _ => None, + }) + } +} + +impl ArchivedICalendarValue { + pub fn size(&self) -> usize { + match self { + ArchivedICalendarValue::Binary(value) => value.len(), + ArchivedICalendarValue::Text(value) => value.len(), + ArchivedICalendarValue::PartialDateTime(_) => std::mem::size_of::(), + ArchivedICalendarValue::RecurrenceRule(_) => { + std::mem::size_of::() + } + _ => VALUE_SIZE, + } + } + + pub fn as_text(&self) -> Option<&str> { + match self { + ArchivedICalendarValue::Text(s) => Some(s.as_str()), + ArchivedICalendarValue::Uri(v) => v.as_str(), + ArchivedICalendarValue::CalendarScale(v) => Some(v.as_str()), + ArchivedICalendarValue::Method(v) => Some(v.as_str()), + ArchivedICalendarValue::Classification(v) => Some(v.as_str()), + ArchivedICalendarValue::Status(v) => Some(v.as_str()), + ArchivedICalendarValue::Transparency(v) => Some(v.as_str()), + ArchivedICalendarValue::Action(v) => Some(v.as_str()), + ArchivedICalendarValue::BusyType(v) => Some(v.as_str()), + ArchivedICalendarValue::ParticipantType(v) => Some(v.as_str()), + ArchivedICalendarValue::ResourceType(v) => Some(v.as_str()), + ArchivedICalendarValue::Proximity(v) => Some(v.as_str()), + _ => None, + } + } + + pub fn as_integer(&self) -> Option { + match self { + ArchivedICalendarValue::Integer(i) => Some(i.to_native()), + _ => None, + } + } + + pub fn as_float(&self) -> Option { + match self { + ArchivedICalendarValue::Float(f) => Some(f.to_native()), + _ => None, + } + } + + pub fn as_boolean(&self) -> Option { + match self { + ArchivedICalendarValue::Boolean(b) => Some(*b), + _ => None, + } + } + + pub fn as_partial_date_time(&self) -> Option<&ArchivedPartialDateTime> { + match self { + ArchivedICalendarValue::PartialDateTime(dt) => Some(dt), + _ => None, + } + } + + pub fn as_binary(&self) -> Option<&[u8]> { + match self { + ArchivedICalendarValue::Binary(d) => Some(d.as_slice()), + _ => None, + } + } +} + +impl ArchivedICalendarParameterName { + pub fn as_str(&self) -> &str { + match self { + ArchivedICalendarParameterName::Altrep => "ALTREP", + ArchivedICalendarParameterName::Cn => "CN", + ArchivedICalendarParameterName::Cutype => "CUTYPE", + ArchivedICalendarParameterName::DelegatedFrom => "DELEGATED-FROM", + ArchivedICalendarParameterName::DelegatedTo => "DELEGATED-TO", + ArchivedICalendarParameterName::Dir => "DIR", + ArchivedICalendarParameterName::Fmttype => "FMTTYPE", + ArchivedICalendarParameterName::Fbtype => "FBTYPE", + ArchivedICalendarParameterName::Language => "LANGUAGE", + ArchivedICalendarParameterName::Member => "MEMBER", + ArchivedICalendarParameterName::Partstat => "PARTSTAT", + ArchivedICalendarParameterName::Range => "RANGE", + ArchivedICalendarParameterName::Related => "RELATED", + ArchivedICalendarParameterName::Reltype => "RELTYPE", + ArchivedICalendarParameterName::Role => "ROLE", + ArchivedICalendarParameterName::Rsvp => "RSVP", + ArchivedICalendarParameterName::ScheduleAgent => "SCHEDULE-AGENT", + ArchivedICalendarParameterName::ScheduleForceSend => "SCHEDULE-FORCE-SEND", + ArchivedICalendarParameterName::ScheduleStatus => "SCHEDULE-STATUS", + ArchivedICalendarParameterName::SentBy => "SENT-BY", + ArchivedICalendarParameterName::Tzid => "TZID", + ArchivedICalendarParameterName::Value => "VALUE", + ArchivedICalendarParameterName::Display => "DISPLAY", + ArchivedICalendarParameterName::Email => "EMAIL", + ArchivedICalendarParameterName::Feature => "FEATURE", + ArchivedICalendarParameterName::Label => "LABEL", + ArchivedICalendarParameterName::Size => "SIZE", + ArchivedICalendarParameterName::Filename => "FILENAME", + ArchivedICalendarParameterName::ManagedId => "MANAGED-ID", + ArchivedICalendarParameterName::Order => "ORDER", + ArchivedICalendarParameterName::Schema => "SCHEMA", + ArchivedICalendarParameterName::Derived => "DERIVED", + ArchivedICalendarParameterName::Gap => "GAP", + ArchivedICalendarParameterName::Linkrel => "LINKREL", + ArchivedICalendarParameterName::Jsptr => "JSPTR", + ArchivedICalendarParameterName::Jsid => "JSID", + ArchivedICalendarParameterName::Other(name) => name.as_str(), + } + } +} + +impl ArchivedICalendarEntry { + #[inline] + pub fn parameters( + &self, + prop: &ICalendarParameterName, + ) -> impl Iterator { + self.params.iter().filter_map(move |param| { + if ¶m.name == prop { + Some(¶m.value) + } else { + None + } + }) + } + + pub fn size(&self) -> usize { + self.values.iter().map(|value| value.size()).sum::() + + self + .params + .iter() + .map(|param| std::mem::size_of::() + param.value.size()) + .sum::() + + self.name.as_str().len() + } +} + +impl ArchivedICalendarParameterValue { + pub fn as_text(&self) -> Option<&str> { + match self { + ArchivedICalendarParameterValue::Text(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Bool(v) => Some(if *v { "TRUE" } else { "FALSE" }), + ArchivedICalendarParameterValue::Uri(uri) => uri.as_str(), + ArchivedICalendarParameterValue::Cutype(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Fbtype(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Partstat(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Related(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Reltype(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Role(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::ScheduleAgent(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::ScheduleForceSend(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Value(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Display(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Feature(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Linkrel(v) => Some(v.as_str()), + ArchivedICalendarParameterValue::Duration(_) + | ArchivedICalendarParameterValue::Integer(_) + | ArchivedICalendarParameterValue::Null => None, + } + } + + pub fn size(&self) -> usize { + match self { + ArchivedICalendarParameterValue::Text(v) => v.len(), + ArchivedICalendarParameterValue::Uri(v) => v.size(), + _ => PARAMETER_VALUE_SIZE, + } + } +} + +impl ArchivedUri { + pub fn as_str(&self) -> Option<&str> { + match self { + ArchivedUri::Data(data) => data.content_type.as_deref(), + ArchivedUri::Location(loc) => loc.as_str().into(), + } + } + + pub fn size(&self) -> usize { + match self { + ArchivedUri::Data(data) => { + data.data.len() + data.content_type.as_ref().map(|s| s.len()).unwrap_or(0) + } + ArchivedUri::Location(loc) => loc.len(), + } + } +} + +impl ArchivedICalendarPeriod { + pub fn time_range(&self, tz: Tz) -> Option<(ZonedDateTime, ZonedDateTime)> { + match self { + ArchivedICalendarPeriod::Range { start, end } => { + if let (Some(start), Some(end)) = ( + start + .to_date_time() + .and_then(|start| start.to_date_time_with_tz(tz)), + end.to_date_time() + .and_then(|end| end.to_date_time_with_tz(tz)), + ) { + Some((start, end)) + } else { + None + } + } + ArchivedICalendarPeriod::Duration { start, duration } => start + .to_date_time() + .and_then(|start| start.to_date_time_with_tz(tz)) + .zip(duration.to_nominal()) + .and_then(|(start, duration)| { + start.checked_add_nominal(duration).map(|end| (start, end)) + }), + } + } +} + +impl ArchivedPartialDateTime { + pub fn to_date_time_with_tz(&self, tz: Tz) -> Option { + self.to_date_time() + .and_then(|dt| dt.to_date_time_with_tz(tz)) + } + + #[inline(always)] + pub fn has_date(&self) -> bool { + self.year.is_some() && self.month.is_some() && self.day.is_some() + } + + #[inline(always)] + pub fn has_time(&self) -> bool { + self.hour.is_some() && self.minute.is_some() + } + + #[inline(always)] + pub fn has_zone(&self) -> bool { + self.tz_hour.is_some() + } + + #[inline(always)] + pub fn has_date_and_time(&self) -> bool { + self.has_date() && self.has_time() + } +} + +impl ArchivedICalendarDuration { + /// Returns this duration as a calendar span. + pub fn to_nominal(&self) -> Option { + let days = + i32::try_from(i64::from(self.weeks.to_native()) * 7 + i64::from(self.days.to_native())) + .ok()?; + let seconds = i64::from(self.hours.to_native()) * 3600 + + i64::from(self.minutes.to_native()) * 60 + + i64::from(self.seconds.to_native()); + Some(if self.neg { + NominalDuration::new(-days, -seconds) + } else { + NominalDuration::new(days, seconds) + }) + } + + pub fn as_seconds(&self) -> i64 { + let secs = self.seconds.to_native() as i64 + + self.minutes.to_native() as i64 * 60 + + self.hours.to_native() as i64 * 3600 + + self.days.to_native() as i64 * 86400 + + self.weeks.to_native() as i64 * 604800; + + if self.neg { -secs } else { secs } + } + + pub fn to_native(&self) -> ICalendarDuration { + ICalendarDuration { + neg: self.neg, + weeks: self.weeks.to_native(), + days: self.days.to_native(), + hours: self.hours.to_native(), + minutes: self.minutes.to_native(), + seconds: self.seconds.to_native(), + } + } +} + +impl ArchivedICalendarComponentType { + pub fn is_calendar_object(&self) -> bool { + matches!( + self, + ArchivedICalendarComponentType::VEvent + | ArchivedICalendarComponentType::VTodo + | ArchivedICalendarComponentType::VJournal + | ArchivedICalendarComponentType::VFreebusy + | ArchivedICalendarComponentType::VAvailability + ) + } + + pub(crate) fn converts_links(&self) -> bool { + matches!( + self, + ArchivedICalendarComponentType::VEvent + | ArchivedICalendarComponentType::VTodo + | ArchivedICalendarComponentType::Participant + | ArchivedICalendarComponentType::VLocation + | ArchivedICalendarComponentType::VCalendar + ) + } + + pub fn has_time_ranges(&self) -> bool { + matches!( + self, + ArchivedICalendarComponentType::VEvent + | ArchivedICalendarComponentType::VTodo + | ArchivedICalendarComponentType::VJournal + | ArchivedICalendarComponentType::VFreebusy + ) + } + + pub fn is_scheduling_object(&self) -> bool { + matches!( + self, + ArchivedICalendarComponentType::VEvent + | ArchivedICalendarComponentType::VTodo + | ArchivedICalendarComponentType::VJournal + | ArchivedICalendarComponentType::VFreebusy + ) + } + + pub fn is_event(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VEvent) + } + + pub fn is_todo(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VTodo) + } + + pub fn is_event_or_todo(&self) -> bool { + matches!( + self, + ArchivedICalendarComponentType::VEvent | ArchivedICalendarComponentType::VTodo + ) + } + + pub fn is_journal(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VJournal) + } + + pub fn is_freebusy(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VFreebusy) + } + + pub fn is_location(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VLocation) + } + + pub fn is_alarm(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::VAlarm) + } + + pub fn is_participant(&self) -> bool { + matches!(self, ArchivedICalendarComponentType::Participant) + } +} + +impl ArchivedICalendarMonth { + pub fn is_leap(&self) -> bool { + self.0 < 0 + } + + pub fn month(&self) -> u8 { + self.0.unsigned_abs() + } +} + +impl PartialEq> for ArchivedICalendarPeriod { + fn eq(&self, other: &Box) -> bool { + self == other.as_ref() + } +} diff --git a/crates/calcard/src/icalendar/rkyv_writer.rs b/crates/calcard/src/icalendar/rkyv_writer.rs new file mode 100644 index 00000000..cc2cfa0b --- /dev/null +++ b/crates/calcard/src/icalendar/rkyv_writer.rs @@ -0,0 +1,494 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::{ + common::{ + ArchivedPartialDateTime, CalendarScale, PartialDateTime, + format::{AsciiPush, BufferedWrite, DurationParts}, + stack::ComponentStack, + writer::{ + DOCUMENT_BUFFER, ENTRY_BUFFER, FoldingWriter, LineWriter, VisitedComponents, + write_param_text, write_param_value, write_quoted_param_value, write_text, write_uri, + }, + }, + icalendar::{ + ValueSeparator, + writer::{DATE_BUFFER, DURATION_BUFFER, PERIOD_BUFFER, RRULE_BUFFER}, + }, +}; +use std::{ + fmt::{self, Display, Write}, + slice::Iter, +}; + +impl ArchivedICalendar { + pub fn write_to(&self, out: &mut impl Write) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_components(&mut FoldingWriter::new(buf)) + }) + } + + fn write_components( + &self, + out: &mut FoldingWriter<'_, W>, + ) -> fmt::Result { + let _v = [0.into()]; + let mut component_iter: Iter<'_, rkyv::primitive::ArchivedU32> = _v.iter(); + let mut component_stack = ComponentStack::new(); + let mut visited = VisitedComponents::new(self.components.len()); + + loop { + if let Some(component_id) = component_iter.next() { + let component_id = component_id.to_native() as usize; + let Some(component) = self + .components + .get(component_id) + .filter(|_| visited.insert(component_id)) + else { + continue; + }; + out.write_boundary("BEGIN:", component.component_type.as_str())?; + + for entry in component.entries.iter() { + if !matches!( + entry.name, + ArchivedICalendarProperty::Begin | ArchivedICalendarProperty::End + ) { + entry.write_line(out, true)?; + } + } + + if !component.component_ids.is_empty() { + component_stack.push((component, component_iter)); + component_iter = component.component_ids.iter(); + } else { + out.write_boundary("END:", component.component_type.as_str())?; + } + } else if let Some((component, iter)) = component_stack.pop() { + out.write_boundary("END:", component.component_type.as_str())?; + component_iter = iter; + } else { + break; + } + } + + Ok(()) + } +} + +impl ArchivedICalendarEntry { + pub fn write_to(&self, out: &mut impl Write, with_value: bool) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_line(&mut FoldingWriter::new(buf), with_value) + }) + } + + pub(crate) fn write_line( + &self, + out: &mut FoldingWriter<'_, W>, + with_value: bool, + ) -> fmt::Result { + out.write_atomic(self.name.as_str())?; + + if matches!( + self.values.first().as_ref(), + Some(ArchivedICalendarValue::Binary(_)) + ) { + out.write_atomic(";ENCODING=BASE64")?; + if !self + .params + .iter() + .any(|param| param.name == ArchivedICalendarParameterName::Value) + { + out.write_atomic(";VALUE=BINARY")?; + } + } + + let mut types = None; + let mut last_param: Option<&ArchivedICalendarParameterName> = None; + + for param in self.params.iter() { + if last_param.is_some_and(|last_param| last_param == ¶m.name) { + out.write_atomic(",")?; + } else { + out.write_atomic(";")?; + out.write_atomic(param.name.as_str())?; + if !matches!(param.value, ArchivedICalendarParameterValue::Null) { + out.write_atomic("=")?; + } + last_param = Some(¶m.name); + } + + match ¶m.value { + ArchivedICalendarParameterValue::Text(v) + if matches!(param.name, ArchivedICalendarParameterName::Jsptr) => + { + write_quoted_param_value(out, v, true)?; + } + ArchivedICalendarParameterValue::Text(v) => { + write_param_value(out, v, true)?; + } + ArchivedICalendarParameterValue::Integer(i) => { + out.push_u64(i.to_native())?; + } + ArchivedICalendarParameterValue::Bool(v) => { + let v = if !matches!(param.name, ArchivedICalendarParameterName::Range) { + if *v { "TRUE" } else { "FALSE" } + } else { + "THISANDFUTURE" + }; + out.write_atomic(v)?; + } + ArchivedICalendarParameterValue::Uri(uri) => { + out.write_atomic("\"")?; + uri.write_param_text(out)?; + out.write_atomic("\"")?; + } + ArchivedICalendarParameterValue::Cutype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Fbtype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Partstat(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Related(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Reltype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Role(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::ScheduleAgent(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::ScheduleForceSend(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Value(v) => { + types = Some(v); + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Display(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Feature(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Duration(v) => { + out.push_duration(v.parts())?; + } + ArchivedICalendarParameterValue::Linkrel(v) => { + write_param_value(out, v.as_str(), true)?; + } + ArchivedICalendarParameterValue::Null => { + last_param = None; + } + } + } + + out.write_atomic(":")?; + + if with_value { + let (default_type, separator) = self.name.default_types(); + let separator = if !matches!(separator, ValueSeparator::Comma) { + ";" + } else { + "," + }; + let default_type = default_type.unwrap_ical(); + + for (pos, value) in self.values.iter().enumerate() { + if pos > 0 { + out.write_atomic(separator)?; + } + + let text = match value { + ArchivedICalendarValue::Binary(v) => { + out.write_base64(v)?; + continue; + } + ArchivedICalendarValue::Boolean(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + continue; + } + ArchivedICalendarValue::Uri(v) => { + v.write_value(out)?; + continue; + } + ArchivedICalendarValue::PartialDateTime(v) => { + PartialDateTime::from(v).push_ical( + out, + &ICalendarValueType::from(types.unwrap_or(&default_type)), + )?; + continue; + } + ArchivedICalendarValue::Duration(v) => { + out.push_duration(v.parts())?; + continue; + } + ArchivedICalendarValue::RecurrenceRule(v) => { + v.push_to(out)?; + continue; + } + ArchivedICalendarValue::Period(v) => { + v.push_to(out)?; + continue; + } + ArchivedICalendarValue::Float(v) => { + write!(out, "{v}")?; + continue; + } + ArchivedICalendarValue::Integer(v) => { + out.push_i64(v.to_native())?; + continue; + } + ArchivedICalendarValue::Text(v) => { + match types.unwrap_or(&default_type) { + ArchivedICalendarValueType::Uri + | ArchivedICalendarValueType::CalAddress => { + write_uri(out, v)?; + } + ArchivedICalendarValueType::Recur => write_text(out, v, false, false)?, + _ => write_text(out, v, true, true)?, + } + continue; + } + ArchivedICalendarValue::CalendarScale(v) => v.as_str(), + ArchivedICalendarValue::Method(v) => v.as_str(), + ArchivedICalendarValue::Classification(v) => v.as_str(), + ArchivedICalendarValue::Status(v) => v.as_str(), + ArchivedICalendarValue::Transparency(v) => v.as_str(), + ArchivedICalendarValue::Action(v) => v.as_str(), + ArchivedICalendarValue::BusyType(v) => v.as_str(), + ArchivedICalendarValue::ParticipantType(v) => v.as_str(), + ArchivedICalendarValue::ResourceType(v) => v.as_str(), + ArchivedICalendarValue::Proximity(v) => v.as_str(), + }; + + out.write_atomic(text)?; + } + } + out.end_line() + } +} + +impl ArchivedUri { + fn write_value(&self, out: &mut impl LineWriter) -> fmt::Result { + match self { + ArchivedUri::Data(v) => { + out.write_str("data:")?; + out.write_str(v.content_type.as_deref().unwrap_or_default())?; + out.write_str(";")?; + out.write_atomic("base64,")?; + out.write_base64(&v.data) + } + ArchivedUri::Location(v) => write_uri(out, v), + } + } + + fn write_param_text(&self, out: &mut impl LineWriter) -> fmt::Result { + match self { + ArchivedUri::Data(v) => { + out.write_str("data:")?; + write_param_text(out, v.content_type.as_deref().unwrap_or_default(), true)?; + out.write_str(";")?; + out.write_atomic("base64,")?; + out.write_base64(&v.data) + } + ArchivedUri::Location(v) => write_param_text(out, v, true), + } + } +} + +impl ArchivedICalendarRecurrenceRule { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + buf.push_ascii_str("FREQ=")?; + buf.push_ascii_str(self.freq.as_str())?; + if let Some(until) = self.until.as_ref() { + buf.push_ascii_str(";UNTIL=")?; + PartialDateTime::from(until).push_ical( + buf, + if until.has_date_and_time() { + &ICalendarValueType::DateTime + } else { + &ICalendarValueType::Date + }, + )?; + } + if let Some(count) = self.count.as_ref().filter(|c| **c > 0) { + buf.push_ascii_str(";COUNT=")?; + buf.push_u64(count.to_native().into())?; + } + if let Some(interval) = self.interval.as_ref() { + buf.push_ascii_str(";INTERVAL=")?; + buf.push_u64(interval.to_native().into())?; + } + buf.push_list(";BYSECOND=", &self.bysecond, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYMINUTE=", &self.byminute, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYHOUR=", &self.byhour, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYDAY=", &self.byday, |buf, item| item.push_to(buf))?; + buf.push_list(";BYMONTHDAY=", &self.bymonthday, |buf, item| { + buf.push_i64((*item).into()) + })?; + buf.push_list(";BYYEARDAY=", &self.byyearday, |buf, item| { + buf.push_i64(item.to_native().into()) + })?; + buf.push_list(";BYWEEKNO=", &self.byweekno, |buf, item| { + buf.push_i64((*item).into()) + })?; + buf.push_list(";BYMONTH=", &self.bymonth, |buf, item| item.push_to(buf))?; + buf.push_list(";BYSETPOS=", &self.bysetpos, |buf, item| { + buf.push_i64(item.to_native().into()) + })?; + if let Some(wkst) = self.wkst.as_ref() { + buf.push_ascii_str(";WKST=")?; + buf.push_ascii_str(wkst.as_str())?; + } + if let Some(rscale) = self.rscale.as_ref() { + buf.push_ascii_str(";RSCALE=")?; + buf.push_ascii_str(rscale.as_str())?; + } else if self.skip.is_some() { + buf.push_ascii_str(";RSCALE=")?; + buf.push_ascii_str(CalendarScale::Gregorian.as_str())?; + } + if let Some(skip) = self.skip.as_ref() { + buf.push_ascii_str(";SKIP=")?; + buf.push_ascii_str(skip.as_str())?; + } + + Ok(()) + } +} + +impl Display for ArchivedICalendarRecurrenceRule { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ArchivedICalendarDay { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + if let Some(ordwk) = self.ordwk.as_ref() { + buf.push_i64(ordwk.to_native().into())?; + } + buf.push_ascii_str(self.weekday.as_str()) + } +} + +impl Display for ArchivedICalendarDay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ArchivedICalendarMonth { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + buf.push_u64(self.month().into())?; + if self.is_leap() { + buf.push_byte(b'L')?; + } + Ok(()) + } +} + +impl Display for ArchivedICalendarMonth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ArchivedICalendarPeriod { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + match self { + ArchivedICalendarPeriod::Range { start, end } => { + PartialDateTime::from(start).push_ical(buf, &ICalendarValueType::DateTime)?; + buf.push_byte(b'/')?; + PartialDateTime::from(end).push_ical(buf, &ICalendarValueType::DateTime) + } + ArchivedICalendarPeriod::Duration { start, duration } => { + PartialDateTime::from(start).push_ical(buf, &ICalendarValueType::DateTime)?; + buf.push_byte(b'/')?; + buf.push_duration(duration.parts()) + } + } + } +} + +impl Display for ArchivedICalendarPeriod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ArchivedICalendarDuration { + fn parts(&self) -> DurationParts { + DurationParts { + neg: self.neg, + weeks: self.weeks.to_native(), + days: self.days.to_native(), + hours: self.hours.to_native(), + minutes: self.minutes.to_native(), + seconds: self.seconds.to_native(), + } + } +} + +impl Display for ArchivedICalendarDuration { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| buf.push_duration(self.parts())) + } +} + +impl ArchivedPartialDateTime { + pub fn format_as_ical( + &self, + out: &mut impl Write, + fmt: &ArchivedICalendarValueType, + ) -> fmt::Result { + PartialDateTime::from(self).format_as_ical(out, &ICalendarValueType::from(fmt)) + } +} + +impl From<&ArchivedICalendarValueType> for ICalendarValueType { + fn from(value_type: &ArchivedICalendarValueType) -> Self { + match value_type { + ArchivedICalendarValueType::Binary => ICalendarValueType::Binary, + ArchivedICalendarValueType::Boolean => ICalendarValueType::Boolean, + ArchivedICalendarValueType::CalAddress => ICalendarValueType::CalAddress, + ArchivedICalendarValueType::Date => ICalendarValueType::Date, + ArchivedICalendarValueType::DateTime => ICalendarValueType::DateTime, + ArchivedICalendarValueType::Duration => ICalendarValueType::Duration, + ArchivedICalendarValueType::Float => ICalendarValueType::Float, + ArchivedICalendarValueType::Integer => ICalendarValueType::Integer, + ArchivedICalendarValueType::Period => ICalendarValueType::Period, + ArchivedICalendarValueType::Recur => ICalendarValueType::Recur, + ArchivedICalendarValueType::Text => ICalendarValueType::Text, + ArchivedICalendarValueType::Time => ICalendarValueType::Time, + ArchivedICalendarValueType::Unknown => ICalendarValueType::Unknown, + ArchivedICalendarValueType::Uri => ICalendarValueType::Uri, + ArchivedICalendarValueType::UtcOffset => ICalendarValueType::UtcOffset, + ArchivedICalendarValueType::XmlReference => ICalendarValueType::XmlReference, + ArchivedICalendarValueType::Uid => ICalendarValueType::Uid, + } + } +} + +impl Display for ArchivedICalendar { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_to(f) + } +} diff --git a/crates/calcard/src/icalendar/timezone.rs b/crates/calcard/src/icalendar/timezone.rs new file mode 100644 index 00000000..edbc360b --- /dev/null +++ b/crates/calcard/src/icalendar/timezone.rs @@ -0,0 +1,700 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDay, ICalendarEntry, + ICalendarFrequency, ICalendarMonth, ICalendarPeriod, ICalendarProperty, + ICalendarRecurrenceRule, ICalendarValue, ICalendarWeekday, +}; +use crate::{ + common::{ + PartialDateTime, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::ICalendarParameterName, +}; +use jiff::{ + Timestamp, + civil::{self, Date, Weekday}, + tz::{Offset, TimeZone}, +}; +use std::{collections::HashMap, ops::Range, str::FromStr}; + +pub struct TzResolver { + tzs: HashMap, + default: Tz, +} + +impl TzResolver +where + T: std::borrow::Borrow + std::hash::Hash + Eq, +{ + pub fn resolve(&self, tz_name: &str) -> Option { + self.tzs + .get(tz_name) + .copied() + .or_else(|| Tz::from_str(tz_name).ok()) + } + + pub fn resolve_or_default(&self, tz_name: Option<&str>) -> Tz { + tz_name + .and_then(|tz_name| { + self.tzs + .get(tz_name) + .copied() + .or_else(|| Tz::from_str(tz_name).ok()) + }) + .unwrap_or(self.default) + } + + pub fn with_default(mut self, default: impl Into) -> Self { + self.default = default.into(); + self + } +} + +impl ICalendar { + pub fn timezones(&self) -> impl Iterator { + self.components + .iter() + .filter(|comp| matches!(comp.component_type, ICalendarComponentType::VTimezone)) + } + + pub fn is_timezone(&self) -> bool { + self.timezones().count() == 1 + } + + pub fn build_tz_resolver(&self) -> TzResolver<&'_ str> { + TzResolver { + tzs: self.timezones().filter_map(|tz| tz.timezone()).collect(), + default: Tz::Floating, + } + } + + pub fn build_owned_tz_resolver(&self) -> TzResolver { + TzResolver { + tzs: self + .timezones() + .filter_map(|tz| tz.timezone()) + .map(|(name, tz)| (name.to_string(), tz)) + .collect(), + default: Tz::Floating, + } + } +} + +impl ICalendarComponent { + pub fn timezone(&self) -> Option<(&str, Tz)> { + let mut tz_name = None; + let mut tz_id = None; + let mut tz_lic = None; + let mut tz_cdo_id = None; + + for entry in &self.entries { + match (&entry.name, entry.values.first()) { + (ICalendarProperty::Tzid, Some(ICalendarValue::Text(id))) => { + tz_id = Tz::from_str(id).ok(); + tz_name = Some(id.as_str()); + } + (ICalendarProperty::Other(value), Some(ICalendarValue::Text(id))) => { + hashify::fnc_map!(value.as_bytes(), + "X-LIC-LOCATION" => { + tz_lic = Tz::from_str(id.strip_prefix("SystemV/").unwrap_or(id.as_str())).ok(); + }, + "X-MICROSOFT-CDO-TZID" => { + tz_cdo_id = Tz::from_ms_cdo_zone_id(id); + }, + _ => {} + ); + } + _ => (), + } + } + + tz_name.zip(tz_id.or(tz_lic).or(tz_cdo_id)) + } +} + +impl ICalendarPeriod { + pub fn time_range(&self, tz: Tz) -> Option<(ZonedDateTime, ZonedDateTime)> { + match self { + ICalendarPeriod::Range { start, end } => start + .to_date_time_with_tz(tz) + .zip(end.to_date_time_with_tz(tz)), + ICalendarPeriod::Duration { start, duration } => start + .to_date_time_with_tz(tz) + .zip(duration.to_nominal()) + .and_then(|(start, duration)| { + start.checked_add_nominal(duration).map(|end| (start, end)) + }), + } + } +} + +impl ICalendarEntry { + pub fn tz_id(&self) -> Option<&str> { + self.parameters(&ICalendarParameterName::Tzid) + .find_map(|v| v.as_text()) + } +} + +const TZ_MIN_YEAR: i32 = 1900; +const TZ_MAX_FUTURE_YEARS: i32 = 100; +const TZ_RANGE_MARGIN: i64 = 86400; +const TZ_RULE_ACTIVE_SECONDS: i64 = 366 * 86400; +const MAX_OBSERVANCES: usize = 4096; + +impl ICalendar { + pub fn add_timezone(&mut self, tz_id: &str, from: i64, to: i64) -> Option { + self.calendar_root()?; + + let tz = Tz::from_str(tz_id).ok()?; + let observances = match tz { + Tz::Iana(_) => build_observances(tz.time_zone()?, from, to), + Tz::Fixed(offset) => { + let offset = round_to_minute(offset.seconds()); + vec![TzObservance { + at: from, + from_offset: offset, + to_offset: offset, + is_dst: false, + name: None, + rrule: None, + }] + } + Tz::Floating => return None, + }; + if observances.is_empty() { + return None; + } + + let tz_component_id = u32::try_from(self.components.len()).ok()?; + let first_component_id = tz_component_id.checked_add(1)?; + let last_component_id = + first_component_id.checked_add(u32::try_from(observances.len()).ok()?)?; + let insert_at = self + .calendar_root()? + .component_ids + .iter() + .position(|component_id| { + self.component_by_id(*component_id) + .is_none_or(|comp| comp.component_type != ICalendarComponentType::VTimezone) + }); + + self.components.reserve(observances.len() + 1); + self.components.push(ICalendarComponent { + component_type: ICalendarComponentType::VTimezone, + entries: vec![ + ICalendarEntry::new(ICalendarProperty::Tzid).with_value(tz_id.to_string()), + ], + component_ids: (first_component_id..last_component_id).collect(), + }); + self.components + .extend(observances.into_iter().map(TzObservance::into_component)); + + let root_ids = &mut self.components.first_mut()?.component_ids; + root_ids.insert(insert_at.unwrap_or(root_ids.len()), tz_component_id); + + Some(tz_component_id) + } + + pub fn add_missing_timezones(&mut self) -> usize { + let mut defined: Vec<&str> = Vec::new(); + for component in self.timezones() { + for entry in &component.entries { + if entry.name == ICalendarProperty::Tzid + && let Some(ICalendarValue::Text(tz_id)) = entry.values.first() + { + defined.push(tz_id.as_str()); + } + } + } + + let mut referenced: Vec = Vec::new(); + let mut range: Option<(i64, i64)> = None; + let mut recurrence_span = 0; + let mut is_unbounded = false; + + for component in &self.components { + if matches!( + component.component_type, + ICalendarComponentType::VTimezone + | ICalendarComponentType::Standard + | ICalendarComponentType::Daylight + ) { + continue; + } + + for entry in &component.entries { + for param in &entry.params { + if param.name == ICalendarParameterName::Tzid + && let Some(tz_id) = param.value.as_text() + && !defined.contains(&tz_id) + && !referenced.iter().any(|id| id == tz_id) + { + referenced.push(tz_id.to_string()); + } + } + + match &entry.name { + ICalendarProperty::Rrule => { + for value in &entry.values { + if let ICalendarValue::RecurrenceRule(rule) = value { + match (&rule.until, &rule.count) { + (Some(until), _) => expand_range(&mut range, until), + (None, Some(count)) => { + recurrence_span = recurrence_span.max(recurrence_span_of( + &rule.freq, + *count, + rule.interval, + )); + } + (None, None) => is_unbounded = true, + } + } + } + } + ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Due + | ICalendarProperty::RecurrenceId + | ICalendarProperty::Exdate + | ICalendarProperty::Rdate => { + for value in &entry.values { + match value { + ICalendarValue::PartialDateTime(dt) => { + expand_range(&mut range, dt); + } + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, end } => { + expand_range(&mut range, start); + expand_range(&mut range, end); + } + ICalendarPeriod::Duration { start, .. } => { + expand_range(&mut range, start); + } + }, + _ => (), + } + } + } + _ => (), + } + } + } + + if referenced.is_empty() { + return 0; + } + + let now = Timestamp::now().as_second(); + let (min, max) = range.unwrap_or((now, now)); + let max = max.saturating_add(recurrence_span); + let current_year = year_of(now); + let max_year = current_year.saturating_add(TZ_MAX_FUTURE_YEARS); + let from_year = year_of(min).clamp(TZ_MIN_YEAR, max_year); + let to_year = if is_unbounded { + year_of(max).max(current_year) + } else { + year_of(max) + } + .saturating_add(2) + .clamp(from_year.saturating_add(1), max_year.saturating_add(2)); + let from = start_of_year(from_year).saturating_sub(TZ_RANGE_MARGIN); + let to = start_of_year(to_year); + + let mut added = 0; + for tz_id in referenced { + if self.add_timezone(&tz_id, from, to).is_some() { + added += 1; + } + } + added + } +} + +#[derive(PartialEq, Eq)] +struct TzRuleKey { + month: u8, + ordwk: i16, + weekday: ICalendarWeekday, + hour: u8, + minute: u8, + second: u8, + from_offset: i32, + to_offset: i32, +} + +struct TzObservance { + at: i64, + from_offset: i32, + to_offset: i32, + is_dst: bool, + name: Option, + rrule: Option, +} + +impl TzObservance { + fn onset(&self) -> civil::DateTime { + naive_utc(self.at.saturating_add(self.from_offset as i64)) + } + + fn rule_key(&self) -> Option<(i32, TzRuleKey)> { + let onset = self.onset(); + let day = u32::from(onset.day().unsigned_abs()); + let ordwk = if day.saturating_add(7) + > days_in_month( + i32::from(onset.year()), + u32::from(onset.month().unsigned_abs()), + ) { + -1 + } else { + ((day - 1) / 7 + 1) as i16 + }; + + Some(( + i32::from(onset.year()), + TzRuleKey { + month: u8::try_from(onset.month()).ok()?, + ordwk, + weekday: weekday_of(onset.weekday()), + hour: u8::try_from(onset.hour()).ok()?, + minute: u8::try_from(onset.minute()).ok()?, + second: u8::try_from(onset.second()).ok()?, + from_offset: self.from_offset, + to_offset: self.to_offset, + }, + )) + } + + fn into_component(self) -> ICalendarComponent { + let mut component = ICalendarComponent::new(if self.is_dst { + ICalendarComponentType::Daylight + } else { + ICalendarComponentType::Standard + }); + + component + .entries + .push(ICalendarEntry::new(ICalendarProperty::Dtstart).with_value( + PartialDateTime::from_naive_timestamp(naive_seconds(self.onset())), + )); + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Tzoffsetfrom) + .with_value(offset_to_partial(self.from_offset)), + ); + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Tzoffsetto) + .with_value(offset_to_partial(self.to_offset)), + ); + if let Some(name) = self.name { + component + .entries + .push(ICalendarEntry::new(ICalendarProperty::Tzname).with_value(name)); + } + if let Some(rrule) = self.rrule { + component + .entries + .push(ICalendarEntry::new(ICalendarProperty::Rrule).with_value(rrule)); + } + + component + } +} + +/// Collects the observances a zone goes through between two instants. +fn build_observances(tz: &TimeZone, from: i64, to: i64) -> Vec { + let Ok(start) = Timestamp::from_second(from) else { + return Vec::new(); + }; + let initial = tz.to_offset_info(start); + let mut observances = Vec::with_capacity(estimated_observances(from, to)); + observances.push(TzObservance { + at: from, + from_offset: round_to_minute(initial.offset().seconds()), + to_offset: round_to_minute(initial.offset().seconds()), + is_dst: initial.dst().is_dst(), + name: abbreviation(initial.abbreviation()), + rrule: None, + }); + + let mut previous = initial.offset(); + for transition in tz.following(start) { + let at = transition.timestamp().as_second(); + if at >= to || observances.len() >= MAX_OBSERVANCES { + break; + } + observances.push(TzObservance { + at, + from_offset: round_to_minute(previous.seconds()), + to_offset: round_to_minute(transition.offset().seconds()), + is_dst: transition.dst().is_dst(), + name: abbreviation(transition.abbreviation()), + rrule: None, + }); + previous = transition.offset(); + } + + label_observances(&mut observances); + + collapse_observances(observances, to) +} + +fn abbreviation(name: &str) -> Option { + (!name.is_empty() + && !name.starts_with(['+', '-']) + && name.bytes().all(|b| b.is_ascii_alphabetic())) + .then(|| name.to_string()) +} + +fn label_observances(observances: &mut [TzObservance]) { + let mut daylight_offset = None; + let mut standard_offset = None; + + for observance in observances.iter() { + let slot = if observance.is_dst { + &mut daylight_offset + } else { + &mut standard_offset + }; + *slot = Some(slot.map_or(observance.to_offset, |offset: i32| { + offset.max(observance.to_offset) + })); + } + + if let (Some(daylight_offset), Some(standard_offset)) = (daylight_offset, standard_offset) + && daylight_offset < standard_offset + { + for observance in observances.iter_mut() { + observance.is_dst = !observance.is_dst; + } + } +} + +fn collapse_observances(mut observances: Vec, to: i64) -> Vec { + let mut discard = vec![false; observances.len()]; + let mut unbounded: [Option<(usize, i64)>; 2] = [None, None]; + let mut trailing: [Option<(usize, u8, i16, ICalendarWeekday)>; 2] = [None, None]; + + let mut onsets: Vec<(usize, i32, TzRuleKey)> = Vec::new(); + let mut runs: Vec> = Vec::new(); + + for is_dst in [false, true] { + onsets.clear(); + runs.clear(); + + for (index, observance) in observances.iter().enumerate() { + if index == 0 || observance.is_dst != is_dst { + continue; + } + let Some((year, key)) = observance.rule_key() else { + continue; + }; + let extends = onsets + .last() + .is_some_and(|(last_index, last_year, last_key)| { + last_year.saturating_add(1) == year + && last_key == &key + && observances[*last_index].name == observance.name + }); + + onsets.push((index, year, key)); + match runs.last_mut() { + Some(run) if extends => run.end = onsets.len(), + _ => runs.push(onsets.len() - 1..onsets.len()), + } + } + + for (position, run) in runs.iter().enumerate() { + let (Some((first, _, key)), Some((last, _, _))) = + (onsets.get(run.start), onsets.get(run.end - 1)) + else { + continue; + }; + if run.len() < 2 { + continue; + } + + let last_at = observances[*last].at; + let is_expired = last_at.saturating_add(TZ_RULE_ACTIVE_SECONDS) < to; + let is_final = position + 1 == runs.len() && !is_expired; + observances[*first].rrule = Some(ICalendarRecurrenceRule { + freq: ICalendarFrequency::Yearly, + until: (!is_final).then(|| PartialDateTime::from_utc_timestamp(last_at)), + bymonth: vec![ICalendarMonth::new(key.month, false)], + byday: vec![ICalendarDay { + ordwk: Some(key.ordwk), + weekday: key.weekday, + }], + ..Default::default() + }); + if is_final { + unbounded[usize::from(is_dst)] = Some((*first, last_at)); + } + + for (index, _, _) in &onsets[run.start + 1..run.end] { + discard[*index] = true; + } + } + + if let Some(run) = runs.last() + && run.len() == 1 + && let Some((index, _, key)) = onsets.get(run.start) + { + trailing[usize::from(is_dst)] = Some((*index, key.month, key.ordwk, key.weekday)); + } + } + + for is_dst in [false, true] { + let class = usize::from(is_dst); + let (None, Some((other_index, other_at))) = + (unbounded[class], unbounded[usize::from(!is_dst)]) + else { + continue; + }; + + match trailing[class] { + Some((index, month, ordwk, weekday)) + if observances[index].at.saturating_add(TZ_RULE_ACTIVE_SECONDS) >= to => + { + observances[index].rrule = Some(ICalendarRecurrenceRule { + freq: ICalendarFrequency::Yearly, + bymonth: vec![ICalendarMonth::new(month, false)], + byday: vec![ICalendarDay { + ordwk: Some(ordwk), + weekday, + }], + ..Default::default() + }); + unbounded[class] = Some((index, observances[index].at)); + } + _ => { + if let Some(rrule) = observances[other_index].rrule.as_mut() { + rrule.until = Some(PartialDateTime::from_utc_timestamp(other_at)); + unbounded[usize::from(!is_dst)] = None; + } + } + } + } + + let mut index = 0; + observances.retain(|_| { + let keep = !discard[index]; + index += 1; + keep + }); + + observances +} + +fn estimated_observances(from: i64, to: i64) -> usize { + let years = to.saturating_sub(from) / TZ_RULE_ACTIVE_SECONDS; + + (years.clamp(0, 1024) as usize) * 2 + 2 +} + +/// Returns a civil datetime as seconds since the Unix epoch, read as UTC. +fn naive_seconds(date_time: civil::DateTime) -> i64 { + Offset::UTC + .to_timestamp(date_time) + .map_or(0, |timestamp| timestamp.as_second()) +} + +fn round_to_minute(offset: i32) -> i32 { + let minutes = (offset.unsigned_abs() + 30) / 60 * 60; + + if offset < 0 { + -(minutes as i32) + } else { + minutes as i32 + } +} + +fn offset_to_partial(offset: i32) -> PartialDateTime { + let seconds = offset.unsigned_abs(); + + PartialDateTime { + tz_hour: u8::try_from(seconds / 3600).ok(), + tz_minute: u8::try_from((seconds % 3600) / 60).ok(), + tz_minus: offset < 0, + ..Default::default() + } +} + +fn recurrence_span_of(freq: &ICalendarFrequency, count: u32, interval: Option) -> i64 { + let unit = match freq { + ICalendarFrequency::Yearly => 366 * 86400, + ICalendarFrequency::Monthly => 31 * 86400, + ICalendarFrequency::Weekly => 7 * 86400, + ICalendarFrequency::Daily => 86400, + ICalendarFrequency::Hourly => 3600, + ICalendarFrequency::Minutely => 60, + ICalendarFrequency::Secondly => 1, + }; + + i64::from(count) + .saturating_mul(i64::from(interval.unwrap_or(1))) + .saturating_mul(unit) +} + +fn expand_range(range: &mut Option<(i64, i64)>, value: &PartialDateTime) { + let Some(timestamp) = value.to_date_time().map(|result| { + naive_seconds(result.date_time) + - result + .offset + .map_or(0, |offset| i64::from(offset.seconds())) + }) else { + return; + }; + + match range { + Some((min, max)) => { + *min = (*min).min(timestamp); + *max = (*max).max(timestamp); + } + None => { + *range = Some((timestamp, timestamp)); + } + } +} + +fn naive_utc(timestamp: i64) -> civil::DateTime { + Timestamp::from_second(timestamp).map_or(civil::DateTime::default(), |timestamp| { + Offset::UTC.to_datetime(timestamp) + }) +} + +fn year_of(timestamp: i64) -> i32 { + i32::from(naive_utc(timestamp).year()) +} + +fn start_of_year(year: i32) -> i64 { + i16::try_from(year) + .ok() + .and_then(|year| civil::DateTime::new(year, 1, 1, 0, 0, 0, 0).ok()) + .map_or(0, naive_seconds) +} + +fn days_in_month(year: i32, month: u32) -> u32 { + i16::try_from(year) + .ok() + .zip(i8::try_from(month).ok()) + .and_then(|(year, month)| Date::new(year, month, 1).ok()) + .map_or(31, |date| u32::from(date.days_in_month().unsigned_abs())) +} + +fn weekday_of(weekday: Weekday) -> ICalendarWeekday { + match weekday { + Weekday::Monday => ICalendarWeekday::Monday, + Weekday::Tuesday => ICalendarWeekday::Tuesday, + Weekday::Wednesday => ICalendarWeekday::Wednesday, + Weekday::Thursday => ICalendarWeekday::Thursday, + Weekday::Friday => ICalendarWeekday::Friday, + Weekday::Saturday => ICalendarWeekday::Saturday, + Weekday::Sunday => ICalendarWeekday::Sunday, + } +} + diff --git a/crates/calcard/src/icalendar/types.rs b/crates/calcard/src/icalendar/types.rs new file mode 100644 index 00000000..e79dad38 --- /dev/null +++ b/crates/calcard/src/icalendar/types.rs @@ -0,0 +1,1512 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::{Entry, Parser}; +use jiff::civil::Weekday; +use std::borrow::Cow; + +impl IanaParse for ICalendarProperty { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "CALSCALE" => Some(ICalendarProperty::Calscale), + "METHOD" => Some(ICalendarProperty::Method), + "PRODID" => Some(ICalendarProperty::Prodid), + "VERSION" => Some(ICalendarProperty::Version), + "ATTACH" => Some(ICalendarProperty::Attach), + "CATEGORIES" => Some(ICalendarProperty::Categories), + "CLASS" => Some(ICalendarProperty::Class), + "COMMENT" => Some(ICalendarProperty::Comment), + "DESCRIPTION" => Some(ICalendarProperty::Description), + "GEO" => Some(ICalendarProperty::Geo), + "LOCATION" => Some(ICalendarProperty::Location), + "PERCENT-COMPLETE" => Some(ICalendarProperty::PercentComplete), + "PRIORITY" => Some(ICalendarProperty::Priority), + "RESOURCES" => Some(ICalendarProperty::Resources), + "STATUS" => Some(ICalendarProperty::Status), + "SUMMARY" => Some(ICalendarProperty::Summary), + "COMPLETED" => Some(ICalendarProperty::Completed), + "DTEND" => Some(ICalendarProperty::Dtend), + "DUE" => Some(ICalendarProperty::Due), + "DTSTART" => Some(ICalendarProperty::Dtstart), + "DURATION" => Some(ICalendarProperty::Duration), + "FREEBUSY" => Some(ICalendarProperty::Freebusy), + "TRANSP" => Some(ICalendarProperty::Transp), + "TZID" => Some(ICalendarProperty::Tzid), + "TZNAME" => Some(ICalendarProperty::Tzname), + "TZOFFSETFROM" => Some(ICalendarProperty::Tzoffsetfrom), + "TZOFFSETTO" => Some(ICalendarProperty::Tzoffsetto), + "TZURL" => Some(ICalendarProperty::Tzurl), + "ATTENDEE" => Some(ICalendarProperty::Attendee), + "CONTACT" => Some(ICalendarProperty::Contact), + "ORGANIZER" => Some(ICalendarProperty::Organizer), + "RECURRENCE-ID" => Some(ICalendarProperty::RecurrenceId), + "RELATED-TO" => Some(ICalendarProperty::RelatedTo), + "URL" => Some(ICalendarProperty::Url), + "UID" => Some(ICalendarProperty::Uid), + "EXDATE" => Some(ICalendarProperty::Exdate), + "EXRULE" => Some(ICalendarProperty::Exrule), + "RDATE" => Some(ICalendarProperty::Rdate), + "RRULE" => Some(ICalendarProperty::Rrule), + "ACTION" => Some(ICalendarProperty::Action), + "REPEAT" => Some(ICalendarProperty::Repeat), + "TRIGGER" => Some(ICalendarProperty::Trigger), + "CREATED" => Some(ICalendarProperty::Created), + "DTSTAMP" => Some(ICalendarProperty::Dtstamp), + "LAST-MODIFIED" => Some(ICalendarProperty::LastModified), + "SEQUENCE" => Some(ICalendarProperty::Sequence), + "REQUEST-STATUS" => Some(ICalendarProperty::RequestStatus), + "XML" => Some(ICalendarProperty::Xml), + "TZUNTIL" => Some(ICalendarProperty::Tzuntil), + "TZID-ALIAS-OF" => Some(ICalendarProperty::TzidAliasOf), + "BUSYTYPE" => Some(ICalendarProperty::Busytype), + "NAME" => Some(ICalendarProperty::Name), + "REFRESH-INTERVAL" => Some(ICalendarProperty::RefreshInterval), + "SOURCE" => Some(ICalendarProperty::Source), + "COLOR" => Some(ICalendarProperty::Color), + "IMAGE" => Some(ICalendarProperty::Image), + "CONFERENCE" => Some(ICalendarProperty::Conference), + "CALENDAR-ADDRESS" => Some(ICalendarProperty::CalendarAddress), + "LOCATION-TYPE" => Some(ICalendarProperty::LocationType), + "PARTICIPANT-TYPE" => Some(ICalendarProperty::ParticipantType), + "RESOURCE-TYPE" => Some(ICalendarProperty::ResourceType), + "STRUCTURED-DATA" => Some(ICalendarProperty::StructuredData), + "STYLED-DESCRIPTION" => Some(ICalendarProperty::StyledDescription), + "ACKNOWLEDGED" => Some(ICalendarProperty::Acknowledged), + "PROXIMITY" => Some(ICalendarProperty::Proximity), + "CONCEPT" => Some(ICalendarProperty::Concept), + "LINK" => Some(ICalendarProperty::Link), + "REFID" => Some(ICalendarProperty::Refid), + "COORDINATES" => Some(ICalendarProperty::Coordinates), + "SHOW-WITHOUT-TIME" => Some(ICalendarProperty::ShowWithoutTime), + "JSID" => Some(ICalendarProperty::Jsid), + "JSPROP" => Some(ICalendarProperty::Jsprop), + "BEGIN" => Some(ICalendarProperty::Begin), + "END" => Some(ICalendarProperty::End), + "ESTIMATED-DURATION" => Some(ICalendarProperty::EstimatedDuration), + "REASON" => Some(ICalendarProperty::Reason), + "SUBSTATE" => Some(ICalendarProperty::Substate), + "TASK-MODE" => Some(ICalendarProperty::TaskMode), + _ => None, + ) + } +} + +impl ICalendarProperty { + pub fn as_str(&self) -> &str { + match self { + ICalendarProperty::Calscale => "CALSCALE", + ICalendarProperty::Method => "METHOD", + ICalendarProperty::Prodid => "PRODID", + ICalendarProperty::Version => "VERSION", + ICalendarProperty::Attach => "ATTACH", + ICalendarProperty::Categories => "CATEGORIES", + ICalendarProperty::Class => "CLASS", + ICalendarProperty::Comment => "COMMENT", + ICalendarProperty::Description => "DESCRIPTION", + ICalendarProperty::Geo => "GEO", + ICalendarProperty::Location => "LOCATION", + ICalendarProperty::PercentComplete => "PERCENT-COMPLETE", + ICalendarProperty::Priority => "PRIORITY", + ICalendarProperty::Resources => "RESOURCES", + ICalendarProperty::Status => "STATUS", + ICalendarProperty::Summary => "SUMMARY", + ICalendarProperty::Completed => "COMPLETED", + ICalendarProperty::Dtend => "DTEND", + ICalendarProperty::Due => "DUE", + ICalendarProperty::Dtstart => "DTSTART", + ICalendarProperty::Duration => "DURATION", + ICalendarProperty::Freebusy => "FREEBUSY", + ICalendarProperty::Transp => "TRANSP", + ICalendarProperty::Tzid => "TZID", + ICalendarProperty::Tzname => "TZNAME", + ICalendarProperty::Tzoffsetfrom => "TZOFFSETFROM", + ICalendarProperty::Tzoffsetto => "TZOFFSETTO", + ICalendarProperty::Tzurl => "TZURL", + ICalendarProperty::Attendee => "ATTENDEE", + ICalendarProperty::Contact => "CONTACT", + ICalendarProperty::Organizer => "ORGANIZER", + ICalendarProperty::RecurrenceId => "RECURRENCE-ID", + ICalendarProperty::RelatedTo => "RELATED-TO", + ICalendarProperty::Url => "URL", + ICalendarProperty::Uid => "UID", + ICalendarProperty::Exdate => "EXDATE", + ICalendarProperty::Exrule => "EXRULE", + ICalendarProperty::Rdate => "RDATE", + ICalendarProperty::Rrule => "RRULE", + ICalendarProperty::Action => "ACTION", + ICalendarProperty::Repeat => "REPEAT", + ICalendarProperty::Trigger => "TRIGGER", + ICalendarProperty::Created => "CREATED", + ICalendarProperty::Dtstamp => "DTSTAMP", + ICalendarProperty::LastModified => "LAST-MODIFIED", + ICalendarProperty::Sequence => "SEQUENCE", + ICalendarProperty::RequestStatus => "REQUEST-STATUS", + ICalendarProperty::Xml => "XML", + ICalendarProperty::Tzuntil => "TZUNTIL", + ICalendarProperty::TzidAliasOf => "TZID-ALIAS-OF", + ICalendarProperty::Busytype => "BUSYTYPE", + ICalendarProperty::Name => "NAME", + ICalendarProperty::RefreshInterval => "REFRESH-INTERVAL", + ICalendarProperty::Source => "SOURCE", + ICalendarProperty::Color => "COLOR", + ICalendarProperty::Image => "IMAGE", + ICalendarProperty::Conference => "CONFERENCE", + ICalendarProperty::CalendarAddress => "CALENDAR-ADDRESS", + ICalendarProperty::LocationType => "LOCATION-TYPE", + ICalendarProperty::ParticipantType => "PARTICIPANT-TYPE", + ICalendarProperty::ResourceType => "RESOURCE-TYPE", + ICalendarProperty::StructuredData => "STRUCTURED-DATA", + ICalendarProperty::StyledDescription => "STYLED-DESCRIPTION", + ICalendarProperty::Acknowledged => "ACKNOWLEDGED", + ICalendarProperty::Proximity => "PROXIMITY", + ICalendarProperty::Concept => "CONCEPT", + ICalendarProperty::Link => "LINK", + ICalendarProperty::Refid => "REFID", + ICalendarProperty::Begin => "BEGIN", + ICalendarProperty::End => "END", + ICalendarProperty::Coordinates => "COORDINATES", + ICalendarProperty::ShowWithoutTime => "SHOW-WITHOUT-TIME", + ICalendarProperty::Jsid => "JSID", + ICalendarProperty::Jsprop => "JSPROP", + ICalendarProperty::EstimatedDuration => "ESTIMATED-DURATION", + ICalendarProperty::Reason => "REASON", + ICalendarProperty::Substate => "SUBSTATE", + ICalendarProperty::TaskMode => "TASK-MODE", + ICalendarProperty::Other(s) => s.as_str(), + } + } + + pub fn into_string(self) -> Cow<'static, str> { + match self { + ICalendarProperty::Calscale => "CALSCALE", + ICalendarProperty::Method => "METHOD", + ICalendarProperty::Prodid => "PRODID", + ICalendarProperty::Version => "VERSION", + ICalendarProperty::Attach => "ATTACH", + ICalendarProperty::Categories => "CATEGORIES", + ICalendarProperty::Class => "CLASS", + ICalendarProperty::Comment => "COMMENT", + ICalendarProperty::Description => "DESCRIPTION", + ICalendarProperty::Geo => "GEO", + ICalendarProperty::Location => "LOCATION", + ICalendarProperty::PercentComplete => "PERCENT-COMPLETE", + ICalendarProperty::Priority => "PRIORITY", + ICalendarProperty::Resources => "RESOURCES", + ICalendarProperty::Status => "STATUS", + ICalendarProperty::Summary => "SUMMARY", + ICalendarProperty::Completed => "COMPLETED", + ICalendarProperty::Dtend => "DTEND", + ICalendarProperty::Due => "DUE", + ICalendarProperty::Dtstart => "DTSTART", + ICalendarProperty::Duration => "DURATION", + ICalendarProperty::Freebusy => "FREEBUSY", + ICalendarProperty::Transp => "TRANSP", + ICalendarProperty::Tzid => "TZID", + ICalendarProperty::Tzname => "TZNAME", + ICalendarProperty::Tzoffsetfrom => "TZOFFSETFROM", + ICalendarProperty::Tzoffsetto => "TZOFFSETTO", + ICalendarProperty::Tzurl => "TZURL", + ICalendarProperty::Attendee => "ATTENDEE", + ICalendarProperty::Contact => "CONTACT", + ICalendarProperty::Organizer => "ORGANIZER", + ICalendarProperty::RecurrenceId => "RECURRENCE-ID", + ICalendarProperty::RelatedTo => "RELATED-TO", + ICalendarProperty::Url => "URL", + ICalendarProperty::Uid => "UID", + ICalendarProperty::Exdate => "EXDATE", + ICalendarProperty::Exrule => "EXRULE", + ICalendarProperty::Rdate => "RDATE", + ICalendarProperty::Rrule => "RRULE", + ICalendarProperty::Action => "ACTION", + ICalendarProperty::Repeat => "REPEAT", + ICalendarProperty::Trigger => "TRIGGER", + ICalendarProperty::Created => "CREATED", + ICalendarProperty::Dtstamp => "DTSTAMP", + ICalendarProperty::LastModified => "LAST-MODIFIED", + ICalendarProperty::Sequence => "SEQUENCE", + ICalendarProperty::RequestStatus => "REQUEST-STATUS", + ICalendarProperty::Xml => "XML", + ICalendarProperty::Tzuntil => "TZUNTIL", + ICalendarProperty::TzidAliasOf => "TZID-ALIAS-OF", + ICalendarProperty::Busytype => "BUSYTYPE", + ICalendarProperty::Name => "NAME", + ICalendarProperty::RefreshInterval => "REFRESH-INTERVAL", + ICalendarProperty::Source => "SOURCE", + ICalendarProperty::Color => "COLOR", + ICalendarProperty::Image => "IMAGE", + ICalendarProperty::Conference => "CONFERENCE", + ICalendarProperty::CalendarAddress => "CALENDAR-ADDRESS", + ICalendarProperty::LocationType => "LOCATION-TYPE", + ICalendarProperty::ParticipantType => "PARTICIPANT-TYPE", + ICalendarProperty::ResourceType => "RESOURCE-TYPE", + ICalendarProperty::StructuredData => "STRUCTURED-DATA", + ICalendarProperty::StyledDescription => "STYLED-DESCRIPTION", + ICalendarProperty::Acknowledged => "ACKNOWLEDGED", + ICalendarProperty::Proximity => "PROXIMITY", + ICalendarProperty::Concept => "CONCEPT", + ICalendarProperty::Link => "LINK", + ICalendarProperty::Refid => "REFID", + ICalendarProperty::Begin => "BEGIN", + ICalendarProperty::End => "END", + ICalendarProperty::Coordinates => "COORDINATES", + ICalendarProperty::ShowWithoutTime => "SHOW-WITHOUT-TIME", + ICalendarProperty::Jsid => "JSID", + ICalendarProperty::Jsprop => "JSPROP", + ICalendarProperty::EstimatedDuration => "ESTIMATED-DURATION", + ICalendarProperty::Reason => "REASON", + ICalendarProperty::Substate => "SUBSTATE", + ICalendarProperty::TaskMode => "TASK-MODE", + ICalendarProperty::Other(s) => return Cow::Owned(s), + } + .into() + } +} + +impl ICalendarParameterName { + pub fn try_parse(input: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(input, + b"ALTREP" => Some(ICalendarParameterName::Altrep), + b"CN" => Some(ICalendarParameterName::Cn), + b"CUTYPE" => Some(ICalendarParameterName::Cutype), + b"DELEGATED-FROM" => Some(ICalendarParameterName::DelegatedFrom), + b"DELEGATED-TO" => Some(ICalendarParameterName::DelegatedTo), + b"DIR" => Some(ICalendarParameterName::Dir), + b"FMTTYPE" => Some(ICalendarParameterName::Fmttype), + b"FBTYPE" => Some(ICalendarParameterName::Fbtype), + b"LANGUAGE" => Some(ICalendarParameterName::Language), + b"MEMBER" => Some(ICalendarParameterName::Member), + b"PARTSTAT" => Some(ICalendarParameterName::Partstat), + b"RANGE" => Some(ICalendarParameterName::Range), + b"RELATED" => Some(ICalendarParameterName::Related), + b"RELTYPE" => Some(ICalendarParameterName::Reltype), + b"ROLE" => Some(ICalendarParameterName::Role), + b"RSVP" => Some(ICalendarParameterName::Rsvp), + b"SCHEDULE-AGENT" => Some(ICalendarParameterName::ScheduleAgent), + b"SCHEDULE-FORCE-SEND" => Some(ICalendarParameterName::ScheduleForceSend), + b"SCHEDULE-STATUS" => Some(ICalendarParameterName::ScheduleStatus), + b"SENT-BY" => Some(ICalendarParameterName::SentBy), + b"TZID" => Some(ICalendarParameterName::Tzid), + b"VALUE" => Some(ICalendarParameterName::Value), + b"DISPLAY" => Some(ICalendarParameterName::Display), + b"EMAIL" => Some(ICalendarParameterName::Email), + b"FEATURE" => Some(ICalendarParameterName::Feature), + b"LABEL" => Some(ICalendarParameterName::Label), + b"SIZE" => Some(ICalendarParameterName::Size), + b"FILENAME" => Some(ICalendarParameterName::Filename), + b"MANAGED-ID" => Some(ICalendarParameterName::ManagedId), + b"ORDER" => Some(ICalendarParameterName::Order), + b"SCHEMA" => Some(ICalendarParameterName::Schema), + b"DERIVED" => Some(ICalendarParameterName::Derived), + b"GAP" => Some(ICalendarParameterName::Gap), + b"LINKREL" => Some(ICalendarParameterName::Linkrel), + b"JSPTR" => Some(ICalendarParameterName::Jsptr), + b"JSID" => Some(ICalendarParameterName::Jsid), + _ => None, + ) + } + + pub fn as_str(&self) -> &str { + match self { + ICalendarParameterName::Altrep => "ALTREP", + ICalendarParameterName::Cn => "CN", + ICalendarParameterName::Cutype => "CUTYPE", + ICalendarParameterName::DelegatedFrom => "DELEGATED-FROM", + ICalendarParameterName::DelegatedTo => "DELEGATED-TO", + ICalendarParameterName::Dir => "DIR", + ICalendarParameterName::Fmttype => "FMTTYPE", + ICalendarParameterName::Fbtype => "FBTYPE", + ICalendarParameterName::Language => "LANGUAGE", + ICalendarParameterName::Member => "MEMBER", + ICalendarParameterName::Partstat => "PARTSTAT", + ICalendarParameterName::Range => "RANGE", + ICalendarParameterName::Related => "RELATED", + ICalendarParameterName::Reltype => "RELTYPE", + ICalendarParameterName::Role => "ROLE", + ICalendarParameterName::Rsvp => "RSVP", + ICalendarParameterName::ScheduleAgent => "SCHEDULE-AGENT", + ICalendarParameterName::ScheduleForceSend => "SCHEDULE-FORCE-SEND", + ICalendarParameterName::ScheduleStatus => "SCHEDULE-STATUS", + ICalendarParameterName::SentBy => "SENT-BY", + ICalendarParameterName::Tzid => "TZID", + ICalendarParameterName::Value => "VALUE", + ICalendarParameterName::Display => "DISPLAY", + ICalendarParameterName::Email => "EMAIL", + ICalendarParameterName::Feature => "FEATURE", + ICalendarParameterName::Label => "LABEL", + ICalendarParameterName::Size => "SIZE", + ICalendarParameterName::Filename => "FILENAME", + ICalendarParameterName::ManagedId => "MANAGED-ID", + ICalendarParameterName::Order => "ORDER", + ICalendarParameterName::Schema => "SCHEMA", + ICalendarParameterName::Derived => "DERIVED", + ICalendarParameterName::Gap => "GAP", + ICalendarParameterName::Linkrel => "LINKREL", + ICalendarParameterName::Jsptr => "JSPTR", + ICalendarParameterName::Jsid => "JSID", + ICalendarParameterName::Other(name) => name.as_str(), + } + } + + pub fn into_string(self) -> Cow<'static, str> { + match self { + ICalendarParameterName::Altrep => "ALTREP", + ICalendarParameterName::Cn => "CN", + ICalendarParameterName::Cutype => "CUTYPE", + ICalendarParameterName::DelegatedFrom => "DELEGATED-FROM", + ICalendarParameterName::DelegatedTo => "DELEGATED-TO", + ICalendarParameterName::Dir => "DIR", + ICalendarParameterName::Fmttype => "FMTTYPE", + ICalendarParameterName::Fbtype => "FBTYPE", + ICalendarParameterName::Language => "LANGUAGE", + ICalendarParameterName::Member => "MEMBER", + ICalendarParameterName::Partstat => "PARTSTAT", + ICalendarParameterName::Range => "RANGE", + ICalendarParameterName::Related => "RELATED", + ICalendarParameterName::Reltype => "RELTYPE", + ICalendarParameterName::Role => "ROLE", + ICalendarParameterName::Rsvp => "RSVP", + ICalendarParameterName::ScheduleAgent => "SCHEDULE-AGENT", + ICalendarParameterName::ScheduleForceSend => "SCHEDULE-FORCE-SEND", + ICalendarParameterName::ScheduleStatus => "SCHEDULE-STATUS", + ICalendarParameterName::SentBy => "SENT-BY", + ICalendarParameterName::Tzid => "TZID", + ICalendarParameterName::Value => "VALUE", + ICalendarParameterName::Display => "DISPLAY", + ICalendarParameterName::Email => "EMAIL", + ICalendarParameterName::Feature => "FEATURE", + ICalendarParameterName::Label => "LABEL", + ICalendarParameterName::Size => "SIZE", + ICalendarParameterName::Filename => "FILENAME", + ICalendarParameterName::ManagedId => "MANAGED-ID", + ICalendarParameterName::Order => "ORDER", + ICalendarParameterName::Schema => "SCHEMA", + ICalendarParameterName::Derived => "DERIVED", + ICalendarParameterName::Gap => "GAP", + ICalendarParameterName::Linkrel => "LINKREL", + ICalendarParameterName::Jsptr => "JSPTR", + ICalendarParameterName::Jsid => "JSID", + ICalendarParameterName::Other(name) => return Cow::Owned(name), + } + .into() + } + + pub fn parse(input: &str) -> Self { + Self::try_parse(input.as_bytes()) + .unwrap_or_else(|| ICalendarParameterName::Other(input.to_string())) + } +} + +impl IanaParse for ICalendarFrequency { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, ICalendarFrequency, + b"SECONDLY" => ICalendarFrequency::Secondly, + b"MINUTELY" => ICalendarFrequency::Minutely, + b"HOURLY" => ICalendarFrequency::Hourly, + b"DAILY" => ICalendarFrequency::Daily, + b"WEEKLY" => ICalendarFrequency::Weekly, + b"MONTHLY" => ICalendarFrequency::Monthly, + b"YEARLY" => ICalendarFrequency::Yearly, + ) + .copied() + } +} + +impl IanaString for ICalendarFrequency { + fn as_str(&self) -> &'static str { + match self { + ICalendarFrequency::Secondly => "SECONDLY", + ICalendarFrequency::Minutely => "MINUTELY", + ICalendarFrequency::Hourly => "HOURLY", + ICalendarFrequency::Daily => "DAILY", + ICalendarFrequency::Weekly => "WEEKLY", + ICalendarFrequency::Monthly => "MONTHLY", + ICalendarFrequency::Yearly => "YEARLY", + } + } +} + +impl IanaParse for ICalendarSkip { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, ICalendarSkip, + b"OMIT" => ICalendarSkip::Omit, + b"BACKWARD" => ICalendarSkip::Backward, + b"FORWARD" => ICalendarSkip::Forward, + ) + .copied() + } +} + +impl IanaString for ICalendarSkip { + fn as_str(&self) -> &'static str { + match self { + ICalendarSkip::Omit => "OMIT", + ICalendarSkip::Backward => "BACKWARD", + ICalendarSkip::Forward => "FORWARD", + } + } +} + +impl IanaParse for ICalendarWeekday { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, ICalendarWeekday, + b"SU" => ICalendarWeekday::Sunday, + b"MO" => ICalendarWeekday::Monday, + b"TU" => ICalendarWeekday::Tuesday, + b"WE" => ICalendarWeekday::Wednesday, + b"TH" => ICalendarWeekday::Thursday, + b"FR" => ICalendarWeekday::Friday, + b"SA" => ICalendarWeekday::Saturday, + ) + .copied() + } +} + +impl IanaString for ICalendarWeekday { + fn as_str(&self) -> &'static str { + match self { + ICalendarWeekday::Sunday => "SU", + ICalendarWeekday::Monday => "MO", + ICalendarWeekday::Tuesday => "TU", + ICalendarWeekday::Wednesday => "WE", + ICalendarWeekday::Thursday => "TH", + ICalendarWeekday::Friday => "FR", + ICalendarWeekday::Saturday => "SA", + } + } +} + +impl From for Weekday { + fn from(value: ICalendarWeekday) -> Self { + match value { + ICalendarWeekday::Sunday => Weekday::Sunday, + ICalendarWeekday::Monday => Weekday::Monday, + ICalendarWeekday::Tuesday => Weekday::Tuesday, + ICalendarWeekday::Wednesday => Weekday::Wednesday, + ICalendarWeekday::Thursday => Weekday::Thursday, + ICalendarWeekday::Friday => Weekday::Friday, + ICalendarWeekday::Saturday => Weekday::Saturday, + } + } +} + +impl IanaParse for ICalendarAction { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "AUDIO" => Some(ICalendarAction::Audio), + "DISPLAY" => Some(ICalendarAction::Display), + "EMAIL" => Some(ICalendarAction::Email), + "PROCEDURE" => Some(ICalendarAction::Procedure), + _ => None, + ) + } +} + +impl IanaString for ICalendarAction { + fn as_str(&self) -> &'static str { + match self { + ICalendarAction::Audio => "AUDIO", + ICalendarAction::Display => "DISPLAY", + ICalendarAction::Email => "EMAIL", + ICalendarAction::Procedure => "PROCEDURE", + } + } +} + +impl IanaParse for ICalendarUserTypes { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "INDIVIDUAL" => Some(ICalendarUserTypes::Individual), + "GROUP" => Some(ICalendarUserTypes::Group), + "RESOURCE" => Some(ICalendarUserTypes::Resource), + "ROOM" => Some(ICalendarUserTypes::Room), + "UNKNOWN" => Some(ICalendarUserTypes::Unknown), + _ => None, + ) + } +} + +impl IanaString for ICalendarUserTypes { + fn as_str(&self) -> &'static str { + match self { + ICalendarUserTypes::Individual => "INDIVIDUAL", + ICalendarUserTypes::Group => "GROUP", + ICalendarUserTypes::Resource => "RESOURCE", + ICalendarUserTypes::Room => "ROOM", + ICalendarUserTypes::Unknown => "UNKNOWN", + } + } +} + +impl IanaParse for ICalendarClassification { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "PUBLIC" => Some(ICalendarClassification::Public), + "PRIVATE" => Some(ICalendarClassification::Private), + "CONFIDENTIAL" => Some(ICalendarClassification::Confidential), + _ => None, + ) + } +} + +impl IanaString for ICalendarClassification { + fn as_str(&self) -> &'static str { + match self { + ICalendarClassification::Public => "PUBLIC", + ICalendarClassification::Private => "PRIVATE", + ICalendarClassification::Confidential => "CONFIDENTIAL", + } + } +} + +impl IanaParse for ICalendarComponentType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "VCALENDAR" => Some(ICalendarComponentType::VCalendar), + "VEVENT" => Some(ICalendarComponentType::VEvent), + "VTODO" => Some(ICalendarComponentType::VTodo), + "VJOURNAL" => Some(ICalendarComponentType::VJournal), + "VFREEBUSY" => Some(ICalendarComponentType::VFreebusy), + "VTIMEZONE" => Some(ICalendarComponentType::VTimezone), + "VALARM" => Some(ICalendarComponentType::VAlarm), + "STANDARD" => Some(ICalendarComponentType::Standard), + "DAYLIGHT" => Some(ICalendarComponentType::Daylight), + "VAVAILABILITY" => Some(ICalendarComponentType::VAvailability), + "AVAILABLE" => Some(ICalendarComponentType::Available), + "PARTICIPANT" => Some(ICalendarComponentType::Participant), + "VLOCATION" => Some(ICalendarComponentType::VLocation), + "VRESOURCE" => Some(ICalendarComponentType::VResource), + "VSTATUS" => Some(ICalendarComponentType::VStatus), + _ => None + ) + } +} + +impl ICalendarComponentType { + pub(crate) fn entry_capacity(&self) -> usize { + match self { + ICalendarComponentType::VCalendar + | ICalendarComponentType::VTimezone + | ICalendarComponentType::VAlarm + | ICalendarComponentType::Standard + | ICalendarComponentType::Daylight + | ICalendarComponentType::Available => 8, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + | ICalendarComponentType::VAvailability + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VResource + | ICalendarComponentType::VStatus + | ICalendarComponentType::Other(_) => 16, + } + } + + pub fn as_str(&self) -> &str { + match self { + ICalendarComponentType::VCalendar => "VCALENDAR", + ICalendarComponentType::VEvent => "VEVENT", + ICalendarComponentType::VTodo => "VTODO", + ICalendarComponentType::VJournal => "VJOURNAL", + ICalendarComponentType::VFreebusy => "VFREEBUSY", + ICalendarComponentType::VTimezone => "VTIMEZONE", + ICalendarComponentType::VAlarm => "VALARM", + ICalendarComponentType::Standard => "STANDARD", + ICalendarComponentType::Daylight => "DAYLIGHT", + ICalendarComponentType::VAvailability => "VAVAILABILITY", + ICalendarComponentType::Available => "AVAILABLE", + ICalendarComponentType::Participant => "PARTICIPANT", + ICalendarComponentType::VLocation => "VLOCATION", + ICalendarComponentType::VResource => "VRESOURCE", + ICalendarComponentType::VStatus => "VSTATUS", + ICalendarComponentType::Other(s) => s.as_str(), + } + } + + pub fn into_string(self) -> Cow<'static, str> { + match self { + ICalendarComponentType::VCalendar => "VCALENDAR", + ICalendarComponentType::VEvent => "VEVENT", + ICalendarComponentType::VTodo => "VTODO", + ICalendarComponentType::VJournal => "VJOURNAL", + ICalendarComponentType::VFreebusy => "VFREEBUSY", + ICalendarComponentType::VTimezone => "VTIMEZONE", + ICalendarComponentType::VAlarm => "VALARM", + ICalendarComponentType::Standard => "STANDARD", + ICalendarComponentType::Daylight => "DAYLIGHT", + ICalendarComponentType::VAvailability => "VAVAILABILITY", + ICalendarComponentType::Available => "AVAILABLE", + ICalendarComponentType::Participant => "PARTICIPANT", + ICalendarComponentType::VLocation => "VLOCATION", + ICalendarComponentType::VResource => "VRESOURCE", + ICalendarComponentType::VStatus => "VSTATUS", + ICalendarComponentType::Other(s) => return Cow::Owned(s), + } + .into() + } +} + +impl IanaParse for ICalendarDisplayType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "BADGE" => Some(ICalendarDisplayType::Badge), + "GRAPHIC" => Some(ICalendarDisplayType::Graphic), + "FULLSIZE" => Some(ICalendarDisplayType::Fullsize), + "THUMBNAIL" => Some(ICalendarDisplayType::Thumbnail), + _ => None, + ) + } +} + +impl IanaString for ICalendarDisplayType { + fn as_str(&self) -> &'static str { + match self { + ICalendarDisplayType::Badge => "BADGE", + ICalendarDisplayType::Graphic => "GRAPHIC", + ICalendarDisplayType::Fullsize => "FULLSIZE", + ICalendarDisplayType::Thumbnail => "THUMBNAIL", + } + } +} + +impl IanaParse for ICalendarFeatureType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "AUDIO" => Some(ICalendarFeatureType::Audio), + "CHAT" => Some(ICalendarFeatureType::Chat), + "FEED" => Some(ICalendarFeatureType::Feed), + "MODERATOR" => Some(ICalendarFeatureType::Moderator), + "PHONE" => Some(ICalendarFeatureType::Phone), + "SCREEN" => Some(ICalendarFeatureType::Screen), + "VIDEO" => Some(ICalendarFeatureType::Video), + _ => None, + ) + } +} + +impl IanaString for ICalendarFeatureType { + fn as_str(&self) -> &'static str { + match self { + ICalendarFeatureType::Audio => "AUDIO", + ICalendarFeatureType::Chat => "CHAT", + ICalendarFeatureType::Feed => "FEED", + ICalendarFeatureType::Moderator => "MODERATOR", + ICalendarFeatureType::Phone => "PHONE", + ICalendarFeatureType::Screen => "SCREEN", + ICalendarFeatureType::Video => "VIDEO", + } + } +} + +impl IanaParse for ICalendarFreeBusyType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "FREE" => Some(ICalendarFreeBusyType::Free), + "BUSY" => Some(ICalendarFreeBusyType::Busy), + "BUSY-UNAVAILABLE" => Some(ICalendarFreeBusyType::BusyUnavailable), + "BUSY-TENTATIVE" => Some(ICalendarFreeBusyType::BusyTentative), + _ => None, + ) + } +} + +impl IanaString for ICalendarFreeBusyType { + fn as_str(&self) -> &'static str { + match self { + ICalendarFreeBusyType::Free => "FREE", + ICalendarFreeBusyType::Busy => "BUSY", + ICalendarFreeBusyType::BusyUnavailable => "BUSY-UNAVAILABLE", + ICalendarFreeBusyType::BusyTentative => "BUSY-TENTATIVE", + } + } +} + +impl IanaParse for ICalendarMethod { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "PUBLISH" => Some(ICalendarMethod::Publish), + "REQUEST" => Some(ICalendarMethod::Request), + "REPLY" => Some(ICalendarMethod::Reply), + "ADD" => Some(ICalendarMethod::Add), + "CANCEL" => Some(ICalendarMethod::Cancel), + "REFRESH" => Some(ICalendarMethod::Refresh), + "COUNTER" => Some(ICalendarMethod::Counter), + "DECLINECOUNTER" => Some(ICalendarMethod::Declinecounter), + _ => None, + ) + } +} + +impl IanaString for ICalendarMethod { + fn as_str(&self) -> &'static str { + match self { + ICalendarMethod::Publish => "PUBLISH", + ICalendarMethod::Request => "REQUEST", + ICalendarMethod::Reply => "REPLY", + ICalendarMethod::Add => "ADD", + ICalendarMethod::Cancel => "CANCEL", + ICalendarMethod::Refresh => "REFRESH", + ICalendarMethod::Counter => "COUNTER", + ICalendarMethod::Declinecounter => "DECLINECOUNTER", + } + } +} + +impl IanaParse for ICalendarRelated { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "START" => Some(ICalendarRelated::Start), + "END" => Some(ICalendarRelated::End), + _ => None, + ) + } +} + +impl IanaString for ICalendarRelated { + fn as_str(&self) -> &'static str { + match self { + ICalendarRelated::Start => "START", + ICalendarRelated::End => "END", + } + } +} + +impl IanaParse for ICalendarParticipantType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "ACTIVE" => Some(ICalendarParticipantType::Active), + "INACTIVE" => Some(ICalendarParticipantType::Inactive), + "SPONSOR" => Some(ICalendarParticipantType::Sponsor), + "CONTACT" => Some(ICalendarParticipantType::Contact), + "BOOKING-CONTACT" => Some(ICalendarParticipantType::BookingContact), + "EMERGENCY-CONTACT" => Some(ICalendarParticipantType::EmergencyContact), + "PUBLICITY-CONTACT" => Some(ICalendarParticipantType::PublicityContact), + "PLANNER-CONTACT" => Some(ICalendarParticipantType::PlannerContact), + "PERFORMER" => Some(ICalendarParticipantType::Performer), + "SPEAKER" => Some(ICalendarParticipantType::Speaker), + _ => None, + ) + } +} + +impl IanaString for ICalendarParticipantType { + fn as_str(&self) -> &'static str { + match self { + ICalendarParticipantType::Active => "ACTIVE", + ICalendarParticipantType::Inactive => "INACTIVE", + ICalendarParticipantType::Sponsor => "SPONSOR", + ICalendarParticipantType::Contact => "CONTACT", + ICalendarParticipantType::BookingContact => "BOOKING-CONTACT", + ICalendarParticipantType::EmergencyContact => "EMERGENCY-CONTACT", + ICalendarParticipantType::PublicityContact => "PUBLICITY-CONTACT", + ICalendarParticipantType::PlannerContact => "PLANNER-CONTACT", + ICalendarParticipantType::Performer => "PERFORMER", + ICalendarParticipantType::Speaker => "SPEAKER", + } + } +} + +impl IanaParse for ICalendarParticipationRole { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "CHAIR" => Some(ICalendarParticipationRole::Chair), + "REQ-PARTICIPANT" => Some(ICalendarParticipationRole::ReqParticipant), + "OPT-PARTICIPANT" => Some(ICalendarParticipationRole::OptParticipant), + "NON-PARTICIPANT" => Some(ICalendarParticipationRole::NonParticipant), + "OWNER" => Some(ICalendarParticipationRole::Owner), + _ => None, + ) + } +} + +impl IanaString for ICalendarParticipationRole { + fn as_str(&self) -> &'static str { + match self { + ICalendarParticipationRole::Chair => "CHAIR", + ICalendarParticipationRole::ReqParticipant => "REQ-PARTICIPANT", + ICalendarParticipationRole::OptParticipant => "OPT-PARTICIPANT", + ICalendarParticipationRole::NonParticipant => "NON-PARTICIPANT", + ICalendarParticipationRole::Owner => "OWNER", + } + } +} + +impl IanaParse for ICalendarStatus { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "TENTATIVE" => Some(ICalendarStatus::Tentative), + "CONFIRMED" => Some(ICalendarStatus::Confirmed), + "CANCELLED" => Some(ICalendarStatus::Cancelled), + "NEEDS-ACTION" => Some(ICalendarStatus::NeedsAction), + "COMPLETED" => Some(ICalendarStatus::Completed), + "IN-PROCESS" => Some(ICalendarStatus::InProcess), + "DRAFT" => Some(ICalendarStatus::Draft), + "FINAL" => Some(ICalendarStatus::Final), + "FAILED" => Some(ICalendarStatus::Failed), + "PENDING" => Some(ICalendarStatus::Pending), + _ => None + ) + } +} + +impl IanaString for ICalendarStatus { + fn as_str(&self) -> &'static str { + match self { + ICalendarStatus::Tentative => "TENTATIVE", + ICalendarStatus::Confirmed => "CONFIRMED", + ICalendarStatus::Cancelled => "CANCELLED", + ICalendarStatus::NeedsAction => "NEEDS-ACTION", + ICalendarStatus::Completed => "COMPLETED", + ICalendarStatus::InProcess => "IN-PROCESS", + ICalendarStatus::Draft => "DRAFT", + ICalendarStatus::Final => "FINAL", + ICalendarStatus::Failed => "FAILED", + ICalendarStatus::Pending => "PENDING", + } + } +} + +impl IanaParse for ICalendarParticipationStatus { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "NEEDS-ACTION" => Some(ICalendarParticipationStatus::NeedsAction), + "ACCEPTED" => Some(ICalendarParticipationStatus::Accepted), + "DECLINED" => Some(ICalendarParticipationStatus::Declined), + "TENTATIVE" => Some(ICalendarParticipationStatus::Tentative), + "DELEGATED" => Some(ICalendarParticipationStatus::Delegated), + "COMPLETED" => Some(ICalendarParticipationStatus::Completed), + "IN-PROCESS" => Some(ICalendarParticipationStatus::InProcess), + "FAILED" => Some(ICalendarParticipationStatus::Failed), + _ => None + ) + } +} + +impl IanaString for ICalendarParticipationStatus { + fn as_str(&self) -> &'static str { + match self { + ICalendarParticipationStatus::NeedsAction => "NEEDS-ACTION", + ICalendarParticipationStatus::Accepted => "ACCEPTED", + ICalendarParticipationStatus::Declined => "DECLINED", + ICalendarParticipationStatus::Tentative => "TENTATIVE", + ICalendarParticipationStatus::Delegated => "DELEGATED", + ICalendarParticipationStatus::Completed => "COMPLETED", + ICalendarParticipationStatus::InProcess => "IN-PROCESS", + ICalendarParticipationStatus::Failed => "FAILED", + } + } +} + +impl IanaParse for ICalendarProximityValue { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "ARRIVE" => Some(ICalendarProximityValue::Arrive), + "DEPART" => Some(ICalendarProximityValue::Depart), + "CONNECT" => Some(ICalendarProximityValue::Connect), + "DISCONNECT" => Some(ICalendarProximityValue::Disconnect), + _ => None, + ) + } +} + +impl IanaString for ICalendarProximityValue { + fn as_str(&self) -> &'static str { + match self { + ICalendarProximityValue::Arrive => "ARRIVE", + ICalendarProximityValue::Depart => "DEPART", + ICalendarProximityValue::Connect => "CONNECT", + ICalendarProximityValue::Disconnect => "DISCONNECT", + } + } +} + +impl IanaParse for ICalendarRelationshipType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "CHILD" => Some(ICalendarRelationshipType::Child), + "PARENT" => Some(ICalendarRelationshipType::Parent), + "SIBLING" => Some(ICalendarRelationshipType::Sibling), + "SNOOZE" => Some(ICalendarRelationshipType::Snooze), + "CONCEPT" => Some(ICalendarRelationshipType::Concept), + "DEPENDS-ON" => Some(ICalendarRelationshipType::DependsOn), + "FINISHTOFINISH" => Some(ICalendarRelationshipType::Finishtofinish), + "FINISHTOSTART" => Some(ICalendarRelationshipType::Finishtostart), + "FIRST" => Some(ICalendarRelationshipType::First), + "NEXT" => Some(ICalendarRelationshipType::Next), + "REFID" => Some(ICalendarRelationshipType::Refid), + "STARTTOFINISH" => Some(ICalendarRelationshipType::Starttofinish), + "STARTTOSTART" => Some(ICalendarRelationshipType::Starttostart), + _ => None, + ) + } +} + +impl IanaString for ICalendarRelationshipType { + fn as_str(&self) -> &'static str { + match self { + ICalendarRelationshipType::Child => "CHILD", + ICalendarRelationshipType::Parent => "PARENT", + ICalendarRelationshipType::Sibling => "SIBLING", + ICalendarRelationshipType::Snooze => "SNOOZE", + ICalendarRelationshipType::Concept => "CONCEPT", + ICalendarRelationshipType::DependsOn => "DEPENDS-ON", + ICalendarRelationshipType::Finishtofinish => "FINISHTOFINISH", + ICalendarRelationshipType::Finishtostart => "FINISHTOSTART", + ICalendarRelationshipType::First => "FIRST", + ICalendarRelationshipType::Next => "NEXT", + ICalendarRelationshipType::Refid => "REFID", + ICalendarRelationshipType::Starttofinish => "STARTTOFINISH", + ICalendarRelationshipType::Starttostart => "STARTTOSTART", + } + } +} + +impl IanaParse for ICalendarResourceType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "PROJECTOR" => Some(ICalendarResourceType::Projector), + "ROOM" => Some(ICalendarResourceType::Room), + "REMOTE-CONFERENCE-AUDIO" => Some(ICalendarResourceType::RemoteConferenceAudio), + "REMOTE-CONFERENCE-VIDEO" => Some(ICalendarResourceType::RemoteConferenceVideo), + _ => None, + ) + } +} + +impl IanaString for ICalendarResourceType { + fn as_str(&self) -> &'static str { + match self { + ICalendarResourceType::Projector => "PROJECTOR", + ICalendarResourceType::Room => "ROOM", + ICalendarResourceType::RemoteConferenceAudio => "REMOTE-CONFERENCE-AUDIO", + ICalendarResourceType::RemoteConferenceVideo => "REMOTE-CONFERENCE-VIDEO", + } + } +} + +impl IanaParse for ICalendarScheduleAgentValue { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "SERVER" => Some(ICalendarScheduleAgentValue::Server), + "CLIENT" => Some(ICalendarScheduleAgentValue::Client), + "NONE" => Some(ICalendarScheduleAgentValue::None), + _ => None, + ) + } +} + +impl IanaString for ICalendarScheduleAgentValue { + fn as_str(&self) -> &'static str { + match self { + ICalendarScheduleAgentValue::Server => "SERVER", + ICalendarScheduleAgentValue::Client => "CLIENT", + ICalendarScheduleAgentValue::None => "NONE", + } + } +} + +impl IanaParse for ICalendarScheduleForceSendValue { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "REQUEST" => Some(ICalendarScheduleForceSendValue::Request), + "REPLY" => Some(ICalendarScheduleForceSendValue::Reply), + _ => None, + ) + } +} + +impl IanaString for ICalendarScheduleForceSendValue { + fn as_str(&self) -> &'static str { + match self { + ICalendarScheduleForceSendValue::Request => "REQUEST", + ICalendarScheduleForceSendValue::Reply => "REPLY", + } + } +} + +impl IanaParse for ICalendarValueType { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, ICalendarValueType, + "BINARY" => ICalendarValueType::Binary, + "BOOLEAN" => ICalendarValueType::Boolean, + "CAL-ADDRESS" => ICalendarValueType::CalAddress, + "DATE" => ICalendarValueType::Date, + "DATE-TIME" => ICalendarValueType::DateTime, + "DURATION" => ICalendarValueType::Duration, + "FLOAT" => ICalendarValueType::Float, + "INTEGER" => ICalendarValueType::Integer, + "PERIOD" => ICalendarValueType::Period, + "RECUR" => ICalendarValueType::Recur, + "TEXT" => ICalendarValueType::Text, + "TIME" => ICalendarValueType::Time, + "UNKNOWN" => ICalendarValueType::Unknown, + "URI" => ICalendarValueType::Uri, + "UTC-OFFSET" => ICalendarValueType::UtcOffset, + "XML-REFERENCE" => ICalendarValueType::XmlReference, + "UID" => ICalendarValueType::Uid, + ) + .copied() + } +} + +impl IanaString for ICalendarValueType { + fn as_str(&self) -> &'static str { + match self { + ICalendarValueType::Binary => "BINARY", + ICalendarValueType::Boolean => "BOOLEAN", + ICalendarValueType::CalAddress => "CAL-ADDRESS", + ICalendarValueType::Date => "DATE", + ICalendarValueType::DateTime => "DATE-TIME", + ICalendarValueType::Duration => "DURATION", + ICalendarValueType::Float => "FLOAT", + ICalendarValueType::Integer => "INTEGER", + ICalendarValueType::Period => "PERIOD", + ICalendarValueType::Recur => "RECUR", + ICalendarValueType::Text => "TEXT", + ICalendarValueType::Time => "TIME", + ICalendarValueType::Unknown => "UNKNOWN", + ICalendarValueType::Uri => "URI", + ICalendarValueType::UtcOffset => "UTC-OFFSET", + ICalendarValueType::XmlReference => "XML-REFERENCE", + ICalendarValueType::Uid => "UID", + } + } +} + +impl IanaParse for ICalendarTransparency { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "OPAQUE" => Some(ICalendarTransparency::Opaque), + "TRANSPARENT" => Some(ICalendarTransparency::Transparent), + _ => None, + ) + } +} + +impl IanaString for ICalendarTransparency { + fn as_str(&self) -> &'static str { + match self { + ICalendarTransparency::Opaque => "OPAQUE", + ICalendarTransparency::Transparent => "TRANSPARENT", + } + } +} + +impl ICalendarProperty { + pub(crate) fn parameter_capacity(&self) -> usize { + match self { + ICalendarProperty::Attendee => 8, + _ => 0, + } + } + + // Returns the default value type and whether the property is multi-valued. + pub(crate) fn default_types(&self) -> (ValueType, ValueSeparator) { + match self { + ICalendarProperty::Calscale => (ValueType::CalendarScale, ValueSeparator::None), + ICalendarProperty::Method => (ValueType::Method, ValueSeparator::None), + ICalendarProperty::Prodid => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Version => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Attach => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Categories => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::Comma, + ), + ICalendarProperty::Class => (ValueType::Classification, ValueSeparator::None), + ICalendarProperty::Comment => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Description => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Geo => ( + ValueType::Ical(ICalendarValueType::Float), + ValueSeparator::Semicolon, + ), + ICalendarProperty::Location => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::PercentComplete => ( + ValueType::Ical(ICalendarValueType::Integer), + ValueSeparator::None, + ), + ICalendarProperty::Priority => ( + ValueType::Ical(ICalendarValueType::Integer), + ValueSeparator::None, + ), + ICalendarProperty::Resources => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::Comma, + ), + ICalendarProperty::Status => (ValueType::Status, ValueSeparator::None), + ICalendarProperty::Summary => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Completed => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Dtend => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Due => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Dtstart => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Duration => ( + ValueType::Ical(ICalendarValueType::Duration), + ValueSeparator::None, + ), + ICalendarProperty::Freebusy => ( + ValueType::Ical(ICalendarValueType::Period), + ValueSeparator::Comma, + ), + ICalendarProperty::Transp => (ValueType::Transparency, ValueSeparator::None), + ICalendarProperty::Tzid => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Tzname => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Tzoffsetfrom => ( + ValueType::Ical(ICalendarValueType::UtcOffset), + ValueSeparator::None, + ), + ICalendarProperty::Tzoffsetto => ( + ValueType::Ical(ICalendarValueType::UtcOffset), + ValueSeparator::None, + ), + ICalendarProperty::Tzurl => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Attendee => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Contact => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Organizer => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::RecurrenceId => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::RelatedTo => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Url => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Uid => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Exdate => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::Comma, + ), + ICalendarProperty::Exrule => ( + ValueType::Ical(ICalendarValueType::Recur), + ValueSeparator::None, + ), + ICalendarProperty::Rdate => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::Comma, + ), + ICalendarProperty::Rrule => ( + ValueType::Ical(ICalendarValueType::Recur), + ValueSeparator::None, + ), + ICalendarProperty::Action => (ValueType::Action, ValueSeparator::None), + ICalendarProperty::Repeat => ( + ValueType::Ical(ICalendarValueType::Integer), + ValueSeparator::None, + ), + ICalendarProperty::Trigger => ( + ValueType::Ical(ICalendarValueType::Duration), + ValueSeparator::None, + ), + ICalendarProperty::Created => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Dtstamp => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::LastModified => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Sequence => ( + ValueType::Ical(ICalendarValueType::Integer), + ValueSeparator::None, + ), + ICalendarProperty::RequestStatus => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::Semicolon, + ), + ICalendarProperty::Xml => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Tzuntil => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::TzidAliasOf => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Busytype => (ValueType::BusyType, ValueSeparator::None), + ICalendarProperty::Name => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::RefreshInterval => ( + ValueType::Ical(ICalendarValueType::Duration), + ValueSeparator::None, + ), + ICalendarProperty::Source => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Color => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Image => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Conference => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::CalendarAddress => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::LocationType => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::Comma, + ), + ICalendarProperty::ParticipantType => { + (ValueType::ParticipantType, ValueSeparator::None) + } + ICalendarProperty::ResourceType => (ValueType::ResourceType, ValueSeparator::None), + ICalendarProperty::StructuredData => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::StyledDescription => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Acknowledged => ( + ValueType::Ical(ICalendarValueType::DateTime), + ValueSeparator::None, + ), + ICalendarProperty::Proximity => (ValueType::Proximity, ValueSeparator::None), + ICalendarProperty::Concept => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Link => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Refid => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Begin => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::End => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::Other(_) => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::Semicolon, + ), + ICalendarProperty::Coordinates => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::ShowWithoutTime => ( + ValueType::Ical(ICalendarValueType::Boolean), + ValueSeparator::None, + ), + ICalendarProperty::Jsid | ICalendarProperty::Jsprop => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::EstimatedDuration => ( + ValueType::Ical(ICalendarValueType::Duration), + ValueSeparator::None, + ), + ICalendarProperty::Reason => ( + ValueType::Ical(ICalendarValueType::Uri), + ValueSeparator::None, + ), + ICalendarProperty::Substate => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + ICalendarProperty::TaskMode => ( + ValueType::Ical(ICalendarValueType::Text), + ValueSeparator::None, + ), + } + } +} + +impl ValueType { + pub fn unwrap_ical(self) -> ICalendarValueType { + match self { + ValueType::Ical(value) => value, + _ => ICalendarValueType::Text, + } + } +} + +impl ICalendar { + pub fn parse(value: impl AsRef) -> Result { + let mut parser = Parser::new(value.as_ref()); + match parser.entry() { + Entry::ICalendar(icalendar) => Ok(icalendar), + other => Err(other), + } + } +} + +impl Hash for ICalendarValue { + fn hash(&self, state: &mut H) { + match self { + ICalendarValue::Binary(value) => { + state.write_u8(0); + value.hash(state); + } + ICalendarValue::Boolean(value) => { + state.write_u8(1); + value.hash(state); + } + ICalendarValue::Uri(value) => { + state.write_u8(2); + value.hash(state); + } + ICalendarValue::PartialDateTime(value) => { + state.write_u8(3); + value.hash(state); + } + ICalendarValue::Duration(value) => { + state.write_u8(4); + value.hash(state); + } + ICalendarValue::RecurrenceRule(value) => { + state.write_u8(5); + value.hash(state); + } + ICalendarValue::Period(value) => { + state.write_u8(6); + value.hash(state); + } + ICalendarValue::Float(value) => { + state.write_u8(7); + value.to_bits().hash(state); + } + ICalendarValue::Integer(value) => { + state.write_u8(8); + value.hash(state); + } + ICalendarValue::Text(value) => { + state.write_u8(9); + value.hash(state); + } + ICalendarValue::CalendarScale(value) => { + state.write_u8(10); + value.hash(state); + } + ICalendarValue::Method(value) => { + state.write_u8(11); + value.hash(state); + } + ICalendarValue::Classification(value) => { + state.write_u8(12); + value.hash(state); + } + ICalendarValue::Status(value) => { + state.write_u8(13); + value.hash(state); + } + ICalendarValue::Transparency(value) => { + state.write_u8(14); + value.hash(state); + } + ICalendarValue::Action(value) => { + state.write_u8(15); + value.hash(state); + } + ICalendarValue::BusyType(value) => { + state.write_u8(16); + value.hash(state); + } + ICalendarValue::ParticipantType(value) => { + state.write_u8(17); + value.hash(state); + } + ICalendarValue::ResourceType(value) => { + state.write_u8(18); + value.hash(state); + } + ICalendarValue::Proximity(value) => { + state.write_u8(19); + value.hash(state); + } + } + } +} + +impl Eq for ICalendarValue {} + +#[allow(clippy::derive_ord_xor_partial_ord)] +impl Ord for ICalendarValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal) + } +} diff --git a/crates/calcard/src/icalendar/utils.rs b/crates/calcard/src/icalendar/utils.rs new file mode 100644 index 00000000..adbe65e6 --- /dev/null +++ b/crates/calcard/src/icalendar/utils.rs @@ -0,0 +1,583 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +use super::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ICalendarParameterName, ICalendarProperty, ICalendarRecurrenceRule, ICalendarStatus, + ICalendarTransparency, ICalendarValue, PARAMETER_VALUE_SIZE, Uri, VALUE_SIZE, +}; +use crate::{ + common::{IanaString, IanaType, PartialDateTime, timezone::NominalDuration}, + icalendar::{ICalendarParameterValue, ICalendarValueType}, +}; + +pub fn strip_mailto_scheme(value: &str) -> &str { + value + .split_once(':') + .filter(|(scheme, _)| scheme.eq_ignore_ascii_case("mailto")) + .map_or(value, |(_, address)| address) +} + +impl ICalendar { + pub fn uids(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| { + component.component_type.is_calendar_object() + || component.component_type == ICalendarComponentType::VCalendar + }) + .filter_map(|component| component.uid()) + } + + pub fn size(&self) -> usize { + self.components + .iter() + .map(|component| component.size()) + .sum() + } + + pub fn component_by_id(&self, id: u32) -> Option<&ICalendarComponent> { + self.components.get(id as usize) + } + + pub(crate) fn calendar_root(&self) -> Option<&ICalendarComponent> { + self.components + .first() + .filter(|root| root.component_type == ICalendarComponentType::VCalendar) + } + + pub(crate) fn has_calendar_components(&self) -> bool { + self.calendar_root().is_some_and(|root| { + root.component_ids + .iter() + .any(|id| *id != 0 && self.component_by_id(*id).is_some()) + }) + } + + pub fn alarms_for_id(&self, id: u32) -> impl Iterator { + self.component_by_id(id) + .map_or(&[][..], |c| c.component_ids.as_slice()) + .iter() + .filter_map(|id| { + self.component_by_id(*id) + .filter(|c| c.component_type == ICalendarComponentType::VAlarm) + }) + } +} + +impl ICalendarComponent { + pub fn uid(&self) -> Option<&str> { + self.property(&ICalendarProperty::Uid) + .and_then(|e| e.values.first()) + .and_then(|v| v.as_text()) + } + + pub fn jsid(&self) -> Option<&str> { + self.property(&ICalendarProperty::Jsid) + .and_then(|e| e.values.first()) + .and_then(|v| v.as_text()) + } + + pub fn property(&self, prop: &ICalendarProperty) -> Option<&ICalendarEntry> { + self.entries.iter().find(|entry| &entry.name == prop) + } + + pub fn property_mut(&mut self, prop: &ICalendarProperty) -> Option<&mut ICalendarEntry> { + self.entries.iter_mut().find(|entry| &entry.name == prop) + } + + pub fn has_property(&self, prop: &ICalendarProperty) -> bool { + self.entries.iter().any(|entry| &entry.name == prop) + } + + pub fn properties<'x, 'y: 'x>( + &'x self, + prop: &'y ICalendarProperty, + ) -> impl Iterator + 'x { + self.entries.iter().filter(move |entry| &entry.name == prop) + } + + pub fn properties_mut<'x, 'y: 'x>( + &'x mut self, + prop: &'y ICalendarProperty, + ) -> impl Iterator + 'x { + self.entries + .iter_mut() + .filter(move |entry| &entry.name == prop) + } + + pub fn size(&self) -> usize { + self.entries.iter().map(|entry| entry.size()).sum() + } + + pub fn is_recurrent(&self) -> bool { + self.entries.iter().any(|entry| { + matches!( + entry.name, + ICalendarProperty::Rrule | ICalendarProperty::Rdate + ) + }) + } + + pub fn is_recurrence_override(&self) -> bool { + self.entries + .iter() + .any(|entry| matches!(entry.name, ICalendarProperty::RecurrenceId)) + } + + pub fn is_recurrent_or_override(&self) -> bool { + self.entries.iter().any(|entry| { + matches!( + entry.name, + ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::RecurrenceId + ) + }) + } + + pub fn status(&self) -> Option<&ICalendarStatus> { + self.entries + .iter() + .find_map(|entry| match (&entry.name, entry.values.first()) { + (ICalendarProperty::Status, Some(ICalendarValue::Status(status))) => Some(status), + _ => None, + }) + } + + pub fn transparency(&self) -> Option<&ICalendarTransparency> { + self.entries + .iter() + .find_map(|entry| match (&entry.name, entry.values.first()) { + (ICalendarProperty::Transp, Some(ICalendarValue::Transparency(trans))) => { + Some(trans) + } + _ => None, + }) + } +} + +impl ICalendarValue { + pub(crate) fn binary_bytes(&self) -> Option<&[u8]> { + match self { + ICalendarValue::Binary(data) => Some(data.as_slice()), + ICalendarValue::Uri(Uri::Data(data)) => Some(data.data.as_slice()), + _ => None, + } + .filter(|data| !data.is_empty()) + } + + pub fn size(&self) -> usize { + match self { + ICalendarValue::Binary(value) => value.len(), + ICalendarValue::Text(value) => value.len(), + ICalendarValue::PartialDateTime(_) => std::mem::size_of::(), + ICalendarValue::RecurrenceRule(_) => std::mem::size_of::(), + _ => VALUE_SIZE, + } + } + + pub fn as_text(&self) -> Option<&str> { + match self { + ICalendarValue::Text(s) => Some(s.as_str()), + ICalendarValue::Uri(v) => v.as_str(), + ICalendarValue::CalendarScale(v) => Some(v.as_str()), + ICalendarValue::Method(v) => Some(v.as_str()), + ICalendarValue::Classification(v) => Some(v.as_str()), + ICalendarValue::Status(v) => Some(v.as_str()), + ICalendarValue::Transparency(v) => Some(v.as_str()), + ICalendarValue::Action(v) => Some(v.as_str()), + ICalendarValue::BusyType(v) => Some(v.as_str()), + ICalendarValue::ParticipantType(v) => Some(v.as_str()), + ICalendarValue::ResourceType(v) => Some(v.as_str()), + ICalendarValue::Proximity(v) => Some(v.as_str()), + _ => None, + } + } + + pub fn into_text(self) -> Option> { + match self { + ICalendarValue::Text(s) => Some(Cow::Owned(s)), + ICalendarValue::Uri(v) => Some(Cow::Owned(v.into_unwrapped_string())), + ICalendarValue::CalendarScale(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Method(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Classification(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Status(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Transparency(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Action(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::BusyType(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::ParticipantType(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::ResourceType(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Proximity(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarValue::Integer(i) => Some(Cow::Owned(i.to_string())), + ICalendarValue::Float(f) => Some(Cow::Owned(f.to_string())), + ICalendarValue::Boolean(b) => Some(Cow::Borrowed(if b { "TRUE" } else { "FALSE" })), + ICalendarValue::PartialDateTime(dt) => dt.to_rfc3339().map(Cow::Owned), + ICalendarValue::RecurrenceRule(rrule) => Some(Cow::Owned(rrule.to_string())), + ICalendarValue::Binary(value) => String::from_utf8(value).ok().map(Cow::Owned), + ICalendarValue::Duration(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::Period(value) => Some(Cow::Owned(value.to_string())), + } + } + + pub fn into_partial_date_time(self) -> Option { + match self { + ICalendarValue::PartialDateTime(dt) => Some(dt), + _ => None, + } + } + + pub fn as_integer(&self) -> Option { + match self { + ICalendarValue::Integer(i) => Some(*i), + _ => None, + } + } + + pub fn as_float(&self) -> Option { + match self { + ICalendarValue::Float(f) => Some(*f), + _ => None, + } + } + + pub fn as_boolean(&self) -> Option { + match self { + ICalendarValue::Boolean(b) => Some(*b), + _ => None, + } + } + + pub fn as_partial_date_time(&self) -> Option<&PartialDateTime> { + match self { + ICalendarValue::PartialDateTime(dt) => Some(dt), + _ => None, + } + } + + pub fn as_binary(&self) -> Option<&[u8]> { + match self { + ICalendarValue::Binary(d) => Some(d.as_slice()), + _ => None, + } + } +} + +impl ICalendarEntry { + #[inline] + pub fn parameters( + &self, + prop: &ICalendarParameterName, + ) -> impl Iterator { + self.params.iter().filter_map(move |param| { + if ¶m.name == prop { + Some(¶m.value) + } else { + None + } + }) + } + + #[inline] + pub fn parameter(&self, prop: &ICalendarParameterName) -> Option<&ICalendarParameterValue> { + self.params.iter().find_map(move |param| { + if ¶m.name == prop { + Some(¶m.value) + } else { + None + } + }) + } + + #[inline] + pub fn has_parameter(&self, prop: &ICalendarParameterName) -> bool { + self.params.iter().any(|param| ¶m.name == prop) + } + + pub fn jsid(&self) -> Option<&str> { + self.parameter(&ICalendarParameterName::Jsid) + .and_then(|v| v.as_text()) + } + + pub fn is_derived(&self) -> bool { + self.parameter(&ICalendarParameterName::Derived) + .is_some_and(|v| matches!(v, ICalendarParameterValue::Bool(true))) + } + + pub fn calendar_address(&self) -> Option<&str> { + self.values + .first() + .and_then(|v| v.as_text()) + .map(strip_mailto_scheme) + } + + pub fn size(&self) -> usize { + self.values.iter().map(|value| value.size()).sum::() + + self + .params + .iter() + .map(|param| std::mem::size_of::() + param.value.size()) + .sum::() + + self.name.as_str().len() + } +} + +impl ICalendarParameterValue { + pub fn as_text(&self) -> Option<&str> { + match self { + ICalendarParameterValue::Text(v) => Some(v.as_str()), + ICalendarParameterValue::Bool(v) => Some(if *v { "TRUE" } else { "FALSE" }), + ICalendarParameterValue::Uri(uri) => uri.as_str(), + ICalendarParameterValue::Cutype(v) => Some(v.as_str()), + ICalendarParameterValue::Fbtype(v) => Some(v.as_str()), + ICalendarParameterValue::Partstat(v) => Some(v.as_str()), + ICalendarParameterValue::Related(v) => Some(v.as_str()), + ICalendarParameterValue::Reltype(v) => Some(v.as_str()), + ICalendarParameterValue::Role(v) => Some(v.as_str()), + ICalendarParameterValue::ScheduleAgent(v) => Some(v.as_str()), + ICalendarParameterValue::ScheduleForceSend(v) => Some(v.as_str()), + ICalendarParameterValue::Value(v) => Some(v.as_str()), + ICalendarParameterValue::Display(v) => Some(v.as_str()), + ICalendarParameterValue::Feature(v) => Some(v.as_str()), + ICalendarParameterValue::Linkrel(v) => Some(v.as_str()), + ICalendarParameterValue::Duration(_) + | ICalendarParameterValue::Integer(_) + | ICalendarParameterValue::Null => None, + } + } + + pub fn into_text(self) -> Option> { + match self { + ICalendarParameterValue::Text(v) => Some(Cow::Owned(v)), + ICalendarParameterValue::Bool(v) => { + Some(Cow::Borrowed(if v { "TRUE" } else { "FALSE" })) + } + ICalendarParameterValue::Uri(uri) => Some(Cow::Owned(uri.into_unwrapped_string())), + ICalendarParameterValue::Cutype(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Fbtype(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Partstat(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Related(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Reltype(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Role(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::ScheduleAgent(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::ScheduleForceSend(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Value(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Display(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Feature(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Linkrel(v) => Some(Cow::Borrowed(v.as_str())), + ICalendarParameterValue::Duration(v) => Some(Cow::Owned(v.to_string())), + ICalendarParameterValue::Integer(v) => Some(Cow::Owned(v.to_string())), + ICalendarParameterValue::Null => None, + } + } + + pub fn into_value_type(self) -> Option> { + match self { + ICalendarParameterValue::Value(v) => Some(IanaType::Iana(v)), + ICalendarParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_integer(&self) -> Option { + match self { + ICalendarParameterValue::Integer(v) => Some(*v), + ICalendarParameterValue::Text(v) => v.parse().ok(), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match self { + ICalendarParameterValue::Bool(v) => Some(*v), + _ => None, + } + } + + pub fn size(&self) -> usize { + match self { + ICalendarParameterValue::Text(v) => v.len(), + ICalendarParameterValue::Uri(v) => v.size(), + _ => PARAMETER_VALUE_SIZE, + } + } +} + +impl ICalendarComponentType { + pub fn is_calendar_object(&self) -> bool { + matches!( + self, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + | ICalendarComponentType::VAvailability + ) + } + + pub(crate) fn converts_links(&self) -> bool { + matches!( + self, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VCalendar + ) + } + + pub fn has_time_ranges(&self) -> bool { + matches!( + self, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + ) + } + + pub fn is_scheduling_object(&self) -> bool { + matches!( + self, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + ) + } + + pub fn is_event(&self) -> bool { + matches!(self, ICalendarComponentType::VEvent) + } + + pub fn is_todo(&self) -> bool { + matches!(self, ICalendarComponentType::VTodo) + } + + pub fn is_event_or_todo(&self) -> bool { + matches!( + self, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo + ) + } + + pub fn is_journal(&self) -> bool { + matches!(self, ICalendarComponentType::VJournal) + } + + pub fn is_freebusy(&self) -> bool { + matches!(self, ICalendarComponentType::VFreebusy) + } + + pub fn is_location(&self) -> bool { + matches!(self, ICalendarComponentType::VLocation) + } + + pub fn is_alarm(&self) -> bool { + matches!(self, ICalendarComponentType::VAlarm) + } + + pub fn is_participant(&self) -> bool { + matches!(self, ICalendarComponentType::Participant) + } +} + +impl Uri { + pub fn as_str(&self) -> Option<&str> { + match self { + Uri::Data(data) => data.content_type.as_deref(), + Uri::Location(loc) => loc.as_str().into(), + } + } + + pub fn size(&self) -> usize { + match self { + Uri::Data(data) => { + data.data.len() + data.content_type.as_ref().map(|s| s.len()).unwrap_or(0) + } + Uri::Location(loc) => loc.len(), + } + } +} + +impl ICalendarDuration { + pub fn from_seconds(seconds: i64) -> Self { + let mut secs = seconds; + let neg = secs < 0; + if neg { + secs = -secs; + } + let weeks = (secs / 604800) as u32; + secs %= 604800; + let days = (secs / 86400) as u32; + secs %= 86400; + let hours = (secs / 3600) as u32; + secs %= 3600; + let minutes = (secs / 60) as u32; + secs %= 60; + let seconds = secs as u32; + + Self { + weeks, + days, + hours, + minutes, + seconds, + neg, + } + } + + pub fn from_days(days: i64) -> Self { + let neg = days < 0; + let days = days.unsigned_abs(); + let (weeks, days) = if days.is_multiple_of(7) { + (days / 7, 0) + } else { + (0, days) + }; + + Self { + weeks: weeks as u32, + days: days as u32, + hours: 0, + minutes: 0, + seconds: 0, + neg, + } + } + + /// Returns this duration as a calendar span. + pub fn to_nominal(&self) -> Option { + let days = i32::try_from(i64::from(self.weeks) * 7 + i64::from(self.days)).ok()?; + let seconds = + i64::from(self.hours) * 3600 + i64::from(self.minutes) * 60 + i64::from(self.seconds); + Some(if self.neg { + NominalDuration::new(-days, -seconds) + } else { + NominalDuration::new(days, seconds) + }) + } + + pub fn as_seconds(&self) -> i64 { + let secs = self.seconds as i64 + + self.minutes as i64 * 60 + + self.hours as i64 * 3600 + + self.days as i64 * 86400 + + self.weeks as i64 * 604800; + + if self.neg { -secs } else { secs } + } + + pub fn is_empty(&self) -> bool { + self.weeks == 0 + && self.days == 0 + && self.hours == 0 + && self.minutes == 0 + && self.seconds == 0 + } +} diff --git a/crates/calcard/src/icalendar/writer.rs b/crates/calcard/src/icalendar/writer.rs new file mode 100644 index 00000000..07423b82 --- /dev/null +++ b/crates/calcard/src/icalendar/writer.rs @@ -0,0 +1,524 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ICalendar, ICalendarDay, ICalendarDuration, ICalendarEntry, ICalendarPeriod, + ICalendarRecurrenceRule, ICalendarValueType, +}; +use crate::{ + common::{ + CalendarScale, IanaString, PartialDateTime, + format::{AsciiPush, BufferedWrite, DurationParts}, + stack::ComponentStack, + writer::{ + DOCUMENT_BUFFER, ENTRY_BUFFER, FoldingWriter, LineWriter, VisitedComponents, + write_param_text, write_param_value, write_quoted_param_value, write_text, write_uri, + }, + }, + icalendar::{ + ICalendarMonth, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, + ICalendarValue, Uri, ValueSeparator, + }, +}; +use std::{ + fmt::{self, Display, Write}, + slice::Iter, +}; + +pub(crate) const RRULE_BUFFER: usize = 256; +pub(crate) const DURATION_BUFFER: usize = 64; +pub(crate) const DATE_BUFFER: usize = 32; +pub(crate) const PERIOD_BUFFER: usize = 96; + +impl ICalendar { + pub fn write_to(&self, out: &mut impl Write) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_components(&mut FoldingWriter::new(buf)) + }) + } + + fn write_components( + &self, + out: &mut FoldingWriter<'_, W>, + ) -> fmt::Result { + let mut component_iter: Iter<'_, u32> = [0].iter(); + let mut component_stack = ComponentStack::new(); + let mut visited = VisitedComponents::new(self.components.len()); + + loop { + if let Some(component_id) = component_iter.next() { + let component_id = *component_id as usize; + let Some(component) = self + .components + .get(component_id) + .filter(|_| visited.insert(component_id)) + else { + continue; + }; + out.write_boundary("BEGIN:", component.component_type.as_str())?; + + for entry in &component.entries { + if !matches!( + entry.name, + ICalendarProperty::Begin | ICalendarProperty::End + ) { + entry.write_line(out, true)?; + } + } + + if !component.component_ids.is_empty() { + component_stack.push((component, component_iter)); + component_iter = component.component_ids.iter(); + } else { + out.write_boundary("END:", component.component_type.as_str())?; + } + } else if let Some((component, iter)) = component_stack.pop() { + out.write_boundary("END:", component.component_type.as_str())?; + component_iter = iter; + } else { + break; + } + } + + Ok(()) + } +} + +impl ICalendarEntry { + pub fn write_to(&self, out: &mut impl Write) -> fmt::Result { + self.write_with_value(out, true) + } + + pub fn write_with_value(&self, out: &mut impl Write, with_value: bool) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_line(&mut FoldingWriter::new(buf), with_value) + }) + } + + pub(crate) fn write_line( + &self, + out: &mut FoldingWriter<'_, W>, + with_value: bool, + ) -> fmt::Result { + out.write_atomic(self.name.as_str())?; + + if matches!(self.values.first(), Some(ICalendarValue::Binary(_))) { + out.write_atomic(";ENCODING=BASE64")?; + if !self + .params + .iter() + .any(|param| param.name == ICalendarParameterName::Value) + { + out.write_atomic(";VALUE=BINARY")?; + } + } + + let mut types = None; + let mut last_param: Option<&ICalendarParameterName> = None; + + for param in &self.params { + if last_param.is_some_and(|last_param| last_param == ¶m.name) { + out.write_atomic(",")?; + } else { + out.write_atomic(";")?; + out.write_atomic(param.name.as_str())?; + if !matches!(param.value, ICalendarParameterValue::Null) { + out.write_atomic("=")?; + } + last_param = Some(¶m.name); + } + + match ¶m.value { + ICalendarParameterValue::Text(v) + if matches!(param.name, ICalendarParameterName::Jsptr) => + { + write_quoted_param_value(out, v, true)?; + } + ICalendarParameterValue::Text(v) => { + write_param_value(out, v, true)?; + } + ICalendarParameterValue::Integer(i) => { + out.push_u64(*i)?; + } + ICalendarParameterValue::Bool(v) => { + let v = if !matches!(param.name, ICalendarParameterName::Range) { + if *v { "TRUE" } else { "FALSE" } + } else { + "THISANDFUTURE" + }; + out.write_atomic(v)?; + } + ICalendarParameterValue::Uri(uri) => { + out.write_atomic("\"")?; + uri.write_param_text(out)?; + out.write_atomic("\"")?; + } + ICalendarParameterValue::Cutype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Fbtype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Partstat(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Related(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Reltype(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Role(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::ScheduleAgent(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::ScheduleForceSend(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Value(v) => { + types = Some(v); + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Display(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Feature(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Duration(v) => { + out.push_duration(v.parts())?; + } + ICalendarParameterValue::Linkrel(v) => { + write_param_value(out, v.as_str(), true)?; + } + ICalendarParameterValue::Null => { + last_param = None; + } + } + } + + out.write_atomic(":")?; + + if !with_value { + return out.end_line(); + } + + let (default_type, separator) = self.name.default_types(); + let separator = if !matches!(separator, ValueSeparator::Comma) { + ";" + } else { + "," + }; + let default_type = default_type.unwrap_ical(); + + for (pos, value) in self.values.iter().enumerate() { + if pos > 0 { + out.write_atomic(separator)?; + } + + let text = match value { + ICalendarValue::Binary(v) => { + out.write_base64(v)?; + continue; + } + ICalendarValue::Boolean(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + continue; + } + ICalendarValue::Uri(v) => { + v.write_value(out)?; + continue; + } + ICalendarValue::PartialDateTime(v) => { + v.push_ical(out, types.unwrap_or(&default_type))?; + continue; + } + ICalendarValue::Duration(v) => { + out.push_duration(v.parts())?; + continue; + } + ICalendarValue::RecurrenceRule(v) => { + v.push_to(out)?; + continue; + } + ICalendarValue::Period(v) => { + v.push_to(out)?; + continue; + } + ICalendarValue::Float(v) => { + write!(out, "{v}")?; + continue; + } + ICalendarValue::Integer(v) => { + out.push_i64(*v)?; + continue; + } + ICalendarValue::Text(v) => { + match types.unwrap_or(&default_type) { + ICalendarValueType::Uri | ICalendarValueType::CalAddress => { + write_uri(out, v)?; + } + ICalendarValueType::Recur => write_text(out, v, false, false)?, + _ => write_text(out, v, true, true)?, + } + continue; + } + ICalendarValue::CalendarScale(v) => v.as_str(), + ICalendarValue::Method(v) => v.as_str(), + ICalendarValue::Classification(v) => v.as_str(), + ICalendarValue::Status(v) => v.as_str(), + ICalendarValue::Transparency(v) => v.as_str(), + ICalendarValue::Action(v) => v.as_str(), + ICalendarValue::BusyType(v) => v.as_str(), + ICalendarValue::ParticipantType(v) => v.as_str(), + ICalendarValue::ResourceType(v) => v.as_str(), + ICalendarValue::Proximity(v) => v.as_str(), + }; + + out.write_atomic(text)?; + } + + out.end_line() + } +} + +impl Uri { + fn write_value(&self, out: &mut impl LineWriter) -> fmt::Result { + match self { + Uri::Data(v) => { + out.write_str("data:")?; + out.write_str(v.content_type.as_deref().unwrap_or_default())?; + out.write_str(";")?; + out.write_atomic("base64,")?; + out.write_base64(&v.data) + } + Uri::Location(v) => write_uri(out, v), + } + } + + fn write_param_text(&self, out: &mut impl LineWriter) -> fmt::Result { + match self { + Uri::Data(v) => { + out.write_str("data:")?; + write_param_text(out, v.content_type.as_deref().unwrap_or_default(), true)?; + out.write_str(";")?; + out.write_atomic("base64,")?; + out.write_base64(&v.data) + } + Uri::Location(v) => write_param_text(out, v, true), + } + } +} + +impl Uri { + pub fn to_unwrapped_string(&self) -> String { + match self { + Uri::Data(v) => v.to_unwrapped_string(), + Uri::Location(v) => v.to_string(), + } + } + + pub fn into_unwrapped_string(self) -> String { + match self { + Uri::Data(v) => v.to_unwrapped_string(), + Uri::Location(v) => v, + } + } +} + +impl ICalendarRecurrenceRule { + pub(crate) fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + buf.push_ascii_str("FREQ=")?; + buf.push_ascii_str(self.freq.as_str())?; + if let Some(until) = &self.until { + buf.push_ascii_str(";UNTIL=")?; + until.push_ical( + buf, + if until.has_date_and_time() { + &ICalendarValueType::DateTime + } else { + &ICalendarValueType::Date + }, + )?; + } + if let Some(count) = self.count.filter(|c| *c > 0) { + buf.push_ascii_str(";COUNT=")?; + buf.push_u64(count.into())?; + } + if let Some(interval) = self.interval { + buf.push_ascii_str(";INTERVAL=")?; + buf.push_u64(interval.into())?; + } + buf.push_list(";BYSECOND=", &self.bysecond, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYMINUTE=", &self.byminute, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYHOUR=", &self.byhour, |buf, item| { + buf.push_u64((*item).into()) + })?; + buf.push_list(";BYDAY=", &self.byday, |buf, item| item.push_to(buf))?; + buf.push_list(";BYMONTHDAY=", &self.bymonthday, |buf, item| { + buf.push_i64((*item).into()) + })?; + buf.push_list(";BYYEARDAY=", &self.byyearday, |buf, item| { + buf.push_i64((*item).into()) + })?; + buf.push_list(";BYWEEKNO=", &self.byweekno, |buf, item| { + buf.push_i64((*item).into()) + })?; + buf.push_list(";BYMONTH=", &self.bymonth, |buf, item| item.push_to(buf))?; + buf.push_list(";BYSETPOS=", &self.bysetpos, |buf, item| { + buf.push_i64((*item).into()) + })?; + if let Some(wkst) = self.wkst { + buf.push_ascii_str(";WKST=")?; + buf.push_ascii_str(wkst.as_str())?; + } + if let Some(rscale) = &self.rscale { + buf.push_ascii_str(";RSCALE=")?; + buf.push_ascii_str(rscale.as_str())?; + } else if self.skip.is_some() { + buf.push_ascii_str(";RSCALE=")?; + buf.push_ascii_str(CalendarScale::Gregorian.as_str())?; + } + if let Some(skip) = &self.skip { + buf.push_ascii_str(";SKIP=")?; + buf.push_ascii_str(skip.as_str())?; + } + + Ok(()) + } +} + +impl Display for ICalendarRecurrenceRule { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ICalendarDay { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + if let Some(ordwk) = self.ordwk { + buf.push_i64(ordwk.into())?; + } + buf.push_ascii_str(self.weekday.as_str()) + } +} + +impl Display for ICalendarDay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ICalendarMonth { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + buf.push_u64(self.month().into())?; + if self.is_leap() { + buf.push_byte(b'L')?; + } + Ok(()) + } +} + +impl Display for ICalendarMonth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ICalendarPeriod { + fn push_to(&self, buf: &mut impl AsciiPush) -> fmt::Result { + match self { + ICalendarPeriod::Range { start, end } => { + start.push_ical(buf, &ICalendarValueType::DateTime)?; + buf.push_byte(b'/')?; + end.push_ical(buf, &ICalendarValueType::DateTime) + } + ICalendarPeriod::Duration { start, duration } => { + start.push_ical(buf, &ICalendarValueType::DateTime)?; + buf.push_byte(b'/')?; + buf.push_duration(duration.parts()) + } + } + } +} + +impl Display for ICalendarPeriod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| self.push_to(buf)) + } +} + +impl ICalendarDuration { + pub(crate) fn parts(&self) -> DurationParts { + DurationParts { + neg: self.neg, + weeks: self.weeks, + days: self.days, + hours: self.hours, + minutes: self.minutes, + seconds: self.seconds, + } + } +} + +impl Display for ICalendarDuration { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_buffered::(|buf| buf.push_duration(self.parts())) + } +} + +impl PartialDateTime { + pub fn format_as_ical(&self, out: &mut impl Write, fmt: &ICalendarValueType) -> fmt::Result { + out.write_buffered::(|buf| self.push_ical(buf, fmt)) + } + + pub(crate) fn push_ical( + &self, + buf: &mut impl AsciiPush, + fmt: &ICalendarValueType, + ) -> fmt::Result { + if matches!(fmt, ICalendarValueType::Date | ICalendarValueType::DateTime) { + buf.push_4_digits(self.year.unwrap_or_default())?; + buf.push_2_digits(self.month.unwrap_or_default())?; + buf.push_2_digits(self.day.unwrap_or_default())?; + } + + if matches!(fmt, ICalendarValueType::DateTime) { + buf.push_byte(b'T')?; + } + + if matches!(fmt, ICalendarValueType::DateTime | ICalendarValueType::Time) { + buf.push_2_digits(self.hour.unwrap_or_default())?; + buf.push_2_digits(self.minute.unwrap_or_default())?; + buf.push_2_digits(self.second.unwrap_or_default())?; + + if matches!((self.tz_hour, self.tz_minute), (Some(0), Some(0))) { + buf.push_byte(b'Z')?; + } + } + + if matches!(fmt, ICalendarValueType::UtcOffset) { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(self.tz_hour.unwrap_or_default())?; + buf.push_2_digits(self.tz_minute.unwrap_or_default())?; + } + + Ok(()) + } +} + +impl Display for ICalendar { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_to(f) + } +} + diff --git a/crates/calcard/src/jscalendar/export/convert.rs b/crates/calcard/src/jscalendar/export/convert.rs new file mode 100644 index 00000000..2dda1643 --- /dev/null +++ b/crates/calcard/src/jscalendar/export/convert.rs @@ -0,0 +1,3337 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::datecalc::rrule::RRule; +use crate::{ + common::{ + IanaParse, PartialDateTime, + blob::BlobResolver, + export::{ExportError, RejectedPatch}, + jsprop::text::PointerString, + timezone::{NominalDuration, Tz, ZonedDateTime}, + }, + icalendar::*, + jscalendar::{ + export::{ConvertedComponent, ExportContext, ExportOptions, State}, + ext::{JSCalendarKeyExt, JSCalendarMapExt, JSCalendarObjectExt}, + overrides::OverrideTemplate, + *, + }, +}; +use ahash::AHashSet; +use jiff::{civil, tz::Offset}; +use jmap_tools::{Key, Map, Value}; +use smallvec::smallvec; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum JSPropSet { + Exists, + Missing, +} + +impl JSPropSet { + fn of(exists: bool) -> Self { + if exists { + JSPropSet::Exists + } else { + JSPropSet::Missing + } + } +} + +impl<'x, I: JSCalendarId, B: JSCalendarId> JSCalendar<'x, I, B> { + pub fn into_icalendar(self) -> Result { + self.into_icalendar_with(ExportOptions::default()) + } + + pub fn into_icalendar_with>( + self, + options: ExportOptions, + ) -> Result { + self.into_icalendar_with_report(options) + .map(|(ical, _)| ical) + } + + /// Converts to iCalendar, also returning the recurrence override patches + /// that were rejected, as required by draft-ietf-calext-jscalendarbis-20, + /// Section 1.5.9. Each rejected override degrades to an `RDATE`. + pub fn into_icalendar_with_report>( + self, + mut options: ExportOptions, + ) -> Result<(ICalendar, Vec), ExportError> { + let mut context = options.context::(); + if let Some(blobs) = &mut context.blobs { + blobs.add_uses(self.blob_ids()); + } + let entries = self.0.into_object().ok_or(ExportError::NotGroup)?; + let entries = if entries.is_type_or_untyped(&[JSCalendarType::Group]) { + entries + } else if entries.is_type_or_untyped(&[JSCalendarType::Event, JSCalendarType::Task]) { + Map::from_iter([( + Key::Property(JSCalendarProperty::Entries), + Value::Array(vec![Value::Object(entries)]), + )]) + } else { + return Err(ExportError::NotGroup); + }; + let mut ical = ICalendar::default(); + + ical.from_jscalendar( + State { + tz: Default::default(), + tz_end: Default::default(), + tz_rid: Default::default(), + start: Default::default(), + recurrence_id: Default::default(), + uid: Default::default(), + entries, + is_date: false, + default_component_type: ICalendarComponentType::VCalendar, + }, + None, + &mut context, + ); + + context.into_result(ical) + } + + pub fn into_inner(self) -> Value<'x, JSCalendarProperty, JSCalendarValue> { + self.0 + } +} + +impl ICalendar { + #[allow(clippy::wrong_self_convention)] + pub(super) fn from_jscalendar( + &mut self, + mut state: State<'_, I, B>, + mut parent_component: Option<&mut ICalendarComponent>, + options: &mut ExportContext<'_, B>, + ) { + if options.has_failed() { + return; + } + let mut root_conversions = None; + let mut locale = None; + let mut uid = state.uid.map(Cow::Borrowed); + let mut organizer_address = None; + let mut organizer_params: Vec = Vec::new(); + let mut organizer_sent_by = None; + let mut privacy = None; + let mut organizer_participant_id = None; + let mut organizer_unmapped_roles = None; + let mut organizer_owner = None; + let mut has_organizer_participant = false; + let mut alarms_without_text = Vec::new(); + let mut main_location_id = None; + let mut start = None; + let mut component_type = None; + let mut description_content_type = None; + let mut overrides = None; + let mut is_show_without_time = false; + let mut has_time_component = false; + let mut has_time_zone = false; + let entry_capacity = state.entry_capacity(); + let mut override_template = match state + .entries + .lookup(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + { + Some(Value::Object(overrides)) + if matches!( + state.default_component_type, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo + ) => + { + let instances = OverrideTemplate::uses(&state.entries, overrides); + (instances > 0).then(|| { + let template = OverrideTemplate::new(&state.entries, instances); + if let Some(blobs) = &mut options.blobs { + blobs.add_uses(template.blob_uses()); + } + template + }) + } + _ => None, + }; + + for (key, value) in state.entries.as_mut_vec() { + match (key, value) { + (Key::Property(JSCalendarProperty::ICalendar), Value::Object(obj)) => { + root_conversions = + ConvertedComponent::try_from_object(std::mem::take(obj.as_mut_vec())); + } + (Key::Property(JSCalendarProperty::MainLocationId), Value::Str(text)) => { + main_location_id = Some(std::mem::take(text)); + } + (Key::Property(JSCalendarProperty::TimeZone), Value::Str(text)) => { + state.tz = Tz::from_str(text.as_ref()).ok(); + has_time_zone = true; + } + (Key::Property(JSCalendarProperty::EndTimeZone), Value::Str(text)) => { + state.tz_end = Tz::from_str(text.as_ref()).ok(); + } + (Key::Property(JSCalendarProperty::RecurrenceIdTimeZone), Value::Str(text)) => { + state.tz_rid = Tz::from_str(text.as_ref()).ok(); + } + (Key::Property(JSCalendarProperty::Uid), Value::Str(text)) => { + uid = Some(std::mem::take(text)); + } + (Key::Property(JSCalendarProperty::Locale), Value::Str(text)) => { + locale = Some(std::mem::take(text)); + } + (Key::Property(JSCalendarProperty::OrganizerCalendarAddress), Value::Str(text)) => { + organizer_address = Some(std::mem::take(text)); + } + (Key::Property(JSCalendarProperty::SentBy), Value::Str(text)) => { + organizer_sent_by = Some(std::mem::take(text)); + } + ( + Key::Property(JSCalendarProperty::Privacy), + Value::Element(JSCalendarValue::Privacy(value)), + ) => { + privacy = Some(*value); + } + (Key::Property(JSCalendarProperty::Privacy), Value::Str(_)) => { + privacy = Some(JSCalendarPrivacy::Private); + } + (Key::Property(JSCalendarProperty::DescriptionContentType), Value::Str(text)) => { + description_content_type = Some(std::mem::take(text)); + } + ( + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(dt)), + ) if matches!( + state.default_component_type, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo + ) => + { + has_time_component |= !dt.is_start_of_day(); + start = dt.to_naive_date_time(); + } + ( + Key::Property(JSCalendarProperty::Due | JSCalendarProperty::RecurrenceId), + Value::Element(JSCalendarValue::DateTime(dt)), + ) => { + has_time_component |= !dt.is_start_of_day(); + } + ( + Key::Property( + JSCalendarProperty::Duration | JSCalendarProperty::EstimatedDuration, + ), + Value::Element(JSCalendarValue::Duration(duration)), + ) => { + has_time_component |= + duration.hours != 0 || duration.minutes != 0 || duration.seconds != 0; + } + (Key::Property(JSCalendarProperty::ShowWithoutTime), Value::Bool(value)) => { + is_show_without_time = *value; + } + (Key::Property(JSCalendarProperty::RecurrenceRule), Value::Object(obj)) => { + if let Some(Value::Element(JSCalendarValue::DateTime(dt))) = + obj.lookup(&Key::Property(JSCalendarProperty::Until)) + { + has_time_component |= !dt.is_start_of_day(); + } + } + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(typ)), + ) => { + component_type = typ.to_icalendar_component_type(); + } + (Key::Property(JSCalendarProperty::RecurrenceOverrides), Value::Object(obj)) + if matches!( + state.default_component_type, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo + ) => + { + has_time_component |= obj.iter().any(|(key, _)| { + matches!(key, Key::Property(JSCalendarProperty::DateTime(dt)) + if !dt.is_start_of_day()) + }); + overrides = Some(std::mem::take(obj)); + } + _ => (), + } + } + + if let Some(overrides) = &mut overrides { + for (_, value) in overrides.iter_mut() { + let Value::Object(obj) = value else { + continue; + }; + + obj.as_mut_vec().retain(|(key, value)| match key { + Key::Property(JSCalendarProperty::Privacy) => { + let override_privacy = match value { + Value::Element(JSCalendarValue::Privacy(value)) => *value, + _ => JSCalendarPrivacy::Private, + }; + if override_privacy > privacy.unwrap_or(JSCalendarPrivacy::Public) { + privacy = Some(override_privacy); + } + false + } + key => !key.is_forbidden_override_key(), + }); + } + } + + /* + In the iCalendar component, the value data types of the DTSTART, DUE and + RECURRENCE-ID properties all MUST be of the same form, either DATE, or DATE WITH + LOCAL TIME, or DATE WITH UTC TIME. It MUST be DATE if the "showWithoutTime" + property value is "true", the "timeZone" property value is not set, and the time + component is zero in the values of the "start", "due", "duration", + "estimatedDuration", "recurrenceId" properties, and the RecurrenceRule object's + "until" property, and in any key of the "recurrenceOverrides" property value. + */ + + let series_is_date = state.is_date; + let is_override = state.recurrence_id.is_some(); + state.is_date = (series_is_date && !is_override) + || (is_show_without_time && !has_time_zone && !has_time_component); + if is_override + && (state.is_date != series_is_date + || state.tz.is_none() + != state + .recurrence_id + .is_some_and(|recurrence_id| recurrence_id.timezone().is_floating())) + && let Some(conversions) = &mut root_conversions + { + conversions.converted_props.retain(|(keys, _)| { + !matches!( + keys.as_slice(), + [Key::Property( + JSCalendarProperty::Start + | JSCalendarProperty::Due + | JSCalendarProperty::Duration + )] + ) + }); + } + + // Build component + let mut component = + ICalendarComponent::new(component_type.unwrap_or(state.default_component_type)); + component.entries.reserve_exact(entry_capacity); + if parent_component.is_none() { + debug_assert!(self.components.is_empty()); + self.components + .push(ICalendarComponent::new(ICalendarComponentType::VCalendar)); + } + + // Add start date + if let Some(start) = + start.and_then(|dt| state.tz.unwrap_or_default().resolve_local_datetime(dt)) + { + state.start = Some(start); + } + if let Some(dt) = state.start { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Dtstart) + .import_converted(&[JSCalendarProperty::Start], &mut root_conversions) + .with_date(dt, state.is_date), + ); + } + + // Add UID + if let Some(uid) = &uid { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Uid) + .import_converted(&[JSCalendarProperty::Uid], &mut root_conversions) + .with_value(uid.clone().into_owned()), + ); + } + + let mut show_without_time = None; + let mut add_recurrence_id = state.recurrence_id.is_some(); + for (key, value) in state.entries.into_vec() { + let Key::Property(property) = key else { + component.insert_jsprop(&[key.to_string().as_ref()], value); + continue; + }; + + match (&property, value, &component.component_type) { + ( + JSCalendarProperty::Links, + Value::Object(obj), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + component.import_links(obj, &mut root_conversions, options); + } + ( + JSCalendarProperty::Participants, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + for (name, value) in obj.into_vec() { + let Value::Object(mut value) = value else { + continue; + }; + + let mut item_conversions = ConvertedComponent::build(&mut value); + let has_address = matches!( + value.lookup(&Key::Property(JSCalendarProperty::CalendarAddress)), + Some(Value::Str(_)) + ); + let mut entry = ICalendarEntry::new(ICalendarProperty::Attendee); + entry.params.reserve_exact(value.len() + 1); + let mut participant = + ICalendarComponent::new(ICalendarComponentType::Participant); + let mut participant_name = None; + let mut calendar_address = None; + let mut status = None; + let mut progress = None; + let mut description = None; + let mut description_content_type = None; + let mut is_uuid5_key = false; + let mut references_key = false; + let mut has_owner_role = false; + let mut unmapped_roles = Vec::new(); + + for (sub_property, value) in value.into_vec() { + match (sub_property, value) { + ( + Key::Property(JSCalendarProperty::CalendarAddress), + Value::Str(text), + ) => { + is_uuid5_key = name.is_uuid5_of(text.as_bytes()); + calendar_address = Some(text); + } + ( + Key::Property(JSCalendarProperty::DelegatedFrom), + Value::Object(obj), + ) if has_address => { + for key in obj.into_expanded_boolean_set() { + entry.params.push(ICalendarParameter::delegated_from( + Uri::parse(key.into_string()), + )); + } + } + ( + Key::Property(JSCalendarProperty::DelegatedFrom), + Value::Str(text), + ) if has_address => { + entry.params.push(ICalendarParameter::delegated_from( + Uri::parse(text.into_owned()), + )); + } + ( + Key::Property(JSCalendarProperty::DelegatedTo), + Value::Object(obj), + ) if has_address => { + for key in obj.into_expanded_boolean_set() { + entry.params.push(ICalendarParameter::delegated_to( + Uri::parse(key.into_string()), + )); + } + } + ( + Key::Property(JSCalendarProperty::DelegatedTo), + Value::Str(text), + ) if has_address => { + entry.params.push(ICalendarParameter::delegated_to( + Uri::parse(text.into_owned()), + )); + } + (Key::Property(JSCalendarProperty::Email), Value::Str(text)) + if has_address => + { + entry + .params + .push(ICalendarParameter::email(text.into_owned())); + } + ( + Key::Property(JSCalendarProperty::ExpectReply), + Value::Bool(value), + ) if has_address => { + entry.params.push(ICalendarParameter::rsvp(value)); + } + ( + Key::Property(JSCalendarProperty::Kind), + Value::Element(JSCalendarValue::ParticipantKind(kind)), + ) if has_address => { + entry.params.push(ICalendarParameter::cutype(match kind { + JSCalendarParticipantKind::Individual => { + ICalendarUserTypes::Individual + } + JSCalendarParticipantKind::Group => { + ICalendarUserTypes::Group + } + JSCalendarParticipantKind::Resource => { + ICalendarUserTypes::Resource + } + JSCalendarParticipantKind::Location => { + ICalendarUserTypes::Room + } + })); + } + (Key::Property(JSCalendarProperty::Kind), Value::Str(text)) + if has_address && text.is_param_token() => + { + entry + .params + .push(ICalendarParameter::cutype(text.to_lowercase())); + } + ( + Key::Property(JSCalendarProperty::MemberOf), + Value::Object(obj), + ) if has_address => { + for key in obj.into_expanded_boolean_set() { + entry.params.push(ICalendarParameter::member(Uri::parse( + key.into_string(), + ))); + } + } + (Key::Property(JSCalendarProperty::Name), Value::Str(text)) => { + if has_address { + entry.params.push(ICalendarParameter::cn( + ICalendarParameterValue::Null, + )); + } + participant_name = Some(text); + } + ( + Key::Property(JSCalendarProperty::ParticipationStatus), + Value::Element(JSCalendarValue::ParticipationStatus(status_)), + ) if has_address => { + if status.is_none() && progress.is_none() { + entry.params.push(ICalendarParameter::partstat( + ICalendarParameterValue::Null, + )); + } + status = Some(status_); + } + ( + Key::Property(JSCalendarProperty::Progress), + Value::Element(JSCalendarValue::Progress(progress_)), + ) if has_address => { + if status.is_none() && progress.is_none() { + entry.params.push(ICalendarParameter::partstat( + ICalendarParameterValue::Null, + )); + } + progress = Some(progress_); + } + (Key::Property(JSCalendarProperty::Roles), Value::Object(obj)) + if has_address => + { + for key in obj.into_expanded_boolean_set() { + let ical_role = match &key { + Key::Property(JSCalendarProperty::ParticipantRole( + role, + )) => Some(match role { + JSCalendarParticipantRole::Optional => { + ICalendarParticipationRole::OptParticipant + } + JSCalendarParticipantRole::Informational => { + ICalendarParticipationRole::NonParticipant + } + JSCalendarParticipantRole::Chair => { + ICalendarParticipationRole::Chair + } + JSCalendarParticipantRole::Required => { + ICalendarParticipationRole::ReqParticipant + } + JSCalendarParticipantRole::Owner => { + ICalendarParticipationRole::Owner + } + }), + _ => None, + }; + if let Some(role) = ical_role { + has_owner_role |= + role == ICalendarParticipationRole::Owner; + entry.params.push(ICalendarParameter::role(role)); + } else { + unmapped_roles.push(key.into_owned()); + } + } + } + (Key::Property(JSCalendarProperty::SentBy), Value::Str(text)) + if has_address => + { + entry.params.push(ICalendarParameter::sent_by(Uri::parse( + text.into_owned(), + ))); + } + ( + Key::Property(JSCalendarProperty::Description), + Value::Str(text), + ) => { + description = Some(text); + } + ( + Key::Property(JSCalendarProperty::DescriptionContentType), + Value::Str(text), + ) => { + description_content_type = Some(text); + } + (Key::Property(JSCalendarProperty::Links), Value::Object(obj)) => { + participant.import_links(obj, &mut item_conversions, options); + } + ( + Key::Property(JSCalendarProperty::PercentComplete), + Value::Number(number), + ) => match number + .as_i64() + .filter(|percent| (0..=100).contains(percent)) + { + Some(percent) => { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::PercentComplete) + .with_value(percent) + .import_converted( + &[JSCalendarProperty::PercentComplete], + &mut item_conversions, + ), + ); + } + None => { + participant.insert_jsprop::( + &[JSCalendarProperty::PercentComplete:: + .to_string() + .as_ref()], + Value::Number(number), + ); + } + }, + ( + Key::Property( + JSCalendarProperty::Type | JSCalendarProperty::ICalendar, + ), + _, + ) => {} + (sub_property, value) => { + if item_conversions.is_none() && has_address { + references_key |= component.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } else { + participant.insert_jsprop( + &[sub_property.to_string().as_ref()], + value, + ); + } + } + } + } + + match (description, description_content_type) { + (Some(description), Some(content_type)) => { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::StyledDescription) + .with_param(ICalendarParameter::fmttype( + content_type.into_owned(), + )) + .with_value(description.into_owned()) + .import_converted( + &[JSCalendarProperty::Description], + &mut item_conversions, + ), + ); + } + (Some(description), None) => { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::Description) + .with_value(description.into_owned()) + .import_converted( + &[JSCalendarProperty::Description], + &mut item_conversions, + ), + ); + } + _ => {} + } + + let mut partstat = match (status, progress) { + ( + Some(JSCalendarParticipationStatus::Accepted) | None, + Some(progress), + ) => match progress { + JSCalendarProgress::NeedsAction => { + Some(ICalendarParticipationStatus::NeedsAction) + } + JSCalendarProgress::InProcess => { + Some(ICalendarParticipationStatus::InProcess) + } + JSCalendarProgress::Completed => { + Some(ICalendarParticipationStatus::Completed) + } + JSCalendarProgress::Failed => { + Some(ICalendarParticipationStatus::Failed) + } + // Not a valid Participant progress value per section 4.4.5 + JSCalendarProgress::Cancelled => None, + }, + (Some(status), _) => Some(match status { + JSCalendarParticipationStatus::Accepted => { + ICalendarParticipationStatus::Accepted + } + JSCalendarParticipationStatus::Declined => { + ICalendarParticipationStatus::Declined + } + JSCalendarParticipationStatus::NeedsAction => { + ICalendarParticipationStatus::NeedsAction + } + JSCalendarParticipationStatus::Tentative => { + ICalendarParticipationStatus::Tentative + } + JSCalendarParticipationStatus::Delegated => { + ICalendarParticipationStatus::Delegated + } + }), + _ => None, + }; + + let has_component = !has_address + || !participant.entries.is_empty() + || item_conversions.is_some(); + let has_entry = has_address + && (partstat.is_some() + || item_conversions.is_none() + || entry.params.iter().any(|param| { + !matches!(param.value, ICalendarParameterValue::Null) + })); + + entry + .params + .retain_mut(|param| match (¶m.name, ¶m.value) { + ( + ICalendarParameterName::Partstat, + ICalendarParameterValue::Null, + ) => partstat.take().is_some_and(|partstat| { + param.value = ICalendarParameterValue::Partstat(partstat); + true + }), + (ICalendarParameterName::Cn, ICalendarParameterValue::Null) => { + participant_name.as_ref().filter(|_| has_entry).is_some_and( + |participant_name| { + param.value = ICalendarParameterValue::Text( + participant_name.to_string(), + ); + true + }, + ) + } + _ => true, + }); + + let is_organizer = has_address + && organizer_address.is_some() + && organizer_address == calendar_address + && !has_organizer_participant; + has_organizer_participant |= is_organizer; + + /* + The "owner" role needs not convert to the ROLE parameter of the + ATTENDEE property if the Participant object converts to both the + ATTENDEE and ORGANIZER property. + */ + + let stripped_owner = if is_organizer { + entry + .params + .iter() + .position(|param| { + matches!( + (¶m.name, ¶m.value), + ( + ICalendarParameterName::Role, + ICalendarParameterValue::Role( + ICalendarParticipationRole::Owner + ) + ) + ) + }) + .inspect(|&position| { + entry.params.remove(position); + }) + } else { + None + }; + + if let Some(calendar_address) = calendar_address { + let calendar_address = + ICalendarValue::Uri(Uri::parse(calendar_address.into_owned())); + + if has_component { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::CalendarAddress) + .with_value(calendar_address.clone()), + ); + } + if has_entry { + entry.values.push(calendar_address); + } + } + + if let Some(participant_name) = participant_name + && has_component + { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::Summary) + .with_value(participant_name.into_owned()) + .import_converted( + &[JSCalendarProperty::Name], + &mut item_conversions, + ), + ); + } + + /* + The Participant object converts to an ATTENDEE property and PARTICIPANT + component, unless it is fully represented by the ORGANIZER property: + its "calendarAddress" property value matches the + "organizerCalendarAddress" property value, it does not have any role + but the "owner" role set, and it only has properties set that convert + to ORGANIZER parameters. + */ + + let is_organizer_only = is_organizer + && !has_component + && unmapped_roles.is_empty() + && entry.params.iter().all(|param| { + matches!( + param.name, + ICalendarParameterName::Cn + | ICalendarParameterName::Email + | ICalendarParameterName::SentBy + ) + }); + + let participant_id = name.into_string(); + + if !unmapped_roles.is_empty() { + references_key = true; + if entry + .params + .iter() + .any(|param| param.name == ICalendarParameterName::Role) + { + component.insert_roles_jsprop::( + &participant_id, + unmapped_roles, + JSPropSet::Exists, + ); + } else if is_organizer { + organizer_unmapped_roles = + Some((participant_id.clone(), unmapped_roles)); + } else { + component.insert_roles_jsprop::( + &participant_id, + unmapped_roles, + JSPropSet::Missing, + ); + } + } + + if is_organizer_only { + organizer_params.append(&mut entry.params); + if !is_uuid5_key || references_key { + organizer_params + .push(ICalendarParameter::jsid(participant_id.clone())); + } + organizer_participant_id = Some(participant_id.clone()); + organizer_owner = + Some(OrganizerOwner::Organizer(participant_id.clone())); + } else if is_organizer { + organizer_params.extend( + entry + .params + .iter() + .filter(|param| { + matches!( + param.name, + ICalendarParameterName::Cn + | ICalendarParameterName::Email + | ICalendarParameterName::SentBy + ) + }) + .cloned(), + ); + if has_owner_role { + organizer_owner = + Some(OrganizerOwner::Organizer(participant_id.clone())); + } + } + + if has_component { + if !is_uuid5_key || references_key { + participant.entries.push( + ICalendarEntry::new(ICalendarProperty::Jsid) + .with_value(participant_id.clone()), + ); + } + if let Some(item_conversions) = item_conversions { + participant = item_conversions.apply_conversions(participant, self); + } + component + .component_ids + .push(self.push_component(participant)); + } + + if !entry.values.is_empty() && !is_organizer_only { + let mut attendee = entry.import_converted_with_id( + &[JSCalendarProperty::Participants], + &mut root_conversions, + Some(participant_id.as_str()), + ); + if !is_uuid5_key || references_key { + attendee.add_param(ICalendarParameter::jsid(participant_id)); + } + if let Some(position) = stripped_owner { + organizer_owner = Some(OrganizerOwner::Attendee { + entry: component.entries.len(), + position, + }); + } + + component.entries.push(attendee); + } + } + } + ( + JSCalendarProperty::Alerts, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let mut alerts = Vec::new(); + let mut has_related_to = false; + + for (name, value) in obj.into_vec() { + let Value::Object(mut value) = value else { + continue; + }; + + let mut item_conversions = ConvertedComponent::build(&mut value); + let mut alert = ICalendarComponent::new(ICalendarComponentType::VAlarm); + let mut has_action = false; + + for (sub_property, value) in value.into_vec() { + match (sub_property, value) { + ( + Key::Property( + JSCalendarProperty::Type | JSCalendarProperty::ICalendar, + ), + _, + ) => {} + ( + Key::Property(JSCalendarProperty::Acknowledged), + Value::Element(JSCalendarValue::DateTime(dt)), + ) => { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Acknowledged) + .with_value(PartialDateTime::from_utc_timestamp( + dt.timestamp, + )) + .import_converted( + &[JSCalendarProperty::Acknowledged], + &mut item_conversions, + ), + ); + } + ( + Key::Property(JSCalendarProperty::Action), + Value::Element(JSCalendarValue::AlertAction(action)), + ) => { + has_action = true; + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Action) + .with_value(ICalendarValue::Action(match action { + JSCalendarAlertAction::Display => { + ICalendarAction::Display + } + JSCalendarAlertAction::Email => { + ICalendarAction::Email + } + })) + .import_converted( + &[JSCalendarProperty::Action], + &mut item_conversions, + ), + ); + } + ( + Key::Property(JSCalendarProperty::RelatedTo), + Value::Object(obj), + ) => { + has_related_to = true; + alert.import_relations(obj, &mut item_conversions); + } + ( + Key::Property(JSCalendarProperty::Trigger), + Value::Object(obj), + ) => { + let mut offset = None; + let mut rel_to = None; + let mut when = None; + + for (key, value) in obj.into_vec() { + match (key, value) { + ( + Key::Property(JSCalendarProperty::Offset), + Value::Element(JSCalendarValue::Duration(value)), + ) => { + offset = Some(value); + } + ( + Key::Property(JSCalendarProperty::RelativeTo), + Value::Element(JSCalendarValue::RelativeTo(value)), + ) => { + rel_to = Some(value); + } + ( + Key::Property(JSCalendarProperty::When), + Value::Element(JSCalendarValue::DateTime(value)), + ) => { + when = Some(value); + } + (Key::Property(JSCalendarProperty::Type), _) => {} + (key, value) => { + alert.insert_jsprop( + &[ + JSCalendarProperty::Trigger:: + .to_string() + .as_ref(), + key.to_string().as_ref(), + ], + value, + ); + } + } + } + + if let Some(when) = when { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Trigger) + .with_param(ICalendarParameter::value( + ICalendarValueType::DateTime, + )) + .with_value(PartialDateTime::from_utc_timestamp( + when.timestamp, + )) + .import_converted( + &[ + JSCalendarProperty::Trigger, + JSCalendarProperty::When, + ], + &mut item_conversions, + ), + ); + } else if let Some(offset) = offset { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Trigger) + .with_param_opt(rel_to.map(|rel_to| { + ICalendarParameter::related(match rel_to { + JSCalendarRelativeTo::Start => { + ICalendarRelated::Start + } + JSCalendarRelativeTo::End => { + ICalendarRelated::End + } + }) + })) + .with_value(offset) + .import_converted( + &[ + JSCalendarProperty::Trigger, + JSCalendarProperty::Offset, + ], + &mut item_conversions, + ), + ); + } + } + (sub_property, value) => { + has_action |= matches!( + sub_property, + Key::Property(JSCalendarProperty::Action) + ); + alert + .insert_jsprop(&[sub_property.to_string().as_ref()], value); + } + } + } + + if let Some(item_conversions) = item_conversions { + alert = item_conversions.apply_conversions(alert, self); + } + + if !alert.entries.is_empty() { + if !has_action && !alert.has_property(&ICalendarProperty::Action) { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Action).with_value( + ICalendarValue::Action(ICalendarAction::Display), + ), + ); + } + if !alert.has_property(&ICalendarProperty::Trigger) { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Trigger) + .with_value(ICalendarDuration::default()), + ); + } + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Jsid) + .with_value(name.into_string()), + ); + alerts.push(alert); + } + } + + // Map RELATED-TO properties + let mut alert_mappings = Vec::new(); + if has_related_to { + let related_ids = alerts + .iter() + .flat_map(|alert| alert.properties(&ICalendarProperty::RelatedTo)) + .filter_map(|related_to| related_to.values.first()) + .filter_map(ICalendarValue::as_text) + .map(str::to_string) + .collect::>(); + for alert in &mut alerts { + if !alert.has_property(&ICalendarProperty::Uid) + && let Some(alert_uid) = alert + .jsid() + .filter(|jsid| related_ids.contains(*jsid)) + .map(|jsid| { + uuid5(format!( + "{}/{jsid}", + uid.as_deref().unwrap_or_default() + )) + }) + { + alert.entries.push( + ICalendarEntry::new(ICalendarProperty::Uid) + .with_value(alert_uid), + ); + } + } + alert_mappings = alerts + .iter() + .filter_map(|alert| { + Some((alert.jsid()?.to_string(), alert.uid()?.to_string())) + }) + .collect(); + } + for mut alert in alerts { + if has_related_to { + for related_to in alert.properties_mut(&ICalendarProperty::RelatedTo) { + if let Some(ICalendarValue::Text(related_to)) = + related_to.values.first_mut() + && let Some((_, uid)) = + alert_mappings.iter().find(|(jsid, _)| jsid == related_to) + { + *related_to = uid.clone(); + } + } + } + let needs_text = alert.missing_alarm_text(); + let alert_id = self.push_component(alert); + if needs_text { + alarms_without_text.push(alert_id); + } + component.component_ids.push(alert_id); + } + } + ( + JSCalendarProperty::Keywords, + Value::Object(obj), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Categories) + .with_values( + obj.into_expanded_boolean_set() + .map(|v| ICalendarValue::Text(v.into_string())), + ) + .import_converted( + &[JSCalendarProperty::Keywords], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Privacy, + Value::Element(JSCalendarValue::Privacy(_)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => {} + ( + JSCalendarProperty::Color, + Value::Str(text), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Color) + .with_value(text.into_owned()) + .import_converted(&[JSCalendarProperty::Color], &mut root_conversions), + ); + } + ( + JSCalendarProperty::Categories, + Value::Object(obj), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + for value in obj.into_expanded_boolean_set() { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Concept) + .with_value(ICalendarValue::Text(value.into_string())) + .import_converted( + &[JSCalendarProperty::Categories], + &mut root_conversions, + ), + ); + } + } + ( + JSCalendarProperty::VirtualLocations, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + for (name, value) in obj.into_vec() { + let mut entry = ICalendarEntry::new(ICalendarProperty::Conference); + let mut is_uuid5_key = false; + let mut references_key = false; + + for (sub_property, value) in value.into_expanded_object() { + match (sub_property, value) { + ( + Key::Property(JSCalendarProperty::Features), + Value::Object(obj), + ) => { + let mut vendor_features = Vec::new(); + for key in obj.into_expanded_boolean_set() { + let value = match key { + Key::Property( + JSCalendarProperty::VirtualLocationFeature(feature), + ) => ICalendarParameterValue::Feature(match feature { + JSCalendarVirtualLocationFeature::Audio => { + ICalendarFeatureType::Audio + } + JSCalendarVirtualLocationFeature::Chat => { + ICalendarFeatureType::Chat + } + JSCalendarVirtualLocationFeature::Feed => { + ICalendarFeatureType::Feed + } + JSCalendarVirtualLocationFeature::Moderator => { + ICalendarFeatureType::Moderator + } + JSCalendarVirtualLocationFeature::Phone => { + ICalendarFeatureType::Phone + } + JSCalendarVirtualLocationFeature::Screen => { + ICalendarFeatureType::Screen + } + JSCalendarVirtualLocationFeature::Video => { + ICalendarFeatureType::Video + } + }), + other if other.to_string().is_param_token() => { + ICalendarParameterValue::Text(other.into_string()) + } + other => { + vendor_features.push(other.into_owned()); + continue; + } + }; + entry.params.push(ICalendarParameter::feature(value)); + } + if !vendor_features.is_empty() { + references_key = true; + let features = JSPropSet::of( + entry.has_parameter(&ICalendarParameterName::Feature), + ); + component.insert_set_jsprop::( + &[ + JSCalendarProperty::VirtualLocations:: + .to_string() + .as_ref(), + name.to_string().as_ref(), + JSCalendarProperty::Features:: + .to_string() + .as_ref(), + ], + vendor_features, + features, + ); + } + } + (Key::Property(JSCalendarProperty::Name), Value::Str(text)) => { + entry + .params + .push(ICalendarParameter::label(text.into_owned())); + } + (Key::Property(JSCalendarProperty::Uri), Value::Str(text)) => { + is_uuid5_key = name.is_uuid5_of(text.as_bytes()); + entry + .values + .push(ICalendarValue::Uri(Uri::parse(text.into_owned()))); + } + ( + Key::Property( + JSCalendarProperty::Type | JSCalendarProperty::ICalendar, + ), + _, + ) => {} + (sub_property, value) => { + references_key |= component.insert_jsprop( + &[ + JSCalendarProperty::VirtualLocations:: + .to_string() + .as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + if !is_uuid5_key || references_key { + entry.add_param(ICalendarParameter::jsid(name.into_string())); + } + + component.entries.push( + entry + .import_converted( + &[JSCalendarProperty::VirtualLocations], + &mut root_conversions, + ) + .with_uri_value_type(), + ); + } + } + ( + JSCalendarProperty::Title, + Value::Str(text), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Summary) + .with_param_opt( + locale + .as_ref() + .map(|locale| ICalendarParameter::language(locale.to_string())), + ) + .with_value(text.into_owned()) + .import_converted(&[JSCalendarProperty::Title], &mut root_conversions), + ); + } + ( + JSCalendarProperty::Title, + Value::Str(text), + ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Name) + .with_param_opt( + locale + .as_ref() + .map(|locale| ICalendarParameter::language(locale.to_string())), + ) + .with_value(text.into_owned()) + .import_converted(&[JSCalendarProperty::Title], &mut root_conversions), + ); + } + ( + JSCalendarProperty::Locations, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let has_multi_location = obj.len() > 1; + for (name, value) in obj.into_vec() { + let Value::Object(mut value) = value else { + continue; + }; + let mut item_conversions = ConvertedComponent::build(&mut value); + let is_main_location = main_location_id + .as_ref() + .is_some_and(|l| l == &name.to_string()); + let mut is_uuid5_key = false; + let mut references_key = false; + let first_entry = component.entries.len(); + let mut location = (item_conversions + .as_ref() + .is_some_and(|v| v.name.is_location()) + || value.iter().any(|(k, v)| match (k, v) { + ( + Key::Property( + JSCalendarProperty::Links + | JSCalendarProperty::LocationTypes + | JSCalendarProperty::Description, + ), + _, + ) => true, + ( + Key::Property(JSCalendarProperty::Coordinates), + Value::Str(uri), + ) => uri.strip_prefix("geo:").is_none_or(|v| { + !v.as_bytes() + .iter() + .all(|b| matches!(b, b'0'..=b'9' | b'.' | b',' | b'-')) + }), + _ => false, + }) + || (has_multi_location && !is_main_location)) + .then_some(ICalendarComponent::new(ICalendarComponentType::VLocation)); + + for (sub_property, value) in value.into_vec() { + match (sub_property, value) { + ( + Key::Property(JSCalendarProperty::LocationTypes), + Value::Object(obj), + ) => { + if let Some(location) = &mut location { + location.entries.push( + ICalendarEntry::new(ICalendarProperty::LocationType) + .import_converted( + &[JSCalendarProperty::LocationTypes], + &mut item_conversions, + ) + .with_values(obj.into_expanded_boolean_set().map( + |v| ICalendarValue::Text(v.into_string()), + )), + ); + } + } + (Key::Property(JSCalendarProperty::Name), Value::Str(text)) => { + is_uuid5_key |= name.is_uuid5_of(text.as_bytes()); + + if let Some(location) = &mut location { + location.entries.push( + ICalendarEntry::new(ICalendarProperty::Name) + .import_converted( + &[JSCalendarProperty::Name], + &mut item_conversions, + ) + .with_value(text.clone().into_owned()), + ); + } + + if location.is_none() || is_main_location { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Location) + .with_param_opt((!is_uuid5_key).then(|| { + ICalendarParameter::jsid( + name.clone().into_string(), + ) + })) + .with_param_opt( + (has_multi_location) + .then(|| ICalendarParameter::derived(true)), + ) + .with_value(text.into_owned()) + .import_converted( + &[JSCalendarProperty::Locations], + &mut root_conversions, + ), + ); + } + } + ( + Key::Property(JSCalendarProperty::Description), + Value::Str(text), + ) => { + if let Some(location) = &mut location { + location.entries.push( + ICalendarEntry::new(ICalendarProperty::Description) + .import_converted( + &[JSCalendarProperty::Description], + &mut item_conversions, + ) + .with_value(text.into_owned()), + ); + } + } + ( + Key::Property(JSCalendarProperty::Coordinates), + Value::Str(text), + ) => { + is_uuid5_key |= name.is_uuid5_of(text.as_bytes()); + if let Some(location) = &mut location { + let entry = + ICalendarEntry::new(ICalendarProperty::Coordinates) + .import_converted( + &[JSCalendarProperty::Coordinates], + &mut item_conversions, + ); + location.entries.push( + if entry.name == ICalendarProperty::Geo { + entry.with_values(parse_geo(text)) + } else { + entry + .with_value(Uri::parse(text.into_owned())) + .with_uri_value_type() + }, + ); + } else { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Geo) + .with_param_opt((!is_uuid5_key).then(|| { + ICalendarParameter::jsid( + name.clone().into_string(), + ) + })) + .with_values(parse_geo(text)) + .import_converted( + &[JSCalendarProperty::Locations], + &mut root_conversions, + ), + ); + } + } + (Key::Property(JSCalendarProperty::Links), Value::Object(obj)) => { + if let Some(location) = &mut location { + location.import_links(obj, &mut item_conversions, options); + } + } + ( + Key::Property( + JSCalendarProperty::Type | JSCalendarProperty::ICalendar, + ), + _, + ) => {} + (sub_property, value) => { + if let Some(location) = &mut location { + location.insert_jsprop( + &[sub_property.to_string().as_ref()], + value, + ); + } else { + references_key |= component.insert_jsprop( + &[ + JSCalendarProperty::Locations:: + .to_string() + .as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + } + + if references_key { + for entry in + component + .entries + .iter_mut() + .skip(first_entry) + .filter(|entry| { + entry.name != ICalendarProperty::Jsprop + && !entry.has_parameter(&ICalendarParameterName::Jsid) + }) + { + entry.add_param(ICalendarParameter::jsid( + name.to_string().into_owned(), + )); + } + } + + if let Some(mut location) = location { + if !is_uuid5_key { + location.entries.push( + ICalendarEntry::new(ICalendarProperty::Jsid) + .with_value(name.to_string().into_owned()), + ); + } + if let Some(item_conversions) = item_conversions { + location = item_conversions.apply_conversions(location, self); + } + component.component_ids.push(self.push_component(location)); + } + } + } + ( + JSCalendarProperty::Created, + Value::Element(JSCalendarValue::DateTime(dt)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Created) + .with_value(PartialDateTime::from_utc_timestamp(dt.timestamp)) + .import_converted( + &[JSCalendarProperty::Created], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Description, + Value::Str(text), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + if let Some(description_content_type) = description_content_type.take() { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::StyledDescription) + .with_value(text.into_owned()) + .with_param(ICalendarParameter::fmttype( + description_content_type.into_owned(), + )) + .import_converted( + &[JSCalendarProperty::Description], + &mut root_conversions, + ), + ); + } else { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Description) + .with_value(text.into_owned()) + .import_converted( + &[JSCalendarProperty::Description], + &mut root_conversions, + ), + ); + } + } + ( + JSCalendarProperty::RecurrenceRule, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let mut rrule = ICalendarRecurrenceRule::default(); + + for (key, value) in obj.into_vec() { + let Key::Property(key) = key else { + continue; + }; + match (key, value) { + ( + JSCalendarProperty::Frequency, + Value::Element(JSCalendarValue::Frequency(value)), + ) => { + rrule.freq = value; + } + ( + JSCalendarProperty::Until, + Value::Element(JSCalendarValue::DateTime(value)), + ) => { + rrule.until = value + .to_naive_date_time() + .and_then(|dt| { + state.tz.unwrap_or_default().resolve_local_datetime(dt) + }) + .map(|dt| { + if state.is_date { + PartialDateTime::from_date_timestamp( + dt.naive_timestamp(), + ) + } else if dt.timezone().is_floating() { + PartialDateTime::from_naive_timestamp( + dt.naive_timestamp(), + ) + } else { + PartialDateTime::from_utc_timestamp(dt.timestamp()) + } + }); + } + (JSCalendarProperty::Count, Value::Number(value)) + if value.as_u64().is_some_and(|count| { + count >= 1 && u32::try_from(count).is_ok() + }) => + { + rrule.count = + value.as_u64().and_then(|count| u32::try_from(count).ok()); + } + (JSCalendarProperty::Interval, Value::Number(value)) + if value.as_u64().is_some_and(|interval| { + interval >= 1 && u16::try_from(interval).is_ok() + }) => + { + rrule.interval = value + .as_u64() + .and_then(|interval| u16::try_from(interval).ok()); + } + (JSCalendarProperty::BySecond, Value::Array(value)) => { + rrule.bysecond = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| u8::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByMinute, Value::Array(value)) => { + rrule.byminute = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| u8::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByHour, Value::Array(value)) => { + rrule.byhour = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| u8::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByDay, Value::Array(value)) + if value.iter().all(|item| { + !matches!( + item.as_object().and_then(|item| item.lookup( + &Key::Property(JSCalendarProperty::NthOfPeriod) + )), + Some(Value::Number(nth)) if nth + .as_i64() + .is_none_or(|nth| i16::try_from(nth).is_err()) + ) + }) => + { + for item in value { + let mut weekday = None; + let mut ordwk = None; + + for (key, value) in item.into_expanded_object() { + match (key, value) { + ( + Key::Property(JSCalendarProperty::Day), + Value::Element(JSCalendarValue::Weekday(value)), + ) => { + weekday = Some(value); + } + ( + Key::Property(JSCalendarProperty::NthOfPeriod), + Value::Number(value), + ) => { + ordwk = value + .as_i64() + .and_then(|nth| i16::try_from(nth).ok()); + } + _ => {} + } + } + + if let Some(weekday) = weekday { + rrule.byday.push(ICalendarDay { weekday, ordwk }); + } + } + } + (JSCalendarProperty::ByMonthDay, Value::Array(value)) => { + rrule.bymonthday = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| i8::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByYearDay, Value::Array(value)) => { + rrule.byyearday = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| i16::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByWeekNo, Value::Array(value)) => { + rrule.byweekno = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| i8::try_from(n).ok())) + .collect(); + } + (JSCalendarProperty::ByMonth, Value::Array(value)) => { + rrule.bymonth = value + .into_iter() + .filter_map(|v| { + v.as_str().and_then(|v| ICalendarMonth::parse(v.as_bytes())) + }) + .collect(); + } + (JSCalendarProperty::BySetPosition, Value::Array(value)) => { + rrule.bysetpos = value + .into_iter() + .filter_map(|v| v.as_i64().and_then(|n| i32::try_from(n).ok())) + .collect(); + } + ( + JSCalendarProperty::FirstDayOfWeek, + Value::Element(JSCalendarValue::Weekday(value)), + ) => { + rrule.wkst = Some(value); + } + ( + JSCalendarProperty::Rscale, + Value::Element(JSCalendarValue::CalendarScale(value)), + ) => { + rrule.rscale = Some(value); + } + ( + JSCalendarProperty::Skip, + Value::Element(JSCalendarValue::Skip(value)), + ) => { + rrule.skip = Some(value); + } + (JSCalendarProperty::Type | JSCalendarProperty::ICalendar, _) => {} + (key, value) => { + component.insert_jsprop( + &[ + JSCalendarProperty::RecurrenceRule:: + .to_string() + .as_ref(), + key.to_string().as_ref(), + ], + value, + ); + } + } + } + + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Rrule) + .with_value(rrule) + .import_converted( + &[JSCalendarProperty::RecurrenceRule], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Updated, + Value::Element(JSCalendarValue::DateTime(dt)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Dtstamp) + .with_value(PartialDateTime::from_utc_timestamp(dt.timestamp)) + .import_converted( + &[JSCalendarProperty::Updated], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Updated, + Value::Element(JSCalendarValue::DateTime(dt)), + ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::LastModified) + .with_value(PartialDateTime::from_utc_timestamp(dt.timestamp)) + .import_converted( + &[JSCalendarProperty::Updated], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Due, + Value::Element(JSCalendarValue::DateTime(dt)), + ICalendarComponentType::VTodo, + ) => { + if let Some(dt) = dt + .to_naive_date_time() + .and_then(|dt| state.tz.unwrap_or_default().resolve_local_datetime(dt)) + { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Due) + .import_converted(&[JSCalendarProperty::Due], &mut root_conversions) + .with_date(dt, state.is_date), + ); + } + } + ( + JSCalendarProperty::RecurrenceId, + Value::Element(JSCalendarValue::DateTime(dt)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + if let Some(dt) = dt.to_naive_date_time().and_then(|dt| { + state + .tz_rid + .or(state.tz) + .unwrap_or_default() + .resolve_local_datetime(dt) + }) { + add_recurrence_id = false; + component.entries.push( + ICalendarEntry::new(ICalendarProperty::RecurrenceId) + .import_converted( + &[JSCalendarProperty::RecurrenceId], + &mut root_conversions, + ) + .with_date(dt, state.is_date), + ); + } + } + ( + JSCalendarProperty::Duration, + Value::Element(JSCalendarValue::Duration(duration)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let end_property = if state.tz_end.is_none() { + ICalendarProperty::Duration + } else { + ICalendarProperty::Dtend + }; + let mut entry = ICalendarEntry::new(end_property.clone()) + .import_converted(&[JSCalendarProperty::Duration], &mut root_conversions); + if entry.name == ICalendarProperty::Dtstart { + if component.component_type == ICalendarComponentType::VEvent + && state.is_date + && state.start.is_some() + && duration.to_nominal() == Some(NominalDuration::DAY) + { + continue; + } + entry = ICalendarEntry::new(end_property); + } + if entry.name == ICalendarProperty::Dtend { + if let Some(end) = state.start.and_then(|start| end_after(start, &duration)) + { + component.entries.push( + entry.with_date( + state + .tz_end + .map(|tz_end| end.with_timezone(tz_end)) + .unwrap_or(end), + state.is_date, + ), + ); + } else { + component.insert_jsprop::( + &[property.to_string().as_ref()], + Value::Element(JSCalendarValue::Duration(duration)), + ); + } + } else { + component.entries.push(entry.with_value(duration)); + } + } + ( + JSCalendarProperty::EstimatedDuration, + Value::Element(JSCalendarValue::Duration(duration)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::EstimatedDuration) + .with_value(duration) + .import_converted( + &[JSCalendarProperty::EstimatedDuration], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Method, + Value::Element(JSCalendarValue::Method(method)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + if let Some(parent_component) = &mut parent_component + && !parent_component.has_property(&ICalendarProperty::Method) + { + parent_component.entries.push( + ICalendarEntry::new(ICalendarProperty::Method).with_value(method), + ); + } + } + ( + JSCalendarProperty::PercentComplete, + Value::Number(number), + ICalendarComponentType::VTodo, + ) => match number + .as_i64() + .filter(|percent| (0..=100).contains(percent)) + { + Some(percent) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::PercentComplete) + .with_value(percent) + .import_converted( + &[JSCalendarProperty::PercentComplete], + &mut root_conversions, + ), + ); + } + None => { + component.insert_jsprop::( + &[property.to_string().as_ref()], + Value::Number(number), + ); + } + }, + ( + JSCalendarProperty::Priority, + Value::Number(number), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => match number + .as_i64() + .filter(|priority| (0..=9).contains(priority)) + { + Some(priority) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Priority) + .with_value(priority) + .import_converted( + &[JSCalendarProperty::Priority], + &mut root_conversions, + ), + ); + } + None => { + component.insert_jsprop::( + &[property.to_string().as_ref()], + Value::Number(number), + ); + } + }, + ( + JSCalendarProperty::Sequence, + Value::Number(number), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => match number + .as_i64() + .filter(|sequence| (0..=i64::from(i32::MAX)).contains(sequence)) + { + Some(sequence) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Sequence) + .with_value(sequence) + .import_converted( + &[JSCalendarProperty::Sequence], + &mut root_conversions, + ), + ); + } + None => { + component.insert_jsprop::( + &[property.to_string().as_ref()], + Value::Number(number), + ); + } + }, + ( + JSCalendarProperty::ProdId, + Value::Str(text), + ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Prodid) + .with_value(text.into_owned()) + .import_converted(&[JSCalendarProperty::ProdId], &mut root_conversions), + ); + } + ( + JSCalendarProperty::RelatedTo, + Value::Object(obj), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.import_relations(obj, &mut root_conversions); + } + ( + JSCalendarProperty::ShowWithoutTime, + Value::Bool(value), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + show_without_time = Some( + ICalendarEntry::new(ICalendarProperty::ShowWithoutTime) + .with_value(value) + .import_converted( + &[JSCalendarProperty::ShowWithoutTime], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Progress, + Value::Element(JSCalendarValue::Progress(value)), + ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Status) + .with_value(match value { + JSCalendarProgress::NeedsAction => ICalendarStatus::NeedsAction, + JSCalendarProgress::InProcess => ICalendarStatus::InProcess, + JSCalendarProgress::Completed => ICalendarStatus::Completed, + JSCalendarProgress::Failed => ICalendarStatus::Failed, + JSCalendarProgress::Cancelled => ICalendarStatus::Cancelled, + }) + .import_converted( + &[JSCalendarProperty::Progress], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Status, + Value::Element(JSCalendarValue::EventStatus(value)), + ICalendarComponentType::VEvent, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Status) + .with_value(match value { + JSCalendarEventStatus::Confirmed => ICalendarStatus::Confirmed, + JSCalendarEventStatus::Cancelled => ICalendarStatus::Cancelled, + JSCalendarEventStatus::Tentative => ICalendarStatus::Tentative, + }) + .import_converted(&[JSCalendarProperty::Status], &mut root_conversions), + ); + } + ( + JSCalendarProperty::Status | JSCalendarProperty::Progress, + Value::Str(text), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Status) + .with_value( + ICalendarStatus::parse(text.as_bytes()) + .map(ICalendarValue::Status) + .unwrap_or_else(|| ICalendarValue::Text(text.into_owned())), + ) + .import_converted(&[JSCalendarProperty::Status], &mut root_conversions), + ); + } + ( + JSCalendarProperty::Source, + Value::Str(text), + ICalendarComponentType::VCalendar, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Source) + .with_value(Uri::parse(text.into_owned())) + .import_converted(&[JSCalendarProperty::Source], &mut root_conversions), + ); + } + ( + JSCalendarProperty::FreeBusyStatus, + Value::Element(JSCalendarValue::FreeBusyStatus(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Transp) + .with_value(match value { + JSCalendarFreeBusyStatus::Free => { + ICalendarTransparency::Transparent + } + JSCalendarFreeBusyStatus::Busy => ICalendarTransparency::Opaque, + }) + .import_converted( + &[JSCalendarProperty::FreeBusyStatus], + &mut root_conversions, + ), + ); + } + ( + JSCalendarProperty::Entries, + Value::Array(items), + ICalendarComponentType::VCalendar, + ) => { + for item in items { + if let Some(entries) = item.into_object().filter(|entries| { + entries + .is_type_or_untyped(&[JSCalendarType::Event, JSCalendarType::Task]) + }) { + self.from_jscalendar( + State { + tz: None, + tz_end: None, + tz_rid: None, + start: None, + recurrence_id: None, + uid: None, + entries, + is_date: false, + default_component_type: ICalendarComponentType::VEvent, + }, + Some(&mut component), + options, + ); + } + } + } + + // Skip previously processed properties + ( + JSCalendarProperty::Type + | JSCalendarProperty::ICalendar + | JSCalendarProperty::Uid + | JSCalendarProperty::MainLocationId + | JSCalendarProperty::Start + | JSCalendarProperty::TimeZone + | JSCalendarProperty::EndTimeZone + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::Locale + | JSCalendarProperty::OrganizerCalendarAddress + | JSCalendarProperty::SentBy + | JSCalendarProperty::DescriptionContentType + | JSCalendarProperty::Version + | JSCalendarProperty::Excluded + | JSCalendarProperty::RecurrenceOverrides, + _, + _, + ) + | (_, Value::Null, _) => {} + (property, value, _) => { + if !property.is_metadata() { + component.insert_jsprop(&[property.to_string().as_ref()], value); + } + } + } + } + + if !alarms_without_text.is_empty() + && let Some(text) = component + .property(&ICalendarProperty::Summary) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_text) + .or(uid.as_deref()) + .map(str::to_string) + { + for alarm_id in alarms_without_text { + if let Some(alarm) = self.components.get_mut(alarm_id as usize) { + alarm.add_alarm_text(&text); + } + } + } + + // Process recurrence overrides + if let Some(overrides) = overrides { + let mut exdates = RecurrenceDates::default(); + let mut rdates = RecurrenceDates::default(); + let converted_props = root_conversions.as_ref().map(|conv| { + conv.converted_props + .iter() + .filter_map(|(keys, _)| match (keys.first()?, keys.get(1)?) { + ( + Key::Property(JSCalendarProperty::RecurrenceOverrides), + Key::Property(JSCalendarProperty::DateTime(dt)), + ) => Some(*dt), + _ => None, + }) + .collect::>() + }); + + let non_instances = start.map(|start| { + component.non_recurrence_instances( + start, + state.tz.unwrap_or_default(), + state.is_date, + overrides + .as_vec() + .iter() + .filter_map(|(key, patch)| match (key, patch) { + ( + Key::Property(JSCalendarProperty::DateTime(dt)), + Value::Object(patch), + ) if !patch + .contains_true(&Key::Property(JSCalendarProperty::Excluded)) => + { + Some(dt.timestamp) + } + _ => None, + }), + options.max_expansions, + &mut options.unproductive_budget, + ) + }); + + for (key, value) in overrides.into_vec() { + if options.has_failed() { + break; + } + let (Key::Property(JSCalendarProperty::DateTime(jsdt)), Value::Object(mut obj)) = + (key, value) + else { + continue; + }; + obj.as_mut_vec().retain(|(key, value)| { + !matches!(key, Key::Property(JSCalendarProperty::Excluded)) + || matches!(value, Value::Bool(true)) + }); + let Some(dt) = jsdt + .to_naive_date_time() + .and_then(|dt| state.tz.unwrap_or_default().resolve_local_datetime(dt)) + else { + continue; + }; + let has_converted_prop = converted_props + .as_ref() + .is_some_and(|props| props.contains(&jsdt)); + + // RDATE with value type PERIOD + if has_converted_prop + && obj.len() == 1 + && let Some(Value::Element(JSCalendarValue::Duration(duration))) = + obj.lookup(&Key::Property(JSCalendarProperty::Duration)) + { + let duration = duration.clone(); + let mut entry = ICalendarEntry::new(ICalendarProperty::Rdate) + .import_converted( + &[ + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::DateTime(jsdt), + ], + &mut root_conversions, + ) + .with_period(dt, duration); + entry.name = ICalendarProperty::Rdate; + component.entries.push(entry); + continue; + } + + if !obj.is_empty() { + if !obj.contains_true(&Key::Property(JSCalendarProperty::Excluded)) { + let Some(template) = &mut override_template else { + continue; + }; + let instance = template.instance(jsdt); + let mut instance = match template.apply_patch(instance, obj) { + Ok(instance) => instance, + Err(pointer) => { + options.reject_patch(jsdt.to_rfc3339(), pointer); + if non_instances + .as_ref() + .is_none_or(|keys| keys.contains(&jsdt.timestamp)) + { + push_recurrence_date( + &mut rdates, + &mut component, + &mut root_conversions, + jsdt, + dt, + has_converted_prop, + ); + } + continue; + } + }; + if non_instances + .as_ref() + .is_some_and(|keys| keys.contains(&jsdt.timestamp)) + { + rdates.push(dt); + } + if let Some(privacy) = privacy { + instance.upsert( + Key::Property(JSCalendarProperty::Privacy), + Value::Element(JSCalendarValue::Privacy(privacy)), + ); + } + self.from_jscalendar( + State { + tz: None, + tz_end: None, + tz_rid: state.tz_rid, + start: None, + recurrence_id: Some(dt), + entries: instance, + uid: uid.as_deref(), + is_date: state.is_date, + default_component_type: component.component_type.clone(), + }, + parent_component.as_deref_mut(), + options, + ); + } else { + // EXDATE + if has_converted_prop { + exdates.push_converted( + &mut component, + ICalendarEntry::new(ICalendarProperty::Exdate).import_converted( + &[ + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::DateTime(jsdt), + ], + &mut root_conversions, + ), + dt, + ); + } else { + exdates.push(dt); + } + } + } else if non_instances + .as_ref() + .is_none_or(|keys| keys.contains(&jsdt.timestamp)) + { + // RDATE + push_recurrence_date( + &mut rdates, + &mut component, + &mut root_conversions, + jsdt, + dt, + has_converted_prop, + ); + } + } + + rdates.write_to(&mut component, ICalendarProperty::Rdate, state.is_date); + exdates.write_to(&mut component, ICalendarProperty::Exdate, state.is_date); + } + + // Add organizer, if not present in participants + if let Some(organizer_address) = organizer_address { + let mut organizer = ICalendarEntry::new(ICalendarProperty::Organizer); + if let Some(sent_by) = organizer_sent_by + && !organizer_params + .iter() + .any(|param| param.name == ICalendarParameterName::SentBy) + { + organizer_params.push(ICalendarParameter::sent_by(Uri::parse( + sent_by.into_owned(), + ))); + } + organizer.params = organizer_params; + organizer.values = smallvec![ICalendarValue::Uri(Uri::parse( + organizer_address.into_owned(), + ))]; + + let mut creates_owner = organizer.params.iter().any(|param| { + matches!( + param.name, + ICalendarParameterName::Cn + | ICalendarParameterName::Email + | ICalendarParameterName::SentBy + | ICalendarParameterName::Jsid + ) + }) || !component.entries.iter().any(|entry| { + entry.name == ICalendarProperty::Attendee + && entry.params.iter().any(|param| { + matches!( + param.value, + ICalendarParameterValue::Role(ICalendarParticipationRole::Owner) + ) + }) + }); + let mut restores_owner = false; + if !creates_owner { + match organizer_owner { + Some(OrganizerOwner::Attendee { entry, position }) => { + if let Some(attendee) = component.entries.get_mut(entry) { + attendee.params.insert( + position.min(attendee.params.len()), + ICalendarParameter::role(ICalendarParticipationRole::Owner), + ); + restores_owner = true; + } + } + Some(OrganizerOwner::Organizer(participant_id)) => { + organizer + .params + .push(ICalendarParameter::jsid(participant_id)); + creates_owner = true; + } + None => {} + } + } + + if let Some((participant_id, roles)) = organizer_unmapped_roles { + component.insert_roles_jsprop::( + &participant_id, + roles, + JSPropSet::of(creates_owner || restores_owner), + ); + } + + // Restore the parameters of Participant objects that only convert to ORGANIZER + if let Some(participant_id) = organizer_participant_id { + organizer = organizer.import_converted_with_id( + &[JSCalendarProperty::Participants], + &mut root_conversions, + Some(participant_id.as_str()), + ); + organizer.name = ICalendarProperty::Organizer; + } + + component.entries.push(organizer); + } + + // Add parent recurrence ID + if add_recurrence_id && let Some(dt) = state.recurrence_id { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::RecurrenceId) + .import_converted(&[JSCalendarProperty::RecurrenceId], &mut root_conversions) + .with_date(dt, series_is_date), + ); + } + + // Add showWithoutTime: true unless there are times with DATE types + if let Some(show_without_time) = show_without_time + && matches!( + show_without_time.values.first(), + Some(ICalendarValue::Boolean(true)) + ) + { + let mut has_date = false; + let mut has_date_time = false; + for entry in component.entries.iter().filter(|entry| { + matches!( + entry.name, + ICalendarProperty::Dtstart | ICalendarProperty::Due + ) + }) { + if matches!( + entry.parameter(&ICalendarParameterName::Value), + Some(ICalendarParameterValue::Value(ICalendarValueType::Date)) + ) { + has_date = true; + } else { + has_date_time = true; + } + } + if has_date_time { + component + .entries + .push(show_without_time.with_value_type(ICalendarValueType::Boolean)); + } else if !has_date { + component.insert_jsprop::( + &[JSCalendarProperty::ShowWithoutTime:: + .to_string() + .as_ref()], + Value::Bool(true), + ); + } + } + + if matches!( + component.component_type, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo + ) { + let has_preserved_class = root_conversions.as_mut().is_some_and(|conversions| { + conversions.retain_class_property(matches!( + privacy, + None | Some(JSCalendarPrivacy::Private) + )) + }); + + if !has_preserved_class && let Some(privacy) = privacy { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Class) + .with_value(match privacy { + JSCalendarPrivacy::Public => ICalendarClassification::Public, + JSCalendarPrivacy::Private => ICalendarClassification::Private, + JSCalendarPrivacy::Secret => ICalendarClassification::Confidential, + }) + .import_converted(&[JSCalendarProperty::Privacy], &mut root_conversions), + ); + } + } + + if let Some(root_conversions) = root_conversions { + component = root_conversions.apply_conversions(component, self); + } + + // The VERSION property is required by RFC 5545 and has no JSCalendar counterpart + if component.component_type == ICalendarComponentType::VCalendar + && !component + .entries + .iter() + .any(|entry| entry.name == ICalendarProperty::Version) + { + component.entries.push( + ICalendarEntry::new(ICalendarProperty::Version).with_value("2.0".to_string()), + ); + } + + if let Some(parent_component) = parent_component { + parent_component + .component_ids + .push(self.push_component(component)); + } else if let Some(root) = self.components.first_mut() { + *root = component; + } + } + + pub fn push_component(&mut self, component: ICalendarComponent) -> u32 { + let comp_num = self.components.len(); + self.components.push(component); + comp_num as u32 + } +} + +impl ICalendarComponent { + fn import_links( + &mut self, + obj: Map<'_, JSCalendarProperty, JSCalendarValue>, + conversion: &mut Option>, + options: &mut ExportContext<'_, B>, + ) { + let mut has_url = self + .entries + .iter() + .any(|entry| entry.name == ICalendarProperty::Url); + let mut has_links = false; + let mut jsprop_links = Vec::new(); + + for (name, value) in obj.into_vec() { + let Value::Object(mut link) = value else { + jsprop_links.push((name.into_owned(), value.into_owned())); + continue; + }; + let has_blob = matches!( + link.lookup(&Key::Property(JSCalendarProperty::BlobId)), + Some(Value::Element(JSCalendarValue::BlobId(_)) | Value::Str(_)) + ); + if has_blob && options.blobs.is_none() { + jsprop_links.push((name.into_owned(), Value::Object(link).into_owned())); + continue; + } + let has_rel = match link.lookup(&Key::Property(JSCalendarProperty::Rel)) { + None => false, + Some(Value::Element(JSCalendarValue::LinkRelation(_))) => true, + Some(Value::Str(rel)) if rel.is_link_relation_type() => true, + Some(_) + if has_blob + || matches!( + link.lookup(&Key::Property(JSCalendarProperty::Display)), + Some(Value::Object(_)) + ) => + { + true + } + Some(_) => { + jsprop_links.push((name.into_owned(), Value::Object(link).into_owned())); + continue; + } + }; + let uri = if has_blob { + None + } else { + let Some(href) = + link.as_mut_vec() + .iter_mut() + .find_map(|(key, value)| match (key, value) { + (Key::Property(JSCalendarProperty::Href), Value::Str(href)) => { + Some(href) + } + _ => None, + }) + else { + jsprop_links.push((name.into_owned(), Value::Object(link).into_owned())); + continue; + }; + match Uri::parse(std::mem::take(href).into_owned()) { + Uri::Location(text) if text.is_data_uri() => { + *href = Cow::Owned(text); + jsprop_links.push((name.into_owned(), Value::Object(link).into_owned())); + continue; + } + uri => Some(uri), + } + }; + let mut entry = ICalendarEntry::new(ICalendarProperty::Link); + let mut link_rel = None; + let mut rel_text = None; + let mut title = None; + let mut has_display = false; + let mut references_key = false; + let mut blob = None; + + for (sub_property, value) in link.into_vec() { + match (sub_property, value) { + ( + Key::Property(JSCalendarProperty::Type | JSCalendarProperty::ICalendar), + _, + ) => {} + ( + Key::Property(JSCalendarProperty::BlobId), + Value::Element(JSCalendarValue::BlobId(blob_id)), + ) if has_blob => { + blob = Some( + options + .blobs + .as_mut() + .and_then(|blobs| blobs.resolve(&blob_id)) + .ok_or_else(|| blob_id.to_string()), + ); + } + (Key::Property(JSCalendarProperty::BlobId), Value::Str(text)) if has_blob => { + blob = Some(Err(text.into_owned())); + } + (Key::Property(JSCalendarProperty::Href | JSCalendarProperty::Size), _) + if has_blob => {} + (Key::Property(JSCalendarProperty::Href), Value::Str(_)) => {} + (Key::Property(JSCalendarProperty::ContentType), Value::Str(text)) => { + entry + .params + .push(ICalendarParameter::fmttype(text.into_owned())); + } + (Key::Property(JSCalendarProperty::Size), Value::Number(number)) => { + match number.as_u64() { + Some(size) => entry.params.push(ICalendarParameter::size(size)), + None => { + references_key |= self.insert_jsprop::( + &[ + JSCalendarProperty::Links::.to_string().as_ref(), + name.to_string().as_ref(), + JSCalendarProperty::Size::.to_string().as_ref(), + ], + Value::Number(number), + ); + } + } + } + ( + Key::Property(JSCalendarProperty::Rel), + Value::Element(JSCalendarValue::LinkRelation(relation)), + ) => { + entry + .params + .push(ICalendarParameter::linkrel(ICalendarParameterValue::Null)); + link_rel = Some(relation); + } + (Key::Property(JSCalendarProperty::Rel), Value::Str(text)) => { + entry + .params + .push(ICalendarParameter::linkrel(ICalendarParameterValue::Null)); + rel_text = Some(text.into_owned()); + } + (Key::Property(JSCalendarProperty::Display), Value::Object(obj)) => { + has_display = true; + let mut vendor_displays = Vec::new(); + for key in obj.into_expanded_boolean_set() { + let value = match key { + Key::Property(JSCalendarProperty::LinkDisplay(display)) => { + ICalendarParameterValue::Display(match display { + JSCalendarLinkDisplay::Badge => ICalendarDisplayType::Badge, + JSCalendarLinkDisplay::Graphic => { + ICalendarDisplayType::Graphic + } + JSCalendarLinkDisplay::Fullsize => { + ICalendarDisplayType::Fullsize + } + JSCalendarLinkDisplay::Thumbnail => { + ICalendarDisplayType::Thumbnail + } + }) + } + other if other.to_string().is_param_token() => { + ICalendarParameterValue::Text(other.into_string()) + } + other => { + vendor_displays.push(other.into_owned()); + continue; + } + }; + entry.params.push(ICalendarParameter::display(value)); + } + if !vendor_displays.is_empty() { + references_key = true; + let displays = JSPropSet::of( + entry.has_parameter(&ICalendarParameterName::Display), + ); + self.insert_set_jsprop::( + &[ + JSCalendarProperty::Links::.to_string().as_ref(), + name.to_string().as_ref(), + JSCalendarProperty::Display::.to_string().as_ref(), + ], + vendor_displays, + displays, + ); + } + } + (Key::Property(JSCalendarProperty::Title), Value::Str(text)) => { + entry + .params + .push(ICalendarParameter::filename(ICalendarParameterValue::Null)); + title = Some(text.into_owned()); + } + (sub_property, value) => { + references_key |= self.insert_jsprop( + &[ + JSCalendarProperty::Links::.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + /* + Which iCalendar property to choose is implementation-specific. As a guideline, + stopping at the first matching criteria: + + 1. Choose ATTACH if the "rel" property value is "enclosure". + 2. Choose IMAGE if the "rel" property value is "icon". + 3. Choose URL if the "rel" property value is "describedby" and no URL property + is set in the iCalendar component. + 4. Choose LINK otherwise. + */ + + entry.name = match link_rel { + Some(LinkRelation::Enclosure) => ICalendarProperty::Attach, + Some(LinkRelation::Icon) => ICalendarProperty::Image, + Some(LinkRelation::Describedby) => ICalendarProperty::Url, + None if has_display => ICalendarProperty::Image, + None if !has_rel => ICalendarProperty::Attach, + _ => ICalendarProperty::Link, + }; + + let is_uuid5_key = match (blob, uri) { + (Some(Ok(bytes)), _) => { + if !options.embed(&bytes) { + continue; + } + if !matches!(entry.name, ICalendarProperty::Image) { + entry.name = ICalendarProperty::Attach; + } + let is_uuid5_key = name.is_uuid5_of(&bytes); + entry.values = smallvec![ICalendarValue::Binary(bytes)]; + is_uuid5_key + } + (Some(Err(blob_id)), _) => { + options.fail(ExportError::UnresolvedBlob { blob_id }); + continue; + } + (None, Some(Uri::Data(data))) => { + if !options.embed(&data.data) { + continue; + } + let is_uuid5_key = name.is_uuid5_of(&data.data); + entry.values = smallvec![ICalendarValue::Uri(Uri::Data(data))]; + is_uuid5_key + } + (None, Some(Uri::Location(href))) => { + let is_uuid5_key = name.is_uuid5_of(href.as_bytes()); + entry.values = smallvec![ICalendarValue::Uri(Uri::Location(href))]; + is_uuid5_key + } + (None, None) => false, + }; + + let link_id = name.into_string(); + let mut entry = entry.import_converted_with_id( + &[JSCalendarProperty::Links], + conversion, + Some(link_id.as_str()), + ); + + // At most one URL property is allowed per component + if entry.name == ICalendarProperty::Url { + if !has_url { + has_url = true; + } else { + entry.name = ICalendarProperty::Link; + } + } + + /* + The LINKREL parameter is not specified for the ATTACH, IMAGE and URL properties. + For interoperability, implementations SHOULD NOT set the LINKREL parameter on + these properties. + */ + + let is_binary = matches!(entry.values.first(), Some(ICalendarValue::Binary(_))); + let has_linkrel = match entry.name { + ICalendarProperty::Link => true, + ICalendarProperty::Attach | ICalendarProperty::Image => { + is_binary + && !matches!(link_rel, Some(LinkRelation::Enclosure | LinkRelation::Icon)) + } + _ => false, + }; + let mut linkrel = match (link_rel, rel_text) { + (Some(relation), _) if has_linkrel => { + Some(ICalendarParameterValue::Linkrel(relation)) + } + (None, Some(text)) if has_linkrel && text.is_link_relation_type() => { + Some(ICalendarParameterValue::Text(text)) + } + (None, Some(text)) => { + references_key |= self.insert_jsprop::( + &[ + JSCalendarProperty::Links::.to_string().as_ref(), + link_id.as_str(), + JSCalendarProperty::Rel::.to_string().as_ref(), + ], + Value::Str(text.into()), + ); + None + } + _ => None, + }; + + if !is_uuid5_key || references_key { + entry.add_param(ICalendarParameter::jsid(link_id)); + } + + /* + The "title" property converts to the FILENAME parameter for the ATTACH and IMAGE + properties. It converts to the LABEL parameter for the LINK and URL properties. + */ + + let title_name = if matches!( + entry.name, + ICalendarProperty::Attach | ICalendarProperty::Image + ) { + ICalendarParameterName::Filename + } else { + ICalendarParameterName::Label + }; + entry + .params + .retain_mut(|param| match (¶m.name, ¶m.value) { + (ICalendarParameterName::Linkrel, ICalendarParameterValue::Null) => { + linkrel.take().is_some_and(|linkrel| { + param.value = linkrel; + true + }) + } + (ICalendarParameterName::Filename, ICalendarParameterValue::Null) => { + title.take().is_some_and(|title| { + param.name = title_name.clone(); + param.value = ICalendarParameterValue::Text(title); + true + }) + } + _ => true, + }); + + if entry.name == ICalendarProperty::Link + && !entry.has_parameter(&ICalendarParameterName::Linkrel) + { + entry.add_param(ICalendarParameter::linkrel(LinkRelation::Enclosure)); + } + + let entry = if is_binary { + entry.with_value_type(ICalendarValueType::Binary) + } else { + entry.with_uri_value_type() + }; + + has_links = true; + self.entries.push(entry); + } + + if has_links { + for (name, value) in jsprop_links { + self.insert_jsprop( + &[ + JSCalendarProperty::Links::.to_string().as_ref(), + name.to_string().as_ref(), + ], + value, + ); + } + } else if !jsprop_links.is_empty() { + self.insert_jsprop( + &[JSCalendarProperty::Links::.to_string().as_ref()], + Value::Object(Map::from(jsprop_links)), + ); + } + } + + fn import_relations( + &mut self, + obj: Map<'_, JSCalendarProperty, JSCalendarValue>, + conversion: &mut Option>, + ) { + for (name, value) in obj.into_vec() { + let mut entry = ICalendarEntry::new(ICalendarProperty::RelatedTo); + + for (sub_property, value) in value.into_expanded_object() { + match (sub_property, value) { + (Key::Property(JSCalendarProperty::Relation), Value::Object(obj)) => { + let mut vendor_relations = Vec::new(); + for key in obj.into_expanded_boolean_set() { + let value = match key { + Key::Property(JSCalendarProperty::RelationValue(relation)) => { + ICalendarParameterValue::Reltype(match relation { + JSCalendarRelation::First => { + ICalendarRelationshipType::First + } + JSCalendarRelation::Next => ICalendarRelationshipType::Next, + JSCalendarRelation::Child => { + ICalendarRelationshipType::Child + } + JSCalendarRelation::Parent => { + ICalendarRelationshipType::Parent + } + JSCalendarRelation::Snooze => { + ICalendarRelationshipType::Snooze + } + }) + } + other if other.to_string().is_param_token() => { + ICalendarParameterValue::Text(other.into_string()) + } + other => { + vendor_relations.push(other.into_owned()); + continue; + } + }; + entry.params.push(ICalendarParameter::reltype(value)); + } + if !vendor_relations.is_empty() { + let relations = JSPropSet::of( + entry.has_parameter(&ICalendarParameterName::Reltype), + ); + self.insert_set_jsprop::( + &[ + JSCalendarProperty::RelatedTo::.to_string().as_ref(), + name.to_string().as_ref(), + JSCalendarProperty::Relation::.to_string().as_ref(), + ], + vendor_relations, + relations, + ); + } + } + ( + Key::Property(JSCalendarProperty::Type | JSCalendarProperty::ICalendar), + _, + ) => {} + (sub_property, value) => { + self.insert_jsprop( + &[ + JSCalendarProperty::RelatedTo::.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + self.entries.push( + entry + .with_value(name.into_string()) + .import_converted(&[JSCalendarProperty::RelatedTo], conversion), + ); + } + } + + fn insert_roles_jsprop( + &mut self, + participant_id: &str, + roles: Vec>>, + set: JSPropSet, + ) { + self.insert_set_jsprop::( + &[ + JSCalendarProperty::Participants::.to_string().as_ref(), + participant_id, + JSCalendarProperty::Roles::.to_string().as_ref(), + ], + roles, + set, + ); + } + + fn insert_set_jsprop( + &mut self, + path: &[&str], + keys: Vec>>, + set: JSPropSet, + ) { + if set == JSPropSet::Exists { + for key in keys { + self.insert_encoded_jsprop::( + String::from_pointer(path.iter().copied().chain([key.to_string().as_ref()])), + Value::Bool(true), + ); + } + } else { + self.insert_jsprop::( + path, + Value::Object(Map::from( + keys.into_iter() + .map(|key| (key, Value::Bool(true))) + .collect::>(), + )), + ); + } + } + + fn insert_jsprop( + &mut self, + path: &[&str], + value: Value<'_, JSCalendarProperty, JSCalendarValue>, + ) -> bool { + if path.last().copied() == Some(JSCalendarProperty::::Excluded.to_string().as_ref()) { + return false; + } + self.insert_encoded_jsprop(String::from_pointer(path.iter().copied()), value) + } + + fn insert_encoded_jsprop( + &mut self, + pointer: String, + value: Value<'_, JSCalendarProperty, JSCalendarValue>, + ) -> bool { + let Some(value) = (!value.is_null()) + .then(|| serde_json::to_string(&value).ok()) + .flatten() + else { + return false; + }; + self.entries.push( + ICalendarEntry::new(ICalendarProperty::Jsprop) + .with_param(ICalendarParameter::jsptr(pointer)) + .with_value(value), + ); + true + } + + fn missing_alarm_text(&self) -> bool { + match self + .property(&ICalendarProperty::Action) + .and_then(|entry| entry.values.first()) + { + Some(ICalendarValue::Action(ICalendarAction::Display)) => { + !self.has_property(&ICalendarProperty::Description) + } + Some(ICalendarValue::Action(ICalendarAction::Email)) => { + !self.has_property(&ICalendarProperty::Description) + || !self.has_property(&ICalendarProperty::Summary) + } + _ => false, + } + } + + fn add_alarm_text(&mut self, text: &str) { + let is_email = matches!( + self.property(&ICalendarProperty::Action) + .and_then(|entry| entry.values.first()), + Some(ICalendarValue::Action(ICalendarAction::Email)) + ); + if !self.has_property(&ICalendarProperty::Description) { + self.entries.push( + ICalendarEntry::new(ICalendarProperty::Description) + .with_param(ICalendarParameter::derived(true)) + .with_value(text.to_string()), + ); + } + if is_email && !self.has_property(&ICalendarProperty::Summary) { + self.entries.push( + ICalendarEntry::new(ICalendarProperty::Summary) + .with_param(ICalendarParameter::derived(true)) + .with_value(text.to_string()), + ); + } + } +} + +impl ICalendarEntry { + fn with_value_type(mut self, value_type: ICalendarValueType) -> Self { + if !self.has_parameter(&ICalendarParameterName::Value) { + self.params.push(ICalendarParameter::value(value_type)); + } + self + } + + fn with_uri_value_type(self) -> Self { + if matches!( + self.name, + ICalendarProperty::Conference + | ICalendarProperty::Coordinates + | ICalendarProperty::Image + | ICalendarProperty::Link + ) && matches!(self.values.first(), Some(ICalendarValue::Uri(_))) + { + self.with_value_type(ICalendarValueType::Uri) + } else { + self + } + } +} + +enum OrganizerOwner { + Attendee { entry: usize, position: usize }, + Organizer(String), +} + +trait ParameterText { + fn is_param_token(&self) -> bool; + fn is_data_uri(&self) -> bool; + fn is_link_relation_type(&self) -> bool; +} + +impl ParameterText for str { + fn is_param_token(&self) -> bool { + !self.is_empty() + && self + .bytes() + .all(|ch| ch.is_ascii_alphanumeric() || ch == b'-') + } + + fn is_data_uri(&self) -> bool { + self.get(..5) + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("data:")) + } + + fn is_link_relation_type(&self) -> bool { + self.is_param_token() + || self.split_once(':').is_some_and(|(scheme, _)| { + scheme + .bytes() + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && scheme + .bytes() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'+' | b'-' | b'.')) + }) + } +} + +/// Returns the end of an event that starts at `start` and lasts `duration`. +/// +/// RFC 5545 section 3.3.6 keeps the day and week components of a duration +/// nominal, so the end depends on where the start falls. +fn end_after(start: ZonedDateTime, duration: &ICalendarDuration) -> Option { + start.checked_add_nominal(duration.to_nominal()?) +} + +fn parse_geo(text: Cow<'_, str>) -> Vec { + if let Some((a, b)) = text + .strip_prefix("geo:") + .and_then(|v| v.trim().split_once(',')) + .and_then(|(a, b)| { + let a = a.parse::().ok().filter(|a| a.is_finite())?; + let b = b.parse::().ok().filter(|b| b.is_finite())?; + Some((a, b)) + }) + { + vec![ICalendarValue::Float(a), ICalendarValue::Float(b)] + } else { + vec![ICalendarValue::Text(text.into_owned())] + } +} + +fn push_recurrence_date( + rdates: &mut RecurrenceDates, + component: &mut ICalendarComponent, + conversions: &mut Option>, + jsdt: JSCalendarDateTime, + dt: ZonedDateTime, + has_converted_prop: bool, +) { + if has_converted_prop { + rdates.push_converted( + component, + ICalendarEntry::new(ICalendarProperty::Rdate).import_converted( + &[ + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::DateTime(jsdt), + ], + conversions, + ), + dt, + ); + } else { + rdates.push(dt); + } +} + +#[derive(Default)] +struct RecurrenceDates { + dates: Vec, + converted: Vec<(usize, Vec)>, +} + +impl RecurrenceDates { + fn push(&mut self, dt: ZonedDateTime) { + match self.converted.last_mut() { + Some((_, dates)) => dates.push(dt), + None => self.dates.push(dt), + } + } + + fn push_converted( + &mut self, + component: &mut ICalendarComponent, + entry: ICalendarEntry, + dt: ZonedDateTime, + ) { + self.converted.push((component.entries.len(), vec![dt])); + component.entries.push(entry); + } + + fn write_to( + self, + component: &mut ICalendarComponent, + property: ICalendarProperty, + is_date: bool, + ) { + for (index, dates) in self.converted { + if let Some(entry) = component.entries.get_mut(index) { + let converted = std::mem::replace(entry, ICalendarEntry::new(property.clone())); + *entry = converted.with_dates(dates, is_date); + } + } + if !self.dates.is_empty() { + component + .entries + .push(ICalendarEntry::new(property).with_dates(self.dates, is_date)); + } + } +} + +impl ICalendarComponent { + /// Returns the candidate wall clock readings the rule does not generate. + /// + /// The rule is expanded in `tz` rather than as a floating series, because + /// an UNTIL value is written in UTC: comparing it against a floating + /// instance would place the bound an offset away from where it belongs and + /// drop the last instance of the series. Instances are then compared by + /// wall clock reading, which is the space the candidates are keyed in. + fn non_recurrence_instances( + &self, + start: civil::DateTime, + tz: Tz, + is_date: bool, + candidates: impl Iterator, + max_expansions: usize, + unproductive_budget: &mut usize, + ) -> AHashSet { + let start_timestamp = Offset::UTC + .to_timestamp(start) + .map_or(0, |ts| ts.as_second()); + let mut pending = candidates + .filter(|candidate| *candidate != start_timestamp) + .collect::>(); + + if let Some(rule) = self + .entries + .iter() + .find_map(|entry| match entry.values.first() { + Some(ICalendarValue::RecurrenceRule(rule)) + if entry.name == ICalendarProperty::Rrule => + { + Some(rule.as_ref()) + } + _ => None, + }) + && let Some(zoned_start) = tz.from_local(start) + && let Ok(rule) = RRule::from_ical(rule, zoned_start, is_date) + && let Some(last) = pending + .iter() + .filter(|candidate| **candidate > start_timestamp) + .max() + .copied() + { + let mut instances = rule.iter().with_unproductive_budget(*unproductive_budget); + for date in instances.by_ref().take(max_expansions) { + let date = date.naive_timestamp(); + if date > last || (pending.remove(&date) && pending.is_empty()) { + break; + } + } + *unproductive_budget = instances.unproductive_budget(); + } + + pending + } +} + diff --git a/crates/calcard/src/jscalendar/export/mod.rs b/crates/calcard/src/jscalendar/export/mod.rs new file mode 100644 index 00000000..bbf1b746 --- /dev/null +++ b/crates/calcard/src/jscalendar/export/mod.rs @@ -0,0 +1,211 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::timezone::ZonedDateTime; +use crate::{ + common::{ + blob::{BlobOptions, BlobResolver, BlobResolverFn, NoBlobIds, ResolvedBlobs}, + export::{EmbeddedBudget, ExportError, RejectedPatch}, + timezone::Tz, + }, + datecalc::MAX_UNPRODUCTIVE_WORK, + icalendar::{ICalendar, ICalendarComponentType}, + jscalendar::{JSCalendarId, JSCalendarProperty, JSCalendarValue}, +}; +use jmap_tools::{Key, Map, Value}; +use std::hash::Hash; + +pub mod convert; +pub mod params; +pub mod props; + +struct State<'x, I: JSCalendarId, B: JSCalendarId> { + entries: Map<'x, JSCalendarProperty, JSCalendarValue>, + default_component_type: ICalendarComponentType, + uid: Option<&'x str>, + tz: Option, + tz_end: Option, + tz_rid: Option, + start: Option, + recurrence_id: Option, + is_date: bool, +} + +#[allow(clippy::type_complexity)] +struct ConvertedComponent<'x, I: JSCalendarId, B: JSCalendarId> { + pub(super) name: ICalendarComponentType, + pub(super) converted_props: Vec<( + Vec>>, + Value<'x, JSCalendarProperty, JSCalendarValue>, + )>, + pub(super) converted_props_count: usize, + pub(super) properties: Vec, JSCalendarValue>>, + pub(super) components: Vec, JSCalendarValue>>, +} + +#[derive(Debug, Clone, Copy)] +pub struct ExportOptions { + max_expansions: usize, + blobs: BlobOptions, +} + +pub(super) struct ExportContext<'a, B> { + max_expansions: usize, + unproductive_budget: usize, + blobs: Option>, + budget: EmbeddedBudget, + error: Option, + rejected_patches: Vec, +} + +const DEFAULT_MAX_EXPANSIONS: usize = 3000; +const ENTRY_CAPACITY_EXTRA: usize = 1; + +impl Default for ExportOptions { + fn default() -> Self { + Self { + max_expansions: DEFAULT_MAX_EXPANSIONS, + blobs: BlobOptions::default(), + } + } +} + +impl ExportOptions { + pub fn new() -> Self { + Self::default() + } +} + +impl ExportOptions { + pub fn max_expansions(mut self, max_expansions: usize) -> Self { + self.max_expansions = max_expansions; + self + } + + pub fn with_blob_resolver(self, blob_resolver: F) -> ExportOptions> + where + F: FnMut(&B) -> Option>, + { + self.with_resolver(BlobResolverFn(blob_resolver)) + } + + pub fn with_resolver(self, blob_resolver: T) -> ExportOptions { + ExportOptions { + max_expansions: self.max_expansions, + blobs: self.blobs.with_handler(blob_resolver), + } + } + + pub fn max_embedded_size(mut self, max_embedded_size: usize) -> Self { + self.blobs = self.blobs.with_max_embedded_size(max_embedded_size); + self + } + + pub(super) fn context(&mut self) -> ExportContext<'_, B> + where + R: BlobResolver, + { + ExportContext { + max_expansions: self.max_expansions, + unproductive_budget: MAX_UNPRODUCTIVE_WORK, + budget: self.blobs.budget(), + blobs: self.blobs.resolved_blobs(), + error: None, + rejected_patches: Vec::new(), + } + } +} + +impl State<'_, I, B> { + fn entry_capacity(&self) -> usize { + self.entries + .iter() + .map(|(key, value)| match (key, value) { + ( + Key::Property( + JSCalendarProperty::Participants + | JSCalendarProperty::Links + | JSCalendarProperty::Locations + | JSCalendarProperty::VirtualLocations + | JSCalendarProperty::Categories + | JSCalendarProperty::RelatedTo + | JSCalendarProperty::RecurrenceOverrides, + ), + Value::Object(members), + ) => members.len(), + (Key::Property(JSCalendarProperty::ICalendar), Value::Object(ical)) => ical + .iter() + .filter_map(|(key, value)| match (key, value) { + (Key::Property(JSCalendarProperty::Properties), Value::Array(items)) => { + Some(items.len()) + } + _ => None, + }) + .sum(), + ( + Key::Property( + JSCalendarProperty::Type + | JSCalendarProperty::Alerts + | JSCalendarProperty::Entries + | JSCalendarProperty::TimeZone + | JSCalendarProperty::EndTimeZone + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::Locale + | JSCalendarProperty::MainLocationId + | JSCalendarProperty::DescriptionContentType + | JSCalendarProperty::SentBy + | JSCalendarProperty::Version + | JSCalendarProperty::Excluded + | JSCalendarProperty::Method + | JSCalendarProperty::Metadata + | JSCalendarProperty::PrivateMetadata, + ), + _, + ) => 0, + _ => 1, + }) + .sum::() + + ENTRY_CAPACITY_EXTRA + } +} + +impl ExportContext<'_, B> { + fn fail(&mut self, error: ExportError) { + if self.error.is_none() { + self.error = Some(error); + } + } + + fn has_failed(&self) -> bool { + self.error.is_some() + } + + fn reject_patch(&mut self, recurrence_id: String, pointer: String) { + self.rejected_patches.push(RejectedPatch { + recurrence_id, + pointer, + }); + } + + fn embed(&mut self, data: &[u8]) -> bool { + match self.budget.take(data) { + Err(error) => { + self.blobs = None; + self.fail(error); + false + } + Ok(()) => true, + } + } + + fn into_result(self, ical: ICalendar) -> Result<(ICalendar, Vec), ExportError> { + match self.error { + Some(error) => Err(error), + None if ical.has_calendar_components() => Ok((ical, self.rejected_patches)), + None => Err(ExportError::NoComponents), + } + } +} diff --git a/crates/calcard/src/jscalendar/export/params.rs b/crates/calcard/src/jscalendar/export/params.rs new file mode 100644 index 00000000..5a0fc333 --- /dev/null +++ b/crates/calcard/src/jscalendar/export/params.rs @@ -0,0 +1,441 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + IanaParse, LinkRelation, PartialDateTime, + parser::Boolean, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::*, + jscalendar::{ + JSCalendarDateTime, JSCalendarId, JSCalendarProperty, JSCalendarValue, + export::ConvertedComponent, ext::JSCalendarKeyExt, + }, + jscontact::export::params::ParamValue, +}; +use jiff::{Timestamp, civil, tz::Offset}; +use jmap_tools::{Key, Value}; +use std::str::FromStr; + +impl ICalendarEntry { + pub(super) fn import_converted( + self, + path: &[JSCalendarProperty], + conversions: &mut Option>, + ) -> Self { + self.import_converted_with_id(path, conversions, None) + } + + pub(super) fn import_converted_with_id( + mut self, + path: &[JSCalendarProperty], + conversions: &mut Option>, + id: Option<&str>, + ) -> Self { + let Some(conversions) = conversions + .as_mut() + .filter(|c| c.converted_props_count < c.converted_props.len()) + else { + return self; + }; + + // Obtain jsid + let value = self.values.first().and_then(|v| v.as_text()); + let jsid = if matches!(self.name, ICalendarProperty::RelatedTo) { + value + } else { + id.or_else(|| self.jsid()) + }; + + let mut matched_once = false; + + 'outer: for (keys, value) in conversions.converted_props.iter_mut() { + if matches!(value, Value::Null) { + continue; + } + + for (pos, item) in path.iter().enumerate() { + if !keys + .iter() + .any(|k| matches!(k, Key::Property(p) if p == item)) + { + if pos == 0 && matched_once { + // Array is sorted, so if we didn't match the first item, + // we won't match any further. + break 'outer; + } else { + continue 'outer; + } + } else { + matched_once = true; + } + } + + if jsid + .map(Key::Borrowed) + .is_none_or(|prop_id| keys.iter().any(|k| k.same_key(&prop_id))) + { + self.import_converted_properties(std::mem::take(value)); + conversions.converted_props_count += 1; + break; + } + } + + // Update binary contents + if self + .parameter(&ICalendarParameterName::Value) + .is_some_and(|v| { + matches!( + v, + ICalendarParameterValue::Value(ICalendarValueType::Binary) + ) + }) + && let Some(ICalendarValue::Uri(Uri::Data(data))) = self.values.first_mut() + { + let bin = ICalendarValue::Binary(std::mem::take(&mut data.data)); + if let Some(content_type) = data.content_type.take() + && !self.has_parameter(&ICalendarParameterName::Fmttype) + { + self.params.push(ICalendarParameter::fmttype(content_type)); + } + self.values[0] = bin; + } + + self + } + + pub(super) fn import_converted_properties( + &mut self, + props: Value<'_, JSCalendarProperty, JSCalendarValue>, + ) { + for (key, value) in props.into_expanded_object() { + match key { + Key::Property(JSCalendarProperty::Name) => { + if let Some(name) = value.into_string() { + self.name = ICalendarProperty::parse(name.as_bytes()) + .unwrap_or(ICalendarProperty::Other(name.to_ascii_uppercase())); + } + } + Key::Property(JSCalendarProperty::Parameters) => { + self.import_jcal_params(value); + } + _ => {} + } + } + } + + pub(super) fn import_jcal_params( + &mut self, + params: Value<'_, JSCalendarProperty, JSCalendarValue>, + ) { + for (key, value) in params.into_expanded_object() { + let (first, rest) = match value { + Value::Array(values) => (None, values), + value => (Some(value), Vec::new()), + }; + let mut values = first + .into_iter() + .chain(rest) + .filter_map(ParamValue::try_from_value) + .peekable(); + + if values.peek().is_none() { + continue; + } + + let key = key.to_string(); + let Some(param) = ICalendarParameterName::try_parse(key.as_bytes()) else { + let key = key.into_owned(); + + for value in values { + self.params.push(ICalendarParameter { + name: ICalendarParameterName::Other(key.to_ascii_uppercase()), + value: value.into_string().into_owned().into(), + }); + } + + continue; + }; + + for value in values { + let value = match ¶m { + ICalendarParameterName::Value => { + let value = value.into_string(); + ICalendarValueType::parse(value.as_bytes()) + .map(ICalendarParameterValue::Value) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Size | ICalendarParameterName::Order => { + match value.into_number() { + Ok(n) => ICalendarParameterValue::Integer(n.unsigned_abs()), + Err(value) => { + ICalendarParameterValue::Text(value.into_string().into_owned()) + } + } + } + ICalendarParameterName::Rsvp | ICalendarParameterName::Derived => { + let value = value.into_string(); + Boolean::parse(value.as_bytes()) + .map(|v| ICalendarParameterValue::Bool(v.0)) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Altrep + | ICalendarParameterName::DelegatedFrom + | ICalendarParameterName::DelegatedTo + | ICalendarParameterName::Dir + | ICalendarParameterName::Member + | ICalendarParameterName::SentBy + | ICalendarParameterName::Schema => { + ICalendarParameterValue::Uri(Uri::parse(value.into_string().into_owned())) + } + ICalendarParameterName::Range => { + let value = value.into_string(); + if value.eq_ignore_ascii_case("THISANDFUTURE") { + ICalendarParameterValue::Bool(true) + } else { + ICalendarParameterValue::Text(value.into_owned()) + } + } + ICalendarParameterName::Gap => { + let value = value.into_string(); + ICalendarDuration::parse(value.as_bytes()) + .map(ICalendarParameterValue::Duration) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Cutype => { + let value = value.into_string(); + ICalendarUserTypes::parse(value.as_bytes()) + .map(ICalendarParameterValue::Cutype) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Fbtype => { + let value = value.into_string(); + ICalendarFreeBusyType::parse(value.as_bytes()) + .map(ICalendarParameterValue::Fbtype) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Partstat => { + let value = value.into_string(); + ICalendarParticipationStatus::parse(value.as_bytes()) + .map(ICalendarParameterValue::Partstat) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Related => { + let value = value.into_string(); + ICalendarRelated::parse(value.as_bytes()) + .map(ICalendarParameterValue::Related) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Reltype => { + let value = value.into_string(); + ICalendarRelationshipType::parse(value.as_bytes()) + .map(ICalendarParameterValue::Reltype) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Role => { + let value = value.into_string(); + ICalendarParticipationRole::parse(value.as_bytes()) + .map(ICalendarParameterValue::Role) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::ScheduleAgent => { + let value = value.into_string(); + ICalendarScheduleAgentValue::parse(value.as_bytes()) + .map(ICalendarParameterValue::ScheduleAgent) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::ScheduleForceSend => { + let value = value.into_string(); + ICalendarScheduleForceSendValue::parse(value.as_bytes()) + .map(ICalendarParameterValue::ScheduleForceSend) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Display => { + let value = value.into_string(); + ICalendarDisplayType::parse(value.as_bytes()) + .map(ICalendarParameterValue::Display) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Feature => { + let value = value.into_string(); + ICalendarFeatureType::parse(value.as_bytes()) + .map(ICalendarParameterValue::Feature) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + ICalendarParameterName::Linkrel => { + let value = value.into_string(); + LinkRelation::parse(value.as_bytes()) + .map(ICalendarParameterValue::Linkrel) + .unwrap_or_else(|| ICalendarParameterValue::Text(value.into_owned())) + } + _ => ICalendarParameterValue::Text(value.into_string().into_owned()), + }; + + self.params.push(ICalendarParameter { + name: param.clone(), + value, + }); + } + } + } +} + +impl ICalendarEntry { + /* + A DATE value has no time and no timezone: the TZID parameter MUST NOT be applied to + it (Section 3.2.19 of [RFC5545]). + */ + + pub(super) fn with_date(mut self, dt: ZonedDateTime, is_date: bool) -> Self { + if !is_date { + return self.with_date_time(dt); + } + + debug_assert!(self.values.is_empty()); + if !self.has_parameter(&ICalendarParameterName::Value) { + self.params + .push(ICalendarParameter::value(ICalendarValueType::Date)); + } + self.values + .push(PartialDateTime::from_date_timestamp(dt.naive_timestamp()).into()); + self + } + + pub(super) fn with_dates(mut self, dts: Vec, is_date: bool) -> Self { + if !is_date { + return self.with_date_times(dts); + } + + debug_assert!(self.values.is_empty()); + if !self.has_parameter(&ICalendarParameterName::Value) { + self.params + .push(ICalendarParameter::value(ICalendarValueType::Date)); + } + for dt in dts { + self.values + .push(PartialDateTime::from_date_timestamp(dt.naive_timestamp()).into()); + } + self + } + + pub(super) fn with_date_time(mut self, dt: ZonedDateTime) -> Self { + debug_assert!(self.values.is_empty()); + + self.remove_period_value_type(); + + // Best effort to restore the original timezone + let tz_id = self.tz_id(); + let has_tz_id = tz_id.is_some(); + self.insert_date( + dt, + has_tz_id, + !has_tz_id, + tz_id.and_then(|id| Tz::from_str(id).ok()), + ); + self + } + + pub(super) fn with_period(mut self, dt: ZonedDateTime, duration: ICalendarDuration) -> Self { + self = self.with_date_time(dt); + + if let Some(ICalendarValue::PartialDateTime(start)) = self.values.pop() { + self.values.push(ICalendarValue::Period(Box::new( + ICalendarPeriod::Duration { start, duration }, + ))); + } + + if !self.has_parameter(&ICalendarParameterName::Value) { + self.params + .push(ICalendarParameter::value(ICalendarValueType::Period)); + } + + self + } + + pub(super) fn with_date_times(mut self, dts: Vec) -> Self { + debug_assert!(self.values.is_empty()); + + self.remove_period_value_type(); + let tz_id = self.tz_id(); + let has_tz_id = tz_id.is_some(); + let entry_tz = tz_id + .and_then(|id| Tz::from_str(id).ok()) + .or_else(|| dts.first().map(|dt| dt.timezone())); + + for (pos, dt) in dts.into_iter().enumerate() { + self.insert_date(dt, has_tz_id, pos == 0, entry_tz); + } + + self + } + + fn remove_period_value_type(&mut self) { + self.params.retain(|param| { + !matches!( + (¶m.name, ¶m.value), + ( + ICalendarParameterName::Value, + ICalendarParameterValue::Value(ICalendarValueType::Period) + ) + ) + }); + } + + fn insert_date( + &mut self, + mut dt: ZonedDateTime, + has_tz_id: bool, + add_tz_id: bool, + entry_tz: Option, + ) { + if let Some(tz) = entry_tz + && tz != dt.timezone() + { + dt = dt.with_timezone(tz); + } + + let tz = dt.timezone(); + if has_tz_id { + self.values + .push(PartialDateTime::from_naive_timestamp(dt.naive_timestamp()).into()); + } else if tz.is_utc() { + self.values + .push(PartialDateTime::from_utc_timestamp(dt.timestamp()).into()); + } else { + if add_tz_id && let Some(tz_name) = tz.name() { + self.params + .push(ICalendarParameter::tzid(tz_name.into_owned())); + } + self.values + .push(PartialDateTime::from_naive_timestamp(dt.naive_timestamp()).into()); + } + } +} + +impl JSCalendarDateTime { + pub fn to_utc_date_time(&self) -> Option { + Timestamp::from_second(self.timestamp) + .ok() + .map(|timestamp| Tz::UTC.from_timestamp(timestamp)) + } + + pub fn to_naive_date_time(&self) -> Option { + Timestamp::from_second(self.timestamp) + .ok() + .map(|timestamp| Offset::UTC.to_datetime(timestamp)) + } +} + +impl From for PartialDateTime { + fn from(dt: JSCalendarDateTime) -> Self { + if !dt.is_local { + Self::from_utc_timestamp(dt.timestamp) + } else { + Self::from_naive_timestamp(dt.timestamp) + } + } +} diff --git a/crates/calcard/src/jscalendar/export/props.rs b/crates/calcard/src/jscalendar/export/props.rs new file mode 100644 index 00000000..0db9bdfd --- /dev/null +++ b/crates/calcard/src/jscalendar/export/props.rs @@ -0,0 +1,403 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Parser, + common::{ + CalendarScale, IanaParse, IanaType, PartialDateTime, jsprop::text::ConvertedKeys, + parser::Integer, + }, + icalendar::*, + jscalendar::{JSCalendarId, JSCalendarProperty, JSCalendarValue, export::ConvertedComponent}, +}; +use jmap_tools::{Key, Map, Value}; +use smallvec::{SmallVec, smallvec}; + +impl ConvertedComponent<'_, I, B> { + pub(super) fn apply_conversions( + self, + mut component: ICalendarComponent, + ical: &mut ICalendar, + ) -> ICalendarComponent { + if !self.properties.is_empty() { + component.import_properties(self.properties); + } + if !self.components.is_empty() { + let mut jcal_components = self.components.into_iter(); + let mut stack = Vec::new(); + + loop { + if let Some(jcal_component) = jcal_components.next() { + let Some(items) = jcal_component.into_array() else { + continue; + }; + let mut items = items.into_iter(); + + if let ( + Some(Value::Str(name)), + Some(Value::Array(properties)), + Some(Value::Array(child_components)), + ) = (items.next(), items.next(), items.next()) + { + // Process the component + let mut sub_component = ICalendarComponent::new( + ICalendarComponentType::parse(name.as_bytes()).unwrap_or_else(|| { + ICalendarComponentType::Other(name.to_ascii_uppercase()) + }), + ); + sub_component.import_properties(properties); + if !child_components.is_empty() { + stack.push((jcal_components, component)); + jcal_components = child_components.into_iter(); + component = sub_component; + } else { + component + .component_ids + .push(ical.push_component(sub_component)); + } + } + } else if let Some((next_components, mut parent_component)) = stack.pop() { + parent_component + .component_ids + .push(ical.push_component(component)); + jcal_components = next_components; + component = parent_component; + } else { + break; + } + } + } + + component + } + + pub(super) fn retain_class_property(&mut self, keep_unknown: bool) -> bool { + let mut has_class = false; + self.properties.retain(|property| { + let Some([Value::Str(name), _, _, value, ..]) = property.as_array() else { + return true; + }; + if ICalendarProperty::parse(name.as_bytes()) != Some(ICalendarProperty::Class) { + return true; + } + let keep = keep_unknown + && !has_class + && matches!(value, Value::Str(value) + if ICalendarClassification::parse(value.as_bytes()).is_none()); + has_class |= keep; + keep + }); + has_class + } +} + +impl ICalendarComponent { + pub(super) fn import_properties( + &mut self, + props: Vec, JSCalendarValue>>, + ) { + for prop in props.into_iter().flat_map(|prop| prop.into_array()) { + let mut prop = prop.into_iter(); + let Some(name) = prop + .next() + .and_then(|v| v.into_string()) + .map(|name| { + ICalendarProperty::parse(name.as_bytes()) + .unwrap_or(ICalendarProperty::Other(name.to_ascii_uppercase())) + }) + .filter(|name| !matches!(name, ICalendarProperty::Begin | ICalendarProperty::End)) + else { + continue; + }; + let Some(params) = prop.next() else { + continue; + }; + let Some(value_type) = + prop.next() + .and_then(|v| v.into_string()) + .map(|v| match ICalendarValueType::parse(v.as_bytes()) { + Some(v) => IanaType::Iana(v), + None => IanaType::Other(v.to_ascii_uppercase()), + }) + else { + continue; + }; + + let (default_type, _) = name.default_types(); + let convert_type = value_type + .iana() + .filter(|&v| v != &ICalendarValueType::Unknown) + .map(|v| ValueType::Ical(*v)) + .unwrap_or(default_type); + let Some(values) = prop.next().and_then(|v| match v { + Value::Array(arr) => Some( + arr.into_iter() + .filter_map(|v| convert_value(v, &convert_type).ok()) + .collect::>(), + ), + v => convert_value(v, &convert_type).ok().map(|v| smallvec![v]), + }) else { + continue; + }; + + let mut entry = ICalendarEntry::new(name); + entry.import_jcal_params(params); + entry.values = values; + if convert_type != default_type { + entry.params.push(ICalendarParameter::value(value_type)); + } + self.entries.push(entry); + } + } +} + +pub(super) fn convert_value<'x, I: JSCalendarId, B: JSCalendarId>( + value: Value<'x, JSCalendarProperty, JSCalendarValue>, + value_type: &'_ ValueType, +) -> Result, JSCalendarValue>> { + match value { + Value::Element(e) => match e { + JSCalendarValue::CalendarScale(v) => Ok(ICalendarValue::CalendarScale(v)), + JSCalendarValue::DateTime(v) => Ok(ICalendarValue::PartialDateTime(v.into())), + JSCalendarValue::Duration(v) => Ok(ICalendarValue::Duration(v)), + JSCalendarValue::Method(v) => Ok(ICalendarValue::Method(v)), + JSCalendarValue::AlertAction(_) + | JSCalendarValue::FreeBusyStatus(_) + | JSCalendarValue::ParticipantKind(_) + | JSCalendarValue::ParticipationStatus(_) + | JSCalendarValue::Privacy(_) + | JSCalendarValue::Progress(_) + | JSCalendarValue::RelativeTo(_) + | JSCalendarValue::ScheduleAgent(_) + | JSCalendarValue::EventStatus(_) + | JSCalendarValue::Frequency(_) + | JSCalendarValue::Skip(_) + | JSCalendarValue::Weekday(_) + | JSCalendarValue::Month(_) + | JSCalendarValue::LinkRelation(_) + | JSCalendarValue::Type(_) + | JSCalendarValue::Id(_) + | JSCalendarValue::BlobId(_) + | JSCalendarValue::IdReference(_) => Err(Value::Element(e)), + }, + Value::Str(s) => { + match value_type { + ValueType::CalendarScale => { + if let Some(value) = CalendarScale::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::CalendarScale(value)); + } + } + ValueType::Method => { + if let Some(value) = ICalendarMethod::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Method(value)); + } + } + ValueType::Classification => { + if let Some(value) = ICalendarClassification::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Classification(value)); + } + } + ValueType::Status => { + if let Some(value) = ICalendarStatus::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Status(value)); + } + } + ValueType::Transparency => { + if let Some(value) = ICalendarTransparency::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Transparency(value)); + } + } + ValueType::Action => { + if let Some(value) = ICalendarAction::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Action(value)); + } + } + ValueType::BusyType => { + if let Some(value) = ICalendarFreeBusyType::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::BusyType(value)); + } + } + ValueType::ParticipantType => { + if let Some(value) = ICalendarParticipantType::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::ParticipantType(value)); + } + } + ValueType::ResourceType => { + if let Some(value) = ICalendarResourceType::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::ResourceType(value)); + } + } + ValueType::Proximity => { + if let Some(value) = ICalendarProximityValue::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Proximity(value)); + } + } + ValueType::Ical(typ) => match typ { + ICalendarValueType::Uri | ICalendarValueType::CalAddress => { + return Ok(ICalendarValue::Uri(Uri::parse(s))); + } + ICalendarValueType::Date => { + let mut dt = PartialDateTime::default(); + if dt.parse_ical_date(&mut s.as_ref().as_bytes().iter().peekable()) { + return Ok(ICalendarValue::PartialDateTime(dt)); + } + } + ICalendarValueType::Time => { + let mut dt = PartialDateTime::default(); + if dt.parse_ical_time(&mut s.as_ref().as_bytes().iter().peekable()) { + return Ok(ICalendarValue::PartialDateTime(dt)); + } + } + ICalendarValueType::DateTime => { + let mut dt = PartialDateTime::default(); + if dt.parse_timestamp(&mut s.as_ref().as_bytes().iter().peekable(), false) { + return Ok(ICalendarValue::PartialDateTime(dt)); + } + } + ICalendarValueType::UtcOffset => { + let mut dt = PartialDateTime::default(); + if dt.parse_zone(&mut s.as_ref().as_bytes().iter().peekable()) { + return Ok(ICalendarValue::PartialDateTime(dt)); + } + } + ICalendarValueType::Duration => { + if let Some(duration) = ICalendarDuration::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Duration(duration)); + } + } + ICalendarValueType::Float => { + if let Ok(float) = s.as_ref().parse::() + && float.is_finite() + { + return Ok(ICalendarValue::Float(float)); + } + } + ICalendarValueType::Integer => { + if let Some(integer) = Integer::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Integer(integer.0)); + } + } + ICalendarValueType::Period => { + if let Some(period) = ICalendarPeriod::parse(s.as_ref().as_bytes()) { + return Ok(ICalendarValue::Period(Box::new(period))); + } + } + ICalendarValueType::Recur => { + let mut parser = Parser::new(s.as_ref()); + if let Ok(recur) = parser.rrule() { + return Ok(ICalendarValue::RecurrenceRule(Box::new(recur))); + } + } + ICalendarValueType::Boolean => { + if s.eq_ignore_ascii_case("true") { + return Ok(ICalendarValue::Boolean(true)); + } else if s.eq_ignore_ascii_case("false") { + return Ok(ICalendarValue::Boolean(false)); + } + } + ICalendarValueType::Binary => { + return Ok(match Uri::parse(s) { + Uri::Data(data) => ICalendarValue::Binary(data.data), + Uri::Location(text) => ICalendarValue::Text(text), + }); + } + ICalendarValueType::Text + | ICalendarValueType::Unknown + | ICalendarValueType::XmlReference + | ICalendarValueType::Uid => (), + }, + } + + Ok(ICalendarValue::Text(s.into_owned())) + } + Value::Bool(b) => Ok(ICalendarValue::Boolean(b)), + Value::Number(n) => { + if let Some(integer) = n.as_i64() { + Ok(ICalendarValue::Integer(integer)) + } else if let Some(integer) = n.as_u64() { + Ok(ICalendarValue::Text(integer.to_string())) + } else { + n.as_f64() + .filter(|float| float.is_finite()) + .map(ICalendarValue::Float) + .ok_or(Value::Number(n)) + } + } + value => Err(value), + } +} + +impl<'x, I: JSCalendarId, B: JSCalendarId> ConvertedComponent<'x, I, B> { + #[allow(clippy::type_complexity)] + pub(super) fn try_from_object( + obj: Vec<( + Key<'x, JSCalendarProperty>, + Value<'x, JSCalendarProperty, JSCalendarValue>, + )>, + ) -> Option { + let mut converted = ConvertedComponent { + name: ICalendarComponentType::Other(String::new()), + converted_props: Vec::new(), + converted_props_count: 0, + properties: Vec::new(), + components: Vec::new(), + }; + let mut has_name = false; + for (sub_property, value) in obj { + match (sub_property, value) { + (Key::Property(JSCalendarProperty::ConvertedProperties), Value::Object(obj)) => { + let obj = obj.into_vec(); + converted.converted_props.reserve(obj.len()); + for (key, value) in obj { + converted + .converted_props + .push((key.into_converted_keys(), value)); + } + } + (Key::Property(JSCalendarProperty::Properties), Value::Array(array)) => { + converted.properties = array; + } + (Key::Property(JSCalendarProperty::Components), Value::Array(array)) => { + converted.components = array; + } + (Key::Property(JSCalendarProperty::Name), Value::Str(text)) => { + converted.name = + ICalendarComponentType::parse(text.as_bytes()).unwrap_or_else(|| { + ICalendarComponentType::Other(text.to_ascii_uppercase()) + }); + has_name = true; + } + _ => {} + } + } + + if !converted.converted_props.is_empty() { + converted + .converted_props + .sort_unstable_by(|a, b| a.0.cmp(&b.0)); + } + + (!converted.properties.is_empty() + || !converted.components.is_empty() + || has_name + || !converted.converted_props.is_empty()) + .then_some(converted) + } + + pub(super) fn build( + entries: &mut Map<'x, JSCalendarProperty, JSCalendarValue>, + ) -> Option { + for (property, value) in entries.as_mut_vec() { + if let (Key::Property(JSCalendarProperty::ICalendar), Value::Object(obj)) = + (property, value) + { + return Self::try_from_object(std::mem::take(obj.as_mut_vec())); + } + } + + None + } +} diff --git a/crates/calcard/src/jscalendar/ext.rs b/crates/calcard/src/jscalendar/ext.rs new file mode 100644 index 00000000..99e10e8e --- /dev/null +++ b/crates/calcard/src/jscalendar/ext.rs @@ -0,0 +1,317 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::jscalendar::{JSCalendarId, JSCalendarProperty, JSCalendarType, JSCalendarValue, Uuid5}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; +use std::{ + mem::{self, discriminant}, + str::FromStr, +}; +use uuid::fmt::Hyphenated; + +pub(crate) trait JSCalendarKeyExt { + fn is_forbidden_override_key(&self) -> bool; + fn is_ignored_override_key(&self) -> bool; + fn is_series_converted_property(&self) -> bool; + fn is_uuid5_of(&self, value: &[u8]) -> bool; + fn matches_pointer_item(&self, item: &JsonPointerItem>) -> bool; + fn pointer_index(&self) -> Option; + fn same_key(&self, other: &Key<'_, JSCalendarProperty>) -> bool; + fn same_data_key(&self, other: &Key<'_, JSCalendarProperty>) -> bool; +} + +pub(crate) trait JSCalendarMapExt { + fn is_type_or_untyped(&self, types: &[JSCalendarType]) -> bool; +} + +pub(crate) trait JSCalendarObjectExt<'x, I: JSCalendarId, B: JSCalendarId> { + fn key_position(&self, key: &Key<'_, JSCalendarProperty>) -> Option; + fn lookup( + &self, + key: &Key<'_, JSCalendarProperty>, + ) -> Option<&Value<'x, JSCalendarProperty, JSCalendarValue>>; + fn lookup_mut( + &mut self, + key: &Key<'_, JSCalendarProperty>, + ) -> Option<&mut Value<'x, JSCalendarProperty, JSCalendarValue>>; + fn upsert( + &mut self, + key: Key<'x, JSCalendarProperty>, + value: Value<'x, JSCalendarProperty, JSCalendarValue>, + ) -> Option, JSCalendarValue>>; + fn upsert_or_get_mut( + &mut self, + key: Key<'x, JSCalendarProperty>, + value: impl FnOnce() -> Value<'x, JSCalendarProperty, JSCalendarValue>, + ) -> &mut Value<'x, JSCalendarProperty, JSCalendarValue>; + fn contains_true(&self, key: &Key<'_, JSCalendarProperty>) -> bool; +} + +pub(crate) trait JSCalendarValueExt: Sized { + fn apply_jsprop(&mut self, pointer: &[JsonPointerItem>], patch: Self); +} + +pub trait JSCalendarPatch { + fn is_instance_patch(&self) -> bool; +} + +impl JSCalendarKeyExt for Key<'_, JSCalendarProperty> { + fn is_forbidden_override_key(&self) -> bool { + match self { + Key::Property(JSCalendarProperty::Pointer(pointer)) => { + JSCalendarProperty::is_forbidden_override_pointer(pointer) + } + Key::Property(property) => property.is_forbidden_override_patch(), + _ => false, + } + } + + fn is_ignored_override_key(&self) -> bool { + matches!( + self, + Key::Property(property) if property.is_forbidden_override_patch() + || matches!(property, JSCalendarProperty::Excluded) + ) + } + + fn is_series_converted_property(&self) -> bool { + let first = match self { + Key::Property(JSCalendarProperty::Pointer(ptr)) => { + return matches!( + ptr.iter() + .find(|item| !matches!(item, JsonPointerItem::Root)), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::RecurrenceRule + ))) + ); + } + Key::Property(property) => { + return matches!( + property, + JSCalendarProperty::RecurrenceOverrides | JSCalendarProperty::RecurrenceRule + ); + } + Key::Borrowed(key) => key.split('/').next(), + Key::Owned(key) => key.split('/').next(), + }; + + matches!( + first.and_then(|name| JSCalendarProperty::::from_str(name).ok()), + Some(JSCalendarProperty::RecurrenceOverrides | JSCalendarProperty::RecurrenceRule) + ) + } + + fn is_uuid5_of(&self, value: &[u8]) -> bool { + let name = self.to_string(); + name.len() == Hyphenated::LENGTH && { + let mut buffer = [0u8; Hyphenated::LENGTH]; + value.uuid5_hyphenated(&mut buffer) == name.as_ref() + } + } + + fn matches_pointer_item(&self, item: &JsonPointerItem>) -> bool { + match item { + JsonPointerItem::Key(item) => item.same_key(self), + JsonPointerItem::Number(number) => self.pointer_index() == Some(*number), + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => { + false + } + } + } + + fn pointer_index(&self) -> Option { + let name = self.as_string_key()?; + (!name.is_empty() + && name.bytes().all(|byte| byte.is_ascii_digit()) + && (name == "0" || !name.starts_with('0'))) + .then(|| name.parse().ok()) + .flatten() + } + + #[inline(always)] + fn same_key(&self, other: &Key<'_, JSCalendarProperty>) -> bool { + match (self, other) { + (Key::Property(a), Key::Property(b)) if !a.has_data() && !b.has_data() => { + discriminant(a) == discriminant(b) + } + _ => self.same_data_key(other), + } + } + + #[inline(never)] + fn same_data_key(&self, other: &Key<'_, JSCalendarProperty>) -> bool { + self == other + } +} + +impl<'x, I: JSCalendarId, B: JSCalendarId> JSCalendarObjectExt<'x, I, B> + for Map<'x, JSCalendarProperty, JSCalendarValue> +{ + #[inline] + fn key_position(&self, key: &Key<'_, JSCalendarProperty>) -> Option { + self.as_vec().iter().position(|(k, _)| k.same_key(key)) + } + + #[inline] + fn lookup( + &self, + key: &Key<'_, JSCalendarProperty>, + ) -> Option<&Value<'x, JSCalendarProperty, JSCalendarValue>> { + self.as_vec() + .iter() + .find_map(|(k, v)| k.same_key(key).then_some(v)) + } + + #[inline] + fn lookup_mut( + &mut self, + key: &Key<'_, JSCalendarProperty>, + ) -> Option<&mut Value<'x, JSCalendarProperty, JSCalendarValue>> { + self.as_mut_vec() + .iter_mut() + .find_map(|(k, v)| k.same_key(key).then_some(v)) + } + + #[inline] + fn upsert( + &mut self, + key: Key<'x, JSCalendarProperty>, + value: Value<'x, JSCalendarProperty, JSCalendarValue>, + ) -> Option, JSCalendarValue>> { + match self.lookup_mut(&key) { + Some(current) => Some(mem::replace(current, value)), + None => { + self.insert_unchecked(key, value); + None + } + } + } + + #[inline] + fn upsert_or_get_mut( + &mut self, + key: Key<'x, JSCalendarProperty>, + value: impl FnOnce() -> Value<'x, JSCalendarProperty, JSCalendarValue>, + ) -> &mut Value<'x, JSCalendarProperty, JSCalendarValue> { + let position = match self.key_position(&key) { + Some(position) => position, + None => { + self.insert_unchecked(key, value()); + self.len() - 1 + } + }; + &mut self.as_mut_vec()[position].1 + } + + #[inline] + fn contains_true(&self, key: &Key<'_, JSCalendarProperty>) -> bool { + self.as_vec() + .iter() + .any(|(k, v)| matches!(v, Value::Bool(true)) && k.same_key(key)) + } +} + +impl JSCalendarMapExt + for Map<'_, JSCalendarProperty, JSCalendarValue> +{ + fn is_type_or_untyped(&self, types: &[JSCalendarType]) -> bool { + match self.lookup(&Key::Property(JSCalendarProperty::Type)) { + None => true, + Some(Value::Element(JSCalendarValue::Type(typ))) => types.contains(typ), + Some(_) => false, + } + } +} + +impl JSCalendarPatch + for Value<'_, JSCalendarProperty, JSCalendarValue> +{ + fn is_instance_patch(&self) -> bool { + self.as_object().is_some_and(|patch| { + let excluded = Key::Property(JSCalendarProperty::Excluded); + !patch.contains_true(&excluded) + && patch + .keys() + .any(|key| !key.same_key(&excluded) && !key.is_forbidden_override_key()) + }) + } +} + +impl JSCalendarValueExt + for Value<'_, JSCalendarProperty, JSCalendarValue> +{ + fn apply_jsprop(&mut self, pointer: &[JsonPointerItem>], patch: Self) { + let (Some((last, parents)), false) = (pointer.split_last(), patch.is_null()) else { + return; + }; + let mut current = self; + for item in parents { + let Value::Object(obj) = current else { + return; + }; + let Some(next) = obj + .iter_mut() + .find_map(|(key, value)| key.matches_pointer_item(item).then_some(value)) + else { + return; + }; + current = next; + } + let Value::Object(obj) = current else { + return; + }; + if obj.iter().any(|(key, _)| key.matches_pointer_item(last)) { + return; + } + let key = match last { + JsonPointerItem::Key(key) => key.clone(), + JsonPointerItem::Number(number) => Key::Owned(number.to_string()), + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => { + return; + } + }; + obj.insert_unchecked(key, patch); + } +} + +impl JSCalendarProperty { + pub(crate) fn is_series_property(&self) -> bool { + matches!( + self, + JSCalendarProperty::RecurrenceRule + | JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::RecurrenceId + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::Excluded + ) + } + + pub(crate) fn is_member_map(&self) -> bool { + matches!( + self, + JSCalendarProperty::Participants + | JSCalendarProperty::Locations + | JSCalendarProperty::VirtualLocations + | JSCalendarProperty::Alerts + | JSCalendarProperty::Links + | JSCalendarProperty::RelatedTo + ) + } + + pub(crate) fn member_pointer( + &self, + keys: [Key<'static, JSCalendarProperty>; N], + ) -> Key<'static, JSCalendarProperty> { + let mut items = Vec::with_capacity(N + 1); + items.push(JsonPointerItem::Key(Key::Property(self.clone()))); + items.extend(keys.into_iter().map(|key| match key.pointer_index() { + Some(index) => JsonPointerItem::Number(index), + None => JsonPointerItem::Key(key), + })); + Key::Property(JSCalendarProperty::Pointer(JsonPointer::new(items))) + } +} diff --git a/crates/calcard/src/jscalendar/import/convert.rs b/crates/calcard/src/jscalendar/import/convert.rs new file mode 100644 index 00000000..62190fe4 --- /dev/null +++ b/crates/calcard/src/jscalendar/import/convert.rs @@ -0,0 +1,2312 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + blob::{BlobIdGenerator, NoBlobIds}, + export::ImportError, + jsprop::{ + JSPropPointer, + text::{IntoAsciiLowercase, PointerString}, + }, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::{timezone::TzResolver, *}, + jscalendar::{ + ext::JSCalendarObjectExt, + import::{ + EntryState, ImportContext, ImportOptions, State, params::ExtractParams, + props::ICalendarBinary, + }, + *, + }, +}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; +use smallvec::{IntoIter, SmallVec}; +use std::mem; + +impl ICalendar { + pub fn into_jscalendar(self) -> JSCalendar<'static, I, B> { + self.convert_jscalendar::(ImportOptions::default()) + .0 + } + + pub fn into_jscalendar_with>( + self, + options: ImportOptions, + ) -> Result, ImportError> { + match self.convert_jscalendar(options) { + (js_calendar, false) => Ok(js_calendar), + (_, true) => Err(ImportError::BlobIdFailed), + } + } + + fn convert_jscalendar>( + mut self, + mut options: ImportOptions, + ) -> (JSCalendar<'static, I, B>, bool) { + let tz_resolver = self.build_owned_tz_resolver(); + let mut context = options.context(); + if let Some(blob_ids) = &mut context.blob_ids { + blob_ids.expect_sizes(self.binary_link_sizes()); + } + let js_calendar = JSCalendar( + self.to_jscalendar(&tz_resolver, 0, &mut context) + .into_root_object(), + ); + let has_failed = context + .blob_ids + .as_ref() + .is_some_and(|blob_ids| blob_ids.has_failed()); + + (js_calendar, has_failed) + } +} + +impl ICalendar { + #[allow(clippy::wrong_self_convention)] + pub(super) fn to_jscalendar( + &mut self, + tz_resolver: &TzResolver, + component_id: u32, + options: &mut ImportContext<'_, B>, + ) -> State { + let mut state = State { + include_ical_components: options.include_ical_components, + ..Default::default() + }; + + // Take component + let Some(component) = self.components.get_mut(component_id as usize) else { + return state; + }; + let mut entries = std::mem::take(&mut component.entries); + state.component_type = component.component_type.clone(); + + // Process subcomponents + let mut group_components = Vec::new(); + let mut alarm_component_ids = Vec::new(); + let mut alarm_with_related = false; + let mut unsupported_component_ids = Vec::new(); + let mut uid_jsid_mappings = Vec::new(); + let mut has_locations = false; + let mut task_anchors = None; + let component_ids = std::mem::take(&mut component.component_ids); + if state.component_type == ICalendarComponentType::VCalendar { + options.task_series = self.task_series(&component_ids); + } + + for component_id in component_ids { + let Some(component) = self.components.get_mut(component_id as usize) else { + continue; + }; + match (&state.component_type, &component.component_type) { + ( + ICalendarComponentType::VCalendar, + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + group_components.push(self.to_jscalendar(tz_resolver, component_id, options)); + } + ( + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + component_type @ (ICalendarComponentType::Participant + | ICalendarComponentType::VLocation), + ) => { + let entries = state.get_mut_object_or_insert(match component_type { + ICalendarComponentType::Participant => JSCalendarProperty::Participants, + ICalendarComponentType::VLocation => { + has_locations = true; + JSCalendarProperty::Locations + } + _ => unreachable!(), + }); + let mut subcomponent_state = + self.to_jscalendar(tz_resolver, component_id, options); + let jsid = subcomponent_state.jsid.take(); + let subcomponent = subcomponent_state.into_object(); + entries.insert_named(jsid, subcomponent); + } + ( + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ICalendarComponentType::VAlarm, + ) => { + if state.component_type == ICalendarComponentType::VTodo + && !task_anchors + .get_or_insert_with(|| TaskAnchors::new(&entries, &options.task_series)) + .cover(component) + { + unsupported_component_ids.push(component_id); + continue; + } + let mut jsid = None; + + for entry in &mut component.entries { + match &entry.name { + ICalendarProperty::RelatedTo => { + alarm_with_related = true; + } + ICalendarProperty::Jsid => { + jsid = std::mem::take(&mut entry.values) + .pop() + .and_then(|v| v.into_text()); + } + _ => {} + } + } + let jsid = jsid.unwrap_or_else(|| { + Cow::Owned(format!("k{}", alarm_component_ids.len() + 1)) + }); + if let Some(uid) = component.uid() { + uid_jsid_mappings.push((uid.to_string(), jsid.clone())); + } + alarm_component_ids.push((component_id, jsid)); + } + _ => { + unsupported_component_ids.push(component_id); + } + } + } + + // Process alarms + if !alarm_component_ids.is_empty() { + for (component_id, jsid) in alarm_component_ids { + if alarm_with_related + && let Some(alarm) = self.components.get_mut(component_id as usize) + { + for entry in &mut alarm.entries { + if matches!(entry.name, ICalendarProperty::RelatedTo) { + for value in &mut entry.values { + if let Some((_, jsid)) = value.as_text().and_then(|uid| { + uid_jsid_mappings.iter().find(|(u, _)| u == uid) + }) { + *value = ICalendarValue::Text(jsid.to_string()); + } + } + } + } + } + + state + .get_mut_object_or_insert(JSCalendarProperty::Alerts) + .insert_unchecked( + Key::from(jsid), + self.to_jscalendar(tz_resolver, component_id, options) + .into_object(), + ); + } + } + + // Build group component list + let mut group_objects = Vec::with_capacity(group_components.len()); + if !group_components.is_empty() { + // Order by UID placing the main components first + group_components.sort_by(|a, b| match a.uid.cmp(&b.uid) { + std::cmp::Ordering::Equal => a + .recurrence_id + .is_some() + .cmp(&b.recurrence_id.is_some()) + .then_with(|| a.recurrence_sequence().cmp(&b.recurrence_sequence())), + other => other, + }); + let mut group_components = group_components.into_iter().peekable(); + + // Bundle recurrence overrides together by UID + while let Some(mut component) = group_components.next() { + let mut instances = Vec::new(); + + if component.uid.is_some() && component.recurrence_id.is_none() { + let mut privacy = component.privacy(); + + while let Some(mut recurrence) = group_components + .next_if(|r| r.uid == component.uid && r.recurrence_id.is_some()) + { + let Some(recurrence_id) = recurrence.recurrence_id.take() else { + continue; + }; + let recurrence_id = + JSCalendarDateTime::local_in(recurrence_id, component.tz_start); + let _ = recurrence.uid.take(); + + privacy = privacy.max(recurrence.privacy()); + recurrence.inherit_time_zone(component.tz_start); + recurrence.set_is_recurrence_instance(); + recurrence.remove_forbidden_override_patches(); + component + .recurrence_overrides + .push((recurrence_id, recurrence)); + } + + if let Some(privacy) = privacy { + component.raise_privacy(privacy); + } + } else if component.uid.is_some() { + while let Some(instance) = + group_components.next_if(|next| next.uid == component.uid) + { + instances.push(instance); + } + + if let Some(privacy) = instances + .iter() + .filter_map(State::privacy) + .max() + .max(component.privacy()) + { + component.raise_privacy(privacy); + for instance in &mut instances { + instance.raise_privacy(privacy); + } + } + } + + component.start_at_recurrence_id(); + if options.return_first { + return component; + } + group_objects.push(component.into_object()); + group_objects.extend(instances.into_iter().map(|mut instance| { + instance.start_at_recurrence_id(); + instance.into_object() + })); + } + } + + // Build unsupported component jcal + if options.include_ical_components && !unsupported_component_ids.is_empty() { + let mut component_iter = unsupported_component_ids.into_iter(); + let mut component_stack = Vec::with_capacity(4); + let mut components = vec![]; + + loop { + if let Some(component_id) = component_iter.next() { + let Some(component) = self.components.get_mut(component_id as usize) else { + continue; + }; + + let mut entries = Vec::with_capacity(component.entries.len()); + for entry in std::mem::take(&mut component.entries) { + entries.push(EntryState::new(entry, false).into_jcal()); + } + + components.push(Value::Array(vec![ + Value::Str( + mem::take(&mut component.component_type) + .into_string() + .into_ascii_lowercase() + .into(), + ), + Value::Array(entries), + Value::Array(vec![]), + ])); + + if !component.component_ids.is_empty() + && component_stack.len() + 1 < MAX_ICAL_COMPONENT_DEPTH + { + component_stack.push((components, component_iter)); + component_iter = std::mem::take(&mut component.component_ids).into_iter(); + components = vec![]; + } + } else if let Some((mut parent_components, iter)) = component_stack.pop() { + if let Some(parent_component) = parent_components + .last_mut() + .and_then(|v| v.as_array_mut()) + .and_then(|v| v.last_mut()) + .and_then(|v| v.as_array_mut()) + { + if !parent_component.is_empty() { + parent_component.extend(components); + } else { + *parent_component = components; + } + } else { + debug_assert!(false, "Invalid component stack state"); + } + components = parent_components; + component_iter = iter; + } else { + break; + } + } + + state.ical_components = Some(Value::Array(components)); + } + + let mut main_location_id = None; + + entries.sort_by_key(|entry| match &entry.name { + ICalendarProperty::Dtstart | ICalendarProperty::Jsid => 0, + ICalendarProperty::RecurrenceId => 1, + ICalendarProperty::Dtend + | ICalendarProperty::Due + | ICalendarProperty::Location + | ICalendarProperty::Name + | ICalendarProperty::Attendee => 2, + _ => 3, + }); + let mut start_date = None; + let mut start_is_date = false; + let mut has_owner = false; + let is_todo = state.component_type == ICalendarComponentType::VTodo; + + for entry in entries { + let mut entry = EntryState::new(entry, state.include_ical_components); + state.has_end |= match (&entry.entry.name, entry.entry.values.first()) { + (ICalendarProperty::Dtend, Some(ICalendarValue::PartialDateTime(value))) => { + value.has_date() + } + (ICalendarProperty::Due, Some(ICalendarValue::PartialDateTime(value))) => { + is_todo && value.has_date() + } + (ICalendarProperty::Duration, Some(ICalendarValue::Duration(_))) => true, + _ => false, + }; + let mut values = std::mem::take(&mut entry.entry.values).into_iter(); + let mut value = values.next(); + let is_link = matches!( + entry.entry.name, + ICalendarProperty::Attach | ICalendarProperty::Image + ) && state.component_type.converts_links(); + let link_value_type = match &value { + Some(ICalendarValue::Binary(_)) => ICalendarValueType::Binary, + _ => ICalendarValueType::Uri, + }; + let mut link_id = (is_link && entry.entry.jsid().is_none()) + .then(|| { + value + .as_ref() + .and_then(ICalendarValue::binary_bytes) + .map(uuid5) + }) + .flatten(); + + if let Some(blob_ids) = options.blob_ids.as_mut() + && is_link + && let Some(ICalendarBinary { + data, + content_type: data_content_type, + is_data_uri, + }) = value + .take_if(|value| value.is_binary()) + .and_then(ICalendarValue::into_binary) + { + let content_type = entry + .entry + .parameter(&ICalendarParameterName::Fmttype) + .and_then(|param| param.as_text()) + .or(data_content_type.as_deref()); + match blob_ids.blob_id(data, content_type) { + Ok(generated) => { + state.map_blob_link( + &mut entry, + generated, + data_content_type, + link_id.take(), + link_value_type, + ); + state.add_conversion_props(entry); + continue; + } + Err(data) => { + value = Some( + ICalendarBinary { + data, + content_type: data_content_type, + is_data_uri, + } + .into_value(), + ); + } + } + } + + let media_type = if is_link && link_value_type == ICalendarValueType::Binary { + if !entry.entry.has_parameter(&ICalendarParameterName::Value) { + entry + .entry + .add_param(ICalendarParameter::value(ICalendarValueType::Binary)); + } + entry + .entry + .parameter(&ICalendarParameterName::Fmttype) + .and_then(|param| param.as_text()) + .map(str::to_string) + } else { + None + }; + + match ( + &entry.entry.name, + value.map(|v| v.uri_to_string(media_type)), + &state.component_type, + ) { + ( + ICalendarProperty::Acknowledged, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VAlarm, + ) if value.has_date_and_time() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Acknowledged), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + value.to_timestamp().unwrap(), + false, + ))), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Acknowledged); + } + ( + ICalendarProperty::Action, + Some(ICalendarValue::Action( + value @ (ICalendarAction::Display | ICalendarAction::Email), + )), + ICalendarComponentType::VAlarm, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Action), + Value::Element(JSCalendarValue::AlertAction(match value { + ICalendarAction::Display => JSCalendarAlertAction::Display, + ICalendarAction::Email => JSCalendarAlertAction::Email, + _ => unreachable!(), + })), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Action); + } + ( + ICalendarProperty::Attach, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VCalendar, + ) => { + entry.set_map_name(); + state.map_named_entry_with_id( + &mut entry, + if link_value_type == ICalendarValueType::Binary { + &[ + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Size, + ICalendarParameterName::Linkrel, + ICalendarParameterName::Jsid, + ] + } else { + &[ + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Size, + ICalendarParameterName::Jsid, + ] + }, + JSCalendarProperty::Links, + [ + ( + Key::Property(JSCalendarProperty::Href), + Value::Str(uri.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Link)), + ), + ( + Key::Property(JSCalendarProperty::Rel), + Value::Element(JSCalendarValue::LinkRelation( + LinkRelation::Enclosure, + )), + ), + ], + link_id, + ); + } + ( + ICalendarProperty::Image, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VCalendar, + ) => { + entry.set_map_name(); + state.map_named_entry_with_id( + &mut entry, + if link_value_type == ICalendarValueType::Binary { + &[ + ICalendarParameterName::Display, + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Size, + ICalendarParameterName::Linkrel, + ICalendarParameterName::Jsid, + ] + } else { + &[ + ICalendarParameterName::Display, + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Size, + ICalendarParameterName::Jsid, + ] + }, + JSCalendarProperty::Links, + [ + ( + Key::Property(JSCalendarProperty::Href), + Value::Str(uri.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Link)), + ), + ( + Key::Property(JSCalendarProperty::Rel), + Value::Element(JSCalendarValue::LinkRelation(LinkRelation::Icon)), + ), + ], + link_id, + ); + } + ( + ICalendarProperty::Link, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VCalendar, + ) => { + entry.set_map_name(); + state.map_named_entry( + &mut entry, + &[ + ICalendarParameterName::Fmttype, + ICalendarParameterName::Label, + ICalendarParameterName::Linkrel, + ICalendarParameterName::Jsid, + ], + JSCalendarProperty::Links, + [ + ( + Key::Property(JSCalendarProperty::Href), + Value::Str(uri.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Link)), + ), + ], + ); + } + ( + ICalendarProperty::Url, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VLocation + | ICalendarComponentType::VCalendar, + ) => { + entry.set_map_name(); + state.map_named_entry( + &mut entry, + &[ICalendarParameterName::Label, ICalendarParameterName::Jsid], + JSCalendarProperty::Links, + [ + ( + Key::Property(JSCalendarProperty::Href), + Value::Str(uri.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Link)), + ), + ( + Key::Property(JSCalendarProperty::Rel), + Value::Element(JSCalendarValue::LinkRelation( + LinkRelation::Describedby, + )), + ), + ], + ); + } + ( + ICalendarProperty::Attendee, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let is_task = matches!(&state.component_type, ICalendarComponentType::VTodo); + let mut progress = None; + + for param in &mut entry.entry.params { + match (¶m.name, &mut param.value) { + ( + ICalendarParameterName::Partstat, + ICalendarParameterValue::Partstat(partstat), + ) if is_task => { + let status = match partstat { + ICalendarParticipationStatus::Completed => { + JSCalendarProgress::Completed + } + ICalendarParticipationStatus::InProcess => { + JSCalendarProgress::InProcess + } + ICalendarParticipationStatus::Failed => { + JSCalendarProgress::Failed + } + _ => { + continue; + } + }; + *partstat = ICalendarParticipationStatus::Accepted; + progress = Some(( + Key::Property(JSCalendarProperty::Progress), + Value::Element(JSCalendarValue::Progress(status)), + )); + } + (ICalendarParameterName::Role, ICalendarParameterValue::Role(role)) => { + has_owner |= matches!(role, ICalendarParticipationRole::Owner); + } + _ => {} + } + } + + state.map_named_entry( + &mut entry, + &[ + ICalendarParameterName::Cn, + ICalendarParameterName::Cutype, + ICalendarParameterName::DelegatedFrom, + ICalendarParameterName::DelegatedTo, + ICalendarParameterName::Email, + ICalendarParameterName::Member, + ICalendarParameterName::Partstat, + ICalendarParameterName::Role, + ICalendarParameterName::Rsvp, + ICalendarParameterName::SentBy, + ICalendarParameterName::Jsid, + ], + JSCalendarProperty::Participants, + [ + Some(( + Key::Property(JSCalendarProperty::CalendarAddress), + Value::Str(uri.into()), + )), + Some(( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Participant)), + )), + progress, + ] + .into_iter() + .flatten(), + ); + } + ( + ICalendarProperty::Organizer, + Some(ICalendarValue::Text(uri)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::OrganizerCalendarAddress), + Value::Str(uri.clone().into()), + ); + + if let Some(sent_by) = entry + .entry + .params + .iter() + .find(|param| param.name == ICalendarParameterName::SentBy) + .and_then(|param| param.value.as_text()) + { + state.entries.insert( + Key::Property(JSCalendarProperty::SentBy), + Value::Str(sent_by.to_string().into()), + ); + } + + /* + It also converts to a Participant object in the "participants" property, + if any of its parameters convert to a Participant object property, or + if none of the ATTENDEE properties in the iCalendar component have + the ROLE parameter set to "OWNER" + + An ORGANIZER property, an ATTENDEE property, and a PARTICIPANT component + in the same iCalendar component all convert to the same Participant + object, if their converted "calendarAddress" property values are equal. + */ + + if !has_owner + || entry.entry.params.iter().any(|p| { + matches!( + p.name, + ICalendarParameterName::Cn + | ICalendarParameterName::Email + | ICalendarParameterName::SentBy + | ICalendarParameterName::Jsid + ) + }) + { + let participant_id = state.find_participant_by_address(uri.as_ref()); + + state.map_named_entry_with_id( + &mut entry, + &[ + ICalendarParameterName::Cn, + ICalendarParameterName::Email, + ICalendarParameterName::SentBy, + ICalendarParameterName::Jsid, + ], + JSCalendarProperty::Participants, + [ + ( + Key::Property(JSCalendarProperty::CalendarAddress), + Value::Str(uri.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type( + JSCalendarType::Participant, + )), + ), + ( + Key::Property(JSCalendarProperty::Roles), + Value::Object(Map::from(vec![( + Key::Property(JSCalendarProperty::ParticipantRole( + JSCalendarParticipantRole::Owner, + )), + Value::Bool(true), + )])), + ), + ], + participant_id, + ); + } else { + continue; + } + } + ( + ICalendarProperty::CalendarAddress, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::Participant, + ) => { + if state.jsid.is_none() { + state.jsid = Some(uuid5(&value)); + } + state.entries.insert( + Key::Property(JSCalendarProperty::CalendarAddress), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::CalendarAddress); + entry.set_map_name(); + } + ( + ICalendarProperty::Categories, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + let obj = state + .entries + .entry(Key::Property(JSCalendarProperty::Keywords)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap(); + obj.upsert(Key::Owned(value), Value::Bool(true)); + for value in values { + if let Some(value) = value.into_text() { + obj.upsert(Key::from(value), Value::Bool(true)); + } + } + entry.set_converted_to_property(&JSCalendarProperty::::Keywords); + } + ( + ICalendarProperty::Class, + Some(ICalendarValue::Classification(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.merge_privacy(match value { + ICalendarClassification::Public => JSCalendarPrivacy::Public, + ICalendarClassification::Private => JSCalendarPrivacy::Private, + ICalendarClassification::Confidential => JSCalendarPrivacy::Secret, + }); + entry.set_converted_to_property(&JSCalendarProperty::::Privacy); + } + ( + ICalendarProperty::Class, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.merge_privacy(JSCalendarPrivacy::Private); + entry.entry.values = values.prepend(Some(ICalendarValue::Text(value))); + } + ( + ICalendarProperty::Color, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Color), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Color); + } + ( + ICalendarProperty::Concept, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) => { + let obj = state + .entries + .entry(Key::Property(JSCalendarProperty::Categories)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap(); + obj.upsert(Key::Owned(value), Value::Bool(true)); + for value in values { + if let Some(value) = value.into_text() { + obj.upsert(Key::from(value), Value::Bool(true)); + } + } + entry.set_converted_to_property(&JSCalendarProperty::::Categories); + } + ( + ICalendarProperty::Conference, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.map_named_entry( + &mut entry, + &[ + ICalendarParameterName::Jsid, + ICalendarParameterName::Feature, + ICalendarParameterName::Label, + ], + JSCalendarProperty::VirtualLocations, + [ + ( + Key::Property(JSCalendarProperty::Uri), + Value::Str(value.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type( + JSCalendarType::VirtualLocation, + )), + ), + ], + ); + } + ( + ICalendarProperty::Coordinates, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VLocation, + ) => { + if state.jsid.is_none() { + state.jsid = Some(uuid5(&value)); + } + state.entries.insert( + Key::Property(JSCalendarProperty::Coordinates), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Coordinates); + } + ( + ICalendarProperty::Geo, + Some(ICalendarValue::Float(coord1)), + ICalendarComponentType::VLocation, + ) if !entry.entry.is_derived() => { + let coord2 = values.next().and_then(|v| v.as_float()).unwrap_or_default(); + + state.entries.insert( + Key::Property(JSCalendarProperty::Coordinates), + Value::Str(format!("geo:{coord1},{coord2}").into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Coordinates); + entry.set_map_name(); + } + ( + ICalendarProperty::Geo, + Some(ICalendarValue::Float(coord1)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if !entry.entry.is_derived() => { + let coord2 = values.next().and_then(|v| v.as_float()).unwrap_or_default(); + if entry.entry.jsid().is_none() + && let Some(main_location_id) = main_location_id.take() + { + entry + .entry + .add_param(ICalendarParameter::jsid(main_location_id)); + } + + entry.set_map_name(); + state.map_named_entry( + &mut entry, + &[ICalendarParameterName::Jsid], + JSCalendarProperty::Locations, + [ + ( + Key::Property(JSCalendarProperty::Coordinates), + Value::Str(format!("geo:{coord1},{coord2}").into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Location)), + ), + ], + ); + } + ( + ICalendarProperty::Name, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VLocation, + ) => { + if state.jsid.is_none() { + state.jsid = Some(uuid5(&value)); + } + state.entries.insert( + Key::Property(JSCalendarProperty::Name), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Name); + state.set_map_component(); + } + ( + ICalendarProperty::Name, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar, + ) + | ( + ICalendarProperty::Summary, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state + .entries + .extract_params(&mut entry.entry, &[ICalendarParameterName::Language]); + state.entries.insert( + Key::Property(JSCalendarProperty::Title), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Title); + } + ( + ICalendarProperty::Summary, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::Participant, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Name), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Name); + state.set_map_component(); + } + ( + ICalendarProperty::Location, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + let location_id = if let Some(location_id) = entry.entry.jsid() { + main_location_id = Some(location_id.to_string()); + location_id + } else { + main_location_id = Some(uuid5(&value)); + main_location_id.as_deref().unwrap() + }; + + if has_locations { + state.entries.insert( + Key::Property(JSCalendarProperty::MainLocationId), + Value::Str(location_id.to_string().into()), + ); + } + + state.map_named_entry( + &mut entry, + &[ + ICalendarParameterName::Jsid, + ICalendarParameterName::Derived, + ], + JSCalendarProperty::Locations, + [ + ( + Key::Property(JSCalendarProperty::Name), + Value::Str(value.into()), + ), + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Location)), + ), + ], + ); + } + ( + ICalendarProperty::LocationType, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VLocation, + ) => { + let obj = state + .entries + .entry(Key::Property(JSCalendarProperty::LocationTypes)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap(); + obj.upsert(Key::Owned(value), Value::Bool(true)); + for value in values { + if let Some(value) = value.into_text() { + obj.upsert(Key::from(value), Value::Bool(true)); + } + } + entry.set_converted_to_property(&JSCalendarProperty::::LocationTypes); + } + + ( + ICalendarProperty::LastModified, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VCalendar, + ) if value.has_date_and_time() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Updated), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + value.to_timestamp().unwrap(), + false, + ))), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Updated); + } + ( + ICalendarProperty::Created, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VCalendar, + ) if value.has_date_and_time() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Created), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + value.to_timestamp().unwrap(), + false, + ))), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Created); + } + ( + ICalendarProperty::Description, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::Participant + | ICalendarComponentType::VCalendar + | ICalendarComponentType::VLocation, + ) if !entry.entry.is_derived() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Description), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Description); + if matches!(state.component_type, ICalendarComponentType::Participant) { + state.set_map_component(); + } + } + ( + ICalendarProperty::StyledDescription, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if !entry.entry.is_derived() + && entry + .entry + .parameter(&ICalendarParameterName::Fmttype) + .is_none_or(|v| v.as_text().is_some_and(|v| v.starts_with("text/"))) => + { + state + .entries + .extract_params(&mut entry.entry, &[ICalendarParameterName::Fmttype]); + state.entries.insert( + Key::Property(JSCalendarProperty::Description), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Description); + if !state + .entries + .contains_key(&Key::Property(JSCalendarProperty::DescriptionContentType)) + { + entry.set_map_name(); + } + } + ( + ICalendarProperty::Dtstart, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if value.has_date() => { + let tzid = entry.entry.tz_id(); + state.tz_start = tzid.and_then(|v| tz_resolver.resolve(v)); + if let Some(dt) = value + .to_date_time() + .and_then(|dt| dt.resolve_with_tz(state.tz_start.unwrap_or_default())) + { + state.has_dates = true; + state.entries.insert( + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + dt.naive_timestamp(), + true, + ))), + ); + + // Remove IANA TZ references + if tzid.is_some() + && state.tz_start.and_then(|tz| tz.name()).as_deref() == tzid + { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Tzid); + } + + if !value.has_time() { + state.entries.insert( + Key::Property(JSCalendarProperty::ShowWithoutTime), + Value::Bool(true), + ); + } + entry.set_converted_to_property(&JSCalendarProperty::::Start); + start_date = Some(dt); + start_is_date = !value.has_time(); + + if state.tz_start.is_none() { + state.tz_start = dt.timezone().to_resolved(); + } + } else { + state.tz_start = None; + entry.entry.values = + values.prepend(Some(ICalendarValue::PartialDateTime(value))); + } + } + ( + ICalendarProperty::Dtend, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent, + ) if value.has_date() && start_date.or(state.recurrence_id).is_some() => { + let tzid = entry.entry.tz_id(); + state.tz_end = tzid.and_then(|v| tz_resolver.resolve(v)).or(state.tz_start); + if let Some((delta, dt, start)) = value + .to_date_time() + .and_then(|dt| dt.to_date_time_with_tz(state.tz_end.unwrap_or_default())) + .and_then(|dt| { + let start = start_date.or(state.recurrence_id)?; + let delta = dt.signed_duration_since(start).as_secs(); + (delta > 0).then_some((delta, dt, start)) + }) + { + state.has_dates = true; + + /* + If the value type of the DTEND property is DATE, then the duration is + the number of days or weeks between the DTSTART and DTEND property + values. If the value type is DATE-TIME, then the duration is the + timespan between the DTSTART and DTEND property values when converted + to UTC time. + */ + + let days = if !value.has_time() { + i64::from(dt.days_since(start)) + } else { + 0 + }; + let duration = if days > 0 { + ICalendarDuration::from_days(days) + } else { + ICalendarDuration::from_seconds(delta) + }; + + state.entries.insert( + Key::Property(JSCalendarProperty::Duration), + Value::Element(JSCalendarValue::Duration(duration)), + ); + + // Remove IANA TZ references + if tzid.is_some() + && state.tz_end.and_then(|tz| tz.name()).as_deref() == tzid + { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Tzid); + } + + if !value.has_time() { + state.entries.insert( + Key::Property(JSCalendarProperty::ShowWithoutTime), + Value::Bool(true), + ); + } + entry.set_converted_to_property(&JSCalendarProperty::::Duration); + entry.set_map_name(); + + if state.tz_end.is_none() { + state.tz_end = dt.timezone().to_resolved(); + } + if state.tz_start.is_none() { + state.tz_start = state.tz_end; + } + } else { + state.tz_end = None; + entry.entry.values = + values.prepend(Some(ICalendarValue::PartialDateTime(value))); + } + } + ( + ICalendarProperty::Due, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VTodo, + ) if value.has_date() => { + let tzid = entry.entry.tz_id(); + let due_tz = tzid.and_then(|v| tz_resolver.resolve(v)).or(state.tz_start); + if let Some(dt) = value + .to_date_time() + .and_then(|dt| dt.to_date_time_with_tz(due_tz.unwrap_or_default())) + { + // Remove IANA TZ references + if tzid.is_some() + && (state.tz_start.is_none() || due_tz == state.tz_start) + && due_tz.and_then(|tz| tz.name()).as_deref() == tzid + { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Tzid); + } + + state.has_dates = true; + if state.tz_start.is_none() { + state.tz_start = dt.timezone().to_resolved(); + } + + state.entries.insert( + Key::Property(JSCalendarProperty::Due), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + dt.with_timezone(state.tz_start.unwrap_or_default()) + .naive_timestamp(), + true, + ))), + ); + state.due = value.has_time().then_some(dt); + + if !value.has_time() { + state.entries.insert( + Key::Property(JSCalendarProperty::ShowWithoutTime), + Value::Bool(true), + ); + } + + entry.set_converted_to_property(&JSCalendarProperty::::Due); + } else { + entry.entry.values = + values.prepend(Some(ICalendarValue::PartialDateTime(value))); + } + } + ( + ICalendarProperty::Dtstamp, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if value.has_date_and_time() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Updated), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + value.to_timestamp().unwrap_or_default(), + false, + ))), + ); + + entry.set_converted_to_property(&JSCalendarProperty::::Updated); + } + ( + ICalendarProperty::Duration, + Some(ICalendarValue::Duration(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Duration), + Value::Element(JSCalendarValue::Duration(value)), + ); + + entry.set_converted_to_property(&JSCalendarProperty::::Duration); + } + ( + ICalendarProperty::EstimatedDuration, + Some(ICalendarValue::Duration(value)), + ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::EstimatedDuration), + Value::Element(JSCalendarValue::Duration(value)), + ); + + entry.set_converted_to_property(&JSCalendarProperty::::EstimatedDuration); + } + ( + ICalendarProperty::RecurrenceId, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if value.has_date() => { + let tzid = entry.entry.tz_id(); + let rid_tz = tzid.and_then(|v| tz_resolver.resolve(v)); + + if let Some(dt) = value + .to_date_time() + .and_then(|dt| dt.resolve_with_tz(rid_tz.unwrap_or_default())) + { + state.has_dates = true; + state.recurrence_id = Some(dt); + state.recurrence_id_is_date = !value.has_time(); + if start_date.is_none() && !value.has_time() { + state.entries.insert( + Key::Property(JSCalendarProperty::ShowWithoutTime), + Value::Bool(true), + ); + } + + // Remove IANA TZ references + if tzid.is_some() && rid_tz.and_then(|tz| tz.name()).as_deref() == tzid { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Tzid); + } + if !value.has_time() { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Value); + } + + entry.set_converted_to_property(&JSCalendarProperty::::RecurrenceId); + } else { + entry.entry.values = + values.prepend(Some(ICalendarValue::PartialDateTime(value))); + } + } + ( + ICalendarProperty::Rdate, + Some(value @ ICalendarValue::Period(_)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if !is_todo || start_date.is_some() => { + /* + The property value converts to the key in the "recurrenceOverrides" + property value, for values of type PERIOD this only applies to the + period start. The duration of a PERIOD converts to the PatchObject's + "duration" property. + */ + + let tz = entry + .entry + .tz_id() + .and_then(|v| tz_resolver.resolve(v)) + .or(state.tz_start) + .unwrap_or_default(); + + // Restore the values, they are preserved as-is unless every period converts + entry.entry.values = values.prepend(Some(value)); + + if let Some((start, _)) = entry + .entry + .values + .first() + .and_then(|value| period_to_date_time(value, tz)) + && entry + .entry + .values + .iter() + .skip(1) + .all(|value| period_to_date_time(value, tz).is_some()) + { + state.has_dates = true; + if state.tz_start.is_none() { + state.tz_start = start.timezone().to_resolved(); + } + + let tz_start = state.tz_start.unwrap_or_default(); + let overrides_name = + JSCalendarProperty::RecurrenceOverrides::.to_string(); + + // Each period converts to its own recurrence override + for pos in 0..entry.entry.values.len() { + let (dt, duration) = + period_to_date_time(&entry.entry.values[pos], tz).unwrap(); + let key = Key::Property(JSCalendarProperty::DateTime( + JSCalendarDateTime::new( + dt.with_timezone(tz_start).naive_timestamp(), + true, + ), + )); + let key_name = key.to_string().into_owned(); + + state.insert_recurrence_override( + key, + Value::Object(Map::from(vec![( + Key::Property(JSCalendarProperty::Duration), + Value::Element(JSCalendarValue::Duration(duration)), + )])), + ); + + if pos == 0 { + entry.set_map_name(); + entry.set_converted_to(|| { + String::from_pointer([ + overrides_name.as_ref(), + key_name.as_str(), + ]) + }); + } else { + state.add_period_conversion_prop(format!( + "{}/{}", + overrides_name.as_ref(), + key_name + )); + } + } + } + } + ( + ICalendarProperty::Rdate | ICalendarProperty::Exdate, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if value.has_date() && (!is_todo || start_date.is_some()) => { + let tzid = entry.entry.tz_id(); + let tz = tzid.and_then(|v| tz_resolver.resolve(v)).or(state.tz_start); + + if let Some(dt) = value + .to_date_time() + .and_then(|dt| dt.resolve_with_tz(tz.unwrap_or_default())) + { + state.has_dates = true; + + // Remove IANA TZ references + if tzid.is_some() + && tz == state.tz_start + && tz.and_then(|tz| tz.name()).as_deref() == tzid + { + entry + .entry + .params + .retain(|p| p.name != ICalendarParameterName::Tzid); + } + + if state.tz_start.is_none() { + state.tz_start = dt.timezone().to_resolved(); + } + + let overrides = state + .entries + .entry(Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap(); + let value = Value::Object(Map::from( + if matches!(entry.entry.name, ICalendarProperty::Exdate) { + vec![( + Key::Property(JSCalendarProperty::Excluded), + Value::Bool(true), + )] + } else { + vec![] + }, + )); + + for (pos, dt) in [dt] + .into_iter() + .chain(values.filter_map(|v| { + v.into_partial_date_time() + .and_then(|dt| dt.to_date_time()) + .and_then(|dt| dt.resolve_with_tz(tz.unwrap_or_default())) + })) + .enumerate() + { + let key = Key::Property(JSCalendarProperty::DateTime( + JSCalendarDateTime::local_in(dt, state.tz_start), + )); + + if pos == 0 { + entry.set_converted_to(|| { + String::from_pointer([ + JSCalendarProperty::RecurrenceOverrides:: + .to_string() + .as_ref(), + key.to_string().as_ref(), + ]) + }); + } + + overrides.upsert(key, value.clone()); + } + } else { + entry.entry.values = + values.prepend(Some(ICalendarValue::PartialDateTime(value))); + } + } + ( + ICalendarProperty::Rrule, + Some(ICalendarValue::RecurrenceRule(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if !is_todo || start_date.is_some() => { + let mut rrule = Map::from(Vec::with_capacity(4)); + + rrule.insert_unchecked( + JSCalendarProperty::Frequency, + Value::Element(JSCalendarValue::Frequency(value.freq)), + ); + + for (key, value) in [ + ( + JSCalendarProperty::Until, + value + .until + .as_ref() + .and_then(|v| { + v.to_date_time_with_tz(state.tz_start.unwrap_or_default()) + }) + .map(|dt| { + Value::Element(JSCalendarValue::DateTime( + JSCalendarDateTime::new( + dt.with_timezone(state.tz_start.unwrap_or_default()) + .naive_timestamp(), + true, + ), + )) + }), + ), + ( + JSCalendarProperty::Count, + value.count.map(|v| Value::Number((v as u64).into())), + ), + ( + JSCalendarProperty::Interval, + value.interval.map(|v| Value::Number((v as u64).into())), + ), + ( + JSCalendarProperty::FirstDayOfWeek, + value + .wkst + .map(|v| Value::Element(JSCalendarValue::Weekday(v))), + ), + ( + JSCalendarProperty::Rscale, + value + .rscale + .map(|v| Value::Element(JSCalendarValue::CalendarScale(v))), + ), + ( + JSCalendarProperty::Skip, + value.skip.map(|v| Value::Element(JSCalendarValue::Skip(v))), + ), + ] { + if let Some(value) = value { + rrule.insert_unchecked(key, value); + } + } + + for (key, values) in [ + ( + JSCalendarProperty::BySecond, + (!value.bysecond.is_empty()).then(|| { + value + .bysecond + .iter() + .map(|&v| Value::Number((v as u64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByMinute, + (!value.byminute.is_empty()).then(|| { + value + .byminute + .iter() + .map(|&v| Value::Number((v as u64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByHour, + (!value.byhour.is_empty()).then(|| { + value + .byhour + .iter() + .map(|&v| Value::Number((v as u64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByDay, + (!value.byday.is_empty()).then(|| { + value + .byday + .iter() + .map(|v| { + Value::Object(Map::from_iter( + [ + Some(( + Key::Property(JSCalendarProperty::Day), + Value::Element(JSCalendarValue::Weekday( + v.weekday, + )), + )), + v.ordwk.map(|v| { + ( + Key::Property( + JSCalendarProperty::NthOfPeriod, + ), + Value::Number((v as i64).into()), + ) + }), + ] + .into_iter() + .flatten(), + )) + }) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByMonthDay, + (!value.bymonthday.is_empty()).then(|| { + value + .bymonthday + .iter() + .map(|&v| Value::Number((v as i64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByYearDay, + (!value.byyearday.is_empty()).then(|| { + value + .byyearday + .iter() + .map(|&v| Value::Number((v as i64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByWeekNo, + (!value.byweekno.is_empty()).then(|| { + value + .byweekno + .iter() + .map(|&v| Value::Number((v as i64).into())) + .collect::>() + }), + ), + ( + JSCalendarProperty::ByMonth, + (!value.bymonth.is_empty()).then(|| { + value + .bymonth + .iter() + .map(|&v| Value::Element(JSCalendarValue::Month(v))) + .collect::>() + }), + ), + ( + JSCalendarProperty::BySetPosition, + (!value.bysetpos.is_empty()).then(|| { + value + .bysetpos + .iter() + .map(|&v| Value::Number((v as i64).into())) + .collect::>() + }), + ), + ] { + if let Some(values) = values { + rrule.insert_unchecked(key, Value::Array(values)); + } + } + + state.entries.insert( + Key::Property(JSCalendarProperty::RecurrenceRule), + Value::Object(rrule), + ); + + entry.set_converted_to_property(&JSCalendarProperty::::RecurrenceRule); + } + ( + ICalendarProperty::Method, + Some(ICalendarValue::Method(value)), + ICalendarComponentType::VCalendar, + ) => { + for object in &mut group_objects { + object.as_object_mut().unwrap().insert_unchecked( + Key::Property(JSCalendarProperty::Method), + Value::Element(JSCalendarValue::Method(value.clone())), + ); + } + continue; + } + ( + ICalendarProperty::PercentComplete, + Some(ICalendarValue::Integer(value)), + ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::PercentComplete), + Value::Number(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::PercentComplete); + } + ( + ICalendarProperty::Priority, + Some(ICalendarValue::Integer(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Priority), + Value::Number(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Priority); + } + ( + ICalendarProperty::Sequence, + Some(ICalendarValue::Integer(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Sequence), + Value::Number(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Sequence); + } + ( + ICalendarProperty::Version, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar, + ) if value == JSCALENDAR_VERSION && entry.entry.params.is_empty() => { + continue; + } + ( + ICalendarProperty::Prodid, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::ProdId), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::ProdId); + } + ( + ICalendarProperty::RelatedTo, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VAlarm, + ) => { + let mut rels: Map<'_, JSCalendarProperty, JSCalendarValue> = + Map::from(vec![]); + for param in std::mem::take(&mut entry.entry.params) { + match (param.name, param.value) { + ( + ICalendarParameterName::Reltype, + ICalendarParameterValue::Reltype(value), + ) => { + rels.upsert( + match value { + ICalendarRelationshipType::Child => { + Key::Property(JSCalendarProperty::RelationValue( + JSCalendarRelation::Child, + )) + } + ICalendarRelationshipType::Parent => { + Key::Property(JSCalendarProperty::RelationValue( + JSCalendarRelation::Parent, + )) + } + ICalendarRelationshipType::Snooze => { + Key::Property(JSCalendarProperty::RelationValue( + JSCalendarRelation::Snooze, + )) + } + ICalendarRelationshipType::First => { + Key::Property(JSCalendarProperty::RelationValue( + JSCalendarRelation::First, + )) + } + ICalendarRelationshipType::Next => { + Key::Property(JSCalendarProperty::RelationValue( + JSCalendarRelation::Next, + )) + } + other => Key::Borrowed(other.as_str()), + }, + Value::Bool(true), + ); + } + ( + ICalendarParameterName::Reltype, + ICalendarParameterValue::Text(value), + ) => { + rels.upsert(Key::Owned(value), Value::Bool(true)); + } + (name, value) => { + entry.entry.params.push(ICalendarParameter { name, value }); + } + } + } + + entry.set_converted_to(|| { + String::from_pointer([ + JSCalendarProperty::RelatedTo::.to_string().as_ref(), + value.as_str(), + ]) + }); + + state + .entries + .entry(Key::Property(JSCalendarProperty::RelatedTo)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap() + .upsert( + Key::Owned(value), + Value::Object(Map::from(if !rels.is_empty() { + vec![( + Key::Property(JSCalendarProperty::Relation), + Value::Object(rels), + )] + } else { + vec![] + })), + ); + } + ( + ICalendarProperty::ShowWithoutTime, + Some(ICalendarValue::Boolean(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) if !is_todo || state.recurrence_id.is_some() || state.has_start_or_due() => { + state.entries.insert( + Key::Property(JSCalendarProperty::ShowWithoutTime), + Value::Bool(value), + ); + entry.set_converted_to_property(&JSCalendarProperty::::ShowWithoutTime); + } + ( + ICalendarProperty::Status, + Some(ICalendarValue::Status(value)), + ICalendarComponentType::VEvent, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Status), + match value { + ICalendarStatus::Tentative => Value::Element( + JSCalendarValue::EventStatus(JSCalendarEventStatus::Tentative), + ), + ICalendarStatus::Confirmed => Value::Element( + JSCalendarValue::EventStatus(JSCalendarEventStatus::Confirmed), + ), + ICalendarStatus::Cancelled => Value::Element( + JSCalendarValue::EventStatus(JSCalendarEventStatus::Cancelled), + ), + other => Value::Str(other.as_str().into()), + }, + ); + entry.set_converted_to_property(&JSCalendarProperty::::Status); + } + ( + ICalendarProperty::Status, + Some(ICalendarValue::Status(value)), + ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Progress), + match value { + ICalendarStatus::NeedsAction => Value::Element( + JSCalendarValue::Progress(JSCalendarProgress::NeedsAction), + ), + ICalendarStatus::Completed => Value::Element( + JSCalendarValue::Progress(JSCalendarProgress::Completed), + ), + ICalendarStatus::InProcess => Value::Element( + JSCalendarValue::Progress(JSCalendarProgress::InProcess), + ), + ICalendarStatus::Failed => Value::Element(JSCalendarValue::Progress( + JSCalendarProgress::Failed, + )), + ICalendarStatus::Cancelled => Value::Element( + JSCalendarValue::Progress(JSCalendarProgress::Cancelled), + ), + other => Value::Str(other.as_str().into()), + }, + ); + entry.set_converted_to_property(&JSCalendarProperty::::Progress); + } + ( + ICalendarProperty::Status, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar | ICalendarComponentType::VTodo, + ) => { + let prop = if matches!(state.component_type, ICalendarComponentType::VTodo) { + JSCalendarProperty::Progress + } else { + JSCalendarProperty::Status + }; + + entry.set_converted_to_property(&prop); + state + .entries + .insert(Key::Property(prop), Value::Str(value.into())); + } + ( + ICalendarProperty::Source, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::Source), + Value::Str(value.into()), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Source); + } + ( + ICalendarProperty::Transp, + Some(ICalendarValue::Transparency(value)), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo, + ) => { + state.entries.insert( + Key::Property(JSCalendarProperty::FreeBusyStatus), + Value::Element(JSCalendarValue::FreeBusyStatus(match value { + ICalendarTransparency::Opaque => JSCalendarFreeBusyStatus::Busy, + ICalendarTransparency::Transparent => JSCalendarFreeBusyStatus::Free, + })), + ); + entry.set_converted_to_property(&JSCalendarProperty::::FreeBusyStatus); + } + ( + ICalendarProperty::Trigger, + Some(ICalendarValue::Duration(value)), + ICalendarComponentType::VAlarm, + ) => { + let mut obj = Map::from(vec![ + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::OffsetTrigger)), + ), + ( + Key::Property(JSCalendarProperty::Offset), + Value::Element(JSCalendarValue::Duration(value)), + ), + ]); + + for param in std::mem::take(&mut entry.entry.params) { + match (param.name, param.value) { + ( + ICalendarParameterName::Related, + ICalendarParameterValue::Related(value), + ) => { + obj.upsert( + Key::Property(JSCalendarProperty::RelativeTo), + Value::Element(JSCalendarValue::RelativeTo(match value { + ICalendarRelated::Start => JSCalendarRelativeTo::Start, + ICalendarRelated::End => JSCalendarRelativeTo::End, + })), + ); + } + (name, value) => { + entry.entry.params.push(ICalendarParameter { name, value }); + } + } + } + + state.entries.insert( + Key::Property(JSCalendarProperty::Trigger), + Value::Object(obj), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Trigger); + } + ( + ICalendarProperty::Trigger, + Some(ICalendarValue::PartialDateTime(value)), + ICalendarComponentType::VAlarm, + ) if value.has_date_and_time() => { + state.entries.insert( + Key::Property(JSCalendarProperty::Trigger), + Value::Object(Map::from(vec![ + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type( + JSCalendarType::AbsoluteTrigger, + )), + ), + ( + Key::Property(JSCalendarProperty::When), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + value.to_timestamp().unwrap_or_default(), + false, + ))), + ), + ])), + ); + entry.set_converted_to_property(&JSCalendarProperty::::Trigger); + } + ( + ICalendarProperty::Uid, + Some(ICalendarValue::Text(value)), + ICalendarComponentType::VCalendar + | ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo, + ) => { + state.uid = Some(value); + continue; + } + (ICalendarProperty::Jsid, Some(ICalendarValue::Text(value)), _) => { + state.jsid = Some(value); + continue; + } + (ICalendarProperty::Jsid, _, _) => { + continue; + } + (ICalendarProperty::Jsprop, Some(ICalendarValue::Text(value)), component_type) => { + let ptr = entry.entry.params.iter().find_map(|param| match param { + ICalendarParameter { + name: ICalendarParameterName::Jsptr, + value: ICalendarParameterValue::Text(ptr), + } => Some(JsonPointer::>::parse(ptr)), + _ => None, + }); + let is_calendar_object = matches!( + component_type, + ICalendarComponentType::VCalendar + | ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + ); + if ptr.as_ref().is_some_and(|ptr| { + JSCalendarProperty::sets_misplaced_excluded(ptr) + || (is_calendar_object + && (JSCalendarProperty::is_metadata_pointer(ptr) + || matches!( + ptr.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::Version + ))) + ))) + }) { + continue; + } + let is_dateless_task = + is_todo && state.recurrence_id.is_none() && !state.has_start_or_due(); + if let Some(ptr) = ptr.filter(|ptr| { + ptr.as_slice().iter().all(|item| { + matches!(item, JsonPointerItem::Key(_) | JsonPointerItem::Number(_)) + }) && !(is_dateless_task + && matches!( + ptr.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::ShowWithoutTime + | JSCalendarProperty::TimeZone + ))) + )) + }) && let Some(patch) = ptr.parse_jsprop_value(&value) + { + state.patch_objects.push((ptr, patch)); + continue; + } + entry.entry.values = values.prepend(Some(ICalendarValue::Text(value))); + } + ( + ICalendarProperty::Description | ICalendarProperty::Summary, + _, + ICalendarComponentType::VAlarm, + ) if entry.entry.is_derived() => { + continue; + } + (ICalendarProperty::Begin | ICalendarProperty::End, _, _) => { + continue; + } + + (_, value, _) => { + entry.entry.values = values.prepend(value); + } + } + + state.add_conversion_props(entry); + } + + if start_is_date { + state.default_to_one_day(); + } + + if state.tz_start.is_none() + && !state + .entries + .contains_key(&Key::Property(JSCalendarProperty::Start)) + && !state + .entries + .contains_key(&Key::Property(JSCalendarProperty::Due)) + { + state.tz_start = state + .recurrence_id + .and_then(|recurrence_id| recurrence_id.timezone().to_resolved()); + } + + if !group_objects.is_empty() { + state.entries.insert( + Key::Property(JSCalendarProperty::Entries), + Value::Array(group_objects), + ); + } + + state + } +} + +#[derive(Clone, Copy, Default)] +pub(super) struct TaskAnchors { + start: bool, + due: bool, +} + +impl TaskAnchors { + fn new(entries: &[ICalendarEntry], series: &[(String, TaskAnchors)]) -> Self { + let mut has_start = false; + let mut has_due = false; + let mut has_duration = false; + let mut has_recurrence_id = false; + let mut uid = None; + for entry in entries { + let converts = || { + matches!( + entry.values.first(), + Some(ICalendarValue::PartialDateTime(value)) + if value.has_date() && value.to_date_time().is_some() + ) + }; + match entry.name { + ICalendarProperty::Dtstart => has_start |= converts(), + ICalendarProperty::Due => has_due |= converts(), + ICalendarProperty::RecurrenceId => has_recurrence_id |= converts(), + ICalendarProperty::Duration => { + has_duration |= + matches!(entry.values.first(), Some(ICalendarValue::Duration(_))); + } + ICalendarProperty::Uid => { + uid = entry.values.first().and_then(ICalendarValue::as_text); + } + _ => {} + } + } + if !has_recurrence_id { + return Self { + start: has_start, + due: has_due, + }; + } + match uid.and_then(|uid| { + series + .binary_search_by(|(series_uid, _)| series_uid.as_str().cmp(uid)) + .ok() + .and_then(|position| series.get(position)) + }) { + Some((_, master)) => Self { + start: has_start || master.start, + due: has_due || (master.due && !has_start && !has_duration), + }, + None => Self { + start: has_start || !has_due, + due: has_due, + }, + } + } + + fn cover(self, alarm: &ICalendarComponent) -> bool { + alarm + .entries + .iter() + .filter(|entry| { + entry.name == ICalendarProperty::Trigger + && matches!(entry.values.first(), Some(ICalendarValue::Duration(_))) + }) + .all(|trigger| { + let related = trigger.params.iter().rev().find_map(|param| match param { + ICalendarParameter { + name: ICalendarParameterName::Related, + value: ICalendarParameterValue::Related(related), + } => Some(related), + _ => None, + }); + match related { + Some(ICalendarRelated::End) => self.due, + _ => self.start, + } + }) + } +} + +impl ICalendar { + fn task_series(&self, component_ids: &[u32]) -> Vec<(String, TaskAnchors)> { + let tasks = || { + component_ids + .iter() + .filter_map(|id| self.components.get(*id as usize)) + .filter(|component| component.component_type == ICalendarComponentType::VTodo) + }; + if !tasks().any(|task| task.has_property(&ICalendarProperty::RecurrenceId)) { + return Vec::new(); + } + let mut series = tasks() + .filter(|task| !task.has_property(&ICalendarProperty::RecurrenceId)) + .filter_map(|task| { + Some(( + task.uid()?.to_string(), + TaskAnchors::new(&task.entries, &[]), + )) + }) + .collect::>(); + series.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + series + } +} + +trait PrependValue { + fn prepend(self, first: Option) -> SmallVec<[ICalendarValue; 1]>; +} + +impl PrependValue for IntoIter<[ICalendarValue; 1]> { + fn prepend(self, first: Option) -> SmallVec<[ICalendarValue; 1]> { + let mut values = SmallVec::with_capacity(usize::from(first.is_some()) + self.len()); + values.extend(first); + values.extend(self); + values + } +} + +fn period_to_date_time( + value: &ICalendarValue, + tz: Tz, +) -> Option<(ZonedDateTime, ICalendarDuration)> { + match value { + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, end } => { + let start = start.to_date_time()?.to_date_time_with_tz(tz)?; + let end = end.to_date_time()?.to_date_time_with_tz(tz)?; + + Some(( + start, + ICalendarDuration::from_seconds(end.signed_duration_since(start).as_secs()), + )) + } + ICalendarPeriod::Duration { start, duration } => Some(( + start.to_date_time()?.to_date_time_with_tz(tz)?, + duration.clone(), + )), + }, + _ => None, + } +} + diff --git a/crates/calcard/src/jscalendar/import/mod.rs b/crates/calcard/src/jscalendar/import/mod.rs new file mode 100644 index 00000000..15758042 --- /dev/null +++ b/crates/calcard/src/jscalendar/import/mod.rs @@ -0,0 +1,249 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashMap; +use jmap_tools::{JsonPointer, Key, Map, Value}; + +use crate::common::timezone::ZonedDateTime; +use crate::{ + common::{ + blob::{BlobIdFn, BlobIdGenerator, BlobIds, BlobOptions, NoBlobIds}, + jsprop::ordered::OrderedMap, + timezone::Tz, + }, + icalendar::{ + ICalendarComponentType, ICalendarEntry, ICalendarParameterName, ICalendarProperty, + }, + jscalendar::{ + JSCalendarDateTime, JSCalendarId, JSCalendarProperty, JSCalendarValue, + ext::JSCalendarObjectExt, + }, +}; + +pub mod convert; +pub mod params; +pub mod props; + +const PROPERTY_MAP_CAPACITY: usize = 16; + +#[derive(Default)] +#[allow(clippy::type_complexity)] +struct State { + component_type: ICalendarComponentType, + entries: PropertyMap, + ical_converted_properties: OrderedMap>, + ical_properties: Vec, JSCalendarValue>>, + ical_components: Option, JSCalendarValue>>, + recurrence_overrides: Vec<(JSCalendarDateTime, State)>, + patch_objects: Vec<( + JsonPointer>, + Value<'static, JSCalendarProperty, JSCalendarValue>, + )>, + link_ids: LinkIds, + parameters: Map<'static, JSCalendarProperty, JSCalendarValue>, + jsid: Option, + uid: Option, + recurrence_id: Option, + recurrence_id_is_date: bool, + due: Option, + tz_start: Option, + tz_end: Option, + has_dates: bool, + has_end: bool, + map_component: bool, + is_recurrence_instance: bool, + include_ical_components: bool, +} + +struct PropertyMap( + Map<'static, JSCalendarProperty, JSCalendarValue>, +); + +struct PropertyEntry<'a, I: JSCalendarId, B: JSCalendarId> { + map: &'a mut Map<'static, JSCalendarProperty, JSCalendarValue>, + key: Key<'static, JSCalendarProperty>, +} + +impl Default for PropertyMap { + fn default() -> Self { + Self(Map::from(Vec::with_capacity(PROPERTY_MAP_CAPACITY))) + } +} + +impl PropertyMap { + #[inline] + fn insert( + &mut self, + key: Key<'static, JSCalendarProperty>, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ) -> Option, JSCalendarValue>> { + self.0.upsert(key, value) + } + + #[inline] + fn entry(&mut self, key: Key<'static, JSCalendarProperty>) -> PropertyEntry<'_, I, B> { + PropertyEntry { + map: &mut self.0, + key, + } + } + + #[inline] + fn get( + &self, + key: &Key<'_, JSCalendarProperty>, + ) -> Option<&Value<'static, JSCalendarProperty, JSCalendarValue>> { + self.0.lookup(key) + } + + #[inline] + fn contains_key(&self, key: &Key<'_, JSCalendarProperty>) -> bool { + self.0.key_position(key).is_some() + } + + fn retain( + &mut self, + mut keep: impl FnMut( + &Key<'static, JSCalendarProperty>, + &mut Value<'static, JSCalendarProperty, JSCalendarValue>, + ) -> bool, + ) { + self.0 + .as_mut_vec() + .retain_mut(|(key, value)| keep(key, value)); + } + + #[inline] + fn into_map(self) -> Map<'static, JSCalendarProperty, JSCalendarValue> { + self.0 + } +} + +impl<'a, I: JSCalendarId, B: JSCalendarId> PropertyEntry<'a, I, B> { + #[inline] + fn or_insert_with( + self, + value: impl FnOnce() -> Value<'static, JSCalendarProperty, JSCalendarValue>, + ) -> &'a mut Value<'static, JSCalendarProperty, JSCalendarValue> { + let position = match self.map.key_position(&self.key) { + Some(position) => position, + None => { + self.map.insert_unchecked(self.key, value()); + self.map.len() - 1 + } + }; + &mut self.map.as_mut_vec()[position].1 + } +} + +#[derive(Debug, Default)] +struct ICalendarConvertedProperty { + name: Option, + params: ICalendarParams, +} + +#[derive(Debug, Default)] +struct ICalendarParams( + Vec<(ICalendarParameterName, ParamValues)>, +); + +#[derive(Debug)] +enum ParamValues { + One(Value<'static, JSCalendarProperty, JSCalendarValue>), + Many(Vec, JSCalendarValue>>), +} + +#[derive(Debug, Default)] +struct LinkIds(AHashMap); + +#[derive(Debug, Clone, Copy)] +struct LinkId { + position: usize, + next_suffix: u32, +} + +#[derive(Debug, Clone)] +struct EntryState { + entry: ICalendarEntry, + converted_to: Option, + map_name: bool, + keep_converted_path: bool, +} + +#[derive(Debug, Clone)] +enum ConvertedTo { + Name(&'static str), + Pointer(String), +} + +#[derive(Debug, Clone, Copy)] +pub struct ImportOptions { + include_ical_components: bool, + return_first: bool, + blobs: BlobOptions, +} + +pub(super) struct ImportContext<'a, B> { + include_ical_components: bool, + return_first: bool, + blob_ids: Option>, + task_series: Vec<(String, convert::TaskAnchors)>, +} + +impl Default for ImportOptions { + fn default() -> Self { + Self { + include_ical_components: true, + return_first: false, + blobs: BlobOptions::default(), + } + } +} + +impl ImportOptions { + pub fn new() -> Self { + Self::default() + } +} + +impl ImportOptions { + pub fn include_ical_components(mut self, include: bool) -> Self { + self.include_ical_components = include; + self + } + + pub fn return_first(mut self, return_first: bool) -> Self { + self.return_first = return_first; + self + } + + pub fn with_blob_ids(self, blob_ids: F) -> ImportOptions> + where + F: FnMut(&[u8]) -> Option, + { + self.with_blob_id_generator(BlobIdFn(blob_ids)) + } + + pub fn with_blob_id_generator(self, blob_id_generator: T) -> ImportOptions { + ImportOptions { + include_ical_components: self.include_ical_components, + return_first: self.return_first, + blobs: self.blobs.with_handler(blob_id_generator), + } + } + + pub(super) fn context(&mut self) -> ImportContext<'_, B> + where + G: BlobIdGenerator, + { + ImportContext { + include_ical_components: self.include_ical_components, + return_first: self.return_first, + blob_ids: self.blobs.blob_ids(), + task_series: Vec::new(), + } + } +} diff --git a/crates/calcard/src/jscalendar/import/params.rs b/crates/calcard/src/jscalendar/import/params.rs new file mode 100644 index 00000000..c424ab3a --- /dev/null +++ b/crates/calcard/src/jscalendar/import/params.rs @@ -0,0 +1,482 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{Data, IanaString, IanaType, jsprop::text::IntoAsciiLowercase}, + icalendar::{ + ICalendarDisplayType, ICalendarEntry, ICalendarFeatureType, ICalendarParameter, + ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationRole, + ICalendarParticipationStatus, ICalendarProperty, ICalendarRelated, ICalendarUserTypes, + ICalendarValue, ICalendarValueType, Uri, + }, + jscalendar::{ + JSCalendarId, JSCalendarLinkDisplay, JSCalendarParticipantKind, JSCalendarParticipantRole, + JSCalendarParticipationStatus, JSCalendarProperty, JSCalendarRelativeTo, JSCalendarValue, + JSCalendarVirtualLocationFeature, + ext::JSCalendarObjectExt, + import::{ICalendarParams, ParamValues, PropertyMap}, + }, +}; +use jmap_tools::{Key, Map, Value}; +use std::{borrow::Cow, mem}; + +pub(super) trait ExtractParams { + fn extract_params( + &mut self, + entry: &mut ICalendarEntry, + extract: &[ICalendarParameterName], + ) -> Option; +} + +pub(super) trait ParamTarget { + fn set( + &mut self, + property: JSCalendarProperty, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ); + + fn set_flag( + &mut self, + property: JSCalendarProperty, + flag: Key<'static, JSCalendarProperty>, + ); +} + +impl ParamTarget for PropertyMap { + fn set( + &mut self, + property: JSCalendarProperty, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ) { + self.insert(Key::Property(property), value); + } + + fn set_flag( + &mut self, + property: JSCalendarProperty, + flag: Key<'static, JSCalendarProperty>, + ) { + if let Some(flags) = self + .entry(Key::Property(property)) + .or_insert_with(Value::new_object) + .as_object_mut() + { + flags.upsert(flag, Value::Bool(true)); + } + } +} + +impl ParamTarget + for Map<'static, JSCalendarProperty, JSCalendarValue> +{ + fn set( + &mut self, + property: JSCalendarProperty, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ) { + self.upsert(Key::Property(property), value); + } + + fn set_flag( + &mut self, + property: JSCalendarProperty, + flag: Key<'static, JSCalendarProperty>, + ) { + if let Some(flags) = self + .upsert_or_get_mut(Key::Property(property), Value::new_object) + .as_object_mut() + { + flags.upsert(flag, Value::Bool(true)); + } + } +} + +impl> ExtractParams for T { + fn extract_params( + &mut self, + entry: &mut ICalendarEntry, + extract: &[ICalendarParameterName], + ) -> Option { + let mut jsid = None; + let is_styled_description = matches!(entry.name, ICalendarProperty::StyledDescription); + let is_conference = matches!(entry.name, ICalendarProperty::Conference); + + entry.params.retain_mut(|param| { + if !extract.contains(¶m.name) { + return true; + } + + match param.name { + ICalendarParameterName::Cn => { + if let Some(text) = param.take_value().into_text() { + self.set(JSCalendarProperty::Name, Value::Str(text)); + } + } + ICalendarParameterName::Cutype => { + let kind = match param.take_value() { + ICalendarParameterValue::Text(value) => Value::Str(value.into()), + ICalendarParameterValue::Cutype(value) => match value { + ICalendarUserTypes::Individual => { + Value::Element(JSCalendarValue::ParticipantKind( + JSCalendarParticipantKind::Individual, + )) + } + ICalendarUserTypes::Group => Value::Element( + JSCalendarValue::ParticipantKind(JSCalendarParticipantKind::Group), + ), + ICalendarUserTypes::Resource => { + Value::Element(JSCalendarValue::ParticipantKind( + JSCalendarParticipantKind::Resource, + )) + } + ICalendarUserTypes::Room => { + Value::Element(JSCalendarValue::ParticipantKind( + JSCalendarParticipantKind::Location, + )) + } + ICalendarUserTypes::Unknown => { + Value::Str(value.as_str().to_lowercase().into()) + } + }, + _ => return false, + }; + self.set(JSCalendarProperty::Kind, kind); + } + ICalendarParameterName::DelegatedFrom => { + if let Some(text) = param.take_value().into_text() { + self.set_flag(JSCalendarProperty::DelegatedFrom, Key::from(text)); + } + } + ICalendarParameterName::DelegatedTo => { + if let Some(text) = param.take_value().into_text() { + self.set_flag(JSCalendarProperty::DelegatedTo, Key::from(text)); + } + } + ICalendarParameterName::Email => { + if let Some(text) = param.take_value().into_text() { + self.set(JSCalendarProperty::Email, Value::Str(text)); + } + } + ICalendarParameterName::Rsvp => { + if let Some(boolean) = param.value.as_bool() { + self.set(JSCalendarProperty::ExpectReply, Value::Bool(boolean)); + } + } + ICalendarParameterName::Member => { + if let Some(text) = param.take_value().into_text() { + self.set_flag(JSCalendarProperty::MemberOf, Key::from(text)); + } + } + ICalendarParameterName::Partstat => { + let status = match param.take_value() { + ICalendarParameterValue::Partstat(value) => { + Value::Element(JSCalendarValue::ParticipationStatus(match value { + ICalendarParticipationStatus::NeedsAction => { + JSCalendarParticipationStatus::NeedsAction + } + ICalendarParticipationStatus::Declined => { + JSCalendarParticipationStatus::Declined + } + ICalendarParticipationStatus::Tentative => { + JSCalendarParticipationStatus::Tentative + } + ICalendarParticipationStatus::Delegated => { + JSCalendarParticipationStatus::Delegated + } + ICalendarParticipationStatus::Completed + | ICalendarParticipationStatus::InProcess + | ICalendarParticipationStatus::Failed + | ICalendarParticipationStatus::Accepted => { + JSCalendarParticipationStatus::Accepted + } + })) + } + ICalendarParameterValue::Text(value) => Value::Str(value.into()), + _ => return false, + }; + self.set(JSCalendarProperty::ParticipationStatus, status); + } + ICalendarParameterName::Role => { + let role = match param.take_value() { + ICalendarParameterValue::Role(value) => { + Key::Property(JSCalendarProperty::ParticipantRole(match value { + ICalendarParticipationRole::Chair => { + JSCalendarParticipantRole::Chair + } + ICalendarParticipationRole::ReqParticipant => { + JSCalendarParticipantRole::Required + } + ICalendarParticipationRole::OptParticipant => { + JSCalendarParticipantRole::Optional + } + ICalendarParticipationRole::NonParticipant => { + JSCalendarParticipantRole::Informational + } + ICalendarParticipationRole::Owner => { + JSCalendarParticipantRole::Owner + } + })) + } + ICalendarParameterValue::Text(value) => Key::Owned(value), + _ => return false, + }; + self.set_flag(JSCalendarProperty::Roles, role); + } + ICalendarParameterName::SentBy => { + if let Some(text) = param.take_value().into_text() { + self.set(JSCalendarProperty::SentBy, Value::Str(text)); + } + } + ICalendarParameterName::Fmttype => { + if let Some(text) = param.take_value().into_text() { + self.set( + if is_styled_description { + JSCalendarProperty::DescriptionContentType + } else { + JSCalendarProperty::ContentType + }, + Value::Str(text), + ); + } + } + ICalendarParameterName::Label | ICalendarParameterName::Filename => { + if let Some(text) = param.take_value().into_text() { + self.set( + if is_conference { + JSCalendarProperty::Name + } else { + JSCalendarProperty::Title + }, + Value::Str(text), + ); + } + } + ICalendarParameterName::Size => match param.value.as_integer() { + Some(number) => { + self.set(JSCalendarProperty::Size, Value::Number(number.into())); + } + None => return true, + }, + ICalendarParameterName::Linkrel => match param.take_value() { + ICalendarParameterValue::Linkrel(linkrel) => { + self.set( + JSCalendarProperty::Rel, + Value::Element(JSCalendarValue::LinkRelation(linkrel)), + ); + } + ICalendarParameterValue::Text(value) => { + self.set(JSCalendarProperty::Rel, Value::Str(value.into())); + } + value => { + param.value = value; + return true; + } + }, + ICalendarParameterName::Related => match param.take_value() { + ICalendarParameterValue::Related(related) => { + self.set( + JSCalendarProperty::RelativeTo, + Value::Element(JSCalendarValue::RelativeTo(match related { + ICalendarRelated::Start => JSCalendarRelativeTo::Start, + ICalendarRelated::End => JSCalendarRelativeTo::End, + })), + ); + } + value => { + param.value = value; + return true; + } + }, + ICalendarParameterName::Feature => { + let feature = match param.take_value() { + ICalendarParameterValue::Feature(value) => { + Key::Property(JSCalendarProperty::VirtualLocationFeature(match value { + ICalendarFeatureType::Audio => { + JSCalendarVirtualLocationFeature::Audio + } + ICalendarFeatureType::Chat => { + JSCalendarVirtualLocationFeature::Chat + } + ICalendarFeatureType::Feed => { + JSCalendarVirtualLocationFeature::Feed + } + ICalendarFeatureType::Moderator => { + JSCalendarVirtualLocationFeature::Moderator + } + ICalendarFeatureType::Phone => { + JSCalendarVirtualLocationFeature::Phone + } + ICalendarFeatureType::Screen => { + JSCalendarVirtualLocationFeature::Screen + } + ICalendarFeatureType::Video => { + JSCalendarVirtualLocationFeature::Video + } + })) + } + ICalendarParameterValue::Text(value) => Key::Owned(value), + _ => return false, + }; + self.set_flag(JSCalendarProperty::Features, feature); + } + ICalendarParameterName::Language => { + if let Some(text) = param.take_value().into_text() { + self.set(JSCalendarProperty::Locale, Value::Str(text)); + } + } + ICalendarParameterName::Display => { + let display = match param.take_value() { + ICalendarParameterValue::Display(value) => { + Key::Property(JSCalendarProperty::LinkDisplay(match value { + ICalendarDisplayType::Badge => JSCalendarLinkDisplay::Badge, + ICalendarDisplayType::Graphic => JSCalendarLinkDisplay::Graphic, + ICalendarDisplayType::Fullsize => JSCalendarLinkDisplay::Fullsize, + ICalendarDisplayType::Thumbnail => JSCalendarLinkDisplay::Thumbnail, + })) + } + ICalendarParameterValue::Text(value) => Key::Owned(value), + value => { + param.value = value; + return true; + } + }; + self.set_flag(JSCalendarProperty::Display, display); + } + ICalendarParameterName::Jsid => { + jsid = param.take_value().into_text().map(Cow::into_owned); + } + ICalendarParameterName::Range + | ICalendarParameterName::Reltype + | ICalendarParameterName::Dir + | ICalendarParameterName::Fbtype + | ICalendarParameterName::ScheduleAgent + | ICalendarParameterName::ScheduleForceSend + | ICalendarParameterName::ScheduleStatus + | ICalendarParameterName::Tzid + | ICalendarParameterName::Value + | ICalendarParameterName::ManagedId + | ICalendarParameterName::Order + | ICalendarParameterName::Schema + | ICalendarParameterName::Derived + | ICalendarParameterName::Gap + | ICalendarParameterName::Jsptr + | ICalendarParameterName::Other(_) + | ICalendarParameterName::Altrep => {} + } + false + }); + + jsid + } +} + +impl ICalendarParameter { + fn take_value(&mut self) -> ICalendarParameterValue { + mem::replace(&mut self.value, ICalendarParameterValue::Null) + } +} + +impl ICalendarParams { + pub(super) fn push( + &mut self, + name: ICalendarParameterName, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ) { + match self.0.iter_mut().find(|(param, _)| param == &name) { + Some((_, values)) => values.push(value), + None => self.0.push((name, ParamValues::One(value))), + } + } + + pub(super) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(super) fn into_jscalendar_value( + self, + ) -> Option, JSCalendarValue>> { + (!self.0.is_empty()).then(|| { + self.0 + .into_iter() + .map(|(param, values)| { + let value = match values { + ParamValues::One(value) => value, + ParamValues::Many(values) => Value::Array(values), + }; + ( + Key::Owned(param.into_string().into_ascii_lowercase()), + value, + ) + }) + .collect() + }) + } +} + +impl ParamValues { + fn push(&mut self, value: Value<'static, JSCalendarProperty, JSCalendarValue>) { + match self { + ParamValues::Many(values) => values.push(value), + ParamValues::One(first) => { + let first = mem::replace(first, Value::Null); + *self = ParamValues::Many(vec![first, value]); + } + } + } +} + +impl ICalendarValue { + pub(super) fn into_jscalendar_value( + self, + value_type: Option<&IanaType>, + ) -> Value<'static, JSCalendarProperty, JSCalendarValue> { + match self { + ICalendarValue::Text(v) => Value::Str(v.into()), + ICalendarValue::Integer(v) => Value::Number(v.into()), + ICalendarValue::Float(v) => Value::Number(v.into()), + ICalendarValue::Boolean(v) => Value::Bool(v), + ICalendarValue::PartialDateTime(v) => { + let mut out = String::new(); + let _ = v.format_as_ical( + &mut out, + value_type.and_then(|v| v.iana()).unwrap_or( + match (v.has_date(), v.has_time(), v.has_zone()) { + (true, true, _) => &ICalendarValueType::DateTime, + (true, _, _) => &ICalendarValueType::Date, + (_, true, _) => &ICalendarValueType::Time, + (_, _, true) => &ICalendarValueType::UtcOffset, + _ => &ICalendarValueType::Text, + }, + ), + ); + Value::Str(out.into()) + } + ICalendarValue::Binary(v) => Value::Str( + Uri::Data(Box::new(Data { + content_type: None, + data: v, + })) + .into_unwrapped_string() + .into(), + ), + ICalendarValue::Uri(v) => Value::Str(v.into_unwrapped_string().into()), + ICalendarValue::Duration(v) => Value::Str(v.to_string().into()), + ICalendarValue::RecurrenceRule(v) => Value::Str(v.to_string().into()), + ICalendarValue::Period(v) => Value::Str(v.to_string().into()), + ICalendarValue::CalendarScale(v) => Value::Str(v.as_str().into()), + ICalendarValue::Method(v) => Value::Str(v.as_str().into()), + ICalendarValue::Classification(v) => Value::Str(v.as_str().into()), + ICalendarValue::Status(v) => Value::Str(v.as_str().into()), + ICalendarValue::Transparency(v) => Value::Str(v.as_str().into()), + ICalendarValue::Action(v) => Value::Str(v.as_str().into()), + ICalendarValue::BusyType(v) => Value::Str(v.as_str().into()), + ICalendarValue::ParticipantType(v) => Value::Str(v.as_str().into()), + ICalendarValue::ResourceType(v) => Value::Str(v.as_str().into()), + ICalendarValue::Proximity(v) => Value::Str(v.as_str().into()), + } + } +} + diff --git a/crates/calcard/src/jscalendar/import/props.rs b/crates/calcard/src/jscalendar/import/props.rs new file mode 100644 index 00000000..9acbf27a --- /dev/null +++ b/crates/calcard/src/jscalendar/import/props.rs @@ -0,0 +1,917 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + Data, IanaString, IanaType, LinkRelation, + blob::GeneratedBlobId, + jsprop::text::{IntoAsciiLowercase, PointerString}, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::{ + ICalendar, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, ICalendarValue, + ICalendarValueType, Uri, + }, + jscalendar::{ + JSCALENDAR_VERSION, JSCalendarDateTime, JSCalendarId, JSCalendarPrivacy, + JSCalendarProperty, JSCalendarType, JSCalendarValue, + ext::{JSCalendarKeyExt, JSCalendarObjectExt, JSCalendarValueExt}, + import::{ + ConvertedTo, EntryState, ICalendarConvertedProperty, ICalendarParams, LinkId, LinkIds, + State, params::ExtractParams, + }, + overrides::{Inherited, OverrideDiff}, + uuid5, + }, +}; +use ahash::AHashMap; +use jmap_tools::{JsonPointerItem, Key, Map, Property, Value}; +use std::{borrow::Cow, collections::hash_map::Entry, mem}; + +impl State { + pub(super) fn map_named_entry( + &mut self, + entry: &mut EntryState, + extract: &[ICalendarParameterName], + top_property_name: JSCalendarProperty, + values: impl IntoIterator< + Item = ( + Key<'static, JSCalendarProperty>, + Value<'static, JSCalendarProperty, JSCalendarValue>, + ), + >, + ) { + self.map_named_entry_with_id(entry, extract, top_property_name, values, None) + } + + pub(super) fn map_named_entry_with_id( + &mut self, + entry: &mut EntryState, + extract: &[ICalendarParameterName], + top_property_name: JSCalendarProperty, + values: impl IntoIterator< + Item = ( + Key<'static, JSCalendarProperty>, + Value<'static, JSCalendarProperty, JSCalendarValue>, + ), + >, + default_id: Option, + ) { + // Obtain main property and value + let mut values = values.into_iter().peekable(); + let (property, value) = match values.peek() { + Some((property, Value::Str(s))) => (property, s.as_ref()), + Some((property, _)) => (property, "unknown"), + _ => { + panic!("Cannot generate jsid without a value"); + } + }; + + let mut parameters = mem::take(&mut self.parameters); + let is_link = top_property_name == JSCalendarProperty::Links; + let js_id = match parameters.extract_params(&mut entry.entry, extract) { + Some(js_id) => js_id, + None => { + let js_id = default_id.unwrap_or_else(|| uuid5(value)); + if is_link { + self.link_ids.unique(js_id) + } else { + js_id + } + } + }; + + entry.set_converted_to(|| { + String::from_pointer([ + top_property_name.to_cow().as_ref(), + js_id.as_str(), + property.to_string().as_ref(), + ]) + }); + + let capacity = values.size_hint().1.unwrap_or_default() + parameters.len(); + let new_object = || Value::Object(Map::from(Vec::with_capacity(capacity))); + if let Some(obj) = self + .entries + .entry(Key::Property(top_property_name)) + .or_insert_with(Value::new_object) + .as_object_mut() + .and_then(|objects| { + if is_link { + self.link_ids.entry(objects, js_id, new_object) + } else { + Some(objects.upsert_or_get_mut(Key::Owned(js_id), new_object)) + } + }) + .and_then(Value::as_object_mut) + { + for (key, value) in values.chain(parameters.as_mut_vec().drain(..)) { + if let Some(current_value) = obj.lookup_mut(&key) { + match (value, current_value) { + (Value::Object(new_obj), Value::Object(existing_obj)) => { + for (key, value) in new_obj.into_vec() { + existing_obj.upsert(key, value); + } + } + (value, current_value) => { + *current_value = value; + } + } + } else { + obj.insert_unchecked(key, value); + } + } + } + parameters.as_mut_vec().clear(); + self.parameters = parameters; + } + + pub(super) fn map_blob_link( + &mut self, + entry: &mut EntryState, + generated: GeneratedBlobId, + content_type: Option, + default_id: Option, + value_type: ICalendarValueType, + ) { + let (rel, extract): (_, &[ICalendarParameterName]) = match (&entry.entry.name, value_type) { + (ICalendarProperty::Image, ICalendarValueType::Binary) => ( + LinkRelation::Icon, + &[ + ICalendarParameterName::Display, + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Linkrel, + ICalendarParameterName::Jsid, + ], + ), + (ICalendarProperty::Image, _) => ( + LinkRelation::Icon, + &[ + ICalendarParameterName::Display, + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Jsid, + ], + ), + (_, ICalendarValueType::Binary) => ( + LinkRelation::Enclosure, + &[ + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Linkrel, + ICalendarParameterName::Jsid, + ], + ), + _ => ( + LinkRelation::Enclosure, + &[ + ICalendarParameterName::Fmttype, + ICalendarParameterName::Filename, + ICalendarParameterName::Jsid, + ], + ), + }; + entry.entry.params.retain(|param| { + !matches!( + param.name, + ICalendarParameterName::Value | ICalendarParameterName::Size + ) + }); + entry.set_map_name(); + + self.map_named_entry_with_id( + entry, + extract, + JSCalendarProperty::Links, + [ + Some(( + Key::Property(JSCalendarProperty::BlobId), + Value::Element(JSCalendarValue::BlobId(generated.blob_id)), + )), + Some(( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Link)), + )), + Some(( + Key::Property(JSCalendarProperty::Rel), + Value::Element(JSCalendarValue::LinkRelation(rel)), + )), + Some(( + Key::Property(JSCalendarProperty::Size), + Value::Number((generated.size as u64).into()), + )), + content_type.map(|content_type| { + ( + Key::Property(JSCalendarProperty::ContentType), + Value::Str(content_type.into()), + ) + }), + ] + .into_iter() + .flatten(), + default_id, + ); + } + + pub(super) fn insert_recurrence_override( + &mut self, + key: Key<'static, JSCalendarProperty>, + value: Value<'static, JSCalendarProperty, JSCalendarValue>, + ) { + self.entries + .entry(Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .or_insert_with(Value::new_object) + .as_object_mut() + .unwrap() + .upsert(key, value); + } + + pub(super) fn add_period_conversion_prop(&mut self, converted_to: String) { + if self.include_ical_components { + let mut params = ICalendarParams::default(); + params.push( + ICalendarParameterName::Value, + Value::Str(ICalendarValueType::Period.as_str().into()), + ); + self.ical_converted_properties.insert_if_absent( + converted_to, + ICalendarConvertedProperty { + name: Some(ICalendarProperty::Rdate), + params, + }, + ); + } + } + + pub(super) fn add_conversion_props(&mut self, mut entry: EntryState) { + if self.include_ical_components { + if let Some(converted_to) = entry.converted_to.take() { + if entry.map_name || !entry.entry.params.is_empty() { + let mut value_type = None; + + let converted_to = converted_to.into_string(); + match self.ical_converted_properties.get_mut(&converted_to) { + Some(conv_prop) => { + entry.jcal_parameters(&mut conv_prop.params, &mut value_type); + } + None => { + let mut params = ICalendarParams::default(); + entry.jcal_parameters(&mut params, &mut value_type); + if let Some(value_type) = value_type { + params.push( + ICalendarParameterName::Value, + Value::Str(value_type.into_string()), + ); + } + if !params.is_empty() || entry.map_name { + self.ical_converted_properties.push( + converted_to, + ICalendarConvertedProperty { + name: if entry.map_name { + Some(entry.entry.name) + } else { + None + }, + params, + }, + ); + } + } + } + } + } else { + self.ical_properties.push(entry.into_jcal()); + } + } + } + + pub(super) fn into_object( + mut self, + ) -> Value<'static, JSCalendarProperty, JSCalendarValue> { + let mut ical_obj = Map::from(Vec::new()); + if !self.ical_converted_properties.is_empty() { + let mut converted_properties = + Map::from(Vec::with_capacity(self.ical_converted_properties.len())); + + for (converted_to, props) in self.ical_converted_properties { + let mut obj = Map::from(Vec::with_capacity(2)); + if let Some(params) = props.params.into_jscalendar_value() { + obj.upsert( + Key::Property(JSCalendarProperty::Parameters), + Value::Object(params), + ); + } + if let Some(name) = props.name { + obj.upsert( + Key::Property(JSCalendarProperty::Name), + Value::Str(name.into_string().into_ascii_lowercase().into()), + ); + } + + converted_properties.insert_unchecked(Key::Owned(converted_to), Value::Object(obj)); + } + + ical_obj.insert_unchecked( + Key::Property(JSCalendarProperty::ConvertedProperties), + Value::Object(converted_properties), + ); + } + + if !self.ical_properties.is_empty() { + ical_obj.insert_unchecked( + Key::Property(JSCalendarProperty::Properties), + Value::Array(self.ical_properties), + ); + } + + if let Some(components) = self.ical_components { + ical_obj.insert_unchecked(Key::Property(JSCalendarProperty::Components), components); + } + + if !ical_obj.is_empty() || self.map_component { + ical_obj.insert_unchecked( + Key::Property(JSCalendarProperty::Name), + Value::Str(self.component_type.as_str().to_ascii_lowercase().into()), + ); + self.entries.insert( + Key::Property(JSCalendarProperty::ICalendar), + Value::Object(ical_obj), + ); + } + + if self.has_dates { + /* + If a date-time value does not have a timezone, then the timezone is not set + in JSCalendar. + */ + + if let Some(tz) = self.tz_start.and_then(|tz| tz.name()) { + self.entries + .insert(Key::Property(JSCalendarProperty::TimeZone), Value::Str(tz)); + } + + if self.tz_end.is_some() + && self.tz_start.is_some() + && self.tz_end != self.tz_start + && let Some(tz) = self.tz_end.and_then(|tz| tz.name()) + { + self.entries.insert( + Key::Property(JSCalendarProperty::EndTimeZone), + Value::Str(tz), + ); + } + + if let Some(recurrence_id) = self.recurrence_id { + self.entries.insert( + Key::Property(JSCalendarProperty::RecurrenceId), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + recurrence_id.naive_timestamp(), + true, + ))), + ); + if !self.is_recurrence_instance + && let Some(tz) = recurrence_id + .timezone() + .to_resolved() + .and_then(|tz| tz.name()) + { + self.entries.insert( + Key::Property(JSCalendarProperty::RecurrenceIdTimeZone), + Value::Str(tz), + ); + } + } + } + + if let Some(uid) = self.uid { + self.entries.insert( + Key::Property(JSCalendarProperty::Uid), + Value::Str(uid.into()), + ); + } + + if !self.is_recurrence_instance + && let Some(component_type) = self.component_type.to_jscalendar_type() + { + self.entries.insert( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(component_type)), + ); + } + + let mut obj = Value::Object(self.entries.into_map()); + let (override_patches, patches): (Vec<_>, Vec<_>) = + self.patch_objects.into_iter().partition(|(pointer, _)| { + !self.recurrence_overrides.is_empty() + && matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceOverrides + ))) + ) + }); + for (pointer, patch) in patches { + obj.apply_jsprop(pointer.as_slice(), patch); + } + + if !self.recurrence_overrides.is_empty() + && let Value::Object(base) = &mut obj + { + let overrides_key = Key::Property(JSCalendarProperty::RecurrenceOverrides); + let mut overrides = match base.key_position(&overrides_key) { + Some(pos) => base.as_mut_vec().remove(pos).1, + None => Value::new_object(), + }; + if let Value::Object(overrides) = &mut overrides { + let diff = OverrideDiff::new(base); + let duration_key = Key::Property(JSCalendarProperty::Duration); + let mut slots = overrides + .as_vec() + .iter() + .enumerate() + .filter_map(|(position, (key, value))| match key { + Key::Property(JSCalendarProperty::DateTime(recurrence_id)) => Some(( + *recurrence_id, + ( + position, + value + .as_object() + .and_then(|period| period.lookup(&duration_key)) + .cloned(), + ), + )), + _ => None, + }) + .collect::>(); + for (recurrence_id, recurrence) in self.recurrence_overrides { + let inherited = recurrence.inherited(); + let instance = recurrence.into_object().into_object().unwrap_or_default(); + let mut patch = diff.diff(recurrence_id, instance, inherited); + match slots.entry(recurrence_id) { + Entry::Occupied(slot) => { + let (position, period_duration) = slot.get(); + if inherited == Inherited::StartAndEnd + && let Some(duration) = period_duration + { + patch.upsert(duration_key.clone(), duration.clone()); + } + if let Some((_, value)) = overrides.as_mut_vec().get_mut(*position) { + *value = Value::Object(patch); + } + } + Entry::Vacant(slot) => { + slot.insert((overrides.as_vec().len(), None)); + overrides.insert_unchecked( + Key::Property(JSCalendarProperty::DateTime(recurrence_id)), + Value::Object(patch), + ); + } + } + } + } + base.insert_unchecked(overrides_key, overrides); + } + + for (pointer, patch) in override_patches { + obj.apply_jsprop(pointer.as_slice(), patch); + } + + obj + } + + pub(super) fn into_root_object( + self, + ) -> Value<'static, JSCalendarProperty, JSCalendarValue> { + let has_version = matches!( + self.component_type, + ICalendarComponentType::VCalendar + | ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + ); + let mut obj = self.into_object(); + if has_version && let Value::Object(map) = &mut obj { + map.upsert( + Key::Property(JSCalendarProperty::Version), + Value::Str(JSCALENDAR_VERSION.into()), + ); + } + obj + } + + pub(super) fn set_map_component(&mut self) { + self.map_component = true; + } + + pub(super) fn set_is_recurrence_instance(&mut self) { + self.is_recurrence_instance = true; + } + + fn inherited(&self) -> Inherited { + if self + .entries + .contains_key(&Key::Property(JSCalendarProperty::Start)) + { + Inherited::Nothing + } else if self.has_end { + Inherited::Start + } else { + Inherited::StartAndEnd + } + } + + pub(super) fn has_start_or_due(&self) -> bool { + self.entries + .contains_key(&Key::Property(JSCalendarProperty::Start)) + || self + .entries + .contains_key(&Key::Property(JSCalendarProperty::Due)) + } + + pub(super) fn inherit_time_zone(&mut self, tz: Option) { + if self + .entries + .contains_key(&Key::Property(JSCalendarProperty::Start)) + || (tz.is_none() && self.due.is_some()) + { + return; + } + if let Some(due) = self.due { + self.entries.insert( + Key::Property(JSCalendarProperty::Due), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::local_in( + due, tz, + ))), + ); + } + self.tz_start = tz; + } + + pub(super) fn start_at_recurrence_id(&mut self) { + if let Some(recurrence_id) = self.recurrence_id + && !self.has_start_or_due() + { + self.entries.insert( + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + recurrence_id.naive_timestamp(), + true, + ))), + ); + if self.recurrence_id_is_date { + self.default_to_one_day(); + } + } + } + + pub(super) fn default_to_one_day(&mut self) { + if self.has_end || self.component_type != ICalendarComponentType::VEvent { + return; + } + self.entries.insert( + Key::Property(JSCalendarProperty::Duration), + Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_days(1))), + ); + if self.include_ical_components { + self.ical_converted_properties.insert( + JSCalendarProperty::Duration::.to_string().into_owned(), + ICalendarConvertedProperty { + name: Some(ICalendarProperty::Dtstart), + ..Default::default() + }, + ); + } + } + + pub(super) fn recurrence_sequence(&self) -> Option { + self.recurrence_id.map(|_| { + self.entries + .get(&Key::Property(JSCalendarProperty::Sequence)) + .and_then(Value::as_i64) + .unwrap_or_default() + }) + } + + pub(super) fn privacy(&self) -> Option { + match self + .entries + .get(&Key::Property(JSCalendarProperty::Privacy))? + { + Value::Element(JSCalendarValue::Privacy(privacy)) => Some(*privacy), + _ => Some(JSCalendarPrivacy::Private), + } + } + + pub(super) fn raise_privacy(&mut self, privacy: JSCalendarPrivacy) { + if self.privacy().unwrap_or(JSCalendarPrivacy::Public) < privacy { + self.entries.insert( + Key::Property(JSCalendarProperty::Privacy), + Value::Element(JSCalendarValue::Privacy(privacy)), + ); + self.patch_objects.retain(|(pointer, _)| { + !matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::Privacy + ))) + ) + }); + } + } + + pub(super) fn merge_privacy(&mut self, privacy: JSCalendarPrivacy) { + let privacy = self + .privacy() + .map_or(privacy, |current| current.max(privacy)); + self.entries.insert( + Key::Property(JSCalendarProperty::Privacy), + Value::Element(JSCalendarValue::Privacy(privacy)), + ); + } + + pub(super) fn remove_forbidden_override_patches(&mut self) { + self.entries.retain(|key, _| { + !matches!(key, Key::Property(property) if property.is_forbidden_override_patch()) + }); + self.patch_objects + .retain(|(pointer, _)| !JSCalendarProperty::is_forbidden_override_pointer(pointer)); + self.ical_converted_properties.retain(|converted_to, _| { + !Key::>::Borrowed(converted_to).is_series_converted_property() + }); + } + + pub(super) fn find_participant_by_address(&self, address: &str) -> Option { + self.entries + .get(&Key::Property(JSCalendarProperty::Participants))? + .as_object()? + .iter() + .find_map(|(key, value)| { + match value + .as_object()? + .lookup(&Key::Property(JSCalendarProperty::CalendarAddress))? + { + Value::Str(value) if value == address => Some(key.to_string().into_owned()), + _ => None, + } + }) + } + + #[inline] + pub(super) fn get_mut_object_or_insert( + &mut self, + key: JSCalendarProperty, + ) -> &mut Map<'static, JSCalendarProperty, JSCalendarValue> { + self.entries + .entry(Key::Property(key)) + .or_insert_with(|| Value::Object(Map::from(Vec::new()))) + .as_object_mut() + .unwrap() + } +} + +impl JSCalendarDateTime { + pub(super) fn local_in(dt: ZonedDateTime, tz: Option) -> Self { + JSCalendarDateTime::new( + if dt.timezone().is_floating() || tz == Some(dt.timezone()) { + dt.naive_timestamp() + } else { + dt.with_timezone(tz.unwrap_or_default()).naive_timestamp() + }, + true, + ) + } +} + +impl EntryState { + pub(super) fn new(entry: ICalendarEntry, keep_converted_path: bool) -> Self { + Self { + entry, + converted_to: None, + map_name: false, + keep_converted_path, + } + } + + pub(super) fn set_converted_to(&mut self, converted_to: impl FnOnce() -> String) { + self.converted_to = Some(ConvertedTo::Pointer( + if self.keep_converted_path && (self.map_name || !self.entry.params.is_empty()) { + converted_to() + } else { + String::new() + }, + )); + } + + pub(super) fn set_converted_to_property( + &mut self, + property: &JSCalendarProperty, + ) { + self.converted_to = Some(match property.static_name() { + Some(name) => ConvertedTo::Name(name), + None => ConvertedTo::Pointer(String::from_pointer([property.to_string().as_ref()])), + }); + } + + pub(super) fn set_map_name(&mut self) { + self.map_name = true; + } + + pub(super) fn into_jcal( + mut self, + ) -> Value<'static, JSCalendarProperty, JSCalendarValue> { + let mut value_type = None; + let mut params = ICalendarParams::default(); + + self.jcal_parameters(&mut params, &mut value_type); + + let values = if self.entry.values.len() == 1 { + self.entry + .values + .into_iter() + .next() + .unwrap() + .into_jscalendar_value(value_type.as_ref()) + } else { + let mut values = Vec::with_capacity(self.entry.values.len()); + for value in self.entry.values { + values.push(value.into_jscalendar_value(value_type.as_ref())); + } + Value::Array(values) + }; + Value::Array(vec![ + Value::Str(self.entry.name.into_string().into_ascii_lowercase().into()), + Value::Object( + params + .into_jscalendar_value() + .unwrap_or(Map::from(Vec::new())), + ), + Value::Str( + value_type + .map(|v| v.into_string()) + .unwrap_or(Cow::Borrowed("unknown")), + ), + values, + ]) + } + + pub(super) fn jcal_parameters( + &mut self, + params: &mut ICalendarParams, + value_type: &mut Option>, + ) { + if self.entry.params.is_empty() { + return; + } + let (default_type, _) = self.entry.name.default_types(); + let default_type = default_type.unwrap_ical(); + + for param in std::mem::take(&mut self.entry.params) { + if matches!(param.name, ICalendarParameterName::Value) { + if let Some(v) = param + .value + .into_value_type() + .filter(|v| !v.is_iana_and(|v| v == &default_type)) + { + *value_type = Some(v); + } + } else if let (ICalendarParameterName::Range, ICalendarParameterValue::Bool(true)) = + (¶m.name, ¶m.value) + { + params.push(param.name, Value::Str("THISANDFUTURE".into())); + } else if let Some(value) = param.value.into_text() { + params.push(param.name, Value::Str(value)); + } + } + } +} + +impl ConvertedTo { + fn into_string(self) -> String { + match self { + ConvertedTo::Name(name) => String::from_pointer([name]), + ConvertedTo::Pointer(pointer) => pointer, + } + } +} + +impl LinkIds { + fn unique(&mut self, base: String) -> String { + let Some(mut suffix) = self.0.get(&base).map(|link| link.next_suffix) else { + return base; + }; + let unique = loop { + let candidate = format!("{base}-{suffix}"); + suffix = suffix.saturating_add(1); + if !self.0.contains_key(&candidate) { + break candidate; + } + }; + if let Some(link) = self.0.get_mut(&base) { + link.next_suffix = suffix; + } + unique + } + + fn entry<'x, I: JSCalendarId, B: JSCalendarId>( + &mut self, + links: &'x mut Map<'static, JSCalendarProperty, JSCalendarValue>, + key: String, + value: impl FnOnce() -> Value<'static, JSCalendarProperty, JSCalendarValue>, + ) -> Option<&'x mut Value<'static, JSCalendarProperty, JSCalendarValue>> { + let position = match self.0.get(&key) { + Some(link) => link.position, + None => { + let position = links.len(); + links.insert_unchecked(Key::Owned(key.clone()), value()); + self.0.insert( + key, + LinkId { + position, + next_suffix: 2, + }, + ); + position + } + }; + links.as_mut_vec().get_mut(position).map(|(_, value)| value) + } +} + +impl ICalendar { + pub(super) fn binary_link_sizes(&self) -> impl Iterator + '_ { + self.blob_binaries().map(<[u8]>::len) + } +} + +impl ICalendarEntry { + pub fn participant_id(&self) -> Option> { + if let Some(jsid) = self.jsid() { + return Some(Cow::Borrowed(jsid)); + } + match self.values.first()? { + ICalendarValue::Text(address) => Some(Cow::Owned(uuid5(address))), + ICalendarValue::Uri(Uri::Location(address)) => Some(Cow::Owned(uuid5(address))), + ICalendarValue::Uri(uri) => Some(Cow::Owned(uuid5(uri.to_unwrapped_string()))), + _ => None, + } + } +} + +pub(super) struct ICalendarBinary { + pub(super) data: Vec, + pub(super) content_type: Option, + pub(super) is_data_uri: bool, +} + +impl ICalendarBinary { + pub(super) fn into_value(self) -> ICalendarValue { + if self.is_data_uri { + ICalendarValue::Uri(Uri::Data(Box::new(Data { + content_type: self.content_type, + data: self.data, + }))) + } else { + ICalendarValue::Binary(self.data) + } + } +} + +impl ICalendarValue { + pub(super) fn is_binary(&self) -> bool { + self.binary_bytes().is_some() + } + + pub(super) fn into_binary(self) -> Option { + match self { + ICalendarValue::Binary(data) => Some(ICalendarBinary { + data, + content_type: None, + is_data_uri: false, + }), + ICalendarValue::Uri(Uri::Data(data)) => Some(ICalendarBinary { + data: data.data, + content_type: data.content_type, + is_data_uri: true, + }), + _ => None, + } + } + + pub(super) fn uri_to_string(self, media_type: Option) -> Self { + match self { + ICalendarValue::Uri(uri) => ICalendarValue::Text(uri.into_unwrapped_string()), + ICalendarValue::Binary(data) => ICalendarValue::Text( + Uri::Data(Box::new(Data { + content_type: media_type, + data, + })) + .into_unwrapped_string(), + ), + other => other, + } + } +} diff --git a/crates/calcard/src/jscalendar/mod.rs b/crates/calcard/src/jscalendar/mod.rs new file mode 100644 index 00000000..7fc0e681 --- /dev/null +++ b/crates/calcard/src/jscalendar/mod.rs @@ -0,0 +1,508 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod export; +pub mod ext; +pub mod import; +#[doc(hidden)] +pub mod overrides; +pub mod parser; +pub mod types; + +use crate::{ + common::{ + CalendarScale, IanaString, LinkRelation, elements::Elements, jsprop::text::AsciiString, + }, + icalendar::{ + ICalendarComponentType, ICalendarDuration, ICalendarFrequency, ICalendarMethod, + ICalendarMonth, ICalendarSkip, ICalendarWeekday, + }, +}; +use jmap_tools::{JsonPointer, Key, Map, Value}; +use serde::Serialize; +use sha1::{Digest, Sha1}; +use std::{borrow::Cow, fmt::Debug, fmt::Display, hash::Hash, str::FromStr}; +use uuid::{Builder, fmt::Hyphenated}; + +pub const JSCALENDAR_VERSION: &str = "2.0"; + +pub(crate) const MAX_ICAL_COMPONENT_DEPTH: usize = 32; +const RFC3339_CAPACITY: usize = 21; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +#[repr(transparent)] +pub struct JSCalendar<'x, I: JSCalendarId, B: JSCalendarId>( + pub Value<'x, JSCalendarProperty, JSCalendarValue>, +); + +pub trait JSCalendarId: + FromStr + Sized + Serialize + Display + Clone + Eq + Hash + Ord + Debug + Default +{ +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarValue { + Type(JSCalendarType), + DateTime(JSCalendarDateTime), + Duration(ICalendarDuration), + AlertAction(JSCalendarAlertAction), + FreeBusyStatus(JSCalendarFreeBusyStatus), + ParticipantKind(JSCalendarParticipantKind), + ParticipationStatus(JSCalendarParticipationStatus), + Privacy(JSCalendarPrivacy), + Progress(JSCalendarProgress), + RelativeTo(JSCalendarRelativeTo), + ScheduleAgent(JSCalendarScheduleAgent), + EventStatus(JSCalendarEventStatus), + LinkRelation(LinkRelation), + Frequency(ICalendarFrequency), + CalendarScale(CalendarScale), + Skip(ICalendarSkip), + Weekday(ICalendarWeekday), + Month(ICalendarMonth), + Method(ICalendarMethod), + Id(I), + BlobId(B), + IdReference(String), +} + +impl Default for JSCalendarValue { + fn default() -> Self { + JSCalendarValue::Type(JSCalendarType::default()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct JSCalendarDateTime { + pub timestamp: i64, + pub is_local: bool, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarProperty { + // JMAP for Calendar Properties + Id, + BaseEventId, + CalendarIds, + IsDraft, + IsOrigin, + UtcStart, + UtcEnd, + UseDefaultAlerts, + MayInviteSelf, + MayInviteOthers, + HideAttendees, + BlobId, + Metadata, + PrivateMetadata, + + // JSCalendar Properties + #[default] + Type, + Acknowledged, + Action, + Alerts, + ByDay, + ByHour, + ByMinute, + ByMonth, + ByMonthDay, + BySecond, + BySetPosition, + ByWeekNo, + ByYearDay, + CalendarAddress, + Categories, + Color, + ContentType, + Coordinates, + Count, + Created, + Day, + DelegatedFrom, + DelegatedTo, + Description, + DescriptionContentType, + Display, + Due, + Duration, + Email, + Entries, + EstimatedDuration, + Excluded, + ExpectReply, + Features, + FirstDayOfWeek, + FreeBusyStatus, + Frequency, + Href, + Interval, + InvitedBy, + Keywords, + Kind, + Links, + Locale, + Locations, + LocationTypes, + MemberOf, + Method, + Name, + NthOfPeriod, + Offset, + Participants, + ParticipationComment, + ParticipationStatus, + PercentComplete, + Priority, + Privacy, + ProdId, + Progress, + RecurrenceId, + RecurrenceIdTimeZone, + RecurrenceOverrides, + Rel, + RelatedTo, + Relation, + RelativeTo, + ReplyTo, + RequestStatus, + Roles, + Rscale, + SentBy, + ScheduleAgent, + ScheduleForceSend, + ScheduleSequence, + ScheduleStatus, + ScheduleUpdated, + SendTo, + Sequence, + ShowWithoutTime, + Size, + Skip, + Source, + Start, + Status, + TimeZone, + Title, + Trigger, + Uid, + Until, + Updated, + Uri, + Version, + VirtualLocations, + When, + EndTimeZone, + MainLocationId, + OrganizerCalendarAddress, + RecurrenceRule, + ICalendar, + Properties, + Parameters, + ConvertedProperties, + ValueType, + Components, + DateTime(JSCalendarDateTime), + LinkDisplay(JSCalendarLinkDisplay), + VirtualLocationFeature(JSCalendarVirtualLocationFeature), + ParticipantRole(JSCalendarParticipantRole), + RelationValue(JSCalendarRelation), + LinkRelation(LinkRelation), + Pointer(JsonPointer>), + IdValue(I), + IdReference(String), +} + +impl JSCalendarId for T where + T: FromStr + Sized + Serialize + Display + Clone + Eq + Hash + Ord + Debug + Default +{ +} + +impl From for JSCalendarProperty { + fn from(id: I) -> Self { + JSCalendarProperty::IdValue(id) + } +} + +impl From for JSCalendarValue { + fn from(id: I) -> Self { + JSCalendarValue::Id(id) + } +} + +impl<'x, I: JSCalendarId, B: JSCalendarId> Default for JSCalendar<'x, I, B> { + fn default() -> Self { + Self(Value::Object(Map::from(vec![ + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Group)), + ), + ( + Key::Property(JSCalendarProperty::Entries), + Value::Array(vec![]), + ), + ]))) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarType { + #[default] + Event, + Task, + Group, + Alert, + Boolean, + Duration, + Id, + Int, + LocalDateTime, + Link, + Location, + NDay, + Number, + Participant, + PatchObject, + RecurrenceRule, + Relation, + SignedDuration, + String, + TimeZone, + TimeZoneId, + TimeZoneRule, + UnsignedInt, + UTCDateTime, + VirtualLocation, + ICalComponent, + OffsetTrigger, + AbsoluteTrigger, +} + +// JSCalendar Enum Values for action (Context: Alert) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarAlertAction { + Display, + Email, +} + +// JSCalendar Enum Values for display (Context: Link) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarLinkDisplay { + Badge, + Graphic, + Fullsize, + Thumbnail, +} + +// JSCalendar Enum Values for features (Context: VirtualLocation) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarVirtualLocationFeature { + Audio, + Chat, + Feed, + Moderator, + Phone, + Screen, + Video, +} + +// JSCalendar Enum Values for freeBusyStatus (Context: Event, Task) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarFreeBusyStatus { + Free, + Busy, +} + +// JSCalendar Enum Values for kind (Context: Participant) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarParticipantKind { + Individual, + Group, + Resource, + Location, +} + +// JSCalendar Enum Values for participationStatus (Context: Participant) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarParticipationStatus { + NeedsAction, + Accepted, + Declined, + Tentative, + Delegated, +} + +// JSCalendar Enum Values for privacy (Context: Event, Task) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarPrivacy { + Public, + Private, + Secret, +} + +// JSCalendar Enum Values for progress (Context: Task, Participant) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarProgress { + NeedsAction, + InProcess, + Completed, + Failed, + Cancelled, +} + +// JSCalendar Enum Values for relation (Context: Relation) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarRelation { + First, + Next, + Child, + Parent, + Snooze, +} + +// JSCalendar Enum Values for relativeTo (Context: OffsetTrigger, Location) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarRelativeTo { + Start, + End, +} + +// JSCalendar Enum Values for roles (Context: Participant) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarParticipantRole { + Owner, + Optional, + Informational, + Chair, + Required, +} + +// JSCalendar Enum Values for scheduleAgent (Context: Participant) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarScheduleAgent { + Server, + Client, + None, +} + +// JSCalendar Enum Values for status (Context: Event) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSCalendarEventStatus { + Confirmed, + Cancelled, + Tentative, +} + +impl JSCalendarDateTime { + pub fn new(timestamp: i64, is_local: bool) -> Self { + Self { + timestamp, + is_local, + } + } + + pub fn is_start_of_day(&self) -> bool { + self.timestamp.rem_euclid(86400) == 0 + } + + pub fn to_rfc3339(&self) -> String { + let mut text = AsciiString::with_capacity(RFC3339_CAPACITY); + let _ = self.push_rfc3339(&mut text); + text.into_string() + } +} + +impl JSCalendar<'_, I, B> { + pub fn blob_ids(&self) -> impl Iterator { + Elements::new(&self.0).filter_map(|element| match element { + JSCalendarValue::BlobId(blob_id) => Some(blob_id), + _ => None, + }) + } +} + +impl ICalendarComponentType { + pub fn to_jscalendar_type(&self) -> Option { + match &self { + ICalendarComponentType::VCalendar => Some(JSCalendarType::Group), + ICalendarComponentType::VEvent => Some(JSCalendarType::Event), + ICalendarComponentType::VTodo => Some(JSCalendarType::Task), + ICalendarComponentType::VAlarm => Some(JSCalendarType::Alert), + ICalendarComponentType::Participant => Some(JSCalendarType::Participant), + ICalendarComponentType::VLocation => Some(JSCalendarType::Location), + ICalendarComponentType::Standard + | ICalendarComponentType::Daylight + | ICalendarComponentType::VAvailability + | ICalendarComponentType::Available + | ICalendarComponentType::VResource + | ICalendarComponentType::VStatus + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + | ICalendarComponentType::VTimezone + | ICalendarComponentType::Other(_) => None, + } + } +} + +impl JSCalendarType { + pub fn to_icalendar_component_type(&self) -> Option { + match &self { + JSCalendarType::Group => Some(ICalendarComponentType::VCalendar), + JSCalendarType::Event => Some(ICalendarComponentType::VEvent), + JSCalendarType::Task => Some(ICalendarComponentType::VTodo), + JSCalendarType::Alert => Some(ICalendarComponentType::VAlarm), + JSCalendarType::Participant => Some(ICalendarComponentType::Participant), + JSCalendarType::Location => Some(ICalendarComponentType::VLocation), + _ => None, + } + } +} + +// 7f1e1965-ae73-4454-b088-232c90730ce2 +static JSCAL_NAMESPACE: uuid::Uuid = uuid::Uuid::from_bytes([ + 127, 30, 25, 101, 174, 115, 68, 84, 176, 136, 35, 44, 144, 115, 12, 226, +]); + +pub(crate) trait Uuid5 { + fn uuid5_hyphenated<'b>(&self, buffer: &'b mut [u8; Hyphenated::LENGTH]) -> &'b str; +} + +#[inline] +pub fn uuid5(text: impl AsRef<[u8]>) -> String { + let mut buffer = [0u8; Hyphenated::LENGTH]; + text.as_ref().uuid5_hyphenated(&mut buffer).to_string() +} + +impl Uuid5 for [u8] { + #[inline] + fn uuid5_hyphenated<'b>(&self, buffer: &'b mut [u8; Hyphenated::LENGTH]) -> &'b str { + let digest: [u8; 20] = Sha1::new() + .chain_update(JSCAL_NAMESPACE.as_bytes()) + .chain_update(self) + .finalize() + .into(); + let [bytes @ .., _, _, _, _] = digest; + Builder::from_sha1_bytes(bytes) + .into_uuid() + .hyphenated() + .encode_lower(buffer) + } +} + +#[cfg(all(test, stalwart_tests))] +impl std::fmt::Display for JSCalendar<'_, I, B> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = serde_json::to_string_pretty(&self.0).map_err(|_| std::fmt::Error)?; + write!(f, "{}", s) + } +} + +impl Display for JSCalendarProperty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.to_string().as_ref()) + } +} + diff --git a/crates/calcard/src/jscalendar/overrides.rs b/crates/calcard/src/jscalendar/overrides.rs new file mode 100644 index 00000000..d625c04c --- /dev/null +++ b/crates/calcard/src/jscalendar/overrides.rs @@ -0,0 +1,693 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{IanaParse, elements::Elements, jsprop::text::ConvertedKeys}, + icalendar::ICalendarProperty, + jscalendar::{ + JSCalendarDateTime, JSCalendarId, JSCalendarProperty, JSCalendarValue, + ext::{JSCalendarKeyExt, JSCalendarObjectExt, JSCalendarPatch}, + }, +}; +use ahash::AHashMap; +use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Map, Value}; +use std::{borrow::Cow, ptr, str::FromStr}; + +type JSCalendarMap<'x, I, B> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type JSCalendarObject<'x, I, B> = Value<'x, JSCalendarProperty, JSCalendarValue>; + +#[derive(Debug, Clone, Copy, Default)] +struct SeriesDates { + start: Option, + due: Option, +} + +pub(crate) struct OverrideTemplate<'x, I: JSCalendarId, B: JSCalendarId> { + entries: JSCalendarMap<'x, I, B>, + dates: SeriesDates, + has_participants: bool, + uses: usize, +} + +impl SeriesDates { + fn new(base: &JSCalendarMap<'_, I, B>) -> Self { + let mut dates = SeriesDates::default(); + for (key, value) in base.iter() { + match (key, value) { + ( + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(dt)), + ) => dates.start = Some(*dt), + ( + Key::Property(JSCalendarProperty::Due), + Value::Element(JSCalendarValue::DateTime(dt)), + ) => dates.due = Some(*dt), + _ => {} + } + } + dates + } + + fn instance( + &self, + recurrence_id: JSCalendarDateTime, + ) -> (Option, Option) { + let Some(start) = self.start else { + return (None, self.due); + }; + let rid = JSCalendarDateTime::new(recurrence_id.timestamp, true); + ( + Some(rid), + self.due.map(|due| { + JSCalendarDateTime::new( + due.timestamp + (rid.timestamp - start.timestamp), + due.is_local, + ) + }), + ) + } +} + +trait OverrideMapExt: Sized { + fn last_object(&self, property: &JSCalendarProperty) -> Option<&Self>; + fn converted_override_dates(&self) -> Vec; + fn reaches_template(&self, is_converted: impl FnMut() -> bool) -> bool; +} + +impl OverrideMapExt for JSCalendarMap<'_, I, B> { + fn last_object(&self, property: &JSCalendarProperty) -> Option<&Self> { + self.as_vec() + .iter() + .rev() + .find_map(|(key, value)| match (key, value) { + (Key::Property(key), Value::Object(obj)) if key == property => Some(obj), + _ => None, + }) + } + + fn converted_override_dates(&self) -> Vec { + self.last_object(&JSCalendarProperty::ICalendar) + .into_iter() + .flat_map(|ical| ical.iter()) + .filter_map(|(key, value)| match (key, value) { + (Key::Property(JSCalendarProperty::ConvertedProperties), Value::Object(props)) => { + Some(props.keys()) + } + _ => None, + }) + .flatten() + .filter_map(|key| match key.clone().into_converted_keys().as_slice() { + [ + Key::Property(JSCalendarProperty::RecurrenceOverrides), + Key::Property(JSCalendarProperty::DateTime(dt)), + .., + ] => Some(*dt), + _ => None, + }) + .collect() + } + + fn reaches_template(&self, mut is_converted: impl FnMut() -> bool) -> bool { + let exported = || { + self.iter().filter(|(key, value)| { + !key.is_forbidden_override_key() + && (!matches!(key, Key::Property(JSCalendarProperty::Excluded)) + || matches!(value, Value::Bool(true))) + }) + }; + let mut members = exported(); + match (members.next(), members.next()) { + (None, _) => false, + (Some((key, Value::Element(JSCalendarValue::Duration(_)))), None) + if key.same_key(&Key::Property(JSCalendarProperty::Duration)) && is_converted() => + { + false + } + _ => !exported().any(|(key, value)| { + matches!(value, Value::Bool(true)) + && key.same_key(&Key::Property(JSCalendarProperty::Excluded)) + }), + } + } +} + +impl<'x, I: JSCalendarId, B: JSCalendarId> OverrideTemplate<'x, I, B> { + pub(crate) fn uses( + base: &JSCalendarMap<'x, I, B>, + overrides: &JSCalendarMap<'x, I, B>, + ) -> usize { + let instances = overrides + .values() + .filter(|patch| patch.is_instance_patch()) + .count(); + if instances == 0 + || !base + .last_object(&JSCalendarProperty::RecurrenceOverrides) + .is_some_and(|exported| ptr::eq(exported, overrides)) + { + return instances; + } + let mut converted = None; + let has_instance = overrides.iter().any(|(key, patch)| match (key, patch) { + (Key::Property(JSCalendarProperty::DateTime(dt)), Value::Object(patch)) => patch + .reaches_template(|| { + converted + .get_or_insert_with(|| base.converted_override_dates()) + .contains(dt) + }), + _ => false, + }); + if has_instance { instances } else { 0 } + } + + pub(crate) fn new(base: &JSCalendarMap<'x, I, B>, uses: usize) -> Self { + let mut entries = Map::from(Vec::with_capacity(base.len())); + let mut has_participants = false; + + for (key, value) in base.iter() { + match (key, value) { + (Key::Property(property), _) if property.is_series_property() => {} + (Key::Property(JSCalendarProperty::ICalendar), Value::Object(obj)) => { + if let Some(obj) = instance_ical_clone(obj) { + entries.insert_unchecked(key.clone(), Value::Object(obj)); + } + } + (Key::Property(JSCalendarProperty::Participants), Value::Object(obj)) => { + has_participants = !obj.is_empty(); + entries.insert_unchecked(key.clone(), value.clone()); + } + _ => { + entries.insert_unchecked(key.clone(), value.clone()); + } + } + } + + Self { + entries, + dates: SeriesDates::new(base), + has_participants, + uses, + } + } + + pub(crate) fn blob_uses(&self) -> impl Iterator { + Elements::from_values(self.entries.values()) + .filter_map(|element| match element { + JSCalendarValue::BlobId(blob_id) => Some(blob_id), + _ => None, + }) + .flat_map(|blob_id| std::iter::repeat_n(blob_id, self.uses)) + } + + pub(crate) fn instance( + &mut self, + recurrence_id: JSCalendarDateTime, + ) -> JSCalendarMap<'x, I, B> { + let (start, due) = self.dates.instance(recurrence_id); + for (property, dt) in [ + (JSCalendarProperty::Start, start), + (JSCalendarProperty::Due, due), + ] { + if let Some(dt) = dt + && let Some(value) = self.entries.lookup_mut(&Key::Property(property)) + { + *value = Value::Element(JSCalendarValue::DateTime(dt)); + } + } + + self.uses = self.uses.saturating_sub(1); + if self.uses == 0 { + std::mem::take(&mut self.entries) + } else { + self.entries.clone() + } + } + + pub(crate) fn apply_patch( + &self, + instance: JSCalendarMap<'x, I, B>, + patch: JSCalendarMap<'x, I, B>, + ) -> Result, String> { + let mut instance = Value::Object(instance); + let mut invalid_pointer = None; + + for (key, value) in patch.into_vec() { + match key { + Key::Property(JSCalendarProperty::Pointer(pointer)) => { + if !instance.patch_jptr(pointer.iter(), value) && invalid_pointer.is_none() { + invalid_pointer = Some(pointer.to_string()); + } + } + Key::Property(JSCalendarProperty::Excluded) => {} + key => { + if let Value::Object(entries) = &mut instance { + if value.is_null() { + if let Some(pos) = entries.key_position(&key) { + entries.as_mut_vec().remove(pos); + } + } else { + entries.upsert(key, value); + } + } + } + } + } + + let Value::Object(mut instance) = instance else { + return Err(invalid_pointer.unwrap_or_default()); + }; + + if self.has_participants + && instance + .lookup(&Key::Property(JSCalendarProperty::Participants)) + .and_then(Value::as_object) + .is_none_or(|participants| participants.is_empty()) + { + instance.as_mut_vec().retain(|(key, _)| { + !matches!( + key, + Key::Property( + JSCalendarProperty::OrganizerCalendarAddress | JSCalendarProperty::SentBy + ) + ) + }); + } + + match invalid_pointer { + Some(pointer) => Err(pointer), + None => Ok(instance), + } + } +} + +fn instance_ical_clone<'x, I: JSCalendarId, B: JSCalendarId>( + obj: &JSCalendarMap<'x, I, B>, +) -> Option> { + let mut obj = obj.clone(); + retain_instance_ical(&mut obj).then_some(obj) +} + +fn retain_instance_ical( + obj: &mut JSCalendarMap<'_, I, B>, +) -> bool { + let mut has_contents = false; + obj.as_mut_vec().retain_mut(|(key, value)| { + let keep = match (key, value) { + (Key::Property(JSCalendarProperty::ConvertedProperties), Value::Object(props)) => { + props + .as_mut_vec() + .retain(|(key, _)| !key.is_series_converted_property()); + !props.is_empty() + } + (Key::Property(JSCalendarProperty::Properties), Value::Array(props)) => { + props.retain(|property| !is_series_ical_property(property)); + !props.is_empty() + } + (Key::Property(JSCalendarProperty::Name), _) => return true, + _ => true, + }; + has_contents |= keep; + keep + }); + has_contents +} + +fn is_series_ical_property( + property: &JSCalendarObject<'_, I, B>, +) -> bool { + property + .as_array() + .and_then(|items| items.first()) + .and_then(Value::as_str) + .is_some_and(|name| { + matches!( + ICalendarProperty::parse(name.as_bytes()), + Some( + ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exdate + | ICalendarProperty::Exrule + | ICalendarProperty::RecurrenceId + ) + ) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Inherited { + Nothing, + Start, + StartAndEnd, +} + +impl Inherited { + fn covers(self, property: &JSCalendarProperty) -> bool { + match (self, property) { + (Inherited::Nothing, _) => false, + ( + _, + JSCalendarProperty::Start + | JSCalendarProperty::TimeZone + | JSCalendarProperty::ShowWithoutTime, + ) => true, + ( + Inherited::StartAndEnd, + JSCalendarProperty::Duration + | JSCalendarProperty::Due + | JSCalendarProperty::EndTimeZone, + ) => true, + _ => false, + } + } + + fn covers_key(self, key: &Key<'_, JSCalendarProperty>) -> bool { + match key { + Key::Property(JSCalendarProperty::Pointer(pointer)) => { + let mut items = pointer + .iter() + .filter(|item| !matches!(item, JsonPointerItem::Root)); + matches!( + (items.next(), items.next()), + (Some(JsonPointerItem::Key(Key::Property(property))), None) + if self.covers(property) + ) + } + Key::Property(property) => self.covers(property), + Key::Borrowed(_) | Key::Owned(_) => { + JSCalendarProperty::::from_str(key.to_string().as_ref()) + .is_ok_and(|property| self.covers(&property)) + } + } + } +} + +pub struct OverrideDiff<'a, I: JSCalendarId, B: JSCalendarId> { + base: &'a JSCalendarMap<'static, I, B>, + base_ical: Option>, + dates: SeriesDates, +} + +impl<'a, I: JSCalendarId, B: JSCalendarId> OverrideDiff<'a, I, B> { + pub fn new(base: &'a JSCalendarMap<'static, I, B>) -> Self { + Self { + base, + base_ical: base + .lookup(&Key::Property(JSCalendarProperty::ICalendar)) + .and_then(Value::as_object) + .and_then(instance_ical_clone), + dates: SeriesDates::new(base), + } + } + + pub fn diff( + &self, + recurrence_id: JSCalendarDateTime, + mut instance: JSCalendarMap<'static, I, B>, + inherited: Inherited, + ) -> JSCalendarMap<'static, I, B> { + let (start, due) = self.dates.instance(recurrence_id); + let mut patch = Map::from(Vec::new()); + let ical_key = Key::Property(JSCalendarProperty::ICalendar); + + for (key, _) in self.base.iter() { + if !key.is_ignored_override_key() + && !matches!( + key, + Key::Property(JSCalendarProperty::Start | JSCalendarProperty::ICalendar) + ) + && !inherited.covers_key(key) + && instance.key_position(key).is_none() + { + patch.insert_unchecked(key.to_owned(), Value::Null); + } + } + + if inherited != Inherited::Nothing + && let Some(base_ical) = &self.base_ical + { + inherit_date_conversions(&mut instance, base_ical, inherited); + } + let has_ical = match instance.lookup_mut(&ical_key) { + Some(Value::Object(obj)) => retain_instance_ical(obj), + _ => false, + }; + if !has_ical { + if let Some(pos) = instance.key_position(&ical_key) { + instance.as_mut_vec().remove(pos); + } + if self.base_ical.is_some() { + patch.insert_unchecked(ical_key, Value::Null); + } + } + + for (key, value) in instance.into_vec() { + if key.is_ignored_override_key() { + continue; + } + let base_value = match (&key, &value) { + ( + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(dt)), + ) if start == Some(*dt) => continue, + ( + Key::Property(JSCalendarProperty::Due), + Value::Element(JSCalendarValue::DateTime(dt)), + ) if due == Some(*dt) => continue, + (Key::Property(JSCalendarProperty::ICalendar), Value::Object(ical)) + if self + .base_ical + .as_ref() + .is_some_and(|base| same_members(base, ical)) => + { + continue; + } + ( + Key::Property( + JSCalendarProperty::Start + | JSCalendarProperty::Due + | JSCalendarProperty::ICalendar, + ), + _, + ) => None, + (key, _) => self.base.lookup(key), + }; + + match (base_value, key, value) { + ( + Some(Value::Object(base_members)), + Key::Property(property), + Value::Object(members), + ) if property.is_member_map() => { + diff_members(&mut patch, property, base_members, members); + } + (Some(base_value), _, value) if same_value(base_value, &value) => {} + (_, key, value) => { + patch.insert_unchecked(key, value); + } + } + } + + patch + } +} + +fn inherit_date_conversions( + instance: &mut JSCalendarMap<'static, I, B>, + base_ical: &JSCalendarMap<'static, I, B>, + inherited: Inherited, +) { + let converted_key = Key::Property(JSCalendarProperty::ConvertedProperties); + let Some(Value::Object(base_converted)) = base_ical.lookup(&converted_key) else { + return; + }; + let mut conversions = base_converted + .iter() + .filter(|(key, _)| inherited.covers_key(key)) + .peekable(); + if conversions.peek().is_none() { + return; + } + + let Value::Object(ical) = instance.upsert_or_get_mut( + Key::Property(JSCalendarProperty::ICalendar), + Value::new_object, + ) else { + return; + }; + let name_key = Key::Property(JSCalendarProperty::Name); + if ical.key_position(&name_key).is_none() + && let Some(name) = base_ical.lookup(&name_key) + { + ical.insert_unchecked(name_key, name.clone()); + } + let Value::Object(converted) = ical.upsert_or_get_mut(converted_key, Value::new_object) else { + return; + }; + for (key, value) in conversions { + if converted.key_position(key).is_none() { + converted.insert_unchecked(key.clone(), value.clone()); + } + } +} + +fn diff_members( + patch: &mut JSCalendarMap<'static, I, B>, + property: JSCalendarProperty, + base_members: &JSCalendarMap<'static, I, B>, + members: JSCalendarMap<'static, I, B>, +) { + let base_members = base_members.as_vec(); + let mut matched = vec![false; base_members.len()]; + let mut next_pos = 0; + let mut index: Option, usize>> = None; + + for (id, member) in members.into_vec() { + let pos = if base_members + .get(next_pos) + .is_some_and(|(base_id, _)| base_id.same_key(&id)) + { + Some(next_pos) + } else { + index + .get_or_insert_with(|| { + base_members + .iter() + .enumerate() + .map(|(pos, (id, _))| (id.to_string(), pos)) + .collect() + }) + .get(id.to_string().as_ref()) + .copied() + }; + + let Some((pos, base_member)) = + pos.and_then(|pos| base_members.get(pos).map(|(_, member)| (pos, member))) + else { + patch.insert_unchecked(property.member_pointer([id]), member); + continue; + }; + if let Some(matched) = matched.get_mut(pos) { + *matched = true; + } + next_pos = pos + 1; + + match (base_member, member) { + (base_member, member) if same_value(base_member, &member) => {} + (Value::Object(base_member), Value::Object(member)) + if property == JSCalendarProperty::Participants + && base_member.lookup(&Key::Property(JSCalendarProperty::CalendarAddress)) + != member.lookup(&Key::Property(JSCalendarProperty::CalendarAddress)) => + { + patch.insert_unchecked(property.member_pointer([id]), Value::Object(member)); + } + (Value::Object(base_member), Value::Object(member)) => { + for (sub_property, _) in base_member.iter() { + if member.key_position(sub_property).is_none() { + patch.insert_unchecked( + property.member_pointer([id.clone(), sub_property.to_owned()]), + Value::Null, + ); + } + } + for (sub_property, value) in member.into_vec() { + match (base_member.lookup(&sub_property), value) { + (Some(base_value), value) if same_value(base_value, &value) => {} + (Some(Value::Object(base_set)), Value::Object(set)) + if is_boolean_set(base_set) && is_boolean_set(&set) => + { + for (key, _) in base_set.iter() { + if set.key_position(key).is_none() { + patch.insert_unchecked( + property.member_pointer([ + id.clone(), + sub_property.clone(), + key.to_owned(), + ]), + Value::Null, + ); + } + } + for (key, value) in set.into_vec() { + if !base_set + .lookup(&key) + .is_some_and(|base| same_value(base, &value)) + { + patch.insert_unchecked( + property.member_pointer([ + id.clone(), + sub_property.clone(), + key, + ]), + value, + ); + } + } + } + (_, value) => { + patch.insert_unchecked( + property.member_pointer([id.clone(), sub_property]), + value, + ); + } + } + } + } + (_, member) => { + patch.insert_unchecked(property.member_pointer([id]), member); + } + } + } + + for ((id, _), matched) in base_members.iter().zip(matched) { + if !matched { + patch.insert_unchecked(property.member_pointer([id.to_owned()]), Value::Null); + } + } +} + +fn same_value( + a: &JSCalendarObject<'_, I, B>, + b: &JSCalendarObject<'_, I, B>, +) -> bool { + match (a, b) { + (Value::Object(a), Value::Object(b)) => same_members(a, b), + (Value::Array(a), Value::Array(b)) => { + a.len() == b.len() && a.iter().zip(b).all(|(a, b)| same_value(a, b)) + } + _ => a == b, + } +} + +fn same_members( + a: &JSCalendarMap<'_, I, B>, + b: &JSCalendarMap<'_, I, B>, +) -> bool { + if a.len() != b.len() { + return false; + } + for (pos, ((a_key, a_value), (b_key, b_value))) in a.iter().zip(b.iter()).enumerate() { + if !a_key.same_key(b_key) { + let index = b + .iter() + .skip(pos) + .map(|(key, value)| (key.to_string(), value)) + .collect::>(); + return index.len() == a.len() - pos + && a.iter().skip(pos).all(|(key, a_value)| { + index + .get(key.to_string().as_ref()) + .is_some_and(|b_value| same_value(a_value, b_value)) + }); + } else if !same_value(a_value, b_value) { + return false; + } + } + true +} + +fn is_boolean_set(set: &JSCalendarMap<'_, I, B>) -> bool { + set.values().all(|value| matches!(value, Value::Bool(_))) +} diff --git a/crates/calcard/src/jscalendar/parser.rs b/crates/calcard/src/jscalendar/parser.rs new file mode 100644 index 00000000..105cad37 --- /dev/null +++ b/crates/calcard/src/jscalendar/parser.rs @@ -0,0 +1,550 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + CalendarScale, IanaParse, IanaString, IdReference, LinkRelation, + format::AsciiPush, + jsprop::text::{DisplayEq, Expect, IdReferenceText, PointerText, PointerTextExt}, + }, + icalendar::{ + ICalendarDuration, ICalendarFrequency, ICalendarMethod, ICalendarMonth, ICalendarSkip, + ICalendarWeekday, + }, + jscalendar::{ + JSCalendar, JSCalendarAlertAction, JSCalendarDateTime, JSCalendarEventStatus, + JSCalendarFreeBusyStatus, JSCalendarId, JSCalendarLinkDisplay, JSCalendarParticipantKind, + JSCalendarParticipantRole, JSCalendarParticipationStatus, JSCalendarPrivacy, + JSCalendarProgress, JSCalendarProperty, JSCalendarRelation, JSCalendarRelativeTo, + JSCalendarScheduleAgent, JSCalendarType, JSCalendarValue, JSCalendarVirtualLocationFeature, + }, +}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property as _, Value}; +use mail_parser::DateTime; +use serde::Serializer; +use std::{ + borrow::Cow, + fmt::{self, Write}, + mem::discriminant, + ops::RangeInclusive, + str::{FromStr, from_utf8}, +}; + +const FOUR_DIGIT_YEARS: RangeInclusive = -62_167_219_200..=253_402_300_799; +pub(crate) const VALUE_TEXT: usize = 64; +pub(crate) const KEY_TEXT: usize = 128; + +impl<'x, I: JSCalendarId, B: JSCalendarId> JSCalendar<'x, I, B> { + pub fn parse(json: &'x str) -> Result { + Value::parse_json(json).map(JSCalendar) + } + + pub fn to_string_pretty(&self) -> String { + serde_json::to_string_pretty(&self.0).unwrap_or_default() + } +} + +impl Element for JSCalendarValue { + type Property = JSCalendarProperty; + + fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { + if let Key::Property(prop) = key { + match prop.patch_or_prop() { + JSCalendarProperty::Type => JSCalendarType::from_str(value) + .ok() + .map(JSCalendarValue::Type), + JSCalendarProperty::Created + | JSCalendarProperty::Updated + | JSCalendarProperty::Acknowledged + | JSCalendarProperty::ScheduleUpdated + | JSCalendarProperty::When + | JSCalendarProperty::UtcStart + | JSCalendarProperty::UtcEnd => { + JSCalendarDateTime::from_rfc3339(value, false).map(JSCalendarValue::DateTime) + } + JSCalendarProperty::Due + | JSCalendarProperty::RecurrenceId + | JSCalendarProperty::Start + | JSCalendarProperty::Until => { + JSCalendarDateTime::from_rfc3339(value, true).map(JSCalendarValue::DateTime) + } + JSCalendarProperty::Duration + | JSCalendarProperty::EstimatedDuration + | JSCalendarProperty::Offset => { + ICalendarDuration::parse(value.as_bytes()).map(JSCalendarValue::Duration) + } + JSCalendarProperty::Action => JSCalendarAlertAction::from_str(value) + .ok() + .map(JSCalendarValue::AlertAction), + JSCalendarProperty::FreeBusyStatus => JSCalendarFreeBusyStatus::from_str(value) + .ok() + .map(JSCalendarValue::FreeBusyStatus), + JSCalendarProperty::Kind => JSCalendarParticipantKind::from_str(value) + .ok() + .map(JSCalendarValue::ParticipantKind), + JSCalendarProperty::ParticipationStatus => { + JSCalendarParticipationStatus::from_str(value) + .ok() + .map(JSCalendarValue::ParticipationStatus) + } + JSCalendarProperty::Privacy => JSCalendarPrivacy::from_str(value) + .ok() + .map(JSCalendarValue::Privacy), + JSCalendarProperty::Progress => JSCalendarProgress::from_str(value) + .ok() + .map(JSCalendarValue::Progress), + JSCalendarProperty::RelativeTo => JSCalendarRelativeTo::from_str(value) + .ok() + .map(JSCalendarValue::RelativeTo), + JSCalendarProperty::ScheduleAgent => JSCalendarScheduleAgent::from_str(value) + .ok() + .map(JSCalendarValue::ScheduleAgent), + JSCalendarProperty::Status => JSCalendarEventStatus::from_str(value) + .ok() + .map(JSCalendarValue::EventStatus), + JSCalendarProperty::Rel => { + LinkRelation::parse(value.as_bytes()).map(JSCalendarValue::LinkRelation) + } + JSCalendarProperty::Frequency => { + ICalendarFrequency::parse(value.as_bytes()).map(JSCalendarValue::Frequency) + } + JSCalendarProperty::FirstDayOfWeek | JSCalendarProperty::Day => { + ICalendarWeekday::parse(value.as_bytes()).map(JSCalendarValue::Weekday) + } + JSCalendarProperty::Skip => { + ICalendarSkip::parse(value.as_bytes()).map(JSCalendarValue::Skip) + } + JSCalendarProperty::Rscale => { + CalendarScale::parse(value.as_bytes()).map(JSCalendarValue::CalendarScale) + } + JSCalendarProperty::ByMonth => { + ICalendarMonth::parse(value.as_bytes()).map(JSCalendarValue::Month) + } + JSCalendarProperty::Method => { + ICalendarMethod::parse(value.as_bytes()).map(JSCalendarValue::Method) + } + JSCalendarProperty::Id | JSCalendarProperty::BaseEventId => { + match IdReference::parse(value) { + IdReference::Value(value) => JSCalendarValue::Id(value).into(), + IdReference::Reference(value) => JSCalendarValue::IdReference(value).into(), + IdReference::Error => None, + } + } + JSCalendarProperty::BlobId => match IdReference::parse(value) { + IdReference::Value(value) => JSCalendarValue::BlobId(value).into(), + IdReference::Reference(value) => JSCalendarValue::IdReference(value).into(), + IdReference::Error => None, + }, + _ => None, + } + } else if key + .as_string_key() + .is_some_and(|name| name.ends_with(BLOB_ID_SUFFIX)) + { + match IdReference::parse(value) { + IdReference::Value(value) => JSCalendarValue::BlobId(value).into(), + IdReference::Reference(value) => JSCalendarValue::IdReference(value).into(), + IdReference::Error => None, + } + } else { + None + } + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + JSCalendarValue::Type(v) => v.as_str().into(), + JSCalendarValue::DateTime(v) => v.to_rfc3339().into(), + JSCalendarValue::Duration(v) => v.to_string().into(), + JSCalendarValue::AlertAction(v) => v.as_str().into(), + JSCalendarValue::FreeBusyStatus(v) => v.as_str().into(), + JSCalendarValue::ParticipantKind(v) => v.as_str().into(), + JSCalendarValue::ParticipationStatus(v) => v.as_str().into(), + JSCalendarValue::Privacy(v) => v.as_str().into(), + JSCalendarValue::Progress(v) => v.as_str().into(), + JSCalendarValue::RelativeTo(v) => v.as_str().into(), + JSCalendarValue::ScheduleAgent(v) => v.as_str().into(), + JSCalendarValue::EventStatus(v) => v.as_str().into(), + JSCalendarValue::LinkRelation(v) => v.as_str().into(), + JSCalendarValue::Frequency(v) => v.as_js_str().into(), + JSCalendarValue::CalendarScale(v) => v.as_js_str().into(), + JSCalendarValue::Skip(v) => v.as_js_str().into(), + JSCalendarValue::Weekday(v) => v.as_js_str().into(), + JSCalendarValue::Month(v) => v.to_string().into(), + JSCalendarValue::Method(v) => v.as_js_str().into(), + JSCalendarValue::Id(v) => v.to_string().into(), + JSCalendarValue::BlobId(v) => v.to_string().into(), + JSCalendarValue::IdReference(s) => s.to_id_reference().into(), + } + } + + #[inline] + fn serialize_text(&self, serializer: S) -> Result { + match self { + JSCalendarValue::DateTime(_) + | JSCalendarValue::Duration(_) + | JSCalendarValue::Month(_) + | JSCalendarValue::Id(_) + | JSCalendarValue::BlobId(_) + | JSCalendarValue::IdReference(_) => self.serialize_data_text(serializer), + _ => serializer.serialize_str(&self.to_cow()), + } + } +} + +impl JSCalendarValue { + #[inline(never)] + fn serialize_data_text(&self, serializer: S) -> Result { + StackText::::serialize( + serializer, + |text| match self { + JSCalendarValue::DateTime(date_time) => date_time.push_rfc3339(text), + JSCalendarValue::Duration(duration) => text.push_duration(duration.parts()), + JSCalendarValue::Month(month) => write!(text, "{month}"), + JSCalendarValue::Id(id) => write!(text, "{id}"), + JSCalendarValue::BlobId(id) => write!(text, "{id}"), + JSCalendarValue::IdReference(id) => text.push_id_reference(id), + _ => Err(fmt::Error), + }, + || self.to_cow(), + ) + } +} + +impl jmap_tools::Property for JSCalendarProperty { + fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) if key.is_metadata() => None, + Some(Key::Property(key)) => match key.patch_or_prop() { + JSCalendarProperty::RecurrenceOverrides => { + JSCalendarDateTime::from_rfc3339(value, true).map(JSCalendarProperty::DateTime) + } + JSCalendarProperty::Display => JSCalendarLinkDisplay::from_str(value) + .ok() + .map(JSCalendarProperty::LinkDisplay), + JSCalendarProperty::Features => JSCalendarVirtualLocationFeature::from_str(value) + .ok() + .map(JSCalendarProperty::VirtualLocationFeature), + JSCalendarProperty::Roles => JSCalendarParticipantRole::from_str(value) + .ok() + .map(JSCalendarProperty::ParticipantRole), + JSCalendarProperty::Relation => JSCalendarRelation::from_str(value) + .ok() + .map(JSCalendarProperty::RelationValue), + JSCalendarProperty::ConvertedProperties => { + JsonPointer::parse_nested(value, depth).map(JSCalendarProperty::Pointer) + } + JSCalendarProperty::DateTime(_) if value.contains('/') => { + JsonPointer::parse_nested(value, depth).map(JSCalendarProperty::Pointer) + } + JSCalendarProperty::DateTime(_) | JSCalendarProperty::Entries => { + JSCalendarProperty::parse_name(value) + } + JSCalendarProperty::CalendarIds => match IdReference::parse(value) { + IdReference::Value(value) => JSCalendarProperty::IdValue(value).into(), + IdReference::Reference(value) => JSCalendarProperty::IdReference(value).into(), + IdReference::Error => None, + }, + _ => JSCalendarProperty::parse_nested_name(value), + }, + None if value.contains('/') => { + JsonPointer::parse_nested(value, depth).map(JSCalendarProperty::Pointer) + } + None => JSCalendarProperty::parse_name(value), + Some(_) => JSCalendarProperty::parse_nested_name(value), + } + } + + fn is_opaque(&self) -> bool { + self.is_metadata() + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + JSCalendarProperty::DateTime(dt) => dt.to_rfc3339().into(), + JSCalendarProperty::Pointer(pointer) => pointer.to_text().into(), + JSCalendarProperty::IdReference(id) => id.to_id_reference().into(), + _ => self.to_string(), + } + } + + #[inline] + fn key_eq(&self, other: &Self) -> bool { + match (self, other) { + (a, b) if !a.has_data() && !b.has_data() => discriminant(a) == discriminant(b), + (JSCalendarProperty::DateTime(a), JSCalendarProperty::DateTime(b)) => a.same_rfc3339(b), + _ => self.data_key_eq(other), + } + } + + #[inline] + fn key_eq_str(&self, other: &str) -> bool { + if !self.has_data() { + self.to_string() == other + } else { + self.data_key_eq_str(other) + } + } + + #[inline] + fn serialize_text(&self, serializer: S) -> Result { + match self { + JSCalendarProperty::DateTime(_) + | JSCalendarProperty::Pointer(_) + | JSCalendarProperty::IdValue(_) + | JSCalendarProperty::IdReference(_) => self.serialize_data_text(serializer), + _ => serializer.serialize_str(&self.to_cow()), + } + } +} + +pub(crate) const BLOB_ID_SUFFIX: &str = "/blobId"; + +impl JSCalendarProperty { + #[inline(never)] + fn data_key_eq(&self, other: &Self) -> bool { + match (self, other) { + (JSCalendarProperty::Pointer(a), JSCalendarProperty::Pointer(b)) if a == b => true, + _ => match (self.static_name(), other.static_name()) { + (Some(a), Some(b)) => a == b, + (Some(a), None) => other.data_key_eq_str(a), + (None, Some(b)) => self.data_key_eq_str(b), + (None, None) => other.data_key_eq_str(&self.to_string()), + }, + } + } + + #[inline(never)] + fn data_key_eq_str(&self, other: &str) -> bool { + match self { + JSCalendarProperty::DateTime(date_time) => date_time.rfc3339_eq(other), + JSCalendarProperty::IdValue(id) => id.display_eq(other), + JSCalendarProperty::Pointer(pointer) => pointer.text_eq(other), + JSCalendarProperty::IdReference(reference) => { + other.strip_prefix('#') == Some(reference.as_str()) + } + JSCalendarProperty::LinkRelation(relation) => relation.as_str() == other, + _ => self.to_string() == other, + } + } + + #[inline(never)] + fn serialize_data_text(&self, serializer: S) -> Result { + StackText::::serialize( + serializer, + |text| match self { + JSCalendarProperty::DateTime(date_time) => date_time.push_rfc3339(text), + JSCalendarProperty::Pointer(pointer) => { + text.push_pointer(pointer, Self::push_pointer_key) + } + JSCalendarProperty::IdValue(id) => write!(text, "{id}"), + JSCalendarProperty::IdReference(id) => text.push_id_reference(id), + _ => Err(fmt::Error), + }, + || self.to_cow(), + ) + } + + fn push_pointer_key(&self, text: &mut StackText) -> fmt::Result { + match self { + JSCalendarProperty::DateTime(date_time) => date_time.push_rfc3339(text), + property => text.push_pointer_token(&property.to_cow()), + } + } + + fn patch_or_prop(&self) -> &JSCalendarProperty { + let mut property = self; + while let JSCalendarProperty::Pointer(ptr) = property { + match ptr.last() { + Some(JsonPointerItem::Key(Key::Property(inner))) => property = inner, + _ => break, + } + } + property + } +} + +impl JSCalendarDateTime { + pub(crate) fn from_rfc3339(value: &str, is_local: bool) -> Option { + let timestamp = match value.as_bytes() { + [ + y0, + y1, + y2, + y3, + b'-', + m0, + m1, + b'-', + d0, + d1, + b'T', + h0, + h1, + b':', + i0, + i1, + b':', + s0, + s1, + rest @ .., + ] if matches!(rest, [] | [b'Z']) + && [y0, y1, y2, y3, m0, m1, d0, d1, h0, h1, i0, i1, s0, s1] + .iter() + .all(|digit| digit.is_ascii_digit()) => + { + let pair = |high: &u8, low: &u8| (high - b'0') * 10 + (low - b'0'); + DateTime { + year: u16::from(pair(y0, y1)) * 100 + u16::from(pair(y2, y3)), + month: pair(m0, m1), + day: pair(d0, d1), + hour: pair(h0, h1), + minute: pair(i0, i1), + second: pair(s0, s1), + tz_before_gmt: false, + tz_hour: 0, + tz_minute: 0, + } + .to_timestamp_local() + } + _ => DateTime::parse_rfc3339(value)?.to_timestamp_local(), + }; + Some(JSCalendarDateTime { + timestamp, + is_local, + }) + } + + #[inline] + fn same_rfc3339(&self, other: &Self) -> bool { + self.is_local == other.is_local + && (self.timestamp == other.timestamp + || (!FOUR_DIGIT_YEARS.contains(&self.timestamp) + || !FOUR_DIGIT_YEARS.contains(&other.timestamp)) + && self.same_wide_rfc3339(other)) + } + + #[inline(never)] + fn same_wide_rfc3339(&self, other: &Self) -> bool { + self.to_rfc3339() == other.to_rfc3339() + } + + fn rfc3339_eq(&self, text: &str) -> bool { + let mut expect = Expect::new(text); + self.push_rfc3339(&mut expect).is_ok() && expect.is_complete() + } + + pub(crate) fn push_rfc3339(&self, text: &mut impl AsciiPush) -> fmt::Result { + let dt = DateTime::from_timestamp(self.timestamp); + text.push_4_digits(dt.year)?; + text.push_byte(b'-')?; + text.push_2_digits(dt.month)?; + text.push_byte(b'-')?; + text.push_2_digits(dt.day)?; + text.push_byte(b'T')?; + text.push_2_digits(dt.hour)?; + text.push_byte(b':')?; + text.push_2_digits(dt.minute)?; + text.push_byte(b':')?; + text.push_2_digits(dt.second)?; + if self.is_local { + Ok(()) + } else { + text.push_byte(b'Z') + } + } +} + +pub(crate) struct StackText { + len: usize, + bytes: [u8; N], +} + +impl StackText { + pub(crate) fn serialize( + serializer: S, + push: impl FnOnce(&mut Self) -> fmt::Result, + fallback: impl FnOnce() -> Cow<'static, str>, + ) -> Result { + let mut text = StackText { + len: 0, + bytes: [0; N], + }; + match push(&mut text).ok().and_then(|()| text.as_str()) { + Some(text) => serializer.serialize_str(text), + None => serializer.serialize_str(&fallback()), + } + } + + fn as_str(&self) -> Option<&str> { + self.bytes + .get(..self.len) + .and_then(|bytes| from_utf8(bytes).ok()) + } + + pub(crate) fn push_pointer( + &mut self, + pointer: &JsonPointer

, + push_key: impl Fn(&P, &mut Self) -> fmt::Result, + ) -> fmt::Result { + for (pos, item) in pointer.as_slice().iter().enumerate() { + if pos > 0 { + self.push_byte(b'/')?; + } + match item { + JsonPointerItem::Root => {} + JsonPointerItem::Wildcard => self.push_byte(b'*')?, + JsonPointerItem::Invalid(invalid) => self.write_str(invalid)?, + JsonPointerItem::Key(Key::Property(property)) => push_key(property, self)?, + JsonPointerItem::Key(Key::Borrowed(key)) => self.push_pointer_token(key)?, + JsonPointerItem::Key(Key::Owned(key)) => self.push_pointer_token(key)?, + JsonPointerItem::Number(number) => self.push_u64(*number)?, + } + } + Ok(()) + } + + pub(crate) fn push_id_reference(&mut self, id: &str) -> fmt::Result { + self.push_byte(b'#')?; + self.write_str(id) + } + + #[inline(always)] + fn push_bytes(&mut self, bytes: &[u8]) -> fmt::Result { + let end = self.len + bytes.len(); + self.bytes + .get_mut(self.len..end) + .ok_or(fmt::Error)? + .copy_from_slice(bytes); + self.len = end; + Ok(()) + } +} + +impl PointerText for StackText {} + +impl AsciiPush for StackText { + #[inline(always)] + fn push_ascii(&mut self, bytes: &[u8]) -> fmt::Result { + self.push_bytes(bytes) + } +} + +impl Write for StackText { + #[inline(always)] + fn write_str(&mut self, text: &str) -> fmt::Result { + self.push_bytes(text.as_bytes()) + } +} + diff --git a/crates/calcard/src/jscalendar/types.rs b/crates/calcard/src/jscalendar/types.rs new file mode 100644 index 00000000..75191f99 --- /dev/null +++ b/crates/calcard/src/jscalendar/types.rs @@ -0,0 +1,995 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::jsprop::{property_names, text::PointerProperty}, + jscalendar::*, +}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use std::str::FromStr; + +impl PointerProperty for JSCalendarProperty { + fn into_pointer(self) -> Result, Self> { + match self { + JSCalendarProperty::Pointer(pointer) => Ok(pointer), + property => Err(property), + } + } +} + +impl FromStr for JSCalendarProperty { + type Err = (); + + fn from_str(s: &str) -> Result { + JSCalendarProperty::parse_name(s) + .or_else(|| JSCalendarProperty::parse_metadata_selector(s)) + .ok_or(()) + } +} + +impl JSCalendarProperty { + property_names!( + JSCalendarProperty, + "@type" => Type, + "acknowledged" => Acknowledged, + "action" => Action, + "alerts" => Alerts, + "baseEventId" => BaseEventId, + "byDay" => ByDay, + "byHour" => ByHour, + "byMinute" => ByMinute, + "byMonth" => ByMonth, + "byMonthDay" => ByMonthDay, + "bySecond" => BySecond, + "bySetPosition" => BySetPosition, + "byWeekNo" => ByWeekNo, + "byYearDay" => ByYearDay, + "calendarAddress" => CalendarAddress, + "calendarIds" => CalendarIds, + "categories" => Categories, + "color" => Color, + "contentType" => ContentType, + "coordinates" => Coordinates, + "count" => Count, + "created" => Created, + "day" => Day, + "delegatedFrom" => DelegatedFrom, + "delegatedTo" => DelegatedTo, + "description" => Description, + "descriptionContentType" => DescriptionContentType, + "display" => Display, + "due" => Due, + "duration" => Duration, + "email" => Email, + "entries" => Entries, + "estimatedDuration" => EstimatedDuration, + "excluded" => Excluded, + "expectReply" => ExpectReply, + "features" => Features, + "firstDayOfWeek" => FirstDayOfWeek, + "freeBusyStatus" => FreeBusyStatus, + "frequency" => Frequency, + "hideAttendees" => HideAttendees, + "href" => Href, + "id" => Id, + "interval" => Interval, + "invitedBy" => InvitedBy, + "isDraft" => IsDraft, + "isOrigin" => IsOrigin, + "keywords" => Keywords, + "kind" => Kind, + "links" => Links, + "locale" => Locale, + "locations" => Locations, + "locationTypes" => LocationTypes, + "mayInviteOthers" => MayInviteOthers, + "mayInviteSelf" => MayInviteSelf, + "memberOf" => MemberOf, + "method" => Method, + "name" => Name, + "nthOfPeriod" => NthOfPeriod, + "offset" => Offset, + "participants" => Participants, + "participationComment" => ParticipationComment, + "participationStatus" => ParticipationStatus, + "percentComplete" => PercentComplete, + "priority" => Priority, + "privacy" => Privacy, + "prodId" => ProdId, + "progress" => Progress, + "recurrenceId" => RecurrenceId, + "recurrenceIdTimeZone" => RecurrenceIdTimeZone, + "recurrenceOverrides" => RecurrenceOverrides, + "rel" => Rel, + "relatedTo" => RelatedTo, + "relation" => Relation, + "relativeTo" => RelativeTo, + "replyTo" => ReplyTo, + "requestStatus" => RequestStatus, + "roles" => Roles, + "rscale" => Rscale, + "sentBy" => SentBy, + "scheduleAgent" => ScheduleAgent, + "scheduleForceSend" => ScheduleForceSend, + "scheduleSequence" => ScheduleSequence, + "scheduleStatus" => ScheduleStatus, + "scheduleUpdated" => ScheduleUpdated, + "sendTo" => SendTo, + "sequence" => Sequence, + "showWithoutTime" => ShowWithoutTime, + "size" => Size, + "skip" => Skip, + "source" => Source, + "start" => Start, + "status" => Status, + "timeZone" => TimeZone, + "title" => Title, + "trigger" => Trigger, + "uid" => Uid, + "until" => Until, + "updated" => Updated, + "uri" => Uri, + "useDefaultAlerts" => UseDefaultAlerts, + "utcEnd" => UtcEnd, + "utcStart" => UtcStart, + "version" => Version, + "virtualLocations" => VirtualLocations, + "when" => When, + "endTimeZone" => EndTimeZone, + "mainLocationId" => MainLocationId, + "organizerCalendarAddress" => OrganizerCalendarAddress, + "recurrenceRule" => RecurrenceRule, + "properties" => Properties, + "components" => Components, + "valueType" => ValueType, + "convertedProperties" => ConvertedProperties, + "parameters" => Parameters, + "iCalendar" => ICalendar, + "blobId" => BlobId, + ); + + fn parse_metadata_selector(s: &str) -> Option { + s.split_once('/') + .and_then(|(root, _)| JSCalendarProperty::::parse_name(root)) + .filter(JSCalendarProperty::is_metadata) + .map(|_| JSCalendarProperty::Pointer(JsonPointer::parse(s))) + } + + pub(crate) fn is_metadata(&self) -> bool { + match self { + JSCalendarProperty::Metadata | JSCalendarProperty::PrivateMetadata => true, + JSCalendarProperty::Pointer(pointer) => { + JSCalendarProperty::is_metadata_pointer(pointer) + } + _ => false, + } + } + + pub(crate) fn sets_misplaced_excluded(pointer: &JsonPointer) -> bool { + let items = match pointer.as_slice() { + [JsonPointerItem::Root, items @ ..] => items, + items => items, + }; + let Some((JsonPointerItem::Key(Key::Property(JSCalendarProperty::Excluded)), parents)) = + items.split_last() + else { + return false; + }; + if matches!( + parents, + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::RecurrenceOverrides)), + JsonPointerItem::Key(Key::Property(JSCalendarProperty::DateTime(_))) + ] + ) { + return false; + } + let mut is_member_key = false; + parents.iter().all(|item| { + let is_structural = is_member_key + || matches!( + item, + JsonPointerItem::Key(Key::Property(_)) | JsonPointerItem::Number(_) + ); + is_member_key = matches!( + item, + JsonPointerItem::Key(Key::Property(property)) + if property.is_member_map() || *property == JSCalendarProperty::RelatedTo + ); + is_structural + }) + } + + pub(crate) fn is_metadata_pointer(pointer: &JsonPointer) -> bool { + pointer + .first() + .and_then(JsonPointerItem::as_property_key) + .is_some_and(JSCalendarProperty::is_metadata) + } + + pub fn to_string(&self) -> Cow<'static, str> { + match self { + JSCalendarProperty::Type => "@type", + JSCalendarProperty::Acknowledged => "acknowledged", + JSCalendarProperty::Action => "action", + JSCalendarProperty::Alerts => "alerts", + JSCalendarProperty::BaseEventId => "baseEventId", + JSCalendarProperty::ByDay => "byDay", + JSCalendarProperty::ByHour => "byHour", + JSCalendarProperty::ByMinute => "byMinute", + JSCalendarProperty::ByMonth => "byMonth", + JSCalendarProperty::ByMonthDay => "byMonthDay", + JSCalendarProperty::BySecond => "bySecond", + JSCalendarProperty::BySetPosition => "bySetPosition", + JSCalendarProperty::ByWeekNo => "byWeekNo", + JSCalendarProperty::ByYearDay => "byYearDay", + JSCalendarProperty::CalendarAddress => "calendarAddress", + JSCalendarProperty::CalendarIds => "calendarIds", + JSCalendarProperty::Categories => "categories", + JSCalendarProperty::Color => "color", + JSCalendarProperty::ContentType => "contentType", + JSCalendarProperty::Coordinates => "coordinates", + JSCalendarProperty::Count => "count", + JSCalendarProperty::Created => "created", + JSCalendarProperty::Day => "day", + JSCalendarProperty::DelegatedFrom => "delegatedFrom", + JSCalendarProperty::DelegatedTo => "delegatedTo", + JSCalendarProperty::Description => "description", + JSCalendarProperty::DescriptionContentType => "descriptionContentType", + JSCalendarProperty::Display => "display", + JSCalendarProperty::Due => "due", + JSCalendarProperty::Duration => "duration", + JSCalendarProperty::Email => "email", + JSCalendarProperty::Entries => "entries", + JSCalendarProperty::EstimatedDuration => "estimatedDuration", + JSCalendarProperty::Excluded => "excluded", + JSCalendarProperty::ExpectReply => "expectReply", + JSCalendarProperty::Features => "features", + JSCalendarProperty::FirstDayOfWeek => "firstDayOfWeek", + JSCalendarProperty::FreeBusyStatus => "freeBusyStatus", + JSCalendarProperty::Frequency => "frequency", + JSCalendarProperty::HideAttendees => "hideAttendees", + JSCalendarProperty::Href => "href", + JSCalendarProperty::Id => "id", + JSCalendarProperty::Interval => "interval", + JSCalendarProperty::InvitedBy => "invitedBy", + JSCalendarProperty::IsDraft => "isDraft", + JSCalendarProperty::IsOrigin => "isOrigin", + JSCalendarProperty::Keywords => "keywords", + JSCalendarProperty::Kind => "kind", + JSCalendarProperty::Links => "links", + JSCalendarProperty::Locale => "locale", + JSCalendarProperty::Locations => "locations", + JSCalendarProperty::LocationTypes => "locationTypes", + JSCalendarProperty::MayInviteOthers => "mayInviteOthers", + JSCalendarProperty::MayInviteSelf => "mayInviteSelf", + JSCalendarProperty::MemberOf => "memberOf", + JSCalendarProperty::Method => "method", + JSCalendarProperty::Name => "name", + JSCalendarProperty::NthOfPeriod => "nthOfPeriod", + JSCalendarProperty::Offset => "offset", + JSCalendarProperty::Participants => "participants", + JSCalendarProperty::ParticipationComment => "participationComment", + JSCalendarProperty::ParticipationStatus => "participationStatus", + JSCalendarProperty::PercentComplete => "percentComplete", + JSCalendarProperty::Priority => "priority", + JSCalendarProperty::Privacy => "privacy", + JSCalendarProperty::ProdId => "prodId", + JSCalendarProperty::Progress => "progress", + JSCalendarProperty::RecurrenceId => "recurrenceId", + JSCalendarProperty::RecurrenceIdTimeZone => "recurrenceIdTimeZone", + JSCalendarProperty::RecurrenceOverrides => "recurrenceOverrides", + JSCalendarProperty::Rel => "rel", + JSCalendarProperty::RelatedTo => "relatedTo", + JSCalendarProperty::Relation => "relation", + JSCalendarProperty::RelativeTo => "relativeTo", + JSCalendarProperty::ReplyTo => "replyTo", + JSCalendarProperty::RequestStatus => "requestStatus", + JSCalendarProperty::Roles => "roles", + JSCalendarProperty::Rscale => "rscale", + JSCalendarProperty::SentBy => "sentBy", + JSCalendarProperty::ScheduleAgent => "scheduleAgent", + JSCalendarProperty::ScheduleForceSend => "scheduleForceSend", + JSCalendarProperty::ScheduleSequence => "scheduleSequence", + JSCalendarProperty::ScheduleStatus => "scheduleStatus", + JSCalendarProperty::ScheduleUpdated => "scheduleUpdated", + JSCalendarProperty::SendTo => "sendTo", + JSCalendarProperty::Sequence => "sequence", + JSCalendarProperty::ShowWithoutTime => "showWithoutTime", + JSCalendarProperty::Size => "size", + JSCalendarProperty::Skip => "skip", + JSCalendarProperty::Source => "source", + JSCalendarProperty::Start => "start", + JSCalendarProperty::Status => "status", + JSCalendarProperty::TimeZone => "timeZone", + JSCalendarProperty::Title => "title", + JSCalendarProperty::Trigger => "trigger", + JSCalendarProperty::Uid => "uid", + JSCalendarProperty::Until => "until", + JSCalendarProperty::Updated => "updated", + JSCalendarProperty::Uri => "uri", + JSCalendarProperty::UseDefaultAlerts => "useDefaultAlerts", + JSCalendarProperty::UtcEnd => "utcEnd", + JSCalendarProperty::UtcStart => "utcStart", + JSCalendarProperty::Version => "version", + JSCalendarProperty::VirtualLocations => "virtualLocations", + JSCalendarProperty::When => "when", + JSCalendarProperty::EndTimeZone => "endTimeZone", + JSCalendarProperty::MainLocationId => "mainLocationId", + JSCalendarProperty::OrganizerCalendarAddress => "organizerCalendarAddress", + JSCalendarProperty::RecurrenceRule => "recurrenceRule", + JSCalendarProperty::Properties => "properties", + JSCalendarProperty::Components => "components", + JSCalendarProperty::ValueType => "valueType", + JSCalendarProperty::ConvertedProperties => "convertedProperties", + JSCalendarProperty::Parameters => "parameters", + JSCalendarProperty::ICalendar => "iCalendar", + JSCalendarProperty::BlobId => "blobId", + JSCalendarProperty::Metadata => "metadata", + JSCalendarProperty::PrivateMetadata => "privateMetadata", + JSCalendarProperty::LinkDisplay(v) => v.as_str(), + JSCalendarProperty::VirtualLocationFeature(v) => v.as_str(), + JSCalendarProperty::ParticipantRole(v) => v.as_str(), + JSCalendarProperty::RelationValue(v) => v.as_str(), + JSCalendarProperty::LinkRelation(v) => v.as_str(), + JSCalendarProperty::DateTime(dt) => return dt.to_rfc3339().into(), + JSCalendarProperty::Pointer(pointer) => return Cow::Owned(pointer.to_string()), + JSCalendarProperty::IdValue(id) => return id.to_string().into(), + JSCalendarProperty::IdReference(s) => return format!("#{}", s).into(), + } + .into() + } + + #[inline] + pub(crate) fn static_name(&self) -> Option<&'static str> { + match self { + JSCalendarProperty::DateTime(_) + | JSCalendarProperty::Pointer(_) + | JSCalendarProperty::IdValue(_) + | JSCalendarProperty::IdReference(_) => None, + JSCalendarProperty::LinkRelation(v) => Some(v.as_str()), + property => match property.to_string() { + Cow::Borrowed(name) => Some(name), + Cow::Owned(_) => None, + }, + } + } + + #[inline] + pub(crate) fn has_data(&self) -> bool { + match self { + JSCalendarProperty::DateTime(_) + | JSCalendarProperty::LinkDisplay(_) + | JSCalendarProperty::VirtualLocationFeature(_) + | JSCalendarProperty::ParticipantRole(_) + | JSCalendarProperty::RelationValue(_) + | JSCalendarProperty::LinkRelation(_) + | JSCalendarProperty::Pointer(_) + | JSCalendarProperty::IdValue(_) + | JSCalendarProperty::IdReference(_) => true, + JSCalendarProperty::Id + | JSCalendarProperty::BaseEventId + | JSCalendarProperty::CalendarIds + | JSCalendarProperty::IsDraft + | JSCalendarProperty::IsOrigin + | JSCalendarProperty::UtcStart + | JSCalendarProperty::UtcEnd + | JSCalendarProperty::UseDefaultAlerts + | JSCalendarProperty::MayInviteSelf + | JSCalendarProperty::MayInviteOthers + | JSCalendarProperty::HideAttendees + | JSCalendarProperty::BlobId + | JSCalendarProperty::Metadata + | JSCalendarProperty::PrivateMetadata + | JSCalendarProperty::Type + | JSCalendarProperty::Acknowledged + | JSCalendarProperty::Action + | JSCalendarProperty::Alerts + | JSCalendarProperty::ByDay + | JSCalendarProperty::ByHour + | JSCalendarProperty::ByMinute + | JSCalendarProperty::ByMonth + | JSCalendarProperty::ByMonthDay + | JSCalendarProperty::BySecond + | JSCalendarProperty::BySetPosition + | JSCalendarProperty::ByWeekNo + | JSCalendarProperty::ByYearDay + | JSCalendarProperty::CalendarAddress + | JSCalendarProperty::Categories + | JSCalendarProperty::Color + | JSCalendarProperty::ContentType + | JSCalendarProperty::Coordinates + | JSCalendarProperty::Count + | JSCalendarProperty::Created + | JSCalendarProperty::Day + | JSCalendarProperty::DelegatedFrom + | JSCalendarProperty::DelegatedTo + | JSCalendarProperty::Description + | JSCalendarProperty::DescriptionContentType + | JSCalendarProperty::Display + | JSCalendarProperty::Due + | JSCalendarProperty::Duration + | JSCalendarProperty::Email + | JSCalendarProperty::Entries + | JSCalendarProperty::EstimatedDuration + | JSCalendarProperty::Excluded + | JSCalendarProperty::ExpectReply + | JSCalendarProperty::Features + | JSCalendarProperty::FirstDayOfWeek + | JSCalendarProperty::FreeBusyStatus + | JSCalendarProperty::Frequency + | JSCalendarProperty::Href + | JSCalendarProperty::Interval + | JSCalendarProperty::InvitedBy + | JSCalendarProperty::Keywords + | JSCalendarProperty::Kind + | JSCalendarProperty::Links + | JSCalendarProperty::Locale + | JSCalendarProperty::Locations + | JSCalendarProperty::LocationTypes + | JSCalendarProperty::MemberOf + | JSCalendarProperty::Method + | JSCalendarProperty::Name + | JSCalendarProperty::NthOfPeriod + | JSCalendarProperty::Offset + | JSCalendarProperty::Participants + | JSCalendarProperty::ParticipationComment + | JSCalendarProperty::ParticipationStatus + | JSCalendarProperty::PercentComplete + | JSCalendarProperty::Priority + | JSCalendarProperty::Privacy + | JSCalendarProperty::ProdId + | JSCalendarProperty::Progress + | JSCalendarProperty::RecurrenceId + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::Rel + | JSCalendarProperty::RelatedTo + | JSCalendarProperty::Relation + | JSCalendarProperty::RelativeTo + | JSCalendarProperty::ReplyTo + | JSCalendarProperty::RequestStatus + | JSCalendarProperty::Roles + | JSCalendarProperty::Rscale + | JSCalendarProperty::SentBy + | JSCalendarProperty::ScheduleAgent + | JSCalendarProperty::ScheduleForceSend + | JSCalendarProperty::ScheduleSequence + | JSCalendarProperty::ScheduleStatus + | JSCalendarProperty::ScheduleUpdated + | JSCalendarProperty::SendTo + | JSCalendarProperty::Sequence + | JSCalendarProperty::ShowWithoutTime + | JSCalendarProperty::Size + | JSCalendarProperty::Skip + | JSCalendarProperty::Source + | JSCalendarProperty::Start + | JSCalendarProperty::Status + | JSCalendarProperty::TimeZone + | JSCalendarProperty::Title + | JSCalendarProperty::Trigger + | JSCalendarProperty::Uid + | JSCalendarProperty::Until + | JSCalendarProperty::Updated + | JSCalendarProperty::Uri + | JSCalendarProperty::Version + | JSCalendarProperty::VirtualLocations + | JSCalendarProperty::When + | JSCalendarProperty::EndTimeZone + | JSCalendarProperty::MainLocationId + | JSCalendarProperty::OrganizerCalendarAddress + | JSCalendarProperty::RecurrenceRule + | JSCalendarProperty::ICalendar + | JSCalendarProperty::Properties + | JSCalendarProperty::Parameters + | JSCalendarProperty::ConvertedProperties + | JSCalendarProperty::ValueType + | JSCalendarProperty::Components => false, + } + } + + pub(crate) fn is_forbidden_override_patch(&self) -> bool { + matches!( + self, + JSCalendarProperty::Type + | JSCalendarProperty::Method + | JSCalendarProperty::OrganizerCalendarAddress + | JSCalendarProperty::Privacy + | JSCalendarProperty::ProdId + | JSCalendarProperty::RecurrenceId + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::SentBy + | JSCalendarProperty::Uid + | JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::RecurrenceRule + ) + } + + pub(crate) fn is_forbidden_override_pointer(pointer: &JsonPointer) -> bool { + let mut items = pointer + .iter() + .filter(|item| !matches!(item, JsonPointerItem::Root)); + + match (items.next(), items.next(), items.next(), items.next()) { + ( + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceOverrides | JSCalendarProperty::RecurrenceRule, + ))), + .., + ) => true, + (Some(JsonPointerItem::Key(Key::Property(property))), None, ..) => { + property.is_forbidden_override_patch() + } + ( + Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants))), + Some(_), + Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::CalendarAddress))), + None, + ) => true, + _ => false, + } + } +} + +impl FromStr for JSCalendarType { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarType, + "Event" => JSCalendarType::Event, + "Task" => JSCalendarType::Task, + "Group" => JSCalendarType::Group, + "Alert" => JSCalendarType::Alert, + "Boolean" => JSCalendarType::Boolean, + "Duration" => JSCalendarType::Duration, + "Id" => JSCalendarType::Id, + "Int" => JSCalendarType::Int, + "LocalDateTime" => JSCalendarType::LocalDateTime, + "Link" => JSCalendarType::Link, + "Location" => JSCalendarType::Location, + "NDay" => JSCalendarType::NDay, + "Number" => JSCalendarType::Number, + "Participant" => JSCalendarType::Participant, + "PatchObject" => JSCalendarType::PatchObject, + "RecurrenceRule" => JSCalendarType::RecurrenceRule, + "Relation" => JSCalendarType::Relation, + "SignedDuration" => JSCalendarType::SignedDuration, + "String" => JSCalendarType::String, + "TimeZone" => JSCalendarType::TimeZone, + "TimeZoneId" => JSCalendarType::TimeZoneId, + "TimeZoneRule" => JSCalendarType::TimeZoneRule, + "UnsignedInt" => JSCalendarType::UnsignedInt, + "UTCDateTime" => JSCalendarType::UTCDateTime, + "VirtualLocation" => JSCalendarType::VirtualLocation, + "ICalComponent" => JSCalendarType::ICalComponent, + "AbsoluteTrigger" => JSCalendarType::AbsoluteTrigger, + "OffsetTrigger" => JSCalendarType::OffsetTrigger, + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarType { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarType::Event => "Event", + JSCalendarType::Task => "Task", + JSCalendarType::Group => "Group", + JSCalendarType::Alert => "Alert", + JSCalendarType::Boolean => "Boolean", + JSCalendarType::Duration => "Duration", + JSCalendarType::Id => "Id", + JSCalendarType::Int => "Int", + JSCalendarType::LocalDateTime => "LocalDateTime", + JSCalendarType::Link => "Link", + JSCalendarType::Location => "Location", + JSCalendarType::NDay => "NDay", + JSCalendarType::Number => "Number", + JSCalendarType::Participant => "Participant", + JSCalendarType::PatchObject => "PatchObject", + JSCalendarType::RecurrenceRule => "RecurrenceRule", + JSCalendarType::Relation => "Relation", + JSCalendarType::SignedDuration => "SignedDuration", + JSCalendarType::String => "String", + JSCalendarType::TimeZone => "TimeZone", + JSCalendarType::TimeZoneId => "TimeZoneId", + JSCalendarType::TimeZoneRule => "TimeZoneRule", + JSCalendarType::UnsignedInt => "UnsignedInt", + JSCalendarType::UTCDateTime => "UTCDateTime", + JSCalendarType::VirtualLocation => "VirtualLocation", + JSCalendarType::ICalComponent => "ICalComponent", + JSCalendarType::OffsetTrigger => "OffsetTrigger", + JSCalendarType::AbsoluteTrigger => "AbsoluteTrigger", + } + } +} + +impl FromStr for JSCalendarAlertAction { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarAlertAction, + "display" => JSCalendarAlertAction::Display, + "email" => JSCalendarAlertAction::Email + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarAlertAction { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarAlertAction::Display => "display", + JSCalendarAlertAction::Email => "email", + } + } +} + +impl FromStr for JSCalendarLinkDisplay { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarLinkDisplay, + "badge" => JSCalendarLinkDisplay::Badge, + "graphic" => JSCalendarLinkDisplay::Graphic, + "fullsize" => JSCalendarLinkDisplay::Fullsize, + "thumbnail" => JSCalendarLinkDisplay::Thumbnail + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarLinkDisplay { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarLinkDisplay::Badge => "badge", + JSCalendarLinkDisplay::Graphic => "graphic", + JSCalendarLinkDisplay::Fullsize => "fullsize", + JSCalendarLinkDisplay::Thumbnail => "thumbnail", + } + } +} + +impl FromStr for JSCalendarVirtualLocationFeature { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarVirtualLocationFeature, + "audio" => JSCalendarVirtualLocationFeature::Audio, + "chat" => JSCalendarVirtualLocationFeature::Chat, + "feed" => JSCalendarVirtualLocationFeature::Feed, + "moderator" => JSCalendarVirtualLocationFeature::Moderator, + "phone" => JSCalendarVirtualLocationFeature::Phone, + "screen" => JSCalendarVirtualLocationFeature::Screen, + "video" => JSCalendarVirtualLocationFeature::Video + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarVirtualLocationFeature { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarVirtualLocationFeature::Audio => "audio", + JSCalendarVirtualLocationFeature::Chat => "chat", + JSCalendarVirtualLocationFeature::Feed => "feed", + JSCalendarVirtualLocationFeature::Moderator => "moderator", + JSCalendarVirtualLocationFeature::Phone => "phone", + JSCalendarVirtualLocationFeature::Screen => "screen", + JSCalendarVirtualLocationFeature::Video => "video", + } + } +} + +impl FromStr for JSCalendarFreeBusyStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarFreeBusyStatus, + "free" => JSCalendarFreeBusyStatus::Free, + "busy" => JSCalendarFreeBusyStatus::Busy + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarFreeBusyStatus { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarFreeBusyStatus::Free => "free", + JSCalendarFreeBusyStatus::Busy => "busy", + } + } +} + +impl FromStr for JSCalendarParticipantKind { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarParticipantKind, + "individual" => JSCalendarParticipantKind::Individual, + "group" => JSCalendarParticipantKind::Group, + "resource" => JSCalendarParticipantKind::Resource, + "location" => JSCalendarParticipantKind::Location + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarParticipantKind { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarParticipantKind::Individual => "individual", + JSCalendarParticipantKind::Group => "group", + JSCalendarParticipantKind::Resource => "resource", + JSCalendarParticipantKind::Location => "location", + } + } +} + +impl FromStr for JSCalendarParticipationStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarParticipationStatus, + "needs-action" => JSCalendarParticipationStatus::NeedsAction, + "accepted" => JSCalendarParticipationStatus::Accepted, + "declined" => JSCalendarParticipationStatus::Declined, + "tentative" => JSCalendarParticipationStatus::Tentative, + "delegated" => JSCalendarParticipationStatus::Delegated + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarParticipationStatus { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarParticipationStatus::NeedsAction => "needs-action", + JSCalendarParticipationStatus::Accepted => "accepted", + JSCalendarParticipationStatus::Declined => "declined", + JSCalendarParticipationStatus::Tentative => "tentative", + JSCalendarParticipationStatus::Delegated => "delegated", + } + } +} + +impl FromStr for JSCalendarPrivacy { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarPrivacy, + "public" => JSCalendarPrivacy::Public, + "private" => JSCalendarPrivacy::Private, + "secret" => JSCalendarPrivacy::Secret + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarPrivacy { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarPrivacy::Public => "public", + JSCalendarPrivacy::Private => "private", + JSCalendarPrivacy::Secret => "secret", + } + } +} + +impl FromStr for JSCalendarProgress { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarProgress, + "needs-action" => JSCalendarProgress::NeedsAction, + "in-process" => JSCalendarProgress::InProcess, + "completed" => JSCalendarProgress::Completed, + "failed" => JSCalendarProgress::Failed, + "cancelled" => JSCalendarProgress::Cancelled, + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarProgress { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarProgress::NeedsAction => "needs-action", + JSCalendarProgress::InProcess => "in-process", + JSCalendarProgress::Completed => "completed", + JSCalendarProgress::Failed => "failed", + JSCalendarProgress::Cancelled => "cancelled", + } + } +} + +impl FromStr for JSCalendarRelation { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarRelation, + "first" => JSCalendarRelation::First, + "next" => JSCalendarRelation::Next, + "child" => JSCalendarRelation::Child, + "parent" => JSCalendarRelation::Parent, + "snooze" => JSCalendarRelation::Snooze, + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarRelation { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarRelation::First => "first", + JSCalendarRelation::Next => "next", + JSCalendarRelation::Child => "child", + JSCalendarRelation::Parent => "parent", + JSCalendarRelation::Snooze => "snooze", + } + } +} + +impl FromStr for JSCalendarRelativeTo { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarRelativeTo, + "start" => JSCalendarRelativeTo::Start, + "end" => JSCalendarRelativeTo::End + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarRelativeTo { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarRelativeTo::Start => "start", + JSCalendarRelativeTo::End => "end", + } + } +} + +impl FromStr for JSCalendarParticipantRole { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarParticipantRole, + "owner" => JSCalendarParticipantRole::Owner, + "optional" => JSCalendarParticipantRole::Optional, + "informational" => JSCalendarParticipantRole::Informational, + "chair" => JSCalendarParticipantRole::Chair, + "required" => JSCalendarParticipantRole::Required, + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarParticipantRole { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarParticipantRole::Owner => "owner", + JSCalendarParticipantRole::Optional => "optional", + JSCalendarParticipantRole::Informational => "informational", + JSCalendarParticipantRole::Chair => "chair", + JSCalendarParticipantRole::Required => "required", + } + } +} + +impl FromStr for JSCalendarScheduleAgent { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarScheduleAgent, + "server" => JSCalendarScheduleAgent::Server, + "client" => JSCalendarScheduleAgent::Client, + "none" => JSCalendarScheduleAgent::None + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarScheduleAgent { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarScheduleAgent::Server => "server", + JSCalendarScheduleAgent::Client => "client", + JSCalendarScheduleAgent::None => "none", + } + } +} + +impl FromStr for JSCalendarEventStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSCalendarEventStatus, + "confirmed" => JSCalendarEventStatus::Confirmed, + "cancelled" => JSCalendarEventStatus::Cancelled, + "tentative" => JSCalendarEventStatus::Tentative + ) + .copied() + .ok_or(()) + } +} + +impl JSCalendarEventStatus { + pub fn as_str(&self) -> &'static str { + match self { + JSCalendarEventStatus::Confirmed => "confirmed", + JSCalendarEventStatus::Cancelled => "cancelled", + JSCalendarEventStatus::Tentative => "tentative", + } + } +} + +impl ICalendarFrequency { + pub fn as_js_str(&self) -> &'static str { + match self { + ICalendarFrequency::Secondly => "secondly", + ICalendarFrequency::Minutely => "minutely", + ICalendarFrequency::Hourly => "hourly", + ICalendarFrequency::Daily => "daily", + ICalendarFrequency::Weekly => "weekly", + ICalendarFrequency::Monthly => "monthly", + ICalendarFrequency::Yearly => "yearly", + } + } +} + +impl ICalendarSkip { + pub fn as_js_str(&self) -> &'static str { + match self { + ICalendarSkip::Omit => "omit", + ICalendarSkip::Backward => "backward", + ICalendarSkip::Forward => "forward", + } + } +} + +impl ICalendarWeekday { + pub fn as_js_str(&self) -> &'static str { + match self { + ICalendarWeekday::Sunday => "su", + ICalendarWeekday::Monday => "mo", + ICalendarWeekday::Tuesday => "tu", + ICalendarWeekday::Wednesday => "we", + ICalendarWeekday::Thursday => "th", + ICalendarWeekday::Friday => "fr", + ICalendarWeekday::Saturday => "sa", + } + } +} + +impl ICalendarMethod { + pub fn as_js_str(&self) -> &'static str { + match self { + ICalendarMethod::Publish => "publish", + ICalendarMethod::Request => "request", + ICalendarMethod::Reply => "reply", + ICalendarMethod::Add => "add", + ICalendarMethod::Cancel => "cancel", + ICalendarMethod::Refresh => "refresh", + ICalendarMethod::Counter => "counter", + ICalendarMethod::Declinecounter => "declinecounter", + } + } +} diff --git a/crates/calcard/src/jscontact/export/convert.rs b/crates/calcard/src/jscontact/export/convert.rs new file mode 100644 index 00000000..56e00992 --- /dev/null +++ b/crates/calcard/src/jscontact/export/convert.rs @@ -0,0 +1,2143 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + Data, IanaParse, IanaType, + blob::BlobResolver, + export::ExportError, + jsprop::{ordered::OrderedMap, text::ConvertedKeys}, + timezone::Tz, + }, + jscontact::{ + JSContact, JSContactId, JSContactKind, JSContactProperty, JSContactValue, + export::{ + ExportContext, ExportOptions, State, + props::{ + U32Value, build_path, convert_anniversary, convert_value, find_text_param, map_kind, + }, + }, + }, + vcard::{ + Jscomp, VCard, VCardEntry, VCardLevel, VCardParameter, VCardParameterValue, VCardPhonetic, + VCardProperty, VCardType, VCardValue, VCardValueType, ValueType, + }, +}; +use jmap_tools::{JsonPointer, Key, Map, Value}; +use smallvec::smallvec; +use std::{collections::HashMap, str::FromStr}; + +impl<'x, I, B> JSContact<'x, I, B> +where + I: JSContactId, + B: JSContactId, +{ + pub fn into_vcard(self) -> Result { + self.into_vcard_with(ExportOptions::default()) + } + + pub fn into_vcard_with>( + self, + mut options: ExportOptions, + ) -> Result { + let context = options.context::(); + self.build_vcard(context) + } + + fn build_vcard(self, mut context: ExportContext<'_, B>) -> Result { + if let Some(blobs) = &mut context.blobs { + blobs.add_uses(self.blob_ids()); + } + let mut state = State::::default(); + let mut properties = self + .0 + .into_object() + .ok_or(ExportError::NotGroup)? + .into_vec(); + state.vcard.entries.reserve( + properties + .iter() + .map(|(key, value)| match (key, value) { + (Key::Property(JSContactProperty::VCard), Value::Object(obj)) => obj + .iter() + .filter_map(|(_, value)| value.as_array()) + .map(<[_]>::len) + .sum(), + ( + Key::Property( + JSContactProperty::Metadata | JSContactProperty::PrivateMetadata, + ), + _, + ) => 0, + (_, Value::Object(obj)) => obj.len(), + _ => 1, + }) + .sum::() + + 1, + ); + let mut localized_properties: OrderedMap> = OrderedMap::default(); + + for (property, value) in &mut properties { + match (property, value) { + (Key::Property(JSContactProperty::Localizations), Value::Object(obj)) => { + for (lang, value) in std::mem::take(obj.as_mut_vec()) { + let mut localizations: Value< + '_, + JSContactProperty, + JSContactValue, + > = Value::Object(Map::from(Vec::with_capacity(3))); + for (key, value) in value.into_expanded_object() { + let ptr = match key { + Key::Property(JSContactProperty::Pointer(ptr)) => ptr, + _ => JsonPointer::parse(key.to_string().as_ref()), + }; + + let ptr_path = ptr.to_string(); + if let Some(value) = build_path( + &mut localizations, + ptr.into_inner().into_iter().peekable(), + value, + ) { + state.insert_jsprop( + &[ + JSContactProperty::Localizations::.to_string().as_ref(), + lang.to_string().as_ref(), + ptr_path.as_str(), + ], + value, + ); + } + } + if let Some(localizations) = + localizations.into_object().filter(|x| !x.is_empty()) + { + localized_properties + .insert(lang.into_string(), localizations.into_vec()); + } + } + } + (Key::Property(JSContactProperty::VCard), Value::Object(obj)) => { + for (sub_property, value) in std::mem::take(obj.as_mut_vec()) { + match sub_property { + Key::Property(JSContactProperty::ConvertedProperties) => { + for (key, value) in value.into_expanded_object() { + state + .converted_props + .push((key.into_converted_keys(), value)); + } + } + Key::Property(JSContactProperty::Properties) => { + if let Some(value) = value.into_array() { + state.import_properties(value); + } + } + _ => { + state.insert_jsprop( + &[ + JSContactProperty::VCard::.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + if !state.converted_props.is_empty() { + state.converted_props.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + } + } + _ => {} + } + } + + for (keys, _) in &state.converted_props { + for key in keys { + if let Key::Property(property) = key + && !state.converted_heads.contains_key(property) + { + state.converted_heads.push(property.clone(), ()); + } + } + } + + // Resolve organizational mappings + let mut org_mappings: HashMap = HashMap::new(); + for (keys, value) in &state.converted_props { + match (keys.first(), keys.get(2)) { + ( + Some(Key::Property(JSContactProperty::Titles)), + Some(Key::Property(JSContactProperty::Name)), + ) => { + if let Some(group) = find_text_param(value, "group") { + org_mappings.entry(group.into_owned()).or_default().0 = + keys[1].to_string().into_owned(); + } + } + (Some(Key::Property(JSContactProperty::Organizations)), _) => { + if let Some(group) = find_text_param(value, "group") { + org_mappings.entry(group.into_owned()).or_default().1 = true; + } + } + _ => {} + } + } + + // Localization maps + let mut has_fn = false; + + let has_localizations = !localized_properties.is_empty(); + let mut name_pos_map: [Option; 20] = [None; 20]; + let mut adr_pos_map: HashMap; 20]> = HashMap::new(); + let mut places_map: HashMap)> = + HashMap::new(); + + for (properties, language) in [(properties, None)].into_iter().chain( + localized_properties + .into_iter() + .map(|(lang, properties)| (properties, Some(lang))), + ) { + let is_localized = language.is_some(); + state.language = language; + + for (property, value) in properties { + let Key::Property(property) = property else { + state.insert_jsprop(&[property.to_string().as_ref()], value); + continue; + }; + + match property { + JSContactProperty::Uid + | JSContactProperty::Kind + | JSContactProperty::Language + | JSContactProperty::ProdId + | JSContactProperty::Created + | JSContactProperty::Updated => { + match convert_value(value, &ValueType::Vcard(VCardValueType::Text)) { + Ok(value) => { + let vcard_property = match property { + JSContactProperty::Uid => VCardProperty::Uid, + JSContactProperty::Kind => VCardProperty::Kind, + JSContactProperty::Language => VCardProperty::Language, + JSContactProperty::ProdId => VCardProperty::Prodid, + JSContactProperty::Created => VCardProperty::Created, + JSContactProperty::Updated => VCardProperty::Rev, + _ => unreachable!(), + }; + state.insert_vcard( + &[property], + VCardEntry::new(vcard_property).with_value(value), + ); + } + Err(value) => { + state.insert_jsprop(&[property.to_string().as_ref()], value); + } + } + } + JSContactProperty::Directories => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new( + map_kind(&value, [(JSContactKind::Entry, VCardProperty::Source)]) + .unwrap_or(VCardProperty::OrgDirectory), + ); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Kind) => {} + Key::Property(JSContactProperty::Uri) => { + if let Some(text) = value.into_string() { + entry.values.push(VCardValue::Text(text.into_owned())); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.params.push(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::ListAs) + if value.as_index().is_some() => + { + if let Some(index) = value.as_index() { + entry.params.push(VCardParameter::index(index)); + } + } + Key::Property(JSContactProperty::MediaType) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::mediatype(text.into_owned())); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::label(text.into_owned())); + } + } + Key::Property(JSContactProperty::Contexts) + if entry.name == VCardProperty::OrgDirectory => + { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Directories], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Anniversaries => { + let mut unconverted = Vec::new(); + let mut has_converted = false; + + for (name, value) in value.into_expanded_object() { + if let Some((vcard_property, mut vcard_place_property)) = map_kind( + &value, + [ + ( + JSContactKind::Birth, + (VCardProperty::Bday, Some(VCardProperty::Birthplace)), + ), + ( + JSContactKind::Death, + (VCardProperty::Deathdate, Some(VCardProperty::Deathplace)), + ), + (JSContactKind::Wedding, (VCardProperty::Anniversary, None)), + ], + ) + .or_else(|| { + if is_localized { + places_map.get(name.to_string().as_ref()).cloned() + } else { + None + } + }) { + let mut date = None; + let mut place = None; + let mut calscale = None; + let mut members = Vec::new(); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Date) => { + match convert_anniversary(value) { + Ok((date_, calscale_)) => { + date = Some(date_); + calscale = calscale_; + } + Err(value) => members.push((sub_property, value)), + } + } + Key::Property(JSContactProperty::Place) + if vcard_place_property.is_some() + && value.as_object().is_some_and(|place| { + place.iter().any(|(key, value)| { + matches!( + key, + Key::Property( + JSContactProperty::Full + | JSContactProperty::Coordinates + ) + ) && matches!( + value, + Value::Str(_) | Value::Element(_) + ) + }) + }) => + { + if has_localizations && !is_localized { + places_map + .entry(name.to_string().into_owned()) + .or_insert_with(|| { + ( + vcard_property.clone(), + vcard_place_property.clone(), + ) + }); + } + + for (place_property, value) in + value.into_expanded_object() + { + match place_property { + Key::Property(JSContactProperty::Full) + if vcard_place_property.is_some() => + { + if let Some(text) = value.into_string() { + place = Some( + VCardEntry::new( + vcard_place_property + .take() + .unwrap(), + ) + .with_value(text.into_owned()), + ); + } + } + Key::Property( + JSContactProperty::Coordinates, + ) if vcard_place_property.is_some() => { + if let Some(text) = value.into_string() { + place = Some( + VCardEntry::new( + vcard_place_property + .take() + .unwrap(), + ) + .with_value(text.into_owned()) + .with_param(VCardParameter::value( + VCardValueType::Uri, + )), + ); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + place_property.to_string().as_ref(), + ], + value, + ); + } + } + } + } + _ => members.push((sub_property, value)), + } + } + + if date.is_none() && place.is_none() && !is_localized { + unconverted + .push((name, Value::Object(members.into_iter().collect()))); + continue; + } + + for (sub_property, value) in members { + if sub_property != Key::Property(JSContactProperty::Kind) { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + + // Add place if it exists + if let Some(place) = place { + state.insert_vcard( + &[ + JSContactProperty::Anniversaries, + JSContactProperty::Place, + ], + place.with_param(VCardParameter::prop_id( + name.to_string().into_owned(), + )), + ); + } + + // Add date + if let Some(date) = date { + let mut entry = VCardEntry::new(vcard_property); + if let Some(calscale) = calscale { + entry.params.push(VCardParameter::calscale(calscale)); + } + state.insert_vcard( + &[ + JSContactProperty::Anniversaries, + JSContactProperty::Date, + ], + entry + .with_param(VCardParameter::prop_id(name.into_string())) + .with_value(VCardValue::PartialDateTime(date)), + ); + } + has_converted = true; + } else { + unconverted.push((name, value)); + } + } + + if has_converted || is_localized { + for (name, value) in unconverted { + state.insert_jsprop( + &[property.to_string().as_ref(), name.to_string().as_ref()], + value, + ); + } + } else if !unconverted.is_empty() { + state.insert_jsprop( + &[property.to_string().as_ref()], + Value::Object(unconverted.into_iter().collect()), + ); + } + } + JSContactProperty::Name => { + let mut params = Vec::new(); + let mut parts: Vec>> = vec![None; 7]; + let mut jscomps = vec![Jscomp::Separator(String::new())]; + let mut num_parts = 0; + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Components) => { + let items = value.into_array().unwrap_or_default(); + jscomps.reserve(items.len()); + for (index, components) in items.into_iter().enumerate() { + let mut comp_value = None; + let mut comp_phonetic = None; + let mut comp_pos = None; + let mut is_separator = false; + + for (component, value) in components.into_expanded_object() + { + match (component, value) { + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind( + JSContactKind::Separator, + )), + ) => { + is_separator = true; + } + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ) => { + comp_pos = kind.to_vcard_n_pos(); + } + ( + Key::Property(JSContactProperty::Value), + value, + ) => { + comp_value = value.into_string(); + } + ( + Key::Property(JSContactProperty::Phonetic), + value, + ) if is_localized => { + comp_phonetic = value.into_string(); + } + (component, value) => { + state.insert_jsprop( + &[ + JSContactProperty::Name:: + .to_string() + .as_ref(), + JSContactProperty::Components:: + .to_string() + .as_ref(), + index.to_string().as_str(), + component.to_string().as_ref(), + ], + value, + ); + } + } + } + + if has_localizations + && !is_localized + && let Some(value) = name_pos_map.get_mut(index) + { + *value = comp_pos; + } + + if let Some(comp_value) = comp_value.or(comp_phonetic) { + if let Some(comp_pos) = comp_pos.or_else(|| { + if is_localized && !is_separator { + name_pos_map + .get(index) + .copied() + .unwrap_or_default() + } else { + None + } + }) { + let part = &mut parts[comp_pos]; + if let Some(part) = part { + let value = part.len() as u32; + part.push(comp_value.into_owned()); + jscomps.push(Jscomp::Entry { + position: comp_pos as u32, + value, + }); + } else { + *part = Some(vec![comp_value.into_owned()]); + jscomps.push(Jscomp::Entry { + position: comp_pos as u32, + value: 0, + }); + } + num_parts = num_parts.max(comp_pos + 1); + } else if is_separator { + jscomps.push(Jscomp::Separator( + comp_value.into_owned(), + )); + } + } + } + } + Key::Property(JSContactProperty::Full) => { + if let Some(text) = value.into_string() { + has_fn = true; + state.insert_vcard( + &[JSContactProperty::Name, JSContactProperty::Full], + VCardEntry::new(VCardProperty::Fn) + .with_value(text.into_owned()), + ); + } + } + Key::Property(JSContactProperty::SortAs) => { + let mut surname = None; + let mut given = None; + + for (key, value) in value.into_expanded_object() { + match key { + Key::Property(JSContactProperty::SortAsKind( + JSContactKind::Surname, + )) => { + surname = value.into_string(); + } + Key::Property(JSContactProperty::SortAsKind( + JSContactKind::Given, + )) => { + given = value.into_string(); + } + _ => { + state.insert_jsprop( + &[ + JSContactProperty::Name:: + .to_string() + .as_ref(), + JSContactProperty::SortAs:: + .to_string() + .as_ref(), + key.to_string().as_ref(), + ], + value, + ); + } + } + } + + if let (Some(surname), Some(given)) = + (surname.as_ref(), given.as_ref()) + { + params.push(VCardParameter::sort_as(format!( + "{surname},{given}" + ))); + } else if let Some(surname) = surname.or(given) { + params.push(VCardParameter::sort_as(surname.into_owned())); + } + } + Key::Property(JSContactProperty::PhoneticSystem) => { + if let Ok(phonetic_system) = + IanaType::::try_from(value) + { + params.push(VCardParameter::phonetic(phonetic_system)); + } + } + Key::Property(JSContactProperty::PhoneticScript) => { + if let Some(phonetic_script) = value.into_string() { + params.push(VCardParameter::script( + phonetic_script.into_owned(), + )); + } + } + Key::Property(JSContactProperty::DefaultSeparator) => { + if let Some(separator) = value.into_string() { + jscomps[0] = Jscomp::Separator(separator.into_owned()); + } + } + Key::Property(JSContactProperty::IsOrdered) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + if !has_fn { + let default_separator = match jscomps.first() { + Some(Jscomp::Separator(sep)) if !sep.is_empty() => sep.as_str(), + _ => " ", + }; + let mut full = String::new(); + let mut prev_value = false; + for jscomp in jscomps.iter().skip(1) { + match jscomp { + Jscomp::Entry { position, value } => { + if let Some(component) = parts + .get(*position as usize) + .and_then(|part| part.as_ref()) + .and_then(|part| part.get(*value as usize)) + { + if prev_value { + full.push_str(default_separator); + } + full.push_str(component); + prev_value = true; + } + } + Jscomp::Separator(sep) => { + full.push_str(sep); + prev_value = false; + } + } + } + + if !full.is_empty() { + state.insert_vcard( + &[JSContactProperty::Name, JSContactProperty::Full], + VCardEntry::new(VCardProperty::Fn) + .with_param(VCardParameter::derived(true)) + .with_value(full), + ); + has_fn = true; + } + } + + if num_parts > 0 || !params.is_empty() { + if jscomps.len() > 1 { + params.push(VCardParameter::jscomps(jscomps)); + } + + // To vCard: add any "surname2" NameComponent to the Family name component, after all "surname" values. + // To vCard: add any "generation" NameComponent to the Honorific suffix component. + for (from_pos, to_pos) in [(5, 0), (6, 4)] { + if let Some(v) = &parts[from_pos] { + let v = v.clone(); + if let Some(to) = &mut parts[to_pos] { + to.extend(v); + } else { + parts[to_pos] = Some(v); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Name, JSContactProperty::Components], + VCardEntry::new(VCardProperty::N) + .with_params(params) + .with_values( + parts.into_iter().map(VCardValue::from_components), + ), + ); + } + } + JSContactProperty::SpeakToAs => { + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::GrammaticalGender) => { + if let Ok(gender) = convert_value(value, &ValueType::GramGender) + { + state.insert_vcard( + &[ + JSContactProperty::SpeakToAs, + JSContactProperty::GrammaticalGender, + ], + VCardEntry::new(VCardProperty::Gramgender) + .with_value(gender), + ); + } + } + Key::Property(JSContactProperty::Pronouns) => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Pronouns); + + for (pronoun_property, value) in + value.into_expanded_object() + { + match pronoun_property { + Key::Property(JSContactProperty::Pronouns) => { + if let Some(value) = value + .into_owned_string() + .map(VCardValue::Text) + { + entry.values = smallvec![value]; + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + sub_property.to_string().as_ref(), + name.to_string().as_ref(), + pronoun_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[ + JSContactProperty::SpeakToAs, + JSContactProperty::Pronouns, + ], + entry.with_param(VCardParameter::prop_id( + name.into_string(), + )), + ); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + } + JSContactProperty::Nicknames => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Nickname); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Name) => { + if let Some(name) = + value.into_owned_string().map(VCardValue::Text) + { + entry.values = smallvec![name]; + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Nicknames], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Media => { + let mut unconverted = Vec::new(); + let mut has_converted = false; + + for (name, value) in value.into_expanded_object() { + if let Some(entry_type) = map_kind( + &value, + [ + (JSContactKind::Photo, VCardProperty::Photo), + (JSContactKind::Logo, VCardProperty::Logo), + (JSContactKind::Sound, VCardProperty::Sound), + ], + ) { + if context.blobs.is_none() + && matches!( + value.as_object_and_get(&Key::Property( + JSContactProperty::BlobId + )), + Some( + Value::Element(JSContactValue::BlobId(_)) + | Value::Str(_) + ) + ) + { + unconverted.push((name, value)); + continue; + } + let mut entry = VCardEntry::new(entry_type); + let resolved = match value + .as_object_and_get(&Key::Property(JSContactProperty::BlobId)) + { + Some(Value::Element(JSContactValue::BlobId(blob_id))) => Some( + context + .blobs + .as_mut() + .and_then(|blobs| blobs.resolve(blob_id)) + .ok_or_else(|| blob_id.to_string()), + ), + Some(Value::Str(text)) => Some(Err(text.to_string())), + _ => None, + }; + let data = match resolved { + Some(Ok(data)) => { + if !context.embed(&data) { + continue; + } + Some(data) + } + Some(Err(blob_id)) => { + context.fail(ExportError::UnresolvedBlob { blob_id }); + continue; + } + None if matches!( + value.as_object_and_get(&Key::Property( + JSContactProperty::Uri + )), + Some(Value::Str(_)) + ) => + { + None + } + _ => { + unconverted.push((name, value)); + continue; + } + }; + let mut blob_media_type = None; + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::BlobId) + if data.is_some() => {} + Key::Property(JSContactProperty::Uri) if data.is_some() => { + if !value.as_str().is_some_and(|uri| { + uri.as_bytes().get(..5).is_some_and(|scheme| { + scheme.eq_ignore_ascii_case(b"data:") + }) + }) { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + Key::Property(JSContactProperty::MediaType) + if data.is_some() => + { + blob_media_type = value.into_owned_string(); + } + Key::Property(JSContactProperty::Uri) => { + if let Some(uri) = value.into_string() { + entry.values = match Data::try_parse(uri.as_bytes()) + { + Some(data) if context.embed(&data.data) => { + smallvec![VCardValue::Binary(Box::new( + data + ))] + } + Some(_) => continue, + None => { + smallvec![VCardValue::Text( + uri.into_owned() + )] + } + }; + } + } + Key::Property(JSContactProperty::MediaType) => { + if let Some(text) = value.into_string() { + entry.params.push(VCardParameter::mediatype( + text.into_owned(), + )); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::label(text.into_owned())); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Kind) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + if let Some(data) = data { + entry.values = smallvec![VCardValue::Binary(Box::new(Data { + content_type: blob_media_type, + data, + }))]; + } + + state.insert_vcard( + &[JSContactProperty::Media], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + has_converted = true; + } else { + unconverted.push((name, value)); + } + } + + if has_converted || state.language.is_some() { + for (name, value) in unconverted { + state.insert_jsprop( + &[property.to_string().as_ref(), name.to_string().as_ref()], + value, + ); + } + } else if !unconverted.is_empty() { + state.insert_jsprop( + &[property.to_string().as_ref()], + Value::Object(unconverted.into_iter().collect()), + ); + } + } + JSContactProperty::Addresses => { + for (name, value) in value.into_expanded_object() { + let mut params = vec![]; + let mut parts: Vec>> = vec![None; 18]; + let mut jscomps = vec![Jscomp::Separator(String::new())]; + let mut num_parts = 0; + let mut timezone = None; + let mut geo = None; + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Components) => { + let items = value.into_array().unwrap_or_default(); + jscomps.reserve(items.len()); + for (index, components) in items.into_iter().enumerate() { + let mut comp_value = None; + let mut comp_phonetic = None; + let mut comp_pos = None; + let mut is_separator = false; + + for (key, value) in components.into_expanded_object() { + match (key, value) { + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind( + JSContactKind::Separator, + )), + ) => { + is_separator = true; + } + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ) => { + comp_pos = kind.to_vcard_adr_pos(); + } + ( + Key::Property(JSContactProperty::Value), + value, + ) => { + comp_value = value.into_string(); + } + ( + Key::Property(JSContactProperty::Phonetic), + value, + ) if is_localized => { + comp_phonetic = value.into_string(); + } + (key, value) => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + index.to_string().as_str(), + key.to_string().as_ref(), + ], + value, + ); + } + } + } + + if has_localizations + && !is_localized + && let Some(value) = adr_pos_map + .entry(name.to_string().into_owned()) + .or_default() + .get_mut(index) + { + *value = comp_pos; + } + + if let Some(comp_value) = comp_value.or(comp_phonetic) { + if let Some(comp_pos) = comp_pos.or_else(|| { + if is_localized && !is_separator { + adr_pos_map + .get(name.to_string().as_ref()) + .and_then(|map| map.get(index).copied()) + .unwrap_or_default() + } else { + None + } + }) { + let part = &mut parts[comp_pos]; + if let Some(part) = part { + let value = part.len() as u32; + part.push(comp_value.into_owned()); + jscomps.push(Jscomp::Entry { + position: comp_pos as u32, + value, + }); + } else { + *part = Some(vec![comp_value.into_owned()]); + jscomps.push(Jscomp::Entry { + position: comp_pos as u32, + value: 0, + }); + } + num_parts = num_parts.max(comp_pos + 1); + } else if is_separator { + jscomps.push(Jscomp::Separator( + comp_value.into_owned(), + )); + } + } + } + } + Key::Property(JSContactProperty::Full) => { + if let Some(value) = value.into_string() { + params.push(VCardParameter::label(value.into_owned())); + } + } + Key::Property(JSContactProperty::Coordinates) => { + geo = value.into_string(); + } + Key::Property(JSContactProperty::TimeZone) => { + timezone = value.into_string(); + } + Key::Property(JSContactProperty::CountryCode) => { + if let Some(value) = value.into_string() { + params.push(VCardParameter::cc(value.into_owned())); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(value) = value.as_pref() { + params.push(VCardParameter::pref(value)); + } + } + Key::Property(JSContactProperty::PhoneticSystem) => { + if let Ok(phonetic_system) = + IanaType::::try_from(value) + { + params.push(VCardParameter::phonetic(phonetic_system)); + } + } + Key::Property(JSContactProperty::PhoneticScript) => { + if let Some(value) = value.into_string() { + params.push(VCardParameter::script(value.into_owned())); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &VCardProperty::Adr, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut params, + ); + } + Key::Property(JSContactProperty::DefaultSeparator) => { + if let Some(value) = value.into_string() { + jscomps[0] = Jscomp::Separator(value.into_owned()); + } + } + Key::Property(JSContactProperty::IsOrdered) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + if let Some(timezone) = timezone { + if state.converted_props.iter().any(|(prop, _)| { + prop.len() == 3 + && prop[0] == Key::Property(JSContactProperty::Addresses) + && prop[1] == name + && prop[2] == Key::Property(JSContactProperty::TimeZone) + }) { + state.insert_vcard( + &[ + JSContactProperty::Addresses, + JSContactProperty::TimeZone, + ], + VCardEntry::new(VCardProperty::Tz).with_value( + Tz::from_str(&timezone) + .map(|tz| { + VCardValue::PartialDateTime(tz.offset_parts()) + }) + .unwrap_or_else(|_| { + VCardValue::Text(timezone.into_owned()) + }), + ), + ); + } else { + params.push(VCardParameter::tz(timezone.into_owned())); + } + } + + if let Some(geo) = geo { + if state.converted_props.iter().any(|(prop, _)| { + prop.len() == 3 + && prop[0] == Key::Property(JSContactProperty::Addresses) + && prop[1] == name + && prop[2] == Key::Property(JSContactProperty::Coordinates) + }) { + state.insert_vcard( + &[ + JSContactProperty::Addresses, + JSContactProperty::Coordinates, + ], + VCardEntry::new(VCardProperty::Geo) + .with_value(geo.into_owned()), + ); + } else { + params.push(VCardParameter::geo(geo.into_owned())); + } + } + + if num_parts > 0 || !params.is_empty() { + params.push(VCardParameter::prop_id(name.into_string())); + + if jscomps.len() > 1 { + params.push(VCardParameter::jscomps(jscomps)); + } + + /* + + apartment To vCard: set the values of the following components: + room, floor, apartment, building + + name To vCard: set the values of the following components: + number, name, block, direction, landmark, subdistrict, district + + */ + + for (comp_target, source) in [ + (1, [7, 8, 9, 12].as_slice()), + (2, [10, 11, 13, 17, 16, 14, 15].as_slice()), + ] { + let mut target = vec![]; + + for comp_pos in source { + if let Some(v) = &parts[*comp_pos] { + target.extend(v.iter().cloned()); + } + } + + if !target.is_empty() { + parts[comp_target] = Some(target); + } + } + + state.insert_vcard( + &[JSContactProperty::Addresses, JSContactProperty::Components], + VCardEntry::new(VCardProperty::Adr) + .with_params(params) + .with_values( + parts.into_iter().map(VCardValue::from_components), + ), + ); + } + } + } + JSContactProperty::Organizations => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Org); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Name) => { + if let Some(value) = + value.into_owned_string().map(VCardValue::Text) + { + if entry.values.is_empty() { + entry.values = smallvec![value]; + } else { + entry.values.insert(0, value); + } + } + } + Key::Property(JSContactProperty::Units) => { + for (index, item) in value + .into_array() + .unwrap_or_default() + .into_iter() + .enumerate() + { + for (key, value) in item.into_expanded_object() { + match key { + Key::Property(JSContactProperty::Name) => { + if let Some(value) = value + .into_owned_string() + .map(VCardValue::Text) + { + entry.values.push(value); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + index.to_string().as_ref(), + key.to_string().as_ref(), + ], + value, + ); + } + } + } + } + } + Key::Property(JSContactProperty::SortAs) => { + if let Some(sort_as) = value.into_string() { + entry.add_param(VCardParameter::sort_as( + sort_as.into_owned(), + )); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Organizations], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Members => { + let mut has_members = false; + for name in value.into_expanded_boolean_set() { + has_members = true; + state.insert_vcard( + &[JSContactProperty::Members], + VCardEntry::new(VCardProperty::Member) + .with_value(name.into_string()), + ); + } + + if !has_members { + state.insert_jsprop( + &[property.to_string().as_ref()], + Value::Object(Map::from(Vec::new())), + ); + } + } + JSContactProperty::Keywords => { + let mut keywords = Vec::new(); + for name in value.into_expanded_boolean_set() { + keywords.push(VCardValue::Text(name.into_string())); + } + + state.insert_vcard( + &[JSContactProperty::Keywords], + VCardEntry::new(VCardProperty::Categories).with_values(keywords), + ); + } + JSContactProperty::Titles => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new( + map_kind( + &value, + [ + (JSContactKind::Title, VCardProperty::Title), + (JSContactKind::Role, VCardProperty::Role), + ], + ) + .unwrap_or(VCardProperty::Title), + ); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Name) => { + if let Some(value) = + value.into_owned_string().map(VCardValue::Text) + { + entry.values = smallvec![value]; + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::OrganizationId) + if org_mappings.values().any(|(id, has_mapping)| { + *has_mapping && id == name.to_string().as_ref() + }) => {} + Key::Property(JSContactProperty::Kind) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Titles], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Emails => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Email); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Address) => { + if let Some(email) = value.into_string() { + entry.values = + smallvec![VCardValue::Text(email.into_owned())]; + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(label) = value.into_string() { + entry + .params + .push(VCardParameter::label(label.into_owned())); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Emails], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::OnlineServices => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Socialprofile); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Uri) => { + if let Some(service) = value.into_string() { + entry.values = + smallvec![VCardValue::Text(service.into_owned())]; + } + } + Key::Property(JSContactProperty::Service) => { + if let Some(service) = value.into_string() { + entry.add_param(VCardParameter::service_type( + service.into_owned(), + )); + } + } + Key::Property(JSContactProperty::User) => { + if let Some(username) = value.into_string() { + entry.add_param(VCardParameter::username( + username.into_owned(), + )); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + Key::Property(JSContactProperty::Label) => { + if let Some(label) = value.into_string() { + entry.add_param(VCardParameter::label( + label.into_owned(), + )); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::OnlineServices], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Phones => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Tel); + let mut types = vec![]; + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Number) => { + if let Some(email) = value.into_string() { + entry.values = + smallvec![VCardValue::Text(email.into_owned())]; + } + } + Key::Property( + bucket @ (JSContactProperty::Contexts + | JSContactProperty::Features), + ) => { + state.convert_types( + &VCardProperty::Tel, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + bucket, + value, + &mut types, + ); + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(label) = value.into_string() { + entry + .params + .push(VCardParameter::label(label.into_owned())); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + entry.params.append(&mut types); + state.insert_vcard( + &[JSContactProperty::Phones], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::PreferredLanguages => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Lang); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Language) => { + if let Some(lang) = value.into_string() { + entry.values = + smallvec![VCardValue::Text(lang.into_owned())]; + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + Key::Property(JSContactProperty::Label) => { + if let Some(label) = value.into_string() { + entry + .params + .push(VCardParameter::label(label.into_owned())); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::PreferredLanguages], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Calendars => { + for (name, value) in value.into_expanded_object() { + if let Some(entry_type) = map_kind( + &value, + [ + (JSContactKind::FreeBusy, VCardProperty::Fburl), + (JSContactKind::Calendar, VCardProperty::Caluri), + ], + ) { + let mut entry = VCardEntry::new(entry_type); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Uri) => { + if let Some(value) = + value.into_owned_string().map(VCardValue::Text) + { + entry.values = smallvec![value]; + } + } + Key::Property(JSContactProperty::MediaType) => { + if let Some(text) = value.into_string() { + entry.params.push(VCardParameter::mediatype( + text.into_owned(), + )); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::label(text.into_owned())); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + Key::Property(JSContactProperty::Kind) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Calendars], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } else { + state.insert_jsprop( + &[property.to_string().as_ref(), name.to_string().as_ref()], + value, + ); + } + } + } + JSContactProperty::SchedulingAddresses + | JSContactProperty::CryptoKeys + | JSContactProperty::Links => { + let vcard_property = match property { + JSContactProperty::SchedulingAddresses => VCardProperty::Caladruri, + JSContactProperty::CryptoKeys => VCardProperty::Key, + JSContactProperty::Links => VCardProperty::Url, + _ => unreachable!(), + }; + + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(vcard_property.clone()); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Uri) => { + if let Some(uri) = value.into_owned_string() { + let data = matches!(vcard_property, VCardProperty::Key) + .then(|| Data::try_parse(uri.as_bytes())) + .flatten() + .filter(|data| context.embed(&data.data)); + entry.values = smallvec![match data { + Some(data) => VCardValue::Binary(Box::new(data)), + None => VCardValue::Text(uri), + }]; + } + } + Key::Property(JSContactProperty::MediaType) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::mediatype(text.into_owned())); + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(text) = value.into_string() { + entry + .params + .push(VCardParameter::label(text.into_owned())); + } + } + Key::Property(JSContactProperty::Pref) + if value.as_pref().is_some() => + { + if let Some(pref) = value.as_pref() { + entry.add_param(VCardParameter::pref(pref)); + } + } + Key::Property(JSContactProperty::Contexts) => { + state.convert_types( + &entry.name, + [ + property.to_string().as_ref(), + name.to_string().as_ref(), + ], + JSContactProperty::Contexts, + value, + &mut entry.params, + ); + } + Key::Property(JSContactProperty::Kind) => { + if matches!( + value, + Value::Element(JSContactValue::Kind( + JSContactKind::Contact + )) + ) { + entry.name = VCardProperty::ContactUri; + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + std::slice::from_ref(&property), + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::Notes => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Note); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Note) => { + if let Some(text) = value.into_string() { + entry.values = + smallvec![VCardValue::Text(text.into_owned())]; + } + } + Key::Property(JSContactProperty::Created) => { + if let Value::Element(JSContactValue::Timestamp(t)) = value + { + entry.params.push(VCardParameter::created( + VCardParameterValue::Timestamp(t), + )); + } + } + Key::Property(JSContactProperty::Author) => { + for (name, value) in value.into_expanded_object() { + match name { + Key::Property(JSContactProperty::Name) => { + if let Some(name) = value.into_string() { + entry.add_param( + VCardParameter::author_name( + name.into_owned(), + ), + ); + } + } + Key::Property(JSContactProperty::Uri) => { + if let Some(uri) = value.into_string() { + entry.add_param(VCardParameter::author( + uri.into_owned(), + )); + } + } + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + name.to_string().as_ref(), + ], + value, + ); + } + } + } + } + + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::Notes], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } + } + JSContactProperty::PersonalInfo => { + for (name, value) in value.into_expanded_object() { + if let Some(entry_type) = map_kind( + &value, + [ + (JSContactKind::Expertise, VCardProperty::Expertise), + (JSContactKind::Hobby, VCardProperty::Hobby), + (JSContactKind::Interest, VCardProperty::Interest), + ], + ) { + let mut entry = VCardEntry::new(entry_type); + + for (sub_property, value) in value.into_expanded_object() { + match sub_property { + Key::Property(JSContactProperty::Value) => { + if let Some(value) = + value.into_owned_string().map(VCardValue::Text) + { + entry.values = smallvec![value]; + } + } + Key::Property(JSContactProperty::Label) => { + if let Some(label) = value.into_string() { + entry.add_param(VCardParameter::label( + label.into_owned(), + )); + } + } + Key::Property(JSContactProperty::ListAs) => { + if let Some(pref) = value.as_u32() { + entry.add_param(VCardParameter::index(pref)); + } + } + Key::Property(JSContactProperty::Level) => { + if let Ok(level) = + IanaType::::try_from(value) + { + let level = if matches!( + entry.name, + VCardProperty::Expertise + ) { + match level { + IanaType::Iana(VCardLevel::High) => { + IanaType::Iana(VCardLevel::Expert) + } + IanaType::Iana(VCardLevel::Medium) => { + IanaType::Iana(VCardLevel::Average) + } + IanaType::Iana(VCardLevel::Low) => { + IanaType::Iana(VCardLevel::Beginner) + } + _ => level, + } + } else { + level + }; + entry.params.push(VCardParameter::level(level)); + } + } + Key::Property(JSContactProperty::Kind) => {} + _ => { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + } + + state.insert_vcard( + &[JSContactProperty::PersonalInfo], + entry.with_param(VCardParameter::prop_id(name.into_string())), + ); + } else { + state.insert_jsprop( + &[property.to_string().as_ref(), name.to_string().as_ref()], + value, + ); + } + } + } + JSContactProperty::RelatedTo => { + for (name, value) in value.into_expanded_object() { + let mut entry = VCardEntry::new(VCardProperty::Related) + .with_value(name.to_string().into_owned()); + + for (sub_property, value) in value.into_expanded_object() { + if let Key::Property(JSContactProperty::Relation) = sub_property { + for typ in value.into_expanded_boolean_set() { + let typ = typ.to_string(); + match VCardType::parse(typ.as_ref().as_bytes()) { + Some(typ) => { + entry.params.push(VCardParameter::typ(typ)) + } + None => { + entry + .params + .push(VCardParameter::typ(typ.into_owned())); + } + } + } + } else { + state.insert_jsprop( + &[ + property.to_string().as_ref(), + name.to_string().as_ref(), + sub_property.to_string().as_ref(), + ], + value, + ); + } + } + + state.insert_vcard(&[JSContactProperty::RelatedTo], entry); + } + } + JSContactProperty::Type + | JSContactProperty::Version + | JSContactProperty::Localizations + | JSContactProperty::VCard => (), + _ => { + if !matches!(value, Value::Null) && !property.is_metadata() { + state.insert_jsprop(&[property.to_string().as_ref()], value); + } + } + } + } + } + + if !has_fn { + state.insert_vcard( + &[JSContactProperty::Name, JSContactProperty::Full], + VCardEntry::new(VCardProperty::Fn) + .with_param(VCardParameter::derived(true)) + .with_value(String::new()), + ); + } + + context.into_result(state.into_vcard()) + } + + pub fn into_inner(self) -> Value<'x, JSContactProperty, JSContactValue> { + self.0 + } +} + diff --git a/crates/calcard/src/jscontact/export/entry.rs b/crates/calcard/src/jscontact/export/entry.rs new file mode 100644 index 00000000..3734b5a7 --- /dev/null +++ b/crates/calcard/src/jscontact/export/entry.rs @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + CalendarScale, IanaParse, + parser::{Boolean, Timestamp}, + }, + jscontact::{JSContactId, JSContactProperty, JSContactValue, export::params::ParamValue}, + vcard::{ + VCardEntry, VCardLevel, VCardParameter, VCardParameterName, VCardParameterValue, + VCardPhonetic, VCardProperty, VCardType, VCardValueType, + }, +}; +use jmap_tools::{Key, Value}; +use std::mem; + +impl VCardEntry { + pub(super) fn import_converted_properties( + &mut self, + props: Value<'_, JSContactProperty, JSContactValue>, + ) where + I: JSContactId, + B: JSContactId, + { + for (key, value) in props.into_expanded_object() { + match key { + Key::Property(JSContactProperty::Name) => { + if let Some(name) = value.into_string() { + self.name = VCardProperty::parse(name.as_bytes()) + .unwrap_or(VCardProperty::Other(name.to_ascii_uppercase())); + } + } + Key::Property(JSContactProperty::Parameters) => { + self.import_jcard_params(value); + } + _ => {} + } + } + } + + pub(super) fn import_jcard_params( + &mut self, + params: Value<'_, JSContactProperty, JSContactValue>, + ) where + I: JSContactId, + B: JSContactId, + { + for (key, value) in params.into_expanded_object() { + let (single, many) = match value { + Value::Array(values) => (None, values), + value => (Some(value), Vec::new()), + }; + let mut values = single + .into_iter() + .chain(many) + .filter_map(ParamValue::try_from_value) + .peekable(); + + if values.peek().is_none() { + continue; + } + + let key = key.to_string(); + let Some(param) = VCardParameterName::try_parse(key.as_bytes()) else { + let mut name = key.to_ascii_uppercase(); + + while let Some(value) = values.next() { + let name = if values.peek().is_some() { + name.clone() + } else { + mem::take(&mut name) + }; + self.params.push(VCardParameter { + name: VCardParameterName::Other(name), + value: value.into_string().into_owned().into(), + }); + } + + continue; + }; + + for value in values { + let value = match ¶m { + VCardParameterName::Value => { + let value = value.into_string(); + VCardValueType::parse(value.as_bytes()) + .map(VCardParameterValue::ValueType) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Pref | VCardParameterName::Index => { + match value.into_number() { + Ok(n) => u32::try_from(n).map_or_else( + |_| VCardParameterValue::Text(n.to_string()), + VCardParameterValue::Integer, + ), + Err(value) => { + VCardParameterValue::Text(value.into_string().into_owned()) + } + } + } + VCardParameterName::Calscale => { + let value = value.into_string(); + CalendarScale::parse(value.as_bytes()) + .map(VCardParameterValue::Calscale) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Phonetic => { + let value = value.into_string(); + VCardPhonetic::parse(value.as_bytes()) + .map(VCardParameterValue::Phonetic) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Level => { + let value = value.into_string(); + VCardLevel::parse(value.as_bytes()) + .map(VCardParameterValue::Level) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Created => { + let value = value.into_string(); + Timestamp::parse(value.as_bytes()) + .map(|v| VCardParameterValue::Timestamp(v.0)) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Derived => { + let value = value.into_string(); + Boolean::parse(value.as_bytes()) + .map(|v| VCardParameterValue::Bool(v.0)) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Type => { + let value = value.into_string(); + VCardType::parse(value.as_bytes()) + .map(VCardParameterValue::Type) + .unwrap_or_else(|| VCardParameterValue::Text(value.into_owned())) + } + VCardParameterName::Group => { + self.group = Some(value.into_string().into_owned()); + continue; + } + _ => VCardParameterValue::Text(value.into_string().into_owned()), + }; + + self.params.push(VCardParameter { + name: param.clone(), + value, + }); + } + } + } +} diff --git a/crates/calcard/src/jscontact/export/mod.rs b/crates/calcard/src/jscontact/export/mod.rs new file mode 100644 index 00000000..d1ccfbea --- /dev/null +++ b/crates/calcard/src/jscontact/export/mod.rs @@ -0,0 +1,121 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + blob::{BlobOptions, BlobResolver, BlobResolverFn, NoBlobIds, ResolvedBlobs}, + export::{EmbeddedBudget, ExportError}, + jsprop::ordered::OrderedMap, + }, + jscontact::{JSContactId, JSContactProperty, JSContactValue}, + vcard::VCard, +}; +use jmap_tools::{Key, Value}; +use std::hash::Hash; + +pub mod convert; +pub mod entry; +pub mod params; +pub mod props; + +#[derive(Default)] +#[allow(clippy::type_complexity)] +struct State<'x, I, B> +where + I: JSContactId, + B: JSContactId, +{ + pub(super) vcard: VCard, + pub(super) converted_props: Vec<( + Vec>>, + Value<'x, JSContactProperty, JSContactValue>, + )>, + pub(super) converted_props_count: usize, + pub(super) converted_heads: OrderedMap, ()>, + pub(super) language: Option, +} + +#[derive(Debug, Clone, Copy)] +pub struct ExportOptions { + blobs: BlobOptions, +} + +impl Default for ExportOptions { + fn default() -> Self { + Self { + blobs: BlobOptions::default(), + } + } +} + +pub(super) struct ExportContext<'a, B> { + blobs: Option>, + budget: EmbeddedBudget, + error: Option, +} + +impl ExportOptions { + pub fn new() -> Self { + Self::default() + } +} + +impl ExportOptions { + pub fn with_blob_resolver(self, blob_resolver: F) -> ExportOptions> + where + F: FnMut(&B) -> Option>, + { + self.with_resolver(BlobResolverFn(blob_resolver)) + } + + pub fn with_resolver(self, blob_resolver: T) -> ExportOptions { + ExportOptions { + blobs: self.blobs.with_handler(blob_resolver), + } + } + + pub fn max_embedded_size(mut self, max_embedded_size: usize) -> Self { + self.blobs = self.blobs.with_max_embedded_size(max_embedded_size); + self + } + + pub(super) fn context(&mut self) -> ExportContext<'_, B> + where + R: BlobResolver, + { + ExportContext { + budget: self.blobs.budget(), + blobs: self.blobs.resolved_blobs(), + error: None, + } + } +} + +impl ExportContext<'_, B> { + fn fail(&mut self, error: ExportError) { + if self.error.is_none() { + self.error = Some(error); + } + } + + fn embed(&mut self, data: &[u8]) -> bool { + match self.budget.take(data) { + Err(error) => { + self.blobs = None; + self.fail(error); + false + } + Ok(()) => true, + } + } + + fn into_result(self, vcard: VCard) -> Result { + match self.error { + Some(error) => Err(error), + None => Ok(vcard), + } + } +} diff --git a/crates/calcard/src/jscontact/export/params.rs b/crates/calcard/src/jscontact/export/params.rs new file mode 100644 index 00000000..76add5ee --- /dev/null +++ b/crates/calcard/src/jscontact/export/params.rs @@ -0,0 +1,365 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{IanaParse, IanaType}, + jscontact::{ + Context, Feature, JSContactId, JSContactProperty, JSContactValue, + export::{State, props::convert_value}, + }, + vcard::{ + VCard, VCardEntry, VCardParameter, VCardParameterName, VCardParameterValue, VCardProperty, + VCardType, VCardValueType, ValueType, + }, +}; +use jmap_tools::{Element, JsonPointer, Key, Property, Value}; +use smallvec::{SmallVec, smallvec}; +use std::{borrow::Cow, mem}; + +impl<'x, I, B> State<'x, I, B> +where + I: JSContactId, + B: JSContactId, +{ + pub(super) fn insert_vcard(&mut self, path: &[JSContactProperty], mut entry: VCardEntry) { + if self.converted_props_count < self.converted_props.len() { + // Obtain propId + let mut prop_id = + if matches!(entry.name, VCardProperty::Member | VCardProperty::Related) { + entry.values.first().and_then(|v| v.as_text()) + } else { + entry.prop_id() + }; + + // Try mapping X-ABLabel + if let Some(prop_id_) = prop_id { + let mut remove_pos = None; + for (param_pos, param) in entry.params.iter().enumerate() { + if let (VCardParameterName::Label, VCardParameterValue::Text(label)) = + (¶m.name, ¶m.value) + { + if self.converted_props.iter().any(|(prop, _)| { + prop.len() == 3 + && prop[0].to_string() == path[0].to_string() + && prop[1] == prop_id_ + && prop[2] == Key::Property(JSContactProperty::Label) + }) { + self.insert_vcard( + &[path[0].clone(), JSContactProperty::Label], + VCardEntry::new(VCardProperty::Other("X-ABLabel".into())) + .with_value(label.to_string()), + ); + remove_pos = Some(param_pos); + } + break; + } + } + + if let Some(pos) = remove_pos { + entry.params.swap_remove(pos); + prop_id = + if matches!(entry.name, VCardProperty::Member | VCardProperty::Related) { + entry.values.first().and_then(|v| v.as_text()) + } else { + entry.prop_id() + }; + } + } + + let skip_tz_geo = matches!(entry.name, VCardProperty::Adr); + let mut matched_once = false; + let may_match = path + .first() + .is_none_or(|head| self.converted_heads.contains_key(head)); + + if may_match { + 'outer: for (keys, value) in self.converted_props.iter_mut() { + let is_localized_key = keys.first().is_some_and(|k| { + matches!(k, Key::Property(JSContactProperty::Localizations)) + }); + + if let Some(lang) = &self.language { + if !is_localized_key || keys.get(1).is_none_or(|k| *k != lang.as_str()) { + continue; + } + } else if is_localized_key { + continue; + } + if matches!(value, Value::Null) { + continue; + } + + for (pos, item) in path.iter().enumerate() { + if !keys + .iter() + .any(|k| matches!(k, Key::Property(p) if p == item)) + { + if pos == 0 && matched_once { + break 'outer; + } else { + continue 'outer; + } + } else { + matched_once = true; + } + } + + if prop_id + .map(Key::Borrowed) + .is_none_or(|prop_id| keys.iter().any(|k| k == &prop_id)) + && (!skip_tz_geo + || !keys.iter().any(|k| { + matches!( + k, + Key::Property( + JSContactProperty::TimeZone + | JSContactProperty::Coordinates + ) + ) + })) + { + entry.import_converted_properties(mem::take(value)); + self.converted_props_count += 1; + break; + } + } + } + } + + if let Some(lang) = &self.language { + entry.params.push(VCardParameter::language(lang.clone())); + } + + self.vcard.entries.push(entry); + } + + pub(super) fn convert_types( + &mut self, + vcard_property: &VCardProperty, + [property, id]: [&str; 2], + bucket: JSContactProperty, + value: Value<'x, JSContactProperty, JSContactValue>, + params: &mut Vec, + ) { + let mut has_mapped = false; + let mut has_unmapped = false; + let mut has_unquotable_key = false; + + let is_mapped = + |key: &Key<'_, JSContactProperty>, + value: &Value<'x, JSContactProperty, JSContactValue>| { + value.as_bool() == Some(true) + && bucket + .vcard_type(vcard_property, key.to_string().as_ref()) + .is_some() + }; + + for (key, item) in value.as_object().into_iter().flat_map(|obj| obj.iter()) { + let key = key.to_string(); + match bucket + .vcard_type(vcard_property, key.as_ref()) + .filter(|_| item.as_bool() == Some(true)) + { + Some(IanaType::Iana(typ)) => { + params.push(VCardParameter::typ(typ)); + has_mapped = true; + } + Some(IanaType::Other(typ)) => { + params.push(VCardParameter::typ(typ.to_ascii_uppercase())); + has_mapped = true; + } + None => { + has_unmapped = true; + has_unquotable_key |= key + .chars() + .any(|ch| ch.is_control() || matches!(ch, '"' | '\\')); + } + } + } + + if !has_unmapped { + return; + } + + let bucket_name = bucket.to_string(); + if has_mapped && !has_unquotable_key && self.language.is_none() { + for (key, item) in value.into_expanded_object() { + if !is_mapped(&key, &item) { + self.insert_jsprop( + &[property, id, bucket_name.as_ref(), key.to_string().as_ref()], + item, + ); + } + } + } else { + self.insert_jsprop( + &[property, id, bucket_name.as_ref()], + Value::Object(value.into_expanded_object().collect()), + ); + } + } + + pub(super) fn insert_jsprop( + &mut self, + path: &[&str], + value: Value<'x, JSContactProperty, JSContactValue>, + ) { + let path = if let Some(lang) = &self.language { + JsonPointer::>::encode([ + JSContactProperty::Localizations::.to_string().as_ref(), + lang.as_str(), + JsonPointer::>::encode(path).as_str(), + ]) + } else { + JsonPointer::>::encode(path) + }; + + self.vcard.entries.push( + VCardEntry::new(VCardProperty::Jsprop) + .with_param(VCardParameter::jsptr(path)) + .with_value(serde_json::to_string(&value).unwrap_or_default()), + ); + } + + pub(super) fn import_properties( + &mut self, + props: Vec, JSContactValue>>, + ) { + for prop in props.into_iter().flat_map(|prop| prop.into_array()) { + let mut prop = prop.into_iter(); + let Some(name) = prop + .next() + .and_then(|v| v.into_string()) + .map(|name| { + VCardProperty::parse(name.as_bytes()).unwrap_or_else(|| { + let mut name = name.into_owned(); + name.make_ascii_uppercase(); + VCardProperty::Other(name) + }) + }) + .filter(|name| !matches!(name, VCardProperty::Begin | VCardProperty::End)) + else { + continue; + }; + let Some(params) = prop.next() else { + continue; + }; + let Some(value_type) = prop + .next() + .and_then(|v| v.into_string()) + .map(|v| VCardValueType::parse(v.as_bytes())) + else { + continue; + }; + + let (default_type, _) = name.default_types(); + let convert_type = value_type.map(ValueType::Vcard).unwrap_or(default_type); + + let Some(values) = prop.next().and_then(|v| match v { + Value::Array(arr) => Some( + arr.into_iter() + .filter_map(|v| convert_value(v, &convert_type).ok()) + .collect::>(), + ), + v => convert_value(v, &convert_type).ok().map(|v| smallvec![v]), + }) else { + continue; + }; + + let mut entry = VCardEntry::new(name); + entry.import_jcard_params(params); + entry.values = values; + if convert_type != default_type + && let Some(value_type) = value_type + { + entry.params.push(VCardParameter::value( + IanaType::::Iana(value_type), + )); + } + self.vcard.entries.push(entry); + } + } + + pub(super) fn into_vcard(self) -> VCard { + self.vcard + } +} + +impl JSContactProperty { + fn vcard_type<'k>( + &self, + property: &VCardProperty, + key: &'k str, + ) -> Option> { + if key.is_empty() + || !key + .bytes() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == b'-') + { + return None; + } + + match (self, VCardType::parse(key.as_bytes())) { + (JSContactProperty::Features, Some(typ)) => Feature::from_vcard_type(property, &typ) + .is_some_and(|feature| feature.as_str() == key) + .then_some(IanaType::Iana(typ)), + (JSContactProperty::Features, None) => { + Feature::from_vcard_type(property, &VCardType::Cell) + .is_some_and(|feature| feature.as_str() == key) + .then_some(IanaType::Iana(VCardType::Cell)) + } + (JSContactProperty::Contexts, Some(typ)) => (Feature::from_vcard_type(property, &typ) + .is_none() + && !matches!(typ, VCardType::Home | VCardType::Cell) + && (!matches!(typ, VCardType::Billing | VCardType::Delivery) + || property == &VCardProperty::Adr)) + .then_some(IanaType::Iana(typ)), + (JSContactProperty::Contexts, None) if key == Context::Private.as_str() => { + Some(IanaType::Iana(VCardType::Home)) + } + (JSContactProperty::Contexts, None) => Some(IanaType::Other(key)), + _ => None, + } + } +} + +pub(crate) enum ParamValue<'x> { + Text(Cow<'x, str>), + Number(i64), + Bool(bool), +} + +impl<'x> ParamValue<'x> { + pub(crate) fn try_from_value(value: Value<'x, P, E>) -> Option { + match value { + Value::Str(s) => Some(Self::Text(s)), + Value::Number(n) => Some(match n.as_i64() { + Some(number) => Self::Number(number), + None => Self::Text(Value::::Number(n).to_string().into()), + }), + Value::Bool(b) => Some(Self::Bool(b)), + Value::Element(e) => Some(Self::Text(e.to_cow().to_string().into())), + _ => None, + } + } + + pub(crate) fn into_string(self) -> Cow<'x, str> { + match self { + Self::Text(s) => s, + Self::Number(n) => n.to_string().into(), + Self::Bool(b) => if b { "true" } else { "false" }.to_string().into(), + } + } + + pub(crate) fn into_number(self) -> Result { + match self { + Self::Number(n) => Ok(n), + Self::Text(s) => s.parse().map_err(|_| Self::Text(s)), + _ => Err(self), + } + } +} + diff --git a/crates/calcard/src/jscontact/export/props.rs b/crates/calcard/src/jscontact/export/props.rs new file mode 100644 index 00000000..ac49efe0 --- /dev/null +++ b/crates/calcard/src/jscontact/export/props.rs @@ -0,0 +1,426 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{CalendarScale, IanaParse, IanaType, PartialDateTime, parser::Integer}, + icalendar::Uri, + jscontact::{ + JSContactGrammaticalGender, JSContactId, JSContactKind, JSContactLevel, + JSContactPhoneticSystem, JSContactProperty, JSContactValue, + }, + vcard::{ + VCardGramGender, VCardKind, VCardLevel, VCardPhonetic, VCardSex, VCardValue, + VCardValueType, ValueType, + }, +}; +use jmap_tools::{JsonPointerItem, Key, Map, Value}; +use std::{borrow::Cow, iter::Peekable, vec::IntoIter}; + +impl VCardValue { + pub(super) fn from_components(components: Option>) -> Self { + match components { + Some(components) if components.len() > 1 => VCardValue::Component(components), + Some(components) => VCardValue::Text(components.into_iter().next().unwrap_or_default()), + None => VCardValue::Text(String::new()), + } + } +} + +pub(super) fn build_path<'x, I, B>( + obj: &mut Value<'x, JSContactProperty, JSContactValue>, + mut ptr: Peekable>>>, + value: Value<'x, JSContactProperty, JSContactValue>, +) -> Option, JSContactValue>> +where + I: JSContactId, + B: JSContactId, +{ + if let Some(item) = ptr.next() { + match item { + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => {} + JsonPointerItem::Key(key) => match obj { + Value::Object(obj) => { + return build_path( + obj.insert_or_get_mut( + key, + if matches!(ptr.peek(), Some(JsonPointerItem::Key(_))) { + Value::Object(Map::from(Vec::new())) + } else { + Value::Array(Vec::new()) + }, + ), + ptr, + value, + ); + } + Value::Null => { + *obj = Value::Object(Map::from(vec![(key, Value::Null)])); + return build_path( + &mut obj + .as_object_mut() + .unwrap() + .as_mut_vec() + .last_mut() + .unwrap() + .1, + ptr, + value, + ); + } + _ => {} + }, + JsonPointerItem::Number(idx) => { + if let Some(arr) = obj.as_array_mut() { + if (idx as usize) < arr.len() { + return build_path(&mut arr[idx as usize], ptr, value); + } + + if idx < 20 { + arr.resize_with(idx as usize + 1, || Value::Null); + return build_path(&mut arr[idx as usize], ptr, value); + } + } + } + } + Some(value) + } else { + *obj = value; + None + } +} + +#[allow(clippy::type_complexity)] +pub(super) fn convert_anniversary( + value: Value<'_, JSContactProperty, JSContactValue>, +) -> Result< + (PartialDateTime, Option), + Value<'_, JSContactProperty, JSContactValue>, +> +where + I: JSContactId, + B: JSContactId, +{ + let mut date = PartialDateTime::default(); + let mut calendar_scale = None; + let mut is_valid = true; + let Some(object) = value.as_object() else { + return Err(value); + }; + + for (key, value) in object.as_vec() { + match key { + Key::Property(JSContactProperty::Day) => { + if let Value::Number(day) = value { + date.day = day + .as_u64() + .filter(|day| (1..=31).contains(day)) + .map(|day| day as u8); + is_valid &= date.day.is_some(); + } + } + Key::Property(JSContactProperty::Month) => { + if let Value::Number(month) = value { + date.month = month + .as_u64() + .filter(|month| (1..=12).contains(month)) + .map(|month| month as u8); + is_valid &= date.month.is_some(); + } + } + Key::Property(JSContactProperty::Year) => { + if let Value::Number(year) = value { + date.year = year + .as_u64() + .filter(|year| *year <= 9999) + .map(|year| year as u16); + is_valid &= date.year.is_some(); + } + } + Key::Property(JSContactProperty::CalendarScale) => { + if let Value::Element(JSContactValue::CalendarScale(scale)) = value { + calendar_scale = Some(scale.clone()); + } + } + Key::Property(JSContactProperty::Utc) => { + if let Value::Element(JSContactValue::Timestamp(timestamp)) = value { + return Ok((PartialDateTime::from_utc_timestamp(*timestamp), None)); + } + } + _ => {} + } + } + + if is_valid && (date.year.is_some() || date.month.is_some() || date.day.is_some()) { + Ok((date, calendar_scale)) + } else { + Err(value) + } +} + +pub(super) fn convert_value<'x, I, B>( + value: Value<'x, JSContactProperty, JSContactValue>, + value_type: &'_ ValueType, +) -> Result, JSContactValue>> +where + I: JSContactId, + B: JSContactId, +{ + match value { + Value::Element(e) => match e { + JSContactValue::Timestamp(t) => Ok(VCardValue::PartialDateTime( + PartialDateTime::from_utc_timestamp(t), + )), + JSContactValue::GrammaticalGender(g) => Ok(VCardValue::GramGender(match g { + JSContactGrammaticalGender::Animate => VCardGramGender::Animate, + JSContactGrammaticalGender::Common => VCardGramGender::Common, + JSContactGrammaticalGender::Feminine => VCardGramGender::Feminine, + JSContactGrammaticalGender::Inanimate => VCardGramGender::Inanimate, + JSContactGrammaticalGender::Masculine => VCardGramGender::Masculine, + JSContactGrammaticalGender::Neuter => VCardGramGender::Neuter, + })), + JSContactValue::Kind(k) => match k { + JSContactKind::Individual => Ok(VCardValue::Kind(VCardKind::Individual)), + JSContactKind::Group => Ok(VCardValue::Kind(VCardKind::Group)), + JSContactKind::Location => Ok(VCardValue::Kind(VCardKind::Location)), + JSContactKind::Org => Ok(VCardValue::Kind(VCardKind::Org)), + JSContactKind::Application => Ok(VCardValue::Kind(VCardKind::Application)), + JSContactKind::Device => Ok(VCardValue::Kind(VCardKind::Device)), + _ => Err(Value::Element(JSContactValue::Kind(k))), + }, + JSContactValue::Level(_) + | JSContactValue::Type(_) + | JSContactValue::Relation(_) + | JSContactValue::PhoneticSystem(_) + | JSContactValue::CalendarScale(_) + | JSContactValue::BlobId(_) + | JSContactValue::Id(_) + | JSContactValue::IdReference(_) => Err(Value::Element(e)), + }, + Value::Str(s) => { + match value_type { + ValueType::Kind => { + if let Some(kind) = VCardKind::parse(s.as_ref().as_bytes()) { + return Ok(VCardValue::Kind(kind)); + } + } + ValueType::Sex => { + if let Some(sex) = VCardSex::parse(s.as_ref().as_bytes()) { + return Ok(VCardValue::Sex(sex)); + } + } + ValueType::GramGender => { + if let Some(gender) = VCardGramGender::parse(s.as_ref().as_bytes()) { + return Ok(VCardValue::GramGender(gender)); + } + } + ValueType::Vcard(typ) => match typ { + VCardValueType::Boolean => { + if s.eq_ignore_ascii_case("true") { + return Ok(VCardValue::Boolean(true)); + } else if s.eq_ignore_ascii_case("false") { + return Ok(VCardValue::Boolean(false)); + } + } + VCardValueType::Date => { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date(&mut s.as_ref().as_bytes().iter().peekable()); + if !dt.is_null() { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::DateAndOrTime => { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date_and_or_time( + &mut s.as_ref().as_bytes().iter().peekable(), + ); + if !dt.is_null() { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::DateTime => { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date_time(&mut s.as_ref().as_bytes().iter().peekable()); + if !dt.is_null() { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::Time => { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_time(&mut s.as_ref().as_bytes().iter().peekable(), false); + if !dt.is_null() { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::Timestamp => { + let mut dt = PartialDateTime::default(); + if dt.parse_timestamp(&mut s.as_ref().as_bytes().iter().peekable(), true) { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::UtcOffset => { + let mut dt = PartialDateTime::default(); + dt.parse_zone(&mut s.as_ref().as_bytes().iter().peekable()); + if !dt.is_null() { + return Ok(VCardValue::PartialDateTime(dt)); + } + } + VCardValueType::Float => { + if let Ok(float) = s.as_ref().parse::() + && float.is_finite() + { + return Ok(VCardValue::Float(float)); + } + } + VCardValueType::Integer => { + if let Some(integer) = Integer::parse(s.as_ref().as_bytes()) { + return Ok(VCardValue::Integer(integer.0)); + } + } + VCardValueType::Uri => { + return Ok(match Uri::parse(s) { + Uri::Data(data) => VCardValue::Binary(data), + Uri::Location(text) => VCardValue::Text(text), + }); + } + VCardValueType::LanguageTag | VCardValueType::Text => (), + }, + } + + Ok(VCardValue::Text(s.into_owned())) + } + Value::Bool(b) => Ok(VCardValue::Boolean(b)), + Value::Number(n) => match n.as_i64() { + Some(integer) => Ok(VCardValue::Integer(integer)), + None => n + .as_f64() + .filter(|float| n.is_f64() && float.is_finite()) + .map(VCardValue::Float) + .ok_or(Value::Number(n)), + }, + value => Err(value), + } +} + +pub(super) trait U32Value { + fn as_u32(&self) -> Option; + fn as_pref(&self) -> Option; + fn as_index(&self) -> Option; +} + +impl U32Value for Value<'_, JSContactProperty, JSContactValue> +where + I: JSContactId, + B: JSContactId, +{ + fn as_u32(&self) -> Option { + self.as_u64().and_then(|value| u32::try_from(value).ok()) + } + + fn as_pref(&self) -> Option { + self.as_u32().filter(|pref| (1..=100).contains(pref)) + } + + fn as_index(&self) -> Option { + self.as_u32().filter(|index| *index >= 1) + } +} + +pub(super) fn map_kind( + value: &Value<'_, JSContactProperty, JSContactValue>, + types: impl IntoIterator, +) -> Option +where + I: JSContactId, + B: JSContactId, +{ + value + .as_object() + .and_then(|obj| obj.get(&Key::Property(JSContactProperty::Kind))) + .and_then(|v| match v { + Value::Element(JSContactValue::Kind(kind)) => { + types.into_iter().find_map(|(js_kind, vcard_property)| { + if js_kind == *kind { + Some(vcard_property) + } else { + None + } + }) + } + + _ => None, + }) +} + +impl TryFrom, JSContactValue>> + for IanaType +where + I: JSContactId, + B: JSContactId, +{ + type Error = (); + fn try_from( + value: Value<'_, JSContactProperty, JSContactValue>, + ) -> Result { + match value { + Value::Element(JSContactValue::PhoneticSystem(system)) => { + Ok(IanaType::Iana(match system { + JSContactPhoneticSystem::Ipa => VCardPhonetic::Ipa, + JSContactPhoneticSystem::Jyut => VCardPhonetic::Jyut, + JSContactPhoneticSystem::Piny => VCardPhonetic::Piny, + JSContactPhoneticSystem::Script => VCardPhonetic::Script, + })) + } + Value::Str(text) => match VCardPhonetic::parse(text.as_ref().as_bytes()) { + Some(phonetic) => Ok(IanaType::Iana(phonetic)), + None => Ok(IanaType::Other(text.to_ascii_uppercase())), + }, + _ => Err(()), + } + } +} + +impl TryFrom, JSContactValue>> + for IanaType +where + I: JSContactId, + B: JSContactId, +{ + type Error = (); + + fn try_from( + value: Value<'_, JSContactProperty, JSContactValue>, + ) -> Result { + match value { + Value::Element(JSContactValue::Level(level)) => Ok(IanaType::Iana(match level { + JSContactLevel::High => VCardLevel::High, + JSContactLevel::Low => VCardLevel::Low, + JSContactLevel::Medium => VCardLevel::Medium, + })), + Value::Str(text) => match VCardLevel::parse(text.as_ref().as_bytes()) { + Some(level) => Ok(IanaType::Iana(level)), + None => Ok(IanaType::Other(text.to_ascii_uppercase())), + }, + _ => Err(()), + } + } +} + +pub(super) fn find_text_param<'x, I, B>( + value: &'x Value<'x, JSContactProperty, JSContactValue>, + name: &str, +) -> Option> +where + I: JSContactId, + B: JSContactId, +{ + value + .as_object() + .and_then(|obj| obj.get(&Key::Property(JSContactProperty::Parameters))) + .and_then(|obj| obj.as_object()) + .and_then(|obj| obj.get_ignore_case(name)) + .and_then(|obj| obj.as_str()) +} + diff --git a/crates/calcard/src/jscontact/import/convert.rs b/crates/calcard/src/jscontact/import/convert.rs new file mode 100644 index 00000000..c2808c9f --- /dev/null +++ b/crates/calcard/src/jscontact/import/convert.rs @@ -0,0 +1,1486 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + blob::{BlobIdGenerator, BlobIds, NoBlobIds}, + export::ImportError, + jsprop::{JSPropPointer, text::PointerString}, + }, + jscontact::{ + JSContact, JSContactId, JSContactKind, JSContactProperty, JSContactValue, + import::{EntryState, ExtractedParams, ImportOptions, State, props::NamedKey}, + }, + vcard::{ + Jscomp, VCard, VCardParameter, VCardParameterName, VCardParameterValue, VCardProperty, + VCardValue, VCardValueType, + }, +}; +use jmap_tools::{JsonPointer, Key, Map, Property, Value}; +use smallvec::smallvec; +use std::mem; + +impl VCard { + pub fn into_jscontact(self) -> JSContact<'static, I, B> + where + I: JSContactId, + B: JSContactId, + { + self.convert_jscontact::(ImportOptions::default()) + .0 + } + + pub fn into_jscontact_with( + self, + options: ImportOptions, + ) -> Result, ImportError> + where + I: JSContactId, + B: JSContactId, + G: BlobIdGenerator, + { + match self.convert_jscontact(options) { + (js_contact, false) => Ok(js_contact), + (_, true) => Err(ImportError::BlobIdFailed), + } + } + + fn convert_jscontact( + self, + mut options: ImportOptions, + ) -> (JSContact<'static, I, B>, bool) + where + I: JSContactId, + B: JSContactId, + G: BlobIdGenerator, + { + let include_vcard_parameters = options.include_vcard_parameters; + let mut blob_ids = options.blob_ids::(); + if let Some(blob_ids) = &mut blob_ids { + blob_ids.expect_sizes(self.blob_binaries().map(<[u8]>::len)); + } + let js_contact = self.build_jscontact(include_vcard_parameters, &mut blob_ids); + let has_failed = blob_ids.is_some_and(|blob_ids| blob_ids.has_failed()); + + (js_contact, has_failed) + } + + fn build_jscontact( + mut self, + include_vcard_parameters: bool, + blob_ids: &mut Option>, + ) -> JSContact<'static, I, B> + where + I: JSContactId, + B: JSContactId, + { + let mut state = State::new(&mut self, include_vcard_parameters); + + for entry in self.entries { + let mut entry = EntryState::new(entry, state.include_vcard_converted); + + match &entry.entry.name { + VCardProperty::Kind => { + if !state.has_property(JSContactProperty::Kind) + && let Some(kind) = entry.to_kind() + { + state.entries.insert(JSContactProperty::Kind, kind); + entry.set_converted_to(|| { + String::from_pointer([JSContactProperty::Kind::.to_cow().as_ref()]) + }); + } + } + VCardProperty::Source => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::PropId, + VCardParameterName::Pref, + VCardParameterName::Mediatype, + VCardParameterName::Label, + ], + JSContactProperty::Directories, + JSContactProperty::Uri, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(JSContactKind::Entry)), + )), + None, + ], + ); + } + VCardProperty::OrgDirectory => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::PropId, + VCardParameterName::Pref, + VCardParameterName::Index, + VCardParameterName::Label, + VCardParameterName::Type, + ], + JSContactProperty::Directories, + JSContactProperty::Uri, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(JSContactKind::Directory)), + )), + None, + ], + ); + } + VCardProperty::Anniversary | VCardProperty::Bday | VCardProperty::Deathdate => { + let kind = match &entry.entry.name { + VCardProperty::Anniversary => JSContactKind::Wedding, + VCardProperty::Bday => JSContactKind::Birth, + VCardProperty::Deathdate => JSContactKind::Death, + _ => unreachable!(), + }; + + state.map_named_entry( + &mut entry, + &[VCardParameterName::PropId, VCardParameterName::Calscale], + JSContactProperty::Anniversaries, + JSContactProperty::Date, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + )), + None, + ], + ); + } + VCardProperty::Birthplace | VCardProperty::Deathplace => { + // Only text and GEO can be mapped + let is_geo = matches!(entry.entry.values.first(), Some(VCardValue::Text(uri)) if uri.starts_with("geo:")); + if (is_geo || !entry.entry.is_type(&VCardValueType::Uri)) + && let Some(text) = entry.to_text() + { + // Extract language and value type + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + &[VCardParameterName::Language, VCardParameterName::PropId], + state.default_language.as_deref(), + ); + + let prop_id = params.prop_id(); + let alt_id = params.alt_id(); + + let kind = Value::Element(JSContactValue::Kind( + if entry.entry.name == VCardProperty::Birthplace { + JSContactKind::Birth + } else { + JSContactKind::Death + }, + )); + + let patch_prop = if params.language.is_some() { + &entry.entry.name + } else if entry.entry.name == VCardProperty::Birthplace { + &VCardProperty::Bday + } else { + &VCardProperty::Deathdate + }; + let mut patch_id = prop_id + .as_deref() + .filter(|prop_id| state.has_prop_id(patch_prop, prop_id)) + .or_else(|| { + state.find_prop_id( + patch_prop, + entry.entry.group.as_deref(), + alt_id.as_deref(), + ) + }) + .map(|prop_id| prop_id.to_string()); + + if patch_id.is_none() && params.language.is_some() { + entry + .entry + .params + .push(VCardParameter::language(params.language.take().unwrap())); + } + + let entries = + state.get_mut_object_or_insert(JSContactProperty::Anniversaries); + + if let Some(lang) = params.language() { + let path = format!( + "{}/{}/{}/{}", + JSContactProperty::Anniversaries::.to_cow().as_ref(), + patch_id.unwrap(), + JSContactProperty::Place::.to_cow().as_ref(), + JSContactProperty::Full::.to_cow().as_ref() + ); + + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Localizations::.to_cow().as_ref(), + lang.as_str(), + path.as_str(), + ]) + }); + + state.localize(lang, |locale| locale.push((path, text))); + } else { + // Place needs to be wrapped in an option to dance around the borrow checker + let prop_name = if !is_geo { + JSContactProperty::Full + } else { + JSContactProperty::Coordinates + }; + let mut place = + Some(Map::from(vec![(Key::Property(prop_name.clone()), text)])); + + if let Some(anniversary) = patch_id.clone().and_then(|patch_id| { + entries + .get_mut(&Key::Owned(patch_id)) + .and_then(|v| v.as_object_mut()) + }) { + anniversary.insert( + Key::Property(JSContactProperty::Place), + Value::Object(place.take().unwrap()), + ); + } + + if place.is_some() + && let Some((key, anniversary)) = + entries.iter_mut().find_map(|(k, v)| { + let obj = v.as_object_mut()?; + if obj.iter().any(|(k, v)| { + k == &Key::Property(JSContactProperty::Kind) + && v == &kind + }) && !obj + .contains_key(&Key::Property(JSContactProperty::Place)) + { + Some((k, obj)) + } else { + None + } + }) + { + anniversary.insert( + Key::Property(JSContactProperty::Place), + Value::Object(place.take().unwrap()), + ); + patch_id = Some(key.to_string().into_owned()); + } + + if let Some(place) = place { + let key = entries.named_key(prop_id); + entries.insert_unchecked( + Key::Owned(key.clone()), + Value::Object(Map::from(vec![ + (Key::Property(JSContactProperty::Kind), kind), + ( + Key::Property(JSContactProperty::Place), + Value::Object(place), + ), + ])), + ); + patch_id = Some(key); + } + + let patch_id = patch_id.unwrap(); + + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Anniversaries::.to_cow().as_ref(), + patch_id.as_ref(), + JSContactProperty::Place::.to_cow().as_ref(), + prop_name.to_cow().as_ref(), + ]) + }); + + state.prop_ids.track( + &entry.entry, + JSContactProperty::Anniversaries, + alt_id, + &patch_id, + ); + } + } + } + VCardProperty::Fn => { + if (!state.has_fn + || (entry + .non_default_language(state.default_language.as_deref()) + .is_some() + && !state.has_fn_localization)) + && let Some(text) = entry.to_text() + { + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + &[VCardParameterName::Language], + state.default_language.as_deref(), + ); + if let Some(lang) = params.language { + let path = format!( + "{}/{}", + JSContactProperty::Name::.to_cow().as_ref(), + JSContactProperty::Full::.to_cow().as_ref() + ); + + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Localizations::.to_cow().as_ref(), + lang.as_str(), + path.as_str(), + ]) + }); + state.has_fn_localization = true; + state.localize(lang, |locale| locale.push((path, text))); + } else { + state.has_fn = true; + state + .get_mut_object_or_insert(JSContactProperty::Name) + .insert(Key::Property(JSContactProperty::Full), text); + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Name::.to_cow().as_ref(), + JSContactProperty::Full::.to_cow().as_ref(), + ]) + }); + } + } + } + VCardProperty::N => { + /* + Only process this N component if it matches the main ALTID and: + - It is the first N component + - Or it has a language that has not been processed yet + */ + if state.name_alt_id.as_deref() == entry.entry.alt_id() + && (!state.has_n + || (entry + .non_default_language(state.default_language.as_deref()) + .is_some() + && !state.has_n_localization)) + { + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + &[ + VCardParameterName::Language, + VCardParameterName::Phonetic, + VCardParameterName::Script, + VCardParameterName::SortAs, + VCardParameterName::Jscomps, + ], + state.default_language.as_deref(), + ); + + if params.language.is_some() && !state.has_n { + entry + .entry + .params + .push(VCardParameter::language(params.language.take().unwrap())); + } + + /* + + 0 = family names (also known as surnames); + 1 = given names; + 2 = additional names; + 3 = honorific prefixes; + 4 = honorific suffixes; + 5 = secondary surname; + 6 = generation + + */ + + let mut default_separator = None; + let mut is_ordered = false; + let mut components = Vec::new(); + + if !params.jscomps.is_empty() { + let mut is_valid = true; + let mut has_entry = false; + + for (pos, jscomp) in + std::mem::take(&mut params.jscomps).into_iter().enumerate() + { + match jscomp { + Jscomp::Entry { position, value } => { + if let Some(kind) = + JSContactKind::from_vcard_n_pos(position as usize) + { + let value = + match entry.entry.values.get(position as usize) { + Some(VCardValue::Text(text)) if value == 0 => { + Some(text.as_str()) + } + Some(VCardValue::Component(values)) => values + .get(value as usize) + .map(|v| v.as_str()), + _ => None, + }; + + if let Some(value) = value { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(value.to_string().into()), + ), + ]))); + has_entry = true; + continue; + } + } + + is_valid = false; + break; + } + Jscomp::Separator(sep) => { + if !sep.is_empty() { + if pos > 0 { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind( + JSContactKind::Separator, + )), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(sep.into()), + ), + ]))); + } else { + default_separator = sep.into(); + } + } + } + } + } + + if is_valid && has_entry { + is_ordered = true; + entry.entry.values.clear(); + } + } + + if !is_ordered { + let has_secondary_parts = + entry + .entry + .values + .iter() + .enumerate() + .any(|(comp_id, value)| { + matches!( + JSContactKind::from_vcard_n_pos(comp_id), + Some( + JSContactKind::Surname2 | JSContactKind::Generation + ) + ) && match value { + VCardValue::Text(text) => !text.is_empty(), + VCardValue::Component(text_list) => { + !text_list.is_empty() + } + _ => false, + } + }); + + components = if has_secondary_parts { + let mut components_ = Vec::with_capacity(7); + for (comp_id, value) in entry.entry.values.iter().enumerate() { + if let Some(kind) = JSContactKind::from_vcard_n_pos(comp_id) { + match value { + VCardValue::Text(text) if !text.is_empty() => { + components_.push((kind, text.as_str())); + } + VCardValue::Component(text_list) => { + for text in text_list { + components_.push((kind, text.as_str())); + } + } + _ => {} + } + } + } + + components_ + .iter() + .filter(|(kind, value)| match kind { + JSContactKind::Credential => !components_ + .contains(&(JSContactKind::Generation, value)), + JSContactKind::Surname => { + !components_.contains(&(JSContactKind::Surname2, value)) + } + _ => true, + }) + .map(|(kind, value)| { + Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(*kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(value.to_string().into()), + ), + ])) + }) + .collect::>() + } else { + let mut parts = Vec::with_capacity(entry.entry.values.len()); + for (comp_id, value) in + mem::take(&mut entry.entry.values).into_iter().enumerate() + { + if let Some(kind) = JSContactKind::from_vcard_n_pos(comp_id) { + match value { + VCardValue::Text(text) if !text.is_empty() => { + parts.push((kind, text)); + } + VCardValue::Component(text_list) => { + parts.extend( + text_list.into_iter().map(|text| (kind, text)), + ); + } + _ => {} + } + } + } + parts + .into_iter() + .map(|(kind, value)| { + Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(value.into()), + ), + ])) + }) + .collect() + }; + } + + if let Some(lang) = params.language() { + let path = format!( + "{}/{}", + JSContactProperty::Name::.to_cow().as_ref(), + JSContactProperty::Components::.to_cow().as_ref(), + ); + + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Localizations::.to_cow().as_ref(), + lang.as_str(), + path.as_str(), + ]) + }); + + state.has_n_localization = true; + state.localize(lang, |locale| { + if !components.is_empty() { + locale.push((path, Value::Array(components))); + } + + params.members::(&entry.entry.name).for_each( + |prop, value| { + locale.push(( + format!( + "{}/{}", + JSContactProperty::Name::.to_cow().as_ref(), + prop.to_string() + ), + value, + )); + }, + ); + }); + } else { + state.has_n = true; + { + let members = params.members::(&entry.entry.name); + + if !members.is_empty() || !components.is_empty() { + let name = state + .entries + .get_mut_object_or_insert(JSContactProperty::Name); + members.append_to(name.as_mut_vec()); + if !components.is_empty() { + name.insert( + Key::Property(JSContactProperty::Components), + Value::Array(components), + ); + } + if is_ordered { + if let Some(default_separator) = default_separator { + name.insert_unchecked( + Key::Property(JSContactProperty::DefaultSeparator), + Value::Str(default_separator.into()), + ); + } + name.insert_unchecked( + Key::Property(JSContactProperty::IsOrdered), + Value::Bool(true), + ); + } + } + } + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Name::.to_cow().as_ref(), + JSContactProperty::Components::.to_cow().as_ref(), + ]) + }); + } + } + } + VCardProperty::Gramgender => { + if !state.has_gram_gender + && let Some(gram_gender) = entry.to_gram_gender() + { + ExtractedParams::default().extract( + &mut entry.entry.params, + &[], + state.default_language.as_deref(), + ); + state + .get_mut_object_or_insert(JSContactProperty::SpeakToAs) + .insert(JSContactProperty::GrammaticalGender, gram_gender); + state.has_gram_gender = true; + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::SpeakToAs::.to_cow().as_ref(), + JSContactProperty::GrammaticalGender::.to_cow().as_ref(), + ]) + }); + } + } + VCardProperty::Pronouns => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Language, + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Pref, + ], + JSContactProperty::SpeakToAs, + JSContactProperty::Pronouns, + [None, None], + ); + } + VCardProperty::Nickname => { + for value in std::mem::take(&mut entry.entry.values) { + let mut value_entry = entry.clone(); + value_entry.entry.values = smallvec![value]; + + state.map_named_entry( + &mut value_entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Language, + ], + JSContactProperty::Nicknames, + JSContactProperty::Name, + [None, None], + ); + state.add_conversion_props(value_entry); + } + continue; + } + VCardProperty::Photo | VCardProperty::Logo | VCardProperty::Sound => { + let kind = match &entry.entry.name { + VCardProperty::Photo => JSContactKind::Photo, + VCardProperty::Logo => JSContactKind::Logo, + VCardProperty::Sound => JSContactKind::Sound, + _ => unreachable!(), + }; + if !blob_ids + .as_mut() + .is_some_and(|blob_ids| state.map_blob_media(&mut entry, kind, blob_ids)) + { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Mediatype, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + ], + JSContactProperty::Media, + JSContactProperty::Uri, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + )), + None, + ], + ); + } + } + VCardProperty::Adr => { + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + &[ + VCardParameterName::Language, + VCardParameterName::Phonetic, + VCardParameterName::Script, + VCardParameterName::Label, + VCardParameterName::Geo, + VCardParameterName::Tz, + VCardParameterName::Cc, + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Pref, + VCardParameterName::Type, + VCardParameterName::Jscomps, + ], + state.default_language.as_deref(), + ); + + let prop_id = params.prop_id(); + let alt_id = params.alt_id(); + + // Locate the address to patch or reset language + let addr_patch = if params.language.is_some() { + let addr_patch = prop_id + .as_deref() + .filter(|prop_id| state.has_prop_id(&entry.entry.name, prop_id)) + .or_else(|| { + state.find_prop_id( + &entry.entry.name, + entry.entry.group.as_deref(), + alt_id.as_deref(), + ) + }); + + if addr_patch.is_none() { + entry + .entry + .params + .push(VCardParameter::language(params.language().unwrap())); + } + addr_patch + } else { + None + }; + + /* + + 0 = ADR-component-pobox ";" + 1 = ADR-component-ext ";" + 2 = ADR-component-street ";" + 3 = ADR-component-locality ";" + 4 = ADR-component-region ";" + 5 = ADR-component-code ";" + 6 = ADR-component-country ";" + 7 = ADR-component-room ";" + 8 = ADR-component-apartment ";" + 9 = ADR-component-floor ";" + 10 = ADR-component-streetnumber ";" + 11 = ADR-component-streetname ";" + 12 = ADR-component-building ";" + 13 = ADR-component-block ";" + 14 = ADR-component-subdistrict ";" + 15 = ADR-component-district ";" + 16 = ADR-component-landmark ";" + 17 = ADR-component-direction ";" + + */ + + let mut default_separator = None; + let mut is_ordered = false; + let mut components = Vec::new(); + + if !params.jscomps.is_empty() { + let mut is_valid = true; + let mut has_entry = false; + + for (pos, jscomp) in + std::mem::take(&mut params.jscomps).into_iter().enumerate() + { + match jscomp { + Jscomp::Entry { position, value } => { + if let Some(kind) = + JSContactKind::from_vcard_adr_pos(position as usize) + { + let value = match entry.entry.values.get(position as usize) + { + Some(VCardValue::Text(text)) if value == 0 => { + Some(text.as_str()) + } + Some(VCardValue::Component(values)) => { + values.get(value as usize).map(|v| v.as_str()) + } + _ => None, + }; + + if let Some(value) = value { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(value.to_string().into()), + ), + ]))); + has_entry = true; + continue; + } + } + + is_valid = false; + break; + } + Jscomp::Separator(sep) => { + if !sep.is_empty() { + if pos > 0 { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind( + JSContactKind::Separator, + )), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(sep.into()), + ), + ]))); + } else { + default_separator = sep.into(); + } + } + } + } + } + + if is_valid && has_entry { + is_ordered = true; + entry.entry.values.clear(); + } + } + + if !is_ordered { + components.clear(); + + let is_rfc9554_adr = entry.entry.values.iter().skip(7).any(|v| match v { + VCardValue::Text(text) => !text.is_empty(), + VCardValue::Component(text_list) => !text_list.is_empty(), + _ => false, + }); + + for (pos, value) in entry.entry.values.drain(..).enumerate() { + if is_rfc9554_adr && (pos == 1 || pos == 2) { + // From vCard: ignore if the ADR structured value is of the format defined in [RFC9554]. Otherwise, convert to "apartment". + // From vCard: ignore if the ADR structured value is of the format defined in [RFC9554]. Otherwise, convert to "name". + continue; + } + + if let Some(kind) = JSContactKind::from_vcard_adr_pos(pos) { + match value { + VCardValue::Text(text) if !text.is_empty() => { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(text.into()), + ), + ]))); + } + VCardValue::Component(text_list) => { + for text in text_list { + components.push(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + ), + ( + Key::Property(JSContactProperty::Value), + Value::Str(text.into()), + ), + ]))); + } + } + _ => {} + } + } + } + } + + if let Some(lang) = params.language() { + let path = format!( + "{}/{}/{}", + JSContactProperty::Addresses::.to_cow().as_ref(), + addr_patch.unwrap(), + JSContactProperty::Components::.to_cow().as_ref(), + ); + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Localizations::.to_cow().as_ref(), + lang.as_str(), + path.as_str(), + ]) + }); + + state.localize(lang, |locale| { + let base_path = path + .rsplit_once('/') + .map_or(path.as_str(), |(base, _)| base); + params.members::(&entry.entry.name).for_each( + |prop_name, value| { + locale.push(( + format!("{}/{}", base_path, prop_name.to_string()), + value, + )); + }, + ); + + if !components.is_empty() { + locale.push((path, Value::Array(components))); + } + }); + } else { + let entries = state + .entries + .get_mut_object_or_insert(JSContactProperty::Addresses); + let members = params.members::(&entry.entry.name); + let mut addr = Vec::with_capacity( + members.len() + + usize::from(!components.is_empty()) + + usize::from(is_ordered) + + usize::from(is_ordered && default_separator.is_some()), + ); + members.append_to(&mut addr); + let mut addr = Map::from(addr); + if !components.is_empty() { + addr.insert_unchecked( + Key::Property(JSContactProperty::Components), + Value::Array(components), + ); + } + if is_ordered { + if let Some(default_separator) = default_separator { + addr.insert_unchecked( + Key::Property(JSContactProperty::DefaultSeparator), + Value::Str(default_separator.into()), + ); + } + addr.insert_unchecked( + Key::Property(JSContactProperty::IsOrdered), + Value::Bool(true), + ); + } + + let prop_id = entries.named_key(prop_id); + + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Addresses::.to_cow().as_ref(), + prop_id.as_str(), + JSContactProperty::Components::.to_cow().as_ref(), + ]) + }); + + state.prop_ids.track( + &entry.entry, + JSContactProperty::Addresses, + alt_id, + &prop_id, + ); + entries.insert_unchecked(Key::Owned(prop_id), Value::Object(addr)); + } + } + VCardProperty::Email => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + ], + JSContactProperty::Emails, + JSContactProperty::Address, + [None, None], + ); + } + VCardProperty::Impp => { + entry.set_map_name(); + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + VCardParameterName::ServiceType, + VCardParameterName::Username, + ], + JSContactProperty::OnlineServices, + JSContactProperty::Uri, + [None, None], + ); + } + VCardProperty::Lang => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + ], + JSContactProperty::PreferredLanguages, + JSContactProperty::Language, + [None, None], + ); + } + VCardProperty::Language | VCardProperty::Prodid | VCardProperty::Uid => { + let property = match &entry.entry.name { + VCardProperty::Language => JSContactProperty::Language, + VCardProperty::Prodid => JSContactProperty::ProdId, + VCardProperty::Uid => JSContactProperty::Uid, + _ => unreachable!(), + }; + + if !state.entries.contains(&property) + && let Some(text) = entry.to_text() + { + entry.set_converted_to(|| { + String::from_pointer([property.to_cow().as_ref()]) + }); + state.entries.insert(property, text); + } + } + VCardProperty::Socialprofile => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + VCardParameterName::ServiceType, + VCardParameterName::Username, + ], + JSContactProperty::OnlineServices, + JSContactProperty::Uri, + [None, None], + ); + } + VCardProperty::Tel => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + ], + JSContactProperty::Phones, + JSContactProperty::Number, + [None, None], + ); + } + VCardProperty::ContactUri => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + VCardParameterName::Mediatype, + ], + JSContactProperty::Links, + JSContactProperty::Uri, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(JSContactKind::Contact)), + )), + None, + ], + ); + } + VCardProperty::Title => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Language, + ], + JSContactProperty::Titles, + JSContactProperty::Name, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(JSContactKind::Title)), + )), + None, + ], + ); + } + VCardProperty::Role => { + let prop_id = state + .find_prop_id( + &VCardProperty::Org, + entry.entry.group.as_deref(), + entry.entry.alt_id(), + ) + .map(|s| s.to_string()); + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Language, + ], + JSContactProperty::Titles, + JSContactProperty::Name, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(JSContactKind::Role)), + )), + prop_id.map(|prop_id| { + ( + Key::Property(JSContactProperty::OrganizationId), + Value::Str(prop_id.into()), + ) + }), + ], + ); + } + VCardProperty::Hobby | VCardProperty::Interest | VCardProperty::Expertise => { + let kind = match &entry.entry.name { + VCardProperty::Expertise => JSContactKind::Expertise, + VCardProperty::Hobby => JSContactKind::Hobby, + VCardProperty::Interest => JSContactKind::Interest, + _ => unreachable!(), + }; + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Level, + VCardParameterName::Index, + VCardParameterName::Label, + VCardParameterName::PropId, + VCardParameterName::Altid, + VCardParameterName::Language, + ], + JSContactProperty::PersonalInfo, + JSContactProperty::Value, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + )), + None, + ], + ); + } + VCardProperty::Org => { + let units = entry + .text_parts_borrowed() + .skip(1) + .map(|unit| { + Value::Object(Map::from(vec![( + Key::Property(JSContactProperty::Name), + Value::Str(unit.to_string().into()), + )])) + }) + .collect::>(); + + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::PropId, + VCardParameterName::SortAs, + ], + JSContactProperty::Organizations, + JSContactProperty::Name, + [ + (!units.is_empty()).then_some(( + Key::Property(JSContactProperty::Units), + Value::Array(units), + )), + None, + ], + ); + } + VCardProperty::Member | VCardProperty::Categories => { + let key = if entry.entry.name == VCardProperty::Member { + JSContactProperty::Members + } else { + JSContactProperty::Keywords + }; + + entry.set_converted_to(|| String::from_pointer([key.to_cow().as_ref()])); + + let obj = state.get_mut_object_or_insert(key); + + for key in entry.entry.values.drain(..).filter_map(|v| v.into_text()) { + obj.insert(Key::from(key), Value::Bool(true)); + } + } + VCardProperty::Created | VCardProperty::Rev => { + let property = match &entry.entry.name { + VCardProperty::Created => JSContactProperty::Created, + VCardProperty::Rev => JSContactProperty::Updated, + _ => unreachable!(), + }; + + if !state.entries.contains(&property) + && let Some(text) = entry.to_timestamp() + { + entry.set_converted_to(|| { + String::from_pointer([property.to_cow().as_ref()]) + }); + state.entries.insert(property, text); + } + } + VCardProperty::Note => { + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Author, + VCardParameterName::AuthorName, + VCardParameterName::Created, + VCardParameterName::Language, + VCardParameterName::PropId, + ], + JSContactProperty::Notes, + JSContactProperty::Note, + [None, None], + ); + } + VCardProperty::Url | VCardProperty::Key | VCardProperty::Caladruri => { + let prop = match &entry.entry.name { + VCardProperty::Url => JSContactProperty::Links, + VCardProperty::Key => JSContactProperty::CryptoKeys, + VCardProperty::Caladruri => JSContactProperty::SchedulingAddresses, + _ => unreachable!(), + }; + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + VCardParameterName::Mediatype, + ], + prop, + JSContactProperty::Uri, + [None, None], + ); + } + VCardProperty::Fburl | VCardProperty::Caluri => { + let kind = match &entry.entry.name { + VCardProperty::Fburl => JSContactKind::FreeBusy, + VCardProperty::Caluri => JSContactKind::Calendar, + _ => unreachable!(), + }; + state.map_named_entry( + &mut entry, + &[ + VCardParameterName::Type, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + VCardParameterName::Mediatype, + ], + JSContactProperty::Calendars, + JSContactProperty::Uri, + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + )), + None, + ], + ); + } + VCardProperty::Related => { + if let Some(text) = entry.to_string() { + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + &[VCardParameterName::Type], + state.default_language.as_deref(), + ); + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::RelatedTo::.to_cow().as_ref(), + text.as_ref(), + ]) + }); + state + .get_mut_object_or_insert(JSContactProperty::RelatedTo) + .insert( + Key::from(text), + Value::Object(Map::from(vec![( + Key::Property(JSContactProperty::Relation), + Value::Object(params.types().into_iter().fold( + Map::new(), + |mut relation, typ| { + relation.insert( + Key::Owned(typ.into_string().to_ascii_lowercase()), + Value::Bool(true), + ); + relation + }, + )), + )])), + ); + } + } + VCardProperty::Tz | VCardProperty::Geo => { + let (key, value) = match &entry.entry.name { + VCardProperty::Tz => { + (Key::Property(JSContactProperty::TimeZone), entry.to_tz()) + } + VCardProperty::Geo => ( + Key::Property(JSContactProperty::Coordinates), + entry.to_text(), + ), + _ => unreachable!(), + }; + + if let Some(value) = value { + let prop_id = state + .find_prop_id( + &VCardProperty::Adr, + entry.entry.group.as_deref(), + entry.entry.alt_id(), + ) + .map(|s| s.to_string()); + + entry.set_map_name(); + + let addresses = + state.get_mut_object_or_insert(JSContactProperty::Addresses); + if let Some(addr) = prop_id + .as_deref() + .and_then(|prop_id| { + addresses + .as_mut_vec() + .iter_mut() + .find_map(|(member, value)| { + (*member == prop_id).then_some(value) + }) + }) + .and_then(|v| v.as_object_mut()) + .filter(|v| !v.contains_key(&key)) + { + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Addresses::.to_cow().as_ref(), + prop_id.as_deref().unwrap_or_default(), + key.to_string().as_ref(), + ]) + }); + addr.insert(key, value); + } else { + let prop_id = addresses.named_key(None); + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Addresses::.to_cow().as_ref(), + prop_id.as_str(), + key.to_string().as_ref(), + ]) + }); + addresses.insert_unchecked( + Key::Owned(prop_id), + Value::Object(Map::from(vec![(key, value)])), + ); + } + } + } + VCardProperty::Other(name) + if name.eq_ignore_ascii_case("X-ABLabel") && entry.entry.group.is_some() => + { + if let Some(prop) = state.find_entry_by_group(entry.entry.group.as_deref()) { + let prop_id = prop.prop_id.to_string(); + let prop_js = prop.prop_js.clone(); + + if let Some(obj) = state + .entries + .get_mut(&prop_js) + .and_then(|v| v.as_object_mut()) + .and_then(|v| v.get_mut(&Key::Owned(prop_id.clone()))) + .and_then(|v| v.as_object_mut()) + .filter(|v| !v.contains_key(&Key::Property(JSContactProperty::Label))) + && let Some(value) = entry.to_text() + { + obj.insert_unchecked(Key::Property(JSContactProperty::Label), value); + entry.set_map_name(); + entry.set_converted_to(|| { + String::from_pointer([ + prop_js.to_cow().as_ref(), + prop_id.as_str(), + JSContactProperty::Label::.to_cow().as_ref(), + ]) + }); + } + } + } + VCardProperty::Jsprop => { + if let Some(VCardParameter { + name: VCardParameterName::Jsptr, + value: VCardParameterValue::Text(ptr), + }) = entry.entry.params.first() + { + let ptr = JsonPointer::>::parse(ptr); + + if JSContactProperty::is_metadata_pointer(&ptr) { + continue; + } + if let Some(VCardValue::Text(text)) = entry.entry.values.first() + && let Some(patch) = ptr.parse_jsprop_value(text) + { + state.patch_objects.push((ptr, patch)); + continue; + } + } + } + VCardProperty::Version + | VCardProperty::Xml + | VCardProperty::Gender + | VCardProperty::Clientpidmap + | VCardProperty::Other(_) => (), + VCardProperty::Begin | VCardProperty::End => { + continue; + } + } + + state.add_conversion_props(entry); + } + + state.into_jscontact() + } +} diff --git a/crates/calcard/src/jscontact/import/entry.rs b/crates/calcard/src/jscontact/import/entry.rs new file mode 100644 index 00000000..69d579ac --- /dev/null +++ b/crates/calcard/src/jscontact/import/entry.rs @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{IanaString, IanaType, writer::write_jscomps}, + jscontact::{ + JSContactGrammaticalGender, JSContactId, JSContactKind, JSContactProperty, JSContactType, + JSContactValue, + import::{EntryState, VCardParams}, + }, + vcard::{ + VCardEntry, VCardGramGender, VCardKind, VCardParameterName, VCardValue, VCardValueType, + }, +}; +use jmap_tools::{Key, Map, Value}; +use std::borrow::Cow; + +#[allow(clippy::wrong_self_convention)] +impl EntryState { + pub(super) fn new(entry: VCardEntry, keep_converted_path: bool) -> Self { + Self { + entry, + converted_to: None, + map_name: false, + keep_converted_path, + } + } + + pub(super) fn has_conversion_params(&self) -> bool { + self.map_name || !self.entry.params.is_empty() || self.entry.group.is_some() + } + + pub(super) fn jcal_parameters( + &mut self, + params: &mut VCardParams, + value_type: &mut Option>, + ) { + if self.entry.params.is_empty() && self.entry.group.is_none() { + return; + } + let (default_type, _) = self.entry.name.default_types(); + let default_type = default_type.unwrap_vcard(); + + for param in std::mem::take(&mut self.entry.params) { + let value = match ¶m.name { + VCardParameterName::Value => { + if let Some(v) = param + .value + .into_value_type() + .filter(|v| !v.is_iana_and(|v| v == &default_type)) + { + *value_type = Some(v); + } + continue; + } + + VCardParameterName::Created => { + let Some(IanaType::Iana(timestamp)) = param.value.into_timestamp() else { + continue; + }; + + Value::Element(JSContactValue::Timestamp(timestamp)) + } + VCardParameterName::Jscomps => { + let Some(v) = param.value.into_jscomps() else { + continue; + }; + let mut jscomps = String::new(); + let _ = write_jscomps(&mut jscomps, &v); + Value::Str(jscomps.into()) + } + VCardParameterName::Other(name) if name.eq_ignore_ascii_case("group") => { + // The GROUP parameter (Section 7.1 of [RFC7095]) does not convert to JSContact. + // It exclusively is for use in jCard and MUST NOT be set in a vCard. + continue; + } + VCardParameterName::Group => { + // The GROUP parameter (Section 7.1 of [RFC7095]) does not convert to JSContact. + // It exclusively is for use in jCard and MUST NOT be set in a vCard. + continue; + } + _ => Value::Str(param.value.into_text()), + }; + + params.push(param.name, value); + } + + if let Some(group) = self.entry.group.take() { + params.set(VCardParameterName::Group, vec![group.into()]); + } + } + + pub(super) fn set_converted_to(&mut self, converted_to: impl FnOnce() -> String) { + self.converted_to = Some( + if self.keep_converted_path && self.has_conversion_params() { + converted_to() + } else { + String::new() + }, + ); + } + + pub(super) fn set_map_name(&mut self) { + self.map_name = true; + } + + pub(super) fn to_text( + &mut self, + ) -> Option, JSContactValue>> { + self.to_string().map(Into::into) + } + + pub(super) fn to_string(&mut self) -> Option> { + let mut values = std::mem::take(&mut self.entry.values).into_iter(); + match values.next()? { + VCardValue::Text(v) => Some(Cow::Owned(v)), + VCardValue::Component(v) => Some(Cow::Owned(v.join(","))), + VCardValue::Binary(data) => Some(Cow::Owned(data.to_unwrapped_string())), + VCardValue::Sex(v) => Some(v.as_str().into()), + VCardValue::GramGender(v) => Some(v.as_str().into()), + VCardValue::Kind(v) => Some(v.as_str().into()), + other => { + self.entry.values.push(other); + self.entry.values.extend(values); + None + } + } + } + + pub(super) fn text_parts_borrowed(&self) -> impl Iterator + '_ { + self.entry.values.iter().filter_map(|value| value.as_text()) + } + + pub(super) fn to_kind( + &mut self, + ) -> Option, JSContactValue>> { + let mut values = std::mem::take(&mut self.entry.values).into_iter(); + match values.next()? { + VCardValue::Kind(v) => Value::Element(JSContactValue::Kind(match v { + VCardKind::Application => JSContactKind::Application, + VCardKind::Device => JSContactKind::Device, + VCardKind::Group => JSContactKind::Group, + VCardKind::Individual => JSContactKind::Individual, + VCardKind::Location => JSContactKind::Location, + VCardKind::Org => JSContactKind::Org, + })) + .into(), + VCardValue::Text(v) => Value::Str(v.into()).into(), + VCardValue::Component(v) => Value::Str(v.join(",").into()).into(), + other => { + self.entry.values.push(other); + self.entry.values.extend(values); + None + } + } + } + + pub(super) fn to_gram_gender( + &mut self, + ) -> Option, JSContactValue>> { + let mut values = std::mem::take(&mut self.entry.values).into_iter(); + match values.next()? { + VCardValue::GramGender(v) => { + Value::Element(JSContactValue::GrammaticalGender(match v { + VCardGramGender::Animate => JSContactGrammaticalGender::Animate, + VCardGramGender::Common => JSContactGrammaticalGender::Common, + VCardGramGender::Feminine => JSContactGrammaticalGender::Feminine, + VCardGramGender::Inanimate => JSContactGrammaticalGender::Inanimate, + VCardGramGender::Masculine => JSContactGrammaticalGender::Masculine, + VCardGramGender::Neuter => JSContactGrammaticalGender::Neuter, + })) + .into() + } + VCardValue::Text(v) => Value::Str(v.into()).into(), + VCardValue::Component(v) => Value::Str(v.join(",").into()).into(), + other => { + self.entry.values.push(other); + self.entry.values.extend(values); + None + } + } + } + + pub(super) fn to_timestamp( + &mut self, + ) -> Option, JSContactValue>> { + if let Some(VCardValue::PartialDateTime(dt)) = self.entry.values.first() + && let Some(timestamp) = dt.to_timestamp() + { + return Value::Element(JSContactValue::Timestamp(timestamp)).into(); + } + + None + } + + pub(super) fn to_tz( + &mut self, + ) -> Option, JSContactValue>> { + let mut values = std::mem::take(&mut self.entry.values).into_iter(); + match values.next()? { + VCardValue::PartialDateTime(v) if v.has_zone() => { + let hour = v.tz_hour.unwrap_or_default(); + + Value::Str( + if hour != 0 { + format!("Etc/GMT{}{}", if v.tz_minus { "+" } else { "-" }, hour) + } else { + "Etc/UTC".to_string() + } + .into(), + ) + .into() + } + VCardValue::Text(v) => Value::Str(v.into()).into(), + VCardValue::Component(v) => Value::Str(v.join(",").into()).into(), + other => { + self.entry.values.push(other); + self.entry.values.extend(values); + None + } + } + } + + pub(super) fn to_anniversary( + &mut self, + ) -> Option, JSContactValue>> { + if let Some(VCardValue::PartialDateTime(dt)) = self.entry.values.first() { + if let Some(timestamp) = dt.to_timestamp() { + Some(Value::Object(Map::from(vec![ + ( + Key::Property(JSContactProperty::Type), + Value::Element(JSContactValue::Type(JSContactType::Timestamp)), + ), + ( + Key::Property(JSContactProperty::Utc), + Value::Element(JSContactValue::Timestamp(timestamp)), + ), + ]))) + } else if dt.day.is_some() || dt.month.is_some() || dt.year.is_some() { + let mut props = Map::from(vec![( + Key::Property(JSContactProperty::Type), + Value::Element(JSContactValue::Type(JSContactType::PartialDate)), + )]); + if let Some(year) = dt.year { + props.insert_unchecked( + Key::Property(JSContactProperty::Year), + Value::Number((year as u64).into()), + ); + } + if let Some(month) = dt.month { + props.insert_unchecked( + Key::Property(JSContactProperty::Month), + Value::Number((month as u64).into()), + ); + } + if let Some(day) = dt.day { + props.insert_unchecked( + Key::Property(JSContactProperty::Day), + Value::Number((day as u64).into()), + ); + } + + Some(Value::Object(props)) + } else { + None + } + } else { + None + } + } + + pub(super) fn non_default_language(&self, default_language: Option<&str>) -> Option<&str> { + self.entry + .language() + .filter(|&lang| Some(lang) != default_language) + } +} diff --git a/crates/calcard/src/jscontact/import/mod.rs b/crates/calcard/src/jscontact/import/mod.rs new file mode 100644 index 00000000..4885fa3d --- /dev/null +++ b/crates/calcard/src/jscontact/import/mod.rs @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + CalendarScale, IanaType, + blob::{BlobIdFn, BlobIdGenerator, BlobIds, BlobOptions, NoBlobIds}, + jsprop::ordered::OrderedMap, + }, + jscontact::{JSContactId, JSContactProperty, JSContactValue}, + vcard::{ + Jscomp, VCardEntry, VCardLevel, VCardParameterName, VCardPhonetic, VCardProperty, VCardType, + }, +}; +use jmap_tools::{Element, JsonPointer, Key, Map, Property, Value}; + +pub mod convert; +pub mod entry; +pub mod params; +pub mod props; + +type JSValue = Value<'static, JSContactProperty, JSContactValue>; + +type Member = (Key<'static, JSContactProperty>, JSValue); + +type PatchObject = (JsonPointer>, JSValue); + +type Localization = Vec<(String, JSValue)>; + +struct CardProperties(Vec>) +where + I: JSContactId, + B: JSContactId; + +struct State +where + I: JSContactId, + B: JSContactId, +{ + entries: CardProperties, + vcard_converted_properties: OrderedMap>, + vcard_properties: Vec>, + patch_objects: Vec>, + localizations: OrderedMap>, + default_language: Option, + prop_ids: PropIds, + name_alt_id: Option, + has_fn: bool, + has_fn_localization: bool, + has_n: bool, + has_n_localization: bool, + has_gram_gender: bool, + include_vcard_converted: bool, +} + +#[derive(Debug, Clone)] +struct EntryState { + entry: VCardEntry, + converted_to: Option, + map_name: bool, + keep_converted_path: bool, +} + +struct PropIdKey +where + I: JSContactId, +{ + prop: VCardProperty, + prop_js: JSContactProperty, + group: Option, + alt_id: Option, + prop_id: String, +} + +struct PropIds +where + I: JSContactId, +{ + keys: Vec>, + is_enabled: bool, +} + +#[derive(Debug, Default)] +struct VCardConvertedProperty +where + I: JSContactId, + B: JSContactId, +{ + name: Option, + params: VCardParams, +} + +#[derive(Debug, Default)] +struct VCardParams(OrderedMap>>) +where + I: JSContactId, + B: JSContactId; + +#[derive(Default)] +struct ExtractedParams { + language: Option, + pref: Option, + author: Option, + author_name: Option, + phonetic_system: Option>, + phonetic_script: Option, + media_type: Option, + calscale: Option>, + sort_as: Option, + prop_id: Option, + alt_id: Option, + types: Vec>, + geo: Option, + tz: Option, + index: Option, + level: Option>, + country_code: Option, + created: Option, + label: Option, + service_type: Option, + username: Option, + jscomps: Vec, +} + +impl CardProperties +where + I: JSContactId, + B: JSContactId, +{ + fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + fn position(&self, property: &JSContactProperty) -> Option { + self.0 + .iter() + .position(|(key, _)| matches!(key, Key::Property(existing) if existing == property)) + } + + fn contains(&self, property: &JSContactProperty) -> bool { + self.position(property).is_some() + } + + fn get_mut(&mut self, property: &JSContactProperty) -> Option<&mut JSValue> { + self.0.iter_mut().find_map(|(key, value)| { + matches!(key, Key::Property(existing) if existing == property).then_some(value) + }) + } + + fn insert(&mut self, property: JSContactProperty, value: JSValue) { + match self.get_mut(&property) { + Some(existing) => *existing = value, + None => self.0.push((Key::Property(property), value)), + } + } + + fn get_mut_object_or_insert( + &mut self, + property: JSContactProperty, + ) -> &mut Map<'static, JSContactProperty, JSContactValue> { + let index = self.position(&property).unwrap_or_else(|| { + self.0.push(( + Key::Property(property), + Value::Object(Map::from(Vec::new())), + )); + self.0.len() - 1 + }); + self.0[index].1.object_or_replace() + } + + fn into_map(self) -> Map<'static, JSContactProperty, JSContactValue> { + Map::from(self.0) + } +} + +impl VCardParams +where + I: JSContactId, + B: JSContactId, +{ + fn push(&mut self, name: VCardParameterName, value: JSValue) { + match self.0.get_mut(&name) { + Some(values) => values.push(value), + None => self.0.push(name, vec![value]), + } + } + + fn set(&mut self, name: VCardParameterName, values: Vec>) { + self.0.insert(name, values); + } +} + +trait ObjectValue<'x, P: Property, E: Element> { + fn object_or_replace(&mut self) -> &mut Map<'x, P, E>; +} + +impl<'x, P: Property, E: Element> ObjectValue<'x, P, E> for Value<'x, P, E> { + #[inline] + fn object_or_replace(&mut self) -> &mut Map<'x, P, E> { + match self { + Value::Object(object) => object, + value => { + *value = Value::Object(Map::from(Vec::new())); + value.object_or_replace() + } + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ImportOptions { + include_vcard_parameters: bool, + blobs: BlobOptions, +} + +impl Default for ImportOptions { + fn default() -> Self { + Self { + include_vcard_parameters: true, + blobs: BlobOptions::default(), + } + } +} + +impl ImportOptions { + pub fn new() -> Self { + Self::default() + } +} + +impl ImportOptions { + pub fn include_vcard_parameters(mut self, include: bool) -> Self { + self.include_vcard_parameters = include; + self + } + + pub fn with_blob_ids(self, blob_ids: F) -> ImportOptions> + where + F: FnMut(&[u8]) -> Option, + { + self.with_blob_id_generator(BlobIdFn(blob_ids)) + } + + pub fn with_blob_id_generator(self, blob_id_generator: T) -> ImportOptions { + ImportOptions { + include_vcard_parameters: self.include_vcard_parameters, + blobs: self.blobs.with_handler(blob_id_generator), + } + } + + pub(super) fn blob_ids(&mut self) -> Option> + where + G: BlobIdGenerator, + { + self.blobs.blob_ids() + } +} diff --git a/crates/calcard/src/jscontact/import/params.rs b/crates/calcard/src/jscontact/import/params.rs new file mode 100644 index 00000000..b9d9fe6b --- /dev/null +++ b/crates/calcard/src/jscontact/import/params.rs @@ -0,0 +1,410 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{IanaType, jsprop::text::IntoAsciiLowercase}, + jscontact::{ + Context, Feature, JSContactId, JSContactKind, JSContactLevel, JSContactPhoneticSystem, + JSContactProperty, JSContactValue, + import::{ExtractedParams, Member, VCardParams}, + }, + vcard::{ + VCardLevel, VCardParameter, VCardParameterName, VCardParameterValue, VCardPhonetic, + VCardProperty, VCardType, + }, +}; +use jmap_tools::{Key, Map, Value}; +use std::{mem, str::FromStr}; + +impl Feature { + pub(crate) fn from_vcard_type(property: &VCardProperty, typ: &VCardType) -> Option { + match (property, typ) { + (VCardProperty::Tel, VCardType::Cell) => Some(Feature::Mobile), + (VCardProperty::Tel, VCardType::Fax) => Some(Feature::Fax), + (VCardProperty::Tel, VCardType::MainNumber) => Some(Feature::MainNumber), + (VCardProperty::Tel, VCardType::Pager) => Some(Feature::Pager), + (VCardProperty::Tel, VCardType::Text) => Some(Feature::Text), + (VCardProperty::Tel, VCardType::Textphone) => Some(Feature::TextPhone), + (VCardProperty::Tel, VCardType::Video) => Some(Feature::Video), + (VCardProperty::Tel, VCardType::Voice) => Some(Feature::Voice), + _ => None, + } + } +} + +impl JSContactProperty { + pub(crate) fn from_vcard_type( + property: &VCardProperty, + typ: IanaType, + ) -> (Self, Key<'static, Self>) { + let feature = match &typ { + IanaType::Iana(typ) => Feature::from_vcard_type(property, typ), + IanaType::Other(_) => None, + }; + if let Some(feature) = feature { + return ( + JSContactProperty::Features, + Key::Property(JSContactProperty::Feature(feature)), + ); + } + + let key = match typ { + IanaType::Iana(VCardType::Home) => { + Key::Property(JSContactProperty::Context(Context::Private)) + } + IanaType::Iana(VCardType::Work) => { + Key::Property(JSContactProperty::Context(Context::Work)) + } + IanaType::Iana(VCardType::Billing) => { + Key::Property(JSContactProperty::Context(Context::Billing)) + } + IanaType::Iana(VCardType::Delivery) => { + Key::Property(JSContactProperty::Context(Context::Delivery)) + } + IanaType::Iana(VCardType::Cell) => Key::Borrowed("mobile"), + typ => { + let key = typ.into_string().to_ascii_lowercase(); + match Context::from_str(&key) { + Ok(context) => Key::Property(JSContactProperty::Context(context)), + Err(()) => Key::Owned(key), + } + } + }; + (JSContactProperty::Contexts, key) + } +} + +impl VCardParameter { + pub(super) fn take_value(&mut self) -> VCardParameterValue { + mem::replace(&mut self.value, VCardParameterValue::Null) + } + + pub(super) fn as_media_type(&self) -> Option<&str> { + match (&self.name, &self.value) { + (VCardParameterName::Mediatype, VCardParameterValue::Text(media_type)) + if !media_type.is_empty() => + { + Some(media_type) + } + _ => None, + } + } +} + +impl ExtractedParams { + pub(super) fn prop_id(&mut self) -> Option { + self.prop_id.take() + } + + pub(super) fn alt_id(&mut self) -> Option { + self.alt_id.take() + } + + pub(super) fn language(&mut self) -> Option { + self.language.take() + } + + pub(super) fn types(&mut self) -> Vec> { + std::mem::take(&mut self.types) + } + + pub(super) fn members( + &mut self, + property: &VCardProperty, + ) -> ParamMembers<'_, I, B> { + let mut contexts: Option>> = None; + let mut features: Option>> = None; + + for typ in std::mem::take(&mut self.types) { + let (bucket, key) = JSContactProperty::::from_vcard_type(property, typ); + let keys = if bucket == JSContactProperty::Features { + features.get_or_insert_default() + } else { + contexts.get_or_insert_default() + }; + if keys.iter().all(|(existing, _)| existing != &key) { + keys.push((key, Value::Bool(true))); + } + } + + ParamMembers { + contexts, + features, + is_name: matches!(property, VCardProperty::N), + is_address: matches!(property, VCardProperty::Adr), + params: self, + } + } +} + +pub(super) struct ParamMembers<'x, I: JSContactId, B: JSContactId> { + contexts: Option>>, + features: Option>>, + is_name: bool, + is_address: bool, + params: &'x mut ExtractedParams, +} + +impl ParamMembers<'_, I, B> { + pub(super) fn len(&self) -> usize { + let p = &*self.params; + [ + self.contexts.is_some(), + self.features.is_some(), + p.language.is_some(), + p.pref.is_some(), + p.author.is_some() || p.author_name.is_some(), + p.media_type.is_some(), + p.phonetic_system.is_some(), + p.phonetic_script.is_some(), + p.calscale.is_some(), + p.sort_as.is_some(), + p.geo.is_some(), + p.tz.is_some(), + p.index.is_some(), + p.level.is_some(), + p.country_code.is_some(), + p.created.is_some(), + p.label.is_some(), + p.service_type.is_some(), + p.username.is_some(), + ] + .into_iter() + .filter(|&present| present) + .count() + } + + pub(super) fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub(super) fn append_to(self, members: &mut Vec>) { + self.for_each(|key, value| members.push((key, value))); + } + + pub(super) fn for_each( + self, + mut emit: impl FnMut( + Key<'static, JSContactProperty>, + Value<'static, JSContactProperty, JSContactValue>, + ), + ) { + let p = self.params; + + if let Some(contexts) = self.contexts { + emit( + Key::Property(JSContactProperty::Contexts), + Value::Object(Map::from(contexts)), + ); + } + if let Some(features) = self.features { + emit( + Key::Property(JSContactProperty::Features), + Value::Object(Map::from(features)), + ); + } + if let Some(language) = p.language.take() { + emit( + Key::Property(JSContactProperty::Language), + Value::Str(language.into()), + ); + } + if let Some(pref) = p.pref.take() { + emit( + Key::Property(JSContactProperty::Pref), + Value::Number((pref as u64).into()), + ); + } + if p.author.is_some() || p.author_name.is_some() { + emit( + Key::Property(JSContactProperty::Author), + Value::Object(Map::from_iter( + [ + p.author_name.take().map(|name| { + ( + Key::Property(JSContactProperty::Name), + Value::Str(name.into()), + ) + }), + p.author.take().map(|author| { + ( + Key::Property(JSContactProperty::Uri), + Value::Str(author.into()), + ) + }), + ] + .into_iter() + .flatten(), + )), + ); + } + if let Some(media_type) = p.media_type.take() { + emit( + Key::Property(JSContactProperty::MediaType), + Value::Str(media_type.into()), + ); + } + if let Some(phonetic_system) = p.phonetic_system.take() { + emit( + Key::Property(JSContactProperty::PhoneticSystem), + match phonetic_system { + IanaType::Iana(value) => { + Value::Element(JSContactValue::PhoneticSystem(match value { + VCardPhonetic::Ipa => JSContactPhoneticSystem::Ipa, + VCardPhonetic::Jyut => JSContactPhoneticSystem::Jyut, + VCardPhonetic::Piny => JSContactPhoneticSystem::Piny, + VCardPhonetic::Script => JSContactPhoneticSystem::Script, + })) + } + IanaType::Other(value) => Value::Str(value.into()), + }, + ); + } + if let Some(phonetic_script) = p.phonetic_script.take() { + emit( + Key::Property(JSContactProperty::PhoneticScript), + Value::Str(phonetic_script.into()), + ); + } + if let Some(calscale) = p.calscale.take() { + emit( + Key::Property(JSContactProperty::CalendarScale), + match calscale { + IanaType::Iana(value) => Value::Element(JSContactValue::CalendarScale(value)), + IanaType::Other(value) => Value::Str(value.into()), + }, + ); + } + if let Some(sort_as) = p.sort_as.take() { + emit( + Key::Property(JSContactProperty::SortAs), + if self.is_name { + if let Some((surname, given)) = + sort_as.split_once(',').and_then(|(surname, given)| { + let surname = surname.trim(); + let given = given.trim(); + + if surname.is_empty() || given.is_empty() { + None + } else { + Some((surname.to_string(), given.to_string())) + } + }) + { + Value::Object(Map::from_iter(vec![ + ( + Key::Property(JSContactProperty::SortAsKind( + JSContactKind::Surname, + )), + Value::Str(surname.into()), + ), + ( + Key::Property(JSContactProperty::SortAsKind(JSContactKind::Given)), + Value::Str(given.into()), + ), + ])) + } else { + Value::Object(Map::from_iter(vec![( + Key::Property(JSContactProperty::SortAsKind(JSContactKind::Surname)), + Value::Str(sort_as.into()), + )])) + } + } else { + Value::Str(sort_as.into()) + }, + ); + } + if let Some(geo) = p.geo.take() { + emit( + Key::Property(JSContactProperty::Coordinates), + Value::Str(geo.into()), + ); + } + if let Some(tz) = p.tz.take() { + emit( + Key::Property(JSContactProperty::TimeZone), + Value::Str(tz.into()), + ); + } + if let Some(index) = p.index.take() { + emit( + Key::Property(JSContactProperty::ListAs), + Value::Number((index as u64).into()), + ); + } + if let Some(level) = p.level.take() { + emit( + Key::Property(JSContactProperty::Level), + match level { + IanaType::Iana(value) => Value::Element(JSContactValue::Level(match value { + VCardLevel::Beginner | VCardLevel::Low => JSContactLevel::Low, + VCardLevel::Average | VCardLevel::Medium => JSContactLevel::Medium, + VCardLevel::Expert | VCardLevel::High => JSContactLevel::High, + })), + IanaType::Other(value) => Value::Str(value.into()), + }, + ); + } + if let Some(country_code) = p.country_code.take() { + emit( + Key::Property(JSContactProperty::CountryCode), + Value::Str(country_code.into()), + ); + } + if let Some(created) = p.created.take() { + emit( + Key::Property(JSContactProperty::Created), + Value::Element(JSContactValue::Timestamp(created)), + ); + } + if let Some(label) = p.label.take() { + emit( + Key::Property(if self.is_address { + JSContactProperty::Full + } else { + JSContactProperty::Label + }), + Value::Str(label.into()), + ); + } + if let Some(service_type) = p.service_type.take() { + emit( + Key::Property(JSContactProperty::Service), + Value::Str(service_type.into()), + ); + } + if let Some(username) = p.username.take() { + emit( + Key::Property(JSContactProperty::User), + Value::Str(username.into()), + ); + } + } +} + +impl VCardParams { + pub(super) fn into_jscontact_value( + self, + ) -> Option, JSContactValue>> { + if !self.0.is_empty() { + let mut obj = Map::from(Vec::with_capacity(self.0.len())); + + for (param, value) in self.0 { + let value = match <[_; 1]>::try_from(value) { + Ok([value]) => value, + Err(values) => Value::Array(values), + }; + obj.insert_unchecked( + Key::Owned(param.into_string().into_ascii_lowercase()), + value, + ); + } + Some(obj) + } else { + None + } + } +} diff --git a/crates/calcard/src/jscontact/import/props.rs b/crates/calcard/src/jscontact/import/props.rs new file mode 100644 index 00000000..84a10bcb --- /dev/null +++ b/crates/calcard/src/jscontact/import/props.rs @@ -0,0 +1,965 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + Data, IanaString, IanaType, + blob::{BlobIds, sniff_media_type}, + format::AsciiPush, + jsprop::{ + ordered::OrderedMap, + text::{AsciiString, PointerString}, + }, + }, + jscontact::{ + JSContact, JSContactId, JSContactKind, JSContactProperty, JSContactType, JSContactValue, + import::{ + CardProperties, EntryState, ExtractedParams, Localization, Member, PropIdKey, PropIds, + State, VCardConvertedProperty, VCardParams, + }, + }, + vcard::{ + VCard, VCardEntry, VCardParameter, VCardParameterName, VCardParameterValue, VCardProperty, + VCardValue, VCardValueType, + }, +}; +use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Map, Property, Value}; +use std::{ + borrow::Cow, + hash::{Hash, Hasher}, + iter::once, + mem, +}; + +const NUMBERED_KEY_CAPACITY: usize = 8; + +#[derive(Clone, Copy)] +struct LanguageTag<'x>(&'x str); + +impl State +where + I: JSContactId, + B: JSContactId, +{ + pub(super) fn new(vcard: &mut VCard, include_vcard_converted: bool) -> Self { + let mut entries = CardProperties::with_capacity(vcard.entries.len().min(20) + 4); + + entries.insert( + JSContactProperty::Type, + Value::Element(JSContactValue::Type(JSContactType::Card)), + ); + entries.insert(JSContactProperty::Version, Value::Str("1.0".into())); + + let mut default_language = None; + let mut language_counts: OrderedMap, usize> = OrderedMap::default(); + let mut language_count = 0; + let mut language_first_found = None; + let mut langful_names: OrderedMap<&VCardProperty, ()> = OrderedMap::default(); + let mut name_alt_ids: OrderedMap<&str, usize> = OrderedMap::default(); + let mut has_prop_id_lookups = false; + for entry in &vcard.entries { + has_prop_id_lookups |= entry.looks_up_prop_ids(); + if let Some(lang) = entry.language() { + language_counts.upsert(LanguageTag(lang), || 0, |count| *count += 1); + if language_first_found.is_none() { + language_first_found = Some(lang); + } + language_count += 1; + langful_names.insert_if_absent(&entry.name, ()); + } + + match &entry.name { + VCardProperty::Language => { + if let Some(VCardValue::Text(lang)) = entry.values.first() { + default_language = Some(lang.to_ascii_lowercase()); + } + } + VCardProperty::N => { + if let Some(alt_id) = entry.alt_id() { + name_alt_ids.upsert(alt_id, || 0, |count| *count += 1); + } + } + _ => (), + } + } + + let mut name_alt_id = None; + let mut name_alt_id_count = 0; + for (&alt_id, &count) in name_alt_ids.iter() { + if count > name_alt_id_count { + name_alt_id = Some(alt_id); + name_alt_id_count = count; + } + } + let name_alt_id = name_alt_id.map(str::to_string); + + if default_language.is_none() + && language_count > 1 + && let Some(min_count) = language_counts.iter().map(|(_, &count)| count).min() + && let Some((LanguageTag(most_used), max_count)) = + language_counts.into_iter().max_by_key(|&(_, count)| count) + && max_count + > vcard + .entries + .iter() + .filter(|entry| { + entry.language().is_none() && langful_names.contains_key(&entry.name) + }) + .count() + { + let lang = if max_count == min_count + && let Some(first) = language_first_found + { + first + } else { + most_used + } + .to_ascii_lowercase(); + + default_language = Some(lang.clone()); + vcard + .entries + .push(VCardEntry::new(VCardProperty::Language).with_value(lang)); + } + + let sort_key = |entry: &VCardEntry| { + let lang = entry.language(); + let weight = u32::from(lang.is_some() && default_language.as_deref() != lang); + + match &entry.name { + VCardProperty::Birthplace | VCardProperty::Deathplace | VCardProperty::Role => { + weight + 2 + } + VCardProperty::Other(name) if name.eq_ignore_ascii_case("X-ABLabel") => weight + 3, + _ => weight, + } + }; + if !vcard.entries.is_sorted_by_key(sort_key) { + vcard.entries.sort_by_key(sort_key); + } + + Self { + entries, + default_language, + localizations: Default::default(), + prop_ids: PropIds::new(has_prop_id_lookups, &vcard.entries), + vcard_converted_properties: Default::default(), + vcard_properties: Default::default(), + patch_objects: Default::default(), + name_alt_id, + has_fn: false, + has_n: false, + has_n_localization: false, + has_fn_localization: false, + has_gram_gender: false, + include_vcard_converted, + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn map_named_entry( + &mut self, + entry: &mut EntryState, + extract: &[VCardParameterName], + top_property_name: JSContactProperty, + value_property_name: JSContactProperty, + extra_properties: [Option>; 2], + ) { + let value = if !matches!( + entry.entry.name, + VCardProperty::Anniversary | VCardProperty::Bday | VCardProperty::Deathdate + ) { + entry.to_text() + } else { + entry.to_anniversary() + }; + + if let Some(value) = value { + self.map_named_value( + entry, + extract, + top_property_name, + value_property_name, + value, + extra_properties, + ); + } + } + + pub(super) fn map_blob_media( + &mut self, + entry: &mut EntryState, + kind: JSContactKind, + blob_ids: &mut BlobIds<'_, B>, + ) -> bool { + if !matches!( + entry.entry.values.first(), + Some(VCardValue::Binary(data)) if !data.data.is_empty() + ) { + return false; + } + let mut values = mem::take(&mut entry.entry.values).into_iter(); + let Some(VCardValue::Binary(data)) = values.next() else { + return false; + }; + let media_type_param = entry + .entry + .params + .iter() + .find_map(|param| param.as_media_type()); + let media_type = match media_type_param.or(data + .content_type + .as_deref() + .filter(|media_type| !media_type.is_empty())) + { + Some(media_type) => Cow::Owned(media_type.to_string()), + None => Cow::Borrowed(sniff_media_type(&data.data)), + }; + let has_media_type_param = media_type_param.is_some(); + + match blob_ids.blob_id(data.data, Some(media_type.as_ref())) { + Ok(generated) => { + entry.entry.params.retain(|param| { + param.name != VCardParameterName::Mediatype || param.as_media_type().is_some() + }); + self.map_named_value( + entry, + &[ + VCardParameterName::Mediatype, + VCardParameterName::Pref, + VCardParameterName::PropId, + VCardParameterName::Label, + ], + JSContactProperty::Media, + JSContactProperty::BlobId, + Value::Element(JSContactValue::BlobId(generated.blob_id)), + [ + Some(( + Key::Property(JSContactProperty::Kind), + Value::Element(JSContactValue::Kind(kind)), + )), + (!has_media_type_param).then(|| { + ( + Key::Property(JSContactProperty::MediaType), + Value::Str(media_type.into_owned().into()), + ) + }), + ], + ); + true + } + Err(bytes) => { + entry.entry.values = once(VCardValue::Binary(Box::new(Data { + content_type: data.content_type, + data: bytes, + }))) + .chain(values) + .collect(); + false + } + } + } + + fn map_named_value( + &mut self, + entry: &mut EntryState, + extract: &[VCardParameterName], + top_property_name: JSContactProperty, + value_property_name: JSContactProperty, + value: Value<'static, JSContactProperty, JSContactValue>, + extra_properties: [Option>; 2], + ) { + let mut params = ExtractedParams::default(); + params.extract( + &mut entry.entry.params, + extract, + self.default_language.as_deref(), + ); + let prop_id = params.prop_id(); + let alt_id = params.alt_id(); + let sub_property = top_property_name.sub_property(); + + if let Some(language) = params.language() { + if let Some(patch) = prop_id + .as_deref() + .filter(|prop_id| self.has_prop_id(&entry.entry.name, prop_id)) + .or_else(|| { + self.find_prop_id( + &entry.entry.name, + entry.entry.group.as_deref(), + alt_id.as_deref(), + ) + }) + .map(|prop_id| { + if let Some(sub_property) = sub_property.as_ref() { + format!( + "{}/{}/{}/{}", + top_property_name.to_cow().as_ref(), + sub_property.to_cow().as_ref(), + prop_id, + value_property_name.to_cow().as_ref() + ) + } else { + format!( + "{}/{}/{}", + top_property_name.to_cow().as_ref(), + prop_id, + value_property_name.to_cow().as_ref() + ) + } + }) + { + entry.set_converted_to(|| { + String::from_pointer([ + JSContactProperty::Localizations::.to_cow().as_ref(), + language.as_str(), + patch.as_str(), + ]) + }); + + self.localize(language, |localizations| { + let base_path = patch + .rsplit_once('/') + .map_or(patch.as_str(), |(base, _)| base); + + params + .members::(&entry.entry.name) + .for_each(|prop, value| { + localizations + .push((format!("{}/{}", base_path, prop.to_string()), value)); + }); + + localizations.push((patch, value)); + }); + return; + } else { + entry.entry.params.push(VCardParameter::language(language)); + } + } + + let mut entries = self + .entries + .get_mut_object_or_insert(top_property_name.clone()); + if let Some(sub_property) = sub_property.clone() { + let Some(sub_entries) = entries + .insert_or_get_mut(sub_property, Value::Object(Map::from(vec![]))) + .as_object_mut() + else { + return; + }; + entries = sub_entries; + } + + let members = params.members::(&entry.entry.name); + let mut obj = + Vec::with_capacity(1 + extra_properties.iter().flatten().count() + members.len()); + obj.push((Key::Property(value_property_name.clone()), value)); + obj.extend(extra_properties.into_iter().flatten()); + members.append_to(&mut obj); + let prop_id = entries.named_key(prop_id); + + if let Some(sub_property) = sub_property { + entry.set_converted_to(|| { + String::from_pointer([ + top_property_name.to_cow().as_ref(), + sub_property.to_cow().as_ref(), + prop_id.as_str(), + value_property_name.to_cow().as_ref(), + ]) + }); + } else { + entry.set_converted_to(|| { + String::from_pointer([ + top_property_name.to_cow().as_ref(), + prop_id.as_str(), + value_property_name.to_cow().as_ref(), + ]) + }); + } + + self.prop_ids + .track(&entry.entry, top_property_name, alt_id, &prop_id); + entries.insert_unchecked(Key::Owned(prop_id), Value::Object(Map::from(obj))); + } + + #[inline] + pub(super) fn get_mut_object_or_insert( + &mut self, + key: JSContactProperty, + ) -> &mut Map<'static, JSContactProperty, JSContactValue> { + self.entries.get_mut_object_or_insert(key) + } + + #[inline] + pub(super) fn has_property(&self, key: JSContactProperty) -> bool { + self.entries.contains(&key) + } + + pub(super) fn localize( + &mut self, + language: String, + update: impl FnOnce(&mut Localization), + ) { + self.localizations.upsert(language, Vec::new, update); + } + + pub(super) fn add_conversion_props(&mut self, mut entry: EntryState) { + if self.include_vcard_converted { + if let Some(converted_to) = entry.converted_to.take() { + if entry.has_conversion_params() { + let mut value_type = None; + + match self.vcard_converted_properties.get_mut(&converted_to) { + Some(conv_prop) => { + entry.jcal_parameters(&mut conv_prop.params, &mut value_type); + } + None => { + let mut params = VCardParams::default(); + entry.jcal_parameters(&mut params, &mut value_type); + if let Some(value_type) = value_type { + params.set( + VCardParameterName::Value, + vec![Value::Str(value_type.into_string())], + ); + } + if !params.0.is_empty() || entry.map_name { + self.vcard_converted_properties.push( + converted_to, + VCardConvertedProperty { + name: if entry.map_name { + Some(entry.entry.name) + } else { + None + }, + params, + }, + ); + } + } + } + } + } else { + let mut value_type = None; + let mut params = VCardParams::default(); + + entry.jcal_parameters(&mut params, &mut value_type); + + let mut values = entry.entry.values.into_iter(); + let values = match (values.next(), values.len()) { + (Some(value), 0) => value.into_jscontact_value(value_type.as_ref()), + (first, _) => Value::Array( + first + .into_iter() + .chain(values) + .map(|value| value.into_jscontact_value(value_type.as_ref())) + .collect(), + ), + }; + let name = match entry.entry.name { + VCardProperty::Other(mut name) => { + name.make_ascii_lowercase(); + name + } + name => name.as_str().to_ascii_lowercase(), + }; + self.vcard_properties.push(Value::Array(vec![ + Value::Str(name.into()), + Value::Object( + params + .into_jscontact_value() + .unwrap_or(Map::from(Vec::new())), + ), + Value::Str( + value_type + .map(|v| v.into_string()) + .unwrap_or(Cow::Borrowed("unknown")), + ), + values, + ])); + } + } + } + + pub(super) fn find_prop_id( + &self, + prop: &VCardProperty, + group: Option<&str>, + alt_id: Option<&str>, + ) -> Option<&str> { + self.prop_ids + .keys + .iter() + .find(|p| { + p.prop == *prop && p.group.as_deref() == group && p.alt_id.as_deref() == alt_id + }) + .map(|p| p.prop_id.as_str()) + } + + pub(super) fn find_entry_by_group(&self, group: Option<&str>) -> Option<&PropIdKey> { + self.prop_ids + .keys + .iter() + .find(|p| p.group.as_deref() == group) + } + + pub(super) fn has_prop_id(&self, prop: &VCardProperty, prop_id: &str) -> bool { + self.prop_ids + .keys + .iter() + .any(|p| p.prop == *prop && p.prop_id == prop_id) + } + + pub(super) fn into_jscontact(mut self) -> JSContact<'static, I, B> { + if !self.localizations.is_empty() { + self.entries.insert( + JSContactProperty::Localizations, + Value::Object( + self.localizations + .into_iter() + .map(|(lang, locals)| { + ( + Key::Owned(lang), + Value::Object( + locals + .into_iter() + .map(|(key, value)| (Key::Owned(key), value)) + .collect(), + ), + ) + }) + .collect(), + ), + ); + } + + let mut vcard_obj = Map::from(Vec::new()); + if !self.vcard_converted_properties.is_empty() { + let mut converted_properties = + Map::from(Vec::with_capacity(self.vcard_converted_properties.len())); + + for (converted_to, props) in self.vcard_converted_properties { + let mut obj = Map::from(Vec::with_capacity(2)); + if let Some(params) = props.params.into_jscontact_value() { + obj.insert( + Key::Property(JSContactProperty::Parameters), + Value::Object(params), + ); + } + if let Some(name) = props.name { + obj.insert( + Key::Property(JSContactProperty::Name), + Value::Str(name.as_str().to_ascii_lowercase().into()), + ); + } + + converted_properties.insert_unchecked(Key::Owned(converted_to), Value::Object(obj)); + } + + vcard_obj.insert_unchecked( + Key::Property(JSContactProperty::ConvertedProperties), + Value::Object(converted_properties), + ); + } + + if !self.vcard_properties.is_empty() { + vcard_obj.insert_unchecked( + Key::Property(JSContactProperty::Properties), + Value::Array(self.vcard_properties), + ); + } + + if !vcard_obj.is_empty() { + self.entries + .insert(JSContactProperty::VCard, Value::Object(vcard_obj)); + } + + let mut obj = Value::Object(self.entries.into_map()); + if !self.patch_objects.is_empty() { + for (ptr, patch) in self.patch_objects { + patch_card(&mut obj, ptr.as_slice(), patch); + } + } + + JSContact(obj) + } +} + +impl PartialEq for LanguageTag<'_> { + fn eq(&self, other: &Self) -> bool { + self.0.eq_ignore_ascii_case(other.0) + } +} + +impl Eq for LanguageTag<'_> {} + +impl Hash for LanguageTag<'_> { + fn hash(&self, state: &mut H) { + for byte in self.0.bytes() { + state.write_u8(byte.to_ascii_lowercase()); + } + state.write_usize(self.0.len()); + } +} + +impl JSContactProperty +where + I: JSContactId, +{ + pub(super) fn sub_property(&self) -> Option> { + match self { + JSContactProperty::SpeakToAs => Some(JSContactProperty::Pronouns), + _ => None, + } + } +} + +impl VCardValue { + pub(super) fn into_jscontact_value( + self, + value_type: Option<&IanaType>, + ) -> Value<'static, JSContactProperty, JSContactValue> { + match self { + VCardValue::Text(v) => Value::Str(v.into()), + VCardValue::Component(v) => Value::Str(v.join(",").into()), + VCardValue::Integer(v) => Value::Number(v.into()), + VCardValue::Float(v) => Value::Number(v.into()), + VCardValue::Boolean(v) => Value::Bool(v), + VCardValue::PartialDateTime(v) => { + let mut out = String::new(); + let _ = v.format_as_vcard( + &mut out, + value_type + .and_then(|v| v.iana()) + .unwrap_or(if v.has_date() && v.has_time() { + &VCardValueType::Timestamp + } else { + &VCardValueType::DateAndOrTime + }), + ); + Value::Str(out.into()) + } + VCardValue::Binary(v) => Value::Str(v.to_unwrapped_string().into()), + VCardValue::Sex(v) => Value::Str(v.as_str().into()), + VCardValue::GramGender(v) => Value::Str(v.as_str().into()), + VCardValue::Kind(v) => Value::Str(v.as_str().into()), + } + } +} + +impl ExtractedParams { + pub(super) fn extract( + &mut self, + params: &mut Vec, + extract: &[VCardParameterName], + default_language: Option<&str>, + ) { + let p = self; + + params.retain_mut(|param| match ¶m.name { + VCardParameterName::Language => { + let v = param.take_value().into_text().to_ascii_lowercase(); + if p.language.is_none() + && default_language.is_none_or(|lang| lang != v) + && extract.contains(&VCardParameterName::Language) + { + p.language = Some(v); + false + } else { + param.value = VCardParameterValue::Text(v); + true + } + } + VCardParameterName::Pref + if p.pref.is_none() && extract.contains(&VCardParameterName::Pref) => + { + p.pref = param.value.as_integer().and_then(|v| v.into_iana()); + false + } + VCardParameterName::Author + if p.author.is_none() && extract.contains(&VCardParameterName::Author) => + { + p.author = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::AuthorName + if p.author_name.is_none() && extract.contains(&VCardParameterName::AuthorName) => + { + p.author_name = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Mediatype + if p.media_type.is_none() && extract.contains(&VCardParameterName::Mediatype) => + { + p.media_type = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Calscale + if p.calscale.is_none() && extract.contains(&VCardParameterName::Calscale) => + { + p.calscale = param.take_value().into_calscale(); + false + } + VCardParameterName::SortAs + if p.sort_as.is_none() && extract.contains(&VCardParameterName::SortAs) => + { + p.sort_as = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Geo + if p.geo.is_none() && extract.contains(&VCardParameterName::Geo) => + { + p.geo = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Tz + if p.tz.is_none() && extract.contains(&VCardParameterName::Tz) => + { + p.tz = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Index + if p.index.is_none() && extract.contains(&VCardParameterName::Index) => + { + p.index = param.value.as_integer().and_then(|v| v.into_iana()); + false + } + VCardParameterName::Level + if p.level.is_none() && extract.contains(&VCardParameterName::Level) => + { + p.level = param.take_value().into_level(); + false + } + VCardParameterName::Cc + if p.country_code.is_none() && extract.contains(&VCardParameterName::Cc) => + { + p.country_code = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Created + if p.created.is_none() && extract.contains(&VCardParameterName::Created) => + { + p.created = param + .take_value() + .into_timestamp() + .and_then(|v| v.into_iana()); + false + } + VCardParameterName::Label + if p.label.is_none() && extract.contains(&VCardParameterName::Label) => + { + p.label = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Phonetic + if p.phonetic_system.is_none() + && extract.contains(&VCardParameterName::Phonetic) => + { + p.phonetic_system = param.take_value().into_phonetic(); + false + } + VCardParameterName::Script + if p.phonetic_script.is_none() && extract.contains(&VCardParameterName::Script) => + { + p.phonetic_script = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::ServiceType + if p.service_type.is_none() + && extract.contains(&VCardParameterName::ServiceType) => + { + p.service_type = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Username + if p.username.is_none() && extract.contains(&VCardParameterName::Username) => + { + p.username = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::PropId + if p.prop_id.is_none() && extract.contains(&VCardParameterName::PropId) => + { + p.prop_id = Some(param.take_value().into_text().into_owned()); + false + } + VCardParameterName::Altid if p.alt_id.is_none() => { + p.alt_id = param.value.as_text().map(|v| v.to_string()); + true + } + VCardParameterName::Type if extract.contains(&VCardParameterName::Type) => { + if let Some(typ) = param.take_value().into_type() { + p.types.push(typ); + } + false + } + VCardParameterName::Jscomps if extract.contains(&VCardParameterName::Jscomps) => { + if let Some(jscomps) = param.take_value().into_jscomps() { + p.jscomps = jscomps; + } + false + } + _ => true, + }); + } +} + +pub(super) trait NamedKey { + fn named_key(&self, key: Option) -> String; +} + +impl NamedKey + for Map<'static, JSContactProperty, JSContactValue> +{ + fn named_key(&self, key: Option) -> String { + let number = self.len() as u64 + 1; + let key = key.unwrap_or_else(|| { + let mut key = AsciiString::with_capacity(NUMBERED_KEY_CAPACITY); + let _ = key.push_byte(b'k').and_then(|()| key.push_u64(number)); + key.into_string() + }); + if self.contains_key(&Key::Borrowed(key.as_str())) { + let mut key = AsciiString::from(key); + let _ = key.push_byte(b'-').and_then(|()| key.push_u64(number)); + key.into_string() + } else { + key + } + } +} + +impl PropIds { + fn new(is_enabled: bool, entries: &[VCardEntry]) -> Self { + Self { + keys: if is_enabled { + Vec::with_capacity( + entries + .iter() + .filter(|entry| entry.name.is_named_property()) + .count(), + ) + } else { + Vec::new() + }, + is_enabled, + } + } + + pub(super) fn track( + &mut self, + entry: &VCardEntry, + prop_js: JSContactProperty, + alt_id: Option, + prop_id: &str, + ) { + if self.is_enabled { + self.keys.push(PropIdKey { + prop_id: prop_id.to_string(), + prop_js, + prop: entry.name.clone(), + group: entry.group.clone(), + alt_id, + }); + } + } +} + +impl VCardEntry { + fn looks_up_prop_ids(&self) -> bool { + match &self.name { + VCardProperty::Birthplace + | VCardProperty::Deathplace + | VCardProperty::Role + | VCardProperty::Tz + | VCardProperty::Geo => true, + VCardProperty::Other(name) + if self.group.is_some() && name.eq_ignore_ascii_case("X-ABLabel") => + { + true + } + _ => self + .params + .iter() + .any(|param| param.name == VCardParameterName::Language), + } + } +} + +impl VCardProperty { + fn is_named_property(&self) -> bool { + matches!( + self, + VCardProperty::Source + | VCardProperty::OrgDirectory + | VCardProperty::Anniversary + | VCardProperty::Bday + | VCardProperty::Deathdate + | VCardProperty::Birthplace + | VCardProperty::Deathplace + | VCardProperty::Pronouns + | VCardProperty::Nickname + | VCardProperty::Photo + | VCardProperty::Logo + | VCardProperty::Sound + | VCardProperty::Adr + | VCardProperty::Email + | VCardProperty::Impp + | VCardProperty::Lang + | VCardProperty::Socialprofile + | VCardProperty::Tel + | VCardProperty::ContactUri + | VCardProperty::Title + | VCardProperty::Role + | VCardProperty::Hobby + | VCardProperty::Interest + | VCardProperty::Expertise + | VCardProperty::Org + | VCardProperty::Note + | VCardProperty::Url + | VCardProperty::Key + | VCardProperty::Caladruri + | VCardProperty::Fburl + | VCardProperty::Caluri + ) + } +} + +fn patch_card( + value: &mut Value<'static, JSContactProperty, JSContactValue>, + pointer: &[JsonPointerItem>], + patch: Value<'static, JSContactProperty, JSContactValue>, +) -> bool { + match (pointer.split_first(), value) { + (Some((JsonPointerItem::Root, rest)), value) => patch_card(value, rest, patch), + (Some((JsonPointerItem::Number(index), rest)), Value::Array(items)) => { + match (items.get_mut(*index as usize), rest.is_empty()) { + (Some(item), false) => patch_card(item, rest, patch), + (Some(item), true) if !patch.is_null() => { + *item = patch; + true + } + _ => false, + } + } + (Some((JsonPointerItem::Key(_) | JsonPointerItem::Number(_), rest)), value) + if rest + .iter() + .all(|item| !matches!(item, JsonPointerItem::Number(_))) => + { + value.patch_jptr(pointer.iter().peekable(), patch) + } + (Some((JsonPointerItem::Key(key), rest)), Value::Object(obj)) => obj + .get_mut(key) + .is_some_and(|item| patch_card(item, rest, patch)), + (Some((JsonPointerItem::Number(index), rest)), Value::Object(obj)) => { + let index = index.to_string(); + obj.iter_mut() + .find(|(key, _)| key.to_string() == index) + .is_some_and(|(_, item)| patch_card(item, rest, patch)) + } + _ => false, + } +} + diff --git a/crates/calcard/src/jscontact/mod.rs b/crates/calcard/src/jscontact/mod.rs new file mode 100644 index 00000000..83ba7584 --- /dev/null +++ b/crates/calcard/src/jscontact/mod.rs @@ -0,0 +1,440 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::{CalendarScale, elements::Elements}; +use jmap_tools::{JsonPointer, Value}; +use serde::Serialize; +use std::{ + borrow::Cow, + fmt::{Debug, Display}, + hash::Hash, + str::FromStr, +}; + +pub mod export; +pub mod import; +pub mod parser; +pub mod types; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +#[repr(transparent)] +pub struct JSContact<'x, I, B>(pub Value<'x, JSContactProperty, JSContactValue>) +where + I: JSContactId, + B: JSContactId; + +pub trait JSContactId: + FromStr + Sized + Serialize + Display + Clone + Eq + Hash + Ord + Debug + Default +{ +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactValue { + Id(I), + BlobId(B), + IdReference(String), + Timestamp(i64), + Type(JSContactType), + GrammaticalGender(JSContactGrammaticalGender), + Kind(JSContactKind), + Level(JSContactLevel), + Relation(JSContactRelation), + PhoneticSystem(JSContactPhoneticSystem), + CalendarScale(CalendarScale), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactProperty { + Type, + Address, + AddressBookIds, + Addresses, + Anniversaries, + Author, + BlobId, + Calendars, + CalendarScale, + Components, + Contexts, + ConvertedProperties, + Coordinates, + CountryCode, + Created, + CryptoKeys, + Date, + Day, + DefaultSeparator, + Directories, + Emails, + Extra, + Features, + Full, + GrammaticalGender, + Id, + IsOrdered, + Keywords, + Kind, + Label, + Language, + Level, + Links, + ListAs, + Localizations, + Media, + MediaType, + Members, + Metadata, + Month, + Name, + Nicknames, + Note, + Notes, + Number, + OnlineServices, + OrganizationId, + Organizations, + Parameters, + PersonalInfo, + Phones, + Phonetic, + PhoneticScript, + PhoneticSystem, + Place, + Pref, + PreferredLanguages, + PrivateMetadata, + ProdId, + Pronouns, + Properties, + RelatedTo, + Relation, + SchedulingAddresses, + Service, + SortAs, + SpeakToAs, + TimeZone, + Titles, + Uid, + Units, + Updated, + Uri, + User, + Utc, + VCard, + Value, + Version, + Year, + IdValue(I), + IdReference(String), + Context(Context), + Feature(Feature), + SortAsKind(JSContactKind), + Pointer(JsonPointer>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactType { + Address, + AddressComponent, + Anniversary, + Author, + Boolean, + Calendar, + Card, + CryptoKey, + Directory, + EmailAddress, + Id, + Int, + JCardProp, + LanguagePref, + Link, + Media, + Name, + NameComponent, + Nickname, + Note, + Number, + OnlineService, + Organization, + OrgUnit, + PartialDate, + PatchObject, + PersonalInfo, + Phone, + Pronouns, + Relation, + Resource, + SchedulingAddress, + SpeakToAs, + String, + Timestamp, + Title, + UnsignedInt, + UTCDateTime, +} + +impl FromStr for JSContactType { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactType, + "Address" => JSContactType::Address, + "AddressComponent" => JSContactType::AddressComponent, + "Anniversary" => JSContactType::Anniversary, + "Author" => JSContactType::Author, + "Boolean" => JSContactType::Boolean, + "Calendar" => JSContactType::Calendar, + "Card" => JSContactType::Card, + "CryptoKey" => JSContactType::CryptoKey, + "Directory" => JSContactType::Directory, + "EmailAddress" => JSContactType::EmailAddress, + "Id" => JSContactType::Id, + "Int" => JSContactType::Int, + "JCardProp" => JSContactType::JCardProp, + "LanguagePref" => JSContactType::LanguagePref, + "Link" => JSContactType::Link, + "Media" => JSContactType::Media, + "Name" => JSContactType::Name, + "NameComponent" => JSContactType::NameComponent, + "Nickname" => JSContactType::Nickname, + "Note" => JSContactType::Note, + "Number" => JSContactType::Number, + "OnlineService" => JSContactType::OnlineService, + "Organization" => JSContactType::Organization, + "OrgUnit" => JSContactType::OrgUnit, + "PartialDate" => JSContactType::PartialDate, + "PatchObject" => JSContactType::PatchObject, + "PersonalInfo" => JSContactType::PersonalInfo, + "Phone" => JSContactType::Phone, + "Pronouns" => JSContactType::Pronouns, + "Relation" => JSContactType::Relation, + "Resource" => JSContactType::Resource, + "SchedulingAddress" => JSContactType::SchedulingAddress, + "SpeakToAs" => JSContactType::SpeakToAs, + "String" => JSContactType::String, + "Timestamp" => JSContactType::Timestamp, + "Title" => JSContactType::Title, + "UnsignedInt" => JSContactType::UnsignedInt, + "UTCDateTime" => JSContactType::UTCDateTime, + ) + .copied() + .ok_or(()) + } +} + +impl JSContactType { + pub fn as_str(&self) -> &'static str { + match self { + JSContactType::Address => "Address", + JSContactType::AddressComponent => "AddressComponent", + JSContactType::Anniversary => "Anniversary", + JSContactType::Author => "Author", + JSContactType::Boolean => "Boolean", + JSContactType::Calendar => "Calendar", + JSContactType::Card => "Card", + JSContactType::CryptoKey => "CryptoKey", + JSContactType::Directory => "Directory", + JSContactType::EmailAddress => "EmailAddress", + JSContactType::Id => "Id", + JSContactType::Int => "Int", + JSContactType::JCardProp => "JCardProp", + JSContactType::LanguagePref => "LanguagePref", + JSContactType::Link => "Link", + JSContactType::Media => "Media", + JSContactType::Name => "Name", + JSContactType::NameComponent => "NameComponent", + JSContactType::Nickname => "Nickname", + JSContactType::Note => "Note", + JSContactType::Number => "Number", + JSContactType::OnlineService => "OnlineService", + JSContactType::Organization => "Organization", + JSContactType::OrgUnit => "OrgUnit", + JSContactType::PartialDate => "PartialDate", + JSContactType::PatchObject => "PatchObject", + JSContactType::PersonalInfo => "PersonalInfo", + JSContactType::Phone => "Phone", + JSContactType::Pronouns => "Pronouns", + JSContactType::Relation => "Relation", + JSContactType::Resource => "Resource", + JSContactType::SchedulingAddress => "SchedulingAddress", + JSContactType::SpeakToAs => "SpeakToAs", + JSContactType::String => "String", + JSContactType::Timestamp => "Timestamp", + JSContactType::Title => "Title", + JSContactType::UnsignedInt => "UnsignedInt", + JSContactType::UTCDateTime => "UTCDateTime", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Context { + Billing, + Delivery, + Private, + Work, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Feature { + Fax, + MainNumber, + Mobile, + Pager, + Text, + TextPhone, + Video, + Voice, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactGrammaticalGender { + Animate, + Common, + Feminine, + Inanimate, + Masculine, + Neuter, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactKind { + Apartment, + Block, + Building, + Country, + Direction, + District, + Floor, + Landmark, + Locality, + Name, + Number, + Postcode, + PostOfficeBox, + Region, + Room, + Separator, + Subdistrict, + Birth, + Death, + Wedding, + Calendar, + FreeBusy, + Application, + Device, + Group, + Individual, + Location, + Org, + Directory, + Entry, + Contact, + Logo, + Photo, + Sound, + Credential, + Generation, + Given, + Given2, + Surname, + Surname2, + Title, + Expertise, + Hobby, + Interest, + Role, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactLevel { + High, + Low, + Medium, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactRelation { + Acquaintance, + Agent, + Child, + Colleague, + Contact, + CoResident, + CoWorker, + Crush, + Date, + Emergency, + Friend, + Kin, + Me, + Met, + Muse, + Neighbor, + Parent, + Sibling, + Spouse, + Sweetheart, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JSContactPhoneticSystem { + Ipa, + Jyut, + Piny, + Script, +} + +#[cfg(all(test, stalwart_tests))] +impl std::fmt::Display for JSContact<'_, I, B> +where + I: JSContactId, + B: JSContactId, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = serde_json::to_string_pretty(&self.0).map_err(|_| std::fmt::Error)?; + write!(f, "{}", s) + } +} + +impl JSContactId for T where + T: FromStr + Sized + Serialize + Display + Clone + Eq + Hash + Ord + Debug + Default +{ +} + +impl From for JSContactProperty { + fn from(id: I) -> Self { + JSContactProperty::IdValue(id) + } +} + +impl From for JSContactValue { + fn from(id: I) -> Self { + JSContactValue::Id(id) + } +} + +impl JSContact<'_, I, B> { + pub fn blob_ids(&self) -> impl Iterator { + Elements::new(&self.0).filter_map(|element| match element { + JSContactValue::BlobId(blob_id) => Some(blob_id), + _ => None, + }) + } +} + +impl<'x, I: JSContactId, B: JSContactId> Default for JSContact<'x, I, B> { + fn default() -> Self { + Self(Value::Object(jmap_tools::Map::new())) + } +} + +impl Display for JSContactProperty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.to_string().as_ref()) + } +} + diff --git a/crates/calcard/src/jscontact/parser.rs b/crates/calcard/src/jscontact/parser.rs new file mode 100644 index 00000000..d0387393 --- /dev/null +++ b/crates/calcard/src/jscontact/parser.rs @@ -0,0 +1,399 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + CalendarScale, IanaParse, IdReference, + jsprop::text::{DisplayEq, IdReferenceText, PointerText, PointerTextExt}, + }, + jscalendar::{ + JSCalendarDateTime, + parser::{BLOB_ID_SUFFIX, KEY_TEXT, StackText, VALUE_TEXT}, + }, + jscontact::{ + Context, Feature, JSContact, JSContactGrammaticalGender, JSContactId, JSContactKind, + JSContactLevel, JSContactPhoneticSystem, JSContactProperty, JSContactRelation, + JSContactType, JSContactValue, + }, +}; +use jiff::Timestamp; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property as _, Value}; +use serde::Serializer; +use std::{ + borrow::Cow, + fmt::{self, Write}, + mem::discriminant, + str::FromStr, +}; + +impl<'x, I, B> JSContact<'x, I, B> +where + I: JSContactId, + B: JSContactId, +{ + pub fn parse(json: &'x str) -> Result { + Value::parse_json(json).map(JSContact) + } + + pub fn to_string_pretty(&self) -> String { + serde_json::to_string_pretty(&self.0).unwrap_or_default() + } +} + +impl Element for JSContactValue +where + I: JSContactId, + B: JSContactId, +{ + type Property = JSContactProperty; + + fn try_parse

{ + pub name: P, + pub test: Presence>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CardFilter { + pub test: FilterTest, + pub prop_filters: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CardPropFilter { + pub name: VCardPropertyWithGroup, + pub test: Presence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CardPropMatch { + pub test: FilterTest, + pub text_matches: Vec, + pub param_filters: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct TextMatch { pub match_type: MatchType, pub value: String, pub collation: Collation, pub negate: bool, + #[cfg_attr(all(test, stalwart_tests), serde(skip))] + pub(crate) needle: String, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Acl { pub aces: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct AclPrincipalPropSet { pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PrincipalMatch { pub principal_properties: PrincipalMatchProperties, pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum PrincipalMatchProperties { Properties(Vec), Self_, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PrincipalPropertySearch { pub property_search: Vec, pub properties: Vec, pub apply_to_principal_collection_set: bool, + pub test: FilterTest, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PropertySearch { - pub property: DavProperty, + pub properties: Vec, pub match_: String, } impl PropertyUpdate { pub fn has_changes(&self) -> bool { - !self.set.is_empty() || !self.remove.is_empty() + !self.ops.is_empty() + } +} + +impl PropertyUpdateOp { + pub fn property(&self) -> &DavProperty { + match self { + PropertyUpdateOp::Set(value) => &value.property, + PropertyUpdateOp::Remove(property) => property, + } } } @@ -242,49 +405,6 @@ impl FreeBusyQuery { } } -pub trait DavDeadProperty { - fn to_dav_values(&self, output: &mut Vec); -} - -impl DavDeadProperty for ArchivedDeadProperty { - fn to_dav_values(&self, output: &mut Vec) { - let mut depth: u32 = 0; - let mut tags = Vec::new(); - let mut tag_start = None; - - for tag in self.0.iter() { - match tag { - ArchivedDeadPropertyTag::ElementStart(start) => { - if depth == 0 { - tag_start = Some(DeadElementTag::from(start)); - } else { - tags.push(tag.into()); - } - - depth += 1; - } - ArchivedDeadPropertyTag::ElementEnd => { - depth = depth.saturating_sub(1); - - if depth > 0 { - tags.push(tag.into()); - } else if let Some(tag_start) = tag_start.take() { - output.push(DavPropertyValue::new( - DavProperty::DeadProperty(tag_start), - DavValue::DeadProperty(DeadProperty(std::mem::take(&mut tags))), - )); - } - } - ArchivedDeadPropertyTag::Text(_) => { - if tag_start.is_some() { - tags.push(tag.into()); - } - } - } - } - } -} - impl Condition<'_> { pub fn is_none_match(&self) -> bool { match self { diff --git a/crates/dav-proto/src/schema/response.rs b/crates/dav-proto/src/schema/response.rs index cd507a28..50414a42 100644 --- a/crates/dav-proto/src/schema/response.rs +++ b/crates/dav-proto/src/schema/response.rs @@ -7,11 +7,7 @@ use super::{ Namespaces, property::{DavProperty, Privilege}, - request::{DavPropertyValue, Filter}, -}; -use calcard::{ - icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, - vcard::{VCardParameterName, VCardProperty}, + request::DavPropertyValue, }; use hyper::StatusCode; use std::{borrow::Cow, fmt::Display}; @@ -24,7 +20,10 @@ pub struct MultiStatus { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Response { pub href: Href, pub typ: ResponseType, @@ -34,7 +33,10 @@ pub struct Response { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum ResponseType { PropStat(List), Status { href: List, status: Status }, @@ -45,12 +47,18 @@ pub enum ResponseType { pub struct Status(pub StatusCode); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct Location(pub Href); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct ResponseDescription(pub String); @@ -58,12 +66,18 @@ pub struct ResponseDescription(pub String); pub struct SyncToken(pub String); #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct Href(pub String); #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct List(pub Vec); @@ -74,7 +88,10 @@ pub struct MkColResponse { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PropStat { pub prop: Prop, pub status: Status, @@ -83,7 +100,10 @@ pub struct PropStat { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct Prop(pub List); @@ -105,7 +125,10 @@ pub struct ScheduleResponseItem { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct SupportedPrivilege { pub privilege: Privilege, pub abstract_: bool, @@ -114,7 +137,10 @@ pub struct SupportedPrivilege { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Ace { pub principal: Principal, pub invert: bool, @@ -124,14 +150,20 @@ pub struct Ace { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum GrantDeny { Grant(List), Deny(List), } #[derive(Debug, Default, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum Principal { Href(Href), Response(Response), @@ -144,7 +176,10 @@ pub enum Principal { } #[derive(Debug, Default, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct AclRestrictions { pub grant_only: bool, pub no_invert: bool, @@ -153,7 +188,10 @@ pub struct AclRestrictions { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum RequiredPrincipal { All, Authenticated, @@ -164,14 +202,20 @@ pub enum RequiredPrincipal { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PrincipalSearchPropertySet { pub namespaces: Namespaces, pub properties: List, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PrincipalSearchProperty { pub name: DavProperty, pub description: String, @@ -183,7 +227,10 @@ pub struct ErrorResponse { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum Condition { Base(BaseCondition), Cal(CalCondition), @@ -191,7 +238,10 @@ pub enum Condition { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum BaseCondition { NoConflictingLock(List), LockTokenSubmitted(List), @@ -221,14 +271,20 @@ pub enum BaseCondition { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Resource { pub href: Href, pub privilege: Privilege, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum CalCondition { CalendarCollectionLocationOk, ValidCalendarData, @@ -238,9 +294,7 @@ pub enum CalCondition { NoUidConflict(Href), InitializeCalendarCollection, SupportedCalendarData, - SupportedFilter( - Vec, ICalendarProperty, ICalendarParameterName>>, - ), + SupportedFilter, SupportedCollation(String), SupportedCalendarComponent, MinDateTime, @@ -259,11 +313,14 @@ pub enum CalCondition { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum CardCondition { SupportedAddressData, SupportedAddressDataConversion, - SupportedFilter(Vec>), + SupportedFilter, SupportedCollation(String), ValidAddressData, NoUidConflict(Href), @@ -350,7 +407,7 @@ impl CalCondition { CalCondition::NoUidConflict(_) => "NoUidConflict", CalCondition::InitializeCalendarCollection => "InitializeCalendarCollection", CalCondition::SupportedCalendarData => "SupportedCalendarData", - CalCondition::SupportedFilter(_) => "SupportedFilter", + CalCondition::SupportedFilter => "SupportedFilter", CalCondition::SupportedCollation(_) => "SupportedCollation", CalCondition::MinDateTime => "MinDateTime", CalCondition::MaxDateTime => "MaxDateTime", @@ -375,7 +432,7 @@ impl CardCondition { match self { CardCondition::SupportedAddressData => "SupportedAddressData", CardCondition::SupportedAddressDataConversion => "SupportedAddressDataConversion", - CardCondition::SupportedFilter(_) => "SupportedFilter", + CardCondition::SupportedFilter => "SupportedFilter", CardCondition::SupportedCollation(_) => "SupportedCollation", CardCondition::ValidAddressData => "ValidAddressData", CardCondition::NoUidConflict(_) => "NoUidConflict", @@ -395,36 +452,3 @@ impl Condition { } } -#[cfg(test)] -mod serde_impl { - use super::Status; - use hyper::StatusCode; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - impl Serialize for Status { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - // Serialize the status code as a u16 - serializer.serialize_u16(self.0.as_u16()) - } - } - - impl<'de> Deserialize<'de> for Status { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // Deserialize as u16 - let status_value = u16::deserialize(deserializer)?; - - // Convert u16 to StatusCode - let status_code = StatusCode::try_from(status_value).map_err(|_| { - serde::de::Error::custom(format!("Invalid status code: {}", status_value)) - })?; - - Ok(Status(status_code)) - } - } -} diff --git a/crates/dav/Cargo.toml b/crates/dav/Cargo.toml index 527f3aaf..9b075ea8 100644 --- a/crates/dav/Cargo.toml +++ b/crates/dav/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dav" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -9,25 +9,25 @@ common = { path = "../common" } store = { path = "../store" } utils = { path = "../utils" } groupware = { path = "../groupware" } +nlp = { path = "../nlp" } directory = { path = "../directory" } registry = { path = "../registry" } http_proto = { path = "../http-proto" } types = { path = "../types" } trc = { path = "../trc" } -calcard = { version = "0.3", features = ["rkyv"] } -hashify = { version = "0.2" } -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } +calcard = { path = "../calcard", features = ["rkyv"] } +hashify = { version = "0.3" } +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } percent-encoding = "2.3.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = "0.10.0" -chrono = "0.4.45" +tokio = { version = "1.54", features = ["time"] } [dev-dependencies] [features] -test_mode = [] dev_mode = [] -enterprise = [] +enterprise = ["dav-proto/enterprise", "common/enterprise", "store/enterprise", "utils/enterprise", "groupware/enterprise", "directory/enterprise", "registry/enterprise", "http_proto/enterprise", "trc/enterprise"] [lints] workspace = true diff --git a/crates/dav/src/calendar/copy_move.rs b/crates/dav/src/calendar/copy_move.rs index 46e9e19c..3f761722 100644 --- a/crates/dav/src/calendar/copy_move.rs +++ b/crates/dav/src/calendar/copy_move.rs @@ -4,35 +4,55 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::assert_is_unique_uid; +use super::{CalendarComponentSupport, assert_event_privacy_access, assert_is_unique_uid}; use crate::{ DavError, DavMethod, common::{ + ContainerOperation, assert_parent_limit, + dead::{ + ContainerRequest, ContainerWrites, DeadContainers, check_growth, container_kinds, + copy_growth, read_container, + }, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; -use calcard::common::timezone::Tz; -use common::{DavName, Server, auth::AccessToken}; +use common::{ + DavName, GroupwareResources, Server, + auth::AccessToken, + storage::{batch::MAX_BATCH_DOCUMENTS, index::PresenceFlags, metadata::StoredEntry}, +}; use dav_proto::{Depth, RequestHeaders}; use groupware::{ DestroyArchive, cache::GroupwareCache, - calendar::{Calendar, CalendarEvent, CalendarPreferences, Timezone}, + calendar::{ + CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarEventContent, CalendarPreferences, + EVENT_HAS_ALARMS, Timezone, + alerts::{CalendarSettings, DefaultAlertsResolver}, + identity::{CalendarAddresses, ParticipantIdentityAddresses}, + privacy::{EventPrivacy, EventViewer}, + rights::{EventAcl, StoredEventOwnership}, + schedule::{AlarmTasks, EventAlarmScheduler, EventAlarmUsers}, + storage::{DirectChangeNotification, NotificationQuota}, + }, }; use http_proto::HttpResponse; use hyper::StatusCode; -use store::write::{BatchBuilder, now}; +use registry::schema::enums::StorageQuota; +use store::write::{BatchBuilder, PendingId, now}; use store::{ ValueKey, - write::{AlignedBytes, Archive, ValueClass}, + roaring::RoaringBitmap, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, - field::PrincipalField, + field::{CalendarEventField, PrincipalField}, + metadata::MetadataKinds, }; pub(crate) trait CalendarCopyMoveRequestHandler: Sync + Send { @@ -58,7 +78,7 @@ impl CalendarCopyMoveRequestHandler for Server { .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::Calendar, @@ -71,7 +91,7 @@ impl CalendarCopyMoveRequestHandler for Server { let from_resource = from_resources .by_path(from_resource_name) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; - #[cfg(not(any(feature = "dev_mode", feature = "test_mode")))] + #[cfg(not(any(feature = "dev_mode", stalwart_tests)))] if is_move && from_resource.is_container() && self @@ -88,19 +108,29 @@ impl CalendarCopyMoveRequestHandler for Server { } // Validate ACL - if !access_token.is_member(from_account_id) + let is_from_owner = access_token.is_member(from_account_id); + if !from_resource.is_container() { + assert_event_privacy_access( + from_resource.resource.event_flags(), + EventViewer::new(is_from_owner), + )?; + } + if !is_from_owner && !from_resources.has_access_to_container( access_token, if from_resource.is_container() { from_resource.document_id() } else { - from_resource.parent_id().unwrap() + from_resource + .parent_id() + .ok_or(DavError::Code(StatusCode::NOT_FOUND))? }, Acl::ReadItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } + let container_operation = ContainerOperation::from_move(is_move); // Validate destination let destination = self @@ -118,10 +148,18 @@ impl CalendarCopyMoveRequestHandler for Server { let to_account_id = destination .account_id .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; + let is_to_owner = access_token.is_member(to_account_id); + if !from_resource.is_container() + && !is_to_owner + && !EventPrivacy::from_flags(from_resource.resource.event_flags().unwrap_or_default()) + .is_public() + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } let to_resources = if to_account_id == from_account_id { from_resources.clone() } else { - self.fetch_dav_resources( + self.fetch_groupware_resources( access_token.account_id(), to_account_id, SyncCollection::Calendar, @@ -135,6 +173,11 @@ impl CalendarCopyMoveRequestHandler for Server { .resource .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resource = to_resources.by_path(destination_resource_name); + let visible_to_resource = to_resource.filter(|resource| { + is_to_owner + || EventPrivacy::from_flags(resource.resource.event_flags().unwrap_or_default()) + != EventPrivacy::Secret + }); self.validate_headers( access_token, headers, @@ -152,7 +195,7 @@ impl CalendarCopyMoveRequestHandler for Server { }, ResourceState { account_id: to_account_id, - collection: to_resource + collection: visible_to_resource .map(|r| { if r.is_container() { Collection::Calendar @@ -161,7 +204,11 @@ impl CalendarCopyMoveRequestHandler for Server { } }) .unwrap_or(Collection::Calendar), - document_id: Some(to_resource.map(|r| r.document_id()).unwrap_or(u32::MAX)), + document_id: Some( + visible_to_resource + .map(|r| r.document_id()) + .unwrap_or(u32::MAX), + ), path: destination_resource_name, ..Default::default() }, @@ -188,29 +235,23 @@ impl CalendarCopyMoveRequestHandler for Server { match (from_resource.is_container(), to_resource.is_container()) { (true, true) => { - let from_children_ids = from_resources - .subtree(from_resource_name) - .filter(|r| !r.is_container()) - .map(|r| r.document_id()) - .collect::>(); - let to_document_ids = to_resources + let from_children_ids = container_operation.event_ids( + &from_resources, + from_resource_name, + EventViewer::new(is_from_owner), + )?; + let to_children = to_resources .subtree(destination_resource_name) .filter(|r| !r.is_container()) - .map(|r| r.document_id()) + .map(|r| (r.document_id(), r.resource.metadata_kinds())) .collect::>(); // Validate ACLs - if !access_token.is_member(to_account_id) - || (!access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( - access_token, - from_resource.document_id(), - if is_move { - Acl::RemoveItems - } else { - Acl::ReadItems - }, - )) + if !is_to_owner + || (!is_from_owner + && !from_resources + .container_acl(access_token, from_resource.document_id()) + .contains_all(container_operation.required_acls())) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -219,13 +260,15 @@ impl CalendarCopyMoveRequestHandler for Server { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), from_children_ids, from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), - to_document_ids, + to_children, new_name, is_move, ) @@ -237,8 +280,9 @@ impl CalendarCopyMoveRequestHandler for Server { let to_calendar_id = to_resource.parent_id().unwrap(); // Validate ACL - if (!access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( + let mut own_events = OwnEventAccess::new(self, access_token); + let may_take = access_token.is_member(from_account_id) + || from_resources.has_access_to_container( access_token, from_calendar_id, if is_move { @@ -246,21 +290,50 @@ impl CalendarCopyMoveRequestHandler for Server { } else { Acl::ReadItems }, - )) - || (!access_token.is_member(to_account_id) - && !to_resources.has_access_to_container( - access_token, - to_calendar_id, - Acl::RemoveItems, - )) - { + ) + || (is_move + && own_events + .may_manage( + &from_resources, + from_calendar_id, + from_account_id, + from_resource.document_id(), + ) + .await?); + let may_overwrite = may_take + && (access_token.is_member(to_account_id) + || to_resources + .container_acl(access_token, to_calendar_id) + .contains_all([Acl::AddItems, Acl::RemoveItems].into_iter()) + || (own_events + .may_manage( + &to_resources, + to_calendar_id, + to_account_id, + to_resource.document_id(), + ) + .await? + && own_events + .may_manage( + &to_resources, + to_calendar_id, + from_account_id, + from_resource.document_id(), + ) + .await?)); + if !may_overwrite { return Err(DavError::Code(StatusCode::FORBIDDEN)); } + assert_event_privacy_access( + to_resource.resource.event_flags(), + EventViewer::new(is_to_owner), + )?; if is_move { move_event( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), from_calendar_id, @@ -276,6 +349,7 @@ impl CalendarCopyMoveRequestHandler for Server { copy_event( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), to_account_id, @@ -301,8 +375,9 @@ impl CalendarCopyMoveRequestHandler for Server { // Validate ACL let from_calendar_id = from_resource.parent_id().unwrap(); let to_calendar_id = parent_resource.document_id(); - if (!access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( + let mut own_events = OwnEventAccess::new(self, access_token); + let may_take = access_token.is_member(from_account_id) + || from_resources.has_access_to_container( access_token, from_calendar_id, if is_move { @@ -310,14 +385,32 @@ impl CalendarCopyMoveRequestHandler for Server { } else { Acl::ReadItems }, - )) - || (!access_token.is_member(to_account_id) - && !to_resources.has_access_to_container( + ) + || (is_move + && own_events + .may_manage( + &from_resources, + from_calendar_id, + from_account_id, + from_resource.document_id(), + ) + .await?); + let may_add = may_take + && (access_token.is_member(to_account_id) + || to_resources.has_access_to_container( access_token, to_calendar_id, Acl::AddItems, - )) - { + ) + || own_events + .may_manage( + &to_resources, + to_calendar_id, + from_account_id, + from_resource.document_id(), + ) + .await?); + if !may_add { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -329,6 +422,7 @@ impl CalendarCopyMoveRequestHandler for Server { move_event( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), from_calendar_id, @@ -356,6 +450,7 @@ impl CalendarCopyMoveRequestHandler for Server { copy_event( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), to_account_id, @@ -372,36 +467,32 @@ impl CalendarCopyMoveRequestHandler for Server { } // Shared users cannot create containers - if !access_token.is_member(to_account_id) { + if !is_to_owner { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate ACLs - if !access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( - access_token, - from_resource.document_id(), - if is_move { - Acl::RemoveItems - } else { - Acl::ReadItems - }, - ) + if !is_from_owner + && !from_resources + .container_acl(access_token, from_resource.document_id()) + .contains_all(container_operation.required_acls()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Copy/move container - let from_children_ids = from_resources - .subtree(from_resource_name) - .filter(|r| !r.is_container()) - .map(|r| r.document_id()) - .collect::>(); + let from_children_ids = container_operation.event_ids( + &from_resources, + from_resource_name, + EventViewer::new(is_from_owner), + )?; if is_move { if from_account_id != to_account_id { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { @@ -432,6 +523,8 @@ impl CalendarCopyMoveRequestHandler for Server { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { @@ -455,10 +548,56 @@ impl CalendarCopyMoveRequestHandler for Server { } } +struct OwnEventAccess<'x> { + server: &'x Server, + access_token: &'x AccessToken, + identities: Option, +} + +impl<'x> OwnEventAccess<'x> { + fn new(server: &'x Server, access_token: &'x AccessToken) -> Self { + OwnEventAccess { + server, + access_token, + identities: None, + } + } + + async fn may_manage( + &mut self, + resources: &GroupwareResources, + calendar_id: u32, + account_id: u32, + document_id: u32, + ) -> crate::Result { + if !EventAcl::for_calendar(resources, self.access_token, calendar_id).may_manage_own_items() + { + return Ok(false); + } + let identities = match self.identities.as_ref() { + Some(identities) => identities, + None => self.identities.get_or_insert( + self.server + .account_identity_addresses(self.access_token.account_id()) + .await + .caused_by(trc::location!())?, + ), + }; + + Ok(self + .server + .stored_event_ownership(account_id, document_id, identities) + .await + .caused_by(trc::location!())? + .may_write_own()) + } +} + #[allow(clippy::too_many_arguments)] async fn copy_event( server: &Server, access_token: &AccessToken, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, to_account_id: u32, @@ -469,7 +608,7 @@ async fn copy_event( // Fetch event let event_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_document_id, @@ -484,23 +623,24 @@ async fn copy_event( // Validate UID assert_is_unique_uid( - server, - server - .fetch_dav_resources( - access_token.account_id(), - to_account_id, - SyncCollection::Calendar, - ) - .await - .caused_by(trc::location!())? - .as_ref(), + to_resources, + access_token, to_account_id, to_calendar_id, - event.inner.data.event.uids().next(), - ) - .await?; + Some(event.inner.uid.as_str()).filter(|uid| !uid.is_empty()), + )?; + let checked_content = server + .assert_stored_event_supported( + from_account_id, + from_document_id, + to_account_id, + to_calendar_id, + ) + .await?; - let changed_by = access_token.account_tenant_ids(); + let changed_by = access_token.changed_by(); + let event_flags = event.inner.flags.to_native(); + let mut quota = NotificationQuota::default(); if from_account_id == to_account_id { let mut new_event = event .deserialize::() @@ -509,17 +649,84 @@ async fn copy_event( name: new_name.to_string(), parent_id: to_calendar_id, }); + server + .notify_calendar_addition( + access_token, + to_account_id, + PendingId::Assigned(from_document_id), + to_calendar_id, + event_flags, + (from_account_id, from_document_id), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + assert_parent_limit( + event.inner.names.len(), + new_event.names.len(), + server.core.groupware.max_calendars_per_event, + )?; + server + .reschedule_event_alarms( + from_account_id, + from_document_id, + event.inner, + None, + &new_event + .names + .iter() + .map(DavName::parent_id) + .collect::>(), + &mut AlarmTasks::None, + &mut DefaultAlertsResolver::default(), + &mut batch, + ) + .await + .caused_by(trc::location!())?; new_event - .update( + .update_meta( changed_by, event, from_account_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; } else { - let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); + let content_ = match checked_content { + Some(content_) => content_, + None => server + .store() + .get_value::>(ValueKey::property( + from_account_id, + Collection::CalendarEvent, + from_document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, + }; + let mut new_content = content_ + .deserialize::() + .caused_by(trc::location!())?; + new_content + .preferences + .retain(|preferences| preferences.account_id == access_token.account_id()); + let next_email_alarms = server + .next_event_alarms( + to_account_id, + &EventAlarmUsers::new(to_account_id, &new_content)? + .with_event_flags(event.inner.flags.to_native()), + &new_content.data, + &[to_calendar_id], + now() as i64, + &mut DefaultAlertsResolver::default(), + ) + .await + .caused_by(trc::location!())?; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; @@ -527,17 +734,62 @@ async fn copy_event( name: new_name.to_string(), parent_id: to_calendar_id, }]; - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) + let container = read_container( + server, + from_account_id, + Collection::CalendarEvent, + from_document_id, + event.inner.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + container + .as_ref() + .map_or(0, |container| copy_growth(container, None)), + ) + .await + .caused_by(trc::location!())?; + new_event.set_metadata_kinds(container_kinds(container.as_ref())); + if to_document_id.is_none() { + server.assert_object_quota( + &*server.account(to_account_id).await?, + StorageQuota::MaxCalendarEvents, + 1, + || to_resources.resources.count(false), + )?; + } + let to_document_id = batch.reserve_document_id(to_account_id, Collection::CalendarEvent); + server + .notify_calendar_addition( + access_token, + to_account_id, + to_document_id.into(), + to_calendar_id, + event_flags, + (from_account_id, from_document_id), + &mut quota, + &mut batch, + ) .await .caused_by(trc::location!())?; + if let Some(container) = &container { + let mut copies = ContainerWrites::new(to_account_id, Collection::CalendarEvent); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } new_event .insert( + new_content, changed_by, to_account_id, to_document_id, - next_email_alarm, + None, + next_email_alarms, &mut batch, ) .caused_by(trc::location!())?; @@ -547,7 +799,7 @@ async fn copy_event( // Overwrite event on destination let event_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::CalendarEvent, to_document_id, @@ -563,16 +815,31 @@ async fn copy_event( .await .caused_by(trc::location!())?; + server + .notify_calendar_removal( + access_token, + to_account_id, + to_document_id, + to_calendar_id, + event.inner.flags.to_native(), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; DestroyArchive(event) .delete( + server, &account_info, to_account_id, to_document_id, to_calendar_id, None, + None, false, &mut batch, ) + .await .caused_by(trc::location!())?; } @@ -585,7 +852,6 @@ async fn copy_event( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); response } @@ -594,6 +860,7 @@ async fn copy_event( async fn move_event( server: &Server, access_token: &AccessToken, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, from_calendar_id: u32, @@ -607,7 +874,7 @@ async fn move_event( // Fetch event let event_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_document_id, @@ -632,42 +899,55 @@ async fn move_event( || to_document_id.is_none() { assert_is_unique_uid( - server, - server - .fetch_dav_resources( - access_token.account_id(), - to_account_id, - SyncCollection::Calendar, - ) - .await - .caused_by(trc::location!())? - .as_ref(), + to_resources, + access_token, to_account_id, to_calendar_id, - event.inner.data.event.uids().next(), - ) - .await?; + Some(event.inner.uid.as_str()).filter(|uid| !uid.is_empty()), + )?; } + let checked_content = if from_account_id != to_account_id || from_calendar_id != to_calendar_id + { + server + .assert_stored_event_supported( + from_account_id, + from_document_id, + to_account_id, + to_calendar_id, + ) + .await? + } else { + None + }; let account_info = server .account_info(access_token.account_id()) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); + let event_flags = event.inner.flags.to_native(); + let mut quota = NotificationQuota::default(); + if from_calendar_id != to_calendar_id || from_account_id != to_account_id { + server + .notify_calendar_removal( + access_token, + from_account_id, + from_document_id, + from_calendar_id, + event_flags, + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } if from_account_id == to_account_id { - let mut name_idx = None; - for (idx, name) in event.inner.names.iter().enumerate() { - if name.parent_id == from_calendar_id { - name_idx = Some(idx); - break; - } - } - - let name_idx = if let Some(name_idx) = name_idx { - name_idx - } else { - return Err(DavError::Code(StatusCode::NOT_FOUND)); - }; + let name_idx = event + .inner + .names + .iter() + .position(|name| name.parent_id == from_calendar_id) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let mut new_event = event .deserialize::() @@ -677,18 +957,82 @@ async fn move_event( name: new_name.to_string(), parent_id: to_calendar_id, }); + if from_calendar_id != to_calendar_id { + server + .notify_calendar_addition( + access_token, + to_account_id, + PendingId::Assigned(from_document_id), + to_calendar_id, + event_flags, + (from_account_id, from_document_id), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + server + .reschedule_event_alarms( + from_account_id, + from_document_id, + event.inner, + None, + &new_event + .names + .iter() + .map(DavName::parent_id) + .collect::>(), + &mut AlarmTasks::None, + &mut DefaultAlertsResolver::default(), + &mut batch, + ) + .await + .caused_by(trc::location!())?; new_event - .update( - access_token.account_tenant_ids(), + .update_meta( + access_token.changed_by(), event.clone(), from_account_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); } else { - let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); + let content_ = match checked_content { + Some(content_) => content_, + None => server + .store() + .get_value::>(ValueKey::property( + from_account_id, + Collection::CalendarEvent, + from_document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, + }; + let mut new_content = content_ + .deserialize::() + .caused_by(trc::location!())?; + new_content + .preferences + .retain(|preferences| preferences.account_id == access_token.account_id()); + let next_email_alarms = server + .next_event_alarms( + to_account_id, + &EventAlarmUsers::new(to_account_id, &new_content)? + .with_event_flags(event.inner.flags.to_native()), + &new_content.data, + &[to_calendar_id], + now() as i64, + &mut DefaultAlertsResolver::default(), + ) + .await + .caused_by(trc::location!())?; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; @@ -696,30 +1040,93 @@ async fn move_event( name: new_name.to_string(), parent_id: to_calendar_id, }]; + let container = read_container( + server, + from_account_id, + Collection::CalendarEvent, + from_document_id, + event.inner.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + container + .as_ref() + .map_or(0, |container| copy_growth(container, None)), + ) + .await + .caused_by(trc::location!())?; + new_event.set_metadata_kinds(container_kinds(container.as_ref())); + let source_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::CalendarEvent, + container + .as_ref() + .map(|container| StoredEntry::from_container(from_document_id, container)), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(event) - .delete( + .delete_with_cleanup( + server, &account_info, from_account_id, from_document_id, from_calendar_id, + Some(content_), + &mut AlarmTasks::None, from_resource_path.into(), - false, + &source_cleanup, &mut batch, ) - .caused_by(trc::location!())?; - - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; + + let to_account = server.account(to_account_id).await?; + if to_document_id.is_none() + && server + .object_quota_limit(&to_account, StorageQuota::MaxCalendarEvents) + .is_some() + { + server.assert_object_quota(&to_account, StorageQuota::MaxCalendarEvents, 1, || { + to_resources.resources.count(false) + })?; + } + + let to_document_id = batch.reserve_document_id(to_account_id, Collection::CalendarEvent); + server + .notify_calendar_addition( + access_token, + to_account_id, + to_document_id.into(), + to_calendar_id, + event_flags, + (from_account_id, from_document_id), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + if let Some(container) = &container { + let mut copies = ContainerWrites::new(to_account_id, Collection::CalendarEvent); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } new_event .insert( - access_token.account_tenant_ids(), + new_content, + access_token.changed_by(), to_account_id, to_document_id, - next_email_alarm, + None, + next_email_alarms, &mut batch, ) .caused_by(trc::location!())?; @@ -729,7 +1136,7 @@ async fn move_event( // Overwrite event on destination let event_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::CalendarEvent, to_document_id, @@ -741,16 +1148,31 @@ async fn move_event( .to_unarchived::() .caused_by(trc::location!())?; + server + .notify_calendar_removal( + access_token, + to_account_id, + to_document_id, + to_calendar_id, + event.inner.flags.to_native(), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; DestroyArchive(event) .delete( + server, &account_info, to_account_id, to_document_id, to_calendar_id, None, + None, false, &mut batch, ) + .await .caused_by(trc::location!())?; } @@ -763,7 +1185,6 @@ async fn move_event( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); response } @@ -781,7 +1202,7 @@ async fn rename_event( // Fetch event let event_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, @@ -806,11 +1227,12 @@ async fn rename_event( let mut batch = BatchBuilder::new(); new_event - .update( - access_token.account_tenant_ids(), + .update_meta( + access_token.changed_by(), event, account_id, document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -819,7 +1241,6 @@ async fn rename_event( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } @@ -828,20 +1249,22 @@ async fn rename_event( async fn copy_container( server: &Server, access_token: &AccessToken, + from_resources: &GroupwareResources, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, from_children_ids: Vec, from_resource_path: String, to_account_id: u32, to_document_id: Option, - to_children_ids: Vec, + to_children: Vec<(u32, MetadataKinds)>, new_name: &str, remove_source: bool, ) -> crate::Result { // Fetch calendar let calendar_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::Calendar, from_document_id, @@ -855,42 +1278,104 @@ async fn copy_container( let mut calendar = old_calendar .deserialize::() .caused_by(trc::location!())?; + let calendar_container = read_container( + server, + from_account_id, + Collection::Calendar, + from_document_id, + old_calendar.inner.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + calendar.set_metadata_kinds(container_kinds(calendar_container.as_ref())); + let event_containers = if from_account_id != to_account_id { + let mut request = ContainerRequest::default(); + for &document_id in &from_children_ids { + if from_resources + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + { + request.insert(from_account_id, Collection::CalendarEvent, document_id); + } + } + request.load(server).await.caused_by(trc::location!())? + } else { + DeadContainers::default() + }; + let mut event_copies = ContainerWrites::new(to_account_id, Collection::CalendarEvent); + let source_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::CalendarEvent, + event_containers + .stored_entries(from_account_id, Collection::CalendarEvent) + .filter(|_| remove_source), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + calendar_container + .as_ref() + .map_or(0, |container| copy_growth(container, None)) + + event_containers.stored_len(), + ) + .await + .caused_by(trc::location!())?; + + // Validate quota + let to_account = server.account(to_account_id).await?; + let has_container_quota = to_document_id.is_none() + && server + .object_quota_limit(&to_account, StorageQuota::MaxCalendars) + .is_some(); + let has_item_quota = from_account_id != to_account_id + && server + .object_quota_limit(&to_account, StorageQuota::MaxCalendarEvents) + .is_some(); + let to_alarm_ids = if to_document_id.is_some() { + to_resources.event_ids_with_flags(EVENT_HAS_ALARMS) + } else { + RoaringBitmap::new() + }; + let to_resources = (has_container_quota || has_item_quota).then_some(to_resources); + if has_container_quota && let Some(to_resources) = &to_resources { + server.assert_object_quota(&to_account, StorageQuota::MaxCalendars, 1, || { + to_resources.resources.count(true) + })?; + } + let deleted_events = match &to_resources { + Some(to_resources) if has_item_quota => to_children + .iter() + .filter(|(document_id, _)| { + to_resources + .resources + .find(*document_id, false) + .is_some_and(|item| item.child_names().len() <= 1) + }) + .count(), + _ => 0, + }; // Prepare write batch let mut batch = BatchBuilder::new(); - if remove_source { - DestroyArchive(old_calendar) - .delete( - access_token.account_tenant_ids(), - from_account_id, - from_document_id, - from_resource_path.into(), - &mut batch, - ) - .caused_by(trc::location!())?; - - // Reset default calendar id - let default_calendar_id = server - .store() - .get_value::(ValueKey { - account_id: from_account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), - }) - .await - .caused_by(trc::location!())?; - if default_calendar_id.is_some_and(|id| id == from_document_id) { - batch - .with_account_id(from_account_id) - .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultCalendarId); - } - } - - let preference = calendar.preferences.into_iter().next().unwrap(); + let personal_id = access_token.personal_id(from_account_id, Collection::Calendar); + let preference = calendar + .preferences + .iter() + .position(|preferences| preferences.account_id == personal_id) + .or_else(|| { + calendar + .preferences + .iter() + .position(|preferences| preferences.account_id == from_account_id) + }) + .map(|idx| calendar.preferences.swap_remove(idx)) + .unwrap_or_default(); + let mut default_alerts = DefaultAlertsResolver::default(); calendar.name = new_name.to_string(); calendar.acls.clear(); calendar.preferences = vec![CalendarPreferences { @@ -900,7 +1385,7 @@ async fn copy_container( default_alerts: preference.default_alerts, sort_order: 0, color: preference.color, - flags: 0, + flags: CALENDAR_SUBSCRIBED, time_zone: Timezone::Default, }]; @@ -913,7 +1398,7 @@ async fn copy_container( // Overwrite destination let calendar_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::Calendar, to_document_id, @@ -924,14 +1409,33 @@ async fn copy_container( let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; + let cleanup = server + .preload_container_cleanup( + access_token.changed_by(), + to_account_id, + Collection::CalendarEvent, + &to_children + .iter() + .filter(|(_, kinds)| !kinds.is_empty()) + .map(|(document_id, _)| *document_id) + .collect(), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(calendar) .delete_with_events( server, + access_token, &account_info, to_account_id, to_document_id, - to_children_ids, + to_children + .into_iter() + .map(|(document_id, _)| document_id) + .collect(), + &to_alarm_ids, + &cleanup, None, false, &mut batch, @@ -940,17 +1444,27 @@ async fn copy_container( .caused_by(trc::location!())?; } - to_document_id + PendingId::Assigned(to_document_id) } else { - server - .store() - .assign_document_ids(to_account_id, Collection::Calendar, 1) - .await - .caused_by(trc::location!())? + PendingId::Slot(batch.reserve_document_id(to_account_id, Collection::Calendar)) }; + let parent_id = to_document_id; + let alarm_calendar_id = parent_id.assigned().unwrap_or(u32::MAX); + default_alerts.set_calendar_settings( + to_account_id, + alarm_calendar_id, + CalendarSettings::from(&calendar), + ); + if let Some(container) = &calendar_container { + let mut copies = ContainerWrites::new(to_account_id, Collection::Calendar); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } calendar .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), to_account_id, to_document_id, &mut batch, @@ -959,109 +1473,233 @@ async fn copy_container( // Copy children let mut required_space = 0; - for from_child_document_id in from_children_ids { - if let Some(event_) = server - .store() - .get_value::>(ValueKey::archive( + let mut created_events = 0; + let mut from_children_ids = from_children_ids; + from_children_ids.sort_unstable(); + from_children_ids.dedup(); + let is_cross_account = from_account_id != to_account_id; + let alarm_ids = from_resources.event_ids_with_flags(EVENT_HAS_ALARMS); + let mut content_ids = Vec::new(); + for chunk in from_children_ids.chunks(MAX_BATCH_DOCUMENTS) { + content_ids.clear(); + content_ids.extend( + chunk + .iter() + .copied() + .filter(|document_id| is_cross_account || alarm_ids.contains(*document_id)), + ); + let mut archives = server + .archive_chunk( from_account_id, Collection::CalendarEvent, - from_child_document_id, - )) - .await? - { - let event = event_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_name = None; - - for name in event.inner.names.iter() { - if name.parent_id == to_document_id { - continue; - } else if name.parent_id == from_document_id { - new_name = Some(name.name.to_string()); - } - } - let new_name = if let Some(new_name) = new_name { - DavName { - name: new_name, - parent_id: to_document_id, - } - } else { - continue; - }; - let event = event_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_event = event - .deserialize::() - .caused_by(trc::location!())?; - - if from_account_id == to_account_id { - if remove_source { - new_event - .names - .retain(|name| name.parent_id != from_document_id); - } - - new_event.names.push(new_name); - new_event - .update( - access_token.account_tenant_ids(), - event, - from_account_id, - from_child_document_id, - &mut batch, - ) + chunk, + Some((CalendarEventField::Content.field(), &content_ids)), + ) + .await + .caused_by(trc::location!())?; + let mut tasks = if !is_cross_account || remove_source { + AlarmTasks::for_chunk(from_account_id, chunk, &alarm_ids, &archives)? + } else { + AlarmTasks::None + }; + for &from_child_document_id in chunk { + if let Some(event_) = archives.take_archive(from_child_document_id) { + let event = event_ + .to_unarchived::() .caused_by(trc::location!())?; - } else { - let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); - if remove_source { - DestroyArchive(event) - .delete( - &account_info, + let mut new_name = None; + + for name in event.inner.names.iter() { + if parent_id + .assigned() + .is_some_and(|parent| name.parent_id == parent) + { + continue; + } else if name.parent_id == from_document_id { + new_name = Some(name.name.to_string()); + } + } + let new_name = if let Some(new_name) = new_name { + DavName { + name: new_name, + parent_id: parent_id.assigned().unwrap_or_default(), + } + } else { + continue; + }; + let mut new_event = event + .deserialize::() + .caused_by(trc::location!())?; + + if from_account_id == to_account_id { + if remove_source { + new_event + .names + .retain(|name| name.parent_id != from_document_id); + } + + let calendar_ids = new_event + .names + .iter() + .map(DavName::parent_id) + .chain([alarm_calendar_id]) + .collect::>(); + new_event.names.push(new_name); + assert_parent_limit( + event.inner.names.len(), + new_event.names.len(), + server.core.groupware.max_calendars_per_event, + )?; + server + .reschedule_event_alarms( from_account_id, from_child_document_id, - from_document_id, - None, - false, + event.inner, + archives.take_value(from_child_document_id), + &calendar_ids, + &mut tasks, + &mut default_alerts, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + new_event + .update_meta( + access_token.changed_by(), + event, + from_account_id, + from_child_document_id, + parent_id.slot(), + &mut batch, + ) + .caused_by(trc::location!())?; + } else { + let content_ = archives + .take_value(from_child_document_id) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let mut new_content = content_ + .deserialize::() + .caused_by(trc::location!())?; + new_content + .preferences + .retain(|preferences| preferences.account_id == access_token.account_id()); + let next_email_alarms = server + .next_event_alarms( + to_account_id, + &EventAlarmUsers::new(to_account_id, &new_content)? + .with_event_flags(event.inner.flags.to_native()), + &new_content.data, + &[alarm_calendar_id], + now() as i64, + &mut default_alerts, + ) + .await + .caused_by(trc::location!())?; + + if remove_source { + DestroyArchive(event) + .delete_with_cleanup( + server, + &account_info, + from_account_id, + from_child_document_id, + from_document_id, + Some(content_), + &mut tasks, + None, + &source_cleanup, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + let to_document_id = + batch.reserve_document_id(to_account_id, Collection::CalendarEvent); + let container = event_containers.get( + from_account_id, + Collection::CalendarEvent, + from_child_document_id, + ); + new_event.set_metadata_kinds(container_kinds(container)); + if let Some(container) = container { + event_copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } + new_event.names = vec![new_name]; + required_space += new_event.size as u64; + created_events += 1; + new_event + .insert( + new_content, + access_token.changed_by(), + to_account_id, + to_document_id, + parent_id.slot(), + next_email_alarms, &mut batch, ) .caused_by(trc::location!())?; } - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) - .await - .caused_by(trc::location!())?; - new_event.names = vec![new_name]; - required_space += new_event.size as u64; - new_event - .insert( - access_token.account_tenant_ids(), - to_account_id, - to_document_id, - next_email_alarm, - &mut batch, - ) - .caused_by(trc::location!())?; } } } + if has_item_quota + && created_events > deleted_events + && let Some(to_resources) = &to_resources + { + server.assert_object_quota( + &to_account, + StorageQuota::MaxCalendarEvents, + created_events - deleted_events, + || to_resources.resources.count(false), + )?; + } + if from_account_id != to_account_id && required_space > 0 { server - .has_available_quota( - server.account(to_account_id).await?.as_ref(), - required_space, - ) + .has_available_quota(&to_account, required_space) .await?; } + if remove_source { + let calendar_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::Calendar, + calendar_container + .as_ref() + .map(|container| StoredEntry::from_container(from_document_id, container)), + ) + .await + .caused_by(trc::location!())?; + DestroyArchive(old_calendar) + .delete_with_cleanup( + server, + access_token.changed_by(), + from_account_id, + from_document_id, + from_resource_path.into(), + &calendar_cleanup, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + + batch + .with_account_id(from_account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear_if_equals(PrincipalField::DefaultCalendarId, from_document_id); + } + server .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); if !is_overwrite { Ok(HttpResponse::new(StatusCode::CREATED)) @@ -1082,7 +1720,7 @@ async fn rename_container( // Fetch calendar let calendar_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, @@ -1101,7 +1739,7 @@ async fn rename_container( let mut batch = BatchBuilder::new(); new_calendar .update( - access_token.account_tenant_ids(), + access_token.changed_by(), calendar, account_id, document_id, @@ -1113,7 +1751,6 @@ async fn rename_container( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } diff --git a/crates/dav/src/calendar/delete.rs b/crates/dav/src/calendar/delete.rs index 520c3da2..9b7d860a 100644 --- a/crates/dav/src/calendar/delete.rs +++ b/crates/dav/src/calendar/delete.rs @@ -4,10 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::assert_event_privacy_access; use crate::{ DavError, DavMethod, common::{ - ETag, + ContainerOperation, ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, @@ -17,21 +18,28 @@ use dav_proto::RequestHeaders; use groupware::{ DestroyArchive, cache::GroupwareCache, - calendar::{Calendar, CalendarEvent}, + calendar::{ + Calendar, CalendarEvent, CalendarEventContent, EVENT_HAS_ALARMS, + identity::ParticipantIdentityAddresses, + notification::{NotificationAudience, may_have_viewers}, + privacy::EventViewer, + rights::EventAcl, + storage::{DirectChange, DirectChangeNotification, NotificationQuota}, + }, }; use http_proto::HttpResponse; use hyper::StatusCode; use registry::schema::enums::Permission; -use store::write::{BatchBuilder, ValueClass}; +use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, - field::PrincipalField, + field::{CalendarEventField, PrincipalField}, }; pub(crate) trait CalendarDeleteRequestHandler: Sync + Send { @@ -59,7 +67,7 @@ impl CalendarDeleteRequestHandler for Server { .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, @@ -72,9 +80,7 @@ impl CalendarDeleteRequestHandler for Server { .by_path(delete_path) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let document_id = delete_resource.document_id(); - let account_info = self - .scheduling_account_info(access_token.account_id(), account_id) - .await?; + let account_info = self.account_info(account_id).await?; let send_itip = self.core.groupware.itip_enabled && !headers.no_schedule_reply && !account_info.addresses().is_empty() @@ -84,7 +90,7 @@ impl CalendarDeleteRequestHandler for Server { let mut batch = BatchBuilder::new(); if delete_resource.is_container() { // Deleting the default calendar is not allowed - #[cfg(not(any(feature = "dev_mode", feature = "test_mode")))] + #[cfg(not(any(feature = "dev_mode", stalwart_tests)))] if self .core .groupware @@ -100,7 +106,7 @@ impl CalendarDeleteRequestHandler for Server { let calendar_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, @@ -114,15 +120,21 @@ impl CalendarDeleteRequestHandler for Server { .caused_by(trc::location!())?; // Validate ACL - if !access_token.is_member(account_id) + let is_owner = access_token.is_member(account_id); + if !is_owner && !calendar .inner .acls .effective_acl(access_token) - .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) + .contains_all(ContainerOperation::Remove.required_acls()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } + let children_ids = ContainerOperation::Remove.event_ids( + &resources, + delete_path, + EventViewer::new(is_owner), + )?; // Validate headers self.validate_headers( @@ -142,17 +154,33 @@ impl CalendarDeleteRequestHandler for Server { .await?; // Delete calendar and events + let cleanup = self + .preload_container_cleanup( + account_info.changed_by(), + account_id, + Collection::CalendarEvent, + &children_ids + .iter() + .copied() + .filter(|&document_id| { + resources + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(calendar) .delete_with_events( self, + access_token, &account_info, account_id, document_id, - resources - .subtree(delete_path) - .filter(|r| !r.is_container()) - .map(|r| r.document_id()) - .collect::>(), + children_ids, + &resources.event_ids_with_flags(EVENT_HAS_ALARMS), + &cleanup, resources.format_resource(delete_resource).into(), send_itip, &mut batch, @@ -161,35 +189,33 @@ impl CalendarDeleteRequestHandler for Server { .caused_by(trc::location!())?; // Reset default calendar id - let default_calendar_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), - }) - .await - .caused_by(trc::location!())?; - if default_calendar_id.is_some_and(|id| id == document_id) { - batch - .with_account_id(account_id) - .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultCalendarId); - } + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear_if_equals(PrincipalField::DefaultCalendarId, document_id); } else { // Validate ACL - let calendar_id = delete_resource.parent_id().unwrap(); - if !access_token.is_member(account_id) - && !resources.has_access_to_container(access_token, calendar_id, Acl::RemoveItems) + let calendar_id = delete_resource + .parent_id() + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let is_owner = access_token.is_member(account_id); + assert_event_privacy_access( + delete_resource.resource.event_flags(), + EventViewer::new(is_owner), + )?; + let may_remove_items = is_owner + || resources.has_access_to_container(access_token, calendar_id, Acl::RemoveItems); + if !may_remove_items + && !EventAcl::for_calendar(&resources, access_token, calendar_id) + .may_manage_own_items() { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let event_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, @@ -197,6 +223,48 @@ impl CalendarDeleteRequestHandler for Server { .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let event = event_ + .to_unarchived::() + .caused_by(trc::location!())?; + let event_flags = event.inner.flags.to_native(); + let may_notify = may_have_viewers( + access_token, + account_id, + &resources, + &[calendar_id], + NotificationAudience::of_event(event_flags), + ); + let content_ = if !may_remove_items || may_notify { + self.store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + } else { + None + }; + let previous_event = content_ + .as_ref() + .map(|content| content.deserialize::()) + .transpose() + .caused_by(trc::location!())? + .map(|content| content.data.event); + if !may_remove_items { + let identities = self + .account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())?; + if !previous_event + .as_ref() + .is_some_and(|ical| identities.event_ownership(ical).may_write_own()) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + } // Validate headers self.validate_headers( @@ -206,7 +274,7 @@ impl CalendarDeleteRequestHandler for Server { account_id, collection: Collection::CalendarEvent, document_id: document_id.into(), - etag: event_.etag().into(), + etag: format!("\"{}\"", event.inner.etag.to_native()).into(), path: delete_path, ..Default::default() }], @@ -216,9 +284,6 @@ impl CalendarDeleteRequestHandler for Server { .await?; // Validate schedule tag - let event = event_ - .to_unarchived::() - .caused_by(trc::location!())?; if headers.if_schedule_tag.is_some() && event.inner.schedule_tag.as_ref().map(|t| t.to_native()) != headers.if_schedule_tag @@ -227,21 +292,40 @@ impl CalendarDeleteRequestHandler for Server { } // Delete event + if let Some(previous) = previous_event { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Destroyed { + event_id: document_id, + previous, + calendar_ids: vec![calendar_id], + event_flags, + }, + Some(&resources), + &mut NotificationQuota::default(), + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } DestroyArchive(event) .delete( + self, &account_info, account_id, document_id, calendar_id, + content_, resources.format_resource(delete_resource).into(), send_itip, &mut batch, ) + .await .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } diff --git a/crates/dav/src/calendar/filter.rs b/crates/dav/src/calendar/filter.rs new file mode 100644 index 00000000..728ff347 --- /dev/null +++ b/crates/dav/src/calendar/filter.rs @@ -0,0 +1,375 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + query::{CalendarQueryHandler, EventTimeRange}, + view::{AlarmView, CalendarView, ChildIdView, ComponentView, EntryView, ParameterView}, +}; +use crate::common::{ + EventContent, + search::{Candidates, IndexedText, QueryScope, TextIndex}, +}; +use calcard::{ + common::timezone::Tz, + icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, +}; +use common::storage::dav::CachedUid; +use dav_proto::schema::request::{ + CalendarPropFilter, CalendarPropMatch, CompFilter, CompFilterMatch, ParamFilter, Presence, + PropValueMatch, +}; +use groupware::calendar::{EVENT_HAS_ALARMS, EVENT_HAS_UNBOUNDED_TODO}; +use store::search::{CalendarSearchField, SearchField}; +use types::TimeRange; + +pub(crate) trait CalendarFilterPlan { + fn candidates(&self, scope: &QueryScope<'_>, index: TextIndex<'_>) -> Candidates; + fn time_ranges(&self) -> FilterTimeRanges; +} + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct FilterTimeRanges { + pub range: Option, + pub has_alarms: bool, +} + +const ALARM_EXPANSION_SLACK: i64 = 86_400; + +trait AlarmReach { + fn widen_for_alarms(&self, range: TimeRange) -> TimeRange; +} + +trait PropFilterPlan { + fn candidates( + &self, + component: &ICalendarComponentType, + scope: &QueryScope<'_>, + index: TextIndex<'_>, + ) -> Candidates; +} + +trait CalendarIndexField { + fn search_field(&self) -> Option<(SearchField, IndexedText)>; +} + +trait EntryFilter: EntryView { + fn matches_prop(&self, test: &CalendarPropMatch, default_tz: Tz) -> bool; + fn matches_param(&self, filter: &ParamFilter) -> bool; +} + +impl CalendarQueryHandler { + pub fn matches_content(&self, content: &EventContent<'_>, filter: &CompFilter) -> bool { + match content.view() { + Some(view) => self.matches(view, filter), + None => self.matches(content.stored(), filter), + } + } + + pub fn matches(&self, event: &V, filter: &CompFilter) -> bool { + match (&filter.test, event.components().first()) { + (Presence::IsDefined(test), Some(root)) if root.component_type() == &filter.name => { + self.component_matches(event, 0, root, test) + } + _ => false, + } + } + + fn component_matches( + &self, + event: &V, + comp_id: u32, + component: &V::Component, + test: &CompFilterMatch, + ) -> bool { + test.time_range + .as_ref() + .is_none_or(|range| self.component_in_range(event, comp_id, component, range)) + && test.prop_filters.iter().all(|filter| { + let mut entries = component + .entries() + .iter() + .filter(|entry| entry.is_named(&filter.name)) + .peekable(); + match &filter.test { + Presence::IsNotDefined => entries.peek().is_none(), + Presence::IsDefined(test) if entries.peek().is_some() => { + entries.any(|entry| entry.matches_prop(test, self.default_tz)) + } + Presence::IsDefined(test) => { + test.param_filters.is_empty() + && matches!(&test.value, Some(PropValueMatch::TimeRange(range)) + if component + .effective_end(&filter.name, self.default_tz) + .is_some_and(|timestamp| { + range.start <= timestamp && range.end > timestamp + })) + } + } + }) + && test + .comp_filters + .iter() + .all(|filter| self.child_filter_matches(event, component, filter)) + } + + fn child_filter_matches( + &self, + event: &V, + parent: &V::Component, + filter: &CompFilter, + ) -> bool { + let components = event.components(); + let mut children = parent.child_ids().iter().filter_map(|child_id| { + let child_id = child_id.child_id(); + components + .get(child_id as usize) + .filter(|child| child.component_type() == &filter.name) + .map(|child| (child_id, child)) + }); + + match &filter.test { + Presence::IsNotDefined => children.next().is_none(), + Presence::IsDefined(test) => children + .any(|(child_id, child)| self.component_matches(event, child_id, child, test)), + } + } + + fn component_in_range( + &self, + event: &V, + comp_id: u32, + component: &V::Component, + range: &TimeRange, + ) -> bool { + if component.is_type(ICalendarComponentType::VAlarm) { + let repetition = component.alarm_repetition(); + event + .alarms() + .iter() + .filter(|alarm| alarm.id() == comp_id) + .any( + |alarm| match alarm.delta().fixed_timestamp(self.default_tz) { + Some(timestamp) => repetition.triggers_in(timestamp, range), + None => self + .expansions_of(alarm.parent_id()) + .iter() + .any(|expansion| { + alarm.timestamp(expansion, self.default_tz).is_some_and( + |timestamp| repetition.triggers_in(timestamp, range), + ) + }), + }, + ) + } else if component.is_type(ICalendarComponentType::VCalendar) { + !self.undated_todos.is_empty() + || self.expanded_times.iter().any(|expansion| { + range.is_in_range(expansion.flags.condition(), expansion.start, expansion.end) + }) + } else { + self.undated_todos.contains(&comp_id) + || (self.dateless_events_match + && component.is_type(ICalendarComponentType::VEvent) + && !component + .entries() + .iter() + .any(|entry| entry.is_named(&ICalendarProperty::Dtstart))) + || self.expansions_of(comp_id).iter().any(|expansion| { + range.is_in_range(expansion.flags.condition(), expansion.start, expansion.end) + }) + } + } +} + +impl EntryFilter for E { + fn matches_prop(&self, test: &CalendarPropMatch, default_tz: Tz) -> bool { + test.value.as_ref().is_none_or(|value| match value { + PropValueMatch::TimeRange(range) => self + .date_time_timestamp(default_tz) + .is_some_and(|timestamp| range.start <= timestamp && range.end > timestamp), + PropValueMatch::Text(text_match) => text_match.is_match_any(self.text_values()), + }) && test + .param_filters + .iter() + .all(|filter| self.matches_param(filter)) + } + + fn matches_param(&self, filter: &ParamFilter) -> bool { + let mut params = self + .params() + .iter() + .filter(|param| param.is_named(&filter.name)) + .peekable(); + match &filter.test { + Presence::IsNotDefined => params.peek().is_none(), + Presence::IsDefined(None) => params.peek().is_some(), + Presence::IsDefined(Some(text_match)) => { + params.peek().is_some() + && text_match.is_match_any(params.filter_map(|param| param.text())) + } + } + } +} + +impl CalendarFilterPlan for CompFilter { + fn candidates(&self, scope: &QueryScope<'_>, index: TextIndex<'_>) -> Candidates { + let Presence::IsDefined(test) = &self.test else { + return Candidates::All; + }; + + let time_range = test.time_range.as_ref().map(|range| match self.name { + ICalendarComponentType::VEvent + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy => { + scope.matching(|resource| resource.resource.is_in_time_range(range)) + } + ICalendarComponentType::VTodo | ICalendarComponentType::VCalendar => { + scope.matching(|resource| { + resource.resource.is_in_time_range(range) + || resource + .event_flags() + .is_some_and(|flags| flags & EVENT_HAS_UNBOUNDED_TODO != 0) + }) + } + ICalendarComponentType::VAlarm => scope.matching(|resource| { + resource + .event_flags() + .is_some_and(|flags| flags & EVENT_HAS_ALARMS != 0) + }), + _ => Candidates::All, + }); + + Candidates::and( + time_range + .into_iter() + .chain( + test.prop_filters + .iter() + .map(|filter| filter.candidates(&self.name, scope, index)), + ) + .chain( + test.comp_filters + .iter() + .map(|filter| filter.candidates(scope, index)), + ), + ) + } + + fn time_ranges(&self) -> FilterTimeRanges { + let Presence::IsDefined(test) = &self.test else { + return FilterTimeRanges::default(); + }; + test.comp_filters.iter().map(CompFilter::time_ranges).fold( + FilterTimeRanges { + range: test.time_range, + has_alarms: test.time_range.is_some() + && self.name == ICalendarComponentType::VAlarm, + }, + FilterTimeRanges::union, + ) + } +} + +impl FilterTimeRanges { + pub fn union(self, other: FilterTimeRanges) -> Self { + FilterTimeRanges { + range: match (self.range, other.range) { + (Some(range), Some(other)) => Some(range.union(other)), + (range, other) => range.or(other), + }, + has_alarms: self.has_alarms || other.has_alarms, + } + } + + pub fn expansion_range(&self, event: &V) -> Option { + self.range.map(|range| { + if self.has_alarms { + event.widen_for_alarms(range) + } else { + range + } + }) + } +} + +impl AlarmReach for V { + fn widen_for_alarms(&self, range: TimeRange) -> TimeRange { + let components = self.components(); + let margin = self + .alarms() + .iter() + .filter_map(|alarm| { + let repetition = components + .get(alarm.id() as usize) + .map(ComponentView::alarm_repetition) + .unwrap_or_default(); + alarm + .trigger_offset() + .map(|offset| offset.saturating_add(repetition.span())) + }) + .max() + .unwrap_or_default() + .saturating_add(ALARM_EXPANSION_SLACK); + TimeRange { + start: range.start.saturating_sub(margin), + end: range.end.saturating_add(margin), + } + } +} + +impl PropFilterPlan for CalendarPropFilter { + fn candidates( + &self, + component: &ICalendarComponentType, + scope: &QueryScope<'_>, + index: TextIndex<'_>, + ) -> Candidates { + let Presence::IsDefined(CalendarPropMatch { + value: Some(PropValueMatch::Text(text_match)), + .. + }) = &self.test + else { + return Candidates::All; + }; + if text_match.negate { + return Candidates::All; + } + + match &self.name { + ICalendarProperty::Uid + if component.is_scheduling_object() + || *component == ICalendarComponentType::VAvailability => + { + scope.matching(|resource| { + resource.uid().is_none_or(|uid| { + uid.is_empty() || uid.is_hashed_uid() || text_match.is_match(uid) + }) + }) + } + name if component.is_scheduling_object() => match name.search_field() { + Some((field, text)) => index.candidates(field, text, text_match), + None => Candidates::All, + }, + _ => Candidates::All, + } + } +} + +impl CalendarIndexField for ICalendarProperty { + fn search_field(&self) -> Option<(SearchField, IndexedText)> { + let (field, text) = match self { + ICalendarProperty::Summary => (CalendarSearchField::Title, IndexedText::Stemmed), + ICalendarProperty::Description => { + (CalendarSearchField::Description, IndexedText::Stemmed) + } + ICalendarProperty::Location => (CalendarSearchField::Location, IndexedText::Plain), + ICalendarProperty::Organizer => (CalendarSearchField::Owner, IndexedText::Identifier), + ICalendarProperty::Attendee => (CalendarSearchField::Attendee, IndexedText::Identifier), + _ => return None, + }; + Some((field.into(), text)) + } +} + diff --git a/crates/dav/src/calendar/freebusy.rs b/crates/dav/src/calendar/freebusy.rs index 3c76a7f8..aafb2775 100644 --- a/crates/dav/src/calendar/freebusy.rs +++ b/crates/dav/src/calendar/freebusy.rs @@ -5,7 +5,7 @@ */ use super::query::CalendarQueryHandler; -use crate::{DavError, calendar::query::is_resource_in_time_range, common::uri::DavUriResource}; +use crate::{DavError, calendar::query::EventTimeRange, common::uri::DavUriResource}; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ @@ -16,26 +16,35 @@ use calcard::{ ICalendarProperty, ICalendarTransparency, ICalendarValue, }, }; -use common::{DavResourcePath, DavResources, PROD_ID, Server, auth::AccessToken}; +use common::{ + DavResourcePath, GroupwareResourceRef, GroupwareResources, PROD_ID, Server, auth::AccessToken, +}; use dav_proto::{RequestHeaders, schema::request::FreeBusyQuery}; -use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; +use groupware::{ + cache::GroupwareCache, + calendar::{ + CalendarEventContent, + availability::{AvailabilityCalendars, ComponentBusy, Inclusion, PrincipalRole}, + identity::ParticipantIdentityAddresses, + privacy::EventPrivacy, + }, +}; use http_proto::HttpResponse; use hyper::StatusCode; use std::str::FromStr; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; use store::{ ahash::AHashMap, + roaring::RoaringBitmap, write::{now, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ - TimeRange, + OverlapCondition, TimeRange, acl::Acl, collection::{Collection, SyncCollection}, + field::CalendarEventField, }; +use utils::map::bitmap::Bitmap; pub(crate) trait CalendarFreebusyRequestHandler: Sync + Send { fn handle_calendar_freebusy_request( @@ -49,12 +58,23 @@ pub(crate) trait CalendarFreebusyRequestHandler: Sync + Send { &self, access_token: &AccessToken, request: FreeBusyQuery, - resources: &DavResources, + resources: &GroupwareResources, account_id: u32, - resource: DavResourcePath<'_>, + scope: FreeBusyScope<'_>, ) -> impl Future> + Send; } +pub(crate) enum FreeBusyScope<'x> { + Collection(DavResourcePath<'x>), + Account(AvailabilityCalendars<'x>), +} + +struct SelectedEvent { + privacy: EventPrivacy, + tz: Tz, + attending_only: bool, +} + impl CalendarFreebusyRequestHandler for Server { async fn handle_calendar_freebusy_request( &self, @@ -64,12 +84,18 @@ impl CalendarFreebusyRequestHandler for Server { ) -> crate::Result { // Validate URI let resource_ = self - .validate_uri(access_token, headers.uri) + .parse_uri(headers.uri, StatusCode::NOT_FOUND) .await? .into_owned_uri()?; let account_id = resource_.account_id; + let has_account_access = access_token.has_access(account_id, resource_.collection); + let access_error = if has_account_access { + StatusCode::NOT_FOUND + } else { + StatusCode::FORBIDDEN + }; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, @@ -82,208 +108,312 @@ impl CalendarFreebusyRequestHandler for Server { .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) - .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + .filter(|resource| { + has_account_access + || (resource.is_container() + && resources.has_access_to_container( + access_token, + resource.document_id(), + Bitmap::from_iter([Acl::ReadItems, Acl::SchedulingReadFreeBusy]), + )) + }) + .ok_or(DavError::Code(access_error))?; if !resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } - self.build_freebusy_object(access_token, request, &resources, account_id, resource) - .await - .map(|ical| { - HttpResponse::new(StatusCode::OK) - .with_content_type("text/calendar; charset=utf-8") - .with_text_body(ical.to_string()) - }) + self.build_freebusy_object( + access_token, + request, + &resources, + account_id, + FreeBusyScope::Collection(resource), + ) + .await + .map(|ical| { + HttpResponse::new(StatusCode::OK) + .with_content_type("text/calendar; charset=utf-8") + .with_text_body(ical.to_string()) + }) } async fn build_freebusy_object( &self, access_token: &AccessToken, request: FreeBusyQuery, - resources: &DavResources, + resources: &GroupwareResources, account_id: u32, - resource: DavResourcePath<'_>, + scope: FreeBusyScope<'_>, ) -> crate::Result { - // Obtain shared ids - let shared_ids = if !access_token.is_member(account_id) { - resources - .shared_items( - access_token, - [Acl::ReadItems, Acl::SchedulingReadFreeBusy], - false, - ) - .into() - } else { - None + let is_owner = access_token.is_member(account_id); + let event_privacy = |resource: &GroupwareResourceRef<'_>| { + if is_owner { + EventPrivacy::Public + } else { + EventPrivacy::from_flags(resource.event_flags().unwrap_or_default()) + } }; - // Build FreeBusy component - let default_tz = resource - .resource - .calendar_preferences(account_id) - .map(|p| p.tz) - .unwrap_or(Tz::UTC); let mut entries = Vec::with_capacity(6); if let Some(range) = request.range { entries.push(ICalendarEntry { name: ICalendarProperty::Dtstart, params: vec![], - values: vec![ICalendarValue::PartialDateTime(Box::new( + values: [ICalendarValue::PartialDateTime( PartialDateTime::from_utc_timestamp(range.start), - ))], + )] + .into(), }); entries.push(ICalendarEntry { name: ICalendarProperty::Dtend, params: vec![], - values: vec![ICalendarValue::PartialDateTime(Box::new( + values: [ICalendarValue::PartialDateTime( PartialDateTime::from_utc_timestamp(range.end), - ))], + )] + .into(), }); entries.push(ICalendarEntry { name: ICalendarProperty::Dtstamp, params: vec![], - values: vec![ICalendarValue::PartialDateTime(Box::new( + values: [ICalendarValue::PartialDateTime( PartialDateTime::from_utc_timestamp(now() as i64), - ))], + )] + .into(), }); - let document_ids = resources - .children(resource.document_id()) - .filter(|resource| { - shared_ids - .as_ref() - .is_none_or(|ids| ids.contains(resource.document_id())) - && is_resource_in_time_range(resource.resource, &range) - }) - .map(|resource| resource.document_id()) - .collect::>(); + let mut document_ids = RoaringBitmap::new(); + let mut selected = AHashMap::new(); + let mut role = PrincipalRole::Owner; + match scope { + FreeBusyScope::Collection(collection) => { + let shared_acls = + Bitmap::from_iter([Acl::ReadItems, Acl::SchedulingReadFreeBusy]); + let tz = collection + .resource + .calendar_preferences(account_id) + .map_or(Tz::UTC, |p| p.tz); + for resource in resources.children(collection.document_id()) { + let privacy = event_privacy(&resource.resource); + if privacy != EventPrivacy::Secret + && (is_owner + || resources.is_shared_item( + &resource.resource, + access_token, + &shared_acls, + true, + )) + && resource.resource.resource.is_in_time_range(&range) + { + let document_id = resource.document_id(); + document_ids.insert(document_id); + selected.insert( + document_id, + SelectedEvent { + privacy, + tz, + attending_only: false, + }, + ); + } + } + } + FreeBusyScope::Account(mut calendars) => { + role = calendars.role(); + for resource in resources.resources.iter() { + if !resource.resource.is_in_time_range(&range) { + continue; + } + let privacy = event_privacy(&resource); + if privacy == EventPrivacy::Secret { + continue; + } + let Some(calendar) = calendars.relevant_calendar(resource.child_names()) + else { + continue; + }; + let document_id = resource.document_id(); + document_ids.insert(document_id); + selected.insert( + document_id, + SelectedEvent { + privacy, + tz: calendar.tz, + attending_only: calendar.inclusion == Inclusion::Attending, + }, + ); + } + } + } + let identities = if selected.values().any(|event| event.attending_only) { + Some( + self.account_identity_addresses(account_id) + .await + .caused_by(trc::location!())?, + ) + } else { + None + }; let mut fb_entries: AHashMap> = - AHashMap::with_capacity(document_ids.len()); + AHashMap::with_capacity(4); let max_instances = self.core.groupware.max_ical_instances; let mut total_instances: usize = 0; - for document_id in document_ids { - let Some(archive) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEvent, - document_id, - )) - .await - .caused_by(trc::location!())? - else { - continue; - }; - let event = archive - .unarchive::() - .caused_by(trc::location!())?; + if !document_ids.is_empty() { + self.archives( + account_id, + Collection::CalendarEvent, + CalendarEventField::Content.field(), + &document_ids, + |document_id, archive| { + let Some(&SelectedEvent { + privacy, + tz: default_tz, + attending_only, + }) = selected.get(&document_id) + else { + return Ok(true); + }; + let event = archive + .unarchive::() + .caused_by(trc::location!())?; - /* - Only VEVENT components without a TRANSP property or with the TRANSP - property set to OPAQUE, and VFREEBUSY components SHOULD be considered - in generating the free busy time information. - */ - let mut components = event - .data - .event - .components - .iter() - .enumerate() - .filter(|(_, comp)| { - (matches!(comp.component_type, ArchivedICalendarComponentType::VEvent) - && comp - .transparency() - .is_none_or(|t| t == &ICalendarTransparency::Opaque)) - || matches!( - comp.component_type, - ArchivedICalendarComponentType::VFreebusy - ) - }) - .peekable(); - - if components.peek().is_none() { - continue; - } - - let events = - CalendarQueryHandler::new(event, Some(range), default_tz).into_expanded_times(); - - if events.is_empty() { - continue; - } - - total_instances = total_instances.saturating_add(events.len()); - if total_instances > max_instances { - return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); - } - - for (component_id, component) in components { - let component_id = component_id as u32; - match component.component_type { - ArchivedICalendarComponentType::VEvent => { - let fbtype = match component.status() { - Some(ArchivedICalendarStatus::Cancelled) => continue, - Some(ArchivedICalendarStatus::Tentative) => { - ICalendarFreeBusyType::BusyTentative + /* + Only VEVENT components without a TRANSP property or with the TRANSP + property set to OPAQUE, and VFREEBUSY components SHOULD be considered + in generating the free busy time information. + */ + let mut components = event + .data + .event + .components + .iter() + .enumerate() + .filter(|(_, comp)| match comp.component_type { + ArchivedICalendarComponentType::VEvent => { + comp.transparency() + .is_none_or(|t| t == &ICalendarTransparency::Opaque) + && (!attending_only + || identities.as_ref().is_some_and(|identities| { + ComponentBusy::parse(comp, identities, role) + .is_some_and(|busy| busy.is_attending()) + })) } - _ => ICalendarFreeBusyType::Busy, - }; + ArchivedICalendarComponentType::VFreebusy => !attending_only, + _ => false, + }) + .peekable(); - let mut events_in_range = Vec::new(); - for event in &events { - if event.comp_id == component_id - && range.is_in_range(false, event.start, event.end) - { - events_in_range.push((event.start, event.end)); - } - } - - if !events_in_range.is_empty() { - fb_entries - .entry(fbtype) - .or_default() - .extend(events_in_range); - } + if components.peek().is_none() { + return Ok(true); } - ArchivedICalendarComponentType::VFreebusy => { - for entry in component.entries.iter() { - if matches!(entry.name, ArchivedICalendarProperty::Freebusy) { - let mut fb_in_range = - freebusy_in_range_utc(entry, &range, default_tz).peekable(); - if fb_in_range.peek().is_some() { - let fb_type = entry - .params - .iter() - .find_map(|param| { - if let ( - ArchivedICalendarParameterName::Fbtype, - ArchivedICalendarParameterValue::Fbtype(param), - ) = (¶m.name, ¶m.value) - { - rkyv_deserialize(param).ok() - } else { - None - } - }) - .unwrap_or(ICalendarFreeBusyType::Busy); - fb_entries.entry(fb_type).or_default().extend(fb_in_range); + let events = CalendarQueryHandler::new(event, Some(range), default_tz); + + if events.expanded_times.is_empty() { + return Ok(true); + } + + total_instances = + total_instances.saturating_add(events.expanded_times.len()); + if total_instances > max_instances { + return Ok(false); + } + + for (component_id, component) in components { + let component_id = component_id as u32; + match component.component_type { + ArchivedICalendarComponentType::VEvent => { + let fbtype = match component.status() { + Some(ArchivedICalendarStatus::Cancelled) => continue, + Some(ArchivedICalendarStatus::Tentative) + if privacy.is_public() => + { + ICalendarFreeBusyType::BusyTentative + } + _ => ICalendarFreeBusyType::Busy, + }; + + let mut events_in_range = events + .expansions_of(component_id) + .iter() + .filter(|event| { + event.start < event.end + && range.is_in_range( + OverlapCondition::Event, + event.start, + event.end, + ) + }) + .map(|event| (event.start, event.end)) + .peekable(); + + if events_in_range.peek().is_some() { + fb_entries + .entry(fbtype) + .or_default() + .extend(events_in_range); } } + ArchivedICalendarComponentType::VFreebusy => { + for entry in component.entries.iter() { + if matches!(entry.name, ArchivedICalendarProperty::Freebusy) + { + let mut fb_in_range = + freebusy_in_range_utc(entry, &range, default_tz) + .peekable(); + if fb_in_range.peek().is_some() { + let fb_type = entry + .params + .iter() + .find_map(|param| { + if let ( + ArchivedICalendarParameterName::Fbtype, + ArchivedICalendarParameterValue::Fbtype( + param, + ), + ) = (¶m.name, ¶m.value) + { + rkyv_deserialize(param).ok() + } else { + None + } + }) + .filter(|fb_type| { + privacy.is_public() + || *fb_type + == ICalendarFreeBusyType::Free + }) + .unwrap_or(ICalendarFreeBusyType::Busy); + + fb_entries + .entry(fb_type) + .or_default() + .extend(fb_in_range); + } + } + } + } + _ => {} } } - _ => {} - } - } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + + if total_instances > max_instances { + return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); } for (fbtype, events_in_range) in fb_entries { entries.push(ICalendarEntry { name: ICalendarProperty::Freebusy, params: vec![ICalendarParameter::fbtype(fbtype)], - values: merge_intervals(events_in_range), + values: merge_intervals(events_in_range).into(), }); } } @@ -297,12 +427,12 @@ impl CalendarFreebusyRequestHandler for Server { ICalendarEntry { name: ICalendarProperty::Version, params: vec![], - values: vec![ICalendarValue::Text("2.0".to_string())], + values: [ICalendarValue::Text("2.0".to_string())].into(), }, ICalendarEntry { name: ICalendarProperty::Prodid, params: vec![], - values: vec![ICalendarValue::Text(PROD_ID.to_string())], + values: [ICalendarValue::Text(PROD_ID.to_string())].into(), }, ], component_ids: vec![1], @@ -346,10 +476,10 @@ fn merge_intervals(mut intervals: Vec<(i64, i64)>) -> Vec { } fn build_ical_value(from: i64, to: i64) -> ICalendarValue { - ICalendarValue::Period(ICalendarPeriod::Range { + ICalendarValue::Period(Box::new(ICalendarPeriod::Range { start: PartialDateTime::from_utc_timestamp(from), end: PartialDateTime::from_utc_timestamp(to), - }) + })) } pub(crate) fn freebusy_in_range( @@ -367,7 +497,7 @@ pub(crate) fn freebusy_in_range( period.time_range(tz).and_then(|(start, end)| { let start = start.timestamp(); let end = end.timestamp(); - if range.is_in_range(false, start, end) { + if range.overlaps(start, end) { rkyv_deserialize(value).ok() } else { None @@ -394,7 +524,7 @@ fn freebusy_in_range_utc( period.time_range(tz).and_then(|(start, end)| { let start = start.timestamp(); let end = end.timestamp(); - if range.is_in_range(false, start, end) { + if start < end && range.overlaps(start, end) { Some((start, end)) } else { None diff --git a/crates/dav/src/calendar/get.rs b/crates/dav/src/calendar/get.rs index f786aeaf..2ba517dd 100644 --- a/crates/dav/src/calendar/get.rs +++ b/crates/dav/src/calendar/get.rs @@ -6,25 +6,33 @@ use crate::{ DavError, DavMethod, + calendar::CalendarEventView, common::{ - ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; -use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; +use groupware::{ + SizeWriter, + cache::GroupwareCache, + calendar::{ + CalendarEvent, CalendarEventContent, EVENT_HAS_ALARMS, alerts::DefaultAlertsResolver, + privacy::EventPrivacy, + }, +}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::CalendarEventField, }; pub(crate) trait CalendarGetRequestHandler: Sync + Send { @@ -50,7 +58,7 @@ impl CalendarGetRequestHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, @@ -69,10 +77,19 @@ impl CalendarGetRequestHandler for Server { } // Validate ACL - if !access_token.is_member(account_id) + let is_owner = access_token.is_member(account_id); + if !is_owner + && EventPrivacy::from_flags(resource.resource.event_flags().unwrap_or_default()) + == EventPrivacy::Secret + { + return Err(DavError::Code(StatusCode::NOT_FOUND)); + } + if !is_owner && !resources.has_access_to_container( access_token, - resource.parent_id().unwrap(), + resource + .parent_id() + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, Acl::ReadItems, ) { @@ -80,22 +97,35 @@ impl CalendarGetRequestHandler for Server { } // Fetch event - let event_ = self - .store() - .get_value::>(ValueKey::archive( + let (event_, prefetched) = if is_head || !headers.if_.is_empty() { + ( + self.store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + resource.document_id(), + )) + .await + .caused_by(trc::location!())?, + None, + ) + } else { + self.archive_and_field( account_id, Collection::CalendarEvent, resource.document_id(), - )) + CalendarEventField::Content.field(), + ) .await .caused_by(trc::location!())? - .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + }; + let event_ = event_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .unarchive::() .caused_by(trc::location!())?; // Validate headers - let etag = event_.etag(); + let etag = format!("\"{}\"", event.etag.to_native()); let schedule_tag = event.schedule_tag.as_ref().map(|tag| tag.to_native()); self.validate_headers( access_token, @@ -119,12 +149,50 @@ impl CalendarGetRequestHandler for Server { .with_schedule_tag_opt(schedule_tag) .with_last_modified(Rfc1123DateTime::new(i64::from(event.modified)).to_string()); - let ical = event.data.event.to_string(); - - if !is_head { - Ok(response.with_binary_body(ical)) - } else { - Ok(response.with_content_length(ical.len())) + let has_alarms = event.flags.to_native() & EVENT_HAS_ALARMS != 0; + if is_head && is_owner && !has_alarms { + return Ok(response.with_content_length(event.size.to_native() as usize)); } + + let content_ = match prefetched { + Some(content) => content, + None => self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + resource.document_id(), + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, + }; + let content = content_ + .to_unarchived::() + .caused_by(trc::location!())?; + let view = self + .calendar_event_view( + access_token, + account_id, + event, + content.inner, + &mut DefaultAlertsResolver::default(), + ) + .await?; + + if is_head { + return Ok(match view { + Some((view, _)) => response.with_content_length(SizeWriter::ical(&view.data.event)), + None => response.with_content_length(event.size.to_native() as usize), + }); + } + + let mut body = String::with_capacity(event.size.to_native() as usize); + let _ = match view { + Some((view, _)) => view.data.event.write_to(&mut body), + None => content.inner.data.event.write_to(&mut body), + }; + Ok(response.with_binary_body(body)) } } diff --git a/crates/dav/src/calendar/mkcol.rs b/crates/dav/src/calendar/mkcol.rs index 7d3846c1..84bbb59b 100644 --- a/crates/dav/src/calendar/mkcol.rs +++ b/crates/dav/src/calendar/mkcol.rs @@ -9,24 +9,29 @@ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ExtractETag, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, storage::index::PresenceFlags}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, }; use groupware::{ cache::GroupwareCache, - calendar::{Calendar, CalendarPreferences}, + calendar::{CALENDAR_SUBSCRIBED, Calendar, CalendarPreferences}, }; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use store::write::BatchBuilder; use trc::AddContext; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + metadata::MetadataKinds, +}; pub(crate) trait CalendarMkColRequestHandler: Sync + Send { fn handle_calendar_mkcol_request( @@ -55,21 +60,27 @@ impl CalendarMkColRequestHandler for Server { .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); - } else if name.contains('/') - || self - .fetch_dav_resources( - access_token.account_id(), - account_id, - SyncCollection::Calendar, - ) - .await - .caused_by(trc::location!())? - .by_path(name) - .is_some() - { + } + let resources = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::Calendar, + ) + .await + .caused_by(trc::location!())?; + if name.contains('/') || resources.by_path(name).is_some() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } + // Validate quota + self.assert_object_quota( + &*self.account(account_id).await?, + StorageQuota::MaxCalendars, + 1, + || resources.resources.count(true), + )?; + // Validate headers self.validate_headers( access_token, @@ -92,24 +103,46 @@ impl CalendarMkColRequestHandler for Server { preferences: vec![CalendarPreferences { account_id, name: name.to_string(), + flags: CALENDAR_SUBSCRIBED, ..Default::default() }], ..Default::default() }; // Apply MKCOL properties + let personal_id = access_token.personal_id(account_id, Collection::Calendar); + calendar.preferences_mut(personal_id).flags |= CALENDAR_SUBSCRIBED; + let mut batch = BatchBuilder::new(); + let document_id = batch.reserve_document_id(account_id, Collection::Calendar); let mut return_prop_stat = None; let mut is_mkcalendar = false; - if let Some(mkcol) = request { + let mut dead_write = None; + if let Some(mut mkcol) = request { let mut prop_stat = PropStatBuilder::default(); is_mkcalendar = mkcol.is_mkcalendar; - if !self.apply_calendar_properties( - access_token.personal_id(account_id, Collection::Calendar), + let dead = DeadPatch::take_values(&mut mkcol.props, DisplayName::Live); + self.apply_calendar_properties( + personal_id, &mut calendar, false, mkcol.props, &mut prop_stat, - ) { + ); + dead_write = dead + .apply( + self, + DeadTarget::container( + account_id, + Collection::Calendar, + document_id, + MetadataKinds::NONE, + ), + &mut prop_stat, + ) + .await + .caused_by(trc::location!())?; + if prop_stat.has_errors() { + prop_stat.fail_dependencies(); return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CalDav) @@ -123,22 +156,28 @@ impl CalendarMkColRequestHandler for Server { } // Prepare write batch - let mut batch = BatchBuilder::new(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Calendar, 1) - .await - .caused_by(trc::location!())?; + calendar.sync_owner_preferences(account_id, personal_id); + if let Some(dead_write) = &dead_write { + calendar.set_metadata_kinds(dead_write.kinds); + } + if let Some(write) = dead_write.and_then(|dead_write| dead_write.write) { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } calendar .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, &mut batch, ) .caused_by(trc::location!())?; - let etag = batch.etag(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let etag = self + .commit_batch(batch) + .await + .caused_by(trc::location!())? + .etag(); if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) diff --git a/crates/dav/src/calendar/mod.rs b/crates/dav/src/calendar/mod.rs index 064d3762..e4569275 100644 --- a/crates/dav/src/calendar/mod.rs +++ b/crates/dav/src/calendar/mod.rs @@ -6,6 +6,7 @@ pub mod copy_move; pub mod delete; +pub mod filter; pub mod freebusy; pub mod get; pub mod mkcol; @@ -13,17 +14,41 @@ pub mod proppatch; pub mod query; pub mod scheduling; pub mod update; +pub mod view; -use crate::{DavError, DavErrorCondition}; -use common::{DavResources, Server}; +use crate::{ + DavError, DavErrorCondition, + common::{ContainerOperation, MergedOverrides}, +}; +use calcard::icalendar::{ + ArchivedICalendar, ArchivedICalendarComponentType, ICalendar, ICalendarComponent, + ICalendarComponentType, +}; +use common::{ArchivedDavName, GroupwareResources, Server, auth::AccessToken}; use dav_proto::schema::{ property::{CalDavProperty, CalendarData, DavProperty, WebDavProperty}, response::CalCondition, }; -use groupware::scheduling::ItipError; +use groupware::{ + calendar::{ + Alarm, ArchivedCalendarEvent, ArchivedCalendarEventContent, Calendar, CalendarEventContent, + EVENT_HAS_ALARMS, SupportedComponent, + alarm::ExpandAlarm, + alerts::{DefaultAlertsResolver, DefaultAlertsView, ICalendarDefaultAlerts}, + compare::{ComparisonScope, RecurrenceComponents}, + privacy::{EventPrivacy, EventViewer, ICalendarPrivacy, PrivacyDenied}, + user::{ICalendarUserData, UserDataError, UserDataView}, + }, + scheduling::ItipError, +}; use hyper::StatusCode; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes, serialize::rkyv_deserialize}, +}; use trc::AddContext; use types::{collection::Collection, field::CalendarEventField}; +use utils::map::bitmap::Bitmap; pub(crate) static CALENDAR_CONTAINER_PROPS: [DavProperty; 31] = [ DavProperty::WebDav(WebDavProperty::CreationDate), @@ -87,37 +112,367 @@ pub(crate) static CALENDAR_ITEM_PROPS: [DavProperty; 20] = [ })), ]; -pub(crate) async fn assert_is_unique_uid( - server: &Server, - resources: &DavResources, +pub(crate) fn assert_is_unique_uid( + resources: &GroupwareResources, + access_token: &AccessToken, account_id: u32, calendar_id: u32, uid: Option<&str>, ) -> crate::Result<()> { - if let Some(uid) = uid { - let hits = server - .document_ids_matching( - account_id, - Collection::CalendarEvent, - CalendarEventField::Uid, - uid.as_bytes(), - ) - .await - .caused_by(trc::location!())?; + let Some(uid) = uid else { + return Ok(()); + }; - if !hits.is_empty() { - for path in resources.children(calendar_id) { - if hits.contains(path.document_id()) { - return Err(DavError::Condition(DavErrorCondition::new( - StatusCode::PRECONDITION_FAILED, - CalCondition::NoUidConflict(resources.format_resource(path).into()), - ))); + let is_owner = access_token.is_member(account_id); + let mut has_hidden_conflict = false; + for path in resources.children_with_uid(calendar_id, uid) { + if is_owner + || EventPrivacy::from_flags(path.resource.event_flags().unwrap_or_default()) + != EventPrivacy::Secret + { + return Err(DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + CalCondition::NoUidConflict(resources.format_resource(path).into()), + ))); + } + has_hidden_conflict = true; + } + + if has_hidden_conflict { + Err(DavError::Code(StatusCode::FORBIDDEN)) + } else { + Ok(()) + } +} + +impl ContainerOperation { + pub(crate) fn event_ids( + self, + resources: &GroupwareResources, + path: &str, + viewer: EventViewer, + ) -> crate::Result> { + let mut document_ids = Vec::new(); + for resource in resources.subtree(path).filter(|r| !r.is_container()) { + match EventPrivacy::from_flags(resource.resource.event_flags().unwrap_or_default()) { + _ if viewer.is_owner() => document_ids.push(resource.document_id()), + EventPrivacy::Public => document_ids.push(resource.document_id()), + EventPrivacy::Secret if self == ContainerOperation::Copy => {} + EventPrivacy::Private | EventPrivacy::Secret => { + return Err(DavError::Code(StatusCode::FORBIDDEN)); } } } + Ok(document_ids) + } +} + +pub(crate) struct SupportedComponents(Bitmap); + +impl SupportedComponents { + fn is_restricted(&self) -> bool { + !self.0.is_empty() } - Ok(()) + pub(crate) fn assert_supports(&self, component: SupportedComponent) -> crate::Result<()> { + if !self.is_restricted() || self.0.contains(component) { + Ok(()) + } else { + Err(DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + CalCondition::SupportedCalendarComponent, + ))) + } + } +} + +trait CalendarObjectComponent { + fn object_component(&self) -> SupportedComponent; +} + +impl CalendarObjectComponent for ArchivedICalendar { + fn object_component(&self) -> SupportedComponent { + self.components + .iter() + .find_map(|component| match component.component_type { + ArchivedICalendarComponentType::VEvent => Some(SupportedComponent::VEvent), + ArchivedICalendarComponentType::VTodo => Some(SupportedComponent::VTodo), + ArchivedICalendarComponentType::VJournal => Some(SupportedComponent::VJournal), + ArchivedICalendarComponentType::VFreebusy => Some(SupportedComponent::VFreebusy), + ArchivedICalendarComponentType::VAvailability => { + Some(SupportedComponent::VAvailability) + } + _ => None, + }) + .unwrap_or(SupportedComponent::Other) + } +} + +pub(crate) trait CalendarComponentSupport: Sync + Send { + fn supported_components( + &self, + account_id: u32, + calendar_id: u32, + ) -> impl Future> + Send; + + fn assert_stored_event_supported( + &self, + event_account_id: u32, + event_document_id: u32, + account_id: u32, + calendar_id: u32, + ) -> impl Future>>> + Send; +} + +impl CalendarComponentSupport for Server { + async fn supported_components( + &self, + account_id: u32, + calendar_id: u32, + ) -> crate::Result { + self.store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + calendar_id, + )) + .await + .caused_by(trc::location!())? + .map(|archive| { + archive + .unarchive::() + .map(|calendar| calendar.supported_components.to_native()) + }) + .transpose() + .caused_by(trc::location!()) + .map(|supported_components| { + SupportedComponents(Bitmap::from(supported_components.unwrap_or_default())) + }) + .map_err(Into::into) + } + + async fn assert_stored_event_supported( + &self, + event_account_id: u32, + event_document_id: u32, + account_id: u32, + calendar_id: u32, + ) -> crate::Result>> { + let supported_components = self.supported_components(account_id, calendar_id).await?; + if !supported_components.is_restricted() { + return Ok(None); + } + + let content = self + .store() + .get_value::>(ValueKey::property( + event_account_id, + Collection::CalendarEvent, + event_document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + supported_components.assert_supports( + content + .unarchive::() + .caused_by(trc::location!())? + .data + .event + .object_component(), + )?; + Ok(Some(content)) + } +} + +pub(crate) fn assert_event_privacy_access( + flags: Option, + viewer: EventViewer, +) -> crate::Result<()> { + flags + .map(EventPrivacy::from_flags) + .unwrap_or_default() + .check_access(viewer) + .map_err(|denied| { + DavError::Code(match denied { + PrivacyDenied::Forbidden => StatusCode::FORBIDDEN, + PrivacyDenied::NotFound => StatusCode::NOT_FOUND, + }) + }) +} + +pub(crate) fn assert_event_privacy_allowed( + ical: &ICalendar, + viewer: EventViewer, +) -> crate::Result<()> { + if ical.privacy().may_be_set_by(viewer) { + Ok(()) + } else { + Err(DavError::Code(StatusCode::FORBIDDEN)) + } +} + +pub(crate) fn serves_ical(view: &ICalendar, submitted: &ICalendar) -> bool { + view == submitted || ComparisonScope::SharedData.component_eq(view, 0, submitted, 0) +} + +pub(crate) trait CalendarEventView: Sync + Send { + fn calendar_event_view( + &self, + access_token: &AccessToken, + account_id: u32, + event: &ArchivedCalendarEvent, + content: &ArchivedCalendarEventContent, + default_alerts: &mut DefaultAlertsResolver, + ) -> impl Future>> + Send; +} + +impl CalendarEventView for Server { + async fn calendar_event_view( + &self, + access_token: &AccessToken, + account_id: u32, + event: &ArchivedCalendarEvent, + content: &ArchivedCalendarEventContent, + default_alerts: &mut DefaultAlertsResolver, + ) -> crate::Result> { + let flags = event.flags.to_native(); + let is_owner = access_token.is_member(account_id); + let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let is_private_view = !is_owner && EventPrivacy::from_flags(flags) != EventPrivacy::Public; + let preferences = content.preferences(personal_id); + let use_default_alerts = preferences.is_some_and(|p| p.use_default_alerts()); + if !is_private_view + && !use_default_alerts + && (is_owner + || (flags & EVENT_HAS_ALARMS == 0 + && preferences.is_none() + && !content + .data + .event + .components + .iter() + .any(|component| component.component_type.is_alarm()))) + { + return Ok(None); + } + + if is_private_view { + let mut view = rkyv_deserialize::<_, CalendarEventContent>(content) + .map(CalendarEventContent::into_private_view) + .caused_by(trc::location!())?; + let merged_overrides = view.data.event.blank_unreachable_components(); + return Ok(Some((view, merged_overrides))); + } + + let resolved_default_alerts = default_alerts + .resolve_for_ical_view( + self, + account_id, + personal_id, + content, + event.names.iter().map(ArchivedDavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + let mut view = + rkyv_deserialize::<_, CalendarEventContent>(content).caused_by(trc::location!())?; + if !is_owner { + let preferences = view + .preferences + .iter() + .find(|p| p.account_id == personal_id); + view.data + .event + .apply_user_data(preferences, UserDataView::AlertsOnly); + } + view.data + .event + .apply_default_alerts(&resolved_default_alerts, DefaultAlertsView::ICalendar); + let merged_overrides = view.data.event.blank_unreachable_components(); + view.data.alarms = view.data.event.served_alarms(); + Ok(Some((view, merged_overrides))) + } +} + +trait UnreachableComponents { + fn blank_unreachable_components(&mut self) -> MergedOverrides; +} + +impl UnreachableComponents for ICalendar { + fn blank_unreachable_components(&mut self) -> MergedOverrides { + let mut is_reachable = vec![false; self.components.len()]; + let mut pending = vec![0u32]; + while let Some(component_id) = pending.pop() { + match is_reachable.get_mut(component_id as usize) { + Some(is_reachable) if !*is_reachable => *is_reachable = true, + _ => continue, + } + if let Some(component) = self.components.get(component_id as usize) { + pending.extend_from_slice(&component.component_ids); + } + } + + let mut merged_overrides = MergedOverrides::default(); + for (component_id, (component, is_reachable)) in + self.components.iter_mut().zip(&is_reachable).enumerate() + { + if *is_reachable { + continue; + } + if component.component_type.is_event_or_todo() && component.is_recurrence_override() { + merged_overrides.push(component_id as u32); + } + *component = ICalendarComponent { + component_type: ICalendarComponentType::Other(Default::default()), + entries: vec![], + component_ids: vec![], + }; + } + merged_overrides.set_base(self.base_component_id()); + merged_overrides + } +} + +trait ServedAlarms { + fn served_alarms(&self) -> Box<[Alarm]>; +} + +impl ServedAlarms for ICalendar { + fn served_alarms(&self) -> Box<[Alarm]> { + self.components + .iter() + .enumerate() + .filter(|(_, component)| component.component_type.is_event_or_todo()) + .flat_map(|(parent_id, component)| { + component.component_ids.iter().filter_map(move |alarm_id| { + self.components + .get(*alarm_id as usize) + .filter(|alarm| alarm.component_type.is_alarm()) + .and_then(|alarm| alarm.expand_alarm(*alarm_id as u16, parent_id as u16)) + }) + }) + .collect() + } +} + +pub(crate) fn user_data_error(err: UserDataError) -> DavError { + DavError::Condition( + DavErrorCondition::new( + StatusCode::PRECONDITION_FAILED, + CalCondition::ValidCalendarData, + ) + .with_details(match err { + UserDataError::TooManyKeywords => "Too many categories", + UserDataError::KeywordTooLong => "Category too long", + UserDataError::TooManyAlerts => "Too many alarms", + UserDataError::TooManyInstances => { + "Too many recurrence instances with personal properties" + } + UserDataError::InvalidColor => "Invalid color", + }), + ) } pub(crate) trait ItipPrecondition { @@ -143,6 +498,7 @@ impl ItipPrecondition for ItipError { | ItipError::OutOfSequence | ItipError::UnknownParticipant(_) | ItipError::UnsupportedMethod(_) => Some(CalCondition::ValidSchedulingMessage), + ItipError::TooManyRecipients => Some(CalCondition::MaxAttendeesPerInstance), _ => None, } } diff --git a/crates/dav/src/calendar/proppatch.rs b/crates/dav/src/calendar/proppatch.rs index efaf5edf..52d15d28 100644 --- a/crates/dav/src/calendar/proppatch.rs +++ b/crates/dav/src/calendar/proppatch.rs @@ -6,26 +6,36 @@ use crate::{ DavError, DavMethod, PropStatBuilder, + calendar::assert_event_privacy_access, common::{ ETag, ExtractETag, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use calcard::common::timezone::Tz; -use common::{Server, auth::AccessToken}; +use common::{ + Server, + auth::AccessToken, + storage::index::{PresenceFlags, RewritePresence}, +}; use dav_proto::{ RequestHeaders, Return, schema::{ Namespace, property::{CalDavProperty, DavProperty, DavValue, ResourceType, WebDavProperty}, - request::{DavPropertyValue, PropertyUpdate}, + request::{DavPropertyValue, PropertyUpdate, PropertyUpdateOp}, response::{BaseCondition, CalCondition, MultiStatus, Response}, }, }; use groupware::{ cache::GroupwareCache, - calendar::{Calendar, CalendarEvent, SupportedComponent, Timezone}, + calendar::{ + Calendar, CalendarEvent, SupportedComponent, Timezone, + alerts::{CalendarAlarmsReschedule, CalendarSettings}, + privacy::EventViewer, + }, }; use http_proto::HttpResponse; use hyper::StatusCode; @@ -33,7 +43,7 @@ use std::str::FromStr; use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -55,17 +65,16 @@ pub(crate) trait CalendarPropPatchRequestHandler: Sync + Send { personal_id: u32, calendar: &mut Calendar, is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool; + ); fn apply_event_properties( &self, event: &mut CalendarEvent, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool; + ); } impl CalendarPropPatchRequestHandler for Server { @@ -83,7 +92,7 @@ impl CalendarPropPatchRequestHandler for Server { let uri = headers.uri; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, @@ -106,11 +115,21 @@ impl CalendarPropPatchRequestHandler for Server { } // Verify ACL - if !access_token.is_member(account_id) { + let is_owner = access_token.is_member(account_id); + if !is_owner { let (acl, document_id) = if resource.is_container() { (Acl::Modify, resource.document_id()) } else { - (Acl::ModifyItems, resource.parent_id().unwrap()) + assert_event_privacy_access( + resource.resource.event_flags(), + EventViewer::new(is_owner), + )?; + ( + Acl::ModifyItems, + resource + .parent_id() + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, + ) }; if !resources.has_access_to_container(access_token, document_id, acl) { @@ -121,7 +140,7 @@ impl CalendarPropPatchRequestHandler for Server { // Fetch archive let archive = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, collection, document_id, @@ -129,6 +148,18 @@ impl CalendarPropPatchRequestHandler for Server { .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let etag = if resource.is_container() { + archive.etag() + } else { + format!( + "\"{}\"", + archive + .unarchive::() + .caused_by(trc::location!())? + .etag + .to_native() + ) + }; // Validate headers self.validate_headers( @@ -138,7 +169,7 @@ impl CalendarPropPatchRequestHandler for Server { account_id, collection, document_id: document_id.into(), - etag: archive.etag().into(), + etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], @@ -147,107 +178,203 @@ impl CalendarPropPatchRequestHandler for Server { ) .await?; - let is_success; + let dead = DeadPatch::take(&mut request.ops, DisplayName::Live); + let has_live_changes = !request.ops.is_empty(); let mut batch = BatchBuilder::new(); let mut items = PropStatBuilder::default(); - let etag = if resource.is_container() { + let (is_success, etag) = if resource.is_container() { // Deserialize let calendar = archive .to_unarchived::() .caused_by(trc::location!())?; - let mut new_calendar = archive - .deserialize::() - .caused_by(trc::location!())?; let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let is_member = access_token.is_member(account_id); - // Remove properties - if !request.set_first && !request.remove.is_empty() { - remove_calendar_properties( - personal_id, - &mut new_calendar, - std::mem::take(&mut request.remove), + // Apply live properties + let new_calendar = if has_live_changes { + let mut new_calendar = archive + .deserialize::() + .caused_by(trc::location!())?; + if is_member { + new_calendar.inherit_owner_preferences(account_id, personal_id); + } + for op in request.ops { + match op { + PropertyUpdateOp::Set(value) => self.apply_calendar_properties( + personal_id, + &mut new_calendar, + true, + [value], + &mut items, + ), + PropertyUpdateOp::Remove(property) => remove_calendar_properties( + personal_id, + &mut new_calendar, + [property], + &mut items, + ), + } + } + Some(new_calendar) + } else { + None + }; + + // Apply dead properties + let mut dead_write = dead + .apply( + self, + DeadTarget::container( + account_id, + Collection::Calendar, + document_id, + calendar.inner.metadata_kinds(), + ), &mut items, - ); - } + ) + .await + .caused_by(trc::location!())?; - // Set properties - is_success = self.apply_calendar_properties( - personal_id, - &mut new_calendar, - true, - request.set, - &mut items, - ); - - // Remove properties - if is_success && !request.remove.is_empty() { - remove_calendar_properties( - personal_id, - &mut new_calendar, - request.remove, - &mut items, - ); - } - - if is_success { - new_calendar - .update( - access_token.account_tenant_ids(), - calendar, + if items.has_errors() { + (false, etag) + } else { + if let Some(mut new_calendar) = new_calendar { + if is_member { + new_calendar.sync_owner_preferences(account_id, personal_id); + } + if let Some(dead_write) = &dead_write { + new_calendar.set_metadata_kinds(dead_write.kinds); + } + self.reschedule_calendar_alarms( + &resources, + account_id, + document_id, + CalendarSettings::from(calendar.inner), + CalendarSettings::from(&new_calendar), + &mut batch, + ) + .await + .caused_by(trc::location!())?; + if let Some(write) = dead_write.and_then(|dead_write| dead_write.write) { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + new_calendar + .update( + access_token.changed_by(), + calendar, + account_id, + document_id, + &mut batch, + ) + .caused_by(trc::location!())?; + } else if let Some(dead_write) = dead_write.as_mut() { + if let Some(write) = dead_write.write.take() { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + Calendar::rewrite_presence( + &calendar, + dead_write.kinds, account_id, document_id, &mut batch, ) - .caused_by(trc::location!())? - .etag() - } else { - calendar.etag().into() + .caused_by(trc::location!())?; + } + (true, batch.etag().unwrap_or(etag)) } } else { // Deserialize let event = archive .to_unarchived::() .caused_by(trc::location!())?; - let mut new_event = archive - .deserialize::() + + // Apply live properties + let new_event = if has_live_changes { + let mut new_event = archive + .deserialize::() + .caused_by(trc::location!())?; + for op in request.ops { + match op { + PropertyUpdateOp::Set(value) => { + self.apply_event_properties(&mut new_event, [value], &mut items) + } + PropertyUpdateOp::Remove(property) => { + remove_event_properties(&mut new_event, [property], &mut items) + } + } + } + Some(new_event) + } else { + None + }; + + // Apply dead properties + let mut dead_write = dead + .apply( + self, + DeadTarget::item( + account_id, + Collection::CalendarEvent, + document_id, + event.inner.metadata_kinds(), + ), + &mut items, + ) + .await .caused_by(trc::location!())?; - // Remove properties - if !request.set_first && !request.remove.is_empty() { - remove_event_properties( - &mut new_event, - std::mem::take(&mut request.remove), - &mut items, - ); - } - - // Set properties - is_success = self.apply_event_properties(&mut new_event, true, request.set, &mut items); - - // Remove properties - if is_success && !request.remove.is_empty() { - remove_event_properties(&mut new_event, request.remove, &mut items); - } - - if is_success { - new_event - .update( - access_token.account_tenant_ids(), - event, + if items.has_errors() { + (false, etag) + } else { + let mut new_etag = None; + if let Some(write) = dead_write + .as_mut() + .and_then(|dead_write| dead_write.write.take()) + { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + if let Some(mut new_event) = new_event { + if let Some(dead_write) = &dead_write { + new_event.set_metadata_kinds(dead_write.kinds); + } + new_etag = new_event + .update_meta( + access_token.changed_by(), + event, + account_id, + document_id, + None, + &mut batch, + ) + .caused_by(trc::location!())? + .into(); + } else if let Some(dead_write) = &dead_write { + CalendarEvent::rewrite_presence( + &event, + dead_write.kinds, account_id, document_id, &mut batch, ) - .caused_by(trc::location!())? - .etag() - } else { - event.etag().into() + .caused_by(trc::location!())?; + } + (true, new_etag.unwrap_or(etag)) } }; if is_success { - self.commit_batch(batch).await.caused_by(trc::location!())?; + if !batch.is_empty() { + self.commit_batch(batch).await.caused_by(trc::location!())?; + } + } else { + items.fail_dependencies(); } if headers.ret != Return::Minimal || !is_success { @@ -257,9 +384,9 @@ impl CalendarPropPatchRequestHandler for Server { .with_namespace(Namespace::CalDav) .to_string(), ) - .with_etag_opt(etag)) + .with_etag(etag)) } else { - Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) + Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag(etag)) } } @@ -268,11 +395,9 @@ impl CalendarPropPatchRequestHandler for Server { personal_id: u32, calendar: &mut Calendar, is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool { - let mut has_errors = false; - + ) { for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { @@ -285,7 +410,6 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } } ( @@ -301,8 +425,6 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - - has_errors = true; } } ( @@ -315,7 +437,6 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } else if !ical.is_timezone() { items.insert_precondition_failed_with_description( property.property, @@ -323,7 +444,6 @@ impl CalendarPropPatchRequestHandler for Server { CalCondition::ValidCalendarData, "Invalid calendar timezone", ); - has_errors = true; } else { calendar.preferences_mut(personal_id).time_zone = Timezone::Custom(ical); items.insert_ok(property.property); @@ -341,7 +461,6 @@ impl CalendarPropPatchRequestHandler for Server { CalCondition::ValidTimezone, "Invalid timezone ID", ); - has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { @@ -362,7 +481,6 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); - has_errors = true; } else { items.insert_ok(property.property); } @@ -388,7 +506,6 @@ impl CalendarPropPatchRequestHandler for Server { CalCondition::SupportedCalendarComponent, "At least one supported component must be specified", ); - has_errors = true; } } else { items.insert_precondition_failed_with_description( @@ -397,29 +514,6 @@ impl CalendarPropPatchRequestHandler for Server { CalCondition::SupportedCalendarComponent, "Property cannot be modified", ); - has_errors = true; - } - } - (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) - if self.core.groupware.dead_property_size.is_some() => - { - if is_update { - calendar.dead_properties.remove_element(dead); - } - - if calendar.dead_properties.size() + values.size() + dead.size() - < self.core.groupware.dead_property_size.unwrap() - { - calendar.dead_properties.add_element(dead.clone(), values.0); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - - has_errors = true; } } (_, DavValue::Null) => { @@ -431,23 +525,17 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::CONFLICT, "Property cannot be modified", ); - has_errors = true; } } } - - !has_errors } fn apply_event_properties( &self, event: &mut CalendarEvent, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool { - let mut has_errors = false; - + ) { for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { @@ -460,34 +548,12 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { event.created = dt; items.insert_ok(property.property); } - (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) - if self.core.groupware.dead_property_size.is_some() => - { - if is_update { - event.dead_properties.remove_element(dead); - } - - if event.dead_properties.size() + values.size() + dead.size() - < self.core.groupware.dead_property_size.unwrap() - { - event.dead_properties.add_element(dead.clone(), values.0); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - has_errors = true; - } - } (_, DavValue::Null) => { items.insert_ok(property.property); } @@ -497,29 +563,22 @@ impl CalendarPropPatchRequestHandler for Server { StatusCode::CONFLICT, "Property cannot be modified", ); - has_errors = true; } } } - - !has_errors } } fn remove_event_properties( event: &mut CalendarEvent, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::WebDav(WebDavProperty::DisplayName) => { event.display_name = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::DeadProperty(dead) => { - event.dead_properties.remove_element(dead); - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } _ => { items.insert_error_with_description( @@ -535,23 +594,19 @@ fn remove_event_properties( fn remove_calendar_properties( personal_id: u32, calendar: &mut Calendar, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::CalDav(CalDavProperty::CalendarDescription) => { calendar.preferences_mut(personal_id).description = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } DavProperty::CalDav(CalDavProperty::CalendarTimezone) | DavProperty::CalDav(CalDavProperty::TimezoneId) => { calendar.preferences_mut(personal_id).time_zone = Timezone::Default; - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::DeadProperty(dead) => { - calendar.dead_properties.remove_element(dead); - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } _ => { items.insert_error_with_description( diff --git a/crates/dav/src/calendar/query.rs b/crates/dav/src/calendar/query.rs index 0d14cc71..f938e337 100644 --- a/crates/dav/src/calendar/query.rs +++ b/crates/dav/src/calendar/query.rs @@ -4,46 +4,55 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::freebusy::freebusy_in_range; +use super::{ + filter::{CalendarFilterPlan, FilterTimeRanges}, + view::{CalendarView, ChildIdView, ComponentView, EntryView}, +}; use crate::{ DavError, common::{ - CalendarFilter, DavQuery, - propfind::{PropFindItem, PropFindRequestHandler}, + CalendarQueryFilter, DavQuery, EventContent, + propfind::PropFindRequestHandler, + search::{QueryScope, TextIndex}, uri::DavUriResource, }, }; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ - ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarEntry, - ArchivedICalendarParameter, ArchivedICalendarProperty, ArchivedICalendarValue, - ICalendarComponentType, ICalendarEntry, ICalendarParameterName, ICalendarProperty, - ICalendarValue, + ICalendar, ICalendarComponentType, ICalendarEntry, ICalendarProperty, ICalendarValue, }, }; -use common::{DavResource, Server, auth::AccessToken}; +use common::{DavResourcePath, GroupwareResource, Server, auth::AccessToken}; +use compact_str::ToCompactString; use dav_proto::{ RequestHeaders, schema::{ property::{CalDavProperty, CalendarData, DavProperty}, - request::{CalendarQuery, Filter, FilterOp, PropFind, Timezone}, + request::{CalendarQuery, PropFind, Timezone}, response::MultiStatus, }, }; use groupware::{ cache::GroupwareCache, - calendar::{ArchivedCalendarEvent, expand::CalendarEventExpansion}, + calendar::{ + ArchivedCalendarEvent, ArchivedCalendarEventContent, + ArchivedCalendarEventNotificationContent, CalendarEventContent, CalendarEventData, + EVENT_HAS_UNBOUNDED_TODO, EVENT_SECRET, + expand::{CalendarEventExpansion, MAX_UTC_OFFSET}, + }, }; use http_proto::HttpResponse; use hyper::StatusCode; use std::{fmt::Write, slice::Iter, str::FromStr}; -use store::{ - ahash::{AHashMap, AHashSet}, - write::serialize::rkyv_deserialize, -}; +use store::write::{SearchIndex, serialize::rkyv_deserialize}; use trc::AddContext; -use types::{TimeRange, acl::Acl, collection::SyncCollection}; +use types::{ + OverlapCondition, OverlapRule, TimeRange, + acl::Acl, + collection::{Collection, SyncCollection}, +}; +use utils::map::bitmap::Bitmap; pub(crate) trait CalendarQueryRequestHandler: Sync + Send { fn handle_calendar_query_request( @@ -54,6 +63,18 @@ pub(crate) trait CalendarQueryRequestHandler: Sync + Send { ) -> impl Future> + Send; } +pub(crate) trait QueryTimezone { + fn resolve(&self) -> Option; +} + +pub(crate) trait EventTimeRange { + fn is_in_time_range(&self, range: &TimeRange) -> bool; +} + +pub(crate) trait CalendarDataRange { + fn calendar_data_range(&self) -> Option; +} + impl CalendarQueryRequestHandler for Server { async fn handle_calendar_query_request( &self, @@ -67,11 +88,16 @@ impl CalendarQueryRequestHandler for Server { .await? .into_owned_uri()?; let account_id = resource_.account_id; + let collection = if resource_.collection == Collection::CalendarEventNotification { + Collection::CalendarEventNotification + } else { + Collection::Calendar + }; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, - SyncCollection::Calendar, + SyncCollection::from(collection), ) .await .caused_by(trc::location!())?; @@ -83,409 +109,346 @@ impl CalendarQueryRequestHandler for Server { return Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(MultiStatus::not_found(headers.uri).to_string())); }; - if !resource.is_container() { - return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); - } - // Obtain shared ids - let shared_ids = if !access_token.is_member(account_id) { - resources - .shared_items(access_token, [Acl::ReadItems], false) - .into() + let is_owner = access_token.is_member(account_id); + let read_items = Bitmap::from_iter([Acl::ReadItems]); + let is_visible = |item: &DavResourcePath<'_>| { + is_owner + || (item.resource.event_flags().unwrap_or_default() & EVENT_SECRET == 0 + && resources.is_shared_item(&item.resource, access_token, &read_items, false)) + }; + let scope = if resource.is_container() { + QueryScope::new( + resources + .children(resource.document_id()) + .filter(is_visible), + ) } else { - None + QueryScope::new(std::iter::once(resource).filter(is_visible)) }; - // Pre-filter by date range - let filter_range = extract_filter_range(&request); - - // Obtain document ids in folder - let mut items = Vec::with_capacity(16); - for resource in resources.children(resource.document_id()) { - if shared_ids - .as_ref() - .is_none_or(|ids| ids.contains(resource.document_id())) - && filter_range - .as_ref() - .is_none_or(|range| is_resource_in_time_range(resource.resource, range)) - { - items.push(PropFindItem::new( - resources.format_resource(resource), - account_id, - resource, - )); + let filter_ranges = request + .filter + .as_ref() + .map(CalendarFilterPlan::time_ranges) + .unwrap_or_default(); + let items = match &request.filter { + Some(filter) if collection == Collection::Calendar && !scope.is_empty() => { + let candidates = + filter.candidates(&scope, TextIndex::new(self, SearchIndex::Calendar)); + scope + .resolve( + self, + SearchIndex::Calendar, + account_id, + candidates, + &resources, + ) + .await } - } + _ => scope.into_items(&resources, account_id), + }; - // Extract the time range from the request - let max_time_range = extract_data_range(&request.properties, filter_range); + let filter = CalendarQueryFilter { + expansion: filter_ranges.union(FilterTimeRanges { + range: request.properties.calendar_data_range(), + has_alarms: false, + }), + timezone: request.timezone.resolve(), + filter: request.filter, + }; self.handle_dav_query( access_token, - DavQuery::calendar_query(request, max_time_range, items, headers), + DavQuery::calendar_query(request.properties, filter, collection, items, headers), ) .await } } -pub(crate) fn is_resource_in_time_range(resource: &DavResource, filter: &TimeRange) -> bool { - // Check whether the resource has a time range and if it overlaps with the filter - if let Some((start, end)) = resource.event_time_range() { - ((filter.start < end) || (filter.start <= start)) - && (filter.end > start || filter.end >= end) - } else { - // If the resource does not have a time range, it is not in the range - false +impl QueryTimezone for Timezone { + fn resolve(&self) -> Option { + match self { + Timezone::Id(id) => Tz::from_str(id).ok(), + Timezone::Name(value) => Tz::from_str(value).ok().or_else(|| { + ICalendar::parse(value).ok().and_then(|ical| { + ical.timezones() + .find_map(|timezone| timezone.timezone().map(|(_, tz)| tz)) + }) + }), + Timezone::None => None, + } } } -fn extract_filter_range(query: &CalendarQuery) -> Option { - let mut range = TimeRange { - start: i64::MAX, - end: i64::MIN, - }; +impl EventTimeRange for GroupwareResource { + fn is_in_time_range(&self, range: &TimeRange) -> bool { + self.event_time_range().is_some_and(|(start, end)| { + range.touches( + start.saturating_sub(MAX_UTC_OFFSET), + end.saturating_add(MAX_UTC_OFFSET), + ) + }) + } +} - for filter in &query.filters { - let op = match filter { - Filter::Component { op, .. } => op, - Filter::Property { op, .. } => op, - Filter::Parameter { op, .. } => op, - _ => continue, +impl CalendarDataRange for PropFind { + fn calendar_data_range(&self) -> Option { + let props = match self { + PropFind::AllProp(props) | PropFind::Prop(props) => props, + PropFind::PropName => return None, }; - if let FilterOp::TimeRange(date_range) = op { - if date_range.start < range.start { - range.start = date_range.start; - } - if date_range.end > range.end { - range.end = date_range.end; - } - } - } - if range.start != i64::MAX { - Some(range) - } else { - None + props + .iter() + .find_map(|prop| match prop { + DavProperty::CalDav(CalDavProperty::CalendarData(data)) => Some(data), + _ => None, + }) + .and_then(CalendarDataRange::calendar_data_range) } } -fn extract_data_range(propfind: &PropFind, filter_range: Option) -> Option { - let props = match propfind { - PropFind::AllProp(props) | PropFind::Prop(props) => props, - PropFind::PropName => &[][..], - }; - - for prop in props { - if let DavProperty::CalDav(CalDavProperty::CalendarData(data)) = prop { - let mut range = filter_range.unwrap_or(TimeRange { - start: i64::MAX, - end: i64::MIN, - }); - - for data_range in [&data.expand, &data.limit_recurrence, &data.limit_freebusy] - .into_iter() - .flatten() - { - if data_range.start < range.start { - range.start = data_range.start; - } - if data_range.end > range.end { - range.end = data_range.end; - } - } - - return if range.start != i64::MAX { - Some(range) - } else { - None - }; - } +impl CalendarDataRange for CalendarData { + fn calendar_data_range(&self) -> Option { + [self.expand, self.limit_recurrence, self.limit_freebusy] + .into_iter() + .flatten() + .reduce(TimeRange::union) } - - filter_range } -pub fn try_parse_tz(tz: &Timezone) -> Option { - match tz { - Timezone::Name(value) | Timezone::Id(value) => Tz::from_str(value).ok(), - Timezone::None => None, +impl CalendarQueryFilter { + pub fn matches_scheduling_message( + &self, + content: &ArchivedCalendarEventNotificationContent, + default_tz: Tz, + max_instances: usize, + ) -> trc::Result { + let Some(filter) = &self.filter else { + return Ok(true); + }; + let Some(ical) = content.calendar_data() else { + return Ok(false); + }; + let time_ranges = filter.time_ranges(); + if time_ranges.range.is_none() { + return Ok(CalendarQueryHandler::for_scheduling(default_tz).matches(ical, filter)); + } + + let content = CalendarEventContent { + data: CalendarEventData::new(rkyv_deserialize(ical)?, Tz::Floating, max_instances), + ..Default::default() + }; + let mut handler = CalendarQueryHandler::for_event_data(&content, time_ranges, default_tz); + handler.dateless_events_match = true; + + Ok(handler.matches(&content, filter)) } } pub(crate) struct CalendarQueryHandler { - default_tz: Tz, - expanded_times: Vec, + pub(super) default_tz: Tz, + pub(super) expanded_times: Vec, + pub(super) undated_todos: Vec, + pub(super) dateless_events_match: bool, } impl CalendarQueryHandler { pub fn new( - event: &ArchivedCalendarEvent, - max_time_range: Option, + event: &ArchivedCalendarEventContent, + expansion_range: Option, default_tz: Tz, ) -> Self { Self { default_tz, - expanded_times: max_time_range - .map(|max_time_range| { + expanded_times: expansion_range + .map(|expansion_range| { event .data - .expand(default_tz, max_time_range) + .expand(default_tz, expansion_range, OverlapRule::CalDav) .unwrap_or_else(|| { trc::event!( Calendar(trc::CalendarEvent::RuleExpansionError), - Reason = "chrono error", - Details = event.data.event.to_string(), + Reason = "Failed to expand stored time ranges", + Contents = event.data.event.to_compact_string(), + ResourceUid = event + .data + .event + .uids() + .next() + .map(|uid| uid.to_compact_string()), ); vec![] }) }) .unwrap_or_default(), + undated_todos: Vec::new(), + dateless_events_match: false, + } + .indexed() + } + + pub fn for_event_data( + content: &CalendarEventContent, + expansion: FilterTimeRanges, + default_tz: Tz, + ) -> Self { + CalendarQueryHandler { + default_tz, + expanded_times: expansion + .expansion_range(content) + .and_then(|range| content.data.expand(default_tz, range, OverlapRule::CalDav)) + .unwrap_or_default(), + undated_todos: content.data.undated_todos().collect(), + dateless_events_match: false, + } + .indexed() + } + + pub fn for_scheduling(default_tz: Tz) -> Self { + CalendarQueryHandler { + default_tz, + expanded_times: Vec::new(), + undated_todos: Vec::new(), + dateless_events_match: true, } } - pub fn filter(&mut self, event: &ArchivedCalendarEvent, filters: &CalendarFilter) -> bool { - let ical = &event.data.event; - let mut is_all = true; - let mut matches_one = false; - - for filter in filters { - match filter { - Filter::AnyOf => { - is_all = false; - } - Filter::AllOf => { - is_all = true; - } - Filter::Property { prop, op, comp } => { - let mut properties = find_components(ical, comp) - .flat_map(|(_, comp)| find_properties(comp, prop)) - .peekable(); - - let result = if properties.peek().is_some() { - properties.any(|entry| { - match op { - FilterOp::Exists => true, - FilterOp::Undefined => false, - FilterOp::TextMatch(text_match) => { - let mut matched_any = false; - - for value in entry.values.iter() { - if let Some(text) = value.as_text() - && text_match.matches(text) - { - matched_any = true; - break; - } - } - - matched_any - } - FilterOp::TimeRange(range) => { - if let Some(ArchivedICalendarValue::PartialDateTime(date)) = - entry.values.first() - { - let tz = entry - .tz_id() - .and_then(|tz_id| Tz::from_str(tz_id).ok()) - .unwrap_or(self.default_tz); - - if let Some(date) = date - .to_date_time() - .and_then(|date| date.to_date_time_with_tz(tz)) - { - let timestamp = date.timestamp(); - // RFC4791#9.9: start <= DTSTART AND end > DTSTART - range.start <= timestamp && range.end > timestamp - } else { - false - } - } else { - false - } - } - } - }) - } else { - matches!(op, FilterOp::Undefined) - }; - - if result { - matches_one = true; - } else if is_all { - return false; - } - } - Filter::Parameter { - prop, - param, - op, - comp, - } => { - let mut parameters = find_components(ical, comp) - .flat_map(|(_, comp)| { - find_properties(comp, prop) - .filter_map(|entry| find_parameter(entry, param)) - }) - .peekable(); - - let result = if parameters.peek().is_some() { - parameters.any(|entry| match op { - FilterOp::Exists => true, - FilterOp::Undefined => false, - FilterOp::TextMatch(text_match) => { - if let Some(text) = entry.value.as_text() { - text_match.matches(text) - } else { - false - } - } - FilterOp::TimeRange(_) => false, - }) - } else { - matches!(op, FilterOp::Undefined) - }; - - if result { - matches_one = true; - } else if is_all { - return false; - } - } - Filter::Component { comp, op } => { - let result = match op { - FilterOp::Exists => find_components(ical, comp).next().is_some(), - FilterOp::Undefined => find_components(ical, comp).next().is_none(), - FilterOp::TimeRange(range) => { - if !matches!(comp.last(), Some(ICalendarComponentType::VAlarm)) { - let matching_comp_ids = find_components(ical, comp) - .map(|(id, comp)| (id as u32, &comp.component_type)) - .collect::>(); - - !matching_comp_ids.is_empty() - && self.expanded_times.iter().any(|event| { - matching_comp_ids.get(&event.comp_id).is_some_and(|ct| { - range.is_in_range( - ct == &&ICalendarComponentType::VTodo, - event.start, - event.end, - ) - }) - }) - } else { - let matching_comp_ids = event - .data - .alarms - .iter() - .map(|alarm| alarm.parent_id.to_native() as u32) - .collect::>(); - - !matching_comp_ids.is_empty() - && self.expanded_times.iter().any(|time| { - matching_comp_ids.contains(&time.comp_id) - && event.data.alarms.iter().any(|alarm| { - alarm.parent_id.to_native() as u32 == time.comp_id - && alarm - .delta - .to_timestamp( - time.start, - time.end, - self.default_tz, - ) - .is_some_and(|timestamp| { - range.is_in_range( - false, timestamp, timestamp, - ) - }) - }) - }) - } - } - FilterOp::TextMatch(_) => false, - }; - - if result { - matches_one = true; - } else if is_all { - return false; - } + pub fn for_content( + event: &ArchivedCalendarEvent, + content: &EventContent<'_>, + expansion: FilterTimeRanges, + default_tz: Tz, + ) -> Self { + let stored = content.stored(); + let expansion_range = match content.view() { + Some(view) => expansion.expansion_range(view), + None => expansion.expansion_range(stored), + }; + let mut handler = Self::new(stored, expansion_range, default_tz); + if event.flags.to_native() & EVENT_HAS_UNBOUNDED_TODO != 0 { + handler.undated_todos.extend(stored.data.undated_todos()); + } + if let Some(merged_overrides) = content + .merged_overrides() + .filter(|merged_overrides| !merged_overrides.is_empty()) + { + for expansion in &mut handler.expanded_times { + if let Some(base_id) = merged_overrides.base_of(expansion.comp_id) { + expansion.comp_id = base_id; } } + handler = handler.indexed(); } - - is_all || matches_one + handler } - pub fn serialize_ical( - &mut self, - event: &ArchivedCalendarEvent, + fn indexed(mut self) -> Self { + self.expanded_times + .sort_unstable_by_key(|expansion| (expansion.comp_id, expansion.start)); + self + } + + pub(super) fn expansions_of(&self, comp_id: u32) -> &[CalendarEventExpansion] { + let from = self + .expanded_times + .partition_point(|expansion| expansion.comp_id < comp_id); + let to = self + .expanded_times + .partition_point(|expansion| expansion.comp_id <= comp_id); + self.expanded_times.get(from..to).unwrap_or_default() + } + + pub fn serialize_content( + &self, + content: &EventContent<'_>, + size: u32, data: &CalendarData, instances_limit: &mut usize, ) -> Option { - let mut out = String::with_capacity(event.size.to_native() as usize); - let _v = [0.into()]; - let mut component_iter: Iter<'_, rkyv::rend::u32_le> = _v.iter(); - let mut component_stack: Vec<(&ArchivedICalendarComponent, Iter<'_, rkyv::rend::u32_le>)> = - Vec::with_capacity(4); - - if data.expand.is_some() { - self.expanded_times.sort_unstable_by_key(|a| a.start); + match content.view() { + Some(view) => self.serialize_ical(view, size, data, instances_limit), + None => self.serialize_ical(content.stored(), size, data, instances_limit), } + } + + pub fn serialize_ical( + &self, + event: &V, + size: u32, + data: &CalendarData, + instances_limit: &mut usize, + ) -> Option { + let mut out = String::with_capacity(size as usize); + let root = [ChildIdOf::::from(0u32)]; + let mut component_iter: Iter<'_, ChildIdOf> = root.iter(); + let mut component_stack: Vec<(&V::Component, Iter<'_, ChildIdOf>)> = + Vec::with_capacity(4); loop { if let Some(component_id) = component_iter.next() { - let component_id = component_id.to_native(); - let component = event - .data - .event - .components - .get(component_id as usize) - .unwrap(); + let component_id = component_id.child_id(); + let Some(component) = + event + .components() + .get(component_id as usize) + .filter(|component| { + data.expand.is_none() + || !component.is_type(ICalendarComponentType::VTimezone) + }) + else { + continue; + }; + let is_todo = component.is_type(ICalendarComponentType::VTodo); // Limit recurrence override if let Some(limit_recurrence) = &data.limit_recurrence && component.is_recurrence_override() - && !self.expanded_times.iter().any(|event| { - event.comp_id == component_id - && limit_recurrence.is_in_range( - component.component_type == ICalendarComponentType::VTodo, - event.start, - event.end, - ) + && !self.expansions_of(component_id).iter().any(|expansion| { + limit_recurrence.is_in_range( + expansion.flags.condition(), + expansion.start, + expansion.end, + ) }) { continue; } // Limit freebusy - if let Some(limit_recurrence) = &data.limit_freebusy - && component.component_type == ICalendarComponentType::VFreebusy - && !self.expanded_times.iter().any(|event| { - event.comp_id == component_id - && limit_recurrence.is_in_range(false, event.start, event.end) + let is_freebusy = component.is_type(ICalendarComponentType::VFreebusy); + if let Some(limit_freebusy) = &data.limit_freebusy + && is_freebusy + && !self.expansions_of(component_id).iter().any(|expansion| { + limit_freebusy.is_in_range( + OverlapCondition::Event, + expansion.start, + expansion.end, + ) }) { continue; } // Filter entries + let is_calendar = component.is_type(ICalendarComponentType::VCalendar); let mut entries = component - .entries + .entries() .iter() .filter_map(|entry| { - if data.properties.is_empty() - || component.component_type == ICalendarComponentType::VCalendar - { + if data.properties.is_empty() || is_calendar { Some((entry, true)) } else { data.properties .iter() .find(|prop| { prop.component.as_ref().is_none_or(|comp| { - comp == &component.component_type - || component_stack.iter().any(|(parent_comp, _)| { - comp == &parent_comp.component_type - }) - }) && prop.name.as_ref().is_none_or(|name| name == &entry.name) + component.component_type() == comp + || component_stack + .iter() + .any(|(parent, _)| parent.component_type() == comp) + }) && prop.name.as_ref().is_none_or(|name| entry.is_named(name)) }) .map(|prop| (entry, !prop.no_value)) } @@ -493,39 +456,68 @@ impl CalendarQueryHandler { .peekable(); // Expand recurrences - let component_name = component.component_type.as_str(); - if let Some(expand) = &data - .expand - .filter(|_| component.component_type.has_time_ranges()) - { + let component_name = component.type_name(); + if let Some(expand) = &data.expand.filter(|_| { + component.has_time_ranges() && !self.undated_todos.contains(&component_id) + }) { let is_recurrent = component.is_recurrent(); let is_recurrent_or_override = is_recurrent || component.is_recurrence_override(); - let is_todo = component.component_type == ICalendarComponentType::VTodo; let mut has_duration = false; let entries = entries - .filter(|(entry, _)| match &entry.name { - ArchivedICalendarProperty::Dtstart - | ArchivedICalendarProperty::Dtend - | ArchivedICalendarProperty::Exdate - | ArchivedICalendarProperty::Exrule - | ArchivedICalendarProperty::Rdate - | ArchivedICalendarProperty::Rrule - | ArchivedICalendarProperty::RecurrenceId => false, - ArchivedICalendarProperty::Due - | ArchivedICalendarProperty::Completed - | ArchivedICalendarProperty::Created => is_recurrent, - ArchivedICalendarProperty::Duration => { + .filter(|(entry, _)| { + let name = entry.name(); + if name == &ICalendarProperty::Duration { has_duration = true; true + } else if is_todo && name == &ICalendarProperty::Due { + false + } else if name == &ICalendarProperty::Due + || name == &ICalendarProperty::Completed + || name == &ICalendarProperty::Created + { + is_recurrent + } else { + !(name == &ICalendarProperty::Dtstart + || name == &ICalendarProperty::Dtend + || name == &ICalendarProperty::Exdate + || name == &ICalendarProperty::Exrule + || name == &ICalendarProperty::Rdate + || name == &ICalendarProperty::Rrule + || name == &ICalendarProperty::RecurrenceId) } - _ => true, }) .collect::>(); - for event in &self.expanded_times { - if event.comp_id == component_id - && (!is_recurrent_or_override - || expand.is_in_range(is_todo, event.start, event.end)) + let end_property = + if has_duration || component.is_type(ICalendarComponentType::VJournal) { + None + } else if is_todo { + Some(ICalendarProperty::Due) + } else { + Some(ICalendarProperty::Dtend) + }; + let mut date_entry = ICalendarEntry { + name: ICalendarProperty::Dtstart, + params: vec![], + values: [ICalendarValue::PartialDateTime(Default::default())].into(), + }; + let mut write_date = + |out: &mut String, name: ICalendarProperty, timestamp: i64| { + date_entry.name = name; + if let Some(ICalendarValue::PartialDateTime(value)) = + date_entry.values.first_mut() + { + *value = PartialDateTime::from_utc_timestamp(timestamp); + } + let _ = date_entry.write_to(out); + }; + for expansion in self.expansions_of(component_id) { + if !is_recurrent_or_override + || expand.is_in_range( + expansion.flags.condition(), + expansion.start, + expansion.end, + ) { if *instances_limit > 0 { *instances_limit -= 1; @@ -535,82 +527,71 @@ impl CalendarQueryHandler { let _ = write!(&mut out, "BEGIN:{component_name}\r\n"); // Write DTSTART, DTEND and RECURRENCE-ID - let mut entry = ICalendarEntry { - name: ICalendarProperty::Dtstart, - params: vec![], - values: vec![ICalendarValue::PartialDateTime(Box::new( - PartialDateTime::from_utc_timestamp(event.start), - ))], - }; - let _ = entry.write_to(&mut out); + write_date(&mut out, ICalendarProperty::Dtstart, expansion.start); if is_recurrent_or_override { - entry.name = ICalendarProperty::RecurrenceId; - let _ = entry.write_to(&mut out); + write_date( + &mut out, + ICalendarProperty::RecurrenceId, + expansion.recurrence_id().utc, + ); } - if !has_duration { - entry.name = ICalendarProperty::Dtend; - entry.values = vec![ICalendarValue::PartialDateTime(Box::new( - PartialDateTime::from_utc_timestamp(event.end), - ))]; - let _ = entry.write_to(&mut out); + if let Some(end_property) = &end_property + && if is_todo { + matches!( + expansion.flags.condition(), + OverlapCondition::TodoStartDue + | OverlapCondition::TodoStartDuration + ) + } else { + expansion.end > expansion.start + } + { + write_date(&mut out, end_property.clone(), expansion.end); } // Write other component entries for (entry, with_value) in &entries { - let _ = entry.write_to(&mut out, *with_value); + entry.write_value(&mut out, *with_value); } let _ = write!(&mut out, "END:{component_name}\r\n"); } } } else if entries.peek().is_some() - || (component.component_type == ICalendarComponentType::VCalendar - && !component.component_ids.is_empty()) + || (is_calendar && !component.child_ids().is_empty()) { let _ = write!(&mut out, "BEGIN:{component_name}\r\n"); match data.limit_freebusy { - Some(range) - if component.component_type == ICalendarComponentType::VFreebusy => - { + Some(range) if is_freebusy => { // Filter freebusy for (entry, with_value) in entries { - if matches!(entry.name, ArchivedICalendarProperty::Freebusy) { - let mut fb_in_range = - freebusy_in_range(entry, &range, self.default_tz) - .peekable(); - if fb_in_range.peek().is_none() { - continue; - } else { - let _ = ICalendarEntry { - name: ICalendarProperty::Freebusy, - params: rkyv_deserialize(&entry.params) - .ok() - .unwrap_or_default(), - values: fb_in_range.collect(), - } - .write_to(&mut out); + if entry.is_named(&ICalendarProperty::Freebusy) { + if let Some(entry) = + entry.freebusy_in_range(&range, self.default_tz) + { + let _ = entry.write_to(&mut out); } } else { - let _ = entry.write_to(&mut out, with_value); + entry.write_value(&mut out, with_value); } } } _ => { for (entry, with_value) in entries { - let _ = entry.write_to(&mut out, with_value); + entry.write_value(&mut out, with_value); } } } - if !component.component_ids.is_empty() { + if !component.child_ids().is_empty() { component_stack.push((component, component_iter)); - component_iter = component.component_ids.iter(); - } else if component.component_ids.is_empty() { + component_iter = component.child_ids().iter(); + } else { let _ = write!(&mut out, "END:{component_name}\r\n"); } } } else if let Some((component, iter)) = component_stack.pop() { - let _ = write!(&mut out, "END:{}\r\n", component.component_type.as_str()); + let _ = write!(&mut out, "END:{}\r\n", component.type_name()); component_iter = iter; } else { break; @@ -619,39 +600,6 @@ impl CalendarQueryHandler { Some(out) } - - pub fn into_expanded_times(self) -> Vec { - self.expanded_times - } } -#[inline(always)] -fn find_components<'x>( - ical: &'x ArchivedICalendar, - comp: &[ICalendarComponentType], -) -> impl Iterator { - // TODO: Properly expand the component type path - let comp = comp.last().unwrap_or(&ICalendarComponentType::VCalendar); - ical.components - .iter() - .enumerate() - .filter(move |(_, entry)| { - comp == &ICalendarComponentType::VCalendar || &entry.component_type == comp - }) -} - -#[inline(always)] -fn find_properties<'x>( - comp: &'x ArchivedICalendarComponent, - prop: &ICalendarProperty, -) -> impl Iterator { - comp.entries.iter().filter(move |entry| &entry.name == prop) -} - -#[inline(always)] -fn find_parameter<'x>( - entry: &'x ArchivedICalendarEntry, - name: &ICalendarParameterName, -) -> Option<&'x ArchivedICalendarParameter> { - entry.params.iter().find(|param| param.name == *name) -} +type ChildIdOf = <::Component as ComponentView>::ChildId; diff --git a/crates/dav/src/calendar/scheduling.rs b/crates/dav/src/calendar/scheduling.rs index ffc950aa..4cbad0a2 100644 --- a/crates/dav/src/calendar/scheduling.rs +++ b/crates/dav/src/calendar/scheduling.rs @@ -6,9 +6,8 @@ use crate::{ DavError, DavErrorCondition, DavMethod, - calendar::freebusy::CalendarFreebusyRequestHandler, + calendar::freebusy::{CalendarFreebusyRequestHandler, FreeBusyScope}, common::{ - ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, @@ -30,17 +29,27 @@ use dav_proto::{ }, }; use groupware::{ - DestroyArchive, cache::GroupwareCache, calendar::CalendarEventNotification, strip_mailto_scheme, + DestroyArchive, + cache::GroupwareCache, + calendar::{ + CalendarEventNotification, CalendarEventNotificationContent, + availability::AvailabilityCalendars, + }, + strip_mailto_scheme, }; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::Permission; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use store::{ahash::AHashMap, write::BatchBuilder}; use trc::AddContext; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + field::CalendarNotificationField, +}; use utils::sanitize_email; pub(crate) trait CalendarEventNotificationHandler: Sync + Send { @@ -61,7 +70,7 @@ pub(crate) trait CalendarEventNotificationHandler: Sync + Send { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], ) -> impl Future> + Send; } @@ -79,7 +88,7 @@ impl CalendarEventNotificationHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::CalendarEventNotification, @@ -105,7 +114,7 @@ impl CalendarEventNotificationHandler for Server { // Fetch event let event_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, resource.document_id(), @@ -118,7 +127,7 @@ impl CalendarEventNotificationHandler for Server { .caused_by(trc::location!())?; // Validate headers - let etag = event_.etag(); + let etag = format!("\"{}\"", event.etag.to_native()); self.validate_headers( access_token, headers, @@ -140,13 +149,30 @@ impl CalendarEventNotificationHandler for Server { .with_etag(etag) .with_last_modified(Rfc1123DateTime::new(i64::from(event.modified)).to_string()); - let ical = event.event.to_string(); - - if !is_head { - Ok(response.with_binary_body(ical)) - } else { - Ok(response.with_content_length(ical.len())) + if is_head { + return Ok(response.with_content_length(event.size.to_native() as usize)); } + + let content_ = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEventNotification, + resource.document_id(), + CalendarNotificationField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let content = content_ + .unarchive::() + .caused_by(trc::location!())?; + + let mut body = String::with_capacity(event.size.to_native() as usize); + if let Some(ical) = content.calendar_data() { + let _ = ical.write_to(&mut body); + } + Ok(response.with_binary_body(body)) } async fn handle_scheduling_delete_request( @@ -165,7 +191,7 @@ impl CalendarEventNotificationHandler for Server { .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::CalendarEventNotification, @@ -189,7 +215,7 @@ impl CalendarEventNotificationHandler for Server { let document_id = resource.document_id(); let event_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, @@ -197,6 +223,9 @@ impl CalendarEventNotificationHandler for Server { .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let event = event_ + .to_unarchived::() + .caused_by(trc::location!())?; // Validate headers self.validate_headers( @@ -206,7 +235,7 @@ impl CalendarEventNotificationHandler for Server { account_id, collection: Collection::CalendarEventNotification, document_id: document_id.into(), - etag: event_.etag().into(), + etag: format!("\"{}\"", event.inner.etag.to_native()).into(), path: delete_path, ..Default::default() }], @@ -215,15 +244,11 @@ impl CalendarEventNotificationHandler for Server { ) .await?; - let event = event_ - .to_unarchived::() - .caused_by(trc::location!())?; - // Delete event let mut batch = BatchBuilder::new(); DestroyArchive(event) .delete( - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, &mut batch, @@ -239,7 +264,7 @@ impl CalendarEventNotificationHandler for Server { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], ) -> crate::Result { // Validate URI let resource = self @@ -257,7 +282,7 @@ impl CalendarEventNotificationHandler for Server { CalCondition::MaxResourceSize(self.core.groupware.max_ical_size as u32), ))); } - let itip_raw = std::str::from_utf8(&bytes).map_err(|_| { + let itip_raw = std::str::from_utf8(bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, @@ -367,63 +392,86 @@ impl CalendarEventNotificationHandler for Server { let mut response = ScheduleResponse::default(); for (email, attendee) in attendees { - if let Some(account_id) = self + let account_id = match self .account_id_from_email(&email, true) .await .caused_by(trc::location!())? { + Some(account_id) + if access_token.is_member(account_id) + || access_token.has_permission(Permission::SysAccountQuery) => + { + Some(account_id) + } + Some(account_id) => self + .try_account(account_id) + .await + .caused_by(trc::location!())? + .filter(|account| account.is_same_tenant(access_token)) + .map(|_| account_id), + None => None, + }; + if let Some(account_id) = account_id { let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await .caused_by(trc::location!())?; - if let Some(resource) = self - .core - .groupware - .default_calendar_name - .as_ref() - .and_then(|name| resources.by_path(name)) - { - let mut free_busy = self - .build_freebusy_object( - access_token, - FreeBusyQuery::new(from_date.timestamp(), to_date.timestamp()), - &resources, - account_id, - resource, - ) - .await?; - - // Add iTIP method - free_busy.components[0].entries.push(ICalendarEntry { - name: ICalendarProperty::Method, - params: vec![], - values: vec![ICalendarValue::Method(ICalendarMethod::Reply)], - }); - - // Add properties - let component = &mut free_busy.components[1]; - component.entries.push(organizer.clone()); - component.entries.push(attendee.clone()); - if let Some(uid) = uid { - component.entries.push(uid.clone()); - } - + let Some(calendars) = + AvailabilityCalendars::owner(&resources, account_id, access_token) + else { response.items.0.push(ScheduleResponseItem { recipient: Href(format!("mailto:{email}")), - request_status: "2.0;Success".into(), - calendar_data: Some(free_busy.to_string()), - }); - } else { - response.items.0.push(ScheduleResponseItem { - recipient: Href(format!("mailto:{email}")), - request_status: "3.7;Default calendar not found".into(), + request_status: "3.8;No authority".into(), calendar_data: None, }); + continue; + }; + let mut free_busy = match self + .build_freebusy_object( + access_token, + FreeBusyQuery::new(from_date.timestamp(), to_date.timestamp()), + &resources, + account_id, + FreeBusyScope::Account(calendars), + ) + .await + { + Ok(free_busy) => free_busy, + Err(DavError::Code(code)) if code == StatusCode::PAYLOAD_TOO_LARGE => { + response.items.0.push(ScheduleResponseItem { + recipient: Href(format!("mailto:{email}")), + request_status: "3.10;Request entity too large".into(), + calendar_data: None, + }); + continue; + } + Err(err) => return Err(err), + }; + + // Add iTIP method + free_busy.components[0].entries.push(ICalendarEntry { + name: ICalendarProperty::Method, + params: vec![], + values: [ICalendarValue::Method(ICalendarMethod::Reply)].into(), + }); + + // Add properties + let component = &mut free_busy.components[1]; + component.entries.push(organizer.clone()); + component.entries.push(attendee.clone()); + if let Some(uid) = uid { + component.entries.push(uid.clone()); } + + response.items.0.push(ScheduleResponseItem { + recipient: Href(format!("mailto:{email}")), + request_status: "2.0;Success".into(), + calendar_data: Some(free_busy.to_string()), + }); } else { response.items.0.push(ScheduleResponseItem { recipient: Href(format!("mailto:{email}")), diff --git a/crates/dav/src/calendar/update.rs b/crates/dav/src/calendar/update.rs index e6a6dd85..3c401b85 100644 --- a/crates/dav/src/calendar/update.rs +++ b/crates/dav/src/calendar/update.rs @@ -4,56 +4,67 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::assert_is_unique_uid; +use super::{ + CalendarComponentSupport, CalendarEventView, assert_event_privacy_access, + assert_event_privacy_allowed, assert_is_unique_uid, serves_ical, user_data_error, +}; use crate::{ DavError, DavErrorCondition, DavMethod, calendar::ItipPrecondition, common::{ - ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, - fix_percent_encoding, }; use calcard::{ Entry, Parser, common::timezone::Tz, icalendar::{ICalendar, ICalendarComponentType}, }; -use common::{DavName, Server, auth::AccessToken}; +use common::{ArchivedDavName, DavName, Server, auth::AccessToken}; +use compact_str::ToCompactString; use dav_proto::{ RequestHeaders, Return, schema::{property::Rfc1123DateTime, response::CalCondition}, }; use groupware::{ cache::GroupwareCache, - calendar::{CalendarEvent, CalendarEventData, itip::ItipSendStatus}, + calendar::{ + CalendarEvent, CalendarEventContent, CalendarEventData, SupportedComponent, + alerts::{DefaultAlertsResolver, DefaultAlertsView, ICalendarDefaultAlerts}, + identity::ParticipantIdentityAddresses, + itip::ItipSendStatus, + notification::{NotificationAudience, may_have_viewers}, + privacy::EventViewer, + rights::{EventAcl, EventChanges}, + schedule::{AlarmTasks, EventAlarmScheduler, EventAlarmUsers}, + storage::{DirectChange, DirectChangeNotification, NotificationQuota}, + user::{ICalendarUserData, UpdatedPolicy, UserDataSplit, UserDataUpdate, UserDataView}, + }, scheduling::{ ItipMessages, event_create::itip_create, event_update::itip_update, - itip::itip_set_unreachable_status, + itip::itip_set_unreachable_status, recipient::RecipientPolicy, }, }; use http_proto::HttpResponse; use hyper::StatusCode; -use std::collections::HashSet; +use registry::schema::enums::StorageQuota; use store::write::{BatchBuilder, now}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::CalendarEventField, }; +use utils::map::bitmap::Bitmap; pub(crate) trait CalendarUpdateRequestHandler: Sync + Send { fn handle_calendar_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], is_patch: bool, ) -> impl Future> + Send; } @@ -63,7 +74,7 @@ impl CalendarUpdateRequestHandler for Server { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], _is_patch: bool, ) -> crate::Result { // Validate URI @@ -73,18 +84,16 @@ impl CalendarUpdateRequestHandler for Server { .into_owned_uri()?; let account_id = resource.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await .caused_by(trc::location!())?; - let resource_name = fix_percent_encoding( - resource - .resource - .ok_or(DavError::Code(StatusCode::CONFLICT))?, - ); + let resource_name = resource + .resource + .ok_or(DavError::Code(StatusCode::CONFLICT))?; if bytes.len() > self.core.groupware.max_ical_size { return Err(DavError::Condition(DavErrorCondition::new( @@ -92,7 +101,7 @@ impl CalendarUpdateRequestHandler for Server { CalCondition::MaxResourceSize(self.core.groupware.max_ical_size as u32), ))); } - let ical_raw = std::str::from_utf8(&bytes).map_err(|_| { + let ical_raw = std::str::from_utf8(bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, @@ -102,7 +111,7 @@ impl CalendarUpdateRequestHandler for Server { ) })?; - let ical = match Parser::new(ical_raw).entry() { + let mut ical = match Parser::new(ical_raw).entry() { Entry::ICalendar(ical) => ical, _ => { return Err(DavError::Condition( @@ -114,13 +123,29 @@ impl CalendarUpdateRequestHandler for Server { )); } }; + let attachments_size = ical.embedded_size(); + if self.core.groupware.max_attachments_size != 0 + && attachments_size > self.core.groupware.max_attachments_size + { + return Err(DavError::Condition( + DavErrorCondition::new( + StatusCode::PRECONDITION_FAILED, + CalCondition::MaxResourceSize(self.core.groupware.max_attachments_size as u32), + ) + .with_details(format!( + "The size of the embedded attachments ({attachments_size} bytes) exceeds the maximum of {} bytes.", + self.core.groupware.max_attachments_size + )), + )); + } + let personal_id = access_token.personal_id(account_id, Collection::Calendar); let account_info = self - .scheduling_account_info(access_token.account_id(), account_id) + .account_info(account_id) .await .caused_by(trc::location!())?; - if let Some(resource) = resources.by_path(resource_name.as_ref()) { + if let Some(resource) = resources.by_path(resource_name) { if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } @@ -128,26 +153,43 @@ impl CalendarUpdateRequestHandler for Server { // Validate ACL let parent_id = resource.parent_id().unwrap(); let document_id = resource.document_id(); - if !access_token.is_member(account_id) - && !resources.has_access_to_container(access_token, parent_id, Acl::ModifyItems) + let is_owner = access_token.is_member(account_id); + if !is_owner + && !resources.has_access_to_container( + access_token, + parent_id, + Bitmap::from_iter([ + Acl::ModifyItems, + Acl::ModifyItemsOwn, + Acl::ModifyPrivateProperties, + Acl::ModifyRSVP, + ]), + ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } + let viewer = EventViewer::new(is_owner); + assert_event_privacy_access(resource.resource.event_flags(), viewer)?; + assert_event_privacy_allowed(&ical, viewer)?; // Update - let event_ = self - .store() - .get_value::>(ValueKey::archive( + let (event_, content_) = self + .archive_and_field( account_id, Collection::CalendarEvent, document_id, - )) + CalendarEventField::Content.field(), + ) .await - .caused_by(trc::location!())? - .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + .caused_by(trc::location!())?; + let event_ = event_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; + let content_ = content_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let content = content_ + .to_unarchived::() + .caused_by(trc::location!())?; // Validate headers match self @@ -158,8 +200,8 @@ impl CalendarUpdateRequestHandler for Server { account_id, collection: Collection::CalendarEvent, document_id: Some(document_id), - etag: event.etag().into(), - path: resource_name.as_ref(), + etag: format!("\"{}\"", event.inner.etag.to_native()).into(), + path: resource_name, ..Default::default() }], Default::default(), @@ -171,30 +213,47 @@ impl CalendarUpdateRequestHandler for Server { Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { + let body = match self + .calendar_event_view( + access_token, + account_id, + event.inner, + content.inner, + &mut DefaultAlertsResolver::default(), + ) + .await? + { + Some((view, _)) => view.data.event.to_string(), + None => content.inner.data.event.to_string(), + }; return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_content_type("text/calendar; charset=utf-8") - .with_etag(event.etag()) + .with_etag(format!("\"{}\"", event.inner.etag.to_native())) .with_last_modified( Rfc1123DateTime::new(i64::from(event.inner.modified)).to_string(), ) .with_header("Preference-Applied", "return=representation") - .with_binary_body(event.inner.data.event.to_string())); + .with_binary_body(body)); } Err(e) => return Err(e), } - if ical == event.inner.data.event { + if ical == content.inner.data.event { // No changes, return existing event return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } // Validate iCal - if event.inner.data.event.uids().next().unwrap_or_default() != validate_ical(&ical)? { + let (uid, component) = validate_ical(&ical)?; + if event.inner.uid.as_str() != uid { return Err(DavError::Condition(DavErrorCondition::new( - StatusCode::PRECONDITION_FAILED, + StatusCode::FORBIDDEN, CalCondition::NoUidConflict(resources.format_resource(resource).into()), ))); } + self.supported_components(account_id, parent_id) + .await? + .assert_supports(component)?; // Validate schedule tag if headers.if_schedule_tag.is_some() @@ -206,39 +265,190 @@ impl CalendarUpdateRequestHandler for Server { // Obtain previous alarm let now = now() as i64; - let prev_email_alarm = event.inner.data.next_alarm(now, Tz::Floating); + let calendar_ids = event + .inner + .names + .iter() + .map(ArchivedDavName::parent_id) + .collect::>(); + let mut default_alerts_resolver = + DefaultAlertsResolver::with_resources(account_id, resources.clone()); + let prev_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, content.inner)? + .with_event_flags(event.inner.flags.to_native()), + &content.inner.data, + &calendar_ids, + now, + &mut default_alerts_resolver, + ) + .await + .caused_by(trc::location!())?; // Build event - let mut next_email_alarm = None; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; - let old_ical = new_event.data.event; - new_event.size = bytes.len() as u32; - new_event.data = CalendarEventData::new( - ical, - Tz::Floating, - self.core.groupware.max_ical_instances, - &mut next_email_alarm, - ); + let mut new_content = content + .deserialize::() + .caused_by(trc::location!())?; + + let writer_default_alerts = default_alerts_resolver + .resolve_for_content( + self, + account_id, + personal_id, + &new_content, + event.inner.names.iter().map(ArchivedDavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + let has_default_alerts = writer_default_alerts.is_enabled(); + let submitted_ical = ical.clone(); + ical.strip_default_alerts(&writer_default_alerts); + let sharee_alerts_view = |content: &CalendarEventContent| { + let mut view = content.data.event.clone(); + view.apply_user_data(content.preferences(personal_id), UserDataView::AlertsOnly); + view + }; + let mut sharee_view = None; + if has_default_alerts { + if is_owner { + ical.restore_hidden_alerts(&new_content.data.event, &writer_default_alerts); + } else { + ical.restore_hidden_alerts( + sharee_view.insert(sharee_alerts_view(&new_content)), + &writer_default_alerts, + ); + } + } + let changes_shared_user_data = + !is_owner && !ical.has_same_user_data(&new_content.data.event); + if is_owner || changes_shared_user_data { + ical.validate_user_data_changes(&new_content.data.event) + .map_err(user_data_error)?; + } + + if !is_owner { + let identities = self + .account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())?; + let ownership = identities.event_ownership(&new_content.data.event); + let acl = EventAcl::for_calendars( + &resources, + access_token, + event.inner.names.iter().map(ArchivedDavName::parent_id), + ); + if !acl.may_write(ownership) { + if changes_shared_user_data { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + let event_flags = event.inner.flags.to_native(); + let sharee_view = + sharee_view.unwrap_or_else(|| sharee_alerts_view(&new_content)); + let mut changes = EventChanges::between( + &sharee_view.into_jscalendar(), + &ical.clone().into_jscalendar(), + event_flags, + event_flags, + &identities, + ); + changes.classify_recurrence_sets(&new_content.data.event, &ical); + changes + .assert_allowed(&acl, ownership, event_flags) + .map_err(|_| DavError::Code(StatusCode::FORBIDDEN))?; + } + } + + let is_user_data_only = if is_owner { + false + } else { + let stored_preferences = new_content + .preferences + .iter() + .find(|p| p.account_id == personal_id); + let (preferences, split) = ical + .split_user_data( + &new_content.data.event, + UserDataUpdate { + account_id: personal_id, + view: UserDataView::AlertsOnly, + previous: stored_preferences, + current: stored_preferences, + updated: UpdatedPolicy::Server(now), + }, + ) + .map_err(user_data_error)?; + new_content.set_preferences(preferences); + split == UserDataSplit::UserDataOnly + }; + let old_ical = if is_user_data_only { + None + } else { + let old_ical = std::mem::take(&mut new_content.data.event); + let owner_default_alerts = default_alerts_resolver + .resolve_for_content( + self, + account_id, + account_id, + &new_content, + event.inner.names.iter().map(ArchivedDavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + new_content.data = CalendarEventData::new_with_default_alerts( + ical, + Tz::Floating, + self.core.groupware.max_ical_instances, + &owner_default_alerts, + ); + Some(old_ical) + }; + let next_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &new_content)? + .with_event_flags(new_event.flags), + &new_content.data, + &calendar_ids, + now, + &mut default_alerts_resolver, + ) + .await + .caused_by(trc::location!())?; // Scheduling let mut itip_messages = None; - let itip_status = ItipSendStatus::resolve( - self, - access_token, - &account_info, - new_event.data.event_range_end(), - ); + let itip_status = if is_user_data_only { + ItipSendStatus::NotRequested + } else { + ItipSendStatus::resolve( + self, + access_token, + &account_info, + new_content.data.event_range_end(), + ) + }; if itip_status.is_send() { - let result = if new_event.schedule_tag.is_some() { + let policy = RecipientPolicy::new(&self.core.groupware, new_event.flags); + let result = if let Some(old_ical) = old_ical + .as_ref() + .filter(|_| new_event.schedule_tag.is_some()) + { itip_update( - &mut new_event.data.event, - &old_ical, + &mut new_content.data.event, + old_ical, account_info.addresses(), + policy, ) } else { - itip_create(&mut new_event.data.event, account_info.addresses()) + itip_create( + &mut new_content.data.event, + account_info.addresses(), + policy, + ) }; match result { @@ -285,7 +495,7 @@ impl CalendarUpdateRequestHandler for Server { Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, DocumentId = document_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); // Event changed, but there are no iTIP messages to send @@ -295,7 +505,7 @@ impl CalendarUpdateRequestHandler for Server { } } - itip_set_unreachable_status(&mut new_event.data.event, account_info.addresses()); + itip_set_unreachable_status(&mut new_content.data.event, account_info.addresses()); } else if let Some(reason) = itip_status.reason() { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), @@ -305,61 +515,122 @@ impl CalendarUpdateRequestHandler for Server { ); } - // Validate quota - let extra_bytes = - (bytes.len() as u64).saturating_sub(u32::from(event.inner.size) as u64); - if extra_bytes > 0 { - self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) - .await?; - } + let account = self.account(account_id).await?; // Prepare write batch let mut batch = BatchBuilder::new(); - let schedule_tag = new_event.schedule_tag; - let etag = new_event - .update( - access_token.account_tenant_ids(), - event, + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); + let notify_event_flags = new_event.flags | event.inner.flags.to_native(); + if let Some(old_ical) = old_ical.filter(|_| { + may_have_viewers( + access_token, account_id, - document_id, + &resources, + &calendar_ids, + NotificationAudience::of_event(notify_event_flags), + ) + }) { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Updated { + event_id: document_id, + previous: old_ical, + current: new_content.data.event.clone(), + calendar_ids: calendar_ids.clone(), + event_flags: notify_event_flags, + }, + Some(&resources), + &mut NotificationQuota::default(), &mut batch, ) - .caused_by(trc::location!())? - .etag(); - if prev_email_alarm != next_email_alarm { - if let Some(prev_alarm) = prev_email_alarm { - prev_alarm.delete_task(&mut batch); - } - if let Some(next_alarm) = next_email_alarm { - next_alarm.write_task(&mut batch); - } + .await + .caused_by(trc::location!())?; } + let serves_submitted_ical = if is_owner && !has_default_alerts { + serves_ical(&new_content.data.event, &submitted_ical) + } else { + let view_default_alerts = default_alerts_resolver + .resolve_for_ical_view( + self, + account_id, + personal_id, + &new_content, + calendar_ids.iter().copied(), + ) + .await + .caused_by(trc::location!())?; + let mut view = new_content.data.event.clone(); + if !is_owner { + view.apply_user_data( + new_content.preferences(personal_id), + UserDataView::AlertsOnly, + ); + } + view.apply_default_alerts(&view_default_alerts, DefaultAlertsView::ICalendar); + serves_ical(&view, &submitted_ical) + }; + + let schedule_tag = new_event.schedule_tag; + let etag = new_event + .update_full( + new_content, + access_token.changed_by(), + event, + content.inner, + account_id, + document_id, + None, + &mut batch, + ) + .caused_by(trc::location!())?; + self.replace_event_alarms( + account_id, + document_id, + prev_email_alarms, + next_email_alarms, + &mut AlarmTasks::None, + &mut batch, + ) + .await + .caused_by(trc::location!())?; if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT) - .with_etag_opt(etag) + .with_etag_opt(serves_submitted_ical.then_some(etag)) .with_schedule_tag_opt(schedule_tag)) - } else if let Some((Some(parent), name)) = resources.map_parent(resource_name.as_ref()) { + } else if let Some((Some(parent), name)) = resources.map_parent(resource_name) { if !parent.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL - if !access_token.is_member(account_id) + let is_owner = access_token.is_member(account_id); + if !is_owner && !resources.has_access_to_container( access_token, parent.document_id(), Acl::AddItems, ) + && !(EventAcl::for_calendar(&resources, access_token, parent.document_id()) + .may_manage_own_items() + && self + .account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())? + .event_ownership(&ical) + .may_write_own()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } + assert_event_privacy_allowed(&ical, EventViewer::new(is_owner))?; + ical.validate_user_data().map_err(user_data_error)?; // Validate headers self.validate_headers( @@ -369,7 +640,7 @@ impl CalendarUpdateRequestHandler for Server { account_id, collection: resource.collection, document_id: Some(u32::MAX), - path: resource_name.as_ref(), + path: resource_name, ..Default::default() }], Default::default(), @@ -378,31 +649,63 @@ impl CalendarUpdateRequestHandler for Server { .await?; // Validate ical object + let (uid, component) = validate_ical(&ical)?; assert_is_unique_uid( - self, &resources, + access_token, account_id, parent.document_id(), - validate_ical(&ical)?.into(), - ) - .await?; + Some(uid), + )?; + self.supported_components(account_id, parent.document_id()) + .await? + .assert_supports(component)?; + + // Validate object quota + let account = self.account(account_id).await?; + self.assert_object_quota(&account, StorageQuota::MaxCalendarEvents, 1, || { + resources.resources.count(false) + })?; + + let submitted_ical = ical.clone(); + let mut preferences = Vec::new(); + if !is_owner { + let extracted = ical + .extract_user_data(personal_id) + .map_err(user_data_error)?; + ical.apply_user_data(None, UserDataView::Full); + if !extracted.is_empty() { + preferences.push(extracted); + } + } // Build event - let mut next_email_alarm = None; let mut event = CalendarEvent { names: vec![DavName { name: name.to_string(), parent_id: parent.document_id(), }], + ..Default::default() + }; + let mut content = CalendarEventContent { data: CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, - &mut next_email_alarm, ), - size: bytes.len() as u32, - ..Default::default() + preferences, }; + let next_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &content)?, + &content.data, + &[parent.document_id()], + now() as i64, + &mut DefaultAlertsResolver::with_resources(account_id, resources.clone()), + ) + .await + .caused_by(trc::location!())?; // Scheduling let mut itip_messages = None; @@ -410,10 +713,14 @@ impl CalendarUpdateRequestHandler for Server { self, access_token, &account_info, - event.data.event_range_end(), + content.data.event_range_end(), ); if itip_status.is_send() { - match itip_create(&mut event.data.event, account_info.addresses()) { + match itip_create( + &mut content.data.event, + account_info.addresses(), + RecipientPolicy::new(&self.core.groupware, event.flags), + ) { Ok(messages) => { if messages.iter().map(|r| r.to.len()).sum::() < self.core.groupware.itip_outbound_max_recipients @@ -441,12 +748,12 @@ impl CalendarUpdateRequestHandler for Server { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } - itip_set_unreachable_status(&mut event.data.event, account_info.addresses()); + itip_set_unreachable_status(&mut content.data.event, account_info.addresses()); } else if let Some(reason) = itip_status.reason() { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), @@ -455,43 +762,65 @@ impl CalendarUpdateRequestHandler for Server { ); } - // Validate quota - if !bytes.is_empty() { - self.has_available_quota( - self.account(account_id).await?.as_ref(), - bytes.len() as u64, - ) - .await?; - } // Prepare write batch let mut batch = BatchBuilder::new(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::CalendarEvent, 1) + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); + let document_id = batch.reserve_document_id(account_id, Collection::CalendarEvent); + let notify_calendar_ids = event.calendar_ids().collect::>(); + if may_have_viewers( + access_token, + account_id, + &resources, + ¬ify_calendar_ids, + NotificationAudience::of_event(event.flags), + ) { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Created { + event_id: document_id.into(), + current: content.data.event.clone(), + calendar_ids: notify_calendar_ids, + event_flags: event.flags, + }, + Some(&resources), + &mut NotificationQuota::default(), + &mut batch, + ) .await .caused_by(trc::location!())?; + } + let serves_submitted_ical = if is_owner { + serves_ical(&content.data.event, &submitted_ical) + } else { + let mut view = content.data.event.clone(); + view.apply_user_data(content.preferences(personal_id), UserDataView::AlertsOnly); + serves_ical(&view, &submitted_ical) + }; + let schedule_tag = event.schedule_tag; let etag = event .insert( - access_token.account_tenant_ids(), + content, + access_token.changed_by(), account_id, document_id, - next_email_alarm, + None, + next_email_alarms, &mut batch, ) - .caused_by(trc::location!())? - .etag(); + .caused_by(trc::location!())?; if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED) - .with_etag_opt(etag) + .with_etag_opt(serves_submitted_ical.then_some(etag)) .with_schedule_tag_opt(schedule_tag)) } else { Err(DavError::Code(StatusCode::CONFLICT))? @@ -499,38 +828,41 @@ impl CalendarUpdateRequestHandler for Server { } } -fn validate_ical(ical: &ICalendar) -> crate::Result<&str> { - // Validate UIDs - let mut uids = HashSet::with_capacity(1); - - // Validate component types - let mut types: [u8; 5] = [0; 5]; +fn validate_ical(ical: &ICalendar) -> crate::Result<(&str, SupportedComponent)> { + let mut uid = None; + let mut has_mixed_uids = false; + let mut object_type = None; + let mut has_mixed_types = false; for comp in &ical.components { - *(match comp.component_type { - ICalendarComponentType::VEvent => &mut types[0], - ICalendarComponentType::VTodo => &mut types[1], - ICalendarComponentType::VJournal => &mut types[2], - ICalendarComponentType::VFreebusy => &mut types[3], - ICalendarComponentType::VAvailability => &mut types[4], + let component = match comp.component_type { + ICalendarComponentType::VEvent => SupportedComponent::VEvent, + ICalendarComponentType::VTodo => SupportedComponent::VTodo, + ICalendarComponentType::VJournal => SupportedComponent::VJournal, + ICalendarComponentType::VFreebusy => SupportedComponent::VFreebusy, + ICalendarComponentType::VAvailability => SupportedComponent::VAvailability, _ => { continue; } - }) += 1; + }; + has_mixed_types |= object_type + .replace(component) + .is_some_and(|previous| previous != component); - if let Some(uid) = comp.uid() { - uids.insert(uid); + if let Some(comp_uid) = comp.uid() { + has_mixed_uids |= uid + .replace(comp_uid) + .is_some_and(|previous| previous != comp_uid); } } - if uids.len() == 1 && types.iter().filter(|&&v| v == 0).count() == 4 { - Ok(uids.iter().next().unwrap()) - } else { - Err(DavError::Condition( + match (uid.filter(|_| !has_mixed_uids), object_type) { + (Some(uid), Some(object_type)) if !has_mixed_types => Ok((uid, object_type)), + _ => Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::ValidCalendarObjectResource, ) .with_details("iCalendar must contain exactly one UID and same component types"), - )) + )), } } diff --git a/crates/dav/src/calendar/view.rs b/crates/dav/src/calendar/view.rs new file mode 100644 index 00000000..120a7745 --- /dev/null +++ b/crates/dav/src/calendar/view.rs @@ -0,0 +1,550 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::freebusy::freebusy_in_range; +use calcard::{ + common::timezone::Tz, + icalendar::{ + ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarComponentType, + ArchivedICalendarEntry, ArchivedICalendarParameter, ArchivedICalendarParameterName, + ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarValue, + ArchivedICalendarValueType, ICalendarComponent, ICalendarComponentType, ICalendarEntry, + ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, + ICalendarValue, ICalendarValueType, + }, +}; +use groupware::calendar::{ + Alarm, AlarmDelta, ArchivedAlarm, ArchivedCalendarEventContent, CalendarEventContent, + expand::CalendarEventExpansion, +}; +use rkyv::primitive::ArchivedU32; +use std::borrow::Cow; +use store::write::serialize::rkyv_deserialize; +use types::TimeRange; + +const SECONDS_PER_DAY: i64 = 86_400; + +pub(crate) trait CalendarView { + type Component: ComponentView; + type Alarm: AlarmView; + + fn components(&self) -> &[Self::Component]; + fn alarms(&self) -> &[Self::Alarm]; +} + +pub(crate) trait ComponentView { + type Type: PartialEq; + type Entry: EntryView; + type ChildId: ChildIdView; + + fn component_type(&self) -> &Self::Type; + fn type_name(&self) -> &str; + fn has_time_ranges(&self) -> bool; + fn is_recurrent(&self) -> bool; + fn is_recurrence_override(&self) -> bool; + fn entries(&self) -> &[Self::Entry]; + fn child_ids(&self) -> &[Self::ChildId]; + + fn is_type(&self, component_type: ICalendarComponentType) -> bool { + self.component_type() == &component_type + } + + fn effective_end(&self, property: &ICalendarProperty, default_tz: Tz) -> Option { + let applies = match property { + ICalendarProperty::Dtend => self.is_type(ICalendarComponentType::VEvent), + ICalendarProperty::Due => self.is_type(ICalendarComponentType::VTodo), + _ => false, + }; + if !applies { + return None; + } + let mut start = None; + let mut duration = None; + for entry in self.entries() { + if entry.is_named(&ICalendarProperty::Dtstart) { + start = entry.date_time_timestamp(default_tz); + } else if entry.is_named(&ICalendarProperty::Duration) { + duration = entry.duration_seconds(); + } + } + start?.checked_add(duration?) + } + + fn alarm_repetition(&self) -> AlarmRepetition { + let mut count = None; + let mut interval = None; + for entry in self.entries() { + if entry.is_named(&ICalendarProperty::Repeat) { + count = entry.integer_value(); + } else if entry.is_named(&ICalendarProperty::Duration) { + interval = entry.duration_seconds(); + } + } + AlarmRepetition::new(count, interval) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AlarmRepetition { + count: u64, + interval: u64, +} + +pub(crate) trait EntryView { + type Name: PartialEq; + type Param: ParameterView; + + fn name(&self) -> &Self::Name; + fn params(&self) -> &[Self::Param]; + fn text_values(&self) -> impl Iterator>; + fn date_time_timestamp(&self, default_tz: Tz) -> Option; + fn integer_value(&self) -> Option; + fn duration_seconds(&self) -> Option; + fn freebusy_in_range(&self, range: &TimeRange, default_tz: Tz) -> Option; + fn write_value(&self, out: &mut String, with_value: bool); + + fn is_named(&self, property: &ICalendarProperty) -> bool { + self.name() == property + } +} + +pub(crate) trait ParameterView { + type Name: PartialEq; + + fn name(&self) -> &Self::Name; + fn text(&self) -> Option>; + + fn is_named(&self, name: &ICalendarParameterName) -> bool { + self.name() == name + } +} + +pub(crate) trait AlarmView { + fn id(&self) -> u32; + fn parent_id(&self) -> u32; + fn delta(&self) -> AlarmDelta; + fn timestamp(&self, expansion: &CalendarEventExpansion, default_tz: Tz) -> Option; + + fn trigger_offset(&self) -> Option { + match self.delta() { + AlarmDelta::Start(offset) | AlarmDelta::End(offset) => Some( + i64::from(offset.days.unsigned_abs()) * SECONDS_PER_DAY + + i64::from(offset.seconds.unsigned_abs()), + ), + AlarmDelta::FixedUtc(_) | AlarmDelta::FixedFloating(_) => None, + } + } +} + +pub(crate) trait ChildIdView: Sized + From { + fn child_id(&self) -> u32; +} + +impl ChildIdView for u32 { + fn child_id(&self) -> u32 { + *self + } +} + +impl ChildIdView for ArchivedU32 { + fn child_id(&self) -> u32 { + self.to_native() + } +} + +impl CalendarView for CalendarEventContent { + type Component = ICalendarComponent; + type Alarm = Alarm; + + fn components(&self) -> &[Self::Component] { + &self.data.event.components + } + + fn alarms(&self) -> &[Self::Alarm] { + &self.data.alarms + } +} + +impl CalendarView for ArchivedCalendarEventContent { + type Component = ArchivedICalendarComponent; + type Alarm = ArchivedAlarm; + + fn components(&self) -> &[Self::Component] { + &self.data.event.components + } + + fn alarms(&self) -> &[Self::Alarm] { + &self.data.alarms + } +} + +impl CalendarView for ArchivedICalendar { + type Component = ArchivedICalendarComponent; + type Alarm = ArchivedAlarm; + + fn components(&self) -> &[Self::Component] { + &self.components + } + + fn alarms(&self) -> &[Self::Alarm] { + &[] + } +} + +impl ComponentView for ICalendarComponent { + type Type = ICalendarComponentType; + type Entry = ICalendarEntry; + type ChildId = u32; + + fn component_type(&self) -> &Self::Type { + &self.component_type + } + + fn type_name(&self) -> &str { + self.component_type.as_str() + } + + fn has_time_ranges(&self) -> bool { + self.component_type.has_time_ranges() + } + + fn is_recurrent(&self) -> bool { + ICalendarComponent::is_recurrent(self) + } + + fn is_recurrence_override(&self) -> bool { + ICalendarComponent::is_recurrence_override(self) + } + + fn entries(&self) -> &[Self::Entry] { + &self.entries + } + + fn child_ids(&self) -> &[Self::ChildId] { + &self.component_ids + } +} + +impl ComponentView for ArchivedICalendarComponent { + type Type = ArchivedICalendarComponentType; + type Entry = ArchivedICalendarEntry; + type ChildId = ArchivedU32; + + fn component_type(&self) -> &Self::Type { + &self.component_type + } + + fn type_name(&self) -> &str { + self.component_type.as_str() + } + + fn has_time_ranges(&self) -> bool { + self.component_type.has_time_ranges() + } + + fn is_recurrent(&self) -> bool { + ArchivedICalendarComponent::is_recurrent(self) + } + + fn is_recurrence_override(&self) -> bool { + ArchivedICalendarComponent::is_recurrence_override(self) + } + + fn entries(&self) -> &[Self::Entry] { + &self.entries + } + + fn child_ids(&self) -> &[Self::ChildId] { + &self.component_ids + } +} + +impl EntryView for ICalendarEntry { + type Name = ICalendarProperty; + type Param = ICalendarParameter; + + fn name(&self) -> &Self::Name { + &self.name + } + + fn params(&self) -> &[Self::Param] { + &self.params + } + + fn text_values(&self) -> impl Iterator> { + self.values.iter().filter_map(ValueText::value_text) + } + + fn date_time_timestamp(&self, default_tz: Tz) -> Option { + self.values + .first() + .and_then(|value| value.as_partial_date_time()) + .and_then(|date| date.to_date_time()) + .and_then(|date| date.to_date_time_with_tz(entry_tz(self.tz_id(), default_tz))) + .map(|date| date.timestamp()) + } + + fn integer_value(&self) -> Option { + self.values.first().and_then(ICalendarValue::as_integer) + } + + fn duration_seconds(&self) -> Option { + match self.values.first() { + Some(ICalendarValue::Duration(duration)) => Some(duration.as_seconds()), + _ => None, + } + } + + fn freebusy_in_range(&self, range: &TimeRange, default_tz: Tz) -> Option { + let tz = entry_tz(self.tz_id(), default_tz); + let values = self + .values + .iter() + .filter(|value| { + matches!(value, ICalendarValue::Period(period) + if period.time_range(tz).is_some_and(|(start, end)| { + range.overlaps(start.timestamp(), end.timestamp()) + })) + }) + .cloned() + .collect::>(); + + (!values.is_empty()).then(|| ICalendarEntry { + name: ICalendarProperty::Freebusy, + params: self.params.clone(), + values: values.into(), + }) + } + + fn write_value(&self, out: &mut String, with_value: bool) { + let _ = self.write_with_value(out, with_value); + } +} + +impl EntryView for ArchivedICalendarEntry { + type Name = ArchivedICalendarProperty; + type Param = ArchivedICalendarParameter; + + fn name(&self) -> &Self::Name { + &self.name + } + + fn params(&self) -> &[Self::Param] { + &self.params + } + + fn text_values(&self) -> impl Iterator> { + self.values.iter().filter_map(ValueText::value_text) + } + + fn date_time_timestamp(&self, default_tz: Tz) -> Option { + self.values + .first() + .and_then(|value| value.as_partial_date_time()) + .and_then(|date| date.to_date_time()) + .and_then(|date| date.to_date_time_with_tz(entry_tz(self.tz_id(), default_tz))) + .map(|date| date.timestamp()) + } + + fn integer_value(&self) -> Option { + self.values + .first() + .and_then(ArchivedICalendarValue::as_integer) + } + + fn duration_seconds(&self) -> Option { + match self.values.first() { + Some(ArchivedICalendarValue::Duration(duration)) => Some(duration.as_seconds()), + _ => None, + } + } + + fn freebusy_in_range(&self, range: &TimeRange, default_tz: Tz) -> Option { + let mut values = freebusy_in_range(self, range, default_tz).peekable(); + values.peek().is_some().then(|| ICalendarEntry { + name: ICalendarProperty::Freebusy, + params: rkyv_deserialize(&self.params).ok().unwrap_or_default(), + values: values.collect(), + }) + } + + fn write_value(&self, out: &mut String, with_value: bool) { + let _ = self.write_to(out, with_value); + } +} + +impl ParameterView for ICalendarParameter { + type Name = ICalendarParameterName; + + fn name(&self) -> &Self::Name { + &self.name + } + + fn text(&self) -> Option> { + match &self.value { + ICalendarParameterValue::Integer(value) => Some(Cow::Owned(value.to_string())), + ICalendarParameterValue::Duration(value) => Some(Cow::Owned(value.to_string())), + value => value.as_text().map(Cow::Borrowed), + } + } +} + +impl ParameterView for ArchivedICalendarParameter { + type Name = ArchivedICalendarParameterName; + + fn name(&self) -> &Self::Name { + &self.name + } + + fn text(&self) -> Option> { + match &self.value { + ArchivedICalendarParameterValue::Integer(value) => { + Some(Cow::Owned(value.to_native().to_string())) + } + ArchivedICalendarParameterValue::Duration(value) => Some(Cow::Owned(value.to_string())), + value => value.as_text().map(Cow::Borrowed), + } + } +} + +trait ValueText { + fn value_text(&self) -> Option>; +} + +impl ValueText for ICalendarValue { + fn value_text(&self) -> Option> { + match self { + ICalendarValue::Integer(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::Float(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::Boolean(value) => { + Some(Cow::Borrowed(if *value { "TRUE" } else { "FALSE" })) + } + ICalendarValue::Duration(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::RecurrenceRule(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::Period(value) => Some(Cow::Owned(value.to_string())), + ICalendarValue::PartialDateTime(value) => { + let value_type = match (value.has_date(), value.has_time()) { + (true, true) => ICalendarValueType::DateTime, + (true, false) => ICalendarValueType::Date, + (false, true) => ICalendarValueType::Time, + (false, false) => ICalendarValueType::UtcOffset, + }; + let mut text = String::with_capacity(20); + value + .format_as_ical(&mut text, &value_type) + .ok() + .map(|_| Cow::Owned(text)) + } + value => value.as_text().map(Cow::Borrowed), + } + } +} + +impl ValueText for ArchivedICalendarValue { + fn value_text(&self) -> Option> { + match self { + ArchivedICalendarValue::Integer(value) => { + Some(Cow::Owned(value.to_native().to_string())) + } + ArchivedICalendarValue::Float(value) => Some(Cow::Owned(value.to_native().to_string())), + ArchivedICalendarValue::Boolean(value) => { + Some(Cow::Borrowed(if *value { "TRUE" } else { "FALSE" })) + } + ArchivedICalendarValue::Duration(value) => Some(Cow::Owned(value.to_string())), + ArchivedICalendarValue::RecurrenceRule(value) => { + Some(Cow::Owned(value.as_ref().to_string())) + } + ArchivedICalendarValue::Period(value) => Some(Cow::Owned(value.to_string())), + ArchivedICalendarValue::PartialDateTime(value) => { + let value_type = match (value.has_date(), value.has_time()) { + (true, true) => ArchivedICalendarValueType::DateTime, + (true, false) => ArchivedICalendarValueType::Date, + (false, true) => ArchivedICalendarValueType::Time, + (false, false) => ArchivedICalendarValueType::UtcOffset, + }; + let mut text = String::with_capacity(20); + value + .format_as_ical(&mut text, &value_type) + .ok() + .map(|_| Cow::Owned(text)) + } + value => value.as_text().map(Cow::Borrowed), + } + } +} + +impl AlarmRepetition { + fn new(count: Option, interval: Option) -> Self { + match ( + count.and_then(|count| u64::try_from(count).ok()), + interval.and_then(|interval| u64::try_from(interval).ok()), + ) { + (Some(count), Some(interval)) if count > 0 && interval > 0 => { + AlarmRepetition { count, interval } + } + _ => AlarmRepetition::default(), + } + } + + pub fn span(&self) -> i64 { + i64::try_from(self.count.saturating_mul(self.interval)).unwrap_or(i64::MAX) + } + + pub fn triggers_in(&self, first: i64, range: &TimeRange) -> bool { + if first >= range.start { + first < range.end + } else if self.count == 0 { + false + } else { + let steps = range.start.abs_diff(first).div_ceil(self.interval); + steps <= self.count + && i128::from(first) + i128::from(steps) * i128::from(self.interval) + < i128::from(range.end) + } + } +} + +impl AlarmView for Alarm { + fn id(&self) -> u32 { + u32::from(self.id) + } + + fn parent_id(&self) -> u32 { + u32::from(self.parent_id) + } + + fn delta(&self) -> AlarmDelta { + self.delta.clone() + } + + fn timestamp(&self, expansion: &CalendarEventExpansion, default_tz: Tz) -> Option { + expansion.alarm_time(&self.delta, default_tz) + } +} + +impl AlarmView for ArchivedAlarm { + fn id(&self) -> u32 { + u32::from(self.id.to_native()) + } + + fn parent_id(&self) -> u32 { + u32::from(self.parent_id.to_native()) + } + + fn delta(&self) -> AlarmDelta { + AlarmDelta::from(&self.delta) + } + + fn timestamp(&self, expansion: &CalendarEventExpansion, default_tz: Tz) -> Option { + expansion.alarm_time(&AlarmDelta::from(&self.delta), default_tz) + } +} + +fn entry_tz(tz_id: Option<&str>, default_tz: Tz) -> Tz { + tz_id + .and_then(|tz_id| tz_id.parse().ok()) + .unwrap_or(default_tz) +} diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index eb4bb9e8..791ccdd8 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -6,32 +6,43 @@ use super::assert_is_unique_uid; use crate::{ - DavError, DavMethod, + DavError, DavErrorCondition, DavMethod, common::{ + ContainerOperation, assert_parent_limit, + dead::{ + ContainerRequest, ContainerWrites, DeadContainers, check_growth, container_kinds, + copy_growth, read_container, + }, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; -use common::{DavName, Server, auth::AccessToken}; -use dav_proto::{Depth, RequestHeaders}; +use common::{ + DavName, GroupwareResources, Server, + auth::AccessToken, + storage::{batch::MAX_BATCH_DOCUMENTS, index::PresenceFlags, metadata::StoredEntry}, +}; +use dav_proto::{Depth, RequestHeaders, schema::response::CardCondition}; use groupware::{ DestroyArchive, cache::GroupwareCache, - contact::{AddressBook, AddressBookPreferences, ContactCard}, + contact::{AddressBook, AddressBookPreferences, ContactCard, ContactCardContent}, }; use http_proto::HttpResponse; use hyper::StatusCode; -use store::write::BatchBuilder; +use registry::schema::enums::StorageQuota; +use store::write::{BatchBuilder, PendingId}; use store::{ ValueKey, - write::{AlignedBytes, Archive, ValueClass}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, - field::PrincipalField, + field::{ContactField, PrincipalField}, + metadata::MetadataKinds, }; pub(crate) trait CardCopyMoveRequestHandler: Sync + Send { @@ -57,7 +68,7 @@ impl CardCopyMoveRequestHandler for Server { .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::AddressBook, @@ -105,7 +116,7 @@ impl CardCopyMoveRequestHandler for Server { let to_resources = if to_account_id == from_account_id { from_resources.clone() } else { - self.fetch_dav_resources( + self.fetch_groupware_resources( access_token.account_id(), to_account_id, SyncCollection::AddressBook, @@ -177,24 +188,20 @@ impl CardCopyMoveRequestHandler for Server { .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); - let to_document_ids = to_resources + let to_children = to_resources .subtree(destination_resource_name) .filter(|r| !r.is_container()) - .map(|r| r.document_id()) + .map(|r| (r.document_id(), r.resource.metadata_kinds())) .collect::>(); // Validate ACLs if !access_token.is_member(to_account_id) || (!access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( - access_token, - from_resource.document_id(), - if is_move { - Acl::RemoveItems - } else { - Acl::ReadItems - }, - )) + && !from_resources + .container_acl(access_token, from_resource.document_id()) + .contains_all( + ContainerOperation::from_move(is_move).required_acls(), + )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -203,13 +210,15 @@ impl CardCopyMoveRequestHandler for Server { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), from_children_ids, from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), - to_document_ids, + to_children, new_name, is_move, ) @@ -232,11 +241,9 @@ impl CardCopyMoveRequestHandler for Server { }, )) || (!access_token.is_member(to_account_id) - && !to_resources.has_access_to_container( - access_token, - to_addressbook_id, - Acl::RemoveItems, - )) + && !to_resources + .container_acl(access_token, to_addressbook_id) + .contains_all([Acl::AddItems, Acl::RemoveItems].into_iter())) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -245,6 +252,7 @@ impl CardCopyMoveRequestHandler for Server { move_card( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), from_addressbook_id, @@ -259,6 +267,7 @@ impl CardCopyMoveRequestHandler for Server { copy_card( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), to_account_id, @@ -312,6 +321,7 @@ impl CardCopyMoveRequestHandler for Server { move_card( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), from_addressbook_id, @@ -338,6 +348,7 @@ impl CardCopyMoveRequestHandler for Server { copy_card( self, access_token, + &to_resources, from_account_id, from_resource.document_id(), to_account_id, @@ -360,15 +371,9 @@ impl CardCopyMoveRequestHandler for Server { // Validate ACLs if !access_token.is_member(from_account_id) - && !from_resources.has_access_to_container( - access_token, - from_resource.document_id(), - if is_move { - Acl::RemoveItems - } else { - Acl::ReadItems - }, - ) + && !from_resources + .container_acl(access_token, from_resource.document_id()) + .contains_all(ContainerOperation::from_move(is_move).required_acls()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -384,6 +389,8 @@ impl CardCopyMoveRequestHandler for Server { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { @@ -414,6 +421,8 @@ impl CardCopyMoveRequestHandler for Server { copy_container( self, access_token, + &from_resources, + &to_resources, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { @@ -441,6 +450,7 @@ impl CardCopyMoveRequestHandler for Server { async fn copy_card( server: &Server, access_token: &AccessToken, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, to_account_id: u32, @@ -451,7 +461,7 @@ async fn copy_card( // Fetch card let card_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_document_id, @@ -466,21 +476,27 @@ async fn copy_card( // Validate UID assert_is_unique_uid( - server, - server - .fetch_dav_resources( - access_token.account_id(), - to_account_id, - SyncCollection::AddressBook, - ) - .await - .caused_by(trc::location!())? - .as_ref(), - to_account_id, + to_resources, to_addressbook_id, - card.inner.card.uid(), - ) - .await?; + Some(card.inner.uid.as_str()).filter(|uid| !uid.is_empty()), + )?; + if from_account_id == to_account_id + && card + .inner + .names + .iter() + .any(|name| name.parent_id == to_addressbook_id) + { + return Err(DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + CardCondition::NoUidConflict( + to_resources + .format_resource_path_by_parent(from_document_id, to_addressbook_id) + .unwrap_or_default() + .into(), + ), + ))); + } if from_account_id == to_account_id { let mut new_card = card @@ -490,12 +506,18 @@ async fn copy_card( name: new_name.to_string(), parent_id: to_addressbook_id, }); + assert_parent_limit( + card.inner.names.len(), + new_card.names.len(), + server.core.groupware.max_address_books_per_card, + )?; new_card - .update( - access_token.account_tenant_ids(), + .update_meta( + access_token.changed_by(), card, from_account_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -507,16 +529,65 @@ async fn copy_card( name: new_name.to_string(), parent_id: to_addressbook_id, }]; - let to_document_id = server + let container = read_container( + server, + from_account_id, + Collection::ContactCard, + from_document_id, + new_card.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + container + .as_ref() + .map_or(0, |container| copy_growth(container, None)), + ) + .await + .caused_by(trc::location!())?; + new_card.set_metadata_kinds(container_kinds(container.as_ref())); + let content_ = server .store() - .assign_document_ids(to_account_id, Collection::ContactCard, 1) + .get_value::>(ValueKey::property( + from_account_id, + Collection::ContactCard, + from_document_id, + ContactField::Content, + )) .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let new_content = content_ + .deserialize::() .caused_by(trc::location!())?; + + if to_document_id.is_none() { + server.assert_object_quota( + &*server.account(to_account_id).await?, + StorageQuota::MaxContactCards, + 1, + || to_resources.resources.count(false), + )?; + } + + let to_document_id = batch.reserve_document_id(to_account_id, Collection::ContactCard); + if let Some(container) = &container { + let mut copies = ContainerWrites::new(to_account_id, Collection::ContactCard); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } new_card .insert( - access_token.account_tenant_ids(), + new_content, + server.core.groupware.vcard_version, + access_token.changed_by(), to_account_id, to_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -526,7 +597,7 @@ async fn copy_card( // Overwrite card on destination let card_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::ContactCard, to_document_id, @@ -540,13 +611,15 @@ async fn copy_card( DestroyArchive(card) .delete( - access_token.account_tenant_ids(), + server, + access_token.changed_by(), to_account_id, to_document_id, to_addressbook_id, None, &mut batch, ) + .await .caused_by(trc::location!())?; } @@ -559,7 +632,6 @@ async fn copy_card( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); response } @@ -568,6 +640,7 @@ async fn copy_card( async fn move_card( server: &Server, access_token: &AccessToken, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, from_addressbook_id: u32, @@ -580,7 +653,7 @@ async fn move_card( // Fetch card let card_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_document_id, @@ -598,21 +671,10 @@ async fn move_card( || to_document_id.is_none() { assert_is_unique_uid( - server, - server - .fetch_dav_resources( - access_token.account_id(), - to_account_id, - SyncCollection::AddressBook, - ) - .await - .caused_by(trc::location!())? - .as_ref(), - to_account_id, + to_resources, to_addressbook_id, - card.inner.card.uid(), - ) - .await?; + Some(card.inner.uid.as_str()).filter(|uid| !uid.is_empty()), + )?; } let mut batch = BatchBuilder::new(); @@ -640,11 +702,12 @@ async fn move_card( parent_id: to_addressbook_id, }); new_card - .update( - access_token.account_tenant_ids(), + .update_meta( + access_token.changed_by(), card.clone(), from_account_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -657,28 +720,94 @@ async fn move_card( name: new_name.to_string(), parent_id: to_addressbook_id, }]; + let container = read_container( + server, + from_account_id, + Collection::ContactCard, + from_document_id, + new_card.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + container + .as_ref() + .map_or(0, |container| copy_growth(container, None)), + ) + .await + .caused_by(trc::location!())?; + new_card.set_metadata_kinds(container_kinds(container.as_ref())); + let source_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::ContactCard, + container + .as_ref() + .map(|container| StoredEntry::from_container(from_document_id, container)), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(card) - .delete( - access_token.account_tenant_ids(), + .delete_with_cleanup( + server, + access_token.changed_by(), from_account_id, from_document_id, from_addressbook_id, + None, from_resource_path.into(), + &source_cleanup, &mut batch, ) - .caused_by(trc::location!())?; - - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; + + let content_ = server + .store() + .get_value::>(ValueKey::property( + from_account_id, + Collection::ContactCard, + from_document_id, + ContactField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let new_content = content_ + .deserialize::() + .caused_by(trc::location!())?; + + let to_account = server.account(to_account_id).await?; + if to_document_id.is_none() + && server + .object_quota_limit(&to_account, StorageQuota::MaxContactCards) + .is_some() + { + server.assert_object_quota(&to_account, StorageQuota::MaxContactCards, 1, || { + to_resources.resources.count(false) + })?; + } + + let to_document_id = batch.reserve_document_id(to_account_id, Collection::ContactCard); + if let Some(container) = &container { + let mut copies = ContainerWrites::new(to_account_id, Collection::ContactCard); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } new_card .insert( - access_token.account_tenant_ids(), + new_content, + server.core.groupware.vcard_version, + access_token.changed_by(), to_account_id, to_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -688,7 +817,7 @@ async fn move_card( // Overwrite card on destination let card_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::ContactCard, to_document_id, @@ -702,13 +831,15 @@ async fn move_card( DestroyArchive(card) .delete( - access_token.account_tenant_ids(), + server, + access_token.changed_by(), to_account_id, to_document_id, to_addressbook_id, None, &mut batch, ) + .await .caused_by(trc::location!())?; } @@ -721,7 +852,6 @@ async fn move_card( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); response } @@ -739,7 +869,7 @@ async fn rename_card( // Fetch card let card_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, @@ -764,11 +894,12 @@ async fn rename_card( let mut batch = BatchBuilder::new(); new_card - .update( - access_token.account_tenant_ids(), + .update_meta( + access_token.changed_by(), card, account_id, document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -777,7 +908,6 @@ async fn rename_card( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } @@ -786,20 +916,22 @@ async fn rename_card( async fn copy_container( server: &Server, access_token: &AccessToken, + from_resources: &GroupwareResources, + to_resources: &GroupwareResources, from_account_id: u32, from_document_id: u32, from_children_ids: Vec, from_resource_path: String, to_account_id: u32, to_document_id: Option, - to_children_ids: Vec, + to_children: Vec<(u32, MetadataKinds)>, new_name: &str, remove_source: bool, ) -> crate::Result { // Fetch book let book_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::AddressBook, from_document_id, @@ -813,41 +945,85 @@ async fn copy_container( let mut book = old_book .deserialize::() .caused_by(trc::location!())?; + let book_container = read_container( + server, + from_account_id, + Collection::AddressBook, + from_document_id, + old_book.inner.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + book.set_metadata_kinds(container_kinds(book_container.as_ref())); + let card_containers = if from_account_id != to_account_id { + let mut request = ContainerRequest::default(); + for &document_id in &from_children_ids { + if from_resources + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + { + request.insert(from_account_id, Collection::ContactCard, document_id); + } + } + request.load(server).await.caused_by(trc::location!())? + } else { + DeadContainers::default() + }; + let mut card_copies = ContainerWrites::new(to_account_id, Collection::ContactCard); + let source_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::ContactCard, + card_containers + .stored_entries(from_account_id, Collection::ContactCard) + .filter(|_| remove_source), + ) + .await + .caused_by(trc::location!())?; + check_growth( + server, + to_account_id, + book_container + .as_ref() + .map_or(0, |container| copy_growth(container, None)) + + card_containers.stored_len(), + ) + .await + .caused_by(trc::location!())?; + + // Validate quota + let to_account = server.account(to_account_id).await?; + let has_container_quota = to_document_id.is_none() + && server + .object_quota_limit(&to_account, StorageQuota::MaxAddressBooks) + .is_some(); + let has_item_quota = from_account_id != to_account_id + && server + .object_quota_limit(&to_account, StorageQuota::MaxContactCards) + .is_some(); + let to_resources = (has_container_quota || has_item_quota).then_some(to_resources); + if has_container_quota && let Some(to_resources) = &to_resources { + server.assert_object_quota(&to_account, StorageQuota::MaxAddressBooks, 1, || { + to_resources.resources.count(true) + })?; + } + let deleted_cards = match &to_resources { + Some(to_resources) if has_item_quota => to_children + .iter() + .filter(|(document_id, _)| { + to_resources + .resources + .find(*document_id, false) + .is_some_and(|item| item.child_names().len() <= 1) + }) + .count(), + _ => 0, + }; // Prepare write batch let mut batch = BatchBuilder::new(); - if remove_source { - DestroyArchive(old_book) - .delete( - access_token.account_tenant_ids(), - from_account_id, - from_document_id, - from_resource_path.into(), - &mut batch, - ) - .caused_by(trc::location!())?; - - // Reset default address book id - let default_book_id = server - .store() - .get_value::(ValueKey { - account_id: from_account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), - }) - .await - .caused_by(trc::location!())?; - if default_book_id.is_some_and(|id| id == from_document_id) { - batch - .with_account_id(from_account_id) - .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultAddressBookId); - } - } - let preference = book.preferences.into_iter().next().unwrap(); book.name = new_name.to_string(); book.subscribers.clear(); @@ -864,7 +1040,7 @@ async fn copy_container( // Overwrite destination let book_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::AddressBook, to_document_id, @@ -875,14 +1051,31 @@ async fn copy_container( let book = book_ .to_unarchived::() .caused_by(trc::location!())?; + let cleanup = server + .preload_container_cleanup( + access_token.changed_by(), + to_account_id, + Collection::ContactCard, + &to_children + .iter() + .filter(|(_, kinds)| !kinds.is_empty()) + .map(|(document_id, _)| *document_id) + .collect(), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(book) .delete_with_cards( server, - access_token.account_tenant_ids(), + access_token.changed_by(), to_account_id, to_document_id, - to_children_ids, + to_children + .into_iter() + .map(|(document_id, _)| document_id) + .collect(), + &cleanup, None, &mut batch, ) @@ -890,16 +1083,20 @@ async fn copy_container( .caused_by(trc::location!())?; } - to_document_id + PendingId::Assigned(to_document_id) } else { - server - .store() - .assign_document_ids(to_account_id, Collection::AddressBook, 1) - .await - .caused_by(trc::location!())? + PendingId::Slot(batch.reserve_document_id(to_account_id, Collection::AddressBook)) }; + let parent_id = to_document_id; + if let Some(container) = &book_container { + let mut copies = ContainerWrites::new(to_account_id, Collection::AddressBook); + copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } book.insert( - access_token.account_tenant_ids(), + access_token.changed_by(), to_account_id, to_document_id, &mut batch, @@ -908,107 +1105,185 @@ async fn copy_container( // Copy children let mut required_space = 0; - for from_child_document_id in from_children_ids { - if let Some(card_) = server - .store() - .get_value::>(ValueKey::archive( + let mut created_cards = 0; + let mut from_children_ids = from_children_ids; + from_children_ids.sort_unstable(); + from_children_ids.dedup(); + let is_cross_account = from_account_id != to_account_id; + for chunk in from_children_ids.chunks(MAX_BATCH_DOCUMENTS) { + let mut archives = server + .archive_chunk( from_account_id, Collection::ContactCard, - from_child_document_id, - )) - .await? - { - let card = card_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_name = None; - - for name in card.inner.names.iter() { - if name.parent_id == to_document_id { - continue; - } else if name.parent_id == from_document_id { - new_name = Some(name.name.to_string()); - } - } - let new_name = if let Some(new_name) = new_name { - DavName { - name: new_name, - parent_id: to_document_id, - } - } else { - continue; - }; - let card = card_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_card = card - .deserialize::() - .caused_by(trc::location!())?; - - if from_account_id == to_account_id { - if remove_source { - new_card - .names - .retain(|name| name.parent_id != from_document_id); - } - - new_card.names.push(new_name); - new_card - .update( - access_token.account_tenant_ids(), - card, - from_account_id, - from_child_document_id, - &mut batch, - ) + chunk, + is_cross_account.then_some((ContactField::Content.field(), chunk)), + ) + .await + .caused_by(trc::location!())?; + for &from_child_document_id in chunk { + if let Some(card_) = archives.take_archive(from_child_document_id) { + let card = card_ + .to_unarchived::() .caused_by(trc::location!())?; - } else { - if remove_source { - DestroyArchive(card) - .delete( - access_token.account_tenant_ids(), + let mut new_name = None; + + for name in card.inner.names.iter() { + if parent_id + .assigned() + .is_some_and(|parent| name.parent_id == parent) + { + continue; + } else if name.parent_id == from_document_id { + new_name = Some(name.name.to_string()); + } + } + let new_name = if let Some(new_name) = new_name { + DavName { + name: new_name, + parent_id: parent_id.assigned().unwrap_or_default(), + } + } else { + continue; + }; + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + + if from_account_id == to_account_id { + if remove_source { + new_card + .names + .retain(|name| name.parent_id != from_document_id); + } + + new_card.names.push(new_name); + assert_parent_limit( + card.inner.names.len(), + new_card.names.len(), + server.core.groupware.max_address_books_per_card, + )?; + new_card + .update_meta( + access_token.changed_by(), + card, from_account_id, from_child_document_id, - from_document_id, - None, + parent_id.slot(), + &mut batch, + ) + .caused_by(trc::location!())?; + } else { + let content_ = archives + .take_value(from_child_document_id) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let new_content = content_ + .deserialize::() + .caused_by(trc::location!())?; + + if remove_source { + DestroyArchive(card) + .delete_with_cleanup( + server, + access_token.changed_by(), + from_account_id, + from_child_document_id, + from_document_id, + Some(content_), + None, + &source_cleanup, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + + let to_document_id = + batch.reserve_document_id(to_account_id, Collection::ContactCard); + let container = card_containers.get( + from_account_id, + Collection::ContactCard, + from_child_document_id, + ); + new_card.set_metadata_kinds(container_kinds(container)); + if let Some(container) = container { + card_copies + .copy(server, container, to_document_id, None, &mut batch) + .await + .caused_by(trc::location!())?; + } + new_card.names = vec![new_name]; + required_space += new_card.size as u64; + created_cards += 1; + new_card + .insert( + new_content, + server.core.groupware.vcard_version, + access_token.changed_by(), + to_account_id, + to_document_id, + parent_id.slot(), &mut batch, ) .caused_by(trc::location!())?; } - - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::ContactCard, 1) - .await - .caused_by(trc::location!())?; - new_card.names = vec![new_name]; - required_space += new_card.size as u64; - new_card - .insert( - access_token.account_tenant_ids(), - to_account_id, - to_document_id, - &mut batch, - ) - .caused_by(trc::location!())?; } } } + if has_item_quota + && created_cards > deleted_cards + && let Some(to_resources) = &to_resources + { + server.assert_object_quota( + &to_account, + StorageQuota::MaxContactCards, + created_cards - deleted_cards, + || to_resources.resources.count(false), + )?; + } + if from_account_id != to_account_id && required_space > 0 { server - .has_available_quota( - server.account(to_account_id).await?.as_ref(), - required_space, - ) + .has_available_quota(&to_account, required_space) .await?; } + if remove_source { + let book_cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::AddressBook, + book_container + .as_ref() + .map(|container| StoredEntry::from_container(from_document_id, container)), + ) + .await + .caused_by(trc::location!())?; + DestroyArchive(old_book) + .delete_with_cleanup( + server, + access_token.changed_by(), + from_account_id, + from_document_id, + from_resource_path.into(), + &book_cleanup, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + + batch + .with_account_id(from_account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear_if_equals(PrincipalField::DefaultAddressBookId, from_document_id); + } + server .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); if !is_overwrite { Ok(HttpResponse::new(StatusCode::CREATED)) @@ -1029,7 +1304,7 @@ async fn rename_container( // Fetch book let book_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, @@ -1048,7 +1323,7 @@ async fn rename_container( let mut batch = BatchBuilder::new(); new_book .update( - access_token.account_tenant_ids(), + access_token.changed_by(), book, account_id, document_id, @@ -1060,7 +1335,6 @@ async fn rename_container( .commit_batch(batch) .await .caused_by(trc::location!())?; - server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } diff --git a/crates/dav/src/card/delete.rs b/crates/dav/src/card/delete.rs index 8a25b48d..13875c5d 100644 --- a/crates/dav/src/card/delete.rs +++ b/crates/dav/src/card/delete.rs @@ -7,7 +7,7 @@ use crate::{ DavError, DavMethod, common::{ - ETag, + ContainerOperation, ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, @@ -21,10 +21,10 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; -use store::write::{BatchBuilder, ValueClass}; +use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -58,7 +58,7 @@ impl CardDeleteRequestHandler for Server { .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -77,7 +77,7 @@ impl CardDeleteRequestHandler for Server { if delete_resource.is_container() { let book_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, @@ -96,7 +96,7 @@ impl CardDeleteRequestHandler for Server { .inner .acls .effective_acl(access_token) - .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) + .contains_all(ContainerOperation::Remove.required_acls()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -119,17 +119,35 @@ impl CardDeleteRequestHandler for Server { .await?; // Delete addressbook and cards + let children = resources + .subtree(delete_path) + .filter(|r| !r.is_container()) + .map(|r| (r.document_id(), r.resource.metadata_kinds())) + .collect::>(); + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::ContactCard, + &children + .iter() + .filter(|(_, kinds)| !kinds.is_empty()) + .map(|(document_id, _)| *document_id) + .collect(), + ) + .await + .caused_by(trc::location!())?; DestroyArchive(book) .delete_with_cards( self, - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, - resources - .subtree(delete_path) - .filter(|r| !r.is_container()) - .map(|r| r.document_id()) - .collect::>(), + children + .into_iter() + .map(|(document_id, _)| document_id) + .collect(), + &cleanup, resources.format_resource(delete_resource).into(), &mut batch, ) @@ -137,23 +155,11 @@ impl CardDeleteRequestHandler for Server { .caused_by(trc::location!())?; // Reset default address book id - let default_book_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), - }) - .await - .caused_by(trc::location!())?; - if default_book_id.is_some_and(|id| id == document_id) { - batch - .with_account_id(account_id) - .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultAddressBookId); - } + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear_if_equals(PrincipalField::DefaultAddressBookId, document_id); } else { // Validate ACL let addressbook_id = delete_resource.parent_id().unwrap(); @@ -169,7 +175,7 @@ impl CardDeleteRequestHandler for Server { let card_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, @@ -177,6 +183,9 @@ impl CardDeleteRequestHandler for Server { .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let card = card_ + .to_unarchived::() + .caused_by(trc::location!())?; // Validate headers self.validate_headers( @@ -186,7 +195,7 @@ impl CardDeleteRequestHandler for Server { account_id, collection: Collection::ContactCard, document_id: document_id.into(), - etag: card_.etag().into(), + etag: format!("\"{}\"", card.inner.etag.to_native()).into(), path: delete_path, ..Default::default() }], @@ -196,24 +205,21 @@ impl CardDeleteRequestHandler for Server { .await?; // Delete card - DestroyArchive( - card_ - .to_unarchived::() - .caused_by(trc::location!())?, - ) - .delete( - access_token.account_tenant_ids(), - account_id, - document_id, - addressbook_id, - resources.format_resource(delete_resource).into(), - &mut batch, - ) - .caused_by(trc::location!())?; + DestroyArchive(card) + .delete( + self, + access_token.changed_by(), + account_id, + document_id, + addressbook_id, + resources.format_resource(delete_resource).into(), + &mut batch, + ) + .await + .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } diff --git a/crates/dav/src/card/filter.rs b/crates/dav/src/card/filter.rs new file mode 100644 index 00000000..bb192032 --- /dev/null +++ b/crates/dav/src/card/filter.rs @@ -0,0 +1,236 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::common::search::{Candidates, IndexedText, QueryScope, TextIndex}; +use calcard::vcard::{ + ArchivedVCard, ArchivedVCardEntry, ArchivedVCardParameterValue, ArchivedVCardValue, + ArchivedVCardValueType, VCardParameterName, VCardProperty, +}; +use common::storage::dav::CachedUid; +use dav_proto::schema::request::{ + CardFilter, CardPropFilter, CardPropMatch, FilterTest, ParamFilter, Presence, + VCardPropertyWithGroup, +}; +use std::{borrow::Cow, convert::identity}; +use store::search::{ContactSearchField, SearchField}; + +pub(crate) trait CardFilterMatch { + fn matches(&self, card: &ArchivedVCard) -> bool; +} + +pub(crate) trait CardFilterPlan { + fn candidates(&self, scope: &QueryScope<'_>, index: TextIndex<'_>) -> Candidates; +} + +trait EntryMatch { + fn matches_entry(&self, entry: &ArchivedVCardEntry) -> bool; +} + +trait EntryText { + fn text_values(&self) -> impl Iterator>; +} + +trait ValueText { + fn value_text(&self) -> Option>; +} + +trait ContactIndexField { + fn search_field(&self) -> Option<(SearchField, IndexedText)>; +} + +impl CardFilterMatch for CardFilter { + fn matches(&self, card: &ArchivedVCard) -> bool { + let mut results = self.prop_filters.iter().map(|filter| filter.matches(card)); + match self.test { + FilterTest::AnyOf => self.prop_filters.is_empty() || results.any(identity), + FilterTest::AllOf => results.all(identity), + } + } +} + +impl CardFilterMatch for CardPropFilter { + fn matches(&self, card: &ArchivedVCard) -> bool { + let mut entries = card + .entries + .iter() + .filter(|entry| self.name.matches_entry(entry)); + match &self.test { + Presence::IsNotDefined => entries.next().is_none(), + Presence::IsDefined(test) => entries.any(|entry| test.matches_entry(entry)), + } + } +} + +impl EntryMatch for VCardPropertyWithGroup { + fn matches_entry(&self, entry: &ArchivedVCardEntry) -> bool { + entry.name == self.name + && self.group.as_deref().is_none_or(|group| { + entry + .group + .as_ref() + .is_some_and(|entry_group| entry_group.eq_ignore_ascii_case(group)) + }) + } +} + +impl EntryMatch for CardPropMatch { + fn matches_entry(&self, entry: &ArchivedVCardEntry) -> bool { + let mut results = self + .text_matches + .iter() + .map(|text_match| text_match.is_match_any(entry.text_values())) + .chain( + self.param_filters + .iter() + .map(|filter| filter.matches_entry(entry)), + ); + match self.test { + FilterTest::AnyOf => { + (self.text_matches.is_empty() && self.param_filters.is_empty()) + || results.any(identity) + } + FilterTest::AllOf => results.all(identity), + } + } +} + +impl EntryMatch for ParamFilter { + fn matches_entry(&self, entry: &ArchivedVCardEntry) -> bool { + let mut params = entry + .params + .iter() + .filter(|param| param.name == self.name) + .peekable(); + match &self.test { + Presence::IsNotDefined => params.peek().is_none(), + Presence::IsDefined(None) => params.peek().is_some(), + Presence::IsDefined(Some(text_match)) => { + params.peek().is_some() + && text_match.is_match_any(params.filter_map(|param| param.value.value_text())) + } + } + } +} + +impl EntryText for ArchivedVCardEntry { + fn text_values(&self) -> impl Iterator> { + self.values.iter().flat_map(|value| { + let (text, items) = match value { + ArchivedVCardValue::Component(items) => (None, items.as_slice()), + value => (value.value_text(), &[][..]), + }; + text.into_iter() + .chain(items.iter().map(|item| Cow::Borrowed(item.as_str()))) + }) + } +} + +impl ValueText for ArchivedVCardValue { + fn value_text(&self) -> Option> { + match self { + ArchivedVCardValue::Integer(value) => Some(Cow::Owned(value.to_native().to_string())), + ArchivedVCardValue::Float(value) => Some(Cow::Owned(value.to_native().to_string())), + ArchivedVCardValue::Boolean(value) => { + Some(Cow::Borrowed(if *value { "TRUE" } else { "FALSE" })) + } + ArchivedVCardValue::PartialDateTime(value) => { + let mut text = String::with_capacity(20); + value + .format_as_vcard(&mut text, &ArchivedVCardValueType::DateAndOrTime) + .ok() + .map(|_| Cow::Owned(text)) + } + value => value.as_text().map(Cow::Borrowed), + } + } +} + +impl ValueText for ArchivedVCardParameterValue { + fn value_text(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Integer(value) => { + Some(Cow::Owned(value.to_native().to_string())) + } + ArchivedVCardParameterValue::Bool(value) => { + Some(Cow::Borrowed(if *value { "TRUE" } else { "FALSE" })) + } + value => value.as_text().map(Cow::Borrowed), + } + } +} + +impl CardFilterPlan for CardFilter { + fn candidates(&self, scope: &QueryScope<'_>, index: TextIndex<'_>) -> Candidates { + if self.prop_filters.is_empty() { + return Candidates::All; + } + let candidates = self + .prop_filters + .iter() + .map(|filter| filter.candidates(scope, index)); + match self.test { + FilterTest::AnyOf => Candidates::or(candidates), + FilterTest::AllOf => Candidates::and(candidates), + } + } +} + +impl CardFilterPlan for CardPropFilter { + fn candidates(&self, scope: &QueryScope<'_>, index: TextIndex<'_>) -> Candidates { + let Presence::IsDefined(test) = &self.test else { + return Candidates::All; + }; + if test.text_matches.is_empty() + || (test.test == FilterTest::AnyOf && !test.param_filters.is_empty()) + { + return Candidates::All; + } + + let is_uid = self.name.name == VCardProperty::Uid && self.name.group.is_none(); + let field = self.name.name.search_field(); + let candidates = test.text_matches.iter().map(|text_match| { + if text_match.negate { + Candidates::All + } else if is_uid { + scope.matching(|resource| { + resource.uid().is_none_or(|uid| { + uid.is_empty() || uid.is_hashed_uid() || text_match.is_match(uid) + }) + }) + } else if let Some((field, text)) = &field { + index.candidates(field.clone(), *text, text_match) + } else { + Candidates::All + } + }); + + match test.test { + FilterTest::AnyOf => Candidates::or(candidates), + FilterTest::AllOf => Candidates::and(candidates), + } + } +} + +impl ContactIndexField for VCardProperty { + fn search_field(&self) -> Option<(SearchField, IndexedText)> { + let (field, text) = match self { + VCardProperty::Fn | VCardProperty::N => (ContactSearchField::Name, IndexedText::Plain), + VCardProperty::Nickname => (ContactSearchField::Nickname, IndexedText::Plain), + VCardProperty::Org => (ContactSearchField::Organization, IndexedText::Plain), + VCardProperty::Email => (ContactSearchField::Email, IndexedText::Identifier), + VCardProperty::Tel => (ContactSearchField::Phone, IndexedText::Identifier), + VCardProperty::Impp | VCardProperty::Socialprofile => { + (ContactSearchField::OnlineService, IndexedText::Identifier) + } + VCardProperty::Adr => (ContactSearchField::Address, IndexedText::Plain), + VCardProperty::Note => (ContactSearchField::Note, IndexedText::Stemmed), + VCardProperty::Member => (ContactSearchField::Member, IndexedText::Identifier), + _ => return None, + }; + Some((field.into(), text)) + } +} + diff --git a/crates/dav/src/card/get.rs b/crates/dav/src/card/get.rs index 702ca9b4..537ef66e 100644 --- a/crates/dav/src/card/get.rs +++ b/crates/dav/src/card/get.rs @@ -7,24 +7,27 @@ use crate::{ DavError, DavMethod, common::{ - ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; -use groupware::{cache::GroupwareCache, contact::ContactCard}; +use groupware::{ + cache::GroupwareCache, + contact::{ContactCard, ContactCardContent}, +}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::ContactField, }; pub(crate) trait CardGetRequestHandler: Sync + Send { @@ -50,7 +53,7 @@ impl CardGetRequestHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -80,22 +83,35 @@ impl CardGetRequestHandler for Server { } // Fetch card - let card_ = self - .store() - .get_value::>(ValueKey::archive( + let (card_, prefetched) = if is_head || !headers.if_.is_empty() { + ( + self.store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + resource.document_id(), + )) + .await + .caused_by(trc::location!())?, + None, + ) + } else { + self.archive_and_field( account_id, Collection::ContactCard, resource.document_id(), - )) + ContactField::Content.field(), + ) .await .caused_by(trc::location!())? - .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + }; + let card_ = card_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .unarchive::() .caused_by(trc::location!())?; // Validate headers - let etag = card_.etag(); + let etag = format!("\"{}\"", card.etag.to_native()); self.validate_headers( access_token, headers, @@ -117,13 +133,34 @@ impl CardGetRequestHandler for Server { .with_etag(etag) .with_last_modified(Rfc1123DateTime::new(i64::from(card.modified)).to_string()); - let mut vcard = String::with_capacity(128); - let _ = card.card.write_to( - &mut vcard, - headers - .vcard_version - .unwrap_or(self.core.groupware.vcard_version), - ); + let version = headers + .vcard_version + .unwrap_or(self.core.groupware.vcard_version); + + if is_head && version == self.core.groupware.vcard_version { + return Ok(response.with_content_length(card.size.to_native() as usize)); + } + + let content_ = match prefetched { + Some(content) => content, + None => self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::ContactCard, + resource.document_id(), + ContactField::Content, + )) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, + }; + let content = content_ + .unarchive::() + .caused_by(trc::location!())?; + + let mut vcard = String::with_capacity(card.size.to_native() as usize); + let _ = content.card.write_to(&mut vcard, version); if !is_head { Ok(response.with_binary_body(vcard)) diff --git a/crates/dav/src/card/mkcol.rs b/crates/dav/src/card/mkcol.rs index b76e6a17..c868a7b9 100644 --- a/crates/dav/src/card/mkcol.rs +++ b/crates/dav/src/card/mkcol.rs @@ -9,11 +9,12 @@ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ExtractETag, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, storage::index::PresenceFlags}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, @@ -24,9 +25,13 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use store::write::BatchBuilder; use trc::AddContext; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + metadata::MetadataKinds, +}; pub(crate) trait CardMkColRequestHandler: Sync + Send { fn handle_card_mkcol_request( @@ -55,21 +60,27 @@ impl CardMkColRequestHandler for Server { .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); - } else if name.contains('/') - || self - .fetch_dav_resources( - access_token.account_id(), - account_id, - SyncCollection::AddressBook, - ) - .await - .caused_by(trc::location!())? - .by_path(name) - .is_some() - { + } + let resources = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::AddressBook, + ) + .await + .caused_by(trc::location!())?; + if name.contains('/') || resources.by_path(name).is_some() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } + // Validate quota + self.assert_object_quota( + &*self.account(account_id).await?, + StorageQuota::MaxAddressBooks, + 1, + || resources.resources.count(true), + )?; + // Validate headers self.validate_headers( access_token, @@ -98,16 +109,34 @@ impl CardMkColRequestHandler for Server { }; // Apply MKCOL properties + let mut batch = BatchBuilder::new(); + let document_id = batch.reserve_document_id(account_id, Collection::AddressBook); let mut return_prop_stat = None; - if let Some(mkcol) = request { + let mut dead_write = None; + if let Some(mut mkcol) = request { let mut prop_stat = PropStatBuilder::default(); - if !self.apply_addressbook_properties( + let dead = DeadPatch::take_values(&mut mkcol.props, DisplayName::Live); + self.apply_addressbook_properties( access_token.personal_id(account_id, Collection::AddressBook), &mut book, - false, mkcol.props, &mut prop_stat, - ) { + ); + dead_write = dead + .apply( + self, + DeadTarget::container( + account_id, + Collection::AddressBook, + document_id, + MetadataKinds::NONE, + ), + &mut prop_stat, + ) + .await + .caused_by(trc::location!())?; + if prop_stat.has_errors() { + prop_stat.fail_dependencies(); return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CardDav) @@ -120,21 +149,26 @@ impl CardMkColRequestHandler for Server { } // Prepare write batch - let mut batch = BatchBuilder::new(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::AddressBook, 1) - .await - .caused_by(trc::location!())?; + if let Some(dead_write) = &dead_write { + book.set_metadata_kinds(dead_write.kinds); + } + if let Some(write) = dead_write.and_then(|dead_write| dead_write.write) { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } book.insert( - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, &mut batch, ) .caused_by(trc::location!())?; - let etag = batch.etag(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let etag = self + .commit_batch(batch) + .await + .caused_by(trc::location!())? + .etag(); if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) diff --git a/crates/dav/src/card/mod.rs b/crates/dav/src/card/mod.rs index 6c14e1c0..3394ab8f 100644 --- a/crates/dav/src/card/mod.rs +++ b/crates/dav/src/card/mod.rs @@ -5,17 +5,16 @@ */ use crate::{DavError, DavErrorCondition}; -use common::{DavResources, Server}; +use common::GroupwareResources; use dav_proto::schema::{ property::{CardDavProperty, DavProperty, WebDavProperty}, response::CardCondition, }; use hyper::StatusCode; -use trc::AddContext; -use types::{collection::Collection, field::ContactField}; pub mod copy_move; pub mod delete; +pub mod filter; pub mod get; pub mod mkcol; pub mod proppatch; @@ -74,34 +73,48 @@ pub(crate) static CARD_ITEM_PROPS: [DavProperty; 20] = [ }), ]; -pub(crate) async fn assert_is_unique_uid( - server: &Server, - resources: &DavResources, - account_id: u32, +pub(crate) fn assert_is_unique_uid( + resources: &GroupwareResources, addressbook_id: u32, uid: Option<&str>, ) -> crate::Result<()> { - if let Some(uid) = uid { - let hits = server - .document_ids_matching( - account_id, - Collection::ContactCard, - ContactField::Uid, - uid.as_bytes(), - ) - .await - .caused_by(trc::location!())?; - if !hits.is_empty() { - for path in resources.children(addressbook_id) { - if hits.contains(path.document_id()) { - return Err(DavError::Condition(DavErrorCondition::new( - StatusCode::PRECONDITION_FAILED, - CardCondition::NoUidConflict(resources.format_resource(path).into()), - ))); - } - } - } + let Some(uid) = uid else { + return Ok(()); + }; + + if let Some(path) = resources.children_with_uid(addressbook_id, uid).next() { + return Err(DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + CardCondition::NoUidConflict(resources.format_resource(path).into()), + ))); } Ok(()) } + +const URN_UUID_PREFIX: &str = "urn:uuid:"; +const UUID_GROUP_LENGTHS: [usize; 5] = [8, 4, 4, 4, 12]; + +pub(crate) fn is_same_uid(uid: &str, other: &str) -> bool { + uid == other + || matches!( + (uuid_of(uid), uuid_of(other)), + (Some(uuid), Some(other_uuid)) if uuid.eq_ignore_ascii_case(other_uuid) + ) +} + +fn uuid_of(uid: &str) -> Option<&str> { + let uuid = uid + .split_at_checked(URN_UUID_PREFIX.len()) + .filter(|(prefix, _)| prefix.eq_ignore_ascii_case(URN_UUID_PREFIX)) + .map_or(uid, |(_, uuid)| uuid); + let mut groups = uuid.split('-'); + + (UUID_GROUP_LENGTHS.iter().all(|&len| { + groups + .next() + .is_some_and(|group| group.len() == len && group.bytes().all(|b| b.is_ascii_hexdigit())) + }) && groups.next().is_none()) + .then_some(uuid) +} + diff --git a/crates/dav/src/card/proppatch.rs b/crates/dav/src/card/proppatch.rs index 3e518b98..7298ebb4 100644 --- a/crates/dav/src/card/proppatch.rs +++ b/crates/dav/src/card/proppatch.rs @@ -8,17 +8,22 @@ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ETag, ExtractETag, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; -use common::{Server, auth::AccessToken}; +use common::{ + Server, + auth::AccessToken, + storage::index::{PresenceFlags, RewritePresence}, +}; use dav_proto::{ RequestHeaders, Return, schema::{ Namespace, property::{CardDavProperty, DavProperty, DavValue, ResourceType, WebDavProperty}, - request::{DavPropertyValue, PropertyUpdate}, + request::{DavPropertyValue, PropertyUpdate, PropertyUpdateOp}, response::{BaseCondition, MultiStatus, Response}, }, }; @@ -31,7 +36,7 @@ use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -51,18 +56,16 @@ pub(crate) trait CardPropPatchRequestHandler: Sync + Send { &self, personal_id: u32, address_book: &mut AddressBook, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool; + ); fn apply_card_properties( &self, card: &mut ContactCard, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool; + ); } impl CardPropPatchRequestHandler for Server { @@ -80,7 +83,7 @@ impl CardPropPatchRequestHandler for Server { let uri = headers.uri; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -118,7 +121,7 @@ impl CardPropPatchRequestHandler for Server { // Fetch archive let archive = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, collection, document_id, @@ -126,6 +129,18 @@ impl CardPropPatchRequestHandler for Server { .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let etag = if resource.is_container() { + archive.etag() + } else { + format!( + "\"{}\"", + archive + .unarchive::() + .caused_by(trc::location!())? + .etag + .to_native() + ) + }; // Validate headers self.validate_headers( @@ -135,7 +150,7 @@ impl CardPropPatchRequestHandler for Server { account_id, collection, document_id: document_id.into(), - etag: archive.etag().into(), + etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], @@ -144,107 +159,183 @@ impl CardPropPatchRequestHandler for Server { ) .await?; - let is_success; + let dead = DeadPatch::take(&mut request.ops, DisplayName::Live); + let has_live_changes = !request.ops.is_empty(); let mut batch = BatchBuilder::new(); let mut items = PropStatBuilder::default(); - let etag = if resource.is_container() { + let (is_success, etag) = if resource.is_container() { // Deserialize let book = archive .to_unarchived::() .caused_by(trc::location!())?; - let mut new_book = archive - .deserialize::() - .caused_by(trc::location!())?; let personal_id = access_token.personal_id(account_id, Collection::AddressBook); - // Remove properties - if !request.set_first && !request.remove.is_empty() { - remove_addressbook_properties( - personal_id, - &mut new_book, - std::mem::take(&mut request.remove), + // Apply live properties + let new_book = if has_live_changes { + let mut new_book = archive + .deserialize::() + .caused_by(trc::location!())?; + for op in request.ops { + match op { + PropertyUpdateOp::Set(value) => self.apply_addressbook_properties( + personal_id, + &mut new_book, + [value], + &mut items, + ), + PropertyUpdateOp::Remove(property) => remove_addressbook_properties( + personal_id, + &mut new_book, + [property], + &mut items, + ), + } + } + Some(new_book) + } else { + None + }; + + // Apply dead properties + let mut dead_write = dead + .apply( + self, + DeadTarget::container( + account_id, + Collection::AddressBook, + document_id, + book.inner.metadata_kinds(), + ), &mut items, - ); - } + ) + .await + .caused_by(trc::location!())?; - // Set properties - is_success = self.apply_addressbook_properties( - personal_id, - &mut new_book, - true, - request.set, - &mut items, - ); - - // Remove properties - if is_success && !request.remove.is_empty() { - remove_addressbook_properties( - personal_id, - &mut new_book, - request.remove, - &mut items, - ); - } - - if is_success { - new_book - .update( - access_token.account_tenant_ids(), - book, + if items.has_errors() { + (false, etag) + } else { + if let Some(write) = dead_write + .as_mut() + .and_then(|dead_write| dead_write.write.take()) + { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + if let Some(mut new_book) = new_book { + if let Some(dead_write) = &dead_write { + new_book.set_metadata_kinds(dead_write.kinds); + } + new_book + .update( + access_token.changed_by(), + book, + account_id, + document_id, + &mut batch, + ) + .caused_by(trc::location!())?; + } else if let Some(dead_write) = &dead_write { + AddressBook::rewrite_presence( + &book, + dead_write.kinds, account_id, document_id, &mut batch, ) - .caused_by(trc::location!())? - .etag() - } else { - book.etag().into() + .caused_by(trc::location!())?; + } + (true, batch.etag().unwrap_or(etag)) } } else { // Deserialize let card = archive .to_unarchived::() .caused_by(trc::location!())?; - let mut new_card = archive - .deserialize::() + + // Apply live properties + let new_card = if has_live_changes { + let mut new_card = archive + .deserialize::() + .caused_by(trc::location!())?; + for op in request.ops { + match op { + PropertyUpdateOp::Set(value) => { + self.apply_card_properties(&mut new_card, [value], &mut items) + } + PropertyUpdateOp::Remove(property) => { + remove_card_properties(&mut new_card, [property], &mut items) + } + } + } + Some(new_card) + } else { + None + }; + + // Apply dead properties + let mut dead_write = dead + .apply( + self, + DeadTarget::item( + account_id, + Collection::ContactCard, + document_id, + card.inner.metadata_kinds(), + ), + &mut items, + ) + .await .caused_by(trc::location!())?; - // Remove properties - if !request.set_first && !request.remove.is_empty() { - remove_card_properties( - &mut new_card, - std::mem::take(&mut request.remove), - &mut items, - ); - } - - // Set properties - is_success = self.apply_card_properties(&mut new_card, true, request.set, &mut items); - - // Remove properties - if is_success && !request.remove.is_empty() { - remove_card_properties(&mut new_card, request.remove, &mut items); - } - - if is_success { - new_card - .update( - access_token.account_tenant_ids(), - card, + if items.has_errors() { + (false, etag) + } else { + let mut new_etag = None; + if let Some(write) = dead_write + .as_mut() + .and_then(|dead_write| dead_write.write.take()) + { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + if let Some(mut new_card) = new_card { + if let Some(dead_write) = &dead_write { + new_card.set_metadata_kinds(dead_write.kinds); + } + new_etag = new_card + .update_meta( + access_token.changed_by(), + card, + account_id, + document_id, + None, + &mut batch, + ) + .caused_by(trc::location!())? + .into(); + } else if let Some(dead_write) = &dead_write { + ContactCard::rewrite_presence( + &card, + dead_write.kinds, account_id, document_id, &mut batch, ) - .caused_by(trc::location!())? - .etag() - } else { - card.etag().into() + .caused_by(trc::location!())?; + } + (true, new_etag.unwrap_or(etag)) } }; if is_success { - self.commit_batch(batch).await.caused_by(trc::location!())?; + if !batch.is_empty() { + self.commit_batch(batch).await.caused_by(trc::location!())?; + } + } else { + items.fail_dependencies(); } if headers.ret != Return::Minimal || !is_success { @@ -254,9 +345,9 @@ impl CardPropPatchRequestHandler for Server { .with_namespace(Namespace::CardDav) .to_string(), ) - .with_etag_opt(etag)) + .with_etag(etag)) } else { - Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) + Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag(etag)) } } @@ -264,12 +355,9 @@ impl CardPropPatchRequestHandler for Server { &self, personal_id: u32, address_book: &mut AddressBook, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool { - let mut has_errors = false; - + ) { for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { @@ -282,7 +370,6 @@ impl CardPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } } ( @@ -298,8 +385,6 @@ impl CardPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - - has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { @@ -318,34 +403,10 @@ impl CardPropPatchRequestHandler for Server { StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); - has_errors = true; } else { items.insert_ok(property.property); } } - (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) - if self.core.groupware.dead_property_size.is_some() => - { - if is_update { - address_book.dead_properties.remove_element(dead); - } - - if address_book.dead_properties.size() + values.size() + dead.size() - < self.core.groupware.dead_property_size.unwrap() - { - address_book - .dead_properties - .add_element(dead.clone(), values.0); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - has_errors = true; - } - } (_, DavValue::Null) => { items.insert_ok(property.property); } @@ -355,23 +416,17 @@ impl CardPropPatchRequestHandler for Server { StatusCode::CONFLICT, "Property cannot be modified", ); - has_errors = true; } } } - - !has_errors } fn apply_card_properties( &self, card: &mut ContactCard, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool { - let mut has_errors = false; - + ) { for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { @@ -384,34 +439,12 @@ impl CardPropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { card.created = dt; items.insert_ok(property.property); } - (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) - if self.core.groupware.dead_property_size.is_some() => - { - if is_update { - card.dead_properties.remove_element(dead); - } - - if card.dead_properties.size() + values.size() + dead.size() - < self.core.groupware.dead_property_size.unwrap() - { - card.dead_properties.add_element(dead.clone(), values.0); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - has_errors = true; - } - } (_, DavValue::Null) => { items.insert_ok(property.property); } @@ -421,29 +454,22 @@ impl CardPropPatchRequestHandler for Server { StatusCode::CONFLICT, "Property cannot be modified", ); - has_errors = true; } } } - - !has_errors } } fn remove_card_properties( card: &mut ContactCard, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::WebDav(WebDavProperty::DisplayName) => { card.display_name = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::DeadProperty(dead) => { - card.dead_properties.remove_element(dead); - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } _ => { items.insert_error_with_description( @@ -459,22 +485,18 @@ fn remove_card_properties( fn remove_addressbook_properties( personal_id: u32, book: &mut AddressBook, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::CardDav(CardDavProperty::AddressbookDescription) => { book.preferences_mut(personal_id).description = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } DavProperty::WebDav(WebDavProperty::DisplayName) => { book.preferences_mut(personal_id).name.clear(); - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::DeadProperty(dead) => { - book.dead_properties.remove_element(dead); - items.insert_with_status(property, StatusCode::NO_CONTENT); + items.insert_ok(property); } _ => { items.insert_error_with_description( diff --git a/crates/dav/src/card/query.rs b/crates/dav/src/card/query.rs index 75ccb2d9..dece2129 100644 --- a/crates/dav/src/card/query.rs +++ b/crates/dav/src/card/query.rs @@ -4,33 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::filter::CardFilterPlan; use crate::{ DavError, common::{ - AddressbookFilter, DavQuery, - propfind::{PropFindItem, PropFindRequestHandler}, + DavQuery, + propfind::PropFindRequestHandler, + search::{QueryScope, TextIndex}, uri::DavUriResource, }, }; -use calcard::vcard::{ - ArchivedVCard, ArchivedVCardEntry, ArchivedVCardParameter, VCardParameterName, VCardProperty, - VCardVersion, -}; -use common::{Server, auth::AccessToken}; +use calcard::vcard::{ArchivedVCard, VCardProperty, VCardVersion}; +use common::{DavResourcePath, Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, - schema::{ - property::CardDavPropertyName, - request::{AddressbookQuery, Filter, FilterOp, VCardPropertyWithGroup}, - response::MultiStatus, - }, + schema::{property::CardDavPropertyName, request::AddressbookQuery, response::MultiStatus}, }; use groupware::cache::GroupwareCache; use http_proto::HttpResponse; use hyper::StatusCode; use std::fmt::Write; +use store::write::SearchIndex; use trc::AddContext; use types::{acl::Acl, collection::SyncCollection}; +use utils::map::bitmap::Bitmap; pub(crate) trait CardQueryRequestHandler: Sync + Send { fn handle_card_query_request( @@ -55,7 +52,7 @@ impl CardQueryRequestHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -70,33 +67,34 @@ impl CardQueryRequestHandler for Server { return Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(MultiStatus::not_found(headers.uri).to_string())); }; - if !resource.is_container() { - return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); - } - // Obtain shared ids - let shared_ids = if !access_token.is_member(account_id) { - resources - .shared_items(access_token, [Acl::ReadItems], false) - .into() + let is_owner = access_token.is_member(account_id); + let read_items = Bitmap::from_iter([Acl::ReadItems]); + let is_visible = |item: &DavResourcePath<'_>| { + is_owner || resources.is_shared_item(&item.resource, access_token, &read_items, false) + }; + let scope = if resource.is_container() { + QueryScope::new( + resources + .children(resource.document_id()) + .filter(is_visible), + ) } else { - None + QueryScope::new(std::iter::once(resource).filter(is_visible)) }; - // Obtain document ids in folder - let mut items = Vec::with_capacity(16); - for resource in resources.children(resource.document_id()) { - if shared_ids - .as_ref() - .is_none_or(|ids| ids.contains(resource.document_id())) - { - items.push(PropFindItem::new( - resources.format_resource(resource), - account_id, - resource, - )); - } - } + let candidates = request + .filter + .candidates(&scope, TextIndex::new(self, SearchIndex::Contacts)); + let items = scope + .resolve( + self, + SearchIndex::Contacts, + account_id, + candidates, + &resources, + ) + .await; self.handle_dav_query( access_token, @@ -106,119 +104,34 @@ impl CardQueryRequestHandler for Server { } } -pub(crate) fn vcard_query(card: &ArchivedVCard, filters: &AddressbookFilter) -> bool { - let mut is_all = true; - let mut matches_one = false; - - for filter in filters { - match filter { - Filter::AnyOf => { - is_all = false; - } - Filter::AllOf => { - is_all = true; - } - Filter::Property { prop, op, .. } => { - let mut properties = find_properties(card, prop).peekable(); - let result = if properties.peek().is_some() { - properties.any(|entry| match op { - FilterOp::Exists => true, - FilterOp::Undefined => false, - FilterOp::TextMatch(text_match) => { - let mut matched_any = false; - - for value in entry.values.iter() { - if let Some(text) = value.as_text() - && text_match.matches(text) - { - matched_any = true; - break; - } - } - - matched_any - } - FilterOp::TimeRange(_) => false, - }) - } else { - matches!(op, FilterOp::Undefined) - }; - - if result { - matches_one = true; - } else if is_all { - return false; - } - } - Filter::Parameter { - prop, param, op, .. - } => { - let mut properties = find_properties(card, prop) - .filter_map(|entry| find_parameter(entry, param)) - .peekable(); - let result = if properties.peek().is_some() { - properties.any(|entry| match op { - FilterOp::Exists => true, - FilterOp::Undefined => false, - FilterOp::TextMatch(text_match) => { - if let Some(text) = entry.value.as_text() { - text_match.matches(text) - } else { - false - } - } - FilterOp::TimeRange(_) => false, - }) - } else { - matches!(op, FilterOp::Undefined) - }; - - if result { - matches_one = true; - } else if is_all { - return false; - } - } - Filter::Component { .. } => {} - } - } - - is_all || matches_one -} - -#[inline(always)] -fn find_properties<'x>( - card: &'x ArchivedVCard, - prop: &VCardPropertyWithGroup, -) -> impl Iterator { - card.entries - .iter() - .filter(move |entry| entry.name == prop.name && entry.group == prop.group) -} - -#[inline(always)] -fn find_parameter<'x>( - entry: &'x ArchivedVCardEntry, - name: &VCardParameterName, -) -> Option<&'x ArchivedVCardParameter> { - entry.params.iter().find(|param| param.name == *name) -} +const PARTIAL_VCARD_CAPACITY: usize = 128; pub(crate) fn serialize_vcard_with_props( card: &ArchivedVCard, props: &[CardDavPropertyName], version: VCardVersion, + size_hint: usize, ) -> String { - let mut vcard = String::with_capacity(128); + let mut vcard = String::with_capacity(if props.is_empty() { + size_hint + } else { + PARTIAL_VCARD_CAPACITY + }); if !props.is_empty() { let _ = write!(&mut vcard, "BEGIN:VCARD\r\n"); - let is_v4 = matches!(version, VCardVersion::V4_0); for entry in card.entries.iter() { for item in props { - if entry.name == item.name && entry.group == item.group { + if entry.name == item.name + && item.group.as_deref().is_none_or(|group| { + entry + .group + .as_ref() + .is_some_and(|entry_group| entry_group.eq_ignore_ascii_case(group)) + }) + { if item.name != VCardProperty::Version { - let _ = entry.write_to(&mut vcard, !item.no_value, is_v4); + let _ = entry.write_with_version(&mut vcard, !item.no_value, version); } else { let _ = write!(&mut vcard, "VERSION:{version}\r\n"); } diff --git a/crates/dav/src/card/update.rs b/crates/dav/src/card/update.rs index 0bb0c6fb..f3f73beb 100644 --- a/crates/dav/src/card/update.rs +++ b/crates/dav/src/card/update.rs @@ -4,35 +4,37 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::assert_is_unique_uid; +use super::{assert_is_unique_uid, is_same_uid}; use crate::{ DavError, DavErrorCondition, DavMethod, common::{ - ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, - fix_percent_encoding, }; -use calcard::{Entry, Parser}; +use calcard::{ + Entry, Parser, + vcard::{VCardProperty, VCardValue}, +}; use common::{DavName, Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{property::Rfc1123DateTime, response::CardCondition}, }; -use groupware::{cache::GroupwareCache, contact::ContactCard}; +use groupware::{ + cache::GroupwareCache, + contact::{ContactCard, ContactCardContent}, +}; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use store::write::BatchBuilder; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::ContactField, }; pub(crate) trait CardUpdateRequestHandler: Sync + Send { @@ -40,7 +42,7 @@ pub(crate) trait CardUpdateRequestHandler: Sync + Send { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], is_patch: bool, ) -> impl Future> + Send; } @@ -50,7 +52,7 @@ impl CardUpdateRequestHandler for Server { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], _is_patch: bool, ) -> crate::Result { // Validate URI @@ -60,18 +62,16 @@ impl CardUpdateRequestHandler for Server { .into_owned_uri()?; let account_id = resource.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await .caused_by(trc::location!())?; - let resource_name = fix_percent_encoding( - resource - .resource - .ok_or(DavError::Code(StatusCode::CONFLICT))?, - ); + let resource_name = resource + .resource + .ok_or(DavError::Code(StatusCode::CONFLICT))?; if bytes.len() > self.core.groupware.max_vcard_size { return Err(DavError::Condition(DavErrorCondition::new( @@ -79,7 +79,7 @@ impl CardUpdateRequestHandler for Server { CardCondition::MaxResourceSize(self.core.groupware.max_vcard_size as u32), ))); } - let vcard_raw = std::str::from_utf8(&bytes).map_err(|_| { + let vcard_raw = std::str::from_utf8(bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, @@ -89,7 +89,7 @@ impl CardUpdateRequestHandler for Server { ) })?; - let vcard = match Parser::new(vcard_raw).strict().entry() { + let mut vcard = match Parser::new(vcard_raw).strict().entry() { Entry::VCard(vcard) => vcard, _ => { return Err(DavError::Condition( @@ -101,8 +101,23 @@ impl CardUpdateRequestHandler for Server { )); } }; + let media_size = vcard.embedded_size(); + if self.core.groupware.max_media_size != 0 + && media_size > self.core.groupware.max_media_size + { + return Err(DavError::Condition( + DavErrorCondition::new( + StatusCode::PRECONDITION_FAILED, + CardCondition::MaxResourceSize(self.core.groupware.max_media_size as u32), + ) + .with_details(format!( + "The size of the embedded media ({media_size} bytes) exceeds the maximum of {} bytes.", + self.core.groupware.max_media_size + )), + )); + } - if let Some(resource) = resources.by_path(resource_name.as_ref()) { + if let Some(resource) = resources.by_path(resource_name) { if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } @@ -117,19 +132,23 @@ impl CardUpdateRequestHandler for Server { } // Update - let card_ = self - .store() - .get_value::>(ValueKey::archive( + let (card_, content_) = self + .archive_and_field( account_id, Collection::ContactCard, document_id, - )) + ContactField::Content.field(), + ) .await - .caused_by(trc::location!())? - .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + .caused_by(trc::location!())?; + let card_ = card_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; + let content_ = content_.ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let content = content_ + .to_unarchived::() + .caused_by(trc::location!())?; // Validate headers match self @@ -140,8 +159,8 @@ impl CardUpdateRequestHandler for Server { account_id, collection: Collection::ContactCard, document_id: Some(document_id), - etag: card.etag().into(), - path: resource_name.as_ref(), + etag: format!("\"{}\"", card.inner.etag.to_native()).into(), + path: resource_name, ..Default::default() }], Default::default(), @@ -153,8 +172,8 @@ impl CardUpdateRequestHandler for Server { Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { - let mut vcard = String::with_capacity(128); - let _ = card.inner.card.write_to( + let mut vcard = String::with_capacity(u32::from(card.inner.size) as usize); + let _ = content.inner.card.write_to( &mut vcard, headers .vcard_version @@ -163,7 +182,7 @@ impl CardUpdateRequestHandler for Server { return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_content_type("text/vcard; charset=utf-8") - .with_etag(card.etag()) + .with_etag(format!("\"{}\"", card.inner.etag.to_native())) .with_last_modified( Rfc1123DateTime::new(i64::from(card.inner.modified)).to_string(), ) @@ -174,49 +193,64 @@ impl CardUpdateRequestHandler for Server { } // Validate UID - match (card.inner.card.uid(), vcard.uid()) { - (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} - (None, None) | (None, Some(_)) => {} + let restored_uid = match ( + content.inner.card.uid().filter(|uid| !uid.is_empty()), + vcard.uid().filter(|uid| !uid.is_empty()), + ) { + (Some(stored_uid), Some(uid)) if is_same_uid(stored_uid, uid) => { + (stored_uid != uid).then_some(stored_uid) + } + (None, uid) => { + assert_is_unique_uid(&resources, parent_id, uid)?; + None + } _ => { return Err(DavError::Condition(DavErrorCondition::new( - StatusCode::PRECONDITION_FAILED, + StatusCode::FORBIDDEN, CardCondition::NoUidConflict(resources.format_resource(resource).into()), ))); } - } - - // Validate quota - let extra_bytes = - (bytes.len() as u64).saturating_sub(u32::from(card.inner.size) as u64); - if extra_bytes > 0 { - self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) - .await?; + }; + if let Some(stored_uid) = restored_uid + && let Some(value) = vcard + .entries + .iter_mut() + .find(|entry| entry.name == VCardProperty::Uid) + .and_then(|entry| entry.values.first_mut()) + { + *value = VCardValue::Text(stored_uid.to_string()); } // Build node - let mut new_card = card + let new_card = card .deserialize::() .caused_by(trc::location!())?; - new_card.size = bytes.len() as u32; - new_card.card = vcard; + let new_content = ContactCardContent { card: vcard }; + + let account = self.account(account_id).await?; // Prepare write batch let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); let etag = new_card - .update( - access_token.account_tenant_ids(), + .update_full( + new_content, + self.core.groupware.vcard_version, + access_token.changed_by(), card, + content.inner, account_id, document_id, + None, &mut batch, ) - .caused_by(trc::location!())? - .etag(); + .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); - Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) - } else if let Some((Some(parent), name)) = resources.map_parent(resource_name.as_ref()) { + Ok(HttpResponse::new(StatusCode::NO_CONTENT) + .with_etag_opt(restored_uid.is_none().then_some(etag))) + } else if let Some((Some(parent), name)) = resources.map_parent(resource_name) { if !parent.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } @@ -240,7 +274,7 @@ impl CardUpdateRequestHandler for Server { account_id, collection: resource.collection, document_id: Some(u32::MAX), - path: resource_name.as_ref(), + path: resource_name, ..Default::default() }], Default::default(), @@ -249,23 +283,14 @@ impl CardUpdateRequestHandler for Server { .await?; // Validate UID - assert_is_unique_uid( - self, - &resources, - account_id, - parent.document_id(), - vcard.uid(), - ) - .await?; + assert_is_unique_uid(&resources, parent.document_id(), vcard.uid())?; + + // Validate object quota + let account = self.account(account_id).await?; + self.assert_object_quota(&account, StorageQuota::MaxContactCards, 1, || { + resources.resources.count(false) + })?; - // Validate quota - if !bytes.is_empty() { - self.has_available_quota( - self.account(account_id).await?.as_ref(), - bytes.len() as u64, - ) - .await?; - } // Build node let card = ContactCard { @@ -273,31 +298,29 @@ impl CardUpdateRequestHandler for Server { name: name.to_string(), parent_id: parent.document_id(), }], - card: vcard, - size: bytes.len() as u32, ..Default::default() }; + let content = ContactCardContent { card: vcard }; // Prepare write batch let mut batch = BatchBuilder::new(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::ContactCard, 1) - .await - .caused_by(trc::location!())?; + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); + let document_id = batch.reserve_document_id(account_id, Collection::ContactCard); let etag = card .insert( - access_token.account_tenant_ids(), + content, + self.core.groupware.vcard_version, + access_token.changed_by(), account_id, document_id, + None, &mut batch, ) - .caused_by(trc::location!())? - .etag(); + .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; - self.notify_task_queue(); - Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) + Ok(HttpResponse::new(StatusCode::CREATED).with_etag(etag)) } else { Err(DavError::Code(StatusCode::CONFLICT))? } diff --git a/crates/dav/src/common/acl.rs b/crates/dav/src/common/acl.rs index b3a5c796..45f5866c 100644 --- a/crates/dav/src/common/acl.rs +++ b/crates/dav/src/common/acl.rs @@ -6,10 +6,19 @@ use super::ArchivedResource; use crate::{ - DavError, DavErrorCondition, DavResourceName, common::uri::DavUriResource, + DavError, DavErrorCondition, DavResourceName, + common::{propfind::ExpandedPrincipals, uri::DavUriResource}, + file::{DavFileResource, is_symlink}, principal::propfind::PrincipalPropFind, }; -use common::{DavResources, Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{ + Server, + auth::AccessToken, + sharing::{ + EffectiveAcl, + grants::{ShareUpdate, ShareViolation}, + }, +}; use dav_proto::{ RequestHeaders, schema::{ @@ -25,9 +34,9 @@ use hyper::StatusCode; use rkyv::vec::ArchivedVec; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; -use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use types::{ acl::{Acl, AclGrant, ArchivedAclGrant}, @@ -53,6 +62,7 @@ pub(crate) trait DavAclHandler: Sync + Send { fn validate_and_map_aces( &self, access_token: &AccessToken, + owner_id: u32, acl: dav_proto::schema::request::Acl, collection: Collection, ) -> impl Future>> + Send; @@ -62,20 +72,10 @@ pub(crate) trait DavAclHandler: Sync + Send { access_token: &AccessToken, account_id: u32, grants: &ArchivedVec, - expand: Option<&PropFind>, + expand: Option<(&PropFind, &mut ExpandedPrincipals)>, ) -> impl Future>> + Send; } -pub(crate) trait ResourceAcl { - fn validate_and_map_parent_acl( - &self, - access_token: &AccessToken, - is_member: bool, - parent_id: Option, - check_acls: impl Into> + Send, - ) -> crate::Result; -} - impl DavAclHandler for Server { async fn handle_acl_request( &self, @@ -98,21 +98,27 @@ impl DavAclHandler for Server { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let resources = self - .fetch_dav_resources(access_token.account_id(), account_id, collection.into()) + .fetch_groupware_resources(access_token.account_id(), account_id, collection.into()) .await .caused_by(trc::location!())?; let resource = resource_ .resource .and_then(|r| resources.by_path(r)) + .filter(|r| { + collection != Collection::FileNode + || (!is_symlink(r) + && (access_token.is_member(account_id) + || resources.file_discoverable(access_token, r.document_id()))) + }) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; - if !resource.resource.is_container() && !matches!(collection, Collection::FileNode) { + if !resource.resource.is_container() { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Fetch node let archive = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, collection, resource.document_id(), @@ -121,28 +127,48 @@ impl DavAclHandler for Server { .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; - let container = - ArchivedResource::from_archive(&archive, collection).caused_by(trc::location!())?; + let container = ArchivedResource::from_archive(&archive, None, collection) + .caused_by(trc::location!())?; // Validate ACL - let acls = container.acls().unwrap(); - if !access_token.is_member(account_id) - && !acls.effective_acl(access_token).contains(Acl::Share) - { + let acls = container + .acls() + .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; + let actor = (!access_token.is_member(account_id)).then(|| { + if collection == Collection::FileNode { + resources.file_acl(access_token, resource.document_id()) + } else { + acls.effective_acl(access_token) + } + }); + if actor.is_some_and(|actor| !actor.contains(Acl::Share)) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate ACEs let grants = self - .validate_and_map_aces(access_token, request, collection) + .validate_and_map_aces(access_token, account_id, request, collection) .await?; + ShareUpdate { + collection, + owner_id: account_id, + actor, + current: acls, + max_shares: self.core.groupware.max_shares_per_item, + } + .validate(&grants) + .map_err(|violation| { + DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + match violation { + ShareViolation::TooManyShares(_) => BaseCondition::LimitedNumberOfAces, + ShareViolation::OwnerShared => BaseCondition::AllowedPrincipal, + ShareViolation::RightNotHeld => BaseCondition::NoAceConflict, + }, + )) + })?; if grants.len() != acls.len() || acls.iter().zip(grants.iter()).any(|(a, b)| a != b) { - // Refresh ACLs - self.refresh_archived_acls(&grants, acls) - .await - .caused_by(trc::location!())?; - let mut batch = BatchBuilder::new(); match container { ArchivedResource::Calendar(calendar) => { @@ -152,7 +178,7 @@ impl DavAclHandler for Server { new_calendar.acls = grants; new_calendar .update( - access_token.account_tenant_ids(), + access_token.changed_by(), calendar, account_id, resource.document_id(), @@ -167,7 +193,7 @@ impl DavAclHandler for Server { new_book.acls = grants; new_book .update( - access_token.account_tenant_ids(), + access_token.changed_by(), book, account_id, resource.document_id(), @@ -181,7 +207,7 @@ impl DavAclHandler for Server { new_node.acls = grants; new_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), node, account_id, resource.document_id(), @@ -189,6 +215,9 @@ impl DavAclHandler for Server { &mut batch, ) .caused_by(trc::location!())?; + if resource.is_container() { + resources.log_descendant_updates(&mut batch, account_id, resource.path()); + } } _ => unreachable!(), } @@ -224,7 +253,7 @@ impl DavAclHandler for Server { let archive = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( uri.account_id, uri.collection, uri.resource, @@ -286,12 +315,20 @@ impl DavAclHandler for Server { Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] async fn validate_and_map_aces( &self, access_token: &AccessToken, + owner_id: u32, acl: dav_proto::schema::request::Acl, collection: Collection, ) -> crate::Result> { + if acl.aces.len() > self.core.groupware.max_shares_per_item { + return Err(DavError::Condition(DavErrorCondition::new( + StatusCode::FORBIDDEN, + BaseCondition::LimitedNumberOfAces, + ))); + } let mut grants = Vec::with_capacity(acl.aces.len()); for ace in acl.aces { if ace.invert { @@ -320,7 +357,7 @@ impl DavAclHandler for Server { }; let mut acls = Bitmap::::default(); - for privilege in privileges { + for privilege in &privileges { match privilege { Privilege::Read => { acls.insert(Acl::Read); @@ -328,23 +365,23 @@ impl DavAclHandler for Server { } Privilege::Write => { acls.insert(Acl::Modify); - acls.insert(Acl::Delete); acls.insert(Acl::AddItems); acls.insert(Acl::ModifyItems); acls.insert(Acl::RemoveItems); } Privilege::WriteContent => { - acls.insert(Acl::AddItems); - acls.insert(Acl::Modify); acls.insert(Acl::ModifyItems); } Privilege::WriteProperties => { acls.insert(Acl::Modify); } - Privilege::ReadCurrentUserPrivilegeSet - | Privilege::Unlock - | Privilege::Bind - | Privilege::Unbind => {} + Privilege::Bind => { + acls.insert(Acl::AddItems); + } + Privilege::Unbind => { + acls.insert(Acl::RemoveItems); + } + Privilege::ReadCurrentUserPrivilegeSet | Privilege::Unlock => {} Privilege::All => { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, @@ -423,24 +460,18 @@ impl DavAclHandler for Server { )) })?; - // Verify that the principal is a valid principal - /*let principal = self - .directory() - .query(QueryParams::id(principal_id).with_return_member_of(false)) + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + let grantee = self + .try_account(principal_id) .await .caused_by(trc::location!())? .ok_or_else(|| { DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, - BaseCondition::AllowedPrincipal, + BaseCondition::RecognizedPrincipal, )) })?; - if !matches!(principal.typ(), Type::Individual | Type::Group) { - return Err(DavError::Condition(DavErrorCondition::new( - StatusCode::FORBIDDEN, - BaseCondition::AllowedPrincipal, - ))); - }*/ + grants.push(AclGrant { account_id: principal_id, @@ -456,16 +487,21 @@ impl DavAclHandler for Server { access_token: &AccessToken, account_id: u32, grants: &ArchivedVec, - expand: Option<&PropFind>, + mut expand: Option<(&PropFind, &mut ExpandedPrincipals)>, ) -> crate::Result> { let mut aces = Vec::with_capacity(grants.len()); if access_token.is_member(account_id) || grants.effective_acl(access_token).contains(Acl::Share) { for grant in grants.iter() { + let privileges = current_user_privilege_set(Bitmap::::from(&grant.grants)); + if privileges.is_empty() { + continue; + } let grant_account_id = u32::from(grant.account_id); - let principal = if let Some(expand) = expand { - self.expand_principal(access_token, grant_account_id, expand) + let principal = if let Some((propfind, principals)) = &mut expand { + principals + .expand(self, access_token, grant_account_id, propfind) .await? .map(Principal::Response) .unwrap_or_else(|| { @@ -474,25 +510,24 @@ impl DavAclHandler for Server { DavResourceName::Principal.base_path(), ))) }) - } else { - let grant_account = self - .account(grant_account_id) - .await - .caused_by(trc::location!())?; - + } else if let Some(grant_account) = self + .try_account(grant_account_id) + .await + .caused_by(trc::location!())? + { Principal::Href(Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(grant_account.name(), RFC_3986), ))) + } else { + Principal::Href(Href(format!( + "{}/_{grant_account_id}/", + DavResourceName::Principal.base_path(), + ))) }; - aces.push(Ace::new( - principal, - GrantDeny::grant(current_user_privilege_set(Bitmap::::from( - &grant.grants, - ))), - )); + aces.push(Ace::new(principal, GrantDeny::grant(privileges))); } } @@ -500,33 +535,6 @@ impl DavAclHandler for Server { } } -impl ResourceAcl for DavResources { - fn validate_and_map_parent_acl( - &self, - access_token: &AccessToken, - is_member: bool, - parent_id: Option, - check_acls: impl Into> + Send, - ) -> crate::Result { - match parent_id { - Some(parent_id) => { - if is_member || self.has_access_to_container(access_token, parent_id, check_acls) { - Ok(parent_id + 1) - } else { - Err(DavError::Code(StatusCode::FORBIDDEN)) - } - } - None => { - if is_member { - Ok(0) - } else { - Err(DavError::Code(StatusCode::FORBIDDEN)) - } - } - } - } -} - pub(crate) trait Privileges { fn current_privilege_set( &self, @@ -552,31 +560,46 @@ impl Privileges for AccessToken { } pub(crate) fn current_user_privilege_set(acl_bitmap: Bitmap) -> Vec { - let mut acls = AHashSet::with_capacity(16); - for grant in acl_bitmap { - match grant { - Acl::Read | Acl::ReadItems => { - acls.insert(Privilege::Read); - acls.insert(Privilege::ReadCurrentUserPrivilegeSet); - } - Acl::Modify => { - acls.insert(Privilege::WriteProperties); - } - Acl::ModifyItems => { - acls.insert(Privilege::WriteContent); - } - Acl::Delete | Acl::RemoveItems => { - acls.insert(Privilege::Write); - } - Acl::Share => { - acls.insert(Privilege::ReadAcl); - acls.insert(Privilege::WriteAcl); - } - Acl::SchedulingReadFreeBusy => { - acls.insert(Privilege::ReadFreeBusy); - } - _ => {} + let mut privileges = Vec::with_capacity(13); + if acl_bitmap.contains(Acl::Read) || acl_bitmap.contains(Acl::ReadItems) { + privileges.push(Privilege::Read); + privileges.push(Privilege::ReadCurrentUserPrivilegeSet); + } + let mut write = true; + for (acl, privilege) in [ + (Acl::Modify, Privilege::WriteProperties), + (Acl::ModifyItems, Privilege::WriteContent), + (Acl::AddItems, Privilege::Bind), + (Acl::RemoveItems, Privilege::Unbind), + ] { + if acl_bitmap.contains(acl) { + privileges.push(privilege); + } else { + write = false; } } - acls.into_iter().collect() + if write { + privileges.push(Privilege::Write); + } + if acl_bitmap.contains(Acl::Share) { + privileges.push(Privilege::ReadAcl); + privileges.push(Privilege::WriteAcl); + } + let free_busy = acl_bitmap.contains(Acl::SchedulingReadFreeBusy); + let invite = acl_bitmap.contains(Acl::SchedulingInvite); + let reply = acl_bitmap.contains(Acl::SchedulingReply); + if free_busy { + privileges.push(Privilege::ReadFreeBusy); + } + if invite { + privileges.push(Privilege::ScheduleDeliverInvite); + } + if reply { + privileges.push(Privilege::ScheduleDeliverReply); + } + if free_busy && invite && reply { + privileges.push(Privilege::ScheduleDeliver); + } + privileges } + diff --git a/crates/dav/src/common/dead/containers.rs b/crates/dav/src/common/dead/containers.rs new file mode 100644 index 00000000..179f2a0d --- /dev/null +++ b/crates/dav/src/common/dead/containers.rs @@ -0,0 +1,262 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{ + Server, + storage::metadata::{ + MetadataContainers, MetadataLog, MetadataWrite, StoredContainer, StoredEntry, + }, +}; +use store::{ + ahash::AHashMap, + roaring::RoaringBitmap, + write::{BatchBuilder, PendingId, metadata::MetadataBuf}, +}; +use trc::AddContext; +use types::{ + collection::Collection, + metadata::{EncodedMetadata, MetadataKinds}, +}; + +type GroupKey = (u32, Collection); + +#[derive(Debug, Default)] +pub(crate) struct ContainerRequest { + groups: AHashMap, +} + +#[derive(Debug, Default)] +pub(crate) struct DeadContainers { + groups: Vec<(GroupKey, MetadataContainers)>, +} + +#[derive(Debug)] +pub(crate) struct ContainerWrites { + account_id: u32, + collection: Collection, +} + +impl ContainerRequest { + pub fn insert(&mut self, account_id: u32, collection: Collection, document_id: u32) { + self.groups + .entry((account_id, collection)) + .or_default() + .insert(document_id); + } + + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + pub async fn load(self, server: &Server) -> trc::Result { + let mut groups = Vec::new(); + for (key @ (account_id, collection), documents) in self.groups { + let containers = server + .load_metadata_containers(account_id, collection, &documents) + .await + .caused_by(trc::location!())?; + if !containers.is_empty() { + groups.push((key, containers)); + } + } + Ok(DeadContainers { groups }) + } +} + +impl DeadContainers { + pub fn get( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + ) -> Option<&MetadataBuf> { + self.group(account_id, collection) + .and_then(|containers| containers.get(document_id)) + } + + pub fn stored_len(&self) -> u64 { + self.groups + .iter() + .flat_map(|(_, containers)| containers.iter()) + .map(|(_, container)| u64::from(container.stored_len())) + .sum() + } + + pub fn stored_entries( + &self, + account_id: u32, + collection: Collection, + ) -> impl Iterator + '_ { + self.group(account_id, collection) + .into_iter() + .flat_map(MetadataContainers::iter) + .map(|(document_id, container)| StoredEntry::from_container(document_id, container)) + } + + fn group(&self, account_id: u32, collection: Collection) -> Option<&MetadataContainers> { + self.groups + .iter() + .find(|(key, _)| *key == (account_id, collection)) + .map(|(_, containers)| containers) + } +} + +impl ContainerWrites { + pub fn new(account_id: u32, collection: Collection) -> Self { + ContainerWrites { + account_id, + collection, + } + } + + pub async fn copy( + &mut self, + server: &Server, + source: &MetadataBuf, + document_id: impl Into, + previous: Option, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.build( + server, + document_id.into(), + previous, + EncodedMetadata::from_view(&source.view()), + batch, + ) + .await + } + + pub async fn clear( + &mut self, + server: &Server, + document_id: impl Into, + previous: StoredContainer, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.build(server, document_id.into(), Some(previous), None, batch) + .await + } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + pub async fn release( + &mut self, + server: &Server, + entry: StoredEntry, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + batch + .with_account_id(self.account_id) + .with_collection(self.collection); + #[cfg(not(feature = "enterprise"))] + entry.release(batch); + Ok(()) + } + + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + async fn build( + &mut self, + server: &Server, + document_id: PendingId, + previous: Option, + next: Option, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + MetadataWrite { + account_id: self.account_id, + collection: self.collection, + previous, + next, + log: MetadataLog::None, + } + .build(document_id, batch) + .caused_by(trc::location!())?; + Ok(()) + } +} + +pub(crate) fn copy_growth(source: &MetadataBuf, previous: Option<&StoredEntry>) -> u64 { + u64::from(source.stored_len()) + .saturating_sub(previous.map_or(0, |previous| u64::from(previous.size))) +} + +pub(crate) async fn check_growth(server: &Server, account_id: u32, growth: u64) -> trc::Result<()> { + if growth > 0 { + let account = server + .account(account_id) + .await + .caused_by(trc::location!())?; + server + .has_available_quota(&account, growth) + .await + .caused_by(trc::location!()) + } else { + Ok(()) + } +} + +pub(crate) async fn read_container( + server: &Server, + account_id: u32, + collection: Collection, + document_id: u32, + kinds: MetadataKinds, +) -> trc::Result> { + if kinds.is_empty() { + Ok(None) + } else { + server + .metadata_container(account_id, collection, document_id) + .await + } +} + +pub(crate) async fn stored_entry_of( + server: &Server, + account_id: u32, + collection: Collection, + document_id: u32, + kinds: MetadataKinds, +) -> trc::Result> { + if kinds.is_empty() { + return Ok(None); + } + server + .stored_metadata_entries( + account_id, + collection, + &RoaringBitmap::from_iter([document_id]), + ) + .await + .map(|entries| entries.get(document_id).copied()) +} + +pub(crate) fn edited_container( + entry: StoredEntry, + account_id: u32, + kinds: MetadataKinds, +) -> trc::Result { + entry + .hash + .map(|hash| StoredContainer { + size: entry.size, + kinds, + hash, + }) + .ok_or_else(|| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Metadata container carries no trailer") + .account_id(account_id) + .document_id(entry.document_id) + .caused_by(trc::location!()) + }) +} + +pub(crate) fn container_kinds(container: Option<&MetadataBuf>) -> MetadataKinds { + container.map_or(MetadataKinds::NONE, MetadataBuf::kinds) +} diff --git a/crates/dav/src/common/dead/mod.rs b/crates/dav/src/common/dead/mod.rs new file mode 100644 index 00000000..ece9190a --- /dev/null +++ b/crates/dav/src/common/dead/mod.rs @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod containers; +mod patch; + +pub(crate) use containers::{ + ContainerRequest, ContainerWrites, DeadContainers, check_growth, container_kinds, copy_growth, + edited_container, read_container, stored_entry_of, +}; +pub(crate) use patch::{DeadPatch, DisplayName}; + +use common::storage::metadata::MetadataLog; +use store::write::PendingId; +use types::{collection::Collection, metadata::MetadataKinds}; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct DeadTarget { + pub account_id: u32, + pub collection: Collection, + pub document_id: PendingId, + pub kinds: MetadataKinds, + pub log: MetadataLog, +} + +impl DeadTarget { + pub fn item( + account_id: u32, + collection: Collection, + document_id: impl Into, + kinds: MetadataKinds, + ) -> Self { + DeadTarget { + account_id, + collection, + document_id: document_id.into(), + kinds, + log: MetadataLog::Item { prefix: None }, + } + } + + pub fn container( + account_id: u32, + collection: Collection, + document_id: impl Into, + kinds: MetadataKinds, + ) -> Self { + DeadTarget { + account_id, + collection, + document_id: document_id.into(), + kinds, + log: MetadataLog::Container, + } + } +} diff --git a/crates/dav/src/common/dead/patch.rs b/crates/dav/src/common/dead/patch.rs new file mode 100644 index 00000000..c85113e9 --- /dev/null +++ b/crates/dav/src/common/dead/patch.rs @@ -0,0 +1,352 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::DeadTarget; +use crate::PropStatBuilder; +use common::{ + Server, + storage::{ + dav::{DISPLAY_NAME_PROPERTY, FilePresence}, + metadata::{ContainerChange, MetadataWrite}, + }, +}; +use dav_proto::schema::{ + property::{DavProperty, DavValue, WebDavProperty}, + request::{DavPropertyValue, PropertyUpdateOp}, + response::BaseCondition, +}; +use hyper::StatusCode; +use std::{borrow::Cow, mem}; +use store::write::PendingId; +use types::metadata::{ + EncodedMetadata, MetadataBuilder, MetadataKinds, MetadataScope, MetadataView, XmlError, + XmlName, XmlNode, XmlValue, +}; + + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DisplayName { + Live, + Stored, +} + +#[derive(Debug)] +enum DeadOp { + Set { + property: DavProperty, + value: XmlValue<'static>, + }, + Remove { + property: DavProperty, + }, +} + +#[derive(Debug, Default)] +pub(crate) struct DeadPatch { + ops: Vec, +} + +#[derive(Debug)] +pub(crate) struct DeadWrite { + pub write: Option, + pub kinds: MetadataKinds, + pub file_presence: FilePresence, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Outcome { + Set, + Removed, + TooLarge, + Invalid(XmlError), + StorageFull, + OverQuota, +} + +impl DeadPatch { + pub fn take(ops: &mut Vec, display_name: DisplayName) -> Self { + let mut dead = Vec::new(); + ops.retain_mut(|op| { + let taken = match op { + PropertyUpdateOp::Set(value) => take_set(value, display_name), + PropertyUpdateOp::Remove(property) => take_remove(property, display_name), + }; + taken.map(|op| dead.push(op)).is_none() + }); + DeadPatch { ops: dead } + } + + pub fn take_values(values: &mut Vec, display_name: DisplayName) -> Self { + let mut dead = Vec::new(); + values.retain_mut(|value| { + take_set(value, display_name) + .map(|op| dead.push(op)) + .is_none() + }); + DeadPatch { ops: dead } + } + + pub fn is_empty(&self) -> bool { + self.ops.is_empty() + } + + fn fail(self, items: &mut PropStatBuilder) { + for op in self.ops { + items.insert_with_status(op.into_property(), StatusCode::FAILED_DEPENDENCY); + } + } + + pub async fn apply( + self, + server: &Server, + target: DeadTarget, + items: &mut PropStatBuilder, + ) -> trc::Result> { + if self.is_empty() { + Ok(None) + } else if items.has_errors() { + self.fail(items); + Ok(None) + } else { + self.prepare(server, target, items).await + } + } + + async fn prepare( + self, + server: &Server, + target: DeadTarget, + items: &mut PropStatBuilder, + ) -> trc::Result> { + let owner = server.account(target.account_id).await?; + let current = match target.document_id { + PendingId::Assigned(document_id) if !target.kinds.is_empty() => { + server + .metadata_container(target.account_id, target.collection, document_id) + .await? + } + _ => None, + }; + let current_view = current.as_ref().map(|current| current.view()); + let mut builder = current_view + .as_ref() + .map_or_else(MetadataBuilder::new, MetadataBuilder::from_view); + let limits = server.core.metadata.limits(); + let live_property_size = server.core.groupware.live_property_size; + let previous_entries = builder.len(); + let previous_len = current_view + .as_ref() + .map_or(0, |current| current.as_bytes().len()); + let entry_bound = limits.entry_bound(previous_entries); + + let ops = self.ops; + let mut outcomes = Vec::with_capacity(ops.len()); + let mut is_full = false; + let mut pending_removes = ops + .iter() + .filter(|op| matches!(op, DeadOp::Remove { .. })) + .count(); + for op in &ops { + outcomes.push(match op { + DeadOp::Set { .. } if is_full => Outcome::StorageFull, + DeadOp::Set { property, value } => { + if is_live_display_name(property) && text_len(value) > live_property_size { + Outcome::TooLarge + } else { + match builder.set_dav(dead_name(property), value) { + Ok(_) if entry_bound.check(builder.len(), pending_removes).is_err() => { + builder.remove_dav(&dead_name(property)); + is_full = true; + Outcome::StorageFull + } + Ok(size) if limits.check_entry_size(size).is_ok() => Outcome::Set, + Ok(_) => Outcome::TooLarge, + Err(err) => Outcome::Invalid(err), + } + } + } + DeadOp::Remove { property } => { + pending_removes = pending_removes.saturating_sub(1); + builder.remove_dav(&dead_name(property)); + Outcome::Removed + } + }); + } + + let mut change = None; + if outcomes.iter().all(|outcome| outcome.is_success()) { + change = ContainerChange::new(current.as_ref(), builder.encode(), builder.edit()); + let failure = match &change { + Some(change) + if change.next().is_some_and(|next| { + limits + .check_edit(MetadataScope::Shared, previous_len, previous_entries, next) + .is_err() + }) => + { + Some(Outcome::StorageFull) + } + Some(change) if !server.has_metadata_quota(&owner, change).await? => { + Some(Outcome::OverQuota) + } + _ => None, + }; + if let Some(failure) = failure { + for outcome in &mut outcomes { + if *outcome == Outcome::Set { + *outcome = failure; + } + } + } + } + drop(builder); + + let is_success = outcomes.iter().all(|outcome| outcome.is_success()); + for (op, outcome) in ops.into_iter().zip(outcomes) { + outcome.report(op.into_property(), items); + } + if !is_success { + return Ok(None); + } + + let view = match &change { + Some(change) => change.next().map(EncodedMetadata::view), + None => current_view, + }; + let kinds = view + .as_ref() + .map_or(MetadataKinds::NONE, MetadataView::kinds); + let file_presence = view + .as_ref() + .map_or(FilePresence::NONE, FilePresence::from_view); + Ok(Some(DeadWrite { + write: change.map(|change| change.into_write(&owner, target.collection, target.log)), + kinds, + file_presence, + })) + } +} + +impl DeadOp { + fn into_property(self) -> DavProperty { + match self { + DeadOp::Set { property, .. } | DeadOp::Remove { property, .. } => property, + } + } +} + +impl Outcome { + fn is_success(self) -> bool { + matches!(self, Outcome::Set | Outcome::Removed) + } + + fn report(self, property: DavProperty, items: &mut PropStatBuilder) { + match self { + Outcome::Set => { + items.insert_ok(property); + } + Outcome::Removed => { + items.insert_ok(property); + } + Outcome::TooLarge => { + items.insert_error_with_description( + property, + StatusCode::INSUFFICIENT_STORAGE, + "Property value is too long", + ); + } + Outcome::Invalid(err) => { + items.insert_error_with_description( + property, + StatusCode::CONFLICT, + err.to_string(), + ); + } + Outcome::StorageFull => { + items.insert_error_with_description( + property, + StatusCode::INSUFFICIENT_STORAGE, + "Property storage limit exceeded", + ); + } + Outcome::OverQuota => { + items.insert_precondition_failed( + property, + StatusCode::INSUFFICIENT_STORAGE, + BaseCondition::QuotaNotExceeded, + ); + } + } + } +} + +fn take_set(value: &mut DavPropertyValue, display_name: DisplayName) -> Option { + let dead = match (&value.property, &mut value.value, display_name) { + (DavProperty::Dead(_), DavValue::Dead(dead), _) => mem::take(dead.as_mut()), + ( + DavProperty::WebDav(WebDavProperty::DisplayName), + DavValue::String(text), + DisplayName::Stored, + ) => text_value(mem::take(text)), + (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::Null, DisplayName::Stored) => { + XmlValue::default() + } + _ => return None, + }; + Some(DeadOp::Set { + property: take_property(&mut value.property), + value: dead, + }) +} + +fn take_remove(property: &mut DavProperty, display_name: DisplayName) -> Option { + match (&*property, display_name) { + (DavProperty::Dead(_), _) + | (DavProperty::WebDav(WebDavProperty::DisplayName), DisplayName::Stored) => { + Some(DeadOp::Remove { + property: take_property(property), + }) + } + _ => None, + } +} + +fn take_property(property: &mut DavProperty) -> DavProperty { + mem::replace(property, DavProperty::WebDav(WebDavProperty::GetETag)) +} + +fn dead_name(property: &DavProperty) -> XmlName<'_> { + match property { + DavProperty::Dead(name) => XmlName::borrowed(name.namespace(), name.name()), + _ => DISPLAY_NAME_PROPERTY, + } +} + +fn text_value(text: String) -> XmlValue<'static> { + XmlValue { + children: if text.is_empty() { + Vec::new() + } else { + vec![XmlNode::Text(Cow::Owned(text))] + }, + ..Default::default() + } +} + +fn is_live_display_name(property: &DavProperty) -> bool { + matches!(property, DavProperty::WebDav(WebDavProperty::DisplayName)) +} + +fn text_len(value: &XmlValue<'_>) -> usize { + value + .children + .iter() + .map(|child| match child { + XmlNode::Text(text) => text.len(), + XmlNode::Element(_) => 0, + }) + .sum() +} diff --git a/crates/dav/src/common/lock.rs b/crates/dav/src/common/lock.rs index a0c9d330..7db94340 100644 --- a/crates/dav/src/common/lock.rs +++ b/crates/dav/src/common/lock.rs @@ -4,28 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::ETag; -use super::uri::{DavUriResource, OwnedUri, UriResource, Urn}; +use super::quoted_etag; +use super::uri::{DavUriResource, OwnedUri, SyncToken, UriResource, Urn, canonical_dav_uri}; +use crate::file::is_symlink; use crate::{DavError, DavErrorCondition, DavMethod}; use common::KV_LOCK_DAV; -use common::{Server, auth::AccessToken}; +use common::{GroupwareResources, Server, auth::AccessToken}; use dav_proto::schema::property::{ActiveLock, LockScope, WebDavProperty}; use dav_proto::schema::request::DavPropertyValue; use dav_proto::schema::response::{BaseCondition, List, PropResponse}; use dav_proto::{Condition, Depth, Timeout}; use dav_proto::{RequestHeaders, schema::request::LockInfo}; -use groupware::cache::GroupwareCache; +use groupware::{cache::GroupwareCache, calendar::privacy::EventPrivacy}; use http_proto::HttpResponse; use hyper::StatusCode; -use std::collections::HashMap; -use store::ValueKey; +use std::borrow::Cow; +use std::sync::Arc; use store::dispatch::lookup::KeyValue; use store::write::serialize::rkyv_deserialize; -use store::write::{AlignedBytes, Archive, Archiver, now}; +use store::write::{Archive, ArchiveBytes, ArchiveCompression, Archiver, Compression, now}; use store::{Serialize, U32_LEN}; use trc::AddContext; -use types::collection::Collection; -use types::dead_property::DeadProperty; +use types::collection::{Collection, SyncCollection}; +use types::metadata::{DavValueView, EncodedDavValue, XmlValue}; +use utils::map::vec_map::VecMap; #[derive(Debug, Default, Clone)] pub struct ResourceState<'x> { @@ -40,7 +42,11 @@ pub struct ResourceState<'x> { #[derive(Debug, Default, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub(crate) struct LockData { - locks: HashMap, + locks: VecMap, +} + +impl ArchiveCompression for LockData { + const COMPRESSION: Compression = Compression::None; } #[derive(Debug, Default, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] @@ -54,7 +60,7 @@ pub(crate) struct LockItem { expires: u64, depth_infinity: bool, exclusive: bool, - owner_dav: Option, + owner_dav: Option>, } struct LockCache<'x> { @@ -65,7 +71,8 @@ struct LockCache<'x> { enum LockArchive<'x> { Unarchived(&'x ArchivedLockData), - Archived(Archive), + Archived(Archive), + Absent, } #[derive(Default)] @@ -117,14 +124,6 @@ impl LockRequestHandler for Server { return Err(DavError::Code(StatusCode::FORBIDDEN)); } - let resources = vec![ResourceState { - account_id, - collection: resource.collection, - path: resource_path, - ..Default::default() - }]; - - let mut base_path = None; let is_lock_request = !matches!(lock_info, LockRequest::Unlock); let if_lock_token = headers .if_ @@ -138,9 +137,40 @@ impl LockRequestHandler for Server { } }) .unwrap_or_default(); + let is_file = resource.collection == Collection::FileNode; + let is_new_lock = is_lock_request && if_lock_token == 0; + let lock_status = if is_file || is_new_lock { + match self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::from(resource.collection), + ) + .await + .caused_by(trc::location!())? + .by_path(resource_path) + { + Some(existing) if is_file && is_symlink(&existing) => { + return Err(DavError::Code(StatusCode::NOT_FOUND)); + } + None if is_new_lock => StatusCode::CREATED, + _ => StatusCode::OK, + } + } else { + StatusCode::OK + }; + + let resources = vec![ResourceState { + account_id, + collection: resource.collection, + path: resource_path, + ..Default::default() + }]; + + let mut base_path = None; let mut lock_data = if let Some(lock_data) = self .in_memory_store() - .key_get::>(resource_hash.as_slice()) + .key_get::>(resource_hash.as_slice()) .await .caused_by(trc::location!())? { @@ -152,7 +182,7 @@ impl LockRequestHandler for Server { access_token, headers, resources, - LockCaches::new_shared(account_id, resource.collection, lock_data), + LockCaches::new_shared(account_id, resource.collection, Some(lock_data)), if is_lock_request { DavMethod::LOCK } else { @@ -188,11 +218,10 @@ impl LockRequestHandler for Server { } // Validate lock_info - if lock_info.owner.as_ref().is_some_and(|o| { - o.size() > self.core.groupware.dead_property_size.unwrap_or(512) - }) { - return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); - } + check_owner_size( + lock_info.owner.as_ref(), + self.core.metadata.limits().max_entry_size, + )?; if self.core.groupware.max_locks_per_user > 0 && lock_data @@ -217,7 +246,7 @@ impl LockRequestHandler for Server { access_token, headers, resources, - Default::default(), + LockCaches::new_shared(account_id, resource.collection, None), DavMethod::LOCK, ) .await?; @@ -252,10 +281,7 @@ impl LockRequestHandler for Server { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } } else { - let locks = lock_data - .locks - .entry(resource_path.to_string()) - .or_insert_with(Default::default); + let locks = lock_data.locks.get_mut_or_insert(resource_path.to_string()); locks.0.push(LockItem::default()); locks.0.last_mut().unwrap() }; @@ -263,35 +289,36 @@ impl LockRequestHandler for Server { lock_item.expires = expires; if let LockRequest::Lock(lock_info) = lock_info { // Validate lock_info - if lock_info.owner.as_ref().is_some_and(|o| { - o.size() > self.core.groupware.dead_property_size.unwrap_or(512) - }) { - return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); - } + check_owner_size( + lock_info.owner.as_ref(), + self.core.metadata.limits().max_entry_size, + )?; lock_item.lock_id = store::rand::random::() ^ expires; lock_item.owner = access_token.account_id(); lock_item.depth_infinity = matches!(headers.depth, Depth::Infinity); - lock_item.owner_dav = lock_info.owner; + lock_item.owner_dav = lock_info + .owner + .as_ref() + .map(XmlValue::encode) + .transpose() + .map_err(|_| DavError::Code(StatusCode::BAD_REQUEST))? + .map(Vec::into_boxed_slice); lock_item.exclusive = matches!(lock_info.lock_scope, LockScope::Exclusive); } let base_path = base_path.get_or_insert_with(|| headers.base_uri().unwrap_or_default()); let active_lock = lock_item.to_active_lock(format!("{base_path}/{resource_path}")); - HttpResponse::new(if if_lock_token == 0 { - StatusCode::CREATED - } else { - StatusCode::OK - }) - .with_lock_token(&active_lock.lock_token.as_ref().unwrap().0) - .with_xml_body( - PropResponse::new(vec![DavPropertyValue::new( - WebDavProperty::LockDiscovery, - vec![active_lock], - )]) - .to_string(), - ) + HttpResponse::new(lock_status) + .with_lock_token(&active_lock.lock_token.as_ref().unwrap().0) + .with_xml_body( + PropResponse::new(vec![DavPropertyValue::new( + WebDavProperty::LockDiscovery, + vec![active_lock], + )]) + .to_string(), + ) } else { let lock_id = headers .lock_token @@ -322,7 +349,7 @@ impl LockRequestHandler for Server { .serialize() .caused_by(trc::location!())?, ) - .expires(max_expire), + .expires(max_expire.saturating_sub(now)), ) .await .caused_by(trc::location!())?; @@ -345,6 +372,11 @@ impl LockRequestHandler for Server { method: DavMethod, ) -> crate::Result<()> { let no_if_headers = headers.if_.is_empty(); + let if_resources = headers + .if_ + .iter() + .map(|if_| if_.resource.map(canonical_if_resource)) + .collect::>(); match method { DavMethod::GET | DavMethod::HEAD if no_if_headers => { // Return early for GET/HEAD requests without If headers @@ -382,32 +414,35 @@ impl LockRequestHandler for Server { DavMethod::GET | DavMethod::HEAD | DavMethod::LOCK | DavMethod::UNLOCK ) { let mut base_path = None; + let submitted_tokens = headers + .if_ + .iter() + .flat_map(|if_| if_.list.iter()) + .filter_map(|cond| match cond { + Condition::StateToken { token, .. } => Some(*token), + _ => None, + }) + .collect::>(); - 'outer: for (pos, resource) in resources.iter().enumerate() { + for (pos, resource) in resources.iter().enumerate() { if pos == 0 && matches!(method, DavMethod::COPY) { continue; } if let Some(idx) = locks.find_cache_pos(self, resource).await? { - let mut failed_locks = Vec::new(); - - for (lock_path, lock_item) in locks.find_locks_by_pos(idx, resource, true)? { - let lock_token = lock_item.urn().to_string(); - if headers.if_.iter().any(|if_| { - if_.resource - .is_none_or(|r| { - r.trim_end_matches('/').ends_with(lock_path)}) - && if_.list.iter().any(|cond| matches!(cond, Condition::StateToken { token, .. } if token == &lock_token)) - }) { - break 'outer; - } else { - let base_path = base_path.get_or_insert_with(|| { - headers.base_uri() - .unwrap_or_default() - }); - failed_locks.push(format!("{base_path}/{lock_path}").into()); - } - } + let failed_locks = locks + .find_locks_by_pos(idx, resource, true)? + .into_iter() + .filter(|(_, lock_item)| { + let lock_token = lock_item.urn().to_string(); + !submitted_tokens.contains(&lock_token.as_str()) + }) + .map(|(lock_path, _)| { + let base_path = base_path + .get_or_insert_with(|| headers.base_uri().unwrap_or_default()); + format!("{base_path}/{lock_path}").into() + }) + .collect::>(); if !failed_locks.is_empty() { lock_response = Err(DavErrorCondition::new( @@ -432,15 +467,16 @@ impl LockRequestHandler for Server { path: "", ..Default::default() }; + let mut cached_resources: Option<(u32, SyncCollection, Arc)> = None; - 'outer: for if_ in &headers.if_ { + 'outer: for (if_, if_resource) in headers.if_.iter().zip(&if_resources) { if if_.list.is_empty() { continue; } let mut resource_state = &mut resource_not_found; - if let Some(resource) = if_.resource { + if let Some(resource) = if_resource.as_deref() { if let Some(resource) = self .validate_uri(access_token, resource) .await @@ -523,17 +559,38 @@ impl LockRequestHandler for Server { if let Some(document_id) = resource_state.document_id.filter(|&id| id != u32::MAX) - && let Some(archive) = self - .store() - .get_value::>(ValueKey::archive( - resource_state.account_id, - resource_state.collection, - document_id, - )) - .await - .caused_by(trc::location!())? { - resource_state.etag = archive.etag().into(); + let resources = groupware_resources_cached( + self, + access_token, + resource_state.account_id, + resource_state.collection, + &mut cached_resources, + ) + .await?; + + let resource = match resource_state.collection { + Collection::Calendar | Collection::AddressBook => { + resources.container_resource_by_id(document_id) + } + Collection::CalendarEvent + | Collection::ContactCard + | Collection::CalendarEventNotification => { + resources.item_by_id(document_id) + } + _ => resources.resources.find_any(document_id), + }; + + let is_owner = access_token.is_member(resource_state.account_id); + if let Some(resource) = resource.filter(|resource| { + is_owner + || EventPrivacy::from_flags( + resource.event_flags().unwrap_or_default(), + ) + .is_public() + }) { + resource_state.etag = quoted_etag(resource.etag().into()).into(); + } } } @@ -552,16 +609,16 @@ impl LockRequestHandler for Server { // Fetch sync token if needs_sync_token && resource_state.sync_token.is_none() { - let id = self - .fetch_dav_resources( - access_token.account_id(), - resource_state.account_id, - resource_state.collection.into(), - ) - .await - .caused_by(trc::location!())? - .highest_change_id; - resource_state.sync_token = Some(Urn::Sync { id, seq: 0 }.to_string()); + let id = groupware_resources_cached( + self, + access_token, + resource_state.account_id, + resource_state.collection, + &mut cached_resources, + ) + .await? + .highest_change_id; + resource_state.sync_token = Some(Urn::Sync(SyncToken::State(id)).to_string()); } } @@ -619,28 +676,38 @@ impl LockRequestHandler for Server { impl LockData { pub fn remove_lock(&mut self, lock_id: u64) -> bool { - for (lock_path, lock_items) in self.locks.iter_mut() { - for (idx, lock_item) in lock_items.0.iter().enumerate() { - if lock_item.lock_id == lock_id { - lock_items.0.swap_remove(idx); - if lock_items.0.is_empty() { - let lock_path = lock_path.clone(); - self.locks.remove(&lock_path); - } - return true; - } - } + let Some((pos, idx)) = self + .locks + .inner + .iter() + .enumerate() + .find_map(|(pos, entry)| { + entry + .value + .0 + .iter() + .position(|lock_item| lock_item.lock_id == lock_id) + .map(|idx| (pos, idx)) + }) + else { + return false; + }; + + let lock_items = &mut self.locks.inner[pos].value; + lock_items.0.swap_remove(idx); + if lock_items.0.is_empty() { + self.locks.swap_remove(pos); } - false + true } pub fn remove_expired(&mut self) -> u64 { let mut max_expire = 0; let now = now(); - self.locks.retain(|_, locks| { - locks.0.retain(|lock| { + self.locks.inner.retain_mut(|entry| { + entry.value.0.retain(|lock| { if lock.expires > now { max_expire = std::cmp::max(max_expire, lock.expires); true @@ -649,7 +716,7 @@ impl LockData { } }); - !locks.0.is_empty() + !entry.value.0.is_empty() }); max_expire @@ -657,27 +724,37 @@ impl LockData { } impl<'x> LockArchive<'x> { - fn unarchive(&'x self) -> trc::Result<&'x ArchivedLockData> { + fn new(lock_data: Option<&'x ArchivedLockData>) -> Self { + lock_data.map_or(LockArchive::Absent, LockArchive::Unarchived) + } + + fn unarchive(&'x self) -> trc::Result> { match self { - LockArchive::Unarchived(archived_lock_data) => Ok(archived_lock_data), - LockArchive::Archived(archive) => { - archive.unarchive::().caused_by(trc::location!()) - } + LockArchive::Unarchived(archived_lock_data) => Ok(Some(archived_lock_data)), + LockArchive::Archived(archive) => archive + .unarchive::() + .caused_by(trc::location!()) + .map(Some), + LockArchive::Absent => Ok(None), } } + + fn is_absent(&self) -> bool { + matches!(self, LockArchive::Absent) + } } impl<'x> LockCaches<'x> { pub(self) fn new_shared( account_id: u32, collection: Collection, - lock_data: &'x ArchivedLockData, + lock_data: Option<&'x ArchivedLockData>, ) -> Self { Self { caches: vec![LockCache { account_id, collection, - lock_archive: LockArchive::Unarchived(lock_data), + lock_archive: LockArchive::new(lock_data), }], } } @@ -690,7 +767,7 @@ impl<'x> LockCaches<'x> { Ok(LockCache { account_id: cache.account_id, collection: cache.collection, - lock_archive: LockArchive::Unarchived( + lock_archive: LockArchive::new( cache.lock_archive.unarchive().caused_by(trc::location!())?, ), }) @@ -713,17 +790,19 @@ impl<'x> LockCaches<'x> { server: &Server, resource_state: &ResourceState<'_>, ) -> trc::Result> { - if let Some(idx) = self.is_cached(resource_state) { - Ok(Some(idx)) + let pos = if let Some(pos) = self.is_cached(resource_state) { + pos } else if resource_state.collection != Collection::None { - if self.insert_lock_data(server, resource_state).await? { - Ok(Some(self.caches.len() - 1)) - } else { - Ok(None) - } + self.insert_lock_data(server, resource_state).await? } else { - Ok(None) - } + return Ok(None); + }; + + Ok(self + .caches + .get(pos) + .filter(|cache| !cache.lock_archive.is_absent()) + .map(|_| pos)) } fn find_locks_by_pos( @@ -732,36 +811,50 @@ impl<'x> LockCaches<'x> { resource_state: &'x ResourceState<'_>, include_children: bool, ) -> trc::Result> { - self.caches[pos] + let Some(cache) = self.caches.get(pos) else { + return Ok(Vec::new()); + }; + + Ok(cache .lock_archive - .unarchive() + .unarchive()? .map(|l| l.find_locks(resource_state.path, include_children)) + .unwrap_or_default()) } async fn insert_lock_data( &mut self, server: &Server, resource_state: &ResourceState<'_>, - ) -> trc::Result { - if let Some(lock_archive) = server + ) -> trc::Result { + let lock_archive = server .in_memory_store() - .key_get::>(resource_state.lock_key().as_slice()) + .key_get::>(resource_state.lock_key().as_slice()) .await .caused_by(trc::location!())? - { - self.caches.push(LockCache { - account_id: resource_state.account_id, - collection: resource_state.collection, - lock_archive: LockArchive::Archived(lock_archive), - }); + .map_or(LockArchive::Absent, LockArchive::Archived); + self.caches.push(LockCache { + account_id: resource_state.account_id, + collection: resource_state.collection, + lock_archive, + }); - Ok(true) - } else { - Ok(false) - } + Ok(self.caches.len() - 1) } } +fn check_owner_size(owner: Option<&XmlValue<'_>>, max_size: usize) -> crate::Result<()> { + match owner.map(XmlValue::encoded_len) { + Some(Ok(size)) if size > max_size => Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)), + Some(Err(_)) => Err(DavError::Code(StatusCode::BAD_REQUEST)), + _ => Ok(()), + } +} + +fn owner_value(bytes: &[u8]) -> Option { + DavValueView::parse(bytes).map(|owner| owner.to_encoded()) +} + impl LockItem { pub fn to_active_lock(&self, href: String) -> ActiveLock { ActiveLock::new( @@ -777,7 +870,7 @@ impl LockItem { } else { Depth::Zero }) - .with_owner_opt(self.owner_dav.clone()) + .with_owner_opt(self.owner_dav.as_deref().and_then(owner_value)) .with_timeout(self.expires.saturating_sub(now())) .with_lock_token(self.urn().to_string()) } @@ -787,6 +880,36 @@ impl LockItem { } } +impl LockData { + pub fn active_locks(&self, resource: &str, base_uri: &str) -> Vec { + let now = now(); + let mut resource_part = resource; + let mut active_locks = Vec::new(); + + loop { + if let Some(locks) = self.locks.get(resource_part) { + active_locks.extend( + locks + .0 + .iter() + .filter(|lock| { + lock.expires > now && (resource == resource_part || lock.depth_infinity) + }) + .map(|lock| lock.to_active_lock(format!("{base_uri}/{resource_part}"))), + ); + } + + if let Some((resource_part_, _)) = resource_part.rsplit_once('/') { + resource_part = resource_part_; + } else { + break; + } + } + + active_locks + } +} + impl ArchivedLockData { pub fn find_locks<'x: 'y, 'y>( &'x self, @@ -837,25 +960,6 @@ impl ArchivedLockData { } impl ArchivedLockItem { - pub fn to_active_lock(&self, href: String) -> ActiveLock { - ActiveLock::new( - href, - if self.exclusive { - LockScope::Exclusive - } else { - LockScope::Shared - }, - ) - .with_depth(if self.depth_infinity { - Depth::Infinity - } else { - Depth::Zero - }) - .with_owner_opt(self.owner_dav.as_ref().map(Into::into)) - .with_timeout(u64::from(self.expires).saturating_sub(now())) - .with_lock_token(self.urn().to_string()) - } - pub fn urn(&self) -> Urn { Urn::Lock(self.lock_id.into()) } @@ -873,6 +977,35 @@ impl ResourceState<'_> { } } +fn canonical_if_resource(uri: &str) -> Cow<'_, str> { + canonical_dav_uri(uri).unwrap_or(Cow::Borrowed(uri)) +} + +async fn groupware_resources_cached( + server: &Server, + access_token: &AccessToken, + account_id: u32, + collection: Collection, + cache: &mut Option<(u32, SyncCollection, Arc)>, +) -> trc::Result> { + let sync_collection = SyncCollection::from(collection); + + if let Some((cached_account_id, cached_collection, resources)) = cache.as_ref() + && *cached_account_id == account_id + && *cached_collection == sync_collection + { + return Ok(resources.clone()); + } + + let resources = server + .fetch_groupware_resources(access_token.account_id(), account_id, sync_collection) + .await + .caused_by(trc::location!())?; + *cache = Some((account_id, sync_collection, resources.clone())); + + Ok(resources) +} + pub(crate) fn build_lock_key(account_id: u32, collection: Collection) -> Vec { let mut result = Vec::with_capacity(U32_LEN + 2); result.push(KV_LOCK_DAV); @@ -890,3 +1023,4 @@ impl PartialEq for ResourceState<'_> { } impl Eq for ResourceState<'_> {} + diff --git a/crates/dav/src/common/mod.rs b/crates/dav/src/common/mod.rs index ecc32fc6..80051e8c 100644 --- a/crates/dav/src/common/mod.rs +++ b/crates/dav/src/common/mod.rs @@ -4,9 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use calcard::{ - icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, - vcard::{VCardParameterName, VCardVersion}, +use crate::calendar::filter::FilterTimeRanges; +use calcard::{common::timezone::Tz, vcard::VCardVersion}; +use common::{ + GroupwareResourceMetadata, GroupwareResourceRef, storage::dav::FILE_DEFAULT_MEDIA_TYPE, }; use dav_proto::{ Depth, RequestHeaders, Return, @@ -14,33 +15,79 @@ use dav_proto::{ Namespace, property::{DavProperty, ReportSet, ResourceType}, request::{ - AddressbookQuery, CalendarQuery, ExpandProperty, Filter, MultiGet, PropFind, - SyncCollection, Timezone, VCardPropertyWithGroup, + AddressbookQuery, CardFilter, CompFilter, ExpandProperty, MultiGet, PropFind, + SyncCollection, }, }, }; use groupware::{ + SizeWriter, calendar::{ - ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarEventNotification, Calendar, - CalendarEvent, CalendarEventNotification, + ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarEventContent, + ArchivedCalendarEventNotification, ArchivedCalendarEventNotificationContent, Calendar, + CalendarEvent, CalendarEventContent, CalendarEventNotification, + CalendarEventNotificationContent, + }, + contact::{ + AddressBook, ArchivedAddressBook, ArchivedContactCard, ArchivedContactCardContent, + ContactCard, ContactCardContent, }, - contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}, file::{ArchivedFileNode, FileNode}, }; +use hyper::StatusCode; use propfind::PropFindItem; use rkyv::vec::ArchivedVec; -use store::write::{AlignedBytes, Archive, BatchBuilder, Operation, ValueClass, ValueOp}; +use std::fmt::Write; +use store::write::{Archive, ArchiveBytes, AssignedIds, BatchBuilder}; use types::{ - TimeRange, acl::ArchivedAclGrant, collection::Collection, dead_property::ArchivedDeadProperty, - field::Field, + acl::{Acl, ArchivedAclGrant}, + collection::Collection, }; -use uri::{OwnedUri, Urn}; +use uri::{OwnedUri, SyncToken, Urn}; pub mod acl; +pub mod dead; pub mod lock; pub mod propfind; +pub mod search; pub mod uri; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ContainerOperation { + Copy, + Remove, +} + +impl ContainerOperation { + pub(crate) fn from_move(is_move: bool) -> Self { + if is_move { + ContainerOperation::Remove + } else { + ContainerOperation::Copy + } + } + + pub(crate) fn required_acls(self) -> impl Iterator { + let acls: &'static [Acl] = match self { + ContainerOperation::Copy => &[Acl::ReadItems], + ContainerOperation::Remove => &[Acl::Delete], + }; + acls.iter().copied() + } +} + +pub(crate) fn assert_parent_limit( + prev_count: usize, + count: usize, + max: usize, +) -> crate::Result<()> { + if count <= max || count <= prev_count { + Ok(()) + } else { + Err(crate::DavError::Code(StatusCode::FORBIDDEN)) + } +} + #[derive(Debug)] pub(crate) struct DavQuery<'x> { pub uri: &'x str, @@ -60,10 +107,7 @@ pub(crate) enum SyncType { #[default] None, Initial, - From { - id: u64, - seq: u32, - }, + Token(SyncToken), } #[derive(Default, Debug)] @@ -82,18 +126,17 @@ pub(crate) enum DavQueryResource<'x> { None, } -pub(crate) type AddressbookFilter = Vec>; -pub(crate) type CalendarFilter = - Vec, ICalendarProperty, ICalendarParameterName>>; - #[derive(Debug)] pub(crate) enum DavQueryFilter { - Addressbook(AddressbookFilter), - Calendar { - filter: CalendarFilter, - max_time_range: Option, - timezone: Timezone, - }, + Addressbook(CardFilter), + Calendar(CalendarQueryFilter), +} + +#[derive(Debug)] +pub(crate) struct CalendarQueryFilter { + pub filter: Option, + pub expansion: FilterTimeRanges, + pub timezone: Option, } pub(crate) trait ETag { @@ -106,37 +149,30 @@ pub(crate) trait ExtractETag { impl ETag for Archive { fn etag(&self) -> String { - format!("\"{}\"", self.version.hash().unwrap_or_default()) + quoted_etag(self.version.hash().unwrap_or_default().into()) } } impl ExtractETag for BatchBuilder { fn etag(&self) -> Option { - let p_value = u8::from(Field::ARCHIVE); - for op in self.ops().iter().rev() { - match op { - Operation::Value { - class: ValueClass::Property(p_id), - op: ValueOp::Set(value), - } if *p_id == p_value => { - return Archive::::extract_hash(value) - .map(|hash| format!("\"{}\"", hash)); - } - Operation::Value { - class: ValueClass::Property(p_id), - op: ValueOp::SetFnc(set_fnc), - } if *p_id == p_value => { - return Archive::::extract_hash(set_fnc.params().bytes(0)) - .map(|hash| format!("\"{}\"", hash)); - } - _ => {} - } - } - - None + self.last_archive_hash() + .map(|hash| quoted_etag(hash.into())) } } +impl ExtractETag for AssignedIds { + fn etag(&self) -> Option { + self.last_archive_hash() + .map(|hash| quoted_etag(hash.into())) + } +} + +pub(crate) fn quoted_etag(hash: u64) -> String { + let mut etag = String::with_capacity(22); + let _ = write!(etag, "\"{hash}\""); + etag +} + pub(crate) trait DavCollection { fn namespace(&self) -> Namespace; } @@ -168,7 +204,7 @@ impl<'x> DavQuery<'x> { }, ret: headers.ret, depth_no_root: headers.depth_no_root, - uri: headers.uri, + uri: headers.raw_uri, vcard_version: headers.vcard_version, sync_type: Default::default(), limit: Default::default(), @@ -189,7 +225,7 @@ impl<'x> DavQuery<'x> { propfind: multiget.properties, ret: headers.ret, depth_no_root: headers.depth_no_root, - uri: headers.uri, + uri: headers.raw_uri, vcard_version: headers.vcard_version, sync_type: Default::default(), depth: Default::default(), @@ -205,7 +241,7 @@ impl<'x> DavQuery<'x> { ) -> Self { Self { resource: DavQueryResource::Query { - filter: DavQueryFilter::Addressbook(query.filters), + filter: DavQueryFilter::Addressbook(query.filter), parent_collection: Collection::AddressBook, items, }, @@ -213,7 +249,7 @@ impl<'x> DavQuery<'x> { limit: query.limit, ret: headers.ret, depth_no_root: headers.depth_no_root, - uri: headers.uri, + uri: headers.raw_uri, vcard_version: headers.vcard_version, sync_type: Default::default(), depth: Default::default(), @@ -222,25 +258,22 @@ impl<'x> DavQuery<'x> { } pub fn calendar_query( - query: CalendarQuery, - max_time_range: Option, + propfind: PropFind, + filter: CalendarQueryFilter, + parent_collection: Collection, items: Vec, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Query { - filter: DavQueryFilter::Calendar { - filter: query.filters, - timezone: query.timezone, - max_time_range, - }, - parent_collection: Collection::Calendar, + filter: DavQueryFilter::Calendar(filter), + parent_collection, items, }, - propfind: query.properties, + propfind, ret: headers.ret, depth_no_root: headers.depth_no_root, - uri: headers.uri, + uri: headers.raw_uri, sync_type: Default::default(), depth: Default::default(), limit: Default::default(), @@ -262,7 +295,7 @@ impl<'x> DavQuery<'x> { .as_deref() .and_then(Urn::parse) .and_then(|urn| urn.try_unwrap_sync()) - .map(|(id, seq)| SyncType::From { id, seq }) + .map(SyncType::Token) .unwrap_or(SyncType::Initial), depth: match changes.depth { Depth::One => 1, @@ -273,7 +306,7 @@ impl<'x> DavQuery<'x> { ret: headers.ret, depth_no_root: headers.depth_no_root, expand: false, - uri: headers.uri, + uri: headers.raw_uri, vcard_version: headers.vcard_version, } } @@ -285,9 +318,7 @@ impl<'x> DavQuery<'x> { ) -> Self { let mut props = Vec::with_capacity(expand.properties.len()); for item in expand.properties { - if !matches!(item.property, DavProperty::DeadProperty(_)) - && !props.contains(&item.property) - { + if !matches!(item.property, DavProperty::Dead(_)) && !props.contains(&item.property) { props.push(item.property); } } @@ -302,7 +333,7 @@ impl<'x> DavQuery<'x> { ret: headers.ret, depth_no_root: headers.depth_no_root, expand: true, - uri: headers.uri, + uri: headers.raw_uri, sync_type: Default::default(), limit: Default::default(), vcard_version: headers.vcard_version, @@ -316,47 +347,264 @@ impl<'x> DavQuery<'x> { pub(crate) enum ArchivedResource<'x> { Calendar(Archive<&'x ArchivedCalendar>), - CalendarEvent(Archive<&'x ArchivedCalendarEvent>), - CalendarEventNotification(Archive<&'x ArchivedCalendarEventNotification>), + CalendarEvent(Archive<&'x ArchivedCalendarEvent>, Option>), + CalendarEventNotification( + Archive<&'x ArchivedCalendarEventNotification>, + Option<&'x ArchivedCalendarEventNotificationContent>, + ), CalendarEventNotificationCollection(bool), AddressBook(Archive<&'x ArchivedAddressBook>), - ContactCard(Archive<&'x ArchivedContactCard>), + ContactCard( + Archive<&'x ArchivedContactCard>, + Option<&'x ArchivedContactCardContent>, + ), FileNode(Archive<&'x ArchivedFileNode>), + Cached(CachedItem<'x>), +} + +#[derive(Clone, Copy)] +pub(crate) struct CachedItem<'x> { + etag: u32, + created: i64, + modified: i64, + kind: CachedKind<'x>, +} + +#[derive(Clone, Copy)] +enum CachedKind<'x> { + CalendarEvent { + flags: u16, + }, + ContactCard, + File { + size: Option, + media_type: Option<&'x str>, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CacheFit { + Exact, + InexactMediaType, + Inexact, +} + +impl<'x> CachedItem<'x> { + pub fn new(collection: Collection, resource: GroupwareResourceRef<'x>) -> Option { + let kind = match collection { + Collection::CalendarEvent => CachedKind::CalendarEvent { + flags: resource.event_flags().unwrap_or_default(), + }, + Collection::ContactCard => CachedKind::ContactCard, + Collection::FileNode => CachedKind::File { + size: resource.size(), + media_type: if resource.file_flags()?.has_exact_media_type() { + resource.media_type() + } else { + None + }, + }, + _ => return None, + }; + Some(CachedItem { + etag: resource.etag(), + created: resource.created_at()?, + modified: resource.modified_at()?, + kind, + }) + } + + pub fn event_flags(&self) -> Option { + match self.kind { + CachedKind::CalendarEvent { flags } => Some(flags), + CachedKind::ContactCard | CachedKind::File { .. } => None, + } + } + + fn content_length(&self) -> Option { + match self.kind { + CachedKind::File { size, .. } => size, + CachedKind::CalendarEvent { .. } | CachedKind::ContactCard => None, + } + } + + fn content_type(&self) -> Option<&'x str> { + match self.kind { + CachedKind::CalendarEvent { .. } => Some("text/calendar"), + CachedKind::ContactCard => Some("text/vcard"), + CachedKind::File { media_type, .. } => media_type, + } + } + + pub fn fit(resource: &GroupwareResourceRef<'_>) -> CacheFit { + match resource.resource.data { + GroupwareResourceMetadata::CalendarEvent { + modified_at: i32::MIN | i32::MAX, + .. + } + | GroupwareResourceMetadata::ContactCard { + modified_at: i32::MIN | i32::MAX, + .. + } + | GroupwareResourceMetadata::File { + created_delta: i32::MIN | i32::MAX, + .. + } => CacheFit::Inexact, + GroupwareResourceMetadata::File { flags, .. } => { + if resource.created_at().is_none() { + CacheFit::Inexact + } else if flags.has_exact_media_type() { + CacheFit::Exact + } else { + CacheFit::InexactMediaType + } + } + _ => CacheFit::Exact, + } + } +} + +pub(crate) enum EventContent<'x> { + Stored(&'x ArchivedCalendarEventContent), + View { + stored: &'x ArchivedCalendarEventContent, + view: CalendarEventContent, + merged_overrides: MergedOverrides, + }, +} + +#[derive(Debug, Default)] +pub(crate) struct MergedOverrides { + base_id: Option, + ids: Vec, +} + +impl MergedOverrides { + pub fn push(&mut self, component_id: u32) { + self.ids.push(component_id); + } + + pub fn set_base(&mut self, base_id: Option) { + self.base_id = base_id; + self.ids.sort_unstable(); + } + + pub fn base_of(&self, component_id: u32) -> Option { + self.base_id + .filter(|_| self.ids.binary_search(&component_id).is_ok()) + } + + pub fn is_empty(&self) -> bool { + self.ids.is_empty() || self.base_id.is_none() + } +} + +impl<'x> EventContent<'x> { + pub fn attach_view(&mut self, view: CalendarEventContent, merged_overrides: MergedOverrides) { + *self = EventContent::View { + stored: self.stored(), + view, + merged_overrides, + }; + } + + pub fn stored(&self) -> &'x ArchivedCalendarEventContent { + match self { + EventContent::Stored(stored) | EventContent::View { stored, .. } => stored, + } + } + + pub fn view(&self) -> Option<&CalendarEventContent> { + match self { + EventContent::Stored(_) => None, + EventContent::View { view, .. } => Some(view), + } + } + + pub fn merged_overrides(&self) -> Option<&MergedOverrides> { + match self { + EventContent::Stored(_) => None, + EventContent::View { + merged_overrides, .. + } => Some(merged_overrides), + } + } + + pub fn to_ical_string(&self) -> String { + match self { + EventContent::Stored(stored) => stored.data.event.to_string(), + EventContent::View { view, .. } => view.data.event.to_string(), + } + } } impl<'x> ArchivedResource<'x> { pub fn from_archive( - archive: &'x Archive, + archive: &'x Archive, + content: Option<&'x Archive>, collection: Collection, ) -> trc::Result { match collection { Collection::Calendar => archive .to_unarchived::() .map(ArchivedResource::Calendar), - Collection::CalendarEvent => archive - .to_unarchived::() - .map(ArchivedResource::CalendarEvent), - Collection::CalendarEventNotification => archive - .to_unarchived::() - .map(ArchivedResource::CalendarEventNotification), + Collection::CalendarEvent => { + let content = content + .map(|content| content.unarchive::()) + .transpose()? + .map(EventContent::Stored); + archive + .to_unarchived::() + .map(|meta| ArchivedResource::CalendarEvent(meta, content)) + } + Collection::CalendarEventNotification => { + let content = content + .map(|content| content.unarchive::()) + .transpose()?; + archive + .to_unarchived::() + .map(|meta| ArchivedResource::CalendarEventNotification(meta, content)) + } Collection::AddressBook => archive .to_unarchived::() .map(ArchivedResource::AddressBook), Collection::FileNode => archive .to_unarchived::() .map(ArchivedResource::FileNode), - Collection::ContactCard => archive - .to_unarchived::() - .map(ArchivedResource::ContactCard), + Collection::ContactCard => { + let content = content + .map(|content| content.unarchive::()) + .transpose()?; + archive + .to_unarchived::() + .map(|meta| ArchivedResource::ContactCard(meta, content)) + } _ => unreachable!(), } } + pub fn etag(&self) -> String { + let hash = match self { + ArchivedResource::CalendarEvent(archive, _) => archive.inner.etag.to_native(), + ArchivedResource::ContactCard(archive, _) => archive.inner.etag.to_native(), + ArchivedResource::CalendarEventNotification(archive, _) => { + archive.inner.etag.to_native() + } + ArchivedResource::Calendar(archive) => archive.version.hash().unwrap_or_default(), + ArchivedResource::AddressBook(archive) => archive.version.hash().unwrap_or_default(), + ArchivedResource::FileNode(archive) => archive.inner.etag.to_native(), + ArchivedResource::CalendarEventNotificationCollection(_) => 0, + ArchivedResource::Cached(item) => item.etag, + }; + + quoted_etag(hash.into()) + } + pub fn acls(&self) -> Option<&ArchivedVec> { match self { Self::Calendar(archive) => Some(&archive.inner.acls), Self::AddressBook(archive) => Some(&archive.inner.acls), - Self::FileNode(archive) => Some(&archive.inner.acls), + Self::FileNode(archive) if archive.inner.is_directory() => Some(&archive.inner.acls), _ => None, } } @@ -364,53 +612,57 @@ impl<'x> ArchivedResource<'x> { pub fn created(&self) -> i64 { match self { ArchivedResource::Calendar(archive) => archive.inner.created.to_native(), - ArchivedResource::CalendarEvent(archive) => archive.inner.created.to_native(), + ArchivedResource::CalendarEvent(archive, _) => archive.inner.created.to_native(), ArchivedResource::AddressBook(archive) => archive.inner.created.to_native(), - ArchivedResource::ContactCard(archive) => archive.inner.created.to_native(), + ArchivedResource::ContactCard(archive, _) => archive.inner.created.to_native(), ArchivedResource::FileNode(archive) => archive.inner.created.to_native(), - ArchivedResource::CalendarEventNotification(archive) => { + ArchivedResource::CalendarEventNotification(archive, _) => { archive.inner.created.to_native() } ArchivedResource::CalendarEventNotificationCollection(_) => 1634515200, + ArchivedResource::Cached(item) => item.created, } } pub fn modified(&self) -> i64 { match self { ArchivedResource::Calendar(archive) => archive.inner.modified.to_native(), - ArchivedResource::CalendarEvent(archive) => archive.inner.modified.to_native(), + ArchivedResource::CalendarEvent(archive, _) => archive.inner.modified.to_native(), ArchivedResource::AddressBook(archive) => archive.inner.modified.to_native(), - ArchivedResource::ContactCard(archive) => archive.inner.modified.to_native(), + ArchivedResource::ContactCard(archive, _) => archive.inner.modified.to_native(), ArchivedResource::FileNode(archive) => archive.inner.modified.to_native(), - ArchivedResource::CalendarEventNotification(archive) => { + ArchivedResource::CalendarEventNotification(archive, _) => { archive.inner.modified.to_native() } ArchivedResource::CalendarEventNotificationCollection(_) => 1634515200, + ArchivedResource::Cached(item) => item.modified, } } - pub fn dead_properties(&self) -> Option<&ArchivedDeadProperty> { - match self { - ArchivedResource::Calendar(archive) => Some(&archive.inner.dead_properties), - ArchivedResource::CalendarEvent(archive) => Some(&archive.inner.dead_properties), - ArchivedResource::AddressBook(archive) => Some(&archive.inner.dead_properties), - ArchivedResource::ContactCard(archive) => Some(&archive.inner.dead_properties), - ArchivedResource::FileNode(archive) => Some(&archive.inner.dead_properties), - ArchivedResource::CalendarEventNotification(_) - | ArchivedResource::CalendarEventNotificationCollection(_) => None, - } + pub fn is_file(&self) -> bool { + matches!( + self, + ArchivedResource::FileNode(_) + | ArchivedResource::Cached(CachedItem { + kind: CachedKind::File { .. }, + .. + }) + ) } pub fn content_length(&self) -> Option { match self { - ArchivedResource::FileNode(archive) => { - archive.inner.file.as_ref().map(|f| f.size.to_native()) - } - ArchivedResource::CalendarEvent(archive) => archive.inner.size.to_native().into(), - ArchivedResource::CalendarEventNotification(archive) => { + ArchivedResource::FileNode(archive) => archive.inner.file().map(|f| f.size.to_native()), + ArchivedResource::CalendarEvent(archive, content) => content + .as_ref() + .and_then(EventContent::view) + .map(|view| SizeWriter::ical(&view.data.event) as u32) + .or_else(|| archive.inner.size.to_native().into()), + ArchivedResource::CalendarEventNotification(archive, _) => { archive.inner.size.to_native().into() } - ArchivedResource::ContactCard(archive) => archive.inner.size.to_native().into(), + ArchivedResource::ContactCard(archive, _) => archive.inner.size.to_native().into(), + ArchivedResource::Cached(item) => item.content_length(), ArchivedResource::AddressBook(_) | ArchivedResource::Calendar(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, @@ -421,13 +673,12 @@ impl<'x> ArchivedResource<'x> { match self { ArchivedResource::FileNode(archive) => archive .inner - .file - .as_ref() - .and_then(|f| f.media_type.as_deref()), - ArchivedResource::CalendarEvent(_) | ArchivedResource::CalendarEventNotification(_) => { - "text/calendar".into() - } - ArchivedResource::ContactCard(_) => "text/vcard".into(), + .file() + .map(|f| f.media_type.as_deref().unwrap_or(FILE_DEFAULT_MEDIA_TYPE)), + ArchivedResource::CalendarEvent(..) + | ArchivedResource::CalendarEventNotification(..) => "text/calendar".into(), + ArchivedResource::ContactCard(..) => "text/vcard".into(), + ArchivedResource::Cached(item) => item.content_type(), ArchivedResource::AddressBook(_) | ArchivedResource::Calendar(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, @@ -436,17 +687,19 @@ impl<'x> ArchivedResource<'x> { pub fn display_name(&self, account_id: u32) -> Option<&str> { match self { - ArchivedResource::Calendar(archive) => { - Some(archive.inner.preferences(account_id).name.as_str()) - } - ArchivedResource::CalendarEvent(archive) => archive.inner.display_name.as_deref(), + ArchivedResource::Calendar(archive) => archive + .inner + .preferences(account_id) + .map(|preferences| preferences.name.as_str()), + ArchivedResource::CalendarEvent(archive, _) => archive.inner.display_name.as_deref(), ArchivedResource::AddressBook(archive) => { Some(archive.inner.preferences(account_id).name.as_str()) } - ArchivedResource::ContactCard(archive) => archive.inner.display_name.as_deref(), - ArchivedResource::FileNode(archive) => archive.inner.display_name.as_deref(), - ArchivedResource::CalendarEventNotification(_) - | ArchivedResource::CalendarEventNotificationCollection(_) => None, + ArchivedResource::ContactCard(archive, _) => archive.inner.display_name.as_deref(), + ArchivedResource::FileNode(_) + | ArchivedResource::CalendarEventNotification(..) + | ArchivedResource::CalendarEventNotificationCollection(_) + | ArchivedResource::Cached(_) => None, } } @@ -471,7 +724,7 @@ impl<'x> ArchivedResource<'x> { ReportSet::AddressbookMultiGet, ] .into(), - ArchivedResource::FileNode(archive) if archive.inner.file.is_none() => vec![ + ArchivedResource::FileNode(archive) if archive.inner.is_directory() => vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, @@ -495,7 +748,7 @@ impl<'x> ArchivedResource<'x> { ArchivedResource::AddressBook(_) => { vec![ResourceType::Collection, ResourceType::AddressBook].into() } - ArchivedResource::FileNode(archive) if archive.inner.file.is_none() => { + ArchivedResource::FileNode(archive) if archive.inner.is_directory() => { vec![ResourceType::Collection].into() } ArchivedResource::CalendarEventNotificationCollection(true) => { @@ -517,4 +770,11 @@ impl SyncType { pub fn is_none_or_initial(&self) -> bool { matches!(self, SyncType::None | SyncType::Initial) } + + pub fn is_incremental(&self) -> bool { + matches!( + self, + SyncType::Token(SyncToken::State(_) | SyncToken::ChangesPage { .. }) + ) + } } diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs deleted file mode 100644 index 79dad936..00000000 --- a/crates/dav/src/common/propfind.rs +++ /dev/null @@ -1,1825 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::{ - ArchivedResource, DavCollection, DavQuery, DavQueryFilter, ETag, SyncType, - acl::{DavAclHandler, Privileges}, - lock::{LockData, build_lock_key}, - uri::{UriResource, Urn}, -}; -use crate::{ - DavError, DavErrorCondition, - calendar::{ - CALENDAR_CONTAINER_PROPS, CALENDAR_ITEM_PROPS, - query::{CalendarQueryHandler, try_parse_tz}, - }, - card::{ - CARD_CONTAINER_PROPS, CARD_ITEM_PROPS, - query::{serialize_vcard_with_props, vcard_query}, - }, - common::{DavQueryResource, acl::current_user_privilege_set, uri::DavUriResource}, - file::{FILE_CONTAINER_PROPS, FILE_ITEM_PROPS}, - principal::{ - CurrentUserPrincipal, - propfind::{PrincipalPropFind, build_home_set}, - }, -}; -use calcard::{common::timezone::Tz, icalendar::ICalendarComponentType}; -use common::{ - DavResourcePath, DavResources, Server, - auth::{AccessToken, AccountCache}, -}; -use dav_proto::{ - Depth, RequestHeaders, - parser::header::dav_base_uri, - requests::NsDeadProperty, - schema::{ - Collation, Namespace, - property::{ - ActiveLock, CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, - PrincipalProperty, Privilege, ReportSet, ResourceType, Rfc1123DateTime, - SupportedCollation, SupportedLock, WebDavProperty, - }, - request::{DavDeadProperty, DavPropertyValue, PropFind}, - response::{ - AclRestrictions, BaseCondition, Href, List, MultiStatus, PropStat, Response, - SupportedPrivilege, - }, - }, -}; -use groupware::calendar::{SCHEDULE_INBOX_ID, SupportedComponent}; -use groupware::{ - DavCalendarResource, DavResourceName, cache::GroupwareCache, calendar::ArchivedTimezone, -}; -use http_proto::HttpResponse; -use hyper::StatusCode; -use registry::schema::{enums::Permission, prelude::ObjectType}; -use std::{borrow::Cow, sync::Arc}; -use store::{ - ValueKey, - registry::RegistryQuery, - write::{AlignedBytes, Archive}, -}; -use store::{ - ahash::AHashMap, - query::log::{Change, Query}, - roaring::RoaringBitmap, -}; -use trc::AddContext; -use types::{ - acl::Acl, - collection::{Collection, SyncCollection}, - dead_property::DeadProperty, -}; -use utils::map::bitmap::Bitmap; - -pub(crate) trait PropFindRequestHandler: Sync + Send { - fn handle_propfind_request( - &self, - access_token: &AccessToken, - headers: &RequestHeaders<'_>, - request: PropFind, - ) -> impl Future> + Send; - - fn handle_dav_query( - &self, - access_token: &AccessToken, - query: DavQuery<'_>, - ) -> impl Future> + Send; - - fn dav_quota( - &self, - account_id: u32, - ) -> impl Future> + Send; -} - -pub(crate) struct PropFindData { - pub accounts: AHashMap, -} - -#[derive(Default)] -pub(crate) struct PropFindAccountData { - pub resources: Option>, - pub quota: Option, - pub owner: Option, - pub locks: Option>, - pub locks_not_found: bool, -} - -#[derive(Clone, Default)] -pub(crate) struct PropFindAccountQuota { - pub used: u64, - pub available: Option, -} - -#[derive(Debug)] -pub(crate) struct PropFindItem { - pub name: String, - pub account_id: u32, - pub document_id: u32, - pub parent_id: Option, - pub is_container: bool, -} - -impl PropFindRequestHandler for Server { - async fn handle_propfind_request( - &self, - access_token: &AccessToken, - headers: &RequestHeaders<'_>, - request: PropFind, - ) -> crate::Result { - // Validate URI - let resource = self.validate_uri(access_token, headers.uri).await?; - - // Reject Infinity depth for certain queries - let return_children = match headers.depth { - Depth::One | Depth::None => true, - Depth::Zero => false, - Depth::Infinity => match resource.collection { - Collection::Principal => true, - Collection::Calendar | Collection::AddressBook - if resource.account_id.is_some() && resource.resource.is_some() => - { - true - } - Collection::CalendarEventNotification if resource.account_id.is_some() => true, - _ => { - return Err(DavErrorCondition::new( - StatusCode::FORBIDDEN, - BaseCondition::PropFindFiniteDepth, - ) - .into()); - } - }, - }; - - // List shared resources - if let Some(account_id) = resource.account_id { - match resource.collection { - Collection::FileNode - | Collection::Calendar - | Collection::AddressBook - | Collection::CalendarEventNotification => { - // Validate permissions - access_token.enforce_permission(match resource.collection { - Collection::FileNode => Permission::DavFilePropFind, - Collection::Calendar - | Collection::CalendarEvent - | Collection::CalendarEventNotification => Permission::DavCalPropFind, - Collection::AddressBook | Collection::ContactCard => { - Permission::DavCardPropFind - } - _ => unreachable!(), - })?; - - self.handle_dav_query( - access_token, - DavQuery::propfind( - UriResource::new_owned( - resource.collection, - account_id, - resource.resource, - ), - request, - headers, - ), - ) - .await - } - Collection::Principal => { - let mut response = MultiStatus::new(Vec::with_capacity(16)); - - if resource.resource.is_some() { - response.add_response(Response::new_status( - [headers.uri.to_string()], - StatusCode::NOT_FOUND, - )); - } else if access_token.has_account_access(account_id) - || (self.core.groupware.allow_directory_query - && access_token.has_permission(Permission::DavPrincipalList)) - || access_token.has_permission(Permission::SysAccountQuery) - { - self.prepare_principal_propfind_response( - access_token, - Collection::Principal, - [account_id].into_iter(), - &request, - &mut response, - ) - .await?; - } else { - response.add_response(Response::new_status( - [headers.uri.to_string()], - StatusCode::FORBIDDEN, - )); - } - - Ok(HttpResponse::new(StatusCode::MULTI_STATUS) - .with_xml_body(response.to_string())) - } - _ => unreachable!(), - } - } else { - let mut response = MultiStatus::new(Vec::with_capacity(16)); - - // Add container info - if !headers.depth_no_root { - add_base_collection_response( - self, - &request, - resource.collection, - access_token, - &mut response, - ) - .await?; - } - - if return_children { - let ids = if !matches!(resource.collection, Collection::Principal) { - // Validate permissions - access_token.enforce_permission(match resource.collection { - Collection::FileNode => Permission::DavFilePropFind, - Collection::Calendar - | Collection::CalendarEvent - | Collection::CalendarEventNotification => Permission::DavCalPropFind, - Collection::AddressBook | Collection::ContactCard => { - Permission::DavCardPropFind - } - _ => unreachable!(), - })?; - RoaringBitmap::from_iter( - access_token.all_ids_by_collection(resource.collection), - ) - } else if (self.core.groupware.allow_directory_query - && access_token.has_permission(Permission::DavPrincipalList)) - || access_token.has_permission(Permission::SysAccountQuery) - { - // Return all principals - self.registry() - .query::( - RegistryQuery::new(ObjectType::Account) - .with_tenant(access_token.tenant_id()), - ) - .await - .caused_by(trc::location!())? - } else { - RoaringBitmap::from_iter(access_token.all_ids()) - }; - - self.prepare_principal_propfind_response( - access_token, - resource.collection, - ids.into_iter(), - &request, - &mut response, - ) - .await?; - } - - Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) - } - } - - async fn handle_dav_query( - &self, - access_token: &AccessToken, - mut query: DavQuery<'_>, - ) -> crate::Result { - let mut response = MultiStatus::new(Vec::with_capacity(16)); - let mut data = PropFindData::new(); - let collection_container; - let collection_children; - let sync_collection; - let mut query_filter = None; - let mut limit = std::cmp::min( - query.limit.unwrap_or(u32::MAX) as usize, - self.core.groupware.max_results, - ); - let mut is_sync_limited = false; - let mut is_propfind = false; - let mut ical_instances_limit = self.core.groupware.max_ical_instances; - - let paths = match std::mem::take(&mut query.resource) { - DavQueryResource::Uri(resource) => { - collection_container = resource.collection; - collection_children = collection_container.child_collection().unwrap(); - sync_collection = SyncCollection::from(collection_container); - is_propfind = true; - - get( - self, - access_token, - collection_container, - collection_children, - sync_collection, - &query, - &mut data, - &mut response, - resource, - limit, - &mut is_sync_limited, - ) - .await? - } - DavQueryResource::Multiget { - hrefs, - parent_collection, - } => { - collection_container = parent_collection; - collection_children = collection_container.child_collection().unwrap(); - sync_collection = SyncCollection::from(collection_container); - - multiget( - self, - access_token, - collection_container, - collection_children, - sync_collection, - &mut data, - &mut response, - hrefs, - ) - .await? - } - DavQueryResource::Query { - filter, - parent_collection, - items, - } => { - query_filter = Some(filter); - collection_container = parent_collection; - collection_children = collection_container.child_collection().unwrap(); - sync_collection = SyncCollection::from(collection_container); - - items - } - DavQueryResource::None => unreachable!(), - }; - response.set_namespace(collection_container.namespace()); - - let mut skip_not_found = query.expand; - let properties = match &query.propfind { - PropFind::PropName => { - let (container_props, children_props) = match collection_container { - Collection::FileNode => { - (FILE_CONTAINER_PROPS.as_slice(), FILE_ITEM_PROPS.as_slice()) - } - Collection::Calendar | Collection::CalendarEventNotification => ( - CALENDAR_CONTAINER_PROPS.as_slice(), - CALENDAR_ITEM_PROPS.as_slice(), - ), - Collection::AddressBook => { - (CARD_CONTAINER_PROPS.as_slice(), CARD_ITEM_PROPS.as_slice()) - } - _ => unreachable!(), - }; - - for property in container_props.iter().chain(children_props) { - response.set_namespace(property.namespace()); - } - - for item in paths { - let props = if item.is_container { - container_props - .iter() - .cloned() - .map(DavPropertyValue::empty) - .collect::>() - } else { - children_props - .iter() - .cloned() - .map(DavPropertyValue::empty) - .collect::>() - }; - - response.add_response(Response::new_propstat( - item.name, - vec![PropStat::new_list(props)], - )); - } - - return Ok( - HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string()) - ); - } - PropFind::AllProp(items) => { - skip_not_found = true; - let mut result = Vec::with_capacity(items.len() + DavProperty::ALL_PROPS.len()); - result.extend(DavProperty::ALL_PROPS); - result.extend(items.iter().filter(|field| !field.is_all_prop()).cloned()); - result - } - PropFind::Prop(items) => items.clone(), - }; - - for property in &properties { - response.set_namespace(property.namespace()); - } - - let is_scheduling = collection_container == Collection::CalendarEventNotification; - let account_info = self - .account(access_token.account_id()) - .await - .caused_by(trc::location!())?; - 'outer: for item in paths { - let account_id = item.account_id; - let personal_id = access_token.personal_id(account_id, collection_container); - let document_id = item.document_id; - let collection = if item.is_container { - collection_container - } else { - collection_children - }; - - // Unarchive resource - let archive_; - let archive = if is_scheduling && item.is_container { - archive_ = Archive::default(); - ArchivedResource::CalendarEventNotificationCollection( - item.document_id == SCHEDULE_INBOX_ID, - ) - } else if let Some(archive) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - collection, - document_id, - )) - .await - .caused_by(trc::location!())? - { - archive_ = archive; - ArchivedResource::from_archive(&archive_, collection).caused_by(trc::location!())? - } else { - response.add_response(Response::new_status([item.name], StatusCode::NOT_FOUND)); - continue; - }; - - // Filter - let mut calendar_filter = None; - if let Some(query_filter) = &query_filter { - match (query_filter, &archive) { - (DavQueryFilter::Addressbook(filter), ArchivedResource::ContactCard(card)) - if !vcard_query(&card.inner.card, filter) => - { - continue; - } - ( - DavQueryFilter::Calendar { - filter, - timezone, - max_time_range, - }, - ArchivedResource::CalendarEvent(event), - ) => { - let default_tz = if let Some(tz) = try_parse_tz(timezone) { - tz - } else if let Some(calendar_id) = item.parent_id { - data.resources(self, access_token, account_id, SyncCollection::Calendar) - .await - .caused_by(trc::location!())? - .calendar_default_tz(calendar_id, account_id) - .unwrap_or(Tz::UTC) - } else { - Tz::UTC - }; - let mut query_handler = - CalendarQueryHandler::new(event.inner, *max_time_range, default_tz); - if !query_handler.filter(event.inner, filter) { - continue; - } - calendar_filter = Some(query_handler); - } - _ => (), - } - } - - // Fill properties - let dead_properties = archive.dead_properties(); - let mut fields = Vec::with_capacity(properties.len()); - let mut fields_not_found = Vec::new(); - for property in &properties { - match property { - DavProperty::WebDav(dav_property) => match dav_property { - WebDavProperty::CreationDate => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::Timestamp(archive.created()), - )); - } - WebDavProperty::DisplayName => { - if let Some(name) = archive.display_name(personal_id) { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String(name.to_string()), - )); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::GetContentLanguage => { - if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::GetContentLength => { - if let Some(value) = archive.content_length() { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::Uint64(value as u64), - )); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::GetContentType => { - if let Some(value) = archive.content_type() { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String(value.to_string()), - )); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::GetETag => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String(archive_.etag()), - )); - } - WebDavProperty::GetCTag => { - if item.is_container { - let ctag = data - .resources(self, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())? - .highest_change_id; - - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String(format!("\"{ctag}\"")), - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::GetLastModified => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::Rfc1123Date(Rfc1123DateTime::new(archive.modified())), - )); - } - WebDavProperty::ResourceType => { - if let Some(resource_type) = archive.resource_type() { - fields.push(DavPropertyValue::new(property.clone(), resource_type)); - } else { - fields.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::LockDiscovery => { - if let Some(locks) = data - .locks(self, account_id, collection_container, &item) - .await - .caused_by(trc::location!())? - { - fields.push(DavPropertyValue::new(property.clone(), locks)); - } else { - fields.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::SupportedLock => { - if !is_scheduling { - fields.push(DavPropertyValue::new( - property.clone(), - SupportedLock::default(), - )); - } else { - fields.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::SupportedReportSet => { - if let Some(report_set) = archive.supported_report_set() { - fields.push(DavPropertyValue::new(property.clone(), report_set)); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::SyncToken => { - let sync_token = data - .resources(self, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())? - .sync_token(); - - fields.push(DavPropertyValue::new(property.clone(), sync_token)); - } - WebDavProperty::CurrentUserPrincipal => { - if !query.expand { - fields.push(DavPropertyValue::new( - property.clone(), - vec![account_info.current_user_principal()], - )); - } else { - fields.push(DavPropertyValue::new( - property.clone(), - self.expand_principal( - access_token, - access_token.account_id(), - &query.propfind, - ) - .await? - .map(|r| DavValue::Response(Box::new(r))) - .unwrap_or(DavValue::Null), - )); - } - } - WebDavProperty::QuotaAvailableBytes => { - let available = if item.is_container { - data.quota(self, account_id) - .await - .caused_by(trc::location!())? - .available - } else { - None - }; - - if let Some(available) = available { - fields.push(DavPropertyValue::new(property.clone(), available)); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::QuotaUsedBytes => { - if item.is_container { - fields.push(DavPropertyValue::new( - property.clone(), - data.quota(self, account_id) - .await - .caused_by(trc::location!())? - .used, - )); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::Owner => { - if !query.expand { - fields.push(DavPropertyValue::new( - property.clone(), - vec![ - data.owner(self, &account_info, account_id) - .await - .caused_by(trc::location!())?, - ], - )); - } else { - fields.push(DavPropertyValue::new( - property.clone(), - self.expand_principal( - access_token, - account_id, - &query.propfind, - ) - .await? - .map(|r| DavValue::Response(Box::new(r))) - .unwrap_or(DavValue::Null), - )); - } - } - WebDavProperty::Group => { - fields.push(DavPropertyValue::empty(property.clone())); - } - WebDavProperty::SupportedPrivilegeSet => { - if !is_scheduling { - fields.push(DavPropertyValue::new( - property.clone(), - vec![SupportedPrivilege::all_privileges( - collection_container == Collection::Calendar, - )], - )); - } else { - fields.push(DavPropertyValue::new( - property.clone(), - vec![SupportedPrivilege::all_scheduling_privileges(matches!( - archive, - ArchivedResource::CalendarEventNotification(_) - | ArchivedResource::CalendarEventNotificationCollection( - true - ) - ))], - )); - } - } - WebDavProperty::CurrentUserPrivilegeSet => { - let privileges = if is_scheduling { - Privilege::scheduling( - matches!( - archive, - ArchivedResource::CalendarEventNotification(_) - | ArchivedResource::CalendarEventNotificationCollection( - true - ) - ), - access_token.is_member(account_id), - ) - } else if access_token.is_member(account_id) { - Privilege::all(matches!( - collection, - Collection::Calendar | Collection::CalendarEvent - )) - } else if let Some(acls) = archive.acls() { - access_token.current_privilege_set( - account_id, - acls, - collection_container == Collection::Calendar, - ) - } else if let Some(parent_id) = item.parent_id { - current_user_privilege_set( - data.resources(self, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())? - .container_acl(access_token, parent_id), - ) - } else { - vec![] - }; - - if !privileges.is_empty() { - fields.push(DavPropertyValue::new(property.clone(), privileges)); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::Acl => { - if let Some(acls) = archive.acls() { - let aces = self - .resolve_ace( - access_token, - account_id, - acls, - query.expand.then_some(&query.propfind), - ) - .await?; - - fields.push(DavPropertyValue::new(property.clone(), aces)); - } else if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - WebDavProperty::AclRestrictions => { - fields.push(DavPropertyValue::new( - property.clone(), - AclRestrictions::default() - .with_no_invert() - .with_grant_only(), - )); - } - WebDavProperty::InheritedAclSet => { - fields.push(DavPropertyValue::empty(property.clone())); - } - WebDavProperty::PrincipalCollectionSet => { - fields.push(DavPropertyValue::new( - property.clone(), - vec![Href( - DavResourceName::Principal.collection_path().to_string(), - )], - )); - } - }, - DavProperty::DeadProperty(tag) => { - if let Some(value) = - dead_properties.and_then(|props| props.find_tag(&tag.name)) - { - fields.push(DavPropertyValue::new(property.clone(), value)); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - DavProperty::CardDav(card_property) => match (card_property, &archive) { - ( - CardDavProperty::AddressbookDescription, - ArchivedResource::AddressBook(book), - ) => { - if let Some(desc) = - book.inner.preferences(personal_id).description.as_deref() - { - fields.push(DavPropertyValue::new( - property.clone(), - desc.to_string(), - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - ( - CardDavProperty::SupportedAddressData, - ArchivedResource::AddressBook(_), - ) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::SupportedAddressData, - )); - } - ( - CardDavProperty::SupportedCollationSet, - ArchivedResource::AddressBook(_), - ) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::Collations(List(vec![ - SupportedCollation { - collation: Collation::AsciiCasemap, - namespace: Namespace::CardDav, - }, - SupportedCollation { - collation: Collation::UnicodeCasemap, - namespace: Namespace::CardDav, - }, - ])), - )); - } - (CardDavProperty::MaxResourceSize, ArchivedResource::AddressBook(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - self.core.groupware.max_vcard_size as u64, - )); - } - ( - CardDavProperty::AddressData { - properties, - version, - }, - ArchivedResource::ContactCard(card), - ) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::CData(serialize_vcard_with_props( - &card.inner.card, - properties, - (*version) - .or(query.vcard_version) - .unwrap_or(self.core.groupware.vcard_version), - )), - )); - } - _ => { - if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - }, - DavProperty::CalDav(cal_property) => match (cal_property, &archive) { - ( - CalDavProperty::CalendarDescription, - ArchivedResource::Calendar(calendar), - ) => { - if let Some(desc) = calendar - .inner - .preferences(personal_id) - .description - .as_deref() - { - fields.push(DavPropertyValue::new( - property.clone(), - desc.to_string(), - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - ( - CalDavProperty::CalendarTimezone, - ArchivedResource::Calendar(calendar), - ) => { - if let ArchivedTimezone::Custom(tz) = - &calendar.inner.preferences(personal_id).time_zone - { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::CData(tz.to_string()), - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - (CalDavProperty::TimezoneId, ArchivedResource::Calendar(calendar)) => { - if let ArchivedTimezone::IANA(tz) = - &calendar.inner.preferences(personal_id).time_zone - { - fields.push(DavPropertyValue::new( - property.clone(), - Tz::from_id(tz.to_native()).unwrap_or(Tz::UTC).to_string(), - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - ( - CalDavProperty::SupportedCalendarComponentSet, - ArchivedResource::Calendar(calendar), - ) => { - let supported_components = - calendar.inner.supported_components.to_native(); - fields.push(DavPropertyValue::new( - property.clone(), - if supported_components != 0 { - DavValue::Components(List( - Bitmap::::from(supported_components) - .into_iter() - .map(ICalendarComponentType::from) - .map(Comp) - .collect(), - )) - } else { - DavValue::all_calendar_components() - }, - )); - } - (CalDavProperty::SupportedCalendarData, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::SupportedCalendarData, - )); - } - (CalDavProperty::SupportedCollationSet, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::Collations(List(vec![ - SupportedCollation { - collation: Collation::AsciiCasemap, - namespace: Namespace::CalDav, - }, - SupportedCollation { - collation: Collation::UnicodeCasemap, - namespace: Namespace::CalDav, - }, - ])), - )); - } - (CalDavProperty::MaxResourceSize, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - self.core.groupware.max_ical_size as u64, - )); - } - (CalDavProperty::MinDateTime, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String("0001-01-01T00:00:00Z".to_string()), - )); - } - (CalDavProperty::MaxDateTime, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String("9999-12-31T23:59:59Z".to_string()), - )); - } - (CalDavProperty::MaxInstances, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - self.core.groupware.max_ical_instances as u64, - )); - } - ( - CalDavProperty::MaxAttendeesPerInstance, - ArchivedResource::Calendar(_), - ) => { - fields.push(DavPropertyValue::new( - property.clone(), - self.core.groupware.max_ical_attendees_per_instance as u64, - )); - } - ( - CalDavProperty::CalendarData(data), - ArchivedResource::CalendarEvent(event), - ) => { - if calendar_filter.is_some() || !data.properties.is_empty() { - if let Some(ical) = calendar_filter - .get_or_insert_with(|| { - CalendarQueryHandler::new(event.inner, None, Tz::UTC) - }) - .serialize_ical(event.inner, data, &mut ical_instances_limit) - { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::CData(ical), - )); - } else { - limit = 0; - break 'outer; - } - } else { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::CData(event.inner.data.event.to_string()), - )); - } - } - ( - CalDavProperty::CalendarData(_), - ArchivedResource::CalendarEventNotification(event), - ) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::CData(event.inner.event.to_string()), - )); - } - (CalDavProperty::ScheduleTag, ArchivedResource::CalendarEvent(event)) - if event.inner.schedule_tag.is_some() => - { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::String(format!( - "\"{}\"", - event.inner.schedule_tag.as_ref().unwrap() - )), - )); - } - (CalDavProperty::ScheduleCalendarTransp, ArchivedResource::Calendar(_)) => { - fields.push(DavPropertyValue::new( - property.clone(), - DavValue::DeadProperty(DeadProperty::single_with_ns( - Namespace::CalDav, - "opaque", - )), - )); - } - ( - CalDavProperty::ScheduleDefaultCalendarURL, - ArchivedResource::CalendarEventNotificationCollection(true), - ) => { - if let Some(default_cal) = &self.core.groupware.default_calendar_name { - fields.push(DavPropertyValue::new( - property.clone(), - vec![Href(format!( - "{}/{}/{default_cal}/", - DavResourceName::Cal.base_path(), - item.name.split('/').nth(3).unwrap_or_default() - ))], - )); - } else { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - - _ => { - if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - }, - - property => { - if !skip_not_found { - fields_not_found.push(DavPropertyValue::empty(property.clone())); - } - } - } - } - - // Add dead properties - if skip_not_found - && let Some(dead_properties) = - dead_properties.filter(|dead_properties| !dead_properties.0.is_empty()) - { - dead_properties.to_dav_values(&mut fields); - } - - // Add response - let mut prop_stat = Vec::with_capacity(2); - if !fields.is_empty() { - prop_stat.push(PropStat::new_list(fields)); - } - if !fields_not_found.is_empty() && !query.is_minimal() { - prop_stat - .push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); - } - if prop_stat.is_empty() { - prop_stat.push(PropStat::new_list(vec![])); - } - response.add_response(Response::new_propstat(item.name, prop_stat)); - - limit -= 1; - if limit == 0 { - break; - } - } - - if limit == 0 || is_sync_limited { - response.add_response( - Response::new_status([query.uri], StatusCode::INSUFFICIENT_STORAGE) - .with_error(BaseCondition::NumberOfMatchesWithinLimit) - .with_response_description(if ical_instances_limit > 0 { - format!( - "The number of matches exceeds the limit of {}", - query - .limit - .unwrap_or(self.core.groupware.max_results as u32) - ) - } else { - format!( - "The number of recurrence instances exceeds the limit of {}", - query - .limit - .unwrap_or(self.core.groupware.max_ical_instances as u32) - ) - }), - ); - } - - if !is_propfind || !response.response.0.is_empty() || !query.sync_type.is_none() { - Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) - } else { - Ok(HttpResponse::new(StatusCode::NOT_FOUND)) - } - } - - async fn dav_quota(&self, account_id: u32) -> trc::Result { - let account = self.account(account_id).await.caused_by(trc::location!())?; - let used = self - .get_used_quota_account(account_id) - .await - .caused_by(trc::location!())? - .max(0) as u64; - let mut available = - (account.quota_disk > 0).then(|| account.quota_disk.saturating_sub(used)); - - if let Some(tenant_id) = account.id_tenant { - let tenant = self.tenant(tenant_id).await.caused_by(trc::location!())?; - - if tenant.quota_disk > 0 { - let tenant_used = self - .get_used_quota_tenant(tenant_id) - .await - .caused_by(trc::location!())? - .max(0) as u64; - let tenant_available = tenant.quota_disk.saturating_sub(tenant_used); - - available = Some(available.map_or(tenant_available, |available| { - available.min(tenant_available) - })); - } - } - - Ok(PropFindAccountQuota { used, available }) - } -} -#[allow(clippy::too_many_arguments)] -async fn get( - server: &Server, - access_token: &AccessToken, - collection_container: Collection, - collection_children: Collection, - sync_collection: SyncCollection, - query: &DavQuery<'_>, - data: &mut PropFindData, - response: &mut MultiStatus, - resource: UriResource>, - limit: usize, - is_sync_limited: &mut bool, -) -> crate::Result> { - let container_has_children = collection_children != collection_container; - response.set_namespace(collection_container.namespace()); - - let account_id = resource.account_id; - let resources = data - .resources(server, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())?; - - // Obtain document ids - let mut display_containers = if !access_token.is_member(account_id) { - resources - .shared_containers( - access_token, - [if container_has_children { - Acl::ReadItems - } else { - Acl::Read - }], - true, - ) - .into() - } else { - None - }; - let mut display_children = display_containers - .as_ref() - .filter(|_| container_has_children) - .map(|containers| { - RoaringBitmap::from_iter(resources.resources.iter().filter_map(|r| { - if r.child_names() - .is_some_and(|n| n.iter().any(|n| containers.contains(n.parent_id))) - { - Some(r.document_id) - } else { - None - } - })) - }); - - // Filter by changelog - let is_sync = match query.sync_type { - SyncType::From { id, seq } => { - let changes = server - .store() - .changes(account_id, sync_collection.into(), Query::Since(id)) - .await - .caused_by(trc::location!())?; - let mut vanished: Vec = Vec::new(); - - // Merge changes - let mut total_changes = 0; - let mut maybe_has_vanished = false; - if container_has_children { - let mut container_changes = RoaringBitmap::new(); - let mut item_changes = RoaringBitmap::new(); - - for change in changes.changes { - match change { - Change::InsertItem(id) => { - item_changes.insert(id as u32); - } - Change::UpdateItem(id) => { - maybe_has_vanished = true; - item_changes.insert(id as u32); - } - Change::InsertContainer(id) => { - container_changes.insert(id as u32); - } - Change::UpdateContainer(id) => { - maybe_has_vanished = true; - container_changes.insert(id as u32); - } - Change::DeleteContainer(_) | Change::DeleteItem(_) => { - maybe_has_vanished = true; - } - Change::UpdateContainerProperty(_) => (), - } - } - - for (document_ids, changes) in [ - (&mut display_containers, container_changes), - (&mut display_children, item_changes), - ] { - if let Some(document_ids) = document_ids { - *document_ids &= changes; - total_changes += document_ids.len() as usize; - } else { - total_changes += changes.len() as usize; - *document_ids = Some(changes); - } - } - } else { - let changes = RoaringBitmap::from_iter(changes.changes.iter().filter_map( - |change| match change { - Change::InsertItem(id) | Change::InsertContainer(id) => Some(*id as u32), - Change::UpdateItem(id) | Change::UpdateContainer(id) => { - maybe_has_vanished = true; - Some(*id as u32) - } - Change::DeleteContainer(_) | Change::DeleteItem(_) => { - maybe_has_vanished = true; - None - } - _ => None, - }, - )); - if let Some(document_ids) = &mut display_containers { - *document_ids &= changes; - total_changes += document_ids.len() as usize; - } else { - total_changes += changes.len() as usize; - display_containers = Some(changes); - } - } - - if maybe_has_vanished - && let Some(vanished_collection) = sync_collection.vanished_collection() - { - vanished = server - .store() - .vanished(account_id, vanished_collection.into(), Query::Since(id)) - .await - .caused_by(trc::location!())?; - total_changes += vanished.len(); - } - - // Truncate changes - if total_changes > limit { - let mut offset = limit * seq as usize; - let mut total_changes = 0; - - // Add vanished items to response - for item in vanished { - if offset > 0 { - offset -= 1; - } else if total_changes < limit { - response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); - total_changes += 1; - } else { - *is_sync_limited = true; - } - } - - // Add items to document set - for document_ids in [&mut display_containers, &mut display_children] - .into_iter() - .flatten() - { - let mut new_document_ids = RoaringBitmap::new(); - for id in document_ids.iter() { - if offset > 0 { - offset -= 1; - } else if total_changes < limit { - new_document_ids.insert(id); - total_changes += 1; - } else { - *is_sync_limited = true; - } - } - *document_ids = new_document_ids; - } - - if *is_sync_limited { - response.set_sync_token(Urn::Sync { id, seq: seq + 1 }.to_string()); - } - } else { - // Add vanished items to response - for item in vanished { - response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); - } - } - - if !*is_sync_limited { - response.set_sync_token(resources.sync_token()); - } - - true - } - SyncType::Initial => { - response.set_sync_token(resources.sync_token()); - false - } - SyncType::None => false, - }; - - let mut results = Vec::new(); - if let Some(resource) = resource.resource { - results = resources - .subtree_with_depth(resource, query.depth) - .filter(|item| { - display_containers.as_ref().is_none_or(|containers| { - if container_has_children { - if item.is_container() { - containers.contains(item.document_id()) - } else { - display_children - .as_ref() - .is_some_and(|children| children.contains(item.document_id())) - } - } else { - containers.contains(item.document_id()) - } - }) && (!query.depth_no_root || item.path() != resource) - }) - .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) - .collect::>(); - } else { - if !query.depth_no_root && query.sync_type.is_none_or_initial() { - server - .prepare_principal_propfind_response( - access_token, - collection_container, - [account_id].into_iter(), - &query.propfind, - response, - ) - .await?; - } - - if query.depth != 0 { - results = resources - .tree_with_depth(query.depth - 1) - .filter(|item| { - display_containers.as_ref().is_none_or(|containers| { - if container_has_children { - if item.is_container() { - containers.contains(item.document_id()) - } else { - display_children - .as_ref() - .is_some_and(|children| children.contains(item.document_id())) - } - } else { - containers.contains(item.document_id()) - } - }) - }) - .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) - .collect::>(); - - // Assisted discovery: - // If 'bob' has access to 'jane' and `bill` calendars, a query to '/dav/cal/bob' will return: - // - /dav/cal/bob/default - // - /dav/cal/jane/default - // - /dav/cal/bill/default - // This is invalid but it's the only workaround for clients which do not support multiple home-sets - if server.core.groupware.assisted_discovery - && !is_sync - && account_id == access_token.account_id() - && matches!( - sync_collection, - SyncCollection::Calendar | SyncCollection::AddressBook - ) - { - for shared_account_id in access_token.all_ids_by_collection(collection_container) { - if shared_account_id == access_token.account_id() { - continue; - } - let shared_resources = data - .resources(server, access_token, shared_account_id, sync_collection) - .await - .caused_by(trc::location!())?; - let shared_containers = - (!access_token.is_member(shared_account_id)).then(|| { - shared_resources.shared_containers( - access_token, - [if container_has_children { - Acl::ReadItems - } else { - Acl::Read - }], - true, - ) - }); - if shared_containers - .as_ref() - .is_none_or(|containers| !containers.is_empty()) - { - results.extend( - shared_resources - .tree_with_depth(query.depth - 1) - .filter(|item| { - item.is_container() - && shared_containers.as_ref().is_none_or(|containers| { - containers.contains(item.document_id()) - }) - }) - .map(|item| { - PropFindItem::new( - shared_resources.format_resource(item), - shared_account_id, - item, - ) - }), - ); - } - } - } - } - } - - Ok(results) -} - -#[allow(clippy::too_many_arguments)] -async fn multiget( - server: &Server, - access_token: &AccessToken, - collection_container: Collection, - collection_children: Collection, - sync_collection: SyncCollection, - data: &mut PropFindData, - response: &mut MultiStatus, - hrefs: Vec, -) -> crate::Result> { - let mut paths = Vec::with_capacity(hrefs.len() * 2); - let mut shared_folders_by_account: AHashMap> = - AHashMap::with_capacity(3); - - for item in hrefs { - let resource = match server - .validate_uri(access_token, &item) - .await - .and_then(|r| r.into_owned_uri()) - { - Ok(resource) => resource, - Err(DavError::Code(code)) => { - response.add_response(Response::new_status([item], code)); - continue; - } - Err(err) => { - return Err(err); - } - }; - - let account_id = resource.account_id; - let resources = data - .resources(server, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())?; - - let document_ids = if !access_token.is_member(account_id) { - if let Some(document_ids) = shared_folders_by_account.get(&account_id) { - document_ids.clone().into() - } else { - let document_ids = Arc::new(resources.shared_containers( - access_token, - [if collection_children == collection_container { - Acl::ReadItems - } else { - Acl::Read - }], - true, - )); - shared_folders_by_account.insert(account_id, document_ids.clone()); - document_ids.into() - } - } else { - None - }; - - if let Some(resource) = resource.resource.and_then(|name| resources.by_path(name)) { - if !resource.is_container() { - if document_ids - .as_ref() - .is_none_or(|docs| docs.contains(resource.parent_id().unwrap())) - { - paths.push(PropFindItem::new( - resources.format_resource(resource), - account_id, - resource, - )); - } else { - response.add_response( - Response::new_status([item], StatusCode::FORBIDDEN) - .with_response_description( - "Not enough permissions to access this shared resource", - ), - ); - } - } else { - response.add_response( - Response::new_status([item], StatusCode::FORBIDDEN) - .with_response_description("Multiget not allowed for collections"), - ); - } - } else { - response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); - } - } - - Ok(paths) -} - -impl PropFindItem { - pub fn new(name: String, account_id: u32, resource: DavResourcePath<'_>) -> Self { - Self { - name, - account_id, - document_id: resource.document_id(), - parent_id: resource.parent_id(), - is_container: resource.is_container(), - } - } -} - -impl PropFindData { - pub fn new() -> Self { - Self { - accounts: AHashMap::with_capacity(2), - } - } - - pub async fn quota( - &mut self, - server: &Server, - account_id: u32, - ) -> trc::Result { - let data = self.accounts.entry(account_id).or_default(); - - if data.quota.is_none() { - data.quota = server.dav_quota(account_id).await?.into(); - } - - Ok(data.quota.clone().unwrap()) - } - - pub async fn owner( - &mut self, - server: &Server, - account_info: &AccountCache, - account_id: u32, - ) -> trc::Result { - let data = self.accounts.entry(account_id).or_default(); - - if data.owner.is_none() { - data.owner = server - .owner_href(account_info, account_id) - .await - .caused_by(trc::location!())? - .into(); - } - - Ok(data.owner.clone().unwrap()) - } - - pub async fn resources( - &mut self, - server: &Server, - access_token: &AccessToken, - account_id: u32, - sync_collection: SyncCollection, - ) -> trc::Result> { - let data = self.accounts.entry(account_id).or_default(); - - if data.resources.is_none() { - let resources = server - .fetch_dav_resources(access_token.account_id(), account_id, sync_collection) - .await - .caused_by(trc::location!())?; - data.resources = resources.into(); - } - - Ok(data.resources.clone().unwrap()) - } - - pub async fn locks( - &mut self, - server: &Server, - account_id: u32, - collection_container: Collection, - item: &PropFindItem, - ) -> trc::Result>> { - let data = self.accounts.entry(account_id).or_default(); - - if data.locks.is_none() && !data.locks_not_found { - data.locks = server - .in_memory_store() - .key_get::>( - build_lock_key(account_id, collection_container).as_slice(), - ) - .await - .caused_by(trc::location!())?; - if data.locks.is_none() { - data.locks_not_found = true; - } - } - - if let Some(lock_data) = &data.locks { - let base_uri = dav_base_uri(&item.name).unwrap_or_default(); - lock_data.unarchive::().map(|locks| { - locks - .find_locks(&item.name.strip_prefix(base_uri).unwrap()[1..], false) - .iter() - .map(|(path, lock)| lock.to_active_lock(format!("{base_uri}/{path}"))) - .collect::>() - .into() - }) - } else { - Ok(None) - } - } -} - -pub(crate) trait SyncTokenUrn { - fn sync_token(&self) -> String; -} - -impl SyncTokenUrn for DavResources { - fn sync_token(&self) -> String { - Urn::Sync { - id: self.highest_change_id, - seq: 0, - } - .to_string() - } -} - -async fn add_base_collection_response( - server: &Server, - request: &PropFind, - collection: Collection, - access_token: &AccessToken, - response: &mut MultiStatus, -) -> trc::Result<()> { - let properties = match request { - PropFind::PropName => { - response.add_response(Response::new_propstat( - DavResourceName::from(collection).collection_path(), - vec![PropStat::new_list(vec![ - DavPropertyValue::empty(DavProperty::WebDav(WebDavProperty::ResourceType)), - DavPropertyValue::empty(DavProperty::WebDav( - WebDavProperty::CurrentUserPrincipal, - )), - DavPropertyValue::empty(DavProperty::WebDav( - WebDavProperty::SupportedReportSet, - )), - ])], - )); - return Ok(()); - } - PropFind::AllProp(include) => { - let mut props = Vec::with_capacity(include.len() + 2); - props.extend([ - DavProperty::WebDav(WebDavProperty::ResourceType), - DavProperty::WebDav(WebDavProperty::SupportedReportSet), - ]); - for prop in include { - if !props.contains(prop) { - props.push(prop.clone()); - } - } - Cow::Owned(props) - } - PropFind::Prop(items) => Cow::Borrowed(items.as_slice()), - }; - - let mut fields = Vec::with_capacity(properties.len()); - let mut fields_not_found = Vec::new(); - let account_info = server - .account(access_token.account_id()) - .await - .caused_by(trc::location!())?; - - for prop in properties.iter() { - response.set_namespace(prop.namespace()); - match prop { - DavProperty::WebDav(WebDavProperty::ResourceType) => { - fields.push(DavPropertyValue::new( - prop.clone(), - vec![ResourceType::Collection], - )); - } - DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal) => { - fields.push(DavPropertyValue::new( - prop.clone(), - vec![account_info.current_user_principal()], - )); - } - DavProperty::Principal(PrincipalProperty::CalendarHomeSet) => { - let hrefs = build_home_set( - server, - access_token, - account_info.name(), - access_token.account_id(), - true, - ) - .await - .caused_by(trc::location!())?; - - fields.push(DavPropertyValue::new(prop.clone(), hrefs)); - } - DavProperty::Principal(PrincipalProperty::AddressbookHomeSet) => { - let hrefs = build_home_set( - server, - access_token, - account_info.name(), - access_token.account_id(), - false, - ) - .await - .caused_by(trc::location!())?; - - fields.push(DavPropertyValue::new(prop.clone(), hrefs)); - } - DavProperty::WebDav(WebDavProperty::SupportedReportSet) => { - let reports = match collection { - Collection::Principal => ReportSet::principal(), - Collection::Calendar | Collection::CalendarEvent => ReportSet::calendar(), - Collection::AddressBook | Collection::ContactCard => ReportSet::addressbook(), - _ => ReportSet::file(), - }; - - fields.push(DavPropertyValue::new(prop.clone(), reports)); - response.set_namespace(collection.namespace()); - } - _ => { - fields_not_found.push(DavPropertyValue::empty(prop.clone())); - } - } - } - - let mut prop_stat = Vec::with_capacity(2); - - if !fields.is_empty() { - prop_stat.push(PropStat::new_list(fields)); - } - - if !fields_not_found.is_empty() { - prop_stat.push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); - } - - response.add_response(Response::new_propstat( - DavResourceName::from(collection).collection_path(), - prop_stat, - )); - - Ok(()) -} diff --git a/crates/dav/src/common/propfind/data.rs b/crates/dav/src/common/propfind/data.rs new file mode 100644 index 00000000..ed3b9a29 --- /dev/null +++ b/crates/dav/src/common/propfind/data.rs @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{PropFindItem, PropFindRequestHandler}; +use crate::{ + common::{ + lock::{LockData, build_lock_key}, + uri::{SyncToken, Urn}, + }, + principal::propfind::PrincipalPropFind, +}; +use common::{ + GroupwareResources, Server, + auth::{AccessToken, AccountCache}, + sharing::file::FileNodeAccess, + storage::dav::canonical_dav_resource_uri, +}; +use dav_proto::{ + parser::header::dav_base_uri, + schema::{ + property::ActiveLock, + request::PropFind, + response::{Href, Response}, + }, +}; +use groupware::cache::GroupwareCache; +use std::{borrow::Cow, sync::Arc}; +use store::{ + ahash::AHashMap, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::collection::{Collection, SyncCollection}; + +pub(crate) struct PropFindData { + pub accounts: AHashMap, + pub principals: ExpandedPrincipals, +} + +#[derive(Default)] +pub(crate) struct ExpandedPrincipals(AHashMap>); + +#[derive(Default)] +pub(crate) struct PropFindAccountData { + pub resources: Option>, + pub quota: Option, + pub owner: Option, + pub locks: Option, + pub locks_not_found: bool, + pub file_access: Option, +} + +#[derive(Clone, Default)] +pub(crate) struct PropFindAccountQuota { + pub used: u64, + pub available: Option, +} + +impl PropFindData { + pub fn new() -> Self { + Self { + accounts: AHashMap::with_capacity(2), + principals: ExpandedPrincipals::default(), + } + } + + pub async fn quota( + &mut self, + server: &Server, + account_id: u32, + ) -> trc::Result { + let data = self.accounts.entry(account_id).or_default(); + + if let Some(quota) = &data.quota { + Ok(quota.clone()) + } else { + let quota = server.dav_quota(account_id).await?; + data.quota = Some(quota.clone()); + Ok(quota) + } + } + + pub async fn owner( + &mut self, + server: &Server, + account_info: &AccountCache, + account_id: u32, + ) -> trc::Result { + let data = self.accounts.entry(account_id).or_default(); + + if let Some(owner) = &data.owner { + Ok(owner.clone()) + } else { + let owner = server + .owner_href(account_info, account_id) + .await + .caused_by(trc::location!())?; + data.owner = Some(owner.clone()); + Ok(owner) + } + } + + pub async fn resources( + &mut self, + server: &Server, + access_token: &AccessToken, + account_id: u32, + sync_collection: SyncCollection, + min_change_id: u64, + ) -> trc::Result> { + let data = self.accounts.entry(account_id).or_default(); + + if let Some(resources) = &data.resources { + Ok(resources.clone()) + } else { + let resources = server + .fetch_groupware_resources_at( + access_token.account_id(), + account_id, + sync_collection, + min_change_id, + ) + .await + .caused_by(trc::location!())?; + data.resources = Some(resources.clone()); + Ok(resources) + } + } + + pub async fn locks( + &mut self, + server: &Server, + account_id: u32, + collection_container: Collection, + item: &PropFindItem, + ) -> trc::Result>> { + let data = self.accounts.entry(account_id).or_default(); + + if data.locks.is_none() && !data.locks_not_found { + data.locks = server + .in_memory_store() + .key_get::>( + build_lock_key(account_id, collection_container).as_slice(), + ) + .await + .caused_by(trc::location!())? + .map(|lock_data| lock_data.deserialize::()) + .transpose() + .caused_by(trc::location!())?; + if data.locks.is_none() { + data.locks_not_found = true; + } + } + + let Some(lock_data) = &data.locks else { + return Ok(None); + }; + let base_uri = dav_base_uri(&item.name).unwrap_or_default(); + let name = if collection_container == Collection::FileNode { + canonical_dav_resource_uri(&item.name).unwrap_or(Cow::Borrowed(item.name.as_str())) + } else { + Cow::Borrowed(item.name.as_str()) + }; + let Some(path) = name + .strip_prefix(base_uri) + .and_then(|path| path.strip_prefix('/')) + .map(|path| path.trim_end_matches('/')) + else { + return Ok(None); + }; + Ok(Some(lock_data.active_locks(path, base_uri))) + } +} + +impl ExpandedPrincipals { + pub async fn expand( + &mut self, + server: &Server, + access_token: &AccessToken, + account_id: u32, + propfind: &PropFind, + ) -> crate::Result> { + if let Some(response) = self.0.get(&account_id) { + return Ok(response.clone()); + } + let response = server + .expand_principal(access_token, account_id, propfind) + .await?; + self.0.insert(account_id, response.clone()); + Ok(response) + } +} + +pub(crate) trait SyncTokenUrn { + fn sync_token(&self) -> String; +} + +impl SyncTokenUrn for GroupwareResources { + fn sync_token(&self) -> String { + Urn::Sync(SyncToken::State(self.highest_change_id)).to_string() + } +} diff --git a/crates/dav/src/common/propfind/dead.rs b/crates/dav/src/common/propfind/dead.rs new file mode 100644 index 00000000..b1ceabe3 --- /dev/null +++ b/crates/dav/src/common/propfind/dead.rs @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::PropFindItem; +use crate::common::dead::{ContainerRequest, DeadContainers}; +use common::{DavResourcePath, Server, auth::AccessToken, storage::dav::DISPLAY_NAME_PROPERTY}; +use dav_proto::schema::{ + property::{DavProperty, WebDavProperty}, + request::DavPropertyValue, +}; +use groupware::calendar::privacy::EventPrivacy; +use store::write::metadata::MetadataBuf; +use types::{collection::Collection, metadata::MetadataKinds}; + + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct DeadPresence { + properties: bool, + display_name: bool, + is_private: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct DeadDemand { + pub properties: bool, + pub display_name: bool, +} + +impl DeadPresence { + pub fn from_resource(resource: &DavResourcePath<'_>) -> Self { + let resource = resource.resource; + match resource.file_flags() { + Some(flags) => { + let presence = flags.presence(); + DeadPresence { + properties: presence.has_dead_properties(), + display_name: presence.has_dav_display_name(), + is_private: false, + } + } + None => DeadPresence { + properties: resource.metadata_kinds().intersects(MetadataKinds::DAV), + display_name: false, + is_private: resource + .event_flags() + .is_some_and(|flags| !EventPrivacy::from_flags(flags).is_public()), + }, + } + } + + pub fn is_hidden_from(&self, access_token: &AccessToken, account_id: u32) -> bool { + self.is_private && !access_token.is_member(account_id) + } +} + +impl DeadDemand { + pub fn new( + properties: &[DavProperty], + is_allprop: bool, + is_propname: bool, + collection: Collection, + ) -> Self { + let is_file = collection == Collection::FileNode; + DeadDemand { + properties: is_allprop + || is_propname + || properties + .iter() + .any(|property| matches!(property, DavProperty::Dead(_))), + display_name: is_file + && (is_allprop + || properties.iter().any(|property| { + matches!(property, DavProperty::WebDav(WebDavProperty::DisplayName)) + })), + } + } + + pub fn is_empty(&self) -> bool { + !self.properties && !self.display_name + } + + fn wants(&self, item: &PropFindItem) -> bool { + (self.properties && item.dead.properties && !item.is_discover_only) + || (self.display_name && item.dead.display_name) + } + + pub async fn load( + &self, + server: &Server, + access_token: &AccessToken, + items: &[PropFindItem], + collection_of: impl Fn(&PropFindItem) -> Collection, + ) -> trc::Result { + if self.is_empty() { + return Ok(DeadContainers::default()); + } + let mut request = ContainerRequest::default(); + for item in items { + if self.wants(item) && !item.dead.is_hidden_from(access_token, item.account_id) { + request.insert(item.account_id, collection_of(item), item.document_id); + } + } + if request.is_empty() { + Ok(DeadContainers::default()) + } else { + request.load(server).await + } + } +} + +pub(crate) fn dead_properties( + container: &MetadataBuf, + collection: Collection, +) -> impl Iterator + '_ { + let skip_display_name = collection == Collection::FileNode; + container + .view() + .dav() + .filter(move |(name, _)| !(skip_display_name && *name == DISPLAY_NAME_PROPERTY)) + .map(|(name, value)| { + DavPropertyValue::new(DavProperty::Dead(name.into_owned()), value.to_encoded()) + }) +} + +pub(crate) fn dead_property_names( + container: &MetadataBuf, + collection: Collection, +) -> impl Iterator + '_ { + let skip_display_name = collection == Collection::FileNode; + container + .view() + .dav() + .filter(move |(name, _)| !(skip_display_name && *name == DISPLAY_NAME_PROPERTY)) + .map(|(name, _)| DavPropertyValue::empty(DavProperty::Dead(name.into_owned()))) +} diff --git a/crates/dav/src/common/propfind/get.rs b/crates/dav/src/common/propfind/get.rs new file mode 100644 index 00000000..d49c5894 --- /dev/null +++ b/crates/dav/src/common/propfind/get.rs @@ -0,0 +1,656 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{PropFindItem, data::PropFindData}; +use crate::{ + DavError, DavErrorCondition, + common::{ + DavCollection, DavQuery, SyncType, + uri::{DavUriResource, SyncCursor, SyncToken, UriResource, Urn, canonical_dav_uri}, + }, + file::is_symlink, + principal::propfind::PrincipalPropFind, +}; +use common::{DavResourcePath, Server, auth::AccessToken}; +use dav_proto::schema::response::{BaseCondition, MultiStatus, Response}; +use groupware::calendar::EVENT_SECRET; +use hyper::StatusCode; +use std::{borrow::Cow, sync::Arc}; +use store::{ + ahash::AHashMap, + query::log::{Change, Changes, Query}, + roaring::RoaringBitmap, +}; +use trc::AddContext; +use types::{ + acl::Acl, + collection::{Collection, SyncCollection}, +}; + +#[allow(clippy::too_many_arguments)] +pub(super) async fn get( + server: &Server, + access_token: &AccessToken, + collection_container: Collection, + collection_children: Collection, + sync_collection: SyncCollection, + query: &DavQuery<'_>, + data: &mut PropFindData, + response: &mut MultiStatus, + resource: UriResource>, + limit: usize, + is_sync_limited: &mut bool, +) -> crate::Result> { + let container_has_children = collection_children != collection_container; + response.set_namespace(collection_container.namespace()); + + let account_id = resource.account_id; + let min_change_id = match query.sync_type { + SyncType::Token( + SyncToken::State(snapshot) + | SyncToken::ChangesPage { snapshot, .. } + | SyncToken::InitialPage { snapshot, .. }, + ) => snapshot, + _ => 0, + }; + let resources = data + .resources( + server, + access_token, + account_id, + sync_collection, + min_change_id, + ) + .await + .caused_by(trc::location!())?; + + // Obtain document ids + let is_file = sync_collection == SyncCollection::FileNode; + let mut file_readable = None; + let mut display_containers = if is_file && !access_token.is_member(account_id) { + let access = resources.file_access(access_token); + file_readable = Some(access.readable.clone()); + let discoverable = access.discoverable.clone(); + data.accounts.entry(account_id).or_default().file_access = Some(access); + Some(discoverable) + } else if !access_token.is_member(account_id) { + resources + .shared_containers( + access_token, + [if container_has_children { + Acl::ReadItems + } else { + Acl::Read + }], + true, + ) + .into() + } else { + None + }; + let visible_containers = display_containers + .as_ref() + .filter(|_| query.sync_type.is_incremental()) + .cloned(); + let mut display_children = display_containers + .as_ref() + .filter(|_| container_has_children) + .map(|containers| { + let hide_secret = sync_collection == SyncCollection::Calendar; + containers + .iter() + .flat_map(|container_id| resources.children(container_id)) + .filter(|child| { + !hide_secret + || child + .resource + .event_flags() + .is_none_or(|flags| flags & EVENT_SECRET == 0) + }) + .map(|child| child.document_id()) + .collect::() + }); + + // Filter by changelog + let incremental = match query.sync_type { + SyncType::Token(SyncToken::State(id)) => Some((id, resources.highest_change_id.max(id), 0)), + SyncType::Token(SyncToken::ChangesPage { + from, + snapshot, + offset, + }) => Some((from, snapshot, offset)), + _ => None, + }; + let is_sync = match incremental { + Some((id, snapshot, offset)) => { + let changes = if id < snapshot { + let changes = server + .store() + .changes( + account_id, + sync_collection.into(), + Query::Range(id, snapshot), + ) + .await + .caused_by(trc::location!())?; + if changes.needs_full_rebuild(id) { + return Err(DavErrorCondition::new( + StatusCode::FORBIDDEN, + BaseCondition::ValidSyncToken, + ) + .into()); + } + changes + } else { + Changes::default() + }; + + let mut maybe_has_vanished = false; + let mut container_changes = RoaringBitmap::new(); + let mut item_changes = RoaringBitmap::new(); + let mut updates = RoaringBitmap::new(); + let track_hidden_items = + display_containers.is_some() && (container_has_children || is_file); + for change in changes.changes { + match change { + Change::InsertItem(id) | Change::UpdateItemMetadata(id) + if container_has_children => + { + item_changes.insert(id as u32); + } + Change::UpdateItem(id) if container_has_children => { + maybe_has_vanished = true; + item_changes.insert(id as u32); + if track_hidden_items { + updates.insert(id as u32); + } + } + Change::InsertItem(id) + | Change::InsertContainer(id) + | Change::UpdateItemMetadata(id) => { + container_changes.insert(id as u32); + } + Change::UpdateItem(id) | Change::UpdateContainer(id) => { + maybe_has_vanished = true; + container_changes.insert(id as u32); + if track_hidden_items && !container_has_children { + updates.insert(id as u32); + } + } + Change::DeleteContainer(_) | Change::DeleteItem(_) => { + maybe_has_vanished = true; + } + Change::UpdateContainerPartial(id, partial) => { + if partial.has_metadata() { + container_changes.insert(id as u32); + } + } + } + } + + let vanished: Vec = match sync_collection.vanished_collection() { + Some(vanished_collection) if maybe_has_vanished => server + .store() + .vanished( + account_id, + vanished_collection.into(), + Query::Range(id, snapshot), + ) + .await + .caused_by(trc::location!())?, + _ => Vec::new(), + }; + + let scope = resource.resource.map_or_else( + || resources.base_path.clone(), + |path| resources.format_collection(path), + ); + let is_visible_href = |href: &str| { + href.starts_with(scope.as_str()) + && visible_containers.as_ref().is_none_or(|containers| { + let Some(path) = href.strip_prefix(resources.base_path.as_str()) else { + return false; + }; + match path.trim_end_matches('/').rsplit_once('/') { + Some((parent, _)) => resources + .by_path(parent) + .is_some_and(|parent| containers.contains(parent.document_id())), + None => false, + } + }) + }; + let hidden_href = |document_id: u32| { + if !updates.contains(document_id) { + return None; + } + let containers = display_containers.as_ref()?; + if container_has_children { + let parent_id = resources + .item_by_id(document_id)? + .child_names() + .iter() + .map(|name| name.parent_id) + .find(|parent_id| containers.contains(*parent_id))?; + resources.format_resource_path_by_parent(document_id, parent_id) + } else { + let resource = resources.any_resource_path_by_id(document_id)?; + match resource.parent_id() { + Some(parent_id) if containers.contains(parent_id) => { + Some(resources.format_resource(resource)) + } + _ => None, + } + } + .filter(|href| is_visible_href(href)) + }; + + let mut removed = Vec::new(); + let mut page_containers = RoaringBitmap::new(); + let mut page_items = RoaringBitmap::new(); + let mut emitted = 0; + let mut next_offset = None; + for (entry, position) in vanished + .into_iter() + .map(SyncEntry::Removed) + .chain(container_changes.iter().map(SyncEntry::Container)) + .chain(item_changes.iter().map(SyncEntry::Item)) + .skip(usize::try_from(offset).unwrap_or(usize::MAX)) + .zip(offset..) + { + let entry = match entry { + SyncEntry::Removed(href) => { + is_visible_href(&href).then_some(SyncEntry::Removed(href)) + } + SyncEntry::Container(document_id) => { + if display_containers + .as_ref() + .is_none_or(|containers| containers.contains(document_id)) + { + Some(SyncEntry::Container(document_id)) + } else if container_has_children { + None + } else { + hidden_href(document_id).map(SyncEntry::Removed) + } + } + SyncEntry::Item(document_id) => { + if display_children + .as_ref() + .is_none_or(|children| children.contains(document_id)) + { + Some(SyncEntry::Item(document_id)) + } else { + hidden_href(document_id).map(SyncEntry::Removed) + } + } + }; + if let Some(entry) = entry { + if emitted == limit { + next_offset = Some(position); + break; + } + match entry { + SyncEntry::Removed(href) => removed.push(href), + SyncEntry::Container(document_id) => { + page_containers.insert(document_id); + } + SyncEntry::Item(document_id) => { + page_items.insert(document_id); + } + } + emitted += 1; + } + } + display_containers = Some(page_containers); + display_children = Some(page_items); + for href in removed { + response.add_response(Response::new_status([href], StatusCode::NOT_FOUND)); + } + + *is_sync_limited = next_offset.is_some(); + response.set_sync_token( + Urn::Sync(match next_offset { + Some(offset) => SyncToken::ChangesPage { + from: id, + snapshot, + offset, + }, + None => SyncToken::State(snapshot), + }) + .to_string(), + ); + + true + } + None => false, + }; + let mut initial = match query.sync_type { + SyncType::Initial => Some(InitialPaging { + snapshot: resources.highest_change_id, + after: None, + boundary: None, + }), + SyncType::Token(SyncToken::InitialPage { snapshot, after }) => Some(InitialPaging { + snapshot, + after: Some(after), + boundary: None, + }), + _ => None, + }; + + let mut results = Vec::new(); + if let Some(resource) = resource.resource { + let paths = resources + .subtree_with_depth(resource, query.depth) + .filter(|item| { + display_containers.as_ref().is_none_or(|containers| { + if container_has_children { + if item.is_container() { + containers.contains(item.document_id()) + } else { + display_children + .as_ref() + .is_some_and(|children| children.contains(item.document_id())) + } + } else { + containers.contains(item.document_id()) + } + }) && (!query.depth_no_root || item.path() != resource) + && (!is_file || !is_symlink(item)) + }); + results = collect_page(paths, account_id, initial.as_mut(), limit, |item| { + let is_discover_only = is_discover_only(file_readable.as_ref(), &item); + let href = if is_file && item.path() == resource { + requested_href(query.uri, item.is_container()) + } else { + resources.format_resource(item) + }; + PropFindItem::new(href, account_id, item).with_discover_only(is_discover_only) + }); + } else { + if !query.depth_no_root && query.sync_type.is_none_or_initial() { + server + .prepare_principal_propfind_response( + access_token, + collection_container, + [account_id].into_iter(), + &query.propfind, + response, + ) + .await?; + } + + if query.depth != 0 { + let paths = resources.tree_with_depth(query.depth - 1).filter(|item| { + display_containers.as_ref().is_none_or(|containers| { + if container_has_children { + if item.is_container() { + containers.contains(item.document_id()) + } else { + display_children + .as_ref() + .is_some_and(|children| children.contains(item.document_id())) + } + } else { + containers.contains(item.document_id()) + } + }) && (!is_file || !is_symlink(item)) + }); + results = collect_page(paths, account_id, initial.as_mut(), limit, |item| { + let is_discover_only = is_discover_only(file_readable.as_ref(), &item); + PropFindItem::new(resources.format_resource(item), account_id, item) + .with_discover_only(is_discover_only) + }); + + // Assisted discovery: + // If 'bob' has access to 'jane' and `bill` calendars, a query to '/dav/cal/bob' will return: + // - /dav/cal/bob/default + // - /dav/cal/jane/default + // - /dav/cal/bill/default + // This is invalid but it's the only workaround for clients which do not support multiple home-sets + if server.core.groupware.assisted_discovery + && !is_sync + && account_id == access_token.account_id() + && matches!( + sync_collection, + SyncCollection::Calendar | SyncCollection::AddressBook + ) + { + for shared_account_id in access_token.all_ids_by_collection(collection_container) { + if shared_account_id == access_token.account_id() { + continue; + } + let shared_resources = data + .resources(server, access_token, shared_account_id, sync_collection, 0) + .await + .caused_by(trc::location!())?; + let shared_containers = + (!access_token.is_member(shared_account_id)).then(|| { + shared_resources.shared_containers( + access_token, + [if container_has_children { + Acl::ReadItems + } else { + Acl::Read + }], + true, + ) + }); + if shared_containers + .as_ref() + .is_none_or(|containers| !containers.is_empty()) + { + results.extend( + shared_resources + .tree_with_depth(query.depth - 1) + .filter(|item| { + item.is_container() + && shared_containers.as_ref().is_none_or(|containers| { + containers.contains(item.document_id()) + }) + }) + .map(|item| { + PropFindItem::new( + shared_resources.format_resource(item), + shared_account_id, + item, + ) + }), + ); + } + } + } + } + } + + if let Some(paging) = initial { + results.retain(|item| { + let cursor = item.sync_cursor(); + paging.after.is_none_or(|after| cursor > after) + && paging.boundary.is_none_or(|boundary| cursor <= boundary) + }); + let boundary = + select_page(&mut results, limit, PropFindItem::sync_cursor).or(paging.boundary); + *is_sync_limited = boundary.is_some(); + response.set_sync_token( + Urn::Sync(match boundary { + Some(after) => SyncToken::InitialPage { + snapshot: paging.snapshot, + after, + }, + None => SyncToken::State(paging.snapshot), + }) + .to_string(), + ); + } + + Ok(results) +} + +enum SyncEntry { + Removed(String), + Container(u32), + Item(u32), +} + +struct InitialPaging { + snapshot: u64, + after: Option, + boundary: Option, +} + +fn collect_page<'x>( + paths: impl Iterator>, + account_id: u32, + paging: Option<&mut InitialPaging>, + limit: usize, + to_item: impl FnMut(DavResourcePath<'x>) -> PropFindItem, +) -> Vec { + match paging { + Some(paging) => { + let mut paths = paths + .filter(|path| { + paging + .after + .is_none_or(|after| SyncCursor::from_path(account_id, path) > after) + }) + .collect::>(); + paging.boundary = select_page(&mut paths, limit, |path| { + SyncCursor::from_path(account_id, path) + }); + paths.into_iter().map(to_item).collect() + } + None => paths.map(to_item).collect(), + } +} + +fn select_page( + items: &mut Vec, + limit: usize, + cursor: impl Fn(&T) -> SyncCursor, +) -> Option { + if items.len() <= limit { + return None; + } + let boundary = cursor(items.select_nth_unstable_by_key(limit - 1, &cursor).1); + items.retain(|item| cursor(item) <= boundary); + Some(boundary) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn multiget( + server: &Server, + access_token: &AccessToken, + collection_container: Collection, + collection_children: Collection, + sync_collection: SyncCollection, + data: &mut PropFindData, + response: &mut MultiStatus, + hrefs: Vec, +) -> crate::Result> { + let mut paths = Vec::with_capacity(hrefs.len() * 2); + let mut shared_folders_by_account: AHashMap> = + AHashMap::with_capacity(3); + + for item in hrefs { + let uri = canonical_dav_uri(&item).unwrap_or(Cow::Borrowed(item.as_str())); + let resource = match server + .validate_uri(access_token, &uri) + .await + .and_then(|r| r.into_owned_uri()) + { + Ok(resource) => resource, + Err(DavError::Code(code)) => { + response.add_response(Response::new_status([item], code)); + continue; + } + Err(err) => { + return Err(err); + } + }; + + let account_id = resource.account_id; + let resources = data + .resources(server, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())?; + + let document_ids = if !access_token.is_member(account_id) { + if let Some(document_ids) = shared_folders_by_account.get(&account_id) { + document_ids.clone().into() + } else { + let document_ids = Arc::new(resources.shared_containers( + access_token, + [if collection_children == collection_container { + Acl::ReadItems + } else { + Acl::Read + }], + true, + )); + shared_folders_by_account.insert(account_id, document_ids.clone()); + document_ids.into() + } + } else { + None + }; + + if let Some(resource) = resource + .resource + .and_then(|name| resources.by_path(name)) + .filter(|resource| { + access_token.is_member(account_id) + || resource + .resource + .event_flags() + .is_none_or(|flags| flags & EVENT_SECRET == 0) + }) + { + if !resource.is_container() { + if document_ids.as_ref().is_none_or(|docs| { + resource + .parent_id() + .is_some_and(|parent_id| docs.contains(parent_id)) + }) { + paths.push(PropFindItem::new( + resources.format_resource(resource), + account_id, + resource, + )); + } else { + response.add_response( + Response::new_status([item], StatusCode::FORBIDDEN) + .with_response_description( + "Not enough permissions to access this shared resource", + ), + ); + } + } else { + response.add_response( + Response::new_status([item], StatusCode::FORBIDDEN) + .with_response_description("Multiget not allowed for collections"), + ); + } + } else { + response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); + } + } + + Ok(paths) +} + +pub(crate) fn requested_href(uri: &str, is_container: bool) -> String { + if is_container && !uri.ends_with('/') { + format!("{uri}/") + } else { + uri.to_string() + } +} + +fn is_discover_only(readable: Option<&RoaringBitmap>, item: &DavResourcePath<'_>) -> bool { + readable.is_some_and(|readable| !readable.contains(item.document_id())) +} + diff --git a/crates/dav/src/common/propfind/item.rs b/crates/dav/src/common/propfind/item.rs new file mode 100644 index 00000000..dcc21b43 --- /dev/null +++ b/crates/dav/src/common/propfind/item.rs @@ -0,0 +1,803 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + PropFindContext, PropFindItem, PropFindState, data::SyncTokenUrn, dead::dead_properties, +}; +use crate::{ + calendar::{ + filter::FilterTimeRanges, + query::{CalendarDataRange, CalendarQueryHandler}, + }, + card::query::serialize_vcard_with_props, + common::{ + ArchivedResource, + acl::{DavAclHandler, Privileges, current_user_privilege_set}, + quoted_etag, + }, + principal::CurrentUserPrincipal, +}; +use calcard::{common::timezone::Tz, icalendar::ICalendarComponentType}; +use common::{Server, storage::dav::DISPLAY_NAME_PROPERTY}; +use dav_proto::schema::{ + Collation, Namespace, + property::{ + CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, Privilege, Rfc1123DateTime, + SupportedCollation, SupportedLock, WebDavProperty, + }, + request::DavPropertyValue, + response::{ + Ace, AclRestrictions, BaseCondition, Href, List, PropStat, Response, SupportedPrivilege, + }, +}; +use groupware::{ + DavCalendarResource, DavResourceName, + calendar::{ + ArchivedTimezone, SupportedComponent, + availability::{Inclusion, PrincipalRole}, + privacy::EventPrivacy, + }, +}; +use hyper::StatusCode; +use std::borrow::Cow; +use store::write::metadata::MetadataBuf; +use trc::AddContext; +use types::{acl::Acl, collection::Collection}; +use utils::map::bitmap::Bitmap; + +pub(super) trait PropFindItemBuilder: Sync + Send { + fn add_propfind_item( + &self, + ctx: &PropFindContext<'_>, + state: &mut PropFindState, + item: PropFindItem, + archive: &ArchivedResource<'_>, + container: Option<&MetadataBuf>, + calendar_filter: Option, + ) -> impl Future> + Send; +} + +impl PropFindItemBuilder for Server { + async fn add_propfind_item( + &self, + ctx: &PropFindContext<'_>, + state: &mut PropFindState, + item: PropFindItem, + archive: &ArchivedResource<'_>, + container: Option<&MetadataBuf>, + mut calendar_filter: Option, + ) -> crate::Result { + let PropFindState { + data, + response, + ical_instances_limit, + } = state; + let PropFindContext { + access_token, + query, + properties, + account_info, + collection_container, + collection_children, + sync_collection, + is_scheduling, + skip_not_found, + is_allprop, + } = *ctx; + let account_id = item.account_id; + let personal_id = access_token.personal_id(account_id, collection_container); + let collection = if item.is_container { + collection_container + } else { + collection_children + }; + + // Fill properties + let event_flags = match archive { + ArchivedResource::CalendarEvent(event, _) => Some(event.inner.flags.to_native()), + ArchivedResource::Cached(item) => item.event_flags(), + _ => None, + }; + let is_private_view = event_flags.is_some_and(|flags| { + !access_token.is_member(account_id) && !EventPrivacy::from_flags(flags).is_public() + }); + let container = container.filter(|_| !is_private_view); + let mut fields = Vec::with_capacity(properties.len()); + let mut fields_not_found = Vec::new(); + let mut fields_too_large = Vec::new(); + for property in properties { + if item.is_discover_only { + match property { + DavProperty::WebDav( + WebDavProperty::ResourceType | WebDavProperty::DisplayName, + ) => {} + DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet) => { + fields.push(DavPropertyValue::new( + property.clone(), + Vec::::new(), + )); + continue; + } + _ => { + if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + continue; + } + } + } + match property { + DavProperty::WebDav(dav_property) => match dav_property { + WebDavProperty::CreationDate => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::Timestamp(archive.created()), + )); + } + WebDavProperty::DisplayName => { + let name = if archive.is_file() { + container + .and_then(|container| { + container.view().dav_property(&DISPLAY_NAME_PROPERTY) + }) + .and_then(|value| value.text()) + } else { + archive + .display_name(personal_id) + .filter(|_| !is_private_view) + .map(Cow::Borrowed) + }; + if let Some(name) = name { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String(name.into_owned()), + )); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::GetContentLanguage => { + if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::GetContentLength => { + let content_length = match archive { + ArchivedResource::CalendarEvent(_, None) if is_private_view => None, + _ => archive.content_length(), + }; + if let Some(value) = content_length { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::Uint64(value as u64), + )); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::GetContentType => { + if let Some(value) = archive.content_type() { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String(value.to_string()), + )); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::GetETag => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String(archive.etag()), + )); + } + WebDavProperty::GetCTag => { + if item.is_container { + let ctag = data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .highest_change_id; + + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String(quoted_etag(ctag)), + )); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::GetLastModified => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::Rfc1123Date(Rfc1123DateTime::new(archive.modified())), + )); + } + WebDavProperty::ResourceType => { + if let Some(resource_type) = archive.resource_type() { + fields.push(DavPropertyValue::new(property.clone(), resource_type)); + } else { + fields.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::LockDiscovery => { + if let Some(locks) = data + .locks(self, account_id, collection_container, &item) + .await + .caused_by(trc::location!())? + { + fields.push(DavPropertyValue::new(property.clone(), locks)); + } else { + fields.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::SupportedLock => { + if !is_scheduling { + fields.push(DavPropertyValue::new( + property.clone(), + SupportedLock::default(), + )); + } else { + fields.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::SupportedReportSet => { + if let Some(report_set) = archive.supported_report_set() { + fields.push(DavPropertyValue::new(property.clone(), report_set)); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::SyncToken => { + let sync_token = data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .sync_token(); + + fields.push(DavPropertyValue::new(property.clone(), sync_token)); + } + WebDavProperty::CurrentUserPrincipal => { + if !query.expand { + fields.push(DavPropertyValue::new( + property.clone(), + vec![account_info.current_user_principal()], + )); + } else { + fields.push(DavPropertyValue::new( + property.clone(), + data.principals + .expand( + self, + access_token, + access_token.account_id(), + &query.propfind, + ) + .await? + .map(|r| DavValue::Response(Box::new(r))) + .unwrap_or(DavValue::Null), + )); + } + } + WebDavProperty::QuotaAvailableBytes => { + let available = if item.is_container { + data.quota(self, account_id) + .await + .caused_by(trc::location!())? + .available + } else { + None + }; + + if let Some(available) = available { + fields.push(DavPropertyValue::new(property.clone(), available)); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::QuotaUsedBytes => { + if item.is_container { + fields.push(DavPropertyValue::new( + property.clone(), + data.quota(self, account_id) + .await + .caused_by(trc::location!())? + .used, + )); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::Owner => { + if !query.expand { + fields.push(DavPropertyValue::new( + property.clone(), + vec![ + data.owner(self, account_info, account_id) + .await + .caused_by(trc::location!())?, + ], + )); + } else { + fields.push(DavPropertyValue::new( + property.clone(), + data.principals + .expand(self, access_token, account_id, &query.propfind) + .await? + .map(|r| DavValue::Response(Box::new(r))) + .unwrap_or(DavValue::Null), + )); + } + } + WebDavProperty::Group => { + fields.push(DavPropertyValue::empty(property.clone())); + } + WebDavProperty::SupportedPrivilegeSet => { + if !is_scheduling { + fields.push(DavPropertyValue::new( + property.clone(), + vec![SupportedPrivilege::all_privileges( + collection_container == Collection::Calendar, + )], + )); + } else { + fields.push(DavPropertyValue::new( + property.clone(), + vec![SupportedPrivilege::all_scheduling_privileges(matches!( + archive, + ArchivedResource::CalendarEventNotification(..) + | ArchivedResource::CalendarEventNotificationCollection( + true + ) + ))], + )); + } + } + WebDavProperty::CurrentUserPrivilegeSet => { + let privileges = if is_scheduling { + Privilege::scheduling( + matches!( + archive, + ArchivedResource::CalendarEventNotification(..) + | ArchivedResource::CalendarEventNotificationCollection( + true + ) + ), + access_token.is_member(account_id), + ) + } else if access_token.is_member(account_id) { + Privilege::all(matches!( + collection, + Collection::Calendar | Collection::CalendarEvent + )) + } else if archive.is_file() { + let mut acl = match data + .accounts + .get(&account_id) + .and_then(|account| account.file_access.as_ref()) + { + Some(access) => access.acl(item.document_id), + None => data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .file_acl(access_token, item.document_id), + }; + if !item.is_container { + acl.remove(Acl::Share); + } + current_user_privilege_set(acl) + } else if let Some(acls) = archive.acls() { + access_token.current_privilege_set( + account_id, + acls, + collection_container == Collection::Calendar, + ) + } else if let Some(parent_id) = item.parent_id { + current_user_privilege_set( + data.resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .container_acl(access_token, parent_id), + ) + } else { + vec![] + }; + + if !privileges.is_empty() { + fields.push(DavPropertyValue::new(property.clone(), privileges)); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::Acl => { + if let Some(acls) = archive.acls() { + let aces = self + .resolve_ace( + access_token, + account_id, + acls, + query + .expand + .then_some((&query.propfind, &mut data.principals)), + ) + .await?; + + fields.push(DavPropertyValue::new(property.clone(), aces)); + } else if archive.is_file() { + fields.push(DavPropertyValue::new(property.clone(), Vec::::new())); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + WebDavProperty::AclRestrictions => { + fields.push(DavPropertyValue::new( + property.clone(), + AclRestrictions::default() + .with_no_invert() + .with_grant_only(), + )); + } + WebDavProperty::InheritedAclSet => { + fields.push(DavPropertyValue::empty(property.clone())); + } + WebDavProperty::PrincipalCollectionSet => { + fields.push(DavPropertyValue::new( + property.clone(), + vec![Href( + DavResourceName::Principal.collection_path().to_string(), + )], + )); + } + }, + DavProperty::Dead(name) => { + if let Some(value) = + container.and_then(|container| container.view().dav_property(name)) + { + fields.push(DavPropertyValue::new(property.clone(), value.to_encoded())); + } else if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + DavProperty::CardDav(card_property) => match (card_property, &archive) { + ( + CardDavProperty::AddressbookDescription, + ArchivedResource::AddressBook(book), + ) => { + if let Some(desc) = + book.inner.preferences(personal_id).description.as_deref() + { + fields.push(DavPropertyValue::new(property.clone(), desc.to_string())); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + (CardDavProperty::SupportedAddressData, ArchivedResource::AddressBook(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::SupportedAddressData, + )); + } + (CardDavProperty::SupportedCollationSet, ArchivedResource::AddressBook(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::Collations(List(vec![ + SupportedCollation { + collation: Collation::AsciiCasemap, + namespace: Namespace::CardDav, + }, + SupportedCollation { + collation: Collation::Octet, + namespace: Namespace::CardDav, + }, + SupportedCollation { + collation: Collation::UnicodeCasemap, + namespace: Namespace::CardDav, + }, + ])), + )); + } + (CardDavProperty::MaxResourceSize, ArchivedResource::AddressBook(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + self.core.groupware.max_vcard_size as u64, + )); + } + ( + CardDavProperty::AddressData { + properties, + version, + }, + ArchivedResource::ContactCard(card, Some(content)), + ) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::CData(serialize_vcard_with_props( + &content.card, + properties, + (*version) + .or(query.vcard_version) + .unwrap_or(self.core.groupware.vcard_version), + u32::from(card.inner.size) as usize, + )), + )); + } + _ => { + if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + }, + DavProperty::CalDav(cal_property) => match (cal_property, &archive) { + (CalDavProperty::CalendarDescription, ArchivedResource::Calendar(calendar)) => { + if let Some(desc) = calendar + .inner + .preferences(personal_id) + .and_then(|preferences| preferences.description.as_deref()) + { + fields.push(DavPropertyValue::new(property.clone(), desc.to_string())); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + (CalDavProperty::CalendarTimezone, ArchivedResource::Calendar(calendar)) => { + if let Some(ArchivedTimezone::Custom(tz)) = calendar + .inner + .preferences(personal_id) + .map(|preferences| &preferences.time_zone) + { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::CData(tz.to_string()), + )); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + (CalDavProperty::TimezoneId, ArchivedResource::Calendar(calendar)) => { + if let Some(ArchivedTimezone::IANA(tz)) = calendar + .inner + .preferences(personal_id) + .map(|preferences| &preferences.time_zone) + { + fields.push(DavPropertyValue::new( + property.clone(), + Tz::from_id(tz.to_native()).unwrap_or(Tz::UTC).to_string(), + )); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + ( + CalDavProperty::SupportedCalendarComponentSet, + ArchivedResource::Calendar(calendar), + ) => { + let supported_components = calendar.inner.supported_components.to_native(); + fields.push(DavPropertyValue::new( + property.clone(), + if supported_components != 0 { + DavValue::Components(List( + Bitmap::::from(supported_components) + .into_iter() + .map(ICalendarComponentType::from) + .map(Comp) + .collect(), + )) + } else { + DavValue::all_calendar_components() + }, + )); + } + (CalDavProperty::SupportedCalendarData, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::SupportedCalendarData, + )); + } + (CalDavProperty::SupportedCollationSet, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::Collations(List(vec![ + SupportedCollation { + collation: Collation::AsciiCasemap, + namespace: Namespace::CalDav, + }, + SupportedCollation { + collation: Collation::Octet, + namespace: Namespace::CalDav, + }, + SupportedCollation { + collation: Collation::UnicodeCasemap, + namespace: Namespace::CalDav, + }, + ])), + )); + } + (CalDavProperty::MaxResourceSize, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + self.core.groupware.max_ical_size as u64, + )); + } + (CalDavProperty::MinDateTime, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String("0001-01-01T00:00:00Z".to_string()), + )); + } + (CalDavProperty::MaxDateTime, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String("9999-12-31T23:59:59Z".to_string()), + )); + } + (CalDavProperty::MaxInstances, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + self.core.groupware.max_ical_instances as u64, + )); + } + (CalDavProperty::MaxAttendeesPerInstance, ArchivedResource::Calendar(_)) => { + fields.push(DavPropertyValue::new( + property.clone(), + self.core.groupware.max_ical_attendees_per_instance as u64, + )); + } + ( + CalDavProperty::CalendarData(calendar_data), + ArchivedResource::CalendarEvent(event, Some(content)), + ) => { + let data_range = calendar_data.calendar_data_range(); + if calendar_filter.is_none() + && (data_range.is_some() || !calendar_data.properties.is_empty()) + { + let default_tz = match (data_range, item.parent_id) { + (Some(_), Some(calendar_id)) => data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .calendar_default_tz(calendar_id, account_id) + .unwrap_or(Tz::UTC), + _ => Tz::UTC, + }; + calendar_filter = Some(CalendarQueryHandler::for_content( + event.inner, + content, + FilterTimeRanges { + range: data_range, + has_alarms: false, + }, + default_tz, + )); + } + if let Some(handler) = &calendar_filter { + if let Some(ical) = handler.serialize_content( + content, + event.inner.size.to_native(), + calendar_data, + ical_instances_limit, + ) { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::CData(ical), + )); + } else if query.sync_type.is_none() { + return Ok(false); + } else { + fields_too_large.push(DavPropertyValue::empty(property.clone())); + } + } else { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::CData(content.to_ical_string()), + )); + } + } + ( + CalDavProperty::CalendarData(_), + ArchivedResource::CalendarEventNotification(_, Some(content)), + ) => { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::CData( + content + .calendar_data() + .map(|ical| ical.to_string()) + .unwrap_or_default(), + ), + )); + } + (CalDavProperty::ScheduleTag, ArchivedResource::CalendarEvent(event, _)) + if event.inner.schedule_tag.is_some() => + { + fields.push(DavPropertyValue::new( + property.clone(), + DavValue::String(quoted_etag( + event + .inner + .schedule_tag + .as_ref() + .map_or(0, |tag| tag.to_native().into()), + )), + )); + } + ( + CalDavProperty::ScheduleCalendarTransp, + ArchivedResource::Calendar(calendar), + ) => { + let flags = calendar.inner.personal_flags(account_id, true); + fields.push(DavPropertyValue::new( + property.clone(), + if Inclusion::from_flags(flags, PrincipalRole::Owner).is_some() { + DavValue::Opaque + } else { + DavValue::Transparent + }, + )); + } + ( + CalDavProperty::ScheduleDefaultCalendarURL, + ArchivedResource::CalendarEventNotificationCollection(true), + ) => { + if let Some(default_cal) = &self.core.groupware.default_calendar_name { + fields.push(DavPropertyValue::new( + property.clone(), + vec![Href(format!( + "{}/{}/{default_cal}/", + DavResourceName::Cal.base_path(), + item.name.split('/').nth(3).unwrap_or_default() + ))], + )); + } else { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + + _ => { + if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + }, + + property => { + if !skip_not_found { + fields_not_found.push(DavPropertyValue::empty(property.clone())); + } + } + } + } + + if is_allprop + && !item.is_discover_only + && let Some(container) = container + { + fields.extend(dead_properties(container, collection)); + } + + // Add response + let mut prop_stat = Vec::with_capacity(2); + if !fields.is_empty() { + prop_stat.push(PropStat::new_list(fields)); + } + if !fields_not_found.is_empty() && !query.is_minimal() { + prop_stat.push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); + } + if !fields_too_large.is_empty() { + prop_stat.push( + PropStat::new_list(fields_too_large) + .with_status(StatusCode::INSUFFICIENT_STORAGE) + .with_error(BaseCondition::NumberOfMatchesWithinLimit), + ); + } + if prop_stat.is_empty() { + prop_stat.push(PropStat::new_list(vec![])); + } + response.add_response(Response::new_propstat(item.name, prop_stat)); + + Ok(true) + } +} diff --git a/crates/dav/src/common/propfind/load.rs b/crates/dav/src/common/propfind/load.rs new file mode 100644 index 00000000..93aed186 --- /dev/null +++ b/crates/dav/src/common/propfind/load.rs @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{PropFindContext, PropFindItem}; +use crate::common::{ArchivedResource, CacheFit}; +use common::{Server, auth::AccessToken}; +use dav_proto::schema::property::{CalDavProperty, CardDavProperty, DavProperty, WebDavProperty}; +use groupware::calendar::{ + EVENT_HAS_ALARMS, EVENT_PRIVATE, EVENT_SECRET, EVENT_USES_DEFAULT_ALERTS, +}; +use store::{ + ahash::AHashMap, + dispatch::ReadShape, + roaring::RoaringBitmap, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{ + collection::Collection, + field::{CalendarEventField, CalendarNotificationField, ContactField, Field}, +}; + + +pub(super) const PROPFIND_BATCH_SIZE: usize = 256; + +type ArchiveKey = (u32, u8, u32); + +pub(super) struct ArchiveLoader { + collection_container: Collection, + collection_children: Collection, + is_scheduling: bool, + needs_content: bool, + needs_content_length: bool, + needs_media_type: bool, + content_field: Option, + cached_items: bool, +} + +pub(super) struct BatchArchives { + metadata: AHashMap>, + contents: AHashMap>, +} + +struct ContentDemand { + is_owner: bool, +} + +struct ArchiveGroup { + metadata: RoaringBitmap, + contents: RoaringBitmap, + demand: ContentDemand, +} + +impl ArchiveLoader { + pub fn new(ctx: &PropFindContext<'_>, has_filter: bool) -> Self { + let properties = ctx.properties; + ArchiveLoader { + cached_items: !has_filter + && !ctx.is_scheduling + && matches!( + ctx.collection_children, + Collection::CalendarEvent | Collection::ContactCard | Collection::FileNode + ) + && properties + .iter() + .all(|property| is_cache_answerable(ctx.collection_children, property)), + collection_container: ctx.collection_container, + collection_children: ctx.collection_children, + is_scheduling: ctx.is_scheduling, + needs_content: has_filter + || properties.iter().any(|property| { + matches!( + property, + DavProperty::CardDav(CardDavProperty::AddressData { .. }) + | DavProperty::CalDav(CalDavProperty::CalendarData(_)) + ) + }), + needs_content_length: ctx.collection_children == Collection::CalendarEvent + && properties.iter().any(|property| { + matches!( + property, + DavProperty::WebDav(WebDavProperty::GetContentLength) + ) + }), + needs_media_type: properties.iter().any(|property| { + matches!( + property, + DavProperty::WebDav(WebDavProperty::GetContentType) + ) + }), + content_field: match ctx.collection_children { + Collection::CalendarEvent => Some(CalendarEventField::Content.field()), + Collection::CalendarEventNotification => { + Some(CalendarNotificationField::Content.field()) + } + Collection::ContactCard => Some(ContactField::Content.field()), + _ => None, + }, + } + } + + pub fn needs_event_view(&self) -> bool { + self.collection_children == Collection::CalendarEvent + && (self.needs_content || self.needs_content_length) + } + + pub fn is_cached(&self, item: &PropFindItem) -> bool { + self.cached_items + && !item.is_container + && match item.cache_fit { + CacheFit::Exact => true, + CacheFit::InexactMediaType => !self.needs_media_type, + CacheFit::Inexact => false, + } + } + + pub fn collection_of(&self, item: &PropFindItem) -> Collection { + if item.is_container { + self.collection_container + } else { + self.collection_children + } + } + + pub async fn load( + &self, + server: &Server, + access_token: &AccessToken, + batch: &[PropFindItem], + ) -> trc::Result { + let content_field = self + .content_field + .filter(|_| self.needs_content || self.needs_content_length); + let mut groups: AHashMap<(u32, Collection), ArchiveGroup> = AHashMap::with_capacity(4); + for item in batch { + if (self.is_scheduling && item.is_container) || self.is_cached(item) { + continue; + } + let collection = self.collection_of(item); + let group = groups + .entry((item.account_id, collection)) + .or_insert_with(|| ArchiveGroup { + metadata: RoaringBitmap::new(), + contents: RoaringBitmap::new(), + demand: ContentDemand { + is_owner: access_token.is_member(item.account_id), + }, + }); + group.metadata.insert(item.document_id); + if content_field.is_some() + && collection == self.collection_children + && (self.needs_content || group.demand.is_met_by(item.event_flags)) + { + group.contents.insert(item.document_id); + } + } + + let mut metadata = AHashMap::with_capacity(batch.len()); + let mut contents = AHashMap::new(); + for (&(account_id, collection), group) in &groups { + let collection_id = u8::from(collection); + match content_field.filter(|_| !group.contents.is_empty()) { + Some(content_field) => { + server + .archives_fields( + account_id, + collection, + ReadShape::Ranges(server.core.storage.data.scan_gap()), + &[ + (content_field, &group.contents), + (Field::ARCHIVE, &group.metadata), + ], + |field, document_id, archive| { + if field == Field::ARCHIVE { + metadata + .insert((account_id, collection_id, document_id), archive); + } else { + contents + .insert((account_id, collection_id, document_id), archive); + } + Ok(true) + }, + ) + .await + } + None => { + server + .archives( + account_id, + collection, + Field::ARCHIVE, + &group.metadata, + |document_id, archive| { + metadata.insert((account_id, collection_id, document_id), archive); + Ok(true) + }, + ) + .await + } + } + .caused_by(trc::location!())?; + } + + Ok(BatchArchives { metadata, contents }) + } +} + +fn is_cache_answerable(collection: Collection, property: &DavProperty) -> bool { + match property { + DavProperty::WebDav(WebDavProperty::DisplayName | WebDavProperty::GetContentLength) => { + collection == Collection::FileNode + } + DavProperty::WebDav(WebDavProperty::Acl) => collection != Collection::FileNode, + DavProperty::WebDav(_) | DavProperty::Dead(_) => true, + _ => false, + } +} + +impl BatchArchives { + pub fn resource( + &self, + collection: Collection, + item: &PropFindItem, + ) -> trc::Result>> { + let key = (item.account_id, collection.into(), item.document_id); + self.metadata + .get(&key) + .map(|archive| { + ArchivedResource::from_archive(archive, self.contents.get(&key), collection) + }) + .transpose() + } +} + +impl ContentDemand { + fn view_flags(&self) -> u16 { + if self.is_owner { + EVENT_USES_DEFAULT_ALERTS + } else { + EVENT_HAS_ALARMS | EVENT_PRIVATE | EVENT_SECRET + } + } + + fn is_met_by(&self, event_flags: u16) -> bool { + event_flags & self.view_flags() != 0 + } +} diff --git a/crates/dav/src/common/propfind/mod.rs b/crates/dav/src/common/propfind/mod.rs new file mode 100644 index 00000000..a9f0f39f --- /dev/null +++ b/crates/dav/src/common/propfind/mod.rs @@ -0,0 +1,857 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod data; +mod dead; +mod get; +mod item; +mod load; + +pub(crate) use data::{ExpandedPrincipals, PropFindAccountQuota, PropFindData, SyncTokenUrn}; +pub(crate) use get::requested_href; + +use super::{ + ArchivedResource, CacheFit, CachedItem, DavCollection, DavQuery, DavQueryFilter, + DavQueryResource, + uri::{DavUriResource, SyncCursor, UriResource}, +}; +use crate::{ + DavError, DavErrorCondition, + calendar::{ + CALENDAR_CONTAINER_PROPS, CALENDAR_ITEM_PROPS, CalendarEventView, + query::CalendarQueryHandler, + }, + card::{CARD_CONTAINER_PROPS, CARD_ITEM_PROPS, filter::CardFilterMatch}, + file::{FILE_CONTAINER_PROPS, FILE_ITEM_PROPS}, + principal::{ + CurrentUserPrincipal, + propfind::{PrincipalPropFind, build_home_set}, + }, +}; +use calcard::common::timezone::Tz; +use common::{ + DavResourcePath, Server, + auth::{AccessToken, AccountCache}, +}; +use dav_proto::{ + Depth, RequestHeaders, + schema::{ + property::{DavProperty, PrincipalProperty, ReportSet, ResourceType, WebDavProperty}, + request::{DavPropertyValue, PropFind}, + response::{BaseCondition, MultiStatus, PropStat, Response}, + }, +}; +use dead::{DeadDemand, DeadPresence, dead_property_names}; +use get::{get, multiget}; +use groupware::{ + DavCalendarResource, DavResourceName, + calendar::{SCHEDULE_INBOX_ID, alerts::DefaultAlertsResolver}, +}; +use http_proto::HttpResponse; +use hyper::StatusCode; +use item::PropFindItemBuilder; +use load::{ArchiveLoader, PROPFIND_BATCH_SIZE}; +use registry::schema::{enums::Permission, prelude::ObjectType}; +use std::borrow::Cow; +use store::{registry::RegistryQuery, roaring::RoaringBitmap, write::metadata::MetadataBuf}; +use trc::AddContext; +use types::collection::{Collection, SyncCollection}; + +pub(crate) trait PropFindRequestHandler: Sync + Send { + fn handle_propfind_request( + &self, + access_token: &AccessToken, + headers: &RequestHeaders<'_>, + request: PropFind, + ) -> impl Future> + Send; + + fn handle_dav_query( + &self, + access_token: &AccessToken, + query: DavQuery<'_>, + ) -> impl Future> + Send; + + fn dav_quota( + &self, + account_id: u32, + ) -> impl Future> + Send; +} + +#[derive(Debug)] +pub(crate) struct PropFindItem { + pub name: String, + pub account_id: u32, + pub document_id: u32, + pub parent_id: Option, + pub is_container: bool, + pub is_discover_only: bool, + pub cache_fit: CacheFit, + pub event_flags: u16, + pub dead: DeadPresence, +} + +pub(crate) struct PropFindState { + pub data: PropFindData, + pub response: MultiStatus, + pub ical_instances_limit: usize, +} + +#[derive(Clone, Copy)] +pub(crate) struct PropFindContext<'x> { + pub access_token: &'x AccessToken, + pub query: &'x DavQuery<'x>, + pub properties: &'x [DavProperty], + pub account_info: &'x AccountCache, + pub collection_container: Collection, + pub collection_children: Collection, + pub sync_collection: SyncCollection, + pub is_scheduling: bool, + pub skip_not_found: bool, + pub is_allprop: bool, +} + +impl PropFindRequestHandler for Server { + async fn handle_propfind_request( + &self, + access_token: &AccessToken, + headers: &RequestHeaders<'_>, + request: PropFind, + ) -> crate::Result { + // Validate URI + let resource = self.validate_uri(access_token, headers.uri).await?; + + // Reject Infinity depth for certain queries + let return_children = match headers.depth { + Depth::One | Depth::None => true, + Depth::Zero => false, + Depth::Infinity => match resource.collection { + Collection::Principal => true, + Collection::Calendar | Collection::AddressBook + if resource.account_id.is_some() && resource.resource.is_some() => + { + true + } + Collection::CalendarEventNotification if resource.account_id.is_some() => true, + _ => { + return Err(DavErrorCondition::new( + StatusCode::FORBIDDEN, + BaseCondition::PropFindFiniteDepth, + ) + .into()); + } + }, + }; + + // List shared resources + if let Some(account_id) = resource.account_id { + match resource.collection { + Collection::FileNode + | Collection::Calendar + | Collection::AddressBook + | Collection::CalendarEventNotification => { + // Validate permissions + access_token.enforce_permission(match resource.collection { + Collection::FileNode => Permission::DavFilePropFind, + Collection::Calendar + | Collection::CalendarEvent + | Collection::CalendarEventNotification => Permission::DavCalPropFind, + Collection::AddressBook | Collection::ContactCard => { + Permission::DavCardPropFind + } + _ => unreachable!(), + })?; + + self.handle_dav_query( + access_token, + DavQuery::propfind( + UriResource::new_owned( + resource.collection, + account_id, + resource.resource, + ), + request, + headers, + ), + ) + .await + } + Collection::Principal => { + if !access_token.is_member(account_id) + && !access_token.has_permission(Permission::SysAccountQuery) + && self + .try_account(account_id) + .await + .caused_by(trc::location!())? + .is_none_or(|account| !account.is_same_tenant(access_token)) + { + return Err(DavError::Code(StatusCode::NOT_FOUND)); + } + + let mut response = MultiStatus::new(Vec::with_capacity(16)); + + if resource.resource.is_some() { + response.add_response(Response::new_status( + [headers.uri.to_string()], + StatusCode::NOT_FOUND, + )); + } else if access_token.has_account_access(account_id) + || (self.core.groupware.allow_directory_query + && access_token.has_permission(Permission::DavPrincipalList)) + || access_token.has_permission(Permission::SysAccountQuery) + { + self.prepare_principal_propfind_response( + access_token, + Collection::Principal, + [account_id].into_iter(), + &request, + &mut response, + ) + .await?; + } else { + response.add_response(Response::new_status( + [headers.uri.to_string()], + StatusCode::FORBIDDEN, + )); + } + + Ok(HttpResponse::new(StatusCode::MULTI_STATUS) + .with_xml_body(response.to_string())) + } + _ => unreachable!(), + } + } else { + let mut response = MultiStatus::new(Vec::with_capacity(16)); + + // Add container info + if !headers.depth_no_root { + add_base_collection_response( + self, + &request, + resource.collection, + access_token, + &mut response, + ) + .await?; + } + + if return_children { + let ids = if !matches!(resource.collection, Collection::Principal) { + // Validate permissions + access_token.enforce_permission(match resource.collection { + Collection::FileNode => Permission::DavFilePropFind, + Collection::Calendar + | Collection::CalendarEvent + | Collection::CalendarEventNotification => Permission::DavCalPropFind, + Collection::AddressBook | Collection::ContactCard => { + Permission::DavCardPropFind + } + _ => unreachable!(), + })?; + RoaringBitmap::from_iter( + access_token.all_ids_by_collection(resource.collection), + ) + } else if (self.core.groupware.allow_directory_query + && access_token.has_permission(Permission::DavPrincipalList)) + || access_token.has_permission(Permission::SysAccountQuery) + { + // Return all principals + let query = RegistryQuery::new(ObjectType::Account); + self.registry() + .query::(query) + .await + .caused_by(trc::location!())? + } else { + RoaringBitmap::from_iter(access_token.all_ids()) + }; + + self.prepare_principal_propfind_response( + access_token, + resource.collection, + ids.into_iter(), + &request, + &mut response, + ) + .await?; + } + + Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) + } + } + + async fn handle_dav_query( + &self, + access_token: &AccessToken, + mut query: DavQuery<'_>, + ) -> crate::Result { + let mut response = MultiStatus::new(Vec::with_capacity(16)); + let mut data = PropFindData::new(); + let collection_container; + let collection_children; + let sync_collection; + let mut query_filter = None; + let limit = std::cmp::min( + query.limit.unwrap_or(u32::MAX) as usize, + self.core.groupware.max_results, + ) + .max(1); + let mut is_sync_limited = false; + let mut is_propfind = false; + + let mut paths = match std::mem::take(&mut query.resource) { + DavQueryResource::Uri(resource) => { + collection_container = resource.collection; + collection_children = collection_container.child_collection().unwrap(); + sync_collection = SyncCollection::from(collection_container); + is_propfind = true; + + get( + self, + access_token, + collection_container, + collection_children, + sync_collection, + &query, + &mut data, + &mut response, + resource, + limit, + &mut is_sync_limited, + ) + .await? + } + DavQueryResource::Multiget { + hrefs, + parent_collection, + } => { + collection_container = parent_collection; + collection_children = collection_container.child_collection().unwrap(); + sync_collection = SyncCollection::from(collection_container); + + multiget( + self, + access_token, + collection_container, + collection_children, + sync_collection, + &mut data, + &mut response, + hrefs, + ) + .await? + } + DavQueryResource::Query { + filter, + parent_collection, + items, + } => { + query_filter = Some(filter); + collection_container = parent_collection; + collection_children = collection_container.child_collection().unwrap(); + sync_collection = SyncCollection::from(collection_container); + + items + } + DavQueryResource::None => unreachable!(), + }; + response.set_namespace(collection_container.namespace()); + + let mut skip_not_found = query.expand; + let mut is_allprop = false; + let (properties, property_names) = match &query.propfind { + PropFind::PropName => { + let (container_props, children_props) = match collection_container { + Collection::FileNode => { + (FILE_CONTAINER_PROPS.as_slice(), FILE_ITEM_PROPS.as_slice()) + } + Collection::Calendar | Collection::CalendarEventNotification => ( + CALENDAR_CONTAINER_PROPS.as_slice(), + CALENDAR_ITEM_PROPS.as_slice(), + ), + Collection::AddressBook => { + (CARD_CONTAINER_PROPS.as_slice(), CARD_ITEM_PROPS.as_slice()) + } + _ => unreachable!(), + }; + + for property in container_props.iter().chain(children_props) { + response.set_namespace(property.namespace()); + } + + if query_filter.is_none() { + let demand = DeadDemand::new(&[], false, true, collection_container); + let mut paths = paths.into_iter(); + loop { + let len = paths.len().min(PROPFIND_BATCH_SIZE); + if len == 0 { + break; + } + let containers = demand + .load( + self, + access_token, + paths.as_slice().get(..len).unwrap_or_default(), + |item| { + collection_of(item, collection_container, collection_children) + }, + ) + .await + .caused_by(trc::location!())?; + for item in paths.by_ref().take(len) { + let collection = + collection_of(&item, collection_container, collection_children); + let container = + containers.get(item.account_id, collection, item.document_id); + response.add_response(item.into_property_names( + container_props, + children_props, + container, + collection, + )); + } + } + + return Ok(HttpResponse::new(StatusCode::MULTI_STATUS) + .with_xml_body(response.to_string())); + } + + (Vec::new(), Some((container_props, children_props))) + } + PropFind::AllProp(items) => { + skip_not_found = true; + is_allprop = true; + let mut result = Vec::with_capacity(items.len() + DavProperty::ALL_PROPS.len()); + result.extend(DavProperty::ALL_PROPS); + result.extend(items.iter().filter(|field| !field.is_all_prop()).cloned()); + (result, None) + } + PropFind::Prop(items) => (items.clone(), None), + }; + + for property in &properties { + response.set_namespace(property.namespace()); + } + + let account_info = self + .account(access_token.account_id()) + .await + .caused_by(trc::location!())?; + let ctx = PropFindContext { + access_token, + query: &query, + properties: &properties, + account_info: &account_info, + collection_container, + collection_children, + sync_collection, + is_scheduling: collection_container == Collection::CalendarEventNotification, + skip_not_found, + is_allprop, + }; + let loader = ArchiveLoader::new(&ctx, query_filter.is_some()); + let dead_demand = DeadDemand::new( + &properties, + is_allprop, + property_names.is_some(), + collection_container, + ); + let needs_event_view = loader.needs_event_view(); + + let mut is_truncated = + query_filter.is_none() && query.sync_type.is_none() && paths.len() > limit; + if is_truncated { + paths.truncate(limit); + } + if paths.len() > PROPFIND_BATCH_SIZE { + paths.sort_unstable_by_key(PropFindItem::storage_order); + } + + let mut remaining = if query.sync_type.is_none() { + limit + } else { + usize::MAX + }; + let mut state = PropFindState { + data, + response, + ical_instances_limit: self.core.groupware.max_ical_instances, + }; + let mut instances_exceeded = false; + let mut default_alerts = DefaultAlertsResolver::default(); + let mut paths = paths.into_iter(); + + 'outer: loop { + let len = paths.len().min(PROPFIND_BATCH_SIZE); + if len == 0 { + break; + } + let batch = paths.as_slice().get(..len).unwrap_or_default(); + let archives = loader + .load(self, access_token, batch) + .await + .caused_by(trc::location!())?; + let containers = dead_demand + .load(self, access_token, batch, |item| loader.collection_of(item)) + .await + .caused_by(trc::location!())?; + + for item in paths.by_ref().take(len) { + let account_id = item.account_id; + let collection = loader.collection_of(&item); + + // Unarchive resource + let cached_resources; + let archive = if ctx.is_scheduling && item.is_container { + Some(ArchivedResource::CalendarEventNotificationCollection( + item.document_id == SCHEDULE_INBOX_ID, + )) + } else if loader.is_cached(&item) { + cached_resources = state + .data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())?; + cached_resources + .item_by_id(item.document_id) + .and_then(|resource| CachedItem::new(collection, resource)) + .map(ArchivedResource::Cached) + } else { + archives + .resource(collection, &item) + .caused_by(trc::location!())? + }; + let Some(mut archive) = archive else { + state + .response + .add_response(Response::new_status([item.name], StatusCode::NOT_FOUND)); + continue; + }; + if needs_event_view + && let ArchivedResource::CalendarEvent(event, Some(content)) = &mut archive + && let Some((view, merged_overrides)) = self + .calendar_event_view( + access_token, + account_id, + event.inner, + content.stored(), + &mut default_alerts, + ) + .await? + { + content.attach_view(view, merged_overrides); + } + + // Filter + let mut calendar_filter = None; + if let Some(query_filter) = &query_filter { + match (query_filter, &archive) { + ( + DavQueryFilter::Addressbook(filter), + ArchivedResource::ContactCard(_, Some(content)), + ) => { + if !filter.matches(&content.card) { + continue; + } + } + ( + DavQueryFilter::Calendar(filter), + ArchivedResource::CalendarEvent(event, Some(content)), + ) => { + let default_tz = match (filter.timezone, item.parent_id) { + (Some(tz), _) => tz, + (None, Some(calendar_id)) => state + .data + .resources(self, access_token, account_id, sync_collection, 0) + .await + .caused_by(trc::location!())? + .calendar_default_tz(calendar_id, account_id) + .unwrap_or(Tz::UTC), + (None, None) => Tz::UTC, + }; + let query_handler = CalendarQueryHandler::for_content( + event.inner, + content, + filter.expansion, + default_tz, + ); + if filter.filter.as_ref().is_some_and(|filter| { + !query_handler.matches_content(content, filter) + }) { + continue; + } + calendar_filter = Some(query_handler); + } + ( + DavQueryFilter::Calendar(filter), + ArchivedResource::CalendarEventNotification(_, Some(content)), + ) => { + if !filter + .matches_scheduling_message( + content, + filter.timezone.unwrap_or(Tz::UTC), + self.core.groupware.max_ical_instances, + ) + .caused_by(trc::location!())? + { + continue; + } + } + _ => continue, + } + } + + if remaining == 0 { + is_truncated = true; + break 'outer; + } + remaining -= 1; + + let container = containers + .get(account_id, collection, item.document_id) + .filter(|_| !item.dead.is_hidden_from(access_token, account_id)); + if let Some((container_props, children_props)) = property_names { + state.response.add_response(item.into_property_names( + container_props, + children_props, + container, + collection, + )); + } else if !self + .add_propfind_item(&ctx, &mut state, item, &archive, container, calendar_filter) + .await? + { + instances_exceeded = true; + break 'outer; + } + } + } + + let mut response = state.response; + if is_truncated || instances_exceeded || is_sync_limited { + response.add_response( + Response::new_status([query.uri], StatusCode::INSUFFICIENT_STORAGE) + .with_error(BaseCondition::NumberOfMatchesWithinLimit) + .with_response_description(if instances_exceeded { + format!( + "The number of recurrence instances exceeds the limit of {}", + self.core.groupware.max_ical_instances + ) + } else { + format!("The number of matches exceeds the limit of {limit}") + }), + ); + } + + if !is_propfind || !response.response.0.is_empty() || !query.sync_type.is_none() { + Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) + } else { + Ok(HttpResponse::new(StatusCode::NOT_FOUND)) + } + } + + async fn dav_quota(&self, account_id: u32) -> trc::Result { + let account = self.account(account_id).await.caused_by(trc::location!())?; + let used = self + .get_used_quota_account(account_id) + .await + .caused_by(trc::location!())? + .max(0) as u64; + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut available = + (account.quota_disk > 0).then(|| account.quota_disk.saturating_sub(used)); + + + Ok(PropFindAccountQuota { used, available }) + } +} + +impl PropFindItem { + pub fn new(name: String, account_id: u32, resource: DavResourcePath<'_>) -> Self { + Self { + name, + account_id, + document_id: resource.document_id(), + parent_id: resource.parent_id(), + is_container: resource.is_container(), + is_discover_only: false, + cache_fit: CachedItem::fit(&resource.resource), + event_flags: resource.resource.event_flags().unwrap_or_default(), + dead: DeadPresence::from_resource(&resource), + } + } + + pub fn with_discover_only(mut self, is_discover_only: bool) -> Self { + self.is_discover_only = is_discover_only; + self + } + + fn storage_order(&self) -> (u32, bool, u32) { + (self.account_id, !self.is_container, self.document_id) + } + + pub(super) fn sync_cursor(&self) -> SyncCursor { + SyncCursor::new( + self.account_id, + self.is_container, + self.document_id, + self.parent_id, + ) + } + + fn into_property_names( + self, + container_props: &[DavProperty], + children_props: &[DavProperty], + container: Option<&MetadataBuf>, + collection: Collection, + ) -> Response { + let props = if self.is_container { + container_props + } else { + children_props + }; + let mut names = Vec::with_capacity(props.len()); + names.extend(props.iter().cloned().map(DavPropertyValue::empty)); + if let Some(container) = container.filter(|_| !self.is_discover_only) { + names.extend(dead_property_names(container, collection)); + } + + Response::new_propstat(self.name, vec![PropStat::new_list(names)]) + } +} + +fn collection_of( + item: &PropFindItem, + collection_container: Collection, + collection_children: Collection, +) -> Collection { + if item.is_container { + collection_container + } else { + collection_children + } +} + +async fn add_base_collection_response( + server: &Server, + request: &PropFind, + collection: Collection, + access_token: &AccessToken, + response: &mut MultiStatus, +) -> trc::Result<()> { + let properties = match request { + PropFind::PropName => { + response.add_response(Response::new_propstat( + DavResourceName::from(collection).collection_path(), + vec![PropStat::new_list(vec![ + DavPropertyValue::empty(DavProperty::WebDav(WebDavProperty::ResourceType)), + DavPropertyValue::empty(DavProperty::WebDav( + WebDavProperty::CurrentUserPrincipal, + )), + DavPropertyValue::empty(DavProperty::WebDav( + WebDavProperty::SupportedReportSet, + )), + ])], + )); + return Ok(()); + } + PropFind::AllProp(include) => { + let mut props = Vec::with_capacity(include.len() + 2); + props.extend([ + DavProperty::WebDav(WebDavProperty::ResourceType), + DavProperty::WebDav(WebDavProperty::SupportedReportSet), + ]); + for prop in include { + if !props.contains(prop) { + props.push(prop.clone()); + } + } + Cow::Owned(props) + } + PropFind::Prop(items) => Cow::Borrowed(items.as_slice()), + }; + + let mut fields = Vec::with_capacity(properties.len()); + let mut fields_not_found = Vec::new(); + let account_info = server + .account(access_token.account_id()) + .await + .caused_by(trc::location!())?; + + for prop in properties.iter() { + response.set_namespace(prop.namespace()); + match prop { + DavProperty::WebDav(WebDavProperty::ResourceType) => { + fields.push(DavPropertyValue::new( + prop.clone(), + vec![ResourceType::Collection], + )); + } + DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal) => { + fields.push(DavPropertyValue::new( + prop.clone(), + vec![account_info.current_user_principal()], + )); + } + DavProperty::Principal(PrincipalProperty::CalendarHomeSet) => { + let hrefs = build_home_set( + server, + access_token, + account_info.name(), + access_token.account_id(), + true, + ) + .await + .caused_by(trc::location!())?; + + fields.push(DavPropertyValue::new(prop.clone(), hrefs)); + } + DavProperty::Principal(PrincipalProperty::AddressbookHomeSet) => { + let hrefs = build_home_set( + server, + access_token, + account_info.name(), + access_token.account_id(), + false, + ) + .await + .caused_by(trc::location!())?; + + fields.push(DavPropertyValue::new(prop.clone(), hrefs)); + } + DavProperty::WebDav(WebDavProperty::SupportedReportSet) => { + let reports = match collection { + Collection::Principal => ReportSet::principal(), + Collection::Calendar | Collection::CalendarEvent => ReportSet::calendar(), + Collection::AddressBook | Collection::ContactCard => ReportSet::addressbook(), + _ => ReportSet::file(), + }; + + fields.push(DavPropertyValue::new(prop.clone(), reports)); + response.set_namespace(collection.namespace()); + } + _ => { + fields_not_found.push(DavPropertyValue::empty(prop.clone())); + } + } + } + + let mut prop_stat = Vec::with_capacity(2); + + if !fields.is_empty() { + prop_stat.push(PropStat::new_list(fields)); + } + + if !fields_not_found.is_empty() { + prop_stat.push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); + } + + response.add_response(Response::new_propstat( + DavResourceName::from(collection).collection_path(), + prop_stat, + )); + + Ok(()) +} diff --git a/crates/dav/src/common/search.rs b/crates/dav/src/common/search.rs new file mode 100644 index 00000000..cf307fee --- /dev/null +++ b/crates/dav/src/common/search.rs @@ -0,0 +1,353 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::propfind::PropFindItem; +use common::{DavResourcePath, GroupwareResourceRef, GroupwareResources, Server}; +use dav_proto::schema::{MatchType, request::TextMatch}; +use groupware::strip_mailto_scheme; +use nlp::language::Language; +use store::{ + ahash::AHashSet, + roaring::RoaringBitmap, + search::{SearchField, SearchFilter, SearchQuery, TextMatch as IndexMatch, tokenize::tokenize}, + write::SearchIndex, +}; +use utils::cheeky_hash::CheekyHash; + +pub(crate) enum Candidates { + All, + Set(RoaringBitmap), + Text(SearchFilter), + And(Vec), + Or(Vec), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum IndexedText { + Plain, + Identifier, + Stemmed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SearchBackend { + Internal, + External, +} + +#[derive(Clone, Copy)] +pub(crate) struct TextIndex<'x> { + fields: Option<&'x AHashSet>, + default_language: Language, + backend: SearchBackend, + splits_identifiers: bool, +} + +pub(crate) struct QueryScope<'x> { + items: Vec>, + document_ids: RoaringBitmap, +} + +impl<'x> TextIndex<'x> { + pub fn new(server: &'x Server, index: SearchIndex) -> Self { + let search_store = server.search_store(); + TextIndex { + fields: if search_store.is_meilisearch() { + None + } else { + server.core.email.index_fields.get(&index) + }, + default_language: server.core.email.default_language, + backend: if search_store.internal_fts().is_some() { + SearchBackend::Internal + } else { + SearchBackend::External + }, + splits_identifiers: search_store.splits_identifiers(), + } + } + + pub fn candidates( + &self, + field: SearchField, + text: IndexedText, + text_match: &TextMatch, + ) -> Candidates { + if text_match.negate + || !self.contains(&field) + || (text == IndexedText::Identifier && !self.splits_identifiers) + { + return Candidates::All; + } + let (value, has_scheme) = if text == IndexedText::Identifier { + let address = strip_mailto_scheme(&text_match.value); + (address, address.len() != text_match.value.len()) + } else { + (text_match.value.as_str(), false) + }; + let language = if text == IndexedText::Stemmed { + Language::detect(value.to_string(), self.default_language).1 + } else { + Language::None + }; + let starts_at_word = has_scheme + || matches!( + text_match.match_type, + MatchType::Equals | MatchType::StartsWith + ) + || value.starts_with(|ch: char| !ch.is_alphanumeric()); + let ends_at_word = matches!( + text_match.match_type, + MatchType::Equals | MatchType::EndsWith + ) || value.ends_with(|ch: char| !ch.is_alphanumeric()); + + let mut words = Vec::with_capacity(4); + tokenize(value, Language::None, |token| { + words.push(token.word); + true + }); + + let term = |word: &str, op: IndexMatch| { + Candidates::Text(SearchFilter::Text { + field: field.clone(), + op, + value: word.to_string(), + language, + }) + }; + let last = words.len().saturating_sub(1); + let is_single = words.len() == 1; + + Candidates::and(words.iter().enumerate().filter_map(|(position, word)| { + let partial_start = position == 0 && !starts_at_word; + let partial_end = position == last && !ends_at_word; + match (partial_start, partial_end) { + (true, false) if !is_single => None, + (_, false) => Some(term(word, IndexMatch::Standard)), + (_, true) => Some(Candidates::or([ + term(word, IndexMatch::Standard), + term(self.prefix_of(word), IndexMatch::Prefix), + ])), + } + })) + } + + fn prefix_of<'w>(&self, word: &'w str) -> &'w str { + word.get(..word.floor_char_boundary(self.backend.max_prefix_len())) + .unwrap_or(word) + } + + fn contains(&self, field: &SearchField) -> bool { + self.fields + .is_some_and(|fields| fields.is_empty() || fields.contains(field)) + } +} + +impl SearchBackend { + fn max_prefix_len(self) -> usize { + match self { + SearchBackend::Internal => CheekyHash::HASH_SIZE, + SearchBackend::External => usize::MAX, + } + } +} + +impl<'x> QueryScope<'x> { + pub fn new(items: impl Iterator>) -> Self { + let items = items.collect::>(); + let document_ids = items.iter().map(DavResourcePath::document_id).collect(); + QueryScope { + items, + document_ids, + } + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + pub fn matching(&self, predicate: impl Fn(&GroupwareResourceRef<'x>) -> bool) -> Candidates { + Candidates::Set( + self.items + .iter() + .filter(|item| predicate(&item.resource)) + .map(DavResourcePath::document_id) + .collect(), + ) + } + + pub async fn resolve( + self, + server: &Server, + index: SearchIndex, + account_id: u32, + candidates: Candidates, + resources: &GroupwareResources, + ) -> Vec { + let QueryScope { + items, + document_ids, + } = self; + let matches = candidates + .resolve(server, index, account_id, document_ids) + .await; + + items + .into_iter() + .filter(|item| matches.contains(item.document_id())) + .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) + .collect() + } + + pub fn into_items(self, resources: &GroupwareResources, account_id: u32) -> Vec { + self.items + .into_iter() + .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) + .collect() + } +} + +impl Candidates { + pub fn and(items: impl IntoIterator) -> Self { + let mut set: Option = None; + let mut children = Vec::new(); + for item in items { + match item { + Candidates::All => {} + Candidates::Set(items) => match &mut set { + Some(set) => *set &= items, + None => set = Some(items), + }, + Candidates::And(items) => children.extend(items), + item => children.push(item), + } + } + if let Some(set) = set { + if set.is_empty() { + return Candidates::Set(set); + } + children.push(Candidates::Set(set)); + } + if children.len() > 1 { + Candidates::And(children) + } else { + children.pop().unwrap_or(Candidates::All) + } + } + + pub fn or(items: impl IntoIterator) -> Self { + let mut set: Option = None; + let mut children = Vec::new(); + for item in items { + match item { + Candidates::All => return Candidates::All, + Candidates::Set(items) => match &mut set { + Some(set) => *set |= items, + None => set = Some(items), + }, + Candidates::Or(items) => children.extend(items), + item => children.push(item), + } + } + if let Some(set) = set.filter(|set| !set.is_empty()) { + children.push(Candidates::Set(set)); + } + if children.len() > 1 { + Candidates::Or(children) + } else { + children + .pop() + .unwrap_or_else(|| Candidates::Set(RoaringBitmap::new())) + } + } + + pub async fn resolve( + self, + server: &Server, + index: SearchIndex, + account_id: u32, + scope: RoaringBitmap, + ) -> RoaringBitmap { + match self { + Candidates::All => scope, + candidates if candidates.has_text() => { + let mut filters = Vec::with_capacity(8); + candidates.into_filters(&mut filters); + match server + .search_store() + .filter_account( + SearchQuery::new(index) + .with_account_id(account_id) + .with_mask(scope.clone()) + .with_filters(filters), + ) + .await + { + Ok(matches) => matches, + Err(err) => { + trc::error!(err.caused_by(trc::location!())); + scope + } + } + } + candidates => { + let mut matches = candidates.evaluate(&scope); + matches &= &scope; + matches + } + } + } + + fn has_text(&self) -> bool { + match self { + Candidates::Text(_) => true, + Candidates::And(items) | Candidates::Or(items) => { + items.iter().any(Candidates::has_text) + } + Candidates::All | Candidates::Set(_) => false, + } + } + + fn into_filters(self, filters: &mut Vec) { + match self { + Candidates::All => {} + Candidates::Set(set) => filters.push(SearchFilter::is_in_set(set)), + Candidates::Text(filter) => filters.push(filter), + Candidates::And(items) => { + filters.push(SearchFilter::And); + for item in items { + item.into_filters(filters); + } + filters.push(SearchFilter::End); + } + Candidates::Or(items) => { + filters.push(SearchFilter::Or); + for item in items { + item.into_filters(filters); + } + filters.push(SearchFilter::End); + } + } + } + + fn evaluate(self, scope: &RoaringBitmap) -> RoaringBitmap { + match self { + Candidates::All | Candidates::Text(_) => scope.clone(), + Candidates::Set(set) => set, + Candidates::And(items) => items + .into_iter() + .map(|item| item.evaluate(scope)) + .reduce(|a, b| a & b) + .unwrap_or_else(|| scope.clone()), + Candidates::Or(items) => items + .into_iter() + .map(|item| item.evaluate(scope)) + .reduce(|a, b| a | b) + .unwrap_or_default(), + } + } +} + diff --git a/crates/dav/src/common/uri.rs b/crates/dav/src/common/uri.rs index 6b811eb1..56e2a7b9 100644 --- a/crates/dav/src/common/uri.rs +++ b/crates/dav/src/common/uri.rs @@ -5,11 +5,15 @@ */ use crate::{DavError, DavResourceName}; -use common::{Server, auth::AccessToken}; +use common::{ + DavResourcePath, Server, + auth::AccessToken, + storage::dav::{DavFileNameError, canonical_calcard_uri, canonical_dav_resource_uri}, +}; use groupware::cache::GroupwareCache; use http_proto::request::decode_path_element; use hyper::StatusCode; -use std::fmt::Display; +use std::{borrow::Cow, fmt::Display}; use trc::AddContext; use types::collection::Collection; @@ -22,7 +26,51 @@ pub(crate) struct UriResource { pub(crate) enum Urn { Lock(u64), - Sync { id: u64, seq: u32 }, + Sync(SyncToken), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SyncToken { + State(u64), + ChangesPage { + from: u64, + snapshot: u64, + offset: u64, + }, + InitialPage { + snapshot: u64, + after: SyncCursor, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SyncCursor(u128); + +impl SyncCursor { + const PARENT_BITS: u32 = 33; + const DOCUMENT_BITS: u32 = 32; + + pub fn new( + account_id: u32, + is_container: bool, + document_id: u32, + parent_id: Option, + ) -> Self { + let parent = parent_id.map_or(0, |parent_id| u128::from(parent_id) + 1); + let document = u128::from(document_id) << Self::PARENT_BITS; + let is_item = u128::from(!is_container) << (Self::PARENT_BITS + Self::DOCUMENT_BITS); + let account = u128::from(account_id) << (Self::PARENT_BITS + Self::DOCUMENT_BITS + 1); + SyncCursor(account | is_item | document | parent) + } + + pub fn from_path(account_id: u32, path: &DavResourcePath<'_>) -> Self { + SyncCursor::new( + account_id, + path.is_container(), + path.document_id(), + path.parent_id(), + ) + } } pub(crate) type UnresolvedUri<'x> = UriResource, Option<&'x str>>; @@ -30,6 +78,12 @@ pub(crate) type OwnedUri<'x> = UriResource>; pub(crate) type DocumentUri = UriResource; pub(crate) trait DavUriResource: Sync + Send { + fn parse_uri<'x>( + &self, + uri: &'x str, + error_status: StatusCode, + ) -> impl Future>> + Send; + fn validate_uri_with_status<'x>( &self, access_token: &AccessToken, @@ -65,6 +119,21 @@ impl DavUriResource for Server { access_token: &AccessToken, uri: &'x str, error_status: StatusCode, + ) -> crate::Result> { + let resource = self.parse_uri(uri, error_status).await?; + if let Some(account_id) = resource.account_id + && resource.collection != Collection::Principal + && !access_token.has_access(account_id, resource.collection) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + Ok(resource) + } + + async fn parse_uri<'x>( + &self, + uri: &'x str, + error_status: StatusCode, ) -> crate::Result> { let (_, uri_parts) = uri .split_once("/dav/") @@ -97,13 +166,6 @@ impl DavUriResource for Server { .ok_or(DavError::Code(error_status))? }; - // Validate access - if resource.collection != Collection::Principal - && !access_token.has_access(account_id, resource.collection) - { - return Err(DavError::Code(StatusCode::FORBIDDEN)); - } - // Obtain remaining path resource.account_id = Some(account_id); resource.resource = uri_parts.next(); @@ -119,7 +181,7 @@ impl DavUriResource for Server { ) -> trc::Result> { if let Some(resource) = uri.resource { if let Some(resource) = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), uri.account_id, uri.collection.into(), @@ -172,11 +234,19 @@ impl OwnedUri<'_> { } } -/*impl UriResource { - pub fn collection_path(&self) -> &'static str { - DavResourceName::from(self.collection).collection_path() +pub(crate) fn canonical_dav_uri(uri: &str) -> Result, DavFileNameError> { + match uri + .split_once("/dav/") + .and_then(|(_, path)| path.split_once('/')) + .and_then(|(collection, _)| DavResourceName::parse(collection)) + { + Some(DavResourceName::File) => canonical_dav_resource_uri(uri), + Some(DavResourceName::Cal | DavResourceName::Card) => Ok(canonical_calcard_uri(uri)), + Some(DavResourceName::Principal | DavResourceName::Scheduling) | None => { + Ok(Cow::Borrowed(uri)) + } } -}*/ +} impl Urn { pub fn try_extract_sync_id(token: &str) -> Option<&str> { @@ -190,17 +260,7 @@ impl Urn { let (kind, id) = inbox.split_once(':')?; match kind { "davlock" => u64::from_str_radix(id, 16).ok().map(Urn::Lock), - "davsync" => { - if let Some((id, seq)) = id.split_once(':') { - let id = u64::from_str_radix(id, 16).ok()?; - let seq = u32::from_str_radix(seq, 16).ok()?; - Some(Urn::Sync { id, seq }) - } else { - u64::from_str_radix(id, 16) - .ok() - .map(|id| Urn::Sync { id, seq: 0 }) - } - } + "davsync" => SyncToken::parse(id).map(Urn::Sync), _ => None, } } @@ -212,9 +272,9 @@ impl Urn { } } - pub fn try_unwrap_sync(&self) -> Option<(u64, u32)> { + pub fn try_unwrap_sync(&self) -> Option { match self { - Urn::Sync { id, seq } => Some((*id, *seq)), + Urn::Sync(token) => Some(*token), _ => None, } } @@ -224,13 +284,44 @@ impl Display for Urn { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Urn::Lock(id) => write!(f, "urn:inbuxa:davlock:{id:x}",), - Urn::Sync { id, seq } => { - if *seq == 0 { - write!(f, "urn:inbuxa:davsync:{id:x}") - } else { - write!(f, "urn:inbuxa:davsync:{id:x}:{seq:x}") - } + Urn::Sync(token) => write!(f, "urn:inbuxa:davsync:{token}"), + } + } +} + +impl SyncToken { + fn parse(input: &str) -> Option { + let mut parts = input.splitn(3, ':'); + let id = u64::from_str_radix(parts.next()?, 16).ok()?; + match (parts.next(), parts.next()) { + (None, None) => Some(SyncToken::State(id)), + (Some(after), None) => Some(SyncToken::InitialPage { + snapshot: id, + after: SyncCursor(u128::from_str_radix(after.strip_prefix('i')?, 16).ok()?), + }), + (Some(offset), Some(snapshot)) => Some(SyncToken::ChangesPage { + from: id, + snapshot: u64::from_str_radix(snapshot, 16).ok()?, + offset: u64::from_str_radix(offset, 16).ok()?, + }), + (None, Some(_)) => None, + } + } +} + +impl Display for SyncToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SyncToken::State(id) => write!(f, "{id:x}"), + SyncToken::ChangesPage { + from, + snapshot, + offset, + } => write!(f, "{from:x}:{offset:x}:{snapshot:x}"), + SyncToken::InitialPage { snapshot, after } => { + write!(f, "{snapshot:x}:i{:x}", after.0) } } } } + diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index 0130e4ab..995c0534 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -9,31 +9,44 @@ use crate::{ DavError, DavMethod, common::{ ExtractETag, + dead::{ + ContainerRequest, ContainerWrites, copy_growth, edited_container, read_container, + stored_entry_of, + }, lock::{LockRequestHandler, ResourceState}, uri::{DavUriResource, UriResource}, }, - file::{DavFileResource, FileItemId}, + file::{DavFileResource, FileItemId, file_name_from_uri, is_symlink}, }; use common::{ - DavResourcePath, DavResources, Server, auth::AccessToken, storage::index::ObjectIndexBuilder, + DavResourcePath, GroupwareResources, Server, + auth::{AccessToken, AccountCache, ChangedBy}, + storage::{ + batch::MAX_BATCH_DOCUMENTS, + dav::{FilePresence, MAX_FILE_NODE_DEPTH}, + index::ObjectIndexBuilder, + metadata::{ContainerCleanup, StoredContainer, StoredEntry}, + }, }; use dav_proto::{Depth, RequestHeaders}; use groupware::{DestroyArchive, cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use std::sync::Arc; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes, metadata::MetadataBuf}, }; use store::{ ahash::AHashMap, - write::{BatchBuilder, now}, + write::{BatchBuilder, Slot}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, + metadata::MetadataKinds, }; pub(crate) trait FileCopyMoveRequestHandler: Sync + Send { @@ -59,34 +72,29 @@ impl FileCopyMoveRequestHandler for Server { .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; - let from_resource = from_resources.map_resource::(&from_resource_)?; - let from_resource_name = from_resource_.resource.unwrap(); - - // Validate source ACLs - if !access_token.is_member(from_account_id) { - let shared = from_resources.shared_containers( - access_token, - if is_move { - [Acl::Read, Acl::Delete].as_slice().iter().copied() - } else { - [Acl::Read].as_slice().iter().copied() - }, - false, - ); - - for resource in from_resources.subtree(from_resource_.resource.unwrap()) { - if !shared.contains(resource.document_id()) { - return Err(DavError::Code(StatusCode::FORBIDDEN)); - } - } + let from_access = (!access_token.is_member(from_account_id)) + .then(|| from_resources.file_access(access_token)); + if let Some(access) = &from_access + && let Some(path) = from_resource_.resource + { + from_resources.hide_undiscoverable( + &access.discoverable, + path, + StatusCode::NOT_FOUND, + StatusCode::NOT_FOUND, + )?; } + let from_resource = from_resources.map_resource::(&from_resource_)?; + let from_resource_name = from_resource_ + .resource + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate destination let destination = self @@ -104,10 +112,11 @@ impl FileCopyMoveRequestHandler for Server { let to_account_id = destination .account_id .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; - let to_resources = if to_account_id == from_account_id { + let is_same_account = to_account_id == from_account_id; + let to_resources = if is_same_account { from_resources.clone() } else { - self.fetch_dav_resources( + self.fetch_groupware_resources( access_token.account_id(), to_account_id, SyncCollection::FileNode, @@ -120,62 +129,104 @@ impl FileCopyMoveRequestHandler for Server { let destination_resource_name = destination .resource .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; - if from_account_id == to_account_id - && (from_resource_name == destination_resource_name - || from_resource_name - .strip_prefix(destination_resource_name) - .is_some_and(|v| v.is_empty() || v.starts_with('/'))) - { - return Ok(HttpResponse::new(StatusCode::BAD_GATEWAY)); + if is_same_account { + if is_same_or_descendant(from_resource_name, destination_resource_name) { + return Ok(HttpResponse::new(StatusCode::BAD_GATEWAY)); + } else if is_same_or_descendant(destination_resource_name, from_resource_name) { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + } + + let to_access_owned = (!is_same_account && !access_token.is_member(to_account_id)) + .then(|| to_resources.file_access(access_token)); + let to_access = if is_same_account { + from_access.as_ref() + } else { + to_access_owned.as_ref() + }; + if let Some(access) = to_access { + to_resources.hide_undiscoverable( + &access.discoverable, + destination_resource_name, + StatusCode::FORBIDDEN, + StatusCode::CONFLICT, + )?; } // Check if the resource exists + let parent = to_resources.map_directory_parent(destination_resource_name)?; let mut delete_destination = None; - let mut destination = if let Some((destination, new_name)) = - to_resources.map_parent(destination_resource_name) - { - if let Some(mut existing_destination) = to_resources - .by_path(destination_resource_name) - .map(Destination::from_dav_resource) - { - if !headers.overwrite_fail { - existing_destination.account_id = to_account_id; - delete_destination = Some(existing_destination); - } else { - return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED)); - } + if let Some(existing) = to_resources.by_path(destination_resource_name) { + if is_symlink(&existing) { + return Err(DavError::Code(StatusCode::CONFLICT)); + } else if headers.overwrite_fail { + return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED)); } - - let mut destination = destination - .map(Destination::from_dav_resource) - .unwrap_or_default(); - destination.new_name = Some(new_name.to_string()); - destination - } else { - return Err(DavError::Code(StatusCode::CONFLICT)); + delete_destination = Some(ExistingDestination { + document_id: existing.document_id(), + is_container: existing.is_container(), + }); + } + let name = file_name_from_uri( + headers + .destination + .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?, + )?; + let copy_depth = match (is_move, headers.depth) { + (false, Depth::Zero) => 0, + (false, Depth::One) => 1, + _ => usize::MAX, }; + if destination_resource_name.split('/').count() + + subtree_height(&from_resources, from_resource_name, copy_depth) + > MAX_FILE_NODE_DEPTH + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + let mut destination = parent + .map(Destination::from_dav_resource) + .unwrap_or_default(); + destination.new_name = Some(name); destination.account_id = to_account_id; + let changes_parent = + !is_same_account || from_resource.resource.parent_id != destination.document_id; + + // Validate source ACLs + if let Some(access) = &from_access { + let removes_source = is_move && !is_same_account; + if from_resources.subtree(from_resource_name).any(|resource| { + !access.has_acl(resource.document_id(), Acl::Read) + || (removes_source + && !access.may_delete(resource.document_id(), resource.parent_id())) + }) || (is_move + && (!access.has_acl(from_resource.resource.document_id, Acl::Modify) + || (changes_parent + && !removes_source + && from_resource + .resource + .parent_id + .is_none_or(|parent_id| !access.has_acl(parent_id, Acl::RemoveItems))))) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + } // Validate destination ACLs - if let Some(document_id) = destination.document_id { - if let Some(delete_destination) = &delete_destination - && !access_token.is_member(to_account_id) - && !to_resources.has_access_to_container( - access_token, - delete_destination.document_id.unwrap(), - Acl::Delete, - ) + if let Some(to_access) = to_access { + let adds_to_parent = changes_parent || !is_move; + if (adds_to_parent + && destination + .document_id + .is_none_or(|parent_id| !to_access.has_acl(parent_id, Acl::AddItems))) + || (delete_destination.is_some() + && !to_access.may_destroy( + to_resources + .subtree(destination_resource_name) + .map(|resource| (resource.document_id(), resource.parent_id())), + )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } - - if !access_token.is_member(to_account_id) - && !to_resources.has_access_to_container(access_token, document_id, Acl::Modify) - { - return Err(DavError::Code(StatusCode::FORBIDDEN)); - } - } else if !access_token.is_member(to_account_id) { - return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers @@ -196,8 +247,7 @@ impl FileCopyMoveRequestHandler for Server { document_id: Some( delete_destination .as_ref() - .and_then(|d| d.document_id) - .unwrap_or(u32::MAX), + .map_or(u32::MAX, |existing| existing.document_id), ), path: destination_resource_name, ..Default::default() @@ -212,42 +262,88 @@ impl FileCopyMoveRequestHandler for Server { ) .await?; - if delete_destination.is_none() - && from_account_id == destination.account_id - && from_resource.resource.parent_id == destination.document_id - && destination.new_name.is_some() - && is_move - { + let from_href = from_resources + .by_path(from_resource_name) + .map(|resource| from_resources.format_resource(resource)) + .unwrap_or_default(); + + if delete_destination.is_none() && !changes_parent && is_move { // Rename - let from_resource_path = if from_resource.resource.is_container { - from_resources.format_collection(from_resource_name) - } else { - from_resources.format_item(from_resource_name) - }; return rename_item( self, access_token, + &from_resources, from_resource, - from_resource_path, + from_resource_name, + from_href, destination, ) .await; } // Validate quota - if !is_move || from_account_id != to_account_id { + let mut quota = None; + if !is_move || !is_same_account { let space_needed = from_resources .subtree(from_resource_name) .map(|a| a.size() as u64) .sum::(); - self.has_available_quota(self.account(to_account_id).await?.as_ref(), space_needed) - .await?; + let to_account = self.account(to_account_id).await?; + + let files_limit = self.object_quota_limit(&to_account, StorageQuota::MaxFiles); + let folders_limit = self.object_quota_limit(&to_account, StorageQuota::MaxFolders); + let is_item_overwrite = delete_destination + .as_ref() + .is_some_and(|d| !d.is_container && !from_resource.resource.is_container); + if (files_limit.is_some() || folders_limit.is_some()) && !is_item_overwrite { + let (created_files, created_folders) = if !from_resource.resource.is_container { + (1, 0) + } else { + match (is_move, headers.depth) { + (false, Depth::Zero) => (0, 1), + (false, Depth::One) => { + count_nodes(from_resources.subtree_with_depth(from_resource_name, 1)) + } + _ => count_nodes(from_resources.subtree(from_resource_name)), + } + }; + let (deleted_files, deleted_folders) = if delete_destination.is_some() { + count_nodes(to_resources.subtree(destination_resource_name)) + } else { + (0, 0) + }; + let new_files = created_files.saturating_sub(deleted_files); + let new_folders = created_folders.saturating_sub(deleted_folders); + + if new_files > 0 || new_folders > 0 { + let used_folders = to_resources.resources.count(true); + let used_files = to_resources.resources.len().saturating_sub(used_folders); + if new_files > 0 { + self.assert_object_quota( + &to_account, + StorageQuota::MaxFiles, + new_files, + || used_files, + )?; + } + if new_folders > 0 { + self.assert_object_quota( + &to_account, + StorageQuota::MaxFolders, + new_folders, + || used_folders, + )?; + } + } + } + quota = Some((to_account, space_needed)); } // Delete collection let is_overwrite = delete_destination .as_ref() .is_some_and(|d| d.is_container || from_resource.resource.is_container); + let mut overwritten = None; if is_overwrite { delete_destination = None; // Find ids to delete @@ -256,19 +352,29 @@ impl FileCopyMoveRequestHandler for Server { .collect::>(); if !ids.is_empty() { ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); - let mut sorted_ids = Vec::with_capacity(ids.len()); - sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); - DestroyArchive(sorted_ids) - .delete( - self, - access_token.account_tenant_ids(), - destination.account_id, - None, + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + to_account_id, + Collection::FileNode, + &ids.iter() + .filter(|a| !a.resource.metadata_kinds().is_empty()) + .map(|a| a.document_id()) + .collect(), ) .await .caused_by(trc::location!())?; + let mut sorted_ids = Vec::with_capacity(ids.len()); + sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); + overwritten = Some(Overwritten { + changed_by: access_token.changed_by(), + account_id: to_account_id, + ids: sorted_ids, + cleanup, + }); } } + let transfer = Transfer { quota, overwritten }; match (from_resource.resource.is_container, is_move) { (true, true) => { @@ -278,8 +384,10 @@ impl FileCopyMoveRequestHandler for Server { from_resources, from_resource, from_resource_name, + from_href, destination, headers.depth, + transfer, ) .await } @@ -290,20 +398,24 @@ impl FileCopyMoveRequestHandler for Server { from_resources, from_resource, from_resource_name, + from_href, destination, headers.depth, false, + transfer, ) .await } (false, true) => { - if let Some(delete_destination) = delete_destination { + if let Some(existing) = delete_destination { overwrite_and_delete_item( self, access_token, from_resource, - from_resources.format_item(from_resource_name), - delete_destination, + from_href, + existing.document_id, + destination, + transfer, ) .await } else { @@ -311,18 +423,26 @@ impl FileCopyMoveRequestHandler for Server { self, access_token, from_resource, - from_resources.format_item(from_resource_name), + from_href, destination, + transfer, ) .await } } - (false, false) => { - if let Some(delete_destination) = delete_destination { - overwrite_item(self, access_token, from_resource, delete_destination).await + if let Some(existing) = delete_destination { + overwrite_item( + self, + access_token, + from_resource, + existing.document_id, + destination, + transfer, + ) + .await } else { - copy_item(self, access_token, from_resource, destination).await + copy_item(self, access_token, from_resource, destination, transfer).await } } } @@ -336,34 +456,94 @@ impl FileCopyMoveRequestHandler for Server { } } -#[derive(Debug)] +fn is_same_or_descendant(path: &str, ancestor: &str) -> bool { + path.strip_prefix(ancestor) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('/')) +} + +fn subtree_height(resources: &GroupwareResources, path: &str, depth: usize) -> usize { + if depth == 0 { + return 0; + } + let base = path.split('/').count(); + resources + .subtree(path) + .map(|resource| resource.path().split('/').count().saturating_sub(base)) + .max() + .unwrap_or_default() + .min(depth) +} + +fn count_nodes<'x>(nodes: impl Iterator>) -> (usize, usize) { + nodes.fold((0, 0), |(files, folders), node| { + if node.is_container() { + (files, folders + 1) + } else { + (files + 1, folders) + } + }) +} + +#[derive(Debug, Default)] pub(crate) struct Destination { pub account_id: u32, pub new_name: Option, pub document_id: Option, - pub is_container: bool, } -impl Default for Destination { - fn default() -> Self { - Self { - account_id: Default::default(), - document_id: Default::default(), - new_name: Default::default(), - is_container: true, +#[derive(Debug)] +struct ExistingDestination { + document_id: u32, + is_container: bool, +} + +struct Transfer { + quota: Option<(Arc, u64)>, + overwritten: Option, +} + +struct Overwritten { + changed_by: ChangedBy, + account_id: u32, + ids: Vec, + cleanup: ContainerCleanup, +} + +impl Transfer { + async fn prepare(self, server: &Server, growth: u64) -> crate::Result<()> { + if let Some((account, bytes)) = &self.quota { + server + .has_available_quota(account, bytes.saturating_add(growth)) + .await?; } + if let Some(overwritten) = self.overwritten { + DestroyArchive(overwritten.ids) + .delete( + server, + overwritten.changed_by, + overwritten.account_id, + &overwritten.cleanup, + None, + ) + .await + .caused_by(trc::location!())?; + } + Ok(()) } } // Moves a container under an existing container +#[allow(clippy::too_many_arguments)] async fn move_container( server: &Server, access_token: &AccessToken, - from_resources: Arc, + from_resources: Arc, from_resource: UriResource, from_resource_name: &str, + from_href: String, destination: Destination, depth: Depth, + transfer: Transfer, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; @@ -371,9 +551,10 @@ async fn move_container( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); if from_account_id == to_account_id { + transfer.prepare(server, 0).await?; let node_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -392,7 +573,7 @@ async fn move_container( let mut batch = BatchBuilder::new(); let etag = new_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), node, from_account_id, from_document_id, @@ -401,10 +582,10 @@ async fn move_container( ) .caused_by(trc::location!())? .etag(); - batch.with_account_id(from_account_id).log_vanished_item( - VanishedCollection::FileNode, - from_resources.format_collection(from_resource_name), - ); + from_resources.log_descendant_updates(&mut batch, from_account_id, from_resource_name); + batch + .with_account_id(from_account_id) + .log_vanished_item(VanishedCollection::FileNode, from_href); server .commit_batch(batch) .await @@ -418,9 +599,11 @@ async fn move_container( from_resources, from_resource, from_resource_name, + from_href, destination, depth, true, + transfer, ) .await } @@ -430,16 +613,18 @@ async fn move_container( async fn copy_container( server: &Server, access_token: &AccessToken, - from_resources: Arc, + from_resources: Arc, from_resource: UriResource, from_resource_name: &str, + from_href: String, mut destination: Destination, depth: Depth, delete_source: bool, + transfer: Transfer, ) -> crate::Result { let infinity_copy = match depth { Depth::Zero if !delete_source => { - return copy_item(server, access_token, from_resource, destination).await; + return copy_item(server, access_token, from_resource, destination, transfer).await; } Depth::One if !delete_source => false, _ => true, @@ -450,17 +635,31 @@ async fn copy_container( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); // Obtain files to copy + let mut container_request = ContainerRequest::default(); + let mut select = |r: DavResourcePath<'_>| { + if !r.resource.metadata_kinds().is_empty() { + container_request.insert(from_account_id, Collection::FileNode, r.document_id()); + } + (r.document_id(), r.hierarchy_seq()) + }; let mut copy_files = if infinity_copy { from_resources .subtree(from_resource_name) - .map(|r| (r.document_id(), r.hierarchy_seq())) + .map(&mut select) .collect::>() } else { from_resources .subtree_with_depth(from_resource_name, 1) - .map(|r| (r.document_id(), r.hierarchy_seq())) + .map(&mut select) .collect::>() }; + let containers = container_request + .load(server) + .await + .caused_by(trc::location!())?; + transfer.prepare(server, containers.stored_len()).await?; + let mut copies = ContainerWrites::new(to_account_id, Collection::FileNode); + let mut removals = ContainerWrites::new(from_account_id, Collection::FileNode); // Top-down copy let mut batch = BatchBuilder::new(); @@ -471,65 +670,85 @@ async fn copy_container( Vec::new() }; copy_files.sort_unstable_by_key(|a| a.1); - let now = now() as i64; - let mut next_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::FileNode, copy_files.len() as u64) - .await - .caused_by(trc::location!())?; - for (document_id, _) in copy_files.into_iter() { - let node_ = server - .store() - .get_value::>(ValueKey::archive( - from_account_id, - Collection::FileNode, - document_id, - )) + let mut new_slots = batch + .reserve_document_ids(to_account_id, Collection::FileNode, copy_files.len() as u32) + .iter(); + let mut chunk_ids = Vec::with_capacity(copy_files.len().min(MAX_BATCH_DOCUMENTS)); + for chunk in copy_files.chunks(MAX_BATCH_DOCUMENTS) { + chunk_ids.clear(); + chunk_ids.extend(chunk.iter().map(|(document_id, _)| *document_id)); + chunk_ids.sort_unstable(); + chunk_ids.dedup(); + let mut archives = server + .archive_chunk(from_account_id, Collection::FileNode, &chunk_ids, None) .await - .caused_by(trc::location!())? - .ok_or(DavError::Code(StatusCode::NOT_FOUND))? - .into_deserialized::() .caused_by(trc::location!())?; + for (&(document_id, _), new_slot) in chunk.iter().zip(new_slots.by_ref()) { + let node_ = archives + .take_archive(document_id) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))? + .into_deserialized::() + .caused_by(trc::location!())?; - // Build node - let mut node = if !delete_source { - node_.inner - } else { - let node = node_.inner.clone(); - delete_files.push((document_id, node_)); - node - }; - node.modified = now; - node.created = now; - if let Some(new_name) = destination.new_name.take() { - node.name = new_name; + // Build node + let mut node = if !delete_source { + node_.inner + } else { + let node = node_.inner.clone(); + delete_files.push((document_id, node_)); + node + }; + let container = containers.get(from_account_id, Collection::FileNode, document_id); + node.set_presence(file_presence(container)); + let set_accessed = node.accessed == 0; + node.stamp_insert(true, true, set_accessed); + node.acls.clear(); + if let Some(new_name) = destination.new_name.take() { + node.name = new_name; + } + let pending_parent = id_map.get(&node.parent_id).copied(); + if pending_parent.is_none() { + node.parent_id = parent_id; + } + + // Prepare write batch + if let Some(container) = container { + copies + .copy(server, container, new_slot, None, &mut batch) + .await + .caused_by(trc::location!())?; + } + let builder = ObjectIndexBuilder::<(), _>::new() + .with_changes(node) + .with_changed_by(access_token.changed_by()); + batch + .with_account_id(to_account_id) + .with_collection(Collection::FileNode) + .create_document(new_slot) + .custom(builder.with_pending_id_opt(pending_parent)) + .caused_by(trc::location!())? + .commit_point(); + id_map.insert(document_id + 1, new_slot); } - node.parent_id = if let Some(&prev_document_id) = id_map.get(&node.parent_id) { - prev_document_id - } else { - parent_id - }; - - // Prepare write batch - let new_document_id = next_document_id; - next_document_id -= 1; - batch - .with_account_id(to_account_id) - .with_collection(Collection::FileNode) - .with_document(new_document_id) - .custom( - ObjectIndexBuilder::<(), _>::new() - .with_changes(node) - .with_changed_by(access_token.account_tenant_ids()), - ) - .caused_by(trc::location!())? - .commit_point(); - id_map.insert(document_id + 1, new_document_id + 1); } // Delete nodes if !delete_files.is_empty() { for (document_id, node) in delete_files.into_iter().rev() { + if let Some(container) = + containers.get(from_account_id, Collection::FileNode, document_id) + { + removals + .clear( + server, + document_id, + StoredContainer::from(container), + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + // Delete record batch .with_account_id(from_account_id) @@ -537,16 +756,15 @@ async fn copy_container( .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() - .with_changed_by(access_token.account_tenant_ids()) + .with_changed_by(access_token.changed_by()) .with_current(node), ) .caused_by(trc::location!())? .commit_point(); } - batch.with_account_id(from_account_id).log_vanished_item( - VanishedCollection::FileNode, - from_resources.format_collection(from_resource_name), - ); + batch + .with_account_id(from_account_id) + .log_vanished_item(VanishedCollection::FileNode, from_href); } // Write changes @@ -561,22 +779,24 @@ async fn copy_container( } // Overwrites the contents of one file with another, then deletes the original +#[allow(clippy::too_many_arguments)] async fn overwrite_and_delete_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, from_resource_path: String, + to_document_id: u32, destination: Destination, + transfer: Transfer, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; - let to_document_id = destination.document_id.unwrap(); // dest_node is the current file at the destination let dest_node_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::FileNode, to_document_id, @@ -592,7 +812,7 @@ async fn overwrite_and_delete_item( // source_node is the file to be copied let source_node__ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -606,17 +826,30 @@ async fn overwrite_and_delete_item( let mut source_node = source_node_ .deserialize::() .caused_by(trc::location!())?; - source_node.name = if let Some(new_name) = destination.new_name { - new_name - } else { - dest_node.inner.name.to_string() - }; + if let Some(new_name) = destination.new_name { + source_node.name = new_name; + } + if from_account_id != to_account_id { + source_node.acls.clear(); + } source_node.parent_id = dest_node.inner.parent_id.into(); let mut batch = BatchBuilder::new(); + let source = replace_container( + server, + from_account_id, + from_document_id, + &mut source_node, + to_account_id, + to_document_id, + dest_node.inner.metadata_kinds(), + transfer, + &mut batch, + ) + .await?; let etag = source_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), dest_node, to_account_id, to_document_id, @@ -625,14 +858,26 @@ async fn overwrite_and_delete_item( ) .caused_by(trc::location!())? .etag(); + let cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::FileNode, + source, + ) + .await + .caused_by(trc::location!())?; DestroyArchive(source_node_) - .delete( - access_token.account_tenant_ids(), + .delete_with_cleanup( + server, + access_token.changed_by(), from_account_id, from_document_id, &mut batch, from_resource_path, + &cleanup, ) + .await .caused_by(trc::location!())?; server .commit_batch(batch) @@ -647,17 +892,18 @@ async fn overwrite_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, + to_document_id: u32, destination: Destination, + transfer: Transfer, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; - let to_document_id = destination.document_id.unwrap(); // dest_node is the current file at the destination let dest_node_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( to_account_id, Collection::FileNode, to_document_id, @@ -673,7 +919,7 @@ async fn overwrite_item( // source_node is the file to be copied let mut source_node = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -683,16 +929,27 @@ async fn overwrite_item( .ok_or(DavError::Code(StatusCode::NOT_FOUND))? .deserialize::() .caused_by(trc::location!())?; - source_node.name = if let Some(new_name) = destination.new_name { - new_name - } else { - dest_node.inner.name.to_string() - }; + if let Some(new_name) = destination.new_name { + source_node.name = new_name; + } + source_node.acls.clear(); source_node.parent_id = dest_node.inner.parent_id.into(); let mut batch = BatchBuilder::new(); + replace_container( + server, + from_account_id, + from_document_id, + &mut source_node, + to_account_id, + to_document_id, + dest_node.inner.metadata_kinds(), + transfer, + &mut batch, + ) + .await?; let etag = source_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), dest_node, to_account_id, to_document_id, @@ -716,6 +973,7 @@ async fn move_item( from_resource: UriResource, from_resource_path: String, destination: Destination, + transfer: Transfer, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; @@ -724,7 +982,7 @@ async fn move_item( let node_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -744,10 +1002,10 @@ async fn move_item( let mut batch = BatchBuilder::new(); let etag = if from_account_id == to_account_id { // Destination is in the same account: just update the parent id - batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); - new_node + transfer.prepare(server, 0).await?; + let etag = new_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), node, from_account_id, from_document_id, @@ -755,17 +1013,27 @@ async fn move_item( &mut batch, ) .caused_by(trc::location!())? - .etag() + .etag(); + batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); + etag } else { // Destination is in a different account: insert a new node, then delete the old one - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; + new_node.acls.clear(); + let to_document_id = batch.reserve_document_id(to_account_id, Collection::FileNode); + let source = copy_node_container( + server, + from_account_id, + from_document_id, + &mut new_node, + to_account_id, + to_document_id, + transfer, + &mut batch, + ) + .await?; let etag = new_node .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), to_account_id, to_document_id, true, @@ -774,14 +1042,26 @@ async fn move_item( ) .caused_by(trc::location!())? .etag(); + let cleanup = server + .container_cleanup_from( + access_token.changed_by(), + from_account_id, + Collection::FileNode, + source, + ) + .await + .caused_by(trc::location!())?; DestroyArchive(node) - .delete( - access_token.account_tenant_ids(), + .delete_with_cleanup( + server, + access_token.changed_by(), from_account_id, from_document_id, &mut batch, from_resource_path, + &cleanup, ) + .await .caused_by(trc::location!())?; etag }; @@ -799,6 +1079,7 @@ async fn copy_item( access_token: &AccessToken, from_resource: UriResource, destination: Destination, + transfer: Transfer, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; @@ -807,7 +1088,7 @@ async fn copy_item( let mut node = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -818,18 +1099,26 @@ async fn copy_item( .deserialize::() .caused_by(trc::location!())?; node.parent_id = parent_id; + node.acls.clear(); if let Some(new_name) = destination.new_name { node.name = new_name; } let mut batch = BatchBuilder::new(); - let to_document_id = server - .store() - .assign_document_ids(to_account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; + let to_document_id = batch.reserve_document_id(to_account_id, Collection::FileNode); + copy_node_container( + server, + from_account_id, + from_document_id, + &mut node, + to_account_id, + to_document_id, + transfer, + &mut batch, + ) + .await?; let etag = node .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), to_account_id, to_document_id, true, @@ -850,7 +1139,9 @@ async fn copy_item( async fn rename_item( server: &Server, access_token: &AccessToken, + from_resources: &GroupwareResources, from_resource: UriResource, + from_resource_name: &str, from_resource_path: String, destination: Destination, ) -> crate::Result { @@ -859,7 +1150,7 @@ async fn rename_item( let node_ = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, @@ -877,7 +1168,7 @@ async fn rename_item( let mut batch = BatchBuilder::new(); let etag = new_node .update( - access_token.account_tenant_ids(), + access_token.changed_by(), node, from_account_id, from_document_id, @@ -886,7 +1177,12 @@ async fn rename_item( ) .caused_by(trc::location!())? .etag(); - batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); + if from_resource.resource.is_container { + from_resources.log_descendant_updates(&mut batch, from_account_id, from_resource_name); + } + batch + .with_account_id(from_account_id) + .log_vanished_item(VanishedCollection::FileNode, from_resource_path); server .commit_batch(batch) .await @@ -895,12 +1191,116 @@ async fn rename_item( Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } +fn file_presence(container: Option<&MetadataBuf>) -> FilePresence { + container.map_or(FilePresence::NONE, |container| { + FilePresence::from_view(&container.view()) + }) +} + +#[allow(clippy::too_many_arguments)] +async fn copy_node_container( + server: &Server, + from_account_id: u32, + from_document_id: u32, + node: &mut FileNode, + to_account_id: u32, + to_document_id: Slot, + transfer: Transfer, + batch: &mut BatchBuilder, +) -> crate::Result> { + let source = read_container( + server, + from_account_id, + Collection::FileNode, + from_document_id, + node.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + node.set_presence(file_presence(source.as_ref())); + transfer + .prepare( + server, + source + .as_ref() + .map_or(0, |source| copy_growth(source, None)), + ) + .await?; + if let Some(source) = &source { + ContainerWrites::new(to_account_id, Collection::FileNode) + .copy(server, source, to_document_id, None, batch) + .await + .caused_by(trc::location!())?; + } + Ok(source + .as_ref() + .map(|source| StoredEntry::from_container(from_document_id, source))) +} + +#[allow(clippy::too_many_arguments)] +async fn replace_container( + server: &Server, + from_account_id: u32, + from_document_id: u32, + node: &mut FileNode, + to_account_id: u32, + to_document_id: u32, + to_kinds: MetadataKinds, + transfer: Transfer, + batch: &mut BatchBuilder, +) -> crate::Result> { + let source = read_container( + server, + from_account_id, + Collection::FileNode, + from_document_id, + node.metadata_kinds(), + ) + .await + .caused_by(trc::location!())?; + let previous = stored_entry_of( + server, + to_account_id, + Collection::FileNode, + to_document_id, + to_kinds, + ) + .await + .caused_by(trc::location!())?; + node.set_presence(file_presence(source.as_ref())); + transfer + .prepare( + server, + source + .as_ref() + .map_or(0, |source| copy_growth(source, previous.as_ref())), + ) + .await?; + + let mut writes = ContainerWrites::new(to_account_id, Collection::FileNode); + match (&source, previous) { + (Some(source), previous) => { + let previous = previous + .map(|previous| edited_container(previous, to_account_id, to_kinds)) + .transpose()?; + writes + .copy(server, source, to_document_id, previous, batch) + .await + } + (None, Some(previous)) => writes.release(server, previous, batch).await, + (None, None) => Ok(()), + } + .caused_by(trc::location!())?; + Ok(source + .as_ref() + .map(|source| StoredEntry::from_container(from_document_id, source))) +} + impl FromDavResource for Destination { fn from_dav_resource(item: DavResourcePath<'_>) -> Self { Destination { account_id: u32::MAX, document_id: Some(item.document_id()), - is_container: item.is_container(), new_name: None, } } diff --git a/crates/dav/src/file/delete.rs b/crates/dav/src/file/delete.rs index 942a8fe0..03925e46 100644 --- a/crates/dav/src/file/delete.rs +++ b/crates/dav/src/file/delete.rs @@ -10,14 +10,16 @@ use crate::{ lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, + file::DavFileResource, }; use common::{Server, auth::AccessToken}; use dav_proto::RequestHeaders; use groupware::{DestroyArchive, cache::GroupwareCache}; use http_proto::HttpResponse; use hyper::StatusCode; +use store::roaring::RoaringBitmap; use trc::AddContext; -use types::{acl::Acl, collection::SyncCollection}; +use types::collection::{Collection, SyncCollection}; pub(crate) trait FileDeleteRequestHandler: Sync + Send { fn handle_file_delete_request( @@ -44,7 +46,7 @@ impl FileDeleteRequestHandler for Server { .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, @@ -52,31 +54,47 @@ impl FileDeleteRequestHandler for Server { .await .caused_by(trc::location!())?; + let access = + (!access_token.is_member(account_id)).then(|| resources.file_access(access_token)); + if let Some(access) = &access { + resources.hide_undiscoverable( + &access.discoverable, + delete_path, + StatusCode::NOT_FOUND, + StatusCode::NOT_FOUND, + )?; + } + // Find ids to delete let mut ids = resources.subtree(delete_path).collect::>(); - if ids.is_empty() { + if ids.is_empty() || ids.first().is_some_and(super::is_symlink) { return Err(DavError::Code(StatusCode::NOT_FOUND)); } // Sort ids descending from the deepest to the root ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); - let (document_id, full_delete_path) = ids + let Some((document_id, full_delete_path)) = ids .last() .map(|a| (a.document_id(), resources.format_resource(*a))) - .unwrap(); - let mut sorted_ids = Vec::with_capacity(ids.len()); - sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); + else { + return Err(DavError::Code(StatusCode::NOT_FOUND)); + }; // Validate ACLs - if !access_token.is_member(account_id) { - let permissions = resources.shared_containers(access_token, [Acl::Delete], false); - if permissions.len() < sorted_ids.len() as u64 - || !sorted_ids.iter().all(|id| permissions.contains(*id)) - { - return Err(DavError::Code(StatusCode::FORBIDDEN)); - } + if let Some(access) = &access + && !access.may_destroy(ids.iter().map(|a| (a.document_id(), a.parent_id()))) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); } + let flagged = ids + .iter() + .filter(|a| !a.resource.metadata_kinds().is_empty()) + .map(|a| a.document_id()) + .collect::(); + let mut sorted_ids = Vec::with_capacity(ids.len()); + sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); + // Validate headers self.validate_headers( access_token, @@ -93,11 +111,21 @@ impl FileDeleteRequestHandler for Server { ) .await?; + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::FileNode, + &flagged, + ) + .await + .caused_by(trc::location!())?; DestroyArchive(sorted_ids) .delete( self, - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, + &cleanup, full_delete_path.into(), ) .await?; diff --git a/crates/dav/src/file/get.rs b/crates/dav/src/file/get.rs index 934aa49c..4353b1d5 100644 --- a/crates/dav/src/file/get.rs +++ b/crates/dav/src/file/get.rs @@ -7,24 +7,24 @@ use crate::{ DavError, DavMethod, common::{ - ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, - file::DavFileResource, + file::{DavFileResource, file_etag}, }; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{Server, auth::AccessToken, storage::dav::FILE_DEFAULT_MEDIA_TYPE}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, - write::{AlignedBytes, Archive, now}, + write::{Archive, ArchiveBytes, now}, }; use trc::AddContext; use types::{ acl::Acl, + blob_hash::BlobHash, collection::{Collection, SyncCollection}, }; @@ -51,19 +51,29 @@ impl FileGetRequestHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let files = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; + if !access_token.is_member(account_id) + && let Some(path) = resource_.resource + { + files.hide_undiscoverable_for( + access_token, + path, + StatusCode::NOT_FOUND, + StatusCode::NOT_FOUND, + )?; + } let resource = files.map_resource(&resource_)?; // Fetch node let node_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::FileNode, resource.resource, @@ -75,14 +85,16 @@ impl FileGetRequestHandler for Server { // Validate ACL if !access_token.is_member(account_id) - && !node.acls.effective_acl(access_token).contains(Acl::Read) + && !files + .file_acl(access_token, resource.resource) + .contains(Acl::Read) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } - let (hash, size, content_type) = if let Some(file) = node.file.as_ref() { + let (hash, size, content_type) = if let Some(file) = node.file() { ( - file.blob_hash.0.as_ref(), + BlobHash::from(&file.blob_hash), u32::from(file.size) as usize, file.media_type.as_ref().map(|s| s.as_str()), ) @@ -91,7 +103,7 @@ impl FileGetRequestHandler for Server { }; // Validate headers - let etag = node_.etag(); + let etag = file_etag(node); self.validate_headers( access_token, headers, @@ -135,7 +147,7 @@ impl FileGetRequestHandler for Server { }; let response = HttpResponse::new(StatusCode::OK) - .with_content_type(content_type.unwrap_or("application/octet-stream")) + .with_content_type(content_type.unwrap_or(FILE_DEFAULT_MEDIA_TYPE)) .with_etag(etag) .with_last_modified(last_modified) .with_accept_ranges(); @@ -145,8 +157,11 @@ impl FileGetRequestHandler for Server { } let contents = self - .blob_store() - .get_blob(hash, byte_range.clone().unwrap_or(0..usize::MAX)) + .fetch_blob( + &hash, + byte_range.clone().unwrap_or(0..usize::MAX), + Some(size as u64), + ) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/mkcol.rs b/crates/dav/src/file/mkcol.rs index e9d9249c..6c3adb5b 100644 --- a/crates/dav/src/file/mkcol.rs +++ b/crates/dav/src/file/mkcol.rs @@ -6,16 +6,20 @@ use super::proppatch::FilePropPatchRequestHandler; use crate::{ - DavMethod, PropStatBuilder, + DavError, DavMethod, PropStatBuilder, common::{ ExtractETag, - acl::ResourceAcl, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, - file::DavFileResource, + file::{DavFileResource, file_name_from_uri, is_symlink, validate_file_parent_acl}, +}; +use common::{ + Server, + auth::AccessToken, + storage::{dav::MAX_FILE_NODE_DEPTH, index::ObjectIndexBuilder}, }; -use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, @@ -23,11 +27,13 @@ use dav_proto::{ use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use store::write::{BatchBuilder, now}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + metadata::MetadataKinds, }; pub(crate) trait FileMkColRequestHandler: Sync + Send { @@ -53,22 +59,53 @@ impl FileMkColRequestHandler for Server { .into_owned_uri()?; let account_id = resource_.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; - let resource = resources.map_parent_resource(&resource_)?; + let path = resource_ + .resource + .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?; + if !access_token.is_member(account_id) { + resources.hide_undiscoverable_for( + access_token, + path, + StatusCode::FORBIDDEN, + StatusCode::CONFLICT, + )?; + } + if let Some(existing) = resources.by_path(path) { + return Err(DavError::Code(if is_symlink(&existing) { + StatusCode::CONFLICT + } else { + StatusCode::METHOD_NOT_ALLOWED + })); + } + let parent = resources.map_directory_parent(path)?; // Validate and map parent ACL - let parent_id = resources.validate_and_map_parent_acl( + let parent_id = validate_file_parent_acl( + &resources, access_token, access_token.is_member(account_id), - resource.resource.0, + parent.map(|parent| parent.document_id()), Acl::AddItems, )?; + let name = file_name_from_uri(headers.uri)?; + if path.split('/').count() > MAX_FILE_NODE_DEPTH { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + + // Validate quota + self.assert_object_quota( + &*self.account(account_id).await?, + StorageQuota::MaxFolders, + 1, + || resources.resources.count(true), + )?; // Validate headers self.validate_headers( @@ -76,9 +113,9 @@ impl FileMkColRequestHandler for Server { headers, vec![ResourceState { account_id, - collection: resource.collection, + collection: resource_.collection, document_id: Some(u32::MAX), - path: resource_.resource.unwrap(), + path, ..Default::default() }], Default::default(), @@ -90,20 +127,38 @@ impl FileMkColRequestHandler for Server { let now = now(); let mut node = FileNode { parent_id, - name: resource.resource.1.to_string(), - display_name: None, - file: None, + name, created: now as i64, modified: now as i64, - dead_properties: Default::default(), - acls: Default::default(), + accessed: now as i64, + changed: now as i64, + ..Default::default() }; // Apply MKCOL properties + let mut batch = BatchBuilder::new(); + let document_id = batch.reserve_document_id(account_id, Collection::FileNode); let mut return_prop_stat = None; - if let Some(mkcol) = request { + let mut dead_write = None; + if let Some(mut mkcol) = request { let mut prop_stat = PropStatBuilder::default(); - if !self.apply_file_properties(&mut node, false, mkcol.props, &mut prop_stat) { + let dead = DeadPatch::take_values(&mut mkcol.props, DisplayName::Stored); + self.apply_file_properties(&mut node, mkcol.props, &mut prop_stat); + dead_write = dead + .apply( + self, + DeadTarget::item( + account_id, + Collection::FileNode, + document_id, + MetadataKinds::NONE, + ), + &mut prop_stat, + ) + .await + .caused_by(trc::location!())?; + if prop_stat.has_errors() { + prop_stat.fail_dependencies(); return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::Dav) @@ -116,20 +171,25 @@ impl FileMkColRequestHandler for Server { } // Prepare write batch - let document_id = self - .store() - .assign_document_ids(account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; - let mut batch = BatchBuilder::new(); + if let Some(dead_write) = &dead_write { + node.set_presence(dead_write.file_presence); + } + if let Some(write) = dead_write.and_then(|dead_write| dead_write.write) { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .with_document(document_id) + .create_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(node)) .caused_by(trc::location!())?; - let etag = batch.etag(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let etag = self + .commit_batch(batch) + .await + .caused_by(trc::location!())? + .etag(); if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) diff --git a/crates/dav/src/file/mod.rs b/crates/dav/src/file/mod.rs index a5c0a5a8..58783f95 100644 --- a/crates/dav/src/file/mod.rs +++ b/crates/dav/src/file/mod.rs @@ -6,11 +6,24 @@ use crate::{ DavError, - common::uri::{OwnedUri, UriResource}, + common::{ + quoted_etag, + uri::{OwnedUri, UriResource}, + }, +}; +use common::{ + DavResourcePath, GroupwareResources, + auth::AccessToken, + storage::dav::{DavFileNameError, FILE_KIND_SYMLINK, dav_file_name}, }; -use common::{DavResourcePath, DavResources}; use dav_proto::schema::property::{DavProperty, WebDavProperty}; +use groupware::file::ArchivedFileNode; use hyper::StatusCode; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use types::{ + acl::Acl, + collection::{Collection, SyncCollection}, +}; pub mod copy_move; pub mod delete; @@ -81,14 +94,28 @@ pub(crate) trait DavFileResource { fn map_parent<'x>(&self, resource: &'x str) -> Option<(Option>, &'x str)>; - #[allow(clippy::type_complexity)] - fn map_parent_resource<'x, T: FromDavResource>( + fn map_directory_parent(&self, resource: &str) -> crate::Result>>; + + fn log_descendant_updates(&self, batch: &mut BatchBuilder, account_id: u32, path: &str); + + fn hide_undiscoverable( &self, - resource: &OwnedUri<'x>, - ) -> crate::Result, &'x str)>>; + discoverable: &RoaringBitmap, + path: &str, + hidden_target: StatusCode, + hidden_parent: StatusCode, + ) -> crate::Result<()>; + + fn hide_undiscoverable_for( + &self, + access_token: &AccessToken, + path: &str, + hidden_target: StatusCode, + hidden_parent: StatusCode, + ) -> crate::Result<()>; } -impl DavFileResource for DavResources { +impl DavFileResource for GroupwareResources { fn map_resource( &self, resource: &OwnedUri<'_>, @@ -96,6 +123,7 @@ impl DavFileResource for DavResources { resource .resource .and_then(|r| self.by_path(r)) + .filter(|r| !is_symlink(r)) .map(|r| UriResource { collection: resource.collection, account_id: resource.account_id, @@ -114,25 +142,118 @@ impl DavFileResource for DavResources { Some((parent, child)) } - fn map_parent_resource<'x, T: FromDavResource>( - &self, - resource: &OwnedUri<'x>, - ) -> crate::Result, &'x str)>> { - if let Some(r) = resource.resource { - if self.by_path(r).is_none() { - self.map_parent(r) - .map(|(parent, child)| UriResource { - collection: resource.collection, - account_id: resource.account_id, - resource: (parent.map(T::from_dav_resource), child), - }) - .ok_or(DavError::Code(StatusCode::CONFLICT)) - } else { - Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) - } - } else { - Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) + fn map_directory_parent(&self, resource: &str) -> crate::Result>> { + self.map_parent(resource) + .map(|(parent, _)| parent) + .filter(|parent| parent.as_ref().is_none_or(DavResourcePath::is_container)) + .ok_or(DavError::Code(StatusCode::CONFLICT)) + } + + fn log_descendant_updates(&self, batch: &mut BatchBuilder, account_id: u32, path: &str) { + let mut descendants = self + .subtree(path) + .filter(|descendant| descendant.path() != path) + .peekable(); + if descendants.peek().is_none() { + return; } + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode); + for descendant in descendants { + batch + .with_document(descendant.document_id()) + .log_item_update(SyncCollection::FileNode, None); + } + batch.commit_point(); + } + + fn hide_undiscoverable( + &self, + discoverable: &RoaringBitmap, + path: &str, + hidden_target: StatusCode, + hidden_parent: StatusCode, + ) -> crate::Result<()> { + hide_unless(self, path, hidden_target, hidden_parent, |document_id| { + discoverable.contains(document_id) + }) + } + + fn hide_undiscoverable_for( + &self, + access_token: &AccessToken, + path: &str, + hidden_target: StatusCode, + hidden_parent: StatusCode, + ) -> crate::Result<()> { + hide_unless(self, path, hidden_target, hidden_parent, |document_id| { + self.file_discoverable(access_token, document_id) + }) + } +} + +fn hide_unless( + resources: &GroupwareResources, + path: &str, + hidden_target: StatusCode, + hidden_parent: StatusCode, + is_discoverable: impl Fn(u32) -> bool, +) -> crate::Result<()> { + let mut current = Some(path); + let mut status = hidden_target; + while let Some(path) = current { + if let Some(resource) = resources.by_path(path) { + return if is_discoverable(resource.document_id()) { + Ok(()) + } else { + Err(DavError::Code(status)) + }; + } + status = hidden_parent; + current = path.rsplit_once('/').map(|(parent, _)| parent); + } + Ok(()) +} + +pub(crate) fn file_name_from_uri(uri: &str) -> crate::Result { + let segment = uri + .trim_end_matches('/') + .rsplit_once('/') + .map_or(uri, |(_, segment)| segment); + dav_file_name(segment).map_err(|err| { + DavError::Code(match err { + DavFileNameError::TooLong => StatusCode::URI_TOO_LONG, + DavFileNameError::Empty + | DavFileNameError::InvalidCharacter + | DavFileNameError::ReservedName => StatusCode::BAD_REQUEST, + }) + }) +} + +pub(crate) fn file_etag(node: &ArchivedFileNode) -> String { + quoted_etag(node.etag.to_native().into()) +} + +pub(crate) fn is_symlink(resource: &DavResourcePath<'_>) -> bool { + resource.resource.file_kind() == Some(FILE_KIND_SYMLINK) +} + +pub(crate) fn validate_file_parent_acl( + resources: &GroupwareResources, + access_token: &AccessToken, + is_member: bool, + parent_id: Option, + acl: Acl, +) -> crate::Result { + match parent_id { + Some(parent_id) + if is_member || resources.file_acl(access_token, parent_id).contains(acl) => + { + Ok(parent_id + 1) + } + None if is_member => Ok(0), + _ => Err(DavError::Code(StatusCode::FORBIDDEN)), } } diff --git a/crates/dav/src/file/proppatch.rs b/crates/dav/src/file/proppatch.rs index 8f76b646..6fd563a6 100644 --- a/crates/dav/src/file/proppatch.rs +++ b/crates/dav/src/file/proppatch.rs @@ -7,18 +7,20 @@ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ - ETag, ExtractETag, + ExtractETag, + dead::{DeadPatch, DeadTarget, DisplayName}, lock::{LockRequestHandler, ResourceState}, + propfind::requested_href, uri::DavUriResource, }, - file::DavFileResource, + file::{DavFileResource, FileItemId, file_etag}, }; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{ property::{DavProperty, DavValue, ResourceType, WebDavProperty}, - request::{DavPropertyValue, PropertyUpdate}, + request::{DavPropertyValue, PropertyUpdate, PropertyUpdateOp}, response::{BaseCondition, MultiStatus, Response}, }, }; @@ -28,7 +30,7 @@ use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -47,10 +49,9 @@ pub(crate) trait FilePropPatchRequestHandler: Sync + Send { fn apply_file_properties( &self, file: &mut FileNode, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool; + ); } impl FilePropPatchRequestHandler for Server { @@ -65,17 +66,29 @@ impl FilePropPatchRequestHandler for Server { .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; - let uri = headers.uri; + let uri = headers.raw_uri; let account_id = resource_.account_id; let files = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; - let resource = files.map_resource(&resource_)?; + if !access_token.is_member(account_id) + && let Some(path) = resource_.resource + { + files.hide_undiscoverable_for( + access_token, + path, + StatusCode::NOT_FOUND, + StatusCode::NOT_FOUND, + )?; + } + let resource = files.map_resource::(&resource_)?; + let document_id = resource.resource.document_id; + let href = requested_href(uri, resource.resource.is_container); if !request.has_changes() { return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); @@ -84,10 +97,10 @@ impl FilePropPatchRequestHandler for Server { // Fetch node let node_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::FileNode, - resource.resource, + document_id, )) .await .caused_by(trc::location!())? @@ -95,16 +108,27 @@ impl FilePropPatchRequestHandler for Server { let node = node_ .to_unarchived::() .caused_by(trc::location!())?; + let current_etag = file_etag(node.inner); // Validate ACL - if !access_token.is_member(account_id) - && !node - .inner - .acls - .effective_acl(access_token) - .contains(Acl::Modify) - { - return Err(DavError::Code(StatusCode::FORBIDDEN)); + if !access_token.is_member(account_id) { + let acl = files.file_acl(access_token, document_id); + let (mut content, mut properties) = (false, false); + for property in request.ops.iter().map(PropertyUpdateOp::property) { + if matches!( + property, + DavProperty::WebDav(WebDavProperty::GetContentType) + ) { + content = true; + } else { + properties = true; + } + } + if (content && !acl.contains(Acl::ModifyItems)) + || (properties && !acl.contains(Acl::Modify)) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } } // Validate headers @@ -114,8 +138,8 @@ impl FilePropPatchRequestHandler for Server { vec![ResourceState { account_id, collection: resource.collection, - document_id: resource.resource.into(), - etag: node_.etag().into(), + document_id: document_id.into(), + etag: current_etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], @@ -124,91 +148,112 @@ impl FilePropPatchRequestHandler for Server { ) .await?; - // Deserialize + // Apply live properties + let dead = DeadPatch::take(&mut request.ops, DisplayName::Stored); + let has_live_changes = !request.ops.is_empty(); let mut new_node = node.deserialize::().caused_by(trc::location!())?; - - // Remove properties let mut items = PropStatBuilder::default(); - if !request.set_first && !request.remove.is_empty() { - remove_file_properties( - &mut new_node, - std::mem::take(&mut request.remove), + for op in request.ops { + match op { + PropertyUpdateOp::Set(value) => { + self.apply_file_properties(&mut new_node, [value], &mut items) + } + PropertyUpdateOp::Remove(property) => { + remove_file_properties(&mut new_node, [property], &mut items) + } + } + } + + // Apply dead properties + let mut dead_write = dead + .apply( + self, + DeadTarget::item( + account_id, + Collection::FileNode, + document_id, + node.inner.metadata_kinds(), + ), &mut items, - ); - } - - // Set properties - let is_success = self.apply_file_properties(&mut new_node, true, request.set, &mut items); - - // Remove properties - if is_success && !request.remove.is_empty() { - remove_file_properties(&mut new_node, request.remove, &mut items); - } + ) + .await + .caused_by(trc::location!())?; + let is_success = !items.has_errors(); let etag = if is_success { let mut batch = BatchBuilder::new(); - let etag = new_node - .update( - access_token.account_tenant_ids(), - node, + if let Some(write) = dead_write + .as_mut() + .and_then(|dead_write| dead_write.write.take()) + { + write + .build(document_id.into(), &mut batch) + .caused_by(trc::location!())?; + } + if has_live_changes { + if let Some(dead_write) = &dead_write { + new_node.set_presence(dead_write.file_presence); + } + new_node + .update( + access_token.changed_by(), + node, + account_id, + document_id, + false, + &mut batch, + ) + .caused_by(trc::location!())?; + } else if let Some(dead_write) = &dead_write { + FileNode::rewrite_presence( + &node, + dead_write.file_presence, account_id, - resource.resource, - true, + document_id, &mut batch, ) - .caused_by(trc::location!())? - .etag(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + .caused_by(trc::location!())?; + } + let etag = batch.etag().unwrap_or(current_etag); + if !batch.is_empty() { + self.commit_batch(batch).await.caused_by(trc::location!())?; + } etag } else { - node_.etag().into() + items.fail_dependencies(); + current_etag }; if headers.ret != Return::Minimal || !is_success { Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body( - MultiStatus::new(vec![Response::new_propstat(uri, items.build())]).to_string(), + MultiStatus::new(vec![Response::new_propstat(href, items.build())]).to_string(), ) - .with_etag_opt(etag)) + .with_etag(etag)) } else { - Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) + Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag(etag)) } } fn apply_file_properties( &self, file: &mut FileNode, - is_update: bool, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, - ) -> bool { - let mut has_errors = false; - + ) { for property in properties { match (&property.property, property.value) { - (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { - if name.len() <= self.core.groupware.live_property_size { - file.display_name = Some(name); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - - has_errors = true; - } - } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { file.created = dt; items.insert_ok(property.property); } (DavProperty::WebDav(WebDavProperty::GetContentType), DavValue::String(name)) - if file.file.is_some() => + if file.file().is_some() => { if name.len() <= self.core.groupware.live_property_size { - file.file.as_mut().unwrap().media_type = Some(name); + if let Some(file) = file.file_mut() { + file.media_type = Some(name); + } items.insert_ok(property.property); } else { items.insert_error_with_description( @@ -216,45 +261,22 @@ impl FilePropPatchRequestHandler for Server { StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); - has_errors = true; } } ( DavProperty::WebDav(WebDavProperty::ResourceType), DavValue::ResourceTypes(types), - ) if file.file.is_none() => { + ) if file.is_directory() => { if types.0.len() != 1 || types.0.first() != Some(&ResourceType::Collection) { items.insert_precondition_failed( property.property, StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); - has_errors = true; } else { items.insert_ok(property.property); } } - (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) - if self.core.groupware.dead_property_size.is_some() => - { - if is_update { - file.dead_properties.remove_element(dead); - } - - if file.dead_properties.size() + values.size() + dead.size() - < self.core.groupware.dead_property_size.unwrap() - { - file.dead_properties.add_element(dead.clone(), values.0); - items.insert_ok(property.property); - } else { - items.insert_error_with_description( - property.property, - StatusCode::INSUFFICIENT_STORAGE, - "Property value is too long", - ); - has_errors = true; - } - } (_, DavValue::Null) => { items.insert_ok(property.property); } @@ -264,33 +286,24 @@ impl FilePropPatchRequestHandler for Server { StatusCode::CONFLICT, "Property cannot be modified", ); - has_errors = true; } } } - - !has_errors } } fn remove_file_properties( node: &mut FileNode, - properties: Vec, + properties: impl IntoIterator, items: &mut PropStatBuilder, ) { for property in properties { match &property { - DavProperty::WebDav(WebDavProperty::DisplayName) => { - node.display_name = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::WebDav(WebDavProperty::GetContentType) if node.file.is_some() => { - node.file.as_mut().unwrap().media_type = None; - items.insert_with_status(property, StatusCode::NO_CONTENT); - } - DavProperty::DeadProperty(dead) => { - node.dead_properties.remove_element(dead); - items.insert_with_status(property, StatusCode::NO_CONTENT); + DavProperty::WebDav(WebDavProperty::GetContentType) if node.file().is_some() => { + if let Some(file) = node.file_mut() { + file.media_type = None; + } + items.insert_ok(property); } _ => { items.insert_error_with_description( diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 85fffe97..f2a34a20 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -7,27 +7,29 @@ use crate::{ DavError, DavMethod, common::{ - ETag, ExtractETag, - acl::ResourceAcl, + ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, - file::DavFileResource, + file::{DavFileResource, file_etag, file_name_from_uri, is_symlink, validate_file_parent_acl}, }; use common::{ - Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, + Server, + auth::AccessToken, + storage::{dav::MAX_FILE_NODE_DEPTH, index::ObjectIndexBuilder}, }; use dav_proto::{RequestHeaders, Return, schema::property::Rfc1123DateTime}; use groupware::{ cache::GroupwareCache, - file::{FileNode, FileProperties}, + file::{FileNode, FileNodeContent, FileProperties}, }; use http_proto::HttpResponse; use hyper::StatusCode; +use registry::schema::enums::StorageQuota; use store::write::{BatchBuilder, now}; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -41,7 +43,7 @@ pub(crate) trait FileUpdateRequestHandler: Sync + Send { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], is_patch: bool, ) -> impl Future> + Send; } @@ -51,7 +53,7 @@ impl FileUpdateRequestHandler for Server { &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, - bytes: Vec, + bytes: &[u8], _is_patch: bool, ) -> crate::Result { // Validate URI @@ -61,7 +63,7 @@ impl FileUpdateRequestHandler for Server { .into_owned_uri()?; let account_id = resource.account_id; let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, @@ -76,14 +78,24 @@ impl FileUpdateRequestHandler for Server { return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); } - if let Some(document_id) = resources - .by_path(resource_name.as_ref()) - .map(|r| r.document_id()) - { + if !access_token.is_member(account_id) { + resources.hide_undiscoverable_for( + access_token, + resource_name.as_ref(), + StatusCode::FORBIDDEN, + StatusCode::CONFLICT, + )?; + } + + if let Some(existing) = resources.by_path(resource_name.as_ref()) { + if is_symlink(&existing) { + return Err(DavError::Code(StatusCode::CONFLICT)); + } + let document_id = existing.document_id(); // Update let node_ = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::FileNode, document_id, @@ -97,11 +109,9 @@ impl FileUpdateRequestHandler for Server { // Validate ACL if !access_token.is_member(account_id) - && !node - .inner - .acls - .effective_acl(access_token) - .contains(Acl::Modify) + && !resources + .file_acl(access_token, document_id) + .contains(Acl::ModifyItems) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } @@ -115,7 +125,7 @@ impl FileUpdateRequestHandler for Server { account_id, collection: resource.collection, document_id: Some(document_id), - etag: node.etag().into(), + etag: file_etag(node.inner).into(), path: resource_name, ..Default::default() }], @@ -128,10 +138,16 @@ impl FileUpdateRequestHandler for Server { Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { - let file = node.inner.file.as_ref().unwrap(); + let file = node + .inner + .file() + .ok_or(DavError::Code(StatusCode::PRECONDITION_FAILED))?; let contents = self - .blob_store() - .get_blob(file.blob_hash.0.as_slice(), 0..usize::MAX) + .fetch_blob( + &BlobHash::from(&file.blob_hash), + 0..usize::MAX, + Some(u64::from(file.size.to_native())), + ) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::PRECONDITION_FAILED))?; @@ -143,7 +159,7 @@ impl FileUpdateRequestHandler for Server { .map(|v| v.as_str()) .unwrap_or("application/octet-stream"), ) - .with_etag(node.etag()) + .with_etag(file_etag(node.inner)) .with_last_modified( Rfc1123DateTime::new(i64::from(node.inner.modified)).to_string(), ) @@ -154,51 +170,48 @@ impl FileUpdateRequestHandler for Server { } // Verify that the node is a file - if let Some(file) = node.inner.file.as_ref() { - if BlobHash::generate(&bytes).as_slice() == file.blob_hash.0.as_slice() { - return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); - } - } else { + let Some(file) = node.inner.file() else { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); + }; + let blob_hash = BlobHash::generate(bytes); + if blob_hash.as_slice() == file.blob_hash.0.as_slice() { + return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } - // Validate quota - let extra_bytes = (bytes.len() as u64) - .saturating_sub(u32::from(node.inner.file.as_ref().unwrap().size) as u64); - if extra_bytes > 0 { - self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) - .await?; - } + let account = self.account(account_id).await?; // Write blob let (blob_hash, blob_hold) = self - .put_temporary_blob(account_id, &bytes, 60) + .hold_blob(account_id, blob_hash, bytes, 60) .await .caused_by(trc::location!())?; // Build node let mut new_node = node.deserialize::().caused_by(trc::location!())?; - let new_file = new_node.file.as_mut().unwrap(); - new_file.blob_hash = blob_hash; - new_file.media_type = headers - .content_type - .filter(|ct| !ct.is_empty() && *ct != "application/octet-stream") - .map(|v| v.to_string()); - new_file.size = bytes.len() as u32; - new_node.modified = now() as i64; + if let Some(new_file) = new_node.file_mut() { + new_file.blob_hash = blob_hash; + new_file.media_type = headers + .content_type + .filter(|ct| !ct.is_empty() && *ct != "application/octet-stream") + .map(|v| v.to_string()); + new_file.size = bytes.len() as u32; + } + new_node.stamp_update(true); // Prepare write batch let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); batch - .with_account_id(account_id) .with_collection(Collection::FileNode) - .with_document(document_id) - .clear(blob_hold) + .with_document(document_id); + blob_hold + .release(&mut batch) .custom( ObjectIndexBuilder::new() .with_current(node) .with_changes(new_node) - .with_changed_by(access_token.account_tenant_ids()), + .with_changed_by(access_token.changed_by()), ) .caused_by(trc::location!())?; let etag = batch.etag(); @@ -208,23 +221,21 @@ impl FileUpdateRequestHandler for Server { } else { // Insert let orig_resource_name = resource_name; - let (parent, resource_name) = resources - .map_parent(orig_resource_name.as_ref()) - .ok_or(DavError::Code(StatusCode::CONFLICT))?; + let parent = resources.map_directory_parent(orig_resource_name)?; + let name = file_name_from_uri(headers.uri)?; + if orig_resource_name.split('/').count() > MAX_FILE_NODE_DEPTH { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } // Validate ACL - let parent_id = resources.validate_and_map_parent_acl( + let parent_id = validate_file_parent_acl( + &resources, access_token, access_token.is_member(account_id), - parent.map(|r| r.document_id()), + parent.map(|parent| parent.document_id()), Acl::AddItems, )?; - // Verify that parent is a collection - if parent.as_ref().is_some_and(|r| !r.is_container()) { - return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); - } - // Validate headers self.validate_headers( access_token, @@ -241,18 +252,16 @@ impl FileUpdateRequestHandler for Server { ) .await?; - // Validate quota - if !bytes.is_empty() { - self.has_available_quota( - self.account(account_id).await?.as_ref(), - bytes.len() as u64, - ) - .await?; - } + // Validate object quota + let account = self.account(account_id).await?; + self.assert_object_quota(&account, StorageQuota::MaxFiles, 1, || { + resources.resources.count(false) + })?; + // Write blob let (blob_hash, blob_hold) = self - .put_temporary_blob(account_id, &bytes, 60) + .hold_blob(account_id, BlobHash::generate(bytes), bytes, 60) .await .caused_by(trc::location!())?; @@ -260,9 +269,8 @@ impl FileUpdateRequestHandler for Server { let now = now(); let node = FileNode { parent_id, - name: resource_name.to_string(), - display_name: None, - file: Some(FileProperties { + name, + content: FileNodeContent::File(FileProperties { blob_hash, size: bytes.len() as u32, media_type: headers.content_type.map(|v| v.to_string()), @@ -270,34 +278,32 @@ impl FileUpdateRequestHandler for Server { }), created: now as i64, modified: now as i64, - dead_properties: Default::default(), - acls: parent - .as_ref() - .and_then(|p| p.resource.acls()) - .map(|acls| acls.to_vec()) - .unwrap_or_default(), + accessed: now as i64, + changed: now as i64, + ..Default::default() }; // Prepare write batch let mut batch = BatchBuilder::new(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; + let document_id = batch.reserve_document_id(account_id, Collection::FileNode); + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); batch - .with_account_id(account_id) .with_collection(Collection::FileNode) - .with_document(document_id) - .clear(blob_hold) + .create_document(document_id); + blob_hold + .release(&mut batch) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) - .with_changed_by(access_token.account_tenant_ids()), + .with_changed_by(access_token.changed_by()), ) .caused_by(trc::location!())?; - let etag = batch.etag(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let etag = self + .commit_batch(batch) + .await + .caused_by(trc::location!())? + .etag(); Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } diff --git a/crates/dav/src/lib.rs b/crates/dav/src/lib.rs index dc4e6d3a..0983c313 100644 --- a/crates/dav/src/lib.rs +++ b/crates/dav/src/lib.rs @@ -16,9 +16,8 @@ use dav_proto::schema::{ request::DavPropertyValue, response::{Condition, List, Prop, PropStat, ResponseDescription, Status}, }; -use groupware::{DavResourceName, RFC_3986, is_uri_segment}; +use groupware::DavResourceName; use hyper::{Method, StatusCode}; -use std::borrow::Cow; use store::ahash::AHashMap; pub(crate) type Result = std::result::Result; @@ -121,24 +120,41 @@ impl DavMethod { Method::POST => Some(DavMethod::POST), Method::PATCH => Some(DavMethod::PATCH), Method::HEAD => Some(DavMethod::HEAD), - _ => { - hashify::tiny_map!(method.as_str().as_bytes(), - "PROPFIND" => DavMethod::PROPFIND, - "PROPPATCH" => DavMethod::PROPPATCH, - "REPORT" => DavMethod::REPORT, - "MKCOL" => DavMethod::MKCOL, - "MKCALENDAR" => DavMethod::MKCALENDAR, - "COPY" => DavMethod::COPY, - "MOVE" => DavMethod::MOVE, - "LOCK" => DavMethod::LOCK, - "UNLOCK" => DavMethod::UNLOCK, - "ACL" => DavMethod::ACL - ) - } + _ => hashify::map!(method.as_str().as_bytes(), DavMethod, + "PROPFIND" => DavMethod::PROPFIND, + "PROPPATCH" => DavMethod::PROPPATCH, + "REPORT" => DavMethod::REPORT, + "MKCOL" => DavMethod::MKCOL, + "MKCALENDAR" => DavMethod::MKCALENDAR, + "COPY" => DavMethod::COPY, + "MOVE" => DavMethod::MOVE, + "LOCK" => DavMethod::LOCK, + "UNLOCK" => DavMethod::UNLOCK, + "ACL" => DavMethod::ACL + ) + .copied(), } } #[inline] + pub fn is_write(self) -> bool { + matches!( + self, + DavMethod::PUT + | DavMethod::POST + | DavMethod::PATCH + | DavMethod::DELETE + | DavMethod::PROPPATCH + | DavMethod::MKCOL + | DavMethod::MKCALENDAR + | DavMethod::COPY + | DavMethod::MOVE + | DavMethod::LOCK + | DavMethod::UNLOCK + | DavMethod::ACL + ) + } + pub fn has_body(self) -> bool { matches!( self, @@ -221,6 +237,33 @@ impl PropStatBuilder { self } + pub fn has_errors(&self) -> bool { + self.propstats + .keys() + .any(|(status, _, _)| !status.is_success()) + } + + pub fn fail_dependencies(&mut self) { + let succeeded = self + .propstats + .keys() + .filter(|(status, _, _)| status.is_success()) + .cloned() + .collect::>(); + let mut failed = Vec::new(); + for key in succeeded { + if let Some(props) = self.propstats.remove(&key) { + failed.extend(props); + } + } + if !failed.is_empty() { + self.propstats + .entry((StatusCode::FAILED_DEPENDENCY, None, None)) + .or_default() + .extend(failed); + } + } + pub fn build(self) -> Vec { self.propstats .into_iter() @@ -234,23 +277,3 @@ impl PropStatBuilder { } } -// Workaround for Apple bug with missing percent encoding in paths -pub(crate) fn fix_percent_encoding(path: &'_ str) -> Cow<'_, str> { - let (parent, name) = if let Some((parent, name)) = path.rsplit_once('/') { - (Some(parent), name) - } else { - (None, path) - }; - - if is_uri_segment(name) { - path.into() - } else { - let name = percent_encoding::utf8_percent_encode(name, RFC_3986); - - if let Some(parent) = parent { - Cow::Owned(format!("{parent}/{name}")) - } else { - Cow::Owned(name.to_string()) - } - } -} diff --git a/crates/dav/src/principal/propfind.rs b/crates/dav/src/principal/propfind.rs index ba43d1f6..49e8bf24 100644 --- a/crates/dav/src/principal/propfind.rs +++ b/crates/dav/src/principal/propfind.rs @@ -7,7 +7,10 @@ use super::CurrentUserPrincipal; use crate::{ DavResourceName, - common::propfind::{PropFindRequestHandler, SyncTokenUrn}, + common::{ + propfind::{PropFindRequestHandler, SyncTokenUrn}, + quoted_etag, + }, }; use common::{ Server, @@ -25,6 +28,7 @@ use dav_proto::schema::{ use groupware::RFC_3986; use groupware::cache::GroupwareCache; use hyper::StatusCode; +use registry::schema::enums::Permission; use std::borrow::Cow; use trc::AddContext; use types::collection::Collection; @@ -118,7 +122,19 @@ impl PrincipalPropFind for Server { let mut fields = Vec::with_capacity(properties.len()); let mut fields_not_found = Vec::new(); - let account = self.account(account_id).await.caused_by(trc::location!())?; + let Some(account) = self + .try_account(account_id) + .await + .caused_by(trc::location!())? + else { + continue; + }; + if !account.is_same_tenant(access_token) + && !access_token.is_member(account_id) + && !access_token.has_permission(Permission::SysAccountQuery) + { + continue; + } // Fetch quota let quota = if needs_quota { @@ -179,7 +195,7 @@ impl PrincipalPropFind for Server { } WebDavProperty::SyncToken if !is_principal => { let sync_token = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, collection.into(), @@ -192,7 +208,7 @@ impl PrincipalPropFind for Server { } WebDavProperty::GetCTag if !is_principal => { let ctag = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, collection.into(), @@ -203,7 +219,7 @@ impl PrincipalPropFind for Server { fields.push(DavPropertyValue::new( property.clone(), - DavValue::String(format!("\"{ctag}\"")), + DavValue::String(quoted_etag(ctag)), )); } WebDavProperty::Owner => { @@ -379,13 +395,21 @@ impl PrincipalPropFind for Server { async fn owner_href(&self, account_info: &AccountCache, account_id: u32) -> trc::Result { if account_info.account_id() == account_id { Ok(account_info.current_user_principal()) - } else { - let account_info = self.account(account_id).await.caused_by(trc::location!())?; + } else if let Some(account_info) = self + .try_account(account_id) + .await + .caused_by(trc::location!())? + { Ok(Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(account_info.name(), RFC_3986), ))) + } else { + Ok(Href(format!( + "{}/_{account_id}/", + DavResourceName::Principal.base_path(), + ))) } } } diff --git a/crates/dav/src/principal/propsearch.rs b/crates/dav/src/principal/propsearch.rs index def9d590..1e6b50bb 100644 --- a/crates/dav/src/principal/propsearch.rs +++ b/crates/dav/src/principal/propsearch.rs @@ -7,13 +7,14 @@ use super::propfind::PrincipalPropFind; use common::{Server, auth::AccessToken}; use dav_proto::schema::{ - property::{DavProperty, WebDavProperty}, - request::{PrincipalPropertySearch, PropFind}, + property::{DavProperty, PrincipalProperty, WebDavProperty}, + request::{FilterTest, PrincipalPropertySearch, PropFind}, response::MultiStatus, }; +use groupware::strip_mailto_scheme; use http_proto::HttpResponse; use hyper::StatusCode; -use registry::schema::prelude::{ObjectType, Property}; +use registry::schema::prelude::{ObjectType, Permission, Property}; use store::{registry::RegistryQuery, roaring::RoaringBitmap}; use trc::AddContext; use types::collection::Collection; @@ -24,6 +25,13 @@ pub(crate) trait PrincipalPropSearch: Sync + Send { access_token: &AccessToken, request: PrincipalPropertySearch, ) -> impl Future> + Send; + + fn principal_property_matches( + &self, + access_token: &AccessToken, + property: &DavProperty, + text: &str, + ) -> impl Future> + Send; } impl PrincipalPropSearch for Server { @@ -32,29 +40,32 @@ impl PrincipalPropSearch for Server { access_token: &AccessToken, mut request: PrincipalPropertySearch, ) -> crate::Result { - let mut search_for = None; - - for prop_search in request.property_search { - if matches!( - prop_search.property, - DavProperty::WebDav(WebDavProperty::DisplayName) - ) && !prop_search.match_.is_empty() - { - search_for = Some(prop_search.match_); + let mut matches: Option = None; + 'outer: for search in &request.property_search { + for property in &search.properties { + if request.test == FilterTest::AllOf + && matches.as_ref().is_some_and(RoaringBitmap::is_empty) + { + break 'outer; + } + let ids = self + .principal_property_matches(access_token, property, &search.match_) + .await?; + matches = Some(match (matches, request.test) { + (None, _) => ids, + (Some(current), FilterTest::AllOf) => current & ids, + (Some(current), FilterTest::AnyOf) => current | ids, + }); } } let mut response = MultiStatus::new(Vec::with_capacity(16)); - if let Some(search_for) = search_for { - let ids = self - .registry() - .query::( - RegistryQuery::new(ObjectType::Account) - .with_tenant(access_token.tenant_id()) - .text(Property::Text, search_for), - ) - .await - .caused_by(trc::location!())?; + if let Some(mut ids) = matches { + if !self.core.groupware.allow_directory_query + && !access_token.has_permission(Permission::SysAccountQuery) + { + ids &= RoaringBitmap::from_iter(access_token.all_ids()); + } if !ids.is_empty() { if request.properties.is_empty() { @@ -76,4 +87,47 @@ impl PrincipalPropSearch for Server { Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + async fn principal_property_matches( + &self, + access_token: &AccessToken, + property: &DavProperty, + text: &str, + ) -> crate::Result { + let text = match property { + DavProperty::WebDav(WebDavProperty::DisplayName) => text.trim(), + DavProperty::Principal(PrincipalProperty::CalendarUserAddressSet) => { + let address = strip_mailto_scheme(text.trim()); + if address.contains('@') { + let mut matches = RoaringBitmap::new(); + if let Some(account_id) = self + .account_id_from_email(address, false) + .await + .caused_by(trc::location!())? + { + #[cfg(not(feature = "enterprise"))] + let is_allowed = true; + if is_allowed { + matches.insert(account_id); + } + } + return Ok(matches); + } + address + } + _ => return Ok(RoaringBitmap::new()), + }; + + if text.is_empty() { + return Ok(RoaringBitmap::new()); + } + + let query = RegistryQuery::new(ObjectType::Account); + self.registry() + .query::(query.text(Property::Text, text)) + .await + .caused_by(trc::location!()) + .map_err(Into::into) + } } diff --git a/crates/dav/src/request.rs b/crates/dav/src/request.rs index 2aaa3cbb..3c6154a1 100644 --- a/crates/dav/src/request.rs +++ b/crates/dav/src/request.rs @@ -24,7 +24,7 @@ use crate::{ acl::DavAclHandler, lock::{LockRequest, LockRequestHandler}, propfind::PropFindRequestHandler, - uri::DavUriResource, + uri::{DavUriResource, UriResource, canonical_dav_uri}, }, file::{ copy_move::FileCopyMoveRequestHandler, delete::FileDeleteRequestHandler, @@ -33,26 +33,38 @@ use crate::{ }, principal::{matching::PrincipalMatching, propsearch::PrincipalPropSearch}, }; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, network::memory::MEMORY_RETRY_AFTER}; use compact_str::{CompactString, ToCompactString}; use dav_proto::{ RequestHeaders, - parser::{DavParser, tokenizer::Tokenizer}, + parser::{DavParser, FilterError, header::strip_query_and_fragment, tokenizer::Tokenizer}, schema::{ Namespace, - property::WebDavProperty, + property::{DavProperty, PrincipalProperty, WebDavProperty}, request::{Acl, LockInfo, MkCol, PropFind, PropertyUpdate, Report}, response::{ - BaseCondition, ErrorResponse, List, PrincipalSearchProperty, PrincipalSearchPropertySet, + BaseCondition, CalCondition, CardCondition, ErrorResponse, PrincipalSearchProperty, + PrincipalSearchPropertySet, }, }, }; use http_proto::{HttpRequest, HttpResponse, HttpSessionData, request::fetch_body}; use hyper::{StatusCode, header}; use registry::schema::enums::Permission; -use std::time::Instant; +use std::{ + ops::RangeInclusive, + time::{Duration, Instant}, +}; +use store::{ + rand::{self, RngExt}, + write::unavailable_retry_after, +}; use trc::{EventType, LimitEvent, StoreEvent, WebDavEvent}; -use types::collection::Collection; +use types::collection::{Collection, SyncCollection}; + +const MAX_CONFLICT_RETRIES: u32 = 3; +const CONFLICT_RETRY_BACKOFF_MS: RangeInclusive = 1..=15; +const CONFLICT_RETRY_AFTER: u64 = 1; pub trait DavRequestHandler: Sync + Send { fn handle_dav_request( @@ -72,7 +84,7 @@ pub(crate) trait DavRequestDispatcher: Sync + Send { access_token: AccessToken, resource: DavResourceName, method: DavMethod, - body: Vec, + body: &[u8], ) -> impl Future> + Send; } @@ -83,12 +95,12 @@ impl DavRequestDispatcher for Server { access_token: AccessToken, resource: DavResourceName, method: DavMethod, - body: Vec, + body: &[u8], ) -> crate::Result { // Dispatch match method { DavMethod::PROPFIND => { - let request = PropFind::parse(&mut Tokenizer::new(&body))?; + let request = PropFind::parse(&mut Tokenizer::new(body))?; self.handle_propfind_request(&access_token, headers, request) .await @@ -150,7 +162,7 @@ impl DavRequestDispatcher for Server { } DavResourceName::Principal => Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), }, - DavMethod::REPORT => match Report::parse(&mut Tokenizer::new(&body))? { + DavMethod::REPORT => match Report::parse(&mut Tokenizer::new(body))? { Report::SyncCollection(sync_collection) => { // Validate permissions let access_token = @@ -178,17 +190,6 @@ impl DavRequestDispatcher for Server { } Report::AclPrincipalPropSet(report) => { // Validate permissions - if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::DavPrincipalAcl) - { - return Err(DavError::Condition( - DavErrorCondition::new( - StatusCode::FORBIDDEN, - BaseCondition::NeedPrivileges(List(Default::default())), - ) - .with_details("The administrator has disabled directory queries."), - )); - } let access_token = access_token.assert_has_permission(Permission::DavPrincipalAcl)?; @@ -197,17 +198,6 @@ impl DavRequestDispatcher for Server { } Report::PrincipalMatch(report) => { // Validate permissions - if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::DavPrincipalMatch) - { - return Err(DavError::Condition( - DavErrorCondition::new( - StatusCode::FORBIDDEN, - BaseCondition::NeedPrivileges(List(Default::default())), - ) - .with_details("The administrator has disabled directory queries."), - )); - } let access_token = access_token.assert_has_permission(Permission::DavPrincipalMatch)?; @@ -217,18 +207,6 @@ impl DavRequestDispatcher for Server { Report::PrincipalPropertySearch(report) => { if resource == DavResourceName::Principal { // Validate permissions - if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::DavPrincipalSearch) - { - return Err(DavError::Condition( - DavErrorCondition::new( - StatusCode::FORBIDDEN, - BaseCondition::NeedPrivileges(List(Default::default())), - ) - .with_details("The administrator has disabled directory queries."), - )); - } - let access_token = access_token.assert_has_permission(Permission::DavPrincipalSearch)?; @@ -245,10 +223,19 @@ impl DavRequestDispatcher for Server { .assert_has_permission(Permission::DavPrincipalSearchPropSet)?; Ok(HttpResponse::new(StatusCode::OK).with_xml_body( - PrincipalSearchPropertySet::new(vec![PrincipalSearchProperty::new( - WebDavProperty::DisplayName, - "Account or Group name", - )]) + PrincipalSearchPropertySet::new(vec![ + PrincipalSearchProperty::new( + WebDavProperty::DisplayName, + "Account or Group name", + ), + PrincipalSearchProperty::new( + DavProperty::Principal( + PrincipalProperty::CalendarUserAddressSet, + ), + "Calendar user address", + ), + ]) + .with_namespace(Namespace::CalDav) .to_string(), )) } else { @@ -289,7 +276,15 @@ impl DavRequestDispatcher for Server { self.handle_dav_query( &access_token, - DavQuery::multiget(report, Collection::Calendar, headers), + DavQuery::multiget( + report, + if resource == DavResourceName::Scheduling { + Collection::CalendarEventNotification + } else { + Collection::Calendar + }, + headers, + ), ) .await } @@ -326,7 +321,7 @@ impl DavRequestDispatcher for Server { } }, DavMethod::PROPPATCH => { - let request = PropertyUpdate::parse(&mut Tokenizer::new(&body))?; + let request = PropertyUpdate::parse(&mut Tokenizer::new(body))?; match resource { DavResourceName::Card => { // Validate permissions @@ -359,7 +354,7 @@ impl DavRequestDispatcher for Server { } DavMethod::MKCOL => { let request = if !body.is_empty() { - Some(MkCol::parse(&mut Tokenizer::new(&body))?) + Some(MkCol::parse(&mut Tokenizer::new(body))?) } else { None }; @@ -527,7 +522,7 @@ impl DavRequestDispatcher for Server { self.handle_calendar_mkcol_request( &access_token, headers, - Some(MkCol::parse(&mut Tokenizer::new(&body))?), + Some(MkCol::parse(&mut Tokenizer::new(body))?), ) .await } @@ -546,7 +541,7 @@ impl DavRequestDispatcher for Server { &access_token, headers, if !body.is_empty() { - LockRequest::Lock(LockInfo::parse(&mut Tokenizer::new(&body))?) + LockRequest::Lock(LockInfo::parse(&mut Tokenizer::new(body))?) } else { LockRequest::Refresh }, @@ -577,7 +572,7 @@ impl DavRequestDispatcher for Server { self.handle_acl_request( &access_token, headers, - Acl::parse(&mut Tokenizer::new(&body))?, + Acl::parse(&mut Tokenizer::new(body))?, ) .await } @@ -629,16 +624,63 @@ impl DavRequestHandler for Server { }; // Parse headers - let mut headers = RequestHeaders::new(request.uri().path()); + let mut destinations = request.headers().get_all("destination").iter(); + let destination = destinations.next().and_then(|value| value.to_str().ok()); + if destinations.next().is_some() { + return HttpResponse::new(StatusCode::BAD_REQUEST); + } + let raw_uri = request.uri().path(); + let (uri, canonical_destination) = match ( + canonical_dav_uri(raw_uri), + destination + .map(|value| canonical_dav_uri(strip_query_and_fragment(value))) + .transpose(), + ) { + (Ok(uri), Ok(destination)) => (uri, destination), + _ => return HttpResponse::new(StatusCode::BAD_REQUEST), + }; + let mut headers = RequestHeaders::new(&uri); + headers.raw_uri = raw_uri; for (key, value) in request.headers() { headers.parse(key.as_str(), value.to_str().unwrap_or_default()); } + if let Some(destination) = canonical_destination.as_deref() { + headers.destination = Some(destination); + } let start_time = Instant::now(); - match self - .dispatch_dav_request(&headers, access_token, resource, method, body) - .await - { + let mut retries = 0; + let result = loop { + match self + .dispatch_dav_request(&headers, access_token.clone(), resource, method, &body) + .await + { + Err(DavError::Internal(err)) + if err.is_assertion_failure() + && method.is_write() + && retries < MAX_CONFLICT_RETRIES => + { + retries += 1; + let collection = SyncCollection::from(Collection::from(resource)); + for uri in [Some(headers.uri), headers.destination] + .into_iter() + .flatten() + { + if let Ok(UriResource { + account_id: Some(account_id), + .. + }) = self.validate_uri(&access_token, uri).await + { + self.inner.mark_cache_stale(account_id, collection); + } + } + let backoff = rand::rng().random_range(CONFLICT_RETRY_BACKOFF_MS); + tokio::time::sleep(Duration::from_millis(backoff)).await; + } + result => break result, + } + }; + match result { Ok(response) => { let event = WebDavEvent::from(method); @@ -647,7 +689,7 @@ impl DavRequestHandler for Server { SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), - Details = &headers, + Headers = &headers, Result = response.status().as_u16(), Elapsed = start_time.elapsed(), ); @@ -656,6 +698,7 @@ impl DavRequestHandler for Server { } Err(DavError::Internal(err)) => { let err_type = err.event_type(); + let retry_after = unavailable_retry_after(&err); trc::error!( err.span_id(session.session_id) @@ -664,26 +707,36 @@ impl DavRequestHandler for Server { .ctx(trc::Key::Elapsed, start_time.elapsed()) ); + #[cfg(not(feature = "enterprise"))] + let is_over_quota = matches!(err_type, EventType::Limit(LimitEvent::Quota)); + match err_type { - EventType::Limit(LimitEvent::Quota | LimitEvent::TenantQuota) => { - HttpResponse::new(StatusCode::PRECONDITION_FAILED) - .with_xml_body( - ErrorResponse::new(BaseCondition::QuotaNotExceeded) - .with_namespace(match resource { - DavResourceName::Card => Namespace::CardDav, - DavResourceName::Cal | DavResourceName::Scheduling => { - Namespace::CalDav - } - DavResourceName::File | DavResourceName::Principal => { - Namespace::Dav - } - }) - .to_string(), - ) - .with_no_cache() - } + _ if is_over_quota => HttpResponse::new(StatusCode::INSUFFICIENT_STORAGE) + .with_xml_body( + ErrorResponse::new(BaseCondition::QuotaNotExceeded) + .with_namespace(match resource { + DavResourceName::Card => Namespace::CardDav, + DavResourceName::Cal | DavResourceName::Scheduling => { + Namespace::CalDav + } + DavResourceName::File | DavResourceName::Principal => { + Namespace::Dav + } + }) + .to_string(), + ) + .with_no_cache(), EventType::Store(StoreEvent::AssertValueFailed) => { - HttpResponse::new(StatusCode::CONFLICT) + HttpResponse::new(StatusCode::SERVICE_UNAVAILABLE) + .with_header("Retry-After", CONFLICT_RETRY_AFTER.to_string()) + } + _ if let Some(retry_after) = retry_after => { + HttpResponse::new(StatusCode::SERVICE_UNAVAILABLE) + .with_header("Retry-After", retry_after.to_string()) + } + EventType::Limit(LimitEvent::MemoryBudget) => { + HttpResponse::new(StatusCode::SERVICE_UNAVAILABLE) + .with_header("Retry-After", MEMORY_RETRY_AFTER.to_string()) } EventType::Security(_) => HttpResponse::new(StatusCode::FORBIDDEN), _ => HttpResponse::new(StatusCode::INTERNAL_SERVER_ERROR), @@ -701,7 +754,7 @@ impl DavRequestHandler for Server { SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), - Details = &headers, + Headers = &headers, Result = result.as_u16(), Reason = err.to_compact_string(), Elapsed = start_time.elapsed(), @@ -717,7 +770,7 @@ impl DavRequestHandler for Server { SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), - Details = &headers, + Headers = &headers, Code = condition.code.as_u16(), Result = CompactString::const_new(condition.condition.display_name()), Reason = condition.details, @@ -748,7 +801,7 @@ impl DavRequestHandler for Server { SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), - Details = &headers, + Headers = &headers, Result = code.as_u16(), Elapsed = start_time.elapsed(), ); @@ -761,7 +814,33 @@ impl DavRequestHandler for Server { impl From for DavError { fn from(err: dav_proto::parser::Error) -> Self { - DavError::Parse(err) + match err { + dav_proto::parser::Error::Filter(FilterError::Invalid(Namespace::CalDav)) => { + DavErrorCondition::new(StatusCode::FORBIDDEN, CalCondition::ValidFilter).into() + } + dav_proto::parser::Error::Filter(FilterError::UnsupportedCollation( + Namespace::CalDav, + collation, + )) => DavErrorCondition::new( + StatusCode::FORBIDDEN, + CalCondition::SupportedCollation(collation), + ) + .into(), + dav_proto::parser::Error::Filter(FilterError::UnsupportedCollation(_, collation)) => { + DavErrorCondition::new( + StatusCode::FORBIDDEN, + CardCondition::SupportedCollation(collation), + ) + .into() + } + dav_proto::parser::Error::Filter(FilterError::TooComplex(Namespace::CalDav)) => { + DavErrorCondition::new(StatusCode::FORBIDDEN, CalCondition::SupportedFilter).into() + } + dav_proto::parser::Error::Filter(FilterError::TooComplex(_)) => { + DavErrorCondition::new(StatusCode::FORBIDDEN, CardCondition::SupportedFilter).into() + } + err => DavError::Parse(err), + } } } diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 2f6e6781..0c7cafb0 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "directory" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -8,9 +8,7 @@ utils = { path = "../utils" } store = { path = "../store" } trc = { path = "../trc" } registry = { path = "../registry" } -mail-parser = { version = "0.11" } -mail-builder = { version = "1.0" } -tokio = { version = "1.53", features = ["net"] } +tokio = { version = "1.54", features = ["net", "rt", "sync", "time"] } ldap3 = { version = "0.12", default-features = false, features = ["tls-rustls-aws-lc-rs"] } deadpool = { version = "0.13", features = ["managed", "rt_tokio_1"] } ahash = { version = "0.8" } @@ -25,16 +23,19 @@ serde = { version = "1.0", features = ["derive"]} totp-rs = { version = "6.0.0", features = ["otpauth"] } reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"] } serde_json = "1.0" -base64 = "0.23" +encodify = "1.0" nohash-hasher = "0.2.0" -jsonwebtoken = { version = "11.0.0", features = ["aws_lc_rs"] } +jsonwebtoken = { version = "11.1.0", features = ["aws_lc_rs"] } +compact_str = "0.10.0" +hashify = "0.3" +form_urlencoded = "1.2.2" +parking_lot = "0.13.0" [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } [features] -test_mode = [] -enterprise = [] +enterprise = ["utils/enterprise", "store/enterprise", "trc/enterprise", "registry/enterprise"] mysql = [] postgres = [] sqlite = [] diff --git a/crates/directory/src/backend/ldap/config.rs b/crates/directory/src/backend/ldap/config.rs index e26c1a60..b4309316 100644 --- a/crates/directory/src/backend/ldap/config.rs +++ b/crates/directory/src/backend/ldap/config.rs @@ -4,11 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Bind, LdapConnectionManager, LdapDirectory, LdapFilter, LdapFilterItem, LdapMappings}; +use super::{ + Bind, LdapConnectionManager, LdapDirectory, LdapFilter, LdapFilterItem, LdapMappings, + failover::LdapServers, +}; use crate::Directory; use deadpool::{Runtime, managed::Pool}; use ldap3::LdapConnSettings; use registry::schema::structs; +use std::sync::Arc; impl LdapDirectory { pub async fn open(config: structs::LdapDirectory) -> Result { @@ -29,19 +33,23 @@ impl LdapDirectory { None }; - let manager = LdapConnectionManager::new( - config.url, - LdapConnSettings::new() - .set_conn_timeout(config.timeout.into_inner()) - .set_starttls(config.use_tls) - .set_no_tls_verify(config.allow_invalid_certs), - bind_dn, - ); + let timeout = config.timeout.into_inner(); + let settings = LdapConnSettings::new() + .set_conn_timeout(timeout) + .set_starttls(config.use_tls) + .set_no_tls_verify(config.allow_invalid_certs); + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut urls = vec![config.url]; + let servers = Arc::new(LdapServers::new(urls.into_boxed_slice())); + let bind_manager = config + .bind_authentication + .then(|| LdapConnectionManager::new(servers.clone(), settings.clone(), None, timeout)); + let manager = LdapConnectionManager::new(servers.clone(), settings, bind_dn, timeout); let mut mappings = LdapMappings { base_dn: config.base_dn, filter_login: LdapFilter::new(&config.filter_login)?, - filter_mailbox: LdapFilter::new(&config.filter_mailbox)?, + filter_recipient: LdapFilter::new(&config.filter_recipient)?, filter_member_of: if let Some(filter) = config.filter_member_of { Some(LdapFilter::new(&filter)?) } else { @@ -111,19 +119,32 @@ impl LdapDirectory { } mappings.attrs_principal = attrs_principal; - let pool = Pool::builder(manager) - .runtime(Runtime::Tokio1) - .max_size(config.pool_max_connections as usize) - .create_timeout(config.pool_timeout_create.into_inner().into()) - .wait_timeout(config.pool_timeout_wait.into_inner().into()) - .recycle_timeout(config.pool_timeout_recycle.into_inner().into()) - .build() - .map_err(|err| format!("Failed to build LDAP pool: {err}"))?; + let create_timeout = if servers.server_count() > 1 { + config + .pool_timeout_create + .into_inner() + .max(timeout) + .saturating_mul(u32::try_from(servers.server_count()).unwrap_or(u32::MAX)) + } else { + config.pool_timeout_create.into_inner() + }; + let build_pool = |manager: LdapConnectionManager| { + Pool::builder(manager) + .runtime(Runtime::Tokio1) + .max_size(config.pool_max_connections as usize) + .create_timeout(create_timeout.into()) + .wait_timeout(config.pool_timeout_wait.into_inner().into()) + .recycle_timeout(config.pool_timeout_recycle.into_inner().into()) + .build() + .map_err(|err| format!("Failed to build LDAP pool: {err}")) + }; Ok(Directory::Ldap(LdapDirectory { mappings, - pool, - auth_bind: config.bind_authentication, + pool: build_pool(manager)?, + bind_pool: bind_manager.map(build_pool).transpose()?, + servers, + timeout, })) } } @@ -132,9 +153,9 @@ impl LdapFilter { pub(super) fn new(value: &str) -> Result { let mut filter = Vec::new(); let mut token = String::new(); - let mut value = value.chars(); + let mut chars = value.chars(); - while let Some(ch) = value.next() { + while let Some(ch) = chars.next() { match ch { '?' => { // For backwards compatibility, we treat '?' as a placeholder for the full value. @@ -149,7 +170,7 @@ impl LdapFilter { filter.push(LdapFilterItem::Static(token)); token = String::new(); } - for ch in value.by_ref() { + for ch in chars.by_ref() { if ch == '}' { break; } else { diff --git a/crates/directory/src/backend/ldap/failover.rs b/crates/directory/src/backend/ldap/failover.rs new file mode 100644 index 00000000..6ad5dd98 --- /dev/null +++ b/crates/directory/src/backend/ldap/failover.rs @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{ + iter::once, + sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + time::Instant, +}; + +const PRIMARY_RETRY_AFTER_MS: u64 = 60_000; + +pub(crate) struct LdapServers { + urls: Box<[String]>, + failover: AtomicBool, + active: AtomicUsize, + last_failure: AtomicU64, + retry_primary_after: u64, + epoch: Instant, +} + +impl LdapServers { + pub fn new(urls: Box<[String]>) -> Self { + Self { + failover: AtomicBool::new(urls.len() > 1), + urls, + active: AtomicUsize::new(0), + last_failure: AtomicU64::new(0), + retry_primary_after: PRIMARY_RETRY_AFTER_MS, + epoch: Instant::now(), + } + } + + pub fn server_count(&self) -> usize { + self.urls.len() + } + + pub fn primary(&self) -> &str { + self.urls.first().map_or("", String::as_str) + } + + pub fn has_failover(&self) -> bool { + self.failover.load(Ordering::Relaxed) + } + + pub fn disable_failover(&self) { + self.failover.store(false, Ordering::Relaxed); + self.active.store(0, Ordering::Relaxed); + } + + pub fn active(&self) -> usize { + self.active.load(Ordering::Relaxed) + } + + pub fn start(&self) -> usize { + if self.has_failover() { + self.active() + } else { + 0 + } + } + + pub fn connect_order(&self, start: usize) -> impl Iterator { + let count = if self.has_failover() { + self.urls.len() + } else { + 1 + }; + + once(start) + .chain((0..self.urls.len()).filter(move |&server| server != start)) + .take(count) + .filter_map(|server| self.urls.get(server).map(|url| (server, url.as_str()))) + } + + pub fn connected(&self, start: usize, server: usize) { + if server != start + && self + .active + .compare_exchange(start, server, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + if server != 0 { + self.last_failure.store(self.now(), Ordering::Relaxed); + } + self.log_switch(server); + } + } + + pub fn primary_recovered(&self) { + if self.has_failover() && self.active.swap(0, Ordering::Relaxed) != 0 { + self.log_switch(0); + } + } + + pub fn claim_primary_probe(&self) -> bool { + if self.active() == 0 || !self.has_failover() { + return false; + } + let last_failure = self.last_failure.load(Ordering::Relaxed); + let now = self.now(); + now.saturating_sub(last_failure) >= self.retry_primary_after + && self + .last_failure + .compare_exchange(last_failure, now, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + } + + fn log_switch(&self, server: usize) { + trc::event!( + Store(trc::StoreEvent::LdapWarning), + Reason = "Switched active LDAP server", + Url = self.urls.get(server).cloned().unwrap_or_default() + ); + } + + fn now(&self) -> u64 { + u64::try_from(self.epoch.elapsed().as_millis()).unwrap_or(u64::MAX) + } +} + diff --git a/crates/directory/src/backend/ldap/lookup.rs b/crates/directory/src/backend/ldap/lookup.rs index 1ce1e702..9afdf3bf 100644 --- a/crates/directory/src/backend/ldap/lookup.rs +++ b/crates/directory/src/backend/ldap/lookup.rs @@ -4,14 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{LdapDirectory, LdapMappings}; -use crate::{Account, Credentials, Group, IntoError, Recipient, core::secret::verify_secret_hash}; -use ldap3::{Ldap, LdapConnAsync, ResultEntry, Scope, SearchEntry}; +use super::{ + LdapDirectory, LdapMappings, + pool::{is_transport_error, with_conn}, +}; +use crate::{ + Account, Credentials, Group, IntoError, Recipient, + core::{secret::verify_secret_hash, verifier::PasswordVerifier}, +}; +use compact_str::CompactString; +use deadpool::managed::Object; +use ldap3::{Ldap, ResultEntry, Scope, SearchEntry}; +use std::time::Duration; use store::xxhash_rust; use utils::sanitize_email; impl LdapDirectory { - pub async fn authenticate(&self, credentials: &Credentials) -> trc::Result { + pub async fn authenticate( + &self, + credentials: &Credentials, + verifier: &PasswordVerifier, + ) -> trc::Result { let (username, secret) = match credentials { Credentials::Basic { username, secret, .. @@ -21,87 +34,114 @@ impl LdapDirectory { if secret.is_empty() { return Err(trc::AuthEvent::Failed .into_err() - .details("Empty secret rejected")); + .reason("Empty secret rejected")); } - let mut conn = self.pool.get().await.map_err(|err| err.into_error())?; - - let mut result = if self.auth_bind { - let filter = self.mappings.filter_login.build(username); - if let Some(mut result) = self.find_object(&mut conn, &filter).await? { - // Perform bind auth using the found dn - let (auth_bind_conn, mut ldap) = LdapConnAsync::with_settings( - self.pool.manager().settings.clone(), - &self.pool.manager().address, - ) + let filter = self.mappings.filter_login.build(username); + let login = { + let mut conn = self.pool.get().await.map_err(|err| err.into_error())?; + match self + .authenticate_on(&mut conn.ldap, &filter, username, secret) .await - .map_err(|err| err.into_error().caused_by(trc::location!()))?; - - ldap3::drive!(auth_bind_conn); - - if ldap - .simple_bind(&result.dn, secret) - .await - .map_err(|err| err.into_error().caused_by(trc::location!()))? - .success() - .is_ok() - { - if result.account.email.is_empty() { - result.account.email = - sanitize_email(username).unwrap_or_else(|| username.to_lowercase()); - } - result - } else { - return Err(trc::AuthEvent::Failed - .into_err() - .details("Secret rejected during auth bind using lookup filter") - .details(vec![result.dn, filter])); + { + Err(err) + | Ok(LdapLogin::VerifyHash { + groups: Err(err), .. + }) if is_transport_error(&err) => { + trc::event!( + Store(trc::StoreEvent::LdapWarning), + Reason = "LDAP operation failed, retrying on a new connection", + CausedBy = err + ); + drop(Object::take(conn)); + let mut conn = self.pool.get().await.map_err(|err| err.into_error())?; + self.authenticate_on(&mut conn.ldap, &filter, username, secret) + .await } - } else { - return Err(trc::AuthEvent::Failed - .into_err() - .details("Auth bind lookup filter yielded no results") - .details(vec![filter])); + login => login, } - } else { - let filter = self.mappings.filter_login.build(username); - if let Some(mut result) = self.find_object(&mut conn, &filter).await? { - if let Some(account_secret) = &result.account.secret { - if !verify_secret_hash(account_secret, secret.as_bytes()).await? { - return Err(trc::AuthEvent::Failed - .into_err() - .details("Invalid credentials") - .details(vec![filter])); - } - } else { - return Err(trc::AuthEvent::Error - .into_err() - .details("Account does not have a secret") - .details(vec![filter])); - } - if result.account.email.is_empty() { - result.account.email = - sanitize_email(username).unwrap_or_else(|| username.to_lowercase()); - } - result - } else { - return Err(trc::AuthEvent::Failed - .into_err() - .details("Authentication filter yielded no results") - .details(vec![filter])); + }?; + + match login { + LdapLogin::Bound(account) => Ok(account), + LdapLogin::VerifyHash { + account, + hash, + groups, + } => { + let verified = verify_secret_hash(verifier, &hash, secret.as_bytes()).await?; + hash_login(account, hash, groups, verified, filter) } - }; - - self.add_group_membership(&mut conn, &mut result).await?; - - Ok(result.account) + } } pub async fn recipient(&self, address: &str) -> trc::Result { - let mut conn = self.pool.get().await.map_err(|err| err.into_error())?; - let filter = self.mappings.filter_mailbox.build(address); - if let Some(mut result) = self.find_object(&mut conn, &filter).await? { + with_conn!(&self.pool, |conn| self.recipient_on(conn, address)) + } + + async fn authenticate_on( + &self, + conn: &mut Ldap, + filter: &str, + username: &str, + secret: &str, + ) -> trc::Result { + let Some(mut result) = self.find_object(conn, filter).await? else { + return Err(trc::AuthEvent::Failed + .into_err() + .reason(if self.bind_pool.is_some() { + "Auth bind lookup filter yielded no results" + } else { + "Authentication filter yielded no results" + }) + .ctx(trc::Key::Query, filter.to_string())); + }; + + let hash = if let Some(bind_pool) = &self.bind_pool { + let is_bound: bool = with_conn!(bind_pool, |bind_conn| bind_on( + bind_conn, + &result.dn, + secret, + self.timeout + ))?; + if !is_bound { + return Err(trc::AuthEvent::Failed + .into_err() + .reason("Secret rejected during auth bind using lookup filter") + .ctx(trc::Key::Name, result.dn) + .ctx(trc::Key::Query, filter.to_string())); + } + None + } else { + Some(result.account.secret.take().ok_or_else(|| { + trc::AuthEvent::Failed + .into_err() + .reason("Account does not have a secret") + .ctx(trc::Key::Query, filter.to_string()) + })?) + }; + + if result.account.email.is_empty() { + result.account.email = + sanitize_email(username).unwrap_or_else(|| username.to_lowercase()); + } + + let groups = self.add_group_membership(conn, &mut result).await; + + match hash { + Some(hash) => Ok(LdapLogin::VerifyHash { + account: result.account, + hash, + groups, + }), + None => groups.map(|()| LdapLogin::Bound(result.account)), + } + } + + async fn recipient_on(&self, conn: &mut Ldap, address: &str) -> trc::Result { + let filter = self.mappings.filter_recipient.build(address); + if let Some(mut result) = self.find_object(conn, &filter).await? { if !result.is_group { - self.add_group_membership(&mut conn, &mut result).await?; + self.add_group_membership(conn, &mut result).await?; Ok(Recipient::Account(result.account)) } else { Ok(Recipient::Group(Group { @@ -114,7 +154,7 @@ impl LdapDirectory { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Mailbox filter yielded no results", - Details = filter + Query = filter ); Ok(Recipient::Invalid) } @@ -129,6 +169,7 @@ impl LdapDirectory { let mut groups = Vec::new(); for name in group_dns.into_iter().filter(|name| name.contains('=')) { let (rs, _res) = conn + .with_timeout(self.timeout) .search( &name, Scope::Base, @@ -159,6 +200,7 @@ impl LdapDirectory { } else if let Some(filter) = &self.mappings.filter_member_of { let filter = filter.build(&result.dn); let rs = conn + .with_timeout(self.timeout) .search( &self.mappings.base_dn, Scope::Subtree, @@ -180,7 +222,7 @@ impl LdapDirectory { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Group entry missing valid email attribute", - Details = v + Address = v ); None }) @@ -203,27 +245,57 @@ impl LdapDirectory { impl LdapDirectory { async fn find_object(&self, conn: &mut Ldap, filter: &str) -> trc::Result> { - conn.search( - &self.mappings.base_dn, - Scope::Subtree, - filter, - &self.mappings.attrs_principal, - ) - .await - .map_err(|err| err.into_error().caused_by(trc::location!()))? - .success() - .map(|(rs, _)| { - trc::event!( - Store(trc::StoreEvent::LdapQuery), - Details = filter.to_string(), - Result = rs.first().map(result_to_trace).unwrap_or_default() - ); + conn.with_timeout(self.timeout) + .search( + &self.mappings.base_dn, + Scope::Subtree, + filter, + &self.mappings.attrs_principal, + ) + .await + .map_err(|err| err.into_error().caused_by(trc::location!()))? + .success() + .map(|(rs, _)| { + trc::event!( + Store(trc::StoreEvent::LdapQuery), + Query = CompactString::from(filter), + Result = rs.first().map(result_to_trace).unwrap_or_default() + ); - rs.into_iter() - .next() - .map(|entry| self.mappings.map_entry(SearchEntry::construct(entry))) + rs.into_iter() + .next() + .map(|entry| self.mappings.map_entry(SearchEntry::construct(entry))) + }) + .map_err(|err| err.into_error().caused_by(trc::location!())) + } +} + +enum LdapLogin { + Bound(Account), + VerifyHash { + account: Account, + hash: String, + groups: trc::Result<()>, + }, +} + +fn hash_login( + mut account: Account, + hash: String, + groups: trc::Result<()>, + verified: bool, + filter: String, +) -> trc::Result { + if verified { + groups.map(|()| { + account.secret = Some(hash); + account }) - .map_err(|err| err.into_error().caused_by(trc::location!())) + } else { + Err(trc::AuthEvent::Failed + .into_err() + .reason("Invalid credentials") + .ctx(trc::Key::Query, filter)) } } @@ -296,6 +368,16 @@ impl LdapMappings { } } +async fn bind_on(conn: &mut Ldap, dn: &str, secret: &str, timeout: Duration) -> trc::Result { + Ok(conn + .with_timeout(timeout) + .simple_bind(dn, secret) + .await + .map_err(|err| err.into_error().caused_by(trc::location!()))? + .success() + .is_ok()) +} + fn result_to_trace(rs: &ResultEntry) -> trc::Value { let se = SearchEntry::construct(rs.clone()); se.attrs @@ -305,3 +387,4 @@ fn result_to_trace(rs: &ResultEntry) -> trc::Value { .collect::>() .into() } + diff --git a/crates/directory/src/backend/ldap/mod.rs b/crates/directory/src/backend/ldap/mod.rs index 26ed54ea..798e9a19 100644 --- a/crates/directory/src/backend/ldap/mod.rs +++ b/crates/directory/src/backend/ldap/mod.rs @@ -5,23 +5,38 @@ */ use deadpool::managed::Pool; -use ldap3::{LdapConnSettings, ldap_escape}; +use failover::LdapServers; +use ldap3::{Ldap, LdapConnSettings, ldap_escape}; +use std::{sync::Arc, time::Duration}; pub mod config; +pub mod failover; pub mod lookup; pub mod pool; pub struct LdapDirectory { pool: Pool, mappings: LdapMappings, - auth_bind: bool, + bind_pool: Option>, + servers: Arc, + timeout: Duration, +} + +impl LdapDirectory { + pub fn has_failover(&self) -> bool { + self.servers.has_failover() + } + + pub fn disable_failover(&self) { + self.servers.disable_failover(); + } } #[derive(Debug, Default)] pub struct LdapMappings { base_dn: String, filter_login: LdapFilter, - filter_mailbox: LdapFilter, + filter_recipient: LdapFilter, filter_member_of: Option, attr_class: Vec, attr_groups: Vec, @@ -79,9 +94,19 @@ impl LdapFilter { } pub(crate) struct LdapConnectionManager { - address: String, + servers: Arc, + connector: Arc, +} + +pub(crate) struct LdapConnector { settings: LdapConnSettings, bind_dn: Option, + timeout: Duration, +} + +pub(crate) struct LdapConnection { + ldap: Ldap, + server: usize, } pub(crate) struct Bind { @@ -90,11 +115,19 @@ pub(crate) struct Bind { } impl LdapConnectionManager { - pub fn new(address: String, settings: LdapConnSettings, bind_dn: Option) -> Self { + pub fn new( + servers: Arc, + settings: LdapConnSettings, + bind_dn: Option, + timeout: Duration, + ) -> Self { Self { - address, - settings, - bind_dn, + servers, + connector: Arc::new(LdapConnector { + settings, + bind_dn, + timeout, + }), } } } @@ -105,21 +138,3 @@ impl Bind { } } -#[cfg(test)] -mod tests { - use super::LdapFilter; - - #[test] - fn filter_placeholders_are_escaped() { - let filter = LdapFilter::new("(&(uid={local})(dc={domain})(mail=?))").unwrap(); - - assert_eq!( - filter.build("*)(uid=*@ex)(dc=*"), - "(&(uid=\\2a\\29\\28uid=\\2a)(dc=ex\\29\\28dc=\\2a)(mail=\\2a\\29\\28uid=\\2a@ex\\29\\28dc=\\2a))" - ); - assert_eq!( - filter.build("john@example.com"), - "(&(uid=john)(dc=example.com)(mail=john@example.com))" - ); - } -} diff --git a/crates/directory/src/backend/ldap/pool.rs b/crates/directory/src/backend/ldap/pool.rs index a8913b79..1d543225 100644 --- a/crates/directory/src/backend/ldap/pool.rs +++ b/crates/directory/src/backend/ldap/pool.rs @@ -4,37 +4,152 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::LdapConnectionManager; +use super::{LdapConnection, LdapConnectionManager, LdapConnector}; use deadpool::managed; use ldap3::{Ldap, LdapConnAsync, LdapError, exop::WhoAmI}; +use std::{io, time::Duration}; +use tokio::time::timeout; +use trc::{EventType, Key, StoreEvent}; + +const RECYCLE_PROBE_AFTER_IDLE: Duration = Duration::from_secs(5); impl managed::Manager for LdapConnectionManager { - type Type = Ldap; + type Type = LdapConnection; type Error = LdapError; - async fn create(&self) -> Result { - let (conn, mut ldap) = - LdapConnAsync::with_settings(self.settings.clone(), &self.address).await?; + async fn create(&self) -> Result { + self.probe_primary(); - ldap3::drive!(conn); + let start = self.servers.start(); + let mut last_error = None; - if let Some(bind) = &self.bind_dn { - ldap.simple_bind(&bind.dn, &bind.password) - .await? - .success()?; + for (server, url) in self.servers.connect_order(start) { + match self + .connector + .connect(url, self.servers.has_failover()) + .await + { + Ok(ldap) => { + self.servers.connected(start, server); + return Ok(LdapConnection { ldap, server }); + } + Err(err) => { + if self.servers.has_failover() { + trc::event!( + Store(trc::StoreEvent::LdapWarning), + Reason = "Failed to connect to LDAP server", + Url = url.to_string(), + Details = err.to_string() + ); + } + last_error = Some(err); + } + } } - Ok(ldap) + Err(last_error + .unwrap_or_else(|| LdapError::from(io::Error::other("no LDAP server is configured")))) } async fn recycle( &self, - conn: &mut Ldap, - _: &managed::Metrics, + conn: &mut LdapConnection, + metrics: &managed::Metrics, ) -> managed::RecycleResult { - conn.extended(WhoAmI) - .await - .map(|_| ()) - .map_err(managed::RecycleError::Backend) + self.probe_primary(); + + if conn.ldap.is_closed() { + Err(managed::RecycleError::message("LDAP connection closed")) + } else if conn.server != self.servers.active() { + Err(managed::RecycleError::message("LDAP server switched")) + } else if metrics.last_used() < RECYCLE_PROBE_AFTER_IDLE { + Ok(()) + } else { + conn.ldap + .extended(WhoAmI) + .await + .map(|_| ()) + .map_err(managed::RecycleError::Backend) + } } } + +impl LdapConnectionManager { + fn probe_primary(&self) { + if !self.servers.claim_primary_probe() { + return; + } + + let servers = self.servers.clone(); + let connector = self.connector.clone(); + tokio::spawn(async move { + match connector.connect(servers.primary(), true).await { + Ok(mut ldap) => { + let _ = ldap.unbind().await; + servers.primary_recovered(); + } + Err(err) => { + trc::event!( + Store(trc::StoreEvent::LdapWarning), + Reason = "Primary LDAP server is still unreachable", + Url = servers.primary().to_string(), + Details = err.to_string() + ); + } + } + }); + } +} + +impl LdapConnector { + async fn connect(&self, url: &str, verify: bool) -> Result { + timeout(self.timeout, async { + let (conn, mut ldap) = + Box::pin(LdapConnAsync::with_settings(self.settings.clone(), url)).await?; + + ldap3::drive!(conn); + + if let Some(bind) = &self.bind_dn { + ldap.simple_bind(&bind.dn, &bind.password) + .await? + .success()?; + } else if verify { + ldap.extended(WhoAmI).await?; + } + + Ok(ldap) + }) + .await? + } +} + +macro_rules! with_conn { + ($pool:expr, |$conn:ident| $operation:expr) => {{ + let pool = $pool; + let mut conn = pool.get().await.map_err(|err| err.into_error())?; + let result = { + let $conn = &mut conn.ldap; + $operation.await + }; + match result { + Err(err) if crate::backend::ldap::pool::is_transport_error(&err) => { + trc::event!( + Store(trc::StoreEvent::LdapWarning), + Reason = "LDAP operation failed, retrying on a new connection", + CausedBy = err + ); + drop(deadpool::managed::Object::take(conn)); + let mut conn = pool.get().await.map_err(|err| err.into_error())?; + let $conn = &mut conn.ldap; + $operation.await + } + result => result, + } + }}; +} + +pub(crate) use with_conn; + +pub(crate) fn is_transport_error(err: &trc::Error) -> bool { + err.matches(EventType::Store(StoreEvent::LdapError)) && err.value(Key::Code).is_none() +} diff --git a/crates/directory/src/backend/oidc/config.rs b/crates/directory/src/backend/oidc/config.rs index 5fdadb17..0753cb4c 100644 --- a/crates/directory/src/backend/oidc/config.rs +++ b/crates/directory/src/backend/oidc/config.rs @@ -5,31 +5,95 @@ */ use crate::Directory; -use crate::backend::oidc::lookup::fetch_jwks_keys; use crate::backend::oidc::{ - CachedKey, DiscoveryDocument, JwksCache, OidcConfig, OidcDiscovery, OidcError, OpenIdDirectory, + Conformance, ConformanceGaps, DiscoveryDocument, ExternalScopes, IntrospectionAuth, JwksState, + METADATA_MAX_SIZE, OAUTH_METADATA_PATH, OidcConfig, OidcDiscovery, OidcError, OpaqueValidation, + OpenIdDirectory, + jwks::{FetchedKeys, fetch_jwks}, + openid_configuration_url, read_body, }; -use ahash::AHashMap; -use registry::schema::structs; -use std::sync::Arc; +use compact_str::{CompactString, format_compact}; +use encodify::base64::STANDARD; +use parking_lot::RwLock; +use registry::schema::{ + enums::{OidcIntrospectionAuth, OidcOpaqueTokenValidation, OidcUnscopedTokenPolicy}, + structs, +}; +use reqwest::{ + Method, StatusCode, Url, + header::{CONTENT_TYPE, HeaderMap}, +}; +use serde::Serialize; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; use trc::AuthEvent; -use utils::Client; +use utils::{ + http::http_client_builder, + outbound::{OutboundClient, OutboundPolicy, Redirects}, +}; const DISCOVERY_RETRY_FOR: Duration = Duration::from_secs(30); const DISCOVERY_RETRY_INTERVAL: Duration = Duration::from_secs(3); +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +struct Metadata { + document: DiscoveryDocument, + url: String, + is_json: bool, +} + +enum MetadataFetch { + Found(Box), + Failed(String), +} + +#[derive(Serialize)] +struct WebDiscoveryDocument<'x> { + #[serde(flatten)] + document: &'x DiscoveryDocument, + stalwart_required_scopes: &'x [&'x str], +} impl OpenIdDirectory { pub async fn open(config: structs::OidcDirectory) -> Result { + let opaque_validation = match config.opaque_token_validation { + OidcOpaqueTokenValidation::None => OpaqueValidation::Reject, + OidcOpaqueTokenValidation::UserInfo => OpaqueValidation::UserInfo, + OidcOpaqueTokenValidation::Introspection => { + OpaqueValidation::Introspection(introspection_auth(&config).await?) + } + }; + + #[cfg(not(feature = "enterprise"))] + let outbound_policy = OutboundPolicy::Trusted; + + if matches!(opaque_validation, OpaqueValidation::UserInfo) + && config.unscoped_tokens == OidcUnscopedTokenPolicy::Reject + { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(config.issuer_url.as_str()), + Reason = "Opaque tokens validated through UserInfo carry no scopes and are always rejected while unscopedTokens is set to reject" + ); + } + Self::new(OidcConfig { - issue_url: config.issuer_url, - require_aud: config.require_audience, - require_scopes: config.require_scopes.into_inner(), - claim_email: config.claim_username, + issuer_url: config.issuer_url, + claim_username: config.claim_username, claim_name: config.claim_name, claim_groups: config.claim_groups, - default_domain: config.username_domain, + username_domain: config.username_domain, + allow_invalid_certs: config.allow_invalid_certs, + outbound_policy, + accepted_audiences: config + .accepted_audiences + .into_iter() + .map(String::into_boxed_str) + .collect(), + opaque_validation, + unscoped_tokens: config.unscoped_tokens, + skip_audience_validation: config.skip_audience_validation, + allow_id_token_claims: config.allow_id_token_claims, + lenient_issuer_match: config.lenient_issuer_match, }) .await .map(Directory::OpenId) @@ -37,21 +101,23 @@ impl OpenIdDirectory { } pub async fn new(config: OidcConfig) -> Result { - let http = utils::http::http_client_builder(false) - .user_agent("Stalwart/1.0") - .timeout(Duration::from_secs(30)) - .build() - .map_err(|e| OidcError::Network(format!("HTTP client build failed: {e}")))?; + let http = OutboundClient::new( + http_client_builder(config.allow_invalid_certs) + .user_agent("Stalwart/1.0") + .timeout(HTTP_TIMEOUT), + config.outbound_policy, + Redirects::None, + ); let started_at = Instant::now(); - let (document, keys) = loop { + let (discovery, issuers, keys) = loop { match Self::discover(&http, &config).await { Ok(discovery) => break discovery, Err(err) if err.is_transient() && started_at.elapsed() < DISCOVERY_RETRY_FOR => { trc::event!( Auth(AuthEvent::Warning), - Url = config.issue_url.to_string(), - Reason = format!( + Url = CompactString::from(config.issuer_url.as_str()), + Reason = format_compact!( "{err}, retrying in {} seconds", DISCOVERY_RETRY_INTERVAL.as_secs() ) @@ -63,115 +129,393 @@ impl OpenIdDirectory { }; Ok(Self { - discovery: OidcDiscovery { - url: config.issue_url.clone(), - document, - }, config, + discovery, + issuers, http, - cache: RwLock::new(JwksCache { - keys, - last_updated: Instant::now(), - }), + jwks: RwLock::new(JwksState::new(keys)), + jwks_refresh: tokio::sync::Mutex::new(()), }) } async fn discover( - http: &Client, + http: &OutboundClient, config: &OidcConfig, - ) -> Result<(DiscoveryDocument, AHashMap>), OidcError> { - let discovery_url = format!( - "{}/.well-known/openid-configuration", - config.issue_url.trim_end_matches('/') - ); - let discovery_bytes = http - .get(&discovery_url) - .send() - .await - .map_err(|e| OidcError::Network(format!("Discovery fetch failed: {e}")))? - .error_for_status() - .map_err(|e| OidcError::Provider(format!("Discovery HTTP error: {e}")))? - .bytes() - .await - .map_err(|e| OidcError::Provider(format!("Discovery HTTP error: {e}")))?; - let discovery: DiscoveryDocument = serde_json::from_slice(&discovery_bytes) - .map_err(|e| OidcError::Provider(format!("Discovery JSON parse error: {e}")))?; + ) -> Result<(OidcDiscovery, Box<[String]>, FetchedKeys), OidcError> { + let Metadata { + mut document, + url, + is_json, + } = fetch_metadata(http, &config.issuer_url).await?; - let normalised_issue = config.issue_url.trim_end_matches('/'); - let normalised_issuer = discovery.issuer.trim_end_matches('/'); - if normalised_issuer != normalised_issue { + if !issuer_matches( + &config.issuer_url, + &document.issuer, + config.lenient_issuer_match, + ) { return Err(OidcError::Config(format!( - "Issuer mismatch: discovery document says '{}' but configured issue_url is '{}'", - discovery.issuer, config.issue_url, + "Issuer mismatch: metadata at '{url}' says '{}' but the configured issuer is '{}'", + document.issuer, config.issuer_url, ))); } - if let Some(supported) = &discovery.scopes_supported { - for scope in &config.require_scopes { - if !supported.contains(scope) { - trc::event!( - Auth(AuthEvent::Warning), - Url = config.issue_url.to_string(), - Reason = format!( - "Required scope '{}' is not in scopes_supported from the IdP", - scope - ) - ); + let oidc_url = openid_configuration_url(&document.issuer); + if (document.userinfo_endpoint.is_none() || document.end_session_endpoint.is_none()) + && url != oidc_url + { + let reason = match fetch_document(http, &oidc_url, false).await { + Ok(MetadataFetch::Found(oidc)) if oidc.document.issuer == document.issuer => { + if document.userinfo_endpoint.is_none() { + document.userinfo_endpoint = oidc.document.userinfo_endpoint; + } + if document.end_session_endpoint.is_none() { + document.end_session_endpoint = oidc.document.end_session_endpoint; + } + None } + Ok(MetadataFetch::Found(_)) => Some(format_compact!( + "the OpenID configuration has a different issuer" + )), + Ok(MetadataFetch::Failed(reason)) => Some(format_compact!("{reason}")), + Err(err) => Some(format_compact!("{err}")), + }; + if let Some(reason) = reason { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(oidc_url.as_str()), + Reason = format_compact!( + "Could not complete userinfo_endpoint and end_session_endpoint: {reason}" + ) + ); } } - if let Some(supported) = &discovery.claims_supported { + if matches!(config.opaque_validation, OpaqueValidation::Introspection(_)) + && document.introspection_endpoint.is_none() + { + return Err(OidcError::Config( + "opaqueTokenValidation is set to introspection but the IdP metadata has no introspection_endpoint".to_string(), + )); + } + + if let Some(supported) = &document.claims_supported { let check = |name: &str, label: &str| { if !supported.iter().any(|c| c == name) { trc::event!( Auth(AuthEvent::Warning), - Url = config.issue_url.to_string(), - Reason = format!( - "Configured {} claim '{}' is not in claims_supported from the IdP", - label, name + Url = CompactString::from(config.issuer_url.as_str()), + Reason = format_compact!( + "Configured {label} claim '{name}' is not in claims_supported from the IdP" ) ); } }; - check(&config.claim_email, "claim_email"); + check(&config.claim_username, "claimUsername"); if let Some(n) = &config.claim_name { - check(n, "claim_name"); + check(n, "claimName"); } if let Some(g) = &config.claim_groups { - check(g, "claim_groups"); + check(g, "claimGroups"); } } - /*{ - let cache = Arc::clone(&cache); - let http = http.clone(); - let jwks_uri = discovery.jwks_uri.clone(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(24 * 3600)); - interval.tick().await; - loop { - interval.tick().await; - match fetch_jwks_keys(&http, &jwks_uri).await { - Ok(new_keys) => { - let mut guard = cache.write().await; - guard.keys = new_keys; - guard.last_updated = Instant::now(); - } - Err(e) => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!("Background JWKS refresh failed: {e}") - ); - } - } - } + let conformance = conformance(&document, is_json); + log_conformance(&document.issuer, conformance); + + let mut issuers = Vec::with_capacity(2); + issuers.push(document.issuer.clone()); + if config.lenient_issuer_match { + issuers.push(match document.issuer.strip_suffix('/') { + Some(issuer) => issuer.to_string(), + None => format!("{}/", document.issuer), }); - }*/ + } - let keys = fetch_jwks_keys(http, &discovery.jwks_uri).await?; + let keys = fetch_jwks(http, &document.jwks_uri, &issuers).await?; - Ok((discovery, keys)) + let web_document = serde_json::to_vec(&WebDiscoveryDocument { + document: &document, + stalwart_required_scopes: &required_scopes(config, &document), + }) + .map_err(|err| OidcError::Provider(format!("Failed to serialize metadata: {err}")))? + .into(); + + Ok(( + OidcDiscovery { + document, + metadata_url: url, + conformance, + web_document, + }, + issuers.into_boxed_slice(), + keys, + )) } } + +async fn introspection_auth(config: &structs::OidcDirectory) -> Result { + if config.introspection_auth_method == OidcIntrospectionAuth::None { + return Ok(IntrospectionAuth::Unauthenticated); + } + let secret = config.introspection_client_secret.secret().await?; + let (Some(client_id), Some(secret)) = ( + config + .introspection_client_id + .as_deref() + .filter(|id| !id.is_empty()), + secret.as_deref().filter(|secret| !secret.is_empty()), + ) else { + return Err( + "introspectionClientId and introspectionClientSecret are required for the selected introspectionAuthMethod" + .to_string(), + ); + }; + + let client_id = form_urlencoded::byte_serialize(client_id.as_bytes()).collect::(); + let secret = form_urlencoded::byte_serialize(secret.as_bytes()).collect::(); + Ok(match config.introspection_auth_method { + OidcIntrospectionAuth::ClientSecretPost => IntrospectionAuth::Post { + credentials: format!("&client_id={client_id}&client_secret={secret}").into_boxed_str(), + }, + _ => IntrospectionAuth::Basic { + authorization: format!("Basic {}", STANDARD.encode(format!("{client_id}:{secret}"))) + .into_boxed_str(), + }, + }) +} + +async fn fetch_metadata(http: &OutboundClient, issuer: &str) -> Result { + let oauth_url = oauth_metadata_url(issuer)?; + let oauth_failure = match fetch_document(http, &oauth_url, true).await? { + MetadataFetch::Found(metadata) => return Ok(*metadata), + MetadataFetch::Failed(reason) => reason, + }; + + let oidc_url = openid_configuration_url(issuer); + match fetch_document(http, &oidc_url, false).await? { + MetadataFetch::Found(metadata) => Ok(*metadata), + MetadataFetch::Failed(oidc_failure) => Err(OidcError::Provider(format!( + "Metadata not found: {oauth_url}: {oauth_failure}; {oidc_url}: {oidc_failure}" + ))), + } +} + +async fn fetch_document( + http: &OutboundClient, + url: &str, + require_json: bool, +) -> Result { + let response = http + .request(Method::GET, url) + .map_err(|err| OidcError::Config(format!("Invalid metadata URL '{url}': {err}")))? + .send() + .await + .map_err(|e| OidcError::Network(format!("Metadata fetch from '{url}' failed: {e}")))?; + + let status = response.status(); + if status != StatusCode::OK { + return Ok(MetadataFetch::Failed(format!("HTTP {status}"))); + } + let is_json = is_json_content(response.headers()); + if require_json && !is_json { + return Ok(MetadataFetch::Failed( + "Content-Type is not application/json".to_string(), + )); + } + let bytes = read_body(response, METADATA_MAX_SIZE, "Metadata").await?; + Ok(match serde_json::from_slice::(&bytes) { + Ok(document) => MetadataFetch::Found(Box::new(Metadata { + document, + url: url.to_string(), + is_json, + })), + Err(err) => MetadataFetch::Failed(format!("Invalid metadata: {err}")), + }) +} + +fn oauth_metadata_url(issuer: &str) -> Result { + let mut url = Url::parse(issuer) + .map_err(|err| OidcError::Config(format!("Invalid issuer URL '{issuer}': {err}")))?; + let path = url.path(); + let mut metadata_path = String::with_capacity(OAUTH_METADATA_PATH.len() + path.len()); + metadata_path.push_str(OAUTH_METADATA_PATH); + metadata_path.push_str(path.strip_suffix('/').unwrap_or(path)); + url.set_path(&metadata_path); + url.set_query(None); + url.set_fragment(None); + Ok(url.into()) +} + +fn is_json_content(headers: &HeaderMap) -> bool { + headers + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|media| media.trim().eq_ignore_ascii_case("application/json")) + }) +} + +fn issuer_matches(configured: &str, discovered: &str, lenient: bool) -> bool { + configured == discovered + || (lenient + && configured.strip_suffix('/').unwrap_or(configured) + == discovered.strip_suffix('/').unwrap_or(discovered)) +} + +fn is_public_issuer(issuer: &str) -> bool { + Url::parse(issuer).is_ok_and(|url| { + url.scheme() == "https" + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && (url.as_str() == issuer + || (url.path() == "/" && url.as_str().strip_suffix('/') == Some(issuer))) + }) +} + +fn conformance(document: &DiscoveryDocument, is_json: bool) -> Conformance { + let contains = |list: &Option>, value: &str| { + list.as_ref() + .is_some_and(|list| list.iter().any(|item| item == value)) + }; + + let mut gaps = ConformanceGaps::default(); + for (gap, failed) in [ + (ConformanceGaps::ISSUER, !is_public_issuer(&document.issuer)), + (ConformanceGaps::CONTENT_TYPE, !is_json), + ( + ConformanceGaps::REGISTRATION_ENDPOINT, + document + .registration_endpoint + .as_deref() + .is_none_or(str::is_empty), + ), + ( + ConformanceGaps::AUTHORIZATION_ENDPOINT, + document.authorization_endpoint.is_empty(), + ), + ( + ConformanceGaps::TOKEN_ENDPOINT, + document.token_endpoint.is_empty(), + ), + ( + ConformanceGaps::RESPONSE_TYPE_CODE, + !contains(&document.response_types_supported, "code"), + ), + ( + ConformanceGaps::GRANT_TYPES, + !contains(&document.grant_types_supported, "authorization_code") + || !contains(&document.grant_types_supported, "refresh_token"), + ), + ( + ConformanceGaps::AUTH_METHOD_NONE, + !contains(&document.token_endpoint_auth_methods_supported, "none"), + ), + ( + ConformanceGaps::PKCE_S256, + !contains(&document.code_challenge_methods_supported, "S256"), + ), + ( + ConformanceGaps::ISS_PARAMETER, + document.authorization_response_iss_parameter_supported != Some(true), + ), + ] { + if failed { + gaps.insert(gap); + } + } + + let mut scopes = ExternalScopes::NONE; + for scope in document.scopes_supported.iter().flatten() { + scopes.insert(ExternalScopes::from_name(scope)); + } + + Conformance { gaps, scopes } +} + +fn log_conformance(issuer: &str, conformance: Conformance) { + let mut missing = String::new(); + for name in conformance.gaps.names() { + if !missing.is_empty() { + missing.push_str(", "); + } + missing.push_str(name); + } + let mut missing_scopes = String::new(); + for name in ExternalScopes::ALL.difference(conformance.scopes).names() { + if !missing_scopes.is_empty() { + missing_scopes.push_str(", "); + } + missing_scopes.push_str(name); + } + + if !conformance.gaps.is_empty() || !conformance.scopes.contains(ExternalScopes::MAIL) { + if !missing_scopes.is_empty() { + if !missing.is_empty() { + missing.push_str(", "); + } + missing.push_str("scopes_supported: "); + missing.push_str(&missing_scopes); + } + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(issuer), + Reason = format_compact!( + "The identity provider does not conform to OAuth Public, PACC documents for its domains will only advertise password authentication. Missing: {missing}" + ) + ); + } else if !missing_scopes.is_empty() { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(issuer), + Reason = format_compact!( + "The identity provider does not support {missing_scopes}, PACC documents for its domains will only advertise password authentication while the matching services are published" + ) + ); + } +} + +fn required_scopes<'x>(config: &'x OidcConfig, document: &DiscoveryDocument) -> Vec<&'x str> { + let mut scopes = Vec::with_capacity(4); + scopes.push("openid"); + let email_fallback = + (config.claim_username != "email" && config.username_domain.is_none()).then_some("email"); + for claim in [ + Some(config.claim_username.as_str()), + email_fallback, + config.claim_name.as_deref(), + config.claim_groups.as_deref(), + ] + .into_iter() + .flatten() + { + let Some((scope, is_standard)) = hashify::map!( + claim.as_bytes(), + (&'static str, bool), + "email" => ("email", true), + "email_verified" => ("email", true), + "preferred_username" => ("profile", true), + "name" => ("profile", true), + "given_name" => ("profile", true), + "family_name" => ("profile", true), + "nickname" => ("profile", true), + "groups" => ("groups", false), + ) else { + continue; + }; + let is_supported = match &document.scopes_supported { + Some(supported) => supported.iter().any(|item| item == scope), + None => *is_standard, + }; + if is_supported && !scopes.contains(scope) { + scopes.push(scope); + } + } + scopes +} + diff --git a/crates/directory/src/backend/oidc/jwks.rs b/crates/directory/src/backend/oidc/jwks.rs new file mode 100644 index 00000000..e8abd1e0 --- /dev/null +++ b/crates/directory/src/backend/oidc/jwks.rs @@ -0,0 +1,304 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::backend::oidc::{ + CachedKey, JwksState, KeySet, METADATA_MAX_SIZE, OidcError, OpenIdDirectory, read_body, +}; +use ahash::AHashMap; +use compact_str::{CompactString, format_compact}; +use jsonwebtoken::{ + Algorithm, DecodingKey, Validation, + jwk::{self, JwkSet}, +}; +use reqwest::{ + Method, + header::{CACHE_CONTROL, HeaderMap}, +}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use trc::AuthEvent; +use utils::outbound::OutboundClient; + +const JWKS_DEFAULT_MAX_AGE: Duration = Duration::from_secs(3600); +const JWKS_MIN_MAX_AGE: Duration = Duration::from_secs(300); +const JWKS_MAX_MAX_AGE: Duration = Duration::from_secs(86400); +const JWKS_RETRY_INTERVAL: Duration = Duration::from_secs(300); +const VALIDATION_LEEWAY: u64 = 60; +const REQUIRED_CLAIMS: [&str; 2] = ["exp", "iss"]; + +pub(crate) struct FetchedKeys { + pub keys: KeySet, + pub max_age: Duration, +} + +enum KeyLookup { + Current(Arc), + Stale(Arc), + Missing { retry: bool }, +} + +impl KeySet { + pub(crate) fn candidates<'x>( + &'x self, + kid: Option<&str>, + ) -> impl Iterator { + let by_kid = kid.and_then(|kid| self.keys.get(kid)); + let all = kid.is_none().then(|| self.keys.values()); + by_kid + .into_iter() + .chain(all.into_iter().flatten()) + .map(|key| (&key.decoding_key, &key.validation)) + } + + fn has_candidates(&self, kid: Option<&str>) -> bool { + match kid { + Some(kid) => self.keys.contains_key(kid), + None => !self.keys.is_empty(), + } + } +} + +impl JwksState { + pub(crate) fn new(fetched: FetchedKeys) -> Self { + let now = Instant::now(); + JwksState { + keys: Arc::new(fetched.keys), + fetched_at: now, + attempted_at: now, + max_age: fetched.max_age, + } + } +} + +impl OpenIdDirectory { + pub(crate) async fn signing_keys(&self, kid: Option<&str>) -> Result, OidcError> { + match self.lookup_keys(kid) { + KeyLookup::Current(keys) => Ok(keys), + KeyLookup::Missing { retry: false } => Err(missing_key(kid)), + KeyLookup::Stale(keys) => { + let Ok(_guard) = self.jwks_refresh.try_lock() else { + return Ok(keys); + }; + match self.lookup_keys(kid) { + KeyLookup::Stale(keys) => Ok(self.refresh_keys().await.unwrap_or(keys)), + KeyLookup::Current(keys) => Ok(keys), + KeyLookup::Missing { .. } => Err(missing_key(kid)), + } + } + KeyLookup::Missing { retry: true } => { + let _guard = self.jwks_refresh.lock().await; + match self.lookup_keys(kid) { + KeyLookup::Current(keys) | KeyLookup::Stale(keys) => Ok(keys), + KeyLookup::Missing { retry: false } => Err(missing_key(kid)), + KeyLookup::Missing { retry: true } => { + let keys = self.refresh_keys().await?; + if keys.has_candidates(kid) { + Ok(keys) + } else { + Err(missing_key(kid)) + } + } + } + } + } + } + + fn lookup_keys(&self, kid: Option<&str>) -> KeyLookup { + let now = Instant::now(); + let state = self.jwks.read(); + let retry = now.saturating_duration_since(state.attempted_at) >= JWKS_RETRY_INTERVAL; + if !state.keys.has_candidates(kid) { + KeyLookup::Missing { retry } + } else if retry && now.saturating_duration_since(state.fetched_at) >= state.max_age { + KeyLookup::Stale(state.keys.clone()) + } else { + KeyLookup::Current(state.keys.clone()) + } + } + + async fn refresh_keys(&self) -> Result, OidcError> { + self.jwks.write().attempted_at = Instant::now(); + match fetch_jwks(&self.http, &self.discovery.document.jwks_uri, &self.issuers).await { + Ok(fetched) => { + let keys = Arc::new(fetched.keys); + let mut state = self.jwks.write(); + state.keys = keys.clone(); + state.fetched_at = Instant::now(); + state.max_age = fetched.max_age; + Ok(keys) + } + Err(err) => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(self.discovery.document.jwks_uri.as_str()), + Reason = format_compact!("JWKS refresh failed, keeping cached keys: {err}") + ); + Err(err) + } + } + } +} + +fn missing_key(kid: Option<&str>) -> OidcError { + if kid.is_some() { + OidcError::TokenValidation("Unknown key id".to_string()) + } else { + OidcError::Provider("JWKS contains no usable keys".to_string()) + } +} + +pub(crate) async fn fetch_jwks( + http: &OutboundClient, + jwks_uri: &str, + issuers: &[String], +) -> Result { + let response = http + .request(Method::GET, jwks_uri) + .map_err(|err| OidcError::Config(format!("Invalid JWKS URL: {err}")))? + .send() + .await + .map_err(|e| OidcError::Network(format!("JWKS fetch failed: {e}")))? + .error_for_status() + .map_err(|e| OidcError::Provider(format!("JWKS HTTP error: {e}")))?; + let max_age = cache_max_age(response.headers()); + let jwks_bytes = read_body(response, METADATA_MAX_SIZE, "JWKS").await?; + let jwks: JwkSet = serde_json::from_slice(&jwks_bytes) + .map_err(|e| OidcError::Provider(format!("JWKS JSON parse error: {e}")))?; + + let mut keys = AHashMap::with_capacity(jwks.keys.len()); + let mut synthetic_id: u64 = 0; + + for key in &jwks.keys { + if let Some(pk_use) = &key.common.public_key_use + && pk_use != &jwk::PublicKeyUse::Signature + { + continue; + } + + let algorithm = match &key.algorithm { + jwk::AlgorithmParameters::RSA(_) => match key.common.key_algorithm { + Some(jwk::KeyAlgorithm::RS256) => Algorithm::RS256, + Some(jwk::KeyAlgorithm::RS384) => Algorithm::RS384, + Some(jwk::KeyAlgorithm::RS512) => Algorithm::RS512, + Some(jwk::KeyAlgorithm::PS256) => Algorithm::PS256, + Some(jwk::KeyAlgorithm::PS384) => Algorithm::PS384, + Some(jwk::KeyAlgorithm::PS512) => Algorithm::PS512, + None => Algorithm::RS256, + Some(other) => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(jwks_uri), + Reason = format_compact!("Unsupported RSA key algorithm {:?}", other) + ); + continue; + } + }, + jwk::AlgorithmParameters::EllipticCurve(ec) => match ec.curve { + jwk::EllipticCurve::P256 => Algorithm::ES256, + jwk::EllipticCurve::P384 => Algorithm::ES384, + _ => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(jwks_uri), + Reason = format_compact!("Unsupported EC curve {:?}", ec.curve) + ); + continue; + } + }, + jwk::AlgorithmParameters::OctetKeyPair(_) => Algorithm::EdDSA, + jwk::AlgorithmParameters::OctetKey(_) => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(jwks_uri), + Reason = format_compact!( + "Symmetric (HMAC) key found in JWKS (kid={:?}), skipping, HMAC is not accepted", + key.common.key_id + ) + ); + continue; + } + _ => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(jwks_uri), + Reason = format_compact!( + "Unrecognised key type in JWKS (kid={:?}), skipping", + key.common.key_id + ) + ); + continue; + } + }; + + let decoding_key = match DecodingKey::from_jwk(key) { + Ok(decoding_key) => decoding_key, + Err(e) => { + trc::event!( + Auth(AuthEvent::Warning), + Url = CompactString::from(jwks_uri), + Reason = format_compact!( + "Failed to build DecodingKey from JWK (kid={:?}): {e}", + key.common.key_id + ) + ); + continue; + } + }; + + let kid: Box = match &key.common.key_id { + Some(id) => id.as_str().into(), + None => { + let id = format!("_synthetic_{synthetic_id}").into_boxed_str(); + synthetic_id += 1; + id + } + }; + + keys.insert( + kid, + CachedKey { + decoding_key, + validation: validation(algorithm, issuers), + }, + ); + } + + Ok(FetchedKeys { + keys: KeySet { keys }, + max_age, + }) +} + +fn validation(algorithm: Algorithm, issuers: &[String]) -> Validation { + let mut validation = Validation::new(algorithm); + validation.validate_aud = false; + validation.set_issuer(issuers); + validation.set_required_spec_claims(&REQUIRED_CLAIMS); + validation.leeway = VALIDATION_LEEWAY; + validation +} + +fn cache_max_age(headers: &HeaderMap) -> Duration { + headers + .get(CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + value.split(',').find_map(|directive| { + let (name, seconds) = directive.trim().split_once('=')?; + if name.trim().eq_ignore_ascii_case("max-age") { + seconds.trim().trim_matches('"').parse::().ok() + } else { + None + } + }) + }) + .map_or(JWKS_DEFAULT_MAX_AGE, |seconds| { + Duration::from_secs(seconds).clamp(JWKS_MIN_MAX_AGE, JWKS_MAX_MAX_AGE) + }) +} + diff --git a/crates/directory/src/backend/oidc/lookup.rs b/crates/directory/src/backend/oidc/lookup.rs index 64d9012e..eaf6fbfa 100644 --- a/crates/directory/src/backend/oidc/lookup.rs +++ b/crates/directory/src/backend/oidc/lookup.rs @@ -6,45 +6,80 @@ use crate::{ Account, Credentials, - backend::oidc::{CachedKey, OidcError, OpenIdDirectory}, + backend::oidc::{ + CLAIMS_MAX_SIZE, ERROR_MAX_SIZE, ExternalScopes, ExternalToken, IntrospectionAuth, + OidcError, OpaqueValidation, OpenIdDirectory, TokenAccess, TokenContext, read_body, + }, }; -use ahash::AHashMap; -use jsonwebtoken::{ - Algorithm, DecodingKey, Header, Validation, decode, decode_header, - jwk::{self, JwkSet}, +use jsonwebtoken::{Algorithm, Header, decode, decode_header, errors::ErrorKind}; +use registry::schema::enums::OidcUnscopedTokenPolicy; +use reqwest::{ + Method, StatusCode, + header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE}, }; -use reqwest::Client; use serde_json::Value; -use std::time::Instant; -use std::{sync::Arc, time::Duration}; use trc::AuthEvent; +use utils::HttpLimitResponse; + +const CLIENT_CLAIMS: [&str; 4] = ["azp", "client_id", "cid", "appid"]; +const ACCESS_TOKEN_CLAIMS: [&str; 5] = ["scope", "scp", "client_id", "cid", "appid"]; +const NON_ACCESS_MARKERS: [(&str, &str); 12] = [ + ("typ", "id"), + ("typ", "refresh"), + ("typ", "offline"), + ("token_use", "id"), + ("token_use", "id_token"), + ("token_use", "refresh"), + ("token_use", "refresh_token"), + ("tokenType", "id-token"), + ("tokenType", "refresh-token"), + ("type", "id-token"), + ("token_type", "id_token"), + ("token_type", "refresh_token"), +]; impl OpenIdDirectory { - pub async fn authenticate(&self, credentials: &Credentials) -> trc::Result { - match credentials { - Credentials::Bearer { token, .. } if token.is_empty() => { - Err(AuthEvent::Failed.into_err().reason("Empty token rejected")) - } - Credentials::Bearer { token, .. } => if let Ok(header) = decode_header(token) { - self.authenticate_jwt(token, header).await - } else { - #[cfg(feature = "test_mode")] - let token = token.strip_prefix(".").unwrap_or(token); - self.authenticate_opaque(token).await - } - .map_err(|err| match err { - OidcError::AuthorizationFailed(reason) => { - AuthEvent::Failed.into_err().reason(reason) - } - err => AuthEvent::Error.into_err().reason(err), - }), - _ => Err(AuthEvent::Error - .into_err() - .reason("Unsupported credentials type for OIDC backend")), - } + pub async fn authenticate(&self, _credentials: &Credentials) -> trc::Result { + Err(AuthEvent::Error + .into_err() + .reason("Unsupported credentials type for OIDC backend")) } - async fn authenticate_jwt(&self, token: &str, header: Header) -> Result { + pub fn accepts_issuer(&self, issuer: &str) -> bool { + self.issuers.iter().any(|accepted| accepted == issuer) + } + + pub async fn authenticate_token( + &self, + token: &str, + ctx: &TokenContext<'_>, + ) -> trc::Result { + if token.is_empty() { + return Err(AuthEvent::Failed.into_err().reason("Empty token rejected")); + } + + if let Ok(header) = decode_header(token) { + self.authenticate_jwt(token, header, ctx).await + } else { + #[cfg(stalwart_tests)] + let token = token.strip_prefix(".").unwrap_or(token); + self.authenticate_opaque(token, ctx).await + } + .map_err(|err| match err { + OidcError::AuthorizationFailed(reason) | OidcError::TokenValidation(reason) => { + AuthEvent::Failed.into_err().reason(reason) + } + OidcError::TokenExpired(reason) => AuthEvent::TokenExpired.into_err().reason(reason), + err => AuthEvent::Error.into_err().reason(err), + }) + } + + async fn authenticate_jwt( + &self, + token: &str, + header: Header, + ctx: &TokenContext<'_>, + ) -> Result { if matches!( header.alg, Algorithm::HS256 | Algorithm::HS384 | Algorithm::HS512 @@ -54,144 +89,262 @@ impl OpenIdDirectory { )); } - let candidates = self.get_key(header.kid.as_deref()).await?; + let kid = header.kid.as_deref(); + let keys = self.signing_keys(kid).await?; let mut last_err = None; - for cached in &candidates { - let dk = &cached.decoding_key; - let alg = cached.algorithm; - let mut validation = Validation::new(alg); - - if let Some(aud) = &self.config.require_aud { - validation.set_audience(&[aud]); - } else { - validation.validate_aud = false; - } - - validation.set_issuer(&[&self.discovery.document.issuer]); - validation.leeway = 60; - - match decode::(token, dk, &validation) { + let mut claims = None; + for (decoding_key, validation) in keys.candidates(kid) { + match decode::(token, decoding_key, validation) { Ok(token_data) => { - if self.config.require_aud.is_some() && token_data.claims.get("aud").is_none() { - last_err = Some(jsonwebtoken::errors::Error::from( - jsonwebtoken::errors::ErrorKind::InvalidAudience, - )); - continue; - } + claims = Some(token_data.claims); + break; + } + Err(err) => last_err = Some(err), + } + } + drop(keys); - self.validate_scopes(&token_data.claims)?; + let Some(mut claims) = claims else { + return Err(match last_err { + Some(err) + if matches!( + err.kind(), + ErrorKind::ExpiredSignature | ErrorKind::ImmatureSignature + ) => + { + OidcError::TokenExpired(format!("JWT validation failed: {err}")) + } + Some(err) => OidcError::TokenValidation(format!("JWT validation failed: {err}")), + None => OidcError::TokenValidation("Unknown key id".to_string()), + }); + }; - let mut claims = token_data.claims; - let missing_username = - is_claim_missing(&claims, Some(&self.config.claim_email)); - let jwt_email = self.resolve_email(&claims).ok(); - let missing_profile = - is_claim_missing(&claims, self.config.claim_name.as_ref()) - || is_claim_missing(&claims, self.config.claim_groups.as_ref()); + check_token_type( + header.typ.as_deref(), + &claims, + self.config.allow_id_token_claims, + )?; + self.check_audience(&claims, ctx)?; + let access = self.token_access(client_claim(&claims)?, token_scopes(&claims), ctx)?; + let expires_at = claims.get("exp").and_then(Value::as_u64); - if jwt_email.is_none() || missing_username || missing_profile { - match self.fetch_userinfo(token).await { - Ok(userinfo) => { - if let (Some(base), Value::Object(extra)) = - (claims.as_object_mut(), userinfo) - { - for (key, value) in extra { - if base.get(&key).is_none_or(Value::is_null) { - base.insert(key, value); - } - } - } + let username = self.resolve_username(&claims); + let missing_username = is_claim_missing(&claims, Some(&self.config.claim_username)); + if (username.is_err() + || missing_username + || is_claim_missing(&claims, self.config.claim_name.as_deref()) + || is_claim_missing(&claims, self.config.claim_groups.as_deref())) + && let Some(endpoint) = self.discovery.document.userinfo_endpoint.as_deref() + { + match self.fetch_userinfo(endpoint, token).await { + Ok(Value::Object(userinfo)) => { + if let Value::Object(base) = &mut claims { + for (key, value) in userinfo { + if base.get(&key).is_none_or(Value::is_null) { + base.insert(key, value); } - Err(err) if jwt_email.is_none() => return Err(err), - Err(_) => {} } } - - let email = match jwt_email { - Some(email) if !missing_username => email, - _ => self.resolve_email(&claims)?, - }; - return self.build_account(email, &claims); - } - Err(e) => { - last_err = Some(e); } + Ok(_) => {} + Err(err) if username.is_err() => return Err(err), + Err(_) => {} } } - Err(OidcError::TokenValidation(format!( - "JWT validation failed: {}", - last_err.map(|e| e.to_string()).unwrap_or_default() - ))) + let username = match username { + Ok(username) if !missing_username => username, + _ => self.resolve_username(&claims)?, + }; + + Ok(ExternalToken { + account: self.build_account(username, &claims), + access, + expires_at, + }) } - async fn authenticate_opaque(&self, token: &str) -> Result { - let claims = self.fetch_userinfo(token).await?; - self.build_account(self.resolve_email(&claims)?, &claims) + async fn authenticate_opaque( + &self, + token: &str, + ctx: &TokenContext<'_>, + ) -> Result { + match &self.config.opaque_validation { + OpaqueValidation::Reject => Err(OidcError::TokenValidation( + "Opaque access tokens are not accepted, only JWT access tokens are validated" + .to_string(), + )), + OpaqueValidation::UserInfo => { + let access = self.token_access(None, ExternalScopes::NONE, ctx)?; + let endpoint = self + .discovery + .document + .userinfo_endpoint + .as_deref() + .ok_or_else(|| { + OidcError::Config("The IdP metadata has no userinfo_endpoint".to_string()) + })?; + let claims = self.fetch_userinfo(endpoint, token).await?; + let username = self.resolve_username(&claims)?; + Ok(ExternalToken { + account: self.build_account(username, &claims), + access, + expires_at: None, + }) + } + OpaqueValidation::Introspection(auth) => self.introspect(token, auth, ctx).await, + } } - async fn get_key(&self, kid: Option<&str>) -> Result>, OidcError> { - { - let guard = self.cache.read().await; + async fn introspect( + &self, + token: &str, + auth: &IntrospectionAuth, + ctx: &TokenContext<'_>, + ) -> Result { + let endpoint = self + .discovery + .document + .introspection_endpoint + .as_deref() + .ok_or_else(|| { + OidcError::Config("The IdP metadata has no introspection_endpoint".to_string()) + })?; - if let Some(kid) = kid { - if let Some(cached) = guard.keys.get(kid) { - return Ok(vec![cached.clone()]); - } - - if guard.last_updated.elapsed() < Duration::from_secs(300) { - return Err(OidcError::TokenValidation("Unknown key id".to_string())); - } - } else { - let all: Vec<_> = guard.keys.values().cloned().collect(); - if !all.is_empty() { - return Ok(all); - } + let mut body = String::with_capacity(token.len() + 64); + body.push_str("token="); + body.extend(form_urlencoded::byte_serialize(token.as_bytes())); + body.push_str("&token_type_hint=access_token"); + let mut request = self + .http + .request(Method::POST, endpoint) + .map_err(|err| OidcError::Config(format!("Invalid introspection URL: {err}")))? + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .header(ACCEPT, "application/json"); + match auth { + IntrospectionAuth::Basic { authorization } => { + request = request.header(AUTHORIZATION, authorization.as_ref()); } + IntrospectionAuth::Post { credentials } => body.push_str(credentials), + IntrospectionAuth::Unauthenticated => {} } - let new_keys = fetch_jwks_keys(&self.http, &self.discovery.document.jwks_uri).await?; + let response = request + .body(body) + .send() + .await + .map_err(|e| OidcError::Network(format!("Introspection request failed: {e}")))?; + let status = response.status(); + if !status.is_success() { + return Err(OidcError::Provider(format!( + "Introspection endpoint returned HTTP {status}" + ))); + } + let bytes = read_body(response, CLAIMS_MAX_SIZE, "Introspection").await?; + let claims = serde_json::from_slice::(&bytes) + .map_err(|e| OidcError::Provider(format!("Introspection JSON parse error: {e}")))?; + + if claims.get("active").and_then(Value::as_bool) != Some(true) { + return Err(OidcError::TokenValidation( + "Token is not active".to_string(), + )); + } + if has_non_access_marker(&claims) { + return Err(OidcError::TokenValidation( + "Introspected token is not an access token".to_string(), + )); + } + if let Some(issuer) = claims.get("iss").and_then(Value::as_str) + && !self.accepts_issuer(issuer) { - let mut guard = self.cache.write().await; - guard.keys = new_keys; - guard.last_updated = Instant::now(); + return Err(OidcError::TokenValidation(format!( + "Introspected token was issued by '{issuer}'" + ))); } - let guard = self.cache.read().await; - if let Some(kid) = kid { - if let Some(cached) = guard.keys.get(kid) { - Ok(vec![cached.clone()]) - } else { - Err(OidcError::TokenValidation( - "Unknown key id after refresh".to_string(), - )) + self.check_audience(&claims, ctx)?; + let access = self.token_access(client_claim(&claims)?, token_scopes(&claims), ctx)?; + let username = self.resolve_username(&claims)?; + + Ok(ExternalToken { + account: self.build_account(username, &claims), + access, + expires_at: claims.get("exp").and_then(Value::as_u64), + }) + } + + fn check_audience(&self, claims: &Value, ctx: &TokenContext<'_>) -> Result<(), OidcError> { + if self.config.skip_audience_validation { + return Ok(()); + } + let is_accepted = |audience: &str| { + self.config + .accepted_audiences + .iter() + .any(|accepted| accepted.as_ref() == audience) + || (ctx.is_resource)(audience) + }; + let accepted = match claims.get("aud") { + Some(Value::String(audience)) => is_accepted(audience), + Some(Value::Array(audiences)) => { + audiences.iter().filter_map(Value::as_str).any(is_accepted) } + _ => false, + }; + if accepted { + Ok(()) } else { - let all: Vec<_> = guard.keys.values().cloned().collect(); - if all.is_empty() { - Err(OidcError::Provider( - "JWKS contains no usable keys".to_string(), - )) - } else { - Ok(all) + Err(OidcError::AuthorizationFailed(format!( + "Token audience {} matches neither a resource identifier of this server nor acceptedAudiences", + claims.get("aud").unwrap_or(&Value::Null) + ))) + } + } + + fn token_access( + &self, + client_id: Option<&str>, + scopes: ExternalScopes, + ctx: &TokenContext<'_>, + ) -> Result { + if client_id.is_some_and(|client_id| (ctx.is_first_party)(client_id)) { + Ok(TokenAccess::Unrestricted) + } else if !scopes.is_empty() { + Ok(TokenAccess::Scoped(scopes)) + } else { + match self.config.unscoped_tokens { + OidcUnscopedTokenPolicy::Reject => Err(OidcError::AuthorizationFailed( + "Token carries none of the urn:ietf:params:oauth:scope:{mail,contacts,calendars,files} scopes and unscopedTokens is set to reject".to_string(), + )), + OidcUnscopedTokenPolicy::Interoperable => { + Ok(TokenAccess::Scoped(ExternalScopes::ALL)) + } + OidcUnscopedTokenPolicy::Full => Ok(TokenAccess::Unrestricted), } } } - async fn fetch_userinfo(&self, token: &str) -> Result { + async fn fetch_userinfo(&self, endpoint: &str, token: &str) -> Result { let resp = self .http - .get(&self.discovery.document.userinfo_endpoint) + .request(Method::GET, endpoint) + .map_err(|err| OidcError::Config(format!("Invalid UserInfo URL: {err}")))? .bearer_auth(token) .send() .await .map_err(|e| OidcError::Network(format!("UserInfo request failed: {e}")))?; let status = resp.status(); - if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { - let reason = resp.text().await.unwrap_or_default(); + if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN { + let reason = resp + .bytes_with_limit(ERROR_MAX_SIZE) + .await + .ok() + .flatten() + .unwrap_or_default(); return Err(OidcError::AuthorizationFailed(format!( - "Token rejected by UserInfo endpoint with status {status}: {reason}" + "Token rejected by UserInfo endpoint with status {status}: {}", + String::from_utf8_lossy(&reason) ))); } if !status.is_success() { @@ -200,37 +353,14 @@ impl OpenIdDirectory { ))); } - let bytes = resp - .bytes() - .await - .map_err(|e| OidcError::Provider(format!("UserInfo HTTP error: {e}")))?; + let bytes = read_body(resp, CLAIMS_MAX_SIZE, "UserInfo").await?; - serde_json::from_slice::(&bytes) + serde_json::from_slice::(&bytes) .map_err(|e| OidcError::Provider(format!("UserInfo JSON parse error: {e}"))) } - fn validate_scopes(&self, claims: &serde_json::Value) -> Result<(), OidcError> { - if !self.config.require_scopes.is_empty() { - let token_scopes = extract_scopes(claims); - - for required in &self.config.require_scopes { - if !token_scopes.iter().any(|s| s == required) { - return Err(OidcError::AuthorizationFailed(format!( - "Missing required scope '{required}', present scopes: {token_scopes:?}" - ))); - } - } - } - - Ok(()) - } - - fn build_account( - &self, - email: String, - claims: &serde_json::Value, - ) -> Result { - Ok(Account { + fn build_account(&self, email: String, claims: &Value) -> Account { + Account { email, email_aliases: Vec::new(), secret: None, @@ -243,7 +373,7 @@ impl OpenIdDirectory { .map(|groups| { groups .into_iter() - .map(|group| match &self.config.default_domain { + .map(|group| match &self.config.username_domain { Some(domain) if !group.contains('@') => format!("{group}@{domain}"), _ => group, }) @@ -257,23 +387,23 @@ impl OpenIdDirectory { .and_then(|v| v.as_str()) .filter(|name| !name.is_empty()) .map(|s| s.to_string()), - }) + } } - fn resolve_email(&self, claims: &serde_json::Value) -> Result { + fn resolve_username(&self, claims: &Value) -> Result { if let Some(val) = claims - .get(&self.config.claim_email) + .get(&self.config.claim_username) .and_then(|v| v.as_str()) { if val.contains('@') { return Ok(val.to_string()); } - if let Some(domain) = &self.config.default_domain { + if let Some(domain) = &self.config.username_domain { return Ok(format!("{val}@{domain}")); } } - if self.config.claim_email != "email" + if self.config.claim_username != "email" && let Some(val) = claims.get("email").and_then(|v| v.as_str()) && val.contains('@') { @@ -286,181 +416,109 @@ impl OpenIdDirectory { } } -pub(super) async fn fetch_jwks_keys( - http: &Client, - jwks_uri: &str, -) -> Result>, OidcError> { - let jwks_bytes = http - .get(jwks_uri) - .send() - .await - .map_err(|e| OidcError::Network(format!("JWKS fetch failed: {e}")))? - .error_for_status() - .map_err(|e| OidcError::Provider(format!("JWKS HTTP error: {e}")))? - .bytes() - .await - .map_err(|e| OidcError::Provider(format!("JWKS HTTP error: {e}")))?; - let jwks: JwkSet = serde_json::from_slice(&jwks_bytes) - .map_err(|e| OidcError::Provider(format!("JWKS JSON parse error: {e}")))?; +fn check_token_type( + header_typ: Option<&str>, + claims: &Value, + allow_id_token_claims: bool, +) -> Result<(), OidcError> { + let claim_is = |name: &str, value: &str| claim_equals(claims, name, value); - let mut map = AHashMap::new(); - let mut synthetic_id: u64 = 0; + if has_non_access_marker(claims) { + Err(OidcError::TokenValidation( + "ID and refresh tokens are not accepted as access tokens".to_string(), + )) + } else if header_typ.is_some_and(|typ| { + typ.eq_ignore_ascii_case("at+jwt") || typ.eq_ignore_ascii_case("application/at+jwt") + }) || claim_is("typ", "bearer") + || claim_is("token_use", "access") + || claim_is("tokenType", "access-token") + || allow_id_token_claims + { + Ok(()) + } else if claims.get("nonce").is_some() || claims.get("at_hash").is_some() { + Err(OidcError::TokenValidation( + "Token carries ID token claims (nonce or at_hash), enable allowIdTokenClaims if the IdP issues such access tokens".to_string(), + )) + } else if ACCESS_TOKEN_CLAIMS + .iter() + .any(|claim| claims.get(claim).is_some_and(|value| !value.is_null())) + { + Ok(()) + } else { + Err(OidcError::TokenValidation( + "Token has no scope or client_id claim and may be an ID token, enable allowIdTokenClaims if the IdP issues such access tokens".to_string(), + )) + } +} - for key in &jwks.keys { - if let Some(pk_use) = &key.common.public_key_use - && pk_use != &jwk::PublicKeyUse::Signature - { - continue; +fn claim_equals(claims: &Value, name: &str, value: &str) -> bool { + claims + .get(name) + .and_then(Value::as_str) + .is_some_and(|claim| claim.eq_ignore_ascii_case(value)) +} + +fn has_non_access_marker(claims: &Value) -> bool { + NON_ACCESS_MARKERS + .iter() + .any(|(name, value)| claim_equals(claims, name, value)) +} + +fn client_claim(claims: &Value) -> Result, OidcError> { + let mut client_id = None; + for value in CLIENT_CLAIMS + .iter() + .filter_map(|claim| claims.get(claim).and_then(Value::as_str)) + { + match client_id { + None => client_id = Some(value), + Some(previous) if previous == value => {} + Some(_) => { + return Err(OidcError::TokenValidation( + "Token carries conflicting client claims".to_string(), + )); + } } - - let algorithm = match &key.algorithm { - jwk::AlgorithmParameters::RSA(_) => match key.common.key_algorithm { - Some(jwk::KeyAlgorithm::RS256) => Algorithm::RS256, - Some(jwk::KeyAlgorithm::RS384) => Algorithm::RS384, - Some(jwk::KeyAlgorithm::RS512) => Algorithm::RS512, - Some(jwk::KeyAlgorithm::PS256) => Algorithm::PS256, - Some(jwk::KeyAlgorithm::PS384) => Algorithm::PS384, - Some(jwk::KeyAlgorithm::PS512) => Algorithm::PS512, - None => Algorithm::RS256, - Some(other) => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!("Unsupported RSA key algorithm {:?}", other) - ); - continue; - } - }, - jwk::AlgorithmParameters::EllipticCurve(ec) => match ec.curve { - jwk::EllipticCurve::P256 => Algorithm::ES256, - jwk::EllipticCurve::P384 => Algorithm::ES384, - _ => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!("Unsupported EC curve {:?}", ec.curve) - ); - continue; - } - }, - jwk::AlgorithmParameters::OctetKeyPair(_) => Algorithm::EdDSA, - jwk::AlgorithmParameters::OctetKey(_) => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!( - "Symmetric (HMAC) key found in JWKS (kid={:?}), skipping — HMAC is not accepted", - key.common.key_id - ) - ); - continue; - } - _ => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!( - "Unrecognised key type in JWKS (kid={:?}), skipping", - key.common.key_id - ) - ); - continue; - } - }; - - let decoding_key = match DecodingKey::from_jwk(key) { - Ok(decoding_key) => decoding_key, - Err(e) => { - trc::event!( - Auth(AuthEvent::Warning), - Url = jwks_uri.to_string(), - Reason = format!( - "Failed to build DecodingKey from JWK (kid={:?}): {e}", - key.common.key_id - ) - ); - continue; - } - }; - - map.insert( - match &key.common.key_id { - Some(id) => id.clone(), - None => { - let id = format!("_synthetic_{synthetic_id}"); - synthetic_id += 1; - id - } - }, - CachedKey { - decoding_key, - algorithm, - } - .into(), - ); } - - Ok(map) + Ok(client_id) } -fn extract_scopes(claims: &serde_json::Value) -> Vec { - match claims.get("scope") { - Some(serde_json::Value::String(s)) => s.split_whitespace().map(|s| s.to_string()).collect(), - Some(serde_json::Value::Array(arr)) => arr - .iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect(), - _ => Vec::new(), +fn token_scopes(claims: &Value) -> ExternalScopes { + let mut scopes = ExternalScopes::NONE; + match claims + .get("scope") + .filter(|value| !value.is_null()) + .or_else(|| claims.get("scp")) + { + Some(Value::String(names)) => { + for name in names.split_ascii_whitespace() { + scopes.insert(ExternalScopes::from_name(name)); + } + } + Some(Value::Array(names)) => { + for name in names.iter().filter_map(Value::as_str) { + scopes.insert(ExternalScopes::from_name(name)); + } + } + _ => {} } + scopes } -fn extract_string_list(value: &serde_json::Value) -> Option> { +fn extract_string_list(value: &Value) -> Option> { match value { - serde_json::Value::Array(arr) => Some( + Value::Array(arr) => Some( arr.iter() .filter_map(|v| v.as_str().map(|s| s.to_string())) .collect(), ), - serde_json::Value::String(s) => Some(s.split_whitespace().map(|s| s.to_string()).collect()), + Value::String(s) => Some(s.split_whitespace().map(|s| s.to_string()).collect()), + Value::Object(map) => Some(map.keys().cloned().collect()), _ => None, } } -#[inline(always)] -fn is_claim_missing(claims: &serde_json::Value, claim: Option<&String>) -> bool { - claim.is_some_and(|claim| claims.get(claim).is_none_or(serde_json::Value::is_null)) +fn is_claim_missing(claims: &Value, claim: Option<&str>) -> bool { + claim.is_some_and(|claim| claims.get(claim).is_none_or(Value::is_null)) } -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_string_list_distinguishes_absent_from_empty() { - assert_eq!( - extract_string_list(&json!(["sales", "support"])), - Some(vec!["sales".to_string(), "support".to_string()]) - ); - assert_eq!( - extract_string_list(&json!("sales support")), - Some(vec!["sales".to_string(), "support".to_string()]) - ); - assert_eq!(extract_string_list(&json!([])), Some(vec![])); - assert_eq!(extract_string_list(&json!("")), Some(vec![])); - assert_eq!(extract_string_list(&json!(null)), None); - assert_eq!(extract_string_list(&json!({"groups": []})), None); - assert_eq!(extract_string_list(&json!(42)), None); - } - - #[test] - fn is_claim_missing_treats_null_as_absent() { - let claims = json!({"groups": null, "name": "John Doe", "roles": []}); - - assert!(is_claim_missing(&claims, Some(&"groups".to_string()))); - assert!(is_claim_missing(&claims, Some(&"unknown".to_string()))); - assert!(!is_claim_missing(&claims, Some(&"name".to_string()))); - assert!(!is_claim_missing(&claims, Some(&"roles".to_string()))); - assert!(!is_claim_missing(&claims, None)); - } -} diff --git a/crates/directory/src/backend/oidc/mod.rs b/crates/directory/src/backend/oidc/mod.rs index 0ddb9310..02372b9f 100644 --- a/crates/directory/src/backend/oidc/mod.rs +++ b/crates/directory/src/backend/oidc/mod.rs @@ -4,68 +4,305 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::Account; use ahash::AHashMap; -use jsonwebtoken::{Algorithm, DecodingKey}; +use jsonwebtoken::{DecodingKey, Validation}; +use parking_lot::RwLock; +use registry::schema::enums::OidcUnscopedTokenPolicy; use serde::{Deserialize, Serialize}; -use std::{fmt, sync::Arc, time::Instant}; -use tokio::sync::RwLock; -use utils::Client; +use std::{ + fmt, + sync::Arc, + time::{Duration, Instant}, +}; +use utils::{ + HttpLimitResponse, + outbound::{OutboundClient, OutboundPolicy}, +}; pub mod config; +pub mod jwks; pub mod lookup; +pub const OPENID_CONFIGURATION_PATH: &str = "/.well-known/openid-configuration"; +pub const OAUTH_METADATA_PATH: &str = "/.well-known/oauth-authorization-server"; + +pub(crate) const METADATA_MAX_SIZE: usize = 256 * 1024; +pub(crate) const CLAIMS_MAX_SIZE: usize = 64 * 1024; +pub(crate) const ERROR_MAX_SIZE: usize = 512; + +pub(crate) async fn read_body( + response: reqwest::Response, + limit: usize, + what: &str, +) -> Result, OidcError> { + match response.bytes_with_limit(limit).await { + Ok(Some(bytes)) => Ok(bytes), + Ok(None) => Err(OidcError::Provider(format!( + "{what} response exceeds {limit} bytes" + ))), + Err(err) => Err(OidcError::Network(format!("{what} request failed: {err}"))), + } +} + +pub fn openid_configuration_url(issuer: &str) -> String { + let issuer = issuer.trim_end_matches('/'); + let mut url = String::with_capacity(issuer.len() + OPENID_CONFIGURATION_PATH.len()); + url.push_str(issuer); + url.push_str(OPENID_CONFIGURATION_PATH); + url +} + pub struct OidcConfig { - pub issue_url: String, - pub require_aud: Option, - pub require_scopes: Vec, - pub claim_email: String, + pub issuer_url: String, + pub claim_username: String, pub claim_name: Option, pub claim_groups: Option, - pub default_domain: Option, + pub username_domain: Option, + pub allow_invalid_certs: bool, + pub outbound_policy: OutboundPolicy, + pub accepted_audiences: Box<[Box]>, + pub opaque_validation: OpaqueValidation, + pub unscoped_tokens: OidcUnscopedTokenPolicy, + pub skip_audience_validation: bool, + pub allow_id_token_claims: bool, + pub lenient_issuer_match: bool, +} + +pub enum OpaqueValidation { + Reject, + UserInfo, + Introspection(IntrospectionAuth), +} + +pub enum IntrospectionAuth { + Basic { authorization: Box }, + Post { credentials: Box }, + Unauthenticated, } pub struct OidcDiscovery { - pub url: String, pub document: DiscoveryDocument, + pub metadata_url: String, + pub conformance: Conformance, + pub web_document: Arc<[u8]>, +} + +impl OidcDiscovery { + pub fn configuration_url(&self) -> &str { + &self.metadata_url + } } #[derive(Deserialize, Serialize)] pub struct DiscoveryDocument { pub issuer: String, pub jwks_uri: String, - pub userinfo_endpoint: String, pub token_endpoint: String, pub authorization_endpoint: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub userinfo_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub end_session_endpoint: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub registration_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub introspection_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub scopes_supported: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub claims_supported: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub code_challenge_methods_supported: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_endpoint_auth_methods_supported: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub grant_types_supported: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response_types_supported: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authorization_response_iss_parameter_supported: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_id_metadata_document_supported: Option, } -struct CachedKey { +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Conformance { + pub gaps: ConformanceGaps, + pub scopes: ExternalScopes, +} + +impl Conformance { + pub fn serves_oauth_public(&self, required: ExternalScopes) -> bool { + self.gaps.is_empty() && self.scopes.contains(required) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ConformanceGaps(u16); + +impl ConformanceGaps { + pub const ISSUER: Self = Self(1 << 0); + pub const CONTENT_TYPE: Self = Self(1 << 1); + pub const REGISTRATION_ENDPOINT: Self = Self(1 << 2); + pub const AUTHORIZATION_ENDPOINT: Self = Self(1 << 3); + pub const TOKEN_ENDPOINT: Self = Self(1 << 4); + pub const RESPONSE_TYPE_CODE: Self = Self(1 << 5); + pub const GRANT_TYPES: Self = Self(1 << 6); + pub const AUTH_METHOD_NONE: Self = Self(1 << 7); + pub const PKCE_S256: Self = Self(1 << 8); + pub const ISS_PARAMETER: Self = Self(1 << 9); + + const NAMES: [(Self, &'static str); 10] = [ + (Self::ISSUER, "issuer is not a valid https identifier"), + ( + Self::CONTENT_TYPE, + "metadata is not served as application/json", + ), + (Self::REGISTRATION_ENDPOINT, "registration_endpoint"), + (Self::AUTHORIZATION_ENDPOINT, "authorization_endpoint"), + (Self::TOKEN_ENDPOINT, "token_endpoint"), + (Self::RESPONSE_TYPE_CODE, "response_types_supported: code"), + ( + Self::GRANT_TYPES, + "grant_types_supported: authorization_code, refresh_token", + ), + ( + Self::AUTH_METHOD_NONE, + "token_endpoint_auth_methods_supported: none", + ), + (Self::PKCE_S256, "code_challenge_methods_supported: S256"), + ( + Self::ISS_PARAMETER, + "authorization_response_iss_parameter_supported", + ), + ]; + + pub fn is_empty(self) -> bool { + self.0 == 0 + } + + pub fn insert(&mut self, gap: Self) { + self.0 |= gap.0; + } + + pub fn contains(self, gap: Self) -> bool { + self.0 & gap.0 == gap.0 + } + + pub fn names(self) -> impl Iterator { + Self::NAMES + .into_iter() + .filter(move |(gap, _)| self.contains(*gap)) + .map(|(_, name)| name) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ExternalScopes(u8); + +impl ExternalScopes { + pub const NONE: Self = Self(0); + pub const MAIL: Self = Self(1 << 0); + pub const CONTACTS: Self = Self(1 << 1); + pub const CALENDARS: Self = Self(1 << 2); + pub const FILES: Self = Self(1 << 3); + pub const ALL: Self = Self(Self::MAIL.0 | Self::CONTACTS.0 | Self::CALENDARS.0 | Self::FILES.0); + + const NAMES: [(Self, &'static str); 4] = [ + (Self::MAIL, "urn:ietf:params:oauth:scope:mail"), + (Self::CONTACTS, "urn:ietf:params:oauth:scope:contacts"), + (Self::CALENDARS, "urn:ietf:params:oauth:scope:calendars"), + (Self::FILES, "urn:ietf:params:oauth:scope:files"), + ]; + + pub fn from_name(name: &str) -> Self { + hashify::map!( + name.as_bytes(), + ExternalScopes, + "urn:ietf:params:oauth:scope:mail" => ExternalScopes::MAIL, + "urn:ietf:params:oauth:scope:contacts" => ExternalScopes::CONTACTS, + "urn:ietf:params:oauth:scope:calendars" => ExternalScopes::CALENDARS, + "urn:ietf:params:oauth:scope:files" => ExternalScopes::FILES, + ) + .copied() + .unwrap_or_default() + } + + pub const fn bits(self) -> u8 { + self.0 + } + + pub fn is_empty(self) -> bool { + self.0 == 0 + } + + pub fn insert(&mut self, scopes: Self) { + self.0 |= scopes.0; + } + + pub fn contains(self, scopes: Self) -> bool { + self.0 & scopes.0 == scopes.0 + } + + pub fn difference(self, scopes: Self) -> Self { + Self(self.0 & !scopes.0) + } + + pub fn names(self) -> impl Iterator { + Self::NAMES + .into_iter() + .filter(move |(scope, _)| self.contains(*scope)) + .map(|(_, name)| name) + } +} + +pub struct TokenContext<'x> { + pub is_resource: &'x (dyn Fn(&str) -> bool + Sync), + pub is_first_party: &'x (dyn Fn(&str) -> bool + Sync), +} + +pub struct ExternalToken { + pub account: Account, + pub access: TokenAccess, + pub expires_at: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TokenAccess { + Unrestricted, + Scoped(ExternalScopes), +} + +pub(crate) struct CachedKey { decoding_key: DecodingKey, - algorithm: Algorithm, + validation: Validation, } -struct JwksCache { - keys: AHashMap>, - last_updated: Instant, +pub(crate) struct KeySet { + keys: AHashMap, CachedKey>, +} + +struct JwksState { + keys: Arc, + fetched_at: Instant, + attempted_at: Instant, + max_age: Duration, } pub struct OpenIdDirectory { config: OidcConfig, pub discovery: OidcDiscovery, - http: Client, - cache: RwLock, + issuers: Box<[String]>, + http: OutboundClient, + jwks: RwLock, + jwks_refresh: tokio::sync::Mutex<()>, } #[derive(Debug)] pub enum OidcError { TokenValidation(String), + TokenExpired(String), AuthorizationFailed(String), Network(String), Provider(String), @@ -82,6 +319,7 @@ impl fmt::Display for OidcError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { OidcError::TokenValidation(msg) => write!(f, "Token validation error: {msg}"), + OidcError::TokenExpired(msg) => write!(f, "Token expired: {msg}"), OidcError::AuthorizationFailed(msg) => write!(f, "Authorization failed: {msg}"), OidcError::Network(msg) => write!(f, "Network error: {msg}"), OidcError::Provider(msg) => write!(f, "Provider error: {msg}"), diff --git a/crates/directory/src/backend/sql/lookup.rs b/crates/directory/src/backend/sql/lookup.rs index 16cf4f54..18568153 100644 --- a/crates/directory/src/backend/sql/lookup.rs +++ b/crates/directory/src/backend/sql/lookup.rs @@ -5,13 +5,21 @@ */ use super::{SqlDirectory, SqlMappings}; -use crate::{Account, Credentials, Recipient, core::secret::verify_secret_hash}; +use crate::{ + Account, Credentials, Recipient, + core::{secret::verify_secret_hash, verifier::PasswordVerifier}, +}; +use compact_str::CompactString; use store::{NamedRows, Rows, Value}; use trc::AddContext; use utils::sanitize_email; impl SqlDirectory { - pub async fn authenticate(&self, credentials: &Credentials) -> trc::Result { + pub async fn authenticate( + &self, + credentials: &Credentials, + verifier: &PasswordVerifier, + ) -> trc::Result { let (username, secret) = match credentials { Credentials::Basic { username, secret, .. @@ -19,14 +27,14 @@ impl SqlDirectory { Credentials::Bearer { .. } => { return Err(trc::AuthEvent::Error .into_err() - .details("Unsupported credentials type for SQL authentication")); + .reason("Unsupported credentials type for SQL authentication")); } }; if secret.is_empty() { return Err(trc::AuthEvent::Failed .into_err() - .details("Empty secret rejected") - .ctx(trc::Key::AccountName, username.to_string())); + .reason("Empty secret rejected") + .ctx(trc::Key::AccountName, CompactString::from(username))); } let Recipient::Account(mut account) = self.mappings.row_to_account( @@ -37,59 +45,27 @@ impl SqlDirectory { ) else { return Err(trc::AuthEvent::Failed .into_err() - .details("SQL login query did not return an account") - .ctx(trc::Key::AccountName, username.to_string())); + .reason("SQL login query did not return an account") + .ctx(trc::Key::AccountName, CompactString::from(username))); }; // Validate secret if let Some(account_secret) = &account.secret { - if !verify_secret_hash(account_secret, secret.as_bytes()).await? { + if !verify_secret_hash(verifier, account_secret, secret.as_bytes()).await? { return Err(trc::AuthEvent::Failed .into_err() - .details("Invalid credentials") - .ctx(trc::Key::AccountName, username.to_string())); + .reason("Invalid credentials") + .ctx(trc::Key::AccountName, CompactString::from(username))); } } else { - return Err(trc::AuthEvent::Error + return Err(trc::AuthEvent::Failed .into_err() - .details("Account does not have a secret") - .ctx(trc::Key::AccountName, username.to_string())); + .reason("Account does not have a secret") + .ctx(trc::Key::AccountName, CompactString::from(username))); } - // Obtain members - if let Some(query) = &self.mappings.query_member_of { - let members = account.groups.get_or_insert_default(); - for row in self - .sql_store - .sql_query::(query, vec![username.into()]) - .await - .caused_by(trc::location!())? - .rows - { - if let Some(Value::Text(address)) = row.values.first() - && let Some(email) = sanitize_email(address) - { - members.push(email); - } - } - } - - // Obtain emails - if let Some(query) = &self.mappings.query_email_aliases { - account.email_aliases.extend( - self.sql_store - .sql_query::(query, vec![username.into()]) - .await - .caused_by(trc::location!())? - .rows - .into_iter() - .flat_map(|v| { - v.values - .into_iter() - .filter_map(|v| sanitize_email(v.to_str().as_ref())) - }), - ); - } + let (members, aliases) = self.members_and_aliases(username).await?; + account.add_members_and_aliases(members, aliases); if account.email.is_empty() { account.email = sanitize_email(username).unwrap_or_else(|| username.to_lowercase()); @@ -108,47 +84,57 @@ impl SqlDirectory { match recipient { Recipient::Account(mut account) => { - // Obtain members - if let Some(query) = &self.mappings.query_member_of { - let members = account.groups.get_or_insert_default(); - for row in self - .sql_store - .sql_query::(query, vec![account.email.as_str().into()]) - .await - .caused_by(trc::location!())? - .rows - { - if let Some(Value::Text(address)) = row.values.first() - && let Some(email) = sanitize_email(address) - { - members.push(email); - } - } - } - - // Obtain emails - if let Some(query) = &self.mappings.query_email_aliases { - account.email_aliases.extend( - self.sql_store - .sql_query::(query, vec![account.email.as_str().into()]) - .await - .caused_by(trc::location!())? - .rows - .into_iter() - .flat_map(|v| { - v.values - .into_iter() - .filter_map(|v| sanitize_email(v.to_str().as_ref())) - }), - ); - } - + let (members, aliases) = self.members_and_aliases(&account.email).await?; + account.add_members_and_aliases(members, aliases); Ok(Recipient::Account(account)) } Recipient::Group(group) => Ok(Recipient::Group(group)), Recipient::Invalid => Ok(Recipient::Invalid), } } + + async fn members_and_aliases(&self, key: &str) -> trc::Result<(Option, Option)> { + let members = match &self.mappings.query_member_of { + Some(query) => Some(self.query_rows(query, key).await?), + None => None, + }; + let aliases = match &self.mappings.query_email_aliases { + Some(query) => Some(self.query_rows(query, key).await?), + None => None, + }; + Ok((members, aliases)) + } + + async fn query_rows(&self, query: &str, key: &str) -> trc::Result { + self.sql_store + .sql_query::(query, vec![key.into()]) + .await + .caused_by(trc::location!()) + } +} + +impl Account { + fn add_members_and_aliases(&mut self, members: Option, aliases: Option) { + if let Some(members) = members { + let groups = self.groups.get_or_insert_default(); + for row in members.rows { + if let Some(Value::Text(address)) = row.values.first() + && let Some(email) = sanitize_email(address) + { + groups.push(email); + } + } + } + + if let Some(aliases) = aliases { + self.email_aliases + .extend(aliases.rows.into_iter().flat_map(|v| { + v.values + .into_iter() + .filter_map(|v| sanitize_email(v.to_str().as_ref())) + })); + } + } } impl SqlMappings { diff --git a/crates/directory/src/core/dispatch.rs b/crates/directory/src/core/dispatch.rs index 8ecd887a..27f62455 100644 --- a/crates/directory/src/core/dispatch.rs +++ b/crates/directory/src/core/dispatch.rs @@ -4,21 +4,44 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Account, Credentials, Directory, Recipient, backend::oidc::OidcDiscovery}; +use crate::{ + Account, Credentials, Directory, Recipient, + backend::oidc::{ExternalToken, OidcDiscovery, TokenContext}, + core::verifier::PasswordVerifier, +}; use registry::schema::enums::DirectoryType; use trc::AddContext; impl Directory { - pub async fn authenticate(&self, credentials: &Credentials) -> trc::Result { + pub async fn authenticate( + &self, + credentials: &Credentials, + verifier: &PasswordVerifier, + ) -> trc::Result { match &self { - Directory::Ldap(store) => store.authenticate(credentials).await, - Directory::Sql(store) => store.authenticate(credentials).await, + Directory::Ldap(store) => store.authenticate(credentials, verifier).await, + Directory::Sql(store) => store.authenticate(credentials, verifier).await, Directory::OpenId(store) => store.authenticate(credentials).await, Directory::Unavailable(directory) => Err(directory.error()), } .caused_by(trc::location!()) } + pub async fn authenticate_token( + &self, + token: &str, + ctx: &TokenContext<'_>, + ) -> trc::Result { + match &self { + Directory::OpenId(store) => store.authenticate_token(token, ctx).await, + Directory::Unavailable(directory) => Err(directory.error()), + _ => Err(trc::AuthEvent::Error + .into_err() + .reason("Directory does not support bearer tokens")), + } + .caused_by(trc::location!()) + } + pub async fn recipient(&self, address: &str) -> trc::Result { match &self { Directory::Ldap(store) => store.recipient(address).await, @@ -45,6 +68,13 @@ impl Directory { } } + pub fn accepts_oidc_issuer(&self, issuer: &str) -> bool { + match &self { + Directory::OpenId(directory) => directory.accepts_issuer(issuer), + _ => false, + } + } + pub fn oidc_discovery_document(&self) -> Option<&OidcDiscovery> { match &self { Directory::OpenId(directory) => Some(&directory.discovery), diff --git a/crates/directory/src/core/mod.rs b/crates/directory/src/core/mod.rs index 799b5fd9..538c09bb 100644 --- a/crates/directory/src/core/mod.rs +++ b/crates/directory/src/core/mod.rs @@ -8,3 +8,4 @@ pub mod config; pub mod dispatch; pub mod sasl; pub mod secret; +pub mod verifier; diff --git a/crates/directory/src/core/sasl.rs b/crates/directory/src/core/sasl.rs index 9e7763ea..111b53cb 100644 --- a/crates/directory/src/core/sasl.rs +++ b/crates/directory/src/core/sasl.rs @@ -5,35 +5,19 @@ */ use crate::Credentials; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use encodify::base64::URL_SAFE_NO_PAD; impl Credentials { pub fn decode_sasl_challenge_plain(challenge: &[u8]) -> Option { - let mut username = Vec::new(); - let mut secret = Vec::new(); - let mut arg_num = 0; - for &ch in challenge { - if ch != 0 { - if arg_num == 1 { - username.push(ch); - } else if arg_num == 2 { - secret.push(ch); - } - } else { - arg_num += 1; - } - } + let mut args = challenge.split(|&ch| ch == 0).skip(1); + let username = args.next().filter(|username| !username.is_empty())?; + let secret = args.next().filter(|secret| !secret.is_empty())?; - match (String::from_utf8(username), String::from_utf8(secret)) { - (Ok(username), Ok(secret)) if !username.is_empty() && !secret.is_empty() => { - Some(Credentials::Basic { - username, - secret, - mfa_token: None, - }) - } - _ => None, - } + Some(Credentials::Basic { + username: std::str::from_utf8(username).ok()?.to_string(), + secret: std::str::from_utf8(secret).ok()?.to_string(), + mfa_token: None, + }) } pub fn decode_sasl_challenge_oauth(challenge: &[u8]) -> Option { @@ -125,55 +109,3 @@ fn extract_email_from_jwt(token: &str) -> Option { .find(|v| v.contains('@')) } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_extract_oauth_bearer() { - let input = b"auth=Bearer validtoken"; - let result = extract_oauth_bearer(input); - assert_eq!(result, Some(("validtoken".to_string(), None))); - - let input = b"auth=Invalid validtoken"; - let result = extract_oauth_bearer(input); - assert_eq!(result, None); - - let input = b"auth=Bearer"; - let result = extract_oauth_bearer(input); - assert_eq!(result, None); - - let input = b""; - let result = extract_oauth_bearer(input); - assert_eq!(result, None); - - let input = b"auth=Bearer token1\x01auth=Bearer token2"; - let result = extract_oauth_bearer(input); - assert_eq!(result, Some(("token1".to_string(), None))); - - let input = b"auth=Bearer VALIDTOKEN"; - let result = extract_oauth_bearer(input); - assert_eq!(result, Some(("VALIDTOKEN".to_string(), None))); - - let input = b"auth=Bearer token with spaces"; - let result = extract_oauth_bearer(input); - assert_eq!(result, Some(("token with spaces".to_string(), None))); - - let input = b"auth=Bearer token_with_special_chars!@#"; - let result = extract_oauth_bearer(input); - assert_eq!( - result, - Some(("token_with_special_chars!@#".to_string(), None)) - ); - - let input = "n,a=user@example.com,\x01host=server.example.com\x01port=143\x01auth=Bearer vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==\x01\x01"; - let result = extract_oauth_bearer(input.as_bytes()); - assert_eq!( - result, - Some(( - "vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==".to_string(), - Some("user@example.com".to_string()) - )) - ); - } -} diff --git a/crates/directory/src/core/secret.rs b/crates/directory/src/core/secret.rs index f0d2c8fb..1deeeb71 100644 --- a/crates/directory/src/core/secret.rs +++ b/crates/directory/src/core/secret.rs @@ -7,9 +7,9 @@ use argon2::Argon2; use argon2::PasswordHash; use argon2::PasswordHasher; -use argon2::PasswordVerifier; -use mail_builder::encoders::Base64Encoder; -use mail_parser::decoders::base64::base64_decode; +use argon2::PasswordVerifier as _; +use compact_str::CompactString; +use encodify::base64::{Base64, LENIENT, Padding, STANDARD}; use pbkdf2::Pbkdf2; use pwhash::{bcrypt, bsdi_crypt, md5_crypt, sha1_crypt, sha256_crypt, sha512_crypt, unix_crypt}; use registry::schema::enums::PasswordHashAlgorithm; @@ -18,9 +18,10 @@ use sha1::Digest; use sha1::Sha1; use sha2::Sha256; use sha2::Sha512; -use tokio::sync::oneshot; use totp_rs::Totp; +use super::verifier::PasswordVerifier; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SecretVerificationResult { Valid, @@ -29,6 +30,7 @@ pub enum SecretVerificationResult { } pub async fn verify_mfa_secret_hash( + verifier: &PasswordVerifier, totp_uri: Option<&str>, totp_token: Option<&str>, hashed_secret: &str, @@ -36,13 +38,9 @@ pub async fn verify_mfa_secret_hash( ) -> trc::Result { if let Some(totp_uri) = totp_uri { if let Some(totp_token) = totp_token { - let result = verify_secret_hash(hashed_secret, secret.as_bytes()).await? + let result = verify_secret_hash(verifier, hashed_secret, secret.as_bytes()).await? && Totp::from_url(totp_uri) - .map_err(|err| { - trc::AuthEvent::Error - .reason(err) - .details(totp_uri.to_string()) - })? + .map_err(|err| trc::AuthEvent::Error.reason(err))? .check_current(totp_token) .is_some(); Ok(if result { @@ -52,7 +50,7 @@ pub async fn verify_mfa_secret_hash( }) } else if !hashed_secret.is_empty() && !secret.is_empty() - && verify_secret_hash(hashed_secret, secret.as_bytes()).await? + && verify_secret_hash(verifier, hashed_secret, secret.as_bytes()).await? { // Only let the client know if the TOTP code is missing // if the password is correct @@ -62,7 +60,7 @@ pub async fn verify_mfa_secret_hash( Ok(SecretVerificationResult::Invalid) } } else if !hashed_secret.is_empty() && !secret.is_empty() { - if verify_secret_hash(hashed_secret, secret.as_bytes()).await? { + if verify_secret_hash(verifier, hashed_secret, secret.as_bytes()).await? { Ok(SecretVerificationResult::Valid) } else { Ok(SecretVerificationResult::Invalid) @@ -72,168 +70,207 @@ pub async fn verify_mfa_secret_hash( } } -async fn verify_hash_prefix(hashed_secret: &str, secret: &[u8]) -> trc::Result { - let is_argon = hashed_secret.starts_with("$argon2"); - let is_pbkdf2 = !is_argon && hashed_secret.starts_with("$pbkdf2"); - let is_scrypt = !is_argon && !is_pbkdf2 && hashed_secret.starts_with("$scrypt"); +#[derive(Debug, Clone, Copy)] +enum CryptScheme { + Argon2, + Pbkdf2, + Scrypt, + Bcrypt, + Sha512Crypt, + Sha256Crypt, + Sha1Crypt, + Md5Crypt, + BsdiCrypt, + UnixCrypt, +} - if is_argon || is_pbkdf2 || is_scrypt { - let (tx, rx) = oneshot::channel(); +impl CryptScheme { + fn from_prefix(hashed_secret: &str) -> Option { + if hashed_secret.starts_with("$argon2") { + Some(CryptScheme::Argon2) + } else if hashed_secret.starts_with("$pbkdf2") { + Some(CryptScheme::Pbkdf2) + } else if hashed_secret.starts_with("$scrypt") { + Some(CryptScheme::Scrypt) + } else if hashed_secret.starts_with("$2") { + Some(CryptScheme::Bcrypt) + } else if hashed_secret.starts_with("$6$") { + Some(CryptScheme::Sha512Crypt) + } else if hashed_secret.starts_with("$5$") { + Some(CryptScheme::Sha256Crypt) + } else if hashed_secret.starts_with("$sha1") { + Some(CryptScheme::Sha1Crypt) + } else if hashed_secret.starts_with("$1") { + Some(CryptScheme::Md5Crypt) + } else { + None + } + } + + fn verify(self, hashed_secret: &str, secret: &[u8]) -> trc::Result { + match self { + CryptScheme::Argon2 | CryptScheme::Pbkdf2 | CryptScheme::Scrypt => { + let hash = PasswordHash::new(hashed_secret) + .map_err(|err| trc::AuthEvent::Error.reason(err))?; + Ok(match self { + CryptScheme::Argon2 => Argon2::default().verify_password(secret, &hash), + CryptScheme::Pbkdf2 => Pbkdf2::default().verify_password(secret, &hash), + _ => Scrypt::default().verify_password(secret, &hash), + } + .is_ok()) + } + CryptScheme::Bcrypt => Ok(bcrypt::verify(secret, hashed_secret)), + CryptScheme::Sha512Crypt => Ok(sha512_crypt::verify(secret, hashed_secret)), + CryptScheme::Sha256Crypt => Ok(sha256_crypt::verify(secret, hashed_secret)), + CryptScheme::Sha1Crypt => Ok(sha1_crypt::verify(secret, hashed_secret)), + CryptScheme::Md5Crypt => Ok(md5_crypt::verify(secret, hashed_secret)), + CryptScheme::BsdiCrypt => Ok(bsdi_crypt::verify(secret, hashed_secret)), + CryptScheme::UnixCrypt => Ok(unix_crypt::verify(secret, hashed_secret)), + } + } + + async fn verify_blocking( + self, + verifier: &PasswordVerifier, + hashed_secret: &str, + secret: &[u8], + ) -> trc::Result { let secret = secret.to_vec(); let hashed_secret = hashed_secret.to_string(); - tokio::task::spawn_blocking(move || match PasswordHash::new(&hashed_secret) { - Ok(hash) => { - let result = if is_argon { - Argon2::default().verify_password(&secret, &hash) - } else if is_pbkdf2 { - Pbkdf2::default().verify_password(&secret, &hash) - } else { - Scrypt::default().verify_password(&secret, &hash) - }; - - tx.send(Ok(result.is_ok())).ok(); - } - Err(err) => { - tx.send(Err(trc::AuthEvent::Error - .reason(err) - .details(hashed_secret))) - .ok(); - } - }); - - match rx.await { - Ok(result) => result, - Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError) - .caused_by(trc::location!()) - .reason(err)), - } - } else if hashed_secret.starts_with("$2") { - // Blowfish crypt - Ok(bcrypt::verify(secret, hashed_secret)) - } else if hashed_secret.starts_with("$6$") { - // SHA-512 crypt - Ok(sha512_crypt::verify(secret, hashed_secret)) - } else if hashed_secret.starts_with("$5$") { - // SHA-256 crypt - Ok(sha256_crypt::verify(secret, hashed_secret)) - } else if hashed_secret.starts_with("$sha1") { - // SHA-1 crypt - Ok(sha1_crypt::verify(secret, hashed_secret)) - } else if hashed_secret.starts_with("$1") { - // MD5 based hash - Ok(md5_crypt::verify(secret, hashed_secret)) - } else { - Err(trc::AuthEvent::Error - .into_err() - .details(hashed_secret.to_string())) + verifier + .run_blocking(move || self.verify(&hashed_secret, &secret)) + .await? } } -pub async fn verify_secret_hash(hashed_secret: &str, secret: &[u8]) -> trc::Result { +async fn verify_hash_prefix( + verifier: &PasswordVerifier, + hashed_secret: &str, + secret: &[u8], +) -> trc::Result { + match CryptScheme::from_prefix(hashed_secret) { + Some(scheme) => { + scheme + .verify_blocking(verifier, hashed_secret, secret) + .await + } + None => Err(unrecognized_hash_format()), + } +} + +#[cold] +fn unrecognized_hash_format() -> trc::Error { + trc::AuthEvent::Error + .into_err() + .reason("Unrecognized password hash format") +} + +fn digest_matches(digest: &[u8], encoded: &str) -> bool { + let mut buf = [0u8; MAX_ENCODED_DIGEST_LEN]; + STANDARD + .encode_slice(digest, &mut buf) + .ok() + .and_then(|len| buf.get(..len)) + .is_some_and(|expected| expected == encoded.as_bytes()) +} + +const MAX_ENCODED_DIGEST_LEN: usize = STANDARD.encoded_len(64); + +pub async fn verify_secret_hash( + verifier: &PasswordVerifier, + hashed_secret: &str, + secret: &[u8], +) -> trc::Result { if hashed_secret.starts_with('$') { - verify_hash_prefix(hashed_secret, secret).await + verify_hash_prefix(verifier, hashed_secret, secret).await } else if hashed_secret.starts_with('_') { - // Enhanced DES-based hash - Ok(bsdi_crypt::verify(secret, hashed_secret)) + CryptScheme::BsdiCrypt + .verify_blocking(verifier, hashed_secret, secret) + .await } else if let Some(hashed_secret) = hashed_secret.strip_prefix('{') { if let Some((algo, hashed_secret)) = hashed_secret.split_once('}') { - match algo.to_ascii_uppercase().as_str() { - "ARGON2" | "ARGON2I" | "ARGON2ID" | "PBKDF2" => { - verify_hash_prefix(hashed_secret, secret).await - } + hashify::fnc_map_ignore_case!(algo.as_bytes(), + "ARGON2" => { + verify_hash_prefix(verifier, hashed_secret, secret).await + }, + "ARGON2I" => { + verify_hash_prefix(verifier, hashed_secret, secret).await + }, + "ARGON2ID" => { + verify_hash_prefix(verifier, hashed_secret, secret).await + }, + "PBKDF2" => { + verify_hash_prefix(verifier, hashed_secret, secret).await + }, "SHA" => { - // SHA-1 let mut hasher = Sha1::new(); hasher.update(secret); - Ok(String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() - == hashed_secret) - } + Ok(digest_matches(&hasher.finalize(), hashed_secret)) + }, "SSHA" => { - // Salted SHA-1 - let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); + let decoded = LENIENT.decode(hashed_secret).unwrap_or_default(); let hash = decoded.get(..20).unwrap_or_default(); let salt = decoded.get(20..).unwrap_or_default(); let mut hasher = Sha1::new(); hasher.update(secret); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) - } + }, "SHA256" => { - // Verify hash let mut hasher = Sha256::new(); hasher.update(secret); - Ok(String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() - == hashed_secret) - } + Ok(digest_matches(&hasher.finalize(), hashed_secret)) + }, "SSHA256" => { - // Salted SHA-256 - let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); + let decoded = LENIENT.decode(hashed_secret).unwrap_or_default(); let hash = decoded.get(..32).unwrap_or_default(); let salt = decoded.get(32..).unwrap_or_default(); let mut hasher = Sha256::new(); hasher.update(secret); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) - } + }, "SHA512" => { - // SHA-512 let mut hasher = Sha512::new(); hasher.update(secret); - Ok(String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() - == hashed_secret) - } + Ok(digest_matches(&hasher.finalize(), hashed_secret)) + }, "SSHA512" => { - // Salted SHA-512 - let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); + let decoded = LENIENT.decode(hashed_secret).unwrap_or_default(); let hash = decoded.get(..64).unwrap_or_default(); let salt = decoded.get(64..).unwrap_or_default(); let mut hasher = Sha512::new(); hasher.update(secret); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) - } + }, "MD5" => { - // MD5 - let digest = md5::compute(secret); - Ok(String::from_utf8( - Base64Encoder::new().encode(&digest[..]).unwrap_or_default(), - ) - .unwrap() - == hashed_secret) - } + Ok(digest_matches(&md5::compute(secret)[..], hashed_secret)) + }, "CRYPT" => { if hashed_secret.starts_with('$') { - verify_hash_prefix(hashed_secret, secret).await + verify_hash_prefix(verifier, hashed_secret, secret).await } else { - // Unix crypt - Ok(unix_crypt::verify(secret, hashed_secret)) + CryptScheme::UnixCrypt + .verify_blocking(verifier, hashed_secret, secret) + .await } + }, + "PLAIN" => { + Ok(hashed_secret.as_bytes() == secret) + }, + "CLEAR" => { + Ok(hashed_secret.as_bytes() == secret) + }, + _ => { + Err(trc::AuthEvent::Error + .ctx(trc::Key::Reason, "Unsupported password hash algorithm") + .ctx(trc::Key::Name, CompactString::from(algo))) } - "PLAIN" | "CLEAR" => Ok(hashed_secret.as_bytes() == secret), - _ => Err(trc::AuthEvent::Error - .ctx(trc::Key::Reason, "Unsupported algorithm") - .details(hashed_secret.to_string())), - } + ) } else { - Err(trc::AuthEvent::Error - .into_err() - .details(hashed_secret.to_string())) + Err(unrecognized_hash_format()) } } else if !hashed_secret.is_empty() { Ok(hashed_secret.as_bytes() == secret) @@ -242,49 +279,42 @@ pub async fn verify_secret_hash(hashed_secret: &str, secret: &[u8]) -> trc::Resu } } -pub async fn hash_secret(algorithm: PasswordHashAlgorithm, secret: Vec) -> trc::Result { - let (tx, rx) = oneshot::channel(); - - tokio::task::spawn_blocking(move || { - let result = match algorithm { - PasswordHashAlgorithm::Argon2id => { - let hasher = Argon2::default(); - hasher - .hash_password(secret.as_slice()) - .map(|h| h.to_string()) - } - PasswordHashAlgorithm::Bcrypt => { - return tx - .send(bcrypt::hash(secret.as_slice()).map_err(|err| { +pub async fn hash_secret( + verifier: &PasswordVerifier, + algorithm: PasswordHashAlgorithm, + secret: Vec, +) -> trc::Result { + verifier + .run_blocking(move || { + let result = match algorithm { + PasswordHashAlgorithm::Argon2id => { + let hasher = Argon2::default(); + hasher + .hash_password(secret.as_slice()) + .map(|h| h.to_string()) + } + PasswordHashAlgorithm::Bcrypt => { + return bcrypt::hash(secret.as_slice()).map_err(|err| { trc::AuthEvent::Error .reason(err) .details("Bcrypt hash failed") - })) - .ok() - .unwrap_or(()); - } - PasswordHashAlgorithm::Scrypt => Scrypt::default() - .hash_password(secret.as_slice()) - .map(|h| h.to_string()), - PasswordHashAlgorithm::Pbkdf2 => Pbkdf2::default() - .hash_password(secret.as_slice()) - .map(|h| h.to_string()), - }; + }); + } + PasswordHashAlgorithm::Scrypt => Scrypt::default() + .hash_password(secret.as_slice()) + .map(|h| h.to_string()), + PasswordHashAlgorithm::Pbkdf2 => Pbkdf2::default() + .hash_password(secret.as_slice()) + .map(|h| h.to_string()), + }; - tx.send(result.map_err(|err| { - trc::AuthEvent::Error - .reason(err) - .details("Password hash failed") - })) - .ok(); - }); - - match rx.await { - Ok(result) => result, - Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError) - .caused_by(trc::location!()) - .reason(err)), - } + result.map_err(|err| { + trc::AuthEvent::Error + .reason(err) + .details("Password hash failed") + }) + }) + .await? } pub fn is_password_hash(s: &str) -> bool { @@ -397,7 +427,7 @@ fn is_sha1_crypt(body: &str) -> bool { } fn is_ldap_hash(scheme: &str, body: &str) -> bool { - match scheme.to_ascii_uppercase().as_str() { + hashify::fnc_map_ignore_case!(scheme.as_bytes(), "SHA" => b64_decoded_len_eq(body, 20), "SSHA" => b64_decoded_len_ge(body, 21), "SHA256" => b64_decoded_len_eq(body, 32), @@ -405,10 +435,13 @@ fn is_ldap_hash(scheme: &str, body: &str) -> bool { "SHA512" => b64_decoded_len_eq(body, 64), "SSHA512" => b64_decoded_len_ge(body, 65), "MD5" => b64_decoded_len_eq(body, 16), - "ARGON2" | "ARGON2I" | "ARGON2ID" | "PBKDF2" => is_complete_phc(body), + "ARGON2" => is_complete_phc(body), + "ARGON2I" => is_complete_phc(body), + "ARGON2ID" => is_complete_phc(body), + "PBKDF2" => is_complete_phc(body), "CRYPT" => is_password_hash(body) || is_unix_des_crypt(body), - _ => false, - } + _ => false + ) } fn is_unix_des_crypt(s: &str) -> bool { @@ -417,231 +450,15 @@ fn is_unix_des_crypt(s: &str) -> bool { || (bytes.len() == 20 && bytes[0] == b'_' && bytes[1..].iter().copied().all(is_crypt_b64)) } +const STANDARD_OPTIONAL_PAD: Base64 = STANDARD.with_padding(Padding::Optional); + fn b64_decoded_len_eq(body: &str, len: usize) -> bool { - b64_decode_loose(body) - .map(|d| d.len() == len) - .unwrap_or(false) + STANDARD_OPTIONAL_PAD.decoded_len(body) == Ok(len) } fn b64_decoded_len_ge(body: &str, min: usize) -> bool { - b64_decode_loose(body) - .map(|d| d.len() >= min) - .unwrap_or(false) + STANDARD_OPTIONAL_PAD + .decoded_len(body) + .is_ok_and(|len| len >= min) } -fn b64_decode_loose(s: &str) -> Option> { - use base64::Engine; - use base64::engine::general_purpose::STANDARD; - use base64::engine::general_purpose::STANDARD_NO_PAD; - STANDARD - .decode(s) - .ok() - .or_else(|| STANDARD_NO_PAD.decode(s).ok()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn b64(bytes: &[u8]) -> String { - String::from_utf8(Base64Encoder::new().encode(bytes).unwrap()).unwrap() - } - - #[test] - fn is_password_hash_detects_phc_strings() { - let argon = Argon2::default() - .hash_password(b"hello") - .unwrap() - .to_string(); - assert!(is_password_hash(&argon), "argon2 not detected: {argon}"); - - let pbkdf = Pbkdf2::default() - .hash_password(b"hello") - .unwrap() - .to_string(); - assert!(is_password_hash(&pbkdf), "pbkdf2 not detected: {pbkdf}"); - - let scr = Scrypt::default() - .hash_password(b"hello") - .unwrap() - .to_string(); - assert!(is_password_hash(&scr), "scrypt not detected: {scr}"); - } - - #[test] - fn is_password_hash_detects_crypt_variants() { - let bc = bcrypt::hash("hello").unwrap(); - assert!(is_password_hash(&bc), "bcrypt not detected: {bc}"); - assert!(bcrypt::verify("hello", &bc)); - - let md5 = "$1$5pZSV9va$azfrPr6af3Fc7dLblQXVa0"; - assert!(is_password_hash(md5)); - assert!(md5_crypt::verify("password", md5)); - - let sha256 = "$5$WH1ABM5sKhxbkgCK$sOnTVjQn1Y3EWibd8gWqqJqjH.KaFrxJE5rijqxcPp7"; - assert!(is_password_hash(sha256)); - assert!(sha256_crypt::verify("test", sha256)); - - let sha256_rounds = - "$5$rounds=11858$WH1ABM5sKhxbkgCK$aTQsjPkz0rBsH3lQlJxw9HDTDXPKBxC0LlVeV69P.t1"; - assert!(is_password_hash(sha256_rounds)); - assert!(sha256_crypt::verify("test", sha256_rounds)); - - let s512 = sha512_crypt::hash("hello").unwrap(); - assert!(is_password_hash(&s512), "sha512_crypt not detected: {s512}"); - assert!(sha512_crypt::verify("hello", &s512)); - - let s1 = sha1_crypt::hash("hello").unwrap(); - assert!(is_password_hash(&s1), "sha1_crypt not detected: {s1}"); - assert!(sha1_crypt::verify("hello", &s1)); - - let bsdi = "_J9..K0AyUubDkQmPLeM"; - assert!(is_password_hash(bsdi), "bsdi_crypt not detected: {bsdi}"); - } - - #[test] - fn is_password_hash_detects_ldap_schemes() { - let mut h = Sha1::new(); - h.update(b"hello"); - let sha = b64(&h.finalize()[..]); - assert!(is_password_hash(&format!("{{SHA}}{sha}"))); - - let mut h = Sha1::new(); - h.update(b"hello"); - h.update(b"saltbytes"); - let mut buf = h.finalize().to_vec(); - buf.extend_from_slice(b"saltbytes"); - let ssha = b64(&buf); - assert!(is_password_hash(&format!("{{SSHA}}{ssha}"))); - - let mut h = Sha256::new(); - h.update(b"hello"); - let sha256 = b64(&h.finalize()[..]); - assert!(is_password_hash(&format!("{{SHA256}}{sha256}"))); - - let mut h = Sha256::new(); - h.update(b"hello"); - h.update(b"saltbytes"); - let mut buf = h.finalize().to_vec(); - buf.extend_from_slice(b"saltbytes"); - let ssha256 = b64(&buf); - assert!(is_password_hash(&format!("{{SSHA256}}{ssha256}"))); - - let mut h = Sha512::new(); - h.update(b"hello"); - let sha512 = b64(&h.finalize()[..]); - assert!(is_password_hash(&format!("{{SHA512}}{sha512}"))); - - let mut h = Sha512::new(); - h.update(b"hello"); - h.update(b"saltbytes"); - let mut buf = h.finalize().to_vec(); - buf.extend_from_slice(b"saltbytes"); - let ssha512 = b64(&buf); - assert!(is_password_hash(&format!("{{SSHA512}}{ssha512}"))); - - let digest = md5::compute(b"hello"); - let md5b = b64(&digest[..]); - assert!(is_password_hash(&format!("{{MD5}}{md5b}"))); - - let inner = sha512_crypt::hash("hello").unwrap(); - assert!(is_password_hash(&format!("{{CRYPT}}{inner}"))); - assert!(is_password_hash(&format!("{{crypt}}{inner}"))); - - assert!(is_password_hash( - "{CRYPT}$1$5pZSV9va$azfrPr6af3Fc7dLblQXVa0" - )); - assert!(is_password_hash("{CRYPT}abcdefghij012")); - assert!(is_password_hash("{CRYPT}_J9..K0AyUubDkQmPLeM")); - - let a = Argon2::default() - .hash_password(b"hello") - .unwrap() - .to_string(); - assert!(is_password_hash(&format!("{{ARGON2ID}}{a}"))); - assert!(is_password_hash(&format!("{{ARGON2}}{a}"))); - assert!(is_password_hash(&format!("{{ARGON2I}}{a}"))); - - let p = Pbkdf2::default() - .hash_password(b"hello") - .unwrap() - .to_string(); - assert!(is_password_hash(&format!("{{PBKDF2}}{p}"))); - - let mut h = Sha1::new(); - h.update(b"hello"); - let sha_lc = b64(&h.finalize()[..]); - assert!(is_password_hash(&format!("{{sha}}{sha_lc}"))); - - let mut h = Sha256::new(); - h.update(b"hello"); - h.update(b"saltbytes"); - let mut buf = h.finalize().to_vec(); - buf.extend_from_slice(b"saltbytes"); - let ssha256_lc = b64(&buf); - assert!(is_password_hash(&format!("{{ssha256}}{ssha256_lc}"))); - - let digest = md5::compute(b"hello"); - let md5_mc = b64(&digest[..]); - assert!(is_password_hash(&format!("{{Md5}}{md5_mc}"))); - } - - #[test] - fn is_password_hash_rejects_passwords() { - let not_hashes = [ - "", - "hello", - "p@ssw0rd!", - "password123", - "correct horse battery staple", - "$myPassword", - "$1incomplete", - "$1$", - "$1$short", - "$1$abc$tooshorthash", - "$5$", - "$5$nohashpart$", - "$5$saltonly$alsotooshort", - "$6$", - "$$$", - "$$argon2$", - "$argon2id$broken", - "$argon2id$v=19$bad", - "$2", - "$2y$", - "$2y$10$short", - "$2z$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", - "$sha1$", - "$sha1$notdigits$salt$hash", - "{", - "{}", - "{}foo", - "{SHA}", - "{SHA}not!valid!base!64", - "{SHA}aGVsbG8=", - "{MD5}", - "{MD5}aGVsbG8=", - "{SHA256}aGVsbG8=", - "{SHA512}aGVsbG8=", - "{SSHA}aGVsbG8=", - "{UNKNOWN}whatever", - "{PLAIN}stillplain", - "{plain}stillplain", - "{CLEAR}stillplain", - "{clear}stillplain", - "{CRYPT}plainpw", - "{CRYPT}", - "{CRYPT}toolongtobeunixcryptbutshortbsdi", - "{ARGON2ID}notaphcstring", - "_short", - "_notvalidbsdi", - "regular_password", - "1234567890123", - "abcdefghij012", - "$5$rounds=$saltvalue$abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJK", - ]; - for p in not_hashes { - assert!(!is_password_hash(p), "false positive: {p:?}"); - } - } -} diff --git a/crates/directory/src/core/verifier.rs b/crates/directory/src/core/verifier.rs new file mode 100644 index 00000000..bd00a167 --- /dev/null +++ b/crates/directory/src/core/verifier.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{ + collections::VecDeque, + fmt::Display, + num::NonZeroUsize, + panic::{AssertUnwindSafe, catch_unwind}, + sync::{Arc, Mutex, MutexGuard, PoisonError}, + thread::available_parallelism, +}; +use tokio::sync::oneshot; + +const FALLBACK_WORKERS: NonZeroUsize = NonZeroUsize::MIN.saturating_add(3); + +#[derive(Clone)] +pub struct PasswordVerifier { + pool: Arc, +} + +struct Pool { + max_workers: usize, + state: Mutex, +} + +#[derive(Default)] +struct PoolState { + workers: usize, + queue: VecDeque, +} + +type Job = Box; + +impl PasswordVerifier { + pub fn new(max_workers: NonZeroUsize) -> Self { + Self { + pool: Arc::new(Pool { + max_workers: max_workers.get(), + state: Mutex::default(), + }), + } + } + + pub(crate) async fn run_blocking( + &self, + task: impl FnOnce() -> T + Send + 'static, + ) -> trc::Result { + let (tx, rx) = oneshot::channel(); + self.pool.submit(Box::new(move || { + if !tx.is_closed() { + let _ = tx.send(task()); + } + })); + rx.await.map_err(thread_error) + } +} + +impl Pool { + fn submit(self: &Arc, job: Job) { + let mut state = self.lock(); + if state.workers < self.max_workers { + state.workers += 1; + drop(state); + let pool = self.clone(); + tokio::task::spawn_blocking(move || pool.work(job)); + } else { + state.queue.push_back(job); + } + } + + fn work(&self, mut job: Job) { + loop { + let _ = catch_unwind(AssertUnwindSafe(job)); + let mut state = self.lock(); + match state.queue.pop_front() { + Some(next) => job = next, + None => { + state.workers -= 1; + return; + } + } + } + } + + fn lock(&self) -> MutexGuard<'_, PoolState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl Default for PasswordVerifier { + fn default() -> Self { + Self::new(available_parallelism().unwrap_or(FALLBACK_WORKERS)) + } +} + +#[cold] +fn thread_error(err: impl Display) -> trc::Error { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .caused_by(trc::location!()) + .reason(err) +} + diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index 48b4448e..9769ed27 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "email" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -10,33 +10,38 @@ store = { path = "../store" } trc = { path = "../trc" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } -jmap-tools = { version = "0.1" } +jmap-tools = { path = "../jmap-tools" } common = { path = "../common" } groupware = { path = "../groupware" } registry = { path = "../registry" } -mail-parser = { version = "0.11", features = ["full_encoding"] } -mail-builder = { version = "1.0" } -sieve-rs = { version = "0.7", features = ["rkyv"] } -tokio = { version = "1.53", features = ["net", "macros"] } +text-extract = { path = "../text-extract" } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +encodify = "1.0" +sieve-rs = { path = "../sieve" } +tokio = { version = "1.54", features = ["net", "macros"] } aes = "0.9" aes-gcm = "0.11.1" chacha20poly1305 = "0.11" cbc = { version = "0.2", features = ["alloc"] } -rasn = "0.28" -rasn-cms = "0.28" -rasn-pkix = "0.28" +rasn = "0.29" +rasn-cms = "0.29" +rasn-pkix = "0.29" rsa = { version = "0.9.10", features = ["sha2"] } rand = "0.8" sequoia-openpgp = { version = "2.4", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } -hashify = "0.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } +hashify = "0.3" +rkyv.workspace = true +compact_str = "0.10.0" +tinyvec = { version = "1.13.3", features = ["alloc"] } +bytecount = "0.6" [features] -test_mode = [] -enterprise = [] +enterprise = ["utils/enterprise", "store/enterprise", "trc/enterprise", "jmap_proto/enterprise", "common/enterprise", "groupware/enterprise", "registry/enterprise"] [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } +serde_json = "1.0" [lints] workspace = true diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index fd7f539e..e2373f8f 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -4,27 +4,93 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ArchivedMessageData, MessageData}; +use crate::{ + cache::mailbox::MailboxCacheAccess, + message::messagedata::{ + EmailMessageData, KEYWORD_BITS, KeywordDiff, KeywordsIter, MessageData, SERVER_SET_KEYWORDS, + }, +}; use common::{ - MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken, - sharing::EffectiveAcl, + CustomKeywords, MessageCache, MessageStoreCache, MessagesCache, Server, auth::AccessToken, + cache::email::MessageRef, +}; +use compact_str::CompactString; +use std::sync::Arc; +use store::{ + ahash::{AHashMap, AHashSet}, + roaring::RoaringBitmap, }; -use store::write::{AlignedBytes, Archive}; -use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ acl::Acl, - collection::Collection, - keyword::{Keyword, OTHER}, + keyword::{DRAFT, Keyword, SEEN}, + metadata::MetadataKinds, }; use utils::map::bitmap::Bitmap; +pub(crate) const HAS_CUSTOM_KEYWORDS: u32 = 1 << 31; +const READ_KEYWORDS: u32 = (1 << SEEN) | (1 << DRAFT); + +pub struct KeywordWrites { + writes: Vec, + mailboxes_changed: bool, +} + +pub(crate) struct KeywordWrite { + pub position: u32, + pub document_id: u32, + pub keywords: u32, + pub commit_point: u32, +} + +const _: () = assert!(size_of::() <= 16); + +impl KeywordWrites { + pub fn with_capacity(capacity: usize) -> Self { + KeywordWrites { + writes: Vec::with_capacity(capacity), + mailboxes_changed: false, + } + } + + pub fn push( + &mut self, + message: MessageRef<'_>, + diff: &KeywordDiff, + commit_point: usize, + ) -> bool { + let (Some(keywords), Ok(commit_point)) = ( + diff.keyword_bits(message.keywords()), + u32::try_from(commit_point), + ) else { + return false; + }; + self.writes.push(KeywordWrite { + position: message.position(), + document_id: message.document_id(), + keywords, + commit_point, + }); + true + } + + pub fn set_mailboxes_changed(&mut self) { + self.mailboxes_changed = true; + } + + pub(crate) fn writes(&self) -> &[KeywordWrite] { + &self.writes + } + + pub(crate) fn mailboxes_changed(&self) -> bool { + self.mailboxes_changed + } +} + struct MessagesCacheBuilder { pub change_id: u64, pub items: Vec, - pub index: AHashMap, - pub keywords: Vec>, - pub size: u64, + pub keywords: Vec, } pub(crate) async fn update_email_cache( @@ -33,41 +99,133 @@ pub(crate) async fn update_email_cache( changed_ids: &AHashMap, store_cache: &MessageStoreCache, ) -> trc::Result { - let mut new_cache = MessagesCacheBuilder { - index: AHashMap::with_capacity(store_cache.emails.items.len()), - items: Vec::with_capacity(store_cache.emails.items.len()), - size: 0, - change_id: 0, - keywords: store_cache.emails.keywords.to_vec(), - }; + let mut fetch_ids = changed_ids + .iter() + .filter(|(_, is_update)| **is_update) + .map(|(document_id, _)| *document_id) + .collect::>(); + fetch_ids.sort_unstable(); + let mut fetched: AHashMap = AHashMap::with_capacity(fetch_ids.len()); + let mut fetched_keywords: Vec = Vec::new(); + + if !fetch_ids.is_empty() { + server + .message_datas(account_id, &fetch_ids, |document_id, mut data| { + if !data.keywords_extra.is_empty() { + fetched_keywords.push(CustomKeywords { + names: std::mem::take(&mut data.keywords_extra).into_boxed_slice(), + document_id, + }); + data.keywords |= HAS_CUSTOM_KEYWORDS; + } + + fetched.insert( + document_id, + MessageCache::new( + document_id, + data.mailboxes, + data.keywords, + data.thread_id, + data.change_id, + data.size, + data.received_at, + data.sent_at, + ), + ); + Ok(true) + }) + .await + .caused_by(trc::location!())?; + fetched_keywords.sort_unstable_by_key(|k| k.document_id); + } + + Ok(merge_email_cache( + &store_cache.emails, + changed_ids, + fetched, + fetched_keywords, + )) +} + +fn merge_email_cache( + cache: &MessagesCache, + changed_ids: &AHashMap, + mut fetched: AHashMap, + fetched_keywords: Vec, +) -> MessagesCache { + let mut inserts = Vec::new(); + let mut deletes = Vec::new(); for (document_id, is_update) in changed_ids { - if *is_update - && let Some(archive) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - *document_id, - )) - .await - .caused_by(trc::location!())? - { - insert_item( - &mut new_cache, - *document_id, - archive.to_unarchived::()?, - ); + if !*is_update { + deletes.push(*document_id); + } else if !cache.contains(*document_id) { + if let Some(item) = fetched.remove(document_id) { + inserts.push(item); + } + } else if !fetched.contains_key(document_id) { + deletes.push(*document_id); } } + inserts.sort_unstable_by_key(MessageCache::sort_rank); + deletes.sort_unstable(); - for item in &store_cache.emails.items { - if !changed_ids.contains_key(&item.document_id) { - email_insert(&mut new_cache, item.clone()); - } + let keywords = merge_custom_keywords(cache, changed_ids, fetched_keywords); + + let patches = fetched + .into_iter() + .filter_map(|(document_id, record)| { + cache + .position(document_id) + .map(|position| (position, record)) + }) + .collect::>(); + + if inserts.is_empty() && deletes.is_empty() { + cache.patch(0, &patches, keywords) + } else { + cache.splice(0, &deletes, &patches, &inserts, keywords) + } +} + +fn merge_custom_keywords( + cache: &MessagesCache, + changed_ids: &AHashMap, + fetched_keywords: Vec, +) -> Arc<[CustomKeywords]> { + if fetched_keywords.is_empty() + && !changed_ids.keys().any(|document_id| { + cache + .keywords() + .binary_search_by_key(document_id, |entry| entry.document_id) + .is_ok() + }) + { + return cache.shared_keywords(); } - Ok(new_cache.build()) + let mut retained = cache + .keywords() + .iter() + .filter(|k| !changed_ids.contains_key(&k.document_id)) + .peekable(); + let mut keywords = Vec::with_capacity(cache.keywords().len() + fetched_keywords.len()); + let mut added = fetched_keywords.into_iter().peekable(); + loop { + match (retained.peek(), added.peek()) { + (Some(old), Some(new)) => { + if old.document_id <= new.document_id { + keywords.push(retained.next().unwrap().clone()); + } else { + keywords.push(added.next().unwrap()); + } + } + (Some(_), None) => keywords.push(retained.next().unwrap().clone()), + (None, Some(_)) => keywords.push(added.next().unwrap()), + (None, None) => break, + } + } + keywords.into() } pub(crate) async fn full_email_cache_build( @@ -77,173 +235,313 @@ pub(crate) async fn full_email_cache_build( // Build cache let mut cache = MessagesCacheBuilder { items: Vec::with_capacity(16), - index: AHashMap::with_capacity(16), - keywords: Vec::new(), - size: 0, + keywords: Default::default(), change_id: 0, }; server - .archives( - account_id, - Collection::Email, - &(), - |document_id, archive| { - insert_item( - &mut cache, + .message_datas(account_id, &(), |document_id, mut data| { + if !data.keywords_extra.is_empty() { + cache.keywords.push(CustomKeywords { + names: data.keywords_extra.into_boxed_slice(), document_id, - archive.to_unarchived::()?, - ); - Ok(true) - }, - ) + }); + data.keywords |= HAS_CUSTOM_KEYWORDS; + } + + cache.items.push(MessageCache::new( + document_id, + data.mailboxes, + data.keywords, + data.thread_id, + data.change_id, + data.size, + data.received_at, + data.sent_at, + )); + Ok(true) + }) .await .caused_by(trc::location!())?; Ok(cache.build()) } -fn insert_item( - cache: &mut MessagesCacheBuilder, - document_id: u32, - archive: Archive<&ArchivedMessageData>, -) { - let message = archive.inner; - let mut item = MessageCache { - mailboxes: message - .mailboxes - .iter() - .map(|m| MessageUidCache { - mailbox_id: m.mailbox_id.to_native(), - uid: m.uid.to_native(), - }) - .collect(), - keywords: 0, - thread_id: message.thread_id.to_native(), - change_id: archive.version.change_id().unwrap_or_default(), - document_id, - size: message.size.to_native(), +impl MessagesCacheBuilder { + pub fn build(mut self) -> MessagesCache { + self.items.sort_unstable_by_key(MessageCache::sort_rank); + self.keywords.sort_unstable_by_key(|k| k.document_id); + MessagesCache::new(self.change_id, self.items, self.keywords) + } +} + +pub fn thread_keywords( + cache: &MessageStoreCache, + keyword: &Keyword, + match_all: bool, + documents: Option<&RoaringBitmap>, + visible: Option<&RoaringBitmap>, +) -> RoaringBitmap { + let is_visible = |document_id: u32| visible.is_none_or(|visible| visible.contains(document_id)); + let mut tagged = RoaringBitmap::new(); + let mut untagged = RoaringBitmap::new(); + let mut tally = |item: MessageRef<'_>| { + if cache.has_keyword(item, keyword) { + tagged.insert(item.thread_id()); + } else if match_all { + untagged.insert(item.thread_id()); + } }; - for keyword in message.keywords.iter() { - match keyword.id() { - Ok(id) => { - item.keywords |= 1 << id; + let members = match documents { + Some(documents) => { + let members = documents + .iter() + .filter_map(|document_id| { + cache + .email_by_id(&document_id) + .map(|item| (document_id, item.thread_id())) + }) + .collect::>(); + let threads = members + .iter() + .map(|(_, thread_id)| *thread_id) + .collect::>(); + for item in cache.emails.iter().filter(|item| { + is_visible(item.document_id()) && threads.contains(&item.thread_id()) + }) { + tally(item); } - Err(custom) => { - if let Some(idx) = cache.keywords.iter().position(|k| **k == *custom) { - item.keywords |= 1 << (OTHER + idx); - } else if cache.keywords.len() < (128 - OTHER) { - cache.keywords.push(custom.into()); - item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); - } + members + } + None => { + let capacity = visible + .map_or(cache.emails.len(), |visible| { + usize::try_from(visible.len()).unwrap_or(usize::MAX) + }) + .min(cache.emails.len()); + let mut members = Vec::with_capacity(capacity); + for item in cache + .emails + .iter() + .filter(|item| is_visible(item.document_id())) + { + members.push((item.document_id(), item.thread_id())); + tally(item); + } + members + } + }; + tagged -= untagged; + members + .into_iter() + .filter(|(_, thread_id)| tagged.contains(*thread_id)) + .map(|(document_id, _)| document_id) + .collect() +} + +pub enum SearchOperator { + LowerThan, + LowerEqualThan, + GreaterThan, + GreaterEqualThan, + Equal, + Contains, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct MailboxCounters { + pub total: usize, + pub unread: usize, + pub threads: AHashSet, + pub unread_threads: AHashSet, +} + +impl MailboxCounters { + #[inline] + fn add(&mut self, thread_id: u32, is_unread: bool, count_threads: bool) { + self.total += 1; + self.unread += usize::from(is_unread); + if count_threads { + self.threads.insert(thread_id); + if is_unread { + self.unread_threads.insert(thread_id); } } } - - email_insert(cache, item); } -impl MessagesCacheBuilder { - pub fn build(mut self) -> MessagesCache { - self.index.shrink_to_fit(); - MessagesCache { - change_id: self.change_id, - items: self.items.into_boxed_slice(), - index: self.index, - keywords: self.keywords.into_boxed_slice(), - size: self.size, +pub enum MessageAccess { + All, + Mailboxes(RoaringBitmap), +} + +impl MessageAccess { + #[inline] + pub fn allows(&self, message: MessageRef<'_>) -> bool { + match self { + MessageAccess::All => true, + MessageAccess::Mailboxes(mailboxes) => message + .mailboxes() + .iter() + .any(|uid| mailboxes.contains(uid.mailbox_id)), + } + } + + #[inline] + pub fn allows_mailbox(&self, mailbox_id: u32) -> bool { + match self { + MessageAccess::All => true, + MessageAccess::Mailboxes(mailboxes) => mailboxes.contains(mailbox_id), + } + } + + pub fn is_empty(&self) -> bool { + match self { + MessageAccess::All => false, + MessageAccess::Mailboxes(mailboxes) => mailboxes.is_empty(), } } } pub trait MessageCacheAccess { - fn email_by_id(&self, id: &u32) -> Option<&MessageCache>; + fn email_by_id(&self, id: &u32) -> Option>; fn has_email_id(&self, id: &u32) -> bool; - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; + fn custom_keywords(&self, message: MessageRef<'_>) -> &[CompactString]; - fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator; + fn message_data(&self, document_id: u32) -> Option; - fn in_thread(&self, thread_id: u32) -> impl Iterator; + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator>; - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; + fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator>; - fn without_keyword(&self, keyword: &Keyword) -> impl Iterator; + fn in_thread(&self, thread_id: u32) -> impl Iterator>; + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator>; + + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator>; fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator; + ) -> impl Iterator>; fn in_mailbox_without_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator; + ) -> impl Iterator>; fn email_document_ids(&self) -> RoaringBitmap; + fn mailbox_counters( + &self, + mailbox_ids: impl IntoIterator, + count_threads: bool, + ) -> AHashMap; + fn shared_messages( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap; - fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; + fn expand_keywords(&self, message: MessageRef<'_>) -> impl Iterator; - fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; + fn has_keyword(&self, message: MessageRef<'_>, keyword: &Keyword) -> bool; + + fn keyword_count(&self, message: MessageRef<'_>) -> usize; + + fn metadata_kinds(&self, message: MessageRef<'_>) -> MetadataKinds; + + fn with_metadata(&self) -> impl Iterator>; + + fn received(&self, date: i64, comp: SearchOperator) -> impl Iterator>; + + fn sent(&self, date: i64, comp: SearchOperator) -> impl Iterator>; + + fn size(&self, size: u32, comp: SearchOperator) -> impl Iterator>; } impl MessageCacheAccess for MessageStoreCache { - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator> { self.emails - .items .iter() - .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) + .filter(move |m| m.mailboxes().iter().any(|m| m.mailbox_id == mailbox_id)) } - fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator { - self.emails.items.iter().filter(move |m| { - m.mailboxes + fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator> { + self.emails.iter().filter(move |m| { + m.mailboxes() .iter() .any(|mb| mailbox_ids.contains(&mb.mailbox_id)) }) } - fn in_thread(&self, thread_id: u32) -> impl Iterator { + fn in_thread(&self, thread_id: u32) -> impl Iterator> { self.emails - .items .iter() - .filter(move |m| m.thread_id == thread_id) + .filter(move |m| m.thread_id() == thread_id) } - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator> { self.emails - .items .iter() - .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)) + .filter(move |m| self.has_keyword(*m, keyword)) } - fn without_keyword(&self, keyword: &Keyword) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator> { self.emails - .items .iter() - .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)) + .filter(move |m| !self.has_keyword(*m, keyword)) + } + + fn received(&self, date: i64, comp: SearchOperator) -> impl Iterator> { + self.emails.iter().filter(move |m| match comp { + SearchOperator::LowerThan => (m.received_at() as i64) < date, + SearchOperator::LowerEqualThan => (m.received_at() as i64) <= date, + SearchOperator::GreaterThan => (m.received_at() as i64) > date, + SearchOperator::GreaterEqualThan => (m.received_at() as i64) >= date, + SearchOperator::Equal => (m.received_at() as i64) == date, + SearchOperator::Contains => unreachable!(), + }) + } + + fn sent(&self, date: i64, comp: SearchOperator) -> impl Iterator> { + self.emails.iter().filter(move |m| { + let sent_at = m.received_at() as i64 + m.sent_at() as i64; + match comp { + SearchOperator::LowerThan => sent_at < date, + SearchOperator::LowerEqualThan => sent_at <= date, + SearchOperator::GreaterThan => sent_at > date, + SearchOperator::GreaterEqualThan => sent_at >= date, + SearchOperator::Equal => sent_at == date, + SearchOperator::Contains => unreachable!(), + } + }) + } + + fn size(&self, size: u32, comp: SearchOperator) -> impl Iterator> { + self.emails.iter().filter(move |m| match comp { + SearchOperator::LowerThan => m.size() < size, + SearchOperator::LowerEqualThan => m.size() <= size, + SearchOperator::GreaterThan => m.size() > size, + SearchOperator::GreaterEqualThan => m.size() >= size, + SearchOperator::Equal => m.size() == size, + SearchOperator::Contains => unreachable!(), + }) } fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.emails.items.iter().filter(move |m| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0) + ) -> impl Iterator> { + self.emails.iter().filter(move |m| { + m.mailboxes().iter().any(|uid| uid.mailbox_id == mailbox_id) + && self.has_keyword(*m, keyword) }) } @@ -251,11 +549,10 @@ impl MessageCacheAccess for MessageStoreCache { &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.emails.items.iter().filter(move |m| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0) + ) -> impl Iterator> { + self.emails.iter().filter(move |m| { + m.mailboxes().iter().any(|uid| uid.mailbox_id == mailbox_id) + && !self.has_keyword(*m, keyword) }) } @@ -264,93 +561,117 @@ impl MessageCacheAccess for MessageStoreCache { access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap { - let check_acls = check_acls.into(); - let mut shared_messages = RoaringBitmap::new(); - for mailbox in &self.mailboxes.items { - if mailbox - .acls - .as_slice() - .effective_acl(access_token) - .contains_all(check_acls) - { - shared_messages.extend( - self.in_mailbox(mailbox.document_id) - .map(|item| item.document_id), - ); - } + let access = MessageAccess::Mailboxes(self.shared_mailboxes(access_token, check_acls)); + if access.is_empty() { + return RoaringBitmap::new(); } - shared_messages + self.emails + .iter() + .filter(|message| access.allows(*message)) + .map(|message| message.document_id()) + .collect() } fn email_document_ids(&self) -> RoaringBitmap { - RoaringBitmap::from_iter(self.emails.index.keys()) + RoaringBitmap::from_sorted_iter(self.emails.document_ids()) + .unwrap_or_else(|_| RoaringBitmap::from_iter(self.emails.document_ids())) } - fn email_by_id(&self, id: &u32) -> Option<&MessageCache> { - self.emails - .index - .get(id) - .and_then(|idx| self.emails.items.get(*idx as usize)) + fn mailbox_counters( + &self, + mailbox_ids: impl IntoIterator, + count_threads: bool, + ) -> AHashMap { + let mut counters = mailbox_ids + .into_iter() + .map(|mailbox_id| (mailbox_id, MailboxCounters::default())) + .collect::>(); + if counters.is_empty() { + return counters; + } + + for message in self.emails.iter() { + let mailboxes = message.mailboxes(); + let mut is_unread = None; + for (pos, uid) in mailboxes.iter().enumerate() { + if let Some(counter) = counters.get_mut(&uid.mailbox_id) + && !mailboxes + .iter() + .take(pos) + .any(|prev| prev.mailbox_id == uid.mailbox_id) + { + let is_unread = + *is_unread.get_or_insert_with(|| message.keywords() & READ_KEYWORDS == 0); + counter.add(message.thread_id(), is_unread, count_threads); + } + } + } + + counters + } + + fn email_by_id(&self, id: &u32) -> Option> { + self.emails.by_id(*id) } fn has_email_id(&self, id: &u32) -> bool { - self.emails.index.contains_key(id) + self.emails.contains(*id) } - fn expand_keywords(&self, message: &MessageCache) -> impl Iterator { - KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) { - Ok(keyword) => keyword, - Err(id) => Keyword::Other(self.emails.keywords[id - OTHER].clone()), + fn custom_keywords(&self, message: MessageRef<'_>) -> &[CompactString] { + if message.keywords() & HAS_CUSTOM_KEYWORDS != 0 { + self.emails.custom_keywords_of(message.document_id()) + } else { + &[] + } + } + + fn message_data(&self, document_id: u32) -> Option { + let message = self.email_by_id(&document_id)?; + + Some(MessageData { + mailboxes: message.mailboxes().iter().copied().collect(), + keywords: message.keywords() & !HAS_CUSTOM_KEYWORDS, + keywords_extra: self.custom_keywords(message).to_vec(), + thread_id: message.thread_id(), + size: message.size(), + received_at: message.received_at(), + sent_at: message.sent_at(), + change_id: message.change_id(), }) } - fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { - keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) + fn expand_keywords(&self, message: MessageRef<'_>) -> impl Iterator { + KeywordsIter(message.keywords() & KEYWORD_BITS).chain( + self.custom_keywords(message) + .iter() + .map(|name| Keyword::Other(name.clone())), + ) } -} -fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) { - let id = item.document_id; - if let Some(idx) = cache.index.get(&id) { - cache.items[*idx as usize] = item; - } else { - cache.size += (std::mem::size_of::() - + (std::mem::size_of::() * 2) - + (item.mailboxes.len() * std::mem::size_of::())) - as u64; - - let idx = cache.items.len() as u32; - cache.items.push(item); - cache.index.insert(id, idx); + fn keyword_count(&self, message: MessageRef<'_>) -> usize { + (message.keywords() & KEYWORD_BITS & !SERVER_SET_KEYWORDS).count_ones() as usize + + self.custom_keywords(message).len() } -} -#[inline] -fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { - match keyword.id() { - Ok(id) => Some(id), - Err(name) => cache - .emails - .keywords + fn metadata_kinds(&self, message: MessageRef<'_>) -> MetadataKinds { + MessageData::metadata_kinds_of(message.keywords()) + } + + fn with_metadata(&self) -> impl Iterator> { + self.emails .iter() - .position(|k| **k == *name) - .map(|idx| (OTHER + idx) as u32), + .filter(|message| MessageData::has_metadata(message.keywords())) } -} -#[derive(Clone, Copy, Debug)] -struct KeywordsIter(u128); - -impl Iterator for KeywordsIter { - type Item = usize; - - fn next(&mut self) -> Option { - if self.0 != 0 { - let item = 127 - self.0.leading_zeros(); - self.0 ^= 1 << item; - Some(item as usize) - } else { - None + fn has_keyword(&self, message: MessageRef<'_>, keyword: &Keyword) -> bool { + match keyword.id() { + Ok(id) => (message.keywords() & (1 << id)) != 0, + Err(name) => self + .custom_keywords(message) + .iter() + .any(|n| n.as_str() == name), } } } + diff --git a/crates/email/src/cache/mailbox.rs b/crates/email/src/cache/mailbox.rs index a315a6db..73a4aeb3 100644 --- a/crates/email/src/cache/mailbox.rs +++ b/crates/email/src/cache/mailbox.rs @@ -4,24 +4,128 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::MessageCacheFetch; use crate::mailbox::{ArchivedMailbox, Mailbox, manage::MailboxFnc}; use common::{ MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; +use std::sync::Arc; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + query::log::{Change, Query}, + write::{Archive, ArchiveBytes}, }; use store::{ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, - collection::Collection, + collection::{Collection, SyncCollection}, + field::Field, special_use::SpecialUse, }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; +#[derive(Debug, Clone)] +pub struct AccountMailboxes { + pub mailboxes: Arc, + pub change_id: u64, +} + +pub trait MailboxCacheFetch: Sync + Send { + fn fetch_mailboxes( + &self, + account_id: u32, + known: Option<&AccountMailboxes>, + ) -> impl Future> + Send; +} + +impl MailboxCacheFetch for Server { + async fn fetch_mailboxes( + &self, + account_id: u32, + known: Option<&AccountMailboxes>, + ) -> trc::Result { + if let Some(cache) = self.get_resident_messages(account_id).await? { + return Ok(AccountMailboxes { + mailboxes: cache.mailboxes.clone(), + change_id: cache.last_change_id, + }); + } + + let Some(known) = known else { + return build_account_mailboxes(self, account_id).await; + }; + let changes = self + .core + .storage + .data + .changes( + account_id, + SyncCollection::Email.into(), + Query::Since(known.change_id), + ) + .await + .caused_by(trc::location!())?; + if changes.needs_full_rebuild(known.change_id) { + return build_account_mailboxes(self, account_id).await; + } + + let mut changed_ids = AHashMap::new(); + for change in &changes.changes { + match change { + Change::InsertContainer(id) | Change::UpdateContainer(id) => { + changed_ids.insert(*id as u32, true); + } + Change::UpdateContainerPartial(id, partial) if partial.has_metadata() => { + changed_ids.insert(*id as u32, true); + } + Change::DeleteContainer(id) => { + changed_ids.insert(*id as u32, false); + } + _ => {} + } + } + + let change_id = changes.to_change_id.max(known.change_id); + if changed_ids.is_empty() { + return Ok(AccountMailboxes { + mailboxes: known.mailboxes.clone(), + change_id, + }); + } + + let mut mailboxes = update_mailbox_cache(self, account_id, &changed_ids, &known.mailboxes) + .await + .caused_by(trc::location!())?; + mailboxes.change_id = changes.container_change_id.unwrap_or(change_id); + Ok(AccountMailboxes { + mailboxes: Arc::new(mailboxes), + change_id, + }) + } +} + +async fn build_account_mailboxes( + server: &Server, + account_id: u32, +) -> trc::Result { + let change_id = server + .core + .storage + .data + .get_last_change_id(account_id, SyncCollection::Email.into()) + .await + .caused_by(trc::location!())? + .unwrap_or_default(); + let mut mailboxes = full_mailbox_cache_build(server, account_id).await?; + mailboxes.change_id = change_id; + Ok(AccountMailboxes { + mailboxes: Arc::new(mailboxes), + change_id, + }) +} + struct MailboxesCacheBuilder { pub change_id: u64, pub index: AHashMap, @@ -33,20 +137,14 @@ pub(crate) async fn update_mailbox_cache( server: &Server, account_id: u32, changed_ids: &AHashMap, - store_cache: &MessageStoreCache, + mailboxes: &MailboxesCache, ) -> trc::Result { - let mut new_cache = MailboxesCacheBuilder { - items: Vec::with_capacity(store_cache.mailboxes.items.len()), - index: AHashMap::with_capacity(store_cache.mailboxes.items.len()), - size: 0, - change_id: 0, - }; - + let mut changes = Vec::with_capacity(changed_ids.len()); for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, *document_id, @@ -54,23 +152,137 @@ pub(crate) async fn update_mailbox_cache( .await .caused_by(trc::location!())? { - insert_item( - &mut new_cache, - *document_id, - archive.unarchive::()?, - ); + changes.push(mailbox_item(*document_id, archive.unarchive::()?)); } } - for item in store_cache.mailboxes.items.iter() { + Ok(apply_mailbox_changes(mailboxes, changed_ids, changes)) +} + +fn apply_mailbox_changes( + mailboxes: &MailboxesCache, + changed_ids: &AHashMap, + changes: Vec, +) -> MailboxesCache { + if let Some(appended) = appended_mailboxes(mailboxes, changed_ids, &changes) { + extend_mailbox_cache(mailboxes, changes, appended) + } else { + rebuild_mailbox_cache(mailboxes, changed_ids, changes) + } +} + +fn appended_mailboxes( + mailboxes: &MailboxesCache, + changed_ids: &AHashMap, + changes: &[MailboxCache], +) -> Option { + let mut updated = 0; + let mut new_ids = Vec::new(); + for item in changes { + if let Some(current) = mailboxes.mailbox_by_id(&item.document_id) { + if current.name != item.name + || current.parent_id != item.parent_id + || current.role != item.role + { + return None; + } + updated += 1; + } else { + new_ids.push(item.document_id); + } + } + + if changed_ids + .keys() + .filter(|id| mailboxes.has_mailbox_id(id)) + .count() + != updated + { + return None; + } + + if !new_ids.is_empty() { + new_ids.sort_unstable(); + if mailboxes.items.iter().any(|item| { + item.parent_id() + .is_some_and(|parent_id| new_ids.binary_search(&parent_id).is_ok()) + }) { + return None; + } + } + + Some(new_ids.len()) +} + +fn extend_mailbox_cache( + mailboxes: &MailboxesCache, + changes: Vec, + appended: usize, +) -> MailboxesCache { + let first_new = mailboxes.items.len(); + let mut cache = MailboxesCacheBuilder { + items: Vec::with_capacity(first_new + appended), + index: mailboxes.index.clone(), + size: mailboxes.size, + change_id: 0, + }; + cache.items.extend_from_slice(&mailboxes.items); + cache.index.reserve(appended); + + let mut topological_sort = TopologicalSort::with_capacity(appended); + for mut item in changes { + if let Some(idx) = cache.index.get(&item.document_id) + && let Some(current) = cache.items.get_mut(*idx as usize) + { + item.name = current.name.clone(); + item.path = current.path.clone(); + *current = item; + } else { + let idx = cache.items.len(); + item.path = base_path(&item, idx); + topological_sort.insert(sort_parent(&item), item.document_id + 1); + cache.index.insert(item.document_id, idx as u32); + cache.items.push(item); + } + } + + resolve_paths(&mut cache, topological_sort, first_new); + cache.size += cache + .items + .get(first_new..) + .unwrap_or_default() + .iter() + .map(item_size) + .sum::(); + + cache.build() +} + +fn rebuild_mailbox_cache( + mailboxes: &MailboxesCache, + changed_ids: &AHashMap, + changes: Vec, +) -> MailboxesCache { + let mut cache = MailboxesCacheBuilder { + items: Vec::with_capacity(mailboxes.items.len()), + index: AHashMap::with_capacity(mailboxes.items.len()), + size: 0, + change_id: 0, + }; + + for item in changes { + mailbox_insert(&mut cache, item); + } + + for item in mailboxes.items.iter() { if !changed_ids.contains_key(&item.document_id) { - mailbox_insert(&mut new_cache, item.clone()); + mailbox_insert(&mut cache, item.clone()); } } - build_tree(&mut new_cache); + build_tree(&mut cache); - Ok(new_cache.build()) + cache.build() } pub(crate) async fn full_mailbox_cache_build( @@ -89,6 +301,7 @@ pub(crate) async fn full_mailbox_cache_build( .archives( account_id, Collection::Mailbox, + Field::ARCHIVE, &(), |document_id, archive| { insert_item(&mut cache, document_id, archive.unarchive::()?); @@ -107,6 +320,7 @@ pub(crate) async fn full_mailbox_cache_build( .archives( account_id, Collection::Mailbox, + Field::ARCHIVE, &(), |document_id, archive| { insert_item(&mut cache, document_id, archive.unarchive::()?); @@ -123,12 +337,17 @@ pub(crate) async fn full_mailbox_cache_build( } fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &ArchivedMailbox) { + mailbox_insert(cache, mailbox_item(document_id, mailbox)); +} + +fn mailbox_item(document_id: u32, mailbox: &ArchivedMailbox) -> MailboxCache { let parent_id = mailbox.parent_id.to_native(); - let item = MailboxCache { + let name: Arc = mailbox.name.as_str().into(); + MailboxCache { document_id, - name: mailbox.name.as_str().into(), - path: "".into(), - role: (&mailbox.role).into(), + path: name.clone(), + name, + role: mailbox.role(), parent_id: if parent_id > 0 { parent_id - 1 } else { @@ -149,59 +368,85 @@ fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &Ar grants: Bitmap::from(&acl.grants), }) .collect(), - }; - - mailbox_insert(cache, item); + metadata_kinds: mailbox.metadata_kinds(), + } } fn build_tree(cache: &mut MailboxesCacheBuilder) { - cache.size = 0; let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); for (idx, mailbox) in cache.items.iter_mut().enumerate() { - topological_sort.insert( - if mailbox.parent_id == u32::MAX { - 0 - } else { - mailbox.parent_id + 1 - }, - mailbox.document_id + 1, - ); - mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { - "INBOX".into() - } else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") { - format!("INBOX {}", idx + 1) - } else { - mailbox.name.clone() - }; - - cache.size += item_size(mailbox); + topological_sort.insert(sort_parent(mailbox), mailbox.document_id + 1); + mailbox.path = base_path(mailbox, idx); } + resolve_paths(cache, topological_sort, 0); + + cache.size = cache.items.iter().map(item_size).sum(); +} + +fn sort_parent(mailbox: &MailboxCache) -> u32 { + if mailbox.parent_id == u32::MAX { + 0 + } else { + mailbox.parent_id + 1 + } +} + +fn base_path(mailbox: &MailboxCache, idx: usize) -> Arc { + if matches!(mailbox.role, SpecialUse::Inbox) { + "INBOX".into() + } else if mailbox.is_root() && mailbox.name.eq_ignore_ascii_case("inbox") { + format!("INBOX {}", idx + 1).into() + } else { + mailbox.name.clone() + } +} + +fn resolve_paths( + cache: &mut MailboxesCacheBuilder, + topological_sort: TopologicalSort, + first_idx: usize, +) { + let mut path = String::new(); for folder_id in topological_sort.into_iterator() { - if folder_id != 0 { - let folder_id = folder_id - 1; - if let Some((path, parent_path)) = by_id(cache, &folder_id) - .and_then(|folder| { - folder - .parent_id() - .map(|parent_id| (&folder.path, parent_id)) - }) - .and_then(|(path, parent_id)| { - by_id(cache, &parent_id).map(|folder| (path, &folder.path)) - }) - { - let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1); - new_path.push_str(parent_path.as_str()); - new_path.push('/'); - new_path.push_str(path.as_str()); - let folder = by_id_mut(cache, &folder_id).unwrap(); - folder.path = new_path; + if let Some(folder_id) = folder_id.checked_sub(1) + && let Some(idx) = cache.index.get(&folder_id).map(|idx| *idx as usize) + && idx >= first_idx + && let Some(folder) = cache.items.get(idx) + && let Some(parent) = folder + .parent_id() + .and_then(|parent_id| by_id(cache, &parent_id)) + { + path.clear(); + path.push_str(&parent.path); + path.push('/'); + path.push_str(&folder.path); + if let Some(folder) = cache.items.get_mut(idx) { + folder.path = path.as_str().into(); } } } } +#[cfg(all(test, stalwart_tests))] +pub(crate) fn mailboxes_cache_from(mailboxes: &[(u32, Mailbox)]) -> MailboxesCache { + let mut cache = MailboxesCacheBuilder { + items: Default::default(), + index: Default::default(), + size: 0, + change_id: 0, + }; + for (document_id, mailbox) in mailboxes { + let bytes = rkyv::to_bytes::(mailbox).expect("the mailbox archives"); + let archived = rkyv::access::(&bytes) + .expect("the archive validates"); + insert_item(&mut cache, *document_id, archived); + } + build_tree(&mut cache); + cache.build() +} + impl MailboxesCacheBuilder { fn build(mut self) -> MailboxesCache { self.index.shrink_to_fit(); @@ -218,6 +463,8 @@ pub trait MailboxCacheAccess { fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>; fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>; fn mailbox_by_path(&self, name: &str) -> Option<&MailboxCache>; + fn mailbox_by_folded_path(&self, path: &str) -> Option<&MailboxCache>; + fn mailbox_by_lowercase_path(&self, lowercase_path: &str) -> Option<&MailboxCache>; fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>; fn shared_mailboxes( &self, @@ -227,23 +474,43 @@ pub trait MailboxCacheAccess { fn has_mailbox_id(&self, id: &u32) -> bool; } -impl MailboxCacheAccess for MessageStoreCache { +impl MailboxCacheAccess for MailboxesCache { fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> { - self.mailboxes - .items + self.items .iter() .find(|m| m.name.eq_ignore_ascii_case(name)) } fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> { - self.mailboxes - .items + self.items .iter() .find(|m| m.path.eq_ignore_ascii_case(path)) } + fn mailbox_by_folded_path(&self, path: &str) -> Option<&MailboxCache> { + let mut folded_path = String::with_capacity(path.len()); + let mut found = None; + for name in path.split('/').map(str::trim) { + if !folded_path.is_empty() { + folded_path.push('/'); + } + folded_path.extend(name.chars().flat_map(char::to_lowercase)); + found = Some(self.mailbox_by_lowercase_path(&folded_path)?); + } + found + } + + fn mailbox_by_lowercase_path(&self, lowercase_path: &str) -> Option<&MailboxCache> { + self.items.iter().find(|m| { + m.path + .chars() + .flat_map(char::to_lowercase) + .eq(lowercase_path.chars()) + }) + } + fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { - self.mailboxes.items.iter().find(|m| &m.role == role) + self.items.iter().find(|m| &m.role == role) } fn shared_mailboxes( @@ -254,8 +521,7 @@ impl MailboxCacheAccess for MessageStoreCache { let check_acls = check_acls.into(); RoaringBitmap::from_iter( - self.mailboxes - .items + self.items .iter() .filter(|m| { m.acls @@ -268,14 +534,51 @@ impl MailboxCacheAccess for MessageStoreCache { } fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> { - self.mailboxes - .index + self.index .get(id) - .and_then(|idx| self.mailboxes.items.get(*idx as usize)) + .and_then(|idx| self.items.get(*idx as usize)) } fn has_mailbox_id(&self, id: &u32) -> bool { - self.mailboxes.index.contains_key(id) + self.index.contains_key(id) + } +} + +impl MailboxCacheAccess for MessageStoreCache { + fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_id(id) + } + + fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_name(name) + } + + fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_path(path) + } + + fn mailbox_by_folded_path(&self, path: &str) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_folded_path(path) + } + + fn mailbox_by_lowercase_path(&self, lowercase_path: &str) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_lowercase_path(lowercase_path) + } + + fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { + self.mailboxes.mailbox_by_role(role) + } + + fn shared_mailboxes( + &self, + access_token: &AccessToken, + check_acls: impl Into> + Sync + Send, + ) -> RoaringBitmap { + self.mailboxes.shared_mailboxes(access_token, check_acls) + } + + fn has_mailbox_id(&self, id: &u32) -> bool { + self.mailboxes.has_mailbox_id(id) } } @@ -287,14 +590,6 @@ fn by_id<'x>(cache: &'x MailboxesCacheBuilder, id: &u32) -> Option<&'x MailboxCa .and_then(|idx| cache.items.get(*idx as usize)) } -#[inline(always)] -fn by_id_mut<'x>(cache: &'x mut MailboxesCacheBuilder, id: &u32) -> Option<&'x mut MailboxCache> { - cache - .index - .get(id) - .and_then(|idx| cache.items.get_mut(*idx as usize)) -} - fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) { let id = item.document_id; if let Some(idx) = cache.index.get(&id) { @@ -306,17 +601,16 @@ fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) { } } -#[inline(always)] fn item_size(item: &MailboxCache) -> u64 { - (std::mem::size_of::() - + (if item.name.len() > std::mem::size_of::() { - item.name.len() - } else { - 0 - }) - + (if item.path.len() > std::mem::size_of::() { - item.path.len() - } else { - 0 - })) as u64 + let path_size = if Arc::ptr_eq(&item.name, &item.path) { + 0 + } else { + shared_str_size(&item.path) + }; + (size_of::() + shared_str_size(&item.name) + path_size) as u64 } + +fn shared_str_size(value: &Arc) -> usize { + 2 * size_of::() + value.len() +} + diff --git a/crates/email/src/cache/mod.rs b/crates/email/src/cache/mod.rs index 089b8386..fb48ec53 100644 --- a/crates/email/src/cache/mod.rs +++ b/crates/email/src/cache/mod.rs @@ -4,17 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{MessageStoreCache, Server, UpdateLock, cache::LockResult}; -use email::{full_email_cache_build, update_email_cache}; +use common::{ + ItemDelta, MessageCache, MessageStoreCache, MessageUpdateLock, MessagesCache, Server, + Verification, + cache::{ + refresh::MAX_DELTA_IDS, + swap::{BLOCKING_CODEC_THRESHOLD, SwapKey, writer::SwapChanges}, + }, +}; +use email::{KeywordWrites, full_email_cache_build, update_email_cache}; use mailbox::{full_mailbox_cache_build, update_mailbox_cache}; use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; use store::{ ahash::AHashMap, - query::log::{Change, Query}, + query::log::{Change, Changes, Query}, + write::AssignedIds, }; -use trc::{AddContext, StoreEvent}; +use trc::{AddContext, CacheEvent}; use types::collection::SyncCollection; -use utils::cache::Cache; pub mod email; pub mod mailbox; @@ -24,122 +31,374 @@ pub trait MessageCacheFetch: Sync + Send { &self, account_id: u32, ) -> impl Future>> + Send; + + fn get_resident_messages( + &self, + account_id: u32, + ) -> impl Future>>> + Send; + + fn fresh_messages(&self, account_id: u32) -> Option>; + + fn apply_keyword_writes( + &self, + account_id: u32, + cache: &Arc, + writes: &KeywordWrites, + assigned_ids: &AssignedIds, + ) -> bool; } impl MessageCacheFetch for Server { + fn fresh_messages(&self, account_id: u32) -> Option> { + let caches = &self.inner.cache; + caches + .peek_messages(account_id) + .filter(|cache| cache.is_fresh(&caches.revalidate)) + } + async fn get_cached_messages(&self, account_id: u32) -> trc::Result> { - let cache_store = &self.inner.cache.messages; - let mut cache = match cache_store.get_value_or_guard_async(&account_id).await { - Ok(cache) => cache, - Err(guard) => { - let start_time = Instant::now(); - let cache = full_cache_build(self, account_id, Arc::new(UpdateLock::new())).await?; - - if guard.insert(cache.clone()).is_err() { - cache_store.update(account_id, cache.clone()); - } - warn_if_uncacheable(cache_store, account_id, &cache); - - trc::event!( - Store(StoreEvent::CacheMiss), - AccountId = account_id, - Collection = SyncCollection::Email.as_str(), - Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], - ChangeId = cache.last_change_id, - Elapsed = start_time.elapsed(), - ); - - return Ok(cache); - } + let cache = match self.inner.cache.resident_messages(account_id) { + Some(cache) => cache, + None => match load_cache(self, account_id).await? { + LoadedCache::Resident(cache) => cache, + LoadedCache::Built(cache) => return Ok(cache), + }, }; + revalidate_messages(self, account_id, cache).await + } - // Obtain current state + async fn get_resident_messages( + &self, + account_id: u32, + ) -> trc::Result>> { + match self.inner.cache.resident_messages(account_id) { + Some(cache) => revalidate_messages(self, account_id, cache).await.map(Some), + None => Ok(None), + } + } + + fn apply_keyword_writes( + &self, + account_id: u32, + cache: &Arc, + writes: &KeywordWrites, + assigned_ids: &AssignedIds, + ) -> bool { let start_time = Instant::now(); - let changes = self - .core - .storage - .data - .changes( - account_id, - SyncCollection::Email.into(), - Query::Since(cache.last_change_id), - ) - .await - .caused_by(trc::location!())?; - - // Regenerate cache if the change log has been truncated - if changes.is_truncated { - let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; - cache_store.update(account_id, cache.clone()); - warn_if_uncacheable(cache_store, account_id, &cache); - - trc::event!( - Store(StoreEvent::CacheStale), - AccountId = account_id, - Collection = SyncCollection::Email.as_str(), - ChangeId = cache.last_change_id, - Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], - Elapsed = start_time.elapsed(), - ); - - return Ok(cache); - } - - // Verify changes - if changes.changes.is_empty() { - trc::event!( - Store(StoreEvent::CacheHit), - AccountId = account_id, - Collection = SyncCollection::Email.as_str(), - ChangeId = cache.last_change_id, - Elapsed = start_time.elapsed(), - ); - - return Ok(cache); - } - - // Lock for updates - let lock = cache.update_lock.clone(); - let _permit = match lock.acquire(cache.last_change_id).await? { - LockResult::Acquired(permit) => permit, - LockResult::Stale(permit) => { - cache = cache_store.peek(&account_id).unwrap_or(cache.clone()); - if cache.last_change_id >= changes.to_change_id { - trc::event!( - Store(StoreEvent::CacheHit), - AccountId = account_id, - Collection = SyncCollection::Email.as_str(), - ChangeId = cache.last_change_id, - Elapsed = start_time.elapsed(), - ); - return Ok(cache); - } - - permit - } + let change_ids = assigned_ids + .change_ids(account_id, SyncCollection::Email) + .collect::>(); + let (Some(&last_change_id), Some(last_write)) = (change_ids.last(), writes.writes().last()) + else { + return false; }; - let mut cache = cache.as_ref().clone(); + if last_write.commit_point as usize + 1 != change_ids.len() + || !change_ids + .iter() + .copied() + .eq(cache.last_change_id + 1..=last_change_id) + { + return false; + } - let mut changed_items: AHashMap = AHashMap::with_capacity(changes.changes.len()); - let mut changed_containers: AHashMap = - AHashMap::with_capacity(changes.changes.len()); + let caches = &self.inner.cache; + let Some(_permit) = cache.update_lock.try_update() else { + return false; + }; + if !caches + .peek_messages(account_id) + .is_some_and(|resident| Arc::ptr_eq(&resident, cache)) + { + return false; + } + + let mut patches = Vec::with_capacity(writes.writes().len()); + for write in writes.writes() { + let Some(message) = cache + .emails + .at(write.position) + .filter(|message| message.document_id() == write.document_id) + else { + return false; + }; + if message.keywords() != write.keywords { + let Some(&change_id) = change_ids.get(write.commit_point as usize) else { + return false; + }; + patches.push(( + write.position, + MessageCache::new( + write.document_id, + message.mailboxes().iter().copied().collect(), + write.keywords, + message.thread_id(), + change_id, + message.size(), + message.received_at(), + message.sent_at(), + ), + )); + } + } + + let refresh = if writes.writes().len() <= MAX_DELTA_IDS { + let mut view_ids = writes + .writes() + .iter() + .map(|write| write.document_id) + .collect::>(); + view_ids.sort_unstable(); + view_ids.dedup(); + ItemDelta::new(cache.emails.change_id, last_change_id, &view_ids) + .map(|delta| cache.refresh.advance(delta)) + } else { + None + } + .unwrap_or_default(); + + let emails = cache + .emails + .patch(last_change_id, &patches, cache.emails.shared_keywords()); + let mailboxes = if writes.mailboxes_changed() { + let mut mailboxes = cache.mailboxes.as_ref().clone(); + mailboxes.change_id = last_change_id; + Arc::new(mailboxes) + } else { + cache.mailboxes.clone() + }; + let updated = Arc::new(MessageStoreCache { + size: emails.size + mailboxes.size, + emails: Arc::new(emails), + mailboxes, + update_lock: cache.update_lock.clone(), + refresh: Arc::new(refresh), + last_change_id, + verification: Verification::default(), + }); + caches.publish_messages( + account_id, + &updated, + Some(SwapChanges::applied( + writes.writes().len(), + cache.last_change_id, + last_change_id, + updated.emails.len(), + )), + ); + + trc::event!( + Cache(CacheEvent::Update), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = last_change_id, + Details = vec![writes.writes().len(), 0], + Total = vec![updated.emails.len(), updated.mailboxes.items.len()], + Elapsed = start_time.elapsed(), + ); + + true + } +} + +async fn revalidate_messages( + server: &Server, + account_id: u32, + cache: Arc, +) -> trc::Result> { + let caches = &server.inner.cache; + + // Serve the snapshot without revalidating while it is within the freshness window + let start_time = Instant::now(); + let revalidate = &caches.revalidate; + let served = |cache: Arc| { + trc::event!( + Cache(CacheEvent::Hit), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = cache.last_change_id, + Elapsed = start_time.elapsed(), + ); + Ok(cache) + }; + if cache.is_fresh(revalidate) { + return served(cache); + } + + let lock = cache.update_lock.clone(); + let entry_epoch = lock.epoch(); + let (_permit, cache) = match lock.try_update() { + Some(permit) => (permit, cache), + None => loop { + let released = lock.released(); + let permit = lock.try_update(); + if permit.is_none() { + released.await; + } + match (caches.peek_messages(account_id), permit) { + (Some(current), _) + if current.is_fresh(revalidate) + || (Arc::ptr_eq(¤t.update_lock, &lock) + && current + .verification + .is_verified_since(entry_epoch, revalidate)) => + { + return served(current); + } + (current, Some(permit)) => break (permit, current.unwrap_or(cache)), + _ => {} + } + }, + }; + + // Obtain current state + let verification = Verification::capture(revalidate, &cache.update_lock); + let changes = server + .core + .storage + .data + .changes( + account_id, + SyncCollection::Email.into(), + Query::Since(cache.last_change_id), + ) + .await + .caused_by(trc::location!())?; + + // Regenerate cache if the change log has been truncated + if changes.needs_full_rebuild(cache.last_change_id) { + let mut cache = + full_cache_build(server, account_id, cache.update_lock.clone(), verification).await?; + if !is_resident_lock(server, account_id, &cache.update_lock) + && let Some(cache) = Arc::get_mut(&mut cache) + { + cache.verification = Verification::default(); + } + caches.publish_messages( + account_id, + &cache, + Some(SwapChanges::rebuilt( + cache.last_change_id, + cache.emails.len(), + )), + ); + + trc::event!( + Cache(CacheEvent::Stale), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = cache.last_change_id, + Total = vec![cache.emails.len(), cache.mailboxes.items.len()], + Elapsed = start_time.elapsed(), + ); + + return Ok(cache); + } + + // Verify changes + if changes.changes.is_empty() { + let is_current = verification.is_current(&cache.update_lock); + let is_advanced = changes.to_change_id > cache.last_change_id; + let cache = if is_current || is_advanced { + let mut verified = cache.as_ref().clone(); + if is_current && is_resident_lock(server, account_id, &verified.update_lock) { + verified.verification = verification; + } + if is_advanced { + verified.last_change_id = changes.to_change_id; + if changes.item_change_id.is_some() { + verified.refresh = Arc::new(verified.refresh.with_new_uid_next()); + } + } + let verified = Arc::new(verified); + caches.publish_messages(account_id, &verified, None); + verified + } else { + cache + }; + + trc::event!( + Cache(CacheEvent::Hit), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = cache.last_change_id, + Elapsed = start_time.elapsed(), + ); + + return Ok(cache); + } + + let change_set = ChangeSet::classify(changes); + let base_change_id = cache.last_change_id; + let mut cache = change_set.apply(server, account_id, cache.as_ref()).await?; + if is_resident_lock(server, account_id, &cache.update_lock) { + cache.verification = verification; + } + + let cache = Arc::new(cache); + caches.publish_messages( + account_id, + &cache, + Some(SwapChanges::applied( + change_set.items.len(), + base_change_id, + cache.last_change_id, + cache.emails.len(), + )), + ); + + trc::event!( + Cache(CacheEvent::Update), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = cache.last_change_id, + Details = vec![change_set.items.len(), change_set.containers.len()], + Total = vec![cache.emails.len(), cache.mailboxes.items.len()], + Elapsed = start_time.elapsed(), + ); + + Ok(cache) +} + +struct ChangeSet { + items: AHashMap, + containers: AHashMap, + view_ids: Vec, + has_container_property_changes: bool, + item_change_id: Option, + container_change_id: Option, + to_change_id: u64, +} + +impl ChangeSet { + fn classify(changes: Changes) -> Self { + let mut items: AHashMap = AHashMap::with_capacity(changes.changes.len()); + let mut containers: AHashMap = AHashMap::with_capacity(changes.changes.len()); + let mut view_ids = Vec::with_capacity(changes.changes.len()); let mut has_container_property_changes = false; for change in changes.changes { match change { - Change::InsertItem(id) => match changed_items.entry(id as u32) { - Entry::Occupied(mut entry) => { - *entry.get_mut() = true; + Change::InsertItem(id) => { + view_ids.push(id as u32); + match items.entry(id as u32) { + Entry::Occupied(mut entry) => { + *entry.get_mut() = true; + } + Entry::Vacant(entry) => { + entry.insert(true); + } } - Entry::Vacant(entry) => { - entry.insert(true); - } - }, + } Change::UpdateItem(id) => { - changed_items.insert(id as u32, true); + view_ids.push(id as u32); + items.insert(id as u32, true); + } + Change::UpdateItemMetadata(id) => { + items.insert(id as u32, true); } Change::DeleteItem(id) => { - match changed_items.entry(id as u32) { + view_ids.push(id as u32); + match items.entry(id as u32) { Entry::Occupied(mut entry) => { // Thread reassignment *entry.get_mut() = true; @@ -150,78 +409,246 @@ impl MessageCacheFetch for Server { } } Change::InsertContainer(id) | Change::UpdateContainer(id) => { - changed_containers.insert(id as u32, true); + containers.insert(id as u32, true); } Change::DeleteContainer(id) => { - changed_containers.insert(id as u32, false); + containers.insert(id as u32, false); } - Change::UpdateContainerProperty(_) => { - has_container_property_changes = true; + Change::UpdateContainerPartial(id, partial) => { + if partial.has_metadata() { + containers.insert(id as u32, true); + } else { + has_container_property_changes = true; + } } } } - if !changed_items.is_empty() { + view_ids.sort_unstable(); + view_ids.dedup(); + + ChangeSet { + items, + containers, + view_ids, + has_container_property_changes, + item_change_id: changes.item_change_id, + container_change_id: changes.container_change_id, + to_change_id: changes.to_change_id, + } + } + + async fn apply( + &self, + server: &Server, + account_id: u32, + previous: &MessageStoreCache, + ) -> trc::Result { + let mut cache = previous.clone(); + + if !self.items.is_empty() { let mut email_cache = - update_email_cache(self, account_id, &changed_items, &cache).await?; - email_cache.change_id = changes.item_change_id.unwrap_or(changes.to_change_id); + update_email_cache(server, account_id, &self.items, &cache).await?; + email_cache.change_id = self.item_change_id.unwrap_or(self.to_change_id); + cache.refresh = Arc::new( + ItemDelta::new( + previous.emails.change_id, + email_cache.change_id, + &self.view_ids, + ) + .map(|delta| previous.refresh.advance(delta)) + .unwrap_or_default(), + ); cache.emails = Arc::new(email_cache); + } else if self.item_change_id.is_some() + || !self.containers.is_empty() + || self.has_container_property_changes + { + cache.refresh = Arc::new(previous.refresh.with_new_uid_next()); } - if !changed_containers.is_empty() { + if !self.containers.is_empty() { let mut mailbox_cache = - update_mailbox_cache(self, account_id, &changed_containers, &cache).await?; - mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id); + update_mailbox_cache(server, account_id, &self.containers, &cache.mailboxes) + .await?; + mailbox_cache.change_id = self.container_change_id.unwrap_or(self.to_change_id); cache.mailboxes = Arc::new(mailbox_cache); - } else if has_container_property_changes { + } else if self.has_container_property_changes { let mut mailbox_cache = cache.mailboxes.as_ref().clone(); - mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id); + mailbox_cache.change_id = self.container_change_id.unwrap_or(self.to_change_id); cache.mailboxes = Arc::new(mailbox_cache); } + cache.size = cache.emails.size + cache.mailboxes.size; - cache.last_change_id = changes.to_change_id; - - cache.update_lock.set_revision(cache.last_change_id); - let cache = Arc::new(cache); - cache_store.update(account_id, cache.clone()); - warn_if_uncacheable(cache_store, account_id, &cache); - - trc::event!( - Store(StoreEvent::CacheUpdate), - AccountId = account_id, - Collection = SyncCollection::Email.as_str(), - ChangeId = cache.last_change_id, - Details = vec![changed_items.len(), changed_containers.len()], - Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], - Elapsed = start_time.elapsed(), - ); + cache.last_change_id = self.to_change_id; Ok(cache) } } -#[inline(always)] -fn warn_if_uncacheable( - cache_store: &Cache>, +fn is_resident_lock( + server: &Server, account_id: u32, - cache: &Arc, -) { - let capacity = cache_store.weight_capacity(); - if cache.size > capacity { + update_lock: &Arc, +) -> bool { + server + .inner + .cache + .peek_messages(account_id) + .is_some_and(|resident| Arc::ptr_eq(&resident.update_lock, update_lock)) +} + +enum LoadedCache { + Resident(Arc), + Built(Arc), +} + +async fn load_cache(server: &Server, account_id: u32) -> trc::Result { + Box::pin(load_missing_cache(server, account_id)).await +} + +async fn load_missing_cache(server: &Server, account_id: u32) -> trc::Result { + let caches = &server.inner.cache; + let guard = match caches.messages.get_value_or_guard_async(&account_id).await { + Ok(cache) => return Ok(LoadedCache::Resident(cache)), + Err(guard) => guard, + }; + + if let Some(cache) = caches.messages_large.lookup(account_id) { + drop(guard); + return Ok(LoadedCache::Resident(cache)); + } + + let start_time = Instant::now(); + if let Some(cache) = restore_cache_build(server, account_id).await { + caches.publish_loaded_messages(guard, account_id, &cache, None); + return Ok(LoadedCache::Resident(cache)); + } + + let cache = full_cache_build(server, account_id, Arc::default(), Default::default()).await?; + caches.publish_loaded_messages( + guard, + account_id, + &cache, + Some(SwapChanges::rebuilt( + cache.last_change_id, + cache.emails.len(), + )), + ); + + trc::event!( + Cache(CacheEvent::Miss), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + Total = vec![cache.emails.len(), cache.mailboxes.items.len()], + ChangeId = cache.last_change_id, + Elapsed = start_time.elapsed(), + ); + + Ok(LoadedCache::Built(cache)) +} + +async fn restore_cache_build(server: &Server, account_id: u32) -> Option> { + if !server.inner.cache.swap.is_enabled() { + return None; + } + + Box::pin(restore_swapped_cache(server, account_id)).await +} + +async fn restore_swapped_cache(server: &Server, account_id: u32) -> Option> { + let start_time = Instant::now(); + let snapshot = match server + .inner + .cache + .swap + .load(SwapKey::messages(account_id)) + .await + { + Ok(Some(snapshot)) => snapshot, + Ok(None) => return None, + Err(err) => { + trc::error!( + err.details("Failed to read the message cache snapshot") + .ctx(trc::Key::AccountId, account_id) + ); + return None; + } + }; + + let snapshot_len = snapshot.len(); + let emails = if snapshot_len >= BLOCKING_CODEC_THRESHOLD { + tokio::task::spawn_blocking(move || MessagesCache::from_snapshot(&snapshot)) + .await + .ok() + .flatten() + } else { + MessagesCache::from_snapshot(&snapshot) + } + .or_else(|| { trc::event!( - Store(StoreEvent::CacheEntryTooLarge), + Cache(CacheEvent::SwapMiss), AccountId = account_id, Collection = SyncCollection::Email.as_str(), - Size = cache.size, - Limit = capacity, + Details = "Discarded an unreadable message cache snapshot", ); - } + None + })?; + + let snapshot_change_id = emails.change_id; + let mut mailboxes = full_mailbox_cache_build(server, account_id) + .await + .caused_by(trc::location!()) + .inspect_err(|err| { + trc::error!(err.clone()); + }) + .ok()?; + mailboxes.change_id = snapshot_change_id; + + let size = emails.size + mailboxes.size; + let cache = MessageStoreCache { + update_lock: Arc::default(), + refresh: Default::default(), + emails: Arc::new(emails), + mailboxes: Arc::new(mailboxes), + last_change_id: snapshot_change_id, + size, + verification: Default::default(), + }; + + trc::event!( + Cache(CacheEvent::SwapHit), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + ChangeId = snapshot_change_id, + Total = vec![cache.emails.len(), cache.mailboxes.items.len()], + Size = snapshot_len, + Elapsed = start_time.elapsed(), + ); + + Some(Arc::new(cache)) } async fn full_cache_build( server: &Server, account_id: u32, - update_lock: Arc, + update_lock: Arc, + verification: Verification, +) -> trc::Result> { + Box::pin(build_full_cache( + server, + account_id, + update_lock, + verification, + )) + .await +} + +async fn build_full_cache( + server: &Server, + account_id: u32, + update_lock: Arc, + verification: Verification, ) -> trc::Result> { let last_change_id = server .core @@ -236,13 +663,15 @@ async fn full_cache_build( let size = emails.size + mailboxes.size; emails.change_id = last_change_id; mailboxes.change_id = last_change_id; - update_lock.set_revision(last_change_id); Ok(Arc::new(MessageStoreCache { update_lock, + refresh: Default::default(), emails: Arc::new(emails), mailboxes: Arc::new(mailboxes), last_change_id, size, + verification, })) } + diff --git a/crates/email/src/identity/index.rs b/crates/email/src/identity/index.rs index c42d1fcb..36d84094 100644 --- a/crates/email/src/identity/index.rs +++ b/crates/email/src/identity/index.rs @@ -5,7 +5,11 @@ */ use super::{ArchivedIdentity, Identity}; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use common::storage::index::{ + IndexValue, IndexableAndSerializableObject, IndexableObject, SerializableObject, + serialize_object, +}; +use store::write::{ArchiveCompression, BatchBuilder, Compression, Dictionary, Slot}; use types::collection::SyncCollection; impl IndexableObject for Identity { @@ -33,3 +37,13 @@ impl IndexableAndSerializableObject for Identity { false } } + +impl ArchiveCompression for Identity { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Common)); +} + +impl SerializableObject for Identity { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs index 815803d7..21f8087f 100644 --- a/crates/email/src/lib.rs +++ b/crates/email/src/lib.rs @@ -10,6 +10,7 @@ pub mod cache; pub mod identity; pub mod mailbox; pub mod message; +pub mod presence; pub mod push; pub mod sieve; pub mod submission; diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index aac155a8..faf28a35 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -7,18 +7,20 @@ use super::*; use crate::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::{delete::EmailDeletion, metadata::MessageData}, + message::{ + delete::EmailDeletion, + messagedata::{EmailMessageData, MessageData}, + }, }; use common::{ - Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, -}; -use registry::schema::{ - enums::IndexDocumentType, - structs::{Task, TaskIndexDocument, TaskStatus}, + Server, + auth::AccessToken, + sharing::EffectiveAcl, + storage::{index::ObjectIndexBuilder, metadata::ContainerCleanup}, }; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes, SearchIndex}, }; use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; @@ -35,6 +37,7 @@ pub trait MailboxDestroy: Sync + Send { document_id: u32, access_token: &AccessToken, remove_emails: bool, + containers: Option<&ContainerCleanup>, ) -> impl Future, MailboxDestroyError>>> + Send; } @@ -54,9 +57,10 @@ impl MailboxDestroy for Server { document_id: u32, access_token: &AccessToken, remove_emails: bool, + containers: Option<&ContainerCleanup>, ) -> trc::Result, MailboxDestroyError>> { // Internal folders cannot be deleted - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] if [INBOX_ID, TRASH_ID, JUNK_ID].contains(&document_id) { return Ok(Err(MailboxDestroyError::CannotDestroy)); } @@ -80,96 +84,99 @@ impl MailboxDestroy for Server { batch.with_account_id(account_id); - let message_ids = - RoaringBitmap::from_iter(cache.in_mailbox(document_id).map(|m| m.document_id)); + let mut message_ids = RoaringBitmap::new(); + let mut flagged_ids = RoaringBitmap::new(); + for message in cache.in_mailbox(document_id) { + message_ids.insert(message.document_id()); + if message.mailboxes().len() == 1 && !cache.metadata_kinds(message).is_empty() { + flagged_ids.insert(message.document_id()); + } + } if !message_ids.is_empty() { if remove_emails { // If the message is in multiple mailboxes, untag it from the current mailbox, // otherwise delete it. + let message_containers = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::Email, + &flagged_ids, + ) + .await + .caused_by(trc::location!())?; let mut deleted_ids = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); - self.archives( - account_id, - Collection::Email, - &message_ids, - |message_id, message_data_| { - // Remove mailbox from list - let prev_message_data = message_data_ - .to_unarchived::() - .caused_by(trc::location!())?; - if !prev_message_data - .inner - .mailboxes - .iter() - .any(|id| id.mailbox_id == document_id) - { - return Ok(true); + self.message_datas(account_id, &message_ids, |message_id, prev_message_data| { + // Remove mailbox from list + if !prev_message_data + .mailboxes + .iter() + .any(|id| id.mailbox_id == document_id) + { + return Ok(true); + } + + if prev_message_data.mailboxes.len() == 1 { + // Delete message + for mailbox in prev_message_data.mailboxes.iter() { + batch.log_vanished_item( + VanishedCollection::Email, + (mailbox.mailbox_id, mailbox.uid), + ); } - - if prev_message_data.inner.mailboxes.len() == 1 { - // Delete message - for mailbox in prev_message_data.inner.mailboxes.iter() { - batch.log_vanished_item( - VanishedCollection::Email, - (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), - ); - } - deleted_ids.insert(message_id); - thread_ids.insert(prev_message_data.inner.thread_id.to_native()); - batch - .with_collection(Collection::Email) - .with_document(message_id) - .custom( - ObjectIndexBuilder::<_, ()>::new() - .with_changed_by(access_token.account_tenant_ids()) - .with_current(prev_message_data), - ) - .caused_by(trc::location!())? - .schedule_task(Task::UnindexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: message_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::now(), - })) - .commit_point(); - } else { - let new_message_data = MessageData { - mailboxes: prev_message_data - .inner - .mailboxes - .iter() - .filter(|m| m.mailbox_id != document_id) - .map(|m| m.to_native()) - .collect(), - keywords: prev_message_data - .inner - .keywords - .iter() - .map(|k| k.to_native()) - .collect(), - thread_id: prev_message_data.inner.thread_id.to_native(), - size: prev_message_data.inner.size.to_native(), - }; - - // Untag message from mailbox - batch - .with_collection(Collection::Email) - .with_document(message_id) - .custom( - ObjectIndexBuilder::new() - .with_changed_by(access_token.account_tenant_ids()) - .with_changes(new_message_data) - .with_current(prev_message_data), - ) - .caused_by(trc::location!())? - .commit_point(); + deleted_ids.insert(message_id); + thread_ids.insert(prev_message_data.thread_id); + batch.with_collection(Collection::Email); + if !prev_message_data.metadata_kinds().is_empty() { + message_containers + .release_or_assert_absent(&mut batch, account_id, message_id); } + batch + .with_document(message_id) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_changed_by(access_token.changed_by()) + .with_current(prev_message_data), + ) + .caused_by(trc::location!())? + .queue_document_unindex(SearchIndex::Email, account_id, message_id) + .commit_point(); + } else { + let new_message_data = MessageData { + mailboxes: prev_message_data + .mailboxes + .iter() + .filter(|m| m.mailbox_id != document_id) + .copied() + .collect(), + keywords: prev_message_data.keywords, + thread_id: prev_message_data.thread_id, + size: prev_message_data.size, + keywords_extra: prev_message_data.keywords_extra.to_vec(), + received_at: prev_message_data.received_at, + sent_at: prev_message_data.sent_at, + change_id: prev_message_data.change_id, + }; - Ok(true) - }, - ) + // Untag message from mailbox + batch + .with_collection(Collection::Email) + .with_document(message_id) + .custom( + ObjectIndexBuilder::new() + .with_changed_by(access_token.changed_by()) + .with_changes(new_message_data) + .with_current(prev_message_data), + ) + .caused_by(trc::location!())? + .commit_point(); + } + + Ok(true) + }) .await .caused_by(trc::location!())?; @@ -184,7 +191,7 @@ impl MailboxDestroy for Server { // Obtain mailbox if let Some(mailbox_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, document_id, @@ -205,26 +212,40 @@ impl MailboxDestroy for Server { } batch .with_account_id(account_id) - .with_collection(Collection::Mailbox) + .with_collection(Collection::Mailbox); + if !mailbox.inner.metadata_kinds().is_empty() { + match containers { + Some(containers) => { + containers.release_or_assert_absent(&mut batch, account_id, document_id) + } + None => self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::Mailbox, + &RoaringBitmap::from_iter([document_id]), + ) + .await + .caused_by(trc::location!())? + .release_or_assert_absent(&mut batch, account_id, document_id), + } + } + batch .with_document(document_id) .clear(MailboxField::UidCounter) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(mailbox)) .caused_by(trc::location!())?; } else { return Ok(Err(MailboxDestroyError::NotFound)); - }; + } if !batch.is_empty() { match self .commit_batch(batch) .await - .and_then(|ids| ids.last_change_id(account_id)) + .map(|ids| ids.last_change_id(account_id, SyncCollection::Email)) { - Ok(change_id) => { - self.notify_task_queue(); - - Ok(Ok(Some(change_id))) - } + Ok(change_id) => Ok(Ok(Some(change_id))), Err(err) if err.is_assertion_failure() => { Ok(Err(MailboxDestroyError::AssertionFailed)) } diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 0bf10a13..a8770d49 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -5,8 +5,37 @@ */ use super::{ArchivedMailbox, Mailbox}; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; -use types::{acl::AclGrant, collection::SyncCollection}; +use common::storage::index::{ + ArchivedIdField, IndexValue, IndexableAndSerializableObject, IndexableObject, + SerializableObject, serialize_object, +}; +use store::{ + U32_LEN, + write::{ArchiveCompression, BatchBuilder, Compression, IndexPropertyClass, Slot, ValueClass}, + xxhash_rust::xxh3::xxh3_128, +}; +use types::{ + acl::AclGrant, collection::SyncCollection, field::MailboxField, metadata::MetadataKinds, +}; +use utils::hash128::Hash128; + +pub(crate) fn name_index_class(lowercase_name: &[u8]) -> ValueClass { + ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: MailboxField::Name.into(), + hash: Hash128::from(xxh3_128(lowercase_name)), + }) +} + +fn name_index(name: &str, stored_parent_id: u32) -> IndexValue<'static> { + let parent_id = stored_parent_id.checked_sub(1).unwrap_or(u32::MAX); + let mut value = Vec::with_capacity(U32_LEN + name.len()); + value.extend_from_slice(&parent_id.to_be_bytes()); + value.extend(name.bytes().map(|byte| byte.to_ascii_lowercase())); + IndexValue::ParentProperty { + field: name_index_class(value.get(U32_LEN..).unwrap_or_default()), + value, + } +} impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { @@ -17,9 +46,14 @@ impl IndexableObject for Mailbox { IndexValue::Acl { value: (&self.acls).into(), }, + name_index(&self.name, self.parent_id), ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + Mailbox::metadata_kinds(self) + } } impl IndexableObject for &ArchivedMailbox { @@ -36,13 +70,32 @@ impl IndexableObject for &ArchivedMailbox { .collect::>() .into(), }, + name_index(&self.name, self.parent_id.to_native()), ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedMailbox::metadata_kinds(self) + } } impl IndexableAndSerializableObject for Mailbox { + const PENDING_ID_FIELD: Option = + Some(ArchivedIdField::new(std::mem::offset_of!(ArchivedMailbox, parent_id)).one_based()); + fn is_versioned() -> bool { false } } + +impl ArchiveCompression for Mailbox { + const COMPRESSION: Compression = Compression::None; +} + +impl SerializableObject for Mailbox { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} + diff --git a/crates/email/src/mailbox/lookup.rs b/crates/email/src/mailbox/lookup.rs new file mode 100644 index 00000000..e3f1d44f --- /dev/null +++ b/crates/email/src/mailbox/lookup.rs @@ -0,0 +1,326 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ARCHIVE_ID, DRAFTS_ID, INBOX_ID, JUNK_ID, Mailbox, SENT_ID, TRASH_ID, index::name_index_class, +}; +use crate::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; +use common::{MessageStoreCache, Server}; +use std::{future::Future, pin::Pin, sync::Arc}; +use store::{ + IterateParams, Key, U32_LEN, ValueKey, + write::{Archive, ArchiveBytes, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::{collection::Collection, special_use::SpecialUse}; + +const ROOT_ID: u32 = u32::MAX; +const INBOX_NAME: &[u8] = b"inbox"; +const MAX_PATH_SEGMENTS: usize = 64; +const MAX_JOINED_SEGMENTS: usize = 16; + +pub trait MailboxLookup: Sync + Send { + fn mailbox_id_by_path( + &self, + account_id: u32, + path: &str, + ) -> impl Future>> + Send; + + fn mailbox_id_by_role( + &self, + account_id: u32, + role: SpecialUse, + ) -> impl Future>> + Send; + + fn mailbox_id_exists( + &self, + account_id: u32, + document_id: u32, + ) -> impl Future> + Send; +} + +impl MailboxLookup for Server { + async fn mailbox_id_by_path(&self, account_id: u32, path: &str) -> trc::Result> { + let mut lookup = PathLookup::new(path); + loop { + if let Some(document_id) = lookup.resolve() { + return Ok(Some(document_id)); + } + if !lookup.next_pass() { + return Ok(None); + } + fetch_names(self, account_id, &mut lookup).await?; + } + } + + async fn mailbox_id_by_role( + &self, + account_id: u32, + role: SpecialUse, + ) -> trc::Result> { + let default_id = match role { + SpecialUse::Inbox => return Ok(Some(INBOX_ID)), + SpecialUse::Trash => return Ok(Some(TRASH_ID)), + SpecialUse::None => return Ok(None), + SpecialUse::Junk => Some(JUNK_ID), + SpecialUse::Drafts => Some(DRAFTS_ID), + SpecialUse::Sent => Some(SENT_ID), + SpecialUse::Archive => Some(ARCHIVE_ID), + SpecialUse::Important + | SpecialUse::Memos + | SpecialUse::Scheduled + | SpecialUse::Snoozed + | SpecialUse::Shared => None, + }; + + if let Some(cache) = self.fresh_messages(account_id) { + return Ok(cache + .mailbox_by_role(&role) + .map(|mailbox| mailbox.document_id)); + } + + if let Some(document_id) = default_id + && let Some(archive) = self + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + document_id, + )) + .await + .caused_by(trc::location!())? + && archive + .unarchive::() + .caused_by(trc::location!())? + .role() + == role + { + return Ok(Some(document_id)); + } + + let cache: Pin>> + Send + '_>> = + Box::pin(self.get_cached_messages(account_id)); + cache.await.caused_by(trc::location!()).map(|cache| { + cache + .mailbox_by_role(&role) + .map(|mailbox| mailbox.document_id) + }) + } + + async fn mailbox_id_exists(&self, account_id: u32, document_id: u32) -> trc::Result { + self.store() + .key_exists(ValueKey::archive( + account_id, + Collection::Mailbox, + document_id, + )) + .await + .caused_by(trc::location!()) + } +} + +struct PathLookup { + segments: Vec>, + queries: Vec, + candidates: Vec, + pass: LookupPass, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LookupPass { + Start, + Segments, + Joined, +} + +struct NameQuery { + name: Box<[u8]>, + spans: Vec<(usize, usize)>, + fetched: bool, +} + +struct Candidate { + start: usize, + end: usize, + document_id: u32, + parent_id: u32, +} + +impl PathLookup { + fn new(path: &str) -> Self { + PathLookup { + segments: path + .split('/') + .map(|segment| segment.as_bytes().to_ascii_lowercase().into_boxed_slice()) + .collect(), + queries: Vec::new(), + candidates: Vec::new(), + pass: LookupPass::Start, + } + } + + fn next_pass(&mut self) -> bool { + match self.pass { + LookupPass::Start if self.segments.len() <= MAX_PATH_SEGMENTS => { + self.pass = LookupPass::Segments; + self.query_spans(|start, end| end == start + 1); + true + } + LookupPass::Segments if (2..=MAX_JOINED_SEGMENTS).contains(&self.segments.len()) => { + self.pass = LookupPass::Joined; + self.query_spans(|start, end| end > start + 1); + true + } + LookupPass::Start | LookupPass::Segments | LookupPass::Joined => false, + } + } + + fn span_name(&self, start: usize, end: usize) -> Box<[u8]> { + let segments = self.segments.get(start..end).unwrap_or_default(); + let mut name = Vec::with_capacity( + segments + .iter() + .map(|segment| segment.len() + 1) + .sum::(), + ); + if let Some((first, rest)) = segments.split_first() { + name.extend_from_slice(first); + for segment in rest { + name.push(b'/'); + name.extend_from_slice(segment); + } + } + name.into_boxed_slice() + } + + fn is_inbox_span(&self, start: usize, end: usize) -> bool { + start == 0 + && end == 1 + && self + .segments + .first() + .is_some_and(|name| **name == *INBOX_NAME) + } + + fn query_spans(&mut self, include: impl Fn(usize, usize) -> bool) { + let num_segments = self.segments.len(); + for end in 1..=num_segments { + for start in 0..end { + if !include(start, end) || self.is_inbox_span(start, end) { + continue; + } + let name = self.span_name(start, end); + match self.queries.iter_mut().find(|query| query.name == name) { + Some(query) => query.spans.push((start, end)), + None => self.queries.push(NameQuery { + name, + spans: vec![(start, end)], + fetched: false, + }), + } + } + } + } + + fn add_candidate(&mut self, name: &[u8], document_id: u32, parent_id: u32) { + if let Some(query) = self.queries.iter().find(|query| *query.name == *name) { + self.candidates + .extend(query.spans.iter().map(|&(start, end)| Candidate { + start, + end, + document_id, + parent_id, + })); + } + } + + fn resolve(&self) -> Option { + let num_segments = self.segments.len(); + let mut reach: Vec> = Vec::with_capacity(num_segments + 1); + reach.push(vec![ROOT_ID]); + for end in 1..=num_segments { + let mut matched = Vec::new(); + for (start, parents) in reach.iter().enumerate() { + if parents.is_empty() { + continue; + } + if self.is_inbox_span(start, end) { + matched.push(INBOX_ID); + continue; + } + matched.extend( + self.candidates + .iter() + .filter(|candidate| { + candidate.start == start + && candidate.end == end + && candidate.document_id != INBOX_ID + && parents.contains(&candidate.parent_id) + }) + .map(|candidate| candidate.document_id), + ); + } + matched.sort_unstable(); + matched.dedup(); + reach.push(matched); + } + reach.last().and_then(|ids| ids.first().copied()) + } +} + +async fn fetch_names(server: &Server, account_id: u32, lookup: &mut PathLookup) -> trc::Result<()> { + let mut ranges = lookup + .queries + .iter_mut() + .filter(|query| !query.fetched) + .map(|query| { + query.fetched = true; + let class = name_index_class(&query.name); + let begin = ValueKey { + account_id, + collection: Collection::Mailbox.into(), + document_id: u32::MAX, + class: class.clone(), + }; + let end = ValueKey { + account_id, + collection: Collection::Mailbox.into(), + document_id: 0, + class, + }; + (begin.serialize(0), IterateParams::new(begin, end)) + }) + .collect::>(); + ranges.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + ranges.dedup_by(|(a, _), (b, _)| a == b); + + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = !key.deserialize_be_u32(key.len() - U32_LEN)?; + let parent_id = value.deserialize_be_u32(0)?; + lookup.add_candidate( + value.get(U32_LEN..).unwrap_or_default(), + document_id, + parent_id, + ); + Ok(true) + }; + + let store = server.store(); + if ranges.len() > 1 { + store + .iterate_many( + ranges.into_iter().map(|(_, params)| params).collect(), + &mut collect, + ) + .await + } else if let Some((_, params)) = ranges.pop() { + store.iterate(params, &mut collect).await + } else { + Ok(()) + } + .caused_by(trc::location!()) +} + diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index 3bc662a9..f6a7fe6b 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -5,7 +5,7 @@ */ use super::*; -use crate::cache::MessageCacheFetch; +use crate::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use common::{Server, storage::index::ObjectIndexBuilder}; use registry::schema::enums::StorageQuota; use std::future::Future; @@ -28,7 +28,7 @@ pub trait MailboxFnc: Sync + Send { impl MailboxFnc for Server { async fn create_system_folders(&self, account_id: u32) -> trc::Result<()> { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if account_id == 0 { return Ok(()); } @@ -41,6 +41,9 @@ impl MailboxFnc for Server { // Create mailboxes let mut last_document_id = ARCHIVE_ID; for folder in &self.core.email.default_folders { + if !folder.create { + continue; + } let document_id = match folder.special_use { SpecialUse::Inbox => INBOX_ID, SpecialUse::Trash => TRASH_ID, @@ -68,17 +71,10 @@ impl MailboxFnc for Server { .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object)) .caused_by(trc::location!())?; } - self.store() - .assign_document_ids(account_id, Collection::Mailbox, (ARCHIVE_ID + 1) as u64) - .await - .caused_by(trc::location!())?; - self.core - .storage - .data - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; + batch.reserve_document_ids(account_id, Collection::Mailbox, last_document_id + 1); + + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(()) } @@ -92,32 +88,20 @@ impl MailboxFnc for Server { let mut next_parent_id = 0; let mut create_paths = Vec::with_capacity(2); + let mut found_path = String::with_capacity(path.len()); let mut path = path.split('/').map(|v| v.trim()); - let mut found_path = String::with_capacity(16); - { - while let Some(name) = path.next() { - if !found_path.is_empty() { - found_path.push('/'); - } + while let Some(name) = path.next() { + if !found_path.is_empty() { + found_path.push('/'); + } + found_path.extend(name.chars().flat_map(char::to_lowercase)); - for ch in name.chars() { - for ch in ch.to_lowercase() { - found_path.push(ch); - } - } - - if let Some(item) = cache - .mailboxes - .items - .iter() - .find(|item| item.path.to_lowercase() == found_path) - { - next_parent_id = item.document_id + 1; - } else { - create_paths.push(name.to_string()); - create_paths.extend(path.map(|v| v.to_string())); - break; - } + if let Some(item) = cache.mailbox_by_lowercase_path(&found_path) { + next_parent_id = item.document_id + 1; + } else { + create_paths.push(name.to_string()); + create_paths.extend(path.map(|v| v.to_string())); + break; } } @@ -137,28 +121,33 @@ impl MailboxFnc for Server { return Ok(None); } - let mut next_document_id = self - .store() - .assign_document_ids(account_id, Collection::Mailbox, create_paths.len() as u64) - .await - .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); - for name in create_paths { - let document_id = next_document_id; - next_document_id -= 1; + let slots = batch.reserve_document_ids( + account_id, + Collection::Mailbox, + create_paths.len() as u32, + ); + let mut parent_slot = None; + + for (name, slot) in create_paths.into_iter().zip(slots.iter()) { + let mut mailbox = Mailbox::new(name); + if parent_slot.is_none() { + mailbox.parent_id = next_parent_id; + } + let builder = ObjectIndexBuilder::<(), _>::new().with_changes(mailbox); + batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .with_document(document_id) - .custom( - ObjectIndexBuilder::<(), _>::new() - .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), - ) + .create_document(slot) + .custom(builder.with_pending_id_opt(parent_slot)) .caused_by(trc::location!())?; - next_parent_id = document_id + 1; + parent_slot = Some(slot); } - self.commit_batch(batch).await.caused_by(trc::location!())?; + let assigned_ids = self.commit_batch(batch).await.caused_by(trc::location!())?; + + return Ok(parent_slot.map(|slot| assigned_ids.slot(slot))); } Ok(Some(next_parent_id - 1)) diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 49658c25..3ce9a137 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -4,11 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use types::{acl::AclGrant, special_use::SpecialUse}; +use common::storage::index::PresenceFlags; +use store::{ + Deserialize, Serialize, + write::{Archive, ArchiveBytes, Archiver, BatchBuilder, MergeResult}, +}; +use trc::AddContext; +use types::{acl::AclGrant, collection::SyncCollection, field::Field, special_use::SpecialUse}; pub mod destroy; pub mod index; +pub mod lookup; pub mod manage; +pub mod role; pub const INBOX_ID: u32 = 0; pub const TRASH_ID: u32 = 1; @@ -17,43 +25,54 @@ pub const DRAFTS_ID: u32 = 3; pub const SENT_ID: u32 = 4; pub const ARCHIVE_ID: u32 = 5; +const ROLE_MASK: u16 = 0x000F; +pub(crate) const METADATA_SHIFT: u32 = 4; +pub(crate) const METADATA_MASK: u16 = 0x00F0; + +const _: () = assert!(((Mailbox::TRACKED.bits() as u16) << METADATA_SHIFT) & !METADATA_MASK == 0); + #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(derive(Debug))] pub struct Mailbox { pub name: String, - pub role: SpecialUse, pub parent_id: u32, pub sort_order: Option, pub uid_validity: u32, pub subscribers: Vec, pub acls: Vec, -} - -#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy)] -#[rkyv(derive(Debug, Clone, Copy))] -pub struct UidMailbox { - pub mailbox_id: u32, - pub uid: u32, + pub flags: u16, } impl Mailbox { pub fn new(name: impl Into) -> Self { Mailbox { name: name.into(), - role: SpecialUse::None, parent_id: 0, sort_order: None, uid_validity: rand::random::(), subscribers: vec![], acls: vec![], + flags: 0, } } pub fn with_role(mut self, role: SpecialUse) -> Self { - self.role = role; + self.set_role(role); self } + pub fn role(&self) -> SpecialUse { + Self::role_of(self.flags) + } + + pub fn set_role(&mut self, role: SpecialUse) { + self.flags = (self.flags & !ROLE_MASK) | u16::from(role.id()); + } + + fn role_of(flags: u16) -> SpecialUse { + SpecialUse::from_id((flags & ROLE_MASK) as u8) + } + pub fn with_parent_id(mut self, parent_id: u32) -> Self { self.parent_id = parent_id; self @@ -88,25 +107,42 @@ impl Mailbox { } impl ArchivedMailbox { + pub fn role(&self) -> SpecialUse { + Mailbox::role_of(self.flags.to_native()) + } + pub fn is_subscribed(&self, subscriber: u32) -> bool { self.subscribers.iter().any(|x| u32::from(x) == subscriber) } } -impl PartialEq for UidMailbox { - fn eq(&self, other: &Self) -> bool { - self.mailbox_id == other.mailbox_id - } +pub fn merge_subscription(batch: &mut BatchBuilder, subscriber: u32, subscribe: bool) { + batch.log_container_update(SyncCollection::Email); + batch.merge_fnc(Field::ARCHIVE, move |_, bytes| { + let Some(bytes) = bytes else { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .details("Mailbox no longer exists.") + .caused_by(trc::location!())); + }; + + let mut mailbox = as Deserialize>::deserialize(bytes) + .and_then(|archive| archive.deserialize::()) + .caused_by(trc::location!())?; + + let changed = if subscribe { + mailbox.add_subscriber(subscriber) + } else { + let was_subscribed = mailbox.is_subscribed(subscriber); + mailbox.remove_subscriber(subscriber); + was_subscribed + }; + + if changed { + Archiver::new(mailbox).serialize().map(MergeResult::Update) + } else { + Ok(MergeResult::Skip) + } + }); } -impl Eq for UidMailbox {} - -impl UidMailbox { - pub fn new(mailbox_id: u32, uid: u32) -> Self { - UidMailbox { mailbox_id, uid } - } - - pub fn new_unassigned(mailbox_id: u32) -> Self { - UidMailbox { mailbox_id, uid: 0 } - } -} diff --git a/crates/email/src/mailbox/role.rs b/crates/email/src/mailbox/role.rs new file mode 100644 index 00000000..c53bcc32 --- /dev/null +++ b/crates/email/src/mailbox/role.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{INBOX_ID, JUNK_ID, TRASH_ID}; +use crate::cache::mailbox::MailboxCacheAccess; +use common::MessageStoreCache; +use std::fmt::{self, Display}; +use types::special_use::SpecialUse; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoleChange { + Create { + role: SpecialUse, + }, + Update { + document_id: u32, + current: SpecialUse, + role: SpecialUse, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoleChangeError { + InUse(SpecialUse), + Protected, +} + +impl RoleChange { + pub fn validate(self, cache: &MessageStoreCache) -> Result<(), RoleChangeError> { + let role = match self { + RoleChange::Update { current, role, .. } if current == role => return Ok(()), + RoleChange::Create { role } | RoleChange::Update { role, .. } => role, + }; + + if role != SpecialUse::None && cache.mailbox_by_role(&role).is_some() { + Err(RoleChangeError::InUse(role)) + } else if let RoleChange::Update { + document_id: INBOX_ID | TRASH_ID | JUNK_ID, + .. + } = self + { + Err(RoleChangeError::Protected) + } else { + Ok(()) + } + } +} + +impl Display for RoleChangeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RoleChangeError::InUse(role) => write!( + f, + "A mailbox with role '{}' already exists.", + role.as_str().unwrap_or_default() + ), + RoleChangeError::Protected => f.write_str( + "You are not allowed to change the role of Inbox, Junk or Trash folders.", + ), + } + } +} + diff --git a/crates/email/src/message/body.rs b/crates/email/src/message/body.rs index dcfaac9c..5e9736cc 100644 --- a/crates/email/src/message/body.rs +++ b/crates/email/src/message/body.rs @@ -4,309 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ - ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType, - PART_ENCODING_BASE64, PART_ENCODING_QP, PART_SIZE_MASK, -}; -use jmap_proto::object::email::{EmailProperty, EmailValue}; -use jmap_tools::{Map, Value}; -use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; -use types::blob::BlobId; -use utils::chained_bytes::ChainedBytes; - -use super::headers::HeaderToValue; - -pub trait ToBodyPart { - fn to_body_part( - &self, - part_id: u32, - properties: &[EmailProperty], - raw_message: &ChainedBytes<'_>, - blob_id: &BlobId, - blob_body_offset: isize, - ) -> Value<'static, EmailProperty, EmailValue>; -} - -impl ToBodyPart for Vec> { - fn to_body_part( - &self, - part_id: u32, - properties: &[EmailProperty], - raw_message: &ChainedBytes<'_>, - blob_id: &BlobId, - blob_body_offset: isize, - ) -> Value<'static, EmailProperty, EmailValue> { - let mut parts = vec![part_id].into_iter(); - let mut parts_stack = Vec::new(); - let mut subparts = Vec::with_capacity(1); - - loop { - if let Some((part_id, part)) = parts - .next() - .map(|part_id| (part_id, &self[part_id as usize])) - { - let mut values = Map::with_capacity(properties.len()); - let multipart = if let PartType::Multipart(parts) = &part.body { - parts.into() - } else { - None - }; - - for property in properties { - let value = match property { - EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), - EmailProperty::BlobId if multipart.is_none() => { - let base_offset = blob_id.start_offset() as isize + blob_body_offset; - BlobId::new_section( - blob_id.hash.clone(), - blob_id.class.clone(), - (part.offset_body as isize + base_offset) as usize, - (part.offset_end as isize + base_offset) as usize, - part.encoding as u8, - ) - .into() - } - EmailProperty::Size if multipart.is_none() => match &part.body { - PartType::Text(text) | PartType::Html(text) => text.len(), - PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(), - PartType::Message(message) => message.root_part().raw_len() as usize, - PartType::Multipart(_) => 0, - } - .into(), - EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(), - EmailProperty::Type => part - .content_type() - .map(|ct| { - ct.subtype() - .map(|st| format!("{}/{}", ct.ctype(), st)) - .unwrap_or_else(|| ct.ctype().to_string()) - }) - .or_else(|| match &part.body { - PartType::Text(_) => Some("text/plain".to_string()), - PartType::Html(_) => Some("text/html".to_string()), - PartType::Message(_) => Some("message/rfc822".to_string()), - _ => None, - }) - .into(), - EmailProperty::Charset => part - .content_type() - .and_then(|ct| ct.attribute("charset")) - .or(match &part.body { - PartType::Text(_) | PartType::Html(_) => Some("us-ascii"), - _ => None, - }) - .map(|v| v.to_string()) - .into(), - EmailProperty::Disposition => part - .content_disposition() - .map(|cd| cd.ctype()) - .map(|v| v.to_string()) - .into(), - EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(), - EmailProperty::Language => match part.content_language() { - HeaderValue::Text(text) => vec![text.to_string()].into(), - HeaderValue::TextList(list) => list - .iter() - .map(|text| text.to_string().into()) - .collect::>>() - .into(), - _ => Value::Null, - }, - EmailProperty::Location => { - part.content_location().map(|v| v.to_string()).into() - } - EmailProperty::Header(_) => { - part.headers.header_to_value(property, raw_message) - } - EmailProperty::Headers => part.headers.headers_to_value(raw_message), - EmailProperty::SubParts => continue, - _ => Value::Null, - }; - values.insert_unchecked(property.clone(), value); - } - - subparts.push(values); - - if let Some(multipart) = multipart { - if parts_stack.len() == 10_000 { - debug_assert!(false, "Too much nesting in message metadata"); - return Value::Null; - } - let multipart = multipart.clone(); - parts_stack.push(( - parts, - std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), - )); - parts = multipart.into_iter(); - } - } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { - prev_subparts - .last_mut() - .unwrap() - .insert_unchecked(EmailProperty::SubParts, subparts); - parts = prev_parts; - subparts = prev_subparts; - } else { - return subparts.pop().map(Into::into).unwrap_or_default(); - } - } - } -} - -impl ToBodyPart for ArchivedMessageMetadataContents { - fn to_body_part( - &self, - part_id: u32, - properties: &[EmailProperty], - raw_message: &ChainedBytes<'_>, - blob_id: &BlobId, - blob_body_offset: isize, - ) -> Value<'static, EmailProperty, EmailValue> { - let mut parts = vec![part_id].into_iter(); - let mut parts_stack = Vec::new(); - let mut subparts = Vec::with_capacity(1); - - loop { - if let Some((part_id, part)) = parts - .next() - .map(|part_id| (part_id, &self.parts[part_id as usize])) - { - let mut values = Map::with_capacity(properties.len()); - let multipart = if let ArchivedMetadataPartType::Multipart(parts) = &part.body { - parts.into() - } else { - None - }; - - for property in properties { - let value = match property { - EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), - EmailProperty::BlobId if multipart.is_none() => { - let base_offset = blob_id.start_offset() as isize + blob_body_offset; - let flags = part.flags.to_native(); - let encoding = if flags & PART_ENCODING_BASE64 != 0 { - 2 - } else if flags & PART_ENCODING_QP != 0 { - 1 - } else { - 0 - }; - BlobId::new_section( - blob_id.hash.clone(), - blob_id.class.clone(), - (u32::from(part.offset_body) as isize + base_offset) as usize, - (u32::from(part.offset_end) as isize + base_offset) as usize, - encoding, - ) - .into() - } - EmailProperty::Size if multipart.is_none() => { - (part.flags.to_native() & PART_SIZE_MASK).into() - } - EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(), - EmailProperty::Type => part - .content_type() - .map(|ct| { - ct.subtype() - .map(|st| format!("{}/{}", ct.ctype(), st)) - .unwrap_or_else(|| ct.ctype().to_string()) - }) - .or_else(|| match &part.body { - ArchivedMetadataPartType::Text => Some("text/plain".to_string()), - ArchivedMetadataPartType::Html => Some("text/html".to_string()), - ArchivedMetadataPartType::Message(_) => { - Some("message/rfc822".to_string()) - } - _ => None, - }) - .into(), - EmailProperty::Charset => { - part.content_type() - .and_then(|ct| ct.attribute("charset")) - .or(match &part.body { - ArchivedMetadataPartType::Text - | ArchivedMetadataPartType::Html => Some("us-ascii"), - _ => None, - }) - .map(|v| v.to_string()) - .into() - } - EmailProperty::Disposition => part - .content_disposition() - .map(|cd| cd.ctype()) - .map(|v| v.to_string()) - .into(), - EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(), - EmailProperty::Language => match part.content_language() { - ArchivedMetadataHeaderValue::Text(text) => { - vec![text.to_string()].into() - } - ArchivedMetadataHeaderValue::TextList(list) => list - .iter() - .map(|text| text.to_string().into()) - .collect::>>() - .into(), - _ => Value::Null, - }, - EmailProperty::Location => { - part.content_location().map(|v| v.to_string()).into() - } - EmailProperty::Header(_) => part.header_to_value(property, raw_message), - EmailProperty::Headers => part.headers_to_value(raw_message), - EmailProperty::SubParts => continue, - _ => Value::Null, - }; - values.insert_unchecked(property.clone(), value); - } - - subparts.push(values); - - if let Some(multipart) = multipart { - if parts_stack.len() == 10_000 { - debug_assert!(false, "Too much nesting in message metadata"); - return Value::Null; - } - let multipart = multipart - .iter() - .map(|id| u16::from(id) as u32) - .collect::>(); - parts_stack.push(( - parts, - std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), - )); - parts = multipart.into_iter(); - } - } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { - prev_subparts - .last_mut() - .unwrap() - .insert_unchecked(EmailProperty::SubParts, subparts); - parts = prev_parts; - subparts = prev_subparts; - } else { - return subparts.pop().map(Into::into).unwrap_or_default(); - } - } - } -} - -pub trait TruncateBody { - fn truncate(&self, max_len: usize) -> (bool, String); -} - -impl TruncateBody for PartType<'_> { - fn truncate(&self, max_len: usize) -> (bool, String) { - match self { - PartType::Text(text) => truncate_plain(text, max_len), - PartType::Html(html) => truncate_html(html, max_len), - PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { - PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len) - } - _ => (false, "".into()), - } - } -} - pub fn truncate_plain(text: &str, mut max_len: usize) -> (bool, String) { if max_len != 0 && text.len() > max_len { let add_dots = max_len > 6; diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 5fa16be4..4fcb61bd 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -4,34 +4,37 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - ingest::{EmailIngest, IngestedEmail}, - metadata::{MessageData, MessageMetadata}, +use super::ingest::{EmailIngest, IngestedEmail}; +use crate::message::{ + index::extractors::VisitText, + ingest::ThreadInfo, + ingest_metadata::IngestMetadata, + messagedata::{MessageData, PendingMessageData}, + metadata::{HeaderId, MetadataRow}, + sortkeys::MessageSortKeys, }; -use crate::{ - mailbox::UidMailbox, - message::{ - index::extractors::VisitTextArchived, - ingest::ThreadInfo, - metadata::{ - MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue, - }, - }, +use common::{ + MessageUid, Server, + auth::AccountCache, + storage::{index::ObjectIndexBuilder, metadata::PrivateMetadataCommit}, }; -use common::{Server, storage::index::ObjectIndexBuilder}; -use mail_parser::parsers::fields::thread::thread_name; +use mail_parser::HeaderForm; use registry::{ schema::{ - enums::IndexDocumentType, - structs::{Task, TaskIndexDocument, TaskMergeThreads, TaskStatus}, + enums::StorageQuota, + structs::{Task, TaskMergeThreads, TaskStatus}, }, types::map::Map, }; -use store::write::{BatchBuilder, IndexPropertyClass, ValueClass}; use store::{ - ValueKey, - write::{AlignedBytes, Archive}, + Deserialize, + write::{PendingId, QueueDocumentId, SearchIndex}, }; +use store::{ + write::{BatchBuilder, IndexPropertyClass, ValueClass}, + xxhash_rust::xxh3::xxh3_128, +}; +use tinyvec::TinyVec; use trc::AddContext; use types::{ blob::{BlobClass, BlobId}, @@ -39,10 +42,12 @@ use types::{ field::EmailField, keyword::Keyword, }; -use utils::cheeky_hash::CheekyHash; + +mod sources; + +pub use sources::{CopySource, CopySources}; pub enum CopyMessageError { - NotFound, OverQuota, AlreadyExists(u32), } @@ -51,12 +56,12 @@ pub trait EmailCopy: Sync + Send { #[allow(clippy::too_many_arguments)] fn copy_message( &self, - from_account_id: u32, - from_message_id: u32, - to_account_id: u32, + source: CopySource, + to_account: &AccountCache, mailboxes: Vec, keywords: Vec, - received_at: Option, + received_at: u64, + containers: Option>, session_id: u64, ) -> impl Future>> + Send; } @@ -65,164 +70,159 @@ impl EmailCopy for Server { #[allow(clippy::too_many_arguments)] async fn copy_message( &self, - from_account_id: u32, - from_message_id: u32, - to_account_id: u32, + source: CopySource, + to_account: &AccountCache, mailboxes: Vec, keywords: Vec, - received_at: Option, + received_at: u64, + mut containers: Option>, session_id: u64, ) -> trc::Result> { - // Obtain metadata - let mut metadata = if let Some(metadata) = self - .store() - .get_value::>(ValueKey::property( - from_account_id, - Collection::Email, - from_message_id, - EmailField::Metadata, - )) - .await? - { - metadata - .deserialize::() - .caused_by(trc::location!())? - } else { - return Ok(Err(CopyMessageError::NotFound)); - }; - - // Check quota - let size = metadata.root_part().offset_end; - let to_account = self.account(to_account_id).await?; - match self.has_available_quota(&to_account, size as u64).await { - Ok(_) => (), - Err(err) => { - if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) - || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) - { - trc::error!(err.account_id(to_account_id).span_id(session_id)); - return Ok(Err(CopyMessageError::OverQuota)); - } else { - return Err(err); - } - } + let to_account_id = to_account.id; + let (metadata_bytes, sort_keys_bytes) = source.into_parts(); + let row = MetadataRow::deserialize(&metadata_bytes).caused_by(trc::location!())?; + let metadata = row.unarchive().caused_by(trc::location!())?; + if let Some(containers) = containers.as_deref_mut() { + containers.shared = containers + .shared + .take() + .filter(|container| container.kinds().intersects(MessageData::TRACKED_METADATA)); } - // Set receivedAt - if let Some(received_at) = received_at { - metadata.rcvd_attach = (metadata.rcvd_attach & MESSAGE_HAS_ATTACHMENT) - | (received_at & MESSAGE_RECEIVED_MASK); - } + let size = u32::try_from(metadata.size()).unwrap_or(u32::MAX); // Obtain threadId + let headers = row.raw_headers().caused_by(trc::location!())?; let mut message_ids = Vec::new(); - let mut subject = ""; - for header in &metadata.contents[0].parts[0].headers { - match &header.name { - MetadataHeaderName::MessageId => { - header.value.visit_text(|id| { - if !id.is_empty() { - message_ids.push(CheekyHash::new(id.as_bytes())); - } - }); - } - MetadataHeaderName::InReplyTo - | MetadataHeaderName::References - | MetadataHeaderName::ResentMessageId => { - header.value.visit_text(|id| { - if !id.is_empty() { - message_ids.push(CheekyHash::new(id.as_bytes())); - } - }); - } - MetadataHeaderName::Subject if subject.is_empty() => { - subject = thread_name(match &header.value { - MetadataHeaderValue::Text(text) => text.as_ref(), - MetadataHeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().as_ref() - } - _ => "", - }); - } - _ => (), + let root_headers = metadata.root().root_part().headers(); + for header in root_headers.iter() { + if matches!( + header.id(), + HeaderId::MESSAGE_ID + | HeaderId::IN_REPLY_TO + | HeaderId::REFERENCES + | HeaderId::RESENT_MESSAGE_ID + ) && let Some(raw) = headers.get(header.value_range()) + { + HeaderForm::MessageIds.parse(raw).value().visit_text(|id| { + if !id.is_empty() { + message_ids.push(xxh3_128(id.as_bytes())); + } + }); } } + let envelope = metadata.root().envelope(); + let subject = metadata.thread_subject_with(root_headers, &headers); + let sent_at = envelope.datetime().map(|date| date.to_timestamp()); message_ids.sort_unstable(); message_ids.dedup(); // Obtain threadId let thread_result = self - .find_thread_id(to_account_id, subject, &message_ids) + .find_thread_id(to_account_id, &subject, &message_ids) .await .caused_by(trc::location!())?; - if let Some(&existing) = thread_result.duplicate_ids.first() { + if let Some(&existing) = thread_result.duplicate_ids.iter().min() { return Ok(Err(CopyMessageError::AlreadyExists(existing))); } + // Assign id let mut email = IngestedEmail { size: size as usize, ..Default::default() }; - let blob_hash = metadata.blob_hash.clone(); + let blob_hash = metadata.blob_hash(); - // Assign IMAP UIDs - let mut mailbox_ids = Vec::with_capacity(mailboxes.len()); - email.imap_uids = Vec::with_capacity(mailboxes.len()); - let mut ids = self - .assign_email_ids(to_account_id, mailboxes.iter().copied(), true) - .await - .caused_by(trc::location!())?; - let document_id = ids.next().unwrap(); - for (uid, mailbox_id) in ids.zip(mailboxes.iter().copied()) { - mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); - email.imap_uids.push(uid); + let mut keywords_flags = 0; + let mut keywords_extra = Vec::new(); + for keyword in keywords { + match keyword.into_id() { + Ok(id) => keywords_flags |= 1 << id, + Err(name) => keywords_extra.push(name), + } } // Prepare batch let mut batch = BatchBuilder::new(); batch.with_account_id(to_account_id); + self.limit_disk_quota(&mut batch, to_account); + self.limit_object_quota(&mut batch, to_account, StorageQuota::MaxEmails); + + // Reserve a document id and one IMAP UID per target mailbox + let document_slot = batch.reserve_document_id(to_account_id, Collection::Email); + let uid_slots = batch.reserve_uids(to_account_id, mailboxes.iter().copied()); + let mailbox_ids: TinyVec<[MessageUid; 2]> = mailboxes + .iter() + .copied() + .map(MessageUid::new_unassigned) + .collect(); // Determine thread id - let tenant_id = to_account.tenant_id(); - let thread_id = if let Some(thread_id) = thread_result.thread_id { - thread_id - } else { + let thread_slot = if thread_result.thread_id.is_none() { batch .with_collection(Collection::Thread) - .with_document(document_id) + .create_document(document_slot) .log_container_insert(SyncCollection::Thread); - document_id + Some(document_slot) + } else { + None }; + + let mut data = PendingMessageData { + data: MessageData { + mailboxes: mailbox_ids, + keywords: keywords_flags, + thread_id: thread_result.thread_id.unwrap_or_default(), + size, + keywords_extra, + received_at, + sent_at: MessageData::sent_at_offset(sent_at, received_at), + change_id: 0, + }, + uid_slots, + thread_slot, + change_id: None, + }; + if let Some(container) = containers + .as_deref() + .and_then(|containers| containers.shared.as_ref()) + { + data.data.set_metadata_kinds(container.kinds()); + } + let thread_info = ThreadInfo { + thread_id: data.thread_id(), + ref_ids: &message_ids, + }; + + let index_builder = ObjectIndexBuilder::<(), _>::new(); batch .with_collection(Collection::Email) - .with_document(document_id) - .custom( - ObjectIndexBuilder::<(), _>::new() - .with_tenant_id(tenant_id) - .with_changes(MessageData { - mailboxes: mailbox_ids.into_boxed_slice(), - keywords: keywords.into_boxed_slice(), - thread_id, - size, - }), - ) + .create_document(document_slot) + .custom(index_builder.with_changes(data)) .caused_by(trc::location!())? .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_result.thread_hash, }), - ThreadInfo::serialize(thread_id, &message_ids), + thread_info, ) - .schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: to_account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::now(), - })); + .queue_document_index(SearchIndex::Email, to_account_id, QueueDocumentId::Current); + + let mut private_commit = PrivateMetadataCommit::default(); + if let Some(containers) = containers { + containers + .build( + &mut batch, + to_account_id, + PendingId::Slot(document_slot), + &mut private_commit, + ) + .caused_by(trc::location!())?; + } // Merge threads if necessary if !thread_result.merge_ids.is_empty() { @@ -234,25 +234,36 @@ impl EmailCopy for Server { })); } - metadata - .index(&mut batch, true) - .caused_by(trc::location!())?; + let sort_keys = match sort_keys_bytes { + Some(sort_keys) => sort_keys, + None => MessageSortKeys::from_envelope(envelope).serialize(), + }; + + metadata.index_verbatim(&mut batch, metadata_bytes, sort_keys); // Insert and obtain ids - let change_id = self - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())? - .last_change_id(to_account_id)?; + let queues = batch.queue_notify(); + let assigned_ids = match self.store().write_batch(&mut batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { + trc::error!(err.account_id(to_account_id).span_id(session_id)); + return Ok(Err(CopyMessageError::OverQuota)); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; + self.inner.mark_caches_stale_from(&assigned_ids); + self.private_metadata_committed(private_commit, &assigned_ids) + .await; + let document_id = assigned_ids.slot(document_slot); // Request indexing - self.notify_task_queue(); + self.notify_queues(queues).await; // Update response email.document_id = document_id; - email.thread_id = thread_id; - email.change_id = change_id; + email.thread_id = thread_result.thread_id.unwrap_or(document_id); + email.change_id = assigned_ids.last_change_id(to_account_id, SyncCollection::Email); + email.imap_uids = assigned_ids.slots(uid_slots).collect(); email.blob_id = BlobId::new( blob_hash, BlobClass::Linked { diff --git a/crates/email/src/message/copy/sources.rs b/crates/email/src/message/copy/sources.rs new file mode 100644 index 00000000..672cf442 --- /dev/null +++ b/crates/email/src/message/copy/sources.rs @@ -0,0 +1,214 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::Server; +use store::{ + IterateParams, Key, U32_LEN, ValueKey, + dispatch::{DocumentSet, ReadShape}, + write::{AnyKey, ValueClass, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::{collection::Collection, field::EmailField}; + +const CHUNK_SIZE: usize = 256; +const CHUNK_BYTES: usize = 4 * 1024 * 1024; + +pub struct CopySource { + metadata: Vec, + sort_keys: Option>, +} + +pub struct CopySources { + account_id: u32, + order: Vec, + cursor: usize, + rows: Vec<(u32, Option)>, + deferred: Vec, +} + +enum ReadInto { + Replace, + Merge, +} + +impl CopySource { + pub fn metadata(&self) -> &[u8] { + &self.metadata + } + + pub(super) fn into_parts(self) -> (Vec, Option>) { + (self.metadata, self.sort_keys) + } +} + +impl CopySources { + pub fn new(account_id: u32, order: Vec) -> Self { + CopySources { + account_id, + order, + cursor: 0, + rows: Vec::new(), + deferred: Vec::new(), + } + } + + pub async fn take( + &mut self, + server: &Server, + document_id: u32, + ) -> trc::Result> { + if let Some(source) = self.remove(document_id) { + return Ok(Some(source)); + } + + let (documents, into) = if self.deferred.binary_search(&document_id).is_ok() { + (std::mem::take(&mut self.deferred), ReadInto::Merge) + } else if let Some(offset) = self + .order + .get(self.cursor..) + .and_then(|pending| pending.iter().position(|id| *id == document_id)) + { + let start = self.cursor + offset; + let end = start.saturating_add(CHUNK_SIZE).min(self.order.len()); + self.cursor = end; + let mut chunk = self.order.get(start..end).unwrap_or_default().to_vec(); + chunk.sort_unstable(); + chunk.dedup(); + (chunk, ReadInto::Replace) + } else { + (vec![document_id], ReadInto::Merge) + }; + self.read(server, documents, document_id, into).await?; + + Ok(self.remove(document_id)) + } + + fn remove(&mut self, document_id: u32) -> Option { + self.rows + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| self.rows.get_mut(idx)) + .and_then(|(_, source)| source.take()) + } + + async fn read( + &mut self, + server: &Server, + documents: Vec, + required: u32, + into: ReadInto, + ) -> trc::Result<()> { + let account_id = self.account_id; + let collection: u8 = Collection::Email.into(); + let store = server.store(); + let gaps = match store.read_shape() { + ReadShape::Keys => documents + .iter() + .map(|&document_id| (document_id, document_id)) + .collect(), + ReadShape::Ranges(gap) => documents.scan_ranges(gap), + }; + let metadata_field: u8 = EmailField::Metadata.into(); + let sort_keys_field: u8 = EmailField::SortKeys.into(); + let ranges = [metadata_field, sort_keys_field] + .into_iter() + .flat_map(|field| { + let class = ValueClass::Immutable(field); + let subspace = class.subspace(collection); + let key = move |document_id| { + ValueKey { + account_id, + collection, + document_id, + class: class.clone(), + } + .serialize(0) + }; + gaps.iter().map(move |&(from_document_id, to_document_id)| { + let begin = key(from_document_id); + let end = if from_document_id == to_document_id { + begin.clone() + } else { + let mut end = key(to_document_id); + end.push(u8::MAX); + end + }; + IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { subspace, key: end }, + ) + }) + }) + .collect::>(); + + if matches!(into, ReadInto::Replace) { + self.rows.clear(); + self.deferred.clear(); + } + let mut fetched = Vec::with_capacity(documents.len()); + let rows = &mut fetched; + let deferred = &mut self.deferred; + let mut bytes = 0usize; + store + .iterate_many(ranges, |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if !documents.contains(document_id) { + return Ok(true); + } + let field = key + .len() + .checked_sub(U32_LEN + 1) + .and_then(|offset| key.get(offset).copied()); + if field == Some(metadata_field) { + if bytes < CHUNK_BYTES || document_id == required { + bytes = bytes.saturating_add(value.len()); + rows.push(( + document_id, + Some(CopySource { + metadata: value.to_vec(), + sort_keys: None, + }), + )); + } else { + deferred.push(document_id); + } + } else if let Some(source) = rows + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| rows.get_mut(idx)) + .and_then(|(_, source)| source.as_mut()) + { + source.sort_keys = Some(value.to_vec()); + } + Ok(true) + }) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + })?; + + if self.rows.is_empty() { + self.rows = fetched; + } else { + for (document_id, source) in fetched { + match self.rows.binary_search_by_key(&document_id, |(id, _)| *id) { + Ok(idx) => { + if let Some((_, slot)) = self.rows.get_mut(idx) { + *slot = source; + } + } + Err(idx) => self.rows.insert(idx, (document_id, source)), + } + } + } + Ok(()) + } +} diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index fd0b95e7..f80e6796 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -15,8 +15,9 @@ use common::auth::{ ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305, ACCOUNT_FLAG_ENCRYPT_APPEND, ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, EncryptionKeys, }; -use mail_builder::{encoders::Base64Encoder, mime::make_boundary}; -use mail_parser::{Message, MimeHeaders, PartType}; +use encodify::base64::MIME; +use mail_builder::mime::make_boundary; +use mail_parser::{Message, PartKind}; use openpgp::{ parse::Parse, serialize::stream, @@ -34,7 +35,7 @@ use rasn_cms::{ }; use rsa::{Oaep, Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey, sha2::Sha256}; use sequoia_openpgp as openpgp; -use std::io::Cursor; +use std::{fmt::Display, io::Cursor}; const AES256_GCM: &Oid = Oid::JOINT_ISO_ITU_T_COUNTRY_US_ORGANIZATION_GOV_CSOR_NIST_ALGORITHMS_AES256_GCM; @@ -60,6 +61,7 @@ pub trait EncryptMessage { flags: u64, ) -> Result, EncryptMessageError>; fn is_encrypted(&self) -> bool; + fn split_mime_headers(&self) -> (Vec, Vec); } impl EncryptMessage for Message<'_> { @@ -75,24 +77,7 @@ impl EncryptMessage for Message<'_> { )); } - let root = self.root_part(); - let raw_message = self.raw_message(); - let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); - let mut inner_message = Vec::with_capacity(raw_message.len()); - - // Move MIME headers and body to inner message - for header in root.headers() { - (if header.name.is_mime_header() { - &mut inner_message - } else { - &mut outer_message - }) - .extend_from_slice( - &raw_message[header.offset_field() as usize..header.offset_end() as usize], - ); - } - inner_message.extend_from_slice(b"\r\n"); - inner_message.extend_from_slice(&raw_message[root.raw_body_offset() as usize..]); + let (mut outer_message, inner_message) = self.split_mime_headers(); // Encrypt inner message if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 { @@ -217,13 +202,14 @@ impl EncryptMessage for Message<'_> { // Encrypt contents (TODO: use rayon) let (encrypted_contents, mac, key, nonce) = tokio::task::spawn_blocking(move || { - let (encrypted_contents, mac) = cipher.encrypt(&key, &nonce, &inner_message); - (encrypted_contents, mac, key, nonce) + cipher + .encrypt(&key, &nonce, &inner_message) + .map(|(encrypted_contents, mac)| (encrypted_contents, mac, key, nonce)) }) .await .map_err(|err| { EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) - })?; + })??; // Encrypt key using public keys let key_encryption_algorithm = cipher.key_encryption_algorithm()?; @@ -332,25 +318,44 @@ impl EncryptMessage for Message<'_> { ) .as_bytes(), ); - Base64Encoder::new() - .wrap_lines() - .encode_to_writer(&pkcs7, &mut outer_message) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) - })?; + MIME.encode_append(&pkcs7, &mut outer_message); } Ok(outer_message) } + fn split_mime_headers(&self) -> (Vec, Vec) { + let root = self.root_part(); + let raw_message = self.raw(); + let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); + let mut inner_message = Vec::with_capacity(raw_message.len()); + for header in root.headers() { + let target = if header.name().is_mime_header() { + &mut inner_message + } else { + &mut outer_message + }; + let field = raw_message + .get(header.offset_field() as usize..header.offset_end() as usize) + .unwrap_or_default(); + target.extend_from_slice(field); + if !field.ends_with(b"\n") { + target.extend_from_slice(b"\r\n"); + } + } + inner_message.extend_from_slice(b"\r\n"); + inner_message.extend_from_slice( + raw_message + .get(root.offset_body() as usize..) + .unwrap_or_default(), + ); + (outer_message, inner_message) + } + fn is_encrypted(&self) -> bool { if self.content_type().is_some_and(|ct| { - let main_type = ct.c_type.as_ref(); - let sub_type = ct - .c_subtype - .as_ref() - .map(|s| s.as_ref()) - .unwrap_or_default(); + let main_type = ct.ctype(); + let sub_type = ct.subtype().unwrap_or_default(); (main_type.eq_ignore_ascii_case("application") && (sub_type.eq_ignore_ascii_case("pkcs7-mime") @@ -366,16 +371,18 @@ impl EncryptMessage for Message<'_> { return true; } - if self.parts.len() <= 2 { + let root = self.root(); + let num_parts = root.parts().count(); + if num_parts <= 2 { let mut text_part = None; let mut is_multipart = false; - for part in &self.parts { - match &part.body { - PartType::Text(text) => { - text_part = Some(text.as_ref()); + for part in root.parts() { + match part.kind() { + PartKind::Text => { + text_part = part.text(); } - PartType::Multipart(_) => { + PartKind::Multipart => { is_multipart = true; } _ => (), @@ -384,7 +391,7 @@ impl EncryptMessage for Message<'_> { match text_part { Some(text) - if (self.parts.len() == 1 || is_multipart) + if (num_parts == 1 || is_multipart) && text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") => { return true; @@ -434,6 +441,11 @@ impl EncryptionFlags for u64 { } } +#[cold] +fn encrypt_error(algorithm: &str, err: impl Display) -> EncryptMessageError { + EncryptMessageError::Error(format!("{algorithm} encryption failed: {err}")) +} + #[derive(Clone, Copy, PartialEq, Eq)] pub enum SymmetricCipher { Aes128Cbc, @@ -466,43 +478,46 @@ impl SymmetricCipher { ) } - fn encrypt(self, key: &[u8], nonce: &[u8], contents: &[u8]) -> (Vec, Option>) { + fn encrypt( + self, + key: &[u8], + nonce: &[u8], + contents: &[u8], + ) -> Result<(Vec, Option>), EncryptMessageError> { match self { - SymmetricCipher::Aes128Cbc => ( + SymmetricCipher::Aes128Cbc => { cbc::Encryptor::::new_from_slices(key, nonce) - .expect("invalid key or iv length") - .encrypt_padded_vec::(contents), - None, - ), - SymmetricCipher::Aes256Cbc => ( + .map(|cipher| (cipher.encrypt_padded_vec::(contents), None)) + .map_err(|err| encrypt_error("AES-128-CBC", err)) + } + SymmetricCipher::Aes256Cbc => { cbc::Encryptor::::new_from_slices(key, nonce) - .expect("invalid key or iv length") - .encrypt_padded_vec::(contents), - None, - ), + .map(|cipher| (cipher.encrypt_padded_vec::(contents), None)) + .map_err(|err| encrypt_error("AES-256-CBC", err)) + } SymmetricCipher::Aes256Gcm => { - let cipher = Aes256Gcm::new_from_slice(key).expect("invalid key length"); + let cipher = Aes256Gcm::new_from_slice(key) + .map_err(|err| encrypt_error("AES-256-GCM", err))?; + let nonce = nonce + .try_into() + .map_err(|err| encrypt_error("AES-256-GCM", err))?; let mut buffer = contents.to_vec(); let tag = cipher - .encrypt_inout_detached( - nonce.try_into().expect("invalid nonce length"), - b"", - buffer.as_mut_slice().into(), - ) - .expect("AES-GCM encryption failed"); - (buffer, Some(tag.to_vec())) + .encrypt_inout_detached(nonce, b"", buffer.as_mut_slice().into()) + .map_err(|err| encrypt_error("AES-256-GCM", err))?; + Ok((buffer, Some(tag.to_vec()))) } SymmetricCipher::ChaCha20Poly1305 => { - let cipher = ChaCha20Poly1305::new_from_slice(key).expect("invalid key length"); + let cipher = ChaCha20Poly1305::new_from_slice(key) + .map_err(|err| encrypt_error("ChaCha20-Poly1305", err))?; + let nonce = nonce + .try_into() + .map_err(|err| encrypt_error("ChaCha20-Poly1305", err))?; let mut buffer = contents.to_vec(); let tag = cipher - .encrypt_inout_detached( - nonce.try_into().expect("invalid nonce length"), - b"", - buffer.as_mut_slice().into(), - ) - .expect("ChaCha20-Poly1305 encryption failed"); - (buffer, Some(tag.to_vec())) + .encrypt_inout_detached(nonce, b"", buffer.as_mut_slice().into()) + .map_err(|err| encrypt_error("ChaCha20-Poly1305", err))?; + Ok((buffer, Some(tag.to_vec()))) } } } @@ -596,3 +611,4 @@ fn encode_null() -> Result, EncryptMessageError> { EncryptMessageError::Error(format!("Failed to encode NULL parameters: {}", err)) }) } + diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 6e437d06..e4c0104e 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -4,16 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::metadata::MessageData; use crate::cache::{MessageCacheFetch, email::MessageCacheAccess}; -use common::{Server, storage::index::ObjectIndexBuilder}; +use crate::message::messagedata::EmailMessageData; +use crate::sieve::SeenIdHash; +use crate::submission::{DeliveryResults, EmailSubmission}; +use common::{MessageStoreCache, Server, auth::ChangedBy, storage::index::ObjectIndexBuilder}; use groupware::calendar::storage::ItipAutoExpunge; -use registry::schema::enums::IndexDocumentType; -use registry::schema::structs::{Task, TaskIndexDocument, TaskStatus}; -use std::future::Future; +use std::{future::Future, time::Duration}; use store::write::key::DeserializeBigEndian; -use store::write::{IndexPropertyClass, now}; -use store::{IterateParams, U32_LEN, U64_LEN, ValueKey}; +use store::write::{Archive, ArchiveBytes, IndexPropertyClass, SearchIndex, now}; +use store::{Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey}; use store::{ roaring::RoaringBitmap, write::{BatchBuilder, ValueClass}, @@ -21,12 +21,13 @@ use store::{ use trc::AddContext; use types::collection::{Collection, SyncCollection, VanishedCollection}; use types::field::{EmailField, EmailSubmissionField}; +use utils::snowflake::SnowflakeIdGenerator; pub trait EmailDeletion: Sync + Send { fn emails_delete( &self, account_id: u32, - tenant_id: Option, + changed_by: ChangedBy, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> impl Future> + Send; @@ -45,6 +46,12 @@ pub trait EmailDeletion: Sync + Send { hold_period: u64, ) -> impl Future> + Send; + fn purge_sieve_seen_ids( + &self, + account_id: u32, + cutoff: u64, + ) -> impl Future> + Send; + fn log_emptied_threads( &self, account_id: u32, @@ -58,56 +65,58 @@ impl EmailDeletion for Server { async fn emails_delete( &self, account_id: u32, - tenant_id: Option, + changed_by: ChangedBy, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> trc::Result { + let cache = self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + let flagged_ids = document_ids + .iter() + .filter(|document_id| { + cache + .email_by_id(document_id) + .is_some_and(|message| !cache.metadata_kinds(message).is_empty()) + }) + .collect::(); + let containers = self + .preload_container_cleanup(changed_by, account_id, Collection::Email, &flagged_ids) + .await + .caused_by(trc::location!())?; let mut deleted_ids = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); - self.archives( - account_id, - Collection::Email, - &document_ids, - |document_id, data_| { - // Add changes to batch - let metadata = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - for mailbox in metadata.inner.mailboxes.iter() { - batch.log_vanished_item( - VanishedCollection::Email, - (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), - ); - } - thread_ids.insert(metadata.inner.thread_id.to_native()); - batch - .with_document(document_id) - .custom( - ObjectIndexBuilder::<_, ()>::new() - .with_tenant_id(tenant_id) - .with_current(metadata), - ) - .caused_by(trc::location!())? - .schedule_task(Task::UnindexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::now(), - })) - .commit_point(); + self.message_datas(account_id, &document_ids, |document_id, metadata| { + // Add changes to batch + for mailbox in metadata.mailboxes.iter() { + batch.log_vanished_item( + VanishedCollection::Email, + (mailbox.mailbox_id, mailbox.uid), + ); + } + thread_ids.insert(metadata.thread_id); + if !metadata.metadata_kinds().is_empty() { + containers.release_or_assert_absent(batch, account_id, document_id); + } + let index_builder = ObjectIndexBuilder::<_, ()>::new(); + batch + .with_document(document_id) + .custom(index_builder.with_current(metadata)) + .caused_by(trc::location!())? + .queue_document_unindex(SearchIndex::Email, account_id, document_id) + .commit_point(); - deleted_ids.insert(document_id); + deleted_ids.insert(document_id); - Ok(true) - }, - ) + Ok(true) + }) .await?; - self.log_emptied_threads(account_id, batch, thread_ids, &deleted_ids) - .await?; + log_threads_emptied(&cache, batch, account_id, &thread_ids, &deleted_ids); let not_destroyed = if document_ids.len() == deleted_ids.len() { RoaringBitmap::new() @@ -131,18 +140,7 @@ impl EmailDeletion for Server { .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - for thread_id in &thread_ids { - if cache - .in_thread(thread_id) - .all(|message| deleted_ids.contains(message.document_id)) - { - batch - .with_account_id(account_id) - .with_collection(Collection::Thread) - .with_document(thread_id) - .log_container_delete(SyncCollection::Thread); - } - } + log_threads_emptied(&cache, batch, account_id, &thread_ids, deleted_ids); } Ok(()) @@ -179,6 +177,49 @@ impl EmailDeletion for Server { .await .caused_by(trc::location!())?; + self.purge_private_metadata(account_id, self.core.email.changes_max_history) + .await + .caused_by(trc::location!())?; + + // Purge expired Sieve duplicate and vacation ids + self.purge_sieve_seen_ids(account_id, now()) + .await + .caused_by(trc::location!())?; + + Ok(()) + } + + async fn purge_sieve_seen_ids(&self, account_id: u32, cutoff: u64) -> trc::Result<()> { + let (from, to) = SeenIdHash::account_range(account_id); + let mut expired = Vec::new(); + self.store() + .iterate(IterateParams::new(from, to).ascending(), |key, value| { + if u64::deserialize(value)? <= cutoff { + expired.push(SeenIdHash::from_key(key).ok_or_else(|| { + trc::Error::corrupted_key(key, value.into(), trc::location!()) + })?); + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + + let mut expired = expired.into_iter().peekable(); + while expired.peek().is_some() { + let mut batch = BatchBuilder::new(); + SeenIdHash::with_context(account_id, &mut batch); + for hash in expired.by_ref() { + batch.clear(hash.class()); + if batch.is_large_batch() { + break; + } + } + self.store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + } + Ok(()) } @@ -228,22 +269,20 @@ impl EmailDeletion for Server { // Delete messages let mut batch = BatchBuilder::new(); - let tenant_id = self - .account(account_id) - .await - .caused_by(trc::location!())? - .tenant_id(); - self.emails_delete(account_id, tenant_id, &mut batch, destroy_ids) + let changed_by = ChangedBy { + account_id, + }; + self.emails_delete(account_id, changed_by, &mut batch, destroy_ids) .await?; self.commit_batch(batch).await?; - self.notify_task_queue(); Ok(()) } async fn purge_email_submissions(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { - // Filter messages by received date + let expire_until = now().saturating_sub(hold_period); let mut destroy_ids = Vec::new(); + let mut live_queue_ids = Vec::new(); self.store() .iterate( IterateParams::new( @@ -258,21 +297,22 @@ impl EmailDeletion for Server { }, ValueKey { account_id, - collection: Collection::Email.into(), + collection: Collection::EmailSubmission.into(), document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), - value: now().saturating_sub(hold_period), + value: u64::MAX, }), }, ) - .ascending() - .no_values(), - |key, _| { - destroy_ids.push(( - key.deserialize_be_u32(key.len() - U32_LEN)?, - key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, - )); + .ascending(), + |key, value| { + let send_at = key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?; + if send_at <= expire_until { + destroy_ids.push((key.deserialize_be_u32(key.len() - U32_LEN)?, send_at)); + } else { + live_queue_ids.push(value.deserialize_be_u64(METADATA_QUEUE_ID_OFFSET)?); + } Ok(true) }, @@ -280,7 +320,29 @@ impl EmailDeletion for Server { .await .caused_by(trc::location!())?; - if destroy_ids.is_empty() { + let mut orphan_queue_ids = Vec::new(); + if let Some(max_queue_id) = + SnowflakeIdGenerator::from_duration(Duration::from_secs(ORPHAN_RESULT_GRACE)) + { + live_queue_ids.sort_unstable(); + let (from_key, to_key) = DeliveryResults::range(account_id, max_queue_id); + self.store() + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let queue_id = key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?; + if live_queue_ids.binary_search(&queue_id).is_err() { + orphan_queue_ids.push(queue_id); + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + + if destroy_ids.is_empty() && orphan_queue_ids.is_empty() { return Ok(()); } @@ -291,21 +353,46 @@ impl EmailDeletion for Server { Total = destroy_ids.len(), ); - // Delete messages let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::EmailSubmission); + for queue_id in orphan_queue_ids { + DeliveryResults::clear(&mut batch, account_id, queue_id); + batch.commit_point(); + } for (document_id, send_at) in destroy_ids { batch - .with_document(document_id) - .clear(EmailSubmissionField::Metadata) - .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { + .with_account_id(account_id) + .with_collection(Collection::EmailSubmission) + .with_document(document_id); + + if let Some(submission) = self + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::EmailSubmission, + document_id, + )) + .await + .caused_by(trc::location!())? + { + let submission = submission + .to_unarchived::() + .caused_by(trc::location!())?; + let queue_id = submission.inner.queue_id.as_ref().map(u64::from); + batch + .custom(ObjectIndexBuilder::<_, ()>::new().with_current(submission)) + .caused_by(trc::location!())?; + if let Some(queue_id) = queue_id { + DeliveryResults::clear(&mut batch, account_id, queue_id); + } + } else { + batch.clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: send_at, - })) - .commit_point(); + })); + } + + batch.commit_point(); } self.commit_batch(batch).await?; @@ -313,3 +400,30 @@ impl EmailDeletion for Server { Ok(()) } } + +const METADATA_QUEUE_ID_OFFSET: usize = U32_LEN * 3; +const ORPHAN_RESULT_GRACE: u64 = 86400; +const EMPTIED_THREADS_PER_COMMIT: usize = 10_000; + +fn log_threads_emptied( + cache: &MessageStoreCache, + batch: &mut BatchBuilder, + account_id: u32, + thread_ids: &RoaringBitmap, + deleted_ids: &RoaringBitmap, +) { + let emptied = cache.emails.emptied_threads(thread_ids, deleted_ids); + for (pos, chunk) in emptied.chunks(EMPTIED_THREADS_PER_COMMIT).enumerate() { + if pos > 0 { + batch.add_commit_point(); + } + batch + .with_account_id(account_id) + .with_collection(Collection::Thread); + for &thread_id in chunk { + batch + .with_document(thread_id) + .log_container_delete(SyncCollection::Thread); + } + } +} diff --git a/crates/email/src/message/delivery.rs b/crates/email/src/message/delivery.rs index f65faeca..e5e82b51 100644 --- a/crates/email/src/message/delivery.rs +++ b/crates/email/src/message/delivery.rs @@ -13,6 +13,7 @@ use common::{ }; use mail_parser::MessageParser; use registry::schema::enums::Permission; +use sieve::{MessageSource, compiler::grammar::actions::action_redirect::Notify}; use std::{borrow::Cow, future::Future}; use store::ahash::AHashMap; use types::blob_hash::BlobHash; @@ -22,13 +23,26 @@ pub const ORCPT_ADDR_TYPE: &str = "rfc822;"; #[derive(Debug)] pub struct IngestMessage { pub sender_address: String, - pub sender_authenticated: bool, + pub sender_authentication: SenderAuthentication, pub recipients: Vec, pub message_blob: BlobHash, pub message_size: u64, pub session_id: u64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SenderAuthentication { + Unauthenticated, + DmarcPass, + Submission, +} + +impl SenderAuthentication { + pub fn is_authenticated(&self) -> bool { + !matches!(self, SenderAuthentication::Unauthenticated) + } +} + #[derive(Debug)] pub struct IngestRecipient { pub address: String, @@ -66,10 +80,20 @@ pub struct LocalDeliveryResult { pub autogenerated: Vec, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AutogeneratedReturnPath { + Account, + Null, +} + pub struct AutogeneratedMessage { - pub sender_address: String, + pub account_address: String, + pub return_path: AutogeneratedReturnPath, pub recipients: Vec, pub message: Vec, + pub notify: Notify, + pub source: MessageSource, + pub exploded: bool, } pub trait MailDelivery: Sync + Send { @@ -147,7 +171,7 @@ impl MailDelivery for Server { Err(err) => { trc::error!( err.details("Failed to lookup recipient.") - .ctx(trc::Key::To, rcpt.address.to_string()) + .ctx(trc::Key::To, rcpt.address) .span_id(message.session_id) .caused_by(trc::location!()) ); @@ -186,25 +210,27 @@ impl MailDelivery for Server { received_at: None, source: IngestSource::Smtp { deliver_to: &rcpt.address, - is_sender_authenticated: message.sender_authenticated, + envelope_from: &message.sender_address, + sender_authentication: message.sender_authentication, is_spam: rcpt.is_spam(), }, + metadata: None, session_id: message.session_id, }) .await } Ok(Some(active_script)) => { - self.sieve_script_ingest( + Box::pin(self.sieve_script_ingest( &access_token, &message.message_blob, &raw_message, &message.sender_address, - message.sender_authenticated, + message.sender_authentication, &rcpt, message.session_id, active_script, &mut result.autogenerated, - ) + )) .await } Err(err) => Err(err), @@ -235,11 +261,6 @@ impl MailDelivery for Server { reason: "Mailbox over quota.".into(), } } - trc::EventType::Limit(trc::LimitEvent::TenantQuota) => { - LocalDeliveryStatus::TemporaryFailure { - reason: "Organization over quota.".into(), - } - } trc::EventType::Security(trc::SecurityEvent::Unauthorized) => { LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], @@ -268,7 +289,7 @@ impl MailDelivery for Server { }; trc::error!( - err.ctx(trc::Key::To, rcpt.address.to_string()) + err.ctx(trc::Key::To, rcpt.address) .span_id(message.session_id) ); diff --git a/crates/email/src/message/headers.rs b/crates/email/src/message/headers.rs index 3e0ae756..fd56597e 100644 --- a/crates/email/src/message/headers.rs +++ b/crates/email/src/message/headers.rs @@ -4,12 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue}; use jmap_proto::{ object::email::{EmailProperty, EmailValue, HeaderForm, HeaderProperty}, types::date::UTCDate, }; -use jmap_tools::{Key, Map, Value}; +use jmap_tools::{Key, Value}; use mail_builder::{ MessageBuilder, headers::{ @@ -21,20 +20,6 @@ use mail_builder::{ url::URL, }, }; -use mail_parser::{Addr, DateTime, Group, Header, HeaderName, HeaderValue, parsers::MessageStream}; -use utils::chained_bytes::ChainedBytes; - -pub trait HeaderToValue { - fn header_to_value( - &self, - property: &EmailProperty, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue>; - fn headers_to_value( - &self, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue>; -} pub trait ValueToHeader<'x> { fn try_into_grouped_addresses(self) -> Option>; @@ -50,105 +35,6 @@ pub trait BuildHeader<'x>: Sized { ) -> Result; } -impl HeaderToValue for Vec> { - fn header_to_value( - &self, - property: &EmailProperty, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue> { - let (header_name, form, all) = match property { - EmailProperty::Header(header) => ( - HeaderName::parse(header.header.as_str()) - .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())), - header.form, - header.all, - ), - EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), - EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false), - EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false), - EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), - EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), - EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), - EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false), - EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), - EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), - EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false), - EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false), - _ => return Value::Null, - }; - - let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured(); - let mut headers = Vec::new(); - let header_name = header_name.as_str(); - for header in self.iter().rev() { - if header.name.as_str().eq_ignore_ascii_case(header_name) { - let raw_header; - let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) { - raw_header = - raw_message.get(header.offset_start as usize..header.offset_end as usize); - - if let Some(bytes) = &raw_header { - let bytes = bytes.as_ref(); - match form { - HeaderForm::Raw => { - HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) - } - HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), - HeaderForm::Addresses - | HeaderForm::GroupedAddresses - | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), - HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), - HeaderForm::Date => MessageStream::new(bytes).parse_date(), - } - } else { - HeaderValue::Empty - } - } else { - header.value.clone() - }; - headers.push(header_value.into_form(&form)); - if !all { - break; - } - } - } - - if !all { - headers.pop().unwrap_or_default() - } else { - if headers.len() > 1 { - headers.reverse(); - } - Value::Array(headers) - } - } - - fn headers_to_value( - &self, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue> { - let mut headers = Vec::with_capacity(self.len()); - for header in self.iter() { - headers.push(Value::Object( - Map::with_capacity(2) - .with_key_value(EmailProperty::Name, header.name().to_string()) - .with_key_value( - EmailProperty::Value, - String::from_utf8_lossy( - raw_message - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default() - .as_ref() - .trim_end(), - ) - .into_owned(), - ), - )); - } - headers.into() - } -} - impl<'x> ValueToHeader<'x> for Value<'x, EmailProperty, EmailValue> { fn try_into_grouped_addresses(self) -> Option> { let mut obj = self.into_object()?; @@ -305,119 +191,6 @@ impl<'x> BuildHeader<'x> for MessageBuilder<'x> { } } -impl HeaderToValue for ArchivedMessageMetadataPart { - fn header_to_value( - &self, - property: &EmailProperty, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue> { - let (header_name, form, all) = match property { - EmailProperty::Header(header) => ( - HeaderName::parse(header.header.as_str()) - .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())), - header.form, - header.all, - ), - EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), - EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false), - EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false), - EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), - EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), - EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), - EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false), - EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), - EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), - EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false), - EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false), - _ => return Value::Null, - }; - - let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured(); - let mut headers = Vec::new(); - let header_name = header_name.as_str(); - for header in self.headers.iter().rev() { - if header.name.as_str().eq_ignore_ascii_case(header_name) { - let raw_header; - let header_value = - if is_raw || matches!(header.value, ArchivedMetadataHeaderValue::Empty) { - raw_header = raw_message.get(header.value_range()); - - if let Some(bytes) = &raw_header { - let bytes = bytes.as_ref(); - match form { - HeaderForm::Raw => { - HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) - } - HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), - HeaderForm::Addresses - | HeaderForm::GroupedAddresses - | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), - HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), - HeaderForm::Date => MessageStream::new(bytes).parse_date(), - } - } else { - HeaderValue::Empty - } - } else { - HeaderValue::from(&header.value) - }; - headers.push(header_value.into_form(&form)); - if !all { - break; - } - } - } - - if !all { - headers.pop().unwrap_or_default() - } else { - if headers.len() > 1 { - headers.reverse(); - } - Value::Array(headers) - } - } - - fn headers_to_value( - &self, - raw_message: &ChainedBytes<'_>, - ) -> Value<'static, EmailProperty, EmailValue> { - let mut headers = Vec::with_capacity(self.headers.len()); - for header in self.headers.iter() { - headers.push(Value::Object( - Map::with_capacity(2) - .with_key_value(EmailProperty::Name, header.name.as_str().to_string()) - .with_key_value( - EmailProperty::Value, - String::from_utf8_lossy( - raw_message - .get(header.value_range()) - .unwrap_or_default() - .as_ref() - .trim_end(), - ) - .into_owned(), - ), - )); - } - headers.into() - } -} - -trait ByteTrim { - fn trim_end(&self) -> Self; -} - -impl ByteTrim for &[u8] { - fn trim_end(&self) -> Self { - let mut end = self.len(); - while end > 0 && self[end - 1].is_ascii_whitespace() { - end -= 1; - } - &self[..end] - } -} - #[inline] pub(crate) fn unwrap_date(value: Value<'_, EmailProperty, EmailValue>) -> Option { match value { @@ -425,118 +198,3 @@ pub(crate) fn unwrap_date(value: Value<'_, EmailProperty, EmailValue>) -> Option _ => None, } } - -pub trait IntoForm { - fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue>; -} - -impl IntoForm for HeaderValue<'_> { - fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue> { - match (self, form) { - (HeaderValue::Text(text), HeaderForm::Raw | HeaderForm::Text) => { - text.into_owned().into() - } - (HeaderValue::TextList(texts), HeaderForm::Raw | HeaderForm::Text) => { - texts.join(", ").into() - } - (HeaderValue::Text(text), HeaderForm::MessageIds) => { - Value::Array(vec![text.into_owned().into()]) - } - (HeaderValue::TextList(texts), HeaderForm::MessageIds) => { - Value::Array(texts.into_iter().map(|t| t.into_owned().into()).collect()) - } - (HeaderValue::DateTime(datetime), HeaderForm::Date) => from_mail_datetime(datetime), - (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::URLs) => { - Value::Array( - addrlist - .into_iter() - .filter_map(|addr| match addr { - Addr { - address: Some(addr), - .. - } if addr.contains(':') => Some(addr.into_owned().into()), - _ => None, - }) - .collect(), - ) - } - (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::Addresses) => { - from_mail_addrlist(addrlist) - } - ( - HeaderValue::Address(mail_parser::Address::Group(grouplist)), - HeaderForm::Addresses, - ) => Value::Array( - grouplist - .into_iter() - .flat_map(|group| group.addresses.into_iter().map(from_mail_addr)) - .collect(), - ), - ( - HeaderValue::Address(mail_parser::Address::List(addrlist)), - HeaderForm::GroupedAddresses, - ) => Value::Array(vec![ - Map::with_capacity(2) - .with_key_value(EmailProperty::Name, Value::Null) - .with_key_value(EmailProperty::Addresses, from_mail_addrlist(addrlist)) - .into(), - ]), - ( - HeaderValue::Address(mail_parser::Address::Group(grouplist)), - HeaderForm::GroupedAddresses, - ) => Value::Array( - grouplist - .into_iter() - .map(from_mail_group) - .collect::>>(), - ), - - _ => Value::Null, - } - } -} - -fn from_mail_datetime(date: DateTime) -> Value<'static, EmailProperty, EmailValue> { - Value::Element(EmailValue::Date(UTCDate { - year: date.year, - month: date.month, - day: date.day, - hour: date.hour, - minute: date.minute, - second: date.second, - tz_before_gmt: date.tz_before_gmt, - tz_hour: date.tz_hour, - tz_minute: date.tz_minute, - })) -} - -fn from_mail_addr(value: Addr<'_>) -> Value<'static, EmailProperty, EmailValue> { - Value::Object( - Map::with_capacity(2) - .with_key_value(EmailProperty::Name, value.name.map(|v| v.into_owned())) - .with_key_value( - EmailProperty::Email, - value.address.unwrap_or_default().into_owned(), - ), - ) -} - -fn from_mail_group(group: Group<'_>) -> Value<'static, EmailProperty, EmailValue> { - Value::Object( - Map::with_capacity(2) - .with_key_value(EmailProperty::Name, group.name.map(|v| v.into_owned())) - .with_key_value( - EmailProperty::Addresses, - from_mail_addrlist(group.addresses), - ), - ) -} - -fn from_mail_addrlist(addrlist: Vec>) -> Value<'static, EmailProperty, EmailValue> { - Value::Array( - addrlist - .into_iter() - .map(from_mail_addr) - .collect::>>(), - ) -} diff --git a/crates/email/src/message/index/attachment.rs b/crates/email/src/message/index/attachment.rs new file mode 100644 index 00000000..826c5939 --- /dev/null +++ b/crates/email/src/message/index/attachment.rs @@ -0,0 +1,202 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::message::metadata::{ArchivedMessageMetadata, PartKind, PartSource, PartView}; +use common::config::mailstore::email::ExtractLimits; +use nlp::language::{ + Language, + detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, +}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use store::search::{EmailSearchField, IndexDocument, SearchField}; +use text_extract::{Extractor, Format, Hints, Limits}; + +const MESSAGE_TEXT_DOCUMENTS: usize = 2; +const MAX_LANGUAGE_SAMPLE: usize = 64 << 10; + +pub(super) struct AttachmentText { + extractor: Option, + limits: Limits, + text: String, + remaining_text: usize, + remaining_decompressed: u64, +} + +pub fn document_limits(config: &ExtractLimits) -> Limits { + Limits { + max_input_bytes: config.max_document_size, + max_output_bytes: config.max_text_size, + max_entries: config.max_archive_entries, + max_parts: config.max_parts, + max_part_bytes: config.max_part_size.min(config.max_decompressed_size), + max_total_bytes: config.max_decompressed_size, + max_rtf_depth: config.max_rtf_depth, + max_pdf_objects: config.max_pdf_objects, + } +} + +impl<'a> PartView<'a> { + pub fn extraction_hints(&self) -> Hints<'a> { + let mut hints = Hints::new(); + if let Some(content_type) = self.content_type() { + hints = hints.with_media_type_parts( + content_type.ctype(), + content_type.subtype().unwrap_or_default(), + ); + } + if let Some(name) = self.attachment_name() { + hints = hints.with_file_name(name); + } + hints + } + + pub fn has_extractable_text(&self) -> bool { + match self.kind() { + PartKind::Binary => self.extraction_hints().may_be_supported(), + PartKind::Text => self.is_rtf_text(), + _ => false, + } + } + + fn is_rtf_text(&self) -> bool { + match self.content_type() { + Some(content_type) if content_type.ctype().eq_ignore_ascii_case("text") => content_type + .subtype() + .is_some_and(|subtype| subtype.eq_ignore_ascii_case("rtf")), + _ => self.extraction_hints().format() == Some(Format::Rtf), + } + } +} + +impl ArchivedMessageMetadata { + pub fn has_extractable_attachments(&self) -> bool { + self.parts().any(|part| part.has_extractable_text()) + } +} + +impl AttachmentText { + pub(super) fn new(config: &ExtractLimits) -> Self { + AttachmentText { + extractor: None, + limits: document_limits(config), + text: String::new(), + remaining_text: config.max_text_size.saturating_mul(MESSAGE_TEXT_DOCUMENTS), + remaining_decompressed: config.max_decompressed_size, + } + } + + pub(super) fn index_binary( + &mut self, + part: PartView<'_>, + source: &PartSource<'_>, + document: &mut IndexDocument, + detector: &mut LanguageDetector, + language: Language, + ) { + let hints = part.extraction_hints(); + if !hints.may_be_supported() + || self.remaining_text == 0 + || part.decoded_size() as usize > self.limits.max_input_bytes + { + return; + } + if self.extract(&part.decoded(source), hints) { + self.index(document, detector, language); + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn index_rtf( + &mut self, + part: PartView<'_>, + contents: &str, + document: &mut IndexDocument, + detector: &mut LanguageDetector, + language: Language, + field: EmailSearchField, + ) -> bool { + if part.is_rtf_text() && self.extract(contents.as_bytes(), part.extraction_hints()) { + self.index_field(document, detector, language, field); + true + } else { + false + } + } + + fn extract(&mut self, contents: &[u8], hints: Hints<'_>) -> bool { + if self.remaining_text == 0 + || (self.remaining_decompressed == 0 && hints.format() != Some(Format::Rtf)) + { + return false; + } + let defaults = &self.limits; + let extractor = self + .extractor + .get_or_insert_with(|| Extractor::new(defaults.clone())); + let limits = extractor.limits_mut(); + limits.max_output_bytes = defaults.max_output_bytes.min(self.remaining_text); + limits.max_total_bytes = defaults.max_total_bytes.min(self.remaining_decompressed); + self.text.clear(); + let text = &mut self.text; + match catch_unwind(AssertUnwindSafe(|| { + extractor.extract(contents, hints, text) + })) { + Ok(Ok(extraction)) => { + self.remaining_text = self.remaining_text.saturating_sub(extraction.bytes_written); + self.remaining_decompressed = self + .remaining_decompressed + .saturating_sub(extraction.bytes_decompressed); + !self.text.is_empty() + } + Ok(Err(failure)) => { + self.remaining_decompressed = self + .remaining_decompressed + .saturating_sub(failure.bytes_decompressed); + false + } + Err(_) => { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Attachment text extraction panicked", + ); + self.extractor = None; + self.remaining_text = 0; + false + } + } + } + + fn index( + &self, + document: &mut IndexDocument, + detector: &mut LanguageDetector, + language: Language, + ) { + self.index_field(document, detector, language, EmailSearchField::Attachment); + } + + fn index_field( + &self, + document: &mut IndexDocument, + detector: &mut LanguageDetector, + language: Language, + field: EmailSearchField, + ) { + if language.is_unknown() { + detector.detect(language_sample(&self.text), MIN_LANGUAGE_SCORE); + } + document.index_text(SearchField::Email(field), &self.text, language); + } +} + +fn language_sample(text: &str) -> &str { + if text.len() <= MAX_LANGUAGE_SAMPLE { + return text; + } + text.get(..text.floor_char_boundary(MAX_LANGUAGE_SAMPLE)) + .unwrap_or_default() +} + diff --git a/crates/email/src/message/index/extractors.rs b/crates/email/src/message/index/extractors.rs index 8b969da6..bb35e061 100644 --- a/crates/email/src/message/index/extractors.rs +++ b/crates/email/src/message/index/extractors.rs @@ -4,33 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ - ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue, - MetadataHeaderName, MetadataHeaderValue, -}; -use mail_parser::{Addr, Address, Group, HeaderValue}; +use crate::message::metadata::PartView; +use mail_parser::{Address, HeaderValue, Mailbox}; use nlp::language::Language; -use rkyv::option::ArchivedOption; -use std::borrow::Cow; -impl ArchivedMessageMetadataContents { - pub fn is_html_part(&self, part_id: u16) -> bool { - self.html_body.iter().any(|&id| id == part_id) - } - - pub fn is_text_part(&self, part_id: u16) -> bool { - self.text_body.iter().any(|&id| id == part_id) - } -} - -impl ArchivedMessageMetadataPart { +impl PartView<'_> { pub fn language(&self) -> Option { - self.header_value(&MetadataHeaderName::ContentLanguage) - .and_then(|v| { - Language::from_iso_639(v.as_text()?) - .unwrap_or(Language::Unknown) - .into() - }) + self.content_language() + .last() + .map(|language| Language::from_iso_639(language).unwrap_or(Language::Unknown)) } } @@ -47,47 +29,42 @@ pub trait VisitText { fn into_visit_text(self, visitor: impl FnMut(String)); } +fn visit_mailbox(mailbox: Mailbox<'_>, visitor: &mut impl FnMut(AddressElement, &str)) { + if let Some(name) = mailbox.name() { + visitor(AddressElement::Name, name); + } + if let Some(addr) = mailbox.address() { + visitor(AddressElement::Address, addr); + } +} + impl VisitText for HeaderValue<'_> { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { - match self { - HeaderValue::Address(Address::List(addr_list)) => { - for addr in addr_list { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - HeaderValue::Address(Address::Group(groups)) => { - for group in groups { - if let Some(name) = &group.name { - visitor(AddressElement::GroupName, name); - } - - for addr in &group.addresses { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); + if let HeaderValue::Address(addresses) = self { + for address in addresses.iter() { + match address { + Address::Mailbox(mailbox) => visit_mailbox(mailbox, &mut visitor), + Address::Group(group) => { + if let Some(name) = group.name() { + visitor(AddressElement::GroupName, name); } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); + for mailbox in group.mailboxes() { + visit_mailbox(mailbox, &mut visitor); } } } } - _ => (), } } fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) { - match &self { + match self { HeaderValue::Text(text) => { - visitor(text.as_ref()); + visitor(text); } HeaderValue::TextList(texts) => { - for text in texts { - visitor(text.as_ref()); + for text in texts.iter() { + visitor(text); } } _ => (), @@ -95,197 +72,6 @@ impl VisitText for HeaderValue<'_> { } fn into_visit_text(self, mut visitor: impl FnMut(String)) { - match self { - HeaderValue::Text(text) => { - visitor(text.into_owned()); - } - HeaderValue::TextList(texts) => { - for text in texts { - visitor(text.into_owned()); - } - } - _ => (), - } - } -} - -pub trait VisitTextArchived { - fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str)); - fn visit_text(&self, visitor: impl FnMut(&str)); -} - -impl VisitTextArchived for MetadataHeaderValue { - fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { - match self { - MetadataHeaderValue::AddressList(addr_list) => { - for addr in addr_list.iter() { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - MetadataHeaderValue::AddressGroup(groups) => { - for group in groups.iter() { - if let Some(name) = &group.name { - visitor(AddressElement::GroupName, name); - } - - for addr in group.addresses.iter() { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - } - _ => (), - } - } - - fn visit_text(&self, mut visitor: impl FnMut(&str)) { - match &self { - MetadataHeaderValue::Text(text) => { - visitor(text.as_ref()); - } - MetadataHeaderValue::TextList(texts) => { - for text in texts.iter() { - visitor(text.as_ref()); - } - } - _ => (), - } - } -} - -impl VisitTextArchived for ArchivedMetadataHeaderValue { - fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { - match self { - ArchivedMetadataHeaderValue::AddressList(addr_list) => { - for addr in addr_list.iter() { - if let ArchivedOption::Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let ArchivedOption::Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - ArchivedMetadataHeaderValue::AddressGroup(groups) => { - for group in groups.iter() { - if let ArchivedOption::Some(name) = &group.name { - visitor(AddressElement::GroupName, name); - } - - for addr in group.addresses.iter() { - if let ArchivedOption::Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let ArchivedOption::Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - } - _ => (), - } - } - - fn visit_text(&self, mut visitor: impl FnMut(&str)) { - match &self { - ArchivedMetadataHeaderValue::Text(text) => { - visitor(text.as_ref()); - } - ArchivedMetadataHeaderValue::TextList(texts) => { - for text in texts.iter() { - visitor(text.as_ref()); - } - } - _ => (), - } - } -} - -pub trait TrimTextValue { - fn trim_text(self, length: usize) -> Self; -} - -impl TrimTextValue for HeaderValue<'_> { - fn trim_text(self, length: usize) -> Self { - match self { - HeaderValue::Address(Address::List(v)) => { - HeaderValue::Address(Address::List(v.trim_text(length))) - } - HeaderValue::Address(Address::Group(v)) => { - HeaderValue::Address(Address::Group(v.trim_text(length))) - } - HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)), - HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)), - v => v, - } - } -} - -impl TrimTextValue for Addr<'_> { - fn trim_text(self, length: usize) -> Self { - Self { - name: self.name.map(|v| v.trim_text(length)), - address: self.address.map(|v| v.trim_text(length)), - } - } -} - -impl TrimTextValue for Group<'_> { - fn trim_text(self, length: usize) -> Self { - Self { - name: self.name.map(|v| v.trim_text(length)), - addresses: self.addresses.trim_text(length), - } - } -} - -impl TrimTextValue for &str { - fn trim_text(self, length: usize) -> Self { - if self.len() < length { - self - } else { - let mut index = 0; - - for (i, _) in self.char_indices() { - if i > length { - break; - } - index = i; - } - - &self[..index] - } - } -} - -impl TrimTextValue for Cow<'_, str> { - fn trim_text(self, length: usize) -> Self { - if self.len() < length { - self - } else { - let mut result = String::with_capacity(length); - for (i, c) in self.char_indices() { - if i > length { - break; - } - result.push(c); - } - result.into() - } - } -} - -impl TrimTextValue for Vec { - fn trim_text(self, length: usize) -> Self { - self.into_iter().map(|v| v.trim_text(length)).collect() + self.visit_text(|text| visitor(text.to_string())); } } diff --git a/crates/email/src/message/index/metadata.rs b/crates/email/src/message/index/metadata.rs index 50eff879..8aa72953 100644 --- a/crates/email/src/message/index/metadata.rs +++ b/crates/email/src/message/index/metadata.rs @@ -5,245 +5,91 @@ */ use crate::message::{ - index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH}, - metadata::{ - ArchivedMessageMetadata, ArchivedMessageMetadataPart, ArchivedMetadataHeaderName, - MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, - MessageMetadataPart, build_metadata_contents, - }, + index::{IndexMessage, PendingMessageData}, + metadata::{ArchivedMessageMetadata, ExtraHeaders, MessageMetadata}, + sortkeys::MessageSortKeys, }; use common::storage::index::ObjectIndexBuilder; -use mail_parser::{ - PartType, - decoders::html::html_to_text, - parsers::{fields::thread::thread_name, preview::preview_text}, -}; use store::{ - Serialize, - write::{Archiver, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass}, + write::{BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass, blob::BlobLinkValue}, + xxhash_rust::xxh3::xxh3_128, }; use trc::AddContext; -use types::{blob_hash::BlobHash, field::EmailField}; -use utils::cheeky_hash::CheekyHash; - -impl MessageMetadata { - #[inline(always)] - pub fn root_part(&self) -> &MessageMetadataPart { - &self.contents[0].parts[0] - } - - pub fn index(self, batch: &mut BatchBuilder, set: bool) -> trc::Result<()> { - if set { - batch - .set( - BlobOp::Link { - hash: self.blob_hash.clone(), - to: BlobLink::Document, - }, - Vec::new(), - ) - .set(EmailField::Metadata, Archiver::new(self).serialize()?); - } else { - batch - .clear(BlobOp::Link { - hash: self.blob_hash.clone(), - to: BlobLink::Document, - }) - .clear(EmailField::Metadata); - } - - Ok(()) - } -} +use types::{ + blob_hash::BlobHash, + field::EmailField, + keyword::{HASATTACHMENT, HASNOATTACHMENT}, +}; +use utils::hash128::Hash128; impl ArchivedMessageMetadata { - #[inline(always)] - pub fn root_part(&self) -> &ArchivedMessageMetadataPart { - &self.contents[0].parts[0] + pub fn index_verbatim(&self, batch: &mut BatchBuilder, metadata: Vec, sort_keys: Vec) { + batch + .set( + BlobOp::Link { + hash: self.blob_hash(), + to: BlobLink::Document, + }, + BlobLinkValue::new(self.blob_size()), + ) + .set(EmailField::SortKeys, sort_keys) + .set(EmailField::Metadata, metadata); } - pub fn unindex(&self, batch: &mut BatchBuilder) { - // Delete metadata - let thread_name = self - .contents - .first() - .and_then(|c| c.parts.first()) - .and_then(|p| { - p.headers.iter().rev().find_map(|h| { - if let ArchivedMetadataHeaderName::Subject = &h.name { - h.value.as_text() - } else { - None - } - }) - }) - .map(thread_name) - .unwrap_or_default(); - + pub fn unindex(&self, batch: &mut BatchBuilder, thread_name: &str) { batch .clear(EmailField::Metadata) + .clear(EmailField::SortKeys) .clear(ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), - hash: CheekyHash::new(if !thread_name.is_empty() { - thread_name - } else { - "!" - }), + hash: Hash128::from(xxh3_128( + if !thread_name.is_empty() { + thread_name + } else { + "!" + } + .as_bytes(), + )), })) .clear(BlobOp::Link { - hash: BlobHash::from(&self.blob_hash), + hash: self.blob_hash(), to: BlobLink::Document, }); } } impl IndexMessage for BatchBuilder { - fn index_message<'x>( + fn index_message( &mut self, - tenant_id: Option, - mut message: mail_parser::Message<'x>, - extra_headers: Vec, - mut extra_headers_parsed: Vec>, + message: &mail_parser::Message<'_>, + extra_headers: &ExtraHeaders, blob_hash: BlobHash, - data: MessageData, - received_at: u64, + mut data: PendingMessageData, ) -> trc::Result<&mut Self> { - let mut has_attachments = false; - let mut preview = None; - let preview_part_id = message - .text_body - .first() - .or_else(|| message.html_body.first()) - .copied() - .unwrap_or(u32::MAX); - - for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { - let part_id = part_id as u32; - match &part.body { - mail_parser::PartType::Text(text) => { - if part_id == preview_part_id { - preview = - preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); - } - - if !message.text_body.contains(&part_id) - && !message.html_body.contains(&part_id) - { - has_attachments = true; - } - } - mail_parser::PartType::Html(html) => { - let text = html_to_text(html); - if part_id == preview_part_id { - preview = - preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); - } - - if !message.text_body.contains(&part_id) - && !message.html_body.contains(&part_id) - { - has_attachments = true; - } - } - mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_) - if !has_attachments => - { - has_attachments = true; - } - _ => {} - } + let built = MessageMetadata::build(message, extra_headers, blob_hash.clone()); + if built.has_attachments { + data.data.keywords |= 1 << HASATTACHMENT; + } else { + data.data.keywords |= 1 << HASNOATTACHMENT; } - // Build raw headers - let root_part = message.root_part(); - let mut raw_headers = Vec::with_capacity( - (root_part.offset_body - root_part.offset_header) as usize + extra_headers.len(), - ); - raw_headers.extend_from_slice(&extra_headers); - raw_headers.extend_from_slice( - message - .raw_message - .as_ref() - .get(root_part.offset_header as usize..root_part.offset_body as usize) - .unwrap_or_default(), - ); - - // Add additional headers to message - let blob_body_offset = if !extra_headers.is_empty() { - // Add extra headers to root part - let offset_start = extra_headers.len() as u32; - let mut part_iter_stack = Vec::new(); - let mut part_iter = message.parts.iter_mut(); - - loop { - if let Some(part) = part_iter.next() { - // Increment header offsets - for header in part.headers.iter_mut() { - header.offset_field += offset_start; - header.offset_start += offset_start; - header.offset_end += offset_start; - } - - // Adjust part offsets - part.offset_body += offset_start; - part.offset_end += offset_start; - part.offset_header += offset_start; - - if let PartType::Message(sub_message) = &mut part.body - && sub_message.root_part().offset_header != 0 - { - part_iter_stack.push(part_iter); - part_iter = sub_message.parts.iter_mut(); - } - } else if let Some(iter) = part_iter_stack.pop() { - part_iter = iter; - } else { - break; - } - } - - // Add extra headers to root part - let root_part = &mut message.parts[0]; - extra_headers_parsed.append(&mut root_part.headers); - root_part.offset_header = 0; - root_part.headers = extra_headers_parsed; - root_part.offset_body - offset_start - } else { - message.root_part().offset_body - }; - - // Build metadata - let metadata = MessageMetadata { - preview: preview.unwrap_or_default().into_owned().into_boxed_str(), - raw_headers: raw_headers.into_boxed_slice(), - contents: build_metadata_contents(message), - blob_hash, - blob_body_offset, - rcvd_attach: (if has_attachments { - MESSAGE_HAS_ATTACHMENT - } else { - 0 - }) | (received_at & MESSAGE_RECEIVED_MASK), - }; - + let index_builder = ObjectIndexBuilder::<(), _>::new(); self.set( BlobOp::Link { - hash: metadata.blob_hash.clone(), + hash: blob_hash, to: BlobLink::Document, }, - Vec::new(), - ) - .custom( - ObjectIndexBuilder::<(), _>::new() - .with_tenant_id(tenant_id) - .with_changes(data), + BlobLinkValue::new(message.raw().len()), ) + .custom(index_builder.with_changes(data)) .caused_by(trc::location!())? + .set( + EmailField::SortKeys, + MessageSortKeys::from_message(message).serialize(), + ) .set( EmailField::Metadata, - Archiver::new(metadata) - .serialize() - .caused_by(trc::location!())?, + built.encode().caused_by(trc::location!())?, ); Ok(self) diff --git a/crates/email/src/message/index/mod.rs b/crates/email/src/message/index/mod.rs index c0c397db..d8d1f022 100644 --- a/crates/email/src/message/index/mod.rs +++ b/crates/email/src/message/index/mod.rs @@ -6,26 +6,91 @@ use crate::{ mailbox::{JUNK_ID, TRASH_ID}, - message::metadata::{ArchivedMessageData, MessageData}, + message::{ + messagedata::{MessageData, PendingMessageData}, + metadata::ExtraHeaders, + }, +}; +use common::storage::index::{ + CurrentObject, IndexItem, IndexValue, IndexableObject, SerializableObject, +}; +use registry::schema::enums::StorageQuota; +use store::{ + Serialize, U64_LEN, + write::{ + BatchBuilder, Patch, PatchSource, PendingId, Slot, SlotRange, assert::AssertValue, + metadata::MetadataClass, now, + }, +}; +use types::{ + blob_hash::BlobHash, + collection::SyncCollection, + field::{EmailField, Field}, + metadata::MetadataKinds, }; -use common::storage::index::{IndexItem, IndexValue, IndexableObject}; -use store::write::now; -use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField}; +pub mod attachment; pub mod extractors; pub mod metadata; pub mod search; +pub mod subject; pub(super) const MAX_MESSAGE_PARTS: usize = 1000; pub const PREVIEW_LENGTH: usize = 256; +impl CurrentObject for MessageData { + fn assert(&self, batch: &mut BatchBuilder) { + batch.assert_value(Field::ARCHIVE, AssertValue::U64(self.change_id)); + } + + fn index_values(&self) -> impl Iterator> { + IndexableObject::index_values(self) + } + + fn clear(&self, batch: &mut BatchBuilder) { + batch.clear(Field::ARCHIVE); + if !IndexableObject::metadata_kinds(self).is_empty() { + batch.clear(MetadataClass::Shared); + } + } +} + +impl SerializableObject for MessageData { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + debug_assert!(pending_id.is_none(), "message data carries no pending id"); + let payload = self.serialize()?; + let offset = (payload.len() - U64_LEN) as u32; + batch.set( + Field::ARCHIVE, + ( + payload, + vec![Patch { + offset, + source: PatchSource::ChangeIdBe, + }], + ), + ); + Ok(()) + } +} + impl IndexableObject for MessageData { fn index_values(&self) -> impl Iterator> { + self.index_values_in_thread(PendingId::Assigned(self.thread_id)) + } + + fn metadata_kinds(&self) -> MetadataKinds { + MessageData::metadata_kinds(self) + } +} + +impl MessageData { + fn index_values_in_thread(&self, thread_id: PendingId) -> impl Iterator> { let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); let mut is_in_trash = false; for mailbox in &self.mailboxes { - mailboxes.push(mailbox.mailbox_id); + mailboxes.push(PendingId::Assigned(mailbox.mailbox_id)); is_in_trash |= mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID; } @@ -39,13 +104,14 @@ impl IndexableObject for MessageData { }, }, IndexValue::Quota { used: self.size }, + IndexValue::ObjectCount(StorageQuota::MaxEmails), IndexValue::LogItem { sync_collection: SyncCollection::Email, - prefix: self.thread_id.into(), + prefix: Some(thread_id), }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Thread, - ids: vec![self.thread_id], + ids: vec![thread_id], }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Email, @@ -56,56 +122,50 @@ impl IndexableObject for MessageData { } } -impl IndexableObject for &ArchivedMessageData { - fn index_values(&self) -> impl Iterator> { - let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); - let mut is_in_trash = false; - - for mailbox in self.mailboxes.iter() { - let mailbox_id = mailbox.mailbox_id.to_native(); - mailboxes.push(mailbox_id); - is_in_trash |= mailbox_id == TRASH_ID || mailbox_id == JUNK_ID; +impl PendingMessageData { + pub fn new(data: MessageData) -> Self { + PendingMessageData { + data, + uid_slots: SlotRange::default(), + thread_slot: None, + change_id: None, } + } - [ - IndexValue::Property { - field: EmailField::DeletedAt.into(), - value: if is_in_trash { - IndexItem::from(now()) - } else { - IndexItem::None - }, - }, - IndexValue::Quota { - used: self.size.to_native(), - }, - IndexValue::LogItem { - sync_collection: SyncCollection::Email, - prefix: self.thread_id.to_native().into(), - }, - IndexValue::LogContainerProperty { - sync_collection: SyncCollection::Thread, - ids: vec![self.thread_id.to_native()], - }, - IndexValue::LogContainerProperty { - sync_collection: SyncCollection::Email, - ids: mailboxes, - }, - ] - .into_iter() + pub fn thread_id(&self) -> PendingId { + match self.thread_slot { + Some(thread_slot) => PendingId::Slot(thread_slot), + None => PendingId::Assigned(self.data.thread_id), + } + } +} + +impl IndexableObject for PendingMessageData { + fn index_values(&self) -> impl Iterator> { + self.data.index_values_in_thread(self.thread_id()) + } +} + +impl SerializableObject for PendingMessageData { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + debug_assert!(pending_id.is_none(), "message data carries no pending id"); + debug_assert_eq!( + self.uid_slots.len(), + self.data.pending_uids(), + "unassigned uids and reserved uid slots disagree" + ); + batch.set(Field::ARCHIVE, self); + Ok(()) } } pub(super) trait IndexMessage { - #[allow(clippy::too_many_arguments)] - fn index_message<'x>( + fn index_message( &mut self, - tenant_id: Option, - message: mail_parser::Message<'x>, - extra_headers: Vec, - extra_headers_parsed: Vec>, + message: &mail_parser::Message<'_>, + extra_headers: &ExtraHeaders, blob_hash: BlobHash, - data: MessageData, - received_at: u64, + data: PendingMessageData, ) -> trc::Result<&mut Self>; } + diff --git a/crates/email/src/message/index/search.rs b/crates/email/src/message/index/search.rs index a7ed9aeb..8e95ee5d 100644 --- a/crates/email/src/message/index/search.rs +++ b/crates/email/src/message/index/search.rs @@ -5,14 +5,17 @@ */ use crate::message::{ - index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived}, + index::{ + MAX_MESSAGE_PARTS, attachment::AttachmentText, extractors::VisitText, + subject::search_subject, + }, metadata::{ - ArchivedMessageMetadata, ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, - ArchivedMetadataPartType, DecodedPartContent, MESSAGE_HAS_ATTACHMENT, - MESSAGE_RECEIVED_MASK, MetadataHeaderName, + AddressHeader, AddressItem, Addresses, ArchivedMessageMetadata, EnvelopeView, HeaderId, + HeaderList, Occurrence, PartFlags, PartKind, PartSource, PartView, }, }; -use mail_parser::{DateTime, decoders::html::html_to_text, parsers::fields::thread::thread_name}; +use common::config::mailstore::email::ExtractLimits; +use mail_parser::{HeaderForm, HeaderValue, html_to_text}; use nlp::{ language::{ Language, @@ -20,303 +23,113 @@ use nlp::{ }, tokenizers::word::WordTokenizer, }; +use std::{borrow::Cow, str::from_utf8}; use store::{ ahash::AHashSet, backend::MAX_TOKEN_LENGTH, search::{EmailSearchField, IndexDocument, SearchField}, write::SearchIndex, }; -use utils::chained_bytes::ChainedBytes; + +struct DocumentBuilder<'x> { + document: IndexDocument, + detector: LanguageDetector, + attachments: AttachmentText, + index_fields: &'x AHashSet, +} impl ArchivedMessageMetadata { + #[allow(clippy::too_many_arguments)] pub fn index_document( &self, account_id: u32, document_id: u32, - raw_message: &[u8], + headers: &[u8], + blob: &[u8], index_fields: &AHashSet, default_language: Language, + extract_limits: &ExtractLimits, ) -> IndexDocument { - let mut detector = LanguageDetector::new(); + let mut builder = DocumentBuilder { + document: IndexDocument::new(SearchIndex::Email) + .with_account_id(account_id) + .with_document_id(document_id), + detector: LanguageDetector::new(), + attachments: AttachmentText::new(extract_limits), + index_fields, + }; + let raw = self.raw_message(Some(headers), blob); + let source = PartSource::Raw(raw); + let root = self.root(); let mut language = Language::Unknown; - let message_contents = &self.contents[0]; - let mut document = IndexDocument::new(SearchIndex::Email) - .with_account_id(account_id) - .with_document_id(document_id); - let raw_message = ChainedBytes::new(self.raw_headers.as_ref()).with_last( - raw_message - .get(self.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::ReceivedAt)) - { - document.index_unsigned( - SearchField::Email(EmailSearchField::ReceivedAt), - self.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, - ); - } - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::Size)) - { - document.index_unsigned( - SearchField::Email(EmailSearchField::Size), - raw_message.len() as u32, - ); - } - - for (part_id, part) in message_contents - .parts - .iter() - .take(MAX_MESSAGE_PARTS) - .enumerate() - { + for part in root.parts().take(MAX_MESSAGE_PARTS) { let part_language = part.language().unwrap_or(language); - if part_id == 0 { + if part.id() == 0 { language = part_language; - - for header in part.headers.iter().rev() { - match &header.name { - ArchivedMetadataHeaderName::From => { - if index_fields.is_empty() - || index_fields - .contains(&SearchField::Email(EmailSearchField::From)) - { - header.value.visit_addresses(|_, value| { - document.index_text( - SearchField::Email(EmailSearchField::From), - value, - Language::None, - ); - }); - } - } - ArchivedMetadataHeaderName::To => { - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::To)) - { - header.value.visit_addresses(|_, value| { - document.index_text( - SearchField::Email(EmailSearchField::To), - value, - Language::None, - ); - }); - } - } - ArchivedMetadataHeaderName::Cc => { - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::Cc)) - { - header.value.visit_addresses(|_, value| { - document.index_text( - SearchField::Email(EmailSearchField::Cc), - value, - Language::None, - ); - }); - } - } - ArchivedMetadataHeaderName::Bcc => { - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::Bcc)) - { - header.value.visit_addresses(|_, value| { - document.index_text( - SearchField::Email(EmailSearchField::Bcc), - value, - Language::None, - ); - }); - } - } - ArchivedMetadataHeaderName::Subject => { - if (index_fields.is_empty() - || index_fields - .contains(&SearchField::Email(EmailSearchField::Subject))) - && let Some(subject) = header.value.as_text() - { - let subject = thread_name(subject); - - if part_language.is_unknown() { - detector.detect(subject, MIN_LANGUAGE_SCORE); - } - - document.index_text( - SearchField::Email(EmailSearchField::Subject), - subject, - part_language, - ); - } - } - ArchivedMetadataHeaderName::Date => { - if (index_fields.is_empty() - || index_fields - .contains(&SearchField::Email(EmailSearchField::SentAt))) - && let Some(date) = header.value.as_datetime() - { - document.index_integer( - SearchField::Email(EmailSearchField::SentAt), - DateTime::from(date).to_timestamp(), - ); - } - } - _ => { - #[cfg(not(feature = "test_mode"))] - let index_headers = index_fields - .contains(&SearchField::Email(EmailSearchField::Headers)); - - #[cfg(feature = "test_mode")] - let index_headers = true; - - if index_headers { - let mut value = String::new(); - match &header.value { - ArchivedMetadataHeaderValue::AddressList(_) - | ArchivedMetadataHeaderValue::AddressGroup(_) => { - header.value.visit_addresses(|_, addr| { - if !value.is_empty() { - value.push(' '); - } - value.push_str(addr); - }); - } - ArchivedMetadataHeaderValue::Text(_) - | ArchivedMetadataHeaderValue::TextList(_) => { - header.value.visit_text(|text| { - if !value.is_empty() { - value.push(' '); - } - value.push_str(text); - }); - } - _ => { - if let Some(raw_value) = - raw_message.get(header.value_range()) - { - let raw_value = std::str::from_utf8(raw_value.as_ref()) - .unwrap_or_default(); - - for word in - WordTokenizer::new(raw_value, MAX_TOKEN_LENGTH) - { - if !value.is_empty() { - value.push(' '); - } - value.push_str(word.word.as_ref()); - } - } - } - } - - document.insert_key_value( - EmailSearchField::Headers, - header.name.as_str(), - value, - ); - } - } - } - } + builder.index_envelope(root.envelope().addresses_of_interest()); + builder.index_root_headers(part.headers(), headers, part_language); } - let part_id = part_id as u16; - match &part.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - let text = match (part.decode_contents(&raw_message), &part.body) { - (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text, - (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => { - html_to_text(html.as_ref()).into() - } - _ => unreachable!(), + match part.kind() { + PartKind::Text | PartKind::Html => { + let field = if part.flags().contains(PartFlags::IN_TEXT_BODY) + || part.flags().contains(PartFlags::IN_HTML_BODY) + { + EmailSearchField::Body + } else { + EmailSearchField::Attachment }; - - if message_contents.is_html_part(part_id) - || message_contents.is_text_part(part_id) - { - if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::Body)) - { - if part_language.is_unknown() { - detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); - } - - document.index_text( - SearchField::Email(EmailSearchField::Body), - text.as_ref(), - part_language, - ); - } - } else if index_fields.is_empty() - || index_fields.contains(&SearchField::Email(EmailSearchField::Attachment)) - { - if part_language.is_unknown() { - detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); - } - - document.index_text( - SearchField::Email(EmailSearchField::Attachment), - text.as_ref(), - part_language, - ); + if !builder.is_enabled(field.clone()) { + continue; } + let Some(text) = part.plain_text(&source) else { + continue; + }; + builder.index_part_text(part, &text, part_language, field); } - ArchivedMetadataPartType::Message(nested_message_id) - if index_fields.is_empty() - || index_fields - .contains(&SearchField::Email(EmailSearchField::Attachment)) => - { - let nested_message = self.message_id(*nested_message_id); - let nested_message_language = nested_message - .root_part() - .language() - .unwrap_or(Language::Unknown); - if let Some(ArchivedMetadataHeaderValue::Text(subject)) = nested_message - .root_part() - .header_value(&MetadataHeaderName::Subject) - { - if nested_message_language.is_unknown() { - detector.detect(subject.as_ref(), MIN_LANGUAGE_SCORE); - } - - document.index_text( - SearchField::Email(EmailSearchField::Attachment), - subject.as_ref(), - nested_message_language, - ); + PartKind::Binary if builder.is_enabled(EmailSearchField::Attachment) => { + builder.attachments.index_binary( + part, + &source, + &mut builder.document, + &mut builder.detector, + part_language, + ); + } + PartKind::Message if builder.is_enabled(EmailSearchField::Attachment) => { + let Some(nested) = part.nested() else { + continue; + }; + let nested_language = + nested.root_part().language().unwrap_or(Language::Unknown); + let nested_source = self.source(nested, raw); + if let Some(subject) = nested.envelope().subject() { + builder.index_text(EmailSearchField::Attachment, subject, nested_language); } - - for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) { - let language = sub_part.language().unwrap_or(nested_message_language); - match &sub_part.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - let text = match ( - sub_part.decode_contents(&raw_message), - &sub_part.body, - ) { - ( - DecodedPartContent::Text(text), - ArchivedMetadataPartType::Text, - ) => text, - ( - DecodedPartContent::Text(html), - ArchivedMetadataPartType::Html, - ) => html_to_text(html.as_ref()).into(), - _ => unreachable!(), - }; - - if language.is_unknown() { - detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); + let Some(nested_source) = nested_source else { + continue; + }; + for sub_part in nested.parts().take(MAX_MESSAGE_PARTS) { + let language = sub_part.language().unwrap_or(nested_language); + match sub_part.kind() { + PartKind::Text | PartKind::Html => { + if let Some(text) = sub_part.plain_text(&nested_source) { + builder.index_part_text( + sub_part, + &text, + language, + EmailSearchField::Attachment, + ); } - - document.index_text( - SearchField::Email(EmailSearchField::Attachment), - text.as_ref(), - language, - ); } + PartKind::Binary => builder.attachments.index_binary( + sub_part, + &nested_source, + &mut builder.document, + &mut builder.detector, + language, + ), _ => (), } } @@ -325,20 +138,229 @@ impl ArchivedMessageMetadata { } } - #[cfg(not(feature = "test_mode"))] - document.set_unknown_language( - detector + #[cfg(not(stalwart_tests))] + builder.document.set_unknown_language( + builder + .detector .most_frequent_language() .unwrap_or(default_language), ); - #[cfg(feature = "test_mode")] - document.set_unknown_language(default_language); - - document.index_bool( - EmailSearchField::HasAttachment, - self.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT != 0, - ); - document + #[cfg(stalwart_tests)] + builder.document.set_unknown_language(default_language); + builder.document } } + +impl DocumentBuilder<'_> { + fn is_enabled(&self, field: EmailSearchField) -> bool { + self.index_fields.is_empty() || self.index_fields.contains(&SearchField::Email(field)) + } + + fn index_text(&mut self, field: EmailSearchField, text: &str, language: Language) { + if language.is_unknown() { + self.detector.detect(text, MIN_LANGUAGE_SCORE); + } + self.document + .index_text(SearchField::Email(field), text, language); + } + + fn index_part_text( + &mut self, + part: PartView<'_>, + text: &str, + language: Language, + field: EmailSearchField, + ) { + if self.is_enabled(field.clone()) + && !self.attachments.index_rtf( + part, + text, + &mut self.document, + &mut self.detector, + language, + field.clone(), + ) + { + self.index_text(field, text, language); + } + } + + fn index_envelope<'a>(&mut self, fields: [(EmailSearchField, Addresses<'a>); 4]) { + for (field, addresses) in fields { + if self.is_enabled(field.clone()) { + let search_field = SearchField::Email(field); + addresses.visit_text(|text| { + self.document + .index_text(search_field.clone(), text, Language::None); + }); + } + } + } + + fn indexes_headers(&self) -> bool { + #[cfg(not(stalwart_tests))] + let index_headers = self + .index_fields + .contains(&SearchField::Email(EmailSearchField::Headers)); + + #[cfg(stalwart_tests)] + let index_headers = true; + + index_headers + } + + fn index_root_headers(&mut self, root: HeaderList<'_>, headers: &[u8], language: Language) { + let index_headers = self.indexes_headers(); + let index_subject = self.is_enabled(EmailSearchField::Subject); + for header in root.iter().rev() { + match header.id() { + id @ (HeaderId::FROM | HeaderId::TO | HeaderId::CC | HeaderId::BCC) => { + if index_headers && let Some(name) = id.as_str() { + self.document + .insert_key_value(EmailSearchField::Headers, name, ""); + } + } + HeaderId::SUBJECT => { + if index_headers { + self.document + .insert_key_value(EmailSearchField::Headers, "subject", ""); + } + if index_subject + && let Some(raw) = headers.get(header.value_range()) + && let Some(subject) = HeaderForm::Text.parse(raw).value().as_text() + { + self.index_text( + EmailSearchField::Subject, + &search_subject(subject), + language, + ); + } + } + id if index_headers => { + let Some(field) = headers.get(header.field_range()) else { + continue; + }; + let name = match id.as_str() { + Some(name) => Cow::Borrowed(name), + None => String::from_utf8_lossy(header.raw_name_in(field)), + }; + let value = id.key_value(header.raw_value_in(field)); + self.document + .insert_key_value(EmailSearchField::Headers, name, value); + } + _ => {} + } + } + } +} + +impl HeaderId { + fn key_value_form(self) -> Option { + match self { + HeaderId::REPLY_TO | HeaderId::SENDER | HeaderId::LIST_ID => { + Some(HeaderForm::Addresses) + } + HeaderId::COMMENTS + | HeaderId::CONTENT_DESCRIPTION + | HeaderId::CONTENT_LOCATION + | HeaderId::CONTENT_TRANSFER_ENCODING => Some(HeaderForm::Text), + HeaderId::IN_REPLY_TO + | HeaderId::MESSAGE_ID + | HeaderId::REFERENCES + | HeaderId::RESENT_MESSAGE_ID + | HeaderId::CONTENT_ID => Some(HeaderForm::MessageIds), + HeaderId::KEYWORDS | HeaderId::CONTENT_LANGUAGE => Some(HeaderForm::CommaList), + _ => None, + } + } + + fn key_value(self, raw: &[u8]) -> String { + let mut value = String::with_capacity(raw.len()); + let mut push = |text: &str| { + if !value.is_empty() { + value.push(' '); + } + value.push_str(text); + }; + if let Some(form) = self.key_value_form() { + let parsed = form.parse(raw); + match parsed.value() { + parsed @ HeaderValue::Address(_) => { + parsed.visit_addresses(|_, text| push(text)); + return value; + } + parsed @ (HeaderValue::Text(_) | HeaderValue::TextList(_)) => { + parsed.visit_text(&mut push); + return value; + } + _ => {} + } + } + for word in WordTokenizer::new(from_utf8(raw).unwrap_or_default(), MAX_TOKEN_LENGTH) { + push(word.word.as_ref()); + } + value + } +} + +impl<'a> PartView<'a> { + pub fn plain_text<'x>(&self, source: &PartSource<'x>) -> Option> { + let text = self.text(source)?.text; + Some(if self.kind() == PartKind::Html { + Cow::Owned(html_to_text(&text)) + } else { + text + }) + } +} + +impl<'a> Addresses<'a> { + fn visit_text(&self, mut visitor: impl FnMut(&'a str)) { + for item in self.iter() { + match item { + AddressItem::Mailbox(mailbox) => { + mailbox + .name + .into_iter() + .chain(mailbox.address) + .for_each(&mut visitor); + } + AddressItem::Group(group) => { + group.name.into_iter().for_each(&mut visitor); + for mailbox in group.members.mailboxes() { + mailbox + .name + .into_iter() + .chain(mailbox.address) + .for_each(&mut visitor); + } + } + } + } + } +} + +impl<'a> EnvelopeView<'a> { + fn addresses_of_interest(&self) -> [(EmailSearchField, Addresses<'a>); 4] { + [ + ( + EmailSearchField::From, + self.addresses(AddressHeader::From, Occurrence::All), + ), + ( + EmailSearchField::To, + self.addresses(AddressHeader::To, Occurrence::All), + ), + ( + EmailSearchField::Cc, + self.addresses(AddressHeader::Cc, Occurrence::All), + ), + ( + EmailSearchField::Bcc, + self.addresses(AddressHeader::Bcc, Occurrence::All), + ), + ] + } +} + diff --git a/crates/email/src/message/index/subject.rs b/crates/email/src/message/index/subject.rs new file mode 100644 index 00000000..5532ab9b --- /dev/null +++ b/crates/email/src/message/index/subject.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use mail_parser::thread_name; +use std::{borrow::Cow, iter::once}; + +pub fn search_subject(subject: &str) -> Cow<'_, str> { + let name = thread_name(subject); + let Some(offset) = name.as_ptr().addr().checked_sub(subject.as_ptr().addr()) else { + return Cow::Borrowed(name); + }; + let Some((prefix, _)) = subject.split_at_checked(offset) else { + return Cow::Borrowed(name); + }; + let Some((markers, tail)) = prefix.rsplit_once(':') else { + return Cow::Borrowed(subject.get(..offset + name.len()).map_or(name, str::trim)); + }; + let mut kept = markers + .split(':') + .filter_map(|segment| { + segment + .trim_end() + .rsplit_once(char::is_whitespace) + .map(|(words, _)| words.trim()) + }) + .chain(once(tail.trim())) + .filter(|words| !words.is_empty()) + .peekable(); + if kept.peek().is_none() { + return Cow::Borrowed(name); + } + let mut text = String::with_capacity(subject.len()); + for words in kept { + text.push_str(words); + text.push(' '); + } + text.push_str(name); + Cow::Owned(text) +} + +pub fn search_subject_query(text: &str) -> Cow<'_, str> { + match search_subject(text) { + subject if subject.is_empty() => Cow::Borrowed(text), + subject => subject, + } +} + diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 2d2bc6ce..1cc63b47 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -6,53 +6,60 @@ use super::crypto::{EncryptMessage, EncryptMessageError}; use crate::{ - cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, - mailbox::{INBOX_ID, JUNK_ID, SENT_ID, TRASH_ID, UidMailbox}, + mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, message::{ crypto::EncryptionFlags, - index::{IndexMessage, extractors::VisitText}, - metadata::{MessageData, MessageMetadata}, + delivery::SenderAuthentication, + index::IndexMessage, + ingest_metadata::IngestMetadata, + messagedata::{EmailMessageData, MessageData, PendingMessageData}, + metadata::{AddressHeader, ExtraHeaders, HeaderId, MetadataStructure, Occurrence}, + thread::ThreadFields, }, }; -use common::{Server, auth::AccessToken}; +use common::{ + MAX_RECEIVED_AT, MessageUid, Server, auth::AccessToken, + storage::metadata::PrivateMetadataCommit, +}; +use compact_str::{CompactString, ToCompactString}; use groupware::{ calendar::itip::{ItipIngest, ItipIngestError}, scheduling::{ItipError, ItipMessages}, }; -use mail_parser::{ - Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, - parsers::fields::thread::thread_name, -}; +use mail_parser::{Message, MessageParser, thread_name}; +use recheck::PendingThread; use registry::{ schema::{ - enums::IndexDocumentType, + enums::StorageQuota, prelude::{ObjectType, Permission, Property}, - structs::{SpamTrainingSample, Task, TaskIndexDocument, TaskMergeThreads, TaskStatus}, + structs::{SpamTrainingSample, Task, TaskMergeThreads, TaskStatus}, }, types::{EnumImpl, ObjectImpl, datetime::UTCDateTime, id::ObjectId, map::Map}, }; use std::future::Future; -use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; -use store::write::{AlignedBytes, Archive, RegistryClass}; +use std::{borrow::Cow, cmp::Ordering, time::Instant}; use store::{ - IndexKeyPrefix, IterateParams, SerializeInfallible, U32_LEN, ValueKey, + Deserialize, IterateParams, SerializeInfallible, U32_LEN, U128_LEN, ValueKey, ahash::AHashMap, + roaring::RoaringBitmap, write::{ - AssignedId, AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass, - key::DeserializeBigEndian, now, + AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, Patch, PatchSource, + PendingId, QueueDocumentId, SearchIndex, SizedSetValue, Slot, SlotRange, ValueClass, + blob::BlobLinkValue, key::DeserializeBigEndian, now, }, }; +use store::{write::RegistryClass, xxhash_rust::xxh3::xxh3_128}; +use tinyvec::TinyVec; use trc::{AddContext, MessageIngestEvent, SpamEvent}; use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, - field::{ContactField, EmailField, MailboxField}, + field::{ContactField, EmailField}, id::Id, keyword::Keyword, - special_use::SpecialUse, }; -use utils::{cheeky_hash::CheekyHash, sanitize_email, snowflake::SnowflakeIdGenerator}; +use utils::{hash128::Hash128, sanitize_email, snowflake::SnowflakeIdGenerator}; #[derive(Default)] pub struct IngestedEmail { @@ -62,6 +69,28 @@ pub struct IngestedEmail { pub blob_id: BlobId, pub size: usize, pub imap_uids: Vec, + pub thread_recheck: Option, +} + +pub enum PendingIngest { + Queued(PendingEmail), + Duplicate(IngestedEmail), +} + +pub struct PendingEmail { + account_id: u32, + document_slot: Slot, + uid_slots: SlotRange, + thread_id: PendingId, + blob_hash: BlobHash, + size: u64, + thread_recheck: Option, + private_commit: PrivateMetadataCommit, + event: MessageIngestEvent, + mailbox_ids: Vec, + message_id: Option, + session_id: u64, + start_time: Instant, } pub struct IngestEmail<'x> { @@ -73,6 +102,7 @@ pub struct IngestEmail<'x> { pub keywords: Vec, pub received_at: Option, pub source: IngestSource<'x>, + pub metadata: Option>, pub session_id: u64, } @@ -80,7 +110,8 @@ pub struct IngestEmail<'x> { pub enum IngestSource<'x> { Smtp { deliver_to: &'x str, - is_sender_authenticated: bool, + envelope_from: &'x str, + sender_authentication: SenderAuthentication, is_spam: bool, }, Jmap { @@ -97,24 +128,28 @@ pub trait EmailIngest: Sync + Send { &self, params: IngestEmail, ) -> impl Future> + Send; + fn email_ingest_into( + &self, + params: &mut IngestEmail, + batch: &mut BatchBuilder, + group_thread: Option, + ) -> impl Future> + Send; + fn email_ingest_finish( + &self, + pending: PendingEmail, + assigned_ids: &AssignedIds, + ) -> impl Future + Send; fn find_thread_id( &self, account_id: u32, thread_name: &str, - message_ids: &[CheekyHash], + message_ids: &[u128], ) -> impl Future> + Send; - fn assign_email_ids( - &self, - account_id: u32, - mailbox_ids: impl IntoIterator + Sync + Send, - generate_email_id: bool, - ) -> impl Future + 'static>> + Send; - fn add_account_spam_sample( + fn add_account_spam_samples( &self, batch: &mut BatchBuilder, account_id: u32, - document_id: u32, - is_spam: bool, + samples: SpamSamples, span_id: u64, ) -> impl Future> + Send; @@ -124,6 +159,7 @@ pub trait EmailIngest: Sync + Send { account_id: u32, batch: &mut BatchBuilder, hash: BlobHash, + size: u64, from: String, subject: String, is_spam: bool, @@ -132,131 +168,204 @@ pub trait EmailIngest: Sync + Send { ); } +#[derive(Default)] +pub struct SpamSamples { + documents: RoaringBitmap, + spam: RoaringBitmap, + prepared: Vec, +} + +pub struct SpamSample { + hash: BlobHash, + size: u64, + from: String, + subject: String, + is_spam: bool, +} + +impl SpamSample { + pub fn from_metadata(bytes: &[u8], is_spam: bool) -> trc::Result { + let structure = MetadataStructure::deserialize(bytes)?; + let metadata = structure.unarchive().caused_by(trc::location!())?; + let envelope = metadata.root().envelope(); + Ok(SpamSample { + hash: metadata.blob_hash(), + size: metadata.blob_size() as u64, + from: envelope + .addresses(AddressHeader::From, Occurrence::Last) + .first() + .and_then(|mailbox| mailbox.address) + .unwrap_or_default() + .to_string(), + subject: thread_name(envelope.subject().unwrap_or_default()).to_string(), + is_spam, + }) + } +} + +impl SpamSamples { + pub fn add(&mut self, document_id: u32, is_spam: bool) { + self.documents.insert(document_id); + if is_spam { + self.spam.insert(document_id); + } else { + self.spam.remove(document_id); + } + } + + pub fn add_sample(&mut self, sample: SpamSample) { + self.prepared.push(sample); + } + + pub fn is_empty(&self) -> bool { + self.documents.is_empty() && self.prepared.is_empty() + } +} + pub struct ThreadResult { pub thread_id: Option, - pub thread_hash: CheekyHash, + pub thread_hash: Hash128, pub merge_ids: Vec, pub duplicate_ids: Vec, + pub matched_ids: Vec, + pub bucket_full: bool, } impl EmailIngest for Server { - #[allow(clippy::blocks_in_conditions)] async fn email_ingest(&self, mut params: IngestEmail<'_>) -> trc::Result { + let mut batch = BatchBuilder::new(); + let pending = match self + .email_ingest_into(&mut params, &mut batch, None) + .await? + { + PendingIngest::Queued(pending) => pending, + PendingIngest::Duplicate(email) => return Ok(email), + }; + let queues = batch.queue_notify(); + let assigned_ids = self + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + self.inner.mark_caches_stale_from(&assigned_ids); + let email = self.email_ingest_finish(pending, &assigned_ids).await; + self.notify_queues(queues).await; + Ok(email) + } + + #[allow(clippy::blocks_in_conditions)] + async fn email_ingest_into( + &self, + params: &mut IngestEmail<'_>, + batch: &mut BatchBuilder, + group_thread: Option, + ) -> trc::Result { // Check quota let start_time = Instant::now(); let account_id = params.access_token.account_id(); - let tenant_id = params.access_token.tenant_id(); let mut raw_message_len = params.raw_message.len() as u64; let account = self.account(account_id).await.caused_by(trc::location!())?; - self.has_available_quota(&account, raw_message_len) - .await - .caused_by(trc::location!())?; + let limits = &self.core.email.limits; + if matches!( + params.source, + IngestSource::Smtp { .. } | IngestSource::Restore + ) { + params.mailbox_ids.truncate(limits.mailboxes_per_email); + if !params.keywords.is_empty() { + params.keywords = Keyword::unique( + std::mem::take(&mut params.keywords) + .into_iter() + .filter(|keyword| limits.is_keyword_allowed(keyword)), + ); + params.keywords.truncate(limits.keywords_per_email); + } + } else if let Err(err) = limits.validate_email(params.mailbox_ids.len(), ¶ms.keywords) { + return Err( + trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) + .ctx(trc::Key::Code, 550) + .ctx(trc::Key::Reason, err.to_string()), + ); + } + let mut metadata = params.metadata.take(); + if let Some(metadata) = metadata.as_deref_mut() { + metadata.shared = metadata + .shared + .take() + .filter(|container| container.kinds().intersects(MessageData::TRACKED_METADATA)); + } // Parse message let mut raw_message = Cow::from(params.raw_message); - let mut message = params.message.ok_or_else(|| { + let mut message = params.message.take().ok_or_else(|| { trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) .ctx(trc::Key::Code, 550) .ctx(trc::Key::Reason, "Failed to parse e-mail message.") })?; // Obtain message references and thread name - let mut message_id = None; - let mut message_ids = Vec::new(); - let thread_result = { - let mut subject = ""; - for header in message.root_part().headers().iter().rev() { - match &header.name { - HeaderName::MessageId => header.value.visit_text(|id| { - if !id.is_empty() { - if message_id.is_none() { - message_id = id.to_string().into(); - } - message_ids.push(CheekyHash::new(id.as_bytes())); - } - }), - HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId => { - header.value.visit_text(|id| { - if !id.is_empty() { - message_ids.push(CheekyHash::new(id.as_bytes())); - } - }); - } - HeaderName::Subject if subject.is_empty() => { - subject = thread_name(match &header.value { - HeaderValue::Text(text) => text.as_ref(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().as_ref() - } - _ => "", - }); - } - _ => (), - } + let fields = ThreadFields::scan(&message); + let mut thread_result = self + .find_thread_id(account_id, fields.subject, &fields.message_ids) + .await?; + let group_slot = match group_thread { + Some(PendingId::Slot(slot)) if thread_result.thread_id.is_none() => Some(slot), + Some(PendingId::Assigned(thread_id)) if thread_result.thread_id.is_none() => { + thread_result.thread_id = Some(thread_id); + None } - - message_ids.sort_unstable(); - message_ids.dedup(); - - self.find_thread_id(account_id, subject, &message_ids) - .await? + _ => None, }; + let ThreadFields { + message_id, + sent_at, + message_ids, + .. + } = fields; // Skip duplicate messages for SMTP ingestion - if !thread_result.duplicate_ids.is_empty() && params.source.is_smtp() { - // Fetch cached messages - let cache = self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; + if !thread_result.duplicate_ids.is_empty() + && params.source.is_smtp() + && Box::pin(checks::has_duplicate_in( + self, + account_id, + &thread_result.duplicate_ids, + params.mailbox_ids.first().copied().unwrap_or(INBOX_ID), + )) + .await + .caused_by(trc::location!())? + { + trc::event!( + MessageIngest(MessageIngestEvent::Duplicate), + SpanId = params.session_id, + AccountId = account_id, + MessageId = message_id, + ); - // Skip duplicate messages - let target_mailbox_id = params.mailbox_ids.first().copied().unwrap_or(INBOX_ID); - if cache - .in_mailboxes(&[target_mailbox_id, JUNK_ID]) - .any(|m| thread_result.duplicate_ids.contains(&m.document_id)) - { - trc::event!( - MessageIngest(MessageIngestEvent::Duplicate), - SpanId = params.session_id, - AccountId = account_id, - MessageId = message_id, - ); - - return Ok(IngestedEmail { - document_id: 0, - thread_id: 0, - change_id: u64::MAX, - blob_id: BlobId::default(), - imap_uids: Vec::new(), - size: 0, - }); - } + return Ok(PendingIngest::Duplicate(IngestedEmail { + document_id: 0, + thread_id: 0, + change_id: u64::MAX, + blob_id: BlobId::default(), + imap_uids: Vec::new(), + size: 0, + thread_recheck: None, + })); } // Spam classification and training let mut train_spam = None; - let mut extra_headers = String::new(); - let mut extra_headers_parsed = Vec::new(); + let mut extra_headers = ExtraHeaders::default(); let mut itip_messages = Vec::new(); let is_spam = match params.source { IngestSource::Smtp { deliver_to, - is_sender_authenticated, + envelope_from, + sender_authentication, mut is_spam, } => { // Add delivered to header if self.core.smtp.session.data.add_delivered_to { - extra_headers = format!("Delivered-To: {deliver_to}\r\n"); - extra_headers_parsed.push(Header { - name: HeaderName::DeliveredTo, - value: HeaderValue::Text(deliver_to.into()), - offset_field: 0, - offset_start: 13, - offset_end: extra_headers.len() as u32, - }); + extra_headers.push(HeaderId::DELIVERED_TO, deliver_to); } // Spam training on confirmed false positives @@ -272,7 +381,7 @@ impl EmailIngest for Server { .and_then(|s| s.address()) .and_then(sanitize_email) && sender != deliver_to - && is_sender_authenticated + && sender_authentication.is_authenticated() && self .document_exists( account_id, @@ -300,60 +409,39 @@ impl EmailIngest for Server { if is_spam && self.core.spam.trusted_reply && let Some(thread_id) = thread_result.thread_id + && Box::pin(checks::is_trusted_reply( + self, + account_id, + thread_id, + &thread_result.matched_ids, + )) + .await + .caused_by(trc::location!())? { - let cache = self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; - let sent_folder_id = cache - .mailbox_by_role(&SpecialUse::Sent) - .map(|m| m.document_id) - .unwrap_or(SENT_ID); - - if cache - .in_thread(thread_id) - .any(|m| m.mailboxes.iter().any(|mb| mb.mailbox_id == sent_folder_id)) + is_spam = false; + if self + .core + .spam + .classifier + .as_ref() + .is_some_and(|c| c.auto_learn_reply_ham) { - is_spam = false; - if self - .core - .spam - .classifier - .as_ref() - .is_some_and(|c| c.auto_learn_reply_ham) - { - train_spam = Some(false); - } - overridden = Some("trusted-reply"); + train_spam = Some(false); } + overridden = Some("trusted-reply"); } // Add Spam-Status header - const HEADER: &str = "X-Spam-Status"; - let offset_field = extra_headers.len(); - let offset_start = offset_field + HEADER.len() + 1; let result = if is_spam { "Yes" } else { "No" }; if let Some(reason) = overridden { - let _ = write!( - &mut extra_headers, - "{HEADER}: {result}, reason={reason}\r\n", + extra_headers.push( + HeaderId::X_SPAM_STATUS, + &format!("{result}, reason={reason}"), ); } else { - let _ = write!(&mut extra_headers, "{HEADER}: {result}\r\n",); + extra_headers.push(HeaderId::X_SPAM_STATUS, result); } - extra_headers_parsed.push(Header { - name: HeaderName::Other(HEADER.into()), - value: HeaderValue::Text( - extra_headers[offset_start + 1..extra_headers.len() - 2] - .to_string() - .into(), - ), - offset_field: offset_field as u32, - offset_start: offset_start as u32, - offset_end: extra_headers.len() as u32, - }); - if is_spam && params.mailbox_ids == [INBOX_ID] { params.mailbox_ids[0] = JUNK_ID; params.keywords.push(Keyword::Junk); @@ -363,41 +451,49 @@ impl EmailIngest for Server { // iMIP processing if self.core.groupware.itip_enabled && !is_spam - && is_sender_authenticated + && sender_authentication.is_authenticated() && params .access_token .has_permission(Permission::CalendarSchedulingReceive) { - let account_info = self - .build_account_info(account.clone()) - .await - .caused_by(trc::location!())?; + let mut account_info = None; let mut sender = None; - for part in &message.parts { + for part in message.root().parts() { if part.content_type().is_some_and(|ct| { ct.ctype().eq_ignore_ascii_case("text") && ct .subtype() .is_some_and(|st| st.eq_ignore_ascii_case("calendar")) && ct.has_attribute("method") - }) && let Some(itip_message) = part.text_contents() + }) && let Some(itip_message) = part.text() { if itip_message.len() < self.core.groupware.itip_inbound_max_ical_size { if let Some(sender) = sender.get_or_insert_with(|| { - message - .from() - .and_then(|s| s.first()) - .and_then(|s| s.address()) - .and_then(sanitize_email) + if sender_authentication == SenderAuthentication::Submission { + sanitize_email(envelope_from) + } else { + message + .from() + .and_then(|s| s.first()) + .and_then(|s| s.address()) + .and_then(sanitize_email) + } }) { - match self - .itip_ingest( - &account_info, - sender, - deliver_to, - itip_message, - ) - .await + let account_info = match &account_info { + Some(account_info) => account_info, + None => account_info.insert( + self.build_account_info(account.clone()) + .await + .caused_by(trc::location!())?, + ), + }; + match Box::pin(self.itip_ingest( + account_info, + sender, + deliver_to, + &itip_message, + )) + .await { Ok(message) => { if let Some(message) = message { @@ -406,7 +502,7 @@ impl EmailIngest for Server { trc::event!( Calendar(trc::CalendarEvent::ItipMessageReceived), SpanId = params.session_id, - From = sender.to_string(), + From = CompactString::from(&*sender), AccountId = account_id, ); } @@ -420,9 +516,9 @@ impl EmailIngest for Server { trc::CalendarEvent::ItipMessageError ), SpanId = params.session_id, - From = sender.to_string(), + From = CompactString::from(&*sender), AccountId = account_id, - Details = err.to_string(), + Details = err.to_compact_string(), ) } } @@ -440,7 +536,7 @@ impl EmailIngest for Server { .from() .and_then(|a| a.first()) .and_then(|a| a.address()) - .map(|a| a.to_string()), + .map(CompactString::from), AccountId = account_id, Details = "iMIP message too large", Limit = self.core.groupware.itip_inbound_max_ical_size, @@ -474,18 +570,12 @@ impl EmailIngest for Server { params.received_at = message .root_part() .headers() - .iter() - .filter_map(|header| { - if let (HeaderName::Received, HeaderValue::Received(received)) = - (&header.name, &header.value) - { - received - .date - .filter(|dt| dt.is_valid()) - .map(|dt| dt.to_timestamp() as u64) - } else { - None - } + .all_received() + .filter_map(|received| { + received + .date() + .filter(|dt| dt.is_valid()) + .map(|dt| dt.to_timestamp() as u64) }) .max(); } @@ -507,7 +597,7 @@ impl EmailIngest for Server { }; let is_encrypted = if do_encrypt && !message.is_encrypted() - && let Some(encrypt_keys) = &account.encryption_key + && let Some(encrypt_keys) = account.encryption_key() { match message.encrypt(encrypt_keys, account.flags).await { Ok(new_raw_message) => { @@ -529,22 +619,6 @@ impl EmailIngest for Server { train_spam = None; } - // Remove contents from parsed message - for part in &mut message.parts { - match &mut part.body { - PartType::Text(txt) | PartType::Html(txt) => { - *txt = Cow::from(""); - } - PartType::Binary(bin) | PartType::InlineBinary(bin) => { - *bin = Cow::from(&[][..]); - } - PartType::Message(_) => { - part.body = PartType::Binary(Cow::from(&[][..])); - } - PartType::Multipart(_) => (), - } - } - true } Err(EncryptMessageError::Error(err)) => { @@ -565,57 +639,88 @@ impl EmailIngest for Server { let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash { (blob_hash.clone(), None) } else { - self.put_temporary_blob(account_id, raw_message.as_ref(), 60) - .await - .map(|(hash, op)| (hash, Some(op))) - .caused_by(trc::location!())? - }; - - // Assign IMAP UIDs - let mut mailbox_ids = Vec::with_capacity(params.mailbox_ids.len()); - let mut imap_uids = Vec::with_capacity(params.mailbox_ids.len()); - let mut ids = self - .assign_email_ids(account_id, params.mailbox_ids.iter().copied(), true) + self.hold_blob( + account_id, + BlobHash::generate(raw_message.as_ref()), + raw_message.as_ref(), + 60, + ) .await - .caused_by(trc::location!())?; - let document_id = ids.next().unwrap(); - for (uid, mailbox_id) in ids.zip(params.mailbox_ids.iter().copied()) { - mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); - imap_uids.push(uid); - } + .map(|(hash, hold)| (hash, Some(hold))) + .caused_by(trc::location!())? + }; // Build write batch - let mut batch = BatchBuilder::new(); - let mailbox_ids_event = mailbox_ids - .iter() - .map(|m| trc::Value::from(m.mailbox_id)) - .collect::>(); batch.with_account_id(account_id); + self.limit_disk_quota(batch, &account); + self.limit_object_quota(batch, &account, StorageQuota::MaxEmails); + + // Reserve a document id and one IMAP UID per target mailbox + let document_slot = batch.reserve_document_id(account_id, Collection::Email); + let uid_slots = batch.reserve_uids(account_id, params.mailbox_ids.iter().copied()); + let mailbox_ids: TinyVec<[MessageUid; 2]> = params + .mailbox_ids + .iter() + .copied() + .map(MessageUid::new_unassigned) + .collect(); // Determine thread id - let thread_id = if let Some(thread_id) = thread_result.thread_id { - thread_id - } else { - batch - .with_collection(Collection::Thread) - .with_document(document_id) - .log_container_insert(SyncCollection::Thread); - document_id + let thread_slot = match (thread_result.thread_id, group_slot) { + (_, Some(slot)) => Some(slot), + (None, None) => { + batch + .with_collection(Collection::Thread) + .create_document(document_slot) + .log_container_insert(SyncCollection::Thread); + Some(document_slot) + } + (Some(_), None) => None, }; - let data = MessageData { - mailboxes: mailbox_ids.into_boxed_slice(), - keywords: params.keywords.into_boxed_slice(), - thread_id, - size: (message.raw_message.len() + extra_headers.len()) as u32, + let mut keywords = 0; + let mut keywords_extra = Vec::new(); + let received_at = params.received_at.unwrap_or_else(now).min(MAX_RECEIVED_AT); + for keyword in std::mem::take(&mut params.keywords) { + match keyword.into_id() { + Ok(id) => keywords |= 1 << id, + Err(name) => keywords_extra.push(name), + } + } + let mut data = PendingMessageData { + data: MessageData { + mailboxes: mailbox_ids, + keywords, + keywords_extra, + thread_id: thread_result.thread_id.unwrap_or_default(), + size: (message.raw().len() + extra_headers.len()) as u32, + received_at, + sent_at: MessageData::sent_at_offset(sent_at, received_at), + change_id: 0, + }, + uid_slots, + thread_slot, + change_id: None, }; + if let Some(container) = metadata + .as_deref() + .and_then(|metadata| metadata.shared.as_ref()) + { + data.data.set_metadata_kinds(container.kinds()); + } + let thread_ref = data.thread_id(); // Request spam training if let Some(learn_spam) = train_spam { self.add_spam_sample( account_id, - &mut batch, + batch, params.blob_hash.unwrap_or(&blob_hash).clone(), + if params.blob_hash.is_some() { + params.raw_message.len() + } else { + raw_message.len() + } as u64, message .from() .and_then(|s| s.first()) @@ -629,93 +734,148 @@ impl EmailIngest for Server { ); } + let thread_info = ThreadInfo { + thread_id: thread_ref, + ref_ids: &message_ids, + }; batch .with_collection(Collection::Email) - .with_document(document_id) - .index_message( - tenant_id, - message, - extra_headers.into_bytes(), - extra_headers_parsed, - blob_hash.clone(), - data, - params.received_at.unwrap_or_else(now), - ) - .caused_by(trc::location!())? + .create_document(document_slot); + #[cfg(not(feature = "enterprise"))] + batch + .index_message(&message, &extra_headers, blob_hash.clone(), data) + .caused_by(trc::location!())?; + batch .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_result.thread_hash, }), - ThreadInfo::serialize(thread_id, &message_ids), + thread_info, ) - .schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::now(), - })); + .queue_document_index(SearchIndex::Email, account_id, QueueDocumentId::Current); + + let mut private_commit = PrivateMetadataCommit::default(); + if let Some(metadata) = metadata { + metadata + .build( + batch, + account_id, + PendingId::Slot(document_slot), + &mut private_commit, + ) + .caused_by(trc::location!())?; + } if let Some(blob_hold) = blob_hold { - batch.clear(blob_hold); + blob_hold.release(batch); } // Merge threads if necessary - if !thread_result.merge_ids.is_empty() - || matches!( - params.source, - IngestSource::Jmap { .. } | IngestSource::Imap { .. } - ) - { + let thread_recheck = if !thread_result.merge_ids.is_empty() { batch.schedule_task(Task::MergeThreads(TaskMergeThreads { account_id: account_id.into(), status: TaskStatus::now(), thread_name: thread_result.thread_hash.to_string(), message_ids: Map::new(message_ids.into_iter().map(|id| id.to_string()).collect()), })); - } + None + } else if !message_ids.is_empty() + && matches!( + params.source, + IngestSource::Jmap { .. } | IngestSource::Imap { .. } + ) + { + Some(PendingThread::new( + thread_result.thread_hash, + message_ids, + thread_result.bucket_full, + )) + } else { + None + }; // Add iTIP responses to batch if !itip_messages.is_empty() { ItipMessages::new(itip_messages) - .queue(&mut batch) + .queue(batch) .caused_by(trc::location!())?; } - // Insert and obtain ids - let change_id = self - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())? - .last_change_id(account_id)?; - - // Request FTS index - self.notify_task_queue(); - - trc::event!( - MessageIngest(match params.source { - IngestSource::Smtp { .. } => + Ok(PendingIngest::Queued(PendingEmail { + account_id, + document_slot, + uid_slots, + thread_id: thread_ref, + blob_hash, + size: raw_message_len, + thread_recheck, + private_commit, + event: match params.source { + IngestSource::Smtp { .. } => { if !is_spam { MessageIngestEvent::Ham } else { MessageIngestEvent::Spam - }, + } + } IngestSource::Jmap { .. } | IngestSource::Restore => MessageIngestEvent::JmapAppend, IngestSource::Imap { .. } => MessageIngestEvent::ImapAppend, - }), - SpanId = params.session_id, + }, + mailbox_ids: std::mem::take(&mut params.mailbox_ids), + message_id, + session_id: params.session_id, + start_time, + })) + } + + async fn email_ingest_finish( + &self, + pending: PendingEmail, + assigned_ids: &AssignedIds, + ) -> IngestedEmail { + let PendingEmail { + account_id, + document_slot, + uid_slots, + thread_id, + blob_hash, + size, + thread_recheck, + private_commit, + event, + mailbox_ids, + message_id, + session_id, + start_time, + } = pending; + self.private_metadata_committed(private_commit, assigned_ids) + .await; + let change_id = assigned_ids.last_change_id(account_id, SyncCollection::Email); + let document_id = assigned_ids.slot(document_slot); + let thread_id = match thread_id { + PendingId::Assigned(thread_id) => thread_id, + PendingId::Slot(slot) => assigned_ids.slot(slot), + }; + let imap_uids = assigned_ids.slots(uid_slots).collect::>(); + + trc::event!( + MessageIngest(event), + SpanId = session_id, AccountId = account_id, DocumentId = document_id, - MailboxId = mailbox_ids_event, + MailboxId = mailbox_ids + .iter() + .map(|mailbox_id| trc::Value::from(*mailbox_id)) + .collect::>(), BlobId = blob_hash.to_hex(), ChangeId = change_id, MessageId = message_id, - Size = raw_message_len, + Size = size, Elapsed = start_time.elapsed(), ); - Ok(IngestedEmail { + IngestedEmail { document_id, thread_id, change_id, @@ -728,26 +888,25 @@ impl EmailIngest for Server { }, section: None, }, - size: raw_message_len as usize, + size: size as usize, imap_uids, - }) + thread_recheck, + } } async fn find_thread_id( &self, account_id: u32, thread_name: &str, - message_ids: &[CheekyHash], + message_ids: &[u128], ) -> trc::Result { let mut result = ThreadResult { thread_id: None, - thread_hash: CheekyHash::new(if !thread_name.is_empty() { - thread_name - } else { - "!" - }), + thread_hash: thread_hash(thread_name), merge_ids: vec![], duplicate_ids: vec![], + matched_ids: vec![], + bucket_full: false, }; if message_ids.is_empty() { @@ -755,21 +914,17 @@ impl EmailIngest for Server { } // Find thread ids - let key_len = IndexKeyPrefix::len() + result.thread_hash.len() + U32_LEN; + let thread_class = IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: result.thread_hash, + }; + let key_len = thread_class.key_len(); let document_id_pos = key_len - U32_LEN; let mut thread_merge = ThreadMerge::new(); + let mut rows = 0; self.store() .iterate( IterateParams::new( - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Hash { - property: EmailField::Threading.into(), - hash: result.thread_hash, - }), - }, ValueKey { account_id, collection: Collection::Email.into(), @@ -779,29 +934,40 @@ impl EmailIngest for Server { hash: result.thread_hash, }), }, + ValueKey { + account_id, + collection: Collection::Email.into(), + document_id: 0, + class: ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: result.thread_hash, + }), + }, ) - .ascending(), + .limit(THREAD_SCAN_WINDOW), |key, value| { + rows += 1; if key.len() == key_len { // Find matching references let references = value.get(U32_LEN..).unwrap_or_default(); if has_message_id(message_ids, references) { - let document_id = key.deserialize_be_u32(document_id_pos)?; + let document_id = !key.deserialize_be_u32(document_id_pos)?; let thread_id = value.deserialize_be_u32(0)?; - if message_ids.len() == references.len() / CheekyHash::HASH_SIZE + if message_ids.len() == references.len() / U128_LEN && references - .as_chunks::<{ CheekyHash::HASH_SIZE }>() + .as_chunks::() .0 .iter() .zip(message_ids.iter()) - .all(|(a, b)| a == b.as_raw_bytes()) + .all(|(a, b)| u128::from_be_bytes(*a) == *b) { result.duplicate_ids.push(document_id); } thread_merge.add(thread_id, document_id); + result.matched_ids.push(document_id); } } @@ -810,6 +976,7 @@ impl EmailIngest for Server { ) .await .caused_by(trc::location!())?; + result.bucket_full = rows >= THREAD_SCAN_WINDOW; match thread_merge.num_thread_ids() { 0 => Ok(result), @@ -828,88 +995,59 @@ impl EmailIngest for Server { } } - async fn assign_email_ids( - &self, - account_id: u32, - mailbox_ids: impl IntoIterator + Sync + Send, - generate_email_id: bool, - ) -> trc::Result + 'static> { - // Increment UID next - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - - let mut expected_ids = 0; - if generate_email_id { - batch - .with_collection(Collection::Email) - .add_and_get(ValueClass::DocumentId, 1); - expected_ids += 1; - } - - batch.with_collection(Collection::Mailbox); - - for mailbox_id in mailbox_ids { - batch - .with_document(mailbox_id) - .add_and_get(MailboxField::UidCounter, 1); - expected_ids += 1; - } - - let ids = if expected_ids > 0 { - self.core.storage.data.write(batch.build_all()).await? - } else { - AssignedIds::default() - }; - if ids.ids.len() == expected_ids { - Ok(ids.ids.into_iter().map(|id| match id { - AssignedId::Counter(id) => id as u32, - AssignedId::ChangeId(_) => unreachable!(), - })) - } else { - Err(trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No all document ids were generated")) - } - } - - async fn add_account_spam_sample( + async fn add_account_spam_samples( &self, batch: &mut BatchBuilder, account_id: u32, - document_id: u32, - is_spam: bool, + samples: SpamSamples, span_id: u64, ) -> trc::Result<()> { - if self.core.spam.classifier.is_some() - && let Some(archive) = self - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await - .caused_by(trc::location!())? - { - let metadata = archive - .to_unarchived::() - .caused_by(trc::location!())?; - let part = metadata.inner.root_part(); + if self.core.spam.classifier.is_none() || samples.is_empty() { + return Ok(()); + } + let SpamSamples { + documents, + spam, + prepared, + } = samples; + batch.with_account_id(account_id); + for sample in prepared { self.add_spam_sample( account_id, batch, - (&metadata.inner.blob_hash).into(), - part.from().unwrap_or_default().to_string(), - thread_name(part.subject().unwrap_or_default()).to_string(), - is_spam, + sample.hash, + sample.size, + sample.from, + sample.subject, + sample.is_spam, true, span_id, ); + batch.commit_point(); + } + if documents.is_empty() { + return Ok(()); } - Ok(()) + self.message_metadata(account_id, &documents, |document_id, bytes| { + let sample = SpamSample::from_metadata(bytes, spam.contains(document_id))?; + self.add_spam_sample( + account_id, + batch, + sample.hash, + sample.size, + sample.from, + sample.subject, + sample.is_spam, + true, + span_id, + ); + batch.commit_point(); + Ok(true) + }) + .await + .caused_by(trc::location!()) } fn add_spam_sample( @@ -917,6 +1055,7 @@ impl EmailIngest for Server { account_id: u32, batch: &mut BatchBuilder, hash: BlobHash, + size: u64, from: String, subject: String, is_spam: bool, @@ -945,7 +1084,10 @@ impl EmailIngest for Server { hash, to: BlobLink::Temporary { until }, }, - ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), + BlobLinkValue::new(size).with_object_id(ObjectId::new( + ObjectType::SpamTrainingSample, + item_id.into(), + )), ) .set( ValueClass::Registry(RegistryClass::Item { object_id, item_id }), @@ -972,26 +1114,45 @@ impl EmailIngest for Server { } } -pub fn has_message_id(a: &[CheekyHash], b: &[u8]) -> bool { - let mut i = 0; - let mut j = 0; +pub fn thread_hash(thread_name: &str) -> Hash128 { + Hash128::from(xxh3_128( + if !thread_name.is_empty() { + thread_name + } else { + "!" + } + .as_bytes(), + )) +} - let a_len = a.len(); - let b_len = b.len() / CheekyHash::HASH_SIZE; +pub fn has_message_id(a: &[u128], b: &[u8]) -> bool { + sorted_intersect( + a.iter().copied(), + b.as_chunks::() + .0 + .iter() + .map(|chunk| u128::from_be_bytes(*chunk)), + ) +} - while i < a_len && j < b_len { - match a[i] - .as_raw_bytes() - .as_slice() - .cmp(&b[j * CheekyHash::HASH_SIZE..(j + 1) * CheekyHash::HASH_SIZE]) - { - std::cmp::Ordering::Equal => return true, - std::cmp::Ordering::Less => i += 1, - std::cmp::Ordering::Greater => j += 1, +fn sorted_intersect(mut a: impl Iterator, mut b: impl Iterator) -> bool { + let (Some(mut left), Some(mut right)) = (a.next(), b.next()) else { + return false; + }; + + loop { + match left.cmp(&right) { + Ordering::Equal => return true, + Ordering::Less => match a.next() { + Some(next) => left = next, + None => return false, + }, + Ordering::Greater => match b.next() { + Some(next) => right = next, + None => return false, + }, } } - - false } impl IngestSource<'_> { @@ -1000,19 +1161,35 @@ impl IngestSource<'_> { } } -pub struct ThreadInfo; +pub struct ThreadInfo<'x> { + pub thread_id: PendingId, + pub ref_ids: &'x [u128], +} -impl ThreadInfo { - pub fn serialize(thread_id: u32, ref_ids: &[CheekyHash]) -> Vec { - let mut buf = Vec::with_capacity(U32_LEN + 1 + ref_ids.len() * CheekyHash::HASH_SIZE); - buf.extend_from_slice(&thread_id.to_be_bytes()); - for ref_id in ref_ids { - buf.extend_from_slice(ref_id.as_raw_bytes()); +impl From> for SizedSetValue { + fn from(info: ThreadInfo<'_>) -> Self { + let mut buf = Vec::with_capacity(U32_LEN + 1 + info.ref_ids.len() * U128_LEN); + buf.extend_from_slice(&info.thread_id.assigned().unwrap_or_default().to_be_bytes()); + for ref_id in info.ref_ids { + buf.extend_from_slice(ref_id.to_be_bytes().as_slice()); + } + + match info.thread_id { + PendingId::Slot(slot) => ( + buf, + vec![Patch { + offset: 0, + source: PatchSource::SlotBeU32(slot), + }], + ) + .into(), + PendingId::Assigned(_) => buf.into(), } - buf } } +const THREAD_SCAN_WINDOW: usize = 256; + pub struct ThreadMerge { entries: AHashMap>, } @@ -1095,3 +1272,7 @@ impl ThreadMerge { } } } + +mod checks; +pub mod group; +pub mod recheck; diff --git a/crates/email/src/message/ingest/checks.rs b/crates/email/src/message/ingest/checks.rs new file mode 100644 index 00000000..ac3f79f2 --- /dev/null +++ b/crates/email/src/message/ingest/checks.rs @@ -0,0 +1,137 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + mailbox::{JUNK_ID, Mailbox, SENT_ID, lookup::MailboxLookup}, + message::messagedata::{EmailMessageData, MessageData}, +}; +use common::Server; +use store::{ + Deserialize, IterateParams, U32_LEN, ValueKey, + dispatch::DocumentSet, + write::{Archive, ArchiveBytes, ValueClass, key::DeserializeBigEndian}, +}; +use tinyvec::TinyVec; +use trc::AddContext; +use types::{collection::Collection, field::Field, special_use::SpecialUse}; + +pub(super) async fn has_duplicate_in( + server: &Server, + account_id: u32, + duplicate_ids: &[u32], + mailbox_id: u32, +) -> trc::Result { + let is_target = |target: u32| target == mailbox_id || target == JUNK_ID; + if let Some(cache) = server.fresh_messages(account_id) { + return Ok(duplicate_ids.iter().any(|document_id| { + cache.email_by_id(document_id).is_some_and(|message| { + message + .mailboxes() + .iter() + .any(|mailbox| is_target(mailbox.mailbox_id)) + }) + })); + } + + let documents = sorted_ids(duplicate_ids); + let mut found = false; + server + .message_datas(account_id, &documents, |_, data| { + found |= data + .mailboxes + .iter() + .any(|mailbox| is_target(mailbox.mailbox_id)); + Ok(true) + }) + .await + .caused_by(trc::location!())?; + Ok(found) +} + +pub(super) async fn is_trusted_reply( + server: &Server, + account_id: u32, + thread_id: u32, + matched_ids: &[u32], +) -> trc::Result { + if let Some(cache) = server.fresh_messages(account_id) { + let sent_id = cache + .mailbox_by_role(&SpecialUse::Sent) + .map_or(SENT_ID, |mailbox| mailbox.document_id); + return Ok(cache.in_thread(thread_id).any(|message| { + message + .mailboxes() + .iter() + .any(|mailbox| mailbox.mailbox_id == sent_id) + })); + } + + let documents = sorted_ids(matched_ids); + if documents.is_empty() { + return Ok(false); + } + + let store = server.store(); + let email_key = |document_id| ValueKey { + account_id, + collection: Collection::Email.into(), + document_id, + class: ValueClass::Property(Field::ARCHIVE.into()), + }; + let sent_key = ValueKey::archive(account_id, Collection::Mailbox, SENT_ID); + let mut ranges = documents + .scan_ranges(store.scan_gap()) + .into_iter() + .map(|(from, to)| IterateParams::new(email_key(from), email_key(to))) + .collect::>(); + ranges.push(IterateParams::new(sent_key.clone(), sent_key)); + + let mailbox_collection: u8 = Collection::Mailbox.into(); + let mut mailbox_ids: TinyVec<[u32; 4]> = TinyVec::new(); + let mut sent_has_role = false; + store + .iterate_many(ranges, |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + let collection = key + .len() + .checked_sub(U32_LEN + 2) + .and_then(|offset| key.get(offset).copied()); + if collection == Some(mailbox_collection) { + let archive = as Deserialize>::deserialize(value)?; + sent_has_role = archive.unarchive::()?.role() == SpecialUse::Sent; + } else if documents.contains(document_id) { + for mailbox in MessageData::deserialize(value)?.mailboxes { + if !mailbox_ids.contains(&mailbox.mailbox_id) { + mailbox_ids.push(mailbox.mailbox_id); + } + } + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + + if mailbox_ids.is_empty() { + Ok(false) + } else if sent_has_role { + Ok(mailbox_ids.contains(&SENT_ID)) + } else { + let sent_id = server + .mailbox_id_by_role(account_id, SpecialUse::Sent) + .await + .caused_by(trc::location!())? + .unwrap_or(SENT_ID); + Ok(mailbox_ids.contains(&sent_id)) + } +} + +fn sorted_ids(ids: &[u32]) -> Vec { + let mut ids = ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + ids +} diff --git a/crates/email/src/message/ingest/group.rs b/crates/email/src/message/ingest/group.rs new file mode 100644 index 00000000..ca231e3b --- /dev/null +++ b/crates/email/src/message/ingest/group.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + EmailIngest, IngestEmail, IngestedEmail, PendingIngest, sorted_intersect, thread_hash, +}; +use crate::message::thread::ThreadFields; +use common::Server; +use mail_parser::Message; +use store::write::{BatchBuilder, PendingId}; +use trc::{AddContext, EventType, LimitEvent, MessageIngestEvent}; +use utils::hash128::Hash128; + +const MAX_GROUP_EMAILS: usize = 32; +const MAX_GROUP_BYTES: usize = 8 * 1024 * 1024; + +pub struct IngestGroup { + batch: BatchBuilder, + pending: Vec, + threads: Vec<(ThreadKey, PendingId)>, + size: usize, + max_emails: usize, +} + +#[derive(Default)] +pub struct ThreadKey { + hash: Hash128, + message_ids: Vec, +} + +impl ThreadKey { + pub fn new(message: &Message<'_>) -> Self { + let fields = ThreadFields::scan(message); + ThreadKey { + hash: thread_hash(fields.subject), + message_ids: fields.message_ids, + } + } +} + +impl Default for IngestGroup { + fn default() -> Self { + IngestGroup::with_max_emails(MAX_GROUP_EMAILS) + } +} + +impl IngestGroup { + pub fn with_max_emails(max_emails: usize) -> Self { + IngestGroup { + batch: BatchBuilder::new(), + pending: Vec::new(), + threads: Vec::new(), + size: 0, + max_emails, + } + } + + pub fn must_flush_before(&self, size: usize) -> bool { + !self.pending.is_empty() + && (self.pending.len() >= self.max_emails + || self.size.saturating_add(size) > MAX_GROUP_BYTES + || self.batch.is_large_batch()) + } + + fn pending_thread(&self, key: &ThreadKey) -> Option { + if key.message_ids.is_empty() { + return None; + } + let mut threads = self + .threads + .iter() + .filter(|(pending, _)| { + pending.hash == key.hash + && sorted_intersect( + pending.message_ids.iter().copied(), + key.message_ids.iter().copied(), + ) + }) + .map(|(_, thread_id)| *thread_id); + let first = threads.next()?; + threads.all(|thread_id| thread_id == first).then_some(first) + } + + pub async fn ingest( + &mut self, + server: &Server, + mut params: IngestEmail<'_>, + key: ThreadKey, + ) -> trc::Result<()> { + let size = params.raw_message.len(); + let group_thread = self.pending_thread(&key); + match server + .email_ingest_into(&mut params, &mut self.batch, group_thread) + .await + { + Ok(pending) => { + if let PendingIngest::Queued(email) = &pending + && !key.message_ids.is_empty() + { + self.threads.push((key, email.thread_id)); + } + self.pending.push(pending); + self.size = self.size.saturating_add(size); + Ok(()) + } + Err(err) => { + if !is_rejection(&err) { + self.discard(); + } + Err(err) + } + } + } + + fn discard(&mut self) { + self.batch = BatchBuilder::new(); + self.pending.clear(); + self.threads.clear(); + self.size = 0; + } + + pub async fn flush(&mut self, server: &Server) -> trc::Result> { + let mut batch = std::mem::take(&mut self.batch); + let pending = std::mem::take(&mut self.pending); + self.threads.clear(); + self.size = 0; + if pending.is_empty() { + return Ok(Vec::new()); + } + + let queues = batch.queue_notify(); + let assigned_ids = server + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + server.inner.mark_caches_stale_from(&assigned_ids); + + let mut ingested = Vec::with_capacity(pending.len()); + for pending in pending { + ingested.push(match pending { + PendingIngest::Queued(pending) => { + server.email_ingest_finish(pending, &assigned_ids).await + } + PendingIngest::Duplicate(email) => email, + }); + } + server.notify_queues(queues).await; + + Ok(ingested) + } +} + +fn is_rejection(err: &trc::Error) -> bool { + match err.as_ref() { + EventType::MessageIngest(MessageIngestEvent::Error) + | EventType::Limit(LimitEvent::Quota) => true, + _ => false, + } +} diff --git a/crates/email/src/message/ingest/recheck.rs b/crates/email/src/message/ingest/recheck.rs new file mode 100644 index 00000000..5fa76798 --- /dev/null +++ b/crates/email/src/message/ingest/recheck.rs @@ -0,0 +1,262 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{IngestedEmail, THREAD_SCAN_WINDOW, has_message_id}; +use common::Server; +use registry::{ + schema::structs::{Task, TaskMergeThreads, TaskStatus}, + types::map::Map, +}; +use std::ops::Range; +use store::{ + IterateParams, Key, U32_LEN, ValueKey, + write::{BatchBuilder, IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, +}; +use tinyvec::TinyVec; +use trc::AddContext; +use types::{collection::Collection, field::EmailField}; +use utils::hash128::Hash128; + +#[derive(Debug)] +pub struct PendingThread { + hash: Hash128, + message_ids: Vec, + bucket_full: bool, +} + +#[derive(Debug, Default)] +pub struct ThreadRecheck { + account_id: u32, + pending: Vec, +} + +struct Bucket { + thread: PendingThread, + thread_ids: TinyVec<[u32; 2]>, +} + +type BucketRange = IterateParams>; + +struct Group { + prefix: Vec, + hash: Hash128, + buckets: Range, + message_ids: Vec, + rows: usize, + full: bool, +} + +impl PendingThread { + pub(super) fn new(hash: Hash128, message_ids: Vec, bucket_full: bool) -> Self { + PendingThread { + hash, + message_ids, + bucket_full, + } + } +} + +impl ThreadRecheck { + pub fn push(&mut self, account_id: u32, email: &mut IngestedEmail) { + if let Some(thread) = email.thread_recheck.take() { + self.account_id = account_id; + self.pending.push(thread); + } + } + + pub async fn finish(self, server: &Server, session_id: u64) { + if let Err(err) = self.run(server).await { + trc::error!(err.span_id(session_id).details("Failed to recheck threads")); + } + } + + async fn run(self, server: &Server) -> trc::Result<()> { + if self.pending.is_empty() { + return Ok(()); + } + let account_id = self.account_id; + let mut detector = ForkDetector::new(account_id, self.pending); + let (mut small, full) = detector.ranges(); + let mut observe = |key: &[u8], value: &[u8]| detector.observe(key, value).map(|_| true); + + let store = server.store(); + if small.len() > 1 { + store.iterate_many(small, &mut observe).await + } else if let Some(range) = small.pop() { + store + .iterate(range.limit(THREAD_SCAN_WINDOW), &mut observe) + .await + } else { + Ok(()) + } + .caused_by(trc::location!())?; + for range in full { + store + .iterate(range, &mut observe) + .await + .caused_by(trc::location!())?; + } + + let mut forks = detector.forks().peekable(); + if forks.peek().is_none() { + return Ok(()); + } + + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + for thread in forks { + batch.schedule_task(Task::MergeThreads(TaskMergeThreads { + account_id: account_id.into(), + status: TaskStatus::now(), + thread_name: thread.hash.to_string(), + message_ids: Map::new( + thread + .message_ids + .into_iter() + .map(|id| id.to_string()) + .collect(), + ), + })); + } + + let queues = batch.queue_notify(); + server + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + server.notify_queues(queues).await; + Ok(()) + } +} + +struct ForkDetector { + account_id: u32, + buckets: Vec, + groups: Vec, +} + +impl ForkDetector { + fn new(account_id: u32, mut pending: Vec) -> Self { + pending.sort_unstable_by_key(|thread| thread.hash); + let mut groups: Vec = Vec::new(); + for (idx, thread) in pending.iter().enumerate() { + match groups.last_mut() { + Some(group) if group.hash == thread.hash => { + group.buckets.end = idx + 1; + group.full |= thread.bucket_full; + group.message_ids.extend_from_slice(&thread.message_ids); + } + _ => { + let mut prefix = thread_key(account_id, thread.hash, 0).serialize(0); + prefix.truncate(prefix.len() - U32_LEN); + groups.push(Group { + prefix, + hash: thread.hash, + buckets: idx..idx + 1, + message_ids: thread.message_ids.clone(), + rows: 0, + full: thread.bucket_full, + }); + } + } + } + for group in &mut groups { + if group.buckets.len() > 1 { + group.message_ids.sort_unstable(); + group.message_ids.dedup(); + } + } + groups.sort_unstable_by(|a, b| a.prefix.cmp(&b.prefix)); + + ForkDetector { + account_id, + buckets: pending + .into_iter() + .map(|thread| Bucket { + thread, + thread_ids: TinyVec::new(), + }) + .collect(), + groups, + } + } + + fn ranges(&self) -> (Vec, Vec) { + let range = |group: &Group| { + IterateParams::new( + thread_key(self.account_id, group.hash, u32::MAX), + thread_key(self.account_id, group.hash, 0), + ) + }; + let mut small = Vec::with_capacity(self.groups.len()); + let mut full = Vec::new(); + for group in &self.groups { + if group.full { + full.push(range(group).limit(THREAD_SCAN_WINDOW)); + } else { + small.push(range(group)); + } + } + (small, full) + } + + fn observe(&mut self, key: &[u8], value: &[u8]) -> trc::Result<()> { + let Some(group) = key + .len() + .checked_sub(U32_LEN) + .and_then(|len| key.get(..len)) + .and_then(|prefix| { + self.groups + .binary_search_by(|group| group.prefix.as_slice().cmp(prefix)) + .ok() + }) + .and_then(|idx| self.groups.get_mut(idx)) + else { + return Ok(()); + }; + group.rows += 1; + let references = value.get(U32_LEN..).unwrap_or_default(); + if group.rows > THREAD_SCAN_WINDOW || !has_message_id(&group.message_ids, references) { + return Ok(()); + } + + let thread_id = value.deserialize_be_u32(0)?; + for bucket in self + .buckets + .get_mut(group.buckets.clone()) + .unwrap_or_default() + { + if bucket.thread_ids.len() < 2 + && !bucket.thread_ids.contains(&thread_id) + && has_message_id(&bucket.thread.message_ids, references) + { + bucket.thread_ids.push(thread_id); + } + } + Ok(()) + } + + fn forks(self) -> impl Iterator { + self.buckets + .into_iter() + .filter(|bucket| bucket.thread_ids.len() > 1) + .map(|bucket| bucket.thread) + } +} + +fn thread_key(account_id: u32, hash: Hash128, document_id: u32) -> ValueKey { + ValueKey { + account_id, + collection: Collection::Email.into(), + document_id, + class: ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash, + }), + } +} + diff --git a/crates/email/src/message/ingest_metadata.rs b/crates/email/src/message/ingest_metadata.rs new file mode 100644 index 00000000..d60cf4db --- /dev/null +++ b/crates/email/src/message/ingest_metadata.rs @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::storage::metadata::{ + MetadataLog, MetadataWrite, PrivateMetadataCommit, PrivateMetadataWrite, +}; +use store::write::{BatchBuilder, PendingId}; +use types::{collection::Collection, metadata::EncodedMetadata}; + +#[derive(Debug, Default)] +pub struct IngestMetadata { + pub shared: Option, + pub private: Option, +} + +impl IngestMetadata { + pub fn new( + shared: Option, + private: Option, + ) -> Option> { + (shared.is_some() || private.is_some()) + .then(|| Box::new(IngestMetadata { shared, private })) + } + + pub(crate) fn build( + self, + batch: &mut BatchBuilder, + account_id: u32, + document_id: PendingId, + commit: &mut PrivateMetadataCommit, + ) -> trc::Result<()> { + if let Some(container) = self.shared { + MetadataWrite { + account_id, + collection: Collection::Email, + previous: None, + next: Some(container), + log: MetadataLog::None, + } + .build(document_id, batch)?; + } + if let Some(write) = self.private { + write.build(document_id, batch, commit)?; + } + Ok(()) + } +} + diff --git a/crates/email/src/message/jmap/body.rs b/crates/email/src/message/jmap/body.rs new file mode 100644 index 00000000..b7fc4cdc --- /dev/null +++ b/crates/email/src/message/jmap/body.rs @@ -0,0 +1,229 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BodyValueOptions, EmailRender, JmapMap, JmapValue, str_value}; +use crate::message::{ + body::{truncate_html, truncate_plain}, + metadata::{BodyList, MAX_NESTING, PartFlags, PartKind, PartSource, PartView}, +}; +use jmap_proto::object::email::{EmailProperty, EmailValue}; +use jmap_tools::{Key, Map, Value}; +use std::borrow::Cow; +use types::blob::{BlobId, EncodedRange}; + +struct Level<'a> { + parent: Option>, + next: usize, + values: Vec>, +} + +impl BodyValueOptions { + pub(super) fn fetches(&self, part: &PartView<'_>) -> bool { + if !part.is_text() + || !part + .content_type() + .is_none_or(|ct| ct.ctype().eq_ignore_ascii_case("text")) + { + return false; + } + let flags = part.flags(); + self.fetch_all + || (self.fetch_html && flags.contains(PartFlags::IN_HTML_BODY)) + || (self.fetch_text && flags.contains(PartFlags::IN_TEXT_BODY)) + } +} + +impl<'a> EmailRender<'a, '_> { + pub(super) fn body_list_values(&self, list: BodyList) -> JmapValue<'a> { + Value::Array( + self.root + .body_list(list) + .map(|part| self.body_part(part)) + .collect(), + ) + } + + pub(super) fn body_part(&self, root: PartView<'a>) -> JmapValue<'a> { + let mut levels = vec![Level { + parent: None, + next: 0, + values: Vec::with_capacity(1), + }]; + loop { + let Some(level) = levels.last_mut() else { + return Value::Null; + }; + let next = match level.parent { + None => (level.next == 0).then_some(root), + Some(parent) => parent.child(level.next), + }; + if let Some(part) = next { + level.next += 1; + level.values.push(Value::Object(self.part_values(part))); + if part.is_multipart() && levels.len() < MAX_NESTING { + levels.push(Level { + parent: Some(part), + next: 0, + values: Vec::with_capacity(part.children().len()), + }); + } + } else if levels.len() > 1 { + let Some(finished) = levels.pop() else { + return Value::Null; + }; + if let Some(Value::Object(object)) = + levels.last_mut().and_then(|level| level.values.last_mut()) + { + object.insert_unchecked(EmailProperty::SubParts, Value::Array(finished.values)); + } + } else { + return levels + .pop() + .and_then(|level| level.values.into_iter().next()) + .unwrap_or_default(); + } + } + } + + fn part_values(&self, part: PartView<'a>) -> JmapMap<'a> { + let multipart = part.is_multipart(); + let mut values = Map::with_capacity(self.body_properties.len()); + for property in self.body_properties { + let value = match property { + EmailProperty::PartId if !multipart => { + Value::Str(Cow::Owned(part.id().to_string())) + } + EmailProperty::BlobId if !multipart => self.part_blob_id(part), + EmailProperty::Size if !multipart => Value::Number(part.decoded_size().into()), + EmailProperty::Size => Value::Number((part.body_range().len() as u64).into()), + EmailProperty::Name => str_value(part.attachment_name()), + EmailProperty::Type => match part.content_type() { + Some(content_type) => Value::Str(content_type.mime_type()), + None => match part.kind() { + PartKind::Text => Value::Str(Cow::Borrowed("text/plain")), + PartKind::Html => Value::Str(Cow::Borrowed("text/html")), + PartKind::Message => Value::Str(Cow::Borrowed("message/rfc822")), + _ => Value::Null, + }, + }, + EmailProperty::Charset => match part.content_type() { + Some(content_type) if content_type.ctype().eq_ignore_ascii_case("text") => { + str_value(content_type.attribute("charset").or(Some("us-ascii"))) + } + Some(_) => Value::Null, + None => str_value(part.is_text().then_some("us-ascii")), + }, + EmailProperty::Disposition => { + str_value(part.content_disposition().map(|cd| cd.ctype())) + } + EmailProperty::Cid => str_value(part.content_id()), + EmailProperty::Language => { + let languages = part.content_language(); + if languages.is_present() { + Value::Array( + languages + .iter() + .map(|language| Value::Str(Cow::Borrowed(language))) + .collect(), + ) + } else { + Value::Null + } + } + EmailProperty::Location => str_value(part.content_location()), + EmailProperty::Header(_) | EmailProperty::Headers => { + self.part_header(part, property) + } + EmailProperty::SubParts if multipart => continue, + _ => Value::Null, + }; + values.insert_unchecked(property.clone(), value); + } + values + } + + fn part_blob_id(&self, part: PartView<'a>) -> JmapValue<'a> { + self.inner_blob_id(part).map_or(Value::Null, |blob_id| { + Value::Element(EmailValue::BlobId(blob_id)) + }) + } + + pub(super) fn inner_blob_id(&self, part: PartView<'a>) -> Option { + let encoded = |part: PartView<'a>, start: usize| { + EncodedRange::new(start, part.body_range().len(), part.encoding().as_u8()) + }; + let chain = self.meta.source_chain(part.message())?; + let mut ids = chain.ids(); + let outermost = match ids.next() { + Some(id) => self.meta.part(id)?, + None => part, + }; + let start = self + .meta + .blob_range(outermost.body_range())? + .start + .checked_sub(self.blob_prefix)?; + let outermost = encoded(outermost, start); + if chain.is_empty() { + return self.blob_id.inner_part(&[], outermost); + } + let mut containers = Vec::with_capacity(chain.len()); + containers.push(outermost); + for id in ids { + let container = self.meta.part(id)?; + containers.push(encoded(container, container.offset_body())); + } + self.blob_id + .inner_part(&containers, encoded(part, part.offset_body())) + } + + fn part_header(&self, part: PartView<'a>, property: &EmailProperty) -> JmapValue<'a> { + let Some(raw) = self.raw else { + return Value::Null; + }; + let Some(source) = self.meta.source(part.message(), raw) else { + return Value::Null; + }; + let headers = part.headers(); + match &source { + PartSource::Raw(raw) => headers.jmap_value(property, *raw), + PartSource::Decoded(bytes) => { + headers.jmap_value(property, bytes.as_slice()).into_owned() + } + } + } + + pub(super) fn body_values(&self) -> JmapValue<'a> { + if !self.options.fetches_any() { + return Value::Object(Map::new()); + } + let Some(raw) = self.raw else { + return Value::Null; + }; + let source = PartSource::Raw(raw); + let mut body_values = Map::with_capacity(self.root.parts().len()); + for part in self.root.parts() { + if !self.options.fetches(&part) { + continue; + } + let Some(decoded) = part.text(&source) else { + continue; + }; + let (is_truncated, value) = match part.kind() { + PartKind::Html => truncate_html(&decoded.text, self.options.max_bytes), + _ => truncate_plain(&decoded.text, self.options.max_bytes), + }; + body_values.insert_unchecked( + Key::Owned(part.id().to_string()), + Map::with_capacity(3) + .with_key_value(EmailProperty::IsEncodingProblem, decoded.has_problems) + .with_key_value(EmailProperty::IsTruncated, is_truncated) + .with_key_value(EmailProperty::Value, value), + ); + } + Value::Object(body_values) + } +} diff --git a/crates/email/src/message/jmap/estimate.rs b/crates/email/src/message/jmap/estimate.rs new file mode 100644 index 00000000..007d809d --- /dev/null +++ b/crates/email/src/message/jmap/estimate.rs @@ -0,0 +1,221 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + +use super::{BodyValueOptions, JmapMap, JmapValue}; +use crate::message::metadata::ArchivedMessageMetadata; +use jmap_proto::object::email::EmailProperty; +use jmap_tools::{Key, Value}; + +const NODE_BYTES: u64 = size_of::<(Key<'static, EmailProperty>, JmapValue<'static>)>() as u64; +const PROPERTY_BYTES: u64 = 2 * NODE_BYTES; +const ADDRESS_BYTES: u64 = 4 * NODE_BYTES; +const PART_PROPERTY_BYTES: u64 = 2 * NODE_BYTES; +const BODY_VALUE_BYTES: u64 = 4 * NODE_BYTES; +const LARGE_EMAIL_BYTES: u64 = 64 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EmailEstimate { + content: bool, + fixed: u64, + envelope_properties: u64, + header_properties: u64, + structure_properties: u64, + part_bytes: u64, + body_values: Option, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct EmailCost { + pub bytes: u64, + pub light: u64, + body_values: u64, + headers: u64, +} + +impl EmailEstimate { + pub fn new( + properties: &[EmailProperty], + body_properties: &[EmailProperty], + options: &BodyValueOptions, + ) -> Self { + let mut estimate = EmailEstimate { + content: false, + fixed: properties.len() as u64 * PROPERTY_BYTES, + envelope_properties: 0, + header_properties: 0, + structure_properties: 0, + part_bytes: 0, + body_values: None, + }; + let mut body_headers = 0; + for property in body_properties { + estimate.part_bytes += PART_PROPERTY_BYTES; + match property { + EmailProperty::Header(header) => { + estimate.part_bytes += header.header.len() as u64; + body_headers += 1; + } + EmailProperty::Headers => body_headers += 1, + _ => {} + } + } + for property in properties { + match property { + EmailProperty::Header(header) => { + estimate.fixed += header.header.len() as u64; + estimate.header_properties += 1; + } + EmailProperty::Headers => estimate.header_properties += 1, + EmailProperty::Subject + | EmailProperty::MessageId + | EmailProperty::InReplyTo + | EmailProperty::References + | EmailProperty::Sender + | EmailProperty::From + | EmailProperty::To + | EmailProperty::Cc + | EmailProperty::Bcc + | EmailProperty::ReplyTo => estimate.envelope_properties = 1, + EmailProperty::TextBody + | EmailProperty::HtmlBody + | EmailProperty::Attachments + | EmailProperty::BodyStructure => estimate.structure_properties += 1, + EmailProperty::BodyValues if options.fetches_any() => { + estimate.body_values = Some(*options); + } + _ => {} + } + } + estimate.content = estimate.header_properties != 0 + || estimate.body_values.is_some() + || (body_headers != 0 && estimate.structure_properties != 0); + if body_headers != 0 { + estimate.header_properties += body_headers * estimate.structure_properties; + } + estimate + } + + pub fn cost(&self, metadata: &ArchivedMessageMetadata) -> EmailCost { + let strings = metadata.strings.len() as u64; + let mut cost = EmailCost { + bytes: self.fixed, + light: 0, + body_values: 0, + headers: self.header_properties * metadata.headers_len() as u64, + }; + if self.envelope_properties != 0 { + cost.bytes += strings + + metadata.addresses.len() as u64 * ADDRESS_BYTES + + metadata.texts.len() as u64 * NODE_BYTES; + } + if self.structure_properties != 0 { + cost.bytes += self.structure_properties + * (strings + + metadata.parts.len() as u64 * self.part_bytes + + metadata.params.len() as u64 * NODE_BYTES); + } + if !self.content && cost.bytes < LARGE_EMAIL_BYTES { + return EmailCost { + light: cost.bytes, + ..EmailCost::default() + }; + } + if let Some(options) = &self.body_values { + let max_bytes = options.max_bytes as u64; + for part in metadata.root().parts().filter(|part| options.fetches(part)) { + let decoded = u64::from(part.decoded_size()); + cost.body_values += if max_bytes != 0 { + decoded.min(max_bytes) + } else { + decoded + }; + cost.bytes += BODY_VALUE_BYTES; + } + } + cost.bytes += cost.body_values + cost.headers; + cost + } + + pub fn overrun(&self, email: &JmapMap<'_>, cost: &EmailCost) -> u64 { + if !self.content { + return 0; + } + let mut body_values = 0; + let mut headers = 0; + for (key, value) in email.iter() { + match key { + Key::Property(EmailProperty::BodyValues) => { + if let Value::Object(values) = value { + body_values += values + .iter() + .filter_map(|(_, value)| match value { + Value::Object(value) => { + match value.get(&Key::Property(EmailProperty::Value)) { + Some(Value::Str(text)) => Some(text.len() as u64), + _ => None, + } + } + _ => None, + }) + .sum::(); + } + } + Key::Property(EmailProperty::Header(_) | EmailProperty::Headers) => { + headers += tree_bytes(value); + } + Key::Property( + EmailProperty::TextBody + | EmailProperty::HtmlBody + | EmailProperty::Attachments + | EmailProperty::BodyStructure, + ) if self.header_properties != 0 => { + headers += part_header_bytes(value); + } + _ => {} + } + } + body_values.saturating_sub(cost.body_values) + headers.saturating_sub(cost.headers) + } +} + +fn tree_bytes(value: &JmapValue<'_>) -> u64 { + match value { + Value::Str(text) => text.len() as u64, + Value::Array(items) => items.iter().map(|item| NODE_BYTES + tree_bytes(item)).sum(), + Value::Object(map) => map + .iter() + .map(|(key, value)| NODE_BYTES + key_bytes(key) + tree_bytes(value)) + .sum(), + _ => 0, + } +} + +fn part_header_bytes(value: &JmapValue<'_>) -> u64 { + match value { + Value::Array(items) => items.iter().map(part_header_bytes).sum(), + Value::Object(map) => map + .iter() + .map(|(key, value)| match key { + Key::Property(EmailProperty::Header(_) | EmailProperty::Headers) => { + NODE_BYTES + key_bytes(key) + tree_bytes(value) + } + Key::Property(EmailProperty::SubParts) => part_header_bytes(value), + _ => 0, + }) + .sum(), + _ => 0, + } +} + +fn key_bytes(key: &Key<'_, EmailProperty>) -> u64 { + match key { + Key::Property(EmailProperty::Header(header)) => header.header.len() as u64, + Key::Property(_) => 0, + Key::Borrowed(key) => key.len() as u64, + Key::Owned(key) => key.len() as u64, + } +} diff --git a/crates/email/src/message/jmap/header.rs b/crates/email/src/message/jmap/header.rs new file mode 100644 index 00000000..5ba244f6 --- /dev/null +++ b/crates/email/src/message/jmap/header.rs @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{JmapValue, date_value}; +use crate::message::metadata::{HeaderId, HeaderList, HeaderView, RawMessage}; +use jmap_proto::object::email::{EmailProperty, HeaderForm, HeaderProperty}; +use jmap_tools::{Map, Value}; +use mail_parser::{AddressList, HeaderForm as ParseForm, HeaderValue, Mailbox, ParsedValue}; +use std::{borrow::Cow, ops::Range, str::from_utf8}; + +pub trait FieldSource<'s>: Copy { + fn field(self, range: Range) -> Option<&'s [u8]>; +} + +#[derive(Debug, Clone, Copy)] +pub struct HeaderBytes<'s>(pub &'s [u8]); + +impl<'s> FieldSource<'s> for &'s [u8] { + #[inline] + fn field(self, range: Range) -> Option<&'s [u8]> { + self.get(range) + } +} + +impl<'s> FieldSource<'s> for RawMessage<'s> { + #[inline] + fn field(self, range: Range) -> Option<&'s [u8]> { + self.contiguous(range) + } +} + +impl HeaderList<'_> { + pub fn jmap_value<'s>( + &self, + property: &EmailProperty, + source: impl FieldSource<'s>, + ) -> JmapValue<'s> { + match property { + EmailProperty::Header(header) => self.jmap_header(header, source), + EmailProperty::Headers => self.jmap_headers(source), + _ => Value::Null, + } + } + + pub fn jmap_header<'s>( + &self, + header: &HeaderProperty, + source: impl FieldSource<'s>, + ) -> JmapValue<'s> { + let id = HeaderId::parse(header.header.as_bytes()); + let name = header.header.as_bytes(); + let lookup = |view: HeaderView<'_>| -> Option> { + if id.is_known() { + (view.id() == id).then(|| { + source + .field(view.value_range()) + .map_or(Value::Null, |raw| HeaderBytes(raw).jmap_value(header.form)) + }) + } else if view.id() == HeaderId::OTHER { + let field = source.field(view.field_range())?; + view.raw_name_in(field) + .eq_ignore_ascii_case(name) + .then(|| HeaderBytes(view.raw_value_in(field)).jmap_value(header.form)) + } else { + None + } + }; + if header.all { + Value::Array(self.iter().filter_map(lookup).collect()) + } else { + self.iter().rev().find_map(lookup).unwrap_or(Value::Null) + } + } + + pub fn jmap_headers<'s>(&self, source: impl FieldSource<'s>) -> JmapValue<'s> { + Value::Array( + self.iter() + .map(|header| { + let field = source.field(header.field_range()).unwrap_or_default(); + Value::Object( + Map::with_capacity(2) + .with_key_value( + EmailProperty::Name, + String::from_utf8_lossy(header.raw_name_in(field)), + ) + .with_key_value( + EmailProperty::Value, + HeaderBytes(header.raw_value_in(field)).raw_text(), + ), + ) + }) + .collect(), + ) + } +} + +impl<'s> HeaderBytes<'s> { + pub fn message_ids(self) -> JmapValue<'s> { + self.ids(ParseForm::MessageIds.parse(self.0).value()) + } + + pub fn jmap_value(self, form: HeaderForm) -> JmapValue<'s> { + let parse_form = match form { + HeaderForm::Raw => { + return Value::Str(self.raw_text()); + } + HeaderForm::Text => ParseForm::Text, + HeaderForm::Addresses | HeaderForm::GroupedAddresses | HeaderForm::URLs => { + ParseForm::Addresses + } + HeaderForm::MessageIds => ParseForm::MessageIds, + HeaderForm::Date => ParseForm::Date, + }; + self.form(&parse_form.parse(self.0), form) + } + + pub fn raw_text(self) -> Cow<'s, str> { + let raw = self.0.trim_ascii_end(); + if raw.contains(&0) { + let kept: Vec = raw.iter().copied().filter(|&byte| byte != 0).collect(); + Cow::Owned( + String::from_utf8(kept) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()), + ) + } else { + String::from_utf8_lossy(raw) + } + } + + fn form(self, parsed: &ParsedValue<'_>, form: HeaderForm) -> JmapValue<'s> { + match (parsed.value(), form) { + (HeaderValue::Text(text), HeaderForm::Raw | HeaderForm::Text) => { + Value::Str(self.borrow(text)) + } + (HeaderValue::TextList(list), HeaderForm::Raw | HeaderForm::Text) => { + let mut joined = String::new(); + for (pos, item) in list.iter().enumerate() { + if pos > 0 { + joined.push_str(", "); + } + joined.push_str(item); + } + Value::Str(Cow::Owned(joined)) + } + (HeaderValue::Text(text), HeaderForm::MessageIds) => { + Value::Array(vec![Value::Str(self.borrow(text))]) + } + (value @ HeaderValue::TextList(_), HeaderForm::MessageIds) => self.ids(value), + (HeaderValue::DateTime(date), HeaderForm::Date) => date_value(&date), + (HeaderValue::Address(list), HeaderForm::URLs) if !list.has_groups() => Value::Array( + list.mailboxes() + .filter_map(|mailbox| { + mailbox + .address() + .filter(|address| address.contains(':')) + .map(|address| Value::Str(self.borrow(address))) + }) + .collect(), + ), + (HeaderValue::Address(list), HeaderForm::Addresses) => Value::Array( + list.mailboxes() + .map(|mailbox| self.mailbox(mailbox)) + .collect(), + ), + (HeaderValue::Address(list), HeaderForm::GroupedAddresses) => self.grouped(list), + _ => Value::Null, + } + } + + fn grouped(self, list: AddressList<'_>) -> JmapValue<'s> { + let group = |name: JmapValue<'s>, addresses: Vec>| { + Value::Object( + Map::with_capacity(2) + .with_key_value(EmailProperty::Name, name) + .with_key_value(EmailProperty::Addresses, Value::Array(addresses)), + ) + }; + if !list.has_groups() { + return Value::Array(vec![group( + Value::Null, + list.mailboxes() + .map(|mailbox| self.mailbox(mailbox)) + .collect(), + )]); + } + Value::Array( + list.groups() + .map(|(name, members)| { + group( + name.map_or(Value::Null, |name| Value::Str(self.borrow(name))), + members.map(|mailbox| self.mailbox(mailbox)).collect(), + ) + }) + .collect(), + ) + } + + fn mailbox(self, mailbox: Mailbox<'_>) -> JmapValue<'s> { + Value::Object( + Map::with_capacity(2) + .with_key_value( + EmailProperty::Name, + mailbox + .name() + .map_or(Value::Null, |name| Value::Str(self.borrow(name))), + ) + .with_key_value( + EmailProperty::Email, + Value::Str(self.borrow(mailbox.address().unwrap_or_default())), + ), + ) + } + + fn ids(self, value: HeaderValue<'_>) -> JmapValue<'s> { + match value { + HeaderValue::TextList(list) => Value::Array( + list.iter() + .map(|item| Value::Str(self.borrow(item))) + .collect(), + ), + HeaderValue::Text(text) => Value::Array(vec![Value::Str(self.borrow(text))]), + _ => Value::Null, + } + } + + fn borrow(self, text: &str) -> Cow<'s, str> { + let start = text.as_ptr().addr(); + let base = self.0.as_ptr().addr(); + start + .checked_sub(base) + .and_then(|offset| self.0.get(offset..offset + text.len())) + .and_then(|bytes| from_utf8(bytes).ok()) + .filter(|borrowed| *borrowed == text) + .map_or_else(|| Cow::Owned(text.to_owned()), Cow::Borrowed) + } +} diff --git a/crates/email/src/message/jmap/mod.rs b/crates/email/src/message/jmap/mod.rs new file mode 100644 index 00000000..dd6db30e --- /dev/null +++ b/crates/email/src/message/jmap/mod.rs @@ -0,0 +1,254 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod body; +pub mod estimate; +pub mod header; + + +use crate::message::metadata::{ + AddressHeader, Addresses, ArchivedMessageMetadata, BodyList, HeaderId, HeaderList, MessageView, + Occurrence, RawMessage, TextItems, +}; +use header::{FieldSource, HeaderBytes}; +use jmap_proto::{ + object::email::{EmailProperty, EmailValue}, + types::date::UTCDate, +}; +use jmap_tools::{Map, Value}; +use std::borrow::Cow; +use types::blob::BlobId; + +pub type JmapValue<'a> = Value<'a, EmailProperty, EmailValue>; +pub type JmapMap<'a> = Map<'a, EmailProperty, EmailValue>; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BodyValueOptions { + pub fetch_text: bool, + pub fetch_html: bool, + pub fetch_all: bool, + pub max_bytes: usize, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct EmailNeeds { + pub headers: bool, + pub blob: bool, +} + +pub struct EmailRender<'a, 'p> { + meta: &'a ArchivedMessageMetadata, + root: MessageView<'a>, + headers: Option<&'a [u8]>, + raw: Option>, + blob_id: &'p BlobId, + blob_prefix: usize, + body_properties: &'p [EmailProperty], + options: &'p BodyValueOptions, +} + +impl BodyValueOptions { + pub fn fetches_any(&self) -> bool { + self.fetch_text || self.fetch_html || self.fetch_all + } +} + +impl EmailNeeds { + pub fn new( + properties: &[EmailProperty], + body_properties: &[EmailProperty], + options: &BodyValueOptions, + ) -> Self { + let body_headers = body_properties + .iter() + .any(|property| matches!(property, EmailProperty::Header(_) | EmailProperty::Headers)); + let mut needs = EmailNeeds::default(); + for property in properties { + match property { + EmailProperty::Header(_) | EmailProperty::Headers | EmailProperty::References => { + needs.headers = true; + } + EmailProperty::TextBody + | EmailProperty::HtmlBody + | EmailProperty::Attachments + | EmailProperty::BodyStructure + if body_headers => + { + needs.headers = true; + needs.blob = true; + } + EmailProperty::BodyValues if options.fetches_any() => { + needs.blob = true; + } + _ => {} + } + } + needs + } +} + +impl<'a, 'p> EmailRender<'a, 'p> { + pub fn new( + meta: &'a ArchivedMessageMetadata, + headers: Option<&'a [u8]>, + blob: Option<&'a [u8]>, + blob_id: &'p BlobId, + body_properties: &'p [EmailProperty], + options: &'p BodyValueOptions, + ) -> Self { + let headers = headers.filter(|headers| headers.len() == meta.headers_len()); + EmailRender { + meta, + root: meta.root(), + headers, + raw: blob.map(|blob| meta.raw_message(headers, blob)), + blob_id, + blob_prefix: 0, + body_properties, + options, + } + } + + pub fn root_headers(&self) -> HeaderList<'a> { + self.root.root_part().headers() + } + + pub fn with_blob_prefix(mut self, prefix: usize) -> Self { + self.blob_prefix = prefix; + self + } + + pub fn renders(property: &EmailProperty) -> bool { + matches!( + property, + EmailProperty::BlobId + | EmailProperty::Preview + | EmailProperty::Subject + | EmailProperty::SentAt + | EmailProperty::MessageId + | EmailProperty::InReplyTo + | EmailProperty::References + | EmailProperty::Sender + | EmailProperty::From + | EmailProperty::To + | EmailProperty::Cc + | EmailProperty::Bcc + | EmailProperty::ReplyTo + | EmailProperty::Header(_) + | EmailProperty::Headers + | EmailProperty::TextBody + | EmailProperty::HtmlBody + | EmailProperty::Attachments + | EmailProperty::BodyStructure + | EmailProperty::BodyValues + ) + } + + pub fn value(&self, property: &EmailProperty) -> Option> { + let envelope = self.root.envelope(); + Some(match property { + EmailProperty::BlobId => Value::Element(EmailValue::BlobId(self.blob_id.clone())), + EmailProperty::Preview => Value::Str(Cow::Borrowed( + self.meta.stored_preview().unwrap_or_default(), + )), + EmailProperty::Subject => str_value(envelope.subject()), + EmailProperty::SentAt => envelope + .datetime() + .map_or(Value::Null, |date| date_value(&date)), + EmailProperty::MessageId => envelope.message_id().jmap_ids(), + EmailProperty::InReplyTo => envelope.in_reply_to().jmap_ids(), + EmailProperty::References => self.headers.map_or(Value::Null, |headers| { + self.root_headers() + .last(HeaderId::REFERENCES) + .and_then(|header| headers.field(header.value_range())) + .map_or(Value::Null, |raw| HeaderBytes(raw).message_ids()) + }), + EmailProperty::Sender => envelope + .addresses(AddressHeader::Sender, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::From => envelope + .addresses(AddressHeader::From, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::To => envelope + .addresses(AddressHeader::To, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::Cc => envelope + .addresses(AddressHeader::Cc, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::Bcc => envelope + .addresses(AddressHeader::Bcc, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::ReplyTo => envelope + .addresses(AddressHeader::ReplyTo, Occurrence::Last) + .jmap_mailboxes(), + EmailProperty::Header(_) | EmailProperty::Headers => { + self.headers.map_or(Value::Null, |headers| { + self.root_headers().jmap_value(property, headers) + }) + } + EmailProperty::TextBody => self.body_list_values(BodyList::Text), + EmailProperty::HtmlBody => self.body_list_values(BodyList::Html), + EmailProperty::Attachments => self.body_list_values(BodyList::Attachments), + EmailProperty::BodyStructure => self.body_part(self.root.root_part()), + EmailProperty::BodyValues => self.body_values(), + _ => return None, + }) + } +} + +fn str_value(text: Option<&str>) -> JmapValue<'_> { + text.map_or(Value::Null, |text| Value::Str(Cow::Borrowed(text))) +} + +impl<'a> TextItems<'a> { + pub fn jmap_ids(&self) -> JmapValue<'a> { + if self.is_empty() { + Value::Null + } else { + Value::Array( + self.iter() + .map(|item| Value::Str(Cow::Borrowed(item))) + .collect(), + ) + } + } +} + +impl<'a> Addresses<'a> { + pub fn jmap_mailboxes(&self) -> JmapValue<'a> { + if self.is_empty() { + return Value::Null; + } + Value::Array( + self.mailboxes() + .map(|mailbox| { + Value::Object( + Map::with_capacity(2) + .with_key_value(EmailProperty::Name, str_value(mailbox.name)) + .with_key_value( + EmailProperty::Email, + Value::Str(Cow::Borrowed(mailbox.address.unwrap_or_default())), + ), + ) + }) + .collect(), + ) + } +} + +fn date_value<'a>(date: &mail_parser::DateTime) -> JmapValue<'a> { + Value::Element(EmailValue::Date(UTCDate { + year: date.year, + month: date.month, + day: date.day, + hour: date.hour, + minute: date.minute, + second: date.second, + tz_before_gmt: date.tz_before_gmt, + tz_hour: date.tz_hour, + tz_minute: date.tz_minute, + })) +} diff --git a/crates/email/src/message/messagedata.rs b/crates/email/src/message/messagedata.rs new file mode 100644 index 00000000..52c5170f --- /dev/null +++ b/crates/email/src/message/messagedata.rs @@ -0,0 +1,801 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + cache::email::HAS_CUSTOM_KEYWORDS, + message::metadata::{ArchivedMessageMetadata, MetadataRow, MetadataStructure}, +}; +use common::{ + MessageUid, Server, + config::mailstore::limits::{EmailLimitError, EmailLimits}, + storage::batch::DocumentBatch, +}; +use compact_str::CompactString; +use std::borrow::Cow; +use store::{ + Deserialize, IterateParams, Key, Serialize, U32_LEN, U64_LEN, ValueKey, + dispatch::{DocumentSet, ScanGap}, + write::{ + AnyKey, AssignedIds, BatchBuilder, MergeResult, PendingId, SerializeWithIds, SetValue, + SizedSetValue, Slot, SlotRange, ValueClass, key::DeserializeBigEndian, serialize::RawValue, + }, +}; +use tinyvec::TinyVec; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + field::{EmailField, Field}, + keyword::{HASATTACHMENT, HASNOATTACHMENT, Keyword, OTHER}, +}; +use utils::codec::leb128::{Leb128Reader, Leb128Writer}; + +pub const SERVER_SET_KEYWORDS: u32 = (1 << HASATTACHMENT) | (1 << HASNOATTACHMENT); +pub const KEYWORD_BITS: u32 = (1 << OTHER) - 1; +pub const HAS_JMAP_METADATA: u32 = 1 << 30; +const METADATA_SCAN_GAP: ScanGap = ScanGap::new(1, 2); + +const _: () = assert!(KEYWORD_BITS & HAS_JMAP_METADATA == 0); + +#[derive(Debug, Clone)] +pub struct MessageData { + pub mailboxes: TinyVec<[MessageUid; 2]>, + pub keywords: u32, + pub keywords_extra: Vec, + pub thread_id: u32, + pub size: u32, + pub received_at: u64, + pub sent_at: i32, + pub change_id: u64, +} + +pub trait EmailMessageData: Sync + Send { + fn message_datas( + &self, + account_id: u32, + documents: &I, + cb: CB, + ) -> impl Future> + Send + where + I: DocumentSet + Send + Sync, + CB: FnMut(u32, MessageData) -> trc::Result + Send + Sync; + + fn message_metadata( + &self, + account_id: u32, + documents: &I, + cb: CB, + ) -> impl Future> + Send + where + I: DocumentSet + Send + Sync, + CB: FnMut(u32, &[u8]) -> trc::Result + Send + Sync; + + fn read_metadata_batch( + &self, + account_id: u32, + form: MetadataForm, + batch: &mut DocumentBatch, + ) -> impl Future> + Send; +} + +pub enum MetadataValue { + Structure(MetadataStructure), + Row(MetadataRow), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetadataForm { + Structure, + Row, +} + +impl EmailMessageData for Server { + async fn message_datas( + &self, + account_id: u32, + documents: &I, + mut cb: CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: FnMut(u32, MessageData) -> trc::Result + Send + Sync, + { + let collection: u8 = Collection::Email.into(); + let ranges = documents + .scan_ranges(self.core.storage.data.scan_gap()) + .into_iter() + .map(|(from_document_id, to_document_id)| { + IterateParams::new( + ValueKey { + account_id, + collection, + document_id: from_document_id, + class: ValueClass::Property(Field::ARCHIVE.into()), + }, + ValueKey { + account_id, + collection, + document_id: to_document_id, + class: ValueClass::Property(Field::ARCHIVE.into()), + }, + ) + }) + .collect::>(); + + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + MessageData::deserialize(value).and_then(|archive| cb(document_id, archive)) + } else { + Ok(true) + } + }; + + match ranges.len() { + 0 => Ok(()), + 1 => { + self.core + .storage + .data + .iterate(ranges.into_iter().next().unwrap(), &mut collect) + .await + } + _ => { + self.core + .storage + .data + .iterate_many(ranges, &mut collect) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } + + async fn message_metadata( + &self, + account_id: u32, + documents: &I, + mut cb: CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: FnMut(u32, &[u8]) -> trc::Result + Send + Sync, + { + let collection: u8 = Collection::Email.into(); + let class = ValueClass::Immutable(EmailField::Metadata.into()); + let subspace = class.subspace(collection); + let key = |document_id| ValueKey { + account_id, + collection, + document_id, + class: class.clone(), + }; + let params = |from_document_id, to_document_id| { + let begin = key(from_document_id).serialize(0); + let end = if from_document_id == to_document_id { + begin.clone() + } else { + let mut end = key(to_document_id).serialize(0); + end.push(u8::MAX); + end + }; + IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { subspace, key: end }, + ) + }; + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + cb(document_id, value) + } else { + Ok(true) + } + }; + let store = &self.core.storage.data; + + if documents.len() == 1 { + let document_id = documents.min(); + return store + .get_value::(key(document_id)) + .await + .and_then(|value| { + value.map_or(Ok(()), |RawValue(value)| { + cb(document_id, &value).map(|_| ()) + }) + }) + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }); + } + + let ranges = documents.scan_ranges(METADATA_SCAN_GAP); + match ranges.as_slice() { + [] => Ok(()), + &[(from_document_id, to_document_id)] => { + store + .iterate(params(from_document_id, to_document_id), &mut collect) + .await + } + _ => { + store + .iterate_many( + ranges + .iter() + .map(|&(from_document_id, to_document_id)| { + params(from_document_id, to_document_id) + }) + .collect(), + &mut collect, + ) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } + + async fn read_metadata_batch( + &self, + account_id: u32, + form: MetadataForm, + batch: &mut DocumentBatch, + ) -> trc::Result<()> { + let Some((pending, mut sink)) = batch.begin_read() else { + return Ok(()); + }; + if let &[document_id] = pending.as_slice() { + let key = ValueKey::immutable( + account_id, + Collection::Email, + document_id, + EmailField::Metadata, + ); + let store = self.store(); + let metadata = match form { + MetadataForm::Structure => store + .get_value::(key) + .await? + .map(MetadataValue::Structure), + MetadataForm::Row => store + .get_value::(key) + .await? + .map(MetadataValue::Row), + }; + if let Some(metadata) = metadata { + let bytes = metadata.decoded_len(); + sink.push(document_id, metadata, bytes); + } + } else { + self.message_metadata(account_id, pending, |document_id, value| { + MetadataValue::decode(value, form).map(|metadata| { + let bytes = metadata.decoded_len(); + sink.push(document_id, metadata, bytes) + }) + }) + .await?; + } + batch.end_read(); + Ok(()) + } +} + +impl MetadataValue { + pub fn decode(bytes: &[u8], form: MetadataForm) -> trc::Result { + match form { + MetadataForm::Structure => { + MetadataStructure::deserialize(bytes).map(MetadataValue::Structure) + } + MetadataForm::Row => MetadataRow::deserialize(bytes).map(MetadataValue::Row), + } + } + + pub fn unarchive(&self) -> trc::Result<&ArchivedMessageMetadata> { + match self { + MetadataValue::Structure(structure) => structure.unarchive(), + MetadataValue::Row(row) => row.unarchive(), + } + } + + pub fn decoded_len(&self) -> usize { + match self { + MetadataValue::Structure(structure) => structure.decoded_len(), + MetadataValue::Row(row) => row.decoded_len(), + } + } + + pub fn raw_headers(&self) -> trc::Result>> { + match self { + MetadataValue::Structure(_) => Ok(None), + MetadataValue::Row(row) => row.raw_headers().map(Some), + } + } +} + +impl MessageData { + pub fn sent_at_offset(sent_at: Option, received_at: u64) -> i32 { + sent_at.map_or(0, |sent_at| { + let received_at = i64::try_from(received_at).unwrap_or(i64::MAX); + sent_at + .saturating_sub(received_at) + .clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32 + }) + } + + pub fn set_keywords(&mut self, keywords: Vec) { + self.keywords &= SERVER_SET_KEYWORDS | !KEYWORD_BITS; + self.keywords_extra.clear(); + for keyword in keywords { + match keyword.into_id() { + Ok(id) => self.keywords |= 1 << id, + Err(name) => { + if !self + .keywords_extra + .iter() + .any(|k| k.as_str() == name.as_str()) + { + self.keywords_extra.push(name); + } + } + } + } + } + + pub fn add_keyword(&mut self, keyword: Keyword) -> bool { + match keyword.into_id() { + Ok(id) => { + let id = 1 << id; + if self.keywords & id == 0 { + self.keywords |= id; + true + } else { + false + } + } + Err(name) => { + if !self + .keywords_extra + .iter() + .any(|k| k.as_str() == name.as_str()) + { + self.keywords_extra.push(name); + true + } else { + false + } + } + } + } + + pub fn remove_keyword(&mut self, keyword: &Keyword) -> bool { + match keyword.id() { + Ok(id) => { + let id = 1 << id; + if self.keywords & id != 0 { + self.keywords &= !id; + true + } else { + false + } + } + Err(name) => { + let prev_len = self.keywords_extra.len(); + self.keywords_extra.retain(|k| k.as_str() != name); + self.keywords_extra.len() != prev_len + } + } + } + + pub fn set_mailboxes(&mut self, mailboxes: TinyVec<[MessageUid; 2]>) { + self.mailboxes = mailboxes; + } + + pub fn add_mailbox(&mut self, mailbox: MessageUid) { + if !self.has_mailbox_id(mailbox.mailbox_id) { + self.mailboxes.push(mailbox); + } + } + + pub fn remove_mailbox(&mut self, mailbox: u32) { + self.mailboxes.retain(|m| m.mailbox_id != mailbox); + } + + pub fn has_keyword(&self, keyword: &Keyword) -> bool { + match keyword.id() { + Ok(id) => (self.keywords & (1 << id)) != 0, + Err(name) => self.keywords_extra.iter().any(|k| k.as_str() == name), + } + } + + pub fn keyword_count(&self) -> usize { + (self.keywords & KEYWORD_BITS & !SERVER_SET_KEYWORDS).count_ones() as usize + + self.keywords_extra.len() + } + + pub fn validate_limits( + &self, + prev_data: &MessageData, + limits: &EmailLimits, + ) -> Result<(), EmailLimitError> { + limits.validate_mailbox_change(prev_data.mailboxes.len(), self.mailboxes.len())?; + limits.validate_keyword_change(prev_data.keyword_count(), self.keyword_count())?; + limits.validate_keyword_names( + self.keywords_extra + .iter() + .filter(|keyword| !prev_data.keywords_extra.contains(keyword)) + .map(|keyword| keyword.as_str()), + ) + } + + pub fn has_keyword_changes(&self, prev_data: &MessageData) -> bool { + (self.keywords ^ prev_data.keywords) & KEYWORD_BITS != 0 + || self.keywords_extra != prev_data.keywords_extra + } + + pub fn keyword_diff(&self, prev_data: &MessageData) -> KeywordDiff { + KeywordDiff::Patch { + added: self.keywords & !prev_data.keywords & KEYWORD_BITS, + removed: prev_data.keywords & !self.keywords & KEYWORD_BITS, + added_extra: self + .keywords_extra + .iter() + .filter(|k| !prev_data.keywords_extra.contains(*k)) + .cloned() + .collect(), + removed_extra: prev_data + .keywords_extra + .iter() + .filter(|k| !self.keywords_extra.contains(*k)) + .cloned() + .collect(), + } + } + + pub fn added_keywords(&self, prev_data: &MessageData) -> impl Iterator { + KeywordsIter(self.keywords & !prev_data.keywords & KEYWORD_BITS) + } + + pub fn removed_keywords(&self, prev_data: &MessageData) -> impl Iterator { + KeywordsIter(prev_data.keywords & !self.keywords & KEYWORD_BITS) + } + + pub fn keywords(&self) -> impl Iterator { + KeywordsIter(self.keywords & KEYWORD_BITS) + .chain(self.keywords_extra.iter().cloned().map(Keyword::Other)) + } + + pub fn added_mailboxes(&self, prev_data: &MessageData) -> impl Iterator { + self.mailboxes.iter().filter(|m| { + prev_data + .mailboxes + .iter() + .all(|pm| pm.mailbox_id != m.mailbox_id) + }) + } + + pub fn removed_mailboxes<'x>( + &'x self, + prev_data: &'x MessageData, + ) -> impl Iterator { + prev_data.mailboxes.iter().filter(|m| { + self.mailboxes + .iter() + .all(|pm| pm.mailbox_id != m.mailbox_id) + }) + } + + pub fn has_mailbox_changes(&self, prev_data: &MessageData) -> bool { + self.mailboxes.len() != prev_data.mailboxes.len() + || !self.mailboxes.iter().all(|m| { + prev_data + .mailboxes + .iter() + .any(|pm| pm.mailbox_id == m.mailbox_id) + }) + } + + pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { + self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + } + + pub fn message_uid(&self, mailbox_id: u32) -> Option { + self.mailboxes + .iter() + .find(|m| m.mailbox_id == mailbox_id) + .map(|m| m.uid) + } +} + +impl Serialize for MessageData { + fn serialize(&self) -> trc::Result> { + self.serialize_with_change_id(self.change_id) + } +} + +#[derive(Debug)] +pub enum KeywordDiff { + Replace(Vec), + Patch { + added: u32, + removed: u32, + added_extra: Vec, + removed_extra: Vec, + }, +} + +impl KeywordDiff { + pub fn replace(keywords: Vec) -> Self { + KeywordDiff::Replace(keywords) + } + + pub fn added(keyword: Keyword) -> Self { + let (added, added_extra) = match keyword.into_id() { + Ok(id) => (1 << id, Vec::new()), + Err(name) => (0, vec![name]), + }; + + KeywordDiff::Patch { + added, + removed: 0, + added_extra, + removed_extra: Vec::new(), + } + } + + pub fn keyword_bits(&self, keywords: u32) -> Option { + match self { + KeywordDiff::Replace(list) if keywords & HAS_CUSTOM_KEYWORDS == 0 => { + list.iter().try_fold( + keywords & (SERVER_SET_KEYWORDS | !KEYWORD_BITS), + |bits, keyword| keyword.id().ok().map(|id| bits | (1 << id)), + ) + } + KeywordDiff::Patch { + added, + removed, + added_extra, + removed_extra, + } if added_extra.is_empty() && removed_extra.is_empty() => { + Some((keywords | (added & KEYWORD_BITS)) & !(removed & KEYWORD_BITS)) + } + _ => None, + } + } + + fn apply(&self, data: &mut MessageData) -> bool { + let prev_keywords = data.keywords; + + match self { + KeywordDiff::Replace(keywords) => { + let prev_extra = data.keywords_extra.clone(); + data.set_keywords(keywords.clone()); + + data.keywords != prev_keywords || data.keywords_extra != prev_extra + } + KeywordDiff::Patch { + added, + removed, + added_extra, + removed_extra, + } => { + let prev_extra_len = data.keywords_extra.len(); + + data.keywords = + (data.keywords | (added & KEYWORD_BITS)) & !(removed & KEYWORD_BITS); + if !removed_extra.is_empty() { + data.keywords_extra.retain(|k| !removed_extra.contains(k)); + } + + let mut extra_changed = data.keywords_extra.len() != prev_extra_len; + for keyword in added_extra { + if !data.keywords_extra.contains(keyword) { + data.keywords_extra.push(keyword.clone()); + extra_changed = true; + } + } + + data.keywords != prev_keywords || extra_changed + } + } + } +} + +pub fn merge_keywords(batch: &mut BatchBuilder, thread_id: u32, diff: KeywordDiff) { + batch.log_item_update(SyncCollection::Email, Some(PendingId::Assigned(thread_id))); + batch.merge_fnc(Field::ARCHIVE, move |ids, bytes| { + let Some(bytes) = bytes else { + return Ok(MergeResult::Skip); + }; + + let mut data = MessageData::deserialize(bytes)?; + if !diff.apply(&mut data) { + return Ok(MergeResult::Skip); + } + + data.serialize_with_change_id(ids.current_change_id()) + .map(MergeResult::Update) + }); +} + +impl MessageData { + pub fn serialize_with_change_id(&self, change_id: u64) -> trc::Result> { + Ok(self.serialize_resolved(change_id, None, SlotRange::default(), None)) + } + + pub fn size_hint(&self) -> usize { + std::mem::size_of::() + + (std::mem::size_of::() * self.mailboxes.len().saturating_sub(2)) + + (self + .keywords_extra + .iter() + .map(|k| k.len() + 1) + .sum::()) + } + + pub fn pending_uids(&self) -> usize { + self.mailboxes.iter().filter(|mb| mb.uid == 0).count() + } + + fn serialize_resolved( + &self, + change_id: u64, + ids: Option<&AssignedIds>, + uid_slots: SlotRange, + thread_slot: Option, + ) -> Vec { + let mut out = Vec::with_capacity(self.size_hint()); + let mut uid_slots = uid_slots.iter(); + + out.push_leb128(self.mailboxes.len()); + for mb in self.mailboxes.iter() { + out.push_leb128(mb.mailbox_id); + let uid = match ids { + Some(ids) if mb.uid == 0 => uid_slots.next().map_or(mb.uid, |slot| ids.slot(slot)), + _ => mb.uid, + }; + out.push_leb128(uid); + } + + out.push_leb128(match (ids, thread_slot) { + (Some(ids), Some(thread_slot)) => ids.slot(thread_slot), + _ => self.thread_id, + }); + + out.push_leb128(self.size); + out.push_leb128(self.received_at); + out.push_leb128(((self.sent_at << 1) ^ (self.sent_at >> 31)) as u32); + + out.extend_from_slice(&self.keywords.to_be_bytes()); + + for s in self.keywords_extra.iter() { + out.push_leb128(s.len()); + out.extend_from_slice(s.as_bytes()); + } + + out.extend_from_slice(&change_id.to_be_bytes()); + + out + } +} + +pub struct PendingMessageData { + pub data: MessageData, + pub uid_slots: SlotRange, + pub thread_slot: Option, + pub change_id: Option, +} + +impl SerializeWithIds for PendingMessageData { + fn serialize_with_ids(&mut self, ids: &AssignedIds) -> trc::Result<(Vec, Option)> { + let change_id = match self.change_id { + Some(change_id) => change_id, + None => ids.try_current_change_id().ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx( + trc::Key::Reason, + "No change id was allocated for this message", + ) + })?, + }; + + Ok(( + self.data + .serialize_resolved(change_id, Some(ids), self.uid_slots, self.thread_slot), + None, + )) + } + + fn size_hint(&self) -> usize { + self.data.size_hint() + } +} + +impl From for SizedSetValue { + fn from(data: PendingMessageData) -> Self { + SetValue::serializable(data) + } +} + +impl Deserialize for MessageData { + fn deserialize(data: &[u8]) -> trc::Result { + deserialize(data) + .ok_or_else(|| trc::Error::corrupted_key(b"", data.into(), trc::location!())) + } +} + +#[inline(always)] +fn deserialize(mut bytes: &[u8]) -> Option { + let num_mailboxes: usize = bytes.read_leb128()?; + let mut data = MessageData { + mailboxes: TinyVec::with_capacity(num_mailboxes), + keywords: 0, + keywords_extra: Vec::new(), + thread_id: 0, + size: 0, + received_at: 0, + sent_at: 0, + change_id: 0, + }; + for _ in 0..num_mailboxes { + let mailbox_id: u32 = bytes.read_leb128()?; + let uid: u32 = bytes.read_leb128()?; + data.mailboxes.push(MessageUid { mailbox_id, uid }); + } + + data.thread_id = bytes.read_leb128()?; + data.size = bytes.read_leb128()?; + data.received_at = bytes.read_leb128()?; + data.sent_at = { + let v: u32 = bytes.read_leb128()?; + ((v >> 1) as i32) ^ -((v & 1) as i32) + }; + + let (keywords, mut bytes) = bytes.split_first_chunk::()?; + data.keywords = u32::from_be_bytes(*keywords); + + while bytes.len() > U64_LEN { + let len: usize = bytes.read_leb128()?; + let (text, rest) = bytes.split_at_checked(len)?; + data.keywords_extra + .push(CompactString::from(std::str::from_utf8(text).ok()?)); + bytes = rest; + } + + data.change_id = u64::from_be_bytes(*bytes.first_chunk::()?); + + Some(data) +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct KeywordsIter(pub u32); + +impl Iterator for KeywordsIter { + type Item = Keyword; + + fn next(&mut self) -> Option { + while self.0 != 0 { + let item = 31 - self.0.leading_zeros(); + self.0 ^= 1 << item; + if let Ok(keyword) = Keyword::try_from_id(item as usize) { + return Some(keyword); + } + } + None + } +} + diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs deleted file mode 100644 index 1110fc6b..00000000 --- a/crates/email/src/message/metadata.rs +++ /dev/null @@ -1,1320 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::mailbox::{ArchivedUidMailbox, UidMailbox}; -use common::storage::index::IndexableAndSerializableObject; -use mail_parser::{ - Addr, Address, Attribute, ContentType, DateTime, Encoding, Group, HeaderName, HeaderValue, - PartType, - decoders::{ - base64::base64_decode, charsets::map::charset_decoder, - quoted_printable::quoted_printable_decode, - }, -}; -use rkyv::{boxed::ArchivedBox, rend::u16_le}; -use std::{borrow::Cow, collections::VecDeque, ops::Range}; -use types::{ - blob_hash::BlobHash, - keyword::{ArchivedKeyword, Keyword}, -}; -use utils::chained_bytes::ChainedBytes; - -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -pub struct MessageData { - pub mailboxes: Box<[UidMailbox]>, - pub keywords: Box<[Keyword]>, - pub thread_id: u32, - pub size: u32, -} - -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -pub struct MessageMetadata { - pub contents: Box<[MessageMetadataContents]>, - pub rcvd_attach: u64, - pub blob_hash: BlobHash, - pub blob_body_offset: u32, - pub preview: Box, - pub raw_headers: Box<[u8]>, -} - -pub const MESSAGE_HAS_ATTACHMENT: u64 = 1 << 63; -pub const MESSAGE_RECEIVED_MASK: u64 = !MESSAGE_HAS_ATTACHMENT; - -impl IndexableAndSerializableObject for MessageData { - fn is_versioned() -> bool { - true - } -} - -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -pub struct MessageMetadataContents { - pub html_body: Box<[u16]>, - pub text_body: Box<[u16]>, - pub attachments: Box<[u16]>, - pub parts: Box<[MessageMetadataPart]>, -} - -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -pub struct MessageMetadataPart { - pub headers: Box<[MetadataHeader]>, - pub body: MetadataPartType, - pub flags: u32, - pub offset_header: u32, - pub offset_body: u32, - pub offset_end: u32, -} - -pub const PART_ENCODING_BASE64: u32 = 1 << 31; -pub const PART_ENCODING_QP: u32 = 1 << 30; -pub const PART_ENCODING_PROBLEM: u32 = 1 << 29; -pub const PART_SIZE_MASK: u32 = !(PART_ENCODING_BASE64 | PART_ENCODING_QP | PART_ENCODING_PROBLEM); - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataHeader { - pub name: MetadataHeaderName, - pub value: MetadataHeaderValue, - pub base_offset: u32, - pub start: u16, - pub end: u16, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub enum MetadataHeaderName { - Other(Box), - Subject, - From, - To, - Cc, - Date, - Bcc, - ReplyTo, - Sender, - Comments, - InReplyTo, - Keywords, - Received, - MessageId, - References, - ReturnPath, - MimeVersion, - ContentDescription, - ContentId, - ContentLanguage, - ContentLocation, - ContentTransferEncoding, - ContentType, - ContentDisposition, - ResentTo, - ResentFrom, - ResentBcc, - ResentCc, - ResentSender, - ResentDate, - ResentMessageId, - ListArchive, - ListHelp, - ListId, - ListOwner, - ListPost, - ListSubscribe, - ListUnsubscribe, - DkimSignature, - ArcAuthenticationResults, - ArcMessageSignature, - ArcSeal, - - // Delivery/Routing - DeliveredTo, - XOriginalTo, - ReturnReceiptTo, - DispositionNotificationTo, - ErrorsTo, - - // Authentication - AuthenticationResults, - ReceivedSpf, - - // Spam/Virus - XSpamStatus, - XSpamScore, - XSpamFlag, - XSpamResult, - - // Priority - Importance, - Priority, - XPriority, - XMSMailPriority, - - // Client/Agent - XMailer, - UserAgent, - XMimeOLE, - - // Network/Origin - XOriginatingIp, - XForwardedTo, - XForwardedFor, - - // Auto-response - AutoSubmitted, - XAutoResponseSuppress, - Precedence, - - // Organization/Threading - Organization, - ThreadIndex, - ThreadTopic, - - // List (additional) - ListUnsubscribePost, - FeedbackId, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub enum MetadataHeaderValue { - AddressList(Box<[MetadataAddress]>), - AddressGroup(Box<[MetadataAddressGroup]>), - Text(Box), - TextList(Box<[Box]>), - DateTime(MetadataDateTime), - ContentType(MetadataContentType), - Empty, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataDateTime { - pub year: u16, - pub month: u8, - pub day: u8, - pub hour: u8, - pub minute: u8, - pub second: u8, - pub tz_hour: i8, - pub tz_minute: u8, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataAddress { - pub name: Option>, - pub address: Option>, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataAddressGroup { - pub name: Option>, - pub addresses: Box<[MetadataAddress]>, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataContentType { - pub c_type: Box, - pub c_subtype: Option>, - pub attributes: Box<[MetadataAttribute]>, -} - -#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -#[rkyv(compare(PartialEq))] -pub struct MetadataAttribute { - pub name: Box, - pub value: Box, -} - -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -pub enum MetadataPartType { - Text, - Html, - Binary, - InlineBinary, - Message(u16), - Multipart(Box<[u16]>), -} - -impl MessageMetadataContents { - pub fn root_part(&self) -> &MessageMetadataPart { - &self.parts[0] - } -} - -#[derive(Debug)] -pub struct DecodedParts<'x> { - pub raw_messages: Vec>, - pub parts: Vec>, -} - -#[derive(Debug)] -pub enum DecodedRawMessage<'x> { - Borrowed(ChainedBytes<'x>), - Owned(Vec), -} - -#[derive(Debug)] -pub struct DecodedPart<'x> { - pub message_id: usize, - pub part_offset: usize, - pub content: DecodedPartContent<'x>, -} - -#[derive(Debug)] -pub enum DecodedPartContent<'x> { - Text(Cow<'x, str>), - Binary(Cow<'x, [u8]>), -} - -impl<'x> DecodedParts<'x> { - #[inline] - pub fn raw_message(&self, message_id: usize) -> Option<&DecodedRawMessage<'x>> { - self.raw_messages.get(message_id) - } - - #[inline] - pub fn raw_message_section( - &'_ self, - message_id: usize, - range: Range, - ) -> Option> { - self.raw_messages.get(message_id).and_then(|m| m.get(range)) - } - - #[inline] - pub fn part(&self, message_id: usize, part_offset: usize) -> Option<&DecodedPartContent<'x>> { - self.parts - .iter() - .find(|p| p.message_id == message_id && p.part_offset == part_offset) - .map(|p| &p.content) - } - - #[inline] - pub fn text_part(&self, message_id: usize, part_offset: usize) -> Option<&str> { - self.part(message_id, part_offset).and_then(|p| match p { - DecodedPartContent::Text(text) => Some(text.as_ref()), - DecodedPartContent::Binary(_) => None, - }) - } - - #[inline] - pub fn binary_part(&self, message_id: usize, part_offset: usize) -> Option<&[u8]> { - self.part(message_id, part_offset).map(|p| match p { - DecodedPartContent::Text(part) => part.as_bytes(), - DecodedPartContent::Binary(binary) => binary.as_ref(), - }) - } - - #[inline] - pub fn transfer_decoded_contents( - &self, - message_id: usize, - part: &ArchivedMessageMetadataPart, - ) -> Option> { - match self.raw_messages.get(message_id)? { - DecodedRawMessage::Borrowed(chain) => Some(part.contents(chain)), - DecodedRawMessage::Owned(vec) => Some(Cow::Owned( - part.contents(&ChainedBytes::new(vec)).into_owned(), - )), - } - } -} - -impl DecodedPartContent<'_> { - pub fn as_bytes(&self) -> &[u8] { - match self { - DecodedPartContent::Text(text) => text.as_bytes(), - DecodedPartContent::Binary(binary) => binary, - } - } - - #[allow(clippy::len_without_is_empty)] - pub fn len(&self) -> usize { - match self { - DecodedPartContent::Text(text) => text.len(), - DecodedPartContent::Binary(binary) => binary.len(), - } - } - - pub fn as_str(&self) -> &str { - match self { - DecodedPartContent::Text(text) => text, - DecodedPartContent::Binary(binary) => std::str::from_utf8(binary).unwrap_or_default(), - } - } -} - -impl<'x> DecodedRawMessage<'x> { - pub fn get(&'_ self, index: Range) -> Option> { - match self { - DecodedRawMessage::Borrowed(bytes) => bytes.get(index), - DecodedRawMessage::Owned(vec) => vec.get(index).map(Cow::Borrowed), - } - } -} - -impl ArchivedMessageMetadata { - #[inline(always)] - pub fn message_id(&self, message_id: u16_le) -> &ArchivedMessageMetadataContents { - &self.contents[u16::from(message_id) as usize] - } - - pub fn decode_contents<'x>(&self, raw: ChainedBytes<'x>) -> DecodedParts<'x> { - let mut result = DecodedParts { - raw_messages: Vec::with_capacity(self.contents.len()), - parts: Vec::new(), - }; - - for _ in 0..self.contents.len() { - result - .raw_messages - .push(DecodedRawMessage::Borrowed(raw.clone())); - } - - for (message_id, contents) in self.contents.iter().enumerate() { - for part in contents.parts.iter() { - let part_offset = u32::from(part.offset_header) as usize; - match &part.body { - ArchivedMetadataPartType::Text - | ArchivedMetadataPartType::Html - | ArchivedMetadataPartType::Binary - | ArchivedMetadataPartType::InlineBinary => { - match result.raw_messages.get(message_id).unwrap() { - DecodedRawMessage::Borrowed(bytes) => { - result.parts.push(DecodedPart { - message_id, - part_offset, - content: part.decode_contents(bytes), - }); - } - DecodedRawMessage::Owned(bytes) => { - result.parts.push(DecodedPart { - message_id, - part_offset, - content: match part.decode_contents(&ChainedBytes::new(bytes)) { - DecodedPartContent::Text(text) => { - DecodedPartContent::Text(text.into_owned().into()) - } - DecodedPartContent::Binary(binary) => { - DecodedPartContent::Binary(binary.into_owned().into()) - } - }, - }); - } - } - } - ArchivedMetadataPartType::Message(nested_message_id) => { - let sub_contents = - if (part.flags & (PART_ENCODING_BASE64 | PART_ENCODING_QP)) != 0 { - match result.raw_messages.get(message_id).unwrap() { - DecodedRawMessage::Borrowed(bytes) => { - part.contents(bytes).into_owned() - } - DecodedRawMessage::Owned(bytes) => { - let bytes = ChainedBytes::new(bytes); - part.contents(&bytes).into_owned() - } - } - } else if let Some(DecodedRawMessage::Owned(bytes)) = - result.raw_messages.get(message_id) - { - bytes.clone() - } else { - continue; - }; - - result.raw_messages[usize::from(*nested_message_id)] = - DecodedRawMessage::Owned(sub_contents); - } - _ => {} - } - } - } - - result - } -} - -impl ArchivedMessageMetadataPart { - pub fn contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> Cow<'x, [u8]> { - let bytes = raw_message.get(self.body_to_end()).unwrap_or_default(); - - if (self.flags & PART_ENCODING_BASE64) != 0 { - base64_decode(bytes.as_ref()).unwrap_or_default().into() - } else if (self.flags & PART_ENCODING_QP) != 0 { - quoted_printable_decode(bytes.as_ref()) - .unwrap_or_default() - .into() - } else { - bytes - } - } - - #[inline(always)] - pub fn body_to_end(&self) -> Range { - (self.offset_body.to_native() as usize)..(self.offset_end.to_native() as usize) - } - - #[inline(always)] - pub fn header_to_end(&self) -> Range { - self.offset_header.to_native() as usize..self.offset_end.to_native() as usize - } - - #[inline(always)] - pub fn header_to_body(&self) -> Range { - self.offset_header.to_native() as usize..self.offset_body.to_native() as usize - } - - pub fn decode_contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> DecodedPartContent<'x> { - let bytes = self.contents(raw_message); - - match self.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - DecodedPartContent::Text( - match ( - bytes, - self.header_value(&MetadataHeaderName::ContentType) - .and_then(|c| c.as_content_type()) - .and_then(|ct| { - ct.attribute("charset") - .and_then(|c| charset_decoder(c.as_bytes())) - }), - ) { - (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(), - (Cow::Owned(vec), None) => String::from_utf8(vec) - .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) - .into(), - (Cow::Borrowed(bytes), Some(charset_decoder)) => { - charset_decoder(bytes).into() - } - (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes), - }, - ) - } - ArchivedMetadataPartType::Binary => DecodedPartContent::Binary(bytes), - ArchivedMetadataPartType::InlineBinary => DecodedPartContent::Binary(bytes), - ArchivedMetadataPartType::Message(_) | ArchivedMetadataPartType::Multipart(_) => { - unreachable!() - } - } - } -} - -pub fn build_metadata_contents( - message: mail_parser::Message<'_>, -) -> Box<[MessageMetadataContents]> { - let mut messages = VecDeque::from([message]); - let mut message_id = 0; - let mut contents = Vec::new(); - - while let Some(message) = messages.pop_front() { - let mut parts = Vec::with_capacity(message.parts.len()); - - for part in message.parts { - let (size, body) = match part.body { - PartType::Text(contents) => (contents.len(), MetadataPartType::Text), - PartType::Html(contents) => (contents.len(), MetadataPartType::Html), - PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), - PartType::InlineBinary(contents) => { - (contents.len(), MetadataPartType::InlineBinary) - } - PartType::Message(message) => { - let message_len = message.root_part().raw_len(); - messages.push_back(message); - message_id += 1; - - (message_len as usize, MetadataPartType::Message(message_id)) - } - PartType::Multipart(parts) => ( - 0, - MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()), - ), - }; - - let flags = match part.encoding { - Encoding::None => 0, - Encoding::QuotedPrintable => PART_ENCODING_QP, - Encoding::Base64 => PART_ENCODING_BASE64, - } | (if part.is_encoding_problem { - PART_ENCODING_PROBLEM - } else { - 0 - }) | (size as u32 & PART_SIZE_MASK); - - parts.push(MessageMetadataPart { - headers: part - .headers - .into_iter() - .map(|hdr| MetadataHeader { - value: if matches!( - &hdr.name, - HeaderName::Subject - | HeaderName::From - | HeaderName::To - | HeaderName::Cc - | HeaderName::Date - | HeaderName::Bcc - | HeaderName::ReplyTo - | HeaderName::Sender - | HeaderName::Comments - | HeaderName::InReplyTo - | HeaderName::Keywords - | HeaderName::MessageId - | HeaderName::References - | HeaderName::ResentMessageId - | HeaderName::ContentDescription - | HeaderName::ContentId - | HeaderName::ContentLanguage - | HeaderName::ContentLocation - | HeaderName::ContentTransferEncoding - | HeaderName::ContentType - | HeaderName::ContentDisposition - | HeaderName::ListId - ) { - hdr.value - } else { - HeaderValue::Empty - } - .into(), - name: hdr.name.into(), - base_offset: hdr.offset_field, - start: (hdr.offset_start - hdr.offset_field) as u16, - end: (hdr.offset_end - hdr.offset_field) as u16, - }) - .collect(), - body, - flags, - offset_header: part.offset_header, - offset_body: part.offset_body, - offset_end: part.offset_end, - }); - } - contents.push(MessageMetadataContents { - html_body: message.html_body.into_iter().map(|c| c as u16).collect(), - text_body: message.text_body.into_iter().map(|c| c as u16).collect(), - attachments: message.attachments.into_iter().map(|c| c as u16).collect(), - parts: parts.into_boxed_slice(), - }); - } - contents.into_boxed_slice() -} - -impl ArchivedMessageMetadataPart { - pub fn is_message(&self) -> bool { - matches!(self.body, ArchivedMetadataPartType::Message(_)) - } - - pub fn sub_parts(&self) -> Option<&ArchivedBox<[u16_le]>> { - if let ArchivedMetadataPartType::Multipart(parts) = &self.body { - Some(parts) - } else { - None - } - } - - pub fn raw_len(&self) -> usize { - (u32::from(self.offset_end)).saturating_sub(u32::from(self.offset_header)) as usize - } - - pub fn header_values( - &self, - name: &MetadataHeaderName, - ) -> impl Iterator + Sync + Send { - self.headers.iter().filter_map(move |header| { - if &header.name == name { - Some(&header.value) - } else { - None - } - }) - } - - pub fn header_value(&self, name: &MetadataHeaderName) -> Option<&ArchivedMetadataHeaderValue> { - self.headers.iter().rev().find_map(move |header| { - if &header.name == name { - Some(&header.value) - } else { - None - } - }) - } - - pub fn from(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::From) - .and_then(|header| header.as_single_address()) - .and_then(|addr| addr.address.as_deref()) - } - - pub fn subject(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::Subject) - .and_then(|header| header.as_text()) - } - - pub fn date(&self) -> Option { - self.header_value(&MetadataHeaderName::Date) - .and_then(|header| header.as_datetime()) - .map(|dt| dt.into()) - } - - pub fn message_id(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::MessageId) - .and_then(|header| header.as_text()) - } - - pub fn in_reply_to(&self) -> &ArchivedMetadataHeaderValue { - self.header_value(&MetadataHeaderName::InReplyTo) - .unwrap_or(&ArchivedMetadataHeaderValue::Empty) - } - - pub fn content_description(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::ContentDescription) - .and_then(|header| header.as_text()) - } - - pub fn content_disposition(&self) -> Option<&ArchivedMetadataContentType> { - self.header_value(&MetadataHeaderName::ContentDisposition) - .and_then(|header| header.as_content_type()) - } - - pub fn content_id(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::ContentId) - .and_then(|header| header.as_text()) - } - - pub fn content_transfer_encoding(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::ContentTransferEncoding) - .and_then(|header| header.as_text()) - } - - pub fn content_type(&self) -> Option<&ArchivedMetadataContentType> { - self.header_value(&MetadataHeaderName::ContentType) - .and_then(|header| header.as_content_type()) - } - - pub fn is_text_mime_type(&self) -> bool { - matches!( - self.body, - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html - ) && self - .content_type() - .is_none_or(|ct| ct.ctype().eq_ignore_ascii_case("text")) - } - - pub fn content_language(&self) -> &ArchivedMetadataHeaderValue { - self.header_value(&MetadataHeaderName::ContentLanguage) - .unwrap_or(&ArchivedMetadataHeaderValue::Empty) - } - - pub fn content_location(&self) -> Option<&str> { - self.header_value(&MetadataHeaderName::ContentLocation) - .and_then(|header| header.as_text()) - } - - pub fn attachment_name(&self) -> Option<&str> { - self.content_disposition() - .and_then(|cd| cd.attribute("filename")) - .or_else(|| self.content_type().and_then(|ct| ct.attribute("name"))) - } -} - -impl From> for MetadataHeaderName { - fn from(value: HeaderName<'_>) -> Self { - match value { - HeaderName::Subject => MetadataHeaderName::Subject, - HeaderName::From => MetadataHeaderName::From, - HeaderName::To => MetadataHeaderName::To, - HeaderName::Cc => MetadataHeaderName::Cc, - HeaderName::Date => MetadataHeaderName::Date, - HeaderName::Bcc => MetadataHeaderName::Bcc, - HeaderName::ReplyTo => MetadataHeaderName::ReplyTo, - HeaderName::Sender => MetadataHeaderName::Sender, - HeaderName::Comments => MetadataHeaderName::Comments, - HeaderName::InReplyTo => MetadataHeaderName::InReplyTo, - HeaderName::Keywords => MetadataHeaderName::Keywords, - HeaderName::Received => MetadataHeaderName::Received, - HeaderName::MessageId => MetadataHeaderName::MessageId, - HeaderName::References => MetadataHeaderName::References, - HeaderName::ReturnPath => MetadataHeaderName::ReturnPath, - HeaderName::MimeVersion => MetadataHeaderName::MimeVersion, - HeaderName::ContentDescription => MetadataHeaderName::ContentDescription, - HeaderName::ContentId => MetadataHeaderName::ContentId, - HeaderName::ContentLanguage => MetadataHeaderName::ContentLanguage, - HeaderName::ContentLocation => MetadataHeaderName::ContentLocation, - HeaderName::ContentTransferEncoding => MetadataHeaderName::ContentTransferEncoding, - HeaderName::ContentType => MetadataHeaderName::ContentType, - HeaderName::ContentDisposition => MetadataHeaderName::ContentDisposition, - HeaderName::ResentTo => MetadataHeaderName::ResentTo, - HeaderName::ResentFrom => MetadataHeaderName::ResentFrom, - HeaderName::ResentBcc => MetadataHeaderName::ResentBcc, - HeaderName::ResentCc => MetadataHeaderName::ResentCc, - HeaderName::ResentSender => MetadataHeaderName::ResentSender, - HeaderName::ResentDate => MetadataHeaderName::ResentDate, - HeaderName::ResentMessageId => MetadataHeaderName::ResentMessageId, - HeaderName::ListArchive => MetadataHeaderName::ListArchive, - HeaderName::ListHelp => MetadataHeaderName::ListHelp, - HeaderName::ListId => MetadataHeaderName::ListId, - HeaderName::ListOwner => MetadataHeaderName::ListOwner, - HeaderName::ListPost => MetadataHeaderName::ListPost, - HeaderName::ListSubscribe => MetadataHeaderName::ListSubscribe, - HeaderName::ListUnsubscribe => MetadataHeaderName::ListUnsubscribe, - HeaderName::DkimSignature => MetadataHeaderName::DkimSignature, - HeaderName::ArcAuthenticationResults => MetadataHeaderName::ArcAuthenticationResults, - HeaderName::ArcMessageSignature => MetadataHeaderName::ArcMessageSignature, - HeaderName::ArcSeal => MetadataHeaderName::ArcSeal, - - // Delivery/Routing - HeaderName::DeliveredTo => MetadataHeaderName::DeliveredTo, - HeaderName::DispositionNotificationTo => MetadataHeaderName::DispositionNotificationTo, - HeaderName::ErrorsTo => MetadataHeaderName::ErrorsTo, - - // Authentication - HeaderName::AuthenticationResults => MetadataHeaderName::AuthenticationResults, - HeaderName::ReceivedSpf => MetadataHeaderName::ReceivedSpf, - - // Priority - HeaderName::Importance => MetadataHeaderName::Importance, - HeaderName::Priority => MetadataHeaderName::Priority, - - // Auto-response - HeaderName::AutoSubmitted => MetadataHeaderName::AutoSubmitted, - - // Organization/Threading - HeaderName::Organization => MetadataHeaderName::Organization, - - // List (additional) - HeaderName::ListUnsubscribePost => MetadataHeaderName::ListUnsubscribePost, - - HeaderName::Other(value) => { - let name = hashify::tiny_map_ignore_case!(value.as_bytes(), - // Delivery/Routing - "X-Original-To" => MetadataHeaderName::XOriginalTo, - "Return-Receipt-To" => MetadataHeaderName::ReturnReceiptTo, - - // Spam/Virus - "X-Spam-Status" => MetadataHeaderName::XSpamStatus, - "X-Spam-Score" => MetadataHeaderName::XSpamScore, - "X-Spam-Flag" => MetadataHeaderName::XSpamFlag, - "X-Spam-Result" => MetadataHeaderName::XSpamResult, - - // Priority - "X-Priority" => MetadataHeaderName::XPriority, - "X-MSMail-Priority" => MetadataHeaderName::XMSMailPriority, - - // Client/Agent - "X-Mailer" => MetadataHeaderName::XMailer, - "User-Agent" => MetadataHeaderName::UserAgent, - "X-MimeOLE" => MetadataHeaderName::XMimeOLE, - - // Network/Origin - "X-Originating-IP" => MetadataHeaderName::XOriginatingIp, - "X-Forwarded-To" => MetadataHeaderName::XForwardedTo, - "X-Forwarded-For" => MetadataHeaderName::XForwardedFor, - - // Auto-response - "X-Auto-Response-Suppress" => MetadataHeaderName::XAutoResponseSuppress, - "Precedence" => MetadataHeaderName::Precedence, - - // Organization/Threading - "Thread-Index" => MetadataHeaderName::ThreadIndex, - "Thread-Topic" => MetadataHeaderName::ThreadTopic, - - // List (additional) - "Feedback-ID" => MetadataHeaderName::FeedbackId, - ); - name.unwrap_or_else(|| { - MetadataHeaderName::Other(value.into_owned().into_boxed_str()) - }) - } - other => MetadataHeaderName::Other(other.as_str().to_string().into_boxed_str()), - } - } -} - -impl From> for MetadataHeaderValue { - fn from(value: HeaderValue<'_>) -> Self { - match value { - HeaderValue::Address(address) => match address { - Address::List(address) => MetadataHeaderValue::AddressList( - address - .into_iter() - .map(|a| MetadataAddress { - name: a.name.map(|a| a.into_owned().into_boxed_str()), - address: a.address.map(|a| a.into_owned().into_boxed_str()), - }) - .collect(), - ), - Address::Group(groups) => MetadataHeaderValue::AddressGroup( - groups - .into_iter() - .map(|g| MetadataAddressGroup { - name: g.name.map(|a| a.into_owned().into_boxed_str()), - addresses: g - .addresses - .into_iter() - .map(|a| MetadataAddress { - name: a.name.map(|a| a.into_owned().into_boxed_str()), - address: a.address.map(|a| a.into_owned().into_boxed_str()), - }) - .collect(), - }) - .collect(), - ), - }, - HeaderValue::Text(text) => { - MetadataHeaderValue::Text(text.into_owned().into_boxed_str()) - } - HeaderValue::TextList(texts) => MetadataHeaderValue::TextList( - texts - .into_iter() - .map(|v| v.into_owned().into_boxed_str()) - .collect(), - ), - HeaderValue::DateTime(dt) => MetadataHeaderValue::DateTime(MetadataDateTime { - year: dt.year, - month: dt.month, - day: dt.day, - hour: dt.hour, - minute: dt.minute, - second: dt.second, - tz_hour: (if dt.tz_before_gmt { -1 } else { 1 }) * dt.tz_hour as i8, - tz_minute: dt.tz_minute, - }), - HeaderValue::ContentType(ct) => MetadataHeaderValue::ContentType(MetadataContentType { - c_type: ct.c_type.into_owned().into_boxed_str(), - c_subtype: ct.c_subtype.map(|v| v.into_owned().into_boxed_str()), - attributes: ct - .attributes - .unwrap_or_default() - .into_iter() - .map(|a| MetadataAttribute { - name: a.name.into_owned().into_boxed_str(), - value: a.value.into_owned().into_boxed_str(), - }) - .collect(), - }), - HeaderValue::Received(_) | HeaderValue::Empty => MetadataHeaderValue::Empty, - } - } -} - -impl From<&ArchivedMetadataDateTime> for DateTime { - fn from(dt: &ArchivedMetadataDateTime) -> Self { - DateTime { - year: dt.year.to_native(), - month: dt.month, - day: dt.day, - hour: dt.hour, - minute: dt.minute, - second: dt.second, - tz_before_gmt: dt.tz_hour < 0, - tz_hour: dt.tz_hour.unsigned_abs(), - tz_minute: dt.tz_minute, - } - } -} - -impl ArchivedMessageMetadataContents { - pub fn root_part(&self) -> &ArchivedMessageMetadataPart { - &self.parts[0] - } -} - -#[derive(Default)] -pub struct MessageDataBuilder { - pub mailboxes: Vec, - pub keywords: Vec, - pub thread_id: u32, - pub size: u32, -} - -impl MessageDataBuilder { - pub fn set_keywords(&mut self, keywords: Vec) { - self.keywords = keywords; - } - - pub fn add_keyword(&mut self, keyword: Keyword) -> bool { - if !self.keywords.contains(&keyword) { - self.keywords.push(keyword); - true - } else { - false - } - } - - pub fn remove_keyword(&mut self, keyword: &Keyword) -> bool { - let prev_len = self.keywords.len(); - self.keywords.retain(|k| k != keyword); - self.keywords.len() != prev_len - } - - pub fn set_mailboxes(&mut self, mailboxes: Vec) { - self.mailboxes = mailboxes; - } - - pub fn add_mailbox(&mut self, mailbox: UidMailbox) { - if !self.mailboxes.contains(&mailbox) { - self.mailboxes.push(mailbox); - } - } - - pub fn remove_mailbox(&mut self, mailbox: u32) { - self.mailboxes.retain(|m| m.mailbox_id != mailbox); - } - - pub fn has_keyword(&self, keyword: &Keyword) -> bool { - self.keywords.iter().any(|k| k == keyword) - } - - pub fn has_keyword_changes(&self, prev_data: &ArchivedMessageData) -> bool { - self.keywords.len() != prev_data.keywords.len() - || !self - .keywords - .iter() - .all(|k| prev_data.keywords.iter().any(|pk| pk == k)) - } - - pub fn added_keywords( - &self, - prev_data: &ArchivedMessageData, - ) -> impl Iterator { - self.keywords - .iter() - .filter(|k| prev_data.keywords.iter().all(|pk| pk != *k)) - } - - pub fn removed_keywords<'x>( - &'x self, - prev_data: &'x ArchivedMessageData, - ) -> impl Iterator { - prev_data - .keywords - .iter() - .filter(|k| self.keywords.iter().all(|pk| pk != *k)) - } - - pub fn added_mailboxes( - &self, - prev_data: &ArchivedMessageData, - ) -> impl Iterator { - self.mailboxes.iter().filter(|m| { - prev_data - .mailboxes - .iter() - .all(|pm| pm.mailbox_id != m.mailbox_id) - }) - } - - pub fn removed_mailboxes<'x>( - &'x self, - prev_data: &'x ArchivedMessageData, - ) -> impl Iterator { - prev_data.mailboxes.iter().filter(|m| { - self.mailboxes - .iter() - .all(|pm| pm.mailbox_id != m.mailbox_id) - }) - } - - pub fn has_mailbox_changes(&self, prev_data: &ArchivedMessageData) -> bool { - self.mailboxes.len() != prev_data.mailboxes.len() - || !self.mailboxes.iter().all(|m| { - prev_data - .mailboxes - .iter() - .any(|pm| pm.mailbox_id == m.mailbox_id) - }) - } - - pub fn seal(self) -> MessageData { - MessageData { - mailboxes: self.mailboxes.into_boxed_slice(), - keywords: self.keywords.into_boxed_slice(), - thread_id: self.thread_id, - size: self.size, - } - } -} - -impl MessageData { - pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { - self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - } -} - -impl ArchivedMessageData { - pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { - self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - } - - pub fn message_uid(&self, mailbox_id: u32) -> Option { - self.mailboxes - .iter() - .find(|m| m.mailbox_id == mailbox_id) - .map(|m| m.uid.to_native()) - } - - pub fn to_builder(&self) -> MessageDataBuilder { - MessageDataBuilder { - mailboxes: self.mailboxes.iter().map(|m| m.to_native()).collect(), - keywords: self.keywords.iter().map(|k| k.to_native()).collect(), - thread_id: self.thread_id.to_native(), - size: self.size.to_native(), - } - } -} - -impl ArchivedMetadataContentType { - pub fn ctype(&self) -> &str { - &self.c_type - } - - pub fn subtype(&self) -> Option<&str> { - self.c_subtype.as_ref().map(|s| s.as_ref()) - } - - pub fn attribute(&self, name: &str) -> Option<&str> { - self.attributes - .iter() - .find(|a| *a.name == *name) - .map(|a| a.value.as_ref()) - } - - /// Returns `true` when the provided attribute name is present - pub fn has_attribute(&self, name: &str) -> bool { - self.attributes.iter().any(|a| *a.name == *name) - } - - pub fn is_attachment(&self) -> bool { - self.c_type.eq_ignore_ascii_case("attachment") - } - - pub fn is_inline(&self) -> bool { - self.c_type.eq_ignore_ascii_case("inline") - } -} - -impl ArchivedMetadataHeaderValue { - pub fn is_empty(&self) -> bool { - self == &MetadataHeaderValue::Empty - } - - pub fn as_text(&self) -> Option<&str> { - match self { - ArchivedMetadataHeaderValue::Text(s) => Some(s.as_ref()), - ArchivedMetadataHeaderValue::TextList(l) => l.last().map(|v| v.as_ref()), - _ => None, - } - } - - pub fn as_text_list(&self) -> Option<&[ArchivedBox]> { - match self { - ArchivedMetadataHeaderValue::Text(s) => Some(std::slice::from_ref(s)), - ArchivedMetadataHeaderValue::TextList(l) => Some(l.as_ref()), - _ => None, - } - } - - pub fn as_content_type(&self) -> Option<&ArchivedMetadataContentType> { - match self { - ArchivedMetadataHeaderValue::ContentType(c) => Some(c), - _ => None, - } - } - - pub fn as_datetime(&self) -> Option<&ArchivedMetadataDateTime> { - match self { - ArchivedMetadataHeaderValue::DateTime(d) => Some(d), - _ => None, - } - } - - pub fn as_single_address(&self) -> Option<&ArchivedMetadataAddress> { - match self { - ArchivedMetadataHeaderValue::AddressList(list) => list.first(), - ArchivedMetadataHeaderValue::AddressGroup(groups) => { - groups.first().and_then(|g| g.addresses.first()) - } - _ => None, - } - } -} - -impl ArchivedUidMailbox { - pub fn to_native(&self) -> UidMailbox { - UidMailbox { - mailbox_id: self.mailbox_id.to_native(), - uid: self.uid.to_native(), - } - } -} - -impl ArchivedMetadataHeader { - #[inline(always)] - pub fn value_range(&self) -> Range { - (self.base_offset.to_native() as usize + self.start.to_native() as usize) - ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) - } - - #[inline(always)] - pub fn name_value_range(&self) -> Range { - (self.base_offset.to_native() as usize) - ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) - } -} - -impl ArchivedMetadataHeaderName { - pub fn is_mime_header(&self) -> bool { - matches!( - self, - ArchivedMetadataHeaderName::ContentDescription - | ArchivedMetadataHeaderName::ContentId - | ArchivedMetadataHeaderName::ContentLanguage - | ArchivedMetadataHeaderName::ContentLocation - | ArchivedMetadataHeaderName::ContentTransferEncoding - | ArchivedMetadataHeaderName::ContentType - | ArchivedMetadataHeaderName::ContentDisposition - ) - } - - pub fn as_str(&self) -> &str { - match self { - ArchivedMetadataHeaderName::Subject => "Subject", - ArchivedMetadataHeaderName::From => "From", - ArchivedMetadataHeaderName::To => "To", - ArchivedMetadataHeaderName::Cc => "Cc", - ArchivedMetadataHeaderName::Date => "Date", - ArchivedMetadataHeaderName::Bcc => "Bcc", - ArchivedMetadataHeaderName::ReplyTo => "Reply-To", - ArchivedMetadataHeaderName::Sender => "Sender", - ArchivedMetadataHeaderName::Comments => "Comments", - ArchivedMetadataHeaderName::InReplyTo => "In-Reply-To", - ArchivedMetadataHeaderName::Keywords => "Keywords", - ArchivedMetadataHeaderName::Received => "Received", - ArchivedMetadataHeaderName::MessageId => "Message-ID", - ArchivedMetadataHeaderName::References => "References", - ArchivedMetadataHeaderName::ReturnPath => "Return-Path", - ArchivedMetadataHeaderName::MimeVersion => "MIME-Version", - ArchivedMetadataHeaderName::ContentDescription => "Content-Description", - ArchivedMetadataHeaderName::ContentId => "Content-ID", - ArchivedMetadataHeaderName::ContentLanguage => "Content-Language", - ArchivedMetadataHeaderName::ContentLocation => "Content-Location", - ArchivedMetadataHeaderName::ContentTransferEncoding => "Content-Transfer-Encoding", - ArchivedMetadataHeaderName::ContentType => "Content-Type", - ArchivedMetadataHeaderName::ContentDisposition => "Content-Disposition", - ArchivedMetadataHeaderName::ResentTo => "Resent-To", - ArchivedMetadataHeaderName::ResentFrom => "Resent-From", - ArchivedMetadataHeaderName::ResentBcc => "Resent-Bcc", - ArchivedMetadataHeaderName::ResentCc => "Resent-Cc", - ArchivedMetadataHeaderName::ResentSender => "Resent-Sender", - ArchivedMetadataHeaderName::ResentDate => "Resent-Date", - ArchivedMetadataHeaderName::ResentMessageId => "Resent-Message-ID", - ArchivedMetadataHeaderName::ListArchive => "List-Archive", - ArchivedMetadataHeaderName::ListHelp => "List-Help", - ArchivedMetadataHeaderName::ListId => "List-ID", - ArchivedMetadataHeaderName::ListOwner => "List-Owner", - ArchivedMetadataHeaderName::ListPost => "List-Post", - ArchivedMetadataHeaderName::ListSubscribe => "List-Subscribe", - ArchivedMetadataHeaderName::ListUnsubscribe => "List-Unsubscribe", - ArchivedMetadataHeaderName::ArcAuthenticationResults => "ARC-Authentication-Results", - ArchivedMetadataHeaderName::ArcMessageSignature => "ARC-Message-Signature", - ArchivedMetadataHeaderName::ArcSeal => "ARC-Seal", - ArchivedMetadataHeaderName::DkimSignature => "DKIM-Signature", - ArchivedMetadataHeaderName::DeliveredTo => "Delivered-To", - ArchivedMetadataHeaderName::XOriginalTo => "X-Original-To", - ArchivedMetadataHeaderName::ReturnReceiptTo => "Return-Receipt-To", - ArchivedMetadataHeaderName::DispositionNotificationTo => "Disposition-Notification-To", - ArchivedMetadataHeaderName::ErrorsTo => "Errors-To", - ArchivedMetadataHeaderName::AuthenticationResults => "Authentication-Results", - ArchivedMetadataHeaderName::ReceivedSpf => "Received-SPF", - ArchivedMetadataHeaderName::XSpamStatus => "X-Spam-Status", - ArchivedMetadataHeaderName::XSpamScore => "X-Spam-Score", - ArchivedMetadataHeaderName::XSpamFlag => "X-Spam-Flag", - ArchivedMetadataHeaderName::XSpamResult => "X-Spam-Result", - ArchivedMetadataHeaderName::Importance => "Importance", - ArchivedMetadataHeaderName::Priority => "Priority", - ArchivedMetadataHeaderName::XPriority => "X-Priority", - ArchivedMetadataHeaderName::XMSMailPriority => "X-MSMail-Priority", - ArchivedMetadataHeaderName::XMailer => "X-Mailer", - ArchivedMetadataHeaderName::UserAgent => "User-Agent", - ArchivedMetadataHeaderName::XMimeOLE => "X-MimeOLE", - ArchivedMetadataHeaderName::XOriginatingIp => "X-Originating-IP", - ArchivedMetadataHeaderName::XForwardedTo => "X-Forwarded-To", - ArchivedMetadataHeaderName::XForwardedFor => "X-Forwarded-For", - ArchivedMetadataHeaderName::AutoSubmitted => "Auto-Submitted", - ArchivedMetadataHeaderName::XAutoResponseSuppress => "X-Auto-Response-Suppress", - ArchivedMetadataHeaderName::Precedence => "Precedence", - ArchivedMetadataHeaderName::Organization => "Organization", - ArchivedMetadataHeaderName::ThreadIndex => "Thread-Index", - ArchivedMetadataHeaderName::ThreadTopic => "Thread-Topic", - ArchivedMetadataHeaderName::ListUnsubscribePost => "List-Unsubscribe-Post", - ArchivedMetadataHeaderName::FeedbackId => "Feedback-ID", - ArchivedMetadataHeaderName::Other(name) => name.as_ref(), - } - } -} - -impl From<&ArchivedMetadataHeaderValue> for HeaderValue<'static> { - fn from(value: &ArchivedMetadataHeaderValue) -> Self { - match value { - ArchivedMetadataHeaderValue::AddressList(addr) => HeaderValue::Address(Address::List( - addr.as_ref().iter().map(Into::into).collect(), - )), - ArchivedMetadataHeaderValue::AddressGroup(addr) => HeaderValue::Address( - Address::Group(addr.as_ref().iter().map(Into::into).collect()), - ), - ArchivedMetadataHeaderValue::Text(text) => HeaderValue::Text(text.to_string().into()), - ArchivedMetadataHeaderValue::TextList(textlist) => HeaderValue::TextList( - textlist - .as_ref() - .iter() - .map(|s| s.to_string().into()) - .collect(), - ), - ArchivedMetadataHeaderValue::DateTime(dt) => HeaderValue::DateTime(dt.into()), - ArchivedMetadataHeaderValue::ContentType(ct) => HeaderValue::ContentType(ct.into()), - ArchivedMetadataHeaderValue::Empty => HeaderValue::Empty, - } - } -} - -impl From<&ArchivedMetadataAddress> for Addr<'static> { - fn from(value: &ArchivedMetadataAddress) -> Self { - Addr { - name: value.name.as_ref().map(|n| n.to_string().into()), - address: value.address.as_ref().map(|a| a.to_string().into()), - } - } -} - -impl From<&ArchivedMetadataAddressGroup> for Group<'static> { - fn from(value: &ArchivedMetadataAddressGroup) -> Self { - Group { - name: value.name.as_ref().map(|n| n.to_string().into()), - addresses: value.addresses.as_ref().iter().map(Into::into).collect(), - } - } -} - -impl From<&ArchivedMetadataContentType> for ContentType<'static> { - fn from(value: &ArchivedMetadataContentType) -> Self { - ContentType { - c_type: value.ctype().to_string().into(), - c_subtype: value.subtype().map(|s| s.to_string().into()), - attributes: Some( - value - .attributes - .iter() - .map(|a| Attribute { - name: a.name.to_string().into(), - value: a.value.to_string().into(), - }) - .collect(), - ), - } - } -} diff --git a/crates/email/src/message/metadata/build.rs b/crates/email/src/message/metadata/build.rs new file mode 100644 index 00000000..730a193e --- /dev/null +++ b/crates/email/src/message/metadata/build.rs @@ -0,0 +1,817 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + AddressEntry, AddressField, ArchivedAddressEntry, ArchivedHeaderEntry, ArchivedMessageEntry, + ArchivedMessageMetadata, ArchivedParamEntry, ArchivedPartEntry, ArchivedStr, ContentTypeEntry, + Envelope, GROUP_BIT, HeaderEntry, HeaderId, MAX_ADDRESS_ENTRIES, MAX_FIELD_ADDRESSES, + MAX_HEADER_ENTRIES, MAX_PARAMS, MAX_PART_ENTRIES, MAX_PROTECTED_VALUE_LEN, MAX_TEXT_ITEMS, + MAX_VALUE_LEN, MessageEntry, MessageMetadata, MetadataDate, NONE, ParamEntry, PartEntry, + PartFlags, PartInfo, PartKind, Span, Str, TransferEncoding, +}; +use crate::message::index::PREVIEW_LENGTH; +use mail_parser::{ + Address, AddressList, ContentType, Encoding, Header, HeaderName, HeaderValue, Message, + MessagePart, PartKind as ParsedKind, Source, +}; +use std::mem::{size_of, take}; +use store::{U32_LEN, write::compress::MAX_ARCHIVE_SIZE}; +use types::blob_hash::BlobHash; + +pub const MAX_POOL_LEN: usize = 4 * 1024 * 1024; + +const CONTENT_TYPE_PROTECTED: &[&str] = &["charset", "name"]; +const CONTENT_DISPOSITION_PROTECTED: &[&str] = &["filename"]; +const PART_PROTECTED_PARAMS: usize = + CONTENT_TYPE_PROTECTED.len() + CONTENT_DISPOSITION_PROTECTED.len(); +const PART_PROTECTED_VALUES: usize = 2 * 2 + 2 * PART_PROTECTED_PARAMS; +const PART_POOL_OVERFLOW: usize = PART_PROTECTED_VALUES * MAX_PROTECTED_VALUE_LEN + 2; +const PART_IDS: usize = 4; +const MAX_STRUCTURE_LEN: usize = size_of::() + + MAX_PART_ENTRIES + * (size_of::() + + size_of::() + + PART_IDS * U32_LEN + + PART_PROTECTED_PARAMS * size_of::() + + PART_POOL_OVERFLOW) + + MAX_HEADER_ENTRIES * size_of::() + + MAX_ADDRESS_ENTRIES * size_of::() + + MAX_TEXT_ITEMS * size_of::() + + MAX_PARAMS * size_of::() + + MAX_POOL_LEN; + +const _: () = { + assert!(MAX_VALUE_LEN <= Str::MAX_LEN); + assert!(MAX_PROTECTED_VALUE_LEN <= MAX_VALUE_LEN); + assert!(MAX_PART_ENTRIES <= Span::MAX_LEN); + assert!(MAX_HEADER_ENTRIES <= Span::MAX_LEN); + assert!(MAX_FIELD_ADDRESSES <= Span::MAX_LEN); + assert!(MAX_FIELD_ADDRESSES < GROUP_BIT as usize); + assert!(MAX_TEXT_ITEMS <= Span::MAX_LEN); + assert!(MAX_PARAMS + CONTENT_TYPE_PROTECTED.len() <= Span::MAX_LEN); + assert!(MAX_PARAMS + CONTENT_DISPOSITION_PROTECTED.len() <= Span::MAX_LEN); + assert!(MAX_STRUCTURE_LEN < MAX_ARCHIVE_SIZE); +}; + +#[derive(Debug, Default, Clone)] +pub struct ExtraHeaders { + bytes: Vec, + fields: Vec, +} + +pub struct NewMetadata { + pub metadata: MessageMetadata, + pub raw_headers: Vec, + pub has_attachments: bool, +} + +impl ExtraHeaders { + pub fn push(&mut self, name: HeaderId, value: &str) -> &mut Self { + let Some(label) = name.as_str() else { + return self; + }; + let offset_field = self.bytes.len() as u32; + self.bytes.extend_from_slice(label.as_bytes()); + self.bytes.push(b':'); + let offset_value = self.bytes.len() as u32; + self.bytes.push(b' '); + self.bytes + .extend(value.bytes().filter(|&byte| !matches!(byte, b'\r' | b'\n'))); + self.bytes.extend_from_slice(b"\r\n"); + self.fields.push(HeaderEntry { + name, + offset_field, + offset_value, + offset_end: self.bytes.len() as u32, + }); + self + } + + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } +} + +#[derive(Clone, Copy)] +struct Shift(u32); + +impl Shift { + const NONE: Shift = Shift(0); + + #[inline] + fn apply(self, offset: u32) -> u32 { + offset.saturating_add(self.0) + } +} + +#[derive(Default, Clone, Copy)] +struct Occurrences<'m> { + last: Option>, + count: u32, +} + +impl<'m> Occurrences<'m> { + #[inline] + fn record(&mut self, header: Header<'m>) { + self.last = Some(header); + self.count += 1; + } +} + +#[derive(Default)] +struct FieldScan<'m> { + date: Option>, + subject: Option>, + in_reply_to: Option>, + message_id: Option>, + addresses: [Occurrences<'m>; 6], + content_id: Option>, + content_description: Option>, + content_transfer_encoding: Option>, + content_language: Option>, + content_location: Option>, + content_md5: Option>, +} + +const ADDRESS_FIELDS: [HeaderName<'static>; 6] = [ + HeaderName::From, + HeaderName::Sender, + HeaderName::ReplyTo, + HeaderName::To, + HeaderName::Cc, + HeaderName::Bcc, +]; + +impl<'m> FieldScan<'m> { + #[inline] + fn record(&mut self, header: Header<'m>, name: &HeaderName<'_>, envelope: bool) { + let slot = match name { + HeaderName::ContentId => &mut self.content_id, + HeaderName::ContentDescription => &mut self.content_description, + HeaderName::ContentTransferEncoding => &mut self.content_transfer_encoding, + HeaderName::ContentLanguage => &mut self.content_language, + HeaderName::ContentLocation => &mut self.content_location, + HeaderName::ContentMd5 => &mut self.content_md5, + HeaderName::Date if envelope => &mut self.date, + HeaderName::Subject if envelope => &mut self.subject, + HeaderName::InReplyTo if envelope => &mut self.in_reply_to, + HeaderName::MessageId if envelope => &mut self.message_id, + HeaderName::From if envelope => return self.addresses[0].record(header), + HeaderName::Sender if envelope => return self.addresses[1].record(header), + HeaderName::ReplyTo if envelope => return self.addresses[2].record(header), + HeaderName::To if envelope => return self.addresses[3].record(header), + HeaderName::Cc if envelope => return self.addresses[4].record(header), + HeaderName::Bcc if envelope => return self.addresses[5].record(header), + _ => return, + }; + *slot = Some(header); + } +} + +struct Builder<'m> { + metadata: MessageMetadata, + message: &'m Message<'m>, + remap: Vec, + order: Vec, + message_ends: Vec, + shift: Shift, +} + +impl MessageMetadata { + pub fn build(message: &Message<'_>, extra: &ExtraHeaders, blob_hash: BlobHash) -> NewMetadata { + let raw = message.raw(); + let root = message.root().root_part(); + let block = raw.get(..root.offset_body() as usize).unwrap_or_default(); + let mut raw_headers = Vec::with_capacity(extra.len() + block.len()); + raw_headers.extend_from_slice(extra.as_bytes()); + raw_headers.extend_from_slice(block); + + let mut builder = Builder::new(message, extra, block.len()); + builder.metadata.blob_hash = blob_hash; + builder.metadata.blob_body_offset = root.offset_body(); + builder.metadata.preview = message + .root() + .body_preview(PREVIEW_LENGTH) + .as_deref() + .map_or(Str::NONE, |preview| builder.push_whole(preview)); + builder.number(); + builder.messages(); + builder.parts(extra); + + debug_assert!( + builder + .metadata + .parts + .first() + .is_none_or(|part| part.offset_body as usize == raw_headers.len()) + ); + + let has_attachments = builder.has_attachments(); + NewMetadata { + metadata: builder.metadata, + raw_headers, + has_attachments, + } + } +} + +impl<'m> Builder<'m> { + fn new(message: &'m Message<'m>, extra: &ExtraHeaders, block_len: usize) -> Self { + let parts_len = message.parts().len().min(MAX_PART_ENTRIES); + let messages_len = message.messages().len().min(MAX_PART_ENTRIES); + Builder { + metadata: MessageMetadata { + messages: Vec::with_capacity(messages_len), + parts: Vec::with_capacity(parts_len), + headers: Vec::new(), + ids: Vec::with_capacity(parts_len * 2), + addresses: Vec::with_capacity(8), + texts: Vec::with_capacity(8), + params: Vec::with_capacity(parts_len * 2), + strings: String::with_capacity( + (block_len / 8).min(MAX_POOL_LEN) + PREVIEW_LENGTH + 64, + ), + ..Default::default() + }, + message, + remap: Vec::new(), + order: Vec::new(), + message_ends: Vec::new(), + shift: Shift(extra.len() as u32), + } + } + + fn number(&mut self) { + let messages_len = self.message.messages().len(); + let parts_len = self.message.parts().len(); + let mut cursor = vec![0u32; messages_len + 1]; + let mut headers_len = 0usize; + for part in self.message.parts() { + if let Some(count) = cursor.get_mut(part.message().id() as usize + 1) { + *count += 1; + } + headers_len += part.headers().len(); + } + let mut total = 0u32; + for slot in cursor.iter_mut() { + total += *slot; + *slot = total; + } + self.metadata + .headers + .reserve(headers_len.min(MAX_HEADER_ENTRIES) + 2); + let stored = parts_len.min(MAX_PART_ENTRIES); + self.remap = vec![NONE; parts_len]; + self.order = vec![NONE; stored]; + for part in self.message.parts() { + let Some(next) = cursor.get_mut(part.message().id() as usize) else { + continue; + }; + let new_id = *next; + *next += 1; + if let (Some(remap), Some(order)) = ( + self.remap.get_mut(part.id() as usize), + self.order.get_mut(new_id as usize), + ) { + *remap = new_id; + *order = part.id(); + } + } + cursor.truncate(messages_len); + self.message_ends = cursor; + } + + fn has_attachments(&self) -> bool { + let root = self.message.root(); + root.has_attachments() + || root.attachments().any(|part| { + part.is_message() + || part.is_content_type("message", "rfc822") + || part.is_content_type("message", "global") + }) + } + + #[inline] + fn new_id(&self, id: u32) -> u32 { + self.remap.get(id as usize).copied().unwrap_or(NONE) + } + + fn messages(&mut self) { + let stored = self.order.len() as u32; + let mut start = 0u32; + let message_ends = take(&mut self.message_ends); + for (message, end) in self + .message + .messages() + .zip(message_ends.iter()) + .take(MAX_PART_ENTRIES) + { + let end = (*end).min(stored).max(start); + let container = message + .container() + .map_or(NONE, |part| self.new_id(part.id())); + let source = match message.source() { + Source::Raw => NONE, + Source::Decoded(part) => self.new_id(part), + }; + let text_body = self.push_part_ids(message.text_body()); + let html_body = self.push_part_ids(message.html_body()); + let attachments = self.push_part_ids(message.attachments()); + self.metadata.messages.push(MessageEntry { + parts: Span::between(start, end as usize), + container, + source, + text_body, + html_body, + attachments, + envelope: Envelope::EMPTY, + }); + start = end; + } + } + + fn push_part_ids(&mut self, parts: impl Iterator>) -> Span { + let start = self.metadata.ids.len(); + let limit = start + Span::MAX_LEN; + for part in parts { + let id = self.new_id(part.id()); + if id != NONE && self.metadata.ids.len() < limit { + self.metadata.ids.push(id); + } + } + Span::between(start as u32, self.metadata.ids.len()) + } + + fn parts(&mut self, extra: &ExtraHeaders) { + let order = take(&mut self.order); + for (new_id, old_id) in order.iter().enumerate() { + let Some(part) = self.message.part(*old_id) else { + self.metadata.parts.push(PartEntry::EMPTY); + continue; + }; + let message = part.message(); + let message_id = message.id(); + let shift = match message.source() { + Source::Raw => self.shift, + Source::Decoded(_) => Shift::NONE, + }; + let is_message_root = self + .metadata + .messages + .get(message_id as usize) + .is_some_and(|entry| entry.parts.start as usize == new_id); + + let headers_start = self.metadata.headers.len() as u32; + let mut scan = FieldScan::default(); + let mut flags = PartFlags::default(); + if new_id == 0 { + let kept = extra.fields.len().min(MAX_HEADER_ENTRIES); + self.metadata + .headers + .extend_from_slice(extra.fields.get(..kept).unwrap_or_default()); + } + for header in part.headers() { + let name = header.name(); + scan.record(header, &name, is_message_root); + if self.metadata.headers.len() < MAX_HEADER_ENTRIES { + self.metadata.headers.push(HeaderEntry { + name: HeaderId::parse(name.as_str().as_bytes()), + offset_field: shift.apply(header.offset_field()), + offset_value: shift.apply(header.offset_start()), + offset_end: shift.apply(header.offset_end()), + }); + } + } + let headers = Span::between(headers_start, self.metadata.headers.len()); + + if is_message_root { + let envelope = self.envelope(part, &scan); + if let Some(entry) = self.metadata.messages.get_mut(message_id as usize) { + entry.envelope = envelope; + } + } + + let (kind, children) = match part.kind() { + ParsedKind::Text => (PartKind::Text, Span::EMPTY), + ParsedKind::Html => (PartKind::Html, Span::EMPTY), + ParsedKind::Binary => (PartKind::Binary, Span::EMPTY), + ParsedKind::InlineBinary => (PartKind::InlineBinary, Span::EMPTY), + ParsedKind::Multipart => (PartKind::Multipart, self.push_part_ids(part.children())), + ParsedKind::Message(nested) => ( + PartKind::Message, + Span { + start: nested.id(), + len: 0, + }, + ), + }; + + let (decoded_size, lines) = match kind { + PartKind::Multipart => (0, 0), + PartKind::Text + | PartKind::Html + | PartKind::Binary + | PartKind::InlineBinary + | PartKind::Message => ( + clamp(part.decoded_len()), + clamp(bytecount::count(part.raw_body(), b'\n')), + ), + }; + + if part.in_text_body() { + flags |= PartFlags::IN_TEXT_BODY; + } + if part.in_html_body() { + flags |= PartFlags::IN_HTML_BODY; + } + if part.is_attachment() { + flags |= PartFlags::ATTACHMENT; + } + if !part.has_known_transfer_encoding() { + flags |= PartFlags::UNKNOWN_TRANSFER_ENCODING; + } + + let content_type = self.push_content_type(part.content_type(), CONTENT_TYPE_PROTECTED); + let content_disposition = + self.push_content_type(part.content_disposition(), CONTENT_DISPOSITION_PROTECTED); + let content_id = scan + .content_id + .and_then(|header| header.value().as_text()) + .map_or(Str::NONE, |id| self.push_bracketed(id)); + let content_description = self.push_text(scan.content_description); + let content_transfer_encoding = self.push_text(scan.content_transfer_encoding); + let content_location = self.push_text(scan.content_location); + let content_md5 = self.push_text(scan.content_md5); + let content_language = self.push_text_list(scan.content_language, false); + + self.metadata.parts.push(PartEntry { + offset_header: if new_id == 0 { + 0 + } else { + shift.apply(part.offset_header()) + }, + offset_body: shift.apply(part.offset_body()), + offset_end: shift.apply(part.offset_end()), + message: message_id, + headers, + children, + decoded_size, + lines, + content_type, + content_disposition, + content_id, + content_description, + content_transfer_encoding, + content_location, + content_md5, + content_language, + info: PartInfo::new( + kind, + match part.encoding() { + Encoding::None => TransferEncoding::None, + Encoding::QuotedPrintable => TransferEncoding::QuotedPrintable, + Encoding::Base64 => TransferEncoding::Base64, + }, + flags, + ), + }); + } + self.order = order; + } + + fn envelope(&mut self, part: MessagePart<'m>, scan: &FieldScan<'m>) -> Envelope { + let date = scan + .date + .and_then(|header| header.value().as_datetime()) + .map(MetadataDate::from); + let date_raw = match (scan.date, date) { + (Some(header), None) => self.push_unfolded(header.raw_value()), + _ => Str::NONE, + }; + let subject = scan + .subject + .and_then(|header| header.value().as_text()) + .map_or(Str::NONE, |subject| self.push_str(subject)); + let message_id = self.push_ids(scan.message_id); + let in_reply_to = self.push_ids(scan.in_reply_to); + let mut envelope = Envelope { + date, + date_raw, + subject, + in_reply_to, + message_id, + ..Envelope::EMPTY + }; + for (index, (occurrences, name)) in scan.addresses.iter().zip(ADDRESS_FIELDS).enumerate() { + let field = match occurrences.count { + 0 => AddressField { + entries: Span { + start: self.metadata.addresses.len() as u32, + len: 0, + }, + last: 0, + }, + 1 => self.push_address_field(occurrences.last), + _ => self.push_address_field( + part.headers().iter().filter(|header| header.name() == name), + ), + }; + match index { + 0 => envelope.from = field, + 1 => envelope.sender = field, + 2 => envelope.reply_to = field, + 3 => envelope.to = field, + 4 => envelope.cc = field, + _ => envelope.bcc = field, + } + } + envelope + } + + fn push_address_field( + &mut self, + headers: impl IntoIterator>, + ) -> AddressField { + let start = self.metadata.addresses.len() as u32; + let limit = MAX_ADDRESS_ENTRIES.min(start as usize + MAX_FIELD_ADDRESSES); + let mut last = 0; + for header in headers { + last = self.metadata.addresses.len() as u32 - start; + if let Some(list) = header.value().as_address() { + self.push_address_list(list, limit); + } + } + AddressField { + entries: Span::between(start, self.metadata.addresses.len()), + last, + } + } + + fn push_address_list(&mut self, list: AddressList<'m>, limit: usize) { + for item in list.iter() { + if self.metadata.addresses.len() >= limit { + return; + } + match item { + Address::Mailbox(mailbox) => { + let entry = AddressEntry { + name: self.push_opt(mailbox.name()), + address: self.push_opt(mailbox.address()), + group: 0, + }; + self.metadata.addresses.push(entry); + } + Address::Group(group) => { + let group_index = self.metadata.addresses.len(); + let entry = AddressEntry { + name: self.push_opt(group.name()), + address: Str::NONE, + group: GROUP_BIT, + }; + self.metadata.addresses.push(entry); + let mut members = 0u16; + for mailbox in group.mailboxes() { + if self.metadata.addresses.len() >= limit { + break; + } + let entry = AddressEntry { + name: self.push_opt(mailbox.name()), + address: self.push_opt(mailbox.address()), + group: 0, + }; + self.metadata.addresses.push(entry); + members += 1; + } + if let Some(entry) = self.metadata.addresses.get_mut(group_index) { + entry.group = GROUP_BIT | members; + } + } + } + } + } + + fn push_text_list(&mut self, header: Option>, bracketed: bool) -> Span { + header + .and_then(|header| self.push_text_items(header, bracketed)) + .unwrap_or(Span::ABSENT) + } + + fn push_text_items(&mut self, header: Header<'m>, bracketed: bool) -> Option { + let start = self.metadata.texts.len() as u32; + let offered = match header.value() { + HeaderValue::TextList(list) => { + for (index, item) in list.iter().enumerate() { + if !self.push_item(item, index, bracketed) { + break; + } + } + list.len() + } + HeaderValue::Text(item) => { + self.push_item(item, 0, bracketed); + 1 + } + _ => return None, + }; + let stored = self.metadata.texts.len(); + (offered == 0 || stored > start as usize).then(|| Span::between(start, stored)) + } + + fn push_ids(&mut self, header: Option>) -> Span { + let Some(header) = header else { + return Span::ABSENT; + }; + match header.value() { + HeaderValue::TextList(_) | HeaderValue::Text(_) => { + self.push_text_items(header, true).unwrap_or(Span::ABSENT) + } + _ => Span { + start: self.metadata.texts.len() as u32, + len: 0, + }, + } + } + + fn push_unfolded(&mut self, raw: &[u8]) -> Str { + let text = String::from_utf8_lossy(raw.trim_ascii()); + if text.contains(['\r', '\n']) { + let unfolded = text.replace(['\r', '\n'], ""); + self.push_str(&unfolded) + } else { + self.push_str(&text) + } + } + + fn push_item(&mut self, item: &str, index: usize, bracketed: bool) -> bool { + if self.metadata.texts.len() >= MAX_TEXT_ITEMS { + return false; + } + let text = if bracketed { + if index > 0 { + self.push_raw(" "); + } + self.push_bracketed(item) + } else { + self.push_str(item) + }; + if text.len == Str::ABSENT_LEN { + return false; + } + self.metadata.texts.push(text); + true + } + + fn push_content_type( + &mut self, + value: Option>, + protected: &[&str], + ) -> ContentTypeEntry { + let Some(value) = value else { + return ContentTypeEntry::NONE; + }; + let ctype = self.push_protected(value.ctype()); + let subtype = value.subtype().map_or(Str::NONE, |subtype| { + self.metadata.strings.push('/'); + self.push_protected(subtype) + }); + let start = self.metadata.params.len() as u32; + let mut stored = 0u32; + for (name, value) in value.attributes() { + let slot = protected + .iter() + .position(|candidate| *candidate == name) + .map(|slot| 1u32 << slot) + .filter(|bit| stored & bit == 0); + let entry = match slot { + Some(bit) => { + stored |= bit; + ParamEntry { + name: self.push_protected(name), + value: self.push_protected(value), + } + } + None if self.metadata.params.len() < MAX_PARAMS => ParamEntry { + name: self.push_str(name), + value: self.push_str(value), + }, + None => continue, + }; + self.metadata.params.push(entry); + } + ContentTypeEntry { + ctype, + subtype, + params: Span::between(start, self.metadata.params.len()), + } + } + + fn push_text(&mut self, header: Option>) -> Str { + header + .and_then(|header| header.value().as_text()) + .map_or(Str::NONE, |text| self.push_str(text)) + } + + fn push_opt(&mut self, text: Option<&str>) -> Str { + text.map_or(Str::NONE, |text| self.push_str(text)) + } + + fn push_bracketed(&mut self, text: &str) -> Str { + if self.push_raw("<") { + let inner = self.push_str(text); + self.push_raw(">"); + inner + } else { + self.push_str(text) + } + } + + fn push_raw(&mut self, text: &str) -> bool { + let fits = self.metadata.strings.len() + text.len() <= MAX_POOL_LEN; + if fits { + self.metadata.strings.push_str(text); + } + fits + } + + #[inline] + fn pool_budget(&self) -> usize { + MAX_POOL_LEN + .saturating_sub(self.metadata.strings.len()) + .min(MAX_VALUE_LEN) + } + + fn push_str(&mut self, text: &str) -> Str { + self.push_within(text, self.pool_budget()) + } + + fn push_protected(&mut self, text: &str) -> Str { + self.push_within(text, self.pool_budget().max(MAX_PROTECTED_VALUE_LEN)) + } + + fn push_whole(&mut self, text: &str) -> Str { + if text.len() <= self.pool_budget() { + self.push_str(text) + } else { + Str::NONE + } + } + + fn push_within(&mut self, text: &str, limit: usize) -> Str { + let kept = floor_str(text, limit.min(Str::MAX_LEN)); + if kept.is_empty() && !text.is_empty() { + return Str::NONE; + } + let start = self.metadata.strings.len() as u32; + self.metadata.strings.push_str(kept); + Str::new(start, kept.len()) + } +} + +impl PartEntry { + pub const EMPTY: PartEntry = PartEntry { + offset_header: 0, + offset_body: 0, + offset_end: 0, + message: 0, + headers: Span::EMPTY, + children: Span::EMPTY, + decoded_size: 0, + lines: 0, + content_type: ContentTypeEntry::NONE, + content_disposition: ContentTypeEntry::NONE, + content_id: Str::NONE, + content_description: Str::NONE, + content_transfer_encoding: Str::NONE, + content_location: Str::NONE, + content_md5: Str::NONE, + content_language: Span::ABSENT, + info: PartInfo::new(PartKind::Text, TransferEncoding::None, PartFlags(0)), + }; +} + +#[inline] +pub(super) fn clamp(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + +pub(crate) fn floor_str(text: &str, max: usize) -> &str { + if text.len() <= max { + return text; + } + let mut end = max; + while !text.is_char_boundary(end) { + end -= 1; + } + text.get(..end).unwrap_or_default() +} diff --git a/crates/email/src/message/metadata/header_id.rs b/crates/email/src/message/metadata/header_id.rs new file mode 100644 index 00000000..c868255e --- /dev/null +++ b/crates/email/src/message/metadata/header_id.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt; + +#[derive( + rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, +)] +#[rkyv(derive(Debug, Clone, Copy, PartialEq, Eq))] +pub struct HeaderId(pub u8); + +macro_rules! header_ids { + ($($id:ident, $name:literal;)+) => { + #[allow(non_camel_case_types, clippy::upper_case_acronyms, dead_code)] + #[repr(u8)] + enum Slot { + OTHER, + $($id,)+ + } + + impl HeaderId { + pub const OTHER: HeaderId = HeaderId(Slot::OTHER as u8); + $(pub const $id: HeaderId = HeaderId(Slot::$id as u8);)+ + + const NAMES: &'static [&'static str] = &["", $($name,)+]; + + pub fn parse(name: &[u8]) -> HeaderId { + hashify::fnc_map_ignore_case!(name, + $($name => HeaderId::$id,)+ + _ => HeaderId::OTHER + ) + } + } + }; +} + +header_ids! { + SUBJECT, "Subject"; + FROM, "From"; + TO, "To"; + CC, "Cc"; + DATE, "Date"; + BCC, "Bcc"; + REPLY_TO, "Reply-To"; + SENDER, "Sender"; + COMMENTS, "Comments"; + IN_REPLY_TO, "In-Reply-To"; + KEYWORDS, "Keywords"; + RECEIVED, "Received"; + MESSAGE_ID, "Message-ID"; + REFERENCES, "References"; + RETURN_PATH, "Return-Path"; + MIME_VERSION, "MIME-Version"; + CONTENT_DESCRIPTION, "Content-Description"; + CONTENT_ID, "Content-ID"; + CONTENT_LANGUAGE, "Content-Language"; + CONTENT_LOCATION, "Content-Location"; + CONTENT_TRANSFER_ENCODING, "Content-Transfer-Encoding"; + CONTENT_TYPE, "Content-Type"; + CONTENT_DISPOSITION, "Content-Disposition"; + RESENT_TO, "Resent-To"; + RESENT_FROM, "Resent-From"; + RESENT_BCC, "Resent-Bcc"; + RESENT_CC, "Resent-Cc"; + RESENT_SENDER, "Resent-Sender"; + RESENT_DATE, "Resent-Date"; + RESENT_MESSAGE_ID, "Resent-Message-ID"; + LIST_ARCHIVE, "List-Archive"; + LIST_HELP, "List-Help"; + LIST_ID, "List-ID"; + LIST_OWNER, "List-Owner"; + LIST_POST, "List-Post"; + LIST_SUBSCRIBE, "List-Subscribe"; + LIST_UNSUBSCRIBE, "List-Unsubscribe"; + DKIM_SIGNATURE, "DKIM-Signature"; + ARC_AUTHENTICATION_RESULTS, "ARC-Authentication-Results"; + ARC_MESSAGE_SIGNATURE, "ARC-Message-Signature"; + ARC_SEAL, "ARC-Seal"; + DELIVERED_TO, "Delivered-To"; + X_ORIGINAL_TO, "X-Original-To"; + RETURN_RECEIPT_TO, "Return-Receipt-To"; + DISPOSITION_NOTIFICATION_TO, "Disposition-Notification-To"; + ERRORS_TO, "Errors-To"; + AUTHENTICATION_RESULTS, "Authentication-Results"; + RECEIVED_SPF, "Received-SPF"; + X_SPAM_STATUS, "X-Spam-Status"; + X_SPAM_SCORE, "X-Spam-Score"; + X_SPAM_FLAG, "X-Spam-Flag"; + X_SPAM_RESULT, "X-Spam-Result"; + IMPORTANCE, "Importance"; + PRIORITY, "Priority"; + X_PRIORITY, "X-Priority"; + XMS_MAIL_PRIORITY, "X-MSMail-Priority"; + X_MAILER, "X-Mailer"; + USER_AGENT, "User-Agent"; + X_MIME_OLE, "X-MimeOLE"; + X_ORIGINATING_IP, "X-Originating-IP"; + X_FORWARDED_TO, "X-Forwarded-To"; + X_FORWARDED_FOR, "X-Forwarded-For"; + AUTO_SUBMITTED, "Auto-Submitted"; + X_AUTO_RESPONSE_SUPPRESS, "X-Auto-Response-Suppress"; + PRECEDENCE, "Precedence"; + ORGANIZATION, "Organization"; + THREAD_INDEX, "Thread-Index"; + THREAD_TOPIC, "Thread-Topic"; + LIST_UNSUBSCRIBE_POST, "List-Unsubscribe-Post"; + FEEDBACK_ID, "Feedback-ID"; + ACCEPT_LANGUAGE, "Accept-Language"; + ARCHIVED_AT, "Archived-At"; + ORIGINAL_RECIPIENT, "Original-Recipient"; + ORIGINAL_SUBJECT, "Original-Subject"; + TLS_REQUIRED, "TLS-Required"; + AUTHOR, "Author"; + DKIM2_SIGNATURE, "DKIM2-Signature"; + MESSAGE_INSTANCE, "Message-Instance"; + X_UNIVERSALLY_UNIQUE_IDENTIFIER, "X-Universally-Unique-Identifier"; + X_VIRUS_SCANNED, "X-Virus-Scanned"; + X_MAIL_FROM, "X-MailFrom"; + X_MAILMAN_VERSION, "X-Mailman-Version"; + MESSAGE_ID_HASH, "Message-ID-Hash"; + X_MESSAGE_ID_HASH, "X-Message-ID-Hash"; + X_MAILMAN_RULE_MISSES, "X-Mailman-Rule-Misses"; + X_SPAM_LEVEL, "X-Spam-Level"; + X_RECEIVED, "X-Received"; + X_GM_MESSAGE_STATE, "X-Gm-Message-State"; + X_GOOGLE_DKIM_SIGNATURE, "X-Google-DKIM-Signature"; + X_MAILBOX_LINE, "X-Mailbox-Line"; + X_GM_GG, "X-Gm-Gg"; + X_ORIGINAL_FROM, "X-Original-From"; + X_GOOGLE_SMTP_SOURCE, "X-Google-Smtp-Source"; + X_MS_TNEF_CORRELATOR, "X-MS-TNEF-Correlator"; + X_MS_HAS_ATTACH, "X-MS-Has-Attach"; + X_ME_PROXY, "X-ME-Proxy"; + X_ME_PROXY_CAUSE, "X-ME-Proxy-Cause"; + X_ME_SENDER, "X-ME-Sender"; + X_GM_FEATURES, "X-Gm-Features"; + X_MS_EXCHANGE_TRANSPORT_CROSS_TENANT_HEADERS_STAMPED, "X-MS-Exchange-Transport-CrossTenantHeadersStamped"; + X_MS_PUBLIC_TRAFFIC_TYPE, "X-MS-PublicTrafficType"; + X_ORIGINATOR_ORG, "X-OriginatorOrg"; + X_MS_EXCHANGE_CROSS_TENANT_NETWORK_MESSAGE_ID, "X-MS-Exchange-CrossTenant-Network-Message-Id"; + X_MS_EXCHANGE_CROSS_TENANT_AUTH_SOURCE, "X-MS-Exchange-CrossTenant-AuthSource"; + X_MS_EXCHANGE_CROSS_TENANT_AUTH_AS, "X-MS-Exchange-CrossTenant-AuthAs"; + X_MS_EXCHANGE_CROSS_TENANT_ORIGINAL_ARRIVAL_TIME, "X-MS-Exchange-CrossTenant-OriginalArrivalTime"; + X_MICROSOFT_ANTISPAM, "X-Microsoft-Antispam"; + X_MS_EXCHANGE_CROSS_TENANT_ID, "X-MS-Exchange-CrossTenant-Id"; + X_MS_TRAFFIC_TYPE_DIAGNOSTIC, "X-MS-TrafficTypeDiagnostic"; + X_MICROSOFT_ANTISPAM_MESSAGE_INFO, "X-Microsoft-Antispam-Message-Info"; + X_MS_OFFICE365_FILTERING_CORRELATION_ID, "X-MS-Office365-Filtering-Correlation-Id"; + X_MS_EXCHANGE_CROSS_TENANT_FROM_ENTITY_HEADER, "X-MS-Exchange-CrossTenant-FromEntityHeader"; + X_MS_EXCHANGE_ANTI_SPAM_MESSAGE_DATA_CHUNK_COUNT, "X-MS-Exchange-AntiSpam-MessageData-ChunkCount"; + X_MS_EXCHANGE_ANTI_SPAM_MESSAGE_DATA0, "X-MS-Exchange-AntiSpam-MessageData-0"; + X_MS_EXCHANGE_SENDER_AD_CHECK, "X-MS-Exchange-SenderADCheck"; + X_FOREFRONT_ANTISPAM_REPORT, "X-Forefront-Antispam-Report"; + X_MS_EXCHANGE_ANTI_SPAM_RELAY, "X-MS-Exchange-AntiSpam-Relay"; + X_MS_EXCHANGE_CROSS_TENANT_MAILBOX_TYPE, "X-MS-Exchange-CrossTenant-MailboxType"; + X_MS_EXCHANGE_CROSS_TENANT_USER_PRINCIPAL_NAME, "X-MS-Exchange-CrossTenant-UserPrincipalName"; + X_GIT_HUB_REASON, "X-GitHub-Reason"; + X_GIT_HUB_RECIPIENT_ADDRESS, "X-GitHub-Recipient-Address"; + X_GIT_HUB_RECIPIENT, "X-GitHub-Recipient"; + DESTINATIONS, "Destinations"; + X_GIT_HUB_SENDER, "X-GitHub-Sender"; + X_GIT_HUB_NOTIFY_PLATFORM, "X-GitHub-Notify-Platform"; + X_THREAD_ID, "X-ThreadId"; + X_SES_OUTGOING, "X-SES-Outgoing"; + X_PROOFPOINT_VIRUS_VERSION, "X-Proofpoint-Virus-Version"; + X_GIT_HUB_ASSIGNEES, "X-GitHub-Assignees"; + X_GIT_HUB_LABELS, "X-GitHub-Labels"; + BIMI_SELECTOR, "BIMI-Selector"; + X_STRIPE_EID, "X-Stripe-EID"; + X_ATTACHMENT_ID, "X-Attachment-Id"; + X_TEST_ID_TRACKER, "X-Test-IDTracker"; + X_IETF_ID_TRACKER, "X-IETF-IDTracker"; + X_PM_MESSAGE_ID, "X-Pm-Message-ID"; + X_ANTI_ABUSE, "X-AntiAbuse"; + AUTOCRYPT, "Autocrypt"; + X_COMPLAINTS_TO, "X-Complaints-To"; + X_PROOFPOINT_ORIG_GUID, "X-Proofpoint-ORIG-GUID"; + X_PROOFPOINT_GUID, "X-Proofpoint-GUID"; + X_GIT_HUB_ISSUE_STATE, "X-GitHub-IssueState"; + X_MS_REACTIONS, "X-MS-Reactions"; + X_PM_MTA_POOL, "X-Pm-MTA-Pool"; + X_PM_RCPT, "X-Pm-RCPT"; + X_PROOFPOINT_SPAM_DETAILS_ENC, "X-Proofpoint-Spam-Details-Enc"; + X_PM_MESSAGE_OPTIONS, "X-Pm-Message-Options"; + X_SENDER_ID, "X-Sender-ID"; + X_GIT_HUB_PULL_REQUEST_STATUS, "X-GitHub-PullRequestStatus"; + X_AUTHORITY_ANALYSIS, "X-Authority-Analysis"; + X_FORWARDED_ENCRYPTED, "X-Forwarded-Encrypted"; + X_MS_EXCHANGE_ANTI_SPAM_EXTERNAL_HOP_MESSAGE_DATA_CHUNK_COUNT, "X-MS-Exchange-AntiSpam-ExternalHop-MessageData-ChunkCount"; + X_MS_EXCHANGE_ANTI_SPAM_EXTERNAL_HOP_MESSAGE_DATA0, "X-MS-Exchange-AntiSpam-ExternalHop-MessageData-0"; + X_MS_EXCHANGE_CROSS_TENANT_RMS_PERSISTED_CONSUMER_ORG, "X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg"; + X_EXCHANGE_ROUTING_POLICY_CHECKED, "X-Exchange-RoutingPolicyChecked"; + X_MS_EXCHANGE_CROSS_TENANT_ORIGINAL_ATTRIBUTED_TENANT_CONNECTING_IP, "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp"; + X_MS_EXCHANGE_MESSAGE_SENT_REPRESENTING_TYPE, "X-MS-Exchange-MessageSentRepresentingType"; + X_MS_EXCHANGE_TRANSPORT_CROSS_TENANT_HEADERS_STRIPPED, "X-MS-Exchange-Transport-CrossTenantHeadersStripped"; + X_MS_OFFICE365_FILTERING_CORRELATION_ID_PRVS, "X-MS-Office365-Filtering-Correlation-Id-Prvs"; + X_MICROSOFT_ANTISPAM_MESSAGE_INFO_ORIGINAL, "X-Microsoft-Antispam-Message-Info-Original"; + X_MS_EXCHANGE_AUTHENTICATION_RESULTS, "X-MS-Exchange-Authentication-Results"; + X_FOREFRONT_ANTISPAM_REPORT_UNTRUSTED, "X-Forefront-Antispam-Report-Untrusted"; + X_MS_EXCHANGE_ATP_MESSAGE_PROPERTIES, "X-MS-Exchange-AtpMessageProperties"; + X_MICROSOFT_ANTISPAM_UNTRUSTED, "X-Microsoft-Antispam-Untrusted"; + X_MS_EXCHANGE_ANTI_SPAM_MESSAGE_DATA1, "X-MS-Exchange-AntiSpam-MessageData-1"; + X_MISSIVE_ID, "X-Missive-Id"; + X_IADB_IP_REVERSE, "X-IADB-IP-Reverse"; + X_REPORT_ABUSE, "X-Report-Abuse"; + DKIM_FILTER, "DKIM-Filter"; + X_ME_RECEIVED, "X-ME-Received"; + X_PROOFPOINT_SPAM_DETAILS, "X-Proofpoint-Spam-Details"; + CONTENT_MD5, "Content-MD5"; +} + +impl HeaderId { + pub fn as_str(self) -> Option<&'static str> { + (self.0 != 0) + .then(|| HeaderId::NAMES.get(self.0 as usize).copied()) + .flatten() + } + + pub fn is_known(self) -> bool { + self.0 != 0 && (self.0 as usize) < HeaderId::NAMES.len() + } + + pub fn is_mime(self) -> bool { + matches!( + self, + HeaderId::CONTENT_DESCRIPTION + | HeaderId::CONTENT_ID + | HeaderId::CONTENT_LANGUAGE + | HeaderId::CONTENT_LOCATION + | HeaderId::CONTENT_TRANSFER_ENCODING + | HeaderId::CONTENT_TYPE + | HeaderId::CONTENT_DISPOSITION + ) + } + + pub fn bit(self) -> (usize, u64) { + ((self.0 >> 6) as usize, 1u64 << (self.0 & 63)) + } +} + +impl From<&ArchivedHeaderId> for HeaderId { + fn from(value: &ArchivedHeaderId) -> Self { + HeaderId(value.0) + } +} + +impl fmt::Display for HeaderId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str().unwrap_or("Other")) + } +} + diff --git a/crates/email/src/message/metadata/mod.rs b/crates/email/src/message/metadata/mod.rs new file mode 100644 index 00000000..ed1a6c0c --- /dev/null +++ b/crates/email/src/message/metadata/mod.rs @@ -0,0 +1,416 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod build; +pub mod header_id; +pub mod row; +pub mod source; +pub mod view; + + +use std::ops::{BitOr, BitOrAssign}; +use types::{blob::MAX_SECTION_CONTAINERS, blob_hash::BlobHash}; + +pub use build::{ExtraHeaders, MAX_POOL_LEN, NewMetadata}; +pub use header_id::HeaderId; +pub use row::{MetadataRow, MetadataStructure}; +pub use source::{DecodedText, PartSource, RawMessage, SourceChain}; +pub use view::{ + AddressHeader, AddressItem, Addresses, BodyList, ContentTypeView, EnvelopeView, Group, + HeaderList, HeaderMatcher, HeaderView, Mailbox, MessageView, Occurrence, PartView, TextItems, +}; + +pub use common::config::mailstore::limits::MAX_HEADER_ENTRIES; + +pub const NONE: u32 = u32::MAX; +pub const GROUP_BIT: u16 = 1 << 15; +pub const MAX_PART_ENTRIES: usize = 16_384; +pub const MAX_ADDRESS_ENTRIES: usize = Span::MAX_LEN; +pub const MAX_FIELD_ADDRESSES: usize = GROUP_BIT as usize - 1; +pub const MAX_TEXT_ITEMS: usize = 4_096; +pub const MAX_PARAMS: usize = 4_096; +pub const MAX_VALUE_LEN: usize = Str::MAX_LEN; +pub const MAX_PROTECTED_VALUE_LEN: usize = 127; +pub const MAX_SOURCE_DEPTH: usize = MAX_SECTION_CONTAINERS; +pub const MAX_NESTING: usize = 256; + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[rkyv(derive(Debug))] +pub struct Str { + pub start: u32, + pub len: u16, +} + +#[derive( + rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default, +)] +#[rkyv(derive(Debug))] +pub struct Span { + pub start: u32, + pub len: u16, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Default)] +pub struct MessageMetadata { + pub blob_hash: BlobHash, + pub blob_body_offset: u32, + pub preview: Str, + pub messages: Vec, + pub parts: Vec, + pub headers: Vec, + pub ids: Vec, + pub addresses: Vec, + pub texts: Vec, + pub params: Vec, + pub strings: String, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct MessageEntry { + pub parts: Span, + pub container: u32, + pub source: u32, + pub text_body: Span, + pub html_body: Span, + pub attachments: Span, + pub envelope: Envelope, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct Envelope { + pub date: Option, + pub date_raw: Str, + pub subject: Str, + pub from: AddressField, + pub sender: AddressField, + pub reply_to: AddressField, + pub to: AddressField, + pub cc: AddressField, + pub bcc: AddressField, + pub in_reply_to: Span, + pub message_id: Span, +} + +#[derive( + rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default, +)] +#[rkyv(derive(Debug))] +pub struct AddressField { + pub entries: Span, + pub last: u32, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[rkyv(derive(Debug))] +pub struct MetadataDate { + pub year: u16, + pub month: u8, + pub day: u8, + pub hour: u8, + pub minute: u8, + pub second: u8, + pub tz_hour: u8, + pub tz_minute: TzMinute, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[rkyv(derive(Debug, Clone, Copy, PartialEq, Eq))] +pub struct TzMinute(u8); + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct PartEntry { + pub offset_header: u32, + pub offset_body: u32, + pub offset_end: u32, + pub message: u32, + pub headers: Span, + pub children: Span, + pub decoded_size: u32, + pub lines: u32, + pub content_type: ContentTypeEntry, + pub content_disposition: ContentTypeEntry, + pub content_id: Str, + pub content_description: Str, + pub content_transfer_encoding: Str, + pub content_location: Str, + pub content_md5: Str, + pub content_language: Span, + pub info: PartInfo, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[rkyv(derive(Debug, Clone, Copy, PartialEq, Eq))] +pub struct PartInfo(u16); + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct ContentTypeEntry { + pub ctype: Str, + pub subtype: Str, + pub params: Span, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct HeaderEntry { + pub name: HeaderId, + pub offset_field: u32, + pub offset_value: u32, + pub offset_end: u32, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct AddressEntry { + pub name: Str, + pub address: Str, + pub group: u16, +} + +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug))] +pub struct ParamEntry { + pub name: Str, + pub value: Str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum PartKind { + Text = 0, + Html = 1, + Binary = 2, + InlineBinary = 3, + Multipart = 4, + Message = 5, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum TransferEncoding { + None = 0, + QuotedPrintable = 1, + Base64 = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct PartFlags(pub u8); + +impl PartFlags { + pub const IN_TEXT_BODY: PartFlags = PartFlags(1); + pub const IN_HTML_BODY: PartFlags = PartFlags(1 << 1); + pub const ATTACHMENT: PartFlags = PartFlags(1 << 2); + pub const UNKNOWN_TRANSFER_ENCODING: PartFlags = PartFlags(1 << 3); + + pub fn contains(self, other: PartFlags) -> bool { + self.0 & other.0 == other.0 + } +} + +impl BitOr for PartFlags { + type Output = PartFlags; + + fn bitor(self, other: PartFlags) -> PartFlags { + PartFlags(self.0 | other.0) + } +} + +impl BitOrAssign for PartFlags { + fn bitor_assign(&mut self, other: PartFlags) { + self.0 |= other.0; + } +} + +impl Str { + pub const ABSENT_LEN: u16 = u16::MAX; + pub const MAX_LEN: usize = Str::ABSENT_LEN as usize - 1; + pub const NONE: Str = Str { + start: 0, + len: Str::ABSENT_LEN, + }; + + #[inline] + fn new(start: u32, len: usize) -> Str { + debug_assert!(len <= Str::MAX_LEN); + Str { + start, + len: len.min(Str::MAX_LEN) as u16, + } + } +} + +impl Default for Str { + fn default() -> Self { + Str::NONE + } +} + +impl Span { + pub const MAX_LEN: usize = u16::MAX as usize - 1; + pub const EMPTY: Span = Span { start: 0, len: 0 }; + pub const ABSENT: Span = Span { + start: NONE, + len: 0, + }; + + #[inline] + fn between(start: u32, end: usize) -> Span { + let len = end.saturating_sub(start as usize); + debug_assert!(len <= Span::MAX_LEN); + Span { + start, + len: len.min(Span::MAX_LEN) as u16, + } + } +} + +impl PartKind { + #[inline] + fn from_bits(bits: u16) -> PartKind { + match bits { + 0 => PartKind::Text, + 1 => PartKind::Html, + 3 => PartKind::InlineBinary, + 4 => PartKind::Multipart, + 5 => PartKind::Message, + _ => PartKind::Binary, + } + } +} + +impl TransferEncoding { + #[inline] + fn from_bits(bits: u16) -> TransferEncoding { + match bits { + 1 => TransferEncoding::QuotedPrintable, + 2 => TransferEncoding::Base64, + _ => TransferEncoding::None, + } + } +} + +impl PartInfo { + const FLAGS: u16 = 0x00ff; + const KIND_SHIFT: u32 = 8; + const KIND: u16 = 0x7 << PartInfo::KIND_SHIFT; + const ENCODING_SHIFT: u32 = 11; + const ENCODING: u16 = 0x3 << PartInfo::ENCODING_SHIFT; + + pub const fn new(kind: PartKind, encoding: TransferEncoding, flags: PartFlags) -> Self { + PartInfo( + flags.0 as u16 + | (kind as u16) << PartInfo::KIND_SHIFT + | (encoding as u16) << PartInfo::ENCODING_SHIFT, + ) + } + + #[inline] + pub fn kind(self) -> PartKind { + PartKind::from_bits((self.0 & PartInfo::KIND) >> PartInfo::KIND_SHIFT) + } + + #[inline] + pub fn encoding(self) -> TransferEncoding { + TransferEncoding::from_bits((self.0 & PartInfo::ENCODING) >> PartInfo::ENCODING_SHIFT) + } + + #[inline] + pub fn flags(self) -> PartFlags { + PartFlags((self.0 & PartInfo::FLAGS) as u8) + } +} + +impl BitOrAssign for PartInfo { + fn bitor_assign(&mut self, flags: PartFlags) { + self.0 |= u16::from(flags.0); + } +} + +impl TzMinute { + const BEFORE_GMT: u8 = 1 << 7; + const MINUTE: u8 = !TzMinute::BEFORE_GMT; + + #[inline] + pub fn new(minute: u8, before_gmt: bool) -> Self { + TzMinute(minute.min(TzMinute::MINUTE) | if before_gmt { TzMinute::BEFORE_GMT } else { 0 }) + } + + #[inline] + pub fn minute(self) -> u8 { + self.0 & TzMinute::MINUTE + } + + #[inline] + pub fn is_before_gmt(self) -> bool { + self.0 & TzMinute::BEFORE_GMT != 0 + } +} + +impl ContentTypeEntry { + pub const NONE: ContentTypeEntry = ContentTypeEntry { + ctype: Str::NONE, + subtype: Str::NONE, + params: Span::EMPTY, + }; +} + +impl AddressField { + pub const EMPTY: AddressField = AddressField { + entries: Span::EMPTY, + last: 0, + }; +} + +impl Envelope { + pub const EMPTY: Envelope = Envelope { + date: None, + date_raw: Str::NONE, + subject: Str::NONE, + from: AddressField::EMPTY, + sender: AddressField::EMPTY, + reply_to: AddressField::EMPTY, + to: AddressField::EMPTY, + cc: AddressField::EMPTY, + bcc: AddressField::EMPTY, + in_reply_to: Span::ABSENT, + message_id: Span::ABSENT, + }; +} + +impl From for MetadataDate { + fn from(date: mail_parser::DateTime) -> Self { + MetadataDate { + year: date.year, + month: date.month, + day: date.day, + hour: date.hour, + minute: date.minute, + second: date.second, + tz_hour: date.tz_hour, + tz_minute: TzMinute::new(date.tz_minute, date.tz_before_gmt), + } + } +} + +impl From<&ArchivedMetadataDate> for mail_parser::DateTime { + fn from(date: &ArchivedMetadataDate) -> Self { + let tz_minute = TzMinute(date.tz_minute.0); + mail_parser::DateTime { + year: date.year.to_native(), + month: date.month, + day: date.day, + hour: date.hour, + minute: date.minute, + second: date.second, + tz_before_gmt: tz_minute.is_before_gmt(), + tz_hour: date.tz_hour, + tz_minute: tz_minute.minute(), + } + } +} diff --git a/crates/email/src/message/metadata/row.rs b/crates/email/src/message/metadata/row.rs new file mode 100644 index 00000000..32ff99b6 --- /dev/null +++ b/crates/email/src/message/metadata/row.rs @@ -0,0 +1,213 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedAddressEntry, ArchivedHeaderEntry, ArchivedMessageEntry, ArchivedMessageMetadata, + ArchivedParamEntry, ArchivedPartEntry, ArchivedStr, MessageMetadata, build::NewMetadata, +}; +use rkyv::{api::high::to_bytes_in, rancor}; +use std::{borrow::Cow, mem::size_of}; +use store::{ + Deserialize, U32_LEN, + write::{ + Archive, ArchiveBytes, Dictionary, + compress::{compress, compress_watermark, decompress_into}, + }, +}; + +const TRAILER_LEN: usize = U32_LEN + 1; +const ROW_MARKER: u8 = 0x80; +const HEADERS_COMPRESSED: u8 = 0x01; +const HEADERS_DICTIONARY: Option = Some(Dictionary::Email); +const STRUCTURE_COMPRESS_WATERMARK: usize = 32 * 1024; + +pub struct MetadataStructure(Archive); + +pub struct MetadataRow { + sections: Archive, + headers_len: usize, + headers_compressed: bool, +} + +#[derive(Debug, Clone, Copy)] +struct RowLayout { + structure_len: usize, + headers_end: usize, + headers_compressed: bool, +} + +impl RowLayout { + fn parse(bytes: &[u8]) -> trc::Result { + let (payload, trailer) = bytes + .split_last_chunk::() + .ok_or_else(RowLayout::corrupted)?; + let [a0, a1, a2, a3, flags] = *trailer; + let structure_len = u32::from_le_bytes([a0, a1, a2, a3]) as usize; + if flags & ROW_MARKER == 0 + || flags & !(ROW_MARKER | HEADERS_COMPRESSED) != 0 + || structure_len > payload.len() + { + return Err(RowLayout::corrupted()); + } + Ok(RowLayout { + structure_len, + headers_end: payload.len(), + headers_compressed: flags & HEADERS_COMPRESSED != 0, + }) + } + + fn structure(self, bytes: &[u8]) -> trc::Result<&[u8]> { + bytes + .get(..self.structure_len) + .ok_or_else(RowLayout::corrupted) + } + + fn headers(self, bytes: &[u8]) -> &[u8] { + bytes + .get(self.structure_len..self.headers_end) + .unwrap_or_default() + } + + fn corrupted() -> trc::Error { + trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid message metadata row") + .caused_by(trc::location!()) + } +} + +impl Deserialize for MetadataStructure { + fn deserialize(bytes: &[u8]) -> trc::Result { + RowLayout::parse(bytes)? + .structure(bytes) + .and_then( as Deserialize>::deserialize) + .map(MetadataStructure) + } +} + +impl Deserialize for MetadataRow { + fn deserialize(bytes: &[u8]) -> trc::Result { + let layout = RowLayout::parse(bytes)?; + let headers = layout.headers(bytes); + Ok(MetadataRow { + sections: Archive::::deserialize_with_prefix( + headers, + layout.structure(bytes)?, + )?, + headers_len: headers.len(), + headers_compressed: layout.headers_compressed, + }) + } +} + +impl MetadataStructure { + pub fn unarchive(&self) -> trc::Result<&ArchivedMessageMetadata> { + self.0.unarchive::() + } + + pub fn archive(&self) -> &Archive { + &self.0 + } + + pub fn decoded_len(&self) -> usize { + self.0.inner.len() + } +} + +impl MetadataRow { + pub fn encode(metadata: MessageMetadata, raw_headers: &[u8]) -> trc::Result> { + let capacity = metadata.archived_len() + TRAILER_LEN + raw_headers.len() + TRAILER_LEN; + let structure = to_bytes_in::<_, rancor::Error>(&metadata, Vec::with_capacity(capacity)) + .map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + })?; + let mut row = + Archive::::serialize_raw(structure, STRUCTURE_COMPRESS_WATERMARK)?; + let structure_len = u32::try_from(row.len()).map_err(|_| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("Message metadata too large") + .caused_by(trc::location!()) + })?; + let compressed = if raw_headers.len() >= compress_watermark(HEADERS_DICTIONARY) { + compress(HEADERS_DICTIONARY, raw_headers, 0).map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + })? + } else { + Vec::new() + }; + let (section, flags) = if !compressed.is_empty() && compressed.len() < raw_headers.len() { + (compressed.as_slice(), ROW_MARKER | HEADERS_COMPRESSED) + } else { + (raw_headers, ROW_MARKER) + }; + row.reserve_exact(section.len() + TRAILER_LEN); + row.extend_from_slice(section); + row.extend_from_slice(&structure_len.to_le_bytes()); + row.push(flags); + Ok(row) + } + + pub fn unarchive(&self) -> trc::Result<&ArchivedMessageMetadata> { + self.sections.unarchive::() + } + + pub fn headers_compressed(&self) -> bool { + self.headers_compressed + } + + pub fn decoded_len(&self) -> usize { + self.sections.inner.len() + } + + pub fn raw_headers(&self) -> trc::Result> { + let stored = self + .sections + .inner + .get(..self.headers_len) + .unwrap_or_default(); + if self.headers_compressed { + let expected = self.unarchive()?.headers_len(); + let mut inflated = Vec::new(); + decompress_into(stored, &mut inflated, expected).map_err(|err| { + trc::StoreEvent::DecompressError + .caused_by(trc::location!()) + .reason(err) + })?; + if inflated.len() == expected { + Ok(Cow::Owned(inflated)) + } else { + Err(RowLayout::corrupted()) + } + } else { + Ok(Cow::Borrowed(stored)) + } + } +} + +impl MessageMetadata { + pub(super) fn archived_len(&self) -> usize { + size_of::() + + self.messages.len() * size_of::() + + self.parts.len() * size_of::() + + self.headers.len() * size_of::() + + self.ids.len() * U32_LEN + + self.addresses.len() * size_of::() + + self.texts.len() * size_of::() + + self.params.len() * size_of::() + + self.strings.len() + } +} + +impl NewMetadata { + pub fn encode(self) -> trc::Result> { + MetadataRow::encode(self.metadata, &self.raw_headers) + } +} diff --git a/crates/email/src/message/metadata/source.rs b/crates/email/src/message/metadata/source.rs new file mode 100644 index 00000000..86a35a46 --- /dev/null +++ b/crates/email/src/message/metadata/source.rs @@ -0,0 +1,313 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedMessageMetadata, HeaderId, MAX_SOURCE_DEPTH, PartFlags, TransferEncoding, + view::{MessageView, PartView}, +}; +use mail_parser::{Charset, Encoding}; +use std::{borrow::Cow, ops::Range}; +use utils::chained_bytes::ChainedBytes; + +impl TransferEncoding { + #[inline] + fn parser(self) -> Encoding { + Encoding::from(self) + } + + #[inline] + pub fn decode(self, bytes: &[u8]) -> Cow<'_, [u8]> { + self.parser().decode(bytes) + } + + #[inline] + pub fn decode_checked(self, bytes: &[u8]) -> (Cow<'_, [u8]>, bool) { + self.parser().decode_checked(bytes) + } + + #[inline] + pub fn decode_append(self, bytes: &[u8], out: &mut Vec) { + self.parser().decode_append(bytes, out) + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct RawMessage<'x> { + bytes: ChainedBytes<'x>, + base: usize, +} + +#[derive(Debug, Clone)] +pub enum PartSource<'x> { + Raw(RawMessage<'x>), + Decoded(Vec), +} + +#[derive(Debug, Clone, Copy)] +pub struct SourceChain { + ids: [u32; MAX_SOURCE_DEPTH], + depth: usize, +} + +#[derive(Debug, Clone)] +pub struct DecodedText<'x> { + pub text: Cow<'x, str>, + pub has_problems: bool, +} + +impl<'x> RawMessage<'x> { + #[inline] + pub fn len(&self) -> usize { + self.base + self.bytes.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn has_headers(&self) -> bool { + self.base == 0 + } + + #[inline] + pub fn whole(&self) -> Option> { + self.has_headers().then_some(self.bytes) + } + + #[inline] + pub fn view(&self, range: Range) -> Option> { + self.bytes + .view(range.start.checked_sub(self.base)?..range.end.checked_sub(self.base)?) + } + + #[inline] + pub fn get(&self, range: Range) -> Option> { + self.bytes + .get(range.start.checked_sub(self.base)?..range.end.checked_sub(self.base)?) + } + + #[inline] + pub fn contiguous(&self, range: Range) -> Option<&'x [u8]> { + match self.view(range)?.segments() { + [bytes, []] | [[], bytes] => Some(bytes), + _ => None, + } + } +} + +impl<'x> PartSource<'x> { + pub fn get(&self, range: Range) -> Option> { + match self { + PartSource::Raw(raw) => raw.get(range), + PartSource::Decoded(bytes) => bytes.get(range).map(Cow::Borrowed), + } + } + + pub fn slice(&self, range: Range) -> Option> { + match self { + PartSource::Raw(raw) => raw.get(range), + PartSource::Decoded(bytes) => bytes.get(range).map(|bytes| Cow::Owned(bytes.to_vec())), + } + } + + pub fn decode(&self, range: Range, encoding: TransferEncoding) -> Cow<'x, [u8]> { + match self { + PartSource::Raw(raw) => match raw.get(range) { + Some(Cow::Borrowed(bytes)) => encoding.decode(bytes), + Some(Cow::Owned(bytes)) => match encoding.decode(&bytes) { + Cow::Borrowed(_) => Cow::Owned(bytes), + Cow::Owned(decoded) => Cow::Owned(decoded), + }, + None => Cow::Borrowed(&[]), + }, + PartSource::Decoded(bytes) => bytes + .get(range) + .map(|bytes| Cow::Owned(encoding.decode(bytes).into_owned())) + .unwrap_or_default(), + } + } + + fn decode_checked( + &self, + range: Range, + encoding: TransferEncoding, + ) -> (Cow<'x, [u8]>, bool) { + let owned = |bytes: &[u8]| { + let (decoded, malformed) = encoding.decode_checked(bytes); + (Cow::Owned(decoded.into_owned()), malformed) + }; + match self { + PartSource::Raw(raw) => match raw.get(range) { + Some(Cow::Borrowed(bytes)) => encoding.decode_checked(bytes), + Some(Cow::Owned(bytes)) => owned(&bytes), + None => (Cow::Borrowed(&[][..]), false), + }, + PartSource::Decoded(bytes) => bytes + .get(range) + .map_or((Cow::Borrowed(&[][..]), false), owned), + } + } + + pub fn len(&self) -> usize { + match self { + PartSource::Raw(raw) => raw.len(), + PartSource::Decoded(bytes) => bytes.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn is_decoded(&self) -> bool { + matches!(self, PartSource::Decoded(_)) + } +} + +impl SourceChain { + #[inline] + pub fn len(&self) -> usize { + self.depth + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.depth == 0 + } + + #[inline] + pub fn ids(&self) -> impl ExactSizeIterator + use<'_> { + self.ids + .get(..self.depth) + .unwrap_or_default() + .iter() + .rev() + .copied() + } +} + +impl ArchivedMessageMetadata { + pub fn raw_message<'x>(&self, headers: Option<&'x [u8]>, blob: &'x [u8]) -> RawMessage<'x> { + let headers_len = self.headers_len(); + match headers.filter(|headers| headers.len() == headers_len) { + Some(headers) => RawMessage { + bytes: ChainedBytes::from_blob(headers, blob, self.blob_body_offset()), + base: 0, + }, + None => RawMessage { + bytes: ChainedBytes::from_blob(&[], blob, self.blob_body_offset()), + base: headers_len, + }, + } + } + + pub fn source_chain(&self, message: MessageView<'_>) -> Option { + let mut ids = [0u32; MAX_SOURCE_DEPTH]; + let mut depth = 0; + let mut current = message; + while let Some(part) = current.source_part() { + *ids.get_mut(depth)? = part.id(); + depth += 1; + current = part.message(); + } + Some(SourceChain { ids, depth }) + } + + pub fn source<'x>( + &self, + message: MessageView<'_>, + raw: RawMessage<'x>, + ) -> Option> { + let chain = self.source_chain(message)?; + if chain.is_empty() { + return Some(PartSource::Raw(raw)); + } + + let mut buffer: Option> = None; + for id in chain.ids() { + let part = self.part(id)?; + let range = part.body_range(); + let mut decoded = Vec::new(); + match &buffer { + None => part + .encoding() + .decode_append(&raw.get(range)?, &mut decoded), + Some(outer) => part + .encoding() + .decode_append(outer.get(range)?, &mut decoded), + } + buffer = Some(decoded); + } + buffer.map(PartSource::Decoded) + } + + pub fn strip_root_fields<'b>(&self, blob: &'b [u8], id: HeaderId) -> Cow<'b, [u8]> { + let mut ranges = self + .root() + .root_part() + .headers() + .all(id) + .filter_map(|header| self.blob_range(header.field_range())) + .filter(|range| range.end <= blob.len()) + .peekable(); + if ranges.peek().is_none() { + return Cow::Borrowed(blob); + } + let mut stripped = Vec::with_capacity(blob.len()); + let mut cursor = 0; + for range in ranges { + if let Some(kept) = blob.get(cursor..range.start) { + stripped.extend_from_slice(kept); + cursor = range.end; + } + } + stripped.extend_from_slice(blob.get(cursor..).unwrap_or_default()); + Cow::Owned(stripped) + } +} + +impl<'a> PartView<'a> { + pub fn decoded<'x>(&self, source: &PartSource<'x>) -> Cow<'x, [u8]> { + source.decode(self.body_range(), self.encoding()) + } + + pub fn text<'x>(&self, source: &PartSource<'x>) -> Option> { + if !self.is_text() { + return None; + } + let (decoded, mut has_problems) = source.decode_checked(self.body_range(), self.encoding()); + if self.flags().contains(PartFlags::UNKNOWN_TRANSFER_ENCODING) { + has_problems = true; + } + let charset = match self.charset() { + None => Charset::default(), + Some(label) => Charset::from_label(label.as_bytes()).unwrap_or_else(|| { + has_problems = true; + Charset::default() + }), + }; + let (text, malformed) = match decoded { + Cow::Borrowed(bytes) => charset.decode_checked(bytes), + Cow::Owned(bytes) if charset == Charset::Utf8 => match String::from_utf8(bytes) { + Ok(text) => (Cow::Owned(text), false), + Err(err) => { + let (text, malformed) = charset.decode_checked(err.as_bytes()); + (Cow::Owned(text.into_owned()), malformed) + } + }, + Cow::Owned(bytes) => { + let (text, malformed) = charset.decode_checked(&bytes); + (Cow::Owned(text.into_owned()), malformed) + } + }; + Some(DecodedText { + text, + has_problems: has_problems || malformed, + }) + } +} diff --git a/crates/email/src/message/metadata/view.rs b/crates/email/src/message/metadata/view.rs new file mode 100644 index 00000000..390f11b9 --- /dev/null +++ b/crates/email/src/message/metadata/view.rs @@ -0,0 +1,1049 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedAddressEntry, ArchivedAddressField, ArchivedContentTypeEntry, ArchivedEnvelope, + ArchivedHeaderEntry, ArchivedMessageEntry, ArchivedMessageMetadata, ArchivedMetadataDate, + ArchivedParamEntry, ArchivedPartEntry, ArchivedPartInfo, ArchivedSpan, ArchivedStr, BlobHash, + GROUP_BIT, HeaderId, NONE, PartEntry, PartFlags, PartInfo, PartKind, Str, TransferEncoding, +}; +use rkyv::{ + option::ArchivedOption, + primitive::{ArchivedU16, ArchivedU32}, +}; +use std::{borrow::Cow, iter::from_fn, ops::Range}; + +const fn le(value: u32) -> ArchivedU32 { + ArchivedU32::from_native(value) +} + +const fn le16(value: u16) -> ArchivedU16 { + ArchivedU16::from_native(value) +} + +const EMPTY_SPAN: ArchivedSpan = ArchivedSpan { + start: le(0), + len: le16(0), +}; + +const ABSENT_SPAN: ArchivedSpan = ArchivedSpan { + start: le(NONE), + len: le16(0), +}; + +const ABSENT_STR: ArchivedStr = ArchivedStr { + start: le(0), + len: le16(Str::ABSENT_LEN), +}; + +const EMPTY_ADDRESSES: ArchivedAddressField = ArchivedAddressField { + entries: EMPTY_SPAN, + last: le(0), +}; + +const NO_CONTENT_TYPE: ArchivedContentTypeEntry = ArchivedContentTypeEntry { + ctype: ABSENT_STR, + subtype: ABSENT_STR, + params: EMPTY_SPAN, +}; + +static EMPTY_MESSAGE: ArchivedMessageEntry = ArchivedMessageEntry { + parts: EMPTY_SPAN, + container: le(NONE), + source: le(NONE), + text_body: EMPTY_SPAN, + html_body: EMPTY_SPAN, + attachments: EMPTY_SPAN, + envelope: ArchivedEnvelope { + date: ArchivedOption::None, + date_raw: ABSENT_STR, + subject: ABSENT_STR, + from: EMPTY_ADDRESSES, + sender: EMPTY_ADDRESSES, + reply_to: EMPTY_ADDRESSES, + to: EMPTY_ADDRESSES, + cc: EMPTY_ADDRESSES, + bcc: EMPTY_ADDRESSES, + in_reply_to: ABSENT_SPAN, + message_id: ABSENT_SPAN, + }, +}; + +static EMPTY_PART: ArchivedPartEntry = ArchivedPartEntry { + offset_header: le(0), + offset_body: le(0), + offset_end: le(0), + message: le(0), + headers: EMPTY_SPAN, + children: EMPTY_SPAN, + decoded_size: le(0), + lines: le(0), + content_type: NO_CONTENT_TYPE, + content_disposition: NO_CONTENT_TYPE, + content_id: ABSENT_STR, + content_description: ABSENT_STR, + content_transfer_encoding: ABSENT_STR, + content_location: ABSENT_STR, + content_md5: ABSENT_STR, + content_language: ABSENT_SPAN, + info: ArchivedPartInfo(le16(PartEntry::EMPTY.info.0)), +}; + +impl ArchivedSpan { + #[inline] + pub fn range(&self) -> Range { + let start = self.start.to_native() as usize; + start..start.saturating_add(self.len.to_native() as usize) + } + + #[inline] + pub fn is_absent(&self) -> bool { + self.start.to_native() == NONE + } + + #[inline] + fn slice<'a, T>(&self, items: &'a [T]) -> &'a [T] { + items.get(self.range()).unwrap_or_default() + } +} + +impl ArchivedStr { + #[inline] + pub fn range(&self) -> Option> { + let len = self.len.to_native(); + (len != Str::ABSENT_LEN).then(|| { + let start = self.start.to_native() as usize; + start..start + len as usize + }) + } + + #[inline] + pub fn get<'a>(&self, pool: &'a str) -> Option<&'a str> { + pool.get(self.range()?) + } + + #[inline] + pub fn bracketed<'a>(&self, pool: &'a str) -> Option<&'a str> { + let range = self.range()?; + pool.get(range.start.checked_sub(1)?..range.end + 1) + .filter(|text| text.starts_with('<') && text.ends_with('>')) + } +} + +impl ArchivedPartInfo { + #[inline] + pub fn native(&self) -> PartInfo { + PartInfo(self.0.to_native()) + } +} + +impl TransferEncoding { + pub fn as_u8(self) -> u8 { + match self { + TransferEncoding::None => 0, + TransferEncoding::QuotedPrintable => 1, + TransferEncoding::Base64 => 2, + } + } +} + +impl From for mail_parser::Encoding { + fn from(value: TransferEncoding) -> Self { + match value { + TransferEncoding::None => mail_parser::Encoding::None, + TransferEncoding::QuotedPrintable => mail_parser::Encoding::QuotedPrintable, + TransferEncoding::Base64 => mail_parser::Encoding::Base64, + } + } +} + +#[derive(Clone, Copy)] +pub struct MessageView<'a> { + meta: &'a ArchivedMessageMetadata, + id: u32, + entry: &'a ArchivedMessageEntry, +} + +#[derive(Clone, Copy)] +pub struct PartView<'a> { + meta: &'a ArchivedMessageMetadata, + id: u32, + entry: &'a ArchivedPartEntry, +} + +#[derive(Clone, Copy)] +pub struct EnvelopeView<'a> { + meta: &'a ArchivedMessageMetadata, + entry: &'a ArchivedEnvelope, +} + +#[derive(Clone, Copy)] +pub struct Addresses<'a> { + pool: &'a str, + entries: &'a [ArchivedAddressEntry], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Mailbox<'a> { + pub name: Option<&'a str>, + pub address: Option<&'a str>, +} + +#[derive(Clone, Copy)] +pub struct Group<'a> { + pub name: Option<&'a str>, + pub members: Addresses<'a>, +} + +#[derive(Clone, Copy)] +pub enum AddressItem<'a> { + Mailbox(Mailbox<'a>), + Group(Group<'a>), +} + +#[derive(Clone, Copy)] +pub struct TextItems<'a> { + pool: &'a str, + items: &'a [ArchivedStr], + present: bool, +} + +#[derive(Clone, Copy)] +pub struct ContentTypeView<'a> { + pool: &'a str, + entry: &'a ArchivedContentTypeEntry, + params: &'a [ArchivedParamEntry], +} + +#[derive(Clone, Copy)] +pub struct HeaderList<'a> { + entries: &'a [ArchivedHeaderEntry], +} + +#[derive(Clone, Copy)] +pub struct HeaderView<'a> { + entry: &'a ArchivedHeaderEntry, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AddressHeader { + From, + Sender, + ReplyTo, + To, + Cc, + Bcc, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BodyList { + Text, + Html, + Attachments, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Occurrence { + Last, + All, +} + +#[derive(Debug, Clone, Default)] +pub struct HeaderMatcher { + known: [u64; 4], + other: Vec>, +} + +impl ArchivedMessageMetadata { + #[inline] + pub fn pool(&self) -> &str { + self.strings.as_str() + } + + pub fn root(&self) -> MessageView<'_> { + self.message(0).unwrap_or(MessageView { + meta: self, + id: 0, + entry: &EMPTY_MESSAGE, + }) + } + + #[inline] + pub fn message(&self, id: u32) -> Option> { + self.messages.get(id as usize).map(|entry| MessageView { + meta: self, + id, + entry, + }) + } + + #[inline] + pub fn part(&self, id: u32) -> Option> { + self.parts.get(id as usize).map(|entry| PartView { + meta: self, + id, + entry, + }) + } + + #[inline] + fn part_or_empty(&self, id: u32) -> PartView<'_> { + PartView { + meta: self, + id, + entry: self.parts.get(id as usize).unwrap_or(&EMPTY_PART), + } + } + + pub fn parts(&self) -> impl ExactSizeIterator> { + self.parts.iter().enumerate().map(|(id, entry)| PartView { + meta: self, + id: id as u32, + entry, + }) + } + + pub fn messages(&self) -> impl ExactSizeIterator> { + self.messages + .iter() + .enumerate() + .map(|(id, entry)| MessageView { + meta: self, + id: id as u32, + entry, + }) + } + + pub fn preview(&self) -> &str { + self.preview.get(self.pool()).unwrap_or_default() + } + + pub fn stored_preview(&self) -> Option<&str> { + self.preview.get(self.pool()) + } + + pub fn blob_hash(&self) -> BlobHash { + BlobHash::from(&self.blob_hash) + } + + pub fn blob_body_offset(&self) -> usize { + self.blob_body_offset.to_native() as usize + } + + pub fn headers_len(&self) -> usize { + self.parts + .first() + .map_or(0, |part| part.offset_body.to_native() as usize) + } + + #[inline] + pub fn extra_headers_len(&self) -> usize { + self.headers_len().saturating_sub(self.blob_body_offset()) + } + + pub fn size(&self) -> usize { + self.parts.first().map_or(0, |part| { + part.offset_end + .to_native() + .saturating_sub(part.offset_header.to_native()) as usize + }) + } + + pub fn blob_size(&self) -> usize { + self.size().saturating_sub(self.extra_headers_len()) + } + + pub fn blob_range(&self, range: Range) -> Option> { + let root_body = self.headers_len(); + let base = self.blob_body_offset(); + let start = range.start.checked_add(base)?.checked_sub(root_body)?; + let end = range.end.checked_add(base)?.checked_sub(root_body)?; + Some(start..end) + } + + fn ids(&self, span: &ArchivedSpan) -> impl ExactSizeIterator> { + span.slice(&self.ids) + .iter() + .map(move |id| self.part_or_empty(id.to_native())) + } +} + +impl<'a> MessageView<'a> { + #[inline] + pub fn id(&self) -> u32 { + self.id + } + + #[inline] + pub fn metadata(&self) -> &'a ArchivedMessageMetadata { + self.meta + } + + pub fn root_part(&self) -> PartView<'a> { + self.meta.part_or_empty(self.entry.parts.start.to_native()) + } + + pub fn parts(&self) -> impl ExactSizeIterator> + use<'a> { + let meta = self.meta; + let range = self.entry.parts.range(); + let start = range.start as u32; + let entries = meta.parts.get(range).unwrap_or_default(); + entries + .iter() + .enumerate() + .map(move |(index, entry)| PartView { + meta, + id: start + index as u32, + entry, + }) + } + + pub fn part_ids(&self) -> Range { + let range = self.entry.parts.range(); + range.start as u32..range.end as u32 + } + + pub fn text_body(&self) -> impl ExactSizeIterator> + use<'a> { + self.meta.ids(&self.entry.text_body) + } + + pub fn html_body(&self) -> impl ExactSizeIterator> + use<'a> { + self.meta.ids(&self.entry.html_body) + } + + pub fn attachments(&self) -> impl ExactSizeIterator> + use<'a> { + self.meta.ids(&self.entry.attachments) + } + + pub fn body_list( + &self, + list: BodyList, + ) -> impl ExactSizeIterator> + use<'a> { + self.meta.ids(match list { + BodyList::Text => &self.entry.text_body, + BodyList::Html => &self.entry.html_body, + BodyList::Attachments => &self.entry.attachments, + }) + } + + pub fn container(&self) -> Option> { + self.meta.part(self.entry.container.to_native()) + } + + pub fn source_part(&self) -> Option> { + let source = self.entry.source.to_native(); + (source != NONE).then(|| self.meta.part(source)).flatten() + } + + pub fn is_raw_source(&self) -> bool { + self.entry.source.to_native() == NONE + } + + #[inline] + pub fn envelope(&self) -> EnvelopeView<'a> { + EnvelopeView { + meta: self.meta, + entry: &self.entry.envelope, + } + } +} + +impl<'a> EnvelopeView<'a> { + pub fn date(&self) -> Option<&'a ArchivedMetadataDate> { + self.entry.date.as_ref() + } + + pub fn datetime(&self) -> Option { + self.date().map(mail_parser::DateTime::from) + } + + pub fn date_raw(&self) -> Option<&'a str> { + self.entry.date_raw.get(self.meta.pool()) + } + + pub fn subject(&self) -> Option<&'a str> { + self.entry.subject.get(self.meta.pool()) + } + + pub fn addresses(&self, header: AddressHeader, occurrence: Occurrence) -> Addresses<'a> { + let field = match header { + AddressHeader::From => &self.entry.from, + AddressHeader::Sender => &self.entry.sender, + AddressHeader::ReplyTo => &self.entry.reply_to, + AddressHeader::To => &self.entry.to, + AddressHeader::Cc => &self.entry.cc, + AddressHeader::Bcc => &self.entry.bcc, + }; + let entries = field.entries.slice(&self.meta.addresses); + let entries = match occurrence { + Occurrence::All => entries, + Occurrence::Last => entries + .get(field.last.to_native() as usize..) + .unwrap_or_default(), + }; + Addresses { + pool: self.meta.pool(), + entries, + } + } + + pub fn in_reply_to(&self) -> TextItems<'a> { + TextItems::new(self.meta, &self.entry.in_reply_to) + } + + pub fn message_id(&self) -> TextItems<'a> { + TextItems::new(self.meta, &self.entry.message_id) + } +} + +impl<'a> Addresses<'a> { + #[inline] + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + #[inline] + fn mailbox(&self, entry: &'a ArchivedAddressEntry) -> Mailbox<'a> { + Mailbox { + name: entry.name.get(self.pool), + address: entry.address.get(self.pool), + } + } + + pub fn iter(&self) -> impl Iterator> + use<'a> { + let pool = self.pool; + let mut rest = self.entries; + from_fn(move || { + let (first, tail) = rest.split_first()?; + let group = first.group.to_native(); + if group & GROUP_BIT != 0 { + let members = ((group & !GROUP_BIT) as usize).min(tail.len()); + let (members, tail) = tail.split_at(members); + rest = tail; + Some(AddressItem::Group(Group { + name: first.name.get(pool), + members: Addresses { + pool, + entries: members, + }, + })) + } else { + rest = tail; + Some(AddressItem::Mailbox(Mailbox { + name: first.name.get(pool), + address: first.address.get(pool), + })) + } + }) + } + + pub fn mailboxes(&self) -> impl DoubleEndedIterator> + use<'a> { + let this = *self; + self.entries + .iter() + .filter(|entry| entry.group.to_native() & GROUP_BIT == 0) + .map(move |entry| this.mailbox(entry)) + } + + pub fn first(&self) -> Option> { + self.mailboxes().next() + } + + pub fn has_groups(&self) -> bool { + self.entries + .iter() + .any(|entry| entry.group.to_native() & GROUP_BIT != 0) + } + + pub fn groups(&self) -> impl Iterator, Addresses<'a>)> + use<'a> { + let pool = self.pool; + let mut rest = self.entries; + from_fn(move || { + let (first, tail) = rest.split_first()?; + let group = first.group.to_native(); + let (name, run, tail) = if group & GROUP_BIT != 0 { + let members = ((group & !GROUP_BIT) as usize).min(tail.len()); + let (members, tail) = tail.split_at(members); + (first.name.get(pool), members, tail) + } else { + let len = rest + .iter() + .take_while(|entry| entry.group.to_native() & GROUP_BIT == 0) + .count(); + let (run, tail) = rest.split_at(len); + (None, run, tail) + }; + rest = tail; + Some((name, Addresses { pool, entries: run })) + }) + } + + pub fn has_imap_address(&self) -> bool { + self.entries.iter().any(|entry| { + entry.group.to_native() & GROUP_BIT != 0 || entry.address.range().is_some() + }) + } +} + +impl<'a> TextItems<'a> { + fn new(meta: &'a ArchivedMessageMetadata, span: &ArchivedSpan) -> Self { + TextItems { + pool: meta.pool(), + items: span.slice(&meta.texts), + present: !span.is_absent(), + } + } + + #[inline] + pub fn is_present(&self) -> bool { + self.present + } + + #[inline] + pub fn len(&self) -> usize { + self.items.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + pub fn iter(&self) -> impl ExactSizeIterator + DoubleEndedIterator + use<'a> { + let pool = self.pool; + self.items + .iter() + .map(move |item| item.get(pool).unwrap_or_default()) + } + + pub fn last(&self) -> Option<&'a str> { + self.items.last()?.get(self.pool) + } + + pub fn last_bracketed(&self) -> Option<&'a str> { + self.items.last()?.bracketed(self.pool) + } + + pub fn joined_bracketed(&self) -> Option<&'a str> { + let first = self.items.first()?.range()?; + let last = self.items.last()?.range()?; + self.pool + .get(first.start.checked_sub(1)?..last.end + 1) + .filter(|text| text.starts_with('<') && text.ends_with('>')) + } +} + +impl<'a> ContentTypeView<'a> { + pub fn ctype(&self) -> &'a str { + self.entry.ctype.get(self.pool).unwrap_or_default() + } + + pub fn subtype(&self) -> Option<&'a str> { + self.entry.subtype.get(self.pool) + } + + pub fn mime_type(&self) -> Cow<'a, str> { + let Some(ctype) = self.entry.ctype.range() else { + return Cow::Borrowed(""); + }; + match self.entry.subtype.range() { + None => Cow::Borrowed(self.pool.get(ctype).unwrap_or_default()), + Some(subtype) if subtype.start == ctype.end + 1 => self + .pool + .get(ctype.start..subtype.end) + .map(Cow::Borrowed) + .unwrap_or_default(), + Some(subtype) => Cow::Owned(format!( + "{}/{}", + self.pool.get(ctype).unwrap_or_default(), + self.pool.get(subtype).unwrap_or_default() + )), + } + } + + pub fn attributes( + &self, + ) -> impl ExactSizeIterator + DoubleEndedIterator + use<'a> { + let pool = self.pool; + self.params.iter().map(move |param| { + ( + param.name.get(pool).unwrap_or_default(), + param.value.get(pool).unwrap_or_default(), + ) + }) + } + + pub fn has_attributes(&self) -> bool { + !self.params.is_empty() + } + + pub fn attribute(&self, name: &str) -> Option<&'a str> { + self.attributes() + .find(|(param, _)| *param == name) + .map(|(_, value)| value) + } + + pub fn is_inline(&self) -> bool { + self.ctype().eq_ignore_ascii_case("inline") + } + + pub fn is_attachment(&self) -> bool { + self.ctype().eq_ignore_ascii_case("attachment") + } +} + +impl<'a> PartView<'a> { + #[inline] + pub fn id(&self) -> u32 { + self.id + } + + #[inline] + pub fn metadata(&self) -> &'a ArchivedMessageMetadata { + self.meta + } + + #[inline] + pub fn kind(&self) -> PartKind { + self.entry.info.native().kind() + } + + #[inline] + pub fn is_multipart(&self) -> bool { + self.kind() == PartKind::Multipart + } + + #[inline] + pub fn is_message(&self) -> bool { + self.kind() == PartKind::Message + } + + #[inline] + pub fn is_text(&self) -> bool { + matches!(self.kind(), PartKind::Text | PartKind::Html) + } + + #[inline] + pub fn encoding(&self) -> TransferEncoding { + self.entry.info.native().encoding() + } + + #[inline] + pub fn flags(&self) -> PartFlags { + self.entry.info.native().flags() + } + + pub fn message(&self) -> MessageView<'a> { + self.meta + .message(self.entry.message.to_native()) + .unwrap_or(MessageView { + meta: self.meta, + id: NONE, + entry: &EMPTY_MESSAGE, + }) + } + + pub fn is_message_root(&self) -> bool { + self.meta + .messages + .get(self.entry.message.to_native() as usize) + .is_some_and(|message| message.parts.start.to_native() == self.id) + } + + pub fn children(&self) -> impl ExactSizeIterator> + use<'a> { + let span = if self.is_multipart() { + &self.entry.children + } else { + &EMPTY_SPAN + }; + self.meta.ids(span) + } + + pub fn child(&self, index: usize) -> Option> { + if self.is_multipart() { + self.entry + .children + .slice(&self.meta.ids) + .get(index) + .and_then(|id| self.meta.part(id.to_native())) + } else { + None + } + } + + pub fn nested(&self) -> Option> { + if self.is_message() { + self.meta.message(self.entry.children.start.to_native()) + } else { + None + } + } + + pub fn headers(&self) -> HeaderList<'a> { + HeaderList { + entries: self.entry.headers.slice(&self.meta.headers), + } + } + + fn content(&self, entry: &'a ArchivedContentTypeEntry) -> Option> { + entry.ctype.range().map(|_| ContentTypeView { + pool: self.meta.pool(), + entry, + params: entry.params.slice(&self.meta.params), + }) + } + + pub fn content_type(&self) -> Option> { + self.content(&self.entry.content_type) + } + + pub fn content_disposition(&self) -> Option> { + self.content(&self.entry.content_disposition) + } + + pub fn content_id(&self) -> Option<&'a str> { + self.entry.content_id.get(self.meta.pool()) + } + + pub fn content_id_bracketed(&self) -> Option<&'a str> { + self.entry.content_id.bracketed(self.meta.pool()) + } + + pub fn content_description(&self) -> Option<&'a str> { + self.entry.content_description.get(self.meta.pool()) + } + + pub fn content_transfer_encoding(&self) -> Option<&'a str> { + self.entry.content_transfer_encoding.get(self.meta.pool()) + } + + pub fn content_language(&self) -> TextItems<'a> { + TextItems::new(self.meta, &self.entry.content_language) + } + + pub fn content_location(&self) -> Option<&'a str> { + self.entry.content_location.get(self.meta.pool()) + } + + pub fn content_md5(&self) -> Option<&'a str> { + self.entry.content_md5.get(self.meta.pool()) + } + + pub fn charset(&self) -> Option<&'a str> { + self.content_type()?.attribute("charset") + } + + pub fn attachment_name(&self) -> Option<&'a str> { + self.content_disposition() + .and_then(|disposition| disposition.attribute("filename")) + .or_else(|| self.content_type()?.attribute("name")) + } + + #[inline] + pub fn decoded_size(&self) -> u32 { + self.entry.decoded_size.to_native() + } + + #[inline] + pub fn lines(&self) -> u32 { + self.entry.lines.to_native() + } + + #[inline] + pub fn offset_header(&self) -> usize { + self.entry.offset_header.to_native() as usize + } + + #[inline] + pub fn offset_body(&self) -> usize { + self.entry.offset_body.to_native() as usize + } + + #[inline] + pub fn offset_end(&self) -> usize { + self.entry.offset_end.to_native() as usize + } + + #[inline] + pub fn header_range(&self) -> Range { + self.offset_header()..self.offset_body().max(self.offset_header()) + } + + #[inline] + pub fn body_range(&self) -> Range { + self.offset_body()..self.offset_end().max(self.offset_body()) + } + + #[inline] + pub fn raw_range(&self) -> Range { + self.offset_header()..self.offset_end().max(self.offset_header()) + } +} + +impl<'a> HeaderList<'a> { + #[inline] + pub fn len(&self) -> usize { + self.entries.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn iter( + &self, + ) -> impl ExactSizeIterator> + DoubleEndedIterator + use<'a> { + self.entries.iter().map(|entry| HeaderView { entry }) + } + + pub fn last(&self, id: HeaderId) -> Option> { + self.entries + .iter() + .rev() + .find(|entry| entry.name.0 == id.0) + .map(|entry| HeaderView { entry }) + } + + pub fn last_named(&self, name: &str, source: &[u8]) -> Option> { + let id = HeaderId::parse(name.as_bytes()); + self.iter().rev().find(|header| { + if id.is_known() { + header.id() == id + } else { + header.id() == HeaderId::OTHER + && header + .raw_name(source) + .eq_ignore_ascii_case(name.as_bytes()) + } + }) + } + + pub fn last_parsed<'s>( + &self, + id: HeaderId, + source: &'s [u8], + form: mail_parser::HeaderForm, + ) -> Option> { + self.last(id)?.raw_value(source).map(|raw| form.parse(raw)) + } + + pub fn all(&self, id: HeaderId) -> impl DoubleEndedIterator> + use<'a> { + self.entries + .iter() + .filter(move |entry| entry.name.0 == id.0) + .map(|entry| HeaderView { entry }) + } +} + +impl<'a> HeaderView<'a> { + #[inline] + pub fn id(&self) -> HeaderId { + HeaderId(self.entry.name.0) + } + + #[inline] + pub fn field_range(&self) -> Range { + let start = self.entry.offset_field.to_native() as usize; + start..(self.entry.offset_end.to_native() as usize).max(start) + } + + #[inline] + pub fn value_range(&self) -> Range { + let start = self.entry.offset_value.to_native() as usize; + start..(self.entry.offset_end.to_native() as usize).max(start) + } + + pub fn raw_name<'r>(&self, source: &'r [u8]) -> &'r [u8] { + let start = self.entry.offset_field.to_native() as usize; + let end = (self.entry.offset_value.to_native() as usize).saturating_sub(1); + source + .get(start..end.max(start)) + .unwrap_or_default() + .trim_ascii_end() + } + + pub fn raw_value<'r>(&self, source: &'r [u8]) -> Option<&'r [u8]> { + source.get(self.value_range()) + } + + pub fn raw_name_in<'f>(&self, field: &'f [u8]) -> &'f [u8] { + let name_len = self + .value_range() + .start + .saturating_sub(self.field_range().start) + .saturating_sub(1); + field.get(..name_len).unwrap_or_default().trim_ascii_end() + } + + pub fn raw_value_in<'f>(&self, field: &'f [u8]) -> &'f [u8] { + let offset = self + .value_range() + .start + .saturating_sub(self.field_range().start); + field.get(offset..).unwrap_or_default() + } +} + +impl HeaderMatcher { + pub fn new<'x>(names: impl IntoIterator) -> Self { + let mut matcher = HeaderMatcher::default(); + for name in names { + matcher.add_name(name); + } + matcher + } + + pub fn add_name(&mut self, name: &str) { + let id = HeaderId::parse(name.as_bytes()); + if id.is_known() { + self.add_id(id); + } else { + self.other + .push(name.as_bytes().to_ascii_lowercase().into_boxed_slice()); + } + } + + pub fn add_id(&mut self, id: HeaderId) { + let (word, bit) = id.bit(); + if let Some(slot) = self.known.get_mut(word) { + *slot |= bit; + } + } + + #[inline] + pub fn matches_id(&self, id: HeaderId) -> bool { + let (word, bit) = id.bit(); + self.known.get(word).is_some_and(|slot| slot & bit != 0) + } + + #[inline] + pub fn matches(&self, header: HeaderView<'_>, source: &[u8]) -> bool { + let id = header.id(); + if id.is_known() { + self.matches_id(id) + } else { + !self.other.is_empty() && self.matches_other(header.raw_name(source)) + } + } + + #[inline] + pub fn matches_other(&self, name: &[u8]) -> bool { + self.other + .iter() + .any(|other| other.eq_ignore_ascii_case(name)) + } + + #[inline] + pub fn matches_field(&self, header: HeaderView<'_>, field: &[u8]) -> bool { + let id = header.id(); + if id.is_known() { + self.matches_id(id) + } else { + self.matches_other(header.raw_name_in(field)) + } + } +} diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs index b75cdb52..c6cd1911 100644 --- a/crates/email/src/message/mod.rs +++ b/crates/email/src/message/mod.rs @@ -12,4 +12,9 @@ pub mod delivery; pub mod headers; pub mod index; pub mod ingest; +pub mod ingest_metadata; +pub mod jmap; +pub mod messagedata; pub mod metadata; +pub mod sortkeys; +pub mod thread; diff --git a/crates/email/src/message/sortkeys.rs b/crates/email/src/message/sortkeys.rs new file mode 100644 index 00000000..d02d2412 --- /dev/null +++ b/crates/email/src/message/sortkeys.rs @@ -0,0 +1,560 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + cache::email::{MessageCacheAccess, thread_keywords}, + message::metadata::{AddressHeader, EnvelopeView, Mailbox, Occurrence}, +}; +use common::{MessageStoreCache, Server, cache::email::MessageRef}; +use mail_parser::{HeaderName, ParsedValue, thread_name}; +use std::collections::BinaryHeap; +use store::{ + IterateParams, U32_LEN, ValueKey, + ahash::AHashMap, + dispatch::DocumentSet, + roaring::RoaringBitmap, + search::{QueryResults, SearchComparator, SearchQuery}, + write::{ValueClass, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::{collection::Collection, field::EmailField, keyword::Keyword}; + +pub const MAX_SORT_KEY_LEN: usize = 40; +pub const MAX_CACHE_SORT_KEYS: usize = 4; +const SORT_KEY_FIELDS: usize = 3; +const SIGN_BIT: u64 = 1 << 63; + +pub type SortKey = [u8; MAX_SORT_KEY_LEN]; + +#[derive(Debug, Clone, Default)] +pub struct MessageSortKeys { + pub from: String, + pub to: String, + pub subject: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MessageSortField { + From, + To, + Subject, +} + +pub enum MessageCacheField { + ReceivedAt, + SentAt, + Size, + Keyword(Keyword), + ThreadKeyword { keyword: Keyword, match_all: bool }, +} + +pub enum MessageComparator { + Search(SearchComparator), + SortKey { + field: MessageSortField, + ascending: bool, + }, + Cache { + field: MessageCacheField, + ascending: bool, + }, +} + +pub trait EmailSortKeys: Sync + Send { + fn query_emails( + &self, + account_id: u32, + cache: &MessageStoreCache, + query: SearchQuery, + comparators: Vec, + ) -> impl Future>> + Send; + + fn message_sort_ranks( + &self, + account_id: u32, + documents: &RoaringBitmap, + fields: &[MessageSortField], + ) -> impl Future>; SORT_KEY_FIELDS]>> + Send; +} + +impl EmailSortKeys for Server { + async fn query_emails( + &self, + account_id: u32, + cache: &MessageStoreCache, + query: SearchQuery, + comparators: Vec, + ) -> trc::Result> { + let results = self + .search_store() + .filter_account(query) + .await + .caused_by(trc::location!())?; + + if results.len() < 2 || comparators.is_empty() { + return Ok(results.into_iter().collect()); + } + + let mut sort_key_fields: Vec = Vec::with_capacity(SORT_KEY_FIELDS); + for comparator in &comparators { + if let MessageComparator::SortKey { field, .. } = comparator + && !sort_key_fields.as_slice().contains(field) + { + sort_key_fields.push(*field); + } + } + let mut sort_key_ranks = if !sort_key_fields.is_empty() { + self.message_sort_ranks(account_id, &results, &sort_key_fields) + .await + .caused_by(trc::location!())? + } else { + Default::default() + }; + + let mut sorted_comparators = Vec::with_capacity(comparators.len()); + for comparator in comparators { + sorted_comparators.push(match comparator { + MessageComparator::Search(comparator) => comparator, + MessageComparator::SortKey { field, ascending } => SearchComparator::sorted_set( + sort_key_ranks[field.index()].take().unwrap_or_default(), + ascending, + ), + MessageComparator::Cache { field, ascending } => { + cache_comparator(cache, &results, field, ascending) + } + }); + } + + Ok(QueryResults::new(results, sorted_comparators).into_sorted()) + } + + async fn message_sort_ranks( + &self, + account_id: u32, + documents: &RoaringBitmap, + fields: &[MessageSortField], + ) -> trc::Result<[Option>; SORT_KEY_FIELDS]> { + let collection: u8 = Collection::Email.into(); + let class = ValueClass::Immutable(EmailField::SortKeys.into()); + let mut sort_keys: [Vec<(SortKey, u32)>; SORT_KEY_FIELDS] = Default::default(); + let ranges = documents + .scan_ranges(self.core.storage.data.scan_gap()) + .into_iter() + .map(|(from_document_id, to_document_id)| { + IterateParams::new( + ValueKey { + account_id, + collection, + document_id: from_document_id, + class: class.clone(), + }, + ValueKey { + account_id, + collection, + document_id: to_document_id, + class: class.clone(), + }, + ) + }) + .collect::>(); + + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + for field in fields { + if let Some(sort_key) = MessageSortKeys::deserialize_field(value, *field) { + let sort_key = &sort_key.as_bytes()[..sort_key.len().min(MAX_SORT_KEY_LEN)]; + let mut padded_sort_key: SortKey = [0u8; MAX_SORT_KEY_LEN]; + padded_sort_key[..sort_key.len()].copy_from_slice(sort_key); + sort_keys[field.index()].push((padded_sort_key, document_id)); + } + } + } + Ok(true) + }; + + match ranges.len() { + 0 => Ok(()), + 1 => { + self.core + .storage + .data + .iterate(ranges.into_iter().next().unwrap(), &mut collect) + .await + } + _ => { + self.core + .storage + .data + .iterate_many(ranges, &mut collect) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + })?; + + let mut ranks: [Option>; SORT_KEY_FIELDS] = Default::default(); + for field in fields { + let sort_keys = &mut sort_keys[field.index()]; + sort_keys.sort_unstable_by_key(|(sort_key, _)| *sort_key); + + let mut field_ranks = AHashMap::with_capacity(sort_keys.len()); + let mut rank = 1u32; + let mut prev_sort_key: Option<&SortKey> = None; + for (sort_key, document_id) in sort_keys.iter() { + if prev_sort_key.is_some_and(|prev| prev != sort_key) { + rank += 1; + } + field_ranks.insert(*document_id, rank); + prev_sort_key = Some(sort_key); + } + ranks[field.index()] = Some(field_ranks); + } + + Ok(ranks) + } +} + +impl MessageSortField { + #[inline(always)] + fn index(&self) -> usize { + match self { + MessageSortField::From => 0, + MessageSortField::To => 1, + MessageSortField::Subject => 2, + } + } +} + +pub struct CacheSorter { + keys: Vec, +} + +struct CacheSortKey { + field: CacheSortField, + ascending: bool, +} + +enum CacheSortField { + ReceivedAt, + SentAt, + Size, + Keyword(Keyword), + DocumentSet(RoaringBitmap), +} + +pub struct TopMessages(std::vec::IntoIter); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct RankedMessage { + key: [u64; MAX_CACHE_SORT_KEYS], + document_id: u32, + thread_id: u32, +} + +impl CacheSorter { + pub fn supports(comparators: &[MessageComparator]) -> bool { + comparators.len() <= MAX_CACHE_SORT_KEYS + && comparators.iter().all(|comparator| { + matches!( + comparator, + MessageComparator::Cache { .. } + | MessageComparator::Search(SearchComparator::DocumentSet { .. }) + ) + }) + } + + pub fn new( + cache: &MessageStoreCache, + comparators: Vec, + ) -> Result> { + if !CacheSorter::supports(&comparators) { + return Err(comparators); + } + let keys = comparators + .into_iter() + .filter_map(|comparator| match comparator { + MessageComparator::Cache { field, ascending } => Some(CacheSortKey { + field: match field { + MessageCacheField::ReceivedAt => CacheSortField::ReceivedAt, + MessageCacheField::SentAt => CacheSortField::SentAt, + MessageCacheField::Size => CacheSortField::Size, + MessageCacheField::Keyword(keyword) => CacheSortField::Keyword(keyword), + MessageCacheField::ThreadKeyword { keyword, match_all } => { + CacheSortField::DocumentSet(thread_keywords( + cache, &keyword, match_all, None, None, + )) + } + }, + ascending, + }), + MessageComparator::Search(SearchComparator::DocumentSet { set, ascending }) => { + Some(CacheSortKey { + field: CacheSortField::DocumentSet(set), + ascending, + }) + } + MessageComparator::Search(SearchComparator::SortedSet { .. }) + | MessageComparator::SortKey { .. } => None, + }) + .collect(); + Ok(CacheSorter { keys }) + } + + pub fn top<'x>( + &self, + cache: &MessageStoreCache, + messages: impl Iterator>, + count: usize, + ) -> (TopMessages, usize) { + let mut heap = BinaryHeap::with_capacity(count.min(cache.emails.len())); + let mut total = 0; + for message in messages { + total += 1; + if count == 0 { + continue; + } + let ranked = self.rank(cache, message); + if heap.len() < count { + heap.push(ranked); + } else if let Some(mut last) = heap.peek_mut() + && ranked < *last + { + *last = ranked; + } + } + (TopMessages(heap.into_sorted_vec().into_iter()), total) + } + + fn rank(&self, cache: &MessageStoreCache, message: MessageRef<'_>) -> RankedMessage { + let mut key = [0u64; MAX_CACHE_SORT_KEYS]; + for (slot, sort_key) in key.iter_mut().zip(&self.keys) { + let value = match &sort_key.field { + CacheSortField::ReceivedAt => message.received_at(), + CacheSortField::SentAt => (message.sent_at_absolute() as u64) ^ SIGN_BIT, + CacheSortField::Size => u64::from(message.size()), + CacheSortField::Keyword(keyword) => u64::from(cache.has_keyword(message, keyword)), + CacheSortField::DocumentSet(documents) => { + u64::from(documents.contains(message.document_id())) + } + }; + *slot = if sort_key.ascending { value } else { !value }; + } + RankedMessage { + key, + document_id: message.document_id(), + thread_id: message.thread_id(), + } + } +} + +impl Iterator for TopMessages { + type Item = (u32, u32); + + fn next(&mut self) -> Option { + self.0 + .next() + .map(|ranked| (ranked.document_id, ranked.thread_id)) + } +} + +pub fn cache_comparator( + cache: &MessageStoreCache, + documents: &RoaringBitmap, + field: MessageCacheField, + ascending: bool, +) -> SearchComparator { + match field { + MessageCacheField::ReceivedAt => SearchComparator::sorted_set( + documents + .iter() + .filter_map(|document_id| { + cache + .email_by_id(&document_id) + .map(|item| (document_id, item.received_at() as u32)) + }) + .collect(), + ascending, + ), + MessageCacheField::Size => SearchComparator::sorted_set( + documents + .iter() + .filter_map(|document_id| { + cache + .email_by_id(&document_id) + .map(|item| (document_id, item.size())) + }) + .collect(), + ascending, + ), + MessageCacheField::SentAt => { + let mut sorted = documents + .iter() + .filter_map(|document_id| { + cache.email_by_id(&document_id).map(|item| { + ( + item.received_at() as i64 + item.sent_at() as i64, + document_id, + ) + }) + }) + .collect::>(); + sorted.sort_unstable_by_key(|(sent_at, _)| *sent_at); + + let mut set = AHashMap::with_capacity(sorted.len()); + let mut rank = 1u32; + let mut prev_sent_at = None; + for (sent_at, document_id) in sorted { + if prev_sent_at.is_some_and(|prev| prev != sent_at) { + rank += 1; + } + set.insert(document_id, rank); + prev_sent_at = Some(sent_at); + } + + SearchComparator::sorted_set(set, ascending) + } + MessageCacheField::Keyword(keyword) => SearchComparator::set( + documents + .iter() + .filter(|document_id| { + cache + .email_by_id(document_id) + .is_some_and(|item| cache.has_keyword(item, &keyword)) + }) + .collect(), + ascending, + ), + MessageCacheField::ThreadKeyword { keyword, match_all } => SearchComparator::set( + thread_keywords(cache, &keyword, match_all, Some(documents), None), + ascending, + ), + } +} + +impl MessageSortKeys { + pub fn new(from: Option>, to: Option>, subject: Option<&str>) -> Self { + MessageSortKeys { + from: from.map(Mailbox::sort_key).unwrap_or_default(), + to: to.map(Mailbox::sort_key).unwrap_or_default(), + subject: subject + .map(|subject| sort_key([thread_name(subject)])) + .unwrap_or_default(), + } + } + + pub fn from_message(message: &mail_parser::Message<'_>) -> Self { + let headers = message.root().headers(); + let first_mailbox = |name: HeaderName<'static>| { + headers + .get(name) + .and_then(|header| header.value().as_address()) + .and_then(|list| list.mailboxes().next()) + .map(|mailbox| Mailbox { + name: mailbox.name(), + address: mailbox.address(), + }) + }; + MessageSortKeys::new( + first_mailbox(HeaderName::From), + first_mailbox(HeaderName::To), + headers.subject(), + ) + } + + pub fn from_envelope(envelope: EnvelopeView<'_>) -> Self { + MessageSortKeys::new( + envelope + .addresses(AddressHeader::From, Occurrence::Last) + .first(), + envelope + .addresses(AddressHeader::To, Occurrence::Last) + .first(), + envelope.subject(), + ) + } + + pub fn serialize(&self) -> Vec { + let sort_keys = [&self.from, &self.to, &self.subject]; + let mut out = Vec::with_capacity( + SORT_KEY_FIELDS + self.from.len() + self.to.len() + self.subject.len(), + ); + for sort_key in sort_keys { + debug_assert!(sort_key.len() <= MAX_SORT_KEY_LEN); + out.push(sort_key.len() as u8); + } + for sort_key in sort_keys { + out.extend_from_slice(sort_key.as_bytes()); + } + out + } + + pub fn deserialize_field(bytes: &[u8], field: MessageSortField) -> Option<&str> { + let offset = SORT_KEY_FIELDS + + bytes + .get(..field.index())? + .iter() + .map(|len| *len as usize) + .sum::(); + + std::str::from_utf8(bytes.get(offset..offset + *bytes.get(field.index())? as usize)?).ok() + } +} + +impl<'x> Mailbox<'x> { + pub fn first_of(parsed: &'x ParsedValue<'_>) -> Option { + parsed + .value() + .as_address()? + .mailboxes() + .next() + .map(|mailbox| Mailbox { + name: mailbox.name(), + address: mailbox.address(), + }) + } + + fn sort_key(self) -> String { + sort_key([ + self.name.unwrap_or_default(), + self.address.unwrap_or_default(), + ]) + } +} + +fn sort_key(values: [&str; N]) -> String { + let mut sort_key = String::with_capacity(MAX_SORT_KEY_LEN); + let mut pending_space = false; + + for value in values { + for ch in value.chars() { + if ch.is_whitespace() { + pending_space = !sort_key.is_empty(); + continue; + } else if pending_space { + if sort_key.len() + 1 > MAX_SORT_KEY_LEN { + return sort_key; + } + sort_key.push(' '); + pending_space = false; + } + + for ch in ch.to_uppercase() { + if sort_key.len() + ch.len_utf8() > MAX_SORT_KEY_LEN { + return sort_key; + } + sort_key.push(ch); + } + } + pending_space = !sort_key.is_empty(); + } + + sort_key +} + diff --git a/crates/email/src/message/thread.rs b/crates/email/src/message/thread.rs new file mode 100644 index 00000000..70ca7243 --- /dev/null +++ b/crates/email/src/message/thread.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::message::{ + index::extractors::VisitText, + metadata::{ArchivedMessageMetadata, HeaderId, HeaderList, MAX_VALUE_LEN, build::floor_str}, +}; +use mail_parser::{HeaderForm, HeaderName, HeaderValue, Message, thread_name}; +use std::borrow::Cow; +use store::xxhash_rust::xxh3::xxh3_128; + +#[derive(Debug, Default)] +pub struct ThreadFields<'m> { + pub subject: &'m str, + pub message_id: Option, + pub message_ids: Vec, + pub sent_at: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ThreadSubject<'x> { + Resolved(&'x str), + InHeaders, +} + +impl<'m> ThreadFields<'m> { + pub fn scan(message: &'m Message<'_>) -> Self { + let mut fields = ThreadFields::default(); + let mut date_seen = false; + for header in message.root_part().headers().iter().rev() { + let value = header.value(); + match header.name() { + HeaderName::MessageId => value.visit_text(|id| { + if !id.is_empty() { + if fields.message_id.is_none() { + fields.message_id = id.to_string().into(); + } + fields.message_ids.push(xxh3_128(id.as_bytes())); + } + }), + HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { + value.visit_text(|id| { + if !id.is_empty() { + fields.message_ids.push(xxh3_128(id.as_bytes())); + } + }); + } + HeaderName::Subject if fields.subject.is_empty() => { + fields.subject = subject_thread_name(value); + } + HeaderName::Date if !date_seen => { + date_seen = true; + fields.sent_at = value.as_datetime().map(|date| date.to_timestamp()); + } + _ => (), + } + } + fields.message_ids.sort_unstable(); + fields.message_ids.dedup(); + fields + } +} + +impl ArchivedMessageMetadata { + pub fn thread_subject(&self) -> ThreadSubject<'_> { + let root = self.root(); + let name = thread_name(root.envelope().subject().unwrap_or_default()); + if !name.is_empty() + || root + .root_part() + .headers() + .all(HeaderId::SUBJECT) + .nth(1) + .is_none() + { + ThreadSubject::Resolved(name) + } else { + ThreadSubject::InHeaders + } + } + + pub fn thread_subject_in(&self, headers: &[u8]) -> Cow<'_, str> { + self.thread_subject_with(self.root().root_part().headers(), headers) + } + + pub fn thread_subject_with(&self, root: HeaderList<'_>, headers: &[u8]) -> Cow<'_, str> { + if let ThreadSubject::Resolved(name) = self.thread_subject() { + return Cow::Borrowed(name); + } + root.iter() + .rev() + .filter(|header| header.id() == HeaderId::SUBJECT) + .find_map(|header| { + let parsed = HeaderForm::Text.parse(header.raw_value(headers)?); + let name = subject_thread_name(parsed.value()); + (!name.is_empty()).then(|| name.to_string()) + }) + .map_or(Cow::Borrowed(""), Cow::Owned) + } +} + +fn subject_thread_name<'x>(value: HeaderValue<'x>) -> &'x str { + let subject = match value { + HeaderValue::Text(text) => text, + HeaderValue::TextList(list) => list.first().unwrap_or_default(), + _ => "", + }; + thread_name(floor_str(subject, MAX_VALUE_LEN)) +} + diff --git a/crates/email/src/presence.rs b/crates/email/src/presence.rs new file mode 100644 index 00000000..5688c762 --- /dev/null +++ b/crates/email/src/presence.rs @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + mailbox::{ArchivedMailbox, METADATA_MASK, METADATA_SHIFT, Mailbox}, + message::messagedata::{HAS_JMAP_METADATA, MessageData}, + sieve::{ArchivedSieveScript, SieveScript}, +}; +use common::storage::{index::PresenceFlags, metadata::MetadataPresence}; +use store::{ + Deserialize, + write::{BatchBuilder, MergeResult}, +}; +use types::{ + collection::{Collection, SyncCollection}, + field::Field, + metadata::MetadataKinds, +}; + +impl MessageData { + pub const TRACKED_METADATA: MetadataKinds = MetadataKinds::JMAP; + + pub fn metadata_kinds(&self) -> MetadataKinds { + Self::metadata_kinds_of(self.keywords) + } + + pub fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + if kinds.intersects(Self::TRACKED_METADATA) { + self.keywords |= HAS_JMAP_METADATA; + } else { + self.keywords &= !HAS_JMAP_METADATA; + } + } + + pub(crate) fn has_metadata(keywords: u32) -> bool { + keywords & HAS_JMAP_METADATA != 0 + } + + pub(crate) fn metadata_kinds_of(keywords: u32) -> MetadataKinds { + if Self::has_metadata(keywords) { + MetadataKinds::JMAP + } else { + MetadataKinds::NONE + } + } +} + +impl Mailbox { + pub fn metadata_kinds(&self) -> MetadataKinds { + Self::metadata_kinds_of(self.flags) + } + + fn metadata_kinds_of(flags: u16) -> MetadataKinds { + tracked_kinds( + ((flags & METADATA_MASK) >> METADATA_SHIFT) as u8, + Self::TRACKED, + ) + } +} + +impl PresenceFlags for Mailbox { + const COLLECTION: Collection = Collection::Mailbox; + const TRACKED: MetadataKinds = MetadataKinds::JMAP.union(MetadataKinds::IMAP); + + fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.flags = (self.flags & !METADATA_MASK) + | (u16::from(kinds.bits() & Self::TRACKED.bits()) << METADATA_SHIFT); + } +} + +impl ArchivedMailbox { + pub fn metadata_kinds(&self) -> MetadataKinds { + Mailbox::metadata_kinds_of(self.flags.to_native()) + } +} + +impl SieveScript { + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.metadata_flags, Self::TRACKED) + } +} + +impl PresenceFlags for SieveScript { + const COLLECTION: Collection = Collection::SieveScript; + const TRACKED: MetadataKinds = MetadataKinds::JMAP; + + fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.metadata_flags = kinds.bits() & Self::TRACKED.bits(); + } +} + +impl ArchivedSieveScript { + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.metadata_flags, SieveScript::TRACKED) + } +} + +fn tracked_kinds(flags: u8, tracked: MetadataKinds) -> MetadataKinds { + MetadataKinds::from_bits(flags & tracked.bits()).unwrap_or_default() +} + +fn tracked_target(presence: MetadataPresence, tracked: MetadataKinds) -> MetadataKinds { + tracked_kinds(presence.after.bits(), tracked) +} + +fn merge_metadata_kinds(batch: &mut BatchBuilder, account_id: u32, kinds: MetadataKinds) { + let group = SyncCollection::Email.change_group(); + batch.merge_fnc(Field::ARCHIVE, move |ids, bytes| { + apply_metadata_kinds(bytes, kinds, ids.change_id(account_id, group)) + }); +} + +fn apply_metadata_kinds( + bytes: Option<&[u8]>, + kinds: MetadataKinds, + change_id: Option, +) -> trc::Result { + let change_id = change_id.ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("No change id was allocated for the message presence update.") + .caused_by(trc::location!()) + })?; + let bytes = bytes.ok_or_else(|| { + trc::StoreEvent::AssertValueFailed + .into_err() + .details("Message no longer exists.") + .caused_by(trc::location!()) + })?; + let mut data = MessageData::deserialize(bytes)?; + let keywords = data.keywords; + data.set_metadata_kinds(kinds); + if data.keywords != keywords { + data.serialize_with_change_id(change_id) + .map(MergeResult::Update) + } else { + Ok(MergeResult::Skip) + } +} + +pub fn update_email_presence( + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + current: MetadataKinds, + presence: MetadataPresence, +) { + let target = tracked_target(presence, MessageData::TRACKED_METADATA); + if presence.has_changed_for(MessageData::TRACKED_METADATA) + || tracked_kinds(current.bits(), MessageData::TRACKED_METADATA) != target + { + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(document_id); + merge_metadata_kinds(batch, account_id, target); + } +} + diff --git a/crates/email/src/push/mod.rs b/crates/email/src/push/mod.rs index 20e572b6..3203e0af 100644 --- a/crates/email/src/push/mod.rs +++ b/crates/email/src/push/mod.rs @@ -92,3 +92,8 @@ impl Urgency { } } } + +impl store::write::ArchiveCompression for PushSubscriptions { + const COMPRESSION: store::write::Compression = + store::write::Compression::Zstd(Some(store::write::Dictionary::Common)); +} diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index 730fb43e..d9f6d055 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -9,10 +9,11 @@ use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::write::BatchBuilder; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + roaring::RoaringBitmap, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; -use types::{collection::Collection, field::SieveField}; +use types::collection::Collection; pub trait SieveScriptDelete: Sync + Send { fn sieve_script_delete( @@ -35,7 +36,7 @@ impl SieveScriptDelete for Server { // Fetch record if let Some(obj_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -43,18 +44,29 @@ impl SieveScriptDelete for Server { .await? { // Delete record + let script = obj_ + .to_unarchived::() + .caused_by(trc::location!())?; batch .with_account_id(account_id) - .with_collection(Collection::SieveScript) + .with_collection(Collection::SieveScript); + if !script.inner.metadata_kinds().is_empty() { + self.preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::SieveScript, + &RoaringBitmap::from_iter([document_id]), + ) + .await + .caused_by(trc::location!())? + .release_or_assert_absent(batch, account_id, document_id); + } + batch .with_document(document_id) - .clear(SieveField::Ids) .custom( ObjectIndexBuilder::<_, ()>::new() - .with_current( - obj_.to_unarchived::() - .caused_by(trc::location!())?, - ) - .with_changed_by(access_token.account_tenant_ids()), + .with_current(script) + .with_changed_by(access_token.changed_by()), ) .caused_by(trc::location!())? .commit_point(); diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index 2e5753aa..066f1fb3 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -5,8 +5,12 @@ */ use super::{ArchivedSieveScript, SieveScript}; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; -use types::{collection::SyncCollection, field::SieveField}; +use common::storage::index::{ + IndexValue, IndexableAndSerializableObject, IndexableObject, SerializableObject, + serialize_object, +}; +use store::write::{ArchiveCompression, BatchBuilder, Compression, Dictionary, Slot}; +use types::{collection::SyncCollection, field::SieveField, metadata::MetadataKinds}; impl IndexableObject for SieveScript { fn index_values(&self) -> impl Iterator> { @@ -17,6 +21,7 @@ impl IndexableObject for SieveScript { }, IndexValue::Blob { value: self.blob_hash.clone(), + size: u64::from(self.size), }, IndexValue::LogItem { sync_collection: SyncCollection::SieveScript, @@ -26,6 +31,10 @@ impl IndexableObject for SieveScript { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + SieveScript::metadata_kinds(self) + } } impl IndexableAndSerializableObject for SieveScript { @@ -43,6 +52,7 @@ impl IndexableObject for &ArchivedSieveScript { }, IndexValue::Blob { value: (&self.blob_hash).into(), + size: u64::from(self.size.to_native()), }, IndexValue::LogItem { sync_collection: SyncCollection::SieveScript, @@ -54,4 +64,18 @@ impl IndexableObject for &ArchivedSieveScript { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedSieveScript::metadata_kinds(self) + } +} + +impl ArchiveCompression for SieveScript { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Sieve)); +} + +impl SerializableObject for SieveScript { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } } diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 12c17c00..3a6fbb31 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -4,31 +4,39 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ActiveScript, SeenIdHash, SieveScript}; +use super::{ActiveScript, ArchivedSieveScript, Bytecode, SeenId, SeenIdHash, SieveScript}; use crate::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, - mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc}, + mailbox::{INBOX_ID, TRASH_ID, lookup::MailboxLookup, manage::MailboxFnc}, message::{ - delivery::{AutogeneratedMessage, IngestRecipient}, + delivery::{ + AutogeneratedMessage, AutogeneratedReturnPath, IngestRecipient, SenderAuthentication, + }, ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, }, + sieve::VacationResponse, }; use common::{ - Server, auth::AccessToken, config::mailstore::spamfilter::spam_status, + MessageStoreCache, Server, + auth::{AccessToken, active_script::CachedActiveScript}, + config::mailstore::spamfilter::spam_status, scripts::plugins::PluginContext, }; +use compact_str::{CompactString, ToCompactString}; use mail_builder::headers::date::Date; use mail_parser::{HeaderName, MessageParser}; -use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; +use sieve::{ + Arena, Context, Envelope, Handler, Input, Mailbox, MessageSource, Recipient, Reply, + ScriptArena, Sieve, SieveAction, Status, + compiler::grammar::actions::action_redirect::Notify, + runtime::{Script, Variable}, +}; use std::{borrow::Cow, sync::Arc}; use std::{future::Future, str::FromStr}; use store::{ - Deserialize, Serialize, ValueKey, + Serialize, ValueKey, ahash::AHashMap, - dispatch::lookup::KeyValue, - write::{ - AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, - }, + write::{Archive, ArchiveBytes, ArchiveVersion, Archiver, BatchBuilder, ValueClass, now}, }; use trc::{AddContext, SieveEvent, SmtpEvent}; use types::{ @@ -47,6 +55,447 @@ struct SieveMessage<'x> { pub flags: Vec, } +struct SieveHandler<'x, 'r> { + server: &'x Server, + session_id: u64, + script_hash: u32, + cache: Option>, + resolved: Vec<(MailboxQuery, Option)>, + has_mailboxes: bool, + mail_from: &'x str, + server_name: &'x str, + mail_max_size: usize, + max_received_headers: usize, + received_headers: usize, + messages: Vec>, + autogenerated: Vec, + checked_ids: AHashMap, + seen_ids: Vec, + reject_reason: Option, + do_discard: bool, + do_deliver: bool, + do_redirect: bool, + redirect_copies: usize, + pending: Option>, +} + +enum PendingWork<'x> { + Include { + name: String, + }, + DuplicateId { + hash: SeenIdHash, + expiry: u64, + last: bool, + }, + Function { + id: u32, + arguments: Vec>, + }, + MailboxExists(MailboxExistsRequest), + FileInto(FileIntoRequest), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum MailboxQuery { + Path(Box), + Role(SpecialUse), + Id(u32), +} + +struct MailboxExistsRequest { + mailboxes: Vec>, + roles: Vec>, +} + +struct FileIntoRequest { + path: MailboxQuery, + mailbox_id: Option, + special_use: Option, + create: bool, + flags: Vec, + message_id: usize, +} + +enum FileIntoTarget { + Mailbox(u32), + Create, +} + +enum Unresolved<'x> { + Query(&'x MailboxQuery), + Cache, +} + +impl MailboxQuery { + fn from_id(id: &str) -> Option { + Id::from_str(id) + .ok() + .map(|id| MailboxQuery::Id(id.document_id())) + } + + fn from_role(role: &str) -> Option { + SpecialUse::parse_use_attr(role).map(MailboxQuery::Role) + } + + async fn resolve(&self, server: &Server, account_id: u32) -> trc::Result> { + match self { + MailboxQuery::Path(path) => server.mailbox_id_by_path(account_id, path).await, + MailboxQuery::Role(role) => server.mailbox_id_by_role(account_id, *role).await, + MailboxQuery::Id(id) => server + .mailbox_id_exists(account_id, *id) + .await + .map(|exists| exists.then_some(*id)), + } + } +} + +impl SieveHandler<'_, '_> { + fn answer(&self, query: &MailboxQuery) -> Option> { + match query { + MailboxQuery::Role(SpecialUse::Inbox) => return Some(Some(INBOX_ID)), + MailboxQuery::Role(SpecialUse::Trash) => return Some(Some(TRASH_ID)), + _ => {} + } + + match &self.cache { + Some(cache) => Some(match query { + MailboxQuery::Path(path) => cache.mailbox_by_path(path).map(|m| m.document_id), + MailboxQuery::Role(role) => cache.mailbox_by_role(role).map(|m| m.document_id), + MailboxQuery::Id(id) => cache.has_mailbox_id(id).then_some(*id), + }), + None => self + .resolved + .iter() + .find(|(resolved, _)| resolved == query) + .map(|(_, document_id)| *document_id), + } + } + + fn mailbox_exists_answer<'q>( + &self, + request: &'q MailboxExistsRequest, + ) -> Result { + let mut roles = Vec::with_capacity(request.roles.len()); + for role in &request.roles { + roles.push(match role { + Some(query) => self.answer(query).ok_or(query)?.unwrap_or(u32::MAX), + None => u32::MAX, + }); + } + + if !request.mailboxes.is_empty() { + for mailbox in &request.mailboxes { + let document_id = match mailbox { + Some(query) => self.answer(query).ok_or(query)?, + None => None, + }; + if !document_id + .is_some_and(|document_id| roles.is_empty() || roles.contains(&document_id)) + { + return Ok(false); + } + } + Ok(true) + } else { + Ok(!roles.is_empty() && !roles.contains(&u32::MAX)) + } + } + + fn file_into_target<'q>( + &self, + request: &'q FileIntoRequest, + ) -> Result> { + for query in [&request.mailbox_id, &request.special_use] + .into_iter() + .flatten() + { + if let Some(document_id) = self.answer(query).ok_or(Unresolved::Query(query))? { + return Ok(FileIntoTarget::Mailbox(document_id)); + } + } + + if request.create { + let Some(cache) = &self.cache else { + return Err(Unresolved::Cache); + }; + let MailboxQuery::Path(path) = &request.path else { + return Ok(FileIntoTarget::Mailbox(INBOX_ID)); + }; + Ok(cache + .mailbox_by_folded_path(path) + .map_or(FileIntoTarget::Create, |mailbox| { + FileIntoTarget::Mailbox(mailbox.document_id) + })) + } else { + self.answer(&request.path) + .ok_or(Unresolved::Query(&request.path)) + .map(|document_id| FileIntoTarget::Mailbox(document_id.unwrap_or(INBOX_ID))) + } + } + + fn file_into( + &mut self, + mailbox_id: u32, + flags: Vec, + message_id: usize, + is_file_into: bool, + ) { + if let Some(message) = self.messages.get_mut(message_id) { + message.flags = flags; + if !message.file_into.contains(&mailbox_id) { + message.file_into.push(mailbox_id); + } + message.did_file_into |= is_file_into; + self.do_deliver = true; + } else { + trc::event!( + Sieve(SieveEvent::UnexpectedError), + Details = "Unknown message id.", + MessageId = message_id, + SpanId = self.session_id + ); + } + } + + fn send_message( + &mut self, + source: MessageSource, + recipient: Recipient<'_>, + notify: Notify, + message_id: usize, + ) { + let (sender_address, return_path) = if source == MessageSource::Vacation { + ("", AutogeneratedReturnPath::Null) + } else { + (self.mail_from, AutogeneratedReturnPath::Account) + }; + let Some(message) = self.messages.get(message_id) else { + trc::event!( + Sieve(SieveEvent::UnexpectedError), + Details = "Unknown message id.", + MessageId = message_id, + SpanId = self.session_id + ); + return; + }; + + if self.received_headers >= self.max_received_headers { + trc::event!( + Smtp(SmtpEvent::LoopDetected), + From = CompactString::from(self.mail_from), + Total = self.received_headers, + Limit = self.max_received_headers, + SpanId = self.session_id, + ); + return; + } + + let recipients: Vec = match recipient { + Recipient::Address(rcpt) => vec![rcpt.to_string()], + Recipient::Group(rcpts) => rcpts.iter().map(|rcpt| rcpt.to_string()).collect(), + Recipient::List(_) => return, + }; + + if message.raw_message.len() > self.mail_max_size { + trc::event!( + Sieve(SieveEvent::MessageTooLarge), + From = CompactString::from(self.mail_from), + To = recipients + .iter() + .map(|r| trc::Value::String(r.as_str().into())) + .collect::>(), + Size = message.raw_message.len(), + Limit = self.mail_max_size, + SpanId = self.session_id, + ); + return; + } + + trc::event!( + Sieve(SieveEvent::SendMessage), + From = CompactString::from(sender_address), + To = recipients + .iter() + .map(|r| trc::Value::String(r.as_str().into())) + .collect::>(), + Size = message.raw_message.len(), + SpanId = self.session_id + ); + + let mut raw_message = Vec::with_capacity(160 + message.raw_message.len()); + write_received_header(&mut raw_message, self.server_name, self.session_id); + raw_message.extend_from_slice(message.raw_message.as_ref()); + + if source == MessageSource::Redirect { + self.redirect_copies += recipients.len(); + } + self.autogenerated.push(AutogeneratedMessage { + account_address: self.mail_from.to_string(), + return_path, + recipients, + message: raw_message, + notify, + source, + exploded: false, + }); + self.do_redirect = true; + } +} + +fn keywords(flags: &[&str]) -> Vec { + flags + .iter() + .filter_map(|flag| { + Keyword::try_parse(flag) + .or_else(|| Keyword::is_atom(flag.as_bytes()).then(|| Keyword::from_str(flag))) + }) + .collect() +} + +impl<'x> Handler<'x> for SieveHandler<'x, '_> { + fn include_script( + &mut self, + _: &Context<'x>, + name: Script<'_>, + optional: bool, + ) -> Reply>> { + match name { + Script::Personal(name) => { + self.pending = Some(PendingWork::Include { + name: name.to_string(), + }); + Reply::Pending + } + Script::Global(name) => Reply::Ready(if optional { + self.server.core.sieve.untrusted_script(name) + } else { + self.server + .get_untrusted_sieve_script(name, self.session_id) + }), + } + } + + fn mailbox_exists( + &mut self, + _: &Context<'x>, + mailboxes: &[Mailbox<'_>], + special_use: &[&str], + ) -> Reply { + let request = MailboxExistsRequest { + mailboxes: mailboxes + .iter() + .map(|mailbox| match mailbox { + Mailbox::Name(name) => Some(MailboxQuery::Path((*name).into())), + Mailbox::Id(id) => MailboxQuery::from_id(id), + }) + .collect(), + roles: special_use + .iter() + .map(|role| MailboxQuery::from_role(role)) + .collect(), + }; + + match self.mailbox_exists_answer(&request).ok() { + Some(exists) => Reply::Ready(exists), + None => { + self.pending = Some(PendingWork::MailboxExists(request)); + Reply::Pending + } + } + } + + fn duplicate_id(&mut self, _: &Context<'x>, id: &str, expiry: u64, last: bool) -> Reply { + if expiry == 0 { + return Reply::Ready(false); + } + + let hash = SeenIdHash::new(self.script_hash, id); + match self.checked_ids.get(&hash) { + Some(result) => Reply::Ready(*result), + None => { + self.pending = Some(PendingWork::DuplicateId { hash, expiry, last }); + Reply::Pending + } + } + } + + fn function( + &mut self, + _: &Context<'x>, + id: u32, + arguments: &[Variable<'x>], + ) -> Reply> { + self.pending = Some(PendingWork::Function { + id, + arguments: arguments.to_vec(), + }); + Reply::Pending + } + + fn action(&mut self, _: &Context<'x>, action: SieveAction<'_>) -> Reply<()> { + match action { + SieveAction::Keep { flags, message_id } => { + let flags = keywords(flags); + self.file_into(INBOX_ID, flags, message_id, false); + } + SieveAction::Discard => { + self.do_discard = true; + } + SieveAction::Reject { reason, .. } => { + self.reject_reason = reason.to_string().into(); + self.do_discard = true; + } + SieveAction::FileInto { + folder, + flags, + mailbox_id, + special_use, + create, + message_id, + } => { + let request = FileIntoRequest { + path: MailboxQuery::Path(folder.into()), + mailbox_id: mailbox_id.and_then(MailboxQuery::from_id), + special_use: special_use.and_then(MailboxQuery::from_role), + create, + flags: keywords(flags), + message_id, + }; + match self.file_into_target(&request).ok() { + Some(FileIntoTarget::Mailbox(target_id)) => { + self.file_into(target_id, request.flags, message_id, true) + } + _ => { + self.pending = Some(PendingWork::FileInto(request)); + return Reply::Pending; + } + } + } + SieveAction::SendMessage { + source, + recipient, + notify, + message_id, + .. + } => { + self.send_message(source, recipient, notify, message_id); + } + SieveAction::CreatedMessage { message, .. } => { + self.messages.push(SieveMessage { + raw_message: message.into(), + file_into: Vec::new(), + did_file_into: false, + flags: Vec::new(), + }); + } + SieveAction::Notify { .. } | SieveAction::SetEnvelope { .. } => (), + } + + Reply::Ready(()) + } +} + pub trait SieveScriptIngest: Sync + Send { #[allow(clippy::too_many_arguments)] fn sieve_script_ingest( @@ -55,7 +504,7 @@ pub trait SieveScriptIngest: Sync + Send { blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, - envelope_from_authenticated: bool, + sender_authentication: SenderAuthentication, envelope_to: &IngestRecipient, session_id: u64, active_script: ActiveScript, @@ -76,13 +525,22 @@ pub trait SieveScriptIngest: Sync + Send { &self, account_id: u32, name: &str, - ) -> impl Future>> + Send; + ) -> impl Future>>> + Send; - fn sieve_script_compile( + fn sieve_script_load( &self, account_id: u32, document_id: u32, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; + + fn sieve_script_recompile( + &self, + account_id: u32, + document_id: u32, + version: ArchiveVersion, + archive: &Archive, + script_object: &ArchivedSieveScript, + ) -> impl Future> + Send; } impl SieveScriptIngest for Server { @@ -93,74 +551,84 @@ impl SieveScriptIngest for Server { blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, - envelope_from_authenticated: bool, + sender_authentication: SenderAuthentication, envelope_to: &IngestRecipient, session_id: u64, active_script: ActiveScript, autogenerated: &mut Vec, ) -> trc::Result { // Parse message - let message = if let Some(message) = MessageParser::new().parse(raw_message) { - message - } else { - return Err( - trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) - .ctx(trc::Key::Code, 550) - .ctx(trc::Key::Reason, "Failed to parse e-mail message."), - ); - }; + let message = MessageParser::new().parse(raw_message).ok_or_else(|| { + trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) + .ctx(trc::Key::Code, 550) + .ctx(trc::Key::Reason, "Failed to parse e-mail message.") + })?; - let received_headers = message - .headers() - .iter() - .filter(|header| matches!(header.name, HeaderName::Received)) - .count(); + let received_headers = message.headers().all(HeaderName::Received).count(); // Obtain mailboxIds let account_id = access_token.account_id(); - let mut cache = self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; - - // Create Sieve instance - let orcpt = envelope_to.orcpt_parameter(); - let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message); + let cache = self.fresh_messages(account_id); // Set account name and email let account_info = self.account(account_id).await.caused_by(trc::location!())?; - let mail_from = account_info.name().to_string(); - instance.set_user_full_name( + let mail_from = account_info.name(); + + // Load the compiled script + let included = ScriptArena::new(); + let mut resolved_includes: Vec<(String, &Sieve<'static>)> = Vec::new(); + let mut recompiled = None; + let script = verified_script(self, account_id, &active_script, &mut recompiled).await?; + + // Create Sieve instance + let mut arena = Arena::new(); + let mut ctx = self + .core + .sieve + .untrusted_runtime + .filter_parsed(&message, &script, &mut arena); + ctx.set_user_full_name( account_info .description() .unwrap_or_else(|| account_info.name()), ); - instance.set_user_address(&mail_from); - - // Set envelope - instance.set_envelope(Envelope::From, envelope_from); - instance.set_envelope(Envelope::To, envelope_to.address.as_str()); - if let Some(orcpt) = &orcpt { - instance.set_envelope(Envelope::Orcpt, orcpt.as_str()); + ctx.set_user_address(mail_from); + ctx.set_envelope(Envelope::From, envelope_from); + ctx.set_envelope(Envelope::To, envelope_to.address.as_str()); + if let Some(orcpt) = envelope_to.orcpt_parameter() { + ctx.set_envelope(Envelope::Orcpt, orcpt); } - instance.set_spam_status(spam_status(envelope_to.spam_percentage)); + ctx.set_spam_status(spam_status(envelope_to.spam_percentage)); - let mut input = Input::script( - active_script.script_name.to_string(), - active_script.script.clone(), - ); + let mut handler = SieveHandler { + server: self, + session_id, + script_hash: active_script.version.hash().unwrap_or_default(), + cache, + resolved: Vec::new(), + has_mailboxes: false, + mail_from, + server_name: self.core.network.server_name.as_str(), + mail_max_size: self.core.email.mail_max_size, + max_received_headers: self.core.sieve.max_received_headers, + received_headers, + messages: vec![SieveMessage { + raw_message: raw_message.into(), + file_into: Vec::new(), + did_file_into: false, + flags: Vec::new(), + }], + autogenerated: Vec::new(), + checked_ids: AHashMap::new(), + seen_ids: Vec::new(), + reject_reason: None, + do_discard: false, + do_deliver: false, + do_redirect: false, + redirect_copies: 0, + pending: None, + }; - let mut do_discard = false; - let mut do_deliver = false; - let mut do_redirect = false; - - let mut reject_reason = None; - let mut messages: Vec = vec![SieveMessage { - raw_message: raw_message.into(), - file_into: Vec::new(), - flags: Vec::new(), - did_file_into: false, - }]; let mut ingested_message = IngestedEmail { document_id: 0, thread_id: 0, @@ -168,337 +636,126 @@ impl SieveScriptIngest for Server { blob_id: Default::default(), size: raw_message.len(), imap_uids: Vec::new(), + thread_recheck: None, }; - let mut checked_ids: AHashMap = AHashMap::new(); - while let Some(event) = instance.run(input) { - match event { - Ok(event) => match event { - Event::IncludeScript { name, .. } => match &name { - sieve::Script::Personal(name_) => { - if let Ok(Some(script)) = - self.sieve_script_get_by_name(account_id, name_).await + loop { + match ctx.run(&mut handler) { + Ok(Status::Finished) => break, + Ok(Status::Pending) => { + let input = match handler.pending.take() { + Some(PendingWork::Include { name }) => { + let script = if let Some((_, script)) = resolved_includes + .iter() + .find(|(resolved, _)| *resolved == name) { - input = Input::script(name, script); - } else { - input = false.into(); - } - } - sieve::Script::Global(name_) => { - if let Some(script) = self.get_untrusted_sieve_script(name_, session_id) + Some(*script) + } else if let Some(script) = + self.sieve_script_get_by_name(account_id, &name).await? { - input = Input::script(name, script.clone()); + let script = included.push(script); + resolved_includes.push((name, script)); + Some(script) } else { - input = false.into(); - } + None + }; + + Input::Script(script) } - }, - Event::MailboxExists { - mailboxes, - special_use, - } => { - if !mailboxes.is_empty() { - let mut special_use_ids = Vec::with_capacity(special_use.len()); - for role in special_use.iter().map(|v| SpecialUse::parse_use_attr(v)) { - special_use_ids.push(match role { - Some(SpecialUse::Inbox) => INBOX_ID, - Some(SpecialUse::Trash) => TRASH_ID, - Some(role) => cache - .mailbox_by_role(&role) - .map(|m| m.document_id) - .unwrap_or(u32::MAX), - None => u32::MAX, + Some(PendingWork::DuplicateId { hash, expiry, last }) => { + let now = now(); + let exists = self + .store() + .get_value::(hash.key(account_id)) + .await + .caused_by(trc::location!())? + .is_some_and(|expires_at| expires_at > now); + + if !exists || last { + handler.seen_ids.push(SeenId { + hash, + expires_at: now.saturating_add(expiry), }); } - let mut result = true; - for mailbox in mailboxes { - match mailbox { - Mailbox::Name(name) => { - if !matches!( - cache.mailbox_by_path(&name), - Some(item) if special_use_ids.is_empty() || - special_use_ids.contains(&item.document_id) - ) { - result = false; - break; - } - } - Mailbox::Id(id) => { - if !matches!(Id::from_str(&id), Ok(id) if - cache.has_mailbox_id(&id.document_id()) && - (special_use_ids.is_empty() || - special_use_ids.contains(&id.document_id()))) - { - result = false; - break; - } - } - } - } - input = result.into(); - } else if !special_use.is_empty() { - let mut result = true; + handler.checked_ids.insert(hash, exists); - for role in special_use.iter().map(|v| SpecialUse::parse_use_attr(v)) { - match role { - Some(SpecialUse::Inbox | SpecialUse::Trash) => {} - Some(other) if cache.mailbox_by_role(&other).is_some() => {} - _ => { - result = false; - break; - } - } - } - input = result.into(); - } else { - input = false.into(); + Input::Bool(exists) } - } - Event::DuplicateId { id, expiry, last } => { - let id_hash = SeenIdHash::new( - account_id, - active_script.version.hash().unwrap_or_default(), - &id, - ); - if let Some(result) = checked_ids.get(&id_hash) { - input = (*result).into(); - } else { - let exists = self - .in_memory_store() - .key_exists(id_hash.key()) + Some(PendingWork::Function { id, arguments }) => { + self.core + .run_plugin( + id, + PluginContext { + session_id, + server: self, + message: ctx.message(), + modifications: &mut Vec::new(), + access_token: access_token.into(), + arguments, + }, + ) .await - .caused_by(trc::location!())?; - - if !exists || last { - self.in_memory_store() - .key_set(KeyValue::new(id_hash.key(), vec![]).expires(expiry)) - .await - .caused_by(trc::location!())?; - } - - checked_ids.insert(id_hash, exists); - input = exists.into(); } - } - Event::Discard => { - do_discard = true; - input = true.into(); - } - Event::Reject { reason, .. } => { - reject_reason = reason.into(); - do_discard = true; - input = true.into(); - } - Event::Keep { flags, message_id } => { - if let Some(message) = messages.get_mut(message_id) { - message.flags = flags.into_iter().map(Keyword::from).collect(); - if !message.file_into.contains(&INBOX_ID) { - message.file_into.push(INBOX_ID); - } - do_deliver = true; - } else { - trc::event!( - Sieve(SieveEvent::UnexpectedError), - Details = "Unknown message id.", - MessageId = message_id, - SpanId = session_id - ); - } - input = true.into(); - } - Event::FileInto { - folder, - flags, - mailbox_id, - special_use, - create, - message_id, - } => { - let mut target_id = u32::MAX; - - // Find mailbox by Id - if let Some(mailbox_id) = mailbox_id.and_then(|m| Id::from_str(&m).ok()) { - let mailbox_id = mailbox_id.document_id(); - if cache.has_mailbox_id(&mailbox_id) { - target_id = mailbox_id; - } - } - - // Find mailbox by role - if target_id == u32::MAX - && let Some(special_use) = - special_use.as_deref().and_then(SpecialUse::parse_use_attr) - { - match special_use { - SpecialUse::Inbox => { - target_id = INBOX_ID; - } - SpecialUse::Trash => { - target_id = TRASH_ID; - } - role => { - if let Some(item) = cache.mailbox_by_role(&role) { - target_id = item.document_id; + Some(PendingWork::MailboxExists(request)) => { + let exists = loop { + match handler.mailbox_exists_answer(&request) { + Ok(exists) => break exists, + Err(query) => { + resolve_mailbox(self, account_id, &mut handler, query) + .await?; } } - } - } - - // Find mailbox by name - if target_id == u32::MAX { - if !create { - if let Some(m) = cache.mailbox_by_path(&folder) { - target_id = m.document_id; - } - } else if let Some(document_id) = self - .mailbox_create_path(account_id, &folder) - .await - .caused_by(trc::location!())? - { - cache = self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; - target_id = document_id; - } - } - - // Default to Inbox - if target_id == u32::MAX { - target_id = INBOX_ID; - } - - if let Some(message) = messages.get_mut(message_id) { - message.flags = flags.into_iter().map(Keyword::from).collect(); - if !message.file_into.contains(&target_id) { - message.file_into.push(target_id); - } - message.did_file_into = true; - do_deliver = true; - } else { - trc::event!( - Sieve(SieveEvent::UnexpectedError), - Details = "Unknown message id.", - MessageId = message_id, - SpanId = session_id - ); - } - input = true.into(); - } - Event::SendMessage { - recipient, - message_id, - .. - } => { - input = true.into(); - if let Some(message) = messages.get(message_id) { - if received_headers >= self.core.sieve.max_received_headers { - trc::event!( - Smtp(SmtpEvent::LoopDetected), - From = mail_from.clone(), - Total = received_headers, - Limit = self.core.sieve.max_received_headers, - SpanId = session_id, - ); - - continue; - } - - let recipients: Vec = match recipient { - Recipient::Address(rcpt) => vec![rcpt], - Recipient::Group(rcpts) => rcpts, - Recipient::List(_) => { - // Not yet implemented - continue; - } }; - if message.raw_message.len() <= self.core.email.mail_max_size { - trc::event!( - Sieve(SieveEvent::SendMessage), - From = mail_from.clone(), - To = recipients - .iter() - .map(|r| trc::Value::String(r.as_str().into())) - .collect::>(), - Size = message.raw_message.len(), - SpanId = session_id - ); - - let mut raw_message = - Vec::with_capacity(160 + message.raw_message.len()); - write_received_header( - &mut raw_message, - &self.core.network.server_name, - session_id, - ); - raw_message.extend_from_slice(message.raw_message.as_ref()); - - autogenerated.push(AutogeneratedMessage { - sender_address: mail_from.clone(), - recipients, - message: raw_message, - }); - do_redirect = true; - } else { - trc::event!( - Sieve(SieveEvent::MessageTooLarge), - From = mail_from.clone(), - To = recipients - .iter() - .map(|r| trc::Value::String(r.as_str().into())) - .collect::>(), - Size = message.raw_message.len(), - Limit = self.core.email.mail_max_size, - SpanId = session_id, - ); - } - } else { - trc::event!( - Sieve(SieveEvent::UnexpectedError), - Details = "Unknown message id.", - MessageId = message_id, - SpanId = session_id - ); - - continue; + Input::Bool(exists) } - } - Event::ListContains { .. } - | Event::Notify { .. } - | Event::SetEnvelope { .. } => { - // Not allowed - input = false.into(); - } - Event::Function { id, arguments } => { - input = self - .core - .run_plugin( - id, - PluginContext { - session_id, - server: self, - message: instance.message(), - modifications: &mut Vec::new(), - access_token: access_token.into(), - arguments, - }, - ) - .await; - } - Event::CreatedMessage { message, .. } => { - messages.push(SieveMessage { - raw_message: message.into(), - file_into: Vec::new(), - flags: Vec::new(), - did_file_into: false, - }); - input = true.into(); - } - }, + Some(PendingWork::FileInto(request)) => { + let target = loop { + match handler.file_into_target(&request) { + Ok(target) => break target, + Err(Unresolved::Query(query)) => { + resolve_mailbox(self, account_id, &mut handler, query) + .await?; + } + Err(Unresolved::Cache) => { + handler.cache = Some( + self.get_cached_messages(account_id) + .await + .caused_by(trc::location!())?, + ); + } + } + }; + let target_id = match (target, &request.path) { + (FileIntoTarget::Mailbox(target_id), _) => target_id, + (FileIntoTarget::Create, MailboxQuery::Path(folder)) => { + match self + .mailbox_create_path(account_id, folder) + .await + .caused_by(trc::location!())? + { + Some(document_id) => { + handler.cache = None; + handler.resolved.clear(); + document_id + } + None => INBOX_ID, + } + } + (FileIntoTarget::Create, _) => INBOX_ID, + }; + handler.file_into(target_id, request.flags, request.message_id, true); - #[cfg(feature = "test_mode")] + Input::Continue + } + None => Input::Continue, + }; + + ctx.resume(input); + } + + #[cfg(stalwart_tests)] Err(sieve::runtime::RuntimeError::ScriptErrorMessage(err)) => { panic!("Sieve test failed: {}", err); } @@ -506,18 +763,35 @@ impl SieveScriptIngest for Server { Err(err) => { trc::event!( Sieve(SieveEvent::RuntimeError), - Reason = err.to_string(), + Reason = err.to_compact_string(), SpanId = session_id ); - - input = true.into(); } } } + let SieveHandler { + mut messages, + autogenerated: mut generated, + seen_ids, + reject_reason, + do_discard, + do_deliver, + do_redirect, + redirect_copies, + .. + } = handler; + let message_changed = ctx.has_message_changed(); + drop(ctx); + let mut message = Some(message); + // Fail-safe, no discard and no keep seen, assume that something went wrong and file anyway. - if !do_deliver && !do_discard && !do_redirect { - messages[0].file_into.push(INBOX_ID); + if !do_deliver + && !do_discard + && !do_redirect + && let Some(message) = messages.first_mut() + { + message.file_into.push(INBOX_ID); } // Deliver messages @@ -526,8 +800,11 @@ impl SieveScriptIngest for Server { for (message_id, sieve_message) in messages.into_iter().enumerate() { if !sieve_message.file_into.is_empty() { // Parse message if needed - let (blob_hash, message) = if message_id == 0 && !instance.has_message_changed() { - (blob_hash.into(), instance.take_message()) + let (blob_hash, message) = if message_id == 0 + && !message_changed + && let Some(message) = message.take() + { + (blob_hash.into(), message) } else if let Some(message) = MessageParser::new().parse(sieve_message.raw_message.as_ref()) { @@ -554,9 +831,11 @@ impl SieveScriptIngest for Server { received_at: None, source: IngestSource::Smtp { deliver_to: envelope_to.address.as_str(), - is_sender_authenticated: envelope_from_authenticated, + envelope_from, + sender_authentication, is_spam: envelope_to.is_spam() && !sieve_message.did_file_into, }, + metadata: None, session_id, }) .await @@ -572,6 +851,30 @@ impl SieveScriptIngest for Server { } } + if reject_reason.is_some() || has_delivered || last_temp_error.is_none() { + if !seen_ids.is_empty() { + let mut batch = BatchBuilder::new(); + SeenId::write_all(&seen_ids, account_id, &mut batch); + if let Err(err) = self.store().write_batch(&mut batch).await { + trc::error!( + err.account_id(account_id) + .span_id(session_id) + .caused_by(trc::location!()) + .details("Failed to record Sieve duplicate and vacation ids") + ); + } + } + if redirect_copies + usize::from(has_delivered) > 1 { + for message in generated + .iter_mut() + .filter(|message| message.source == MessageSource::Redirect) + { + message.exploded = true; + } + } + autogenerated.append(&mut generated); + } + if let Some(reject_reason) = reject_reason { Err( trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) @@ -600,30 +903,22 @@ impl SieveScriptIngest for Server { } async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result> { - // Find the currently active script - if let Some(document_id) = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::ActiveScriptId.into()), - }) - .await - .caused_by(trc::location!())? - { - if let Some(script) = self.sieve_script_compile(account_id, document_id).await? { - Ok(Some(ActiveScript { - document_id, - script: Arc::new(script.script), - script_name: script.name, - version: script.version, - })) - } else { - Ok(None) + let account = self.account(account_id).await.caused_by(trc::location!())?; + let document_id = match account.active_script.get() { + CachedActiveScript::Known(document_id) => document_id, + CachedActiveScript::Unknown(ticket) => { + let document_id = self + .sieve_script_get_active_id(account_id) + .await + .caused_by(trc::location!())?; + account.active_script.fill(ticket, document_id); + document_id } - } else { - Ok(None) + }; + + match document_id { + Some(document_id) => self.sieve_script_load(account_id, document_id).await, + None => Ok(None), } } @@ -631,9 +926,9 @@ impl SieveScriptIngest for Server { &self, account_id: u32, name: &str, - ) -> trc::Result> { + ) -> trc::Result>> { // Find the script by name - if let Some(document_id) = self + let Some(document_id) = self .document_ids_matching( account_id, Collection::SieveScript, @@ -643,47 +938,51 @@ impl SieveScriptIngest for Server { .await .caused_by(trc::location!())? .min() - { - self.sieve_script_compile(account_id, document_id) - .await - .map(|script| script.map(|s| s.script)) - } else { - Ok(None) - } + else { + return Ok(None); + }; + + let Some(active_script) = self.sieve_script_load(account_id, document_id).await? else { + return Ok(None); + }; + let mut recompiled = None; + + verified_script(self, account_id, &active_script, &mut recompiled) + .await + .map(|script| Some(script.into_owned())) } - #[allow(clippy::blocks_in_conditions)] - async fn sieve_script_compile( + async fn sieve_script_load( &self, account_id: u32, document_id: u32, - ) -> trc::Result> { - // Obtain script object - let Some(script_object) = self + ) -> trc::Result> { + Ok(self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? - else { - return Ok(None); - }; + .map(|archive| ActiveScript { + document_id, + version: archive.version, + script: Bytecode::Stored(archive), + })) + } - // Obtain the sieve script length - let version = script_object.version; - let unarchived_script = script_object - .unarchive::() - .caused_by(trc::location!())?; - let script_offset = u32::from(unarchived_script.size) as usize; - - // Obtain the sieve script blob + async fn sieve_script_recompile( + &self, + account_id: u32, + document_id: u32, + version: ArchiveVersion, + archive: &Archive, + script_object: &ArchivedSieveScript, + ) -> trc::Result { let script_bytes = self - .core - .storage - .blob - .get_blob(unarchived_script.blob_hash.0.as_ref(), 0..usize::MAX) + .blob_store() + .get_blob(script_object.blob_hash.0.as_ref(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -693,88 +992,123 @@ impl SieveScriptIngest for Server { .document_id(document_id) })?; - // Obtain the precompiled script - if let Some(script) = script_bytes.get(script_offset..).and_then(|bytes| { - as Deserialize>::deserialize(bytes) - .ok()? - .deserialize::() - .ok() - }) { - Ok(Some(CompiledScript { - script, - name: unarchived_script.name.as_str().into(), - version, - })) - } else { - // Deserialization failed, probably because the script compiler version changed - match self.core.sieve.untrusted_compiler.compile( - script_bytes.get(0..script_offset).ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?, - ) { - Ok(sieve) => { - // Store updated compiled sieve script - let sieve = Archiver::new(sieve).untrusted(); - let compiled_bytes = sieve.serialize().caused_by(trc::location!())?; - let mut updated_sieve_bytes = - Vec::with_capacity(script_offset + compiled_bytes.len()); - updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); - updated_sieve_bytes.extend_from_slice(&compiled_bytes); - - // Store updated blob - let (new_blob_hash, new_blob_hold) = self - .put_temporary_blob(account_id, &updated_sieve_bytes, 60) - .await?; - let mut new_script_object = - rkyv::deserialize(unarchived_script).caused_by(trc::location!())?; - let blob_hash = - std::mem::replace(&mut new_script_object.blob_hash, new_blob_hash.clone()); - let new_archive = Archiver::new(new_script_object); - - // Update script object - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::SieveScript) - .with_document(document_id) - .assert_value(SieveField::Archive, &script_object) - .set( - SieveField::Archive, - new_archive.serialize().caused_by(trc::location!())?, - ) - .clear(BlobOp::Link { - hash: blob_hash, - to: BlobLink::Document, - }) - .set( - BlobOp::Link { - hash: new_blob_hash, - to: BlobLink::Document, - }, - Vec::new(), - ) - .clear(new_blob_hold); - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - - Ok(Some(CompiledScript { - script: sieve.into_inner(), - name: new_archive.into_inner().name, - version, - })) - } - Err(error) => Err(trc::StoreEvent::UnexpectedError + let script = self + .core + .sieve + .untrusted_compiler + .compile(&script_bytes) + .map_err(|err| { + trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) - .reason(error) - .details("Failed to compile Sieve script")), - } + .reason(err) + .details("Failed to compile Sieve script") + })?; + + let new_script_object = SieveScript { + name: script_object.name.to_string(), + blob_hash: BlobHash::from(&script_object.blob_hash), + size: script_object.size.to_native(), + vacation_response: script_object.vacation_response.as_ref().map(|vr| { + VacationResponse { + from_date: vr.from_date.as_ref().map(|v| v.to_native()), + to_date: vr.to_date.as_ref().map(|v| v.to_native()), + subject: vr.subject.as_ref().map(|v| v.to_string()), + text_body: vr.text_body.as_ref().map(|v| v.to_string()), + html_body: vr.html_body.as_ref().map(|v| v.to_string()), + } + }), + script: script.to_bytes(), + metadata_flags: script_object.metadata_flags, + }; + let bytecode = new_script_object.script.clone(); + let bytes = Archiver::new(new_script_object) + .serialize() + .caused_by(trc::location!())?; + + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .with_document(document_id) + .assert_value(SieveField::Archive, archive) + .set(SieveField::Archive, bytes); + self.commit_batch(batch).await.caused_by(trc::location!())?; + + Ok(ActiveScript { + document_id, + version, + script: Bytecode::Compiled(bytecode), + }) + } +} + +async fn resolve_mailbox( + server: &Server, + account_id: u32, + handler: &mut SieveHandler<'_, '_>, + query: &MailboxQuery, +) -> trc::Result<()> { + let document_id = query + .resolve(server, account_id) + .await + .caused_by(trc::location!())?; + if document_id.is_none() && !handler.has_mailboxes { + if server + .mailbox_id_exists(account_id, INBOX_ID) + .await + .caused_by(trc::location!())? + { + handler.has_mailboxes = true; + } else { + handler.cache = Some( + server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?, + ); + return Ok(()); } } + handler.resolved.push((query.clone(), document_id)); + Ok(()) +} + +async fn verified_script<'x>( + server: &Server, + account_id: u32, + script: &'x ActiveScript, + recompiled: &'x mut Option, +) -> trc::Result> { + if let Ok(sieve) = Sieve::from_bytes(script.bytecode()?) { + return Ok(sieve); + } + + let script = match &script.script { + Bytecode::Stored(archive) => { + let script_object = archive + .unarchive::() + .caused_by(trc::location!())?; + &*recompiled.insert( + server + .sieve_script_recompile( + account_id, + script.document_id, + script.version, + archive, + script_object, + ) + .await?, + ) + } + Bytecode::Compiled(_) => script, + }; + + Sieve::from_bytes(script.bytecode()?).map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + .details("Failed to load compiled Sieve script") + }) } fn write_received_header(buf: &mut Vec, hostname: &str, id: u64) { @@ -786,9 +1120,3 @@ fn write_received_header(buf: &mut Vec, hostname: &str, id: u64) { buf.extend_from_slice(Date::now().to_rfc822().as_bytes()); buf.extend_from_slice(b"\r\n"); } - -pub struct CompiledScript { - pub script: Sieve, - pub name: String, - pub version: ArchiveVersion, -} diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs index 873614a6..ca983cda 100644 --- a/crates/email/src/sieve/mod.rs +++ b/crates/email/src/sieve/mod.rs @@ -4,22 +4,42 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::KV_SIEVE_ID; use sieve::Sieve; -use std::sync::Arc; -use store::{blake3, write::ArchiveVersion}; -use types::blob_hash::BlobHash; +use store::{ + SerializeInfallible, U32_LEN, ValueKey, blake3, + write::{Archive, ArchiveBytes, ArchiveVersion, BatchBuilder, IndexPropertyClass, ValueClass}, +}; +use trc::AddContext; +use types::{blob_hash::BlobHash, collection::Collection, field::SieveField}; +use utils::hash128::Hash128; pub mod delete; pub mod index; pub mod ingest; -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct ActiveScript { pub document_id: u32, pub version: ArchiveVersion, - pub script_name: String, - pub script: Arc, + pub script: Bytecode, +} + +#[derive(Debug)] +pub enum Bytecode { + Stored(Archive), + Compiled(Vec), +} + +impl ActiveScript { + pub fn bytecode(&self) -> trc::Result<&[u8]> { + match &self.script { + Bytecode::Stored(archive) => archive + .unarchive::() + .caused_by(trc::location!()) + .map(|script| script.script.as_ref()), + Bytecode::Compiled(bytes) => Ok(bytes), + } + } } #[derive( @@ -31,6 +51,8 @@ pub struct SieveScript { pub blob_hash: BlobHash, pub size: u32, pub vacation_response: Option, + pub script: Vec, + pub metadata_flags: u8, } #[derive( @@ -52,6 +74,8 @@ impl SieveScript { blob_hash, vacation_response: None, size: 0, + script: Vec::new(), + metadata_flags: 0, } } @@ -74,31 +98,83 @@ impl SieveScript { self.vacation_response = Some(vacation_response); self } + + pub fn with_script(mut self, script: &Sieve<'_>) -> Self { + self.set_script(script); + self + } + + pub fn set_script(&mut self, script: &Sieve<'_>) { + self.script = script.to_bytes(); + } } -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] #[repr(transparent)] -pub struct SeenIdHash(pub [u8; 32]); +pub struct SeenIdHash(pub Hash128); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SeenId { + pub hash: SeenIdHash, + pub expires_at: u64, +} + +const SEEN_ID_DOCUMENT_ID: u32 = 0; +const SEEN_ID_HASH_OFFSET: usize = U32_LEN + 3; impl SeenIdHash { - pub fn new(account_id: u32, hash: u32, id: &str) -> Self { + pub fn new(script_hash: u32, id: &str) -> Self { let mut hasher = blake3::Hasher::new(); - hasher.update(&account_id.to_be_bytes()); - hasher.update(&hash.to_be_bytes()); + hasher.update(&script_hash.to_be_bytes()); hasher.update(id.as_bytes()); - SeenIdHash(hasher.finalize().into()) + let mut hash = [0u8; Hash128::LEN]; + hasher.finalize_xof().fill(&mut hash); + SeenIdHash(Hash128::from_be_bytes(hash)) } - pub fn key(&self) -> Vec { - let mut result = Vec::with_capacity(self.0.len() + 1); - result.push(KV_SIEVE_ID); - result.extend_from_slice(&self.0); - result + pub fn class(&self) -> ValueClass { + ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: SieveField::SeenId.into(), + hash: self.0, + }) + } + + pub fn key(&self, account_id: u32) -> ValueKey { + ValueKey { + account_id, + collection: Collection::SieveScript.into(), + document_id: SEEN_ID_DOCUMENT_ID, + class: self.class(), + } + } + + pub fn account_range(account_id: u32) -> (ValueKey, ValueKey) { + ( + SeenIdHash(Hash128::default()).key(account_id), + SeenIdHash(Hash128::from(u128::MAX)).key(account_id), + ) + } + + pub fn from_key(key: &[u8]) -> Option { + key.get(SEEN_ID_HASH_OFFSET..SEEN_ID_HASH_OFFSET + Hash128::LEN) + .and_then(Hash128::from_be_slice) + .map(SeenIdHash) + } + + pub fn with_context(account_id: u32, batch: &mut BatchBuilder) -> &mut BatchBuilder { + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .with_document(SEEN_ID_DOCUMENT_ID) } } -impl AsRef<[u8]> for SeenIdHash { - fn as_ref(&self) -> &[u8] { - &self.0 +impl SeenId { + pub fn write_all(seen_ids: &[SeenId], account_id: u32, batch: &mut BatchBuilder) { + let batch = SeenIdHash::with_context(account_id, batch); + for seen_id in seen_ids { + batch.set(seen_id.hash.class(), seen_id.expires_at.serialize()); + } } } + diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index f80e2676..5e068736 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -5,10 +5,17 @@ */ use super::{ArchivedEmailSubmission, EmailSubmission}; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use common::storage::index::{ + IndexValue, IndexableAndSerializableObject, IndexableObject, SerializableObject, + serialize_object, +}; +use registry::schema::enums::StorageQuota; use store::{ U32_LEN, U64_LEN, - write::{IndexPropertyClass, ValueClass, key::KeySerializer}, + write::{ + ArchiveCompression, BatchBuilder, Compression, Dictionary, IndexPropertyClass, Slot, + ValueClass, key::KeySerializer, + }, }; use types::{collection::SyncCollection, field::EmailSubmissionField}; @@ -33,6 +40,7 @@ impl IndexableObject for EmailSubmission { sync_collection: SyncCollection::EmailSubmission, prefix: None, }, + IndexValue::ObjectCount(StorageQuota::MaxEmailSubmissions), ] .into_iter() } @@ -59,6 +67,7 @@ impl IndexableObject for &ArchivedEmailSubmission { sync_collection: SyncCollection::EmailSubmission, prefix: None, }, + IndexValue::ObjectCount(StorageQuota::MaxEmailSubmissions), ] .into_iter() } @@ -69,3 +78,13 @@ impl IndexableAndSerializableObject for EmailSubmission { false } } + +impl ArchiveCompression for EmailSubmission { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Common)); +} + +impl SerializableObject for EmailSubmission { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} diff --git a/crates/email/src/submission/mod.rs b/crates/email/src/submission/mod.rs index ed59eaed..0ef95df1 100644 --- a/crates/email/src/submission/mod.rs +++ b/crates/email/src/submission/mod.rs @@ -4,6 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use store::{ + ValueKey, + write::{ArchiveCompression, BatchBuilder, Compression, IndexPropertyClass, ValueClass}, +}; +use types::{collection::Collection, field::EmailSubmissionField}; use utils::map::vec_map::VecMap; pub mod index; @@ -47,6 +52,18 @@ pub struct DeliveryStatus { pub displayed: bool, } +pub const QUEUED_REPLY: &str = "250 2.1.5 Queued"; + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub struct DeliveryResults(pub Box<[DeliveryResult]>); + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub struct DeliveryResult { + pub address: Box, + pub delivered: Delivered, + pub smtp_reply: Box, +} + #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] @@ -70,11 +87,12 @@ pub enum UndoStatus { impl UndoStatus { pub fn parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), - "pending" => UndoStatus::Pending, - "final" => UndoStatus::Final, - "canceled" => UndoStatus::Canceled, - "cancelled" => UndoStatus::Canceled, + hashify::fnc_map!(s.as_bytes(), + "pending" => Some(UndoStatus::Pending), + "final" => Some(UndoStatus::Final), + "canceled" => Some(UndoStatus::Canceled), + "cancelled" => Some(UndoStatus::Canceled), + _ => None, ) } @@ -113,16 +131,84 @@ impl ArchivedUndoStatus { } } +impl DeliveryResults { + pub fn key(account_id: u32, queue_id: u64) -> ValueKey { + Self::key_at(account_id, queue_id, 0) + } + + pub fn write(batch: &mut BatchBuilder, account_id: u32, queue_id: u64, value: Vec) { + batch + .with_account_id(account_id) + .with_collection(Collection::EmailSubmission) + .with_document(0) + .set(Self::class(queue_id), value); + } + + pub fn clear(batch: &mut BatchBuilder, account_id: u32, queue_id: u64) { + batch + .with_account_id(account_id) + .with_collection(Collection::EmailSubmission) + .with_document(0) + .clear(Self::class(queue_id)); + } + + pub fn range( + account_id: u32, + max_queue_id: u64, + ) -> (ValueKey, ValueKey) { + ( + Self::key_at(account_id, 0, 0), + Self::key_at(account_id, max_queue_id, u32::MAX), + ) + } + + fn key_at(account_id: u32, queue_id: u64, document_id: u32) -> ValueKey { + ValueKey { + account_id, + collection: Collection::EmailSubmission.into(), + document_id, + class: Self::class(queue_id), + } + } + + fn class(queue_id: u64) -> ValueClass { + ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailSubmissionField::DeliveryResult.into(), + value: queue_id, + }) + } +} + +impl ArchiveCompression for DeliveryResults { + const COMPRESSION: Compression = Compression::None; +} + +impl From<&ArchivedDeliveryResult> for DeliveryStatus { + fn from(value: &ArchivedDeliveryResult) -> Self { + DeliveryStatus { + smtp_reply: value.smtp_reply.to_string(), + delivered: Delivered::from(&value.delivered), + displayed: false, + } + } +} + +impl From<&ArchivedDelivered> for Delivered { + fn from(value: &ArchivedDelivered) -> Self { + match value { + ArchivedDelivered::Queued => Delivered::Queued, + ArchivedDelivered::Yes => Delivered::Yes, + ArchivedDelivered::No => Delivered::No, + ArchivedDelivered::Unknown => Delivered::Unknown, + } + } +} + impl From<&ArchivedDeliveryStatus> for DeliveryStatus { fn from(value: &ArchivedDeliveryStatus) -> Self { DeliveryStatus { smtp_reply: value.smtp_reply.to_string(), - delivered: match value.delivered { - ArchivedDelivered::Queued => Delivered::Queued, - ArchivedDelivered::Yes => Delivered::Yes, - ArchivedDelivered::No => Delivered::No, - ArchivedDelivered::Unknown => Delivered::Unknown, - }, + delivered: Delivered::from(&value.delivered), displayed: value.displayed, } } diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml index 6b53b8ff..e5a7d898 100644 --- a/crates/groupware/Cargo.toml +++ b/crates/groupware/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "groupware" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -11,24 +11,26 @@ types = { path = "../types" } trc = { path = "../trc" } nlp = { path = "../nlp" } registry = { path = "../registry" } -calcard = { version = "0.3", features = ["rkyv"] } -hashify = "0.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } +calcard = { path = "../calcard", features = ["rkyv"] } +jmap-tools = { path = "../jmap-tools" } +hashify = "0.3" +text-extract = { path = "../text-extract" } +tokio = { version = "1.54", features = ["net", "macros"] } +rkyv.workspace = true percent-encoding = "2.3.2" compact_str = "0.10.0" ahash = { version = "0.8" } -indexmap = "2.14.1" -chrono = "0.4.45" +indexmap = "2.14.2" +jiff = "0.2.38" icu_locale_core = "2" icu_provider = { version = "2", features = ["sync"] } icu_plurals = "2" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" -icu_datetime = { version = "~2.3", features = ["unstable_chrono_0_4"] } +icu_datetime = { version = "~2.4", features = ["unstable_jiff_0_2"] } [features] -test_mode = [] -enterprise = [] +enterprise = ["utils/enterprise", "store/enterprise", "common/enterprise", "trc/enterprise", "registry/enterprise"] [dev-dependencies] diff --git a/crates/groupware/src/cache/calcard.rs b/crates/groupware/src/cache/calcard.rs index f9dbe14c..0dd3739a 100644 --- a/crates/groupware/src/cache/calcard.rs +++ b/crates/groupware/src/cache/calcard.rs @@ -4,27 +4,29 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::GroupwareCache; +use super::{ChunkAccumulator, GroupwareCache}; use crate::{ - DavResourceName, RFC_3986, + DavResourceName, calendar::{ - ArchivedCalendar, ArchivedCalendarEvent, Calendar, CalendarEvent, SCHEDULE_INBOX_ID, - SCHEDULE_OUTBOX_ID, storage::ItipAutoExpunge, + ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarEventNotification, Calendar, + CalendarEvent, CalendarEventNotification, DIRECT_NOTIFICATION_PARENT_ID, + EVENT_NOTIFICATION_IS_DIRECT, SCHEDULE_INBOX_ID, SCHEDULE_OUTBOX_ID, }, contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}, - encode_path_segment, }; use calcard::common::timezone::Tz; use common::{ - DavName, DavPath, DavResource, DavResourceMetadata, DavResources, Server, - TinyCalendarPreferences, UpdateLock, + ArenaRef, DavName, DavPath, GroupwareResource, GroupwareResourceMetadata, GroupwareResources, + NO_ID, PathIndex, ResourceStore, Server, TinyCalendarPreferences, UpdateLock, + storage::dav::{CONTAINER_FLAG, CachedUid, ResourceChunkBuilder, canonical_calcard_segment}, }; use std::sync::Arc; -use store::ahash::{AHashMap, AHashSet}; +use store::ahash::AHashMap; use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, + field::Field, }; use utils::map::bitmap::Bitmap; @@ -36,7 +38,7 @@ pub(super) async fn build_calcard_resources( container_collection: Collection, item_collection: Collection, update_lock: Arc, -) -> trc::Result { +) -> trc::Result { let is_calendar = matches!(sync_collection, SyncCollection::Calendar); let owner_account_info = server.account(account_id).await?; let access_account_info = if account_id == access_account_id { @@ -44,25 +46,8 @@ pub(super) async fn build_calcard_resources( } else { server.account(access_account_id).await? }; - let mut cache = DavResources { - base_path: format!( - "{}/{}/", - if is_calendar { - DavResourceName::Cal - } else { - DavResourceName::Card - } - .base_path(), - percent_encoding::utf8_percent_encode(owner_account_info.name(), RFC_3986), - ), - paths: AHashSet::with_capacity(16), - resources: Vec::with_capacity(16), - item_change_id: 0, - container_change_id: 0, - highest_change_id: 0, - size: std::mem::size_of::() as u64, - update_lock, - }; + let base_path = + DavResourceName::from(sync_collection).account_base_path(owner_account_info.name()); let mut is_first_check = true; loop { @@ -74,109 +59,162 @@ pub(super) async fn build_calcard_resources( .await .caused_by(trc::location!())? .unwrap_or_default(); - cache.item_change_id = last_change_id; - cache.container_change_id = last_change_id; - cache.highest_change_id = last_change_id; - cache.update_lock.set_revision(last_change_id); + update_lock.set_revision(last_change_id); + let mut containers = ChunkAccumulator::default(); server .archives( account_id, container_collection, + Field::ARCHIVE, &(), |document_id, archive| { - let resource = if is_calendar { - resource_from_calendar(archive.unarchive::()?, document_id) + let etag = archive.version.hash().unwrap_or_default(); + let builder = containers.current(); + if is_calendar { + push_calendar(builder, archive.unarchive::()?, document_id, etag); } else { - resource_from_addressbook(archive.unarchive::()?, document_id) - }; - let path = DavPath { - path: encode_path_segment(resource.container_name().unwrap()).into_owned(), - parent_id: None, - hierarchy_seq: 1, - resource_idx: cache.resources.len(), - }; - - cache.size += (std::mem::size_of::() - + std::mem::size_of::() - + (path.path.len()) * 2) as u64; - cache.paths.insert(path); - cache.resources.push(resource); - + push_addressbook( + builder, + archive.unarchive::()?, + document_id, + etag, + ); + } Ok(true) }, ) .await .caused_by(trc::location!())?; - if cache.paths.is_empty() { + if containers.is_empty() { if is_first_check { - if is_calendar { + let created = if is_calendar { server .create_default_calendar(&access_account_info, &owner_account_info) - .await?; + .await } else { server .create_default_addressbook(&access_account_info, &owner_account_info) - .await?; + .await + }; + if let Err(err) = created + && !err.is_assertion_failure() + { + return Err(err); } is_first_check = false; continue; } else { + let mut cache = GroupwareResources { + base_path, + paths: Default::default(), + resources: ResourceStore::from_sorted(containers.finish(), Vec::new(), false), + item_change_id: last_change_id, + container_change_id: last_change_id, + highest_change_id: last_change_id, + size: 0, + update_lock, + verification: Default::default(), + }; + cache.recompute_size(); return Ok(cache); } } - let parent_range = cache.resources.len(); + let mut items = ChunkAccumulator::default(); server - .archives(account_id, item_collection, &(), |document_id, archive| { - let resource = if is_calendar { - resource_from_event(archive.unarchive::()?, document_id) - } else { - resource_from_card(archive.unarchive::()?, document_id) - }; - let resource_idx = cache.resources.len(); - - for name in resource.child_names().unwrap_or_default().iter() { - if let Some(parent) = - cache.resources.get(..parent_range).and_then(|resources| { - resources.iter().find(|r| r.document_id == name.parent_id) - }) - { - let path = DavPath { - path: format!( - "{}/{}", - encode_path_segment(parent.container_name().unwrap()), - encode_path_segment(&name.name) - ), - parent_id: Some(name.parent_id), - hierarchy_seq: 0, - resource_idx, - }; - - cache.size += (std::mem::size_of::() - + name.name.len() - + path.path.len()) as u64; - cache.paths.insert(path); + .archives( + account_id, + item_collection, + Field::ARCHIVE, + &(), + |document_id, archive| { + let builder = items.current(); + if is_calendar { + push_event(builder, archive.unarchive::()?, document_id); + } else { + push_card(builder, archive.unarchive::()?, document_id); } - } - cache.size += std::mem::size_of::() as u64; - cache.resources.push(resource); - - Ok(true) - }) + Ok(true) + }, + ) .await .caused_by(trc::location!())?; + let resources = ResourceStore::from_sorted(containers.finish(), items.finish(), false); + let paths = build_calcard_paths(&resources); + let mut cache = GroupwareResources { + base_path, + paths: Arc::new(paths), + resources, + item_change_id: last_change_id, + container_change_id: last_change_id, + highest_change_id: last_change_id, + size: 0, + update_lock, + verification: Default::default(), + }; + cache.recompute_size(); return Ok(cache); } } +pub(super) fn build_calcard_paths(resources: &ResourceStore) -> PathIndex { + let mut names: AHashMap = AHashMap::with_capacity(16); + for resource in resources.iter() { + if resource.is_container() + && let Some(name) = resource.container_name() + { + names.insert( + resource.document_id(), + canonical_calcard_segment(name).into_owned(), + ); + } + } + + let mut entries: Vec<(String, DavPath)> = Vec::with_capacity(resources.len()); + for resource in resources.iter() { + if resource.is_container() { + if let Some(name) = names.get(&resource.document_id()) { + entries.push(( + name.clone(), + DavPath { + path: ArenaRef::default(), + parent_id: NO_ID, + hierarchy_seq: 1 | CONTAINER_FLAG, + document_id: resource.document_id(), + }, + )); + } + } else { + for name in resource.child_names() { + if let Some(parent) = names.get(&name.parent_id) { + entries.push(( + format!( + "{parent}/{}", + canonical_calcard_segment(resource.child_name_at(name)) + ), + DavPath { + path: ArenaRef::default(), + parent_id: name.parent_id, + hierarchy_seq: 0, + document_id: resource.document_id(), + }, + )); + } + } + } + } + + PathIndex::pack(entries) +} + pub(super) async fn build_scheduling_resources( server: &Server, account_id: u32, update_lock: Arc, -) -> trc::Result { +) -> trc::Result { let last_change_id = server .core .storage @@ -187,218 +225,270 @@ pub(super) async fn build_scheduling_resources( .unwrap_or_default(); let account_info = server.account(account_id).await?; - let item_ids = server - .itip_ids(account_id) + update_lock.set_revision(last_change_id); + + let mut containers = ChunkAccumulator::default(); + for document_id in [SCHEDULE_OUTBOX_ID, SCHEDULE_INBOX_ID] { + push_scheduling_container(containers.current(), document_id); + } + + let mut items = ChunkAccumulator::default(); + server + .archives( + account_id, + Collection::CalendarEventNotification, + Field::ARCHIVE, + &(), + |document_id, archive| { + push_scheduling( + items.current(), + archive.unarchive::()?, + document_id, + ); + Ok(true) + }, + ) .await .caused_by(trc::location!())?; - update_lock.set_revision(last_change_id); - let mut cache = DavResources { - base_path: format!( - "{}/{}/", - DavResourceName::Scheduling.base_path(), - percent_encoding::utf8_percent_encode(account_info.name(), RFC_3986), - ), - paths: AHashSet::with_capacity((2 + item_ids.len()) as usize), - resources: Vec::with_capacity((2 + item_ids.len()) as usize), + let resources = ResourceStore::from_sorted(containers.finish(), items.finish(), false); + let paths = build_scheduling_paths(&resources); + let mut cache = GroupwareResources { + base_path: DavResourceName::Scheduling.account_base_path(account_info.name()), + paths: Arc::new(paths), + resources, item_change_id: last_change_id, container_change_id: last_change_id, highest_change_id: last_change_id, - size: std::mem::size_of::() as u64, + size: 0, update_lock, + verification: Default::default(), }; - - for (document_id, is_container) in item_ids - .into_iter() - .map(|document_id| (document_id, false)) - .chain([(SCHEDULE_INBOX_ID, true), (SCHEDULE_OUTBOX_ID, true)]) - { - let path = path_from_scheduling(document_id, cache.resources.len(), is_container); - cache.size += (std::mem::size_of::() + (path.path.len() * 2)) as u64 - + std::mem::size_of::() as u64; - cache.paths.insert(path); - cache - .resources - .push(resource_from_scheduling(document_id, is_container)); - } - + cache.recompute_size(); Ok(cache) } -pub(super) fn build_simple_hierarchy(cache: &mut DavResources) { - cache.paths = AHashSet::with_capacity(cache.resources.len()); - let name_idx = cache - .resources - .iter() - .filter_map(|resource| { - resource - .container_name() - .map(|name| (resource.document_id, name)) - }) - .collect::>(); - - for (resource_idx, resource) in cache.resources.iter().enumerate() { - match &resource.data { - DavResourceMetadata::Calendar { name, .. } - | DavResourceMetadata::AddressBook { name, .. } => { - let path = DavPath { - path: encode_path_segment(name).into_owned(), - parent_id: None, - hierarchy_seq: 1, - resource_idx, - }; - cache.size += - (std::mem::size_of::() + name.len() + path.path.len()) as u64; - cache.paths.insert(path); - } - DavResourceMetadata::CalendarEvent { names, .. } - | DavResourceMetadata::ContactCard { names } => { - for name in names { - if let Some(parent_name) = name_idx.get(&name.parent_id) { - let path = DavPath { - path: format!( - "{}/{}", - encode_path_segment(parent_name), - encode_path_segment(&name.name) - ), - parent_id: Some(name.parent_id), - hierarchy_seq: 0, - resource_idx, - }; - cache.size += (std::mem::size_of::() - + name.name.len() - + path.path.len()) as u64; - cache.paths.insert(path); - } - } - } - _ => unreachable!(), - } - cache.size += std::mem::size_of::() as u64; - } -} - -pub(super) fn resource_from_calendar(calendar: &ArchivedCalendar, document_id: u32) -> DavResource { - DavResource { - document_id, - data: DavResourceMetadata::Calendar { - name: calendar.name.to_string(), - acls: calendar - .acls - .iter() - .map(|acl| AclGrant { - account_id: acl.account_id.to_native(), - grants: Bitmap::from(&acl.grants), - }) - .collect(), - preferences: calendar - .preferences - .iter() - .map(|pref| TinyCalendarPreferences { - account_id: pref.account_id.to_native(), - flags: pref.flags.to_native(), - tz: pref.time_zone.tz().unwrap_or(Tz::UTC), - }) - .collect(), - }, - } -} - -pub(super) fn resource_from_event(event: &ArchivedCalendarEvent, document_id: u32) -> DavResource { - let (start, duration) = event.data.event_range().unwrap_or_default(); - DavResource { - document_id, - data: DavResourceMetadata::CalendarEvent { - names: event - .names - .iter() - .map(|name| DavName { - name: name.name.to_string(), - parent_id: name.parent_id.to_native(), - }) - .collect(), - start, - duration, - }, - } -} - -pub(super) fn resource_from_scheduling(document_id: u32, is_container: bool) -> DavResource { - DavResource { - document_id, - data: DavResourceMetadata::CalendarEventNotification { - names: if !is_container { - [DavName { - name: format!("{document_id}.ics"), +pub(super) fn build_scheduling_paths(resources: &ResourceStore) -> PathIndex { + let mut entries: Vec<(String, DavPath)> = Vec::with_capacity(resources.len()); + for resource in resources.iter() { + if resource.is_container() { + entries.push(( + if resource.document_id() == SCHEDULE_INBOX_ID { + "inbox".to_string() + } else { + "outbox".to_string() + }, + DavPath { + path: ArenaRef::default(), + parent_id: NO_ID, + hierarchy_seq: 1 | CONTAINER_FLAG, + document_id: resource.document_id(), + }, + )); + } else if resource + .child_names() + .iter() + .all(|name| name.parent_id == SCHEDULE_INBOX_ID) + { + entries.push(( + format!("inbox/{}.ics", resource.document_id()), + DavPath { + path: ArenaRef::default(), parent_id: SCHEDULE_INBOX_ID, - }] - .into_iter() - .collect() - } else { - Default::default() - }, - }, + hierarchy_seq: 0, + document_id: resource.document_id(), + }, + )); + } } + PathIndex::pack(entries) } -pub(super) fn path_from_scheduling( +pub(super) fn push_calendar( + builder: &mut ResourceChunkBuilder, + calendar: &ArchivedCalendar, document_id: u32, - resource_idx: usize, - is_container: bool, -) -> DavPath { - if is_container { - DavPath { - path: if document_id == SCHEDULE_INBOX_ID { - "inbox".to_string() - } else { - "outbox".to_string() - }, - parent_id: None, - hierarchy_seq: 1, - resource_idx, - } - } else { - DavPath { - path: format!("inbox/{document_id}.ics"), - parent_id: Some(SCHEDULE_INBOX_ID), - hierarchy_seq: 0, - resource_idx, - } - } + etag: u32, +) { + let name = builder.push_str(calendar.name.as_str()); + let acls = builder.push_acls( + &calendar + .acls + .iter() + .map(|acl| AclGrant { + account_id: acl.account_id.to_native(), + grants: Bitmap::from(&acl.grants), + }) + .collect::>(), + ); + let preferences = builder.push_prefs( + &calendar + .preferences + .iter() + .map(|pref| TinyCalendarPreferences { + account_id: pref.account_id.to_native(), + flags: pref.flags.to_native(), + tz: pref.time_zone.tz().unwrap_or(Tz::UTC), + }) + .collect::>(), + ); + builder.records.push(GroupwareResource { + document_id, + data: GroupwareResourceMetadata::Calendar { + name, + acls, + preferences, + etag, + metadata: calendar.metadata_kinds(), + }, + }); } -pub(super) fn resource_from_addressbook( +pub(super) fn push_addressbook( + builder: &mut ResourceChunkBuilder, book: &ArchivedAddressBook, document_id: u32, -) -> DavResource { - DavResource { + etag: u32, +) { + let name = builder.push_str(book.name.as_str()); + let acls = builder.push_acls( + &book + .acls + .iter() + .map(|acl| AclGrant { + account_id: acl.account_id.to_native(), + grants: Bitmap::from(&acl.grants), + }) + .collect::>(), + ); + builder.records.push(GroupwareResource { document_id, - data: DavResourceMetadata::AddressBook { - name: book.name.to_string(), - acls: book - .acls - .iter() - .map(|acl| AclGrant { - account_id: acl.account_id.to_native(), - grants: Bitmap::from(&acl.grants), - }) - .collect(), + data: GroupwareResourceMetadata::AddressBook { + name, + acls, + etag, + metadata: book.metadata_kinds(), }, - } + }); } -pub(super) fn resource_from_card(card: &ArchivedContactCard, document_id: u32) -> DavResource { - DavResource { +pub(super) fn push_event( + builder: &mut ResourceChunkBuilder, + event: &ArchivedCalendarEvent, + document_id: u32, +) { + let created_at = event.created.to_native(); + let names = builder.push_names( + &event + .names + .iter() + .map(|name| DavName { + name: name.name.to_string(), + parent_id: name.parent_id.to_native(), + }) + .collect::>(), + ); + let uid = builder.push_uid(&event.uid.as_str().cached_uid(), names); + builder.records.push(GroupwareResource { document_id, - data: DavResourceMetadata::ContactCard { - names: card - .names - .iter() - .map(|name| DavName { - name: name.name.to_string(), - parent_id: name.parent_id.to_native(), - }) - .collect(), + data: GroupwareResourceMetadata::CalendarEvent { + names, + start: event.start.to_native(), + duration: event.duration.to_native(), + created_at, + modified_at: event + .modified + .to_native() + .saturating_sub(created_at) + .clamp(i32::MIN as i64, i32::MAX as i64) as i32, + uid, + etag: event.etag.to_native(), + flags: event.flags.to_native(), }, - } + }); } + +pub(super) fn push_card( + builder: &mut ResourceChunkBuilder, + card: &ArchivedContactCard, + document_id: u32, +) { + let created_at = card.created.to_native(); + let names = builder.push_names( + &card + .names + .iter() + .map(|name| DavName { + name: name.name.to_string(), + parent_id: name.parent_id.to_native(), + }) + .collect::>(), + ); + let uid = builder.push_uid(&card.uid.as_str().cached_uid(), names); + builder.records.push(GroupwareResource { + document_id, + data: GroupwareResourceMetadata::ContactCard { + names, + created_at, + modified_at: card + .modified + .to_native() + .saturating_sub(created_at) + .clamp(i32::MIN as i64, i32::MAX as i64) as i32, + uid, + etag: card.etag.to_native(), + flags: card.flags.to_native(), + }, + }); +} + +pub(super) fn push_scheduling( + builder: &mut ResourceChunkBuilder, + event: &ArchivedCalendarEventNotification, + document_id: u32, +) { + let names = builder.push_names(&[DavName { + name: format!("{document_id}.ics"), + parent_id: if event.flags.to_native() & EVENT_NOTIFICATION_IS_DIRECT == 0 { + SCHEDULE_INBOX_ID + } else { + DIRECT_NOTIFICATION_PARENT_ID + }, + }]); + let calendar_ids_len = event.calendar_ids.len() as u16; + let principals = builder.push_principals(event.calendar_ids().chain(event.dismissed_ids())); + builder.records.push(GroupwareResource { + document_id, + data: GroupwareResourceMetadata::CalendarEventNotification { + names, + created_at: event.created.to_native(), + event_id: event + .event_id + .as_ref() + .map(|v| v.to_native()) + .unwrap_or(u32::MAX), + etag: event.etag.to_native(), + changed_by: event.changed_by_id(), + principals, + calendar_ids_len, + flags: event.flags.to_native(), + }, + }); +} + +pub(super) fn push_scheduling_container(builder: &mut ResourceChunkBuilder, document_id: u32) { + builder.records.push(GroupwareResource { + document_id, + data: GroupwareResourceMetadata::CalendarEventNotification { + names: ArenaRef::default(), + created_at: 0, + event_id: u32::MAX, + etag: 0, + changed_by: NO_ID, + principals: ArenaRef::default(), + calendar_ids_len: 0, + flags: 0, + }, + }); +} + diff --git a/crates/groupware/src/cache/file.rs b/crates/groupware/src/cache/file.rs index 09885bd8..bbccdd10 100644 --- a/crates/groupware/src/cache/file.rs +++ b/crates/groupware/src/cache/file.rs @@ -4,17 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::ChunkAccumulator; use crate::{ - DavResourceName, RFC_3986, encode_path_segment, - file::{ArchivedFileNode, FileNode}, + DavResourceName, + file::{ArchivedFileNode, FileNode, FileNodeRole}, +}; +use common::{ + ArenaRef, DavPath, FileFlags, GroupwareResource, GroupwareResourceMetadata, GroupwareResources, + NO_ID, PathIndex, ResourceStore, Server, UpdateLock, + storage::dav::{CONTAINER_FLAG, ResourceChunkBuilder, canonical_path_segment}, }; -use common::{DavPath, DavResource, DavResourceMetadata, DavResources, Server, UpdateLock}; use std::sync::Arc; -use store::ahash::{AHashMap, AHashSet}; +use store::ahash::AHashMap; use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, + field::Field, + media_type::media_type_essence, }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; @@ -22,7 +29,7 @@ pub(super) async fn build_file_resources( server: &Server, account_id: u32, update_lock: Arc, -) -> trc::Result { +) -> trc::Result { let last_change_id = server .core .storage @@ -33,18 +40,19 @@ pub(super) async fn build_file_resources( .unwrap_or_default(); let account_info = server.account(account_id).await?; - let mut resources = Vec::with_capacity(16); + let mut nodes = ChunkAccumulator::default(); server .archives( account_id, Collection::FileNode, + Field::ARCHIVE, &(), |document_id, archive| { - resources.push(resource_from_file( + push_file( + nodes.current(), archive.unarchive::()?, document_id, - )); - + ); Ok(true) }, ) @@ -52,44 +60,44 @@ pub(super) async fn build_file_resources( .caused_by(trc::location!())?; update_lock.set_revision(last_change_id); - let mut files = DavResources { - base_path: format!( - "{}/{}/", - DavResourceName::File.base_path(), - percent_encoding::utf8_percent_encode(account_info.name(), RFC_3986), - ), - size: std::mem::size_of::() as u64, - paths: AHashSet::with_capacity(resources.len()), + let resources = ResourceStore::from_sorted(nodes.finish(), Vec::new(), true); + let paths = build_nested_hierarchy(&resources); + let mut files = GroupwareResources { + base_path: DavResourceName::File.account_base_path(account_info.name()), + size: 0, + paths: Arc::new(paths), resources, item_change_id: last_change_id, container_change_id: last_change_id, highest_change_id: last_change_id, update_lock, + verification: Default::default(), }; - - build_nested_hierarchy(&mut files); + files.recompute_size(); Ok(files) } -pub(super) fn build_nested_hierarchy(resources: &mut DavResources) { - let mut topological_sort = TopologicalSort::with_capacity(resources.resources.len()); - let mut names = AHashMap::with_capacity(resources.resources.len()); +pub(super) fn build_nested_hierarchy(resources: &ResourceStore) -> PathIndex { + let mut topological_sort = TopologicalSort::with_capacity(resources.len()); + let mut names: AHashMap = + AHashMap::with_capacity(resources.len()); - for (resource_idx, resource) in resources.resources.iter().enumerate() { - if let DavResourceMetadata::File { parent_id, .. } = resource.data { + for resource in resources.iter() { + if let GroupwareResourceMetadata::File { parent_id, .. } = resource.resource.data { topological_sort.insert( - parent_id.map(|id| id + 1).unwrap_or_default(), - resource.document_id + 1, + if parent_id != NO_ID { parent_id + 1 } else { 0 }, + resource.document_id() + 1, ); names.insert( - resource.document_id, - DavPath { - path: encode_path_segment(resource.container_name().unwrap()).into_owned(), + resource.document_id(), + ( + canonical_path_segment(resource.container_name().unwrap_or_default()) + .into_owned(), parent_id, - hierarchy_seq: 0, - resource_idx, - }, + 0, + resource.is_container(), + ), ); } } @@ -99,183 +107,91 @@ pub(super) fn build_nested_hierarchy(resources: &mut DavResources) { let folder_id = folder_id - 1; let path = names .get(&folder_id) - .and_then(|folder| folder.parent_id.map(|parent_id| (&folder.path, parent_id))) - .and_then(|(name, parent_id)| { + .filter(|(_, parent_id, _, _)| *parent_id != NO_ID) + .and_then(|(name, parent_id, _, _)| { names - .get(&parent_id) - .map(|parent| format!("{}/{}", parent.path, name)) + .get(parent_id) + .map(|(parent, _, _, _)| format!("{parent}/{name}")) }); if let Some(folder) = names.get_mut(&folder_id) { if let Some(path) = path { - folder.path = path; + folder.0 = path; } - folder.hierarchy_seq = hierarchy_sequence as u32; + folder.2 = hierarchy_sequence as u32; } } } - resources.paths = names - .into_values() - .inspect(|v| { - resources.size += (std::mem::size_of::() - + std::mem::size_of::() - + std::mem::size_of::() - + std::mem::size_of::() - + v.path.len()) as u64; - }) - .collect(); + let entries = names + .into_iter() + .map( + |(document_id, (path, parent_id, hierarchy_seq, is_container))| { + ( + path, + DavPath { + path: ArenaRef::default(), + parent_id, + hierarchy_seq: hierarchy_seq + | if is_container { CONTAINER_FLAG } else { 0 }, + document_id, + }, + ) + }, + ) + .collect::>(); + + PathIndex::pack(entries) } -pub(super) fn resource_from_file(node: &ArchivedFileNode, document_id: u32) -> DavResource { +pub fn push_file(builder: &mut ResourceChunkBuilder, node: &ArchivedFileNode, document_id: u32) { let parent_id = node.parent_id.to_native(); - DavResource { - document_id, - data: DavResourceMetadata::File { - name: node.name.as_str().to_string(), - size: node.file.as_ref().map(|f| f.size.to_native()), - parent_id: if parent_id > 0 { - Some(parent_id - 1) - } else { - None - }, - acls: node - .acls - .iter() - .map(|acl| AclGrant { - account_id: acl.account_id.to_native(), - grants: Bitmap::from(&acl.grants), - }) - .collect(), - }, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const MISSING_DOCUMENT_ID: u32 = 9; - - fn folder(document_id: u32, name: &str, parent_id: Option) -> DavResource { - DavResource { - document_id, - data: DavResourceMetadata::File { - name: name.to_string(), - size: None, - parent_id, - acls: Default::default(), - }, - } - } - - fn file(document_id: u32, name: &str, parent_id: Option) -> DavResource { - DavResource { - document_id, - data: DavResourceMetadata::File { - name: name.to_string(), - size: Some(1024), - parent_id, - acls: Default::default(), - }, - } - } - - fn build(resources: Vec) -> DavResources { - let mut files = DavResources { - base_path: "/dav/file/john/".to_string(), - paths: AHashSet::with_capacity(resources.len()), - resources, - item_change_id: 0, - container_change_id: 0, - highest_change_id: 0, - size: 0, - update_lock: Arc::new(UpdateLock::new()), - }; - build_nested_hierarchy(&mut files); - files - } - - fn sorted_paths(files: &DavResources) -> Vec<&str> { - let mut paths = files - .paths + let file = node.file(); + let stored_media_type = file.and_then(|file| file.media_type.as_deref()); + let media_type = stored_media_type.and_then(media_type_essence); + let (name, media_type_id, extra_len) = + builder.push_file_name(node.name.as_str(), media_type.as_deref()); + let is_media_type_exact = stored_media_type + .is_none_or(|stored| !media_type_id.is_none() && media_type.as_deref() == Some(stored)); + let acls = builder.push_acls( + &node + .acls .iter() - .map(|path| path.path.as_str()) - .collect::>(); - paths.sort_unstable(); - paths - } - - fn hierarchy_seq(files: &DavResources, path: &str) -> u32 { - files.paths.get(path).expect(path).hierarchy_seq - } - - #[test] - fn nested_hierarchy() { - let files = build(vec![ - folder(0, "docs", None), - folder(1, "reports", Some(0)), - file(2, "q1.txt", Some(1)), - file(3, "readme.txt", None), - ]); - - assert_eq!( - sorted_paths(&files), - ["docs", "docs/reports", "docs/reports/q1.txt", "readme.txt"] - ); - assert!(hierarchy_seq(&files, "docs") < hierarchy_seq(&files, "docs/reports")); - assert!( - hierarchy_seq(&files, "docs/reports") < hierarchy_seq(&files, "docs/reports/q1.txt") - ); - } - - #[test] - fn nested_hierarchy_percent_encodes_paths() { - let files = build(vec![ - folder(0, "My Documents", None), - folder(1, "Berichte 2026", Some(0)), - file(2, "Ünterlagen Q1.txt", Some(1)), - folder(3, "My%20Folder", None), - file(4, "file(1)+a:b.txt", Some(3)), - ]); - - assert_eq!( - sorted_paths(&files), - [ - "My%20Documents", - "My%20Documents/Berichte%202026", - "My%20Documents/Berichte%202026/%C3%9Cnterlagen%20Q1.txt", - "My%20Folder", - "My%20Folder/file(1)+a:b.txt", - ] - ); - assert_eq!( - files.format_resource(files.by_path("My%20Documents").unwrap()), - "/dav/file/john/My%20Documents/" - ); - assert_eq!( - files.format_resource( - files - .by_path("My%20Documents/Berichte%202026/%C3%9Cnterlagen%20Q1.txt") - .unwrap() - ), - "/dav/file/john/My%20Documents/Berichte%202026/%C3%9Cnterlagen%20Q1.txt" - ); - } - - #[test] - fn nested_hierarchy_with_dangling_parent() { - let files = build(vec![ - folder(0, "docs", None), - folder(1, "reports", Some(MISSING_DOCUMENT_ID)), - file(2, "q1.txt", Some(1)), - file(3, "orphan.txt", Some(MISSING_DOCUMENT_ID)), - ]); - - assert_eq!( - sorted_paths(&files), - ["docs", "orphan.txt", "reports", "reports/q1.txt"] - ); - assert!(hierarchy_seq(&files, "reports") < hierarchy_seq(&files, "reports/q1.txt")); - } + .map(|acl| AclGrant { + account_id: acl.account_id.to_native(), + grants: Bitmap::from(&acl.grants), + }) + .collect::>(), + ); + let modified = node.modified.to_native(); + let flags = FileFlags::new( + node.kind_id(), + file.is_some_and(|f| f.executable), + node.role().map_or(0, FileNodeRole::id), + media_type_id, + extra_len, + ) + .with_presence(node.presence()); + builder.records.push(GroupwareResource { + document_id, + data: GroupwareResourceMetadata::File { + name, + size: file.map_or(NO_ID, |f| f.size.to_native()), + parent_id: if parent_id > 0 { parent_id - 1 } else { NO_ID }, + acls, + etag: node.etag.to_native(), + modified, + created_delta: node + .created + .to_native() + .saturating_sub(modified) + .clamp(i32::MIN as i64, i32::MAX as i64) as i32, + flags: if is_media_type_exact { + flags.with_exact_media_type() + } else { + flags + }, + }, + }); } + diff --git a/crates/groupware/src/cache/mod.rs b/crates/groupware/src/cache/mod.rs index 1d746fd4..4e201fb9 100644 --- a/crates/groupware/src/cache/mod.rs +++ b/crates/groupware/src/cache/mod.rs @@ -5,44 +5,113 @@ */ use crate::{ - cache::calcard::{build_scheduling_resources, path_from_scheduling, resource_from_scheduling}, - calendar::{CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarPreferences}, + DavResourceName, RFC_3986, + cache::calcard::{build_scheduling_paths, build_scheduling_resources, push_scheduling}, + calendar::{ + CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarEventNotification, + CalendarPreferences, + }, contact::{AddressBook, AddressBookPreferences, ContactCard}, file::FileNode, }; -use ahash::AHashSet; use calcard::{ - build_calcard_resources, build_simple_hierarchy, resource_from_addressbook, - resource_from_calendar, resource_from_card, resource_from_event, + build_calcard_paths, build_calcard_resources, push_addressbook, push_calendar, push_card, + push_event, }; use common::{ - DavResource, DavResources, Server, UpdateLock, auth::AccountCache, cache::LockResult, + DAV_CHUNK, GroupwareResources, ResourceChunk, Server, UpdateLock, Verification, + auth::AccountCache, + cache::{ + LockResult, + swap::{ + BLOCKING_CODEC_THRESHOLD, SwapKey, SwapPart, + writer::{SwapChanges, SwapTarget}, + }, + }, + storage::{ + batch::MAX_BATCH_DOCUMENTS, + dav::{ResourceChunkBuilder, hierarchy::PathUpdate}, + }, }; -use file::{build_file_resources, build_nested_hierarchy, resource_from_file}; -use std::{sync::Arc, time::Instant}; +use file::{build_file_resources, build_nested_hierarchy, push_file}; +use registry::schema::enums::StorageQuota; +use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; use store::{ - SerializeInfallible, ValueKey, + IterateParams, U32_LEN, ValueKey, ahash::AHashMap, query::log::{Change, Query}, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, + write::{ + Archive, ArchiveBytes, BatchBuilder, PendingId, ValueClass, assert::AssertValue, + key::DeserializeBigEndian, + }, }; -use trc::{AddContext, StoreEvent}; +use trc::{AddContext, CacheEvent}; use types::{ collection::{Collection, SyncCollection}, field::PrincipalField, }; -use utils::cache::Cache; pub mod calcard; pub mod file; +impl DavResourceName { + pub fn account_base_path(&self, account_name: &str) -> String { + format!( + "{}/{}/", + self.base_path(), + percent_encoding::utf8_percent_encode(account_name, RFC_3986) + ) + } +} + +#[derive(Default)] +pub struct ChunkAccumulator { + builders: Vec, +} + +impl ChunkAccumulator { + pub fn current(&mut self) -> &mut ResourceChunkBuilder { + if self + .builders + .last() + .is_none_or(|builder| builder.len() >= DAV_CHUNK) + { + self.builders.push(ResourceChunkBuilder::with_capacity(64)); + } + self.builders.last_mut().unwrap() + } + + pub fn is_empty(&self) -> bool { + self.builders.iter().all(|builder| builder.is_empty()) + } + + pub fn len(&self) -> usize { + self.builders.iter().map(|builder| builder.len()).sum() + } + + pub fn finish(self) -> Vec { + self.builders + .into_iter() + .filter(|builder| !builder.is_empty()) + .collect() + } +} + pub trait GroupwareCache: Sync + Send { - fn fetch_dav_resources( + fn fetch_groupware_resources( &self, access_account_id: u32, account_id: u32, collection: SyncCollection, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; + + fn fetch_groupware_resources_at( + &self, + access_account_id: u32, + account_id: u32, + collection: SyncCollection, + min_change_id: u64, + ) -> impl Future>> + Send; fn create_default_addressbook( &self, @@ -58,64 +127,67 @@ pub trait GroupwareCache: Sync + Send { fn get_or_create_default_calendar( &self, - access_account_id: u32, - account_id: u32, + account_info_access: &AccountCache, + account_info_owner: &AccountCache, ) -> impl Future>> + Send; - fn cached_dav_resources( + fn cached_groupware_resources( &self, account_id: u32, collection: SyncCollection, - ) -> Option>; + ) -> Option>; + + fn count_documents( + &self, + account_id: u32, + collection: Collection, + limit: usize, + ) -> impl Future> + Send; } impl GroupwareCache for Server { - async fn fetch_dav_resources( + fn fetch_groupware_resources( &self, access_account_id: u32, account_id: u32, collection: SyncCollection, - ) -> trc::Result> { - let cache_store = match collection { - SyncCollection::Calendar => &self.inner.cache.events, - SyncCollection::AddressBook => &self.inner.cache.contacts, - SyncCollection::FileNode => &self.inner.cache.files, - SyncCollection::CalendarEventNotification => &self.inner.cache.scheduling, - _ => unreachable!(), + ) -> impl Future>> + Send { + self.fetch_groupware_resources_at(access_account_id, account_id, collection, 0) + } + + async fn fetch_groupware_resources_at( + &self, + access_account_id: u32, + account_id: u32, + collection: SyncCollection, + min_change_id: u64, + ) -> trc::Result> { + let caches = &self.inner.cache; + let mut cache = match caches.resident_resources(account_id, collection) { + Some(cache) => cache, + None => match load_cache(self, account_id, collection, access_account_id).await? { + LoadedCache::Resident(cache) => cache, + LoadedCache::Built(cache) => return Ok(cache), + }, }; - let mut cache = match cache_store.get_value_or_guard_async(&account_id).await { - Ok(cache) => cache, - Err(guard) => { - let start_time = Instant::now(); - let cache = full_cache_build( - self, - account_id, - collection, - Arc::new(UpdateLock::new()), - access_account_id, - ) - .await?; - if guard.insert(cache.clone()).is_err() { - cache_store.update(account_id, cache.clone()); - } - warn_if_uncacheable(cache_store, account_id, collection, &cache); + // Serve the snapshot without revalidating while it is within the freshness window + let start_time = Instant::now(); + let revalidate = &self.inner.cache.revalidate; + if cache.is_fresh(revalidate) && cache.highest_change_id >= min_change_id { + trc::event!( + Cache(CacheEvent::Hit), + AccountId = account_id, + Collection = collection.as_str(), + ChangeId = cache.highest_change_id, + Elapsed = start_time.elapsed(), + ); - trc::event!( - Store(StoreEvent::CacheMiss), - AccountId = account_id, - Collection = collection.as_str(), - Total = cache.resources.len(), - ChangeId = cache.highest_change_id, - Elapsed = start_time.elapsed(), - ); - - return Ok(cache); - } - }; + return Ok(cache); + } // Obtain current state - let start_time = Instant::now(); + let verification = Verification::capture(revalidate, &cache.update_lock); let changes = self .core .storage @@ -129,20 +201,28 @@ impl GroupwareCache for Server { .caused_by(trc::location!())?; // Regenerate cache if the change log has been truncated - if changes.is_truncated { + if changes.needs_full_rebuild(cache.highest_change_id) { let cache = full_cache_build( self, account_id, collection, cache.update_lock.clone(), access_account_id, + verification, ) .await?; - cache_store.update(account_id, cache.clone()); - warn_if_uncacheable(cache_store, account_id, collection, &cache); + caches.publish_resources( + account_id, + collection, + &cache, + Some(SwapChanges::rebuilt( + cache.highest_change_id, + cache.resources.len(), + )), + ); trc::event!( - Store(StoreEvent::CacheStale), + Cache(CacheEvent::Stale), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, @@ -155,8 +235,18 @@ impl GroupwareCache for Server { // Verify changes if changes.changes.is_empty() { + let cache = if verification.is_current(&cache.update_lock) { + let mut verified = cache.as_ref().clone(); + verified.verification = verification; + let verified = Arc::new(verified); + caches.publish_resources(account_id, collection, &verified, None); + verified + } else { + cache + }; + trc::event!( - Store(StoreEvent::CacheHit), + Cache(CacheEvent::Hit), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, @@ -171,10 +261,12 @@ impl GroupwareCache for Server { let _permit = match lock.acquire(cache.highest_change_id).await? { LockResult::Acquired(permit) => permit, LockResult::Stale(permit) => { - cache = cache_store.peek(&account_id).unwrap_or(cache.clone()); + cache = caches + .peek_resources(account_id, collection) + .unwrap_or(cache); if cache.highest_change_id >= changes.to_change_id { trc::event!( - Store(StoreEvent::CacheHit), + Cache(CacheEvent::Hit), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, @@ -188,132 +280,44 @@ impl GroupwareCache for Server { }; let num_changes = changes.changes.len(); - let cache = if !matches!(collection, SyncCollection::CalendarEventNotification) { - let mut updated_resources = AHashMap::with_capacity(8); - let has_no_children = collection == SyncCollection::FileNode; + let base_change_id = cache.highest_change_id; + let mut updated_resources = AHashMap::with_capacity(8); + let has_no_children = collection == SyncCollection::FileNode; + let mut staging = ResourceChunkBuilder::with_capacity(8); - process_changes( - self, - account_id, - collection, - has_no_children, - &mut updated_resources, - changes.changes, - ) - .await?; + process_changes( + self, + account_id, + collection, + has_no_children, + &mut staging, + &mut updated_resources, + changes.changes, + ) + .await?; - let mut rebuild_hierarchy = false; - let mut resources = Vec::with_capacity(cache.resources.len()); - - for resource in &cache.resources { - let is_container = has_no_children || resource.is_container(); - if let Some(updated_resource) = - updated_resources.remove(&(is_container, resource.document_id)) - { - if let Some(updated_resource) = updated_resource { - rebuild_hierarchy = - rebuild_hierarchy || updated_resource.has_hierarchy_changes(resource); - resources.push(updated_resource); - } else { - // Deleted resource - rebuild_hierarchy = true; - } - } else { - resources.push(resource.clone()); - } - } - - // Add new resources - for resource in updated_resources.into_values().flatten() { - resources.push(resource); - rebuild_hierarchy = true; - } - - if rebuild_hierarchy { - let mut cache = DavResources { - base_path: cache.base_path.clone(), - paths: Default::default(), - resources, - item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), - container_change_id: changes - .container_change_id - .unwrap_or(cache.container_change_id), - highest_change_id: changes.to_change_id, - size: std::mem::size_of::() as u64, - update_lock: lock.clone(), - }; - - if matches!(collection, SyncCollection::FileNode) { - build_nested_hierarchy(&mut cache); - } else { - build_simple_hierarchy(&mut cache); - } - cache - } else { - DavResources { - base_path: cache.base_path.clone(), - paths: cache.paths.clone(), - resources, - item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), - container_change_id: changes - .container_change_id - .unwrap_or(cache.container_change_id), - highest_change_id: changes.to_change_id, - size: cache.size, - update_lock: lock.clone(), - } - } - } else { - let mut delete_ids = AHashSet::with_capacity(changes.changes.len()); - let mut resources = Vec::with_capacity(cache.resources.len()); - let mut paths = AHashSet::with_capacity(cache.paths.len()); - - for change in changes.changes { - match change { - Change::InsertItem(document_id) => { - let document_id = document_id as u32; - paths.insert(path_from_scheduling(document_id, resources.len(), false)); - resources.push(resource_from_scheduling(document_id, false)); - } - Change::DeleteItem(document_id) => { - delete_ids.insert(document_id as u32); - } - _ => {} - } - } - - for resource in &cache.resources { - if !delete_ids.contains(&resource.document_id) { - paths.insert(path_from_scheduling( - resource.document_id, - resources.len(), - resource.is_container(), - )); - resources.push(resource.clone()); - } - } - - DavResources { - base_path: cache.base_path.clone(), - paths, - resources, - item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), - container_change_id: changes - .container_change_id - .unwrap_or(cache.container_change_id), - highest_change_id: changes.to_change_id, - size: cache.size, - update_lock: cache.update_lock.clone(), - } - }; + let staging = staging.finish(); + let mut cache = rebuild_cache(&cache, collection, &staging, &updated_resources); + cache.item_change_id = changes.item_change_id.unwrap_or(cache.item_change_id); + cache.container_change_id = changes + .container_change_id + .unwrap_or(cache.container_change_id); + cache.highest_change_id = changes.to_change_id; + cache.update_lock = lock.clone(); + cache.verification = verification; cache.update_lock.set_revision(cache.highest_change_id); let cache = Arc::new(cache); - cache_store.update(account_id, cache.clone()); - warn_if_uncacheable(cache_store, account_id, collection, &cache); + let swap_changes = SwapChanges::applied( + num_changes, + base_change_id, + cache.highest_change_id, + cache.resources.len(), + ); + caches.publish_resources(account_id, collection, &cache, Some(swap_changes)); trc::event!( - Store(StoreEvent::CacheUpdate), + Cache(CacheEvent::Update), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, @@ -334,10 +338,7 @@ impl GroupwareCache for Server { let mut batch = BatchBuilder::new(); let account_id = account_info_owner.account_id(); let account_name = account_info_owner.name(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::AddressBook, 1) - .await?; + let document_id = batch.reserve_document_id(account_id, Collection::AddressBook); AddressBook { name: name.clone(), preferences: vec![AddressBookPreferences { @@ -357,7 +358,7 @@ impl GroupwareCache for Server { ..Default::default() } .insert( - account_info_access.account_tenant_ids(), + account_info_access.changed_by(), account_id, document_id, &mut batch, @@ -366,13 +367,14 @@ impl GroupwareCache for Server { batch .with_collection(Collection::Principal) .with_document(0) + .assert_value(PrincipalField::DefaultAddressBookId, AssertValue::None) .set( PrincipalField::DefaultAddressBookId, - document_id.serialize(), + PendingId::Slot(document_id), ); - self.commit_batch(batch).await?; - Ok(Some(document_id)) + let ids = self.commit_batch(batch).await?; + Ok(Some(ids.slot(document_id))) } else { Ok(None) } @@ -387,10 +389,7 @@ impl GroupwareCache for Server { let mut batch = BatchBuilder::new(); let account_id = account_info_owner.account_id(); let account_name = account_info_owner.name(); - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Calendar, 1) - .await?; + let document_id = batch.reserve_document_id(account_id, Collection::Calendar); Calendar { name: name.clone(), preferences: vec![CalendarPreferences { @@ -410,7 +409,7 @@ impl GroupwareCache for Server { ..Default::default() } .insert( - account_info_access.account_tenant_ids(), + account_info_access.changed_by(), account_id, document_id, &mut batch, @@ -420,10 +419,14 @@ impl GroupwareCache for Server { batch .with_collection(Collection::Principal) .with_document(0) - .set(PrincipalField::DefaultCalendarId, document_id.serialize()); + .assert_value(PrincipalField::DefaultCalendarId, AssertValue::None) + .set( + PrincipalField::DefaultCalendarId, + PendingId::Slot(document_id), + ); - self.commit_batch(batch).await?; - Ok(Some(document_id)) + let ids = self.commit_batch(batch).await?; + Ok(Some(ids.slot(document_id))) } else { Ok(None) } @@ -431,133 +434,429 @@ impl GroupwareCache for Server { async fn get_or_create_default_calendar( &self, - access_account_id: u32, - account_id: u32, + account_info_access: &AccountCache, + account_info_owner: &AccountCache, ) -> trc::Result> { - let default_calendar_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), - }) + let account_id = account_info_owner.account_id(); + if let Some(calendar_id) = any_calendar_id(self, account_id).await? { + return Ok(Some(calendar_id)); + } + + match self + .create_default_calendar(account_info_access, account_info_owner) .await - .caused_by(trc::location!())?; - if default_calendar_id.is_some() { - Ok(default_calendar_id) - } else { - self.fetch_dav_resources(access_account_id, account_id, SyncCollection::Calendar) - .await - .map(|c| c.document_ids(true).next()) + { + Err(err) if err.is_assertion_failure() => any_calendar_id(self, account_id).await, + result => result, } } + async fn count_documents( + &self, + account_id: u32, + collection: Collection, + limit: usize, + ) -> trc::Result { + let caches = &self.inner.cache; + let counted = match collection { + Collection::CalendarEvent => { + Some((SyncCollection::Calendar, StorageQuota::MaxCalendarEvents)) + } + Collection::CalendarEventNotification => Some(( + SyncCollection::CalendarEventNotification, + StorageQuota::MaxCalendarEventNotifications, + )), + _ => None, + }; + if let Some((collection, object)) = counted { + return match caches + .peek_resources(account_id, collection) + .filter(|resources| resources.is_fresh(&caches.revalidate)) + { + Some(resources) => Ok(resources.resources.count(false)), + None => self + .object_count(account_id, object) + .await + .caused_by(trc::location!()), + }; + } + + let cached = match collection { + Collection::Calendar => Some(SyncCollection::Calendar), + Collection::AddressBook | Collection::ContactCard => Some(SyncCollection::AddressBook), + _ => None, + } + .filter(|collection| caches.has_resources(account_id, *collection)); + if let Some(sync_collection) = cached { + let is_container = matches!(collection, Collection::Calendar | Collection::AddressBook); + return self + .fetch_groupware_resources(account_id, account_id, sync_collection) + .await + .map(|resources| resources.resources.count(is_container)); + } + + let mut count = 0; + self.store() + .iterate( + IterateParams::new( + ValueKey::archive(account_id, collection, 0), + ValueKey::archive(account_id, collection, u32::MAX), + ) + .no_values() + .limit(limit), + |_, _| { + count += 1; + Ok(count < limit) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| count) + } + #[inline(always)] - fn cached_dav_resources( + fn cached_groupware_resources( &self, account_id: u32, collection: SyncCollection, - ) -> Option> { - (match collection { - SyncCollection::Calendar => &self.inner.cache.events, - SyncCollection::AddressBook => &self.inner.cache.contacts, - SyncCollection::FileNode => &self.inner.cache.files, - _ => unreachable!(), - }) - .get(&account_id) + ) -> Option> { + self.inner.cache.resident_resources(account_id, collection) } } +async fn any_calendar_id(server: &Server, account_id: u32) -> trc::Result> { + if let Some(calendar_id) = server + .store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), + }) + .await + .caused_by(trc::location!())? + { + return Ok(Some(calendar_id)); + } + + let mut calendar_id = None; + server + .store() + .iterate( + IterateParams::new( + ValueKey::archive(account_id, Collection::Calendar, 0), + ValueKey::archive(account_id, Collection::Calendar, u32::MAX), + ) + .no_values() + .ascending() + .only_first(), + |key, _| { + calendar_id = key.deserialize_be_u32(key.len() - U32_LEN).map(Some)?; + Ok(false) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| calendar_id) +} + +fn rebuild_cache( + previous: &GroupwareResources, + collection: SyncCollection, + staging: &ResourceChunk, + changes: &AHashMap<(bool, u32), Option>, +) -> GroupwareResources { + let mut updated = GroupwareResources { + base_path: previous.base_path.clone(), + paths: previous.paths.clone(), + resources: previous.resources.rebuild(staging, changes), + item_change_id: previous.item_change_id, + container_change_id: previous.container_change_id, + highest_change_id: previous.highest_change_id, + size: 0, + update_lock: previous.update_lock.clone(), + verification: Default::default(), + }; + + match updated.patch_paths(&previous.resources, staging, changes) { + PathUpdate::Shared => {} + PathUpdate::Patched(paths) => updated.paths = Arc::new(paths), + PathUpdate::Rebuild => { + updated.paths = Arc::new(match collection { + SyncCollection::FileNode => build_nested_hierarchy(&updated.resources), + SyncCollection::CalendarEventNotification => { + build_scheduling_paths(&updated.resources) + } + _ => build_calcard_paths(&updated.resources), + }) + } + } + + updated.recompute_size(); + updated +} + async fn process_changes( server: &Server, account_id: u32, collection: SyncCollection, has_no_children: bool, - updated_resources: &mut AHashMap<(bool, u32), Option>, + staging: &mut ResourceChunkBuilder, + updated_resources: &mut AHashMap<(bool, u32), Option>, changes: Vec, ) -> trc::Result<()> { - for change in changes { + let mut item_ids = Vec::new(); + for change in changes.into_iter().rev() { match change { - Change::InsertItem(id) | Change::UpdateItem(id) => { + Change::InsertItem(id) | Change::UpdateItem(id) | Change::UpdateItemMetadata(id) => { let document_id = id as u32; - if let Some(archive) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - collection.collection(false), - document_id, - )) - .await - .caused_by(trc::location!())? + if let Entry::Vacant(entry) = + updated_resources.entry((has_no_children, document_id)) { - updated_resources.insert( - (has_no_children, document_id), - Some(resource_from_archive( - archive, - document_id, - collection, - false, - )?), - ); - } else { - updated_resources.insert((has_no_children, document_id), None); + entry.insert(None); + item_ids.push(document_id); } } Change::DeleteItem(id) => { - updated_resources.insert((has_no_children, id as u32), None); + updated_resources + .entry((has_no_children, id as u32)) + .or_insert(None); } Change::InsertContainer(id) | Change::UpdateContainer(id) => { - let document_id = id as u32; - if let Some(archive) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - collection.collection(true), - document_id, - )) - .await - .caused_by(trc::location!())? - { - updated_resources.insert( - (true, document_id), - Some(resource_from_archive( - archive, - document_id, - collection, - true, - )?), - ); - } else { - updated_resources.insert((true, document_id), None); - } + fetch_container( + server, + account_id, + collection, + id as u32, + staging, + updated_resources, + ) + .await?; + } + Change::UpdateContainerPartial(id, partial) if partial.has_metadata() => { + fetch_container( + server, + account_id, + collection, + id as u32, + staging, + updated_resources, + ) + .await?; } Change::DeleteContainer(id) => { - updated_resources.insert((true, id as u32), None); + updated_resources.entry((true, id as u32)).or_insert(None); + } + Change::UpdateContainerPartial(..) => (), + } + } + + item_ids.sort_unstable(); + for chunk in item_ids.chunks(MAX_BATCH_DOCUMENTS) { + let mut archives = server + .archive_chunk(account_id, collection.collection(false), chunk, None) + .await + .caused_by(trc::location!())?; + for &document_id in chunk { + if let Some(archive) = archives.take_archive(document_id) { + let slot = staging.len() as u32; + push_from_archive(staging, archive, document_id, collection, false)?; + updated_resources.insert((has_no_children, document_id), Some(slot)); } - Change::UpdateContainerProperty(_) => (), } } Ok(()) } -#[inline(always)] -fn warn_if_uncacheable( - cache_store: &Cache>, +async fn fetch_container( + server: &Server, account_id: u32, collection: SyncCollection, - cache: &Arc, -) { - let capacity = cache_store.weight_capacity(); - if cache.size > capacity { + document_id: u32, + staging: &mut ResourceChunkBuilder, + updated_resources: &mut AHashMap<(bool, u32), Option>, +) -> trc::Result<()> { + let Entry::Vacant(entry) = updated_resources.entry((true, document_id)) else { + return Ok(()); + }; + let slot = if let Some(archive) = server + .store() + .get_value::>(ValueKey::archive( + account_id, + collection.collection(true), + document_id, + )) + .await + .caused_by(trc::location!())? + { + let slot = staging.len() as u32; + push_from_archive(staging, archive, document_id, collection, true)?; + Some(slot) + } else { + None + }; + entry.insert(slot); + Ok(()) +} + +enum LoadedCache { + Resident(Arc), + Built(Arc), +} + +async fn load_cache( + server: &Server, + account_id: u32, + collection: SyncCollection, + access_account_id: u32, +) -> trc::Result { + Box::pin(load_missing_cache( + server, + account_id, + collection, + access_account_id, + )) + .await +} + +async fn load_missing_cache( + server: &Server, + account_id: u32, + collection: SyncCollection, + access_account_id: u32, +) -> trc::Result { + let caches = &server.inner.cache; + let Some(resources) = caches.resources(collection) else { + return Err(trc::StoreEvent::UnexpectedError + .into_err() + .details("Unsupported groupware collection") + .caused_by(trc::location!())); + }; + let guard = match resources.get_value_or_guard_async(&account_id).await { + Ok(cache) => return Ok(LoadedCache::Resident(cache)), + Err(guard) => guard, + }; + + if let Some(cache) = caches + .resources_large + .lookup(SwapTarget::new(account_id, collection)) + { + drop(guard); + return Ok(LoadedCache::Resident(cache)); + } + + let start_time = Instant::now(); + if let Some(cache) = restore_cache_build(server, account_id, collection).await { + caches.publish_loaded_resources(guard, account_id, collection, &cache, None); + return Ok(LoadedCache::Resident(cache)); + } + + let cache = full_cache_build( + server, + account_id, + collection, + Arc::new(UpdateLock::new()), + access_account_id, + Default::default(), + ) + .await?; + caches.publish_loaded_resources( + guard, + account_id, + collection, + &cache, + Some(SwapChanges::rebuilt( + cache.highest_change_id, + cache.resources.len(), + )), + ); + + trc::event!( + Cache(CacheEvent::Miss), + AccountId = account_id, + Collection = collection.as_str(), + Total = cache.resources.len(), + ChangeId = cache.highest_change_id, + Elapsed = start_time.elapsed(), + ); + + Ok(LoadedCache::Built(cache)) +} + +async fn restore_cache_build( + server: &Server, + account_id: u32, + collection: SyncCollection, +) -> Option> { + if !server.inner.cache.swap.is_enabled() { + return None; + } + + Box::pin(restore_swapped_cache(server, account_id, collection)).await +} + +async fn restore_swapped_cache( + server: &Server, + account_id: u32, + collection: SyncCollection, +) -> Option> { + let start_time = Instant::now(); + let key = SwapKey::new(account_id, collection, SwapPart::Resources); + let snapshot = match server.inner.cache.swap.load(key).await { + Ok(Some(snapshot)) => snapshot, + Ok(None) => return None, + Err(err) => { + trc::error!( + err.details("Failed to read the resource cache snapshot") + .ctx(trc::Key::AccountId, account_id) + .ctx(trc::Key::Collection, collection.as_str()) + ); + return None; + } + }; + + let snapshot_len = snapshot.len(); + let restored = if snapshot_len >= BLOCKING_CODEC_THRESHOLD { + tokio::task::spawn_blocking(move || GroupwareResources::from_snapshot(&snapshot)) + .await + .ok() + .flatten() + } else { + GroupwareResources::from_snapshot(&snapshot) + } + .or_else(|| { trc::event!( - Store(StoreEvent::CacheEntryTooLarge), + Cache(CacheEvent::SwapMiss), AccountId = account_id, Collection = collection.as_str(), - Size = cache.size, - Limit = capacity, + Details = "Discarded an unreadable resource cache snapshot", ); + None + })?; + + let base_path = DavResourceName::from(collection) + .account_base_path(server.account(account_id).await.ok()?.name()); + if restored.base_path != base_path { + trc::event!( + Cache(CacheEvent::SwapMiss), + AccountId = account_id, + Collection = collection.as_str(), + Details = "Discarded a resource cache snapshot with a stale base path", + ); + return None; } + + trc::event!( + Cache(CacheEvent::SwapHit), + AccountId = account_id, + Collection = collection.as_str(), + ChangeId = restored.highest_change_id, + Total = restored.resources.len(), + Size = snapshot_len, + Elapsed = start_time.elapsed(), + ); + + Some(Arc::new(restored)) } async fn full_cache_build( @@ -566,7 +865,27 @@ async fn full_cache_build( collection: SyncCollection, update_lock: Arc, access_account_id: u32, -) -> trc::Result> { + verification: Verification, +) -> trc::Result> { + Box::pin(build_full_cache( + server, + account_id, + collection, + update_lock, + access_account_id, + verification, + )) + .await +} + +async fn build_full_cache( + server: &Server, + account_id: u32, + collection: SyncCollection, + update_lock: Arc, + access_account_id: u32, + verification: Verification, +) -> trc::Result> { match collection { SyncCollection::Calendar => { build_calcard_resources( @@ -598,56 +917,78 @@ async fn full_cache_build( } _ => unreachable!(), } - .map(Arc::new) + .map(|mut cache| { + cache.verification = verification; + Arc::new(cache) + }) } -fn resource_from_archive( - archive: Archive, +fn push_from_archive( + builder: &mut ResourceChunkBuilder, + archive: Archive, document_id: u32, collection: SyncCollection, is_container: bool, -) -> trc::Result { - Ok(match collection { +) -> trc::Result<()> { + let etag = archive.version.hash().unwrap_or_default(); + + match collection { SyncCollection::Calendar => { if is_container { - resource_from_calendar( + push_calendar( + builder, archive .unarchive::() .caused_by(trc::location!())?, document_id, - ) + etag, + ); } else { - resource_from_event( + push_event( + builder, archive .unarchive::() .caused_by(trc::location!())?, document_id, - ) + ); } } SyncCollection::AddressBook => { if is_container { - resource_from_addressbook( + push_addressbook( + builder, archive .unarchive::() .caused_by(trc::location!())?, document_id, - ) + etag, + ); } else { - resource_from_card( + push_card( + builder, archive .unarchive::() .caused_by(trc::location!())?, document_id, - ) + ); } } - SyncCollection::FileNode => resource_from_file( + SyncCollection::FileNode => push_file( + builder, archive .unarchive::() .caused_by(trc::location!())?, document_id, ), + SyncCollection::CalendarEventNotification => push_scheduling( + builder, + archive + .unarchive::() + .caused_by(trc::location!())?, + document_id, + ), _ => unreachable!(), - }) + } + Ok(()) } + diff --git a/crates/groupware/src/calendar/alarm.rs b/crates/groupware/src/calendar/alarm.rs index 2bc59fd3..217a5160 100644 --- a/crates/groupware/src/calendar/alarm.rs +++ b/crates/groupware/src/calendar/alarm.rs @@ -5,25 +5,115 @@ */ use super::{ - Alarm, AlarmDelta, ArchivedAlarmDelta, ArchivedCalendarEventData, expand::resolve_local, + ALARM_EMAIL, Alarm, AlarmDelta, AlarmOffset, ArchivedAlarmDelta, ArchivedAlarmOffset, + ArchivedCalendarEventContent, ArchivedCalendarEventData, ArchivedEventPreferences, + ArchivedEventUserData, CalendarEventData, DefaultAlert, EventPreferences, PREF_HAS_ALERTS, + alerts::{DefaultAlerts, SnoozeAlarm}, + expand::{ + ComponentRecurrenceId, NaiveTimestamp, RangeFlags, RecurrenceKey, RecurrenceShift, + SECONDS_PER_DAY, resolve_local, + }, + user::BASE_INSTANCE, }; use calcard::{ - common::timezone::Tz, + common::timezone::{NominalDuration, Tz, ZonedDateTime}, icalendar::{ - ICalendarComponent, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, - ICalendarRelated, ICalendarValue, + ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarEntry, ICalendar, + ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, ICalendarRelated, + ICalendarValue, }, }; use std::str::FromStr; -use store::write::bitpack::BitpackIterator; -use utils::codec::leb128::Leb128Reader; +use store::write::{TaskId, bitpack::BitpackIterator}; +use utils::codec::leb128::Leb128; + +const TZ_BOUND: i64 = 24 * 60 * 60; +const ALARM_KIND_SHIFT: u32 = 16; +const ALARM_INDEX_MASK: u64 = u16::MAX as u64; +const ALARM_KIND_STORED: u64 = 0; +const ALARM_KIND_PERSONAL: u64 = 1; +const ALARM_KIND_DEFAULT: u64 = 2; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AlarmId { + Stored(u16), + Personal(u16), + Default(u16), +} + +impl AlarmId { + pub fn to_task_id(self) -> u64 { + let (kind, index) = match self { + AlarmId::Stored(index) => (ALARM_KIND_STORED, index), + AlarmId::Personal(index) => (ALARM_KIND_PERSONAL, index), + AlarmId::Default(index) => (ALARM_KIND_DEFAULT, index), + }; + (kind << ALARM_KIND_SHIFT) | index as u64 + } + + pub fn from_task_id(task_id: u64) -> Option { + let index = (task_id & ALARM_INDEX_MASK) as u16; + match task_id >> ALARM_KIND_SHIFT { + ALARM_KIND_STORED => Some(AlarmId::Stored(index)), + ALARM_KIND_PERSONAL => Some(AlarmId::Personal(index)), + ALARM_KIND_DEFAULT => Some(AlarmId::Default(index)), + _ => None, + } + } + + pub fn index(self) -> u16 { + match self { + AlarmId::Stored(index) | AlarmId::Personal(index) | AlarmId::Default(index) => index, + } + } + + pub fn is_default(self) -> bool { + matches!(self, AlarmId::Default(_)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum AlarmTarget { + Owner, + Sharee(u32), +} + +impl AlarmTarget { + pub fn for_account(owner_id: u32, target_id: u32) -> Self { + if target_id == owner_id { + AlarmTarget::Owner + } else { + AlarmTarget::Sharee(target_id) + } + } + + pub fn sharee_id(self) -> Option { + match self { + AlarmTarget::Owner => None, + AlarmTarget::Sharee(account_id) => Some(account_id), + } + } + + pub fn account_id(self, owner_id: u32) -> u32 { + self.sharee_id().unwrap_or(owner_id) + } + + pub fn task_id(self) -> TaskId { + match self { + AlarmTarget::Owner => TaskId::Document, + AlarmTarget::Sharee(account_id) => TaskId::DocumentTarget(account_id), + } + } +} #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CalendarAlarm { - pub alarm_id: u16, + pub alarm_id: AlarmId, pub event_id: u16, pub alarm_time: i64, pub typ: CalendarAlarmType, + pub target: AlarmTarget, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -33,34 +123,691 @@ pub enum CalendarAlarmType { event_start_tz: u16, event_end: i64, event_end_tz: u16, + recurrence_id: Option, }, Display { recurrence_id: Option, }, } -impl ArchivedCalendarEventData { - pub fn next_alarm(&self, start_time: i64, default_tz: Tz) -> Option { - if self.alarms.is_empty() { +impl CalendarAlarm { + pub fn with_target(mut self, target: AlarmTarget) -> Self { + self.target = target; + self + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PersonalAlarm { + id: u16, + delta: AlarmDelta, + is_email_alert: bool, + snoozed_alarms: Vec, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PersonalAlarms { + base: Vec, + instances: Vec<(u32, Vec)>, +} + +impl PersonalAlarm { + fn new(alert: &ICalendarComponent, siblings: &[ICalendarComponent], id: u16) -> Option { + alert.expand_alarm(id, 0).map(|alarm| PersonalAlarm { + id, + is_email_alert: alarm.is_email(), + delta: alarm.delta, + snoozed_alarms: alert + .snoozed_alarms() + .flat_map(|parent| { + std::iter::once(parent).chain( + siblings + .iter() + .filter(move |sibling| { + sibling.component_type == ICalendarComponentType::VAlarm + && sibling.uid() == Some(parent) + }) + .filter_map(ICalendarComponent::jsid), + ) + }) + .map(str::to_string) + .collect(), + }) + } + + fn is_kept_with(&self, defaults: &DefaultAlerts) -> bool { + self.snoozed_alarms + .iter() + .any(|id| defaults.contains_id(id)) + } +} + +impl PersonalAlarms { + pub fn new(preferences: &EventPreferences) -> Self { + let mut alarms = PersonalAlarms::default(); + for instance in preferences + .instances + .iter() + .filter(|instance| instance.flags & PREF_HAS_ALERTS != 0) + { + let expanded = instance + .alerts + .iter() + .zip(0u16..) + .filter_map(|(alert, id)| PersonalAlarm::new(alert, &instance.alerts, id)) + .collect::>(); + if instance.recurrence_key == BASE_INSTANCE { + alarms.base = expanded; + } else { + alarms.instances.push((instance.recurrence_key, expanded)); + } + } + alarms + } + + pub fn is_empty(&self) -> bool { + self.base.is_empty() && self.instances.iter().all(|(_, alarms)| alarms.is_empty()) + } + + fn scoped(&self) -> impl Iterator { + self.base + .iter() + .map(|alarm| (alarm, AlarmScope::Base)) + .chain(self.instances.iter().flat_map(|(key, alarms)| { + alarms + .iter() + .map(move |alarm| (alarm, AlarmScope::Instance(*key))) + })) + } + + fn has_instance(&self, recurrence_key: u32) -> bool { + self.instances.iter().any(|(key, _)| *key == recurrence_key) + } +} + +impl ArchivedEventPreferences { + pub fn alerts_instance(&self, recurrence_key: Option) -> Option<&ArchivedEventUserData> { + recurrence_key + .and_then(|key| self.instance_with_alerts(key)) + .or_else(|| self.instance_with_alerts(BASE_INSTANCE)) + } + + pub fn personal_alert( + &self, + recurrence_key: Option, + index: u16, + ) -> Option<&ArchivedICalendarComponent> { + self.alerts_instance(recurrence_key) + .and_then(|instance| instance.alerts.get(index as usize)) + } + + fn instance_with_alerts(&self, recurrence_key: u32) -> Option<&ArchivedEventUserData> { + self.instances + .binary_search_by_key(&recurrence_key, |instance| { + instance.recurrence_key.to_native() + }) + .ok() + .and_then(|idx| self.instances.get(idx)) + .filter(|instance| instance.flags.to_native() & PREF_HAS_ALERTS != 0) + } +} + +#[derive(Clone, Copy)] +pub enum TriggeredAlarm<'x> { + Component(&'x ArchivedICalendarComponent), + Default { + alert: &'x DefaultAlert, + acknowledged: Option<&'x ArchivedICalendarComponent>, + }, +} + +impl<'x> TriggeredAlarm<'x> { + pub fn component(&self) -> Option<&'x ArchivedICalendarComponent> { + match self { + TriggeredAlarm::Component(component) => Some(component), + TriggeredAlarm::Default { .. } => None, + } + } + + pub fn alert_id(&self) -> Option<&'x str> { + match self { + TriggeredAlarm::Component(component) => component.alarm_jsid(), + TriggeredAlarm::Default { alert, .. } => Some(alert.id.as_str()), + } + } + + pub fn acknowledged(&self) -> Option { + match self { + TriggeredAlarm::Component(component) + | TriggeredAlarm::Default { + acknowledged: Some(component), + .. + } => component.acknowledged(), + TriggeredAlarm::Default { + acknowledged: None, .. + } => None, + } + } +} + +pub trait ArchivedAlarmComponent { + fn alarm_jsid(&self) -> Option<&str>; + + fn acknowledged(&self) -> Option; +} + +impl ArchivedAlarmComponent for ArchivedICalendarComponent { + fn alarm_jsid(&self) -> Option<&str> { + self.property(&ICalendarProperty::Jsid) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_text()) + } + + fn acknowledged(&self) -> Option { + self.property(&ICalendarProperty::Acknowledged) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_partial_date_time()) + .and_then(|date_time| date_time.to_date_time_with_tz(Tz::UTC)) + .map(|date_time| date_time.timestamp()) + } +} + +impl ArchivedCalendarEventContent { + pub fn triggered_alarm<'x>( + &'x self, + target: AlarmTarget, + alarm_id: AlarmId, + comp_id: u16, + recurrence_key: Option, + defaults: &'x DefaultAlerts, + ) -> Option> { + let components = &self.data.event.components; + let preferences = target + .sharee_id() + .and_then(|account_id| self.preferences(account_id)); + match alarm_id { + AlarmId::Default(index) => { + let alert = defaults.get(index)?; + let is_acknowledged_copy = |alarm: &&ArchivedICalendarComponent| { + alarm.alarm_jsid() == Some(alert.id.as_str()) + }; + let acknowledged = if target.sharee_id().is_some() { + preferences + .and_then(|preferences| preferences.alerts_instance(recurrence_key)) + .and_then(|instance| instance.alerts.iter().find(is_acknowledged_copy)) + } else { + components + .get(comp_id as usize) + .into_iter() + .flat_map(|component| component.component_ids.iter()) + .filter_map(|id| components.get(id.to_native() as usize)) + .find(is_acknowledged_copy) + }; + Some(TriggeredAlarm::Default { + alert, + acknowledged, + }) + } + AlarmId::Personal(index) => preferences? + .personal_alert(recurrence_key, index) + .map(TriggeredAlarm::Component), + AlarmId::Stored(index) => components + .get(index as usize) + .filter(|component| component.component_type == ICalendarComponentType::VAlarm) + .map(TriggeredAlarm::Component), + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AlarmRecurrence { + is_recurrent: bool, + shift: Option, + tz: Tz, + flags: RangeFlags, + recurrence_tz: Tz, +} + +impl AlarmRecurrence { + pub fn recurrence_id(&self, start_date_naive: i64) -> Option { + match (self.is_recurrent, &self.shift) { + (false, _) => None, + (true, Some(shift)) => shift + .recurrence_id(self.tz, self.flags, start_date_naive, self.recurrence_tz) + .map(|recurrence_id| recurrence_id.naive), + (true, None) => Some(start_date_naive), + } + } + + pub fn recurrence_key(&self, start_date_naive: i64) -> Option { + self.recurrence_id(start_date_naive) + .and_then(RecurrenceKey::from_recurrence_id) + .map(RecurrenceKey::prefix) + } + + pub fn is_recurrent(&self) -> bool { + self.is_recurrent + } + + fn of( + component: Option<&C>, + own_tz: Tz, + tz: Tz, + flags: RangeFlags, + ) -> Self { + component.map_or_else(AlarmRecurrence::default, |component| AlarmRecurrence { + is_recurrent: component.is_recurrent_or_override(), + shift: component + .timestamp(&ICalendarProperty::Dtstart) + .zip(component.recurrence_id(own_tz)) + .zip(component.recurrence_tz(own_tz)) + .and_then(|((start, recurrence_id), recurrence_tz)| { + RecurrenceShift::new(own_tz, start, recurrence_id, recurrence_tz) + }), + tz, + flags, + recurrence_tz: component.recurrence_tz(tz).unwrap_or(tz), + }) + } +} + +pub trait CalendarEntryView { + fn as_boolean(&self) -> Option; + + fn is_date_only(&self) -> bool; + + fn timestamp(&self) -> Option; +} + +pub trait CalendarComponentView { + type Entry: CalendarEntryView; + + fn entry(&self, property: &ICalendarProperty) -> Option<&Self::Entry>; + + fn is_event_or_todo(&self) -> bool; + + fn is_recurrence_override(&self) -> bool; + + fn is_recurrent_or_override(&self) -> bool; + + fn timestamp(&self, property: &ICalendarProperty) -> Option { + self.entry(property).and_then(CalendarEntryView::timestamp) + } + + fn is_shown_without_time(&self) -> bool { + self.entry(&ICalendarProperty::ShowWithoutTime) + .and_then(CalendarEntryView::as_boolean) + .unwrap_or(false) + || self + .entry(&ICalendarProperty::Dtstart) + .is_some_and(CalendarEntryView::is_date_only) + } +} + +pub trait CalendarView { + type Component: CalendarComponentView; + + fn components(&self) -> impl Iterator; + + fn main_component(&self) -> Option<&Self::Component> { + self.components() + .find(|component| component.is_event_or_todo() && !component.is_recurrence_override()) + } +} + +macro_rules! calendar_view { + ($calendar:ty, $component:ty, $entry:ty) => { + impl CalendarEntryView for $entry { + fn as_boolean(&self) -> Option { + self.values.first().and_then(|value| value.as_boolean()) + } + + fn is_date_only(&self) -> bool { + self.values + .first() + .and_then(|value| value.as_partial_date_time()) + .is_some_and(|date_time| !date_time.has_time()) + } + + fn timestamp(&self) -> Option { + self.values + .first()? + .as_partial_date_time()? + .to_date_time() + .map(|date_time| date_time.date_time.naive_timestamp()) + } + } + + impl CalendarComponentView for $component { + type Entry = $entry; + + fn entry(&self, property: &ICalendarProperty) -> Option<&Self::Entry> { + self.property(property) + } + + fn is_event_or_todo(&self) -> bool { + self.component_type.is_event_or_todo() + } + + fn is_recurrence_override(&self) -> bool { + <$component>::is_recurrence_override(self) + } + + fn is_recurrent_or_override(&self) -> bool { + <$component>::is_recurrent_or_override(self) + } + } + + impl CalendarView for $calendar { + type Component = $component; + + fn components(&self) -> impl Iterator { + self.components.iter() + } + } + }; +} + +calendar_view!(ICalendar, ICalendarComponent, ICalendarEntry); +calendar_view!( + ArchivedICalendar, + ArchivedICalendarComponent, + ArchivedICalendarEntry +); + +#[derive(Debug, Clone, Copy)] +pub enum AlarmSource<'x> { + Stored(&'x DefaultAlerts), + Personal(&'x PersonalAlarms, &'x DefaultAlerts), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AlarmScope { + All, + Base, + Instance(u32), +} + +impl AlarmScope { + fn is_occurrence(self) -> bool { + matches!(self, AlarmScope::Instance(_)) + } + + fn is_behind(self, recurrence_key: Option) -> bool { + matches!( + (self, recurrence_key), + (AlarmScope::Instance(key), Some(current)) if current > key + ) + } +} + +impl AlarmSource<'_> { + pub(crate) fn may_have_alarms(&self, has_stored_alarms: bool) -> bool { + match self { + AlarmSource::Stored(defaults) => has_stored_alarms || defaults.has_alerts(), + AlarmSource::Personal(alarms, defaults) if defaults.is_enabled() => { + defaults.has_alerts() + || alarms + .scoped() + .any(|(alarm, _)| alarm.is_kept_with(defaults)) + } + AlarmSource::Personal(alarms, _) => !alarms.is_empty(), + } + } + + fn has_floating_triggers(&self, data: &D) -> bool { + match self { + AlarmSource::Stored(_) => data.has_floating_stored_alarms(), + AlarmSource::Personal(alarms, _) => alarms + .scoped() + .any(|(alarm, _)| matches!(alarm.delta, AlarmDelta::FixedFloating(_))), + } + } + + fn uses_stored_alarms(&self) -> bool { + matches!(self, AlarmSource::Stored(_)) + } + + fn shared_candidates(&self, candidates: &mut Vec) { + let (defaults, personal) = match self { + AlarmSource::Stored(defaults) => (*defaults, None), + AlarmSource::Personal(alarms, defaults) => (*defaults, Some(*alarms)), + }; + candidates.extend(defaults.indexed_alerts().map(|(index, alert)| { + AlarmCandidate::new( + AlarmId::Default(index), + AlarmScope::All, + alert.delta(), + alert.is_email(), + ) + })); + if let Some(alarms) = personal { + let keep_snoozes_only = defaults.is_enabled(); + candidates.extend( + alarms + .scoped() + .filter(|(alarm, _)| !keep_snoozes_only || alarm.is_kept_with(defaults)) + .map(|(alarm, scope)| { + AlarmCandidate::new( + AlarmId::Personal(alarm.id), + scope, + alarm.delta.clone(), + alarm.is_email_alert, + ) + }), + ); + } + } + + fn applies(&self, scope: AlarmScope, recurrence_key: Option) -> bool { + match (scope, self) { + (AlarmScope::All, _) => true, + (AlarmScope::Instance(key), _) => recurrence_key == Some(key), + (AlarmScope::Base, AlarmSource::Personal(alarms, _)) => { + recurrence_key.is_none_or(|key| !alarms.has_instance(key)) + } + (AlarmScope::Base, AlarmSource::Stored(_)) => true, + } + } +} + +pub struct AlarmRange<'x> { + pub comp_id: u16, + pub start_tz: u16, + pub end_tz: u16, + pub duration: i64, + pub flags: RangeFlags, + pub instances: &'x [u8], +} + +#[derive(Debug, Clone)] +struct AlarmCandidate { + id: AlarmId, + scope: AlarmScope, + delta: AlarmDelta, + is_email_alert: bool, + naive_bias: Option, + fixed_time: i64, + is_pending: bool, +} + +#[derive(Debug, Default, Clone, Copy)] +struct PendingAlarms { + any: bool, + any_unbounded: bool, + min_bias: Option, +} + +impl AlarmCandidate { + fn new(id: AlarmId, scope: AlarmScope, delta: AlarmDelta, is_email_alert: bool) -> Self { + AlarmCandidate { + id, + scope, + delta, + is_email_alert, + naive_bias: None, + fixed_time: 0, + is_pending: false, + } + } + + fn from_stored(alarm: Alarm) -> Self { + let is_email_alert = alarm.is_email(); + AlarmCandidate::new( + AlarmId::Stored(alarm.id), + AlarmScope::All, + alarm.delta, + is_email_alert, + ) + } + + fn prepare( + &mut self, + duration: i64, + default_tz: Tz, + start_time: i64, + best_so_far: Option, + ) { + match &self.delta { + AlarmDelta::Start(offset) => { + self.naive_bias = Some(offset.naive_seconds()); + self.is_pending = true; + } + AlarmDelta::End(offset) => { + self.naive_bias = Some(offset.naive_seconds() + duration); + self.is_pending = true; + } + AlarmDelta::FixedUtc(_) | AlarmDelta::FixedFloating(_) => { + self.naive_bias = None; + match self.delta.fixed_timestamp(default_tz).filter(|alarm_time| { + *alarm_time > start_time && best_so_far.is_none_or(|best| *alarm_time < best) + }) { + Some(alarm_time) => { + self.fixed_time = alarm_time; + self.is_pending = true; + } + None => self.is_pending = false, + } + } + } + } + + fn alarm_time( + &self, + start: &ZonedDateTime, + end: &ZonedDateTime, + default_tz: Tz, + ) -> Option { + if self.naive_bias.is_some() { + self.delta.to_timestamp(start, end, default_tz) + } else { + Some(self.fixed_time) + } + } +} + +impl PendingAlarms { + fn of(candidates: &[AlarmCandidate]) -> Self { + let mut pending = PendingAlarms::default(); + for candidate in candidates.iter().filter(|candidate| candidate.is_pending) { + pending.any = true; + match candidate.naive_bias { + Some(bias) => { + pending.min_bias = + Some(pending.min_bias.map_or(bias, |min_bias| min_bias.min(bias))); + } + None => pending.any_unbounded = true, + } + } + pending + } + + fn is_past(&self, start_date_naive: i64, best_so_far: Option) -> bool { + if self.any_unbounded { + return false; + } + match (self.min_bias, best_so_far) { + (Some(min_bias), Some(best)) => { + start_date_naive + .saturating_add(min_bias) + .saturating_sub(TZ_BOUND) + >= best + } + _ => false, + } + } +} + +pub trait EventAlarmData { + fn alarm_base_offset(&self) -> i64; + + fn alarm_ranges(&self) -> impl Iterator>; + + fn stored_alarms(&self, comp_id: u16) -> impl Iterator; + + fn has_stored_alarms(&self) -> bool; + + fn has_floating_stored_alarms(&self) -> bool; + + fn component_recurrence( + &self, + comp_id: u16, + component_tz: Tz, + flags: RangeFlags, + ) -> AlarmRecurrence; + + fn needs_default_tz(&self, source: &AlarmSource<'_>) -> bool { + let floating = Tz::Floating.as_id(); + source.may_have_alarms(self.has_stored_alarms()) + && (self + .alarm_ranges() + .any(|range| range.start_tz == floating || range.end_tz == floating) + || source.has_floating_triggers(self)) + } + + fn next_alarm_from( + &self, + start_time: i64, + default_tz: Tz, + source: &AlarmSource<'_>, + ) -> Option { + if !source.may_have_alarms(self.has_stored_alarms()) { return None; } - let base_offset = self.base_offset.to_native(); + let base_offset = self.alarm_base_offset(); let mut next_alarm: Option = None; + let mut shared = Vec::new(); + source.shared_candidates(&mut shared); + let uses_stored_alarms = source.uses_stored_alarms(); + if shared.is_empty() && !uses_stored_alarms { + return None; + } + let mut candidates: Vec = Vec::with_capacity(shared.len()); - 'outer: for range in self.time_ranges.iter() { - let comp_id = range.id.to_native(); - let Some(alarm) = self.alarms.iter().find(|a| a.parent_id == comp_id) else { + for range in self.alarm_ranges() { + let comp_id = range.comp_id; + candidates.clear(); + if uses_stored_alarms { + candidates.extend(self.stored_alarms(comp_id).map(AlarmCandidate::from_stored)); + } + candidates.extend_from_slice(&shared); + if candidates.is_empty() { + continue; + } + + let instances = range.instances; + let Some((offset_or_count, bytes_read)) = u32::from_leb128(instances) else { continue; }; - - let instances = range.instances.as_ref(); - let (offset_or_count, bytes_read) = instances.read_leb128::()?; - - let duration = range.duration.to_native() as i64; - let mut start_tz = Tz::from_id(range.start_tz.to_native())?; - let mut end_tz = Tz::from_id(range.end_tz.to_native())?; - + let duration = range.duration; + let flags = range.flags; + let (Some(mut start_tz), Some(mut end_tz)) = + (Tz::from_id(range.start_tz), Tz::from_id(range.end_tz)) + else { + continue; + }; + let component_tz = start_tz; if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; } @@ -68,93 +815,111 @@ impl ArchivedCalendarEventData { end_tz = default_tz; } - if instances.len() > bytes_read { - // Recurring event - let unpacker = - BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); - for start_offset in unpacker { - let start_date_naive = start_offset as i64 + base_offset; - let end_date_naive = start_date_naive + duration; - let (Some(start), Some(end)) = ( - resolve_local(start_tz, start_date_naive), - resolve_local(end_tz, end_date_naive), - ) else { + let best_so_far = next_alarm.as_ref().map(|next| next.alarm_time); + for candidate in &mut candidates { + candidate.prepare(duration, default_tz, start_time, best_so_far); + } + let recurrence = self.component_recurrence(comp_id, component_tz, flags); + if !recurrence.is_recurrent() { + for candidate in &mut candidates { + candidate.is_pending &= !candidate.scope.is_occurrence(); + } + } + if !PendingAlarms::of(&candidates).any { + continue; + } + + let is_packed = instances.len() > bytes_read; + let packed = is_packed.then(|| { + BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count) + }); + let single = (!is_packed).then_some(offset_or_count); + + for start_date_naive in packed + .into_iter() + .flatten() + .chain(single) + .map(|offset| base_offset.saturating_add(offset as i64)) + { + let best_so_far = next_alarm.as_ref().map(|next| next.alarm_time); + let pending = PendingAlarms::of(&candidates); + if !pending.any || pending.is_past(start_date_naive, best_so_far) { + break; + } + + let end_date_naive = start_date_naive.saturating_add(duration); + let mut instance_key = None; + let mut resolved = None; + + for candidate in &mut candidates { + if !candidate.is_pending { continue; + } + if candidate.scope != AlarmScope::All { + let recurrence_key = *instance_key + .get_or_insert_with(|| recurrence.recurrence_key(start_date_naive)); + if !source.applies(candidate.scope, recurrence_key) { + if candidate.scope.is_behind(recurrence_key) { + candidate.is_pending = false; + } + continue; + } + } + if let Some(naive_bias) = candidate.naive_bias { + let naive_alarm = start_date_naive.saturating_add(naive_bias); + if naive_alarm.saturating_add(TZ_BOUND) <= start_time { + continue; + } + if best_so_far + .is_some_and(|best| naive_alarm.saturating_sub(TZ_BOUND) >= best) + { + candidate.is_pending = false; + continue; + } + } + + let (start, end) = match resolved { + Some(resolved) => resolved, + None => { + let (Some(start), Some(end)) = ( + flags.resolve_start(start_tz, start_date_naive), + flags.resolve_end(end_tz, end_date_naive), + ) else { + break; + }; + *resolved.insert((start, end)) + } }; - if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) + if let Some(alarm_time) = candidate.alarm_time(&start, &end, default_tz) && alarm_time > start_time - && next_alarm + { + if next_alarm .as_ref() .is_none_or(|next| alarm_time < next.alarm_time) - { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, - typ: if alarm.is_email_alert { - CalendarAlarmType::Email { - event_start: start_date_naive, - event_start_tz: start_tz.as_id(), - event_end: end_date_naive, - event_end_tz: end_tz.as_id(), - } - } else { - let comp = - &self.event.components[alarm.parent_id.to_native() as usize]; - - CalendarAlarmType::Display { - recurrence_id: if comp.is_recurrent_or_override() { - start_date_naive.into() - } else { - None - }, - } - }, - }); - continue 'outer; - } - } - } else { - // Single event - let start_date_naive = offset_or_count as i64 + base_offset; - let end_date_naive = start_date_naive + duration; - let (Some(start), Some(end)) = ( - resolve_local(start_tz, start_date_naive), - resolve_local(end_tz, end_date_naive), - ) else { - continue; - }; - - if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) - && alarm_time > start_time - && next_alarm - .as_ref() - .is_none_or(|next| alarm_time < next.alarm_time) - { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, - typ: if alarm.is_email_alert { - CalendarAlarmType::Email { - event_start: start_date_naive, - event_start_tz: start_tz.as_id(), - event_end: end_date_naive, - event_end_tz: end_tz.as_id(), - } - } else { - let comp = &self.event.components[alarm.parent_id.to_native() as usize]; - - CalendarAlarmType::Display { - recurrence_id: if comp.is_recurrent_or_override() { - start_date_naive.into() + { + next_alarm = Some(CalendarAlarm { + alarm_id: candidate.id, + event_id: comp_id, + alarm_time, + typ: if candidate.is_email_alert { + CalendarAlarmType::Email { + event_start: start.timestamp(), + event_start_tz: start_tz.as_id(), + event_end: end.timestamp(), + event_end_tz: end_tz.as_id(), + recurrence_id: recurrence.recurrence_id(start_date_naive), + } } else { - None + CalendarAlarmType::Display { + recurrence_id: recurrence.recurrence_id(start_date_naive), + } }, - } - }, - }); + target: AlarmTarget::Owner, + }); + } + candidate.is_pending = false; + } } } } @@ -163,6 +928,109 @@ impl ArchivedCalendarEventData { } } +impl EventAlarmData for ArchivedCalendarEventData { + fn alarm_base_offset(&self) -> i64 { + self.base_offset.to_native() + } + + fn alarm_ranges(&self) -> impl Iterator> { + self.time_ranges.iter().map(|range| AlarmRange { + comp_id: range.id.to_native(), + start_tz: range.start_tz.to_native(), + end_tz: range.end_tz.to_native(), + duration: range.duration.to_native() as i64, + flags: RangeFlags::from_bits(range.flags), + instances: range.instances.as_ref(), + }) + } + + fn stored_alarms(&self, comp_id: u16) -> impl Iterator { + self.alarms + .iter() + .filter(move |alarm| alarm.parent_id == comp_id) + .map(|alarm| Alarm { + id: alarm.id.to_native(), + parent_id: alarm.parent_id.to_native(), + delta: AlarmDelta::from(&alarm.delta), + flags: alarm.flags.to_native(), + }) + } + + fn has_stored_alarms(&self) -> bool { + !self.alarms.is_empty() + } + + fn has_floating_stored_alarms(&self) -> bool { + self.alarms + .iter() + .any(|alarm| matches!(alarm.delta, ArchivedAlarmDelta::FixedFloating(_))) + } + + fn component_recurrence( + &self, + comp_id: u16, + component_tz: Tz, + flags: RangeFlags, + ) -> AlarmRecurrence { + AlarmRecurrence::of( + self.event.components.get(comp_id as usize), + self.component_tz(u32::from(comp_id)) + .unwrap_or(component_tz), + component_tz, + flags, + ) + } +} + +impl EventAlarmData for CalendarEventData { + fn alarm_base_offset(&self) -> i64 { + self.base_offset + } + + fn alarm_ranges(&self) -> impl Iterator> { + self.time_ranges.iter().map(|range| AlarmRange { + comp_id: range.id, + start_tz: range.start_tz, + end_tz: range.end_tz, + duration: range.duration as i64, + flags: RangeFlags::from_bits(range.flags), + instances: range.instances.as_ref(), + }) + } + + fn stored_alarms(&self, comp_id: u16) -> impl Iterator { + self.alarms + .iter() + .filter(move |alarm| alarm.parent_id == comp_id) + .cloned() + } + + fn has_stored_alarms(&self) -> bool { + !self.alarms.is_empty() + } + + fn has_floating_stored_alarms(&self) -> bool { + self.alarms + .iter() + .any(|alarm| matches!(alarm.delta, AlarmDelta::FixedFloating(_))) + } + + fn component_recurrence( + &self, + comp_id: u16, + component_tz: Tz, + flags: RangeFlags, + ) -> AlarmRecurrence { + AlarmRecurrence::of( + self.event.components.get(comp_id as usize), + self.component_tz(u32::from(comp_id)) + .unwrap_or(component_tz), + component_tz, + flags, + ) + } +} + pub trait ExpandAlarm { fn expand_alarm(&self, id: u16, parent_id: u16) -> Option; } @@ -170,7 +1038,7 @@ pub trait ExpandAlarm { impl ExpandAlarm for ICalendarComponent { fn expand_alarm(&self, id: u16, parent_id: u16) -> Option { let mut trigger = None; - let mut is_email_alert = false; + let mut flags = 0; for entry in self.entries.iter() { match &entry.name { @@ -211,29 +1079,33 @@ impl ExpandAlarm for ICalendarComponent { } ICalendarValue::Duration(duration) => { if trigger_start { - Some(AlarmDelta::Start(duration.as_seconds())) + Some(AlarmDelta::Start(duration.into())) } else { - Some(AlarmDelta::End(duration.as_seconds())) + Some(AlarmDelta::End(duration.into())) } } _ => None, }; } ICalendarProperty::Action => { - is_email_alert = is_email_alert - || entry - .values - .first() - .and_then(|v| v.as_text()) - .is_some_and(|v| v.eq_ignore_ascii_case("email")); + if entry + .values + .first() + .and_then(|v| v.as_text()) + .is_some_and(|v| v.eq_ignore_ascii_case("email")) + { + flags |= ALARM_EMAIL; + } } - ICalendarProperty::Summary | ICalendarProperty::Description => { - is_email_alert = is_email_alert - || entry + ICalendarProperty::Summary | ICalendarProperty::Description + if !entry.is_derived() + && entry .values .first() .and_then(|v| v.as_text()) - .is_some_and(|v| v.contains("@email")); + .is_some_and(|v| v.contains("@email")) => + { + flags |= ALARM_EMAIL; } _ => {} } @@ -243,31 +1115,87 @@ impl ExpandAlarm for ICalendarComponent { id, parent_id, delta, - is_email_alert, + flags, }) } } impl AlarmDelta { - pub fn to_timestamp(&self, start: i64, end: i64, default_tz: Tz) -> Option { + pub fn to_timestamp( + &self, + start: &ZonedDateTime, + end: &ZonedDateTime, + default_tz: Tz, + ) -> Option { + match self { + AlarmDelta::Start(offset) => offset.apply(start), + AlarmDelta::End(offset) => offset.apply(end), + AlarmDelta::FixedUtc(_) | AlarmDelta::FixedFloating(_) => { + self.fixed_timestamp(default_tz) + } + } + } + + pub fn fixed_timestamp(&self, default_tz: Tz) -> Option { match self { - AlarmDelta::Start(delta) => Some(start + delta), - AlarmDelta::End(delta) => Some(end + delta), AlarmDelta::FixedUtc(timestamp) => Some(*timestamp), AlarmDelta::FixedFloating(timestamp) => resolve_local(default_tz, *timestamp), + AlarmDelta::Start(_) | AlarmDelta::End(_) => None, } } } -impl ArchivedAlarmDelta { - pub fn to_timestamp(&self, start: i64, end: i64, default_tz: Tz) -> Option { - match self { - ArchivedAlarmDelta::Start(delta) => Some(start + delta.to_native()), - ArchivedAlarmDelta::End(delta) => Some(end + delta.to_native()), - ArchivedAlarmDelta::FixedUtc(timestamp) => Some(timestamp.to_native()), +impl From<&ArchivedAlarmDelta> for AlarmDelta { + fn from(delta: &ArchivedAlarmDelta) -> Self { + match delta { + ArchivedAlarmDelta::Start(offset) => AlarmDelta::Start(offset.into()), + ArchivedAlarmDelta::End(offset) => AlarmDelta::End(offset.into()), + ArchivedAlarmDelta::FixedUtc(timestamp) => AlarmDelta::FixedUtc(timestamp.to_native()), ArchivedAlarmDelta::FixedFloating(timestamp) => { - resolve_local(default_tz, timestamp.to_native()) + AlarmDelta::FixedFloating(timestamp.to_native()) } } } } + +impl AlarmOffset { + pub fn naive_seconds(&self) -> i64 { + i64::from(self.days) * SECONDS_PER_DAY + i64::from(self.seconds) + } + + pub fn apply(&self, from: &ZonedDateTime) -> Option { + from.checked_add_nominal(NominalDuration::new(self.days, 0)) + .map(|at| at.timestamp() + i64::from(self.seconds)) + } +} + +impl From<&ICalendarDuration> for AlarmOffset { + fn from(duration: &ICalendarDuration) -> Self { + let days = i32::try_from(u64::from(duration.weeks) * 7 + u64::from(duration.days)) + .unwrap_or(i32::MAX); + let seconds = i32::try_from( + u64::from(duration.hours) * 3600 + + u64::from(duration.minutes) * 60 + + u64::from(duration.seconds), + ) + .unwrap_or(i32::MAX); + if duration.neg { + AlarmOffset { + days: -days, + seconds: -seconds, + } + } else { + AlarmOffset { days, seconds } + } + } +} + +impl From<&ArchivedAlarmOffset> for AlarmOffset { + fn from(offset: &ArchivedAlarmOffset) -> Self { + AlarmOffset { + days: offset.days.to_native(), + seconds: offset.seconds.to_native(), + } + } +} + diff --git a/crates/groupware/src/calendar/alerts.rs b/crates/groupware/src/calendar/alerts.rs new file mode 100644 index 00000000..45c22d4c --- /dev/null +++ b/crates/groupware/src/calendar/alerts.rs @@ -0,0 +1,882 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::alarm::{AlarmTarget, CalendarComponentView, CalendarView}; +use super::schedule::{AlarmTasks, EventAlarmContent, EventAlarmScheduler, EventAlarmUsers}; +use super::{ + ALERT_EMAIL, ALERT_RELATIVE_TO_END, ALERT_WITH_TIME, AlarmDelta, ArchivedCalendar, + CALENDAR_SUBSCRIBED, Calendar, CalendarEventContent, DefaultAlert, EVENT_HAS_ALARMS, + EVENT_USES_DEFAULT_ALERTS, default_preference_flags, +}; +use calcard::{ + common::timezone::Tz, + icalendar::{ + ICalendar, ICalendarAction, ICalendarComponent, ICalendarComponentType, ICalendarEntry, + ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, + ICalendarRelated, ICalendarRelationshipType, ICalendarValue, Uri, + }, +}; +use common::{GroupwareResources, Server, storage::batch::MAX_BATCH_DOCUMENTS}; +use std::{collections::hash_map::Entry, sync::Arc}; +use store::{ + ValueKey, + ahash::{AHashMap, AHashSet}, + write::{Archive, ArchiveBytes, BatchBuilder, now}, +}; +use trc::AddContext; +use types::{collection::Collection, field::CalendarEventField}; + +const MAX_DEFAULT_ALERTS: usize = 1024; +const DEFAULT_ALARM_DESCRIPTION: &str = "Reminder"; +const MAILTO_PREFIX: &str = "mailto:"; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct DefaultAlerts { + enabled: bool, + alerts: Vec, + recipient: Option>, +} + +#[derive(Debug, Clone, Copy)] +pub struct DefaultAlarmText<'x> { + pub description: &'x str, + pub recipient: Option<&'x str>, +} + +impl DefaultAlerts { + pub fn disabled() -> Self { + DefaultAlerts::default() + } + + pub(crate) fn merge<'x>( + calendars: impl Iterator, + personal_id: u32, + inherit_from: Option, + with_time: bool, + ) -> Self { + let mut alert_ids = AHashSet::new(); + DefaultAlerts { + enabled: true, + alerts: calendars + .filter_map(|settings| settings.user_or_inherited(personal_id, inherit_from)) + .flat_map(|settings| settings.alerts.iter()) + .filter(|alert| { + (alert.flags & ALERT_WITH_TIME != 0) == with_time + && alert_ids.insert(alert.id.as_str()) + }) + .take(MAX_DEFAULT_ALERTS) + .cloned() + .collect(), + recipient: None, + } + } + + pub fn is_enabled(&self) -> bool { + self.enabled + } + + pub fn has_alerts(&self) -> bool { + !self.alerts.is_empty() + } + + pub fn has_email_alerts(&self) -> bool { + self.alerts.iter().any(DefaultAlert::is_email) + } + + pub fn get(&self, index: u16) -> Option<&DefaultAlert> { + self.alerts.get(index as usize) + } + + pub fn contains_id(&self, id: &str) -> bool { + self.alerts.iter().any(|alert| alert.id == id) + } + + pub fn indexed_alerts(&self) -> impl Iterator { + self.alerts + .iter() + .zip(0u16..) + .map(|(alert, index)| (index, alert)) + } +} + +impl DefaultAlert { + pub fn is_email(&self) -> bool { + self.flags & ALERT_EMAIL != 0 + } + + pub fn delta(&self) -> AlarmDelta { + if self.flags & ALERT_RELATIVE_TO_END != 0 { + AlarmDelta::End((&self.offset).into()) + } else { + AlarmDelta::Start((&self.offset).into()) + } + } + + pub fn to_ical(&self, text: Option>) -> ICalendarComponent { + let mut entries = Vec::with_capacity(6); + entries.push( + ICalendarEntry::new(ICalendarProperty::Jsid) + .with_value(ICalendarValue::Text(self.id.clone())), + ); + entries.push(ICalendarEntry::new(ICalendarProperty::Action).with_value( + if self.is_email() { + ICalendarValue::Action(ICalendarAction::Email) + } else { + ICalendarValue::Action(ICalendarAction::Display) + }, + )); + entries.push( + ICalendarEntry::new(ICalendarProperty::Trigger) + .with_param_opt((self.flags & ALERT_RELATIVE_TO_END != 0).then_some( + ICalendarParameter::related(ICalendarParameterValue::Related( + ICalendarRelated::End, + )), + )) + .with_value(ICalendarValue::Duration(self.offset.clone())), + ); + if let Some(text) = text { + entries.push( + ICalendarEntry::new(ICalendarProperty::Description).with_value(text.description), + ); + if self.is_email() { + entries.push( + ICalendarEntry::new(ICalendarProperty::Summary).with_value(text.description), + ); + if let Some(recipient) = text.recipient { + let mut address = String::with_capacity(MAILTO_PREFIX.len() + recipient.len()); + address.push_str(MAILTO_PREFIX); + address.push_str(recipient); + entries.push( + ICalendarEntry::new(ICalendarProperty::Attendee) + .with_value(Uri::Location(address)), + ); + } + } + } + + ICalendarComponent { + component_type: ICalendarComponentType::VAlarm, + entries, + component_ids: vec![], + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct CalendarSettings(Vec); + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct UserCalendarSettings { + pub account_id: u32, + pub is_subscribed: bool, + pub time_zone: Tz, + pub alerts: Vec, +} + +impl UserCalendarSettings { + fn differs_only_in_alerts(&self, other: &Self) -> bool { + self.is_subscribed == other.is_subscribed && self.time_zone == other.time_zone + } +} + +impl CalendarSettings { + pub fn user(&self, account_id: u32) -> Option<&UserCalendarSettings> { + self.0 + .iter() + .find(|settings| settings.account_id == account_id) + } + + fn user_or_inherited( + &self, + account_id: u32, + inherit_from: Option, + ) -> Option<&UserCalendarSettings> { + self.user(account_id) + .or_else(|| inherit_from.and_then(|account_id| self.user(account_id))) + } + + fn changes<'x>(&'x self, other: &'x Self) -> impl Iterator + 'x { + self.0 + .iter() + .map(|settings| settings.account_id) + .chain( + other + .0 + .iter() + .map(|settings| settings.account_id) + .filter(|account_id| self.user(*account_id).is_none()), + ) + .filter_map(move |account_id| { + let previous = self.user(account_id); + let current = other.user(account_id); + (previous != current).then(|| { + ( + account_id, + previous.zip(current).is_some_and(|(previous, current)| { + previous.differs_only_in_alerts(current) + }), + ) + }) + }) + } + + async fn fetch(server: &Server, account_id: u32, calendar_id: u32) -> trc::Result { + server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + calendar_id, + )) + .await + .caused_by(trc::location!())? + .map(|archive| archive.unarchive::().map(CalendarSettings::from)) + .transpose() + .map(Option::unwrap_or_default) + .caused_by(trc::location!()) + } +} + +impl From<&Calendar> for CalendarSettings { + fn from(calendar: &Calendar) -> Self { + CalendarSettings( + calendar + .preferences + .iter() + .map(|preferences| UserCalendarSettings { + account_id: preferences.account_id, + is_subscribed: preferences.flags & CALENDAR_SUBSCRIBED != 0, + time_zone: preferences.time_zone.tz().unwrap_or_default(), + alerts: preferences.default_alerts.clone(), + }) + .collect(), + ) + } +} + +impl From<&ArchivedCalendar> for CalendarSettings { + fn from(calendar: &ArchivedCalendar) -> Self { + CalendarSettings( + calendar + .preferences + .iter() + .map(|preferences| UserCalendarSettings { + account_id: preferences.account_id.to_native(), + is_subscribed: preferences.flags.to_native() & CALENDAR_SUBSCRIBED != 0, + time_zone: preferences.time_zone.tz().unwrap_or_default(), + alerts: preferences + .default_alerts + .iter() + .map(|alert| DefaultAlert { + id: alert.id.to_string(), + offset: alert.offset.to_native(), + flags: alert.flags.to_native(), + }) + .collect(), + }) + .collect(), + ) + } +} + +#[derive(Default)] +pub struct DefaultAlertsResolver { + calendars: AHashMap<(u32, u32), CalendarSettings>, + resources: Option<(u32, Arc)>, + recipients: AHashMap>>, + members: AHashMap<(u32, u32), bool>, +} + +impl DefaultAlertsResolver { + pub fn with_resources(account_id: u32, resources: Arc) -> Self { + DefaultAlertsResolver { + resources: Some((account_id, resources)), + ..Default::default() + } + } + + pub fn with_calendar( + mut self, + account_id: u32, + calendar_id: u32, + settings: CalendarSettings, + ) -> Self { + self.set_calendar_settings(account_id, calendar_id, settings); + self + } + + pub fn set_calendar_settings( + &mut self, + account_id: u32, + calendar_id: u32, + settings: CalendarSettings, + ) { + self.calendars.insert((account_id, calendar_id), settings); + } + + async fn settings( + &mut self, + server: &Server, + account_id: u32, + calendar_id: u32, + ) -> trc::Result<&CalendarSettings> { + Ok(match self.calendars.entry((account_id, calendar_id)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + entry.insert(CalendarSettings::fetch(server, account_id, calendar_id).await?) + } + }) + } + + fn cached_subscription( + &self, + account_id: u32, + personal_id: u32, + calendar_id: u32, + ) -> Option> { + if let Some(settings) = self.calendars.get(&(account_id, calendar_id)) { + return Some( + settings + .user(personal_id) + .map(|settings| settings.is_subscribed), + ); + } + self.resources + .as_ref() + .filter(|(resources_account_id, _)| *resources_account_id == account_id) + .and_then(|(_, resources)| resources.container_resource_by_id(calendar_id)) + .map(|calendar| { + calendar + .personal_calendar_preferences(personal_id) + .map(|preferences| preferences.flags & CALENDAR_SUBSCRIBED != 0) + }) + } + + async fn is_member( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + ) -> trc::Result { + if personal_id == account_id { + return Ok(true); + } + Ok(match self.members.entry((account_id, personal_id)) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => *entry.insert( + server + .account(personal_id) + .await + .caused_by(trc::location!())? + .id_member_of + .contains(&account_id), + ), + }) + } + + pub async fn is_subscribed( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + calendar_ids: impl IntoIterator, + ) -> trc::Result { + let mut default_subscription = None; + for calendar_id in calendar_ids { + let is_subscribed = match self.cached_subscription(account_id, personal_id, calendar_id) + { + Some(is_subscribed) => is_subscribed, + None => self + .settings(server, account_id, calendar_id) + .await? + .user(personal_id) + .map(|settings| settings.is_subscribed), + }; + let is_subscribed = match is_subscribed { + Some(is_subscribed) => is_subscribed, + None => match default_subscription { + Some(is_subscribed) => is_subscribed, + None => { + let is_member = self.is_member(server, account_id, personal_id).await?; + *default_subscription + .insert(default_preference_flags(is_member) & CALENDAR_SUBSCRIBED != 0) + } + }, + }; + if is_subscribed { + return Ok(true); + } + } + Ok(false) + } + + pub async fn time_zone( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + calendar_ids: impl IntoIterator, + ) -> trc::Result { + let inherit_from = self + .is_member(server, account_id, personal_id) + .await? + .then_some(account_id); + for calendar_id in calendar_ids { + if let Some(time_zone) = self + .settings(server, account_id, calendar_id) + .await? + .user_or_inherited(personal_id, inherit_from) + .map(|settings| settings.time_zone) + .filter(|time_zone| !time_zone.is_floating()) + { + return Ok(time_zone); + } + } + Ok(Tz::Floating) + } + + pub async fn resolve( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + use_default_alerts: bool, + calendar_ids: impl IntoIterator + Clone, + with_time: bool, + ) -> trc::Result { + if !use_default_alerts { + return Ok(DefaultAlerts::disabled()); + } + + for calendar_id in calendar_ids.clone() { + self.settings(server, account_id, calendar_id).await?; + } + let inherit_from = self + .is_member(server, account_id, personal_id) + .await? + .then_some(account_id); + + Ok(DefaultAlerts::merge( + calendar_ids + .into_iter() + .filter_map(|calendar_id| self.calendars.get(&(account_id, calendar_id))), + personal_id, + inherit_from, + with_time, + )) + } + + async fn recipient( + &mut self, + server: &Server, + personal_id: u32, + ) -> trc::Result>> { + Ok(match self.recipients.entry(personal_id) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let account = server + .account(personal_id) + .await + .caused_by(trc::location!())?; + entry + .insert((!account.name.is_empty()).then(|| Arc::from(account.name.as_ref()))) + .clone() + } + }) + } +} + +impl DefaultAlertsResolver { + pub async fn resolve_for_content( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + content: &C, + calendar_ids: impl IntoIterator + Clone, + ) -> trc::Result { + self.resolve( + server, + account_id, + personal_id, + content.uses_default_alerts(personal_id), + calendar_ids, + !content.shows_without_time(), + ) + .await + } + + pub async fn resolve_for_ical_view( + &mut self, + server: &Server, + account_id: u32, + personal_id: u32, + content: &C, + calendar_ids: impl IntoIterator + Clone, + ) -> trc::Result { + let mut default_alerts = self + .resolve_for_content(server, account_id, personal_id, content, calendar_ids) + .await?; + if default_alerts.has_email_alerts() { + default_alerts.recipient = self.recipient(server, personal_id).await?; + } + Ok(default_alerts) + } +} + +pub trait CalendarAlarmsReschedule: Sync + Send { + #[allow(clippy::too_many_arguments)] + fn reschedule_calendar_alarms( + &self, + resources: &Arc, + account_id: u32, + calendar_id: u32, + previous: CalendarSettings, + current: CalendarSettings, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; +} + +impl CalendarAlarmsReschedule for Server { + async fn reschedule_calendar_alarms( + &self, + resources: &Arc, + account_id: u32, + calendar_id: u32, + previous: CalendarSettings, + current: CalendarSettings, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let mut targets = Vec::new(); + let mut required_flags = 0; + for (changed_id, differs_only_in_alerts) in previous.changes(¤t) { + targets.push(AlarmTarget::for_account(account_id, changed_id)); + required_flags |= if differs_only_in_alerts { + EVENT_USES_DEFAULT_ALERTS + } else { + EVENT_HAS_ALARMS + }; + } + if targets.is_empty() { + return Ok(()); + } + + let mut previous_resolver = DefaultAlertsResolver::with_resources( + account_id, + resources.clone(), + ) + .with_calendar(account_id, calendar_id, previous); + let mut current_resolver = DefaultAlertsResolver::with_resources( + account_id, + resources.clone(), + ) + .with_calendar(account_id, calendar_id, current); + let now = now() as i64; + let mut calendar_ids = Vec::new(); + + let mut children = resources + .children(calendar_id) + .filter_map(|resource| { + resource + .resource + .event_flags() + .filter(|flags| flags & required_flags != 0) + .map(|flags| (resource.document_id(), flags, resource.resource)) + }) + .collect::>(); + children.sort_unstable_by_key(|(document_id, ..)| *document_id); + let mut content_ids = Vec::new(); + + for chunk in children.chunks(MAX_BATCH_DOCUMENTS) { + content_ids.clear(); + content_ids.extend(chunk.iter().map(|(document_id, ..)| *document_id)); + let mut contents = self + .archive_chunk( + account_id, + Collection::CalendarEvent, + &[], + Some((CalendarEventField::Content.field(), &content_ids)), + ) + .await + .caused_by(trc::location!())?; + let mut tasks = AlarmTasks::pending( + content_ids + .iter() + .flat_map(|document_id| { + targets + .iter() + .map(|target| target.task_id().resolve(account_id, *document_id)) + }) + .collect(), + ); + + for &(document_id, event_flags, resource) in chunk { + let Some(content_) = contents.take_value(document_id) else { + continue; + }; + let content = content_ + .unarchive::() + .caused_by(trc::location!())?; + calendar_ids.clear(); + calendar_ids.extend(resource.child_names().iter().map(|name| name.parent_id)); + + for target in targets.iter().copied() { + let users = EventAlarmUsers::for_target(account_id, content, target)? + .with_event_flags(event_flags); + let previous_alarm = self + .next_event_alarms( + account_id, + &users, + &content.data, + &calendar_ids, + now, + &mut previous_resolver, + ) + .await? + .into_alarm(); + let next_alarm = self + .next_event_alarms( + account_id, + &users, + &content.data, + &calendar_ids, + now, + &mut current_resolver, + ) + .await? + .into_alarm(); + self.replace_user_alarm( + account_id, + document_id, + previous_alarm, + next_alarm, + &mut tasks, + batch, + ) + .await?; + } + } + } + + Ok(()) + } +} + +pub trait ICalendarShowWithoutTime { + fn shows_without_time(&self) -> bool; +} + +impl ICalendarShowWithoutTime for T { + fn shows_without_time(&self) -> bool { + self.main_component() + .is_some_and(CalendarComponentView::is_shown_without_time) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DefaultAlertsView { + Replace, + Merge, + ICalendar, +} + +pub trait ICalendarDefaultAlerts { + fn apply_default_alerts(&mut self, defaults: &DefaultAlerts, view: DefaultAlertsView); + + fn strip_default_alerts(&mut self, defaults: &DefaultAlerts); + + fn is_kept_with_default_alerts( + &self, + alarm: &ICalendarComponent, + defaults: &DefaultAlerts, + ) -> bool; + + fn restore_hidden_alerts(&mut self, previous: &ICalendar, defaults: &DefaultAlerts); +} + +impl ICalendarDefaultAlerts for ICalendar { + fn apply_default_alerts(&mut self, defaults: &DefaultAlerts, view: DefaultAlertsView) { + if !defaults.is_enabled() { + return; + } + + for comp_id in 0..self.components.len() { + let Some(component) = self.components.get(comp_id) else { + continue; + }; + if !component.component_type.is_event_or_todo() { + continue; + } + + let text = (view == DefaultAlertsView::ICalendar).then(|| DefaultAlarmText { + description: component + .property(&ICalendarProperty::Summary) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_text()) + .filter(|summary| !summary.is_empty()) + .unwrap_or(DEFAULT_ALARM_DESCRIPTION), + recipient: defaults.recipient.as_deref(), + }); + let mut stored_defaults: Vec<(&str, &[ICalendarEntry])> = Vec::new(); + let mut kept_ids = Vec::new(); + let mut removed_ids = Vec::new(); + for alarm_id in &component.component_ids { + let Some(alarm) = self + .components + .get(*alarm_id as usize) + .filter(|c| c.component_type == ICalendarComponentType::VAlarm) + else { + continue; + }; + removed_ids.push(*alarm_id); + if let Some(jsid) = alarm.jsid().filter(|id| defaults.contains_id(id)) { + stored_defaults.push((jsid, &alarm.entries)); + } else if view == DefaultAlertsView::Merge + || self.is_kept_with_default_alerts(alarm, defaults) + { + kept_ids.push(*alarm_id); + } + } + + let mut new_alarms = Vec::with_capacity(defaults.alerts.len()); + for alert in &defaults.alerts { + let mut alarm = alert.to_ical(text); + if let Some((_, entries)) = stored_defaults.iter().find(|(id, _)| *id == alert.id) { + let generated = alarm.entries.len(); + for entry in entries.iter() { + if !alarm + .entries + .iter() + .take(generated) + .any(|generated| generated.name == entry.name) + { + alarm.entries.push(entry.clone()); + } + } + } + new_alarms.push(alarm); + } + + let mut new_ids = Vec::with_capacity(new_alarms.len()); + for alarm in new_alarms { + new_ids.push(self.components.len() as u32); + self.components.push(alarm); + } + + if let Some(component) = self.components.get_mut(comp_id) { + component + .component_ids + .retain(|id| !removed_ids.contains(id) || kept_ids.contains(id)); + component.component_ids.extend(new_ids); + } + } + } + + fn strip_default_alerts(&mut self, defaults: &DefaultAlerts) { + if !defaults.is_enabled() { + return; + } + + let redundant = (0u32..) + .zip(&self.components) + .filter(|(_, c)| { + c.component_type == ICalendarComponentType::VAlarm + && c.jsid().is_some_and(|id| defaults.contains_id(id)) + && !c.has_property(&ICalendarProperty::Acknowledged) + }) + .map(|(id, _)| id) + .collect::>(); + + if !redundant.is_empty() && !self.remove_component_ids(&redundant) { + for component in &mut self.components { + component.component_ids.retain(|id| !redundant.contains(id)); + } + } + } + + fn is_kept_with_default_alerts( + &self, + alarm: &ICalendarComponent, + defaults: &DefaultAlerts, + ) -> bool { + alarm.snoozed_alarms().any(|parent| { + defaults.contains_id(parent) + || self.components.iter().any(|c| { + c.component_type == ICalendarComponentType::VAlarm + && c.uid() == Some(parent) + && c.jsid().is_some_and(|id| defaults.contains_id(id)) + }) + }) + } + fn restore_hidden_alerts(&mut self, previous: &ICalendar, defaults: &DefaultAlerts) { + if !defaults.is_enabled() { + return; + } + + for comp_id in 0..self.components.len() { + let Some(component) = self + .components + .get(comp_id) + .filter(|c| c.component_type.is_event_or_todo()) + else { + continue; + }; + let recurrence_id = component.property(&ICalendarProperty::RecurrenceId); + let Some(previous_component) = previous.components.iter().find(|c| { + c.component_type == component.component_type + && c.property(&ICalendarProperty::RecurrenceId) == recurrence_id + }) else { + continue; + }; + let hidden = previous_component + .component_ids + .iter() + .filter_map(|id| previous.components.get(*id as usize)) + .filter(|alarm| { + alarm.component_type == ICalendarComponentType::VAlarm + && alarm.jsid().is_none_or(|id| !defaults.contains_id(id)) + && !previous.is_kept_with_default_alerts(alarm, defaults) + && !component + .component_ids + .iter() + .filter_map(|id| self.components.get(*id as usize)) + .any(|existing| existing == *alarm) + }) + .cloned() + .collect::>(); + + let first_id = self.components.len() as u32; + let new_ids = first_id..first_id + hidden.len() as u32; + self.components.extend(hidden); + if let Some(component) = self.components.get_mut(comp_id) { + component.component_ids.extend(new_ids); + } + } + } +} + +pub trait SnoozeAlarm { + fn snoozed_alarms(&self) -> impl Iterator; +} + +impl SnoozeAlarm for ICalendarComponent { + fn snoozed_alarms(&self) -> impl Iterator { + self.entries + .iter() + .filter(|entry| { + entry.name == ICalendarProperty::RelatedTo + && entry.params.iter().any(|param| { + param.name == ICalendarParameterName::Reltype + && matches!( + param.value, + ICalendarParameterValue::Reltype(ICalendarRelationshipType::Snooze) + ) + }) + }) + .filter_map(|entry| entry.values.first().and_then(|value| value.as_text())) + } +} + diff --git a/crates/groupware/src/calendar/availability.rs b/crates/groupware/src/calendar/availability.rs new file mode 100644 index 00000000..c7d123aa --- /dev/null +++ b/crates/groupware/src/calendar/availability.rs @@ -0,0 +1,423 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CALENDAR_AVAILABILITY_ALL, CALENDAR_AVAILABILITY_ATTENDING, CALENDAR_AVAILABILITY_NONE, + CALENDAR_SUBSCRIBED, default_preference_flags, identity::CalendarAddresses, +}; +use calcard::{ + common::timezone::Tz, + icalendar::{ + ArchivedICalendarComponent, ArchivedICalendarParameterValue, + ArchivedICalendarParticipationStatus, ArchivedICalendarProperty, ArchivedICalendarStatus, + ArchivedICalendarTransparency, ArchivedICalendarValue, ICalendarParameterName, + }, +}; +use common::{CachedName, GroupwareResources, auth::AccessToken}; +use store::{ahash::AHashMap, roaring::RoaringBitmap}; +use types::acl::Acl; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AvailabilityPreference { + All, + Attending, + None, +} + +impl AvailabilityPreference { + pub fn from_flags(flags: u16) -> Option { + if flags & CALENDAR_AVAILABILITY_ALL != 0 { + Some(AvailabilityPreference::All) + } else if flags & CALENDAR_AVAILABILITY_ATTENDING != 0 { + Some(AvailabilityPreference::Attending) + } else if flags & CALENDAR_AVAILABILITY_NONE != 0 { + Some(AvailabilityPreference::None) + } else { + None + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrincipalRole { + Owner, + Sharee, +} + +impl PrincipalRole { + pub fn of(principal: &AccessToken, account_id: u32) -> Self { + if principal.is_member(account_id) { + PrincipalRole::Owner + } else { + PrincipalRole::Sharee + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Inclusion { + Attending, + All, +} + +impl Inclusion { + pub fn from_flags(flags: u16, role: PrincipalRole) -> Option { + if flags & CALENDAR_SUBSCRIBED == 0 { + return None; + } + match (AvailabilityPreference::from_flags(flags), role) { + (Some(AvailabilityPreference::All), _) | (None, PrincipalRole::Owner) => { + Some(Inclusion::All) + } + (Some(AvailabilityPreference::Attending), _) => Some(Inclusion::Attending), + (Some(AvailabilityPreference::None), _) | (None, PrincipalRole::Sharee) => None, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct RelevantCalendar { + pub inclusion: Inclusion, + pub tz: Tz, +} + +impl RelevantCalendar { + pub fn includes(&self, busy: &ComponentBusy) -> bool { + self.inclusion == Inclusion::All || busy.is_attending() + } +} + +enum CalendarAccess { + All, + Shared(RoaringBitmap), +} + +impl CalendarAccess { + fn contains(&self, calendar_id: u32) -> bool { + match self { + CalendarAccess::All => true, + CalendarAccess::Shared(calendar_ids) => calendar_ids.contains(calendar_id), + } + } +} + +pub struct AvailabilityCalendars<'x> { + resources: &'x GroupwareResources, + access_token: &'x AccessToken, + account_id: u32, + principal_id: u32, + role: PrincipalRole, + free_busy: CalendarAccess, + subscribable: CalendarAccess, + readable: Option, + relevance: AHashMap>, +} + +impl<'x> AvailabilityCalendars<'x> { + pub fn new( + resources: &'x GroupwareResources, + account_id: u32, + access_token: &'x AccessToken, + principal: &AccessToken, + ) -> Option { + let free_busy = Self::free_busy_access(resources, account_id, access_token)?; + let role = PrincipalRole::of(principal, account_id); + let subscribable = match role { + PrincipalRole::Owner => CalendarAccess::All, + PrincipalRole::Sharee => CalendarAccess::Shared(resources.shared_containers( + principal, + [Acl::Read, Acl::ReadItems], + true, + )), + }; + Some(Self::build( + resources, + account_id, + access_token, + principal.account_id(), + role, + free_busy, + subscribable, + )) + } + + pub fn owner( + resources: &'x GroupwareResources, + account_id: u32, + access_token: &'x AccessToken, + ) -> Option { + let free_busy = Self::free_busy_access(resources, account_id, access_token)?; + Some(Self::build( + resources, + account_id, + access_token, + account_id, + PrincipalRole::Owner, + free_busy, + CalendarAccess::All, + )) + } + + fn free_busy_access( + resources: &GroupwareResources, + account_id: u32, + access_token: &AccessToken, + ) -> Option { + if access_token.is_member(account_id) { + Some(CalendarAccess::All) + } else { + let calendar_ids = resources.shared_containers( + access_token, + [Acl::ReadItems, Acl::SchedulingReadFreeBusy], + true, + ); + (!calendar_ids.is_empty()).then_some(CalendarAccess::Shared(calendar_ids)) + } + } + + fn build( + resources: &'x GroupwareResources, + account_id: u32, + access_token: &'x AccessToken, + principal_id: u32, + role: PrincipalRole, + free_busy: CalendarAccess, + subscribable: CalendarAccess, + ) -> Self { + AvailabilityCalendars { + resources, + access_token, + account_id, + principal_id, + role, + free_busy, + subscribable, + readable: None, + relevance: AHashMap::new(), + } + } + + pub fn role(&self) -> PrincipalRole { + self.role + } + + pub fn relevant_calendar(&mut self, names: &[CachedName]) -> Option { + names + .iter() + .filter_map(|name| self.calendar_relevance(name.parent_id)) + .reduce(|best, calendar| { + if calendar.inclusion > best.inclusion { + calendar + } else { + best + } + }) + } + + fn calendar_relevance(&mut self, calendar_id: u32) -> Option { + if let Some(relevance) = self.relevance.get(&calendar_id) { + return *relevance; + } + let relevance = self.evaluate(calendar_id); + self.relevance.insert(calendar_id, relevance); + relevance + } + + fn evaluate(&self, calendar_id: u32) -> Option { + if !self.free_busy.contains(calendar_id) || !self.subscribable.contains(calendar_id) { + return None; + } + let calendar = self.resources.container_resource_by_id(calendar_id)?; + let (flags, tz) = match calendar.personal_calendar_preferences(self.principal_id) { + Some(preferences) => (preferences.flags, preferences.tz), + None if self.role == PrincipalRole::Owner => ( + default_preference_flags(true), + calendar.calendar_preferences(self.principal_id)?.tz, + ), + None => return None, + }; + Inclusion::from_flags(flags, self.role).map(|inclusion| RelevantCalendar { inclusion, tz }) + } + + pub fn may_read_items(&mut self, names: &[CachedName]) -> bool { + let AvailabilityCalendars { + resources, + access_token, + account_id, + readable, + .. + } = self; + let readable = readable.get_or_insert_with(|| { + if access_token.is_member(*account_id) { + CalendarAccess::All + } else { + CalendarAccess::Shared(resources.shared_containers( + access_token, + [Acl::ReadItems], + true, + )) + } + }); + names.iter().any(|name| readable.contains(name.parent_id)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BusyLevel { + Confirmed, + Tentative, + Unavailable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Participation { + Attendee(ParticipationStatus), + Organizer, + None, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ParticipationStatus { + Accepted, + Tentative, + Declined, + Other, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EventStatus { + Confirmed, + Tentative, + Other, +} + +#[derive(Debug, Clone, Copy)] +pub struct ComponentBusy { + status: EventStatus, + participation: Participation, + has_attendees: bool, + is_recurrent_or_override: bool, +} + +impl ComponentBusy { + pub fn parse( + component: &ArchivedICalendarComponent, + identities: &CalendarAddresses, + role: PrincipalRole, + ) -> Option { + if !component.component_type.is_event() { + return None; + } + let is_principal = |value: &ArchivedICalendarValue| { + value + .as_text() + .is_some_and(|address| identities.contains(address)) + }; + let mut busy = ComponentBusy { + status: EventStatus::Confirmed, + participation: Participation::None, + has_attendees: false, + is_recurrent_or_override: false, + }; + + for entry in component.entries.iter() { + match (&entry.name, entry.values.first()) { + ( + ArchivedICalendarProperty::Status, + Some(ArchivedICalendarValue::Status(status)), + ) => { + busy.status = match status { + ArchivedICalendarStatus::Cancelled => return None, + ArchivedICalendarStatus::Confirmed => EventStatus::Confirmed, + ArchivedICalendarStatus::Tentative => EventStatus::Tentative, + _ => EventStatus::Other, + }; + } + ( + ArchivedICalendarProperty::Transp, + Some(ArchivedICalendarValue::Transparency( + ArchivedICalendarTransparency::Transparent, + )), + ) if role == PrincipalRole::Owner => { + return None; + } + (ArchivedICalendarProperty::Organizer, Some(value)) => { + if busy.participation == Participation::None && is_principal(value) { + busy.participation = Participation::Organizer; + } + } + (ArchivedICalendarProperty::Attendee, Some(value)) => { + busy.has_attendees = true; + if is_principal(value) { + busy.participation = Participation::Attendee( + match entry.parameters(&ICalendarParameterName::Partstat).next() { + Some(ArchivedICalendarParameterValue::Partstat( + ArchivedICalendarParticipationStatus::Accepted, + )) => ParticipationStatus::Accepted, + Some(ArchivedICalendarParameterValue::Partstat( + ArchivedICalendarParticipationStatus::Tentative, + )) => ParticipationStatus::Tentative, + Some(ArchivedICalendarParameterValue::Partstat( + ArchivedICalendarParticipationStatus::Declined, + )) => ParticipationStatus::Declined, + _ => ParticipationStatus::Other, + }, + ); + } + } + ( + ArchivedICalendarProperty::Rrule + | ArchivedICalendarProperty::Rdate + | ArchivedICalendarProperty::RecurrenceId, + _, + ) => { + busy.is_recurrent_or_override = true; + } + _ => (), + } + } + + Some(busy) + } + + pub fn is_attending(&self) -> bool { + matches!( + self.participation, + Participation::Organizer + | Participation::Attendee( + ParticipationStatus::Accepted | ParticipationStatus::Tentative + ) + ) + } + + pub fn is_recurrent_or_override(&self) -> bool { + self.is_recurrent_or_override + } + + fn participation_status(&self) -> ParticipationStatus { + match self.participation { + Participation::Attendee(status) => status, + Participation::Organizer => ParticipationStatus::Accepted, + Participation::None if !self.has_attendees => ParticipationStatus::Accepted, + Participation::None => ParticipationStatus::Other, + } + } + + pub fn busy_level(&self) -> BusyLevel { + let participation_status = self.participation_status(); + if self.status == EventStatus::Tentative + || participation_status == ParticipationStatus::Tentative + { + BusyLevel::Tentative + } else if self.status == EventStatus::Confirmed + && participation_status == ParticipationStatus::Accepted + { + BusyLevel::Confirmed + } else { + BusyLevel::Unavailable + } + } +} diff --git a/crates/groupware/src/calendar/color.rs b/crates/groupware/src/calendar/color.rs new file mode 100644 index 00000000..c01710d7 --- /dev/null +++ b/crates/groupware/src/calendar/color.rs @@ -0,0 +1,236 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::{self, Display}; + +pub const MAX_OTHER_COLOR_LEN: usize = 32; + +macro_rules! css_colors { + ($($name:tt => $variant:ident),* $(,)?) => { + #[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Hash, + )] + #[rkyv(compare(PartialEq), derive(Debug))] + pub enum CssColor { + $($variant,)* + Rgb(u32), + Other(String), + } + + impl CssColor { + fn named(value: &str) -> Option { + hashify::fnc_map_ignore_case!( + value.as_bytes(), + $($name => Some(CssColor::$variant),)* + _ => None + ) + } + + fn as_named_str(&self) -> Option<&'static str> { + match self { + $(CssColor::$variant => Some($name),)* + CssColor::Rgb(_) | CssColor::Other(_) => None, + } + } + } + }; +} + +css_colors!( + "aliceblue" => Aliceblue, + "antiquewhite" => Antiquewhite, + "aqua" => Aqua, + "aquamarine" => Aquamarine, + "azure" => Azure, + "beige" => Beige, + "bisque" => Bisque, + "black" => Black, + "blanchedalmond" => Blanchedalmond, + "blue" => Blue, + "blueviolet" => Blueviolet, + "brown" => Brown, + "burlywood" => Burlywood, + "cadetblue" => Cadetblue, + "chartreuse" => Chartreuse, + "chocolate" => Chocolate, + "coral" => Coral, + "cornflowerblue" => Cornflowerblue, + "cornsilk" => Cornsilk, + "crimson" => Crimson, + "cyan" => Cyan, + "darkblue" => Darkblue, + "darkcyan" => Darkcyan, + "darkgoldenrod" => Darkgoldenrod, + "darkgray" => Darkgray, + "darkgreen" => Darkgreen, + "darkgrey" => Darkgrey, + "darkkhaki" => Darkkhaki, + "darkmagenta" => Darkmagenta, + "darkolivegreen" => Darkolivegreen, + "darkorange" => Darkorange, + "darkorchid" => Darkorchid, + "darkred" => Darkred, + "darksalmon" => Darksalmon, + "darkseagreen" => Darkseagreen, + "darkslateblue" => Darkslateblue, + "darkslategray" => Darkslategray, + "darkslategrey" => Darkslategrey, + "darkturquoise" => Darkturquoise, + "darkviolet" => Darkviolet, + "deeppink" => Deeppink, + "deepskyblue" => Deepskyblue, + "dimgray" => Dimgray, + "dimgrey" => Dimgrey, + "dodgerblue" => Dodgerblue, + "firebrick" => Firebrick, + "floralwhite" => Floralwhite, + "forestgreen" => Forestgreen, + "fuchsia" => Fuchsia, + "gainsboro" => Gainsboro, + "ghostwhite" => Ghostwhite, + "gold" => Gold, + "goldenrod" => Goldenrod, + "gray" => Gray, + "green" => Green, + "greenyellow" => Greenyellow, + "grey" => Grey, + "honeydew" => Honeydew, + "hotpink" => Hotpink, + "indianred" => Indianred, + "indigo" => Indigo, + "ivory" => Ivory, + "khaki" => Khaki, + "lavender" => Lavender, + "lavenderblush" => Lavenderblush, + "lawngreen" => Lawngreen, + "lemonchiffon" => Lemonchiffon, + "lightblue" => Lightblue, + "lightcoral" => Lightcoral, + "lightcyan" => Lightcyan, + "lightgoldenrodyellow" => Lightgoldenrodyellow, + "lightgray" => Lightgray, + "lightgreen" => Lightgreen, + "lightgrey" => Lightgrey, + "lightpink" => Lightpink, + "lightsalmon" => Lightsalmon, + "lightseagreen" => Lightseagreen, + "lightskyblue" => Lightskyblue, + "lightslategray" => Lightslategray, + "lightslategrey" => Lightslategrey, + "lightsteelblue" => Lightsteelblue, + "lightyellow" => Lightyellow, + "lime" => Lime, + "limegreen" => Limegreen, + "linen" => Linen, + "magenta" => Magenta, + "maroon" => Maroon, + "mediumaquamarine" => Mediumaquamarine, + "mediumblue" => Mediumblue, + "mediumorchid" => Mediumorchid, + "mediumpurple" => Mediumpurple, + "mediumseagreen" => Mediumseagreen, + "mediumslateblue" => Mediumslateblue, + "mediumspringgreen" => Mediumspringgreen, + "mediumturquoise" => Mediumturquoise, + "mediumvioletred" => Mediumvioletred, + "midnightblue" => Midnightblue, + "mintcream" => Mintcream, + "mistyrose" => Mistyrose, + "moccasin" => Moccasin, + "navajowhite" => Navajowhite, + "navy" => Navy, + "oldlace" => Oldlace, + "olive" => Olive, + "olivedrab" => Olivedrab, + "orange" => Orange, + "orangered" => Orangered, + "orchid" => Orchid, + "palegoldenrod" => Palegoldenrod, + "palegreen" => Palegreen, + "paleturquoise" => Paleturquoise, + "palevioletred" => Palevioletred, + "papayawhip" => Papayawhip, + "peachpuff" => Peachpuff, + "peru" => Peru, + "pink" => Pink, + "plum" => Plum, + "powderblue" => Powderblue, + "purple" => Purple, + "red" => Red, + "rosybrown" => Rosybrown, + "royalblue" => Royalblue, + "saddlebrown" => Saddlebrown, + "salmon" => Salmon, + "sandybrown" => Sandybrown, + "seagreen" => Seagreen, + "seashell" => Seashell, + "sienna" => Sienna, + "silver" => Silver, + "skyblue" => Skyblue, + "slateblue" => Slateblue, + "slategray" => Slategray, + "slategrey" => Slategrey, + "snow" => Snow, + "springgreen" => Springgreen, + "steelblue" => Steelblue, + "tan" => Tan, + "teal" => Teal, + "thistle" => Thistle, + "tomato" => Tomato, + "turquoise" => Turquoise, + "violet" => Violet, + "wheat" => Wheat, + "white" => White, + "whitesmoke" => Whitesmoke, + "yellow" => Yellow, + "yellowgreen" => Yellowgreen, +); + +impl CssColor { + pub fn parse(value: &str) -> Option { + if !value.bytes().any(|byte| byte.is_ascii_uppercase()) { + if let Some(rgb) = value + .strip_prefix('#') + .filter(|hex| hex.len() == 6 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())) + .and_then(|hex| u32::from_str_radix(hex, 16).ok()) + { + return Some(CssColor::Rgb(rgb)); + } + if let Some(color) = CssColor::named(value) { + return Some(color); + } + } + + (!value.is_empty() + && value.len() <= MAX_OTHER_COLOR_LEN + && value + .bytes() + .all(|byte| byte.is_ascii_graphic() || byte == b' ')) + .then(|| CssColor::Other(value.to_string())) + } + + pub fn is_standard(&self) -> bool { + match self { + CssColor::Other(value) => { + value.strip_prefix('#').is_some_and(|hex| { + matches!(hex.len(), 3 | 6) && hex.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) || CssColor::named(value).is_some() + } + _ => true, + } + } +} + +impl Display for CssColor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CssColor::Rgb(rgb) => write!(f, "#{rgb:06x}"), + CssColor::Other(value) => f.write_str(value), + named => f.write_str(named.as_named_str().unwrap_or_default()), + } + } +} + diff --git a/crates/groupware/src/calendar/compare.rs b/crates/groupware/src/calendar/compare.rs new file mode 100644 index 00000000..8e3b8c49 --- /dev/null +++ b/crates/groupware/src/calendar/compare.rs @@ -0,0 +1,417 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{expand::NaiveTimestamp, user::UserDataEntry}; +use ahash::AHashMap; +use calcard::{ + common::PartialDateTime, + icalendar::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarParameter, + ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, ICalendarValue, + ICalendarValueType, Uri, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ComparisonScope { + SharedData, + SchedulingData, +} + +impl ComparisonScope { + pub fn component_eq(self, a: &ICalendar, a_id: u32, b: &ICalendar, b_id: u32) -> bool { + let (Some(a_component), Some(b_component)) = + (a.component_by_id(a_id), b.component_by_id(b_id)) + else { + return false; + }; + a_component.component_type == b_component.component_type + && self + .entries(a_component) + .same_entries(&self.entries(b_component)) + && self.children_eq(a, a_component, b, b_component) + } + + fn children_eq( + self, + a: &ICalendar, + a_component: &ICalendarComponent, + b: &ICalendar, + b_component: &ICalendarComponent, + ) -> bool { + let (a_children, b_children) = + (self.children(a, a_component), self.children(b, b_component)); + if a_children.len() != b_children.len() { + return false; + } + let mut candidates: AHashMap, Vec> = + AHashMap::with_capacity(b_children.len()); + for b_child in b_children { + let Some(key) = b.child_key(b_child) else { + return false; + }; + candidates.entry(key).or_default().push(b_child); + } + + a_children.into_iter().all(|a_child| { + let Some(candidates) = a + .child_key(a_child) + .and_then(|key| candidates.get_mut(&key)) + else { + return false; + }; + match candidates + .iter() + .position(|b_child| self.component_eq(a, a_child, b, *b_child)) + { + Some(index) => { + candidates.swap_remove(index); + true + } + None => false, + } + }) + } + + fn entries(self, component: &ICalendarComponent) -> Vec<&ICalendarEntry> { + let mut entries = component + .entries + .iter() + .filter(|entry| match self { + ComparisonScope::SharedData => !entry.name.is_volatile(), + ComparisonScope::SchedulingData => { + !entry.name.is_volatile() + && !entry.is_user_data() + && entry.name != ICalendarProperty::Sequence + } + }) + .collect::>(); + entries.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + entries + } + + fn children(self, ical: &ICalendar, component: &ICalendarComponent) -> Vec { + component + .component_ids + .iter() + .copied() + .filter(|id| { + self == ComparisonScope::SharedData + || ical + .component_by_id(*id) + .is_some_and(|c| c.component_type != ICalendarComponentType::VAlarm) + }) + .collect() + } +} + +type ChildKey<'x> = (&'x ICalendarComponentType, Option); + +trait ChildComponentKey { + fn child_key(&self, comp_id: u32) -> Option>; +} + +impl ChildComponentKey for ICalendar { + fn child_key(&self, comp_id: u32) -> Option> { + let component = self.component_by_id(comp_id)?; + Some(( + &component.component_type, + component + .property(&ICalendarProperty::RecurrenceId) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_partial_date_time) + .and_then(PartialDateTime::naive_timestamp), + )) + } +} + +pub trait RecurrenceComponents { + fn base_component_id(&self) -> Option; +} + +impl RecurrenceComponents for ICalendar { + fn base_component_id(&self) -> Option { + self.components + .iter() + .position(|component| { + component.component_type.is_event_or_todo() && !component.is_recurrence_override() + }) + .map(|id| id as u32) + } +} + +pub struct RedundantOverrides<'x> { + ical: &'x ICalendar, + base: &'x ICalendarComponent, + base_start: &'x ICalendarEntry, + base_entries: Vec<&'x ICalendarEntry>, +} + +impl<'x> RedundantOverrides<'x> { + pub fn new(ical: &'x ICalendar) -> Option { + let base = ical.component_by_id(ical.base_component_id()?)?; + Some(RedundantOverrides { + ical, + base, + base_start: base.property(&ICalendarProperty::Dtstart)?, + base_entries: base.occurrence_entries(), + }) + } + + pub fn is_redundant(&self, override_id: u32) -> bool { + let Some(component) = self.ical.component_by_id(override_id) else { + return false; + }; + let (Some(recurrence_id), Some(start)) = ( + component.property(&ICalendarProperty::RecurrenceId), + component.property(&ICalendarProperty::Dtstart), + ) else { + return false; + }; + let Some(shift) = self.base_start.shift_to(start) else { + return false; + }; + + start.values == recurrence_id.values + && start.tz_id() == recurrence_id.tz_id() + && [ICalendarProperty::Dtend, ICalendarProperty::Due] + .iter() + .all( + |name| match (self.base.property(name), component.property(name)) { + (Some(base_entry), Some(entry)) => { + base_entry.shift_to(entry) == Some(shift) + } + (None, None) => true, + _ => false, + }, + ) + && component + .occurrence_entries() + .same_entries(&self.base_entries) + && ComparisonScope::SharedData.children_eq(self.ical, component, self.ical, self.base) + } +} + +pub trait EntryEquivalence { + fn is_equivalent(&self, other: &Self) -> bool; + + fn shift_to(&self, other: &Self) -> Option; +} + +impl EntryEquivalence for ICalendarEntry { + fn is_equivalent(&self, other: &Self) -> bool { + self.name == other.name + && self.values.len() == other.values.len() + && self.values.iter().zip(&other.values).all(|(value, other)| { + value == other + || (self.name.is_calendar_address() + && value + .as_address() + .is_some_and(|address| other.as_address() == Some(address))) + }) + && self.has_equivalent_params(other) + } + + fn shift_to(&self, other: &Self) -> Option { + let ([ICalendarValue::PartialDateTime(from)], [ICalendarValue::PartialDateTime(to)]) = + (self.values.as_slice(), other.values.as_slice()) + else { + return None; + }; + (from.has_same_shape(to) && self.has_equivalent_params(other)) + .then(|| to.naive_timestamp()?.checked_sub(from.naive_timestamp()?)) + .flatten() + } +} + +pub trait NaiveDateTime { + fn naive_timestamp(&self) -> Option; + + fn has_same_shape(&self, other: &Self) -> bool; + + fn shifted(&self, shift: i64) -> Option; +} + +impl NaiveDateTime for PartialDateTime { + fn naive_timestamp(&self) -> Option { + self.to_date_time() + .map(|date_time| date_time.date_time.naive_timestamp()) + } + + fn has_same_shape(&self, other: &Self) -> bool { + self.has_date() == other.has_date() + && self.has_time() == other.has_time() + && self.tz_hour == other.tz_hour + && self.tz_minute == other.tz_minute + && self.tz_minus == other.tz_minus + } + + fn shifted(&self, shift: i64) -> Option { + let timestamp = self.naive_timestamp()?.checked_add(shift)?; + let shifted = if self.has_time() { + PartialDateTime::from_naive_timestamp(timestamp) + } else { + PartialDateTime::from_date_timestamp(timestamp) + }; + Some(PartialDateTime { + second: shifted.second.filter(|_| self.second.is_some()), + tz_hour: self.tz_hour, + tz_minute: self.tz_minute, + tz_minus: self.tz_minus, + ..shifted + }) + } +} + +pub trait EntryList { + fn same_entries(&self, other: &Self) -> bool; +} + +impl EntryList for [&ICalendarEntry] { + fn same_entries(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + let mut matched = vec![false; other.len()]; + self.iter().all(|entry| { + let start = other.partition_point(|candidate| candidate.name < entry.name); + other + .get(start..) + .unwrap_or_default() + .iter() + .zip(matched.get_mut(start..).unwrap_or_default()) + .take_while(|(candidate, _)| candidate.name == entry.name) + .find(|(candidate, is_matched)| !**is_matched && entry.is_equivalent(candidate)) + .map(|(_, is_matched)| *is_matched = true) + .is_some() + }) + } +} + +trait OccurrenceEntries { + fn occurrence_entries(&self) -> Vec<&ICalendarEntry>; +} + +impl OccurrenceEntries for ICalendarComponent { + fn occurrence_entries(&self) -> Vec<&ICalendarEntry> { + let mut entries = self + .entries + .iter() + .filter(|entry| { + !entry.name.is_volatile() + && !matches!( + entry.name, + ICalendarProperty::RecurrenceId + | ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exrule + | ICalendarProperty::Exdate + | ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Due + ) + }) + .collect::>(); + entries.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + entries + } +} + +trait EntryParameters { + fn significant_params(&self) -> impl Iterator + Clone; + + fn has_equivalent_params(&self, other: &Self) -> bool; +} + +impl EntryParameters for ICalendarEntry { + fn significant_params(&self) -> impl Iterator + Clone { + self.params.iter().filter(|param| match ¶m.value { + ICalendarParameterValue::Value(value_type) + if param.name == ICalendarParameterName::Value => + { + self.values.is_empty() + || !self.values.iter().all(|value| value.is_of_type(value_type)) + } + _ => true, + }) + } + + fn has_equivalent_params(&self, other: &Self) -> bool { + let (params, other_params) = (self.significant_params(), other.significant_params()); + params.clone().count() == other_params.clone().count() + && params.clone().all(|param| { + params.clone().filter(|item| *item == param).count() + == other_params.clone().filter(|item| *item == param).count() + }) + } +} + +trait ValueClass { + fn is_of_type(&self, value_type: &ICalendarValueType) -> bool; + + fn as_address(&self) -> Option<&str>; +} + +impl ValueClass for ICalendarValue { + fn is_of_type(&self, value_type: &ICalendarValueType) -> bool { + match (value_type, self) { + (ICalendarValueType::Binary, ICalendarValue::Binary(_)) + | (ICalendarValueType::Boolean, ICalendarValue::Boolean(_)) + | (ICalendarValueType::CalAddress, ICalendarValue::Uri(_) | ICalendarValue::Text(_)) + | (ICalendarValueType::Duration, ICalendarValue::Duration(_)) + | (ICalendarValueType::Float, ICalendarValue::Float(_)) + | (ICalendarValueType::Integer, ICalendarValue::Integer(_)) + | (ICalendarValueType::Period, ICalendarValue::Period(_)) + | (ICalendarValueType::Recur, ICalendarValue::RecurrenceRule(_)) + | (ICalendarValueType::Text, ICalendarValue::Text(_)) + | (ICalendarValueType::Uri, ICalendarValue::Uri(_)) => true, + (ICalendarValueType::Date, ICalendarValue::PartialDateTime(value)) => { + value.has_date() && !value.has_time() + } + (ICalendarValueType::DateTime, ICalendarValue::PartialDateTime(value)) => { + value.has_date() && value.has_time() + } + (ICalendarValueType::Time, ICalendarValue::PartialDateTime(value)) => { + !value.has_date() && value.has_time() + } + _ => false, + } + } + + fn as_address(&self) -> Option<&str> { + match self { + ICalendarValue::Uri(Uri::Location(address)) | ICalendarValue::Text(address) => { + Some(address) + } + _ => None, + } + } +} + +pub trait PropertyClass { + fn is_volatile(&self) -> bool; + + fn is_calendar_address(&self) -> bool; +} + +impl PropertyClass for ICalendarProperty { + fn is_volatile(&self) -> bool { + matches!( + self, + ICalendarProperty::Dtstamp | ICalendarProperty::LastModified + ) + } + + fn is_calendar_address(&self) -> bool { + matches!( + self, + ICalendarProperty::Organizer + | ICalendarProperty::Attendee + | ICalendarProperty::CalendarAddress + ) + } +} + diff --git a/crates/groupware/src/calendar/dates.rs b/crates/groupware/src/calendar/dates.rs index 44b92849..428bf71e 100644 --- a/crates/groupware/src/calendar/dates.rs +++ b/crates/groupware/src/calendar/dates.rs @@ -5,142 +5,103 @@ */ use super::{ - ArchivedCalendarEventData, ArchivedTimezone, CalendarEventData, Timezone, - alarm::{CalendarAlarm, ExpandAlarm}, + Alarm, ArchivedCalendarEventData, ArchivedTimezone, CalendarEventData, Timezone, + alarm::ExpandAlarm, + alerts::{DefaultAlerts, ICalendarDefaultAlerts}, + expand::RangeFlags, + overlap::ICalendarOverlap, }; -use crate::calendar::{ComponentTimeRange, alarm::CalendarAlarmType}; +use crate::calendar::ComponentTimeRange; use calcard::{ common::timezone::Tz, - icalendar::{ICalendar, ICalendarComponentType, dates::TimeOrDelta}, + icalendar::{ICalendar, ICalendarComponentType}, }; use compact_str::ToCompactString; use indexmap::IndexMap; use store::{ ahash::{AHashMap, RandomState}, - write::{key::KeySerializer, now}, + write::key::KeySerializer, }; +use types::OverlapCondition; +#[cfg(all(test, stalwart_tests))] +use utils::codec::leb128::Leb128; const MAX_TIME_SPAN: i64 = u32::MAX as i64; impl CalendarEventData { - pub fn new( + pub fn new(ical: ICalendar, default_tz: Tz, max_expansions: usize) -> Self { + Self::new_with_default_alerts(ical, default_tz, max_expansions, &DefaultAlerts::disabled()) + } + + pub fn new_with_default_alerts( ical: ICalendar, default_tz: Tz, max_expansions: usize, - next_email_alarm: &mut Option, + default_alerts: &DefaultAlerts, ) -> Self { let mut ranges = TimeRanges::default(); - let now = now() as i64; let expanded = ical.expand_dates(default_tz, max_expansions); - let mut groups: IndexMap<(u16, u16, u16, i32), Vec, RandomState> = + let mut groups: IndexMap<(u16, u16, u16, i32, RangeFlags), Vec, RandomState> = IndexMap::with_capacity_and_hasher(16, RandomState::default()); - let mut alarms = AHashMap::with_capacity(16); + let mut components: AHashMap, OverlapCondition)> = + AHashMap::with_capacity(16); for event in expanded.events { - let start_naive = event.start.naive_local(); - let start_tz = event.start.timezone().as_id(); - let start_timestamp_utc = event.start.timestamp(); - let start_timestamp_naive = start_naive.and_utc().timestamp(); - let (end_timestamp_utc, end_timestamp_naive, end_tz) = match event.end { - TimeOrDelta::Time(time) => { - let end_naive = time.naive_local(); - let end_timestamp_utc = time.timestamp(); - let end_timestamp_naive = end_naive.and_utc().timestamp(); - ( - end_timestamp_utc, - end_timestamp_naive, - time.timezone().as_id(), - ) - } - TimeOrDelta::Delta(delta) => { - let delta = delta.num_seconds(); - ( - start_timestamp_utc + delta, - start_timestamp_naive + delta, - start_tz, - ) - } + let Ok(comp_id) = u16::try_from(event.comp_id) else { + continue; }; + let start_tz = event.start.timezone().as_id(); + let end_tz = event.end.timezone().as_id(); + let (start_timestamp_utc, end_timestamp_utc) = event.timestamps(); + let (start_timestamp_naive, end_timestamp_naive) = event.naive_timestamps(); - // Expand alarms - let mut min = std::cmp::min(start_timestamp_utc, end_timestamp_utc); - let mut max = std::cmp::max(start_timestamp_utc, end_timestamp_utc); - for alarm in alarms.entry(event.comp_id).or_insert_with(|| { - ical.component_by_id(event.comp_id) + let (_, condition) = components.entry(comp_id).or_insert_with(|| { + // Expand alarms + let alarms = ical + .component_by_id(event.comp_id) .map_or(&[][..], |c| c.component_ids.as_slice()) .iter() .filter_map(|alarm_id| { + let id = u16::try_from(*alarm_id).ok()?; ical.component_by_id(*alarm_id).and_then(|alarm| { - if alarm.component_type == ICalendarComponentType::VAlarm { - alarm.expand_alarm(*alarm_id as u16, event.comp_id as u16) + if alarm.component_type == ICalendarComponentType::VAlarm + && (!default_alerts.is_enabled() + || ical.is_kept_with_default_alerts(alarm, default_alerts)) + { + alarm.expand_alarm(id, comp_id) } else { None } }) }) - .collect::>() - }) { - if let Some(alarm_time) = - alarm - .delta - .to_timestamp(start_timestamp_utc, end_timestamp_utc, default_tz) - { - if alarm_time < min { - min = alarm_time; - } - if alarm_time > max { - max = alarm_time; - } - if alarm_time > now - && next_email_alarm - .as_ref() - .is_none_or(|next| alarm_time < next.alarm_time) - { - *next_email_alarm = Some(CalendarAlarm { - alarm_id: alarm.id, - event_id: alarm.parent_id, - alarm_time, - typ: if alarm.is_email_alert { - CalendarAlarmType::Email { - event_start: start_timestamp_naive, - event_end: end_timestamp_naive, - event_start_tz: start_tz, - event_end_tz: end_tz, - } - } else { - CalendarAlarmType::Display { - recurrence_id: if ical.components[alarm.parent_id as usize] - .is_recurrent_or_override() - { - start_timestamp_naive.into() - } else { - None - }, - } - }, - }); - } - } - } + .collect::>(); + (alarms, ical.overlap_condition(event.comp_id)) + }); + let condition = condition.for_instance(start_timestamp_utc, end_timestamp_utc); ranges.update_base_offset(start_timestamp_naive, end_timestamp_naive); - ranges.update_utc_min_max(min, max); + ranges.update_utc_min_max( + std::cmp::min(start_timestamp_utc, end_timestamp_utc), + std::cmp::max(start_timestamp_utc, end_timestamp_utc), + ); groups .entry(( start_tz, end_tz, - event.comp_id as u16, + comp_id, (end_timestamp_naive - start_timestamp_naive) .clamp(i32::MIN as i64, i32::MAX as i64) as i32, + RangeFlags::of(&event.start, &event.end).with_condition(condition), )) .or_default() .push(start_timestamp_naive); } let mut events = Vec::with_capacity(groups.len()); - for ((start_tz, end_tz, id, duration), mut instances) in groups { + for ((start_tz, end_tz, id, duration, flags), mut instances) in groups { instances.sort_unstable(); + instances.dedup(); instances.truncate(instances.partition_point(|instance| { instance.saturating_sub(ranges.base_offset) <= MAX_TIME_SPAN })); @@ -169,6 +130,7 @@ impl CalendarEventData { start_tz, end_tz, duration, + flags: flags.bits(), instances: instances.into_boxed_slice(), }); } @@ -181,93 +143,122 @@ impl CalendarEventData { .into_iter() .map(|e| e.error.to_compact_string()) .collect::>(), - Details = ical.to_string(), + Contents = ical.to_compact_string(), + ResourceUid = ical.uids().next().map(|uid| uid.to_compact_string()), Limit = max_expansions, ); } + let (base_offset, base_time_utc, duration) = if ranges.is_empty() { + (0, 0, 0) + } else { + ( + ranges.base_offset, + ranges + .min_time_utc + .saturating_sub(ranges.base_offset) + .clamp(0, MAX_TIME_SPAN) as u32, + ranges + .max_time_utc + .saturating_sub(ranges.min_time_utc) + .clamp(0, MAX_TIME_SPAN) as u32, + ) + }; + CalendarEventData { event: ical, time_ranges: events.into_boxed_slice(), - alarms: alarms + alarms: components .into_values() - .flatten() + .flat_map(|(alarms, _)| alarms) .collect::>() .into_boxed_slice(), - base_offset: ranges.base_offset, - base_time_utc: (ranges.min_time_utc - ranges.base_offset).clamp(0, MAX_TIME_SPAN) - as u32, - duration: (ranges.max_time_utc - ranges.min_time_utc).clamp(0, MAX_TIME_SPAN) as u32, + base_offset, + base_time_utc, + duration, } } pub fn event_range(&self) -> Option<(i64, u32)> { - if self.base_offset != 0 { - Some((self.base_offset + self.base_time_utc as i64, self.duration)) + if !self.time_ranges.is_empty() { + Some((self.event_range_start(), self.duration)) } else { None } } + + pub fn has_floating_start(&self) -> bool { + let floating = Tz::Floating.as_id(); + !self.time_ranges.is_empty() + && self + .time_ranges + .iter() + .all(|range| range.start_tz == floating) + } } -#[derive(Default, Debug)] +#[derive(Debug)] struct TimeRanges { max_time_utc: i64, min_time_utc: i64, base_offset: i64, } +impl Default for TimeRanges { + fn default() -> Self { + TimeRanges { + max_time_utc: i64::MIN, + min_time_utc: i64::MAX, + base_offset: i64::MAX, + } + } +} + impl TimeRanges { pub fn update_base_offset(&mut self, t1: i64, t2: i64) { - let offset = std::cmp::min(t1, t2); - if offset < self.base_offset || self.base_offset == 0 { - self.base_offset = offset; - } + self.base_offset = self.base_offset.min(t1).min(t2); } pub fn update_utc_min_max(&mut self, min: i64, max: i64) { - if min < self.min_time_utc || self.min_time_utc == 0 { - self.min_time_utc = min; - } - if max > self.max_time_utc { - self.max_time_utc = max; - } - if min < self.base_offset || self.base_offset == 0 { - self.base_offset = min; - } + self.min_time_utc = self.min_time_utc.min(min); + self.max_time_utc = self.max_time_utc.max(max); + self.base_offset = self.base_offset.min(min); + } + + pub fn is_empty(&self) -> bool { + self.min_time_utc > self.max_time_utc } } impl ArchivedCalendarEventData { pub fn event_range(&self) -> Option<(i64, u32)> { - if self.base_offset != 0 { - Some(( - self.base_offset.to_native() + self.base_time_utc.to_native() as i64, - self.duration.to_native(), - )) + if !self.time_ranges.is_empty() { + Some((self.event_range_start(), self.duration.to_native())) } else { None } } pub fn event_range_start(&self) -> i64 { - self.base_offset.to_native() + self.base_time_utc.to_native() as i64 + self.base_offset + .to_native() + .saturating_add(self.base_time_utc.to_native() as i64) } pub fn event_range_end(&self) -> i64 { - self.base_offset.to_native() - + self.base_time_utc.to_native() as i64 - + self.duration.to_native() as i64 + self.event_range_start() + .saturating_add(self.duration.to_native() as i64) } } impl CalendarEventData { pub fn event_range_start(&self) -> i64 { - self.base_offset + self.base_time_utc as i64 + self.base_offset.saturating_add(self.base_time_utc as i64) } pub fn event_range_end(&self) -> i64 { - self.base_offset + self.base_time_utc as i64 + self.duration as i64 + self.event_range_start() + .saturating_add(self.duration as i64) } } @@ -296,3 +287,4 @@ impl ArchivedTimezone { } } } + diff --git a/crates/groupware/src/calendar/expand.rs b/crates/groupware/src/calendar/expand.rs index 9314ada7..6bd1ecca 100644 --- a/crates/groupware/src/calendar/expand.rs +++ b/crates/groupware/src/calendar/expand.rs @@ -4,21 +4,49 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::ArchivedCalendarEventData; +use super::{AlarmDelta, ArchivedCalendarEventData}; use crate::calendar::CalendarEventData; use ahash::AHashSet; use calcard::{ - common::{DateTimeResult, timezone::Tz}, - icalendar::{ArchivedICalendarComponent, ICalendarComponent, ICalendarProperty}, + common::{ + DateTimeResult, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::{ + ArchivedICalendarComponent, ArchivedICalendarParameterName, ICalendarComponent, + ICalendarParameterName, ICalendarProperty, ICalendarValue, + }, }; -use chrono::{DateTime, TimeZone}; +use jiff::{SignedDuration, Timestamp, civil::DateTime}; use std::str::FromStr; use store::write::bitpack::BitpackIterator; -use types::TimeRange; -use utils::codec::leb128::Leb128Reader; +use types::{OverlapCondition, OverlapRule, TimeRange}; +use utils::codec::leb128::Leb128; const RECURRENCE_KEY_EPOCH: i64 = -2208988800; const RECURRENCE_KEY_GRANULARITY: i64 = 60; +pub const SECONDS_PER_DAY: i64 = 86_400; +const NAIVE_EPOCH: DateTime = DateTime::constant(1970, 1, 1, 0, 0, 0, 0); +pub const MAX_UTC_OFFSET: i64 = SECONDS_PER_DAY; +const EXPANSION_SKIP_MARGIN: i64 = 2 * MAX_UTC_OFFSET; + +pub trait NaiveTimestamp: Sized { + fn naive_timestamp(&self) -> i64; + + fn from_naive_timestamp(timestamp: i64) -> Option; +} + +impl NaiveTimestamp for DateTime { + fn naive_timestamp(&self) -> i64 { + self.duration_since(NAIVE_EPOCH).as_secs() + } + + fn from_naive_timestamp(timestamp: i64) -> Option { + NAIVE_EPOCH + .checked_add(SignedDuration::from_secs(timestamp)) + .ok() + } +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RecurrenceKey(u32); @@ -35,6 +63,10 @@ impl RecurrenceKey { .map(RecurrenceKey) } + pub fn to_naive_timestamp(self) -> i64 { + RECURRENCE_KEY_EPOCH + (self.0 as i64 - 1) * RECURRENCE_KEY_GRANULARITY + } + pub fn from_prefix(prefix: u32) -> Option { (prefix != 0).then_some(RecurrenceKey(prefix)) } @@ -50,8 +82,23 @@ pub struct RecurrenceId { pub naive: i64, } +impl RecurrenceId { + fn from_utc(utc: i64, tz: Tz) -> Option { + Some(RecurrenceId { + utc, + naive: tz + .from_timestamp(Timestamp::from_second(utc).ok()?) + .naive_timestamp(), + }) + } +} + pub trait ComponentRecurrenceId { fn recurrence_id(&self, fallback_tz: Tz) -> Option; + + fn recurrence_tz(&self, fallback_tz: Tz) -> Option; + + fn this_and_future_tz(&self, fallback_tz: Tz) -> Option; } impl ComponentRecurrenceId for ArchivedICalendarComponent { @@ -67,6 +114,20 @@ impl ComponentRecurrenceId for ArchivedICalendarComponent { fallback_tz, ) } + + fn recurrence_tz(&self, fallback_tz: Tz) -> Option { + self.property(&ICalendarProperty::RecurrenceId) + .map(|entry| resolve_tz(entry.tz_id(), fallback_tz)) + } + + fn this_and_future_tz(&self, fallback_tz: Tz) -> Option { + let entry = self.property(&ICalendarProperty::RecurrenceId)?; + entry + .params + .iter() + .any(|param| param.name == ArchivedICalendarParameterName::Range) + .then(|| resolve_tz(entry.tz_id(), fallback_tz)) + } } impl ComponentRecurrenceId for ICalendarComponent { @@ -82,6 +143,26 @@ impl ComponentRecurrenceId for ICalendarComponent { fallback_tz, ) } + + fn recurrence_tz(&self, fallback_tz: Tz) -> Option { + self.property(&ICalendarProperty::RecurrenceId) + .map(|entry| resolve_tz(entry.tz_id(), fallback_tz)) + } + + fn this_and_future_tz(&self, fallback_tz: Tz) -> Option { + let entry = self.property(&ICalendarProperty::RecurrenceId)?; + entry + .params + .iter() + .any(|param| param.name == ICalendarParameterName::Range) + .then(|| resolve_tz(entry.tz_id(), fallback_tz)) + } +} + +fn resolve_tz(tz_id: Option<&str>, fallback_tz: Tz) -> Tz { + tz_id + .and_then(|tz_id| Tz::from_str(tz_id).ok()) + .unwrap_or(fallback_tz) } fn resolve_recurrence_id( @@ -89,32 +170,132 @@ fn resolve_recurrence_id( date_time: DateTimeResult, fallback_tz: Tz, ) -> Option { - let tz = tz_id - .and_then(|tz_id| Tz::from_str(tz_id).ok()) - .unwrap_or(fallback_tz); - let date_time = date_time.to_date_time_with_tz(tz)?.with_timezone(&tz); + let tz = resolve_tz(tz_id, fallback_tz); + let date_time = date_time.to_date_time_with_tz(tz)?.with_timezone(tz); Some(RecurrenceId { utc: date_time.timestamp(), - naive: date_time.naive_local().and_utc().timestamp(), + naive: date_time.naive_timestamp(), }) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RecurrenceShift { + tz: Tz, + nominal: i64, + exact: i64, +} + +impl RecurrenceShift { + pub(crate) fn new( + tz: Tz, + start_naive: i64, + recurrence_id: RecurrenceId, + recurrence_tz: Tz, + ) -> Option { + let recurrence_naive = if recurrence_tz == tz { + recurrence_id.naive + } else { + RecurrenceId::from_utc(recurrence_id.utc, tz)?.naive + }; + Some(RecurrenceShift { + tz, + nominal: start_naive - recurrence_naive, + exact: resolve_local(tz, start_naive)? - recurrence_id.utc, + }) + } + + pub(crate) fn recurrence_id( + &self, + tz: Tz, + flags: RangeFlags, + start_naive: i64, + recurrence_tz: Tz, + ) -> Option { + if tz == self.tz { + let naive = start_naive - self.nominal; + let utc = resolve_local(tz, naive)?; + if recurrence_tz == tz { + Some(RecurrenceId { utc, naive }) + } else { + RecurrenceId::from_utc(utc, recurrence_tz) + } + } else { + RecurrenceId::from_utc( + flags.resolve_start(tz, start_naive)?.timestamp() - self.exact, + recurrence_tz, + ) + } + } +} + +#[derive(Debug, Default)] +struct ThisAndFutureShifts(Vec<(u32, RecurrenceShift)>); + +impl ThisAndFutureShifts { + fn recurrence_id( + &mut self, + comp_id: u32, + flags: RangeFlags, + own_recurrence_id: Option, + component_tz: Tz, + start_naive: i64, + recurrence_tz: Tz, + ) -> Option { + match own_recurrence_id { + Some(own_recurrence_id) => { + self.0.extend( + RecurrenceShift::new( + component_tz, + start_naive, + own_recurrence_id, + recurrence_tz, + ) + .map(|shift| (comp_id, shift)), + ); + None + } + None => self + .0 + .iter() + .find(|(id, _)| *id == comp_id) + .and_then(|(_, shift)| { + shift.recurrence_id(component_tz, flags, start_naive, recurrence_tz) + }), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CalendarEventExpansion { pub comp_id: u32, pub own_recurrence_id: Option, + pub series_recurrence_id: Option, pub start: i64, pub end: i64, pub start_naive: i64, + pub end_naive: i64, + pub start_tz: Tz, + pub end_tz: Tz, + pub flags: RangeFlags, } impl CalendarEventExpansion { + pub fn alarm_time(&self, delta: &AlarmDelta, default_tz: Tz) -> Option { + delta.to_timestamp( + &self.flags.resolve_start(self.start_tz, self.start_naive)?, + &self.flags.resolve_end(self.end_tz, self.end_naive)?, + default_tz, + ) + } + pub fn recurrence_id(&self) -> RecurrenceId { - self.own_recurrence_id.unwrap_or(RecurrenceId { - utc: self.start, - naive: self.start_naive, - }) + self.own_recurrence_id + .or(self.series_recurrence_id) + .unwrap_or(RecurrenceId { + utc: self.start, + naive: self.start_naive, + }) } pub fn recurrence_key(&self) -> Option { @@ -123,28 +304,72 @@ impl CalendarEventExpansion { } impl ArchivedCalendarEventData { - pub fn expand(&self, default_tz: Tz, limit: TimeRange) -> Option> { - let mut expansion = Vec::with_capacity(self.time_ranges.len()); - let base_offset = self.base_offset.to_native(); + pub fn component_tz(&self, comp_id: u32) -> Option { + self.time_ranges + .iter() + .find(|range| range.id.to_native() as u32 == comp_id) + .and_then(|range| Tz::from_id(range.start_tz.to_native())) + } - 'outer: for (range_index, range) in self.time_ranges.iter().enumerate() { - let instances = range.instances.as_ref(); - let (offset_or_count, bytes_read) = instances.read_leb128::()?; + pub fn expand( + &self, + default_tz: Tz, + limit: TimeRange, + rule: OverlapRule, + ) -> Option> { + self.expand_instances(default_tz, limit, rule) + } +} - let comp_id = range.id.to_native() as u32; - let component = self.event.components.get(comp_id as usize)?; - let duration = range.duration.to_native() as i64; - let component_tz = Tz::from_id(range.start_tz.to_native())?; - let mut own_recurrence_id = self - .time_ranges - .iter() +#[derive(Clone, Copy)] +struct TimeRangeView<'x> { + id: u16, + start_tz: u16, + end_tz: u16, + duration: i32, + flags: u8, + instances: &'x [u8], +} + +trait ExpansionSource { + type Component: ComponentRecurrenceId; + + fn base_offset(&self) -> i64; + + fn time_ranges(&self) -> impl Iterator> + Clone; + + fn component(&self, comp_id: u32) -> Option<&Self::Component>; + + fn expand_instances( + &self, + default_tz: Tz, + limit: TimeRange, + rule: OverlapRule, + ) -> Option> { + let time_ranges = self.time_ranges(); + let mut expansion = Vec::with_capacity(time_ranges.size_hint().0); + let base_offset = self.base_offset(); + let mut shifts = ThisAndFutureShifts::default(); + + 'outer: for (range_index, range) in time_ranges.clone().enumerate() { + let instances = range.instances; + let (offset_or_count, bytes_read) = u32::from_leb128(instances)?; + + let comp_id = range.id as u32; + let component = self.component(comp_id)?; + let duration = range.duration as i64; + let flags = RangeFlags::from_bits(range.flags); + let condition = flags.condition(); + let component_tz = Tz::from_id(range.start_tz)?; + let mut own_recurrence_id = time_ranges + .clone() .take(range_index) .all(|prior| prior.id != range.id) .then(|| component.recurrence_id(component_tz)) .flatten(); + let this_and_future_tz = component.this_and_future_tz(component_tz); let mut start_tz = component_tz; - let mut end_tz = Tz::from_id(range.end_tz.to_native())?; - let is_todo = component.component_type.is_todo(); + let mut end_tz = Tz::from_id(range.end_tz)?; if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; @@ -154,45 +379,84 @@ impl ArchivedCalendarEventData { } if instances.len() > bytes_read { + let skip_before = (limit.start <= limit.end + && (rule == OverlapRule::Jmap || condition != OverlapCondition::TodoCreated)) + .then(|| limit.start.saturating_sub(EXPANSION_SKIP_MARGIN)); let unpacker = BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); for start_offset in unpacker { let own_recurrence_id = own_recurrence_id.take(); - let start_date_naive = start_offset as i64 + base_offset; - let end_date_naive = start_date_naive + duration; + let start_date_naive = base_offset.saturating_add(start_offset as i64); + let end_date_naive = start_date_naive.saturating_add(duration); + let series_recurrence_id = this_and_future_tz.and_then(|recurrence_tz| { + shifts.recurrence_id( + comp_id, + flags, + own_recurrence_id, + component_tz, + start_date_naive, + recurrence_tz, + ) + }); + if skip_before + .is_some_and(|before| start_date_naive < before && end_date_naive < before) + { + continue; + } let (Some(start), Some(end)) = ( - resolve_local(start_tz, start_date_naive), - resolve_local(end_tz, end_date_naive), + flags.resolve_start(start_tz, start_date_naive), + flags.resolve_end(end_tz, end_date_naive), ) else { continue; }; + let (start, end) = (start.timestamp(), end.timestamp()); - if limit.is_in_range(is_todo, start, end) { + if limit.matches(rule, condition, start, end) { expansion.push(CalendarEventExpansion { comp_id, own_recurrence_id, + series_recurrence_id, start, end, start_naive: start_date_naive, + end_naive: end_date_naive, + start_tz, + end_tz, + flags, }); } else if start > limit.end { continue 'outer; } } } else { - let start_date_naive = offset_or_count as i64 + base_offset; - let end_date_naive = start_date_naive + duration; + let start_date_naive = base_offset.saturating_add(offset_or_count as i64); + let end_date_naive = start_date_naive.saturating_add(duration); + let series_recurrence_id = this_and_future_tz.and_then(|recurrence_tz| { + shifts.recurrence_id( + comp_id, + flags, + own_recurrence_id, + component_tz, + start_date_naive, + recurrence_tz, + ) + }); if let (Some(start), Some(end)) = ( - resolve_local(start_tz, start_date_naive), - resolve_local(end_tz, end_date_naive), - ) && limit.is_in_range(is_todo, start, end) + flags.resolve_start(start_tz, start_date_naive), + flags.resolve_end(end_tz, end_date_naive), + ) && limit.matches(rule, condition, start.timestamp(), end.timestamp()) { expansion.push(CalendarEventExpansion { comp_id, own_recurrence_id, - start, - end, + series_recurrence_id, + start: start.timestamp(), + end: end.timestamp(), start_naive: start_date_naive, + end_naive: end_date_naive, + start_tz, + end_tz, + flags, }); } } @@ -202,7 +466,62 @@ impl ArchivedCalendarEventData { } } +impl ExpansionSource for ArchivedCalendarEventData { + type Component = ArchivedICalendarComponent; + + fn base_offset(&self) -> i64 { + self.base_offset.to_native() + } + + fn time_ranges(&self) -> impl Iterator> + Clone { + self.time_ranges.iter().map(|range| TimeRangeView { + id: range.id.to_native(), + start_tz: range.start_tz.to_native(), + end_tz: range.end_tz.to_native(), + duration: range.duration.to_native(), + flags: range.flags, + instances: range.instances.as_ref(), + }) + } + + fn component(&self, comp_id: u32) -> Option<&Self::Component> { + self.event.components.get(comp_id as usize) + } +} + +impl ExpansionSource for CalendarEventData { + type Component = ICalendarComponent; + + fn base_offset(&self) -> i64 { + self.base_offset + } + + fn time_ranges(&self) -> impl Iterator> + Clone { + self.time_ranges.iter().map(|range| TimeRangeView { + id: range.id, + start_tz: range.start_tz, + end_tz: range.end_tz, + duration: range.duration, + flags: range.flags, + instances: range.instances.as_ref(), + }) + } + + fn component(&self, comp_id: u32) -> Option<&Self::Component> { + self.event.components.get(comp_id as usize) + } +} + impl CalendarEventData { + pub fn expand( + &self, + default_tz: Tz, + limit: TimeRange, + rule: OverlapRule, + ) -> Option> { + self.expand_instances(default_tz, limit, rule) + } + pub fn component_tz(&self, comp_id: u32) -> Option { self.time_ranges .iter() @@ -217,13 +536,15 @@ impl CalendarEventData { ) -> Option> { let mut expansion = Vec::with_capacity(keys.len()); let base_offset = self.base_offset; + let mut shifts = ThisAndFutureShifts::default(); for (range_index, range) in self.time_ranges.iter().enumerate() { let instances = range.instances.as_ref(); - let (offset_or_count, bytes_read) = instances.read_leb128::()?; + let (offset_or_count, bytes_read) = u32::from_leb128(instances)?; let comp_id = range.id as u32; let component = self.event.components.get(comp_id as usize)?; let duration = range.duration as i64; + let flags = RangeFlags::from_bits(range.flags); let component_tz = Tz::from_id(range.start_tz)?; let mut own_recurrence_id = self .time_ranges @@ -232,6 +553,7 @@ impl CalendarEventData { .all(|prior| prior.id != range.id) .then(|| component.recurrence_id(component_tz)) .flatten(); + let this_and_future_tz = component.this_and_future_tz(component_tz); let mut start_tz = component_tz; let mut end_tz = Tz::from_id(range.end_tz)?; @@ -243,26 +565,42 @@ impl CalendarEventData { } let mut push_instance = |own_recurrence_id: Option, start_offset: u32| { - let start_date_naive = start_offset as i64 + base_offset; - let recurrence_id_naive = - own_recurrence_id.map_or(start_date_naive, |recurrence_id| recurrence_id.naive); + let start_date_naive = base_offset.saturating_add(start_offset as i64); + let series_recurrence_id = this_and_future_tz.and_then(|recurrence_tz| { + shifts.recurrence_id( + comp_id, + flags, + own_recurrence_id, + component_tz, + start_date_naive, + recurrence_tz, + ) + }); + let recurrence_id_naive = own_recurrence_id + .or(series_recurrence_id) + .map_or(start_date_naive, |recurrence_id| recurrence_id.naive); if RecurrenceKey::from_recurrence_id(recurrence_id_naive) .is_none_or(|key| !keys.contains(&key)) { return; } - let end_date_naive = start_date_naive + duration; + let end_date_naive = start_date_naive.saturating_add(duration); if let (Some(start), Some(end)) = ( - resolve_local(start_tz, start_date_naive), - resolve_local(end_tz, end_date_naive), + flags.resolve_start(start_tz, start_date_naive), + flags.resolve_end(end_tz, end_date_naive), ) { expansion.push(CalendarEventExpansion { comp_id, own_recurrence_id, - start, - end, + series_recurrence_id, + start: start.timestamp(), + end: end.timestamp(), start_naive: start_date_naive, + end_naive: end_date_naive, + start_tz, + end_tz, + flags, }); } }; @@ -287,197 +625,167 @@ impl CalendarEventData { Some(expansion) } - pub fn expand_single(&self, comp_id: u32, default_tz: Tz) -> Option { - let range = self.time_ranges.iter().find(|r| r.id as u32 == comp_id)?; - let instances = range.instances.as_ref(); - let (offset_or_count, bytes_read) = instances.read_leb128::()?; - let component_tz = Tz::from_id(range.start_tz)?; - let own_recurrence_id = self + pub fn expand_base(&self, default_tz: Tz) -> Option { + let (comp_id, component) = self .event .components - .get(comp_id as usize) - .and_then(|component| component.recurrence_id(component_tz)); - let mut start_tz = component_tz; - let mut end_tz = Tz::from_id(range.end_tz)?; - - if start_tz.is_floating() && !default_tz.is_floating() { - start_tz = default_tz; - } - if end_tz.is_floating() && !default_tz.is_floating() { - end_tz = default_tz; - } - let start_offset = if instances.len() > bytes_read { - let mut unpacker = - BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); - unpacker.next()? - } else { - offset_or_count + .iter() + .enumerate() + .filter(|(_, component)| component.component_type.is_scheduling_object()) + .min_by_key(|(_, component)| component.is_recurrence_override())?; + let comp_id = comp_id as u32; + let dtstart = component.property(&ICalendarProperty::Dtstart)?; + let start = dtstart.values.first()?.as_partial_date_time()?; + let start_date_time = start.to_date_time()?; + let start_date_naive = start_date_time.date_time.naive_timestamp(); + let effective_tz = |tz: Tz| { + if tz.is_floating() && !default_tz.is_floating() { + default_tz + } else { + tz + } }; - let start_date_naive = start_offset as i64 + self.base_offset; - let end_date_naive = start_date_naive + range.duration as i64; - let start = resolve_local(start_tz, start_date_naive)?; - let end = resolve_local(end_tz, end_date_naive)?; + let (component_tz, end_zoned) = match self + .time_ranges + .iter() + .find(|range| range.id as u32 == comp_id) + { + Some(range) => ( + Tz::from_id(range.start_tz)?, + RangeFlags::from_bits(range.flags).resolve_end( + effective_tz(Tz::from_id(range.end_tz)?), + start_date_naive.saturating_add(range.duration as i64), + )?, + ), + None => { + let resolver = self.event.build_tz_resolver(); + let start_tz = start_date_time + .tz() + .unwrap_or_else(|| resolver.resolve_or_default(dtstart.tz_id())); + let dtend = component.property(&ICalendarProperty::Dtend); + let end = dtend + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_partial_date_time) + .and_then(|value| value.to_date_time()); + let end_tz = end + .as_ref() + .and_then(|end| end.tz()) + .or_else(|| { + dtend + .and_then(|entry| entry.tz_id()) + .map(|tz_id| resolver.resolve_or_default(Some(tz_id))) + }) + .unwrap_or(start_tz); + let end = match ( + end, + component + .property(&ICalendarProperty::Duration) + .and_then(|entry| entry.values.first()), + ) { + (Some(end), _) => effective_tz(end_tz).from_local(end.date_time)?, + (None, Some(ICalendarValue::Duration(duration))) => effective_tz(start_tz) + .from_local(start_date_time.date_time)? + .checked_add_nominal(duration.to_nominal()?)?, + (None, _) if !start.has_time() => resolve_local_zoned( + effective_tz(end_tz), + start_date_naive + SECONDS_PER_DAY, + )?, + (None, _) => resolve_local_zoned(effective_tz(end_tz), start_date_naive)?, + }; + (start_tz, end) + } + }; + let start_zoned = resolve_local_zoned(effective_tz(component_tz), start_date_naive)?; Some(CalendarEventExpansion { comp_id, - own_recurrence_id, - start, - end, + own_recurrence_id: component.recurrence_id(component_tz), + series_recurrence_id: None, + start: start_zoned.timestamp(), + end: end_zoned.timestamp(), start_naive: start_date_naive, + end_naive: end_zoned.naive_timestamp(), + start_tz: start_zoned.timezone(), + end_tz: end_zoned.timezone(), + flags: RangeFlags::of(&start_zoned, &end_zoned), }) } } -impl Default for CalendarEventExpansion { - fn default() -> Self { - Self { - comp_id: u32::MAX, - own_recurrence_id: None, - start: i64::MAX, - end: i64::MAX, - start_naive: i64::MAX, +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub struct RangeFlags(u8); + +impl RangeFlags { + const LATER_START: u8 = 1; + const LATER_END: u8 = 1 << 1; + const CONDITION_SHIFT: u8 = 2; + const CONDITION_MASK: u8 = 0b111 << Self::CONDITION_SHIFT; + + pub fn of(start: &ZonedDateTime, end: &ZonedDateTime) -> Self { + let mut flags = RangeFlags::default(); + if Self::is_later(start) { + flags.0 |= Self::LATER_START; } + if Self::is_later(end) { + flags.0 |= Self::LATER_END; + } + flags + } + + pub fn with_condition(self, condition: OverlapCondition) -> Self { + RangeFlags( + (self.0 & !Self::CONDITION_MASK) + | (((condition as u8) << Self::CONDITION_SHIFT) & Self::CONDITION_MASK), + ) + } + + pub fn condition(self) -> OverlapCondition { + OverlapCondition::ALL + .get(usize::from( + (self.0 & Self::CONDITION_MASK) >> Self::CONDITION_SHIFT, + )) + .copied() + .unwrap_or_default() + } + + pub fn from_bits(bits: u8) -> Self { + RangeFlags(bits) + } + + pub fn bits(self) -> u8 { + self.0 + } + + pub fn resolve_start(self, tz: Tz, naive_secs: i64) -> Option { + self.resolve(Self::LATER_START, tz, naive_secs) + } + + pub fn resolve_end(self, tz: Tz, naive_secs: i64) -> Option { + self.resolve(Self::LATER_END, tz, naive_secs) + } + + fn resolve(self, flag: u8, tz: Tz, naive_secs: i64) -> Option { + let local = DateTime::from_naive_timestamp(naive_secs)?; + if self.0 & flag != 0 { + tz.from_local_later(local) + } else { + tz.from_local(local) + } + } + + fn is_later(date_time: &ZonedDateTime) -> bool { + date_time + .timezone() + .from_local(date_time.naive_local()) + .is_some_and(|earlier| earlier.timestamp() < date_time.timestamp()) } } pub fn resolve_local(tz: Tz, naive_secs: i64) -> Option { - tz.from_local_datetime(&DateTime::from_timestamp(naive_secs, 0)?.naive_local()) - .earliest() - .map(|dt| dt.timestamp()) + resolve_local_zoned(tz, naive_secs).map(|dt| dt.timestamp()) } -#[cfg(test)] -mod tests { - use super::*; - use calcard::{Entry, Parser}; - use chrono::NaiveDate; - - fn naive(year: i32, month: u32, day: u32, hour: u32, minute: u32, second: u32) -> i64 { - NaiveDate::from_ymd_opt(year, month, day) - .and_then(|date| date.and_hms_opt(hour, minute, second)) - .map(|date_time| date_time.and_utc().timestamp()) - .expect("valid date") - } - - fn key(year: i32, month: u32, day: u32, hour: u32, minute: u32) -> RecurrenceKey { - RecurrenceKey::from_recurrence_id(naive(year, month, day, hour, minute, 0)) - .expect("representable recurrence id") - } - - fn event_data(ical: &str) -> CalendarEventData { - let entry = Parser::new(ical).entry(); - let Entry::ICalendar(ical) = entry else { - panic!("failed to parse iCalendar: {entry:?}"); - }; - CalendarEventData::new(ical, Tz::UTC, 1000, &mut None) - } - - fn expand_key(data: &CalendarEventData, key: RecurrenceKey) -> Vec<(u32, i64)> { - let mut keys = AHashSet::from_iter([key]); - data.expand_from_ids(&mut keys, Tz::UTC) - .expect("expansion") - .into_iter() - .map(|expansion| (expansion.comp_id, expansion.start_naive)) - .collect() - } - - const MASTER: &str = concat!( - "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//Test//EN\r\n", - "BEGIN:VEVENT\r\nUID:u@example.com\r\nDTSTAMP:20270101T000000Z\r\n", - "DTSTART:20270301T090000Z\r\nDTEND:20270301T100000Z\r\n", - "RRULE:FREQ=WEEKLY;COUNT=5\r\nSUMMARY:Weekly\r\nEND:VEVENT\r\n", - ); - - const OVERRIDE: &str = concat!( - "BEGIN:VEVENT\r\nUID:u@example.com\r\nDTSTAMP:20270101T000000Z\r\n", - "RECURRENCE-ID:20270308T090000Z\r\nDTSTART:20270308T140000Z\r\n", - "DTEND:20270308T150000Z\r\nSUMMARY:Moved\r\nEND:VEVENT\r\n", - ); - - #[test] - fn recurrence_key_encoding() { - assert_eq!( - RecurrenceKey::from_recurrence_id(RECURRENCE_KEY_EPOCH), - Some(RecurrenceKey(1)) - ); - assert_eq!( - RecurrenceKey::from_recurrence_id(RECURRENCE_KEY_EPOCH - 1), - None - ); - assert_eq!(RecurrenceKey::from_recurrence_id(i64::MAX), None); - assert_eq!(RecurrenceKey::from_prefix(0), None); - assert_eq!( - RecurrenceKey::from_prefix(key(2027, 3, 15, 9, 0).prefix()), - Some(key(2027, 3, 15, 9, 0)) - ); - assert_ne!(key(2027, 3, 15, 9, 0), key(2027, 3, 15, 9, 1)); - assert_eq!( - RecurrenceKey::from_recurrence_id(naive(2027, 3, 15, 9, 0, 30)), - Some(key(2027, 3, 15, 9, 0)) - ); - } - - #[test] - fn recurrence_keys_survive_an_override() { - let before = event_data(&format!("{MASTER}END:VCALENDAR\r\n")); - let after = event_data(&format!("{MASTER}{OVERRIDE}END:VCALENDAR\r\n")); - - for (day, comp_id) in [(1, 1), (15, 1), (22, 1), (29, 1)] { - let key = key(2027, 3, day, 9, 0); - let start_naive = naive(2027, 3, day, 9, 0, 0); - assert_eq!(expand_key(&before, key), [(comp_id, start_naive)]); - assert_eq!(expand_key(&after, key), [(comp_id, start_naive)]); - } - - let overridden = key(2027, 3, 8, 9, 0); - assert_eq!( - expand_key(&before, overridden), - [(1, naive(2027, 3, 8, 9, 0, 0))] - ); - assert_eq!( - expand_key(&after, overridden), - [(2, naive(2027, 3, 8, 14, 0, 0))] - ); - } - - #[test] - fn this_and_future_instances_get_distinct_keys() { - const THIS_AND_FUTURE: &str = concat!( - "BEGIN:VEVENT\r\nUID:u@example.com\r\nDTSTAMP:20270101T000000Z\r\n", - "RECURRENCE-ID;RANGE=THISANDFUTURE:20270315T090000Z\r\n", - "DTSTART:20270315T100000Z\r\nDTEND:20270315T113000Z\r\n", - "SUMMARY:Longer\r\nEND:VEVENT\r\n", - ); - let data = event_data(&format!("{MASTER}{THIS_AND_FUTURE}END:VCALENDAR\r\n")); - - assert_eq!( - expand_key(&data, key(2027, 3, 15, 9, 0)), - [(2, naive(2027, 3, 15, 10, 0, 0))] - ); - assert_eq!( - expand_key(&data, key(2027, 3, 22, 10, 0)), - [(2, naive(2027, 3, 22, 10, 0, 0))] - ); - assert_eq!( - expand_key(&data, key(2027, 3, 29, 10, 0)), - [(2, naive(2027, 3, 29, 10, 0, 0))] - ); - assert_eq!( - expand_key(&data, key(2027, 3, 1, 9, 0)), - [(1, naive(2027, 3, 1, 9, 0, 0))] - ); - } - - #[test] - fn unmatched_recurrence_keys_are_reported_back() { - let data = event_data(&format!("{MASTER}END:VCALENDAR\r\n")); - let missing = key(2027, 4, 5, 9, 0); - let present = key(2027, 3, 15, 9, 0); - let mut keys = AHashSet::from_iter([missing, present]); - - let expansion = data.expand_from_ids(&mut keys, Tz::UTC).expect("expansion"); - - assert_eq!(expansion.len(), 1); - assert_eq!(keys.into_iter().collect::>(), [missing]); - } +pub fn resolve_local_zoned(tz: Tz, naive_secs: i64) -> Option { + tz.from_local(DateTime::from_naive_timestamp(naive_secs)?) } + diff --git a/crates/groupware/src/calendar/hidden.rs b/crates/groupware/src/calendar/hidden.rs new file mode 100644 index 00000000..b9db4b5a --- /dev/null +++ b/crates/groupware/src/calendar/hidden.rs @@ -0,0 +1,547 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + identity::CalendarAddresses, + participants::{ParticipantObject, ParticipantPointer}, + rights::EventValueCompare, +}; +use ahash::AHashSet; +use calcard::jscalendar::{JSCalendarProperty, JSCalendarValue}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; +use store::rand; +use types::{blob::BlobId, id::Id}; + +type EventKey<'x> = Key<'x, JSCalendarProperty>; +type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct HiddenAdditions { + pub base: bool, + pub overrides: bool, + pub ignored: usize, +} + +pub trait HiddenParticipantsMerge<'x> { + fn merge_visible_changes( + &mut self, + before: &EventValue<'x>, + after: EventValue<'x>, + ) -> HiddenAdditions; +} + +#[derive(Default)] +struct ParticipantRegistry { + ids: AHashSet, + addresses: AHashSet, + renamed: Vec<(String, Option)>, + ignored: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ParticipantScope { + Base, + Occurrence, +} + +struct OccurrenceBase<'a, 'x> { + full: Option<&'a EventMap<'x>>, + visible: Option<&'a EventMap<'x>>, +} + +impl<'x> HiddenParticipantsMerge<'x> for EventValue<'x> { + fn merge_visible_changes( + &mut self, + before: &EventValue<'x>, + after: EventValue<'x>, + ) -> HiddenAdditions { + let mut additions = HiddenAdditions::default(); + let (Some(full), Some(before), Value::Object(mut after)) = + (self.as_object_mut(), before.as_object(), after) + else { + return additions; + }; + let participants_key = Key::Property(JSCalendarProperty::Participants); + let overrides_key = Key::Property(JSCalendarProperty::RecurrenceOverrides); + let after_participants = after.remove(&participants_key); + let after_overrides = after.remove(&overrides_key); + let mut registry = ParticipantRegistry::new(full); + + for key in before.keys().filter(|key| { + **key != participants_key && **key != overrides_key && !after.contains_key(key) + }) { + full.remove(key); + } + for (key, value) in after.into_vec() { + match before.get(&key) { + Some(before_value) if before_value.same_value(&value) => {} + Some(before_value) if key == Key::Property(JSCalendarProperty::ICalendar) => { + match full.get_mut(&key) { + Some(full_value) => full_value.merge_object(before_value, value), + None => { + full.insert(key, value); + } + } + } + _ => { + full.insert(key, value); + } + } + } + + let before_participants = before.get(&participants_key); + let before_overrides = before.get(&overrides_key); + let has_override_changes = !after_overrides.as_ref().same_value(&before_overrides); + let visible_participants = has_override_changes + .then(|| { + after_participants + .as_ref() + .and_then(Value::as_object) + .cloned() + }) + .flatten(); + if !after_participants.as_ref().same_value(&before_participants) { + let full_participants = full + .insert_or_get_mut(participants_key.clone(), Value::Object(Map::new())) + .as_object_mut(); + if let Some(full_participants) = full_participants { + additions.base = registry.merge_participants( + full_participants, + before_participants.and_then(Value::as_object), + after_participants + .and_then(Value::into_object) + .unwrap_or_default(), + ParticipantScope::Base, + ); + } + if full + .get(&participants_key) + .and_then(Value::as_object) + .is_some_and(Map::is_empty) + { + full.remove(&participants_key); + } + } + + if has_override_changes { + let mut full_overrides = full + .remove(&overrides_key) + .and_then(Value::into_object) + .unwrap_or_default(); + additions.overrides = registry.merge_overrides( + &mut full_overrides, + before_overrides.and_then(Value::as_object), + after_overrides + .and_then(Value::into_object) + .unwrap_or_default(), + OccurrenceBase { + full: full.get(&participants_key).and_then(Value::as_object), + visible: visible_participants.as_ref(), + }, + ); + if !full_overrides.is_empty() { + full.insert(overrides_key, Value::Object(full_overrides)); + } + } + + additions.ignored = registry.ignored; + additions + } +} + +impl ParticipantRegistry { + fn new(full: &EventMap<'_>) -> Self { + let mut registry = ParticipantRegistry::default(); + if let Some(participants) = full + .get(&Key::Property(JSCalendarProperty::Participants)) + .and_then(Value::as_object) + { + registry.extend(participants); + } + for patch in full + .get(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .and_then(Value::as_object) + .into_iter() + .flat_map(|overrides| overrides.values()) + .filter_map(Value::as_object) + { + for (key, value) in patch.iter() { + if *key == Key::Property(JSCalendarProperty::Participants) { + if let Some(participants) = value.as_object() { + registry.extend(participants); + } + } else if let Some((id, pointer)) = key.participant_pointer() { + if pointer.is_empty() { + registry.register_address(value); + } + registry.register_id(&id); + } + } + } + registry + } + + fn extend(&mut self, participants: &EventMap<'_>) { + for (id, participant) in participants.iter() { + self.register_id(&id.to_string()); + self.register_address(participant); + } + } + + fn register_id(&mut self, id: &str) { + if !self.ids.contains(id) { + self.ids.insert(id.to_string()); + } + } + + fn register_address(&mut self, participant: &EventValue<'_>) { + if let Some(address) = participant.calendar_address() { + self.addresses + .insert(CalendarAddresses::normalize(&address).into_owned()); + } + } + + fn has_id(&self, id: &str) -> bool { + self.ids.contains(id) + } + + fn has_address(&self, participant: &EventValue<'_>) -> bool { + participant.calendar_address().is_some_and(|address| { + self.addresses + .contains(CalendarAddresses::normalize(&address).as_ref()) + }) + } + + fn fresh_id(&mut self) -> String { + loop { + let id = format!("{:x}", rand::random::()); + if !self.ids.contains(&id) { + self.ids.insert(id.clone()); + return id; + } + } + } + + fn admit<'x>( + &mut self, + id: EventKey<'x>, + participant: &EventValue<'x>, + is_taken: bool, + scope: ParticipantScope, + ) -> Option> { + let name = id.to_string(); + if self.has_address(participant) { + self.ignored += 1; + if scope == ParticipantScope::Base { + self.renamed.push((name.into_owned(), None)); + } + return None; + } + self.register_address(participant); + if is_taken || self.has_id(&name) { + let fresh_id = self.fresh_id(); + if scope == ParticipantScope::Base { + self.renamed + .push((name.into_owned(), Some(fresh_id.clone()))); + } + Some(Key::Owned(fresh_id)) + } else { + self.ids.insert(name.into_owned()); + Some(id) + } + } + + fn renamed_id(&self, id: &str) -> Option> { + self.renamed + .iter() + .find(|(old_id, _)| old_id == id) + .map(|(_, new_id)| new_id.as_deref()) + } + + fn rename_patch_keys<'x>(&self, patch: EventMap<'x>) -> EventMap<'x> { + if self.renamed.is_empty() { + return patch; + } + let mut renamed = Map::with_capacity(patch.len()); + for (key, value) in patch.into_vec() { + let target = key + .participant_pointer() + .and_then(|(id, rest)| Some((self.renamed_id(&id)?.map(str::to_string), rest))) + .map(|(new_id, rest)| { + new_id.map(|new_id| { + let mut items = Vec::with_capacity(rest.len() + 2); + items.push(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::Participants, + ))); + items.push(JsonPointerItem::Key(Key::Owned(new_id))); + items.extend(rest.iter().cloned()); + Key::Property(JSCalendarProperty::Pointer(JsonPointer::new(items))) + }) + }); + match target { + Some(Some(key)) => renamed.insert_unchecked(key, value), + Some(None) => {} + None => renamed.insert_unchecked(key, value), + } + } + renamed + } + + fn rename_participant_ids<'x>(&self, participants: EventMap<'x>) -> EventMap<'x> { + if self.renamed.is_empty() { + return participants; + } + let mut renamed = Map::with_capacity(participants.len()); + for (id, participant) in participants.into_vec() { + match self.renamed_id(&id.to_string()) { + Some(Some(new_id)) => { + renamed.insert_unchecked(Key::Owned(new_id.to_string()), participant) + } + Some(None) => {} + None => renamed.insert_unchecked(id, participant), + } + } + renamed + } + + fn merge_participants<'x>( + &mut self, + full: &mut EventMap<'x>, + before: Option<&EventMap<'x>>, + after: EventMap<'x>, + scope: ParticipantScope, + ) -> bool { + let mut has_additions = false; + if let Some(before) = before { + for id in before.keys().filter(|id| !after.contains_key(id)) { + full.remove(id); + } + } + for (id, participant) in after.into_vec() { + match before.and_then(|before| before.get(&id)) { + Some(before_participant) if before_participant.same_value(&participant) => {} + Some(before_participant) => match full.get_mut(&id) { + Some(full_participant) => { + full_participant.merge_object(before_participant, participant) + } + None => { + full.insert(id, participant); + } + }, + None => { + has_additions = true; + let is_taken = full.contains_key(&id); + if let Some(id) = self.admit(id, &participant, is_taken, scope) { + full.insert(id, participant); + } + } + } + } + has_additions + } + + fn merge_overrides<'x>( + &mut self, + full: &mut EventMap<'x>, + before: Option<&EventMap<'x>>, + after: EventMap<'x>, + base: OccurrenceBase<'_, 'x>, + ) -> bool { + let mut has_additions = false; + if let Some(before) = before { + for (recurrence_id, before_patch) in before + .iter() + .filter(|(recurrence_id, _)| !after.contains_key(recurrence_id)) + { + let is_empty = match full.get_mut(recurrence_id).and_then(Value::as_object_mut) { + Some(full_patch) => { + for key in before_patch.as_object().into_iter().flat_map(Map::keys) { + full_patch.remove(key); + } + full_patch.is_empty() + } + None => false, + }; + if is_empty { + full.remove(recurrence_id); + } + } + } + + for (recurrence_id, after_patch) in after.into_vec() { + let before_patch = before.and_then(|before| before.get(&recurrence_id)); + if before_patch.is_some_and(|before_patch| before_patch.same_value(&after_patch)) { + continue; + } + let Value::Object(after_patch) = after_patch else { + full.insert(recurrence_id, after_patch); + continue; + }; + let before_patch = before_patch.and_then(Value::as_object); + match full.get_mut(&recurrence_id).and_then(Value::as_object_mut) { + Some(full_patch) => { + has_additions |= self.merge_patch(full_patch, before_patch, after_patch, &base); + } + None => { + let mut full_patch = Map::new(); + has_additions |= + self.merge_patch(&mut full_patch, before_patch, after_patch, &base); + full.insert(recurrence_id, Value::Object(full_patch)); + } + } + } + has_additions + } + + fn merge_patch<'x>( + &mut self, + full: &mut EventMap<'x>, + before: Option<&EventMap<'x>>, + after: EventMap<'x>, + base: &OccurrenceBase<'_, 'x>, + ) -> bool { + let mut has_additions = false; + let after = self.rename_patch_keys(after); + if let Some(before) = before { + for key in before.keys().filter(|key| !after.contains_key(key)) { + full.remove(key); + } + } + for (key, value) in after.into_vec() { + let before_value = before.and_then(|before| before.get(&key)); + if before_value.is_some_and(|before_value| before_value.same_value(&value)) { + continue; + } + if key == Key::Property(JSCalendarProperty::Participants) { + let Value::Object(participants) = value else { + full.insert(key, value); + continue; + }; + let participants = self.rename_participant_ids(participants); + let has_occurrence_set = matches!(full.get(&key), Some(Value::Object(_))); + let visible = if has_occurrence_set { + before_value.and_then(Value::as_object) + } else { + full.insert( + key.clone(), + Value::Object(base.full.cloned().unwrap_or_default()), + ); + base.visible + }; + if let Some(full_participants) = full.get_mut(&key).and_then(Value::as_object_mut) { + has_additions |= self.merge_participants( + full_participants, + visible, + participants, + ParticipantScope::Occurrence, + ); + } + continue; + } + + let added_id = before_value + .is_none() + .then(|| key.participant_pointer()) + .flatten() + .filter(|(_, pointer)| pointer.is_empty() && value.as_object().is_some()) + .map(|(id, _)| id.into_owned()) + .filter(|id| { + base.visible + .is_none_or(|visible| !visible.keys().any(|key| key.to_string() == *id)) + }); + match added_id { + Some(id) => { + has_additions = true; + let is_taken = full.contains_key(&key) + || base.full.is_some_and(|participants| { + participants.keys().any(|key| key.to_string() == id) + }); + if let Some(id) = self.admit( + Key::Owned(id), + &value, + is_taken, + ParticipantScope::Occurrence, + ) { + let id = match id { + Key::Owned(id) => id, + id => id.to_string().into_owned(), + }; + full.remove_participant_patches(&id); + full.insert(participant_key(id), value); + } + } + None => match (before_value, full.get_mut(&key)) { + (Some(before_value), Some(full_value)) => { + full_value.merge_object(before_value, value) + } + _ => { + if let Some((id, [])) = key.participant_pointer() { + let id = id.into_owned(); + full.remove_participant_patches(&id); + } + full.insert(key, value); + } + }, + } + } + has_additions + } +} + +trait ParticipantPatches { + fn remove_participant_patches(&mut self, id: &str); +} + +impl ParticipantPatches for EventMap<'_> { + fn remove_participant_patches(&mut self, id: &str) { + self.as_mut_vec().retain(|(key, _)| { + key.participant_pointer() + .is_none_or(|(patch_id, pointer)| pointer.is_empty() || patch_id != id) + }); + } +} + +trait ThreeWayMerge<'x> { + fn merge_object(&mut self, before: &EventValue<'x>, after: EventValue<'x>); +} + +impl<'x> ThreeWayMerge<'x> for EventValue<'x> { + fn merge_object(&mut self, before: &EventValue<'x>, after: EventValue<'x>) { + let after = match (self.as_object_mut(), before.as_object(), after) { + (Some(full), Some(before), Value::Object(after)) => { + for key in before.keys().filter(|key| !after.contains_key(key)) { + full.remove(key); + } + for (key, value) in after.into_vec() { + match before.get(&key) { + Some(before_value) if before_value.same_value(&value) => {} + Some(before_value) => match full.get_mut(&key) { + Some(full_value) => full_value.merge_object(before_value, value), + None => { + full.insert(key, value); + } + }, + None => { + full.insert(key, value); + } + } + } + return; + } + (_, _, after) => after, + }; + *self = after; + } +} + +fn participant_key(id: String) -> EventKey<'static> { + Key::Property(JSCalendarProperty::Pointer(JsonPointer::new(vec![ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + JsonPointerItem::Key(Key::Owned(id)), + ]))) +} + diff --git a/crates/groupware/src/calendar/identity.rs b/crates/groupware/src/calendar/identity.rs new file mode 100644 index 00000000..9875666f --- /dev/null +++ b/crates/groupware/src/calendar/identity.rs @@ -0,0 +1,468 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ParticipantIdentities, ParticipantIdentityChange, ParticipantIdentityChangeType, + participants::ArchivedParticipantEntry, +}; +use calcard::icalendar::{ + ArchivedICalendar, ArchivedICalendarProperty, ICalendar, ICalendarComponent, ICalendarEntry, + ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationRole, ICalendarProperty, +}; +use common::{Server, auth::AccountInfo}; +use std::borrow::Cow; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{collection::Collection, field::PrincipalField}; + +const MAILTO_SCHEME: &str = "mailto"; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct CalendarAddresses(Vec); + +impl CalendarAddresses { + pub fn from_emails<'x>(emails: impl IntoIterator) -> Self { + let mut addresses = CalendarAddresses::default(); + for email in emails { + addresses.push(CalendarAddresses::normalize(email)); + } + addresses + } + + pub fn contains(&self, calendar_address: &str) -> bool { + let normalized = CalendarAddresses::normalize(calendar_address); + self.0.iter().any(|address| address == normalized.as_ref()) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn push(&mut self, address: Cow<'_, str>) { + if !self.0.iter().any(|existing| existing == address.as_ref()) { + self.0.push(address.into_owned()); + } + } + + pub fn normalize(calendar_address: &str) -> Cow<'_, str> { + let calendar_address = calendar_address.trim(); + let (scheme, rest) = match calendar_address.split_once(':') { + Some((scheme, rest)) if is_scheme(scheme) => (scheme, rest), + _ => { + let mut address = + String::with_capacity(MAILTO_SCHEME.len() + 1 + calendar_address.len()); + address.push_str(MAILTO_SCHEME); + address.push(':'); + address.push_str(&normalize_percent_encoding(calendar_address).to_lowercase()); + return Cow::Owned(address); + } + }; + + if scheme == MAILTO_SCHEME + && !rest + .bytes() + .any(|b| b.is_ascii_uppercase() || b == b'%' || !b.is_ascii()) + { + return Cow::Borrowed(calendar_address); + } + + let scheme = scheme.to_ascii_lowercase(); + let rest = normalize_percent_encoding(rest); + let rest = if scheme == MAILTO_SCHEME { + rest.to_lowercase() + } else { + normalize_hierarchical_part(&rest) + }; + let mut address = String::with_capacity(scheme.len() + 1 + rest.len()); + address.push_str(&scheme); + address.push(':'); + address.push_str(&rest); + Cow::Owned(address) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventOwnership { + NoOwner, + Owner, + NotOwner, +} + +impl CalendarAddresses { + pub fn event_ownership(&self, ical: &ICalendar) -> EventOwnership { + let is_series = |component: &ICalendarComponent| !component.is_recurrence_override(); + let scheduling_components = || { + ical.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + }; + let has_series = scheduling_components().any(is_series); + let mut ownership = EventOwnership::NoOwner; + for entry in scheduling_components() + .filter(|component| is_series(component) == has_series) + .flat_map(|component| component.entries.iter()) + .filter(|entry| entry.is_owner_entry()) + { + if entry + .calendar_address() + .is_some_and(|address| self.contains(address)) + { + return EventOwnership::Owner; + } + ownership = EventOwnership::NotOwner; + } + + ownership + } + + pub fn archived_event_ownership(&self, ical: &ArchivedICalendar) -> EventOwnership { + let scheduling_components = || { + ical.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + }; + let has_series = + scheduling_components().any(|component| !component.is_recurrence_override()); + let mut ownership = EventOwnership::NoOwner; + for entry in scheduling_components() + .filter(|component| component.is_recurrence_override() != has_series) + .flat_map(|component| component.entries.iter()) + .filter(|entry| match entry.name { + ArchivedICalendarProperty::Organizer => true, + ArchivedICalendarProperty::Attendee => entry.has_owner_role(), + _ => false, + }) + { + if entry + .values + .first() + .and_then(|value| value.as_text()) + .is_some_and(|address| self.contains(address)) + { + return EventOwnership::Owner; + } + ownership = EventOwnership::NotOwner; + } + + ownership + } +} + +impl EventOwnership { + pub fn may_write_own(self) -> bool { + self != EventOwnership::NotOwner + } +} + +pub(crate) trait OwnerEntry { + fn is_owner_entry(&self) -> bool; +} + +impl OwnerEntry for ICalendarEntry { + fn is_owner_entry(&self) -> bool { + match self.name { + ICalendarProperty::Organizer => true, + ICalendarProperty::Attendee => { + self.parameters(&ICalendarParameterName::Role).any(|role| { + matches!( + role, + ICalendarParameterValue::Role(ICalendarParticipationRole::Owner) + ) + }) + } + _ => false, + } + } +} + +const MAX_IDENTITY_CHANGES: usize = 16; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct IdentityChanges { + pub created: Vec, + pub updated: Vec, + pub destroyed: Vec, + pub new_change_id: u32, + pub has_more_changes: bool, +} + +struct PendingIdentityChange { + id: u32, + first: ParticipantIdentityChangeType, + last: ParticipantIdentityChangeType, +} + +impl ParticipantIdentities { + pub fn begin_changes(&mut self) { + self.change_id += 1; + } + + pub fn record_change(&mut self, id: u32, change_type: ParticipantIdentityChangeType) { + self.changes.push(ParticipantIdentityChange { + change_id: self.change_id, + id, + change_type, + }); + while self.changes.len() > MAX_IDENTITY_CHANGES { + let Some(oldest) = self.changes.first().map(|change| change.change_id) else { + break; + }; + self.changes.retain(|change| change.change_id != oldest); + self.trimmed_change_id = oldest; + } + } + + pub fn changes_since(&self, since: u32, max_changes: usize) -> Option { + if since > self.change_id || since < self.trimmed_change_id { + return None; + } + + let mut changes = IdentityChanges { + new_change_id: since, + ..Default::default() + }; + let mut pending: Vec = Vec::new(); + let mut total = 0; + for group in self + .changes + .chunk_by(|a, b| a.change_id == b.change_id) + .filter(|group| group.first().is_some_and(|change| change.change_id > since)) + { + if total + group.len() > max_changes { + if total == 0 { + return None; + } + changes.has_more_changes = true; + break; + } + total += group.len(); + for change in group { + match pending.iter_mut().find(|pending| pending.id == change.id) { + Some(pending) => pending.last = change.change_type, + None => pending.push(PendingIdentityChange { + id: change.id, + first: change.change_type, + last: change.change_type, + }), + } + } + if let Some(change) = group.first() { + changes.new_change_id = change.change_id; + } + } + if !changes.has_more_changes { + changes.new_change_id = self.change_id.max(changes.new_change_id); + } + + for pending in pending { + match (pending.first, pending.last) { + ( + ParticipantIdentityChangeType::Created, + ParticipantIdentityChangeType::Destroyed, + ) => {} + (ParticipantIdentityChangeType::Created, _) => changes.created.push(pending.id), + (_, ParticipantIdentityChangeType::Destroyed) => changes.destroyed.push(pending.id), + _ => changes.updated.push(pending.id), + } + } + + Some(changes) + } + + pub fn calendar_addresses(&self, allowed: &CalendarAddresses) -> CalendarAddresses { + let mut addresses = CalendarAddresses::default(); + for address in self + .identities + .iter() + .map(|identity| CalendarAddresses::normalize(&identity.calendar_address)) + { + if allowed.0.iter().any(|allowed| allowed == address.as_ref()) { + addresses.push(address); + } + } + addresses + } +} + +pub trait ParticipantIdentityAddresses: Sync + Send { + fn account_identity_addresses( + &self, + account_id: u32, + ) -> impl Future> + Send; + + fn identity_addresses( + &self, + account_id: u32, + account_info: &AccountInfo, + ) -> impl Future> + Send; +} + +impl ParticipantIdentityAddresses for Server { + async fn account_identity_addresses(&self, account_id: u32) -> trc::Result { + let account_info = self + .account_info(account_id) + .await + .caused_by(trc::location!())?; + self.identity_addresses(account_id, &account_info).await + } + + async fn identity_addresses( + &self, + account_id: u32, + account_info: &AccountInfo, + ) -> trc::Result { + let allowed = + CalendarAddresses::from_emails(account_info.addresses().iter().map(String::as_str)); + let Some(archive) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::Principal, + 0, + PrincipalField::ParticipantIdentities, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(allowed); + }; + + archive + .deserialize::() + .map(|identities| identities.calendar_addresses(&allowed)) + .caused_by(trc::location!()) + } +} + +fn is_scheme(scheme: &str) -> bool { + let mut bytes = scheme.bytes(); + bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) + && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) +} + +fn normalize_percent_encoding(value: &str) -> Cow<'_, str> { + if !value.contains('%') { + return Cow::Borrowed(value); + } + + let mut out = Vec::with_capacity(value.len()); + let mut bytes = value.bytes(); + while let Some(byte) = bytes.next() { + if byte != b'%' { + out.push(byte); + continue; + } + let mut lookahead = bytes.clone(); + match ( + lookahead.next().and_then(hex_value), + lookahead.next().and_then(hex_value), + ) { + (Some(high), Some(low)) => { + bytes = lookahead; + let decoded = (high << 4) | low; + if decoded.is_ascii_alphanumeric() || matches!(decoded, b'-' | b'.' | b'_' | b'~') { + out.push(decoded); + } else { + out.push(b'%'); + out.push(hex_digit(high)); + out.push(hex_digit(low)); + } + } + _ => out.push(byte), + } + } + + String::from_utf8(out).map_or(Cow::Borrowed(value), Cow::Owned) +} + +fn hex_value(byte: u8) -> Option { + char::from(byte).to_digit(16).map(|digit| digit as u8) +} + +fn hex_digit(value: u8) -> u8 { + b"0123456789ABCDEF" + .get(value as usize) + .copied() + .unwrap_or(b'0') +} + +fn normalize_hierarchical_part(rest: &str) -> String { + let path_len = rest + .bytes() + .take_while(|b| !matches!(b, b'?' | b'#')) + .count(); + let (before_query, query) = rest.split_at_checked(path_len).unwrap_or((rest, "")); + let mut out = String::with_capacity(rest.len()); + let path = if let Some(authority_and_path) = before_query.strip_prefix("//") { + let authority_len = authority_and_path + .bytes() + .take_while(|b| *b != b'/') + .count(); + let (authority, path) = authority_and_path + .split_at_checked(authority_len) + .unwrap_or((authority_and_path, "")); + out.push_str("//"); + match authority.rsplit_once('@') { + Some((user_info, host)) => { + out.push_str(user_info); + out.push('@'); + out.push_str(&host.to_ascii_lowercase()); + } + None => out.push_str(&authority.to_ascii_lowercase()), + } + path + } else { + before_query + }; + + if path.starts_with('/') || rest.starts_with("//") { + remove_dot_segments(path, &mut out); + } else { + out.push_str(path); + } + out.push_str(query); + out +} + +fn remove_dot_segments(path: &str, out: &mut String) { + let is_absolute = path.starts_with('/'); + let mut segments = Vec::new(); + let mut has_trailing_slash = false; + let mut iter = path.split('/'); + if is_absolute { + iter.next(); + } + for segment in iter { + match segment { + "." => has_trailing_slash = true, + ".." => { + segments.pop(); + has_trailing_slash = true; + } + segment => { + segments.push(segment); + has_trailing_slash = false; + } + } + } + + if is_absolute { + out.push('/'); + } + for (idx, segment) in segments.iter().enumerate() { + if idx > 0 { + out.push('/'); + } + out.push_str(segment); + } + if has_trailing_slash && !segments.is_empty() { + out.push('/'); + } +} + diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index 74a4d8fe..24e75fc7 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -9,34 +9,50 @@ use super::{ ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, }; use crate::{ + MetaHasher, SizeWriter, calendar::{ - ArchivedCalendarEventNotification, ArchivedChangedBy, ArchivedEventPreferences, - CalendarEventNotification, ChangedBy, EventPreferences, + ArchivedCalendarEventContent, ArchivedCalendarEventNotification, + ArchivedCalendarEventNotificationContent, ArchivedChangedBy, CalendarEventContent, + CalendarEventNotification, CalendarEventNotificationContent, ChangedBy, + EVENT_FLOATING_START, EVENT_HAS_ALARMS, EVENT_HAS_UNBOUNDED_TODO, + EVENT_USES_DEFAULT_ALERTS, EventPreferences, PREF_HAS_ALERTS, + participants::ArchivedParticipantEntry, privacy::ICalendarPrivacy, }, strip_mailto_scheme, }; use ahash::AHashSet; use calcard::icalendar::{ - ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarValue, + ArchivedICalendar, ArchivedICalendarComponentType, ArchivedICalendarParameterValue, + ArchivedICalendarProperty, ArchivedICalendarValue, ICalendar, ICalendarComponentType, ICalendarParameterValue, ICalendarProperty, ICalendarValue, }; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use common::storage::{ + dav::PresenceBits, + index::{ + ArchivedSplitObject, IndexValue, IndexableAndSerializableObject, IndexableObject, + SEARCH_HASH_UNCHANGED, SerializableObject, SplitObject, serialize_object, + }, +}; use nlp::language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }; +use registry::schema::enums::StorageQuota; use store::{ U32_LEN, search::{CalendarSearchField, IndexDocument, SearchField}, - write::{IndexPropertyClass, SearchIndex, ValueClass}, + write::{ArchiveCompression, BatchBuilder, Compression, Dictionary, SearchIndex, Slot}, xxhash_rust::xxh3, }; use types::{ acl::AclGrant, collection::SyncCollection, - field::{CalendarEventField, CalendarNotificationField}, + field::{CalendarEventField, CalendarNotificationField, Field}, + metadata::MetadataKinds, }; +const EVENT_HASHED_FLAGS: u16 = !PresenceBits::CALENDAR_EVENT.mask(); + impl IndexableObject for Calendar { fn index_values(&self) -> impl Iterator> { [ @@ -52,6 +68,10 @@ impl IndexableObject for Calendar { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + Calendar::metadata_kinds(self) + } } impl IndexableObject for &ArchivedCalendar { @@ -74,6 +94,10 @@ impl IndexableObject for &ArchivedCalendar { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedCalendar::metadata_kinds(self) + } } impl IndexableAndSerializableObject for Calendar { @@ -82,55 +106,45 @@ impl IndexableAndSerializableObject for Calendar { } } +impl SerializableObject for Calendar { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} + impl IndexableObject for CalendarEvent { fn index_values(&self) -> impl Iterator> { [ IndexValue::SearchIndex { index: SearchIndex::Calendar, - hash: self - .hashes() - .chain([self.data.event_range_start() as u64]) - .fold(0, |acc, hash| acc ^ hash), + hash: SEARCH_HASH_UNCHANGED, }, IndexValue::Index { field: CalendarEventField::Uid.into(), - value: self.data.event.uids().next().into(), - }, - IndexValue::Quota { - used: self.size() as u32, + value: self.uid.as_str().into(), }, + IndexValue::Quota { used: self.size }, IndexValue::LogItem { sync_collection: SyncCollection::Calendar, prefix: None, }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEvents), ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + CalendarEvent::metadata_kinds(self) + } } impl IndexableObject for &ArchivedCalendarEvent { fn index_values(&self) -> impl Iterator> { - [ - IndexValue::SearchIndex { - index: SearchIndex::Calendar, - hash: self - .hashes() - .chain([self.data.event_range_start() as u64]) - .fold(0, |acc, hash| acc ^ hash), - }, - IndexValue::Index { - field: CalendarEventField::Uid.into(), - value: self.data.event.uids().next().into(), - }, - IndexValue::Quota { - used: self.size() as u32, - }, - IndexValue::LogItem { - sync_collection: SyncCollection::Calendar, - prefix: None, - }, - ] - .into_iter() + self.meta_index_values().into_iter() + } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedCalendarEvent::metadata_kinds(self) } } @@ -138,25 +152,218 @@ impl IndexableAndSerializableObject for CalendarEvent { fn is_versioned() -> bool { true } + + fn set_pending_id(&mut self, document_id: u32) -> trc::Result<()> { + let content_hash = self.etag ^ self.meta_hash(); + self.names + .last_mut() + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Pending calendar id requires a name.") + })? + .parent_id = document_id; + self.etag = content_hash ^ self.meta_hash(); + Ok(()) + } + + fn size_hint(&self) -> usize { + self.meta_size_hint() + } +} + +impl SplitObject for CalendarEvent { + type Content = CalendarEventContent; + type Context = (); + + const CONTENT_FIELD: Field = CalendarEventField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher.u32(self.names.len() as u32); + for name in &self.names { + hasher.str(&name.name).u32(name.parent_id); + } + hasher + .str(&self.uid) + .opt_str(self.display_name.as_deref()) + .i64(self.start) + .u32(self.duration) + .i64(self.created) + .i64(self.modified) + .u32(self.size) + .u16(self.flags & EVENT_HASHED_FLAGS) + .opt_u32(self.schedule_tag) + .finish() + } + + fn set_etag(&mut self, etag: u32) { + self.etag = etag; + } + + fn etag_value(&self) -> u32 { + self.etag + } + + fn refresh_from_content(&mut self, content: &CalendarEventContent, _ctx: ()) { + self.uid = content + .data + .event + .object_uid() + .unwrap_or_default() + .to_string(); + let (start, duration) = content.data.event_range().unwrap_or_default(); + self.start = start; + self.duration = duration; + self.size = SizeWriter::ical(&content.data.event) as u32; + if content.data.has_unbounded_todo() { + self.flags |= EVENT_HAS_UNBOUNDED_TODO; + } else { + self.flags &= !EVENT_HAS_UNBOUNDED_TODO; + } + if content.data.has_floating_start() { + self.flags |= EVENT_FLOATING_START; + } else { + self.flags &= !EVENT_FLOATING_START; + } + let alarm_flags = if content + .preferences + .iter() + .any(EventPreferences::use_default_alerts) + { + EVENT_HAS_ALARMS | EVENT_USES_DEFAULT_ALERTS + } else if !content.data.alarms.is_empty() + || content.preferences.iter().any(|preferences| { + preferences.instances.iter().any(|instance| { + instance.flags & PREF_HAS_ALERTS != 0 && !instance.alerts.is_empty() + }) + }) + { + EVENT_HAS_ALARMS + } else { + 0 + }; + self.flags = (self.flags & !(EVENT_HAS_ALARMS | EVENT_USES_DEFAULT_ALERTS)) | alarm_flags; + self.flags = content.data.event.privacy().apply_to_flags(self.flags); + } + + fn full_index_values<'x>(&'x self, content: &'x CalendarEventContent) -> Vec> { + vec![ + IndexValue::SearchIndex { + index: SearchIndex::Calendar, + hash: content + .hashes() + .chain([content.data.event_range_start() as u64]) + .fold(0, |acc, hash| acc ^ hash), + }, + IndexValue::Index { + field: CalendarEventField::Uid.into(), + value: self.uid.as_str().into(), + }, + IndexValue::Quota { used: self.size }, + IndexValue::LogItem { + sync_collection: SyncCollection::Calendar, + prefix: None, + }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEvents), + ] + } +} + +impl ArchivedSplitObject for ArchivedCalendarEvent { + type ArchivedContent = ArchivedCalendarEventContent; + + const CONTENT_FIELD: Field = CalendarEventField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher.u32(self.names.len() as u32); + for name in self.names.iter() { + hasher.str(&name.name).u32(name.parent_id.to_native()); + } + hasher + .str(&self.uid) + .opt_str(self.display_name.as_deref()) + .i64(self.start.to_native()) + .u32(self.duration.to_native()) + .i64(self.created.to_native()) + .i64(self.modified.to_native()) + .u32(self.size.to_native()) + .u16(self.flags.to_native() & EVENT_HASHED_FLAGS) + .opt_u32(self.schedule_tag.as_ref().map(|tag| tag.to_native())) + .finish() + } + + fn etag(&self) -> u32 { + self.etag.to_native() + } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedCalendarEvent::metadata_kinds(self) + } + + fn meta_index_values(&self) -> Vec> { + vec![ + IndexValue::SearchIndex { + index: SearchIndex::Calendar, + hash: SEARCH_HASH_UNCHANGED, + }, + IndexValue::Index { + field: CalendarEventField::Uid.into(), + value: self.uid.as_str().into(), + }, + IndexValue::Quota { + used: self.size.to_native(), + }, + IndexValue::LogItem { + sync_collection: SyncCollection::Calendar, + prefix: None, + }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEvents), + ] + } + + fn full_index_values<'x>( + &'x self, + content: &'x ArchivedCalendarEventContent, + ) -> Vec> { + vec![ + IndexValue::SearchIndex { + index: SearchIndex::Calendar, + hash: content + .hashes() + .chain([content.data.event_range_start() as u64]) + .fold(0, |acc, hash| acc ^ hash), + }, + IndexValue::Index { + field: CalendarEventField::Uid.into(), + value: self.uid.as_str().into(), + }, + IndexValue::Quota { + used: self.size.to_native(), + }, + IndexValue::LogItem { + sync_collection: SyncCollection::Calendar, + prefix: None, + }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEvents), + ] + } } impl IndexableObject for CalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Quota { - used: self.size() as u32, - }, - IndexValue::Property { - field: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: self.created as u64, - }), - value: self.event_id.unwrap_or(u32::MAX).into(), + IndexValue::Quota { used: self.size }, + IndexValue::Index { + field: CalendarNotificationField::Created.into(), + value: self.created.into(), }, IndexValue::LogItem { sync_collection: SyncCollection::CalendarEventNotification, prefix: None, }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEventNotifications), ] .into_iter() } @@ -164,28 +371,7 @@ impl IndexableObject for CalendarEventNotification { impl IndexableObject for &ArchivedCalendarEventNotification { fn index_values(&self) -> impl Iterator> { - [ - IndexValue::Quota { - used: self.size() as u32, - }, - IndexValue::Property { - field: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: self.created.to_native() as u64, - }), - value: self - .event_id - .as_ref() - .map(|v| v.to_native()) - .unwrap_or(u32::MAX) - .into(), - }, - IndexValue::LogItem { - sync_collection: SyncCollection::CalendarEventNotification, - prefix: None, - }, - ] - .into_iter() + self.meta_index_values().into_iter() } } @@ -193,12 +379,174 @@ impl IndexableAndSerializableObject for CalendarEventNotification { fn is_versioned() -> bool { false } + + fn set_pending_id(&mut self, document_id: u32) -> trc::Result<()> { + let content_hash = self.etag ^ self.meta_hash(); + self.event_id = Some(document_id); + self.etag = content_hash ^ self.meta_hash(); + Ok(()) + } + + fn size_hint(&self) -> usize { + self.meta_size_hint() + } +} + +impl SplitObject for CalendarEventNotification { + type Content = CalendarEventNotificationContent; + type Context = (); + + const CONTENT_FIELD: Field = CalendarNotificationField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + match &self.changed_by { + ChangedBy::PrincipalId(id) => { + hasher.u16(0).u32(*id); + } + ChangedBy::CalendarAddress(address) => { + hasher.u16(1).str(address); + } + } + for calendar_id in &self.calendar_ids { + hasher.u32(*calendar_id); + } + hasher + .opt_u32(self.event_id) + .i64(self.created) + .i64(self.modified) + .u32(self.size) + .u16(self.flags) + .u32(self.calendar_ids.len() as u32) + .finish() + } + + fn set_etag(&mut self, etag: u32) { + self.etag = etag; + } + + fn etag_value(&self) -> u32 { + self.etag + } + + fn refresh_from_content(&mut self, content: &CalendarEventNotificationContent, _ctx: ()) { + self.size = content + .snapshots() + .into_iter() + .flatten() + .filter(|snapshot| !snapshot.components.is_empty()) + .map(SizeWriter::ical) + .sum::() as u32; + } + + fn full_index_values<'x>( + &'x self, + _content: &'x CalendarEventNotificationContent, + ) -> Vec> { + self.index_values().collect() + } +} + +impl ArchivedSplitObject for ArchivedCalendarEventNotification { + type ArchivedContent = ArchivedCalendarEventNotificationContent; + + const CONTENT_FIELD: Field = CalendarNotificationField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + match &self.changed_by { + ArchivedChangedBy::PrincipalId(id) => { + hasher.u16(0).u32(id.to_native()); + } + ArchivedChangedBy::CalendarAddress(address) => { + hasher.u16(1).str(address); + } + } + for calendar_id in self.calendar_ids.iter() { + hasher.u32(calendar_id.to_native()); + } + hasher + .opt_u32(self.event_id.as_ref().map(|id| id.to_native())) + .i64(self.created.to_native()) + .i64(self.modified.to_native()) + .u32(self.size.to_native()) + .u16(self.flags.to_native()) + .u32(self.calendar_ids.len() as u32) + .finish() + } + + fn etag(&self) -> u32 { + self.etag.to_native() + } + + fn meta_index_values(&self) -> Vec> { + vec![ + IndexValue::Quota { + used: self.size.to_native(), + }, + IndexValue::Index { + field: CalendarNotificationField::Created.into(), + value: self.created.to_native().into(), + }, + IndexValue::LogItem { + sync_collection: SyncCollection::CalendarEventNotification, + prefix: None, + }, + IndexValue::ObjectCount(StorageQuota::MaxCalendarEventNotifications), + ] + } + + fn full_index_values<'x>( + &'x self, + _content: &'x ArchivedCalendarEventNotificationContent, + ) -> Vec> { + self.meta_index_values() + } +} + +pub trait ICalendarObjectUid { + fn object_uid(&self) -> Option<&str>; +} + +impl ICalendarObjectUid for ICalendar { + fn object_uid(&self) -> Option<&str> { + self.components + .iter() + .filter(|component| { + matches!( + component.component_type, + ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VJournal + | ICalendarComponentType::VFreebusy + | ICalendarComponentType::VAvailability + ) + }) + .find_map(|component| component.uid()) + } +} + +impl ICalendarObjectUid for ArchivedICalendar { + fn object_uid(&self) -> Option<&str> { + self.components + .iter() + .filter(|component| { + matches!( + component.component_type, + ArchivedICalendarComponentType::VEvent + | ArchivedICalendarComponentType::VTodo + | ArchivedICalendarComponentType::VJournal + | ArchivedICalendarComponentType::VFreebusy + | ArchivedICalendarComponentType::VAvailability + ) + }) + .find_map(|component| component.uid()) + } } impl Calendar { pub fn size(&self) -> usize { - self.dead_properties.size() - + self.preferences.iter().map(|p| p.size()).sum::() + self.preferences.iter().map(|p| p.size()).sum::() + self.name.len() + std::mem::size_of::() } @@ -206,52 +554,53 @@ impl Calendar { impl ArchivedCalendar { pub fn size(&self) -> usize { - self.dead_properties.size() - + self.preferences.iter().map(|p| p.size()).sum::() + self.preferences.iter().map(|p| p.size()).sum::() + self.name.len() + std::mem::size_of::() } } impl CalendarEvent { - pub fn size(&self) -> usize { - self.dead_properties.size() + pub fn meta_size_hint(&self) -> usize { + self.uid.len() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() - + self.preferences.iter().map(|p| p.size()).sum::() - + self.size as usize + std::mem::size_of::() } } impl ArchivedCalendarEvent { - pub fn size(&self) -> usize { - self.dead_properties.size() + pub fn event_range_end(&self) -> i64 { + self.start + .to_native() + .saturating_add(self.duration.to_native() as i64) + } + + pub fn meta_size_hint(&self) -> usize { + self.uid.len() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() - + self.preferences.iter().map(|p| p.size()).sum::() - + self.size.to_native() as usize + std::mem::size_of::() } } impl CalendarEventNotification { - pub fn size(&self) -> usize { + pub fn meta_size_hint(&self) -> usize { (match &self.changed_by { ChangedBy::PrincipalId(_) => U32_LEN, ChangedBy::CalendarAddress(v) => v.len(), - }) + std::mem::size_of::() - + self.size as usize + }) + ((self.calendar_ids.len() + self.dismissed_by.len()) * U32_LEN) + + std::mem::size_of::() } } impl ArchivedCalendarEventNotification { - pub fn size(&self) -> usize { + pub fn meta_size_hint(&self) -> usize { (match &self.changed_by { ArchivedChangedBy::PrincipalId(_) => U32_LEN, ArchivedChangedBy::CalendarAddress(v) => v.len(), - }) + std::mem::size_of::() - + self.size.to_native() as usize + }) + ((self.calendar_ids.len() + self.dismissed_by.len()) * U32_LEN) + + std::mem::size_of::() } } @@ -277,22 +626,6 @@ impl ArchivedCalendarPreferences { } } -impl EventPreferences { - pub fn size(&self) -> usize { - self.alerts.iter().map(|a| a.size()).sum::() - + self.properties.iter().map(|p| p.size()).sum::() - + std::mem::size_of::() - } -} - -impl ArchivedEventPreferences { - pub fn size(&self) -> usize { - self.alerts.iter().map(|a| a.size()).sum::() - + self.properties.iter().map(|p| p.size()).sum::() - + std::mem::size_of::() - } -} - impl Timezone { pub fn size(&self) -> usize { match self { @@ -325,7 +658,7 @@ impl ArchivedDefaultAlert { } } -impl CalendarEvent { +impl CalendarEventContent { pub fn hashes(&self) -> impl Iterator { self.data .event @@ -365,7 +698,7 @@ impl CalendarEvent { } } -impl ArchivedCalendarEvent { +impl ArchivedCalendarEventContent { pub fn hashes(&self) -> impl Iterator { self.data .event @@ -405,7 +738,7 @@ impl ArchivedCalendarEvent { } } -impl ArchivedCalendarEvent { +impl ArchivedCalendarEventContent { pub fn index_document( &self, account_id: u32, @@ -417,12 +750,6 @@ impl ArchivedCalendarEvent { .with_account_id(account_id) .with_document_id(document_id); - if index_fields.is_empty() - || index_fields.contains(&SearchField::Calendar(CalendarSearchField::Start)) - { - document.index_integer(CalendarSearchField::Start, self.data.event_range_start()); - } - let mut detector = LanguageDetector::new(); for component in self .data @@ -432,6 +759,8 @@ impl ArchivedCalendarEvent { .filter(|e| e.component_type.is_scheduling_object()) { for entry in component.entries.iter() { + let is_owner_attendee = matches!(entry.name, ArchivedICalendarProperty::Attendee) + && entry.has_owner_role(); let (is_lang, is_keyword, field) = match entry.name { ArchivedICalendarProperty::Summary => (true, false, CalendarSearchField::Title), ArchivedICalendarProperty::Description => { @@ -446,7 +775,6 @@ impl ArchivedCalendarEvent { ArchivedICalendarProperty::Attendee => { (false, false, CalendarSearchField::Attendee) } - ArchivedICalendarProperty::Uid => (false, true, CalendarSearchField::Uid), _ => continue, }; let field = SearchField::Calendar(field); @@ -481,6 +809,18 @@ impl ArchivedCalendarEvent { } } } + if is_owner_attendee { + let field = SearchField::Calendar(CalendarSearchField::Owner); + if index_fields.is_empty() || index_fields.contains(&field) { + for value in entry.name_and_address() { + document.index_text( + field.clone(), + strip_mailto_scheme(value), + Language::None, + ); + } + } + } } } @@ -493,3 +833,24 @@ impl ArchivedCalendarEvent { document } } + +impl ArchiveCompression for Calendar { + const COMPRESSION: Compression = Compression::None; +} + +impl ArchiveCompression for CalendarEvent { + const COMPRESSION: Compression = Compression::None; +} + +impl ArchiveCompression for CalendarEventContent { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Calendar)); +} + +impl ArchiveCompression for CalendarEventNotification { + const COMPRESSION: Compression = Compression::None; +} + +impl ArchiveCompression for CalendarEventNotificationContent { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Calendar)); +} + diff --git a/crates/groupware/src/calendar/instance_filter.rs b/crates/groupware/src/calendar/instance_filter.rs new file mode 100644 index 00000000..d0e4c07c --- /dev/null +++ b/crates/groupware/src/calendar/instance_filter.rs @@ -0,0 +1,338 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + calendar::{ + identity::CalendarAddresses, + participants::{ArchivedParticipantEntry, ParticipantVisibility}, + }, + strip_mailto_scheme, +}; +use calcard::icalendar::{ + ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarEntry, + ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarValue, +}; +use nlp::language::{ + Language, + detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, +}; +use std::borrow::Cow; +use store::{ + ahash::AHashSet, + search::{CalendarSearchField, SearchFilter, TextMatch, tokenize::tokenize}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AttendeeSearch<'x> { + All, + VisibleTo(&'x CalendarAddresses), +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct QueryWord { + word: String, + stem: Option, +} + +#[derive(Debug)] +struct TextCondition { + fields: &'static [CalendarSearchField], + op: TextMatch, + language: Language, + words: Vec, +} + +#[derive(Debug, Default)] +pub struct InstanceTextFilter { + conditions: Vec, + ordered_fields: Vec, +} + +pub struct EventTextFilter<'x> { + conditions: &'x [TextCondition], + ordered_fields: &'x [CalendarSearchField], + language: Language, + attendees: AttendeeSearch<'x>, +} + +#[derive(Default)] +struct FieldTokens<'a> { + words: AHashSet>, + stemmed: AHashSet>, + ordered: Vec>, +} + +struct ComponentTokens<'a, 'x> { + ordered_fields: &'x [CalendarSearchField], + attendees: AttendeeSearch<'x>, + fields: Vec<(CalendarSearchField, FieldTokens<'a>)>, +} + +impl InstanceTextFilter { + pub fn add(&mut self, fields: &'static [CalendarSearchField], filter: &SearchFilter) { + let SearchFilter::Text { + op, + value, + language, + .. + } = filter + else { + return; + }; + let words = match op { + TextMatch::Prefix => vec![QueryWord { + word: value.to_lowercase(), + stem: None, + }], + TextMatch::Standard | TextMatch::Exact => { + let mut words = Vec::new(); + tokenize(value, *language, |token| { + words.push(QueryWord { + word: token.word.into_owned(), + stem: token.stem.map(Cow::into_owned), + }); + true + }); + if *op == TextMatch::Standard { + words.sort_unstable(); + words.dedup(); + } + words + } + }; + if *op == TextMatch::Exact { + for field in fields { + if !self.ordered_fields.contains(field) { + self.ordered_fields.push(*field); + } + } + } + self.conditions.push(TextCondition { + fields, + op: *op, + language: *language, + words, + }); + } + + pub fn is_empty(&self) -> bool { + self.conditions.is_empty() + } + + pub fn for_event<'x>( + &'x self, + event: &ArchivedICalendar, + default_language: Language, + attendees: AttendeeSearch<'x>, + ) -> EventTextFilter<'x> { + let language = if self + .conditions + .iter() + .flat_map(|condition| condition.fields) + .any(|field| field.is_language_detected()) + { + let mut detector = LanguageDetector::new(); + for value in event + .components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + .flat_map(|component| { + field_values(component, CalendarSearchField::Title, AttendeeSearch::All).chain( + field_values( + component, + CalendarSearchField::Description, + AttendeeSearch::All, + ), + ) + }) + { + detector.detect(value, MIN_LANGUAGE_SCORE); + } + detector + .most_frequent_language() + .unwrap_or(default_language) + } else { + default_language + }; + + EventTextFilter { + conditions: &self.conditions, + ordered_fields: &self.ordered_fields, + language, + attendees, + } + } +} + +impl EventTextFilter<'_> { + pub fn matches(&self, component: &ArchivedICalendarComponent) -> bool { + let mut tokens = ComponentTokens { + ordered_fields: self.ordered_fields, + attendees: self.attendees, + fields: Vec::with_capacity(2), + }; + self.conditions + .iter() + .all(|condition| condition.matches(component, self.language, &mut tokens)) + } +} + +impl<'a> ComponentTokens<'a, '_> { + fn of( + &mut self, + component: &'a ArchivedICalendarComponent, + field: CalendarSearchField, + language: Language, + ) -> &FieldTokens<'a> { + if let Some(index) = self.fields.iter().position(|(cached, _)| *cached == field) { + return &self.fields[index].1; + } + + let keep_order = self.ordered_fields.contains(&field); + let mut tokens = FieldTokens::default(); + for value in field_values(component, field, self.attendees) { + tokenize(value, language, |token| { + if keep_order { + tokens.ordered.push(token.word.clone()); + } + if let Some(stem) = token.stem { + tokens.stemmed.insert(stem); + } + tokens.stemmed.insert(token.word.clone()); + tokens.words.insert(token.word); + true + }); + } + &self.fields.push_mut((field, tokens)).1 + } +} + +impl TextCondition { + fn matches<'a>( + &self, + component: &'a ArchivedICalendarComponent, + event_language: Language, + tokens: &mut ComponentTokens<'a, '_>, + ) -> bool { + !self.words.is_empty() + && self.fields.iter().any(|field| { + let language = if field.is_language_detected() { + event_language + } else { + Language::None + }; + self.matches_tokens(tokens.of(component, *field, language)) + }) + } + + fn matches_tokens(&self, tokens: &FieldTokens<'_>) -> bool { + match self.op { + TextMatch::Standard => { + let is_stemmed = !matches!(self.language, Language::None | Language::Unknown); + self.words + .iter() + .all(|word| word.matches(tokens, is_stemmed)) + } + TextMatch::Exact => tokens.ordered.windows(self.words.len()).any(|window| { + window + .iter() + .zip(&self.words) + .all(|(token, word)| *token == word.word) + }), + TextMatch::Prefix => self.words.iter().all(|word| { + tokens + .stemmed + .iter() + .any(|token| token.starts_with(&word.word)) + }), + } + } +} + +impl QueryWord { + fn matches(&self, tokens: &FieldTokens<'_>, is_stemmed: bool) -> bool { + if is_stemmed { + tokens.stemmed.contains(self.word.as_str()) + || self + .stem + .as_deref() + .is_some_and(|stem| tokens.stemmed.contains(stem)) + } else { + tokens.words.contains(self.word.as_str()) + } + } +} + +trait DetectedLanguage { + fn is_language_detected(&self) -> bool; +} + +impl DetectedLanguage for CalendarSearchField { + fn is_language_detected(&self) -> bool { + matches!( + self, + CalendarSearchField::Title | CalendarSearchField::Description + ) + } +} + +fn field_values<'x>( + component: &'x ArchivedICalendarComponent, + field: CalendarSearchField, + attendees: AttendeeSearch<'_>, +) -> impl Iterator { + component + .entries + .iter() + .filter_map(move |entry| { + let all_texts = match (&entry.name, field) { + (ArchivedICalendarProperty::Summary, CalendarSearchField::Title) + | (ArchivedICalendarProperty::Description, CalendarSearchField::Description) + | (ArchivedICalendarProperty::Location, CalendarSearchField::Location) + | (ArchivedICalendarProperty::Organizer, CalendarSearchField::Owner) => true, + (ArchivedICalendarProperty::Attendee, CalendarSearchField::Owner) + if entry.has_owner_role() => + { + false + } + (ArchivedICalendarProperty::Attendee, CalendarSearchField::Attendee) => { + match attendees { + AttendeeSearch::All => true, + AttendeeSearch::VisibleTo(identities) + if entry.is_visible_participant(identities) => + { + false + } + AttendeeSearch::VisibleTo(_) => return None, + } + } + _ => return None, + }; + Some((entry, all_texts)) + }) + .flat_map(|(entry, all_texts)| entry_texts(entry, all_texts)) + .map(strip_mailto_scheme) +} + +fn entry_texts(entry: &ArchivedICalendarEntry, all_texts: bool) -> impl Iterator { + entry.name_and_address().filter(move |_| !all_texts).chain( + entry + .values + .iter() + .filter_map(|value| match value { + ArchivedICalendarValue::Text(value) => Some(value.as_str()), + ArchivedICalendarValue::Uri(uri) => uri.as_str(), + _ => None, + }) + .chain(entry.params.iter().filter_map(|param| match ¶m.value { + ArchivedICalendarParameterValue::Text(value) => Some(value.as_str()), + ArchivedICalendarParameterValue::Uri(uri) => uri.as_str(), + _ => None, + })) + .filter(move |_| all_texts), + ) +} + diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index d08d484f..c8240e7a 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -5,21 +5,27 @@ */ use crate::{ - RFC_3986, + RFC_3986, SizeWriter, cache::GroupwareCache, calendar::{ - CalendarEvent, CalendarEventData, CalendarEventNotification, ChangedBy, - EVENT_HIDE_ATTENDEES, EVENT_NOTIFICATION_IS_CHANGE, + CalendarEvent, CalendarEventContent, CalendarEventData, CalendarEventNotification, + CalendarEventNotificationContent, ChangedBy, EVENT_HIDE_ATTENDEES, + EVENT_NOTIFICATION_IS_CHANGE, + alerts::DefaultAlertsResolver, + notification::NotificationAudience, + privacy::{EventPrivacy, ICalendarPrivacy}, + schedule::{AlarmTasks, EventAlarmScheduler, EventAlarmUsers}, + storage::NotificationQuota, }, scheduling::{ InstanceId, ItipError, ItipMessage, ItipSnapshots, format::{DateStyle, TextFormatter, hyperlink}, - ical_size, inbound::{ MergeAction, MergeResult, itip_import_message, itip_merge_changes, itip_method, itip_process_message, }, - itip::itip_build_envelope, + itip::{itip_add_tz, itip_build_envelope}, + recipient::RecipientPolicy, snapshot::itip_snapshot, }, }; @@ -28,7 +34,8 @@ use calcard::{ icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, - ICalendarParticipationStatus, ICalendarProperty, ICalendarValue, + ICalendarParticipationRole, ICalendarParticipationStatus, ICalendarProperty, + ICalendarValue, }, }; use common::{ @@ -36,11 +43,12 @@ use common::{ auth::{AccessToken, AccountInfo, oauth::GrantType}, i18n, }; -use registry::schema::enums::Permission; +use compact_str::ToCompactString; +use registry::schema::enums::{Permission, StorageQuota}; use std::net::IpAddr; use store::{ - ValueKey, rand, - write::{AlignedBytes, Archive, BatchBuilder, now}, + rand, + write::{Archive, ArchiveBytes, BatchBuilder, now, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ @@ -142,7 +150,7 @@ impl ItipIngest for Server { comp.entries.push(ICalendarEntry { name: ICalendarProperty::Organizer, params: vec![], - values: vec![ICalendarValue::Text(format!("mailto:{recipient}"))], + values: [ICalendarValue::Text(format!("mailto:{recipient}"))].into(), }); } } @@ -150,10 +158,23 @@ impl ItipIngest for Server { } let itip_snapshots = itip_snapshot(&itip, account_info.addresses(), false)?; - if !itip_snapshots.sender_is_organizer_or_attendee(sender) { - return Err(ItipIngestError::Message( - ItipError::SenderIsNotOrganizerNorAttendee, - )); + match itip_method(&itip)? { + ICalendarMethod::Reply | ICalendarMethod::Refresh | ICalendarMethod::Counter => { + if !itip_snapshots.sender_is_organizer_or_attendee(sender) { + return Err(ItipIngestError::Message( + ItipError::SenderIsNotOrganizerNorAttendee, + )); + } + } + ICalendarMethod::Request + | ICalendarMethod::Add + | ICalendarMethod::Cancel + | ICalendarMethod::Publish + | ICalendarMethod::Declinecounter => { + if itip_snapshots.organizer.email.email != sender { + return Err(ItipIngestError::Message(ItipError::SenderIsNotOrganizer)); + } + } } // Obtain changedBy @@ -178,32 +199,38 @@ impl ItipIngest for Server { .next(); if let Some(document_id) = document_id { - if let Some(archive) = self - .store() - .get_value::>(ValueKey::archive( + let (archive, content_archive) = self + .archive_and_field( account_id, Collection::CalendarEvent, document_id, - )) + CalendarEventField::Content.field(), + ) .await - .caused_by(trc::location!())? - { - let event_ = archive - .to_unarchived::() - .caused_by(trc::location!())?; - let event = event_ + .caused_by(trc::location!())?; + if let Some(archive) = archive { + let Some(content_archive) = content_archive else { + return Err(ItipIngestError::Message(ItipError::EventNotFound)); + }; + + let event = archive .deserialize::() .caused_by(trc::location!())?; + let content = content_archive + .deserialize::() + .caused_by(trc::location!())?; // Process the iTIP message - let snapshots = itip_snapshot(&event.data.event, account_info.addresses(), false)?; + let snapshots = + itip_snapshot(&content.data.event, account_info.addresses(), false)?; let is_organizer_update = !itip_snapshots.organizer.email.is_local; match itip_process_message( - &event.data.event, + &content.data.event, snapshots, &itip, itip_snapshots, sender.to_string(), + RecipientPolicy::new(&self.core.groupware, event.flags), )? { MergeResult::Actions(changes) => { commit_itip_merge( @@ -212,10 +239,11 @@ impl ItipIngest for Server { account_id, document_id, &archive, + &content_archive, event, + content, changes, itip, - itip_message.len(), changed_by, is_organizer_update, ) @@ -252,21 +280,11 @@ impl ItipIngest for Server { let mut ical = itip.clone(); itip_import_message(&mut ical)?; - // Validate quota - if self - .has_available_quota( - self.account(account_id).await?.as_ref(), - itip_message.len() as u64, - ) - .await - .is_err() - { - return Err(ItipIngestError::Message(ItipError::QuotaExceeded)); - } + let account = self.account(account_id).await?; // Obtain parent calendar let Some(parent_id) = self - .get_or_create_default_calendar(account_id, account_id) + .get_or_create_default_calendar(account_info.account(), account_info.account()) .await .caused_by(trc::location!())? else { @@ -274,62 +292,92 @@ impl ItipIngest for Server { }; // Build event - let mut next_email_alarm = None; let now = now() as i64; let event = CalendarEvent { names: vec![DavName { name: format!("{}_{}.ics", now, rand::random::()), parent_id, }], + schedule_tag: Some(1), + ..Default::default() + }; + let current = ical.clone(); + let event_content = CalendarEventContent { data: CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, - &mut next_email_alarm, ), - size: itip_message.len() as u32, - schedule_tag: Some(1), ..Default::default() }; + let next_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &event_content)? + .with_event_flags(event.flags), + &event_content.data, + &[parent_id], + now, + &mut DefaultAlertsResolver::default(), + ) + .await + .caused_by(trc::location!())?; - // Obtain document ids - let document_id = self - .store() - .assign_document_ids(account_id, Collection::CalendarEvent, 1) - .await - .caused_by(trc::location!())?; - let itip_document_id = self - .store() - .assign_document_ids(account_id, Collection::CalendarEventNotification, 1) - .await - .caused_by(trc::location!())?; - let itip_message = CalendarEventNotification { - event: itip, - event_id: Some(document_id), + let notification = CalendarEventNotification { + event_id: None, changed_by, - size: itip_message.len() as u32, + calendar_ids: vec![parent_id], + flags: NotificationAudience::of_event(event.flags) + .with_privacy(current.privacy().is_public()) + .flags(), ..Default::default() }; + let notification_content = CalendarEventNotificationContent::Itip { + message: itip, + previous: None, + current: Some(current), + }; // Prepare write batch let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + self.limit_disk_quota(&mut batch, &account); + self.limit_object_quota(&mut batch, &account, StorageQuota::MaxCalendarEvents); + let document_id = batch.reserve_document_id(account_id, Collection::CalendarEvent); event .insert( - account_info.account_tenant_ids(), + event_content, + account_info.changed_by(), account_id, document_id, - next_email_alarm, + None, + next_email_alarms, &mut batch, ) .caused_by(trc::location!())?; - itip_message - .insert( - account_info.account_tenant_ids(), + if NotificationQuota::default() + .reserve( + self, account_id, - itip_document_id, + notification_content.stored_size(), &mut batch, ) - .caused_by(trc::location!())?; + .await + .caused_by(trc::location!())? + { + let itip_document_id = + batch.reserve_document_id(account_id, Collection::CalendarEventNotification); + notification + .insert( + notification_content, + account_info.changed_by(), + account_id, + itip_document_id, + Some(document_id), + &mut batch, + ) + .caused_by(trc::location!())?; + } self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(None) @@ -385,13 +433,13 @@ impl ItipIngest for Server { Err(reason) => return Ok(RsvpResponse::error(reason, language)), }; - let Some(archive) = self - .store() - .get_value::>(ValueKey::archive( + let (Some(archive), Some(content_archive)) = self + .archive_and_field( rsvp.account_id, Collection::CalendarEvent, rsvp.document_id, - )) + CalendarEventField::Content.field(), + ) .await .caused_by(trc::location!())? else { @@ -405,25 +453,31 @@ impl ItipIngest for Server { // Without a participation status this is a request for the invitation details let Some(part_stat) = part_stat else { - let event = archive - .deserialize::() + let flags = archive + .unarchive::() + .caused_by(trc::location!())? + .flags + .to_native(); + let ical = content_archive + .unarchive::() + .and_then(|content| rkyv_deserialize::<_, ICalendar>(&content.data.event)) .caused_by(trc::location!())?; return Ok(build_rsvp_invitation( - &event.data.event, + &ical, &rsvp.attendee, organizer_info.addresses(), - event.flags & EVENT_HIDE_ATTENDEES != 0, + flags & EVENT_HIDE_ATTENDEES != 0, language, )); }; let comment = request.sanitized_comment(); // Locate the attendee within the organizer's copy of the event - let event = archive - .deserialize::() + let content = content_archive + .deserialize::() .caused_by(trc::location!())?; - let Ok(snapshots) = itip_snapshot(&event.data.event, organizer_info.addresses(), false) + let Ok(snapshots) = itip_snapshot(&content.data.event, organizer_info.addresses(), false) else { return Ok(RsvpResponse::error(RsvpError::NotParticipant, language)); }; @@ -453,16 +507,19 @@ impl ItipIngest for Server { return Ok(RsvpResponse::recorded(&part_stat)); } instances.sort_unstable(); + let event = archive + .deserialize::() + .caused_by(trc::location!())?; // Deliver the reply to the organizer without going through the mail queue - let reply = build_rsvp_reply( + let mut reply = build_rsvp_reply( &snapshots, &instances, &rsvp.attendee, &part_stat, comment.as_deref(), ); - let reply_size = ical_size(&reply); + itip_add_tz(&mut reply, &content.data.event); let attendee_copy = http_rsvp_attendee_copy(self, &rsvp, snapshots.uid, remote_ip).await?; let changed_by = if let Some(account_id) = self .account_id_from_email(&rsvp.attendee, true) @@ -477,11 +534,12 @@ impl ItipIngest for Server { let merge = itip_snapshot(&reply, organizer_info.addresses(), false).and_then(|reply_snapshots| { itip_process_message( - &event.data.event, + &content.data.event, snapshots, &reply, reply_snapshots, rsvp.attendee.clone(), + RecipientPolicy::new(&self.core.groupware, event.flags), ) }); let changes = match merge { @@ -503,7 +561,7 @@ impl ItipIngest for Server { AccountId = rsvp.account_id, DocumentId = rsvp.document_id, From = rsvp.attendee.clone(), - Details = err.to_string(), + Details = err.to_compact_string(), ); return Ok(RsvpResponse::error(RsvpError::ServerError, language)); @@ -516,10 +574,11 @@ impl ItipIngest for Server { rsvp.account_id, rsvp.document_id, &archive, + &content_archive, event, + content, changes, reply, - reply_size, changed_by, false, ) @@ -532,7 +591,7 @@ impl ItipIngest for Server { AccountId = rsvp.account_id, DocumentId = rsvp.document_id, From = rsvp.attendee.clone(), - Details = err.to_string(), + Details = err.to_compact_string(), ); return Ok(RsvpResponse::error(RsvpError::ServerError, language)); @@ -561,12 +620,19 @@ async fn http_rsvp_sync_attendee_copy( .archive .to_unarchived::() .caused_by(trc::location!())?; - let mut new_event = event + let content = target + .content_archive + .to_unarchived::() + .caused_by(trc::location!())?; + let new_event = event .deserialize::() .caused_by(trc::location!())?; + let mut new_content = content + .deserialize::() + .caused_by(trc::location!())?; let mut did_change = false; - for component in &mut new_event.data.event.components { + for component in &mut new_content.data.event.components { if !component.component_type.is_scheduling_object() { continue; } @@ -588,7 +654,7 @@ async fn http_rsvp_sync_attendee_copy( ) = (¶m.name, &mut param.value) { has_partstat = true; - if current != part_stat { + if *current != *part_stat { *current = part_stat.clone(); did_change = true; } @@ -609,15 +675,16 @@ async fn http_rsvp_sync_attendee_copy( .account_info(target.account_id) .await .caused_by(trc::location!())?; - new_event.size = ical_size(&new_event.data.event) as u32; - let mut batch = BatchBuilder::new(); new_event - .update( - attendee_info.account_tenant_ids(), + .update_full( + new_content, + attendee_info.changed_by(), event, + content.inner, target.account_id, target.document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -633,7 +700,8 @@ async fn http_rsvp_sync_attendee_copy( struct RsvpTarget { account_id: u32, document_id: u32, - archive: Archive, + archive: Archive, + content_archive: Archive, } async fn http_rsvp_attendee_copy( @@ -687,20 +755,25 @@ async fn http_rsvp_attendee_copy( return Ok(None); }; - Ok(server - .store() - .get_value::>(ValueKey::archive( + let (Some(archive), Some(content_archive)) = server + .archive_and_field( account_id, Collection::CalendarEvent, document_id, - )) + CalendarEventField::Content.field(), + ) .await .caused_by(trc::location!())? - .map(|archive| RsvpTarget { - account_id, - document_id, - archive, - })) + else { + return Ok(None); + }; + + Ok(Some(RsvpTarget { + account_id, + document_id, + archive, + content_archive, + })) } struct RsvpToken { @@ -748,48 +821,100 @@ async fn commit_itip_merge( account_info: &AccountInfo, account_id: u32, document_id: u32, - archive: &Archive, + archive: &Archive, + content_archive: &Archive, mut event: CalendarEvent, + mut content: CalendarEventContent, changes: Vec, itip: ICalendar, - itip_size: usize, changed_by: ChangedBy, is_organizer_update: bool, ) -> Result<(), ItipIngestError> { let event_ = archive .to_unarchived::() .caused_by(trc::location!())?; + let content_ = content_archive + .to_unarchived::() + .caused_by(trc::location!())?; // Merge changes - itip_merge_changes(&mut event.data.event, &itip, changes); + let privacy = EventPrivacy::from_flags(event_.inner.flags.to_native()); + let previous = content.data.event.clone(); + itip_merge_changes(&mut content.data.event, &itip, changes)?; + if content + .data + .event + .components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + .any(|component| { + component + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Class) + .filter_map(|entry| entry.values.first()) + .map(EventPrivacy::from_value) + .max() + .unwrap_or_default() + != privacy + }) + { + content.data.event.set_privacy(privacy); + } + let current = content.data.event.clone(); // Calculate the new ical size - event.size = ical_size(&event.data.event) as u32; - if event.size > server.core.groupware.max_ical_size as u32 { + let new_size = SizeWriter::ical(&content.data.event) as u32; + if new_size > server.core.groupware.max_ical_size as u32 { return Err(ItipIngestError::Message(ItipError::EventTooLarge)); } - // Validate quota - let extra_bytes = (event.size as u64).saturating_sub(event_.inner.size.to_native() as u64); - if extra_bytes > 0 - && server - .has_available_quota(server.account(account_id).await?.as_ref(), extra_bytes) - .await - .is_err() - { - return Err(ItipIngestError::Message(ItipError::QuotaExceeded)); - } + let account = server.account(account_id).await?; // Build event let now = now() as i64; - let prev_email_alarm = event_.inner.data.next_alarm(now, Tz::Floating); - let mut next_email_alarm = None; - event.data = CalendarEventData::new( - event.data.event, + let mut resolver = DefaultAlertsResolver::default(); + let default_alerts = resolver + .resolve_for_content( + server, + account_id, + account_id, + &content, + event.names.iter().map(DavName::parent_id), + ) + .await?; + let calendar_ids = event + .names + .iter() + .map(DavName::parent_id) + .collect::>(); + let prev_email_alarms = server + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, content_.inner)? + .with_event_flags(event_.inner.flags.to_native()), + &content_.inner.data, + &calendar_ids, + now, + &mut resolver, + ) + .await?; + content.data = CalendarEventData::new_with_default_alerts( + content.data.event, Tz::Floating, server.core.groupware.max_ical_instances, - &mut next_email_alarm, + &default_alerts, ); + let next_email_alarms = server + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &content)?.with_event_flags(event.flags), + &content.data, + &calendar_ids, + now, + &mut resolver, + ) + .await?; if is_organizer_update { if let Some(schedule_tag) = &mut event.schedule_tag { *schedule_tag += 1; @@ -799,47 +924,71 @@ async fn commit_itip_merge( } // Build event for schedule inbox - let itip_document_id = server - .store() - .assign_document_ids(account_id, Collection::CalendarEventNotification, 1) - .await - .caused_by(trc::location!())?; - let itip_message = CalendarEventNotification { - event: itip, + let notification = CalendarEventNotification { changed_by, event_id: Some(document_id), - flags: EVENT_NOTIFICATION_IS_CHANGE, - size: itip_size as u32, + calendar_ids: calendar_ids.clone(), + flags: EVENT_NOTIFICATION_IS_CHANGE + | NotificationAudience::of_event(event.flags | event_.inner.flags.to_native()) + .with_privacy(privacy.is_public()) + .flags(), ..Default::default() }; + let notification_content = CalendarEventNotificationContent::Itip { + message: itip, + previous: Some(previous), + current: Some(current), + }; // Prepare write batch let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + server.limit_disk_quota(&mut batch, &account); event - .update( - account_info.account_tenant_ids(), + .update_full( + content, + account_info.changed_by(), event_, + content_.inner, account_id, document_id, + None, &mut batch, ) .caused_by(trc::location!())?; - if prev_email_alarm != next_email_alarm { - if let Some(prev_alarm) = prev_email_alarm { - prev_alarm.delete_task(&mut batch); - } - if let Some(next_alarm) = next_email_alarm { - next_alarm.write_task(&mut batch); - } - } - itip_message - .insert( - account_info.account_tenant_ids(), + server + .replace_event_alarms( account_id, - itip_document_id, + document_id, + prev_email_alarms, + next_email_alarms, + &mut AlarmTasks::None, &mut batch, ) - .caused_by(trc::location!())?; + .await?; + if NotificationQuota::default() + .reserve( + server, + account_id, + notification_content.stored_size(), + &mut batch, + ) + .await + .caused_by(trc::location!())? + { + let itip_document_id = + batch.reserve_document_id(account_id, Collection::CalendarEventNotification); + notification + .insert( + notification_content, + account_info.changed_by(), + account_id, + itip_document_id, + None, + &mut batch, + ) + .caused_by(trc::location!())?; + } server .commit_batch(batch) .await @@ -909,7 +1058,8 @@ fn build_rsvp_reply( values: vec![ ICalendarValue::Text("2.0".to_string()), ICalendarValue::Text("Success".to_string()), - ], + ] + .into(), }); let comp_id = message.components.len() as u32; @@ -1024,7 +1174,9 @@ fn build_rsvp_invitation( .email .email .eq_ignore_ascii_case(&snapshots.organizer.email.email) - && (!hide_attendees || candidate.email.email.eq_ignore_ascii_case(attendee)) + && (!hide_attendees + || candidate.role == Some(&ICalendarParticipationRole::Owner) + || candidate.email.email.eq_ignore_ascii_case(attendee)) }) .map(|candidate| RsvpParticipant { name: candidate.name.map(|name| name.to_string()), @@ -1272,12 +1424,13 @@ impl RsvpResponse { impl RsvpRequest { fn part_stat(&self) -> Result, RsvpError> { match self.partstat.as_deref() { - Some(partstat) => hashify::tiny_map_ignore_case!(partstat.as_bytes(), - "ACCEPTED" => ICalendarParticipationStatus::Accepted, - "DECLINED" => ICalendarParticipationStatus::Declined, - "TENTATIVE" => ICalendarParticipationStatus::Tentative, - "COMPLETED" => ICalendarParticipationStatus::Completed, - "IN-PROCESS" => ICalendarParticipationStatus::InProcess, + Some(partstat) => hashify::fnc_map_ignore_case!(partstat.as_bytes(), + "ACCEPTED" => Some(ICalendarParticipationStatus::Accepted), + "DECLINED" => Some(ICalendarParticipationStatus::Declined), + "TENTATIVE" => Some(ICalendarParticipationStatus::Tentative), + "COMPLETED" => Some(ICalendarParticipationStatus::Completed), + "IN-PROCESS" => Some(ICalendarParticipationStatus::InProcess), + _ => None, ) .map(Some) .ok_or(RsvpError::InvalidPartStat), @@ -1314,6 +1467,20 @@ impl From for ItipIngestError { impl From for ItipIngestError { fn from(err: trc::Error) -> Self { - ItipIngestError::Internal(err) + if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) { + ItipIngestError::Message(ItipError::QuotaExceeded) + } else { + ItipIngestError::Internal(err) + } + } +} + +impl CalendarEventNotificationContent { + fn stored_size(&self) -> u64 { + self.snapshots() + .into_iter() + .flatten() + .map(|ical| SizeWriter::ical(ical) as u64) + .sum() } } diff --git a/crates/groupware/src/calendar/mod.rs b/crates/groupware/src/calendar/mod.rs index 424689b1..05440992 100644 --- a/crates/groupware/src/calendar/mod.rs +++ b/crates/groupware/src/calendar/mod.rs @@ -5,17 +5,34 @@ */ pub mod alarm; +pub mod alerts; +pub mod availability; +pub mod color; +pub mod compare; pub mod dates; pub mod expand; +pub mod hidden; +pub mod identity; pub mod index; +pub mod instance_filter; pub mod itip; +pub mod notification; +pub mod overlap; +pub mod participants; +pub mod privacy; +pub mod rights; +pub mod schedule; +pub mod sequence; pub mod storage; +pub mod user; +use crate::metadata::tracked_kinds; use calcard::icalendar::{ - ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ArchivedICalendar, ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, }; -use common::DavName; -use types::{acl::AclGrant, dead_property::DeadProperty}; +use color::CssColor; +use common::{DavName, NO_ID, storage::dav::PresenceBits}; +use types::{acl::AclGrant, metadata::MetadataKinds}; use utils::map::bitmap::BitmapItem; #[derive( @@ -26,7 +43,7 @@ pub struct Calendar { pub preferences: Vec, pub acls: Vec, pub supported_components: u64, - pub dead_properties: DeadProperty, + pub presence: u8, pub created: i64, pub modified: i64, } @@ -47,7 +64,7 @@ pub enum SupportedComponent { Participant, // [RFC9073, Section 7.1] VLocation, // [RFC9073, Section 7.2] [RFC Errata 7381] VResource, // [RFC9073, Section 7.3] - VStatus, // draft-ietf-calext-ical-tasks-14 + VStatus, // draft-ietf-calext-ical-tasks-17 Other, } @@ -87,6 +104,23 @@ pub struct ParticipantIdentities { pub identities: Vec, pub default_name: String, pub default: u32, + pub change_id: u32, + pub trimmed_change_id: u32, + pub changes: Vec, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParticipantIdentityChange { + pub change_id: u32, + pub id: u32, + pub change_type: ParticipantIdentityChangeType, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParticipantIdentityChangeType { + Created, + Updated, + Destroyed, } #[derive( @@ -104,44 +138,171 @@ pub const ALERT_RELATIVE_TO_END: u16 = 1 << 2; pub const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1; pub const SCHEDULE_OUTBOX_ID: u32 = u32::MAX - 2; +pub const DIRECT_NOTIFICATION_PARENT_ID: u32 = u32::MAX - 3; pub const EVENT_INVITE_SELF: u16 = 1; pub const EVENT_INVITE_OTHERS: u16 = 1 << 1; pub const EVENT_HIDE_ATTENDEES: u16 = 1 << 2; pub const EVENT_DRAFT: u16 = 1 << 3; +pub const EVENT_META_DAV: u16 = PresenceBits::CALENDAR_EVENT.dav(); +pub const EVENT_HAS_ALARMS: u16 = 1 << 5; +pub const EVENT_PRIVATE: u16 = 1 << 6; +pub const EVENT_SECRET: u16 = 1 << 7; +pub const EVENT_USES_DEFAULT_ALERTS: u16 = 1 << 8; +pub const EVENT_HAS_UNBOUNDED_TODO: u16 = 1 << 9; +pub const EVENT_META_JMAP: u16 = PresenceBits::CALENDAR_EVENT.jmap(); +pub const EVENT_FLOATING_START: u16 = 1 << 11; pub const EVENT_NOTIFICATION_IS_DRAFT: u16 = 1; pub const EVENT_NOTIFICATION_IS_CHANGE: u16 = 1 << 1; +pub const EVENT_NOTIFICATION_IS_DESTROY: u16 = 1 << 2; +pub const EVENT_NOTIFICATION_IS_DIRECT: u16 = 1 << 3; +pub const EVENT_NOTIFICATION_PRIVATE: u16 = 1 << 4; +pub const EVENT_NOTIFICATION_HIDE_ATTENDEES: u16 = 1 << 5; pub const PREF_USE_DEFAULT_ALERTS: u16 = 1; +pub const PREF_HAS_USE_DEFAULT_ALERTS: u16 = 1 << 1; +pub const PREF_HAS_COLOR: u16 = 1 << 2; +pub const PREF_HAS_KEYWORDS: u16 = 1 << 3; +pub const PREF_HAS_ALERTS: u16 = 1 << 4; +pub const PREF_HAS_FREE_BUSY: u16 = 1 << 5; +pub const PREF_FREE_BUSY_FREE: u16 = 1 << 6; + +pub const MAX_USER_KEYWORDS: usize = 64; +pub const MAX_USER_KEYWORD_LEN: usize = 128; +pub const MAX_USER_ALERTS: usize = 32; +pub const MAX_USER_INSTANCES: usize = 128; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEvent { pub names: Vec, + pub uid: String, pub display_name: Option, - pub data: CalendarEventData, - pub preferences: Vec, - pub flags: u16, - pub dead_properties: DeadProperty, - pub size: u32, + pub start: i64, + pub duration: u32, pub created: i64, pub modified: i64, + pub size: u32, + pub etag: u32, + pub flags: u16, pub schedule_tag: Option, } +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] +pub struct CalendarEventContent { + pub data: CalendarEventData, + pub preferences: Vec, +} + #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventNotification { - pub event: ICalendar, pub event_id: Option, pub changed_by: ChangedBy, - pub flags: u16, - pub size: u32, + pub calendar_ids: Vec, + pub dismissed_by: Vec, pub created: i64, pub modified: i64, + pub size: u32, + pub etag: u32, + pub flags: u16, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub enum CalendarEventNotificationContent { + Direct { + previous: Option, + current: Option, + }, + Itip { + message: ICalendar, + previous: Option, + current: Option, + }, +} + +impl ArchivedCalendarEventNotification { + pub fn changed_by_id(&self) -> u32 { + match &self.changed_by { + ArchivedChangedBy::PrincipalId(id) => id.to_native(), + ArchivedChangedBy::CalendarAddress(_) => NO_ID, + } + } + + pub fn dismissed_ids(&self) -> impl Iterator { + self.dismissed_by.iter().map(|id| id.to_native()) + } + + pub fn calendar_ids(&self) -> impl Iterator { + self.calendar_ids.iter().map(|id| id.to_native()) + } +} + +impl CalendarEventNotification { + pub fn changed_by_id(&self) -> u32 { + match &self.changed_by { + ChangedBy::PrincipalId(id) => *id, + ChangedBy::CalendarAddress(_) => NO_ID, + } + } +} + +impl Default for CalendarEventNotificationContent { + fn default() -> Self { + CalendarEventNotificationContent::Direct { + previous: None, + current: None, + } + } +} + +impl CalendarEventNotificationContent { + pub fn snapshots(&self) -> [Option<&ICalendar>; 3] { + match self { + CalendarEventNotificationContent::Direct { previous, current } => { + [None, previous.as_ref(), current.as_ref()] + } + CalendarEventNotificationContent::Itip { + message, + previous, + current, + } => [Some(message), previous.as_ref(), current.as_ref()], + } + } +} + +impl ArchivedCalendarEventNotificationContent { + pub fn previous(&self) -> Option<&ArchivedICalendar> { + match self { + ArchivedCalendarEventNotificationContent::Direct { previous, .. } + | ArchivedCalendarEventNotificationContent::Itip { previous, .. } => previous.as_ref(), + } + } + + pub fn current(&self) -> Option<&ArchivedICalendar> { + match self { + ArchivedCalendarEventNotificationContent::Direct { current, .. } + | ArchivedCalendarEventNotificationContent::Itip { current, .. } => current.as_ref(), + } + } + + pub fn calendar_data(&self) -> Option<&ArchivedICalendar> { + self.itip_message() + .or_else(|| self.current()) + .or_else(|| self.previous()) + } + + pub fn itip_message(&self) -> Option<&ArchivedICalendar> { + match self { + ArchivedCalendarEventNotificationContent::Itip { message, .. } => Some(message), + ArchivedCalendarEventNotificationContent::Direct { .. } => None, + } + } } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] @@ -168,18 +329,39 @@ pub struct Alarm { pub id: u16, pub parent_id: u16, pub delta: AlarmDelta, - pub is_email_alert: bool, + pub flags: u16, +} + +pub const ALARM_EMAIL: u16 = 1; + +impl Alarm { + pub fn is_email(&self) -> bool { + self.flags & ALARM_EMAIL != 0 + } +} + +impl ArchivedAlarm { + pub fn is_email(&self) -> bool { + self.flags.to_native() & ALARM_EMAIL != 0 + } } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(compare(PartialEq), derive(Debug))] pub enum AlarmDelta { - Start(i64), - End(i64), + Start(AlarmOffset), + End(AlarmOffset), FixedUtc(i64), FixedFloating(i64), } +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy, PartialEq, Eq)] +#[rkyv(compare(PartialEq), derive(Debug))] +pub struct AlarmOffset { + pub days: i32, + pub seconds: i32, +} + #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] @@ -188,6 +370,7 @@ pub struct ComponentTimeRange { pub start_tz: u16, pub end_tz: u16, pub duration: i32, + pub flags: u8, pub instances: Box<[u8]>, } @@ -196,8 +379,18 @@ pub struct ComponentTimeRange { )] pub struct EventPreferences { pub account_id: u32, + pub updated: i64, + pub instances: Vec, +} + +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] +pub struct EventUserData { + pub recurrence_key: u32, pub flags: u16, - pub properties: Vec, + pub color: Option, + pub keywords: Vec, pub alerts: Vec, } @@ -212,84 +405,171 @@ pub enum Timezone { } impl Calendar { - pub fn preferences(&self, account_id: u32) -> &CalendarPreferences { - if self.preferences.len() == 1 { - &self.preferences[0] - } else { - self.preferences - .iter() - .find(|p| p.account_id == account_id) - .or_else(|| self.preferences.first()) - .unwrap() + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.presence) + } + + pub fn personal_preferences(&self, account_id: u32) -> Option<&CalendarPreferences> { + self.preferences.iter().find(|p| p.account_id == account_id) + } + + pub fn subscribe_member(&mut self, account_id: u32) { + if self.personal_preferences(account_id).is_none() { + self.preferences_mut(account_id).flags |= CALENDAR_SUBSCRIBED; } } + pub fn preferences(&self, account_id: u32) -> &CalendarPreferences { + match self.personal_or_first(account_id) { + Some(preferences) => preferences, + None => CalendarPreferences::empty(), + } + } + + fn personal_or_first(&self, account_id: u32) -> Option<&CalendarPreferences> { + match self.preferences.as_slice() { + [preferences] => Some(preferences), + preferences => preferences + .iter() + .find(|p| p.account_id == account_id) + .or(preferences.first()), + } + } + + pub fn inherit_owner_preferences(&mut self, account_id: u32, personal_id: u32) { + if account_id == personal_id || self.personal_preferences(personal_id).is_some() { + return; + } + let Some(owner) = self.personal_preferences(account_id).cloned() else { + return; + }; + self.preferences.push(CalendarPreferences { + account_id: personal_id, + ..owner + }); + } + + pub fn sync_owner_preferences(&mut self, account_id: u32, personal_id: u32) { + if account_id == personal_id { + return; + } + let Some(member) = self + .preferences + .iter() + .find(|preferences| preferences.account_id == personal_id) + .cloned() + else { + return; + }; + *self.preferences_mut(account_id) = CalendarPreferences { + account_id, + ..member + }; + } + pub fn preferences_mut(&mut self, account_id: u32) -> &mut CalendarPreferences { - let idx = if let Some(idx) = self + if let Some(idx) = self .preferences .iter() .position(|p| p.account_id == account_id) { - idx - } else { - let mut preferences = self.preferences[0].clone(); - preferences.account_id = account_id; - self.preferences.push(preferences); - self.preferences.len() - 1 + return &mut self.preferences[idx]; + } + + let inherited = self + .preferences + .first() + .unwrap_or_else(|| CalendarPreferences::empty()); + let preferences = CalendarPreferences { + account_id, + name: inherited.name.clone(), + description: inherited.description.clone(), + color: inherited.color.clone(), + time_zone: inherited.time_zone.clone(), + ..Default::default() + }; + self.preferences.push_mut(preferences) + } +} + +impl CalendarPreferences { + pub fn empty() -> &'static Self { + static EMPTY: CalendarPreferences = CalendarPreferences { + account_id: NO_ID, + name: String::new(), + description: None, + sort_order: 0, + color: None, + flags: 0, + time_zone: Timezone::Default, + default_alerts: Vec::new(), }; - &mut self.preferences[idx] + &EMPTY } } impl ArchivedCalendar { + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.presence) + } + pub fn default_alerts( &self, account_id: u32, with_time: bool, ) -> impl Iterator { - self.preferences(account_id) - .default_alerts - .iter() + self.personal_preferences(account_id) + .into_iter() + .flat_map(|preferences| preferences.default_alerts.iter()) .filter(move |a| (a.flags & ALERT_WITH_TIME != 0) == with_time) } - pub fn preferences(&self, account_id: u32) -> &ArchivedCalendarPreferences { - if self.preferences.len() == 1 { - &self.preferences[0] - } else { - self.preferences + pub fn personal_preferences(&self, account_id: u32) -> Option<&ArchivedCalendarPreferences> { + self.preferences.iter().find(|p| p.account_id == account_id) + } + + pub fn personal_flags(&self, account_id: u32, is_member: bool) -> u16 { + self.personal_preferences(account_id) + .map_or_else(|| default_preference_flags(is_member), |p| p.flags.into()) + } + + pub fn preferences(&self, account_id: u32) -> Option<&ArchivedCalendarPreferences> { + match self.preferences.as_slice() { + [preferences] => Some(preferences), + preferences => preferences .iter() .find(|p| p.account_id == account_id) - .or_else(|| self.preferences.first()) - .unwrap() + .or(preferences.first()), } } } +pub const fn default_preference_flags(is_member: bool) -> u16 { + if is_member { CALENDAR_SUBSCRIBED } else { 0 } +} + impl CalendarEvent { - pub fn preferences(&self, account_id: u32) -> Option<&EventPreferences> { - self.preferences.iter().find(|p| p.account_id == account_id) + pub fn metadata_kinds(&self) -> MetadataKinds { + PresenceBits::CALENDAR_EVENT.kinds(self.flags) } - pub fn preferences_mut(&mut self, account_id: u32) -> &mut EventPreferences { - let idx = if let Some(idx) = self - .preferences - .iter() - .position(|p| p.account_id == account_id) - { - idx - } else { - self.preferences.push(EventPreferences { - account_id, - flags: 0, - properties: Vec::new(), - alerts: Vec::new(), - }); - self.preferences.len() - 1 - }; + pub fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.flags = PresenceBits::CALENDAR_EVENT.apply(self.flags, kinds); + } - &mut self.preferences[idx] + pub fn calendar_ids(&self) -> impl Iterator { + self.names.iter().map(DavName::parent_id) + } + + pub fn all_calendar_ids(&self, prev_data: &ArchivedCalendarEvent) -> Vec { + let mut calendar_ids = self.calendar_ids().collect::>(); + for calendar_id in prev_data.calendar_ids() { + if !calendar_ids.contains(&calendar_id) { + calendar_ids.push(calendar_id); + } + } + calendar_ids } pub fn added_calendar_ids( @@ -325,8 +605,12 @@ impl CalendarEvent { } impl ArchivedCalendarEvent { - pub fn preferences(&self, account_id: u32) -> Option<&ArchivedEventPreferences> { - self.preferences.iter().find(|p| p.account_id == account_id) + pub fn metadata_kinds(&self) -> MetadataKinds { + PresenceBits::CALENDAR_EVENT.kinds(self.flags.to_native()) + } + + pub fn calendar_ids(&self) -> impl Iterator { + self.names.iter().map(|name| name.parent_id.to_native()) } } @@ -437,3 +721,8 @@ impl From for ICalendarComponentType { } } } + +impl store::write::ArchiveCompression for ParticipantIdentities { + const COMPRESSION: store::write::Compression = + store::write::Compression::Zstd(Some(store::write::Dictionary::Common)); +} diff --git a/crates/groupware/src/calendar/notification.rs b/crates/groupware/src/calendar/notification.rs new file mode 100644 index 00000000..162c0525 --- /dev/null +++ b/crates/groupware/src/calendar/notification.rs @@ -0,0 +1,521 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedCalendarEventNotification, CALENDAR_SUBSCRIBED, CalendarEventNotification, + EVENT_HIDE_ATTENDEES, EVENT_NOTIFICATION_HIDE_ATTENDEES, EVENT_NOTIFICATION_PRIVATE, + default_preference_flags, privacy::EventPrivacy, +}; +use crate::{DestroyArchive, cache::GroupwareCache}; +use common::{ + GroupwareResources, Server, + auth::{AccessToken, AccountCache}, + storage::dav::resource::NotificationRef, +}; +use std::sync::Arc; +use store::{ + ValueKey, + roaring::RoaringBitmap, + write::{Archive, ArchiveBytes, BatchBuilder}, +}; +use trc::AddContext; +use types::{ + acl::{Acl, AclGrant}, + collection::{Collection, SyncCollection}, +}; + +pub struct NotificationViewer { + viewer_id: u32, + is_member: bool, + calendar_ids: RoaringBitmap, + known_calendar_ids: RoaringBitmap, + writable_calendar_ids: RoaringBitmap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NotificationAudience { + Readers, + Writers, + Members, +} + +impl NotificationViewer { + pub fn new( + calendars: &GroupwareResources, + access_token: &AccessToken, + account_id: u32, + ) -> Self { + let viewer_id = access_token.account_id(); + let is_member = access_token.is_member(account_id); + let mut calendar_ids = RoaringBitmap::new(); + let mut known_calendar_ids = RoaringBitmap::new(); + let mut writable_calendar_ids = RoaringBitmap::new(); + + for calendar in calendars.resources.iter().filter(|r| r.is_container()) { + known_calendar_ids.insert(calendar.document_id()); + let flags = calendar + .personal_calendar_preferences(viewer_id) + .map_or_else(|| default_preference_flags(is_member), |prefs| prefs.flags); + if flags & CALENDAR_SUBSCRIBED != 0 + && (is_member || has_grant(access_token, calendar.acls(), Acl::ReadItems)) + { + calendar_ids.insert(calendar.document_id()); + } + if !is_member && has_grant(access_token, calendar.acls(), Acl::ModifyItems) { + writable_calendar_ids.insert(calendar.document_id()); + } + } + + NotificationViewer { + viewer_id, + is_member, + calendar_ids, + known_calendar_ids, + writable_calendar_ids, + } + } + + pub fn without_calendars(access_token: &AccessToken, account_id: u32) -> Self { + NotificationViewer { + viewer_id: access_token.account_id(), + is_member: access_token.is_member(account_id), + calendar_ids: RoaringBitmap::new(), + known_calendar_ids: RoaringBitmap::new(), + writable_calendar_ids: RoaringBitmap::new(), + } + } + + pub fn is_member(&self) -> bool { + self.is_member + } + + pub fn can_view(&self, notification: &NotificationRef<'_>) -> bool { + notification.changed_by != self.viewer_id + && !notification.dismissed_by.contains(&self.viewer_id) + && match NotificationAudience::from_flags(notification.flags) { + NotificationAudience::Readers => true, + NotificationAudience::Writers => { + self.is_member + || (!notification.calendar_ids.is_empty() + && notification.calendar_ids.iter().all(|calendar_id| { + self.writable_calendar_ids.contains(*calendar_id) + })) + } + NotificationAudience::Members => self.is_member, + } + && if notification + .calendar_ids + .iter() + .any(|calendar_id| self.known_calendar_ids.contains(*calendar_id)) + { + notification + .calendar_ids + .iter() + .any(|calendar_id| self.calendar_ids.contains(*calendar_id)) + } else { + self.is_member + } + } + + pub fn visible_notifications(&self, notifications: &GroupwareResources) -> RoaringBitmap { + notifications + .resources + .iter() + .filter(|resource| { + !resource.is_container() + && resource + .notification() + .is_some_and(|notification| self.can_view(¬ification)) + }) + .map(|resource| resource.document_id()) + .collect() + } +} + +impl NotificationAudience { + pub fn of_event(event_flags: u16) -> Self { + if !EventPrivacy::from_flags(event_flags).is_public() { + NotificationAudience::Members + } else if event_flags & EVENT_HIDE_ATTENDEES != 0 { + NotificationAudience::Writers + } else { + NotificationAudience::Readers + } + } + + pub fn from_flags(notification_flags: u16) -> Self { + if notification_flags & EVENT_NOTIFICATION_PRIVATE != 0 { + NotificationAudience::Members + } else if notification_flags & EVENT_NOTIFICATION_HIDE_ATTENDEES != 0 { + NotificationAudience::Writers + } else { + NotificationAudience::Readers + } + } + + pub fn with_privacy(self, is_public: bool) -> Self { + if is_public { + self + } else { + NotificationAudience::Members + } + } + + pub fn flags(self) -> u16 { + match self { + NotificationAudience::Readers => 0, + NotificationAudience::Writers => EVENT_NOTIFICATION_HIDE_ATTENDEES, + NotificationAudience::Members => EVENT_NOTIFICATION_PRIVATE, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccountViewers { + Owner, + Members, +} + +impl AccountViewers { + pub fn of(account: &AccountCache) -> Self { + if account.is_user_account() { + AccountViewers::Owner + } else { + AccountViewers::Members + } + } +} + +pub fn may_have_viewers( + access_token: &AccessToken, + account_id: u32, + calendars: &GroupwareResources, + calendar_ids: &[u32], + audience: NotificationAudience, +) -> bool { + has_viewers( + access_token, + account_id, + calendars, + calendar_ids, + audience, + AccountViewers::Members, + ) +} + +pub fn has_viewers( + access_token: &AccessToken, + account_id: u32, + calendars: &GroupwareResources, + calendar_ids: &[u32], + audience: NotificationAudience, + account_viewers: AccountViewers, +) -> bool { + if calendar_ids.is_empty() { + return !access_token.is_account_id(account_id); + } else if !access_token.is_account_id(account_id) + && match account_viewers { + AccountViewers::Owner => is_subscribed(calendars, calendar_ids, account_id), + AccountViewers::Members => true, + } + { + return true; + } + + subscribed_viewers(calendars, calendar_ids, audience) + .any(|reader_id| !is_actor(access_token, reader_id)) +} + +fn is_actor(access_token: &AccessToken, account_id: u32) -> bool { + access_token + .member_ids() + .any(|member_id| member_id == account_id) +} + +fn is_subscribed(calendars: &GroupwareResources, calendar_ids: &[u32], account_id: u32) -> bool { + calendar_ids + .iter() + .filter_map(|calendar_id| calendars.resources.find(*calendar_id, true)) + .any(|calendar| { + calendar + .personal_calendar_preferences(account_id) + .is_some_and(|preferences| preferences.flags & CALENDAR_SUBSCRIBED != 0) + }) +} + +fn subscribed_readers<'x>( + calendars: &'x GroupwareResources, + calendar_ids: &'x [u32], +) -> impl Iterator + 'x { + calendar_ids + .iter() + .filter_map(|calendar_id| calendars.resources.find(*calendar_id, true)) + .flat_map(|calendar| { + let preferences = calendar.all_calendar_preferences(); + calendar + .acls() + .iter() + .filter(|acl| acl.grants.contains(Acl::ReadItems)) + .map(|acl| acl.account_id) + .filter(move |account_id| { + preferences.iter().any(|preferences| { + preferences.account_id == *account_id + && preferences.flags & CALENDAR_SUBSCRIBED != 0 + }) + }) + }) +} + +fn subscribed_viewers<'x>( + calendars: &'x GroupwareResources, + calendar_ids: &'x [u32], + audience: NotificationAudience, +) -> impl Iterator + 'x { + subscribed_readers(calendars, calendar_ids).filter(move |reader_id| match audience { + NotificationAudience::Readers => true, + NotificationAudience::Writers => may_modify_all(calendars, calendar_ids, *reader_id), + NotificationAudience::Members => false, + }) +} + +fn may_modify_all(calendars: &GroupwareResources, calendar_ids: &[u32], account_id: u32) -> bool { + calendar_ids.iter().all(|calendar_id| { + calendars + .resources + .find(*calendar_id, true) + .is_some_and(|calendar| { + calendar.acls().iter().any(|acl| { + acl.account_id == account_id && acl.grants.contains(Acl::ModifyItems) + }) + }) + }) +} + +fn has_grant(access_token: &AccessToken, acls: &[AclGrant], grant: Acl) -> bool { + acls.iter() + .any(|acl| acl.grants.contains(grant) && access_token.is_member(acl.account_id)) +} + +pub trait CalendarNotificationViewers: Sync + Send { + fn notification_viewer( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> impl Future> + Send; + + fn calendars_if_any( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> impl Future>>> + Send; +} + +impl CalendarNotificationViewers for Server { + async fn notification_viewer( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> trc::Result { + Ok( + match self + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + { + Some(calendars) => NotificationViewer::new(&calendars, access_token, account_id), + None => NotificationViewer::without_calendars(access_token, account_id), + }, + ) + } + + async fn calendars_if_any( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> trc::Result>> { + if self + .count_documents(account_id, Collection::Calendar, 1) + .await + .caused_by(trc::location!())? + == 0 + { + return Ok(None); + } + + self.fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::Calendar, + ) + .await + .caused_by(trc::location!()) + .map(Some) + } +} + +pub trait CalendarNotificationReap: Sync + Send { + fn reap_calendar_notifications( + &self, + access_token: &AccessToken, + account_id: u32, + calendar_ids: &[u32], + batch: &mut BatchBuilder, + ) -> impl Future> + Send; +} + +impl CalendarNotificationReap for Server { + async fn reap_calendar_notifications( + &self, + access_token: &AccessToken, + account_id: u32, + calendar_ids: &[u32], + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let notifications = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::CalendarEventNotification, + ) + .await + .caused_by(trc::location!())?; + let is_member = access_token.is_member(account_id); + let orphaned = notifications + .resources + .iter() + .filter(|resource| { + !resource.is_container() + && resource.notification().is_some_and(|notification| { + (is_member || notification.changed_by == account_id) + && !notification.calendar_ids.is_empty() + && notification + .calendar_ids + .iter() + .all(|calendar_id| calendar_ids.contains(calendar_id)) + }) + }) + .map(|resource| resource.document_id()) + .collect::>(); + + for document_id in orphaned { + if let Some(notification) = self + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEventNotification, + document_id, + )) + .await + .caused_by(trc::location!())? + { + DestroyArchive( + notification + .to_unarchived::() + .caused_by(trc::location!())?, + ) + .delete(access_token.changed_by(), account_id, document_id, batch) + .caused_by(trc::location!())?; + } + } + + Ok(()) + } +} + +pub trait CalendarNotificationDismiss: Sync + Send { + fn dismiss_notification( + &self, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + notification: Archive<&ArchivedCalendarEventNotification>, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + fn has_pending_viewers( + &self, + access_token: &AccessToken, + account_id: u32, + notification: &ArchivedCalendarEventNotification, + dismissed_by: &[u32], + ) -> impl Future> + Send; +} + +impl CalendarNotificationDismiss for Server { + async fn dismiss_notification( + &self, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + notification: Archive<&ArchivedCalendarEventNotification>, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let viewer_id = access_token.account_id(); + let is_owner_only = NotificationAudience::from_flags(notification.inner.flags.to_native()) + == NotificationAudience::Members; + let mut dismissed_by = notification.inner.dismissed_ids().collect::>(); + if !dismissed_by.contains(&viewer_id) { + dismissed_by.push(viewer_id); + } + + if !(is_owner_only && access_token.is_member(account_id)) + && self + .has_pending_viewers(access_token, account_id, notification.inner, &dismissed_by) + .await + .caused_by(trc::location!())? + { + let mut updated = notification + .deserialize::() + .caused_by(trc::location!())?; + updated.dismissed_by = dismissed_by; + updated + .update_meta( + access_token.changed_by(), + notification, + account_id, + document_id, + batch, + ) + .caused_by(trc::location!())?; + } else { + DestroyArchive(notification) + .delete(access_token.changed_by(), account_id, document_id, batch) + .caused_by(trc::location!())?; + } + + Ok(()) + } + + async fn has_pending_viewers( + &self, + access_token: &AccessToken, + account_id: u32, + notification: &ArchivedCalendarEventNotification, + dismissed_by: &[u32], + ) -> trc::Result { + let changed_by = notification.changed_by_id(); + let audience = NotificationAudience::from_flags(notification.flags.to_native()); + if changed_by != account_id && !dismissed_by.contains(&account_id) { + return Ok(true); + } else if audience == NotificationAudience::Members || notification.calendar_ids.is_empty() + { + return Ok(false); + } + + let Some(calendars) = self + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + else { + return Ok(false); + }; + + let calendar_ids = notification.calendar_ids().collect::>(); + Ok(subscribed_viewers(&calendars, &calendar_ids, audience) + .any(|account_id| account_id != changed_by && !dismissed_by.contains(&account_id))) + } +} + diff --git a/crates/groupware/src/calendar/overlap.rs b/crates/groupware/src/calendar/overlap.rs new file mode 100644 index 00000000..5e055c47 --- /dev/null +++ b/crates/groupware/src/calendar/overlap.rs @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ArchivedCalendarEventData, CalendarEventData}; +use calcard::icalendar::{ + ArchivedICalendarEntry, ArchivedICalendarProperty, ArchivedICalendarValue, ICalendar, + ICalendarEntry, ICalendarProperty, ICalendarValue, +}; +use types::OverlapCondition; + +pub trait ICalendarOverlap { + fn overlap_condition(&self, comp_id: u32) -> OverlapCondition; +} + +#[derive(Debug, Default, Clone, Copy)] +struct TodoDates { + start: bool, + recurrence_id: bool, + end: bool, + duration: bool, + due: bool, + completed: bool, + created: bool, +} + +trait TodoDateEntry { + fn add_to(&self, dates: &mut TodoDates); +} + +impl ICalendarOverlap for ICalendar { + fn overlap_condition(&self, comp_id: u32) -> OverlapCondition { + let Some(component) = self + .component_by_id(comp_id) + .filter(|component| component.component_type.is_todo()) + else { + return OverlapCondition::Event; + }; + let dates = TodoDates::of(&component.entries); + dates + .inherits_length() + .then(|| { + let uid = component.uid(); + self.components.iter().find(|series| { + series.component_type == component.component_type + && series.uid() == uid + && !series.is_recurrence_override() + && series.entries.iter().any(|entry| { + matches!( + entry.name, + ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exdate + ) + }) + }) + }) + .flatten() + .and_then(|series| TodoDates::of(&series.entries).condition()) + .or_else(|| dates.condition()) + .unwrap_or_default() + } +} + +impl CalendarEventData { + pub fn has_unbounded_todo(&self) -> bool { + self.event.components.iter().any(|component| { + component.component_type.is_todo() && TodoDates::of(&component.entries).is_unbounded() + }) + } + + pub fn undated_todos(&self) -> impl Iterator + '_ { + self.event + .components + .iter() + .zip(0u32..) + .filter(|(component, _)| { + component.component_type.is_todo() + && TodoDates::of(&component.entries).condition().is_none() + }) + .map(|(_, comp_id)| comp_id) + } +} + +impl ArchivedCalendarEventData { + pub fn undated_todos(&self) -> impl Iterator + '_ { + self.event + .components + .iter() + .zip(0u32..) + .filter(|(component, _)| { + component.component_type.is_todo() + && TodoDates::of(component.entries.iter()) + .condition() + .is_none() + }) + .map(|(_, comp_id)| comp_id) + } +} + +impl TodoDates { + fn of<'x, E: TodoDateEntry + 'x>(entries: impl IntoIterator) -> Self { + let mut dates = TodoDates::default(); + for entry in entries { + entry.add_to(&mut dates); + } + dates + } + + fn is_unbounded(self) -> bool { + matches!(self.condition(), None | Some(OverlapCondition::TodoCreated)) + } + + fn inherits_length(self) -> bool { + self.recurrence_id && !(self.start || self.end || self.duration || self.due) + } + + fn condition(self) -> Option { + match ( + self.start || self.recurrence_id, + self.duration, + self.due, + self.completed, + self.created, + ) { + (true, true, _, _, _) => Some(OverlapCondition::TodoStartDuration), + (true, false, true, _, _) => Some(OverlapCondition::TodoStartDue), + (true, false, false, _, _) => Some(OverlapCondition::TodoStart), + (false, _, true, _, _) => Some(OverlapCondition::TodoDue), + (false, _, false, true, true) => Some(OverlapCondition::TodoCreatedCompleted), + (false, _, false, true, false) => Some(OverlapCondition::TodoCompleted), + (false, _, false, false, true) => Some(OverlapCondition::TodoCreated), + (false, _, false, false, false) => None, + } + } +} + +impl TodoDateEntry for ICalendarEntry { + fn add_to(&self, dates: &mut TodoDates) { + let is_date_time = || { + matches!( + self.values.first(), + Some(ICalendarValue::PartialDateTime(value)) if value.to_date_time().is_some() + ) + }; + match self.name { + ICalendarProperty::Dtstart => dates.start |= is_date_time(), + ICalendarProperty::RecurrenceId => dates.recurrence_id |= is_date_time(), + ICalendarProperty::Dtend => dates.end |= is_date_time(), + ICalendarProperty::Due => dates.due |= is_date_time(), + ICalendarProperty::Completed => dates.completed |= is_date_time(), + ICalendarProperty::Created => dates.created |= is_date_time(), + ICalendarProperty::Duration => { + dates.duration |= matches!(self.values.first(), Some(ICalendarValue::Duration(_))); + } + _ => {} + } + } +} + +impl TodoDateEntry for ArchivedICalendarEntry { + fn add_to(&self, dates: &mut TodoDates) { + let is_date_time = || { + matches!( + self.values.first(), + Some(ArchivedICalendarValue::PartialDateTime(value)) if value.to_date_time().is_some() + ) + }; + match self.name { + ArchivedICalendarProperty::Dtstart => dates.start |= is_date_time(), + ArchivedICalendarProperty::RecurrenceId => dates.recurrence_id |= is_date_time(), + ArchivedICalendarProperty::Dtend => dates.end |= is_date_time(), + ArchivedICalendarProperty::Due => dates.due |= is_date_time(), + ArchivedICalendarProperty::Completed => dates.completed |= is_date_time(), + ArchivedICalendarProperty::Created => dates.created |= is_date_time(), + ArchivedICalendarProperty::Duration => { + dates.duration |= matches!( + self.values.first(), + Some(ArchivedICalendarValue::Duration(_)) + ); + } + _ => {} + } + } +} + diff --git a/crates/groupware/src/calendar/participants.rs b/crates/groupware/src/calendar/participants.rs new file mode 100644 index 00000000..9b959dac --- /dev/null +++ b/crates/groupware/src/calendar/participants.rs @@ -0,0 +1,639 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::identity::CalendarAddresses; +use calcard::{ + icalendar::{ + ArchivedICalendarEntry, ArchivedICalendarParameterValue, + ArchivedICalendarParticipationRole, ArchivedICalendarProperty, ArchivedICalendarValue, + ICalendarEntry, ICalendarParameterName, ICalendarParameterValue, + ICalendarParticipationRole, ICalendarProperty, + }, + jscalendar::{JSCalendarParticipantRole, JSCalendarProperty, JSCalendarValue}, +}; +use jmap_tools::{JsonPointer, JsonPointerHandler, JsonPointerItem, Key, Map, Value}; +use std::borrow::Cow; +use types::{blob::BlobId, id::Id}; + +type EventKey<'x> = Key<'x, JSCalendarProperty>; +type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; +type EventPointerItem = JsonPointerItem>; + +pub trait ParticipantVisibility { + fn is_visible_participant(&self, identities: &CalendarAddresses) -> bool; +} + +pub trait VisibleParticipants { + fn retain_visible_participants(&mut self, identities: &CalendarAddresses); +} + +impl ParticipantVisibility for ICalendarEntry { + fn is_visible_participant(&self, identities: &CalendarAddresses) -> bool { + self.name != ICalendarProperty::Attendee + || self.parameters(&ICalendarParameterName::Role).any(|role| { + matches!( + role, + ICalendarParameterValue::Role(ICalendarParticipationRole::Owner) + ) + }) + || self + .calendar_address() + .is_some_and(|address| identities.contains(address)) + } +} + +impl ParticipantVisibility for ArchivedICalendarEntry { + fn is_visible_participant(&self, identities: &CalendarAddresses) -> bool { + !matches!(self.name, ArchivedICalendarProperty::Attendee) + || self.has_owner_role() + || self + .values + .first() + .and_then(|value| value.as_text()) + .is_some_and(|address| identities.contains(address)) + } +} + +pub(crate) trait ArchivedParticipantEntry { + fn has_owner_role(&self) -> bool; + + fn name_and_address(&self) -> impl Iterator; +} + +impl ArchivedParticipantEntry for ArchivedICalendarEntry { + fn has_owner_role(&self) -> bool { + self.parameters(&ICalendarParameterName::Role).any(|role| { + matches!( + role, + ArchivedICalendarParameterValue::Role(ArchivedICalendarParticipationRole::Owner) + ) + }) + } + + fn name_and_address(&self) -> impl Iterator { + self.values + .iter() + .filter_map(|value| match value { + ArchivedICalendarValue::Text(value) => Some(value.as_str()), + ArchivedICalendarValue::Uri(uri) => uri.as_str(), + _ => None, + }) + .chain( + self.parameters(&ICalendarParameterName::Cn) + .filter_map(|value| match value { + ArchivedICalendarParameterValue::Text(value) => Some(value.as_str()), + _ => None, + }), + ) + } +} + +impl ParticipantVisibility for EventValue<'_> { + fn is_visible_participant(&self, identities: &CalendarAddresses) -> bool { + self.is_owner() + || self + .calendar_address() + .is_some_and(|address| identities.contains(&address)) + } +} + +impl VisibleParticipants for EventValue<'_> { + fn retain_visible_participants(&mut self, identities: &CalendarAddresses) { + let Some(event) = self.as_object_mut() else { + return; + }; + let participants_key = Key::Property(JSCalendarProperty::Participants); + let position = event + .as_vec() + .iter() + .position(|(key, _)| key == &participants_key); + let mut participants = position.map(|position| event.as_mut_vec().remove(position)); + let base = participants + .as_ref() + .and_then(|(_, participants)| participants.as_object()); + let base_hidden = base.map_or_else(Vec::new, |base| base.hidden_ids(identities)); + let base_visible = base.map_or_else(Vec::new, |base| base.visible_addresses(identities)); + + if let Some(overrides) = event + .get_mut(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .and_then(Value::as_object_mut) + { + overrides.as_mut_vec().retain_mut(|(_, patch)| { + let Some(patch) = patch.as_object_mut() else { + return true; + }; + let is_empty = patch.is_empty(); + patch.retain_visible_patches(base, &base_hidden, identities); + patch.retain_visible_patch_delegations(&base_visible, identities); + is_empty || !patch.is_empty() + }); + } + + if let Some(ical) = event.get_mut(&Key::Property(JSCalendarProperty::ICalendar)) { + ical.remove_participant_properties(&base_hidden); + } + + if let (Some(position), Some((key, mut value))) = (position, participants.take()) { + if let Some(participants) = value.as_object_mut() { + participants.remove_hidden(identities); + participants.retain_visible_delegations(identities); + } + if value + .as_object() + .is_none_or(|participants| !participants.is_empty()) + { + let position = position.min(event.len()); + event.as_mut_vec().insert(position, (key, value)); + } + } + } +} + +pub(crate) trait ParticipantObject { + fn is_owner(&self) -> bool; + + fn calendar_address(&self) -> Option>; +} + +impl ParticipantObject for EventValue<'_> { + fn is_owner(&self) -> bool { + self.as_object_and_get(&Key::Property(JSCalendarProperty::Roles)) + .is_some_and(EventValue::has_owner_role) + } + + fn calendar_address(&self) -> Option> { + self.as_object_and_get(&Key::Property(JSCalendarProperty::CalendarAddress)) + .and_then(Value::as_str) + } +} + +trait OwnerRole { + fn has_owner_role(&self) -> bool; +} + +impl OwnerRole for EventValue<'_> { + fn has_owner_role(&self) -> bool { + self.as_object_and_get(&Key::Property(JSCalendarProperty::ParticipantRole( + JSCalendarParticipantRole::Owner, + ))) + .and_then(Value::as_bool) + .unwrap_or_default() + } +} + +trait ParticipantMap<'x> { + fn hidden_ids(&self, identities: &CalendarAddresses) -> Vec>; + + fn visible_addresses(&self, identities: &CalendarAddresses) -> Vec>; + + fn retain_visible_delegations(&mut self, identities: &CalendarAddresses); + + fn remove_hidden(&mut self, identities: &CalendarAddresses); + + fn participant(&self, id: &str) -> Option<&EventValue<'x>>; +} + +impl<'x> ParticipantMap<'x> for EventMap<'x> { + fn hidden_ids(&self, identities: &CalendarAddresses) -> Vec> { + self.iter() + .filter(|(_, participant)| !participant.is_visible_participant(identities)) + .map(|(key, _)| key.to_string()) + .collect() + } + + fn visible_addresses(&self, identities: &CalendarAddresses) -> Vec> { + self.iter() + .filter(|(_, participant)| participant.is_visible_participant(identities)) + .filter_map(|(_, participant)| participant.calendar_address()) + .collect() + } + + fn retain_visible_delegations(&mut self, identities: &CalendarAddresses) { + if !self.values().any(EventValue::has_delegates) { + return; + } + let participants = self.as_mut_vec(); + for index in 0..participants.len() { + let (before, rest) = participants.split_at_mut(index); + let Some(((_, participant), after)) = rest.split_first_mut() else { + break; + }; + participant.retain_visible_delegations(|address| { + identities.contains(address) || { + let address = CalendarAddresses::normalize(address); + before + .iter() + .chain(after.iter()) + .filter_map(|(_, participant)| participant.calendar_address()) + .any(|own| CalendarAddresses::normalize(&own) == address) + } + }); + } + } + + fn remove_hidden(&mut self, identities: &CalendarAddresses) { + self.as_mut_vec() + .retain(|(_, participant)| participant.is_visible_participant(identities)); + } + + fn participant(&self, id: &str) -> Option<&EventValue<'x>> { + self.as_vec() + .iter() + .find(|(key, _)| key.to_string() == id) + .map(|(_, participant)| participant) + } +} + +trait ParticipantDelegations { + fn has_delegates(&self) -> bool; + + fn retain_visible_delegations(&mut self, is_visible: impl Fn(&str) -> bool + Copy); + + fn retain_visible_delegates(&mut self, is_visible: impl Fn(&str) -> bool + Copy) -> bool; +} + +impl ParticipantDelegations for EventValue<'_> { + fn has_delegates(&self) -> bool { + self.as_object() + .is_some_and(|participant| participant.keys().any(EventKey::is_delegation)) + } + + fn retain_visible_delegations(&mut self, is_visible: impl Fn(&str) -> bool + Copy) { + if let Some(participant) = self.as_object_mut() { + participant.as_mut_vec().retain_mut(|(key, value)| { + !key.is_delegation() || value.retain_visible_delegates(is_visible) + }); + } + } + + fn retain_visible_delegates(&mut self, is_visible: impl Fn(&str) -> bool + Copy) -> bool { + match self { + Value::Str(address) => is_visible(address), + Value::Array(addresses) => { + addresses + .retain(|address| address.as_str().is_none_or(|address| is_visible(&address))); + !addresses.is_empty() + } + Value::Object(addresses) => { + addresses + .as_mut_vec() + .retain(|(address, _)| is_visible(&address.to_string())); + !addresses.is_empty() + } + _ => true, + } + } +} + +trait VisibleAddress { + fn is_visible_address(&self, visible: &[Cow<'_, str>], identities: &CalendarAddresses) -> bool; +} + +impl VisibleAddress for str { + fn is_visible_address(&self, visible: &[Cow<'_, str>], identities: &CalendarAddresses) -> bool { + let address = CalendarAddresses::normalize(self); + visible + .iter() + .any(|visible| CalendarAddresses::normalize(visible) == address) + || identities.contains(&address) + } +} + +trait ParticipantPatch<'x> { + fn retain_visible_patches( + &mut self, + base: Option<&EventMap<'x>>, + base_hidden: &[Cow<'_, str>], + identities: &CalendarAddresses, + ); + + fn retain_visible_patch_delegations( + &mut self, + base_visible: &[Cow<'_, str>], + identities: &CalendarAddresses, + ); + + fn participant_patches<'y>( + &'y self, + id: &'y str, + ) -> impl Iterator)> + where + 'x: 'y; +} + +impl<'x> ParticipantPatch<'x> for EventMap<'x> { + fn retain_visible_patches( + &mut self, + base: Option<&EventMap<'x>>, + base_hidden: &[Cow<'_, str>], + identities: &CalendarAddresses, + ) { + if let Some(participants) = self.get_mut(&Key::Property(JSCalendarProperty::Participants)) + && let Some(map) = participants.as_object_mut() + { + map.remove_hidden(identities); + if map.is_empty() { + *participants = Value::Null; + } + } + + let mut ids = self + .iter() + .filter_map(|(key, _)| key.participant_pointer()) + .map(|(id, _)| id.into_owned()) + .collect::>(); + ids.sort_unstable(); + ids.dedup(); + + let mut hidden = Cow::Borrowed(base_hidden); + for id in ids { + let base_participant = base.and_then(|base| base.participant(&id)); + let is_base_visible = base_participant + .is_some_and(|participant| participant.is_visible_participant(identities)); + let replacement = self + .participant_patches(&id) + .find(|(pointer, _)| pointer.is_empty()) + .map(|(_, value)| value); + let is_visible = match replacement { + Some(replacement) => replacement.is_visible_participant(identities), + None => base_participant.is_some_and(|participant| { + participant.is_patched_visible(self.participant_patches(&id), identities) + }), + }; + let new_patch = match (is_base_visible, is_visible) { + (true, true) => continue, + (false, true) if replacement.is_some() => { + hidden + .to_mut() + .retain(|hidden_id| hidden_id.as_ref() != id.as_str()); + continue; + } + (false, true) => base_participant.map(|participant| { + let mut participant = participant.clone(); + for (pointer, value) in self.participant_patches(&id) { + participant.patch_jptr(pointer.iter().peekable(), value.clone()); + } + participant + }), + (true, false) => Some(Value::Null), + (false, false) => None, + }; + + self.as_mut_vec().retain(|(key, _)| { + key.participant_pointer() + .is_none_or(|(patch_id, _)| patch_id != id.as_str()) + }); + if new_patch.as_ref().is_some_and(|patch| !patch.is_null()) { + hidden + .to_mut() + .retain(|hidden_id| hidden_id.as_ref() != id.as_str()); + } else if !hidden + .iter() + .any(|hidden_id| hidden_id.as_ref() == id.as_str()) + { + hidden.to_mut().push(Cow::Owned(id.clone())); + } + if let Some(new_patch) = new_patch { + self.insert_unchecked( + Key::Property(JSCalendarProperty::Pointer(JsonPointer::new(vec![ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + JsonPointerItem::Key(Key::Owned(id)), + ]))), + new_patch, + ); + } + } + + if let Some(ical) = self.get_mut(&Key::Property(JSCalendarProperty::ICalendar)) { + ical.remove_participant_properties(&hidden); + } + } + + fn retain_visible_patch_delegations( + &mut self, + base_visible: &[Cow<'_, str>], + identities: &CalendarAddresses, + ) { + let participants_key = Key::Property(JSCalendarProperty::Participants); + if !self.iter().any(|(key, value)| { + if key == &participants_key { + value.as_object().is_some_and(|participants| { + participants.values().any(EventValue::has_delegates) + }) + } else { + match key.participant_pointer() { + Some((_, [])) => value.has_delegates(), + Some((_, [delegation, ..])) => delegation.is_delegation(), + _ => false, + } + } + }) { + return; + } + + let mut visible = Cow::Borrowed(base_visible); + for (key, value) in self.iter() { + if key == &participants_key { + if let Some(participants) = value.as_object() { + visible.to_mut().extend( + participants + .visible_addresses(identities) + .into_iter() + .map(|address| Cow::Owned(address.into_owned())), + ); + } + } else if matches!(key.participant_pointer(), Some((_, []))) + && value.is_visible_participant(identities) + && let Some(address) = value.calendar_address() + { + visible.to_mut().push(Cow::Owned(address.into_owned())); + } + } + + let is_visible = |address: &str| address.is_visible_address(&visible, identities); + self.as_mut_vec().retain_mut(|(key, value)| { + if key == &participants_key { + if let Some(participants) = value.as_object_mut() { + participants.retain_visible_delegations(identities); + } + } else if let Some((_, pointer)) = key.participant_pointer() { + match pointer { + [] => value.retain_visible_delegations(is_visible), + [delegation] if delegation.is_delegation() => { + if !value.retain_visible_delegates(is_visible) { + *value = Value::Null; + } + } + [delegation, delegate] if delegation.is_delegation() => { + return delegate.as_address().is_none_or(|delegate| { + delegate.is_visible_address(&visible, identities) + }); + } + _ => {} + } + } + true + }); + } + + fn participant_patches<'y>( + &'y self, + id: &'y str, + ) -> impl Iterator)> + where + 'x: 'y, + { + self.iter().filter_map(move |(key, value)| { + key.participant_pointer() + .filter(|(patch_id, _)| patch_id == id) + .map(|(_, pointer)| (pointer, value)) + }) + } +} + +trait PatchedParticipant { + fn is_patched_visible<'a, 'v: 'a>( + &'a self, + patches: impl Iterator)>, + identities: &CalendarAddresses, + ) -> bool; +} + +impl PatchedParticipant for EventValue<'_> { + fn is_patched_visible<'a, 'v: 'a>( + &'a self, + patches: impl Iterator)>, + identities: &CalendarAddresses, + ) -> bool { + let mut is_owner = self.is_owner(); + let mut address = self.calendar_address(); + + for (pointer, value) in patches { + match pointer { + [roles] if roles.is_property(&JSCalendarProperty::Roles) => { + is_owner = value.has_owner_role(); + } + [roles, role] + if roles.is_property(&JSCalendarProperty::Roles) + && role.is_property(&JSCalendarProperty::ParticipantRole( + JSCalendarParticipantRole::Owner, + )) => + { + is_owner = value.as_bool().unwrap_or_default(); + } + [calendar_address] + if calendar_address.is_property(&JSCalendarProperty::CalendarAddress) => + { + address = value.as_str(); + } + _ => {} + } + } + + is_owner || address.is_some_and(|address| identities.contains(&address)) + } +} + +pub(crate) trait ParticipantPointer { + fn participant_pointer(&self) -> Option<(Cow<'_, str>, &[EventPointerItem])>; +} + +impl ParticipantPointer for EventKey<'_> { + fn participant_pointer(&self) -> Option<(Cow<'_, str>, &[EventPointerItem])> { + let Key::Property(JSCalendarProperty::Pointer(pointer)) = self else { + return None; + }; + match pointer.as_slice() { + [participants, id, rest @ ..] + if participants.is_property(&JSCalendarProperty::Participants) => + { + let id = match id { + JsonPointerItem::Key(key) => key.to_string(), + JsonPointerItem::Number(number) => Cow::Owned(number.to_string()), + JsonPointerItem::Root + | JsonPointerItem::Wildcard + | JsonPointerItem::Invalid(_) => return None, + }; + Some((id, rest)) + } + _ => None, + } + } +} + +trait PointerProperty { + fn is_property(&self, property: &JSCalendarProperty) -> bool; + + fn as_address(&self) -> Option>; +} + +impl PointerProperty for EventPointerItem { + fn is_property(&self, property: &JSCalendarProperty) -> bool { + matches!(self, JsonPointerItem::Key(Key::Property(item)) if item == property) + } + + fn as_address(&self) -> Option> { + match self { + JsonPointerItem::Key(key) => Some(key.to_string()), + _ => None, + } + } +} + +trait DelegationKey { + fn is_delegation(&self) -> bool; +} + +impl DelegationKey for EventKey<'_> { + fn is_delegation(&self) -> bool { + matches!( + self, + Key::Property(JSCalendarProperty::DelegatedTo | JSCalendarProperty::DelegatedFrom) + ) + } +} + +impl DelegationKey for EventPointerItem { + fn is_delegation(&self) -> bool { + self.is_property(&JSCalendarProperty::DelegatedTo) + || self.is_property(&JSCalendarProperty::DelegatedFrom) + } +} + +trait ICalendarProperties { + fn remove_participant_properties(&mut self, hidden: &[Cow<'_, str>]); +} + +impl ICalendarProperties for EventValue<'_> { + fn remove_participant_properties(&mut self, hidden: &[Cow<'_, str>]) { + if hidden.is_empty() { + return; + } + if let Some(converted_properties) = self + .as_object_mut() + .and_then(|ical| ical.get_mut(&Key::Property(JSCalendarProperty::ConvertedProperties))) + .and_then(Value::as_object_mut) + { + converted_properties.as_mut_vec().retain(|(key, _)| { + key.to_string() + .strip_prefix("participants/") + .and_then(|pointer| pointer.split('/').next()) + .is_none_or(|id| { + let id = if id.contains('~') { + Cow::Owned(id.replace("~1", "/").replace("~0", "~")) + } else { + Cow::Borrowed(id) + }; + !hidden + .iter() + .any(|hidden_id| hidden_id.as_ref() == id.as_ref()) + }) + }); + } + } +} + diff --git a/crates/groupware/src/calendar/privacy.rs b/crates/groupware/src/calendar/privacy.rs new file mode 100644 index 00000000..55147701 --- /dev/null +++ b/crates/groupware/src/calendar/privacy.rs @@ -0,0 +1,285 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CalendarEventContent, CalendarEventData, EVENT_PRIVATE, EVENT_SECRET, + compare::RedundantOverrides, +}; +use calcard::icalendar::{ + ICalendar, ICalendarClassification, ICalendarComponent, ICalendarComponentType, ICalendarEntry, + ICalendarParameterName, ICalendarProperty, ICalendarValue, +}; +use common::auth::AccessToken; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EventPrivacy { + #[default] + Public, + Private, + Secret, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrivacyDenied { + Forbidden, + NotFound, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventViewer { + Owner, + Sharee, +} + +impl EventViewer { + pub fn new(is_owner: bool) -> Self { + if is_owner { + EventViewer::Owner + } else { + EventViewer::Sharee + } + } + + pub fn of(access_token: &AccessToken, account_id: u32) -> Self { + EventViewer::new(access_token.is_member(account_id)) + } + + pub fn is_owner(self) -> bool { + self == EventViewer::Owner + } +} + +impl EventPrivacy { + pub fn from_flags(flags: u16) -> Self { + if flags & EVENT_SECRET != 0 { + EventPrivacy::Secret + } else if flags & EVENT_PRIVATE != 0 { + EventPrivacy::Private + } else { + EventPrivacy::Public + } + } + + pub fn apply_to_flags(self, flags: u16) -> u16 { + let flags = flags & !(EVENT_PRIVATE | EVENT_SECRET); + match self { + EventPrivacy::Public => flags, + EventPrivacy::Private => flags | EVENT_PRIVATE, + EventPrivacy::Secret => flags | EVENT_SECRET, + } + } + + pub fn from_value(value: &ICalendarValue) -> Self { + match value { + ICalendarValue::Classification(ICalendarClassification::Public) => EventPrivacy::Public, + ICalendarValue::Classification(ICalendarClassification::Confidential) => { + EventPrivacy::Secret + } + _ => EventPrivacy::Private, + } + } + + pub fn is_public(self) -> bool { + self == EventPrivacy::Public + } + + pub fn check_access(self, viewer: EventViewer) -> Result<(), PrivacyDenied> { + match self { + _ if viewer.is_owner() => Ok(()), + EventPrivacy::Public => Ok(()), + EventPrivacy::Private => Err(PrivacyDenied::Forbidden), + EventPrivacy::Secret => Err(PrivacyDenied::NotFound), + } + } + + pub fn may_be_set_by(self, viewer: EventViewer) -> bool { + viewer.is_owner() || self.is_public() + } + + fn classification(self) -> ICalendarClassification { + match self { + EventPrivacy::Public => ICalendarClassification::Public, + EventPrivacy::Private => ICalendarClassification::Private, + EventPrivacy::Secret => ICalendarClassification::Confidential, + } + } +} + +impl CalendarEventContent { + pub fn into_private_view(self) -> Self { + CalendarEventContent { + data: CalendarEventData { + event: self.data.event.into_private_view(), + alarms: Box::default(), + ..self.data + }, + preferences: vec![], + } + } +} + +pub trait ICalendarPrivacy { + fn privacy(&self) -> EventPrivacy; + + fn set_privacy(&mut self, privacy: EventPrivacy); + + fn into_private_view(self) -> ICalendar; +} + +impl ICalendarPrivacy for ICalendar { + fn privacy(&self) -> EventPrivacy { + self.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + .flat_map(|component| component.entries.iter()) + .filter(|entry| entry.name == ICalendarProperty::Class) + .filter_map(|entry| entry.values.first()) + .map(EventPrivacy::from_value) + .max() + .unwrap_or_default() + } + + fn set_privacy(&mut self, privacy: EventPrivacy) { + for component in self + .components + .iter_mut() + .filter(|component| component.component_type.is_scheduling_object()) + { + component + .entries + .retain(|entry| entry.name != ICalendarProperty::Class); + if !privacy.is_public() { + component.entries.push(ICalendarEntry { + name: ICalendarProperty::Class, + params: vec![], + values: [ICalendarValue::Classification(privacy.classification())].into(), + }); + } + } + } + + fn into_private_view(self) -> ICalendar { + let is_visible = self + .components + .iter() + .map(|component| is_private_component(&component.component_type)) + .collect::>(); + let components = self + .components + .into_iter() + .zip(&is_visible) + .map(|(component, is_component_visible)| { + if !is_component_visible { + return ICalendarComponent { + component_type: ICalendarComponentType::Other(Default::default()), + entries: vec![], + component_ids: vec![], + }; + } + let entries = component + .entries + .into_iter() + .filter(|entry| is_private_entry(&component.component_type, &entry.name)) + .map(|mut entry| { + entry.params.retain(|param| { + matches!( + param.name, + ICalendarParameterName::Tzid + | ICalendarParameterName::Value + | ICalendarParameterName::Range + ) + }); + entry + }) + .collect(); + let component_ids = component + .component_ids + .into_iter() + .filter(|id| is_visible.get(*id as usize).copied().unwrap_or_default()) + .collect(); + ICalendarComponent { + component_type: component.component_type, + entries, + component_ids, + } + }) + .collect(); + + let mut view = ICalendar { components }; + view.remove_redundant_overrides(); + view + } +} + +trait RedundantOverrideRemoval { + fn remove_redundant_overrides(&mut self); +} + +impl RedundantOverrideRemoval for ICalendar { + fn remove_redundant_overrides(&mut self) { + let Some(overrides) = RedundantOverrides::new(self) else { + return; + }; + let redundant = self + .components + .iter() + .enumerate() + .filter(|(id, component)| { + component.component_type.is_event_or_todo() + && component.is_recurrence_override() + && overrides.is_redundant(*id as u32) + }) + .map(|(id, _)| id as u32) + .collect::>(); + if !redundant.is_empty() { + for component in &mut self.components { + component.component_ids.retain(|id| !redundant.contains(id)); + } + } + } +} + +fn is_private_component(component_type: &ICalendarComponentType) -> bool { + matches!( + component_type, + ICalendarComponentType::VCalendar + | ICalendarComponentType::VEvent + | ICalendarComponentType::VTodo + | ICalendarComponentType::VTimezone + | ICalendarComponentType::Standard + | ICalendarComponentType::Daylight + ) +} + +fn is_private_entry(component_type: &ICalendarComponentType, property: &ICalendarProperty) -> bool { + match component_type { + ICalendarComponentType::VCalendar => matches!( + property, + ICalendarProperty::Version | ICalendarProperty::Prodid | ICalendarProperty::Calscale + ), + ICalendarComponentType::VEvent | ICalendarComponentType::VTodo => matches!( + property, + ICalendarProperty::Uid + | ICalendarProperty::Dtstamp + | ICalendarProperty::Created + | ICalendarProperty::Sequence + | ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Duration + | ICalendarProperty::Due + | ICalendarProperty::EstimatedDuration + | ICalendarProperty::Transp + | ICalendarProperty::Class + | ICalendarProperty::RecurrenceId + | ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exdate + | ICalendarProperty::ShowWithoutTime + ), + _ => true, + } +} + diff --git a/crates/groupware/src/calendar/rights.rs b/crates/groupware/src/calendar/rights.rs new file mode 100644 index 00000000..b27bc109 --- /dev/null +++ b/crates/groupware/src/calendar/rights.rs @@ -0,0 +1,941 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CalendarEventContent, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, + EVENT_INVITE_SELF, + identity::{CalendarAddresses, EventOwnership}, +}; +use calcard::{ + common::timezone::Tz, + icalendar::{ICalendar, ICalendarPeriod, ICalendarProperty, ICalendarValue}, + jscalendar::{JSCalendar, JSCalendarProperty, JSCalendarValue}, +}; +use common::{GroupwareResources, Server, auth::AccessToken}; +use jmap_tools::{JsonPointerItem, Key, Map, Value}; +use std::{borrow::Cow, str::FromStr}; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes, serialize::rkyv_deserialize}, +}; +use trc::AddContext; +use types::{acl::Acl, blob::BlobId, collection::Collection, field::CalendarEventField, id::Id}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventRightsError { + ModifyEvent, + ModifyPersonalProperties, + Reply, + InviteSelf, + InviteOthers, +} + +impl EventRightsError { + pub fn description(self) -> &'static str { + match self { + EventRightsError::ModifyEvent => "You are not allowed to modify this event.", + EventRightsError::ModifyPersonalProperties => { + "You are not allowed to modify personal properties of this event." + } + EventRightsError::Reply => "You are not allowed to reply to this event.", + EventRightsError::InviteSelf => "You are not allowed to add yourself to this event.", + EventRightsError::InviteOthers => "You are not allowed to invite others to this event.", + } + } +} + +type EventKey<'x> = Key<'x, JSCalendarProperty>; +type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; +type MemberChange<'a, 'x> = ( + &'a EventKey<'x>, + Option<&'a EventValue<'x>>, + Option<&'a EventValue<'x>>, +); + +const SHARED_EVENT_FLAGS: u16 = + EVENT_DRAFT | EVENT_INVITE_SELF | EVENT_INVITE_OTHERS | EVENT_HIDE_ATTENDEES; +const VENDOR_PROPERTY_PREFIX: &str = "x-"; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct EventAcl { + write_all: bool, + write_own: bool, + write_own_items: bool, + update_private: bool, + rsvp: bool, +} + +impl EventAcl { + pub fn for_calendars( + cache: &GroupwareResources, + access_token: &AccessToken, + calendar_ids: impl IntoIterator, + ) -> Self { + let mut acl: Option = None; + for calendar_id in calendar_ids { + let calendar = EventAcl::for_calendar(cache, access_token, calendar_id); + acl = Some(match acl { + Some(acl) => EventAcl { + write_all: acl.write_all && calendar.write_all, + write_own: acl.write_own && calendar.write_own, + write_own_items: acl.write_own_items && calendar.write_own_items, + update_private: acl.update_private && calendar.update_private, + rsvp: acl.rsvp && calendar.rsvp, + }, + None => calendar, + }); + } + acl.unwrap_or_default() + } + + pub fn for_calendar( + cache: &GroupwareResources, + access_token: &AccessToken, + calendar_id: u32, + ) -> Self { + let grants = cache.container_acl(access_token, calendar_id); + let write_all = grants.contains(Acl::ModifyItems); + let write_own_items = grants.contains(Acl::ModifyItemsOwn); + EventAcl { + write_all, + write_own: write_all || write_own_items, + write_own_items, + update_private: write_all || grants.contains(Acl::ModifyPrivateProperties), + rsvp: write_all || grants.contains(Acl::ModifyRSVP), + } + } + + pub fn has_any_write(&self) -> bool { + self.write_all || self.write_own || self.update_private || self.rsvp + } + + pub fn may_manage_own_items(&self) -> bool { + self.write_own_items + } + + pub fn may_write(&self, ownership: EventOwnership) -> bool { + self.write_all || (self.write_own && ownership.may_write_own()) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct EventChanges { + flags: u8, +} + +impl EventChanges { + const PER_USER: u8 = 1; + const RSVP: u8 = 1 << 1; + const INVITE_SELF: u8 = 1 << 2; + const INVITE_OTHERS: u8 = 1 << 3; + const INVITE_OTHERS_UNLISTED: u8 = 1 << 4; + const OTHER: u8 = 1 << 5; + const VENDOR: u8 = 1 << 6; + + pub fn between<'x>( + before: &JSCalendar<'x, Id, BlobId>, + after: &JSCalendar<'x, Id, BlobId>, + before_flags: u16, + after_flags: u16, + identities: &CalendarAddresses, + ) -> Self { + let mut changes = EventChanges::default(); + if (before_flags ^ after_flags) & SHARED_EVENT_FLAGS != 0 { + changes.insert(EventChanges::OTHER); + } + match (before.0.as_object(), after.0.as_object()) { + (Some(before), Some(after)) => changes.classify_group(before, after, identities), + _ if !before.0.same_value(&after.0) => changes.insert(EventChanges::OTHER), + _ => {} + } + changes + } + + pub fn from_per_user_patch(patch: &EventValue<'_>) -> Option { + let patch = patch.as_object()?; + patch + .keys() + .all(EventChanges::is_per_user_key) + .then(|| EventChanges { + flags: if patch.is_empty() { + 0 + } else { + EventChanges::PER_USER + }, + }) + } + + fn is_per_user_key(key: &EventKey<'_>) -> bool { + match key { + Key::Property(JSCalendarProperty::Pointer(pointer)) => match pointer.as_slice() { + [JsonPointerItem::Key(Key::Property(property)), ..] if property.is_per_user() => { + true + } + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::RecurrenceOverrides)), + _, + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Pointer(pointer))), + ] => matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property(property))) if property.is_per_user() + ), + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::RecurrenceOverrides)), + _, + JsonPointerItem::Key(Key::Property(property)), + .., + ] => property.is_per_user(), + _ => false, + }, + Key::Property(property) => property.is_per_user(), + _ => false, + } + } + + pub fn is_per_user_only(&self) -> bool { + self.flags & !EventChanges::PER_USER == 0 + } + + pub fn classify_recurrence_sets(&mut self, before: &ICalendar, after: &ICalendar) { + if !self.contains(EventChanges::OTHER) + && RecurrenceSet::new(before) != RecurrenceSet::new(after) + { + self.insert(EventChanges::OTHER); + } + } + + pub fn insert_per_user(&mut self) { + self.insert(EventChanges::PER_USER); + } + + pub fn is_empty(&self) -> bool { + self.flags == 0 + } + + pub fn assert_allowed( + &self, + acl: &EventAcl, + ownership: EventOwnership, + event_flags: u16, + ) -> Result<(), EventRightsError> { + if acl.may_write(ownership) { + return Ok(()); + } + + let denied = if self.is_empty() { + (!acl.has_any_write()).then_some(EventRightsError::ModifyEvent) + } else if self.flags & !EventChanges::VENDOR == 0 { + (!(acl.update_private || acl.rsvp)).then_some(EventRightsError::ModifyEvent) + } else if self.contains(EventChanges::OTHER) { + Some(EventRightsError::ModifyEvent) + } else if self.contains(EventChanges::PER_USER) && !acl.update_private { + Some(EventRightsError::ModifyPersonalProperties) + } else if self.contains(EventChanges::RSVP) && !acl.rsvp { + Some(EventRightsError::Reply) + } else if self.contains(EventChanges::INVITE_SELF) + && !(acl.rsvp && event_flags & EVENT_INVITE_SELF != 0) + { + Some(EventRightsError::InviteSelf) + } else if self.contains(EventChanges::INVITE_OTHERS_UNLISTED) + || (self.contains(EventChanges::INVITE_OTHERS) + && !(acl.rsvp && event_flags & EVENT_INVITE_OTHERS != 0)) + { + Some(EventRightsError::InviteOthers) + } else { + None + }; + + denied.map_or(Ok(()), Err) + } + + fn insert(&mut self, flag: u8) { + self.flags |= flag; + } + + fn contains(&self, flag: u8) -> bool { + self.flags & flag != 0 + } + + fn classify_group<'x>( + &mut self, + before: &EventMap<'x>, + after: &EventMap<'x>, + identities: &CalendarAddresses, + ) { + for (key, before_value, after_value) in Some(before).changes_to(Some(after)) { + match key { + Key::Property( + JSCalendarProperty::Entries + | JSCalendarProperty::ProdId + | JSCalendarProperty::Created + | JSCalendarProperty::Updated, + ) => {} + Key::Property(JSCalendarProperty::ICalendar) + if before_value.same_ical_value(&after_value) => + { + self.insert(EventChanges::VENDOR) + } + _ => self.insert(EventChanges::OTHER), + } + } + + let (before_entries, after_entries) = (before.entries(), after.entries()); + if before_entries.is_empty() || after_entries.is_empty() { + if before_entries.len() != after_entries.len() { + self.insert(EventChanges::OTHER); + } + return; + } + + let mut matched = vec![false; after_entries.len()]; + for before_entry in before_entries { + match after_entries + .iter() + .zip(matched.iter_mut()) + .find(|(after_entry, is_matched)| { + !**is_matched && before_entry.is_same_instance(after_entry) + }) { + Some((after_entry, is_matched)) => { + *is_matched = true; + match (before_entry.as_object(), after_entry.as_object()) { + (Some(before_event), Some(after_event)) => { + self.classify_event(before_event, after_event, identities) + } + _ if !before_entry.same_value(after_entry) => { + self.insert(EventChanges::OTHER) + } + _ => {} + } + } + None => self.insert(EventChanges::OTHER), + } + } + if matched.contains(&false) { + self.insert(EventChanges::OTHER); + } + } + + fn classify_event<'x>( + &mut self, + before: &EventMap<'x>, + after: &EventMap<'x>, + identities: &CalendarAddresses, + ) { + let mut classifier = ChangeClassifier { + before, + after, + identities, + is_participant: before.participants().is_some_and(|participants| { + participants + .values() + .any(|participant| participant.is_identity(identities)) + }), + changes: EventChanges::default(), + }; + for (key, before_value, after_value) in Some(before).changes_to(Some(after)) { + classifier.classify_property(key, before_value, after_value); + } + self.flags |= classifier.changes.flags; + } +} + +pub trait StoredEventOwnership: Sync + Send { + fn stored_event_ownership( + &self, + account_id: u32, + document_id: u32, + identities: &CalendarAddresses, + ) -> impl Future> + Send; +} + +impl StoredEventOwnership for Server { + async fn stored_event_ownership( + &self, + account_id: u32, + document_id: u32, + identities: &CalendarAddresses, + ) -> trc::Result { + let Some(archive) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(EventOwnership::NotOwner); + }; + + archive + .unarchive::() + .and_then(|content| rkyv_deserialize::<_, ICalendar>(&content.data.event)) + .map(|event| identities.event_ownership(&event)) + .caused_by(trc::location!()) + } +} + +struct ChangeClassifier<'a, 'x> { + before: &'a EventMap<'x>, + after: &'a EventMap<'x>, + identities: &'a CalendarAddresses, + is_participant: bool, + changes: EventChanges, +} + +impl<'a, 'x> ChangeClassifier<'a, 'x> { + fn classify_property( + &mut self, + key: &EventKey<'_>, + before: Option<&EventValue<'x>>, + after: Option<&EventValue<'x>>, + ) { + match key { + Key::Property(property) if property.is_volatile() => {} + Key::Property(JSCalendarProperty::ICalendar) => { + self.classify_ical(before, after); + } + Key::Property(property) if property.is_per_user() => { + self.changes.insert(EventChanges::PER_USER); + } + Key::Property(JSCalendarProperty::Participants) => { + self.classify_participants( + before.and_then(Value::as_object), + after.and_then(Value::as_object), + ); + } + Key::Property(JSCalendarProperty::RecurrenceOverrides) => { + self.classify_overrides( + before.and_then(Value::as_object), + after.and_then(Value::as_object), + ); + } + _ => self.changes.insert(EventChanges::OTHER), + } + } + + fn classify_ical(&mut self, before: Option<&EventValue<'x>>, after: Option<&EventValue<'x>>) { + self.changes.insert(if before.same_ical_value(&after) { + EventChanges::VENDOR + } else { + EventChanges::OTHER + }); + } + + fn classify_participants( + &mut self, + before: Option<&EventMap<'x>>, + after: Option<&EventMap<'x>>, + ) { + for (_, before_participant, after_participant) in before.changes_to(after) { + match (before_participant, after_participant) { + (Some(before_participant), Some(after_participant)) => { + self.classify_participant_change(before_participant, after_participant) + } + (None, Some(after_participant)) => self.classify_new_participant(after_participant), + _ => self.changes.insert(EventChanges::OTHER), + } + } + } + + fn classify_participant_change(&mut self, before: &EventValue<'x>, after: &EventValue<'x>) { + let changes_rsvp_only = match (before.as_object(), after.as_object()) { + (Some(before_map), Some(after_map)) => Some(before_map) + .changes_to(Some(after_map)) + .all(|(key, _, _)| matches!(key, Key::Property(property) if property.is_rsvp())), + _ => false, + }; + + if changes_rsvp_only && before.is_identity(self.identities) { + self.changes.insert(EventChanges::RSVP); + } else { + self.changes.insert(EventChanges::OTHER); + } + } + + fn classify_new_participant(&mut self, participant: &EventValue<'x>) { + let has_no_roles = + match participant.as_object_and_get(&Key::Property(JSCalendarProperty::Roles)) { + None => true, + Some(Value::Object(roles)) => roles.is_empty(), + Some(_) => false, + }; + + self.changes.insert(if !has_no_roles { + EventChanges::OTHER + } else if participant.is_identity(self.identities) { + EventChanges::INVITE_SELF + } else if self.is_participant { + EventChanges::INVITE_OTHERS + } else { + EventChanges::INVITE_OTHERS_UNLISTED + }); + } + + fn classify_overrides(&mut self, before: Option<&EventMap<'x>>, after: Option<&EventMap<'x>>) { + let empty = EventMap::new(); + + for (_, before_value, after_value) in before.changes_to(after) { + let (Some(before_patch), Some(after_patch)) = ( + before_value.map_or(Some(&empty), Value::as_object), + after_value.map_or(Some(&empty), Value::as_object), + ) else { + self.changes.insert(EventChanges::OTHER); + continue; + }; + if (before_value.is_none() && after_patch.is_empty()) + || (after_value.is_none() && before_patch.is_empty()) + { + self.changes.insert(EventChanges::OTHER); + continue; + } + + for (entry_key, before_entry, after_entry) in + Some(before_patch).changes_to(Some(after_patch)) + { + self.classify_override_entry(entry_key, before_entry, after_entry); + } + } + } + + fn classify_override_entry( + &mut self, + key: &EventKey<'_>, + before: Option<&EventValue<'x>>, + after: Option<&EventValue<'x>>, + ) { + match key { + Key::Property(JSCalendarProperty::Pointer(pointer)) => { + self.classify_override_pointer(pointer.as_slice(), before, after) + } + Key::Property(JSCalendarProperty::Participants) => { + let base = self.after.participants(); + self.classify_participants( + before.and_then(Value::as_object).or(base), + after.and_then(Value::as_object).or(base), + ); + } + Key::Property(JSCalendarProperty::ICalendar) => { + self.classify_ical(before, after); + } + Key::Property(property) if property.is_volatile() => {} + Key::Property(property) if property.is_per_user() => { + self.changes.insert(EventChanges::PER_USER); + } + _ => self.changes.insert(EventChanges::OTHER), + } + } + + fn classify_override_pointer( + &mut self, + pointer: &[JsonPointerItem>], + before: Option<&EventValue<'x>>, + after: Option<&EventValue<'x>>, + ) { + match pointer { + [JsonPointerItem::Key(Key::Property(property)), ..] if property.is_volatile() => {} + [JsonPointerItem::Key(Key::Property(property)), ..] if property.is_per_user() => { + self.changes.insert(EventChanges::PER_USER); + } + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + id, + ] => { + let Some(id) = id.as_member_key() else { + self.changes.insert(EventChanges::OTHER); + return; + }; + let base = self.base_participant(&id); + match (before.filter(|value| value.as_object().is_some()), after) { + (before, Some(after)) if after.as_object().is_some() => match before.or(base) { + Some(before) => self.classify_participant_change(before, after), + None => self.classify_new_participant(after), + }, + (Some(before), None) => match base { + Some(base) => self.classify_participant_change(before, base), + None => self.changes.insert(EventChanges::OTHER), + }, + _ => self.changes.insert(EventChanges::OTHER), + } + } + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + id, + JsonPointerItem::Key(Key::Property(property)), + ] if property.is_rsvp() + && id + .as_member_key() + .and_then(|id| self.base_participant(&id)) + .is_some_and(|participant| participant.is_identity(self.identities)) => + { + self.changes.insert(EventChanges::RSVP); + } + _ => self.changes.insert(EventChanges::OTHER), + } + } + + fn base_participant(&self, id: &EventKey<'_>) -> Option<&'a EventValue<'x>> { + self.after + .participants() + .and_then(|participants| participants.get(id)) + .or_else(|| { + self.before + .participants() + .and_then(|participants| participants.get(id)) + }) + } +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct RecurrenceSet { + dates: Vec, + exclusions: Vec, +} + +impl RecurrenceSet { + fn new(ical: &ICalendar) -> Self { + let mut set = RecurrenceSet::default(); + let mut resolver = None; + for entry in ical + .components + .iter() + .filter(|component| { + component.component_type.is_scheduling_object() + && !component.is_recurrence_override() + }) + .flat_map(|component| component.entries.iter()) + { + let dates = match entry.name { + ICalendarProperty::Rdate => &mut set.dates, + ICalendarProperty::Exdate => &mut set.exclusions, + _ => continue, + }; + let tz = resolver + .get_or_insert_with(|| ical.build_tz_resolver()) + .resolve_or_default(entry.tz_id()); + dates.extend( + entry + .values + .iter() + .filter_map(|value| match value { + ICalendarValue::PartialDateTime(date_time) => Some(date_time), + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, .. } + | ICalendarPeriod::Duration { start, .. } => Some(start), + }, + _ => None, + }) + .filter_map(|date_time| date_time.to_date_time_with_tz(tz)) + .map(|date_time| date_time.timestamp()), + ); + } + for dates in [&mut set.dates, &mut set.exclusions] { + dates.sort_unstable(); + dates.dedup(); + } + set + } +} + +trait EventMembers<'a, 'x> { + fn changes_to( + self, + after: Option<&'a EventMap<'x>>, + ) -> impl Iterator>; +} + +impl<'a, 'x> EventMembers<'a, 'x> for Option<&'a EventMap<'x>> { + fn changes_to( + self, + after: Option<&'a EventMap<'x>>, + ) -> impl Iterator> { + self.into_iter() + .flat_map(|before| before.iter()) + .map(move |(key, value)| (key, Some(value), after.and_then(|after| after.get(key)))) + .chain( + after + .into_iter() + .flat_map(|after| after.iter()) + .filter(move |(key, _)| self.is_none_or(|before| !before.contains_key(key))) + .map(|(key, value)| (key, None, Some(value))), + ) + .filter(|(_, before, after)| !before.same_value(after)) + } +} + +trait EventObject<'x> { + fn entries(&self) -> &[EventValue<'x>]; + + fn participants(&self) -> Option<&EventMap<'x>>; +} + +impl<'x> EventObject<'x> for EventMap<'x> { + fn entries(&self) -> &[EventValue<'x>] { + self.get(&Key::Property(JSCalendarProperty::Entries)) + .and_then(Value::as_array) + .unwrap_or_default() + } + + fn participants(&self) -> Option<&EventMap<'x>> { + self.get(&Key::Property(JSCalendarProperty::Participants)) + .and_then(Value::as_object) + } +} + +pub(crate) trait EventValueCompare { + fn same_value(&self, other: &Self) -> bool; +} + +impl EventValueCompare for EventValue<'_> { + fn same_value(&self, other: &Self) -> bool { + match (self, other) { + (Value::Object(before), Value::Object(after)) => { + before.len() == after.len() + && before.iter().all(|(key, before)| { + after.get(key).is_some_and(|after| before.same_value(after)) + }) + } + (Value::Array(before), Value::Array(after)) => { + before.len() == after.len() + && before + .iter() + .zip(after) + .all(|(before, after)| before.same_value(after)) + } + _ => self == other, + } + } +} + +impl EventValueCompare for Option<&EventValue<'_>> { + fn same_value(&self, other: &Self) -> bool { + match (self, other) { + (Some(before), Some(after)) => before.same_value(after), + (before, after) => before.is_none() && after.is_none(), + } + } +} + +trait ICalendarCompare { + fn same_ical_value(&self, other: &Self) -> bool; +} + +impl ICalendarCompare for Option<&EventValue<'_>> { + fn same_ical_value(&self, other: &Self) -> bool { + match (self, other) { + (None | Some(Value::Null), None | Some(Value::Null)) => true, + (Some(Value::Object(before)), None | Some(Value::Null)) => { + before.same_ical_value(&EventMap::new()) + } + (None | Some(Value::Null), Some(Value::Object(after))) => { + EventMap::new().same_ical_value(after) + } + (Some(Value::Object(before)), Some(Value::Object(after))) => { + before.same_ical_value(after) + } + _ => self.same_value(other), + } + } +} + +impl ICalendarCompare for EventMap<'_> { + fn same_ical_value(&self, other: &Self) -> bool { + Some(self) + .changes_to(Some(other)) + .all(|(key, before, after)| match key { + Key::Property(JSCalendarProperty::Name) => before.is_none() || after.is_none(), + Key::Property(JSCalendarProperty::Properties) => before + .and_then(Value::as_array) + .unwrap_or_default() + .same_properties(after.and_then(Value::as_array).unwrap_or_default()), + Key::Property(JSCalendarProperty::Components) => before + .and_then(Value::as_array) + .unwrap_or_default() + .same_components(after.and_then(Value::as_array).unwrap_or_default()), + _ => false, + }) + } +} + +trait ICalendarList { + fn same_properties(&self, other: &Self) -> bool; + + fn same_components(&self, other: &Self) -> bool; +} + +impl ICalendarList for [EventValue<'_>] { + fn same_properties(&self, other: &Self) -> bool { + let mut before = self + .iter() + .filter(|property| !property.is_volatile_ical_property()); + let mut after = other + .iter() + .filter(|property| !property.is_volatile_ical_property()); + loop { + match (before.next(), after.next()) { + (Some(before), Some(after)) if before.same_value(after) => {} + (None, None) => return true, + _ => return false, + } + } + } + + fn same_components(&self, other: &Self) -> bool { + let mut before = self + .iter() + .filter(|component| !component.is_iana_timezone()); + let mut after = other + .iter() + .filter(|component| !component.is_iana_timezone()); + loop { + match (before.next(), after.next()) { + (Some(before), Some(after)) if before.same_ical_component(after) => {} + (None, None) => return true, + _ => return false, + } + } + } +} + +trait ICalendarComponentValue { + fn same_ical_component(&self, other: &Self) -> bool; + + fn is_iana_timezone(&self) -> bool; +} + +impl ICalendarComponentValue for EventValue<'_> { + fn same_ical_component(&self, other: &Self) -> bool { + match (self.as_array(), other.as_array()) { + ( + Some([before_name, before_properties, before_components]), + Some([after_name, after_properties, after_components]), + ) => { + before_name.same_value(after_name) + && before_properties + .as_array() + .unwrap_or_default() + .same_properties(after_properties.as_array().unwrap_or_default()) + && before_components + .as_array() + .unwrap_or_default() + .same_components(after_components.as_array().unwrap_or_default()) + } + _ => self.same_value(other), + } + } + + fn is_iana_timezone(&self) -> bool { + matches!( + self.as_array(), + Some([Value::Str(name), Value::Array(properties), ..]) + if name.eq_ignore_ascii_case("vtimezone") + && properties.iter().any(|property| matches!( + property.as_array(), + Some([Value::Str(property), _, _, Value::Str(tz_id), ..]) + if property.eq_ignore_ascii_case("tzid") + && Tz::from_str(tz_id).is_ok() + )) + ) + } +} + +trait EventValueProperties { + fn is_identity(&self, identities: &CalendarAddresses) -> bool; + + fn is_same_instance(&self, other: &Self) -> bool; + + fn is_volatile_ical_property(&self) -> bool; +} + +impl EventValueProperties for EventValue<'_> { + fn is_identity(&self, identities: &CalendarAddresses) -> bool { + self.as_object_and_get(&Key::Property(JSCalendarProperty::CalendarAddress)) + .and_then(Value::as_str) + .is_some_and(|address| identities.contains(&address)) + } + + fn is_same_instance(&self, other: &Self) -> bool { + [JSCalendarProperty::Uid, JSCalendarProperty::RecurrenceId] + .into_iter() + .map(Key::Property) + .all(|key| { + self.as_object_and_get(&key) + .same_value(&other.as_object_and_get(&key)) + }) + } + + fn is_volatile_ical_property(&self) -> bool { + self.as_array() + .and_then(|items| items.first()) + .and_then(Value::as_str) + .is_some_and(|name| { + name.get(..2) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(VENDOR_PROPERTY_PREFIX)) + || hashify::set_ignore_case!( + name.as_bytes(), + "dtstamp", + "last-modified", + "created" + ) + }) + } +} + +trait MemberKey { + fn as_member_key(&self) -> Option>>; +} + +impl MemberKey for JsonPointerItem> { + fn as_member_key(&self) -> Option>> { + match self { + JsonPointerItem::Key(key) => Some(Cow::Borrowed(key)), + JsonPointerItem::Number(number) => Some(Cow::Owned(Key::Owned(number.to_string()))), + JsonPointerItem::Root | JsonPointerItem::Wildcard | JsonPointerItem::Invalid(_) => None, + } + } +} + +trait PropertyClass { + fn is_per_user(&self) -> bool; + + fn is_rsvp(&self) -> bool; + + fn is_volatile(&self) -> bool; +} + +impl PropertyClass for JSCalendarProperty { + fn is_per_user(&self) -> bool { + matches!( + self, + JSCalendarProperty::Keywords + | JSCalendarProperty::Color + | JSCalendarProperty::FreeBusyStatus + | JSCalendarProperty::UseDefaultAlerts + | JSCalendarProperty::Alerts + ) + } + + fn is_rsvp(&self) -> bool { + matches!( + self, + JSCalendarProperty::ParticipationStatus + | JSCalendarProperty::ExpectReply + | JSCalendarProperty::ScheduleSequence + | JSCalendarProperty::ScheduleUpdated + ) + } + + fn is_volatile(&self) -> bool { + matches!( + self, + JSCalendarProperty::Updated | JSCalendarProperty::Created + ) + } +} + diff --git a/crates/groupware/src/calendar/schedule.rs b/crates/groupware/src/calendar/schedule.rs new file mode 100644 index 00000000..e80a5e47 --- /dev/null +++ b/crates/groupware/src/calendar/schedule.rs @@ -0,0 +1,721 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedCalendarEvent, ArchivedCalendarEventContent, ArchivedEventPreferences, + CalendarEventContent, EVENT_DRAFT, EVENT_HAS_ALARMS, EventPreferences, PREF_HAS_ALERTS, + alarm::{AlarmSource, AlarmTarget, CalendarAlarm, EventAlarmData, PersonalAlarms}, + alerts::{DefaultAlerts, DefaultAlertsResolver, ICalendarShowWithoutTime}, +}; +use calcard::common::timezone::Tz; +use common::{ArchivedDavName, Server, storage::chunk::ArchiveChunk}; +use registry::schema::{enums::TaskType, structs::Task}; +use std::cmp::Ordering; +use store::{ + Deserialize, IterateParams, U64_LEN, ValueKey, + roaring::RoaringBitmap, + write::{ + Archive, ArchiveBytes, BatchBuilder, TaskId, TaskQueueClass, ValueClass, + key::DeserializeBigEndian, now, serialize::rkyv_deserialize, + }, +}; +use trc::AddContext; +use types::{collection::Collection, field::CalendarEventField}; + +#[derive(Debug, Default)] +pub struct EventAlarmUsers { + with_time: bool, + is_draft: bool, + owner: Option, + sharees: Vec, +} + +#[derive(Debug)] +struct EventAlarmUser { + target: AlarmTarget, + uses_defaults: bool, + alarms: PersonalAlarms, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct EventAlarms(Vec); + +pub trait EventAlarmPreferences { + fn account_id(&self) -> u32; + + fn uses_default_alerts(&self) -> bool; + + fn has_stored_alerts(&self) -> bool; + + fn personal_alarms(&self) -> trc::Result; +} + +pub trait EventAlarmContent { + type Preferences: EventAlarmPreferences; + + fn shows_without_time(&self) -> bool; + + fn user_preferences(&self, account_id: u32) -> Option<&Self::Preferences>; + + fn all_preferences(&self) -> impl Iterator; + + fn uses_default_alerts(&self, account_id: u32) -> bool { + self.user_preferences(account_id) + .is_some_and(EventAlarmPreferences::uses_default_alerts) + } +} + +impl EventAlarmPreferences for EventPreferences { + fn account_id(&self) -> u32 { + self.account_id + } + + fn uses_default_alerts(&self) -> bool { + EventPreferences::use_default_alerts(self) + } + + fn has_stored_alerts(&self) -> bool { + self.instances + .iter() + .any(|instance| instance.flags & PREF_HAS_ALERTS != 0) + } + + fn personal_alarms(&self) -> trc::Result { + Ok(PersonalAlarms::new(self)) + } +} + +impl EventAlarmPreferences for ArchivedEventPreferences { + fn account_id(&self) -> u32 { + self.account_id.to_native() + } + + fn uses_default_alerts(&self) -> bool { + ArchivedEventPreferences::use_default_alerts(self) + } + + fn has_stored_alerts(&self) -> bool { + self.instances + .iter() + .any(|instance| instance.flags & PREF_HAS_ALERTS != 0) + } + + fn personal_alarms(&self) -> trc::Result { + rkyv_deserialize::<_, EventPreferences>(self) + .caused_by(trc::location!()) + .map(|preferences| PersonalAlarms::new(&preferences)) + } +} + +impl EventAlarmContent for CalendarEventContent { + type Preferences = EventPreferences; + + fn shows_without_time(&self) -> bool { + self.data.event.shows_without_time() + } + + fn user_preferences(&self, account_id: u32) -> Option<&Self::Preferences> { + self.preferences(account_id) + } + + fn all_preferences(&self) -> impl Iterator { + self.preferences.iter() + } +} + +impl EventAlarmContent for ArchivedCalendarEventContent { + type Preferences = ArchivedEventPreferences; + + fn shows_without_time(&self) -> bool { + self.data.event.shows_without_time() + } + + fn user_preferences(&self, account_id: u32) -> Option<&Self::Preferences> { + self.preferences(account_id) + } + + fn all_preferences(&self) -> impl Iterator { + self.preferences.iter() + } +} + +impl EventAlarmUser { + fn owner(account_id: u32, content: &C) -> Self { + EventAlarmUser { + target: AlarmTarget::Owner, + uses_defaults: content.uses_default_alerts(account_id), + alarms: PersonalAlarms::default(), + } + } + + fn has_alarms(&self) -> bool { + match self.target { + AlarmTarget::Owner => true, + AlarmTarget::Sharee(_) => self.uses_defaults || !self.alarms.is_empty(), + } + } + + fn source<'x>(&'x self, defaults: &'x DefaultAlerts) -> AlarmSource<'x> { + match self.target { + AlarmTarget::Owner => AlarmSource::Stored(defaults), + AlarmTarget::Sharee(_) => AlarmSource::Personal(&self.alarms, defaults), + } + } +} + +impl EventAlarmUsers { + pub fn new(account_id: u32, content: &C) -> trc::Result { + let mut sharees = Vec::new(); + for preferences in content + .all_preferences() + .filter(|preferences| sharee_has_alerts(account_id, *preferences)) + { + let sharee = EventAlarmUser { + target: AlarmTarget::Sharee(preferences.account_id()), + uses_defaults: preferences.uses_default_alerts(), + alarms: preferences.personal_alarms()?, + }; + if sharee.has_alarms() { + sharees.push(sharee); + } + } + sharees.sort_unstable_by_key(|sharee| sharee.target); + + Ok(EventAlarmUsers { + with_time: !content.shows_without_time(), + is_draft: false, + owner: Some(EventAlarmUser::owner(account_id, content)), + sharees, + }) + } + + pub fn for_target( + account_id: u32, + content: &C, + target: AlarmTarget, + ) -> trc::Result { + let mut users = EventAlarmUsers { + with_time: !content.shows_without_time(), + is_draft: false, + owner: None, + sharees: Vec::new(), + }; + match target { + AlarmTarget::Owner => { + users.owner = Some(EventAlarmUser::owner(account_id, content)); + } + AlarmTarget::Sharee(sharee_id) => { + if let Some(preferences) = content + .user_preferences(sharee_id) + .filter(|preferences| sharee_has_alerts(account_id, *preferences)) + { + let sharee = EventAlarmUser { + target, + uses_defaults: preferences.uses_default_alerts(), + alarms: preferences.personal_alarms()?, + }; + if sharee.has_alarms() { + users.sharees.push(sharee); + } + } + } + } + + Ok(users) + } + + fn users(&self) -> impl Iterator { + self.owner.iter().chain(self.sharees.iter()) + } + + fn is_empty(&self) -> bool { + self.owner.is_none() && self.sharees.is_empty() + } + + pub fn targets( + account_id: u32, + content: &C, + ) -> impl Iterator { + std::iter::once(AlarmTarget::Owner).chain( + content + .all_preferences() + .filter(move |preferences| sharee_has_alerts(account_id, *preferences)) + .map(|preferences| AlarmTarget::Sharee(preferences.account_id())), + ) + } + + pub fn task_ids( + account_id: u32, + document_id: u32, + content: &C, + ) -> impl Iterator { + Self::targets(account_id, content) + .map(move |target| target.task_id().resolve(account_id, document_id)) + } + + pub fn with_event_flags(mut self, flags: u16) -> Self { + self.is_draft = flags & EVENT_DRAFT != 0; + self + } +} + +fn sharee_has_alerts(account_id: u32, preferences: &P) -> bool { + preferences.account_id() != account_id + && (preferences.uses_default_alerts() || preferences.has_stored_alerts()) +} + +impl EventAlarms { + pub fn into_alarm(mut self) -> Option { + self.0.pop() + } +} + +impl IntoIterator for EventAlarms { + type Item = CalendarAlarm; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +pub trait EventAlarmScheduler: Sync + Send { + fn next_event_alarms( + &self, + account_id: u32, + users: &EventAlarmUsers, + data: &D, + calendar_ids: &[u32], + start_time: i64, + resolver: &mut DefaultAlertsResolver, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn reschedule_event_alarms( + &self, + account_id: u32, + document_id: u32, + event: &ArchivedCalendarEvent, + content: Option>, + calendar_ids: &[u32], + tasks: &mut AlarmTasks, + resolver: &mut DefaultAlertsResolver, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn replace_event_alarms( + &self, + account_id: u32, + document_id: u32, + previous: EventAlarms, + next: EventAlarms, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn replace_user_alarm( + &self, + account_id: u32, + document_id: u32, + previous: Option, + next: Option, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + fn clear_event_alarms( + &self, + account_id: u32, + document_id: u32, + targets: T, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> impl Future> + Send + where + T: IntoIterator + Send, + T::IntoIter: Send; + + fn queued_alarm_due( + &self, + account_id: u32, + document_id: u32, + target: AlarmTarget, + tasks: &mut AlarmTasks, + ) -> impl Future>> + Send; +} + +#[derive(Debug, Default)] +pub enum AlarmTasks { + #[default] + None, + Pending(Vec), + Loaded(Vec<(u64, Option<(TaskType, u64)>)>), +} + +impl AlarmTasks { + pub fn pending(mut task_ids: Vec) -> Self { + task_ids.sort_unstable(); + task_ids.dedup(); + AlarmTasks::Pending(task_ids) + } + + pub fn for_chunk( + account_id: u32, + chunk: &[u32], + alarm_ids: &RoaringBitmap, + archives: &ArchiveChunk, + ) -> trc::Result { + let mut task_ids = Vec::new(); + for &document_id in chunk + .iter() + .filter(|document_id| alarm_ids.contains(**document_id)) + { + if let Some(content) = archives.value(document_id) { + let content = content + .unarchive::() + .caused_by(trc::location!())?; + task_ids.extend(EventAlarmUsers::task_ids(account_id, document_id, content)); + } + } + Ok(AlarmTasks::pending(task_ids)) + } + + fn due(&self, task_id: u64) -> Option> { + match self { + AlarmTasks::Loaded(tasks) => tasks + .binary_search_by_key(&task_id, |(id, _)| *id) + .ok() + .and_then(|idx| tasks.get(idx)) + .map(|(_, due)| *due), + AlarmTasks::None | AlarmTasks::Pending(_) => None, + } + } + + fn is_pending(&self, task_id: u64) -> bool { + matches!(self, AlarmTasks::Pending(task_ids) if task_ids.binary_search(&task_id).is_ok()) + } +} + +impl EventAlarmScheduler for Server { + async fn next_event_alarms( + &self, + account_id: u32, + users: &EventAlarmUsers, + data: &D, + calendar_ids: &[u32], + start_time: i64, + resolver: &mut DefaultAlertsResolver, + ) -> trc::Result { + if !self.core.groupware.alarms_enabled || users.is_draft || users.is_empty() { + return Ok(EventAlarms::default()); + } + + let mut alarms = Vec::new(); + let has_stored_alarms = data.has_stored_alarms(); + for user in users.users() { + let target_id = user.target.account_id(account_id); + let defaults = if user.uses_defaults { + if !resolver + .is_subscribed(self, account_id, target_id, calendar_ids.iter().copied()) + .await? + { + continue; + } + resolver + .resolve( + self, + account_id, + target_id, + true, + calendar_ids.iter().copied(), + users.with_time, + ) + .await? + } else { + let defaults = DefaultAlerts::disabled(); + if !user.source(&defaults).may_have_alarms(has_stored_alarms) + || !resolver + .is_subscribed(self, account_id, target_id, calendar_ids.iter().copied()) + .await? + { + continue; + } + defaults + }; + let source = user.source(&defaults); + let default_tz = if data.needs_default_tz(&source) { + resolver + .time_zone(self, account_id, target_id, calendar_ids.iter().copied()) + .await? + } else { + Tz::Floating + }; + if let Some(alarm) = data.next_alarm_from(start_time, default_tz, &source) { + alarms.push(alarm.with_target(user.target)); + } + } + + Ok(EventAlarms(alarms)) + } + + async fn replace_event_alarms( + &self, + account_id: u32, + document_id: u32, + previous: EventAlarms, + next: EventAlarms, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if previous == next { + return Ok(()); + } + + let mut previous = previous.0.into_iter().peekable(); + let mut next = next.0.into_iter().peekable(); + loop { + let (previous_alarm, next_alarm) = match (previous.peek(), next.peek()) { + (Some(previous_alarm), Some(next_alarm)) => { + match previous_alarm.target.cmp(&next_alarm.target) { + Ordering::Equal => (previous.next(), next.next()), + Ordering::Less => (previous.next(), None), + Ordering::Greater => (None, next.next()), + } + } + (Some(_), None) => (previous.next(), None), + (None, Some(_)) => (None, next.next()), + (None, None) => break, + }; + self.replace_user_alarm( + account_id, + document_id, + previous_alarm, + next_alarm, + tasks, + batch, + ) + .await?; + } + + Ok(()) + } + + async fn replace_user_alarm( + &self, + account_id: u32, + document_id: u32, + previous: Option, + next: Option, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if previous == next { + return Ok(()); + } + let Some(target) = next + .as_ref() + .or(previous.as_ref()) + .map(|alarm| alarm.target) + else { + return Ok(()); + }; + + let queued_due = self + .queued_alarm_due(account_id, document_id, target, tasks) + .await? + .or_else(|| { + previous.map(|previous| (previous.task_type(), previous.alarm_time as u64)) + }); + batch + .with_account_id(account_id) + .with_collection(Collection::CalendarEvent) + .with_document(document_id); + if let Some((typ, due)) = queued_due { + batch.clear_document_task(target.task_id(), typ, due); + } + if let Some(next) = next { + next.write_task(batch)?; + } + + Ok(()) + } + + async fn clear_event_alarms( + &self, + account_id: u32, + document_id: u32, + targets: T, + tasks: &mut AlarmTasks, + batch: &mut BatchBuilder, + ) -> trc::Result<()> + where + T: IntoIterator + Send, + T::IntoIter: Send, + { + for target in targets { + if let Some((typ, due)) = self + .queued_alarm_due(account_id, document_id, target, tasks) + .await? + { + batch + .with_account_id(account_id) + .with_collection(Collection::CalendarEvent) + .with_document(document_id) + .clear_document_task(target.task_id(), typ, due); + } + } + + Ok(()) + } + + async fn queued_alarm_due( + &self, + account_id: u32, + document_id: u32, + target: AlarmTarget, + tasks: &mut AlarmTasks, + ) -> trc::Result> { + let task_id = target.task_id().resolve(account_id, document_id); + if tasks.is_pending(task_id) + && let AlarmTasks::Pending(task_ids) = std::mem::take(tasks) + { + *tasks = AlarmTasks::Loaded(load_alarm_tasks(self, task_ids).await?); + } + if let Some(due) = tasks.due(task_id) { + return Ok(due); + } + queued_task_due(self, task_id).await + } + + async fn reschedule_event_alarms( + &self, + account_id: u32, + document_id: u32, + event: &ArchivedCalendarEvent, + content: Option>, + calendar_ids: &[u32], + tasks: &mut AlarmTasks, + resolver: &mut DefaultAlertsResolver, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if event.flags.to_native() & EVENT_HAS_ALARMS == 0 { + return Ok(()); + } + let content = match content { + Some(content) => content, + None => { + let Some(content) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + content + } + }; + let content = content + .unarchive::() + .caused_by(trc::location!())?; + let users = + EventAlarmUsers::new(account_id, content)?.with_event_flags(event.flags.to_native()); + let now = now() as i64; + let previous_alarms = self + .next_event_alarms( + account_id, + &users, + &content.data, + &event + .names + .iter() + .map(ArchivedDavName::parent_id) + .collect::>(), + now, + resolver, + ) + .await?; + let next_alarms = self + .next_event_alarms( + account_id, + &users, + &content.data, + calendar_ids, + now, + resolver, + ) + .await?; + + self.replace_event_alarms( + account_id, + document_id, + previous_alarms, + next_alarms, + tasks, + batch, + ) + .await + } +} + +async fn queued_task_due(server: &Server, task_id: u64) -> trc::Result> { + server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { + id: TaskId::Assigned(task_id), + }, + ))) + .await + .map(|task| task.map(|task| (task.object_type(), task.due_timestamp()))) + .caused_by(trc::location!()) +} + +async fn load_alarm_tasks( + server: &Server, + task_ids: Vec, +) -> trc::Result)>> { + if let [task_id] = task_ids.as_slice() { + let due = queued_task_due(server, *task_id).await?; + return Ok(vec![(*task_id, due)]); + } + let mut tasks = task_ids + .into_iter() + .map(|task_id| (task_id, None)) + .collect::)>>(); + let ranges = tasks + .iter() + .map(|(task_id, _)| { + let key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Task { + id: TaskId::Assigned(*task_id), + })); + IterateParams::new(key.clone(), key) + }) + .collect(); + server + .store() + .iterate_many(ranges, |key, value| { + let task_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + if let Ok(idx) = tasks.binary_search_by_key(&task_id, |(id, _)| *id) + && let Some((_, due)) = tasks.get_mut(idx) + { + let task = ::deserialize(value)?; + *due = Some((task.object_type(), task.due_timestamp())); + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + Ok(tasks) +} + diff --git a/crates/groupware/src/calendar/sequence.rs b/crates/groupware/src/calendar/sequence.rs new file mode 100644 index 00000000..1795e292 --- /dev/null +++ b/crates/groupware/src/calendar/sequence.rs @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, compare::ComparisonScope, + expand::ComponentRecurrenceId, +}; +use calcard::{ + common::timezone::Tz, + icalendar::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarProperty, ICalendarValue, + }, +}; + +pub const SCHEDULING_EVENT_FLAGS: u16 = + EVENT_INVITE_SELF | EVENT_INVITE_OTHERS | EVENT_HIDE_ATTENDEES; + +pub trait ICalendarSequence { + fn has_scheduling_changes(&self, previous: &ICalendar) -> bool; + + fn increment_sequence(&mut self, previous: &ICalendar); +} + +impl ICalendarSequence for ICalendar { + fn has_scheduling_changes(&self, previous: &ICalendar) -> bool { + !ComparisonScope::SchedulingData.component_eq(self, 0, previous, 0) + || PreviousSequences::new(previous).are_lowered_by(self) + } + + fn increment_sequence(&mut self, previous: &ICalendar) { + let previous = PreviousSequences::new(previous); + for component in self + .components + .iter_mut() + .filter(|component| component.component_type.is_scheduling_object()) + { + let previous_sequence = previous.of(component); + if component.sequence().unwrap_or_default() <= previous_sequence { + component.set_sequence(previous_sequence + 1); + } + } + } +} + +struct PreviousSequences<'x> { + series: Option, + components: Vec<(&'x ICalendarComponentType, Option, Option)>, +} + +impl<'x> PreviousSequences<'x> { + fn new(ical: &'x ICalendar) -> Self { + let components = ical + .components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + .map(|component| { + ( + &component.component_type, + component.recurrence_instant(), + component.sequence(), + ) + }) + .collect::>(); + PreviousSequences { + series: ical + .components + .iter() + .find(|component| { + component.component_type.is_scheduling_object() + && !component.is_recurrence_override() + }) + .and_then(ComponentSequence::sequence), + components, + } + } + + fn are_lowered_by(&self, ical: &ICalendar) -> bool { + ical.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + .any(|component| component.sequence().unwrap_or_default() < self.of(component)) + } + + fn of(&self, component: &ICalendarComponent) -> i64 { + let recurrence_instant = component.recurrence_instant(); + self.components + .iter() + .find(|(component_type, instant, _)| { + **component_type == component.component_type && *instant == recurrence_instant + }) + .map_or(self.series, |(_, _, sequence)| *sequence) + .unwrap_or_default() + } +} + +trait ComponentSequence { + fn sequence(&self) -> Option; + + fn set_sequence(&mut self, sequence: i64); + + fn recurrence_instant(&self) -> Option; +} + +impl ComponentSequence for ICalendarComponent { + fn sequence(&self) -> Option { + self.property(&ICalendarProperty::Sequence) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_integer) + } + + fn recurrence_instant(&self) -> Option { + self.recurrence_id(Tz::Floating).map(|id| id.utc) + } + + fn set_sequence(&mut self, sequence: i64) { + match self + .entries + .iter_mut() + .find(|entry| entry.name == ICalendarProperty::Sequence) + { + Some(entry) => entry.values = [ICalendarValue::Integer(sequence)].into(), + None => self.add_sequence(sequence), + } + } +} + diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index e566bc2b..1f983b12 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -9,40 +9,57 @@ use super::{ alarm::CalendarAlarm, }; use crate::{ - DavResourceName, DestroyArchive, RFC_3986, + DavResourceName, DestroyArchive, RFC_3986, SizeWriter, + cache::GroupwareCache, calendar::{ - ArchivedCalendarEventNotification, CalendarEventNotification, alarm::CalendarAlarmType, + ArchivedCalendarEventContent, ArchivedCalendarEventNotification, CalendarEventContent, + CalendarEventNotification, CalendarEventNotificationContent, ChangedBy as EventChangedBy, + EVENT_DRAFT, EVENT_HAS_ALARMS, EVENT_NOTIFICATION_IS_CHANGE, EVENT_NOTIFICATION_IS_DESTROY, + EVENT_NOTIFICATION_IS_DIRECT, EVENT_NOTIFICATION_IS_DRAFT, + alarm::{AlarmTarget, CalendarAlarmType}, + alerts::DefaultAlertsResolver, + notification::{ + AccountViewers, CalendarNotificationReap, CalendarNotificationViewers, + NotificationAudience, has_viewers, may_have_viewers, + }, + privacy::ICalendarPrivacy, + schedule::{AlarmTasks, EventAlarmScheduler, EventAlarmUsers, EventAlarms}, }, - scheduling::{ItipMessages, event_cancel::itip_cancel}, + scheduling::{ItipMessages, event_cancel::itip_cancel, recipient::RecipientPolicy}, }; -use calcard::common::timezone::Tz; +use calcard::icalendar::ICalendar; use common::{ - Server, - auth::{AccountInfo, AccountTenantIds}, - storage::index::ObjectIndexBuilder, + DavName, GroupwareResources, Server, + auth::{AccessToken, AccountCache, AccountInfo, ChangedBy}, + storage::{ + batch::MAX_BATCH_DOCUMENTS, + index::{GroupwareWrite, ObjectIndexBuilder, SplitCurrent, SplitUpdate}, + metadata::ContainerCleanup, + }, }; use registry::{ + schema::enums::{StorageQuota, TaskType}, schema::structs::{Task, TaskCalendarAlarmEmail, TaskCalendarAlarmNotification, TaskStatus}, - types::{EnumImpl, ObjectImpl, datetime::UTCDateTime}, + types::{ObjectImpl, datetime::UTCDateTime}, }; +use std::sync::Arc; use store::{ - IterateParams, SerializeInfallible, U32_LEN, ValueKey, + IndexKey, IterateParams, U32_LEN, ValueKey, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, Operation, TaskQueueClass, - ValueClass, ValueOp, key::DeserializeBigEndian, now, + Archive, ArchiveBytes, BatchBuilder, Operation, PendingId, SetValue, Slot, TaskId, + TaskQueueClass, ValueClass, ValueOp, key::DeserializeBigEndian, now, + serialize::rkyv_deserialize, }, }; use trc::AddContext; use types::{ collection::{Collection, VanishedCollection}, - field::CalendarNotificationField, + field::{CalendarEventField, CalendarNotificationField, Field}, id::Id, }; pub trait ItipAutoExpunge: Sync + Send { - fn itip_ids(&self, account_id: u32) -> impl Future> + Send; - fn itip_auto_expunge( &self, account_id: u32, @@ -51,65 +68,25 @@ pub trait ItipAutoExpunge: Sync + Send { } impl ItipAutoExpunge for Server { - async fn itip_ids(&self, account_id: u32) -> trc::Result { - let mut document_ids = RoaringBitmap::new(); - self.store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::CalendarEventNotification.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::CalendarEventNotification.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: u64::MAX, - }), - }, - ) - .no_values() - .ascending(), - |key, _| { - document_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!()) - .map(|_| document_ids) - } - async fn itip_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { let mut destroy_ids = RoaringBitmap::new(); + let expire_before = (now().saturating_sub(hold_period) as i64).to_be_bytes(); self.store() .iterate( IterateParams::new( - ValueKey { + IndexKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: 0, - }), + field: CalendarNotificationField::Created.into(), + key: &[][..], }, - ValueKey { + IndexKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: now().saturating_sub(hold_period), - }), + field: CalendarNotificationField::Created.into(), + key: &expire_before[..], }, ) .no_values() @@ -140,28 +117,25 @@ impl ItipAutoExpunge for Server { .account(account_id) .await .caused_by(trc::location!())? - .account_tenant_ids(); + .changed_by(); - for document_id in destroy_ids { - // Fetch event - if let Some(event_) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEventNotification, - document_id, - )) - .await - .caused_by(trc::location!())? - { + self.archives( + account_id, + Collection::CalendarEventNotification, + Field::ARCHIVE, + &destroy_ids, + |document_id, event_| { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(event) .delete(changed_by, account_id, document_id, &mut batch) .caused_by(trc::location!())?; - } - } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; @@ -170,73 +144,104 @@ impl ItipAutoExpunge for Server { } impl CalendarEvent { - pub fn update<'x>( + #[allow(clippy::too_many_arguments)] + pub fn update_full( self, - changed_by: AccountTenantIds, + content: CalendarEventContent, + changed_by: ChangedBy, event: Archive<&ArchivedCalendarEvent>, + event_content: &ArchivedCalendarEventContent, account_id: u32, document_id: u32, - batch: &'x mut BatchBuilder, - ) -> trc::Result<&'x mut BatchBuilder> { + parent_id: Option, + batch: &mut BatchBuilder, + ) -> trc::Result { let mut new_event = self; - - // Build event new_event.modified = now() as i64; - // Prepare write batch + let update = SplitUpdate::full(event, event_content, new_event, content, ())?; + let etag = update.etag(); + batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) - .custom( - ObjectIndexBuilder::new() - .with_current(event) - .with_changes(new_event) - .with_changed_by(changed_by), - ) - .map(|b| b.commit_point()) + .custom(update.into_builder(changed_by, parent_id))? + .commit_point(); + + Ok(etag) } - pub fn insert( + pub fn update_meta( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, + event: Archive<&ArchivedCalendarEvent>, account_id: u32, document_id: u32, - next_alarm: Option, + parent_id: Option, batch: &mut BatchBuilder, - ) -> trc::Result<&mut BatchBuilder> { - // Build event + ) -> trc::Result { + let mut new_event = self; + new_event.modified = now() as i64; + + let update = SplitUpdate::meta_only(event, new_event); + let etag = update.etag(); + + batch + .with_account_id(account_id) + .with_collection(Collection::CalendarEvent) + .with_document(document_id) + .custom(update.into_builder(changed_by, parent_id))? + .commit_point(); + + Ok(etag) + } + + #[allow(clippy::too_many_arguments)] + pub fn insert( + self, + content: CalendarEventContent, + changed_by: ChangedBy, + account_id: u32, + document_id: impl Into, + parent_id: Option, + next_alarms: EventAlarms, + batch: &mut BatchBuilder, + ) -> trc::Result { let mut event = self; let now = now() as i64; event.modified = now; event.created = now; - // Prepare write batch + let changes = GroupwareWrite::full(event, content, ())?; + let etag = format!("\"{}\"", changes.meta().etag); + batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() - .with_changes(event) - .with_changed_by(changed_by), - ) - .map(|batch| { - if let Some(next_alarm) = next_alarm { - next_alarm.write_task(batch); - } + .with_changes(changes) + .with_changed_by(changed_by) + .with_pending_id_opt(parent_id), + )?; - batch.commit_point() - }) + for next_alarm in next_alarms { + next_alarm.write_task(batch)?; + } + batch.commit_point(); + + Ok(etag) } } impl Calendar { pub fn insert( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, - document_id: u32, + document_id: impl Into, batch: &mut BatchBuilder, ) -> trc::Result<&mut BatchBuilder> { // Build address calendar @@ -257,7 +262,7 @@ impl Calendar { batch .with_account_id(account_id) .with_collection(Collection::Calendar) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(calendar) @@ -268,7 +273,7 @@ impl Calendar { pub fn update<'x>( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, calendar: Archive<&ArchivedCalendar>, account_id: u32, document_id: u32, @@ -294,12 +299,35 @@ impl Calendar { } impl CalendarEventNotification { - pub fn insert( + pub fn update_meta( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, + notification: Archive<&ArchivedCalendarEventNotification>, account_id: u32, document_id: u32, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let mut updated = self; + updated.modified = now() as i64; + + batch + .with_account_id(account_id) + .with_collection(Collection::CalendarEventNotification) + .with_document(document_id) + .custom(SplitUpdate::meta_only(notification, updated).into_builder(changed_by, None))? + .commit_point(); + + Ok(()) + } + + pub fn insert( + self, + content: CalendarEventNotificationContent, + changed_by: ChangedBy, + account_id: u32, + document_id: impl Into, + event_id: Option, + batch: &mut BatchBuilder, ) -> trc::Result<&mut BatchBuilder> { // Build event let mut event = self; @@ -311,11 +339,12 @@ impl CalendarEventNotification { batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() - .with_changes(event) - .with_changed_by(changed_by), + .with_changes(GroupwareWrite::full(event, content, ())?) + .with_changed_by(changed_by) + .with_pending_id_opt(event_id), ) .map(|batch| batch.commit_point()) } @@ -326,61 +355,171 @@ impl DestroyArchive> { pub async fn delete_with_events( self, server: &Server, + access_token: &AccessToken, account_info: &AccountInfo, account_id: u32, document_id: u32, - children_ids: Vec, + mut children_ids: Vec, + alarm_ids: &RoaringBitmap, + cleanup: &ContainerCleanup, delete_path: Option, send_itip: bool, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions let calendar_id = document_id; - for document_id in children_ids { - if let Some(event_) = server - .store() - .get_value::>(ValueKey::archive( + let mut resolver = DefaultAlertsResolver::default(); + let mut quota = NotificationQuota::default(); + server + .reap_calendar_notifications(access_token, account_id, &[calendar_id], batch) + .await + .caused_by(trc::location!())?; + let calendars = if !access_token.is_member(account_id) { + server + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + } else { + None + }; + children_ids.sort_unstable(); + children_ids.dedup(); + let content_for_all = send_itip || calendars.is_some(); + let mut content_ids = Vec::new(); + for chunk in children_ids.chunks(MAX_BATCH_DOCUMENTS) { + content_ids.clear(); + content_ids.extend( + chunk + .iter() + .copied() + .filter(|document_id| content_for_all || alarm_ids.contains(*document_id)), + ); + let mut archives = server + .archive_chunk( account_id, Collection::CalendarEvent, - document_id, - )) - .await? - { - DestroyArchive( - event_ - .to_unarchived::() - .caused_by(trc::location!())?, + chunk, + Some((CalendarEventField::Content.field(), &content_ids)), ) - .delete( - account_info, - account_id, - document_id, - calendar_id, - None, - send_itip, - batch, - )?; + .await + .caused_by(trc::location!())?; + let mut tasks = AlarmTasks::for_chunk(account_id, chunk, alarm_ids, &archives)?; + + for &document_id in chunk { + let Some(event_) = archives.take_archive(document_id) else { + continue; + }; + let event = event_ + .to_unarchived::() + .caused_by(trc::location!())?; + let event_flags = event.inner.flags.to_native(); + let mut content = archives.take_value(document_id); + if let Some(calendars) = calendars.as_ref() + && let Some(previous) = server + .stored_event_if_notifiable( + access_token, + account_id, + calendars, + &[calendar_id], + event_flags, + (account_id, document_id), + &mut content, + ) + .await + .caused_by(trc::location!())? + { + content = previous.content; + server + .notify_direct_change( + access_token, + account_id, + DirectChange::Destroyed { + event_id: document_id, + previous: previous.event, + calendar_ids: vec![calendar_id], + event_flags, + }, + Some(calendars), + &mut quota, + batch, + ) + .await + .caused_by(trc::location!())?; + } + DestroyArchive(event) + .delete_from_calendar( + server, + account_info, + account_id, + document_id, + calendar_id, + content, + &mut tasks, + None, + send_itip, + &mut resolver, + cleanup, + batch, + ) + .await?; } } self.delete( - account_info.account_tenant_ids(), + server, + account_info.changed_by(), account_id, document_id, delete_path, batch, ) + .await } - pub fn delete( + pub async fn delete( self, - changed_by: AccountTenantIds, + server: &Server, + changed_by: ChangedBy, account_id: u32, document_id: u32, delete_path: Option, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.delete_with_cleanup( + server, + changed_by, + account_id, + document_id, + delete_path, + &ContainerCleanup::empty(Collection::Calendar), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn delete_with_cleanup( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + delete_path: Option, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, ) -> trc::Result<()> { let calendar = self.0; + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + calendar.inner.metadata_kinds(), + ) + .await?; + // Delete calendar batch .with_account_id(account_id) @@ -403,15 +542,81 @@ impl DestroyArchive> { impl DestroyArchive> { #[allow(clippy::too_many_arguments)] - pub fn delete( + pub async fn delete( self, + server: &Server, account_info: &AccountInfo, account_id: u32, document_id: u32, calendar_id: u32, + content: Option>, delete_path: Option, send_itip: bool, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.delete_from_calendar( + server, + account_info, + account_id, + document_id, + calendar_id, + content, + &mut AlarmTasks::None, + delete_path, + send_itip, + &mut DefaultAlertsResolver::default(), + &ContainerCleanup::empty(Collection::CalendarEvent), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn delete_with_cleanup( + self, + server: &Server, + account_info: &AccountInfo, + account_id: u32, + document_id: u32, + calendar_id: u32, + content: Option>, + tasks: &mut AlarmTasks, + delete_path: Option, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.delete_from_calendar( + server, + account_info, + account_id, + document_id, + calendar_id, + content, + tasks, + delete_path, + false, + &mut DefaultAlertsResolver::default(), + cleanup, + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn delete_from_calendar( + self, + server: &Server, + account_info: &AccountInfo, + account_id: u32, + document_id: u32, + calendar_id: u32, + content: Option>, + tasks: &mut AlarmTasks, + delete_path: Option, + send_itip: bool, + resolver: &mut DefaultAlertsResolver, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, ) -> trc::Result<()> { if let Some(delete_idx) = self .0 @@ -427,19 +632,43 @@ impl DestroyArchive> { .deserialize::() .caused_by(trc::location!())?; new_event.names.swap_remove(delete_idx); + server + .reschedule_event_alarms( + account_id, + document_id, + event.inner, + content, + &new_event + .names + .iter() + .map(DavName::parent_id) + .collect::>(), + tasks, + resolver, + batch, + ) + .await + .caused_by(trc::location!())?; + let update = SplitUpdate::meta_only(event, new_event); batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) - .custom( - ObjectIndexBuilder::new() - .with_changed_by(account_info.account_tenant_ids()) - .with_current(event) - .with_changes(new_event), - ) + .custom(update.into_builder(account_info.changed_by(), None)) .caused_by(trc::location!())?; } else { - self.delete_all(account_info, account_id, document_id, send_itip, batch)?; + self.remove_all( + server, + account_info, + account_id, + document_id, + content, + tasks, + send_itip, + cleanup, + batch, + ) + .await?; } if let Some(delete_path) = delete_path { @@ -453,38 +682,108 @@ impl DestroyArchive> { } #[allow(clippy::too_many_arguments)] - pub fn delete_all( + pub async fn delete_all( self, + server: &Server, account_info: &AccountInfo, account_id: u32, document_id: u32, + content: Option>, send_itip: bool, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.remove_all( + server, + account_info, + account_id, + document_id, + content, + &mut AlarmTasks::None, + send_itip, + &ContainerCleanup::empty(Collection::CalendarEvent), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn remove_all( + self, + server: &Server, + account_info: &AccountInfo, + account_id: u32, + document_id: u32, + content: Option>, + tasks: &mut AlarmTasks, + send_itip: bool, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, ) -> trc::Result<()> { let event = self.0; - // Delete event + let now = now() as i64; + + let has_alarms = event.inner.flags.to_native() & EVENT_HAS_ALARMS != 0; + let send_itip = + send_itip && event.inner.schedule_tag.is_some() && event.inner.event_range_end() > now; + + let content_ = match content { + Some(content) => Some(content), + None if send_itip || has_alarms => server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())?, + None => None, + }; + + let content = content_ + .as_ref() + .map(|content_| content_.to_unarchived::()) + .transpose() + .caused_by(trc::location!())?; + + if has_alarms { + let targets = match &content { + Some(content) => EventAlarmUsers::targets(account_id, content.inner).collect(), + None => vec![AlarmTarget::Owner], + }; + server + .clear_event_alarms(account_id, document_id, targets, tasks, batch) + .await + .caused_by(trc::location!())?; + } + server + .release_or_read( + cleanup, + batch, + account_info.changed_by(), + account_id, + document_id, + event.inner.metadata_kinds(), + ) + .await?; batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id); - // Remove next alarm if it exists - let now = now() as i64; - if let Some(next_alarm) = event.inner.data.next_alarm(now, Tz::Floating) { - next_alarm.delete_task(batch); - } - - // Scheduling - if send_itip - && event.inner.schedule_tag.is_some() - && event.inner.data.event_range_end() > now - { - let event = event - .deserialize::() + if send_itip && let Some(content) = content { + let content = content + .deserialize::() .caused_by(trc::location!())?; - if let Ok(messages) = itip_cancel(&event.data.event, account_info.addresses(), true) { - ItipMessages::new(vec![messages]) + if let Ok(messages) = itip_cancel( + &content.data.event, + account_info.addresses(), + true, + RecipientPolicy::new(&server.core.groupware, event.inner.flags.to_native()), + ) { + ItipMessages::new(messages) .queue(batch) .caused_by(trc::location!())?; } @@ -493,8 +792,8 @@ impl DestroyArchive> { batch .custom( ObjectIndexBuilder::<_, ()>::new() - .with_changed_by(account_info.account_tenant_ids()) - .with_current(event), + .with_changed_by(account_info.changed_by()) + .with_current(SplitCurrent::::MetaOnly(event)), ) .caused_by(trc::location!())?; @@ -506,7 +805,7 @@ impl DestroyArchive> { #[allow(clippy::too_many_arguments)] pub fn delete( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, document_id: u32, batch: &mut BatchBuilder, @@ -519,7 +818,9 @@ impl DestroyArchive> { .custom( ObjectIndexBuilder::<_, ()>::new() .with_changed_by(changed_by) - .with_current(self.0), + .with_current(SplitCurrent::::MetaOnly( + self.0, + )), ) .caused_by(trc::location!())? .commit_point(); @@ -529,70 +830,76 @@ impl DestroyArchive> { } impl CalendarAlarm { - pub fn build_write_ops(&self, account_id: u32, document_id: u32) -> [Operation; 2] { - let task = match &self.typ { + pub fn task(&self, account_id: u32) -> Task { + match &self.typ { CalendarAlarmType::Email { event_start, event_start_tz, event_end, event_end_tz, + recurrence_id, } => Task::CalendarAlarmEmail(TaskCalendarAlarmEmail { account_id: account_id.into(), - document_id: document_id.into(), - alarm_id: self.alarm_id.into(), + document_id: Id::default(), + alarm_id: self.alarm_id.to_task_id(), event_id: self.event_id.into(), event_end: UTCDateTime::from_timestamp(*event_end), event_end_tz: (*event_end_tz).into(), event_start: UTCDateTime::from_timestamp(*event_start), event_start_tz: (*event_start_tz).into(), + target_account_id: self.target.sharee_id().map(Id::from), + recurrence_id: *recurrence_id, status: TaskStatus::at(self.alarm_time), }), CalendarAlarmType::Display { recurrence_id } => { Task::CalendarAlarmNotification(TaskCalendarAlarmNotification { account_id: account_id.into(), - document_id: document_id.into(), - alarm_id: self.alarm_id.into(), + document_id: Id::default(), + alarm_id: self.alarm_id.to_task_id(), event_id: self.event_id.into(), recurrence_id: *recurrence_id, + target_account_id: self.target.sharee_id().map(Id::from), status: TaskStatus::at(self.alarm_time), }) } - }; - let id = Id::from_parts(account_id, document_id).id(); + } + } + + pub fn build_write_ops(&self, account_id: u32, document_id: u32) -> [Operation; 2] { + let mut task = self.task(account_id); + task.set_document_id(Id::from(document_id)); + let id = TaskId::Assigned(self.target.task_id().resolve(account_id, document_id)); [ Operation::Value { class: ValueClass::TaskQueue(TaskQueueClass::Due { id, + typ: task.object_type(), due: self.alarm_time as u64, }), - op: ValueOp::Set(task.object_type().to_id().serialize()), + op: ValueOp::Set(SetValue::Fixed(Vec::new())), }, Operation::Value { class: ValueClass::TaskQueue(TaskQueueClass::Task { id }), - op: ValueOp::Set(task.to_pickled_vec()), + op: ValueOp::Set(SetValue::Fixed(task.to_pickled_vec())), }, ] } - pub fn write_task(&self, batch: &mut BatchBuilder) { - let account_id = batch.last_account_id().unwrap(); - let document_id = batch.last_document_id().unwrap(); - - for op in self.build_write_ops(account_id, document_id) { - batch.any_op(op); - } + pub fn write_task(&self, batch: &mut BatchBuilder) -> trc::Result<()> { + let account_id = batch.last_account_id().ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No account is set for an alarm task.") + })?; + batch.schedule_document_task(self.target.task_id(), self.task(account_id))?; + Ok(()) } - pub fn delete_task(&self, batch: &mut BatchBuilder) { - let account_id = batch.last_account_id().unwrap(); - let document_id = batch.last_document_id().unwrap(); - let id = Id::from_parts(account_id, document_id).id(); - batch - .clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })) - .clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id, - due: self.alarm_time as u64, - })); + pub fn task_type(&self) -> TaskType { + match self.typ { + CalendarAlarmType::Email { .. } => TaskType::CalendarAlarmEmail, + CalendarAlarmType::Display { .. } => TaskType::CalendarAlarmNotification, + } } } @@ -605,7 +912,7 @@ impl ArchivedCalendarEvent { for event_name in self.names.iter() { if let Some(calendar_) = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_info.account_id(), Collection::Calendar, event_name.parent_id.to_native(), @@ -632,3 +939,583 @@ impl ArchivedCalendarEvent { .details("Event is not linked to any calendar")) } } + +pub struct NotifiableEvent { + pub event: ICalendar, + pub content: Option>, +} + +pub enum DirectChange { + Created { + event_id: PendingId, + current: ICalendar, + calendar_ids: Vec, + event_flags: u16, + }, + Updated { + event_id: u32, + previous: ICalendar, + current: ICalendar, + calendar_ids: Vec, + event_flags: u16, + }, + Destroyed { + event_id: u32, + previous: ICalendar, + calendar_ids: Vec, + event_flags: u16, + }, +} + +fn draft_flag(event_flags: u16) -> u16 { + if event_flags & EVENT_DRAFT != 0 { + EVENT_NOTIFICATION_IS_DRAFT + } else { + 0 + } +} + +impl DirectChange { + fn into_notification( + self, + changed_by: u32, + ) -> ( + CalendarEventNotification, + CalendarEventNotificationContent, + Option, + ) { + let (flags, event_id, pending_event_id, previous, current, calendar_ids, event_flags) = + match self { + DirectChange::Created { + event_id, + current, + calendar_ids, + event_flags, + } => { + let (assigned_id, pending_id) = match event_id { + PendingId::Assigned(event_id) => (Some(event_id), None), + PendingId::Slot(slot) => (None, Some(slot)), + }; + + ( + draft_flag(event_flags), + assigned_id, + pending_id, + None, + Some(current), + calendar_ids, + event_flags, + ) + } + DirectChange::Updated { + event_id, + previous, + current, + calendar_ids, + event_flags, + } => ( + EVENT_NOTIFICATION_IS_CHANGE | draft_flag(event_flags), + Some(event_id), + None, + Some(previous), + Some(current), + calendar_ids, + event_flags, + ), + DirectChange::Destroyed { + event_id, + previous, + calendar_ids, + event_flags, + } => ( + EVENT_NOTIFICATION_IS_DESTROY, + Some(event_id), + None, + Some(previous), + None, + calendar_ids, + event_flags, + ), + }; + let flags = flags + | NotificationAudience::of_event(event_flags) + .with_privacy( + [previous.as_ref(), current.as_ref()] + .into_iter() + .flatten() + .all(|snapshot| snapshot.privacy().is_public()), + ) + .flags(); + + ( + CalendarEventNotification { + event_id, + changed_by: EventChangedBy::PrincipalId(changed_by), + calendar_ids, + flags: flags | EVENT_NOTIFICATION_IS_DIRECT, + ..Default::default() + }, + CalendarEventNotificationContent::Direct { previous, current }, + pending_event_id, + ) + } +} + +#[derive(Default)] +pub struct NotificationQuota { + usage: Option, +} + +struct NotificationUsage { + account: Arc, + limit: Option, + stored: usize, + created: usize, + expired: usize, + cursor: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NotificationCursor { + created: [u8; CREATED_LEN], + document_id: u32, +} + +const CREATED_LEN: usize = std::mem::size_of::(); + +impl NotificationQuota { + pub(crate) async fn reserve( + &mut self, + server: &Server, + account_id: u32, + size: u64, + batch: &mut BatchBuilder, + ) -> trc::Result { + let usage = match &mut self.usage { + Some(usage) => usage, + usage @ None => usage.insert(NotificationUsage::load(server, account_id).await?), + }; + + match server.has_available_quota(&usage.account, size).await { + Ok(()) => {} + Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { + return Ok(false); + } + Err(err) => return Err(err.caused_by(trc::location!())), + } + + if let Some(limit) = usage.limit + && usage.stored + usage.created >= limit + usage.expired + { + if limit == 0 { + return Ok(false); + } + let Some(cursor) = server.oldest_notification(account_id, usage.cursor).await? else { + return Ok(false); + }; + let document_id = cursor.document_id; + if let Some(notification) = server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEventNotification, + document_id, + )) + .await + .caused_by(trc::location!())? + { + DestroyArchive( + notification + .to_unarchived::() + .caused_by(trc::location!())?, + ) + .delete(usage.account.changed_by(), account_id, document_id, batch) + .caused_by(trc::location!())?; + } + usage.expired += 1; + usage.cursor = Some(cursor); + } + + usage.created += 1; + Ok(true) + } +} + +impl NotificationUsage { + async fn load(server: &Server, account_id: u32) -> trc::Result { + let account = server + .account(account_id) + .await + .caused_by(trc::location!())?; + let limit = + server.object_quota_limit(&account, StorageQuota::MaxCalendarEventNotifications); + let stored = match limit { + Some(limit) => server + .count_documents(account_id, Collection::CalendarEventNotification, limit) + .await + .caused_by(trc::location!())?, + None => 0, + }; + Ok(NotificationUsage { + account, + limit, + stored, + created: 0, + expired: 0, + cursor: None, + }) + } +} + +pub trait DirectChangeNotification: Sync + Send { + fn notify_direct_change( + &self, + access_token: &AccessToken, + account_id: u32, + change: DirectChange, + calendars: Option<&GroupwareResources>, + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + fn oldest_notification( + &self, + account_id: u32, + after: Option, + ) -> impl Future>> + Send; + + fn may_have_notification_viewers( + &self, + access_token: &AccessToken, + account_id: u32, + notification: &CalendarEventNotification, + calendars: Option<&GroupwareResources>, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn notify_calendar_removal( + &self, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + calendar_id: u32, + event_flags: u16, + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn notify_calendar_addition( + &self, + access_token: &AccessToken, + account_id: u32, + event_id: PendingId, + calendar_id: u32, + event_flags: u16, + source: (u32, u32), + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] + fn stored_event_if_notifiable( + &self, + access_token: &AccessToken, + account_id: u32, + calendars: &GroupwareResources, + calendar_ids: &[u32], + event_flags: u16, + source: (u32, u32), + prefetched: &mut Option>, + ) -> impl Future>> + Send; +} + +impl DirectChangeNotification for Server { + async fn notify_direct_change( + &self, + access_token: &AccessToken, + account_id: u32, + change: DirectChange, + calendars: Option<&GroupwareResources>, + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let (notification, content, pending_event_id) = + change.into_notification(access_token.account_id()); + if !self + .may_have_notification_viewers(access_token, account_id, ¬ification, calendars) + .await + .caused_by(trc::location!())? + { + return Ok(()); + } + let size = content + .snapshots() + .into_iter() + .flatten() + .map(SizeWriter::ical) + .sum::(); + if !quota + .reserve(self, account_id, size as u64, batch) + .await + .caused_by(trc::location!())? + { + return Ok(()); + } + + let document_id = + batch.reserve_document_id(account_id, Collection::CalendarEventNotification); + notification + .insert( + content, + access_token.changed_by(), + account_id, + document_id, + pending_event_id, + batch, + ) + .caused_by(trc::location!())?; + + Ok(()) + } + + async fn notify_calendar_removal( + &self, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + calendar_id: u32, + event_flags: u16, + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let Some(calendars) = self + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + let Some(previous) = self + .stored_event_if_notifiable( + access_token, + account_id, + &calendars, + &[calendar_id], + event_flags, + (account_id, document_id), + &mut None, + ) + .await + .caused_by(trc::location!())? + .map(|notifiable| notifiable.event) + else { + return Ok(()); + }; + + self.notify_direct_change( + access_token, + account_id, + DirectChange::Destroyed { + event_id: document_id, + previous, + calendar_ids: vec![calendar_id], + event_flags, + }, + Some(&calendars), + quota, + batch, + ) + .await + .caused_by(trc::location!()) + } + + async fn notify_calendar_addition( + &self, + access_token: &AccessToken, + account_id: u32, + event_id: PendingId, + calendar_id: u32, + event_flags: u16, + source: (u32, u32), + quota: &mut NotificationQuota, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let Some(calendars) = self + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + let Some(current) = self + .stored_event_if_notifiable( + access_token, + account_id, + &calendars, + &[calendar_id], + event_flags, + source, + &mut None, + ) + .await + .caused_by(trc::location!())? + .map(|notifiable| notifiable.event) + else { + return Ok(()); + }; + + self.notify_direct_change( + access_token, + account_id, + DirectChange::Created { + event_id, + current, + calendar_ids: vec![calendar_id], + event_flags, + }, + Some(&calendars), + quota, + batch, + ) + .await + .caused_by(trc::location!()) + } + + async fn stored_event_if_notifiable( + &self, + access_token: &AccessToken, + account_id: u32, + calendars: &GroupwareResources, + calendar_ids: &[u32], + event_flags: u16, + source: (u32, u32), + prefetched: &mut Option>, + ) -> trc::Result> { + if !may_have_viewers( + access_token, + account_id, + calendars, + calendar_ids, + NotificationAudience::of_event(event_flags), + ) { + return Ok(None); + } + + let (source_account_id, source_document_id) = source; + let content = match prefetched.take() { + Some(content) => content, + None => { + let Some(content) = self + .store() + .get_value::>(ValueKey::property( + source_account_id, + Collection::CalendarEvent, + source_document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + content + } + }; + + content + .unarchive::() + .and_then(|archived| rkyv_deserialize::<_, ICalendar>(&archived.data.event)) + .map(|event| { + Some(NotifiableEvent { + event, + content: Some(content), + }) + }) + .caused_by(trc::location!()) + } + + async fn may_have_notification_viewers( + &self, + access_token: &AccessToken, + account_id: u32, + notification: &CalendarEventNotification, + calendars: Option<&GroupwareResources>, + ) -> trc::Result { + let audience = NotificationAudience::from_flags(notification.flags); + let account_viewers = AccountViewers::of( + self.account(account_id) + .await + .caused_by(trc::location!())? + .as_ref(), + ); + let viewers = |calendars: &GroupwareResources| { + has_viewers( + access_token, + account_id, + calendars, + ¬ification.calendar_ids, + audience, + account_viewers, + ) + }; + + match calendars { + Some(calendars) => Ok(viewers(calendars)), + None => Ok(self + .calendars_if_any(access_token, account_id) + .await + .caused_by(trc::location!())? + .is_some_and(|calendars| viewers(&calendars))), + } + } + + async fn oldest_notification( + &self, + account_id: u32, + after: Option, + ) -> trc::Result> { + let mut oldest = None; + let (from_created, from_document_id) = after.as_ref().map_or((&[][..], 0), |after| { + (&after.created[..], after.document_id) + }); + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: Collection::CalendarEventNotification.into(), + document_id: from_document_id, + field: CalendarNotificationField::Created.into(), + key: from_created, + }, + IndexKey { + account_id, + collection: Collection::CalendarEventNotification.into(), + document_id: u32::MAX, + field: CalendarNotificationField::Created.into(), + key: &u64::MAX.to_be_bytes()[..], + }, + ) + .no_values() + .ascending() + .limit(1 + usize::from(after.is_some())), + |key, _| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if after.is_some_and(|after| after.document_id == document_id) { + return Ok(true); + } + oldest = Some(NotificationCursor { + created: key + .deserialize_be_u64(key.len().saturating_sub(U32_LEN + CREATED_LEN))? + .to_be_bytes(), + document_id, + }); + Ok(false) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(oldest) + } +} diff --git a/crates/groupware/src/calendar/user.rs b/crates/groupware/src/calendar/user.rs new file mode 100644 index 00000000..560bb2b6 --- /dev/null +++ b/crates/groupware/src/calendar/user.rs @@ -0,0 +1,1344 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedCalendarEventContent, ArchivedEventPreferences, CalendarEventContent, + CalendarEventData, EventPreferences, EventUserData, MAX_USER_ALERTS, MAX_USER_INSTANCES, + MAX_USER_KEYWORD_LEN, MAX_USER_KEYWORDS, PREF_FREE_BUSY_FREE, PREF_HAS_ALERTS, PREF_HAS_COLOR, + PREF_HAS_FREE_BUSY, PREF_HAS_KEYWORDS, PREF_HAS_USE_DEFAULT_ALERTS, PREF_USE_DEFAULT_ALERTS, + color::CssColor, + compare::{ + ComparisonScope, EntryList, NaiveDateTime, RecurrenceComponents, RedundantOverrides, + }, + expand::{ComponentRecurrenceId, RecurrenceKey}, +}; +use ahash::AHashSet; +use calcard::{ + common::{PartialDateTime, timezone::Tz}, + icalendar::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, + ICalendarParameterName, ICalendarProperty, ICalendarTransparency, ICalendarValue, + timezone::TzResolver, + }, +}; + +pub const BASE_INSTANCE: u32 = 0; + +const DEFAULT_ALERT_FLAGS: u16 = PREF_HAS_USE_DEFAULT_ALERTS | PREF_USE_DEFAULT_ALERTS; +const MAX_COMPONENT_DEPTH: usize = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UserDataError { + TooManyKeywords, + KeywordTooLong, + TooManyAlerts, + TooManyInstances, + InvalidColor, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UserDataLookup { + Inherited, + Exact, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UserDataView { + Full, + AlertsOnly, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UserDataSplit { + UserDataOnly, + SharedDataChanged, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UpdatedPolicy { + Server(i64), + Client, +} + +#[derive(Debug, Clone, Copy)] +pub struct UserDataUpdate<'x> { + pub account_id: u32, + pub view: UserDataView, + pub previous: Option<&'x EventPreferences>, + pub current: Option<&'x EventPreferences>, + pub updated: UpdatedPolicy, +} + +impl CalendarEventContent { + pub fn preferences(&self, account_id: u32) -> Option<&EventPreferences> { + self.preferences.iter().find(|p| p.account_id == account_id) + } + + pub fn preferences_mut(&mut self, account_id: u32) -> &mut EventPreferences { + let idx = if let Some(idx) = self + .preferences + .iter() + .position(|p| p.account_id == account_id) + { + idx + } else { + self.preferences.push(EventPreferences { + account_id, + ..Default::default() + }); + self.preferences.len() - 1 + }; + + &mut self.preferences[idx] + } + + pub fn set_preferences(&mut self, preferences: EventPreferences) { + let account_id = preferences.account_id; + if preferences.is_empty() { + self.preferences.retain(|p| p.account_id != account_id); + } else if let Some(current) = self + .preferences + .iter_mut() + .find(|p| p.account_id == account_id) + { + *current = preferences; + } else { + self.preferences.push(preferences); + } + } +} + +impl ArchivedCalendarEventContent { + pub fn preferences(&self, account_id: u32) -> Option<&ArchivedEventPreferences> { + self.preferences.iter().find(|p| p.account_id == account_id) + } +} + +impl CalendarEventData { + pub fn apply_user_data(&mut self, preferences: Option<&EventPreferences>, view: UserDataView) { + if let Some(overrides) = self.user_data_overrides(preferences, view) { + overrides.append_to(&mut self.event); + } + self.event.apply_user_data(preferences, view); + } + + pub fn user_data_view( + &self, + preferences: Option<&EventPreferences>, + view: UserDataView, + ) -> ICalendar { + let mut event = self.event.clone(); + if let Some(overrides) = self.user_data_overrides(preferences, view) { + overrides.append_to(&mut event); + } + event.apply_user_data(preferences, view); + event + } + + fn user_data_overrides( + &self, + preferences: Option<&EventPreferences>, + view: UserDataView, + ) -> Option { + let preferences = preferences.filter(|_| view == UserDataView::Full)?; + let mut keys = preferences + .instances + .iter() + .filter_map(|instance| RecurrenceKey::from_prefix(instance.recurrence_key)) + .collect::>(); + let mut instance_keys = InstanceKeys::new(&self.event); + for key in self + .event + .components + .iter() + .filter(|component| { + component.is_user_data_component() && component.is_recurrence_override() + }) + .filter_map(|component| instance_keys.of(component)) + .filter_map(RecurrenceKey::from_prefix) + .collect::>() + { + keys.remove(&key); + } + if keys.is_empty() { + return None; + } + let base_id = self.event.base_component_id()?; + let expansions = self.expand_from_ids(&mut keys, Tz::Floating)?; + let root_id = self + .event + .components + .iter() + .position(|component| component.component_ids.contains(&base_id))?; + let mut overrides = OccurrenceOverrides::new(&self.event, base_id)?; + for expansion in expansions + .iter() + .filter(|expansion| expansion.comp_id == base_id) + { + overrides.push(expansion.start_naive); + } + Some(overrides.into_personal_overrides(root_id)) + } +} + +struct PersonalOverrides { + root_id: usize, + override_ids: Vec, + components: Vec, +} + +impl PersonalOverrides { + fn append_to(self, event: &mut ICalendar) { + event.components.extend(self.components); + if let Some(root) = event.components.get_mut(self.root_id) { + root.component_ids.extend(self.override_ids); + } + } +} + +impl EventPreferences { + pub fn instance(&self, recurrence_key: u32) -> Option<&EventUserData> { + self.instances + .binary_search_by_key(&recurrence_key, |i| i.recurrence_key) + .ok() + .and_then(|idx| self.instances.get(idx)) + } + + pub fn instance_mut( + &mut self, + recurrence_key: u32, + ) -> Result<&mut EventUserData, UserDataError> { + let idx = match self + .instances + .binary_search_by_key(&recurrence_key, |i| i.recurrence_key) + { + Ok(idx) => idx, + Err(idx) => { + if self.instances.len() >= MAX_USER_INSTANCES { + return Err(UserDataError::TooManyInstances); + } + self.instances.insert( + idx, + EventUserData { + recurrence_key, + ..Default::default() + }, + ); + idx + } + }; + self.instances + .get_mut(idx) + .ok_or(UserDataError::TooManyInstances) + } + + pub fn use_default_alerts(&self) -> bool { + self.instance(BASE_INSTANCE) + .is_some_and(|i| i.flags & PREF_USE_DEFAULT_ALERTS != 0) + } + + pub fn set_use_default_alerts(&mut self, value: bool) -> Result<(), UserDataError> { + let instance = self.instance_mut(BASE_INSTANCE)?; + instance.flags |= PREF_HAS_USE_DEFAULT_ALERTS; + if value { + instance.flags |= PREF_USE_DEFAULT_ALERTS; + } else { + instance.flags &= !PREF_USE_DEFAULT_ALERTS; + } + Ok(()) + } + + pub fn inherit_default_alerts( + &mut self, + current: Option<&EventPreferences>, + ) -> Result<(), UserDataError> { + if let Some(flags) = current + .and_then(|preferences| preferences.instance(BASE_INSTANCE)) + .map(|base| base.flags & DEFAULT_ALERT_FLAGS) + .filter(|flags| *flags != 0) + { + self.instance_mut(BASE_INSTANCE)?.flags |= flags; + } + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.updated == 0 && self.instances.iter().all(|i| i.flags == 0) + } + + pub fn size(&self) -> usize { + self.instances.iter().map(|i| i.size()).sum::() + + std::mem::size_of::() + } + + fn stamp_updated(&mut self, update: &UserDataUpdate<'_>, client_updated: Option) { + let previous_updated = update.previous.map_or(0, |previous| previous.updated); + let is_changed = update + .previous + .map_or(!self.instances.is_empty(), |previous| { + previous.instances != self.instances + }); + self.updated = match update.updated { + UpdatedPolicy::Server(now) if is_changed => now, + UpdatedPolicy::Client => client_updated.unwrap_or(previous_updated), + UpdatedPolicy::Server(_) => previous_updated, + }; + } +} + +impl ArchivedEventPreferences { + pub fn use_default_alerts(&self) -> bool { + self.instances + .iter() + .find(|i| i.recurrence_key == BASE_INSTANCE) + .is_some_and(|i| i.flags & PREF_USE_DEFAULT_ALERTS != 0) + } + + pub fn is_free(&self, recurrence_key: u32) -> Option { + [recurrence_key, BASE_INSTANCE] + .into_iter() + .filter_map(|key| { + self.instances + .binary_search_by_key(&key, |i| i.recurrence_key.to_native()) + .ok() + .and_then(|idx| self.instances.get(idx)) + }) + .map(|i| i.flags.to_native()) + .find(|flags| flags & PREF_HAS_FREE_BUSY != 0) + .map(|flags| flags & PREF_FREE_BUSY_FREE != 0) + } + + pub fn size(&self) -> usize { + self.instances + .iter() + .map(|i| { + i.keywords.iter().map(|k| k.len()).sum::() + + i.alerts.iter().map(|a| a.size()).sum::() + + std::mem::size_of::() + }) + .sum::() + + std::mem::size_of::() + } +} + +impl EventUserData { + pub fn set_color(&mut self, color: Option) { + self.flags |= PREF_HAS_COLOR; + self.color = color; + } + + pub fn set_free_busy(&mut self, is_free: bool) { + self.flags |= PREF_HAS_FREE_BUSY; + if is_free { + self.flags |= PREF_FREE_BUSY_FREE; + } else { + self.flags &= !PREF_FREE_BUSY_FREE; + } + } + + pub fn set_keywords<'x>( + &mut self, + keywords: impl IntoIterator, + ) -> Result<(), UserDataError> { + self.keywords.clear(); + for keyword in keywords { + if keyword.len() > MAX_USER_KEYWORD_LEN { + return Err(UserDataError::KeywordTooLong); + } + if self.keywords.len() >= MAX_USER_KEYWORDS { + return Err(UserDataError::TooManyKeywords); + } + if !self.keywords.iter().any(|k| k == keyword) { + self.keywords.push(keyword.to_string()); + } + } + self.flags |= PREF_HAS_KEYWORDS; + Ok(()) + } + + pub fn set_alerts(&mut self, alerts: Vec) -> Result<(), UserDataError> { + if alerts.len() > MAX_USER_ALERTS { + return Err(UserDataError::TooManyAlerts); + } + self.flags |= PREF_HAS_ALERTS; + self.alerts = alerts; + Ok(()) + } + + pub fn size(&self) -> usize { + self.keywords.iter().map(|k| k.len()).sum::() + + self.alerts.iter().map(|a| a.size()).sum::() + + std::mem::size_of::() + } + + fn clear_alerts(&mut self) { + self.flags &= !PREF_HAS_ALERTS; + self.alerts.clear(); + } + + fn remove_inherited(&mut self, base: &EventUserData) { + for flag in [ + PREF_HAS_COLOR, + PREF_HAS_KEYWORDS, + PREF_HAS_FREE_BUSY, + PREF_HAS_ALERTS, + ] { + let (has_self, has_base) = (self.flags & flag != 0, base.flags & flag != 0); + let is_inherited = match (has_self, has_base) { + (false, false) => true, + (true, true) => match flag { + PREF_HAS_COLOR => self.color == base.color, + PREF_HAS_KEYWORDS => self.keywords == base.keywords, + PREF_HAS_FREE_BUSY => { + self.flags & PREF_FREE_BUSY_FREE == base.flags & PREF_FREE_BUSY_FREE + } + _ => self.alerts == base.alerts, + }, + _ => false, + }; + if is_inherited { + self.flags &= !flag; + match flag { + PREF_HAS_COLOR => self.color = None, + PREF_HAS_KEYWORDS => self.keywords.clear(), + PREF_HAS_FREE_BUSY => self.flags &= !PREF_FREE_BUSY_FREE, + _ => self.alerts.clear(), + } + } else { + self.flags |= flag; + } + } + } +} + +pub trait UserDataEntry { + fn is_user_data(&self) -> bool; +} + +impl UserDataEntry for ICalendarEntry { + fn is_user_data(&self) -> bool { + match &self.name { + ICalendarProperty::Color + | ICalendarProperty::Categories + | ICalendarProperty::Transp => true, + ICalendarProperty::Jsprop => self + .parameter(&ICalendarParameterName::Jsptr) + .and_then(|pointer| pointer.as_text()) + .and_then(|pointer| pointer.split('/').next()) + .is_some_and(|property| { + hashify::set!( + property.as_bytes(), + "alerts", + "color", + "freeBusyStatus", + "keywords", + "useDefaultAlerts" + ) + }), + _ => false, + } + } +} + +pub trait ICalendarUserData { + fn validate_user_data(&self) -> Result<(), UserDataError>; + + fn validate_user_data_changes(&self, stored: &ICalendar) -> Result<(), UserDataError>; + + fn apply_user_data(&mut self, preferences: Option<&EventPreferences>, view: UserDataView); + + fn extract_user_data(&self, account_id: u32) -> Result; + + fn extract_user_preferences( + &self, + stored: &ICalendar, + update: UserDataUpdate<'_>, + ) -> Result; + + fn copy_user_data(&mut self, source: &ICalendar) -> Vec; + + fn split_user_data( + &mut self, + stored: &ICalendar, + update: UserDataUpdate<'_>, + ) -> Result<(EventPreferences, UserDataSplit), UserDataError>; + + fn user_data_override( + &self, + comp_id: u32, + recurrence_key: RecurrenceKey, + ) -> Option<&ICalendarComponent>; + + fn has_same_user_data(&self, stored: &ICalendar) -> bool; +} + +impl ICalendarUserData for ICalendar { + fn validate_user_data(&self) -> Result<(), UserDataError> { + self.components + .iter() + .filter(|c| c.is_user_data_component()) + .try_for_each(|component| UserDataUsage::new(self, component).validate(None)) + } + + fn validate_user_data_changes(&self, stored: &ICalendar) -> Result<(), UserDataError> { + let stored_components = UserDataComponents::new(stored); + let mut instance_keys = InstanceKeys::new(self); + self.components + .iter() + .filter(|c| c.is_user_data_component()) + .try_for_each(|component| { + UserDataUsage::new(self, component).validate( + instance_keys + .of(component) + .and_then(|key| { + stored_components.counterpart(key, UserDataLookup::Inherited) + }) + .map(|stored_component| UserDataUsage::new(stored, stored_component)) + .as_ref(), + ) + }) + } + + fn apply_user_data(&mut self, preferences: Option<&EventPreferences>, view: UserDataView) { + let base = preferences.and_then(|p| p.instance(BASE_INSTANCE)); + for (comp_id, key) in self.user_data_component_ids() { + let instance = preferences + .filter(|_| key != BASE_INSTANCE) + .and_then(|p| p.instance(key)); + let resolve = |flag: u16| { + instance + .filter(|i| i.flags & flag != 0) + .or_else(|| base.filter(|b| b.flags & flag != 0)) + }; + + let first_alert_id = self.components.len() as u32; + self.components.extend( + resolve(PREF_HAS_ALERTS) + .into_iter() + .flat_map(|i| i.alerts.iter()) + .map(ICalendarComponent::detached), + ); + let alert_ids = first_alert_id..self.components.len() as u32; + + let entries = (view == UserDataView::Full).then(|| { + [ + resolve(PREF_HAS_COLOR) + .and_then(|i| i.color.as_ref()) + .map(|color| ICalendarEntry { + name: ICalendarProperty::Color, + params: vec![], + values: [ICalendarValue::Text(color.to_string())].into(), + }), + resolve(PREF_HAS_KEYWORDS) + .filter(|i| !i.keywords.is_empty()) + .map(|i| ICalendarEntry { + name: ICalendarProperty::Categories, + params: vec![], + values: i + .keywords + .iter() + .map(|k| ICalendarValue::Text(k.clone())) + .collect(), + }), + resolve(PREF_HAS_FREE_BUSY).map(|i| ICalendarEntry { + name: ICalendarProperty::Transp, + params: vec![], + values: [ICalendarValue::Transparency( + if i.flags & PREF_FREE_BUSY_FREE != 0 { + ICalendarTransparency::Transparent + } else { + ICalendarTransparency::Opaque + }, + )] + .into(), + }), + ] + .into_iter() + .flatten() + .collect::>() + }); + self.replace_user_data(comp_id, alert_ids, entries); + } + } + + fn extract_user_data(&self, account_id: u32) -> Result { + let mut preferences = EventPreferences { + account_id, + ..Default::default() + }; + let base = match self + .base_component_id() + .and_then(|id| self.component_by_id(id)) + { + Some(component) => self.component_user_data(component)?, + None => EventUserData::default(), + }; + + let mut instance_keys = InstanceKeys::new(self); + for component in self + .components + .iter() + .filter(|c| c.is_user_data_component() && c.is_recurrence_override()) + { + let Some(key) = instance_keys.of(component) else { + continue; + }; + let mut data = self.component_user_data(component)?; + data.remove_inherited(&base); + if data.flags != 0 { + *preferences.instance_mut(key)? = EventUserData { + recurrence_key: key, + ..data + }; + } + } + if base.flags != 0 { + *preferences.instance_mut(BASE_INSTANCE)? = base; + } + + Ok(preferences) + } + + fn extract_user_preferences( + &self, + stored: &ICalendar, + update: UserDataUpdate<'_>, + ) -> Result { + let mut preferences = match update.view { + UserDataView::Full => { + let mut preferences = self.extract_user_data(update.account_id)?; + preferences.inherit_default_alerts(update.current)?; + preferences + } + UserDataView::AlertsOnly => { + self.extract_alerts(update.current.cloned().unwrap_or_else(|| EventPreferences { + account_id: update.account_id, + ..Default::default() + }))? + } + }; + preferences.stamp_updated( + &update, + self.base_updated() + .filter(|updated| stored.base_updated() != Some(*updated)), + ); + + Ok(preferences) + } + + fn copy_user_data(&mut self, source: &ICalendar) -> Vec { + let dropped = { + let stored = UserDataComponents::new(self); + let mut instance_keys = InstanceKeys::new(source); + source + .components + .iter() + .filter(|component| component.is_user_data_component()) + .filter_map(|component| { + let recurrence_id = instance_keys.recurrence_id(component)?; + let key = RecurrenceKey::from_recurrence_id(recurrence_id)?.prefix(); + (!stored.contains(key)).then_some(recurrence_id) + }) + .collect::>() + }; + + self.restore_user_data( + &UserDataComponents::new(source), + UserDataView::Full, + UserDataLookup::Exact, + ); + self.drop_unreachable_components(); + dropped + } + + fn split_user_data( + &mut self, + stored: &ICalendar, + update: UserDataUpdate<'_>, + ) -> Result<(EventPreferences, UserDataSplit), UserDataError> { + let preferences = self.extract_user_preferences(stored, update)?; + let stored_components = UserDataComponents::new(stored); + self.restore_user_data(&stored_components, update.view, UserDataLookup::Inherited); + self.prune_user_data_overrides(&stored_components); + + let split = if ComparisonScope::SharedData.component_eq(self, 0, stored, 0) { + UserDataSplit::UserDataOnly + } else { + UserDataSplit::SharedDataChanged + }; + + Ok((preferences, split)) + } + + fn user_data_override( + &self, + comp_id: u32, + recurrence_key: RecurrenceKey, + ) -> Option<&ICalendarComponent> { + if self + .component_by_id(comp_id) + .is_none_or(|c| c.is_recurrence_override()) + { + return None; + } + let mut instance_keys = InstanceKeys::new(self); + self.components + .first()? + .component_ids + .iter() + .filter_map(|id| self.component_by_id(*id)) + .find(|c| { + c.is_user_data_component() + && c.is_recurrence_override() + && instance_keys.of(c) == Some(recurrence_key.prefix()) + }) + } + + fn has_same_user_data(&self, stored: &ICalendar) -> bool { + let stored_components = UserDataComponents::new(stored); + self.user_data_component_ids() + .into_iter() + .filter_map(|(comp_id, key)| self.component_by_id(comp_id).map(|c| (c, key))) + .all(|(component, key)| { + let entries = component.user_data_entries(); + match stored_components.counterpart(key, UserDataLookup::Inherited) { + Some(source) => entries.same_entries(&source.user_data_entries()), + None => entries.is_empty(), + } + }) + } +} + +trait UserDataComponent { + fn is_user_data_component(&self) -> bool; + + fn user_data_entries(&self) -> Vec<&ICalendarEntry>; + + fn detached(&self) -> ICalendarComponent; +} + +impl UserDataComponent for ICalendarComponent { + fn is_user_data_component(&self) -> bool { + self.component_type.is_event_or_todo() + } + + fn user_data_entries(&self) -> Vec<&ICalendarEntry> { + let mut entries = self + .entries + .iter() + .filter(|entry| entry.is_user_data()) + .collect::>(); + entries.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + entries + } + + fn detached(&self) -> ICalendarComponent { + ICalendarComponent { + component_type: self.component_type.clone(), + entries: self.entries.clone(), + component_ids: Vec::new(), + } + } +} + +trait UserDataTree { + fn user_data_component_ids(&self) -> Vec<(u32, u32)>; + + fn alarms<'x>( + &'x self, + component: &'x ICalendarComponent, + ) -> impl Iterator; + + fn component_user_data( + &self, + component: &ICalendarComponent, + ) -> Result; + + fn extract_alerts( + &self, + preferences: EventPreferences, + ) -> Result; + + fn restore_user_data( + &mut self, + stored: &UserDataComponents<'_>, + view: UserDataView, + lookup: UserDataLookup, + ); + + fn drop_unreachable_components(&mut self); + + fn prune_user_data_overrides(&mut self, stored: &UserDataComponents<'_>); + + fn replace_user_data( + &mut self, + comp_id: u32, + alert_ids: impl IntoIterator, + entries: Option>, + ); + + fn base_updated(&self) -> Option; +} + +impl UserDataTree for ICalendar { + fn user_data_component_ids(&self) -> Vec<(u32, u32)> { + let mut instance_keys = InstanceKeys::new(self); + self.components + .iter() + .enumerate() + .filter(|(_, component)| component.is_user_data_component()) + .filter_map(|(id, component)| Some((id as u32, instance_keys.of(component)?))) + .collect() + } + + fn alarms<'x>( + &'x self, + component: &'x ICalendarComponent, + ) -> impl Iterator { + component.component_ids.iter().filter_map(|id| { + self.component_by_id(*id) + .filter(|child| child.component_type == ICalendarComponentType::VAlarm) + .map(|child| (*id, child)) + }) + } + + fn component_user_data( + &self, + component: &ICalendarComponent, + ) -> Result { + let mut data = EventUserData::default(); + let mut keywords: Vec<&str> = Vec::new(); + let mut has_keywords = false; + + for entry in &component.entries { + match &entry.name { + ICalendarProperty::Color => { + if let Some(value) = entry.values.first().and_then(|v| v.as_text()) { + data.set_color(Some( + CssColor::parse(value).ok_or(UserDataError::InvalidColor)?, + )); + } + } + ICalendarProperty::Categories => { + has_keywords = true; + keywords.extend(entry.values.iter().filter_map(|v| v.as_text())); + } + ICalendarProperty::Transp => { + data.set_free_busy(matches!( + entry.values.first(), + Some(ICalendarValue::Transparency( + ICalendarTransparency::Transparent + )) + )); + } + _ => {} + } + } + if has_keywords { + data.set_keywords(keywords)?; + } + + let alerts = self + .alarms(component) + .map(|(_, alarm)| alarm.detached()) + .collect::>(); + if !alerts.is_empty() { + data.set_alerts(alerts)?; + } + + Ok(data) + } + + fn extract_alerts( + &self, + mut preferences: EventPreferences, + ) -> Result { + let base_alerts = self + .base_component_id() + .and_then(|id| self.component_by_id(id)) + .map(|base| { + self.alarms(base) + .map(|(_, alarm)| alarm.detached()) + .collect::>() + }) + .unwrap_or_default(); + + let mut instance_keys = InstanceKeys::new(self); + for component in self + .components + .iter() + .filter(|c| c.is_user_data_component() && c.is_recurrence_override()) + { + let Some(key) = instance_keys.of(component) else { + continue; + }; + let alerts = self + .alarms(component) + .map(|(_, alarm)| alarm.detached()) + .collect::>(); + if alerts != base_alerts { + preferences.instance_mut(key)?.set_alerts(alerts)?; + } else if let Some(instance) = preferences + .instances + .iter_mut() + .find(|instance| instance.recurrence_key == key) + { + instance.clear_alerts(); + } + } + + if !base_alerts.is_empty() { + preferences + .instance_mut(BASE_INSTANCE)? + .set_alerts(base_alerts)?; + } else if let Some(base) = preferences + .instances + .iter_mut() + .find(|instance| instance.recurrence_key == BASE_INSTANCE) + { + base.clear_alerts(); + } + preferences.instances.retain(|instance| instance.flags != 0); + + Ok(preferences) + } + + fn restore_user_data( + &mut self, + stored: &UserDataComponents<'_>, + view: UserDataView, + lookup: UserDataLookup, + ) { + let mut appender = ComponentAppender::new(stored.ical, self.components.len() as u32); + let replacements = self + .user_data_component_ids() + .into_iter() + .filter_map(|(comp_id, key)| match stored.counterpart(key, lookup) { + Some(source) => Some(( + comp_id, + stored + .ical + .alarms(source) + .filter_map(|(alarm_id, _)| appender.copy_tree(alarm_id)) + .collect::>(), + (view == UserDataView::Full).then(|| { + source + .entries + .iter() + .filter(|entry| entry.is_user_data()) + .cloned() + .collect::>() + }), + )), + None if lookup == UserDataLookup::Exact => None, + None => Some(( + comp_id, + Vec::new(), + (view == UserDataView::Full).then(Vec::new), + )), + }) + .collect::>(); + + self.components.extend(appender.into_components()); + for (comp_id, alert_ids, entries) in replacements { + self.replace_user_data(comp_id, alert_ids, entries); + } + } + + fn drop_unreachable_components(&mut self) { + let mut is_reachable = vec![false; self.components.len()]; + let mut pending = vec![0u32]; + while let Some(comp_id) = pending.pop() { + match is_reachable.get_mut(comp_id as usize) { + Some(is_reachable) if !*is_reachable => *is_reachable = true, + _ => continue, + } + if let Some(component) = self.components.get(comp_id as usize) { + pending.extend_from_slice(&component.component_ids); + } + } + + let unreachable = is_reachable + .into_iter() + .enumerate() + .filter(|(_, is_reachable)| !*is_reachable) + .map(|(comp_id, _)| comp_id as u32) + .collect::>(); + if !unreachable.is_empty() && !self.remove_component_ids(&unreachable) { + for component in &mut self.components { + component + .component_ids + .retain(|id| !unreachable.contains(id)); + } + } + } + + fn prune_user_data_overrides(&mut self, stored: &UserDataComponents<'_>) { + let Some(overrides) = RedundantOverrides::new(self) else { + return; + }; + let redundant = self + .user_data_component_ids() + .into_iter() + .filter(|(comp_id, key)| { + *key != BASE_INSTANCE && !stored.contains(*key) && overrides.is_redundant(*comp_id) + }) + .map(|(comp_id, _)| comp_id) + .collect::>(); + + if !redundant.is_empty() { + for component in &mut self.components { + component.component_ids.retain(|id| !redundant.contains(id)); + } + } + } + + fn replace_user_data( + &mut self, + comp_id: u32, + alert_ids: impl IntoIterator, + entries: Option>, + ) { + let Some(mut children) = self + .components + .get_mut(comp_id as usize) + .map(|component| std::mem::take(&mut component.component_ids)) + else { + return; + }; + children.retain(|id| { + self.component_by_id(*id) + .is_none_or(|child| child.component_type != ICalendarComponentType::VAlarm) + }); + children.extend(alert_ids); + + if let Some(component) = self.components.get_mut(comp_id as usize) { + component.component_ids = children; + if let Some(entries) = entries { + component.entries.retain(|entry| !entry.is_user_data()); + component.entries.extend(entries); + } + } + } + + fn base_updated(&self) -> Option { + self.base_component_id() + .and_then(|id| self.component_by_id(id)) + .and_then(|component| component.property(&ICalendarProperty::Dtstamp)) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_partial_date_time()) + .and_then(|value| value.to_timestamp()) + } +} + +struct UserDataComponents<'x> { + ical: &'x ICalendar, + ids: Vec<(u32, u32)>, +} + +struct UserDataUsage<'x> { + ical: &'x ICalendar, + component: &'x ICalendarComponent, +} + +impl<'x> UserDataUsage<'x> { + fn new(ical: &'x ICalendar, component: &'x ICalendarComponent) -> Self { + UserDataUsage { ical, component } + } + + fn keywords(&self) -> impl Iterator { + self.component + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Categories) + .flat_map(|entry| entry.values.iter()) + .filter_map(|value| value.as_text()) + } + + fn colors(&self) -> impl Iterator { + self.component + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Color) + .filter_map(|entry| entry.values.first().and_then(|value| value.as_text())) + } + + fn alerts(&self) -> usize { + self.ical.alarms(self.component).count() + } + + fn validate(&self, stored: Option<&UserDataUsage<'_>>) -> Result<(), UserDataError> { + let mut keywords = 0; + for keyword in self.keywords() { + if keyword.len() > MAX_USER_KEYWORD_LEN + && stored.is_none_or(|stored| stored.keywords().all(|stored| stored != keyword)) + { + return Err(UserDataError::KeywordTooLong); + } + keywords += 1; + } + if keywords > MAX_USER_KEYWORDS + && stored.is_none_or(|stored| keywords > stored.keywords().count()) + { + return Err(UserDataError::TooManyKeywords); + } + if self.colors().any(|color| { + CssColor::parse(color).is_none() + && stored.is_none_or(|stored| stored.colors().all(|stored| stored != color)) + }) { + return Err(UserDataError::InvalidColor); + } + let alerts = self.alerts(); + if alerts > MAX_USER_ALERTS && stored.is_none_or(|stored| alerts > stored.alerts()) { + return Err(UserDataError::TooManyAlerts); + } + Ok(()) + } +} + +struct InstanceKeys<'x> { + ical: &'x ICalendar, + resolver: Option>, +} + +impl<'x> InstanceKeys<'x> { + fn new(ical: &'x ICalendar) -> Self { + InstanceKeys { + ical, + resolver: None, + } + } + + fn of(&mut self, component: &ICalendarComponent) -> Option { + match self.recurrence_id(component) { + Some(recurrence_id) => { + RecurrenceKey::from_recurrence_id(recurrence_id).map(RecurrenceKey::prefix) + } + None if !component.is_recurrence_override() => Some(BASE_INSTANCE), + None => None, + } + } + + fn recurrence_id(&mut self, component: &ICalendarComponent) -> Option { + if !component.is_recurrence_override() { + return None; + } + let tz = self.start_tz(component); + component + .recurrence_id(tz) + .map(|recurrence_id| recurrence_id.naive) + } + + fn start_tz(&mut self, component: &ICalendarComponent) -> Tz { + let Some(entry) = component.property(&ICalendarProperty::Dtstart) else { + return Tz::Floating; + }; + if let Some(tz) = entry + .values + .first() + .and_then(ICalendarValue::as_partial_date_time) + .and_then(PartialDateTime::to_date_time) + .and_then(|start| start.tz()) + { + return tz; + } + let ical = self.ical; + self.resolver + .get_or_insert_with(|| ical.build_tz_resolver()) + .resolve_or_default(entry.tz_id()) + } +} + +impl<'x> UserDataComponents<'x> { + fn new(ical: &'x ICalendar) -> Self { + let mut ids = ical + .user_data_component_ids() + .into_iter() + .map(|(comp_id, key)| (key, comp_id)) + .collect::>(); + ids.sort_unstable(); + UserDataComponents { ical, ids } + } + + fn contains(&self, key: u32) -> bool { + self.position(key).is_some() + } + + fn counterpart(&self, key: u32, lookup: UserDataLookup) -> Option<&'x ICalendarComponent> { + match lookup { + UserDataLookup::Inherited => { + self.position(key).or_else(|| self.position(BASE_INSTANCE)) + } + UserDataLookup::Exact => self.position(key), + } + .and_then(|index| self.ids.get(index)) + .and_then(|(_, comp_id)| self.ical.component_by_id(*comp_id)) + } + + fn position(&self, key: u32) -> Option { + let index = self + .ids + .partition_point(|(stored_key, _)| *stored_key < key); + self.ids + .get(index) + .filter(|(stored_key, _)| *stored_key == key) + .map(|_| index) + } +} + +struct ComponentAppender<'x> { + source: &'x ICalendar, + first_id: u32, + components: Vec, +} + +impl<'x> ComponentAppender<'x> { + fn new(source: &'x ICalendar, first_id: u32) -> Self { + ComponentAppender { + source, + first_id, + components: Vec::new(), + } + } + + fn push(&mut self, component: ICalendarComponent) -> u32 { + let id = self.first_id + self.components.len() as u32; + self.components.push(component); + id + } + + fn copy_tree(&mut self, id: u32) -> Option { + self.copy_subtree(id, 0) + } + + fn copy_subtree(&mut self, id: u32, depth: usize) -> Option { + let source = self.source; + let component = source + .component_by_id(id) + .filter(|_| depth < MAX_COMPONENT_DEPTH)?; + let copy_id = self.push(component.detached()); + let children = component + .component_ids + .iter() + .filter_map(|child_id| self.copy_subtree(*child_id, depth + 1)) + .collect::>(); + if let Some(copy) = self.components.get_mut((copy_id - self.first_id) as usize) { + copy.component_ids = children; + } + Some(copy_id) + } + + fn into_components(self) -> Vec { + self.components + } +} + +struct OccurrenceOverrides<'x> { + appender: ComponentAppender<'x>, + base: &'x ICalendarComponent, + base_start: i64, + override_ids: Vec, +} + +impl<'x> OccurrenceOverrides<'x> { + fn new(ical: &'x ICalendar, base_id: u32) -> Option { + let base = ical.component_by_id(base_id)?; + let base_start = base + .property(&ICalendarProperty::Dtstart)? + .values + .first()? + .as_partial_date_time()? + .naive_timestamp()?; + Some(OccurrenceOverrides { + appender: ComponentAppender::new(ical, ical.components.len() as u32), + base, + base_start, + override_ids: Vec::new(), + }) + } + + fn push(&mut self, start: i64) { + let (base, source) = (self.base, self.appender.source); + let shift = start - self.base_start; + let mut entries = Vec::with_capacity(base.entries.len() + 1); + for entry in &base.entries { + match entry.name { + ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exrule + | ICalendarProperty::Exdate => {} + ICalendarProperty::Dtstart => { + let Some(start) = entry.shifted(shift) else { + return; + }; + entries.push(ICalendarEntry { + name: ICalendarProperty::RecurrenceId, + params: start + .params + .iter() + .filter(|param| { + matches!( + param.name, + ICalendarParameterName::Tzid | ICalendarParameterName::Value + ) + }) + .cloned() + .collect(), + values: start.values.clone(), + }); + entries.push(start); + } + ICalendarProperty::Dtend | ICalendarProperty::Due => { + entries.push(entry.shifted(shift).unwrap_or_else(|| entry.clone())); + } + _ => entries.push(entry.clone()), + } + } + + let children = base + .component_ids + .iter() + .filter(|id| { + source + .component_by_id(**id) + .is_some_and(|child| child.component_type != ICalendarComponentType::VAlarm) + }) + .filter_map(|id| self.appender.copy_tree(*id)) + .collect(); + let override_id = self.appender.push(ICalendarComponent { + component_type: base.component_type.clone(), + entries, + component_ids: children, + }); + self.override_ids.push(override_id); + } + + fn into_personal_overrides(self, root_id: usize) -> PersonalOverrides { + PersonalOverrides { + root_id, + override_ids: self.override_ids, + components: self.appender.into_components(), + } + } +} + +trait ShiftedEntry { + fn shifted(&self, shift: i64) -> Option; +} + +impl ShiftedEntry for ICalendarEntry { + fn shifted(&self, shift: i64) -> Option { + let [ICalendarValue::PartialDateTime(value)] = self.values.as_slice() else { + return None; + }; + Some(ICalendarEntry { + name: self.name.clone(), + params: self.params.clone(), + values: [ICalendarValue::PartialDateTime(value.shifted(shift)?)].into(), + }) + } +} + diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index 985e494f..d017d135 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -4,25 +4,42 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; +use super::{ + AddressBook, ArchivedAddressBook, ArchivedContactCard, ArchivedContactCardContent, ContactCard, + ContactCardContent, +}; +use crate::{MetaHasher, SizeWriter}; use ahash::AHashSet; use calcard::{ common::IanaString, - vcard::{ArchivedVCardProperty, ArchivedVCardValue, VCardProperty}, + vcard::{ArchivedVCardProperty, ArchivedVCardValue, VCardProperty, VCardValue, VCardVersion}, +}; +use common::storage::{ + dav::PresenceBits, + index::{ + ArchivedSplitObject, IndexItem, IndexValue, IndexableAndSerializableObject, + IndexableObject, SEARCH_HASH_UNCHANGED, SerializableObject, SplitObject, serialize_object, + }, }; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use nlp::language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }; use store::{ search::{ContactSearchField, IndexDocument, SearchField}, - write::{IndexPropertyClass, SearchIndex, ValueClass}, + write::{ArchiveCompression, BatchBuilder, Compression, Dictionary, SearchIndex, Slot}, xxhash_rust::xxh3, }; -use types::{acl::AclGrant, collection::SyncCollection, field::ContactField}; +use types::{ + acl::AclGrant, + collection::SyncCollection, + field::{ContactField, Field}, + metadata::MetadataKinds, +}; use utils::sanitize_email; +const CARD_HASHED_FLAGS: u16 = !PresenceBits::CONTACT_CARD.mask(); + impl IndexableObject for AddressBook { fn index_values(&self) -> impl Iterator> { [ @@ -38,6 +55,10 @@ impl IndexableObject for AddressBook { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + AddressBook::metadata_kinds(self) + } } impl IndexableObject for &ArchivedAddressBook { @@ -60,6 +81,10 @@ impl IndexableObject for &ArchivedAddressBook { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedAddressBook::metadata_kinds(self) + } } impl IndexableAndSerializableObject for AddressBook { @@ -68,31 +93,24 @@ impl IndexableAndSerializableObject for AddressBook { } } +impl SerializableObject for AddressBook { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} + impl IndexableObject for ContactCard { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Index { - field: ContactField::Uid.into(), - value: self.card.uid().into(), - }, IndexValue::Index { field: ContactField::Email.into(), - value: self.emails().next().into(), - }, - IndexValue::Property { - field: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: ContactField::CreatedToUpdated.into(), - value: self.created as u64, - }), - value: self.modified.into(), + value: IndexItem::None, }, IndexValue::SearchIndex { index: SearchIndex::Contacts, - hash: self.hashes().fold(0, |acc, hash| acc ^ hash), - }, - IndexValue::Quota { - used: self.size() as u32, + hash: SEARCH_HASH_UNCHANGED, }, + IndexValue::Quota { used: self.size }, IndexValue::LogItem { sync_collection: SyncCollection::AddressBook, prefix: None, @@ -100,39 +118,19 @@ impl IndexableObject for ContactCard { ] .into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ContactCard::metadata_kinds(self) + } } impl IndexableObject for &ArchivedContactCard { fn index_values(&self) -> impl Iterator> { - [ - IndexValue::Index { - field: ContactField::Uid.into(), - value: self.card.uid().into(), - }, - IndexValue::Index { - field: ContactField::Email.into(), - value: self.emails().next().into(), - }, - IndexValue::Property { - field: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: ContactField::CreatedToUpdated.into(), - value: self.created.to_native() as u64, - }), - value: (self.modified.to_native() as u64).into(), - }, - IndexValue::SearchIndex { - index: SearchIndex::Contacts, - hash: self.hashes().fold(0, |acc, hash| acc ^ hash), - }, - IndexValue::Quota { - used: self.size() as u32, - }, - IndexValue::LogItem { - sync_collection: SyncCollection::AddressBook, - prefix: None, - }, - ] - .into_iter() + self.meta_index_values().into_iter() + } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedContactCard::metadata_kinds(self) } } @@ -140,16 +138,159 @@ impl IndexableAndSerializableObject for ContactCard { fn is_versioned() -> bool { true } + + fn set_pending_id(&mut self, document_id: u32) -> trc::Result<()> { + let content_hash = self.etag ^ self.meta_hash(); + self.names + .last_mut() + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Pending address book id requires a name.") + })? + .parent_id = document_id; + self.etag = content_hash ^ self.meta_hash(); + Ok(()) + } + + fn size_hint(&self) -> usize { + self.meta_size_hint() + } +} + +impl SplitObject for ContactCard { + type Content = ContactCardContent; + type Context = VCardVersion; + + const CONTENT_FIELD: Field = ContactField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher.u32(self.names.len() as u32); + for name in &self.names { + hasher.str(&name.name).u32(name.parent_id); + } + hasher + .str(&self.uid) + .opt_str(self.display_name.as_deref()) + .i64(self.created) + .i64(self.modified) + .u32(self.size) + .u16(self.flags & CARD_HASHED_FLAGS) + .finish() + } + + fn set_etag(&mut self, etag: u32) { + self.etag = etag; + } + + fn etag_value(&self) -> u32 { + self.etag + } + + fn refresh_from_content(&mut self, content: &ContactCardContent, ctx: VCardVersion) { + self.uid = content.card.uid().unwrap_or_default().to_string(); + self.size = SizeWriter::vcard(&content.card, ctx) as u32; + } + + fn full_index_values<'x>(&'x self, content: &'x ContactCardContent) -> Vec> { + vec![ + IndexValue::Index { + field: ContactField::Email.into(), + value: content.emails().next().into(), + }, + IndexValue::SearchIndex { + index: SearchIndex::Contacts, + hash: content.hashes().fold(0, |acc, hash| acc ^ hash), + }, + IndexValue::Quota { used: self.size }, + IndexValue::LogItem { + sync_collection: SyncCollection::AddressBook, + prefix: None, + }, + ] + } +} + +impl ArchivedSplitObject for ArchivedContactCard { + type ArchivedContent = ArchivedContactCardContent; + + const CONTENT_FIELD: Field = ContactField::Content.field(); + + fn meta_hash(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher.u32(self.names.len() as u32); + for name in self.names.iter() { + hasher.str(&name.name).u32(name.parent_id.to_native()); + } + hasher + .str(&self.uid) + .opt_str(self.display_name.as_deref()) + .i64(self.created.to_native()) + .i64(self.modified.to_native()) + .u32(self.size.to_native()) + .u16(self.flags.to_native() & CARD_HASHED_FLAGS) + .finish() + } + + fn etag(&self) -> u32 { + self.etag.to_native() + } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedContactCard::metadata_kinds(self) + } + + fn meta_index_values(&self) -> Vec> { + vec![ + IndexValue::Index { + field: ContactField::Email.into(), + value: IndexItem::None, + }, + IndexValue::SearchIndex { + index: SearchIndex::Contacts, + hash: SEARCH_HASH_UNCHANGED, + }, + IndexValue::Quota { + used: self.size.to_native(), + }, + IndexValue::LogItem { + sync_collection: SyncCollection::AddressBook, + prefix: None, + }, + ] + } + + fn full_index_values<'x>( + &'x self, + content: &'x ArchivedContactCardContent, + ) -> Vec> { + vec![ + IndexValue::Index { + field: ContactField::Email.into(), + value: content.emails().next().into(), + }, + IndexValue::SearchIndex { + index: SearchIndex::Contacts, + hash: content.hashes().fold(0, |acc, hash| acc ^ hash), + }, + IndexValue::Quota { + used: self.size.to_native(), + }, + IndexValue::LogItem { + sync_collection: SyncCollection::AddressBook, + prefix: None, + }, + ] + } } impl AddressBook { pub fn size(&self) -> usize { - self.dead_properties.size() - + self - .preferences - .iter() - .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) - .sum::() + self.preferences + .iter() + .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) + .sum::() + self.name.len() + std::mem::size_of::() } @@ -157,26 +298,25 @@ impl AddressBook { impl ArchivedAddressBook { pub fn size(&self) -> usize { - self.dead_properties.size() - + self - .preferences - .iter() - .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) - .sum::() + self.preferences + .iter() + .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) + .sum::() + self.name.len() + std::mem::size_of::() } } impl ContactCard { - pub fn size(&self) -> usize { - self.dead_properties.size() + pub fn meta_size_hint(&self) -> usize { + self.uid.len() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() - + self.size as usize + std::mem::size_of::() } +} +impl ContactCardContent { pub fn hashes(&self) -> impl Iterator { self.card .entries @@ -200,7 +340,15 @@ impl ContactCard { | VCardProperty::Tel ) }) - .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) + .flat_map(|e| e.values.iter()) + .flat_map(|v| { + let (text, items) = match v { + VCardValue::Component(items) => (None, items.as_slice()), + value => (value.as_text(), Default::default()), + }; + text.into_iter() + .chain(items.iter().map(|item| item.as_str())) + }) .map(|v| xxh3::xxh3_64(v.as_bytes())) } @@ -214,14 +362,15 @@ impl ContactCard { } impl ArchivedContactCard { - pub fn size(&self) -> usize { - self.dead_properties.size() + pub fn meta_size_hint(&self) -> usize { + self.uid.len() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() - + self.size.to_native() as usize + std::mem::size_of::() } +} +impl ArchivedContactCardContent { pub fn hashes(&self) -> impl Iterator { self.card .entries @@ -245,7 +394,15 @@ impl ArchivedContactCard { | ArchivedVCardProperty::Tel ) }) - .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) + .flat_map(|e| e.values.iter()) + .flat_map(|v| { + let (text, items) = match v { + ArchivedVCardValue::Component(items) => (None, items.as_slice()), + value => (value.as_text(), Default::default()), + }; + text.into_iter() + .chain(items.iter().map(|item| item.as_str())) + }) .map(|v| xxh3::xxh3_64(v.as_bytes())) } @@ -271,7 +428,9 @@ impl ArchivedContactCard { for entry in self.card.entries.iter() { let (is_text, is_keyword, field) = match entry.name { - ArchivedVCardProperty::N => (false, false, ContactSearchField::Name), + ArchivedVCardProperty::N | ArchivedVCardProperty::Fn => { + (false, false, ContactSearchField::Name) + } ArchivedVCardProperty::Nickname => (false, false, ContactSearchField::Nickname), ArchivedVCardProperty::Org => (false, false, ContactSearchField::Organization), ArchivedVCardProperty::Email => (false, false, ContactSearchField::Email), @@ -282,7 +441,6 @@ impl ArchivedContactCard { ArchivedVCardProperty::Adr => (false, false, ContactSearchField::Address), ArchivedVCardProperty::Note => (true, false, ContactSearchField::Note), ArchivedVCardProperty::Kind => (false, true, ContactSearchField::Kind), - ArchivedVCardProperty::Uid => (false, true, ContactSearchField::Uid), ArchivedVCardProperty::Member => (false, false, ContactSearchField::Member), _ => continue, }; @@ -340,3 +498,16 @@ impl ArchivedContactCard { document } } + +impl ArchiveCompression for AddressBook { + const COMPRESSION: Compression = Compression::None; +} + +impl ArchiveCompression for ContactCard { + const COMPRESSION: Compression = Compression::None; +} + +impl ArchiveCompression for ContactCardContent { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Contact)); +} + diff --git a/crates/groupware/src/contact/mod.rs b/crates/groupware/src/contact/mod.rs index 4ee17752..a1f9b169 100644 --- a/crates/groupware/src/contact/mod.rs +++ b/crates/groupware/src/contact/mod.rs @@ -7,9 +7,10 @@ pub mod index; pub mod storage; +use crate::metadata::tracked_kinds; use calcard::vcard::VCard; -use common::DavName; -use types::{acl::AclGrant, dead_property::DeadProperty}; +use common::{DavName, storage::dav::PresenceBits}; +use types::{acl::AclGrant, metadata::MetadataKinds}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, @@ -19,7 +20,7 @@ pub struct AddressBook { pub name: String, pub preferences: Vec, pub subscribers: Vec, - pub dead_properties: DeadProperty, + pub presence: u8, pub acls: Vec, pub created: i64, pub modified: i64, @@ -36,20 +37,35 @@ pub struct AddressBookPreferences { pub sort_order: u32, } +pub const CARD_META_DAV: u16 = PresenceBits::CONTACT_CARD.dav(); +pub const CARD_META_JMAP: u16 = PresenceBits::CONTACT_CARD.jmap(); + #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ContactCard { pub names: Vec, + pub uid: String, pub display_name: Option, - pub card: VCard, - pub dead_properties: DeadProperty, pub created: i64, pub modified: i64, pub size: u32, + pub etag: u32, + pub flags: u16, +} + +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] +pub struct ContactCardContent { + pub card: VCard, } impl AddressBook { + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.presence) + } + pub fn preferences(&self, account_id: u32) -> &AddressBookPreferences { if self.preferences.len() == 1 { &self.preferences[0] @@ -81,6 +97,10 @@ impl AddressBook { } impl ArchivedAddressBook { + pub fn metadata_kinds(&self) -> MetadataKinds { + tracked_kinds(self.presence) + } + pub fn preferences(&self, account_id: u32) -> &ArchivedAddressBookPreferences { if self.preferences.len() == 1 { &self.preferences[0] @@ -95,6 +115,14 @@ impl ArchivedAddressBook { } impl ContactCard { + pub fn metadata_kinds(&self) -> MetadataKinds { + PresenceBits::CONTACT_CARD.kinds(self.flags) + } + + pub fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.flags = PresenceBits::CONTACT_CARD.apply(self.flags, kinds); + } + pub fn added_addressbook_ids( &self, prev_data: &ArchivedContactCard, @@ -126,3 +154,9 @@ impl ContactCard { .map(|m| m.parent_id) } } + +impl ArchivedContactCard { + pub fn metadata_kinds(&self) -> MetadataKinds { + PresenceBits::CONTACT_CARD.kinds(self.flags.to_native()) + } +} diff --git a/crates/groupware/src/contact/storage.rs b/crates/groupware/src/contact/storage.rs index 01cfb696..aeebd63e 100644 --- a/crates/groupware/src/contact/storage.rs +++ b/crates/groupware/src/contact/storage.rs @@ -4,77 +4,128 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; +use super::{ + AddressBook, ArchivedAddressBook, ArchivedContactCard, ArchivedContactCardContent, ContactCard, + ContactCardContent, +}; use crate::DestroyArchive; -use common::{Server, auth::AccountTenantIds, storage::index::ObjectIndexBuilder}; +use calcard::vcard::VCardVersion; +use common::{ + Server, + auth::ChangedBy, + storage::{ + batch::MAX_BATCH_DOCUMENTS, + index::{GroupwareWrite, ObjectIndexBuilder, SplitCurrent, SplitUpdate}, + metadata::ContainerCleanup, + }, +}; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder, now}, + write::{Archive, ArchiveBytes, BatchBuilder, PendingId, Slot, now}, }; use trc::AddContext; -use types::collection::{Collection, VanishedCollection}; +use types::{ + collection::{Collection, VanishedCollection}, + field::ContactField, +}; impl ContactCard { - pub fn update<'x>( + #[allow(clippy::too_many_arguments)] + pub fn update_full( self, - changed_by: AccountTenantIds, + content: ContactCardContent, + version: VCardVersion, + changed_by: ChangedBy, card: Archive<&ArchivedContactCard>, + card_content: &ArchivedContactCardContent, account_id: u32, document_id: u32, - batch: &'x mut BatchBuilder, - ) -> trc::Result<&'x mut BatchBuilder> { + parent_id: Option, + batch: &mut BatchBuilder, + ) -> trc::Result { let mut new_card = self; - - // Build card new_card.modified = now() as i64; - // Prepare write batch + let update = SplitUpdate::full(card, card_content, new_card, content, version)?; + let etag = update.etag(); + batch .with_account_id(account_id) .with_collection(Collection::ContactCard) .with_document(document_id) - .custom( - ObjectIndexBuilder::new() - .with_current(card) - .with_changes(new_card) - .with_changed_by(changed_by), - ) - .map(|b| b.commit_point()) + .custom(update.into_builder(changed_by, parent_id))? + .commit_point(); + + Ok(etag) } - pub fn insert( + pub fn update_meta( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, + card: Archive<&ArchivedContactCard>, account_id: u32, document_id: u32, + parent_id: Option, batch: &mut BatchBuilder, - ) -> trc::Result<&mut BatchBuilder> { + ) -> trc::Result { + let mut new_card = self; + new_card.modified = now() as i64; + + let update = SplitUpdate::meta_only(card, new_card); + let etag = update.etag(); + + batch + .with_account_id(account_id) + .with_collection(Collection::ContactCard) + .with_document(document_id) + .custom(update.into_builder(changed_by, parent_id))? + .commit_point(); + + Ok(etag) + } + + #[allow(clippy::too_many_arguments)] + pub fn insert( + self, + content: ContactCardContent, + version: VCardVersion, + changed_by: ChangedBy, + account_id: u32, + document_id: impl Into, + parent_id: Option, + batch: &mut BatchBuilder, + ) -> trc::Result { // Build card let mut card = self; let now = now() as i64; card.modified = now; card.created = now; - // Prepare write batch + let changes = GroupwareWrite::full(card, content, version)?; + let etag = format!("\"{}\"", changes.meta().etag); + batch .with_account_id(account_id) .with_collection(Collection::ContactCard) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() - .with_changes(card) - .with_changed_by(changed_by), - ) - .map(|b| b.commit_point()) + .with_changes(changes) + .with_changed_by(changed_by) + .with_pending_id_opt(parent_id), + )? + .commit_point(); + + Ok(etag) } } impl AddressBook { pub fn insert( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, - document_id: u32, + document_id: impl Into, batch: &mut BatchBuilder, ) -> trc::Result<&mut BatchBuilder> { // Build address book @@ -87,7 +138,7 @@ impl AddressBook { batch .with_account_id(account_id) .with_collection(Collection::AddressBook) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(book) @@ -98,7 +149,7 @@ impl AddressBook { pub fn update<'x>( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, book: Archive<&ArchivedAddressBook>, account_id: u32, document_id: u32, @@ -128,53 +179,107 @@ impl DestroyArchive> { pub async fn delete_with_cards( self, server: &Server, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, document_id: u32, - children_ids: Vec, + mut children_ids: Vec, + cleanup: &ContainerCleanup, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions let addressbook_id = document_id; - for document_id in children_ids { - if let Some(card_) = server - .store() - .get_value::>(ValueKey::archive( + children_ids.sort_unstable(); + children_ids.dedup(); + for chunk in children_ids.chunks(MAX_BATCH_DOCUMENTS) { + let mut archives = server + .archive_chunk( account_id, Collection::ContactCard, - document_id, - )) - .await? - { + chunk, + Some((ContactField::Content.field(), chunk)), + ) + .await + .caused_by(trc::location!())?; + for &document_id in chunk { + let Some(card_) = archives.take_archive(document_id) else { + continue; + }; DestroyArchive( card_ .to_unarchived::() .caused_by(trc::location!())?, ) - .delete( + .remove( + server, changed_by, account_id, document_id, addressbook_id, + archives.take_value(document_id), None, + cleanup, batch, - )?; + ) + .await?; } } - self.delete(changed_by, account_id, document_id, delete_path, batch) + self.delete( + server, + changed_by, + account_id, + document_id, + delete_path, + batch, + ) + .await } - pub fn delete( + pub async fn delete( self, - changed_by: AccountTenantIds, + server: &Server, + changed_by: ChangedBy, account_id: u32, document_id: u32, delete_path: Option, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.delete_with_cleanup( + server, + changed_by, + account_id, + document_id, + delete_path, + &ContainerCleanup::empty(Collection::AddressBook), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn delete_with_cleanup( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + delete_path: Option, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, ) -> trc::Result<()> { let book = self.0; + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + book.inner.metadata_kinds(), + ) + .await?; + // Delete addressbook batch .with_account_id(account_id) @@ -198,14 +303,70 @@ impl DestroyArchive> { } impl DestroyArchive> { - pub fn delete( + #[allow(clippy::too_many_arguments)] + pub async fn delete( self, - changed_by: AccountTenantIds, + server: &Server, + changed_by: ChangedBy, account_id: u32, document_id: u32, addressbook_id: u32, delete_path: Option, batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.remove( + server, + changed_by, + account_id, + document_id, + addressbook_id, + None, + delete_path, + &ContainerCleanup::empty(Collection::ContactCard), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn delete_with_cleanup( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + addressbook_id: u32, + content: Option>, + delete_path: Option, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + self.remove( + server, + changed_by, + account_id, + document_id, + addressbook_id, + content, + delete_path, + cleanup, + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn remove( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + addressbook_id: u32, + content: Option>, + delete_path: Option, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, ) -> trc::Result<()> { let card = self.0; if let Some(delete_idx) = card @@ -224,23 +385,35 @@ impl DestroyArchive> { .deserialize::() .caused_by(trc::location!())?; new_card.names.swap_remove(delete_idx); + let update = SplitUpdate::meta_only(card, new_card); batch .with_document(document_id) - .custom( - ObjectIndexBuilder::new() - .with_changed_by(changed_by) - .with_current(card) - .with_changes(new_card), - ) + .custom(update.into_builder(changed_by, None)) .caused_by(trc::location!())?; } else { // Delete card + let content_ = match content { + Some(content) => Some(content), + None => card_content(server, account_id, document_id).await?, + }; + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + card.inner.metadata_kinds(), + ) + .await?; batch + .with_account_id(account_id) + .with_collection(Collection::ContactCard) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_changed_by(changed_by) - .with_current(card), + .with_current(card_current(card, content_.as_ref())?), ) .caused_by(trc::location!())?; } @@ -255,13 +428,52 @@ impl DestroyArchive> { Ok(()) } - pub fn delete_all( + pub async fn delete_all( self, - changed_by: AccountTenantIds, + server: &Server, + changed_by: ChangedBy, account_id: u32, document_id: u32, batch: &mut BatchBuilder, ) -> trc::Result<()> { + self.remove_all( + server, + changed_by, + account_id, + document_id, + None, + &ContainerCleanup::empty(Collection::ContactCard), + batch, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn remove_all( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + content: Option>, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let content_ = match content { + Some(content) => Some(content), + None => card_content(server, account_id, document_id).await?, + }; + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + self.0.inner.metadata_kinds(), + ) + .await?; + batch .with_account_id(account_id) .with_collection(Collection::ContactCard) @@ -269,7 +481,7 @@ impl DestroyArchive> { .custom( ObjectIndexBuilder::<_, ()>::new() .with_changed_by(changed_by) - .with_current(self.0), + .with_current(card_current(self.0, content_.as_ref())?), ) .caused_by(trc::location!()) .map(|b| { @@ -277,3 +489,36 @@ impl DestroyArchive> { }) } } + +async fn card_content( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result>> { + server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::ContactCard, + document_id, + ContactField::Content, + )) + .await + .caused_by(trc::location!()) +} + +fn card_current<'x>( + card: Archive<&'x ArchivedContactCard>, + content: Option<&'x Archive>, +) -> trc::Result> { + match content { + Some(content) => content + .to_unarchived::() + .caused_by(trc::location!()) + .map(|content| SplitCurrent::Full { + meta: card, + content: content.inner, + }), + None => Ok(SplitCurrent::MetaOnly(card)), + } +} diff --git a/crates/groupware/src/file/content.rs b/crates/groupware/src/file/content.rs new file mode 100644 index 00000000..056fd0cb --- /dev/null +++ b/crates/groupware/src/file/content.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use text_extract::Hints; +use types::media_type::media_type_essence; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum FileContentKind { + Document = 1, + Html = 2, + PlainText = 3, +} + +impl FileContentKind { + pub fn detect(file_name: &str, media_type: Option<&str>) -> Option { + let essence = media_type.and_then(media_type_essence); + let parts = essence + .as_deref() + .and_then(|essence| essence.split_once('/')); + match parts { + Some(("text", "html")) => return Some(FileContentKind::Html), + Some(("text", subtype)) + if Hints::new() + .with_media_type_parts("text", subtype) + .format() + .is_some() => + { + return Some(FileContentKind::Document); + } + Some(("text", _)) => return Some(FileContentKind::PlainText), + _ => {} + } + let mut hints = Hints::new().with_file_name(file_name); + if let Some(media_type) = media_type { + hints = hints.with_media_type(media_type); + } + if hints.format().is_some() { + return Some(FileContentKind::Document); + } + match parts { + Some(("application", "xhtml+xml")) => Some(FileContentKind::Html), + None | Some(("application", "octet-stream")) => Self::from_extension(file_name), + Some(("application", sub)) + if matches!( + sub, + "json" | "xml" | "javascript" | "x-sh" | "x-yaml" | "yaml" | "toml" + ) || sub.ends_with("+json") + || sub.ends_with("+xml") => + { + Some(FileContentKind::PlainText) + } + _ => None, + } + } + + fn from_extension(file_name: &str) -> Option { + let (_, extension) = file_name.rsplit_once('.')?; + hashify::map_ignore_case!(extension.as_bytes(), FileContentKind, + b"html" => FileContentKind::Html, + b"htm" => FileContentKind::Html, + b"xhtml" => FileContentKind::Html, + b"txt" => FileContentKind::PlainText, + b"text" => FileContentKind::PlainText, + b"md" => FileContentKind::PlainText, + b"markdown" => FileContentKind::PlainText, + b"rst" => FileContentKind::PlainText, + b"csv" => FileContentKind::PlainText, + b"tsv" => FileContentKind::PlainText, + b"log" => FileContentKind::PlainText, + b"json" => FileContentKind::PlainText, + b"xml" => FileContentKind::PlainText, + b"yaml" => FileContentKind::PlainText, + b"yml" => FileContentKind::PlainText, + b"toml" => FileContentKind::PlainText, + b"ini" => FileContentKind::PlainText, + b"conf" => FileContentKind::PlainText, + b"cfg" => FileContentKind::PlainText, + b"tex" => FileContentKind::PlainText, + b"srt" => FileContentKind::PlainText, + b"vtt" => FileContentKind::PlainText, + b"sql" => FileContentKind::PlainText, + b"sh" => FileContentKind::PlainText, + b"py" => FileContentKind::PlainText, + b"rs" => FileContentKind::PlainText, + b"js" => FileContentKind::PlainText, + b"ts" => FileContentKind::PlainText, + b"css" => FileContentKind::PlainText, + b"c" => FileContentKind::PlainText, + b"h" => FileContentKind::PlainText, + b"cpp" => FileContentKind::PlainText, + b"java" => FileContentKind::PlainText, + b"go" => FileContentKind::PlainText, + b"rb" => FileContentKind::PlainText, + b"php" => FileContentKind::PlainText, + ) + .copied() + } + + pub fn index_hash(kind: Option, blob_hash: &[u8]) -> u64 { + blob_hash + .first_chunk::<8>() + .map_or(0, |bytes| u64::from_le_bytes(*bytes)) + ^ ((kind.map_or(0, |kind| kind as u64)) << 56) + } +} + diff --git a/crates/groupware/src/file/index.rs b/crates/groupware/src/file/index.rs index 7487df65..fad96c53 100644 --- a/crates/groupware/src/file/index.rs +++ b/crates/groupware/src/file/index.rs @@ -4,9 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ArchivedFileNode, FileNode}; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; -use types::{acl::AclGrant, collection::SyncCollection}; +use super::{ + ArchivedFileNode, ArchivedFileNodeContent, FileNode, FileNodeContent, content::FileContentKind, +}; +use common::storage::index::{ + IndexValue, IndexableAndSerializableObject, IndexableObject, SerializableObject, + serialize_object, +}; +use store::write::{ArchiveCompression, BatchBuilder, Compression, SearchIndex, Slot}; +use types::{acl::AclGrant, collection::SyncCollection, metadata::MetadataKinds}; impl IndexableObject for FileNode { fn index_values(&self) -> impl Iterator> { @@ -25,14 +31,28 @@ impl IndexableObject for FileNode { }, ]); - if let Some(file) = &self.file { - values.extend([IndexValue::Blob { - value: file.blob_hash.clone(), - }]); + if let Some(file) = self.file() { + values.extend([ + IndexValue::SearchIndex { + index: SearchIndex::File, + hash: FileContentKind::index_hash( + FileContentKind::detect(&self.name, file.media_type.as_deref()), + file.blob_hash.as_slice(), + ), + }, + IndexValue::Blob { + value: file.blob_hash.clone(), + size: u64::from(file.size), + }, + ]); } values.into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + FileNode::metadata_kinds(self) + } } impl IndexableObject for &ArchivedFileNode { @@ -57,41 +77,88 @@ impl IndexableObject for &ArchivedFileNode { }, ]); - if let Some(file) = self.file.as_ref() { - values.extend([IndexValue::Blob { - value: (&file.blob_hash).into(), - }]); + if let Some(file) = self.file() { + values.extend([ + IndexValue::SearchIndex { + index: SearchIndex::File, + hash: FileContentKind::index_hash( + FileContentKind::detect(&self.name, file.media_type.as_deref()), + file.blob_hash.0.as_slice(), + ), + }, + IndexValue::Blob { + value: (&file.blob_hash).into(), + size: u64::from(file.size.to_native()), + }, + ]); } values.into_iter() } + + fn metadata_kinds(&self) -> MetadataKinds { + ArchivedFileNode::metadata_kinds(self) + } } impl IndexableAndSerializableObject for FileNode { fn is_versioned() -> bool { true } + + fn set_pending_id(&mut self, document_id: u32) -> trc::Result<()> { + self.parent_id = document_id + 1; + self.etag = self.compute_etag(); + Ok(()) + } + + fn size_hint(&self) -> usize { + self.size() + } } impl FileNode { pub fn size(&self) -> usize { - self.dead_properties.size() - + self.display_name.as_ref().map_or(0, |n| n.len()) - + self.name.len() - + self.file.as_ref().map_or(0, |f| f.size as usize) + self.name.len() + + match &self.content { + FileNodeContent::Directory => 0, + FileNodeContent::File(file) => { + file.size as usize + file.media_type.as_ref().map_or(0, |t| t.len()) + } + FileNodeContent::Symlink(target) => target.len(), + } + std::mem::size_of::() } } impl ArchivedFileNode { pub fn size(&self) -> usize { - self.dead_properties.size() - + self.display_name.as_ref().map_or(0, |n| n.len()) - + self.name.len() - + self - .file - .as_ref() - .map_or(0, |f| f.size.to_native() as usize) + self.name.len() + + match &self.content { + ArchivedFileNodeContent::Directory => 0, + ArchivedFileNodeContent::File(file) => { + file.size.to_native() as usize + file.media_type.as_ref().map_or(0, |t| t.len()) + } + ArchivedFileNodeContent::Symlink(target) => target.len(), + } + std::mem::size_of::() } } + +impl ArchiveCompression for FileNode { + const COMPRESSION: Compression = Compression::None; +} + +impl SerializableObject for FileNode { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + let mut node = self; + node.etag = node.compute_etag(); + let etag = node.etag; + serialize_object(node, batch, pending_id)?; + if pending_id.is_none() { + batch.set_archive_hash(Some(etag)); + } + Ok(()) + } +} + diff --git a/crates/groupware/src/file/mod.rs b/crates/groupware/src/file/mod.rs index 8d000cfe..90248b0c 100644 --- a/crates/groupware/src/file/mod.rs +++ b/crates/groupware/src/file/mod.rs @@ -4,10 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod content; pub mod index; pub mod storage; +pub mod symlink; -use types::{acl::AclGrant, blob_hash::BlobHash, dead_property::DeadProperty}; +use crate::MetaHasher; +use common::storage::dav::{FILE_KIND_DIRECTORY, FILE_KIND_FILE, FILE_KIND_SYMLINK, FilePresence}; +use std::{fmt::Display, str::FromStr}; +use types::{acl::AclGrant, blob_hash::BlobHash, metadata::MetadataKinds}; + +const ROLE_SHIFT: u32 = 3; +const ROLE_MASK: u16 = 0xF << ROLE_SHIFT; + +const _: () = assert!(FilePresence::from_bits(u16::MAX).bits() & ROLE_MASK == 0); #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, @@ -16,14 +26,28 @@ use types::{acl::AclGrant, blob_hash::BlobHash, dead_property::DeadProperty}; pub struct FileNode { pub parent_id: u32, pub name: String, - pub display_name: Option, - pub file: Option, + pub content: FileNodeContent, + pub flags: u16, + pub etag: u32, pub created: i64, pub modified: i64, - pub dead_properties: DeadProperty, + pub accessed: i64, + pub changed: i64, + pub unsubscribed: Vec, pub acls: Vec, } +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] +#[rkyv(derive(Debug))] +pub enum FileNodeContent { + #[default] + Directory, + File(FileProperties), + Symlink(String), +} + #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] @@ -34,3 +58,263 @@ pub struct FileProperties { pub media_type: Option, pub executable: bool, } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(u8)] +pub enum FileNodeRole { + Root = 1, + Home = 2, + Temp = 3, + Trash = 4, + Documents = 5, + Downloads = 6, + Music = 7, + Pictures = 8, + Videos = 9, +} + +impl FileNode { + #[inline(always)] + pub fn file(&self) -> Option<&FileProperties> { + match &self.content { + FileNodeContent::File(file) => Some(file), + _ => None, + } + } + + #[inline(always)] + pub fn file_mut(&mut self) -> Option<&mut FileProperties> { + match &mut self.content { + FileNodeContent::File(file) => Some(file), + _ => None, + } + } + + #[inline(always)] + pub fn is_directory(&self) -> bool { + matches!(self.content, FileNodeContent::Directory) + } + + #[inline(always)] + pub fn symlink_target(&self) -> Option<&str> { + match &self.content { + FileNodeContent::Symlink(target) => Some(target), + _ => None, + } + } + + #[inline(always)] + pub fn kind_id(&self) -> u8 { + self.content.kind_id() + } + + pub fn is_subscribed(&self, account_id: u32) -> bool { + !self.unsubscribed.contains(&account_id) + } + + pub fn set_subscribed(&mut self, account_id: u32, subscribed: bool) { + if subscribed { + self.unsubscribed.retain(|id| *id != account_id); + } else if !self.unsubscribed.contains(&account_id) { + self.unsubscribed.push(account_id); + } + } + + pub fn presence(&self) -> FilePresence { + FilePresence::from_bits(self.flags) + } + + pub fn set_presence(&mut self, presence: FilePresence) { + self.flags = presence.apply(self.flags); + } + + pub fn role(&self) -> Option { + Self::role_of(self.flags) + } + + pub fn set_role(&mut self, role: Option) { + self.flags = + (self.flags & !ROLE_MASK) | (u16::from(role.map_or(0, FileNodeRole::id)) << ROLE_SHIFT); + } + + fn role_of(flags: u16) -> Option { + FileNodeRole::from_id(((flags & ROLE_MASK) >> ROLE_SHIFT) as u8) + } + + pub fn metadata_kinds(&self) -> MetadataKinds { + self.presence().kinds() + } + + pub fn compute_etag(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher + .u8(self.kind_id()) + .u32(self.parent_id) + .str(&self.name); + match &self.content { + FileNodeContent::Directory => {} + FileNodeContent::File(file) => { + hasher + .bytes(file.blob_hash.as_slice()) + .u32(file.size) + .opt_str(file.media_type.as_deref()) + .u8(u8::from(file.executable)); + } + FileNodeContent::Symlink(target) => { + hasher.str(target); + } + } + hasher.u8(self.role().map_or(0, FileNodeRole::id)).finish() + } +} + +impl FileNodeContent { + #[inline(always)] + pub fn kind_id(&self) -> u8 { + match self { + FileNodeContent::Directory => FILE_KIND_DIRECTORY, + FileNodeContent::File(_) => FILE_KIND_FILE, + FileNodeContent::Symlink(_) => FILE_KIND_SYMLINK, + } + } +} + +impl ArchivedFileNode { + #[inline(always)] + pub fn file(&self) -> Option<&ArchivedFileProperties> { + match &self.content { + ArchivedFileNodeContent::File(file) => Some(file), + _ => None, + } + } + + #[inline(always)] + pub fn is_directory(&self) -> bool { + matches!(self.content, ArchivedFileNodeContent::Directory) + } + + #[inline(always)] + pub fn symlink_target(&self) -> Option<&str> { + match &self.content { + ArchivedFileNodeContent::Symlink(target) => Some(target.as_str()), + _ => None, + } + } + + #[inline(always)] + pub fn kind_id(&self) -> u8 { + match &self.content { + ArchivedFileNodeContent::Directory => FILE_KIND_DIRECTORY, + ArchivedFileNodeContent::File(_) => FILE_KIND_FILE, + ArchivedFileNodeContent::Symlink(_) => FILE_KIND_SYMLINK, + } + } + + #[inline(always)] + pub fn role(&self) -> Option { + FileNode::role_of(self.flags.to_native()) + } + + pub fn is_subscribed(&self, account_id: u32) -> bool { + !self + .unsubscribed + .iter() + .any(|id| id.to_native() == account_id) + } + + pub fn presence(&self) -> FilePresence { + FilePresence::from_bits(self.flags.to_native()) + } + + pub fn metadata_kinds(&self) -> MetadataKinds { + self.presence().kinds() + } + + pub fn compute_etag(&self) -> u32 { + let mut hasher = MetaHasher::new(); + hasher + .u8(self.kind_id()) + .u32(self.parent_id.to_native()) + .str(&self.name); + match &self.content { + ArchivedFileNodeContent::Directory => {} + ArchivedFileNodeContent::File(file) => { + hasher + .bytes(file.blob_hash.0.as_slice()) + .u32(file.size.to_native()) + .opt_str(file.media_type.as_deref()) + .u8(u8::from(file.executable)); + } + ArchivedFileNodeContent::Symlink(target) => { + hasher.str(target); + } + } + hasher.u8(self.role().map_or(0, FileNodeRole::id)).finish() + } +} + +impl FileNodeRole { + pub fn as_str(&self) -> &'static str { + match self { + FileNodeRole::Root => "root", + FileNodeRole::Home => "home", + FileNodeRole::Temp => "temp", + FileNodeRole::Trash => "trash", + FileNodeRole::Documents => "documents", + FileNodeRole::Downloads => "downloads", + FileNodeRole::Music => "music", + FileNodeRole::Pictures => "pictures", + FileNodeRole::Videos => "videos", + } + } + + pub fn parse(value: &str) -> Option { + hashify::map!(value.as_bytes(), FileNodeRole, + b"root" => FileNodeRole::Root, + b"home" => FileNodeRole::Home, + b"temp" => FileNodeRole::Temp, + b"trash" => FileNodeRole::Trash, + b"documents" => FileNodeRole::Documents, + b"downloads" => FileNodeRole::Downloads, + b"music" => FileNodeRole::Music, + b"pictures" => FileNodeRole::Pictures, + b"videos" => FileNodeRole::Videos, + ) + .copied() + } + + #[inline(always)] + pub fn id(self) -> u8 { + self as u8 + } + + pub fn from_id(id: u8) -> Option { + match id { + 1 => Some(FileNodeRole::Root), + 2 => Some(FileNodeRole::Home), + 3 => Some(FileNodeRole::Temp), + 4 => Some(FileNodeRole::Trash), + 5 => Some(FileNodeRole::Documents), + 6 => Some(FileNodeRole::Downloads), + 7 => Some(FileNodeRole::Music), + 8 => Some(FileNodeRole::Pictures), + 9 => Some(FileNodeRole::Videos), + _ => None, + } + } +} + +impl FromStr for FileNodeRole { + type Err = (); + + fn from_str(s: &str) -> Result { + FileNodeRole::parse(s).ok_or(()) + } +} + +impl Display for FileNodeRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + diff --git a/crates/groupware/src/file/storage.rs b/crates/groupware/src/file/storage.rs index a8f468d6..e20f6b96 100644 --- a/crates/groupware/src/file/storage.rs +++ b/crates/groupware/src/file/storage.rs @@ -6,38 +6,56 @@ use super::{ArchivedFileNode, FileNode}; use crate::DestroyArchive; -use common::{Server, auth::AccountTenantIds, storage::index::ObjectIndexBuilder}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive, BatchBuilder, now}, +use common::{ + Server, + auth::ChangedBy, + storage::{batch::MAX_BATCH_DOCUMENTS, index::ObjectIndexBuilder, metadata::ContainerCleanup}, }; +use store::write::{Archive, BatchBuilder, PendingId, Slot, now}; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; impl FileNode { + pub fn stamp_insert(&mut self, set_created: bool, set_modified: bool, set_accessed: bool) { + let now = now() as i64; + if set_created { + self.created = now; + } + if set_modified { + self.modified = now; + } + if set_accessed { + self.accessed = now; + } + self.changed = now; + } + + pub fn stamp_update(&mut self, set_modified: bool) { + let now = now() as i64; + if set_modified { + self.modified = now; + } + self.changed = now; + } + pub fn insert( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, - document_id: u32, + document_id: impl Into, set_created: bool, set_modified: bool, batch: &mut BatchBuilder, ) -> trc::Result<&mut BatchBuilder> { let mut node = self; - let now = now() as i64; - if set_created { - node.created = now; - } - if set_modified { - node.modified = now; - } + let set_accessed = node.accessed == 0; + node.stamp_insert(set_created, set_modified, set_accessed); // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .with_document(document_id) + .with_pending_document(document_id.into()) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) @@ -46,9 +64,30 @@ impl FileNode { .map(|b| b.commit_point()) } + pub fn insert_stamped( + self, + changed_by: ChangedBy, + account_id: u32, + document_id: Slot, + parent_id: Option, + batch: &mut BatchBuilder, + ) -> trc::Result<&mut BatchBuilder> { + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode) + .create_document(document_id) + .custom( + ObjectIndexBuilder::<(), _>::new() + .with_changes(self) + .with_changed_by(changed_by) + .with_pending_id_opt(parent_id), + ) + .map(|b| b.commit_point()) + } + pub fn update<'x>( self, - changed_by: AccountTenantIds, + changed_by: ChangedBy, node: Archive<&ArchivedFileNode>, account_id: u32, document_id: u32, @@ -56,9 +95,7 @@ impl FileNode { batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { let mut new_node = self; - if set_modified { - new_node.modified = now() as i64; - } + new_node.stamp_update(set_modified); batch .with_account_id(account_id) .with_collection(Collection::FileNode) @@ -71,17 +108,73 @@ impl FileNode { ) .map(|b| b.commit_point()) } + + pub fn update_stamped<'x>( + self, + changed_by: ChangedBy, + node: Archive<&ArchivedFileNode>, + account_id: u32, + document_id: u32, + parent_id: Option, + batch: &'x mut BatchBuilder, + ) -> trc::Result<&'x mut BatchBuilder> { + let builder = ObjectIndexBuilder::new() + .with_current(node) + .with_changes(self) + .with_changed_by(changed_by); + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode) + .with_document(document_id) + .custom(builder.with_pending_id_opt(parent_id)) + .map(|b| b.commit_point()) + } } impl DestroyArchive> { - pub fn delete( + pub async fn delete( self, - changed_by: AccountTenantIds, + server: &Server, + changed_by: ChangedBy, account_id: u32, document_id: u32, batch: &mut BatchBuilder, path: String, ) -> trc::Result<()> { + self.delete_with_cleanup( + server, + changed_by, + account_id, + document_id, + batch, + path, + &ContainerCleanup::empty(Collection::FileNode), + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn delete_with_cleanup( + self, + server: &Server, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + path: String, + cleanup: &ContainerCleanup, + ) -> trc::Result<()> { + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + self.0.inner.metadata_kinds(), + ) + .await?; + // Prepare write batch batch .with_account_id(account_id) @@ -102,14 +195,22 @@ impl DestroyArchive> { pub async fn delete( self, server: &Server, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, + cleanup: &ContainerCleanup, delete_path: Option, ) -> trc::Result<()> { // Process deletions let mut batch = BatchBuilder::new(); - self.delete_batch(server, changed_by, account_id, delete_path, &mut batch) - .await?; + self.delete_batch( + server, + changed_by, + account_id, + cleanup, + delete_path, + &mut batch, + ) + .await?; // Write changes if !batch.is_empty() { server @@ -124,8 +225,9 @@ impl DestroyArchive> { pub async fn delete_batch( self, server: &Server, - changed_by: AccountTenantIds, + changed_by: ChangedBy, account_id: u32, + cleanup: &ContainerCleanup, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { @@ -133,31 +235,47 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::FileNode); - for document_id in self.0 { - if let Some(node) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::FileNode, - document_id, - )) - .await? - { + let mut document_ids = self.0; + document_ids.sort_unstable(); + document_ids.dedup(); + for chunk in document_ids.chunks(MAX_BATCH_DOCUMENTS) { + let mut archives = server + .archive_chunk(account_id, Collection::FileNode, chunk, None) + .await + .caused_by(trc::location!())?; + for &document_id in chunk { + let Some(node_) = archives.take_archive(document_id) else { + continue; + }; + let node = node_ + .to_unarchived::() + .caused_by(trc::location!())?; + server + .release_or_read( + cleanup, + batch, + changed_by, + account_id, + document_id, + node.inner.metadata_kinds(), + ) + .await?; + // Delete record batch .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_changed_by(changed_by) - .with_current( - node.to_unarchived::() - .caused_by(trc::location!())?, - ), + .with_current(node), ) .caused_by(trc::location!())? .commit_point(); } } + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode); if !batch.is_empty() && let Some(delete_path) = delete_path diff --git a/crates/groupware/src/file/symlink.rs b/crates/groupware/src/file/symlink.rs new file mode 100644 index 00000000..86652145 --- /dev/null +++ b/crates/groupware/src/file/symlink.rs @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub const MAX_SYMLINK_TARGET_LEN: usize = 4096; +const SEPARATOR: char = '/'; +const ROOT: &str = "/"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SymlinkTargetError { + Empty, + EmptyElement, + InvalidCharacter, + TooLong, +} + +#[derive(Default)] +pub struct SymlinkTargetBuilder { + target: String, + elements: usize, +} + +impl SymlinkTargetBuilder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + target: String::with_capacity(capacity), + elements: 0, + } + } + + pub fn push(&mut self, element: &str) -> Result<(), SymlinkTargetError> { + if element.contains([SEPARATOR, '\0']) { + return Err(SymlinkTargetError::InvalidCharacter); + } else if element.is_empty() && self.elements > 0 { + return Err(SymlinkTargetError::EmptyElement); + } + if self.elements > 0 { + self.target.push(SEPARATOR); + } + self.target.push_str(element); + self.elements += 1; + if self.target.len() > MAX_SYMLINK_TARGET_LEN { + Err(SymlinkTargetError::TooLong) + } else { + Ok(()) + } + } + + pub fn build(mut self) -> Result { + match self.elements { + 0 => Err(SymlinkTargetError::Empty), + 1 if self.target.is_empty() => { + self.target.push(SEPARATOR); + Ok(self.target) + } + _ => Ok(self.target), + } + } +} + +pub fn symlink_target_elements(target: &str) -> impl Iterator { + let (root, rest) = if target == ROOT { + (Some(""), None) + } else { + (None, Some(target.split(SEPARATOR))) + }; + root.into_iter().chain(rest.into_iter().flatten()) +} + +impl SymlinkTargetError { + pub fn description(&self) -> &'static str { + match self { + SymlinkTargetError::Empty => "target must contain at least one element.", + SymlinkTargetError::EmptyElement => "only the first target element may be empty.", + SymlinkTargetError::InvalidCharacter => { + "target elements must not contain '/' or U+0000." + } + SymlinkTargetError::TooLong => "target is too long.", + } + } +} + diff --git a/crates/groupware/src/lib.rs b/crates/groupware/src/lib.rs index 70b26d35..c9b661c1 100644 --- a/crates/groupware/src/lib.rs +++ b/crates/groupware/src/lib.rs @@ -7,17 +7,24 @@ #![warn(clippy::large_futures)] use calcard::common::timezone::Tz; -use common::DavResources; -use percent_encoding::{AsciiSet, CONTROLS, percent_decode_str, utf8_percent_encode}; +use common::GroupwareResources; +pub use common::storage::dav::RFC_3986; +use percent_encoding::percent_decode_str; use std::borrow::Cow; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + metadata::MetadataKinds, +}; pub mod cache; pub mod calendar; pub mod contact; pub mod file; +pub mod metadata; pub mod scheduling; +pub const METADATA_KINDS: MetadataKinds = MetadataKinds::JMAP.union(MetadataKinds::DAV); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DavResourceName { Card, @@ -27,84 +34,95 @@ pub enum DavResourceName { Scheduling, } -pub const RFC_3986: &AsciiSet = &CONTROLS - .add(b' ') - .add(b'!') - .add(b'"') - .add(b'#') - .add(b'$') - .add(b'%') - .add(b'&') - .add(b'\'') - .add(b'(') - .add(b')') - .add(b'*') - .add(b'+') - .add(b',') - .add(b'/') - .add(b':') - .add(b';') - .add(b'<') - .add(b'=') - .add(b'>') - .add(b'?') - .add(b'@') - .add(b'[') - .add(b'\\') - .add(b']') - .add(b'^') - .add(b'`') - .add(b'{') - .add(b'|') - .add(b'}'); +pub struct SizeWriter(usize); -fn is_pchar(byte: u8) -> bool { - matches!(byte, - b'A'..=b'Z' - | b'a'..=b'z' - | b'0'..=b'9' - | b'-' - | b'.' - | b'_' - | b'~' - | b'!' - | b'$' - | b'&' - | b'\'' - | b'(' - | b')' - | b'*' - | b'+' - | b',' - | b';' - | b'=' - | b':' - | b'@') +impl std::fmt::Write for SizeWriter { + fn write_str(&mut self, text: &str) -> std::fmt::Result { + self.0 += text.len(); + Ok(()) + } } -pub fn is_uri_segment(name: &str) -> bool { - let mut bytes = name.as_bytes().iter(); +impl SizeWriter { + pub fn ical(ical: &calcard::icalendar::ICalendar) -> usize { + let mut writer = SizeWriter(0); + let _ = ical.write_to(&mut writer); + writer.0 + } - while let Some(&byte) = bytes.next() { - if byte == b'%' { - if !bytes.next().is_some_and(u8::is_ascii_hexdigit) - || !bytes.next().is_some_and(u8::is_ascii_hexdigit) - { - return false; + pub fn vcard(vcard: &calcard::vcard::VCard, version: calcard::vcard::VCardVersion) -> usize { + let mut writer = SizeWriter(0); + let _ = vcard.write_to(&mut writer, version); + writer.0 + } +} + +#[derive(Default)] +pub struct MetaHasher(store::xxhash_rust::xxh3::Xxh3); + +impl MetaHasher { + pub fn new() -> Self { + Self::default() + } + + pub fn bytes(&mut self, value: &[u8]) -> &mut Self { + self.0.update(&(value.len() as u64).to_le_bytes()); + self.0.update(value); + self + } + + pub fn str(&mut self, value: &str) -> &mut Self { + self.bytes(value.as_bytes()) + } + + pub fn opt_str(&mut self, value: Option<&str>) -> &mut Self { + match value { + Some(value) => { + self.0.update(&[1]); + self.str(value) + } + None => { + self.0.update(&[0]); + self } - } else if !is_pchar(byte) { - return false; } } - true -} + pub fn u8(&mut self, value: u8) -> &mut Self { + self.0.update(&[value]); + self + } -pub fn encode_path_segment(name: &str) -> Cow<'_, str> { - if is_uri_segment(name) { - Cow::Borrowed(name) - } else { - utf8_percent_encode(name, RFC_3986).into() + pub fn u16(&mut self, value: u16) -> &mut Self { + self.0.update(&value.to_le_bytes()); + self + } + + pub fn u32(&mut self, value: u32) -> &mut Self { + self.0.update(&value.to_le_bytes()); + self + } + + pub fn i64(&mut self, value: i64) -> &mut Self { + self.0.update(&value.to_le_bytes()); + self + } + + pub fn opt_u32(&mut self, value: Option) -> &mut Self { + match value { + Some(value) => { + self.0.update(&[1]); + self.u32(value) + } + None => { + self.0.update(&[0]); + self + } + } + } + + pub fn finish(&self) -> u32 { + self.0.digest() as u32 } } @@ -112,13 +130,14 @@ pub struct DestroyArchive(pub T); impl DavResourceName { pub fn parse(service: &str) -> Option { - hashify::tiny_map!(service.as_bytes(), + hashify::map!(service.as_bytes(), DavResourceName, "card" => DavResourceName::Card, "cal" => DavResourceName::Cal, "file" => DavResourceName::File, "pal" => DavResourceName::Principal, "itip" => DavResourceName::Scheduling, ) + .copied() } pub fn base_path(&self) -> &'static str { @@ -193,11 +212,10 @@ pub trait DavCalendarResource { fn calendar_default_tz(&self, calendar_id: u32, account_id: u32) -> Option; } -impl DavCalendarResource for DavResources { +impl DavCalendarResource for GroupwareResources { fn calendar_default_tz(&self, calendar_id: u32, account_id: u32) -> Option { self.container_resource_by_id(calendar_id) - .and_then(|c| c.calendar_preferences(account_id)) - .map(|p| p.tz) + .and_then(|c| c.calendar_preferences(account_id).map(|p| p.tz)) } } @@ -226,62 +244,3 @@ pub fn extract_addr_spec(value: &str) -> Option<&str> { .map(|(addr, _)| addr.trim()) .filter(|addr| !addr.is_empty()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn path_segments_from_uris_are_preserved() { - for name in [ - "readme.txt", - "My%20Folder", - "%C3%9Cnterlagen.txt", - "file(1).txt", - "a+b.txt", - "Q&A.txt", - "it's.txt", - "mail@host.txt", - "a:b.txt", - "notes;v=2,rev=3!$*=.txt", - "~backup_1-2.txt", - ] { - assert!(is_uri_segment(name), "{name:?}"); - assert_eq!(encode_path_segment(name), name); - } - } - - #[test] - fn path_segments_from_names_are_encoded() { - for (name, expected) in [ - ("My Folder", "My%20Folder"), - ("Ünterlagen.txt", "%C3%9Cnterlagen.txt"), - ("Ünterlagen 2026.txt", "%C3%9Cnterlagen%202026.txt"), - ("100%", "100%25"), - ("100%2", "100%252"), - ("100%zz", "100%25zz"), - ("a/b.txt", "a%2Fb.txt"), - ("ac.txt", "a%3Cb%3Ec.txt"), - ("a\"b#c?d.txt", "a%22b%23c%3Fd.txt"), - ("a\tb.txt", "a%09b.txt"), - ] { - assert!(!is_uri_segment(name), "{name:?}"); - assert_eq!(encode_path_segment(name), expected, "{name:?}"); - } - } - - #[test] - fn encoded_path_segments_are_stable() { - for name in [ - "My Folder", - "Ünterlagen 2026.txt", - "100%", - "a/b.txt", - "file(1).txt", - ] { - let encoded = encode_path_segment(name).into_owned(); - assert!(is_uri_segment(&encoded), "{encoded:?}"); - assert_eq!(encode_path_segment(&encoded), encoded, "{name:?}"); - } - } -} diff --git a/crates/groupware/src/metadata/mod.rs b/crates/groupware/src/metadata/mod.rs new file mode 100644 index 00000000..42f1e14d --- /dev/null +++ b/crates/groupware/src/metadata/mod.rs @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + METADATA_KINDS, + calendar::{ArchivedCalendarEvent, Calendar, CalendarEvent}, + contact::{AddressBook, ArchivedContactCard, ContactCard}, + file::{ArchivedFileNode, FileNode}, +}; +use common::storage::{ + dav::FilePresence, + index::{GroupwareWrite, PresenceFlags, RewritePresence, rewrite_archive}, +}; +use store::write::{Archive, BatchBuilder}; +use trc::AddContext; +use types::{collection::Collection, metadata::MetadataKinds}; + +impl PresenceFlags for Calendar { + const COLLECTION: Collection = Collection::Calendar; + const TRACKED: MetadataKinds = METADATA_KINDS; + + fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.presence = kinds.bits() & Self::TRACKED.bits(); + } +} + +impl PresenceFlags for AddressBook { + const COLLECTION: Collection = Collection::AddressBook; + const TRACKED: MetadataKinds = METADATA_KINDS; + + fn set_metadata_kinds(&mut self, kinds: MetadataKinds) { + self.presence = kinds.bits() & Self::TRACKED.bits(); + } +} + +impl RewritePresence for CalendarEvent { + fn rewrite_presence( + current: &Archive<&ArchivedCalendarEvent>, + kinds: MetadataKinds, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if current.inner.metadata_kinds() == tracked_kinds(kinds.bits()) { + return Ok(()); + } + let mut event = current + .deserialize::() + .caused_by(trc::location!())?; + event.set_metadata_kinds(kinds); + rewrite_archive( + current, + GroupwareWrite::meta_only(event, current.inner), + Collection::CalendarEvent, + account_id, + document_id, + batch, + ) + } +} + +impl RewritePresence for ContactCard { + fn rewrite_presence( + current: &Archive<&ArchivedContactCard>, + kinds: MetadataKinds, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if current.inner.metadata_kinds() == tracked_kinds(kinds.bits()) { + return Ok(()); + } + let mut card = current + .deserialize::() + .caused_by(trc::location!())?; + card.set_metadata_kinds(kinds); + rewrite_archive( + current, + GroupwareWrite::meta_only(card, current.inner), + Collection::ContactCard, + account_id, + document_id, + batch, + ) + } +} + +impl FileNode { + pub fn rewrite_presence( + current: &Archive<&ArchivedFileNode>, + presence: FilePresence, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if current.inner.presence() == presence { + return Ok(()); + } + let mut node = current + .deserialize::() + .caused_by(trc::location!())?; + node.set_presence(presence); + rewrite_archive( + current, + node, + Collection::FileNode, + account_id, + document_id, + batch, + ) + } +} + +pub(crate) fn tracked_kinds(bits: u8) -> MetadataKinds { + MetadataKinds::from_bits(bits & METADATA_KINDS.bits()).unwrap_or(MetadataKinds::NONE) +} + diff --git a/crates/groupware/src/scheduling/attendee.rs b/crates/groupware/src/scheduling/attendee.rs index fcf3d50e..a643bfa5 100644 --- a/crates/groupware/src/scheduling/attendee.rs +++ b/crates/groupware/src/scheduling/attendee.rs @@ -6,7 +6,7 @@ use crate::scheduling::{ Email, InstanceId, ItipEntries, ItipEntryValue, ItipError, ItipMessage, ItipSnapshot, - ItipSnapshots, ItipSummary, + ItipSnapshots, ItipSummary, RecurrenceId, itip::{ ItipExportAs, can_attendee_modify_property, itip_add_tz, itip_build_envelope, itip_export_component, @@ -39,6 +39,7 @@ pub(crate) fn attendee_handle_update( let mut email_rcpt = AHashSet::new(); let mut new_delegates = AHashSet::new(); let mut part_stat = &ICalendarParticipationStatus::NeedsAction; + let mut replied_instances: Vec = Vec::new(); for (instance_id, instance) in &new_itip.components { if let Some(old_instance) = old_itip.components.get(instance_id) { @@ -48,8 +49,7 @@ pub(crate) fn attendee_handle_update( { // Distinguish a genuine add/remove of a restricted property from a value-only drift // caused by a client re-encoding the same property - let old_name_counts = count_entry_names(&old_instance.entries); - let new_name_counts = count_entry_names(&instance.entries); + let mut name_counts = None; // Check added fields let mut send_update = false; @@ -77,6 +77,11 @@ pub(crate) fn attendee_handle_update( message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); + replied_instances.push(InstanceId::Recurrence(RecurrenceId { + entry_id: 0, + date: date.timestamp, + this_and_future: false, + })); } } _ => { @@ -85,8 +90,15 @@ pub(crate) fn attendee_handle_update( &instance.comp.component_type, new_entry.name, ) { - if name_count(&new_name_counts, new_entry.name) - > name_count(&old_name_counts, new_entry.name) + if name_counts + .get_or_insert_with(|| { + NameCounts::new( + &instance.comp.component_type, + &old_instance.entries, + &instance.entries, + ) + }) + .is_added(new_entry.name) { return Err(ItipError::CannotModifyProperty( new_entry.name.clone(), @@ -149,8 +161,10 @@ pub(crate) fn attendee_handle_update( &dt_stamp, instance.sequence.unwrap_or_default(), ItipExportAs::Attendee(attendee_entry_uids), + None, )); mail_from = Some(&local_attendee.email.email); + replied_instances.push(instance_id.clone()); } // Check removed fields @@ -158,8 +172,15 @@ pub(crate) fn attendee_handle_update( if !can_attendee_modify_property( &instance.comp.component_type, removed_entry.name, - ) && name_count(&old_name_counts, removed_entry.name) - > name_count(&new_name_counts, removed_entry.name) + ) && name_counts + .get_or_insert_with(|| { + NameCounts::new( + &instance.comp.component_type, + &old_instance.entries, + &instance.entries, + ) + }) + .is_removed(removed_entry.name) { // Removing these properties is not allowed return Err(ItipError::CannotModifyProperty( @@ -202,8 +223,10 @@ pub(crate) fn attendee_handle_update( &dt_stamp, instance.sequence.unwrap_or_default(), ItipExportAs::Attendee(attendee_entry_uids), + None, )); mail_from = Some(&local_attendee.email.email); + replied_instances.push(instance_id.clone()); } else { return Err(ItipError::CannotModifyInstance); } @@ -226,6 +249,7 @@ pub(crate) fn attendee_handle_update( message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); + replied_instances.push(instance_id.clone()); } } else { // Removing instances is not allowed @@ -240,15 +264,20 @@ pub(crate) fn attendee_handle_update( // Add timezones if needed itip_add_tz(&mut message, new_ical); + replied_instances.sort_unstable(); + replied_instances.dedup(); + let mut responses = vec![ItipMessage { from: from.to_string(), from_organizer: false, to: email_rcpt.into_iter().map(|e| e.to_string()).collect(), summary: ItipSummary::Rsvp { part_stat: part_stat.clone(), - current: new_itip - .main_instance_or_default() - .build_summary(Some(&new_itip.organizer), &[]), + current: new_itip.build_instances_summary( + &replied_instances, + Some(&new_itip.organizer), + &[], + ), }, message, }]; @@ -363,15 +392,111 @@ pub(crate) fn attendee_decline<'x>( }) } -fn count_entry_names<'x>(entries: &'x ItipEntries<'x>) -> AHashMap<&'x ICalendarProperty, usize> { - let mut counts = AHashMap::with_capacity(entries.len()); - for entry in entries { - *counts.entry(entry.name).or_insert(0) += 1; +pub(crate) struct OrganizerSequences(Vec<(u16, Option)>); + +impl OrganizerSequences { + pub(crate) fn of(old_itip: &ItipSnapshots<'_>, new_itip: &ItipSnapshots<'_>) -> Self { + OrganizerSequences( + new_itip + .components + .iter() + .filter_map(|(instance_id, instance)| { + let sequence = match (old_itip.components.get(instance_id), instance_id) { + (Some(old_instance), _) => old_instance.sequence, + (None, InstanceId::Recurrence(recurrence_id)) => { + old_itip.occurrence_source(recurrence_id)?.sequence + } + (None, InstanceId::Main) => return None, + }; + (sequence.unwrap_or_default() != instance.sequence.unwrap_or_default()) + .then_some((instance.comp_id, sequence)) + }) + .collect(), + ) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn restore(self, ical: &mut ICalendar) { + for (comp_id, sequence) in self.0 { + let Some(component) = ical.components.get_mut(usize::from(comp_id)) else { + continue; + }; + match sequence { + Some(sequence) => match component.property_mut(&ICalendarProperty::Sequence) { + Some(entry) => entry.values = [ICalendarValue::Integer(sequence)].into(), + None => component.add_sequence(sequence), + }, + None => component + .entries + .retain(|entry| entry.name != ICalendarProperty::Sequence), + } + } } - counts } -#[inline] -fn name_count(counts: &AHashMap<&ICalendarProperty, usize>, name: &ICalendarProperty) -> usize { - counts.get(name).copied().unwrap_or(0) +struct NameCounts<'x> { + old: AHashMap<&'x ICalendarProperty, usize>, + new: AHashMap<&'x ICalendarProperty, usize>, + duration_slot: &'static ICalendarProperty, } + +impl<'x> NameCounts<'x> { + fn new( + component_type: &ICalendarComponentType, + old: &ItipEntries<'x>, + new: &ItipEntries<'x>, + ) -> Self { + let duration_slot = match component_type { + ICalendarComponentType::VEvent => &ICalendarProperty::Dtend, + ICalendarComponentType::VTodo => &ICalendarProperty::Due, + _ => &ICalendarProperty::Duration, + }; + NameCounts { + old: Self::count(old, duration_slot), + new: Self::count(new, duration_slot), + duration_slot, + } + } + + fn count( + entries: &ItipEntries<'x>, + duration_slot: &'static ICalendarProperty, + ) -> AHashMap<&'x ICalendarProperty, usize> { + let mut counts = AHashMap::with_capacity(entries.len()); + for entry in entries { + *counts + .entry(Self::slot(entry.name, duration_slot)) + .or_insert(0) += 1; + } + counts + } + + fn slot<'y>( + name: &'y ICalendarProperty, + duration_slot: &'static ICalendarProperty, + ) -> &'y ICalendarProperty { + if matches!(name, ICalendarProperty::Duration) { + duration_slot + } else { + name + } + } + + fn get(counts: &AHashMap<&ICalendarProperty, usize>, name: &ICalendarProperty) -> usize { + counts.get(name).copied().unwrap_or(0) + } + + fn is_added(&self, name: &ICalendarProperty) -> bool { + let slot = Self::slot(name, self.duration_slot); + Self::get(&self.new, slot) > Self::get(&self.old, slot) + } + + fn is_removed(&self, name: &ICalendarProperty) -> bool { + let slot = Self::slot(name, self.duration_slot); + Self::get(&self.old, slot) > Self::get(&self.new, slot) + } +} + diff --git a/crates/groupware/src/scheduling/event_cancel.rs b/crates/groupware/src/scheduling/event_cancel.rs index 27eac0a7..656776c2 100644 --- a/crates/groupware/src/scheduling/event_cancel.rs +++ b/crates/groupware/src/scheduling/event_cancel.rs @@ -5,9 +5,10 @@ */ use crate::scheduling::{ - InstanceId, ItipError, ItipMessage, ItipSummary, + Email, InstanceId, ItipError, ItipMessage, ItipSummary, attendee::attendee_decline, itip::{itip_add_tz, itip_build_envelope}, + recipient::{RecipientPolicy, itip_messages_per_recipient}, snapshot::itip_snapshot, }; use ahash::AHashSet; @@ -19,11 +20,19 @@ use calcard::{ }, }; +#[derive(Debug, Clone, Copy)] +pub(crate) enum CancelScope<'x> { + Event, + Instance(&'x [&'x str]), + Attendees(&'x [&'x str]), +} + pub fn itip_cancel( ical: &ICalendar, account_emails: &[String], is_deletion: bool, -) -> Result, ItipError> { + policy: RecipientPolicy, +) -> Result>, ItipError> { // Prepare iTIP message let itip = itip_snapshot(ical, account_emails, false)?; let dt_stamp = PartialDateTime::now(); @@ -39,16 +48,14 @@ pub fn itip_cancel( // Fetch guest emails let mut recipients = AHashSet::new(); - let mut cancel_guests = AHashSet::new(); let mut component_type = &ICalendarComponentType::VEvent; let mut sequence = 0; for (instance_id, comp) in &itip.components { component_type = &comp.comp.component_type; for attendee in &comp.attendees { if attendee.send_update_messages() { - recipients.insert(attendee.email.email.clone()); + recipients.insert(attendee.email.email.as_str()); } - cancel_guests.insert(&attendee.email); } // Increment sequence if needed @@ -59,23 +66,25 @@ pub fn itip_cancel( if !recipients.is_empty() && component_type != &ICalendarComponentType::VFreebusy { let instance = itip.main_instance_or_default(); - message.components.push(build_cancel_component( + message.components.push(CancelScope::Event.build_component( instance.comp, sequence, dt_stamp, - &[], )); // Add timezones itip_add_tz(&mut message, ical); - Ok(ItipMessage { - to: recipients.into_iter().collect(), - summary: ItipSummary::Cancel(instance.build_summary(None, &[])), - from: itip.organizer.email.email, - from_organizer: true, - message, - }) + itip_messages_per_recipient( + vec![ItipMessage { + to: recipients.into_iter().map(str::to_string).collect(), + summary: ItipSummary::Cancel(instance.build_summary(None, &[])), + from: itip.organizer.email.email, + from_organizer: true, + message, + }], + policy, + ) } else { Err(ItipError::NothingToSend) } @@ -88,6 +97,7 @@ pub fn itip_cancel( // Decline attendance for all instances that have local attendees let mut mail_from = None; let mut email_rcpt = AHashSet::new(); + let mut declined_instances = Vec::with_capacity(itip.components.len()); for (instance_id, comp) in &itip.components { if let Some((cancel_comp, attendee_email)) = attendee_decline( instance_id, @@ -102,82 +112,85 @@ pub fn itip_cancel( message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); + declined_instances.push(instance_id); } } if let Some(from) = mail_from { + declined_instances.sort_unstable(); // Add timezone information if needed itip_add_tz(&mut message, ical); email_rcpt.insert(&itip.organizer.email.email); - Ok(ItipMessage { + Ok(vec![ItipMessage { from: from.to_string(), from_organizer: false, to: email_rcpt.into_iter().map(|e| e.to_string()).collect(), summary: ItipSummary::Rsvp { part_stat: ICalendarParticipationStatus::Declined, - current: itip.main_instance_or_default().build_summary(None, &[]), + current: itip.build_instances_summary(&declined_instances, None, &[]), }, message, - }) + }]) } else { Err(ItipError::NothingToSend) } } } -pub(crate) fn build_cancel_component( - component: &ICalendarComponent, - sequence: i64, - dt_stamp: PartialDateTime, - attendees: &[&str], -) -> ICalendarComponent { - let mut cancel_comp = ICalendarComponent { - component_type: component.component_type.clone(), - entries: Vec::with_capacity(7), - component_ids: vec![], - }; - cancel_comp.add_property( - ICalendarProperty::Status, - ICalendarValue::Status(ICalendarStatus::Cancelled), - ); - cancel_comp.add_dtstamp(dt_stamp); - cancel_comp.add_sequence(sequence); - cancel_comp.entries.extend( - component - .entries - .iter() - .filter(|e| match e.name { - ICalendarProperty::Organizer - | ICalendarProperty::Uid - | ICalendarProperty::Summary - | ICalendarProperty::Dtstart - | ICalendarProperty::Dtend - | ICalendarProperty::Duration - | ICalendarProperty::Due - | ICalendarProperty::RecurrenceId - | ICalendarProperty::Created - | ICalendarProperty::LastModified - | ICalendarProperty::Description - | ICalendarProperty::Location => true, - ICalendarProperty::Attendee => { - attendees.is_empty() - || e.values +impl CancelScope<'_> { + pub(crate) fn build_component( + self, + component: &ICalendarComponent, + sequence: i64, + dt_stamp: PartialDateTime, + ) -> ICalendarComponent { + let mut cancel_comp = ICalendarComponent { + component_type: component.component_type.clone(), + entries: Vec::with_capacity(7), + component_ids: vec![], + }; + if !matches!(self, CancelScope::Attendees(_)) { + cancel_comp.add_property( + ICalendarProperty::Status, + ICalendarValue::Status(ICalendarStatus::Cancelled), + ); + } + cancel_comp.add_dtstamp(dt_stamp); + cancel_comp.add_sequence(sequence); + cancel_comp.entries.extend( + component + .entries + .iter() + .filter(|e| match e.name { + ICalendarProperty::Organizer + | ICalendarProperty::Uid + | ICalendarProperty::Summary + | ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Duration + | ICalendarProperty::Due + | ICalendarProperty::RecurrenceId + | ICalendarProperty::Created + | ICalendarProperty::LastModified + | ICalendarProperty::Description + | ICalendarProperty::Location => true, + ICalendarProperty::Attendee => match self { + CancelScope::Event => true, + CancelScope::Instance(attendees) | CancelScope::Attendees(attendees) => e + .values .first() .and_then(|v| v.as_text()) - .is_some_and(|email| { - attendees.iter().any(|attendee| { - email - .strip_suffix(attendee) - .is_some_and(|v| v.ends_with(':') || v.is_empty()) - }) - }) - } - _ => false, - }) - .cloned(), - ); + .and_then(|address| Email::new(address, &[])) + .is_some_and(|email| attendees.contains(&email.email.as_str())), + }, + _ => false, + }) + .cloned(), + ); - cancel_comp + cancel_comp + } } + diff --git a/crates/groupware/src/scheduling/event_create.rs b/crates/groupware/src/scheduling/event_create.rs index 8fe59b40..d161c345 100644 --- a/crates/groupware/src/scheduling/event_create.rs +++ b/crates/groupware/src/scheduling/event_create.rs @@ -5,7 +5,11 @@ */ use crate::scheduling::{ - ItipError, ItipMessage, itip::itip_finalize, organizer::organizer_request_full, + ItipError, ItipMessage, + attendee::attendee_handle_update, + itip::itip_finalize, + organizer::organizer_request_full, + recipient::{RecipientPolicy, itip_messages_per_recipient}, snapshot::itip_snapshot, }; use calcard::icalendar::ICalendar; @@ -13,6 +17,7 @@ use calcard::icalendar::ICalendar; pub fn itip_create( ical: &mut ICalendar, account_emails: &[String], + policy: RecipientPolicy, ) -> Result>, ItipError> { let itip = itip_snapshot(ical, account_emails, false)?; if !itip.organizer.is_server_scheduling { @@ -20,8 +25,28 @@ pub fn itip_create( } else if !itip.organizer.email.is_local { Err(ItipError::NotOrganizer) } else { - organizer_request_full(ical, &itip, None, true).inspect(|_| { + organizer_request_full(ical, &itip, None, true) + .and_then(|messages| itip_messages_per_recipient(messages, policy)) + .inspect(|_| { + itip_finalize(ical, &[]); + }) + } +} + +pub fn itip_attendee_create( + ical: &mut ICalendar, + account_emails: &[String], +) -> Result>, ItipError> { + let new_itip = itip_snapshot(ical, account_emails, false)?; + if !new_itip.organizer.is_server_scheduling { + Err(ItipError::OtherSchedulingAgent) + } else if new_itip.organizer.email.is_local { + Err(ItipError::NothingToSend) + } else { + let old_itip = new_itip.unanswered(); + attendee_handle_update(ical, old_itip, new_itip).inspect(|_| { itip_finalize(ical, &[]); }) } } + diff --git a/crates/groupware/src/scheduling/event_update.rs b/crates/groupware/src/scheduling/event_update.rs index 5e97950f..779253a1 100644 --- a/crates/groupware/src/scheduling/event_update.rs +++ b/crates/groupware/src/scheduling/event_update.rs @@ -5,8 +5,13 @@ */ use crate::scheduling::{ - ItipError, ItipMessage, attendee::attendee_handle_update, event_cancel::itip_cancel, - itip::itip_finalize, organizer::organizer_handle_update, snapshot::itip_snapshot, + ItipError, ItipMessage, + attendee::{OrganizerSequences, attendee_handle_update}, + event_cancel::itip_cancel, + itip::itip_finalize, + organizer::organizer_handle_update, + recipient::{RecipientPolicy, itip_messages_per_recipient}, + snapshot::itip_snapshot, }; use calcard::icalendar::ICalendar; @@ -14,6 +19,7 @@ pub fn itip_update( ical: &mut ICalendar, old_ical: &ICalendar, account_emails: &[String], + policy: RecipientPolicy, ) -> Result>, ItipError> { let old_itip = itip_snapshot(old_ical, account_emails, false)?; match itip_snapshot(ical, account_emails, false) { @@ -24,8 +30,17 @@ pub fn itip_update( Err(ItipError::OrganizerMismatch) } else if old_itip.organizer.email.is_local { organizer_handle_update(old_ical, ical, old_itip, new_itip, &mut sequences) + .and_then(|messages| itip_messages_per_recipient(messages, policy)) } else { - attendee_handle_update(ical, old_itip, new_itip) + let organizer_sequences = OrganizerSequences::of(&old_itip, &new_itip); + if organizer_sequences.is_empty() { + attendee_handle_update(ical, old_itip, new_itip) + } else { + drop(new_itip); + organizer_sequences.restore(ical); + itip_snapshot(ical, account_emails, false) + .and_then(|new_itip| attendee_handle_update(ical, old_itip, new_itip)) + } } .inspect(|_| { itip_finalize(ical, &sequences); @@ -39,7 +54,7 @@ pub fn itip_update( | ItipError::OtherSchedulingAgent => { if old_itip.organizer.email.is_local { // RFC 6638 does not support replacing the organizer, so we cancel the event - itip_cancel(old_ical, account_emails, false).map(|message| vec![message]) + itip_cancel(old_ical, account_emails, false, policy) } else { Err(ItipError::CannotModifyAddress) } diff --git a/crates/groupware/src/scheduling/format.rs b/crates/groupware/src/scheduling/format.rs index ad873d51..d3ee9b10 100644 --- a/crates/groupware/src/scheduling/format.rs +++ b/crates/groupware/src/scheduling/format.rs @@ -9,11 +9,15 @@ use calcard::{ common::timezone::Tz, icalendar::{ICalendarDay, ICalendarFrequency, ICalendarRecurrenceRule, ICalendarWeekday}, }; -use chrono::{DateTime, NaiveDate, TimeZone, Weekday}; use common::i18n::{self, Locale, PluralForms}; +use compact_str::ToCompactString; use icu_datetime::{DateTimeFormatter, fieldsets}; use icu_locale_core::{Locale as IcuLocale, locale}; use icu_plurals::{PluralCategory, PluralRuleType, PluralRules, PluralRulesOptions}; +use jiff::{ + Timestamp, + civil::{Date, ISOWeekDate, Weekday}, +}; use std::fmt::Write; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -43,7 +47,7 @@ impl TextFormatter { .into_err() .caused_by(trc::location!()) .details(detail) - .ctx(trc::Key::Reason, err.to_string()) + .ctx(trc::Key::Reason, err.to_compact_string()) } }; let plural_prefs = (&icu_locale).into(); @@ -54,7 +58,7 @@ impl TextFormatter { .into_err() .caused_by(trc::location!()) .details("Failed to load plural rules") - .ctx(trc::Key::Reason, err.to_string()) + .ctx(trc::Key::Reason, err.to_compact_string()) }) }; @@ -93,11 +97,7 @@ impl TextFormatter { DateStyle::Long => (&self.weekday, &self.date_long), }; let local = tz - .from_utc_datetime( - &DateTime::from_timestamp(time.start, 0) - .unwrap_or_default() - .naive_local(), - ) + .from_timestamp(Timestamp::from_second(time.start).unwrap_or_default()) .naive_local(); let _ = write!( out, @@ -316,22 +316,22 @@ impl TextFormatter { fn write_weekday(&self, out: &mut String, weekday: ICalendarWeekday) { let weekday = match weekday { - ICalendarWeekday::Monday => Weekday::Mon, - ICalendarWeekday::Tuesday => Weekday::Tue, - ICalendarWeekday::Wednesday => Weekday::Wed, - ICalendarWeekday::Thursday => Weekday::Thu, - ICalendarWeekday::Friday => Weekday::Fri, - ICalendarWeekday::Saturday => Weekday::Sat, - ICalendarWeekday::Sunday => Weekday::Sun, + ICalendarWeekday::Monday => Weekday::Monday, + ICalendarWeekday::Tuesday => Weekday::Tuesday, + ICalendarWeekday::Wednesday => Weekday::Wednesday, + ICalendarWeekday::Thursday => Weekday::Thursday, + ICalendarWeekday::Friday => Weekday::Friday, + ICalendarWeekday::Saturday => Weekday::Saturday, + ICalendarWeekday::Sunday => Weekday::Sunday, }; - if let Some(date) = NaiveDate::from_isoywd_opt(2024, 1, weekday) { + if let Ok(date) = ISOWeekDate::new(2024, 1, weekday).map(ISOWeekDate::date) { let _ = write!(out, "{}", self.weekday.format(&date)); } } fn write_month(&self, out: &mut String, month: u8) { - if let Some(date) = NaiveDate::from_ymd_opt(2024, month.clamp(1, 12) as u32, 1) { + if let Ok(date) = Date::new(2024, month.clamp(1, 12) as i8, 1) { let _ = write!(out, "{}", self.month.format(&date)); } } @@ -407,188 +407,15 @@ fn write_number(out: &mut String, template: &str, placeholder: &str, value: u64) pub fn hyperlink(value: &str) -> Option<&str> { let (scheme, _) = value.split_once(':')?; - ["https", "http", "tel", "sip", "sips", "xmpp"] - .iter() - .any(|candidate| scheme.eq_ignore_ascii_case(candidate)) - .then_some(value) + hashify::set_ignore_case!( + scheme.as_bytes(), + "https", + "http", + "tel", + "sip", + "sips", + "xmpp" + ) + .then_some(value) } -#[cfg(test)] -mod tests { - use super::{PluralCategoryExt, PluralForms, TextFormatter, i18n}; - use calcard::icalendar::{ - ICalendarDay, ICalendarFrequency, ICalendarRecurrenceRule, ICalendarWeekday, - }; - use icu_plurals::PluralCategory; - - fn rule(freq: ICalendarFrequency, interval: Option) -> ICalendarRecurrenceRule { - ICalendarRecurrenceRule { - freq, - interval, - ..Default::default() - } - } - - fn format(language: &str, rule: &ICalendarRecurrenceRule) -> String { - let mut out = String::new(); - TextFormatter::new(language) - .expect("formatter") - .recurrence(&mut out, rule); - out - } - - #[test] - fn every_shipped_locale_loads_icu_data() { - for locale in i18n::ALL_LOCALES { - let formatter = TextFormatter::new(locale.name) - .unwrap_or_else(|err| panic!("{}: {err:?}", locale.name)); - assert_eq!(formatter.locale.name, locale.name); - - let mut out = String::new(); - formatter.recurrence( - &mut out, - &ICalendarRecurrenceRule { - freq: ICalendarFrequency::Monthly, - interval: Some(2), - count: Some(5), - bymonthday: vec![3, -1], - byday: vec![ICalendarDay { - ordwk: Some(2), - weekday: ICalendarWeekday::Monday, - }], - ..Default::default() - }, - ); - assert!(!out.is_empty(), "{} produced no text", locale.name); - } - } - - #[test] - fn plural_form_selects_category_and_falls_back_to_other() { - let forms = PluralForms { - zero: "many", - one: "single", - two: "many", - few: "a few", - many: "many", - other: "many", - }; - assert_eq!(PluralCategory::One.plural_form(&forms), "single"); - assert_eq!(PluralCategory::Few.plural_form(&forms), "a few"); - assert_eq!(PluralCategory::Many.plural_form(&forms), "many"); - assert_eq!(PluralCategory::Other.plural_form(&forms), "many"); - - // Categories a locale omits are filled from "other" at build time - let polish = i18n::locale("pl-PL").expect("locale must exist"); - assert_eq!(polish.calendar_rrule_secondly.many, "Co $n sekund"); - assert_eq!( - polish.calendar_rrule_secondly.zero, - polish.calendar_rrule_secondly.other - ); - } - - #[test] - fn frequency_uses_cardinal_plural_rules() { - assert_eq!( - format("en", &rule(ICalendarFrequency::Weekly, None)), - "Every week" - ); - assert_eq!( - format("en", &rule(ICalendarFrequency::Weekly, Some(2))), - "Every 2 weeks" - ); - - // Polish distinguishes one / few / many, unlike English - assert_eq!( - format("pl", &rule(ICalendarFrequency::Weekly, Some(1))), - "Co tydzień" - ); - assert_eq!( - format("pl", &rule(ICalendarFrequency::Weekly, Some(2))), - "Co 2 tygodnie" - ); - assert_eq!( - format("pl", &rule(ICalendarFrequency::Weekly, Some(5))), - "Co 5 tygodni" - ); - } - - #[test] - fn weekday_names_are_localized_without_translation_keys() { - let mut byday = rule(ICalendarFrequency::Weekly, None); - byday.byday = vec![ICalendarDay { - weekday: ICalendarWeekday::Monday, - ordwk: None, - }]; - - assert_eq!(format("en", &byday), "Every week on Monday"); - assert_eq!(format("es", &byday), "Cada semana los lunes"); - } - - #[test] - fn ordinal_weekday_uses_ordinal_plural_rules() { - let mut byday = rule(ICalendarFrequency::Monthly, None); - byday.byday = vec![ICalendarDay { - weekday: ICalendarWeekday::Tuesday, - ordwk: Some(2), - }]; - assert_eq!(format("en", &byday), "Every month on the 2nd Tuesday"); - - byday.byday = vec![ICalendarDay { - weekday: ICalendarWeekday::Tuesday, - ordwk: Some(3), - }]; - assert_eq!(format("en", &byday), "Every month on the 3rd Tuesday"); - - byday.byday = vec![ICalendarDay { - weekday: ICalendarWeekday::Tuesday, - ordwk: Some(-1), - }]; - assert_eq!( - format("en", &byday), - "Every month on the 1st Tuesday from the end" - ); - } - - #[test] - fn count_is_pluralized_and_appended() { - let mut counted = rule(ICalendarFrequency::Daily, None); - counted.count = Some(1); - assert_eq!(format("en", &counted), "Every day, 1 time"); - - counted.count = Some(5); - assert_eq!(format("en", &counted), "Every day, 5 times"); - } - - #[test] - fn multiple_days_are_joined_with_the_localized_conjunction() { - let mut byday = rule(ICalendarFrequency::Weekly, None); - byday.byday = vec![ - ICalendarDay { - weekday: ICalendarWeekday::Monday, - ordwk: None, - }, - ICalendarDay { - weekday: ICalendarWeekday::Wednesday, - ordwk: None, - }, - ICalendarDay { - weekday: ICalendarWeekday::Friday, - ordwk: None, - }, - ]; - - assert_eq!( - format("en", &byday), - "Every week on Monday, Wednesday and Friday" - ); - } - - #[test] - fn unknown_language_falls_back_to_english_rules() { - assert_eq!( - format("zz", &rule(ICalendarFrequency::Weekly, Some(3))), - "Every 3 weeks" - ); - } -} diff --git a/crates/groupware/src/scheduling/inbound.rs b/crates/groupware/src/scheduling/inbound.rs index 21521aac..39575390 100644 --- a/crates/groupware/src/scheduling/inbound.rs +++ b/crates/groupware/src/scheduling/inbound.rs @@ -5,7 +5,9 @@ */ use crate::scheduling::{ - InstanceId, ItipError, ItipMessage, ItipSnapshots, organizer::organizer_request_full, + Attendee, InstanceId, ItipError, ItipMessage, ItipSnapshot, ItipSnapshots, RecurrenceId, + organizer::organizer_request_full, + recipient::{RecipientPolicy, itip_messages_per_recipient, series::Series}, }; use ahash::AHashSet; use calcard::icalendar::{ @@ -35,13 +37,25 @@ pub enum MergeAction { parameters: Vec, }, AddComponent { - component: ICalendarComponent, + components: Vec, }, RemoveComponent { component_id: u16, }, } +enum EntryChange { + Parameters { + entry_id: u16, + remove: Vec, + add: Vec, + }, + Entries { + remove: AHashSet, + add: Vec, + }, +} + pub enum MergeResult { Actions(Vec), Message(ItipMessage), @@ -54,6 +68,7 @@ pub fn itip_process_message( itip: &ICalendar, itip_snapshots: ItipSnapshots<'_>, sender: String, + policy: RecipientPolicy, ) -> Result { if snapshots.organizer.email != itip_snapshots.organizer.email { return Err(ItipError::OrganizerMismatch); @@ -69,25 +84,50 @@ pub fn itip_process_message( } match method { ICalendarMethod::Reply => { - handle_reply(&snapshots, &itip_snapshots, &sender, &mut merge_actions)?; + handle_reply( + ical, + &snapshots, + &itip_snapshots, + &sender, + policy.max_instances, + &mut merge_actions, + )?; } ICalendarMethod::Refresh => { - return organizer_request_full(ical, &snapshots, None, false).and_then( - |messages| { - messages - .into_iter() - .next() - .map(|mut message| { - message.to = vec![sender]; - MergeResult::Message(message) - }) - .ok_or(ItipError::NothingToSend) - }, - ); + if !snapshots + .components + .values() + .any(|instance| instance.attendee_by_email(&sender).is_some()) + { + return Err(ItipError::SenderIsNotParticipant(sender)); + } + let message = organizer_request_full(ical, &snapshots, None, false)? + .into_iter() + .next() + .ok_or(ItipError::NothingToSend)?; + return itip_messages_per_recipient( + vec![ItipMessage { + to: vec![sender], + ..message + }], + policy, + )? + .into_iter() + .next() + .map(MergeResult::Message) + .ok_or(ItipError::NothingToSend); } _ => return Err(ItipError::UnsupportedMethod(method.clone())), } } else { + if matches!( + method, + ICalendarMethod::Request | ICalendarMethod::Add | ICalendarMethod::Cancel + ) && snapshots.organizer.email.email != sender + { + return Err(ItipError::SenderIsNotOrganizer); + } + // Handle organizer and attendees updates match method { ICalendarMethod::Request => { @@ -138,7 +178,7 @@ pub fn itip_process_message( } else { // Add instance merge_actions.push(MergeAction::AddComponent { - component: ICalendarComponent { + components: vec![ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries @@ -149,7 +189,7 @@ pub fn itip_process_message( .cloned() .collect(), component_ids: vec![], - }, + }], }); } } @@ -170,7 +210,7 @@ pub fn itip_process_message( if !snapshots.components.contains_key(instance_id) { let itip_component = &itip.components[itip_snapshot.comp_id as usize]; merge_actions.push(MergeAction::AddComponent { - component: ICalendarComponent { + components: vec![ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries @@ -181,7 +221,7 @@ pub fn itip_process_message( .cloned() .collect(), component_ids: vec![], - }, + }], }); } } @@ -222,9 +262,10 @@ pub fn itip_process_message( .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], - values: vec![ICalendarValue::Status( + values: [ICalendarValue::Status( ICalendarStatus::Cancelled, - )], + )] + .into(), }]) .collect(), }); @@ -236,7 +277,7 @@ pub fn itip_process_message( } else { let itip_component = itip_snapshot.comp; merge_actions.push(MergeAction::AddComponent { - component: ICalendarComponent { + components: vec![ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries @@ -251,13 +292,14 @@ pub fn itip_process_message( .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], - values: vec![ICalendarValue::Status( + values: [ICalendarValue::Status( ICalendarStatus::Cancelled, - )], + )] + .into(), }]) .collect(), component_ids: vec![], - }, + }], }); } } @@ -294,9 +336,10 @@ pub fn itip_process_message( .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], - values: vec![ICalendarValue::Status( + values: [ICalendarValue::Status( ICalendarStatus::Cancelled, - )], + )] + .into(), }]) .collect(), }); @@ -311,7 +354,14 @@ pub fn itip_process_message( }) }) => { - handle_reply(&snapshots, &itip_snapshots, &sender, &mut merge_actions)?; + handle_reply( + ical, + &snapshots, + &itip_snapshots, + &sender, + policy.max_instances, + &mut merge_actions, + )?; } _ => return Err(ItipError::UnsupportedMethod(method.clone())), } @@ -347,202 +397,316 @@ pub fn itip_import_message(ical: &mut ICalendar) -> Result<(), ItipError> { } fn handle_reply( + ical: &ICalendar, snapshots: &ItipSnapshots<'_>, itip_snapshots: &ItipSnapshots<'_>, sender: &str, + max_instances: usize, merge_actions: &mut Vec, ) -> Result<(), ItipError> { + let replied_dates = itip_snapshots + .components + .keys() + .filter_map(|instance_id| match instance_id { + InstanceId::Recurrence(recurrence_id) => Some(recurrence_id.date), + InstanceId::Main => None, + }) + .collect::>(); + let mut series = None; + for (instance_id, itip_snapshot) in &itip_snapshots.components { - if let Some(snapshot) = snapshots.components.get(instance_id) { - if let (Some(attendee), Some(updated_attendee)) = ( - snapshot.attendee_by_email(sender), - itip_snapshot.attendee_by_email(sender), - ) { - let itip_component = itip_snapshot.comp; - let changed_part_stat = attendee.part_stat != updated_attendee.part_stat; - let changed_rsvp = attendee.rsvp != updated_attendee.rsvp; - let changed_delegated_to = attendee.delegated_to != updated_attendee.delegated_to; - let has_request_status = !itip_snapshot.request_status.is_empty(); - - if changed_part_stat || changed_rsvp || changed_delegated_to || has_request_status { - // Update participant status - let mut add_parameters = Vec::new(); - let mut remove_parameters = Vec::new(); - if changed_part_stat { - remove_parameters.push(ICalendarParameterName::Partstat); - if let Some(part_stat) = updated_attendee.part_stat { - add_parameters.push(ICalendarParameter::partstat(part_stat.clone())); - } - } - - if changed_rsvp { - remove_parameters.push(ICalendarParameterName::Rsvp); - if let Some(rsvp) = updated_attendee.rsvp { - add_parameters.push(ICalendarParameter::rsvp(rsvp)); - } - } - - if changed_delegated_to { - remove_parameters.push(ICalendarParameterName::DelegatedTo); - if !updated_attendee.delegated_to.is_empty() { - add_parameters.extend(updated_attendee.delegated_to.iter().map( - |email| { - ICalendarParameter::delegated_to(Uri::Location( - email.to_string(), - )) - }, - )); - } - } - - // RFC 6638 3.2.5: the status defaults to 2.0 when the reply carries none - remove_parameters.push(ICalendarParameterName::ScheduleStatus); - add_parameters.push(ICalendarParameter::schedule_status( - if has_request_status { - itip_snapshot.request_status.join(",") - } else { - "2.0".to_string() - }, - )); - - merge_actions.push(MergeAction::RemoveParameters { - component_id: snapshot.comp_id, - entry_id: attendee.entry_id, - parameters: remove_parameters, - }); - merge_actions.push(MergeAction::AddParameters { - component_id: snapshot.comp_id, - entry_id: attendee.entry_id, - parameters: add_parameters, - }); - - // Add unknown delegated attendees - for delegated_to in &updated_attendee.delegated_to { - if let Some(itip_delegated) = - itip_snapshot.attendee_by_email(&delegated_to.email) - { - if let Some(delegated) = snapshot.attendee_by_email(&delegated_to.email) - { - if delegated != itip_delegated { - merge_actions.push(MergeAction::RemoveParameters { - component_id: snapshot.comp_id, - entry_id: delegated.entry_id, - parameters: vec![ - ICalendarParameterName::DelegatedTo, - ICalendarParameterName::DelegatedFrom, - ICalendarParameterName::Partstat, - ICalendarParameterName::Rsvp, - ICalendarParameterName::ScheduleStatus, - ICalendarParameterName::Role, - ], - }); - merge_actions.push(MergeAction::AddParameters { - component_id: snapshot.comp_id, - entry_id: delegated.entry_id, - parameters: itip_component.entries - [itip_delegated.entry_id as usize] - .params - .iter() - .filter(|param| { - matches!( - param.name, - ICalendarParameterName::DelegatedTo - | ICalendarParameterName::DelegatedFrom - | ICalendarParameterName::Partstat - | ICalendarParameterName::Rsvp - | ICalendarParameterName::ScheduleStatus - | ICalendarParameterName::Role - ) - }) - .cloned() - .collect(), - }); - } - } else { - merge_actions.push(MergeAction::AddEntries { - component_id: snapshot.comp_id, - entries: vec![ - itip_component.entries[itip_delegated.entry_id as usize] - .clone(), - ], - }); - } - } - } - } - - // Add changed properties for VTODO - if snapshot.comp.component_type == ICalendarComponentType::VTodo { - let mut remove_entries = AHashSet::new(); - let mut add_entries = Vec::new(); - - for entry in itip_component.entries.iter() { - if matches!( - entry.name, - ICalendarProperty::PercentComplete - | ICalendarProperty::Status - | ICalendarProperty::Completed - ) { - remove_entries.insert(entry.name.clone()); - add_entries.push(entry.clone()); - } - } - - if !add_entries.is_empty() { - merge_actions.push(MergeAction::RemoveEntries { - component_id: snapshot.comp_id, - entries: remove_entries, - }); - merge_actions.push(MergeAction::AddEntries { - component_id: snapshot.comp_id, - entries: add_entries, - }); - } - } - } else { - return Err(ItipError::SenderIsNotParticipant(sender.to_string())); - } - } else if itip_snapshot.attendee_by_email(sender).is_some() { - // Add component - let itip_component = itip_snapshot.comp; - let is_todo = itip_component.component_type == ICalendarComponentType::VTodo; - merge_actions.push(MergeAction::AddComponent { - component: ICalendarComponent { - component_type: itip_component.component_type.clone(), - entries: itip_component - .entries - .iter() - .filter(|entry| { - matches!( - entry.name, - ICalendarProperty::Organizer - | ICalendarProperty::Attendee - | ICalendarProperty::Uid - | ICalendarProperty::Dtstamp - | ICalendarProperty::Sequence - | ICalendarProperty::RecurrenceId - ) || (is_todo - && matches!( - entry.name, - ICalendarProperty::PercentComplete - | ICalendarProperty::Status - | ICalendarProperty::Completed - )) - }) - .cloned() - .collect(), - component_ids: vec![], - }, - }); - } else { + let Some(updated_attendee) = itip_snapshot.attendee_by_email(sender) else { return Err(ItipError::SenderIsNotParticipant(sender.to_string())); + }; + + if let Some((answered_id, snapshot)) = snapshots.matching_instance(instance_id) { + let attendee = snapshot + .attendee_by_email(sender) + .ok_or_else(|| ItipError::SenderIsNotParticipant(sender.to_string()))?; + let participation = + EntryChange::participation(snapshot, attendee, itip_snapshot, updated_attendee); + if !participation.is_empty() { + for (other_id, other) in &snapshots.components { + if let InstanceId::Recurrence(other_recurrence) = other_id + && answered_id.covers(other_recurrence) + && !replied_dates.contains(&other_recurrence.date) + && let Some(other_attendee) = other.attendee_by_email(sender) + && other_attendee == attendee + { + for change in EntryChange::participation( + other, + other_attendee, + itip_snapshot, + updated_attendee, + ) { + change.push_actions(other.comp_id, merge_actions); + } + } + } + } + for change in participation + .into_iter() + .chain(EntryChange::todo_progress(snapshot, itip_snapshot)) + { + change.push_actions(snapshot.comp_id, merge_actions); + } + } else if let InstanceId::Recurrence(recurrence_id) = instance_id + && let Some(source) = snapshots.occurrence_source(recurrence_id) + { + let attendee = source + .attendee_by_email(sender) + .ok_or_else(|| ItipError::SenderIsNotParticipant(sender.to_string()))?; + let progress = EntryChange::todo_progress(source, itip_snapshot); + let reports_progress = matches!( + &progress, + Some(EntryChange::Entries { add, .. }) + if add.iter().any(|entry| !source.comp.entries.contains(entry)) + ); + if !attendee.answers_differently(updated_attendee) && !reports_progress { + continue; + } + let series = series.get_or_insert_with(|| { + Series::new( + ical, + snapshots.main_instance().map(|main| main.comp), + max_instances, + ) + }); + if !series.has_occurrence(recurrence_id.date) { + continue; + } + + let mut occurrence = source.comp.clone(); + for change in + EntryChange::participation(source, attendee, itip_snapshot, updated_attendee) + .into_iter() + .chain(progress) + { + change.apply(&mut occurrence); + } + if let Some(components) = itip_snapshot + .comp + .entries + .get(usize::from(recurrence_id.entry_id)) + .and_then(|entry| series.occurrence(occurrence, entry, recurrence_id.date)) + { + merge_actions.push(MergeAction::AddComponent { components }); + } } } Ok(()) } -pub fn itip_merge_changes(ical: &mut ICalendar, itip: &ICalendar, changes: Vec) { +impl EntryChange { + fn participation( + snapshot: &ItipSnapshot<'_>, + attendee: &Attendee<'_>, + itip_snapshot: &ItipSnapshot<'_>, + updated_attendee: &Attendee<'_>, + ) -> Vec { + let mut changes = Vec::new(); + let itip_component = itip_snapshot.comp; + let changed_part_stat = attendee.part_stat != updated_attendee.part_stat; + let changed_rsvp = attendee.rsvp != updated_attendee.rsvp; + let changed_delegated_to = attendee.delegated_to != updated_attendee.delegated_to; + let has_request_status = !itip_snapshot.request_status.is_empty(); + + if changed_part_stat || changed_rsvp || changed_delegated_to || has_request_status { + // Update participant status + let mut add_parameters = Vec::new(); + let mut remove_parameters = Vec::new(); + if changed_part_stat { + remove_parameters.push(ICalendarParameterName::Partstat); + if let Some(part_stat) = updated_attendee.part_stat { + add_parameters.push(ICalendarParameter::partstat(part_stat.clone())); + } + } + + if changed_rsvp { + remove_parameters.push(ICalendarParameterName::Rsvp); + if let Some(rsvp) = updated_attendee.rsvp { + add_parameters.push(ICalendarParameter::rsvp(rsvp)); + } + } + + if changed_delegated_to { + remove_parameters.push(ICalendarParameterName::DelegatedTo); + if !updated_attendee.delegated_to.is_empty() { + add_parameters.extend(updated_attendee.delegated_to.iter().map(|email| { + ICalendarParameter::delegated_to(Uri::Location(email.to_string())) + })); + } + } + + // RFC 6638 4.2: the status defaults to 2.0 when the reply carries none + remove_parameters.push(ICalendarParameterName::ScheduleStatus); + add_parameters.push(ICalendarParameter::schedule_status(if has_request_status { + itip_snapshot.request_status.join(",") + } else { + "2.0".to_string() + })); + + changes.push(EntryChange::Parameters { + entry_id: attendee.entry_id, + remove: remove_parameters, + add: add_parameters, + }); + + // Add unknown delegated attendees + for delegated_to in &updated_attendee.delegated_to { + if let Some(itip_delegated) = itip_snapshot.attendee_by_email(&delegated_to.email) { + if let Some(delegated) = snapshot.attendee_by_email(&delegated_to.email) { + if delegated != itip_delegated { + changes.push(EntryChange::Parameters { + entry_id: delegated.entry_id, + remove: vec![ + ICalendarParameterName::DelegatedTo, + ICalendarParameterName::DelegatedFrom, + ICalendarParameterName::Partstat, + ICalendarParameterName::Rsvp, + ICalendarParameterName::ScheduleStatus, + ICalendarParameterName::Role, + ], + add: itip_component.entries[itip_delegated.entry_id as usize] + .params + .iter() + .filter(|param| { + matches!( + param.name, + ICalendarParameterName::DelegatedTo + | ICalendarParameterName::DelegatedFrom + | ICalendarParameterName::Partstat + | ICalendarParameterName::Rsvp + | ICalendarParameterName::ScheduleStatus + | ICalendarParameterName::Role + ) + }) + .cloned() + .collect(), + }); + } + } else { + changes.push(EntryChange::Entries { + remove: AHashSet::new(), + add: vec![ + itip_component.entries[itip_delegated.entry_id as usize].clone(), + ], + }); + } + } + } + } + + changes + } + + fn todo_progress( + snapshot: &ItipSnapshot<'_>, + itip_snapshot: &ItipSnapshot<'_>, + ) -> Option { + // Add changed properties for VTODO + if snapshot.comp.component_type != ICalendarComponentType::VTodo { + return None; + } + + let mut remove_entries = AHashSet::new(); + let mut add_entries = Vec::new(); + for entry in itip_snapshot.comp.entries.iter() { + if matches!( + entry.name, + ICalendarProperty::PercentComplete + | ICalendarProperty::Status + | ICalendarProperty::Completed + ) { + remove_entries.insert(entry.name.clone()); + add_entries.push(entry.clone()); + } + } + + (!add_entries.is_empty()).then_some(EntryChange::Entries { + remove: remove_entries, + add: add_entries, + }) + } + + fn push_actions(self, component_id: u16, merge_actions: &mut Vec) { + match self { + EntryChange::Parameters { + entry_id, + remove, + add, + } => { + merge_actions.push(MergeAction::RemoveParameters { + component_id, + entry_id, + parameters: remove, + }); + merge_actions.push(MergeAction::AddParameters { + component_id, + entry_id, + parameters: add, + }); + } + EntryChange::Entries { remove, add } => { + if !remove.is_empty() { + merge_actions.push(MergeAction::RemoveEntries { + component_id, + entries: remove, + }); + } + merge_actions.push(MergeAction::AddEntries { + component_id, + entries: add, + }); + } + } + } + + fn apply(self, component: &mut ICalendarComponent) { + match self { + EntryChange::Parameters { + entry_id, + remove, + add, + } => { + if let Some(entry) = component.entries.get_mut(usize::from(entry_id)) { + entry.params.retain(|param| !remove.contains(¶m.name)); + entry.params.extend(add); + } + } + EntryChange::Entries { remove, add } => { + component + .entries + .retain(|entry| !remove.contains(&entry.name)); + component.entries.extend(add); + } + } + } +} + +impl InstanceId { + fn covers(&self, occurrence: &RecurrenceId) -> bool { + match self { + InstanceId::Main => true, + InstanceId::Recurrence(range) => range.this_and_future && occurrence.date > range.date, + } + } +} + +impl Attendee<'_> { + fn answers_differently(&self, reply: &Attendee<'_>) -> bool { + self.part_stat != reply.part_stat + || self.rsvp != reply.rsvp + || self.delegated_to != reply.delegated_to + } +} + +pub fn itip_merge_changes( + ical: &mut ICalendar, + itip: &ICalendar, + changes: Vec, +) -> Result<(), ItipError> { let mut remove_component_ids: Vec = Vec::new(); for action in changes { match action { @@ -580,7 +744,7 @@ pub fn itip_merge_changes(ical: &mut ICalendar, itip: &ICalendar, changes: Vec { + MergeAction::AddComponent { mut components } => { let comp_id = ical.components.len() as u32; if let Some(root) = ical .components @@ -588,7 +752,13 @@ pub fn itip_merge_changes(ical: &mut ICalendar, itip: &ICalendar, changes: Vec { @@ -597,11 +767,12 @@ pub fn itip_merge_changes(ical: &mut ICalendar, itip: &ICalendar, changes: Vec Result<&ICalendarMethod, ItipError> { }) .ok_or(ItipError::MissingMethod) } + diff --git a/crates/groupware/src/scheduling/itip.rs b/crates/groupware/src/scheduling/itip.rs index 8191e3f8..4d767338 100644 --- a/crates/groupware/src/scheduling/itip.rs +++ b/crates/groupware/src/scheduling/itip.rs @@ -8,8 +8,8 @@ use crate::scheduling::{Email, ItipMessage, ItipMessages, ItipSummary}; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ - ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, - ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ICalendarMethod, ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationStatus, ICalendarProperty, ICalendarScheduleAgentValue, ICalendarValue, }, @@ -19,6 +19,7 @@ use registry::schema::structs::{ Task, TaskCalendarItipContents, TaskCalendarItipMessage, TaskStatus, }; use store::write::BatchBuilder; +use types::id::Id; pub(crate) fn itip_build_envelope(method: ICalendarMethod) -> ICalendarComponent { ICalendarComponent { @@ -27,17 +28,17 @@ pub(crate) fn itip_build_envelope(method: ICalendarMethod) -> ICalendarComponent ICalendarEntry { name: ICalendarProperty::Version, params: vec![], - values: vec![ICalendarValue::Text("2.0".to_string())], + values: [ICalendarValue::Text("2.0".to_string())].into(), }, ICalendarEntry { name: ICalendarProperty::Prodid, params: vec![], - values: vec![ICalendarValue::Text(PROD_ID.to_string())], + values: [ICalendarValue::Text(PROD_ID.to_string())].into(), }, ICalendarEntry { name: ICalendarProperty::Method, params: vec![], - values: vec![ICalendarValue::Method(method)], + values: [ICalendarValue::Method(method)].into(), }, ], component_ids: Default::default(), @@ -67,7 +68,7 @@ pub fn itip_assign_organizer(ical: &mut ICalendar, organizer_address: &str) -> b component.entries.push(ICalendarEntry { name: ICalendarProperty::Organizer, params: vec![], - values: vec![ICalendarValue::Text(format!("mailto:{organizer_address}"))], + values: [ICalendarValue::Text(format!("mailto:{organizer_address}"))].into(), }); assigned = true; } @@ -213,8 +214,10 @@ pub(crate) fn itip_export_component( dt_stamp: &PartialDateTime, sequence: i64, export_as: ItipExportAs<'_>, + series_length: Option<&ICalendarDuration>, ) -> ICalendarComponent { let is_todo = component.component_type == ICalendarComponentType::VTodo; + let is_event = component.component_type == ICalendarComponentType::VEvent; let mut comp = ICalendarComponent { component_type: component.component_type.clone(), entries: Vec::with_capacity(component.entries.len() + 1), @@ -327,20 +330,64 @@ pub(crate) fn itip_export_component( } } - if matches!(export_as, ItipExportAs::Attendee(_)) { - comp.entries.push(ICalendarEntry { - name: ICalendarProperty::RequestStatus, - params: vec![], - values: vec![ - ICalendarValue::Text("2.0".to_string()), - ICalendarValue::Text("Success".to_string()), - ], - }); + match export_as { + ItipExportAs::Attendee(_) => { + comp.entries.push(ICalendarEntry { + name: ICalendarProperty::RequestStatus, + params: vec![], + values: vec![ + ICalendarValue::Text("2.0".to_string()), + ICalendarValue::Text("Success".to_string()), + ] + .into(), + }); + } + ItipExportAs::Organizer(_) => { + if is_event + && !component.has_property(&ICalendarProperty::Dtstart) + && let Some(recurrence_id) = component.property(&ICalendarProperty::RecurrenceId) + { + comp.entries + .push(itip_date_entry(ICalendarProperty::Dtstart, recurrence_id)); + if !component.has_property(&ICalendarProperty::Dtend) + && !component.has_property(&ICalendarProperty::Duration) + && let Some(series_length) = series_length + { + comp.entries.push(ICalendarEntry { + name: ICalendarProperty::Duration, + params: vec![], + values: [ICalendarValue::Duration(series_length.clone())].into(), + }); + } + } + } } comp } +pub(crate) fn itip_date_entry(name: ICalendarProperty, source: &ICalendarEntry) -> ICalendarEntry { + ICalendarEntry { + name, + params: itip_date_params(source), + values: source.values.clone(), + } +} + +pub(crate) fn itip_date_params(source: &ICalendarEntry) -> Vec { + source + .params + .iter() + .filter(|param| { + matches!( + param.name, + ICalendarParameterName::Tzid | ICalendarParameterName::Value + ) + }) + .cloned() + .collect() +} + pub(crate) fn itip_finalize(ical: &mut ICalendar, scheduling_object_ids: &[u16]) { for comp in ical.components.iter_mut() { if comp.component_type.is_scheduling_object() { @@ -366,7 +413,7 @@ pub(crate) fn itip_finalize(ical: &mut ICalendar, scheduling_object_ids: &[u16]) if let Some(ICalendarValue::Integer(seq)) = entry.values.first_mut() { *seq += 1; } else { - entry.values = vec![ICalendarValue::Integer(1)]; + entry.values = [ICalendarValue::Integer(1)].into(); } found_sequence = true; break; @@ -445,12 +492,17 @@ impl ItipMessages { } pub fn queue(self, batch: &mut BatchBuilder) -> trc::Result<()> { - batch.schedule_task(Task::CalendarItipMessage(TaskCalendarItipMessage { - account_id: batch.last_account_id().unwrap().into(), - document_id: batch.last_document_id().unwrap().into(), + let account_id = batch.last_account_id().ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No account is set for an iTIP task.") + })?; + batch.schedule_task_with_document(Task::CalendarItipMessage(TaskCalendarItipMessage { + account_id: account_id.into(), + document_id: Id::default(), messages: self.messages.into(), status: TaskStatus::now(), - })); + }))?; Ok(()) } diff --git a/crates/groupware/src/scheduling/mod.rs b/crates/groupware/src/scheduling/mod.rs index cdc7cb8c..8c06fc7e 100644 --- a/crates/groupware/src/scheduling/mod.rs +++ b/crates/groupware/src/scheduling/mod.rs @@ -9,15 +9,15 @@ use ahash::{AHashMap, AHashSet}; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ - ICalendar, ICalendarComponent, ICalendarDuration, ICalendarEntry, ICalendarMethod, - ICalendarParameter, ICalendarParticipationRole, ICalendarParticipationStatus, - ICalendarPeriod, ICalendarProperty, ICalendarRecurrenceRule, - ICalendarScheduleForceSendValue, ICalendarStatus, ICalendarUserTypes, ICalendarValue, Uri, + ICalendarComponent, ICalendarDuration, ICalendarEntry, ICalendarMethod, ICalendarParameter, + ICalendarParticipationRole, ICalendarParticipationStatus, ICalendarPeriod, + ICalendarProperty, ICalendarRecurrenceRule, ICalendarScheduleForceSendValue, + ICalendarStatus, ICalendarUserTypes, ICalendarValue, Uri, }, }; use indexmap::IndexSet; use registry::schema::structs::TaskCalendarItipContents; -use std::{fmt::Display, hash::Hash}; +use std::{cmp::Ordering, fmt::Display, hash::Hash}; use utils::sanitize_email; pub mod attendee; @@ -28,16 +28,17 @@ pub mod format; pub mod inbound; pub mod itip; pub mod organizer; +pub mod recipient; pub mod snapshot; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ItipSnapshots<'x> { pub organizer: Organizer<'x>, pub uid: &'x str, pub components: AHashMap>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ItipSnapshot<'x> { pub comp_id: u16, pub comp: &'x ICalendarComponent, @@ -46,17 +47,18 @@ pub struct ItipSnapshot<'x> { pub entries: ItipEntries<'x>, pub sequence: Option, pub request_status: Vec<&'x str>, + pub recurrence_time: Option, } pub type ItipEntries<'x> = IndexSet, ahash::RandomState>; -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ItipEntry<'x> { pub name: &'x ICalendarProperty, pub value: ItipEntryValue<'x>, } -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum ItipEntryValue<'x> { DateTime(ItipDateTime<'x>), Period(&'x ICalendarPeriod), @@ -67,7 +69,7 @@ pub enum ItipEntryValue<'x> { Integer(i64), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ItipDateTime<'x> { pub date: &'x PartialDateTime, pub tz_id: Option<&'x str>, @@ -75,20 +77,20 @@ pub struct ItipDateTime<'x> { pub timestamp: i64, } -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum InstanceId { Main, Recurrence(RecurrenceId), } -#[derive(Debug, PartialOrd, Ord)] +#[derive(Debug, Clone)] pub struct RecurrenceId { pub entry_id: u16, pub date: i64, pub this_and_future: bool, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Attendee<'x> { pub entry_id: u16, pub email: Email, @@ -104,7 +106,7 @@ pub struct Attendee<'x> { pub force_send: Option<&'x ICalendarScheduleForceSendValue>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Organizer<'x> { pub entry_id: u16, pub email: Email, @@ -113,7 +115,7 @@ pub struct Organizer<'x> { pub force_send: Option<&'x ICalendarScheduleForceSendValue>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Email { pub email: String, pub is_local: bool, @@ -140,6 +142,7 @@ pub enum ItipError { OutOfSequence, OrganizerIsLocalAddress, SenderIsNotOrganizerNorAttendee, + SenderIsNotOrganizer, SenderIsNotParticipant(String), UnknownParticipant(String), UnsupportedMethod(ICalendarMethod), @@ -149,6 +152,7 @@ pub enum ItipError { QuotaExceeded, NoDefaultCalendar, AutoAddDisabled, + TooManyRecipients, } #[derive(Debug)] @@ -190,7 +194,7 @@ pub enum ItipValue { Participants(Vec), } -#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ItipTime { pub start: i64, pub tz_id: u16, @@ -259,21 +263,6 @@ impl Email { } } -pub fn ical_size(ical: &ICalendar) -> usize { - struct SizeWriter(usize); - - impl std::fmt::Write for SizeWriter { - fn write_str(&mut self, text: &str) -> std::fmt::Result { - self.0 += text.len(); - Ok(()) - } - } - - let mut writer = SizeWriter(0); - let _ = ical.write_to(&mut writer); - writer.0 -} - impl PartialEq for Attendee<'_> { fn eq(&self, other: &Self) -> bool { self.email == other.email @@ -328,6 +317,18 @@ impl PartialEq for RecurrenceId { impl Eq for RecurrenceId {} +impl Ord for RecurrenceId { + fn cmp(&self, other: &Self) -> Ordering { + (self.date, self.this_and_future).cmp(&(other.date, other.this_and_future)) + } +} + +impl PartialOrd for RecurrenceId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + impl Hash for RecurrenceId { fn hash(&self, state: &mut H) { self.date.hash(state); @@ -335,6 +336,27 @@ impl Hash for RecurrenceId { } } +impl Ord for ItipParticipant { + fn cmp(&self, other: &Self) -> Ordering { + match (self.is_organizer, other.is_organizer) { + (true, false) => Ordering::Less, + (false, true) => Ordering::Greater, + _ => match (self.name.as_deref(), other.name.as_deref()) { + (Some(name), Some(other_name)) => name + .cmp(other_name) + .then_with(|| self.email.cmp(&other.email)), + _ => self.email.cmp(&other.email), + }, + } + } +} + +impl PartialOrd for ItipParticipant { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + impl PartialEq for ItipDateTime<'_> { fn eq(&self, other: &Self) -> bool { self.timestamp == other.timestamp @@ -356,7 +378,7 @@ impl ItipDateTime<'_> { .tz_id .map(|tz_id| vec![ICalendarParameter::tzid(tz_id.to_string())]) .unwrap_or_default(), - values: vec![ICalendarValue::PartialDateTime(Box::new(self.date.clone()))], + values: [ICalendarValue::PartialDateTime(self.date.clone())].into(), } } } @@ -381,6 +403,7 @@ impl ItipError { | ItipError::OutOfSequence | ItipError::UnknownParticipant(_) | ItipError::UnsupportedMethod(_) + | ItipError::TooManyRecipients ) } } @@ -427,6 +450,7 @@ impl Display for ItipError { ItipError::SenderIsNotOrganizerNorAttendee => { write!(f, "Sender is neither organizer nor attendee") } + ItipError::SenderIsNotOrganizer => write!(f, "Sender is not the organizer"), ItipError::UnknownParticipant(participant) => { write!(f, "Unknown participant: {}", participant) } @@ -444,6 +468,10 @@ impl Display for ItipError { ItipError::AutoAddDisabled => { write!(f, "Auto-adding events is disabled for this account") } + ItipError::TooManyRecipients => write!( + f, + "The number of scheduling message recipients exceeds the maximum allowed." + ), } } } diff --git a/crates/groupware/src/scheduling/organizer.rs b/crates/groupware/src/scheduling/organizer.rs index 265b17e0..f1638d13 100644 --- a/crates/groupware/src/scheduling/organizer.rs +++ b/crates/groupware/src/scheduling/organizer.rs @@ -5,20 +5,62 @@ */ use crate::scheduling::{ - InstanceId, ItipError, ItipMessage, ItipSnapshots, ItipSummary, - event_cancel::build_cancel_component, - itip::{ItipExportAs, itip_add_tz, itip_build_envelope, itip_export_component}, + InstanceId, ItipDateTime, ItipEntryValue, ItipError, ItipMessage, ItipSnapshot, ItipSnapshots, + ItipSummary, RecurrenceId, + event_cancel::CancelScope, + itip::{ + ItipExportAs, itip_add_tz, itip_build_envelope, itip_date_entry, itip_date_params, + itip_export_component, + }, }; use ahash::{AHashMap, AHashSet}; use calcard::{ common::PartialDateTime, icalendar::{ - ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarMethod, - ICalendarParticipationStatus, ICalendarProperty, ICalendarStatus, + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, + ICalendarParticipationStatus, ICalendarProperty, ICalendarStatus, ICalendarValue, }, }; use std::collections::hash_map::Entry; +pub(crate) struct SequenceIncrement<'x, 'y> { + previous: &'x ItipSnapshots<'y>, + component_ids: &'x mut Vec, +} + +impl<'x, 'y> SequenceIncrement<'x, 'y> { + fn new(previous: &'x ItipSnapshots<'y>, component_ids: &'x mut Vec) -> Self { + SequenceIncrement { + previous, + component_ids, + } + } + + fn sequence(&self, instance_id: &InstanceId, instance: &ItipSnapshot<'_>) -> i64 { + let sequence = instance.sequence.unwrap_or_default(); + let previous = self + .previous + .components + .get(instance_id) + .or_else(|| self.previous.main_instance()) + .and_then(|previous| previous.sequence) + .unwrap_or_default(); + if sequence > previous { + sequence + } else { + sequence + 1 + } + } + + fn increment(&mut self, instance_id: &InstanceId, instance: &ItipSnapshot<'_>) -> i64 { + let sequence = self.sequence(instance_id, instance); + if sequence != instance.sequence.unwrap_or_default() { + self.component_ids.push(instance.comp_id); + } + sequence + } +} + pub(crate) fn organizer_handle_update( old_ical: &ICalendar, new_ical: &ICalendar, @@ -29,6 +71,9 @@ pub(crate) fn organizer_handle_update( let mut changed_instances: Vec<(&InstanceId, &str, &ICalendarMethod)> = Vec::new(); let mut increment_sequence = false; let mut changed_properties = AHashSet::new(); + let mut added_exclusions: Vec<&ItipDateTime<'_>> = Vec::new(); + let mut exclusion_recipients: Vec<&str> = Vec::new(); + let mut cancelled_dates: Vec = Vec::new(); for (instance_id, instance) in &new_itip.components { if let Some(old_instance) = old_itip.components.get(instance_id) { @@ -74,17 +119,54 @@ pub(crate) fn organizer_handle_update( increment_sequence = true; } - changed_instances.extend(instance.attendees.iter().filter_map(|attendee| { - if attendee.send_update_messages() { - Some(( - instance_id, - attendee.email.email.as_str(), - &ICalendarMethod::Request, - )) - } else { - None + let mut exclusions = Vec::new(); + let mut is_exclusion_only = instance_id == &InstanceId::Main + && changed_entries + && !changed_attendees + && old_instance + .entries + .difference(&instance.entries) + .next() + .is_none(); + + if is_exclusion_only { + for entry in instance.entries.difference(&old_instance.entries) { + match (entry.name, &entry.value) { + (ICalendarProperty::Exdate, ItipEntryValue::DateTime(date)) => { + exclusions.push(date); + } + _ => { + is_exclusion_only = false; + break; + } + } } - })); + } + + if is_exclusion_only { + added_exclusions = exclusions; + exclusion_recipients.extend( + instance + .attendees + .iter() + .filter(|attendee| attendee.send_update_messages()) + .map(|attendee| attendee.email.email.as_str()), + ); + exclusion_recipients.sort_unstable(); + exclusion_recipients.dedup(); + } else { + changed_instances.extend(instance.attendees.iter().filter_map(|attendee| { + if attendee.send_update_messages() { + Some(( + instance_id, + attendee.email.email.as_str(), + &ICalendarMethod::Request, + )) + } else { + None + } + })); + } } } else if instance_id != &InstanceId::Main { changed_properties.insert(&ICalendarProperty::Exdate); @@ -124,13 +206,19 @@ pub(crate) fn organizer_handle_update( })); changed_properties.insert(&ICalendarProperty::Exdate); increment_sequence = true; + + if let InstanceId::Recurrence(recurrence_id) = instance_id { + cancelled_dates.push(recurrence_id.date); + } } else { return Err(ItipError::CannotModifyInstance); } } } - if changed_instances.is_empty() { + added_exclusions.retain(|date| !cancelled_dates.contains(&date.timestamp)); + + if changed_instances.is_empty() && added_exclusions.is_empty() { return Err(ItipError::NothingToSend); } @@ -169,24 +257,23 @@ pub(crate) fn organizer_handle_update( } } - // Build summary of changed properties - let new_summary = new_itip - .main_instance_or_default() - .build_summary(Some(&new_itip.organizer), &[]); - let old_summary = old_itip - .main_instance_or_default() - .build_summary(Some(&old_itip.organizer), &new_summary); - // Prepare full updates let mut messages = Vec::new(); if !send_full_update.is_empty() { match organizer_request_full( new_ical, &new_itip, - increment_sequence.then_some(increment_sequences), + increment_sequence + .then(|| SequenceIncrement::new(&old_itip, &mut *increment_sequences)), false, ) { Ok(messages_) => { + let new_summary = new_itip + .main_instance_or_default() + .build_summary(Some(&new_itip.organizer), &[]); + let old_summary = old_itip + .main_instance_or_default() + .build_summary(Some(&old_itip.organizer), &new_summary); for mut message in messages_ { message.summary = ItipSummary::Update { method: ICalendarMethod::Request, @@ -217,13 +304,20 @@ pub(crate) fn organizer_handle_update( } } + let mut increment = increment_sequence + .then(|| SequenceIncrement::new(&old_itip, &mut *increment_sequences)); + let main_sequence = new_itip + .main_instance() + .map(|instance| match &increment { + Some(increment) => increment.sequence(&InstanceId::Main, instance), + None => instance.sequence.unwrap_or_default(), + }) + .unwrap_or_default(); let dt_stamp = PartialDateTime::now(); + let series_length = new_itip.main_instance().and_then(ItipSnapshot::length); for ((method, instances), emails) in updates { - let (mut ical, mut itip, is_cancel) = if matches!(method, ICalendarMethod::Cancel) { - (old_ical, &old_itip, true) - } else { - (new_ical, &new_itip, false) - }; + let is_cancel = matches!(method, ICalendarMethod::Cancel); + let mut tz_source = if is_cancel { old_ical } else { new_ical }; // Prepare iTIP message let mut message = ICalendar { @@ -231,71 +325,93 @@ pub(crate) fn organizer_handle_update( }; message.components.push(itip_build_envelope(method.clone())); - let mut increment_sequences = Vec::new(); - - for instance_id in instances { - let comp = match itip.components.get(instance_id) { - Some(comp) => comp, - None => { - // New component added with CANCELLED status - ical = new_ical; - itip = &new_itip; - itip.components.get(instance_id).unwrap() + for instance_id in &instances { + let old_instance = old_itip.components.get(*instance_id); + let new_instance = new_itip.components.get(*instance_id); + let component = if is_cancel { + let Some(instance) = old_instance.or(new_instance) else { + continue; + }; + if old_instance.is_none() { + tz_source = new_ical; } - }; - // Prepare component for iTIP - let sequence = if increment_sequence { - comp.sequence.unwrap_or_default() + 1 + let sequence = match (new_instance, &mut increment) { + (None, _) => (instance.sequence.unwrap_or_default() + 1).max(main_sequence), + (Some(new_instance), Some(increment)) => increment + .increment(instance_id, new_instance) + .max(old_instance.map_or(0, |old_instance| { + old_instance.sequence.unwrap_or_default() + 1 + })), + (Some(new_instance), None) => new_instance + .sequence + .or(instance.sequence) + .unwrap_or_default(), + }; + let scope = if old_instance.is_some() + && new_instance.is_some_and(|new_instance| { + !matches!(new_instance.comp.status(), Some(ICalendarStatus::Cancelled)) + }) { + CancelScope::Attendees(&emails) + } else { + CancelScope::Instance(&emails) + }; + scope.build_component(instance.comp, sequence, dt_stamp.clone()) } else { - comp.sequence.unwrap_or_default() - }; - let orig_component = comp.comp; - let component = if !is_cancel { - if increment_sequence { - increment_sequences.push(comp.comp_id); - } + let Some(instance) = new_instance else { + continue; + }; + let sequence = match &mut increment { + Some(increment) => increment.increment(instance_id, instance), + None => instance.sequence.unwrap_or_default(), + }; // Export component with updated sequence and participation status itip_export_component( - orig_component, - itip.uid, + instance.comp, + new_itip.uid, &dt_stamp, sequence, ItipExportAs::Organizer(&ICalendarParticipationStatus::NeedsAction), + series_length.as_ref(), ) - } else { - build_cancel_component(orig_component, sequence, dt_stamp.clone(), &emails) }; // Add component to message let comp_id = message.components.len() as u32; message.components.push(component); - message.components[0].component_ids.push(comp_id); + if let Some(root) = message.components.first_mut() { + root.component_ids.push(comp_id); + } } // Add timezones - itip_add_tz(&mut message, ical); + itip_add_tz(&mut message, tz_source); + + let current_guests = instances + .iter() + .any(|instance_id| new_itip.components.contains_key(*instance_id)) + .then_some(&new_itip.organizer); + let current = new_itip.build_instances_summary(&instances, current_guests, &[]); + let previous = + old_itip.build_instances_summary(&instances, Some(&old_itip.organizer), ¤t); messages.push(ItipMessage { - from: itip.organizer.email.email.clone(), + from: new_itip.organizer.email.email.clone(), from_organizer: true, to: emails.into_iter().map(|e| e.to_string()).collect(), - summary: if method == &ICalendarMethod::Cancel { - ItipSummary::Cancel( - new_summary - .iter() - .chain(old_summary.iter()) - .map(|summary| (&summary.name, summary)) - .collect::>() - .into_values() - .cloned() - .collect(), - ) + summary: if is_cancel { + let mut fields = current; + for field in previous { + if !fields.iter().any(|existing| existing.name == field.name) { + fields.push(field); + } + } + ItipSummary::Cancel(fields) } else { ItipSummary::Update { method: method.clone(), - current: new_summary.clone(), - previous: old_summary.clone(), + current, + previous, } }, message, @@ -303,13 +419,143 @@ pub(crate) fn organizer_handle_update( } } + if !added_exclusions.is_empty() + && !exclusion_recipients.is_empty() + && send_full_update.is_empty() + && let Some(main_instance) = new_itip.main_instance() + && main_instance.entries.iter().any(|entry| { + matches!( + entry.name, + ICalendarProperty::Rrule | ICalendarProperty::Rdate + ) + }) + { + let mut increment = increment_sequence + .then(|| SequenceIncrement::new(&old_itip, &mut *increment_sequences)); + let sequence = match &mut increment { + Some(increment) => increment.increment(&InstanceId::Main, main_instance), + None => main_instance.sequence.unwrap_or_default(), + }; + let dt_stamp = PartialDateTime::now(); + let mut message = ICalendar { + components: Vec::with_capacity(added_exclusions.len() + 1), + }; + message + .components + .push(itip_build_envelope(ICalendarMethod::Cancel)); + let mut cancelled_instances: Vec = Vec::with_capacity(added_exclusions.len()); + let mut exdate_sources: AHashMap<(Option<&str>, &PartialDateTime), &ICalendarEntry> = + AHashMap::with_capacity(added_exclusions.len()); + let mut overrides_by_date = AHashMap::with_capacity(new_itip.components.len()); + + for entry in &main_instance.comp.entries { + if entry.name == ICalendarProperty::Exdate { + let tz_id = entry.tz_id(); + for value in &entry.values { + if let ICalendarValue::PartialDateTime(value) = value { + exdate_sources.entry((tz_id, value)).or_insert(entry); + } + } + } + } + for (instance_id, instance) in &new_itip.components { + if let InstanceId::Recurrence(recurrence_id) = instance_id { + overrides_by_date + .entry(recurrence_id.date) + .or_insert((instance_id, instance)); + } + } + + for date in added_exclusions { + let Some(source) = exdate_sources.get(&(date.tz_id, date.date)).copied() else { + continue; + }; + + let overridden = overrides_by_date.get(&date.timestamp).copied(); + let instance_id = match overridden { + Some((instance_id, _)) => instance_id.clone(), + None => InstanceId::Recurrence(RecurrenceId { + entry_id: 0, + date: date.timestamp, + this_and_future: false, + }), + }; + let overridden = overridden.map(|(_, instance)| instance); + let params = itip_date_params(source); + let value = ICalendarValue::PartialDateTime(date.date.clone()); + let dt_start = match overridden + .and_then(|instance| instance.comp.property(&ICalendarProperty::Dtstart)) + { + Some(entry) => itip_date_entry(ICalendarProperty::Dtstart, entry), + None => ICalendarEntry { + name: ICalendarProperty::Dtstart, + params: params.clone(), + values: [value.clone()].into(), + }, + }; + + let mut component = CancelScope::Instance(&exclusion_recipients).build_component( + main_instance.comp, + sequence, + dt_stamp.clone(), + ); + component.entries.retain(|entry| { + !matches!( + entry.name, + ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Duration + | ICalendarProperty::Due + ) + }); + component.entries.push(ICalendarEntry { + name: ICalendarProperty::RecurrenceId, + params, + values: [value].into(), + }); + component.entries.push(dt_start); + + let comp_id = message.components.len() as u32; + message.components.push(component); + if let Some(root) = message.components.first_mut() { + root.component_ids.push(comp_id); + } + cancelled_instances.push(instance_id); + } + + if !cancelled_instances.is_empty() { + itip_add_tz(&mut message, new_ical); + + let mut fields = new_itip.build_instances_summary(&cancelled_instances, None, &[]); + fields.push(main_instance.guest_field(&new_itip.organizer)); + + messages.push(ItipMessage { + from: new_itip.organizer.email.email.clone(), + from_organizer: true, + to: exclusion_recipients + .into_iter() + .map(|email| email.to_string()) + .collect(), + summary: ItipSummary::Cancel(fields), + message, + }); + } + } + + if messages.is_empty() { + return Err(ItipError::NothingToSend); + } + + increment_sequences.sort_unstable(); + increment_sequences.dedup(); + Ok(messages) } pub(crate) fn organizer_request_full( ical: &ICalendar, itip: &ItipSnapshots<'_>, - mut increment_sequence: Option<&mut Vec>, + mut increment: Option>, is_first_request: bool, ) -> Result>, ItipError> { // Prepare iTIP message @@ -321,19 +567,18 @@ pub(crate) fn organizer_request_full( let mut recipients = AHashSet::new(); let mut copy_components = AHashSet::new(); + let series_length = itip.main_instance().and_then(ItipSnapshot::length); - for comp in itip.components.values() { + for (instance_id, comp) in &itip.components { // Skip private components if comp.attendees.is_empty() { continue; } // Prepare component for iTIP - let sequence = if let Some(increment_sequence) = &mut increment_sequence { - increment_sequence.push(comp.comp_id); - comp.sequence.unwrap_or_default() + 1 - } else { - comp.sequence.unwrap_or_default() + let sequence = match &mut increment { + Some(increment) => increment.increment(instance_id, comp), + None => comp.sequence.unwrap_or_default(), }; let orig_component = &ical.components[comp.comp_id as usize]; let mut component = itip_export_component( @@ -342,6 +587,7 @@ pub(crate) fn organizer_request_full( &dt_stamp, sequence, ItipExportAs::Organizer(&ICalendarParticipationStatus::NeedsAction), + series_length.as_ref(), ); // Add VALARM sub-components @@ -401,3 +647,4 @@ pub(crate) fn organizer_request_full( Err(ItipError::NothingToSend) } } + diff --git a/crates/groupware/src/scheduling/recipient.rs b/crates/groupware/src/scheduling/recipient.rs new file mode 100644 index 00000000..eeaeb2f0 --- /dev/null +++ b/crates/groupware/src/scheduling/recipient.rs @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + calendar::EVENT_HIDE_ATTENDEES, + scheduling::{ItipError, ItipMessage}, +}; +use calcard::icalendar::ICalendar; +use common::config::groupware::GroupwareConfig; +use split::RecipientSplit; + +pub(crate) mod series; +mod split; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AttendeeVisibility { + #[default] + All, + RecipientOnly, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecipientPolicy { + pub visibility: AttendeeVisibility, + pub max_recipients: usize, + pub max_instances: usize, +} + +impl AttendeeVisibility { + pub fn from_event_flags(flags: u16) -> Self { + if flags & EVENT_HIDE_ATTENDEES != 0 { + AttendeeVisibility::RecipientOnly + } else { + AttendeeVisibility::All + } + } +} + +impl RecipientPolicy { + pub fn new(config: &GroupwareConfig, event_flags: u16) -> Self { + RecipientPolicy { + visibility: AttendeeVisibility::from_event_flags(event_flags), + max_recipients: config.itip_outbound_max_recipients, + max_instances: config.max_ical_instances, + } + } +} + +pub(crate) fn itip_messages_per_recipient( + messages: Vec>, + policy: RecipientPolicy, +) -> Result>, ItipError> { + if messages + .iter() + .map(|message| message.to.len()) + .sum::() + >= policy.max_recipients + { + return Err(ItipError::TooManyRecipients); + } + + let mut result = Vec::with_capacity(messages.len()); + for message in messages { + if message.from_organizer { + RecipientSplit::split(message, policy, &mut result); + } else { + result.push(message); + } + } + Ok(result) +} + diff --git a/crates/groupware/src/scheduling/recipient/series.rs b/crates/groupware/src/scheduling/recipient/series.rs new file mode 100644 index 00000000..2987baa5 --- /dev/null +++ b/crates/groupware/src/scheduling/recipient/series.rs @@ -0,0 +1,483 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::scheduling::{ItipTime, itip::itip_date_entry}; +use calcard::{ + common::{ + PartialDateTime, + timezone::{Tz, ZonedDateTime}, + }, + icalendar::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, + ICalendarParameterName, ICalendarPeriod, ICalendarProperty, ICalendarRecurrenceRule, + ICalendarValue, timezone::TzResolver, + }, +}; +use std::{borrow::Cow, cell::OnceCell}; + +mod occurrence; + +const SECONDS_PER_DAY: i64 = 86400; +const EXPANSION_COMPONENT_ID: u32 = 1; + +pub(crate) struct Series<'x> { + ical: &'x ICalendar, + tz: TzResolver<&'x str>, + master: Option>, + max_instances: usize, + occurrences: OnceCell, +} + +struct Master<'x> { + component: &'x ICalendarComponent, + start: Option, + start_form: StartForm, + tz_id: Option<&'x str>, + has_rule: bool, + rdates: Vec, +} + +#[derive(Debug, Clone, Copy)] +enum StartForm { + Date, + Floating, + Utc, +} + +pub(super) struct Recurrence<'x> { + entry: &'x ICalendarEntry, + start: Option, + this_and_future: bool, +} + +struct RuleOccurrences { + starts: Vec, + is_complete: bool, +} + +pub(super) struct SeriesPlan<'x> { + cut: Option, + has_instances: bool, + exdates: Vec<&'x ICalendarEntry>, + removed_rdates: Vec, +} + +trait InstanceStart { + fn instance_start(&self) -> Option<&PartialDateTime>; +} + +impl<'x> Series<'x> { + pub(crate) fn new( + ical: &'x ICalendar, + master: Option<&'x ICalendarComponent>, + max_instances: usize, + ) -> Self { + let tz = ical.build_tz_resolver(); + let master = master.map(|component| Master::new(component, &tz)); + Series { + ical, + tz, + master, + max_instances, + occurrences: OnceCell::new(), + } + } + + pub(super) fn master_component(&self) -> Option<&'x ICalendarComponent> { + self.master.as_ref().map(|master| master.component) + } + + pub(super) fn entry_time(&self, entry: &ICalendarEntry) -> Option { + let tz = self.tz.resolve_or_default(entry.tz_id()); + entry + .values + .first() + .and_then(|value| value.as_partial_date_time()) + .and_then(|date| date.to_date_time_with_tz(tz)) + .map(|date| ItipTime { + start: date.timestamp(), + tz_id: tz.as_id(), + }) + } + + pub(super) fn recurrence(&self, entry: &'x ICalendarEntry) -> Recurrence<'x> { + Recurrence { + entry, + start: entry + .values + .first() + .and_then(|value| value.as_partial_date_time()) + .and_then(|date| self.timestamp(date, entry.tz_id())), + this_and_future: entry + .params + .iter() + .any(|param| param.name == ICalendarParameterName::Range), + } + } + + pub(super) fn plan(&self, excluded: &[&Recurrence<'x>]) -> SeriesPlan<'x> { + let cut = excluded + .iter() + .filter(|recurrence| recurrence.this_and_future) + .filter_map(|recurrence| recurrence.start) + .min(); + let mut plan = SeriesPlan { + cut, + has_instances: cut.is_none_or(|cut| { + self.master + .as_ref() + .and_then(|master| master.start) + .is_none_or(|start| start < cut) + }), + exdates: Vec::new(), + removed_rdates: Vec::new(), + }; + + if plan.has_instances { + for recurrence in excluded + .iter() + .filter(|recurrence| !recurrence.this_and_future) + { + match recurrence.start { + Some(start) if cut.is_some_and(|cut| start >= cut) => {} + Some(start) => { + let is_rdate = self.is_rdate(start); + if is_rdate { + plan.removed_rdates.push(start); + } + if self.is_rule_instance(start, is_rdate) { + plan.exdates.push(recurrence.entry); + } + } + None => plan.exdates.push(recurrence.entry), + } + } + } + + plan + } + + fn timestamp(&self, date: &PartialDateTime, tz_id: Option<&str>) -> Option { + self.zoned(date, tz_id).map(|date| date.timestamp()) + } + + fn zoned(&self, date: &PartialDateTime, tz_id: Option<&str>) -> Option { + date.to_date_time_with_tz(self.tz.resolve_or_default( + tz_id.or_else(|| self.master.as_ref().and_then(|master| master.tz_id)), + )) + } + + pub(crate) fn has_occurrence(&self, start: i64) -> bool { + let Some(master) = &self.master else { + return false; + }; + let is_rule_instance = || { + let occurrences = self.occurrences(); + occurrences.contains(start) + || (master + .start + .is_some_and(|master_start| start > master_start) + && occurrences.is_unknown_at(start)) + }; + let is_instance = self.is_rdate(start) + || (master.has_rule && is_rule_instance()) + || (!master.has_rule && !master.rdates.is_empty() && master.start == Some(start)); + is_instance && !self.is_exdate(start) + } + + fn is_rdate(&self, start: i64) -> bool { + self.master + .as_ref() + .is_some_and(|master| master.rdates.binary_search(&start).is_ok()) + } + + fn is_exdate(&self, start: i64) -> bool { + self.master.as_ref().is_some_and(|master| { + master + .component + .properties(&ICalendarProperty::Exdate) + .any(|entry| { + entry + .values + .iter() + .filter_map(|value| value.as_partial_date_time()) + .any(|date| self.timestamp(date, entry.tz_id()) == Some(start)) + }) + }) + } + + fn is_rule_instance(&self, start: i64, is_rdate: bool) -> bool { + let Some(master) = &self.master else { + return false; + }; + match master.start { + Some(master_start) if start == master_start => true, + Some(master_start) if start < master_start => false, + _ if !master.has_rule => false, + _ => { + let occurrences = self.occurrences(); + occurrences.contains(start) || (!is_rdate && occurrences.is_unknown_at(start)) + } + } + } + + fn occurrences(&self) -> &RuleOccurrences { + self.occurrences.get_or_init(|| match &self.master { + Some(master) if master.has_rule => { + RuleOccurrences::expand(self.ical, master, self.max_instances) + } + _ => RuleOccurrences { + starts: Vec::new(), + is_complete: true, + }, + }) + } + + fn truncate(&self, rule: &ICalendarRecurrenceRule, cut: i64) -> ICalendarRecurrenceRule { + let Some(master) = &self.master else { + return rule.clone(); + }; + if rule.count.is_some() && self.occurrences().ends_before(cut) { + return rule.clone(); + } + + let (until, until_timestamp) = master.start_form.until_before(cut); + let until = match &rule.until { + Some(current) + if current + .to_date_time_with_tz(Tz::Floating) + .is_some_and(|current| current.timestamp() <= until_timestamp) => + { + current.clone() + } + _ => until, + }; + + ICalendarRecurrenceRule { + count: None, + until: Some(until), + ..rule.clone() + } + } +} + +impl<'x> Master<'x> { + fn new(component: &'x ICalendarComponent, tz: &TzResolver<&'x str>) -> Self { + let start_entry = component.property(&ICalendarProperty::Dtstart); + let tz_id = start_entry.and_then(|entry| entry.tz_id()); + let start_date = start_entry + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_partial_date_time()); + let resolve = |date: &PartialDateTime, tz_id: Option<&str>| { + date.to_date_time_with_tz(tz.resolve_or_default(tz_id)) + .map(|date| date.timestamp()) + }; + let mut rdates = component + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Rdate) + .flat_map(|entry| { + let entry_tz_id = entry.tz_id().or(tz_id); + entry + .values + .iter() + .filter_map(|value| value.instance_start()) + .filter_map(move |date| resolve(date, entry_tz_id)) + }) + .collect::>(); + rdates.sort_unstable(); + + Master { + component, + start: start_date.and_then(|date| resolve(date, tz_id)), + start_form: match start_date { + Some(date) if !date.has_time() => StartForm::Date, + Some(date) if !date.has_zone() && tz_id.is_none() => StartForm::Floating, + _ => StartForm::Utc, + }, + tz_id, + has_rule: component.has_property(&ICalendarProperty::Rrule), + rdates, + } + } +} + +impl StartForm { + fn until_before(self, cut: i64) -> (PartialDateTime, i64) { + let last = cut - 1; + match self { + StartForm::Date => ( + PartialDateTime::from_date_timestamp(last), + last.div_euclid(SECONDS_PER_DAY) * SECONDS_PER_DAY, + ), + StartForm::Floating => (PartialDateTime::from_naive_timestamp(last), last), + StartForm::Utc => (PartialDateTime::from_utc_timestamp(last), last), + } + } +} + +impl RuleOccurrences { + fn expand(ical: &ICalendar, master: &Master<'_>, limit: usize) -> Self { + let root = ICalendarComponent { + component_type: ICalendarComponentType::VCalendar, + entries: Vec::new(), + component_ids: vec![EXPANSION_COMPONENT_ID], + }; + let rule = ICalendarComponent { + component_type: master.component.component_type.clone(), + entries: master + .component + .entries + .iter() + .filter(|entry| { + matches!( + entry.name, + ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Duration + | ICalendarProperty::Due + | ICalendarProperty::Rrule + ) + }) + .cloned() + .collect(), + component_ids: Vec::new(), + }; + let timezones = ical + .components + .iter() + .filter(|component| { + component.component_type == ICalendarComponentType::VTimezone + && master.tz_id.is_some() + && component + .property(&ICalendarProperty::Tzid) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_text()) + == master.tz_id + }) + .map(|component| ICalendarComponent { + component_type: ICalendarComponentType::VTimezone, + entries: component.entries.clone(), + component_ids: Vec::new(), + }); + let expanded = ICalendar { + components: [root, rule].into_iter().chain(timezones).collect(), + } + .expand_dates(Tz::Floating, limit.saturating_add(1)); + + let mut starts = expanded + .events + .iter() + .map(|event| event.start.timestamp()) + .collect::>(); + starts.sort_unstable(); + + RuleOccurrences { + is_complete: expanded.errors.is_empty() && starts.len() <= limit, + starts, + } + } + + fn contains(&self, start: i64) -> bool { + self.starts.binary_search(&start).is_ok() + } + + fn is_unknown_at(&self, start: i64) -> bool { + !self.is_complete && self.starts.last().is_none_or(|last| *last < start) + } + + fn ends_before(&self, start: i64) -> bool { + self.is_complete && self.starts.last().is_none_or(|last| *last < start) + } +} + +impl<'x> Recurrence<'x> { + pub(super) fn entry(&self) -> &'x ICalendarEntry { + self.entry + } + + pub(super) fn is_this_and_future(&self) -> bool { + self.this_and_future + } +} + +impl<'x> SeriesPlan<'x> { + pub(super) fn has_instances(&self) -> bool { + self.has_instances + } + + pub(super) fn exdate_count(&self) -> usize { + self.exdates.len() + } + + pub(super) fn entry<'y>( + &self, + series: &Series<'_>, + entry: &'y ICalendarEntry, + ) -> Option> { + match (&entry.name, self.cut) { + (ICalendarProperty::Rdate, cut) if cut.is_some() || !self.removed_rdates.is_empty() => { + let tz_id = entry.tz_id(); + let values = entry + .values + .iter() + .filter(|value| { + value + .instance_start() + .and_then(|date| series.timestamp(date, tz_id)) + .is_none_or(|start| { + cut.is_none_or(|cut| start < cut) + && !self.removed_rdates.contains(&start) + }) + }) + .cloned() + .collect::>(); + (!values.is_empty()).then(|| { + Cow::Owned(ICalendarEntry { + name: ICalendarProperty::Rdate, + params: entry.params.clone(), + values: values.into(), + }) + }) + } + (ICalendarProperty::Rrule, Some(cut)) => Some(Cow::Owned(ICalendarEntry { + name: ICalendarProperty::Rrule, + params: entry.params.clone(), + values: entry + .values + .iter() + .map(|value| match value { + ICalendarValue::RecurrenceRule(rule) => { + ICalendarValue::RecurrenceRule(Box::new(series.truncate(rule, cut))) + } + value => value.clone(), + }) + .collect(), + })), + _ => Some(Cow::Borrowed(entry)), + } + } + + pub(super) fn exdates(&self) -> impl Iterator + '_ { + self.exdates + .iter() + .map(|entry| itip_date_entry(ICalendarProperty::Exdate, entry)) + } +} + +impl InstanceStart for ICalendarValue { + fn instance_start(&self) -> Option<&PartialDateTime> { + match self { + ICalendarValue::PartialDateTime(date) => Some(date), + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, .. } | ICalendarPeriod::Duration { start, .. } => { + Some(start) + } + }, + _ => None, + } + } +} diff --git a/crates/groupware/src/scheduling/recipient/series/occurrence.rs b/crates/groupware/src/scheduling/recipient/series/occurrence.rs new file mode 100644 index 00000000..1c515733 --- /dev/null +++ b/crates/groupware/src/scheduling/recipient/series/occurrence.rs @@ -0,0 +1,287 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Series; +use crate::scheduling::itip::itip_date_params; +use ahash::RandomState; +use calcard::{ + common::{ + PartialDateTime, + timezone::{NominalDuration, Tz, ZonedDateTime}, + }, + icalendar::{ + ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarParameterName, + ICalendarPeriod, ICalendarProperty, ICalendarValue, + }, +}; +use indexmap::IndexSet; +use jiff::Timestamp; + +#[derive(Debug, Clone, Copy)] +enum DateForm { + Date, + Floating, + Utc, + Zoned(Tz), +} + +impl Series<'_> { + pub(crate) fn occurrence( + &self, + mut component: ICalendarComponent, + recurrence_id: &ICalendarEntry, + original_start: i64, + ) -> Option> { + let series_start = self + .master + .as_ref()? + .component + .property(&ICalendarProperty::Dtstart)?; + let series_form = self.date_form(series_start)?; + let series_tz = self.tz.resolve_or_default(series_start.tz_id()); + let original = match self.entry_zoned(recurrence_id) { + Some(original) if original.timestamp() == original_start => original, + _ => series_tz.from_timestamp(Timestamp::from_second(original_start).ok()?), + } + .with_timezone(series_tz); + let source_original = component + .property(&ICalendarProperty::RecurrenceId) + .and_then(|entry| self.entry_zoned(entry)) + .map(|source_original| source_original.with_timezone(series_tz)); + let has_start = component.has_property(&ICalendarProperty::Dtstart); + let has_end = [ + ICalendarProperty::Dtend, + ICalendarProperty::Duration, + ICalendarProperty::Due, + ] + .iter() + .any(|name| component.has_property(name)); + let source_start = component + .property(&ICalendarProperty::Dtstart) + .and_then(|entry| self.entry_zoned(entry)) + .or(source_original)?; + let start = match source_original { + Some(source_original) => shift(original, source_original, source_start)?, + None => original, + }; + + for entry in &mut component.entries { + let instant = match entry.name { + ICalendarProperty::Dtstart => start, + ICalendarProperty::Dtend | ICalendarProperty::Due => { + self.end(entry, source_start, start)? + } + _ => continue, + }; + let form = self.date_form(entry)?; + entry.values = [ICalendarValue::PartialDateTime(form.value(instant))].into(); + } + + let period_end = self.period_end(&component.component_type, original_start); + component.entries.retain(|entry| match entry.name { + ICalendarProperty::Rrule + | ICalendarProperty::Rdate + | ICalendarProperty::Exdate + | ICalendarProperty::Exrule + | ICalendarProperty::RecurrenceId => false, + ICalendarProperty::Dtend | ICalendarProperty::Due | ICalendarProperty::Duration => { + period_end.is_none() + } + _ => true, + }); + if !has_start { + component.entries.push(ICalendarEntry { + name: ICalendarProperty::Dtstart, + params: itip_date_params(series_start), + values: [ICalendarValue::PartialDateTime(series_form.value(start))].into(), + }); + } + let end = match period_end { + Some(period_end) => Some(period_end), + None if !has_start && !has_end => self.series_end(start), + None => None, + }; + component.entries.extend(end); + component.entries.push(ICalendarEntry { + name: ICalendarProperty::RecurrenceId, + params: itip_date_params(series_start) + .into_iter() + .chain( + recurrence_id + .params + .iter() + .filter(|param| param.name == ICalendarParameterName::Range) + .cloned(), + ) + .collect(), + values: [ICalendarValue::PartialDateTime(series_form.value(original))].into(), + }); + + Some(self.with_subcomponents(component)) + } + + fn end( + &self, + entry: &ICalendarEntry, + source_start: ZonedDateTime, + start: ZonedDateTime, + ) -> Option { + let source_end = self.entry_zoned(entry)?; + match self.date_form(entry)? { + DateForm::Date => start + .checked_add_nominal(NominalDuration::new(source_end.days_since(source_start), 0)), + DateForm::Floating | DateForm::Utc | DateForm::Zoned(_) => { + start.checked_add(source_end.signed_duration_since(source_start)) + } + } + } + + fn series_end(&self, start: ZonedDateTime) -> Option { + let master = self.master.as_ref()?.component; + if let Some(duration) = master.property(&ICalendarProperty::Duration) { + return Some(duration.clone()); + } + let series_start = self.entry_zoned(master.property(&ICalendarProperty::Dtstart)?)?; + [ICalendarProperty::Dtend, ICalendarProperty::Due] + .iter() + .find_map(|name| { + let entry = master.property(name)?; + Some(ICalendarEntry { + name: name.clone(), + params: itip_date_params(entry), + values: [ICalendarValue::PartialDateTime( + self.date_form(entry)? + .value(self.end(entry, series_start, start)?), + )] + .into(), + }) + }) + } + + fn period_end( + &self, + component_type: &ICalendarComponentType, + original_start: i64, + ) -> Option { + self.master + .as_ref()? + .component + .properties(&ICalendarProperty::Rdate) + .find_map(|entry| { + let tz_id = entry.tz_id(); + entry.values.iter().find_map(|value| match value { + ICalendarValue::Period(period) => match period.as_ref() { + ICalendarPeriod::Range { start, end } + if self.timestamp(start, tz_id) == Some(original_start) => + { + Some(ICalendarEntry { + name: if component_type == &ICalendarComponentType::VTodo { + ICalendarProperty::Due + } else { + ICalendarProperty::Dtend + }, + params: entry + .params + .iter() + .filter(|param| param.name == ICalendarParameterName::Tzid) + .cloned() + .collect(), + values: [ICalendarValue::PartialDateTime(end.clone())].into(), + }) + } + ICalendarPeriod::Duration { start, duration } + if self.timestamp(start, tz_id) == Some(original_start) => + { + Some(ICalendarEntry { + name: ICalendarProperty::Duration, + params: vec![], + values: [ICalendarValue::Duration(duration.clone())].into(), + }) + } + _ => None, + }, + _ => None, + }) + }) + } + + fn entry_zoned(&self, entry: &ICalendarEntry) -> Option { + self.zoned(entry.values.first()?.as_partial_date_time()?, entry.tz_id()) + } + + fn date_form(&self, entry: &ICalendarEntry) -> Option { + let date = entry.values.first()?.as_partial_date_time()?; + Some(if !date.has_time() { + DateForm::Date + } else if date.has_zone() { + DateForm::Utc + } else if let Some(tz_id) = entry.tz_id() { + DateForm::Zoned(self.tz.resolve_or_default(Some(tz_id))) + } else { + DateForm::Floating + }) + } + + fn with_subcomponents(&self, mut component: ICalendarComponent) -> Vec { + let mut subtree: IndexSet = IndexSet::default(); + let mut pending = component + .component_ids + .iter() + .rev() + .copied() + .collect::>(); + while let Some(id) = pending.pop() { + if let Some(subcomponent) = self.ical.components.get(id as usize) + && subtree.insert(id) + { + pending.extend(subcomponent.component_ids.iter().rev()); + } + } + + let relink = |component_ids: &[u32]| { + component_ids + .iter() + .filter_map(|id| subtree.get_index_of(id)) + .map(|index| index as u32 + 1) + .collect::>() + }; + component.component_ids = relink(&component.component_ids); + let mut components = Vec::with_capacity(subtree.len() + 1); + components.push(component); + components.extend( + subtree + .iter() + .filter_map(|id| self.ical.components.get(*id as usize)) + .map(|subcomponent| ICalendarComponent { + component_type: subcomponent.component_type.clone(), + entries: subcomponent.entries.clone(), + component_ids: relink(&subcomponent.component_ids), + }), + ); + components + } +} + +impl DateForm { + fn value(self, instant: ZonedDateTime) -> PartialDateTime { + match self { + DateForm::Date => PartialDateTime::from_date_timestamp(instant.naive_timestamp()), + DateForm::Floating => PartialDateTime::from_naive_timestamp(instant.naive_timestamp()), + DateForm::Utc => PartialDateTime::from_utc_timestamp(instant.timestamp()), + DateForm::Zoned(tz) => { + PartialDateTime::from_naive_timestamp(instant.with_timezone(tz).naive_timestamp()) + } + } + } +} + +fn shift(instant: ZonedDateTime, from: ZonedDateTime, to: ZonedDateTime) -> Option { + if from.timezone() == to.timezone() { + instant.checked_add_nominal(NominalDuration::between(from, to)) + } else { + instant.checked_add(to.signed_duration_since(from)) + } +} diff --git a/crates/groupware/src/scheduling/recipient/split.rs b/crates/groupware/src/scheduling/recipient/split.rs new file mode 100644 index 00000000..785fa228 --- /dev/null +++ b/crates/groupware/src/scheduling/recipient/split.rs @@ -0,0 +1,886 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + AttendeeVisibility, RecipientPolicy, + series::{Recurrence, Series, SeriesPlan}, +}; +use crate::{ + calendar::identity::OwnerEntry, + scheduling::{ + Email, ItipField, ItipMessage, ItipParticipant, ItipSummary, ItipTime, ItipValue, + event_cancel::CancelScope, + itip::{itip_add_tz, itip_build_envelope}, + }, +}; +use ahash::{AHashMap, RandomState}; +use calcard::{ + common::PartialDateTime, + icalendar::{ + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, + ICalendarParameter, ICalendarParameterName, ICalendarProperty, ICalendarValue, Uri, + }, +}; +use indexmap::IndexMap; +use std::{borrow::Cow, iter}; + +const PARTICIPANTS_POINTER: &str = "participants"; + +pub(super) struct RecipientSplit<'x> { + ical: &'x ICalendar, + visibility: AttendeeVisibility, + addresses: AHashMap, + components: Vec>, + series: Series<'x>, +} + +struct SchedulingComponent<'x> { + id: u32, + recurrence: Option>, + attendees: Vec, + participant_ids: Vec<(u32, Cow<'x, str>)>, + owners: Vec<(usize, EntryOwner<'x>)>, + owner_addresses: Vec, + owner_participant_ids: Vec>, + alarms: Vec<(u32, Option)>, +} + +#[derive(Debug, Clone, Copy)] +enum EntryOwner<'x> { + Address(u32), + Participant(&'x str), + Unknown, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct RecipientView { + included: Vec, + excluded: Vec, +} + +trait AddressReferences { + fn address_references(&self) -> impl Iterator; +} + +impl<'x> RecipientSplit<'x> { + pub(super) fn split( + message: ItipMessage, + policy: RecipientPolicy, + result: &mut Vec>, + ) { + let ItipMessage { + from, + from_organizer, + mut to, + summary, + message: ical, + } = message; + to.sort_unstable(); + + let split = RecipientSplit::new(&ical, policy); + let groups = split.groups(to); + if let [(view, _)] = groups.as_slice() + && split.visibility == AttendeeVisibility::All + && view.is_complete(split.components.len()) + { + result.push(ItipMessage { + from, + from_organizer, + to: groups + .into_iter() + .next() + .map(|(_, recipients)| recipients) + .unwrap_or_default(), + summary, + message: ical, + }); + } else { + result.reserve(groups.len()); + result.extend(groups.into_iter().filter_map(|(view, recipients)| { + split.message(&view, recipients, &from, &summary) + })); + } + } + + fn new(ical: &'x ICalendar, policy: RecipientPolicy) -> Self { + let scheduling_components = || { + ical.components + .first() + .into_iter() + .flat_map(|root| root.component_ids.iter()) + .filter_map(|id| { + ical.components + .get(*id as usize) + .filter(|component| component.component_type.is_scheduling_object()) + .map(|component| (*id, component)) + }) + }; + let series = Series::new( + ical, + scheduling_components() + .find(|(_, component)| !component.is_recurrence_override()) + .map(|(_, component)| component), + policy.max_instances, + ); + let is_hidden = policy.visibility == AttendeeVisibility::RecipientOnly; + let mut addresses = AHashMap::new(); + let mut references = Vec::new(); + let mut components = Vec::new(); + + for (id, component) in scheduling_components() { + let mut scheduling = SchedulingComponent { + id, + recurrence: None, + attendees: Vec::new(), + participant_ids: Vec::new(), + owners: Vec::new(), + owner_addresses: Vec::new(), + owner_participant_ids: Vec::new(), + alarms: Vec::new(), + }; + for (index, entry) in component.entries.iter().enumerate() { + match &entry.name { + ICalendarProperty::Attendee => { + let address = entry + .values + .first() + .and_then(|value| value.as_text()) + .and_then(|value| Email::new(value, &[])) + .map(|email| { + let next_id = addresses.len() as u32; + *addresses.entry(email.email).or_insert(next_id) + }); + if let Some(address) = address { + scheduling.attendees.push(address); + if is_hidden && let Some(participant_id) = entry.participant_id() { + scheduling.participant_ids.push((address, participant_id)); + } + if is_hidden && entry.is_owner_entry() { + scheduling.owner_addresses.push(address); + } + } + if is_hidden { + scheduling.owners.push(( + index, + address.map_or(EntryOwner::Unknown, EntryOwner::Address), + )); + } + } + ICalendarProperty::Organizer if is_hidden => { + if let Some(email) = entry + .values + .first() + .and_then(|value| value.as_text()) + .and_then(|value| Email::new(value, &[])) + { + let next_id = addresses.len() as u32; + scheduling + .owner_addresses + .push(*addresses.entry(email.email).or_insert(next_id)); + } + if let Some(participant_id) = entry.participant_id() { + scheduling.owner_participant_ids.push(participant_id); + } + } + ICalendarProperty::RecurrenceId if scheduling.recurrence.is_none() => { + scheduling.recurrence = Some(series.recurrence(entry)); + } + ICalendarProperty::Jsprop if is_hidden => { + if let Some(owner) = EntryOwner::from_jsprop(entry) { + scheduling.owners.push((index, owner)); + } + } + ICalendarProperty::Other(_) + if is_hidden && entry.address_references().next().is_some() => + { + references.push((components.len(), index, entry)); + } + _ => {} + } + } + scheduling.attendees.sort_unstable(); + scheduling.attendees.dedup(); + components.push(scheduling); + } + + if is_hidden { + for (position, index, entry) in references { + if let Some(owner) = EntryOwner::from_references(entry, &addresses) + && let Some(scheduling) = components.get_mut(position) + { + scheduling.owners.push((index, owner)); + } + } + for (scheduling, (_, component)) in components.iter_mut().zip(scheduling_components()) { + scheduling.owners.sort_by_key(|(index, _)| *index); + scheduling.owner_addresses.sort_unstable(); + scheduling.owner_addresses.dedup(); + for (address, participant_id) in &scheduling.participant_ids { + if scheduling.owner_addresses.binary_search(address).is_ok() { + scheduling + .owner_participant_ids + .push(participant_id.clone()); + } + } + scheduling.alarms = component + .component_ids + .iter() + .filter_map(|id| { + let alarm = ical.components.get(*id as usize).filter(|alarm| { + alarm.component_type == ICalendarComponentType::VAlarm + })?; + let mut owners = alarm + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Attendee) + .map(|entry| { + entry + .values + .first() + .and_then(|value| value.as_text()) + .and_then(|value| Email::new(value, &[])) + .and_then(|email| addresses.get(&email.email).copied()) + }); + let owner = owners.next()?; + Some(( + *id, + owners + .all(|other| other == owner) + .then_some(owner) + .flatten(), + )) + }) + .collect(); + } + } + + RecipientSplit { + ical, + visibility: policy.visibility, + addresses, + components, + series, + } + } + + fn groups(&self, recipients: Vec) -> Vec<(RecipientView, Vec)> { + match self.visibility { + AttendeeVisibility::All => { + let mut groups = + IndexMap::, RandomState>::with_capacity_and_hasher( + recipients.len(), + RandomState::new(), + ); + for recipient in recipients { + groups + .entry(self.view(&recipient)) + .or_default() + .push(recipient); + } + groups.into_iter().collect() + } + AttendeeVisibility::RecipientOnly => recipients + .into_iter() + .map(|recipient| (self.view(&recipient), vec![recipient])) + .collect(), + } + } + + fn view(&self, recipient: &str) -> RecipientView { + let address = self.addresses.get(recipient).copied(); + let mut included = Vec::with_capacity(self.components.len()); + let mut excluded = Vec::new(); + let mut attends_main = false; + for component in &self.components { + if address.is_some_and(|address| component.attendees.binary_search(&address).is_ok()) { + attends_main |= component.recurrence.is_none(); + included.push(component.id); + } else if component.recurrence.is_some() { + excluded.push(component.id); + } + } + if included.is_empty() { + included.extend(self.components.iter().map(|component| component.id)); + } + if !attends_main { + excluded.clear(); + } + RecipientView { included, excluded } + } + + fn message( + &self, + view: &RecipientView, + recipients: Vec, + from: &str, + summary: &ItipSummary, + ) -> Option> { + let root = self.ical.components.first()?; + let recipient = recipients + .first() + .filter(|_| self.visibility == AttendeeVisibility::RecipientOnly) + .map(String::as_str); + let address = recipient.and_then(|recipient| self.addresses.get(recipient).copied()); + let plan = (!view.excluded.is_empty()).then(|| { + let mut excluded = view.excluded.iter().peekable(); + self.series.plan( + &self + .components + .iter() + .filter(|component| excluded.next_if(|id| **id == component.id).is_some()) + .filter_map(|component| component.recurrence.as_ref()) + .collect::>(), + ) + }); + let has_main_instances = plan.as_ref().is_none_or(SeriesPlan::has_instances); + + let mut components = vec![ICalendarComponent::default(); self.ical.components.len()]; + let mut root_ids = Vec::with_capacity(root.component_ids.len()); + let mut children = Vec::new(); + let mut has_scheduling = false; + let mut has_main = false; + let mut scheduling = self.components.iter().peekable(); + let mut included = view.included.iter().peekable(); + + for id in &root.component_ids { + let Some(source) = self.ical.components.get(*id as usize) else { + continue; + }; + match scheduling.next_if(|component| component.id == *id) { + Some(component) => { + let is_main = component.recurrence.is_none(); + if included.next_if(|included| **included == *id).is_some() + && (!is_main || has_main_instances) + && let Some(slot) = components.get_mut(*id as usize) + { + *slot = component.copy( + source, + address, + &self.addresses, + plan.as_ref() + .filter(|_| is_main) + .map(|plan| (plan, &self.series)), + &mut children, + ); + root_ids.push(*id); + has_scheduling = true; + has_main |= is_main; + } + } + None if !source.component_type.is_scheduling_object() => { + root_ids.push(*id); + children.push(*id); + } + None => {} + } + } + + if !has_scheduling { + return match summary { + ItipSummary::Update { current, .. } => { + self.cancel_message(recipients, from, current) + } + _ => None, + }; + } + + let mut is_copied = vec![false; self.ical.components.len()]; + while let Some(id) = children.pop() { + if let (Some(source), Some(slot), Some(is_copied)) = ( + self.ical.components.get(id as usize), + components.get_mut(id as usize), + is_copied.get_mut(id as usize), + ) && id != 0 + && !*is_copied + && !source.component_type.is_scheduling_object() + { + *is_copied = true; + *slot = source.clone(); + children.extend(&source.component_ids); + } + } + if let Some(slot) = components.first_mut() { + *slot = ICalendarComponent { + component_type: root.component_type.clone(), + entries: root.entries.clone(), + component_ids: root_ids, + }; + } + + let message = ICalendar { components }; + let summary = if !has_main && !view.is_complete(self.components.len()) { + summary.for_slice(&self.slice_instances(view), recipient) + } else { + summary.visible_to(recipient) + }; + + Some(ItipMessage { + from: from.to_string(), + from_organizer: true, + to: recipients, + summary, + message, + }) + } + + fn slice_instances(&self, view: &RecipientView) -> SliceInstances { + let mut slice = SliceInstances { + starts: Vec::with_capacity(view.included.len()), + earliest: None, + keeps_rrule: false, + overrides: Vec::new(), + }; + let mut participants: Vec = Vec::new(); + let mut included = view.included.iter().peekable(); + + for component in &self.components { + if included.next_if(|id| **id == component.id).is_none() { + continue; + } + let Some(source) = self.ical.components.get(component.id as usize) else { + continue; + }; + let recurrence = component.recurrence.as_ref(); + slice.keeps_rrule |= recurrence.is_none_or(Recurrence::is_this_and_future); + + if let Some(start) = source + .property(&ICalendarProperty::Dtstart) + .or_else(|| recurrence.map(Recurrence::entry)) + .and_then(|entry| self.series.entry_time(entry)) + { + if slice + .earliest + .as_ref() + .is_none_or(|earliest| start.start < earliest.start) + { + slice.earliest = Some(start); + } + slice.starts.push(start.start); + } + + for entry in &source.entries { + match entry.name { + ICalendarProperty::Organizer | ICalendarProperty::Attendee => { + let is_organizer = entry.name == ICalendarProperty::Organizer; + let Some(email) = entry + .values + .first() + .and_then(|value| value.as_text()) + .and_then(|value| Email::new(value, &[])) + else { + continue; + }; + match participants + .iter_mut() + .find(|participant| participant.email == email.email) + { + Some(participant) => participant.is_organizer |= is_organizer, + None => participants.push(ItipParticipant { + email: email.email, + name: entry + .parameter(&ICalendarParameterName::Cn) + .and_then(|value| value.as_text()) + .map(|name| name.to_string()), + is_organizer, + }), + } + } + ICalendarProperty::Summary + | ICalendarProperty::Description + | ICalendarProperty::Location + | ICalendarProperty::Conference => { + if !slice.overrides.iter().any(|field| field.name == entry.name) + && let Some(value) = ItipValue::from_entry(entry) + { + slice.overrides.push(ItipField { + name: entry.name.clone(), + value, + }); + } + } + _ => {} + } + } + } + + if !participants.is_empty() { + participants.sort_unstable(); + slice.overrides.push(ItipField { + name: ICalendarProperty::Attendee, + value: ItipValue::Participants(participants), + }); + } + + slice + } + + fn cancel_message( + &self, + recipients: Vec, + from: &str, + fields: &[ItipField], + ) -> Option> { + let master = self.series.master_component()?; + let recipient = recipients + .first() + .filter(|_| self.visibility == AttendeeVisibility::RecipientOnly) + .map(String::as_str); + let summary = ItipSummary::Cancel( + fields + .iter() + .map(|field| field.visible_to(recipient)) + .collect(), + ); + let sequence = master + .property(&ICalendarProperty::Sequence) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_integer()) + .unwrap_or_default(); + let dt_stamp = master + .property(&ICalendarProperty::Dtstamp) + .and_then(|entry| entry.values.first()) + .and_then(|value| value.as_partial_date_time()) + .cloned() + .unwrap_or_else(PartialDateTime::now); + let attendees = recipients.iter().map(String::as_str).collect::>(); + let mut envelope = itip_build_envelope(ICalendarMethod::Cancel); + envelope.component_ids.push(1); + let mut message = ICalendar { + components: vec![ + envelope, + CancelScope::Attendees(&attendees).build_component(master, sequence, dt_stamp), + ], + }; + itip_add_tz(&mut message, self.ical); + + Some(ItipMessage { + from: from.to_string(), + from_organizer: true, + summary, + to: recipients, + message, + }) + } +} + +impl SchedulingComponent<'_> { + fn copy( + &self, + source: &ICalendarComponent, + address: Option, + addresses: &AHashMap, + plan: Option<(&SeriesPlan<'_>, &Series<'_>)>, + children: &mut Vec, + ) -> ICalendarComponent { + let participant_id = self + .participant_ids + .iter() + .find(|(owner, _)| Some(*owner) == address) + .map(|(_, participant_id)| participant_id.as_ref()); + let mut owners = self.owners.iter().peekable(); + let mut entries = Vec::with_capacity( + source.entries.len() + plan.map_or(0, |(plan, _)| plan.exdate_count()), + ); + for (index, entry) in source.entries.iter().enumerate() { + let mut entry = Cow::Borrowed(entry); + if let Some((_, owner)) = owners.next_if(|(owner_index, _)| *owner_index == index) + && !owner.is_visible_to(address, participant_id) + { + if !self.is_owner(owner) { + continue; + } + if entry.name == ICalendarProperty::Attendee { + entry + .to_mut() + .params + .retain(|param| self.is_visible_param(param, address, addresses)); + } + } + match plan { + Some((plan, series)) => { + entries.extend(plan.entry(series, &entry).map(|entry| entry.into_owned())) + } + None => entries.push(entry.into_owned()), + } + } + if let Some((plan, _)) = plan { + entries.extend(plan.exdates()); + } + + let mut alarms = self.alarms.iter().peekable(); + let component_ids = source + .component_ids + .iter() + .copied() + .filter(|id| { + alarms + .next_if(|(alarm_id, _)| alarm_id == id) + .is_none_or(|(_, owner)| owner.is_some_and(|owner| Some(owner) == address)) + }) + .collect::>(); + children.extend(&component_ids); + + ICalendarComponent { + component_type: source.component_type.clone(), + entries, + component_ids, + } + } + + fn is_visible_param( + &self, + param: &ICalendarParameter, + address: Option, + addresses: &AHashMap, + ) -> bool { + !matches!( + param.name, + ICalendarParameterName::DelegatedTo | ICalendarParameterName::DelegatedFrom + ) || param + .value + .as_text() + .and_then(|value| Email::new(value, &[])) + .and_then(|email| addresses.get(&email.email).copied()) + .is_some_and(|delegate| { + Some(delegate) == address || self.owner_addresses.binary_search(&delegate).is_ok() + }) + } + + fn is_owner(&self, owner: &EntryOwner<'_>) -> bool { + match owner { + EntryOwner::Address(address) => self.owner_addresses.binary_search(address).is_ok(), + EntryOwner::Participant(segment) => self + .owner_participant_ids + .iter() + .any(|participant_id| EntryOwner::pointer_segment_eq(segment, participant_id)), + EntryOwner::Unknown => false, + } + } +} + +impl<'x> EntryOwner<'x> { + fn from_jsprop(entry: &'x ICalendarEntry) -> Option { + let pointer = entry + .parameter(&ICalendarParameterName::Jsptr) + .and_then(|value| value.as_text()) + .unwrap_or_default(); + let participants = pointer + .strip_prefix(PARTICIPANTS_POINTER) + .filter(|rest| rest.is_empty() || rest.starts_with('/')); + match participants + .and_then(|rest| rest.strip_prefix('/')) + .map(|rest| rest.split_once('/').map_or(rest, |(id, _)| id)) + .filter(|id| !id.is_empty()) + { + Some(participant_id) => Some(EntryOwner::Participant(participant_id)), + None if participants.is_some() + || entry + .values + .iter() + .filter_map(|value| value.as_text()) + .any(|value| value.contains(PARTICIPANTS_POINTER)) => + { + Some(EntryOwner::Unknown) + } + None => None, + } + } + + fn from_references(entry: &ICalendarEntry, addresses: &AHashMap) -> Option { + let mut owners = entry + .address_references() + .filter_map(|reference| Email::new(reference, &[])) + .filter_map(|email| addresses.get(&email.email).copied()); + let owner = owners.next()?; + Some(if owners.all(|other| other == owner) { + EntryOwner::Address(owner) + } else { + EntryOwner::Unknown + }) + } + + fn is_visible_to(&self, address: Option, participant_id: Option<&str>) -> bool { + match self { + EntryOwner::Address(owner) => address == Some(*owner), + EntryOwner::Participant(segment) => participant_id.is_some_and(|participant_id| { + EntryOwner::pointer_segment_eq(segment, participant_id) + }), + EntryOwner::Unknown => false, + } + } + + fn pointer_segment_eq(segment: &str, value: &str) -> bool { + let mut bytes = segment.bytes(); + iter::from_fn(|| match bytes.next()? { + b'~' => match bytes.next() { + Some(b'0') => Some(b'~'), + Some(b'1') => Some(b'/'), + _ => Some(u8::MAX), + }, + byte => Some(byte), + }) + .eq(value.bytes()) + } +} + +impl RecipientView { + fn is_complete(&self, components: usize) -> bool { + self.excluded.is_empty() && self.included.len() == components + } +} + +impl AddressReferences for ICalendarEntry { + fn address_references(&self) -> impl Iterator { + self.params + .iter() + .filter_map(|param| param.value.as_text()) + .chain(self.values.iter().filter_map(|value| value.as_text())) + .filter(|reference| reference.contains('@')) + } +} + +impl ItipField { + fn visible_to(&self, recipient: Option<&str>) -> Self { + match (&self.value, recipient) { + (ItipValue::Participants(participants), Some(recipient)) => ItipField { + name: self.name.clone(), + value: ItipValue::Participants( + participants + .iter() + .filter(|participant| { + participant.is_organizer || participant.email == recipient + }) + .cloned() + .collect(), + ), + }, + _ => self.clone(), + } + } +} + +impl ItipSummary { + fn visible_to(&self, recipient: Option<&str>) -> Self { + let visible = |fields: &[ItipField]| -> Vec { + fields + .iter() + .map(|field| field.visible_to(recipient)) + .collect() + }; + match self { + ItipSummary::Invite(current) => ItipSummary::Invite(visible(current)), + ItipSummary::Update { + method, + current, + previous, + } => { + let current = visible(current); + let previous = previous + .iter() + .map(|field| field.visible_to(recipient)) + .filter(|field| !current.contains(field)) + .collect(); + ItipSummary::Update { + method: method.clone(), + current, + previous, + } + } + ItipSummary::Cancel(current) => ItipSummary::Cancel(visible(current)), + ItipSummary::Rsvp { part_stat, current } => ItipSummary::Rsvp { + part_stat: part_stat.clone(), + current: visible(current), + }, + } + } + + fn for_slice(&self, slice: &SliceInstances, recipient: Option<&str>) -> Self { + match self { + ItipSummary::Invite(current) => ItipSummary::Invite(slice.filter(current, recipient)), + ItipSummary::Update { + method, current, .. + } => ItipSummary::Update { + method: method.clone(), + current: slice.filter(current, recipient), + previous: Vec::new(), + }, + ItipSummary::Cancel(current) => ItipSummary::Cancel(slice.filter(current, recipient)), + ItipSummary::Rsvp { part_stat, current } => ItipSummary::Rsvp { + part_stat: part_stat.clone(), + current: slice.filter(current, recipient), + }, + } + } +} + +struct SliceInstances { + starts: Vec, + earliest: Option, + keeps_rrule: bool, + overrides: Vec, +} + +impl SliceInstances { + fn filter(&self, fields: &[ItipField], recipient: Option<&str>) -> Vec { + let mut filtered = Vec::with_capacity(fields.len() + 1); + let mut has_start = false; + + for field in fields { + match (&field.name, &field.value) { + (ICalendarProperty::Dtstart, ItipValue::Time(start)) => { + if !self.starts.contains(&start.start) { + continue; + } + has_start = true; + } + (ICalendarProperty::Rrule, _) if !self.keeps_rrule => continue, + _ => {} + } + + match self + .overrides + .iter() + .find(|replacement| replacement.name == field.name) + { + Some(replacement) => { + let replacement = replacement.visible_to(recipient); + if !filtered.contains(&replacement) { + filtered.push(replacement); + } + } + None => filtered.push(field.visible_to(recipient)), + } + } + + if !has_start && let Some(earliest) = &self.earliest { + filtered.push(ItipField { + name: ICalendarProperty::Dtstart, + value: ItipValue::Time(*earliest), + }); + } + for replacement in &self.overrides { + if !filtered.iter().any(|field| field.name == replacement.name) { + filtered.push(replacement.visible_to(recipient)); + } + } + + filtered + } +} + +impl ItipValue { + fn from_entry(entry: &ICalendarEntry) -> Option { + entry + .values + .first() + .and_then(|value| match value { + ICalendarValue::Text(text) => Some(text.as_str()), + ICalendarValue::Uri(Uri::Location(uri)) => Some(uri.as_str()), + _ => None, + }) + .map(|text| ItipValue::Text(text.to_string())) + } +} diff --git a/crates/groupware/src/scheduling/snapshot.rs b/crates/groupware/src/scheduling/snapshot.rs index c2624319..b6cdbefb 100644 --- a/crates/groupware/src/scheduling/snapshot.rs +++ b/crates/groupware/src/scheduling/snapshot.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::calendar::expand::SECONDS_PER_DAY; use crate::scheduling::{ Attendee, Email, InstanceId, ItipDateTime, ItipEntry, ItipEntryValue, ItipError, ItipField, ItipParticipant, ItipSnapshot, ItipSnapshots, ItipTime, ItipValue, Organizer, RecurrenceId, }; use ahash::AHashMap; use calcard::icalendar::{ - ICalendar, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, - ICalendarScheduleAgentValue, ICalendarValue, Uri, + ICalendar, ICalendarDuration, ICalendarParameterName, ICalendarParameterValue, + ICalendarParticipationStatus, ICalendarProperty, ICalendarScheduleAgentValue, ICalendarValue, + Uri, }; +use std::borrow::Borrow; pub fn itip_snapshot<'x, 'y>( ical: &'x ICalendar, @@ -56,6 +59,7 @@ pub fn itip_snapshot<'x, 'y>( entries: Default::default(), sequence: Default::default(), request_status: Default::default(), + recurrence_time: None, }; let mut instance_id = InstanceId::Main; @@ -266,16 +270,21 @@ pub fn itip_snapshot<'x, 'y>( } } + let tz = tz_resolver + .get_or_insert_with(|| ical.build_tz_resolver()) + .resolve_or_default(tz_id); + let start = date + .to_date_time_with_tz(tz) + .map(|dt| dt.timestamp()) + .unwrap_or_else(|| date.to_timestamp().unwrap_or_default()); + + sched_comp.recurrence_time = Some(ItipTime { + start, + tz_id: tz.as_id(), + }); instance_id = InstanceId::Recurrence(RecurrenceId { entry_id: entry_id as u16, - date: date - .to_date_time_with_tz( - tz_resolver - .get_or_insert_with(|| ical.build_tz_resolver()) - .resolve_or_default(tz_id), - ) - .map(|dt| dt.timestamp()) - .unwrap_or_else(|| date.to_timestamp().unwrap_or_default()), + date: start, this_and_future, }); } @@ -315,7 +324,7 @@ pub fn itip_snapshot<'x, 'y>( .get_or_insert_with(|| ical.build_tz_resolver()) .resolve_or_default(tz_id); ItipEntryValue::DateTime(ItipDateTime { - date: date.as_ref(), + date, tz_id, tz_code: tz.as_id(), timestamp: date @@ -361,7 +370,19 @@ pub fn itip_snapshot<'x, 'y>( } } -impl ItipSnapshots<'_> { +impl<'x> ItipSnapshots<'x> { + pub(crate) fn unanswered(&self) -> ItipSnapshots<'x> { + ItipSnapshots { + organizer: self.organizer.clone(), + uid: self.uid, + components: self + .components + .iter() + .map(|(instance_id, instance)| (instance_id.clone(), instance.unanswered())) + .collect(), + } + } + pub fn sender_is_organizer_or_attendee(&self, email: &str) -> bool { self.organizer.email.email == email || self.components.values().any(|snapshot| { @@ -380,9 +401,183 @@ impl ItipSnapshots<'_> { self.main_instance() .unwrap_or_else(|| self.components.values().next().unwrap()) } + + pub fn matching_instance( + &self, + instance_id: &InstanceId, + ) -> Option<(&InstanceId, &ItipSnapshot<'x>)> { + self.components + .get_key_value(instance_id) + .or_else(|| match instance_id { + InstanceId::Recurrence(recurrence_id) => { + self.components + .get_key_value(&InstanceId::Recurrence(RecurrenceId { + this_and_future: !recurrence_id.this_and_future, + ..recurrence_id.clone() + })) + } + InstanceId::Main => None, + }) + } + + pub fn occurrence_source(&self, recurrence_id: &RecurrenceId) -> Option<&ItipSnapshot<'x>> { + self.components + .iter() + .filter_map(|(instance_id, instance)| match instance_id { + InstanceId::Recurrence(covering) + if covering.this_and_future && covering.date <= recurrence_id.date => + { + Some((covering.date, instance)) + } + InstanceId::Recurrence(_) | InstanceId::Main => None, + }) + .max_by_key(|(date, _)| *date) + .map(|(_, instance)| instance) + .or_else(|| self.components.get(&InstanceId::Main)) + } + + pub fn instance_time(&self, instance_id: &InstanceId) -> Option { + let instance = self.components.get(instance_id); + if let Some(time) = instance.and_then(ItipSnapshot::start_time) { + return Some(time); + } + + match instance_id { + InstanceId::Main => None, + InstanceId::Recurrence(recurrence_id) => instance + .and_then(|instance| instance.recurrence_time) + .or_else(|| { + self.main_instance() + .and_then(ItipSnapshot::start_time) + .map(|master_start| ItipTime { + start: recurrence_id.date, + tz_id: master_start.tz_id, + }) + }), + } + } + + pub fn build_instance_summary( + &self, + instance_id: &InstanceId, + include_guests: Option<&Organizer<'_>>, + skip_fields: &[ItipField], + ) -> Vec { + let instance = self.components.get(instance_id); + let Some(master) = self.main_instance() else { + return instance.map_or_else(Vec::new, |instance| { + instance.build_summary(include_guests, skip_fields) + }); + }; + let InstanceId::Recurrence(recurrence_id) = instance_id else { + return master.build_summary(include_guests, skip_fields); + }; + + let mut fields = Vec::with_capacity(SUMMARY_IDENTITY.len() + 3); + for name in SUMMARY_IDENTITY { + let source = instance + .filter(|instance| instance.entries.iter().any(|entry| entry.name == &name)) + .unwrap_or(master); + fields.extend(source.summary_fields(&name)); + } + if let Some(time) = self.instance_time(instance_id) { + fields.push(ItipField { + name: ICalendarProperty::Dtstart, + value: ItipValue::Time(time), + }); + } + if recurrence_id.this_and_future { + fields.extend(master.summary_fields(&ICalendarProperty::Rrule)); + } + + instance + .unwrap_or(master) + .finish_summary(fields, include_guests, skip_fields) + } + + pub fn build_instances_summary( + &self, + instance_ids: &[impl Borrow], + include_guests: Option<&Organizer<'_>>, + skip_fields: &[ItipField], + ) -> Vec { + let Some((first, rest)) = instance_ids.split_first() else { + return Vec::new(); + }; + let mut fields = self.build_instance_summary(first.borrow(), include_guests, skip_fields); + + for instance_id in rest { + let Some(time) = self.instance_time(instance_id.borrow()) else { + continue; + }; + let field = ItipField { + name: ICalendarProperty::Dtstart, + value: ItipValue::Time(time), + }; + if !fields.contains(&field) && !skip_fields.contains(&field) { + fields.push(field); + } + } + + fields + } } -impl ItipSnapshot<'_> { +const SUMMARY_IDENTITY: [ICalendarProperty; 4] = [ + ICalendarProperty::Summary, + ICalendarProperty::Description, + ICalendarProperty::Location, + ICalendarProperty::Conference, +]; + +impl<'x> ItipSnapshot<'x> { + pub fn length(&self) -> Option { + let mut start = None; + let mut end = None; + for entry in &self.entries { + match (entry.name, &entry.value) { + (ICalendarProperty::Duration, ItipEntryValue::Duration(duration)) => { + return Some((*duration).clone()); + } + (ICalendarProperty::Dtstart, ItipEntryValue::DateTime(date)) => start = Some(date), + (ICalendarProperty::Dtend, ItipEntryValue::DateTime(date)) => end = Some(date), + _ => {} + } + } + let (start, end) = (start?, end?); + let seconds = end.timestamp - start.timestamp; + (seconds > 0).then(|| { + if start.date.has_time() { + ICalendarDuration::from_seconds(seconds) + } else { + ICalendarDuration::from_days((seconds + SECONDS_PER_DAY / 2) / SECONDS_PER_DAY) + } + }) + } + + fn unanswered(&self) -> ItipSnapshot<'x> { + ItipSnapshot { + comp_id: self.comp_id, + comp: self.comp, + attendees: self + .attendees + .iter() + .map(|attendee| { + let mut attendee = attendee.clone(); + if attendee.email.is_local && attendee.part_stat.is_some() { + attendee.part_stat = Some(&ICalendarParticipationStatus::NeedsAction); + } + attendee + }) + .collect(), + dtstamp: self.dtstamp, + entries: self.entries.clone(), + sequence: self.sequence, + request_status: self.request_status.clone(), + recurrence_time: self.recurrence_time, + } + } + pub fn has_local_attendee(&self) -> bool { self.attendees .iter() @@ -410,7 +605,7 @@ impl ItipSnapshot<'_> { include_guests: Option<&Organizer<'_>>, skip_fields: &[ItipField], ) -> Vec { - let mut fields = Vec::with_capacity(5); + let mut fields = Vec::with_capacity(6); for entry in &self.entries { if matches!( @@ -421,64 +616,81 @@ impl ItipSnapshot<'_> { | ICalendarProperty::Location | ICalendarProperty::Conference | ICalendarProperty::Rrule - ) { - let value = match &entry.value { - ItipEntryValue::DateTime(dt) => ItipValue::Time(ItipTime { - start: dt.timestamp, - tz_id: dt.tz_code, - }), - ItipEntryValue::RRule(rule) => ItipValue::Rrule(Box::new((*rule).clone())), - ItipEntryValue::Text(value) => ItipValue::Text(value.to_string()), - _ => continue, - }; - let field = ItipField { + ) && let Some(value) = entry.value.to_summary_value() + { + fields.push(ItipField { name: entry.name.clone(), value, - }; - - if !skip_fields.contains(&field) { - fields.push(field); - } + }); } } - if let Some(organizer) = include_guests { - let mut attendees = Vec::with_capacity(self.attendees.len()); - for attendee in &self.attendees { - if attendee.email.email != organizer.email.email { - attendees.push(ItipParticipant { - email: attendee.email.email.to_string(), - name: attendee.name.map(|n| n.to_string()), - is_organizer: false, - }); - } + self.finish_summary(fields, include_guests, skip_fields) + } + + pub fn guest_field(&self, organizer: &Organizer<'_>) -> ItipField { + let mut attendees = Vec::with_capacity(self.attendees.len() + 1); + for attendee in &self.attendees { + if attendee.email.email != organizer.email.email { + attendees.push(ItipParticipant { + email: attendee.email.email.clone(), + name: attendee.name.map(|name| name.to_string()), + is_organizer: false, + }); } - attendees.push(ItipParticipant { - email: organizer.email.email.to_string(), - name: organizer.name.map(|n| n.to_string()), - is_organizer: true, - }); - attendees.sort_by(|a, b| { - if a.is_organizer && !b.is_organizer { - std::cmp::Ordering::Less - } else if !a.is_organizer && b.is_organizer { - std::cmp::Ordering::Greater - } else if let (Some(a_name), Some(b_name)) = (a.name.as_deref(), b.name.as_deref()) - { - match a_name.cmp(b_name) { - std::cmp::Ordering::Equal => a.email.cmp(&b.email), - ord => ord, - } - } else { - a.email.cmp(&b.email) - } - }); + } + attendees.push(ItipParticipant { + email: organizer.email.email.clone(), + name: organizer.name.map(|name| name.to_string()), + is_organizer: true, + }); + attendees.sort_unstable(); - let field = ItipField { - name: ICalendarProperty::Attendee, - value: ItipValue::Participants(attendees), - }; + ItipField { + name: ICalendarProperty::Attendee, + value: ItipValue::Participants(attendees), + } + } + fn start_time(&self) -> Option { + self.entries + .iter() + .find_map(|entry| match (entry.name, &entry.value) { + (ICalendarProperty::Dtstart, ItipEntryValue::DateTime(start)) => Some(ItipTime { + start: start.timestamp, + tz_id: start.tz_code, + }), + _ => None, + }) + } + + fn summary_fields<'y>( + &'y self, + name: &'y ICalendarProperty, + ) -> impl Iterator + 'y { + self.entries.iter().filter_map(move |entry| { + if entry.name != name { + return None; + } + entry.value.to_summary_value().map(|value| ItipField { + name: name.clone(), + value, + }) + }) + } + + fn finish_summary( + &self, + mut fields: Vec, + include_guests: Option<&Organizer<'_>>, + skip_fields: &[ItipField], + ) -> Vec { + if !skip_fields.is_empty() { + fields.retain(|field| !skip_fields.contains(field)); + } + + if let Some(organizer) = include_guests { + let field = self.guest_field(organizer); if !skip_fields.contains(&field) { fields.push(field); } @@ -487,3 +699,18 @@ impl ItipSnapshot<'_> { fields } } + +impl ItipEntryValue<'_> { + fn to_summary_value(&self) -> Option { + match self { + ItipEntryValue::DateTime(date) => Some(ItipValue::Time(ItipTime { + start: date.timestamp, + tz_id: date.tz_code, + })), + ItipEntryValue::RRule(rule) => Some(ItipValue::Rrule(Box::new((*rule).clone()))), + ItipEntryValue::Text(value) => Some(ItipValue::Text(value.to_string())), + _ => None, + } + } +} + diff --git a/crates/http-proto/Cargo.toml b/crates/http-proto/Cargo.toml index 87af79ab..cbb9f829 100644 --- a/crates/http-proto/Cargo.toml +++ b/crates/http-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http_proto" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -9,7 +9,7 @@ trc = { path = "../trc" } registry = { path = "../registry" } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } http-body-util = "0.1.5" form_urlencoded = "1.2.2" percent-encoding = "2.3.2" @@ -18,9 +18,8 @@ compact_str = "0.10.0" [dev-dependencies] [features] -test_mode = [] dev_mode = [] -enterprise = [] +enterprise = ["common/enterprise", "trc/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/http-proto/src/body.rs b/crates/http-proto/src/body.rs new file mode 100644 index 00000000..6be5014f --- /dev/null +++ b/crates/http-proto/src/body.rs @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use hyper::body::{Body, Bytes, Frame, SizeHint}; +use std::{ + convert::Infallible, + pin::Pin, + task::{Context, Poll}, +}; + +pub struct PrefixedBody { + prefix: Option, + body: Option, +} + +impl PrefixedBody { + pub fn new(prefix: Bytes, body: Bytes) -> Self { + PrefixedBody { + prefix: Some(prefix).filter(|prefix| !prefix.is_empty()), + body: Some(body).filter(|body| !body.is_empty()), + } + } + + pub fn len(&self) -> usize { + self.prefix.as_ref().map_or(0, Bytes::len) + self.body.as_ref().map_or(0, Bytes::len) + } + + pub fn is_empty(&self) -> bool { + self.prefix.is_none() && self.body.is_none() + } +} + +impl Body for PrefixedBody { + type Data = Bytes; + type Error = Infallible; + + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Infallible>>> { + let this = self.get_mut(); + Poll::Ready( + this.prefix + .take() + .or_else(|| this.body.take()) + .map(|chunk| Ok(Frame::data(chunk))), + ) + } + + fn is_end_stream(&self) -> bool { + self.is_empty() + } + + fn size_hint(&self) -> SizeHint { + SizeHint::with_exact(self.len() as u64) + } +} + diff --git a/crates/http-proto/src/context.rs b/crates/http-proto/src/context.rs index 672ad3e5..4e7ae254 100644 --- a/crates/http-proto/src/context.rs +++ b/crates/http-proto/src/context.rs @@ -7,9 +7,12 @@ use crate::{HttpContext, HttpRequest, HttpSessionData}; use common::{ Server, - expr::{functions::ResolveVariable, *}, + expr::{ + Bump, Variable, + bumpalo::{self, collections::Vec as BumpVec}, + functions::ResolveVariable, + }, }; -use compact_str::{ToCompactString, format_compact}; use hyper::StatusCode; use registry::schema::enums::ExpressionVariable; @@ -19,10 +22,12 @@ impl<'x> HttpContext<'x> { } pub async fn has_endpoint_access(&self, server: &Server) -> StatusCode { + let mut arena = Bump::new(); server .eval_if( &server.core.network.http.allowed_endpoint, self, + &mut arena, self.session.session_id, ) .await @@ -31,37 +36,47 @@ impl<'x> HttpContext<'x> { } impl ResolveVariable for HttpContext<'_> { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { - ExpressionVariable::RemoteIp => self.session.remote_ip.to_compact_string().into(), + ExpressionVariable::RemoteIp => Variable::String( + bumpalo::format!(in arena, "{}", self.session.remote_ip).into_bump_str(), + ), ExpressionVariable::RemotePort => self.session.remote_port.into(), - ExpressionVariable::LocalIp => self.session.local_ip.to_compact_string().into(), + ExpressionVariable::LocalIp => Variable::String( + bumpalo::format!(in arena, "{}", self.session.local_ip).into_bump_str(), + ), ExpressionVariable::LocalPort => self.session.local_port.into(), ExpressionVariable::IsTls => self.session.is_tls.into(), ExpressionVariable::Protocol => { if self.session.is_tls { "https" } else { "http" }.into() } ExpressionVariable::Listener => self.session.instance.id.as_str().into(), - ExpressionVariable::Url => self.req.uri().to_compact_string().into(), + ExpressionVariable::Url => { + Variable::String(bumpalo::format!(in arena, "{}", self.req.uri()).into_bump_str()) + } ExpressionVariable::Path => self.req.uri().path().into(), ExpressionVariable::Method => self.req.method().as_str().into(), - ExpressionVariable::Headers => self - .req - .headers() - .iter() - .map(|(h, v)| { + ExpressionVariable::Headers => { + let headers = self.req.headers(); + let mut items = BumpVec::with_capacity_in(headers.len(), arena); + items.extend(headers.iter().map(|(name, value)| { Variable::String( - format_compact!("{}: {}", h.as_str(), v.to_str().unwrap_or_default()) - .into(), + bumpalo::format!( + in arena, + "{}: {}", + name.as_str(), + value.to_str().unwrap_or_default() + ) + .into_bump_str(), ) - }) - .collect::>() - .into(), + })); + Variable::Array(items.into_bump_slice()) + } _ => Variable::default(), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } diff --git a/crates/http-proto/src/lib.rs b/crates/http-proto/src/lib.rs index 628b5436..bf40e020 100644 --- a/crates/http-proto/src/lib.rs +++ b/crates/http-proto/src/lib.rs @@ -6,6 +6,7 @@ #![warn(clippy::large_futures)] +pub mod body; pub mod context; pub mod request; pub mod response; @@ -13,7 +14,7 @@ pub mod response; pub use form_urlencoded; use common::network::ServerInstance; -use hyper::StatusCode; +use hyper::{StatusCode, body::Bytes}; use std::{net::IpAddr, sync::Arc}; pub type HttpRequest = hyper::Request; @@ -31,7 +32,8 @@ pub struct HtmlResponse { pub enum HttpResponseBody { Text(String), - Binary(Vec), + Binary(Bytes), + Prefixed(body::PrefixedBody), Stream(http_body_util::combinators::BoxBody), WebsocketUpgrade(String), Empty, @@ -61,7 +63,8 @@ pub struct HttpSessionData { pub struct DownloadResponse { pub filename: String, pub content_type: String, - pub blob: Vec, + pub prefix: Option, + pub blob: Bytes, } pub struct JsonProblemResponse(pub StatusCode); diff --git a/crates/http-proto/src/request.rs b/crates/http-proto/src/request.rs index 467dba37..fc80101a 100644 --- a/crates/http-proto/src/request.rs +++ b/crates/http-proto/src/request.rs @@ -6,11 +6,16 @@ use std::borrow::Cow; -use compact_str::ToCompactString; +use compact_str::{CompactString, ToCompactString}; use http_body_util::BodyExt; +use hyper::body::Body; use crate::HttpRequest; +const MIN_FIRST_BODY_ALLOCATION: usize = 4 * 1024; +const MIN_BODY_GROWTH: usize = 64 * 1024; +const BODY_GROWTH_FACTOR: usize = 4; + #[inline] pub fn decode_path_element(item: &str) -> Cow<'_, str> { percent_encoding::percent_decode_str(item) @@ -23,16 +28,43 @@ pub async fn fetch_body( max_size: usize, session_id: u64, ) -> Option> { - let mut bytes = Vec::with_capacity(1024); + let declared = req + .body() + .size_hint() + .exact() + .and_then(|len| usize::try_from(len).ok()) + .filter(|&len| max_size == 0 || len <= max_size); + let mut bytes = Vec::new(); while let Some(Ok(frame)) = req.frame().await { if let Some(data) = frame.data_ref() { - if bytes.len() + data.len() <= max_size || max_size == 0 { + let len = bytes.len() + data.len(); + if len <= max_size || max_size == 0 { + if let Some(declared) = declared + && len > bytes.capacity() + { + if bytes.capacity() == 0 { + bytes = Vec::with_capacity( + len.saturating_mul(BODY_GROWTH_FACTOR) + .max(MIN_FIRST_BODY_ALLOCATION) + .min(declared) + .max(len), + ); + } else { + let target = bytes + .capacity() + .saturating_mul(BODY_GROWTH_FACTOR) + .max(MIN_BODY_GROWTH) + .min(declared) + .max(len); + bytes.reserve_exact(target - bytes.len()); + } + } bytes.extend_from_slice(data); } else { trc::event!( Http(trc::HttpEvent::RequestBody), SpanId = session_id, - Details = req + Headers = req .headers() .iter() .map(|(k, v)| trc::Value::Array(vec![ @@ -40,9 +72,8 @@ pub async fn fetch_body( v.to_str().unwrap_or_default().to_compact_string().into() ])) .collect::>(), - Contents = std::str::from_utf8(&bytes) - .unwrap_or("[binary data]") - .to_string(), + Contents = + CompactString::from(std::str::from_utf8(&bytes).unwrap_or("[binary data]")), Size = bytes.len(), Limit = max_size, ); @@ -55,7 +86,7 @@ pub async fn fetch_body( trc::event!( Http(trc::HttpEvent::RequestBody), SpanId = session_id, - Details = req + Headers = req .headers() .iter() .map(|(k, v)| trc::Value::Array(vec![ @@ -63,9 +94,7 @@ pub async fn fetch_body( v.to_str().unwrap_or_default().to_compact_string().into() ])) .collect::>(), - Contents = std::str::from_utf8(&bytes) - .unwrap_or("[binary data]") - .to_string(), + Contents = CompactString::from(std::str::from_utf8(&bytes).unwrap_or("[binary data]")), Size = bytes.len(), ); diff --git a/crates/http-proto/src/response.rs b/crates/http-proto/src/response.rs index 112aa8f2..6ecaf7b7 100644 --- a/crates/http-proto/src/response.rs +++ b/crates/http-proto/src/response.rs @@ -15,7 +15,7 @@ use serde_json::json; use crate::{ DownloadResponse, HtmlResponse, HttpResponse, HttpResponseBody, JsonProblemResponse, - JsonResponse, ToHttpResponse, + JsonResponse, ToHttpResponse, body::PrefixedBody, }; impl HttpResponse { @@ -121,13 +121,20 @@ impl HttpResponse { self.with_content_length(body_len) } - pub fn with_binary_body(mut self, body: impl Into>) -> Self { + pub fn with_binary_body(mut self, body: impl Into) -> Self { let body = body.into(); let body_len = body.len(); self.body = HttpResponseBody::Binary(body); self.with_content_length(body_len) } + pub fn with_prefixed_body(mut self, prefix: Bytes, body: Bytes) -> Self { + let body = PrefixedBody::new(prefix, body); + let body_len = body.len(); + self.body = HttpResponseBody::Prefixed(body); + self.with_content_length(body_len) + } + pub fn with_stream_body( mut self, stream: http_body_util::combinators::BoxBody, @@ -204,10 +211,33 @@ impl HttpResponse { self } + pub fn with_sandbox_policy(mut self) -> Self { + self.builder = self + .builder + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .header( + header::CONTENT_SECURITY_POLICY, + "default-src 'none'; sandbox", + ); + self + } + + pub fn with_image_sandbox_policy(mut self) -> Self { + self.builder = self + .builder + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .header( + header::CONTENT_SECURITY_POLICY, + "default-src 'none'; style-src 'unsafe-inline'; sandbox", + ); + self + } + pub fn size(&self) -> usize { match &self.body { HttpResponseBody::Text(value) => value.len(), HttpResponseBody::Binary(value) => value.len(), + HttpResponseBody::Prefixed(value) => value.len(), _ => 0, } } @@ -222,11 +252,12 @@ impl HttpResponse { .map_err(|never| match never {}) .boxed(), ), - HttpResponseBody::Binary(body) => self.builder.body( - Full::new(Bytes::from(body)) - .map_err(|never| match never {}) - .boxed(), - ), + HttpResponseBody::Binary(body) => self + .builder + .body(Full::new(body).map_err(|never| match never {}).boxed()), + HttpResponseBody::Prefixed(body) => self + .builder + .body(body.map_err(|never| match never {}).boxed()), HttpResponseBody::Empty => { let has_content_length = self .builder @@ -289,18 +320,21 @@ impl ToHttpResponse for JsonResponse { impl ToHttpResponse for DownloadResponse { fn into_http_response(self) -> HttpResponse { - HttpResponse::new(StatusCode::OK) + let response = HttpResponse::new(StatusCode::OK) .with_content_type(self.content_type) .with_content_disposition(format!( "attachment; filename=\"{}\"", self.filename.replace('\"', "\\\"") )) - .with_cache_control("private, immutable, max-age=31536000") - .with_binary_body(self.blob) + .with_cache_control("private, immutable, max-age=31536000"); + match self.prefix { + Some(prefix) => response.with_prefixed_body(prefix, self.blob), + None => response.with_binary_body(self.blob), + } } } -impl ToHttpResponse for Resource> { +impl> ToHttpResponse for Resource { fn into_http_response(self) -> HttpResponse { HttpResponse::new(StatusCode::OK) .with_content_type(self.content_type.as_ref()) diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index 8c8143a4..c38eed55 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "http" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -20,32 +20,31 @@ types = { path = "../types" } directory = { path = "../directory" } services = { path = "../services" } registry = { path = "../registry" } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } -mail-builder = { version = "1.0" } -mail-auth = { version = "0.13", features = ["generate", "arc"] } -tokio = { version = "1.53", features = ["rt"] } -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } -hyper-util = { version = "0.1.20", features = ["tokio"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +mail-auth = { version = "0.14", features = ["generate", "arc"] } +tokio = { version = "1.54", features = ["rt"] } +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1.21", features = ["tokio"] } http-body-util = "0.1.5" async-stream = "0.3.6" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" -chrono = "0.4" -base64 = "0.23" +encodify = "1.0" sha2 = "0.11" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true form-data = { version = "0.6.0", features = ["sync"], default-features = false } mime = "0.3.17" percent-encoding = "2.3.2" -hashify = { version = "0.2" } +hashify = { version = "0.3" } +compact_str = "0.10.0" [dev-dependencies] flate2 = "1.1" [features] -test_mode = [] dev_mode = [] -enterprise = ["scim/enterprise"] +enterprise = ["scim/enterprise", "store/enterprise", "common/enterprise", "utils/enterprise", "trc/enterprise", "email/enterprise", "smtp/enterprise", "jmap/enterprise", "dav/enterprise", "groupware/enterprise", "http_proto/enterprise", "jmap_proto/enterprise", "directory/enterprise", "services/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/http/src/api/diagnose.rs b/crates/http/src/api/diagnose.rs index 55fa81e7..e586e2ab 100644 --- a/crates/http/src/api/diagnose.rs +++ b/crates/http/src/api/diagnose.rs @@ -12,7 +12,7 @@ use common::{ }, }; use hyper::body::{Bytes, Frame}; -use mail_auth::{DnssecStatus, IpLookupStrategy, mta_sts::TlsRpt}; +use mail_auth::{DnssecStatus, IpLookupStrategy, mta_sts::TlsRptRecord}; use serde::{Deserialize, Serialize}; use smtp::outbound::{ client::{SmtpClient, StartTlsResult}, @@ -29,6 +29,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::{io::AsyncWriteExt, sync::mpsc}; +use utils::{sanitize_domain, sanitize_email}; #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -194,15 +195,62 @@ impl ElapsedMs for Instant { self.elapsed().as_millis() as u64 } } + +#[derive(Debug)] +pub(crate) struct DeliveryTarget(Target); + +#[derive(Debug)] +enum Target { + Domain(String), + Address(String), +} + +impl DeliveryTarget { + pub(crate) fn parse(target: &str) -> Option { + if target + .chars() + .any(|ch| ch.is_whitespace() || ch.is_control() || matches!(ch, '<' | '>')) + { + return None; + } + let target = if target.contains('@') { + DeliveryTarget(Target::Address(sanitize_email(target)?)) + } else { + DeliveryTarget(Target::Domain(sanitize_domain(target)?)) + }; + target + .domain() + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_')) + .then_some(target) + } + + fn domain(&self) -> &str { + match &self.0 { + Target::Domain(domain) => domain, + Target::Address(address) => address + .rsplit_once('@') + .map_or(address.as_str(), |(_, domain)| domain), + } + } + + fn address(&self) -> Option<&str> { + match &self.0 { + Target::Domain(_) => None, + Target::Address(address) => Some(address), + } + } +} + pub(crate) fn spawn_delivery_diagnose( server: Server, - domain_or_email: String, + target: DeliveryTarget, timeout: Duration, ) -> mpsc::Receiver { let (tx, rx) = mpsc::channel(10); tokio::spawn(async move { - let _ = delivery_diagnose(tx, server, domain_or_email, timeout).await; + let _ = delivery_diagnose(tx, server, target, timeout).await; }); rx @@ -211,14 +259,11 @@ pub(crate) fn spawn_delivery_diagnose( async fn delivery_diagnose( tx: mpsc::Sender, server: Server, - domain_or_email: String, + target: DeliveryTarget, timeout: Duration, ) -> Result<(), mpsc::error::SendError> { - let (domain, email) = if let Some((_, domain)) = domain_or_email.rsplit_once('@') { - (domain.to_string(), Some(domain_or_email)) - } else { - (domain_or_email, None) - }; + let domain = target.domain(); + let email = target.address(); let local_host = &server.core.network.server_name; let conn_strategy = server.get_connection_or_default("default", 0); @@ -230,7 +275,7 @@ async fn delivery_diagnose( // Lookup MX let now = Instant::now(); - let mxs = match server.mx_lookup(domain.as_str()).await { + let mxs = match server.mx_lookup(domain).await { Ok(mxs) => mxs, Err(err) => { tx.send(DeliveryStage::MxLookupError { @@ -245,14 +290,14 @@ async fn delivery_diagnose( // Obtain remote host list let mx_config = MxConfig { - max_mx: mxs.rrset.len(), + max_mx: mxs.records.len(), max_multi_homed: 10, ip_lookup_strategy: IpLookupStrategy::Ipv4thenIpv6, }; - let hosts = if let Some(hosts) = mxs.to_remote_hosts(&domain, &mx_config) { + let hosts = if let Some(hosts) = mxs.to_remote_hosts(domain, &mx_config) { tx.send(DeliveryStage::MxLookupSuccess { mxs: mxs - .rrset + .records .iter() .map(|mx| MX { exchanges: mx.exchanges.iter().map(|e| e.to_string()).collect(), @@ -277,7 +322,7 @@ async fn delivery_diagnose( // Fetch MTA-STS policy let now = Instant::now(); tx.send(DeliveryStage::MtaStsFetchStart).await?; - let mta_sts_policy = match server.lookup_mta_sts_policy(&domain, timeout).await { + let mta_sts_policy = match server.lookup_mta_sts_policy(domain, timeout).await { Ok(policy) => { tx.send(DeliveryStage::MtaStsFetchSuccess { policy: policy.as_ref().clone(), @@ -316,7 +361,7 @@ async fn delivery_diagnose( .smtp .resolvers .dns - .txt_lookup::( + .txt_lookup::( format!("_smtp._tls.{domain}."), Some(&server.inner.cache.dns_txt), ) @@ -697,7 +742,7 @@ async fn delivery_diagnose( // Verify recipient let mut is_success = email.is_none(); - if let Some(email) = &email { + if let Some(email) = email { // MAIL FROM tx.send(DeliveryStage::MailFromStart).await?; @@ -783,3 +828,4 @@ async fn delivery_diagnose( Ok(()) } + diff --git a/crates/http/src/api/mod.rs b/crates/http/src/api/mod.rs index 5185d381..10852631 100644 --- a/crates/http/src/api/mod.rs +++ b/crates/http/src/api/mod.rs @@ -7,9 +7,11 @@ pub mod diagnose; use crate::{ - api::diagnose::{DeliveryStage, spawn_delivery_diagnose}, + api::diagnose::{DeliveryStage, DeliveryTarget, spawn_delivery_diagnose}, auth::{ - authenticate::Authenticator, oauth::auth::OAuthApiHandler, permissions::AccountApiHandler, + authenticate::{Authenticator, HttpAuthenticated}, + oauth::{auth::OAuthApiHandler, request_origin}, + permissions::AccountApiHandler, }, }; use common::{ @@ -66,10 +68,21 @@ impl ManagementApi for Server { "auth" if is_post => { self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - Box::pin(self.handle_login_request( + self.handle_login_request( session, + request_origin(req, session), body.ok_or_else(|| trc::LimitEvent::SizeRequest.into_err())?, - )) + ) + .await + } + "auth" if path.get(1).copied() == Some("client") => { + self.is_http_anonymous_request_allowed(session.remote_ip) + .await?; + self.handle_client_info_request( + session, + request_origin(req, session), + req.uri().query().unwrap_or_default(), + ) .await } "calendar" @@ -95,27 +108,42 @@ impl ManagementApi for Server { if let Some(email) = path.get(1).copied() { self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - self.handle_discover_request(session, decode_path_element(email).as_ref()) - .await + self.handle_discover_request( + session, + request_origin(req, session), + decode_path_element(email).as_ref(), + ) + .await } else { Err(trc::ResourceEvent::NotFound.into_err()) } } "account" => { // Authenticate request - let (_in_flight, access_token) = self.authenticate_headers(req, session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(req, session).await?; self.handle_account_request(&access_token).await } "schema" => { // Authenticate request - let (_in_flight, access_token) = self.authenticate_headers(req, session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(req, session).await?; + + #[cfg(not(feature = "enterprise"))] static SCHEMA_JSON: &[u8] = - include_bytes!("../../../../resources/schema/schema.json.gz"); + include_bytes!("../../../../resources/schema/schema_oss.json.gz"); + #[cfg(not(feature = "enterprise"))] const SCHEMA_HASH: &str = - include_str!("../../../../resources/schema/schema.json.sha256"); + include_str!("../../../../resources/schema/schema_oss.json.sha256"); if path.get(1).is_some_and(|hash| hash == &SCHEMA_HASH) { - Ok(Resource::new("application/json", SCHEMA_JSON.to_vec()) + Ok(Resource::new("application/json", SCHEMA_JSON) .into_http_response() .with_immutable_cache() .with_header(CONTENT_ENCODING, "gzip")) @@ -174,11 +202,13 @@ impl ManagementApi for Server { .unwrap_or(30), ); - let mut rx = spawn_delivery_diagnose( - self.clone(), - decode_path_element(target).to_lowercase(), - timeout, - ); + let target = DeliveryTarget::parse(&decode_path_element(target)) + .ok_or_else(|| { + trc::ResourceEvent::BadParameters + .into_err() + .details("Invalid delivery target") + })?; + let mut rx = spawn_delivery_diagnose(self.clone(), target, timeout); Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") @@ -229,26 +259,37 @@ impl ManagementApi for Server { } else { self.authenticate_headers(req, session) .await - .map(|(_, token)| token) + .map(|auth| auth.access_token) } } else { self.authenticate_headers(req, session) .await - .map(|(_, token)| token) + .map(|auth| auth.access_token) } } } pub trait ToManageHttpResponse { - fn into_http_response(self, challenge: AuthChallenge) -> HttpResponse; + fn into_http_response( + self, + challenge: AuthChallenge, + resource_metadata: impl FnOnce() -> String, + ) -> HttpResponse; } impl ToManageHttpResponse for &trc::Error { - fn into_http_response(self, challenge: AuthChallenge) -> HttpResponse { + fn into_http_response( + self, + challenge: AuthChallenge, + resource_metadata: impl FnOnce() -> String, + ) -> HttpResponse { match self.as_ref() { trc::EventType::Auth( - trc::AuthEvent::Failed | trc::AuthEvent::Error | trc::AuthEvent::TokenExpired, - ) => HttpResponse::unauthorized(challenge), + trc::AuthEvent::Failed + | trc::AuthEvent::Error + | trc::AuthEvent::TokenExpired + | trc::AuthEvent::CredentialExpired, + ) => HttpResponse::unauthorized(challenge, &resource_metadata()), _ => self.to_request_error().into_http_response(), } } @@ -266,11 +307,7 @@ pub fn accept_language(req: &HttpRequest) -> &str { .unwrap_or("en") } -const BEARER_CHALLENGE: &str = concat!( - "Bearer realm=\"Stalwart Server\", ", - "resource_metadata=\"/.well-known/oauth-protected-resource\"" -); -const BASIC_CHALLENGE: &str = "Basic realm=\"Stalwart Server\""; +pub(crate) const BASIC_CHALLENGE: &str = "Basic realm=\"Stalwart Server\""; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AuthChallenge { @@ -279,13 +316,15 @@ pub enum AuthChallenge { } pub trait UnauthorizedResponse { - fn unauthorized(challenge: AuthChallenge) -> Self; + fn unauthorized(challenge: AuthChallenge, resource_metadata: &str) -> Self; } impl UnauthorizedResponse for HttpResponse { - fn unauthorized(challenge: AuthChallenge) -> Self { - let response = HttpResponse::new(StatusCode::UNAUTHORIZED) - .with_header(header::WWW_AUTHENTICATE, BEARER_CHALLENGE); + fn unauthorized(challenge: AuthChallenge, resource_metadata: &str) -> Self { + let response = HttpResponse::new(StatusCode::UNAUTHORIZED).with_header( + header::WWW_AUTHENTICATE, + format!("Bearer realm=\"Stalwart Server\", resource_metadata=\"{resource_metadata}\""), + ); if challenge == AuthChallenge::BearerAndBasic { response.with_header(header::WWW_AUTHENTICATE, BASIC_CHALLENGE) diff --git a/crates/http/src/auth/authenticate.rs b/crates/http/src/auth/authenticate.rs index 5d9f2714..41bdac12 100644 --- a/crates/http/src/auth/authenticate.rs +++ b/crates/http/src/auth/authenticate.rs @@ -4,21 +4,31 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; -use common::{HttpAuthCache, Server, auth::AuthRequest, network::limiter::InFlight}; +use common::{ + HttpAuthCache, Server, + auth::{AccessToken, AuthRequest, authentication::Authenticated}, + cache::http_auth::RevocationEpoch, + network::limiter::InFlight, +}; +use compact_str::CompactString; use directory::Credentials; +use encodify::base64::LENIENT; use http_proto::{HttpRequest, HttpSessionData}; use hyper::header; -use mail_parser::decoders::base64::base64_decode; use std::future::Future; -use std::time::{Duration, Instant}; + +pub struct HttpAuthenticated { + pub in_flight: Option, + pub access_token: AccessToken, + pub epoch: RevocationEpoch, +} pub trait Authenticator: Sync + Send { fn authenticate_headers( &self, req: &HttpRequest, session: &HttpSessionData, - ) -> impl Future, AccessToken)>> + Send; + ) -> impl Future> + Send; } impl Authenticator for Server { @@ -26,15 +36,18 @@ impl Authenticator for Server { &self, req: &HttpRequest, session: &HttpSessionData, - ) -> trc::Result<(Option, AccessToken)> { + ) -> trc::Result { + let revocations = &self.inner.cache.grant_revocations; + let epoch = revocations.epoch(); if let Some((mechanism, token)) = req.authorization() { // Check if the credentials are cached if let Some(http_cache) = self.inner.cache.http_auth.get(token) { // Make sure the revision is still valid - if http_cache.expires > Instant::now() { + if self.inner.cache.http_auth.is_live(&http_cache) { let access_token = AccessToken::renew( self.access_token(http_cache.account_id).await?, http_cache.credential_id, + http_cache.token_scope, session.remote_ip, )?; @@ -43,7 +56,11 @@ impl Authenticator for Server { return self .is_http_authenticated_request_allowed(&access_token, session.remote_ip) .await - .map(|in_flight| (in_flight, access_token)); + .map(|in_flight| HttpAuthenticated { + in_flight, + access_token, + epoch, + }); } } @@ -57,7 +74,7 @@ impl Authenticator for Server { trc::AuthEvent::Error .into_err() .details("Failed to decode Basic auth request.") - .id(token.to_string()) + .id(CompactString::from(token)) .caused_by(trc::location!()) })? } else if mechanism.eq_ignore_ascii_case("bearer") { @@ -77,12 +94,15 @@ impl Authenticator for Server { return Err(trc::AuthEvent::Error .into_err() .reason("Unsupported authentication mechanism.") - .details(token.to_string()) + .details(CompactString::from(token)) .caused_by(trc::location!())); }; // Authenticate - let access_token = self + let Authenticated { + access_token, + expires_in, + } = self .authenticate(&AuthRequest::from_credentials( credentials, session.session_id, @@ -91,21 +111,30 @@ impl Authenticator for Server { .await?; // Cache credentials - self.inner.cache.http_auth.insert( - token.into(), + let cache_ttl = expires_in.map_or(self.core.oauth.oauth_expiry_token, |expires_in| { + expires_in.min(self.core.oauth.oauth_expiry_token) + }); + revocations.insert( + &self.inner.cache.http_auth, + token, HttpAuthCache { account_id: access_token.account_id(), - revision: access_token.revision(), + expires: self.inner.cache.http_auth.expires_in(cache_ttl), credential_id: access_token.credential_id(), - expires: Instant::now() - + Duration::from_secs(self.core.oauth.oauth_expiry_token), + revision: access_token.revision(), + token_scope: access_token.token_scope(), }, + epoch, ); // Enforce authenticated rate limit self.is_http_authenticated_request_allowed(&access_token, session.remote_ip) .await - .map(|in_flight| (in_flight, access_token)) + .map(|in_flight| HttpAuthenticated { + in_flight, + access_token, + epoch, + }) } else { // Enforce anonymous rate limit self.is_http_anonymous_request_allowed(session.remote_ip) @@ -144,15 +173,13 @@ impl HttpHeaders for HttpRequest { } fn decode_plain_auth(token: &str) -> Option { - base64_decode(token.as_bytes()) - .and_then(|token| String::from_utf8(token).ok()) - .and_then(|token| { - token - .split_once(':') - .map(|(login, secret)| Credentials::Basic { - username: login.trim().to_lowercase(), - secret: secret.to_string(), - mfa_token: None, - }) - }) + LENIENT.decode_to_string(token).ok().and_then(|token| { + token + .split_once(':') + .map(|(login, secret)| Credentials::Basic { + username: login.trim().to_lowercase(), + secret: secret.to_string(), + mfa_token: None, + }) + }) } diff --git a/crates/http/src/auth/oauth/auth.rs b/crates/http/src/auth/oauth/auth.rs index 5b640e46..43f29c72 100644 --- a/crates/http/src/auth/oauth/auth.rs +++ b/crates/http/src/auth/oauth/auth.rs @@ -4,9 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{DeviceAuthResponse, FormData, MAX_POST_LEN, OAuthCode, PkceCodeChallenge}; +use super::{ + DeviceAuthResponse, ErrorType, FormData, MAX_POST_LEN, OAuthCode, PkceCodeChallenge, + TokenResponse, + authorize::{ + AuthorizationError, granted_scope, is_resource_path, issuer, resource_origin, scope_labels, + validate_authorization, + }, + logout::END_SESSION_PATH, + request_origin, + token::{ClientCredentials, TokenHandler, token_response}, +}; use crate::auth::oauth::{ - OAuthStatus, openid::OpenIdHandler, registration::ClientRegistrationHandler, + OAuthStatus, TOKEN_ENDPOINT_AUTH_METHODS, + openid::{OidcMetadataRequest, OpenIdHandler}, }; use common::{ KV_OAUTH, Server, @@ -14,16 +25,17 @@ use common::{ AuthRequest, authentication::UsernameParts, oauth::{ - CLIENT_ID_MAX_LEN, DEVICE_CODE_LEN, SUPPORTED_SCOPES, USER_CODE_ALPHABET, - USER_CODE_LEN, - client_id::{decode_client_id, scopes_to_mask}, - registration::redirect_uri_matches, + DEVICE_CODE_LEN, USER_CODE_ALPHABET, USER_CODE_LEN, + client::{RequestOrigin, ResolveContext, ResolveMode, ResolvedClient}, + supported_scopes, }, }, }; use directory::Credentials; use http_proto::*; -use std::future::Future; +use hyper::{StatusCode, body::Bytes}; +use registry::schema::enums::OAuthGrantType; +use std::{borrow::Cow, future::Future}; use store::{ Serialize, dispatch::lookup::KeyValue, @@ -35,57 +47,76 @@ use store::{ distr::{Alphanumeric, StandardUniform}, rng, }, - write::AlignedBytes, + write::ArchiveBytes, }; use trc::AddContext; #[derive(Debug, serde::Serialize)] -pub struct ProtectedResourceMetadata { +pub struct ProtectedResourceMetadata<'x> { pub resource: String, - pub authorization_servers: [String; 1], + pub authorization_servers: [Cow<'x, str>; 1], pub scopes_supported: &'static [&'static str], pub bearer_methods_supported: &'static [&'static str], } #[derive(Debug, serde::Serialize)] -pub struct OAuthMetadata { +pub struct OAuthMetadata<'x> { pub issuer: String, pub token_endpoint: String, pub authorization_endpoint: String, pub device_authorization_endpoint: String, pub registration_endpoint: String, pub introspection_endpoint: String, + pub revocation_endpoint: String, + pub end_session_endpoint: String, pub grant_types_supported: &'static [&'static str], pub response_types_supported: &'static [&'static str], pub scopes_supported: &'static [&'static str], pub token_endpoint_auth_methods_supported: &'static [&'static str], + pub revocation_endpoint_auth_methods_supported: &'static [&'static str], pub code_challenge_methods_supported: &'static [&'static str], pub authorization_response_iss_parameter_supported: bool, + pub client_id_metadata_document_supported: bool, + pub protected_resources: &'x [Box], } pub trait OAuthApiHandler: Sync + Send { fn handle_discover_request( &self, session: &HttpSessionData, + origin: RequestOrigin<'_>, account_name: &str, ) -> impl Future> + Send; fn handle_login_request( &self, session: &HttpSessionData, + origin: RequestOrigin<'_>, body: Vec, ) -> impl Future> + Send; + fn handle_client_info_request( + &self, + session: &HttpSessionData, + origin: RequestOrigin<'_>, + query: &str, + ) -> impl Future> + Send; + fn handle_device_auth( &self, req: &mut HttpRequest, session: &HttpSessionData, ) -> impl Future> + Send; - fn handle_oauth_metadata(&self) -> impl Future> + Send; + fn handle_oauth_metadata( + &self, + origin: RequestOrigin<'_>, + ) -> impl Future> + Send; fn handle_oauth_protected_resource( &self, + origin: RequestOrigin<'_>, + path: &str, ) -> impl Future> + Send; } @@ -100,30 +131,24 @@ pub enum LoginRequest { #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] mfa_token: Option, - client_id: String, + query: String, + #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] - redirect_uri: Option, - #[serde(default)] - nonce: Option, - #[serde(default)] - scope: Option, - #[serde(default)] - code_challenge: Option, - #[serde(default)] - code_challenge_method: Option, - #[serde(default)] - state: Option, - #[serde(default)] - resource: Vec, + consent: Option, }, #[serde(rename_all = "camelCase")] AuthDevice { + #[serde(default)] account_name: String, + #[serde(default)] account_secret: String, #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] mfa_token: Option, code: String, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + consent: Option, }, } @@ -131,16 +156,55 @@ pub enum LoginRequest { #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum LoginResponse { - Authenticated { client_code: String, iss: String }, + Authenticated { + location: String, + }, + Redirect { + location: String, + }, + Invalid { + error: ErrorType, + description: Cow<'static, str>, + }, Verified, + Denied, MfaRequired, + Expired, Failure, } +#[derive(Debug, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ClientInfo { + pub name: Option, + pub logo: Option, + pub client_host: Option, + pub redirect_host: Option, + pub scopes: Vec<&'static str>, + pub verified: bool, + pub consent_required: bool, + pub deny_location: Option, +} + +#[derive(Debug, serde::Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum ClientInfoResponse { + Client(ClientInfo), + Error { + error: ErrorType, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option<&'static str>, + redirect: bool, + #[serde(skip_serializing_if = "Option::is_none")] + location: Option, + }, +} + impl OAuthApiHandler for Server { async fn handle_discover_request( &self, session: &HttpSessionData, + origin: RequestOrigin<'_>, account_name: &str, ) -> trc::Result { let username = UsernameParts::new(account_name.trim()); @@ -157,17 +221,25 @@ impl OAuthApiHandler for Server { .await? .and_then(|directory| directory.oidc_discovery_document()) { - Ok(JsonResponse::new(&endpoint.document) - .no_cache() - .into_http_response()) + Ok(HttpResponse::new(StatusCode::OK) + .with_content_type("application/json; charset=utf-8") + .with_binary_body(Bytes::from_owner(endpoint.web_document.clone())) + .with_no_store()) } else { - self.handle_oidc_metadata(!session.is_tls).await + self.handle_oidc_metadata( + OidcMetadataRequest::Discover { + is_tls: session.is_tls, + }, + origin, + ) + .await } } async fn handle_login_request( &self, session: &HttpSessionData, + origin: RequestOrigin<'_>, body: Vec, ) -> trc::Result { let request = serde_json::from_slice::(&body).map_err(|err| { @@ -179,191 +251,86 @@ impl OAuthApiHandler for Server { account_name, account_secret, mfa_token, - client_id, - redirect_uri, - nonce, - scope, - code_challenge, - code_challenge_method, - resource, - .. + query, + consent, } => { - // Validate clientId - if client_id.len() > CLIENT_ID_MAX_LEN { - return Err(trc::AuthEvent::Error - .into_err() - .details("Client ID is too long.")); - } else if redirect_uri - .as_ref() - .is_some_and(|uri| uri.starts_with("http://")) - { - #[cfg(not(feature = "dev_mode"))] - if !self.registry().is_recovery_mode() && code_challenge.is_none() { - return Err(trc::AuthEvent::Error - .into_err() - .details("Redirect URI must be HTTPS.")); - } - } - - // Resolve the client and validate the redirect URI against the registration. - // Stateless client ids are self-describing; otherwise fall back to the registry. - let redirect_uri = redirect_uri.ok_or_else(|| { - trc::AuthEvent::Error - .into_err() - .details("A redirect URI is required.") - })?; - let stateless_client = - decode_client_id(self.core.oauth.oauth_key.as_bytes(), &client_id); - let granted_scope = match &stateless_client { - Some(meta) => { - if !meta - .redirect_uris - .iter() - .any(|uri| redirect_uri_matches(uri, &redirect_uri)) - { - return Err(trc::AuthEvent::Error - .into_err() - .details("Redirect URI does not match the client registration.")); - } - grant_scope(scope.as_deref(), meta.scope_mask) - } - None => scope, - }; - - // Validate Resource Indicators (RFC 8707) - for resource in &resource { - if !is_known_resource( - [self.core.network.server_name.as_str()] - .into_iter() - .chain( - self.core - .network - .info - .services - .values() - .filter_map(|v| v.hostname.as_deref()), - ) - .chain( - self.core - .network - .info - .mxs - .iter() - .filter_map(|mx| mx.hostname.as_deref()), - ), - resource, - ) { - return Err(trc::AuthEvent::Error - .into_err() - .details(format!("Unknown resource indicator: {}", resource))); - } - } - - // Parse and validate PKCE challenge (RFC 7636). - let pkce_challenge = match code_challenge { - Some(challenge) => match code_challenge_method.as_deref().unwrap_or("plain") { - "S256" => PkceCodeChallenge::S256(challenge), - "plain" if stateless_client.is_none() => { - PkceCodeChallenge::Plain(challenge) - } - _ => { - return Err(trc::AuthEvent::Error - .into_err() - .details("Unsupported PKCE code_challenge_method.")); - } - }, - None => { - if stateless_client.is_some() { - return Err(trc::AuthEvent::Error.into_err().details( - "A PKCE code_challenge with the S256 method is required.", - )); - } - PkceCodeChallenge::None - } - }; - - // Authenticate - match self - .authenticate(&AuthRequest { - credentials: Credentials::Basic { - username: account_name, - secret: account_secret, - mfa_token, - }, - session_id: session.session_id, + let issuer = issuer(self, origin); + let authorization = match validate_authorization( + self, + &query, + &ResolveContext { + mode: ResolveMode::Authorize, + origin: Some(origin), remote_ip: session.remote_ip, - }) - .await - { - Ok(access_token) => { - // Registry-backed clients are validated once the account is known - if stateless_client.is_none() - && self - .validate_client_registration( - &client_id, - Some(redirect_uri.as_str()), - access_token.account_id(), - ) - .await? - .is_some() - { - return Err(trc::AuthEvent::Error - .into_err() - .details("Invalid client registration.")); - } - - // Generate client code - let client_code = rng() - .sample_iter(Alphanumeric) - .take(DEVICE_CODE_LEN) - .map(char::from) - .collect::(); - - // Serialize OAuth code - let value = Archiver::new(OAuthCode { - status: OAuthStatus::Authorized, - account_id: access_token.account_id(), - client_id, - nonce, - params: redirect_uri, - code_challenge: pkce_challenge, - scope: granted_scope, - resources: resource, - }) - .untrusted() - .serialize() - .caused_by(trc::location!())?; - - // Insert client code - self.in_memory_store() - .key_set( - KeyValue::with_prefix(KV_OAUTH, client_code.as_bytes(), value) - .expires(self.core.oauth.oauth_expiry_auth_code), - ) - .await?; - - LoginResponse::Authenticated { - client_code, - iss: self.core.network.http.url_https.clone(), - } - } - Err(err) => match *err.as_ref() { - trc::EventType::Auth(trc::AuthEvent::MfaRequired) => { - trc::error!(err.span_id(session.session_id)); - LoginResponse::MfaRequired - } - trc::EventType::Auth(_) => { - trc::error!(err.span_id(session.session_id)); - LoginResponse::Failure - } - trc::EventType::Security(_) => { - trc::error!(err.span_id(session.session_id)); - LoginResponse::Failure - } - _ => { - return Err(err); - } }, + &issuer, + ) + .await? + { + Ok(authorization) => authorization, + Err(AuthorizationError::Invalid { error, description }) => { + return Ok(JsonResponse::new(LoginResponse::Invalid { + error, + description: description.into(), + }) + .no_cache() + .into_http_response()); + } + Err(AuthorizationError::Redirect { location }) => { + return Ok(JsonResponse::new(LoginResponse::Redirect { location }) + .no_cache() + .into_http_response()); + } + }; + + if authorization.client.requires_consent() && consent != Some(true) { + LoginResponse::Redirect { + location: authorization.error_location(ErrorType::AccessDenied, &issuer), + } + } else { + match authenticate_login( + self, + session, + &authorization.client, + account_name, + account_secret, + mfa_token, + ) + .await? + { + Ok(login) => { + let client_code = rng() + .sample_iter(Alphanumeric) + .take(DEVICE_CODE_LEN) + .map(char::from) + .collect::(); + let location = authorization.code_location(&client_code, &issuer); + + let value = Archiver::new(OAuthCode { + status: OAuthStatus::Authorized, + account_id: login.account_id, + credential_version: login.credential_version, + client_id: authorization.client_id, + nonce: authorization.nonce, + params: authorization.redirect_uri, + code_challenge: authorization.code_challenge, + scope: authorization.scope, + }) + .untrusted() + .serialize() + .caused_by(trc::location!())?; + + self.in_memory_store() + .key_set( + KeyValue::with_prefix(KV_OAUTH, client_code.as_bytes(), value) + .expires(self.core.oauth.oauth_expiry_auth_code), + ) + .await?; + + LoginResponse::Authenticated { location } + } + Err(response) => response, + } } } LoginRequest::AuthDevice { @@ -371,97 +338,85 @@ impl OAuthApiHandler for Server { account_secret, mfa_token, code, + consent, } => { - // Obtain code - let mut result = LoginResponse::Failure; - if let Some(auth_code_) = self - .in_memory_store() - .key_get::>(KeyValue::<()>::build_key( - KV_OAUTH, - code.as_bytes(), - )) - .await? - { - let oauth = auth_code_ - .unarchive::() - .caused_by(trc::location!())?; - if oauth.status == OAuthStatus::Pending { - // Authenticate - match self - .authenticate(&AuthRequest { - credentials: Credentials::Basic { - username: account_name, - secret: account_secret, - mfa_token, - }, - session_id: session.session_id, - remote_ip: session.remote_ip, - }) - .await - { - Ok(access_token) => { - let new_oauth_code = OAuthCode { - status: OAuthStatus::Authorized, - account_id: access_token.account_id(), - client_id: oauth.client_id.to_string(), - nonce: oauth.nonce.as_ref().map(|s| s.to_string()), - params: Default::default(), - code_challenge: PkceCodeChallenge::None, - scope: oauth.scope.as_ref().map(|s| s.to_string()), - resources: oauth - .resources - .iter() - .map(|s| s.to_string()) - .collect(), - }; + approve_device( + self, + session, + account_name, + account_secret, + mfa_token, + &code, + consent, + ) + .await? + } + }; - // Delete issued user code - self.in_memory_store() - .key_delete(KeyValue::<()>::build_key( - KV_OAUTH, - code.as_bytes(), - )) - .await?; + Ok(JsonResponse::new(response).no_cache().into_http_response()) + } - // Update device code status - self.in_memory_store() - .key_set( - KeyValue::with_prefix( - KV_OAUTH, - oauth.params.as_bytes(), - Archiver::new(new_oauth_code) - .untrusted() - .serialize() - .caused_by(trc::location!())?, - ) - .expires(self.core.oauth.oauth_expiry_auth_code), - ) - .await?; - - result = LoginResponse::Verified; - } - Err(err) => match *err.as_ref() { - trc::EventType::Auth(trc::AuthEvent::MfaRequired) => { - trc::error!(err.span_id(session.session_id)); - result = LoginResponse::MfaRequired; - } - trc::EventType::Auth(_) => { - trc::error!(err.span_id(session.session_id)); - result = LoginResponse::Failure; - } - trc::EventType::Security(_) => { - trc::error!(err.span_id(session.session_id)); - result = LoginResponse::Failure; - } - _ => { - return Err(err); - } - }, - } + async fn handle_client_info_request( + &self, + session: &HttpSessionData, + origin: RequestOrigin<'_>, + query: &str, + ) -> trc::Result { + let user_code = form_urlencoded::parse(query.as_bytes()) + .find_map(|(name, value)| (name == "user_code").then_some(value)); + let response = if let Some(user_code) = user_code { + device_client_info(self, &user_code, session) + .await? + .map(ClientInfoResponse::Client) + .unwrap_or_else(|| ClientInfoResponse::Error { + error: ErrorType::InvalidGrant, + description: Some("The device code is invalid or has expired."), + redirect: false, + location: None, + }) + } else { + let issuer = issuer(self, origin); + match validate_authorization( + self, + query, + &ResolveContext { + mode: ResolveMode::Authorize, + origin: Some(origin), + remote_ip: session.remote_ip, + }, + &issuer, + ) + .await? + { + Ok(authorization) => { + let client = &authorization.client; + ClientInfoResponse::Client(ClientInfo { + name: client.display_name().map(str::to_string), + logo: client.display_logo().map(str::to_string), + client_host: client.client_host().map(str::to_string), + redirect_host: authorization.redirect_host(), + scopes: authorization.scope_labels(), + verified: client.is_verified(), + consent_required: client.requires_consent(), + deny_location: Some( + authorization.error_location(ErrorType::AccessDenied, &issuer), + ), + }) + } + Err(AuthorizationError::Invalid { error, description }) => { + ClientInfoResponse::Error { + error, + description: Some(description), + redirect: false, + location: None, } } - - result + Err(AuthorizationError::Redirect { location }) => ClientInfoResponse::Error { + error: ErrorType::InvalidRequest, + description: None, + redirect: true, + location: Some(location), + }, } }; @@ -473,20 +428,47 @@ impl OAuthApiHandler for Server { req: &mut HttpRequest, session: &HttpSessionData, ) -> trc::Result { - // Parse form let mut form_data = FormData::from_request(req, MAX_POST_LEN, session.session_id).await?; - let client_id = form_data - .remove("client_id") - .filter(|client_id| client_id.len() <= CLIENT_ID_MAX_LEN) - .ok_or_else(|| { - trc::ResourceEvent::BadParameters - .into_err() - .details("Client ID is missing.") - })?; + let credentials = ClientCredentials::new(req, &form_data); + let Some(client_id) = credentials + .client_id + .as_deref() + .filter(|_| !credentials.is_ambiguous) + .map(str::to_string) + else { + return Ok(token_response( + TokenResponse::error(ErrorType::InvalidRequest), + credentials.used_basic, + )); + }; + let client = match self + .resolve_token_client( + &client_id, + &credentials, + &ResolveContext { + mode: ResolveMode::Token, + origin: Some(request_origin(req, session)), + remote_ip: session.remote_ip, + }, + OAuthGrantType::DeviceCode, + ) + .await? + { + Ok(client) => client, + Err(error) => { + return Ok(token_response( + TokenResponse::error(error), + credentials.used_basic, + )); + } + }; + let used_basic = credentials.used_basic; + drop(credentials); + let scope = match granted_scope(form_data.get("scope"), client.allowed_scopes()) { + Ok(scope) => scope, + Err(error) => return Ok(token_response(TokenResponse::error(error), used_basic)), + }; let nonce = form_data.remove("nonce"); - let scope = form_data - .remove("scope") - .and_then(|scope| grant_scope(Some(&scope), u64::MAX)); // Generate device code let device_code = rng() @@ -513,12 +495,12 @@ impl OAuthApiHandler for Server { let oauth_code = Archiver::new(OAuthCode { status: OAuthStatus::Pending, account_id: u32::MAX, + credential_version: 0, client_id, nonce, params: device_code.clone(), code_challenge: PkceCodeChallenge::None, scope, - resources: Vec::new(), }) .untrusted() .serialize() @@ -554,14 +536,17 @@ impl OAuthApiHandler for Server { .into_http_response()) } - async fn handle_oauth_metadata(&self) -> trc::Result { - let base_url = &self.core.network.http.url_https; + async fn handle_oauth_metadata(&self, origin: RequestOrigin<'_>) -> trc::Result { + let base_url = issuer(self, origin); + let resources = &self.core.network.resources; Ok(JsonResponse::new(OAuthMetadata { authorization_endpoint: format!("{base_url}/login",), token_endpoint: format!("{base_url}/auth/token"), device_authorization_endpoint: format!("{base_url}/auth/device"), introspection_endpoint: format!("{base_url}/auth/introspect"), + revocation_endpoint: format!("{base_url}/auth/revoke"), + end_session_endpoint: format!("{base_url}{END_SESSION_PATH}"), registration_endpoint: format!("{base_url}/auth/register"), grant_types_supported: &[ "authorization_code", @@ -569,27 +554,38 @@ impl OAuthApiHandler for Server { "urn:ietf:params:oauth:grant-type:device_code", ], response_types_supported: &["code"], - scopes_supported: SUPPORTED_SCOPES, - token_endpoint_auth_methods_supported: &[ - "none", - "client_secret_post", - "client_secret_basic", - ], + scopes_supported: supported_scopes(resources.serves_files), + token_endpoint_auth_methods_supported: TOKEN_ENDPOINT_AUTH_METHODS, + revocation_endpoint_auth_methods_supported: TOKEN_ENDPOINT_AUTH_METHODS, code_challenge_methods_supported: &["S256"], authorization_response_iss_parameter_supported: true, - issuer: base_url.to_string(), + client_id_metadata_document_supported: true, + protected_resources: &resources.identifiers, + issuer: base_url.into_owned(), }) .into_http_response() .with_cors_unrestricted()) } - async fn handle_oauth_protected_resource(&self) -> trc::Result { - let base_url = &self.core.network.http.url_https; + async fn handle_oauth_protected_resource( + &self, + origin: RequestOrigin<'_>, + path: &str, + ) -> trc::Result { + let Some(mut resource) = resource_origin(self, origin).filter(|_| is_resource_path(path)) + else { + return Err(trc::ResourceEvent::NotFound.into_err()); + }; + resource.push_str(path); + let authorization_server = match self.external_issuer_for_host(origin.authority).await? { + Some(issuer) => Cow::Owned(issuer), + None => issuer(self, origin), + }; Ok(JsonResponse::new(ProtectedResourceMetadata { - resource: base_url.to_string(), - authorization_servers: [base_url.to_string()], - scopes_supported: SUPPORTED_SCOPES, + resource, + authorization_servers: [authorization_server], + scopes_supported: supported_scopes(self.core.network.resources.serves_files), bearer_methods_supported: &["header"], }) .into_http_response() @@ -597,53 +593,227 @@ impl OAuthApiHandler for Server { } } -fn grant_scope(requested: Option<&str>, registered_mask: u64) -> Option { - let mut granted = String::new(); - for scope in requested.unwrap_or_default().split_ascii_whitespace() { - let bit = scopes_to_mask(scope); - if bit != 0 && registered_mask & bit == bit { - if !granted.is_empty() { - granted.push(' '); - } - granted.push_str(scope); - } +async fn device_client_info( + server: &Server, + user_code: &str, + session: &HttpSessionData, +) -> trc::Result> { + if !is_user_code(user_code) { + return Ok(None); + } + let Some(stored) = server + .in_memory_store() + .key_get::>(KeyValue::<()>::build_key(KV_OAUTH, user_code.as_bytes())) + .await? + else { + return Ok(None); + }; + let oauth = stored + .unarchive::() + .caused_by(trc::location!())?; + if oauth.status != OAuthStatus::Pending { + return Ok(None); } - (!granted.is_empty()).then_some(granted) + match server + .resolve_oauth_client( + oauth.client_id.as_str(), + &ResolveContext { + mode: ResolveMode::Token, + origin: None, + remote_ip: session.remote_ip, + }, + ) + .await? + { + Ok(client) if client.allows_grant(OAuthGrantType::DeviceCode) => Ok(Some(ClientInfo { + name: client.display_name().map(str::to_string), + logo: client.display_logo().map(str::to_string), + client_host: None, + redirect_host: None, + scopes: scope_labels(&client, oauth.scope.as_ref().map(|scope| scope.as_str())), + verified: client.is_verified(), + consent_required: true, + deny_location: None, + })), + _ => Ok(None), + } } -fn is_known_resource<'x>(hostnames: impl IntoIterator, uri: &str) -> bool { - let Some((scheme, rest)) = uri.split_once("://") else { - return false; +struct LoginAccount { + account_id: u32, + credential_version: u64, +} + +async fn authenticate_login( + server: &Server, + session: &HttpSessionData, + client: &ResolvedClient, + username: String, + secret: String, + mfa_token: Option, +) -> trc::Result> { + match server + .authenticate(&AuthRequest { + credentials: Credentials::Basic { + username, + secret, + mfa_token, + }, + session_id: session.session_id, + remote_ip: session.remote_ip, + }) + .await + { + Ok(authenticated) => { + let access_token = authenticated.access_token; + let account_id = access_token.account_id(); + if !client.is_first_party() && access_token.is_main_credential_expired() { + trc::event!( + Auth(trc::AuthEvent::CredentialExpired), + AccountId = account_id, + SpanId = session.session_id, + Reason = "Main credential expired, refusing sign-in to third-party client.", + ); + return Ok(Err(LoginResponse::Expired)); + } + + Ok(Ok(LoginAccount { + account_id, + credential_version: access_token.credential_version(), + })) + } + Err(err) => { + let response = match *err.as_ref() { + trc::EventType::Auth(trc::AuthEvent::MfaRequired) => LoginResponse::MfaRequired, + trc::EventType::Auth(_) | trc::EventType::Security(_) => LoginResponse::Failure, + _ => return Err(err), + }; + trc::error!(err.span_id(session.session_id)); + Ok(Err(response)) + } + } +} + +async fn approve_device( + server: &Server, + session: &HttpSessionData, + account_name: String, + account_secret: String, + mfa_token: Option, + code: &str, + consent: Option, +) -> trc::Result { + let Some(consent) = consent.filter(|_| is_user_code(code)) else { + return Ok(LoginResponse::Failure); }; - let supported = hashify::tiny_map!(scheme.as_bytes(), - b"http" => true, - b"https" => true, - b"smtp" => true, - b"smtps" => true, - b"imap" => true, - b"imaps" => true, - b"pop3" => true, - b"pop3s" => true, - b"caldav" => true, - b"caldavs" => true, - b"webdav" => true, - b"webdavs" => true, - b"carddav" => true, - b"carddavs" => true, - b"sieve" => true, - b"sieves" => true - ) - .unwrap_or(false); + let Some(stored) = server + .in_memory_store() + .key_get::>(KeyValue::<()>::build_key(KV_OAUTH, code.as_bytes())) + .await? + else { + return Ok(LoginResponse::Failure); + }; + let oauth = stored + .unarchive::() + .caused_by(trc::location!())?; + if oauth.status != OAuthStatus::Pending { + return Ok(LoginResponse::Failure); + } - let authority = rest.split_once('/').map_or(rest, |(auth, _)| auth); - let host = authority - .rsplit_once(':') - .filter(|(_, port)| !port.is_empty() && port.as_bytes().iter().all(|c| c.is_ascii_digit())) - .map_or(authority, |(host, _)| host); + let denied = || { + ( + OAuthStatus::Denied, + u32::MAX, + 0, + server.core.oauth.oauth_expiry_user_code, + LoginResponse::Denied, + ) + }; + let (status, account_id, credential_version, expiry, response) = if consent { + match server + .resolve_oauth_client( + oauth.client_id.as_str(), + &ResolveContext { + mode: ResolveMode::Token, + origin: None, + remote_ip: session.remote_ip, + }, + ) + .await? + { + Ok(client) => match authenticate_login( + server, + session, + &client, + account_name, + account_secret, + mfa_token, + ) + .await? + { + Ok(login) => ( + OAuthStatus::Authorized, + login.account_id, + login.credential_version, + server.core.oauth.oauth_expiry_auth_code, + LoginResponse::Verified, + ), + Err(LoginResponse::Expired) => { + let (status, account_id, credential_version, expiry, _) = denied(); + ( + status, + account_id, + credential_version, + expiry, + LoginResponse::Expired, + ) + } + Err(response) => return Ok(response), + }, + Err(_) => denied(), + } + } else { + denied() + }; - supported - && hostnames - .into_iter() - .any(|hostname| host.eq_ignore_ascii_case(hostname)) + let value = Archiver::new(OAuthCode { + status, + account_id, + credential_version, + client_id: oauth.client_id.to_string(), + nonce: oauth.nonce.as_ref().map(|nonce| nonce.to_string()), + params: String::new(), + code_challenge: PkceCodeChallenge::None, + scope: oauth.scope.as_ref().map(|scope| scope.to_string()), + }) + .untrusted() + .serialize() + .caused_by(trc::location!())?; + + if !server + .in_memory_store() + .key_delete(KeyValue::<()>::build_key(KV_OAUTH, code.as_bytes())) + .await? + { + return Ok(LoginResponse::Failure); + } + server + .in_memory_store() + .key_set(KeyValue::with_prefix(KV_OAUTH, oauth.params.as_bytes(), value).expires(expiry)) + .await?; + + Ok(response) } + +fn is_user_code(code: &str) -> bool { + code.len() == USER_CODE_LEN + 1 + && code.bytes().enumerate().all(|(pos, ch)| { + if pos == USER_CODE_LEN / 2 { + ch == b'-' + } else { + USER_CODE_ALPHABET.contains(&ch) + } + }) +} + diff --git a/crates/http/src/auth/oauth/authorize.rs b/crates/http/src/auth/oauth/authorize.rs new file mode 100644 index 00000000..ad65f49d --- /dev/null +++ b/crates/http/src/auth/oauth/authorize.rs @@ -0,0 +1,499 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ErrorType, PkceCodeChallenge}; +use common::{ + Server, + auth::oauth::{ + client::{RequestOrigin, ResolveContext, ResolvedClient}, + registration::redirect_display_host, + scope::OAuthScopes, + }, + manager::app_client::is_safe_authority, +}; +use http_proto::form_urlencoded; +use registry::schema::enums::OAuthGrantType; +use std::borrow::Cow; + +const MAX_QUERY_LEN: usize = 8 * 1024; +const MAX_RESOURCES: usize = 16; +const PKCE_S256_CHALLENGE_LEN: usize = 43; +const LABEL_FULL_ACCESS: &str = "Full access to your account"; +pub const PROTECTED_RESOURCE_PATH: &str = "/.well-known/oauth-protected-resource"; + +#[derive(Debug)] +pub struct Authorization { + pub client: ResolvedClient, + pub client_id: String, + pub redirect_uri: String, + pub state: String, + pub nonce: Option, + pub scope: Option, + pub code_challenge: PkceCodeChallenge, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AuthorizationError { + Invalid { + error: ErrorType, + description: &'static str, + }, + Redirect { + location: String, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Param { + ClientId, + RedirectUri, + ResponseType, + State, + Scope, + Nonce, + CodeChallenge, + CodeChallengeMethod, + Resource, +} + +#[derive(Default)] +struct AuthorizationQuery<'x> { + client_id: Option>, + redirect_uri: Option>, + response_type: Option>, + state: Option>, + scope: Option>, + nonce: Option>, + code_challenge: Option>, + code_challenge_method: Option>, + resources: Vec>, + duplicates: u16, + too_many_resources: bool, +} + +impl Param { + fn parse(name: &str) -> Option { + hashify::map!( + name.as_bytes(), + Param, + "client_id" => Param::ClientId, + "redirect_uri" => Param::RedirectUri, + "response_type" => Param::ResponseType, + "state" => Param::State, + "scope" => Param::Scope, + "nonce" => Param::Nonce, + "code_challenge" => Param::CodeChallenge, + "code_challenge_method" => Param::CodeChallengeMethod, + "resource" => Param::Resource, + ) + .copied() + } + + fn bit(self) -> u16 { + 1 << (self as u16) + } +} + +impl<'x> AuthorizationQuery<'x> { + fn parse(query: &'x str) -> Self { + let mut parsed = AuthorizationQuery::default(); + for (name, value) in form_urlencoded::parse(query.as_bytes()) { + let Some(param) = Param::parse(&name) else { + continue; + }; + let slot = match param { + Param::Resource => { + if parsed.resources.len() < MAX_RESOURCES { + parsed.resources.push(value); + } else { + parsed.too_many_resources = true; + } + continue; + } + Param::ClientId => &mut parsed.client_id, + Param::RedirectUri => &mut parsed.redirect_uri, + Param::ResponseType => &mut parsed.response_type, + Param::State => &mut parsed.state, + Param::Scope => &mut parsed.scope, + Param::Nonce => &mut parsed.nonce, + Param::CodeChallenge => &mut parsed.code_challenge, + Param::CodeChallengeMethod => &mut parsed.code_challenge_method, + }; + if slot.is_some() { + parsed.duplicates |= param.bit(); + } else { + *slot = Some(value); + } + } + parsed + } + + fn is_duplicate(&self, param: Param) -> bool { + self.duplicates & param.bit() != 0 + } +} + +pub async fn validate_authorization( + server: &Server, + query: &str, + ctx: &ResolveContext<'_>, + issuer: &str, +) -> trc::Result> { + if query.len() > MAX_QUERY_LEN { + return Ok(Err(AuthorizationError::invalid( + ErrorType::InvalidRequest, + "The authorization request is too large.", + ))); + } + + let query = AuthorizationQuery::parse(query); + if query.is_duplicate(Param::ClientId) || query.is_duplicate(Param::RedirectUri) { + return Ok(Err(AuthorizationError::invalid( + ErrorType::InvalidRequest, + "The client_id and redirect_uri parameters must not be repeated.", + ))); + } + let Some(client_id) = query.client_id.as_deref().filter(|id| !id.is_empty()) else { + return Ok(Err(AuthorizationError::invalid( + ErrorType::InvalidRequest, + "The client_id parameter is missing.", + ))); + }; + let client = match server.resolve_oauth_client(client_id, ctx).await? { + Ok(client) => client, + Err(rejection) => { + return Ok(Err(AuthorizationError::invalid( + ErrorType::InvalidClient, + rejection.as_str(), + ))); + } + }; + let Some(redirect_uri) = query + .redirect_uri + .as_deref() + .filter(|uri| client.redirect_uri_matches(uri)) + else { + return Ok(Err(AuthorizationError::invalid( + ErrorType::InvalidRequest, + "The redirect_uri parameter is missing or does not match the client registration.", + ))); + }; + + let state = query + .state + .as_deref() + .filter(|state| !state.is_empty() && !query.is_duplicate(Param::State)); + let redirect = |error: ErrorType, description: &str| { + Ok(Err(AuthorizationError::Redirect { + location: error_location(redirect_uri, error, Some(description), state, issuer), + })) + }; + + if query.duplicates != 0 { + return redirect( + ErrorType::InvalidRequest, + "Request parameters must not be repeated.", + ); + } else if query.too_many_resources { + return redirect( + ErrorType::InvalidTarget, + "Too many resource parameters were requested.", + ); + } + match query.response_type.as_deref() { + Some("code") => {} + Some(_) => { + return redirect( + ErrorType::UnsupportedResponseType, + "Only the code response type is supported.", + ); + } + None => { + return redirect( + ErrorType::InvalidRequest, + "The response_type parameter is missing.", + ); + } + } + let Some(state) = state else { + return redirect(ErrorType::InvalidRequest, "The state parameter is missing."); + }; + if !client.allows_grant(OAuthGrantType::AuthorizationCode) { + return redirect( + ErrorType::UnauthorizedClient, + "The client is not allowed to use the authorization code grant.", + ); + } + + let code_challenge = match ( + query.code_challenge.as_deref(), + query.code_challenge_method.as_deref(), + ) { + (Some(challenge), Some("S256")) if is_s256_challenge(challenge) => { + PkceCodeChallenge::S256(challenge.to_string()) + } + (Some(_), Some("S256")) => { + return redirect( + ErrorType::InvalidRequest, + "The code_challenge parameter is not a valid S256 challenge.", + ); + } + (None, None) if client.is_confidential() => PkceCodeChallenge::None, + (None, None) => { + return redirect( + ErrorType::InvalidRequest, + "A PKCE code_challenge is required.", + ); + } + _ => { + return redirect( + ErrorType::InvalidRequest, + "The code_challenge_method must be S256.", + ); + } + }; + + let scope = match granted_scope(query.scope.as_deref(), client.allowed_scopes()) { + Ok(scope) => scope, + Err(error) => { + return redirect(error, "The requested scope is not allowed for this client."); + } + }; + + for resource in &query.resources { + if !is_permitted_resource(server, ctx.origin, resource) { + return redirect( + ErrorType::InvalidTarget, + "The requested resource is not served by this server.", + ); + } + } + + Ok(Ok(Authorization { + client_id: client_id.to_string(), + redirect_uri: redirect_uri.to_string(), + state: state.to_string(), + nonce: query.nonce.map(Cow::into_owned), + scope, + code_challenge, + client, + })) +} + +impl Authorization { + pub fn error_location(&self, error: ErrorType, issuer: &str) -> String { + error_location(&self.redirect_uri, error, None, Some(&self.state), issuer) + } + + pub fn code_location(&self, code: &str, issuer: &str) -> String { + redirect_location( + &self.redirect_uri, + [("code", code), ("state", &self.state), ("iss", issuer)], + ) + } + + pub fn scope_labels(&self) -> Vec<&'static str> { + scope_labels(&self.client, self.scope.as_deref()) + } + + pub fn redirect_host(&self) -> Option { + redirect_display_host(&self.redirect_uri) + } +} + +impl AuthorizationError { + fn invalid(error: ErrorType, description: &'static str) -> Self { + AuthorizationError::Invalid { error, description } + } +} + +pub fn granted_scope( + requested: Option<&str>, + allowed: Option, +) -> Result, ErrorType> { + let mut scope = String::new(); + for item in requested.unwrap_or_default().split_ascii_whitespace() { + if let Some(allowed) = allowed { + let scope = OAuthScopes::from_name(item); + if scope.is_empty() || !allowed.contains(scope) { + return Err(ErrorType::InvalidScope); + } + } + if !scope.is_empty() { + scope.push(' '); + } + scope.push_str(item); + } + Ok((!scope.is_empty()).then_some(scope)) +} + +pub fn scope_labels(client: &ResolvedClient, scope: Option<&str>) -> Vec<&'static str> { + if client.allowed_scopes().is_none() { + return vec![LABEL_FULL_ACCESS]; + } + + let mut seen = OAuthScopes::NONE; + let mut labels = Vec::new(); + for item in scope.unwrap_or_default().split_ascii_whitespace() { + let bit = OAuthScopes::from_name(item); + if !seen.intersects(bit) + && let Some(label) = scope_label(item) + { + seen |= bit; + labels.push(label); + } + } + labels +} + +fn scope_label(scope: &str) -> Option<&'static str> { + hashify::map!( + scope.as_bytes(), + &'static str, + "openid" => "See your account name and email address", + "offline_access" => "Stay connected when you are not using it", + "urn:ietf:params:oauth:scope:mail" => "Read, send and manage your email", + "urn:ietf:params:oauth:scope:contacts" => "Read and manage your contacts", + "urn:ietf:params:oauth:scope:calendars" => "Read and manage your calendars", + "urn:ietf:params:oauth:scope:files" => "Read and manage your files", + ) + .copied() +} + +pub fn issuer<'x>(server: &'x Server, origin: RequestOrigin<'_>) -> Cow<'x, str> { + issuer_for(&server.core.network.http.url_https, origin) +} + +fn issuer_for<'x>(url_https: &'x str, origin: RequestOrigin<'_>) -> Cow<'x, str> { + if !url_https.is_empty() && (origin.is_tls || !cfg!(feature = "dev_mode")) { + Cow::Borrowed(url_https) + } else { + Cow::Owned(format!("{}://{}", origin.scheme(), origin.authority)) + } +} + +pub fn resource_origin(server: &Server, origin: RequestOrigin<'_>) -> Option { + if !is_safe_authority(origin.authority) { + None + } else if server.registry().is_bootstrap_mode() || server.registry().is_recovery_mode() { + Some(format!("{}://{}", origin.scheme(), origin.authority)) + } else if server + .core + .network + .resources + .is_trusted_host(origin.authority) + { + Some(format!("https://{}", origin.authority)) + } else { + None + } +} + +pub fn resource_metadata_url(server: &Server, origin: RequestOrigin<'_>, path: &str) -> String { + let mut url = resource_origin(server, origin) + .unwrap_or_else(|| server.core.network.http.url_https.clone()); + url.push_str(PROTECTED_RESOURCE_PATH); + if path != "/" && is_resource_path(path) { + url.push_str(path); + } + url +} + +pub fn is_resource_path(path: &str) -> bool { + path.is_empty() + || (path.starts_with('/') + && path.bytes().all(|b| { + b.is_ascii_alphanumeric() + || matches!( + b, + b'-' | b'.' + | b'_' + | b'~' + | b'/' + | b'%' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b':' + | b'@' + ) + })) +} + +fn is_permitted_resource( + server: &Server, + origin: Option>, + resource: &str, +) -> bool { + server.core.network.resources.is_known_resource(resource) + || ((server.registry().is_bootstrap_mode() || server.registry().is_recovery_mode()) + && origin.is_some_and(|origin| { + resource + .strip_prefix(origin.scheme()) + .and_then(|rest| rest.strip_prefix("://")) + .and_then(|rest| rest.strip_prefix(origin.authority)) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('/')) + && !resource.contains('#') + })) +} + +fn is_s256_challenge(challenge: &str) -> bool { + challenge.len() == PKCE_S256_CHALLENGE_LEN + && challenge + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) +} + +fn error_location( + redirect_uri: &str, + error: ErrorType, + description: Option<&str>, + state: Option<&str>, + issuer: &str, +) -> String { + let mut params = [ + ("error", error.as_str()), + ("", ""), + ("", ""), + ("iss", issuer), + ]; + if let Some(description) = description { + params[1] = ("error_description", description); + } + if let Some(state) = state { + params[2] = ("state", state); + } + redirect_location( + redirect_uri, + params.into_iter().filter(|(name, _)| !name.is_empty()), + ) +} + +pub(super) fn redirect_location<'x>( + redirect_uri: &str, + params: impl IntoIterator, +) -> String { + let mut location = String::with_capacity(redirect_uri.len() + 192); + location.push_str(redirect_uri); + if !redirect_uri.contains('?') { + location.push('?'); + } else if !redirect_uri.ends_with(['?', '&']) { + location.push('&'); + } + let start = location.len(); + form_urlencoded::Serializer::for_suffix(location, start) + .extend_pairs(params) + .finish() +} + diff --git a/crates/http/src/auth/oauth/logout.rs b/crates/http/src/auth/oauth/logout.rs new file mode 100644 index 00000000..e256c80c --- /dev/null +++ b/crates/http/src/auth/oauth/logout.rs @@ -0,0 +1,177 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + authorize::{issuer, redirect_location}, + request_origin, +}; +use common::{Server, auth::oauth::client::RequestOrigin}; +use http_proto::{ + HttpRequest, HttpResponse, HttpSessionData, form_urlencoded, request::fetch_body, +}; +use hyper::{ + Method, StatusCode, + header::{CONTENT_SECURITY_POLICY, X_FRAME_OPTIONS}, +}; +use std::{borrow::Cow, future::Future}; + +pub const END_SESSION_PATH: &str = "/auth/logout"; +const MAX_LOGOUT_REQUEST_LEN: usize = 8 * 1024; +const LOGGED_OUT_PAGE: &str = concat!( + "", + "", + "Signed out", + "", + "

You have been signed out

", + "

You can close this window.

" +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Param { + ClientId, + PostLogoutRedirectUri, + State, + IdTokenHint, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct EndSessionParams<'x> { + pub client_id: Option>, + pub post_logout_redirect_uri: Option>, + pub state: Option>, + pub id_token_hint: Option>, +} + +pub trait EndSessionHandler: Sync + Send { + fn handle_end_session( + &self, + req: &mut HttpRequest, + session: &HttpSessionData, + ) -> impl Future> + Send; + + fn post_logout_location( + &self, + params: &EndSessionParams<'_>, + origin: RequestOrigin<'_>, + ) -> impl Future>> + Send; +} + +impl EndSessionHandler for Server { + async fn handle_end_session( + &self, + req: &mut HttpRequest, + session: &HttpSessionData, + ) -> trc::Result { + let body = if req.method() == Method::POST { + fetch_body(req, MAX_LOGOUT_REQUEST_LEN, session.session_id).await + } else { + None + }; + let query = match &body { + Some(body) => body.as_slice(), + None => req.uri().query().unwrap_or_default().as_bytes(), + }; + let params = if query.len() <= MAX_LOGOUT_REQUEST_LEN { + EndSessionParams::parse(query) + } else { + None + }; + let location = match params { + Some(params) => { + self.post_logout_location(¶ms, request_origin(req, session)) + .await? + } + None => None, + }; + + Ok(match location { + Some(location) => HttpResponse::new(StatusCode::SEE_OTHER) + .with_location(location) + .with_no_store(), + None => HttpResponse::new(StatusCode::OK) + .with_content_type("text/html; charset=utf-8") + .with_header(X_FRAME_OPTIONS, "DENY") + .with_header( + CONTENT_SECURITY_POLICY, + "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'", + ) + .with_no_store() + .with_text_body(LOGGED_OUT_PAGE), + }) + } + + async fn post_logout_location( + &self, + params: &EndSessionParams<'_>, + origin: RequestOrigin<'_>, + ) -> trc::Result> { + let Some(redirect_uri) = params.post_logout_redirect_uri.as_deref() else { + return Ok(None); + }; + let hint_audience = match params.id_token_hint.as_deref() { + Some(hint) => match self.verify_id_token_hint(hint, &issuer(self, origin)) { + Some(hint) => Some(hint.aud), + None => return Ok(None), + }, + None => None, + }; + let client_id = match (params.client_id.as_deref(), hint_audience.as_deref()) { + (Some(client_id), Some(audience)) if client_id != audience => return Ok(None), + (Some(client_id), _) => client_id, + (None, Some(audience)) => audience, + (None, None) => return Ok(None), + }; + + if !self + .is_post_logout_redirect_uri_allowed(client_id, redirect_uri, origin) + .await? + { + return Ok(None); + } + + Ok(Some(match params.state.as_deref() { + Some(state) => redirect_location(redirect_uri, [("state", state)]), + None => redirect_uri.to_string(), + })) + } +} + +impl Param { + fn parse(name: &str) -> Option { + hashify::map!( + name.as_bytes(), + Param, + "client_id" => Param::ClientId, + "post_logout_redirect_uri" => Param::PostLogoutRedirectUri, + "state" => Param::State, + "id_token_hint" => Param::IdTokenHint, + ) + .copied() + } +} + +impl<'x> EndSessionParams<'x> { + pub fn parse(query: &'x [u8]) -> Option { + let mut params = EndSessionParams::default(); + for (name, value) in form_urlencoded::parse(query) { + let slot = match Param::parse(&name) { + Some(Param::ClientId) => &mut params.client_id, + Some(Param::PostLogoutRedirectUri) => &mut params.post_logout_redirect_uri, + Some(Param::State) => &mut params.state, + Some(Param::IdTokenHint) => &mut params.id_token_hint, + None => continue, + }; + if slot.is_some() { + return None; + } + if !value.is_empty() { + *slot = Some(value); + } + } + Some(params) + } +} + diff --git a/crates/http/src/auth/oauth/mod.rs b/crates/http/src/auth/oauth/mod.rs index d1b91c1b..0ee77e5c 100644 --- a/crates/http/src/auth/oauth/mod.rs +++ b/crates/http/src/auth/oauth/mod.rs @@ -4,12 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use http_proto::{HttpRequest, request::fetch_body}; -use hyper::header::CONTENT_TYPE; +use common::auth::oauth::client::RequestOrigin; +use http_proto::{HttpRequest, HttpSessionData, request::fetch_body}; +use hyper::header::{CONTENT_TYPE, HOST}; use serde::{Deserialize, Serialize}; use utils::map::vec_map::VecMap; pub mod auth; +pub mod authorize; +pub mod logout; pub mod openid; pub mod registration; pub mod token; @@ -31,9 +34,12 @@ pub enum OAuthStatus { Authorized, TokenIssued, Pending, + Denied, } const MAX_POST_LEN: usize = 2048; +pub(crate) const TOKEN_ENDPOINT_AUTH_METHODS: &[&str] = + &["none", "client_secret_post", "client_secret_basic"]; pub struct OAuth { pub key: String, @@ -50,12 +56,12 @@ pub struct OAuth { pub struct OAuthCode { pub status: OAuthStatus, pub account_id: u32, + pub credential_version: u64, pub client_id: String, pub nonce: Option, pub params: String, pub code_challenge: PkceCodeChallenge, pub scope: Option, - pub resources: Vec, } #[derive( @@ -73,7 +79,6 @@ pub struct OAuthCode { pub enum PkceCodeChallenge { None, S256(String), - Plain(String), } #[derive(Debug, Serialize, Deserialize)] @@ -143,13 +148,13 @@ pub struct OAuthResponse { pub expires_in: u64, #[serde(skip_serializing_if = "Option::is_none")] pub refresh_token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub scope: Option, + pub scope: String, + pub username: String, #[serde(skip_serializing_if = "Option::is_none")] pub id_token: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub enum ErrorType { #[serde(rename = "invalid_grant")] InvalidGrant, @@ -171,6 +176,29 @@ pub enum ErrorType { AccessDenied, #[serde(rename = "expired_token")] ExpiredToken, + #[serde(rename = "invalid_target")] + InvalidTarget, + #[serde(rename = "unsupported_response_type")] + UnsupportedResponseType, +} + +impl ErrorType { + pub fn as_str(&self) -> &'static str { + match self { + ErrorType::InvalidGrant => "invalid_grant", + ErrorType::InvalidClient => "invalid_client", + ErrorType::InvalidScope => "invalid_scope", + ErrorType::InvalidRequest => "invalid_request", + ErrorType::UnauthorizedClient => "unauthorized_client", + ErrorType::UnsupportedGrantType => "unsupported_grant_type", + ErrorType::AuthorizationPending => "authorization_pending", + ErrorType::SlowDown => "slow_down", + ErrorType::AccessDenied => "access_denied", + ErrorType::ExpiredToken => "expired_token", + ErrorType::InvalidTarget => "invalid_target", + ErrorType::UnsupportedResponseType => "unsupported_response_type", + } + } } impl TokenResponse { @@ -240,3 +268,19 @@ impl FormData { self.fields.iter() } } + +pub fn request_origin<'x>(req: &'x HttpRequest, session: &HttpSessionData) -> RequestOrigin<'x> { + RequestOrigin { + is_tls: session.is_tls, + authority: req + .headers() + .get(HOST) + .and_then(|host| host.to_str().ok()) + .or_else(|| req.uri().authority().map(|authority| authority.as_str())) + .unwrap_or_default(), + } +} + +impl store::write::ArchiveCompression for OAuthCode { + const COMPRESSION: store::write::Compression = store::write::Compression::None; +} diff --git a/crates/http/src/auth/oauth/openid.rs b/crates/http/src/auth/oauth/openid.rs index 3407f51e..2170a405 100644 --- a/crates/http/src/auth/oauth/openid.rs +++ b/crates/http/src/auth/oauth/openid.rs @@ -4,13 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::oauth::SUPPORTED_SCOPES, auth::oauth::oidc::Userinfo}; +use super::{TOKEN_ENDPOINT_AUTH_METHODS, authorize::issuer, logout::END_SESSION_PATH}; +use common::{ + Server, + auth::oauth::{SCOPE_OPENID, client::RequestOrigin, oidc::Userinfo, supported_scopes}, +}; use http_proto::*; use serde::Serialize; use std::future::Future; #[derive(Debug, Serialize)] -pub struct OpenIdMetadata { +pub struct OpenIdMetadata<'x> { pub issuer: String, pub authorization_endpoint: String, pub token_endpoint: String, @@ -18,17 +22,32 @@ pub struct OpenIdMetadata { pub jwks_uri: String, pub registration_endpoint: String, pub device_authorization_endpoint: String, + pub revocation_endpoint: String, + pub end_session_endpoint: String, pub scopes_supported: &'static [&'static str], pub response_types_supported: &'static [&'static str], pub subject_types_supported: &'static [&'static str], pub grant_types_supported: &'static [&'static str], pub token_endpoint_auth_methods_supported: &'static [&'static str], - pub id_token_signing_alg_values_supported: &'static [&'static str], + pub revocation_endpoint_auth_methods_supported: &'static [&'static str], + pub id_token_signing_alg_values_supported: [&'static str; 1], pub claims_supported: &'static [&'static str], pub code_challenge_methods_supported: &'static [&'static str], pub authorization_response_iss_parameter_supported: bool, + pub client_id_metadata_document_supported: bool, + pub protected_resources: &'x [Box], + #[serde(skip_serializing_if = "Option::is_none")] + pub stalwart_required_scopes: Option<&'static [&'static str]>, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OidcMetadataRequest { + WellKnown, + Discover { is_tls: bool }, +} + +const DISCOVER_REQUIRED_SCOPES: &[&str] = &[SCOPE_OPENID]; + pub trait OpenIdHandler: Sync + Send { fn handle_userinfo_request( &self, @@ -37,7 +56,8 @@ pub trait OpenIdHandler: Sync + Send { fn handle_oidc_metadata( &self, - strip_base_url: bool, + request: OidcMetadataRequest, + origin: RequestOrigin<'_>, ) -> impl Future> + Send; } @@ -57,11 +77,17 @@ impl OpenIdHandler for Server { .into_http_response()) } - async fn handle_oidc_metadata(&self, strip_base_url: bool) -> trc::Result { - let base_url = if strip_base_url { + async fn handle_oidc_metadata( + &self, + request: OidcMetadataRequest, + origin: RequestOrigin<'_>, + ) -> trc::Result { + let strip_base_url = matches!(request, OidcMetadataRequest::Discover { is_tls: false }); + let request_issuer = issuer(self, origin); + let base_url: &str = if strip_base_url { #[cfg(feature = "dev_mode")] { - "http://127.0.0.1:8080" + &request_issuer } #[cfg(not(feature = "dev_mode"))] @@ -69,7 +95,12 @@ impl OpenIdHandler for Server { "" } } else { - &self.core.network.http.url_https + &request_issuer + }; + let issuer = if strip_base_url && !self.core.network.http.url_https.is_empty() { + base_url + } else { + &request_issuer }; Ok(JsonResponse::new(OpenIdMetadata { @@ -79,23 +110,19 @@ impl OpenIdHandler for Server { jwks_uri: format!("{base_url}/auth/jwks.json"), registration_endpoint: format!("{base_url}/auth/register"), device_authorization_endpoint: format!("{base_url}/auth/device"), + revocation_endpoint: format!("{base_url}/auth/revoke"), + end_session_endpoint: format!("{base_url}{END_SESSION_PATH}"), response_types_supported: &["code"], grant_types_supported: &[ "authorization_code", "refresh_token", "urn:ietf:params:oauth:grant-type:device_code", ], - scopes_supported: SUPPORTED_SCOPES, + scopes_supported: supported_scopes(self.core.network.resources.serves_files), subject_types_supported: &["public"], - token_endpoint_auth_methods_supported: &[ - "none", - "client_secret_post", - "client_secret_basic", - ], - id_token_signing_alg_values_supported: &[ - "RS256", "RS384", "RS512", "ES256", "ES384", "PS256", "PS384", "PS512", "HS256", - "HS384", "HS512", - ], + token_endpoint_auth_methods_supported: TOKEN_ENDPOINT_AUTH_METHODS, + revocation_endpoint_auth_methods_supported: TOKEN_ENDPOINT_AUTH_METHODS, + id_token_signing_alg_values_supported: [self.core.oauth.oidc_signature_algorithm_name], claims_supported: &[ "sub", "name", @@ -105,7 +132,11 @@ impl OpenIdHandler for Server { ], code_challenge_methods_supported: &["S256"], authorization_response_iss_parameter_supported: true, - issuer: base_url.to_string(), + client_id_metadata_document_supported: true, + protected_resources: &self.core.network.resources.identifiers, + stalwart_required_scopes: matches!(request, OidcMetadataRequest::Discover { .. }) + .then_some(DISCOVER_REQUIRED_SCOPES), + issuer: issuer.to_string(), }) .into_http_response() .with_cors_unrestricted()) diff --git a/crates/http/src/auth/oauth/registration.rs b/crates/http/src/auth/oauth/registration.rs index c1c56f73..481b002f 100644 --- a/crates/http/src/auth/oauth/registration.rs +++ b/crates/http/src/auth/oauth/registration.rs @@ -8,34 +8,36 @@ use super::ErrorType; use crate::auth::authenticate::Authenticator; use common::{ Server, - auth::{ - BuildAccessToken, - oauth::{ - client_id::{ClientMeta, decode_client_id, encode_client_id, scopes_to_mask}, - registration::{ - ClientRegistrationError, ClientRegistrationRequest, ClientRegistrationResponse, - TokenEndpointAuthMethod, redirect_uri_matches, validate_grant_metadata, - validate_redirect_uri, - }, + auth::oauth::{ + CLIENT_ID_MAX_LEN, + client::ResolvedClient, + client_id::{ClientMeta, encode_client_id}, + registration::{ + ClientRegistrationError, ClientRegistrationRequest, ClientRegistrationResponse, + TokenEndpointAuthMethod, parse_grant_type, parse_oauth_scope, validate_client_name, + validate_grant_metadata, validate_oauth_client, validate_public_client_metadata, + validate_redirect_uri, }, + scope::OAuthScopes, }, }; +use compact_str::CompactString; use directory::core::secret::{hash_secret, verify_secret_hash}; use http_proto::{request::fetch_body, *}; use hyper::StatusCode; +use jmap::registry::mapping::oauth_client::is_superuser; use registry::schema::{ - enums::{PasswordHashAlgorithm, Permission}, - prelude::{ObjectType, Property, UTCDateTime}, + enums::{OAuthGrantType, PasswordHashAlgorithm, Permission}, + prelude::UTCDateTime, structs::OAuthClient, }; -use std::future::Future; +use std::{future::Future, net::IpAddr}; use store::{ rand::{RngExt, distr::Alphanumeric, rng}, registry::write::{RegistryWrite, RegistryWriteResult}, write::now, }; use trc::{AddContext, AuthEvent}; -use types::id::Id; pub trait ClientRegistrationHandler: Sync + Send { fn handle_oauth_registration_request( @@ -44,26 +46,20 @@ pub trait ClientRegistrationHandler: Sync + Send { session: HttpSessionData, ) -> impl Future> + Send; - fn validate_client_registration( - &self, - client_id: &str, - redirect_uri: Option<&str>, - account_id: u32, - ) -> impl Future>> + Send; - fn verify_client_secret( &self, - client_id: &str, + client: &ResolvedClient, client_secret: Option<&str>, + remote_ip: IpAddr, ) -> impl Future>> + Send; } + impl ClientRegistrationHandler for Server { async fn handle_oauth_registration_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> trc::Result { - // Parse request let body = fetch_body(req, 20 * 1024, session.session_id).await; let request = serde_json::from_slice::( body.as_deref().unwrap_or_default(), @@ -72,7 +68,6 @@ impl ClientRegistrationHandler for Server { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; - // Validate redirect URIs and grant metadata (RFC 7591 + OAuth Public Clients profile) if request.redirect_uris.is_empty() { return Ok(registration_error( ClientRegistrationError::invalid_redirect_uri( @@ -80,35 +75,28 @@ impl ClientRegistrationHandler for Server { ), )); } - for uri in &request.redirect_uris { - if let Err(err) = validate_redirect_uri(uri) { - return Ok(registration_error(err)); - } - } - if let Err(err) = validate_grant_metadata(&request) { + if let Err(err) = + validate_grant_metadata(&request).and_then(|_| validate_client_name(&request)) + { return Ok(registration_error(err)); } - let is_public = matches!( + if matches!( request.token_endpoint_auth_method, None | Some(TokenEndpointAuthMethod::None) - ); - - if is_public { - // Public client: issue a stateless, self-describing client id with no database write - if self.core.oauth.allow_anonymous_client_registration { - self.is_http_anonymous_request_allowed(session.remote_ip) - .await?; - } else { - let (_, access_token) = self.authenticate_headers(req, &session).await?; - access_token.enforce_permission(Permission::OAuthClientRegistration)?; + ) { + if let Err(err) = validate_public_client_metadata(&request) { + return Ok(registration_error(err)); } + self.is_http_anonymous_request_allowed(session.remote_ip) + .await?; + let client_id = encode_client_id( self.core.oauth.oauth_key.as_bytes(), &ClientMeta { redirect_uris: request.redirect_uris.clone(), - scope_mask: scopes_to_mask(request.scope.as_deref().unwrap_or_default()), + scopes: OAuthScopes::parse(request.scope.as_deref().unwrap_or_default()), client_name: request.client_name.clone(), }, ) @@ -119,6 +107,13 @@ impl ClientRegistrationHandler for Server { .reason(err) .caused_by(trc::location!()) })?; + if client_id.len() > CLIENT_ID_MAX_LEN { + return Ok(registration_error( + ClientRegistrationError::invalid_client_metadata( + "The client metadata is too large to be encoded in a client id.", + ), + )); + } trc::event!( Auth(AuthEvent::ClientRegistration), @@ -139,47 +134,101 @@ impl ClientRegistrationHandler for Server { .into_http_response()); } - // Confidential client: authenticate and persist the registration - let (_, access_token) = self.authenticate_headers(req, &session).await?; - access_token.enforce_permission(Permission::OAuthClientRegistration)?; - let tenant_id = access_token.tenant_id(); + for uri in &request.redirect_uris { + if let Err(err) = validate_redirect_uri(uri) { + return Ok(registration_error(err)); + } + } + if !matches!( + request.token_endpoint_auth_method, + Some( + TokenEndpointAuthMethod::ClientSecretBasic + | TokenEndpointAuthMethod::ClientSecretPost + ) + ) { + return Ok(registration_error( + ClientRegistrationError::invalid_client_metadata( + "token_endpoint_auth_method must be none, client_secret_basic or client_secret_post.", + ), + )); + } + + let access_token = self.authenticate_headers(req, &session).await?.access_token; + access_token.enforce_permission(Permission::SysOAuthClientCreate)?; + + let mut scopes = Vec::new(); + for scope in request + .scope + .as_deref() + .unwrap_or_default() + .split_ascii_whitespace() + .filter_map(parse_oauth_scope) + { + if !scopes.contains(&scope) { + scopes.push(scope); + } + } + let mut grant_types = Vec::new(); + for grant_type in &request.grant_types { + match parse_grant_type(grant_type) { + Some(grant_type) if !grant_types.contains(&grant_type) => { + grant_types.push(grant_type) + } + Some(_) => {} + None => { + return Ok(registration_error( + ClientRegistrationError::invalid_client_metadata("Unsupported grant type."), + )); + } + } + } + if grant_types.is_empty() { + grant_types = vec![ + OAuthGrantType::AuthorizationCode, + OAuthGrantType::RefreshToken, + ]; + } - // Generate client ID let client_id = rng() .sample_iter(Alphanumeric) .take(20) .map(|ch| char::from(ch.to_ascii_lowercase())) .collect::(); + let mut client = OAuthClient { + client_id: client_id.clone(), + description: request.client_name.clone(), + contacts: request.contacts.clone().into(), + redirect_uris: request.redirect_uris.clone().into(), + logo: request.logo_uri.clone(), + created_at: UTCDateTime::now(), + first_party: false, + scopes: scopes.into(), + grant_types: grant_types.into(), + ..Default::default() + }; + + if let Err((_, err)) = validate_oauth_client(&client, is_superuser(&access_token)) { + return Ok(registration_error(err)); + } - // Generate client secret let client_secret = rng() .sample_iter(Alphanumeric) .take(48) .map(char::from) .collect::(); - let secret_hash = hash_secret( - PasswordHashAlgorithm::Argon2id, - client_secret.clone().into_bytes(), - ) - .await - .caused_by(trc::location!())?; + client.secret = Some( + hash_secret( + &self.inner.data.password_verifier, + PasswordHashAlgorithm::Argon2id, + client_secret.clone().into_bytes(), + ) + .await + .caused_by(trc::location!())?, + ); let result = self .registry() - .write(RegistryWrite::insert( - &OAuthClient { - client_id: client_id.clone(), - description: request.client_name.clone(), - contacts: request.contacts.clone().into(), - member_tenant_id: tenant_id.map(|id| Id::new(id as u64)), - redirect_uris: request.redirect_uris.clone().into(), - logo: request.logo_uri.clone(), - secret: Some(secret_hash), - created_at: UTCDateTime::now(), - ..Default::default() - } - .into(), - )) + .write(RegistryWrite::insert(&client.into())) .await .caused_by(trc::location!())?; @@ -187,13 +236,13 @@ impl ClientRegistrationHandler for Server { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Failed to register OAuth client.") - .reason(result.to_string()) + .reason(result) .caused_by(trc::location!())); } trc::event!( Auth(AuthEvent::ClientRegistration), - Id = client_id.to_string(), + Id = CompactString::from(&client_id), RemoteIp = session.remote_ip ); @@ -212,119 +261,37 @@ impl ClientRegistrationHandler for Server { .into_http_response()) } - async fn validate_client_registration( - &self, - client_id: &str, - redirect_uri: Option<&str>, - account_id: u32, - ) -> trc::Result> { - // Stateless client ids are self-describing and validated at the authorization endpoint - if decode_client_id(self.core.oauth.oauth_key.as_bytes(), client_id).is_some() { - return Ok(None); - } - if !self.core.oauth.require_client_authentication { - return Ok(None); - } - - // Fetch client registration - let found_registration = if let Some(client_id) = self - .registry() - .primary_key( - ObjectType::OAuthClient.into(), - Property::ClientId, - client_id.as_bytes().to_vec(), - ) - .await? - { - if let Some(redirect_uri) = redirect_uri { - let client = self - .registry() - .object::(client_id.id()) - .await? - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .details("OAuth client not found.") - .caused_by(trc::location!()) - .ctx(trc::Key::Id, client_id.id().id()) - })?; - if client - .redirect_uris - .iter() - .any(|uri| redirect_uri_matches(uri, redirect_uri)) - { - return Ok(None); - } - } else { - // Device flow does not require a redirect URI - - return Ok(None); - } - - true - } else { - false - }; - - // Check if the account is allowed to override client registration - if self - .access_token(account_id) - .await - .caused_by(trc::location!())? - .build() - .has_permission(Permission::OAuthClientOverride) - { - return Ok(None); - } - - Ok(Some(if found_registration { - ErrorType::InvalidClient - } else { - ErrorType::InvalidRequest - })) - } - async fn verify_client_secret( &self, - client_id: &str, + client: &ResolvedClient, client_secret: Option<&str>, + remote_ip: IpAddr, ) -> trc::Result> { - // Stateless and unregistered clients have no secret to verify - if decode_client_id(self.core.oauth.oauth_key.as_bytes(), client_id).is_some() { - return Ok(None); - } - let Some(client_id) = self - .registry() - .primary_key( - ObjectType::OAuthClient.into(), - Property::ClientId, - client_id.as_bytes().to_vec(), - ) - .await? - else { - return Ok(None); - }; - let Some(client) = self - .registry() - .object::(client_id.id()) - .await - .caused_by(trc::location!())? - else { - return Ok(None); - }; - - match client.secret.as_deref() { - Some(hash) if !hash.is_empty() => match client_secret { - Some(secret) - if verify_secret_hash(hash, secret.as_bytes()) - .await - .caused_by(trc::location!())? => + match (client.secret_hash(), client_secret) { + (Some(hash), Some(secret)) + if verify_secret_hash( + &self.inner.data.password_verifier, + hash, + secret.as_bytes(), + ) + .await + .caused_by(trc::location!())? => + { + Ok(None) + } + (Some(_), secret) => { + if secret.is_some() + && self.has_auth_fail2ban() + && self.is_auth_fail2banned(remote_ip, None).await? { - Ok(None) + Err(trc::SecurityEvent::AuthenticationBan + .into_err() + .ctx(trc::Key::RemoteIp, remote_ip)) + } else { + Ok(Some(ErrorType::InvalidClient)) } - _ => Ok(Some(ErrorType::InvalidClient)), - }, - _ => Ok(None), + } + (None, _) => Ok(None), } } } diff --git a/crates/http/src/auth/oauth/token.rs b/crates/http/src/auth/oauth/token.rs index 5b499807..8ccf06f3 100644 --- a/crates/http/src/auth/oauth/token.rs +++ b/crates/http/src/auth/oauth/token.rs @@ -6,27 +6,36 @@ use super::{ ArchivedOAuthStatus, ArchivedPkceCodeChallenge, ErrorType, FormData, MAX_POST_LEN, OAuthCode, - OAuthResponse, OAuthStatus, TokenResponse, registration::ClientRegistrationHandler, -}; -use crate::auth::authenticate::HttpHeaders; -use base64::{ - Engine, - engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}, + OAuthResponse, OAuthStatus, TokenResponse, authorize::issuer, + registration::ClientRegistrationHandler, request_origin, }; +use crate::{api::BASIC_CHALLENGE, auth::authenticate::HttpHeaders}; use common::{ KV_OAUTH, Server, auth::{ - AccessToken, - oauth::{GrantType, oidc::StandardClaims}, + RECOVERY_ADMIN_ID, + oauth::{ + GrantType, + client::{ResolveContext, ResolveMode, ResolvedClient}, + grant::{GrantDeletion, GrantRotation, NewOAuthGrant, RECOVERY_ADMIN_GRANT_ID}, + oidc::StandardClaims, + registration::GRANT_TYPE_DEVICE_CODE, + scope::OAuthScopes, + token::{RefreshTokenParams, TokenGrant}, + }, }, + cache::invalidate::CacheInvalidationBuilder, }; +use encodify::base64::{STANDARD, URL_SAFE_NO_PAD}; use http_proto::*; -use hyper::StatusCode; -use sha2::{Digest, Sha256}; -use std::{borrow::Cow, future::Future}; +use hyper::{StatusCode, header::WWW_AUTHENTICATE}; +use percent_encoding::percent_decode_str; +use registry::schema::enums::OAuthGrantType; +use sha2::{Digest, Sha256, digest::Output}; +use std::{borrow::Cow, future::Future, net::IpAddr}; use store::{ dispatch::lookup::KeyValue, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes, now}, }; use trc::AddContext; @@ -40,251 +49,108 @@ pub trait TokenHandler: Sync + Send { fn handle_token_introspect( &self, req: &mut HttpRequest, - access_token: &AccessToken, session_id: u64, ) -> impl Future> + Send; - #[allow(clippy::too_many_arguments)] - fn issue_token( + fn handle_token_revocation( + &self, + req: &mut HttpRequest, + session: HttpSessionData, + ) -> impl Future> + Send; + + fn consume_oauth_code(&self, code: &str) -> impl Future> + Send; + + fn authenticate_token_client( &self, - account_id: u32, client_id: &str, - issuer: String, - nonce: Option, - scope: Option, - with_refresh_token: bool, - with_id_token: bool, - ) -> impl Future> + Send; + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + ) -> impl Future>> + Send; + + fn resolve_token_client( + &self, + client_id: &str, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + grant: OAuthGrantType, + ) -> impl Future>> + Send; + + fn authorization_code_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + issuer: &str, + ) -> impl Future> + Send; + + fn device_code_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + issuer: &str, + ) -> impl Future> + Send; + + fn refresh_token_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + session_id: u64, + ) -> impl Future> + Send; + + fn issue_new_grant( + &self, + request: NewGrantRequest<'_>, + ) -> impl Future> + Send; + + fn grant_response(&self, response: GrantResponse<'_>) -> trc::Result; } impl TokenHandler for Server { - // Token endpoint + async fn consume_oauth_code(&self, code: &str) -> trc::Result { + self.in_memory_store() + .key_delete(KeyValue::<()>::build_key(KV_OAUTH, code.as_bytes())) + .await + } + async fn handle_token_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> trc::Result { - // Parse form let params = FormData::from_request(req, MAX_POST_LEN, session.session_id).await?; + let credentials = ClientCredentials::new(req, ¶ms); + let origin = request_origin(req, &session); + let issuer = issuer(self, origin); + let ctx = ResolveContext { + mode: ResolveMode::Token, + origin: Some(origin), + remote_ip: session.remote_ip, + }; let grant_type = params.get("grant_type").unwrap_or_default(); - let (client_id_cred, client_secret_cred) = client_credentials(req, ¶ms); - let mut response = TokenResponse::error(ErrorType::InvalidGrant); - - let issuer = self.core.network.http.url_https.to_string(); - - if grant_type.eq_ignore_ascii_case("authorization_code") { - response = if let (Some(code), Some(client_id), Some(redirect_uri)) = ( - params.get("code"), - client_id_cred.as_deref(), - params.get("redirect_uri"), - ) { - // Obtain code - match self - .in_memory_store() - .key_get::>(KeyValue::<()>::build_key( - KV_OAUTH, - code.as_bytes(), - )) - .await? - { - Some(auth_code_) => { - let oauth = auth_code_ - .unarchive::() - .caused_by(trc::location!())?; - if client_id != oauth.client_id || redirect_uri != oauth.params { - TokenResponse::error(ErrorType::InvalidClient) - } else if !verify_pkce(&oauth.code_challenge, params.get("code_verifier")) { - TokenResponse::error(ErrorType::InvalidGrant) - } else if oauth.status == OAuthStatus::Authorized { - // Validate client id - if let Some(error) = self - .validate_client_registration( - client_id, - redirect_uri.into(), - oauth.account_id.into(), - ) - .await? - { - TokenResponse::error(error) - } else if let Some(error) = self - .verify_client_secret(client_id, client_secret_cred.as_deref()) - .await? - { - TokenResponse::error(error) - } else { - // Mark this token as issued - self.in_memory_store() - .key_delete(KeyValue::<()>::build_key( - KV_OAUTH, - code.as_bytes(), - )) - .await?; - - // Issue token - self.issue_token( - oauth.account_id.into(), - &oauth.client_id, - issuer, - oauth.nonce.as_ref().map(|s| s.as_str().into()), - oauth.scope.as_ref().map(|s| s.as_str().into()), - true, - true, - ) - .await - .map(TokenResponse::Granted) - .map_err(|err| { - trc::AuthEvent::Error - .into_err() - .details(err) - .caused_by(trc::location!()) - })? - } - } else { - TokenResponse::error(ErrorType::InvalidGrant) - } - } - None => TokenResponse::error(ErrorType::AccessDenied), - } - } else { - TokenResponse::error(ErrorType::InvalidClient) - }; - } else if grant_type.eq_ignore_ascii_case("urn:ietf:params:oauth:grant-type:device_code") { - response = TokenResponse::error(ErrorType::ExpiredToken); - - if let (Some(device_code), Some(client_id)) = - (params.get("device_code"), params.get("client_id")) - { - // Obtain code - if let Some(auth_code_) = self - .in_memory_store() - .key_get::>(KeyValue::<()>::build_key( - KV_OAUTH, - device_code.as_bytes(), - )) - .await? - { - let oauth = auth_code_ - .unarchive::() - .caused_by(trc::location!())?; - response = if oauth.client_id != client_id { - TokenResponse::error(ErrorType::InvalidClient) - } else { - match oauth.status { - ArchivedOAuthStatus::Authorized => { - if let Some(error) = self - .validate_client_registration( - client_id, - None, - oauth.account_id.into(), - ) - .await? - { - TokenResponse::error(error) - } else { - // Mark this token as issued - self.in_memory_store() - .key_delete(KeyValue::<()>::build_key( - KV_OAUTH, - device_code.as_bytes(), - )) - .await?; - - // Issue token - self.issue_token( - oauth.account_id.into(), - &oauth.client_id, - issuer, - oauth.nonce.as_ref().map(|s| s.as_str().into()), - oauth.scope.as_ref().map(|s| s.as_str().into()), - true, - true, - ) - .await - .map(TokenResponse::Granted) - .map_err(|err| { - trc::AuthEvent::Error - .into_err() - .details(err) - .caused_by(trc::location!()) - })? - } - } - ArchivedOAuthStatus::Pending => { - TokenResponse::error(ErrorType::AuthorizationPending) - } - ArchivedOAuthStatus::TokenIssued => { - TokenResponse::error(ErrorType::ExpiredToken) - } - } - }; - } - } + let response = if credentials.is_ambiguous { + TokenResponse::error(ErrorType::InvalidRequest) + } else if grant_type.eq_ignore_ascii_case("authorization_code") { + self.authorization_code_grant(¶ms, &credentials, &ctx, &issuer) + .await? + } else if grant_type.eq_ignore_ascii_case(GRANT_TYPE_DEVICE_CODE) { + self.device_code_grant(¶ms, &credentials, &ctx, &issuer) + .await? } else if grant_type.eq_ignore_ascii_case("refresh_token") { - if let Some(refresh_token) = params.get("refresh_token") { - if let Some(client_id) = client_id_cred.as_deref() - && let Some(error) = self - .verify_client_secret(client_id, client_secret_cred.as_deref()) - .await? - { - return Ok(JsonResponse::with_status( - StatusCode::BAD_REQUEST, - TokenResponse::error(error), - ) - .into_http_response()); - } - response = match self - .validate_access_token(GrantType::RefreshToken.into(), refresh_token) - .await - { - Ok(token_info) => self - .issue_token( - token_info.account_id, - "", - issuer, - None, - None, - token_info.expires_in - <= self.core.oauth.oauth_expiry_refresh_token_renew, - false, - ) - .await - .map(TokenResponse::Granted) - .map_err(|err| { - trc::AuthEvent::Error - .into_err() - .details(err) - .caused_by(trc::location!()) - })?, - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to validate refresh token") - .span_id(session.session_id) - ); - TokenResponse::error(ErrorType::InvalidGrant) - } - }; - } else { - response = TokenResponse::error(ErrorType::InvalidRequest); - } - } + self.refresh_token_grant(¶ms, &credentials, &ctx, session.session_id) + .await? + } else { + TokenResponse::error(ErrorType::UnsupportedGrantType) + }; - Ok(JsonResponse::with_status( - if response.is_error() { - StatusCode::BAD_REQUEST - } else { - StatusCode::OK - }, - response, - ) - .into_http_response()) + Ok(token_response(response, credentials.used_basic)) } async fn handle_token_introspect( &self, req: &mut HttpRequest, - access_token: &AccessToken, session_id: u64, ) -> trc::Result { // Parse token @@ -297,112 +163,572 @@ impl TokenHandler for Server { .details("Client ID is missing.") })?; - self.introspect_access_token(&token, access_token) + self.introspect_access_token(&token) .await .map(|response| JsonResponse::new(response).no_cache().into_http_response()) } - async fn issue_token( + async fn handle_token_revocation( + &self, + req: &mut HttpRequest, + session: HttpSessionData, + ) -> trc::Result { + let params = FormData::from_request(req, MAX_POST_LEN, session.session_id).await?; + let credentials = ClientCredentials::new(req, ¶ms); + let token = match params.get("token") { + Some(token) if !token.is_empty() && !credentials.is_ambiguous => token, + _ => { + return Ok(token_response( + TokenResponse::error(ErrorType::InvalidRequest), + credentials.used_basic, + )); + } + }; + let Some(client_id) = credentials.client_id.as_deref() else { + return Ok(token_response( + TokenResponse::error(ErrorType::InvalidClient), + credentials.used_basic, + )); + }; + + let Some((account_id, grant)) = self + .decode_access_token(None, token) + .ok() + .filter(|info| { + matches!( + info.grant_type, + GrantType::AccessToken | GrantType::RefreshToken + ) + }) + .and_then(|info| info.grant.map(|grant| (info.account_id, grant))) + else { + return Ok(revocation_response()); + }; + let Some(stored) = self.oauth_grant(grant.id).await?.filter(|stored| { + stored.account_id() == account_id && stored.grant.client_id == client_id + }) else { + return Ok(revocation_response()); + }; + + let ctx = ResolveContext { + mode: ResolveMode::Token, + origin: Some(request_origin(req, &session)), + remote_ip: session.remote_ip, + }; + if let Err(error) = self + .authenticate_token_client(client_id, &credentials, &ctx) + .await? + { + return Ok(token_response( + TokenResponse::error(error), + credentials.used_basic, + )); + } + + let mut invalidator = CacheInvalidationBuilder::default(); + self.delete_oauth_grant(stored, GrantDeletion::Unconditional, &mut invalidator) + .await?; + self.invalidate_caches(invalidator).await?; + + Ok(revocation_response()) + } + + async fn authenticate_token_client( &self, - account_id: u32, client_id: &str, - issuer: String, - nonce: Option, - scope: Option, - with_refresh_token: bool, - with_id_token: bool, - ) -> trc::Result { - let credential_version = self + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + ) -> trc::Result> { + let client = match self.resolve_oauth_client(client_id, ctx).await? { + Ok(client) => client, + Err(_) => return Ok(Err(ErrorType::InvalidClient)), + }; + if let Some(error) = self + .verify_client_secret(&client, credentials.client_secret.as_deref(), ctx.remote_ip) + .await? + { + Ok(Err(error)) + } else { + Ok(Ok(client)) + } + } + + async fn resolve_token_client( + &self, + client_id: &str, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + grant: OAuthGrantType, + ) -> trc::Result> { + match self + .authenticate_token_client(client_id, credentials, ctx) + .await? + { + Ok(client) if !client.allows_grant(grant) => Ok(Err(ErrorType::UnauthorizedClient)), + result => Ok(result), + } + } + + async fn authorization_code_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + issuer: &str, + ) -> trc::Result { + let (Some(code), Some(client_id), Some(redirect_uri)) = ( + params.get("code"), + credentials.client_id.as_deref(), + params.get("redirect_uri").filter(|uri| !uri.is_empty()), + ) else { + return Ok(TokenResponse::error(ErrorType::InvalidRequest)); + }; + + let Some(auth_code_) = self + .in_memory_store() + .key_get::>(KeyValue::<()>::build_key(KV_OAUTH, code.as_bytes())) + .await? + else { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + }; + let oauth = auth_code_ + .unarchive::() + .caused_by(trc::location!())?; + if oauth.status != OAuthStatus::Authorized + || client_id != oauth.client_id + || redirect_uri != oauth.params + { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + let client = match self + .resolve_token_client( + client_id, + credentials, + ctx, + OAuthGrantType::AuthorizationCode, + ) + .await? + { + Ok(client) => client, + Err(error) => return Ok(TokenResponse::error(error)), + }; + + if !verify_pkce( + &oauth.code_challenge, + params.get("code_verifier"), + client.is_confidential(), + ) || !self.consume_oauth_code(code).await? + { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + self.issue_new_grant(NewGrantRequest { + account_id: oauth.account_id.into(), + credential_version: oauth.credential_version.into(), + client_id: oauth.client_id.as_str(), + client: &client, + scope: oauth.scope.as_ref().map(|s| s.as_str()), + nonce: oauth.nonce.as_ref().map(|s| s.as_str()), + issuer, + remote_ip: ctx.remote_ip, + }) + .await + } + + async fn device_code_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + issuer: &str, + ) -> trc::Result { + let (Some(device_code), Some(client_id)) = + (params.get("device_code"), credentials.client_id.as_deref()) + else { + return Ok(TokenResponse::error(ErrorType::InvalidRequest)); + }; + + let Some(auth_code_) = self + .in_memory_store() + .key_get::>(KeyValue::<()>::build_key( + KV_OAUTH, + device_code.as_bytes(), + )) + .await? + else { + return Ok(TokenResponse::error(ErrorType::ExpiredToken)); + }; + let oauth = auth_code_ + .unarchive::() + .caused_by(trc::location!())?; + if oauth.client_id != client_id { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + let client = match self + .resolve_token_client(client_id, credentials, ctx, OAuthGrantType::DeviceCode) + .await? + { + Ok(client) => client, + Err(error) => return Ok(TokenResponse::error(error)), + }; + + match oauth.status { + ArchivedOAuthStatus::Authorized => { + if oauth.params.is_empty() && self.consume_oauth_code(device_code).await? { + self.issue_new_grant(NewGrantRequest { + account_id: oauth.account_id.into(), + credential_version: oauth.credential_version.into(), + client_id: oauth.client_id.as_str(), + client: &client, + scope: oauth.scope.as_ref().map(|s| s.as_str()), + nonce: oauth.nonce.as_ref().map(|s| s.as_str()), + issuer, + remote_ip: ctx.remote_ip, + }) + .await + } else { + Ok(TokenResponse::error(ErrorType::InvalidGrant)) + } + } + ArchivedOAuthStatus::Pending => { + Ok(TokenResponse::error(ErrorType::AuthorizationPending)) + } + ArchivedOAuthStatus::Denied => Ok(TokenResponse::error(ErrorType::AccessDenied)), + ArchivedOAuthStatus::TokenIssued => Ok(TokenResponse::error(ErrorType::ExpiredToken)), + } + } + + async fn refresh_token_grant( + &self, + params: &FormData, + credentials: &ClientCredentials<'_>, + ctx: &ResolveContext<'_>, + session_id: u64, + ) -> trc::Result { + let (Some(refresh_token), Some(client_id)) = ( + params.get("refresh_token"), + credentials.client_id.as_deref(), + ) else { + return Ok(TokenResponse::error(ErrorType::InvalidRequest)); + }; + + let token_info = match self + .validate_access_token(GrantType::RefreshToken.into(), refresh_token) + .await + { + Ok(token_info) => token_info, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to validate refresh token") + .span_id(session_id) + ); + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + }; + + let Some(presented) = token_info.grant else { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + }; + let Some(stored) = self + .oauth_grant(presented.id) + .await? + .filter(|stored| stored.account_id() == token_info.account_id) + else { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + }; + if stored.grant.client_id != client_id { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + let client = match self + .resolve_token_client(client_id, credentials, ctx, OAuthGrantType::RefreshToken) + .await? + { + Ok(client) => client, + Err(error) => return Ok(TokenResponse::error(error)), + }; + if client.kind() != stored.grant.client_kind + || (!matches!(client, ResolvedClient::Cimd(None)) + && client.allowed_scopes().is_some_and(|allowed| { + !allowed.contains(OAuthScopes::from_registry(stored.grant.scopes.iter())) + })) + { + let mut invalidator = CacheInvalidationBuilder::default(); + self.delete_oauth_grant(stored, GrantDeletion::Unconditional, &mut invalidator) + .await?; + self.invalidate_caches(invalidator).await?; + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + let granted_scope = token_info.claims.as_deref().unwrap_or_default(); + if params + .get("scope") + .is_some_and(|requested| !is_scope_subset(requested, granted_scope)) + { + return Ok(TokenResponse::error(ErrorType::InvalidScope)); + } + + if token_info.credential_expired + && !client.is_first_party() + && stored.is_current(presented.generation) + { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + + match self + .rotate_oauth_grant(stored, presented.generation) + .await? + { + GrantRotation::Issue { next, rotated_at } => { + let account = self + .account(token_info.account_id) + .await + .caused_by(trc::location!())?; + self.grant_response(GrantResponse { + account_id: token_info.account_id, + account_name: account.name(), + credential_version: token_info.credential_version, + refresh: next, + rotated_at, + scope: granted_scope, + id_token: None, + }) + .map(TokenResponse::Granted) + } + GrantRotation::Revoked | GrantRotation::Rejected => { + Ok(TokenResponse::error(ErrorType::InvalidGrant)) + } + } + } + + async fn issue_new_grant(&self, request: NewGrantRequest<'_>) -> trc::Result { + let account_id = request.account_id; + let scope = request.scope.unwrap_or_default(); + let scopes = OAuthScopes::parse(scope); + let access_token = self .access_token(account_id) .await - .caused_by(trc::location!())? - .credential_version(); + .caused_by(trc::location!())?; + let credential_version = access_token.credential_version(); + if credential_version != request.credential_version + || (!request.client.is_first_party() && access_token.is_main_credential_expired()) + { + return Ok(TokenResponse::error(ErrorType::InvalidGrant)); + } + let grant_id = match self + .create_oauth_grant(NewOAuthGrant { + account_id, + client_id: request.client_id, + client_name: request.client.display_name(), + client_kind: request.client.kind(), + scopes: scopes.interoperable(), + remote_ip: request.remote_ip, + }) + .await + { + Ok(grant_id) => grant_id, + Err(_) if account_id == RECOVERY_ADMIN_ID => RECOVERY_ADMIN_GRANT_ID, + Err(err) => return Err(err), + }; + let account = self.account(account_id).await.caused_by(trc::location!())?; - let account_name = account.name(); + let id_token = if scopes.contains(OAuthScopes::OPENID) { + match self.issue_id_token( + account_id.to_string(), + request.issuer, + request.client_id, + StandardClaims { + nonce: request.nonce.map(str::to_string), + preferred_username: account.name().to_string().into(), + email: account.name().to_string().into(), + email_verified: true, + name: account.description().map(|d| d.to_string()), + }, + ) { + Ok(id_token) => Some(id_token), + Err(err) => { + trc::error!(err); + None + } + } + } else { + None + }; + + self.grant_response(GrantResponse { + account_id, + account_name: account.name(), + credential_version, + refresh: TokenGrant { + id: grant_id, + generation: 0, + }, + rotated_at: now(), + scope, + id_token, + }) + .map(|mut response| { + if grant_id == RECOVERY_ADMIN_GRANT_ID { + response.refresh_token = None; + } + TokenResponse::Granted(response) + }) + } + + fn grant_response(&self, response: GrantResponse<'_>) -> trc::Result { + let account_name = response.account_name; Ok(OAuthResponse { - access_token: self - .encode_access_token( - GrantType::AccessToken, - account_id, - account_name, - self.core.oauth.oauth_expiry_token, - None, - credential_version.into(), - ) - .await?, + access_token: self.encode_grant_access_token( + response.account_id, + account_name, + response.refresh.id, + response.credential_version, + )?, token_type: "bearer".to_string(), expires_in: self.core.oauth.oauth_expiry_token, - refresh_token: if with_refresh_token { - self.encode_access_token( - GrantType::RefreshToken, - account_id, + refresh_token: self + .encode_refresh_token(RefreshTokenParams { + account_id: response.account_id, account_name, - self.core.oauth.oauth_expiry_refresh_token, - None, - credential_version.into(), - ) - .await? - .into() - } else { - None - }, - id_token: if with_id_token { - match self.issue_id_token( - account_id.to_string(), - issuer, - client_id, - StandardClaims { - nonce, - preferred_username: account.name().to_string().into(), - email: account.name().to_string().into(), - description: account.description().map(|d| d.to_string()), - }, - ) { - Ok(id_token) => Some(id_token), - Err(err) => { - trc::error!(err); - None - } - } - } else { - None - }, - scope, + grant: response.refresh, + rotated_at: response.rotated_at, + credential_version: response.credential_version, + scope: Some(response.scope), + })? + .into(), + scope: response.scope.to_string(), + username: account_name.to_string(), + id_token: response.id_token, }) } } -fn client_credentials<'x>( - req: &'x HttpRequest, - params: &'x FormData, -) -> (Option>, Option>) { - let mut client_id = params.get("client_id").map(Cow::Borrowed); - let mut client_secret = params.get("client_secret").map(Cow::Borrowed); - - if (client_id.is_none() || client_secret.is_none()) - && let Some((id, secret)) = req - .authorization_basic() - .and_then(|token| STANDARD.decode(token).ok()) - .and_then(|bytes| String::from_utf8(bytes).ok()) - .and_then(|creds| { - creds - .split_once(':') - .map(|(id, secret)| (id.to_string(), secret.to_string())) - }) - { - if client_id.is_none() { - client_id = Some(Cow::Owned(id)); +fn is_scope_subset(requested: &str, granted: &str) -> bool { + let granted_scopes = OAuthScopes::parse(granted); + requested.split_ascii_whitespace().all(|name| { + let scope = OAuthScopes::from_name(name); + if scope.is_empty() { + granted + .split_ascii_whitespace() + .any(|granted| granted == name) + } else { + granted_scopes.contains(scope) } - if client_secret.is_none() { - client_secret = Some(Cow::Owned(secret)); - } - } - - (client_id, client_secret) + }) } -fn verify_pkce(stored: &ArchivedPkceCodeChallenge, verifier: Option<&str>) -> bool { +pub struct NewGrantRequest<'x> { + account_id: u32, + credential_version: u64, + client_id: &'x str, + client: &'x ResolvedClient, + scope: Option<&'x str>, + nonce: Option<&'x str>, + issuer: &'x str, + remote_ip: IpAddr, +} + +pub struct GrantResponse<'x> { + account_id: u32, + account_name: &'x str, + credential_version: u64, + refresh: TokenGrant, + rotated_at: u64, + scope: &'x str, + id_token: Option, +} + +fn revocation_response() -> HttpResponse { + HttpResponse::new(StatusCode::OK).with_no_store() +} + +pub(crate) fn token_response(response: TokenResponse, used_basic: bool) -> HttpResponse { + let status = match &response { + TokenResponse::Granted(_) => StatusCode::OK, + TokenResponse::Error { + error: ErrorType::InvalidClient, + } => StatusCode::UNAUTHORIZED, + TokenResponse::Error { .. } => StatusCode::BAD_REQUEST, + }; + let response = JsonResponse::with_status(status, response) + .no_cache() + .into_http_response(); + if status == StatusCode::UNAUTHORIZED && used_basic { + response.with_header(WWW_AUTHENTICATE, BASIC_CHALLENGE) + } else { + response + } +} + +pub struct ClientCredentials<'x> { + pub client_id: Option>, + pub client_secret: Option>, + pub used_basic: bool, + pub is_ambiguous: bool, +} + +impl<'x> ClientCredentials<'x> { + pub fn new(req: &'x HttpRequest, params: &'x FormData) -> Self { + Self::parse( + params.get("client_id"), + params.get("client_secret"), + req.authorization_basic(), + ) + } + + fn parse(body_id: Option<&'x str>, body_secret: Option<&'x str>, basic: Option<&str>) -> Self { + let mut credentials = ClientCredentials { + client_id: body_id.filter(|id| !id.is_empty()).map(Cow::Borrowed), + client_secret: body_secret + .filter(|secret| !secret.is_empty()) + .map(Cow::Borrowed), + used_basic: basic.is_some(), + is_ambiguous: false, + }; + let Some(basic) = basic else { + return credentials; + }; + credentials.is_ambiguous = credentials.client_secret.is_some(); + + if let Ok(creds) = STANDARD.decode_to_string(basic) + && let Some((id, secret)) = creds.split_once(':') + { + if !id.is_empty() { + let id = form_decode(id); + match &credentials.client_id { + Some(body_id) => credentials.is_ambiguous |= body_id.as_ref() != id, + None => credentials.client_id = Some(Cow::Owned(id)), + } + } + if credentials.client_secret.is_none() && !secret.is_empty() { + credentials.client_secret = Some(Cow::Owned(form_decode(secret))); + } + } + + credentials + } +} + +fn form_decode(value: &str) -> String { + let value = if value.contains('+') { + Cow::Owned(value.replace('+', " ")) + } else { + Cow::Borrowed(value) + }; + percent_decode_str(&value).decode_utf8_lossy().into_owned() +} + +const PKCE_S256_LEN: usize = URL_SAFE_NO_PAD.encoded_len(size_of::>()); + +fn verify_pkce( + stored: &ArchivedPkceCodeChallenge, + verifier: Option<&str>, + is_confidential: bool, +) -> bool { let is_valid_pkce_challenge = |challenge: &str| { (43..=128).contains(&challenge.len()) && challenge @@ -421,19 +747,17 @@ fn verify_pkce(stored: &ArchivedPkceCodeChallenge, verifier: Option<&str>) -> bo }; match (stored, verifier) { - (ArchivedPkceCodeChallenge::None, None) => true, - (ArchivedPkceCodeChallenge::Plain(expected), Some(verifier)) - if is_valid_pkce_challenge(verifier) => - { - constant_time_eq(expected.as_bytes(), verifier.as_bytes()) - } + (ArchivedPkceCodeChallenge::None, None) => is_confidential, (ArchivedPkceCodeChallenge::S256(expected), Some(verifier)) if is_valid_pkce_challenge(verifier) => { let digest = Sha256::digest(verifier.as_bytes()); - let computed = URL_SAFE_NO_PAD.encode(digest); - constant_time_eq(expected.as_bytes(), computed.as_bytes()) + let mut computed = [0u8; PKCE_S256_LEN]; + URL_SAFE_NO_PAD + .encode_str(digest, &mut computed) + .is_ok_and(|computed| constant_time_eq(expected.as_bytes(), computed.as_bytes())) } _ => false, } } + diff --git a/crates/http/src/auth/permissions.rs b/crates/http/src/auth/permissions.rs index fd86613e..10fa445a 100644 --- a/crates/http/src/auth/permissions.rs +++ b/crates/http/src/auth/permissions.rs @@ -48,77 +48,56 @@ impl AccountApiHandler for Server { false }; let is_recovery_admin = access_token.account_id() == RECOVERY_ADMIN_ID; - let permissions = if !self.registry().is_bootstrap_mode() - && let Some(scope) = access_token.access_scope() - { - let mut permissions = scope.permissions.clone(); + let permissions = + if !self.registry().is_bootstrap_mode() && access_token.access_scope().is_some() { + let mut permissions = access_token.effective_permissions().into_owned(); - for p in [ - Permission::SysDmarcInternalReportUpdate, - Permission::SysDmarcInternalReportCreate, - Permission::SysDmarcExternalReportUpdate, - Permission::SysDmarcExternalReportCreate, - Permission::SysTlsInternalReportUpdate, - Permission::SysTlsInternalReportCreate, - Permission::SysTlsExternalReportUpdate, - Permission::SysTlsExternalReportCreate, - Permission::SysArfExternalReportCreate, - Permission::SysArfExternalReportUpdate, - Permission::SysQueuedMessageCreate, - Permission::SysLogCreate, - Permission::SysLogDestroy, - Permission::SysLogUpdate, - Permission::SysClusterNodeCreate, - Permission::SysClusterNodeUpdate, - Permission::SysClusterNodeDestroy, - Permission::SysBootstrapGet, - Permission::SysBootstrapUpdate, - ] { - permissions.clear(p.to_id() as usize); - } - - if !self.core.groupware.allow_directory_query { - for p in [ - Permission::JmapPrincipalQuery, - Permission::JmapPrincipalQueryChanges, - Permission::JmapPrincipalGet, - Permission::JmapPrincipalGetAvailability, - Permission::JmapPrincipalChanges, - ] { + for p in [Permission::SysBootstrapGet, Permission::SysBootstrapUpdate] { permissions.clear(p.to_id() as usize); } - } - if is_external_directory || is_recovery_admin { - permissions.clear(Permission::SysAccountPasswordGet.to_id() as usize); - permissions.clear(Permission::SysAccountPasswordUpdate.to_id() as usize); - } - - if is_recovery_admin { - for p in [ - Permission::SysAccountSettingsGet, - Permission::SysAccountSettingsUpdate, - Permission::SysApiKeyCreate, - Permission::SysApiKeyUpdate, - Permission::SysApiKeyDestroy, - Permission::SysApiKeyQuery, - Permission::SysApiKeyGet, - Permission::SysAppPasswordCreate, - Permission::SysAppPasswordUpdate, - Permission::SysAppPasswordDestroy, - Permission::SysAppPasswordQuery, - Permission::SysAppPasswordGet, - ] { - permissions.clear(p.to_id() as usize); + if !self.core.groupware.allow_directory_query + && !access_token.has_permission(Permission::SysAccountQuery) + { + for p in [ + Permission::JmapPrincipalQuery, + Permission::JmapPrincipalQueryChanges, + Permission::JmapPrincipalChanges, + ] { + permissions.clear(p.to_id() as usize); + } } - } - permissions.build_permissions_list() - } else if self.registry().is_bootstrap_mode() { - vec![Permission::SysBootstrapGet, Permission::SysBootstrapUpdate] - } else { - Vec::new() - }; + if is_external_directory || is_recovery_admin { + permissions.clear(Permission::SysAccountPasswordGet.to_id() as usize); + permissions.clear(Permission::SysAccountPasswordUpdate.to_id() as usize); + } + + if is_recovery_admin { + for p in [ + Permission::SysAccountSettingsGet, + Permission::SysAccountSettingsUpdate, + Permission::SysApiKeyCreate, + Permission::SysApiKeyUpdate, + Permission::SysApiKeyDestroy, + Permission::SysApiKeyQuery, + Permission::SysApiKeyGet, + Permission::SysAppPasswordCreate, + Permission::SysAppPasswordUpdate, + Permission::SysAppPasswordDestroy, + Permission::SysAppPasswordQuery, + Permission::SysAppPasswordGet, + ] { + permissions.clear(p.to_id() as usize); + } + } + + permissions.build_permissions_list() + } else if self.registry().is_bootstrap_mode() { + vec![Permission::SysBootstrapGet, Permission::SysBootstrapUpdate] + } else { + Vec::new() + }; Ok(JsonResponse::new(Account { permissions, diff --git a/crates/http/src/form/mod.rs b/crates/http/src/form/mod.rs index d727dfe9..e9abf637 100644 --- a/crates/http/src/form/mod.rs +++ b/crates/http/src/form/mod.rs @@ -5,22 +5,24 @@ */ use crate::auth::oauth::FormData; -use chrono::Utc; use common::{ KV_RATE_LIMIT_CONTACT, Server, config::network::{ContactForm, FieldOrDefault}, network::ip_to_bytes, psl, }; -use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery}; +use email::message::delivery::{ + IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery, SenderAuthentication, +}; use http_proto::*; use hyper::StatusCode; -use mail_auth::common::cache::NoCache; +use mail_auth::NoCache; use mail_builder::{ MessageBuilder, headers::{ HeaderType, address::{Address, EmailAddress}, + date::Date, }, mime::make_boundary, }; @@ -28,6 +30,7 @@ use serde_json::json; use std::{borrow::Cow, fmt::Write, future::Future}; use store::write::BatchBuilder; use trc::AddContext; +use types::blob_hash::BlobHash; pub trait FormHandler: Sync + Send { fn handle_contact_form( @@ -54,7 +57,6 @@ impl FormHandler for Server { KV_RATE_LIMIT_CONTACT, &ip_to_bytes(&session.remote_ip), rate, - false, ) .await .caused_by(trc::location!())? @@ -101,7 +103,7 @@ impl FormHandler for Server { .smtp .resolvers .dns - .mx_lookup(domain, None::<&NoCache<_, _>>) + .mx_lookup(domain, None::<&NoCache>) .await .is_err() { @@ -128,11 +130,7 @@ impl FormHandler for Server { body.push_str("\r\n"); } } - let _ = write!( - &mut body, - "Date: {}\r\n", - Utc::now().format("%a, %d %b %Y %T %z") - ); + let _ = write!(&mut body, "Date: {}\r\n", Date::now().to_rfc822()); let _ = write!( &mut body, "IP: {}:{}\r\n", @@ -169,14 +167,14 @@ impl FormHandler for Server { // Reserve and write blob let (message_blob, blob_hold) = self - .put_temporary_blob(u32::MAX, &message, 60) + .hold_blob(u32::MAX, BlobHash::generate(&message), &message, 60) .await .caused_by(trc::location!())?; for result in self .deliver_message(IngestMessage { sender_address: from_email, - sender_authenticated: false, + sender_authentication: SenderAuthentication::Unauthenticated, recipients: form .rcpt_to .iter() @@ -206,9 +204,9 @@ impl FormHandler for Server { // Remove blob hold let mut batch = BatchBuilder::new(); - batch.clear(blob_hold); + blob_hold.release(&mut batch); self.store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; diff --git a/crates/http/src/lib.rs b/crates/http/src/lib.rs index f5adc5b6..2d37a3b8 100644 --- a/crates/http/src/lib.rs +++ b/crates/http/src/lib.rs @@ -10,6 +10,7 @@ pub mod api; pub mod auth; pub mod form; pub mod request; +mod stream; use common::Inner; use std::sync::Arc; diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index 946f8754..146a939b 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -8,20 +8,34 @@ use crate::{ HttpSessionManager, api::{AuthChallenge, ManagementApi, ToManageHttpResponse}, auth::{ - authenticate::{Authenticator, HttpHeaders}, + authenticate::{Authenticator, HttpAuthenticated, HttpHeaders}, oauth::{ - FormData, auth::OAuthApiHandler, openid::OpenIdHandler, - registration::ClientRegistrationHandler, token::TokenHandler, + FormData, + auth::OAuthApiHandler, + authorize::{PROTECTED_RESOURCE_PATH, resource_metadata_url}, + logout::EndSessionHandler, + openid::{OidcMetadataRequest, OpenIdHandler}, + registration::ClientRegistrationHandler, + request_origin, + token::TokenHandler, }, }, form::FormHandler, + stream::InFlightStream, }; use common::{ BuildServer, Inner, KV_ACME, Server, + auth::oauth::{client::RequestOrigin, scope::OAuthScopes}, ipc::PushEvent, - manager::application::Resource, - network::{SessionData, SessionManager, SessionStream}, + manager::{ + app_client::APP_CLIENT_DOCUMENT, + application::{AppClientDocument, Resource}, + }, + network::{ + SessionData, SessionManager, SessionStream, forwarded::ClientIp, memory::MemoryClass, + }, }; +use compact_str::{CompactString, ToCompactString}; use dav::{DavMethod, request::DavRequestHandler}; use groupware::DavResourceName; use http_proto::{ @@ -30,15 +44,19 @@ use http_proto::{ }; use hyper::{ Method, StatusCode, body, - header::{self, CONTENT_ENCODING, CONTENT_TYPE}, + header::{ + self, CONTENT_ENCODING, CONTENT_SECURITY_POLICY, CONTENT_TYPE, HOST, X_FRAME_OPTIONS, + }, server::conn::http1, service::service_fn, }; use hyper_util::rt::{TokioIo, TokioTimer}; use jmap::{ api::{ - ToJmapHttpResponse, event_source::EventSourceHandler, request::RequestHandler, - session::SessionHandler, + ToJmapHttpResponse, + event_source::EventSourceHandler, + request::RequestHandler, + session::{SessionHandler, allowed_capabilities}, }, blob::{download::BlobDownload, upload::BlobUpload}, websocket::upgrade::WebSocketUpgrade, @@ -46,7 +64,7 @@ use jmap::{ use jmap_proto::request::{Request, capability::Session}; use percent_encoding::percent_decode_str; use registry::schema::enums::Permission; -use std::{net::IpAddr, str::FromStr, sync::Arc}; +use std::{str::FromStr, sync::Arc}; use store::dispatch::lookup::KeyValue; use trc::SecurityEvent; use types::{blob::BlobId, id::Id}; @@ -95,8 +113,11 @@ impl ParseHttp for Server { match (path.next().unwrap_or_default(), req.method()) { ("", &Method::POST) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; if let Some(content_type) = req.headers().get(CONTENT_TYPE) { let is_json = content_type @@ -128,52 +149,66 @@ impl ParseHttp for Server { .await .ok_or_else(|| trc::LimitEvent::SizeRequest.into_err())?; + let mut request = Request::parse( + &bytes, + self.core.jmap.request_max_calls, + self.core.jmap.request_max_size, + )?; + request.using = allowed_capabilities(&access_token, request.using); + + let mut reservation = self.memory_reservation(MemoryClass::Read); return Ok(self - .handle_jmap_request( - Request::parse( - &bytes, - self.core.jmap.request_max_calls, - self.core.jmap.request_max_size, - )?, - &access_token, - &session, - ) + .handle_jmap_request(request, &access_token, &session, &mut reservation) .await .into_http_response()); } ("download", &Method::GET) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; + let access_token = + access_token.assert_allows_oauth_scopes(OAuthScopes::INTEROPERABLE)?; if let (Some(_), Some(blob_id), Some(name)) = ( path.next().and_then(|p| Id::from_str(p).ok()), path.next().and_then(BlobId::from_base32), path.next(), ) { - return match self.blob_download(&blob_id, &access_token).await? { - Some(blob) => Ok(DownloadResponse { - filename: name.to_string(), - content_type: req - .uri() - .query() - .and_then(|q| { - form_urlencoded::parse(q.as_bytes()) - .find(|(k, _)| k == "accept") - .map(|(_, v)| v.into_owned()) - }) - .unwrap_or("application/octet-stream".to_string()), - blob, + return match self.blob_contents(&blob_id, &access_token).await? { + Some(contents) => { + let (prefix, blob) = contents.into_http_parts()?; + Ok(DownloadResponse { + filename: name.to_string(), + content_type: req + .uri() + .query() + .and_then(|q| { + form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "accept") + .map(|(_, v)| v.into_owned()) + }) + .unwrap_or("application/octet-stream".to_string()), + prefix, + blob, + } + .into_http_response()) } - .into_http_response()), None => Err(trc::ResourceEvent::NotFound.into_err()), }; } } ("upload", &Method::POST) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; + let access_token = + access_token.assert_allows_oauth_scopes(OAuthScopes::INTEROPERABLE)?; if let Some(account_id) = path.next().and_then(|p| Id::from_str(p).ok()) { return match fetch_body( @@ -205,25 +240,38 @@ impl ParseHttp for Server { } ("eventsource", &Method::GET) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; + let access_token = + access_token.assert_allows_oauth_scopes(OAuthScopes::INTEROPERABLE)?; return self.handle_event_source(req, access_token).await; } ("ws", &Method::GET) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + epoch, + } = self.authenticate_headers(&req, &session).await?; + let access_token = + access_token.assert_allows_oauth_scopes(OAuthScopes::INTEROPERABLE)?; return self - .upgrade_websocket_connection(req, access_token, session) + .upgrade_websocket_connection(req, access_token, epoch, session) .await; } ("session", &Method::GET) => { return if req.headers().contains_key(header::AUTHORIZATION) { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; self.handle_session_resource( self.core.network.http.url_https.to_string(), @@ -267,8 +315,11 @@ impl ParseHttp for Server { ), (Some(resource), Some(method)) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; self.handle_dav_request(req, access_token, &session, resource, method) .await @@ -300,21 +351,36 @@ impl ParseHttp for Server { self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - return self.handle_oauth_metadata().await; + return self + .handle_oauth_metadata(request_origin(&req, &session)) + .await; } ("oauth-protected-resource", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - return self.handle_oauth_protected_resource().await; + return self + .handle_oauth_protected_resource( + request_origin(&req, &session), + req.uri() + .path() + .strip_prefix(PROTECTED_RESOURCE_PATH) + .unwrap_or_default(), + ) + .await; } ("openid-configuration", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - return self.handle_oidc_metadata(false).await; + return self + .handle_oidc_metadata( + OidcMetadataRequest::WellKnown, + request_origin(&req, &session), + ) + .await; } ("acme-challenge", &Method::GET) if self.has_acme_http_providers() => { if let Some(token) = path.next() { @@ -335,8 +401,7 @@ impl ParseHttp for Server { .await?; return if let Some(policy) = &self.core.smtp.session.mta_sts_policy { - Ok(Resource::new("text/plain", policy.to_string().into_bytes()) - .into_http_response()) + Ok(Resource::new("text/plain", policy.text.clone()).into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) }; @@ -351,13 +416,14 @@ impl ParseHttp for Server { req.headers() .get(header::HOST) .and_then(|h| h.to_str().ok()) - .map(|h| h.rsplit_once(':').map_or(h, |(h, _)| h)) + .map(pacc_domain) .unwrap_or_default(), ) .await? .into_bytes(), ) .into_http_response() + .with_sandbox_policy() .with_cors_unrestricted()); } ("mail-v1.xml", &Method::GET) => { @@ -368,7 +434,7 @@ impl ParseHttp for Server { return self .handle_autoconfig_request(req.uri().query()) .await - .map(|resource| resource.into_http_response()); + .map(|resource| resource.into_http_response().with_sandbox_policy()); } ("autoconfig", &Method::GET) if path.next().unwrap_or_default() == "mail" @@ -378,10 +444,14 @@ impl ParseHttp for Server { self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - return self - .handle_autoconfig_request(req.uri().query()) - .await - .map(|resource| resource.into_http_response().with_cors_unrestricted()); + return self.handle_autoconfig_request(req.uri().query()).await.map( + |resource| { + resource + .into_http_response() + .with_sandbox_policy() + .with_cors_unrestricted() + }, + ); } (_, &Method::OPTIONS) => { return Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_cors_unrestricted()); @@ -401,19 +471,27 @@ impl ParseHttp for Server { return self.handle_token_request(&mut req, session).await; } + ("revoke", &Method::POST) => { + self.is_http_anonymous_request_allowed(session.remote_ip) + .await?; + + return self.handle_token_revocation(&mut req, session).await; + } ("introspect", &Method::POST) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let _auth = self.authenticate_headers(&req, &session).await?; return self - .handle_token_introspect(&mut req, &access_token, session.session_id) + .handle_token_introspect(&mut req, session.session_id) .await; } ("userinfo", &Method::GET) => { // Authenticate request - let (_in_flight, access_token) = - self.authenticate_headers(&req, &session).await?; + let HttpAuthenticated { + in_flight: _in_flight, + access_token, + .. + } = self.authenticate_headers(&req, &session).await?; return self .handle_userinfo_request(access_token.account_id()) @@ -424,6 +502,12 @@ impl ParseHttp for Server { .handle_oauth_registration_request(&mut req, session) .await; } + ("logout", &Method::GET | &Method::POST) => { + self.is_http_anonymous_request_allowed(session.remote_ip) + .await?; + + return self.handle_end_session(&mut req, &session).await; + } ("jwks.json", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) @@ -445,7 +529,13 @@ impl ParseHttp for Server { return Ok(match self.handle_api_request(&mut req, &session).await { Ok(response) => response, Err(err) => { - let response = err.into_http_response(AuthChallenge::Bearer); + let response = err.into_http_response(AuthChallenge::Bearer, || { + resource_metadata_url( + self, + request_origin(&req, &session), + req.uri().path(), + ) + }); trc::error!(err.span_id(session.session_id)); response } @@ -462,7 +552,7 @@ impl ParseHttp for Server { return self .handle_autoconfig_request(req.uri().query()) .await - .map(|resource| resource.into_http_response()); + .map(|resource| resource.into_http_response().with_sandbox_policy()); } } "calendar" => { @@ -496,7 +586,7 @@ impl ParseHttp for Server { fetch_body(&mut req, 8192, session.session_id).await, ) .await - .map(|resource| resource.into_http_response()); + .map(|resource| resource.into_http_response().with_sandbox_policy()); } else if document_name.eq_ignore_ascii_case("autodiscover.json") { // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) @@ -509,11 +599,14 @@ impl ParseHttp for Server { return self .handle_autodiscover_v2_request(req.uri().query(), path_email.as_deref()) .await - .map(|result| match result { - Ok(resource) => resource.into_http_response(), - Err(err) => HttpResponse::new(StatusCode::BAD_REQUEST) - .with_content_type("application/json; charset=utf-8") - .with_text_body(err), + .map(|result| { + match result { + Ok(resource) => resource.into_http_response(), + Err(err) => HttpResponse::new(StatusCode::BAD_REQUEST) + .with_content_type("application/json; charset=utf-8") + .with_text_body(err), + } + .with_sandbox_policy() }); } } @@ -552,11 +645,7 @@ impl ParseHttp for Server { "metrics" => match path.next().unwrap_or_default() { "prometheus" => { if let Some(prometheus) = &self.core.metrics.prometheus { - if let Some(auth) = &prometheus.auth - && req - .authorization_basic() - .is_none_or(|secret| secret != auth) - { + if !prometheus.auth.is_authorized(req.authorization_basic()) { return Err(trc::AuthEvent::Failed .into_err() .details("Invalid or missing credentials.") @@ -599,15 +688,39 @@ impl ParseHttp for Server { return Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/html; charset=utf-8") .with_header(CONTENT_ENCODING, "gzip") + .with_header(X_FRAME_OPTIONS, "DENY") + .with_header(CONTENT_SECURITY_POLICY, "frame-ancestors 'none'") .with_binary_body(LOGIN_PAGE)); } external => { if path.next().is_none() { if !external.is_empty() { + if !self.inner.data.applications.has_prefix(external) + && ban_scanned_path(self, req.uri().path(), &session).await? + { + return Err(trc::ResourceEvent::NotFound.into_err()); + } return Ok(HttpResponse::redirect(format!("/{external}/"))); } else if let Some(url) = &self.core.network.http.redirect_root { return Ok(HttpResponse::redirect(url.clone())); } + } else if req.uri().path().get(external.len() + 2..) == Some(APP_CLIENT_DOCUMENT) + && let Some(response) = match self + .application_client_document(external, request_origin(&req, &session)) + { + AppClientDocument::Document(document) => { + Some(Ok(Resource::new("application/json", document) + .into_http_response() + .with_no_cache() + .with_cors_unrestricted())) + } + AppClientDocument::Untrusted => { + Some(Err(trc::ResourceEvent::NotFound.into_err())) + } + AppClientDocument::NotShipped => None, + } + { + return response; } else if let Some(resource) = self .inner .data @@ -632,29 +745,37 @@ impl ParseHttp for Server { } // Block dangerous URLs - let path = req.uri().path(); - if self.is_http_banned_path(path, session.remote_ip).await? { - trc::event!( - Security(SecurityEvent::ScanBan), - SpanId = session.session_id, - RemoteIp = session.remote_ip, - Path = path.to_string(), - ); - } + ban_scanned_path(self, req.uri().path(), &session).await?; Err(trc::ResourceEvent::NotFound.into_err()) } } +async fn ban_scanned_path( + server: &Server, + path: &str, + session: &HttpSessionData, +) -> trc::Result { + let banned = server.is_http_banned_path(path, session.remote_ip).await?; + if banned { + trc::event!( + Security(SecurityEvent::ScanBan), + SpanId = session.session_id, + RemoteIp = session.remote_ip, + Path = CompactString::from(path), + ); + } + Ok(banned) +} + async fn handle_session(inner: Arc, session: SessionData) { - let _in_flight = session.in_flight; let is_tls = session.stream.is_tls(); if let Err(http_err) = http1::Builder::new() .timer(TokioTimer::new()) .keep_alive(true) .serve_connection( - TokioIo::new(session.stream), + TokioIo::new(InFlightStream::new(session.stream, session.in_flight)), service_fn(|req: hyper::Request| { let instance = session.instance.clone(); let inner = inner.clone(); @@ -663,65 +784,62 @@ async fn handle_session(inner: Arc, session: SessionDat let server = inner.build_server(); // Obtain remote IP - let remote_ip = if !server.core.network.http.use_forwarded { - trc::event!( - Http(trc::HttpEvent::RequestUrl), - SpanId = session.session_id, - Url = req.uri().to_string(), - ); - - session.remote_ip - } else if let Some(forwarded_for) = req - .headers() - .get(header::FORWARDED) - .and_then(|h| h.to_str().ok()) - .and_then(|h| { - let h = h.to_ascii_lowercase(); - h.split_once("for=").and_then(|(_, rest)| { - let mut start_ip = usize::MAX; - let mut end_ip = usize::MAX; - - for (pos, ch) in rest.char_indices() { - match ch { - '0'..='9' | 'a'..='f' | ':' | '.' => { - if start_ip == usize::MAX { - start_ip = pos; - } - end_ip = pos; - } - '"' | '[' | ' ' if start_ip == usize::MAX => {} - _ => { - break; - } - } - } - - rest.get(start_ip..=end_ip) - .and_then(|h| h.parse::().ok()) - }) - }) - .or_else(|| { - req.headers() - .get("X-Forwarded-For") - .and_then(|h| h.to_str().ok()) - .map(|h| h.split_once(',').map_or(h, |(ip, _)| ip).trim()) - .and_then(|h| h.parse::().ok()) - }) + let remote_ip = match server + .core + .network + .http + .remote_ip_source + .resolve(req.headers()) { - trc::event!( - Http(trc::HttpEvent::RequestUrl), - SpanId = session.session_id, - RemoteIp = forwarded_for, - Url = req.uri().to_string(), - ); + ClientIp::Peer => { + trc::event!( + Http(trc::HttpEvent::RequestUrl), + SpanId = session.session_id, + Url = req.uri().to_compact_string(), + ); - forwarded_for - } else { - trc::event!( - Http(trc::HttpEvent::XForwardedMissing), - SpanId = session.session_id, - ); - session.remote_ip + session.remote_ip + } + ClientIp::Forwarded(forwarded_for) => { + trc::event!( + Http(trc::HttpEvent::RequestUrl), + SpanId = session.session_id, + RemoteIp = forwarded_for, + Url = req.uri().to_compact_string(), + ); + + forwarded_for + } + ClientIp::Missing if session.remote_ip.is_loopback() => { + trc::event!( + Http(trc::HttpEvent::RequestUrl), + SpanId = session.session_id, + Url = req.uri().to_compact_string(), + ); + + session.remote_ip + } + ClientIp::Missing => { + trc::event!( + Http(trc::HttpEvent::XForwardedMissing), + SpanId = session.session_id, + ); + session.remote_ip + } + ClientIp::Invalid => { + trc::event!( + Http(trc::HttpEvent::XForwardedInvalid), + ListenerId = instance.id.clone(), + RemoteIp = session.remote_ip, + SpanId = session.session_id, + ); + + return Ok::<_, hyper::Error>( + JsonProblemResponse(StatusCode::BAD_REQUEST) + .into_http_response() + .build(), + ); + } }; // Check if the remote IP has been blocked @@ -741,23 +859,44 @@ async fn handle_session(inner: Arc, session: SessionDat } // Parse HTTP request - let response = match Box::pin(server.parse_http_request( - req, - HttpSessionData { - instance, - local_ip: session.local_ip, - local_port: session.local_port, - remote_ip, - remote_port: session.remote_port, - is_tls, - session_id: session.session_id, - }, - )) - .await + let challenge_uri = req.uri().clone(); + let challenge_host = req.headers().get(HOST).cloned(); + let response = match server + .parse_http_request( + req, + HttpSessionData { + instance, + local_ip: session.local_ip, + local_port: session.local_port, + remote_ip, + remote_port: session.remote_port, + is_tls, + session_id: session.session_id, + }, + ) + .await { Ok(response) => response, Err(err) => { - let response = err.into_http_response(AuthChallenge::BearerAndBasic); + let response = + err.into_http_response(AuthChallenge::BearerAndBasic, || { + resource_metadata_url( + &server, + RequestOrigin { + is_tls, + authority: challenge_host + .as_ref() + .and_then(|host| host.to_str().ok()) + .or_else(|| { + challenge_uri + .authority() + .map(|authority| authority.as_str()) + }) + .unwrap_or_default(), + }, + challenge_uri.path(), + ) + }); trc::error!(err.span_id(session.session_id)); response } @@ -769,7 +908,7 @@ async fn handle_session(inner: Arc, session: SessionDat Contents = match response.body() { HttpResponseBody::Text(value) => trc::Value::String(value.as_str().into()), - HttpResponseBody::Binary(_) => + HttpResponseBody::Binary(_) | HttpResponseBody::Prefixed(_) => trc::Value::String("[binary data]".into()), HttpResponseBody::Stream(_) => trc::Value::String("[stream]".into()), _ => trc::Value::None, @@ -800,14 +939,14 @@ async fn handle_session(inner: Arc, session: SessionDat { if http_err.is_parse() { let server = inner.build_server(); - if !server.core.network.http.use_forwarded { + if server.core.network.http.remote_ip_source.is_peer() { match server.is_scanner_fail2banned(session.remote_ip).await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = session.session_id, RemoteIp = session.remote_ip, - Reason = http_err.to_string(), + Reason = http_err.to_compact_string(), ); return; } @@ -825,7 +964,7 @@ async fn handle_session(inner: Arc, session: SessionDat trc::event!( Http(trc::HttpEvent::Error), SpanId = session.session_id, - Reason = http_err.to_string(), + Reason = http_err.to_compact_string(), ); } } @@ -843,42 +982,13 @@ impl SessionManager for HttpSessionManager { } } -#[cfg(test)] -mod tests { - use flate2::read::GzDecoder; - use std::io::Read; +const PACC_HOST_PREFIX: &str = "ua-auto-config."; - const PAGES: [(&str, &[u8], &str); 2] = [ - ( - "calendar-rsvp.html", - super::RSVP_PAGE, - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/calendar-rsvp.html.min" - )), - ), - ( - "login.html", - super::LOGIN_PAGE, - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/login.html.min" - )), - ), - ]; - - #[test] - fn gzipped_static_pages_are_in_sync() { - for (name, gzipped, minified) in PAGES { - let mut decoded = String::new(); - GzDecoder::new(gzipped) - .read_to_string(&mut decoded) - .unwrap_or_else(|err| panic!("{name}.min.gz failed to decompress: {err}")); - - assert_eq!( - decoded, minified, - "{name}.min.gz is stale, re-run resources/scripts/minify_html.sh --gzip" - ); - } - } +fn pacc_domain(host: &str) -> &str { + let host = host.rsplit_once(':').map_or(host, |(host, _)| host); + host.get(..PACC_HOST_PREFIX.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(PACC_HOST_PREFIX)) + .and_then(|_| host.get(PACC_HOST_PREFIX.len()..)) + .unwrap_or(host) } + diff --git a/crates/http/src/stream.rs b/crates/http/src/stream.rs new file mode 100644 index 00000000..e595c883 --- /dev/null +++ b/crates/http/src/stream.rs @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::network::limiter::InFlight; +use std::{ + io::{self, IoSlice}, + pin::Pin, + task::{Context, Poll}, +}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +pub(crate) struct InFlightStream { + stream: T, + _in_flight: InFlight, +} + +impl InFlightStream { + pub(crate) fn new(stream: T, in_flight: InFlight) -> Self { + Self { + stream, + _in_flight: in_flight, + } + } +} + +impl AsyncRead for InFlightStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for InFlightStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_write(cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.stream.is_write_vectored() + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_shutdown(cx) + } +} diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index 4f3678d2..ce9dee05 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -1,18 +1,20 @@ [package] name = "imap_proto" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] types = { path = "../types" } utils = { path = "../utils" } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } ahash = { version = "0.8" } -base64 = "0.23" -chrono = { version = "0.4"} +encodify = "1.0" +jiff = "0.2.38" trc = { path = "../trc" } -hashify = { version = "0.2" } +hashify = { version = "0.3" } compact_str = "0.10.0" +smallvec = { version = "1", features = ["union"] } +itoa = "1" [dev-dependencies] diff --git a/crates/imap-proto/src/lib.rs b/crates/imap-proto/src/lib.rs index a8d8b49b..97f86bbc 100644 --- a/crates/imap-proto/src/lib.rs +++ b/crates/imap-proto/src/lib.rs @@ -4,14 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use protocol::ObjectId; use protocol::capability::Capability; +use protocol::metadata::MetadataCode; use std::borrow::Cow; pub mod parser; pub mod protocol; pub mod receiver; -pub mod utf7; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Command { @@ -81,6 +82,9 @@ pub enum Command { // RFC 10022 UidBatches, + + GetMetadata, + SetMetadata, } impl Command { @@ -110,8 +114,72 @@ impl Command { | Command::Thread(false) ) } + + pub fn as_str(&self) -> &'static str { + let message = self.completed_message(); + &message[..message.len() - COMPLETED_SUFFIX.len()] + } + + pub fn completed_message(&self) -> &'static str { + match self { + Command::UidBatches => "UIDBATCHES completed", + Command::Capability => "CAPABILITY completed", + Command::Noop => "NOOP completed", + Command::Logout => "LOGOUT completed", + Command::StartTls => "STARTTLS completed", + Command::Authenticate => "AUTHENTICATE completed", + Command::Login => "LOGIN completed", + Command::Enable => "ENABLE completed", + Command::Select => "SELECT completed", + Command::Examine => "EXAMINE completed", + Command::Create => "CREATE completed", + Command::Delete => "DELETE completed", + Command::Rename => "RENAME completed", + Command::Subscribe => "SUBSCRIBE completed", + Command::Unsubscribe => "UNSUBSCRIBE completed", + Command::List => "LIST completed", + Command::Namespace => "NAMESPACE completed", + Command::Status => "STATUS completed", + Command::Append => "APPEND completed", + Command::Idle => "IDLE completed", + Command::Close => "CLOSE completed", + Command::Unselect => "UNSELECT completed", + Command::Expunge(false) => "EXPUNGE completed", + Command::Search(false) => "SEARCH completed", + Command::Fetch(false) => "FETCH completed", + Command::Store(false) => "STORE completed", + Command::Copy(false) => "COPY completed", + Command::Move(false) => "MOVE completed", + Command::Sort(false) => "SORT completed", + Command::Thread(false) => "THREAD completed", + Command::Expunge(true) => "UID EXPUNGE completed", + Command::Search(true) => "UID SEARCH completed", + Command::Fetch(true) => "UID FETCH completed", + Command::Store(true) => "UID STORE completed", + Command::Copy(true) => "UID COPY completed", + Command::Move(true) => "UID MOVE completed", + Command::Sort(true) => "UID SORT completed", + Command::Thread(true) => "UID THREAD completed", + Command::Lsub => "LSUB completed", + Command::Check => "CHECK completed", + Command::SetAcl => "SETACL completed", + Command::DeleteAcl => "DELETEACL completed", + Command::GetAcl => "GETACL completed", + Command::ListRights => "LISTRIGHTS completed", + Command::MyRights => "MYRIGHTS completed", + Command::Unauthenticate => "UNAUTHENTICATE completed", + Command::Id => "ID completed", + Command::GetQuota => "GETQUOTA completed", + Command::GetQuotaRoot => "GETQUOTAROOT completed", + Command::GetJmapAccess => "GETJMAPACCESS completed", + Command::GetMetadata => "GETMETADATA completed", + Command::SetMetadata => "SETMETADATA completed", + } + } } +const COMPLETED_SUFFIX: &str = " completed"; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseCode { Alert, @@ -159,7 +227,7 @@ pub enum ResponseCode { // CONDSTORE Modified { - ids: Vec, + ranges: Vec<(u32, u32)>, }, HighestModseq { modseq: u64, @@ -178,16 +246,21 @@ pub enum ResponseCode { TooFew, TooMany, + // LITERAL+, CATENATE + TooBig, + // MESSAGELIMIT MessageLimit { limit: u32, uid: Option, }, + + Metadata(MetadataCode), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusResponse { - pub tag: Option, + pub tag: Option, pub code: Option, pub message: Cow<'static, str>, pub rtype: ResponseType, @@ -234,7 +307,7 @@ impl StatusResponse { } pub fn completed(command: Command) -> Self { - StatusResponse::ok(format!("{} completed", command)) + StatusResponse::ok(command.completed_message()) } pub fn with_code(mut self, code: ResponseCode) -> Self { @@ -242,7 +315,7 @@ impl StatusResponse { self } - pub fn with_tag(mut self, tag: impl Into) -> Self { + pub fn with_tag(mut self, tag: impl Into) -> Self { self.tag = Some(tag.into()); self } diff --git a/crates/imap-proto/src/parser/acl.rs b/crates/imap-proto/src/parser/acl.rs index 3629c978..56954992 100644 --- a/crates/imap-proto/src/parser/acl.rs +++ b/crates/imap-proto/src/parser/acl.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::acl::{self, ModRights, ModRightsOp, Rights}, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; use super::PushUnique; @@ -39,20 +36,17 @@ impl Request { _ => unreachable!(), }; let mut tokens = self.tokens.into_iter(); - let mailbox_name = utf7_maybe_decode( - tokens - .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ); + let mailbox_name = tokens + .next() + .ok_or_else(|| bad(self.tag.clone(), "Missing mailbox name."))? + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?; let identifier = if has_identifier { tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing identifier."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing identifier."))? .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? .into() } else { None @@ -61,10 +55,10 @@ impl Request { ModRights::parse( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing rights."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing rights."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? .into() } else { None @@ -95,6 +89,7 @@ impl ModRights { b'x' => Rights::DeleteMailbox, b't' => Rights::DeleteMessages, b'e' => Rights::Expunge, + b'n' => Rights::Annotate, b'a' => Rights::Administer, // RFC2086 b'd' => Rights::DeleteMessages, @@ -123,100 +118,3 @@ impl ModRights { } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::acl::{self, ModRights, ModRightsOp, Rights}, - receiver::Receiver, - }; - - #[test] - fn parse_acl() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A003 Setacl INBOX/Drafts Byron lrswikda\r\n", - acl::Arguments { - tag: "A003".into(), - mailbox_name: "INBOX/Drafts".into(), - identifier: Some("Byron".into()), - mod_rights: ModRights { - op: ModRightsOp::Replace, - rights: vec![ - Rights::Lookup, - Rights::Read, - Rights::Seen, - Rights::Write, - Rights::Insert, - Rights::CreateMailbox, - Rights::DeleteMessages, - Rights::Administer, - ], - } - .into(), - }, - ), - ( - "A002 SETACL INBOX/Drafts Chris +cda\r\n", - acl::Arguments { - tag: "A002".into(), - mailbox_name: "INBOX/Drafts".into(), - identifier: Some("Chris".into()), - mod_rights: ModRights { - op: ModRightsOp::Add, - rights: vec![ - Rights::CreateMailbox, - Rights::DeleteMessages, - Rights::Administer, - ], - } - .into(), - }, - ), - ( - "A036 SETACL INBOX/Drafts John -lrswicda\r\n", - acl::Arguments { - tag: "A036".into(), - mailbox_name: "INBOX/Drafts".into(), - identifier: Some("John".into()), - mod_rights: ModRights { - op: ModRightsOp::Remove, - rights: vec![ - Rights::Lookup, - Rights::Read, - Rights::Seen, - Rights::Write, - Rights::Insert, - Rights::CreateMailbox, - Rights::DeleteMessages, - Rights::Administer, - ], - } - .into(), - }, - ), - ( - "A001 GETACL INBOX/Drafts\r\n", - acl::Arguments { - tag: "A001".into(), - mailbox_name: "INBOX/Drafts".into(), - identifier: None, - mod_rights: None, - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_acl(false) - .unwrap(), - arguments, - "{:?}", - command - ); - } - } -} diff --git a/crates/imap-proto/src/parser/append.rs b/crates/imap-proto/src/parser/append.rs index 7585ede0..c9f4ec2d 100644 --- a/crates/imap-proto/src/parser/append.rs +++ b/crates/imap-proto/src/parser/append.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - +use super::parse_datetime; use crate::{ Command, protocol::{ @@ -13,11 +12,8 @@ use crate::{ append::{self, Message}, }, receiver::{Request, Token, bad}, - utf7::utf7_maybe_decode, }; -use super::parse_datetime; - enum State { None, Flags, @@ -32,14 +28,11 @@ impl Request { _ => { // Obtain mailbox name let mut tokens = self.tokens.into_iter().peekable(); - let mailbox_name = utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ); + let mailbox_name = tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?; let mut messages = Vec::new(); while tokens.peek().is_some() { @@ -63,7 +56,7 @@ impl Request { State::UTF8 => State::UTF8Data, _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid opening parenthesis found.", )); } @@ -72,7 +65,7 @@ impl Request { Token::ParenthesisClose => match state { State::None | State::UTF8 => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid closing parenthesis found.", )); } @@ -95,43 +88,41 @@ impl Request { message.received_at = Some(date_time); } else { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Failed to parse received time.", )); } } else { - message.message = value; + message.message = value.into_vec(); break; } } State::Flags => { - message.flags.push( - Flag::parse_imap(value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); + let flag = Flag::parse_imap(value) + .map_err(|v| bad(self.tag.clone(), v))?; + if !message.flags.contains(&flag) { + message.flags.push(flag); + } } State::UTF8 => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected parenthesis after UTF8.", )); } State::UTF8Data => { if message.message.is_empty() { - message.message = value; + message.message = value.into_vec(); } else { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid parameter after message literal.", )); } } }, _ => { - return Err(bad( - self.tag.to_compact_string(), - "Invalid arguments.", - )); + return Err(bad(self.tag.clone(), "Invalid arguments.")); } } } @@ -149,203 +140,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::{ - Flag, - append::{self, Message}, - }, - receiver::{Error, Receiver}, - }; - - #[test] - fn parse_append() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A003 APPEND saved-messages (\\Seen) {1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "saved-messages".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![Flag::Seen], - received_at: None, - }], - }, - ), - ( - "A003 APPEND \"hello world\" (\\Seen \\Draft $MDNSent) {1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "hello world".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![Flag::Seen, Flag::Draft, Flag::MDNSent], - received_at: None, - }], - }, - ), - ( - "A003 APPEND \"hi\" ($Junk) \"7-Feb-1994 22:43:04 -0800\" {1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "hi".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![Flag::Junk], - received_at: Some(760689784), - }], - }, - ), - ( - "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" {1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "hi".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![], - received_at: Some(1668977999), - }], - }, - ), - ( - "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" ~{1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "hi".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![], - received_at: Some(1668977999), - }], - }, - ), - ( - "42 APPEND \"Drafts\" (\\Draft) UTF8 (~{5+}\r\nhello)\r\n", - append::Arguments { - tag: "42".into(), - mailbox_name: "Drafts".into(), - messages: vec![Message { - message: vec![b'h', b'e', b'l', b'l', b'o'], - flags: vec![Flag::Draft], - received_at: None, - }], - }, - ), - ( - "42 APPEND \"Drafts\" (\\Draft) \"20-Nov-2022 23:59:59 +0300\" UTF8 (~{5+}\r\nhello)\r\n", - append::Arguments { - tag: "42".into(), - mailbox_name: "Drafts".into(), - messages: vec![Message { - message: vec![b'h', b'e', b'l', b'l', b'o'], - flags: vec![Flag::Draft], - received_at: Some(1668977999), - }], - }, - ), - ( - "A003 APPEND \"&A8g- \\\"&A9QD1APUA9gD3APcA-+\\\"\" (\\Seen) \"7-Feb-1994 22:43:04 -0800\" {1+}\r\na\r\n", - append::Arguments { - tag: "A003".into(), - mailbox_name: "ψ \"ϔϔϔϘϜϜ+\"".into(), - messages: vec![Message { - message: vec![b'a'], - flags: vec![Flag::Seen], - received_at: Some(760689784), - }], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .expect(command) - .parse_append(false) - .expect(command), - arguments, - "{:?}", - command - ); - } - - // Multiappend - for line in [ - "A003 APPEND saved-messages (\\Seen) UTF8 ({329}\r\n", - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@owatagu.example.net\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n)", - " (\\Seen) \"7-Feb-1994 22:43:04 -0800\" {295}\r\n", - "Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST)\r\n", - "From: Joe Mooch \r\n", - "Subject: Re: afternoon meeting\r\n", - "To: foobar@blurdybloop.example.com\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n", - "3:30 is fine with me.\r\n\r\n", - ] { - match receiver.parse(&mut line.as_bytes().iter()) { - Ok(request) => { - assert_eq!( - request.parse_append(false).unwrap(), - append::Arguments { - tag: "A003".into(), - mailbox_name: "saved-messages".into(), - messages: vec![ - Message { - message: concat!( - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@owatagu.example.net\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n", - ) - .as_bytes() - .to_vec(), - flags: vec![Flag::Seen], - received_at: None, - }, - Message { - message: concat!( - "Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST)\r\n", - "From: Joe Mooch \r\n", - "Subject: Re: afternoon meeting\r\n", - "To: foobar@blurdybloop.example.com\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n", - "3:30 is fine with me.\r\n", - ) - .as_bytes() - .to_vec(), - flags: vec![Flag::Seen], - received_at: Some(760689784), - } - ], - }, - ); - } - Err(err) => match err { - Error::NeedsMoreData | Error::NeedsLiteral { .. } => (), - Error::Error { response } => panic!("{:?}", response), - }, - } - } - } -} diff --git a/crates/imap-proto/src/parser/authenticate.rs b/crates/imap-proto/src/parser/authenticate.rs index 82857578..41a7488a 100644 --- a/crates/imap-proto/src/parser/authenticate.rs +++ b/crates/imap-proto/src/parser/authenticate.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::authenticate::{self, Mechanism}, @@ -18,7 +16,7 @@ impl Request { let mut tokens = self.tokens.into_iter(); Ok(authenticate::Arguments { mechanism: Mechanism::parse(&tokens.next().unwrap().unwrap_bytes()) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, params: tokens .filter_map(|token| token.unwrap_string().ok()) .collect(), @@ -32,19 +30,20 @@ impl Request { impl Mechanism { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "PLAIN" => Self::Plain, - "CRAM-MD5" => Self::CramMd5, - "DIGEST-MD5" => Self::DigestMd5, - "SCRAM-SHA-1" => Self::ScramSha1, - "SCRAM-SHA-256" => Self::ScramSha256, - "APOP" => Self::Apop, - "NTLM" => Self::Ntlm, - "GSSAPI" => Self::Gssapi, - "ANONYMOUS" => Self::Anonymous, - "EXTERNAL" => Self::External, - "OAUTHBEARER" => Self::OAuthBearer, - "XOAUTH2" => Self::XOauth2, + hashify::fnc_map_ignore_case!(value, + "PLAIN" => Some(Self::Plain), + "CRAM-MD5" => Some(Self::CramMd5), + "DIGEST-MD5" => Some(Self::DigestMd5), + "SCRAM-SHA-1" => Some(Self::ScramSha1), + "SCRAM-SHA-256" => Some(Self::ScramSha256), + "APOP" => Some(Self::Apop), + "NTLM" => Some(Self::Ntlm), + "GSSAPI" => Some(Self::Gssapi), + "ANONYMOUS" => Some(Self::Anonymous), + "EXTERNAL" => Some(Self::External), + "OAUTHBEARER" => Some(Self::OAuthBearer), + "XOAUTH2" => Some(Self::XOauth2), + _ => None, ) .ok_or_else(|| { format!( @@ -56,43 +55,3 @@ impl Mechanism { } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::authenticate::{self, Mechanism}, - receiver::Receiver, - }; - - #[test] - fn parse_authenticate() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "a002 AUTHENTICATE \"EXTERNAL\" {16+}\r\nfred@example.com\r\n", - authenticate::Arguments { - tag: "a002".into(), - mechanism: Mechanism::External, - params: vec!["fred@example.com".into()], - }, - ), - ( - "A01 AUTHENTICATE PLAIN\r\n", - authenticate::Arguments { - tag: "A01".into(), - mechanism: Mechanism::Plain, - params: vec![], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_authenticate() - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/copy_move.rs b/crates/imap-proto/src/parser/copy_move.rs index e845bf36..2c5f633e 100644 --- a/crates/imap-proto/src/parser/copy_move.rs +++ b/crates/imap-proto/src/parser/copy_move.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::copy_move, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; use super::parse_sequence_set; @@ -24,18 +21,15 @@ impl Request { sequence_set: parse_sequence_set( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing sequence set."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - mailbox_name: utf7_maybe_decode( - tokens - .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + .map_err(|v| bad(self.tag.clone(), v))?, + mailbox_name: tokens + .next() + .ok_or_else(|| bad(self.tag.clone(), "Missing mailbox name."))? + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }) } else { @@ -44,46 +38,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::{Sequence, copy_move}, - receiver::Receiver, - }; - - #[test] - fn parse_copy() { - let mut receiver = Receiver::new(); - - assert_eq!( - receiver - .parse(&mut "A003 COPY 2:4 MEETING\r\n".as_bytes().iter()) - .unwrap() - .parse_copy_move(false) - .unwrap(), - copy_move::Arguments { - sequence_set: Sequence::Range { - start: 2.into(), - end: 4.into(), - }, - mailbox_name: "MEETING".into(), - tag: "A003".into(), - } - ); - assert_eq!( - receiver - .parse(&mut "A003 COPY 2:4 \"You &- Me\"\r\n".as_bytes().iter()) - .unwrap() - .parse_copy_move(false) - .unwrap(), - copy_move::Arguments { - sequence_set: Sequence::Range { - start: 2.into(), - end: 4.into(), - }, - mailbox_name: "You & Me".into(), - tag: "A003".into(), - } - ); - } -} diff --git a/crates/imap-proto/src/parser/create.rs b/crates/imap-proto/src/parser/create.rs index ee939470..b4b00261 100644 --- a/crates/imap-proto/src/parser/create.rs +++ b/crates/imap-proto/src/parser/create.rs @@ -4,85 +4,57 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString, format_compact}; +use compact_str::format_compact; use crate::{ Command, protocol::{create, list::Attribute}, receiver::{Request, Token, bad}, - utf7::utf7_maybe_decode, }; impl Request { pub fn parse_create(self, is_utf8: bool) -> trc::Result { if !self.tokens.is_empty() { let mut tokens = self.tokens.into_iter(); - let mailbox_name = utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ); + let mailbox_name = tokens + .next() + .unwrap() + .unwrap_mailbox_name_strict(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?; let mailbox_role = if let Some(Token::ParenthesisOpen) = tokens.next() { match tokens.next() { Some(Token::Argument(param)) if param.eq_ignore_ascii_case(b"USE") => (), _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Failed to parse, expected 'USE'.", - )); + return Err(bad(self.tag, "Failed to parse, expected 'USE'.")); } } if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected '(' after 'USE'.", - )); + return Err(bad(self.tag, "Expected '(' after 'USE'.")); } match tokens.next() { - Some(Token::Argument(value)) => { - let r = hashify::tiny_map_ignore_case!(value.as_slice(), - "\\Archive" => Some(Attribute::Archive), - "\\Drafts" => Some(Attribute::Drafts), - "\\Junk" => Some(Attribute::Junk), - "\\Sent" => Some(Attribute::Sent), - "\\Trash" => Some(Attribute::Trash), - "\\Important" => Some(Attribute::Important), - "\\Memos" => Some(Attribute::Memos), - "\\Scheduled" => Some(Attribute::Scheduled), - "\\Snoozed" => Some(Attribute::Snoozed), - "\\All" => None, - ); - - match r { - Some(Some(tag)) => Some(tag), - Some(None) => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "A mailbox with the \"\\All\" attribute already exists.", - )); - } - None => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - format_compact!( - "Special use attribute {:?} is not supported.", - String::from_utf8_lossy(&value) - ), - )); - } + Some(Token::Argument(value)) => match Attribute::parse_special_use(&value) { + Some(Attribute::All) => { + return Err(bad( + self.tag, + "A mailbox with the \"\\All\" attribute already exists.", + )); } - } + Some(tag) => Some(tag), + None => { + return Err(bad( + self.tag, + format_compact!( + "Special use attribute {:?} is not supported.", + String::from_utf8_lossy(&value) + ), + )); + } + }, _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Invalid SPECIAL-USE attribute.", - )); + return Err(bad(self.tag, "Invalid SPECIAL-USE attribute.")); } } } else { @@ -100,60 +72,21 @@ impl Request { } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::{create, list::Attribute}, - receiver::Receiver, - }; - - #[test] - fn parse_create() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A142 CREATE 12345\r\n", - create::Arguments { - tag: "A142".into(), - mailbox_name: "12345".into(), - mailbox_role: None, - }, - ), - ( - "A142 CREATE \"my funky mailbox\"\r\n", - create::Arguments { - tag: "A142".into(), - mailbox_name: "my funky mailbox".into(), - mailbox_role: None, - }, - ), - ( - "t1 CREATE \"Important Messages\" (USE (\\Important))\r\n", - create::Arguments { - tag: "t1".into(), - mailbox_name: "Important Messages".into(), - mailbox_role: Some(Attribute::Important), - }, - ), - ( - "A142 CREATE \"Test-ąęć-Test\"\r\n", - create::Arguments { - tag: "A142".into(), - mailbox_name: "Test-ąęć-Test".into(), - mailbox_role: None, - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_create(true) - .unwrap(), - arguments - ); - } +impl Attribute { + pub fn parse_special_use(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, Attribute, + "\\Archive" => Attribute::Archive, + "\\Drafts" => Attribute::Drafts, + "\\Junk" => Attribute::Junk, + "\\Sent" => Attribute::Sent, + "\\Trash" => Attribute::Trash, + "\\Important" => Attribute::Important, + "\\Memos" => Attribute::Memos, + "\\Scheduled" => Attribute::Scheduled, + "\\Snoozed" => Attribute::Snoozed, + "\\All" => Attribute::All, + ) + .copied() } } + diff --git a/crates/imap-proto/src/parser/delete.rs b/crates/imap-proto/src/parser/delete.rs index 58979c04..a40ec44c 100644 --- a/crates/imap-proto/src/parser/delete.rs +++ b/crates/imap-proto/src/parser/delete.rs @@ -4,28 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::delete, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; impl Request { pub fn parse_delete(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(delete::Arguments { - mailbox_name: utf7_maybe_decode( - self.tokens - .into_iter() - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + mailbox_name: self + .tokens + .into_iter() + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), @@ -34,38 +29,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::delete, receiver::Receiver}; - - #[test] - fn parse_delete() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A142 DELETE INBOX\r\n", - delete::Arguments { - mailbox_name: "INBOX".into(), - tag: "A142".into(), - }, - ), - ( - "A142 DELETE \"my funky mailbox\"\r\n", - delete::Arguments { - mailbox_name: "my funky mailbox".into(), - tag: "A142".into(), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_delete(true) - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/enable.rs b/crates/imap-proto/src/parser/enable.rs index ead86aba..aa132cf3 100644 --- a/crates/imap-proto/src/parser/enable.rs +++ b/crates/imap-proto/src/parser/enable.rs @@ -7,100 +7,43 @@ use crate::{ Command, protocol::{capability::Capability, enable}, - receiver::{Request, bad}, + receiver::{Request, Token, bad}, }; -use compact_str::ToCompactString; impl Request { pub fn parse_enable(self) -> trc::Result { - let len = self.tokens.len(); - if len > 0 { - let mut capabilities = Vec::with_capacity(len); - for capability in self.tokens { - capabilities.push( - Capability::parse(&capability.unwrap_bytes()) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); - } - Ok(enable::Arguments { - tag: self.tag, - capabilities, - }) - } else { - Err(self.into_error("Missing arguments.")) + if self.tokens.is_empty() { + return Err(self.into_error("Missing arguments.")); } - } -} -impl Capability { - pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "IMAP4rev2" => Self::IMAP4rev2, - "STARTTLS" => Self::StartTLS, - "LOGINDISABLED" => Self::LoginDisabled, - "CONDSTORE" => Self::CondStore, - "QRESYNC" => Self::QResync, - "UTF8=ACCEPT" => Self::Utf8Accept, - "OBJECTID+" => Self::ObjectIdPlus, - "UIDONLY" => Self::UidOnly, - ) - .ok_or_else(|| { - format!( - "Unsupported capability '{}'.", - String::from_utf8_lossy(value) - ) - .into() + let mut capabilities = Vec::with_capacity(self.tokens.len()); + for token in self.tokens { + match token { + Token::Argument(name) => capabilities.extend(Capability::parse(&name)), + _ => return Err(bad(self.tag, "Invalid capability name.")), + } + } + + Ok(enable::Arguments { + tag: self.tag, + capabilities, }) } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::{capability::Capability, enable}, - receiver::Receiver, - }; - - #[test] - fn parse_enable() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "t2 ENABLE IMAP4rev2 CONDSTORE\r\n", - enable::Arguments { - tag: "t2".into(), - capabilities: vec![Capability::IMAP4rev2, Capability::CondStore], - }, - ), - ( - "t3 ENABLE OBJECTID+\r\n", - enable::Arguments { - tag: "t3".into(), - capabilities: vec![Capability::ObjectIdPlus], - }, - ), - ( - "t4 ENABLE CONDSTORE OBJECTID+ UTF8=ACCEPT\r\n", - enable::Arguments { - tag: "t4".into(), - capabilities: vec![ - Capability::CondStore, - Capability::ObjectIdPlus, - Capability::Utf8Accept, - ], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_enable() - .unwrap(), - arguments, - "Failed to parse {command}" - ); - } +impl Capability { + pub fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "IMAP4rev2" => Some(Self::IMAP4rev2), + "STARTTLS" => Some(Self::StartTLS), + "LOGINDISABLED" => Some(Self::LoginDisabled), + "CONDSTORE" => Some(Self::CondStore), + "QRESYNC" => Some(Self::QResync), + "UTF8=ACCEPT" => Some(Self::Utf8Accept), + "OBJECTID+" => Some(Self::ObjectIdPlus), + "UIDONLY" => Some(Self::UidOnly), + _ => None, + ) } } + diff --git a/crates/imap-proto/src/parser/fetch.rs b/crates/imap-proto/src/parser/fetch.rs index d18986f9..ca909f59 100644 --- a/crates/imap-proto/src/parser/fetch.rs +++ b/crates/imap-proto/src/parser/fetch.rs @@ -10,11 +10,14 @@ use crate::{ protocol::fetch::{self, Attribute, Section}, receiver::{Request, Token, bad}, }; -use compact_str::{CompactString, ToCompactString, format_compact}; +use compact_str::format_compact; use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; +const ATTRIBUTES_INIT_LEN: usize = 8; +const FIELDS_INIT_LEN: usize = 32; + impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_fetch(self) -> trc::Result { @@ -23,14 +26,14 @@ impl Request { } let mut tokens = self.tokens.into_iter().peekable(); - let mut attributes = Vec::new(); + let mut attributes = Vec::with_capacity(tokens.len().min(ATTRIBUTES_INIT_LEN)); let sequence_set = parse_sequence_set( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing sequence set."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let mut in_parentheses = false; @@ -87,7 +90,7 @@ impl Request { let rfc822 = tokens .next() .ok_or_else(|| { - bad(self.tag.to_compact_string(), "Missing RFC822 parameter.") + bad(self.tag.clone(), "Missing RFC822 parameter.") })? .unwrap_bytes(); if rfc822.eq_ignore_ascii_case(b"HEADER") { @@ -98,7 +101,7 @@ impl Request { Attribute::Rfc822Text } else { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!( "Invalid RFC822 parameter {:?}.", String::from_utf8_lossy(&rfc822) @@ -123,13 +126,13 @@ impl Request { .is_none_or( |token| !token.eq_ignore_ascii_case(b"PEEK")) { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected 'PEEK' after '.'.", )); } if tokens.next().is_none_or( |token| !token.is_bracket_open()) { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected '[' after 'BODY.PEEK'", )); } @@ -146,100 +149,14 @@ impl Request { } }; - // Parse section-spect - let mut sections = Vec::new(); - while let Some(token) = tokens.next() { - match token { - Token::BracketClose => break, - Token::Argument(value) => { - let section = if value.eq_ignore_ascii_case(b"HEADER") { - if let Some(Token::Dot) = tokens.peek() { - tokens.next(); - if tokens.next().is_none_or( |token| { - !token.eq_ignore_ascii_case(b"FIELDS") - }) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected 'FIELDS' after 'HEADER.'.", - )); - } - let is_not = if let Some(Token::Dot) = tokens.peek() { - tokens.next(); - if tokens.next().is_none_or( |token| { - !token.eq_ignore_ascii_case(b"NOT") - }) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected 'NOT' after 'HEADER.FIELDS.'.", - )); - } - true - } else { - false - }; - if tokens - .next() - .is_none_or( |token| !token.is_parenthesis_open()) - { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected '(' after 'HEADER.FIELDS'.", - )); - } - let mut fields = Vec::new(); - while let Some(token) = tokens.next() { - match token { - Token::ParenthesisClose => break, - Token::Argument(value) => { - fields.push(String::from_utf8(value).map_err( - |_| bad(self.tag.to_compact_string(),"Invalid UTF-8 in header field name."), - )?); - } - _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected field name.", - )) - } - } - } - Section::HeaderFields { - not: is_not, - fields, - } - } else { - Section::Header - } - } else if value.eq_ignore_ascii_case(b"TEXT") { - Section::Text - } else if value.eq_ignore_ascii_case(b"MIME") { - Section::Mime - } else { - Section::Part { - num: parse_number::(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - } - }; - sections.push(section); - } - Token::Dot => (), - _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - format_compact!( - "Invalid token {:?} found in section-spect.", - token - ), - )) - } - } - } + let sections = Section::parse_section_spec(&mut tokens) + .map_err(|v| bad(self.tag.clone(), v))?; attributes.push_unique(Attribute::BodySection { peek: is_peek, sections, partial: parse_partial(&mut tokens) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, }); }, "BINARY" => { @@ -248,7 +165,7 @@ impl Request { let param = tokens .next() .ok_or({ - bad(self.tag.to_compact_string(),"Missing parameter after 'BINARY.'.") + bad(self.tag.clone(),"Missing parameter after 'BINARY.'.") })? .unwrap_bytes(); if param.eq_ignore_ascii_case(b"PEEK") { @@ -257,7 +174,7 @@ impl Request { (false, true) } else { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, "Expected 'PEEK' or 'SIZE' after 'BINARY.'.", )); } @@ -265,38 +182,17 @@ impl Request { (false, false) }; - // Parse section-part if tokens.next().is_none_or( |token| !token.is_bracket_open()) { - return Err(bad(self.tag.to_compact_string(), "Expected '[' after 'BINARY'.")); - } - let mut sections = Vec::new(); - while let Some(token) = tokens.next() { - match token { - Token::Argument(value) => { - sections.push( - parse_number::(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); - } - Token::Dot => (), - Token::BracketClose => break, - _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - format_compact!( - "Expected part section integer, got {:?}.", - token.to_string() - ), - )) - } - } + return Err(bad(self.tag.clone(), "Expected '[' after 'BINARY'.")); } + let sections = Section::parse_section_part(&mut tokens) + .map_err(|v| bad(self.tag.clone(), v))?; attributes.push_unique(if !is_size { Attribute::Binary { peek: is_peek, sections, partial: parse_partial(&mut tokens) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, } } else { Attribute::BinarySize { sections } @@ -330,7 +226,7 @@ impl Request { }, _ => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!("Invalid attribute {:?}", String::from_utf8_lossy(&value)), )); } @@ -344,25 +240,19 @@ impl Request { if !in_parentheses { in_parentheses = true; } else { - return Err(bad( - self.tag.to_compact_string(), - "Unexpected parenthesis open.", - )); + return Err(bad(self.tag.clone(), "Unexpected parenthesis open.")); } } Token::ParenthesisClose => { if in_parentheses { break; } else { - return Err(bad( - self.tag.to_compact_string(), - "Unexpected parenthesis close.", - )); + return Err(bad(self.tag.clone(), "Unexpected parenthesis close.")); } } _ => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!("Invalid fetch argument {:?}.", token.to_string()), )); } @@ -381,14 +271,11 @@ impl Request { &tokens .next() .ok_or_else(|| { - bad( - self.tag.to_compact_string(), - "Missing CHANGEDSINCE parameter.", - ) + bad(self.tag.clone(), "Missing CHANGEDSINCE parameter.") })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? .into(); } Token::Argument(param) if param.eq_ignore_ascii_case(b"VANISHED") => { @@ -399,7 +286,7 @@ impl Request { } _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), format_compact!("Unsupported parameter '{}'.", token), )); } @@ -416,10 +303,7 @@ impl Request { include_vanished, }) } else { - Err(bad( - CompactString::from_string_buffer(self.tag), - "No data items to fetch specified.", - )) + Err(bad(self.tag, "No data items to fetch specified.")) } } } @@ -459,6 +343,155 @@ pub fn parse_partial(tokens: &mut Peekable>) -> super::Result>) -> super::Result> { + let mut sections = Vec::new(); + let mut position = SectionPosition::Start; + loop { + let token = tokens + .next() + .ok_or_else(|| Cow::from("Expected ']' after section-spec."))?; + position = match (position, token) { + ( + SectionPosition::Start + | SectionPosition::AfterPart + | SectionPosition::AfterText, + Token::BracketClose, + ) => return Ok(sections), + (SectionPosition::AfterPart, Token::Dot) => SectionPosition::AfterDot, + (SectionPosition::Start | SectionPosition::AfterDot, Token::Argument(value)) => { + if let Some(num) = Section::part_number(&value) { + sections.push(Section::Part { num }); + SectionPosition::AfterPart + } else if value.eq_ignore_ascii_case(b"MIME") { + if matches!(position, SectionPosition::Start) { + return Err("Expected a part number before 'MIME'.".into()); + } + sections.push(Section::Mime); + SectionPosition::AfterText + } else { + sections.push(Section::parse_section_msgtext(&value, tokens)?); + SectionPosition::AfterText + } + } + (_, token) => { + return Err( + format!("Unexpected {:?} in section-spec.", token.to_string()).into(), + ); + } + }; + } + } + + fn parse_section_msgtext( + value: &[u8], + tokens: &mut Peekable>, + ) -> super::Result
{ + if value.eq_ignore_ascii_case(b"TEXT") { + return Ok(Section::Text); + } + if !value.eq_ignore_ascii_case(b"HEADER") { + return Err(format!( + "Expected a part number, 'HEADER', 'TEXT' or 'MIME', found {:?}.", + String::from_utf8_lossy(value) + ) + .into()); + } + if tokens.next_if(Token::is_dot).is_none() { + return Ok(Section::Header); + } + if tokens + .next() + .is_none_or(|token| !token.eq_ignore_ascii_case(b"FIELDS")) + { + return Err("Expected 'FIELDS' after 'HEADER.'.".into()); + } + let not = tokens.next_if(Token::is_dot).is_some(); + if not + && tokens + .next() + .is_none_or(|token| !token.eq_ignore_ascii_case(b"NOT")) + { + return Err("Expected 'NOT' after 'HEADER.FIELDS.'.".into()); + } + if tokens + .next() + .is_none_or(|token| !token.is_parenthesis_open()) + { + return Err("Expected '(' after 'HEADER.FIELDS'.".into()); + } + let mut fields = Vec::with_capacity(tokens.len().min(FIELDS_INIT_LEN)); + loop { + match tokens.next() { + Some(Token::ParenthesisClose) if !fields.is_empty() => { + return Ok(Section::HeaderFields { not, fields }); + } + Some(Token::Argument(value)) if Section::is_field_name(&value) => fields.push( + String::from_utf8(value.into_vec()) + .map_err(|_| Cow::from("Invalid UTF-8 in header field name."))?, + ), + Some(Token::Argument(_) | Token::Nil) => { + return Err( + "Header field names must be RFC 5322 field-names (printable US-ASCII except ':').".into(), + ); + } + _ => return Err("Expected a header field name.".into()), + } + } + } + + fn parse_section_part(tokens: &mut Peekable>) -> super::Result> { + let mut sections = Vec::new(); + let mut position = SectionPosition::Start; + loop { + let token = tokens + .next() + .ok_or_else(|| Cow::from("Expected ']' after section-part."))?; + position = match (position, token) { + (SectionPosition::Start | SectionPosition::AfterPart, Token::BracketClose) => { + return Ok(sections); + } + (SectionPosition::AfterPart, Token::Dot) => SectionPosition::AfterDot, + (SectionPosition::Start | SectionPosition::AfterDot, Token::Argument(value)) => { + sections.push(Section::part_number(&value).ok_or_else(|| { + Cow::from(format!( + "Expected a non-zero part number, found {:?}.", + String::from_utf8_lossy(&value) + )) + })?); + SectionPosition::AfterPart + } + (_, token) => { + return Err( + format!("Unexpected {:?} in section-part.", token.to_string()).into(), + ); + } + }; + } + } + + fn is_field_name(value: &[u8]) -> bool { + !value.is_empty() && value.iter().all(|ch| matches!(ch, 33..=57 | 59..=126)) + } + + fn part_number(value: &[u8]) -> Option { + match value { + [b'1'..=b'9', rest @ ..] if rest.iter().all(u8::is_ascii_digit) => { + std::str::from_utf8(value).ok()?.parse().ok() + } + _ => None, + } + } +} + /* fetch = "FETCH" SP sequence-set SP ( @@ -501,328 +534,3 @@ pub fn parse_partial(tokens: &mut Peekable>) -> super::Result ", - "BODY.PEEK[3.2.2.2] ", - "BODY[4.2.TEXT]<4.100> ", - "BINARY[1.2.3] ", - "BINARY.PEEK[4] ", - "BINARY[6.5.4]<100.200> ", - "BINARY.PEEK[7]<9.88> ", - "BINARY.SIZE[9.1]", - ")\r\n" - ), - fetch::Arguments { - tag: "A001".into(), - sequence_set: Sequence::number(1), - attributes: vec![ - Attribute::BodySection { - peek: false, - sections: vec![ - Section::Part { num: 4 }, - Section::Part { num: 2 }, - Section::Header, - ], - partial: Some((0, 20)), - }, - Attribute::BodySection { - peek: true, - sections: vec![ - Section::Part { num: 3 }, - Section::Part { num: 2 }, - Section::Part { num: 2 }, - Section::Part { num: 2 }, - ], - partial: None, - }, - Attribute::BodySection { - peek: false, - sections: vec![ - Section::Part { num: 4 }, - Section::Part { num: 2 }, - Section::Text, - ], - partial: Some((4, 100)), - }, - Attribute::Binary { - peek: false, - sections: vec![1, 2, 3], - partial: None, - }, - Attribute::Binary { - peek: true, - sections: vec![4], - partial: None, - }, - Attribute::Binary { - peek: false, - sections: vec![6, 5, 4], - partial: Some((100, 200)), - }, - Attribute::Binary { - peek: true, - sections: vec![7], - partial: Some((9, 88)), - }, - Attribute::BinarySize { - sections: vec![9, 1], - }, - ], - changed_since: None, - include_vanished: false, - }, - ), - ( - "A001 FETCH 1 ALL\r\n", - fetch::Arguments { - tag: "A001".into(), - sequence_set: Sequence::number(1), - attributes: vec![ - Attribute::Flags, - Attribute::InternalDate, - Attribute::Rfc822Size, - Attribute::Envelope, - ], - changed_since: None, - include_vanished: false, - }, - ), - ( - "A001 FETCH 1 FULL\r\n", - fetch::Arguments { - tag: "A001".into(), - sequence_set: Sequence::number(1), - attributes: vec![ - Attribute::Flags, - Attribute::InternalDate, - Attribute::Rfc822Size, - Attribute::Envelope, - Attribute::Body, - ], - changed_since: None, - include_vanished: false, - }, - ), - ( - "A001 FETCH 1 FAST\r\n", - fetch::Arguments { - tag: "A001".into(), - sequence_set: Sequence::number(1), - attributes: vec![ - Attribute::Flags, - Attribute::InternalDate, - Attribute::Rfc822Size, - ], - changed_since: None, - include_vanished: false, - }, - ), - ( - "s100 UID FETCH 1:* (FLAGS MODSEQ) (CHANGEDSINCE 12345 VANISHED)\r\n", - fetch::Arguments { - tag: "s100".into(), - sequence_set: Sequence::range(1.into(), None), - attributes: vec![Attribute::Flags, Attribute::ModSeq], - changed_since: 12345.into(), - include_vanished: true, - }, - ), - ( - "9 UID FETCH 1:* UID (VANISHED CHANGEDSINCE 1)\r\n", - fetch::Arguments { - tag: "9".into(), - sequence_set: Sequence::range(1.into(), None), - attributes: vec![Attribute::Uid], - changed_since: 1.into(), - include_vanished: true, - }, - ), - ( - "A010 FETCH 1:* (OBJECTID)\r\n", - fetch::Arguments { - tag: "A010".into(), - sequence_set: Sequence::range(1.into(), None), - attributes: vec![Attribute::ObjectId], - changed_since: None, - include_vanished: false, - }, - ), - ( - "A011 FETCH 1 (UID OBJECTID FLAGS)\r\n", - fetch::Arguments { - tag: "A011".into(), - sequence_set: Sequence::number(1), - attributes: vec![Attribute::Uid, Attribute::ObjectId, Attribute::Flags], - changed_since: None, - include_vanished: false, - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_fetch() - .expect(command), - arguments, - "{}", - command - ); - } - } -} diff --git a/crates/imap-proto/src/parser/list.rs b/crates/imap-proto/src/parser/list.rs index dcd1021f..e4fe41d5 100644 --- a/crates/imap-proto/src/parser/list.rs +++ b/crates/imap-proto/src/parser/list.rs @@ -4,17 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString}; - use crate::{ Command, protocol::{ list::{self, ReturnOption, SelectionOption}, + metadata::Entry, status::Status, }, receiver::{Request, Token, bad}, - utf7::utf7_maybe_decode, }; +use std::mem::discriminant; impl Request { #[allow(clippy::while_let_on_iterator)] @@ -28,15 +27,12 @@ impl Request { .next() .unwrap() .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - mailbox_name: utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + .map_err(|v| bad(self.tag.clone(), v))?, + mailbox_name: tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }) } @@ -54,12 +50,12 @@ impl Request { Token::Argument(value) => { selection_options.push( SelectionOption::parse(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, ); } _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid selection option argument.", )); } @@ -67,20 +63,18 @@ impl Request { } tokens .next() - .ok_or_else(|| { - bad(self.tag.to_compact_string(), "Missing reference name.") - })? + .ok_or_else(|| bad(self.tag.clone(), "Missing reference name."))? .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? } token => token .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, }; match tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing mailbox name."))? { Token::ParenthesisOpen => { while let Some(token) = tokens.next() { @@ -90,19 +84,18 @@ impl Request { mailbox_name.push( token .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, ); } } } } token => { - mailbox_name.push(utf7_maybe_decode( + mailbox_name.push( token - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - )); + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, + ); } } @@ -115,7 +108,7 @@ impl Request { .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid return option, expected parenthesis.", )); } @@ -125,14 +118,14 @@ impl Request { Token::ParenthesisClose => break, Token::Argument(value) => { let mut return_option = ReturnOption::parse(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; if let ReturnOption::Status(status) = &mut return_option { if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, "Invalid return option, expected parenthesis after STATUS.", )); } @@ -140,24 +133,42 @@ impl Request { match token { Token::ParenthesisClose => break, Token::Argument(value) => { - status.push(Status::parse(&value).map_err( - |v| bad(self.tag.to_compact_string(), v), - )?); + status.push( + Status::parse(&value) + .map_err(|v| bad(self.tag.clone(), v))?, + ); } _ => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, "Invalid status return option argument.", )); } } } + } else if let ReturnOption::Metadata(entries) = &mut return_option { + if tokens + .next() + .is_none_or(|token| !token.is_parenthesis_open()) + { + return Err(bad( + self.tag, + "Invalid return option, expected parenthesis after METADATA.", + )); + } + *entries = Entry::parse_list(&mut tokens) + .map_err(|v| bad(self.tag.clone(), v))?; + entries.sort_unstable(); + entries.dedup(); } + return_options.retain(|existing| { + discriminant(existing) != discriminant(&return_option) + }); return_options.push(return_option); } _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid return option argument.", )); } @@ -179,11 +190,12 @@ impl Request { impl SelectionOption { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "SUBSCRIBED" => Self::Subscribed, - "REMOTE" => Self::Remote, - "RECURSIVEMATCH" => Self::RecursiveMatch, - "SPECIAL-USE" => Self::SpecialUse, + hashify::fnc_map_ignore_case!(value, + "SUBSCRIBED" => Some(Self::Subscribed), + "REMOTE" => Some(Self::Remote), + "RECURSIVEMATCH" => Some(Self::RecursiveMatch), + "SPECIAL-USE" => Some(Self::SpecialUse), + _ => None, ) .ok_or_else(|| { format!( @@ -197,186 +209,16 @@ impl SelectionOption { impl ReturnOption { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "SUBSCRIBED" => Self::Subscribed, - "CHILDREN" => Self::Children, - "STATUS" => Self::Status(Vec::with_capacity(2)), - "SPECIAL-USE" => Self::SpecialUse, + hashify::fnc_map_ignore_case!(value, + "SUBSCRIBED" => Some(Self::Subscribed), + "CHILDREN" => Some(Self::Children), + "STATUS" => Some(Self::Status(Vec::with_capacity(2))), + "SPECIAL-USE" => Some(Self::SpecialUse), + "METADATA" => Some(Self::Metadata(Vec::new())), + "OBJECTID" => Some(Self::ObjectId), + _ => None, ) .ok_or_else(|| format!("Invalid return option {:?}", String::from_utf8_lossy(value)).into()) } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::{ - list::{self, ReturnOption, SelectionOption}, - status::Status, - }, - receiver::Receiver, - }; - - #[test] - fn parse_list() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A682 LIST \"\" *\r\n", - list::Arguments::Basic { - tag: "A682".into(), - reference_name: "".into(), - mailbox_name: "*".into(), - }, - ), - ( - "A02 LIST (SUBSCRIBED) \"\" \"*\"\r\n", - list::Arguments::Extended { - tag: "A02".into(), - reference_name: "".into(), - mailbox_name: vec!["*".into()], - selection_options: vec![SelectionOption::Subscribed], - return_options: vec![], - }, - ), - ( - "A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n", - list::Arguments::Extended { - tag: "A03".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![], - return_options: vec![ReturnOption::Children], - }, - ), - ( - "A04 LIST (REMOTE) \"\" \"%\" RETURN (CHILDREN)\r\n", - list::Arguments::Extended { - tag: "A04".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![SelectionOption::Remote], - return_options: vec![ReturnOption::Children], - }, - ), - ( - "A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n", - list::Arguments::Extended { - tag: "A05".into(), - reference_name: "".into(), - mailbox_name: vec!["*".into()], - selection_options: vec![SelectionOption::Remote, SelectionOption::Subscribed], - return_options: vec![], - }, - ), - ( - "A06 LIST (REMOTE) \"\" \"*\" RETURN (SUBSCRIBED)\r\n", - list::Arguments::Extended { - tag: "A06".into(), - reference_name: "".into(), - mailbox_name: vec!["*".into()], - selection_options: vec![SelectionOption::Remote], - return_options: vec![ReturnOption::Subscribed], - }, - ), - ( - "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\"\r\n", - list::Arguments::Extended { - tag: "C04".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![ - SelectionOption::Subscribed, - SelectionOption::RecursiveMatch, - ], - return_options: vec![], - }, - ), - ( - "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\" RETURN (CHILDREN)\r\n", - list::Arguments::Extended { - tag: "C04".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![ - SelectionOption::Subscribed, - SelectionOption::RecursiveMatch, - ], - return_options: vec![ReturnOption::Children], - }, - ), - ( - "a1 LIST \"\" (\"foo\")\r\n", - list::Arguments::Extended { - tag: "a1".into(), - reference_name: "".into(), - mailbox_name: vec!["foo".into()], - selection_options: vec![], - return_options: vec![], - }, - ), - ( - "a3.1 LIST \"\" (% music/rock)\r\n", - list::Arguments::Extended { - tag: "a3.1".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into(), "music/rock".into()], - selection_options: vec![], - return_options: vec![], - }, - ), - ( - "BBB LIST \"\" (\"INBOX\" \"Drafts\" \"Sent/%\")\r\n", - list::Arguments::Extended { - tag: "BBB".into(), - reference_name: "".into(), - mailbox_name: vec!["INBOX".into(), "Drafts".into(), "Sent/%".into()], - selection_options: vec![], - return_options: vec![], - }, - ), - ( - "A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n", - list::Arguments::Extended { - tag: "A01".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![], - return_options: vec![ReturnOption::Status(vec![ - Status::Messages, - Status::Unseen, - ])], - }, - ), - ( - concat!( - "A02 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" ", - "% RETURN (CHILDREN STATUS (MESSAGES))\r\n" - ), - list::Arguments::Extended { - tag: "A02".into(), - reference_name: "".into(), - mailbox_name: vec!["%".into()], - selection_options: vec![ - SelectionOption::Subscribed, - SelectionOption::RecursiveMatch, - ], - return_options: vec![ - ReturnOption::Children, - ReturnOption::Status(vec![Status::Messages]), - ], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_list(true) - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/login.rs b/crates/imap-proto/src/parser/login.rs index 37eab322..edd40cfe 100644 --- a/crates/imap-proto/src/parser/login.rs +++ b/crates/imap-proto/src/parser/login.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::login, @@ -22,12 +20,12 @@ impl Request { .next() .unwrap() .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, password: tokens .next() .unwrap() .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }) } @@ -37,40 +35,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::login, receiver::Receiver}; - - #[test] - fn parse_login() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "a001 LOGIN SMITH SESAME\r\n", - login::Arguments { - tag: "a001".into(), - username: "SMITH".into(), - password: "SESAME".into(), - }, - ), - ( - "A001 LOGIN {11+}\r\nFRED FOOBAR {7+}\r\nfat man\r\n", - login::Arguments { - tag: "A001".into(), - username: "FRED FOOBAR".into(), - password: "fat man".into(), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_login() - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/lsub.rs b/crates/imap-proto/src/parser/lsub.rs index ebd601a8..0c4b80a4 100644 --- a/crates/imap-proto/src/parser/lsub.rs +++ b/crates/imap-proto/src/parser/lsub.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::list::{self, SelectionOption}, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; impl Request { @@ -21,17 +18,16 @@ impl Request { Ok(list::Arguments::Extended { reference_name: tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing reference name."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing reference name."))? .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - mailbox_name: vec![utf7_maybe_decode( + .map_err(|v| bad(self.tag.clone(), v))?, + mailbox_name: vec![ tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - )], + .ok_or_else(|| bad(self.tag.clone(), "Missing mailbox name."))? + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, + ], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], tag: self.tag, @@ -42,47 +38,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::list::{self, SelectionOption}, - receiver::Receiver, - }; - - #[test] - fn parse_lsub() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A002 LSUB \"#news.\" \"comp.mail.*\"\r\n", - list::Arguments::Extended { - tag: "A002".into(), - reference_name: "#news.".into(), - mailbox_name: vec!["comp.mail.*".into()], - selection_options: vec![SelectionOption::Subscribed], - return_options: vec![], - }, - ), - ( - "A002 LSUB \"#news.\" \"comp.%\"\r\n", - list::Arguments::Extended { - tag: "A002".into(), - reference_name: "#news.".into(), - mailbox_name: vec!["comp.%".into()], - selection_options: vec![SelectionOption::Subscribed], - return_options: vec![], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_lsub(false) - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/metadata.rs b/crates/imap-proto/src/parser/metadata.rs new file mode 100644 index 00000000..5f22df89 --- /dev/null +++ b/crates/imap-proto/src/parser/metadata.rs @@ -0,0 +1,233 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::parse_number; +use crate::{ + Command, + protocol::metadata::{Depth, Entry, EntryValue, GetArguments, Scope, SetArguments}, + receiver::{Request, Token, bad}, +}; +use compact_str::CompactString; +use std::{borrow::Cow, iter::Peekable, vec::IntoIter}; + +type Tokens = Peekable>; + +impl Request { + pub fn parse_get_metadata(self, is_utf8: bool) -> trc::Result { + let mut arguments = GetArguments { + tag: self.tag, + mailbox_name: CompactString::const_new(""), + max_size: None, + depth: Depth::Zero, + entries: Vec::new(), + }; + arguments + .parse(&mut self.tokens.into_iter().peekable(), is_utf8) + .map_err(|err| bad(arguments.tag.clone(), err))?; + Ok(arguments) + } + + pub fn parse_set_metadata(self, is_utf8: bool) -> trc::Result { + let mut arguments = SetArguments { + tag: self.tag, + mailbox_name: CompactString::const_new(""), + entries: Vec::new(), + }; + arguments + .parse(&mut self.tokens.into_iter(), is_utf8) + .map_err(|err| bad(arguments.tag.clone(), err))?; + Ok(arguments) + } +} + +impl GetArguments { + fn parse(&mut self, tokens: &mut Tokens, is_utf8: bool) -> super::Result<()> { + if tokens.next_if(Token::is_parenthesis_open).is_some() { + self.parse_options(tokens)?; + } + + self.mailbox_name = tokens + .next() + .ok_or("Missing mailbox name.")? + .unwrap_mailbox_name(is_utf8)?; + + let mut is_list = tokens.next_if(Token::is_parenthesis_open).is_some(); + if is_list + && tokens + .peek() + .is_some_and(|token| !token.as_bytes().starts_with(b"/")) + { + self.parse_options(tokens)?; + is_list = tokens.next_if(Token::is_parenthesis_open).is_some(); + } + + if is_list { + self.entries = Entry::parse_list(tokens)?; + if tokens.next().is_some() { + return Err("Unexpected arguments after entry specifiers.".into()); + } + } else { + for token in tokens { + self.entries.push(Entry::parse(token.as_bytes())?); + } + if self.entries.is_empty() { + return Err("Missing entry specifier.".into()); + } + } + + self.entries.sort_unstable(); + self.entries.dedup(); + Ok(()) + } + + fn parse_options(&mut self, tokens: &mut Tokens) -> super::Result<()> { + while let Some(token) = tokens.next() { + match token { + Token::ParenthesisClose => return Ok(()), + Token::Argument(option) => { + hashify::fnc_map_ignore_case!(option.as_slice(), + "MAXSIZE" => { + self.max_size = Some(parse_number( + tokens.next().ok_or("Missing MAXSIZE value.")?.as_bytes(), + )?); + }, + "DEPTH" => { + self.depth = + Depth::parse(tokens.next().ok_or("Missing DEPTH value.")?.as_bytes())?; + }, + _ => { + return Err(format!( + "Unsupported GETMETADATA option {:?}.", + String::from_utf8_lossy(&option) + ) + .into()); + } + ) + } + _ => return Err("Invalid GETMETADATA option.".into()), + } + } + + Err("Unterminated GETMETADATA options.".into()) + } +} + +impl SetArguments { + fn parse(&mut self, tokens: &mut IntoIter, is_utf8: bool) -> super::Result<()> { + self.mailbox_name = tokens + .next() + .ok_or("Missing mailbox name.")? + .unwrap_mailbox_name(is_utf8)?; + + if tokens + .next() + .is_none_or(|token| !token.is_parenthesis_open()) + { + return Err("Expected a parenthesized list of entries and values.".into()); + } + + loop { + let entry = match tokens.next().ok_or("Unterminated entry value list.")? { + Token::ParenthesisClose if !self.entries.is_empty() => break, + Token::ParenthesisOpen | Token::ParenthesisClose => { + return Err("Invalid entry value list.".into()); + } + token => Entry::parse(token.as_bytes())?, + }; + let value = match tokens.next().ok_or("Missing entry value.")? { + Token::Argument(value) if value.eq_ignore_ascii_case(b"NIL") => None, + Token::ParenthesisOpen | Token::ParenthesisClose => { + return Err("Missing entry value.".into()); + } + token => Some(Cow::Owned(token.unwrap_bytes().into_vec())), + }; + self.entries.push(EntryValue { entry, value }); + } + + if tokens.next().is_some() { + return Err("Unexpected arguments after entry values.".into()); + } + + self.entries.reverse(); + self.entries.sort_by(|a, b| a.entry.cmp(&b.entry)); + self.entries.dedup_by(|a, b| a.entry == b.entry); + Ok(()) + } +} + +impl Entry<'static> { + pub fn parse(value: &[u8]) -> super::Result { + if let Some(ch) = value + .iter() + .find(|&&ch| !(0x1a..0x80).contains(&ch) || ch == b'*' || ch == b'%') + { + return Err(format!("Invalid character {:?} in entry name.", *ch as char).into()); + } + + let name = std::str::from_utf8(value).map_err(|_| "Invalid entry name.")?; + let (scope, rest) = name + .strip_prefix('/') + .map(|name| { + name.split_once('/') + .map_or((name, None), |(scope, rest)| (scope, Some(rest))) + }) + .ok_or("Entry names must begin with /shared or /private.")?; + let scope = hashify::map_ignore_case!(scope.as_bytes(), Scope, + "shared" => Scope::Shared, + "private" => Scope::Private, + ) + .copied() + .ok_or("Entry names must begin with /shared or /private.")?; + let rest = rest.ok_or("Entry names must have at least two components.")?; + + let mut path = String::with_capacity(rest.len() + 1); + path.push('/'); + path.push_str(rest); + if path.ends_with('/') || path.contains("//") { + return Err(format!("Invalid entry name {name:?}.").into()); + } + path.make_ascii_lowercase(); + + let mut components = path.split('/').skip(1); + if let (Some("vendor"), _, None) = (components.next(), components.next(), components.next()) + { + return Err("Vendor entry names must have at least four components.".into()); + } + + Ok(Entry { + scope, + path: Cow::Owned(path), + }) + } + + pub(crate) fn parse_list(tokens: &mut impl Iterator) -> super::Result> { + let mut entries = Vec::new(); + for token in tokens { + match token { + Token::ParenthesisClose if !entries.is_empty() => return Ok(entries), + Token::ParenthesisOpen | Token::ParenthesisClose => { + return Err("Invalid entry specifier list.".into()); + } + token => entries.push(Entry::parse(token.as_bytes())?), + } + } + + Err("Unterminated entry specifier list.".into()) + } +} + +impl Depth { + fn parse(value: &[u8]) -> super::Result { + hashify::map_ignore_case!(value, Depth, + "0" => Depth::Zero, + "1" => Depth::One, + "infinity" => Depth::Infinity, + ) + .copied() + .ok_or_else(|| "DEPTH must be 0, 1 or infinity.".into()) + } +} + diff --git a/crates/imap-proto/src/parser/mod.rs b/crates/imap-proto/src/parser/mod.rs index 96cf8a0c..e098f042 100644 --- a/crates/imap-proto/src/parser/mod.rs +++ b/crates/imap-proto/src/parser/mod.rs @@ -15,6 +15,7 @@ pub mod fetch; pub mod list; pub mod login; pub mod lsub; +pub mod metadata; pub mod quota; pub mod rename; pub mod search; @@ -26,63 +27,65 @@ pub mod subscribe; pub mod thread; pub mod uidbatches; -use std::{borrow::Cow, str::FromStr}; - -use chrono::{DateTime, NaiveDate}; - use crate::{ Command, protocol::{Flag, Sequence}, - receiver::CommandParser, + receiver::{ArgumentBytes, CommandParser}, }; +use compact_str::CompactString; +use jiff::{civil::Time, fmt::strtime, tz::Offset}; +use std::{borrow::Cow, str::FromStr}; pub type Result = std::result::Result>; impl CommandParser for Command { fn parse(value: &[u8], uid: bool) -> Option { - hashify::tiny_map!(value, - "CAPABILITY" => Command::Capability, - "NOOP" => Command::Noop, - "LOGOUT" => Command::Logout, - "STARTTLS" => Command::StartTls, - "AUTHENTICATE" => Command::Authenticate, - "LOGIN" => Command::Login, - "ENABLE" => Command::Enable, - "SELECT" => Command::Select, - "EXAMINE" => Command::Examine, - "CREATE" => Command::Create, - "DELETE" => Command::Delete, - "RENAME" => Command::Rename, - "SUBSCRIBE" => Command::Subscribe, - "UNSUBSCRIBE" => Command::Unsubscribe, - "LIST" => Command::List, - "NAMESPACE" => Command::Namespace, - "STATUS" => Command::Status, - "APPEND" => Command::Append, - "IDLE" => Command::Idle, - "CLOSE" => Command::Close, - "UNSELECT" => Command::Unselect, - "EXPUNGE" => Command::Expunge(uid), - "SEARCH" => Command::Search(uid), - "FETCH" => Command::Fetch(uid), - "STORE" => Command::Store(uid), - "COPY" => Command::Copy(uid), - "MOVE" => Command::Move(uid), - "SORT" => Command::Sort(uid), - "THREAD" => Command::Thread(uid), - "LSUB" => Command::Lsub, - "CHECK" => Command::Check, - "SETACL" => Command::SetAcl, - "DELETEACL" => Command::DeleteAcl, - "GETACL" => Command::GetAcl, - "LISTRIGHTS" => Command::ListRights, - "MYRIGHTS" => Command::MyRights, - "UNAUTHENTICATE" => Command::Unauthenticate, - "ID" => Command::Id, - "GETQUOTA" => Command::GetQuota, - "GETQUOTAROOT" => Command::GetQuotaRoot, - "GETJMAPACCESS" => Command::GetJmapAccess, - "UIDBATCHES" => Command::UidBatches, + hashify::fnc_map!(value, + "CAPABILITY" => Some(Command::Capability), + "NOOP" => Some(Command::Noop), + "LOGOUT" => Some(Command::Logout), + "STARTTLS" => Some(Command::StartTls), + "AUTHENTICATE" => Some(Command::Authenticate), + "LOGIN" => Some(Command::Login), + "ENABLE" => Some(Command::Enable), + "SELECT" => Some(Command::Select), + "EXAMINE" => Some(Command::Examine), + "CREATE" => Some(Command::Create), + "DELETE" => Some(Command::Delete), + "RENAME" => Some(Command::Rename), + "SUBSCRIBE" => Some(Command::Subscribe), + "UNSUBSCRIBE" => Some(Command::Unsubscribe), + "LIST" => Some(Command::List), + "NAMESPACE" => Some(Command::Namespace), + "STATUS" => Some(Command::Status), + "APPEND" => Some(Command::Append), + "IDLE" => Some(Command::Idle), + "CLOSE" => Some(Command::Close), + "UNSELECT" => Some(Command::Unselect), + "EXPUNGE" => Some(Command::Expunge(uid)), + "SEARCH" => Some(Command::Search(uid)), + "FETCH" => Some(Command::Fetch(uid)), + "STORE" => Some(Command::Store(uid)), + "COPY" => Some(Command::Copy(uid)), + "MOVE" => Some(Command::Move(uid)), + "SORT" => Some(Command::Sort(uid)), + "THREAD" => Some(Command::Thread(uid)), + "LSUB" => Some(Command::Lsub), + "CHECK" => Some(Command::Check), + "SETACL" => Some(Command::SetAcl), + "DELETEACL" => Some(Command::DeleteAcl), + "GETACL" => Some(Command::GetAcl), + "LISTRIGHTS" => Some(Command::ListRights), + "MYRIGHTS" => Some(Command::MyRights), + "UNAUTHENTICATE" => Some(Command::Unauthenticate), + "ID" => Some(Command::Id), + "GETQUOTA" => Some(Command::GetQuota), + "GETQUOTAROOT" => Some(Command::GetQuotaRoot), + "GETJMAPACCESS" => Some(Command::GetJmapAccess), + "UIDBATCHES" => Some(Command::UidBatches), + "GETMETADATA" => Some(Command::GetMetadata), + "SETMETADATA" => Some(Command::SetMetadata), + _ => None, ) } @@ -90,50 +93,60 @@ impl CommandParser for Command { fn tokenize_brackets(&self) -> bool { matches!(self, Command::Fetch(_)) } + + #[inline(always)] + fn tokenize_nil(&self) -> bool { + matches!(self, Command::SetMetadata) + } } impl Flag { - pub fn parse_imap(value: Vec) -> Result { + pub fn parse_imap(value: ArgumentBytes) -> Result { if !value.is_empty() { - let flag = hashify::tiny_map_ignore_case!(value.as_slice(), - "\\Seen" => Flag::Seen, - "\\Answered" => Flag::Answered, - "\\Flagged" => Flag::Flagged, - "\\Deleted" => Flag::Deleted, - "\\Draft" => Flag::Draft, - "\\Recent" => Flag::Recent, - "\\Important" => Flag::Important, - "$Forwarded" => Flag::Forwarded, - "$MDNSent" => Flag::MDNSent, - "$Junk" => Flag::Junk, - "$NotJunk" => Flag::NotJunk, - "$Phishing" => Flag::Phishing, - "$Important" => Flag::Important, - "$autosent" => Flag::Autosent, - "$canunsubscribe" => Flag::CanUnsubscribe, - "$followed" => Flag::Followed, - "$hasattachment" => Flag::HasAttachment, - "$hasmemo" => Flag::HasMemo, - "$hasnoattachment" => Flag::HasNoAttachment, - "$imported" => Flag::Imported, - "$istrusted" => Flag::IsTrusted, - "$MailFlagBit0" => Flag::MailFlagBit0, - "$MailFlagBit1" => Flag::MailFlagBit1, - "$MailFlagBit2" => Flag::MailFlagBit2, - "$maskedemail" => Flag::MaskedEmail, - "$memo" => Flag::Memo, - "$muted" => Flag::Muted, - "$new" => Flag::New, - "$notify" => Flag::Notify, - "$unsubscribed" => Flag::Unsubscribed, + let flag = hashify::fnc_map_ignore_case!(value.as_slice(), + "\\Seen" => Some(Flag::Seen), + "\\Answered" => Some(Flag::Answered), + "\\Flagged" => Some(Flag::Flagged), + "\\Deleted" => Some(Flag::Deleted), + "\\Draft" => Some(Flag::Draft), + "\\Recent" => Some(Flag::Recent), + "\\Important" => Some(Flag::Important), + "$Forwarded" => Some(Flag::Forwarded), + "$MDNSent" => Some(Flag::MDNSent), + "$Junk" => Some(Flag::Junk), + "$NotJunk" => Some(Flag::NotJunk), + "$Phishing" => Some(Flag::Phishing), + "$Important" => Some(Flag::Important), + "$autosent" => Some(Flag::Autosent), + "$canunsubscribe" => Some(Flag::CanUnsubscribe), + "$followed" => Some(Flag::Followed), + "$hasattachment" => Some(Flag::HasAttachment), + "$hasmemo" => Some(Flag::HasMemo), + "$hasnoattachment" => Some(Flag::HasNoAttachment), + "$imported" => Some(Flag::Imported), + "$istrusted" => Some(Flag::IsTrusted), + "$MailFlagBit0" => Some(Flag::MailFlagBit0), + "$MailFlagBit1" => Some(Flag::MailFlagBit1), + "$MailFlagBit2" => Some(Flag::MailFlagBit2), + "$maskedemail" => Some(Flag::MaskedEmail), + "$memo" => Some(Flag::Memo), + "$muted" => Some(Flag::Muted), + "$new" => Some(Flag::New), + "$notify" => Some(Flag::Notify), + "$unsubscribed" => Some(Flag::Unsubscribed), + _ => None, ); if let Some(flag) = flag { Ok(flag) - } else { - String::from_utf8(value) + } else if value.spilled() { + String::from_utf8(value.into_vec()) .map_err(|_| Cow::from("Invalid UTF-8.")) - .map(|v| Flag::Keyword(v.into_boxed_str())) + .map(|keyword| Flag::Keyword(CompactString::from_string_buffer(keyword))) + } else { + CompactString::from_utf8(value.as_slice()) + .map_err(|_| Cow::from("Invalid UTF-8.")) + .map(Flag::Keyword) } } else { Err(Cow::from("Null flags are not allowed.")) @@ -142,38 +155,39 @@ impl Flag { pub fn parse_jmap(value: String) -> Self { if value.starts_with('$') { - hashify::tiny_map_ignore_case!(value.as_bytes(), - "$seen" => Flag::Seen, - "$draft" => Flag::Draft, - "$flagged" => Flag::Flagged, - "$answered" => Flag::Answered, - "$recent" => Flag::Recent, - "$important" => Flag::Important, - "$phishing" => Flag::Phishing, - "$junk" => Flag::Junk, - "$notjunk" => Flag::NotJunk, - "$deleted" => Flag::Deleted, - "$forwarded" => Flag::Forwarded, - "$mdnsent" => Flag::MDNSent, - "$autosent" => Flag::Autosent, - "$canunsubscribe" => Flag::CanUnsubscribe, - "$followed" => Flag::Followed, - "$hasattachment" => Flag::HasAttachment, - "$hasmemo" => Flag::HasMemo, - "$hasnoattachment" => Flag::HasNoAttachment, - "$imported" => Flag::Imported, - "$istrusted" => Flag::IsTrusted, - "$MailFlagBit0" => Flag::MailFlagBit0, - "$MailFlagBit1" => Flag::MailFlagBit1, - "$MailFlagBit2" => Flag::MailFlagBit2, - "$maskedemail" => Flag::MaskedEmail, - "$memo" => Flag::Memo, - "$muted" => Flag::Muted, - "$new" => Flag::New, - "$notify" => Flag::Notify, - "$unsubscribed" => Flag::Unsubscribed, + hashify::fnc_map_ignore_case!(value.as_bytes(), + "$seen" => Some(Flag::Seen), + "$draft" => Some(Flag::Draft), + "$flagged" => Some(Flag::Flagged), + "$answered" => Some(Flag::Answered), + "$recent" => Some(Flag::Recent), + "$important" => Some(Flag::Important), + "$phishing" => Some(Flag::Phishing), + "$junk" => Some(Flag::Junk), + "$notjunk" => Some(Flag::NotJunk), + "$deleted" => Some(Flag::Deleted), + "$forwarded" => Some(Flag::Forwarded), + "$mdnsent" => Some(Flag::MDNSent), + "$autosent" => Some(Flag::Autosent), + "$canunsubscribe" => Some(Flag::CanUnsubscribe), + "$followed" => Some(Flag::Followed), + "$hasattachment" => Some(Flag::HasAttachment), + "$hasmemo" => Some(Flag::HasMemo), + "$hasnoattachment" => Some(Flag::HasNoAttachment), + "$imported" => Some(Flag::Imported), + "$istrusted" => Some(Flag::IsTrusted), + "$MailFlagBit0" => Some(Flag::MailFlagBit0), + "$MailFlagBit1" => Some(Flag::MailFlagBit1), + "$MailFlagBit2" => Some(Flag::MailFlagBit2), + "$maskedemail" => Some(Flag::MaskedEmail), + "$memo" => Some(Flag::Memo), + "$muted" => Some(Flag::Muted), + "$new" => Some(Flag::New), + "$notify" => Some(Flag::Notify), + "$unsubscribed" => Some(Flag::Unsubscribed), + _ => None, ) - .unwrap_or_else(|| Flag::Keyword(value.into_boxed_str())) + .unwrap_or_else(|| Flag::Keyword(value.into())) } else { let mut keyword = String::with_capacity(value.len()); for c in value.chars() { @@ -183,33 +197,168 @@ impl Flag { keyword.push('_'); } } - Flag::Keyword(keyword.into_boxed_str()) + Flag::Keyword(keyword.into()) } } } +const SECONDS_PER_DAY: i64 = 86_400; + +#[inline(always)] +fn short_month(m0: u8, m1: u8, m2: u8) -> Option { + Some(match (m0 | 32, m1 | 32, m2 | 32) { + (b'j', b'a', b'n') => 1, + (b'f', b'e', b'b') => 2, + (b'm', b'a', b'r') => 3, + (b'a', b'p', b'r') => 4, + (b'm', b'a', b'y') => 5, + (b'j', b'u', b'n') => 6, + (b'j', b'u', b'l') => 7, + (b'a', b'u', b'g') => 8, + (b's', b'e', b'p') => 9, + (b'o', b'c', b't') => 10, + (b'n', b'o', b'v') => 11, + (b'd', b'e', b'c') => 12, + _ => return None, + }) +} + +#[inline(always)] +fn days_in_month(year: i32, month: u32) -> u32 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + _ if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29, + _ => 28, + } +} + +#[inline(always)] +fn days_from_civil(year: i32, month: u32, day: u32) -> i64 { + let year = year - (month <= 2) as i32; + let era = if year >= 0 { year } else { year - 399 } / 400; + let year_of_era = (year - era * 400) as i64; + let month_of_year = (if month > 2 { month - 3 } else { month + 9 }) as i64; + let day_of_year = (153 * month_of_year + 2) / 5 + day as i64 - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era as i64 * 146_097 + day_of_era - 719_468 +} + +#[inline(always)] +fn two_digits(d0: u8, d1: u8) -> Option { + (d0.is_ascii_digit() && d1.is_ascii_digit()).then(|| ((d0 - b'0') * 10 + (d1 - b'0')) as u32) +} + +#[inline(always)] +fn parse_civil_date(value: &[u8]) -> Option<(i64, &[u8])> { + let (day, rest) = match value { + [d0, d1, b'-', rest @ ..] if d0.is_ascii_digit() && d1.is_ascii_digit() => { + (two_digits(*d0, *d1)?, rest) + } + [d0, b'-', rest @ ..] if d0.is_ascii_digit() => ((*d0 - b'0') as u32, rest), + _ => return None, + }; + let [m0, m1, m2, b'-', rest @ ..] = rest else { + return None; + }; + let month = short_month(*m0, *m1, *m2)?; + let [y0, y1, y2, y3, rest @ ..] = rest else { + return None; + }; + let year = (two_digits(*y0, *y1)? * 100 + two_digits(*y2, *y3)?) as i32; + + if day == 0 || day > days_in_month(year, month) { + return None; + } + + Some((days_from_civil(year, month, day), rest)) +} + +fn parse_datetime_fast(value: &[u8]) -> Option { + let (days, rest) = parse_civil_date(value)?; + let [ + b' ', + h0, + h1, + b':', + m0, + m1, + b':', + s0, + s1, + b' ', + sign, + z0, + z1, + z2, + z3, + ] = rest + else { + return None; + }; + let hour = two_digits(*h0, *h1)?; + let minute = two_digits(*m0, *m1)?; + let second = two_digits(*s0, *s1)?; + let zone_hour = two_digits(*z0, *z1)?; + let zone_minute = two_digits(*z2, *z3)?; + + if hour > 23 || minute > 59 || second > 59 || zone_hour > 23 || zone_minute > 59 { + return None; + } + + let zone = (zone_hour * 3600 + zone_minute * 60) as i64; + let zone = match sign { + b'+' => -zone, + b'-' => zone, + _ => return None, + }; + + Some(days * SECONDS_PER_DAY + (hour * 3600 + minute * 60 + second) as i64 + zone) +} + +fn parse_date_fast(value: &[u8]) -> Option { + let (days, rest) = parse_civil_date(value)?; + rest.is_empty().then(|| days * SECONDS_PER_DAY) +} + pub fn parse_datetime(value: &[u8]) -> Result { + if let Some(timestamp) = parse_datetime_fast(value.trim_ascii()) { + return Ok(timestamp); + } + + parse_datetime_slow(value) +} + +#[inline(never)] +fn parse_datetime_slow(value: &[u8]) -> Result { std::str::from_utf8(value) .map_err(|_| Cow::from("Expected date/time, found an invalid UTF-8 string.")) .and_then(|datetime| { - DateTime::parse_from_str(datetime.trim(), "%d-%b-%Y %H:%M:%S %z") + strtime::parse("%d-%b-%Y %H:%M:%S %z", datetime.trim()) + .and_then(|parsed| parsed.to_timestamp()) + .map(|timestamp| timestamp.as_second()) .map_err(|_| Cow::from(format!("Failed to parse date/time '{}'.", datetime))) - .map(|dt| dt.timestamp()) }) } pub fn parse_date(value: &[u8]) -> Result { + if let Some(timestamp) = parse_date_fast(value.trim_ascii()) { + return Ok(timestamp); + } + + parse_date_slow(value) +} + +#[inline(never)] +fn parse_date_slow(value: &[u8]) -> Result { std::str::from_utf8(value) .map_err(|_| Cow::from("Expected date, found an invalid UTF-8 string.")) .and_then(|date| { - NaiveDate::parse_from_str(date.trim(), "%d-%b-%Y") + strtime::parse("%d-%b-%Y", date.trim()) + .and_then(|parsed| parsed.to_date()) + .and_then(|parsed| Offset::UTC.to_timestamp(parsed.to_datetime(Time::midnight()))) + .map(|timestamp| timestamp.as_second()) .map_err(|_| Cow::from(format!("Failed to parse date '{}'.", date))) - .map(|dt| { - dt.and_hms_opt(0, 0, 0) - .unwrap_or_default() - .and_utc() - .timestamp() - }) }) } @@ -223,8 +372,109 @@ pub fn parse_number(value: &[u8]) -> Result { }) } +#[inline(always)] +fn sequence_number(value: &[u8]) -> Option<(u32, &[u8])> { + let digits = value + .iter() + .position(|ch| !ch.is_ascii_digit()) + .unwrap_or(value.len()); + let (number, tail) = value.split_at(digits); + if number.is_empty() || number.len() > 10 { + return None; + } + let number = number + .iter() + .fold(0u64, |acc, ch| acc * 10 + (ch - b'0') as u64); + (number <= u32::MAX as u64).then_some((number as u32, tail)) +} + +#[inline(always)] +fn sequence_item(value: &[u8]) -> Option<(Sequence, &[u8])> { + match value { + [b'$', tail @ ..] => Some((Sequence::SavedSearch, tail)), + [b'*', b':', b'*', tail @ ..] => Some(( + Sequence::Range { + start: None, + end: None, + }, + tail, + )), + [b'*', b':', tail @ ..] => { + let (end, tail) = sequence_number(tail)?; + Some(( + Sequence::Range { + start: None, + end: Some(end), + }, + tail, + )) + } + [b'*', tail @ ..] => Some(( + Sequence::Range { + start: None, + end: None, + }, + tail, + )), + _ => { + let (start, tail) = sequence_number(value)?; + match tail { + [b':', b'*', tail @ ..] => Some(( + Sequence::Range { + start: Some(start), + end: None, + }, + tail, + )), + [b':', tail @ ..] => { + let (end, tail) = sequence_number(tail)?; + Some(( + Sequence::Range { + start: Some(start), + end: Some(end), + }, + tail, + )) + } + _ => Some((Sequence::Number { value: start }, tail)), + } + } + } +} + +fn parse_sequence_set_fast(value: &[u8]) -> Option { + let (first, tail) = sequence_item(value)?; + let mut rest = match tail { + [] => return Some(first), + [b',', rest @ ..] => rest, + _ => return None, + }; + + let commas = rest.iter().filter(|&&ch| ch == b',').count(); + let mut items = Vec::with_capacity(commas + 2); + items.push(first); + loop { + let (item, tail) = sequence_item(rest)?; + items.push(item); + match tail { + [b',', tail @ ..] => rest = tail, + [] => return Some(Sequence::List { items }), + _ => return None, + } + } +} + pub fn parse_sequence_set(value: &[u8]) -> Result { - let mut sequence_set = Vec::new(); + if let Some(sequence_set) = parse_sequence_set_fast(value) { + return Ok(sequence_set); + } + + parse_sequence_set_slow(value) +} + +#[inline(never)] +fn parse_sequence_set_slow(value: &[u8]) -> Result { + let mut sequence_set = Vec::with_capacity(value.iter().filter(|&&ch| ch == b',').count() + 1); let mut range_start = None; let mut token_start = None; @@ -414,89 +664,3 @@ impl PushUnique for Vec { } } -#[cfg(test)] -mod tests { - use crate::{Command, protocol::Sequence, receiver::CommandParser}; - - #[test] - fn parse_command() { - assert_eq!( - Command::parse(b"GETJMAPACCESS", false), - Some(Command::GetJmapAccess) - ); - assert_eq!(Command::parse(b"NOTACOMMAND", false), None); - } - - #[test] - fn parse_sequence_set() { - for (sequence, expected_result) in [ - ("$", Sequence::SavedSearch), - ( - "*", - Sequence::Range { - start: None, - end: None, - }, - ), - ( - "1,3000:3021", - Sequence::List { - items: vec![ - Sequence::Number { value: 1 }, - Sequence::Range { - start: 3000.into(), - end: 3021.into(), - }, - ], - }, - ), - ( - "2,4:7,9,12:*", - Sequence::List { - items: vec![ - Sequence::Number { value: 2 }, - Sequence::Range { - start: 4.into(), - end: 7.into(), - }, - Sequence::Number { value: 9 }, - Sequence::Range { - start: 12.into(), - end: None, - }, - ], - }, - ), - ( - "*:4,5:7", - Sequence::List { - items: vec![ - Sequence::Range { - start: None, - end: 4.into(), - }, - Sequence::Range { - start: 5.into(), - end: 7.into(), - }, - ], - }, - ), - ( - "2,4,5", - Sequence::List { - items: vec![ - Sequence::Number { value: 2 }, - Sequence::Number { value: 4 }, - Sequence::Number { value: 5 }, - ], - }, - ), - ] { - assert_eq!( - super::parse_sequence_set(sequence.as_bytes()).unwrap(), - expected_result - ); - } - } -} diff --git a/crates/imap-proto/src/parser/quota.rs b/crates/imap-proto/src/parser/quota.rs index 2159a73e..8e9928b1 100644 --- a/crates/imap-proto/src/parser/quota.rs +++ b/crates/imap-proto/src/parser/quota.rs @@ -4,28 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::quota, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; impl Request { pub fn parse_get_quota_root(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(quota::Arguments { - name: utf7_maybe_decode( - self.tokens - .into_iter() - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + name: self + .tokens + .into_iter() + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), @@ -42,7 +37,7 @@ impl Request { .next() .unwrap() .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }), 0 => Err(self.into_error("Missing quota root.")), @@ -51,44 +46,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::quota, receiver::Receiver}; - - #[test] - fn parse_quota() { - let mut receiver = Receiver::new(); - - let (command, arguments) = ( - "A142 GETQUOTAROOT INBOX\r\n", - quota::Arguments { - name: "INBOX".into(), - tag: "A142".into(), - }, - ); - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_get_quota_root(true) - .unwrap(), - arguments - ); - - let (command, arguments) = ( - "A142 GETQUOTA \"my funky mailbox\"\r\n", - quota::Arguments { - name: "my funky mailbox".into(), - tag: "A142".into(), - }, - ); - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_get_quota() - .unwrap(), - arguments - ); - } -} diff --git a/crates/imap-proto/src/parser/rename.rs b/crates/imap-proto/src/parser/rename.rs index 4486c586..94468880 100644 --- a/crates/imap-proto/src/parser/rename.rs +++ b/crates/imap-proto/src/parser/rename.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::rename, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; impl Request { @@ -19,22 +16,16 @@ impl Request { 2 => { let mut tokens = self.tokens.into_iter(); Ok(rename::Arguments { - mailbox_name: utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), - new_mailbox_name: utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + mailbox_name: tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, + new_mailbox_name: tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }) } @@ -45,40 +36,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::rename, receiver::Receiver}; - - #[test] - fn parse_rename() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A142 RENAME \"my funky mailbox\" Private\r\n", - rename::Arguments { - mailbox_name: "my funky mailbox".into(), - new_mailbox_name: "Private".into(), - tag: "A142".into(), - }, - ), - ( - "A142 RENAME {1+}\r\na {1+}\r\nb\r\n", - rename::Arguments { - mailbox_name: "a".into(), - new_mailbox_name: "b".into(), - tag: "A142".into(), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_rename(true) - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/search.rs b/crates/imap-proto/src/parser/search.rs index 3a75e77c..9f6d8c36 100644 --- a/crates/imap-proto/src/parser/search.rs +++ b/crates/imap-proto/src/parser/search.rs @@ -8,18 +8,19 @@ use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; -use compact_str::ToCompactString; use mail_parser::decoders::charsets::DecoderFnc; -use mail_parser::decoders::charsets::map::charset_decoder; +use mail_parser::decoders::charsets::charset_decoder; use crate::Command; use crate::protocol::search::{self, Filter}; use crate::protocol::search::{ModSeqEntry, ResultOption}; use crate::protocol::{Flag, ProtocolVersion}; -use crate::receiver::{Request, Token, bad}; +use crate::receiver::{ArgumentBytes, Request, Token, bad}; use super::{parse_date, parse_number, parse_sequence_set}; +const FILTERS_INIT_LEN: usize = 16; + impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_search(self, version: ProtocolVersion) -> trc::Result { @@ -37,15 +38,15 @@ impl Request { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"return") => { tokens.next(); is_esearch = true; - result_options = parse_result_options(&mut tokens) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + result_options = + parse_result_options(&mut tokens).map_err(|v| bad(self.tag.clone(), v))?; } Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"charset") => { tokens.next(); decoder = charset_decoder( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing charset."))? .unwrap_bytes(), ); } @@ -53,14 +54,10 @@ impl Request { } } - let filter = parse_filters(&mut tokens, decoder) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + let filter = parse_filters(&mut tokens, decoder).map_err(|v| bad(self.tag.clone(), v))?; match filter.len() { - 0 => Err(bad( - self.tag.to_compact_string(), - "No filters found in command.", - )), + 0 => Err(bad(self.tag.clone(), "No filters found in command.")), _ => Ok(search::Arguments { tag: self.tag, result_options, @@ -100,7 +97,7 @@ pub fn parse_filters( tokens: &mut Peekable>, decoder: Option, ) -> super::Result> { - let mut filters = Vec::new(); + let mut filters = Vec::with_capacity(tokens.len().min(FILTERS_INIT_LEN)); let mut filters_len = 0; let mut filters_stack = Vec::new(); let mut operator = Filter::And; @@ -382,7 +379,7 @@ pub fn parse_filters( ) .into()); } - let flag = Flag::parse_imap((param[7..]).to_vec())?; + let flag = Flag::parse_imap(ArgumentBytes::from_slice(¶m[7..]))?; let mod_seq_entry = match tokens.next() { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"all") => { ModSeqEntry::All(flag) @@ -499,7 +496,7 @@ pub fn parse_filters( while let Some((mut prev_filters, prev_operator, prev_filters_len)) = filters_stack.pop() { - if operator == Filter::And && (prev_operator != Filter::Or || filters_len == 1) { + if operator == Filter::And && (prev_operator == Filter::And || filters_len == 1) { prev_filters.extend(filters); filters_len += prev_filters_len; } else { @@ -532,20 +529,22 @@ pub fn decode_argument( if let Some(decoder) = decoder { Ok(decoder(&argument)) } else { - Ok(String::from_utf8(argument).map_err(|_| Cow::from("Invalid UTF-8 argument."))?) + Ok(String::from_utf8(argument.into_vec()) + .map_err(|_| Cow::from("Invalid UTF-8 argument."))?) } } impl ResultOption { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!( + hashify::fnc_map_ignore_case!( value, - "min" => Self::Min, - "max" => Self::Max, - "all" => Self::All, - "count" => Self::Count, - "save" => Self::Save, - "context" => Self::Context, + "min" => Some(Self::Min), + "max" => Some(Self::Max), + "all" => Some(Self::All), + "count" => Some(Self::Count), + "save" => Some(Self::Save), + "context" => Some(Self::Context), + _ => None, ) .ok_or_else(|| { format!( @@ -557,334 +556,3 @@ impl ResultOption { } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::{ - Flag, ProtocolVersion, Sequence, - search::{self, Filter, ModSeqEntry, ResultOption}, - }, - receiver::Receiver, - }; - - #[test] - fn parse_search() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - b"A282 SEARCH RETURN (MIN COUNT) FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n" - .to_vec(), - search::Arguments { - tag: "A282".into(), - result_options: vec![ResultOption::Min, ResultOption::Count], - filter: vec![ - Filter::Flagged, - Filter::Since(760060800), - Filter::Not, - Filter::From("Smith".into()), - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - b"A283 SEARCH RETURN () FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n".to_vec(), - search::Arguments { - tag: "A283".into(), - result_options: vec![], - filter: vec![ - Filter::Flagged, - Filter::Since(760060800), - Filter::Not, - Filter::From("Smith".into()), - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - b"A301 SEARCH $ SMALLER 4096\r\n".to_vec(), - search::Arguments { - tag: "A301".into(), - result_options: vec![], - filter: vec![Filter::seq_saved_search(), Filter::Smaller(4096)], - is_esearch: true, - sort: None, - }, - ), - ( - "P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) TEXT {8+}\r\nмать\r\n" - .as_bytes() - .to_vec(), - search::Arguments { - tag: "P283".into(), - result_options: vec![], - filter: vec![ - Filter::Or, - Filter::seq_saved_search(), - Filter::Sequence( - Sequence::List { - items: vec![ - Sequence::number(1), - Sequence::range(3000.into(), 3021.into()), - ], - }, - false, - ), - Filter::End, - Filter::Text("мать".into()), - ], - is_esearch: true, - sort: None, - }, - ), - ( - b"F282 SEARCH RETURN (SAVE) KEYWORD $Junk\r\n".to_vec(), - search::Arguments { - tag: "F282".into(), - result_options: vec![ResultOption::Save], - filter: vec![Filter::Keyword(Flag::Junk)], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"F282 SEARCH OR OR FROM hello@world.com TO ".to_vec(), - b"test@example.com OR BCC jane@foobar.com ".to_vec(), - b"CC john@doe.com\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "F282".into(), - result_options: vec![], - filter: vec![ - Filter::Or, - Filter::Or, - Filter::From("hello@world.com".into()), - Filter::To("test@example.com".into()), - Filter::End, - Filter::Or, - Filter::Bcc("jane@foobar.com".into()), - Filter::Cc("john@doe.com".into()), - Filter::End, - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"abc SEARCH OR SMALLER 10000 OR ".to_vec(), - b"HEADER Subject \"ravioli festival\" ".to_vec(), - b"HEADER From \"dr. ravioli\"\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "abc".into(), - result_options: vec![], - filter: vec![ - Filter::Or, - Filter::Smaller(10000), - Filter::Or, - Filter::Header("Subject".into(), "ravioli festival".into()), - Filter::Header("From".into(), "dr. ravioli".into()), - Filter::End, - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"abc SEARCH (DELETED SEEN ANSWERED) ".to_vec(), - b"NOT (FROM john TO jane BCC bill) ".to_vec(), - b"(1,30:* UID 1,2,3,4 $)\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "abc".into(), - result_options: vec![], - filter: vec![ - Filter::Deleted, - Filter::Seen, - Filter::Answered, - Filter::Not, - Filter::From("john".into()), - Filter::To("jane".into()), - Filter::Bcc("bill".into()), - Filter::End, - Filter::Sequence( - Sequence::List { - items: vec![Sequence::number(1), Sequence::range(30.into(), None)], - }, - false, - ), - Filter::Sequence( - Sequence::List { - items: vec![ - Sequence::number(1), - Sequence::number(2), - Sequence::number(3), - Sequence::number(4), - ], - }, - true, - ), - Filter::seq_saved_search(), - ], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"abc SEARCH *:* UID *:100,100:* ".to_vec(), - b"(FLAGGED (DRAFT (DELETED (ANSWERED)))) ".to_vec(), - b"OR (SENTON 20-Nov-2022) (LARGER 8196)\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "abc".into(), - result_options: vec![], - filter: vec![ - Filter::seq_range(None, None), - Filter::Sequence( - Sequence::List { - items: vec![ - Sequence::range(None, 100.into()), - Sequence::range(100.into(), None), - ], - }, - true, - ), - Filter::Flagged, - Filter::Draft, - Filter::Deleted, - Filter::Answered, - Filter::Or, - Filter::SentOn(1668902400), - Filter::Larger(8196), - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"abc SEARCH NOT (FROM john OR TO jane CC bill) ".to_vec(), - b"OR (UNDELETED ALL) ($ NOT FLAGGED) ".to_vec(), - b"(((KEYWORD \"tps report\")))\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "abc".into(), - result_options: vec![], - filter: vec![ - Filter::Not, - Filter::From("john".into()), - Filter::Or, - Filter::To("jane".into()), - Filter::Cc("bill".into()), - Filter::End, - Filter::End, - Filter::Or, - Filter::And, - Filter::Undeleted, - Filter::All, - Filter::End, - Filter::And, - Filter::seq_saved_search(), - Filter::Not, - Filter::Flagged, - Filter::End, - Filter::End, - Filter::End, - Filter::Keyword(Flag::Keyword("tps report".into())), - ], - is_esearch: true, - sort: None, - }, - ), - ( - [ - b"B283 SEARCH RETURN (SAVE MIN MAX) CHARSET KOI8-R TEXT ".to_vec(), - b"{11+}\r\n\xf0\xd2\xc9\xd7\xc5\xd4, \xcd\xc9\xd2\r\n".to_vec(), - ] - .concat(), - search::Arguments { - tag: "B283".into(), - result_options: vec![ResultOption::Save, ResultOption::Min, ResultOption::Max], - filter: vec![Filter::Text("Привет, мир".into())], - is_esearch: true, - sort: None, - }, - ), - ( - b"B283 SEARCH CHARSET BIG5 FROM \"\xa7A\xa6n\xa1A\xa5@\xac\xc9\"\r\n".to_vec(), - search::Arguments { - tag: "B283".into(), - result_options: vec![], - filter: vec![Filter::From("你好,世界".into())], - is_esearch: true, - sort: None, - }, - ), - ( - b"a SEARCH MODSEQ \"/flags/\\draft\" all 620162338\r\n".to_vec(), - search::Arguments { - tag: "a".into(), - result_options: vec![], - filter: vec![Filter::ModSeq((620162338, ModSeqEntry::All(Flag::Draft)))], - is_esearch: true, - sort: None, - }, - ), - ( - b"t SEARCH OR NOT MODSEQ 720162338 LARGER 50000\r\n".to_vec(), - search::Arguments { - tag: "t".into(), - result_options: vec![], - filter: vec![ - Filter::Or, - Filter::Not, - Filter::ModSeq((720162338, ModSeqEntry::None)), - Filter::End, - Filter::Larger(50000), - Filter::End, - ], - is_esearch: true, - sort: None, - }, - ), - ( - b"5 UID SEARCH BEFORE 1-Dec-2023\r\n".to_vec(), - search::Arguments { - tag: "5".into(), - result_options: vec![], - filter: vec![Filter::Before(1701388800)], - is_esearch: true, - sort: None, - }, - ), - ] { - let command_str = String::from_utf8_lossy(&command).into_owned(); - assert_eq!( - receiver - .parse(&mut command.iter()) - .unwrap() - .parse_search(ProtocolVersion::Rev2) - .expect(&command_str), - arguments, - "{}", - command_str - ); - } - } -} diff --git a/crates/imap-proto/src/parser/select.rs b/crates/imap-proto/src/parser/select.rs index 0e64fc78..ed9ead06 100644 --- a/crates/imap-proto/src/parser/select.rs +++ b/crates/imap-proto/src/parser/select.rs @@ -4,9 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString, format_compact}; -use std::str::FromStr; -use types::id::Id; +use compact_str::format_compact; use crate::{ Command, @@ -15,7 +13,6 @@ use crate::{ select::{self, QResync}, }, receiver::{Request, Token, bad}, - utf7::utf7_maybe_decode, }; use super::{parse_number, parse_sequence_set}; @@ -26,14 +23,11 @@ impl Request { let mut tokens = self.tokens.into_iter().peekable(); // Mailbox name - let mailbox_name = utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ); + let mailbox_name = tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?; // CONDSTORE parameters let mut condstore = false; @@ -51,10 +45,7 @@ impl Request { .next() .is_none_or(|token| !token.is_parenthesis_open()) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Expected '(' after 'QRESYNC'.", - )); + return Err(bad(self.tag, "Expected '(' after 'QRESYNC'.")); } let uid_validity = parse_number::( @@ -62,32 +53,32 @@ impl Request { .next() .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Missing uidvalidity parameter for QRESYNC.", ) })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let modseq = parse_number::( &tokens .next() .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Missing modseq parameter for QRESYNC.", ) })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let mut known_uids = None; let mut seq_match = None; let has_seq_match = match tokens.peek() { Some(Token::Argument(value)) => { known_uids = parse_sequence_set(value) - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? .into(); tokens.next(); if matches!(tokens.peek(), Some(Token::ParenthesisOpen)) { @@ -111,34 +102,31 @@ impl Request { .next() .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Missing known-sequence-set parameter for QRESYNC.", ) })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, parse_sequence_set( &tokens .next() .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Missing known-uid-set parameter for QRESYNC.", ) })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + .map_err(|v| bad(self.tag.clone(), v))?, )); if tokens .next() .is_none_or(|token| !token.is_parenthesis_close()) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Missing ')' for 'QRESYNC'.", - )); + return Err(bad(self.tag, "Missing ')' for 'QRESYNC'.")); } } @@ -146,10 +134,7 @@ impl Request { .next() .is_none_or(|token| !token.is_parenthesis_close()) { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Missing ')' for 'QRESYNC'.", - )); + return Err(bad(self.tag, "Missing ')' for 'QRESYNC'.")); } qresync = QResync { @@ -162,6 +147,7 @@ impl Request { } Token::Argument(param) if param.eq_ignore_ascii_case(b"OBJECTID") => { let mut oid = ObjectId::default(); + let mut is_matchable = true; if matches!(tokens.peek(), Some(Token::ParenthesisOpen)) { tokens.next(); while let Some(token) = tokens.next() { @@ -172,17 +158,21 @@ impl Request { .next() .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected value after OBJECTID key.", ) })? .unwrap_bytes(); - let id = std::str::from_utf8(&value) - .ok() - .and_then(|v| Id::from_str(v).ok()); + let id = ObjectId::decode_id(&value); hashify::fnc_map_ignore_case!(key.as_slice(), - "MAILBOXID" => { oid.mailbox_id = id; }, - "ACCOUNTID" => { oid.account_id = id; }, + "MAILBOXID" => { + is_matchable &= id.is_some(); + oid.mailbox_id = id; + }, + "ACCOUNTID" => { + is_matchable &= id.is_some(); + oid.account_id = id; + }, "EMAILID" => { oid.email_id = id; }, "THREADID" => { oid.thread_id = id; }, _ => {} @@ -190,7 +180,7 @@ impl Request { } _ => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!( "Unexpected value '{}'.", token @@ -200,6 +190,9 @@ impl Request { } } } + if !is_matchable { + oid.mailbox_id = None; + } objectid = Some(oid); } Token::ParenthesisClose => { @@ -207,7 +200,7 @@ impl Request { } _ => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!("Unexpected value '{}'.", token), )); } @@ -216,7 +209,7 @@ impl Request { } Some(token) => { return Err(bad( - CompactString::from_string_buffer(self.tag), + self.tag, format_compact!("Unexpected value '{}'.", token), )); } @@ -236,276 +229,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{ - protocol::{ - ObjectId, Sequence, - select::{self, QResync}, - }, - receiver::Receiver, - }; - use std::str::FromStr; - use types::id::Id; - - #[test] - fn parse_select_objectid() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A1 SELECT \"foo\" (OBJECTID)\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A1".into(), - condstore: false, - qresync: None, - objectid: Some(ObjectId::default()), - }, - ), - ( - "A2 SELECT \"foo\" (OBJECTID (MAILBOXID abc ACCOUNTID xyz))\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A2".into(), - condstore: false, - qresync: None, - objectid: Some(ObjectId { - mailbox_id: Some(Id::from_str("abc").unwrap()), - account_id: Some(Id::from_str("xyz").unwrap()), - ..Default::default() - }), - }, - ), - ( - "A3 EXAMINE \"foo\" (OBJECTID (MAILBOXID abc))\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A3".into(), - condstore: false, - qresync: None, - objectid: Some(ObjectId { - mailbox_id: Some(Id::from_str("abc").unwrap()), - ..Default::default() - }), - }, - ), - ( - "A4 SELECT \"foo\" (CONDSTORE OBJECTID)\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A4".into(), - condstore: true, - qresync: None, - objectid: Some(ObjectId::default()), - }, - ), - ( - "A5 SELECT \"foo\" (OBJECTID (FOOBAR baz MAILBOXID abc))\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A5".into(), - condstore: false, - qresync: None, - objectid: Some(ObjectId { - mailbox_id: Some(Id::from_str("abc").unwrap()), - ..Default::default() - }), - }, - ), - ( - "A6 SELECT \"foo\" (OBJECTID (MAILBOXID 456))\r\n", - select::Arguments { - mailbox_name: "foo".into(), - tag: "A6".into(), - condstore: false, - qresync: None, - objectid: Some(ObjectId::default()), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_select(true) - .unwrap(), - arguments, - "Failed to parse {command}" - ); - } - } - - #[test] - fn parse_select() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A142 SELECT INBOX\r\n", - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "A142".into(), - condstore: false, - qresync: None, - objectid: None, - }, - ), - ( - "A142 SELECT \"my funky mailbox\"\r\n", - select::Arguments { - mailbox_name: "my funky mailbox".into(), - tag: "A142".into(), - condstore: false, - qresync: None, - objectid: None, - }, - ), - ( - "A142 SELECT INBOX (CONDSTORE)\r\n", - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "A142".into(), - condstore: true, - qresync: None, - objectid: None, - }, - ), - ( - "A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198))\r\n", - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "A142".into(), - condstore: false, - qresync: QResync { - uid_validity: 3857529045, - modseq: 20010715194032001, - known_uids: Some(Sequence::Range { - start: Some(1), - end: Some(198), - }), - seq_match: None, - } - .into(), - objectid: None, - }, - ), - ( - concat!( - "A03 SELECT INBOX (QRESYNC (67890007 90060115194045000 ", - "41:211,214:541) CONDSTORE)\r\n" - ), - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "A03".into(), - condstore: true, - qresync: QResync { - uid_validity: 67890007, - modseq: 90060115194045000, - known_uids: Some(Sequence::List { - items: vec![ - Sequence::Range { - start: Some(41), - end: Some(211), - }, - Sequence::Range { - start: Some(214), - end: Some(541), - }, - ], - }), - seq_match: None, - } - .into(), - objectid: None, - }, - ), - ( - concat!( - "B04 SELECT INBOX (QRESYNC (67890007 ", - "90060115194045000 1:29997 (5000,7500,9000,9990:9999 15000,", - "22500,27000,29970,29973,29976,29979,29982,29985,29988,29991,", - "29994,29997)))\r\n" - ), - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "B04".into(), - condstore: false, - qresync: QResync { - uid_validity: 67890007, - modseq: 90060115194045000, - known_uids: Some(Sequence::Range { - start: Some(1), - end: Some(29997), - }), - seq_match: Some(( - Sequence::List { - items: vec![ - Sequence::Number { value: 5000 }, - Sequence::Number { value: 7500 }, - Sequence::Number { value: 9000 }, - Sequence::Range { - start: Some(9990), - end: Some(9999), - }, - ], - }, - Sequence::List { - items: vec![ - Sequence::Number { value: 15000 }, - Sequence::Number { value: 22500 }, - Sequence::Number { value: 27000 }, - Sequence::Number { value: 29970 }, - Sequence::Number { value: 29973 }, - Sequence::Number { value: 29976 }, - Sequence::Number { value: 29979 }, - Sequence::Number { value: 29982 }, - Sequence::Number { value: 29985 }, - Sequence::Number { value: 29988 }, - Sequence::Number { value: 29991 }, - Sequence::Number { value: 29994 }, - Sequence::Number { value: 29997 }, - ], - }, - )), - } - .into(), - objectid: None, - }, - ), - ( - "A12 SELECT \"INBOX\" (QRESYNC (1693237464 16582))\r\n", - select::Arguments { - mailbox_name: "INBOX".into(), - tag: "A12".into(), - condstore: false, - qresync: QResync { - uid_validity: 1693237464, - modseq: 16582, - known_uids: None, - seq_match: None, - } - .into(), - objectid: None, - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap_or_else(|err| panic!( - "Failed to parse command '{}': {:?}", - command, err - )) - .parse_select(true) - .unwrap_or_else(|err| panic!( - "Failed to parse command '{}': {:?}", - command, err - )), - arguments, - "Failed to parse {}", - command - ); - } - } -} diff --git a/crates/imap-proto/src/parser/sort.rs b/crates/imap-proto/src/parser/sort.rs index d3902ce9..6b4984dc 100644 --- a/crates/imap-proto/src/parser/sort.rs +++ b/crates/imap-proto/src/parser/sort.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; -use mail_parser::decoders::charsets::map::charset_decoder; +use mail_parser::decoders::charsets::charset_decoder; use crate::{ Command, @@ -29,8 +28,7 @@ impl Request { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"return") => { tokens.next(); ( - parse_result_options(&mut tokens) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + parse_result_options(&mut tokens).map_err(|v| bad(self.tag.clone(), v))?, true, ) } @@ -42,7 +40,7 @@ impl Request { .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected sort criteria between parentheses.", )); } @@ -56,40 +54,32 @@ impl Request { is_ascending = false; } else { sort.push(Comparator { - sort: Sort::parse(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, + sort: Sort::parse(&value).map_err(|v| bad(self.tag.clone(), v))?, ascending: is_ascending, }); is_ascending = true; } } _ => { - return Err(bad( - self.tag.to_compact_string(), - "Invalid result option argument.", - )); + return Err(bad(self.tag.clone(), "Invalid result option argument.")); } } } if sort.is_empty() { - return Err(bad(self.tag.to_compact_string(), "Missing sort criteria.")); + return Err(bad(self.tag.clone(), "Missing sort criteria.")); } let decoder = charset_decoder( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing charset."))? .unwrap_bytes(), ); - let filter = parse_filters(&mut tokens, decoder) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + let filter = parse_filters(&mut tokens, decoder).map_err(|v| bad(self.tag.clone(), v))?; match filter.len() { - 0 => Err(bad( - self.tag.to_compact_string(), - "No filters found in command.", - )), + 0 => Err(bad(self.tag.clone(), "No filters found in command.")), _ => Ok(Arguments { sort: sort.into(), result_options, @@ -103,140 +93,19 @@ impl Request { impl Sort { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "ARRIVAL" => Self::Arrival, - "CC" => Self::Cc, - "DATE" => Self::Date, - "FROM" => Self::From, - "SIZE" => Self::Size, - "SUBJECT" => Self::Subject, - "TO" => Self::To, - "DISPLAYFROM" => Self::DisplayFrom, - "DISPLAYTO" => Self::DisplayTo, + hashify::fnc_map_ignore_case!(value, + "ARRIVAL" => Some(Self::Arrival), + "CC" => Some(Self::Cc), + "DATE" => Some(Self::Date), + "FROM" => Some(Self::From), + "SIZE" => Some(Self::Size), + "SUBJECT" => Some(Self::Subject), + "TO" => Some(Self::To), + "DISPLAYFROM" => Some(Self::DisplayFrom), + "DISPLAYTO" => Some(Self::DisplayTo), + _ => None, ) .ok_or_else(|| format!("Invalid sort criteria {:?}", String::from_utf8_lossy(value)).into()) } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::{ - Flag, - search::{Arguments, Comparator, Filter, ResultOption, Sort}, - }, - receiver::Receiver, - }; - - #[test] - fn parse_sort() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - b"A282 SORT (SUBJECT) UTF-8 SINCE 1-Feb-1994\r\n".to_vec(), - Arguments { - sort: vec![Comparator { - sort: Sort::Subject, - ascending: true, - }] - .into(), - filter: vec![Filter::Since(760060800)], - result_options: Vec::new(), - is_esearch: false, - tag: "A282".into(), - }, - ), - ( - b"A283 SORT (SUBJECT REVERSE DATE) UTF-8 ALL\r\n".to_vec(), - Arguments { - sort: vec![ - Comparator { - sort: Sort::Subject, - ascending: true, - }, - Comparator { - sort: Sort::Date, - ascending: false, - }, - ] - .into(), - filter: vec![Filter::All], - result_options: Vec::new(), - is_esearch: false, - tag: "A283".into(), - }, - ), - ( - b"A284 SORT (SUBJECT) US-ASCII TEXT \"not in mailbox\"\r\n".to_vec(), - Arguments { - sort: vec![Comparator { - sort: Sort::Subject, - ascending: true, - }] - .into(), - filter: vec![Filter::Text("not in mailbox".into())], - result_options: Vec::new(), - is_esearch: false, - tag: "A284".into(), - }, - ), - ( - [ - b"A284 SORT (REVERSE ARRIVAL FROM) iso-8859-6 SUBJECT ".to_vec(), - b"\"\xe5\xd1\xcd\xc8\xc7 \xc8\xc7\xe4\xd9\xc7\xe4\xe5\"\r\n".to_vec(), - ] - .concat(), - Arguments { - sort: vec![ - Comparator { - sort: Sort::Arrival, - ascending: false, - }, - Comparator { - sort: Sort::From, - ascending: true, - }, - ] - .into(), - filter: vec![Filter::Subject("مرحبا بالعالم".into())], - result_options: Vec::new(), - is_esearch: false, - tag: "A284".into(), - }, - ), - ( - [ - b"E01 UID SORT RETURN (COUNT) (REVERSE DATE) ".to_vec(), - b"UTF-8 UNDELETED UNKEYWORD $Junk\r\n".to_vec(), - ] - .concat(), - Arguments { - sort: vec![Comparator { - sort: Sort::Date, - ascending: false, - }] - .into(), - filter: vec![Filter::Undeleted, Filter::Unkeyword(Flag::Junk)], - result_options: vec![ResultOption::Count], - is_esearch: true, - tag: "E01".into(), - }, - ), - ] { - let command_str = String::from_utf8_lossy(&command).into_owned(); - - assert_eq!( - receiver - .parse(&mut command.iter()) - .unwrap() - .parse_sort() - .expect(&command_str), - arguments, - "{}", - command_str - ); - } - } -} diff --git a/crates/imap-proto/src/parser/status.rs b/crates/imap-proto/src/parser/status.rs index 2f7183c1..7c2e2dd5 100644 --- a/crates/imap-proto/src/parser/status.rs +++ b/crates/imap-proto/src/parser/status.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString}; - use crate::Command; use crate::protocol::status; use crate::protocol::status::Status; use crate::receiver::{Request, Token, bad}; -use crate::utf7::utf7_maybe_decode; impl Request { pub fn parse_status(self, is_utf8: bool) -> trc::Result { @@ -18,14 +15,11 @@ impl Request { 0..=3 => Err(self.into_error("Missing arguments.")), len => { let mut tokens = self.tokens.into_iter(); - let mailbox_name = utf7_maybe_decode( - tokens - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ); + let mailbox_name = tokens + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?; let mut items = Vec::with_capacity(len - 2); if tokens @@ -33,7 +27,7 @@ impl Request { .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected parenthesis after mailbox name.", )); } @@ -43,14 +37,12 @@ impl Request { match token { Token::ParenthesisClose => break, Token::Argument(value) => { - items.push( - Status::parse(&value) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); + items + .push(Status::parse(&value).map_err(|v| bad(self.tag.clone(), v))?); } _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), "Invalid status return option argument.", )); } @@ -64,10 +56,7 @@ impl Request { items, }) } else { - Err(bad( - CompactString::from_string_buffer(self.tag), - "At least one status item is required.", - )) + Err(bad(self.tag, "At least one status item is required.")) } } } @@ -76,18 +65,19 @@ impl Request { impl Status { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "MESSAGES" => Self::Messages, - "UIDNEXT" => Self::UidNext, - "UIDVALIDITY" => Self::UidValidity, - "UNSEEN" => Self::Unseen, - "DELETED" => Self::Deleted, - "SIZE" => Self::Size, - "HIGHESTMODSEQ" => Self::HighestModSeq, - "OBJECTID" => Self::ObjectId, - "RECENT" => Self::Recent, - "DELETED-STORAGE" => Self::DeletedStorage + hashify::map_ignore_case!(value, Status, + "MESSAGES" => Status::Messages, + "UIDNEXT" => Status::UidNext, + "UIDVALIDITY" => Status::UidValidity, + "UNSEEN" => Status::Unseen, + "DELETED" => Status::Deleted, + "SIZE" => Status::Size, + "HIGHESTMODSEQ" => Status::HighestModSeq, + "OBJECTID" => Status::ObjectId, + "RECENT" => Status::Recent, + "DELETED-STORAGE" => Status::DeletedStorage ) + .copied() .ok_or_else(|| { format!( "Invalid status option '{}'.", @@ -98,53 +88,3 @@ impl Status { } } -#[cfg(test)] -mod tests { - use crate::{protocol::status, receiver::Receiver}; - - #[test] - fn parse_status() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A042 STATUS blurdybloop (UIDNEXT MESSAGES)\r\n", - status::Arguments { - tag: "A042".into(), - mailbox_name: "blurdybloop".into(), - items: vec![status::Status::UidNext, status::Status::Messages], - }, - ), - ( - "A043 STATUS foo (OBJECTID)\r\n", - status::Arguments { - tag: "A043".into(), - mailbox_name: "foo".into(), - items: vec![status::Status::ObjectId], - }, - ), - ( - "A044 STATUS foo (MESSAGES OBJECTID UIDVALIDITY)\r\n", - status::Arguments { - tag: "A044".into(), - mailbox_name: "foo".into(), - items: vec![ - status::Status::Messages, - status::Status::ObjectId, - status::Status::UidValidity, - ], - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_status(true) - .unwrap(), - arguments, - "Failed to parse {command}" - ); - } - } -} diff --git a/crates/imap-proto/src/parser/store.rs b/crates/imap-proto/src/parser/store.rs index e9304f66..9fdf23b8 100644 --- a/crates/imap-proto/src/parser/store.rs +++ b/crates/imap-proto/src/parser/store.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString, format_compact}; +use compact_str::format_compact; use crate::{ Command, @@ -25,10 +25,10 @@ impl Request { let sequence_set = parse_sequence_set( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing sequence set."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let mut unchanged_since = None; // CONDSTORE parameters @@ -41,14 +41,11 @@ impl Request { &tokens .next() .ok_or_else(|| { - bad( - self.tag.to_compact_string(), - "Missing UNCHANGEDSINCE parameter.", - ) + bad(self.tag.clone(), "Missing UNCHANGEDSINCE parameter.") })? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))? + .map_err(|v| bad(self.tag.clone(), v))? .into(); } Token::ParenthesisClose => { @@ -56,7 +53,7 @@ impl Request { } _ => { return Err(bad( - self.tag.to_compact_string(), + self.tag.clone(), format_compact!("Unsupported parameter '{}'.", token), )); } @@ -67,24 +64,20 @@ impl Request { // Operation let operation = tokens .next() - .ok_or_else(|| { - bad( - self.tag.to_compact_string(), - "Missing message data item name.", - ) - })? + .ok_or_else(|| bad(self.tag.clone(), "Missing message data item name."))? .unwrap_bytes(); - let (is_silent, operation) = hashify::tiny_map_ignore_case!(operation.as_slice(), - "FLAGS" => (false, Operation::Set), - "FLAGS.SILENT" => (true, Operation::Set), - "+FLAGS" => (false, Operation::Add), - "+FLAGS.SILENT" => (true, Operation::Add), - "-FLAGS" => (false, Operation::Clear), - "-FLAGS.SILENT" => (true, Operation::Clear), + let (is_silent, operation) = hashify::fnc_map_ignore_case!(operation.as_slice(), + "FLAGS" => Some((false, Operation::Set)), + "FLAGS.SILENT" => Some((true, Operation::Set)), + "+FLAGS" => Some((false, Operation::Add)), + "+FLAGS.SILENT" => Some((true, Operation::Add)), + "-FLAGS" => Some((false, Operation::Clear)), + "-FLAGS.SILENT" => Some((true, Operation::Clear)), + _ => None, ) .ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), format_compact!( "Unsupported message data item name: {:?}", String::from_utf8_lossy(&operation) @@ -96,124 +89,43 @@ impl Request { let mut keywords = Vec::new(); match tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing flags to set."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing flags to set."))? { Token::ParenthesisOpen => { for token in tokens { match token { Token::Argument(flag) => { - keywords.push( - Flag::parse_imap(flag) - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); + let flag = + Flag::parse_imap(flag).map_err(|v| bad(self.tag.clone(), v))?; + if !keywords.contains(&flag) { + keywords.push(flag); + } } Token::ParenthesisClose => { break; } _ => { - return Err(bad(self.tag.to_compact_string(), "Unsupported flag.")); + return Err(bad(self.tag.clone(), "Unsupported flag.")); } } } } Token::Argument(flag) => { - keywords.push( - Flag::parse_imap(flag).map_err(|v| bad(self.tag.to_compact_string(), v))?, - ); + keywords.push(Flag::parse_imap(flag).map_err(|v| bad(self.tag.clone(), v))?); } _ => { - return Err(bad( - CompactString::from_string_buffer(self.tag), - "Invalid flags parameter.", - )); + return Err(bad(self.tag, "Invalid flags parameter.")); } } - if !keywords.is_empty() || operation == Operation::Set { - Ok(store::Arguments { - tag: self.tag, - sequence_set, - operation, - is_silent, - keywords, - unchanged_since, - }) - } else { - Err(bad(self.tag.to_compact_string(), "Missing flags to set.")) - } + Ok(store::Arguments { + tag: self.tag, + sequence_set, + operation, + is_silent, + keywords, + unchanged_since, + }) } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::{ - Flag, Sequence, - store::{self, Operation}, - }, - receiver::Receiver, - }; - - #[test] - fn parse_store() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A003 STORE 2:4 +FLAGS (\\Deleted)\r\n", - store::Arguments { - sequence_set: Sequence::Range { - start: 2.into(), - end: 4.into(), - }, - is_silent: false, - operation: Operation::Add, - keywords: vec![Flag::Deleted], - tag: "A003".into(), - unchanged_since: None, - }, - ), - ( - "A004 STORE *:100 -FLAGS.SILENT ($Phishing $Junk)\r\n", - store::Arguments { - sequence_set: Sequence::Range { - start: None, - end: 100.into(), - }, - is_silent: true, - operation: Operation::Clear, - keywords: vec![Flag::Phishing, Flag::Junk], - tag: "A004".into(), - unchanged_since: None, - }, - ), - ( - "d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) +FLAGS.SILENT \\Deleted\r\n", - store::Arguments { - sequence_set: Sequence::List { - items: vec![ - Sequence::Number { value: 7 }, - Sequence::Number { value: 5 }, - Sequence::Number { value: 9 }, - ], - }, - is_silent: true, - operation: Operation::Add, - keywords: vec![Flag::Deleted], - tag: "d105".into(), - unchanged_since: Some(320162338), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_store() - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/subscribe.rs b/crates/imap-proto/src/parser/subscribe.rs index 225632e3..711e1bbc 100644 --- a/crates/imap-proto/src/parser/subscribe.rs +++ b/crates/imap-proto/src/parser/subscribe.rs @@ -4,28 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; - use crate::{ Command, protocol::subscribe, receiver::{Request, bad}, - utf7::utf7_maybe_decode, }; impl Request { pub fn parse_subscribe(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(subscribe::Arguments { - mailbox_name: utf7_maybe_decode( - self.tokens - .into_iter() - .next() - .unwrap() - .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?, - is_utf8, - ), + mailbox_name: self + .tokens + .into_iter() + .next() + .unwrap() + .unwrap_mailbox_name(is_utf8) + .map_err(|v| bad(self.tag.clone(), v))?, tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), @@ -34,38 +29,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::subscribe, receiver::Receiver}; - - #[test] - fn parse_subscribe() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A142 SUBSCRIBE #news.comp.mail.mime\r\n", - subscribe::Arguments { - mailbox_name: "#news.comp.mail.mime".into(), - tag: "A142".into(), - }, - ), - ( - "A142 SUBSCRIBE \"#news.comp.mail.mime\"\r\n", - subscribe::Arguments { - mailbox_name: "#news.comp.mail.mime".into(), - tag: "A142".into(), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_subscribe(true) - .unwrap(), - arguments - ); - } - } -} diff --git a/crates/imap-proto/src/parser/thread.rs b/crates/imap-proto/src/parser/thread.rs index e4978314..a6b4399e 100644 --- a/crates/imap-proto/src/parser/thread.rs +++ b/crates/imap-proto/src/parser/thread.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::ToCompactString; -use mail_parser::decoders::charsets::map::charset_decoder; +use mail_parser::decoders::charsets::charset_decoder; use crate::{ Command, @@ -26,25 +25,21 @@ impl Request { let algorithm = Algorithm::parse( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing threading algorithm."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing threading algorithm."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let decoder = charset_decoder( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing charset."))? .unwrap_bytes(), ); - let filter = parse_filters(&mut tokens, decoder) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + let filter = parse_filters(&mut tokens, decoder).map_err(|v| bad(self.tag.clone(), v))?; match filter.len() { - 0 => Err(bad( - self.tag.to_compact_string(), - "No filters found in command.", - )), + 0 => Err(bad(self.tag.clone(), "No filters found in command.")), _ => Ok(thread::Arguments { algorithm, filter, @@ -56,9 +51,10 @@ impl Request { impl Algorithm { pub fn parse(value: &[u8]) -> super::Result { - hashify::tiny_map_ignore_case!(value, - "ORDEREDSUBJECT" => Self::OrderedSubject, - "REFERENCES" => Self::References, + hashify::fnc_map_ignore_case!(value, + "ORDEREDSUBJECT" => Some(Self::OrderedSubject), + "REFERENCES" => Some(Self::References), + _ => None, ) .ok_or_else(|| { format!( @@ -70,51 +66,3 @@ impl Algorithm { } } -#[cfg(test)] -mod tests { - - use crate::{ - protocol::{ - search::Filter, - thread::{self, Algorithm}, - }, - receiver::Receiver, - }; - - #[test] - fn parse_thread() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - b"A283 THREAD ORDEREDSUBJECT UTF-8 SINCE 5-MAR-2000\r\n".to_vec(), - thread::Arguments { - algorithm: Algorithm::OrderedSubject, - filter: vec![Filter::Since(952214400)], - tag: "A283".into(), - }, - ), - ( - b"A284 THREAD REFERENCES US-ASCII TEXT \"gewp\"\r\n".to_vec(), - thread::Arguments { - algorithm: Algorithm::References, - filter: vec![Filter::Text("gewp".into())], - tag: "A284".into(), - }, - ), - ] { - let command_str = String::from_utf8_lossy(&command).into_owned(); - - assert_eq!( - receiver - .parse(&mut command.iter()) - .unwrap() - .parse_thread() - .expect(&command_str), - arguments, - "{}", - command_str - ); - } - } -} diff --git a/crates/imap-proto/src/parser/uidbatches.rs b/crates/imap-proto/src/parser/uidbatches.rs index 3d5f1618..7129fbf7 100644 --- a/crates/imap-proto/src/parser/uidbatches.rs +++ b/crates/imap-proto/src/parser/uidbatches.rs @@ -9,7 +9,6 @@ use crate::{ protocol::uidbatches, receiver::{Request, bad}, }; -use compact_str::ToCompactString; use super::parse_number; @@ -19,39 +18,33 @@ impl Request { let batch_size = parse_number::( &tokens .next() - .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing batch size."))? + .ok_or_else(|| bad(self.tag.clone(), "Missing batch size."))? .unwrap_bytes(), ) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; if batch_size == 0 { - return Err(bad( - self.tag.to_compact_string(), - "Batch size cannot be zero.", - )); + return Err(bad(self.tag.clone(), "Batch size cannot be zero.")); } let batch_range = match tokens.next() { Some(token) => { let token = token .unwrap_string() - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let (from, to) = token.split_once(':').ok_or_else(|| { bad( - self.tag.to_compact_string(), + self.tag.clone(), "Expected a batch range in the form 'from:to'.", ) })?; let from = parse_number::(from.trim().as_bytes()) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; let to = parse_number::(to.trim().as_bytes()) - .map_err(|v| bad(self.tag.to_compact_string(), v))?; + .map_err(|v| bad(self.tag.clone(), v))?; if from == 0 || to == 0 { - return Err(bad( - self.tag.to_compact_string(), - "Batch numbers start at one.", - )); + return Err(bad(self.tag.clone(), "Batch numbers start at one.")); } Some((from, to)) @@ -60,10 +53,7 @@ impl Request { }; if tokens.next().is_some() { - return Err(bad( - self.tag.to_compact_string(), - "Too many arguments for UIDBATCHES.", - )); + return Err(bad(self.tag.clone(), "Too many arguments for UIDBATCHES.")); } Ok(uidbatches::Arguments { @@ -74,59 +64,3 @@ impl Request { } } -#[cfg(test)] -mod tests { - use crate::{protocol::uidbatches, receiver::Receiver}; - - #[test] - fn parse_uidbatches() { - let mut receiver = Receiver::new(); - - for (command, arguments) in [ - ( - "A143 UIDBATCHES 2000\r\n", - uidbatches::Arguments { - tag: "A143".into(), - batch_size: 2000, - batch_range: None, - }, - ), - ( - "A302 UIDBATCHES 2000 10:20\r\n", - uidbatches::Arguments { - tag: "A302".into(), - batch_size: 2000, - batch_range: Some((10, 20)), - }, - ), - ] { - assert_eq!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_uidbatches() - .unwrap(), - arguments, - "Failed to parse {command}" - ); - } - - for command in [ - "A1 UIDBATCHES\r\n", - "A2 UIDBATCHES abc\r\n", - "A3 UIDBATCHES 2000 10\r\n", - "A4 UIDBATCHES 0\r\n", - "A5 UIDBATCHES 2000 0:20\r\n", - "A6 UIDBATCHES 2000 1:2 junk\r\n", - ] { - assert!( - receiver - .parse(&mut command.as_bytes().iter()) - .unwrap() - .parse_uidbatches() - .is_err(), - "Expected an error for {command}" - ); - } - } -} diff --git a/crates/imap-proto/src/protocol/acl.rs b/crates/imap-proto/src/protocol/acl.rs index 978755b9..b097f7a8 100644 --- a/crates/imap-proto/src/protocol/acl.rs +++ b/crates/imap-proto/src/protocol/acl.rs @@ -4,40 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -/* +use compact_str::CompactString; - l - lookup (mailbox is visible to LIST/LSUB commands, SUBSCRIBE - mailbox) - r - read (SELECT the mailbox, perform STATUS) - s - keep seen/unseen information across sessions (set or clear - \SEEN flag via STORE, also set \SEEN during APPEND/COPY/ - FETCH BODY[...]) - w - write (set or clear flags other than \SEEN and \DELETED via - STORE, also set them during APPEND/COPY) - i - insert (perform APPEND, COPY into mailbox) - p - post (send mail to submission address for mailbox, - not enforced by IMAP4 itself) - k - create mailboxes (CREATE new sub-mailboxes in any - implementation-defined hierarchy, parent mailbox for the new - mailbox name in RENAME) - x - delete mailbox (DELETE mailbox, old mailbox name in RENAME) - t - delete messages (set or clear \DELETED flag via STORE, set - \DELETED flag during APPEND/COPY) - e - perform EXPUNGE and expunge as a part of CLOSE - a - administer (perform SETACL/DELETEACL/GETACL/LISTRIGHTS) - - // RFC2086 - c - create (CREATE new sub-mailboxes in any implementation-defined - hierarchy) - d - delete (STORE DELETED flag, perform EXPUNGE) - -*/ - -use types::acl::Acl; - -use super::quoted_string; -use crate::utf7::utf7_encode; +use super::{quoted_mailbox_name, quoted_string}; use std::fmt::Display; +use types::acl::Acl; +use utils::map::bitmap::Bitmap; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Rights { @@ -51,9 +23,25 @@ pub enum Rights { DeleteMailbox, DeleteMessages, Expunge, + Annotate, Administer, } +const ACL_RIGHTS: [(&[Acl], Rights); 12] = [ + (&[Acl::Read], Rights::Lookup), + (&[Acl::ReadItems], Rights::Read), + (&[Acl::ModifyItems], Rights::Seen), + (&[Acl::ModifyItems], Rights::Write), + (&[Acl::AddItems], Rights::Insert), + (&[Acl::Submit], Rights::Post), + (&[Acl::CreateChild], Rights::CreateMailbox), + (&[Acl::Delete, Acl::Modify], Rights::DeleteMailbox), + (&[Acl::RemoveItems], Rights::DeleteMessages), + (&[Acl::RemoveItems], Rights::Expunge), + (&[Acl::ModifyMetadata], Rights::Annotate), + (&[Acl::Share], Rights::Administer), +]; + #[derive(Debug, PartialEq, Eq, Clone)] pub struct ModRights { pub op: ModRightsOp, @@ -69,28 +57,28 @@ pub enum ModRightsOp { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, - pub identifier: Option, + pub tag: CompactString, + pub mailbox_name: CompactString, + pub identifier: Option, pub mod_rights: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct GetAclResponse { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub permissions: Vec<(String, Vec)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListRightsResponse { - pub mailbox_name: String, - pub identifier: String, + pub mailbox_name: CompactString, + pub identifier: CompactString, pub permissions: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MyRightsResponse { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub rights: Vec, } @@ -98,15 +86,11 @@ impl GetAclResponse { pub fn into_bytes(self, is_utf8: bool) -> Vec { let mut buf = Vec::with_capacity(self.mailbox_name.len() + 10 * self.permissions.len() * 5); buf.extend_from_slice(b"* ACL "); - if is_utf8 { - quoted_string(&mut buf, &self.mailbox_name); - } else { - quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); - } + quoted_mailbox_name(&mut buf, &self.mailbox_name, is_utf8); for (identifier, rights) in self.permissions { - buf.extend_from_slice(b" "); + buf.push(b' '); quoted_string(&mut buf, &identifier); - buf.extend_from_slice(b" "); + buf.push(b' '); for right in rights { buf.push(right.to_char()); @@ -123,15 +107,11 @@ impl ListRightsResponse { self.mailbox_name.len() + self.identifier.len() + 10 * self.permissions.len() * 5, ); buf.extend_from_slice(b"* LISTRIGHTS "); - if is_utf8 { - quoted_string(&mut buf, &self.mailbox_name); - } else { - quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); - } - buf.extend_from_slice(b" "); + quoted_mailbox_name(&mut buf, &self.mailbox_name, is_utf8); + buf.push(b' '); quoted_string(&mut buf, &self.identifier); for rights in self.permissions { - buf.extend_from_slice(b" "); + buf.push(b' '); for right in rights { buf.push(right.to_char()); } @@ -145,12 +125,8 @@ impl MyRightsResponse { pub fn into_bytes(self, is_utf8: bool) -> Vec { let mut buf = Vec::with_capacity(self.mailbox_name.len() + 10 + self.rights.len()); buf.extend_from_slice(b"* MYRIGHTS "); - if is_utf8 { - quoted_string(&mut buf, &self.mailbox_name); - } else { - quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); - } - buf.extend_from_slice(b" "); + quoted_mailbox_name(&mut buf, &self.mailbox_name, is_utf8); + buf.push(b' '); for right in self.rights { buf.push(right.to_char()); } @@ -160,6 +136,28 @@ impl MyRightsResponse { } impl Rights { + pub fn from_acl(acl: Bitmap) -> impl Iterator { + ACL_RIGHTS + .iter() + .filter(move |(acls, _)| acls.iter().any(|item| acl.contains(*item))) + .map(|(_, right)| *right) + } + + pub fn to_acl(&self) -> &'static [Acl] { + match self { + Rights::Lookup => &[Acl::Read], + Rights::Read => &[Acl::ReadItems], + Rights::Seen | Rights::Write => &[Acl::ModifyItems], + Rights::Insert => &[Acl::AddItems], + Rights::Post => &[Acl::Submit], + Rights::CreateMailbox => &[Acl::CreateChild], + Rights::DeleteMailbox => &[Acl::Delete, Acl::Modify], + Rights::DeleteMessages | Rights::Expunge => &[Acl::RemoveItems], + Rights::Annotate => &[Acl::ModifyMetadata], + Rights::Administer => &[Acl::Share], + } + } + pub fn to_char(&self) -> u8 { match self { Rights::Lookup => b'l', @@ -172,6 +170,7 @@ impl Rights { Rights::DeleteMailbox => b'x', Rights::DeleteMessages => b't', Rights::Expunge => b'e', + Rights::Annotate => b'n', Rights::Administer => b'a', } } @@ -190,97 +189,9 @@ impl Display for Rights { Rights::DeleteMailbox => write!(f, "x"), Rights::DeleteMessages => write!(f, "t"), Rights::Expunge => write!(f, "e"), + Rights::Annotate => write!(f, "n"), Rights::Administer => write!(f, "a"), } } } -impl From for Acl { - fn from(value: Rights) -> Self { - match value { - Rights::Lookup => Acl::Read, - Rights::Read => Acl::ReadItems, - Rights::Seen => Acl::ModifyItems, - Rights::Write => Acl::ModifyItems, - Rights::Insert => Acl::AddItems, - Rights::Post => Acl::Submit, - Rights::CreateMailbox => Acl::CreateChild, - Rights::DeleteMailbox => Acl::Delete, - Rights::DeleteMessages => Acl::RemoveItems, - Rights::Expunge => Acl::RemoveItems, - Rights::Administer => Acl::Share, - } - } -} - -#[cfg(test)] -mod tests { - - use crate::protocol::acl::{GetAclResponse, ListRightsResponse, MyRightsResponse, Rights}; - - #[test] - fn serialize_acl() { - assert_eq!( - String::from_utf8( - GetAclResponse { - mailbox_name: "INBOX".into(), - permissions: vec![ - ( - "Fred".into(), - vec![ - Rights::Lookup, - Rights::Read, - Rights::Seen, - Rights::Write, - Rights::Insert, - Rights::CreateMailbox, - Rights::DeleteMessages, - Rights::Administer, - ] - ), - ( - "David".into(), - vec![ - Rights::CreateMailbox, - Rights::DeleteMessages, - Rights::Administer, - ] - ) - ] - } - .into_bytes(true) - ) - .unwrap(), - "* ACL \"INBOX\" \"Fred\" lrswikta \"David\" kta\r\n" - ); - - assert_eq!( - String::from_utf8( - ListRightsResponse { - mailbox_name: "Deleted Items".into(), - identifier: "Fred".into(), - permissions: vec![ - vec![Rights::Lookup, Rights::Read], - vec![Rights::Administer], - vec![Rights::DeleteMailbox] - ] - } - .into_bytes(true) - ) - .unwrap(), - "* LISTRIGHTS \"Deleted Items\" \"Fred\" lr a x\r\n" - ); - - assert_eq!( - String::from_utf8( - MyRightsResponse { - mailbox_name: "Important".into(), - rights: vec![Rights::Lookup, Rights::Read, Rights::DeleteMailbox] - } - .into_bytes(true) - ) - .unwrap(), - "* MYRIGHTS \"Important\" lrx\r\n" - ); - } -} diff --git a/crates/imap-proto/src/protocol/append.rs b/crates/imap-proto/src/protocol/append.rs index cd3ad0da..5b02c4b5 100644 --- a/crates/imap-proto/src/protocol/append.rs +++ b/crates/imap-proto/src/protocol/append.rs @@ -5,11 +5,12 @@ */ use super::Flag; +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub messages: Vec, } diff --git a/crates/imap-proto/src/protocol/authenticate.rs b/crates/imap-proto/src/protocol/authenticate.rs index a65b60e6..fcd3b785 100644 --- a/crates/imap-proto/src/protocol/authenticate.rs +++ b/crates/imap-proto/src/protocol/authenticate.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub mechanism: Mechanism, - pub params: Vec, + pub params: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -28,8 +29,8 @@ pub enum Mechanism { } impl Mechanism { - pub fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(match self { + pub fn as_bytes(&self) -> &'static [u8] { + match self { Mechanism::Plain => b"PLAIN", Mechanism::CramMd5 => b"CRAM-MD5", Mechanism::DigestMd5 => b"DIGEST-MD5", @@ -42,7 +43,11 @@ impl Mechanism { Mechanism::External => b"EXTERNAL", Mechanism::OAuthBearer => b"OAUTHBEARER", Mechanism::XOauth2 => b"XOAUTH2", - }); + } + } + + pub fn serialize(&self, buf: &mut Vec) { + buf.extend_from_slice(self.as_bytes()); } pub fn into_bytes(self) -> Vec { diff --git a/crates/imap-proto/src/protocol/capability.rs b/crates/imap-proto/src/protocol/capability.rs index 4fb7c577..c5d62408 100644 --- a/crates/imap-proto/src/protocol/capability.rs +++ b/crates/imap-proto/src/protocol/capability.rs @@ -5,6 +5,7 @@ */ use super::{ImapResponse, authenticate::Mechanism}; +use crate::protocol::push_int; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { @@ -58,6 +59,16 @@ pub enum Capability { UidBatches, MessageLimit(u32), SaveLimit(u32), + Metadata, + MetadataServer, + ListMetadata, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuthCapability { + Disabled, + Plain, + PlainAndBearer, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -95,7 +106,7 @@ impl Capability { Capability::Binary => b"BINARY", Capability::Unselect => b"UNSELECT", Capability::ACL => b"ACL", - Capability::Rights => b"RIGHTS=texk", + Capability::Rights => b"RIGHTS=texkn", Capability::UIDPlus => b"UIDPLUS", Capability::ESearch => b"ESEARCH", Capability::SASLIR => b"SASL-IR", @@ -127,14 +138,17 @@ impl Capability { Capability::JmapAccess => b"JMAPACCESS", Capability::UidOnly => b"UIDONLY", Capability::UidBatches => b"UIDBATCHES", + Capability::Metadata => b"METADATA", + Capability::MetadataServer => b"METADATA-SERVER", + Capability::ListMetadata => b"LIST-METADATA", Capability::MessageLimit(limit) => { buf.extend_from_slice(b"MESSAGELIMIT="); - buf.extend_from_slice(limit.to_string().as_bytes()); + push_int(buf, *limit); return; } Capability::SaveLimit(limit) => { buf.extend_from_slice(b"SAVELIMIT="); - buf.extend_from_slice(limit.to_string().as_bytes()); + push_int(buf, *limit); return; } }); @@ -143,7 +157,7 @@ impl Capability { pub fn all_capabilities( is_authenticated: bool, offer_tls: bool, - allow_auth: bool, + auth: AuthCapability, message_limit: u32, save_limit: u32, ) -> Vec { @@ -194,14 +208,16 @@ impl Capability { Capability::MessageLimit(message_limit), Capability::SaveLimit(save_limit), ]); - } else if allow_auth { - capabilities.extend([ - Capability::Auth(Mechanism::Plain), - Capability::Auth(Mechanism::OAuthBearer), - Capability::Auth(Mechanism::XOauth2), - ]); } else { - capabilities.push(Capability::LoginDisabled); + match auth { + AuthCapability::PlainAndBearer => capabilities.extend([ + Capability::Auth(Mechanism::Plain), + Capability::Auth(Mechanism::OAuthBearer), + Capability::Auth(Mechanism::XOauth2), + ]), + AuthCapability::Plain => capabilities.push(Capability::Auth(Mechanism::Plain)), + AuthCapability::Disabled => capabilities.push(Capability::LoginDisabled), + } } if offer_tls { capabilities.push(Capability::StartTLS); @@ -212,37 +228,20 @@ impl Capability { } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); + fn serialize_into(&self, buf: &mut Vec) { buf.extend_from_slice(b"* CAPABILITY"); for capability in self.capabilities.iter() { buf.push(b' '); - capability.serialize(&mut buf); + capability.serialize(buf); } buf.extend_from_slice(b"\r\n"); - buf + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 16; + const CAPABILITY_LEN: usize = 20; + + FRAMING_LEN + self.capabilities.len() * CAPABILITY_LEN } } -#[cfg(test)] -mod tests { - use crate::protocol::{ - ImapResponse, - capability::{Capability, Response}, - }; - - #[test] - fn serialize_capability() { - assert_eq!( - &Response { - capabilities: vec![ - Capability::IMAP4rev2, - Capability::StartTLS, - Capability::LoginDisabled - ], - } - .serialize(), - "* CAPABILITY IMAP4rev2 STARTTLS LOGINDISABLED\r\n".as_bytes() - ); - } -} diff --git a/crates/imap-proto/src/protocol/copy_move.rs b/crates/imap-proto/src/protocol/copy_move.rs index 2ae3cdb0..a895599a 100644 --- a/crates/imap-proto/src/protocol/copy_move.rs +++ b/crates/imap-proto/src/protocol/copy_move.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use super::Sequence; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, - pub mailbox_name: String, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/create.rs b/crates/imap-proto/src/protocol/create.rs index 7c9c287c..6f21805e 100644 --- a/crates/imap-proto/src/protocol/create.rs +++ b/crates/imap-proto/src/protocol/create.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use super::list::Attribute; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub mailbox_role: Option, } diff --git a/crates/imap-proto/src/protocol/delete.rs b/crates/imap-proto/src/protocol/delete.rs index 2284bc72..8f1e5fd6 100644 --- a/crates/imap-proto/src/protocol/delete.rs +++ b/crates/imap-proto/src/protocol/delete.rs @@ -3,9 +3,10 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/enable.rs b/crates/imap-proto/src/protocol/enable.rs index 9393a0a0..a3388299 100644 --- a/crates/imap-proto/src/protocol/enable.rs +++ b/crates/imap-proto/src/protocol/enable.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use super::{ImapResponse, capability::Capability}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub capabilities: Vec, } @@ -17,19 +18,26 @@ pub struct Response { } impl ImapResponse for Response { - fn serialize(self) -> Vec { + fn serialize_into(&self, buf: &mut Vec) { if !self.enabled.is_empty() { - let mut buf = Vec::with_capacity(64); buf.extend(b"* ENABLED"); - for capability in self.enabled { + for capability in &self.enabled { buf.push(b' '); - capability.serialize(&mut buf); + capability.serialize(buf); } buf.push(b'\r'); buf.push(b'\n'); - buf + } + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 12; + const CAPABILITY_LEN: usize = 20; + + if self.enabled.is_empty() { + 0 } else { - Vec::new() + FRAMING_LEN + self.enabled.len() * CAPABILITY_LEN } } } diff --git a/crates/imap-proto/src/protocol/expunge.rs b/crates/imap-proto/src/protocol/expunge.rs index 6a3b39db..49e86644 100644 --- a/crates/imap-proto/src/protocol/expunge.rs +++ b/crates/imap-proto/src/protocol/expunge.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ImapResponse, serialize_sequence}; +use super::{ImapResponse, Sequence, push_int, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { @@ -13,35 +13,47 @@ pub struct Response { } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); - self.serialize_to(&mut buf); - buf + fn serialize_into(&self, buf: &mut Vec) { + self.serialize_to(buf); + } + + fn size_hint(&self) -> usize { + const EXPUNGE_LINE_LEN: usize = 21; + const VANISHED_FRAMING_LEN: usize = 24; + const RANGE_LEN: usize = 8; + + if !self.use_vanished { + self.ids.len() * EXPUNGE_LINE_LEN + } else { + VANISHED_FRAMING_LEN + self.ids.len() * RANGE_LEN + } } } impl Response { - pub fn serialize_to(self, buf: &mut Vec) { + pub fn serialize_to(&self, buf: &mut Vec) { if !self.use_vanished { - for (num_deletions, id) in self.ids.into_iter().enumerate() { + for (num_deletions, id) in self.ids.iter().enumerate() { buf.extend_from_slice(b"* "); - buf.extend_from_slice( - id.saturating_sub(num_deletions as u32) - .to_string() - .as_bytes(), - ); + push_int(buf, id.saturating_sub(num_deletions as u32)); buf.extend_from_slice(b" EXPUNGE\r\n"); } } else { - Vanished { - earlier: false, - ids: self.ids, - } - .serialize(buf); + serialize_vanished(buf, false, &self.ids); } } } +fn serialize_vanished(buf: &mut Vec, earlier: bool, ids: &[u32]) { + if earlier { + buf.extend_from_slice(b"* VANISHED (EARLIER) "); + } else { + buf.extend_from_slice(b"* VANISHED "); + } + serialize_sequence(buf, ids); + buf.extend_from_slice(b"\r\n"); +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Vanished { pub earlier: bool, @@ -50,62 +62,64 @@ pub struct Vanished { impl Vanished { pub fn serialize(&self, buf: &mut Vec) { - if self.earlier { - buf.extend_from_slice(b"* VANISHED (EARLIER) "); - } else { - buf.extend_from_slice(b"* VANISHED "); + serialize_vanished(buf, self.earlier, &self.ids); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VanishedFilter { + ranges: Vec<(u32, u32)>, +} + +impl VanishedFilter { + pub fn new(sequence: &Sequence, last_uid: u32) -> Option { + let mut ranges = Vec::new(); + if !Self::collect(sequence, last_uid, &mut ranges) { + return None; } - serialize_sequence(buf, &self.ids); - buf.extend_from_slice(b"\r\n"); + ranges.sort_unstable(); + let mut merged: Vec<(u32, u32)> = Vec::with_capacity(ranges.len()); + for (start, end) in ranges { + match merged.last_mut() { + Some(last) if start <= last.1.saturating_add(1) => last.1 = last.1.max(end), + _ => merged.push((start, end)), + } + } + Some(Self { ranges: merged }) + } + + fn collect(sequence: &Sequence, last_uid: u32, ranges: &mut Vec<(u32, u32)>) -> bool { + match sequence { + Sequence::Number { value } => ranges.push((*value, *value)), + Sequence::Range { + start: Some(start), + end: Some(end), + } => ranges.push((*start.min(end), *start.max(end))), + Sequence::Range { + start: Some(bound), + end: None, + } + | Sequence::Range { + start: None, + end: Some(bound), + } => ranges.push((*bound, u32::MAX)), + Sequence::Range { + start: None, + end: None, + } => ranges.push((last_uid, u32::MAX)), + Sequence::List { items } => { + return items + .iter() + .all(|item| Self::collect(item, last_uid, ranges)); + } + Sequence::SavedSearch => return false, + } + true + } + + pub fn contains(&self, uid: u32) -> bool { + let pos = self.ranges.partition_point(|&(_, end)| end < uid); + self.ranges.get(pos).is_some_and(|&(start, _)| start <= uid) } } -#[cfg(test)] -mod tests { - use crate::protocol::ImapResponse; - - #[test] - fn serialize_expunge() { - assert_eq!( - String::from_utf8( - super::Response { - use_vanished: false, - ids: vec![3, 4, 5] - } - .serialize() - ) - .unwrap(), - concat!("* 3 EXPUNGE\r\n", "* 3 EXPUNGE\r\n", "* 3 EXPUNGE\r\n",) - ); - - assert_eq!( - String::from_utf8( - super::Response { - use_vanished: false, - ids: vec![3, 4, 7, 9, 11] - } - .serialize() - ) - .unwrap(), - concat!( - "* 3 EXPUNGE\r\n", - "* 3 EXPUNGE\r\n", - "* 5 EXPUNGE\r\n", - "* 6 EXPUNGE\r\n", - "* 7 EXPUNGE\r\n", - ) - ); - - assert_eq!( - String::from_utf8( - super::Response { - use_vanished: true, - ids: vec![3, 4, 5] - } - .serialize() - ) - .unwrap(), - "* VANISHED 3:5\r\n" - ); - } -} diff --git a/crates/imap-proto/src/protocol/fetch.rs b/crates/imap-proto/src/protocol/fetch.rs index 8058b039..a55d3bb9 100644 --- a/crates/imap-proto/src/protocol/fetch.rs +++ b/crates/imap-proto/src/protocol/fetch.rs @@ -3,32 +3,70 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::protocol::push_int; +use compact_str::CompactString; use super::{ - Flag, ImapResponse, ObjectId, Sequence, literal_string, quoted_or_literal_encoded_string, - quoted_or_literal_encoded_string_or_nil, quoted_or_literal_string, - quoted_or_literal_string_or_nil, quoted_rfc2822_or_nil, quoted_timestamp, + Flag, ObjectId, Sequence, WriteLiteral, literal_string, quoted_or_literal_string, + quoted_timestamp, }; -use crate::protocol::literal_string_slice; -use mail_parser::DateTime; -use std::borrow::Cow; -use utils::chained_bytes::SliceRange; +use std::borrow::{Borrow, Cow}; +use utils::chained_bytes::ChainedBytes; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, pub attributes: Vec, pub changed_since: Option, pub include_vanished: bool, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Response<'x> { - pub is_uid: bool, - pub is_utf8: bool, - pub items: Vec>, + +impl Flag { + pub fn write_fetch_item>( + buf: &mut Vec, + flags: impl IntoIterator, + ) { + buf.extend_from_slice(b"FLAGS ("); + for (pos, flag) in flags + .into_iter() + .filter(|flag| flag.borrow().is_serializable()) + .enumerate() + { + if pos > 0 { + buf.push(b' '); + } + flag.borrow().serialize(buf); + } + buf.push(b')'); + } } +impl Attribute { + pub fn header_fields(&self) -> Option<&[String]> { + match self { + Attribute::BodySection { sections, .. } => match sections.last() { + Some(Section::HeaderFields { fields, .. }) => Some(fields), + _ => None, + }, + _ => None, + } + } +} + +impl Arguments { + pub fn sets_seen(&self) -> bool { + self.attributes.iter().any(|attribute| { + matches!( + attribute, + Attribute::BodySection { peek: false, .. } + | Attribute::Binary { peek: false, .. } + | Attribute::Rfc822 + | Attribute::Rfc822Text + ) + }) + } +} #[derive(Debug, Clone, PartialEq, Eq)] pub struct FetchItem<'x> { pub id: u32, @@ -80,27 +118,18 @@ pub enum Section { #[derive(Debug, Clone, PartialEq, Eq)] pub enum DataItem<'x> { Binary { - sections: Vec, + sections: Cow<'x, [u32]>, offset: Option, contents: BodyContents<'x>, }, BinarySize { - sections: Vec, + sections: Cow<'x, [u32]>, size: usize, }, - Body { - part: BodyPart<'x>, - }, - BodyStructure { - part: BodyPart<'x>, - }, BodySection { - sections: Vec
, + sections: Cow<'x, [Section]>, origin_octet: Option, - contents: Cow<'x, [u8]>, - }, - Envelope { - envelope: Envelope<'x>, + contents: BodyContents<'x>, }, Flags { flags: Vec, @@ -112,16 +141,16 @@ pub enum DataItem<'x> { uid: u32, }, Rfc822 { - contents: SliceRange<'x>, + contents: ChainedBytes<'x>, }, Rfc822Header { - contents: SliceRange<'x>, + contents: ChainedBytes<'x>, }, Rfc822Size { size: usize, }, Rfc822Text { - contents: SliceRange<'x>, + contents: ChainedBytes<'x>, }, Preview { contents: Option>, @@ -132,464 +161,64 @@ pub enum DataItem<'x> { ObjectId(ObjectId), } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Address<'x> { - Single(EmailAddress<'x>), - Group(AddressGroup<'x>), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AddressGroup<'x> { - pub name: Option>, - pub addresses: Vec>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EmailAddress<'x> { - pub name: Option>, - pub address: Cow<'x, str>, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum BodyContents<'x> { Text(Cow<'x, str>), - Bytes(Cow<'x, [u8]>), + Bytes(ChainedBytes<'x>), + Owned(Vec), } -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct Envelope<'x> { - pub date: Option, - pub subject: Option>, - pub from: Vec>, - pub sender: Vec>, - pub reply_to: Vec>, - pub to: Vec>, - pub cc: Vec>, - pub bcc: Vec>, - pub in_reply_to: Option>, - pub message_id: Option>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -#[allow(clippy::type_complexity)] -pub enum BodyPart<'x> { - Multipart { - body_parts: Vec>, - body_subtype: Cow<'x, str>, - // Extension data - body_parameters: Option, Cow<'x, str>)>>, - extension: BodyPartExtension<'x>, - }, - Basic { - body_type: Option>, - fields: BodyPartFields<'x>, - // Extension data - body_md5: Option>, - extension: BodyPartExtension<'x>, - }, - Text { - fields: BodyPartFields<'x>, - body_size_lines: usize, - // Extension data - body_md5: Option>, - extension: BodyPartExtension<'x>, - }, - Message { - fields: BodyPartFields<'x>, - envelope: Option>>, - body: Option>>, - body_size_lines: usize, - // Extension data - body_md5: Option>, - extension: BodyPartExtension<'x>, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct BodyPartFields<'x> { - pub body_subtype: Option>, - pub body_parameters: Option, Cow<'x, str>)>>, - pub body_id: Option>, - pub body_description: Option>, - pub body_encoding: Option>, - pub body_size_octets: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -#[allow(clippy::type_complexity)] -pub struct BodyPartExtension<'x> { - pub body_disposition: Option<(Cow<'x, str>, Vec<(Cow<'x, str>, Cow<'x, str>)>)>, - pub body_language: Option>>, - pub body_location: Option>, -} - -impl Address<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { +impl<'x> BodyContents<'x> { + pub fn as_chained(&self) -> ChainedBytes<'_> { match self { - Address::Single(addr) => addr.serialize(buf, is_utf8), - Address::Group(addr) => addr.serialize(buf, is_utf8), + BodyContents::Text(text) => ChainedBytes::new(text.as_bytes()), + BodyContents::Bytes(bytes) => *bytes, + BodyContents::Owned(bytes) => ChainedBytes::new(bytes), } } - pub fn into_owned<'y>(self) -> Address<'y> { + pub fn len(&self) -> usize { match self { - Address::Single(addr) => Address::Single(addr.into_owned()), - Address::Group(addr) => Address::Group(addr.into_owned()), + BodyContents::Text(text) => text.len(), + BodyContents::Bytes(bytes) => bytes.len(), + BodyContents::Owned(bytes) => bytes.len(), } } -} -impl EmailAddress<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { - buf.push(b'('); - if let Some(name) = &self.name { - quoted_or_literal_encoded_string(buf, name, is_utf8); - } else { - buf.extend_from_slice(b"NIL"); - } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } - let addr = if let Some((route, addr)) = self.address.split_once(':') { - buf.push(b' '); - quoted_or_literal_string(buf, route); - buf.push(b' '); - addr - } else { - buf.extend_from_slice(b" NIL "); - &self.address + pub fn partial(self, partial: Option<(u32, u32)>) -> Self { + let Some((start, len)) = partial else { + return self; }; - - if let Some((local, host)) = addr.rsplit_once('@') { - quoted_or_literal_string(buf, local); - buf.push(b' '); - quoted_or_literal_string(buf, host); - } else { - quoted_or_literal_string(buf, &self.address); - buf.extend_from_slice(b" \"\""); - } - buf.push(b')'); - } - - pub fn into_owned<'y>(self) -> EmailAddress<'y> { - EmailAddress { - name: self.name.map(|n| n.into_owned().into()), - address: self.address.into_owned().into(), + let start = start as usize; + let end = start.saturating_add(len as usize); + match self { + BodyContents::Bytes(bytes) => { + BodyContents::Bytes(bytes.view(start..end.min(bytes.len())).unwrap_or_default()) + } + BodyContents::Owned(mut bytes) => { + bytes.truncate(end); + bytes.drain(..start.min(bytes.len())); + BodyContents::Owned(bytes) + } + BodyContents::Text(text) => BodyContents::Owned( + text.as_bytes() + .get(start..end.min(text.len())) + .unwrap_or_default() + .to_vec(), + ), } } } -impl AddressGroup<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { - buf.extend_from_slice(b"(NIL NIL "); - if let Some(name) = &self.name { - quoted_or_literal_encoded_string(buf, name, is_utf8); - } else { - buf.extend_from_slice(b"\"\""); - } - buf.extend_from_slice(b" NIL)"); - for addr in &self.addresses { - addr.serialize(buf, is_utf8); - } - buf.extend_from_slice(b"(NIL NIL NIL NIL)"); - } - - pub fn into_owned<'y>(self) -> AddressGroup<'y> { - AddressGroup { - name: self.name.map(|n| n.into_owned().into()), - addresses: self - .addresses - .into_iter() - .map(|addr| addr.into_owned()) - .collect(), - } - } -} - -impl<'x> BodyPart<'x> { - pub fn serialize(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool) { - buf.push(b'('); - match self { - BodyPart::Multipart { - body_parts, - body_subtype, - body_parameters, - extension, - } => { - for part in body_parts.iter() { - part.serialize(buf, is_extended, is_utf8); - } - buf.push(b' '); - quoted_or_literal_string(buf, body_subtype); - if is_extended { - if let Some(body_parameters) = body_parameters { - buf.extend_from_slice(b" ("); - for (pos, (key, value)) in body_parameters.iter().enumerate() { - if pos > 0 { - buf.push(b' '); - } - quoted_or_literal_string(buf, key); - buf.push(b' '); - quoted_or_literal_encoded_string(buf, value, is_utf8); - } - buf.push(b')'); - } else { - buf.extend_from_slice(b" NIL"); - } - buf.push(b' '); - extension.serialize(buf, is_utf8); - } - } - BodyPart::Basic { - body_type, - fields, - body_md5, - extension, - } => { - quoted_or_literal_string_or_nil(buf, body_type.as_deref()); - buf.push(b' '); - fields.serialize(buf, is_utf8); - if is_extended { - buf.push(b' '); - quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); - buf.push(b' '); - extension.serialize(buf, is_utf8); - } - } - BodyPart::Text { - fields, - body_size_lines, - body_md5, - extension, - } => { - buf.extend_from_slice(b"\"text\" "); - fields.serialize(buf, is_utf8); - buf.push(b' '); - buf.extend_from_slice(body_size_lines.to_string().as_bytes()); - if is_extended { - buf.push(b' '); - quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); - buf.push(b' '); - extension.serialize(buf, is_utf8); - } - } - BodyPart::Message { - fields, - envelope, - body, - body_size_lines, - body_md5, - extension, - } => { - buf.extend_from_slice(b"\"message\" "); - fields.serialize(buf, is_utf8); - buf.push(b' '); - if let Some(envelope) = envelope { - envelope.serialize(buf, is_utf8); - } else { - buf.extend_from_slice(b"NIL"); - } - buf.push(b' '); - if let Some(body) = body { - body.serialize(buf, is_extended, is_utf8); - } else { - buf.extend_from_slice(b"NIL"); - } - buf.push(b' '); - buf.extend_from_slice(body_size_lines.to_string().as_bytes()); - if is_extended { - buf.push(b' '); - quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); - buf.push(b' '); - extension.serialize(buf, is_utf8); - } - } - } - buf.push(b')'); - } - - pub fn add_part(&mut self, part: BodyPart<'x>) { - match self { - BodyPart::Multipart { body_parts, .. } => body_parts.push(part), - BodyPart::Message { body, .. } => *body = Box::new(part).into(), - _ => debug_assert!(false, "Cannot add a part to a non-multipart body part"), - } - } - - pub fn set_envelope(&mut self, envelope_: Envelope<'x>) { - match self { - BodyPart::Message { envelope, .. } => *envelope = Some(Box::new(envelope_)), - _ => debug_assert!(false, "Cannot set envelope on a non-message body part"), - } - } - - pub fn into_owned<'y>(self) -> BodyPart<'y> { - match self { - BodyPart::Multipart { - body_parts, - body_subtype, - body_parameters, - extension, - } => BodyPart::Multipart { - body_parts: body_parts.into_iter().map(|v| v.into_owned()).collect(), - body_subtype: body_subtype.into_owned().into(), - body_parameters: body_parameters.map(|b| { - b.into_iter() - .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) - .collect::>() - }), - extension: extension.into_owned(), - }, - BodyPart::Basic { - body_type, - fields, - body_md5, - extension, - } => BodyPart::Basic { - body_type: body_type.map(|v| v.into_owned().into()), - fields: fields.into_owned(), - body_md5: body_md5.map(|v| v.into_owned().into()), - extension: extension.into_owned(), - }, - BodyPart::Text { - fields, - body_size_lines, - body_md5, - extension, - } => BodyPart::Text { - fields: fields.into_owned(), - body_size_lines, - body_md5: body_md5.map(|v| v.into_owned().into()), - extension: extension.into_owned(), - }, - BodyPart::Message { - fields, - envelope, - body, - body_size_lines, - body_md5, - extension, - } => BodyPart::Message { - fields: fields.into_owned(), - envelope: envelope.map(|v| Box::new(v.into_owned())), - body: body.map(|b| Box::new(b.into_owned())), - body_size_lines, - body_md5: body_md5.map(|v| v.into_owned().into()), - extension: extension.into_owned(), - }, - } - } -} - -impl BodyPartFields<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { - quoted_or_literal_string_or_nil(buf, self.body_subtype.as_deref()); - if let Some(body_parameters) = &self.body_parameters { - buf.extend_from_slice(b" ("); - for (pos, (key, value)) in body_parameters.iter().enumerate() { - if pos > 0 { - buf.push(b' '); - } - quoted_or_literal_string(buf, key); - buf.push(b' '); - quoted_or_literal_encoded_string(buf, value, is_utf8); - } - buf.push(b')'); - } else { - buf.extend_from_slice(b" NIL"); - } - for item in [&self.body_id, &self.body_description, &self.body_encoding] { - buf.push(b' '); - quoted_or_literal_encoded_string_or_nil(buf, item.as_deref(), is_utf8); - } - buf.push(b' '); - buf.extend_from_slice(self.body_size_octets.to_string().as_bytes()); - } - - pub fn into_owned<'y>(self) -> BodyPartFields<'y> { - BodyPartFields { - body_subtype: self.body_subtype.map(|v| v.into_owned().into()), - body_parameters: self.body_parameters.map(|b| { - b.into_iter() - .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) - .collect::>() - }), - body_id: self.body_id.map(|v| v.into_owned().into()), - body_description: self.body_description.map(|v| v.into_owned().into()), - body_encoding: self.body_encoding.map(|v| v.into_owned().into()), - body_size_octets: self.body_size_octets, - } - } -} - -impl BodyPartExtension<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { - if let Some((disposition, parameters)) = &self.body_disposition { - buf.push(b'('); - quoted_or_literal_string(buf, disposition); - if !parameters.is_empty() { - buf.extend_from_slice(b" ("); - for (pos, (key, value)) in parameters.iter().enumerate() { - if pos > 0 { - buf.push(b' '); - } - quoted_or_literal_string(buf, key); - buf.push(b' '); - quoted_or_literal_encoded_string(buf, value, is_utf8); - } - buf.extend_from_slice(b"))"); - } else { - buf.extend_from_slice(b" NIL)"); - } - } else { - buf.extend_from_slice(b"NIL"); - } - if let Some(body_language) = &self.body_language { - match body_language.len() { - 0 => buf.extend_from_slice(b" NIL"), - 1 => { - buf.push(b' '); - quoted_or_literal_string(buf, body_language.last().unwrap()); - } - _ => { - buf.extend_from_slice(b" ("); - for (pos, lang) in body_language.iter().enumerate() { - if pos > 0 { - buf.push(b' '); - } - quoted_or_literal_string(buf, lang); - } - buf.push(b')'); - } - } - } else { - buf.extend_from_slice(b" NIL"); - } - buf.push(b' '); - quoted_or_literal_string_or_nil(buf, self.body_location.as_deref()); - } - - pub fn into_owned<'y>(self) -> BodyPartExtension<'y> { - BodyPartExtension { - body_disposition: self.body_disposition.map(|(a, b)| { - ( - a.into_owned().into(), - b.into_iter() - .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) - .collect::>(), - ) - }), - body_language: self - .body_language - .map(|v| v.into_iter().map(|a| a.into_owned().into()).collect()), - body_location: self.body_location.map(|v| v.into_owned().into()), - } - } -} - -impl BodyContents<'_> { - pub fn into_owned<'y>(self) -> BodyContents<'y> { - match self { - BodyContents::Text(text) => BodyContents::Text(text.into_owned().into()), - BodyContents::Bytes(bytes) => BodyContents::Bytes(bytes.into_owned().into()), +impl<'x> From> for BodyContents<'x> { + fn from(bytes: Cow<'x, [u8]>) -> Self { + match bytes { + Cow::Borrowed(bytes) => BodyContents::Bytes(ChainedBytes::new(bytes)), + Cow::Owned(bytes) => BodyContents::Owned(bytes), } } } @@ -598,23 +227,35 @@ impl Section { pub fn serialize(&self, buf: &mut Vec) { match self { Section::Part { num } => { - buf.extend_from_slice(num.to_string().as_bytes()); + push_int(buf, *num); } Section::Header => { buf.extend_from_slice(b"HEADER"); } Section::HeaderFields { not, fields } => { if !not { - buf.extend_from_slice(b"HEADER.FIELDS "); + buf.extend_from_slice(b"HEADER.FIELDS ("); } else { - buf.extend_from_slice(b"HEADER.FIELDS.NOT "); + buf.extend_from_slice(b"HEADER.FIELDS.NOT ("); } - buf.push(b'('); for (pos, field) in fields.iter().enumerate() { if pos > 0 { buf.push(b' '); } - buf.extend_from_slice(field.as_str().to_ascii_uppercase().as_bytes()); + let start = buf.len(); + buf.extend_from_slice(field.as_bytes()); + let is_atom = buf.get_mut(start..).is_some_and(|written| { + let mut is_atom = !written.is_empty(); + for ch in written.iter_mut() { + is_atom &= ATOM_CHAR[*ch as usize]; + ch.make_ascii_uppercase(); + } + is_atom + }); + if !is_atom { + buf.truncate(start); + quoted_or_literal_string(buf, &field.to_ascii_uppercase()); + } } buf.push(b')'); } @@ -628,88 +269,8 @@ impl Section { } } -static DUMMY_ADDRESS: [Address; 1] = [Address::Single(EmailAddress { - name: None, - address: Cow::Borrowed("unknown@localhost"), -})]; - -impl Envelope<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { - buf.push(b'('); - quoted_rfc2822_or_nil(buf, &self.date); - buf.push(b' '); - quoted_or_literal_encoded_string_or_nil(buf, self.subject.as_deref(), is_utf8); - - // Note: [RFC-2822] requires that all messages have a valid - // From header. Therefore, the from, sender, and reply-to - // members in the envelope can not be NIL. - - let from = if !self.from.is_empty() { - &self.from[..] - } else { - &DUMMY_ADDRESS[..] - }; - - self.serialize_addresses(buf, from, is_utf8); - self.serialize_addresses( - buf, - if !self.sender.is_empty() { - &self.sender - } else { - from - }, - is_utf8, - ); - self.serialize_addresses( - buf, - if !self.reply_to.is_empty() { - &self.reply_to - } else { - from - }, - is_utf8, - ); - self.serialize_addresses(buf, &self.to, is_utf8); - self.serialize_addresses(buf, &self.cc, is_utf8); - self.serialize_addresses(buf, &self.bcc, is_utf8); - for item in [&self.in_reply_to, &self.message_id] { - buf.push(b' '); - quoted_or_literal_string_or_nil(buf, item.as_deref()); - } - buf.push(b')'); - } - - fn serialize_addresses(&self, buf: &mut Vec, addresses: &[Address], is_utf8: bool) { - buf.push(b' '); - if !addresses.is_empty() { - buf.push(b'('); - for address in addresses { - address.serialize(buf, is_utf8); - } - buf.push(b')'); - } else { - buf.extend_from_slice(b"NIL"); - } - } - - pub fn into_owned<'y>(self) -> Envelope<'y> { - Envelope { - date: self.date, - subject: self.subject.map(|v| v.into_owned().into()), - from: self.from.into_iter().map(|v| v.into_owned()).collect(), - sender: self.sender.into_iter().map(|v| v.into_owned()).collect(), - reply_to: self.reply_to.into_iter().map(|v| v.into_owned()).collect(), - to: self.to.into_iter().map(|v| v.into_owned()).collect(), - cc: self.cc.into_iter().map(|v| v.into_owned()).collect(), - bcc: self.bcc.into_iter().map(|v| v.into_owned()).collect(), - in_reply_to: self.in_reply_to.map(|v| v.into_owned().into()), - message_id: self.message_id.map(|v| v.into_owned().into()), - } - } -} - impl DataItem<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { + pub fn serialize(&self, buf: &mut Vec) { match self { DataItem::Binary { sections, @@ -721,25 +282,20 @@ impl DataItem<'_> { if pos > 0 { buf.push(b'.'); } - buf.extend_from_slice(section.to_string().as_bytes()); + push_int(buf, *section); } if let Some(offset) = offset { buf.extend_from_slice(b"]<"); - buf.extend_from_slice(offset.to_string().as_bytes()); + push_int(buf, *offset); buf.extend_from_slice(b"> "); } else { buf.extend_from_slice(b"] "); } - match contents { - BodyContents::Text(text) => { - literal_string(buf, text.as_bytes()); - } - BodyContents::Bytes(bytes) => { - buf.extend_from_slice(b"~{"); - buf.extend_from_slice(bytes.len().to_string().as_bytes()); - buf.extend_from_slice(b"}\r\n"); - buf.extend_from_slice(bytes); - } + if let BodyContents::Text(text) = contents { + literal_string(buf, text.as_bytes()); + } else { + buf.push(b'~'); + contents.as_chained().write_literal(buf); } } DataItem::BinarySize { sections, size } => { @@ -748,18 +304,10 @@ impl DataItem<'_> { if pos > 0 { buf.push(b'.'); } - buf.extend_from_slice(section.to_string().as_bytes()); + push_int(buf, *section); } buf.extend_from_slice(b"] "); - buf.extend_from_slice(size.to_string().as_bytes()); - } - DataItem::Body { part } => { - buf.extend_from_slice(b"BODY "); - part.serialize(buf, false, is_utf8); - } - DataItem::BodyStructure { part } => { - buf.extend_from_slice(b"BODYSTRUCTURE "); - part.serialize(buf, true, is_utf8); + push_int(buf, *size); } DataItem::BodySection { sections, @@ -775,50 +323,37 @@ impl DataItem<'_> { } if let Some(origin_octet) = origin_octet { buf.extend_from_slice(b"]<"); - buf.extend_from_slice(origin_octet.to_string().as_bytes()); + push_int(buf, *origin_octet); buf.extend_from_slice(b"> "); } else { buf.extend_from_slice(b"] "); } - literal_string(buf, contents); - } - DataItem::Envelope { envelope } => { - buf.extend_from_slice(b"ENVELOPE "); - envelope.serialize(buf, is_utf8); - } - DataItem::Flags { flags } => { - buf.extend_from_slice(b"FLAGS ("); - for (pos, flag) in flags.iter().enumerate() { - if pos > 0 { - buf.push(b' '); - } - flag.serialize(buf); - } - buf.push(b')'); + contents.as_chained().write_literal(buf); } + DataItem::Flags { flags } => Flag::write_fetch_item(buf, flags), DataItem::InternalDate { date } => { buf.extend_from_slice(b"INTERNALDATE "); quoted_timestamp(buf, *date); } DataItem::Uid { uid } => { buf.extend_from_slice(b"UID "); - buf.extend_from_slice(uid.to_string().as_bytes()); + push_int(buf, *uid); } DataItem::Rfc822 { contents } => { buf.extend_from_slice(b"RFC822 "); - literal_string_slice(buf, contents); + contents.write_literal(buf); } DataItem::Rfc822Header { contents } => { buf.extend_from_slice(b"RFC822.HEADER "); - literal_string_slice(buf, contents); + contents.write_literal(buf); } DataItem::Rfc822Size { size } => { buf.extend_from_slice(b"RFC822.SIZE "); - buf.extend_from_slice(size.to_string().as_bytes()); + push_int(buf, *size); } DataItem::Rfc822Text { contents } => { buf.extend_from_slice(b"RFC822.TEXT "); - literal_string_slice(buf, contents); + contents.write_literal(buf); } DataItem::Preview { contents } => { buf.extend_from_slice(b"PREVIEW "); @@ -830,7 +365,7 @@ impl DataItem<'_> { } DataItem::ModSeq { modseq } => { buf.extend_from_slice(b"MODSEQ ("); - buf.extend_from_slice(modseq.to_string().as_bytes()); + push_int(buf, *modseq); buf.push(b')'); } DataItem::ObjectId(object_id) => { @@ -841,564 +376,113 @@ impl DataItem<'_> { } impl FetchItem<'_> { - pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { + pub fn write_open(buf: &mut Vec, id: u32, is_uidonly: bool) { buf.extend_from_slice(b"* "); - buf.extend_from_slice(self.id.to_string().as_bytes()); - buf.extend_from_slice(if self.is_uidonly { + push_int(buf, id); + buf.extend_from_slice(if is_uidonly { b" UIDFETCH (".as_slice() } else { b" FETCH (".as_slice() }); + } + + pub fn write_close(buf: &mut Vec) { + buf.extend_from_slice(b")\r\n"); + } + + pub fn serialize(&self, buf: &mut Vec) { + Self::write_open(buf, self.id, self.is_uidonly); for (pos, item) in self.items.iter().enumerate() { if pos > 0 { buf.push(b' '); } - item.serialize(buf, is_utf8); + item.serialize(buf); } - buf.extend_from_slice(b")\r\n"); + Self::write_close(buf); } } -impl ImapResponse for Response<'_> { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(128); - for item in &self.items { - item.serialize(&mut buf, self.is_utf8); - } - buf - } -} - -/* - - body = "(" (body-type-1part / body-type-mpart) ")" - - body-type-1part = (body-type-basic / body-type-msg / body-type-text) - [SP body-ext-1part] - - body-type-basic = media-basic SP body-fields - ; MESSAGE subtype MUST NOT be "RFC822" or - ; "GLOBAL" - - body-type-mpart = 1*body SP media-subtype - [SP body-ext-mpart] - ; MULTIPART body part - - body-type-msg = media-message SP body-fields SP envelope - SP body SP body-fld-lines - - body-type-text = media-text SP body-fields SP body-fld-lines - - body-fields = body-fld-param SP body-fld-id SP body-fld-desc SP - body-fld-enc SP body-fld-octets - - media-message = DQUOTE "MESSAGE" DQUOTE SP - DQUOTE ("RFC822" / "GLOBAL") DQUOTE - ; Defined in [MIME-IMT] - - media-basic = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" / - "FONT" / "MESSAGE" / "MODEL" / "VIDEO" ) DQUOTE) - / string) - SP media-subtype - - envelope = "(" env-date SP env-subject SP env-from SP - env-sender SP env-reply-to SP env-to SP env-cc SP - env-bcc SP env-in-reply-to SP env-message-id ")" - - body-fld-lines = number64 - -*/ - -#[cfg(test)] -mod tests { - - use mail_parser::DateTime; - use utils::chained_bytes::SliceRange; - - use crate::protocol::{Flag, ImapResponse}; - - use super::{ - Address, AddressGroup, BodyPart, BodyPartExtension, BodyPartFields, DataItem, EmailAddress, - Envelope, FetchItem, Response, Section, - }; - - #[test] - fn serialize_fetch_data_item() { - for (item, expected_response) in [ - ( - super::DataItem::Envelope { - envelope: Envelope { - date: DateTime::from_timestamp(837570205).into(), - subject: Some("IMAP4rev2 WG mtg summary and minutes".into()), - from: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - sender: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - reply_to: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - to: vec![Address::Single(EmailAddress { - name: None, - address: "imap@cac.washington.edu".into(), - })], - cc: vec![ - Address::Single(EmailAddress { - name: None, - address: "minutes@CNRI.Reston.VA.US".into(), - }), - Address::Single(EmailAddress { - name: Some("John Klensin".into()), - address: "KLENSIN@MIT.EDU".into(), - }), - ], - bcc: vec![], - in_reply_to: None, - message_id: Some("".into()), - }, - }, - concat!( - "ENVELOPE (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", - "\"IMAP4rev2 WG mtg summary and minutes\" ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((NIL NIL \"imap\" \"cac.washington.edu\")) ", - "((NIL NIL \"minutes\" \"CNRI.Reston.VA.US\")", - "(\"John Klensin\" NIL \"KLENSIN\" \"MIT.EDU\")) NIL NIL ", - "\"\")" - ), - ), - ( - super::DataItem::Envelope { - envelope: Envelope { - date: DateTime::from_timestamp(837570205).into(), - subject: Some("Group test".into()), - from: vec![Address::Single(EmailAddress { - name: Some("Bill Foobar".into()), - address: "foobar@example.com".into(), - })], - sender: vec![], - reply_to: vec![], - to: vec![Address::Group(AddressGroup { - name: Some("Friends and Family".into()), - addresses: vec![ - EmailAddress { - name: Some("John Doe".into()), - address: "jdoe@example.com".into(), - }, - EmailAddress { - name: Some("Jane Smith".into()), - address: "jane.smith@example.com".into(), - }, - ], - })], - cc: vec![], - bcc: vec![], - in_reply_to: None, - message_id: Some("".into()), - }, - }, - concat!( - "ENVELOPE (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", - "\"Group test\" ", - "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", - "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", - "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", - "((NIL NIL \"Friends and Family\" NIL)", - "(\"John Doe\" NIL \"jdoe\" \"example.com\")", - "(\"Jane Smith\" NIL \"jane.smith\" \"example.com\")", - "(NIL NIL NIL NIL)) ", - "NIL NIL NIL \"\")" - ), - ), - ( - super::DataItem::Body { - part: BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("PLAIN".into()), - body_parameters: vec![("CHARSET".into(), "US-ASCII".into())].into(), - body_id: None, - body_description: None, - body_encoding: Some("7BIT".into()), - body_size_octets: 2279, - }, - body_size_lines: 48, - body_md5: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - }, - "BODY (\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 2279 48)", - ), - ( - super::DataItem::Body { - part: BodyPart::Message { - fields: BodyPartFields { - body_subtype: Some("RFC822".into()), - body_parameters: None, - body_id: Some("".into()), - body_description: Some("An attached email".into()), - body_encoding: Some("quoted-printable".into()), - body_size_octets: 9323, - }, - envelope: Box::new(Envelope { - date: DateTime::from_timestamp(837570205).into(), - subject: Some("Hello world!".into()), - from: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - sender: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - reply_to: vec![Address::Single(EmailAddress { - name: Some("Terry Gray".into()), - address: "gray@cac.washington.edu".into(), - })], - to: vec![Address::Single(EmailAddress { - name: None, - address: "imap@cac.washington.edu".into(), - })], - cc: vec![], - bcc: vec![], - in_reply_to: None, - message_id: Some("<4234324@domain.com>".into()), - }) - .into(), - body: Box::new(BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("HTML".into()), - body_parameters: None, - body_id: None, - body_description: None, - body_encoding: Some("8BIT".into()), - body_size_octets: 4234, - }, - body_size_lines: 431, - body_md5: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }) - .into(), - body_size_lines: 908, - body_md5: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - }, - concat!( - "BODY (\"message\" \"RFC822\" NIL \"\" \"An attached email\" ", - "\"quoted-printable\" 9323 (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", - "\"Hello world!\" ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", - "((NIL NIL \"imap\" \"cac.washington.edu\")) NIL NIL NIL ", - "\"<4234324@domain.com>\") (\"text\" \"HTML\" NIL NIL NIL ", - "\"8BIT\" 4234 431) 908)" - ), - ), - ( - super::DataItem::Body { - part: BodyPart::Multipart { - body_parts: vec![ - BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("PLAIN".into()), - body_parameters: vec![("CHARSET".into(), "US-ASCII".into())] - .into(), - body_id: None, - body_description: None, - body_encoding: Some("7BIT".into()), - body_size_octets: 1152, - }, - body_size_lines: 23, - body_md5: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("PLAIN".into()), - body_parameters: vec![ - ("CHARSET".into(), "US-ASCII".into()), - ("NAME".into(), "cc.diff".into()), - ] - .into(), - body_id: Some( - "<960723163407.20117h@cac.washington.edu>".into(), - ), - body_description: Some("Compiler diff".into()), - body_encoding: Some("BASE64".into()), - body_size_octets: 4554, - }, - body_size_lines: 73, - body_md5: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - ], - body_subtype: "MIXED".into(), - body_parameters: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - }, - concat!( - "BODY ((\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") ", - "NIL NIL \"7BIT\" 1152 23)", - "(\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\" \"NAME\" \"cc.diff\") ", - "\"<960723163407.20117h@cac.washington.edu>\" \"Compiler diff\" ", - "\"BASE64\" 4554 73) \"MIXED\")", - ), - ), - ( - DataItem::BodyStructure { - part: BodyPart::Multipart { - body_parts: vec![ - BodyPart::Multipart { - body_parts: vec![ - BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("PLAIN".into()), - body_parameters: vec![( - "CHARSET".into(), - "UTF-8".into(), - )] - .into(), - body_id: Some("<111@domain.com>".into()), - body_description: Some("Text part".into()), - body_encoding: Some("7BIT".into()), - body_size_octets: 1152, - }, - body_size_lines: 23, - body_md5: Some("8o3456".into()), - extension: BodyPartExtension { - body_disposition: ("inline".into(), vec![]).into(), - body_language: vec!["en-US".into()].into(), - body_location: Some("right here".into()), - }, - }, - BodyPart::Text { - fields: BodyPartFields { - body_subtype: Some("HTML".into()), - body_parameters: vec![( - "CHARSET".into(), - "UTF-8".into(), - )] - .into(), - body_id: Some("<54535@domain.com>".into()), - body_description: Some("HTML part".into()), - body_encoding: Some("8BIT".into()), - body_size_octets: 45345, - }, - body_size_lines: 994, - body_md5: Some("53454".into()), - extension: BodyPartExtension { - body_disposition: ( - "attachment".into(), - vec![("filename".into(), "myfile.txt".into())], - ) - .into(), - body_language: vec!["en-US".into(), "de-DE".into()] - .into(), - body_location: Some("right there".into()), - }, - }, - ], - body_subtype: "ALTERNATIVE".into(), - body_parameters: vec![( - "x-param".into(), - "a very special parameter".into(), - )] - .into(), - extension: BodyPartExtension { - body_disposition: None, - body_language: vec!["en-US".into()].into(), - body_location: Some("unknown".into()), - }, - }, - BodyPart::Basic { - body_type: Some("APPLICATION".into()), - fields: BodyPartFields { - body_subtype: Some("MSWORD".into()), - body_parameters: vec![( - "NAME".into(), - "chimichangas.docx".into(), - )] - .into(), - body_id: Some("<4444@chimi.changa>".into()), - body_description: Some("Chimichangas recipe".into()), - body_encoding: Some("base64".into()), - body_size_octets: 84723, - }, - body_md5: Some("1234".into()), - extension: BodyPartExtension { - body_disposition: ( - "attachment".into(), - vec![("filename".into(), "chimichangas.docx".into())], - ) - .into(), - body_language: vec!["en-MX".into()].into(), - body_location: Some("secret location".into()), - }, - }, - ], - body_subtype: "MIXED".into(), - body_parameters: None, - extension: BodyPartExtension { - body_disposition: None, - body_language: None, - body_location: None, - }, - }, - }, - concat!( - "BODYSTRUCTURE (((\"text\" \"PLAIN\" (\"CHARSET\" \"UTF-8\") ", - "\"<111@domain.com>\" \"Text part\" \"7BIT\" 1152 23 \"8o3456\" ", - "(\"inline\" NIL) \"en-US\" \"right here\")", - "(\"text\" \"HTML\" (\"CHARSET\" \"UTF-8\") ", - "\"<54535@domain.com>\" \"HTML part\" \"8BIT\" 45345 994 \"53454\" ", - "(\"attachment\" (\"filename\" \"myfile.txt\")) ", - "(\"en-US\" \"de-DE\") ", - "\"right there\") \"ALTERNATIVE\" (\"x-param\" ", - "\"a very special parameter\") ", - "NIL \"en-US\" \"unknown\")", - "(\"APPLICATION\" \"MSWORD\" (\"NAME\" \"chimichangas.docx\") ", - "\"<4444@chimi.changa>\" \"Chimichangas recipe\" \"base64\"", - " 84723 \"1234\" ", - "(\"attachment\" (\"filename\" \"chimichangas.docx\")) \"en-MX\" ", - "\"secret location\") \"MIXED\" NIL NIL NIL NIL)", - ), - ), - ( - super::DataItem::Binary { - sections: vec![1, 2, 3], - offset: 10.into(), - contents: super::BodyContents::Bytes(b"hello".to_vec().into()), - }, - "BINARY[1.2.3]<10> ~{5}\r\nhello", - ), - ( - super::DataItem::Binary { - sections: vec![1, 2, 3], - offset: None, - contents: super::BodyContents::Text("hello".into()), - }, - "BINARY[1.2.3] {5}\r\nhello", - ), - ( - super::DataItem::BodySection { - sections: vec![ - Section::Part { num: 1 }, - Section::Part { num: 2 }, - Section::Mime, - ], - origin_octet: 11.into(), - contents: b"howdy"[..].into(), - }, - "BODY[1.2.MIME]<11> {5}\r\nhowdy", - ), - ( - super::DataItem::BodySection { - sections: vec![Section::HeaderFields { - not: true, - fields: vec!["Subject".into(), "x-special".into()], - }], - origin_octet: None, - contents: b"howdy"[..].into(), - }, - "BODY[HEADER.FIELDS.NOT (SUBJECT X-SPECIAL)] {5}\r\nhowdy", - ), - ( - super::DataItem::BodySection { - sections: vec![Section::HeaderFields { - not: false, - fields: vec!["From".into(), "List-Archive".into()], - }], - origin_octet: None, - contents: b"howdy"[..].into(), - }, - "BODY[HEADER.FIELDS (FROM LIST-ARCHIVE)] {5}\r\nhowdy", - ), - ( - super::DataItem::Flags { - flags: vec![Flag::Seen], - }, - "FLAGS (\\Seen)", - ), - ( - super::DataItem::InternalDate { date: 482374938 }, - "INTERNALDATE \"15-Apr-1985 01:02:18 +0000\"", - ), - ] { - let mut buf = Vec::with_capacity(100); - - item.serialize(&mut buf, false); - - assert_eq!(String::from_utf8(buf).unwrap(), expected_response); - } - } - - #[test] - fn serialize_fetch() { - assert_eq!( - String::from_utf8( - Response { - is_uid: false, - is_utf8: false, - items: vec![FetchItem { - id: 123, - is_uidonly: false, - items: vec![ - super::DataItem::Flags { - flags: vec![Flag::Deleted, Flag::Flagged], - }, - super::DataItem::Uid { uid: 983 }, - super::DataItem::Rfc822Size { size: 443 }, - super::DataItem::Rfc822Text { - contents: SliceRange::Single(&b"hi"[..]), - }, - super::DataItem::Rfc822Header { - contents: SliceRange::Single(&b"header"[..]), - }, - ], - }], - } - .serialize(), - ) - .unwrap(), - concat!( - "* 123 FETCH (FLAGS (\\Deleted \\Flagged) ", - "UID 983 ", - "RFC822.SIZE 443 ", - "RFC822.TEXT {2}\r\nhi ", - "RFC822.HEADER {6}\r\nheader)\r\n", - ) +const fn atom_char_table() -> [bool; 256] { + let mut table = [false; 256]; + let mut ch = 0x21; + while ch < 0x7f { + table[ch] = !matches!( + ch as u8, + b'(' | b')' | b'{' | b'%' | b'*' | b'"' | b'\\' | b']' ); + ch += 1; + } + table +} + +static ATOM_CHAR: [bool; 256] = atom_char_table(); + +const SECTION_FRAMING_LEN: usize = 20; +const ITEM_FRAMING_LEN: usize = 16; +const INT_LEN: usize = 11; +const SIZE_LEN: usize = 20; +const OBJECT_ID_LEN: usize = 96; + +impl Section { + fn size_hint(&self) -> usize { + match self { + Section::Part { .. } => INT_LEN, + Section::Header => 6, + Section::HeaderFields { fields, .. } => { + SECTION_FRAMING_LEN + fields.iter().map(|field| field.len() + 1).sum::() + } + Section::Text | Section::Mime => 4, + } } } + +impl DataItem<'_> { + fn size_hint(&self) -> usize { + match self { + DataItem::Binary { + sections, contents, .. + } => ITEM_FRAMING_LEN + sections.len() * INT_LEN + INT_LEN + contents.len(), + DataItem::BinarySize { sections, .. } => { + ITEM_FRAMING_LEN + sections.len() * INT_LEN + SIZE_LEN + } + DataItem::BodySection { + sections, contents, .. + } => { + ITEM_FRAMING_LEN + + sections + .iter() + .map(|section| section.size_hint() + 1) + .sum::() + + INT_LEN + + contents.len() + } + DataItem::Flags { flags } => ITEM_FRAMING_LEN + flags.len() * 16, + DataItem::InternalDate { .. } => ITEM_FRAMING_LEN + 28, + DataItem::Uid { .. } => ITEM_FRAMING_LEN, + DataItem::Rfc822 { contents } + | DataItem::Rfc822Header { contents } + | DataItem::Rfc822Text { contents } => ITEM_FRAMING_LEN + INT_LEN + contents.len(), + DataItem::Rfc822Size { .. } => ITEM_FRAMING_LEN + SIZE_LEN, + DataItem::Preview { contents } => { + ITEM_FRAMING_LEN + + contents + .as_ref() + .map_or(0, |contents| contents.len() + INT_LEN) + } + DataItem::ModSeq { .. } => ITEM_FRAMING_LEN + SIZE_LEN, + DataItem::ObjectId(_) => OBJECT_ID_LEN, + } + } +} + +impl FetchItem<'_> { + pub(crate) fn size_hint(&self) -> usize { + ITEM_FRAMING_LEN + + self + .items + .iter() + .map(|item| item.size_hint() + 1) + .sum::() + } +} + diff --git a/crates/imap-proto/src/protocol/list.rs b/crates/imap-proto/src/protocol/list.rs index 4446a793..3d49bda4 100644 --- a/crates/imap-proto/src/protocol/list.rs +++ b/crates/imap-proto/src/protocol/list.rs @@ -3,25 +3,26 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - -use crate::utf7::utf7_encode; +use compact_str::CompactString; use super::{ - ImapResponse, quoted_string, + ImapResponse, MailboxObjectId, ObjectId, + metadata::{Entry, EntryValue, serialize_entry_values}, + quoted_mailbox_name, quoted_string, status::{Status, StatusItem}, }; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Arguments { Basic { - tag: String, - reference_name: String, - mailbox_name: String, + tag: CompactString, + reference_name: CompactString, + mailbox_name: CompactString, }, Extended { - tag: String, - reference_name: String, - mailbox_name: Vec, + tag: CompactString, + reference_name: CompactString, + mailbox_name: Vec, selection_options: Vec, return_options: Vec, }, @@ -50,6 +51,8 @@ pub enum ReturnOption { Children, Status(Vec), SpecialUse, + Metadata(Vec>), + ObjectId, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -89,9 +92,11 @@ pub enum Tag { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListItem { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub attributes: Vec, pub tags: Vec, + pub objectid: Option, + pub metadata: Option]>>, } impl Arguments { @@ -106,11 +111,33 @@ impl Arguments { mailbox_name, reference_name, .. - } => mailbox_name.is_empty() && reference_name.is_empty(), + } => { + reference_name.is_empty() + && mailbox_name + .iter() + .all(|mailbox_name| mailbox_name.is_empty()) + } } } - pub fn unwrap_tag(self) -> String { + pub fn requests_objectid(&self) -> bool { + match self { + Arguments::Basic { .. } => false, + Arguments::Extended { return_options, .. } => { + return_options.iter().any(|option| match option { + ReturnOption::ObjectId => true, + ReturnOption::Status(status) => status.contains(&Status::ObjectId), + _ => false, + }) + } + } + } + + pub fn returns_objectid(&self) -> bool { + matches!(self, Arguments::Extended { return_options, .. } if return_options.contains(&ReturnOption::ObjectId)) + } + + pub fn unwrap_tag(self) -> CompactString { match self { Arguments::Basic { tag, .. } => tag, Arguments::Extended { tag, .. } => tag, @@ -127,7 +154,11 @@ impl Attribute { } pub fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(match self { + buf.extend_from_slice(self.as_bytes()); + } + + pub fn as_bytes(&self) -> &'static [u8] { + match self { Attribute::NoInferiors => b"\\NoInferiors", Attribute::NoSelect => b"\\NoSelect", Attribute::Marked => b"\\Marked", @@ -148,7 +179,7 @@ impl Attribute { Attribute::Memos => b"\\Memos", Attribute::Scheduled => b"\\Scheduled", Attribute::Snoozed => b"\\Snoozed", - }); + } } } @@ -156,7 +187,7 @@ impl TryFrom<&str> for Attribute { type Error = (); fn try_from(value: &str) -> Result { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), Attribute, "archive" => Attribute::Archive, "drafts" => Attribute::Drafts, "junk" => Attribute::Junk, @@ -167,6 +198,7 @@ impl TryFrom<&str> for Attribute { "scheduled" => Attribute::Scheduled, "snoozed" => Attribute::Snoozed, ) + .copied() .ok_or(()) } } @@ -204,16 +236,17 @@ impl Tag { } impl ListItem { - pub fn new(name: impl Into) -> Self { + pub fn new(name: impl Into) -> Self { ListItem { mailbox_name: name.into(), attributes: Vec::new(), tags: Vec::new(), + objectid: None, + metadata: None, } } pub fn serialize(&self, buf: &mut Vec, is_rev2: bool, is_utf8: bool, is_lsub: bool) { - let normalized_mailbox_name = utf7_encode(&self.mailbox_name); if !is_lsub { buf.extend_from_slice(b"* LIST ("); } else { @@ -226,189 +259,104 @@ impl ListItem { attr.serialize(buf); } buf.extend_from_slice(b") \"/\" "); - let mut extra_tags = Vec::new(); - if normalized_mailbox_name != self.mailbox_name { - if is_rev2 || is_utf8 { - quoted_string(buf, &self.mailbox_name); - if is_rev2 { - extra_tags.push(Tag::OldName(normalized_mailbox_name)); - } - } else { - quoted_string(buf, &normalized_mailbox_name); - } - } else { - quoted_string(buf, &self.mailbox_name); - } + // RFC 9051 5.1 drops modified UTF-7, so a rev2 or UTF8=ACCEPT client is sent + // the name as stored. OLDNAME is reserved for RENAME and for Net-Unicode + // normalization (RFC 9051 6.3.9.7), neither of which a re-encoding is. + quoted_mailbox_name(buf, &self.mailbox_name, is_rev2 || is_utf8); - if !extra_tags.is_empty() || !self.tags.is_empty() { + if !self.tags.is_empty() || self.objectid.is_some() { buf.extend_from_slice(b" ("); - for (pos, tag) in extra_tags.iter().chain(self.tags.iter()).enumerate() { + for (pos, tag) in self.tags.iter().enumerate() { if pos > 0 { buf.push(b' '); } tag.serialize(buf); } + if let Some(objectid) = &self.objectid { + if !self.tags.is_empty() { + buf.push(b' '); + } + buf.extend_from_slice(b"\"OBJECTID\" "); + ObjectId::from(*objectid).serialize_kvpairs(buf); + } buf.extend_from_slice(b")\r\n"); } else { buf.extend_from_slice(b"\r\n"); } + + if let Some(metadata) = self.metadata.as_deref() + && !metadata.is_empty() + { + serialize_entry_values(buf, &self.mailbox_name, metadata, is_rev2 || is_utf8); + } } } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(100); - - match (self.list_items.is_empty(), self.status_items.is_empty()) { - (false, false) => { - for (list_item, status_item) in self.list_items.iter().zip(self.status_items.iter()) - { - list_item.serialize(&mut buf, self.is_rev2, self.is_utf8, self.is_lsub); - status_item.serialize(&mut buf, self.is_rev2); - } + fn serialize_into(&self, buf: &mut Vec) { + let mut status_items = self.status_items.iter().peekable(); + for list_item in &self.list_items { + list_item.serialize(buf, self.is_rev2, self.is_utf8, self.is_lsub); + if let Some(status_item) = status_items + .next_if(|status_item| status_item.mailbox_name == list_item.mailbox_name) + { + status_item.serialize(buf, self.is_rev2); } - (false, true) => { - for list_item in &self.list_items { - list_item.serialize(&mut buf, self.is_rev2, self.is_utf8, self.is_lsub); - } - } - (true, false) => { - for status_item in &self.status_items { - status_item.serialize(&mut buf, self.is_rev2); - } - } - _ => (), } - - buf - } -} - -#[cfg(test)] -mod tests { - - use crate::protocol::{ - ImapResponse, - status::{Status, StatusItem, StatusItemType}, - }; - - use super::{Attribute, ChildInfo, ListItem, Tag}; - - #[test] - fn serialize_list_item() { - for (response, expected_v2, expected_v1) in [ - ( - super::ListItem { - mailbox_name: "".into(), - attributes: vec![], - tags: vec![], - }, - "* LIST () \"/\" \"\"\r\n", - "* LIST () \"/\" \"\"\r\n", - ), - ( - super::ListItem { - mailbox_name: "中國書店".into(), - attributes: vec![Attribute::NoInferiors, Attribute::Drafts], - tags: vec![], - }, - concat!( - "* LIST (\\NoInferiors \\Drafts) \"/\" \"中國書店\" ", - "(\"OLDNAME\" (\"&Ti1XC2b4Xpc-\"))\r\n" - ), - "* LIST (\\NoInferiors \\Drafts) \"/\" \"&Ti1XC2b4Xpc-\"\r\n", - ), - ( - super::ListItem { - mailbox_name: "☺".into(), - attributes: vec![Attribute::Subscribed, Attribute::Remote], - tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], - }, - concat!( - "* LIST (\\Subscribed \\Remote) \"/\" \"☺\" ", - "(\"OLDNAME\" (\"&Jjo-\") \"CHILDINFO\" (\"SUBSCRIBED\"))\r\n" - ), - concat!( - "* LIST (\\Subscribed \\Remote) \"/\" \"&Jjo-\" ", - "(\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n" - ), - ), - ( - super::ListItem { - mailbox_name: "foo".into(), - attributes: vec![Attribute::HasNoChildren], - tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], - }, - "* LIST (\\HasNoChildren) \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", - "* LIST (\\HasNoChildren) \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", - ), - ] { - let mut buf_1 = Vec::with_capacity(100); - let mut buf_2 = Vec::with_capacity(100); - - response.serialize(&mut buf_1, false, false, false); - response.serialize(&mut buf_2, true, true, false); - - let response_v1 = String::from_utf8(buf_1).unwrap(); - let response_v2 = String::from_utf8(buf_2).unwrap(); - - assert_eq!(response_v2, expected_v2); - assert_eq!(response_v1, expected_v1); + for status_item in status_items { + status_item.serialize(buf, self.is_rev2); } } - #[test] - fn serialize_list() { - let mut response = super::Response { - list_items: vec![ - ListItem { - mailbox_name: "INBOX".into(), - attributes: vec![Attribute::Subscribed], - tags: vec![], - }, - ListItem { - mailbox_name: "foo".into(), - attributes: vec![], - tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], - }, - ], - status_items: vec![ - StatusItem { - mailbox_name: "INBOX".into(), - items: vec![(Status::Messages, StatusItemType::Number(17))], - }, - StatusItem { - mailbox_name: "foo".into(), - items: vec![ - (Status::Messages, StatusItemType::Number(30)), - (Status::Unseen, StatusItemType::Number(29)), - ], - }, - ], - is_lsub: false, - is_rev2: true, - is_utf8: true, - }; - let expected_v2 = concat!( - "* LIST (\\Subscribed) \"/\" \"INBOX\"\r\n", - "* STATUS \"INBOX\" (MESSAGES 17)\r\n", - "* LIST () \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", - "* STATUS \"foo\" (MESSAGES 30 UNSEEN 29)\r\n", - ); - let expected_v1 = concat!( - "* LSUB (\\Subscribed) \"/\" \"INBOX\"\r\n", - "* LSUB () \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", - ); + fn size_hint(&self) -> usize { + const LIST_FRAMING_LEN: usize = 24; + const ATTRIBUTE_LEN: usize = 16; + const TAG_LEN: usize = 24; + const OBJECT_ID_LEN: usize = 56; + const STATUS_FRAMING_LEN: usize = 24; + const STATUS_ITEM_LEN: usize = 24; + const METADATA_FRAMING_LEN: usize = 20; + const METADATA_ENTRY_LEN: usize = 16; - let response_v2 = String::from_utf8(response.clone().serialize()).unwrap(); - response.is_rev2 = false; - response.is_utf8 = false; - response.is_lsub = true; - response.status_items.clear(); - let response_v1 = String::from_utf8(response.serialize()).unwrap(); - - assert_eq!(response_v2, expected_v2); - assert_eq!(response_v1, expected_v1); + self.list_items + .iter() + .map(|item| { + LIST_FRAMING_LEN + + item.mailbox_name.len() * 2 + + item.attributes.len() * ATTRIBUTE_LEN + + item.tags.len() * TAG_LEN + + if item.objectid.is_some() { + OBJECT_ID_LEN + } else { + 0 + } + + match item.metadata.as_deref() { + Some(metadata) if !metadata.is_empty() => { + METADATA_FRAMING_LEN + + item.mailbox_name.len() * 2 + + metadata + .iter() + .map(|entry| { + METADATA_ENTRY_LEN + + entry.entry.path.len() + + entry.value.as_ref().map_or(0, |value| value.len()) + }) + .sum::() + } + _ => 0, + } + }) + .sum::() + + self + .status_items + .iter() + .map(|item| { + STATUS_FRAMING_LEN + + item.mailbox_name.len() * 2 + + item.items.len() * STATUS_ITEM_LEN + }) + .sum::() } } + diff --git a/crates/imap-proto/src/protocol/login.rs b/crates/imap-proto/src/protocol/login.rs index bbe51501..3990e815 100644 --- a/crates/imap-proto/src/protocol/login.rs +++ b/crates/imap-proto/src/protocol/login.rs @@ -3,10 +3,11 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub username: String, - pub password: String, + pub tag: CompactString, + pub username: CompactString, + pub password: CompactString, } diff --git a/crates/imap-proto/src/protocol/metadata.rs b/crates/imap-proto/src/protocol/metadata.rs new file mode 100644 index 00000000..558c0e6f --- /dev/null +++ b/crates/imap-proto/src/protocol/metadata.rs @@ -0,0 +1,177 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{literal_string, literal8_string, push_int, quoted_mailbox_name, quoted_string}; +use compact_str::CompactString; +use std::borrow::Cow; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GetArguments { + pub tag: CompactString, + pub mailbox_name: CompactString, + pub max_size: Option, + pub depth: Depth, + pub entries: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SetArguments { + pub tag: CompactString, + pub mailbox_name: CompactString, + pub entries: Vec>, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Depth { + #[default] + Zero, + One, + Infinity, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Scope { + Shared, + Private, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Entry<'x> { + pub scope: Scope, + pub path: Cow<'x, str>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EntryValue<'x> { + pub entry: Entry<'x>, + pub value: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Response<'x> { + pub mailbox_name: &'x str, + pub entries: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnsolicitedResponse<'x> { + pub mailbox_name: &'x str, + pub entries: Vec>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetadataCode { + LongEntries(u32), + MaxSize(u32), + TooMany, + NoPrivate, +} + +impl Scope { + pub fn as_str(&self) -> &'static str { + match self { + Scope::Shared => "/shared", + Scope::Private => "/private", + } + } +} + +impl Entry<'_> { + pub fn serialize(&self, buf: &mut Vec) { + let scope = self.scope.as_str(); + if self.path.bytes().all(is_astring_char) { + buf.extend_from_slice(scope.as_bytes()); + buf.extend_from_slice(self.path.as_bytes()); + } else { + quoted_string(buf, &[scope, &self.path].concat()); + } + } +} + +impl EntryValue<'_> { + pub fn into_owned(self) -> EntryValue<'static> { + EntryValue { + entry: Entry { + scope: self.entry.scope, + path: Cow::Owned(self.entry.path.into_owned()), + }, + value: self.value.map(|value| Cow::Owned(value.into_owned())), + } + } + + pub fn serialize(&self, buf: &mut Vec) { + self.entry.serialize(buf); + buf.push(b' '); + match self.value.as_deref() { + Some(value) if value.iter().all(|ch| (0x20..0x7f).contains(ch)) => { + quoted_string(buf, std::str::from_utf8(value).unwrap_or_default()) + } + Some(value) if value.contains(&0) => literal8_string(buf, value), + Some(value) => literal_string(buf, value), + None => buf.extend_from_slice(b"NIL"), + } + } +} + +impl Response<'_> { + pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { + serialize_entry_values(buf, self.mailbox_name, &self.entries, is_utf8); + } +} + +pub(crate) fn serialize_entry_values( + buf: &mut Vec, + mailbox_name: &str, + entries: &[EntryValue<'_>], + is_utf8: bool, +) { + buf.extend_from_slice(b"* METADATA "); + quoted_mailbox_name(buf, mailbox_name, is_utf8); + buf.extend_from_slice(b" ("); + for (pos, entry) in entries.iter().enumerate() { + if pos > 0 { + buf.push(b' '); + } + entry.serialize(buf); + } + buf.extend_from_slice(b")\r\n"); +} + +impl UnsolicitedResponse<'_> { + pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { + buf.extend_from_slice(b"* METADATA "); + quoted_mailbox_name(buf, self.mailbox_name, is_utf8); + for entry in &self.entries { + buf.push(b' '); + entry.serialize(buf); + } + buf.extend_from_slice(b"\r\n"); + } +} + +impl MetadataCode { + pub fn serialize(&self, buf: &mut Vec) { + buf.extend_from_slice(self.as_str().as_bytes()); + if let MetadataCode::LongEntries(size) | MetadataCode::MaxSize(size) = self { + buf.push(b' '); + push_int(buf, *size); + } + } + + pub fn as_str(&self) -> &'static str { + match self { + MetadataCode::LongEntries(_) => "METADATA LONGENTRIES", + MetadataCode::MaxSize(_) => "METADATA MAXSIZE", + MetadataCode::TooMany => "METADATA TOOMANY", + MetadataCode::NoPrivate => "METADATA NOPRIVATE", + } + } +} + +fn is_astring_char(ch: u8) -> bool { + ch > 0x20 && ch < 0x7f && !matches!(ch, b'(' | b')' | b'{' | b'"' | b'\\' | b'%' | b'*') +} + diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index d3f4dab2..f414b16d 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -4,15 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Command, ResponseCode, ResponseType, StatusResponse}; -use ahash::AHashSet; -use base64::{Engine, engine::general_purpose::STANDARD}; -use chrono::{DateTime, Utc}; +use crate::{ + Command, ResponseCode, ResponseType, StatusResponse, protocol::metadata::MetadataCode, +}; use compact_str::CompactString; -use std::{cmp::Ordering, fmt::Display}; +use encodify::{base32::STALWART, base64::STANDARD, utf7::IMAP}; +use std::fmt::Display; use types::id::Id; -use types::keyword::{ArchivedKeyword, Keyword}; -use utils::chained_bytes::SliceRange; +use types::keyword::Keyword; +use utils::chained_bytes::ChainedBytes; pub mod acl; pub mod append; @@ -26,6 +26,7 @@ pub mod expunge; pub mod fetch; pub mod list; pub mod login; +pub mod metadata; pub mod namespace; pub mod quota; pub mod rename; @@ -51,7 +52,38 @@ pub struct ObjectId { pub thread_id: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MailboxObjectId { + Empty, + Mailbox { account_id: u32, mailbox_id: u32 }, +} + +const _: () = assert!(size_of::>() <= 12); + +impl From for ObjectId { + fn from(value: MailboxObjectId) -> Self { + match value { + MailboxObjectId::Empty => ObjectId::default(), + MailboxObjectId::Mailbox { + account_id, + mailbox_id, + } => ObjectId { + account_id: Some(Id::from(account_id)), + mailbox_id: Some(Id::from(mailbox_id)), + ..Default::default() + }, + } + } +} + impl ObjectId { + pub fn decode_id(value: &[u8]) -> Option { + match value { + [b'a', _, ..] => None, + _ => STALWART.decode_u64(value).ok().map(Id::new), + } + } + pub fn is_empty(&self) -> bool { self.mailbox_id.is_none() && self.account_id.is_none() @@ -74,7 +106,7 @@ impl ObjectId { } first = false; buf.extend_from_slice(key); - buf.extend_from_slice(value.to_string().as_bytes()); + STALWART.encode_u64_append(value.id(), buf); } } buf.push(b')'); @@ -141,83 +173,92 @@ impl Sequence { _ => false, } } +} - pub fn expand(&self, max_value: u32) -> AHashSet { - match self { - Sequence::Number { value } => AHashSet::from_iter([*value]), - Sequence::List { items } => { - let mut result = AHashSet::with_capacity(items.len()); - for item in items { - match item { - Sequence::Number { value } => { - result.insert(*value); - } - Sequence::Range { start, end } => { - let start = start.unwrap_or(max_value); - let end = end.unwrap_or(max_value); - match start.cmp(&end) { - Ordering::Equal => { - result.insert(start); - } - Ordering::Less => { - result.extend(start..=end); - } - Ordering::Greater => { - result.extend(end..=start); - } - } - } - _ => (), - } - } - result - } - Sequence::Range { start, end } => { - let mut result = AHashSet::new(); - let start = start.unwrap_or(max_value); - let end = end.unwrap_or(max_value); - match start.cmp(&end) { - Ordering::Equal => { - result.insert(start); - } - Ordering::Less => { - result.extend(start..=end); - } - Ordering::Greater => { - result.extend(end..=start); - } - } - result - } - _ => AHashSet::new(), - } +const DEFAULT_RESPONSE_CAPACITY: usize = 128; + +pub trait ImapResponse { + fn serialize_into(&self, buf: &mut Vec); + + fn size_hint(&self) -> usize { + DEFAULT_RESPONSE_CAPACITY + } + + fn serialize(self) -> Vec + where + Self: Sized, + { + let mut buf = Vec::with_capacity(self.size_hint()); + self.serialize_into(&mut buf); + buf } } -pub trait ImapResponse { - fn serialize(self) -> Vec; +const fn byte_class_table(needles: &[u8]) -> [bool; 256] { + let mut table = [false; 256]; + let mut pos = 0; + while pos < needles.len() { + table[needles[pos] as usize] = true; + pos += 1; + } + table +} + +static NEEDS_ESCAPE: [bool; 256] = byte_class_table(b"\\\"\0"); +static NEEDS_LITERAL: [bool; 256] = byte_class_table(b"\\\"\r\n\0"); + +#[inline(always)] +fn find_escape(text: &[u8]) -> Option { + text.iter().position(|&ch| NEEDS_ESCAPE[ch as usize]) +} + +#[cold] +#[inline(never)] +fn without_nul(text: &str) -> String { + text.replace('\0', "") } pub fn quoted_string(buf: &mut Vec, text: &str) { + let mut rest = text.as_bytes(); buf.push(b'"'); - for &c in text.as_bytes() { - if c == b'\\' || c == b'"' { + while let Some(pos) = find_escape(rest) { + let (head, tail) = rest.split_at(pos); + buf.extend_from_slice(head); + let mut tail = tail.iter(); + if let Some(&ch) = tail.next().filter(|&&ch| ch != 0) { buf.push(b'\\'); + buf.push(ch); } - buf.push(c); + rest = tail.as_slice(); } + buf.extend_from_slice(rest); buf.push(b'"'); } -pub fn quoted_or_literal_string(buf: &mut Vec, text: &str) { - if text.as_bytes().iter().any(|ch| b"\\\"\r\n".contains(ch)) { - literal_string(buf, text.as_bytes()) +pub(crate) fn quoted_mailbox_name(buf: &mut Vec, name: &str, is_utf8: bool) { + if is_utf8 { + quoted_string(buf, name); + } else if find_escape(name.as_bytes()).is_some() { + quoted_string(buf, &IMAP.encode(name)); } else { buf.push(b'"'); - buf.extend_from_slice(text.as_bytes()); + IMAP.encode_append(name, buf); buf.push(b'"'); } } + +pub fn quoted_or_literal_string(buf: &mut Vec, text: &str) { + let bytes = text.as_bytes(); + if !bytes.iter().any(|&ch| NEEDS_LITERAL[ch as usize]) { + buf.push(b'"'); + buf.extend_from_slice(bytes); + buf.push(b'"'); + } else if bytes.contains(&0) { + quoted_or_literal_string(buf, &without_nul(text)); + } else { + push_literal(buf, bytes); + } +} pub fn quoted_or_literal_string_or_nil(buf: &mut Vec, text: Option<&str>) { if let Some(text) = text { quoted_or_literal_string(buf, text); @@ -226,13 +267,52 @@ pub fn quoted_or_literal_string_or_nil(buf: &mut Vec, text: Option<&str>) { } } +const CLASS_LITERAL: u8 = 1; +const CLASS_NON_ASCII: u8 = 2; +const CLASS_NUL: u8 = 4; + +const fn text_class_table() -> [u8; 256] { + let mut table = [0u8; 256]; + table[b'\\' as usize] = CLASS_LITERAL; + table[b'"' as usize] = CLASS_LITERAL; + table[b'\r' as usize] = CLASS_LITERAL; + table[b'\n' as usize] = CLASS_LITERAL; + table[0] = CLASS_NUL; + let mut ch = 0x80; + while ch < 256 { + table[ch] |= CLASS_NON_ASCII; + ch += 1; + } + table +} + +static TEXT_CLASS: [u8; 256] = text_class_table(); + +fn push_base64_encoded(buf: &mut Vec, text: &[u8]) { + buf.extend_from_slice(b"\"=?utf-8?B?"); + STANDARD.encode_append(text, buf); + buf.extend_from_slice(b"?=\""); +} + pub fn quoted_or_literal_encoded_string(buf: &mut Vec, text: &str, is_utf8: bool) { - if is_utf8 || text.is_ascii() { + if is_utf8 { quoted_or_literal_string(buf, text); + return; + } + let bytes = text.as_bytes(); + let class = bytes + .iter() + .fold(0u8, |class, &ch| class | TEXT_CLASS[ch as usize]); + if class == 0 { + buf.push(b'"'); + buf.extend_from_slice(bytes); + buf.push(b'"'); + } else if class & CLASS_NUL != 0 { + quoted_or_literal_encoded_string(buf, &without_nul(text), is_utf8); + } else if class & CLASS_NON_ASCII != 0 { + push_base64_encoded(buf, bytes); } else { - buf.extend_from_slice(b"\"=?utf-8?B?"); - buf.extend_from_slice(STANDARD.encode(text.as_bytes()).as_bytes()); - buf.extend_from_slice(b"?=\""); + push_literal(buf, bytes); } } @@ -248,6 +328,28 @@ pub fn quoted_or_literal_encoded_string_or_nil( } } +pub fn quoted_or_literal_raw_string(buf: &mut Vec, text: &str, is_utf8: bool) { + if is_utf8 { + quoted_or_literal_string(buf, text); + } else if !text.bytes().any(|ch| TEXT_CLASS[ch as usize] != 0) { + buf.push(b'"'); + buf.extend_from_slice(text.as_bytes()); + buf.push(b'"'); + } else if text.as_bytes().contains(&0) { + quoted_or_literal_raw_string(buf, &without_nul(text), is_utf8); + } else { + push_literal(buf, text.as_bytes()); + } +} + +pub fn quoted_or_literal_raw_string_or_nil(buf: &mut Vec, text: Option<&str>, is_utf8: bool) { + if let Some(text) = text { + quoted_or_literal_raw_string(buf, text, is_utf8); + } else { + buf.extend_from_slice(b"NIL"); + } +} + pub fn quoted_string_or_nil(buf: &mut Vec, text: Option<&str>) { if let Some(text) = text { quoted_string(buf, text); @@ -257,43 +359,164 @@ pub fn quoted_string_or_nil(buf: &mut Vec, text: Option<&str>) { } pub fn literal_string(buf: &mut Vec, text: &[u8]) { - buf.push(b'{'); - buf.extend_from_slice(text.len().to_string().as_bytes()); - buf.extend_from_slice(b"}\r\n"); - buf.extend_from_slice(text); + if text.contains(&0) { + let text = text + .iter() + .copied() + .filter(|&ch| ch != 0) + .collect::>(); + push_literal(buf, &text); + } else { + push_literal(buf, text); + } } -pub fn literal_string_slice(buf: &mut Vec, text: &SliceRange<'_>) { +pub fn literal8_string(buf: &mut Vec, bytes: &[u8]) { + buf.push(b'~'); + push_literal(buf, bytes); +} + +fn push_literal(buf: &mut Vec, bytes: &[u8]) { buf.push(b'{'); - buf.extend_from_slice(text.len().to_string().as_bytes()); + push_int(buf, bytes.len()); buf.extend_from_slice(b"}\r\n"); - buf.extend(*text); + buf.extend_from_slice(bytes); +} + +const LITERAL_FRAMING_LEN: usize = 24; + +pub trait WriteLiteral { + fn write_literal(&self, buf: &mut Vec); +} + +impl WriteLiteral for ChainedBytes<'_> { + fn write_literal(&self, buf: &mut Vec) { + buf.reserve(self.len() + LITERAL_FRAMING_LEN); + buf.push(b'{'); + push_int(buf, self.len()); + buf.extend_from_slice(b"}\r\n"); + self.extend_into(buf); + } +} + +pub fn push_int(buf: &mut Vec, value: impl itoa::Integer) { + let mut int_buf = itoa::Buffer::new(); + buf.extend_from_slice(int_buf.format(value).as_bytes()); +} + +const MONTHS_ABBREVIATED: [&[u8; 3]; 12] = [ + b"Jan", b"Feb", b"Mar", b"Apr", b"May", b"Jun", b"Jul", b"Aug", b"Sep", b"Oct", b"Nov", b"Dec", +]; + +const DAYS_ABBREVIATED: [&[u8; 3]; 7] = [b"Sun", b"Mon", b"Tue", b"Wed", b"Thu", b"Fri", b"Sat"]; + +#[inline(always)] +fn two_digits(value: u32) -> [u8; 2] { + [b'0' + (value / 10) as u8, b'0' + (value % 10) as u8] +} + +#[inline(always)] +fn push_padded_u8(buf: &mut Vec, value: u8) { + if value >= 100 { + buf.push(b'0' + value / 100); + } + buf.push(b'0' + (value / 10) % 10); + buf.push(b'0' + value % 10); +} + +#[inline(always)] +fn push_plain_u8(buf: &mut Vec, value: u8) { + if value >= 100 { + buf.push(b'0' + value / 100); + } + if value >= 10 { + buf.push(b'0' + (value / 10) % 10); + } + buf.push(b'0' + value % 10); +} + +#[inline(always)] +fn push_padded_u16(buf: &mut Vec, value: u16) { + if value >= 10_000 { + buf.push(b'0' + (value / 10_000) as u8); + } + buf.push(b'0' + ((value / 1_000) % 10) as u8); + buf.push(b'0' + ((value / 100) % 10) as u8); + buf.push(b'0' + ((value / 10) % 10) as u8); + buf.push(b'0' + (value % 10) as u8); +} + +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let shifted = days + 719_468; + let era = if shifted >= 0 { + shifted + } else { + shifted - 146_096 + } / 146_097; + let day_of_era = shifted - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let shifted_month = (5 * day_of_year + 2) / 153; + let day = (day_of_year - (153 * shifted_month + 2) / 5 + 1) as u32; + let month = if shifted_month < 10 { + shifted_month + 3 + } else { + shifted_month - 9 + } as u32; + let year = year_of_era + era * 400; + (if month <= 2 { year + 1 } else { year }, month, day) } pub fn quoted_timestamp(buf: &mut Vec, timestamp: i64) { - buf.push(b'"'); - buf.extend_from_slice( - DateTime::::from_timestamp(timestamp, 0) - .unwrap_or_default() - .format("%d-%b-%Y %H:%M:%S %z") - .to_string() - .as_bytes(), - ); - buf.push(b'"'); + let (year, month, day) = civil_from_days(timestamp.div_euclid(86_400)); + let seconds_of_day = timestamp.rem_euclid(86_400) as u32; + let year = year.clamp(0, 9999) as u32; + let month = MONTHS_ABBREVIATED[(month - 1) as usize]; + let [day_hi, day_lo] = two_digits(day); + let [year_a, year_b] = two_digits(year / 100); + let [year_c, year_d] = two_digits(year % 100); + let [hour_hi, hour_lo] = two_digits(seconds_of_day / 3600); + let [minute_hi, minute_lo] = two_digits((seconds_of_day / 60) % 60); + let [second_hi, second_lo] = two_digits(seconds_of_day % 60); + + buf.extend_from_slice(&[ + b'"', day_hi, day_lo, b'-', month[0], month[1], month[2], b'-', year_a, year_b, year_c, + year_d, b' ', hour_hi, hour_lo, b':', minute_hi, minute_lo, b':', second_hi, second_lo, + b' ', b'+', b'0', b'0', b'0', b'0', b'"', + ]); } pub fn quoted_rfc2822(buf: &mut Vec, timestamp: &mail_parser::DateTime) { + const MAX_LEN: usize = 40; + buf.reserve(MAX_LEN); buf.push(b'"'); - buf.extend_from_slice(timestamp.to_rfc822().as_bytes()); - buf.push(b'"'); -} - -pub fn quoted_rfc2822_or_nil(buf: &mut Vec, timestamp: &Option) { - if let Some(timestamp) = timestamp { - quoted_rfc2822(buf, timestamp); - } else { - buf.extend_from_slice(b"NIL"); + buf.extend_from_slice(DAYS_ABBREVIATED[usize::from(timestamp.day_of_week()) % 7]); + buf.extend_from_slice(b", "); + push_plain_u8(buf, timestamp.day); + buf.push(b' '); + if let Some(month) = MONTHS_ABBREVIATED.get(usize::from(timestamp.month.saturating_sub(1))) { + buf.extend_from_slice(*month); } + buf.push(b' '); + push_padded_u16(buf, timestamp.year); + buf.push(b' '); + push_padded_u8(buf, timestamp.hour); + buf.push(b':'); + push_padded_u8(buf, timestamp.minute); + buf.push(b':'); + push_padded_u8(buf, timestamp.second); + buf.push(b' '); + buf.push( + if timestamp.tz_before_gmt && (timestamp.tz_hour > 0 || timestamp.tz_minute > 0) { + b'-' + } else { + b'+' + }, + ); + push_padded_u8(buf, timestamp.tz_hour); + push_padded_u8(buf, timestamp.tz_minute); + buf.push(b'"'); } #[derive(Debug, Clone, PartialEq, Eq)] @@ -327,10 +550,17 @@ pub enum Flag { New, Notify, Unsubscribed, - Keyword(Box), + Keyword(CompactString), } impl Flag { + pub fn is_serializable(&self) -> bool { + match self { + Flag::Keyword(keyword) => Keyword::is_atom(keyword.as_bytes()), + _ => true, + } + } + pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { Flag::Seen => b"\\Seen", @@ -404,43 +634,6 @@ impl From for Flag { } } -impl From<&ArchivedKeyword> for Flag { - fn from(value: &ArchivedKeyword) -> Self { - match value { - ArchivedKeyword::Seen => Flag::Seen, - ArchivedKeyword::Draft => Flag::Draft, - ArchivedKeyword::Flagged => Flag::Flagged, - ArchivedKeyword::Answered => Flag::Answered, - ArchivedKeyword::Recent => Flag::Recent, - ArchivedKeyword::Important => Flag::Important, - ArchivedKeyword::Phishing => Flag::Phishing, - ArchivedKeyword::Junk => Flag::Junk, - ArchivedKeyword::NotJunk => Flag::NotJunk, - ArchivedKeyword::Deleted => Flag::Deleted, - ArchivedKeyword::Forwarded => Flag::Forwarded, - ArchivedKeyword::MdnSent => Flag::MDNSent, - ArchivedKeyword::Autosent => Flag::Autosent, - ArchivedKeyword::CanUnsubscribe => Flag::CanUnsubscribe, - ArchivedKeyword::Followed => Flag::Followed, - ArchivedKeyword::HasAttachment => Flag::HasAttachment, - ArchivedKeyword::HasMemo => Flag::HasMemo, - ArchivedKeyword::HasNoAttachment => Flag::HasNoAttachment, - ArchivedKeyword::Imported => Flag::Imported, - ArchivedKeyword::IsTrusted => Flag::IsTrusted, - ArchivedKeyword::MailFlagBit0 => Flag::MailFlagBit0, - ArchivedKeyword::MailFlagBit1 => Flag::MailFlagBit1, - ArchivedKeyword::MailFlagBit2 => Flag::MailFlagBit2, - ArchivedKeyword::MaskedEmail => Flag::MaskedEmail, - ArchivedKeyword::Memo => Flag::Memo, - ArchivedKeyword::Muted => Flag::Muted, - ArchivedKeyword::New => Flag::New, - ArchivedKeyword::Notify => Flag::Notify, - ArchivedKeyword::Unsubscribed => Flag::Unsubscribed, - ArchivedKeyword::Other(value) => Flag::Keyword(value.as_ref().into()), - } - } -} - impl From for Keyword { fn from(value: Flag) -> Self { match value { @@ -473,7 +666,7 @@ impl From for Keyword { Flag::New => Keyword::New, Flag::Notify => Keyword::Notify, Flag::Unsubscribed => Keyword::Unsubscribed, - Flag::Keyword(value) => Keyword::from_boxed_other(value), + Flag::Keyword(value) => Keyword::from_compact_string(value), } } } @@ -485,7 +678,7 @@ impl ResponseCode { ResponseCode::AlreadyExists => b"ALREADYEXISTS", ResponseCode::AppendUid { uid_validity, uids } => { buf.extend_from_slice(b"APPENDUID "); - buf.extend_from_slice(uid_validity.to_string().as_bytes()); + push_int(buf, *uid_validity); buf.push(b' '); serialize_sequence(buf, uids); return; @@ -511,7 +704,7 @@ impl ResponseCode { dest_uids, } => { buf.extend_from_slice(b"COPYUID "); - buf.extend_from_slice(uid_validity.to_string().as_bytes()); + push_int(buf, *uid_validity); buf.push(b' '); serialize_sequence(buf, src_uids); buf.push(b' '); @@ -539,9 +732,9 @@ impl ResponseCode { ResponseCode::UidValidity => b"UIDVALIDITY", ResponseCode::Unavailable => b"UNAVAILABLE", ResponseCode::UnknownCte => b"UNKNOWN-CTE", - ResponseCode::Modified { ids } => { + ResponseCode::Modified { ranges } => { buf.extend_from_slice(b"MODIFIED "); - serialize_sequence(buf, ids); + serialize_sequence_ranges(buf, ranges); return; } ResponseCode::ObjectId(object_id) => { @@ -550,22 +743,27 @@ impl ResponseCode { } ResponseCode::HighestModseq { modseq } => { buf.extend_from_slice(b"HIGHESTMODSEQ "); - buf.extend_from_slice(modseq.to_string().as_bytes()); + push_int(buf, *modseq); return; } ResponseCode::UseAttr => b"USEATTR", ResponseCode::UidRequired => b"UIDREQUIRED", ResponseCode::TooFew => b"TOOFEW", ResponseCode::TooMany => b"TOOMANY", + ResponseCode::TooBig => b"TOOBIG", ResponseCode::MessageLimit { limit, uid } => { buf.extend_from_slice(b"MESSAGELIMIT "); - buf.extend_from_slice(limit.to_string().as_bytes()); + push_int(buf, *limit); if let Some(uid) = uid { buf.push(b' '); - buf.extend_from_slice(uid.to_string().as_bytes()); + push_int(buf, *uid); } return; } + ResponseCode::Metadata(code) => { + code.serialize(buf); + return; + } }); } @@ -612,9 +810,36 @@ impl ResponseCode { ResponseCode::UidRequired => "UIDREQUIRED", ResponseCode::TooFew => "TOOFEW", ResponseCode::TooMany => "TOOMANY", + ResponseCode::TooBig => "TOOBIG", ResponseCode::MessageLimit { .. } => "MESSAGELIMIT", + ResponseCode::Metadata(code) => code.as_str(), } } + + fn size_hint(&self) -> usize { + const BRACKETS_LEN: usize = 3; + const INT_LEN: usize = 11; + const CAPABILITY_LEN: usize = 20; + const OBJECT_ID_LEN: usize = 92; + + self.as_str().len() + + BRACKETS_LEN + + match self { + ResponseCode::AppendUid { uids, .. } => INT_LEN * (uids.len() + 1), + ResponseCode::Capability { capabilities } => capabilities.len() * CAPABILITY_LEN, + ResponseCode::CopyUid { + src_uids, + dest_uids, + .. + } => INT_LEN * (src_uids.len() + dest_uids.len() + 1) + 2, + ResponseCode::Modified { ranges } => ranges.len() * (INT_LEN * 2 + 1) + 1, + ResponseCode::ObjectId(_) => OBJECT_ID_LEN, + ResponseCode::HighestModseq { .. } => 21, + ResponseCode::MessageLimit { .. } => INT_LEN * 2 + 1, + ResponseCode::Metadata(_) => INT_LEN + 1, + _ => 0, + } + } } impl ResponseType { @@ -635,7 +860,21 @@ impl ResponseType { impl From for trc::Value { fn from(value: ResponseCode) -> Self { - trc::Value::String(CompactString::const_new(value.as_str())) + trc::Value::String(match &value { + ResponseCode::AppendUid { .. } + | ResponseCode::Capability { .. } + | ResponseCode::CopyUid { .. } + | ResponseCode::Modified { .. } + | ResponseCode::HighestModseq { .. } + | ResponseCode::ObjectId(_) + | ResponseCode::MessageLimit { .. } + | ResponseCode::Metadata(MetadataCode::LongEntries(_) | MetadataCode::MaxSize(_)) => { + let mut buf = Vec::with_capacity(value.size_hint()); + value.serialize(&mut buf); + CompactString::from_utf8_lossy(&buf) + } + _ => CompactString::const_new(value.as_str()), + }) } } @@ -646,27 +885,49 @@ impl From for trc::Value { } impl StatusResponse { - pub fn serialize(self, mut buf: Vec) -> Vec { + pub fn serialize_into(&self, buf: &mut Vec) { if let Some(tag) = &self.tag { buf.extend_from_slice(tag.as_bytes()); } else { buf.push(b'*'); } buf.push(b' '); - self.rtype.serialize(&mut buf); + self.rtype.serialize(buf); buf.push(b' '); if let Some(code) = &self.code { buf.push(b'['); - code.serialize(&mut buf); + code.serialize(buf); buf.extend_from_slice(b"] "); } buf.extend_from_slice(self.message.as_bytes()); buf.extend_from_slice(b"\r\n"); + } + + pub fn serialize(self, mut buf: Vec) -> Vec { + self.serialize_into(&mut buf); buf } pub fn into_bytes(self) -> Vec { - self.serialize(Vec::with_capacity(16)) + let capacity = self.size_hint(); + self.serialize(Vec::with_capacity(capacity)) + } + + pub fn serialize_after(self, response: &impl ImapResponse) -> Vec { + let mut buf = Vec::with_capacity(response.size_hint() + self.size_hint()); + response.serialize_into(&mut buf); + self.serialize_into(&mut buf); + buf + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 4; + + self.tag.as_ref().map_or(1, |tag| tag.len()) + + self.rtype.as_str().len() + + self.message.len() + + self.code.as_ref().map_or(0, ResponseCode::size_hint) + + FRAMING_LEN } } @@ -701,6 +962,9 @@ impl SerializeResponse for trc::Error { trc::EventType::Limit(trc::LimitEvent::Quota) => { Some(ResponseCode::OverQuota.as_str()) } + trc::EventType::Limit(trc::LimitEvent::MemoryBudget) => { + Some(ResponseCode::Unavailable.as_str()) + } trc::EventType::Limit(_) => Some(ResponseCode::Limit.as_str()), trc::EventType::Auth(_) => Some(ResponseCode::AuthenticationFailed.as_str()), trc::EventType::Security(_) => Some(ResponseCode::AuthorizationFailed.as_str()), @@ -734,194 +998,46 @@ impl ProtocolVersion { } pub fn serialize_sequence(buf: &mut Vec, list: &[u32]) { - let mut ids = list.iter().peekable(); - while let Some(&id) = ids.next() { - buf.extend_from_slice(id.to_string().as_bytes()); + let mut rest = list; + while let Some((&id, mut tail)) = rest.split_first() { + push_int(buf, id); + let mut range_id = id; - loop { - match ids.peek() { - Some(&&next_id) if next_id == range_id + 1 => { - range_id += 1; - ids.next(); - } - next => { - if range_id != id { - buf.push(b':'); - buf.extend_from_slice(range_id.to_string().as_bytes()); - } - if next.is_some() { - buf.push(b','); - } - break; - } + while let Some((&next_id, next_tail)) = tail.split_first() { + if next_id != range_id + 1 { + break; } + range_id += 1; + tail = next_tail; + } + + if range_id != id { + buf.push(b':'); + push_int(buf, range_id); + } + if !tail.is_empty() { + buf.push(b','); + } + rest = tail; + } +} + +pub fn serialize_sequence_ranges(buf: &mut Vec, ranges: &[(u32, u32)]) { + for (pos, (from, to)) in ranges.iter().enumerate() { + if pos > 0 { + buf.push(b','); + } + push_int(buf, *from); + if to != from { + buf.push(b':'); + push_int(buf, *to); } } } impl Display for Command { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Command::UidBatches => write!(f, "UIDBATCHES"), - Command::Capability => write!(f, "CAPABILITY"), - Command::Noop => write!(f, "NOOP"), - Command::Logout => write!(f, "LOGOUT"), - Command::StartTls => write!(f, "STARTTLS"), - Command::Authenticate => write!(f, "AUTHENTICATE"), - Command::Login => write!(f, "LOGIN"), - Command::Enable => write!(f, "ENABLE"), - Command::Select => write!(f, "SELECT"), - Command::Examine => write!(f, "EXAMINE"), - Command::Create => write!(f, "CREATE"), - Command::Delete => write!(f, "DELETE"), - Command::Rename => write!(f, "RENAME"), - Command::Subscribe => write!(f, "SUBSCRIBE"), - Command::Unsubscribe => write!(f, "UNSUBSCRIBE"), - Command::List => write!(f, "LIST"), - Command::Namespace => write!(f, "NAMESPACE"), - Command::Status => write!(f, "STATUS"), - Command::Append => write!(f, "APPEND"), - Command::Idle => write!(f, "IDLE"), - Command::Close => write!(f, "CLOSE"), - Command::Unselect => write!(f, "UNSELECT"), - Command::Expunge(false) => write!(f, "EXPUNGE"), - Command::Search(false) => write!(f, "SEARCH"), - Command::Fetch(false) => write!(f, "FETCH"), - Command::Store(false) => write!(f, "STORE"), - Command::Copy(false) => write!(f, "COPY"), - Command::Move(false) => write!(f, "MOVE"), - Command::Sort(false) => write!(f, "SORT"), - Command::Thread(false) => write!(f, "THREAD"), - Command::Expunge(true) => write!(f, "UID EXPUNGE"), - Command::Search(true) => write!(f, "UID SEARCH"), - Command::Fetch(true) => write!(f, "UID FETCH"), - Command::Store(true) => write!(f, "UID STORE"), - Command::Copy(true) => write!(f, "UID COPY"), - Command::Move(true) => write!(f, "UID MOVE"), - Command::Sort(true) => write!(f, "UID SORT"), - Command::Thread(true) => write!(f, "UID THREAD"), - Command::Lsub => write!(f, "LSUB"), - Command::Check => write!(f, "CHECK"), - Command::SetAcl => write!(f, "SETACL"), - Command::DeleteAcl => write!(f, "DELETEACL"), - Command::GetAcl => write!(f, "GETACL"), - Command::ListRights => write!(f, "LISTRIGHTS"), - Command::MyRights => write!(f, "MYRIGHTS"), - Command::Unauthenticate => write!(f, "UNAUTHENTICATE"), - Command::Id => write!(f, "ID"), - Command::GetQuota => write!(f, "GETQUOTA"), - Command::GetQuotaRoot => write!(f, "GETQUOTAROOT"), - Command::GetJmapAccess => write!(f, "GETJMAPACCESS"), - } + f.write_str(self.as_str()) } } -#[cfg(test)] -mod tests { - use crate::parser::parse_sequence_set; - use crate::protocol::{ObjectId, SerializeResponse}; - use types::id::Id; - - #[test] - fn serialize_error_uses_command_tag() { - for (error, expected) in [ - ( - trc::AuthEvent::Failed.into_err().id("a1"), - "a1 NO [AUTHENTICATIONFAILED] ", - ), - ( - trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::Id, 7u32) - .id("a1"), - "a1 NO [AUTHENTICATIONFAILED] ", - ), - ( - trc::AuthEvent::Error - .into_err() - .ctx(trc::Key::Id, "12") - .id("a2"), - "a2 NO [AUTHENTICATIONFAILED] ", - ), - ( - trc::AuthEvent::TooManyAttempts - .into_err() - .caused_by(trc::AuthEvent::Failed.into_err().ctx(trc::Key::Id, 7u32)) - .id("a3"), - "a3 NO [AUTHENTICATIONFAILED] ", - ), - ( - trc::AuthEvent::Failed.into_err().ctx(trc::Key::Id, 7u32), - "* NO [AUTHENTICATIONFAILED] ", - ), - ] { - let response = error.serialize(); - assert!( - response.starts_with(expected.as_bytes()), - "{:?} does not start with {expected:?}", - String::from_utf8_lossy(&response) - ); - } - } - - #[test] - fn serialize_objectid_compound() { - // Empty compound - let mut buf = Vec::new(); - ObjectId::default().serialize(&mut buf); - assert_eq!(String::from_utf8(buf).unwrap(), "OBJECTID ()"); - - // Mailbox context: MAILBOXID + ACCOUNTID - let mut buf = Vec::new(); - ObjectId { - mailbox_id: Some(Id::from(1u32)), - account_id: Some(Id::from(2u32)), - ..Default::default() - } - .serialize(&mut buf); - assert_eq!( - String::from_utf8(buf).unwrap(), - format!( - "OBJECTID (ACCOUNTID {} MAILBOXID {})", - Id::from(2u32), - Id::from(1u32) - ) - ); - - // Message context: EMAILID + THREADID only - let mut buf = Vec::new(); - ObjectId { - email_id: Some(Id::from_parts(3, 4)), - thread_id: Some(Id::from(3u32)), - ..Default::default() - } - .serialize(&mut buf); - assert_eq!( - String::from_utf8(buf).unwrap(), - format!( - "OBJECTID (EMAILID {} THREADID {})", - Id::from_parts(3, 4), - Id::from(3u32) - ) - ); - } - - #[test] - fn sequence_set_contains() { - for (sequence, expected_result, max_value) in [ - ("1,5:10", vec![1, 5, 6, 7, 8, 9, 10], 10), - ("2,4:7,9,12:*", vec![2, 4, 5, 6, 7, 9, 12, 13, 14, 15], 15), - ("*:4,5:7", vec![4, 5, 6, 7], 7), - ("2,4,5", vec![2, 4, 5], 5), - ] { - let sequence = parse_sequence_set(sequence.as_bytes()).unwrap(); - - assert_eq!( - (1..=15) - .filter(|num| sequence.contains(*num, max_value)) - .collect::>(), - expected_result - ); - } - } -} diff --git a/crates/imap-proto/src/protocol/namespace.rs b/crates/imap-proto/src/protocol/namespace.rs index a6e16b6c..da8ed1f5 100644 --- a/crates/imap-proto/src/protocol/namespace.rs +++ b/crates/imap-proto/src/protocol/namespace.rs @@ -11,15 +11,23 @@ pub struct Response { } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); + fn serialize_into(&self, buf: &mut Vec) { if let Some(shared_prefix) = &self.shared_prefix { buf.extend_from_slice(b"* NAMESPACE ((\"\" \"/\")) (("); - quoted_string(&mut buf, shared_prefix); + quoted_string(buf, shared_prefix); buf.extend_from_slice(b" \"/\")) NIL\r\n"); } else { buf.extend_from_slice(b"* NAMESPACE ((\"\" \"/\")) NIL NIL\r\n"); } - buf + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 48; + + FRAMING_LEN + + self + .shared_prefix + .as_ref() + .map_or(0, |prefix| prefix.len() * 2 + 2) } } diff --git a/crates/imap-proto/src/protocol/quota.rs b/crates/imap-proto/src/protocol/quota.rs index 6399544b..105ac999 100644 --- a/crates/imap-proto/src/protocol/quota.rs +++ b/crates/imap-proto/src/protocol/quota.rs @@ -3,17 +3,18 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; -use super::{ImapResponse, capability::QuotaResourceName, quoted_string}; +use super::{ImapResponse, capability::QuotaResourceName, push_int, quoted_string}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub name: String, + pub tag: CompactString, + pub name: CompactString, } pub struct QuotaItem { - pub name: String, + pub name: CompactString, pub resources: Vec, } @@ -24,18 +25,17 @@ pub struct QuotaResource { } pub struct Response { - pub quota_root_items: Vec, + pub quota_root_items: Vec, pub quota_items: Vec, } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); + fn serialize_into(&self, buf: &mut Vec) { if !self.quota_root_items.is_empty() { buf.extend_from_slice(b"* QUOTAROOT"); for item in &self.quota_root_items { buf.push(b' '); - quoted_string(&mut buf, item); + quoted_string(buf, item); } buf.extend_from_slice(b"\r\n"); } @@ -43,7 +43,7 @@ impl ImapResponse for Response { if !self.quota_items.is_empty() { for item in &self.quota_items { buf.extend_from_slice(b"* QUOTA "); - quoted_string(&mut buf, &item.name); + quoted_string(buf, &item.name); buf.extend_from_slice(b" ("); for (pos, resource) in item.resources.iter().enumerate() { if pos > 0 { @@ -67,72 +67,38 @@ impl ImapResponse for Response { } } - buf.extend_from_slice(format!("{used} {total}").as_bytes()); + push_int(buf, used); + buf.push(b' '); + push_int(buf, total); } buf.extend_from_slice(b")\r\n"); } } + } - buf + fn size_hint(&self) -> usize { + const QUOTA_ROOT_FRAMING_LEN: usize = 16; + const QUOTA_FRAMING_LEN: usize = 16; + const RESOURCE_LEN: usize = 64; + + let quota_roots = if self.quota_root_items.is_empty() { + 0 + } else { + QUOTA_ROOT_FRAMING_LEN + + self + .quota_root_items + .iter() + .map(|item| item.len() + 4) + .sum::() + }; + quota_roots + + self + .quota_items + .iter() + .map(|item| { + QUOTA_FRAMING_LEN + item.name.len() + 4 + item.resources.len() * RESOURCE_LEN + }) + .sum::() } } -#[cfg(test)] -mod tests { - use crate::protocol::{ImapResponse, capability::QuotaResourceName}; - - use super::{QuotaItem, QuotaResource}; - - #[test] - fn serialize_quota() { - for (response, expected) in [ - ( - super::Response { - quota_root_items: vec!["INBOX".into(), "#test".into()], - quota_items: vec![], - }, - "* QUOTAROOT \"INBOX\" \"#test\"\r\n", - ), - ( - super::Response { - quota_root_items: vec![], - quota_items: vec![QuotaItem { - name: "INBOX".into(), - resources: vec![QuotaResource { - resource: QuotaResourceName::Storage, - total: 1073741824, - used: 1048576, - }], - }], - }, - "* QUOTA \"INBOX\" (STORAGE 1024 1048576)\r\n", - ), - ( - super::Response { - quota_root_items: vec!["my mailbox".into(), "".into()], - quota_items: vec![QuotaItem { - name: "INBOX".into(), - resources: vec![ - QuotaResource { - resource: QuotaResourceName::Storage, - total: 1073741824, - used: 1048576, - }, - QuotaResource { - resource: QuotaResourceName::Message, - total: 100, - used: 2, - }, - ], - }], - }, - concat!( - "* QUOTAROOT \"my mailbox\" \"\"\r\n", - "* QUOTA \"INBOX\" (STORAGE 1024 1048576 MESSAGE 2 100)\r\n" - ), - ), - ] { - assert_eq!(String::from_utf8(response.serialize()).unwrap(), expected); - } - } -} diff --git a/crates/imap-proto/src/protocol/rename.rs b/crates/imap-proto/src/protocol/rename.rs index 2d8493f7..3076a2c2 100644 --- a/crates/imap-proto/src/protocol/rename.rs +++ b/crates/imap-proto/src/protocol/rename.rs @@ -3,10 +3,11 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, - pub new_mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, + pub new_mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/search.rs b/crates/imap-proto/src/protocol/search.rs index f969cdfe..4d1ed544 100644 --- a/crates/imap-proto/src/protocol/search.rs +++ b/crates/imap-proto/src/protocol/search.rs @@ -3,12 +3,14 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::protocol::push_int; +use compact_str::CompactString; use super::{Flag, Sequence, quoted_string, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub is_esearch: bool, pub sort: Option>, pub result_options: Vec, @@ -108,8 +110,8 @@ pub enum Filter { ModSeq((u64, ModSeqEntry)), // RFC 8474 - ObjectID - EmailId(String), - ThreadId(String), + EmailId(CompactString), + ThreadId(CompactString), // RFC 9738 - MESSAGELIMIT UidAfter(u32), @@ -136,7 +138,20 @@ impl Filter { impl Response { pub fn serialize(self, tag: &str) -> Vec { - let mut buf = Vec::with_capacity(64); + const FRAMING_LEN: usize = 64; + const SEQUENCE_ID_LEN: usize = 6; + const PLAIN_ID_LEN: usize = 8; + + let mut buf = Vec::with_capacity( + FRAMING_LEN + + tag.len() + + self.ids.len() + * if self.is_esearch { + SEQUENCE_ID_LEN + } else { + PLAIN_ID_LEN + }, + ); if self.is_esearch { buf.extend_from_slice(b"* ESEARCH (TAG "); quoted_string(&mut buf, tag); @@ -146,15 +161,15 @@ impl Response { } if let Some(count) = &self.count { buf.extend_from_slice(b" COUNT "); - buf.extend_from_slice(count.to_string().as_bytes()); + push_int(&mut buf, *count); } if let Some(min) = &self.min { buf.extend_from_slice(b" MIN "); - buf.extend_from_slice(min.to_string().as_bytes()); + push_int(&mut buf, *min); } if let Some(max) = &self.max { buf.extend_from_slice(b" MAX "); - buf.extend_from_slice(max.to_string().as_bytes()); + push_int(&mut buf, *max); } if !self.ids.is_empty() { buf.extend_from_slice(b" ALL "); @@ -162,7 +177,7 @@ impl Response { } if let Some(highest_modseq) = self.highest_modseq { buf.extend_from_slice(b" MODSEQ "); - buf.extend_from_slice(highest_modseq.to_string().as_bytes()); + push_int(&mut buf, highest_modseq); } } else { if !self.is_sort { @@ -173,12 +188,12 @@ impl Response { if !self.ids.is_empty() { for id in &self.ids { buf.push(b' '); - buf.extend_from_slice(id.to_string().as_bytes()); + push_int(&mut buf, *id); } } if let Some(highest_modseq) = self.highest_modseq { buf.extend_from_slice(b" (MODSEQ "); - buf.extend_from_slice(highest_modseq.to_string().as_bytes()); + push_int(&mut buf, highest_modseq); buf.push(b')'); } } @@ -187,81 +202,3 @@ impl Response { } } -#[cfg(test)] -mod tests { - - #[test] - fn serialize_search() { - for (mut response, tag, expected_v2, expected_v1) in [ - ( - super::Response { - is_uid: false, - is_esearch: true, - is_sort: false, - ids: vec![2, 10, 11], - min: 2.into(), - max: 11.into(), - count: 3.into(), - highest_modseq: None, - }, - "A283", - "* ESEARCH (TAG \"A283\") COUNT 3 MIN 2 MAX 11 ALL 2,10:11\r\n", - "* SEARCH 2 10 11\r\n", - ), - ( - super::Response { - is_uid: false, - is_esearch: true, - is_sort: false, - ids: vec![ - 1, 2, 3, 5, 10, 11, 12, 13, 90, 92, 93, 94, 95, 96, 97, 98, 99, - ], - min: None, - max: None, - count: None, - highest_modseq: None, - }, - "A283", - "* ESEARCH (TAG \"A283\") ALL 1:3,5,10:13,90,92:99\r\n", - "* SEARCH 1 2 3 5 10 11 12 13 90 92 93 94 95 96 97 98 99\r\n", - ), - ( - super::Response { - is_uid: false, - is_esearch: true, - is_sort: false, - ids: vec![], - min: None, - max: None, - count: None, - highest_modseq: None, - }, - "A283", - "* ESEARCH (TAG \"A283\")\r\n", - "* SEARCH\r\n", - ), - ( - super::Response { - is_uid: false, - is_esearch: true, - is_sort: false, - ids: vec![10, 11, 12, 13, 21], - min: None, - max: None, - count: None, - highest_modseq: 12345.into(), - }, - "A283", - "* ESEARCH (TAG \"A283\") ALL 10:13,21 MODSEQ 12345\r\n", - "* SEARCH 10 11 12 13 21 (MODSEQ 12345)\r\n", - ), - ] { - let response_v2 = String::from_utf8(response.clone().serialize(tag)).unwrap(); - response.is_esearch = false; - let response_v1 = String::from_utf8(response.serialize(tag)).unwrap(); - - assert_eq!(response_v2, expected_v2); - assert_eq!(response_v1, expected_v1); - } - } -} diff --git a/crates/imap-proto/src/protocol/select.rs b/crates/imap-proto/src/protocol/select.rs index 077a062a..19d78ce6 100644 --- a/crates/imap-proto/src/protocol/select.rs +++ b/crates/imap-proto/src/protocol/select.rs @@ -3,15 +3,19 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::protocol::push_int; +use compact_str::CompactString; +use types::acl::Acl; +use utils::map::bitmap::Bitmap; use crate::{ResponseCode, StatusResponse}; -use super::{ImapResponse, ObjectId, Sequence, list::ListItem}; +use super::{Flag, ImapResponse, ObjectId, Sequence, list::ListItem, store::Operation}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub condstore: bool, pub qresync: Option, pub objectid: Option, @@ -41,6 +45,15 @@ pub struct Response { pub closed_previous: bool, pub highest_modseq: Option, pub objectid: Option, + pub permanent_flags: PermanentFlags, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PermanentFlags { + All, + AllButDeleted, + DeletedOnly, + None, } #[derive(Debug, Clone)] @@ -49,15 +62,14 @@ pub struct Exists { } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(100); + fn serialize_into(&self, buf: &mut Vec) { if self.closed_previous { - buf = StatusResponse::ok("Closed previous mailbox") + StatusResponse::ok("Closed previous mailbox") .with_code(ResponseCode::Closed) - .serialize(buf); + .serialize_into(buf); } buf.extend_from_slice(b"* "); - buf.extend_from_slice(self.total_messages.to_string().as_bytes()); + push_int(buf, self.total_messages); if !self.is_rev2 && self.recent_messages > 0 { buf.extend_from_slice( b" EXISTS\r\n* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", @@ -69,34 +81,99 @@ impl ImapResponse for Response { } if self.is_rev2 { self.mailbox - .serialize(&mut buf, self.is_rev2, self.is_utf8, false); + .serialize(buf, self.is_rev2, self.is_utf8, false); } else { buf.extend_from_slice(b"* "); - buf.extend_from_slice(self.recent_messages.to_string().as_bytes()); + push_int(buf, self.recent_messages); buf.extend_from_slice(b" RECENT\r\n"); if self.unseen_seq > 0 { buf.extend_from_slice(b"* OK [UNSEEN "); - buf.extend_from_slice(self.unseen_seq.to_string().as_bytes()); + push_int(buf, self.unseen_seq); buf.extend_from_slice(b"] Unseen messages\r\n"); } } - buf.extend_from_slice( - b"* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", - ); + self.permanent_flags.serialize(buf); buf.extend_from_slice(b"* OK [UIDVALIDITY "); - buf.extend_from_slice(self.uid_validity.to_string().as_bytes()); + push_int(buf, self.uid_validity); buf.extend_from_slice(b"] UIDs valid\r\n* OK [UIDNEXT "); - buf.extend_from_slice(self.uid_next.to_string().as_bytes()); + push_int(buf, self.uid_next); buf.extend_from_slice(b"] Next predicted UID\r\n"); - if let Some(highest_modseq) = self.highest_modseq { - highest_modseq.serialize(&mut buf); + if let Some(highest_modseq) = &self.highest_modseq { + highest_modseq.serialize(buf); } if let Some(objectid) = &self.objectid { buf.extend_from_slice(b"* OK ["); - objectid.serialize(&mut buf); + objectid.serialize(buf); buf.extend_from_slice(b"] Object identifiers\r\n"); } - buf + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 320; + const OBJECT_ID_LEN: usize = 96; + const CLOSED_LEN: usize = 40; + const MODSEQ_LEN: usize = 64; + + FRAMING_LEN + + self.mailbox.mailbox_name.len() * 2 + + self.mailbox.attributes.len() * 16 + + if self.closed_previous { CLOSED_LEN } else { 0 } + + if self.highest_modseq.is_some() { + MODSEQ_LEN + } else { + 0 + } + + if self.objectid.is_some() { + OBJECT_ID_LEN + } else { + 0 + } + } +} + +impl From> for PermanentFlags { + fn from(acl: Bitmap) -> Self { + match ( + acl.contains(Acl::ModifyItems), + acl.contains(Acl::RemoveItems), + ) { + (true, true) => PermanentFlags::All, + (true, false) => PermanentFlags::AllButDeleted, + (false, true) => PermanentFlags::DeletedOnly, + (false, false) => PermanentFlags::None, + } + } +} + +impl PermanentFlags { + pub fn permits(self, flag: &Flag) -> bool { + match self { + PermanentFlags::All => true, + PermanentFlags::AllButDeleted => *flag != Flag::Deleted, + PermanentFlags::DeletedOnly => *flag == Flag::Deleted, + PermanentFlags::None => false, + } + } + + pub fn permits_any(self, operation: &Operation, flags: &[Flag]) -> bool { + match self { + PermanentFlags::None => false, + _ if *operation == Operation::Set || flags.is_empty() => true, + _ => flags.iter().any(|flag| self.permits(flag)), + } + } + + fn serialize(self, buf: &mut Vec) { + buf.extend_from_slice(match self { + PermanentFlags::All => { + b"* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n".as_slice() + } + PermanentFlags::AllButDeleted => { + b"* OK [PERMANENTFLAGS (\\Seen \\Answered \\Flagged \\Draft \\*)] Limited\r\n" + } + PermanentFlags::DeletedOnly => b"* OK [PERMANENTFLAGS (\\Deleted)] Limited\r\n", + PermanentFlags::None => b"* OK [PERMANENTFLAGS ()] No permanent flags permitted\r\n", + }); } } @@ -107,12 +184,12 @@ impl HighestModSeq { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"* OK [HIGHESTMODSEQ "); - buf.extend_from_slice(self.0.to_string().as_bytes()); + push_int(buf, self.0); buf.extend_from_slice(b"] Highest Modseq\r\n"); } pub fn into_bytes(self) -> Vec { - let mut buf = Vec::with_capacity(40); + let mut buf = Vec::with_capacity(64); self.serialize(&mut buf); buf } @@ -121,131 +198,14 @@ impl HighestModSeq { impl Exists { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"* "); - buf.extend_from_slice(self.total_messages.to_string().as_bytes()); + push_int(buf, self.total_messages); buf.extend_from_slice(b" EXISTS\r\n"); } pub fn into_bytes(self) -> Vec { - let mut buf = Vec::with_capacity(15); + let mut buf = Vec::with_capacity(32); self.serialize(&mut buf); buf } } -#[cfg(test)] -mod tests { - use crate::protocol::{ImapResponse, ObjectId, list::ListItem}; - use types::id::Id; - - use super::HighestModSeq; - - #[test] - fn serialize_select() { - let objectid = ObjectId { - mailbox_id: Some(Id::from(1u32)), - account_id: Some(Id::from(2u32)), - ..Default::default() - }; - let mut objectid_line = b"* OK [".to_vec(); - objectid.serialize(&mut objectid_line); - objectid_line.extend_from_slice(b"] Object identifiers\r\n"); - let objectid_line = String::from_utf8(objectid_line).unwrap(); - - for (mut response, _tag, expected_v2, expected_v1) in [ - ( - super::Response { - mailbox: ListItem::new("INBOX"), - total_messages: 172, - recent_messages: 5, - unseen_seq: 3, - uid_validity: 3857529045, - uid_next: 4392, - closed_previous: false, - is_rev2: true, - is_utf8: true, - highest_modseq: HighestModSeq::new(100).into(), - objectid: Some(objectid.clone()), - }, - "A142", - format!( - concat!( - "* 172 EXISTS\r\n", - "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n", - "* LIST () \"/\" \"INBOX\"\r\n", - "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", - "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", - "* OK [UIDNEXT 4392] Next predicted UID\r\n", - "* OK [HIGHESTMODSEQ 100] Highest Modseq\r\n", - "{}" - ), - objectid_line - ), - format!( - concat!( - "* 172 EXISTS\r\n", - "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", - "* 5 RECENT\r\n", - "* OK [UNSEEN 3] Unseen messages\r\n", - "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", - "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", - "* OK [UIDNEXT 4392] Next predicted UID\r\n", - "* OK [HIGHESTMODSEQ 100] Highest Modseq\r\n", - "{}" - ), - objectid_line - ), - ), - ( - super::Response { - mailbox: ListItem::new("~peter/mail/台北/日本語"), - total_messages: 172, - recent_messages: 5, - unseen_seq: 3, - uid_validity: 3857529045, - uid_next: 4392, - closed_previous: true, - is_rev2: true, - is_utf8: true, - highest_modseq: None, - objectid: Some(objectid.clone()), - }, - "A142", - format!( - concat!( - "* OK [CLOSED] Closed previous mailbox\r\n", - "* 172 EXISTS\r\n", - "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n", - "* LIST () \"/\" \"~peter/mail/台北/日本語\" (\"OLDNAME\" ", - "(\"~peter/mail/&U,BTFw-/&ZeVnLIqe-\"))\r\n", - "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", - "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", - "* OK [UIDNEXT 4392] Next predicted UID\r\n", - "{}" - ), - objectid_line - ), - format!( - concat!( - "* OK [CLOSED] Closed previous mailbox\r\n", - "* 172 EXISTS\r\n", - "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", - "* 5 RECENT\r\n", - "* OK [UNSEEN 3] Unseen messages\r\n", - "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", - "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", - "* OK [UIDNEXT 4392] Next predicted UID\r\n", - "{}" - ), - objectid_line - ), - ), - ] { - let response_v2 = String::from_utf8(response.clone().serialize()).unwrap(); - response.is_rev2 = false; - let response_v1 = String::from_utf8(response.serialize()).unwrap(); - - assert_eq!(response_v2, expected_v2); - assert_eq!(response_v1, expected_v1); - } - } -} diff --git a/crates/imap-proto/src/protocol/status.rs b/crates/imap-proto/src/protocol/status.rs index 971adbfc..7faa6cca 100644 --- a/crates/imap-proto/src/protocol/status.rs +++ b/crates/imap-proto/src/protocol/status.rs @@ -3,15 +3,15 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::protocol::push_int; +use compact_str::CompactString; -use crate::utf7::utf7_encode; - -use super::{ObjectId, quoted_string}; +use super::{ObjectId, quoted_mailbox_name}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub items: Vec, } @@ -31,7 +31,7 @@ pub enum Status { #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusItem { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub items: Vec<(Status, StatusItemType)>, } @@ -45,11 +45,7 @@ pub enum StatusItemType { impl StatusItem { pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { buf.extend_from_slice(b"* STATUS "); - if is_utf8 { - quoted_string(buf, &self.mailbox_name); - } else { - quoted_string(buf, &utf7_encode(&self.mailbox_name)); - } + quoted_mailbox_name(buf, &self.mailbox_name, is_utf8); buf.extend_from_slice(b" ("); for (pos, (status_item, value)) in self.items.iter().enumerate() { if pos > 0 { @@ -71,7 +67,7 @@ impl StatusItem { match value { StatusItemType::Number(num) => { - buf.extend_from_slice(num.to_string().as_bytes()); + push_int(buf, *num); } StatusItemType::String(str) => { buf.push(b'('); @@ -87,39 +83,3 @@ impl StatusItem { } } -#[cfg(test)] -mod tests { - use crate::protocol::{ - ObjectId, - status::{Status, StatusItem, StatusItemType}, - }; - use types::id::Id; - - #[test] - fn serialize_status() { - let objectid = ObjectId { - mailbox_id: Some(Id::from(1u32)), - account_id: Some(Id::from(2u32)), - ..Default::default() - }; - let mut kvpairs = Vec::new(); - objectid.serialize_kvpairs(&mut kvpairs); - let kvpairs = String::from_utf8(kvpairs).unwrap(); - - let mut buf = Vec::new(); - StatusItem { - mailbox_name: "blurdybloop".into(), - items: vec![ - (Status::Messages, StatusItemType::Number(231)), - (Status::UidNext, StatusItemType::Number(44292)), - (Status::ObjectId, StatusItemType::ObjectId(objectid.clone())), - ], - } - .serialize(&mut buf, true); - - assert_eq!( - String::from_utf8(buf).unwrap(), - format!("* STATUS \"blurdybloop\" (MESSAGES 231 UIDNEXT 44292 OBJECTID {kvpairs})\r\n") - ); - } -} diff --git a/crates/imap-proto/src/protocol/store.rs b/crates/imap-proto/src/protocol/store.rs index 1094a33b..c33483bd 100644 --- a/crates/imap-proto/src/protocol/store.rs +++ b/crates/imap-proto/src/protocol/store.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use super::{Flag, ImapResponse, Sequence, fetch::FetchItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, pub operation: Operation, pub is_silent: bool, @@ -25,16 +26,17 @@ pub enum Operation { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response<'x> { - pub is_utf8: bool, pub items: Vec>, } impl ImapResponse for Response<'_> { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); + fn serialize_into(&self, buf: &mut Vec) { for item in &self.items { - item.serialize(&mut buf, self.is_utf8); + item.serialize(buf); } - buf + } + + fn size_hint(&self) -> usize { + self.items.iter().map(FetchItem::size_hint).sum() } } diff --git a/crates/imap-proto/src/protocol/subscribe.rs b/crates/imap-proto/src/protocol/subscribe.rs index 2284bc72..8f1e5fd6 100644 --- a/crates/imap-proto/src/protocol/subscribe.rs +++ b/crates/imap-proto/src/protocol/subscribe.rs @@ -3,9 +3,10 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/thread.rs b/crates/imap-proto/src/protocol/thread.rs index 77a3340b..af262dfc 100644 --- a/crates/imap-proto/src/protocol/thread.rs +++ b/crates/imap-proto/src/protocol/thread.rs @@ -3,12 +3,14 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::protocol::push_int; +use compact_str::CompactString; use super::{ImapResponse, search::Filter}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub filter: Vec, pub algorithm: Algorithm, } @@ -26,8 +28,7 @@ pub struct Response { } impl ImapResponse for Response { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(64); + fn serialize_into(&self, buf: &mut Vec) { buf.extend_from_slice(b"* THREAD "); for thread in &self.threads { buf.push(b'('); @@ -35,31 +36,23 @@ impl ImapResponse for Response { if pos > 0 { buf.push(b' '); } - buf.extend_from_slice(id.to_string().as_bytes()); + push_int(buf, *id); } buf.push(b')'); } buf.extend_from_slice(b"\r\n"); - buf + } + + fn size_hint(&self) -> usize { + const FRAMING_LEN: usize = 12; + const ID_LEN: usize = 11; + + FRAMING_LEN + + self + .threads + .iter() + .map(|thread| 2 + thread.len() * ID_LEN) + .sum::() } } -#[cfg(test)] -mod tests { - use crate::protocol::ImapResponse; - - #[test] - fn serialize_thread() { - assert_eq!( - String::from_utf8( - super::Response { - is_uid: true, - threads: vec![vec![2, 10, 11], vec![49], vec![1, 3]], - } - .serialize() - ) - .unwrap(), - "* THREAD (2 10 11)(49)(1 3)\r\n" - ); - } -} diff --git a/crates/imap-proto/src/protocol/uidbatches.rs b/crates/imap-proto/src/protocol/uidbatches.rs index a881119d..86fc59a1 100644 --- a/crates/imap-proto/src/protocol/uidbatches.rs +++ b/crates/imap-proto/src/protocol/uidbatches.rs @@ -3,12 +3,13 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; -use std::fmt::Write; +use super::push_int; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub batch_size: u32, pub batch_range: Option<(u32, u32)>, } @@ -20,39 +21,18 @@ pub struct Response { impl Response { pub fn serialize(self, tag: &str) -> Vec { - let mut buf = String::with_capacity(32 + (self.ranges.len() * 16)); - let _ = write!(&mut buf, "* UIDBATCHES (TAG \"{tag}\")"); + let mut buf = Vec::with_capacity(32 + tag.len() + (self.ranges.len() * 22)); + buf.extend_from_slice(b"* UIDBATCHES (TAG \""); + buf.extend_from_slice(tag.as_bytes()); + buf.extend_from_slice(b"\")"); for (pos, (high, low)) in self.ranges.iter().enumerate() { - let _ = write!(&mut buf, "{}{high}:{low}", if pos == 0 { ' ' } else { ',' }); + buf.push(if pos == 0 { b' ' } else { b',' }); + push_int(&mut buf, *high); + buf.push(b':'); + push_int(&mut buf, *low); } - buf.push_str("\r\n"); - buf.into_bytes() + buf.extend_from_slice(b"\r\n"); + buf } } -#[cfg(test)] -mod tests { - use super::Response; - - #[test] - fn serialize_uidbatches() { - assert_eq!( - String::from_utf8( - Response { - ranges: vec![(215295, 99696), (99695, 20351), (20350, 7830), (7829, 1)], - } - .serialize("A143") - ) - .unwrap(), - concat!( - "* UIDBATCHES (TAG \"A143\") ", - "215295:99696,99695:20351,20350:7830,7829:1\r\n" - ) - ); - - assert_eq!( - String::from_utf8(Response { ranges: vec![] }.serialize("A144")).unwrap(), - "* UIDBATCHES (TAG \"A144\")\r\n" - ); - } -} diff --git a/crates/imap-proto/src/receiver.rs b/crates/imap-proto/src/receiver.rs index 945003a5..724bd746 100644 --- a/crates/imap-proto/src/receiver.rs +++ b/crates/imap-proto/src/receiver.rs @@ -6,8 +6,14 @@ use super::{ResponseCode, ResponseType}; use compact_str::{CompactString, format_compact}; +use encodify::utf7::IMAP; +use smallvec::SmallVec; use std::fmt::Display; +pub type ArgumentBytes = SmallVec<[u8; TOKEN_INLINE_LEN]>; + +const TOKEN_INLINE_LEN: usize = 22; + #[derive(Debug, Clone)] pub enum Error { NeedsMoreData, @@ -17,7 +23,7 @@ pub enum Error { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Request { - pub tag: String, + pub tag: CompactString, pub command: T, pub tokens: Vec, } @@ -25,11 +31,14 @@ pub struct Request { pub trait CommandParser: Sized + Default { fn parse(bytes: &[u8], is_uid: bool) -> Option; fn tokenize_brackets(&self) -> bool; + fn tokenize_nil(&self) -> bool { + false + } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Token { - Argument(Vec), + Argument(ArgumentBytes), ParenthesisOpen, // ( ParenthesisClose, // ) BracketOpen, // [ @@ -38,12 +47,13 @@ pub enum Token { Gt, // > Dot, // . Nil, // NIL + NilString(ArgumentBytes), } impl Default for Request { fn default() -> Self { Self { - tag: String::new(), + tag: CompactString::const_new(""), command: T::default(), tokens: Vec::new(), } @@ -70,14 +80,70 @@ pub struct Receiver { pub max_request_size: usize, pub current_request_size: usize, pub start_state: State, + skipping: bool, } +pub const MAX_UNAUTHENTICATED_REQUEST_SIZE: usize = 64 * 1024; + const ARG_MAX_LEN: usize = 8000; +const ARG_INIT_LEN: usize = 64; +const TAG_MAX_LEN: usize = 128; +const COMMAND_MAX_LEN: usize = 15; +const LITERAL_MAX_DIGITS: usize = 15; +const TOKENS_INIT_LEN: usize = 32; struct ArgumentBuffer { buf: Vec, } +const CLASS_ARGUMENT: u8 = 0b0000_0001; +const CLASS_BRACKET: u8 = 0b0000_0010; +const CLASS_QUOTED: u8 = 0b0000_0100; +const CLASS_TAG: u8 = 0b0000_1000; +const CLASS_COMMAND: u8 = 0b0001_0000; +const CLASS_START: u8 = 0b0010_0000; + +const BYTE_CLASS: [u8; 256] = byte_class_table(); + +const fn byte_class_table() -> [u8; 256] { + let mut table = [0u8; 256]; + let mut index = 0; + while index < 256 { + let ch = index as u8; + let is_whitespace = matches!(ch, b' ' | b'\t' | b'\n' | b'\x0c' | b'\r'); + let mut class = 0; + if is_whitespace || matches!(ch, b'"' | b'{' | b'(' | b')') { + class |= CLASS_ARGUMENT; + } + if matches!(ch, b'[' | b']' | b'<' | b'>' | b'.') { + class |= CLASS_BRACKET; + } + if matches!(ch, b'"' | b'\\' | b'\n') { + class |= CLASS_QUOTED; + } + if matches!(ch, b' ' | b'\t' | b'\r' | b'\n') { + class |= CLASS_TAG; + } + if !matches!(ch, b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') { + class |= CLASS_COMMAND; + } + if !is_whitespace { + class |= CLASS_START; + } + table[index] = class; + index += 1; + } + table +} + +#[inline(always)] +fn run_len(bytes: &[u8], class: u8) -> usize { + bytes + .iter() + .position(|&ch| BYTE_CLASS[ch as usize] & class != 0) + .unwrap_or(bytes.len()) +} + impl Receiver { pub fn new() -> Self { Receiver { @@ -99,7 +165,15 @@ impl Receiver { } } + pub fn is_discarding(&self) -> bool { + matches!(self.state, State::LiteralDiscard { .. }) + } + pub fn error_reset(&mut self, message: impl Into) -> Error { + self.error_reset_with(message, ResponseCode::Parse) + } + + fn error_reset_with(&mut self, message: impl Into, code: ResponseCode) -> Error { let request = std::mem::take(&mut self.request); let err = Error::err( if !request.tag.is_empty() { @@ -108,223 +182,373 @@ impl Receiver { None }, message, + code, ); self.buf = ArgumentBuffer::default(); self.state = self.start_state; self.current_request_size = 0; + self.skipping = false; err } + fn end_skipped_command(&mut self) { + self.skipping = false; + self.request = Request::default(); + self.buf.clear(); + self.current_request_size = 0; + } + + #[cold] + #[inline(never)] + fn literal_size_error(&mut self) -> Error { + self.error_reset_with( + format_compact!( + "Literal exceeds the maximum request size of {} bytes.", + self.max_request_size + ), + ResponseCode::TooBig, + ) + } + + #[inline(never)] + fn request_size_error(&mut self) -> Error { + self.error_reset(format_compact!( + "Request exceeds maximum limit of {} bytes.", + self.max_request_size + )) + } + + #[inline(always)] + fn push_request_token(&mut self, token: Token) { + if self.request.tokens.is_empty() { + self.request.tokens.reserve(TOKENS_INIT_LEN); + } + self.request.tokens.push(token); + } + fn push_argument(&mut self, in_quote: bool) -> Result<(), Error> { if !self.buf.is_empty() { self.current_request_size += self.buf.len(); if self.current_request_size > self.max_request_size { - return Err(self.error_reset(format_compact!( - "Request exceeds maximum limit of {} bytes.", - self.max_request_size - ))); + return Err(self.request_size_error()); } - self.request.tokens.push(Token::Argument(self.buf.take())); + let token = Token::Argument(ArgumentBytes::from_slice(self.buf.as_ref())); + self.push_request_token(token); + self.buf.clear(); } else if in_quote { - self.request.tokens.push(Token::Nil); + self.push_request_token(Token::Nil); } Ok(()) } + fn push_literal(&mut self) -> Result<(), Error> { + if !self.buf.is_empty() { + if self.request.command.tokenize_nil() && self.is_nil() { + return self.push_nil_string(); + } + self.current_request_size += self.buf.len(); + if self.current_request_size > self.max_request_size { + return Err(self.request_size_error()); + } + let token = Token::Argument(ArgumentBytes::from_vec(self.buf.take_moved())); + self.push_request_token(token); + } + Ok(()) + } + + fn is_nil(&self) -> bool { + self.buf.as_ref().eq_ignore_ascii_case(b"NIL") + } + + #[cold] + fn push_nil_string(&mut self) -> Result<(), Error> { + self.current_request_size += self.buf.len(); + if self.current_request_size > self.max_request_size { + return Err(self.request_size_error()); + } + let token = Token::NilString(ArgumentBytes::from_slice(self.buf.as_ref())); + self.push_request_token(token); + self.buf.clear(); + Ok(()) + } + fn push_token(&mut self, token: Token) -> Result<(), Error> { self.current_request_size += 1; if self.current_request_size > self.max_request_size { - return Err(self.error_reset(format_compact!( - "Request exceeds maximum limit of {} bytes.", - self.max_request_size - ))); + return Err(self.request_size_error()); } - self.request.tokens.push(token); + self.push_request_token(token); Ok(()) } pub fn parse(&mut self, bytes: &mut std::slice::Iter<'_, u8>) -> Result, Error> { - #[allow(clippy::while_let_on_iterator)] - while let Some(&ch) = bytes.next() { - match self.state { + let mut state = self.state; + + 'outer: loop { + match state { State::Start => { - if !ch.is_ascii_whitespace() { - // SAFETY: This called just once - self.buf.push_unchecked(ch); - self.state = State::Tag; + match bytes.find(|&&ch| BYTE_CLASS[ch as usize] & CLASS_START != 0) { + Some(&ch) => { + self.buf.push_unchecked(ch); + state = State::Tag; + } + None => break, } } - State::Tag => match ch { - b' ' => { - if !self.buf.is_empty() { - self.request.tag = - String::from_utf8(self.buf.take()).map_err(|_| { - self.error_reset("Tag is not a valid UTF-8 string.") - })?; - self.state = State::Command { is_uid: false }; - } - } - b'\t' | b'\r' => {} - b'\n' => { - return Err(self.error_reset(format_compact!( - "Missing command after tag {:?}, found CRLF instead.", - self.buf.as_str() - ))); - } - _ => { - self.buf.push_checked(ch, 128).map_err(|_| { + State::Tag => { + let slice = bytes.as_slice(); + let (run, tail) = slice.split_at(run_len(slice, CLASS_TAG)); + if let Err(allowed) = self.buf.extend_checked(run, TAG_MAX_LEN) { + *bytes = slice.split_at(allowed + 1).1.iter(); + return Err( self.error_reset("Tag exceeds maximum length of 128 characters.") - })?; + ); } - }, - State::Command { is_uid } => { - if ch.is_ascii_alphanumeric() { - self.buf - .push_checked(ch.to_ascii_uppercase(), 15) - .map_err(|_| { - self.error_reset("Command exceeds maximum length of 15 characters.") - })?; - } else if ch.is_ascii_whitespace() { - if !self.buf.is_empty() { - if !self.buf.as_ref().eq_ignore_ascii_case(b"UID") { - self.request.command = T::parse(self.buf.as_ref(), is_uid) - .ok_or_else(|| { - let err = format_compact!( - "Unrecognized command '{}'.", - String::from_utf8_lossy(self.buf.as_ref()) + let Some((&ch, rest)) = tail.split_first() else { + *bytes = tail.iter(); + break; + }; + *bytes = rest.iter(); + match ch { + b' ' => { + if !self.buf.is_empty() { + match CompactString::from_utf8(self.buf.as_ref()) { + Ok(tag) => { + self.request.tag = tag; + self.buf.clear(); + } + Err(_) => { + return Err( + self.error_reset("Tag is not a valid UTF-8 string.") ); - self.error_reset(err) - })?; - self.buf.clear(); - if ch != b'\n' { - self.state = State::Argument { last_ch: b' ' }; - } else { - self.state = self.start_state; - self.current_request_size = 0; - return Ok(std::mem::take(&mut self.request)); + } } - } else { - self.buf.clear(); - self.state = State::Command { is_uid: true }; + state = State::Command { is_uid: false }; } } - } else { + b'\n' => { + return Err(self.error_reset(format_compact!( + "Missing command after tag {:?}, found CRLF instead.", + self.buf.as_str() + ))); + } + _ => {} + } + } + State::Command { is_uid } => { + let slice = bytes.as_slice(); + let (run, tail) = slice.split_at(run_len(slice, CLASS_COMMAND)); + if let Err(allowed) = self.buf.extend_checked_uppercase(run, COMMAND_MAX_LEN) { + *bytes = slice.split_at(allowed + 1).1.iter(); + return Err( + self.error_reset("Command exceeds maximum length of 15 characters.") + ); + } + let Some((&ch, rest)) = tail.split_first() else { + *bytes = tail.iter(); + break; + }; + *bytes = rest.iter(); + if !ch.is_ascii_whitespace() { return Err(self.error_reset(format_compact!( "Invalid character {:?} in command name.", ch as char ))); } - } - State::Argument { last_ch } => match ch { - b'\"' if last_ch.is_ascii_whitespace() => { - self.push_argument(false)?; - self.state = State::ArgumentQuoted { escaped: false }; - } - b'{' if last_ch.is_ascii_whitespace() - || (last_ch == b'~' && self.buf.len() == 1) => - { - if last_ch != b'~' { - self.push_argument(false)?; + if !self.buf.is_empty() { + if !self.buf.as_ref().eq_ignore_ascii_case(b"UID") { + self.request.command = + T::parse(self.buf.as_ref(), is_uid).ok_or_else(|| { + let err = format_compact!( + "Unrecognized command '{}'.", + String::from_utf8_lossy(self.buf.as_ref()) + ); + self.error_reset(err) + })?; + self.buf.clear(); + if ch != b'\n' { + state = State::Argument { last_ch: b' ' }; + } else { + self.state = self.start_state; + self.current_request_size = 0; + return Ok(std::mem::take(&mut self.request)); + } } else { self.buf.clear(); - } - self.state = State::Literal { non_sync: false }; - } - b'(' => { - self.push_argument(false)?; - self.push_token(Token::ParenthesisOpen)?; - } - b')' => { - self.push_argument(false)?; - self.push_token(Token::ParenthesisClose)?; - } - b'[' if self.request.command.tokenize_brackets() => { - self.push_argument(false)?; - self.push_token(Token::BracketOpen)?; - } - b']' if self.request.command.tokenize_brackets() => { - self.push_argument(false)?; - self.push_token(Token::BracketClose)?; - } - b'<' if self.request.command.tokenize_brackets() => { - self.push_argument(false)?; - self.push_token(Token::Lt)?; - } - b'>' if self.request.command.tokenize_brackets() => { - self.push_argument(false)?; - self.push_token(Token::Gt)?; - } - b'.' if self.request.command.tokenize_brackets() => { - self.push_argument(false)?; - self.push_token(Token::Dot)?; - } - b'\n' => { - self.push_argument(false)?; - self.state = self.start_state; - self.current_request_size = 0; - return Ok(std::mem::take(&mut self.request)); - } - _ if ch.is_ascii_whitespace() => { - self.push_argument(false)?; - self.state = State::Argument { last_ch: ch }; - } - _ => { - self.buf.push_checked(ch, ARG_MAX_LEN).map_err(|_| { - self.error_reset("Argument exceeds maximum length of 8000 bytes.") - })?; - self.state = State::Argument { last_ch: ch }; - } - }, - State::ArgumentQuoted { escaped } => match ch { - b'\"' => { - if !escaped { - self.push_argument(true)?; - self.state = State::Argument { last_ch: b' ' }; - } else { - self.buf - .push_checked(ch, ARG_MAX_LEN) - .map_err(|_| self.error_reset("Quoted argument too long."))?; - self.state = State::ArgumentQuoted { escaped: false }; + state = State::Command { is_uid: true }; } } - b'\\' => { - if escaped { - self.buf - .push_checked(ch, ARG_MAX_LEN) - .map_err(|_| self.error_reset("Quoted argument too long."))?; + } + State::Argument { last_ch } => { + let mut last_ch = last_ch; + let class = if self.request.command.tokenize_brackets() { + CLASS_ARGUMENT | CLASS_BRACKET + } else { + CLASS_ARGUMENT + }; + loop { + let slice = bytes.as_slice(); + let (run, tail) = slice.split_at(run_len(slice, class)); + if let Some(&last) = run.last() { + if let Err(allowed) = self.buf.extend_checked(run, ARG_MAX_LEN) { + *bytes = slice.split_at(allowed + 1).1.iter(); + return Err(self.error_reset( + "Argument exceeds maximum length of 8000 bytes.", + )); + } + last_ch = last; } - self.state = State::ArgumentQuoted { escaped: !escaped }; - } - b'\n' => { - return Err(self.error_reset("Unterminated quoted argument.")); - } - _ => { - if escaped { - // SAFETY: We check the size below - self.buf.push_unchecked(b'\\'); + let Some((&ch, rest)) = tail.split_first() else { + *bytes = tail.iter(); + state = State::Argument { last_ch }; + break 'outer; + }; + *bytes = rest.iter(); + match ch { + b'\"' if last_ch.is_ascii_whitespace() => { + self.push_argument(false)?; + state = State::ArgumentQuoted { escaped: false }; + continue 'outer; + } + b'{' if last_ch.is_ascii_whitespace() + || (last_ch == b'~' && self.buf.len() == 1) => + { + if last_ch != b'~' { + self.push_argument(false)?; + } else { + self.buf.clear(); + } + state = State::Literal { non_sync: false }; + continue 'outer; + } + b'(' => { + self.push_argument(false)?; + self.push_token(Token::ParenthesisOpen)?; + } + b')' => { + self.push_argument(false)?; + self.push_token(Token::ParenthesisClose)?; + } + b'[' => { + self.push_argument(false)?; + self.push_token(Token::BracketOpen)?; + } + b']' => { + self.push_argument(false)?; + self.push_token(Token::BracketClose)?; + } + b'<' => { + self.push_argument(false)?; + self.push_token(Token::Lt)?; + } + b'>' => { + self.push_argument(false)?; + self.push_token(Token::Gt)?; + } + b'.' => { + self.push_argument(false)?; + self.push_token(Token::Dot)?; + } + b'\n' if self.skipping => { + self.end_skipped_command(); + state = self.start_state; + continue 'outer; + } + b'\n' => { + self.push_argument(false)?; + self.state = self.start_state; + self.current_request_size = 0; + return Ok(std::mem::take(&mut self.request)); + } + _ if ch.is_ascii_whitespace() => { + self.push_argument(false)?; + last_ch = ch; + } + _ => { + self.buf.push_checked(ch, ARG_MAX_LEN).map_err(|_| { + self.error_reset( + "Argument exceeds maximum length of 8000 bytes.", + ) + })?; + last_ch = ch; + } } - self.buf - .push_checked(ch, ARG_MAX_LEN) - .map_err(|_| self.error_reset("Quoted argument too long."))?; - self.state = State::ArgumentQuoted { escaped: false }; } - }, + } + State::ArgumentQuoted { escaped } => { + if escaped { + let Some(&ch) = bytes.next() else { break }; + match ch { + b'\n' => { + return Err(self.error_reset("Unterminated quoted argument.")); + } + b'\"' | b'\\' => { + self.buf + .push_checked(ch, ARG_MAX_LEN) + .map_err(|_| self.error_reset("Quoted argument too long."))?; + } + _ => { + self.buf.push_unchecked(b'\\'); + self.buf + .push_checked(ch, ARG_MAX_LEN) + .map_err(|_| self.error_reset("Quoted argument too long."))?; + } + } + state = State::ArgumentQuoted { escaped: false }; + continue; + } + let slice = bytes.as_slice(); + let (run, tail) = slice.split_at(run_len(slice, CLASS_QUOTED)); + if let Err(allowed) = self.buf.extend_checked(run, ARG_MAX_LEN) { + *bytes = slice.split_at(allowed + 1).1.iter(); + return Err(self.error_reset("Quoted argument too long.")); + } + let Some((&ch, rest)) = tail.split_first() else { + *bytes = tail.iter(); + break; + }; + *bytes = rest.iter(); + match ch { + b'\"' => { + if self.request.command.tokenize_nil() && self.is_nil() { + self.push_nil_string()?; + } else { + self.push_argument(true)?; + } + state = State::Argument { last_ch: b' ' }; + } + b'\\' => { + state = State::ArgumentQuoted { escaped: true }; + } + _ => { + return Err(self.error_reset("Unterminated quoted argument.")); + } + } + } State::Literal { non_sync } => { + let Some(&ch) = bytes.next() else { break }; match ch { b'}' => { if !self.buf.is_empty() { let size = self.buf.as_str().parse::().map_err(|_| { self.error_reset("Literal size is not a valid number.") })?; + if self.skipping { + self.buf.clear(); + if non_sync { + state = State::LiteralSeek { size, non_sync }; + } else { + self.end_skipped_command(); + state = self.start_state; + } + continue; + } let oversize = self.current_request_size + size as usize > self.max_request_size; if oversize && !non_sync { - return Err(self.error_reset(format_compact!( - "Literal exceeds the maximum request size of {} bytes.", - self.max_request_size - ))); + return Err(self.literal_size_error()); } - self.state = State::LiteralSeek { size, non_sync }; + state = State::LiteralSeek { size, non_sync }; if !oversize { self.buf.resize_buffer(size as usize); } @@ -335,18 +559,17 @@ impl Receiver { } b'+' => { if !self.buf.is_empty() { - self.state = State::Literal { non_sync: true }; + state = State::Literal { non_sync: true }; } else { return Err(self.error_reset("Invalid non-sync literal.")); } } _ if ch.is_ascii_digit() => { if !non_sync { - self.buf.push_checked(ch, 15).map_err(|_| { + self.buf.push_checked(ch, LITERAL_MAX_DIGITS).map_err(|_| { self.error_reset("Literal size exceeds maximum of 15 digits.") })?; } else { - // Digit found after non-sync '+' flag return Err(self.error_reset("Invalid literal.")); } } @@ -359,20 +582,28 @@ impl Receiver { } } State::LiteralSeek { size, non_sync } => { + let Some(&ch) = bytes.next() else { break }; if ch == b'\n' { - if non_sync + if self.skipping { + state = State::LiteralDiscard { remaining: size }; + } else if non_sync && self.current_request_size + size as usize > self.max_request_size { + let err = self.literal_size_error(); + self.skipping = true; self.state = State::LiteralDiscard { remaining: size }; + return Err(err); } else if size > 0 { - self.state = State::LiteralData { remaining: size }; + state = State::LiteralData { remaining: size }; if !non_sync { + self.state = state; return Err(Error::NeedsLiteral { size }); } } else { - self.state = State::Argument { last_ch: b' ' }; + state = State::Argument { last_ch: b' ' }; self.push_token(Token::Nil)?; if !non_sync { + self.state = state; return Err(Error::NeedsLiteral { size }); } } @@ -383,33 +614,43 @@ impl Receiver { } } State::LiteralDiscard { remaining } => { - if remaining > 1 { - self.state = State::LiteralDiscard { - remaining: remaining - 1, - }; - } else { - return Err(self.error_reset(format_compact!( - "Literal exceeds the maximum request size of {} bytes.", - self.max_request_size - ))); + let available = bytes.as_slice(); + let taken = (remaining as usize).min(available.len()); + *bytes = available.split_at(taken).1.iter(); + let remaining = remaining - taken as u32; + if remaining > 0 { + state = State::LiteralDiscard { remaining }; + break; } + state = State::Argument { last_ch: b' ' }; } State::LiteralData { remaining } => { - // SAFETY: We checked the size before entering this state + let Some(&ch) = bytes.next() else { break }; self.buf.push_unchecked(ch); + let mut remaining = remaining - 1; - if remaining > 1 { - self.state = State::LiteralData { - remaining: remaining - 1, - }; + if remaining > 0 { + let available = bytes.as_slice(); + let taken = (remaining as usize).min(available.len()); + if taken > 0 { + let (run, tail) = available.split_at(taken); + self.buf.extend_from_slice(run); + *bytes = tail.iter(); + remaining -= taken as u32; + } + } + + if remaining > 0 { + state = State::LiteralData { remaining }; } else { - self.push_argument(false)?; - self.state = State::Argument { last_ch: b' ' }; + self.push_literal()?; + state = State::Argument { last_ch: b' ' }; } } } } + self.state = state; Err(Error::NeedsMoreData) } } @@ -417,13 +658,42 @@ impl Receiver { impl ArgumentBuffer { pub fn new() -> Self { ArgumentBuffer { - buf: Vec::with_capacity(10), + buf: Vec::with_capacity(ARG_INIT_LEN), } } pub fn resize_buffer(&mut self, size: usize) { - if self.buf.capacity() < size { - self.buf.reserve(size - self.buf.capacity()); + self.buf.reserve(size.saturating_sub(self.buf.len())); + } + + #[inline(always)] + pub fn extend_from_slice(&mut self, bytes: &[u8]) { + self.buf.extend_from_slice(bytes); + } + + #[inline(always)] + pub fn extend_checked(&mut self, bytes: &[u8], limit: usize) -> Result<(), usize> { + let allowed = limit.saturating_sub(self.buf.len()); + if bytes.len() <= allowed { + self.buf.extend_from_slice(bytes); + Ok(()) + } else { + Err(allowed) + } + } + + #[inline(always)] + pub fn extend_checked_uppercase(&mut self, bytes: &[u8], limit: usize) -> Result<(), usize> { + let allowed = limit.saturating_sub(self.buf.len()); + if bytes.len() <= allowed { + let appended_from = self.buf.len(); + self.buf.extend_from_slice(bytes); + if let Some(appended) = self.buf.get_mut(appended_from..) { + appended.make_ascii_uppercase(); + } + Ok(()) + } else { + Err(allowed) } } @@ -442,10 +712,8 @@ impl ArgumentBuffer { self.buf.push(byte); } - pub fn take(&mut self) -> Vec { - let buf = self.buf.clone(); - self.buf.clear(); - buf + pub fn take_moved(&mut self) -> Vec { + std::mem::replace(&mut self.buf, Vec::with_capacity(ARG_INIT_LEN)) } #[inline(always)] @@ -470,19 +738,45 @@ impl ArgumentBuffer { } impl Token { - pub fn unwrap_string(self) -> crate::parser::Result { + pub fn unwrap_string(self) -> crate::parser::Result { match self { - Token::Argument(value) => { - String::from_utf8(value).map_err(|_| "Invalid UTF-8 in argument.".into()) - } - other => Ok(other.to_string()), + Token::Argument(value) if value.spilled() => String::from_utf8(value.into_vec()) + .map(CompactString::from_string_buffer) + .map_err(|_| "Invalid UTF-8 in argument.".into()), + Token::Argument(value) => CompactString::from_utf8(value.as_slice()) + .map_err(|_| "Invalid UTF-8 in argument.".into()), + other => Ok(CompactString::from(other.to_string())), } } - pub fn unwrap_bytes(self) -> Vec { + pub fn unwrap_mailbox_name(self, is_utf8: bool) -> crate::parser::Result { + let name = self.unwrap_string()?; + if is_utf8 || !name.contains('&') { + Ok(name) + } else { + Ok(IMAP + .lenient() + .decode(name.as_bytes()) + .map(CompactString::from_string_buffer) + .unwrap_or(name)) + } + } + + pub fn unwrap_mailbox_name_strict(self, is_utf8: bool) -> crate::parser::Result { + let name = self.unwrap_string()?; + if is_utf8 || !name.contains('&') { + Ok(name) + } else { + IMAP.decode(name.as_bytes()) + .map(CompactString::from_string_buffer) + .map_err(|err| format!("Invalid modified UTF-7 in mailbox name: {err}.").into()) + } + } + + pub fn unwrap_bytes(self) -> ArgumentBytes { match self { - Token::Argument(value) => value, - other => other.as_bytes().to_vec(), + Token::Argument(value) | Token::NilString(value) => value, + other => ArgumentBytes::from_slice(other.as_bytes()), } } @@ -497,6 +791,7 @@ impl Token { Token::Lt => bytes.eq(b"<"), Token::Dot => bytes.eq(b"."), Token::Nil => bytes.is_empty(), + Token::NilString(value) => value.eq_ignore_ascii_case(bytes), } } @@ -550,7 +845,7 @@ impl Display for Token { impl Token { pub fn as_bytes(&self) -> &[u8] { match self { - Token::Argument(value) => value, + Token::Argument(value) => value.as_slice(), Token::ParenthesisOpen => b"(", Token::ParenthesisClose => b")", Token::BracketOpen => b"[", @@ -559,18 +854,23 @@ impl Token { Token::Lt => b"<", Token::Dot => b".", Token::Nil => b"", + Token::NilString(value) => value.as_slice(), } } } impl Error { - pub fn err(tag: Option>, message: impl Into) -> Self { + pub fn err( + tag: Option>, + message: impl Into, + code: ResponseCode, + ) -> Self { Error::Error { response: trc::ImapEvent::Error .ctx(trc::Key::Details, message) .ctx_opt(trc::Key::Id, tag.map(Into::into)) .ctx(trc::Key::Type, ResponseType::Bad) - .code(ResponseCode::Parse), + .code(code), } } } @@ -584,6 +884,7 @@ impl Default for Receiver { start_state: State::Start, max_request_size: 25 * 1024 * 1024, current_request_size: 0, + skipping: false, } } } @@ -592,13 +893,13 @@ impl Request { pub fn into_error(self, message: impl Into) -> trc::Error { trc::ImapEvent::Error .ctx(trc::Key::Details, message) - .ctx(trc::Key::Id, CompactString::from_string_buffer(self.tag)) + .ctx(trc::Key::Id, self.tag) } pub fn into_parse_error(self, message: impl Into) -> trc::Error { trc::ImapEvent::Error .ctx(trc::Key::Details, message) - .ctx(trc::Key::Id, CompactString::from_string_buffer(self.tag)) + .ctx(trc::Key::Id, self.tag) .ctx(trc::Key::Code, ResponseCode::Parse) .ctx(trc::Key::Type, ResponseType::Bad) } @@ -640,534 +941,3 @@ DQUOTE = %x22 ; " (Double Quote) */ -#[cfg(test)] -mod tests { - - use crate::Command; - - use super::{Error, Receiver, Request, Token}; - - #[test] - fn receiver_parse_ok() { - let mut receiver = Receiver::new(); - - for (frames, expected_requests) in [ - ( - vec!["abcd CAPABILITY\r\n"], - vec![Request { - tag: "abcd".into(), - command: Command::Capability, - tokens: vec![], - }], - ), - ( - vec!["A023 LO", "GOUT\r\n"], - vec![Request { - tag: "A023".into(), - command: Command::Logout, - tokens: vec![], - }], - ), - ( - vec![" A001 AUTHENTICATE GSSAPI \r\n"], - vec![Request { - tag: "A001".into(), - command: Command::Authenticate, - tokens: vec![Token::Argument(b"GSSAPI".to_vec())], - }], - ), - ( - vec!["A03 AUTHENTICATE ", "PLAIN dGVzdAB0ZXN", "0AHRlc3Q=\r\n"], - vec![Request { - tag: "A03".into(), - command: Command::Authenticate, - tokens: vec![ - Token::Argument(b"PLAIN".to_vec()), - Token::Argument(b"dGVzdAB0ZXN0AHRlc3Q=".to_vec()), - ], - }], - ), - ( - vec!["A003 CREATE owatagusiam/\r\n"], - vec![Request { - tag: "A003".into(), - command: Command::Create, - tokens: vec![Token::Argument(b"owatagusiam/".to_vec())], - }], - ), - ( - vec!["A682 LIST \"\" *\r\n"], - vec![Request { - tag: "A682".into(), - command: Command::List, - tokens: vec![Token::Nil, Token::Argument(b"*".to_vec())], - }], - ), - ( - vec!["A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n"], - vec![Request { - tag: "A03".into(), - command: Command::List, - tokens: vec![ - Token::ParenthesisOpen, - Token::ParenthesisClose, - Token::Nil, - Token::Argument(b"%".to_vec()), - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"CHILDREN".to_vec()), - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n"], - vec![Request { - tag: "A05".into(), - command: Command::List, - tokens: vec![ - Token::ParenthesisOpen, - Token::Argument(b"REMOTE".to_vec()), - Token::Argument(b"SUBSCRIBED".to_vec()), - Token::ParenthesisClose, - Token::Nil, - Token::Argument(b"*".to_vec()), - ], - }], - ), - ( - vec!["a1 list \"\" (\"foo\")\r\n"], - vec![Request { - tag: "a1".into(), - command: Command::List, - tokens: vec![ - Token::Nil, - Token::ParenthesisOpen, - Token::Argument(b"foo".to_vec()), - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["a3.1 LIST \"\" (% music/rock)\r\n"], - vec![Request { - tag: "a3.1".into(), - command: Command::List, - tokens: vec![ - Token::Nil, - Token::ParenthesisOpen, - Token::Argument(b"%".to_vec()), - Token::Argument(b"music/rock".to_vec()), - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n"], - vec![Request { - tag: "A01".into(), - command: Command::List, - tokens: vec![ - Token::Nil, - Token::Argument(b"%".to_vec()), - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"STATUS".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"MESSAGES".to_vec()), - Token::Argument(b"UNSEEN".to_vec()), - Token::ParenthesisClose, - Token::ParenthesisClose, - ], - }], - ), - ( - vec![" A01 LiSt \"\" % RETURN ( STATUS ( MESSAGES UNSEEN ) ) \r\n"], - vec![Request { - tag: "A01".into(), - command: Command::List, - tokens: vec![ - Token::Nil, - Token::Argument(b"%".to_vec()), - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"STATUS".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"MESSAGES".to_vec()), - Token::Argument(b"UNSEEN".to_vec()), - Token::ParenthesisClose, - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["A02 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" % RETURN (STATUS (MESSAGES))\r\n"], - vec![Request { - tag: "A02".into(), - command: Command::List, - tokens: vec![ - Token::ParenthesisOpen, - Token::Argument(b"SUBSCRIBED".to_vec()), - Token::Argument(b"RECURSIVEMATCH".to_vec()), - Token::ParenthesisClose, - Token::Nil, - Token::Argument(b"%".to_vec()), - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"STATUS".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"MESSAGES".to_vec()), - Token::ParenthesisClose, - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["A002 CREATE \"INBOX.Sent Mail\"\r\n"], - vec![Request { - tag: "A002".into(), - command: Command::Create, - tokens: vec![Token::Argument(b"INBOX.Sent Mail".to_vec())], - }], - ), - ( - vec!["A002 CREATE \"Maibox \\\"quo\\\\ted\\\" \"\r\n"], - vec![Request { - tag: "A002".into(), - command: Command::Create, - tokens: vec![Token::Argument(b"Maibox \"quo\\ted\" ".to_vec())], - }], - ), - ( - vec!["A004 COPY 2:4 meeting\r\n"], - vec![Request { - tag: "A004".into(), - command: Command::Copy(false), - tokens: vec![ - Token::Argument(b"2:4".to_vec()), - Token::Argument(b"meeting".to_vec()), - ], - }], - ), - ( - vec![ - "A282 SEARCH RETURN (MIN COU", - "NT) FLAGGED SINCE 1-Feb-1994 ", - "NOT FROM \"Smith\"\r\n", - ], - vec![Request { - tag: "A282".into(), - command: Command::Search(false), - tokens: vec![ - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"MIN".to_vec()), - Token::Argument(b"COUNT".to_vec()), - Token::ParenthesisClose, - Token::Argument(b"FLAGGED".to_vec()), - Token::Argument(b"SINCE".to_vec()), - Token::Argument(b"1-Feb-1994".to_vec()), - Token::Argument(b"NOT".to_vec()), - Token::Argument(b"FROM".to_vec()), - Token::Argument(b"Smith".to_vec()), - ], - }], - ), - ( - vec!["F284 UID STORE $ +FLAGS.Silent (\\Deleted)\r\n"], - vec![Request { - tag: "F284".into(), - command: Command::Store(true), - tokens: vec![ - Token::Argument(b"$".to_vec()), - Token::Argument(b"+FLAGS.Silent".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"\\Deleted".to_vec()), - Token::ParenthesisClose, - ], - }], - ), - ( - vec!["A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)])\r\n"], - vec![Request { - tag: "A654".into(), - command: Command::Fetch(false), - tokens: vec![ - Token::Argument(b"2:4".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"FLAGS".to_vec()), - Token::Argument(b"BODY".to_vec()), - Token::BracketOpen, - Token::Argument(b"HEADER".to_vec()), - Token::Dot, - Token::Argument(b"FIELDS".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"DATE".to_vec()), - Token::Argument(b"FROM".to_vec()), - Token::ParenthesisClose, - Token::BracketClose, - Token::ParenthesisClose, - ], - }], - ), - ( - vec![ - "B283 UID SEARCH RETURN (SAVE) CHARSET ", - "KOI8-R (OR $ 1,3000:3021) TEXT \"hello world\"\r\n", - ], - vec![Request { - tag: "B283".into(), - command: Command::Search(true), - tokens: vec![ - Token::Argument(b"RETURN".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"SAVE".to_vec()), - Token::ParenthesisClose, - Token::Argument(b"CHARSET".to_vec()), - Token::Argument(b"KOI8-R".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"OR".to_vec()), - Token::Argument(b"$".to_vec()), - Token::Argument(b"1,3000:3021".to_vec()), - Token::ParenthesisClose, - Token::Argument(b"TEXT".to_vec()), - Token::Argument(b"hello world".to_vec()), - ], - }], - ), - ( - vec![ - "P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) ", - "TEXT {8+}\r\nмать\r\n", - ], - vec![Request { - tag: "P283".into(), - command: Command::Search(false), - tokens: vec![ - Token::Argument(b"CHARSET".to_vec()), - Token::Argument(b"UTF-8".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"OR".to_vec()), - Token::Argument(b"$".to_vec()), - Token::Argument(b"1,3000:3021".to_vec()), - Token::ParenthesisClose, - Token::Argument(b"TEXT".to_vec()), - Token::Argument("мать".to_string().into_bytes()), - ], - }], - ), - ( - vec!["A001 LOGIN {11}\r\n", "FRED FOOBAR {7}\r\n", "fat man\r\n"], - vec![Request { - tag: "A001".into(), - command: Command::Login, - tokens: vec![ - Token::Argument(b"FRED FOOBAR".to_vec()), - Token::Argument(b"fat man".to_vec()), - ], - }], - ), - ( - vec!["TAG3 CREATE \"Test-ąęć-Test\"\r\n"], - vec![Request { - tag: "TAG3".into(), - command: Command::Create, - tokens: vec![Token::Argument("Test-ąęć-Test".as_bytes().to_vec())], - }], - ), - ( - vec!["abc LOGIN {0}\r\n", "\r\n"], - vec![Request { - tag: "abc".into(), - command: Command::Login, - tokens: vec![Token::Nil], - }], - ), - ( - vec!["abc LOGIN {0+}\r\n\r\n"], - vec![Request { - tag: "abc".into(), - command: Command::Login, - tokens: vec![Token::Nil], - }], - ), - ( - vec![ - "A003 APPEND saved-messages (\\Seen) {297+}\r\n", - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@example.com\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", - ], - vec![Request { - tag: "A003".into(), - command: Command::Append, - tokens: vec![ - Token::Argument(b"saved-messages".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"\\Seen".to_vec()), - Token::ParenthesisClose, - Token::Argument( - concat!( - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@example.com\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n" - ) - .as_bytes() - .to_vec(), - ), - ], - }], - ), - ( - vec![ - "A003 APPEND saved-messages (\\Seen) {326}\r\n", - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@owatagu.siam.edu.example\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", - ], - vec![Request { - tag: "A003".into(), - command: Command::Append, - tokens: vec![ - Token::Argument(b"saved-messages".to_vec()), - Token::ParenthesisOpen, - Token::Argument(b"\\Seen".to_vec()), - Token::ParenthesisClose, - Token::Argument( - concat!( - "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", - "From: Fred Foobar \r\n", - "Subject: afternoon meeting\r\n", - "To: mooch@owatagu.siam.edu.example\r\n", - "Message-Id: \r\n", - "MIME-Version: 1.0\r\n", - "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", - "\r\n", - "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n", - ) - .as_bytes() - .to_vec(), - ), - ], - }], - ), - ( - vec!["001 NOOP\r\n002 CAPABILITY\r\nabc LOGIN hello world\r\n"], - vec![ - Request { - tag: "001".into(), - command: Command::Noop, - tokens: vec![], - }, - Request { - tag: "002".into(), - command: Command::Capability, - tokens: vec![], - }, - Request { - tag: "abc".into(), - command: Command::Login, - tokens: vec![ - Token::Argument(b"hello".to_vec()), - Token::Argument(b"world".to_vec()), - ], - }, - ], - ), - ] { - let mut requests = Vec::new(); - for frame in &frames { - let mut bytes = frame.as_bytes().iter(); - loop { - match receiver.parse(&mut bytes) { - Ok(request) => requests.push(request), - Err(Error::NeedsMoreData | Error::NeedsLiteral { .. }) => break, - Err(err) => panic!("{:?} for frames {:#?}", err, frames), - } - } - } - assert_eq!(requests, expected_requests, "{:#?}", frames); - } - } - - #[test] - fn receiver_parse_invalid() { - let mut receiver = Receiver::::new(); - for invalid in [ - //"\r\n", - //" \r \n", - "a001\r\n", - "a001 unknown\r\n", - "a001 login {abc}\r\n", - "a001 login {+30}\r\n", - "a001 login {30} junk\r\n", - ] { - match receiver.parse(&mut invalid.as_bytes().iter()) { - Err(Error::Error { .. }) => {} - result => panic!("Expecter error, got: {:?}", result), - } - } - } - - #[test] - fn receiver_discard_oversized_non_sync_literal() { - for frames in [ - vec![ - "a1 APPEND inbox (\\Draft) {40+}\r\n", - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n", - "b2 NOOP\r\n", - ], - vec![ - "a1 APPEND inbox (\\Draft) {40+}\r\nAAAAAAAAAAAAAAAAAAAA", - "AAAAAAAAAAAAAAAAAAAA\r\nb2 NOOP\r\n", - ], - vec![ - "a1 APPEND inbox (\\Draft) {40+}\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nb2 NOOP\r\n", - ], - ] { - let mut receiver = Receiver::::with_max_request_size(30); - let mut requests = Vec::new(); - let mut errors = 0; - for frame in &frames { - let mut bytes = frame.as_bytes().iter(); - loop { - match receiver.parse(&mut bytes) { - Ok(request) => requests.push(request), - Err(Error::NeedsMoreData | Error::NeedsLiteral { .. }) => break, - Err(Error::Error { .. }) => errors += 1, - } - } - } - - assert_eq!(errors, 1, "expected a single error for {:#?}", frames); - assert_eq!( - requests, - vec![Request { - tag: "b2".into(), - command: Command::Noop, - tokens: vec![], - }], - "connection did not resync for {:#?}", - frames - ); - } - } -} diff --git a/crates/imap-proto/src/utf7.rs b/crates/imap-proto/src/utf7.rs deleted file mode 100644 index f314e153..00000000 --- a/crates/imap-proto/src/utf7.rs +++ /dev/null @@ -1,189 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -// Ported from https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs -// Author: Jeffrey Stedfast - -static UTF_7_RANK: &[u8] = &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 62, 63, 255, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, - 255, 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, -]; - -static UTF_7_MAP: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; - -pub fn utf7_decode(text: &str) -> Option { - let mut bytes: Vec = Vec::with_capacity(text.len()); - let mut bits = 0; - let mut v: u32 = 0; - let mut shifted = false; - let mut text = text.chars().peekable(); - - while let Some(ch) = text.next() { - if shifted { - if ch == '-' { - shifted = false; - bits = 0; - v = 0; - } else if ch as usize > 127 { - return None; - } else { - let rank = *UTF_7_RANK.get(ch as usize)?; - - if rank == 0xff { - return None; - } - - v = (v << 6) | rank as u32; - bits += 6; - - if bits >= 16 { - bytes.push(((v >> (bits - 16)) & 0xffff) as u16); - bits -= 16; - } - } - } else if ch == '&' { - match text.peek() { - Some('-') => { - bytes.push(b'&' as u16); - text.next(); - } - Some(_) => { - shifted = true; - } - None => { - bytes.push(ch as u16); - } - } - } else { - bytes.push(ch as u16); - } - } - - String::from_utf16(&bytes).ok() -} - -pub fn utf7_encode(text: &str) -> String { - let mut result = String::with_capacity(text.len()); - let mut shifted = false; - let mut bits = 0; - let mut u: u32 = 0; - - for ch in text.encode_utf16() { - if (0x20..0x7f).contains(&ch) { - if shifted { - if bits > 0 { - result.push(char::from(UTF_7_MAP[((u << (6 - bits)) & 0x3f) as usize])); - } - result.push('-'); - shifted = false; - bits = 0; - } - - if ch == 0x26 { - result.push_str("&-"); - } else { - result.push((ch as u8) as char); - } - } else { - if !shifted { - result.push('&'); - shifted = true; - } - - u = (u << 16) | ch as u32; - bits += 16; - - while bits >= 6 { - result.push(char::from(UTF_7_MAP[((u >> (bits - 6)) & 0x3f) as usize])); - bits -= 6; - } - } - } - - if shifted { - if bits > 0 { - result.push(char::from(UTF_7_MAP[((u << (6 - bits)) & 0x3f) as usize])); - } - result.push('-'); - } - - result -} - -#[inline(always)] -pub fn utf7_maybe_decode(text: String, is_utf8: bool) -> String { - if is_utf8 { - text - } else { - utf7_decode(&text).unwrap_or(text) - } -} - -#[cfg(test)] -mod tests { - - #[test] - fn utf7_decode() { - for (input, expected_result) in [ - ("~peter/mail/&U,BTFw-/&ZeVnLIqe-", "~peter/mail/台北/日本語"), - ("&U,BTF2XlZyyKng-", "台北日本語"), - ("Hello, World&ACE-", "Hello, World!"), - ("Hi Mom -&Jjo--!", "Hi Mom -☺-!"), - ("&ZeVnLIqe-", "日本語"), - ("Item 3 is &AKM-1.", "Item 3 is £1."), - ("Plus minus &- -&- &--", "Plus minus & -& &-"), - ( - "&APw-ber ihre mi&AN8-liche Lage&ADs- &ACI-wir", - "über ihre mißliche Lage; \"wir", - ), - ( - concat!( - "&ACI-The sayings of Confucius,&ACI- James R. Ware, trans. &U,BTFw-:\n", - "&ZYeB9FH6ckh5Pg-, 1980.\n", - "&Vttm+E6UfZM-, &W4tRQ066bOg-, &UxdOrA-: &Ti1XC2b4Xpc-, 1990." - ), - concat!( - "\"The sayings of Confucius,\" James R. Ware, trans. 台北:\n", - "文致出版社, 1980.\n", - "四書五經, 宋元人注, 北京: 中國書店, 1990." - ), - ), - ("Test-ąęć-Test", "Test-ąęć-Test"), - (r#"&A8g- "&A9QD1APUA9gD3APcA-+""#, "ψ \"ϔϔϔϘϜϜ+\""), - ] { - assert_eq!( - super::utf7_decode(input).expect(input), - expected_result, - "while decoding {:?}", - input - ); - } - } - - #[test] - fn utf7_encode() { - for (expected_result, input) in [ - ("~peter/mail/&U,BTFw-/&ZeVnLIqe-", "~peter/mail/台北/日本語"), - ("&U,BTF2XlZyyKng-", "台北日本語"), - ("Hi Mom -&Jjo--!", "Hi Mom -☺-!"), - ("&ZeVnLIqe-", "日本語"), - ("Item 3 is &AKM-1.", "Item 3 is £1."), - ("Plus minus &- -&- &--", "Plus minus & -& &-"), - ("&VMhUyNg93gQ-", "哈哈😄"), - ] { - assert_eq!( - super::utf7_encode(input), - expected_result, - "while encoding {:?}", - expected_result - ); - } - } -} diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 50468085..1c96b95d 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -14,17 +14,17 @@ email = { path = "../email" } nlp = { path = "../nlp" } utils = { path = "../utils" } registry = { path = "../registry" } -mail-parser = { version = "0.11", features = ["full_encoding"] } -tokio = { version = "1.53", features = ["full"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +encodify = "1.0" +tokio = { version = "1.54", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } -parking_lot = "0.12" +parking_lot = "0.13" ahash = { version = "0.8" } -md5 = "0.8.1" -rand = "0.10.2" +rand = "0.10.3" compact_str = "0.10.0" [features] -test_mode = [] +enterprise = ["directory/enterprise", "trc/enterprise", "store/enterprise", "common/enterprise", "email/enterprise", "utils/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/imap/src/core/client.rs b/crates/imap/src/core/client.rs index 2de60d8d..fe099714 100644 --- a/crates/imap/src/core/client.rs +++ b/crates/imap/src/core/client.rs @@ -4,19 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{iter::Peekable, sync::Arc, vec::IntoIter}; - -use common::{ - KV_RATE_LIMIT_IMAP, - network::{SessionResult, SessionStream}, -}; +use super::{MAX_CONCURRENT_OPS, SelectedAccess, SelectedMailbox, Session, SessionData, State}; +use crate::op::ImapContext; +use common::network::{SessionResult, SessionStream}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, receiver::{self, Request}, }; +use std::{iter::Peekable, sync::Arc, vec::IntoIter}; +use tokio::sync::OwnedSemaphorePermit; use trc::SecurityEvent; -use super::{SelectedMailbox, Session, SessionData, State}; +const UNAUTHENTICATED_LITERAL_BYE: &[u8] = + b"* BYE [TOOBIG] Literal exceeds the maximum size allowed before authentication.\r\n"; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { @@ -30,22 +30,18 @@ impl Session { let mut bytes = bytes.iter(); let mut requests = Vec::with_capacity(2); let mut needs_literal = None; - let mut has_expunge = false; loop { match self.receiver.parse(&mut bytes) { - Ok(request) => match self.is_allowed(request).await { - Ok(request) => { - has_expunge |= - matches!(request.command, Command::Expunge(_) | Command::Close); - requests.push(request); - } - Err(err) => { - if !self.write_error(err).await { + Ok(request) => { + if let Some(failure) = self.state.take_sasl_failure() { + if !self.write_error(failure).await { return SessionResult::Close; } + } else { + requests.push(request); } - }, + } Err(receiver::Error::NeedsMoreData) => { break; } @@ -54,6 +50,20 @@ impl Session { break; } Err(receiver::Error::Error { response }) => { + let is_discarding = self.receiver.is_discarding(); + if is_discarding && matches!(self.state, State::NotAuthenticated { .. }) { + trc::error!(response.span_id(self.session_id)); + self.write_bytes(UNAUTHENTICATED_LITERAL_BYE).await.ok(); + return SessionResult::Close; + } + + if let Some(failure) = self.state.take_sasl_failure() { + if !self.write_error(failure).await { + return SessionResult::Close; + } + break; + } + // Check for port scanners if matches!( (&self.state, response.key(trc::Key::Code)), @@ -86,56 +96,83 @@ impl Session { if !self.write_error(response).await { return SessionResult::Close; } - break; + if !is_discarding { + break; + } } } } let mut requests = requests.into_iter().peekable(); while let Some(request) = requests.next() { + let request = match self.is_allowed(request).await { + Ok(request) => request, + Err(err) => { + if !self.write_error(err).await { + return SessionResult::Close; + } + continue; + } + }; + if let Err(err) = self.assert_selected_readable(&request).await { + if !self.write_error(err).await { + return SessionResult::Close; + } + continue; + } + let result = match request.command { Command::List | Command::Lsub => self - .handle_list(request) + .handle_list(request, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::Select | Command::Examine => self - .handle_select(request) + .handle_select(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Create => self - .handle_create(group_requests(&mut requests, vec![request])) + .handle_create( + group_requests(&mut requests, vec![request]), + self.acquire_write_permit().await, + ) .await .map(|_| SessionResult::Continue), Command::Delete => self - .handle_delete(group_requests(&mut requests, vec![request])) + .handle_delete( + group_requests(&mut requests, vec![request]), + self.acquire_write_permit().await, + ) .await .map(|_| SessionResult::Continue), Command::Rename => self - .handle_rename(request) + .handle_rename(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Status => self - .handle_status(group_requests(&mut requests, vec![request])) + .handle_status( + group_requests(&mut requests, vec![request]), + self.acquire_read_permit().await, + ) .await .map(|_| SessionResult::Continue), Command::Append => self - .handle_append(request) + .handle_append(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Close => self - .handle_close(request) + .handle_close(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Unselect => self - .handle_unselect(request) + .handle_unselect(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Expunge(is_uid) => self - .handle_expunge(request, is_uid) + .handle_expunge(request, is_uid, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Search(is_uid) => self - .handle_search(request, false, is_uid) + .handle_search(request, false, is_uid, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::Fetch(_) => self @@ -143,45 +180,47 @@ impl Session { .await .map(|_| SessionResult::Continue), Command::Store(is_uid) => self - .handle_store(request, is_uid, !has_expunge) + .handle_store(request, is_uid, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Copy(is_uid) => self - .handle_copy_move(request, false, is_uid) + .handle_copy_move(request, false, is_uid, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Move(is_uid) => self - .handle_copy_move(request, true, is_uid) + .handle_copy_move(request, true, is_uid, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Sort(is_uid) => self - .handle_search(request, true, is_uid) + .handle_search(request, true, is_uid, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::Thread(is_uid) => self - .handle_thread(request, is_uid) + .handle_thread(request, is_uid, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::Idle => self - .handle_idle(request) + .handle_idle(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Subscribe => self - .handle_subscribe(request, true) + .handle_subscribe(request, true, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Unsubscribe => self - .handle_subscribe(request, false) + .handle_subscribe(request, false, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Namespace => self .handle_namespace(request) .await .map(|_| SessionResult::Continue), - Command::Authenticate => Box::pin(self.handle_authenticate(request)) + Command::Authenticate => self + .handle_authenticate(request) .await .map(|_| SessionResult::Continue), - Command::Login => Box::pin(self.handle_login(request)) + Command::Login => self + .handle_login(request) .await .map(|_| SessionResult::Continue), Command::Capability => self @@ -201,11 +240,11 @@ impl Session { .await .map(|_| SessionResult::UpgradeTls), Command::Noop => self - .handle_noop(request) + .handle_noop(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Check => self - .handle_noop(request) + .handle_noop(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Logout => self @@ -213,15 +252,15 @@ impl Session { .await .map(|_| SessionResult::Close), Command::SetAcl => self - .handle_set_acl(request) + .handle_set_acl(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::DeleteAcl => self - .handle_set_acl(request) + .handle_set_acl(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::GetAcl => self - .handle_get_acl(request) + .handle_get_acl(request, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::ListRights => self @@ -229,19 +268,19 @@ impl Session { .await .map(|_| SessionResult::Continue), Command::MyRights => self - .handle_my_rights(request) + .handle_my_rights(request, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::GetQuota => self - .handle_get_quota(request) + .handle_get_quota(request, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::GetQuotaRoot => self - .handle_get_quota_root(request) + .handle_get_quota_root(request, self.acquire_read_permit().await) .await .map(|_| SessionResult::Continue), Command::Unauthenticate => self - .handle_unauthenticate(request) + .handle_unauthenticate(request, self.acquire_write_permit().await) .await .map(|_| SessionResult::Continue), Command::Id => self @@ -256,6 +295,14 @@ impl Session { .handle_uidbatches(request) .await .map(|_| SessionResult::Continue), + Command::GetMetadata => self + .handle_get_metadata(request, self.acquire_read_permit().await) + .await + .map(|_| SessionResult::Continue), + Command::SetMetadata => self + .handle_set_metadata(request, self.acquire_write_permit().await) + .await + .map(|_| SessionResult::Continue), }; match result { @@ -280,6 +327,18 @@ impl Session { SessionResult::Continue } + + pub async fn acquire_read_permit(&self) -> Option { + self.op_semaphore.clone().acquire_owned().await.ok() + } + + pub async fn acquire_write_permit(&self) -> Option { + self.op_semaphore + .clone() + .acquire_many_owned(MAX_CONCURRENT_OPS) + .await + .ok() + } } pub fn group_requests( @@ -301,22 +360,13 @@ pub fn group_requests( impl Session { async fn is_allowed(&self, request: Request) -> trc::Result> { let state = &self.state; - // Rate limit request if let State::Authenticated { data } | State::Selected { data, .. } = state - && let Some(rate) = &self.server.core.imap.rate_requests - && data + && let Err(err) = self .server - .in_memory_store() - .is_rate_allowed( - KV_RATE_LIMIT_IMAP, - &data.account_id.to_be_bytes(), - rate, - true, - ) - .await? - .is_some() + .is_imap_request_allowed(&data.access_token) + .await { - return Err(trc::LimitEvent::TooManyRequests.into_err()); + return Err(err.id(request.tag)); } match &request.command { @@ -395,7 +445,9 @@ impl Session { | Command::Unauthenticate | Command::GetQuota | Command::GetQuotaRoot - | Command::GetJmapAccess => { + | Command::GetJmapAccess + | Command::GetMetadata + | Command::SetMetadata => { if let State::Authenticated { .. } | State::Selected { .. } = state { Ok(request) } else { @@ -452,9 +504,48 @@ impl Session { }, } } + + async fn assert_selected_readable(&self, request: &Request) -> trc::Result<()> { + let State::Selected { data, mailbox } = &self.state else { + return Ok(()); + }; + let touches_selected = matches!( + request.command, + Command::Expunge(_) + | Command::Search(_) + | Command::Fetch(_) + | Command::Store(_) + | Command::Copy(_) + | Command::Move(_) + | Command::Sort(_) + | Command::Thread(_) + | Command::UidBatches + | Command::Check + | Command::Noop + | Command::Idle + ); + if !touches_selected || !data.is_acl_shared(mailbox.id.account_id) { + return Ok(()); + } + match data + .selected_access(&mailbox.id) + .await + .imap_ctx(&request.tag, trc::location!())? + { + SelectedAccess::Readable { .. } => Ok(()), + SelectedAccess::Revoked => Err(data.close_revoked().await.id(request.tag.clone())), + } + } } impl State { + pub fn take_sasl_failure(&mut self) -> Option { + match self { + State::NotAuthenticated { sasl_failure, .. } => sasl_failure.take(), + _ => None, + } + } + pub fn auth_failures(&self) -> u32 { match self { State::NotAuthenticated { auth_failures, .. } => *auth_failures, diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 60836b90..49f83144 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -4,27 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Account, MailboxId, MailboxSync, Session, SessionData}; -use crate::core::Mailbox; -use ahash::AHashMap; +use super::{ + AccountCaches, AccountView, CounterDelta, Counters, MailboxChange, MailboxCounters, MailboxId, + MailboxName, MailboxRefresh, MailboxSource, MailboxSync, SelectedAccess, Session, SessionData, +}; use common::{ + MailboxesCache, MessageStoreCache, auth::AccessToken, + config::mailstore::email::DefaultFolder, network::{SessionStream, limiter::InFlight}, sharing::EffectiveAcl, }; +use compact_str::CompactString; use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + cache::{ + MessageCacheFetch, + mailbox::{AccountMailboxes, MailboxCacheAccess, MailboxCacheFetch}, + }, mailbox::INBOX_ID, }; -use imap_proto::protocol::list::Attribute; +use imap_proto::protocol::{MailboxObjectId, ObjectId}; use parking_lot::Mutex; -use std::{collections::BTreeMap, sync::atomic::Ordering}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, +use std::{ + mem, + sync::{Arc, atomic::Ordering}, }; +use store::{ValueKey, roaring::RoaringBitmap, write::ValueClass}; use trc::AddContext; -use types::{acl::Acl, collection::Collection, keyword::Keyword, special_use::SpecialUse}; +use types::{acl::Acl, collection::Collection, special_use::SpecialUse}; + +const REVOKED_BYE: &[u8] = b"* BYE [NOPERM] Read access to the selected mailbox was revoked.\r\n"; impl SessionData { pub async fn new( @@ -32,332 +41,433 @@ impl SessionData { access_token: AccessToken, in_flight: Option, ) -> trc::Result { - let mut session = SessionData { + let data = SessionData { stream_tx: session.stream_tx.clone(), server: session.server.clone(), account_id: access_token.account_id(), session_id: session.session_id, - mailboxes: Mutex::new(vec![]), - state: access_token.state().into(), + mailboxes: Mutex::new(Vec::new()), + state: token_state(&access_token).into(), remote_addr: session.remote_addr, access_token, in_flight, }; - // Fetch mailboxes for the main account - let mut mailboxes = vec![ - session - .fetch_account_mailboxes(session.account_id, None, &session.access_token, None) + let shared = data + .access_token + .shared_accounts(Collection::Mailbox) + .copied() + .collect::>(); + let mut accounts = Vec::with_capacity(shared.len() + 1); + accounts.push( + data.account_view(data.account_id, None, &data.access_token) + .await + .caused_by(trc::location!())?, + ); + for account_id in shared { + let prefix = data.shared_prefix(account_id).await?; + accounts.push( + data.account_view(account_id, Some(prefix), &data.access_token) + .await + .caused_by(trc::location!())?, + ); + } + *data.mailboxes.lock() = accounts; + + Ok(data) + } + + async fn shared_prefix(&self, account_id: u32) -> trc::Result { + Ok(format!( + "{}/{}", + self.server.core.email.shared_folder, + self.server + .account(account_id) .await .caused_by(trc::location!())? - .unwrap(), - ]; - - // Fetch shared mailboxes - for &account_id in session.access_token.shared_accounts(Collection::Mailbox) { - let prefix: String = format!( - "{}/{}", - session.server.core.email.shared_folder, - session - .server - .account(account_id) - .await - .caused_by(trc::location!())? - .name() - ); - mailboxes.push( - session - .fetch_account_mailboxes(account_id, prefix.into(), &session.access_token, None) - .await - .caused_by(trc::location!())? - .unwrap(), - ); - } - - session.mailboxes = Mutex::new(mailboxes); - - Ok(session) + .name() + )) } - async fn fetch_account_mailboxes( + async fn account_view( &self, account_id: u32, - mailbox_prefix: Option, + prefix: Option, access_token: &AccessToken, - current_state: Option, - ) -> trc::Result> { - let cache = self - .server - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; - if current_state.is_some_and(|state| state == cache.last_change_id) { - return Ok(None); - } - - let shared_mailbox_ids = if access_token.is_member(account_id) { - None - } else { - cache.shared_mailboxes(access_token, Acl::Read).into() - }; - - // Build special uses - let mut special_uses = AHashMap::new(); - for mailbox in &cache.mailboxes.items { - if shared_mailbox_ids - .as_ref() - .is_none_or(|ids| ids.contains(mailbox.document_id)) - && !matches!(mailbox.role, SpecialUse::None) - { - special_uses.insert(mailbox.role, mailbox.document_id); - } - } - - // Build account - let mut account = Account { - account_id, - prefix: mailbox_prefix, - mailbox_names: BTreeMap::new(), - mailbox_state: AHashMap::with_capacity(cache.mailboxes.items.len()), - last_change_id: cache.last_change_id, - }; - - for mailbox in &cache.mailboxes.items { - if shared_mailbox_ids - .as_ref() - .is_some_and(|ids| !ids.contains(mailbox.document_id)) - { - continue; - } - - // Build mailbox path and map it to its effective id - let mailbox_name = if let Some(prefix) = &account.prefix { - let mut name = String::with_capacity(prefix.len() + mailbox.path.len() + 1); - name.push_str(prefix.as_str()); - name.push('/'); - name.push_str(mailbox.path.as_str()); - name - } else { - mailbox.path.clone() - }; - let effective_mailbox_id = self + ) -> trc::Result { + let (mailboxes, source) = if access_token.member_ids().any(|id| id == account_id) { + let cache = self .server - .core - .email - .default_folders - .iter() - .find(|f| f.name == mailbox_name || f.aliases.iter().any(|a| a == &mailbox_name)) - .and_then(|f| special_uses.get(&f.special_use)) - .copied() - .unwrap_or(mailbox.document_id); - account - .mailbox_names - .insert(mailbox_name, effective_mailbox_id); - account.mailbox_state.insert( - mailbox.document_id, - Mailbox { - has_children: cache - .mailboxes - .items - .iter() - .any(|child| child.parent_id == mailbox.document_id), - is_subscribed: mailbox.subscribers.contains(&access_token.account_id()), - special_use: match mailbox.role { - SpecialUse::Trash => Some(Attribute::Trash), - SpecialUse::Junk => Some(Attribute::Junk), - SpecialUse::Drafts => Some(Attribute::Drafts), - SpecialUse::Archive => Some(Attribute::Archive), - SpecialUse::Sent => Some(Attribute::Sent), - SpecialUse::Important => Some(Attribute::Important), - SpecialUse::Memos => Some(Attribute::Memos), - SpecialUse::Scheduled => Some(Attribute::Scheduled), - SpecialUse::Snoozed => Some(Attribute::Snoozed), - _ => None, - }, - total_messages: cache.in_mailbox(mailbox.document_id).count() as u64, - total_unseen: cache - .in_mailbox_without_keyword(mailbox.document_id, &Keyword::Seen) - .count() as u64, - total_deleted: cache - .in_mailbox_with_keyword(mailbox.document_id, &Keyword::Deleted) - .count() as u64, - uid_validity: mailbox.uid_validity as u64, - uid_next: self - .get_uid_next(&MailboxId { - account_id, - mailbox_id: mailbox.document_id, - }) - .await - .caused_by(trc::location!())? as u64, - total_deleted_storage: None, - size: None, + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + (cache.mailboxes.clone(), MailboxSource::Messages) + } else { + let fetched = self + .server + .fetch_mailboxes(account_id, None) + .await + .caused_by(trc::location!())?; + ( + fetched.mailboxes, + MailboxSource::Folders { + change_id: fetched.change_id, + verified_at: self.server.inner.cache.revalidate.now(), }, - ); - } - - Ok(account.into()) + ) + }; + Ok(AccountView::build( + &self.server.core.email.default_folders, + mailboxes, + source, + account_id, + prefix, + access_token, + )) } - pub async fn synchronize_mailboxes( + pub async fn synchronize_mailboxes(&self, return_changes: bool) -> trc::Result { + self.synchronize_mailboxes_using(return_changes, AccountCaches::default()) + .await + } + + pub async fn synchronize_mailboxes_using( &self, return_changes: bool, - ) -> trc::Result> { - let mut changes = if return_changes { - MailboxSync::default().into() - } else { - None - }; - - // Obtain access token + caches: AccountCaches, + ) -> trc::Result { let access_token = self .refresh_access_token() .await .caused_by(trc::location!())?; - let state = access_token.state(); + self.synchronize_mailboxes_as(access_token, return_changes, caches) + .await + } + + pub async fn synchronize_mailboxes_as( + &self, + access_token: AccessToken, + return_changes: bool, + mut caches: AccountCaches, + ) -> trc::Result { + let mut changes = return_changes.then(MailboxSync::default); + let state = token_state(&access_token); - // Shared mailboxes might have changed - let mut added_accounts = Vec::new(); if self.state.load(Ordering::Relaxed) != state { - // Remove unlinked shared accounts - let mut added_account_ids = Vec::new(); - { - let mut mailboxes = self.mailboxes.lock(); - let mut new_accounts = Vec::with_capacity(mailboxes.len()); + let added_account_ids = { + let mut accounts = self.mailboxes.lock(); let has_access_to = access_token .shared_accounts(Collection::Mailbox) .copied() .collect::>(); - for account in mailboxes.drain(..) { - if access_token.is_account_id(account.account_id) - || has_access_to.contains(&account.account_id) - { - new_accounts.push(account); - } else { - // Add unshared mailboxes to deleted list - if let Some(changes) = &mut changes { - for (mailbox_name, _) in account.mailbox_names { - changes.deleted.push(mailbox_name); - } - } + accounts.retain_mut(|account| { + let keep = access_token.is_account_id(account.account_id) + || has_access_to.contains(&account.account_id); + if keep { + account.expire_folders(); + } else if let Some(changes) = &mut changes { + let account_id = account.account_id; + changes + .deleted + .extend(mem::take(&mut account.names).into_iter().map(|entry| { + MailboxChange { + account_id, + name: entry.name, + objectid: MailboxObjectId::Empty, + } + })); } - } + keep + }); + has_access_to + .iter() + .copied() + .filter(|account_id| { + !accounts + .iter() + .skip(1) + .any(|account| account.account_id == *account_id) + }) + .collect::>() + }; - // Add new shared account ids - for account_id in has_access_to { - if !new_accounts - .iter() - .skip(1) - .any(|m| m.account_id == account_id) - { - added_account_ids.push(account_id); - } - } - *mailboxes = new_accounts; - } - - // Fetch mailboxes for each new shared account for account_id in added_account_ids { - let prefix: String = format!( - "{}/{}", - self.server.core.email.shared_folder, - self.server - .account(account_id) - .await - .caused_by(trc::location!())? - .name() - ); - added_accounts.push( - self.fetch_account_mailboxes(account_id, prefix.into(), &access_token, None) - .await? - .unwrap(), - ); + let prefix = self.shared_prefix(account_id).await?; + let account = self + .account_view(account_id, Some(prefix), &access_token) + .await?; + if let Some(changes) = &mut changes { + changes.added.extend( + account + .names + .iter() + .map(|entry| entry.to_change(account_id)), + ); + } + self.mailboxes.lock().push(account); } - // Update state self.state.store(state, Ordering::Relaxed); } - // Fetch mailbox changes for all accounts - let mut changed_accounts = Vec::new(); let account_states = self .mailboxes .lock() .iter() - .map(|m| (m.account_id, m.prefix.clone(), m.last_change_id)) + .map(|account| { + ( + account.account_id, + account.mailboxes.clone(), + account.source, + ) + }) .collect::>(); - for (account_id, prefix, last_state) in account_states { - if let Some(changed_account) = self - .fetch_account_mailboxes(account_id, prefix, &access_token, last_state.into()) - .await - .caused_by(trc::location!())? + + for (account_id, mailboxes, source) in account_states { + if let MailboxSource::Folders { verified_at, .. } = source { + if !self.server.inner.cache.revalidate.is_within(verified_at) { + self.refresh_folders(account_id, &access_token, &mut changes) + .await?; + } + continue; + } + + let cache = caches.fetch(&self.server, account_id).await?; + if !Arc::ptr_eq(&cache.mailboxes, &mailboxes) + && !self.refresh_account_structure( + account_id, + &cache.mailboxes, + &access_token, + &mut changes, + ) { - changed_accounts.push(changed_account); + continue; } - } - // Update mailboxes - if !changed_accounts.is_empty() || !added_accounts.is_empty() { - let mut mailboxes = self.mailboxes.lock(); - - for changed_account in changed_accounts { - if let Some(pos) = mailboxes - .iter() - .position(|a| a.account_id == changed_account.account_id) + if let Some(changes) = &mut changes { + let mut accounts = self.mailboxes.lock(); + if let Some(account) = accounts + .iter_mut() + .find(|account| account.account_id == account_id) + && let Some(previous) = account.refresh_counters(&cache) { - // Add changes and deletions - if let Some(changes) = &mut changes { - let old_account = &mailboxes[pos]; - let new_account = &changed_account; - - // Add new mailboxes - for (mailbox_name, mailbox_id) in new_account.mailbox_names.iter() { - if let Some(old_mailbox) = old_account.mailbox_state.get(mailbox_id) { - if let Some(mailbox) = new_account.mailbox_state.get(mailbox_id) - && (mailbox.total_messages != old_mailbox.total_messages - || mailbox.total_unseen != old_mailbox.total_unseen) - { - changes.changed.push(mailbox_name.clone()); - } - } else { - changes.added.push(mailbox_name.clone()); - } - } - - // Add deleted mailboxes - for (mailbox_name, mailbox_id) in &old_account.mailbox_names { - if !new_account.mailbox_state.contains_key(mailbox_id) { - changes.deleted.push(mailbox_name.clone()); - } + for entry in account.names.iter().filter(|entry| { + !changes + .added + .iter() + .any(|added| added.account_id == account_id && added.name == entry.name) + }) { + let before = previous + .get(entry.mailbox_id as usize) + .copied() + .unwrap_or_default(); + let after = account.counters_of(entry.mailbox_id); + if before.total != after.total || before.unseen != after.unseen { + changes.changed.push(entry.name.clone()); } } - - mailboxes[pos] = changed_account; - } else { - // Add newly shared accounts - if let Some(changes) = &mut changes { - changes - .added - .extend(changed_account.mailbox_names.keys().cloned()); - } - - mailboxes.push(changed_account); } } - - if !added_accounts.is_empty() { - // Add newly shared accounts - if let Some(changes) = &mut changes { - for added_account in &added_accounts { - changes - .added - .extend(added_account.mailbox_names.keys().cloned()); - } - } - mailboxes.extend(added_accounts); - } } - Ok(changes) + Ok(MailboxRefresh { + changes, + caches, + access_token, + }) + } + + async fn refresh_folders( + &self, + account_id: u32, + access_token: &AccessToken, + changes: &mut Option, + ) -> trc::Result<()> { + let Some(known) = self.known_folders(account_id) else { + return Ok(()); + }; + let verified_at = self.server.inner.cache.revalidate.now(); + let fetched = self + .server + .fetch_mailboxes(account_id, Some(&known)) + .await + .caused_by(trc::location!())?; + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + && let MailboxSource::Folders { change_id, .. } = account.source + && fetched.change_id >= change_id + { + if !Arc::ptr_eq(&fetched.mailboxes, &account.mailboxes) { + account.restructure( + &self.server.core.email.default_folders, + fetched.mailboxes, + access_token, + changes, + ); + } + account.source = MailboxSource::Folders { + change_id: fetched.change_id, + verified_at, + }; + } + Ok(()) + } + + pub fn expire_folders(&self, account_id: u32) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + { + account.expire_folders(); + } + } + + pub fn expire_folders_of(&self, mailbox_name: &str) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.holds_name(mailbox_name)) + { + account.expire_folders(); + } + } + + pub fn use_messages_of(&self, mailbox_name: &str) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.holds_name(mailbox_name)) + { + account.source = MailboxSource::Messages; + } + } + + fn known_folders(&self, account_id: u32) -> Option { + self.mailboxes + .lock() + .iter() + .find(|account| account.account_id == account_id) + .and_then(|account| match account.source { + MailboxSource::Folders { change_id, .. } => Some(AccountMailboxes { + mailboxes: account.mailboxes.clone(), + change_id, + }), + MailboxSource::Messages => None, + }) + } + + pub fn use_messages(&self, account_id: u32) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + { + account.source = MailboxSource::Messages; + } + } + + fn refresh_account_structure( + &self, + account_id: u32, + mailboxes: &Arc, + access_token: &AccessToken, + changes: &mut Option, + ) -> bool { + let mut accounts = self.mailboxes.lock(); + let Some(account) = accounts + .iter_mut() + .find(|account| account.account_id == account_id) + else { + return false; + }; + account.restructure( + &self.server.core.email.default_folders, + mailboxes.clone(), + access_token, + changes, + ); + true + } + + pub fn ensure_counters(&self, account_id: u32, cache: &MessageStoreCache) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + { + account.ensure_counters(cache); + } + } + + pub fn has_counters_at(&self, account_id: u32, change_id: u64) -> bool { + self.mailboxes + .lock() + .iter() + .find(|account| account.account_id == account_id) + .and_then(|account| account.counters.as_ref()) + .is_some_and(|counters| counters.change_id == change_id) + } + + pub fn take_previous_counters(&self, account_id: u32) -> Option> { + self.mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + .and_then(|account| account.previous_counters.take()) + } + + pub fn apply_counter_deltas( + &self, + account_id: u32, + from_change_id: u64, + to_change_id: u64, + deltas: &[CounterDelta], + rebuilt: bool, + ) { + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + { + account.apply_counter_deltas(from_change_id, to_change_id, deltas, rebuilt); + } + } + + pub async fn seed_mailbox_counters(&self) -> trc::Result<()> { + let account_ids = self + .mailboxes + .lock() + .iter() + .filter(|account| account.source == MailboxSource::Messages) + .map(|account| account.account_id) + .collect::>(); + for account_id in account_ids { + let cache = self + .server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + if let Some(account) = self + .mailboxes + .lock() + .iter_mut() + .find(|account| account.account_id == account_id) + { + account.ensure_counters(&cache); + account.previous_counters = None; + } + } + Ok(()) } pub fn get_mailbox_by_name(&self, mailbox_name: &str) -> Option { @@ -366,73 +476,401 @@ impl SessionData { if account .prefix .as_ref() - .is_none_or(|p| mailbox_name.starts_with(p.as_str())) + .is_none_or(|prefix| mailbox_name.starts_with(prefix.as_str())) { - for (mailbox_name_, mailbox_id_) in account.mailbox_names.iter() { - if (!is_inbox && mailbox_name_ == mailbox_name) - || (is_inbox && *mailbox_id_ == INBOX_ID) - { - return MailboxId { - account_id: account.account_id, - mailbox_id: *mailbox_id_, - } - .into(); + let mailbox_id = if is_inbox { + account.has_mailbox_id(INBOX_ID).then_some(INBOX_ID) + } else { + account.id_by_name(mailbox_name) + }; + if let Some(mailbox_id) = mailbox_id { + return MailboxId { + account_id: account.account_id, + mailbox_id, } + .into(); } } } None } - pub fn get_mailbox_by_id(&self, account_id: u32, mailbox_id: u32) -> Option { - for account in self.mailboxes.lock().iter() { - if account.account_id == account_id - && account.mailbox_names.values().any(|id| *id == mailbox_id) - { - return MailboxId { + pub fn get_mailbox_by_object_id( + &self, + object_id: &ObjectId, + access_token: &AccessToken, + ) -> Option<(MailboxId, CompactString)> { + let mailbox_id = u32::try_from(object_id.mailbox_id?.id()).ok()?; + let account_id = match object_id.account_id { + Some(account_id) => u32::try_from(account_id.id()).ok()?, + None => self.account_id, + }; + let accounts = self.mailboxes.lock(); + let account = accounts + .iter() + .find(|account| account.account_id == account_id)?; + let entry = account + .names + .iter() + .find(|entry| entry.mailbox_id == mailbox_id && !entry.is_alias)?; + let is_readable = access_token.is_member(account_id) + || account.mailbox(mailbox_id).is_some_and(|mailbox| { + mailbox + .acls + .as_slice() + .effective_acl(access_token) + .contains(Acl::ReadItems) + }); + is_readable.then(|| { + ( + MailboxId { account_id, mailbox_id, - } - .into(); - } + }, + entry.name.as_str().into(), + ) + }) + } + + pub fn mailbox_object_id(&self, mailbox: MailboxId, name: &str) -> ObjectId { + let is_alias = self + .mailboxes + .lock() + .iter() + .find(|account| account.account_id == mailbox.account_id) + .and_then(|account| account.alias_target(&self.server.core.email.default_folders, name)) + .is_some_and(|target| target != mailbox.mailbox_id); + if is_alias { + ObjectId::default() + } else { + ObjectId::from(mailbox) } - None + } + + pub fn uid_validity(&self, mailbox: &MailboxId) -> u32 { + self.mailboxes + .lock() + .iter() + .find(|account| account.account_id == mailbox.account_id) + .map_or(0, |account| account.uid_validity(mailbox.mailbox_id)) + } + + pub async fn uid_next( + &self, + cache: &MessageStoreCache, + mailbox: &MailboxId, + ) -> trc::Result { + cache + .refresh + .uid_next + .get_or_read(mailbox.mailbox_id, || async { + self.server + .core + .storage + .data + .get_counter(ValueKey { + account_id: mailbox.account_id, + collection: Collection::Mailbox.into(), + document_id: mailbox.mailbox_id, + class: ValueClass::MailboxUid, + }) + .await + .caused_by(trc::location!()) + .map(|v| (v + 1) as u32) + }) + .await } pub async fn check_mailbox_acl( &self, + cache: Option<&MessageStoreCache>, account_id: u32, - document_id: u32, + mailbox_id: u32, item: Acl, ) -> trc::Result { let access_token = self.refresh_access_token().await?; - Ok(access_token.is_member(account_id) - || self - .server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Mailbox, - document_id, - )) - .await - .and_then(|mailbox| { - if let Some(mailbox) = mailbox { - Ok(Some( - mailbox - .unarchive::()? - .acls - .effective_acl(&access_token) - .contains(item), - )) - } else { - Ok(None) - } - })? - .ok_or_else(|| { - trc::ImapEvent::Error - .caused_by(trc::location!()) - .details("Mailbox no longer exists.") - })?) + self.check_mailbox_acl_as(&access_token, cache, account_id, mailbox_id, item) + .await + } + + pub async fn check_mailbox_acl_as( + &self, + access_token: &AccessToken, + cache: Option<&MessageStoreCache>, + account_id: u32, + mailbox_id: u32, + item: Acl, + ) -> trc::Result { + if access_token.is_member(account_id) { + return Ok(true); + } + let synchronized; + let mailboxes = match cache { + Some(cache) => &cache.mailboxes, + None => { + synchronized = self + .mailboxes + .lock() + .iter() + .find(|account| account.account_id == account_id) + .map(|account| account.mailboxes.clone()) + .ok_or_else(|| { + trc::ImapEvent::Error + .caused_by(trc::location!()) + .details("Mailbox no longer exists.") + })?; + &synchronized + } + }; + mailboxes + .mailbox_by_id(&mailbox_id) + .map(|mailbox| { + mailbox + .acls + .as_slice() + .effective_acl(access_token) + .contains(item) + }) + .ok_or_else(|| { + trc::ImapEvent::Error + .caused_by(trc::location!()) + .details("Mailbox no longer exists.") + }) + } + + pub fn is_acl_shared(&self, account_id: u32) -> bool { + !self.access_token.member_ids().any(|id| id == account_id) + } + + pub async fn selected_access(&self, mailbox: &MailboxId) -> trc::Result { + let cache = self + .server + .get_cached_messages(mailbox.account_id) + .await + .caused_by(trc::location!())?; + let access_token = self + .refresh_access_token() + .await + .caused_by(trc::location!())?; + let is_readable = matches!( + self.check_mailbox_acl_as( + &access_token, + Some(&cache), + mailbox.account_id, + mailbox.mailbox_id, + Acl::ReadItems, + ) + .await, + Ok(true) + ); + Ok(if is_readable { + SelectedAccess::Readable { + access_token, + cache, + } + } else { + SelectedAccess::Revoked + }) + } + + pub async fn close_revoked(&self) -> trc::Error { + self.write_bytes(REVOKED_BYE).await.ok(); + trc::NetworkEvent::Closed + .into_err() + .details("Read access to the selected mailbox was revoked.") } } + +fn token_state(access_token: &AccessToken) -> u32 { + access_token.state() ^ access_token.revision() as u32 +} + +impl AccountView { + pub fn build( + default_folders: &[DefaultFolder], + mailboxes: Arc, + source: MailboxSource, + account_id: u32, + prefix: Option, + access_token: &AccessToken, + ) -> Self { + let visible = if access_token.is_member(account_id) { + None + } else { + Some(mailboxes.shared_mailboxes(access_token, Acl::Read)) + }; + let is_visible = |mailbox_id: u32| { + visible + .as_ref() + .is_none_or(|visible| visible.contains(mailbox_id)) + }; + + let mut parents = RoaringBitmap::new(); + let mut special_uses = Vec::new(); + for mailbox in mailboxes.items.iter() { + if let Some(parent_id) = mailbox.parent_id() { + parents.insert(parent_id); + } + if !matches!(mailbox.role, SpecialUse::None) && is_visible(mailbox.document_id) { + special_uses.push((mailbox.role, mailbox.document_id)); + } + } + + let mut names = Vec::with_capacity(mailboxes.items.len()); + for mailbox in mailboxes + .items + .iter() + .filter(|mailbox| is_visible(mailbox.document_id)) + { + let name = match &prefix { + Some(prefix) => { + let mut name = String::with_capacity(prefix.len() + mailbox.path.len() + 1); + name.push_str(prefix); + name.push('/'); + name.push_str(&mailbox.path); + name + } + None => mailbox.path.to_string(), + }; + let mailbox_id = AccountView::default_folder_role(default_folders, &name) + .and_then(|role| { + special_uses + .iter() + .find(|(special_use, _)| *special_use == role) + .map(|(_, mailbox_id)| *mailbox_id) + }) + .unwrap_or(mailbox.document_id); + names.push(MailboxName { + name, + mailbox_id, + is_alias: mailbox_id != mailbox.document_id, + }); + } + names.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + + AccountView { + account_id, + prefix, + mailboxes, + source, + visible, + names, + parents, + counters: None, + previous_counters: None, + } + } + + fn restructure( + &mut self, + default_folders: &[DefaultFolder], + mailboxes: Arc, + access_token: &AccessToken, + changes: &mut Option, + ) { + if self.keeps_structure(&mailboxes, access_token) { + self.mailboxes = mailboxes; + return; + } + let refreshed = AccountView::build( + default_folders, + mailboxes, + self.source, + self.account_id, + self.prefix.take(), + access_token, + ); + let previous = mem::replace(&mut self.names, refreshed.names); + if let Some(changes) = changes { + let account_id = self.account_id; + for entry in &self.names { + match previous + .binary_search_by(|known| known.name.cmp(&entry.name)) + .ok() + .and_then(|pos| previous.get(pos)) + { + Some(known) if known.object_id(account_id) == entry.object_id(account_id) => {} + Some(known) => { + changes.deleted.push(known.to_change(account_id)); + changes.added.push(entry.to_change(account_id)); + } + None => changes.added.push(entry.to_change(account_id)), + } + } + for entry in previous { + if self.entry_by_name(&entry.name).is_none() { + changes.deleted.push(entry.into_change(account_id)); + } + } + } + self.mailboxes = refreshed.mailboxes; + self.visible = refreshed.visible; + self.parents = refreshed.parents; + self.prefix = refreshed.prefix; + } + + fn keeps_structure(&self, mailboxes: &MailboxesCache, access_token: &AccessToken) -> bool { + self.mailboxes.items.len() == mailboxes.items.len() + && self + .mailboxes + .items + .iter() + .zip(mailboxes.items.iter()) + .all(|(current, next)| { + current.document_id == next.document_id + && current.parent_id == next.parent_id + && current.role == next.role + && current.path == next.path + }) + && match &self.visible { + None => access_token.is_member(self.account_id), + Some(visible) => { + !access_token.is_member(self.account_id) + && mailboxes.shared_mailboxes(access_token, Acl::Read) == *visible + } + } + } + + pub fn ensure_counters(&mut self, cache: &MessageStoreCache) { + if let Some(stale) = self + .counters + .take_if(|counters| counters.change_id != cache.emails.change_id) + { + self.previous_counters.get_or_insert(stale.by_mailbox); + } + if self.counters.is_none() { + self.counters = Some(MailboxCounters::compute(cache)); + } + } + + pub fn refresh_counters(&mut self, cache: &MessageStoreCache) -> Option> { + self.ensure_counters(cache); + self.previous_counters.take() + } + + pub fn apply_counter_deltas( + &mut self, + from_change_id: u64, + to_change_id: u64, + deltas: &[CounterDelta], + rebuilt: bool, + ) { + match &mut self.counters { + Some(counters) if !rebuilt && counters.change_id == from_change_id => { + if self.previous_counters.is_none() { + self.previous_counters = Some(counters.by_mailbox.clone()); + } + counters.apply_deltas(deltas, to_change_id); + } + Some(counters) if counters.change_id == to_change_id => {} + Some(counters) => { + if self.previous_counters.is_none() { + self.previous_counters = Some(std::mem::take(&mut counters.by_mailbox)); + } + self.counters = None; + } + None => {} + } + } +} + diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs deleted file mode 100644 index 9eedb917..00000000 --- a/crates/imap/src/core/message.rs +++ /dev/null @@ -1,332 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::{ - ImapUidToId, Mailbox, MailboxId, MailboxState, NextMailboxState, SelectedMailbox, SessionData, -}; -use crate::core::ImapId; -use ahash::AHashMap; -use common::network::SessionStream; -use email::cache::MessageCacheFetch; -use imap_proto::protocol::{Sequence, expunge, select::Exists}; -use std::collections::BTreeMap; -use store::{ValueKey, roaring::RoaringBitmap, write::ValueClass}; -use trc::AddContext; -use types::{collection::Collection, field::MailboxField}; - -impl SessionData { - pub async fn fetch_messages( - &self, - mailbox: &MailboxId, - current_state: Option, - ) -> trc::Result> { - let cached_messages = self - .server - .get_cached_messages(mailbox.account_id) - .await - .caused_by(trc::location!())?; - - if current_state.is_some_and(|state| state == cached_messages.emails.change_id) { - return Ok(None); - } - - // Obtain UID next and assign UIDs - let uid_map = cached_messages - .emails - .items - .iter() - .filter_map(|item| { - item.mailboxes.iter().find_map(|m| { - if m.mailbox_id == mailbox.mailbox_id { - Some((m.uid, item.document_id)) - } else { - None - } - }) - }) - .collect::>(); - let mut uid_max = 0; - let mut id_to_imap = AHashMap::with_capacity(uid_map.len()); - let mut uid_to_id = AHashMap::with_capacity(uid_map.len()); - - for (seqnum, (uid, message_id)) in uid_map.into_iter().enumerate() { - if uid > uid_max { - uid_max = uid; - } - id_to_imap.insert( - message_id, - ImapId { - uid, - seqnum: seqnum as u32 + 1, - }, - ); - uid_to_id.insert(uid, message_id); - } - - Ok(Some(MailboxState { - total_messages: id_to_imap.len(), - id_to_imap, - uid_to_id, - uid_max, - modseq: cached_messages.emails.change_id, - next_state: None, - })) - } - - pub async fn synchronize_messages(&self, mailbox: &SelectedMailbox) -> trc::Result { - // Obtain current modseq - let mut current_modseq = mailbox.state.lock().modseq; - if let Some(new_state) = self - .fetch_messages(&mailbox.id, current_modseq.into()) - .await? - { - // Synchronize messages - let mut current_state = mailbox.state.lock(); - current_modseq = new_state.modseq; - - // Add missing uids - let mut deletions = current_state - .next_state - .take() - .map(|state| state.deletions) - .unwrap_or_default(); - let mut id_to_imap = AHashMap::with_capacity(current_state.id_to_imap.len()); - for (id, imap_id) in std::mem::take(&mut current_state.id_to_imap) { - if !new_state.uid_to_id.contains_key(&imap_id.uid) { - // Add to deletions - deletions.push(imap_id); - - // Invalidate entries - current_state.uid_to_id.remove(&imap_id.uid); - } else { - id_to_imap.insert(id, imap_id); - } - } - current_state.id_to_imap = id_to_imap; - - // Update state - current_state.modseq = new_state.modseq; - current_state.next_state = Some(Box::new(NextMailboxState { - next_state: new_state, - deletions, - })); - } - - Ok(current_modseq) - } - - pub async fn write_mailbox_changes( - &self, - mailbox: &SelectedMailbox, - use_vanished: bool, - ) -> trc::Result { - // Resync mailbox - let modseq = self.synchronize_messages(mailbox).await?; - let mut buf = Vec::new(); - { - let mut current_state = mailbox.state.lock(); - if let Some(next_state) = current_state.next_state.take() { - if !next_state.deletions.is_empty() { - let mut ids = next_state - .deletions - .into_iter() - .map(|id| if use_vanished { id.uid } else { id.seqnum }) - .collect::>(); - ids.sort_unstable(); - expunge::Response { use_vanished, ids }.serialize_to(&mut buf); - } - if !buf.is_empty() - || next_state - .next_state - .uid_max - .saturating_sub(current_state.uid_max) - > 0 - { - Exists { - total_messages: next_state.next_state.total_messages, - } - .serialize(&mut buf); - } - *current_state = next_state.next_state; - } - } - if !buf.is_empty() { - self.write_bytes(buf).await?; - } - - Ok(modseq) - } - - pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { - self.server - .core - .storage - .data - .get_counter(ValueKey { - account_id: mailbox.account_id, - collection: Collection::Mailbox.into(), - document_id: mailbox.mailbox_id, - class: ValueClass::Property(MailboxField::UidCounter.into()), - }) - .await - .map(|v| (v + 1) as u32) - } - - pub fn mailbox_state(&self, mailbox: &MailboxId) -> Option { - self.mailboxes - .lock() - .iter() - .find(|m| m.account_id == mailbox.account_id) - .and_then(|m| m.mailbox_state.get(&mailbox.mailbox_id)) - .cloned() - } -} - -impl SelectedMailbox { - pub async fn sequence_to_ids( - &self, - sequence: &Sequence, - is_uid: bool, - ) -> trc::Result> { - if !sequence.is_saved_search() { - let mut ids = AHashMap::new(); - let state = self.state.lock(); - let (id_to_imap, uid_max, total_messages) = - if let Some(next) = state.next_state.as_ref() { - ( - &next.next_state.id_to_imap, - next.next_state.uid_max, - next.next_state.total_messages, - ) - } else { - (&state.id_to_imap, state.uid_max, state.total_messages) - }; - - if is_uid { - if !id_to_imap.is_empty() { - for (id, imap_id) in id_to_imap { - if sequence.contains(imap_id.uid, uid_max) { - ids.insert(*id, *imap_id); - } - } - } - } else if !id_to_imap.is_empty() { - for (id, imap_id) in id_to_imap { - if sequence.contains(imap_id.seqnum, total_messages as u32) { - ids.insert(*id, *imap_id); - } - } - } - - Ok(ids) - } else { - let saved_ids = self.get_saved_search().await.ok_or_else(|| { - trc::ImapEvent::Error - .into_err() - .details("No saved search found.") - })?; - let mut ids = AHashMap::with_capacity(saved_ids.len()); - let state = self.state.lock(); - - for imap_id in saved_ids.iter() { - if let Some(id) = state.uid_to_id.get(&imap_id.uid) { - ids.insert(*id, *imap_id); - } - } - - Ok(ids) - } - } - - pub fn uids_in_range(&self, min: Option, max: Option) -> RoaringBitmap { - let state = self.state.lock(); - let id_to_imap = state - .next_state - .as_ref() - .map_or(&state.id_to_imap, |next| &next.next_state.id_to_imap); - - RoaringBitmap::from_iter(id_to_imap.iter().filter_map(|(id, imap_id)| { - (min.is_none_or(|min| imap_id.uid >= min) && max.is_none_or(|max| imap_id.uid <= max)) - .then_some(*id) - })) - } - - pub fn seqnum_to_uid(&self, seqnum: u32) -> Option { - let state = self.state.lock(); - state - .next_state - .as_ref() - .map_or(&state.id_to_imap, |next| &next.next_state.id_to_imap) - .values() - .find(|imap_id| imap_id.seqnum == seqnum) - .map(|imap_id| imap_id.uid) - } - - pub fn uids_descending(&self) -> Vec { - let mut uids = { - let state = self.state.lock(); - state - .next_state - .as_ref() - .map_or(&state.uid_to_id, |next| &next.next_state.uid_to_id) - .keys() - .copied() - .collect::>() - }; - uids.sort_unstable_by(|a, b| b.cmp(a)); - uids - } - - pub async fn sequence_expand_missing(&self, sequence: &Sequence, is_uid: bool) -> Vec { - let mut deleted_ids = Vec::new(); - if !sequence.is_saved_search() { - let state = self.state.lock(); - if is_uid { - for uid in sequence.expand(state.uid_max) { - if !state.uid_to_id.contains_key(&uid) { - deleted_ids.push(uid); - } - } - } else { - for seqnum in sequence.expand(state.total_messages as u32) { - if seqnum > state.total_messages as u32 { - deleted_ids.push(seqnum); - } - } - } - } else if let Some(saved_ids) = self.get_saved_search().await { - let state = self.state.lock(); - for id in saved_ids.iter() { - if !state.uid_to_id.contains_key(&id.uid) { - deleted_ids.push(if is_uid { id.uid } else { id.seqnum }); - } - } - } - deleted_ids.sort_unstable(); - deleted_ids - } - - pub fn append_messages(&self, ids: Vec, modseq: Option) { - let mut mailbox = self.state.lock(); - if modseq.unwrap_or(0) > mailbox.modseq { - let mut uid_max = 0; - for id in ids { - mailbox.total_messages += 1; - let seqnum = mailbox.total_messages as u32; - mailbox.uid_to_id.insert(id.uid, id.uid); - mailbox.id_to_imap.insert( - id.id, - ImapId { - uid: id.uid, - seqnum, - }, - ); - uid_max = id.uid; - } - mailbox.uid_max = uid_max; - } - } -} diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index 174bb450..c7bdef0f 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -4,32 +4,38 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashMap; use common::{ - Inner, Server, + Inner, MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken, + cache::email::MessageRef, + config::mailstore::email::DefaultFolder, network::{ServerInstance, SessionStream, limiter::InFlight}, }; +use email::cache::MessageCacheFetch; use imap_proto::{ Command, - protocol::{ProtocolVersion, list::Attribute}, + protocol::{MailboxObjectId, ObjectId, ProtocolVersion, list::Attribute}, receiver::Receiver, }; use std::{ - collections::BTreeMap, net::IpAddr, sync::{Arc, atomic::AtomicU32}, }; +use store::roaring::RoaringBitmap; use tokio::{ io::{ReadHalf, WriteHalf}, - sync::watch, + sync::{Semaphore, watch}, }; use trc::AddContext; +use types::{keyword::Keyword, special_use::SpecialUse}; pub mod client; pub mod mailbox; -pub mod message; +pub mod rights; pub mod session; +pub mod view; + +pub use view::{ItemChanges, MailboxView, Resolved, Row}; #[derive(Clone)] pub struct ImapSessionManager { @@ -42,8 +48,11 @@ impl ImapSessionManager { } } +pub const MAX_CONCURRENT_OPS: u32 = 5; + pub struct Session { pub server: Server, + pub op_semaphore: Arc, pub instance: Arc, pub receiver: Receiver, pub version: ProtocolVersion, @@ -66,7 +75,7 @@ pub struct SessionData { pub access_token: AccessToken, pub server: Server, pub session_id: u64, - pub mailboxes: parking_lot::Mutex>, + pub mailboxes: parking_lot::Mutex>, pub stream_tx: Arc>>, pub state: AtomicU32, pub remote_addr: IpAddr, @@ -75,7 +84,7 @@ pub struct SessionData { pub struct SelectedMailbox { pub id: MailboxId, - pub state: parking_lot::Mutex, + pub view: parking_lot::Mutex, pub saved_search: parking_lot::Mutex, pub is_select: bool, pub is_condstore: bool, @@ -87,77 +96,148 @@ pub struct MailboxId { pub mailbox_id: u32, } -#[derive(Debug, Clone, Default)] -pub struct Account { +pub struct AccountView { pub account_id: u32, pub prefix: Option, - pub mailbox_names: BTreeMap, - pub mailbox_state: AHashMap, - pub last_change_id: u64, + pub mailboxes: Arc, + pub source: MailboxSource, + pub visible: Option, + pub names: Vec, + pub parents: RoaringBitmap, + pub counters: Option, + pub previous_counters: Option>, } -#[derive(Debug, Default, Clone)] -pub struct Mailbox { - pub has_children: bool, - pub is_subscribed: bool, - pub special_use: Option, - pub total_messages: u64, - pub total_unseen: u64, - pub total_deleted: u64, - pub total_deleted_storage: Option, - pub uid_validity: u64, - pub uid_next: u64, - pub size: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct MailboxState { - pub uid_max: u32, - pub id_to_imap: AHashMap, - pub uid_to_id: AHashMap, - pub total_messages: usize, - pub modseq: u64, - pub next_state: Option>, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MailboxSource { + Messages, + Folders { change_id: u64, verified_at: u64 }, } #[derive(Debug, Clone)] -pub struct NextMailboxState { - pub next_state: MailboxState, - pub deletions: Vec, +pub struct MailboxName { + pub name: String, + pub mailbox_id: u32, + pub is_alias: bool, } -#[derive(Debug, Clone, Copy, Default)] -pub struct ImapId { - pub uid: u32, - pub seqnum: u32, +const _: () = assert!(size_of::() <= 32); + +#[derive(Debug)] +pub struct MailboxChange { + pub account_id: u32, + pub name: String, + pub objectid: MailboxObjectId, +} + +impl MailboxName { + pub fn object_id(&self, account_id: u32) -> MailboxObjectId { + if self.is_alias { + MailboxObjectId::Empty + } else { + MailboxObjectId::Mailbox { + account_id, + mailbox_id: self.mailbox_id, + } + } + } + + pub fn to_change(&self, account_id: u32) -> MailboxChange { + MailboxChange { + account_id, + name: self.name.clone(), + objectid: self.object_id(account_id), + } + } + + pub fn into_change(self, account_id: u32) -> MailboxChange { + MailboxChange { + account_id, + objectid: self.object_id(account_id), + name: self.name, + } + } +} + +impl From for MailboxObjectId { + fn from(mailbox: MailboxId) -> Self { + MailboxObjectId::Mailbox { + account_id: mailbox.account_id, + mailbox_id: mailbox.mailbox_id, + } + } +} + +impl From for ObjectId { + fn from(mailbox: MailboxId) -> Self { + MailboxObjectId::from(mailbox).into() + } +} + +#[derive(Debug, Clone)] +pub struct MailboxCounters { + pub change_id: u64, + pub by_mailbox: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct CounterDelta { + pub mailbox_id: u32, + pub size: u32, + pub is_unseen: bool, + pub is_deleted: bool, + pub add: bool, +} + +#[derive(Clone, Copy)] +pub struct KeywordBits { + seen: u32, + deleted: u32, +} + +#[derive(Default)] +pub struct AccountCaches(Vec<(u32, Arc)>); + +pub struct MailboxRefresh { + pub changes: Option, + pub caches: AccountCaches, + pub access_token: AccessToken, +} + +pub enum SelectedAccess { + Readable { + access_token: AccessToken, + cache: Arc, + }, + Revoked, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Counters { + pub total: u32, + pub unseen: u32, + pub deleted: u32, + pub size: u64, + pub deleted_size: u64, } #[derive(Debug, Default)] pub struct MailboxSync { - pub added: Vec, + pub added: Vec, pub changed: Vec, - pub deleted: Vec, + pub deleted: Vec, } pub enum SavedSearch { - InFlight { - rx: watch::Receiver>>, - }, - Results { - items: Arc>, - }, + InFlight { rx: watch::Receiver>> }, + Results { items: Arc> }, None, } -#[derive(Debug, Clone, Copy, Default)] -pub struct ImapUidToId { - pub uid: u32, - pub id: u32, -} - pub enum State { NotAuthenticated { auth_failures: u32, + sasl_failure: Option, }, Authenticated { data: Arc>, @@ -174,9 +254,11 @@ impl State { new_stream: Arc>>, ) -> Option> { match self { - State::NotAuthenticated { auth_failures } => { - State::NotAuthenticated { auth_failures }.into() + State::NotAuthenticated { auth_failures, .. } => State::NotAuthenticated { + auth_failures, + sasl_failure: None, } + .into(), State::Authenticated { data } => { Arc::try_unwrap(data).ok().map(|data| State::Authenticated { data: Arc::new(data.replace_stream_tx(new_stream)), @@ -195,11 +277,8 @@ impl State { impl SessionData { pub async fn refresh_access_token(&self) -> trc::Result { self.server - .access_token(self.account_id) + .renew_access_token(&self.access_token, self.remote_addr) .await - .and_then(|inner| { - AccessToken::renew(inner, self.access_token.credential_id(), self.remote_addr) - }) .caused_by(trc::location!()) } @@ -221,19 +300,255 @@ impl SessionData { } } -impl MailboxState { - pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { - if let Some(imap_id) = self.id_to_imap.get(&document_id) { - Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) - } else if is_uid { - self.next_state.as_ref().and_then(|s| { - s.next_state - .id_to_imap - .get(&document_id) - .map(|imap_id| (imap_id.uid, *imap_id)) - }) - } else { - None +impl AccountView { + pub fn mailbox(&self, mailbox_id: u32) -> Option<&MailboxCache> { + self.mailboxes + .index + .get(&mailbox_id) + .and_then(|idx| self.mailboxes.items.get(*idx as usize)) + } + + pub fn is_visible(&self, mailbox_id: u32) -> bool { + self.visible + .as_ref() + .is_none_or(|visible| visible.contains(mailbox_id)) + } + + pub fn has_children(&self, mailbox_id: u32) -> bool { + self.parents.contains(mailbox_id) + } + + pub fn entry_by_name(&self, name: &str) -> Option<&MailboxName> { + self.names + .binary_search_by(|entry| entry.name.as_str().cmp(name)) + .ok() + .and_then(|idx| self.names.get(idx)) + } + + pub fn id_by_name(&self, name: &str) -> Option { + self.entry_by_name(name).map(|entry| entry.mailbox_id) + } + + pub fn alias_target(&self, default_folders: &[DefaultFolder], name: &str) -> Option { + let role = AccountView::default_folder_role(default_folders, name) + .filter(|role| !matches!(role, SpecialUse::None))?; + self.mailboxes + .items + .iter() + .find(|mailbox| mailbox.role == role && self.is_visible(mailbox.document_id)) + .map(|mailbox| mailbox.document_id) + } + + pub fn default_folder_role( + default_folders: &[DefaultFolder], + name: &str, + ) -> Option { + default_folders + .iter() + .find(|folder| folder.name == name || folder.aliases.iter().any(|alias| alias == name)) + .map(|folder| folder.special_use) + } + + pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { + self.names + .iter() + .any(|entry| entry.mailbox_id == mailbox_id) + } + + pub fn holds_name(&self, mailbox_name: &str) -> bool { + self.prefix + .as_deref() + .and_then(|prefix| mailbox_name.strip_prefix(prefix)) + .and_then(|rest| rest.strip_prefix('/')) + .is_some_and(|folder| !folder.is_empty()) + } + + pub fn expire_folders(&mut self) { + if let MailboxSource::Folders { verified_at, .. } = &mut self.source { + *verified_at = 0; + } + } + + pub fn is_subscribed(&self, mailbox_id: u32, subscriber: u32) -> bool { + self.mailbox(mailbox_id) + .is_some_and(|mailbox| mailbox.subscribers.contains(&subscriber)) + } + + pub fn special_use(&self, mailbox_id: u32) -> Option { + self.mailbox(mailbox_id) + .and_then(|mailbox| Self::special_use_attribute(&mailbox.role)) + } + + pub fn uid_validity(&self, mailbox_id: u32) -> u32 { + self.mailbox(mailbox_id) + .map_or(0, |mailbox| mailbox.uid_validity) + } + + pub fn special_use_attribute(role: &SpecialUse) -> Option { + match role { + SpecialUse::Trash => Some(Attribute::Trash), + SpecialUse::Junk => Some(Attribute::Junk), + SpecialUse::Drafts => Some(Attribute::Drafts), + SpecialUse::Archive => Some(Attribute::Archive), + SpecialUse::Sent => Some(Attribute::Sent), + SpecialUse::Important => Some(Attribute::Important), + SpecialUse::Memos => Some(Attribute::Memos), + SpecialUse::Scheduled => Some(Attribute::Scheduled), + SpecialUse::Snoozed => Some(Attribute::Snoozed), + _ => None, + } + } + + pub fn counters_of(&self, mailbox_id: u32) -> Counters { + self.counters + .as_ref() + .map_or_else(Counters::default, |counters| counters.get(mailbox_id)) + } +} + +impl MailboxCounters { + pub fn compute(cache: &MessageStoreCache) -> Self { + let slots = cache + .mailboxes + .items + .iter() + .map(|mailbox| mailbox.document_id as usize + 1) + .max() + .unwrap_or(0); + let mut counters = MailboxCounters { + change_id: cache.emails.change_id, + by_mailbox: vec![Counters::default(); slots], + }; + let bits = KeywordBits::new(); + for message in cache.emails.iter() { + counters.adjust(message, bits, true); + } + counters + } + + pub fn apply_deltas(&mut self, deltas: &[CounterDelta], change_id: u64) { + for delta in deltas { + let slot = delta.mailbox_id as usize; + if slot >= self.by_mailbox.len() { + if delta.add { + self.by_mailbox.resize(slot + 1, Counters::default()); + } else { + continue; + } + } + if let Some(counters) = self.by_mailbox.get_mut(slot) { + counters.apply( + delta.add, + delta.is_unseen, + delta.is_deleted, + delta.size as u64, + ); + } + } + self.change_id = change_id; + } + + fn adjust(&mut self, message: MessageRef<'_>, bits: KeywordBits, add: bool) { + let keywords = message.keywords(); + let is_unseen = keywords & bits.seen == 0; + let is_deleted = keywords & bits.deleted != 0; + let size = message.size() as u64; + + for membership in message.mailboxes() { + if let Some(counters) = self.by_mailbox.get_mut(membership.mailbox_id as usize) { + counters.apply(add, is_unseen, is_deleted, size); + } + } + } + + pub fn get(&self, mailbox_id: u32) -> Counters { + self.by_mailbox + .get(mailbox_id as usize) + .copied() + .unwrap_or_default() + } +} + +impl KeywordBits { + pub fn new() -> Self { + KeywordBits { + seen: Keyword::Seen.id().map_or(0, |id| 1u32 << id), + deleted: Keyword::Deleted.id().map_or(0, |id| 1u32 << id), } } } + +impl Default for KeywordBits { + fn default() -> Self { + Self::new() + } +} + +impl CounterDelta { + pub fn push_for( + message: MessageRef<'_>, + bits: KeywordBits, + add: bool, + out: &mut Vec, + ) { + let keywords = message.keywords(); + let is_unseen = keywords & bits.seen == 0; + let is_deleted = keywords & bits.deleted != 0; + let size = message.size(); + out.extend(message.mailboxes().iter().map(|membership| CounterDelta { + mailbox_id: membership.mailbox_id, + size, + is_unseen, + is_deleted, + add, + })); + } +} + +impl Counters { + fn apply(&mut self, add: bool, is_unseen: bool, is_deleted: bool, size: u64) { + if add { + self.total += 1; + self.size += size; + if is_unseen { + self.unseen += 1; + } + if is_deleted { + self.deleted += 1; + self.deleted_size += size; + } + } else { + self.total = self.total.saturating_sub(1); + self.size = self.size.saturating_sub(size); + if is_unseen { + self.unseen = self.unseen.saturating_sub(1); + } + if is_deleted { + self.deleted = self.deleted.saturating_sub(1); + self.deleted_size = self.deleted_size.saturating_sub(size); + } + } + } +} + +impl AccountCaches { + pub fn with(account_id: u32, cache: Arc) -> Self { + AccountCaches(vec![(account_id, cache)]) + } + + pub async fn fetch( + &mut self, + server: &Server, + account_id: u32, + ) -> trc::Result> { + if let Some((_, cache)) = self.0.iter().find(|(id, _)| *id == account_id) { + return Ok(cache.clone()); + } + let cache = server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + self.0.push((account_id, cache.clone())); + Ok(cache) + } +} diff --git a/crates/imap/src/core/rights.rs b/crates/imap/src/core/rights.rs new file mode 100644 index 00000000..12457a8c --- /dev/null +++ b/crates/imap/src/core/rights.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{MessageStoreCache, auth::AccessToken, sharing::EffectiveAcl}; +use email::cache::mailbox::MailboxCacheAccess; +use imap_proto::protocol::select::PermanentFlags; +use types::acl::Acl; +use utils::map::bitmap::Bitmap; + +pub enum MailboxRights<'x> { + Member, + Shared { + access_token: &'x AccessToken, + acls: Vec<(u32, Bitmap)>, + }, +} + +impl<'x> MailboxRights<'x> { + pub fn new(access_token: &'x AccessToken, account_id: u32) -> Self { + if access_token.is_member(account_id) { + MailboxRights::Member + } else { + MailboxRights::Shared { + access_token, + acls: Vec::new(), + } + } + } + + pub fn acl(&mut self, cache: &MessageStoreCache, mailbox_id: u32) -> Bitmap { + let MailboxRights::Shared { access_token, acls } = self else { + return Bitmap::all(); + }; + match acls.binary_search_by_key(&mailbox_id, |(id, _)| *id) { + Ok(index) => acls[index].1, + Err(index) => { + let acl = cache + .mailbox_by_id(&mailbox_id) + .map_or_else(Bitmap::new, |mailbox| { + mailbox.acls.as_slice().effective_acl(access_token) + }); + acls.insert(index, (mailbox_id, acl)); + acl + } + } + } + + pub fn permanent_flags( + &mut self, + cache: &MessageStoreCache, + mailbox_ids: impl IntoIterator, + ) -> PermanentFlags { + if matches!(self, MailboxRights::Member) { + return PermanentFlags::All; + } + let mut acl = Bitmap::all(); + for mailbox_id in mailbox_ids { + acl.intersection(&self.acl(cache, mailbox_id)); + } + PermanentFlags::from(acl) + } +} diff --git a/crates/imap/src/core/session.rs b/crates/imap/src/core/session.rs index 3ba49489..6ab22aa6 100644 --- a/crates/imap/src/core/session.rs +++ b/crates/imap/src/core/session.rs @@ -4,21 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ImapSessionManager, Session, State}; -use crate::{ - GREETING_WITH_TLS, GREETING_WITH_TLS_LOGIN_DISABLED, GREETING_WITHOUT_TLS, - GREETING_WITHOUT_TLS_LOGIN_DISABLED, -}; +use super::{ImapSessionManager, MAX_CONCURRENT_OPS, Session, State}; +use crate::{auth_capability, greeting}; use common::{ BuildServer, network::{SessionData, SessionManager, SessionResult, SessionStream, stream::NullIo}, }; +use compact_str::ToCompactString; use imap_proto::{ + ResponseCode, StatusResponse, protocol::{ProtocolVersion, SerializeResponse}, - receiver::Receiver, + receiver::{MAX_UNAUTHENTICATED_REQUEST_SIZE, Receiver}, }; use std::sync::Arc; +use store::write::is_commit_contention; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Semaphore; use tokio_rustls::server::TlsStream; impl SessionManager for ImapSessionManager { @@ -46,7 +47,7 @@ impl SessionManager for ImapSessionManager { impl Session { pub async fn handle_conn(&mut self) -> bool { - let mut buf = vec![0; 8192]; + let mut buf = vec![0; 65536]; let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { @@ -83,7 +84,7 @@ impl Session { trc::event!( Network(trc::NetworkEvent::ReadError), SpanId = self.session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); break; @@ -123,18 +124,12 @@ impl Session { let is_tls = session.stream.is_tls(); let server = manager.inner.build_server(); let offer_tls = !is_tls && session.instance.acceptor.is_tls(); - let allow_auth = is_tls || server.core.imap.allow_plain_auth; - let greeting = match (offer_tls, allow_auth) { - (true, true) => &GREETING_WITH_TLS, - (true, false) => &GREETING_WITH_TLS_LOGIN_DISABLED, - (false, true) => &GREETING_WITHOUT_TLS, - (false, false) => &GREETING_WITHOUT_TLS_LOGIN_DISABLED, - }; + let greeting = greeting(offer_tls, auth_capability(&server, is_tls)); if let Err(err) = session.stream.write_all(greeting).await { trc::event!( Network(trc::NetworkEvent::WriteError), - Reason = err.to_string(), + Reason = err.to_compact_string(), SpanId = session.session_id, Details = "Failed to write to stream" ); @@ -146,9 +141,13 @@ impl Session { let (stream_rx, stream_tx) = tokio::io::split(session.stream); Ok(Session { - receiver: Receiver::with_max_request_size(server.core.imap.max_request_size), + receiver: Receiver::with_max_request_size(MAX_UNAUTHENTICATED_REQUEST_SIZE), + op_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_OPS as usize)), version: ProtocolVersion::Rev1, - state: State::NotAuthenticated { auth_failures: 0 }, + state: State::NotAuthenticated { + auth_failures: 0, + sasl_failure: None, + }, is_tls, is_condstore: false, is_qresync: false, @@ -201,10 +200,11 @@ impl Session { let (stream_rx, stream_tx) = tokio::io::split(self.instance.tls_accept(stream, self.session_id).await?); let stream_tx = Arc::new(tokio::sync::Mutex::new(stream_tx)); - let receiver = Receiver::with_max_request_size(self.server.core.imap.max_request_size); + let receiver = Receiver::with_max_request_size(self.receiver.max_request_size); Ok(Session { server: self.server, + op_semaphore: self.op_semaphore, instance: self.instance, receiver, version: self.version, @@ -259,7 +259,7 @@ impl Session { pub async fn write_error(&self, err: trc::Error) -> bool { if err.should_write_err() { let disconnect = err.must_disconnect(); - let bytes = err.serialize(); + let bytes = error_response(&err); trc::error!(err.span_id(self.session_id)); if let Err(err) = self.write_bytes(bytes).await { @@ -276,7 +276,41 @@ impl Session { } } +fn error_response(err: &trc::Error) -> Vec { + if is_commit_contention(err) { + let response = + StatusResponse::no("Temporary failure, try again.").with_code(ResponseCode::InUse); + match err + .keys() + .iter() + .rev() + .find_map(|(key, value)| (*key == trc::Key::Id).then_some(value)) + .and_then(|value| value.as_str()) + { + Some(tag) => response.with_tag(tag), + None => response, + } + .into_bytes() + } else { + err.serialize() + } +} + +pub const OUTPUT_FLUSH_THRESHOLD: usize = 64 * 1024; +const OUTPUT_RETAIN_CAPACITY: usize = 4 * OUTPUT_FLUSH_THRESHOLD; + impl super::SessionData { + pub async fn flush_output(&self, output: &mut Vec, force: bool) -> trc::Result<()> { + if !output.is_empty() && (force || output.len() >= OUTPUT_FLUSH_THRESHOLD) { + self.write_bytes(&output).await?; + output.clear(); + if output.capacity() > OUTPUT_RETAIN_CAPACITY { + output.shrink_to(OUTPUT_FLUSH_THRESHOLD); + } + } + Ok(()) + } + pub async fn write_bytes(&self, bytes: impl AsRef<[u8]>) -> trc::Result<()> { let bytes = bytes.as_ref(); @@ -301,7 +335,7 @@ impl super::SessionData { pub async fn write_error(&self, err: trc::Error) -> trc::Result<()> { if err.should_write_err() { - let bytes = err.serialize(); + let bytes = error_response(&err); trc::error!(err.span_id(self.session_id)); self.write_bytes(bytes).await } else { @@ -310,3 +344,4 @@ impl super::SessionData { } } } + diff --git a/crates/imap/src/core/view.rs b/crates/imap/src/core/view.rs new file mode 100644 index 00000000..2b835e01 --- /dev/null +++ b/crates/imap/src/core/view.rs @@ -0,0 +1,885 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CounterDelta, KeywordBits, SelectedMailbox, SessionData}; +use common::{MessageStoreCache, MessagesCache, NO_ID, network::SessionStream}; + +pub use common::MailboxRow as Row; +use imap_proto::protocol::{Sequence, expunge, select::Exists}; +use std::{borrow::Cow, sync::Arc}; +use store::{query::log::Query, roaring::RoaringBitmap}; +use trc::AddContext; +use types::collection::SyncCollection; + +const CHANGED_SCAN_RATIO: usize = 5; +const CHANGED_FALLBACK_RATIO: usize = 8; +const MAX_SCAN_SEQUENCE_ITEMS: usize = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Resolved { + pub seqnum: u32, + pub uid: u32, + pub id: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemChanges<'x> { + Rebuild, + Items(&'x [u32]), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncOutcome { + Unchanged, + Incremental, + Rebuilt, +} + +impl SyncOutcome { + pub fn is_changed(self) -> bool { + !matches!(self, SyncOutcome::Unchanged) + } +} + +pub struct MailboxView { + mailbox_id: u32, + rows: Arc>, + tombstones: Vec, + appended: u32, + uid_max: u32, + snapshot: Arc, + sorted: bool, + mask: Option, +} + +impl Resolved { + #[inline(always)] + pub fn imap_id(&self, is_uid: bool) -> u32 { + if is_uid { self.uid } else { self.seqnum } + } + + #[inline(always)] + pub fn row(&self) -> Row { + Row { + uid: self.uid, + id: self.id, + } + } +} + +impl MailboxView { + pub fn build(cache: &MessageStoreCache, mailbox_id: u32) -> Self { + let rows = cache + .refresh + .mailbox_rows + .get_or_build(&cache.emails, mailbox_id, || { + let mut rows = Self::collect_rows(cache, mailbox_id); + rows.sort_unstable_by_key(|row| row.uid); + rows + }); + + MailboxView { + mailbox_id, + uid_max: rows.last().map_or(0, |row| row.uid), + rows, + tombstones: Vec::new(), + appended: 0, + snapshot: cache.emails.clone(), + sorted: true, + mask: None, + } + } + + fn collect_rows(cache: &MessageStoreCache, mailbox_id: u32) -> Vec { + cache + .emails + .iter() + .filter_map(|message| { + message.uid_in(mailbox_id).map(|uid| Row { + uid, + id: message.document_id(), + }) + }) + .collect() + } + + #[inline(always)] + pub fn change_id(&self) -> u64 { + self.snapshot.change_id + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.rows.len() + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + #[inline(always)] + pub fn rows(&self) -> &[Row] { + &self.rows + } + + #[inline(always)] + pub fn uid_max(&self) -> u32 { + self.uid_max + } + + pub fn has_pending_changes(&self) -> bool { + !self.tombstones.is_empty() || self.appended > 0 || !self.sorted + } + + pub fn live_rows_by_uid(&self) -> Vec { + let mut rows = self + .rows + .iter() + .copied() + .filter(Row::is_live) + .collect::>(); + if !self.sorted { + rows.sort_unstable_by_key(|row| row.uid); + } + rows + } + + fn position_of_uid(&self, uid: u32) -> Option { + if self.sorted { + self.rows.binary_search_by_key(&uid, |row| row.uid).ok() + } else { + self.rows.iter().position(|row| row.uid == uid) + } + } + + fn for_uid_range(&self, lo: u32, hi: u32, mut f: impl FnMut(usize, &Row)) { + if lo > hi { + return; + } + if self.sorted { + let from = self.rows.partition_point(|row| row.uid < lo); + let to = self.rows.partition_point(|row| row.uid <= hi); + if let Some(slice) = self.rows.get(from..to) { + for (offset, row) in slice.iter().enumerate() { + f(from + offset, row); + } + } + } else { + for (position, row) in self.rows.iter().enumerate() { + if (lo..=hi).contains(&row.uid) { + f(position, row); + } + } + } + } + + #[inline(always)] + fn resolved_at(position: usize, row: &Row) -> Option { + row.is_live().then_some(Resolved { + seqnum: position as u32 + 1, + uid: row.uid, + id: row.id, + }) + } + + pub fn uid_to_seqnum(&self, uid: u32) -> Option { + self.position_of_uid(uid) + .map(|position| position as u32 + 1) + } + + pub fn seqnum_to_uid(&self, seqnum: u32) -> Option { + seqnum + .checked_sub(1) + .and_then(|position| self.rows.get(position as usize)) + .map(|row| row.uid) + } + + pub fn resolve( + &self, + sequence: &Sequence, + is_uid: bool, + saved: Option<&[Row]>, + ) -> Vec { + let mut out = Vec::new(); + self.resolve_into(sequence, is_uid, saved, &mut out); + if matches!(sequence, Sequence::List { .. }) { + out.sort_unstable_by_key(|resolved| resolved.seqnum); + out.dedup_by_key(|resolved| resolved.seqnum); + } + out + } + + pub fn resolve_uid(&self, uid: u32) -> Option { + self.position_of_uid(uid) + .and_then(|position| self.rows.get(position).map(|row| (position, row))) + .and_then(|(position, row)| Self::resolved_at(position, row)) + } + + pub fn changed_since_scan( + &self, + emails: &MessagesCache, + changed_since: u64, + sequence: &Sequence, + is_uid: bool, + ) -> Option> { + let mut items = MAX_SCAN_SEQUENCE_ITEMS; + if !self.sorted || !Self::is_scannable(sequence, &mut items) { + return None; + } + let covered = self.covered_by(sequence, is_uid); + if covered.saturating_mul(CHANGED_SCAN_RATIO) < emails.len() { + return None; + } + self.resolve_changed( + emails, + changed_since, + sequence, + is_uid, + covered / CHANGED_FALLBACK_RATIO, + ) + } + + fn is_scannable(sequence: &Sequence, items: &mut usize) -> bool { + match sequence { + Sequence::Number { .. } | Sequence::Range { .. } => { + let available = *items > 0; + *items = items.saturating_sub(1); + available + } + Sequence::List { items: list } => list + .iter() + .all(|sequence| Self::is_scannable(sequence, items)), + Sequence::SavedSearch => false, + } + } + + fn covered_by(&self, sequence: &Sequence, is_uid: bool) -> usize { + match sequence { + Sequence::Number { .. } => 1, + Sequence::Range { start, end } => { + let max = if is_uid { + self.uid_max() + } else { + self.rows.len() as u32 + }; + let (start, end) = (start.unwrap_or(max), end.unwrap_or(max)); + let (lo, hi) = (start.min(end), start.max(end)); + if is_uid { + self.rows + .partition_point(|row| row.uid <= hi) + .saturating_sub(self.rows.partition_point(|row| row.uid < lo)) + } else { + (hi.min(max) as usize + 1).saturating_sub(lo.max(1) as usize) + } + } + Sequence::List { items } => { + items.iter().map(|item| self.covered_by(item, is_uid)).sum() + } + Sequence::SavedSearch => self.rows.len(), + } + } + + fn resolve_changed( + &self, + emails: &MessagesCache, + changed_since: u64, + sequence: &Sequence, + is_uid: bool, + budget: usize, + ) -> Option> { + let mut candidates = Vec::new(); + for message in emails + .iter() + .rev() + .filter(|message| message.change_id() >= changed_since) + { + if let Some(uid) = message.uid_in(self.mailbox_id) { + if candidates.len() == budget { + return None; + } + candidates.push(Resolved { + seqnum: 0, + uid, + id: message.document_id(), + }); + } + } + + let max = if is_uid { + self.uid_max() + } else { + self.rows.len() as u32 + }; + candidates.retain_mut(|candidate| match self.resolve_uid(candidate.uid) { + Some(resolved) + if resolved.id == candidate.id + && sequence.contains(resolved.imap_id(is_uid), max) => + { + *candidate = resolved; + true + } + _ => false, + }); + Some(candidates) + } + + fn resolve_into( + &self, + sequence: &Sequence, + is_uid: bool, + saved: Option<&[Row]>, + out: &mut Vec, + ) { + match sequence { + Sequence::Number { value } => { + let position = if is_uid { + self.position_of_uid(*value) + } else { + value.checked_sub(1).map(|position| position as usize) + }; + if let Some(resolved) = position + .and_then(|position| self.rows.get(position).map(|row| (position, row))) + .and_then(|(position, row)| Self::resolved_at(position, row)) + { + out.push(resolved); + } + } + Sequence::Range { start, end } => { + let max = if is_uid { + self.uid_max() + } else { + self.rows.len() as u32 + }; + let (start, end) = (start.unwrap_or(max), end.unwrap_or(max)); + let (lo, hi) = (start.min(end), start.max(end)); + if is_uid { + self.for_uid_range(lo, hi, |position, row| { + if let Some(resolved) = Self::resolved_at(position, row) { + out.push(resolved); + } + }); + } else { + let from = lo.max(1) as usize - 1; + let to = (hi as usize).min(self.rows.len()); + if let Some(slice) = self.rows.get(from..to) { + out.reserve(slice.len()); + out.extend( + slice + .iter() + .enumerate() + .filter_map(|(offset, row)| Self::resolved_at(from + offset, row)), + ); + } + } + } + Sequence::List { items } => { + for item in items { + self.resolve_into(item, is_uid, saved, out); + } + } + Sequence::SavedSearch => { + if let Some(saved) = saved { + out.reserve(saved.len()); + for row in saved { + if let Some(resolved) = self + .position_of_uid(row.uid) + .and_then(|position| self.rows.get(position).map(|row| (position, row))) + .and_then(|(position, row)| Self::resolved_at(position, row)) + { + out.push(resolved); + } + } + } + } + } + } + + pub fn missing_in( + &self, + sequence: &Sequence, + is_uid: bool, + saved: Option<&[Row]>, + out: &mut Vec<(u32, u32)>, + ) -> bool { + let mut expunged = false; + match sequence { + Sequence::Number { value } => { + let row = if is_uid { + self.position_of_uid(*value) + .and_then(|position| self.rows.get(position)) + } else { + value + .checked_sub(1) + .and_then(|position| self.rows.get(position as usize)) + }; + match row { + Some(row) if row.is_live() => {} + Some(_) => { + expunged = true; + out.push((*value, *value)); + } + None => out.push((*value, *value)), + } + } + Sequence::Range { start, end } => { + let max = if is_uid { + self.uid_max() + } else { + self.rows.len() as u32 + }; + let (start, end) = (start.unwrap_or(max), end.unwrap_or(max)); + let (lo, hi) = (start.min(end), start.max(end)); + if is_uid { + let mut next_expected = lo; + self.for_uid_range(lo, hi, |_, row| { + if self.sorted && row.uid > next_expected { + out.push((next_expected, row.uid - 1)); + } + if !row.is_live() { + expunged = true; + out.push((row.uid, row.uid)); + } + next_expected = row.uid.saturating_add(1); + }); + if self.sorted && next_expected <= hi { + out.push((next_expected, hi)); + } + } else { + let lo = lo.max(1); + let len = self.rows.len() as u32; + for &position in &self.tombstones { + let seqnum = position + 1; + if (lo..=hi).contains(&seqnum) { + expunged = true; + out.push((seqnum, seqnum)); + } + } + if hi > len && lo <= hi { + out.push((lo.max(len + 1), hi)); + } + } + } + Sequence::List { items } => { + for item in items { + expunged |= self.missing_in(item, is_uid, saved, out); + } + } + Sequence::SavedSearch => { + if let Some(saved) = saved { + for row in saved { + match self.position_of_uid(row.uid) { + Some(position) if self.rows.get(position).is_some_and(Row::is_live) => { + } + Some(position) => { + expunged = true; + out.push(if is_uid { + (row.uid, row.uid) + } else { + (position as u32 + 1, position as u32 + 1) + }); + } + None if is_uid => out.push((row.uid, row.uid)), + None => {} + } + } + } + } + } + expunged + } + + pub fn map_result(&self, document_id: u32) -> Option { + self.snapshot + .by_id(document_id) + .and_then(|message| message.uid_in(self.mailbox_id)) + .and_then(|uid| self.position_of_uid(uid)) + .and_then(|position| { + self.rows + .get(position) + .and_then(|row| Self::resolved_at(position, row)) + }) + } + + pub fn map_result_set(&self, set: &RoaringBitmap, mut f: impl FnMut(Resolved)) { + for (position, row) in self.rows.iter().enumerate() { + if row.is_live() && set.contains(row.id) { + f(Resolved { + seqnum: position as u32 + 1, + uid: row.uid, + id: row.id, + }); + } + } + } + + pub fn uids_in_range(&self, min: Option, max: Option) -> RoaringBitmap { + let mut set = RoaringBitmap::new(); + self.for_uid_range(min.unwrap_or(1), max.unwrap_or(u32::MAX), |_, row| { + if row.is_live() { + set.insert(row.id); + } + }); + set + } + + pub fn document_ids(&mut self) -> RoaringBitmap { + self.mask + .get_or_insert_with(|| { + RoaringBitmap::from_iter( + self.rows + .iter() + .filter(|row| row.is_live()) + .map(|row| row.id), + ) + }) + .clone() + } + + pub fn append_local(&mut self, rows: impl IntoIterator) { + let mut rows = rows.into_iter().collect::>(); + rows.sort_unstable_by_key(|row| row.uid); + self.insert_rows(rows); + } + + fn insert_rows(&mut self, rows: Vec) -> bool { + if rows.is_empty() { + return false; + } + if Arc::get_mut(&mut self.rows).is_none() { + let mut owned = Vec::with_capacity(self.rows.len() + rows.len()); + owned.extend_from_slice(&self.rows); + self.rows = Arc::new(owned); + } + + let mut out_of_order = false; + for row in rows { + if row.uid > self.uid_max { + self.push_row(row); + } else if self.position_of_uid(row.uid).is_none() { + self.push_row(row); + self.sorted = false; + out_of_order = true; + } + } + if out_of_order { + trc::event!( + Imap(trc::ImapEvent::Error), + MailboxId = self.mailbox_id, + Details = "A message with a UID below the mailbox maximum was added.", + ); + } + out_of_order + } + + fn push_row(&mut self, row: Row) { + if let Some(mask) = &mut self.mask { + mask.insert(row.id); + } + self.uid_max = self.uid_max.max(row.uid); + Arc::make_mut(&mut self.rows).push(row); + self.appended += 1; + } + + fn tombstone(&mut self, position: usize) { + if self.rows.get(position).is_some_and(Row::is_live) + && let Some(row) = Arc::make_mut(&mut self.rows).get_mut(position) + { + if let Some(mask) = &mut self.mask { + mask.remove(row.id); + } + row.id = NO_ID; + self.tombstones.push(position as u32); + } + } + + pub fn sync( + &mut self, + cache: &MessageStoreCache, + changes: ItemChanges<'_>, + mut changed: Option<&mut Vec>, + mut deltas: Option<&mut Vec>, + ) -> SyncOutcome { + if cache.emails.change_id <= self.snapshot.change_id { + return SyncOutcome::Unchanged; + } + + let outcome = match changes { + ItemChanges::Rebuild => { + self.rebuild(cache); + SyncOutcome::Rebuilt + } + ItemChanges::Items(document_ids) => { + let bits = KeywordBits::new(); + let mut inserted = Vec::new(); + for &document_id in document_ids { + let before_message = self.snapshot.by_id(document_id); + let after_message = cache.emails.by_id(document_id); + if let Some(deltas) = deltas.as_deref_mut() { + if let Some(message) = before_message { + CounterDelta::push_for(message, bits, false, deltas); + } + if let Some(message) = after_message { + CounterDelta::push_for(message, bits, true, deltas); + } + } + let before = before_message.and_then(|message| message.uid_in(self.mailbox_id)); + let after = after_message.and_then(|message| message.uid_in(self.mailbox_id)); + + if let Some(uid) = before + && after != before + && let Some(position) = self.position_of_uid(uid) + { + self.tombstone(position); + } + if let Some(uid) = after { + if after != before { + inserted.push(Row { + uid, + id: document_id, + }); + } + if let Some(changed) = changed.as_deref_mut() { + changed.push(uid); + } + } + } + + if !inserted.is_empty() { + inserted.sort_unstable_by_key(|row| row.uid); + self.insert_rows(inserted); + } + SyncOutcome::Incremental + } + }; + + self.snapshot = cache.emails.clone(); + outcome + } + + fn rebuild(&mut self, cache: &MessageStoreCache) { + let mut fresh = Self::collect_rows(cache, self.mailbox_id); + fresh.sort_unstable_by_key(|row| row.uid); + + let (removed, added) = if self.sorted { + Self::diff_sorted(&self.rows, 0..self.rows.len(), &fresh) + } else { + let mut order = (0..self.rows.len()).collect::>(); + order + .sort_unstable_by_key(|&position| self.rows.get(position).map_or(0, |row| row.uid)); + Self::diff_sorted(&self.rows, order.into_iter(), &fresh) + }; + + for position in removed { + self.tombstone(position); + } + self.insert_rows(added); + } + + fn diff_sorted( + rows: &[Row], + positions: impl Iterator, + fresh: &[Row], + ) -> (Vec, Vec) { + let mut removed = Vec::new(); + let mut added = Vec::new(); + let mut fresh_iter = fresh.iter().peekable(); + for position in positions { + let Some(old) = rows.get(position) else { + continue; + }; + loop { + match fresh_iter.peek() { + Some(new) if new.uid < old.uid => { + added.push(**new); + fresh_iter.next(); + } + Some(new) if new.uid == old.uid => { + fresh_iter.next(); + break; + } + _ => { + removed.push(position); + break; + } + } + } + } + added.extend(fresh_iter.copied()); + (removed, added) + } + + pub fn flush(&mut self, use_vanished: bool, buf: &mut Vec) { + if !self.tombstones.is_empty() { + self.tombstones.sort_unstable(); + self.tombstones.dedup(); + let ids = if use_vanished { + let mut uids = self + .tombstones + .iter() + .filter_map(|&position| self.rows.get(position as usize)) + .map(|row| row.uid) + .collect::>(); + if !self.sorted { + uids.sort_unstable(); + } + uids + } else { + self.tombstones + .iter() + .map(|position| position + 1) + .collect::>() + }; + expunge::Response { use_vanished, ids }.serialize_to(buf); + Arc::make_mut(&mut self.rows).retain(Row::is_live); + self.tombstones.clear(); + self.appended = 0; + Exists { + total_messages: self.rows.len(), + } + .serialize(buf); + } else if self.appended > 0 { + self.appended = 0; + Exists { + total_messages: self.rows.len(), + } + .serialize(buf); + } + } +} + +impl SelectedMailbox { + pub async fn resolve(&self, sequence: &Sequence, is_uid: bool) -> trc::Result> { + let saved = if sequence.is_saved_search() { + Some(self.get_saved_search().await.ok_or_else(|| { + trc::ImapEvent::Error + .into_err() + .details("No saved search found.") + })?) + } else { + None + }; + Ok(self + .view + .lock() + .resolve(sequence, is_uid, saved.as_deref().map(Vec::as_slice))) + } + + pub async fn missing_in(&self, sequence: &Sequence, is_uid: bool) -> (Vec<(u32, u32)>, bool) { + let saved = if sequence.is_saved_search() { + self.get_saved_search().await + } else { + None + }; + let mut out = Vec::new(); + let expunged = self.view.lock().missing_in( + sequence, + is_uid, + saved.as_deref().map(Vec::as_slice), + &mut out, + ); + (out, expunged) + } + + pub fn change_id(&self) -> u64 { + self.view.lock().change_id() + } +} + +impl SessionData { + pub async fn sync_view( + &self, + mailbox: &SelectedMailbox, + cache: &MessageStoreCache, + changed: Option<&mut Vec>, + ) -> trc::Result { + let since = mailbox.change_id(); + if cache.emails.change_id <= since { + return Ok(since); + } + + let document_ids = match cache.refresh.changed_items(since, cache.emails.change_id) { + Some(document_ids) => Some(document_ids), + None => { + let changes = self + .server + .store() + .changes( + mailbox.id.account_id, + SyncCollection::Email.into(), + Query::Since(since), + ) + .await + .caused_by(trc::location!())?; + (!changes.needs_full_rebuild(since)).then(|| { + let mut document_ids = changes + .changes + .iter() + .filter(|change| !change.is_metadata_only()) + .filter_map(|change| change.item_id()) + .map(|item_id| (item_id & u32::MAX as u64) as u32) + .collect::>(); + document_ids.sort_unstable(); + document_ids.dedup(); + Cow::Owned(document_ids) + }) + } + }; + let changes = match &document_ids { + Some(document_ids) => ItemChanges::Items(document_ids), + None => ItemChanges::Rebuild, + }; + + let mut deltas = Vec::new(); + let track_counters = self.has_counters_at(mailbox.id.account_id, since); + let (outcome, change_id) = { + let mut view = mailbox.view.lock(); + let outcome = view.sync( + cache, + changes, + changed, + track_counters.then_some(&mut deltas), + ); + (outcome, view.change_id()) + }; + if outcome.is_changed() && track_counters { + self.apply_counter_deltas( + mailbox.id.account_id, + since, + change_id, + &deltas, + matches!(outcome, SyncOutcome::Rebuilt), + ); + } + Ok(change_id) + } + + pub async fn flush_view( + &self, + mailbox: &SelectedMailbox, + use_vanished: bool, + ) -> trc::Result<()> { + let mut buf = Vec::new(); + mailbox.view.lock().flush(use_vanished, &mut buf); + if !buf.is_empty() { + self.write_bytes(buf).await + } else { + Ok(()) + } + } + + pub async fn write_mailbox_changes( + &self, + mailbox: &SelectedMailbox, + cache: &MessageStoreCache, + use_vanished: bool, + ) -> trc::Result { + let change_id = self.sync_view(mailbox, cache, None).await?; + self.flush_view(mailbox, use_vanished).await?; + Ok(change_id) + } +} + diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index 3a08b978..4188f2d1 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -6,31 +6,46 @@ #![warn(clippy::large_futures)] +use common::Server; +use imap_proto::{ + ResponseCode, StatusResponse, + protocol::capability::{AuthCapability, Capability}, +}; use std::sync::LazyLock; -use imap_proto::{ResponseCode, StatusResponse, protocol::capability::Capability}; - pub mod core; pub mod op; static SERVER_GREETING: &str = "Stalwart IMAP4rev2 at your service."; -pub(crate) static GREETING_WITH_TLS: LazyLock> = - LazyLock::new(|| build_greeting(true, true)); +const AUTH_CAPABILITIES: [AuthCapability; 3] = [ + AuthCapability::Disabled, + AuthCapability::Plain, + AuthCapability::PlainAndBearer, +]; -pub(crate) static GREETING_WITH_TLS_LOGIN_DISABLED: LazyLock> = - LazyLock::new(|| build_greeting(true, false)); +static GREETINGS: LazyLock<[[Vec; 3]; 2]> = LazyLock::new(|| { + [false, true].map(|offer_tls| AUTH_CAPABILITIES.map(|auth| build_greeting(offer_tls, auth))) +}); -pub(crate) static GREETING_WITHOUT_TLS: LazyLock> = - LazyLock::new(|| build_greeting(false, true)); +pub(crate) fn greeting(offer_tls: bool, auth: AuthCapability) -> &'static [u8] { + &GREETINGS[usize::from(offer_tls)][auth as usize] +} -pub(crate) static GREETING_WITHOUT_TLS_LOGIN_DISABLED: LazyLock> = - LazyLock::new(|| build_greeting(false, false)); +pub(crate) fn auth_capability(server: &Server, is_tls: bool) -> AuthCapability { + if !is_tls && !server.core.imap.allow_plain_auth { + AuthCapability::Disabled + } else if server.accepts_bearer_tokens(is_tls) { + AuthCapability::PlainAndBearer + } else { + AuthCapability::Plain + } +} -fn build_greeting(offer_tls: bool, allow_auth: bool) -> Vec { +fn build_greeting(offer_tls: bool, auth: AuthCapability) -> Vec { StatusResponse::ok(SERVER_GREETING) .with_code(ResponseCode::Capability { - capabilities: Capability::all_capabilities(false, offer_tls, allow_auth, 0, 0), + capabilities: Capability::all_capabilities(false, offer_tls, auth, 0, 0), }) .into_bytes() } diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 8dec273c..7373492b 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -10,7 +10,7 @@ use crate::{ spawn_op, }; use common::{ - auth::AccessToken, ipc::CacheInvalidation, network::SessionStream, sharing::EffectiveAcl, + auth::AccessToken, network::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; use compact_str::ToCompactString; @@ -25,8 +25,9 @@ use registry::schema::enums::Permission; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, @@ -35,7 +36,11 @@ use types::{ use utils::map::bitmap::Bitmap; impl Session { - pub async fn handle_get_acl(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_get_acl( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAclGet)?; @@ -44,7 +49,7 @@ impl Session { let is_utf8 = self.version.is_rev2() || self.is_utf8; let data = self.state.session_data(); - spawn_op!(data, { + spawn_op!(permit, data, { let (mailbox_id, mailbox_, _) = data .get_acl_mailbox(&arguments, true) .await @@ -64,22 +69,7 @@ impl Session { .name() .to_string(); - permissions.push(( - account_name, - vec![ - Rights::Read, - Rights::Lookup, - Rights::Insert, - Rights::DeleteMessages, - Rights::Expunge, - Rights::Seen, - Rights::Write, - Rights::CreateMailbox, - Rights::DeleteMailbox, - Rights::Post, - Rights::Administer, - ], - )); + permissions.push((account_name, Rights::from_acl(Bitmap::all()).collect())); } for item in mailbox.inner.acls.iter() { @@ -88,55 +78,19 @@ impl Session { continue; } - let mut rights = Vec::new(); - - for acl in Bitmap::from(&item.grants) { - match acl { - Acl::Read => { - rights.push(Rights::Lookup); - } - Acl::Modify => { - rights.push(Rights::CreateMailbox); - } - Acl::Delete => { - rights.push(Rights::DeleteMailbox); - } - Acl::ReadItems => { - rights.push(Rights::Read); - } - Acl::AddItems => { - rights.push(Rights::Insert); - } - Acl::ModifyItems => { - rights.push(Rights::Write); - rights.push(Rights::Seen); - } - Acl::RemoveItems => { - rights.push(Rights::DeleteMessages); - rights.push(Rights::Expunge); - } - Acl::CreateChild => { - rights.push(Rights::CreateMailbox); - } - Acl::Share => { - rights.push(Rights::Administer); - } - Acl::Submit => { - rights.push(Rights::Post); - } - _ => (), - } - } - - let account_name = data + let Some(account) = data .server - .account(item.account_id.into()) + .try_account(item.account_id.into()) .await .imap_ctx(&arguments.tag, trc::location!())? - .name() - .to_string(); + else { + continue; + }; - permissions.push((account_name, rights)); + permissions.push(( + account.name().to_string(), + Rights::from_acl(Bitmap::from(&item.grants)).collect(), + )); } trc::event!( @@ -154,7 +108,7 @@ impl Session { .with_tag(arguments.tag) .serialize( GetAclResponse { - mailbox_name: arguments.mailbox_name.to_string(), + mailbox_name: arguments.mailbox_name.clone(), permissions, } .into_bytes(is_utf8), @@ -164,16 +118,20 @@ impl Session { }) } - pub async fn handle_my_rights(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_my_rights( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapMyRights)?; let op_start = Instant::now(); let arguments = request.parse_acl(self.is_utf8)?; - let data = self.state.session_data(); let is_utf8 = self.version.is_rev2() || self.is_utf8; + let data = self.state.session_data(); - spawn_op!(data, { + spawn_op!(permit, data, { let (mailbox_id, mailbox_, access_token) = data .get_acl_mailbox(&arguments, false) .await @@ -181,49 +139,12 @@ impl Session { let mailbox = mailbox_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; - let rights = if access_token.is_shared(mailbox_id.account_id) { - let acl = mailbox.inner.acls.effective_acl(&access_token); - let mut rights = Vec::with_capacity(5); - if acl.contains(Acl::ReadItems) { - rights.push(Rights::Read); - rights.push(Rights::Lookup); - } - if acl.contains(Acl::AddItems) { - rights.push(Rights::Insert); - } - if acl.contains(Acl::RemoveItems) { - rights.push(Rights::DeleteMessages); - rights.push(Rights::Expunge); - } - if acl.contains(Acl::ModifyItems) { - rights.push(Rights::Seen); - rights.push(Rights::Write); - } - if acl.contains(Acl::CreateChild) { - rights.push(Rights::CreateMailbox); - } - if acl.contains(Acl::Delete) { - rights.push(Rights::DeleteMailbox); - } - if acl.contains(Acl::Submit) { - rights.push(Rights::Post); - } - rights + let acl = if access_token.is_shared(mailbox_id.account_id) { + mailbox.inner.acls.effective_acl(&access_token) } else { - vec![ - Rights::Read, - Rights::Lookup, - Rights::Insert, - Rights::DeleteMessages, - Rights::Expunge, - Rights::Seen, - Rights::Write, - Rights::CreateMailbox, - Rights::DeleteMailbox, - Rights::Post, - Rights::Administer, - ] + Bitmap::all() }; + let rights = Rights::from_acl(acl).collect::>(); trc::event!( Imap(trc::ImapEvent::MyRights), @@ -243,7 +164,7 @@ impl Session { .with_tag(arguments.tag) .serialize( MyRightsResponse { - mailbox_name: arguments.mailbox_name.to_string(), + mailbox_name: arguments.mailbox_name.clone(), rights, } .into_bytes(is_utf8), @@ -253,7 +174,11 @@ impl Session { }) } - pub async fn handle_set_acl(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_set_acl( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAclSet)?; @@ -262,139 +187,136 @@ impl Session { let arguments = request.parse_acl(self.is_utf8)?; let data = self.state.session_data(); - spawn_op!(data, { - // Validate mailbox - let (mailbox_id, current_mailbox, _) = data - .get_acl_mailbox(&arguments, true) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - let current_mailbox = current_mailbox - .into_deserialized::() - .imap_ctx(&arguments.tag, trc::location!())?; - - // Obtain principal id - let acl_account_id = data - .server - .account_id_from_email(arguments.identifier.as_ref().unwrap(), false) - .await - .imap_ctx(&arguments.tag, trc::location!())? - .ok_or_else(|| { - trc::ImapEvent::Error - .into_err() - .details("Account does not exist") - .id(arguments.tag.to_string()) - .caused_by(trc::location!()) - })?; - - // Prepare changes - let mut mailbox = current_mailbox.inner.clone(); - let (op, rights) = arguments - .mod_rights - .map(|mr| { - ( - mr.op, - Bitmap::from_iter(mr.rights.into_iter().map(Acl::from)), - ) - }) - .unwrap_or_else(|| (ModRightsOp::Replace, Bitmap::new())); - - if let Some(item) = mailbox - .acls - .iter_mut() - .find(|item| item.account_id == acl_account_id) - { - match op { - ModRightsOp::Replace => { - if !rights.is_empty() { - item.grants = rights; - } else { - mailbox - .acls - .retain(|item| item.account_id != acl_account_id); - } - } - ModRightsOp::Add => { - item.grants.union(&rights); - } - ModRightsOp::Remove => { - for right in rights { - item.grants.remove(right); - } - if item.grants.is_empty() { - mailbox - .acls - .retain(|item| item.account_id != acl_account_id); - } - } - } - } else if !rights.is_empty() { - match op { - ModRightsOp::Add | ModRightsOp::Replace => { - mailbox.acls.push(AclGrant { - account_id: acl_account_id, - grants: rights, - }); - } - ModRightsOp::Remove => (), - } - } - - if mailbox.acls.len() > data.server.core.groupware.max_shares_per_item { - return Err(trc::ImapEvent::Error - .into_err() - .details("Maximum shares per item exceeded") - .caused_by(trc::location!())); - } - - let grants = mailbox - .acls - .iter() - .map(|r| trc::Value::from(r.account_id)) - .collect::>(); - - // Write changes - let mut batch = BatchBuilder::new(); - batch - .with_account_id(mailbox_id.account_id) - .with_collection(Collection::Mailbox) - .with_document(mailbox_id.mailbox_id) - .custom( - ObjectIndexBuilder::new() - .with_changes(mailbox) - .with_current(current_mailbox), - ) - .imap_ctx(&arguments.tag, trc::location!())?; - - if !batch.is_empty() { - data.server - .commit_batch(batch) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - } - - // Invalidate ACLs - data.server - .invalidate_caches(CacheInvalidation::AccessToken(acl_account_id).into()) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - trc::event!( - Imap(trc::ImapEvent::SetAcl), - SpanId = data.session_id, - MailboxName = arguments.mailbox_name.clone(), - AccountId = mailbox_id.account_id, - MailboxId = mailbox_id.mailbox_id, - Details = grants, - Elapsed = op_start.elapsed() - ); - - data.write_bytes( - StatusResponse::completed(command) - .with_tag(arguments.tag) - .into_bytes(), - ) + // Validate mailbox + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + let (mailbox_id, current_mailbox, access_token) = data + .get_acl_mailbox(&arguments, true) .await - }) + .imap_ctx(&arguments.tag, trc::location!())?; + let current_mailbox = current_mailbox + .into_deserialized::() + .imap_ctx(&arguments.tag, trc::location!())?; + + // Obtain principal id + let acl_account_id = data + .server + .account_id_from_email(arguments.identifier.as_ref().unwrap(), false) + .await + .imap_ctx(&arguments.tag, trc::location!())? + .ok_or_else(|| { + trc::ImapEvent::Error + .into_err() + .details("Account does not exist") + .id(arguments.tag.clone()) + .caused_by(trc::location!()) + })?; + + // Prepare changes + let mut mailbox = current_mailbox.inner.clone(); + let (op, rights) = arguments + .mod_rights + .map(|mr| { + ( + mr.op, + Bitmap::from_iter( + mr.rights + .iter() + .flat_map(|right| right.to_acl().iter().copied()), + ), + ) + }) + .unwrap_or_else(|| (ModRightsOp::Replace, Bitmap::new())); + + if let Some(item) = mailbox + .acls + .iter_mut() + .find(|item| item.account_id == acl_account_id) + { + match op { + ModRightsOp::Replace => { + if !rights.is_empty() { + item.grants = rights; + } else { + mailbox + .acls + .retain(|item| item.account_id != acl_account_id); + } + } + ModRightsOp::Add => { + item.grants.union(&rights); + } + ModRightsOp::Remove => { + item.grants.difference(&rights); + if item.grants.is_empty() { + mailbox + .acls + .retain(|item| item.account_id != acl_account_id); + } + } + } + } else if !rights.is_empty() && matches!(op, ModRightsOp::Add | ModRightsOp::Replace) { + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + let grantee = data + .server + .account(acl_account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + mailbox.acls.push(AclGrant { + account_id: acl_account_id, + grants: rights, + }); + } + + if mailbox.acls.len() > data.server.core.groupware.max_shares_per_item { + return Err(trc::ImapEvent::Error + .into_err() + .details("Maximum shares per item exceeded") + .caused_by(trc::location!())); + } + + let grants = mailbox + .acls + .iter() + .map(|r| trc::Value::from(r.account_id)) + .collect::>(); + + // Write changes + let mut batch = BatchBuilder::new(); + batch + .with_account_id(mailbox_id.account_id) + .with_collection(Collection::Mailbox) + .with_document(mailbox_id.mailbox_id) + .custom( + ObjectIndexBuilder::new() + .with_changes(mailbox) + .with_current(current_mailbox), + ) + .imap_ctx(&arguments.tag, trc::location!())?; + + if !batch.is_empty() { + data.server + .commit_batch(batch) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + data.expire_folders(mailbox_id.account_id); + } + + trc::event!( + Imap(trc::ImapEvent::SetAcl), + SpanId = data.session_id, + MailboxName = arguments.mailbox_name.clone(), + AccountId = mailbox_id.account_id, + MailboxId = mailbox_id.mailbox_id, + Details = grants, + Elapsed = op_start.elapsed() + ); + + data.write_bytes( + StatusResponse::completed(command) + .with_tag(arguments.tag) + .into_bytes(), + ) + .await } pub async fn handle_list_rights(&mut self, request: Request) -> trc::Result<()> { @@ -427,6 +349,7 @@ impl Session { vec![Rights::CreateMailbox], vec![Rights::DeleteMailbox], vec![Rights::Post], + vec![Rights::Annotate], vec![Rights::Administer], ], } @@ -452,12 +375,16 @@ impl SessionData { &self, arguments: &Arguments, validate: bool, - ) -> trc::Result<(MailboxId, Archive, AccessToken)> { + ) -> trc::Result<(MailboxId, Archive, AccessToken)> { + self.expire_folders_of(&arguments.mailbox_name); + self.synchronize_mailboxes(false) + .await + .caused_by(trc::location!())?; if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index f85645c3..bb413d2a 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -4,21 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ImapContext, ToModSeq}; -use crate::{ - core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}, - spawn_op, +use super::{ImapContext, IntoImapError, ToModSeq}; +use crate::core::{MailboxId, Row, SelectedMailbox, Session, SessionData}; +use common::{ + MAX_RECEIVED_AT, auth::BuildAccessToken, ipc::PushNotification, network::SessionStream, +}; +use email::message::ingest::{ + IngestEmail, IngestSource, IngestedEmail, + group::{IngestGroup, ThreadKey}, + recheck::ThreadRecheck, }; -use common::{auth::BuildAccessToken, ipc::PushNotification, network::SessionStream}; -use email::message::ingest::{EmailIngest, IngestEmail, IngestSource}; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{append::Arguments, select::HighestModSeq}, receiver::Request, }; use mail_parser::MessageParser; -use registry::schema::enums::Permission; +use registry::schema::enums::{Permission, StorageQuota}; use std::{sync::Arc, time::Instant}; +use tokio::sync::OwnedSemaphorePermit; use types::{ acl::Acl, keyword::Keyword, @@ -26,7 +30,11 @@ use types::{ }; impl Session { - pub async fn handle_append(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_append( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAppend)?; @@ -51,6 +59,7 @@ impl Session { } // Refresh mailboxes + data.expire_folders_of(&arguments.mailbox_name); data.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -66,32 +75,48 @@ impl Session { .id(arguments.tag)); }; let is_qresync = self.is_qresync; + let use_vanished = self.is_qresync || self.is_uidonly; - spawn_op!(data, { - let response = data - .append_messages(arguments, selected_mailbox, mailbox, is_qresync, op_start) - .await? - .into_bytes(); + let mut recheck = ThreadRecheck::default(); + let result = match data + .append_messages( + arguments, + selected_mailbox, + mailbox, + is_qresync, + use_vanished, + op_start, + &mut recheck, + ) + .await + { + Ok(response) => data.write_bytes(response.into_bytes()).await, + Err(err) => Err(err), + }; - data.write_bytes(response).await - }) + recheck.finish(&data.server, data.session_id).await; + + result } } impl SessionData { + #[allow(clippy::too_many_arguments)] async fn append_messages( &self, arguments: Arguments, selected_mailbox: Option>, mailbox: MailboxId, is_qresync: bool, + use_vanished: bool, op_start: Instant, + recheck: &mut ThreadRecheck, ) -> trc::Result { // Verify ACLs let account_id = mailbox.account_id; let mailbox_id = mailbox.mailbox_id; if !self - .check_mailbox_acl(account_id, mailbox_id, Acl::AddItems) + .check_mailbox_acl(None, account_id, mailbox_id, Acl::AddItems) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -117,50 +142,115 @@ impl SessionData { .build() }; + // RFC 9738 makes APPEND atomic, so the whole batch must fit in the email quota + if arguments.messages.len() > 1 { + let account = self + .server + .account(account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + if self + .server + .object_quota_limit(&account, StorageQuota::MaxEmails) + .is_some() + { + let used = self + .server + .object_count(account_id, StorageQuota::MaxEmails) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + self.server + .assert_object_quota( + &account, + StorageQuota::MaxEmails, + arguments.messages.len(), + || used, + ) + .map_err(|err| { + err.details("Quota exceeded.") + .code(ResponseCode::OverQuota) + .id(arguments.tag.clone()) + })?; + } + } + // Append messages let mut response = StatusResponse::completed(Command::Append); - let mut created_ids = Vec::with_capacity(arguments.messages.len()); + let mut created = Vec::with_capacity(arguments.messages.len()); let mut last_change_id = None; + let limits = &self.server.core.email.limits; + let grouping = arguments.messages.len() > 1; + let mut group = IngestGroup::with_max_emails(arguments.messages.len()); for message in arguments.messages { - match self - .server - .email_ingest(IngestEmail { - raw_message: &message.message, - message: MessageParser::new().parse(&message.message), - blob_hash: None, - access_token: &access_token, - mailbox_ids: vec![mailbox_id], - keywords: message.flags.into_iter().map(Keyword::from).collect(), - received_at: message.received_at.map(|d| d as u64), - source: IngestSource::Imap { - train_classifier: true, - }, - session_id: self.session_id, - }) - .await + let parsed = MessageParser::new().parse(&message.message); + if let Some(parsed) = &parsed + && let Err(err) = limits.validate_header_section(parsed) { - Ok(email) => { - created_ids.push(ImapUidToId { - uid: email.imap_uids[0], - id: email.document_id, - }); - last_change_id = Some(email.change_id); - } - Err(err) => { - return Err( - if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) { - err.details("Disk quota exceeded.") - .code(ResponseCode::OverQuota) - } else if err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) { - err.details("Organization disk quota exceeded.") - .code(ResponseCode::OverQuota) - } else { - err - } - .id(arguments.tag), - ); + return Err(err.into_imap_error().id(arguments.tag)); + } + let keywords = message + .flags + .into_iter() + .map(Keyword::from) + .collect::>(); + if let Err(err) = limits.validate_email(1, &keywords) { + return Err(err.into_imap_error().id(arguments.tag)); + } + + let received_at = message + .received_at + .map(|received_at| received_at.clamp(0, MAX_RECEIVED_AT as i64) as u64); + + let key = match &parsed { + Some(parsed) if grouping => ThreadKey::new(parsed), + _ => ThreadKey::default(), + }; + if group.must_flush_before(message.message.len()) { + match group.flush(&self.server).await { + Ok(emails) => appended( + emails, + account_id, + recheck, + &mut created, + &mut last_change_id, + ), + Err(err) => return Err(append_error(err).id(arguments.tag)), } } + + if let Err(err) = group + .ingest( + &self.server, + IngestEmail { + raw_message: &message.message, + message: parsed, + blob_hash: None, + access_token: &access_token, + mailbox_ids: vec![mailbox_id], + keywords, + received_at, + source: IngestSource::Imap { + train_classifier: true, + }, + metadata: None, + session_id: self.session_id, + }, + key, + ) + .await + { + return Err(append_error(err).id(arguments.tag)); + } + } + match group.flush(&self.server).await { + Ok(emails) => appended( + emails, + account_id, + recheck, + &mut created, + &mut last_change_id, + ), + Err(err) => return Err(append_error(err).id(arguments.tag)), } // Broadcast changes @@ -183,34 +273,36 @@ impl SessionData { MailboxName = arguments.mailbox_name.clone(), AccountId = account_id, MailboxId = mailbox_id, - DocumentId = created_ids + DocumentId = created .iter() - .map(|r| trc::Value::from(r.id)) + .map(|row| trc::Value::from(row.id)) .collect::>(), Elapsed = op_start.elapsed() ); - if !created_ids.is_empty() { - let uids = created_ids.iter().map(|id| id.uid).collect(); - match selected_mailbox { - Some(selected_mailbox) if selected_mailbox.id == mailbox => { - // Write updated modseq - if is_qresync { - self.write_bytes( - HighestModSeq::new(last_change_id.unwrap_or_default().to_modseq()) - .into_bytes(), - ) - .await?; - } - - selected_mailbox.append_messages(created_ids, last_change_id); + if !created.is_empty() { + let uids = created.iter().map(|row| row.uid).collect(); + if let Some(selected_mailbox) = selected_mailbox + && selected_mailbox.id == mailbox + { + // Write updated modseq + if is_qresync { + self.write_bytes( + HighestModSeq::new(last_change_id.unwrap_or_default().to_modseq()) + .into_bytes(), + ) + .await?; } - _ => {} - }; - let uid_validity = self - .mailbox_state(&mailbox) - .map(|m| m.uid_validity as u32) - .unwrap_or_default(); + + selected_mailbox.view.lock().append_local(created); + + // RFC 9051 6.3.12 requires an untagged EXISTS when the mailbox + // being appended to is the one this connection has selected + self.flush_view(&selected_mailbox, use_vanished) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + } + let uid_validity = self.uid_validity(&mailbox); response = response.with_code(ResponseCode::AppendUid { uid_validity, uids }); } @@ -218,3 +310,35 @@ impl SessionData { Ok(response.with_tag(arguments.tag)) } } + +fn appended( + emails: Vec, + account_id: u32, + recheck: &mut ThreadRecheck, + created: &mut Vec, + last_change_id: &mut Option, +) { + for mut email in emails { + recheck.push(account_id, &mut email); + *last_change_id = Some(email.change_id); + if let Some(&uid) = email.imap_uids.first() { + created.push(Row { + uid, + id: email.document_id, + }); + } + } +} + +fn append_error(err: trc::Error) -> trc::Error { + #[cfg(not(feature = "enterprise"))] + let is_tenant_quota = false; + if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) { + err.details("Quota exceeded.").code(ResponseCode::OverQuota) + } else if is_tenant_quota { + err.details("Organization disk quota exceeded.") + .code(ResponseCode::OverQuota) + } else { + err + } +} diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index f4f61d15..cddf030f 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -6,94 +6,179 @@ use crate::core::{Session, SessionData, State}; use common::{ - auth::AuthRequest, + auth::{ + AuthRequest, + oauth::scope::OAuthScopes, + sasl::{BearerFailure, BearerMechanism, SASL_CANCEL, SASL_EMPTY}, + }, network::{SessionStream, limiter::LimiterResult}, }; +use compact_str::CompactString; use directory::Credentials; +use encodify::base64::LENIENT; use imap_proto::{ - Command, ResponseCode, StatusResponse, - protocol::{authenticate::Mechanism, capability::Capability}, - receiver::{self, Request}, + Command, ResponseCode, ResponseType, StatusResponse, + protocol::authenticate::Mechanism, + receiver::{self, MAX_UNAUTHENTICATED_REQUEST_SIZE, Request}, }; -use mail_parser::decoders::base64::base64_decode; use registry::schema::enums::Permission; use std::sync::Arc; +use tokio::sync::OwnedSemaphorePermit; impl Session { pub async fn handle_authenticate(&mut self, request: Request) -> trc::Result<()> { let mut args = request.parse_authenticate()?; - match args.mechanism { - Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { - if !args.params.is_empty() { - let challenge = base64_decode(args.params.pop().unwrap().as_bytes()) - .ok_or_else(|| { - trc::AuthEvent::Error - .into_err() - .details("Failed to decode challenge.") - .id(args.tag.clone()) - .code(ResponseCode::Parse) - })?; + let bearer = match args.mechanism { + Mechanism::Plain => None, + Mechanism::OAuthBearer => Some(BearerMechanism::OAuthBearer), + Mechanism::XOauth2 => Some(BearerMechanism::XOAuth2), + _ => { + return Err(trc::AuthEvent::Error + .into_err() + .details("Authentication mechanism not supported.") + .id(args.tag) + .code(ResponseCode::Cannot)); + } + }; - let credentials = if args.mechanism == Mechanism::Plain { - Credentials::decode_sasl_challenge_plain(&challenge) - } else { - Credentials::decode_sasl_challenge_oauth(&challenge) - } - .ok_or_else(|| { - trc::AuthEvent::Error - .into_err() - .details("Invalid SASL challenge.") - .id(args.tag.clone()) - })?; + if bearer.is_some() && !self.server.accepts_bearer_tokens(self.is_tls) { + return Err(trc::ImapEvent::Error + .into_err() + .details("OAUTHBEARER and XOAUTH2 require a TLS connection.") + .code(ResponseCode::PrivacyRequired) + .id(args.tag)); + } - self.authenticate(credentials, args.tag).await + match args.params.pop() { + Some(param) if param == SASL_CANCEL => Err(trc::ImapEvent::Error + .into_err() + .details("Authentication cancelled.") + .ctx(trc::Key::Type, ResponseType::Bad) + .id(args.tag)), + Some(param) if param != SASL_EMPTY => { + let Ok(challenge) = LENIENT.decode(param.as_bytes()) else { + return Err(self + .malformed_credentials(&args.tag) + .await + .ctx(trc::Key::Type, ResponseType::Bad)); + }; + let credentials = if bearer.is_none() { + Credentials::decode_sasl_challenge_plain(&challenge) } else { - self.receiver.request = receiver::Request { - tag: args.tag, - command: Command::Authenticate, - tokens: vec![receiver::Token::Argument(args.mechanism.into_bytes())], - }; - self.receiver.state = receiver::State::Argument { last_ch: b' ' }; - self.write_bytes(b"+ \r\n".to_vec()).await + Credentials::decode_sasl_challenge_oauth(&challenge) + }; + match (credentials, bearer) { + (Some(credentials), _) => { + self.authenticate(credentials, args.tag, bearer).await + } + (None, Some(mechanism)) => { + let err = self.malformed_credentials(&args.tag).await; + match BearerFailure::from_error(&err) { + Some(failure) => { + self.bearer_challenge(mechanism, failure, None, err, args.tag) + .await + } + None => Err(err), + } + } + (None, None) => Err(self.malformed_credentials(&args.tag).await), } } - _ => Err(trc::AuthEvent::Error - .into_err() - .details("Authentication mechanism not supported.") - .id(args.tag) - .code(ResponseCode::Cannot)), + _ => { + self.expect_sasl_response(args.tag, args.mechanism); + self.write_bytes(b"+ \r\n".to_vec()).await + } } } - pub async fn authenticate(&mut self, credentials: Credentials, tag: String) -> trc::Result<()> { - // Authenticate - let access_token = self + async fn bearer_challenge( + &mut self, + mechanism: BearerMechanism, + failure: BearerFailure, + credentials: Option<&Credentials>, + err: trc::Error, + tag: CompactString, + ) -> trc::Result<()> { + let challenge = self .server - .authenticate(&AuthRequest::from_credentials( - credentials, - self.session_id, - self.remote_addr, - )) + .bearer_failure_challenge(mechanism, failure, credentials) + .await; + if let State::NotAuthenticated { sasl_failure, .. } = &mut self.state { + *sasl_failure = Some(err); + } + self.expect_sasl_response( + tag, + match mechanism { + BearerMechanism::OAuthBearer => Mechanism::OAuthBearer, + BearerMechanism::XOAuth2 => Mechanism::XOauth2, + }, + ); + self.write_bytes(format!("+ {challenge}\r\n").into_bytes()) .await - .map_err(|err| { - if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { - let auth_failures = self.state.auth_failures(); - if auth_failures < self.server.core.imap.max_auth_failures { - self.state = State::NotAuthenticated { - auth_failures: auth_failures + 1, - }; - } else { - return trc::AuthEvent::TooManyAttempts - .into_err() - .caused_by(err) - .id(tag.clone()); + } + + fn expect_sasl_response(&mut self, tag: CompactString, mechanism: Mechanism) { + self.receiver.request = receiver::Request { + tag, + command: Command::Authenticate, + tokens: vec![receiver::Token::Argument( + receiver::ArgumentBytes::from_slice(mechanism.as_bytes()), + )], + }; + self.receiver.state = receiver::State::Argument { last_ch: b' ' }; + } + + pub async fn authenticate( + &mut self, + credentials: Credentials, + tag: CompactString, + bearer: Option, + ) -> trc::Result<()> { + // Authenticate + let request = AuthRequest::from_credentials(credentials, self.session_id, self.remote_addr); + let access_token = match self.server.authenticate(&request).await { + Ok(authenticated) => match bearer { + Some(mechanism) => { + match authenticated + .access_token + .assert_allows_oauth_scopes(OAuthScopes::MAIL) + { + Ok(access_token) => access_token, + Err(err) => { + return self + .bearer_challenge( + mechanism, + BearerFailure::InsufficientScope, + Some(&request.credentials), + BearerFailure::insufficient_scope(err).id(tag.clone()), + tag, + ) + .await; + } } } - - err.id(tag.clone()) - }) - .and_then(|token| token.assert_has_permission(Permission::ImapAuthenticate))?; + None => authenticated.access_token, + } + .assert_has_permission(Permission::ImapAuthenticate) + .map_err(|err| err.id(tag.clone()))?, + Err(err) => { + let err = self.register_auth_failure(err, &tag); + return match (bearer, BearerFailure::from_error(&err)) { + (Some(mechanism), Some(failure)) => { + self.bearer_challenge( + mechanism, + failure, + Some(&request.credentials), + err, + tag, + ) + .await + } + _ => Err(err), + }; + } + }; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { @@ -114,16 +199,11 @@ impl Session { .map_err(|err| err.id(tag.clone()))?, ), }; + self.receiver.max_request_size = self.server.core.imap.max_request_size; self.write_bytes( StatusResponse::ok("Authentication successful") .with_code(ResponseCode::Capability { - capabilities: Capability::all_capabilities( - true, - !self.is_tls && self.instance.acceptor.is_tls(), - true, - self.server.core.imap.max_messages_per_command, - self.server.core.imap.max_messages_per_save, - ), + capabilities: self.capabilities(), }) .with_tag(tag) .into_bytes(), @@ -131,8 +211,49 @@ impl Session { .await } - pub async fn handle_unauthenticate(&mut self, request: Request) -> trc::Result<()> { - self.state = State::NotAuthenticated { auth_failures: 0 }; + async fn malformed_credentials(&mut self, tag: &CompactString) -> trc::Error { + let err = self + .server + .auth_failure( + trc::AuthEvent::Failed + .into_err() + .details("Invalid SASL response."), + self.remote_addr, + None, + ) + .await; + self.register_auth_failure(err, tag) + } + + fn register_auth_failure(&mut self, err: trc::Error, tag: &CompactString) -> trc::Error { + if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { + let auth_failures = self.state.auth_failures(); + if auth_failures < self.server.core.imap.max_auth_failures { + self.state = State::NotAuthenticated { + auth_failures: auth_failures + 1, + sasl_failure: None, + }; + } else { + return trc::AuthEvent::TooManyAttempts + .into_err() + .caused_by(err) + .id(tag.clone()); + } + } + + err.id(tag.clone()) + } + + pub async fn handle_unauthenticate( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { + self.state = State::NotAuthenticated { + auth_failures: 0, + sasl_failure: None, + }; + self.receiver.max_request_size = MAX_UNAUTHENTICATED_REQUEST_SIZE; self.is_condstore = false; self.is_qresync = false; self.is_utf8 = false; diff --git a/crates/imap/src/op/capability.rs b/crates/imap/src/op/capability.rs index bc4a32ea..adb05e86 100644 --- a/crates/imap/src/op/capability.rs +++ b/crates/imap/src/op/capability.rs @@ -6,12 +6,14 @@ use std::time::Instant; -use crate::core::Session; +use crate::{ + auth_capability, + core::{Session, State}, +}; use common::network::SessionStream; use imap_proto::{ Command, StatusResponse, protocol::{ - ImapResponse, capability::{Capability, Response}, quoted_string, }, @@ -36,22 +38,31 @@ impl Session { self.write_bytes( StatusResponse::completed(Command::Capability) .with_tag(request.tag) - .serialize( - Response { - capabilities: Capability::all_capabilities( - self.state.is_authenticated(), - !self.is_tls && self.instance.acceptor.is_tls(), - self.is_tls || self.server.core.imap.allow_plain_auth, - self.server.core.imap.max_messages_per_command, - self.server.core.imap.max_messages_per_save, - ), - } - .serialize(), - ), + .serialize_after(&Response { + capabilities: self.capabilities(), + }), ) .await } + pub fn capabilities(&self) -> Vec { + let mut capabilities = Capability::all_capabilities( + self.state.is_authenticated(), + !self.is_tls && self.instance.acceptor.is_tls(), + auth_capability(&self.server, self.is_tls), + self.server.core.imap.max_messages_per_command, + self.server.core.imap.max_messages_per_save, + ); + if let State::Authenticated { data } | State::Selected { data, .. } = &self.state + && data + .access_token + .has_permission(Permission::ImapMetadataGet) + { + capabilities.extend([Capability::Metadata, Capability::ListMetadata]); + } + capabilities + } + pub async fn handle_id(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapId)?; diff --git a/crates/imap/src/op/close.rs b/crates/imap/src/op/close.rs index b5a938ff..745cfaa6 100644 --- a/crates/imap/src/op/close.rs +++ b/crates/imap/src/op/close.rs @@ -4,22 +4,49 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - +use super::ImapContext; use crate::core::{Session, State}; use common::network::SessionStream; +use email::cache::MessageCacheFetch; use imap_proto::{Command, StatusResponse, receiver::Request}; -use trc::AddContext; +use registry::schema::enums::Permission; +use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; +use types::acl::Acl; impl Session { - pub async fn handle_close(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_close( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { let op_start = Instant::now(); let (data, mailbox) = self.state.select_data(); - if mailbox.is_select { - data.expunge(mailbox.clone(), None, u32::MAX, op_start) + let mut skipped_expunge = None; + if mailbox.is_select && data.access_token.has_permission(Permission::ImapExpunge) { + let cache = data + .server + .get_cached_messages(mailbox.id.account_id) .await - .caused_by(trc::location!())?; + .imap_ctx(&request.tag, trc::location!())?; + match data + .check_mailbox_acl( + Some(&cache), + mailbox.id.account_id, + mailbox.id.mailbox_id, + Acl::RemoveItems, + ) + .await + { + Ok(true) => { + data.expunge(mailbox.clone(), cache, None, u32::MAX, op_start) + .await + .imap_ctx(&request.tag, trc::location!())?; + } + Ok(false) => {} + Err(err) => skipped_expunge = Some(err), + } } trc::event!( @@ -27,6 +54,7 @@ impl Session { SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, + CausedBy = skipped_expunge, Elapsed = op_start.elapsed() ); diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index f0eeabee..0ad65c87 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -5,19 +5,19 @@ */ use super::ImapContext; -use crate::{ - core::{MailboxId, SelectedMailbox, Session, SessionData}, - spawn_op, +use crate::core::{AccountCaches, MailboxId, Resolved, SelectedMailbox, Session, SessionData}; +use common::{ + MessageStoreCache, MessageUid, ipc::PushNotification, network::SessionStream, + storage::index::ObjectIndexBuilder, }; -use ahash::AHashMap; -use common::{ipc::PushNotification, network::SessionStream, storage::index::ObjectIndexBuilder}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, + mailbox::{JUNK_ID, TRASH_ID}, message::{ - copy::{CopyMessageError, EmailCopy}, - ingest::EmailIngest, - metadata::MessageData, + copy::{CopyMessageError, CopySources, EmailCopy}, + ingest::{EmailIngest, SpamSample, SpamSamples}, + ingest_metadata::IngestMetadata, + messagedata::{MessageData, PendingMessageData}, }, }; use imap_proto::{ @@ -33,15 +33,19 @@ use std::{ use store::{ ValueKey, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{BatchBuilder, Slot, now}, }; -use trc::AddContext; +use tokio::sync::OwnedSemaphorePermit; use types::{ acl::Acl, - collection::{Collection, VanishedCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, + keyword::Keyword, + metadata::EncodedMetadata, type_state::{DataType, StateChange}, }; +const TOO_MANY_MAILBOXES: &str = "Message belongs to too many mailboxes."; +const TOO_MANY_KEYWORDS: &str = "Message exceeds the keyword limits of the destination."; const MAX_MOVE_RETRIES: u32 = 3; const MOVE_RETRY_BACKOFF_MS: RangeInclusive = 1..=15; @@ -51,6 +55,7 @@ impl Session { request: Request, is_move: bool, is_uid: bool, + _permit: Option, ) -> trc::Result<()> { // Validate access self.assert_has_permission(if is_move { @@ -70,47 +75,49 @@ impl Session { self.server.core.imap.max_messages_per_save }; - spawn_op!(data, { - // Refresh mailboxes - data.synchronize_mailboxes(false) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - // Make sure the mailbox exists. - let dest_mailbox = - if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { - mailbox - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details("Destination mailbox does not exist.") - .code(ResponseCode::TryCreate) - .id(arguments.tag)); - }; - - // Check that the destination mailbox is not the same as the source mailbox. - if src_mailbox.id.account_id == dest_mailbox.account_id - && src_mailbox.id.mailbox_id == dest_mailbox.mailbox_id - { - return Err(trc::ImapEvent::Error - .into_err() - .details("Source and destination mailboxes are the same.") - .code(ResponseCode::Cannot) - .id(arguments.tag)); - } - - data.copy_move( - arguments, - src_mailbox, - dest_mailbox, - is_move, - is_uid, - use_vanished, - message_limit, - op_start, - ) + // Refresh mailboxes + data.use_messages_of(&arguments.mailbox_name); + let caches = data + .synchronize_mailboxes(false) .await - }) + .imap_ctx(&arguments.tag, trc::location!())? + .caches; + + // Make sure the mailbox exists. + let dest_mailbox = if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) + { + mailbox + } else { + return Err(trc::ImapEvent::Error + .into_err() + .details("Destination mailbox does not exist.") + .code(ResponseCode::TryCreate) + .id(arguments.tag)); + }; + + // Check that the destination mailbox is not the same as the source mailbox. + if src_mailbox.id.account_id == dest_mailbox.account_id + && src_mailbox.id.mailbox_id == dest_mailbox.mailbox_id + { + return Err(trc::ImapEvent::Error + .into_err() + .details("Source and destination mailboxes are the same.") + .code(ResponseCode::Cannot) + .id(arguments.tag)); + } + + data.copy_move( + arguments, + caches, + src_mailbox, + dest_mailbox, + is_move, + is_uid, + use_vanished, + message_limit, + op_start, + ) + .await } } @@ -119,6 +126,7 @@ impl SessionData { pub async fn copy_move( &self, arguments: Arguments, + mut caches: AccountCaches, src_mailbox: Arc, dest_mailbox: MailboxId, is_move: bool, @@ -127,13 +135,17 @@ impl SessionData { message_limit: u32, op_start: Instant, ) -> trc::Result<()> { - self.synchronize_messages(&src_mailbox) + let mut cache = caches + .fetch(&self.server, src_mailbox.id.account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + self.sync_view(&src_mailbox, &cache, None) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert IMAP ids to JMAP ids. - let ids = src_mailbox - .sequence_to_ids(&arguments.sequence_set, is_uid) + let mut ids = src_mailbox + .resolve(&arguments.sequence_set, is_uid) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -170,6 +182,7 @@ impl SessionData { if is_move && !self .check_mailbox_acl( + Some(&cache), src_mailbox.id.account_id, src_mailbox.id.mailbox_id, Acl::RemoveItems, @@ -189,8 +202,23 @@ impl SessionData { // Verify that the user can append messages to the destination mailbox. let dest_mailbox_id = dest_mailbox.mailbox_id; + let dest_cache = if src_mailbox.id.account_id != dest_mailbox.account_id { + Some( + caches + .fetch(&self.server, dest_mailbox.account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?, + ) + } else { + None + }; if !self - .check_mailbox_acl(dest_mailbox.account_id, dest_mailbox_id, Acl::AddItems) + .check_mailbox_acl( + Some(dest_cache.as_deref().unwrap_or(&cache)), + dest_mailbox.account_id, + dest_mailbox_id, + Acl::AddItems, + ) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -206,13 +234,12 @@ impl SessionData { // RFC 9738 requires the highest UIDs to be processed first when truncating. // COPY is atomic, so it is refused outright rather than partially applied. - let mut ids = ids; let message_limit = message_limit as usize; let mut limited_uid = None; if ids.len() > message_limit { - let mut uids = ids.values().map(|imap_id| imap_id.uid).collect::>(); - let cutoff = uids.len() - message_limit; - let lowest_uid = *uids.select_nth_unstable(cutoff).1; + let cutoff = ids.len() - message_limit; + let (_, lowest, _) = ids.select_nth_unstable_by_key(cutoff, |resolved| resolved.uid); + let lowest_uid = lowest.uid; if !is_move { return self @@ -228,7 +255,8 @@ impl SessionData { .await; } - ids.retain(|_, imap_id| imap_id.uid >= lowest_uid); + ids.drain(..cutoff); + ids.sort_unstable_by_key(|resolved| resolved.uid); limited_uid = Some(lowest_uid); } @@ -244,42 +272,42 @@ impl SessionData { if src_mailbox.id.account_id == dest_mailbox.account_id { // Mailboxes are in the same account let account_id = src_mailbox.id.account_id; - let dest_mailbox_id = UidMailbox::new_unassigned(dest_mailbox_id); - let mut written_uids = AHashMap::with_capacity(ids.len()); + let dest_mailbox_id = MessageUid::new_unassigned(dest_mailbox_id); + let mut written_ids = RoaringBitmap::new(); let mut retries = 0; loop { let mut batch = BatchBuilder::new(); + let mut spam_samples = SpamSamples::default(); + let mut pending_copied_ids: Vec<(u32, Slot)> = Vec::with_capacity(ids.len()); copied_ids.clear(); did_move = false; + error = None; - for (&id, imap_id) in &ids { + for imap_id in &ids { + let id = imap_id.id; // Obtain mailbox tags - let data_ = if let Some(result) = self - .get_message_data(account_id, id) - .await - .imap_ctx(&arguments.tag, trc::location!())? - { - result - } else { + let Some(data) = cache.message_data(id) else { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id, + Collection = Collection::Email, + Details = "Message data not found.", + CausedBy = trc::location!(), + ); continue; }; - // Deserialize - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; - // Make sure the message still belongs to this mailbox if !data - .inner .mailboxes .iter() .any(|mailbox| mailbox.mailbox_id == src_mailbox.id.mailbox_id) { // Moved by a chunk of a previous attempt - if let Some(&uid) = written_uids.get(&id) - && data.inner.message_uid(dest_mailbox_id.mailbox_id) == Some(uid) + if written_ids.contains(id) + && let Some(uid) = data.message_uid(dest_mailbox_id.mailbox_id) { copied_ids.push((imap_id.uid, uid)); did_move = true; @@ -289,16 +317,14 @@ impl SessionData { // If the message is already in the destination mailbox, skip it. if let Some(mailbox) = data - .inner .mailboxes .iter() .find(|mailbox| mailbox.mailbox_id == dest_mailbox_id.mailbox_id) { - let uid = mailbox.uid.to_native(); - copied_ids.push((imap_id.uid, uid)); + copied_ids.push((imap_id.uid, mailbox.uid)); if is_move { - let mut new_data = data.inner.to_builder(); + let mut new_data = data.clone(); new_data.remove_mailbox(src_mailbox.id.mailbox_id); batch .with_account_id(account_id) @@ -307,7 +333,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data.seal()), + .with_changes(new_data), ) .imap_ctx(&arguments.tag, trc::location!())? .log_vanished_item( @@ -315,7 +341,7 @@ impl SessionData { (src_mailbox.id.mailbox_id, imap_id.uid), ) .commit_point(); - written_uids.insert(id, uid); + written_ids.insert(id); did_move = true; } @@ -323,51 +349,51 @@ impl SessionData { } // Prepare changes - let mut new_data = data.inner.to_builder(); + let mut new_data = data.clone(); // Add destination folder new_data.add_mailbox(dest_mailbox_id); if is_move { new_data.remove_mailbox(src_mailbox.id.mailbox_id); } - - // Assign IMAP UIDs - let ids = self + if self .server - .assign_email_ids( - account_id, - new_data - .mailboxes - .iter() - .filter(|m| m.uid == 0) - .map(|m| m.mailbox_id), - false, - ) - .await - .caused_by(trc::location!())?; - - for (uid_mailbox, uid) in new_data - .mailboxes - .iter_mut() - .filter(|m| m.uid == 0) - .zip(ids) + .core + .email + .limits + .validate_mailbox_change(data.mailboxes.len(), new_data.mailboxes.len()) + .is_err() { - copied_ids.push((imap_id.uid, uid)); - written_uids.insert(id, uid); - uid_mailbox.uid = uid; + error = Some((ResponseCode::Limit, TOO_MANY_MAILBOXES)); + continue; } + // Reserve IMAP UIDs + let uid_slots = batch.reserve_uids( + account_id, + new_data + .mailboxes + .iter() + .filter(|m| m.uid == 0) + .map(|m| m.mailbox_id), + ); + pending_copied_ids.extend(uid_slots.iter().map(|slot| (imap_id.uid, slot))); + // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), - ) + .custom(ObjectIndexBuilder::new().with_current(data).with_changes( + PendingMessageData { + data: new_data, + uid_slots, + thread_slot: None, + change_id: None, + }, + )) .imap_ctx(&arguments.tag, trc::location!())?; + written_ids.insert(id); if is_move { batch.log_vanished_item( VanishedCollection::Email, @@ -377,29 +403,11 @@ impl SessionData { // Add message to training queue if dest_mailbox_id.mailbox_id == JUNK_ID { - self.server - .add_account_spam_sample( - &mut batch, - account_id, - id, - true, - self.session_id, - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + spam_samples.add(id, true); } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id.mailbox_id != TRASH_ID { - self.server - .add_account_spam_sample( - &mut batch, - account_id, - id, - false, - self.session_id, - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + spam_samples.add(id, false); } batch.commit_point(); @@ -410,12 +418,32 @@ impl SessionData { } } + self.server + .add_account_spam_samples(&mut batch, account_id, spam_samples, self.session_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + // Write changes match self.server.commit_batch(batch).await { - Ok(_) => break, + Ok(assigned_ids) => { + copied_ids.extend( + pending_copied_ids + .into_iter() + .map(|(src_uid, slot)| (src_uid, assigned_ids.slot(slot))), + ); + break; + } Err(err) => { retry_after_conflict(err, &mut retries, &arguments.tag, trc::location!()) - .await? + .await?; + self.server + .inner + .mark_cache_stale(account_id, SyncCollection::Email); + cache = self + .server + .get_cached_messages(account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; } } } @@ -425,12 +453,6 @@ impl SessionData { let mut dest_change_id = None; let dest_account_id = dest_mailbox.account_id; let mut destroy_ids = RoaringBitmap::new(); - let cache = self - .server - .get_cached_messages(src_account_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - let mut dest_cache = None; let train_spam = if dest_mailbox_id == JUNK_ID { Some(true) } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id != TRASH_ID { @@ -438,22 +460,80 @@ impl SessionData { } else { None }; - let mut train_batch = BatchBuilder::new(); - let mut did_train = false; - train_batch.with_account_id(src_account_id); - 'next_message: for (id, imap_id) in ids { + let mut spam_samples = SpamSamples::default(); + let mut flagged_containers = None; + let dest_account = self + .server + .account(dest_account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + let mut sources = CopySources::new( + src_account_id, + ids.iter().map(|imap_id| imap_id.id).collect(), + ); + 'next_message: for imap_id in &ids { + let id = imap_id.id; + let email = cache.email_by_id(&id); + let keywords = email + .map(|email| cache.expand_keywords(email).collect::>()) + .unwrap_or_default(); + let user_keywords = keywords + .iter() + .filter(|keyword| { + !matches!(keyword, Keyword::HasAttachment | Keyword::HasNoAttachment) + }) + .count(); + if self + .server + .core + .email + .limits + .validate_keywords( + user_keywords, + keywords.iter().filter_map(|keyword| keyword.id().err()), + ) + .is_err() + { + error = Some((ResponseCode::Limit, TOO_MANY_KEYWORDS)); + continue; + } + let shared = if email.is_some_and(|email| !cache.metadata_kinds(email).is_empty()) { + if flagged_containers.is_none() { + flagged_containers = Some( + self.flagged_containers(src_account_id, &cache, &ids) + .await + .imap_ctx(&arguments.tag, trc::location!())?, + ); + } + flagged_containers + .as_mut() + .and_then(|containers| take_container(containers, id)) + } else { + None + }; + let Some(source) = sources + .take(&self.server, id) + .await + .imap_ctx(&arguments.tag, trc::location!())? + else { + continue; + }; + let sample = match train_spam { + Some(is_spam) if self.server.core.spam.classifier.is_some() => Some( + SpamSample::from_metadata(source.metadata(), is_spam) + .imap_ctx(&arguments.tag, trc::location!())?, + ), + _ => None, + }; match self .server .copy_message( - src_account_id, - id, - dest_account_id, + source, + &dest_account, vec![dest_mailbox_id], - cache - .email_by_id(&id) - .map(|e| cache.expand_keywords(e).collect()) - .unwrap_or_default(), - None, + keywords, + email.map(|email| email.received_at()).unwrap_or_else(now), + IngestMetadata::new(shared, None), self.session_id, ) .await @@ -467,21 +547,12 @@ impl SessionData { } } Err(CopyMessageError::AlreadyExists(existing_id)) => { - if dest_cache.is_none() { - dest_cache = self - .server - .get_cached_messages(dest_account_id) - .await - .imap_ctx(&arguments.tag, trc::location!())? - .into(); - } - if let Some(uid) = dest_cache .as_ref() .and_then(|cache| cache.email_by_id(&existing_id)) .and_then(|message| { message - .mailboxes + .mailboxes() .iter() .find(|mailbox| mailbox.mailbox_id == dest_mailbox_id) }) @@ -492,73 +563,72 @@ impl SessionData { let mut retries = 0; loop { - let data_ = if let Some(data_) = self + let Some(data) = self .get_message_data(dest_account_id, existing_id) .await .imap_ctx(&arguments.tag, trc::location!())? - { - data_ - } else { + else { continue 'next_message; }; - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; - if let Some(uid) = data.inner.message_uid(dest_mailbox_id) { + if let Some(uid) = data.message_uid(dest_mailbox_id) { copied_ids.push((imap_id.uid, uid)); break; } - let mut new_data = data.inner.to_builder(); - new_data.add_mailbox(UidMailbox::new_unassigned(dest_mailbox_id)); - - let uids = self + let mut new_data = data.clone(); + new_data.add_mailbox(MessageUid::new_unassigned(dest_mailbox_id)); + if self .server - .assign_email_ids( - dest_account_id, - new_data - .mailboxes - .iter() - .filter(|m| m.uid == 0) - .map(|m| m.mailbox_id), - false, + .core + .email + .limits + .validate_mailbox_change( + data.mailboxes.len(), + new_data.mailboxes.len(), ) - .await - .caused_by(trc::location!())?; - - let mut assigned_uid = 0; - for (uid_mailbox, uid) in new_data - .mailboxes - .iter_mut() - .filter(|m| m.uid == 0) - .zip(uids) + .is_err() { - uid_mailbox.uid = uid; - assigned_uid = uid; + error = Some((ResponseCode::Limit, TOO_MANY_MAILBOXES)); + continue 'next_message; } let mut batch = BatchBuilder::new(); + let uid_slots = batch.reserve_uids( + dest_account_id, + new_data + .mailboxes + .iter() + .filter(|m| m.uid == 0) + .map(|m| m.mailbox_id), + ); + let assigned_slot = uid_slots.iter().last(); + batch .with_account_id(dest_account_id) .with_collection(Collection::Email) .with_document(existing_id) .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), + ObjectIndexBuilder::new().with_current(data).with_changes( + PendingMessageData { + data: new_data, + uid_slots, + thread_slot: None, + change_id: None, + }, + ), ) .imap_ctx(&arguments.tag, trc::location!())?; - match self - .server - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(dest_account_id)) - { - Ok(change_id) => { - dest_change_id = change_id.into(); - copied_ids.push((imap_id.uid, assigned_uid)); + match self.server.commit_batch(batch).await { + Ok(assigned_ids) => { + dest_change_id = assigned_ids + .last_change_id(dest_account_id, SyncCollection::Email) + .into(); + copied_ids.push(( + imap_id.uid, + assigned_slot.map_or(0, |slot| assigned_ids.slot(slot)), + )); break; } Err(err) => { @@ -578,24 +648,10 @@ impl SessionData { error = Some((ResponseCode::OverQuota, "Mailbox quota exceeded")); continue; } - Err(CopyMessageError::NotFound) => { - continue; - } }; - if let Some(is_spam) = train_spam { - self.server - .add_account_spam_sample( - &mut train_batch, - src_account_id, - id, - is_spam, - self.session_id, - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - train_batch.commit_point(); - did_train = true; + if let Some(sample) = sample { + spam_samples.add_sample(sample); } if is_move { @@ -603,11 +659,23 @@ impl SessionData { } } - if did_train { + if !spam_samples.is_empty() { + let mut train_batch = BatchBuilder::new(); self.server - .commit_batch(train_batch) + .add_account_spam_samples( + &mut train_batch, + src_account_id, + spam_samples, + self.session_id, + ) .await .imap_ctx(&arguments.tag, trc::location!())?; + if !train_batch.is_empty() { + self.server + .commit_batch(train_batch) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + } } // Untag or delete emails @@ -620,6 +688,7 @@ impl SessionData { src_account_id, src_mailbox.id.mailbox_id, &destroy_ids, + &cache, &mut batch, ) .await @@ -634,7 +703,15 @@ impl SessionData { &arguments.tag, trc::location!(), ) - .await? + .await?; + self.server + .inner + .mark_cache_stale(src_account_id, SyncCollection::Email); + cache = self + .server + .get_cached_messages(src_account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; } } } @@ -695,10 +772,7 @@ impl SessionData { } // Prepare response - let uid_validity = self - .mailbox_state(&dest_mailbox) - .map(|m| m.uid_validity as u32) - .unwrap_or_default(); + let uid_validity = self.uid_validity(&dest_mailbox); let mut src_uids = Vec::with_capacity(copied_ids.len()); let mut dest_uids = Vec::with_capacity(copied_ids.len()); @@ -744,7 +818,12 @@ impl SessionData { if did_move { // Resynchronize source mailbox on a successful move - self.write_mailbox_changes(&src_mailbox, use_vanished) + let cache = self + .server + .get_cached_messages(src_mailbox.id.account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + self.write_mailbox_changes(&src_mailbox, &cache, use_vanished) .await .imap_ctx(&arguments.tag, trc::location!())?; } @@ -776,15 +855,11 @@ impl SessionData { &self, account_id: u32, id: u32, - ) -> trc::Result>> { + ) -> trc::Result> { if let Some(data) = self .server .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - id, - )) + .get_value::(ValueKey::archive(account_id, Collection::Email, id)) .await? { Ok(Some(data)) @@ -793,7 +868,7 @@ impl SessionData { Store(trc::StoreEvent::NotFound), AccountId = account_id, Collection = Collection::Email, - MessageId = id, + DocumentId = id, SpanId = self.session_id, Details = "Message not found" ); @@ -803,6 +878,49 @@ impl SessionData { } } +impl SessionData { + async fn flagged_containers( + &self, + account_id: u32, + cache: &MessageStoreCache, + ids: &[Resolved], + ) -> trc::Result)>> { + let flagged = ids + .iter() + .filter(|resolved| { + cache + .email_by_id(&resolved.id) + .is_some_and(|email| !cache.metadata_kinds(email).is_empty()) + }) + .map(|resolved| resolved.id) + .collect::(); + let mut containers = Vec::with_capacity(flagged.len() as usize); + self.server + .metadata_containers( + account_id, + Collection::Email, + &flagged, + |document_id, view, _| { + containers.push((document_id, EncodedMetadata::from_view(&view))); + Ok(true) + }, + ) + .await?; + containers.sort_unstable_by_key(|(document_id, _)| *document_id); + Ok(containers) + } +} + +fn take_container( + containers: &mut [(u32, Option)], + document_id: u32, +) -> Option { + let position = containers + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok()?; + containers.get_mut(position)?.1.take() +} + async fn retry_after_conflict( err: trc::Error, retries: &mut u32, diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index 08c5275b..0972ad4e 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -5,25 +5,30 @@ */ use crate::{ - core::{Session, SessionData}, + core::{MailboxId, Session, SessionData}, op::ImapContext, - spawn_op, }; use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; -use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; +use compact_str::format_compact; +use email::{cache::MessageCacheFetch, mailbox::role::RoleChange}; use imap_proto::{ Command, ResponseCode, StatusResponse, - protocol::{ObjectId, create::Arguments, list::Attribute}, + protocol::{create::Arguments, list::Attribute}, receiver::Request, }; use registry::schema::enums::{Permission, StorageQuota}; use std::time::Instant; use store::write::BatchBuilder; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; -use types::{acl::Acl, collection::Collection, id::Id, special_use::SpecialUse}; +use types::{acl::Acl, collection::Collection, special_use::SpecialUse}; impl Session { - pub async fn handle_create(&mut self, requests: Vec>) -> trc::Result<()> { + pub async fn handle_create( + &mut self, + requests: Vec>, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapCreate)?; @@ -31,23 +36,21 @@ impl Session { let is_utf8 = self.is_utf8; let is_objectid = self.is_objectid; - spawn_op!(data, { - for request in requests { - match request.parse_create(is_utf8) { - Ok(argument) => match data.create_folder(argument, is_objectid).await { - Ok(response) => { - data.write_bytes(response.into_bytes()).await?; - } - Err(error) => { - data.write_error(error).await?; - } - }, - Err(err) => data.write_error(err).await?, - } + for request in requests { + match request.parse_create(is_utf8) { + Ok(argument) => match data.create_folder(argument, is_objectid).await { + Ok(response) => { + data.write_bytes(response.into_bytes()).await?; + } + Err(error) => { + data.write_error(error).await?; + } + }, + Err(err) => data.write_error(err).await?, } + } - Ok(()) - }) + Ok(()) } } @@ -60,9 +63,12 @@ impl SessionData { let op_start = Instant::now(); // Refresh mailboxes - self.synchronize_mailboxes(false) + self.expire_folders_of(&arguments.mailbox_name); + let mut caches = self + .synchronize_mailboxes(false) .await - .imap_ctx(&arguments.tag, trc::location!())?; + .imap_ctx(&arguments.tag, trc::location!())? + .caches; // Validate mailbox name let params = self @@ -77,9 +83,8 @@ impl SessionData { .account(params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; - let mailbox_count = self - .server - .get_cached_messages(params.account_id) + let mailbox_count = caches + .fetch(&self.server, params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())? .mailboxes @@ -102,44 +107,45 @@ impl SessionData { } // Build batch - let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); - let mut create_ids = Vec::with_capacity(params.path.len()); - let mut next_document_id = self - .server - .store() - .assign_document_ids( - params.account_id, - Collection::Mailbox, - params.path.len() as u64, - ) - .await - .caused_by(trc::location!())?; + let parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut batch = BatchBuilder::new(); - for (pos, &path_item) in params.path.iter().enumerate() { - let mut mailbox = email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id); + let slots = batch.reserve_document_ids( + params.account_id, + Collection::Mailbox, + params.path.len() as u32, + ); + let mut parent_slot = None; + + for ((pos, &path_item), slot) in params.path.iter().enumerate().zip(slots.iter()) { + let mut mailbox = email::mailbox::Mailbox::new(path_item); + if parent_slot.is_none() { + mailbox.parent_id = parent_id; + } if pos == params.path.len() - 1 && let Some(mailbox_role) = arguments.mailbox_role.map(attr_to_role) { - mailbox.role = mailbox_role; + mailbox.set_role(mailbox_role); } - let mailbox_id = next_document_id; - next_document_id -= 1; + let builder = ObjectIndexBuilder::<(), _>::new().with_changes(mailbox); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .with_document(mailbox_id) - .custom(ObjectIndexBuilder::<(), _>::new().with_changes(mailbox)) + .create_document(slot) + .custom(builder.with_pending_id_opt(parent_slot)) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); - parent_id = mailbox_id + 1; - create_ids.push(mailbox_id); + parent_slot = Some(slot); } - self.server + let assigned_ids = self + .server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; + self.expire_folders(params.account_id); + let create_ids = assigned_ids.slots(slots).collect::>(); + let last_mailbox_id = create_ids.last().copied().unwrap_or_default(); trc::event!( Imap(trc::ImapEvent::CreateMailbox), @@ -156,11 +162,13 @@ impl SessionData { // Build response let response = StatusResponse::ok("Mailbox created.").with_tag(arguments.tag); Ok(if is_objectid { - response.with_code(ResponseCode::ObjectId(ObjectId { - mailbox_id: Some(Id::from(parent_id - 1)), - account_id: Some(Id::from(params.account_id)), - ..Default::default() - })) + response.with_code(ResponseCode::ObjectId(self.mailbox_object_id( + MailboxId { + account_id: params.account_id, + mailbox_id: last_mailbox_id, + }, + arguments.mailbox_name.trim_end_matches('/'), + ))) } else { response }) @@ -182,7 +190,7 @@ impl SessionData { if name.is_empty() { return Err(trc::ImapEvent::Error .into_err() - .details(format!("Invalid folder name '{mailbox_name}'.",))); + .details(format_compact!("Invalid folder name '{mailbox_name}'.",))); } // Build path @@ -244,7 +252,7 @@ impl SessionData { (account, full_path, prefix) } else { #[allow(clippy::unnecessary_literal_unwrap)] - return Err(trc::ImapEvent::Error.into_err().details(format!( + return Err(trc::ImapEvent::Error.into_err().details(format_compact!( "Shared account '{}' not found.", prefix.unwrap_or_default() ))); @@ -265,10 +273,10 @@ impl SessionData { }; // Locate parent mailbox - if account.mailbox_names.contains_key(&full_path) { + if account.id_by_name(&full_path).is_some() { return Err(trc::ImapEvent::Error .into_err() - .details(format!("Mailbox '{}' already exists.", full_path)) + .details(format_compact!("Mailbox '{}' already exists.", full_path)) .code(ResponseCode::AlreadyExists)); } @@ -278,7 +286,7 @@ impl SessionData { let mut create_path = Vec::with_capacity(path.len()); while !path.is_empty() { let mailbox_name: String = path.join("/"); - if let Some(&mailbox_id) = account.mailbox_names.get(&mailbox_name) { + if let Some(mailbox_id) = account.id_by_name(&mailbox_name) { parent_mailbox_id = mailbox_id.into(); parent_mailbox_name = mailbox_name.into(); break; @@ -302,7 +310,7 @@ impl SessionData { // Validate ACLs if let Some(parent_mailbox_id) = parent_mailbox_id { if !self - .check_mailbox_acl(account_id, parent_mailbox_id, Acl::CreateChild) + .check_mailbox_acl(None, account_id, parent_mailbox_id, Acl::CreateChild) .await? { return Err(trc::ImapEvent::Error @@ -329,24 +337,22 @@ impl SessionData { parent_mailbox_id, parent_mailbox_name, special_use: if let Some(mailbox_role) = mailbox_role { - // Make sure role is unique - let special_use = attr_to_role(mailbox_role); - if self - .server - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .mailbox_by_role(&special_use) - .is_some() - { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!( - "A mailbox with role '{}' already exists.", - special_use.as_str().unwrap_or_default() - )) - .code(ResponseCode::UseAttr)); + RoleChange::Create { + role: attr_to_role(mailbox_role), } + .validate( + self.server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .as_ref(), + ) + .map_err(|err| { + trc::ImapEvent::Error + .into_err() + .details(err.to_string()) + .code(ResponseCode::UseAttr) + })?; Some(mailbox_role) } else { None @@ -366,8 +372,7 @@ pub struct CreateParams<'x> { pub is_rename: bool, } -#[inline] -fn attr_to_role(attr: Attribute) -> SpecialUse { +pub(crate) fn attr_to_role(attr: Attribute) -> SpecialUse { match attr { Attribute::Archive => SpecialUse::Archive, Attribute::Drafts => SpecialUse::Drafts, diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index 60eb3567..38f1d459 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -5,10 +5,7 @@ */ use super::ImapContext; -use crate::{ - core::{Session, SessionData}, - spawn_op, -}; +use crate::core::{Session, SessionData}; use common::network::SessionStream; use email::mailbox::destroy::{MailboxDestroy, MailboxDestroyError}; use imap_proto::{ @@ -16,32 +13,35 @@ use imap_proto::{ }; use registry::schema::enums::Permission; use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; impl Session { - pub async fn handle_delete(&mut self, requests: Vec>) -> trc::Result<()> { + pub async fn handle_delete( + &mut self, + requests: Vec>, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapDelete)?; let data = self.state.session_data(); let is_utf8 = self.is_utf8; - spawn_op!(data, { - for request in requests { - match request.parse_delete(is_utf8) { - Ok(argument) => match data.delete_folder(argument).await { - Ok(response) => { - data.write_bytes(response.into_bytes()).await?; - } - Err(error) => { - data.write_error(error).await?; - } - }, - Err(response) => data.write_error(response).await?, - } + for request in requests { + match request.parse_delete(is_utf8) { + Ok(argument) => match data.delete_folder(argument).await { + Ok(response) => { + data.write_bytes(response.into_bytes()).await?; + } + Err(error) => { + data.write_error(error).await?; + } + }, + Err(response) => data.write_error(response).await?, } + } - Ok(()) - }) + Ok(()) } } @@ -50,6 +50,7 @@ impl SessionData { let op_start = Instant::now(); // Refresh mailboxes + self.expire_folders_of(&arguments.mailbox_name); self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -74,7 +75,7 @@ impl SessionData { if let Err(err) = self .server - .mailbox_destroy(account_id, mailbox_id, &access_token, true) + .mailbox_destroy(account_id, mailbox_id, &access_token, true, None) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -104,14 +105,7 @@ impl SessionData { .id(arguments.tag)); } - // Update mailbox cache - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == account_id { - account.mailbox_names.remove(&arguments.mailbox_name); - account.mailbox_state.remove(&mailbox_id); - break; - } - } + self.expire_folders(account_id); trc::event!( Imap(trc::ImapEvent::DeleteMailbox), diff --git a/crates/imap/src/op/enable.rs b/crates/imap/src/op/enable.rs index aa06c512..8c4fba10 100644 --- a/crates/imap/src/op/enable.rs +++ b/crates/imap/src/op/enable.rs @@ -6,13 +6,14 @@ use crate::core::Session; use common::network::SessionStream; +use compact_str::format_compact; use imap_proto::{ Command, StatusResponse, - protocol::{ImapResponse, ProtocolVersion, capability::Capability, enable}, + protocol::{ProtocolVersion, capability::Capability, enable}, receiver::Request, }; use registry::schema::enums::Permission; -use std::time::Instant; +use std::{mem::replace, time::Instant}; impl Session { pub async fn handle_enable(&mut self, request: Request) -> trc::Result<()> { @@ -27,35 +28,27 @@ impl Session { }; for capability in arguments.capabilities { - match capability { + let newly_enabled = match capability { Capability::IMAP4rev2 => { - self.version = ProtocolVersion::Rev2; self.is_utf8 = true; + replace(&mut self.version, ProtocolVersion::Rev2) != ProtocolVersion::Rev2 } Capability::IMAP4rev1 => { - self.version = ProtocolVersion::Rev1; - } - Capability::CondStore => { - self.is_condstore = true; + replace(&mut self.version, ProtocolVersion::Rev1) != ProtocolVersion::Rev1 } + Capability::CondStore => !replace(&mut self.is_condstore, true), Capability::QResync => { - self.is_qresync = true; self.is_condstore = true; + !replace(&mut self.is_qresync, true) } - Capability::Utf8Accept => { - self.is_utf8 = true; - } - Capability::ObjectIdPlus => { - self.is_objectid = true; - } - Capability::UidOnly => { - self.is_uidonly = true; - } - _ => { - continue; - } + Capability::Utf8Accept => !replace(&mut self.is_utf8, true), + Capability::ObjectIdPlus => !replace(&mut self.is_objectid, true), + Capability::UidOnly => !replace(&mut self.is_uidonly, true), + _ => false, + }; + if newly_enabled { + response.enabled.push(capability); } - response.enabled.push(capability); } trc::event!( @@ -64,7 +57,7 @@ impl Session { Details = response .enabled .iter() - .map(|c| trc::Value::from(format!("{c:?}"))) + .map(|c| trc::Value::from(format_compact!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); @@ -72,7 +65,7 @@ impl Session { self.write_bytes( StatusResponse::ok("ENABLE successful.") .with_tag(arguments.tag) - .serialize(response.serialize()), + .serialize_after(&response), ) .await } diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 80133b94..74a8f0c0 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -5,46 +5,61 @@ */ use super::{ImapContext, ToModSeq}; -use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; -use ahash::AHashMap; -use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; +use crate::core::{Resolved, SavedSearch, SelectedMailbox, Session, SessionData}; +use common::{MessageStoreCache, network::SessionStream, storage::index::ObjectIndexBuilder}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::{delete::EmailDeletion, metadata::MessageData}, + message::{delete::EmailDeletion, messagedata::EmailMessageData}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, receiver::{Request, Token}, }; -use registry::schema::{ - enums::{IndexDocumentType, Permission}, - structs::{Task, TaskIndexDocument, TaskStatus}, +use rand::RngExt; +use registry::schema::enums::Permission; +use std::{ + ops::RangeInclusive, + sync::Arc, + time::{Duration, Instant}, }; -use std::{sync::Arc, time::Instant}; -use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ + roaring::RoaringBitmap, + write::{BatchBuilder, SearchIndex}, +}; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; use types::{ acl::Acl, - collection::{Collection, VanishedCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, keyword::Keyword, }; +const MAX_EXPUNGE_RETRIES: u32 = 3; +const EXPUNGE_RETRY_BACKOFF_MS: RangeInclusive = 1..=15; + impl Session { pub async fn handle_expunge( &mut self, request: Request, is_uid: bool, + _permit: Option, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapExpunge)?; let op_start = Instant::now(); let (data, mailbox) = self.state.select_data(); + let cache = data + .server + .get_cached_messages(mailbox.id.account_id) + .await + .imap_ctx(&request.tag, trc::location!())?; // Validate ACL if !data .check_mailbox_acl( + Some(&cache), mailbox.id.account_id, mailbox.id.mailbox_id, Acl::RemoveItems, @@ -62,6 +77,10 @@ impl Session { .id(request.tag)); } + data.sync_view(&mailbox, &cache, None) + .await + .imap_ctx(&request.tag, trc::location!())?; + // Parse sequence to operate on let sequence = match request.tokens.into_iter().next() { Some(Token::Argument(value)) if is_uid => { @@ -74,7 +93,7 @@ impl Session { })?; Some( mailbox - .sequence_to_ids(&sequence, true) + .resolve(&sequence, true) .await .map_err(|err| err.id(request.tag.clone()))?, ) @@ -92,7 +111,7 @@ impl Session { // Expunge let limited_uid = data - .expunge(mailbox.clone(), sequence, message_limit, op_start) + .expunge(mailbox.clone(), cache, sequence, message_limit, op_start) .await .imap_ctx(&request.tag, trc::location!())?; @@ -100,8 +119,13 @@ impl Session { *mailbox.saved_search.lock() = SavedSearch::None; // Synchronize messages + let cache = data + .server + .get_cached_messages(mailbox.id.account_id) + .await + .imap_ctx(&request.tag, trc::location!())?; let modseq = data - .write_mailbox_changes(&mailbox, self.is_qresync || self.is_uidonly) + .write_mailbox_changes(&mailbox, &cache, self.is_qresync || self.is_uidonly) .await .imap_ctx(&request.tag, trc::location!())?; let mut response = @@ -135,79 +159,105 @@ impl SessionData { pub async fn expunge( &self, mailbox: Arc, - sequence: Option>, + mut cache: Arc, + sequence: Option>, message_limit: u32, op_start: Instant, ) -> trc::Result> { - // Obtain message ids let account_id = mailbox.id.account_id; - let mut deleted_ids = RoaringBitmap::from_iter( - self.server - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .in_mailbox_with_keyword(mailbox.id.mailbox_id, &Keyword::Deleted) - .map(|m| m.document_id), - ); + let sequence = sequence.map(|sequence| { + RoaringBitmap::from_iter(sequence.into_iter().map(|resolved| resolved.id)) + }); + let mut retries = 0; - // Filter by sequence - if let Some(sequence) = &sequence { - deleted_ids &= RoaringBitmap::from_iter(sequence.keys()); - } + loop { + // Obtain message ids + let mut deleted_ids = RoaringBitmap::from_iter( + cache + .in_mailbox_with_keyword(mailbox.id.mailbox_id, &Keyword::Deleted) + .map(|m| m.document_id()), + ); - // RFC 9738 requires the highest UIDs to be processed first when truncating. - // Only messages the session has a UID for can be ordered, so the count that - // decides whether to truncate has to come from that same set. - let mut limited_uid = None; - if deleted_ids.len() > message_limit as u64 { - let mut uids = { - let state = mailbox.state.lock(); - deleted_ids - .iter() - .filter_map(|id| state.id_to_imap.get(&id).map(|imap_id| (imap_id.uid, id))) - .collect::>() - }; + // Filter by sequence + if let Some(sequence) = &sequence { + deleted_ids &= sequence; + } - if uids.len() > message_limit as usize { - let cutoff = uids.len() - message_limit as usize; - let (below, lowest, _) = uids.select_nth_unstable(cutoff); - limited_uid = Some(lowest.0); - for (_, id) in below { - deleted_ids.remove(*id); + // RFC 9738 requires the highest UIDs to be processed first when truncating. + let mut limited_uid = None; + if deleted_ids.len() > message_limit as u64 { + let mut uids = { + let view = mailbox.view.lock(); + deleted_ids + .iter() + .filter_map(|id| view.map_result(id).map(|resolved| (resolved.uid, id))) + .collect::>() + }; + + if uids.len() > message_limit as usize { + let cutoff = uids.len() - message_limit as usize; + let (below, lowest, _) = uids.select_nth_unstable(cutoff); + limited_uid = Some(lowest.0); + for (_, id) in below { + deleted_ids.remove(*id); + } } } - } - // Delete ids - let mut batch = BatchBuilder::new(); - let (fully_deleted, thread_ids) = self - .email_untag_or_delete(account_id, mailbox.id.mailbox_id, &deleted_ids, &mut batch) - .await - .caused_by(trc::location!())?; - self.server - .log_emptied_threads(account_id, &mut batch, thread_ids, &fully_deleted) - .await - .caused_by(trc::location!())?; - - trc::event!( - Imap(trc::ImapEvent::Expunge), - SpanId = self.session_id, - AccountId = account_id, - MailboxId = mailbox.id.mailbox_id, - DocumentId = deleted_ids.iter().map(trc::Value::from).collect::>(), - Elapsed = op_start.elapsed() - ); - - // Write changes on source account - if !batch.is_empty() { + // Delete ids + let mut batch = BatchBuilder::new(); + let (fully_deleted, thread_ids) = self + .email_untag_or_delete( + account_id, + mailbox.id.mailbox_id, + &deleted_ids, + &cache, + &mut batch, + ) + .await + .caused_by(trc::location!())?; self.server - .commit_batch(batch) + .log_emptied_threads(account_id, &mut batch, thread_ids, &fully_deleted) .await .caused_by(trc::location!())?; - self.server.notify_task_queue(); - } - Ok(limited_uid) + // Write changes on source account + if !batch.is_empty() + && let Err(err) = self.server.commit_batch(batch).await + { + if !err.is_assertion_failure() { + return Err(err.caused_by(trc::location!())); + } else if retries == MAX_EXPUNGE_RETRIES { + return Err(trc::ImapEvent::Error + .into_err() + .details("Some messages were modified by another process.")); + } + + retries += 1; + let backoff = rand::rng().random_range(EXPUNGE_RETRY_BACKOFF_MS); + tokio::time::sleep(Duration::from_millis(backoff)).await; + self.server + .inner + .mark_cache_stale(account_id, SyncCollection::Email); + cache = self + .server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + continue; + } + + trc::event!( + Imap(trc::ImapEvent::Expunge), + SpanId = self.session_id, + AccountId = account_id, + MailboxId = mailbox.id.mailbox_id, + DocumentId = deleted_ids.iter().map(trc::Value::from).collect::>(), + Elapsed = op_start.elapsed() + ); + + return Ok(limited_uid); + } } pub async fn email_untag_or_delete( @@ -215,8 +265,28 @@ impl SessionData { account_id: u32, mailbox_id: u32, deleted_ids: &RoaringBitmap, + cache: &MessageStoreCache, batch: &mut BatchBuilder, ) -> trc::Result<(RoaringBitmap, RoaringBitmap)> { + let flagged_ids = deleted_ids + .iter() + .filter(|document_id| { + cache.email_by_id(document_id).is_some_and(|message| { + !cache.metadata_kinds(message).is_empty() + && matches!(message.mailboxes(), [only] if only.mailbox_id == mailbox_id) + }) + }) + .collect::(); + let containers = self + .server + .preload_container_cleanup( + self.access_token.changed_by(), + account_id, + Collection::Email, + &flagged_ids, + ) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::Email); @@ -224,62 +294,48 @@ impl SessionData { let mut fully_deleted = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); self.server - .archives( - account_id, - Collection::Email, - deleted_ids, - |document_id, data_| { - let metadata = data_ - .to_unarchived::() - .caused_by(trc::location!())?; + .message_datas(account_id, deleted_ids, |document_id, metadata| { + if let Some(message_uid) = metadata.message_uid(mailbox_id) { + // Add vanished items + batch.with_document(document_id); + batch.log_vanished_item(VanishedCollection::Email, (mailbox_id, message_uid)); - if let Some(message_uid) = metadata.inner.message_uid(mailbox_id) { - // Add vanished items - batch.with_document(document_id); - batch.log_vanished_item( - VanishedCollection::Email, - (mailbox_id, message_uid), - ); - - if metadata.inner.mailboxes.len() == 1 { - // Delete message - fully_deleted.insert(document_id); - thread_ids.insert(metadata.inner.thread_id.to_native()); - batch - .custom( - ObjectIndexBuilder::<_, ()>::new() - .with_changed_by(self.access_token.account_tenant_ids()) - .with_current(metadata), - ) - .caused_by(trc::location!())? - .schedule_task(Task::UnindexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::now(), - })) - .commit_point(); - } else { - // Untag message from this mailbox and remove Deleted flag - let mut new_metadata = metadata.inner.to_builder(); - new_metadata.remove_mailbox(mailbox_id); - new_metadata.remove_keyword(&Keyword::Deleted); - - // Write changes - batch - .custom( - ObjectIndexBuilder::new() - .with_current(metadata) - .with_changes(new_metadata.seal()), - ) - .caused_by(trc::location!())? - .commit_point(); + if metadata.mailboxes.len() == 1 { + // Delete message + fully_deleted.insert(document_id); + thread_ids.insert(metadata.thread_id); + if !metadata.metadata_kinds().is_empty() { + containers.release_or_assert_absent(batch, account_id, document_id); } - } + batch + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_changed_by(self.access_token.changed_by()) + .with_current(metadata), + ) + .caused_by(trc::location!())? + .queue_document_unindex(SearchIndex::Email, account_id, document_id) + .commit_point(); + } else { + // Untag message from this mailbox and remove Deleted flag + let mut new_metadata = metadata.clone(); + new_metadata.remove_mailbox(mailbox_id); + new_metadata.remove_keyword(&Keyword::Deleted); - Ok(true) - }, - ) + // Write changes + batch + .custom( + ObjectIndexBuilder::new() + .with_current(metadata) + .with_changes(new_metadata), + ) + .caused_by(trc::location!())? + .commit_point(); + } + } + + Ok(true) + }) .await .caused_by(trc::location!())?; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs deleted file mode 100644 index 0bd27203..00000000 --- a/crates/imap/src/op/fetch.rs +++ /dev/null @@ -1,1279 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::{FromModSeq, ImapContext}; -use crate::{ - core::{SelectedMailbox, Session, SessionData}, - spawn_op, -}; -use ahash::AHashMap; -use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; -use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::metadata::{ - ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, - ArchivedMetadataPartType, DecodedParts, MESSAGE_RECEIVED_MASK, MessageData, - MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM, - }, -}; -use imap_proto::{ - Command, ResponseCode, ResponseType, StatusResponse, - parser::PushUnique, - protocol::{ - Flag, ObjectId, - expunge::Vanished, - fetch::{ - self, Arguments, Attribute, BodyContents, BodyPart, BodyPartExtension, BodyPartFields, - DataItem, Envelope, FetchItem, Section, - }, - }, - receiver::Request, -}; -use registry::schema::enums::Permission; -use std::{borrow::Cow, sync::Arc, time::Instant}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; -use store::{ - query::log::{Change, Query}, - rkyv::rend::u16_le, - write::BatchBuilder, -}; -use types::{ - acl::Acl, - collection::{Collection, SyncCollection, VanishedCollection}, - field::EmailField, - id::Id, - keyword::Keyword, -}; -use utils::chained_bytes::{ChainedBytes, SliceRange}; - -impl Session { - pub async fn handle_fetch(&mut self, requests: Vec>) -> trc::Result<()> { - // Validate access - self.assert_has_permission(Permission::ImapFetch)?; - - let (data, mailbox) = self.state.select_data(); - let is_qresync = self.is_qresync; - let is_uidonly = self.is_uidonly; - let is_utf8 = self.is_utf8; - let message_limit = self.server.core.imap.max_messages_per_command; - - let mut ops = Vec::with_capacity(requests.len()); - let mut activate_objectid = false; - - for request in requests { - let is_uid = matches!(request.command, Command::Fetch(true)); - match request.parse_fetch() { - Ok(arguments) => { - let enabled_condstore = if !self.is_condstore - && arguments.changed_since.is_some() - || arguments.attributes.contains(&Attribute::ModSeq) - { - self.is_condstore = true; - true - } else { - false - }; - - if arguments.attributes.contains(&Attribute::ObjectId) { - activate_objectid = true; - } - - ops.push(Ok((is_uid, enabled_condstore, arguments))); - } - Err(err) => { - ops.push(Err(err)); - } - } - } - - if activate_objectid && let Some(enabled) = self.activate_objectid() { - self.write_bytes(enabled).await?; - } - - spawn_op!(data, { - for op in ops { - match op { - Ok((is_uid, enabled_condstore, arguments)) => { - match data - .fetch( - arguments, - mailbox.clone(), - is_uid, - is_qresync, - is_uidonly, - enabled_condstore, - is_utf8, - message_limit, - Instant::now(), - ) - .await - { - Ok(response) => { - data.write_bytes(response.into_bytes()).await?; - } - Err(err) => data.write_error(err).await?, - } - } - Err(err) => data.write_error(err).await?, - } - } - - Ok(()) - }) - } -} - -impl SessionData { - #[allow(clippy::too_many_arguments)] - pub async fn fetch( - &self, - mut arguments: Arguments, - mailbox: Arc, - is_uid: bool, - is_qresync: bool, - is_uidonly: bool, - enabled_condstore: bool, - is_utf8: bool, - message_limit: u32, - op_start: Instant, - ) -> trc::Result { - // Validate VANISHED parameter - if arguments.include_vanished { - if !is_qresync { - return Err(trc::ImapEvent::Error - .into_err() - .details("Enable QRESYNC first to use the VANISHED parameter.") - .ctx(trc::Key::Type, ResponseType::Bad) - .id(arguments.tag)); - } else if !is_uid { - return Err(trc::ImapEvent::Error - .into_err() - .details("VANISHED parameter is only available for UID FETCH.") - .ctx(trc::Key::Type, ResponseType::Bad) - .id(arguments.tag)); - } - } - - // Resync messages if needed - let account_id = mailbox.id.account_id; - let mut modseq = self - .synchronize_messages(&mailbox) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - // Convert IMAP ids to JMAP ids. - let mut ids = mailbox - .sequence_to_ids(&arguments.sequence_set, is_uid) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - // Convert state to modseq - if let Some(changed_since) = arguments.changed_since { - // Obtain changes since the modseq. - let changelog = self - .server - .store() - .changes( - account_id, - SyncCollection::Email.into(), - Query::from_modseq(changed_since), - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - // Process changes - let mut changed_ids = AHashMap::new(); - let mut has_vanished = false; - - for change in changelog.changes { - match change { - Change::InsertItem(id) | Change::UpdateItem(id) => { - let id = (id & u32::MAX as u64) as u32; - if let Some(uid) = ids.get(&id) { - changed_ids.insert(id, *uid); - } - if !has_vanished { - has_vanished = matches!(change, Change::UpdateItem(_)); - } - } - Change::DeleteItem(_) => { - has_vanished = true; - } - _ => (), - } - } - - // Send vanished UIDs - if arguments.include_vanished && has_vanished { - // Add to vanished all known destroyed Ids - let vanished = self - .server - .store() - .vanished::<(u32, u32)>( - account_id, - VanishedCollection::Email.into(), - Query::from_modseq(changed_since), - ) - .await - .imap_ctx(&arguments.tag, trc::location!())? - .into_iter() - .filter_map(|(mailbox_id, uid)| { - if mailbox.id.mailbox_id == mailbox_id { - Some(uid) - } else { - None - } - }) - .collect::>(); - - if !vanished.is_empty() { - let mut buf = Vec::with_capacity(vanished.len() * 3); - Vanished { - earlier: true, - ids: vanished, - } - .serialize(&mut buf); - self.write_bytes(buf).await?; - } - } - - // Filter out ids without changes - if changed_ids.is_empty() { - // Condstore was just enabled, return highest modseq. - if enabled_condstore { - self.write_bytes( - StatusResponse::ok("Highest Modseq") - .with_code(ResponseCode::highest_modseq(modseq)) - .into_bytes(), - ) - .await?; - } - - trc::event!( - Imap(trc::ImapEvent::Fetch), - SpanId = self.session_id, - AccountId = account_id, - MailboxId = mailbox.id.mailbox_id, - Elapsed = op_start.elapsed() - ); - - return Ok( - StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag) - ); - } - ids = changed_ids; - arguments.attributes.push_unique(Attribute::ModSeq); - } - - // Build properties list - let mut set_seen_flags = false; - let mut needs_blobs = false; - - for attribute in &arguments.attributes { - match attribute { - Attribute::BodySection { sections, .. } - if sections.first().is_some_and(|s| { - matches!(s, Section::Header | Section::HeaderFields { .. }) - }) => {} - Attribute::Body | Attribute::BodyStructure | Attribute::BinarySize { .. } => { - /* - Note that this did not result in \Seen being set, because - RFC822.HEADER response data occurs as a result of a FETCH - of RFC822.HEADER. BODY[HEADER] response data occurs as a - result of a FETCH of BODY[HEADER] (which sets \Seen) or - BODY.PEEK[HEADER] (which does not set \Seen). - */ - needs_blobs = true; - } - Attribute::BodySection { peek, .. } | Attribute::Binary { peek, .. } => { - if mailbox.is_select && !*peek { - set_seen_flags = true; - } - needs_blobs = true; - } - Attribute::Rfc822Text | Attribute::Rfc822 => { - if mailbox.is_select { - set_seen_flags = true; - } - needs_blobs = true; - } - _ => (), - } - } - - if set_seen_flags - && !self - .check_mailbox_acl( - mailbox.id.account_id, - mailbox.id.mailbox_id, - Acl::ModifyItems, - ) - .await - .imap_ctx(&arguments.tag, trc::location!())? - { - set_seen_flags = false; - } - - if is_uid { - if arguments.attributes.is_empty() { - arguments.attributes.push(Attribute::Flags); - } else if !arguments.attributes.contains(&Attribute::Uid) { - arguments.attributes.insert(0, Attribute::Uid); - } - } - - // Process each message - let mut batch = BatchBuilder::new(); - let mut ids = ids - .into_iter() - .map(|(id, imap_id)| (imap_id.seqnum, imap_id.uid, id)) - .collect::>(); - ids.sort_unstable_by_key(|(seqnum, _, _)| *seqnum); - - // RFC 9738 requires the highest UIDs to be processed first when truncating - let message_limit = message_limit as usize; - let limited_uid = if ids.len() > message_limit { - ids.drain(..ids.len() - message_limit); - ids.first().map(|(_, uid, _)| *uid) - } else { - None - }; - - let fetched_ids = ids - .iter() - .map(|id| trc::Value::from(id.2)) - .collect::>(); - let message_cache = self - .server - .get_cached_messages(account_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - for (seqnum, uid, id) in ids { - // Obtain attributes and keywords - let (metadata_, data) = if let (Some(email), Some(data)) = ( - self.server - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - id, - EmailField::Metadata, - )) - .await - .imap_ctx(&arguments.tag, trc::location!())?, - message_cache.email_by_id(&id), - ) { - (email, data) - } else { - trc::event!( - Store(trc::StoreEvent::NotFound), - AccountId = account_id, - DocumentId = id, - Collection = Collection::Email, - Details = "Message metadata not found.", - CausedBy = trc::location!(), - ); - continue; - }; - let metadata = metadata_ - .unarchive::() - .imap_ctx(&arguments.tag, trc::location!())?; - let raw_body; - - // Fetch and parse blob - let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); - if needs_blobs { - // Retrieve raw message if needed - raw_body = self - .server - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - if let Some(raw_body) = &raw_body { - raw_message.append( - raw_body - .get(metadata.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - } else { - trc::event!( - Store(trc::StoreEvent::NotFound), - AccountId = account_id, - DocumentId = id, - Collection = Collection::Email, - BlobId = metadata.blob_hash.0.as_slice(), - Details = "Blob not found.", - CausedBy = trc::location!(), - ); - - continue; - } - } - - let message = &metadata.contents[0]; - let decoded = metadata.decode_contents(raw_message.clone()); - - // Build response - let mut items = Vec::with_capacity(arguments.attributes.len()); - let set_seen_flag = set_seen_flags && !message_cache.has_keyword(data, &Keyword::Seen); - - for attribute in &arguments.attributes { - match attribute { - Attribute::Envelope => { - items.push(DataItem::Envelope { - envelope: message.envelope(), - }); - } - Attribute::Flags => { - let mut flags = message_cache - .expand_keywords(data) - .map(Flag::from) - .collect::>(); - if set_seen_flag { - flags.push(Flag::Seen); - } - items.push(DataItem::Flags { flags }); - } - Attribute::InternalDate => { - items.push(DataItem::InternalDate { - date: (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, - }); - } - Attribute::Preview { .. } => { - items.push(DataItem::Preview { - contents: if !metadata.preview.is_empty() { - Some(metadata.preview.as_bytes().into()) - } else { - None - }, - }); - } - Attribute::Rfc822Size => { - items.push(DataItem::Rfc822Size { - size: data.size as usize, - }); - } - Attribute::Uid => { - items.push(DataItem::Uid { uid }); - } - Attribute::Rfc822 => { - items.push(DataItem::Rfc822 { - contents: raw_message.get_full_range(), - }); - } - Attribute::Rfc822Header => { - let contents = raw_message.get_slice_range( - 0..u32::from(metadata.root_part().offset_body) as usize, - ); - - if contents != SliceRange::None { - items.push(DataItem::Rfc822Header { contents }); - } - } - Attribute::Rfc822Text => { - items.push(DataItem::Rfc822Text { - contents: raw_message.get_full_range(), - }); - } - Attribute::Body => { - items.push(DataItem::Body { - part: metadata.body_structure(&decoded, false), - }); - } - Attribute::BodyStructure => { - items.push(DataItem::BodyStructure { - part: metadata.body_structure(&decoded, true), - }); - } - Attribute::BodySection { - sections, partial, .. - } => { - if let Some(contents) = metadata.body_section(&decoded, sections, *partial) - { - items.push(DataItem::BodySection { - sections: sections.to_vec(), - origin_octet: partial.map(|(start, _)| start), - contents, - }); - } - } - - Attribute::Binary { - sections, partial, .. - } => match metadata.binary(&decoded, sections, *partial) { - Ok(Some(contents)) => { - items.push(DataItem::Binary { - sections: sections.to_vec(), - offset: partial.map(|(start, _)| start), - contents, - }); - } - Err(_) => { - self.write_error( - trc::ImapEvent::Error - .into_err() - .details(format!( - "Failed to decode part {} of message {}.", - sections - .iter() - .map(|s| s.to_string()) - .collect::>() - .join("."), - if is_uid { uid } else { seqnum } - )) - .code(ResponseCode::UnknownCte), - ) - .await?; - continue; - } - _ => (), - }, - Attribute::BinarySize { sections } => { - if let Some(size) = metadata.binary_size(&decoded, sections) { - items.push(DataItem::BinarySize { - sections: sections.to_vec(), - size, - }); - } - } - Attribute::ModSeq => { - items.push(DataItem::ModSeq { - modseq: data.change_id + 1, - }); - } - Attribute::ObjectId => { - items.push(DataItem::ObjectId(ObjectId { - email_id: Some(Id::from_parts(data.thread_id, id)), - thread_id: Some(Id::from(data.thread_id)), - ..Default::default() - })); - } - } - } - - // Add flags to the response if the message was unseen - if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { - let mut flags = message_cache - .expand_keywords(data) - .map(Flag::from) - .collect::>(); - flags.push(Flag::Seen); - items.push(DataItem::Flags { flags }); - } - - // Serialize fetch item - let mut buf = Vec::with_capacity(128); - FetchItem { - id: if is_uidonly { uid } else { seqnum }, - is_uidonly, - items, - } - .serialize(&mut buf, is_utf8); - self.write_bytes(buf).await?; - - // Add to set flags - if set_seen_flag - && let Some(data_) = self - .server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - id, - )) - .await - .imap_ctx(&arguments.tag, trc::location!())? - { - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; - let mut new_data = data.inner.to_builder(); - new_data.keywords.push(Keyword::Seen); - - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), - ) - .imap_ctx(&arguments.tag, trc::location!())? - .commit_point(); - } - } - - // Set Seen ids - if !batch.is_empty() { - match self - .server - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .imap_ctx(&arguments.tag, trc::location!()) - { - Ok(change_id) => { - modseq = change_id; - } - Err(err) => { - if !err.is_assertion_failure() { - return Err(err.id(arguments.tag)); - } - } - } - } - - trc::event!( - Imap(trc::ImapEvent::Fetch), - SpanId = self.session_id, - AccountId = account_id, - MailboxId = mailbox.id.mailbox_id, - DocumentId = fetched_ids, - Details = arguments - .attributes - .iter() - .map(|c| trc::Value::from(format!("{c:?}"))) - .collect::>(), - Elapsed = op_start.elapsed() - ); - - // Condstore was enabled with this command - if enabled_condstore { - self.write_bytes( - StatusResponse::ok("Highest Modseq") - .with_code(ResponseCode::highest_modseq(modseq)) - .into_bytes(), - ) - .await?; - } - - let response = StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag); - Ok(match limited_uid { - Some(uid) => response.with_code(ResponseCode::MessageLimit { - limit: message_limit as u32, - uid: uid.into(), - }), - None => response, - }) - } -} - -#[allow(clippy::result_unit_err)] -pub trait AsImapDataItem { - fn body_structure(&'_ self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart<'_>; - fn body_section<'x>( - &self, - decoded: &'x DecodedParts<'x>, - sections: &[Section], - partial: Option<(u32, u32)>, - ) -> Option>; - fn binary<'x>( - &self, - decoded: &'x DecodedParts<'x>, - sections: &[u32], - partial: Option<(u32, u32)>, - ) -> Result>, ()>; - fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option; -} - -#[allow(clippy::result_unit_err)] -pub trait AsImapDataItemPart { - fn as_body_part( - &'_ self, - decoded: &DecodedParts<'_>, - message_id: usize, - part_id: usize, - is_extended: bool, - ) -> BodyPart<'_>; - - fn envelope(&'_ self) -> Envelope<'_>; -} - -impl AsImapDataItemPart for ArchivedMessageMetadataContents { - fn as_body_part( - &'_ self, - decoded: &DecodedParts<'_>, - message_id: usize, - part_id: usize, - is_extended: bool, - ) -> BodyPart<'_> { - let part = &self.parts[part_id]; - let body = decoded.raw_message_section(message_id, part.body_to_end()); - let (is_multipart, is_text) = match &part.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => (false, true), - ArchivedMetadataPartType::Multipart(_) => (true, false), - _ => (false, false), - }; - let content_type = part - .header_value(&MetadataHeaderName::ContentType) - .and_then(|ct| ct.as_content_type()); - - let mut body_md5 = None; - let mut extension = BodyPartExtension::default(); - let mut fields = BodyPartFields::default(); - - if !is_multipart || is_extended { - fields.body_parameters = content_type - .as_ref() - .map(|ct| { - ct.attributes - .iter() - .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) - .collect::>() - }) - .filter(|p| !p.is_empty()) - } - - if !is_multipart { - fields.body_subtype = content_type - .as_ref() - .and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into())); - - fields.body_id = part - .header_value(&MetadataHeaderName::ContentId) - .and_then(|id| id.as_text().map(|id| format!("<{}>", id).into())); - - fields.body_description = part - .header_value(&MetadataHeaderName::ContentDescription) - .and_then(|ct| ct.as_text().map(|ct| ct.into())); - - fields.body_encoding = part - .header_value(&MetadataHeaderName::ContentTransferEncoding) - .and_then(|ct| ct.as_text().map(|ct| ct.into())); - - fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0); - - if is_text { - if fields.body_subtype.is_none() { - fields.body_subtype = Some("plain".into()); - } - if fields.body_encoding.is_none() { - fields.body_encoding = Some("7bit".into()); - } - if fields.body_parameters.is_none() { - fields.body_parameters = Some(vec![("charset".into(), "us-ascii".into())]); - } - } - } - - if is_extended { - if !is_multipart { - body_md5 = body - .as_ref() - .map(|b| format!("{:x}", md5::compute(b)).into()); - } - - extension.body_disposition = part - .header_value(&MetadataHeaderName::ContentDisposition) - .and_then(|cd| cd.as_content_type()) - .map(|cd| { - ( - cd.c_type.as_ref().into(), - cd.attributes - .iter() - .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) - .collect::>(), - ) - }); - - extension.body_language = part - .header_value(&MetadataHeaderName::ContentLanguage) - .and_then(|hv| { - hv.as_text_list() - .map(|list| list.iter().map(|item| item.as_ref().into()).collect()) - }); - - extension.body_location = part - .header_value(&MetadataHeaderName::ContentLocation) - .and_then(|ct| ct.as_text().map(|ct| ct.into())); - } - - match &part.body { - ArchivedMetadataPartType::Multipart(parts) => BodyPart::Multipart { - body_parts: Vec::with_capacity(parts.len()), - body_subtype: content_type - .as_ref() - .and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into())) - .unwrap_or_else(|| "".into()), - body_parameters: fields.body_parameters, - extension, - }, - ArchivedMetadataPartType::Message(_) => BodyPart::Message { - fields, - envelope: None, - body: None, - body_size_lines: 0, - body_md5, - extension, - }, - _ => { - if is_text { - BodyPart::Text { - fields, - body_size_lines: body - .as_ref() - .map(|b| b.iter().filter(|&&ch| ch == b'\n').count()) - .unwrap_or(0), - body_md5, - extension, - } - } else { - BodyPart::Basic { - body_type: content_type - .as_ref() - .map(|ct| Cow::from(ct.c_type.as_ref())), - fields, - body_md5, - extension, - } - } - } - } - } - - fn envelope(&'_ self) -> Envelope<'_> { - let headers = self.root_part(); - Envelope { - date: headers.date(), - subject: headers.subject().map(|s| s.into()), - from: headers - .header_values(&MetadataHeaderName::From) - .flat_map(|a| a.as_imap_address()) - .collect(), - sender: headers - .header_values(&MetadataHeaderName::Sender) - .flat_map(|a| a.as_imap_address()) - .collect(), - reply_to: headers - .header_values(&MetadataHeaderName::ReplyTo) - .flat_map(|a| a.as_imap_address()) - .collect(), - to: headers - .header_values(&MetadataHeaderName::To) - .flat_map(|a| a.as_imap_address()) - .collect(), - cc: headers - .header_values(&MetadataHeaderName::Cc) - .flat_map(|a| a.as_imap_address()) - .collect(), - bcc: headers - .header_values(&MetadataHeaderName::Bcc) - .flat_map(|a| a.as_imap_address()) - .collect(), - in_reply_to: headers.in_reply_to().as_text_list().map(|list| { - let mut irt = String::with_capacity(list.len() * 10); - for (pos, l) in list.iter().enumerate() { - if pos > 0 { - irt.push(' '); - } - irt.push('<'); - irt.push_str(l.as_ref()); - irt.push('>'); - } - irt.into() - }), - message_id: headers.message_id().map(|id| format!("<{}>", id).into()), - } - } -} - -impl AsImapDataItem for ArchivedMessageMetadata { - fn body_structure(&'_ self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart<'_> { - let mut stack = Vec::new(); - let base_part = [u16_le::from_native(0)]; - let mut parts = base_part.as_slice().iter(); - let mut message = &self.contents[0]; - let mut root_part = None; - let mut message_id = 0; - - loop { - while let Some(part_id) = parts.next() { - let part_id = u16::from(part_id) as usize; - let mut part = message.as_body_part(decoded, message_id, part_id, is_extended); - - match &message.parts[part_id].body { - ArchivedMetadataPartType::Message(nested_message_id) => { - let nested_message = self.message_id(*nested_message_id); - part.set_envelope(nested_message.envelope()); - if let Some(root_part) = root_part { - if stack.len() == 10_000 { - debug_assert!(false, "Too much nesting in message metadata"); - return root_part; - } - stack.push((root_part, parts, (message, message_id).into())); - } - root_part = part.into(); - parts = base_part.as_slice().iter(); - message = nested_message; - message_id = u16::from(*nested_message_id) as usize; - continue; - } - ArchivedMetadataPartType::Multipart(subparts) => { - if let Some(root_part) = root_part { - if stack.len() == 10_000 { - debug_assert!(false, "Too much nesting in message metadata"); - return root_part; - } - stack.push((root_part, parts, None)); - } - root_part = part.into(); - parts = subparts.iter(); - continue; - } - _ => (), - } - if let Some(root_part) = &mut root_part { - root_part.add_part(part); - } else { - return part; - } - } - if let Some((mut prev_root_part, prev_parts, prev_message)) = stack.pop() { - if let Some((prev_message, prev_message_id)) = prev_message { - message = prev_message; - message_id = prev_message_id; - } - - prev_root_part.add_part(root_part.unwrap()); - parts = prev_parts; - root_part = prev_root_part.into(); - } else { - break; - } - } - - root_part.unwrap() - } - - fn body_section<'x>( - &self, - decoded: &'x DecodedParts<'x>, - sections: &[Section], - partial: Option<(u32, u32)>, - ) -> Option> { - let mut part = self.root_part(); - if sections.is_empty() { - return Some(get_cow_partial_bytes( - decoded.raw_message_section(0, part.header_to_end())?, - partial, - )); - } - - let mut message = &self.contents[0]; - let mut message_id = 0; - let mut sections_iter = sections.iter().enumerate().peekable(); - - while let Some((section_num, section)) = sections_iter.next() { - match section { - Section::Part { num } => { - part = if let Some(sub_part_ids) = part.sub_parts() { - sub_part_ids - .as_ref() - .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) - } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) - { - Some(part) - } else { - None - }?; - - if let ArchivedMetadataPartType::Message(nested_message_id) = &part.body - && let Some(( - _, - Section::Part { .. } - | Section::Header - | Section::HeaderFields { .. } - | Section::Text, - )) = sections_iter.peek() - { - message = self.message_id(*nested_message_id); - part = message.root_part(); - message_id = u16::from(nested_message_id) as usize; - } - } - Section::Header => { - return Some(get_cow_partial_bytes( - decoded.raw_message_section(message_id, part.header_to_body())?, - partial, - )); - } - Section::HeaderFields { not, fields } => { - let mut headers = Vec::with_capacity( - u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) - as usize, - ); - for header in part.headers.iter() { - let header_name = header.name.as_str(); - if fields.iter().any(|f| header_name.eq_ignore_ascii_case(f)) != *not { - headers.extend_from_slice(header_name.as_bytes()); - headers.push(b':'); - headers.extend_from_slice( - &decoded - .raw_message_section(message_id, header.value_range()) - .unwrap_or_default(), - ); - } - } - - headers.extend_from_slice(b"\r\n"); - - return Some(if partial.is_none() { - headers.into() - } else { - get_partial_bytes(&headers, partial).to_vec().into() - }); - } - Section::Text => { - return Some(get_cow_partial_bytes( - decoded.raw_message_section(message_id, part.body_to_end())?, - partial, - )); - } - Section::Mime => { - let mut headers = Vec::with_capacity( - u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) - as usize, - ); - for header in part.headers.iter() { - if header.name.is_mime_header() - || header.name.as_str().starts_with("Content-") - { - headers.extend_from_slice(header.name.as_str().as_bytes()); - headers.extend_from_slice(b":"); - headers.extend_from_slice( - &decoded - .raw_message_section(message_id, header.value_range()) - .unwrap_or_default(), - ); - } - } - headers.extend_from_slice(b"\r\n"); - return Some(if partial.is_none() { - headers.into() - } else { - get_partial_bytes(&headers, partial).to_vec().into() - }); - } - } - } - - // BODY[x] should return both headers and body, but most clients - // expect BODY[x] to return only the body, just like BOXY[x.TEXT] does. - - Some(get_cow_partial_bytes( - decoded.raw_message_section(message_id, part.body_to_end())?, - partial, - )) - } - - fn binary<'x>( - &self, - decoded: &'x DecodedParts<'x>, - sections: &[u32], - partial: Option<(u32, u32)>, - ) -> Result>, ()> { - let mut message = &self.contents[0]; - let mut message_id = 0; - let mut part = self.root_part(); - let mut sections_iter = sections.iter().enumerate().peekable(); - - while let Some((section_num, num)) = sections_iter.next() { - part = if let Some(sub_part_ids) = part.sub_parts() { - if let Some(part) = sub_part_ids - .as_ref() - .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) - { - part - } else { - return Ok(None); - } - } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { - part - } else { - return Ok(None); - }; - - if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = - (&part.body, sections_iter.peek()) - { - message = self.message_id(*nested_message); - part = message.root_part(); - message_id = u16::from(nested_message) as usize; - } - } - - if (part.flags & PART_ENCODING_PROBLEM) == 0 { - let part_offset = u32::from(part.offset_header) as usize; - Ok(match &part.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - BodyContents::Bytes(get_cow_partial_bytes( - decoded - .transfer_decoded_contents(message_id, part) - .unwrap_or_default(), - partial, - )) - .into() - } - ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { - BodyContents::Bytes( - get_partial_bytes( - decoded - .binary_part(message_id, part_offset) - .unwrap_or_default(), - partial, - ) - .into(), - ) - .into() - } - ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({ - { - let part = self.message_id(*message).root_part(); - get_cow_partial_bytes( - decoded - .raw_message_section(message_id, part.header_to_end()) - .unwrap_or_default(), - partial, - ) - } - }) - .into(), - ArchivedMetadataPartType::Multipart(_) => { - BodyContents::Bytes(get_cow_partial_bytes( - decoded - .raw_message_section(message_id, part.header_to_end()) - .unwrap_or_default(), - partial, - )) - .into() - } - }) - } else { - Err(()) - } - } - - fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option { - let mut message = &self.contents[0]; - let mut message_id = 0; - let mut part = self.root_part(); - let mut sections_iter = sections.iter().enumerate().peekable(); - - while let Some((section_num, num)) = sections_iter.next() { - part = if let Some(sub_part_ids) = part.sub_parts() { - sub_part_ids - .as_ref() - .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.as_ref().get(u16::from(pos) as usize)) - } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { - Some(part) - } else { - None - }?; - - if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = - (&part.body, sections_iter.peek()) - { - message = self.message_id(*nested_message); - message_id = u16::from(nested_message) as usize; - part = message.root_part(); - } - } - - match &part.body { - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => decoded - .transfer_decoded_contents(message_id, part) - .map(|p| p.len()) - .unwrap_or_default(), - ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => decoded - .part(message_id, u32::from(part.offset_header) as usize) - .map(|p| p.len()) - .unwrap_or_default(), - ArchivedMetadataPartType::Message(message) => { - self.message_id(*message).root_part().raw_len() - } - ArchivedMetadataPartType::Multipart(_) => part.raw_len(), - } - .into() - } -} - -#[inline(always)] -fn get_partial_bytes(bytes: &[u8], partial: Option<(u32, u32)>) -> &[u8] { - if let Some((start, end)) = partial { - bytes - .get(start as usize..std::cmp::min((start + end) as usize, bytes.len())) - .unwrap_or_default() - } else { - bytes - } -} - -#[inline(always)] -fn get_cow_partial_bytes(bytes: Cow<'_, [u8]>, partial: Option<(u32, u32)>) -> Cow<'_, [u8]> { - if let Some((start, end)) = partial { - let range = start as usize..std::cmp::min((start + end) as usize, bytes.len()); - match bytes { - Cow::Borrowed(bytes) => Cow::Borrowed(bytes.get(range).unwrap_or_default()), - Cow::Owned(bytes) => Cow::Owned(bytes.get(range).unwrap_or_default().to_vec()), - } - } else { - bytes - } -} - -trait AsImapAddress { - fn as_imap_address(&'_ self) -> Vec>; -} - -impl AsImapAddress for ArchivedMetadataHeaderValue { - fn as_imap_address(&'_ self) -> Vec> { - let mut addresses = Vec::new(); - - match self { - ArchivedMetadataHeaderValue::AddressList(list) => { - for addr in list.iter() { - if let Some(email) = addr.address.as_ref() { - addresses.push(fetch::Address::Single(fetch::EmailAddress { - name: addr.name.as_ref().map(|n| n.as_ref().into()), - address: email.as_ref().into(), - })); - } - } - } - ArchivedMetadataHeaderValue::AddressGroup(list) => { - for group in list.iter() { - addresses.push(fetch::Address::Group(fetch::AddressGroup { - name: group.name.as_ref().map(|n| n.as_ref().into()), - addresses: group - .addresses - .iter() - .filter_map(|addr| { - fetch::EmailAddress { - name: addr.name.as_ref().map(|n| n.as_ref().into()), - address: addr.address.as_ref()?.as_ref().into(), - } - .into() - }) - .collect(), - })); - } - } - _ => (), - } - - addresses - } -} diff --git a/crates/imap/src/op/fetch/mod.rs b/crates/imap/src/op/fetch/mod.rs new file mode 100644 index 00000000..f80959ae --- /dev/null +++ b/crates/imap/src/op/fetch/mod.rs @@ -0,0 +1,844 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod section; +pub mod source; +pub mod structure; + +use super::{FromModSeq, ImapContext}; +use crate::{ + core::{ + SelectedMailbox, Session, SessionData, rights::MailboxRights, + session::OUTPUT_FLUSH_THRESHOLD, + }, + spawn_op, +}; +use common::{ + MessageStoreCache, + network::SessionStream, + storage::batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, +}; +use compact_str::{CompactString, format_compact}; +use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::{ + messagedata::{EmailMessageData, KeywordDiff, MetadataForm, MetadataValue, merge_keywords}, + metadata::HeaderMatcher, + }, +}; +use imap_proto::{ + Command, ResponseCode, ResponseType, StatusResponse, + parser::PushUnique, + protocol::{ + Flag, ObjectId, + expunge::{Vanished, VanishedFilter}, + fetch::{Arguments, Attribute, DataItem, FetchItem, Section}, + }, + receiver::Request, +}; +use registry::schema::enums::Permission; +use source::DecodedSources; +use std::{borrow::Cow, fmt::Write, iter, sync::Arc, time::Instant}; +use store::{query::log::Query, write::BatchBuilder}; +use structure::{Binary, ImapMetadata}; +use types::{ + collection::{Collection, SyncCollection}, + id::Id, + keyword::Keyword, +}; + +impl Session { + pub async fn handle_fetch(&mut self, requests: Vec>) -> trc::Result<()> { + // Validate access + self.assert_has_permission(Permission::ImapFetch)?; + + let (data, mailbox) = self.state.select_data(); + let is_qresync = self.is_qresync; + let is_uidonly = self.is_uidonly; + let is_utf8 = self.is_utf8; + let message_limit = self.server.core.imap.max_messages_per_command; + + let mut ops = Vec::with_capacity(requests.len()); + let mut activate_objectid = false; + + for request in requests { + let is_uid = matches!(request.command, Command::Fetch(true)); + match request.parse_fetch() { + Ok(arguments) => { + let enabled_condstore = if !self.is_condstore + && arguments.changed_since.is_some() + || arguments.attributes.contains(&Attribute::ModSeq) + { + self.is_condstore = true; + true + } else { + false + }; + + if arguments.attributes.contains(&Attribute::ObjectId) { + activate_objectid = true; + } + + ops.push(Ok((is_uid, enabled_condstore, arguments))); + } + Err(err) => { + ops.push(Err(err)); + } + } + } + + if activate_objectid && let Some(enabled) = self.activate_objectid() { + self.write_bytes(enabled).await?; + } + + // A non-peek fetch sets \Seen, which makes it a write for the purposes of + // RFC 9051 5.5 ordering. Draining the readers keeps it behind them. + let sets_seen = ops.iter().any(|op| { + op.as_ref() + .is_ok_and(|(_, _, arguments)| arguments.sets_seen()) + }); + let _write_permit; + let permit = if sets_seen { + _write_permit = self.acquire_write_permit().await; + None + } else { + self.acquire_read_permit().await + }; + + spawn_op!(permit, data, { + for op in ops { + match op { + Ok((is_uid, enabled_condstore, arguments)) => { + match data + .fetch( + arguments, + mailbox.clone(), + None, + is_uid, + is_qresync, + is_uidonly, + enabled_condstore, + is_utf8, + message_limit, + Instant::now(), + ) + .await + { + Ok(response) => { + data.write_bytes(response.into_bytes()).await?; + } + Err(err) => data.write_error(err).await?, + } + } + Err(err) => data.write_error(err).await?, + } + } + + Ok(()) + }) + } +} + +impl SessionData { + #[allow(clippy::too_many_arguments)] + pub async fn fetch( + &self, + mut arguments: Arguments, + mailbox: Arc, + cache: Option>, + is_uid: bool, + is_qresync: bool, + is_uidonly: bool, + enabled_condstore: bool, + is_utf8: bool, + message_limit: u32, + op_start: Instant, + ) -> trc::Result { + // Validate VANISHED parameter + if arguments.include_vanished { + if !is_qresync { + return Err(trc::ImapEvent::Error + .into_err() + .details("Enable QRESYNC first to use the VANISHED parameter.") + .ctx(trc::Key::Type, ResponseType::Bad) + .id(arguments.tag)); + } else if !is_uid { + return Err(trc::ImapEvent::Error + .into_err() + .details("VANISHED parameter is only available for UID FETCH.") + .ctx(trc::Key::Type, ResponseType::Bad) + .id(arguments.tag)); + } + } + + // Resync messages if needed + let account_id = mailbox.id.account_id; + let cache = match cache { + Some(cache) => cache, + None => self + .server + .get_cached_messages(account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?, + }; + let mut modseq = self + .sync_view(&mailbox, &cache, None) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + + // Convert IMAP ids to JMAP ids. + let changed_ids = arguments.changed_since.and_then(|changed_since| { + mailbox.view.lock().changed_since_scan( + &cache.emails, + changed_since, + &arguments.sequence_set, + is_uid, + ) + }); + let is_changed_only = changed_ids.is_some(); + let mut ids = match changed_ids { + Some(ids) => ids, + None => mailbox + .resolve(&arguments.sequence_set, is_uid) + .await + .imap_ctx(&arguments.tag, trc::location!())?, + }; + + // Convert state to modseq + if let Some(changed_since) = arguments.changed_since { + // Send vanished UIDs + if arguments.include_vanished && cache.last_change_id >= changed_since { + // Add to vanished all known destroyed Ids + let mut vanished = self + .server + .store() + .vanished_uids( + account_id, + mailbox.id.mailbox_id, + Query::from_modseq(changed_since), + ) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + if !vanished.is_empty() { + let last_uid = mailbox.view.lock().uid_max(); + if let Some(filter) = VanishedFilter::new(&arguments.sequence_set, last_uid) { + vanished.retain(|uid| filter.contains(*uid)); + } + } + + if !vanished.is_empty() { + let mut buf = Vec::with_capacity(vanished.len() * 3); + Vanished { + earlier: true, + ids: vanished, + } + .serialize(&mut buf); + self.write_bytes(buf).await?; + } + } + + // Filter out ids without changes + if !is_changed_only { + ids.retain(|resolved| { + cache + .email_by_id(&resolved.id) + .is_some_and(|message| message.change_id() >= changed_since) + }); + } + if ids.is_empty() { + // Condstore was just enabled, return highest modseq. + if enabled_condstore { + self.write_bytes( + StatusResponse::ok("Highest Modseq") + .with_code(ResponseCode::highest_modseq(modseq)) + .into_bytes(), + ) + .await?; + } + + trc::event!( + Imap(trc::ImapEvent::Fetch), + SpanId = self.session_id, + AccountId = account_id, + MailboxId = mailbox.id.mailbox_id, + Elapsed = op_start.elapsed() + ); + + return Ok( + StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag) + ); + } + arguments.attributes.push_unique(Attribute::ModSeq); + } + + let needs = FetchNeeds::new(&arguments.attributes); + let access_token = if mailbox.is_select && arguments.sets_seen() { + Some( + self.refresh_access_token() + .await + .imap_ctx(&arguments.tag, trc::location!())?, + ) + } else { + None + }; + let mut seen_rights = access_token.as_ref().and_then(|access_token| { + let mut rights = MailboxRights::new(access_token, mailbox.id.account_id); + rights + .permanent_flags(&cache, [mailbox.id.mailbox_id]) + .permits(&Flag::Seen) + .then_some(rights) + }); + + if is_uid { + if arguments.attributes.is_empty() { + arguments.attributes.push(Attribute::Flags); + } else if !arguments.attributes.contains(&Attribute::Uid) { + arguments.attributes.insert(0, Attribute::Uid); + } + } + let matchers = arguments + .attributes + .iter() + .map(|attribute| { + attribute + .header_fields() + .map(|fields| HeaderMatcher::new(fields.iter().map(String::as_str))) + }) + .collect::>(); + + // Process each message + let mut batch = BatchBuilder::new(); + // RFC 9738 requires the highest UIDs to be processed first when truncating + let message_limit = message_limit as usize; + let limited_uid = if ids.len() > message_limit { + let cutoff = ids.len() - message_limit; + ids.select_nth_unstable_by_key(cutoff, |resolved| resolved.uid); + ids.drain(..cutoff); + ids.sort_unstable_by_key(|resolved| resolved.uid); + ids.first().map(|resolved| resolved.uid) + } else { + ids.sort_unstable_by_key(|resolved| resolved.uid); + None + }; + + let form = if needs.headers { + MetadataForm::Row + } else { + MetadataForm::Structure + }; + let mut metadata_batch = DocumentBatch::new(MAX_BATCH_BYTES); + let mut output = Vec::with_capacity((ids.len() * 64).min(OUTPUT_FLUSH_THRESHOLD)); + let mut windows = ids.chunks(MAX_BATCH_DOCUMENTS).peekable(); + let mut budget_error = None; + 'windows: while let Some(window) = windows.next() { + metadata_batch.reset(); + if needs.structure { + for resolved in window { + if cache.has_email_id(&resolved.id) { + metadata_batch.request(resolved.id); + } + } + } + + loop { + self.server + .read_metadata_batch(account_id, form, &mut metadata_batch) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + + for resolved in window { + let (seqnum, uid, id) = (resolved.seqnum, resolved.uid, resolved.id); + let Some(data) = cache.email_by_id(&id) else { + if metadata_batch.is_first_round() { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id, + Collection = Collection::Email, + Details = "Message data not found.", + CausedBy = trc::location!(), + ); + } + continue; + }; + let stored = if needs.structure { + Some(match metadata_batch.lookup(id) { + Lookup::Found(stored) => stored, + Lookup::Missing => { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id, + Collection = Collection::Email, + Details = "Message metadata not found.", + CausedBy = trc::location!(), + ); + continue; + } + Lookup::Skip => continue, + }) + } else if metadata_batch.is_first_round() { + None + } else { + continue; + }; + let set_seen_flag = !cache.has_keyword(data, &Keyword::Seen) + && seen_rights.as_mut().is_some_and(|rights| { + rights + .permanent_flags( + &cache, + data.mailboxes() + .iter() + .map(|membership| membership.mailbox_id), + ) + .permits(&Flag::Seen) + }); + let headers = stored + .map(MetadataValue::raw_headers) + .transpose() + .imap_ctx(&arguments.tag, trc::location!())? + .flatten(); + let metadata = stored + .map(MetadataValue::unarchive) + .transpose() + .imap_ctx(&arguments.tag, trc::location!())?; + + let blob = match metadata.filter(|_| needs.blob) { + Some(metadata) => { + let blob = match self + .server + .fetch_blob( + &metadata.blob_hash(), + 0..usize::MAX, + Some(metadata.size() as u64), + ) + .await + .and_then(|blob| { + blob.map(|mut blob| { + blob.try_reserve_extra(blob.len()).map(|_| blob) + }) + .transpose() + }) { + Ok(blob) => blob, + Err(err) + if err.matches(trc::EventType::Limit( + trc::LimitEvent::MemoryBudget, + )) => + { + budget_error = Some(err); + break 'windows; + } + Err(err) => { + return Err(err).imap_ctx(&arguments.tag, trc::location!()); + } + }; + if blob.is_none() { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id, + Collection = Collection::Email, + BlobId = metadata.blob_hash.0.as_slice(), + Details = "Blob not found.", + CausedBy = trc::location!(), + ); + continue; + } + blob + } + None => None, + }; + let raw = metadata + .map(|metadata| { + metadata.raw_message( + headers.as_deref(), + blob.as_deref().unwrap_or_default(), + ) + }) + .unwrap_or_default(); + + let mut sources = DecodedSources::default(); + let mut message_start = output.len(); + let mut has_items = false; + FetchItem::write_open( + &mut output, + if is_uidonly { uid } else { seqnum }, + is_uidonly, + ); + + for (attribute, matcher) in arguments.attributes.iter().zip(&matchers) { + match attribute { + Attribute::Flags => { + separate(&mut output, &mut has_items); + Flag::write_fetch_item( + &mut output, + cache + .expand_keywords(data) + .map(Flag::from) + .chain(set_seen_flag.then_some(Flag::Seen)), + ); + } + Attribute::InternalDate => { + separate(&mut output, &mut has_items); + DataItem::InternalDate { + date: data.received_at() as i64, + } + .serialize(&mut output); + } + Attribute::Rfc822Size => { + separate(&mut output, &mut has_items); + DataItem::Rfc822Size { + size: data.size() as usize, + } + .serialize(&mut output); + } + Attribute::Uid => { + separate(&mut output, &mut has_items); + DataItem::Uid { uid }.serialize(&mut output); + } + Attribute::ModSeq => { + separate(&mut output, &mut has_items); + DataItem::ModSeq { + modseq: data.change_id() + 1, + } + .serialize(&mut output); + } + Attribute::ObjectId => { + separate(&mut output, &mut has_items); + DataItem::ObjectId(ObjectId { + email_id: Some(Id::from_parts(data.thread_id(), id)), + thread_id: Some(Id::from(data.thread_id())), + ..Default::default() + }) + .serialize(&mut output); + } + Attribute::Envelope => { + if let Some(metadata) = metadata { + separate(&mut output, &mut has_items); + output.extend_from_slice(b"ENVELOPE "); + metadata.write_envelope(&mut output, is_utf8); + } + } + Attribute::Body | Attribute::BodyStructure => { + if let Some(metadata) = metadata { + let is_extended = matches!(attribute, Attribute::BodyStructure); + separate(&mut output, &mut has_items); + output.extend_from_slice(if is_extended { + b"BODYSTRUCTURE ".as_slice() + } else { + b"BODY ".as_slice() + }); + metadata.write_structure(&mut output, is_extended, is_utf8); + } + } + Attribute::Preview { .. } => { + if let Some(metadata) = metadata { + separate(&mut output, &mut has_items); + let preview = metadata.preview(); + DataItem::Preview { + contents: (!preview.is_empty()) + .then(|| preview.as_bytes().into()), + } + .serialize(&mut output); + } + } + Attribute::Rfc822 => { + if let Some(contents) = raw.whole() { + separate(&mut output, &mut has_items); + DataItem::Rfc822 { contents }.serialize(&mut output); + } + } + Attribute::Rfc822Header => { + if let Some(contents) = + metadata.and_then(|metadata| metadata.header(raw)) + { + separate(&mut output, &mut has_items); + DataItem::Rfc822Header { contents }.serialize(&mut output); + } + } + Attribute::Rfc822Text => { + if let Some(contents) = metadata.and_then(|metadata| { + raw.view(metadata.headers_len()..raw.len()) + }) { + separate(&mut output, &mut has_items); + DataItem::Rfc822Text { contents }.serialize(&mut output); + } + } + Attribute::BodySection { + sections, partial, .. + } => { + let Some(metadata) = metadata else { + continue; + }; + if let (Some(matcher), Some(headers), [section], None) = + (matcher, &headers, sections.as_slice(), partial) + { + separate(&mut output, &mut has_items); + metadata.write_header_fields( + &mut output, + headers, + section, + matcher, + ); + } else if let Some(contents) = metadata.body_section( + raw, + &mut sources, + sections, + *partial, + matcher.as_ref(), + ) { + separate(&mut output, &mut has_items); + DataItem::BodySection { + sections: Cow::Borrowed(sections), + origin_octet: partial.map(|(start, _)| start), + contents, + } + .serialize(&mut output); + } + } + Attribute::Binary { + sections, partial, .. + } => { + let Some(metadata) = metadata else { + continue; + }; + match metadata.binary(raw, &mut sources, sections, *partial) { + Binary::Found(contents) => { + separate(&mut output, &mut has_items); + DataItem::Binary { + sections: Cow::Borrowed(sections), + offset: partial.map(|(start, _)| start), + contents, + } + .serialize(&mut output); + } + Binary::Missing => (), + Binary::UnknownCte => { + self.write_unknown_cte( + &mut output, + message_start, + sections, + if is_uid { uid } else { seqnum }, + ) + .await?; + message_start = 0; + } + } + } + Attribute::BinarySize { sections } => { + let Some(metadata) = metadata else { + continue; + }; + match metadata.binary_size(sections) { + Binary::Found(size) => { + separate(&mut output, &mut has_items); + DataItem::BinarySize { + sections: Cow::Borrowed(sections), + size, + } + .serialize(&mut output); + } + Binary::Missing => (), + Binary::UnknownCte => { + self.write_unknown_cte( + &mut output, + message_start, + sections, + if is_uid { uid } else { seqnum }, + ) + .await?; + message_start = 0; + } + } + } + } + } + + // Add flags to the response if the message was unseen + if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { + separate(&mut output, &mut has_items); + Flag::write_fetch_item( + &mut output, + cache + .expand_keywords(data) + .map(Flag::from) + .chain(iter::once(Flag::Seen)), + ); + } + + FetchItem::write_close(&mut output); + self.flush_output(&mut output, false).await?; + + // Add to set flags + if set_seen_flag { + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(id); + merge_keywords( + &mut batch, + data.thread_id(), + KeywordDiff::added(Keyword::Seen), + ); + batch.commit_point(); + } + } + + if !metadata_batch.next_round() { + break; + } + } + + if needs.structure && windows.peek().is_some() { + self.flush_output(&mut output, true).await?; + } + } + + // Set Seen ids + if !batch.is_empty() { + match self + .server + .commit_batch(batch) + .await + .map(|ids| ids.last_change_id(account_id, SyncCollection::Email)) + .imap_ctx(&arguments.tag, trc::location!()) + { + Ok(change_id) => { + modseq = change_id; + } + Err(err) => { + if !err.is_assertion_failure() { + return Err(err.id(arguments.tag)); + } + } + } + } + + self.flush_output(&mut output, true).await?; + + if let Some(err) = budget_error { + return Err(err.ctx(trc::Key::Id, CompactString::from(arguments.tag.as_str()))); + } + + trc::event!( + Imap(trc::ImapEvent::Fetch), + SpanId = self.session_id, + AccountId = account_id, + MailboxId = mailbox.id.mailbox_id, + DocumentId = ids + .iter() + .map(|resolved| trc::Value::from(resolved.id)) + .collect::>(), + Details = arguments + .attributes + .iter() + .map(|c| trc::Value::from(format_compact!("{c:?}"))) + .collect::>(), + Elapsed = op_start.elapsed() + ); + + // Condstore was enabled with this command + if enabled_condstore { + self.write_bytes( + StatusResponse::ok("Highest Modseq") + .with_code(ResponseCode::highest_modseq(modseq)) + .into_bytes(), + ) + .await?; + } + + let response = StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag); + Ok(match limited_uid { + Some(uid) => response.with_code(ResponseCode::MessageLimit { + limit: message_limit as u32, + uid: uid.into(), + }), + None => response, + }) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct FetchNeeds { + pub structure: bool, + pub headers: bool, + pub blob: bool, +} + +impl FetchNeeds { + pub fn new(attributes: &[Attribute]) -> Self { + let mut needs = FetchNeeds::default(); + for attribute in attributes { + let (structure, headers, blob) = match attribute { + Attribute::Envelope + | Attribute::Body + | Attribute::BodyStructure + | Attribute::BinarySize { .. } + | Attribute::Preview { .. } => (true, false, false), + Attribute::Rfc822 => (true, true, true), + Attribute::Rfc822Header => (true, true, false), + Attribute::Rfc822Text => (true, false, true), + Attribute::BodySection { sections, .. } => match sections.as_slice() { + [] | [Section::Part { num: 1 }, Section::Mime] => (true, true, true), + [ + Section::Header | Section::HeaderFields { .. } | Section::Mime, + .., + ] => (true, true, false), + _ => (true, false, true), + }, + Attribute::Binary { sections, .. } => (true, sections.is_empty(), true), + Attribute::Flags + | Attribute::InternalDate + | Attribute::Rfc822Size + | Attribute::Uid + | Attribute::ModSeq + | Attribute::ObjectId => (false, false, false), + }; + needs.structure |= structure; + needs.headers |= headers; + needs.blob |= blob; + } + needs + } +} + +impl SessionData { + async fn write_unknown_cte( + &self, + output: &mut Vec, + message_start: usize, + sections: &[u32], + id: u32, + ) -> trc::Result<()> { + let pending = output.split_off(message_start); + self.flush_output(output, true).await?; + let mut part = String::with_capacity(sections.len() * 3); + for (pos, section) in sections.iter().enumerate() { + if pos > 0 { + part.push('.'); + } + let _ = write!(part, "{section}"); + } + let err = trc::ImapEvent::Error + .into_err() + .details(format_compact!( + "Failed to decode part {part} of message {id}." + )) + .code(ResponseCode::UnknownCte); + self.write_error(err).await?; + output.extend_from_slice(&pending); + Ok(()) + } +} + +fn separate(output: &mut Vec, has_items: &mut bool) { + if *has_items { + output.push(b' '); + } else { + *has_items = true; + } +} + diff --git a/crates/imap/src/op/fetch/section.rs b/crates/imap/src/op/fetch/section.rs new file mode 100644 index 00000000..b906ad75 --- /dev/null +++ b/crates/imap/src/op/fetch/section.rs @@ -0,0 +1,401 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + source::{DecodedSources, SourceView}, + structure::Binary, +}; +use email::message::metadata::{ + HeaderList, HeaderMatcher, HeaderView, MessageView, PartFlags, PartKind, PartView, RawMessage, +}; +use imap_proto::protocol::{ + fetch::{BodyContents, Section}, + push_int, +}; +use std::ops::Range; +use utils::chained_bytes::ChainedBytes; + +const CONTENT_PREFIX: &[u8] = b"content-"; +const MAX_FIELDS_CAPACITY: usize = 16 * 1024; + +pub(super) trait MessageSections { + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + ); + fn header<'x>(&self, raw: RawMessage<'x>) -> Option>; + fn body_section<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[Section], + partial: Option<(u32, u32)>, + matcher: Option<&HeaderMatcher>, + ) -> Option>; + fn binary<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[u32], + partial: Option<(u32, u32)>, + ) -> Binary>; + fn binary_size(&self, sections: &[u32]) -> Binary; +} + +impl MessageSections for MessageView<'_> { + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + ) { + let part = self.root_part(); + let fields = part.headers(); + let gap = part.header_gap(&SourceView::Bytes(headers)); + if gap.is_empty() { + fields.write_header_fields(buf, headers, section, matcher, |_| true); + } else { + fields.write_header_fields(buf, headers, section, matcher, |header| { + !gap.contains(&header.field_range().start) + }); + } + } + + fn header<'x>(&self, raw: RawMessage<'x>) -> Option> { + let part = self.root_part(); + let source = SourceView::Raw(raw); + source.view_without(part.header_range(), part.header_gap(&source)) + } + + fn body_section<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[Section], + partial: Option<(u32, u32)>, + matcher: Option<&HeaderMatcher>, + ) -> Option> { + let mut part = self.root_part(); + if sections.is_empty() { + return raw + .view(part.raw_range()) + .map(|bytes| BodyContents::Bytes(bytes).partial(partial)); + } + + let mut sections_iter = sections.iter().enumerate().peekable(); + while let Some((section_num, section)) = sections_iter.next() { + match section { + Section::Part { num } => { + part = if part.is_multipart() { + part.child((*num).saturating_sub(1) as usize)? + } else if *num == 1 + && (section_num == sections.len() - 1 + || part.is_message() + || (part.is_message_root() + && matches!(sections_iter.peek(), Some((_, Section::Mime))))) + { + part + } else { + return None; + }; + + if part.is_message() + && let Some(( + _, + Section::Part { .. } + | Section::Header + | Section::HeaderFields { .. } + | Section::Text, + )) = sections_iter.peek() + { + part = part.nested()?.root_part(); + } + } + Section::Header => { + let source = sources.source(part.message(), raw)?; + return source + .view_without(part.header_range(), part.header_gap(&source)) + .map(|bytes| BodyContents::Bytes(bytes).partial(partial)); + } + Section::HeaderFields { not, fields } => { + let built; + let matcher = match matcher { + Some(matcher) => matcher, + None => { + built = HeaderMatcher::new(fields.iter().map(String::as_str)); + &built + } + }; + return part.fields_section(sources, raw, partial, |source| { + part.collect_fields(source, |header, field| { + matcher.matches_field(header, field) != *not + }) + }); + } + Section::Text => { + let message = part.message(); + let source = sources.source(message, raw)?; + let mut range = part.body_range(); + if part.is_message_root() + && let Some(container) = message.container() + { + let end = if message + .source_part() + .is_some_and(|source_part| source_part.id() == container.id()) + { + source.len() + } else { + container.offset_end() + }; + if end > range.end { + range.end = end; + } + } + return source + .view(range) + .map(|bytes| BodyContents::Bytes(bytes).partial(partial)); + } + Section::Mime => { + if !part.is_message_root() { + return sources + .source(part.message(), raw)? + .view(part.header_range()) + .map(|bytes| BodyContents::Bytes(bytes).partial(partial)); + } + return part.fields_section(sources, raw, partial, |source| { + part.collect_fields(source, |header, field| header.is_mime_field(field)) + }); + } + } + } + + sources + .source(part.message(), raw)? + .view(part.body_range()) + .map(|bytes| BodyContents::Bytes(bytes).partial(partial)) + } + + fn binary<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[u32], + partial: Option<(u32, u32)>, + ) -> Binary> { + let Some(part) = self.root_part().descend(sections) else { + return Binary::Missing; + }; + if part.flags().contains(PartFlags::UNKNOWN_TRANSFER_ENCODING) { + return Binary::UnknownCte; + } + let contents = match part.kind() { + PartKind::Text | PartKind::Html | PartKind::Binary | PartKind::InlineBinary => { + match sources.leaf(part, raw) { + Some(bytes) => BodyContents::Bytes(bytes), + None => return Binary::Missing, + } + } + PartKind::Message => { + let Some(nested) = part.nested() else { + return Binary::Missing; + }; + match sources.source(nested, raw) { + Some(source) => source + .view(nested.root_part().raw_range()) + .map_or(BodyContents::Owned(Vec::new()), BodyContents::Bytes), + None => return Binary::Missing, + } + } + PartKind::Multipart => match sources.source(part.message(), raw) { + Some(source) => source + .view(part.raw_range()) + .map_or(BodyContents::Owned(Vec::new()), BodyContents::Bytes), + None => return Binary::Missing, + }, + }; + Binary::Found(contents.partial(partial)) + } + + fn binary_size(&self, sections: &[u32]) -> Binary { + let Some(part) = self.root_part().descend(sections) else { + return Binary::Missing; + }; + if part.flags().contains(PartFlags::UNKNOWN_TRANSFER_ENCODING) { + return Binary::UnknownCte; + } + Binary::Found(match part.kind() { + PartKind::Multipart => part.raw_range().len(), + _ => part.decoded_size() as usize, + }) + } +} + +trait PartSections<'a>: Sized { + fn descend(self, sections: &[u32]) -> Option; + fn fields_section<'x>( + &self, + sources: &'x mut DecodedSources, + raw: RawMessage<'x>, + partial: Option<(u32, u32)>, + collect: impl FnOnce(&SourceView<'_>) -> Vec, + ) -> Option>; + fn header_gap(&self, source: &SourceView<'_>) -> Range; + fn collect_fields( + &self, + source: &SourceView<'_>, + select: impl Fn(HeaderView<'_>, &[u8]) -> bool, + ) -> Vec; +} + +impl<'a> PartSections<'a> for PartView<'a> { + fn descend(self, sections: &[u32]) -> Option { + let mut part = self; + let mut sections_iter = sections.iter().enumerate().peekable(); + while let Some((section_num, num)) = sections_iter.next() { + part = if part.is_multipart() { + part.child((*num).saturating_sub(1) as usize)? + } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { + part + } else { + return None; + }; + if part.is_message() && sections_iter.peek().is_some() { + part = part.nested()?.root_part(); + } + } + Some(part) + } + + fn fields_section<'x>( + &self, + sources: &'x mut DecodedSources, + raw: RawMessage<'x>, + partial: Option<(u32, u32)>, + collect: impl FnOnce(&SourceView<'_>) -> Vec, + ) -> Option> { + let source = sources.source(self.message(), raw)?; + Some(BodyContents::Owned(collect(&source)).partial(partial)) + } + + fn header_gap(&self, source: &SourceView<'_>) -> Range { + if !self.is_message_root() { + return 0..0; + } + let message = self.message(); + let prefix = if message.id() == 0 { + message.metadata().extra_headers_len() + } else { + 0 + }; + let start = self.offset_header().saturating_add(prefix); + let first_field = |header: HeaderView<'_>| { + let range = header.field_range(); + (range.start >= start + && (header.id().is_known() + || source + .get(range.clone()) + .is_some_and(|field| header.raw_name_in(&field).is_field_name()))) + .then_some(range.start) + }; + let end = self.headers().iter().find_map(first_field); + start..end.unwrap_or(start) + } + + fn collect_fields( + &self, + source: &SourceView<'_>, + select: impl Fn(HeaderView<'_>, &[u8]) -> bool, + ) -> Vec { + let gap = self.header_gap(source); + let mut headers = Vec::with_capacity(self.header_range().len().min(MAX_FIELDS_CAPACITY)); + for header in self.headers().iter() { + let range = header.field_range(); + if !gap.contains(&range.start) + && let Some(field) = source.get(range) + && select(header, &field) + { + headers.extend_from_slice(&field); + } + } + headers.extend_from_slice(b"\r\n"); + headers + } +} + +trait HeaderFieldsWriter { + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + keep: impl Fn(&HeaderView<'_>) -> bool, + ); +} + +impl HeaderFieldsWriter for HeaderList<'_> { + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + keep: impl Fn(&HeaderView<'_>) -> bool, + ) { + let not = matches!(section, Section::HeaderFields { not: true, .. }); + let selected = || { + self.iter().filter(|header| { + keep(header) + && matcher.matches(*header, headers) != not + && headers.get(header.field_range()).is_some() + }) + }; + let len = selected() + .map(|header| header.field_range().len()) + .sum::() + + 2; + buf.extend_from_slice(b"BODY["); + section.serialize(buf); + buf.extend_from_slice(b"] {"); + push_int(buf, len); + buf.extend_from_slice(b"}\r\n"); + for header in selected() { + if let Some(field) = headers.get(header.field_range()) { + buf.extend_from_slice(field); + } + } + buf.extend_from_slice(b"\r\n"); + } +} + +trait FieldName { + fn is_field_name(&self) -> bool; +} + +impl FieldName for [u8] { + fn is_field_name(&self) -> bool { + !self.is_empty() && self.iter().all(|byte| matches!(byte, 33..=57 | 59..=126)) + } +} + +trait MimeField { + fn is_mime_field(&self, field: &[u8]) -> bool; +} + +impl MimeField for HeaderView<'_> { + fn is_mime_field(&self, field: &[u8]) -> bool { + self.id().is_mime() + || self + .raw_name_in(field) + .get(..CONTENT_PREFIX.len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(CONTENT_PREFIX)) + } +} + diff --git a/crates/imap/src/op/fetch/source.rs b/crates/imap/src/op/fetch/source.rs new file mode 100644 index 00000000..890d0550 --- /dev/null +++ b/crates/imap/src/op/fetch/source.rs @@ -0,0 +1,189 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use email::message::metadata::{ + MAX_SOURCE_DEPTH, MessageView, PartView, RawMessage, TransferEncoding, +}; +use std::{borrow::Cow, ops::Range}; +use utils::chained_bytes::ChainedBytes; + +#[derive(Debug, Default)] +pub struct DecodedSources { + source: Option, + leaf: Option, +} + +#[derive(Debug)] +struct Decoded { + part_id: u32, + bytes: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum SourceView<'x> { + Raw(RawMessage<'x>), + Bytes(&'x [u8]), +} + +impl DecodedSources { + pub(super) fn source<'x>( + &'x mut self, + message: MessageView<'_>, + raw: RawMessage<'x>, + ) -> Option> { + Decoded::source(&mut self.source, message, raw) + } + + pub(super) fn leaf<'x>( + &'x mut self, + part: PartView<'_>, + raw: RawMessage<'x>, + ) -> Option> { + let DecodedSources { source, leaf } = self; + if leaf.as_ref().is_some_and(|leaf| leaf.part_id == part.id()) { + return leaf.as_ref().map(|leaf| ChainedBytes::new(&leaf.bytes)); + } + match Decoded::source(source, part.message(), raw)? + .decode(part.body_range(), part.encoding()) + { + Cow::Borrowed(bytes) => Some(ChainedBytes::new(bytes)), + Cow::Owned(bytes) => Some(ChainedBytes::new( + &leaf + .insert(Decoded { + part_id: part.id(), + bytes, + }) + .bytes, + )), + } + } + + pub fn len(&self) -> usize { + usize::from(self.source.is_some()) + usize::from(self.leaf.is_some()) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[cfg(all(test, stalwart_tests))] + pub(super) fn bytes_held(&self) -> usize { + [&self.source, &self.leaf] + .into_iter() + .flatten() + .map(|decoded| decoded.bytes.len()) + .sum() + } +} + +impl Decoded { + fn source<'x>( + slot: &'x mut Option, + message: MessageView<'_>, + raw: RawMessage<'x>, + ) -> Option> { + let mut chain = [0u32; MAX_SOURCE_DEPTH]; + let mut depth = 0; + let mut is_cached = false; + let mut current = message; + while let Some(part) = current.source_part() { + if slot + .as_ref() + .is_some_and(|decoded| decoded.part_id == part.id()) + { + is_cached = true; + break; + } + *chain.get_mut(depth)? = part.id(); + depth += 1; + current = part.message(); + } + + if !is_cached { + if depth == 0 { + return Some(SourceView::Raw(raw)); + } + *slot = None; + } + + let meta = message.metadata(); + for part_id in chain.get(..depth)?.iter().rev() { + let part = meta.part(*part_id)?; + let range = part.body_range(); + let mut bytes = Vec::new(); + match slot.as_ref() { + None => part.encoding().decode_append(&raw.get(range)?, &mut bytes), + Some(base) => part + .encoding() + .decode_append(base.bytes.get(range)?, &mut bytes), + } + *slot = Some(Decoded { + part_id: *part_id, + bytes, + }); + } + + slot.as_ref() + .map(|decoded| SourceView::Bytes(&decoded.bytes)) + } +} + +impl<'x> SourceView<'x> { + pub(super) fn view(&self, range: Range) -> Option> { + match self { + SourceView::Raw(raw) => raw.view(range), + SourceView::Bytes(bytes) => bytes.get(range).map(ChainedBytes::new), + } + } + + pub(super) fn get(&self, range: Range) -> Option> { + match self { + SourceView::Raw(raw) => raw.get(range), + SourceView::Bytes(bytes) => bytes.get(range).map(Cow::Borrowed), + } + } + + pub(super) fn view_without( + &self, + range: Range, + gap: Range, + ) -> Option> { + if !gap.is_empty() + && let (Some(head), Some(tail)) = ( + self.contiguous(range.start..gap.start), + self.contiguous(gap.end..range.end), + ) + { + return Some(ChainedBytes::chain(head, tail)); + } + self.view(range) + } + + pub(super) fn len(&self) -> usize { + match self { + SourceView::Raw(raw) => raw.len(), + SourceView::Bytes(bytes) => bytes.len(), + } + } + + fn contiguous(&self, range: Range) -> Option<&'x [u8]> { + match self { + SourceView::Raw(raw) => raw.contiguous(range), + SourceView::Bytes(bytes) => bytes.get(range), + } + } + + fn decode(&self, range: Range, encoding: TransferEncoding) -> Cow<'x, [u8]> { + match self.get(range) { + Some(Cow::Borrowed(bytes)) => encoding.decode(bytes), + Some(Cow::Owned(bytes)) => match encoding.decode(&bytes) { + Cow::Borrowed(_) => Cow::Owned(bytes), + Cow::Owned(decoded) => Cow::Owned(decoded), + }, + None => Cow::Borrowed(&[]), + } + } +} diff --git a/crates/imap/src/op/fetch/structure.rs b/crates/imap/src/op/fetch/structure.rs new file mode 100644 index 00000000..9f5f7860 --- /dev/null +++ b/crates/imap/src/op/fetch/structure.rs @@ -0,0 +1,737 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{section::MessageSections, source::DecodedSources}; +use email::message::metadata::{ + AddressHeader, AddressItem, Addresses, ArchivedMessageMetadata, ContentTypeView, EnvelopeView, + HeaderMatcher, MAX_NESTING, Mailbox, MessageView, Occurrence, PartKind, PartView, RawMessage, +}; +use imap_proto::protocol::{ + fetch::{BodyContents, Section}, + push_int, quoted_or_literal_encoded_string, quoted_or_literal_encoded_string_or_nil, + quoted_or_literal_raw_string, quoted_or_literal_raw_string_or_nil, quoted_rfc2822, +}; +use utils::chained_bytes::ChainedBytes; + +const DUMMY_ADDRESS: Mailbox<'static> = Mailbox { + name: None, + address: Some("unknown@localhost"), +}; +const DEFAULT_TRANSFER_ENCODING: &str = "7bit"; +const DEFAULT_TEXT_SUBTYPE: &str = "plain"; +const DEFAULT_MULTIPART_SUBTYPE: &str = "mixed"; +const DEFAULT_MESSAGE_SUBTYPE: &str = "rfc822"; +const DEFAULT_TEXT_PARAMS: &[u8] = b"(\"charset\" \"us-ascii\")"; +const EMPTY_ENVELOPE: &[u8] = b"(NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)"; +const EMPTY_BODY: &[u8] = b"(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 0 0)"; +const EMPTY_BODY_EXTENDED: &[u8] = + b"(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 0 0 NIL NIL NIL NIL)"; +const MESSAGE_FRAME: u32 = u32::MAX; +const NO_PARENT: u32 = u32::MAX; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Binary { + Found(T), + Missing, + UnknownCte, +} + +impl Binary { + pub fn map(self, f: impl FnOnce(T) -> U) -> Binary { + match self { + Binary::Found(value) => Binary::Found(f(value)), + Binary::Missing => Binary::Missing, + Binary::UnknownCte => Binary::UnknownCte, + } + } +} + +pub trait ImapMetadata { + fn write_envelope(&self, buf: &mut Vec, is_utf8: bool); + fn write_structure(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool); + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + ); + fn header<'x>(&self, raw: RawMessage<'x>) -> Option>; + fn body_section<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[Section], + partial: Option<(u32, u32)>, + matcher: Option<&HeaderMatcher>, + ) -> Option>; + fn binary<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[u32], + partial: Option<(u32, u32)>, + ) -> Binary>; + fn binary_size(&self, sections: &[u32]) -> Binary; +} + +impl ImapMetadata for ArchivedMessageMetadata { + fn write_envelope(&self, buf: &mut Vec, is_utf8: bool) { + self.root().envelope().write_imap(buf, is_utf8); + } + + fn write_structure(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool) { + self.root().write_structure(buf, is_extended, is_utf8); + } + + fn write_header_fields( + &self, + buf: &mut Vec, + headers: &[u8], + section: &Section, + matcher: &HeaderMatcher, + ) { + self.root() + .write_header_fields(buf, headers, section, matcher); + } + + fn header<'x>(&self, raw: RawMessage<'x>) -> Option> { + self.root().header(raw) + } + + fn body_section<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[Section], + partial: Option<(u32, u32)>, + matcher: Option<&HeaderMatcher>, + ) -> Option> { + self.root() + .body_section(raw, sources, sections, partial, matcher) + } + + fn binary<'x>( + &self, + raw: RawMessage<'x>, + sources: &'x mut DecodedSources, + sections: &[u32], + partial: Option<(u32, u32)>, + ) -> Binary> { + self.root().binary(raw, sources, sections, partial) + } + + fn binary_size(&self, sections: &[u32]) -> Binary { + self.root().binary_size(sections) + } +} + +#[derive(Clone, Copy)] +struct Frame { + part: u32, + next: u32, +} + +struct FrameStack { + frames: [Frame; MAX_NESTING], + len: usize, +} + +impl FrameStack { + fn new() -> Self { + FrameStack { + frames: [Frame { part: 0, next: 0 }; MAX_NESTING], + len: 0, + } + } + + fn push(&mut self, frame: Frame) -> bool { + match self.frames.get_mut(self.len) { + Some(slot) => { + *slot = frame; + self.len += 1; + true + } + None => false, + } + } + + fn top(&mut self) -> Option<&mut Frame> { + self.frames.get_mut(self.len.checked_sub(1)?) + } + + fn pop(&mut self) -> Option { + let frame = *self.frames.get(self.len.checked_sub(1)?)?; + self.len -= 1; + Some(frame) + } +} + +trait StructureWriter { + fn write_structure(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool); +} + +impl StructureWriter for MessageView<'_> { + fn write_structure(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool) { + let meta = self.metadata(); + let empty_body = if is_extended { + EMPTY_BODY_EXTENDED + } else { + EMPTY_BODY + }; + let mut stack = FrameStack::new(); + let mut next = Some((self.root_part(), NO_PARENT)); + + loop { + if let Some((part, parent)) = next.take() { + match part.kind() { + PartKind::Multipart => { + buf.push(b'('); + if stack.push(Frame { + part: part.id(), + next: 0, + }) { + continue; + } + buf.extend_from_slice(empty_body); + part.write_multipart_tail(buf, is_extended, is_utf8); + } + PartKind::Message => { + buf.push(b'('); + let nested = part.nested(); + let content_type = part.content_type(); + part.write_message_head( + buf, + content_type + .as_ref() + .and_then(|content_type| content_type.message_subtype()) + .unwrap_or(DEFAULT_MESSAGE_SUBTYPE), + content_type, + nested.map(|nested| nested.envelope()), + is_utf8, + ); + match nested { + Some(nested) + if stack.push(Frame { + part: part.id(), + next: MESSAGE_FRAME, + }) => + { + next = Some((nested.root_part(), NO_PARENT)); + continue; + } + _ => { + buf.extend_from_slice(empty_body); + part.write_message_tail(buf, is_extended, is_utf8); + } + } + } + _ => part.write_leaf(buf, parent, empty_body, is_extended, is_utf8), + } + } + + let Some(frame) = stack.top() else { + return; + }; + let Some(part) = meta.part(frame.part) else { + stack.pop(); + continue; + }; + if frame.next != MESSAGE_FRAME + && let Some(child) = part.child(frame.next as usize) + { + frame.next += 1; + next = Some((child, frame.part)); + continue; + } + let has_children = frame.next != 0; + stack.pop(); + if part.is_message() { + part.write_message_tail(buf, is_extended, is_utf8); + } else { + if !has_children { + buf.extend_from_slice(empty_body); + } + part.write_multipart_tail(buf, is_extended, is_utf8); + } + } + } +} + +trait ImapEnvelope { + fn write_imap(&self, buf: &mut Vec, is_utf8: bool); +} + +impl ImapEnvelope for EnvelopeView<'_> { + fn write_imap(&self, buf: &mut Vec, is_utf8: bool) { + buf.push(b'('); + match self.datetime() { + Some(date) => quoted_rfc2822(buf, &date), + None => quoted_or_literal_encoded_string_or_nil(buf, self.date_raw(), is_utf8), + } + buf.push(b' '); + quoted_or_literal_encoded_string_or_nil(buf, self.subject(), is_utf8); + + let from = self + .addresses(AddressHeader::From, Occurrence::All) + .visible(); + from.write_imap_or(buf, None, is_utf8); + self.addresses(AddressHeader::Sender, Occurrence::All) + .visible() + .or(from) + .write_imap_or(buf, None, is_utf8); + self.addresses(AddressHeader::ReplyTo, Occurrence::All) + .visible() + .or(from) + .write_imap_or(buf, None, is_utf8); + for header in [AddressHeader::To, AddressHeader::Cc, AddressHeader::Bcc] { + self.addresses(header, Occurrence::All) + .visible() + .write_imap_or(buf, Some(b"NIL"), is_utf8); + } + + let in_reply_to = self.in_reply_to(); + buf.push(b' '); + quoted_or_literal_raw_string_or_nil( + buf, + if in_reply_to.is_present() && in_reply_to.is_empty() { + Some("") + } else { + in_reply_to.joined_bracketed() + }, + is_utf8, + ); + let message_id = self.message_id(); + buf.push(b' '); + quoted_or_literal_raw_string_or_nil( + buf, + if message_id.is_present() && message_id.is_empty() { + Some("") + } else { + message_id.last_bracketed() + }, + is_utf8, + ); + buf.push(b')'); + } +} + +trait ImapAddresses: Sized { + fn visible(self) -> Option; + fn write_imap(&self, buf: &mut Vec, is_utf8: bool); +} + +impl ImapAddresses for Addresses<'_> { + fn visible(self) -> Option { + self.has_imap_address().then_some(self) + } + + fn write_imap(&self, buf: &mut Vec, is_utf8: bool) { + buf.push(b'('); + for item in self.iter() { + match item { + AddressItem::Mailbox(mailbox) => mailbox.write_imap(buf, is_utf8), + AddressItem::Group(group) => { + buf.extend_from_slice(b"(NIL NIL "); + if let Some(name) = group.name { + quoted_or_literal_encoded_string(buf, name, is_utf8); + } else { + buf.extend_from_slice(b"\"\""); + } + buf.extend_from_slice(b" NIL)"); + for mailbox in group.members.mailboxes() { + mailbox.write_imap(buf, is_utf8); + } + buf.extend_from_slice(b"(NIL NIL NIL NIL)"); + } + } + } + buf.push(b')'); + } +} + +trait ImapAddressField { + fn write_imap_or(self, buf: &mut Vec, fallback: Option<&[u8]>, is_utf8: bool); +} + +impl ImapAddressField for Option> { + fn write_imap_or(self, buf: &mut Vec, fallback: Option<&[u8]>, is_utf8: bool) { + buf.push(b' '); + match (self, fallback) { + (Some(addresses), _) => addresses.write_imap(buf, is_utf8), + (None, Some(fallback)) => buf.extend_from_slice(fallback), + (None, None) => { + buf.push(b'('); + DUMMY_ADDRESS.write_imap(buf, is_utf8); + buf.push(b')'); + } + } + } +} + +trait ImapMailbox { + fn write_imap(&self, buf: &mut Vec, is_utf8: bool); +} + +impl ImapMailbox for Mailbox<'_> { + fn write_imap(&self, buf: &mut Vec, is_utf8: bool) { + let Some(address) = self.address else { + return; + }; + buf.push(b'('); + if let Some(name) = self.name { + quoted_or_literal_encoded_string(buf, name, is_utf8); + } else { + buf.extend_from_slice(b"NIL"); + } + + let addr = if let Some((route, addr)) = address.split_once(':') { + buf.push(b' '); + quoted_or_literal_raw_string(buf, route, is_utf8); + buf.push(b' '); + addr + } else { + buf.extend_from_slice(b" NIL "); + address + }; + + if let Some((local, host)) = addr.rsplit_once('@') { + quoted_or_literal_raw_string(buf, local, is_utf8); + buf.push(b' '); + quoted_or_literal_raw_string(buf, host, is_utf8); + } else { + quoted_or_literal_raw_string(buf, address, is_utf8); + buf.extend_from_slice(b" \"\""); + } + buf.push(b')'); + } +} + +trait ImapContentType { + fn write_parameters(&self, buf: &mut Vec, is_utf8: bool); + fn message_subtype(&self) -> Option<&str>; +} + +impl ImapContentType for ContentTypeView<'_> { + fn message_subtype(&self) -> Option<&str> { + self.subtype().filter(|subtype| { + self.ctype().eq_ignore_ascii_case("message") && subtype.is_message_subtype() + }) + } + + fn write_parameters(&self, buf: &mut Vec, is_utf8: bool) { + for (pos, (key, value)) in self.attributes().enumerate() { + if pos > 0 { + buf.push(b' '); + } + quoted_or_literal_raw_string(buf, key, is_utf8); + buf.push(b' '); + quoted_or_literal_encoded_string(buf, value, is_utf8); + } + } +} + +#[derive(Clone, Copy)] +enum LeafMedia<'a> { + Text { + subtype: &'a str, + params: Option>, + }, + Message { + subtype: &'a str, + params: Option>, + }, + Basic { + ctype: &'a str, + subtype: &'a str, + params: ContentTypeView<'a>, + }, +} + +trait ImapPart { + fn is_digest(&self) -> bool; + fn leaf_media(&self, parent: u32) -> LeafMedia<'_>; + fn write_fields( + &self, + buf: &mut Vec, + params: Option>, + default_params: &[u8], + is_utf8: bool, + ); + fn write_extension(&self, buf: &mut Vec, is_utf8: bool); + fn write_leaf( + &self, + buf: &mut Vec, + parent: u32, + empty_body: &[u8], + is_extended: bool, + is_utf8: bool, + ); + fn write_message_head( + &self, + buf: &mut Vec, + subtype: &str, + params: Option>, + envelope: Option>, + is_utf8: bool, + ); + fn write_message_tail(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool); + fn write_multipart_tail(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool); +} + +impl ImapPart for PartView<'_> { + fn is_digest(&self) -> bool { + self.content_type() + .and_then(|content_type| content_type.subtype()) + .is_some_and(|subtype| subtype.eq_ignore_ascii_case("digest")) + } + + fn leaf_media(&self, parent: u32) -> LeafMedia<'_> { + let Some(content_type) = self.content_type() else { + return if self + .metadata() + .part(parent) + .is_some_and(|parent| parent.is_digest()) + { + LeafMedia::Message { + subtype: DEFAULT_MESSAGE_SUBTYPE, + params: None, + } + } else { + LeafMedia::Text { + subtype: DEFAULT_TEXT_SUBTYPE, + params: None, + } + }; + }; + let ctype = content_type.ctype(); + let Some(subtype) = content_type + .subtype() + .filter(|subtype| !subtype.is_empty() && !ctype.is_empty()) + else { + return LeafMedia::Text { + subtype: DEFAULT_TEXT_SUBTYPE, + params: None, + }; + }; + if ctype.eq_ignore_ascii_case("text") { + LeafMedia::Text { + subtype, + params: Some(content_type), + } + } else if ctype.eq_ignore_ascii_case("multipart") { + LeafMedia::Text { + subtype: DEFAULT_TEXT_SUBTYPE, + params: Some(content_type), + } + } else if ctype.eq_ignore_ascii_case("message") && subtype.is_message_subtype() { + LeafMedia::Message { + subtype, + params: Some(content_type), + } + } else if !ctype.is_media_token() || !subtype.is_media_token() { + LeafMedia::Text { + subtype: DEFAULT_TEXT_SUBTYPE, + params: None, + } + } else { + LeafMedia::Basic { + ctype, + subtype, + params: content_type, + } + } + } + + fn write_fields( + &self, + buf: &mut Vec, + params: Option>, + default_params: &[u8], + is_utf8: bool, + ) { + match params.filter(|content_type| content_type.has_attributes()) { + Some(content_type) => { + buf.push(b'('); + content_type.write_parameters(buf, is_utf8); + buf.push(b')'); + } + None => buf.extend_from_slice(default_params), + } + buf.push(b' '); + quoted_or_literal_encoded_string_or_nil(buf, self.content_id_bracketed(), is_utf8); + buf.push(b' '); + quoted_or_literal_encoded_string_or_nil(buf, self.content_description(), is_utf8); + buf.push(b' '); + quoted_or_literal_encoded_string( + buf, + self.content_transfer_encoding() + .unwrap_or(DEFAULT_TRANSFER_ENCODING), + is_utf8, + ); + buf.push(b' '); + push_int(buf, self.body_range().len()); + } + + fn write_extension(&self, buf: &mut Vec, is_utf8: bool) { + if let Some(disposition) = self.content_disposition() { + buf.push(b'('); + quoted_or_literal_raw_string(buf, disposition.ctype(), is_utf8); + if disposition.has_attributes() { + buf.extend_from_slice(b" ("); + disposition.write_parameters(buf, is_utf8); + buf.extend_from_slice(b"))"); + } else { + buf.extend_from_slice(b" NIL)"); + } + } else { + buf.extend_from_slice(b"NIL"); + } + let languages = self.content_language(); + match languages.len() { + 0 => buf.extend_from_slice(b" NIL"), + 1 => { + buf.push(b' '); + quoted_or_literal_raw_string( + buf, + languages.iter().next().unwrap_or_default(), + is_utf8, + ); + } + _ => { + buf.extend_from_slice(b" ("); + for (pos, language) in languages.iter().enumerate() { + if pos > 0 { + buf.push(b' '); + } + quoted_or_literal_raw_string(buf, language, is_utf8); + } + buf.push(b')'); + } + } + buf.push(b' '); + quoted_or_literal_raw_string_or_nil(buf, self.content_location(), is_utf8); + } + + fn write_leaf( + &self, + buf: &mut Vec, + parent: u32, + empty_body: &[u8], + is_extended: bool, + is_utf8: bool, + ) { + buf.push(b'('); + match self.leaf_media(parent) { + LeafMedia::Text { subtype, params } => { + buf.extend_from_slice(b"\"text\" "); + quoted_or_literal_raw_string(buf, subtype, is_utf8); + buf.push(b' '); + self.write_fields(buf, params, DEFAULT_TEXT_PARAMS, is_utf8); + buf.push(b' '); + push_int(buf, self.lines()); + } + LeafMedia::Message { subtype, params } => { + self.write_message_head(buf, subtype, params, None, is_utf8); + buf.extend_from_slice(empty_body); + buf.push(b' '); + push_int(buf, self.lines()); + } + LeafMedia::Basic { + ctype, + subtype, + params, + } => { + quoted_or_literal_raw_string(buf, ctype, is_utf8); + buf.push(b' '); + quoted_or_literal_raw_string(buf, subtype, is_utf8); + buf.push(b' '); + self.write_fields(buf, Some(params), b"NIL", is_utf8); + } + } + if is_extended { + buf.push(b' '); + quoted_or_literal_raw_string_or_nil(buf, self.content_md5(), is_utf8); + buf.push(b' '); + self.write_extension(buf, is_utf8); + } + buf.push(b')'); + } + + fn write_message_head( + &self, + buf: &mut Vec, + subtype: &str, + params: Option>, + envelope: Option>, + is_utf8: bool, + ) { + buf.extend_from_slice(b"\"message\" "); + quoted_or_literal_raw_string(buf, subtype, is_utf8); + buf.push(b' '); + self.write_fields(buf, params, b"NIL", is_utf8); + buf.push(b' '); + match envelope { + Some(envelope) => envelope.write_imap(buf, is_utf8), + None => buf.extend_from_slice(EMPTY_ENVELOPE), + } + buf.push(b' '); + } + + fn write_message_tail(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool) { + buf.push(b' '); + push_int(buf, self.lines()); + if is_extended { + buf.push(b' '); + quoted_or_literal_raw_string_or_nil(buf, self.content_md5(), is_utf8); + buf.push(b' '); + self.write_extension(buf, is_utf8); + } + buf.push(b')'); + } + + fn write_multipart_tail(&self, buf: &mut Vec, is_extended: bool, is_utf8: bool) { + let content_type = self.content_type(); + buf.push(b' '); + quoted_or_literal_raw_string( + buf, + content_type + .and_then(|content_type| content_type.subtype()) + .filter(|subtype| !subtype.is_empty()) + .unwrap_or(DEFAULT_MULTIPART_SUBTYPE), + is_utf8, + ); + if is_extended { + match content_type.filter(|content_type| content_type.has_attributes()) { + Some(content_type) => { + buf.extend_from_slice(b" ("); + content_type.write_parameters(buf, is_utf8); + buf.push(b')'); + } + None => buf.extend_from_slice(b" NIL"), + } + buf.push(b' '); + self.write_extension(buf, is_utf8); + } + buf.push(b')'); + } +} + +trait MediaToken { + fn is_message_subtype(&self) -> bool; + fn is_media_token(&self) -> bool; +} + +impl MediaToken for str { + fn is_message_subtype(&self) -> bool { + self.eq_ignore_ascii_case("rfc822") || self.eq_ignore_ascii_case("global") + } + + fn is_media_token(&self) -> bool { + !self.is_empty() && !self.contains('\0') + } +} + diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index a1a0eb7a..38d50a0e 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -5,11 +5,19 @@ */ use crate::{ - core::{SelectedMailbox, Session, SessionData, State}, + core::{AccountCaches, SelectedAccess, SelectedMailbox, Session, SessionData, State}, op::ImapContext, }; -use ahash::AHashSet; -use common::{ipc::PushNotification, network::SessionStream}; +use common::{ + auth::AccessToken, + ipc::{ + PushNotification, + subscribers::{PushReceiver, PushSubscribers}, + }, + network::SessionStream, +}; +use compact_str::CompactString; +use email::cache::MessageCacheFetch; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -20,15 +28,26 @@ use imap_proto::{ receiver::Request, }; use registry::schema::enums::Permission; -use std::{sync::Arc, time::Instant}; -use store::query::log::Query; -use tokio::io::AsyncReadExt; +use std::{ + future::pending, + pin::pin, + sync::Arc, + task::{Context, Poll, Waker}, + time::{Duration, Instant}, +}; +use tokio::{io::AsyncReadExt, sync::OwnedSemaphorePermit}; use trc::AddContext; -use types::{collection::SyncCollection, type_state::DataType}; +use types::type_state::DataType; use utils::map::bitmap::Bitmap; +const SHARED_STATUS_INTERVAL: Duration = Duration::from_secs(1); + impl Session { - pub async fn handle_idle(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_idle( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapIdle)?; @@ -47,12 +66,26 @@ impl Session { let is_rev2 = self.version.is_rev2(); let is_utf8 = self.is_utf8; let is_uidonly = self.is_uidonly; + let is_objectid = self.is_objectid; let use_vanished = self.is_qresync || is_uidonly; // Register with push manager - let mut push_rx = self - .server - .subscribe_push_manager(&data.access_token, types) + let primary_id = mailbox + .as_ref() + .map_or(data.account_id, |mailbox| mailbox.id.account_id); + let view_sync = if data.is_acl_shared(primary_id) { + ViewSync::SyncIfReadable + } else { + ViewSync::Sync + }; + let (mut primary_rx, mut shared_rx) = subscribe_idle( + &self.server.inner.ipc.push_subscribers, + &data.access_token, + primary_id, + types, + ); + + data.seed_mailbox_counters() .await .imap_ctx(&request.tag, trc::location!())?; @@ -67,109 +100,321 @@ impl Session { ); let op_start = Instant::now(); - let mut buf = vec![0; 4]; + let timeout_idle = self.server.core.imap.timeout_idle; + let mut window = [0u8; 4]; + let mut byte = [0u8; 1]; + let mut notifications = Vec::new(); + let mut shared_notifications = Vec::new(); + let mut schedule = PassSchedule::Paced(op_start); loop { - tokio::select! { - result = tokio::time::timeout(self.server.core.imap.timeout_idle, self.stream_rx.read_exact(&mut buf)) => { + let wait = schedule.wait(timeout_idle); + let wake = tokio::select! { + result = tokio::time::timeout(wait, self.stream_rx.read(&mut byte)) => { match result { - Ok(Ok(bytes_read)) => { - if bytes_read > 0 { - if (buf[..bytes_read]).windows(4).any(|w| w == b"DONE") { - trc::event!(Imap(trc::ImapEvent::IdleStop), SpanId = self.session_id, Elapsed = op_start.elapsed()); - return self.write_bytes(StatusResponse::completed(Command::Idle) - .with_tag(request.tag) - .into_bytes()).await; + Ok(Ok(0)) => { + return Err(trc::NetworkEvent::Closed.into_err().details("IMAP connection closed by client.").id(request.tag)); + } + Ok(Ok(_)) => { + window.rotate_left(1); + window[3] = byte[0]; + if &window == b"DONE" { + if schedule.is_deferred() + && let Err(err) = data.write_changes(mailbox.as_ref(), true, ViewSync::Skip, use_vanished, is_uidonly, is_rev2, is_utf8, is_objectid).await + { + trc::error!(err.span_id(self.session_id)); } - } else { - return Err(trc::NetworkEvent::Closed.into_err().details("IMAP connection closed by client.").id(request.tag)); + trc::event!(Imap(trc::ImapEvent::IdleStop), SpanId = self.session_id, Elapsed = op_start.elapsed()); + return self.write_bytes(StatusResponse::completed(Command::Idle) + .with_tag(request.tag) + .into_bytes()).await; } + continue; }, Ok(Err(err)) => { return Err(trc::NetworkEvent::ReadError.into_err().reason(err).details("IMAP connection error.").id(request.tag)); }, + Err(_) if wait < timeout_idle => Wake::Due, Err(_) => { self.write_bytes(&b"* BYE IDLE timed out.\r\n"[..]).await.ok(); return Err(trc::NetworkEvent::Timeout.into_err().details("IMAP IDLE timed out.").id(request.tag)); } } } - push_notification = push_rx.recv() => { - if let Some(push_notification) = push_notification { - let mut has_mailbox_changes = false; - let mut has_email_changes = false; + received = primary_rx.recv(&mut notifications) => Wake::received(received), + received = recv_shared(&mut shared_rx, &mut shared_notifications), if !schedule.is_deferred() => Wake::received(received), + }; - match push_notification { - PushNotification::StateChange(state_change) => { - for type_state in state_change.types { - match type_state { - DataType::Email | DataType::EmailDelivery => { - has_email_changes = true; - } - DataType::Mailbox => { - has_mailbox_changes = true; - } - _ => {} - } - } - }, - PushNotification::EmailPush(_) => { - has_email_changes = true; - has_mailbox_changes = true; - }, - PushNotification::CalendarAlert(_) => (), - } + let mut check_mailboxes = match wake { + Wake::Notified => false, + Wake::Due => true, + Wake::Closed => return Err(self.idle_closed(request.tag).await), + }; + let mut changes = IdleChanges::default(); + for notification in notifications.drain(..) { + changes.add(¬ification); + } + let mut shared = IdleChanges::default(); + for notification in shared_notifications.drain(..) { + shared.add(¬ification); + } + check_mailboxes |= + changes.mailboxes || (shared.mailboxes && schedule.admits_shared(Instant::now())); + if check_mailboxes && let Some(shared_rx) = &mut shared_rx { + if !take_pending(shared_rx, &mut shared_notifications) { + return Err(self.idle_closed(request.tag).await); + } + shared_notifications.clear(); + } - if has_mailbox_changes || has_email_changes { - data.write_changes(&mailbox, has_mailbox_changes, has_email_changes, use_vanished, is_uidonly, is_rev2, is_utf8).await?; - } - } else { - self.write_bytes(&b"* BYE Server shutting down.\r\n"[..]).await.ok(); - return Err(trc::NetworkEvent::Closed.into_err().details("IDLE channel closed.").id(request.tag)); + let view = changes.view_sync(view_sync); + if check_mailboxes || view != ViewSync::Skip { + data.write_changes( + mailbox.as_ref(), + check_mailboxes, + view, + use_vanished, + is_uidonly, + is_rev2, + is_utf8, + is_objectid, + ) + .await?; + } + if check_mailboxes && shared_rx.is_some() { + schedule.passed(Instant::now()); + } + } + } + + async fn idle_closed(&self, tag: CompactString) -> trc::Error { + self.write_bytes(&b"* BYE Server shutting down.\r\n"[..]) + .await + .ok(); + trc::NetworkEvent::Closed + .into_err() + .details("IDLE channel closed.") + .id(tag) + } +} + +fn subscribe_idle( + subscribers: &Arc, + access_token: &AccessToken, + primary_id: u32, + types: Bitmap, +) -> (PushReceiver, Option) { + let shared_rx = access_token + .member_ids() + .any(|account_id| account_id != primary_id) + .then(|| { + subscribers.subscribe( + access_token + .member_ids() + .filter(|account_id| *account_id != primary_id), + Bitmap::from_iter([DataType::Mailbox]), + ) + }); + (subscribers.subscribe([primary_id], types), shared_rx) +} + +async fn recv_shared( + shared_rx: &mut Option, + notifications: &mut Vec, +) -> bool { + match shared_rx { + Some(shared_rx) => shared_rx.recv(notifications).await, + None => pending().await, + } +} + +fn take_pending(rx: &mut PushReceiver, notifications: &mut Vec) -> bool { + match pin!(rx.recv(notifications)).poll(&mut Context::from_waker(Waker::noop())) { + Poll::Ready(is_open) => is_open, + Poll::Pending => true, + } +} + +enum Wake { + Notified, + Due, + Closed, +} + +impl Wake { + fn received(received: bool) -> Self { + if received { + Wake::Notified + } else { + Wake::Closed + } + } +} + +#[derive(Default)] +struct IdleChanges { + mailboxes: bool, + emails: bool, +} + +impl IdleChanges { + fn add(&mut self, notification: &PushNotification) { + match notification { + PushNotification::StateChange(state_change) => { + for type_state in state_change.types { + match type_state { + DataType::Email | DataType::EmailDelivery => self.emails = true, + DataType::Mailbox => self.mailboxes = true, + _ => {} } } } + PushNotification::EmailPush(_) => { + self.emails = true; + self.mailboxes = true; + } + PushNotification::ViewerStateChange(_) | PushNotification::CalendarAlert(_) => {} } } + + fn view_sync(&self, sync: ViewSync) -> ViewSync { + match sync { + ViewSync::Sync if self.emails => sync, + ViewSync::SyncIfReadable if self.emails || self.mailboxes => sync, + _ => ViewSync::Skip, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ViewSync { + Skip, + Sync, + SyncIfReadable, +} + +enum PassSchedule { + Paced(Instant), + Deferred(Instant), +} + +impl PassSchedule { + fn is_deferred(&self) -> bool { + matches!(self, PassSchedule::Deferred(_)) + } + + fn wait(&self, timeout: Duration) -> Duration { + match self { + PassSchedule::Deferred(due) => { + due.saturating_duration_since(Instant::now()).min(timeout) + } + PassSchedule::Paced(_) => timeout, + } + } + + fn admits_shared(&mut self, now: Instant) -> bool { + match *self { + PassSchedule::Paced(due) | PassSchedule::Deferred(due) if due > now => { + *self = PassSchedule::Deferred(due); + false + } + _ => true, + } + } + + fn passed(&mut self, now: Instant) { + *self = PassSchedule::Paced(now + SHARED_STATUS_INTERVAL); + } } impl SessionData { #[allow(clippy::too_many_arguments)] pub async fn write_changes( &self, - mailbox: &Option>, + mailbox: Option<&Arc>, check_mailboxes: bool, - check_emails: bool, + view: ViewSync, use_vanished: bool, is_uidonly: bool, is_rev2: bool, is_utf8: bool, + is_objectid: bool, ) -> trc::Result<()> { + // Synchronize the selected mailbox first so the account counters are current + let mut changed_uids = Vec::new(); + let mut access_token = None; + let cache = match (mailbox, view) { + (Some(mailbox), ViewSync::Sync) => Some( + self.server + .get_cached_messages(mailbox.id.account_id) + .await + .caused_by(trc::location!())?, + ), + (Some(mailbox), ViewSync::SyncIfReadable) => { + match self + .selected_access(&mailbox.id) + .await + .caused_by(trc::location!())? + { + SelectedAccess::Readable { + access_token: token, + cache, + } => { + access_token = Some(token); + Some(cache) + } + SelectedAccess::Revoked => return Err(self.close_revoked().await), + } + } + _ => None, + }; + if let (Some(mailbox), Some(cache)) = (mailbox, &cache) { + self.sync_view(mailbox, cache, Some(&mut changed_uids)) + .await + .caused_by(trc::location!())?; + } + // Fetch all changed mailboxes if check_mailboxes { - let changes = self - .synchronize_mailboxes(true) - .await - .caused_by(trc::location!())? - .unwrap(); + let caches = match (mailbox, &cache) { + (Some(mailbox), Some(cache)) => { + AccountCaches::with(mailbox.id.account_id, cache.clone()) + } + _ => AccountCaches::default(), + }; + let refresh = match access_token { + Some(access_token) => { + self.synchronize_mailboxes_as(access_token, true, caches) + .await + } + None => self.synchronize_mailboxes_using(true, caches).await, + } + .caused_by(trc::location!())?; + let changes = refresh.changes.unwrap_or_default(); + let mut caches = refresh.caches; + let access_token = refresh.access_token; let mut buf = Vec::with_capacity(64); // List deleted mailboxes - for mailbox_name in changes.deleted { + for change in changes.deleted { ListItem { - mailbox_name, + mailbox_name: change.name.into(), attributes: vec![Attribute::NonExistent], tags: vec![], + objectid: is_objectid.then_some(change.objectid), + metadata: None, } .serialize(&mut buf, is_rev2, is_utf8, false); } // List added mailboxes - for mailbox_name in changes.added { + for change in changes.added { ListItem { - mailbox_name, + mailbox_name: change.name.into(), attributes: vec![], tags: vec![], + objectid: is_objectid.then_some(change.objectid), + metadata: None, } .serialize(&mut buf, is_rev2, is_utf8, false); } @@ -177,7 +422,9 @@ impl SessionData { for mailbox_name in changes.changed { if let Ok(status) = self .status( - mailbox_name, + &mut caches, + &access_token, + mailbox_name.into(), &[ Status::Messages, Status::Unseen, @@ -197,78 +444,46 @@ impl SessionData { } // Fetch selected mailbox changes - if check_emails { - // Synchronize emails - if let Some(mailbox) = mailbox { - // Obtain changes since last sync - let modseq = mailbox.state.lock().modseq; - let new_state = self - .write_mailbox_changes(mailbox, use_vanished) - .await - .caused_by(trc::location!())?; - if new_state == modseq { - return Ok(()); - } + if let (Some(mailbox), Some(cache)) = (mailbox, cache) { + self.flush_view(mailbox, use_vanished) + .await + .caused_by(trc::location!())?; - // Obtain changed messages - let changelog = self - .server - .store() - .changes( - mailbox.id.account_id, - SyncCollection::Email.into(), - Query::Since(modseq), + if !changed_uids.is_empty() { + changed_uids.sort_unstable(); + changed_uids.dedup(); + let op_start = Instant::now(); + return self + .fetch( + fetch::Arguments { + tag: "".into(), + sequence_set: Sequence::List { + items: changed_uids + .into_iter() + .map(|uid| Sequence::Number { value: uid }) + .collect(), + }, + attributes: vec![fetch::Attribute::Flags, fetch::Attribute::Uid], + changed_since: None, + include_vanished: false, + }, + mailbox.clone(), + Some(cache), + true, + use_vanished, + is_uidonly, + false, + is_utf8, + u32::MAX, + op_start, ) .await - .caused_by(trc::location!())?; - let changed_ids = { - let state = mailbox.state.lock(); - changelog - .changes - .into_iter() - .filter_map(|change| { - change.try_unwrap_item_id().and_then(|item_id| { - state - .id_to_imap - .get(&((item_id & u32::MAX as u64) as u32)) - .map(|id| id.uid) - }) - }) - .collect::>() - }; - - if !changed_ids.is_empty() { - let op_start = Instant::now(); - return self - .fetch( - fetch::Arguments { - tag: "".into(), - sequence_set: Sequence::List { - items: changed_ids - .into_iter() - .map(|uid| Sequence::Number { value: uid }) - .collect(), - }, - attributes: vec![fetch::Attribute::Flags, fetch::Attribute::Uid], - changed_since: None, - include_vanished: false, - }, - mailbox.clone(), - true, - use_vanished, - is_uidonly, - false, - is_utf8, - u32::MAX, - op_start, - ) - .await - .caused_by(trc::location!()) - .map(|_| ()); - } + .caused_by(trc::location!()) + .map(|_| ()); } } Ok(()) } } + diff --git a/crates/imap/src/op/list.rs b/crates/imap/src/op/list.rs index 6ad81800..b5331436 100644 --- a/crates/imap/src/op/list.rs +++ b/crates/imap/src/op/list.rs @@ -4,18 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - +use super::{ImapContext, metadata::ListMetadataTarget}; use crate::{ - core::{Session, SessionData}, + core::{AccountView, MailboxRefresh, Session, SessionData}, spawn_op, }; +use ahash::AHashSet; use common::network::SessionStream; - +use compact_str::{CompactString, format_compact}; use imap_proto::{ - Command, StatusResponse, + Command, ResponseType, StatusResponse, protocol::{ - ImapResponse, ProtocolVersion, + MailboxObjectId, ProtocolVersion, list::{ self, Arguments, Attribute, ChildInfo, ListItem, ReturnOption, SelectionOption, Tag, }, @@ -23,12 +23,16 @@ use imap_proto::{ receiver::Request, }; use registry::schema::enums::Permission; +use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; use trc::StoreEvent; -use super::ImapContext; - impl Session { - pub async fn handle_list(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_list( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { let op_start = Instant::now(); let command = request.command; let is_lsub = command == Command::Lsub; @@ -44,34 +48,43 @@ impl Session { request.parse_lsub(self.is_utf8) }?; + if arguments.requests_objectid() + && let Some(enabled) = self.activate_objectid() + { + self.write_bytes(enabled).await?; + } + if !arguments.is_separator_query() { let data = self.state.session_data(); let version = self.version; let is_utf8 = self.is_utf8; spawn_op!( + permit, data, data.list(arguments, is_lsub, version, is_utf8, op_start) .await ) } else { + let objectid = arguments + .returns_objectid() + .then_some(MailboxObjectId::Empty); self.write_bytes( StatusResponse::completed(command) .with_tag(arguments.unwrap_tag()) - .serialize( - list::Response { - is_rev2: self.version.is_rev2(), - is_utf8: self.is_utf8, - is_lsub, - list_items: vec![ListItem { - mailbox_name: "".into(), - attributes: vec![Attribute::NoSelect], - tags: vec![], - }], - status_items: Vec::new(), - } - .serialize(), - ), + .serialize_after(&list::Response { + is_rev2: self.version.is_rev2(), + is_utf8: self.is_utf8, + is_lsub, + list_items: vec![ListItem { + mailbox_name: "".into(), + attributes: vec![Attribute::NoSelect], + tags: vec![], + objectid, + metadata: None, + }], + status_items: Vec::new(), + }), ) .await } @@ -116,7 +129,12 @@ impl SessionData { }; // Refresh mailboxes - self.synchronize_mailboxes(false) + let MailboxRefresh { + mut caches, + access_token, + .. + } = self + .synchronize_mailboxes(false) .await .imap_ctx(&tag, trc::location!())?; @@ -128,6 +146,8 @@ impl SessionData { let mut include_subscribed = false; let mut include_children = false; let mut include_status = None; + let mut include_metadata = None; + let mut include_objectid = false; for selection_option in &selection_options { match selection_option { SelectionOption::Subscribed => { @@ -158,23 +178,44 @@ impl SessionData { ReturnOption::SpecialUse => { include_special_use = true; } + ReturnOption::Metadata(entries) => { + include_metadata = Some(entries.as_slice()); + } + ReturnOption::ObjectId => { + include_objectid = true; + } } } if recursive_match && !filter_subscribed { return Err(trc::ImapEvent::Error .into_err() .details("RECURSIVEMATCH requires the SUBSCRIBED selection option.") + .ctx(trc::Key::Type, ResponseType::Bad) .id(tag)); } // Append reference name if !patterns.is_empty() && !reference_name.is_empty() { patterns.iter_mut().for_each(|item| { - *item = format!("{}{}", reference_name, item); + *item = format_compact!("{}{}", reference_name, item); }) } + let metadata_access = match include_metadata { + Some(entries) => { + self.assert_metadata_request(&tag, entries.len())?; + access_token + .enforce_permission(Permission::ImapMetadataGet) + .map_err(|err| err.id(tag.clone()))?; + Some(access_token.clone()) + } + None => None, + }; + let mut metadata_targets = Vec::new(); + let mut list_items = Vec::with_capacity(10); + let mut lsub_unmatched = Vec::new(); + let mut child_prefix = String::new(); // Add mailboxes let mut added_shared_folder = false; @@ -192,100 +233,207 @@ impl SessionData { vec![Attribute::NoSelect] }, tags: vec![], + objectid: include_objectid.then_some(MailboxObjectId::Empty), + metadata: None, }); } added_shared_folder = true; } if !filter_subscribed && matches_pattern(&patterns, prefix) { list_items.push(ListItem { - mailbox_name: prefix.clone(), + mailbox_name: prefix.as_str().into(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { vec![Attribute::NoSelect] }, tags: vec![], + objectid: include_objectid.then_some(MailboxObjectId::Empty), + metadata: None, }); } } - for (mailbox_name, mailbox_id) in &account.mailbox_names { + for entry in &account.names { + let mailbox_name = &entry.name; + let mailbox_id = entry.mailbox_id; if matches_pattern(&patterns, mailbox_name) { - let mailbox = if let Some(mailbox) = account.mailbox_state.get(mailbox_id) { - mailbox - } else { + let Some(mailbox) = account.mailbox(mailbox_id) else { trc::event!( Store(StoreEvent::UnexpectedError), Details = "IMAP mailbox no longer present in account state", - Id = *mailbox_id, - Details = account - .mailbox_state - .keys() - .copied() - .map(trc::Value::from) - .collect::>() + MailboxId = mailbox_id, ); continue; }; - let mut has_recursive_match = false; - if recursive_match { - let prefix = format!("{}/", mailbox_name); - for (mailbox_name, mailbox_id) in &account.mailbox_names { - if mailbox_name.starts_with(&prefix) - && account.mailbox_state.get(mailbox_id).unwrap().is_subscribed - { - has_recursive_match = true; - break; - } - } - } - if !filter_subscribed || mailbox.is_subscribed || has_recursive_match { + let is_subscribed = mailbox.subscribers.contains(&self.account_id); + let has_recursive_match = recursive_match && { + child_prefix.clear(); + child_prefix.push_str(mailbox_name); + child_prefix.push('/'); + let start = account + .names + .partition_point(|child| child.name.as_str() < child_prefix.as_str()); + account + .names + .get(start..) + .unwrap_or_default() + .iter() + .take_while(|child| child.name.starts_with(child_prefix.as_str())) + .any(|child| account.is_subscribed(child.mailbox_id, self.account_id)) + }; + if !filter_subscribed || is_subscribed || has_recursive_match { let mut attributes = Vec::with_capacity(2); if include_children { - attributes.push(if mailbox.has_children { + attributes.push(if account.has_children(mailbox_id) { Attribute::HasChildren } else { Attribute::HasNoChildren }); } - if include_subscribed && mailbox.is_subscribed { + if include_subscribed && is_subscribed { attributes.push(Attribute::Subscribed); } if include_special_use { - if let Some(special_use) = &mailbox.special_use { - attributes.push(*special_use); + if let Some(special_use) = + AccountView::special_use_attribute(&mailbox.role) + { + attributes.push(special_use); } else if filter_special_use { continue; } } + if let Some(access_token) = &metadata_access + && (!filter_subscribed || is_subscribed) + && let Some(target) = ListMetadataTarget::new( + list_items.len(), + account.account_id, + mailbox_id, + mailbox, + access_token, + ) + { + metadata_targets.push(target); + } list_items.push(ListItem { - mailbox_name: mailbox_name.clone(), + mailbox_name: mailbox_name.as_str().into(), attributes, tags: if !has_recursive_match { vec![] } else { vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])] }, + objectid: include_objectid.then(|| entry.object_id(account.account_id)), + metadata: None, }); } + } else if is_lsub && account.is_subscribed(mailbox_id, self.account_id) { + lsub_unmatched.push(mailbox_name.clone()); + } + } + } + + // RFC 3501 6.3.9 requires LSUB with % to report the unsubscribed parents + // of a subscribed mailbox, flagged \Noselect + if !lsub_unmatched.is_empty() { + let mut emitted = list_items + .iter() + .map(|item| item.mailbox_name.as_str()) + .collect::>(); + let mut parents = Vec::new(); + for mailbox_name in &lsub_unmatched { + for (pos, _) in mailbox_name.match_indices('/') { + if let Some(parent) = mailbox_name.get(..pos) + && matches_pattern(&patterns, parent) + && emitted.insert(parent) + { + parents.push(parent); + } } } + list_items.extend(parents.into_iter().map(|parent| ListItem { + mailbox_name: parent.into(), + attributes: vec![Attribute::NoSelect], + tags: vec![], + objectid: include_objectid.then_some(MailboxObjectId::Empty), + metadata: None, + })); + } + + if let (Some(requested), Some(access_token)) = (include_metadata, &metadata_access) + && !metadata_targets.is_empty() + { + self.list_metadata( + &tag, + requested, + &metadata_targets, + &mut list_items, + access_token, + ) + .await?; + } + + // RFC 5258 3.5 requires redundant CHILDINFO responses to be suppressed + if recursive_match { + let mut names = list_items + .iter() + .map(|item| item.mailbox_name.as_str()) + .collect::>(); + names.sort_unstable(); + let redundant = list_items + .iter() + .map(|item| { + item.tags.iter().any(|tag| matches!(tag, Tag::ChildInfo(_))) && { + child_prefix.clear(); + child_prefix.push_str(item.mailbox_name.as_str()); + child_prefix.push('/'); + let pos = names.partition_point(|name| *name < child_prefix.as_str()); + names + .get(pos) + .is_some_and(|name| name.starts_with(child_prefix.as_str())) + } + }) + .collect::>(); + + let mut redundant = redundant.into_iter(); + list_items.retain_mut(|item| match redundant.next() { + Some(true) => { + item.tags.clear(); + item.attributes.contains(&Attribute::Subscribed) + } + _ => true, + }); } // Add status response let mut status_items = Vec::new(); if let Some(include_status) = include_status { - for list_item in &list_items { + for list_item in &mut list_items { + let Some(mailbox) = self.get_mailbox_by_name(&list_item.mailbox_name) else { + continue; + }; + let cache = caches + .fetch(&self.server, mailbox.account_id) + .await + .imap_ctx(&tag, trc::location!())?; match self - .status(list_item.mailbox_name.clone(), include_status) + .status_in( + &cache, + &access_token, + mailbox, + list_item.mailbox_name.clone(), + include_status, + ) .await - .imap_ctx(&tag, trc::location!()) { Ok(status_item) => { status_items.push(status_item); } Err(err) => { - self.write_error(err).await?; + if err.matches(trc::EventType::Imap(trc::ImapEvent::Error)) { + list_item.attributes.push(Attribute::NoSelect); + } + trc::error!(err.span_id(self.session_id)); } } } @@ -313,23 +461,20 @@ impl SessionData { Command::Lsub }) .with_tag(tag) - .serialize( - list::Response { - is_rev2: version.is_rev2(), - is_utf8, - is_lsub, - list_items, - status_items, - } - .serialize(), - ), + .serialize_after(&list::Response { + is_rev2: version.is_rev2(), + is_utf8, + is_lsub, + list_items, + status_items, + }), ) .await } } #[allow(clippy::while_let_on_iterator)] -pub fn matches_pattern(patterns: &[String], mailbox_name: &str) -> bool { +pub fn matches_pattern(patterns: &[CompactString], mailbox_name: &str) -> bool { if patterns.is_empty() { return true; } diff --git a/crates/imap/src/op/login.rs b/crates/imap/src/op/login.rs index f3dec4ae..c1682c44 100644 --- a/crates/imap/src/op/login.rs +++ b/crates/imap/src/op/login.rs @@ -20,6 +20,7 @@ impl Session { mfa_token: None, }, arguments.tag, + None, ) .await } diff --git a/crates/imap/src/op/metadata/edit.rs b/crates/imap/src/op/metadata/edit.rs new file mode 100644 index 00000000..27bf59ce --- /dev/null +++ b/crates/imap/src/op/metadata/edit.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::storage::metadata::ContainerChange; +use imap_proto::protocol::metadata::MetadataCode; +use std::borrow::Cow; +use store::write::metadata::MetadataBuf; +use types::metadata::{EntryBound, LimitViolation, MetadataBuilder, MetadataLimits, MetadataScope}; + +struct ContainerEdit<'x> { + builder: MetadataBuilder<'x>, + previous_len: usize, + previous_entries: usize, + entry_bound: EntryBound, + largest: usize, + changed: bool, +} + +pub(crate) fn edit_container<'x>( + previous: Option<&'x MetadataBuf>, + entries: impl Iterator)> + Clone, + scope: MetadataScope, + limits: &MetadataLimits, +) -> Result, MetadataCode> { + let mut edit = ContainerEdit::new(previous, limits); + for (name, _) in entries.clone().filter(|(_, value)| value.is_none()) { + edit.remove(name); + } + for (name, value) in entries.filter_map(|(name, value)| value.map(|value| (name, value))) { + edit.set(name, value, limits)?; + } + edit.finish(previous, scope, limits) +} + +impl<'x> ContainerEdit<'x> { + fn new(previous: Option<&'x MetadataBuf>, limits: &MetadataLimits) -> Self { + let view = previous.map(MetadataBuf::view); + let (previous_len, previous_entries) = view + .as_ref() + .map_or((0, 0), |view| (view.as_bytes().len(), view.len())); + ContainerEdit { + builder: view + .as_ref() + .map_or_else(MetadataBuilder::new, MetadataBuilder::from_view), + previous_len, + previous_entries, + entry_bound: limits.entry_bound(previous_entries), + largest: 0, + changed: false, + } + } + + fn remove(&mut self, name: &str) { + self.changed |= self.builder.remove_imap(name); + } + + fn set( + &mut self, + name: &'x str, + value: &'x [u8], + limits: &MetadataLimits, + ) -> Result<(), MetadataCode> { + limits + .check_entry_size(value.len()) + .map_err(|_| MetadataCode::MaxSize(to_u32(limits.max_entry_size)))?; + self.largest = self.largest.max(value.len()); + if self.builder.imap(name) != Some(value) { + self.builder.set_imap(Cow::Borrowed(name), value); + self.changed = true; + self.entry_bound + .check(self.builder.len(), 0) + .map_err(|_| MetadataCode::TooMany)?; + } + Ok(()) + } + + fn finish( + self, + previous: Option<&MetadataBuf>, + scope: MetadataScope, + limits: &MetadataLimits, + ) -> Result, MetadataCode> { + if !self.changed { + return Ok(None); + } + let edit = self.builder.edit(); + let next = self.builder.encode(); + if let Some(next) = &next { + match limits.check_edit(scope, self.previous_len, self.previous_entries, next) { + Ok(()) => {} + Err(LimitViolation::ContainerSize { size, max }) => { + let available = (max + self.largest).saturating_sub(size); + return Err(MetadataCode::MaxSize(to_u32( + available.min(limits.max_entry_size), + ))); + } + Err(_) => return Err(MetadataCode::TooMany), + } + } + Ok(ContainerChange::new(previous, next, edit)) + } +} + +fn to_u32(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + diff --git a/crates/imap/src/op/metadata/get.rs b/crates/imap/src/op/metadata/get.rs new file mode 100644 index 00000000..4ee6f97c --- /dev/null +++ b/crates/imap/src/op/metadata/get.rs @@ -0,0 +1,173 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + READ_RIGHTS, SERVER_DOCUMENT_ID, SPECIAL_USE_ENTRY, reads_private_container, + select::{EntrySource, PrivateEntries, SelectOptions, select_entries}, + special_use_value, +}; +use crate::{core::SessionData, op::ImapContext}; +use common::network::SessionStream; +use imap_proto::{ + Command, ResponseCode, StatusResponse, + protocol::metadata::{GetArguments, MetadataCode, Response, Scope}, +}; +use registry::schema::enums::Permission; +use std::time::Instant; +use types::{collection::Collection, metadata::MetadataKinds, special_use::SpecialUse}; + +const ADMIN_ENTRY: &str = "/admin"; +const COMMENT_ENTRY: &str = "/comment"; + +impl SessionData { + pub async fn get_metadata( + &self, + arguments: GetArguments, + is_utf8: bool, + op_start: Instant, + ) -> trc::Result<()> { + let tag = arguments.tag.as_str(); + self.assert_metadata_request(tag, arguments.entries.len())?; + + let access_token = self + .refresh_access_token() + .await + .imap_ctx(tag, trc::location!())?; + let viewer = self + .server + .metadata_viewer(&access_token, Permission::ImapMetadataPrivate); + let has_shared = arguments + .entries + .iter() + .any(|entry| entry.scope == Scope::Shared); + let has_private = viewer.is_some() + && arguments + .entries + .iter() + .any(|entry| entry.scope == Scope::Private); + + let (shared, private, role, mailbox_id) = if arguments.mailbox_name.is_empty() { + let private = match viewer { + Some(viewer) if has_private => self + .server + .metadata_container( + viewer.account_id(), + Collection::Principal, + SERVER_DOCUMENT_ID, + ) + .await + .imap_ctx(tag, trc::location!())?, + _ => None, + }; + (None, private, SpecialUse::None, None) + } else { + let mailbox = self + .metadata_mailbox(tag, &arguments.mailbox_name, &access_token) + .await?; + mailbox.assert_rights(tag, &READ_RIGHTS)?; + + let shared = if has_shared && mailbox.kinds.contains(MetadataKinds::IMAP) { + self.server + .metadata_container(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) + .await + .imap_ctx(tag, trc::location!())? + } else { + None + }; + let private = match viewer { + Some(viewer) + if reads_private_container(&arguments.entries, arguments.depth) + && self + .server + .metadata_viewer_state(viewer, mailbox.account_id, Collection::Mailbox) + .await + .imap_ctx(tag, trc::location!())? + .containers + > 0 => + { + self.server + .private_metadata_container( + mailbox.account_id, + viewer.account_id(), + Collection::Mailbox, + mailbox.mailbox_id, + ) + .await + .imap_ctx(tag, trc::location!())? + } + _ => None, + }; + (shared, private, mailbox.role, Some(mailbox.id())) + }; + + let shared_source = match &shared { + Some(container) => EntrySource::new(container.view().imap()), + None if arguments.mailbox_name.is_empty() => { + let metadata = &self.server.core.metadata; + EntrySource::new( + [ + (ADMIN_ENTRY, metadata.imap_server_admin.as_deref()), + (COMMENT_ENTRY, metadata.imap_server_comment.as_deref()), + ] + .into_iter() + .filter_map(|(name, value)| value.map(|value| (name, value.as_bytes()))), + ) + } + None => EntrySource::default(), + }; + let mut private_source = private + .as_ref() + .map(|container| EntrySource::new(container.view().imap())) + .unwrap_or_default(); + if let Some(value) = special_use_value(role) { + private_source.insert(SPECIAL_USE_ENTRY, value); + } + let private_entries = match viewer { + Some(_) => PrivateEntries::All(&private_source), + None if arguments.mailbox_name.is_empty() => PrivateEntries::Hidden, + None => PrivateEntries::SpecialUse(&private_source), + }; + + let selection = select_entries( + &arguments.entries, + SelectOptions { + depth: arguments.depth, + max_size: arguments.max_size, + }, + &shared_source, + private_entries, + ); + + let total = selection.entries.len(); + let mut buf = Vec::with_capacity(128); + if !selection.entries.is_empty() { + Response { + mailbox_name: &arguments.mailbox_name, + entries: selection.entries, + } + .serialize(&mut buf, is_utf8); + } + let response = StatusResponse::completed(Command::GetMetadata).with_tag(tag); + let response = match selection.longest { + Some(longest) => { + response.with_code(ResponseCode::Metadata(MetadataCode::LongEntries(longest))) + } + None => response, + }; + + trc::event!( + Imap(trc::ImapEvent::GetMetadata), + SpanId = self.session_id, + MailboxName = arguments.mailbox_name, + AccountId = mailbox_id.map(|id| id.account_id), + MailboxId = mailbox_id.map(|id| id.mailbox_id), + Total = total, + Elapsed = op_start.elapsed() + ); + + self.write_bytes(response.serialize(buf)).await + } +} diff --git a/crates/imap/src/op/metadata/list.rs b/crates/imap/src/op/metadata/list.rs new file mode 100644 index 00000000..87211d7c --- /dev/null +++ b/crates/imap/src/op/metadata/list.rs @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + READ_RIGHTS, SPECIAL_USE_ENTRY, mailbox_rights, owned_entries, reads_private_container, + select::{EntrySource, PrivateEntries, SelectOptions, select_entries}, + special_use_value, +}; +use crate::{core::SessionData, op::ImapContext}; +use common::{ + MailboxCache, auth::AccessToken, network::SessionStream, storage::metadata::MetadataContainers, +}; +use imap_proto::protocol::{ + list::ListItem, + metadata::{Depth, Entry, Scope}, +}; +use registry::schema::enums::Permission; +use store::roaring::RoaringBitmap; +use types::{collection::Collection, metadata::MetadataKinds, special_use::SpecialUse}; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ListMetadataTarget { + item: usize, + account_id: u32, + mailbox_id: u32, + kinds: MetadataKinds, + role: SpecialUse, +} + +impl ListMetadataTarget { + pub(crate) fn new( + item: usize, + account_id: u32, + mailbox_id: u32, + mailbox: &MailboxCache, + access_token: &AccessToken, + ) -> Option { + let rights = mailbox_rights(access_token, account_id, mailbox); + READ_RIGHTS + .iter() + .all(|right| rights.contains(*right)) + .then_some(ListMetadataTarget { + item, + account_id, + mailbox_id, + kinds: mailbox.metadata_kinds, + role: mailbox.role, + }) + } +} + +impl SessionData { + pub(crate) async fn list_metadata( + &self, + tag: &str, + requested: &[Entry<'static>], + targets: &[ListMetadataTarget], + list_items: &mut [ListItem], + access_token: &AccessToken, + ) -> trc::Result<()> { + let viewer = self + .server + .metadata_viewer(access_token, Permission::ImapMetadataPrivate); + let has_shared = requested.iter().any(|entry| entry.scope == Scope::Shared); + let options = SelectOptions { + depth: Depth::Zero, + max_size: None, + }; + let has_private = viewer.is_some() && reads_private_container(requested, options.depth); + + for account_targets in targets.chunk_by(|a, b| a.account_id == b.account_id) { + let Some(account_id) = account_targets.first().map(|target| target.account_id) else { + continue; + }; + + let flagged = account_targets + .iter() + .filter(|target| has_shared && target.kinds.contains(MetadataKinds::IMAP)) + .map(|target| target.mailbox_id) + .collect::(); + let shared = self + .server + .load_metadata_containers(account_id, Collection::Mailbox, &flagged) + .await + .imap_ctx(tag, trc::location!())?; + + let private = match viewer.filter(|_| has_private) { + Some(viewer) + if self + .server + .metadata_viewer_state(viewer, account_id, Collection::Mailbox) + .await + .imap_ctx(tag, trc::location!())? + .containers + > 0 => + { + let listed = account_targets + .iter() + .map(|target| target.mailbox_id) + .collect::(); + self.server + .load_private_metadata_containers( + account_id, + viewer.account_id(), + Collection::Mailbox, + &listed, + ) + .await + .imap_ctx(tag, trc::location!())? + } + _ => MetadataContainers::default(), + }; + + for target in account_targets { + let shared_source = shared + .get(target.mailbox_id) + .map(|container| EntrySource::new(container.view().imap())) + .unwrap_or_default(); + let mut private_source = private + .get(target.mailbox_id) + .map(|container| EntrySource::new(container.view().imap())) + .unwrap_or_default(); + if let Some(value) = special_use_value(target.role) { + private_source.insert(SPECIAL_USE_ENTRY, value); + } + let private_entries = match viewer { + Some(_) => PrivateEntries::All(&private_source), + None => PrivateEntries::SpecialUse(&private_source), + }; + let selection = select_entries(requested, options, &shared_source, private_entries); + if let Some(item) = list_items.get_mut(target.item) { + item.metadata = owned_entries(selection.entries); + } + } + } + + Ok(()) + } +} diff --git a/crates/imap/src/op/metadata/mod.rs b/crates/imap/src/op/metadata/mod.rs new file mode 100644 index 00000000..8663f09d --- /dev/null +++ b/crates/imap/src/op/metadata/mod.rs @@ -0,0 +1,231 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod edit; +mod get; +mod list; +mod select; +mod set; + +pub(crate) use list::ListMetadataTarget; + +use super::ImapContext; +use crate::{ + core::{AccountView, MailboxId, Session, SessionData}, + spawn_op, +}; +use common::{ + MailboxCache, MessageStoreCache, auth::AccessToken, network::SessionStream, + sharing::EffectiveAcl, +}; +use compact_str::CompactString; +use email::cache::mailbox::MailboxCacheAccess; +use imap_proto::{ + Command, ResponseCode, + protocol::metadata::{Depth, Entry, EntryValue, MetadataCode, Scope}, + receiver::Request, +}; +use registry::schema::enums::Permission; +use std::{sync::Arc, time::Instant}; +use tokio::sync::OwnedSemaphorePermit; +use types::{acl::Acl, metadata::MetadataKinds, special_use::SpecialUse}; +use utils::map::bitmap::Bitmap; + +const SERVER_DOCUMENT_ID: u32 = 0; +const SPECIAL_USE_ENTRY: &str = "/specialuse"; +const READ_RIGHTS: [Acl; 2] = [Acl::Read, Acl::ReadItems]; +const MIN_REQUESTED_ENTRIES: usize = 64; + +struct MetadataMailbox { + account_id: u32, + mailbox_id: u32, + kinds: MetadataKinds, + role: SpecialUse, + rights: Bitmap, + cache: Arc, +} + +impl Session { + pub async fn handle_get_metadata( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { + self.assert_has_permission(Permission::ImapMetadataGet)?; + + let op_start = Instant::now(); + let arguments = request.parse_get_metadata(self.is_utf8)?; + let data = self.state.session_data(); + let is_utf8 = self.is_utf8; + + spawn_op!( + permit, + data, + data.get_metadata(arguments, is_utf8, op_start).await + ) + } + + pub async fn handle_set_metadata( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { + // Get is intentional: private entries only need read access, shared entries enforce ImapMetadataSet in set_metadata + self.assert_has_permission(Permission::ImapMetadataGet)?; + + let op_start = Instant::now(); + let arguments = request.parse_set_metadata(self.is_utf8)?; + let data = self.state.session_data(); + + spawn_op!(permit, data, { + let response = data.set_metadata(arguments, op_start).await?; + data.write_bytes(response.into_bytes()).await + }) + } +} + +impl SessionData { + async fn metadata_mailbox( + &self, + tag: &str, + mailbox_name: &str, + access_token: &AccessToken, + ) -> trc::Result { + self.use_messages_of(mailbox_name); + let mut caches = self + .synchronize_mailboxes_as(access_token.clone(), false, Default::default()) + .await + .imap_ctx(tag, trc::location!())? + .caches; + let mailbox = self + .get_mailbox_by_name(mailbox_name) + .ok_or_else(|| mailbox_not_found(tag))?; + let cache = caches + .fetch(&self.server, mailbox.account_id) + .await + .imap_ctx(tag, trc::location!())?; + let (kinds, role, rights) = cache + .mailbox_by_id(&mailbox.mailbox_id) + .map(|cached| { + ( + cached.metadata_kinds, + cached.role, + mailbox_rights(access_token, mailbox.account_id, cached), + ) + }) + .ok_or_else(|| mailbox_not_found(tag))?; + + Ok(MetadataMailbox { + account_id: mailbox.account_id, + mailbox_id: mailbox.mailbox_id, + kinds, + role, + rights, + cache, + }) + } + + pub(crate) fn assert_metadata_request(&self, tag: &str, entries: usize) -> trc::Result<()> { + if entries + <= self + .server + .core + .metadata + .max_entries + .saturating_mul(2) + .max(MIN_REQUESTED_ENTRIES) + { + Ok(()) + } else { + Err(trc::ImapEvent::Error + .into_err() + .details("Too many metadata entries requested.") + .code(ResponseCode::Limit) + .id(CompactString::from(tag))) + } + } +} + +impl MetadataMailbox { + fn id(&self) -> MailboxId { + MailboxId { + account_id: self.account_id, + mailbox_id: self.mailbox_id, + } + } + + fn assert_rights(&self, tag: &str, rights: &[Acl]) -> trc::Result<()> { + if rights.iter().all(|right| self.rights.contains(*right)) { + Ok(()) + } else { + Err(no_permission(tag)) + } + } + + fn assert_any_right(&self, tag: &str, rights: &[Acl]) -> trc::Result<()> { + if rights.iter().any(|right| self.rights.contains(*right)) { + Ok(()) + } else { + Err(no_permission(tag)) + } + } +} + +#[cold] +fn no_permission(tag: &str) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details("You do not have enough permissions to access annotations on this mailbox.") + .code(ResponseCode::NoPerm) + .id(CompactString::from(tag)) +} + +fn mailbox_rights( + access_token: &AccessToken, + account_id: u32, + mailbox: &MailboxCache, +) -> Bitmap { + if access_token.is_member(account_id) { + Bitmap::all() + } else { + mailbox.acls.as_slice().effective_acl(access_token) + } +} + +fn reads_private_container(entries: &[Entry<'_>], depth: Depth) -> bool { + entries.iter().any(|entry| { + entry.scope == Scope::Private && (depth != Depth::Zero || entry.path != SPECIAL_USE_ENTRY) + }) +} + +fn special_use_value(role: SpecialUse) -> Option<&'static [u8]> { + AccountView::special_use_attribute(&role).map(|attribute| attribute.as_bytes()) +} + +fn mailbox_not_found(tag: &str) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details("Mailbox does not exist.") + .code(ResponseCode::NonExistent) + .id(CompactString::from(tag)) +} + +fn metadata_error(tag: &str, code: MetadataCode) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details(match code { + MetadataCode::MaxSize(_) => "Annotation value or mailbox annotations too large.", + MetadataCode::TooMany => "Too many annotations.", + MetadataCode::NoPrivate => "Private annotations are not supported.", + MetadataCode::LongEntries(_) => "Annotation value too large.", + }) + .code(ResponseCode::Metadata(code)) + .id(CompactString::from(tag)) +} + +fn owned_entries(entries: Vec>) -> Option]>> { + (!entries.is_empty()).then(|| entries.into_iter().map(EntryValue::into_owned).collect()) +} diff --git a/crates/imap/src/op/metadata/select.rs b/crates/imap/src/op/metadata/select.rs new file mode 100644 index 00000000..1f52d0f9 --- /dev/null +++ b/crates/imap/src/op/metadata/select.rs @@ -0,0 +1,202 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::SPECIAL_USE_ENTRY; +use imap_proto::protocol::metadata::{Depth, Entry, EntryValue, Scope}; +use std::borrow::Cow; + +#[derive(Debug, Default)] +pub(crate) struct EntrySource<'x> { + entries: Vec<(&'x str, &'x [u8])>, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum PrivateEntries<'s, 'x> { + All(&'s EntrySource<'x>), + SpecialUse(&'s EntrySource<'x>), + Hidden, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SelectOptions { + pub depth: Depth, + pub max_size: Option, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct Selection<'x> { + pub entries: Vec>, + pub longest: Option, +} + +struct Emitted(Vec); + +impl<'x> EntrySource<'x> { + pub fn new(entries: impl IntoIterator) -> Self { + let mut entries = entries.into_iter().collect::>(); + if !entries.is_sorted_by(|a, b| a.0 < b.0) { + entries.sort_unstable_by(|a, b| a.0.cmp(b.0)); + entries.dedup_by(|a, b| a.0 == b.0); + } + EntrySource { entries } + } + + pub fn insert(&mut self, name: &'x str, value: &'x [u8]) { + match self.position(name) { + Ok(position) => { + if let Some(entry) = self.entries.get_mut(position) { + entry.1 = value; + } + } + Err(position) => self.entries.insert(position, (name, value)), + } + } + + fn position(&self, name: &str) -> Result { + self.entries + .binary_search_by(|(entry, _)| (*entry).cmp(name)) + } + + fn get(&self, name: &str) -> Option<(usize, &'x str, &'x [u8])> { + let position = self.position(name).ok()?; + self.entries + .get(position) + .map(|&(name, value)| (position, name, value)) + } + + fn children<'y>( + &'y self, + prefix: &'y str, + depth: Depth, + ) -> impl Iterator + 'y { + let start = self.entries.partition_point(|(name, _)| *name < prefix); + self.entries + .get(start..) + .unwrap_or_default() + .iter() + .zip(start..) + .map_while(move |(&(name, value), position)| { + name.strip_prefix(prefix) + .map(|child| (position, name, value, child)) + }) + .filter(move |(_, _, _, child)| { + !child.is_empty() && (depth == Depth::Infinity || !child.contains('/')) + }) + .map(|(position, name, value, _)| (position, name, value)) + } +} + +impl<'s, 'x> PrivateEntries<'s, 'x> { + fn source(self) -> Option<&'s EntrySource<'x>> { + match self { + PrivateEntries::All(source) | PrivateEntries::SpecialUse(source) => Some(source), + PrivateEntries::Hidden => None, + } + } + + fn exposes(self, path: &str) -> bool { + match self { + PrivateEntries::All(_) => true, + PrivateEntries::SpecialUse(_) => path == SPECIAL_USE_ENTRY, + PrivateEntries::Hidden => false, + } + } +} + +impl Emitted { + fn new(source: &EntrySource<'_>, depth: Depth) -> Self { + if depth == Depth::Zero { + Emitted(Vec::new()) + } else { + Emitted(vec![false; source.entries.len()]) + } + } + + fn insert(&mut self, position: usize) -> bool { + self.0.is_empty() + || self + .0 + .get_mut(position) + .is_some_and(|emitted| !std::mem::replace(emitted, true)) + } +} + +impl<'x> Selection<'x> { + fn push(&mut self, scope: Scope, name: &'x str, value: &'x [u8], max_size: Option) { + match max_size { + Some(max_size) if value.len() > max_size as usize => { + let size = u32::try_from(value.len()).unwrap_or(u32::MAX); + self.longest = Some(self.longest.map_or(size, |longest| longest.max(size))); + } + _ => self.entries.push(EntryValue { + entry: Entry { + scope, + path: Cow::Borrowed(name), + }, + value: Some(Cow::Borrowed(value)), + }), + } + } +} + +pub(crate) fn select_entries<'x>( + requested: &'x [Entry<'x>], + options: SelectOptions, + shared: &EntrySource<'x>, + private: PrivateEntries<'_, 'x>, +) -> Selection<'x> { + let mut selection = Selection { + entries: Vec::with_capacity(requested.len()), + longest: None, + }; + let private_source = private.source(); + let mut shared_emitted = Emitted::new(shared, options.depth); + let mut private_emitted = private_source.map(|source| Emitted::new(source, options.depth)); + let mut prefix = String::new(); + + for entry in requested { + let (source, emitted) = match (entry.scope, private_source, private_emitted.as_mut()) { + (Scope::Shared, _, _) => (shared, &mut shared_emitted), + (Scope::Private, Some(source), Some(emitted)) if private.exposes(&entry.path) => { + (source, emitted) + } + (Scope::Private, _, _) => continue, + }; + let path = entry.path.as_ref(); + + if !path.is_empty() { + match source.get(path) { + Some((position, name, value)) => { + if emitted.insert(position) { + selection.push(entry.scope, name, value, options.max_size); + } + } + None if options.depth == Depth::Zero => selection.entries.push(EntryValue { + entry: Entry { + scope: entry.scope, + path: Cow::Borrowed(path), + }, + value: None, + }), + None => {} + } + } + + if options.depth != Depth::Zero { + prefix.clear(); + prefix.push_str(path); + prefix.push('/'); + for (position, name, value) in source.children(&prefix, options.depth) { + if emitted.insert(position) { + selection.push(entry.scope, name, value, options.max_size); + } + } + } + } + + selection +} + diff --git a/crates/imap/src/op/metadata/set.rs b/crates/imap/src/op/metadata/set.rs new file mode 100644 index 00000000..c0d3d8cf --- /dev/null +++ b/crates/imap/src/op/metadata/set.rs @@ -0,0 +1,491 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MetadataMailbox, READ_RIGHTS, SERVER_DOCUMENT_ID, SPECIAL_USE_ENTRY, edit::edit_container, + mailbox_not_found, metadata_error, special_use_value, +}; +use crate::{ + core::{MailboxId, SessionData}, + op::{ImapContext, create::attr_to_role}, +}; +use common::{ + auth::AccessToken, + network::SessionStream, + storage::{ + index::{ObjectIndexBuilder, PresenceFlags, RewritePresence}, + metadata::{ + ContainerChange, MetadataLog, MetadataPresence, MetadataViewer, PrivateMetadataCommit, + }, + }, +}; +use compact_str::CompactString; +use email::mailbox::{Mailbox, role::RoleChange}; +use imap_proto::{ + Command, ResponseCode, StatusResponse, + protocol::{ + list::Attribute, + metadata::{EntryValue, MetadataCode, Scope, SetArguments}, + }, +}; +use registry::schema::enums::Permission; +use std::time::Instant; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes, AssignedIds, BatchBuilder, metadata::MetadataBuf}, +}; +use types::{ + acl::Acl, + collection::{Collection, SyncCollection}, + metadata::MetadataScope, + special_use::SpecialUse, +}; + +const SHARED_WRITE_RIGHTS: [Acl; 1] = [Acl::ModifyMetadata]; +const SHARED_ACCESS_RIGHTS: [Acl; 4] = + [Acl::ReadItems, Acl::ModifyItems, Acl::AddItems, Acl::Submit]; +const ROLE_RIGHTS: [Acl; 1] = [Acl::Modify]; +const MODIFIED_ELSEWHERE: &str = "Metadata was modified by another process."; + +impl SessionData { + pub async fn set_metadata( + &self, + arguments: SetArguments, + op_start: Instant, + ) -> trc::Result { + let SetArguments { + tag, + mailbox_name, + entries, + } = arguments; + let access_token = self + .refresh_access_token() + .await + .imap_ctx(&tag, trc::location!())?; + + let has_shared = entries + .iter() + .any(|entry| entry.entry.scope == Scope::Shared); + if has_shared { + access_token + .enforce_permission(Permission::ImapMetadataSet) + .map_err(|err| err.id(tag.clone()))?; + } + let viewer = if needs_viewer(&mailbox_name, &entries) { + Some( + self.server + .metadata_viewer(&access_token, Permission::ImapMetadataPrivate) + .ok_or_else(|| metadata_error(&tag, MetadataCode::NoPrivate))?, + ) + } else { + None + }; + + let mailbox_id = if mailbox_name.is_empty() { + self.set_server_metadata(&tag, &entries, has_shared, viewer) + .await?; + None + } else { + Some( + self.set_mailbox_metadata(&tag, &mailbox_name, &entries, viewer, &access_token) + .await?, + ) + }; + + trc::event!( + Imap(trc::ImapEvent::SetMetadata), + SpanId = self.session_id, + MailboxName = mailbox_name, + AccountId = mailbox_id.map(|id| id.account_id), + MailboxId = mailbox_id.map(|id| id.mailbox_id), + Total = entries.len(), + Elapsed = op_start.elapsed() + ); + + Ok(StatusResponse::completed(Command::SetMetadata).with_tag(tag)) + } + + async fn set_server_metadata( + &self, + tag: &str, + entries: &[EntryValue<'_>], + has_shared: bool, + viewer: Option, + ) -> trc::Result<()> { + if has_shared { + return Err(trc::ImapEvent::Error + .into_err() + .details("Shared server annotations are read-only.") + .code(ResponseCode::NoPerm) + .id(CompactString::from(tag))); + } + let Some(viewer) = viewer else { + return Ok(()); + }; + + let account_id = viewer.account_id(); + let previous = self + .server + .metadata_container(account_id, Collection::Principal, SERVER_DOCUMENT_ID) + .await + .imap_ctx(tag, trc::location!())?; + let Some(change) = self.container_change( + tag, + previous.as_ref(), + entries.iter().map(entry_value), + MetadataScope::Private, + )? + else { + return Ok(()); + }; + + let account = self + .server + .account(account_id) + .await + .imap_ctx(tag, trc::location!())?; + if !self + .server + .has_metadata_quota(&account, &change) + .await + .imap_ctx(tag, trc::location!())? + { + return Err(over_quota(tag)); + } + + let mut batch = BatchBuilder::new(); + change + .into_write(&account, Collection::Principal, MetadataLog::None) + .build(SERVER_DOCUMENT_ID.into(), &mut batch) + .imap_ctx(tag, trc::location!())?; + self.commit_metadata(tag, batch, None, trc::location!()) + .await + .map(|_| ()) + } + + async fn set_mailbox_metadata( + &self, + tag: &str, + mailbox_name: &str, + entries: &[EntryValue<'_>], + viewer: Option, + access_token: &AccessToken, + ) -> trc::Result { + let mailbox = self + .metadata_mailbox(tag, mailbox_name, access_token) + .await?; + let shared_entries = entries + .iter() + .filter(|entry| entry.entry.scope == Scope::Shared) + .map(entry_value); + let private_entries = entries + .iter() + .filter(|entry| { + entry.entry.scope == Scope::Private && entry.entry.path != SPECIAL_USE_ENTRY + }) + .map(entry_value); + let has_shared = shared_entries.clone().next().is_some(); + let has_private = private_entries.clone().next().is_some(); + + if has_shared { + mailbox.assert_rights(tag, &SHARED_WRITE_RIGHTS)?; + mailbox.assert_any_right(tag, &SHARED_ACCESS_RIGHTS)?; + } + if has_private { + mailbox.assert_rights(tag, &READ_RIGHTS)?; + } + let role = match entries.iter().find(|entry| { + entry.entry.scope == Scope::Private && entry.entry.path == SPECIAL_USE_ENTRY + }) { + Some(entry) => self.special_use_change(tag, &mailbox, entry.value.as_deref())?, + None => None, + }; + + let shared = if has_shared { + let previous = if !mailbox.kinds.is_empty() { + self.server + .metadata_container(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) + .await + .imap_ctx(tag, trc::location!())? + } else { + None + }; + self.container_change( + tag, + previous.as_ref(), + shared_entries, + MetadataScope::Shared, + )? + } else { + None + }; + let private = match viewer { + Some(viewer) if has_private => { + let previous = if self + .server + .metadata_viewer_state(viewer, mailbox.account_id, Collection::Mailbox) + .await + .imap_ctx(tag, trc::location!())? + .containers + > 0 + { + self.server + .private_metadata_container( + mailbox.account_id, + viewer.account_id(), + Collection::Mailbox, + mailbox.mailbox_id, + ) + .await + .imap_ctx(tag, trc::location!())? + } else { + None + }; + self.container_change( + tag, + previous.as_ref(), + private_entries, + MetadataScope::Private, + )? + .map(|change| (viewer, change)) + } + _ => None, + }; + + let mut batch = BatchBuilder::new(); + let mut commit = PrivateMetadataCommit::default(); + let mut presence = None; + if let Some(change) = shared { + let owner = self + .server + .account(mailbox.account_id) + .await + .imap_ctx(tag, trc::location!())?; + if !self + .server + .has_metadata_quota(&owner, &change) + .await + .imap_ctx(tag, trc::location!())? + { + return Err(over_quota(tag)); + } + presence = Some( + change + .into_write(&owner, Collection::Mailbox, MetadataLog::Container) + .build(mailbox.mailbox_id.into(), &mut batch) + .imap_ctx(tag, trc::location!())?, + ); + } + if let Some((viewer, change)) = private { + let account = self + .server + .account(viewer.account_id()) + .await + .imap_ctx(tag, trc::location!())?; + if !self + .server + .has_metadata_quota(&account, &change) + .await + .imap_ctx(tag, trc::location!())? + { + return Err(over_quota(tag)); + } + change + .into_private_write( + mailbox.account_id, + &account, + Collection::Mailbox, + MetadataLog::Container, + ) + .build(mailbox.mailbox_id.into(), &mut batch, &mut commit) + .imap_ctx(tag, trc::location!())?; + } + + let presence = presence.filter(|presence| presence.has_changed_for(Mailbox::TRACKED)); + if role.is_some() || presence.is_some() { + self.update_mailbox_archive(tag, &mailbox, role, presence, &mut batch) + .await?; + } + + if !batch.is_empty() { + let assigned_ids = self + .commit_metadata(tag, batch, Some(mailbox.account_id), trc::location!()) + .await?; + self.server + .private_metadata_committed(commit, &assigned_ids) + .await; + } + + Ok(mailbox.id()) + } + + async fn commit_metadata( + &self, + tag: &str, + batch: BatchBuilder, + cached_account_id: Option, + location: &'static str, + ) -> trc::Result { + match self.server.commit_batch(batch).await { + Err(err) if err.is_assertion_failure() => { + if let Some(account_id) = cached_account_id { + self.server + .inner + .mark_cache_stale(account_id, SyncCollection::Email); + } + Err(trc::ImapEvent::Error + .into_err() + .details(MODIFIED_ELSEWHERE) + .id(CompactString::from(tag))) + } + result => result.imap_ctx(tag, location), + } + } + + fn special_use_change( + &self, + tag: &str, + mailbox: &MetadataMailbox, + value: Option<&[u8]>, + ) -> trc::Result> { + mailbox.assert_rights(tag, &ROLE_RIGHTS)?; + let role = parse_special_use(value) + .ok_or_else(|| use_attr_error(tag, "Unsupported special-use attribute."))?; + if role == SpecialUse::None && special_use_value(mailbox.role).is_none() { + return Ok(None); + } + RoleChange::Update { + document_id: mailbox.mailbox_id, + current: mailbox.role, + role, + } + .validate(&mailbox.cache) + .map_err(|err| use_attr_error(tag, err.to_string()))?; + Ok((role != mailbox.role).then_some(role)) + } + + fn container_change<'x>( + &self, + tag: &str, + previous: Option<&'x MetadataBuf>, + entries: impl Iterator)> + Clone, + scope: MetadataScope, + ) -> trc::Result> { + edit_container( + previous, + entries, + scope, + &self.server.core.metadata.limits(), + ) + .map_err(|code| metadata_error(tag, code)) + } + + async fn update_mailbox_archive( + &self, + tag: &str, + mailbox: &MetadataMailbox, + role: Option, + presence: Option, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + let archive = self + .server + .store() + .get_value::>(ValueKey::archive( + mailbox.account_id, + Collection::Mailbox, + mailbox.mailbox_id, + )) + .await + .imap_ctx(tag, trc::location!())? + .ok_or_else(|| mailbox_not_found(tag))?; + let current = archive + .to_unarchived::() + .imap_ctx(tag, trc::location!())?; + + match (role, presence) { + (Some(role), presence) => { + let mut changes = current + .deserialize::() + .imap_ctx(tag, trc::location!())?; + changes.set_role(role); + if let Some(presence) = presence { + changes.set_metadata_kinds(presence.after); + } + batch + .with_account_id(mailbox.account_id) + .with_collection(Collection::Mailbox) + .with_document(mailbox.mailbox_id) + .custom( + ObjectIndexBuilder::new() + .with_current(current) + .with_changes(changes), + ) + .imap_ctx(tag, trc::location!())?; + } + (None, Some(presence)) => { + Mailbox::rewrite_presence( + ¤t, + presence.after, + mailbox.account_id, + mailbox.mailbox_id, + batch, + ) + .imap_ctx(tag, trc::location!())?; + } + (None, None) => {} + } + + Ok(()) + } +} + +fn needs_viewer(mailbox_name: &str, entries: &[EntryValue<'_>]) -> bool { + entries.iter().any(|entry| { + entry.entry.scope == Scope::Private + && (mailbox_name.is_empty() || entry.entry.path != SPECIAL_USE_ENTRY) + }) +} + +fn entry_value<'x>(entry: &'x EntryValue<'_>) -> (&'x str, Option<&'x [u8]>) { + (entry.entry.path.as_ref(), entry.value.as_deref()) +} + +fn over_quota(tag: &str) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details("Quota exceeded.") + .code(ResponseCode::OverQuota) + .id(CompactString::from(tag)) +} + +fn use_attr_error(tag: &str, details: impl Into) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details(details) + .code(ResponseCode::UseAttr) + .id(CompactString::from(tag)) +} + +fn parse_special_use(value: Option<&[u8]>) -> Option { + let mut role = SpecialUse::None; + for attribute in value + .unwrap_or_default() + .split(u8::is_ascii_whitespace) + .filter(|attribute| !attribute.is_empty()) + { + let next = match Attribute::parse_special_use(attribute)? { + Attribute::All => return None, + attribute => attr_to_role(attribute), + }; + if role != SpecialUse::None && role != next { + return None; + } + role = next; + } + Some(role) +} + diff --git a/crates/imap/src/op/mod.rs b/crates/imap/src/op/mod.rs index 70cdee3e..d3926ad4 100644 --- a/crates/imap/src/op/mod.rs +++ b/crates/imap/src/op/mod.rs @@ -5,6 +5,8 @@ */ use ::store::query::log::Query; +use common::config::mailstore::limits::EmailLimitError; +use compact_str::CompactString; use imap_proto::ResponseCode; pub mod acl; @@ -22,6 +24,7 @@ pub mod idle; pub mod list; pub mod login; pub mod logout; +pub mod metadata; pub mod namespace; pub mod noop; pub mod quota; @@ -42,6 +45,19 @@ trait ToModSeq { fn to_modseq(&self) -> u64; } +trait IntoImapError { + fn into_imap_error(self) -> trc::Error; +} + +impl IntoImapError for EmailLimitError { + fn into_imap_error(self) -> trc::Error { + trc::ImapEvent::Error + .into_err() + .details(self.to_string()) + .code(ResponseCode::Limit) + } +} + impl FromModSeq for Query { fn from_modseq(modseq: u64) -> Self { if modseq > 0 { @@ -60,10 +76,11 @@ impl ToModSeq for u64 { #[macro_export] macro_rules! spawn_op { - ($data:expr, $($code:tt)*) => { + ($permit:expr, $data:expr, $($code:tt)*) => { { - tokio::spawn(async move { + let permit = $permit; + let op = async move { let data = &($data); if let Err(err) = (async { @@ -73,7 +90,17 @@ macro_rules! spawn_op { { let _ = data.write_error(err).await; } - }); + }; + + match permit { + Some(permit) => { + tokio::spawn(async move { + let _permit: tokio::sync::OwnedSemaphorePermit = permit; + op.await; + }); + } + None => op.await, + } Ok(())} }; @@ -88,12 +115,12 @@ impl ImapContext for trc::Result { Ok(value) => Ok(value), Err(err) => Err( if !err.matches(trc::EventType::Imap(trc::ImapEvent::Error)) { - err.ctx(trc::Key::Id, tag.to_string()) + err.ctx(trc::Key::Id, CompactString::from(tag)) .ctx(trc::Key::Details, "Internal Server Error") .ctx(trc::Key::Code, ResponseCode::ContactAdmin) .ctx(trc::Key::CausedBy, location) } else { - err.ctx(trc::Key::Id, tag.to_string()) + err.ctx(trc::Key::Id, CompactString::from(tag)) }, ), } diff --git a/crates/imap/src/op/namespace.rs b/crates/imap/src/op/namespace.rs index 33ad5c56..5f3e932b 100644 --- a/crates/imap/src/op/namespace.rs +++ b/crates/imap/src/op/namespace.rs @@ -6,11 +6,7 @@ use crate::core::Session; use common::network::SessionStream; -use imap_proto::{ - Command, StatusResponse, - protocol::{ImapResponse, namespace::Response}, - receiver::Request, -}; +use imap_proto::{Command, StatusResponse, protocol::namespace::Response, receiver::Request}; use registry::schema::enums::Permission; impl Session { @@ -27,16 +23,13 @@ impl Session { self.write_bytes( StatusResponse::completed(Command::Namespace) .with_tag(request.tag) - .serialize( - Response { - shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { - Some(self.server.core.email.shared_folder.as_str().into()) - } else { - None - }, - } - .serialize(), - ), + .serialize_after(&Response { + shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { + Some(self.server.core.email.shared_folder.as_str().into()) + } else { + None + }, + }), ) .await } diff --git a/crates/imap/src/op/noop.rs b/crates/imap/src/op/noop.rs index 7ecf5aa8..64fbeb24 100644 --- a/crates/imap/src/op/noop.rs +++ b/crates/imap/src/op/noop.rs @@ -5,24 +5,33 @@ */ use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; -use crate::core::{Session, State}; +use crate::{ + core::{Session, State}, + op::idle::ViewSync, +}; use common::network::SessionStream; use imap_proto::{Command, StatusResponse, receiver::Request}; impl Session { - pub async fn handle_noop(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_noop( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { let op_start = Instant::now(); if let State::Selected { data, mailbox, .. } = &self.state { data.write_changes( - &Some(mailbox.clone()), + Some(mailbox), false, - true, + ViewSync::Sync, self.is_qresync || self.is_uidonly, self.is_uidonly, self.version.is_rev2(), self.is_utf8, + self.is_objectid, ) .await?; } diff --git a/crates/imap/src/op/quota.rs b/crates/imap/src/op/quota.rs index 34ea9628..0a52e3ce 100644 --- a/crates/imap/src/op/quota.rs +++ b/crates/imap/src/op/quota.rs @@ -4,22 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - use crate::{ core::{Session, SessionData}, op::ImapContext, spawn_op, }; use common::network::SessionStream; +use compact_str::format_compact; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{ - ImapResponse, capability::QuotaResourceName, quota::{Arguments, QuotaItem, QuotaResource, Response}, }, @@ -27,15 +21,20 @@ use imap_proto::{ }; use registry::schema::enums::Permission; use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; impl Session { - pub async fn handle_get_quota(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_get_quota( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; let data = self.state.session_data(); - spawn_op!(data, { + spawn_op!(permit, data, { match request.parse_get_quota() { Ok(argument) => match data.get_quota(argument).await { Ok(response) => { @@ -52,14 +51,18 @@ impl Session { }) } - pub async fn handle_get_quota_root(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_get_quota_root( + &mut self, + request: Request, + permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; let data = self.state.session_data(); let is_utf8 = self.is_utf8; - spawn_op!(data, { + spawn_op!(permit, data, { match request.parse_get_quota_root(is_utf8) { Ok(argument) => match data.get_quota_root(argument).await { Ok(response) => { @@ -96,7 +99,7 @@ impl SessionData { trc::ImapEvent::Error .into_err() .details("Invalid quota root parameter.") - .id(arguments.tag.to_string()) + .id(arguments.tag.clone()) })?; // Obtain access token for mailbox @@ -114,11 +117,9 @@ impl SessionData { trc::event!( Imap(trc::ImapEvent::GetQuota), SpanId = self.session_id, - Id = arguments.name.clone(), - Details = vec![ - trc::Value::from(used_quota), - trc::Value::from(account.disk_quota()) - ], + MailboxName = arguments.name.clone(), + Used = used_quota, + Limit = account.disk_quota(), Elapsed = op_start.elapsed() ); @@ -141,7 +142,7 @@ impl SessionData { Ok(StatusResponse::ok("GETQUOTA successful.") .with_tag(arguments.tag) - .serialize(response.serialize())) + .serialize_after(&response)) } pub async fn get_quota_root(&self, arguments: Arguments) -> trc::Result> { @@ -188,9 +189,9 @@ impl SessionData { // Build response let response = Response { - quota_root_items: vec![arguments.name, format!("#{account_id}")], + quota_root_items: vec![arguments.name, format_compact!("#{account_id}")], quota_items: vec![QuotaItem { - name: format!("#{account_id}"), + name: format_compact!("#{account_id}"), resources: if account.disk_quota() > 0 { vec![QuotaResource { resource: QuotaResourceName::Storage, @@ -205,6 +206,6 @@ impl SessionData { Ok(StatusResponse::ok("GETQUOTAROOT successful.") .with_tag(arguments.tag) - .serialize(response.serialize())) + .serialize_after(&response)) } } diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 704f1000..cfda3c97 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -4,30 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - core::{Session, SessionData}, - spawn_op, -}; +use crate::core::{MailboxId, Session, SessionData}; use common::{network::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder}; -use email::cache::MessageCacheFetch; +use compact_str::format_compact; use imap_proto::{ - Command, ResponseCode, StatusResponse, - protocol::{ObjectId, rename::Arguments}, - receiver::Request, + Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; use registry::schema::enums::{Permission, StorageQuota}; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; -use types::{acl::Acl, collection::Collection, id::Id}; +use types::{acl::Acl, collection::Collection}; use super::ImapContext; impl Session { - pub async fn handle_rename(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_rename( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapRename)?; @@ -36,10 +36,8 @@ impl Session { let data = self.state.session_data(); let is_objectid = self.is_objectid; - spawn_op!(data, { - let response = data.rename_folder(arguments, is_objectid, op_start).await?; - data.write_bytes(response.into_bytes()).await - }) + let response = data.rename_folder(arguments, is_objectid, op_start).await?; + data.write_bytes(response.into_bytes()).await } } @@ -51,9 +49,13 @@ impl SessionData { op_start: Instant, ) -> trc::Result { // Refresh mailboxes - self.synchronize_mailboxes(false) + self.expire_folders_of(&arguments.mailbox_name); + self.expire_folders_of(&arguments.new_mailbox_name); + let mut caches = self + .synchronize_mailboxes(false) .await - .imap_ctx(&arguments.tag, trc::location!())?; + .imap_ctx(&arguments.tag, trc::location!())? + .caches; // Validate mailbox name let mut params = self @@ -66,9 +68,9 @@ impl SessionData { let mailbox_id = { let mut mailbox_id = None; for account in self.mailboxes.lock().iter() { - if let Some(mailbox_id_) = account.mailbox_names.get(&arguments.mailbox_name) { + if let Some(mailbox_id_) = account.id_by_name(&arguments.mailbox_name) { if account.account_id == params.account_id { - mailbox_id = (*mailbox_id_).into(); + mailbox_id = mailbox_id_.into(); break; } else { return Err(trc::ImapEvent::Error @@ -84,7 +86,10 @@ impl SessionData { } else { return Err(trc::ImapEvent::Error .into_err() - .details(format!("Mailbox '{}' not found.", arguments.mailbox_name)) + .details(format_compact!( + "Mailbox '{}' not found.", + arguments.mailbox_name + )) .code(ResponseCode::NonExistent) .id(arguments.tag)); } @@ -94,7 +99,7 @@ impl SessionData { let mailbox_ = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( params.account_id, Collection::Mailbox, mailbox_id, @@ -104,7 +109,10 @@ impl SessionData { .ok_or_else(|| { trc::ImapEvent::Error .into_err() - .details(format!("Mailbox '{}' not found.", arguments.mailbox_name)) + .details(format_compact!( + "Mailbox '{}' not found.", + arguments.mailbox_name + )) .caused_by(trc::location!()) .code(ResponseCode::NonExistent) .id(arguments.tag.clone()) @@ -142,9 +150,8 @@ impl SessionData { .account(params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; - let mailbox_count = self - .server - .get_cached_messages(params.account_id) + let mailbox_count = caches + .fetch(&self.server, params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())? .mailboxes @@ -168,36 +175,31 @@ impl SessionData { } // Build batch - let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); - let mut create_ids = Vec::with_capacity(params.path.len()); - let mut next_document_id = self - .server - .store() - .assign_document_ids( - params.account_id, - Collection::Mailbox, - params.path.len() as u64, - ) - .await - .caused_by(trc::location!())?; + let parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut batch = BatchBuilder::new(); + let slots = batch.reserve_document_ids( + params.account_id, + Collection::Mailbox, + params.path.len() as u32, + ); + let mut parent_slot = None; - for &path_item in params.path.iter() { - let mailbox_id = next_document_id; - next_document_id -= 1; + for (&path_item, slot) in params.path.iter().zip(slots.iter()) { + let mut new_folder = email::mailbox::Mailbox::new(path_item); + if parent_slot.is_none() { + new_folder.parent_id = parent_id; + } + let builder = ObjectIndexBuilder::<(), _>::new().with_changes(new_folder); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .with_document(mailbox_id) - .custom(ObjectIndexBuilder::<(), _>::new().with_changes( - email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), - )) + .create_document(slot) + .custom(builder.with_pending_id_opt(parent_slot)) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); - parent_id = mailbox_id + 1; - create_ids.push(mailbox_id); + parent_slot = Some(slot); } let mut new_mailbox = mailbox @@ -205,16 +207,14 @@ impl SessionData { .caused_by(trc::location!())?; new_mailbox.name = new_mailbox_name.into(); new_mailbox.parent_id = parent_id; - new_mailbox.uid_validity = rand::random::(); + let builder = ObjectIndexBuilder::new() + .with_current(mailbox) + .with_changes(new_mailbox); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) - .custom( - ObjectIndexBuilder::new() - .with_current(mailbox) - .with_changes(new_mailbox), - ) + .custom(builder.with_pending_id_opt(parent_slot)) .imap_ctx(&arguments.tag, trc::location!())?; self.server .commit_batch(batch) @@ -222,6 +222,17 @@ impl SessionData { .imap_ctx(&arguments.tag, trc::location!())?; let account_id = params.account_id; + self.expire_folders(account_id); + + let objectid = is_objectid.then(|| { + self.mailbox_object_id( + MailboxId { + account_id, + mailbox_id, + }, + &arguments.new_mailbox_name, + ) + }); trc::event!( Imap(trc::ImapEvent::RenameMailbox), @@ -233,14 +244,9 @@ impl SessionData { ); let response = StatusResponse::completed(Command::Rename).with_tag(arguments.tag); - Ok(if is_objectid { - response.with_code(ResponseCode::ObjectId(ObjectId { - mailbox_id: Some(Id::from(mailbox_id)), - account_id: Some(Id::from(account_id)), - ..Default::default() - })) - } else { - response + Ok(match objectid { + Some(objectid) => response.with_code(ResponseCode::ObjectId(objectid)), + None => response, }) } } diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index edc3ad28..c4029e48 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -4,17 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{FromModSeq, ToModSeq}; use crate::{ - core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}, + core::{Resolved, Row, SavedSearch, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::network::SessionStream; -use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; +use common::{MessageStoreCache, network::SessionStream}; +use email::{ + cache::{ + MessageCacheFetch, + email::{MessageCacheAccess, SearchOperator}, + }, + message::{ + index::subject::search_subject_query, + sortkeys::{EmailSortKeys, MessageCacheField, MessageComparator, MessageSortField}, + }, +}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ - Sequence, + ObjectId, Sequence, search::{self, Arguments, Comparator, Filter, Response, ResultOption}, }, receiver::Request, @@ -22,19 +30,16 @@ use imap_proto::{ use mail_parser::HeaderName; use nlp::language::Language; use registry::schema::enums::Permission; -use std::{str::FromStr, sync::Arc, time::Instant}; +use std::{sync::Arc, time::Instant}; use store::{ - query::log::Query, roaring::RoaringBitmap, - search::{ - EmailSearchField, SearchComparator, SearchFilter, SearchOperator, SearchQuery, SearchValue, - }, + search::{EmailSearchField, KeyValueMatch, SearchFilter, SearchQuery}, write::{SearchIndex, now}, }; +use tokio::sync::OwnedSemaphorePermit; use tokio::sync::watch; use trc::AddContext; -use types::{collection::SyncCollection, id::Id, keyword::Keyword}; -use utils::map::vec_map::VecMap; +use types::keyword::Keyword; impl Session { pub async fn handle_search( @@ -42,6 +47,7 @@ impl Session { request: Request, is_sort: bool, is_uid: bool, + permit: Option, ) -> trc::Result<()> { let op_start = Instant::now(); let mut arguments = if !is_sort { @@ -84,7 +90,7 @@ impl Session { (None, None) }; - spawn_op!(data, { + spawn_op!(permit, data, { let tag = std::mem::take(&mut arguments.tag); let bytes = match data .search( @@ -135,12 +141,21 @@ impl SessionData { &self, arguments: Arguments, mailbox: Arc, - results_tx: Option>>>, - prev_saved_search: Option>>>, + results_tx: Option>>>, + prev_saved_search: Option>>>, is_uid: bool, message_limit: u32, op_start: Instant, ) -> trc::Result<(search::Response, Option)> { + let cache = self + .server + .get_cached_messages(mailbox.id.account_id) + .await + .caused_by(trc::location!())?; + self.sync_view(&mailbox, &cache, None) + .await + .caused_by(trc::location!())?; + // Run query let is_sort = arguments.sort.is_some(); let (result_set, include_highest_modseq) = self @@ -148,72 +163,81 @@ impl SessionData { arguments.filter, arguments.sort.unwrap_or_default(), &mailbox, + &cache, &prev_saved_search, ) .await?; - - // Obtain modseq - let highest_modseq = if include_highest_modseq { - self.synchronize_messages(&mailbox) - .await? - .to_modseq() - .into() - } else { - None - }; - // Sort and map ids - let mut min: Option<(u32, ImapId)> = None; - let mut max: Option<(u32, ImapId)> = None; - let mut total = 0; - let results_len = result_set.len(); - let mut saved_results = if results_tx.is_some() { - Some(Vec::with_capacity(results_len)) - } else { - None - }; - let mut imap_ids = Vec::with_capacity(results_len); + let find_min = arguments.result_options.contains(&ResultOption::Min); + let find_max = arguments.result_options.contains(&ResultOption::Max); let return_all = arguments.result_options.is_empty() || arguments.result_options.contains(&ResultOption::All); - mailbox.map_search_results( - result_set.into_iter(), + let mut results = SearchResults { is_uid, - return_all - || (saved_results.is_some() + find_min, + find_max, + collect_all: return_all + || (results_tx.is_some() && arguments.result_options.contains(&ResultOption::Count)), - arguments.result_options.contains(&ResultOption::Min), - arguments.result_options.contains(&ResultOption::Max), - &mut min, - &mut max, - &mut total, - &mut imap_ids, - &mut saved_results, - ); + min: None, + max: None, + total: 0, + imap_ids: Vec::with_capacity(result_set.len()), + modseq_cache: include_highest_modseq.then_some(&cache), + highest_modseq: None, + saved_results: results_tx + .is_some() + .then(|| Vec::with_capacity(result_set.len())), + }; + { + let view = mailbox.view.lock(); + if !is_sort && result_set.len() > view.len() / 8 { + let set = RoaringBitmap::from_iter(result_set); + view.map_result_set(&set, |resolved| results.push(resolved)); + } else { + for document_id in result_set { + if let Some(resolved) = view.map_result(document_id) { + results.push(resolved); + } + } + } + } + results.finish(); + let SearchResults { + min, + max, + total, + mut imap_ids, + mut saved_results, + highest_modseq, + .. + } = results; + // RFC 9738 exempts SORT, whose ordering is meaningless once truncated let mut limited_uid = None; if !is_sort { - imap_ids.sort_unstable(); - let message_limit = message_limit as usize; if imap_ids.len() > message_limit { - let threshold = imap_ids[imap_ids.len() - message_limit]; - imap_ids.drain(..imap_ids.len() - message_limit); - limited_uid = if is_uid { + let cutoff = imap_ids.len() - message_limit; + let (_, threshold, _) = imap_ids.select_nth_unstable(cutoff); + let threshold = *threshold; + imap_ids.drain(..cutoff); + imap_ids.sort_unstable(); + let threshold_uid = if is_uid { Some(threshold) } else { - mailbox.seqnum_to_uid(threshold) + mailbox.view.lock().seqnum_to_uid(threshold) }; + limited_uid = threshold_uid; // RFC 9738 requires the saved search to be truncated to match - if let Some(saved_results) = saved_results.as_mut() { - saved_results.retain(|imap_id| { - if is_uid { - imap_id.uid >= threshold - } else { - imap_id.seqnum >= threshold - } - }); + if let (Some(saved_results), Some(threshold_uid)) = + (saved_results.as_mut(), threshold_uid) + { + saved_results.retain(|row| row.uid >= threshold_uid); } + } else { + imap_ids.sort_unstable(); } } @@ -243,8 +267,8 @@ impl SessionData { Ok(( Response { is_uid, - min: min.map(|(id, _)| id), - max: max.map(|(id, _)| id), + min: min.map(|resolved| resolved.imap_id(is_uid)), + max: max.map(|resolved| resolved.imap_id(is_uid)), count: if arguments.result_options.contains(&ResultOption::Count) { Some(total) } else { @@ -264,58 +288,48 @@ impl SessionData { imap_filter: Vec, imap_comparator: Vec, mailbox: &SelectedMailbox, - prev_saved_search: &Option>>>, + cache: &MessageStoreCache, + prev_saved_search: &Option>>>, ) -> trc::Result<(Vec, bool)> { // Obtain message ids let mut filters = Vec::with_capacity(imap_filter.len() + 1); - let cache = self - .server - .get_cached_messages(mailbox.id.account_id) - .await - .caused_by(trc::location!())?; - let message_ids = RoaringBitmap::from_iter( - cache - .in_mailbox(mailbox.id.mailbox_id) - .map(|m| m.document_id), - ); + let message_ids = mailbox.view.lock().document_ids(); // Convert query let mut include_highest_modseq = false; for filter in imap_filter { match filter { Filter::Sequence(sequence, uid_filter) => { - let mut set = RoaringBitmap::new(); - if let (Sequence::SavedSearch, Some(prev_saved_search)) = + let resolved = if let (Sequence::SavedSearch, Some(prev_saved_search)) = (&sequence, &prev_saved_search) { if let Some(prev_saved_search) = prev_saved_search { - let state = mailbox.state.lock(); - for imap_id in prev_saved_search.iter() { - if let Some(id) = state.uid_to_id.get(&imap_id.uid) { - set.insert(*id); - } - } + mailbox.view.lock().resolve( + &sequence, + true, + Some(prev_saved_search.as_slice()), + ) } else { return Err(trc::ImapEvent::Error .into_err() .details("No saved search found.")); } } else { - for id in mailbox.sequence_to_ids(&sequence, uid_filter).await?.keys() { - set.insert(*id); - } - } - filters.push(SearchFilter::is_in_set(set)); + mailbox.resolve(&sequence, uid_filter).await? + }; + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + resolved.iter().map(|resolved| resolved.id), + ))); } Filter::UidAfter(uid) => { filters.push(SearchFilter::is_in_set(match uid.checked_add(1) { - Some(min) => mailbox.uids_in_range(Some(min), None), + Some(min) => mailbox.view.lock().uids_in_range(Some(min), None), None => RoaringBitmap::new(), })); } Filter::UidBefore(uid) => { filters.push(SearchFilter::is_in_set(if uid > 1 { - mailbox.uids_in_range(None, Some(uid - 1)) + mailbox.view.lock().uids_in_range(None, Some(uid - 1)) } else { RoaringBitmap::new() })); @@ -327,104 +341,147 @@ impl SessionData { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .with_keyword(&Keyword::Answered) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Before(date) => { - filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received(date, SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))); } Filter::Deleted => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Deleted).map(|m| m.document_id), + cache + .with_keyword(&Keyword::Deleted) + .map(|m| m.document_id()), ))); } Filter::Draft => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Draft).map(|m| m.document_id), + cache.with_keyword(&Keyword::Draft).map(|m| m.document_id()), ))); } Filter::Flagged => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Flagged).map(|m| m.document_id), + cache + .with_keyword(&Keyword::Flagged) + .map(|m| m.document_id()), ))); } Filter::Keyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .with_keyword(&Keyword::from(keyword)) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Larger(size) => { - filters.push(SearchFilter::gt(EmailSearchField::Size, size)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .size(size, SearchOperator::GreaterThan) + .map(|m| m.document_id()), + ))); } Filter::On(date) => { - filters.push(SearchFilter::And); - filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); - filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400)); - filters.push(SearchFilter::End); + let date_from = date as u64; + let date_to = date_from + 86400; + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .emails + .iter() + .filter(|m| m.received_at() >= date_from && m.received_at() < date_to) + .map(|m| m.document_id()), + ))); } Filter::Seen => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Seen).map(|m| m.document_id), + cache.with_keyword(&Keyword::Seen).map(|m| m.document_id()), ))); } Filter::SentBefore(date) => { - filters.push(SearchFilter::lt(EmailSearchField::SentAt, date)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .sent(date, SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))); } Filter::SentOn(date) => { - filters.push(SearchFilter::And); - filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); - filters.push(SearchFilter::lt(EmailSearchField::SentAt, date + 86400)); - filters.push(SearchFilter::End); + let date_from = date; + let date_to = date_from + 86400; + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .emails + .iter() + .filter(|m| { + let sent_at = m.received_at() as i64 + m.sent_at() as i64; + sent_at >= date_from && sent_at < date_to + }) + .map(|m| m.document_id()), + ))); } Filter::SentSince(date) => { - filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .sent(date, SearchOperator::GreaterEqualThan) + .map(|m| m.document_id()), + ))); } Filter::Since(date) => { - filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received(date, SearchOperator::GreaterEqualThan) + .map(|m| m.document_id()), + ))); } Filter::Smaller(size) => { - filters.push(SearchFilter::lt(EmailSearchField::Size, size)); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .size(size, SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))); } Filter::Unanswered => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Answered) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Undeleted => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Deleted) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Undraft => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Draft) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Unflagged => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Flagged) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Unkeyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::from(keyword)) - .map(|m| m.document_id), + .map(|m| m.document_id()), ))); } Filter::Unseen => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.without_keyword(&Keyword::Seen).map(|m| m.document_id), + cache + .without_keyword(&Keyword::Seen) + .map(|m| m.document_id()), ))); } Filter::Recent => { @@ -447,61 +504,57 @@ impl SessionData { filters.push(SearchFilter::End);*/ } Filter::Older(secs) => { - filters.push(SearchFilter::le( - EmailSearchField::ReceivedAt, - now().saturating_sub(secs as u64), - )); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received( + now().saturating_sub(secs as u64) as i64, + SearchOperator::LowerThan, + ) + .map(|m| m.document_id()), + ))); } Filter::Younger(secs) => { - filters.push(SearchFilter::ge( - EmailSearchField::ReceivedAt, - now().saturating_sub(secs as u64), - )); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received( + now().saturating_sub(secs as u64) as i64, + SearchOperator::GreaterEqualThan, + ) + .map(|m| m.document_id()), + ))); } Filter::ModSeq((modseq, _)) => { - let mut set = RoaringBitmap::new(); - for id in self - .server - .store() - .changes( - mailbox.id.account_id, - SyncCollection::Email.into(), - Query::from_modseq(modseq), - ) - .await? - .changes - .into_iter() - .filter_map(|change| change.try_unwrap_item_id()) - { - let id = (id & u32::MAX as u64) as u32; - if message_ids.contains(id) { - set.insert(id); - } - } - filters.push(SearchFilter::is_in_set(set)); + let mailbox_id = mailbox.id.mailbox_id; + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .emails + .iter() + .filter(|m| m.change_id() + 1 >= modseq && m.has_mailbox_id(mailbox_id)) + .map(|m| m.document_id()), + ))); include_highest_modseq = true; } Filter::EmailId(id) => { - if let Ok(id) = Id::from_str(&id) { - filters.push(SearchFilter::is_in_set( - RoaringBitmap::from_sorted_iter([id.document_id()]).unwrap(), - )); - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!("Failed to parse email id '{id}'.",))); + let mut matches = RoaringBitmap::new(); + if let Some(id) = ObjectId::decode_id(id.as_bytes()) + && cache + .email_by_id(&id.document_id()) + .is_some_and(|message| message.thread_id() == id.prefix_id()) + { + matches.insert(id.document_id()); } + filters.push(SearchFilter::is_in_set(matches)); } Filter::ThreadId(id) => { - if let Ok(id) = Id::from_str(&id) { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.in_thread(id.document_id()).map(|m| m.document_id), - ))); - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!("Failed to parse thread id '{id}'.",))); - } + let matches = match ObjectId::decode_id(id.as_bytes()) + .and_then(|id| u32::try_from(id.id()).ok()) + { + Some(thread_id) => RoaringBitmap::from_iter( + cache.in_thread(thread_id).map(|m| m.document_id()), + ), + None => RoaringBitmap::new(), + }; + filters.push(SearchFilter::is_in_set(matches)); } Filter::Bcc(text) => { filters.push(SearchFilter::has_text( @@ -535,10 +588,14 @@ impl SessionData { if let Some(header) = HeaderName::parse(header) { match header { HeaderName::Subject => { - filters.push(SearchFilter::has_text_detect( - EmailSearchField::Subject, + let (text, language) = Language::detect_non_ascii( value, self.server.core.email.default_language, + ); + filters.push(SearchFilter::has_text( + EmailSearchField::Subject, + search_subject_query(&text), + language, )); } header @ (HeaderName::From @@ -558,27 +615,25 @@ impl SessionData { )); } header => { - let op = if matches!( + let op = if value.is_empty() { + KeyValueMatch::Exists + } else if matches!( header, HeaderName::MessageId | HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId - ) || value.is_empty() - { - SearchOperator::Equal + ) { + KeyValueMatch::Equals(value) } else { - SearchOperator::Contains + KeyValueMatch::Contains(value) }; - filters.push(SearchFilter::cond( - EmailSearchField::Headers, + filters.push(SearchFilter::KeyValue { + field: EmailSearchField::Headers.into(), + key: header.as_str().to_lowercase(), op, - SearchValue::KeyValues( - VecMap::with_capacity(1) - .with_append(header.as_str().to_lowercase(), value), - ), - )); + }); } } } @@ -586,7 +641,7 @@ impl SessionData { Filter::Subject(text) => { filters.push(SearchFilter::has_text_detect( EmailSearchField::Subject, - text, + search_subject_query(&text), self.server.core.email.default_language, )); } @@ -658,41 +713,48 @@ impl SessionData { let mut comparators = Vec::with_capacity(imap_comparator.len()); for comparator in imap_comparator { comparators.push(match comparator.sort { - search::Sort::Arrival => { - SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending) - } + search::Sort::Arrival => MessageComparator::Cache { + field: MessageCacheField::ReceivedAt, + ascending: comparator.ascending, + }, search::Sort::Cc => { return Err(trc::ImapEvent::Error .into_err() .details("Sorting by CC is not supported.")); } - search::Sort::Date => { - SearchComparator::field(EmailSearchField::SentAt, comparator.ascending) - } - search::Sort::From | search::Sort::DisplayFrom => { - SearchComparator::field(EmailSearchField::From, comparator.ascending) - } - search::Sort::Size => { - SearchComparator::field(EmailSearchField::Size, comparator.ascending) - } - search::Sort::Subject => { - SearchComparator::field(EmailSearchField::Subject, comparator.ascending) - } - search::Sort::To | search::Sort::DisplayTo => { - SearchComparator::field(EmailSearchField::To, comparator.ascending) - } + search::Sort::Date => MessageComparator::Cache { + field: MessageCacheField::SentAt, + ascending: comparator.ascending, + }, + search::Sort::From | search::Sort::DisplayFrom => MessageComparator::SortKey { + field: MessageSortField::From, + ascending: comparator.ascending, + }, + search::Sort::Size => MessageComparator::Cache { + field: MessageCacheField::Size, + ascending: comparator.ascending, + }, + search::Sort::Subject => MessageComparator::SortKey { + field: MessageSortField::Subject, + ascending: comparator.ascending, + }, + search::Sort::To | search::Sort::DisplayTo => MessageComparator::SortKey { + field: MessageSortField::To, + ascending: comparator.ascending, + }, }); } // Run query self.server - .search_store() - .query_account( + .query_emails( + mailbox.id.account_id, + cache, SearchQuery::new(SearchIndex::Email) .with_filters(filters) - .with_comparators(comparators) .with_account_id(mailbox.id.account_id) .with_mask(message_ids), + comparators, ) .await .map(|res| (res, include_highest_modseq)) @@ -700,8 +762,69 @@ impl SessionData { } } +struct SearchResults<'x> { + is_uid: bool, + find_min: bool, + find_max: bool, + collect_all: bool, + min: Option, + max: Option, + total: u32, + imap_ids: Vec, + saved_results: Option>, + modseq_cache: Option<&'x MessageStoreCache>, + highest_modseq: Option, +} + +impl SearchResults<'_> { + fn push(&mut self, resolved: Resolved) { + let id = resolved.imap_id(self.is_uid); + if self.find_min && self.min.is_none_or(|min| id < min.imap_id(self.is_uid)) { + self.min = Some(resolved); + } + if self.find_max && self.max.is_none_or(|max| id > max.imap_id(self.is_uid)) { + self.max = Some(resolved); + } + if self.collect_all || !(self.find_min || self.find_max) { + self.imap_ids.push(id); + self.track_modseq(&resolved); + if let Some(saved) = self.saved_results.as_mut() { + saved.push(resolved.row()); + } + } + self.total += 1; + } + + fn finish(&mut self) { + if (self.find_min || self.find_max) && !self.collect_all { + let max = self.max.filter(|max| { + self.min + .is_none_or(|min| min.imap_id(self.is_uid) != max.imap_id(self.is_uid)) + }); + for resolved in [self.min, max].into_iter().flatten() { + self.imap_ids.push(resolved.imap_id(self.is_uid)); + self.track_modseq(&resolved); + if let Some(saved) = self.saved_results.as_mut() { + saved.push(resolved.row()); + } + } + } + } + + fn track_modseq(&mut self, resolved: &Resolved) { + if let Some(cache) = self.modseq_cache + && let Some(item) = cache.email_by_id(&resolved.id) + { + let modseq = item.change_id() + 1; + if self.highest_modseq.is_none_or(|highest| modseq > highest) { + self.highest_modseq = Some(modseq); + } + } + } +} + impl SelectedMailbox { - pub async fn get_saved_search(&self) -> Option>> { + pub async fn get_saved_search(&self) -> Option>> { let mut rx = match &*self.saved_search.lock() { SavedSearch::InFlight { rx } => rx.clone(), SavedSearch::Results { items } => { @@ -715,66 +838,10 @@ impl SelectedMailbox { let v = rx.borrow(); Some(v.clone()) } - - #[allow(clippy::too_many_arguments)] - pub fn map_search_results( - &self, - ids: impl Iterator, - is_uid: bool, - collect_all: bool, - find_min: bool, - find_max: bool, - min: &mut Option<(u32, ImapId)>, - max: &mut Option<(u32, ImapId)>, - total: &mut u32, - imap_ids: &mut Vec, - saved_results: &mut Option>, - ) { - let state = self.state.lock(); - let find_min_or_max = find_min || find_max; - for document_id in ids { - if let Some((id, imap_id)) = state.map_result_id(document_id, is_uid) { - if find_min { - if let Some((prev_min, _)) = min { - if id < *prev_min { - *min = Some((id, imap_id)); - } - } else { - *min = Some((id, imap_id)); - } - } - if find_max { - if let Some((prev_max, _)) = max { - if id > *prev_max { - *max = Some((id, imap_id)); - } - } else { - *max = Some((id, imap_id)); - } - } - if collect_all || !find_min_or_max { - imap_ids.push(id); - if let Some(r) = saved_results.as_mut() { - r.push(imap_id) - } - } - *total += 1; - } - } - if find_min_or_max && !collect_all { - let max = max.filter(|(max_id, _)| min.is_none_or(|(min_id, _)| min_id != *max_id)); - for (id, imap_id) in [*min, max].into_iter().flatten() { - imap_ids.push(id); - if let Some(r) = saved_results.as_mut() { - r.push(imap_id) - } - } - } - } } impl SavedSearch { - pub async fn unwrap(&self) -> Option>> { + pub async fn unwrap(&self) -> Option>> { match self { SavedSearch::InFlight { rx } => { let mut rx = rx.clone(); diff --git a/crates/imap/src/op/select.rs b/crates/imap/src/op/select.rs index 7ad3db48..267c5df9 100644 --- a/crates/imap/src/op/select.rs +++ b/crates/imap/src/op/select.rs @@ -5,23 +5,32 @@ */ use super::{ImapContext, ToModSeq}; -use crate::core::{SavedSearch, SelectedMailbox, Session, State}; +use crate::core::{ + MailboxRefresh, MailboxView, SavedSearch, SelectedMailbox, Session, State, + rights::MailboxRights, +}; use common::network::SessionStream; +use email::cache::mailbox::MailboxCacheAccess; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ ImapResponse, ObjectId, Sequence, fetch, list::ListItem, - select::{HighestModSeq, Response}, + select::{HighestModSeq, PermanentFlags, Response}, }, receiver::Request, }; use registry::schema::enums::Permission; use std::{sync::Arc, time::Instant}; -use types::{acl::Acl, id::Id}; +use tokio::sync::OwnedSemaphorePermit; +use types::acl::Acl; impl Session { - pub async fn handle_select(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_select( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(if request.command == Command::Select { Permission::ImapSelect @@ -46,7 +55,21 @@ impl Session { let want_objectid = self.is_objectid; // Refresh mailboxes - data.synchronize_mailboxes(false) + if let Some(account_id) = arguments + .objectid + .as_ref() + .and_then(|objectid| objectid.account_id) + .and_then(|account_id| u32::try_from(account_id.id()).ok()) + { + data.use_messages(account_id); + } + data.use_messages_of(&arguments.mailbox_name); + let MailboxRefresh { + mut caches, + access_token, + .. + } = data + .synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -54,35 +77,40 @@ impl Session { let mailbox = arguments .objectid .as_ref() - .and_then( - |objectid| match (objectid.account_id, objectid.mailbox_id) { - (Some(account_id), Some(mailbox_id)) => { - data.get_mailbox_by_id(account_id.document_id(), mailbox_id.document_id()) - } - _ => None, - }, - ) - .or_else(|| data.get_mailbox_by_name(&arguments.mailbox_name)); + .and_then(|objectid| data.get_mailbox_by_object_id(objectid, &access_token)) + .or_else(|| { + data.get_mailbox_by_name(&arguments.mailbox_name) + .map(|mailbox| (mailbox, arguments.mailbox_name)) + }); - if let Some(mailbox) = mailbox { - if !data - .check_mailbox_acl(mailbox.account_id, mailbox.mailbox_id, Acl::ReadItems) + if let Some((mailbox, mailbox_name)) = mailbox { + let cache = caches + .fetch(&data.server, mailbox.account_id) .await - .imap_ctx(&arguments.tag, trc::location!())? - { - return Err(trc::ImapEvent::Error - .into_err() - .details("You do not have the required permissions to read this mailbox.") - .code(ResponseCode::NoPerm) - .id(arguments.tag)); + .imap_ctx(&arguments.tag, trc::location!())?; + data.use_messages(mailbox.account_id); + let acl = MailboxRights::new(&access_token, mailbox.account_id) + .acl(&cache, mailbox.mailbox_id); + if !acl.contains(Acl::ReadItems) { + return Err(if cache.has_mailbox_id(&mailbox.mailbox_id) { + trc::ImapEvent::Error + .into_err() + .details("You do not have the required permissions to read this mailbox.") + .code(ResponseCode::NoPerm) + .id(arguments.tag) + } else { + trc::ImapEvent::Error + .into_err() + .details("Mailbox no longer exists.") + .id(arguments.tag) + }); } - - // Try obtaining the mailbox from the cache - let state = data - .fetch_messages(&mailbox, None) + let view = MailboxView::build(&cache, mailbox.mailbox_id); + let uid_validity = data.uid_validity(&mailbox); + let uid_next = data + .uid_next(&cache, &mailbox) .await - .imap_ctx(&arguments.tag, trc::location!())? - .unwrap(); + .imap_ctx(&arguments.tag, trc::location!())?; // Synchronize messages let closed_previous = self.state.close_mailbox(); @@ -91,57 +119,80 @@ impl Session { // Build new state let is_rev2 = self.version.is_rev2(); let is_utf8 = self.is_utf8; - let mailbox_state = data.mailbox_state(&mailbox).unwrap(); - let total_messages = state.total_messages; + let is_read_write = is_select + && acl + .contains_any([Acl::AddItems, Acl::RemoveItems, Acl::ModifyItems].into_iter()); + let total_messages = view.len(); let highest_modseq = if is_condstore { - HighestModSeq::new(state.modseq.to_modseq()).into() + HighestModSeq::new(view.change_id().to_modseq()).into() } else { None }; let mailbox = Arc::new(SelectedMailbox { id: mailbox, - state: parking_lot::Mutex::new(state), + view: parking_lot::Mutex::new(view), saved_search: parking_lot::Mutex::new(SavedSearch::None), is_select, is_condstore, }); // Validate QRESYNC arguments - if let Some(qresync) = arguments.qresync { - if !self.is_qresync { - return Err(trc::ImapEvent::Error - .into_err() - .details("QRESYNC is not enabled.") - .id(arguments.tag)); + let qresync = match arguments.qresync { + Some(qresync) => { + if !self.is_qresync { + return Err(trc::ImapEvent::Error + .into_err() + .details("QRESYNC is not enabled.") + .id(arguments.tag)); + } + if self.is_uidonly && qresync.seq_match.is_some() { + return Err(trc::ImapEvent::Error + .into_err() + .details(concat!( + "The QRESYNC sequence matching parameter ", + "is not allowed once UIDONLY is enabled." + )) + .code(ResponseCode::UidRequired) + .ctx(trc::Key::Type, ResponseType::Bad) + .id(arguments.tag)); + } + (qresync.uid_validity == uid_validity).then_some(qresync) } - if self.is_uidonly && qresync.seq_match.is_some() { - return Err(trc::ImapEvent::Error - .into_err() - .details(concat!( - "The QRESYNC sequence matching parameter ", - "is not allowed once UIDONLY is enabled." - )) - .code(ResponseCode::UidRequired) - .ctx(trc::Key::Type, ResponseType::Bad) - .id(arguments.tag)); - } - if qresync.uid_validity == mailbox_state.uid_validity as u32 { - // Send flags for changed messages + None => None, + }; + + // Build response + let response = Response { + mailbox: ListItem::new(mailbox_name.clone()), + total_messages, + recent_messages: 0, + unseen_seq: 0, + uid_validity, + uid_next, + closed_previous, + is_rev2, + is_utf8, + highest_modseq, + objectid: want_objectid.then(|| ObjectId::from(mailbox.id)), + permanent_flags: PermanentFlags::from(acl), + }; + + let untagged = match qresync { + Some(qresync) => { + self.write_bytes(response.serialize()).await?; data.fetch( fetch::Arguments { tag: "".into(), - sequence_set: qresync - .known_uids - .or_else(|| qresync.seq_match.map(|(_, s)| s)) - .unwrap_or(Sequence::Range { - start: 1.into(), - end: None, - }), + sequence_set: qresync.known_uids.unwrap_or(Sequence::Range { + start: 1.into(), + end: None, + }), attributes: vec![fetch::Attribute::Flags], changed_since: qresync.modseq.into(), include_vanished: true, }, mailbox.clone(), + Some(cache), true, true, self.is_uidonly, @@ -152,53 +203,37 @@ impl Session { ) .await .imap_ctx(&arguments.tag, trc::location!())?; + None } - } + None => Some(response), + }; trc::event!( Imap(trc::ImapEvent::Select), SpanId = self.session_id, - MailboxName = arguments.mailbox_name.clone(), + MailboxName = mailbox_name, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Total = total_messages, - UidNext = mailbox_state.uid_next, - UidValidity = mailbox_state.uid_validity, + UidNext = uid_next, + UidValidity = uid_validity, Elapsed = op_start.elapsed() ); - // Build response - let response = Response { - mailbox: ListItem::new(arguments.mailbox_name), - total_messages, - recent_messages: 0, - unseen_seq: 0, - uid_validity: mailbox_state.uid_validity as u32, - uid_next: mailbox_state.uid_next as u32, - closed_previous, - is_rev2, - is_utf8, - highest_modseq, - objectid: want_objectid.then(|| ObjectId { - mailbox_id: Some(Id::from(mailbox.id.mailbox_id)), - account_id: Some(Id::from(mailbox.id.account_id)), - ..Default::default() - }), - }; - // Update state self.state = State::Selected { data, mailbox }; - self.write_bytes( - StatusResponse::completed(command) - .with_tag(arguments.tag) - .with_code(if is_select { - ResponseCode::ReadWrite - } else { - ResponseCode::ReadOnly - }) - .serialize(response.serialize()), - ) + let completed = StatusResponse::completed(command) + .with_tag(arguments.tag) + .with_code(if is_read_write { + ResponseCode::ReadWrite + } else { + ResponseCode::ReadOnly + }); + self.write_bytes(match untagged { + Some(response) => completed.serialize_after(&response), + None => completed.into_bytes(), + }) .await } else { Err(trc::ImapEvent::Error @@ -209,7 +244,11 @@ impl Session { } } - pub async fn handle_unselect(&mut self, request: Request) -> trc::Result<()> { + pub async fn handle_unselect( + &mut self, + request: Request, + _permit: Option, + ) -> trc::Result<()> { self.state.close_mailbox(); self.state = State::Authenticated { data: self.state.session_data(), diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index a1a699fb..772bb6a1 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -6,28 +6,33 @@ use super::ToModSeq; use crate::{ - core::{Mailbox, Session, SessionData}, + core::{AccountCaches, MailboxId, Session, SessionData}, op::ImapContext, spawn_op, }; -use common::network::SessionStream; -use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; +use common::{MessageStoreCache, auth::AccessToken, network::SessionStream}; +use compact_str::{CompactString, format_compact}; +use email::cache::mailbox::MailboxCacheAccess; use imap_proto::{ Command, ResponseCode, StatusResponse, - parser::PushUnique, protocol::{ - ObjectId, + MailboxObjectId, ObjectId, status::{Status, StatusItem, StatusItemType}, }, receiver::Request, }; use registry::schema::enums::Permission; use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; -use types::{acl::Acl, id::Id, keyword::Keyword}; +use types::acl::Acl; impl Session { - pub async fn handle_status(&mut self, requests: Vec>) -> trc::Result<()> { + pub async fn handle_status( + &mut self, + requests: Vec>, + permit: Option, + ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; @@ -52,32 +57,39 @@ impl Session { } let data = self.state.session_data(); + for arguments in parsed.iter().flatten() { + data.use_messages_of(&arguments.mailbox_name); + } - spawn_op!(data, { - let mut did_sync = false; + spawn_op!(permit, data, { + let mut caches = None; for request in parsed { match request { Ok(arguments) => { let op_start = Instant::now(); - let synchronized = if did_sync { - Ok(()) - } else { - // Refresh mailboxes - data.synchronize_mailboxes(false) - .await - .imap_ctx(&arguments.tag, trc::location!()) - .map(|_| ()) + let synchronized = match &mut caches { + Some(refresh) => Ok(refresh), + None => { + // Refresh mailboxes + data.synchronize_mailboxes(false) + .await + .imap_ctx(&arguments.tag, trc::location!()) + .map(|refresh| caches.insert(refresh)) + } }; // Fetch status let status = match synchronized { - Ok(()) => { - did_sync = true; - data.status(arguments.mailbox_name, &arguments.items) - .await - .imap_ctx(&arguments.tag, trc::location!()) - } + Ok(refresh) => data + .status( + &mut refresh.caches, + &refresh.access_token, + arguments.mailbox_name, + &arguments.items, + ) + .await + .imap_ctx(&arguments.tag, trc::location!()), Err(err) => Err(err), }; @@ -90,7 +102,7 @@ impl Session { Details = arguments .items .iter() - .map(|c| trc::Value::from(format!("{c:?}"))) + .map(|c| trc::Value::from(format_compact!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); @@ -117,7 +129,13 @@ impl Session { } impl SessionData { - pub async fn status(&self, mailbox_name: String, items: &[Status]) -> trc::Result { + pub async fn status( + &self, + caches: &mut AccountCaches, + access_token: &AccessToken, + mailbox_name: CompactString, + items: &[Status], + ) -> trc::Result { // Get mailbox id let mailbox = if let Some(mailbox) = self.get_mailbox_by_name(&mailbox_name) { mailbox @@ -164,8 +182,28 @@ impl SessionData { }; }; + let cache = caches.fetch(&self.server, mailbox.account_id).await?; + + self.status_in(&cache, access_token, mailbox, mailbox_name, items) + .await + } + + pub async fn status_in( + &self, + cache: &MessageStoreCache, + access_token: &AccessToken, + mailbox: MailboxId, + mailbox_name: CompactString, + items: &[Status], + ) -> trc::Result { if !self - .check_mailbox_acl(mailbox.account_id, mailbox.mailbox_id, Acl::ReadItems) + .check_mailbox_acl_as( + access_token, + Some(cache), + mailbox.account_id, + mailbox.mailbox_id, + Acl::ReadItems, + ) .await .caused_by(trc::location!())? { @@ -174,147 +212,87 @@ impl SessionData { .details("You do not have the required permissions to read this mailbox.") .code(ResponseCode::NoPerm)); } + let uid_validity = cache + .mailbox_by_id(&mailbox.mailbox_id) + .map(|cached| cached.uid_validity) + .ok_or_else(|| { + trc::ImapEvent::Error + .into_err() + .details("Mailbox does not exist.") + .code(ResponseCode::NonExistent) + })?; + self.use_messages(mailbox.account_id); - // Make sure all requested fields are up to date - let mut items_update = Vec::with_capacity(items.len()); - let mut items_response = Vec::with_capacity(items.len()); + let uid_next = if items.contains(&Status::UidNext) { + Some(self.uid_next(cache, &mailbox).await?) + } else { + None + }; + let needs_counters = items.iter().any(|item| { + matches!( + item, + Status::Messages + | Status::Unseen + | Status::Deleted + | Status::DeletedStorage + | Status::Size + ) + }); - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == mailbox.account_id { - let mailbox_state = - if let Some(mailbox_state) = account.mailbox_state.get(&mailbox.mailbox_id) { - mailbox_state - } else { - continue; - }; - for item in items { - match item { - Status::Messages => { - items_response.push(( - *item, - StatusItemType::Number(mailbox_state.total_messages), - )); - } - Status::UidNext => { - items_response - .push((*item, StatusItemType::Number(mailbox_state.uid_next))); - } - Status::UidValidity => { - items_response - .push((*item, StatusItemType::Number(mailbox_state.uid_validity))); - } - Status::Unseen => { - items_response - .push((*item, StatusItemType::Number(mailbox_state.total_unseen))); - } - Status::Deleted => { - items_response - .push((*item, StatusItemType::Number(mailbox_state.total_deleted))); - } - Status::DeletedStorage => { - if let Some(value) = mailbox_state.total_deleted_storage { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } - } - Status::Size => { - if let Some(value) = mailbox_state.size { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } - } - Status::HighestModSeq => { - items_response.push(( - *item, - StatusItemType::Number(account.last_change_id.to_modseq()), - )); - } - Status::ObjectId => { - items_response.push(( - *item, - StatusItemType::ObjectId(ObjectId { - mailbox_id: Some(Id::from(mailbox.mailbox_id)), - account_id: Some(Id::from(mailbox.account_id)), - ..Default::default() - }), - )); - } - Status::Recent => { - items_response.push((*item, StatusItemType::Number(0))); - } - } - } - break; - } + if needs_counters { + self.ensure_counters(mailbox.account_id, cache); } - if !items_update.is_empty() { - // Retrieve latest values - let mut values_update = Vec::with_capacity(items_update.len()); + let accounts = self.mailboxes.lock(); + let account = accounts + .iter() + .find(|account| account.account_id == mailbox.account_id) + .ok_or_else(|| { + trc::ImapEvent::Error + .into_err() + .details("Mailbox does not exist.") + .code(ResponseCode::NonExistent) + })?; + let counters = if needs_counters { + account.counters_of(mailbox.mailbox_id) + } else { + Default::default() + }; + let objectid = account.entry_by_name(&mailbox_name).map_or_else( + || MailboxObjectId::from(mailbox), + |entry| entry.object_id(account.account_id), + ); - let cache = self - .server - .get_cached_messages(mailbox.account_id) - .await - .caused_by(trc::location!())?; + let is_deferred = |item: &Status| matches!(item, Status::Size | Status::DeletedStorage); - for item in items_update { - let result = match item { - Status::DeletedStorage => cache - .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) - .map(|x| x.size) - .sum::() as u64, - Status::Size => cache - .in_mailbox(mailbox.mailbox_id) - .map(|x| x.size) - .sum::() as u64, - _ => { - unreachable!() - } - }; - - items_response.push((item, StatusItemType::Number(result))); - values_update.push((item, result)); - } - - // Update cache - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == mailbox.account_id { - let mailbox_state = account - .mailbox_state - .entry(mailbox.mailbox_id) - .or_insert_with(Mailbox::default); - - for (item, value) in values_update { - match item { - Status::DeletedStorage => { - mailbox_state.total_deleted_storage = value.into() - } - Status::Size => mailbox_state.size = value.into(), - Status::Recent => { - items_response - .iter_mut() - .find(|(i, _)| *i == Status::Recent) - .unwrap() - .1 = StatusItemType::Number(0); - } - _ => { - unreachable!() - } - } - } - - break; - } - } - } - - // Generate response Ok(StatusItem { mailbox_name, - items: items_response, + items: items + .iter() + .filter(|item| !is_deferred(item)) + .chain(items.iter().filter(|item| is_deferred(item))) + .map(|item| { + ( + *item, + match item { + Status::Messages => StatusItemType::Number(counters.total as u64), + Status::UidNext => { + StatusItemType::Number(uid_next.unwrap_or_default() as u64) + } + Status::UidValidity => StatusItemType::Number(uid_validity as u64), + Status::Unseen => StatusItemType::Number(counters.unseen as u64), + Status::Deleted => StatusItemType::Number(counters.deleted as u64), + Status::DeletedStorage => StatusItemType::Number(counters.deleted_size), + Status::Size => StatusItemType::Number(counters.size), + Status::HighestModSeq => { + StatusItemType::Number(cache.emails.change_id.to_modseq()) + } + Status::ObjectId => StatusItemType::ObjectId(objectid.into()), + Status::Recent => StatusItemType::Number(0), + }, + ) + }) + .collect(), }) } } diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 8a33843e..0395b6ab 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -4,38 +4,44 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{FromModSeq, ImapContext}; -use crate::{ - core::{SelectedMailbox, Session, SessionData}, - spawn_op, +use super::{ImapContext, IntoImapError}; +use crate::core::{SelectedMailbox, Session, SessionData, rights::MailboxRights}; +use common::{ + MessageStoreCache, + cache::email::MessageRef, + config::mailstore::limits::{EmailLimitError, EmailLimits}, + network::SessionStream, }; -use ahash::AHashSet; -use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; +use compact_str::{CompactString, format_compact}; use email::{ + cache::{ + MessageCacheFetch, + email::{KeywordWrites, MessageCacheAccess}, + }, mailbox::TRASH_ID, - message::{ingest::EmailIngest, metadata::MessageData}, + message::{ + ingest::{EmailIngest, SpamSamples}, + messagedata::{KeywordDiff, SERVER_SET_KEYWORDS, merge_keywords}, + }, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ Flag, ImapResponse, fetch::{DataItem, FetchItem}, + select::PermanentFlags, store::{Arguments, Operation, Response}, }, receiver::Request, }; use registry::schema::enums::Permission; use std::{sync::Arc, time::Instant}; -use store::{ - ValueKey, - query::log::{Change, Query}, - write::{AlignedBytes, Archive, BatchBuilder}, -}; +use store::write::{BatchBuilder, PendingId}; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; use types::{ - acl::Acl, collection::{Collection, SyncCollection}, - keyword::Keyword, + keyword::{DELETED, Keyword}, }; impl Session { @@ -43,7 +49,7 @@ impl Session { &mut self, request: Request, is_uid: bool, - spawn: bool, + _permit: Option, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStore)?; @@ -52,42 +58,199 @@ impl Session { let arguments = request.parse_store()?; let (data, mailbox) = self.state.select_data(); let is_condstore = self.is_condstore || mailbox.is_condstore; - let is_utf8 = self.is_utf8; let is_uidonly = self.is_uidonly; let message_limit = self.server.core.imap.max_messages_per_command; - if spawn { - spawn_op!(data, { - let response = data - .store( - arguments, - mailbox, - is_uid, - is_condstore, - is_utf8, - is_uidonly, - message_limit, - op_start, - ) - .await?; + let response = data + .store( + arguments, + mailbox, + is_uid, + is_condstore, + is_uidonly, + message_limit, + op_start, + ) + .await?; - data.write_bytes(response).await - }) + data.write_bytes(response).await + } +} + +const DELETED_BIT: u32 = 1 << DELETED; + +#[derive(Default)] +struct KeywordEdit { + added: u32, + removed: u32, + added_extra: Vec, + removed_extra: Vec, +} + +impl KeywordEdit { + fn compute( + cache: &MessageStoreCache, + message: MessageRef<'_>, + operation: &Operation, + keywords: &[Keyword], + ) -> Self { + let mut edit = KeywordEdit::default(); + match operation { + Operation::Set => { + edit.add_missing(cache, message, keywords); + for keyword in cache.expand_keywords(message).filter(|keyword| { + !matches!(keyword, Keyword::HasAttachment | Keyword::HasNoAttachment) + && !keywords.contains(keyword) + }) { + match keyword.into_id() { + Ok(id) => edit.removed |= 1 << id, + Err(name) => edit.removed_extra.push(name), + } + } + } + Operation::Add => { + edit.add_missing(cache, message, keywords); + } + Operation::Clear => { + for keyword in keywords + .iter() + .filter(|keyword| cache.has_keyword(message, keyword)) + { + match keyword.id() { + Ok(id) => edit.removed |= 1 << id, + Err(name) => edit.removed_extra.push(name.into()), + } + } + } + } + edit + } + + fn restrict(&mut self, rights: PermanentFlags) { + match rights { + PermanentFlags::All => {} + PermanentFlags::AllButDeleted => { + self.added &= !DELETED_BIT; + self.removed &= !DELETED_BIT; + } + PermanentFlags::DeletedOnly => { + self.added &= DELETED_BIT; + self.removed &= DELETED_BIT; + self.added_extra.clear(); + self.removed_extra.clear(); + } + PermanentFlags::None => *self = KeywordEdit::default(), + } + } + + fn add_missing( + &mut self, + cache: &MessageStoreCache, + message: MessageRef<'_>, + keywords: &[Keyword], + ) { + for keyword in keywords + .iter() + .filter(|keyword| !cache.has_keyword(message, keyword)) + { + match keyword.id() { + Ok(id) => self.added |= 1 << id, + Err(name) => self.added_extra.push(name.into()), + } + } + } + + fn net_change(&self) -> isize { + ((self.added & !SERVER_SET_KEYWORDS).count_ones() as isize + + self.added_extra.len() as isize) + - ((self.removed & !SERVER_SET_KEYWORDS).count_ones() as isize + + self.removed_extra.len() as isize) + } + + fn validate_limits( + &self, + cache: &MessageStoreCache, + message: MessageRef<'_>, + limits: &EmailLimits, + ) -> Result<(), EmailLimitError> { + if self.added & !SERVER_SET_KEYWORDS == 0 && self.added_extra.is_empty() { + return Ok(()); + } + let prev_count = cache.keyword_count(message); + limits.validate_keyword_change( + prev_count, + prev_count.saturating_add_signed(self.net_change()), + )?; + limits.validate_keyword_names(self.added_extra.iter().map(|name| name.as_str())) + } + + fn is_empty(&self) -> bool { + self.added == 0 + && self.removed == 0 + && self.added_extra.is_empty() + && self.removed_extra.is_empty() + } + + fn adds(&self, keyword: &Keyword) -> bool { + match keyword.id() { + Ok(id) => self.added & (1 << id) != 0, + Err(name) => self.added_extra.iter().any(|extra| extra == name), + } + } + + fn removes(&self, keyword: &Keyword) -> bool { + match keyword.id() { + Ok(id) => self.removed & (1 << id) != 0, + Err(name) => self.removed_extra.iter().any(|extra| extra == name), + } + } + + fn spam_training(&self, message: MessageRef<'_>) -> Option { + if self.adds(&Keyword::Junk) { + Some(true) + } else if !message.has_mailbox_id(TRASH_ID) + && (self.adds(&Keyword::NotJunk) || self.removes(&Keyword::Junk)) + { + Some(false) } else { - let response = data - .store( - arguments, - mailbox, - is_uid, - is_condstore, - is_utf8, - is_uidonly, - message_limit, - op_start, - ) - .await?; + None + } + } - data.write_bytes(response).await + fn flags( + &self, + cache: &MessageStoreCache, + message: MessageRef<'_>, + keywords: &[Keyword], + ) -> Vec { + cache + .expand_keywords(message) + .filter(|keyword| !self.removes(keyword)) + .chain( + keywords + .iter() + .filter(|keyword| self.adds(keyword)) + .cloned(), + ) + .map(Flag::from) + .collect() + } + + fn into_diff( + self, + operation: &Operation, + keywords: &[Keyword], + rights: PermanentFlags, + ) -> KeywordDiff { + if *operation == Operation::Set && rights == PermanentFlags::All { + KeywordDiff::replace(keywords.to_vec()) + } else { + KeywordDiff::Patch { + added: self.added, + removed: self.removed, + added_extra: self.added_extra, + removed_extra: self.removed_extra, + } } } } @@ -100,20 +263,24 @@ impl SessionData { mailbox: Arc, is_uid: bool, is_condstore: bool, - is_utf8: bool, is_uidonly: bool, message_limit: u32, op_start: Instant, ) -> trc::Result> { // Resync messages if needed let account_id = mailbox.id.account_id; - self.synchronize_messages(&mailbox) + let cache = self + .server + .get_cached_messages(account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + self.sync_view(&mailbox, &cache, None) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert IMAP ids to JMAP ids. let mut ids = mailbox - .sequence_to_ids(&arguments.sequence_set, is_uid) + .resolve(&arguments.sequence_set, is_uid) .await .imap_ctx(&arguments.tag, trc::location!())?; if ids.is_empty() { @@ -123,14 +290,14 @@ impl SessionData { } // Verify that the user can modify messages in this mailbox. - if !self - .check_mailbox_acl( - mailbox.id.account_id, - mailbox.id.mailbox_id, - Acl::ModifyItems, - ) + let access_token = self + .refresh_access_token() .await - .imap_ctx(&arguments.tag, trc::location!())? + .imap_ctx(&arguments.tag, trc::location!())?; + let mut rights = MailboxRights::new(&access_token, account_id); + if !rights + .permanent_flags(&cache, [mailbox.id.mailbox_id]) + .permits_any(&arguments.operation, &arguments.keywords) { return Err(trc::ImapEvent::Error .into_err() @@ -146,46 +313,32 @@ impl SessionData { let mut response_code = None; let mut unchanged_failed = false; if let Some(unchanged_since) = arguments.unchanged_since { - // Obtain changes since the modseq. - let changelog = self - .server - .store() - .changes( - account_id, - SyncCollection::Email.into(), - Query::from_modseq(unchanged_since), - ) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + let (mut modified, expunged) = + mailbox.missing_in(&arguments.sequence_set, is_uid).await; + unchanged_failed = expunged && !is_uid; - let mut modified = mailbox - .sequence_expand_missing(&arguments.sequence_set, is_uid) - .await; - - // Add all IDs that changed in this mailbox - for (id, is_delete) in changelog.changes.into_iter().filter_map(|change| { - change.item_id().map(|id| { - ( - (id & u32::MAX as u64) as u32, - matches!(change, Change::DeleteItem(_)), - ) - }) - }) { - if let Some(imap_id) = ids.remove(&id) { - if is_uid { - modified.push(imap_id.uid); - } else { - modified.push(imap_id.seqnum); - if is_delete { - unchanged_failed = true; - } - } + ids.retain(|resolved| { + let is_modified = cache + .email_by_id(&resolved.id) + .is_none_or(|message| message.change_id() >= unchanged_since); + if is_modified { + let imap_id = resolved.imap_id(is_uid); + modified.push((imap_id, imap_id)); } - } + !is_modified + }); if !modified.is_empty() { modified.sort_unstable(); - response_code = ResponseCode::Modified { ids: modified }.into(); + modified.dedup_by(|next, prev| { + if next.0 <= prev.1.saturating_add(1) { + prev.1 = prev.1.max(next.1); + true + } else { + false + } + }); + response_code = ResponseCode::Modified { ranges: modified }.into(); } } @@ -205,25 +358,27 @@ impl SessionData { SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, - Type = format!("{:?}", arguments.operation), + Type = format_compact!("{:?}", arguments.operation), Details = arguments .keywords .iter() - .map(|c| trc::Value::from(format!("{c:?}"))) + .map(|c| trc::Value::from(format_compact!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); return Ok(response.into_bytes()); } + // RFC 9738 requires the highest UIDs to be processed first when truncating. let message_limit = message_limit as usize; let mut untagged = Vec::new(); if ids.len() > message_limit { - let mut uids = ids.values().map(|imap_id| imap_id.uid).collect::>(); - let cutoff = uids.len() - message_limit; - let lowest_uid = *uids.select_nth_unstable(cutoff).1; - ids.retain(|_, imap_id| imap_id.uid >= lowest_uid); + let cutoff = ids.len() - message_limit; + ids.select_nth_unstable_by_key(cutoff, |resolved| resolved.uid); + ids.drain(..cutoff); + ids.sort_unstable_by_key(|resolved| resolved.uid); + let lowest_uid = ids.first().map_or(0, |resolved| resolved.uid); let code = ResponseCode::MessageLimit { limit: message_limit as u32, @@ -239,7 +394,6 @@ impl SessionData { } let mut items = Response { - is_utf8, items: Vec::with_capacity(ids.len()), }; @@ -247,102 +401,84 @@ impl SessionData { let set_keywords = arguments .keywords .iter() - .map(|k| Keyword::from(k.clone())) + .map(|keyword| Keyword::from(keyword.clone())) .collect::>(); - let mut changed_mailboxes = AHashSet::new(); + let mut changed_mailboxes: Vec = Vec::new(); + let mut unchanged = Vec::new(); + let is_unchanged_since = arguments.unchanged_since.is_some(); let mut batch = BatchBuilder::new(); + let mut spam_samples = SpamSamples::default(); + let mut writes = self + .server + .core + .storage + .data + .is_embedded() + .then(|| KeywordWrites::with_capacity(ids.len())); - for (id, imap_id) in &ids { + for resolved in &ids { // Obtain message data - let data_ = if let Some(data) = self - .server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - *id, - )) - .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? - { - data - } else { + let Some(message) = cache.email_by_id(&resolved.id) else { continue; }; - // Deserialize - let data = data_ - .to_unarchived::() - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - let mut new_data = data.inner.to_builder(); - // Apply changes - let mut seen_changed = false; - match arguments.operation { - Operation::Set => { - seen_changed = set_keywords.contains(&Keyword::Seen) - != new_data.has_keyword(&Keyword::Seen); - new_data.set_keywords(set_keywords.clone()); - } - Operation::Add => { - for keyword in &set_keywords { - if new_data.add_keyword(keyword.clone()) && keyword == &Keyword::Seen { - seen_changed = true; - } + let mut edit = + KeywordEdit::compute(&cache, message, &arguments.operation, &set_keywords); + let is_requested = !edit.is_empty(); + let message_rights = rights.permanent_flags( + &cache, + message + .mailboxes() + .iter() + .map(|membership| membership.mailbox_id), + ); + edit.restrict(message_rights); + if edit.is_empty() { + if (is_requested && !arguments.is_silent) || is_unchanged_since { + let mut data_items = Vec::with_capacity(3); + if !arguments.is_silent { + data_items.push(DataItem::Flags { + flags: edit.flags(&cache, message, &set_keywords), + }); } - } - Operation::Clear => { - for keyword in &set_keywords { - if new_data.remove_keyword(keyword) && keyword == &Keyword::Seen { - seen_changed = true; - } + if is_uid { + data_items.push(DataItem::Uid { uid: resolved.uid }); } + if is_condstore || is_unchanged_since { + data_items.push(DataItem::ModSeq { + modseq: message.change_id() + 1, + }); + } + unchanged.push(FetchItem { + id: resolved.imap_id(is_uidonly), + is_uidonly, + items: data_items, + }); } - } - - if !new_data.has_keyword_changes(data.inner) { continue; } + if let Err(err) = edit.validate_limits(&cache, message, &self.server.core.email.limits) + { + return Err(err.into_imap_error().id(response.tag.unwrap_or_default())); + } // Train spam filter - let mut train_spam = None; - for keyword in new_data.added_keywords(data.inner) { - if keyword == &Keyword::Junk { - train_spam = Some(true); - break; - } else if keyword == &Keyword::NotJunk && !data.inner.has_mailbox_id(TRASH_ID) { - // Only train as ham if not in Trash (Apple likes to add NotJunk to trashed items, which would be spammy) - train_spam = Some(false); - break; - } - } - if train_spam.is_none() { - for keyword in new_data.removed_keywords(data.inner) { - if keyword == &Keyword::Junk { - if !data.inner.has_mailbox_id(TRASH_ID) { - train_spam = Some(false); - } - break; - } - } - } + let train_spam = edit.spam_training(message); // Convert keywords to flags let flags = if !arguments.is_silent { - new_data - .keywords - .iter() - .cloned() - .map(Flag::from) - .collect::>() + edit.flags(&cache, message, &set_keywords) } else { vec![] }; // Set all current mailboxes as changed if the Seen tag changed - if seen_changed { - for mailbox_id in new_data.mailboxes.iter() { - changed_mailboxes.insert(mailbox_id.mailbox_id); + if edit.adds(&Keyword::Seen) || edit.removes(&Keyword::Seen) { + for membership in message.mailboxes() { + if !changed_mailboxes.contains(&membership.mailbox_id) { + changed_mailboxes.push(membership.mailbox_id); + } } } @@ -350,26 +486,18 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email) - .with_document(*id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), - ) - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + .with_document(resolved.id); + let diff = edit.into_diff(&arguments.operation, &set_keywords, message_rights); + if let Some(pending) = &mut writes + && !pending.push(message, &diff, batch.commit_point_count()) + { + writes = None; + } + merge_keywords(&mut batch, message.thread_id(), diff); // Add spam train task if let Some(learn_spam) = train_spam { - self.server - .add_account_spam_sample( - &mut batch, - account_id, - *id, - learn_spam, - self.session_id, - ) - .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + spam_samples.add(resolved.id, learn_spam); } // Set commit point @@ -379,27 +507,19 @@ impl SessionData { if !arguments.is_silent { let mut data_items = vec![DataItem::Flags { flags }]; if is_uid { - data_items.push(DataItem::Uid { uid: imap_id.uid }); + data_items.push(DataItem::Uid { uid: resolved.uid }); } items.items.push(FetchItem { - id: if is_uidonly { - imap_id.uid - } else { - imap_id.seqnum - }, + id: resolved.imap_id(is_uidonly), is_uidonly, items: data_items, }); } else if is_condstore { items.items.push(FetchItem { - id: if is_uidonly { - imap_id.uid - } else { - imap_id.seqnum - }, + id: resolved.imap_id(is_uidonly), is_uidonly, items: if is_uid { - vec![DataItem::Uid { uid: imap_id.uid }] + vec![DataItem::Uid { uid: resolved.uid }] } else { vec![] }, @@ -407,12 +527,28 @@ impl SessionData { } } - // Log mailbox changes - if !changed_mailboxes.is_empty() { - for parent_id in changed_mailboxes { - batch.log_container_property_change(SyncCollection::Email, parent_id); + if let Some(pending) = &mut writes { + if !spam_samples.is_empty() { + writes = None; + } else if !changed_mailboxes.is_empty() { + pending.set_mailboxes_changed(); } } + self.server + .add_account_spam_samples(&mut batch, account_id, spam_samples, self.session_id) + .await + .imap_ctx( + response.tag.as_deref().unwrap_or_default(), + trc::location!(), + )?; + + // Log mailbox changes + for parent_id in changed_mailboxes { + batch.log_container_property_change( + SyncCollection::Email, + PendingId::Assigned(parent_id), + ); + } // Write changes if !batch.is_empty() { @@ -420,12 +556,16 @@ impl SessionData { .server .commit_batch(batch) .await - .and_then(|ids| ids.last_change_id(mailbox.id.account_id)) .caused_by(trc::location!()) { - Ok(change_id) => { + Ok(assigned_ids) => { + if let Some(writes) = &writes { + self.server + .apply_keyword_writes(account_id, &cache, writes, &assigned_ids); + } if is_condstore { - let modseq = change_id + 1; + let modseq = + assigned_ids.last_change_id(account_id, SyncCollection::Email) + 1; for item in items.items.iter_mut() { item.items.push(DataItem::ModSeq { modseq }); } @@ -441,6 +581,7 @@ impl SessionData { } } } + items.items.extend(unchanged); trc::event!( Imap(trc::ImapEvent::Store), @@ -449,24 +590,24 @@ impl SessionData { MailboxId = mailbox.id.mailbox_id, DocumentId = ids .iter() - .map(|id| trc::Value::from(*id.0)) + .map(|resolved| trc::Value::from(resolved.id)) .collect::>(), - Type = format!("{:?}", arguments.operation), + Type = format_compact!("{:?}", arguments.operation), Details = arguments .keywords .iter() - .map(|c| trc::Value::from(format!("{c:?}"))) + .map(|c| trc::Value::from(format_compact!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); // Send response - let items = items.serialize(); - Ok(response.serialize(if untagged.is_empty() { - items + Ok(if untagged.is_empty() { + response.serialize_after(&items) } else { - untagged.extend_from_slice(&items); - untagged - })) + items.serialize_into(&mut untagged); + response.serialize(untagged) + }) } } + diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 22e1df61..bf6dba5f 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -3,20 +3,20 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use super::ImapContext; -use crate::{ - core::{Session, SessionData}, - spawn_op, -}; -use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; +use crate::core::{Session, SessionData}; +use common::network::SessionStream; +use email::mailbox::merge_subscription; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use registry::schema::enums::Permission; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; +use tokio::sync::OwnedSemaphorePermit; use types::collection::Collection; impl Session { @@ -24,6 +24,7 @@ impl Session { &mut self, request: Request, is_subscribe: bool, + _permit: Option, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapSubscribe)?; @@ -32,30 +33,29 @@ impl Session { let arguments = request.parse_subscribe(self.is_utf8)?; let data = self.state.session_data(); - spawn_op!(data, { - let response = data - .subscribe_folder( - arguments.tag, - arguments.mailbox_name, - is_subscribe, - op_start, - ) - .await?; + let response = data + .subscribe_folder( + arguments.tag, + arguments.mailbox_name, + is_subscribe, + op_start, + ) + .await?; - data.write_bytes(response.into_bytes()).await - }) + data.write_bytes(response.into_bytes()).await } } impl SessionData { pub async fn subscribe_folder( &self, - tag: String, - mailbox_name: String, + tag: CompactString, + mailbox_name: CompactString, subscribe: bool, op_start: Instant, ) -> trc::Result { // Refresh mailboxes + self.expire_folders_of(&mailbox_name); self.synchronize_mailboxes(false) .await .imap_ctx(&tag, trc::location!())?; @@ -74,29 +74,28 @@ impl SessionData { }; // Verify if mailbox is already subscribed/unsubscribed - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == account_id { - if let Some(mailbox) = account.mailbox_state.get(&mailbox_id) - && mailbox.is_subscribed == subscribe - { - return Err(trc::ImapEvent::Error - .into_err() - .details(if subscribe { - "Mailbox is already subscribed." - } else { - "Mailbox is already unsubscribed." - }) - .id(tag)); - } - break; - } + if self + .mailboxes + .lock() + .iter() + .find(|account| account.account_id == account_id) + .is_some_and(|account| account.is_subscribed(mailbox_id, self.account_id) == subscribe) + { + return Err(trc::ImapEvent::Error + .into_err() + .details(if subscribe { + "Mailbox is already subscribed." + } else { + "Mailbox is already unsubscribed." + }) + .id(tag)); } // Obtain mailbox let mailbox_ = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, mailbox_id, @@ -115,41 +114,19 @@ impl SessionData { .to_unarchived::() .imap_ctx(&tag, trc::location!())?; - if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) - || (!subscribe && mailbox.inner.is_subscribed(self.account_id)) - { + if mailbox.inner.is_subscribed(self.account_id) != subscribe { // Build batch - let mut new_mailbox = mailbox.deserialize().imap_ctx(&tag, trc::location!())?; - if subscribe { - new_mailbox.subscribers.push(self.account_id); - } else { - new_mailbox.remove_subscriber(self.account_id); - } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .with_document(mailbox_id) - .custom( - ObjectIndexBuilder::new() - .with_current(mailbox) - .with_changes(new_mailbox), - ) - .imap_ctx(&tag, trc::location!())?; + .with_document(mailbox_id); + merge_subscription(&mut batch, self.account_id, subscribe); self.server .commit_batch(batch) .await .imap_ctx(&tag, trc::location!())?; - - // Update mailbox cache - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == account_id { - if let Some(mailbox) = account.mailbox_state.get_mut(&mailbox_id) { - mailbox.is_subscribed = subscribe; - } - break; - } - } + self.expire_folders(account_id); } trc::event!( diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 94dc836d..5664f9aa 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -13,14 +13,12 @@ use common::network::SessionStream; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, - protocol::{ - ImapResponse, - thread::{Arguments, Response}, - }, + protocol::thread::{Arguments, Response}, receiver::Request, }; use registry::schema::enums::Permission; use std::{sync::Arc, time::Instant}; +use tokio::sync::OwnedSemaphorePermit; use trc::AddContext; impl Session { @@ -28,6 +26,7 @@ impl Session { &mut self, request: Request, is_uid: bool, + permit: Option, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapThread)?; @@ -37,7 +36,7 @@ impl Session { let mut arguments = request.parse_thread()?; let (data, mailbox) = self.state.mailbox_state(); - spawn_op!(data, { + spawn_op!(permit, data, { let tag = std::mem::take(&mut arguments.tag); match data.thread(arguments, mailbox, is_uid, op_start).await { @@ -45,7 +44,7 @@ impl Session { data.write_bytes( StatusResponse::completed(command) .with_tag(tag) - .serialize(response.serialize()), + .serialize_after(&response), ) .await } @@ -63,44 +62,46 @@ impl SessionData { is_uid: bool, op_start: Instant, ) -> trc::Result { + let cache = self + .server + .get_cached_messages(mailbox.id.account_id) + .await + .caused_by(trc::location!())?; + self.sync_view(&mailbox, &cache, None) + .await + .caused_by(trc::location!())?; + // Run query let (result_set, _) = self - .query(arguments.filter, vec![], &mailbox, &None) + .query(arguments.filter, vec![], &mailbox, &cache, &None) .await?; - // Synchronize mailbox - if !result_set.is_empty() { - self.synchronize_messages(&mailbox) - .await - .caused_by(trc::location!())?; - } else { + if result_set.is_empty() { return Ok(Response { is_uid, threads: vec![], }); } - // Lock the cache - let cache = self - .server - .get_cached_messages(mailbox.id.account_id) - .await - .caused_by(trc::location!())?; - // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); - let state = mailbox.state.lock(); - for document_id in result_set { - if let Some(item) = cache.email_by_id(&document_id) - && let Some((imap_id, _)) = state.map_result_id(document_id, is_uid) - { - threads.entry(item.thread_id).or_default().push(imap_id); + { + let view = mailbox.view.lock(); + for document_id in result_set { + if let Some(item) = cache.email_by_id(&document_id) + && let Some(resolved) = view.map_result(document_id) + { + threads + .entry(item.thread_id()) + .or_default() + .push(resolved.imap_id(is_uid)); + } } } let mut threads = threads - .into_iter() - .map(|(_, mut messages)| { + .into_values() + .map(|mut messages| { messages.sort_unstable(); messages }) diff --git a/crates/imap/src/op/uidbatches.rs b/crates/imap/src/op/uidbatches.rs index f80245b0..b8e8dd18 100644 --- a/crates/imap/src/op/uidbatches.rs +++ b/crates/imap/src/op/uidbatches.rs @@ -7,6 +7,8 @@ use super::ImapContext; use crate::core::Session; use common::network::SessionStream; +use compact_str::format_compact; +use email::cache::MessageCacheFetch; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::uidbatches, receiver::Request, }; @@ -26,7 +28,7 @@ impl Session { if arguments.batch_size < min_batch_size { return Err(trc::ImapEvent::Error .into_err() - .details(format!("Minimum batch size is {min_batch_size}.")) + .details(format_compact!("Minimum batch size is {min_batch_size}.")) .code(ResponseCode::TooFew) .id(arguments.tag)); } @@ -50,7 +52,7 @@ impl Session { { return Err(trc::ImapEvent::Error .into_err() - .details(format!( + .details(format_compact!( "A single UIDBATCHES response is limited to {max_uid_batches} ranges." )) .code(ResponseCode::TooMany) @@ -58,43 +60,62 @@ impl Session { } // Resynchronize so that batches reflect the current mailbox contents - data.synchronize_messages(&mailbox) + let cache = data + .server + .get_cached_messages(mailbox.id.account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + data.sync_view(&mailbox, &cache, None) .await .imap_ctx(&arguments.tag, trc::location!())?; - let uids = mailbox.uids_descending(); let batch_size = arguments.batch_size as usize; - let total_batches = uids.len().div_ceil(batch_size); + let ranges = { + let view = mailbox.view.lock(); + let live_rows; + let rows = if view.has_pending_changes() { + live_rows = view.live_rows_by_uid(); + live_rows.as_slice() + } else { + view.rows() + }; + let total_batches = rows.len().div_ceil(batch_size); - if arguments.batch_range.is_none() && total_batches > max_uid_batches as usize { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!( - "A single UIDBATCHES response is limited to {max_uid_batches} ranges." - )) - .code(ResponseCode::TooMany) - .id(arguments.tag)); - } - - // Batch ranges tile the whole UID space, so each range starts right below - // the previous one and the oldest batch always reaches down to UID 1. - let (first, last) = match arguments.batch_range { - Some((from, to)) => ( - (from as usize - 1).min(total_batches), - (to as usize).min(total_batches), - ), - None => (0, total_batches), - }; - let mut ranges = Vec::with_capacity(last.saturating_sub(first)); - let mut high = uids.first().copied().unwrap_or(0); - for batch in 0..last { - let end = ((batch + 1) * batch_size).min(uids.len()); - let low = if end == uids.len() { 1 } else { uids[end - 1] }; - if batch >= first { - ranges.push((high, low)); + if arguments.batch_range.is_none() && total_batches > max_uid_batches as usize { + return Err(trc::ImapEvent::Error + .into_err() + .details(format_compact!( + "A single UIDBATCHES response is limited to {max_uid_batches} ranges." + )) + .code(ResponseCode::TooMany) + .id(arguments.tag)); } - high = low.saturating_sub(1); - } + + // Batch ranges tile the whole UID space, so each range starts right below + // the previous one and the oldest batch always reaches down to UID 1. + let (first, last) = match arguments.batch_range { + Some((from, to)) => ( + (from as usize - 1).min(total_batches), + (to as usize).min(total_batches), + ), + None => (0, total_batches), + }; + let mut ranges = Vec::with_capacity(last.saturating_sub(first)); + let mut high = rows.last().map_or(0, |row| row.uid); + for batch in 0..last { + let end = ((batch + 1) * batch_size).min(rows.len()); + let low = if end == rows.len() { + 1 + } else { + rows.get(rows.len() - end).map_or(1, |row| row.uid) + }; + if batch >= first { + ranges.push((high, low)); + } + high = low.saturating_sub(1); + } + ranges + }; trc::event!( Imap(trc::ImapEvent::UidBatches), diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index f5ba95b2..4a07dcb7 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap_proto" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -8,17 +8,23 @@ utils = { path = "../utils" } types = { path = "../types" } trc = { path = "../trc" } registry = { path = "../registry" } -jmap-tools = { version = "0.1" } -calcard = { version = "0.3" } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } +jmap-tools = { path = "../jmap-tools" } +calcard = { path = "../calcard" } +mail-parser = { version = "1.0", features = ["full_encoding"] } serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.12", features = ["serde"] } serde_json = { version = "1.0", features = ["raw_value"] } -hashify = "0.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } +simdutf8 = "0.1.5" +sonic-rs = "0.5.10" +hashify = "0.3" +rkyv.workspace = true compact_str = { version = "0.10.0", features = ["rkyv", "serde"] } +encodify = "1.0" [dev-dependencies] [lints] workspace = true + +[features] +enterprise = ["utils/enterprise", "trc/enterprise", "registry/enterprise"] diff --git a/crates/jmap-proto/src/error/method.rs b/crates/jmap-proto/src/error/method.rs index 87911650..53690c9e 100644 --- a/crates/jmap-proto/src/error/method.rs +++ b/crates/jmap-proto/src/error/method.rs @@ -120,6 +120,16 @@ impl Serialize for MethodErrorWrapper { ("invalidResultReference", description.unwrap_or_default()) } trc::JmapEvent::Forbidden => ("forbidden", description.unwrap_or_default()), + trc::JmapEvent::ObjectNotFound => ("notFound", description.unwrap_or_default()), + trc::JmapEvent::ExpandDurationTooLarge => { + ("expandDurationTooLarge", description.unwrap_or_default()) + } + trc::JmapEvent::CannotCalculateOccurrences => ( + "cannotCalculateOccurrences", + description.unwrap_or_default(), + ), + trc::JmapEvent::TooLarge => ("tooLarge", description.unwrap_or_default()), + trc::JmapEvent::RateLimit => ("rateLimit", description.unwrap_or_default()), trc::JmapEvent::AccountNotFound => ( "accountNotFound", "The accountId does not correspond to a valid account", @@ -135,6 +145,17 @@ impl Serialize for MethodErrorWrapper { "accountReadOnly", "This method modifies state, but the account is read-only.", ), + trc::JmapEvent::FromAccountNotFound => ( + "fromAccountNotFound", + "The fromAccountId does not correspond to a valid account.", + ), + trc::JmapEvent::FromAccountNotSupportedByMethod => ( + "fromAccountNotSupportedByMethod", + concat!( + "The fromAccountId given corresponds to a valid account, ", + "but the account does not support this data type." + ), + ), trc::JmapEvent::UnknownDataType => ( "unknownDataType", concat!( @@ -164,6 +185,9 @@ impl Serialize for MethodErrorWrapper { "This server is temporarily unavailable.", ), }, + trc::EventType::Limit(trc::LimitEvent::SizeRequest) => { + ("unsupportedFilter", description.unwrap_or_default()) + } _ => ( "serverUnavailable", concat!( diff --git a/crates/jmap-proto/src/error/request.rs b/crates/jmap-proto/src/error/request.rs index 9474bf49..917e6a13 100644 --- a/crates/jmap-proto/src/error/request.rs +++ b/crates/jmap-proto/src/error/request.rs @@ -168,13 +168,6 @@ impl<'x> RequestError<'x> { ) } - pub fn tenant_over_quota() -> Self { - RequestError::blank( - 403, - "Tenant quota exceeded", - "Your organization has exceeded its quota.", - ) - } pub fn too_many_requests() -> Self { RequestError::blank( @@ -371,55 +364,3 @@ impl Display for RequestError<'_> { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rate_limit_headers_match_spec() { - // Spec example: RateLimit-Policy: "burst";q=100;w=60,"daily";q=1000;w=86400 - let mut p1 = String::new(); - RateLimitPolicy::new("burst", 100) - .with_window(60) - .fmt_policy(&mut p1); - assert_eq!(p1, r#""burst";q=100;w=60"#); - - // Spec example: RateLimit-Policy: "peruser";q=65535;qu="content-bytes";w=10 - let mut p2 = String::new(); - RateLimitPolicy::new("peruser", 65535) - .with_window(10) - .with_unit(RateLimitUnit::ContentBytes) - .fmt_policy(&mut p2); - assert_eq!(p2, r#""peruser";q=65535;w=10;qu="content-bytes""#); - - // Spec example: RateLimit: "default";r=50;t=30 - let mut s = String::new(); - RateLimitPolicy::new("default", 100) - .with_remaining(50) - .with_reset(30) - .fmt_state(&mut s); - assert_eq!(s, r#""default";r=50;t=30"#); - - // Two policies in one header - let err = RequestError::too_many_requests() - .with_rate_limit( - RateLimitPolicy::new("burst", 100) - .with_window(60) - .with_reset(30), - ) - .with_rate_limit( - RateLimitPolicy::new("daily", 1000) - .with_window(86400) - .with_reset(3600), - ); - assert_eq!( - err.rate_limit_policy_header().as_deref(), - Some(r#""burst";q=100;w=60, "daily";q=1000;w=86400"#), - ); - assert_eq!( - err.rate_limit_state_header().as_deref(), - Some(r#""burst";r=0;t=30, "daily";r=0;t=3600"#), - ); - assert_eq!(err.retry_after, Some(3600)); - } -} diff --git a/crates/jmap-proto/src/error/set.rs b/crates/jmap-proto/src/error/set.rs index e637be59..bd6edb66 100644 --- a/crates/jmap-proto/src/error/set.rs +++ b/crates/jmap-proto/src/error/set.rs @@ -4,13 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::request::MaybeInvalid; use jmap_tools::{Key, Property}; use registry::types::{ error::{PatchError, ValidationError}, id::ObjectId, }; use std::borrow::Cow; -use types::id::Id; +use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, serde::Serialize)] #[serde(bound(serialize = "InvalidProperty

: serde::Serialize"))] @@ -45,6 +46,10 @@ struct SetErrorInner { #[serde(skip_serializing_if = "Vec::is_empty")] #[serde(rename = "validationErrors")] validation_errors: Vec, + + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(rename = "notFound")] + not_found: Vec>, } #[derive(Debug, Clone)] @@ -174,9 +179,15 @@ impl SetError { object_id: None, linked_objects: Vec::new(), validation_errors: Vec::new(), + not_found: Vec::new(), })) } + pub fn with_not_found(mut self, not_found: Vec>) -> Self { + self.0.not_found = not_found; + self + } + pub fn with_description(mut self, description: impl Into>) -> Self { self.0.description = description.into().into(); self @@ -347,6 +358,7 @@ impl From for SetError { object_id: None, linked_objects: Vec::new(), validation_errors: Vec::new(), + not_found: Vec::new(), })) } } diff --git a/crates/jmap-proto/src/method/availability.rs b/crates/jmap-proto/src/method/availability.rs index 26fe5092..51052dd8 100644 --- a/crates/jmap-proto/src/method/availability.rs +++ b/crates/jmap-proto/src/method/availability.rs @@ -13,14 +13,14 @@ use crate::{ }; use calcard::jscalendar::{JSCalendar, JSCalendarProperty}; use serde::{Deserialize, Deserializer, Serialize}; -use types::{blob::BlobId, id::Id}; +use types::{TimeRange, blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct GetAvailabilityRequest { pub account_id: Id, pub id: Id, - pub utc_start: UTCDate, - pub utc_end: UTCDate, + pub utc_start: Option, + pub utc_end: Option, pub show_details: bool, pub event_properties: Option>>>, } @@ -38,14 +38,24 @@ pub struct BusyPeriod { pub utc_end: UTCDate, pub busy_status: Option, pub event: Option>, + pub account_id: Option, } -#[derive(Debug, Serialize, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)] +#[derive(Debug, Serialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "lowercase")] pub enum BusyStatus { - Confirmed, Tentative, Unavailable, + Confirmed, +} + +impl GetAvailabilityRequest { + pub fn time_range(&self) -> Option { + let start = self.utc_start.as_ref().filter(|date| date.is_valid())?; + let end = self.utc_end.as_ref().filter(|date| date.is_valid())?; + Some(TimeRange::new(start.timestamp(), end.timestamp())) + .filter(|range| range.end > range.start) + } } impl<'de> DeserializeArguments<'de> for GetAvailabilityRequest { @@ -89,3 +99,4 @@ impl<'de> Deserialize<'de> for GetAvailabilityRequest { deserialize_request(deserializer) } } + diff --git a/crates/jmap-proto/src/method/changes.rs b/crates/jmap-proto/src/method/changes.rs index 0e3423c2..b01bc6db 100644 --- a/crates/jmap-proto/src/method/changes.rs +++ b/crates/jmap-proto/src/method/changes.rs @@ -18,6 +18,7 @@ pub struct ChangesRequest { pub account_id: Id, pub since_state: State, pub max_changes: Option, + pub ignore_metadata_only_changes: Option, } #[derive(Debug, Clone, serde::Serialize)] @@ -41,7 +42,6 @@ pub struct ChangesResponse { pub destroyed: Vec, #[serde(rename = "updatedProperties")] - #[serde(skip_serializing_if = "Option::is_none")] pub updated_properties: Option>>, } @@ -60,6 +60,9 @@ impl<'de> DeserializeArguments<'de> for ChangesRequest { b"maxChanges" => { self.max_changes = map.next_value()?; }, + b"ignoreMetadataOnlyChanges" => { + self.ignore_metadata_only_changes = map.next_value()?; + }, _ => { let _ = map.next_value::()?; } @@ -83,3 +86,4 @@ impl ChangesResponse { !self.created.is_empty() || !self.updated.is_empty() || !self.destroyed.is_empty() } } + diff --git a/crates/jmap-proto/src/method/copy.rs b/crates/jmap-proto/src/method/copy.rs index 25d61d10..377c089a 100644 --- a/crates/jmap-proto/src/method/copy.rs +++ b/crates/jmap-proto/src/method/copy.rs @@ -6,15 +6,15 @@ use crate::{ error::set::SetError, - object::{JmapObject, blob::BlobProperty}, + object::{JmapObject, JmapObjectId, blob::BlobProperty}, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, - reference::MaybeIdReference, }, - types::state::State, + response::Response, + types::state::{ExpectedState, State}, }; -use jmap_tools::{Key, Map, Value}; +use jmap_tools::{Element, Key, Map, ObjectAsVec, Property, Value}; use serde::{Deserialize, Deserializer, Serialize}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; @@ -25,7 +25,7 @@ pub struct CopyRequest<'x, T: JmapObject> { pub if_from_in_state: Option, pub account_id: Id, pub if_in_state: Option, - pub create: VecMap, Value<'x, T::Property, T::Element>>, + pub create: VecMap>, pub on_success_destroy_original: Option, pub destroy_from_if_in_state: Option, pub arguments: T::CopyArguments, @@ -47,11 +47,15 @@ pub struct CopyResponse { #[serde(rename = "created")] #[serde(skip_serializing_if = "VecMap::is_empty")] - pub created: VecMap>, + pub created: VecMap>, #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] - pub not_created: VecMap>, + pub not_created: VecMap>, + + #[serde(rename = "destroyed")] + #[serde(skip_serializing_if = "Option::is_none")] + pub destroyed: Option>>, } #[derive(Debug, Clone, Default)] @@ -88,13 +92,13 @@ impl<'de, T: JmapObject> DeserializeArguments<'de> for CopyRequest<'de, T> { self.account_id = crate::request::deserialize_account_id(map)?; }, b"ifInState" => { - self.if_in_state = map.next_value()?; + self.if_in_state = map.next_value::()?.0; }, b"fromAccountId" => { self.from_account_id = crate::request::deserialize_account_id(map)?; }, b"ifFromInState" => { - self.if_from_in_state = map.next_value()?; + self.if_from_in_state = map.next_value::()?.0; }, b"create" => { self.create = map.next_value()?; @@ -103,7 +107,7 @@ impl<'de, T: JmapObject> DeserializeArguments<'de> for CopyRequest<'de, T> { self.on_success_destroy_original = map.next_value()?; }, b"destroyFromIfInState" => { - self.destroy_from_if_in_state = map.next_value()?; + self.destroy_from_if_in_state = map.next_value::()?.0; }, _ => { self.arguments.deserialize_argument(key, map)?; @@ -172,14 +176,71 @@ impl<'de, T: JmapObject> Default for CopyRequest<'de, T> { } impl CopyResponse { - pub fn created(&mut self, id: Id, document_id: impl Into) { + pub fn created(&mut self, id: String, document_id: impl Into) { + self.created_with_properties(id, document_id, []); + } + + pub fn created_with_properties( + &mut self, + id: String, + document_id: impl Into, + properties: impl IntoIterator)>, + ) { let document_id = document_id.into(); - self.created.append( - id, - Value::Object(Map::from(vec![( - Key::Property(T::ID_PROPERTY), - Value::Element(document_id.into()), - )])), - ); + let mut object = Map::from(vec![( + Key::Property(T::ID_PROPERTY), + Value::Element(document_id.into()), + )]); + for (property, value) in properties { + object.insert(property, value); + } + self.created.append(id, Value::Object(object)); + } + + pub fn update_created_ids(&self, response: &mut Response) { + for (create_id, obj) in &self.created { + if let Value::Object(obj) = obj + && let Some(Value::Element(id)) = obj.get(&Key::Property(T::ID_PROPERTY)) + && let Some(id) = id.as_any_id() + { + response.created_ids.insert(create_id.clone(), id); + } + } } } + +pub trait CopySourceId { + fn source_id(&self, id_property: P) -> Option; + + fn take_source_id(&mut self, id_property: P) -> Result>; +} + +impl + JmapObjectId> CopySourceId

for Value<'_, P, E> { + fn source_id(&self, id_property: P) -> Option { + self.as_object() + .and_then(|object| object.get(&Key::Property(id_property))) + .and_then(Value::as_element) + .and_then(JmapObjectId::as_id) + } + + fn take_source_id(&mut self, id_property: P) -> Result> { + let key = Key::Property(id_property.clone()); + self.as_object_mut() + .map(ObjectAsVec::as_mut_vec) + .and_then(|entries| { + entries + .iter() + .position(|(entry, _)| *entry == key) + .map(|position| entries.remove(position).1) + }) + .as_ref() + .and_then(Value::as_element) + .and_then(JmapObjectId::as_id) + .ok_or_else(|| { + SetError::invalid_properties() + .with_property(id_property) + .with_description("Missing or invalid \"id\" property.") + }) + } +} + diff --git a/crates/jmap-proto/src/method/get.rs b/crates/jmap-proto/src/method/get.rs index 942ac336..b2e87d65 100644 --- a/crates/jmap-proto/src/method/get.rs +++ b/crates/jmap-proto/src/method/get.rs @@ -7,7 +7,7 @@ use crate::{ object::JmapObject, request::{ - MaybeInvalid, + IntoValid, MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeIdReference, MaybeResultReference, ResultReference}, }, @@ -17,6 +17,41 @@ use jmap_tools::Value; use serde::{Deserialize, Deserializer}; use types::id::Id; +const LINEAR_DEDUP_LIMIT: usize = 16; + +pub fn unique_properties(properties: impl Iterator) -> Vec

(key: &Key<'_, Self::Property>, value: &str) -> Option { + if let Key::Property(prop) = key { + match prop.patch_or_prop() { + EmailProperty::Id | EmailProperty::ThreadId | EmailProperty::MailboxIds => { + match parse_ref(value) { + MaybeReference::Value(v) => Some(EmailValue::Id(v)), + MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), + MaybeReference::ParseError => None, + } + } + EmailProperty::BlobId => match parse_ref(value) { + MaybeReference::Value(v) => Some(EmailValue::BlobId(v)), + MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), + MaybeReference::ParseError => None, + }, + EmailProperty::Header(HeaderProperty { + form: HeaderForm::Date, + .. + }) + | EmailProperty::ReceivedAt + | EmailProperty::SentAt => UTCDate::from_str(value).ok().map(EmailValue::Date), + _ => None, } } else { - EmailProperty::parse(value, allow_patch) + None } } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { + EmailValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl EmailValue { + pub fn text(&self) -> Text<'_> { + match self { + EmailValue::Id(id) => Text::Id(*id), + EmailValue::Date(utcdate) => Text::Display(utcdate), + EmailValue::BlobId(blob_id) => Text::Display(blob_id), + EmailValue::IdReference(r) => Text::Reference(r), + } + } +} + +impl EmailProperty { + pub fn text(&self) -> Text<'_> { + Text::Static(match self { EmailProperty::Attachments => "attachments", EmailProperty::Bcc => "bcc", EmailProperty::BlobId => "blobId", @@ -167,112 +266,90 @@ impl Property for EmailProperty { EmailProperty::Value => "value", EmailProperty::IsEncodingProblem => "isEncodingProblem", EmailProperty::IsTruncated => "isTruncated", - EmailProperty::Header(header) => return header.to_string().into(), - EmailProperty::Keyword(keyword) => return keyword.to_string().into(), - EmailProperty::IdValue(id) => return id.to_string().into(), - EmailProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), - EmailProperty::IdReference(r) => return format!("#{r}").into(), - } - .into() - } -} - -impl Element for EmailValue { - type Property = EmailProperty; - - fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { - if let Key::Property(prop) = key { - match prop.patch_or_prop() { - EmailProperty::Id | EmailProperty::ThreadId | EmailProperty::MailboxIds => { - match parse_ref(value) { - MaybeReference::Value(v) => Some(EmailValue::Id(v)), - MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), - MaybeReference::ParseError => None, - } - } - EmailProperty::BlobId => match parse_ref(value) { - MaybeReference::Value(v) => Some(EmailValue::BlobId(v)), - MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), - MaybeReference::ParseError => None, - }, - EmailProperty::Header(HeaderProperty { - form: HeaderForm::Date, - .. - }) - | EmailProperty::ReceivedAt - | EmailProperty::SentAt => UTCDate::from_str(value).ok().map(EmailValue::Date), - _ => None, - } - } else { - None - } - } - - fn to_cow(&self) -> Cow<'static, str> { - match self { - EmailValue::Id(id) => id.to_string().into(), - EmailValue::Date(utcdate) => utcdate.to_string().into(), - EmailValue::BlobId(blob_id) => blob_id.to_string().into(), - EmailValue::IdReference(r) => format!("#{r}").into(), - } - } -} - -impl EmailProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - "id" => EmailProperty::Id, - "blobId" => EmailProperty::BlobId, - "threadId" => EmailProperty::ThreadId, - "mailboxIds" => EmailProperty::MailboxIds, - "keywords" => EmailProperty::Keywords, - "size" => EmailProperty::Size, - "receivedAt" => EmailProperty::ReceivedAt, - "name" => EmailProperty::Name, - "email" => EmailProperty::Email, - "addresses" => EmailProperty::Addresses, - "value" => EmailProperty::Value, - "messageId" => EmailProperty::MessageId, - "inReplyTo" => EmailProperty::InReplyTo, - "references" => EmailProperty::References, - "sender" => EmailProperty::Sender, - "from" => EmailProperty::From, - "to" => EmailProperty::To, - "cc" => EmailProperty::Cc, - "bcc" => EmailProperty::Bcc, - "replyTo" => EmailProperty::ReplyTo, - "subject" => EmailProperty::Subject, - "sentAt" => EmailProperty::SentAt, - "textBody" => EmailProperty::TextBody, - "htmlBody" => EmailProperty::HtmlBody, - "attachments" => EmailProperty::Attachments, - "partId" => EmailProperty::PartId, - "headers" => EmailProperty::Headers, - "type" => EmailProperty::Type, - "charset" => EmailProperty::Charset, - "disposition" => EmailProperty::Disposition, - "cid" => EmailProperty::Cid, - "language" => EmailProperty::Language, - "location" => EmailProperty::Location, - "subParts" => EmailProperty::SubParts, - "bodyStructure" => EmailProperty::BodyStructure, - "bodyValues" => EmailProperty::BodyValues, - "isEncodingProblem" => EmailProperty::IsEncodingProblem, - "isTruncated" => EmailProperty::IsTruncated, - "hasAttachment" => EmailProperty::HasAttachment, - "preview" => EmailProperty::Preview - ) - .or_else(|| { - if let Some(header) = value.strip_prefix("header:") { - HeaderProperty::parse(header).map(EmailProperty::Header) - } else if allow_patch && value.contains('/') { - EmailProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + EmailProperty::Metadata => "metadata", + EmailProperty::PrivateMetadata => "privateMetadata", + EmailProperty::Header(header) => return Text::Display(header), + EmailProperty::Keyword(keyword) => return Text::Str(keyword.as_str()), + EmailProperty::IdValue(id) => return Text::Id(*id), + EmailProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + EmailProperty::IdReference(r) => return Text::Reference(r), }) } + fn has_dynamic_text(&self) -> bool { + matches!( + self, + EmailProperty::Pointer(_) + | EmailProperty::IdValue(_) + | EmailProperty::Header(_) + | EmailProperty::Keyword(_) + | EmailProperty::IdReference(_) + ) + } + + property_names!( + EmailProperty, + "id" => EmailProperty::Id, + "blobId" => EmailProperty::BlobId, + "threadId" => EmailProperty::ThreadId, + "mailboxIds" => EmailProperty::MailboxIds, + "keywords" => EmailProperty::Keywords, + "size" => EmailProperty::Size, + "receivedAt" => EmailProperty::ReceivedAt, + "name" => EmailProperty::Name, + "email" => EmailProperty::Email, + "addresses" => EmailProperty::Addresses, + "value" => EmailProperty::Value, + "messageId" => EmailProperty::MessageId, + "inReplyTo" => EmailProperty::InReplyTo, + "references" => EmailProperty::References, + "sender" => EmailProperty::Sender, + "from" => EmailProperty::From, + "to" => EmailProperty::To, + "cc" => EmailProperty::Cc, + "bcc" => EmailProperty::Bcc, + "replyTo" => EmailProperty::ReplyTo, + "subject" => EmailProperty::Subject, + "sentAt" => EmailProperty::SentAt, + "textBody" => EmailProperty::TextBody, + "htmlBody" => EmailProperty::HtmlBody, + "attachments" => EmailProperty::Attachments, + "partId" => EmailProperty::PartId, + "headers" => EmailProperty::Headers, + "type" => EmailProperty::Type, + "charset" => EmailProperty::Charset, + "disposition" => EmailProperty::Disposition, + "cid" => EmailProperty::Cid, + "language" => EmailProperty::Language, + "location" => EmailProperty::Location, + "subParts" => EmailProperty::SubParts, + "bodyStructure" => EmailProperty::BodyStructure, + "bodyValues" => EmailProperty::BodyValues, + "isEncodingProblem" => EmailProperty::IsEncodingProblem, + "isTruncated" => EmailProperty::IsTruncated, + "hasAttachment" => EmailProperty::HasAttachment, + "preview" => EmailProperty::Preview, + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + EmailProperty::parse_name(value).or_else(|| EmailProperty::parse_other(value, patch_depth)) + } + + fn parse_nested(value: &str) -> Option { + EmailProperty::parse_nested_name(value).or_else(|| EmailProperty::parse_other(value, None)) + } + + fn parse_other(value: &str, patch_depth: Option) -> Option { + if let Some(header) = value.strip_prefix("header:") { + HeaderProperty::parse(header).map(EmailProperty::Header) + } else { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(EmailProperty::Pointer) + } + } + fn patch_or_prop(&self) -> &EmailProperty { if let EmailProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() @@ -315,12 +392,77 @@ impl EmailProperty { } } +const FORM_TEXT: u8 = 1; +const FORM_ADDRESSES: u8 = 1 << 1; +const FORM_MESSAGE_IDS: u8 = 1 << 2; +const FORM_DATE: u8 = 1 << 3; +const FORM_URLS: u8 = 1 << 4; +const FORM_ANY: u8 = u8::MAX; + +impl HeaderForm { + fn mask(self) -> u8 { + match self { + HeaderForm::Raw => FORM_ANY, + HeaderForm::Text => FORM_TEXT, + HeaderForm::Addresses | HeaderForm::GroupedAddresses => FORM_ADDRESSES, + HeaderForm::MessageIds => FORM_MESSAGE_IDS, + HeaderForm::Date => FORM_DATE, + HeaderForm::URLs => FORM_URLS, + } + } +} + +impl EmailProperty { + pub fn is_allowed_form(&self) -> bool { + match self { + EmailProperty::Header(header) => header.is_allowed_form(), + _ => true, + } + } +} + impl HeaderProperty { + pub fn is_allowed_form(&self) -> bool { + let allowed = hashify::fnc_map_ignore_case!(self.header.as_bytes(), + "Subject" => FORM_TEXT, + "Comments" => FORM_TEXT, + "Keywords" => FORM_TEXT, + "From" => FORM_ADDRESSES, + "Sender" => FORM_ADDRESSES, + "Reply-To" => FORM_ADDRESSES, + "To" => FORM_ADDRESSES, + "Cc" => FORM_ADDRESSES, + "Bcc" => FORM_ADDRESSES, + "Resent-From" => FORM_ADDRESSES, + "Resent-Sender" => FORM_ADDRESSES, + "Resent-To" => FORM_ADDRESSES, + "Resent-Cc" => FORM_ADDRESSES, + "Resent-Bcc" => FORM_ADDRESSES, + "Message-ID" => FORM_MESSAGE_IDS, + "In-Reply-To" => FORM_MESSAGE_IDS, + "References" => FORM_MESSAGE_IDS, + "Resent-Message-ID" => FORM_MESSAGE_IDS, + "Date" => FORM_DATE, + "Resent-Date" => FORM_DATE, + "List-Help" => FORM_URLS, + "List-Unsubscribe" => FORM_URLS, + "List-Subscribe" => FORM_URLS, + "List-Post" => FORM_URLS, + "List-Owner" => FORM_URLS, + "List-Archive" => FORM_URLS, + "Return-Path" => 0, + "Received" => 0, + _ => FORM_ANY + ); + self.form == HeaderForm::Raw || self.form.mask() & allowed != 0 + } + fn parse(value: &str) -> Option { let mut result = HeaderProperty { form: HeaderForm::Raw, header: String::new(), all: false, + as_raw: false, }; for (pos, value) in value.split(':').enumerate() { @@ -336,7 +478,7 @@ impl HeaderProperty { b"asMessageIds" => { result.form = HeaderForm::MessageIds;}, b"asDate" => { result.form = HeaderForm::Date;}, b"asURLs" => { result.form = HeaderForm::URLs;}, - b"asRaw" => { result.form = HeaderForm::Raw; }, + b"asRaw" => { result.as_raw = true; }, b"all" => { result.all = true; }, _ => { return None; @@ -361,6 +503,9 @@ impl HeaderProperty { impl Display for HeaderProperty { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "header:{}", self.header)?; + if self.as_raw { + f.write_str(":asRaw")?; + } self.form.fmt(f)?; if self.all { write!(f, ":all") } else { Ok(()) } } @@ -380,11 +525,40 @@ impl Display for HeaderForm { } } +impl MetadataProperty for EmailProperty { + fn as_metadata_root(&self) -> Option { + match self { + EmailProperty::Metadata => Some(MetadataRoot::Shared), + EmailProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + EmailProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => EmailProperty::Metadata, + MetadataRoot::Private => EmailProperty::PrivateMetadata, + } + } +} + impl FromStr for EmailProperty { type Err = (); fn from_str(s: &str) -> Result { - EmailProperty::parse(s, false).ok_or(()) + EmailProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| EmailProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) } } @@ -493,7 +667,7 @@ impl JmapObject for Email { type Id = Id; - type Filter = EmailFilter; + type Filter = EmailQueryFilter; type Comparator = EmailComparator; @@ -539,6 +713,12 @@ pub enum EmailFilter { _T(String), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EmailQueryFilter { + Email(EmailFilter), + Metadata(MetadataFilter), +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum EmailComparator { ReceivedAt, @@ -643,13 +823,31 @@ impl<'de> DeserializeArguments<'de> for EmailFilter { } } +impl<'de> DeserializeArguments<'de> for EmailQueryFilter { + fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> + where + A: serde::de::MapAccess<'de>, + { + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => EmailQueryFilter::Metadata(filter), + None => { + let mut filter = EmailFilter::default(); + filter.deserialize_argument(key, map)?; + EmailQueryFilter::Email(filter) + } + }; + + Ok(()) + } +} + impl<'de> DeserializeArguments<'de> for EmailComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"receivedAt" => { *self = EmailComparator::ReceivedAt; @@ -685,7 +883,7 @@ impl<'de> DeserializeArguments<'de> for EmailComparator { *self = EmailComparator::SomeInThreadHaveKeyword(self.take_keyword()); }, _ => { - *self = EmailComparator::_T(key.to_string()); + *self = EmailComparator::_T(value.into_owned()); } ); } else if key == "keyword" { @@ -716,6 +914,12 @@ impl Default for EmailFilter { } } +impl Default for EmailQueryFilter { + fn default() -> Self { + EmailQueryFilter::Email(EmailFilter::default()) + } +} + impl Default for EmailComparator { fn default() -> Self { EmailComparator::_T("".to_string()) @@ -771,6 +975,15 @@ impl Display for EmailFilter { } } +impl Display for EmailQueryFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EmailQueryFilter::Email(filter) => filter.fmt(f), + EmailQueryFilter::Metadata(filter) => f.write_str(filter.as_str()), + } + } +} + impl Display for EmailComparator { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) @@ -805,10 +1018,11 @@ impl Serialize for EmailComparator { } } -impl Filter { +impl Filter { pub fn is_immutable(&self) -> bool { match self { - Filter::Property(f) => f.is_immutable(), + Filter::Property(EmailQueryFilter::Email(f)) => f.is_immutable(), + Filter::Property(EmailQueryFilter::Metadata(_)) => false, Filter::And | Filter::Or | Filter::Not | Filter::Close => true, } } diff --git a/crates/jmap-proto/src/object/email_submission.rs b/crates/jmap-proto/src/object/email_submission.rs index c52780f5..e65ee13a 100644 --- a/crates/jmap-proto/src/object/email_submission.rs +++ b/crates/jmap-proto/src/object/email_submission.rs @@ -10,12 +10,17 @@ use crate::{ email::{EmailProperty, EmailValue}, parse_ref, }, - request::{MaybeInvalid, deserialize::DeserializeArguments, reference::MaybeIdReference}, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments}, + reference::MaybeIdReference, + }, types::date::UTCDate, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property, Value}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property, Value}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{blob::BlobId, id::Id, text::Text}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Default)] @@ -78,33 +83,35 @@ pub enum Displayed { impl Property for EmailSubmissionProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - EmailSubmissionProperty::parse(value, key.is_none()) + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + EmailSubmissionProperty::parse(value, key.is_none().then_some(depth)) } fn to_cow(&self) -> Cow<'static, str> { - match self { - EmailSubmissionProperty::DeliveryStatus => "deliveryStatus", - EmailSubmissionProperty::DsnBlobIds => "dsnBlobIds", - EmailSubmissionProperty::Email => "email", - EmailSubmissionProperty::Envelope => "envelope", - EmailSubmissionProperty::Id => "id", - EmailSubmissionProperty::IdentityId => "identityId", - EmailSubmissionProperty::MdnBlobIds => "mdnBlobIds", - EmailSubmissionProperty::SendAt => "sendAt", - EmailSubmissionProperty::ThreadId => "threadId", - EmailSubmissionProperty::UndoStatus => "undoStatus", - EmailSubmissionProperty::Parameters => "parameters", - EmailSubmissionProperty::SmtpReply => "smtpReply", - EmailSubmissionProperty::Delivered => "delivered", - EmailSubmissionProperty::Displayed => "displayed", - EmailSubmissionProperty::MailFrom => "mailFrom", - EmailSubmissionProperty::RcptTo => "rcptTo", - EmailSubmissionProperty::EmailId => "emailId", - EmailSubmissionProperty::Pointer(json_pointer) => { - return json_pointer.to_string().into(); - } + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -149,45 +156,85 @@ impl Element for EmailSubmissionValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - EmailSubmissionValue::Id(id) => id.to_string().into(), - EmailSubmissionValue::Date(utcdate) => utcdate.to_string().into(), - EmailSubmissionValue::BlobId(blob_id) => blob_id.to_string().into(), - EmailSubmissionValue::IdReference(r) => format!("#{r}").into(), - EmailSubmissionValue::UndoStatus(undo_status) => undo_status.as_str().into(), - EmailSubmissionValue::Delivered(delivered) => delivered.as_str().into(), - EmailSubmissionValue::Displayed(displayed) => displayed.as_str().into(), + EmailSubmissionValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl EmailSubmissionValue { + pub fn text(&self) -> Text<'_> { + match self { + EmailSubmissionValue::Id(id) => Text::Id(*id), + EmailSubmissionValue::Date(utcdate) => Text::Display(utcdate), + EmailSubmissionValue::BlobId(blob_id) => Text::Display(blob_id), + EmailSubmissionValue::IdReference(r) => Text::Reference(r), + EmailSubmissionValue::UndoStatus(undo_status) => Text::Static(undo_status.as_str()), + EmailSubmissionValue::Delivered(delivered) => Text::Static(delivered.as_str()), + EmailSubmissionValue::Displayed(displayed) => Text::Static(displayed.as_str()), } } } impl EmailSubmissionProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - "id" => EmailSubmissionProperty::Id, - "identityId" => EmailSubmissionProperty::IdentityId, - "threadId" => EmailSubmissionProperty::ThreadId, - "emailId" => EmailSubmissionProperty::EmailId, - "envelope" => EmailSubmissionProperty::Envelope, - "mailFrom" => EmailSubmissionProperty::MailFrom, - "rcptTo" => EmailSubmissionProperty::RcptTo, - "email" => EmailSubmissionProperty::Email, - "parameters" => EmailSubmissionProperty::Parameters, - "sendAt" => EmailSubmissionProperty::SendAt, - "undoStatus" => EmailSubmissionProperty::UndoStatus, - "deliveryStatus" => EmailSubmissionProperty::DeliveryStatus, - "smtpReply" => EmailSubmissionProperty::SmtpReply, - "delivered" => EmailSubmissionProperty::Delivered, - "displayed" => EmailSubmissionProperty::Displayed, - "dsnBlobIds" => EmailSubmissionProperty::DsnBlobIds, - "mdnBlobIds" => EmailSubmissionProperty::MdnBlobIds, + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + EmailSubmissionProperty::DeliveryStatus => "deliveryStatus", + EmailSubmissionProperty::DsnBlobIds => "dsnBlobIds", + EmailSubmissionProperty::Email => "email", + EmailSubmissionProperty::Envelope => "envelope", + EmailSubmissionProperty::Id => "id", + EmailSubmissionProperty::IdentityId => "identityId", + EmailSubmissionProperty::MdnBlobIds => "mdnBlobIds", + EmailSubmissionProperty::SendAt => "sendAt", + EmailSubmissionProperty::ThreadId => "threadId", + EmailSubmissionProperty::UndoStatus => "undoStatus", + EmailSubmissionProperty::Parameters => "parameters", + EmailSubmissionProperty::SmtpReply => "smtpReply", + EmailSubmissionProperty::Delivered => "delivered", + EmailSubmissionProperty::Displayed => "displayed", + EmailSubmissionProperty::MailFrom => "mailFrom", + EmailSubmissionProperty::RcptTo => "rcptTo", + EmailSubmissionProperty::EmailId => "emailId", + EmailSubmissionProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!(self, EmailSubmissionProperty::Pointer(_)) + } + + fn parse(value: &str, patch_depth: Option) -> Option { + hashify::fnc_map!(value.as_bytes(), + "id" => Some(EmailSubmissionProperty::Id), + "identityId" => Some(EmailSubmissionProperty::IdentityId), + "threadId" => Some(EmailSubmissionProperty::ThreadId), + "emailId" => Some(EmailSubmissionProperty::EmailId), + "envelope" => Some(EmailSubmissionProperty::Envelope), + "mailFrom" => Some(EmailSubmissionProperty::MailFrom), + "rcptTo" => Some(EmailSubmissionProperty::RcptTo), + "email" => Some(EmailSubmissionProperty::Email), + "parameters" => Some(EmailSubmissionProperty::Parameters), + "sendAt" => Some(EmailSubmissionProperty::SendAt), + "undoStatus" => Some(EmailSubmissionProperty::UndoStatus), + "deliveryStatus" => Some(EmailSubmissionProperty::DeliveryStatus), + "smtpReply" => Some(EmailSubmissionProperty::SmtpReply), + "delivered" => Some(EmailSubmissionProperty::Delivered), + "displayed" => Some(EmailSubmissionProperty::Displayed), + "dsnBlobIds" => Some(EmailSubmissionProperty::DsnBlobIds), + "mdnBlobIds" => Some(EmailSubmissionProperty::MdnBlobIds), + _ => None, ) .or_else(|| { - if allow_patch && value.contains('/') { - EmailSubmissionProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(EmailSubmissionProperty::Pointer) }) } @@ -204,14 +251,15 @@ impl EmailSubmissionProperty { impl UndoStatus { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"pending" => UndoStatus::Pending, - b"final" => UndoStatus::Final, - b"canceled" => UndoStatus::Canceled, + hashify::fnc_map!(value.as_bytes(), + b"pending" => Some(UndoStatus::Pending), + b"final" => Some(UndoStatus::Final), + b"canceled" => Some(UndoStatus::Canceled), + _ => None, ) } - fn as_str(&self) -> &'static str { + pub(crate) fn as_str(&self) -> &'static str { match self { UndoStatus::Pending => "pending", UndoStatus::Final => "final", @@ -222,15 +270,16 @@ impl UndoStatus { impl Delivered { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"queued" => Delivered::Queued, - b"yes" => Delivered::Yes, - b"no" => Delivered::No, - b"unknown" => Delivered::Unknown, + hashify::fnc_map!(value.as_bytes(), + b"queued" => Some(Delivered::Queued), + b"yes" => Some(Delivered::Yes), + b"no" => Some(Delivered::No), + b"unknown" => Some(Delivered::Unknown), + _ => None, ) } - fn as_str(&self) -> &'static str { + pub(crate) fn as_str(&self) -> &'static str { match self { Delivered::Queued => "queued", Delivered::Yes => "yes", @@ -242,13 +291,14 @@ impl Delivered { impl Displayed { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"yes" => Displayed::Yes, - b"unknown" => Displayed::Unknown, + hashify::fnc_map!(value.as_bytes(), + b"yes" => Some(Displayed::Yes), + b"unknown" => Some(Displayed::Unknown), + _ => None, ) } - fn as_str(&self) -> &'static str { + pub(crate) fn as_str(&self) -> &'static str { match self { Displayed::Yes => "yes", Displayed::Unknown => "unknown", @@ -288,7 +338,7 @@ impl FromStr for EmailSubmissionProperty { type Err = (); fn from_str(s: &str) -> Result { - EmailSubmissionProperty::parse(s, false).ok_or(()) + EmailSubmissionProperty::parse(s, None).ok_or(()) } } @@ -385,7 +435,7 @@ impl<'de> DeserializeArguments<'de> for EmailSubmissionComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"emailId" => { @@ -398,7 +448,7 @@ impl<'de> DeserializeArguments<'de> for EmailSubmissionComparator { *self = EmailSubmissionComparator::SentAt; }, _ => { - *self = EmailSubmissionComparator::_T(key.to_string()); + *self = EmailSubmissionComparator::_T(value.into_owned()); } ); } else { diff --git a/crates/jmap-proto/src/object/file_node.rs b/crates/jmap-proto/src/object/file_node.rs index b8d24dbb..fd4a7d83 100644 --- a/crates/jmap-proto/src/object/file_node.rs +++ b/crates/jmap-proto/src/object/file_node.rs @@ -6,14 +6,20 @@ use crate::{ object::{ - AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, + AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, + }, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments}, }, - request::{MaybeInvalid, deserialize::DeserializeArguments}, types::date::UTCDate, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::{acl::Acl, blob::BlobId, id::Id}; +use types::{acl::Acl, blob::BlobId, id::Id, text::Text}; use utils::glob::GlobPattern; #[derive(Debug, Clone, Default)] @@ -27,7 +33,7 @@ pub enum FileNodeProperty { Size, Name, Type, - NodeType, + MediaType, Target, Created, Modified, @@ -38,6 +44,8 @@ pub enum FileNodeProperty { MyRights, ShareWith, IsSubscribed, + Metadata, + PrivateMetadata, IdValue(Id), Rights(FileNodeRight), @@ -52,13 +60,14 @@ pub enum FileNodeRight { MayDelete, MayModifyContent, MayShare, + MaySetMetadata, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FileNodeNodeType { File, Directory, - Symlink, + SymLink, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -84,43 +93,48 @@ pub enum FileNodeValue { impl Property for FileNodeProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - let allow_patch = key.is_none(); - if let Some(Key::Property(key)) = key { - match key.patch_or_prop() { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) => match key.patch_or_prop() { FileNodeProperty::ShareWith => { Id::from_str(value).ok().map(FileNodeProperty::IdValue) } - _ => FileNodeProperty::parse(value, allow_patch), - } - } else { - FileNodeProperty::parse(value, allow_patch) + _ => FileNodeProperty::parse_nested_name(value), + }, + Some(_) => FileNodeProperty::parse_nested_name(value), + None => FileNodeProperty::parse(value, Some(depth)), } } + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() + } + fn to_cow(&self) -> Cow<'static, str> { - match self { - FileNodeProperty::Id => "id", - FileNodeProperty::ParentId => "parentId", - FileNodeProperty::BlobId => "blobId", - FileNodeProperty::Size => "size", - FileNodeProperty::Name => "name", - FileNodeProperty::Type => "type", - FileNodeProperty::NodeType => "nodeType", - FileNodeProperty::Target => "target", - FileNodeProperty::Created => "created", - FileNodeProperty::Modified => "modified", - FileNodeProperty::Accessed => "accessed", - FileNodeProperty::Changed => "changed", - FileNodeProperty::Executable => "executable", - FileNodeProperty::Role => "role", - FileNodeProperty::MyRights => "myRights", - FileNodeProperty::ShareWith => "shareWith", - FileNodeProperty::IsSubscribed => "isSubscribed", - FileNodeProperty::Rights(file_right) => file_right.as_str(), - FileNodeProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), - FileNodeProperty::IdValue(id) => return id.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -133,6 +147,7 @@ impl FileNodeRight { FileNodeRight::MayDelete => "mayDelete", FileNodeRight::MayModifyContent => "mayModifyContent", FileNodeRight::MayShare => "mayShare", + FileNodeRight::MaySetMetadata => "maySetMetadata", } } } @@ -140,18 +155,19 @@ impl FileNodeRight { impl FileNodeNodeType { pub fn as_str(&self) -> &'static str { match self { - FileNodeNodeType::File => "file", - FileNodeNodeType::Directory => "directory", - FileNodeNodeType::Symlink => "symlink", + FileNodeNodeType::File => "File", + FileNodeNodeType::Directory => "Directory", + FileNodeNodeType::SymLink => "SymLink", } } pub fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"file" => FileNodeNodeType::File, - b"directory" => FileNodeNodeType::Directory, - b"symlink" => FileNodeNodeType::Symlink, + hashify::map!(value.as_bytes(), FileNodeNodeType, + b"File" => FileNodeNodeType::File, + b"Directory" => FileNodeNodeType::Directory, + b"SymLink" => FileNodeNodeType::SymLink, ) + .copied() } } @@ -169,6 +185,12 @@ impl Display for FileNodeNodeType { } } +impl Serialize for FileNodeNodeType { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + impl FileNodeRole { pub fn as_str(&self) -> &'static str { match self { @@ -185,7 +207,7 @@ impl FileNodeRole { } pub fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), FileNodeRole, b"root" => FileNodeRole::Root, b"home" => FileNodeRole::Home, b"temp" => FileNodeRole::Temp, @@ -196,6 +218,7 @@ impl FileNodeRole { b"pictures" => FileNodeRole::Pictures, b"videos" => FileNodeRole::Videos, ) + .copied() } } @@ -243,48 +266,97 @@ impl Element for FileNodeValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - FileNodeValue::Id(id) => id.to_string().into(), - FileNodeValue::Date(utcdate) => utcdate.to_string().into(), - FileNodeValue::BlobId(blob_id) => blob_id.to_string().into(), - FileNodeValue::IdReference(r) => format!("#{r}").into(), + FileNodeValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl FileNodeValue { + pub fn text(&self) -> Text<'_> { + match self { + FileNodeValue::Id(id) => Text::Id(*id), + FileNodeValue::Date(utcdate) => Text::Display(utcdate), + FileNodeValue::BlobId(blob_id) => Text::Display(blob_id), + FileNodeValue::IdReference(r) => Text::Reference(r), } } } impl FileNodeProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => FileNodeProperty::Id, - b"parentId" => FileNodeProperty::ParentId, - b"blobId" => FileNodeProperty::BlobId, - b"size" => FileNodeProperty::Size, - b"name" => FileNodeProperty::Name, - b"type" => FileNodeProperty::Type, - b"nodeType" => FileNodeProperty::NodeType, - b"target" => FileNodeProperty::Target, - b"created" => FileNodeProperty::Created, - b"modified" => FileNodeProperty::Modified, - b"accessed" => FileNodeProperty::Accessed, - b"changed" => FileNodeProperty::Changed, - b"executable" => FileNodeProperty::Executable, - b"role" => FileNodeProperty::Role, - b"myRights" => FileNodeProperty::MyRights, - b"shareWith" => FileNodeProperty::ShareWith, - b"isSubscribed" => FileNodeProperty::IsSubscribed, - b"mayRead" => FileNodeProperty::Rights(FileNodeRight::MayRead), - b"mayAddChildren" => FileNodeProperty::Rights(FileNodeRight::MayAddChildren), - b"mayRename" => FileNodeProperty::Rights(FileNodeRight::MayRename), - b"mayDelete" => FileNodeProperty::Rights(FileNodeRight::MayDelete), - b"mayModifyContent" => FileNodeProperty::Rights(FileNodeRight::MayModifyContent), - b"mayShare" => FileNodeProperty::Rights(FileNodeRight::MayShare), + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + FileNodeProperty::Id => "id", + FileNodeProperty::ParentId => "parentId", + FileNodeProperty::BlobId => "blobId", + FileNodeProperty::Size => "size", + FileNodeProperty::Name => "name", + FileNodeProperty::Type => "@type", + FileNodeProperty::MediaType => "mediaType", + FileNodeProperty::Target => "target", + FileNodeProperty::Created => "created", + FileNodeProperty::Modified => "modified", + FileNodeProperty::Accessed => "accessed", + FileNodeProperty::Changed => "changed", + FileNodeProperty::Executable => "executable", + FileNodeProperty::Role => "role", + FileNodeProperty::MyRights => "myRights", + FileNodeProperty::ShareWith => "shareWith", + FileNodeProperty::IsSubscribed => "isSubscribed", + FileNodeProperty::Metadata => "metadata", + FileNodeProperty::PrivateMetadata => "privateMetadata", + FileNodeProperty::Rights(file_right) => file_right.as_str(), + FileNodeProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + FileNodeProperty::IdValue(id) => return Text::Id(*id), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!( + self, + FileNodeProperty::Pointer(_) | FileNodeProperty::IdValue(_) ) - .or_else(|| { - if allow_patch && value.contains('/') { - FileNodeProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + } + + property_names!( + FileNodeProperty, + "id" => FileNodeProperty::Id, + "parentId" => FileNodeProperty::ParentId, + "blobId" => FileNodeProperty::BlobId, + "size" => FileNodeProperty::Size, + "name" => FileNodeProperty::Name, + "@type" => FileNodeProperty::Type, + "mediaType" => FileNodeProperty::MediaType, + "target" => FileNodeProperty::Target, + "created" => FileNodeProperty::Created, + "modified" => FileNodeProperty::Modified, + "accessed" => FileNodeProperty::Accessed, + "changed" => FileNodeProperty::Changed, + "executable" => FileNodeProperty::Executable, + "role" => FileNodeProperty::Role, + "myRights" => FileNodeProperty::MyRights, + "shareWith" => FileNodeProperty::ShareWith, + "isSubscribed" => FileNodeProperty::IsSubscribed, + "mayRead" => FileNodeProperty::Rights(FileNodeRight::MayRead), + "mayAddChildren" => FileNodeProperty::Rights(FileNodeRight::MayAddChildren), + "mayRename" => FileNodeProperty::Rights(FileNodeRight::MayRename), + "mayDelete" => FileNodeProperty::Rights(FileNodeRight::MayDelete), + "mayModifyContent" => FileNodeProperty::Rights(FileNodeRight::MayModifyContent), + "mayShare" => FileNodeProperty::Rights(FileNodeRight::MayShare), + "maySetMetadata" => FileNodeProperty::Rights(FileNodeRight::MaySetMetadata), + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + FileNodeProperty::parse_name(value).or_else(|| { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(FileNodeProperty::Pointer) }) } @@ -299,6 +371,30 @@ impl FileNodeProperty { } } +impl MetadataProperty for FileNodeProperty { + fn as_metadata_root(&self) -> Option { + match self { + FileNodeProperty::Metadata => Some(MetadataRoot::Shared), + FileNodeProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + FileNodeProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => FileNodeProperty::Metadata, + MetadataRoot::Private => FileNodeProperty::PrivateMetadata, + } + } +} + #[derive(Debug, Clone, Default)] pub struct FileNodeSetArguments { pub on_destroy_remove_children: Option, @@ -322,16 +418,23 @@ impl<'de> serde::Deserialize<'de> for OnExists { where D: serde::Deserializer<'de>, { - let value: Option> = Option::deserialize(deserializer)?; - match value.as_deref() { - Some("replace") => Ok(OnExists::Replace), - Some("rename") => Ok(OnExists::Rename), - Some("newest") => Ok(OnExists::Newest), - None | Some("") => Ok(OnExists::Reject), - Some(other) => Err(serde::de::Error::custom(format!( - "Invalid onExists value: {other:?}" - ))), - } + let Some(value) = + Option::>::deserialize(deserializer)?.filter(|value| !value.0.is_empty()) + else { + return Ok(OnExists::Reject); + }; + hashify::fnc_map!(value.0.as_bytes(), + b"replace" => Ok(OnExists::Replace), + b"rename" => Ok(OnExists::Rename), + b"newest" => Ok(OnExists::Newest), + b"unchanged" => Err(serde::de::Error::custom( + "onExists \"unchanged\" requires JMAP Conditional Set, which is not supported", + )), + _ => Err(serde::de::Error::custom(format!( + "Invalid onExists value: {:?}", + value.0 + ))) + ) } } @@ -403,7 +506,12 @@ impl FromStr for FileNodeProperty { type Err = (); fn from_str(s: &str) -> Result { - FileNodeProperty::parse(s, false).ok_or(()) + FileNodeProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| FileNodeProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) } } @@ -449,9 +557,10 @@ impl JmapRight for FileNodeRight { FileNodeRight::MayRead => &[Acl::Read, Acl::ReadItems], FileNodeRight::MayAddChildren => &[Acl::AddItems], FileNodeRight::MayRename => &[Acl::Modify], - FileNodeRight::MayDelete => &[Acl::Delete, Acl::RemoveItems], + FileNodeRight::MayDelete => &[Acl::RemoveItems], FileNodeRight::MayModifyContent => &[Acl::ModifyItems], FileNodeRight::MayShare => &[Acl::Share], + FileNodeRight::MaySetMetadata => &[Acl::ModifyMetadata], } } @@ -463,6 +572,7 @@ impl JmapRight for FileNodeRight { FileNodeRight::MayDelete, FileNodeRight::MayModifyContent, FileNodeRight::MayShare, + FileNodeRight::MaySetMetadata, ] } } @@ -494,10 +604,11 @@ pub enum FileNodeFilter { MaxSize(u64), Name(String), NameMatch(GlobPattern), - Type(String), - TypeMatch(GlobPattern), + MediaType(String), + MediaTypeMatch(GlobPattern), Text(String), Body(String), + Metadata(MetadataFilter), _T(String), } @@ -507,7 +618,7 @@ pub enum FileNodeComparator { Size, Created, Modified, - Type, + MediaType, NodeType, Tree, _T(String), @@ -576,11 +687,11 @@ impl<'de> DeserializeArguments<'de> for FileNodeFilter { b"nameMatch" => { *self = FileNodeFilter::NameMatch(map.next_value()?); }, - b"type" => { - *self = FileNodeFilter::Type(map.next_value()?); + b"mediaType" => { + *self = FileNodeFilter::MediaType(map.next_value()?); }, - b"typeMatch" => { - *self = FileNodeFilter::TypeMatch(map.next_value()?); + b"mediaTypeMatch" => { + *self = FileNodeFilter::MediaTypeMatch(map.next_value()?); }, b"body" => { *self = FileNodeFilter::Body(map.next_value()?); @@ -589,8 +700,13 @@ impl<'de> DeserializeArguments<'de> for FileNodeFilter { *self = FileNodeFilter::Text(map.next_value()?); }, _ => { - *self = FileNodeFilter::_T(key.to_string()); - let _ = map.next_value::()?; + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => FileNodeFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + FileNodeFilter::_T(key.to_string()) + } + }; } ); @@ -604,7 +720,7 @@ impl<'de> DeserializeArguments<'de> for FileNodeComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = FileNodeComparator::Name; @@ -618,8 +734,8 @@ impl<'de> DeserializeArguments<'de> for FileNodeComparator { b"modified" => { *self = FileNodeComparator::Modified; }, - b"type" => { - *self = FileNodeComparator::Type; + b"mediaType" => { + *self = FileNodeComparator::MediaType; }, b"nodeType" => { *self = FileNodeComparator::NodeType; @@ -716,10 +832,11 @@ impl FileNodeFilter { FileNodeFilter::MaxSize(_) => "maxSize", FileNodeFilter::Name(_) => "name", FileNodeFilter::NameMatch(_) => "nameMatch", - FileNodeFilter::Type(_) => "type", - FileNodeFilter::TypeMatch(_) => "typeMatch", + FileNodeFilter::MediaType(_) => "mediaType", + FileNodeFilter::MediaTypeMatch(_) => "mediaTypeMatch", FileNodeFilter::Text(_) => "text", FileNodeFilter::Body(_) => "body", + FileNodeFilter::Metadata(filter) => filter.as_str(), FileNodeFilter::_T(s) => return s.into(), } .into() @@ -733,7 +850,7 @@ impl FileNodeComparator { FileNodeComparator::Size => "size", FileNodeComparator::Created => "created", FileNodeComparator::Modified => "modified", - FileNodeComparator::Type => "type", + FileNodeComparator::MediaType => "mediaType", FileNodeComparator::NodeType => "nodeType", FileNodeComparator::Tree => "tree", FileNodeComparator::_T(s) => s.as_ref(), @@ -746,7 +863,7 @@ impl FileNodeComparator { FileNodeComparator::Size => "size", FileNodeComparator::Created => "created", FileNodeComparator::Modified => "modified", - FileNodeComparator::Type => "type", + FileNodeComparator::MediaType => "mediaType", FileNodeComparator::NodeType => "nodeType", FileNodeComparator::Tree => "tree", FileNodeComparator::_T(s) => return s.into(), diff --git a/crates/jmap-proto/src/object/identity.rs b/crates/jmap-proto/src/object/identity.rs index 4f9b43dd..4d030538 100644 --- a/crates/jmap-proto/src/object/identity.rs +++ b/crates/jmap-proto/src/object/identity.rs @@ -5,9 +5,10 @@ */ use crate::object::{AnyId, JmapObject, JmapObjectId}; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct Identity; @@ -34,22 +35,35 @@ pub enum IdentityValue { impl Property for IdentityProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - IdentityProperty::parse(value, key.is_none()) + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + IdentityProperty::parse(value, key.is_none().then_some(depth)) } fn to_cow(&self) -> Cow<'static, str> { - match self { - IdentityProperty::Bcc => "bcc", - IdentityProperty::Email => "email", - IdentityProperty::HtmlSignature => "htmlSignature", - IdentityProperty::Id => "id", - IdentityProperty::MayDelete => "mayDelete", - IdentityProperty::Name => "name", - IdentityProperty::ReplyTo => "replyTo", - IdentityProperty::TextSignature => "textSignature", - IdentityProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -68,30 +82,58 @@ impl Element for IdentityValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl IdentityValue { + pub fn text(&self) -> Text<'_> { match self { - IdentityValue::Id(id) => id.to_string().into(), + IdentityValue::Id(id) => Text::Id(*id), } } } impl IdentityProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => IdentityProperty::Id, - b"name" => IdentityProperty::Name, - b"email" => IdentityProperty::Email, - b"replyTo" => IdentityProperty::ReplyTo, - b"bcc" => IdentityProperty::Bcc, - b"textSignature" => IdentityProperty::TextSignature, - b"htmlSignature" => IdentityProperty::HtmlSignature, - b"mayDelete" => IdentityProperty::MayDelete, + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + IdentityProperty::Bcc => "bcc", + IdentityProperty::Email => "email", + IdentityProperty::HtmlSignature => "htmlSignature", + IdentityProperty::Id => "id", + IdentityProperty::MayDelete => "mayDelete", + IdentityProperty::Name => "name", + IdentityProperty::ReplyTo => "replyTo", + IdentityProperty::TextSignature => "textSignature", + IdentityProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!(self, IdentityProperty::Pointer(_)) + } + + fn parse(value: &str, patch_depth: Option) -> Option { + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(IdentityProperty::Id), + b"name" => Some(IdentityProperty::Name), + b"email" => Some(IdentityProperty::Email), + b"replyTo" => Some(IdentityProperty::ReplyTo), + b"bcc" => Some(IdentityProperty::Bcc), + b"textSignature" => Some(IdentityProperty::TextSignature), + b"htmlSignature" => Some(IdentityProperty::HtmlSignature), + b"mayDelete" => Some(IdentityProperty::MayDelete), + _ => None, ) .or_else(|| { - if allow_patch && value.contains('/') { - IdentityProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(IdentityProperty::Pointer) }) } @@ -110,7 +152,7 @@ impl FromStr for IdentityProperty { type Err = (); fn from_str(s: &str) -> Result { - IdentityProperty::parse(s, false).ok_or(()) + IdentityProperty::parse(s, None).ok_or(()) } } diff --git a/crates/jmap-proto/src/object/mailbox.rs b/crates/jmap-proto/src/object/mailbox.rs index dbdfc964..dfada287 100644 --- a/crates/jmap-proto/src/object/mailbox.rs +++ b/crates/jmap-proto/src/object/mailbox.rs @@ -6,13 +6,19 @@ use crate::{ object::{ - AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, + AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, + }, + request::{ + deserialize::{CowStr, DeserializeArguments}, + reference::MaybeIdReference, }, - request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{acl::Acl, id::Id, special_use::SpecialUse}; +use types::{acl::Acl, id::Id, special_use::SpecialUse, text::Text}; #[derive(Debug, Clone, Default)] pub struct Mailbox; @@ -31,6 +37,8 @@ pub enum MailboxProperty { ShareWith, MyRights, IsSubscribed, + Metadata, + PrivateMetadata, // Other IdValue(Id), @@ -50,6 +58,7 @@ pub enum MailboxRight { MaySubmit, MayDelete, MayShare, + MaySetMetadata, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -61,38 +70,48 @@ pub enum MailboxValue { impl Property for MailboxProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - let allow_patch = key.is_none(); - if let Some(Key::Property(key)) = key { - match key.patch_or_prop() { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) => match key.patch_or_prop() { MailboxProperty::ShareWith => { Id::from_str(value).ok().map(MailboxProperty::IdValue) } - _ => MailboxProperty::parse(value, allow_patch), - } - } else { - MailboxProperty::parse(value, allow_patch) + _ => MailboxProperty::parse_nested_name(value), + }, + Some(_) => MailboxProperty::parse_nested_name(value), + None => MailboxProperty::parse(value, Some(depth)), } } + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() + } + fn to_cow(&self) -> Cow<'static, str> { - match self { - MailboxProperty::Id => "id", - MailboxProperty::IsSubscribed => "isSubscribed", - MailboxProperty::MyRights => "myRights", - MailboxProperty::Name => "name", - MailboxProperty::ParentId => "parentId", - MailboxProperty::Role => "role", - MailboxProperty::SortOrder => "sortOrder", - MailboxProperty::TotalEmails => "totalEmails", - MailboxProperty::TotalThreads => "totalThreads", - MailboxProperty::UnreadEmails => "unreadEmails", - MailboxProperty::UnreadThreads => "unreadThreads", - MailboxProperty::ShareWith => "shareWith", - MailboxProperty::Rights(mailbox_right) => mailbox_right.as_str(), - MailboxProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), - MailboxProperty::IdValue(id) => return id.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -109,6 +128,7 @@ impl MailboxRight { MailboxRight::MaySubmit => "maySubmit", MailboxRight::MayDelete => "mayDelete", MailboxRight::MayShare => "mayShare", + MailboxRight::MaySetMetadata => "maySetMetadata", } } } @@ -133,46 +153,89 @@ impl Element for MailboxValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl MailboxValue { + pub fn text(&self) -> Text<'_> { match self { - MailboxValue::Id(id) => id.to_string().into(), - MailboxValue::IdReference(r) => format!("#{r}").into(), - MailboxValue::Role(special_use) => special_use.as_str().unwrap_or_default().into(), + MailboxValue::Id(id) => Text::Id(*id), + MailboxValue::IdReference(r) => Text::Reference(r), + MailboxValue::Role(special_use) => { + Text::Static(special_use.as_str().unwrap_or_default()) + } } } } impl MailboxProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => MailboxProperty::Id, - b"name" => MailboxProperty::Name, - b"parentId" => MailboxProperty::ParentId, - b"role" => MailboxProperty::Role, - b"sortOrder" => MailboxProperty::SortOrder, - b"totalEmails" => MailboxProperty::TotalEmails, - b"unreadEmails" => MailboxProperty::UnreadEmails, - b"totalThreads" => MailboxProperty::TotalThreads, - b"unreadThreads" => MailboxProperty::UnreadThreads, - b"shareWith" => MailboxProperty::ShareWith, - b"myRights" => MailboxProperty::MyRights, - b"mayReadItems" => MailboxProperty::Rights(MailboxRight::MayReadItems), - b"mayAddItems" => MailboxProperty::Rights(MailboxRight::MayAddItems), - b"mayRemoveItems" => MailboxProperty::Rights(MailboxRight::MayRemoveItems), - b"maySetSeen" => MailboxProperty::Rights(MailboxRight::MaySetSeen), - b"maySetKeywords" => MailboxProperty::Rights(MailboxRight::MaySetKeywords), - b"mayCreateChild" => MailboxProperty::Rights(MailboxRight::MayCreateChild), - b"mayRename" => MailboxProperty::Rights(MailboxRight::MayRename), - b"maySubmit" => MailboxProperty::Rights(MailboxRight::MaySubmit), - b"mayDelete" => MailboxProperty::Rights(MailboxRight::MayDelete), - b"mayShare" => MailboxProperty::Rights(MailboxRight::MayShare), - b"isSubscribed" => MailboxProperty::IsSubscribed, + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + MailboxProperty::Id => "id", + MailboxProperty::IsSubscribed => "isSubscribed", + MailboxProperty::MyRights => "myRights", + MailboxProperty::Name => "name", + MailboxProperty::ParentId => "parentId", + MailboxProperty::Role => "role", + MailboxProperty::SortOrder => "sortOrder", + MailboxProperty::TotalEmails => "totalEmails", + MailboxProperty::TotalThreads => "totalThreads", + MailboxProperty::UnreadEmails => "unreadEmails", + MailboxProperty::UnreadThreads => "unreadThreads", + MailboxProperty::ShareWith => "shareWith", + MailboxProperty::Metadata => "metadata", + MailboxProperty::PrivateMetadata => "privateMetadata", + MailboxProperty::Rights(mailbox_right) => mailbox_right.as_str(), + MailboxProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + MailboxProperty::IdValue(id) => return Text::Id(*id), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!( + self, + MailboxProperty::Pointer(_) | MailboxProperty::IdValue(_) ) - .or_else(|| { - if allow_patch && value.contains('/') { - MailboxProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + } + + property_names!( + MailboxProperty, + "id" => MailboxProperty::Id, + "name" => MailboxProperty::Name, + "parentId" => MailboxProperty::ParentId, + "role" => MailboxProperty::Role, + "sortOrder" => MailboxProperty::SortOrder, + "totalEmails" => MailboxProperty::TotalEmails, + "unreadEmails" => MailboxProperty::UnreadEmails, + "totalThreads" => MailboxProperty::TotalThreads, + "unreadThreads" => MailboxProperty::UnreadThreads, + "shareWith" => MailboxProperty::ShareWith, + "myRights" => MailboxProperty::MyRights, + "mayReadItems" => MailboxProperty::Rights(MailboxRight::MayReadItems), + "mayAddItems" => MailboxProperty::Rights(MailboxRight::MayAddItems), + "mayRemoveItems" => MailboxProperty::Rights(MailboxRight::MayRemoveItems), + "maySetSeen" => MailboxProperty::Rights(MailboxRight::MaySetSeen), + "maySetKeywords" => MailboxProperty::Rights(MailboxRight::MaySetKeywords), + "mayCreateChild" => MailboxProperty::Rights(MailboxRight::MayCreateChild), + "mayRename" => MailboxProperty::Rights(MailboxRight::MayRename), + "maySubmit" => MailboxProperty::Rights(MailboxRight::MaySubmit), + "mayDelete" => MailboxProperty::Rights(MailboxRight::MayDelete), + "mayShare" => MailboxProperty::Rights(MailboxRight::MayShare), + "maySetMetadata" => MailboxProperty::Rights(MailboxRight::MaySetMetadata), + "isSubscribed" => MailboxProperty::IsSubscribed, + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + MailboxProperty::parse_name(value).or_else(|| { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(MailboxProperty::Pointer) }) } @@ -187,6 +250,30 @@ impl MailboxProperty { } } +impl MetadataProperty for MailboxProperty { + fn as_metadata_root(&self) -> Option { + match self { + MailboxProperty::Metadata => Some(MetadataRoot::Shared), + MailboxProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + MailboxProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => MailboxProperty::Metadata, + MetadataRoot::Private => MailboxProperty::PrivateMetadata, + } + } +} + #[derive(Debug, Clone, Default)] pub struct MailboxSetArguments { pub on_destroy_remove_emails: Option, @@ -238,7 +325,12 @@ impl FromStr for MailboxProperty { type Err = (); fn from_str(s: &str) -> Result { - MailboxProperty::parse(s, false).ok_or(()) + MailboxProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| MailboxProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) } } @@ -309,6 +401,7 @@ pub enum MailboxFilter { Role(Option), HasAnyRole(bool), IsSubscribed(bool), + Metadata(MetadataFilter), _T(String), } @@ -342,8 +435,13 @@ impl<'de> DeserializeArguments<'de> for MailboxFilter { *self = MailboxFilter::IsSubscribed(map.next_value()?); }, _ => { - *self = MailboxFilter::_T(key.to_string()); - let _ = map.next_value::()?; + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => MailboxFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + MailboxFilter::_T(key.to_string()) + } + }; } ); @@ -357,7 +455,7 @@ impl<'de> DeserializeArguments<'de> for MailboxComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"sortOrder" => { *self = MailboxComparator::SortOrder; @@ -369,7 +467,7 @@ impl<'de> DeserializeArguments<'de> for MailboxComparator { *self = MailboxComparator::ParentId; }, _ => { - *self = MailboxComparator::_T(key.to_string()); + *self = MailboxComparator::_T(value.into_owned()); } ); } else { @@ -459,6 +557,7 @@ impl JmapRight for MailboxRight { MailboxRight::MaySubmit => &[Acl::Submit], MailboxRight::MayDelete => &[Acl::Delete], MailboxRight::MayShare => &[Acl::Share], + MailboxRight::MaySetMetadata => &[Acl::ModifyMetadata], } } @@ -474,6 +573,7 @@ impl JmapRight for MailboxRight { MailboxRight::MaySubmit, MailboxRight::MayDelete, MailboxRight::MayShare, + MailboxRight::MaySetMetadata, ] } } diff --git a/crates/jmap-proto/src/object/metadata.rs b/crates/jmap-proto/src/object/metadata.rs new file mode 100644 index 00000000..61d7dac0 --- /dev/null +++ b/crates/jmap-proto/src/object/metadata.rs @@ -0,0 +1,504 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Property, Value}; +use serde::de::{Deserializer, Error, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use std::{borrow::Cow, fmt, str::FromStr}; + +macro_rules! property_names { + ($property:ident, $($name:literal => $value:expr,)*) => { + fn parse_name(value: &str) -> Option { + hashify::fnc_map!(value.as_bytes(), + $($name => Some($value),)* + "metadata" => Some($property::Metadata), + "privateMetadata" => Some($property::PrivateMetadata), + _ => None, + ) + } + + fn parse_nested_name(value: &str) -> Option { + hashify::fnc_map!(value.as_bytes(), + $($name => Some($value),)* + _ => None, + ) + } + }; +} + +pub(crate) use property_names; + +#[derive( + rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, +)] +pub enum MetadataRoot { + Shared, + Private, +} + +pub trait MetadataProperty: Property { + fn as_metadata_root(&self) -> Option; + + fn as_pointer(&self) -> Option<&JsonPointer>; + + fn from_metadata_root(root: MetadataRoot) -> Self; + + fn metadata_pointer(&self) -> Option<(MetadataRoot, &JsonPointer)> { + let pointer = self.as_pointer()?; + let root = pointer.first()?.as_property_key()?.as_metadata_root()?; + Some((root, pointer)) + } + + fn metadata_root(&self) -> Option { + self.as_metadata_root() + .or_else(|| self.metadata_pointer().map(|(root, _)| root)) + } + + fn has_metadata>(object: &Value<'_, Self, E>) -> bool { + object.as_object().is_some_and(|object| { + object + .keys() + .any(|key| key.as_property().and_then(Self::metadata_root).is_some()) + }) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum Selection { + #[default] + None, + All, + Namespaces(Vec>), +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MetadataSelection { + pub shared: Selection, + pub private: Selection, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct MetadataPath { + pub namespace: Box, + pub keys: Box<[Box]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MetadataFilter { + Condition { + root: MetadataRoot, + path: MetadataPath, + condition: MetadataCondition, + }, + Invalid { + root: MetadataRoot, + name: &'static str, + reason: &'static str, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MetadataCondition { + Exists, + TextContains(String), + TextEquals(String), +} + +enum FilterArgument<'de> { + String(Cow<'de, str>), + TextMatch(Result<(MetadataPath, String), &'static str>), + Other, +} + +struct FilterArgumentVisitor; + +impl MetadataRoot { + pub fn parse(value: &str) -> Option { + hashify::fnc_map!(value.as_bytes(), + b"metadata" => Some(MetadataRoot::Shared), + b"privateMetadata" => Some(MetadataRoot::Private), + _ => None, + ) + } + + pub fn as_str(&self) -> &'static str { + match self { + MetadataRoot::Shared => "metadata", + MetadataRoot::Private => "privateMetadata", + } + } + + pub fn from_selector(value: &str) -> Option { + value + .split_once('/') + .and_then(|(root, _)| MetadataRoot::parse(root)) + } +} + +impl Selection { + pub fn is_none(&self) -> bool { + matches!(self, Selection::None) + } + + pub fn contains(&self, namespace: &str) -> bool { + match self { + Selection::None => false, + Selection::All => true, + Selection::Namespaces(namespaces) => namespaces + .binary_search_by(|ns| (**ns).cmp(namespace)) + .is_ok(), + } + } + + fn select_all(&mut self) { + *self = Selection::All; + } + + fn select_namespace(&mut self, namespace: Box) { + match self { + Selection::None => *self = Selection::Namespaces(vec![namespace]), + Selection::All => {} + Selection::Namespaces(namespaces) => namespaces.push(namespace), + } + } + + fn normalize(&mut self) { + if let Selection::Namespaces(namespaces) = self { + namespaces.sort_unstable(); + namespaces.dedup(); + } + } +} + +impl MetadataSelection { + pub fn all() -> Self { + MetadataSelection { + shared: Selection::All, + private: Selection::All, + } + } + + pub fn is_none(&self) -> bool { + self.shared.is_none() && self.private.is_none() + } + + pub fn extract(properties: &mut Vec

) -> trc::Result { + let mut selection = MetadataSelection::default(); + let mut result = Ok(()); + + properties.retain(|property| { + if let Some(root) = property.as_metadata_root() { + selection.root_mut(root).select_all(); + false + } else if let Some((root, pointer)) = property.metadata_pointer() { + match MetadataSelection::namespace(pointer) { + Ok(namespace) => selection.root_mut(root).select_namespace(namespace), + Err(err) => result = Err(err), + } + false + } else { + true + } + }); + + result.map(|_| { + selection.shared.normalize(); + selection.private.normalize(); + selection + }) + } + + pub fn root(&self, root: MetadataRoot) -> &Selection { + match root { + MetadataRoot::Shared => &self.shared, + MetadataRoot::Private => &self.private, + } + } + + fn root_mut(&mut self, root: MetadataRoot) -> &mut Selection { + match root { + MetadataRoot::Shared => &mut self.shared, + MetadataRoot::Private => &mut self.private, + } + } + + fn namespace(pointer: &JsonPointer

) -> trc::Result> { + let [_, namespace] = pointer.as_slice() else { + return Err(trc::JmapEvent::InvalidArguments.into_err().details(format!( + "Metadata subselector {pointer} must have exactly two segments" + ))); + }; + let JsonPointerItem::Key(key) = namespace else { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format!("Invalid metadata subselector {pointer}"))); + }; + + Ok(key.to_string().into()) + } +} + +impl MetadataPath { + fn unescape(segment: &str) -> Option> { + let Some((literal, mut rest)) = segment.split_once('~') else { + return Some(segment.into()); + }; + let mut unescaped = String::with_capacity(segment.len()); + unescaped.push_str(literal); + + loop { + let (escaped, tail) = rest + .strip_prefix('0') + .map(|tail| ('~', tail)) + .or_else(|| rest.strip_prefix('1').map(|tail| ('/', tail)))?; + unescaped.push(escaped); + rest = tail; + + match rest.split_once('~') { + Some((literal, tail)) => { + unescaped.push_str(literal); + rest = tail; + } + None => { + unescaped.push_str(rest); + return Some(unescaped.into_boxed_str()); + } + } + } + } +} + +impl FromStr for MetadataPath { + type Err = &'static str; + + fn from_str(path: &str) -> Result { + let unescape = + |segment| MetadataPath::unescape(segment).ok_or("invalid escape in metadata path"); + let (namespace, keys) = match path.split_once('/') { + Some((namespace, keys)) => (namespace, keys.split('/').map(unescape).collect()), + None => (path, Ok(Box::default())), + }; + + Ok(MetadataPath { + namespace: unescape(namespace)?, + keys: keys?, + }) + } +} + +impl MetadataFilter { + pub fn try_deserialize<'de, A>(key: &str, map: &mut A) -> Result, A::Error> + where + A: MapAccess<'de>, + { + hashify::fnc_map!(key.as_bytes(), + b"metadataExists" => MetadataFilter::deserialize_exists( + MetadataRoot::Shared, + "metadataExists", + map, + ), + b"privateMetadataExists" => MetadataFilter::deserialize_exists( + MetadataRoot::Private, + "privateMetadataExists", + map, + ), + b"metadataTextContains" => MetadataFilter::deserialize_text( + MetadataRoot::Shared, + "metadataTextContains", + MetadataCondition::TextContains, + map, + ), + b"privateMetadataTextContains" => MetadataFilter::deserialize_text( + MetadataRoot::Private, + "privateMetadataTextContains", + MetadataCondition::TextContains, + map, + ), + b"metadataTextEquals" => MetadataFilter::deserialize_text( + MetadataRoot::Shared, + "metadataTextEquals", + MetadataCondition::TextEquals, + map, + ), + b"privateMetadataTextEquals" => MetadataFilter::deserialize_text( + MetadataRoot::Private, + "privateMetadataTextEquals", + MetadataCondition::TextEquals, + map, + ), + _ => Ok(None), + ) + } + + fn deserialize_exists<'de, A>( + root: MetadataRoot, + name: &'static str, + map: &mut A, + ) -> Result, A::Error> + where + A: MapAccess<'de>, + { + let filter = match map.next_value::()? { + FilterArgument::String(path) => { + MetadataPath::from_str(&path).map(|path| MetadataFilter::Condition { + root, + path, + condition: MetadataCondition::Exists, + }) + } + FilterArgument::TextMatch(_) | FilterArgument::Other => { + Err("expected a metadata path string") + } + }; + Ok(Some(filter.unwrap_or_else(|reason| { + MetadataFilter::Invalid { root, name, reason } + }))) + } + + fn deserialize_text<'de, A>( + root: MetadataRoot, + name: &'static str, + condition: fn(String) -> MetadataCondition, + map: &mut A, + ) -> Result, A::Error> + where + A: MapAccess<'de>, + { + let filter = match map.next_value::()? { + FilterArgument::TextMatch(text_match) => { + text_match.map(|(path, value)| MetadataFilter::Condition { + root, + path, + condition: condition(value), + }) + } + FilterArgument::String(_) | FilterArgument::Other => { + Err("expected a MetadataTextMatch object") + } + }; + Ok(Some(filter.unwrap_or_else(|reason| { + MetadataFilter::Invalid { root, name, reason } + }))) + } + + pub fn root(&self) -> MetadataRoot { + match self { + MetadataFilter::Condition { root, .. } | MetadataFilter::Invalid { root, .. } => *root, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + MetadataFilter::Invalid { name, .. } => name, + MetadataFilter::Condition { + root, condition, .. + } => match (root, condition) { + (MetadataRoot::Shared, MetadataCondition::Exists) => "metadataExists", + (MetadataRoot::Shared, MetadataCondition::TextContains(_)) => { + "metadataTextContains" + } + (MetadataRoot::Shared, MetadataCondition::TextEquals(_)) => "metadataTextEquals", + (MetadataRoot::Private, MetadataCondition::Exists) => "privateMetadataExists", + (MetadataRoot::Private, MetadataCondition::TextContains(_)) => { + "privateMetadataTextContains" + } + (MetadataRoot::Private, MetadataCondition::TextEquals(_)) => { + "privateMetadataTextEquals" + } + }, + } + } +} + +impl<'de> serde::Deserialize<'de> for FilterArgument<'de> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(FilterArgumentVisitor) + } +} + +impl<'de> Visitor<'de> for FilterArgumentVisitor { + type Value = FilterArgument<'de>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a metadata filter argument") + } + + fn visit_borrowed_str(self, value: &'de str) -> Result { + Ok(FilterArgument::String(Cow::Borrowed(value))) + } + + fn visit_str(self, value: &str) -> Result { + Ok(FilterArgument::String(Cow::Owned(value.into()))) + } + + fn visit_string(self, value: String) -> Result { + Ok(FilterArgument::String(Cow::Owned(value))) + } + + fn visit_unit(self) -> Result { + Ok(FilterArgument::Other) + } + + fn visit_bool(self, _: bool) -> Result { + Ok(FilterArgument::Other) + } + + fn visit_i64(self, _: i64) -> Result { + Ok(FilterArgument::Other) + } + + fn visit_u64(self, _: u64) -> Result { + Ok(FilterArgument::Other) + } + + fn visit_f64(self, _: f64) -> Result { + Ok(FilterArgument::Other) + } + + fn visit_seq>(self, seq: A) -> Result { + IgnoredAny.visit_seq(seq).map(|_| FilterArgument::Other) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut path = None; + let mut value = None; + let mut error = None; + + while let Some(key) = map.next_key::()? { + let argument = map.next_value::()?; + let field = match &key { + FilterArgument::String(key) => hashify::fnc_map!(key.as_bytes(), + b"path" => Some(&mut path), + b"value" => Some(&mut value), + _ => None, + ), + FilterArgument::TextMatch(_) | FilterArgument::Other => None, + }; + match (field, argument) { + (Some(field), FilterArgument::String(text)) if field.is_none() => { + *field = Some(text); + } + (Some(field), _) if field.is_none() => { + error.get_or_insert("MetadataTextMatch path and value must be strings"); + } + _ => { + error.get_or_insert("unexpected or duplicate MetadataTextMatch field"); + } + } + } + + Ok(FilterArgument::TextMatch(match (error, path, value) { + (Some(reason), _, _) => Err(reason), + (None, Some(path), Some(value)) => { + MetadataPath::from_str(&path).map(|path| (path, value.into_owned())) + } + (None, _, _) => Err("MetadataTextMatch requires a path and a value"), + })) + } +} + diff --git a/crates/jmap-proto/src/object/mod.rs b/crates/jmap-proto/src/object/mod.rs index 938fe1d2..cd6613e9 100644 --- a/crates/jmap-proto/src/object/mod.rs +++ b/crates/jmap-proto/src/object/mod.rs @@ -9,6 +9,7 @@ use jmap_tools::{Element, Null, Property}; use serde::Serialize; use std::{fmt::Debug, str::FromStr}; use types::{acl::Acl, blob::BlobId, id::Id}; +use utils::map::bitmap::Bitmap; pub mod addressbook; pub mod blob; @@ -21,6 +22,7 @@ pub mod email_submission; pub mod file_node; pub mod identity; pub mod mailbox; +pub mod metadata; pub mod participant_identity; pub mod principal; pub mod push_subscription; @@ -58,6 +60,14 @@ pub trait JmapSharedObject: JmapObject { pub trait JmapRight: Clone + Copy + Sized + 'static { fn all_rights() -> &'static [Self]; fn to_acl(&self) -> &'static [Acl]; + fn implied_by_acl(&self) -> &'static [Acl] { + &[] + } + fn is_granted(&self, acls: &Bitmap) -> bool { + let is_granted = |required: &[Acl]| required.iter().all(|acl| acls.contains(*acl)); + is_granted(self.to_acl()) + || (!self.implied_by_acl().is_empty() && is_granted(self.implied_by_acl())) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] diff --git a/crates/jmap-proto/src/object/participant_identity.rs b/crates/jmap-proto/src/object/participant_identity.rs index d66f5596..bef169ce 100644 --- a/crates/jmap-proto/src/object/participant_identity.rs +++ b/crates/jmap-proto/src/object/participant_identity.rs @@ -9,8 +9,9 @@ use crate::{ request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct ParticipantIdentity; @@ -42,6 +43,10 @@ impl Property for ParticipantIdentityProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for ParticipantIdentityValue { @@ -61,19 +66,30 @@ impl Element for ParticipantIdentityValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl ParticipantIdentityValue { + pub fn text(&self) -> Text<'_> { match self { - ParticipantIdentityValue::Id(id) => id.to_string().into(), + ParticipantIdentityValue::Id(id) => Text::Id(*id), } } } impl ParticipantIdentityProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => ParticipantIdentityProperty::Id, - b"name" => ParticipantIdentityProperty::Name, - b"calendarAddress" => ParticipantIdentityProperty::CalendarAddress, - b"isDefault" => ParticipantIdentityProperty::IsDefault + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(ParticipantIdentityProperty::Id), + b"name" => Some(ParticipantIdentityProperty::Name), + b"calendarAddress" => Some(ParticipantIdentityProperty::CalendarAddress), + b"isDefault" => Some(ParticipantIdentityProperty::IsDefault), + _ => None ) } diff --git a/crates/jmap-proto/src/object/principal.rs b/crates/jmap-proto/src/object/principal.rs index e0af7a7b..86929a68 100644 --- a/crates/jmap-proto/src/object/principal.rs +++ b/crates/jmap-proto/src/object/principal.rs @@ -5,12 +5,16 @@ */ use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; use crate::{ object::{AnyId, JmapObject, JmapObjectId}, - request::{capability::Capability, deserialize::DeserializeArguments}, + request::{ + capability::Capability, + deserialize::{CowStr, DeserializeArguments}, + }, }; #[derive(Debug, Clone, Default)] @@ -51,19 +55,23 @@ impl Property for PrincipalProperty { } fn to_cow(&self) -> Cow<'static, str> { - match self { - PrincipalProperty::Capabilities => "capabilities", - PrincipalProperty::Description => "description", - PrincipalProperty::Email => "email", - PrincipalProperty::Id => "id", - PrincipalProperty::Name => "name", - PrincipalProperty::Timezone => "timezone", - PrincipalProperty::Type => "type", - PrincipalProperty::Accounts => "accounts", - PrincipalProperty::Capability(cap) => cap.as_str(), - PrincipalProperty::IdValue(id) => return id.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -83,24 +91,54 @@ impl Element for PrincipalValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl PrincipalValue { + pub fn text(&self) -> Text<'_> { match self { - PrincipalValue::Id(id) => id.to_string().into(), - PrincipalValue::Type(t) => t.as_str().into(), + PrincipalValue::Id(id) => Text::Id(*id), + PrincipalValue::Type(t) => Text::Static(t.as_str()), } } } impl PrincipalProperty { + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + PrincipalProperty::Capabilities => "capabilities", + PrincipalProperty::Description => "description", + PrincipalProperty::Email => "email", + PrincipalProperty::Id => "id", + PrincipalProperty::Name => "name", + PrincipalProperty::Timezone => "timeZone", + PrincipalProperty::Type => "type", + PrincipalProperty::Accounts => "accounts", + PrincipalProperty::Capability(cap) => cap.as_str(), + PrincipalProperty::IdValue(id) => return Text::Id(*id), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!(self, PrincipalProperty::IdValue(_)) + } + pub fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => PrincipalProperty::Id, - b"type" => PrincipalProperty::Type, - b"name" => PrincipalProperty::Name, - b"description" => PrincipalProperty::Description, - b"email" => PrincipalProperty::Email, - b"timeZone" => PrincipalProperty::Timezone, - b"capabilities" => PrincipalProperty::Capabilities, - b"accounts" => PrincipalProperty::Accounts, + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(PrincipalProperty::Id), + b"type" => Some(PrincipalProperty::Type), + b"name" => Some(PrincipalProperty::Name), + b"description" => Some(PrincipalProperty::Description), + b"email" => Some(PrincipalProperty::Email), + b"timeZone" => Some(PrincipalProperty::Timezone), + b"capabilities" => Some(PrincipalProperty::Capabilities), + b"accounts" => Some(PrincipalProperty::Accounts), + _ => None, ) } @@ -122,12 +160,13 @@ impl PrincipalProperty { impl PrincipalType { pub fn parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), - b"individual" => PrincipalType::Individual, - b"group" => PrincipalType::Group, - b"resource" => PrincipalType::Resource, - b"location" => PrincipalType::Location, - b"other" => PrincipalType::Other, + hashify::fnc_map!(s.as_bytes(), + b"individual" => Some(PrincipalType::Individual), + b"group" => Some(PrincipalType::Group), + b"resource" => Some(PrincipalType::Resource), + b"location" => Some(PrincipalType::Location), + b"other" => Some(PrincipalType::Other), + _ => None, ) } @@ -182,6 +221,7 @@ pub enum PrincipalFilter { Text(String), Type(PrincipalType), Timezone(String), + CalendarAddress(String), _T(String), } @@ -217,6 +257,9 @@ impl<'de> DeserializeArguments<'de> for PrincipalFilter { b"timeZone" => { *self = PrincipalFilter::Timezone(map.next_value()?); }, + b"calendarAddress" => { + *self = PrincipalFilter::CalendarAddress(map.next_value()?); + }, _ => { *self = PrincipalFilter::_T(key.to_string()); let _ = map.next_value::()?; @@ -233,7 +276,7 @@ impl<'de> DeserializeArguments<'de> for PrincipalComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = PrincipalComparator::Name; @@ -245,7 +288,7 @@ impl<'de> DeserializeArguments<'de> for PrincipalComparator { *self = PrincipalComparator::Type; }, _ => { - *self = PrincipalComparator::_T(key.to_string()); + *self = PrincipalComparator::_T(value.into_owned()); } ); } else { @@ -323,7 +366,8 @@ impl Display for PrincipalFilter { PrincipalFilter::Name(_) => "name", PrincipalFilter::Text(_) => "text", PrincipalFilter::Type(_) => "type", - PrincipalFilter::Timezone(_) => "timezone", + PrincipalFilter::Timezone(_) => "timeZone", + PrincipalFilter::CalendarAddress(_) => "calendarAddress", PrincipalFilter::_T(other) => other, }) } diff --git a/crates/jmap-proto/src/object/push_subscription.rs b/crates/jmap-proto/src/object/push_subscription.rs index 6137be54..cd7d7706 100644 --- a/crates/jmap-proto/src/object/push_subscription.rs +++ b/crates/jmap-proto/src/object/push_subscription.rs @@ -8,10 +8,11 @@ use crate::object::email::{EmailProperty, HeaderForm, HeaderProperty}; use crate::object::{AnyId, JmapObject, JmapObjectId}; use crate::types::date::UTCDate; use jmap_tools::{Element, JsonPointer, JsonPointerItem}; -use jmap_tools::{Key, Property}; +use jmap_tools::{Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::borrow::Cow; use std::str::FromStr; -use types::{id::Id, type_state::DataType}; +use types::{id::Id, text::Text, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct PushSubscription; @@ -42,11 +43,41 @@ pub enum PushSubscriptionValue { impl Property for PushSubscriptionProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - PushSubscriptionProperty::parse(value, key.is_none()) + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + PushSubscriptionProperty::parse(value, key.is_none().then_some(depth)) } fn to_cow(&self) -> Cow<'static, str> { - match self { + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other + } + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl PushSubscriptionProperty { + pub fn text(&self) -> Text<'_> { + Text::Static(match self { PushSubscriptionProperty::DeviceClientId => "deviceClientId", PushSubscriptionProperty::Expires => "expires", PushSubscriptionProperty::Id => "id", @@ -57,34 +88,33 @@ impl Property for PushSubscriptionProperty { PushSubscriptionProperty::VerificationCode => "verificationCode", PushSubscriptionProperty::P256dh => "p256dh", PushSubscriptionProperty::Auth => "auth", - PushSubscriptionProperty::Pointer(json_pointer) => { - return json_pointer.to_string().into(); - } - } - .into() + PushSubscriptionProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + }) } -} -impl PushSubscriptionProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => PushSubscriptionProperty::Id, - b"deviceClientId" => PushSubscriptionProperty::DeviceClientId, - b"url" => PushSubscriptionProperty::Url, - b"keys" => PushSubscriptionProperty::Keys, - b"p256dh" => PushSubscriptionProperty::P256dh, - b"auth" => PushSubscriptionProperty::Auth, - b"verificationCode" => PushSubscriptionProperty::VerificationCode, - b"expires" => PushSubscriptionProperty::Expires, - b"types" => PushSubscriptionProperty::Types, - b"emailPush" => PushSubscriptionProperty::EmailPush, + fn has_dynamic_text(&self) -> bool { + matches!(self, PushSubscriptionProperty::Pointer(_)) + } + + fn parse(value: &str, patch_depth: Option) -> Option { + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(PushSubscriptionProperty::Id), + b"deviceClientId" => Some(PushSubscriptionProperty::DeviceClientId), + b"url" => Some(PushSubscriptionProperty::Url), + b"keys" => Some(PushSubscriptionProperty::Keys), + b"p256dh" => Some(PushSubscriptionProperty::P256dh), + b"auth" => Some(PushSubscriptionProperty::Auth), + b"verificationCode" => Some(PushSubscriptionProperty::VerificationCode), + b"expires" => Some(PushSubscriptionProperty::Expires), + b"types" => Some(PushSubscriptionProperty::Types), + b"emailPush" => Some(PushSubscriptionProperty::EmailPush), + _ => None, ) .or_else(|| { - if allow_patch && value.contains('/') { - PushSubscriptionProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(PushSubscriptionProperty::Pointer) }) } @@ -122,10 +152,23 @@ impl Element for PushSubscriptionValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - PushSubscriptionValue::Id(id) => id.to_string().into(), - PushSubscriptionValue::Date(utcdate) => utcdate.to_string().into(), - PushSubscriptionValue::Types(data_type) => data_type.as_str().into(), + PushSubscriptionValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl PushSubscriptionValue { + pub fn text(&self) -> Text<'_> { + match self { + PushSubscriptionValue::Id(id) => Text::Id(*id), + PushSubscriptionValue::Date(utcdate) => Text::Display(utcdate), + PushSubscriptionValue::Types(data_type) => Text::Static(data_type.as_str()), } } } @@ -134,7 +177,7 @@ impl FromStr for PushSubscriptionProperty { type Err = (); fn from_str(s: &str) -> Result { - PushSubscriptionProperty::parse(s, false).ok_or(()) + PushSubscriptionProperty::parse(s, None).ok_or(()) } } @@ -266,6 +309,7 @@ impl From<&EmailPushProperty> for EmailProperty { form: (&header.form).into(), header: header.header.clone(), all: header.all, + as_raw: false, }), } } @@ -304,6 +348,7 @@ impl From<&ArchivedEmailPushProperty> for EmailProperty { form: (&header.form).into(), header: header.header.as_str().to_string(), all: header.all, + as_raw: false, }), } } diff --git a/crates/jmap-proto/src/object/quota.rs b/crates/jmap-proto/src/object/quota.rs index 172294e4..d4963296 100644 --- a/crates/jmap-proto/src/object/quota.rs +++ b/crates/jmap-proto/src/object/quota.rs @@ -6,11 +6,12 @@ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, - request::deserialize::DeserializeArguments, + request::deserialize::{CowStr, DeserializeArguments}, }; use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{id::Id, type_state::DataType}; +use types::{id::Id, text::Text, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct Quota; @@ -55,21 +56,26 @@ impl Property for QuotaProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl QuotaProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => QuotaProperty::Id, - b"resourceType" => QuotaProperty::ResourceType, - b"used" => QuotaProperty::Used, - b"name" => QuotaProperty::Name, - b"scope" => QuotaProperty::Scope, - b"types" => QuotaProperty::Types, - b"hardLimit" => QuotaProperty::HardLimit, - b"warnLimit" => QuotaProperty::WarnLimit, - b"softLimit" => QuotaProperty::SoftLimit, - b"description" => QuotaProperty::Description, + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(QuotaProperty::Id), + b"resourceType" => Some(QuotaProperty::ResourceType), + b"used" => Some(QuotaProperty::Used), + b"name" => Some(QuotaProperty::Name), + b"scope" => Some(QuotaProperty::Scope), + b"types" => Some(QuotaProperty::Types), + b"hardLimit" => Some(QuotaProperty::HardLimit), + b"warnLimit" => Some(QuotaProperty::WarnLimit), + b"softLimit" => Some(QuotaProperty::SoftLimit), + b"description" => Some(QuotaProperty::Description), + _ => None, ) } } @@ -90,9 +96,19 @@ impl Element for QuotaValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl QuotaValue { + pub fn text(&self) -> Text<'_> { match self { - QuotaValue::Id(id) => id.to_string().into(), - QuotaValue::Types(data_type) => data_type.as_str().into(), + QuotaValue::Id(id) => Text::Id(*id), + QuotaValue::Types(data_type) => Text::Static(data_type.as_str()), } } } @@ -180,7 +196,7 @@ impl<'de> DeserializeArguments<'de> for QuotaComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = QuotaComparator::Name; @@ -192,7 +208,7 @@ impl<'de> DeserializeArguments<'de> for QuotaComparator { *self = QuotaComparator::Used; }, _ => { - *self = QuotaComparator::_T(key.to_string()); + *self = QuotaComparator::_T(value.into_owned()); } ); } else { diff --git a/crates/jmap-proto/src/object/registry.rs b/crates/jmap-proto/src/object/registry.rs index e4dc73e8..be49976f 100644 --- a/crates/jmap-proto/src/object/registry.rs +++ b/crates/jmap-proto/src/object/registry.rs @@ -6,10 +6,9 @@ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, - request::deserialize::DeserializeArguments, + request::deserialize::{CowStr, DeserializeArguments}, }; use registry::{jmap::RegistryValue, schema::prelude::Property, types::EnumImpl}; -use std::borrow::Cow; use types::id::Id; #[derive(Debug)] @@ -161,7 +160,7 @@ impl<'de> DeserializeArguments<'de> for RegistryComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; if let Some(property) = Property::parse(value.as_ref()) { *self = RegistryComparator::Property(property); @@ -178,12 +177,13 @@ impl<'de> DeserializeArguments<'de> for RegistryComparator { impl RegistryFilterOperator { pub fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), RegistryFilterOperator, b"GreaterThan" => RegistryFilterOperator::GreaterThan, b"GreaterThanOrEqual" => RegistryFilterOperator::GreaterThanOrEqual, b"LessThan" => RegistryFilterOperator::LessThan, b"LessThanOrEqual" => RegistryFilterOperator::LessThanOrEqual, ) + .copied() } } diff --git a/crates/jmap-proto/src/object/search_snippet.rs b/crates/jmap-proto/src/object/search_snippet.rs index 0da60e7f..49dc26e8 100644 --- a/crates/jmap-proto/src/object/search_snippet.rs +++ b/crates/jmap-proto/src/object/search_snippet.rs @@ -5,8 +5,9 @@ */ use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct SearchSnippet; @@ -36,6 +37,10 @@ impl Property for SearchSnippetProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for SearchSnippetValue { @@ -55,18 +60,29 @@ impl Element for SearchSnippetValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl SearchSnippetValue { + pub fn text(&self) -> Text<'_> { match self { - SearchSnippetValue::Id(id) => id.to_string().into(), + SearchSnippetValue::Id(id) => Text::Id(*id), } } } impl SearchSnippetProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"emailId" => SearchSnippetProperty::EmailId, - b"subject" => SearchSnippetProperty::Subject, - b"preview" => SearchSnippetProperty::Preview, + hashify::fnc_map!(value.as_bytes(), + b"emailId" => Some(SearchSnippetProperty::EmailId), + b"subject" => Some(SearchSnippetProperty::Subject), + b"preview" => Some(SearchSnippetProperty::Preview), + _ => None, ) } } diff --git a/crates/jmap-proto/src/object/share_notification.rs b/crates/jmap-proto/src/object/share_notification.rs index afc2944f..73fea1a2 100644 --- a/crates/jmap-proto/src/object/share_notification.rs +++ b/crates/jmap-proto/src/object/share_notification.rs @@ -6,12 +6,13 @@ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, - request::deserialize::DeserializeArguments, + request::deserialize::{CowStr, DeserializeArguments}, types::date::UTCDate, }; use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::{id::Id, type_state::DataType}; +use types::{id::Id, text::Text, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct ShareNotification; @@ -61,6 +62,17 @@ impl Property for ShareNotificationProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + || matches!( + (self, other), + ( + ShareNotificationProperty::Name | ShareNotificationProperty::ChangedByName, + ShareNotificationProperty::Name | ShareNotificationProperty::ChangedByName + ) + ) + } } impl Element for ShareNotificationValue { @@ -86,28 +98,42 @@ impl Element for ShareNotificationValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - ShareNotificationValue::Id(id) => id.to_string().into(), - ShareNotificationValue::Date(date) => date.to_string().into(), - ShareNotificationValue::ObjectType(ty) => ty.as_str().into(), + ShareNotificationValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl ShareNotificationValue { + pub fn text(&self) -> Text<'_> { + match self { + ShareNotificationValue::Id(id) => Text::Id(*id), + ShareNotificationValue::Date(date) => Text::Display(date), + ShareNotificationValue::ObjectType(ty) => Text::Static(ty.as_str()), } } } impl ShareNotificationProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => ShareNotificationProperty::Id, - b"created" => ShareNotificationProperty::Created, - b"changedBy" => ShareNotificationProperty::ChangedBy, - b"name" => ShareNotificationProperty::ChangedByName, - b"email" => ShareNotificationProperty::ChangedByEmail, - b"principalId" => ShareNotificationProperty::ChangedByPrincipalId, - b"objectType" => ShareNotificationProperty::ObjectType, - b"objectAccountId" => ShareNotificationProperty::ObjectAccountId, - b"objectId" => ShareNotificationProperty::ObjectId, - b"oldRights" => ShareNotificationProperty::OldRights, - b"newRights" => ShareNotificationProperty::NewRights + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(ShareNotificationProperty::Id), + b"created" => Some(ShareNotificationProperty::Created), + b"changedBy" => Some(ShareNotificationProperty::ChangedBy), + b"name" => Some(ShareNotificationProperty::ChangedByName), + b"email" => Some(ShareNotificationProperty::ChangedByEmail), + b"principalId" => Some(ShareNotificationProperty::ChangedByPrincipalId), + b"objectType" => Some(ShareNotificationProperty::ObjectType), + b"objectAccountId" => Some(ShareNotificationProperty::ObjectAccountId), + b"objectId" => Some(ShareNotificationProperty::ObjectId), + b"oldRights" => Some(ShareNotificationProperty::OldRights), + b"newRights" => Some(ShareNotificationProperty::NewRights), + _ => None ) } } @@ -192,13 +218,13 @@ impl<'de> DeserializeArguments<'de> for ShareNotificationComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = ShareNotificationComparator::Created; }, _ => { - *self = ShareNotificationComparator::_T(value.to_string()); + *self = ShareNotificationComparator::_T(value.into_owned()); } ); } else { diff --git a/crates/jmap-proto/src/object/sieve.rs b/crates/jmap-proto/src/object/sieve.rs index adaf835c..24c3e963 100644 --- a/crates/jmap-proto/src/object/sieve.rs +++ b/crates/jmap-proto/src/object/sieve.rs @@ -5,12 +5,17 @@ */ use crate::{ - object::{AnyId, DeserializeArguments, JmapObject, JmapObjectId, MaybeReference, parse_ref}, - request::reference::MaybeIdReference, + object::{ + AnyId, DeserializeArguments, JmapObject, JmapObjectId, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, + }, + request::{deserialize::CowStr, reference::MaybeIdReference}, }; -use jmap_tools::{Element, Key, Property}; +use jmap_tools::{Element, JsonPointer, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{blob::BlobId, id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct Sieve; @@ -21,6 +26,9 @@ pub enum SieveProperty { Name, BlobId, IsActive, + Metadata, + PrivateMetadata, + Pointer(JsonPointer), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -31,18 +39,43 @@ pub enum SieveValue { } impl Property for SieveProperty { - fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { - SieveProperty::parse(value) + fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(_) => SieveProperty::parse_nested_name(value), + None => SieveProperty::parse(value, Some(depth)), + } + } + + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() } fn to_cow(&self) -> Cow<'static, str> { - match self { - SieveProperty::BlobId => "blobId", - SieveProperty::Id => "id", - SieveProperty::Name => "name", - SieveProperty::IsActive => "isActive", + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -70,22 +103,80 @@ impl Element for SieveValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl SieveValue { + pub fn text(&self) -> Text<'_> { match self { - SieveValue::Id(id) => id.to_string().into(), - SieveValue::BlobId(blob_id) => blob_id.to_string().into(), - SieveValue::IdReference(r) => format!("#{r}").into(), + SieveValue::Id(id) => Text::Id(*id), + SieveValue::BlobId(blob_id) => Text::Display(blob_id), + SieveValue::IdReference(r) => Text::Reference(r), } } } impl SieveProperty { - fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => SieveProperty::Id, - b"name" => SieveProperty::Name, - b"blobId" => SieveProperty::BlobId, - b"isActive" => SieveProperty::IsActive, - ) + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + SieveProperty::BlobId => "blobId", + SieveProperty::Id => "id", + SieveProperty::Name => "name", + SieveProperty::IsActive => "isActive", + SieveProperty::Metadata => "metadata", + SieveProperty::PrivateMetadata => "privateMetadata", + SieveProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!(self, SieveProperty::Pointer(_)) + } + + property_names!( + SieveProperty, + "id" => SieveProperty::Id, + "name" => SieveProperty::Name, + "blobId" => SieveProperty::BlobId, + "isActive" => SieveProperty::IsActive, + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + SieveProperty::parse_name(value).or_else(|| { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(SieveProperty::Pointer) + }) + } +} + +impl MetadataProperty for SieveProperty { + fn as_metadata_root(&self) -> Option { + match self { + SieveProperty::Metadata => Some(MetadataRoot::Shared), + SieveProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + SieveProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => SieveProperty::Metadata, + MetadataRoot::Private => SieveProperty::PrivateMetadata, + } } } @@ -120,7 +211,12 @@ impl FromStr for SieveProperty { type Err = (); fn from_str(s: &str) -> Result { - SieveProperty::parse(s).ok_or(()) + SieveProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| SieveProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) } } @@ -152,6 +248,7 @@ impl JmapObject for Sieve { pub enum SieveFilter { Name(String), IsActive(bool), + Metadata(MetadataFilter), _T(String), } @@ -175,8 +272,13 @@ impl<'de> DeserializeArguments<'de> for SieveFilter { *self = SieveFilter::IsActive(map.next_value()?); }, _ => { - *self = SieveFilter::_T(key.to_string()); - let _ = map.next_value::()?; + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => SieveFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + SieveFilter::_T(key.to_string()) + } + }; } ); @@ -190,7 +292,7 @@ impl<'de> DeserializeArguments<'de> for SieveComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = SieveComparator::Name; @@ -199,7 +301,7 @@ impl<'de> DeserializeArguments<'de> for SieveComparator { *self = SieveComparator::IsActive; }, _ => { - *self = SieveComparator::_T(key.to_string()); + *self = SieveComparator::_T(value.into_owned()); } ); } else { diff --git a/crates/jmap-proto/src/object/thread.rs b/crates/jmap-proto/src/object/thread.rs index 4815a842..22568033 100644 --- a/crates/jmap-proto/src/object/thread.rs +++ b/crates/jmap-proto/src/object/thread.rs @@ -5,8 +5,9 @@ */ use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; use crate::object::{AnyId, JmapObject, JmapObjectId}; @@ -36,6 +37,10 @@ impl Property for ThreadProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for ThreadValue { @@ -50,17 +55,28 @@ impl Element for ThreadValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl ThreadValue { + pub fn text(&self) -> Text<'_> { match self { - ThreadValue::Id(id) => id.to_string().into(), + ThreadValue::Id(id) => Text::Id(*id), } } } impl ThreadProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => ThreadProperty::Id, - b"emailIds" => ThreadProperty::EmailIds, + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(ThreadProperty::Id), + b"emailIds" => Some(ThreadProperty::EmailIds), + _ => None, ) } } diff --git a/crates/jmap-proto/src/object/vacation_response.rs b/crates/jmap-proto/src/object/vacation_response.rs index be1a9ffb..09a57037 100644 --- a/crates/jmap-proto/src/object/vacation_response.rs +++ b/crates/jmap-proto/src/object/vacation_response.rs @@ -9,8 +9,9 @@ use crate::{ types::date::UTCDate, }; use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::id::Id; +use types::{id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct VacationResponse; @@ -49,6 +50,10 @@ impl Property for VacationResponseProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for VacationResponseValue { @@ -73,23 +78,37 @@ impl Element for VacationResponseValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - VacationResponseValue::Id(id) => id.to_string().into(), - VacationResponseValue::Date(utcdate) => utcdate.to_string().into(), + VacationResponseValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl VacationResponseValue { + pub fn text(&self) -> Text<'_> { + match self { + VacationResponseValue::Id(id) => Text::Id(*id), + VacationResponseValue::Date(utcdate) => Text::Display(utcdate), } } } impl VacationResponseProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => VacationResponseProperty::Id, - b"isEnabled" => VacationResponseProperty::IsEnabled, - b"fromDate" => VacationResponseProperty::FromDate, - b"toDate" => VacationResponseProperty::ToDate, - b"textBody" => VacationResponseProperty::TextBody, - b"htmlBody" => VacationResponseProperty::HtmlBody, - b"subject" => VacationResponseProperty::Subject, + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(VacationResponseProperty::Id), + b"isEnabled" => Some(VacationResponseProperty::IsEnabled), + b"fromDate" => Some(VacationResponseProperty::FromDate), + b"toDate" => Some(VacationResponseProperty::ToDate), + b"textBody" => Some(VacationResponseProperty::TextBody), + b"htmlBody" => Some(VacationResponseProperty::HtmlBody), + b"subject" => Some(VacationResponseProperty::Subject), + _ => None, ) } } diff --git a/crates/jmap-proto/src/references/eval.rs b/crates/jmap-proto/src/references/eval.rs index c3cb663e..401c9f52 100644 --- a/crates/jmap-proto/src/references/eval.rs +++ b/crates/jmap-proto/src/references/eval.rs @@ -11,7 +11,10 @@ use crate::{ jsptr::{EvalResults, ResponsePtr}, }, request::reference::ResultReference, - response::{ChangesResponseMethod, GetResponseMethod, Response, ResponseMethod}, + response::{ + ChangesResponseMethod, CopyResponseMethod, GetResponseMethod, Response, ResponseMethod, + SetResponseMethod, + }, }; use compact_str::format_compact; use jmap_tools::{Element, Key, Property, Value}; @@ -128,11 +131,80 @@ impl Response<'_> { ChangesResponseMethod::ShareNotification(response) => { response.eval_jptr(path, &mut results) } + ChangesResponseMethod::ParticipantIdentity(response) => { + response.eval_jptr(path, &mut results) + } }, ResponseMethod::Query(response) => response.eval_jptr(path, &mut results), ResponseMethod::QueryChanges(response) => { response.eval_jptr(path, &mut results) } + ResponseMethod::Set(response) => match response { + SetResponseMethod::Email(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::Mailbox(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::Identity(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::EmailSubmission(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::PushSubscription(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::Sieve(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::VacationResponse(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::AddressBook(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::ContactCard(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::FileNode(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::ShareNotification(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::Calendar(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::CalendarEvent(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::CalendarEventNotification(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::ParticipantIdentity(response) => { + response.eval_jptr(path, &mut results) + } + SetResponseMethod::Registry(response) => { + response.eval_jptr(path, &mut results) + } + }, + ResponseMethod::Copy(response) => match response { + CopyResponseMethod::Email(response) => { + response.eval_jptr(path, &mut results) + } + CopyResponseMethod::ContactCard(response) => { + response.eval_jptr(path, &mut results) + } + CopyResponseMethod::CalendarEvent(response) => { + response.eval_jptr(path, &mut results) + } + CopyResponseMethod::FileNode(response) => { + response.eval_jptr(path, &mut results) + } + CopyResponseMethod::Blob(_) => false, + }, + ResponseMethod::ImportEmail(response) => response.eval_jptr(path, &mut results), _ => false, }; @@ -206,10 +278,13 @@ where .details("Id reference points to invalid type.")); } } else if let Graph::Some { child_id, graph } = graph { - graph - .entry(child_id.to_string()) - .or_insert_with(Vec::new) - .push(id_ref.to_string()); + let id_ref = id_ref.to_string(); + match graph.get_mut(*child_id) { + Some(parents) => parents.push(id_ref), + None => { + graph.insert(child_id.to_string(), vec![id_ref]); + } + } } else { return Err(trc::JmapEvent::InvalidResultReference .into_err() diff --git a/crates/jmap-proto/src/references/jsptr.rs b/crates/jmap-proto/src/references/jsptr.rs index ed2eeb1b..75df223c 100644 --- a/crates/jmap-proto/src/references/jsptr.rs +++ b/crates/jmap-proto/src/references/jsptr.rs @@ -9,9 +9,12 @@ use crate::{ PropertyWrapper, availability::{BusyPeriod, GetAvailabilityResponse}, changes::ChangesResponse, + copy::CopyResponse, get::GetResponse, + import::ImportEmailResponse, query::QueryResponse, query_changes::{AddedItem, QueryChangesResponse}, + set::SetResponse, }, object::{ AnyId, JmapObject, JmapObjectId, @@ -105,6 +108,7 @@ where } _ => {} }, + Some(JsonPointerItem::Invalid(_)) => return false, Some(JsonPointerItem::Root) | None => match self { Value::Element(e) => { if let Some(id) = e.as_any_id() { @@ -128,6 +132,41 @@ where } } +impl ResponsePtr for Option { + fn eval_jptr(&self, pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { + if let Some(value) = self { + value.eval_jptr(pointer, results); + } + true + } +} + +fn eval_map_jptr<'a, V: ResponsePtr + 'a>( + mut pointer: JsonPointerIter<'_, Null>, + results: &mut EvalResults, + get: impl FnOnce(&str) -> Option<&'a V>, + values: impl Iterator, +) { + match pointer.next() { + Some(JsonPointerItem::Key(key)) => { + if let Some(value) = key.as_string_key().and_then(get) { + value.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Number(n)) => { + if let Some(value) = get(&n.to_string()) { + value.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Wildcard) => { + for value in values { + value.eval_jptr(pointer.clone(), results); + } + } + _ => {} + } +} + impl ResponsePtr for Id { fn eval_jptr(&self, _pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { results.0.push(EvalResult::Id(AnyId::Id(*self))); @@ -188,6 +227,70 @@ impl ResponsePtr for ChangesResponse { } } +impl ResponsePtr for SetResponse { + fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { + match pointer.next().and_then(|item| item.as_string_key()) { + Some("created") => { + eval_map_jptr( + pointer, + results, + |key| self.created.get(key), + self.created.values(), + ); + true + } + Some("updated") => { + eval_map_jptr( + pointer, + results, + |key| Id::from_str(key).ok().and_then(|id| self.updated.get(&id)), + self.updated.values(), + ); + true + } + Some("destroyed") => { + self.destroyed.eval_jptr(pointer, results); + true + } + _ => false, + } + } +} + +impl ResponsePtr for CopyResponse { + fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { + match pointer.next().and_then(|item| item.as_string_key()) { + Some("created") => { + eval_map_jptr( + pointer, + results, + |key| self.created.get(key), + self.created.values(), + ); + true + } + _ => false, + } + } +} + +impl ResponsePtr for ImportEmailResponse { + fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { + match pointer.next().and_then(|item| item.as_string_key()) { + Some("created") => { + eval_map_jptr( + pointer, + results, + |key| self.created.get(key), + self.created.values(), + ); + true + } + _ => false, + } + } +} + impl ResponsePtr for QueryResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { @@ -244,13 +347,13 @@ impl ResponsePtr for CalendarEventNotificationObject { true } Some("calendarEventId") => { - if let Some(id) = &self.calendar_event_id { + if let Some(id) = self.calendar_event_id.value() { results.0.push(EvalResult::Id(AnyId::Id(*id))); } true } Some("event") => { - if let Some(event) = &self.event { + if let Some(event) = self.event.value() { event.0.eval_jptr(pointer, results); } true diff --git a/crates/jmap-proto/src/references/mod.rs b/crates/jmap-proto/src/references/mod.rs index f4fa820f..1fde9c49 100644 --- a/crates/jmap-proto/src/references/mod.rs +++ b/crates/jmap-proto/src/references/mod.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use ahash::{AHashMap, AHashSet}; use compact_str::format_compact; -use std::collections::HashMap; use utils::map::vec_map::VecMap; pub mod eval; @@ -15,14 +15,14 @@ pub mod resolve; pub(crate) enum Graph<'x> { Some { child_id: &'x str, - graph: &'x mut HashMap>, + graph: &'x mut AHashMap>, }, None, } fn topological_sort( create: &mut VecMap, - graph: HashMap>, + graph: AHashMap>, ) -> trc::Result> { // Make sure all references exist for (from_id, to_ids) in graph.iter() { @@ -38,39 +38,47 @@ fn topological_sort( } let mut sorted_create = VecMap::with_capacity(create.len()); + let mut placed = AHashSet::with_capacity(create.len()); let mut it_stack = Vec::new(); - let keys = graph.keys().cloned().collect::>(); - let mut it = keys.iter(); - 'main: loop { - while let Some(from_id) = it.next() { - if let Some(to_ids) = graph.get(from_id) { - it_stack.push((it, from_id)); - if it_stack.len() > 1000 { - return Err(trc::JmapEvent::InvalidArguments - .into_err() - .details("Cyclical references are not allowed.")); + 'main: for root in graph.keys() { + let mut it = std::slice::from_ref(root).iter(); + loop { + while let Some(from_id) = it.next() { + if placed.contains(from_id.as_str()) { + continue; } - it = to_ids.iter(); - continue; - } else if let Some((id, value)) = create.remove_entry(from_id) { - sorted_create.append(id, value); - if create.is_empty() { - break 'main; + if let Some(to_ids) = graph.get(from_id) { + it_stack.push((it, from_id)); + if it_stack.len() > 1000 { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details("Cyclical references are not allowed.")); + } + it = to_ids.iter(); + } else if placed.insert(from_id.as_str()) + && let Some((id, value)) = create.remove_entry(from_id) + { + sorted_create.append(id, value); + if create.is_empty() { + break 'main; + } } } - } - if let Some((prev_it, from_id)) = it_stack.pop() { - it = prev_it; - if let Some((id, value)) = create.remove_entry(from_id) { - sorted_create.append(id, value); - if create.is_empty() { - break 'main; + if let Some((prev_it, from_id)) = it_stack.pop() { + it = prev_it; + if placed.insert(from_id.as_str()) + && let Some((id, value)) = create.remove_entry(from_id) + { + sorted_create.append(id, value); + if create.is_empty() { + break 'main; + } } + } else { + break; } - } else { - break; } } @@ -83,862 +91,3 @@ fn topological_sort( Ok(sorted_create) } -#[cfg(test)] -mod tests { - use crate::references::Graph; - use crate::references::eval::EvalObjectReferences; - use crate::{ - method::{changes::ChangesResponse, get::GetResponse, query::QueryResponse}, - object::{ - email::{EmailProperty, EmailValue}, - mailbox::{MailboxProperty, MailboxValue}, - thread::{ThreadProperty, ThreadValue}, - }, - request::{ - Call, GetRequestMethod, Request, RequestMethod, SetRequestMethod, - reference::{MaybeIdReference, MaybeResultReference}, - }, - response::{ChangesResponseMethod, GetResponseMethod, Response, ResponseMethod}, - }; - use jmap_tools::{Key, Map, Value}; - use std::collections::HashMap; - use types::id::Id; - - #[test] - fn eval_value_references() { - let request = Request::parse( - br##"{ - "using":["urn:ietf:params:jmap:mail"], - "methodCalls": [[ "Email/query", { - "accountId": "a", - "filter": { "inMailbox": "a" }, - "sort": [{ "property": "receivedAt", "isAscending": false }], - "collapseThreads": true, - "position": 0, - "limit": 10, - "calculateTotal": true - }, "t0" ], - [ "Email/get", { - "accountId": "a", - "#ids": { - "resultOf": "t0", - "name": "Email/query", - "path": "/ids" - }, - "properties": [ "threadId" ] - }, "t1" ], - [ "Thread/get", { - "accountId": "a", - "#ids": { - "resultOf": "t1", - "name": "Email/get", - "path": "/list/*/threadId" - } - }, "t2" ], - [ "Email/get", { - "accountId": "a", - "#ids": { - "resultOf": "t2", - "name": "Thread/get", - "path": "/list/*/emailIds" - }, - "properties": [ "from", "receivedAt", "subject" ] - }, "t3" ]] - }"##, - 100, - 1024 * 1024, - ) - .unwrap(); - - let mut response = Response::new( - 1234, - request.created_ids.unwrap_or_default(), - request.method_calls.len(), - ); - - assert_eq!(request.method_calls.len(), 4); - - for (test_num, mut call) in request.method_calls.into_iter().enumerate() { - match test_num { - 0 => { - response.method_responses.push(Call { - id: call.id, - name: call.name, - method: ResponseMethod::Query(QueryResponse { - account_id: Id::new(1), - query_state: Default::default(), - can_calculate_changes: Default::default(), - position: Default::default(), - ids: vec![Id::new(4), Id::new(5)], - total: Default::default(), - limit: Default::default(), - }), - }); - } - 1 => { - response.resolve_references(&mut call.method).unwrap(); - match call.method { - RequestMethod::Get(GetRequestMethod::Email(req)) => { - assert_eq!( - req.ids, - Some(MaybeResultReference::Value(vec![ - MaybeIdReference::Id(Id::new(4)), - MaybeIdReference::Id(Id::new(5)) - ])) - ); - } - _ => panic!("Expected Email Get Request"), - } - response.method_responses.push(Call { - id: call.id, - name: call.name, - method: ResponseMethod::Get(GetResponseMethod::Email(GetResponse { - account_id: Id::new(1).into(), - state: Default::default(), - list: vec![ - Value::Object(Map::from(vec![( - Key::Property(EmailProperty::ThreadId), - Value::Element(EmailValue::Id(Id::new(9))), - )])), - Value::Object(Map::from(vec![( - Key::Property(EmailProperty::ThreadId), - Value::Element(EmailValue::Id(Id::new(10))), - )])), - ], - not_found: Default::default(), - })), - }); - } - 2 => { - response.resolve_references(&mut call.method).unwrap(); - match call.method { - RequestMethod::Get(GetRequestMethod::Thread(req)) => { - assert_eq!( - req.ids, - Some(MaybeResultReference::Value(vec![ - MaybeIdReference::Id(Id::new(9)), - MaybeIdReference::Id(Id::new(10)) - ])) - ); - } - _ => panic!("Expected Thread Get Request"), - } - response.method_responses.push(Call { - id: call.id, - name: call.name, - method: ResponseMethod::Get(GetResponseMethod::Thread(GetResponse { - account_id: Id::new(1).into(), - state: Default::default(), - list: vec![ - Value::Object(Map::from(vec![( - Key::Property(ThreadProperty::EmailIds), - Value::Array(vec![ - Value::Element(ThreadValue::Id(Id::new(100))), - Value::Element(ThreadValue::Id(Id::new(101))), - ]), - )])), - Value::Object(Map::from(vec![( - Key::Property(ThreadProperty::EmailIds), - Value::Array(vec![ - Value::Element(ThreadValue::Id(Id::new(102))), - Value::Element(ThreadValue::Id(Id::new(103))), - ]), - )])), - ], - not_found: Default::default(), - })), - }); - } - 3 => { - response.resolve_references(&mut call.method).unwrap(); - match call.method { - RequestMethod::Get(GetRequestMethod::Email(req)) => { - assert_eq!( - req.ids, - Some(MaybeResultReference::Value(vec![ - MaybeIdReference::Id(Id::new(100)), - MaybeIdReference::Id(Id::new(101)), - MaybeIdReference::Id(Id::new(102)), - MaybeIdReference::Id(Id::new(103)), - ])) - ); - } - _ => panic!("Expected Mailbox Get Request"), - } - } - _ => panic!("Unexpected invocation {}", test_num), - } - } - } - - #[test] - fn eval_property_references() { - let request = Request::parse( - br##"{ - "using":["urn:ietf:params:jmap:mail"], - "methodCalls": [ - ["Mailbox/changes",{ - "accountId":"s", - "sinceState":"srxqk071myhgkyay" - },"0"], - ["Mailbox/get",{ - "accountId":"s", - "#ids":{"name":"Mailbox/changes","path":"/created","resultOf":"0"} - },"1"], - ["Mailbox/get",{ - "accountId":"s", - "#ids":{"name":"Mailbox/changes","path":"/updated","resultOf":"0"}, - "#properties":{"name":"Mailbox/changes","path":"/updatedProperties","resultOf":"0"} - },"2"] - ] - }"##, - 100, - 1024 * 1024, - ) - .unwrap(); - - let mut response = Response::new( - 1234, - request.created_ids.unwrap_or_default(), - request.method_calls.len(), - ); - - assert_eq!(request.method_calls.len(), 3); - - for (test_num, mut call) in request.method_calls.into_iter().enumerate() { - match test_num { - 0 => { - response.method_responses.push(Call { - id: call.id, - name: call.name, - method: ResponseMethod::Changes(ChangesResponseMethod::Mailbox(Box::new( - ChangesResponse { - account_id: Id::new(1), - old_state: Default::default(), - new_state: Default::default(), - has_more_changes: Default::default(), - created: Default::default(), - updated: vec![Id::new(2), Id::new(3)], - destroyed: Default::default(), - updated_properties: Some(vec![ - MailboxProperty::Name.into(), - MailboxProperty::ParentId.into(), - ]), - }, - ))), - }); - } - 1 => { - response.resolve_references(&mut call.method).unwrap(); - match call.method { - RequestMethod::Get(GetRequestMethod::Mailbox(req)) => { - assert_eq!(req.ids, Some(MaybeResultReference::Value(vec![]))); - } - _ => panic!("Expected Mailbox Get Request"), - } - } - 2 => { - response.resolve_references(&mut call.method).unwrap(); - match call.method { - RequestMethod::Get(GetRequestMethod::Mailbox(req)) => { - assert_eq!( - req.ids, - Some(MaybeResultReference::Value(vec![ - MaybeIdReference::Id(Id::new(2)), - MaybeIdReference::Id(Id::new(3)) - ])) - ); - } - _ => panic!("Expected Mailbox Get Request"), - } - } - _ => panic!("Unexpected invocation {}", test_num), - } - } - } - - #[test] - fn eval_create_references() { - let request = Request::parse( - br##"{ - "using": [ - "urn:ietf:params:jmap:core", - "urn:ietf:params:jmap:mail" - ], - "methodCalls": [ - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "a": { - "name": "Folder a", - "parentId": "#b" - }, - "b": { - "name": "Folder b", - "parentId": "#c" - }, - "c": { - "name": "Folder c", - "parentId": "#d" - }, - "d": { - "name": "Folder d", - "parentId": "#e" - }, - "e": { - "name": "Folder e", - "parentId": "#f" - }, - "f": { - "name": "Folder f", - "parentId": "#g" - }, - "g": { - "name": "Folder g", - "parentId": null - } - } - }, - "fulltree" - ], - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "a1": { - "name": "Folder a1", - "parentId": null - }, - "b2": { - "name": "Folder b2", - "parentId": "#a1" - }, - "c3": { - "name": "Folder c3", - "parentId": "#a1" - }, - "d4": { - "name": "Folder d4", - "parentId": "#b2" - }, - "e5": { - "name": "Folder e5", - "parentId": "#b2" - }, - "f6": { - "name": "Folder f6", - "parentId": "#d4" - }, - "g7": { - "name": "Folder g7", - "parentId": "#e5" - } - } - }, - "fulltree2" - ], - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "z": { - "name": "Folder Z", - "parentId": "#x" - }, - "y": { - "name": null - }, - "x": { - "name": "Folder X" - } - } - }, - "xyz" - ], - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "a": { - "name": "Folder a", - "parentId": "#b" - }, - "b": { - "name": "Folder b", - "parentId": "#c" - }, - "c": { - "name": "Folder c", - "parentId": "#d" - }, - "d": { - "name": "Folder d", - "parentId": "#a" - } - } - }, - "circular" - ] - ] - }"##, - 100, - 1024 * 1024, - ) - .unwrap(); - - let response = Response::new( - 1234, - request.created_ids.unwrap_or_default(), - request.method_calls.len(), - ); - - for (test_num, mut call) in request.method_calls.into_iter().enumerate() { - match response.resolve_references(&mut call.method) { - Ok(_) => assert!( - (0..3).contains(&test_num), - "Unexpected invocation {}", - test_num - ), - Err(err) => { - assert_eq!(test_num, 3); - assert!( - err.matches(trc::EventType::Jmap(trc::JmapEvent::InvalidArguments)), - "{:?}", - err - ); - continue; - } - } - - if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { - if test_num == 0 { - assert_eq!( - request - .create - .unwrap() - .into_iter() - .map(|b| b.0) - .collect::>(), - ["g", "f", "e", "d", "c", "b", "a"] - .iter() - .map(|i| i.to_string()) - .collect::>() - ); - } else if test_num == 1 { - let mut pending_ids = vec!["a1", "b2", "d4", "e5", "f6", "c3", "g7"]; - - for (id, _) in request.create.as_ref().unwrap() { - match id.as_str() { - "a1" => (), - "b2" | "c3" => assert!(!pending_ids.contains(&"a1")), - "d4" | "e5" => assert!(!pending_ids.contains(&"b2")), - "f6" => assert!(!pending_ids.contains(&"d4")), - "g7" => assert!(!pending_ids.contains(&"e5")), - _ => panic!("Unexpected ID"), - } - pending_ids.retain(|i| i != id); - } - - if !pending_ids.is_empty() { - panic!( - "Unexpected order: {:?}", - request - .create - .as_ref() - .unwrap() - .iter() - .map(|b| b.0.to_string()) - .collect::>() - ); - } - } else if test_num == 2 { - assert_eq!( - request - .create - .unwrap() - .into_iter() - .map(|b| b.0) - .collect::>(), - ["x", "z", "y"] - .iter() - .map(|i| i.to_string()) - .collect::>() - ); - } - } else { - panic!("Expected Set Mailbox Request"); - } - } - - let request = Request::parse( - br##"{ - "using": [ - "urn:ietf:params:jmap:core", - "urn:ietf:params:jmap:mail" - ], - "methodCalls": [ - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "a": { - "name": "a", - "parentId": "#x" - }, - "b": { - "name": "b", - "parentId": "#y" - }, - "c": { - "name": "c", - "parentId": "#z" - } - } - }, - "ref1" - ], - [ - "Mailbox/set", - { - "accountId": "b", - "create": { - "a1": { - "name": "a1", - "parentId": "#a" - }, - "b2": { - "name": "b2", - "parentId": "#b" - }, - "c3": { - "name": "c3", - "parentId": "#c" - } - } - }, - "red2" - ] - ], - "createdIds": { - "x": "b", - "y": "c", - "z": "d" - } - }"##, - 1024, - 1024 * 1024, - ) - .unwrap(); - - let mut response = Response::new( - 1234, - request.created_ids.unwrap_or_default(), - request.method_calls.len(), - ); - - let mut invocations = request.method_calls.into_iter(); - let mut call = invocations.next().unwrap(); - response.resolve_references(&mut call.method).unwrap(); - - if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { - let create = request - .create - .as_ref() - .unwrap() - .iter() - .map(|(p, v)| { - ( - p.as_str(), - v.as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(), - ) - }) - .collect::>(); - assert_eq!( - *create.get("a").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(1))) - ); - assert_eq!( - *create.get("b").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(2))) - ); - assert_eq!( - *create.get("c").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(3))) - ); - } else { - panic!("Expected Mailbox Set Request"); - } - - response - .created_ids - .insert("a".to_string(), Id::new(5).into()); - response - .created_ids - .insert("b".to_string(), Id::new(6).into()); - response - .created_ids - .insert("c".to_string(), Id::new(7).into()); - - let mut call = invocations.next().unwrap(); - response.resolve_references(&mut call.method).unwrap(); - - if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { - let create = request - .create - .as_ref() - .unwrap() - .iter() - .map(|(p, v)| { - ( - p.as_str(), - v.as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(), - ) - }) - .collect::>(); - assert_eq!( - *create.get("a1").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(5))) - ); - assert_eq!( - *create.get("b2").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(6))) - ); - assert_eq!( - *create.get("c3").unwrap(), - &Value::Element(MailboxValue::Id(Id::new(7))) - ); - } else { - panic!("Expected Mailbox Set Request"); - } - } - - #[test] - fn eval_nested_element_ref() { - let mut created_ids = HashMap::new(); - created_ids.insert("server-1".to_string(), Id::new(42).into()); - let response = Response::new(0, created_ids, 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![ - ( - Key::Property(MailboxProperty::Name), - Value::Str("inbox".into()), - ), - ( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("server-1".into())), - )])), - ), - ])); - - value - .eval_object_references(&response, &mut Graph::None, 0, 5, true) - .unwrap(); - - let nested = value - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(); - assert_eq!(nested, &Value::Element(MailboxValue::Id(Id::new(42)))); - } - - #[test] - fn eval_array_element_ref() { - let mut created_ids = HashMap::new(); - created_ids.insert("a".to_string(), Id::new(1).into()); - created_ids.insert("b".to_string(), Id::new(2).into()); - let response = Response::new(0, created_ids, 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Array(vec![ - Value::Element(MailboxValue::IdReference("a".into())), - Value::Element(MailboxValue::IdReference("b".into())), - ]), - )])); - - value - .eval_object_references(&response, &mut Graph::None, 0, 5, true) - .unwrap(); - - let arr = value - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_array() - .unwrap(); - assert_eq!(arr.len(), 2); - assert_eq!(arr[0], Value::Element(MailboxValue::Id(Id::new(1)))); - assert_eq!(arr[1], Value::Element(MailboxValue::Id(Id::new(2)))); - } - - #[test] - fn eval_nested_array_of_objects_with_element_ref() { - let mut created_ids = HashMap::new(); - created_ids.insert("a".to_string(), Id::new(7).into()); - let response = Response::new(0, created_ids, 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Array(vec![Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("a".into())), - )]))]), - )])); - - value - .eval_object_references(&response, &mut Graph::None, 0, 5, true) - .unwrap(); - - let resolved = value - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_array() - .unwrap()[0] - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(); - assert_eq!(resolved, &Value::Element(MailboxValue::Id(Id::new(7)))); - } - - #[test] - fn eval_graph_collects_nested_ref() { - let response = Response::new(0, HashMap::new(), 0); - let mut graph_map: HashMap> = HashMap::new(); - let child_id = "outer".to_string(); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("inner".into())), - )])), - )])); - - { - let mut graph = Graph::Some { - child_id: &child_id, - graph: &mut graph_map, - }; - value - .eval_object_references(&response, &mut graph, 0, 5, true) - .unwrap(); - } - - assert_eq!(graph_map.get("outer"), Some(&vec!["inner".to_string()])); - } - - #[test] - fn eval_unresolved_nested_ref_errors_without_graph() { - let response = Response::new(0, HashMap::new(), 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("missing".into())), - )])), - )])); - - let err = value - .eval_object_references(&response, &mut Graph::None, 0, 5, true) - .unwrap_err(); - assert!( - err.matches(trc::EventType::Jmap(trc::JmapEvent::InvalidResultReference)), - "{:?}", - err - ); - } - - #[test] - fn eval_depth_limit_blocks_walk_into_inner_object() { - let mut created_ids = HashMap::new(); - created_ids.insert("inner".to_string(), Id::new(99).into()); - let response = Response::new(0, created_ids, 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("inner".into())), - )])), - )])), - )])); - - value - .eval_object_references(&response, &mut Graph::None, 0, 2, true) - .unwrap(); - - let deepest = value - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(); - assert_eq!( - deepest, - &Value::Element(MailboxValue::IdReference("inner".into())) - ); - } - - #[test] - fn eval_depth_limit_substitutes_element_at_max_depth() { - let mut created_ids = HashMap::new(); - created_ids.insert("inner".to_string(), Id::new(99).into()); - let response = Response::new(0, created_ids, 0); - - let mut value: Value<'_, MailboxProperty, MailboxValue> = Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Object(Map::from(vec![( - Key::Property(MailboxProperty::ParentId), - Value::Element(MailboxValue::IdReference("inner".into())), - )])), - )])); - - value - .eval_object_references(&response, &mut Graph::None, 0, 2, true) - .unwrap(); - - let resolved = value - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap() - .as_object() - .unwrap() - .get(&Key::Property(MailboxProperty::ParentId)) - .unwrap(); - assert_eq!(resolved, &Value::Element(MailboxValue::Id(Id::new(99)))); - } -} diff --git a/crates/jmap-proto/src/references/resolve.rs b/crates/jmap-proto/src/references/resolve.rs index 72b0b230..521441bb 100644 --- a/crates/jmap-proto/src/references/resolve.rs +++ b/crates/jmap-proto/src/references/resolve.rs @@ -24,11 +24,14 @@ use crate::{ }, response::Response, }; +use ahash::AHashMap; use compact_str::format_compact; use jmap_tools::{Element, Key, Property, Value}; -use std::collections::HashMap; use types::id::Id; +const EMBEDDED_BLOB_REFERENCE_DEPTH: usize = 8; +const EMAIL_BODY_REFERENCE_DEPTH: usize = 128; + impl Response<'_> { pub fn resolve_references(&self, request: &mut RequestMethod) -> trc::Result<()> { match request { @@ -60,7 +63,9 @@ impl Response<'_> { GetRequestMethod::Registry(request) => request.resolve_references(self)?, }, RequestMethod::Set(request) => match request { - SetRequestMethod::Email(request) => request.resolve_references(self, 2, false)?, + SetRequestMethod::Email(request) => { + request.resolve_references(self, EMAIL_BODY_REFERENCE_DEPTH, false)? + } SetRequestMethod::Mailbox(request) => request.resolve_references(self, 1, false)?, SetRequestMethod::Identity(request) => { request.resolve_references(self, 1, false)? @@ -79,7 +84,7 @@ impl Response<'_> { request.resolve_references(self, 1, false)? } SetRequestMethod::ContactCard(request) => { - request.resolve_references(self, 1, false)? + request.resolve_references(self, EMBEDDED_BLOB_REFERENCE_DEPTH, false)? } SetRequestMethod::FileNode(request) => { request.resolve_references(self, 1, false)? @@ -91,7 +96,7 @@ impl Response<'_> { request.resolve_references(self, 1, false)? } SetRequestMethod::CalendarEvent(request) => { - request.resolve_references(self, 1, false)? + request.resolve_references(self, EMBEDDED_BLOB_REFERENCE_DEPTH, false)? } SetRequestMethod::CalendarEventNotification(request) => { request.resolve_references(self, 1, false)? @@ -102,15 +107,15 @@ impl Response<'_> { SetRequestMethod::Registry(request) => request.resolve_references(self, 5, true)?, }, RequestMethod::Copy(request) => match request { - CopyRequestMethod::Email(request) => request.resolve_references(self, 1, false)?, + CopyRequestMethod::Email(request) => request.resolve_references(self, 2, false)?, CopyRequestMethod::CalendarEvent(request) => { - request.resolve_references(self, 1, false)? + request.resolve_references(self, EMBEDDED_BLOB_REFERENCE_DEPTH, false)? } CopyRequestMethod::ContactCard(request) => { - request.resolve_references(self, 1, false)? + request.resolve_references(self, EMBEDDED_BLOB_REFERENCE_DEPTH, false)? } CopyRequestMethod::FileNode(request) => { - request.resolve_references(self, 1, false)? + request.resolve_references(self, 2, false)? } CopyRequestMethod::Blob(_) => (), }, @@ -259,7 +264,7 @@ impl<'x, T: JmapObject> ResolveSetReference for SetRequest<'x, T> { ) -> trc::Result<()> { // Resolve create references if let Some(create) = &mut self.create { - let mut graph = HashMap::with_capacity(create.len()); + let mut graph = AHashMap::new(); for (id, obj) in create.iter_mut() { obj.eval_object_references( response, @@ -309,12 +314,8 @@ impl<'x, T: JmapObject> ResolveSetReference for CopyRequest<'x, T> { eval_strings: bool, ) -> trc::Result<()> { // Resolve create references - for (id, obj) in self.create.iter_mut() { + for obj in self.create.values_mut() { obj.eval_object_references(response, &mut Graph::None, 0, max_depth, eval_strings)?; - - if let MaybeIdReference::Reference(ir) = id { - *id = MaybeIdReference::Id(response.eval_id_reference(ir)?); - } } Ok(()) @@ -385,7 +386,7 @@ impl ResolveReference for GetSearchSnippetRequest { impl ResolveReference for BlobUploadRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { - let mut graph = HashMap::with_capacity(self.create.len()); + let mut graph = AHashMap::>::new(); for (create_id, object) in self.create.iter_mut() { for data in &mut object.data { if let DataSourceObject::Id { id, .. } = data @@ -403,10 +404,13 @@ impl ResolveReference for BlobUploadRequest { )); } None => { - graph - .entry(create_id.to_string()) - .or_insert_with(Vec::new) - .push(parent_id.to_string()); + let parent_id = parent_id.to_string(); + match graph.get_mut(create_id.as_str()) { + Some(parents) => parents.push(parent_id), + None => { + graph.insert(create_id.to_string(), vec![parent_id]); + } + } } } } diff --git a/crates/jmap-proto/src/request/capability.rs b/crates/jmap-proto/src/request/capability.rs index 7a5de676..f7e909a6 100644 --- a/crates/jmap-proto/src/request/capability.rs +++ b/crates/jmap-proto/src/request/capability.rs @@ -7,13 +7,20 @@ use std::fmt; use crate::{ - object::{email::EmailComparator, file_node::FileNodeComparator}, + object::{ + email::EmailComparator, + file_node::{FileNodeComparator, FileNodeNodeType}, + }, response::serialize::serialize_hex, types::date::UTCDate, }; use ahash::AHashMap; use serde::{Deserialize, Deserializer}; -use types::{id::Id, type_state::DataType}; +use types::{ + id::Id, + metadata::{NamespaceScope, RegisteredNamespace}, + type_state::DataType, +}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, serde::Serialize)] @@ -95,6 +102,8 @@ pub enum Capability { WebPushVapid = 1 << 18, #[serde(rename(serialize = "urn:ietf:params:jmap:emailpush"))] EmailPush = 1 << 19, + #[serde(rename(serialize = "urn:ietf:params:jmap:metadata"))] + Metadata = 1 << 20, } #[derive(Debug, Clone, Copy, Default)] @@ -107,6 +116,15 @@ impl CapabilityIds { } } +impl FromIterator for CapabilityIds { + fn from_iter>(iter: T) -> Self { + CapabilityIds( + iter.into_iter() + .fold(0, |ids, capability| ids | capability as u32), + ) + } +} + #[derive(Debug, Clone, serde::Serialize)] #[serde(untagged)] #[allow(dead_code)] @@ -120,10 +138,12 @@ pub enum Capabilities { Blob(BlobCapabilities), Contacts(ContactsCapabilities), Principals(PrincipalCapabilities), + PrincipalsOwner(PrincipalOwnerCapabilities), PrincipalsAvailability(PrincipalAvailabilityCapabilities), Calendar(CalendarCapabilities), FileNode(FileNodeCapabilities), WebPush(WebPushCapabilities), + Metadata(MetadataCapabilities), Empty(EmptyCapabilities), } @@ -251,27 +271,14 @@ pub struct PrincipalCapabilities { pub current_user_principal_id: Option, } -/*#[derive(Debug, Clone, serde::Serialize)] +#[derive(Debug, Clone, serde::Serialize)] pub struct PrincipalOwnerCapabilities { #[serde(rename(serialize = "accountIdForPrincipal"))] pub account_id_for_principal: Id, - #[serde(rename(serialize = "principalId"))] pub principal_id: Id, } -#[derive(Debug, Clone, serde::Serialize)] -pub struct PrincipalCalendarCapabilities { - #[serde(rename(serialize = "accountIdForPrincipal"))] - pub account_id_for_principal: Option, - #[serde(rename(serialize = "mayGetAvailability"))] - pub may_get_availability: bool, - #[serde(rename(serialize = "mayShareWith"))] - pub may_share_with: bool, - #[serde(rename(serialize = "calendarAddress"))] - pub calendar_address: String, -}*/ - #[derive(Debug, Clone, serde::Serialize)] pub struct FileNodeCapabilities { #[serde(rename(serialize = "maxFileNodeDepth"))] @@ -284,6 +291,8 @@ pub struct FileNodeCapabilities { pub forbidden_node_names: Option>, #[serde(rename(serialize = "fileNodeQuerySortOptions"))] pub file_node_query_sort_options: Vec, + #[serde(rename(serialize = "supportedNodeTypes"))] + pub supported_node_types: &'static [FileNodeNodeType], #[serde(rename(serialize = "mayCreateTopLevelFileNode"))] pub may_create_top_level_file_node: bool, #[serde(rename(serialize = "caseInsensitiveNames"))] @@ -302,6 +311,24 @@ pub struct WebPushCapabilities { pub application_server_key: String, } +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct MetadataCapabilities { + #[serde(rename(serialize = "dataTypes"))] + pub data_types: VecMap, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct DataTypeMetadataInfo { + #[serde(rename(serialize = "namespaces"))] + pub namespaces: Vec<&'static str>, + #[serde(rename(serialize = "supportsVendorNamespaces"))] + pub supports_vendor_namespaces: bool, + #[serde(rename(serialize = "supportsPrivate"))] + pub supports_private: bool, + #[serde(rename(serialize = "maxDepth"))] + pub max_depth: Option, +} + #[derive(Debug, Clone, Default, serde::Serialize)] pub struct EmptyCapabilities {} @@ -334,6 +361,18 @@ impl Capability { Capability::Stalwart => "urn:inbuxa:jmap:registry", Capability::WebPushVapid => "urn:ietf:params:jmap:webpush-vapid", Capability::EmailPush => "urn:ietf:params:jmap:emailpush", + Capability::Metadata => "urn:ietf:params:jmap:metadata", + } + } + + pub fn for_data_type(data_type: DataType) -> Option { + match data_type { + DataType::Email | DataType::Mailbox => Some(Capability::Mail), + DataType::SieveScript => Some(Capability::Sieve), + DataType::Calendar | DataType::CalendarEvent => Some(Capability::Calendars), + DataType::AddressBook | DataType::ContactCard => Some(Capability::Contacts), + DataType::FileNode => Some(Capability::FileNode), + _ => None, } } @@ -358,6 +397,7 @@ impl Capability { Capability::Stalwart, Capability::WebPushVapid, Capability::EmailPush, + Capability::Metadata, ] } } @@ -457,9 +497,59 @@ impl Capabilities { } } +impl MetadataCapabilities { + pub fn to_account_capabilities( + &self, + account_capabilities: CapabilityIds, + supports_private: bool, + ) -> Option { + let data_types = self + .data_types + .iter() + .filter(|(data_type, _)| { + Capability::for_data_type(**data_type) + .is_some_and(|capability| account_capabilities.contains(capability)) + }) + .filter_map(|(data_type, info)| { + info.to_account_info(supports_private) + .map(|info| (*data_type, info)) + }) + .collect::>(); + + (!data_types.is_empty()).then_some(MetadataCapabilities { data_types }) + } +} + +impl DataTypeMetadataInfo { + fn to_account_info(&self, supports_private: bool) -> Option { + let supports_private = self.supports_private && supports_private; + let namespaces = if supports_private { + self.namespaces.clone() + } else { + self.namespaces + .iter() + .copied() + .filter(|name| { + RegisteredNamespace::by_name(name) + .is_none_or(|namespace| namespace.scope != NamespaceScope::PrivateMetadata) + }) + .collect() + }; + + (self.supports_vendor_namespaces || !namespaces.is_empty()).then_some( + DataTypeMetadataInfo { + namespaces, + supports_vendor_namespaces: self.supports_vendor_namespaces, + supports_private, + max_depth: self.max_depth, + }, + ) + } +} + impl Capability { pub fn parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), + hashify::map!(s.as_bytes(), Capability, "urn:ietf:params:jmap:core" => Capability::Core, "urn:ietf:params:jmap:mail" => Capability::Mail, "urn:ietf:params:jmap:submission" => Capability::Submission, @@ -480,7 +570,9 @@ impl Capability { "urn:inbuxa:jmap:registry" => Capability::Stalwart, "urn:ietf:params:jmap:webpush-vapid" => Capability::WebPushVapid, "urn:ietf:params:jmap:emailpush" => Capability::EmailPush, + "urn:ietf:params:jmap:metadata" => Capability::Metadata, ) + .copied() } } @@ -522,3 +614,4 @@ impl<'de> Deserialize<'de> for CapabilityIds { deserializer.deserialize_seq(CapabilityIdsVisitor) } } + diff --git a/crates/jmap-proto/src/request/deserialize.rs b/crates/jmap-proto/src/request/deserialize.rs index 337786aa..c3897042 100644 --- a/crates/jmap-proto/src/request/deserialize.rs +++ b/crates/jmap-proto/src/request/deserialize.rs @@ -5,10 +5,50 @@ */ use serde::{ - Deserializer, + Deserialize, Deserializer, de::{self, MapAccess, Visitor}, }; -use std::{fmt, marker::PhantomData}; +use std::{borrow::Cow, fmt, marker::PhantomData}; + +pub(crate) struct CowStr<'x>(pub Cow<'x, str>); + +impl<'de> Deserialize<'de> for CowStr<'de> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct CowStrVisitor; + + impl<'de> Visitor<'de> for CowStrVisitor { + type Value = CowStr<'de>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_borrowed_str(self, value: &'de str) -> Result { + Ok(CowStr(Cow::Borrowed(value))) + } + + fn visit_str(self, value: &str) -> Result { + Ok(CowStr(Cow::Owned(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result { + Ok(CowStr(Cow::Owned(value))) + } + } + + deserializer.deserialize_str(CowStrVisitor) + } +} + +pub(crate) fn next_lowercase_value<'de, A>(map: &mut A) -> Result +where + A: MapAccess<'de>, +{ + Ok(utils::text::lowercase(&map.next_value::>()?.0).into_owned()) +} pub trait DeserializeArguments<'de> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> @@ -26,6 +66,39 @@ impl<'de> DeserializeArguments<'de> for () { } } +#[derive(Default)] +struct ArgumentForms(u16); + +impl ArgumentForms { + #[inline] + fn insert(&mut self, key: &str) -> bool { + !matches!( + key.as_bytes().first(), + Some(b'#' | b'd' | b'e' | b'i' | b'm' | b'p') + ) || self.insert_named(key) + } + + #[inline] + fn insert_named(&mut self, key: &str) -> bool { + let Some(&(form, conflict)) = hashify::map!(key.as_bytes(), (u16, u16), + "ids" => (1 << 0, 1 << 1), + "#ids" => (1 << 1, 1 << 0), + "properties" => (1 << 2, 1 << 3), + "#properties" => (1 << 3, 1 << 2), + "destroy" => (1 << 4, 1 << 5), + "#destroy" => (1 << 5, 1 << 4), + "emailIds" => (1 << 6, 1 << 7), + "#emailIds" => (1 << 7, 1 << 6), + "mailboxIds" => (1 << 8, 1 << 9), + "#mailboxIds" => (1 << 9, 1 << 8), + ) else { + return true; + }; + self.0 |= form; + self.0 & conflict == 0 + } +} + pub(crate) fn deserialize_request<'de, T, D>(deserializer: D) -> Result where T: DeserializeArguments<'de> + Default, @@ -58,8 +131,14 @@ where A: MapAccess<'de>, { let mut target = T::default(); + let mut forms = ArgumentForms::default(); while let Some(key) = map.next_key::<&str>()? { + if !forms.insert(key) { + return Err(de::Error::custom(format_args!( + "Argument {key:?} cannot be given in both normal and referenced form." + ))); + } target .deserialize_argument(key, &mut map) .map_err(de::Error::custom)?; diff --git a/crates/jmap-proto/src/request/method.rs b/crates/jmap-proto/src/request/method.rs index 3e45c8c0..9b6424f6 100644 --- a/crates/jmap-proto/src/request/method.rs +++ b/crates/jmap-proto/src/request/method.rs @@ -6,7 +6,10 @@ use crate::request::capability::Capability; use registry::{ - schema::prelude::{OBJ_SINGLETON, ObjectType}, + schema::prelude::{ + OBJ_NO_CREATE, OBJ_NO_DESTROY, OBJ_NO_GET, OBJ_NO_QUERY, OBJ_NO_UPDATE, OBJ_SINGLETON, + ObjectType, + }, types::EnumImpl, }; use std::{borrow::Cow, fmt::Display}; @@ -103,6 +106,15 @@ impl MethodName { } } + pub fn capability(&self) -> Capability { + match (self.obj, self.fnc) { + (MethodObject::Principal, MethodFunction::GetAvailability) => { + Capability::PrincipalsAvailability + } + (obj, _) => obj.capability(), + } + } + pub fn as_str(&self) -> Cow<'static, str> { match (self.fnc, self.obj) { (MethodFunction::Get, MethodObject::PushSubscription) => "PushSubscription/get", @@ -244,7 +256,7 @@ impl MethodName { } pub fn parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), + hashify::map!(s.as_bytes(), (MethodObject, MethodFunction), "PushSubscription/get" => (MethodObject::PushSubscription, MethodFunction::Get), "PushSubscription/set" => (MethodObject::PushSubscription, MethodFunction::Set), @@ -354,16 +366,27 @@ impl MethodName { "Core/echo" => (MethodObject::Core, MethodFunction::Echo), - ).or_else(|| { + ).copied().or_else(|| { let (obj, fnc) = s.strip_prefix("x:")?.split_once('/')?; let obj = ObjectType::parse(obj)?; - let fnc = hashify::tiny_map!(fnc.as_bytes(), + let fnc = hashify::map!(fnc.as_bytes(), MethodFunction, "get" => MethodFunction::Get, "set" => MethodFunction::Set, "query" => MethodFunction::Query, - )?; + ).copied()?; - if obj.flags() & OBJ_SINGLETON == 0 || fnc != MethodFunction::Query { + let flags = obj.flags(); + let offered = match fnc { + MethodFunction::Get => flags & OBJ_NO_GET == 0, + MethodFunction::Query => flags & (OBJ_SINGLETON | OBJ_NO_QUERY) == 0, + MethodFunction::Set => { + flags & (OBJ_NO_CREATE | OBJ_NO_UPDATE | OBJ_NO_DESTROY) + != (OBJ_NO_CREATE | OBJ_NO_UPDATE | OBJ_NO_DESTROY) + } + _ => true, + }; + + if offered { (MethodObject::Registry(obj), fnc).into() } else { None diff --git a/crates/jmap-proto/src/request/mod.rs b/crates/jmap-proto/src/request/mod.rs index 42f6224e..17254710 100644 --- a/crates/jmap-proto/src/request/mod.rs +++ b/crates/jmap-proto/src/request/mod.rs @@ -39,8 +39,9 @@ use crate::{ }, request::{capability::CapabilityIds, reference::MaybeIdReference}, }; +use ahash::AHashMap; use jmap_tools::{Null, Value}; -use std::{collections::HashMap, fmt::Debug, str::FromStr}; +use std::{fmt::Debug, str::FromStr}; use types::id::Id; use utils::map::vec_map::VecMap; @@ -60,7 +61,7 @@ where pub struct Request<'x> { pub using: CapabilityIds, pub method_calls: Vec>>, - pub created_ids: Option>, + pub created_ids: Option>, } #[derive(Debug)] diff --git a/crates/jmap-proto/src/request/parser.rs b/crates/jmap-proto/src/request/parser.rs index 19a0dd7a..37555c24 100644 --- a/crates/jmap-proto/src/request/parser.rs +++ b/crates/jmap-proto/src/request/parser.rs @@ -12,16 +12,31 @@ use crate::request::{ CopyRequestMethod, GetRequestMethod, ParseRequestMethod, QueryChangesRequestMethod, QueryRequestMethod, SetRequestMethod, deserialize::DeserializeArguments, }; +use compact_str::{CompactString, ToCompactString}; use serde::{ Deserialize, Deserializer, - de::{self, SeqAccess, Visitor}, + de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}, }; -use std::fmt::{self, Display}; +use serde_json::value::RawValue; +use simdutf8::basic::from_utf8; +use std::{ + borrow::Cow, + fmt::{self, Display}, + marker::PhantomData, +}; +use utils::text::utf8_lossy; + +const MAX_ERROR_EXCERPT: usize = 1024; impl<'x> Request<'x> { pub fn parse(json: &'x [u8], max_calls: usize, max_size: usize) -> trc::Result { if json.len() <= max_size { - match serde_json::from_slice::(json) { + let request = match from_utf8(json) { + Ok(text) => serde_json::from_str::(text) + .or_else(|err| Self::parse_lenient(text, err)), + Err(err) => Err(err.to_compact_string()), + }; + match request { Ok(request) => { if request.method_calls.len() <= max_calls { Ok(request) @@ -29,34 +44,40 @@ impl<'x> Request<'x> { Err(trc::LimitEvent::CallsIn.into_err()) } } - Err(err) => Err(trc::JmapEvent::NotRequest - .into_err() - .reason(err.to_string()) - .details(String::from_utf8_lossy(json).into_owned())), + Err(err) => Err(trc::JmapEvent::NotRequest.into_err().reason(err).details( + CompactString::from(utf8_lossy(json.get(..MAX_ERROR_EXCERPT).unwrap_or(json))), + )), } } else { Err(trc::LimitEvent::SizeRequest.into_err()) } } -} -impl<'de> DeserializeArguments<'de> for Request<'de> { - fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> + #[cold] + #[inline(never)] + fn parse_lenient(text: &'x str, err: serde_json::Error) -> Result { + serde_json::from_str::(text) + .map(|request| request.0) + .map_err(|_| err.to_compact_string()) + } + + pub(crate) fn deserialize_field(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where - A: serde::de::MapAccess<'de>, + A: MapAccess<'x>, + C: CallList<'x>, { hashify::fnc_map!(key.as_bytes(), b"using" => { self.using = map.next_value()?; }, b"methodCalls" => { - self.method_calls = map.next_value()?; + self.method_calls = map.next_value::()?.into_calls(); }, b"createdIds" => { self.created_ids = map.next_value()?; }, _ => { - let _ = map.next_value::()?; + let _ = map.next_value::()?; } ); @@ -64,6 +85,303 @@ impl<'de> DeserializeArguments<'de> for Request<'de> { } } +pub(crate) trait CallList<'de>: Deserialize<'de> { + fn into_calls(self) -> Vec>>; +} + +impl<'de> CallList<'de> for Vec>> { + fn into_calls(self) -> Vec>> { + self + } +} + +impl<'de> CallList<'de> for Vec> { + fn into_calls(self) -> Vec>> { + self.into_iter().map(|call| call.0).collect() + } +} + +impl<'de> DeserializeArguments<'de> for Request<'de> { + fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> + where + A: MapAccess<'de>, + { + self.deserialize_field::>>>(key, map) + } +} + +pub(crate) struct LenientRequest<'x>(pub Request<'x>); + +#[repr(transparent)] +pub(crate) struct LenientCall<'x>(Call>); + +struct MethodArguments(MethodName); + +impl<'de> DeserializeSeed<'de> for MethodArguments { + type Value = RequestMethod<'de>; + + #[inline(always)] + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(match (self.0.fnc, self.0.obj) { + (MethodFunction::Get, MethodObject::Email) => RequestMethod::Get( + GetRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Mailbox) => RequestMethod::Get( + GetRequestMethod::Mailbox(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Thread) => RequestMethod::Get( + GetRequestMethod::Thread(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Identity) => RequestMethod::Get( + GetRequestMethod::Identity(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::EmailSubmission) => RequestMethod::Get( + GetRequestMethod::EmailSubmission(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::PushSubscription) => RequestMethod::Get( + GetRequestMethod::PushSubscription(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::VacationResponse) => RequestMethod::Get( + GetRequestMethod::VacationResponse(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::SieveScript) => RequestMethod::Get( + GetRequestMethod::Sieve(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Principal) => RequestMethod::Get( + GetRequestMethod::Principal(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Quota) => RequestMethod::Get( + GetRequestMethod::Quota(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Blob) => RequestMethod::Get( + GetRequestMethod::Blob(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::Calendar) => RequestMethod::Get( + GetRequestMethod::Calendar(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::CalendarEvent) => RequestMethod::Get( + GetRequestMethod::CalendarEvent(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::CalendarEventNotification) => { + RequestMethod::Get(GetRequestMethod::CalendarEventNotification( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::Get, MethodObject::ParticipantIdentity) => RequestMethod::Get( + GetRequestMethod::ParticipantIdentity(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::AddressBook) => RequestMethod::Get( + GetRequestMethod::AddressBook(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::ContactCard) => RequestMethod::Get( + GetRequestMethod::ContactCard(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::FileNode) => RequestMethod::Get( + GetRequestMethod::FileNode(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::ShareNotification) => RequestMethod::Get( + GetRequestMethod::ShareNotification(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Get, MethodObject::SearchSnippet) => { + RequestMethod::SearchSnippet(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Get, MethodObject::Registry(_)) => RequestMethod::Get( + GetRequestMethod::Registry(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::Email) => RequestMethod::Set( + SetRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::Mailbox) => RequestMethod::Set( + SetRequestMethod::Mailbox(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::Identity) => RequestMethod::Set( + SetRequestMethod::Identity(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::EmailSubmission) => RequestMethod::Set( + SetRequestMethod::EmailSubmission(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::PushSubscription) => RequestMethod::Set( + SetRequestMethod::PushSubscription(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::VacationResponse) => RequestMethod::Set( + SetRequestMethod::VacationResponse(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::SieveScript) => RequestMethod::Set( + SetRequestMethod::Sieve(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::Calendar) => RequestMethod::Set( + SetRequestMethod::Calendar(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::CalendarEvent) => RequestMethod::Set( + SetRequestMethod::CalendarEvent(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::CalendarEventNotification) => { + RequestMethod::Set(SetRequestMethod::CalendarEventNotification( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::Set, MethodObject::ParticipantIdentity) => RequestMethod::Set( + SetRequestMethod::ParticipantIdentity(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::AddressBook) => RequestMethod::Set( + SetRequestMethod::AddressBook(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::ContactCard) => RequestMethod::Set( + SetRequestMethod::ContactCard(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::FileNode) => RequestMethod::Set( + SetRequestMethod::FileNode(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::ShareNotification) => RequestMethod::Set( + SetRequestMethod::ShareNotification(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Set, MethodObject::Registry(_)) => RequestMethod::Set( + SetRequestMethod::Registry(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Email) => RequestMethod::Query( + QueryRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Mailbox) => RequestMethod::Query( + QueryRequestMethod::Mailbox(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::EmailSubmission) => RequestMethod::Query( + QueryRequestMethod::EmailSubmission(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::SieveScript) => RequestMethod::Query( + QueryRequestMethod::Sieve(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Principal) => RequestMethod::Query( + QueryRequestMethod::Principal(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Quota) => RequestMethod::Query( + QueryRequestMethod::Quota(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Calendar) => RequestMethod::Query( + QueryRequestMethod::Calendar(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::CalendarEvent) => RequestMethod::Query( + QueryRequestMethod::CalendarEvent(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::CalendarEventNotification) => { + RequestMethod::Query(QueryRequestMethod::CalendarEventNotification( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::Query, MethodObject::AddressBook) => RequestMethod::Query( + QueryRequestMethod::AddressBook(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::ContactCard) => RequestMethod::Query( + QueryRequestMethod::ContactCard(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::FileNode) => RequestMethod::Query( + QueryRequestMethod::FileNode(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::ShareNotification) => RequestMethod::Query( + QueryRequestMethod::ShareNotification(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Query, MethodObject::Registry(_)) => RequestMethod::Query( + QueryRequestMethod::Registry(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::Email) => RequestMethod::QueryChanges( + QueryChangesRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::Mailbox) => RequestMethod::QueryChanges( + QueryChangesRequestMethod::Mailbox(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::EmailSubmission) => { + RequestMethod::QueryChanges(QueryChangesRequestMethod::EmailSubmission( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::QueryChanges, MethodObject::Principal) => RequestMethod::QueryChanges( + QueryChangesRequestMethod::Principal(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::Quota) => RequestMethod::QueryChanges( + QueryChangesRequestMethod::Quota(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::CalendarEvent) => { + RequestMethod::QueryChanges(QueryChangesRequestMethod::CalendarEvent( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::QueryChanges, MethodObject::CalendarEventNotification) => { + RequestMethod::QueryChanges(QueryChangesRequestMethod::CalendarEventNotification( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::QueryChanges, MethodObject::ContactCard) => { + RequestMethod::QueryChanges(QueryChangesRequestMethod::ContactCard( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::QueryChanges, MethodObject::FileNode) => RequestMethod::QueryChanges( + QueryChangesRequestMethod::FileNode(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::QueryChanges, MethodObject::ShareNotification) => { + RequestMethod::QueryChanges(QueryChangesRequestMethod::ShareNotification( + Deserialize::deserialize(deserializer)?, + )) + } + (MethodFunction::Changes, _) => { + RequestMethod::Changes(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Copy, MethodObject::Email) => RequestMethod::Copy( + CopyRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Copy, MethodObject::Blob) => RequestMethod::Copy( + CopyRequestMethod::Blob(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Copy, MethodObject::CalendarEvent) => RequestMethod::Copy( + CopyRequestMethod::CalendarEvent(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Copy, MethodObject::ContactCard) => RequestMethod::Copy( + CopyRequestMethod::ContactCard(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Copy, MethodObject::FileNode) => RequestMethod::Copy( + CopyRequestMethod::FileNode(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Lookup, MethodObject::Blob) => { + RequestMethod::LookupBlob(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Upload, MethodObject::Blob) => { + RequestMethod::UploadBlob(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Import, MethodObject::Email) => { + RequestMethod::ImportEmail(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Parse, MethodObject::Email) => RequestMethod::Parse( + ParseRequestMethod::Email(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Parse, MethodObject::CalendarEvent) => RequestMethod::Parse( + ParseRequestMethod::CalendarEvent(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Parse, MethodObject::ContactCard) => RequestMethod::Parse( + ParseRequestMethod::ContactCard(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::GetAvailability, MethodObject::Principal) => RequestMethod::Get( + GetRequestMethod::PrincipalAvailability(Deserialize::deserialize(deserializer)?), + ), + (MethodFunction::Validate, MethodObject::SieveScript) => { + RequestMethod::ValidateScript(Deserialize::deserialize(deserializer)?) + } + (MethodFunction::Echo, MethodObject::Core) => { + RequestMethod::Echo(Deserialize::deserialize(deserializer)?) + } + _ => { + IgnoredAny::deserialize(deserializer)?; + RequestMethod::Error( + trc::JmapEvent::UnknownMethod + .into_err() + .details(self.0.as_str()), + ) + } + }) + } +} + struct CallVisitor; impl<'de> Visitor<'de> for CallVisitor { @@ -78,614 +396,20 @@ impl<'de> Visitor<'de> for CallVisitor { V: SeqAccess<'de>, { let method_name = seq - .next_element::>()? + .next_element::>()? .ok_or_else(|| de::Error::invalid_length(0, &self))?; - let name = match MethodName::parse(method_name.as_ref()) { - Some(name) => name, + let (name, method) = match MethodName::parse(method_name.as_ref()) { + Some(name) => match seq.next_element_seed(MethodArguments(name)) { + Ok(Some(method)) => (name, method), + Err(err) => (name, RequestMethod::invalid(err)), + Ok(None) => { + return Err(de::Error::invalid_length(1, &self)); + } + }, None => { - // Ignore the rest of the call - let _ = seq - .next_element::()? + seq.next_element::()? .ok_or_else(|| de::Error::invalid_length(1, &self))?; - let id = seq - .next_element::()? - .ok_or_else(|| de::Error::invalid_length(2, &self))?; - - return Ok(Call { - id, - method: RequestMethod::Error( - trc::JmapEvent::UnknownMethod - .into_err() - .details(method_name.to_string()), - ), - name: MethodName::error(), - }); - } - }; - - let method = match (&name.fnc, &name.obj) { - (MethodFunction::Get, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Email(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Mailbox) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Mailbox(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Thread) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Thread(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Identity) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Identity(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::EmailSubmission) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::EmailSubmission(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::PushSubscription) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::PushSubscription(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::VacationResponse) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::VacationResponse(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::SieveScript) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Sieve(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Principal) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Principal(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Quota) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Quota(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Blob) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Blob(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Calendar) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Calendar(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::CalendarEvent) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::CalendarEvent(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::CalendarEventNotification) => { - match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::Get(GetRequestMethod::CalendarEventNotification(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::Get, MethodObject::ParticipantIdentity) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ParticipantIdentity(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::AddressBook) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::AddressBook(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ContactCard(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::FileNode) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::FileNode(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::ShareNotification) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ShareNotification(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::SearchSnippet) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::SearchSnippet(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Get, MethodObject::Registry(_)) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Registry(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Email(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::Mailbox) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Mailbox(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::Identity) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Identity(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::EmailSubmission) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::EmailSubmission(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::PushSubscription) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::PushSubscription(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::VacationResponse) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::VacationResponse(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::SieveScript) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Sieve(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::Calendar) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Calendar(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::CalendarEvent) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::CalendarEvent(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::CalendarEventNotification) => { - match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::Set(SetRequestMethod::CalendarEventNotification(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::Set, MethodObject::ParticipantIdentity) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ParticipantIdentity(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::AddressBook) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::AddressBook(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ContactCard(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::FileNode) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::FileNode(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::ShareNotification) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ShareNotification(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Set, MethodObject::Registry(_)) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Registry(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Email(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Mailbox) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Mailbox(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::EmailSubmission) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::EmailSubmission(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::SieveScript) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Sieve(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Principal) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Principal(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Quota) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Quota(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Calendar) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Calendar(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::CalendarEvent) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::CalendarEvent(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::CalendarEventNotification) => { - match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::Query(QueryRequestMethod::CalendarEventNotification(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::Query, MethodObject::AddressBook) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::AddressBook(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::ContactCard(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::FileNode) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::FileNode(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::ShareNotification) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::Query(QueryRequestMethod::ShareNotification(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Query, MethodObject::Registry(_)) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Registry(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::Email(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::Mailbox) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::Mailbox(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::EmailSubmission) => { - match seq.next_element() { - Ok(Some(value)) => RequestMethod::QueryChanges( - QueryChangesRequestMethod::EmailSubmission(value), - ), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::QueryChanges, MethodObject::Principal) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::Principal(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::Quota) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::Quota(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::CalendarEvent) => match seq.next_element() - { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::CalendarEvent(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::CalendarEventNotification) => { - match seq.next_element() { - Ok(Some(value)) => RequestMethod::QueryChanges( - QueryChangesRequestMethod::CalendarEventNotification(value), - ), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::QueryChanges, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::ContactCard(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::FileNode) => match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::QueryChanges(QueryChangesRequestMethod::FileNode(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::QueryChanges, MethodObject::ShareNotification) => { - match seq.next_element() { - Ok(Some(value)) => RequestMethod::QueryChanges( - QueryChangesRequestMethod::ShareNotification(value), - ), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::Changes, _) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Changes(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Copy, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::Email(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Copy, MethodObject::Blob) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::Blob(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Copy, MethodObject::CalendarEvent) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::CalendarEvent(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Copy, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::ContactCard(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Copy, MethodObject::FileNode) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::FileNode(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Lookup, MethodObject::Blob) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::LookupBlob(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Upload, MethodObject::Blob) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::UploadBlob(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Import, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::ImportEmail(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Parse, MethodObject::Email) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::Email(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Parse, MethodObject::CalendarEvent) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::CalendarEvent(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Parse, MethodObject::ContactCard) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::ContactCard(value)), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::GetAvailability, MethodObject::Principal) => { - match seq.next_element() { - Ok(Some(value)) => { - RequestMethod::Get(GetRequestMethod::PrincipalAvailability(value)) - } - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - } - } - (MethodFunction::Validate, MethodObject::SieveScript) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::ValidateScript(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - (MethodFunction::Echo, MethodObject::Core) => match seq.next_element() { - Ok(Some(value)) => RequestMethod::Echo(value), - Err(err) => RequestMethod::invalid(err), - Ok(None) => { - return Err(de::Error::invalid_length(1, &self)); - } - }, - _ => { - return Err(de::Error::custom(format!( - "Invalid method function/object combination: {}", - method_name - ))); + (MethodName::error(), RequestMethod::unknown(method_name)) } }; @@ -697,14 +421,106 @@ impl<'de> Visitor<'de> for CallVisitor { } } +struct LenientCallVisitor; + +impl<'de> Visitor<'de> for LenientCallVisitor { + type Value = LenientCall<'de>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an array with 3 elements") + } + + fn visit_seq(self, mut seq: V) -> Result, V::Error> + where + V: SeqAccess<'de>, + { + let method_name = seq + .next_element::>()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let arguments = seq + .next_element::<&'de RawValue>()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + let id = seq + .next_element::()? + .ok_or_else(|| de::Error::invalid_length(2, &self))?; + let (name, method) = match MethodName::parse(method_name.as_ref()) { + Some(name) => { + let mut deserializer = serde_json::Deserializer::from_str(arguments.get()); + match MethodArguments(name).deserialize(&mut deserializer) { + Ok(method) => (name, method), + Err(err) => (name, RequestMethod::invalid(err)), + } + } + None => (MethodName::error(), RequestMethod::unknown(method_name)), + }; + + Ok(LenientCall(Call { id, method, name })) + } +} + +impl<'de> Deserialize<'de> for LenientCall<'de> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(LenientCallVisitor) + } +} + impl RequestMethod<'_> { fn invalid(err: impl Display) -> Self { RequestMethod::Error( trc::JmapEvent::InvalidArguments .into_err() - .details(err.to_string()), + .details(err.to_compact_string()), ) } + + fn unknown(method_name: Cow<'_, str>) -> Self { + RequestMethod::Error( + trc::JmapEvent::UnknownMethod + .into_err() + .details(method_name.into_owned()), + ) + } +} + +struct RequestVisitor { + _calls: PhantomData, +} + +impl<'de, C: CallList<'de>> Visitor<'de> for RequestVisitor { + type Value = Request<'de>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JMAP request object") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut target = Request::default(); + let mut has_using = false; + let mut has_method_calls = false; + + while let Some(key) = map.next_key::<&str>()? { + match key { + "using" => has_using = true, + "methodCalls" => has_method_calls = true, + _ => {} + } + target.deserialize_field::(key, &mut map)?; + } + + if !has_using || !has_method_calls { + return Err(de::Error::custom( + "Request is missing the \"using\" or \"methodCalls\" property.", + )); + } + + Ok(target) + } } impl<'de> Deserialize<'de> for Request<'de> { @@ -712,45 +528,22 @@ impl<'de> Deserialize<'de> for Request<'de> { where D: Deserializer<'de>, { - struct RequestVisitor; + deserializer.deserialize_map(RequestVisitor::>>> { + _calls: PhantomData, + }) + } +} - impl<'de> Visitor<'de> for RequestVisitor { - type Value = Request<'de>; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a JMAP request object") - } - - fn visit_map(self, mut map: A) -> Result - where - A: de::MapAccess<'de>, - { - let mut target = Request::default(); - let mut has_using = false; - let mut has_method_calls = false; - - while let Some(key) = map.next_key::<&str>()? { - match key { - "using" => has_using = true, - "methodCalls" => has_method_calls = true, - _ => {} - } - target - .deserialize_argument(key, &mut map) - .map_err(de::Error::custom)?; - } - - if !has_using || !has_method_calls { - return Err(de::Error::custom( - "Request is missing the \"using\" or \"methodCalls\" property.", - )); - } - - Ok(target) - } - } - - deserializer.deserialize_map(RequestVisitor) +impl<'de> Deserialize<'de> for LenientRequest<'de> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer + .deserialize_map(RequestVisitor::>> { + _calls: PhantomData, + }) + .map(LenientRequest) } } @@ -763,186 +556,3 @@ impl<'de> Deserialize<'de> for Call> { } } -#[cfg(test)] -mod tests { - use crate::request::Request; - - const TEST: &str = r#" - { - "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail" ], - "methodCalls": [ - [ "method1", { - "arg1": "arg1data", - "arg2": "arg2data" - }, "c1" ], - [ "Core/echo", { - "hello": true, - "high": 5 - }, "c2" ], - [ "method3", {"hello": [{"a": {"b": true}}]}, "c3" ] - ], - "createdIds": { - "c1": "m1", - "c2": "m2" - } - } - "#; - - const TEST1: &str = r#" - { - "using": [ - "urn:ietf:params:jmap:core", - "urn:ietf:params:jmap:mail" - ], - "methodCalls": [ - [ - "Email/query", - { - "accountId": "0", - "filter": { "conditions": [ { "hasKeyword": "music", "maxSize": 455 }, { "hasKeyword": "video" }, { "operator": "AND", "conditions": [ { "subject": "test" }, { "minSize": 100 } ] } ], "operator": "OR" }, - "sort": [ - { - "property": "subject", - "isAscending": true - }, - { - "property": "allInThreadHaveKeyword", - "isAscending": false, - "keyword": "$seen" - }, - { - "keyword": "$junk", - "property": "someInThreadHaveKeyword", - "collation": "i;octet", - "isAscending": false - } - ], - "position": 0, - "limit": 10 - }, - "c1" - ] - ], - "createdIds": {} - } - "#; - - const TEST2: &str = r##" - { - "using": [ - "urn:ietf:params:jmap:submission", - "urn:ietf:params:jmap:mail", - "urn:ietf:params:jmap:core" - ], - "methodCalls": [ - [ - "Email/set", - { - "accountId": "c", - "create": { - "c37ee58b-e224-4799-88e6-1d7484e3b782": { - "mailboxIds": { - "9": true - }, - "subject": "test", - "from": [ - { - "name": "Foo", - "email": "foo@bar.com" - } - ], - "to": [ - { - "name": null, - "email": "bar@foo.com" - } - ], - "cc": [], - "bcc": [], - "replyTo": [ - { - "name": null, - "email": "foo@bar.com" - } - ], - "htmlBody": [ - { - "partId": "c37ee58b-e224-4799-88e6-1d7484e3b782", - "type": "text/html" - } - ], - "bodyValues": { - "c37ee58b-e224-4799-88e6-1d7484e3b782": { - "value": "

test email

", - "isEncodingProblem": false, - "isTruncated": false - } - }, - "header:User-Agent:asText": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0" - } - } - }, - "c0" - ], - [ - "EmailSubmission/set", - { - "accountId": "c", - "create": { - "c37ee58b-e224-4799-88e6-1d7484e3b782": { - "identityId": "a", - "emailId": "#c37ee58b-e224-4799-88e6-1d7484e3b782", - "envelope": { - "mailFrom": { - "email": "foo@bar.com" - }, - "rcptTo": [ - { - "email": "bar@foo.com" - } - ] - } - } - }, - "onSuccessUpdateEmail": { - "#c37ee58b-e224-4799-88e6-1d7484e3b782": { - "mailboxIds/d": true, - "mailboxIds/9": null, - "keywords/$seen": true, - "keywords/$draft": null - } - } - }, - "c1" - ] - ] - } - "##; - - const TEST_ESCAPED_SOLIDUS: &str = r#" - { - "using": [ "urn:ietf:params:jmap:core" ], - "methodCalls": [ - [ "Core\/echo", { "hello": true }, "c1" ] - ] - } - "#; - - #[test] - fn parse_request() { - println!("{:#?}", Request::parse(TEST.as_bytes(), 10, 10240)); - println!("{:#?}", Request::parse(TEST1.as_bytes(), 10, 10240)); - println!("{:#?}", Request::parse(TEST2.as_bytes(), 10, 10240)); - } - - #[test] - fn parse_method_name_with_escaped_solidus() { - let request = Request::parse(TEST_ESCAPED_SOLIDUS.as_bytes(), 10, 10240) - .expect("escaped solidus in method name must parse"); - assert_eq!(request.method_calls.len(), 1); - assert!(matches!( - request.method_calls[0].method, - super::RequestMethod::Echo(_) - )); - } -} diff --git a/crates/jmap-proto/src/request/reference.rs b/crates/jmap-proto/src/request/reference.rs index 0b18e3b4..a7034fc0 100644 --- a/crates/jmap-proto/src/request/reference.rs +++ b/crates/jmap-proto/src/request/reference.rs @@ -6,12 +6,14 @@ use super::method::MethodName; use jmap_tools::{JsonPointer, Null}; +use serde::Deserialize; use std::{borrow::Cow, fmt::Display, str::FromStr}; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ResultReference { #[serde(rename = "resultOf")] pub result_of: String, + #[serde(deserialize_with = "deserialize_reference_name")] pub name: MethodName, pub path: JsonPointer, } @@ -29,6 +31,14 @@ pub enum MaybeResultReference { Reference(ResultReference), } +fn deserialize_reference_name<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + >::deserialize(deserializer) + .map(|name| MethodName::parse(name.as_ref()).unwrap_or_else(MethodName::error)) +} + impl Display for ResultReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/crates/jmap-proto/src/request/websocket.rs b/crates/jmap-proto/src/request/websocket.rs index 8726bcd8..56e411c7 100644 --- a/crates/jmap-proto/src/request/websocket.rs +++ b/crates/jmap-proto/src/request/websocket.rs @@ -8,14 +8,20 @@ use super::Request; use crate::{ error::request::{RequestError, RequestErrorType, RequestLimitError}, object::AnyId, - request::{Call, deserialize::DeserializeArguments}, + request::{ + Call, RequestMethod, + parser::{CallList, LenientCall}, + }, response::{Response, ResponseMethod, serialize::serialize_hex, status::PushObject}, }; +use ahash::AHashMap; +use compact_str::{CompactString, ToCompactString, format_compact}; use serde::{ Deserialize, Deserializer, de::{self, MapAccess, Visitor}, }; -use std::{borrow::Cow, collections::HashMap, fmt}; +use simdutf8::basic::from_utf8; +use std::{borrow::Cow, fmt, marker::PhantomData}; use types::type_state::DataType; #[derive(Debug)] @@ -37,8 +43,8 @@ pub struct WebSocketResponse<'x> { session_state: u32, #[serde(rename(deserialize = "createdIds"))] - #[serde(skip_serializing_if = "HashMap::is_empty")] - created_ids: HashMap, + #[serde(skip_serializing_if = "crate::method::ahash_is_empty")] + created_ids: AHashMap, #[serde(rename = "requestId")] #[serde(skip_serializing_if = "Option::is_none")] @@ -104,9 +110,23 @@ enum MessageType { } impl<'x> WebSocketMessage<'x> { + #[cold] + #[inline(never)] + fn parse_lenient(text: &'x str, err: serde_json::Error) -> Result { + serde_json::from_str::(text) + .map(|message| message.0) + .map_err(|_| err.to_compact_string()) + } + pub fn parse(json: &'x [u8], max_calls: usize, max_size: usize) -> trc::Result { if json.len() <= max_size { - match serde_json::from_slice::(json) { + let message = match from_utf8(json) { + Ok(text) => { + serde_json::from_str::(text).or_else(|err| Self::parse_lenient(text, err)) + } + Err(err) => Err(err.to_compact_string()), + }; + match message { Ok(WebSocketMessage::Request(req)) if req.request.method_calls.len() > max_calls => { @@ -115,7 +135,7 @@ impl<'x> WebSocketMessage<'x> { Ok(msg) => Ok(msg), Err(err) => Err(trc::JmapEvent::NotRequest .into_err() - .details(format!("Invalid WebSocket JMAP request {err}"))), + .details(format_compact!("Invalid WebSocket JMAP request {err}"))), } } else { Err(trc::LimitEvent::SizeRequest.into_err()) @@ -128,13 +148,32 @@ impl<'de: 'x, 'x: 'de> Deserialize<'de> for WebSocketMessage<'x> { where D: Deserializer<'de>, { - deserializer.deserialize_map(WebSocketMessageVisitor) + deserializer.deserialize_map(WebSocketMessageVisitor::>>> { + _calls: PhantomData, + }) } } -struct WebSocketMessageVisitor; +struct LenientWebSocketMessage<'x>(WebSocketMessage<'x>); -impl<'de> Visitor<'de> for WebSocketMessageVisitor { +impl<'de> Deserialize<'de> for LenientWebSocketMessage<'de> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer + .deserialize_map(WebSocketMessageVisitor::>> { + _calls: PhantomData, + }) + .map(LenientWebSocketMessage) + } +} + +struct WebSocketMessageVisitor { + _calls: PhantomData, +} + +impl<'de, C: CallList<'de>> Visitor<'de> for WebSocketMessageVisitor { type Value = WebSocketMessage<'de>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { @@ -172,7 +211,7 @@ impl<'de> Visitor<'de> for WebSocketMessageVisitor { request.id = map.next_value()?; }, _ => { - request.request.deserialize_argument(key, &mut map)?; + request.request.deserialize_field::(key, &mut map)?; found_request_keys = true; } ); @@ -193,10 +232,11 @@ impl<'de> Visitor<'de> for WebSocketMessageVisitor { impl MessageType { fn parse(s: &str) -> Self { - hashify::tiny_map!(s.as_bytes(), - b"Request" => MessageType::Request, - b"WebSocketPushEnable" => MessageType::PushEnable, - b"WebSocketPushDisable" => MessageType::PushDisable, + hashify::fnc_map!(s.as_bytes(), + b"Request" => Some(MessageType::Request), + b"WebSocketPushEnable" => Some(MessageType::PushEnable), + b"WebSocketPushDisable" => Some(MessageType::PushDisable), + _ => None, ) .unwrap_or(MessageType::None) } @@ -236,8 +276,8 @@ impl<'x> WebSocketResponse<'x> { } } - pub fn to_json(&self) -> String { - serde_json::to_string(self).unwrap() + pub fn to_json(&self) -> Result { + sonic_rs::to_string(self) } } diff --git a/crates/jmap-proto/src/response/mod.rs b/crates/jmap-proto/src/response/mod.rs index c9b72ab1..31a7ae39 100644 --- a/crates/jmap-proto/src/response/mod.rs +++ b/crates/jmap-proto/src/response/mod.rs @@ -52,8 +52,8 @@ use crate::{ }, request::{Call, method::MethodName}, }; +use ahash::AHashMap; use jmap_tools::{Null, Value}; -use std::collections::HashMap; #[derive(Debug, serde::Serialize)] #[serde(untagged)] @@ -137,6 +137,7 @@ pub enum ChangesResponseMethod { CalendarEvent(Box>), CalendarEventNotification(Box>), ShareNotification(Box>), + ParticipantIdentity(Box>), } #[derive(Debug, serde::Serialize)] @@ -167,12 +168,16 @@ pub struct Response<'x> { pub session_state: u32, #[serde(rename = "createdIds")] - #[serde(skip_serializing_if = "HashMap::is_empty")] - pub created_ids: HashMap, + #[serde(skip_serializing_if = "crate::method::ahash_is_empty")] + pub created_ids: AHashMap, } impl<'x> Response<'x> { - pub fn new(session_state: u32, created_ids: HashMap, capacity: usize) -> Self { + pub fn to_json(&self) -> Result { + sonic_rs::to_string(self) + } + + pub fn new(session_state: u32, created_ids: AHashMap, capacity: usize) -> Self { Response { session_state, created_ids, diff --git a/crates/jmap-proto/src/response/serialize.rs b/crates/jmap-proto/src/response/serialize.rs index 4e70fec8..f9c3f977 100644 --- a/crates/jmap-proto/src/response/serialize.rs +++ b/crates/jmap-proto/src/response/serialize.rs @@ -14,7 +14,7 @@ impl Serialize for Call> { S: serde::Serializer, { let mut seq = serializer.serialize_seq(3.into())?; - seq.serialize_element(&self.name.to_string())?; + seq.serialize_element(self.name.as_str().as_ref())?; seq.serialize_element(&self.method)?; seq.serialize_element(&self.id)?; seq.end() @@ -25,5 +25,6 @@ pub fn serialize_hex(value: &u32, serializer: S) -> Result where S: serde::Serializer, { - format!("{:x}", value).serialize(serializer) + serializer.collect_str(&format_args!("{value:x}")) } + diff --git a/crates/jmap-proto/src/types/date.rs b/crates/jmap-proto/src/types/date.rs index 96877764..d8ced633 100644 --- a/crates/jmap-proto/src/types/date.rs +++ b/crates/jmap-proto/src/types/date.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Display, str::FromStr}; +use std::{ + fmt::Display, + str::{FromStr, from_utf8}, +}; #[derive( rkyv::Archive, @@ -117,9 +120,45 @@ impl FromStr for UTCDate { } impl UTCDate { + fn text(&self) -> DateText { + let mut text = DateText { + bytes: [0; DATE_TEXT_CAPACITY], + len: 0, + }; + text.push_number(self.year, 4); + text.push(b'-'); + text.push_number(self.month.into(), 2); + text.push(b'-'); + text.push_number(self.day.into(), 2); + text.push(b'T'); + text.push_number(self.hour.into(), 2); + text.push(b':'); + text.push_number(self.minute.into(), 2); + text.push(b':'); + text.push_number(self.second.into(), 2); + if self.tz_hour != 0 || self.tz_minute != 0 { + text.push( + if self.tz_before_gmt && (self.tz_hour > 0 || self.tz_minute > 0) { + b'-' + } else { + b'+' + }, + ); + text.push_number(self.tz_hour.into(), 2); + text.push(b':'); + text.push_number(self.tz_minute.into(), 2); + } else { + text.push(b'Z'); + } + text + } + pub fn from_timestamp(timestamp: i64) -> Self { // Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days - let (z, seconds) = ((timestamp / 86400) + 719468, timestamp % 86400); + let (z, seconds) = ( + timestamp.div_euclid(86400) + 719468, + timestamp.rem_euclid(86400), + ); let era: i64 = (if z >= 0 { z } else { z - 146096 }) / 146097; let doe: u64 = (z - era * 146097) as u64; // [0, 146096] let yoe: u64 = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] @@ -189,33 +228,51 @@ impl From<&ArchivedUTCDate> for UTCDate { } } +const DATE_TEXT_CAPACITY: usize = 40; +const MAX_DIGITS: usize = 5; + +struct DateText { + bytes: [u8; DATE_TEXT_CAPACITY], + len: usize, +} + +impl DateText { + fn push(&mut self, byte: u8) { + if let Some(slot) = self.bytes.get_mut(self.len) { + *slot = byte; + self.len += 1; + } + } + + fn push_number(&mut self, value: u16, width: usize) { + let mut digits = [0u8; MAX_DIGITS]; + let mut rest = value; + let mut count = 0; + for digit in digits.iter_mut().rev() { + *digit = b'0' + (rest % 10) as u8; + rest /= 10; + count += 1; + if rest == 0 { + break; + } + } + (count..width).for_each(|_| self.push(b'0')); + if let Some((_, significant)) = digits.split_at_checked(MAX_DIGITS - count) { + significant.iter().for_each(|&digit| self.push(digit)); + } + } + + fn as_str(&self) -> &str { + self.bytes + .get(..self.len) + .and_then(|bytes| from_utf8(bytes).ok()) + .unwrap_or_default() + } +} + impl Display for UTCDate { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.tz_hour != 0 || self.tz_minute != 0 { - write!( - f, - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}{}{:02}:{:02}", - self.year, - self.month, - self.day, - self.hour, - self.minute, - self.second, - if self.tz_before_gmt && (self.tz_hour > 0 || self.tz_minute > 0) { - "-" - } else { - "+" - }, - self.tz_hour, - self.tz_minute, - ) - } else { - write!( - f, - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", - self.year, self.month, self.day, self.hour, self.minute, self.second, - ) - } + f.write_str(self.text().as_str()) } } @@ -224,7 +281,7 @@ impl serde::Serialize for UTCDate { where S: serde::Serializer, { - serializer.serialize_str(self.to_string().as_str()) + serializer.serialize_str(self.text().as_str()) } } @@ -250,29 +307,3 @@ impl From for UTCDate { } } -#[cfg(test)] -mod tests { - use crate::types::date::UTCDate; - use std::str::FromStr; - - #[test] - fn parse_jmap_date() { - for (input, expected_result) in [ - ("1997-11-21T09:55:06-06:00", "1997-11-21T09:55:06-06:00"), - ("1997-11-21T09:55:06+00:00", "1997-11-21T09:55:06Z"), - ("2021-01-01T09:55:06+02:00", "2021-01-01T09:55:06+02:00"), - ("2004-06-28T23:43:45.000Z", "2004-06-28T23:43:45Z"), - ("1997-11-21T09:55:06.123+00:00", "1997-11-21T09:55:06Z"), - ( - "2021-01-01T09:55:06.4567+02:00", - "2021-01-01T09:55:06+02:00", - ), - ] { - let date = UTCDate::from_str(input).unwrap(); - assert_eq!(date.to_string(), expected_result); - - let timestamp = date.timestamp(); - assert_eq!(UTCDate::from_timestamp(timestamp).timestamp(), timestamp); - } - } -} diff --git a/crates/jmap-proto/src/types/state.rs b/crates/jmap-proto/src/types/state.rs index 8c2af28a..a0637f10 100644 --- a/crates/jmap-proto/src/types/state.rs +++ b/crates/jmap-proto/src/types/state.rs @@ -4,11 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use types::ChangeId; -use utils::codec::{ - base32_custom::{Base32Reader, Base32Writer}, - leb128::{Leb128Iterator, Leb128Writer}, +use encodify::base32::STALWART; +use std::{ + borrow::Cow, + fmt::{self, Write}, + io::{Cursor, Write as _}, }; +use types::ChangeId; +use utils::codec::leb128::{Leb128, Leb128Iterator}; + +const MAX_SERIALIZED_LEN: usize = 30; #[derive(Debug, Clone, PartialEq, Eq)] pub struct JMAPIntermediateState { @@ -41,13 +46,13 @@ impl State { match it.next()? { b'n' => Some(State::Initial), b's' => { - let mut reader = Base32Reader::from_iter(it); + let mut reader = STALWART.decoder_from_iter(it); reader .next_leb128::() .map(|change_id| (change_id != 0).then_some(change_id).into()) } b'r' => { - let mut it = Base32Reader::from_iter(it); + let mut it = STALWART.decoder_from_iter(it); if let (Some(from_id), Some(to_id), Some(items_sent)) = ( it.next_leb128::(), @@ -101,7 +106,26 @@ impl serde::Serialize for State { where S: serde::Serializer, { - serializer.serialize_str(self.to_string().as_str()) + serializer.collect_str(self) + } +} + +pub(crate) struct ExpectedState(pub Option); + +impl<'de> serde::Deserialize<'de> for ExpectedState { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Ok(ExpectedState( + >>::deserialize(deserializer)?.map(|value| { + State::parse(&value).unwrap_or(State::Intermediate(JMAPIntermediateState { + from_id: 0, + to_id: 0, + items_sent: 0, + })) + }), + )) } } @@ -115,63 +139,37 @@ impl<'de> serde::Deserialize<'de> for State { } } -impl std::fmt::Display for State { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut writer = Base32Writer::with_capacity(10); +impl fmt::Display for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut bytes = Cursor::new([0u8; MAX_SERIALIZED_LEN]); - match self { - State::Initial => { - writer.push_char('n'); - } + let prefix = match self { + State::Initial => 'n', State::Exact(id) => { - writer.push_char('s'); - writer.write_leb128(*id).unwrap(); + bytes.write_all(&id.to_leb128()).map_err(|_| fmt::Error)?; + 's' } State::Intermediate(intermediate) => { - writer.push_char('r'); - writer.write_leb128(intermediate.from_id).unwrap(); - writer - .write_leb128(intermediate.to_id - intermediate.from_id) - .unwrap(); - writer.write_leb128(intermediate.items_sent).unwrap(); + bytes + .write_all(&intermediate.from_id.to_leb128()) + .map_err(|_| fmt::Error)?; + bytes + .write_all(&(intermediate.to_id - intermediate.from_id).to_leb128()) + .map_err(|_| fmt::Error)?; + bytes + .write_all(&intermediate.items_sent.to_leb128()) + .map_err(|_| fmt::Error)?; + 'r' } - } + }; - f.write_str(&writer.finalize()) + f.write_char(prefix)?; + let len = bytes.position() as usize; + write!( + f, + "{}", + STALWART.display(bytes.get_ref().get(..len).unwrap_or_default()) + ) } } -#[cfg(test)] -mod tests { - use super::State; - use types::ChangeId; - - #[test] - fn test_state_id() { - for id in [ - State::new_initial(), - State::new_exact(1), - State::new_exact(12345678), - State::new_exact(ChangeId::MAX), - State::new_intermediate(0, 0, 1), - State::new_intermediate(1024, 2048, 100), - State::new_intermediate(12345678, 87654321, 1), - State::new_intermediate(0, 0, 12345678), - State::new_intermediate(0, 87654321, 12345678), - State::new_intermediate(12345678, 87654321, 1), - State::new_intermediate(12345678, 87654321, 12345678), - State::new_intermediate(ChangeId::MAX, ChangeId::MAX, ChangeId::MAX as usize), - ] { - assert_eq!(State::parse(&id.to_string()).unwrap(), id); - } - } - - #[test] - fn test_state_zero_change_id_is_initial() { - assert_eq!( - State::parse(&State::new_exact(0).to_string()).unwrap(), - State::Initial - ); - assert_eq!(State::from(None), State::Initial); - } -} diff --git a/crates/jmap-tools/Cargo.toml b/crates/jmap-tools/Cargo.toml new file mode 100644 index 00000000..d7519f04 --- /dev/null +++ b/crates/jmap-tools/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "jmap-tools" +description = "JMAP object parser with JSON Pointer querying and patching support" +version = "0.1.9" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +rkyv = { version = "0.8", optional = true } + +[features] +default = [] +rkyv = ["dep:rkyv"] + +[lints] +workspace = true diff --git a/crates/jmap-tools/src/json/de.rs b/crates/jmap-tools/src/json/de.rs new file mode 100644 index 00000000..b01cf761 --- /dev/null +++ b/crates/jmap-tools/src/json/de.rs @@ -0,0 +1,320 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::value::Value; +use crate::json::key::{self, Key, OpaqueKey}; +use crate::json::object_vec::ObjectAsVec; +use crate::{Element, Property}; +use serde::de::{Deserialize, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use std::borrow::Cow; +use std::marker::PhantomData; +use std::mem; + +type Member<'de, P, E> = (Key<'de, P>, Value<'de, P, E>); + +struct Scratch<'de, P: Property, E: Element> { + members: Vec>, + items: Vec>, + opaque: bool, +} + +impl Scratch<'_, P, E> { + fn new() -> Self { + Scratch { + members: Vec::new(), + items: Vec::new(), + opaque: false, + } + } +} + +impl<'de, P: Property, E: Element> Scratch<'de, P, E> { + #[cold] + #[inline(never)] + fn opaque_seq(&mut self, mut visitor: V) -> Result, V::Error> + where + V: SeqAccess<'de>, + { + let outer = mem::replace(&mut self.opaque, true); + let start = self.items.len(); + + while let Some(item) = visitor.next_element_seed(ValueSeed { + parent_key: None, + scratch: &mut *self, + })? { + self.items.push(item); + } + + self.opaque = outer; + Ok(Value::Array(self.items.split_off(start))) + } + + #[cold] + #[inline(never)] + fn opaque_map(&mut self, mut visitor: V) -> Result, V::Error> + where + V: MapAccess<'de>, + { + let outer = mem::replace(&mut self.opaque, true); + let start = self.members.len(); + + while let Some(key) = visitor.next_key_seed(OpaqueKey(PhantomData))? { + let value = visitor.next_value_seed(ValueSeed { + parent_key: None, + scratch: &mut *self, + })?; + + self.members.push((key, value)); + } + + self.opaque = outer; + Ok(Value::Object(ObjectAsVec(self.members.split_off(start)))) + } +} + +struct ValueSeed<'s, 'k, 'de, P: Property, E: Element> { + parent_key: Option<&'k Key<'k, P>>, + scratch: &'s mut Scratch<'de, P, E>, +} + +impl<'de, P: Property, E: Element> Deserialize<'de> for Value<'de, P, E> { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + ValueSeed { + parent_key: None, + scratch: &mut Scratch::new(), + } + .deserialize(deserializer) + } +} + +impl<'de, P: Property, E: Element> DeserializeSeed<'de> + for ValueSeed<'_, '_, 'de, P, E> +{ + type Value = Value<'de, P, E>; + + #[inline] + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(self) + } +} + +impl<'de, P: Property, E: Element> ValueSeed<'_, '_, 'de, P, E> { + #[inline] + fn string(&self, text: &str) -> Option { + self.parent_key + .filter(|key| !key.is_opaque()) + .and_then(|key| E::try_parse::

(key, text)) + } +} + +impl<'de, P: Property, E: Element> Visitor<'de> for ValueSeed<'_, '_, 'de, P, E> { + type Value = Value<'de, P, E>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("any valid JSON value") + } + + #[inline] + fn visit_bool(self, value: bool) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Bool(value)) + } + + #[inline] + fn visit_i64(self, value: i64) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Number(value.into())) + } + + #[inline] + fn visit_u64(self, value: u64) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Number(value.into())) + } + + #[inline] + fn visit_f64(self, value: f64) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Number(value.into())) + } + + #[inline] + fn visit_string(self, v: String) -> Result + where + ERR: serde::de::Error, + { + Ok(match self.string(&v) { + Some(element) => Value::Element(element), + None => Value::Str(Cow::Owned(v)), + }) + } + + #[inline] + fn visit_str(self, v: &str) -> Result + where + ERR: serde::de::Error, + { + Ok(match self.string(v) { + Some(element) => Value::Element(element), + None => Value::Str(Cow::Owned(v.to_owned())), + }) + } + + #[inline] + fn visit_borrowed_str(self, v: &'de str) -> Result + where + ERR: serde::de::Error, + { + Ok(match self.string(v) { + Some(element) => Value::Element(element), + None => Value::Str(Cow::Borrowed(v)), + }) + } + + #[inline] + fn visit_none(self) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Null) + } + + #[inline] + fn visit_i8(self, v: i8) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as i64).into())) + } + + #[inline] + fn visit_i16(self, v: i16) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as i64).into())) + } + + #[inline] + fn visit_i32(self, v: i32) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as i64).into())) + } + + #[inline] + fn visit_u8(self, v: u8) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as u64).into())) + } + + #[inline] + fn visit_u16(self, v: u16) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as u64).into())) + } + + #[inline] + fn visit_u32(self, v: u32) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as u64).into())) + } + + #[inline] + fn visit_f32(self, v: f32) -> Result + where + ERR: serde::de::Error, + { + Ok(Value::Number((v as f64).into())) + } + + #[inline] + fn visit_some(self, deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + self.deserialize(deserializer) + } + + #[inline] + fn visit_unit(self) -> Result, ERR> + where + ERR: serde::de::Error, + { + Ok(Value::Null) + } + + #[inline] + fn visit_seq(self, mut visitor: V) -> Result, V::Error> + where + V: SeqAccess<'de>, + { + if self.parent_key.is_some_and(Key::is_opaque) { + return self.scratch.opaque_seq(visitor); + } + + let start = self.scratch.items.len(); + + while let Some(elem) = visitor.next_element_seed(ValueSeed { + parent_key: self.parent_key, + scratch: &mut *self.scratch, + })? { + self.scratch.items.push(elem); + } + + Ok(Value::Array(self.scratch.items.split_off(start))) + } + + #[inline] + fn visit_map(self, mut visitor: V) -> Result, V::Error> + where + V: MapAccess<'de>, + { + if self.scratch.opaque || self.parent_key.is_some_and(Key::is_opaque) { + return self.scratch.opaque_map(visitor); + } + + let start = self.scratch.members.len(); + + while let Some(key) = visitor.next_key_seed(key::DeserializationContext { + parent_key: self.parent_key, + })? { + let value = visitor.next_value_seed(ValueSeed { + parent_key: Some(&key), + scratch: &mut *self.scratch, + })?; + + self.scratch.members.push((key, value)); + } + + Ok(Value::Object(ObjectAsVec( + self.scratch.members.split_off(start), + ))) + } +} + + diff --git a/crates/jmap-tools/src/json/deserializer.rs b/crates/jmap-tools/src/json/deserializer.rs new file mode 100644 index 00000000..eba00ccf --- /dev/null +++ b/crates/jmap-tools/src/json/deserializer.rs @@ -0,0 +1,355 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; + +use crate::json::key::Key; +use crate::json::num::N; +use crate::{Element, Property, Value}; +use serde::Deserializer; +use serde::de::{self, IntoDeserializer, MapAccess, SeqAccess, Visitor}; + +impl<'de, P: Property, E: Element> IntoDeserializer<'de, de::value::Error> + for &'de Value<'_, P, E> +{ + type Deserializer = Self; + + fn into_deserializer(self) -> Self::Deserializer { + self + } +} + +impl<'de, P: Property, E: Element> Deserializer<'de> for &'de Value<'_, P, E> { + type Error = de::value::Error; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self { + Value::Null => visitor.visit_unit(), + Value::Bool(b) => visitor.visit_bool(*b), + Value::Number(n) => match n.n { + N::PosInt(u) => visitor.visit_u64(u), + N::NegInt(i) => visitor.visit_i64(i), + N::Float(f) => visitor.visit_f64(f), + }, + Value::Str(s) => visitor.visit_borrowed_str(s), + Value::Array(arr) => { + let seq = SeqDeserializer::new(arr); + visitor.visit_seq(seq) + } + Value::Object(map) => { + let map = MapDeserializer::new(map.as_vec().as_slice()); + visitor.visit_map(map) + } + Value::Element(e) => match e.to_cow() { + Cow::Borrowed(s) => visitor.visit_borrowed_str(s), + Cow::Owned(s) => visitor.visit_string(s), + }, + } + } + + fn deserialize_bool(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_i8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_i16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_i32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_i64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_u8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_u16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_u32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_u64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_f32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_f64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_char(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_string(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_bytes(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_byte_buf(visitor) + } + + fn deserialize_byte_buf(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self { + Value::Null => visitor.visit_none(), + _ => visitor.visit_some(self), + } + } + + fn deserialize_unit(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + Err(de::Error::custom("deserialize_enum is not yet supported")) + } + + fn deserialize_identifier(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_string(visitor) + } + + fn deserialize_ignored_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_unit() + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_unit(visitor) + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } +} + +// Helper struct to deserialize sequences (arrays). +struct SeqDeserializer<'a, 'ctx, P: Property, E: Element> { + iter: std::slice::Iter<'a, Value<'ctx, P, E>>, +} + +impl<'a, 'ctx, P: Property, E: Element> SeqDeserializer<'a, 'ctx, P, E> { + fn new(slice: &'a [Value<'ctx, P, E>]) -> Self { + SeqDeserializer { iter: slice.iter() } + } +} + +impl<'de, 'a: 'de, 'ctx: 'de, P: Property, E: Element> SeqAccess<'de> + for SeqDeserializer<'a, 'ctx, P, E> +{ + type Error = de::value::Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: de::DeserializeSeed<'de>, + { + self.iter + .next() + .map(|value| seed.deserialize(value)) + .transpose() + } +} + +// Helper struct to deserialize maps (objects). +struct MapDeserializer<'a, 'ctx, P: Property, E: Element> { + iter: std::slice::Iter<'a, (Key<'ctx, P>, Value<'ctx, P, E>)>, + value: Option<&'a Value<'ctx, P, E>>, +} + +impl<'a, 'ctx, P: Property, E: Element> MapDeserializer<'a, 'ctx, P, E> { + fn new(map: &'a [(Key<'ctx, P>, Value<'ctx, P, E>)]) -> Self { + MapDeserializer { + iter: map.iter(), + value: None, + } + } +} + +impl<'de, 'a: 'de, 'ctx: 'de, P: Property, E: Element> MapAccess<'de> + for MapDeserializer<'a, 'ctx, P, E> +{ + type Error = de::value::Error; + + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: de::DeserializeSeed<'de>, + { + if let Some((key, value)) = self.iter.next() { + self.value = Some(value); + match key.to_string() { + Cow::Borrowed(key) => seed + .deserialize(de::value::BorrowedStrDeserializer::new(key)) + .map(Some), + Cow::Owned(key) => seed + .deserialize(de::value::StringDeserializer::new(key)) + .map(Some), + } + } else { + Ok(None) + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + match self.value.take() { + Some(value) => seed.deserialize(value), + None => Err(de::Error::custom("value is missing")), + } + } +} + diff --git a/crates/jmap-tools/src/json/index.rs b/crates/jmap-tools/src/json/index.rs new file mode 100644 index 00000000..7c7d76ed --- /dev/null +++ b/crates/jmap-tools/src/json/index.rs @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::{ + Value, + json::value::{Element, Property}, +}; + +/// A type that can be used to index into a `jmap_tools::Value`. +pub trait Index<'v, P: Property, E: Element> { + /// Return None if the key is not already in the array or object. + #[doc(hidden)] + fn index_into(self, v: &'v Value<'v, P, E>) -> Option<&'v Value<'v, P, E>>; +} + +impl<'v, P: Property, E: Element> Index<'v, P, E> for usize { + #[inline] + fn index_into(self, v: &'v Value<'v, P, E>) -> Option<&'v Value<'v, P, E>> { + match v { + Value::Array(vec) => vec.get(self), + _ => None, + } + } +} + +impl<'v, 'a: 'v, P: Property, E: Element> Index<'v, P, E> for &'a str { + #[inline] + fn index_into(self, v: &'v Value<'v, P, E>) -> Option<&'v Value<'v, P, E>> { + match v { + Value::Object(map) => map.iter().find(|(k, _v)| *k == &self).map(|(_k, v)| v), + _ => None, + } + } +} diff --git a/crates/jmap-tools/src/json/key.rs b/crates/jmap-tools/src/json/key.rs new file mode 100644 index 00000000..8059e03d --- /dev/null +++ b/crates/jmap-tools/src/json/key.rs @@ -0,0 +1,290 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::json::value::Property; +use serde::de::{self, DeserializeSeed, Visitor}; +use serde::{Serialize, Serializer}; +use std::borrow::Cow; +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; + +#[derive(Debug, Clone)] +pub enum Key<'x, P: Property> { + Property(P), + Borrowed(&'x str), + Owned(String), +} + +pub(crate) struct DeserializationContext<'x, P: Property> { + pub parent_key: Option<&'x Key<'x, P>>, +} + +impl<'de, 'x, P: Property> DeserializeSeed<'de> for DeserializationContext<'x, P> { + type Value = Key<'de, P>; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(KeyVisitor { context: &self }) + } +} + +struct KeyVisitor<'x, P: Property> { + context: &'x DeserializationContext<'x, P>, +} + +impl<'de, 'x, P: Property> Visitor<'de> for KeyVisitor<'x, P> { + type Value = Key<'de, P>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_borrowed_str(self, value: &'de str) -> Result + where + ERR: de::Error, + { + match P::try_parse(self.context.parent_key, value) { + Some(word) => Ok(Key::Property(word)), + None => Ok(Key::Borrowed(value)), + } + } + + fn visit_str(self, value: &str) -> Result + where + ERR: de::Error, + { + match P::try_parse(self.context.parent_key, value) { + Some(word) => Ok(Key::Property(word)), + None => Ok(Key::Owned(value.to_owned())), + } + } + + fn visit_string(self, value: String) -> Result + where + ERR: de::Error, + { + match P::try_parse(self.context.parent_key, &value) { + Some(word) => Ok(Key::Property(word)), + None => Ok(Key::Owned(value)), + } + } +} + +pub(crate) struct OpaqueKey(pub PhantomData

); + +impl<'de, P: Property> DeserializeSeed<'de> for OpaqueKey

{ + type Value = Key<'de, P>; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(self) + } +} + +impl<'de, P: Property> Visitor<'de> for OpaqueKey

{ + type Value = Key<'de, P>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_borrowed_str(self, value: &'de str) -> Result + where + ERR: de::Error, + { + Ok(Key::Borrowed(value)) + } + + fn visit_str(self, value: &str) -> Result + where + ERR: de::Error, + { + Ok(Key::Owned(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result + where + ERR: de::Error, + { + Ok(Key::Owned(value)) + } +} + +impl Serialize for Key<'_, P> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Key::Property(property) => property.serialize_text(serializer), + Key::Borrowed(text) => serializer.serialize_str(text), + Key::Owned(text) => serializer.serialize_str(text), + } + } +} + +impl PartialEq for Key<'_, P> { + #[inline] + fn eq(&self, other: &Self) -> bool { + match other { + Key::Property(property) => self.matches_property(property), + Key::Borrowed(text) => self.matches_str(text), + Key::Owned(text) => self.matches_str(text), + } + } +} + +impl Eq for Key<'_, P> {} + +impl Hash for Key<'_, P> { + #[inline] + fn hash(&self, state: &mut H) { + match self { + Key::Property(word) => word.key_hash(state), + Key::Borrowed(s) => s.hash(state), + Key::Owned(s) => s.hash(state), + } + } +} + +impl PartialOrd for Key<'_, P> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Key<'_, P> { + fn cmp(&self, other: &Self) -> Ordering { + self.to_string().cmp(&other.to_string()) + } +} + +impl PartialEq<&str> for Key<'_, P> { + #[inline] + fn eq(&self, other: &&str) -> bool { + self.matches_str(other) + } +} + +impl<'x, P: Property> From<&'x str> for Key<'x, P> { + fn from(s: &'x str) -> Self { + match P::try_parse(None, s) { + Some(word) => Key::Property(word), + None => Key::Borrowed(s), + } + } +} + +impl<'x, P: Property> From> for Cow<'x, str> { + fn from(s: Key<'x, P>) -> Self { + match s { + Key::Borrowed(s) => Cow::Borrowed(s), + Key::Owned(s) => Cow::Owned(s), + Key::Property(word) => word.to_cow(), + } + } +} + +impl<'x, P: Property> From> for Key<'x, P> { + fn from(s: Cow<'x, str>) -> Self { + match s { + Cow::Borrowed(s) => Key::Borrowed(s), + Cow::Owned(s) => Key::Owned(s), + } + } +} + +impl Key<'_, P> { + #[inline] + pub(crate) fn matches_property(&self, property: &P) -> bool { + match self { + Key::Property(word) => word.key_eq(property), + Key::Borrowed(text) => property.key_eq_str(text), + Key::Owned(text) => property.key_eq_str(text), + } + } + + #[inline] + pub(crate) fn is_opaque(&self) -> bool { + matches!(self, Key::Property(property) if property.is_opaque()) + } + + #[inline] + pub(crate) fn matches_str(&self, text: &str) -> bool { + match self { + Key::Property(word) => word.key_eq_str(text), + Key::Borrowed(other) => *other == text, + Key::Owned(other) => other == text, + } + } + + pub fn to_string(&self) -> Cow<'_, str> { + match self { + Key::Borrowed(s) => Cow::Borrowed(s), + Key::Owned(s) => Cow::Borrowed(s.as_str()), + Key::Property(word) => word.to_cow(), + } + } + + pub fn into_string(self) -> String { + match self { + Key::Borrowed(s) => s.to_owned(), + Key::Owned(s) => s, + Key::Property(word) => word.to_cow().into_owned(), + } + } + + pub fn try_into_property(self) -> Option

{ + match self { + Key::Property(word) => Some(word), + _ => None, + } + } + + pub fn into_owned(self) -> Key<'static, P> { + match self { + Key::Borrowed(s) => Key::Owned(s.to_owned()), + Key::Owned(s) => Key::Owned(s), + Key::Property(word) => Key::Property(word), + } + } + + pub fn to_owned(&self) -> Key<'static, P> { + match self { + Key::Borrowed(s) => Key::Owned(s.to_string()), + Key::Owned(s) => Key::Owned(s.clone()), + Key::Property(word) => Key::Property(word.clone()), + } + } + + pub fn as_property(&self) -> Option<&P> { + match self { + Key::Property(word) => Some(word), + _ => None, + } + } + + pub fn as_string_key(&self) -> Option<&str> { + match self { + Key::Borrowed(s) => Some(s), + Key::Owned(s) => Some(s.as_str()), + Key::Property(_) => None, + } + } +} + +impl<'x, P: Property> From

for Key<'x, P> { + fn from(word: P) -> Self { + Key::Property(word) + } +} + diff --git a/crates/jmap-tools/src/json/mod.rs b/crates/jmap-tools/src/json/mod.rs new file mode 100644 index 00000000..6e92feab --- /dev/null +++ b/crates/jmap-tools/src/json/mod.rs @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub(crate) mod de; +pub(crate) mod deserializer; +pub(crate) mod index; +pub(crate) mod key; +pub(crate) mod num; +pub(crate) mod object_vec; +pub(crate) mod parser; +#[cfg(feature = "rkyv")] +pub(crate) mod rkyv; +pub(crate) mod ser; +pub(crate) mod value; diff --git a/crates/jmap-tools/src/json/num.rs b/crates/jmap-tools/src/json/num.rs new file mode 100644 index 00000000..89fa3105 --- /dev/null +++ b/crates/jmap-tools/src/json/num.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use core::hash::{Hash, Hasher}; + +/// Represents a JSON number, whether integer or floating point. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct Number { + pub(crate) n: N, +} + +impl From for Number { + fn from(n: N) -> Self { + Self { n } + } +} + +#[derive(Copy, Clone)] +pub(crate) enum N { + PosInt(u64), + /// Always less than zero. + NegInt(i64), + Float(f64), +} + +impl Number { + /// If the `Number` is an integer, represent it as i64 if possible. Returns + /// None otherwise. + pub fn as_u64(&self) -> Option { + match self.n { + N::PosInt(v) => Some(v), + _ => None, + } + } + /// If the `Number` is an integer, represent it as u64 if possible. Returns + /// None otherwise. + pub fn as_i64(&self) -> Option { + match self.n { + N::PosInt(n) => { + if n <= i64::MAX as u64 { + Some(n as i64) + } else { + None + } + } + N::NegInt(v) => Some(v), + _ => None, + } + } + + /// Represents the number as f64 if possible. Returns None otherwise. + pub fn as_f64(&self) -> Option { + match self.n { + N::PosInt(n) => Some(n as f64), + N::NegInt(n) => Some(n as f64), + N::Float(n) => Some(n), + } + } + + /// Returns true if the `Number` is a f64. + pub fn is_f64(&self) -> bool { + matches!(self.n, N::Float(_)) + } + + /// Returns true if the `Number` is a u64. + pub fn is_u64(&self) -> bool { + matches!(self.n, N::PosInt(_)) + } + + /// Returns true if the `Number` is an integer between `i64::MIN` and + /// `i64::MAX`. + pub fn is_i64(&self) -> bool { + match self.n { + N::PosInt(v) => v <= i64::MAX as u64, + N::NegInt(_) => true, + N::Float(_) => false, + } + } + + pub fn cast_to_i64(self) -> i64 { + match self.n { + N::PosInt(v) => i64::try_from(v).unwrap_or(i64::MAX), + N::NegInt(v) => v, + N::Float(v) => v as i64, + } + } + + pub fn cast_to_u64(self) -> u64 { + match self.n { + N::PosInt(v) => v, + N::NegInt(_) => 0, + N::Float(v) => v as u64, + } + } + + pub fn try_cast_to_i64(self) -> Result { + match self.n { + N::PosInt(v) => i64::try_from(v).map_err(|_| v as f64), + N::NegInt(v) => Ok(v), + N::Float(v) => Err(v), + } + } + + pub(crate) fn into_json_value(self) -> serde_json::Value { + match self.n { + N::PosInt(n) => serde_json::Value::Number(n.into()), + N::NegInt(n) => serde_json::Value::Number(n.into()), + N::Float(n) => serde_json::value::Number::from_f64(n) + .map_or(serde_json::Value::Null, serde_json::Value::Number), + } + } +} + +impl PartialEq for N { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (N::PosInt(a), N::PosInt(b)) => a == b, + (N::NegInt(a), N::NegInt(b)) => a == b, + (N::Float(a), N::Float(b)) => a == b || (a.is_nan() && b.is_nan()), + _ => false, + } + } +} + +impl Eq for N {} + +impl Hash for N { + fn hash(&self, h: &mut H) { + match *self { + N::PosInt(i) => i.hash(h), + N::NegInt(i) => i.hash(h), + N::Float(f) => { + let bits = if f == 0.0 { + 0.0f64.to_bits() + } else if f.is_nan() { + f64::NAN.to_bits() + } else { + f.to_bits() + }; + bits.hash(h); + } + } + } +} +impl From for Number { + fn from(val: u64) -> Self { + Self { n: N::PosInt(val) } + } +} + +impl From for Number { + fn from(val: i64) -> Self { + let n = match u64::try_from(val) { + Ok(val) => N::PosInt(val), + Err(_) => N::NegInt(val), + }; + Self { n } + } +} + +impl From for Number { + fn from(val: f64) -> Self { + Self { n: N::Float(val) } + } +} + +impl From for Number { + fn from(val: usize) -> Self { + Self { + n: N::PosInt(val as u64), + } + } +} + +impl From for Number { + fn from(val: isize) -> Self { + Self::from(val as i64) + } +} + +impl From for Number { + fn from(val: u32) -> Self { + Self { + n: N::PosInt(val as u64), + } + } +} + +impl From for Number { + fn from(val: i32) -> Self { + Self::from(i64::from(val)) + } +} + +impl TryFrom for serde_json::value::Number { + type Error = f64; + + fn try_from(num: Number) -> Result { + match num.n { + N::PosInt(n) => Ok(n.into()), + N::NegInt(n) => Ok(n.into()), + N::Float(n) => serde_json::value::Number::from_f64(n).ok_or(n), + } + } +} + diff --git a/crates/jmap-tools/src/json/object_vec.rs b/crates/jmap-tools/src/json/object_vec.rs new file mode 100644 index 00000000..15dd04f7 --- /dev/null +++ b/crates/jmap-tools/src/json/object_vec.rs @@ -0,0 +1,343 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#![allow(clippy::useless_conversion)] +#![allow(clippy::useless_asref)] + +use crate::{ + Value, + json::key::Key, + json::value::{Element, Property}, + pointer::NumberKey, +}; +use std::mem::replace; + +const NAMED_KEY_CAPACITY: usize = 24; + +/// Represents a JSON key/value type. +/// +/// For performance reasons we use a Vec instead of a Hashmap. +/// This comes with a tradeoff of slower key accesses as we need to iterate and compare. +/// +/// The ObjectAsVec struct is a wrapper around a Vec of (&str, Value) pairs. +/// It provides methods to make it easy to migrate from serde_json::Value::Object or +/// serde_json::Map. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct ObjectAsVec<'ctx, P: Property, E: Element>( + pub(crate) Vec<(Key<'ctx, P>, Value<'ctx, P, E>)>, +); + +impl<'ctx, P: Property, E: Element> From, Value<'ctx, P, E>)>> + for ObjectAsVec<'ctx, P, E> +{ + fn from(vec: Vec<(Key<'ctx, P>, Value<'ctx, P, E>)>) -> Self { + ObjectAsVec(vec) + } +} + +impl<'ctx, P: Property, E: Element> FromIterator<(Key<'ctx, P>, Value<'ctx, P, E>)> + for ObjectAsVec<'ctx, P, E> +{ + fn from_iter, Value<'ctx, P, E>)>>(iter: T) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl<'ctx, P: Property, E: Element> ObjectAsVec<'ctx, P, E> { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + /// Access to the underlying Vec. + /// + /// # Note + /// Since KeyStrType can be changed via a feature flag avoid using `as_vec` and use other + /// methods instead. This could be a problem with feature unification, when one crate uses it + /// as `&str` and another uses it as `Cow`, both will get `Cow` + #[inline] + pub fn as_vec(&self) -> &Vec<(Key<'ctx, P>, Value<'ctx, P, E>)> { + &self.0 + } + + #[inline] + pub fn as_mut_vec(&mut self) -> &mut Vec<(Key<'ctx, P>, Value<'ctx, P, E>)> { + &mut self.0 + } + + /// Access to the underlying Vec. Keys are normalized to Cow. + #[inline] + pub fn into_vec(self) -> Vec<(Key<'ctx, P>, Value<'ctx, P, E>)> { + self.0 + } + + /// Returns a reference to the value corresponding to the key. + /// + /// ## Performance + /// As this is backed by a Vec, this searches linearly through the Vec as may be much more + /// expensive than a `Hashmap` for larger Objects. + #[inline] + pub fn get(&self, key: &Key<'_, P>) -> Option<&Value<'ctx, P, E>> { + self.position(key) + .and_then(|pos| self.0.get(pos)) + .map(|(_, v)| v) + } + + #[inline] + pub(crate) fn position(&self, key: &Key<'_, P>) -> Option { + let mut keys = self.0.iter().map(|(k, _)| k); + match key { + Key::Property(property) => keys.position(|k| match k { + Key::Property(word) => word.key_eq(property), + Key::Borrowed(text) => property.key_eq_str(text), + Key::Owned(text) => property.key_eq_str(text), + }), + Key::Borrowed(text) => keys.position(|k| k.matches_str(text)), + Key::Owned(text) => keys.position(|k| k.matches_str(text)), + } + } + + #[inline] + pub fn get_ignore_case(&self, key: &str) -> Option<&Value<'ctx, P, E>> { + self.0.iter().find_map(|(k, v)| { + if k.to_string().eq_ignore_ascii_case(key) { + Some(v) + } else { + None + } + }) + } + + /// Returns a mutable reference to the value corresponding to the key, if it exists. + /// + /// ## Performance + /// As this is backed by a Vec, this searches linearly through the Vec as may be much more + /// expensive than a `Hashmap` for larger Objects. + #[inline] + pub fn get_mut(&mut self, key: &Key<'ctx, P>) -> Option<&mut Value<'ctx, P, E>> { + self.position(key) + .and_then(|pos| self.0.get_mut(pos)) + .map(|(_, v)| v) + } + + /// Returns the key-value pair corresponding to the supplied key. + /// + /// ## Performance + /// As this is backed by a Vec, this searches linearly through the Vec as may be much more + /// expensive than a `Hashmap` for larger Objects. + #[inline] + pub fn get_key_value(&self, key: &Key<'_, P>) -> Option<(&Key<'_, P>, &Value<'ctx, P, E>)> { + self.position(key) + .and_then(|pos| self.0.get(pos)) + .map(|(k, v)| (k, v)) + } + + /// An iterator visiting all key-value pairs + #[inline] + pub fn iter(&self) -> impl Iterator, &Value<'ctx, P, E>)> { + self.0.iter().map(|(k, v)| (k, v)) + } + + #[inline] + pub fn iter_mut( + &mut self, + ) -> impl Iterator, &mut Value<'ctx, P, E>)> { + self.0.iter_mut().map(|(k, v)| (k, v)) + } + + /// Returns the number of elements in the object + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns true if the object contains no elements + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// An iterator visiting all keys + #[inline] + pub fn keys(&self) -> impl Iterator> { + self.0.iter().map(|(k, _)| k) + } + + /// An iterator visiting all values + #[inline] + pub fn values(&self) -> impl Iterator> { + self.0.iter().map(|(_, v)| v) + } + + /// Returns true if the object contains a value for the specified key. + /// + /// ## Performance + /// As this is backed by a Vec, this searches linearly through the Vec as may be much more + /// expensive than a `Hashmap` for larger Objects. + #[inline] + pub fn contains_key(&self, key: &Key<'ctx, P>) -> bool { + self.position(key).is_some() + } + + #[inline] + pub fn contains_key_value(&self, key: &Key<'ctx, P>, value: &Value<'ctx, P, E>) -> bool { + self.0.iter().any(|(k, v)| k == key && v == value) + } + + #[inline] + pub fn contains_any_key(&self, keys: &[Key<'ctx, P>]) -> bool { + self.0.iter().any(|(k, _)| keys.contains(k)) + } + + pub fn remove(&mut self, key: &Key<'ctx, P>) -> Option> { + self.position(key).map(|pos| self.0.swap_remove(pos).1) + } + + /// Inserts a key-value pair into the object. + /// If the object did not have this key present, `None` is returned. + /// If the object did have this key present, the value is updated, and the old value is + /// returned. + /// + /// ## Performance + /// This operation is linear in the size of the Vec because it potentially requires iterating + /// through all elements to find a matching key. + #[inline] + pub fn insert( + &mut self, + key: impl Into>, + value: impl Into>, + ) -> Option> { + let key = key.into(); + match self.position(&key).and_then(|pos| self.0.get_mut(pos)) { + Some((_, v)) => Some(replace(v, value.into())), + None => { + self.0.push((key, value.into())); + None + } + } + } + + /// Inserts a key-value pair into the object if the key does not yet exist, otherwise returns a + /// mutable reference to the existing value. + /// + /// ## Performance + /// This operation might be linear in the size of the Vec because it requires iterating through + /// all elements to find a matching key, and might add to the end if not found. + #[inline] + pub fn insert_or_get_mut( + &mut self, + key: impl Into>, + value: impl Into>, + ) -> &mut Value<'ctx, P, E> { + let key = key.into(); + let pos = match self.position(&key) { + Some(pos) => pos, + None => { + self.0.push((key, value.into())); + self.0.len() - 1 + } + }; + &mut self.0[pos].1 + } + + #[inline] + pub fn insert_unchecked( + &mut self, + key: impl Into>, + value: impl Into>, + ) { + self.0.push((key.into(), value.into())); + } + + #[inline] + pub fn with_key_value( + mut self, + key: impl Into>, + value: impl Into>, + ) -> Self { + self.insert_unchecked(key, value); + self + } + + pub fn insert_named(&mut self, key: Option, value: Value<'ctx, P, E>) -> String { + let mut number = NumberKey::default(); + let suffix = number.format(self.0.len() as u64 + 1); + let mut key = key.unwrap_or_else(|| { + let mut key = String::with_capacity(NAMED_KEY_CAPACITY); + key.push('k'); + key.push_str(suffix); + key + }); + + if self.contains_key(&Key::Borrowed(key.as_str())) { + key.push('-'); + key.push_str(suffix); + } + + self.0.push((Key::Owned(key.clone()), value)); + key + } + + #[inline] + pub fn extend(&mut self, iter: I) + where + I: IntoIterator, Value<'ctx, P, E>)>, + { + self.0.extend(iter); + } + + /// Inserts a key-value pair into the object and returns the mutable reference of the inserted + /// value. + /// + /// ## Note + /// The key must not exist in the object. If the key already exists, the object will contain + /// multiple keys afterwards. + /// + /// ## Performance + /// This operation is amortized constant time, worst case linear time in the size of the Vec + /// because it potentially requires a reallocation to grow the Vec. + #[inline] + pub fn insert_unchecked_and_get_mut( + &mut self, + key: impl Into>, + value: impl Into>, + ) -> &mut Value<'ctx, P, E> { + let key = key.into(); + self.0.push((key, value.into())); + let idx = self.0.len() - 1; + &mut self.0[idx].1 + } +} + +impl<'ctx, P: Property, E: Element> ObjectAsVec<'ctx, P, E> { + pub fn into_expanded_boolean_set(self) -> impl Iterator> { + self.into_vec() + .into_iter() + .filter_map(|(key, value)| value.as_bool().filter(|&b| b).map(|_| key)) + } +} + +impl<'ctx, P: Property, E: Element> From> + for serde_json::Map +{ + fn from(val: ObjectAsVec<'ctx, P, E>) -> Self { + val.iter() + .map(|(key, val)| (key.to_string().into_owned(), val.into())) + .collect() + } +} +impl<'ctx, P: Property, E: Element> From<&ObjectAsVec<'ctx, P, E>> + for serde_json::Map +{ + fn from(val: &ObjectAsVec<'ctx, P, E>) -> Self { + val.iter() + .map(|(key, val)| (key.to_string().into_owned(), val.into())) + .collect() + } +} diff --git a/crates/jmap-tools/src/json/parser.rs b/crates/jmap-tools/src/json/parser.rs new file mode 100644 index 00000000..e1c19ab5 --- /dev/null +++ b/crates/jmap-tools/src/json/parser.rs @@ -0,0 +1,755 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::json::key::Key; +use crate::json::object_vec::ObjectAsVec; +use crate::json::value::Value; +use crate::{Element, Property}; +use std::borrow::Cow; + +const MAX_DEPTH: u8 = 128; +const ONES: u64 = u64::MAX / 255; +const HIGH: u64 = ONES << 7; +const QUOTES: u64 = ONES * b'"' as u64; +const BACKSLASHES: u64 = ONES * b'\\' as u64; +const CONTROLS: u64 = ONES * 0x20; +const SWAR_LIMIT: usize = 128; +const BLOCK: usize = 128; + +#[derive(Clone, Copy)] +enum Code { + EofWhileParsingList, + EofWhileParsingObject, + EofWhileParsingString, + EofWhileParsingValue, + ExpectedColon, + ExpectedListCommaOrEnd, + ExpectedObjectCommaOrEnd, + ExpectedSomeIdent, + ExpectedSomeValue, + InvalidEscape, + InvalidNumber, + NumberOutOfRange, + ControlCharacterWhileParsingString, + KeyMustBeAString, + LoneLeadingSurrogateInHexEscape, + TrailingComma, + TrailingCharacters, + UnexpectedEndOfHexEscape, + RecursionLimitExceeded, +} + +impl Code { + fn as_str(self) -> &'static str { + match self { + Code::EofWhileParsingList => "EOF while parsing a list", + Code::EofWhileParsingObject => "EOF while parsing an object", + Code::EofWhileParsingString => "EOF while parsing a string", + Code::EofWhileParsingValue => "EOF while parsing a value", + Code::ExpectedColon => "expected `:`", + Code::ExpectedListCommaOrEnd => "expected `,` or `]`", + Code::ExpectedObjectCommaOrEnd => "expected `,` or `}`", + Code::ExpectedSomeIdent => "expected ident", + Code::ExpectedSomeValue => "expected value", + Code::InvalidEscape => "invalid escape", + Code::InvalidNumber => "invalid number", + Code::NumberOutOfRange => "number out of range", + Code::ControlCharacterWhileParsingString => { + "control character (\\u0000-\\u001F) found while parsing a string" + } + Code::KeyMustBeAString => "key must be a string", + Code::LoneLeadingSurrogateInHexEscape => "lone leading surrogate in hex escape", + Code::TrailingComma => "trailing comma", + Code::TrailingCharacters => "trailing characters", + Code::UnexpectedEndOfHexEscape => "unexpected end of hex escape", + Code::RecursionLimitExceeded => "recursion limit exceeded", + } + } +} + +struct Failure { + code: Code, + index: usize, +} + +impl Failure { + #[cold] + fn message(self, bytes: &[u8]) -> String { + let head = bytes.get(..self.index).unwrap_or(bytes); + let mut lines = head.rsplit(|&byte| byte == b'\n'); + let column = lines.next().map_or(0, <[u8]>::len); + let line = 1 + lines.count(); + format!("{} at line {line} column {column}", self.code.as_str()) + } +} + +type Parsed = Result; + +enum Text<'x> { + Borrowed(&'x str), + Copied, +} + +#[derive(Clone, Copy)] +struct Unscanned<'x>(&'x [u8]); + +pub(crate) struct Parser<'x, P: Property, E: Element> { + text: &'x str, + bytes: &'x [u8], + index: usize, + depth: u8, + opaque: bool, + scratch: String, + members: Vec<(Key<'x, P>, Value<'x, P, E>)>, + items: Vec>, +} + +impl<'x, P: Property, E: Element> Parser<'x, P, E> { + pub(crate) fn parse(json: &'x str) -> Result, String> { + Parser::new(json) + .document() + .map_err(|failure| failure.message(json.as_bytes())) + } + + fn new(json: &'x str) -> Self { + Parser { + text: json, + bytes: json.as_bytes(), + index: 0, + depth: MAX_DEPTH, + opaque: false, + scratch: String::new(), + members: Vec::new(), + items: Vec::new(), + } + } + + fn document(&mut self) -> Parsed> { + let value = self.value(None)?; + match self.whitespace() { + Some(_) => Err(self.peek_error(Code::TrailingCharacters)), + None => Ok(value), + } + } + + #[inline(always)] + fn peek(&self) -> Option { + self.bytes.get(self.index).copied() + } + + #[inline(always)] + fn peek_or_null(&self) -> u8 { + self.bytes.get(self.index).copied().unwrap_or(0) + } + + #[inline(always)] + fn next_char(&mut self) -> Option { + let byte = self.bytes.get(self.index).copied(); + if byte.is_some() { + self.index += 1; + } + byte + } + + #[inline(always)] + fn eat_char(&mut self) { + self.index += 1; + } + + #[cold] + fn error(&self, code: Code) -> Failure { + Failure { + code, + index: self.index, + } + } + + #[cold] + fn peek_error(&self, code: Code) -> Failure { + Failure { + code, + index: self.bytes.len().min(self.index + 1), + } + } + + #[inline(always)] + fn whitespace(&mut self) -> Option { + loop { + match self.peek() { + Some(b' ' | b'\n' | b'\t' | b'\r') => self.eat_char(), + other => return other, + } + } + } + + fn value(&mut self, parent: Option<&Key<'_, P>>) -> Parsed> { + let Some(peek) = self.whitespace() else { + return Err(self.peek_error(Code::EofWhileParsingValue)); + }; + match peek { + b'"' => { + self.eat_char(); + Ok(match self.string()? { + Text::Borrowed(text) => match Self::element(parent, text) { + Some(element) => Value::Element(element), + None => Value::Str(Cow::Borrowed(text)), + }, + Text::Copied => match Self::element(parent, &self.scratch) { + Some(element) => Value::Element(element), + None => Value::Str(Cow::Owned(self.scratch.as_str().to_owned())), + }, + }) + } + b'{' => { + self.enter()?; + let object = self.object(parent); + self.depth += 1; + let object = object?; + self.eat_char(); + Ok(object) + } + b'[' => { + self.enter()?; + let array = self.array(parent); + self.depth += 1; + let array = array?; + self.eat_char(); + Ok(array) + } + b'-' => { + self.eat_char(); + self.integer(false) + } + b'0'..=b'9' => self.integer(true), + b'n' => { + self.eat_char(); + self.ident(b"ull").map(|_| Value::Null) + } + b't' => { + self.eat_char(); + self.ident(b"rue").map(|_| Value::Bool(true)) + } + b'f' => { + self.eat_char(); + self.ident(b"alse").map(|_| Value::Bool(false)) + } + _ => Err(self.peek_error(Code::ExpectedSomeValue)), + } + } + + #[inline(always)] + fn element(parent: Option<&Key<'_, P>>, text: &str) -> Option { + parent + .filter(|key| !key.is_opaque()) + .and_then(|key| E::try_parse::

(key, text)) + } + + #[inline(always)] + fn enter(&mut self) -> Parsed<()> { + self.depth -= 1; + if self.depth == 0 { + return Err(self.peek_error(Code::RecursionLimitExceeded)); + } + self.eat_char(); + Ok(()) + } + + fn object(&mut self, parent: Option<&Key<'_, P>>) -> Parsed> { + let outer = self.opaque; + let opaque = outer || parent.is_some_and(Key::is_opaque); + self.opaque = opaque; + let start = self.members.len(); + let mut first = true; + loop { + let Some(peek) = self.whitespace() else { + return Err(self.peek_error(Code::EofWhileParsingObject)); + }; + if peek == b'}' { + break; + } + if first { + first = false; + if peek != b'"' { + return Err(self.peek_error(Code::KeyMustBeAString)); + } + } else if peek == b',' { + self.eat_char(); + match self.whitespace() { + Some(b'"') => {} + Some(b'}') => return Err(self.peek_error(Code::TrailingComma)), + Some(_) => return Err(self.peek_error(Code::KeyMustBeAString)), + None => return Err(self.peek_error(Code::EofWhileParsingValue)), + } + } else { + return Err(self.peek_error(Code::ExpectedObjectCommaOrEnd)); + } + self.eat_char(); + let key = match self.string()? { + Text::Borrowed(text) if opaque => Key::Borrowed(text), + Text::Copied if opaque => Key::Owned(self.scratch.as_str().to_owned()), + Text::Borrowed(text) => match P::try_parse(parent, text) { + Some(property) => Key::Property(property), + None => Key::Borrowed(text), + }, + Text::Copied => match P::try_parse(parent, &self.scratch) { + Some(property) => Key::Property(property), + None => Key::Owned(self.scratch.as_str().to_owned()), + }, + }; + match self.whitespace() { + Some(b':') => self.eat_char(), + Some(_) => return Err(self.peek_error(Code::ExpectedColon)), + None => return Err(self.peek_error(Code::EofWhileParsingObject)), + } + let value = self.value(if opaque { None } else { Some(&key) })?; + self.members.push((key, value)); + } + self.opaque = outer; + Ok(Value::Object(ObjectAsVec(self.members.split_off(start)))) + } + + fn array(&mut self, parent: Option<&Key<'_, P>>) -> Parsed> { + let outer = self.opaque; + let opaque = outer || parent.is_some_and(Key::is_opaque); + self.opaque = opaque; + let start = self.items.len(); + let mut first = true; + loop { + let Some(peek) = self.whitespace() else { + return Err(self.peek_error(Code::EofWhileParsingList)); + }; + if peek == b']' { + break; + } + if first { + first = false; + } else if peek == b',' { + self.eat_char(); + match self.whitespace() { + Some(b']') => return Err(self.peek_error(Code::TrailingComma)), + Some(_) => {} + None => return Err(self.peek_error(Code::EofWhileParsingValue)), + } + } else { + return Err(self.peek_error(Code::ExpectedListCommaOrEnd)); + } + let item = self.value(if opaque { None } else { parent })?; + self.items.push(item); + } + self.opaque = outer; + Ok(Value::Array(self.items.split_off(start))) + } + + fn ident(&mut self, ident: &[u8]) -> Parsed<()> { + for expected in ident { + match self.next_char() { + None => return Err(self.error(Code::EofWhileParsingValue)), + Some(next) if next != *expected => return Err(self.error(Code::ExpectedSomeIdent)), + Some(_) => {} + } + } + Ok(()) + } + + #[inline(always)] + fn skip_to_escape(&mut self) { + let rest = self.bytes.get(self.index..).unwrap_or_default(); + let Some((&first, tail)) = rest.split_first() else { + return; + }; + if Unscanned::is_escape(first) { + return; + } + let (head, long) = tail.split_at(tail.len().min(SWAR_LIMIT)); + let mut offset = self.index + 1; + let (chunks, remainder) = head.as_chunks::<8>(); + for chunk in chunks { + let word = u64::from_le_bytes(*chunk); + let controls = word.wrapping_sub(CONTROLS) & !word; + let quotes = word ^ QUOTES; + let quotes = quotes.wrapping_sub(ONES) & !quotes; + let backslashes = word ^ BACKSLASHES; + let backslashes = backslashes.wrapping_sub(ONES) & !backslashes; + let found = (controls | quotes | backslashes) & HIGH; + if found != 0 { + self.index = offset + found.trailing_zeros() as usize / 8; + return; + } + offset += 8; + } + self.index = offset + + if long.is_empty() { + Unscanned(remainder).plain_len() + } else { + Unscanned(long).blockwise_plain_len() + }; + } + + fn string(&mut self) -> Parsed> { + let mut start = self.index; + let mut copied = false; + loop { + self.skip_to_escape(); + match self.peek() { + Some(b'"') => { + let run = self.text.get(start..self.index).unwrap_or_default(); + self.eat_char(); + return Ok(if copied { + self.scratch.push_str(run); + Text::Copied + } else { + Text::Borrowed(run) + }); + } + Some(b'\\') => { + if !copied { + copied = true; + self.scratch.clear(); + } + let run = self.text.get(start..self.index).unwrap_or_default(); + self.scratch.push_str(run); + self.eat_char(); + self.escape()?; + start = self.index; + } + Some(_) => { + self.eat_char(); + return Err(self.error(Code::ControlCharacterWhileParsingString)); + } + None => return Err(self.error(Code::EofWhileParsingString)), + } + } + } + + fn escape(&mut self) -> Parsed<()> { + let Some(byte) = self.next_char() else { + return Err(self.error(Code::EofWhileParsingString)); + }; + let decoded = match byte { + b'"' => '"', + b'\\' => '\\', + b'/' => '/', + b'b' => '\x08', + b'f' => '\x0c', + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'u' => return self.unicode_escape(), + _ => return Err(self.error(Code::InvalidEscape)), + }; + self.scratch.push(decoded); + Ok(()) + } + + fn hex_escape(&mut self) -> Parsed { + match self.bytes.get(self.index..) { + Some([a, b, c, d, ..]) => { + self.index += 4; + match ( + Self::hex_digit(*a), + Self::hex_digit(*b), + Self::hex_digit(*c), + Self::hex_digit(*d), + ) { + (Some(a), Some(b), Some(c), Some(d)) => Ok((a << 12) | (b << 8) | (c << 4) | d), + _ => Err(self.error(Code::InvalidEscape)), + } + } + _ => { + self.index = self.bytes.len(); + Err(self.error(Code::EofWhileParsingString)) + } + } + } + + fn hex_digit(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(u16::from(byte - b'0')), + b'A'..=b'F' => Some(u16::from(byte - b'A' + 10)), + b'a'..=b'f' => Some(u16::from(byte - b'a' + 10)), + _ => None, + } + } + + fn unicode_escape(&mut self) -> Parsed<()> { + let high = self.hex_escape()?; + if (0xDC00..=0xDFFF).contains(&high) { + return Err(self.error(Code::LoneLeadingSurrogateInHexEscape)); + } + if !(0xD800..=0xDBFF).contains(&high) { + self.push_code_point(u32::from(high)); + return Ok(()); + } + for expected in *b"\\u" { + match self.peek() { + Some(byte) if byte == expected => self.eat_char(), + Some(_) => { + self.eat_char(); + return Err(self.error(Code::UnexpectedEndOfHexEscape)); + } + None => return Err(self.error(Code::EofWhileParsingString)), + } + } + let low = self.hex_escape()?; + if !(0xDC00..=0xDFFF).contains(&low) { + return Err(self.error(Code::LoneLeadingSurrogateInHexEscape)); + } + self.push_code_point( + (((u32::from(high) - 0xD800) << 10) | (u32::from(low) - 0xDC00)) + 0x1_0000, + ); + Ok(()) + } + + fn push_code_point(&mut self, code: u32) { + if let Some(ch) = char::from_u32(code) { + self.scratch.push(ch); + } + } + + fn integer(&mut self, positive: bool) -> Parsed> { + let Some(next) = self.next_char() else { + return Err(self.error(Code::EofWhileParsingValue)); + }; + match next { + b'0' => match self.peek_or_null() { + b'0'..=b'9' => Err(self.peek_error(Code::InvalidNumber)), + _ => self.number(positive, 0), + }, + b'1'..=b'9' => { + let mut significand = u64::from(next - b'0'); + loop { + match self.peek_or_null() { + byte @ b'0'..=b'9' => { + let Some(next) = Self::append_digit(significand, byte) else { + return self + .long_integer(positive, significand) + .map(|value| Value::Number(value.into())); + }; + self.eat_char(); + significand = next; + } + _ => return self.number(positive, significand), + } + } + } + _ => Err(self.error(Code::InvalidNumber)), + } + } + + #[inline(always)] + fn append_digit(significand: u64, digit: u8) -> Option { + significand + .checked_mul(10) + .and_then(|value| value.checked_add(u64::from(digit - b'0'))) + } + + fn number(&mut self, positive: bool, significand: u64) -> Parsed> { + Ok(Value::Number(match self.peek_or_null() { + b'.' => self.decimal(positive, significand, 0)?.into(), + b'e' | b'E' => self.exponent(positive, significand, 0)?.into(), + _ if positive => significand.into(), + _ => { + let negative = (significand as i64).wrapping_neg(); + if negative >= 0 { + (-(significand as f64)).into() + } else { + negative.into() + } + } + })) + } + + fn decimal(&mut self, positive: bool, mut significand: u64, before: i32) -> Parsed { + self.eat_char(); + let mut after = 0; + while let byte @ b'0'..=b'9' = self.peek_or_null() { + let Some(next) = Self::append_digit(significand, byte) else { + return self.decimal_overflow(positive, significand, before + after); + }; + self.eat_char(); + significand = next; + after -= 1; + } + if after == 0 { + return Err(match self.peek() { + Some(_) => self.peek_error(Code::InvalidNumber), + None => self.peek_error(Code::EofWhileParsingValue), + }); + } + match self.peek_or_null() { + b'e' | b'E' => self.exponent(positive, significand, before + after), + _ => self.float(positive, significand, before + after), + } + } + + fn exponent(&mut self, positive: bool, significand: u64, starting: i32) -> Parsed { + self.eat_char(); + let positive_exponent = match self.peek_or_null() { + b'+' => { + self.eat_char(); + true + } + b'-' => { + self.eat_char(); + false + } + _ => true, + }; + let mut exponent = match self.next_char() { + Some(byte @ b'0'..=b'9') => i32::from(byte - b'0'), + Some(_) => return Err(self.error(Code::InvalidNumber)), + None => return Err(self.error(Code::EofWhileParsingValue)), + }; + while let byte @ b'0'..=b'9' = self.peek_or_null() { + self.eat_char(); + let Some(next) = exponent + .checked_mul(10) + .and_then(|value| value.checked_add(i32::from(byte - b'0'))) + else { + return self.exponent_overflow(positive, significand == 0, positive_exponent); + }; + exponent = next; + } + let exponent = if positive_exponent { + starting.saturating_add(exponent) + } else { + starting.saturating_sub(exponent) + }; + self.float(positive, significand, exponent) + } + + fn float(&self, positive: bool, significand: u64, mut exponent: i32) -> Parsed { + let mut value = significand as f64; + loop { + match POW10.get(exponent.wrapping_abs() as usize) { + Some(&power) => { + if exponent >= 0 { + value *= power; + if value.is_infinite() { + return Err(self.error(Code::NumberOutOfRange)); + } + } else { + value /= power; + } + break; + } + None => { + if value == 0.0 { + break; + } + if exponent >= 0 { + return Err(self.error(Code::NumberOutOfRange)); + } + value /= 1e308; + exponent += 308; + } + } + } + Ok(if positive { value } else { -value }) + } + + #[cold] + fn long_integer(&mut self, positive: bool, significand: u64) -> Parsed { + let mut exponent = 0; + loop { + match self.peek_or_null() { + b'0'..=b'9' => { + self.eat_char(); + exponent += 1; + } + b'.' => return self.decimal(positive, significand, exponent), + b'e' | b'E' => return self.exponent(positive, significand, exponent), + _ => return self.float(positive, significand, exponent), + } + } + } + + #[cold] + fn decimal_overflow(&mut self, positive: bool, significand: u64, exponent: i32) -> Parsed { + while let b'0'..=b'9' = self.peek_or_null() { + self.eat_char(); + } + match self.peek_or_null() { + b'e' | b'E' => self.exponent(positive, significand, exponent), + _ => self.float(positive, significand, exponent), + } + } + + #[cold] + fn exponent_overflow( + &mut self, + positive: bool, + zero_significand: bool, + positive_exponent: bool, + ) -> Parsed { + if !zero_significand && positive_exponent { + return Err(self.error(Code::NumberOutOfRange)); + } + while let b'0'..=b'9' = self.peek_or_null() { + self.eat_char(); + } + Ok(if positive { 0.0 } else { -0.0 }) + } +} + +impl Unscanned<'_> { + #[inline(always)] + fn is_escape(byte: u8) -> bool { + (byte == b'"') | (byte == b'\\') | (byte < 0x20) + } + + #[inline(always)] + fn plain_len(self) -> usize { + self.0 + .iter() + .position(|&byte| Self::is_escape(byte)) + .unwrap_or(self.0.len()) + } + + #[inline(never)] + fn blockwise_plain_len(self) -> usize { + let mut offset = 0; + let (blocks, remainder) = self.0.as_chunks::(); + for block in blocks { + if block + .iter() + .fold(false, |found, &byte| found | Self::is_escape(byte)) + { + return offset + Unscanned(block).plain_len(); + } + offset += BLOCK; + } + offset + Unscanned(remainder).plain_len() + } +} + +static POW10: [f64; 309] = [ + 1e000, 1e001, 1e002, 1e003, 1e004, 1e005, 1e006, 1e007, 1e008, 1e009, 1e010, 1e011, 1e012, + 1e013, 1e014, 1e015, 1e016, 1e017, 1e018, 1e019, 1e020, 1e021, 1e022, 1e023, 1e024, 1e025, + 1e026, 1e027, 1e028, 1e029, 1e030, 1e031, 1e032, 1e033, 1e034, 1e035, 1e036, 1e037, 1e038, + 1e039, 1e040, 1e041, 1e042, 1e043, 1e044, 1e045, 1e046, 1e047, 1e048, 1e049, 1e050, 1e051, + 1e052, 1e053, 1e054, 1e055, 1e056, 1e057, 1e058, 1e059, 1e060, 1e061, 1e062, 1e063, 1e064, + 1e065, 1e066, 1e067, 1e068, 1e069, 1e070, 1e071, 1e072, 1e073, 1e074, 1e075, 1e076, 1e077, + 1e078, 1e079, 1e080, 1e081, 1e082, 1e083, 1e084, 1e085, 1e086, 1e087, 1e088, 1e089, 1e090, + 1e091, 1e092, 1e093, 1e094, 1e095, 1e096, 1e097, 1e098, 1e099, 1e100, 1e101, 1e102, 1e103, + 1e104, 1e105, 1e106, 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, 1e115, 1e116, + 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, + 1e130, 1e131, 1e132, 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, 1e142, + 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, 1e151, 1e152, 1e153, 1e154, 1e155, + 1e156, 1e157, 1e158, 1e159, 1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, + 1e169, 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, 1e178, 1e179, 1e180, 1e181, + 1e182, 1e183, 1e184, 1e185, 1e186, 1e187, 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, + 1e195, 1e196, 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, 1e205, 1e206, 1e207, + 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, 1e214, 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, + 1e221, 1e222, 1e223, 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, 1e232, 1e233, + 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, + 1e247, 1e248, 1e249, 1e250, 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259, + 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, 1e268, 1e269, 1e270, 1e271, 1e272, + 1e273, 1e274, 1e275, 1e276, 1e277, 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, + 1e286, 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, 1e295, 1e296, 1e297, 1e298, + 1e299, 1e300, 1e301, 1e302, 1e303, 1e304, 1e305, 1e306, 1e307, 1e308, +]; + diff --git a/crates/jmap-tools/src/json/rkyv.rs b/crates/jmap-tools/src/json/rkyv.rs new file mode 100644 index 00000000..68fe2184 --- /dev/null +++ b/crates/jmap-tools/src/json/rkyv.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Element, Property, Value}; +use rkyv::{ + option::ArchivedOption, + primitive::{ArchivedU32, ArchivedU64}, + string::ArchivedString, +}; + +impl<'ctx, P: Property, E: Element, T> From<&ArchivedOption> for Value<'ctx, P, E> +where + for<'x> &'x T: Into>, +{ + fn from(value: &ArchivedOption) -> Self { + match value { + ArchivedOption::Some(value) => value.into(), + ArchivedOption::None => Value::Null, + } + } +} + +impl<'ctx, P: Property, E: Element> From<&ArchivedString> for Value<'ctx, P, E> { + fn from(value: &ArchivedString) -> Self { + Value::Str(value.to_string().into()) + } +} + +impl<'ctx, P: Property, E: Element> From<&ArchivedU32> for Value<'ctx, P, E> { + fn from(value: &ArchivedU32) -> Self { + Value::Number(u32::from(value).into()) + } +} + +impl<'ctx, P: Property, E: Element> From<&ArchivedU64> for Value<'ctx, P, E> { + fn from(value: &ArchivedU64) -> Self { + Value::Number(u64::from(value).into()) + } +} diff --git a/crates/jmap-tools/src/json/ser.rs b/crates/jmap-tools/src/json/ser.rs new file mode 100644 index 00000000..2469579f --- /dev/null +++ b/crates/jmap-tools/src/json/ser.rs @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2021 Pascal Seitz + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::json::num::{N, Number}; +use crate::json::value::Value; +use crate::{Element, Map, Property}; +use serde::ser::{Serialize, Serializer}; + +impl Serialize for Value<'_, P, E> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Value::Null => serializer.serialize_unit(), + Value::Bool(b) => serializer.serialize_bool(*b), + Value::Number(n) => n.serialize(serializer), + Value::Str(s) => serializer.serialize_str(s), + Value::Array(v) => serializer.collect_seq(v), + Value::Object(m) => m.serialize(serializer), + Value::Element(e) => e.serialize_text(serializer), + } + } +} +impl Serialize for Map<'_, P, E> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_map(self.iter()) + } +} + +impl Serialize for Number { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.n { + N::PosInt(n) => serializer.serialize_u64(n), + N::NegInt(n) => serializer.serialize_i64(n), + N::Float(n) => serializer.serialize_f64(n), + } + } +} + diff --git a/crates/jmap-tools/src/json/value.rs b/crates/jmap-tools/src/json/value.rs new file mode 100644 index 00000000..decf6e01 --- /dev/null +++ b/crates/jmap-tools/src/json/value.rs @@ -0,0 +1,599 @@ +use serde::{Serialize, Serializer}; + +use crate::json::index::Index; +use crate::json::key::Key; +use crate::json::num::{N, Number}; +pub use crate::json::object_vec::ObjectAsVec; +use crate::json::parser::Parser; +use crate::pointer::PointerDepth; +use core::fmt; +use core::hash::{Hash, Hasher}; +use std::borrow::Cow; +use std::fmt::{Debug, Display}; +use std::str::FromStr; + +/// Represents any valid JMAP value. +#[derive(Clone, Eq, PartialEq, Hash, Default)] +pub enum Value<'ctx, P: Property, E: Element> { + #[default] + Null, + Bool(bool), + Number(Number), + Element(E), + Str(Cow<'ctx, str>), + Array(Vec>), + Object(ObjectAsVec<'ctx, P, E>), +} + +pub trait Property: Debug + Clone + PartialEq + Eq + PartialOrd + Ord + Hash { + fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option; + fn to_cow(&self) -> Cow<'static, str>; + + fn try_parse_nested(key: Option<&Key<'_, Self>>, value: &str, _: PointerDepth) -> Option { + Self::try_parse(key, value) + } + + fn is_opaque(&self) -> bool { + false + } + + fn key_eq(&self, other: &Self) -> bool { + self.to_cow() == other.to_cow() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.to_cow() == other + } + + fn key_hash(&self, state: &mut H) { + self.to_cow().hash(state) + } + + fn serialize_text(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_cow()) + } +} + +pub trait Element: Clone + PartialEq + Eq + Hash + Debug + Sized { + type Property: Property; + + fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option; + fn to_cow(&self) -> Cow<'static, str>; + + fn serialize_text(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_cow()) + } +} + +impl<'ctx, P: Property, E: Element> Value<'ctx, P, E> { + pub fn new_object() -> Self { + Value::Object(ObjectAsVec::from(Vec::new())) + } + + pub fn new_boolean_set(set: impl IntoIterator, bool)>) -> Self { + let mut obj = Vec::new(); + for (key, value) in set { + obj.push((key, value.into())); + } + Value::Object(obj.into()) + } + + pub fn parse_json(json: &'ctx str) -> Result { + Parser::parse(json) + } + + /// Returns a reference to the value corresponding to the key. + #[inline] + pub fn get>(&'ctx self, index: I) -> &'ctx Value<'ctx, P, E> { + index.index_into(self).unwrap_or(&Value::Null) + } + + pub fn is_object_and_contains_key(&self, key: &Key<'_, P>) -> bool { + match self { + Value::Object(obj) => obj.contains_key(key), + _ => false, + } + } + + pub fn is_object_and_contains_any_key(&self, keys: &[Key<'_, P>]) -> bool { + match self { + Value::Object(obj) => obj.contains_any_key(keys), + _ => false, + } + } + + #[inline] + pub fn as_object_and_get(&self, key: &Key<'_, P>) -> Option<&Value<'ctx, P, E>> { + match self { + Value::Object(obj) => obj.get(key), + _ => None, + } + } + + /// Returns true if `Value` is Value::Null. + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + /// Returns true if `Value` is Value::Array. + pub fn is_array(&self) -> bool { + matches!(self, Value::Array(_)) + } + + /// Returns true if `Value` is Value::Object. + pub fn is_object(&self) -> bool { + matches!(self, Value::Object(_)) + } + + /// Returns true if `Value` is Value::Bool. + pub fn is_bool(&self) -> bool { + matches!(self, Value::Bool(_)) + } + + /// Returns true if `Value` is Value::Number. + pub fn is_number(&self) -> bool { + matches!(self, Value::Number(_)) + } + + /// Returns true if `Value` is Value::Str. + pub fn is_string(&self) -> bool { + matches!(self, Value::Str(_)) + } + + /// Returns true if the Value is an integer between i64::MIN and i64::MAX. + /// For any Value on which is_i64 returns true, as_i64 is guaranteed to return the integer + /// value. + pub fn is_i64(&self) -> bool { + match self { + Value::Number(n) => n.is_i64(), + _ => false, + } + } + + /// Returns true if the Value is an integer between zero and u64::MAX. + /// For any Value on which is_u64 returns true, as_u64 is guaranteed to return the integer + /// value. + pub fn is_u64(&self) -> bool { + match self { + Value::Number(n) => n.is_u64(), + _ => false, + } + } + + /// Returns true if the Value is a f64 number. + pub fn is_f64(&self) -> bool { + match self { + Value::Number(n) => n.is_f64(), + _ => false, + } + } + + /// If the Value is an Array, returns an iterator over the elements in the array. + pub fn iter_array(&self) -> Option>> { + match self { + Value::Array(arr) => Some(arr.iter()), + _ => None, + } + } + + /// If the Value is an Object, returns an iterator over the elements in the object. + pub fn iter_object(&self) -> Option, &Value<'_, P, E>)>> { + match self { + Value::Object(arr) => Some(arr.iter()), + _ => None, + } + } + + /// If the Value is an Array, returns the associated Array. Returns None otherwise. + pub fn as_array(&self) -> Option<&[Value<'ctx, P, E>]> { + match self { + Value::Array(arr) => Some(arr), + _ => None, + } + } + + pub fn as_array_mut(&mut self) -> Option<&mut Vec>> { + match self { + Value::Array(arr) => Some(arr), + _ => None, + } + } + + pub fn into_array(self) -> Option>> { + match self { + Value::Array(arr) => Some(arr), + _ => None, + } + } + + /// If the Value is an Object, returns the associated Object. Returns None otherwise. + pub fn as_object(&self) -> Option<&ObjectAsVec<'ctx, P, E>> { + match self { + Value::Object(obj) => Some(obj), + _ => None, + } + } + + pub fn as_element(&self) -> Option<&E> { + match self { + Value::Element(element) => Some(element), + _ => None, + } + } + + pub fn as_object_mut(&mut self) -> Option<&mut ObjectAsVec<'ctx, P, E>> { + match self { + Value::Object(obj) => Some(obj), + _ => None, + } + } + + pub fn into_object(self) -> Option> { + match self { + Value::Object(obj) => Some(obj), + _ => None, + } + } + + pub fn into_owned(self) -> Value<'static, P, E> { + match self { + Value::Null => Value::Null, + Value::Bool(b) => Value::Bool(b), + Value::Number(n) => Value::Number(n), + Value::Element(e) => Value::Element(e), + Value::Str(s) => Value::Str(Cow::Owned(s.into_owned())), + Value::Array(arr) => { + let owned_arr: Vec> = + arr.into_iter().map(|v| v.into_owned()).collect(); + Value::Array(owned_arr) + } + Value::Object(obj) => Value::Object( + obj.into_vec() + .into_iter() + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect(), + ), + } + } + + pub fn into_expanded_object(self) -> impl Iterator, Value<'ctx, P, E>)> { + self.into_object() + .map(|obj| obj.into_vec()) + .unwrap_or_default() + .into_iter() + } + + pub fn into_expanded_boolean_set(self) -> impl Iterator> { + self.into_object() + .map(|obj| obj.into_vec()) + .unwrap_or_default() + .into_iter() + .filter_map(|(key, value)| value.as_bool().filter(|&b| b).map(|_| key)) + } + + pub fn into_element(self) -> Option { + match self { + Value::Element(element) => Some(element), + _ => None, + } + } + + /// If the Value is a Boolean, returns the associated bool. Returns None otherwise. + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(b) => Some(*b), + _ => None, + } + } + + /// If the Value is a String, returns the associated str. Returns None otherwise. + pub fn as_str(&self) -> Option> { + match self { + Value::Str(text) => Some(text.as_ref().into()), + Value::Element(element) => Some(element.to_cow()), + _ => None, + } + } + + pub fn into_string(self) -> Option> { + match self { + Value::Str(text) => Some(text), + Value::Element(element) => Some(element.to_cow()), + _ => None, + } + } + + pub fn into_owned_string(self) -> Option { + match self { + Value::Str(text) => Some(text.into_owned()), + Value::Element(element) => Some(element.to_cow().into_owned()), + _ => None, + } + } + + /// If the Value is an integer, represent it as i64 if possible. Returns None otherwise. + pub fn as_i64(&self) -> Option { + match self { + Value::Number(n) => n.as_i64(), + _ => None, + } + } + + /// If the Value is an integer, represent it as u64 if possible. Returns None otherwise. + pub fn as_u64(&self) -> Option { + match self { + Value::Number(n) => n.as_u64(), + _ => None, + } + } + + /// If the Value is a number, represent it as f64 if possible. Returns None otherwise. + pub fn as_f64(&self) -> Option { + match self { + Value::Number(n) => n.as_f64(), + _ => None, + } + } +} + +impl From for Value<'_, P, E> { + fn from(val: bool) -> Self { + Value::Bool(val) + } +} + +impl<'a, P: Property, E: Element> From<&'a str> for Value<'a, P, E> { + fn from(val: &'a str) -> Self { + Value::Str(Cow::Borrowed(val)) + } +} + +impl From for Value<'_, P, E> { + fn from(val: String) -> Self { + Value::Str(Cow::Owned(val)) + } +} + +impl<'x, P: Property, E: Element> From> for Value<'x, P, E> { + fn from(val: Cow<'x, str>) -> Self { + Value::Str(val) + } +} + +impl<'a, P: Property, E: Element, T: Into>> From> for Value<'a, P, E> { + fn from(val: Vec) -> Self { + Value::Array(val.into_iter().map(Into::into).collect()) + } +} + +impl<'x, P: Property, E: Element, T: Into>> From> for Value<'x, P, E> { + fn from(val: Option) -> Self { + match val { + Some(v) => v.into(), + None => Value::Null, + } + } +} + +impl<'x, P: Property, E: Element> From for Value<'x, P, E> { + fn from(element: E) -> Self { + Value::Element(element) + } +} + +impl<'a, P: Property, E: Element, T: Clone + Into>> From<&[T]> for Value<'a, P, E> { + fn from(val: &[T]) -> Self { + Value::Array(val.iter().map(Clone::clone).map(Into::into).collect()) + } +} + +impl Debug for Value<'_, P, E> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Value::Null => formatter.write_str("Null"), + Value::Bool(boolean) => write!(formatter, "Bool({})", boolean), + Value::Number(number) => match number.n { + N::PosInt(n) => write!(formatter, "Number({:?})", n), + N::NegInt(n) => write!(formatter, "Number({:?})", n), + N::Float(n) => write!(formatter, "Number({:?})", n), + }, + Value::Str(string) => write!(formatter, "Str({:?})", string), + Value::Array(vec) => { + formatter.write_str("Array ")?; + Debug::fmt(vec, formatter) + } + Value::Object(map) => { + formatter.write_str("Object ")?; + Debug::fmt(map, formatter) + } + Value::Element(element) => write!(formatter, "Element({})", element.to_cow()), + } + } +} + +// We just convert to serde_json::Value to Display +impl Display for Value<'_, P, E> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", serde_json::Value::from(self.clone())) + } +} + +impl From for Value<'_, P, E> { + fn from(val: u64) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: i64) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: f64) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: u32) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: i32) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: usize) -> Self { + Value::Number(val.into()) + } +} + +impl From for Value<'_, P, E> { + fn from(val: isize) -> Self { + Value::Number(val.into()) + } +} + +impl<'ctx, P: Property, E: Element> From> for Value<'ctx, P, E> { + fn from(val: ObjectAsVec<'ctx, P, E>) -> Self { + Value::Object(val) + } +} + +impl From> for serde_json::Value { + fn from(val: Value<'_, P, E>) -> Self { + match val { + Value::Null => serde_json::Value::Null, + Value::Bool(val) => serde_json::Value::Bool(val), + Value::Number(val) => val.into_json_value(), + Value::Str(val) => serde_json::Value::String(val.to_string()), + Value::Array(vals) => { + serde_json::Value::Array(vals.into_iter().map(|val| val.into()).collect()) + } + Value::Object(vals) => serde_json::Value::Object(vals.into()), + Value::Element(element) => serde_json::Value::String(element.to_cow().to_string()), + } + } +} + +impl From<&Value<'_, P, E>> for serde_json::Value { + fn from(val: &Value<'_, P, E>) -> Self { + match val { + Value::Null => serde_json::Value::Null, + Value::Bool(val) => serde_json::Value::Bool(*val), + Value::Number(val) => val.into_json_value(), + Value::Str(val) => serde_json::Value::String(val.to_string()), + Value::Array(vals) => { + serde_json::Value::Array(vals.iter().map(|val| val.into()).collect()) + } + Value::Object(vals) => serde_json::Value::Object(vals.into()), + Value::Element(element) => serde_json::Value::String(element.to_cow().to_string()), + } + } +} + +impl<'ctx, P: Property, E: Element> From<&'ctx serde_json::Value> for Value<'ctx, P, E> { + fn from(value: &'ctx serde_json::Value) -> Self { + match value { + serde_json::Value::Null => Value::Null, + serde_json::Value::Bool(b) => Value::Bool(*b), + serde_json::Value::Number(n) => { + if let Some(n) = n.as_i64() { + Value::Number(n.into()) + } else if let Some(n) = n.as_u64() { + Value::Number(n.into()) + } else if let Some(n) = n.as_f64() { + Value::Number(n.into()) + } else { + unreachable!() + } + } + serde_json::Value::String(val) => Value::Str(Cow::Borrowed(val)), + serde_json::Value::Array(arr) => { + let out: Vec> = arr.iter().map(|v| v.into()).collect(); + Value::Array(out) + } + serde_json::Value::Object(obj) => { + let mut ans = ObjectAsVec(Vec::with_capacity(obj.len())); + for (k, v) in obj { + ans.insert(Key::Borrowed(k.as_str()), v); + } + Value::Object(ans) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +pub struct Null; + +impl Display for Null { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "null") + } +} + +impl AsRef for Null { + fn as_ref(&self) -> &str { + "null" + } +} + +impl FromStr for Null { + type Err = (); + + fn from_str(_: &str) -> Result { + Err(()) + } +} + +impl Property for Null { + fn try_parse(_: Option<&Key<'_, Self>>, _: &str) -> Option { + None + } + + fn to_cow(&self) -> Cow<'static, str> { + "".into() + } +} + +impl Element for Null { + type Property = Null; + + fn try_parse

(_: &Key<'_, Self::Property>, _: &str) -> Option { + None + } + + fn to_cow(&self) -> Cow<'static, str> { + "".into() + } +} + +impl Property for () { + fn try_parse(_: Option<&Key<'_, Self>>, _: &str) -> Option { + None + } + + fn to_cow(&self) -> Cow<'static, str> { + "".into() + } +} + +impl Element for () { + type Property = (); + + fn try_parse

(_: &Key<'_, Self::Property>, _: &str) -> Option { + None + } + + fn to_cow(&self) -> Cow<'static, str> { + "".into() + } +} + diff --git a/crates/jmap-tools/src/lib.rs b/crates/jmap-tools/src/lib.rs new file mode 100644 index 00000000..e1c686d6 --- /dev/null +++ b/crates/jmap-tools/src/lib.rs @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ +#![deny(rust_2018_idioms)] +#![forbid(unsafe_code)] + +mod json; +mod pointer; + +pub use json::key::Key; +pub use json::object_vec::{ObjectAsVec, ObjectAsVec as Map}; +pub use json::value::{Element, Null, Property, Value}; +pub use pointer::{ + JsonPointer, JsonPointerHandler, JsonPointerItem, JsonPointerIter, PointerDepth, +}; diff --git a/crates/jmap-tools/src/pointer/eval.rs b/crates/jmap-tools/src/pointer/eval.rs new file mode 100644 index 00000000..64df7476 --- /dev/null +++ b/crates/jmap-tools/src/pointer/eval.rs @@ -0,0 +1,319 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{JsonPointerHandler, JsonPointerItem, NumberKey}; +use crate::json::key::Key; +use crate::pointer::JsonPointerIter; +use crate::{Element, ObjectAsVec, Property, Value}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::hash::BuildHasher; + +impl<'x, P: Property, E: Element> JsonPointerHandler<'x, P, E> for Value<'x, P, E> { + fn eval_jptr<'y>( + &'y self, + mut pointer: JsonPointerIter<'_, P>, + results: &mut Vec>>, + ) { + match pointer.next() { + Some(JsonPointerItem::Key(key)) => { + if let Value::Object(map) = self + && let Some(v) = map.get(key) + { + v.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Number(n)) => match self { + Value::Array(values) => { + if let Some(v) = usize::try_from(*n).ok().and_then(|index| values.get(index)) { + v.eval_jptr(pointer, results); + } + } + Value::Object(map) => { + let mut buf = NumberKey::default(); + if let Some(v) = map.get(&Key::Borrowed(buf.format(*n))) { + v.eval_jptr(pointer, results); + } + } + _ => {} + }, + Some(JsonPointerItem::Wildcard) => match self { + Value::Array(values) => { + for v in values { + v.eval_jptr(pointer.clone(), results); + } + } + Value::Object(map) => { + for v in map.values() { + v.eval_jptr(pointer.clone(), results); + } + } + _ => {} + }, + Some(JsonPointerItem::Invalid(_)) => {} + Some(JsonPointerItem::Root) | None => { + results.push(Cow::Borrowed(self)); + } + } + } + + fn patch_jptr<'y: 'x>( + &mut self, + mut pointer: JsonPointerIter<'_, P>, + value: Value<'y, P, E>, + ) -> bool { + match pointer.next() { + Some(JsonPointerItem::Key(key)) => { + if let Value::Object(map) = self { + return map.patch_entry(key, || key.clone(), pointer, value); + } + } + Some(JsonPointerItem::Number(n)) => match self { + Value::Array(values) => { + if let Some(item) = usize::try_from(*n) + .ok() + .and_then(|index| values.get_mut(index)) + { + return match (pointer.peek().is_some(), value) { + (true, value) => item.patch_jptr(pointer, value), + (false, Value::Null) => false, + (false, value) => { + *item = value; + true + } + }; + } + } + Value::Object(map) => { + let mut buf = NumberKey::default(); + let digits = buf.format(*n); + return map.patch_entry( + &Key::Borrowed(digits), + || Key::Owned(digits.to_string()), + pointer, + value, + ); + } + _ => {} + }, + Some( + JsonPointerItem::Wildcard | JsonPointerItem::Root | JsonPointerItem::Invalid(_), + ) + | None => (), + } + + false + } + + fn to_value<'y>(&'y self) -> Cow<'y, Value<'x, P, E>> { + Cow::Borrowed(self) + } +} + +impl<'x, P: Property, E: Element> ObjectAsVec<'x, P, E> { + fn patch_entry<'y: 'x>( + &mut self, + key: &Key<'_, P>, + new_key: impl FnOnce() -> Key<'static, P>, + mut pointer: JsonPointerIter<'_, P>, + value: Value<'y, P, E>, + ) -> bool { + let Some(pos) = self.position(key) else { + return match (pointer.peek().is_none(), value) { + (false, _) => false, + (true, Value::Null) => true, + (true, value) => { + self.insert_unchecked(new_key(), value); + true + } + }; + }; + + if pointer.peek().is_some() { + self.0 + .get_mut(pos) + .is_some_and(|(_, item)| item.patch_jptr(pointer, value)) + } else if matches!(value, Value::Null) { + self.0.remove(pos); + true + } else if let Some((_, item)) = self.0.get_mut(pos) { + *item = value; + true + } else { + false + } + } +} + +impl<'x, P: Property, E: Element, T> JsonPointerHandler<'x, P, E> for Vec +where + T: JsonPointerHandler<'x, P, E> + for<'y> TryFrom> + 'static, +{ + fn eval_jptr<'y>( + &'y self, + mut pointer: JsonPointerIter<'_, P>, + results: &mut Vec>>, + ) { + match pointer.next() { + Some(JsonPointerItem::Number(n)) => { + if let Some(v) = usize::try_from(*n).ok().and_then(|index| self.get(index)) { + v.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Wildcard) => { + for v in self { + v.eval_jptr(pointer.clone(), results); + } + } + Some(JsonPointerItem::Root) | None => { + results.push(self.to_value()); + } + _ => {} + } + } + + fn patch_jptr<'y: 'x>( + &mut self, + mut pointer: JsonPointerIter<'_, P>, + value: Value<'y, P, E>, + ) -> bool { + if let Some(JsonPointerItem::Number(n)) = pointer.next() + && let Some(item) = usize::try_from(*n) + .ok() + .and_then(|index| self.get_mut(index)) + { + if pointer.peek().is_some() { + return item.patch_jptr(pointer, value); + } else if !matches!(value, Value::Null) + && let Ok(value) = T::try_from(value) + { + *item = value; + return true; + } + } + false + } + + fn to_value<'y>(&'y self) -> Cow<'y, Value<'x, P, E>> { + Cow::Owned(Value::Array( + self.iter().map(|v| v.to_value().into_owned()).collect(), + )) + } +} + +impl<'x, P: Property, E: Element, T> TryFrom> for Vec +where + T: JsonPointerHandler<'x, P, E> + for<'y> TryFrom> + 'static, +{ + type Error = (); + + fn try_from(value: Value<'x, P, E>) -> Result { + if let Value::Array(arr) = value { + arr.into_iter() + .map(T::try_from) + .collect::, _>>() + .map_err(|_| ()) + } else { + Err(()) + } + } +} + +impl<'x, P: Property, E: Element, T, S: BuildHasher + Default> JsonPointerHandler<'x, P, E> + for HashMap +where + T: JsonPointerHandler<'x, P, E> + for<'y> TryFrom> + 'static, +{ + fn eval_jptr<'y>( + &'y self, + mut pointer: JsonPointerIter<'_, P>, + results: &mut Vec>>, + ) { + match pointer.next() { + Some(JsonPointerItem::Key(key)) => { + if let Some(v) = self.get(key.to_string().as_ref()) { + v.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Number(n)) => { + let mut buf = NumberKey::default(); + if let Some(v) = self.get(buf.format(*n)) { + v.eval_jptr(pointer, results); + } + } + Some(JsonPointerItem::Wildcard) => { + for v in self.values() { + v.eval_jptr(pointer.clone(), results); + } + } + Some(JsonPointerItem::Invalid(_)) => {} + Some(JsonPointerItem::Root) | None => { + results.push(self.to_value()); + } + } + } + + fn patch_jptr<'y: 'x>( + &mut self, + mut pointer: JsonPointerIter<'_, P>, + value: Value<'y, P, E>, + ) -> bool { + let mut buf = NumberKey::default(); + let key = match pointer.next() { + Some(JsonPointerItem::Key(key)) => key.to_string(), + Some(JsonPointerItem::Number(n)) => Cow::Borrowed(buf.format(*n)), + Some( + JsonPointerItem::Wildcard | JsonPointerItem::Root | JsonPointerItem::Invalid(_), + ) + | None => return false, + }; + + if pointer.peek().is_some() { + self.get_mut(key.as_ref()) + .is_some_and(|item| item.patch_jptr(pointer, value)) + } else if matches!(value, Value::Null) { + self.remove(key.as_ref()); + true + } else if let Ok(value) = T::try_from(value) { + if let Some(item) = self.get_mut(key.as_ref()) { + *item = value; + } else { + self.insert(key.into_owned(), value); + } + true + } else { + false + } + } + + fn to_value<'y>(&'y self) -> Cow<'y, Value<'x, P, E>> { + Cow::Owned(Value::Object( + self.iter() + .map(|(k, v)| (Key::Owned(k.to_string()), v.to_value().into_owned())) + .collect(), + )) + } +} + +impl<'x, P: Property, E: Element, T> TryFrom> for HashMap +where + T: JsonPointerHandler<'x, P, E> + for<'y> TryFrom> + 'static, +{ + type Error = (); + + fn try_from(value: Value<'x, P, E>) -> Result { + if let Value::Object(map) = value { + map.into_vec() + .into_iter() + .map(|(k, v)| T::try_from(v).map(|v| (k.to_string().into_owned(), v))) + .collect::, _>>() + .map_err(|_| ()) + } else { + Err(()) + } + } +} + diff --git a/crates/jmap-tools/src/pointer/mod.rs b/crates/jmap-tools/src/pointer/mod.rs new file mode 100644 index 00000000..5adf0aeb --- /dev/null +++ b/crates/jmap-tools/src/pointer/mod.rs @@ -0,0 +1,221 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) mod eval; +pub(crate) mod parser; + +use crate::{Element, Key, Property, Value}; +use std::{ + borrow::Cow, + fmt::{self, Debug, Display, Formatter, Write}, + iter::Peekable, + slice::Iter, + str::from_utf8, +}; + +const ENCODE_CAPACITY: usize = 32; + +pub trait JsonPointerHandler<'x, P: Property, E: Element>: Debug { + fn eval_jptr<'y>( + &'y self, + pointer: JsonPointerIter<'_, P>, + results: &mut Vec>>, + ); + fn patch_jptr<'y: 'x>( + &mut self, + pointer: JsonPointerIter<'_, P>, + value: Value<'y, P, E>, + ) -> bool; + fn to_value<'y>(&'y self) -> Cow<'y, Value<'x, P, E>>; +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct JsonPointer(pub(crate) Vec>); + +pub type JsonPointerIter<'x, P> = Peekable>>; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum JsonPointerItem { + Root, + Wildcard, + Invalid(String), + Key(Key<'static, P>), + Number(u64), +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct PointerDepth(u8); + +impl PointerDepth { + pub const LIMIT: u8 = 16; + + fn inner(self) -> Self { + PointerDepth(self.0 + 1) + } + + fn can_nest(self) -> bool { + self.0 < Self::LIMIT + } +} + +impl JsonPointer

{ + pub fn new(items: Vec>) -> Self { + Self(items) + } + + pub fn iter(&self) -> JsonPointerIter<'_, P> { + self.0.iter().peekable() + } + + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator> { + self.0.into_iter() + } + + pub fn into_inner(self) -> Vec> { + self.0 + } + + pub fn encode(items: I) -> String + where + I: IntoIterator, + T: AsRef, + { + let mut encoded = String::with_capacity(ENCODE_CAPACITY); + for (pos, item) in items.into_iter().enumerate() { + if pos > 0 { + encoded.push('/'); + } + encoded.extend(EscapedToken(item.as_ref()).pieces()); + } + encoded + } + + pub fn first(&self) -> Option<&JsonPointerItem

> { + self.0.first() + } + + pub fn first_mut(&mut self) -> Option<&mut JsonPointerItem

> { + self.0.first_mut() + } + + pub fn last(&self) -> Option<&JsonPointerItem

> { + self.0.last() + } + + pub fn last_mut(&mut self) -> Option<&mut JsonPointerItem

> { + self.0.last_mut() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn as_slice(&self) -> &[JsonPointerItem

] { + &self.0 + } + + pub fn as_mut_slice(&mut self) -> &mut [JsonPointerItem

] { + &mut self.0 + } +} + +impl JsonPointerItem

{ + pub fn as_key(&self) -> Option<&Key<'static, P>> { + match self { + JsonPointerItem::Key(key) => Some(key), + _ => None, + } + } + + pub fn as_property_key(&self) -> Option<&P> { + match self { + JsonPointerItem::Key(Key::Property(key)) => Some(key), + _ => None, + } + } + + pub fn as_string_key(&self) -> Option<&str> { + match self { + JsonPointerItem::Key(Key::Borrowed(key)) => Some(key), + JsonPointerItem::Key(Key::Owned(key)) => Some(key), + _ => None, + } + } + + pub fn to_cow(&self) -> Option> { + match self { + JsonPointerItem::Key(Key::Property(key)) => Some(key.to_cow()), + JsonPointerItem::Key(Key::Borrowed(key)) => Some(Cow::Borrowed(key)), + JsonPointerItem::Key(Key::Owned(key)) => Some(Cow::Borrowed(key)), + _ => None, + } + } +} + +impl Display for JsonPointer

{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + for (i, ptr) in self.0.iter().enumerate() { + if i > 0 { + f.write_char('/')?; + } + + match ptr { + JsonPointerItem::Root => {} + JsonPointerItem::Wildcard => f.write_char('*')?, + JsonPointerItem::Invalid(text) => f.write_str(text)?, + JsonPointerItem::Key(k) => EscapedToken(&k.to_string()) + .pieces() + .try_for_each(|piece| f.write_str(piece))?, + JsonPointerItem::Number(n) => f.write_str(NumberKey::default().format(*n))?, + } + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +struct EscapedToken<'x>(&'x str); + +impl<'x> EscapedToken<'x> { + fn pieces(self) -> impl Iterator { + self.0 + .split_inclusive(['~', '/']) + .flat_map(|piece| match piece.strip_suffix(['~', '/']) { + Some(text) if piece.ends_with('~') => [text, "~0"], + Some(text) => [text, "~1"], + None => [piece, ""], + }) + .filter(|piece| !piece.is_empty()) + } +} + +#[derive(Default)] +pub(crate) struct NumberKey([u8; 20]); + +impl NumberKey { + pub(crate) fn format(&mut self, mut n: u64) -> &str { + let mut pos = self.0.len(); + loop { + pos -= 1; + if let Some(digit) = self.0.get_mut(pos) { + *digit = b'0' + (n % 10) as u8; + } + n /= 10; + if n == 0 || pos == 0 { + break; + } + } + self.0 + .get(pos..) + .and_then(|digits| from_utf8(digits).ok()) + .unwrap_or_default() + } +} diff --git a/crates/jmap-tools/src/pointer/parser.rs b/crates/jmap-tools/src/pointer/parser.rs new file mode 100644 index 00000000..1982b44f --- /dev/null +++ b/crates/jmap-tools/src/pointer/parser.rs @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use std::borrow::Cow; + +#[derive(Clone, Copy)] +struct Token<'x> { + text: &'x str, + digits: bool, + escaped: bool, +} + +impl JsonPointer

{ + pub const MAX_SEGMENTS: usize = 128; + + pub fn parse(value: &str) -> Self { + Self::parse_tokens(value, PointerDepth::default().inner()) + } + + pub fn parse_nested(value: &str, depth: PointerDepth) -> Option { + depth + .can_nest() + .then(|| Self::parse_tokens(value, depth.inner())) + } + + fn parse_tokens(value: &str, depth: PointerDepth) -> Self { + if value.is_empty() { + return JsonPointer(vec![JsonPointerItem::Root]); + } + + let text = value.strip_prefix('/').unwrap_or(value); + let separators = text + .bytes() + .filter(|&byte| byte == b'/') + .take(Self::MAX_SEGMENTS) + .count(); + let (token, mut rest) = Token::split(text); + let first = JsonPointerItem::parse_segment(None, token, depth); + if separators == Self::MAX_SEGMENTS { + return Self::too_long(first); + } + + let mut path = Vec::with_capacity(separators + 1); + let opaque = first.is_opaque(); + path.push(first); + + if opaque { + return Self::parse_opaque_tail(path, rest); + } + + while let Some(text) = rest { + let (token, tail) = Token::split(text); + let item = JsonPointerItem::parse_segment( + path.last().and_then(JsonPointerItem::as_key), + token, + depth, + ); + path.push(item); + rest = tail; + } + + JsonPointer(path) + } + + #[cold] + #[inline(never)] + fn too_long(first: JsonPointerItem

) -> Self { + JsonPointer(vec![first, JsonPointerItem::Invalid(String::new())]) + } + + #[cold] + #[inline(never)] + fn parse_opaque_tail(mut path: Vec>, mut rest: Option<&str>) -> Self { + while let Some(text) = rest { + let (token, tail) = Token::split(text); + path.push(JsonPointerItem::parse_opaque_token(token)); + rest = tail; + } + + JsonPointer(path) + } +} + +impl<'x> Token<'x> { + fn split(text: &'x str) -> (Self, Option<&'x str>) { + let mut digits = true; + let mut escaped = false; + let mut end = text.len(); + for (pos, byte) in text.bytes().enumerate() { + match byte { + b'/' => { + end = pos; + break; + } + b'~' => { + escaped = true; + digits = false; + } + b'0'..=b'9' => {} + _ => digits = false, + } + } + let token = Token { + text: text.get(..end).unwrap_or_default(), + digits, + escaped, + }; + (token, text.get(end + 1..)) + } + + fn unescape(self) -> Option { + let (head, mut rest) = self.text.split_once('~')?; + let mut text = String::with_capacity(self.text.len()); + text.push_str(head); + loop { + match rest.split_at_checked(1) { + Some(("0", tail)) => { + text.push('~'); + rest = tail; + } + Some(("1", tail)) => { + text.push('/'); + rest = tail; + } + _ => return None, + } + + match rest.split_once('~') { + Some((segment, tail)) => { + text.push_str(segment); + rest = tail; + } + None => { + text.push_str(rest); + return Some(text); + } + } + } + } +} + +impl JsonPointerItem

{ + fn is_opaque(&self) -> bool { + matches!(self, JsonPointerItem::Key(key) if key.is_opaque()) + } + + #[inline(always)] + fn parse_segment( + parent: Option<&Key<'static, P>>, + token: Token<'_>, + depth: PointerDepth, + ) -> Self { + if token.text.is_empty() { + JsonPointerItem::empty_segment() + } else { + JsonPointerItem::parse_token(parent, token, depth) + } + } + + #[cold] + #[inline(never)] + fn empty_segment() -> Self { + JsonPointerItem::Key("".into()) + } + + fn parse_opaque_token(token: Token<'_>) -> Self { + if !token.escaped { + JsonPointerItem::Key(Key::Owned(token.text.to_string())) + } else { + match token.unescape() { + Some(text) => JsonPointerItem::Key(Key::Owned(text)), + None => JsonPointerItem::Invalid(token.text.to_string()), + } + } + } + + fn parse_token( + parent: Option<&Key<'static, P>>, + token: Token<'_>, + depth: PointerDepth, + ) -> Self { + let text = token.text; + if token.digits { + match P::try_parse_nested(parent, text, depth) { + Some(prop) => JsonPointerItem::Key(Key::Property(prop)), + None => match text.parse::() { + Ok(index) if text == "0" || !text.starts_with('0') => { + JsonPointerItem::Number(index) + } + _ => JsonPointerItem::Key(Key::Owned(text.to_string())), + }, + } + } else if text == "*" { + JsonPointerItem::Wildcard + } else if !token.escaped { + JsonPointerItem::Key(match P::try_parse_nested(parent, text, depth) { + Some(prop) => Key::Property(prop), + None => Key::Owned(text.to_string()), + }) + } else { + match token.unescape() { + Some(text) => { + JsonPointerItem::Key(match P::try_parse_nested(parent, &text, depth) { + Some(prop) => Key::Property(prop), + None => Key::Owned(text), + }) + } + None => JsonPointerItem::Invalid(text.to_string()), + } + } + } +} + +impl<'de, P: Property> serde::Deserialize<'de> for JsonPointer

{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + >::deserialize(deserializer).map(|s| JsonPointer::parse(s.as_ref())) + } +} + +impl serde::Serialize for JsonPointer

{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(self) + } +} + diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 876e1698..18eb7953 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -19,37 +19,38 @@ spam-filter = { path = "../spam-filter" } email = { path = "../email" } groupware = { path = "../groupware" } registry = { path = "../registry" } -calcard = { version = "0.3" } -smtp-proto = { version = "0.2" } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } -mail-builder = { version = "1.0" } -mail-auth = { version = "0.13", features = ["generate", "arc"] } -sieve-rs = { version = "0.7", features = ["rkyv"] } -jmap-tools = { version = "0.1", features = ["rkyv"] } +calcard = { path = "../calcard" } +smtp-proto = { path = "../smtp-proto" } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +mail-auth = { version = "0.14", features = ["generate", "arc"] } +sieve-rs = { path = "../sieve" } +jmap-tools = { path = "../jmap-tools", features = ["rkyv"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } -hyper-util = { version = "0.1.20", features = ["tokio"] } +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1.21", features = ["tokio"] } http-body-util = "0.1.5" -tokio = { version = "1.53", features = ["rt"] } +tokio = { version = "1.54", features = ["rt"] } futures-util = "0.3.34" async-stream = "0.3.6" -base64 = "0.23" -p256 = { version = "0.13", features = ["ecdh"] } +encodify = "1.0" +p256 = { version = "0.14", features = ["ecdh"] } sha1 = "0.11" sha2 = "0.11" reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"]} tokio-tungstenite = "0.30" tungstenite = "0.30" -chrono = "0.4" -rand = "0.10.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } -hashify = "0.2" +jiff = "0.2.38" +rand = "0.10.3" +rkyv.workspace = true +hashify = "0.3" +compact_str = "0.10.0" +memchr = "2.8" [features] -test_mode = [] dev_mode = [] -enterprise = [] +enterprise = ["store/enterprise", "http_proto/enterprise", "jmap_proto/enterprise", "smtp/enterprise", "utils/enterprise", "common/enterprise", "services/enterprise", "directory/enterprise", "trc/enterprise", "spam-filter/enterprise", "email/enterprise", "groupware/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/jmap/src/addressbook/get.rs b/crates/jmap/src/addressbook/get.rs index 74624b62..f0797939 100644 --- a/crates/jmap/src/addressbook/get.rs +++ b/crates/jmap/src/addressbook/get.rs @@ -4,18 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::acl::JmapRights, changes::state::JmapCacheState}; +use crate::{ + api::{ + acl::JmapRights, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + changes::state::JmapCacheState, +}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{cache::GroupwareCache, contact::AddressBook}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::addressbook::{self, AddressBookProperty, AddressBookValue}, + request::capability::CapabilityIds, }; use jmap_tools::{Map, Value}; use store::{ ValueKey, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, ValueClass}, + write::{Archive, ArchiveBytes, ValueClass}, }; use trc::AddContext; use types::{ @@ -29,6 +36,7 @@ pub trait AddressBookGet: Sync + Send { &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -37,22 +45,33 @@ impl AddressBookGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - AddressBookProperty::Id, - AddressBookProperty::Name, - AddressBookProperty::Description, - AddressBookProperty::SortOrder, - AddressBookProperty::IsDefault, - AddressBookProperty::IsSubscribed, - AddressBookProperty::ShareWith, - AddressBookProperty::MyRights, - ]); + let (properties, selection) = select_properties( + &mut request, + &[ + AddressBookProperty::Id, + AddressBookProperty::Name, + AddressBookProperty::Description, + AddressBookProperty::SortOrder, + AddressBookProperty::IsDefault, + AddressBookProperty::IsSubscribed, + AddressBookProperty::ShareWith, + AddressBookProperty::MyRights, + ], + using, + )?; + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::AddressBook); + let metadata_get = metadata.get(selection); + let metadata_slots = metadata_get.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let personal_id = access_token.personal_id(account_id, Collection::AddressBook); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -63,30 +82,51 @@ impl AddressBookGet for Server { } else { cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) }; - let default_address_book_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), - }) - .await - .caused_by(trc::location!())? - .or_else(|| cache.document_ids(true).min()); + let default_address_book_id = if properties.contains(&AddressBookProperty::IsDefault) { + self.store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), + }) + .await + .caused_by(trc::location!())? + .or_else(|| cache.document_ids(true).min()) + } else { + None + }; let ids = if let Some(ids) = ids { ids } else { - address_book_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + address_book_ids.iter().map(Into::into), + self.core.jmap.get_max_objects, + )? + }; + let mut metadata_values = match &metadata_get { + Some(get) => { + let mut documents = MetadataDocuments::default(); + for document_id in ids + .iter() + .map(|id| id.document_id()) + .filter(|document_id| address_book_ids.contains(*document_id)) + { + documents.insert( + document_id, + cache + .container_resource_by_id(document_id) + .map_or_else(Default::default, |resource| resource.metadata_kinds()), + ); + } + Some(metadata.load(self, account_id, get, &documents).await?) + } + None => None, }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(true).into(), + state: sampled.state(cache.get_state(true)).into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; @@ -100,7 +140,7 @@ impl AddressBookGet for Server { } let _address_book = if let Some(address_book) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, @@ -115,7 +155,7 @@ impl AddressBookGet for Server { let address_book = _address_book .unarchive::() .caused_by(trc::location!())?; - let mut result = Map::with_capacity(properties.len()); + let mut result = Map::with_capacity(properties.len() + metadata_slots); for property in &properties { match property { AddressBookProperty::Id => { @@ -189,6 +229,9 @@ impl AddressBookGet for Server { } } } + if let Some(values) = &mut metadata_values { + values.insert_into(document_id, &mut result); + } response.list.push(result.into()); } diff --git a/crates/jmap/src/addressbook/set.rs b/crates/jmap/src/addressbook/set.rs index 9086aa82..866bcb33 100644 --- a/crates/jmap/src/addressbook/set.rs +++ b/crates/jmap/src/addressbook/set.rs @@ -5,8 +5,18 @@ */ use crate::api::acl::{JmapAcl, JmapRights}; +use crate::api::metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, ObjectMetadata, + PreparedMetadata, is_empty_update, reject_uncommitted, +}; +use crate::api::pending_creates::PendingCreates; use crate::changes::state::JmapCacheState; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{ + Server, + auth::AccessToken, + sharing::{EffectiveAcl, grants::ShareUpdate}, + storage::{batch::MAX_BATCH_DOCUMENTS, index::PresenceFlags, quota::ObjectQuotaUsage}, +}; use groupware::{ DestroyArchive, cache::GroupwareCache, @@ -17,21 +27,22 @@ use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::addressbook::{self, AddressBookProperty, AddressBookValue}, - request::{MaybeInvalid, reference::MaybeIdReference}, + request::{MaybeInvalid, capability::CapabilityIds, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Value}; use rand::{RngExt, distr::Alphanumeric}; +use registry::schema::enums::StorageQuota; use store::{ - SerializeInfallible, ValueKey, + ValueKey, ahash::AHashSet, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, + write::{Archive, ArchiveBytes, BatchBuilder, PendingId}, }; use trc::AddContext; use types::{ - acl::Acl, + acl::{Acl, AclGrant}, collection::{Collection, SyncCollection}, - field::PrincipalField, + field::{ContactField, PrincipalField}, id::Id, }; @@ -41,6 +52,7 @@ pub trait AddressBookSet: Sync + Send { request: SetRequest<'_, addressbook::AddressBook>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -50,24 +62,50 @@ impl AddressBookSet for Server { mut request: SetRequest<'_, addressbook::AddressBook>, access_token: &AccessToken, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::AddressBook); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(true, &request.if_in_state)?); + .with_state(sampled.assert_state(cache.get_state(true), &request.if_in_state)?); + let mut metadata_writer = MetadataWriter::new(metadata, account_id); let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); let is_shared = access_token.is_shared(account_id); - let mut set_default = None; + let mut set_default: Option = None; + let mut created_slots = PendingCreates::new(); + + // Obtain quota + let quota = if request.has_creates() && !is_shared { + let account = self.account(account_id).await.caused_by(trc::location!())?; + self.object_quota_usage(&account, StorageQuota::MaxAddressBooks, || { + cache.resources.count(true) + }) + } else { + ObjectQuotaUsage::unlimited() + }; // Process creates let mut batch = BatchBuilder::new(); - 'create: for (id, object) in request.unwrap_create() { + 'create: for (id, mut object) in request.unwrap_create() { + if !quota.has_room(created_slots.len()) { + response.not_created.append( + id, + SetError::over_quota().with_description(concat!( + "There are too many address books, ", + "please delete some before adding a new one." + )), + ); + continue 'create; + } + if is_shared { response.not_created.append( id, @@ -92,6 +130,14 @@ impl AddressBookSet for Server { }; // Process changes + let metadata_patches = + match metadata_writer.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; if let Err(err) = update_address_book(None, object, &mut address_book, access_token, account_id) { @@ -101,25 +147,61 @@ impl AddressBookSet for Server { // Validate ACLs if !address_book.acls.is_empty() { - if let Err(err) = self.acl_validate(&address_book.acls).await { + if let Err(err) = JmapRights::validate_shares::( + ShareUpdate { + collection: Collection::AddressBook, + owner_id: account_id, + actor: None, + current: &[] as &[AclGrant], + max_shares: self.core.groupware.max_shares_per_item, + }, + &address_book.acls, + ) { + response.not_created.append(id, err); + continue 'create; + } + if let Err(err) = self + .acl_validate( + access_token, + account_id, + &address_book.acls, + &[] as &[AclGrant], + ) + .await + { response.not_created.append(id, err.into()); continue 'create; } - - self.refresh_acls(&address_book.acls, None) - .await - .caused_by(trc::location!())?; } + let prepared_metadata = match metadata_patches { + Some(patches) => match metadata_writer + .prepare_for(self, patches, MetadataAccess::FULL, MetadataTarget::Create) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }, + None => None, + }; + // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::AddressBook, 1) - .await - .caused_by(trc::location!())?; + let document_id = batch.reserve_document_id(account_id, Collection::AddressBook); + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + address_book.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, &mut batch)?; + } address_book .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, &mut batch, @@ -130,14 +212,21 @@ impl AddressBookSet for Server { &request.arguments.on_success_set_is_default && id_ref == &id { - set_default = Some(document_id); + set_default = Some(PendingId::Slot(document_id)); } - response.created(id, document_id); + created_slots.push(id, document_id); } // Process updates - 'update: for (id, object) in request.unwrap_update() { + metadata_writer + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .container_resource_by_id(document_id) + .map(|resource| resource.metadata_kinds()) + }) + .await?; + 'update: for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -150,12 +239,20 @@ impl AddressBookSet for Server { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } + let mut metadata_patches = + match metadata_writer.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }; // Obtain address book let document_id = id.document_id(); let address_book_ = if let Some(address_book_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, @@ -170,6 +267,38 @@ impl AddressBookSet for Server { let address_book = address_book_ .to_unarchived::() .caused_by(trc::location!())?; + + let acl = is_shared.then(|| address_book.inner.acls.effective_acl(access_token)); + if acl.is_some_and(|acl| !acl.contains_any([Acl::Read, Acl::ReadItems].into_iter())) { + response.not_updated.append(id, SetError::not_found()); + continue 'update; + } + let metadata_access = MetadataAccess { + may_write_shared: acl.is_none_or(|acl| acl.contains(Acl::ModifyMetadata)), + may_read: true, + }; + if object + .as_object() + .is_some_and(|object| is_empty_update::(object, id)) + && let Some(patches) = metadata_patches.take() + { + match metadata_writer + .write_metadata_only::( + self, + patches, + metadata_access, + &address_book, + document_id, + &mut batch, + ) + .await? + { + Ok(()) => response.updated.append(id, None), + Err(err) => response.not_updated.append(id, err), + } + continue 'update; + } + let mut new_address_book = address_book .deserialize::() .caused_by(trc::location!())?; @@ -190,34 +319,74 @@ impl AddressBookSet for Server { }; // Validate ACL - if is_shared { - let acl = address_book.inner.acls.effective_acl(access_token); - if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { - response.not_updated.append( - id, - SetError::forbidden() - .with_description("You are not allowed to modify this address book."), - ); - continue 'update; - } + if acl.is_some_and(|acl| has_acl_changes && !acl.contains(Acl::Share)) { + response.not_updated.append( + id, + SetError::forbidden() + .with_description("You are not allowed to share this address book."), + ); + continue 'update; } if has_acl_changes { - if let Err(err) = self.acl_validate(&new_address_book.acls).await { + if let Err(err) = JmapRights::validate_shares::( + ShareUpdate { + collection: Collection::AddressBook, + owner_id: account_id, + actor: acl, + current: &address_book.inner.acls, + max_shares: self.core.groupware.max_shares_per_item, + }, + &new_address_book.acls, + ) { + response.not_updated.append(id, err); + continue 'update; + } + if let Err(err) = self + .acl_validate( + access_token, + account_id, + &new_address_book.acls, + &address_book.inner.acls, + ) + .await + { response.not_updated.append(id, err.into()); continue 'update; } - self.refresh_archived_acls( - &new_address_book.acls, - address_book.inner.acls.as_slice(), - ) - .await - .caused_by(trc::location!())?; } + let prepared_metadata = match metadata_patches { + Some(patches) => match metadata_writer + .prepare_for( + self, + patches, + metadata_access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }, + None => None, + }; + // Update record + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + new_address_book.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, &mut batch)?; + } new_address_book .update( - access_token.account_tenant_ids(), + access_token.changed_by(), address_book, account_id, document_id, @@ -228,25 +397,32 @@ impl AddressBookSet for Server { } // Process deletions - let mut reset_default_address_book = false; + let mut destroyed_books: Vec = Vec::new(); if !will_destroy.is_empty() { let mut destroy_children = AHashSet::new(); let mut destroy_parents = AHashSet::new(); - let default_address_book_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), - }) - .await - .caused_by(trc::location!())?; let on_destroy_remove_contents = request .arguments .on_destroy_remove_contents .unwrap_or(false); + let book_cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::AddressBook, + &will_destroy + .iter() + .map(|id| id.document_id()) + .filter(|&document_id| { + cache + .container_resource_by_id(document_id) + .is_some_and(|book| !book.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; for id in will_destroy { let document_id = id.document_id(); @@ -258,7 +434,7 @@ impl AddressBookSet for Server { let Some(address_book_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, @@ -280,7 +456,7 @@ impl AddressBookSet for Server { .inner .acls .effective_acl(access_token) - .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) + .contains(Acl::Delete) { response.not_destroyed.append( id, @@ -306,66 +482,96 @@ impl AddressBookSet for Server { .container_resource_path_by_id(document_id) .map(|resource| cache.format_resource(resource)); DestroyArchive(address_book) - .delete( - access_token.account_tenant_ids(), + .delete_with_cleanup( + self, + access_token.changed_by(), account_id, document_id, delete_path, + &book_cleanup, &mut batch, ) + .await .caused_by(trc::location!())?; - if default_address_book_id == Some(document_id) { - reset_default_address_book = true; - } + destroyed_books.push(document_id); response.destroyed.push(id); } // Delete children if !destroy_children.is_empty() { - for document_id in destroy_children { - if let Some(card_) = self - .store() - .get_value::>(ValueKey::archive( + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::ContactCard, + &destroy_children + .iter() + .copied() + .filter(|&document_id| { + cache + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; + let mut destroy_children = destroy_children.into_iter().collect::>(); + destroy_children.sort_unstable(); + destroy_children.dedup(); + for chunk in destroy_children.chunks(MAX_BATCH_DOCUMENTS) { + let mut archives = self + .archive_chunk( account_id, Collection::ContactCard, - document_id, - )) - .await? - { - let card = card_ - .to_unarchived::() - .caused_by(trc::location!())?; - - if card - .inner - .names - .iter() - .all(|n| destroy_parents.contains(&n.parent_id.to_native())) - { - // Card only belongs to address books being deleted, delete it - DestroyArchive(card).delete_all( - access_token.account_tenant_ids(), - account_id, - document_id, - &mut batch, - )?; - } else { - // Unlink addressbook id from card - let mut new_card = card - .deserialize::() + chunk, + Some((ContactField::Content.field(), chunk)), + ) + .await + .caused_by(trc::location!())?; + for &document_id in chunk { + if let Some(card_) = archives.take_archive(document_id) { + let card = card_ + .to_unarchived::() .caused_by(trc::location!())?; - new_card + + if card + .inner .names - .retain(|n| !destroy_parents.contains(&n.parent_id)); - new_card.update( - access_token.account_tenant_ids(), - card, - account_id, - document_id, - &mut batch, - )?; + .iter() + .all(|n| destroy_parents.contains(&n.parent_id.to_native())) + { + // Card only belongs to address books being deleted, delete it + DestroyArchive(card) + .remove_all( + self, + access_token.changed_by(), + account_id, + document_id, + archives.take_value(document_id), + &cleanup, + &mut batch, + ) + .await?; + } else { + // Unlink addressbook id from card + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + new_card + .names + .retain(|n| !destroy_parents.contains(&n.parent_id)); + new_card.update_meta( + access_token.changed_by(), + card, + account_id, + document_id, + None, + &mut batch, + )?; + } } } } @@ -374,9 +580,12 @@ impl AddressBookSet for Server { // Set default address book if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_set_is_default { - set_default = Some(id.document_id()); + let document_id = id.document_id(); + if cache.has_container_id(&document_id) && !destroyed_books.contains(&document_id) { + set_default = Some(PendingId::Assigned(document_id)); + } } - if let Some(default_address_book_id) = set_default { + if let Some(default_address_book_id) = set_default.filter(|_| !is_shared) { if response.not_created.is_empty() && response.not_updated.is_empty() && response.not_destroyed.is_empty() @@ -387,27 +596,40 @@ impl AddressBookSet for Server { .with_document(0) .set( PrincipalField::DefaultAddressBookId, - default_address_book_id.serialize(), + default_address_book_id, ); } - } else if reset_default_address_book { + } else if !destroyed_books.is_empty() { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultAddressBookId); + .with_document(0); + for document_id in destroyed_books { + batch.clear_if_equals(PrincipalField::DefaultAddressBookId, document_id); + } } // Write changes - if !batch.is_empty() - && let Ok(change_id) = self - .commit_batch(batch) - .await - .caused_by(trc::location!())? - .last_change_id(account_id) - { - self.notify_task_queue(); - response.new_state = State::Exact(change_id).into(); + if !batch.is_empty() { + let assigned_ids = match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut response, + created_slots.into_create_ids(), + "Another process modified this address book, please try again.", + ); + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; + + created_slots.resolve(&mut response, &assigned_ids); + + if let Some(change_id) = assigned_ids.change_id(account_id, SyncCollection::AddressBook) + { + response.new_state = State::Exact(change_id).into(); + } } Ok(response) @@ -441,8 +663,13 @@ fn update_address_book( (AddressBookProperty::Description, Value::Null) => { address_book.preferences_mut(personal_id).description = None; } - (AddressBookProperty::SortOrder, Value::Number(value)) => { - address_book.preferences_mut(personal_id).sort_order = value.cast_to_u64() as u32; + (AddressBookProperty::SortOrder, Value::Number(value)) + if let Some(sort_order) = value + .as_u64() + .and_then(|sort_order| u32::try_from(sort_order).ok()) + .filter(|sort_order| *sort_order < 1 << 31) => + { + address_book.preferences_mut(personal_id).sort_order = sort_order; } (AddressBookProperty::IsSubscribed, Value::Bool(subscribe)) => { if subscribe { diff --git a/crates/jmap/src/api/acl.rs b/crates/jmap/src/api/acl.rs index 2667bf72..8e6bbe08 100644 --- a/crates/jmap/src/api/acl.rs +++ b/crates/jmap/src/api/acl.rs @@ -4,14 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{ + Server, + auth::AccessToken, + sharing::{ + EffectiveAcl, + grants::{AclGrantLookup, ShareUpdate, ShareViolation}, + }, +}; use jmap_proto::{ error::set::SetError, object::{JmapRight, JmapSharedObject}, }; use jmap_tools::{JsonPointerIter, Key, Map, Property, Value}; -use registry::schema::prelude::ObjectType; -use store::{registry::RegistryQuery, roaring::RoaringBitmap}; use types::{ acl::{Acl, AclGrant}, id::Id, @@ -96,7 +101,7 @@ impl JmapRights { } }; - let acl = right + let right = right .as_property_key() .cloned() .and_then(|p| T::Right::try_from(p).ok()) @@ -107,16 +112,17 @@ impl JmapRights { "Invalid permission {:?}.", right.to_cow().unwrap_or_default() )) - })? - .to_acl() - .iter() - .copied(); + })?; + let acl = right.to_acl().iter().copied(); if let Some(acl_item) = grants.iter_mut().find(|item| item.account_id == account_id) { if is_set { acl_item.grants.insert_many(acl); } else { acl_item.grants.remove_many(acl); + if right.is_implied_by(&acl_item.grants) { + return Err(implied_right::(right)); + } if acl_item.grants.is_empty() { grants.retain(|item| item.account_id != account_id); } @@ -155,23 +161,63 @@ impl JmapRights { T::Right: TryFrom, { let mut acls = Bitmap::new(); + let mut denied = Vec::new(); - for key in value.into_expanded_boolean_set() { - acls.insert_many( - key.as_property() - .and_then(|p| T::Right::try_from(p.clone()).ok()) - .ok_or_else(|| { - SetError::invalid_properties() - .with_property(T::SHARE_WITH_PROPERTY) - .with_description(format!("Invalid permission {:?}.", key.to_string())) - })? - .to_acl() - .iter() - .copied(), - ); + for (key, value) in value.into_expanded_object() { + let right = key + .as_property() + .and_then(|p| T::Right::try_from(p.clone()).ok()); + match (value.as_bool(), right) { + (Some(true), Some(right)) => { + acls.insert_many(right.to_acl().iter().copied()); + } + (Some(false), Some(right)) => { + if !right.implied_by_acl().is_empty() { + denied.push(right); + } + } + (Some(_), None) => { + return Err(SetError::invalid_properties() + .with_property(T::SHARE_WITH_PROPERTY) + .with_description(format!("Invalid permission {:?}.", key.to_string()))); + } + (None, _) => { + return Err(SetError::invalid_properties() + .with_property(T::SHARE_WITH_PROPERTY) + .with_description(format!( + "Permission {:?} must be a boolean.", + key.to_string() + ))); + } + } } - Ok(acls) + match denied.into_iter().find(|right| right.is_implied_by(&acls)) { + Some(right) => Err(implied_right::(right)), + None => Ok(acls), + } + } + + pub fn validate_shares( + update: ShareUpdate<'_, G>, + grants: &[AclGrant], + ) -> Result<(), SetError> { + update + .validate(grants) + .map_err(|violation| match violation { + ShareViolation::TooManyShares(max) => SetError::invalid_properties() + .with_property(T::SHARE_WITH_PROPERTY) + .with_description(format!( + "Maximum number of shares per item exceeded (max: {max})" + )), + ShareViolation::OwnerShared => SetError::invalid_properties() + .with_property(T::SHARE_WITH_PROPERTY) + .with_description( + "An object cannot be shared with the principal that owns it.", + ), + ShareViolation::RightNotHeld => SetError::forbidden() + .with_description("You cannot grant rights that you do not have yourself."), + }) } pub fn all_rights() -> Value<'static, T::Property, T::Element> { @@ -193,7 +239,7 @@ impl JmapRights { for right in T::Right::all_rights() { obj.insert_unchecked( Key::Property((*right).into()), - Value::Bool(right.to_acl().iter().all(|acl| acls.contains(*acl))), + Value::Bool(right.is_granted(&acls)), ); } @@ -211,59 +257,103 @@ impl JmapRights { if access_token.is_member(account_id) || grants.effective_acl(access_token).contains(Acl::Share) { - let mut share_with = Map::with_capacity(grants.len()); - for grant in grants { - share_with.insert_unchecked( - Key::Property(Id::from(grant.account_id).into()), - Self::rights::(grant.grants), - ); - } - - Value::Object(share_with) + Self::grants_value::(grants) } else { Value::Null } } + + pub fn grants_value( + grants: &[AclGrant], + ) -> Value<'static, T::Property, T::Element> + where + T::Property: From, + { + let mut share_with = Map::with_capacity(grants.len()); + for grant in grants { + share_with.insert_unchecked( + Key::Property(Id::from(grant.account_id).into()), + Self::rights::(grant.grants), + ); + } + Value::Object(share_with) + } } pub trait JmapAcl { - fn acl_validate( + fn acl_validate( &self, + access_token: &AccessToken, + owner_id: u32, grants: &[AclGrant], + current: &G, ) -> impl Future> + Send; } pub enum ShareValidationError { MaxSharesExceeded(usize), InvalidAccountId(Id), + Unavailable, } impl JmapAcl for Server { - async fn acl_validate(&self, grants: &[AclGrant]) -> Result<(), ShareValidationError> { + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + async fn acl_validate( + &self, + access_token: &AccessToken, + owner_id: u32, + grants: &[AclGrant], + current: &G, + ) -> Result<(), ShareValidationError> { if grants.len() > self.core.groupware.max_shares_per_item { return Err(ShareValidationError::MaxSharesExceeded( self.core.groupware.max_shares_per_item, )); } - let principal_ids = self - .registry() - .query::(RegistryQuery::new(ObjectType::Account)) - .await - .unwrap_or_default(); - for grant in grants { - if !principal_ids.contains(grant.account_id) { - return Err(ShareValidationError::InvalidAccountId(Id::from( - grant.account_id, - ))); + if !current.grants_of(grant.account_id).is_empty() { + continue; } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + let grantee = match self.try_account(grant.account_id).await { + Ok(Some(grantee)) => grantee, + Ok(None) => { + return Err(ShareValidationError::InvalidAccountId(Id::from( + grant.account_id, + ))); + } + Err(err) => { + trc::error!(err.caused_by(trc::location!())); + return Err(ShareValidationError::Unavailable); + } + }; } Ok(()) } } +trait ImpliedRight { + fn is_implied_by(&self, acls: &Bitmap) -> bool; +} + +impl ImpliedRight for R { + fn is_implied_by(&self, acls: &Bitmap) -> bool { + let implied_by = self.implied_by_acl(); + !implied_by.is_empty() && implied_by.iter().all(|acl| acls.contains(*acl)) + } +} + +fn implied_right(right: T::Right) -> SetError { + SetError::invalid_properties() + .with_property(T::SHARE_WITH_PROPERTY) + .with_description(format!( + "Permission {} cannot be removed while other permissions imply it.", + Key::Property(right.into()).to_string() + )) +} + impl From for SetError { fn from(err: ShareValidationError) -> Self { match err { @@ -273,6 +363,8 @@ impl From for SetError { )), ShareValidationError::InvalidAccountId(id) => SetError::invalid_properties() .with_description(format!("Account id {id} is invalid.")), + ShareValidationError::Unavailable => SetError::invalid_properties() + .with_description("Shares could not be validated, please try again later."), } } } diff --git a/crates/jmap/src/api/auth.rs b/crates/jmap/src/api/auth.rs index 043411c9..90ccaabe 100644 --- a/crates/jmap/src/api/auth.rs +++ b/crates/jmap/src/api/auth.rs @@ -14,7 +14,10 @@ use jmap_proto::{ reference::MaybeResultReference, }, }; -use registry::schema::enums::Permission; +use registry::schema::{ + enums::Permission, + prelude::{OBJ_NO_CREATE, OBJ_NO_DESTROY, OBJ_NO_UPDATE, ObjectType}, +}; use types::{collection::Collection, id::Id}; pub trait JmapAuthorization { @@ -26,16 +29,20 @@ pub trait JmapAuthorization { ) -> trc::Result<()>; fn assert_has_access(&self, to_account_id: Id, to_collection: Collection) -> trc::Result<&Self>; + fn assert_has_source_access( + &self, + from_account_id: Id, + from_collection: Collection, + ) -> trc::Result<&Self>; } impl JmapAuthorization for AccessToken { fn assert_is_member(&self, account_id: Id) -> trc::Result<&Self> { - if self.is_member(account_id.document_id()) { + let document_id = account_id.document_id(); + if self.is_member(document_id) { Ok(self) } else { - Err(trc::JmapEvent::Forbidden - .into_err() - .details(format!("You are not an owner of account {}", account_id))) + Err(account_error(self, document_id)) } } @@ -44,13 +51,24 @@ impl JmapAuthorization for AccessToken { to_account_id: Id, to_collection: Collection, ) -> trc::Result<&Self> { - if self.has_access(to_account_id.document_id(), to_collection) { + let document_id = to_account_id.document_id(); + if self.has_access(document_id, to_collection) { Ok(self) } else { - Err(trc::JmapEvent::Forbidden.into_err().details(format!( - "You do not have access to account {}", - to_account_id - ))) + Err(account_error(self, document_id)) + } + } + + fn assert_has_source_access( + &self, + from_account_id: Id, + from_collection: Collection, + ) -> trc::Result<&Self> { + let document_id = from_account_id.document_id(); + if self.has_access(document_id, from_collection) { + Ok(self) + } else { + Err(source_account_error(self, document_id)) } } @@ -203,14 +221,7 @@ impl JmapAuthorization for AccessToken { let MethodObject::Registry(object_type) = object else { unreachable!() }; - let set_permissions = object_type.set_permission(); - validate_set( - s, - self, - set_permissions[0], - set_permissions[1], - set_permissions[2], - ) + validate_registry_set(s, self, object_type) } }; } @@ -316,6 +327,22 @@ impl JmapAuthorization for AccessToken { } } +fn account_error(access_token: &AccessToken, account_id: u32) -> trc::Error { + if access_token.has_account_access(account_id) { + trc::JmapEvent::AccountNotSupportedByMethod.into_err() + } else { + trc::JmapEvent::AccountNotFound.into_err() + } +} + +fn source_account_error(access_token: &AccessToken, account_id: u32) -> trc::Error { + if access_token.has_account_access(account_id) { + trc::JmapEvent::FromAccountNotSupportedByMethod.into_err() + } else { + trc::JmapEvent::FromAccountNotFound.into_err() + } +} + fn validate_set( set: &SetRequest<'_, T>, access_token: &AccessToken, @@ -323,10 +350,36 @@ fn validate_set( update_permission: Permission, destroy_permission: Permission, ) -> trc::Result<()> { - let can_create = access_token.has_permission(create_permission); - let can_update = access_token.has_permission(update_permission); - let can_destroy = access_token.has_permission(destroy_permission); + check_set( + set, + access_token.has_permission(create_permission), + access_token.has_permission(update_permission), + access_token.has_permission(destroy_permission), + ) +} +fn validate_registry_set( + set: &SetRequest<'_, T>, + access_token: &AccessToken, + object_type: ObjectType, +) -> trc::Result<()> { + let flags = object_type.flags(); + let can_create = + flags & OBJ_NO_CREATE == 0 && access_token.has_permission(object_type.create_permission()); + let can_update = + flags & OBJ_NO_UPDATE == 0 && access_token.has_permission(object_type.update_permission()); + let can_destroy = flags & OBJ_NO_DESTROY == 0 + && access_token.has_permission(object_type.destroy_permission()); + + check_set(set, can_create, can_update, can_destroy) +} + +fn check_set( + set: &SetRequest<'_, T>, + can_create: bool, + can_update: bool, + can_destroy: bool, +) -> trc::Result<()> { if can_create && can_update && can_destroy { Ok(()) } else if !can_create && !can_update && !can_destroy { diff --git a/crates/jmap/src/api/budget.rs b/crates/jmap/src/api/budget.rs new file mode 100644 index 00000000..30d1fe41 --- /dev/null +++ b/crates/jmap/src/api/budget.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + +use common::network::memory::{MEMORY_STEP, MemoryReservation, ReservationScope}; +use std::mem::take; +use trc::AddContext; + +pub const SERIALIZED_COPIES: u64 = 3; +pub const ARCHIVE_EXPANSION: u64 = 2; +const LIGHT_REQUEST_BYTES: u64 = 16 * MEMORY_STEP; + +pub struct GetBudget<'x> { + scope: ReservationScope<'x>, + estimate: u64, + light: u64, +} + +impl<'x> GetBudget<'x> { + pub fn new(reservation: &'x mut MemoryReservation) -> Self { + GetBudget { + scope: reservation.scope(), + estimate: 0, + light: 0, + } + } + + pub fn add(&mut self, bytes: u64) -> trc::Result<()> { + self.estimate = self + .estimate + .saturating_add(bytes.saturating_mul(SERIALIZED_COPIES)); + if self.estimate >= MEMORY_STEP { + self.scope + .try_reserve_extra(self.estimate) + .caused_by(trc::location!()) + } else { + Ok(()) + } + } + + pub fn add_light(&mut self, bytes: u64) -> trc::Result<()> { + self.light = self.light.saturating_add(bytes); + if self.light >= LIGHT_REQUEST_BYTES { + let light = take(&mut self.light); + self.add(light) + } else { + Ok(()) + } + } + + pub fn add_archive(&mut self, archive_bytes: usize, objects: usize) -> trc::Result<()> { + self.add( + (archive_bytes as u64) + .saturating_mul(objects as u64) + .saturating_mul(ARCHIVE_EXPANSION), + ) + } + + pub fn commit(self) { + self.scope.commit(); + } +} diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index d27483c7..85851b5a 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -5,7 +5,11 @@ */ use crate::api::IntoPushObject; -use common::{LONG_1D_SLUMBER, Server, auth::AccessToken, ipc::PushNotification}; +use common::{ + LONG_1D_SLUMBER, Server, + auth::AccessToken, + ipc::{PushNotification, ViewerStateChange}, +}; use http_body_util::{StreamBody, combinators::BoxBody}; use http_proto::*; use hyper::{ @@ -77,9 +81,9 @@ impl EventSourceHandler for Server { } let mut ping = if ping > 0 { - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] let interval = std::cmp::max(ping, 30); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] let interval = ping; let interval_duration = Duration::from_secs(interval as u64); @@ -97,7 +101,8 @@ impl EventSourceHandler for Server { }; // Register with push manager - let mut push_rx = self.subscribe_push_manager(&access_token, types).await?; + access_token.restrict_data_types(&mut types); + let mut push_rx = self.subscribe_push_manager(&access_token, types); let mut changed: VecMap> = VecMap::new(); let throttle = self.core.jmap.event_source_throttle; @@ -108,16 +113,20 @@ impl EventSourceHandler for Server { let mut last_message = Instant::now() - throttle; let mut timeout = ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER); + let mut notifications = Vec::new(); loop { - match tokio::time::timeout(timeout, push_rx.recv()).await { - Ok(Some(notification)) => { + match tokio::time::timeout(timeout, push_rx.recv(&mut notifications)).await { + Ok(true) => for notification in notifications.drain(..) { match notification { - PushNotification::StateChange(state_change) => { + PushNotification::StateChange(state_change) + | PushNotification::ViewerStateChange(ViewerStateChange { + change: state_change, + .. + }) => { + let states = changed.get_mut_or_insert(state_change.account_id.into()); for type_state in state_change.types { - changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, State::Exact(state_change.change_id)); + states.set(type_state, State::Exact(state_change.change_id)); } } PushNotification::CalendarAlert(calendar_alert) => { @@ -128,15 +137,14 @@ impl EventSourceHandler for Server { } PushNotification::EmailPush(email_push) => { let state_change = email_push.to_state_change(); + let states = changed.get_mut_or_insert(state_change.account_id.into()); for type_state in state_change.types { - changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, State::Exact(state_change.change_id)); + states.set(type_state, State::Exact(state_change.change_id)); } } } - } - Ok(None) => { + }, + Ok(false) => { break; } Err(_) => (), diff --git a/crates/jmap/src/api/metadata/apply.rs b/crates/jmap/src/api/metadata/apply.rs new file mode 100644 index 00000000..54771af6 --- /dev/null +++ b/crates/jmap/src/api/metadata/apply.rs @@ -0,0 +1,434 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MetadataSupport, metadata_scope, + patch::{InvalidValue, Patch, PatchKey, PatchValue, value_depth}, + validate::ValidatedPatches, + violation::{Rule, Violation, Violations, limit_description, namespace_path}, + write::MetadataUpdate, +}; +use common::storage::metadata::ContainerChange; +use jmap_proto::{ + error::set::SetError, + object::metadata::{MetadataProperty, MetadataRoot}, +}; +use jmap_tools::{JsonPointer, JsonPointerHandler, Null, Value}; +use std::mem; +use store::write::metadata::MetadataBuf; +use types::metadata::{ + EncodedJson, JsonError, JsonKind, JsonView, LimitViolation, MetadataBuilder, MetadataEdit, + MetadataView, Namespace, +}; + +struct RootContext<'x> { + support: &'x MetadataSupport, + root: MetadataRoot, +} + +impl ValidatedPatches { + pub fn apply( + self, + shared: Option<&MetadataBuf>, + private: Option<&MetadataBuf>, + ) -> Result> { + let mut violations = Violations::default(); + let shared = RootContext { + support: &self.support, + root: MetadataRoot::Shared, + } + .apply(self.shared, shared, &mut violations); + let private = RootContext { + support: &self.support, + root: MetadataRoot::Private, + } + .apply(self.private, private, &mut violations); + + if violations.is_empty() { + Ok(MetadataUpdate::new(shared, private)) + } else { + Err(violations.into_set_error()) + } + } +} + +impl RootContext<'_> { + fn apply( + &self, + patches: Vec, + current: Option<&MetadataBuf>, + violations: &mut Violations, + ) -> Option { + if patches.is_empty() { + return None; + } + + let mut patches = expand(patches); + let edit = if patches.iter().all(|patch| patch.value.is_removal()) { + MetadataEdit::RemovalOnly + } else { + MetadataEdit::Write + }; + let view = current.map(MetadataBuf::view); + let mut builder = view + .as_ref() + .map(MetadataBuilder::from_view) + .unwrap_or_default(); + let entry_bound = self + .support + .limits + .entry_bound(view.as_ref().map_or(0, MetadataView::len)); + let mut removals = patches + .iter() + .filter(|patch| !patch.key.is_deep() && matches!(patch.value, PatchValue::Remove)) + .count(); + let mut is_over_limit = false; + + let mut items = patches + .iter_mut() + .map(|patch| { + let value = mem::replace(&mut patch.value, PatchValue::Clear); + (&patch.key, value) + }) + .peekable(); + while let Some((key, value)) = items.next() { + match (key.namespace.as_deref(), value) { + (None, PatchValue::Clear) => builder.clear_jmap(), + (Some(name), value) if key.is_deep() => { + let mut group = vec![(key, value)]; + while let Some(item) = items.next_if(|(next, _)| { + next.is_deep() && next.namespace.as_deref() == Some(name) + }) { + group.push(item); + } + self.patch_namespace(&mut builder, name, &group, violations); + } + (Some(name), PatchValue::Remove) => { + removals = removals.saturating_sub(1); + if let Ok(namespace) = Namespace::parse(name) { + builder.remove_jmap(&namespace); + } + } + (Some(name), PatchValue::Set(value)) => { + if let Some((namespace, value)) = + self.namespace_value(name, value, view.as_ref(), violations) + && !is_over_limit + { + builder.set_jmap(namespace, value); + if let Err(violation) = entry_bound.check(builder.len(), removals) { + is_over_limit = true; + self.report_container(Rule::Entries, violation, violations); + } + } + } + (_, _) => {} + } + } + + if is_over_limit { + None + } else { + self.finish(builder, view, current, edit, violations) + } + } + + fn namespace_value<'x>( + &self, + name: &'x str, + value: Result, + stored: Option<&MetadataView<'_>>, + violations: &mut Violations, + ) -> Option<(Namespace<'x>, EncodedJson)> { + let namespace = self.namespace(name, violations)?; + if self.support.is_supported(&namespace, self.root) { + return self + .check_value(name, value, violations) + .map(|value| (namespace, value)); + } + + match ( + value, + stored.and_then(|stored| stored.jmap_namespace(&namespace)), + ) { + (Ok(value), Some(stored)) if same_json(value.view(), stored) => { + let kept = if value.as_bytes() == stored.as_bytes() { + value + } else { + stored + .to_value::() + .and_then(|stored| EncodedJson::encode(&stored).ok()) + .unwrap_or(value) + }; + Some((namespace, kept)) + } + (_, Some(_)) => { + self.report_namespace( + name, + "Withdrawn namespaces can only be removed or kept unchanged.", + violations, + ); + None + } + (_, None) => { + self.report_namespace(name, "Unsupported namespace.", violations); + None + } + } + } + + fn patch_namespace<'x>( + &self, + builder: &mut MetadataBuilder<'x>, + name: &'x str, + group: &[(&PatchKey, PatchValue)], + violations: &mut Violations, + ) { + let Some(namespace) = self.namespace(name, violations) else { + return; + }; + let patched = { + let Some(mut value) = builder + .jmap(&namespace) + .and_then(|view| view.to_value::()) + else { + if let Some((key, _)) = group.first() { + self.report_parent(key, "The namespace does not exist.", violations); + } + return; + }; + + for (key, patch) in group { + let patch = match patch { + PatchValue::Set(Ok(encoded)) => encoded.view().to_value().unwrap_or_default(), + PatchValue::Set(Err(invalid)) => { + let depth = u32::try_from(key.pointer.len()) + .unwrap_or(u32::MAX) + .saturating_add(invalid.depth); + self.report_invalid(name, InvalidValue { depth, ..*invalid }, violations); + Value::Null + } + PatchValue::Remove | PatchValue::Replace(_) | PatchValue::Clear => Value::Null, + }; + if !value.patch_jptr(key.pointer.iter(), patch) { + self.report_parent( + key, + "A parent of the patched value does not exist or is not an object.", + violations, + ); + } + } + + EncodedJson::encode_namespace(&value).map_err(|error| InvalidValue { + error, + depth: value_depth(&value, 0), + }) + }; + + if let Some(value) = self.check_value(name, patched, violations) { + builder.set_jmap(namespace, value); + } + } + + fn namespace<'x>(&self, name: &'x str, violations: &mut Violations) -> Option> { + let namespace = Namespace::parse(name).ok(); + if namespace.is_none() { + self.report_namespace(name, "Unsupported namespace.", violations); + } + namespace + } + + fn check_value( + &self, + name: &str, + value: Result, + violations: &mut Violations, + ) -> Option { + match value { + Ok(value) => { + let limits = &self.support.limits; + if let Err(violation) = limits.check_depth(value.depth()) { + self.report_limit(Rule::Depth, name, violation, violations); + } + if let Err(violation) = limits.check_entry_size(value.len()) { + self.report_limit(Rule::EntrySize, name, violation, violations); + } + Some(value) + } + Err(invalid) => { + self.report_invalid(name, invalid, violations); + None + } + } + } + + fn finish( + &self, + builder: MetadataBuilder<'_>, + view: Option>, + current: Option<&MetadataBuf>, + edit: MetadataEdit, + violations: &mut Violations, + ) -> Option { + let change = ContainerChange::new(current, builder.encode(), edit)?; + if let Some(next) = change.next() { + let (previous_len, previous_entries) = view + .as_ref() + .map_or((0, 0), |view| (view.as_bytes().len(), view.len())); + if let Err(violation) = self.support.limits.check_edit( + metadata_scope(self.root), + previous_len, + previous_entries, + next, + ) { + let rule = match violation { + LimitViolation::Entries { .. } => Rule::Entries, + _ => Rule::ContainerSize, + }; + self.report_container(rule, violation, violations); + } + } + Some(change) + } + + fn report_namespace(&self, name: &str, description: &'static str, violations: &mut Violations) { + violations.report(Rule::Namespace, || Violation { + rule: Rule::Namespace, + root: self.root, + path: Some(namespace_path(self.root, name)), + description: description.into(), + }); + } + + fn report_parent( + &self, + key: &PatchKey, + description: &'static str, + violations: &mut Violations, + ) { + violations.report(Rule::Parent, || Violation { + rule: Rule::Parent, + root: self.root, + path: Some(key.to_path()), + description: description.into(), + }); + } + + fn report_limit( + &self, + rule: Rule, + name: &str, + violation: LimitViolation, + violations: &mut Violations, + ) { + violations.report(rule, || Violation { + rule, + root: self.root, + path: Some(namespace_path(self.root, name)), + description: limit_description(violation).into(), + }); + } + + fn report_container(&self, rule: Rule, violation: LimitViolation, violations: &mut Violations) { + violations.report(rule, || Violation { + rule, + root: self.root, + path: None, + description: limit_description(violation).into(), + }); + } + + fn report_invalid(&self, name: &str, invalid: InvalidValue, violations: &mut Violations) { + if let Err(violation) = self.support.limits.check_depth(invalid.depth) { + self.report_limit(Rule::Depth, name, violation, violations); + } + let (rule, description) = match invalid.error { + JsonError::ControlCharacter => ( + Rule::ControlCharacter, + "Metadata keys and values cannot contain control characters.", + ), + JsonError::NestingTooDeep => (Rule::Depth, "Metadata value is nested too deeply."), + JsonError::NotAnObject => (Rule::Namespace, "Namespace values must be objects."), + }; + violations.report(rule, || Violation { + rule, + root: self.root, + path: Some(namespace_path(self.root, name)), + description: description.into(), + }); + } +} + +fn same_json(left: JsonView<'_>, right: JsonView<'_>) -> bool { + if left.as_bytes() == right.as_bytes() { + return true; + } + match (left.kind(), right.kind()) { + (JsonKind::Object, JsonKind::Object) => { + let mut left = left.members().collect::>(); + let mut right = right.members().collect::>(); + left.len() == right.len() && { + left.sort_by_key(|(key, _)| *key); + right.sort_by_key(|(key, _)| *key); + left.into_iter() + .zip(right) + .all(|((left_key, left), (right_key, right))| { + left_key == right_key && same_json(left, right) + }) + } + } + (JsonKind::Array, JsonKind::Array) => { + left.len() == right.len() + && left + .items() + .zip(right.items()) + .all(|(left, right)| same_json(left, right)) + } + (JsonKind::Number, JsonKind::Number) => match (left.as_i64(), right.as_i64()) { + (Some(left), Some(right)) => left == right, + _ => match (left.as_u64(), right.as_u64()) { + (Some(left), Some(right)) => left == right, + _ => left.as_f64() == right.as_f64(), + }, + }, + (JsonKind::String, JsonKind::String) => left.as_str() == right.as_str(), + _ => false, + } +} + +fn expand(patches: Vec) -> Vec { + if !patches + .iter() + .any(|patch| matches!(patch.value, PatchValue::Replace(_))) + { + return patches; + } + let mut expanded = Vec::with_capacity(patches.len()); + for patch in patches { + match patch.value { + PatchValue::Replace(members) => { + let root = patch.key.root; + expanded.reserve(members.len() + 1); + expanded.push(Patch { + key: patch.key, + value: PatchValue::Clear, + }); + expanded.extend(members.into_iter().map(|member| Patch { + key: PatchKey { + root, + namespace: Some(member.namespace), + pointer: JsonPointer::new(Vec::new()), + }, + value: PatchValue::Set(member.value), + })); + } + value => expanded.push(Patch { + key: patch.key, + value, + }), + } + } + expanded +} diff --git a/crates/jmap/src/api/metadata/copy.rs b/crates/jmap/src/api/metadata/copy.rs new file mode 100644 index 00000000..315411d0 --- /dev/null +++ b/crates/jmap/src/api/metadata/copy.rs @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{validate::ValidatedPatches, write::MetadataUpdate}; +use common::storage::metadata::ContainerChange; +use jmap_proto::{error::set::SetError, object::metadata::MetadataProperty}; +use store::write::metadata::MetadataBuf; +use types::metadata::{EncodedMetadata, MetadataBuilder, MetadataKinds}; + +impl MetadataUpdate { + pub fn copied(shared: Option<&MetadataBuf>, private: Option<&MetadataBuf>) -> Self { + MetadataUpdate::new(copied(None, shared), copied(None, private)) + } + + pub fn copied_without_jmap(shared: Option<&MetadataBuf>) -> Self { + MetadataUpdate::new( + shared.and_then(without_jmap).map(ContainerChange::copied), + None, + ) + } +} + +fn without_jmap(source: &MetadataBuf) -> Option { + let view = source.view(); + if view.kinds().contains(MetadataKinds::JMAP) { + let mut builder = MetadataBuilder::from_view(&view); + builder.clear_jmap(); + builder.encode() + } else { + EncodedMetadata::from_view(&view) + } +} + +impl ValidatedPatches { + pub fn apply_to_copy( + self, + shared: Option<&MetadataBuf>, + private: Option<&MetadataBuf>, + ) -> Result> { + let update = self.apply::

(shared, private)?; + Ok(MetadataUpdate::new( + copied(update.shared(), shared), + copied(update.private(), private), + )) + } +} + +fn copied( + patched: Option<&ContainerChange>, + source: Option<&MetadataBuf>, +) -> Option { + match patched { + Some(change) => change.next().cloned(), + None => source.and_then(|source| EncodedMetadata::from_view(&source.view())), + } + .map(ContainerChange::copied) +} + diff --git a/crates/jmap/src/api/metadata/filter.rs b/crates/jmap/src/api/metadata/filter.rs new file mode 100644 index 00000000..001d4a41 --- /dev/null +++ b/crates/jmap/src/api/metadata/filter.rs @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::GroupwareResources; +use jmap_proto::{method::query::Filter, request::deserialize::DeserializeArguments}; +use store::{ + roaring::RoaringBitmap, + search::{SearchFilter, SearchQuery}, + write::SearchIndex, +}; +use types::metadata::MetadataKinds; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResourceScope { + Containers, + Items, + All, +} + +impl ResourceScope { + fn includes(self, is_container: bool) -> bool { + match self { + ResourceScope::Containers => is_container, + ResourceScope::Items => !is_container, + ResourceScope::All => true, + } + } +} + +pub fn flagged_documents( + cache: &GroupwareResources, + scope: ResourceScope, + readable: Option<&RoaringBitmap>, +) -> RoaringBitmap { + cache + .resources + .iter() + .filter(|resource| { + scope.includes(resource.is_container()) + && resource.metadata_kinds().contains(MetadataKinds::JMAP) + && readable.is_none_or(|readable| readable.contains(resource.document_id())) + }) + .map(|resource| resource.document_id()) + .collect() +} + +pub fn filter_containers( + filters: &[Filter], + leaves: Vec, + readable: RoaringBitmap, +) -> RoaringBitmap +where + F: for<'de> DeserializeArguments<'de> + Default, +{ + if filters.is_empty() { + return readable; + } + let mut leaves = leaves.into_iter(); + let filters = filters + .iter() + .map(|filter| match filter { + Filter::Property(_) => SearchFilter::is_in_set(leaves.next().unwrap_or_default()), + Filter::And => SearchFilter::And, + Filter::Or => SearchFilter::Or, + Filter::Not => SearchFilter::Not, + Filter::Close => SearchFilter::End, + }) + .collect(); + SearchQuery::new(SearchIndex::InMemory) + .with_filters(filters) + .with_mask(readable) + .filter() + .into_bitmap() +} diff --git a/crates/jmap/src/api/metadata/get.rs b/crates/jmap/src/api/metadata/get.rs new file mode 100644 index 00000000..065e7c8e --- /dev/null +++ b/crates/jmap/src/api/metadata/get.rs @@ -0,0 +1,236 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::MetadataSupport; +use common::Server; +use jmap_proto::object::metadata::{MetadataProperty, MetadataRoot, MetadataSelection, Selection}; +use jmap_tools::{Element, Key, Map, Value}; +use std::mem; +use store::roaring::RoaringBitmap; +use types::{ + collection::Collection, + metadata::{MetadataKinds, MetadataView, Namespace}, +}; + +#[derive(Debug, Default)] +pub struct MetadataDocuments { + shared: RoaringBitmap, + requested: RoaringBitmap, + pub(super) repeated: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataGet { + collection: Collection, + pub(super) shared: Selection, + pub(super) private: Selection, +} + +type Outputs = Vec<(u32, Value<'static, P, E>)>; + +#[derive(Debug)] +pub struct MetadataValues> { + shared: Option>, + private: Option>, + repeated: Vec, +} + +impl MetadataDocuments { + pub fn insert(&mut self, document_id: u32, kinds: MetadataKinds) { + if !self.requested.insert(document_id) { + self.repeated.push(document_id); + } + if kinds.contains(MetadataKinds::JMAP) { + self.shared.insert(document_id); + } + } + + pub fn shared(&self) -> &RoaringBitmap { + &self.shared + } + + pub fn requested(&self) -> &RoaringBitmap { + &self.requested + } +} + +impl MetadataGet { + pub fn new(support: Option<&MetadataSupport>, selection: MetadataSelection) -> Option { + let support = support?; + let get = MetadataGet { + collection: support.object_type().collection(), + shared: storable(selection.shared), + private: if support.supports_private() { + storable(selection.private) + } else { + Selection::None + }, + }; + (!get.shared.is_none() || !get.private.is_none()).then_some(get) + } + + pub fn wants_shared(&self) -> bool { + !self.shared.is_none() + } + + pub fn wants_private(&self) -> bool { + !self.private.is_none() + } + + pub async fn load( + &self, + server: &Server, + account_id: u32, + private_viewer: Option, + documents: &MetadataDocuments, + ) -> trc::Result> + where + P: MetadataProperty + Send + Sync, + E: Element + Send + Sync, + { + let shared = if self.wants_shared() { + let mut outputs = Vec::new(); + if !documents.shared.is_empty() { + server + .metadata_containers( + account_id, + self.collection, + &documents.shared, + |document_id, view, _| { + push_output(&self.shared, document_id, &view, &mut outputs); + Ok(true) + }, + ) + .await?; + } + Some(outputs) + } else { + None + }; + + let private = match private_viewer { + Some(viewer_id) if self.wants_private() && !documents.requested.is_empty() => { + let mut outputs = Vec::new(); + server + .private_metadata_containers( + account_id, + viewer_id, + self.collection, + &documents.requested, + |document_id, view, _| { + push_output(&self.private, document_id, &view, &mut outputs); + Ok(true) + }, + ) + .await?; + Some(outputs) + } + _ => self.wants_private().then(Vec::new), + }; + + Ok(MetadataValues::new( + shared, + private, + documents.repeated.clone(), + )) + } +} + +fn storable(mut selection: Selection) -> Selection { + if let Selection::Namespaces(names) = &mut selection { + names.retain(|name| Namespace::parse(name).is_ok()); + } + selection +} + +pub(super) fn push_output( + selection: &Selection, + document_id: u32, + view: &MetadataView<'_>, + outputs: &mut Outputs, +) where + P: MetadataProperty, + E: Element, +{ + let mut output = Map::new(); + for (namespace, value) in view.jmap() { + if selection.contains(namespace.name()) + && let Some(value) = value.to_value::() + { + let key = match namespace { + Namespace::Registered(namespace) => Key::Borrowed(namespace.name), + Namespace::Vendor(name) => Key::Owned(name.to_string()), + }; + output.insert_unchecked(key, value.into_owned()); + } + } + if !output.is_empty() { + outputs.push((document_id, Value::Object(output))); + } +} + +impl MetadataValues +where + P: MetadataProperty, + E: Element, +{ + pub(super) fn new( + shared: Option>, + private: Option>, + mut repeated: Vec, + ) -> Self { + repeated.sort_unstable(); + repeated.dedup(); + MetadataValues { + shared: shared.map(sorted), + private: private.map(sorted), + repeated, + } + } + + pub fn insert_into(&mut self, document_id: u32, object: &mut Map<'_, P, E>) { + let is_repeated = self.repeated.binary_search(&document_id).is_ok(); + for (root, outputs) in [ + (MetadataRoot::Shared, self.shared.as_mut()), + (MetadataRoot::Private, self.private.as_mut()), + ] { + if let Some(outputs) = outputs { + object.insert_unchecked( + P::from_metadata_root(root), + take_output(outputs, document_id, is_repeated), + ); + } + } + } +} + +fn sorted(mut outputs: Vec<(u32, T)>) -> Vec<(u32, T)> { + if !outputs.is_sorted_by_key(|(document_id, _)| *document_id) { + outputs.sort_unstable_by_key(|(document_id, _)| *document_id); + } + outputs +} + +fn take_output( + outputs: &mut [(u32, Value<'static, P, E>)], + document_id: u32, + is_repeated: bool, +) -> Value<'static, P, E> +where + P: MetadataProperty, + E: Element, +{ + let output = outputs + .binary_search_by_key(&document_id, |(document_id, _)| *document_id) + .ok() + .and_then(|position| outputs.get_mut(position)) + .map(|(_, output)| output); + match output { + Some(output) if is_repeated => output.clone(), + Some(output) => mem::replace(output, Value::Object(Map::new())), + None => Value::Object(Map::new()), + } +} diff --git a/crates/jmap/src/api/metadata/mod.rs b/crates/jmap/src/api/metadata/mod.rs new file mode 100644 index 00000000..e69c48cc --- /dev/null +++ b/crates/jmap/src/api/metadata/mod.rs @@ -0,0 +1,173 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod apply; +mod copy; +mod filter; +mod get; +mod object; +mod patch; +mod preload; +mod prepared; +mod query; +mod reject; +mod validate; +mod violation; +mod write; +mod writer; + + +pub use common::storage::metadata::MetadataContainers; +pub use filter::{ResourceScope, filter_containers, flagged_documents}; +pub use get::{MetadataDocuments, MetadataGet, MetadataValues}; +pub use object::ObjectMetadata; +pub use patch::{MetadataPatches, is_empty_update}; +pub use preload::{MetadataPreload, PreloadedContainers}; +pub use prepared::PreparedMetadata; +pub use query::{MetadataQuery, PrivateCandidates}; +pub use reject::reject_uncommitted; +pub use validate::{MetadataAccess, ValidatedPatches}; +pub use write::MetadataUpdate; +pub use writer::{MetadataTarget, MetadataWriter, NewMetadata}; + +use common::{ + auth::AccessToken, config::metadata::MetadataConfig, storage::metadata::MetadataViewer, +}; +use jmap_proto::{ + method::get::GetRequest, + object::{ + JmapObject, + metadata::{MetadataProperty, MetadataRoot, MetadataSelection}, + }, + request::capability::{Capability, CapabilityIds}, +}; +use registry::schema::enums::Permission; +use types::{ + collection::Collection, + metadata::{MetadataLimits, MetadataScope, Namespace}, + type_state::DataType, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetadataType { + Email, + Mailbox, + SieveScript, + Calendar, + CalendarEvent, + AddressBook, + ContactCard, + FileNode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetadataSupport { + object: MetadataType, + vendor_namespaces: bool, + private: bool, + writable: bool, + limits: MetadataLimits, + query_max_scan: usize, +} + +impl MetadataType { + pub fn data_type(self) -> DataType { + match self { + MetadataType::Email => DataType::Email, + MetadataType::Mailbox => DataType::Mailbox, + MetadataType::SieveScript => DataType::SieveScript, + MetadataType::Calendar => DataType::Calendar, + MetadataType::CalendarEvent => DataType::CalendarEvent, + MetadataType::AddressBook => DataType::AddressBook, + MetadataType::ContactCard => DataType::ContactCard, + MetadataType::FileNode => DataType::FileNode, + } + } + + pub fn collection(self) -> Collection { + match self { + MetadataType::Email => Collection::Email, + MetadataType::Mailbox => Collection::Mailbox, + MetadataType::SieveScript => Collection::SieveScript, + MetadataType::Calendar => Collection::Calendar, + MetadataType::CalendarEvent => Collection::CalendarEvent, + MetadataType::AddressBook => Collection::AddressBook, + MetadataType::ContactCard => Collection::ContactCard, + MetadataType::FileNode => Collection::FileNode, + } + } +} + +impl MetadataSupport { + pub fn new( + config: &MetadataConfig, + access_token: &AccessToken, + object: MetadataType, + viewer: Option, + ) -> Option { + (config.data_types.contains(object.data_type()) + && access_token.has_permission(Permission::JmapMetadataGet)) + .then(|| MetadataSupport { + object, + vendor_namespaces: config.vendor_namespaces, + private: viewer.is_some(), + writable: access_token.has_permission(Permission::JmapMetadataSet), + limits: config.limits(), + query_max_scan: config.query_max_scan, + }) + } + + pub fn object_type(&self) -> MetadataType { + self.object + } + + pub fn supports_private(&self) -> bool { + self.private + } + + pub fn is_supported(&self, namespace: &Namespace<'_>, root: MetadataRoot) -> bool { + match namespace { + Namespace::Registered(namespace) => { + namespace.is_writable() + && namespace.supports(self.object.data_type(), metadata_scope(root)) + } + Namespace::Vendor(_) => self.vendor_namespaces, + } + } +} + +fn uses_metadata(using: CapabilityIds) -> bool { + using.contains(Capability::Metadata) +} + +pub fn select_properties( + request: &mut GetRequest, + defaults: &[T::Property], + using: CapabilityIds, +) -> trc::Result<(Vec, MetadataSelection)> +where + T: JmapObject, + T::Property: MetadataProperty, +{ + if request.properties.is_none() { + let selection = if uses_metadata(using) { + MetadataSelection::all() + } else { + MetadataSelection::default() + }; + Ok((request.unwrap_properties(defaults), selection)) + } else { + let mut properties = request.unwrap_properties(defaults); + MetadataSelection::extract(&mut properties).map(|selection| (properties, selection)) + } +} + +fn metadata_scope(root: MetadataRoot) -> MetadataScope { + match root { + MetadataRoot::Shared => MetadataScope::Shared, + MetadataRoot::Private => MetadataScope::Private, + } +} diff --git a/crates/jmap/src/api/metadata/object.rs b/crates/jmap/src/api/metadata/object.rs new file mode 100644 index 00000000..e6ecd803 --- /dev/null +++ b/crates/jmap/src/api/metadata/object.rs @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MetadataContainers, MetadataDocuments, MetadataGet, MetadataPatches, MetadataPreload, + MetadataQuery, MetadataSupport, MetadataType, MetadataValues, PreloadedContainers, + PrivateCandidates, uses_metadata, +}; +use crate::changes::state::{MetadataStateManager, ViewerChangeId}; +use common::{ + Server, + auth::AccessToken, + storage::metadata::{MetadataLog, MetadataViewer}, +}; +use jmap_proto::{ + error::set::SetError, + object::metadata::{MetadataFilter, MetadataProperty, MetadataSelection}, + request::{MaybeInvalid, capability::CapabilityIds}, +}; +use jmap_tools::{Element, Value}; +use registry::schema::enums::Permission; +use std::borrow::Borrow; +use store::roaring::RoaringBitmap; +use trc::AddContext; +use types::{collection::Collection, id::Id, metadata::MetadataKinds}; +use utils::map::vec_map::VecMap; + +#[derive(Debug, Clone, Copy)] +pub struct ObjectMetadata { + object: MetadataType, + support: Option, + caller_id: u32, + client: MetadataClient, +} + +#[derive(Debug, Clone, Copy)] +enum MetadataClient { + Aware { viewer: Option }, + Unaware { viewer: Option }, +} + +impl ObjectMetadata { + pub fn new( + server: &Server, + access_token: &AccessToken, + using: CapabilityIds, + object: MetadataType, + ) -> Self { + let viewer = server.metadata_viewer(access_token, Permission::JmapMetadataPrivate); + let support = MetadataSupport::new(&server.core.metadata, access_token, object, viewer); + let viewer = viewer.filter(|_| support.is_some()); + ObjectMetadata { + object, + support, + caller_id: access_token.account_id(), + client: if uses_metadata(using) { + MetadataClient::Aware { viewer } + } else { + MetadataClient::Unaware { viewer } + }, + } + } + + pub fn support(&self) -> Option<&MetadataSupport> { + match self.client { + MetadataClient::Aware { .. } => self.support.as_ref(), + MetadataClient::Unaware { .. } => None, + } + } + + pub fn viewer(&self) -> Option { + match self.client { + MetadataClient::Aware { viewer } => viewer, + MetadataClient::Unaware { .. } => None, + } + } + + pub fn is_listed(&self) -> bool { + self.support.is_some() + } + + pub fn state_viewer(&self) -> Option { + match self.client { + MetadataClient::Aware { viewer } | MetadataClient::Unaware { viewer } => viewer, + } + } + + pub fn caller_id(&self) -> u32 { + self.caller_id + } + + pub fn collection(&self) -> Collection { + self.object.collection() + } + + pub fn log(&self) -> MetadataLog { + match self.object { + MetadataType::Mailbox | MetadataType::Calendar | MetadataType::AddressBook => { + MetadataLog::Container + } + MetadataType::Email + | MetadataType::SieveScript + | MetadataType::CalendarEvent + | MetadataType::ContactCard + | MetadataType::FileNode => MetadataLog::Item { prefix: None }, + } + } + + pub async fn viewer_change_id( + &self, + server: &Server, + account_id: u32, + ) -> trc::Result { + server + .viewer_change_id(self.state_viewer(), account_id, self.collection()) + .await + } + + pub async fn private_candidates( + &self, + server: &Server, + account_id: u32, + ) -> trc::Result> { + match self.viewer() { + Some(viewer) => viewer_candidates(server, viewer, account_id, self.collection()).await, + None => Ok(None), + } + } + + pub fn extract( + &self, + patches: MetadataPatches, + object: &mut Value<'_, P, E>, + ) -> Result, SetError

> + where + P: MetadataProperty, + E: Element, + { + if self.support().is_some() { + patches.extract(object) + } else { + MetadataPatches::reject_unsupported(object).map(|_| None) + } + } + + pub async fn preload_updates( + &self, + server: &Server, + account_id: u32, + updates: Option<&VecMap, Value<'_, P, E>>>, + stored: impl Fn(u32) -> Option, + ) -> trc::Result + where + P: MetadataProperty, + E: Element, + { + if self.support().is_some() { + self.preload( + server, + account_id, + MetadataPreload::from_updates(updates, stored), + ) + .await + } else { + Ok(PreloadedContainers::default()) + } + } + + pub async fn preload( + &self, + server: &Server, + account_id: u32, + preload: MetadataPreload, + ) -> trc::Result { + let collection = self.collection(); + let shared = server + .load_metadata_containers(account_id, collection, &preload.shared) + .await + .caused_by(trc::location!())?; + let private_viewer = if preload.private.is_empty() { + None + } else { + self.private_candidates(server, account_id) + .await? + .map(|candidates| candidates.viewer_id) + }; + let private = match private_viewer { + Some(viewer_id) => server + .load_private_metadata_containers( + account_id, + viewer_id, + collection, + &preload.private, + ) + .await + .caused_by(trc::location!())?, + None => MetadataContainers::default(), + }; + Ok(PreloadedContainers { shared, private }) + } + + pub fn get(&self, selection: MetadataSelection) -> Option { + MetadataGet::new(self.support.as_ref(), selection) + } + + pub async fn load( + &self, + server: &Server, + account_id: u32, + get: &MetadataGet, + documents: &MetadataDocuments, + ) -> trc::Result> + where + P: MetadataProperty + Send + Sync, + E: Element + Send + Sync, + { + let private_viewer = if get.wants_private() && !documents.requested().is_empty() { + match self.client { + MetadataClient::Aware { + viewer: Some(viewer), + } => viewer_candidates(server, viewer, account_id, self.collection()) + .await? + .map(|candidates| candidates.viewer_id), + MetadataClient::Aware { viewer: None } => None, + MetadataClient::Unaware { .. } => Some(self.caller_id), + } + } else { + None + }; + get.load(server, account_id, private_viewer, documents) + .await + } + + pub fn query<'x>( + &self, + filters: impl IntoIterator, + ) -> trc::Result { + MetadataQuery::new(self.support(), filters) + } + + pub async fn evaluate>( + &self, + server: &Server, + account_id: u32, + query: &MetadataQuery, + shared: impl FnOnce() -> B, + ) -> trc::Result> { + if query.is_empty() { + return Ok(Vec::new()); + } + let shared = query.has_shared().then(shared); + let private = if query.has_private() { + self.private_candidates(server, account_id).await? + } else { + None + }; + let empty = RoaringBitmap::new(); + query + .evaluate( + server, + account_id, + shared.as_ref().map_or(&empty, Borrow::borrow), + private, + ) + .await + } +} + +async fn viewer_candidates( + server: &Server, + viewer: MetadataViewer, + owner_id: u32, + collection: Collection, +) -> trc::Result> { + let state = server + .metadata_viewer_state(viewer, owner_id, collection) + .await + .caused_by(trc::location!())?; + Ok((state.containers > 0).then_some(PrivateCandidates { + viewer_id: viewer.account_id(), + containers: state.containers, + })) +} diff --git a/crates/jmap/src/api/metadata/patch.rs b/crates/jmap/src/api/metadata/patch.rs new file mode 100644 index 00000000..76c82c1a --- /dev/null +++ b/crates/jmap/src/api/metadata/patch.rs @@ -0,0 +1,359 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::violation::namespace_path; +use crate::matches_id; +use jmap_proto::{ + error::set::SetError, + object::{ + JmapObject, + metadata::{MetadataProperty, MetadataRoot}, + }, +}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Map, Null, Property, Value}; +use std::{cmp::Ordering, mem}; +use types::{ + id::Id, + metadata::{EncodedJson, JsonError}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum PatchMode { + Create, + #[default] + Update, +} + +#[derive(Debug, Default)] +pub struct MetadataPatches { + mode: PatchMode, + pub(super) patches: Vec, +} + +#[derive(Debug)] +pub(super) struct Patch { + pub key: PatchKey, + pub value: PatchValue, +} + +#[derive(Debug)] +pub(super) struct PatchKey { + pub root: MetadataRoot, + pub namespace: Option, + pub pointer: JsonPointer, +} + +#[derive(Debug)] +pub(super) enum PatchValue { + Remove, + Set(Result), + Replace(Vec), + Clear, +} + +#[derive(Debug)] +pub(super) struct Member { + pub namespace: String, + pub value: Result, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct InvalidValue { + pub error: JsonError, + pub depth: u32, +} + +const MAX_VALUE_NESTING: u32 = 128; +const UNSUPPORTED: &str = "Metadata is not supported on this object."; + +impl MetadataPatches { + pub fn for_create() -> Self { + MetadataPatches { + mode: PatchMode::Create, + patches: Vec::new(), + } + } + + pub fn for_update() -> Self { + MetadataPatches { + mode: PatchMode::Update, + patches: Vec::new(), + } + } + + pub fn is_empty(&self) -> bool { + self.patches.is_empty() + } + + pub fn has_shared(&self) -> bool { + self.has_root(MetadataRoot::Shared) + } + + pub fn has_private(&self) -> bool { + self.has_root(MetadataRoot::Private) + } + + fn has_root(&self, root: MetadataRoot) -> bool { + self.patches.iter().any(|patch| patch.key.root == root) + } + + pub(super) fn reject_unsupported(object: &Value<'_, P, E>) -> Result<(), SetError

> + where + P: MetadataProperty, + E: Element, + { + if !P::has_metadata(object) { + return Ok(()); + } + Err(SetError::invalid_properties() + .with_properties( + object + .as_object() + .into_iter() + .flat_map(|object| object.keys()) + .filter_map(|key| key.as_property()) + .filter(|property| property.metadata_root().is_some()) + .cloned(), + ) + .with_description(UNSUPPORTED)) + } + + pub(super) fn extract( + mut self, + object: &mut Value<'_, P, E>, + ) -> Result, SetError

> + where + P: MetadataProperty, + E: Element, + { + if !P::has_metadata(object) { + return Ok(None); + } + let Value::Object(object) = object else { + return Ok(None); + }; + let entries = mem::take(object.as_mut_vec()); + let remaining = object.as_mut_vec(); + remaining.reserve(entries.len()); + for (key, value) in entries { + match key { + Key::Property(property) if property.metadata_root().is_some() => { + self.push(&property, value)? + } + key => remaining.push((key, value)), + } + } + Ok(Some(self)) + } + + pub fn push(&mut self, property: &P, value: Value<'_, P, E>) -> Result<(), SetError

> + where + P: MetadataProperty, + E: Element, + { + let key = if let Some(root) = property.as_metadata_root() { + PatchKey::root(root) + } else if let Some((root, pointer)) = property.metadata_pointer() { + PatchKey::parse(root, pointer)? + } else { + return Err(SetError::invalid_properties() + .with_property(property.clone()) + .with_description("Not a metadata property.")); + }; + let root = key.root; + + let value = match (key.namespace.as_deref(), value) { + (None, Value::Null) if self.mode == PatchMode::Create => { + return Err(root_error(root, "Metadata must be an object on create.")); + } + (None, Value::Null) => PatchValue::Replace(Vec::new()), + (None, Value::Object(members)) => { + let mut namespaces = Vec::with_capacity(members.len()); + for (namespace, value) in members.into_vec() { + let namespace = namespace.to_string().into_owned(); + if !value.is_object() { + return Err(namespace_error( + root, + &namespace, + "Namespace values must be objects.", + )); + } + namespaces.push(Member { + value: encode(&value), + namespace, + }); + } + PatchValue::Replace(namespaces) + } + (None, _) => return Err(root_error(root, "Metadata must be an object.")), + (Some(_), _) if self.mode == PatchMode::Create => { + return Err(root_error( + root, + "Metadata patches are not allowed on create.", + )); + } + (Some(_), Value::Null) => PatchValue::Remove, + (Some(namespace), value) if key.pointer.is_empty() && !value.is_object() => { + return Err(namespace_error( + root, + namespace, + "Namespace values must be objects.", + )); + } + (Some(_), value) => PatchValue::Set(encode(&value)), + }; + + self.patches.push(Patch { key, value }); + Ok(()) + } +} + +impl PatchValue { + pub(super) fn is_removal(&self) -> bool { + match self { + PatchValue::Remove | PatchValue::Clear => true, + PatchValue::Replace(members) => members.is_empty(), + PatchValue::Set(_) => false, + } + } +} + +impl PatchKey { + fn root(root: MetadataRoot) -> Self { + PatchKey { + root, + namespace: None, + pointer: JsonPointer::new(Vec::new()), + } + } + + fn parse( + root: MetadataRoot, + pointer: &JsonPointer

, + ) -> Result> { + let mut segments = pointer.as_slice().iter().skip(1).map(|item| match item { + JsonPointerItem::Key(key) => Ok(key.to_string().into_owned()), + _ => Err(SetError::invalid_patch() + .with_property(Key::Owned(pointer.to_string())) + .with_description(format!( + "Invalid {} patch pointer {pointer}.", + root.as_str() + ))), + }); + let namespace = segments.next().transpose()?; + let items = segments + .map(|segment| segment.map(|segment| JsonPointerItem::Key(Key::Owned(segment)))) + .collect::, _>>()?; + + Ok(PatchKey { + root, + namespace, + pointer: JsonPointer::new(items), + }) + } + + pub fn cmp_path(&self, other: &Self) -> Ordering { + root_rank(self.root) + .cmp(&root_rank(other.root)) + .then_with(|| self.namespace.cmp(&other.namespace)) + .then_with(|| self.pointer.cmp(&other.pointer)) + } + + pub fn is_prefix_of(&self, other: &Self) -> bool { + self.root == other.root + && match (&self.namespace, &other.namespace) { + (None, _) => true, + (Some(namespace), Some(other_namespace)) => { + namespace == other_namespace + && other + .pointer + .as_slice() + .starts_with(self.pointer.as_slice()) + } + (Some(_), None) => false, + } + } + + pub fn is_deep(&self) -> bool { + !self.pointer.is_empty() + } + + pub fn to_path(&self) -> String { + let mut path = JsonPointer::::encode( + [self.root.as_str()] + .into_iter() + .chain(self.namespace.as_deref()), + ); + if !self.pointer.is_empty() { + path.push('/'); + path.push_str(&self.pointer.to_string()); + } + path + } +} + +fn encode>( + value: &Value<'_, P, E>, +) -> Result { + EncodedJson::encode(value).map_err(|error| InvalidValue { + error, + depth: value_depth(value, 0), + }) +} + +pub(super) fn value_depth>( + value: &Value<'_, P, E>, + nesting: u32, +) -> u32 { + if nesting >= MAX_VALUE_NESTING { + return nesting; + } + match value { + Value::Object(object) => { + 1 + object + .values() + .map(|value| value_depth(value, nesting + 1)) + .max() + .unwrap_or_default() + } + Value::Array(items) => items + .iter() + .map(|value| value_depth(value, nesting + 1)) + .max() + .unwrap_or_default(), + _ => 0, + } +} + +pub fn is_empty_update(object: &Map<'_, T::Property, T::Element>, id: Id) -> bool { + object.iter().all(|(key, value)| { + matches!(key, Key::Property(property) if *property == T::ID_PROPERTY) + && matches_id(value, id) + }) +} + +fn root_rank(root: MetadataRoot) -> u8 { + match root { + MetadataRoot::Shared => 0, + MetadataRoot::Private => 1, + } +} + +fn root_error(root: MetadataRoot, description: &'static str) -> SetError

{ + SetError::invalid_properties() + .with_property(P::from_metadata_root(root)) + .with_description(description) +} + +fn namespace_error( + root: MetadataRoot, + namespace: &str, + description: &'static str, +) -> SetError

{ + SetError::invalid_properties() + .with_property(Key::Owned(namespace_path(root, namespace))) + .with_description(description) +} diff --git a/crates/jmap/src/api/metadata/preload.rs b/crates/jmap/src/api/metadata/preload.rs new file mode 100644 index 00000000..d85efb26 --- /dev/null +++ b/crates/jmap/src/api/metadata/preload.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{MetadataContainers, MetadataPatches}; +use jmap_proto::{ + object::metadata::{MetadataProperty, MetadataRoot}, + request::MaybeInvalid, +}; +use jmap_tools::{Element, Value}; +use store::roaring::RoaringBitmap; +use types::{id::Id, metadata::MetadataKinds}; +use utils::map::vec_map::VecMap; + +#[derive(Debug, Default)] +pub struct MetadataPreload { + pub(super) shared: RoaringBitmap, + pub(super) private: RoaringBitmap, +} + +#[derive(Debug, Default)] +pub struct PreloadedContainers { + pub shared: MetadataContainers, + pub private: MetadataContainers, +} + +impl MetadataPreload { + pub(super) fn from_updates( + updates: Option<&VecMap, Value<'_, P, E>>>, + stored: impl Fn(u32) -> Option, + ) -> Self + where + P: MetadataProperty, + E: Element, + { + let mut preload = MetadataPreload::default(); + for (id, object) in updates.into_iter().flat_map(VecMap::iter) { + if let MaybeInvalid::Value(id) = id + && let Some(object) = object.as_object() + { + let mut roots = object + .keys() + .filter_map(|key| key.as_property().and_then(MetadataProperty::metadata_root)) + .peekable(); + if roots.peek().is_some() + && let Some(kinds) = stored(id.document_id()) + { + for root in roots { + preload.add_root(id.document_id(), root, kinds); + } + } + } + } + preload + } + + pub fn insert(&mut self, document_id: u32, patches: &MetadataPatches, stored: MetadataKinds) { + if patches.has_shared() { + self.add_root(document_id, MetadataRoot::Shared, stored); + } + if patches.has_private() { + self.add_root(document_id, MetadataRoot::Private, stored); + } + } + + pub fn insert_source(&mut self, document_id: u32, stored: MetadataKinds) { + self.add_root(document_id, MetadataRoot::Shared, stored); + self.add_root(document_id, MetadataRoot::Private, stored); + } + + fn add_root(&mut self, document_id: u32, root: MetadataRoot, stored: MetadataKinds) { + match root { + MetadataRoot::Shared if !stored.is_empty() => { + self.shared.insert(document_id); + } + MetadataRoot::Private => { + self.private.insert(document_id); + } + MetadataRoot::Shared => {} + } + } + + pub fn is_empty(&self) -> bool { + self.shared.is_empty() && self.private.is_empty() + } +} diff --git a/crates/jmap/src/api/metadata/prepared.rs b/crates/jmap/src/api/metadata/prepared.rs new file mode 100644 index 00000000..e2e5272e --- /dev/null +++ b/crates/jmap/src/api/metadata/prepared.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::storage::{ + dav::FilePresence, + metadata::{MetadataPresence, MetadataWrite, PrivateMetadataCommit, PrivateMetadataWrite}, +}; +use store::write::{BatchBuilder, PendingId}; +use trc::AddContext; +use types::metadata::{EncodedMetadata, MetadataKinds}; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct PreparedMetadata { + pub(super) shared: Option, + pub(super) private: Option, +} + +impl PreparedMetadata { + pub fn is_empty(&self) -> bool { + self.shared.is_none() && self.private.is_none() + } + + pub fn is_private_only(&self) -> bool { + self.shared.is_none() && self.private.is_some() + } + + pub fn shared_kinds(&self) -> Option { + self.shared.as_ref().map(|write| { + write + .next + .as_ref() + .map_or(MetadataKinds::NONE, EncodedMetadata::kinds) + }) + } + + pub fn file_presence(&self) -> Option { + self.shared.as_ref().map(|write| { + write.next.as_ref().map_or(FilePresence::NONE, |next| { + FilePresence::from_view(&next.view()) + }) + }) + } + + pub fn build( + self, + document_id: impl Into, + batch: &mut BatchBuilder, + commit: &mut PrivateMetadataCommit, + ) -> trc::Result> { + let document_id = document_id.into(); + let presence = self + .shared + .map(|write| write.build(document_id, batch)) + .transpose() + .caused_by(trc::location!())?; + if let Some(write) = self.private { + write + .build(document_id, batch, commit) + .caused_by(trc::location!())?; + } + Ok(presence) + } +} diff --git a/crates/jmap/src/api/metadata/query.rs b/crates/jmap/src/api/metadata/query.rs new file mode 100644 index 00000000..54a99a51 --- /dev/null +++ b/crates/jmap/src/api/metadata/query.rs @@ -0,0 +1,307 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::MetadataSupport; +use common::Server; +use jmap_proto::object::metadata::{MetadataCondition, MetadataFilter, MetadataRoot}; +use memchr::memmem::Finder; +use store::roaring::RoaringBitmap; +use types::{ + collation::unicode_casemap, + collection::Collection, + metadata::{MetadataKinds, MetadataView, Namespace, RegisteredNamespace}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PrivateCandidates { + pub viewer_id: u32, + pub containers: u32, +} + +#[derive(Debug)] +pub struct MetadataQuery { + collection: Collection, + max_scan: usize, + leaves: Vec, +} + +#[derive(Debug)] +struct QueryLeaf { + root: MetadataRoot, + target: Option, + condition: LeafCondition, +} + +#[derive(Debug)] +struct LeafTarget { + namespace: LeafNamespace, + keys: Box<[Box]>, +} + +#[derive(Debug)] +enum LeafNamespace { + Registered(&'static RegisteredNamespace), + Vendor(Box), +} + +#[derive(Debug)] +enum LeafCondition { + Exists, + Contains(Box>), + Equals(Box), +} + +impl MetadataQuery { + pub fn new<'x>( + support: Option<&MetadataSupport>, + filters: impl IntoIterator, + ) -> trc::Result { + let mut filters = filters.into_iter().peekable(); + let Some(support) = support else { + return match filters.peek() { + Some(filter) => Err(unsupported_filter(filter.as_str())), + None => Ok(MetadataQuery { + collection: Collection::None, + max_scan: 0, + leaves: Vec::new(), + }), + }; + }; + + let mut scratch = String::new(); + let leaves = filters + .map(|filter| QueryLeaf::new(support, filter, &mut scratch)) + .collect::>>()?; + + Ok(MetadataQuery { + collection: support.object_type().collection(), + max_scan: support.query_max_scan, + leaves, + }) + } + + pub fn is_empty(&self) -> bool { + self.leaves.is_empty() + } + + pub fn len(&self) -> usize { + self.leaves.len() + } + + pub fn has_shared(&self) -> bool { + self.searches(MetadataRoot::Shared) + } + + pub fn has_private(&self) -> bool { + self.searches(MetadataRoot::Private) + } + + pub async fn evaluate( + &self, + server: &Server, + account_id: u32, + shared: &RoaringBitmap, + private: Option, + ) -> trc::Result> { + self.check_scan_limit(MetadataRoot::Shared, shared.len())?; + if let Some(private) = private { + self.check_scan_limit(MetadataRoot::Private, u64::from(private.containers))?; + } + + let mut results = vec![RoaringBitmap::new(); self.leaves.len()]; + let mut scratch = String::new(); + if self.has_shared() && !shared.is_empty() { + server + .metadata_containers( + account_id, + self.collection, + shared, + |document_id, view, _| { + self.evaluate_container( + MetadataRoot::Shared, + document_id, + &view, + &mut results, + &mut scratch, + ); + Ok(true) + }, + ) + .await?; + } + if let Some(private) = private + && self.has_private() + { + let mut scanned = 0u64; + server + .all_private_metadata_containers( + account_id, + private.viewer_id, + self.collection, + |document_id, view, _| { + scanned += 1; + self.check_scan_limit(MetadataRoot::Private, scanned)?; + self.evaluate_container( + MetadataRoot::Private, + document_id, + &view, + &mut results, + &mut scratch, + ); + Ok(true) + }, + ) + .await?; + } + + Ok(results) + } + + pub(super) fn evaluate_container( + &self, + root: MetadataRoot, + document_id: u32, + view: &MetadataView<'_>, + results: &mut [RoaringBitmap], + scratch: &mut String, + ) { + if view.kinds().contains(MetadataKinds::JMAP) { + for (leaf, result) in self.leaves.iter().zip(results.iter_mut()) { + if leaf.root == root && leaf.matches(view, scratch) { + result.insert(document_id); + } + } + } + } + + fn searches(&self, root: MetadataRoot) -> bool { + self.leaves + .iter() + .any(|leaf| leaf.root == root && leaf.target.is_some()) + } + + pub(super) fn check_scan_limit(&self, root: MetadataRoot, candidates: u64) -> trc::Result<()> { + if candidates > self.max_scan as u64 + && let Some(leaf) = self.leaves.iter().find(|leaf| { + leaf.root == root + && leaf.target.is_some() + && !matches!(leaf.condition, LeafCondition::Exists) + }) + { + Err(trc::JmapEvent::UnsupportedFilter + .into_err() + .details(format!( + "{} is limited to {} candidates.", + leaf.name(), + self.max_scan + ))) + } else { + Ok(()) + } + } +} + +impl QueryLeaf { + fn new( + support: &MetadataSupport, + filter: &MetadataFilter, + scratch: &mut String, + ) -> trc::Result { + if filter.root() == MetadataRoot::Private && !support.supports_private() { + return Err(unsupported_filter(filter.as_str())); + } + let (root, path, condition) = match filter { + MetadataFilter::Condition { + root, + path, + condition, + } => (*root, path, condition), + MetadataFilter::Invalid { name, reason, .. } => { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format!("Invalid {name} condition: {reason}"))); + } + }; + + let target = Namespace::parse(&path.namespace) + .ok() + .map(|namespace| LeafTarget { + namespace: match namespace { + Namespace::Registered(namespace) => LeafNamespace::Registered(namespace), + Namespace::Vendor(name) => LeafNamespace::Vendor(name.into()), + }, + keys: path.keys.clone(), + }); + let condition = match condition { + MetadataCondition::Exists => LeafCondition::Exists, + MetadataCondition::TextContains(text) => { + scratch.clear(); + unicode_casemap(text, scratch); + LeafCondition::Contains(Box::new(Finder::new(scratch.as_bytes()).into_owned())) + } + MetadataCondition::TextEquals(text) => LeafCondition::Equals(text.as_str().into()), + }; + + Ok(QueryLeaf { + root, + target, + condition, + }) + } + + fn matches(&self, view: &MetadataView<'_>, scratch: &mut String) -> bool { + let Some(target) = &self.target else { + return false; + }; + let Some(namespace) = view.jmap_namespace(&target.namespace.as_namespace()) else { + return false; + }; + if target.keys.is_empty() { + return matches!(self.condition, LeafCondition::Exists) && !namespace.is_empty_object(); + } + let Some(value) = target + .keys + .iter() + .try_fold(namespace, |value, key| value.get(key)) + else { + return false; + }; + + match &self.condition { + LeafCondition::Exists => true, + LeafCondition::Contains(needle) => value.as_str().is_some_and(|text| { + scratch.clear(); + unicode_casemap(text, scratch); + needle.find(scratch.as_bytes()).is_some() + }), + LeafCondition::Equals(expected) => value.as_str() == Some(&**expected), + } + } + + fn name(&self) -> &'static str { + match (self.root, &self.condition) { + (MetadataRoot::Shared, LeafCondition::Exists) => "metadataExists", + (MetadataRoot::Shared, LeafCondition::Contains(_)) => "metadataTextContains", + (MetadataRoot::Shared, LeafCondition::Equals(_)) => "metadataTextEquals", + (MetadataRoot::Private, LeafCondition::Exists) => "privateMetadataExists", + (MetadataRoot::Private, LeafCondition::Contains(_)) => "privateMetadataTextContains", + (MetadataRoot::Private, LeafCondition::Equals(_)) => "privateMetadataTextEquals", + } + } +} + +impl LeafNamespace { + fn as_namespace(&self) -> Namespace<'_> { + match self { + LeafNamespace::Registered(namespace) => Namespace::Registered(namespace), + LeafNamespace::Vendor(name) => Namespace::Vendor(name), + } + } +} + +fn unsupported_filter(name: &'static str) -> trc::Error { + trc::JmapEvent::UnsupportedFilter.into_err().details(name) +} diff --git a/crates/jmap/src/api/metadata/reject.rs b/crates/jmap/src/api/metadata/reject.rs new file mode 100644 index 00000000..e8ded7b3 --- /dev/null +++ b/crates/jmap/src/api/metadata/reject.rs @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{error::set::SetError, method::set::SetResponse, object::JmapObject}; +use std::mem; + +pub fn reject_uncommitted( + response: &mut SetResponse, + creates: impl IntoIterator, + description: &'static str, +) { + let error = || SetError::forbidden().with_description(description); + for create_id in creates { + response.not_created.append(create_id, error()); + } + for (id, _) in mem::take(&mut response.updated) { + response.not_updated.append(id, error()); + } + for id in mem::take(&mut response.destroyed) { + response.not_destroyed.append(id, error()); + } +} diff --git a/crates/jmap/src/api/metadata/validate.rs b/crates/jmap/src/api/metadata/validate.rs new file mode 100644 index 00000000..70a566f0 --- /dev/null +++ b/crates/jmap/src/api/metadata/validate.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MetadataSupport, + patch::{MetadataPatches, Patch, PatchValue}, + violation::{Rule, Violation, Violations, namespace_path}, + write::MetadataUpdate, +}; +use jmap_proto::{ + error::set::SetError, + object::metadata::{MetadataProperty, MetadataRoot}, +}; +use store::write::metadata::MetadataBuf; +use types::{ + acl::Acl, + metadata::{Namespace, NamespaceError}, +}; +use utils::map::bitmap::Bitmap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetadataAccess { + pub may_write_shared: bool, + pub may_read: bool, +} + +impl MetadataAccess { + pub const FULL: Self = MetadataAccess { + may_write_shared: true, + may_read: true, + }; + + pub fn from_container_acls(acls: impl IntoIterator>) -> Self { + acls.into_iter().fold( + MetadataAccess { + may_write_shared: true, + may_read: false, + }, + |access, acl| MetadataAccess { + may_write_shared: access.may_write_shared && acl.contains(Acl::ModifyMetadata), + may_read: access.may_read || acl.contains(Acl::ReadItems), + }, + ) + } +} + +#[derive(Debug)] +pub struct ValidatedPatches { + pub(super) support: MetadataSupport, + pub(super) shared: Vec, + pub(super) private: Vec, +} + +impl MetadataPatches { + pub fn validate( + self, + support: &MetadataSupport, + access: MetadataAccess, + ) -> Result> { + let mut patches = self.patches; + patches.sort_unstable_by(|a, b| a.key.cmp_path(&b.key)); + + let mut violations = Violations::default(); + if let Some([first, second]) = patches + .windows(2) + .find(|pair| matches!(pair, [first, second] if first.key.is_prefix_of(&second.key))) + { + violations.report(Rule::Syntax, || Violation { + rule: Rule::Syntax, + root: second.key.root, + path: Some(second.key.to_path()), + description: format!( + "Patch {} overlaps with patch {}.", + second.key.to_path(), + first.key.to_path() + ) + .into(), + }); + return Err(violations.into_set_error()); + } + + let private = patches + .split_off(patches.partition_point(|patch| patch.key.root == MetadataRoot::Shared)); + let shared = patches; + + for (root, patches, is_allowed) in [ + ( + MetadataRoot::Shared, + &shared, + support.writable && access.may_write_shared, + ), + (MetadataRoot::Private, &private, access.may_read), + ] { + if !patches.is_empty() && !is_allowed { + violations.report(Rule::Forbidden, || Violation { + rule: Rule::Forbidden, + root, + path: None, + description: format!("Not allowed to modify {}.", root.as_str()).into(), + }); + } + } + if !private.is_empty() && !support.private { + violations.report(Rule::Private, || Violation { + rule: Rule::Private, + root: MetadataRoot::Private, + path: None, + description: "Private metadata is not supported on this object.".into(), + }); + } + for patch in shared.iter().chain(&private) { + check_namespaces(support, patch, &mut violations); + } + + if violations.is_empty() { + Ok(ValidatedPatches { + support: *support, + shared, + private, + }) + } else { + Err(violations.into_set_error()) + } + } + + pub fn apply( + self, + support: &MetadataSupport, + access: MetadataAccess, + shared: Option<&MetadataBuf>, + private: Option<&MetadataBuf>, + ) -> Result> { + self.validate(support, access)?.apply(shared, private) + } +} + +impl ValidatedPatches { + pub fn has_shared(&self) -> bool { + !self.shared.is_empty() + } + + pub fn has_private(&self) -> bool { + !self.private.is_empty() + } +} + +fn check_namespaces(support: &MetadataSupport, patch: &Patch, violations: &mut Violations) { + let root = patch.key.root; + let mut report = |name: &str, description: &'static str| { + violations.report(Rule::Namespace, || Violation { + rule: Rule::Namespace, + root, + path: Some(namespace_path(root, name)), + description: description.into(), + }) + }; + + match (patch.key.namespace.as_deref(), &patch.value) { + (None, PatchValue::Replace(members)) => { + for member in members { + match Namespace::parse(&member.namespace) { + Ok(_) => {} + Err(NamespaceError::Unregistered) => { + report(&member.namespace, "Unsupported namespace.") + } + Err(NamespaceError::Invalid) => { + report(&member.namespace, "Invalid namespace name.") + } + } + } + } + (Some(name), PatchValue::Remove) if !patch.key.is_deep() => { + if let Err(NamespaceError::Invalid) = Namespace::parse(name) { + report(name, "Invalid namespace name."); + } + } + (Some(name), _) => match Namespace::parse(name) { + Ok(namespace) if support.is_supported(&namespace, root) => {} + Ok(_) | Err(NamespaceError::Unregistered) => report(name, "Unsupported namespace."), + Err(NamespaceError::Invalid) => report(name, "Invalid namespace name."), + }, + (None, _) => {} + } +} diff --git a/crates/jmap/src/api/metadata/violation.rs b/crates/jmap/src/api/metadata/violation.rs new file mode 100644 index 00000000..4a44f217 --- /dev/null +++ b/crates/jmap/src/api/metadata/violation.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{ + error::set::SetError, + object::metadata::{MetadataProperty, MetadataRoot}, +}; +use jmap_tools::{JsonPointer, Key, Null}; +use std::borrow::Cow; +use types::metadata::LimitViolation; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum Rule { + Syntax, + Forbidden, + Private, + Namespace, + Parent, + Depth, + ControlCharacter, + EntrySize, + Entries, + ContainerSize, +} + +#[derive(Debug)] +pub(super) struct Violation { + pub rule: Rule, + pub root: MetadataRoot, + pub path: Option, + pub description: Cow<'static, str>, +} + +#[derive(Debug, Default)] +pub(super) struct Violations(Option); + +impl Violations { + pub fn report(&mut self, rule: Rule, violation: impl FnOnce() -> Violation) { + if self.0.as_ref().is_none_or(|first| rule < first.rule) { + self.0 = Some(violation()); + } + } + + pub fn is_empty(&self) -> bool { + self.0.is_none() + } + + pub fn into_set_error(self) -> SetError

{ + let Some(violation) = self.0 else { + return SetError::invalid_properties(); + }; + let error = match violation.rule { + Rule::Syntax | Rule::Parent => SetError::invalid_patch(), + Rule::Forbidden => SetError::forbidden(), + _ => SetError::invalid_properties(), + }; + match violation.path { + Some(path) => error.with_property(Key::Owned(path)), + None => error.with_property(P::from_metadata_root(violation.root)), + } + .with_description(violation.description) + } +} + +pub(super) fn namespace_path(root: MetadataRoot, name: &str) -> String { + JsonPointer::::encode([root.as_str(), name]) +} + +pub(super) fn limit_description(violation: LimitViolation) -> String { + match violation { + LimitViolation::Depth { depth, max } => { + format!("Metadata value depth {depth} exceeds the maximum of {max}.") + } + LimitViolation::EntrySize { size, max } => { + format!("Metadata value size {size} exceeds the maximum of {max} bytes.") + } + LimitViolation::ContainerSize { size, max } => { + format!("Metadata size {size} exceeds the maximum of {max} bytes.") + } + LimitViolation::Entries { count, max } => { + format!("Metadata entry count {count} exceeds the maximum of {max}.") + } + } +} diff --git a/crates/jmap/src/api/metadata/write.rs b/crates/jmap/src/api/metadata/write.rs new file mode 100644 index 00000000..427130a2 --- /dev/null +++ b/crates/jmap/src/api/metadata/write.rs @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::prepared::PreparedMetadata; +use common::{ + Server, + auth::AccountCache, + storage::metadata::{ContainerChange, MetadataLog, PrivateMetadataWrite}, +}; +use email::message::ingest_metadata::IngestMetadata; +use jmap_proto::{error::set::SetError, object::metadata::MetadataProperty}; +use std::sync::Arc; +use types::collection::Collection; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct MetadataUpdate { + shared: Option, + private: Option, +} + +pub(super) struct ContainerTarget<'x> { + pub owner: &'x AccountCache, + pub viewer_id: u32, + pub viewer: &'x mut Option>, + pub collection: Collection, + pub log: MetadataLog, +} + +impl MetadataUpdate { + pub(super) fn new(shared: Option, private: Option) -> Self { + MetadataUpdate { shared, private } + } + + pub fn is_empty(&self) -> bool { + self.shared.is_none() && self.private.is_none() + } + + pub fn shared(&self) -> Option<&ContainerChange> { + self.shared.as_ref() + } + + pub fn private(&self) -> Option<&ContainerChange> { + self.private.as_ref() + } + + pub(super) async fn prepare( + self, + server: &Server, + target: ContainerTarget<'_>, + ) -> trc::Result>> { + let shared = match self.shared { + Some(change) => { + if !server.has_metadata_quota(target.owner, &change).await? { + return Ok(Err(SetError::over_quota())); + } + Some(change.into_write(target.owner, target.collection, target.log)) + } + None => None, + }; + let private = match self.private { + Some(change) => match target.private_write(server, change).await? { + Ok(write) => Some(write), + Err(err) => return Ok(Err(err)), + }, + None => None, + }; + Ok(Ok(PreparedMetadata { shared, private })) + } + + pub(super) async fn prepare_ingest( + self, + server: &Server, + target: ContainerTarget<'_>, + ) -> trc::Result>, SetError

>> { + let private = match self.private { + Some(change) => match target.private_write(server, change).await? { + Ok(write) => Some(write), + Err(err) => return Ok(Err(err)), + }, + None => None, + }; + Ok(Ok(IngestMetadata::new( + self.shared.and_then(ContainerChange::into_next), + private, + ))) + } +} + +impl ContainerTarget<'_> { + async fn private_write( + self, + server: &Server, + change: ContainerChange, + ) -> trc::Result>> { + let owner = self.owner; + let viewer: &AccountCache = if self.viewer_id == owner.id { + owner + } else { + let viewer = match self.viewer.take() { + Some(viewer) => viewer, + None => server.account(self.viewer_id).await?, + }; + self.viewer.insert(viewer) + }; + if !server.has_metadata_quota(viewer, &change).await? { + return Ok(Err(SetError::over_quota())); + } + Ok(Ok(change.into_private_write( + owner.id, + viewer, + self.collection, + self.log, + ))) + } +} diff --git a/crates/jmap/src/api/metadata/writer.rs b/crates/jmap/src/api/metadata/writer.rs new file mode 100644 index 00000000..5d2bf3a6 --- /dev/null +++ b/crates/jmap/src/api/metadata/writer.rs @@ -0,0 +1,301 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MetadataAccess, MetadataPatches, MetadataPreload, MetadataUpdate, ObjectMetadata, + PreloadedContainers, PreparedMetadata, write::ContainerTarget, +}; +use common::{ + Server, + auth::AccountCache, + storage::{ + index::RewritePresence, + metadata::{MetadataLog, MetadataPresence, PrivateMetadataCommit}, + }, +}; +use email::message::ingest_metadata::IngestMetadata; +use jmap_proto::{error::set::SetError, object::metadata::MetadataProperty, request::MaybeInvalid}; +use jmap_tools::{Element, Value}; +use std::{mem, sync::Arc}; +use store::write::{Archive, AssignedIds, BatchBuilder, PendingId}; +use trc::AddContext; +use types::{id::Id, metadata::MetadataKinds}; +use utils::map::vec_map::VecMap; + +pub struct MetadataWriter { + metadata: ObjectMetadata, + account_id: u32, + owner: Option>, + viewer: Option>, + containers: PreloadedContainers, + commit: PrivateMetadataCommit, +} + +#[derive(Debug)] +pub enum NewMetadata { + Create(MetadataPatches), + Copy { + patches: Option, + source_id: u32, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetadataTarget { + Create, + Update { document_id: u32 }, + UpdateInThread { document_id: u32, thread_id: u32 }, +} + +impl MetadataWriter { + pub fn new(metadata: ObjectMetadata, account_id: u32) -> Self { + MetadataWriter { + metadata, + account_id, + owner: None, + viewer: None, + containers: PreloadedContainers::default(), + commit: PrivateMetadataCommit::default(), + } + } + + pub fn with_owner(mut self, owner: Arc) -> Self { + self.owner = Some(owner); + self + } + + pub fn metadata(&self) -> &ObjectMetadata { + &self.metadata + } + + pub fn extract( + &self, + patches: MetadataPatches, + object: &mut Value<'_, P, E>, + ) -> Result, SetError

> + where + P: MetadataProperty, + E: Element, + { + self.metadata.extract(patches, object) + } + + pub async fn preload( + &mut self, + server: &Server, + account_id: u32, + preload: MetadataPreload, + ) -> trc::Result<()> { + if !preload.is_empty() { + self.containers = self.metadata.preload(server, account_id, preload).await?; + } + Ok(()) + } + + pub async fn preload_updates( + &mut self, + server: &Server, + updates: Option<&VecMap, Value<'_, P, E>>>, + stored: impl Fn(u32) -> Option, + ) -> trc::Result<()> + where + P: MetadataProperty, + E: Element, + { + self.containers = self + .metadata + .preload_updates(server, self.account_id, updates, stored) + .await?; + Ok(()) + } + + pub async fn prepare_for( + &mut self, + server: &Server, + patches: MetadataPatches, + access: MetadataAccess, + target: MetadataTarget, + ) -> trc::Result>> { + let Some(support) = self.metadata.support() else { + return Ok(Ok(PreparedMetadata::default())); + }; + let validated = match patches.validate(support, access) { + Ok(validated) => validated, + Err(err) => return Ok(Err(err)), + }; + let (document_id, log) = match target { + MetadataTarget::Create => (None, MetadataLog::None), + MetadataTarget::Update { document_id } => (Some(document_id), self.metadata.log()), + MetadataTarget::UpdateInThread { + document_id, + thread_id, + } => ( + Some(document_id), + MetadataLog::Item { + prefix: Some(PendingId::Assigned(thread_id)), + }, + ), + }; + let (shared, private) = document_id.map_or((None, None), |document_id| { + ( + self.containers.shared.get(document_id), + self.containers.private.get(document_id), + ) + }); + match validated.apply(shared, private) { + Ok(update) => self.finish_prepare(server, update, log).await, + Err(err) => Ok(Err(err)), + } + } + + pub async fn prepare_ingest( + &mut self, + server: &Server, + metadata: NewMetadata, + access: MetadataAccess, + ) -> trc::Result>, SetError

>> { + let update = match self.new_update(metadata, access) { + Ok(update) if update.is_empty() => return Ok(Ok(None)), + Ok(update) => update, + Err(err) => return Ok(Err(err)), + }; + let target = self.target(server, MetadataLog::None).await?; + update.prepare_ingest(server, target).await + } + + pub async fn prepare_new( + &mut self, + server: &Server, + metadata: Option, + access: MetadataAccess, + ) -> trc::Result, SetError

>> { + let Some(metadata) = metadata else { + return Ok(Ok(None)); + }; + match self.new_update(metadata, access) { + Ok(update) => Ok(self + .finish_prepare(server, update, MetadataLog::None) + .await? + .map(Some)), + Err(err) => Ok(Err(err)), + } + } + + fn new_update( + &self, + metadata: NewMetadata, + access: MetadataAccess, + ) -> Result> { + let (patches, source_id) = match metadata { + NewMetadata::Create(patches) => (Some(patches), None), + NewMetadata::Copy { patches, source_id } => (patches, Some(source_id)), + }; + let shared = source_id.and_then(|source_id| self.containers.shared.get(source_id)); + let Some(support) = self.metadata.support() else { + return Ok(if self.metadata.is_listed() { + MetadataUpdate::copied(shared, None) + } else { + MetadataUpdate::copied_without_jmap(shared) + }); + }; + let validated = patches + .unwrap_or_else(MetadataPatches::for_create) + .validate(support, access)?; + match source_id { + Some(source_id) => validated.apply_to_copy( + shared, + self.containers + .private + .get(source_id) + .filter(|_| access.may_read), + ), + None => validated.apply(None, None), + } + } + + async fn finish_prepare( + &mut self, + server: &Server, + update: MetadataUpdate, + log: MetadataLog, + ) -> trc::Result>> { + if update.is_empty() { + return Ok(Ok(PreparedMetadata::default())); + } + let target = self.target(server, log).await?; + update.prepare(server, target).await + } + + async fn target( + &mut self, + server: &Server, + log: MetadataLog, + ) -> trc::Result> { + let owner = match self.owner.take() { + Some(owner) => owner, + None => server + .account(self.account_id) + .await + .caused_by(trc::location!())?, + }; + Ok(ContainerTarget { + owner: self.owner.insert(owner), + viewer_id: self.metadata.caller_id(), + viewer: &mut self.viewer, + collection: self.metadata.collection(), + log, + }) + } + + pub fn write( + &mut self, + prepared: PreparedMetadata, + document_id: impl Into, + batch: &mut BatchBuilder, + ) -> trc::Result> { + prepared.build(document_id, batch, &mut self.commit) + } + + pub async fn write_metadata_only( + &mut self, + server: &Server, + patches: MetadataPatches, + access: MetadataAccess, + current: &Archive<&T::Archived>, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result>> { + let prepared = match self + .prepare_for( + server, + patches, + access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => prepared, + Err(err) => return Ok(Err(err)), + }; + if let Some(kinds) = prepared.shared_kinds() { + T::rewrite_presence(current, kinds, self.account_id, document_id, batch) + .caused_by(trc::location!())?; + } + self.write(prepared, document_id, batch)?; + batch.commit_point(); + Ok(Ok(())) + } + + pub async fn commit( + &mut self, + server: &Server, + batch: BatchBuilder, + ) -> trc::Result { + server + .commit_metadata_batch(batch, mem::take(&mut self.commit)) + .await + } +} diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 62377d09..625321df 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -6,7 +6,8 @@ use crate::blob::UploadResponse; use calcard::jscalendar::JSCalendarDateTime; -use common::ipc::{CalendarAlert, PushNotification}; +use common::ipc::{CalendarAlert, PushNotification, ViewerStateChange}; +use common::network::memory::MEMORY_RETRY_AFTER; use http_proto::{HttpResponse, JsonResponse, ToHttpResponse}; use hyper::StatusCode; use jmap_proto::{ @@ -15,12 +16,17 @@ use jmap_proto::{ response::{Response, status::PushObject}, types::state::State, }; +use store::write::unavailable_retry_after; use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; pub mod acl; pub mod auth; +pub mod budget; pub mod event_source; +pub mod metadata; +pub mod parent_ref; +pub mod pending_creates; pub mod query; pub mod request; pub mod session; @@ -37,7 +43,20 @@ pub trait ToJmapHttpResponse { impl ToJmapHttpResponse for Response<'_> { fn into_http_response(self) -> HttpResponse { - JsonResponse::new(self).into_http_response() + match self.to_json() { + Ok(body) => HttpResponse::new(StatusCode::OK) + .with_content_type("application/json; charset=utf-8") + .with_text_body(body), + Err(err) => { + trc::error!( + trc::ResourceEvent::Error + .into_err() + .details("Failed to serialize JMAP response") + .reason(err) + ); + RequestError::internal_server_error().into_http_response() + } + } } } @@ -120,7 +139,9 @@ impl ToRequestError for trc::Error { .with_rate_limit(policy) } trc::LimitEvent::Quota => RequestError::over_quota(), - trc::LimitEvent::TenantQuota => RequestError::tenant_over_quota(), + trc::LimitEvent::MemoryBudget => { + RequestError::unavailable().with_retry_after(MEMORY_RETRY_AFTER) + } trc::LimitEvent::BlobQuota => { let mut err = RequestError::over_blob_quota( total.unwrap_or(0) as usize, @@ -190,6 +211,9 @@ impl ToRequestError for trc::Error { RequestError::internal_server_error() } }, + trc::EventType::Store(_) if let Some(retry_after) = unavailable_retry_after(self) => { + RequestError::unavailable().with_retry_after(retry_after) + } trc::EventType::Resource(cause) => match cause { trc::ResourceEvent::NotFound => RequestError::not_found(), trc::ResourceEvent::BadParameters => RequestError::blank( @@ -216,7 +240,11 @@ pub(crate) fn notifications_into_push_objects( let mut objects = Vec::with_capacity(notifications.len()); for notification in notifications { match notification { - PushNotification::StateChange(state_change) => { + PushNotification::StateChange(state_change) + | PushNotification::ViewerStateChange(ViewerStateChange { + change: state_change, + .. + }) => { for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) @@ -246,7 +274,7 @@ pub(crate) fn notifications_into_push_objects( impl IntoPushObject for CalendarAlert { fn into_push_object(self) -> PushObject { PushObject::CalendarAlert { - account_id: self.account_id.into(), + account_id: self.event_account_id.into(), calendar_event_id: self.event_id.into(), uid: self.uid, recurrence_id: self diff --git a/crates/jmap/src/api/parent_ref.rs b/crates/jmap/src/api/parent_ref.rs new file mode 100644 index 00000000..0b4d4c7f --- /dev/null +++ b/crates/jmap/src/api/parent_ref.rs @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::object::AnyId; +use store::{ + ahash::AHashMap, + write::{ + Slot, + log::{PENDING_ID_MARKER, is_pending_id}, + }, +}; +use types::id::Id; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ParentRef { + #[default] + Root, + Node(u32), + Pending(Slot), +} + +impl ParentRef { + pub const ROOT: ParentRef = ParentRef::Root; + + pub fn pending(slot: Slot) -> Self { + ParentRef::Pending(slot) + } + + pub fn concrete(document_id: u32) -> Self { + ParentRef::Node(document_id) + } + + pub fn from_id(id: Id) -> Self { + if is_pending_id(id.id()) { + ParentRef::Pending(Slot::new((id.id() & u32::MAX as u64) as u32)) + } else { + ParentRef::Node(id.document_id()) + } + } + + pub fn from_stored(parent_id: u32) -> Self { + match parent_id { + 0 => ParentRef::Root, + parent_id => ParentRef::Node(parent_id - 1), + } + } + + pub fn to_id(self) -> Option { + match self { + ParentRef::Root => None, + ParentRef::Node(document_id) => Some(Id::from(document_id)), + ParentRef::Pending(slot) => Some(Id::new(PENDING_ID_MARKER | slot.index() as u64)), + } + } + + pub fn is_root(self) -> bool { + matches!(self, ParentRef::Root) + } + + pub fn is_pending(self) -> bool { + matches!(self, ParentRef::Pending(_)) + } + + pub fn slot(self) -> Option { + match self { + ParentRef::Pending(slot) => Some(slot), + _ => None, + } + } + + pub fn document_id(self) -> Option { + match self { + ParentRef::Node(document_id) => Some(document_id), + _ => None, + } + } + + pub fn cache_id(self) -> Option> { + match self { + ParentRef::Root => Some(None), + ParentRef::Node(document_id) => Some(Some(document_id)), + ParentRef::Pending(_) => None, + } + } + + pub fn as_stored(self) -> u32 { + match self { + ParentRef::Root | ParentRef::Pending(_) => 0, + ParentRef::Node(document_id) => document_id + 1, + } + } + + fn as_packed(self) -> u64 { + match self { + ParentRef::Root => 0, + ParentRef::Node(document_id) => document_id as u64 + 1, + ParentRef::Pending(slot) => PENDING_ID_MARKER | slot.index() as u64, + } + } +} + +impl std::hash::Hash for ParentRef { + #[inline(always)] + fn hash(&self, state: &mut H) { + state.write_u64(self.as_packed()); + } +} + +pub struct CreateResolver<'x>(&'x AHashMap); + +impl<'x> CreateResolver<'x> { + pub fn new(pending_creates: &'x AHashMap) -> Self { + CreateResolver(pending_creates) + } + + pub fn created_id(&self, id_ref: &str) -> Option { + self.0 + .get(id_ref) + .and_then(|slot| ParentRef::Pending(*slot).to_id()) + .map(AnyId::Id) + } + + fn contains_slot(&self, slot: Slot) -> bool { + self.0.values().any(|created| *created == slot) + } +} + +impl ParentRef { + pub fn from_client_id(id: Id, resolver: Option<&CreateResolver<'_>>) -> Option { + match ParentRef::from_id(id) { + ParentRef::Pending(slot) + if resolver.is_none_or(|resolver| !resolver.contains_slot(slot)) => + { + None + } + parent => Some(parent), + } + } +} diff --git a/crates/jmap/src/api/pending_creates.rs b/crates/jmap/src/api/pending_creates.rs new file mode 100644 index 00000000..ae922c74 --- /dev/null +++ b/crates/jmap/src/api/pending_creates.rs @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{method::set::SetResponse, object::JmapObject}; +use store::write::{AssignedIds, Slot}; + +#[derive(Debug, Default)] +pub struct PendingCreates(Vec<(String, Slot)>); + +impl PendingCreates { + pub fn new() -> Self { + PendingCreates(Vec::new()) + } + + pub fn push(&mut self, create_id: String, slot: Slot) { + self.0.push((create_id, slot)); + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn into_create_ids(self) -> impl Iterator { + self.0.into_iter().map(|(create_id, _)| create_id) + } + + pub fn resolve(self, response: &mut SetResponse, assigned_ids: &AssignedIds) + where + T: JmapObject, + u32: Into, + { + for (create_id, slot) in self.0 { + response.created(create_id, assigned_ids.slot(slot)); + } + } +} diff --git a/crates/jmap/src/api/query.rs b/crates/jmap/src/api/query.rs index 212e93a5..5852244b 100644 --- a/crates/jmap/src/api/query.rs +++ b/crates/jmap/src/api/query.rs @@ -20,6 +20,7 @@ pub struct QueryResponseBuilder { pub has_anchor: bool, pub anchor_found: bool, index: i32, + position_override: Option, pub response: QueryResponse, } @@ -31,15 +32,9 @@ impl QueryResponseBuilder { query_state: State, request: &QueryRequest, ) -> Self { - let (limit_total, limit) = if let Some(limit) = request.limit { - if limit > 0 { - let limit = std::cmp::min(limit, max_results); - (std::cmp::min(limit, total_results), limit) - } else { - (0, 0) - } - } else { - (std::cmp::min(max_results, total_results), max_results) + let limit_total = match request.limit { + Some(limit) => limit.min(max_results).min(total_results), + None => max_results.min(total_results), }; QueryResponseBuilder { @@ -51,6 +46,7 @@ impl QueryResponseBuilder { anchor_offset: request.anchor_offset.unwrap_or(0), anchor_found: false, index: 0, + position_override: None, response: QueryResponse { account_id: request.account_id, query_state, @@ -62,11 +58,11 @@ impl QueryResponseBuilder { } else { None }, - limit: if total_results > limit { - Some(limit) - } else { - None - }, + limit: match request.limit { + Some(requested) => requested > max_results, + None => total_results > max_results, + } + .then_some(max_results), }, } } @@ -84,11 +80,13 @@ impl QueryResponseBuilder { if self.position >= 0 { if self.position > 0 { self.position -= 1; - } else { + } else if self.response.ids.len() < self.limit { self.response.ids.push(id); if self.response.ids.len() == self.limit { return false; } + } else { + return false; } } else { self.response.ids.push(id); @@ -104,15 +102,17 @@ impl QueryResponseBuilder { if self.anchor_offset >= 0 { if self.anchor_found && current_index >= self.position { + if self.response.ids.len() >= self.limit { + return false; + } self.response.ids.push(id); - if self.limit > 0 && self.response.ids.len() == self.limit { + if self.response.ids.len() == self.limit { return false; } } } else { self.response.ids.push(id); if self.anchor_found - && self.limit > 0 && self.response.ids.len() >= self.position as usize + self.limit { return false; @@ -128,6 +128,11 @@ impl QueryResponseBuilder { } pub fn build(mut self) -> trc::Result { + if let Some(position) = self.position_override { + self.response.position = i32::try_from(position).unwrap_or(i32::MAX); + return Ok(self.response); + } + if self.has_anchor { if !self.anchor_found { return Err(trc::JmapEvent::AnchorNotFound.into_err()); @@ -136,11 +141,7 @@ impl QueryResponseBuilder { let start = self.position.max(0) as usize; if self.anchor_offset < 0 { let start = start.min(self.response.ids.len()); - let end = if self.limit > 0 { - std::cmp::min(start + self.limit, self.response.ids.len()) - } else { - self.response.ids.len() - }; + let end = std::cmp::min(start + self.limit, self.response.ids.len()); self.response.ids = self.response.ids[start..end].to_vec(); } self.response.position = start as i32; @@ -162,11 +163,7 @@ impl QueryResponseBuilder { 0 }; self.response.position = start_offset as i32; - let end_offset = if self.limit > 0 { - std::cmp::min(start_offset + self.limit, self.response.ids.len()) - } else { - self.response.ids.len() - }; + let end_offset = std::cmp::min(start_offset + self.limit, self.response.ids.len()); self.response.ids = self.response.ids[start_offset..end_offset].to_vec(); } @@ -174,3 +171,56 @@ impl QueryResponseBuilder { Ok(self.response) } } + +impl QueryResponseBuilder { + pub fn with_position(mut self, position: usize) -> Self { + self.position_override = Some(position); + self + } +} + +impl QueryResponseBuilder { + pub fn new_streaming( + max_results: usize, + query_state: State, + request: &QueryRequest, + ) -> Self { + let limit = request + .limit + .map_or(max_results, |limit| limit.min(max_results)); + + QueryResponseBuilder { + requested_position: request.position.unwrap_or(0), + position: request.position.unwrap_or(0), + limit, + has_anchor: request.anchor.is_some(), + anchor: request.anchor.map(|anchor| anchor.id()).unwrap_or(0), + anchor_offset: request.anchor_offset.unwrap_or(0), + anchor_found: false, + index: 0, + position_override: None, + response: QueryResponse { + account_id: request.account_id, + query_state, + can_calculate_changes: true, + position: 0, + ids: vec![], + total: request.calculate_total.unwrap_or(false).then_some(0), + limit: request + .limit + .is_none_or(|limit| limit > max_results) + .then_some(max_results), + }, + } + } + + pub fn wants_total(&self) -> bool { + self.response.total.is_some() + } + + pub fn finish(mut self, total: Option) -> trc::Result { + debug_assert_eq!(self.response.total.is_some(), total.is_some()); + self.response.total = self.response.total.and(total); + self.build() + } +} diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 8ccd5445..f74c2ca2 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -45,16 +45,17 @@ use crate::{ thread::get::ThreadGet, vacation::{get::VacationResponseGet, set::VacationResponseSet}, }; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, network::memory::MemoryReservation}; +use compact_str::format_compact; use http_proto::HttpSessionData; use jmap_proto::{ request::{ Call, CopyRequestMethod, GetRequestMethod, INVALID_ACCOUNT_ID, ParseRequestMethod, QueryRequestMethod, Request, RequestMethod, SetRequestMethod, - capability::Capability, + capability::{Capability, CapabilityIds}, method::{MethodName, MethodObject}, }, - response::{Response, ResponseMethod, SetResponseMethod}, + response::{CopyResponseMethod, Response, ResponseMethod, SetResponseMethod}, }; use std::future::Future; use std::time::Instant; @@ -67,8 +68,10 @@ pub trait RequestHandler: Sync + Send { request: Request<'x>, access_token: &AccessToken, session: &HttpSessionData, + reservation: &mut MemoryReservation, ) -> impl Future> + Send; + #[allow(clippy::too_many_arguments)] fn handle_method_call<'x>( &self, method: RequestMethod<'x>, @@ -76,6 +79,8 @@ pub trait RequestHandler: Sync + Send { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> impl Future>> + Send; } @@ -85,6 +90,7 @@ impl RequestHandler for Server { request: Request<'x>, access_token: &AccessToken, session: &HttpSessionData, + reservation: &mut MemoryReservation, ) -> Response<'x> { let add_created_ids = request.created_ids.is_some(); let using = request.using; @@ -106,12 +112,12 @@ impl RequestHandler for Server { } if !matches!(call.method, RequestMethod::Error(_)) { - let capability = call.name.obj.capability(); + let capability = call.name.capability(); if capability != Capability::Stalwart && !using.contains(capability) { response.push_response( call.id, MethodName::error(), - trc::JmapEvent::UnknownMethod.into_err().details(format!( + trc::JmapEvent::UnknownMethod.into_err().details(format_compact!( "Method {} requires capability {} which is not present in the \"using\" property.", call.name, capability.as_str() @@ -133,6 +139,8 @@ impl RequestHandler for Server { access_token, &mut next_call, session, + using, + reservation, ) .await { @@ -189,6 +197,24 @@ impl RequestHandler for Server { } } } + ResponseMethod::Copy(copy_response) => { + // Add created ids + match copy_response { + CopyResponseMethod::Email(copy_response) => { + copy_response.update_created_ids(&mut response); + } + CopyResponseMethod::ContactCard(copy_response) => { + copy_response.update_created_ids(&mut response); + } + CopyResponseMethod::CalendarEvent(copy_response) => { + copy_response.update_created_ids(&mut response); + } + CopyResponseMethod::FileNode(copy_response) => { + copy_response.update_created_ids(&mut response); + } + CopyResponseMethod::Blob(_) => {} + } + } ResponseMethod::ImportEmail(import_response) => { // Add created ids import_response.update_created_ids(&mut response); @@ -241,6 +267,8 @@ impl RequestHandler for Server { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> trc::Result> { let op_start = Instant::now(); @@ -254,13 +282,15 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Email)?; - self.email_get(*req, access_token).await?.into() + self.email_get(*req, access_token, using, reservation) + .await? + .into() } GetRequestMethod::Mailbox(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Mailbox)?; - self.mailbox_get(*req, access_token).await?.into() + self.mailbox_get(*req, access_token, using).await?.into() } GetRequestMethod::Thread(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -288,7 +318,9 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; - self.sieve_script_get(*req).await?.into() + self.sieve_script_get(*req, access_token, using) + .await? + .into() } GetRequestMethod::VacationResponse(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -315,39 +347,48 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::AddressBook)?; - self.address_book_get(*req, access_token).await?.into() + self.address_book_get(*req, access_token, using) + .await? + .into() } GetRequestMethod::ContactCard(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::ContactCard)?; - self.contact_card_get(*req, access_token).await?.into() + self.contact_card_get(*req, access_token, using, reservation) + .await? + .into() } GetRequestMethod::FileNode(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::FileNode)?; - self.file_node_get(*req, access_token).await?.into() + self.file_node_get(*req, access_token, using).await?.into() + } + GetRequestMethod::PrincipalAvailability(mut req) => { + resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; + + self.principal_get_availability(*req, access_token) + .await? + .into() } - GetRequestMethod::PrincipalAvailability(req) => self - .principal_get_availability(*req, access_token) - .await? - .into(), GetRequestMethod::Calendar(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Calendar)?; - self.calendar_get(*req, access_token).await?.into() + self.calendar_get(*req, access_token, using).await?.into() } GetRequestMethod::CalendarEvent(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; - self.calendar_event_get(*req, access_token).await?.into() + self.calendar_event_get(*req, access_token, using, reservation) + .await? + .into() } GetRequestMethod::CalendarEventNotification(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - access_token.assert_is_member(req.account_id)?; + access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_notification_get(*req, access_token) .await? @@ -355,9 +396,11 @@ impl RequestHandler for Server { } GetRequestMethod::ParticipantIdentity(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - access_token.assert_is_member(req.account_id)?; + access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; - self.participant_identity_get(*req).await?.into() + self.participant_identity_get(*req, access_token) + .await? + .into() } GetRequestMethod::ShareNotification(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -369,13 +412,9 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; - Box::pin(self.registry_get( - method_name.obj.unwrap_registry(), - *req, - access_token, - )) - .await? - .into() + self.registry_get(method_name.obj.unwrap_registry(), *req, access_token) + .await? + .into() } }, RequestMethod::Query(req) => match req { @@ -383,13 +422,13 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Email)?; - self.email_query(*req, access_token).await?.into() + self.email_query(*req, access_token, using).await?.into() } QueryRequestMethod::Mailbox(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Mailbox)?; - self.mailbox_query(*req, access_token).await?.into() + self.mailbox_query(*req, access_token, using).await?.into() } QueryRequestMethod::EmailSubmission(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -401,11 +440,14 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; - self.sieve_script_query(*req).await?.into() - } - QueryRequestMethod::Principal(req) => { - self.principal_query(*req, access_token).await?.into() + self.sieve_script_query(*req, access_token, using) + .await? + .into() } + QueryRequestMethod::Principal(req) => self + .principal_query(*req, access_token, using) + .await? + .into(), QueryRequestMethod::Quota(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; @@ -416,35 +458,43 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::AddressBook)?; - self.address_book_query(*req, access_token).await?.into() + self.address_book_query(*req, access_token, using) + .await? + .into() } QueryRequestMethod::ContactCard(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::ContactCard)?; - self.contact_card_query(*req, access_token).await?.into() + self.contact_card_query(*req, access_token, using) + .await? + .into() } QueryRequestMethod::FileNode(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::FileNode)?; - self.file_node_query(*req, access_token).await?.into() + self.file_node_query(*req, access_token, using) + .await? + .into() } QueryRequestMethod::Calendar(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Calendar)?; - self.calendar_query(*req, access_token).await?.into() + self.calendar_query(*req, access_token, using).await?.into() } QueryRequestMethod::CalendarEvent(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; - self.calendar_event_query(*req, access_token).await?.into() + self.calendar_event_query(*req, access_token, using) + .await? + .into() } QueryRequestMethod::CalendarEventNotification(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - access_token.assert_is_member(req.account_id)?; + access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_notification_query(*req, access_token) .await? @@ -460,13 +510,9 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; - Box::pin(self.registry_query( - method_name.obj.unwrap_registry(), - *req, - access_token, - )) - .await? - .into() + self.registry_query(method_name.obj.unwrap_registry(), *req, access_token) + .await? + .into() } }, RequestMethod::Set(req) => match req { @@ -474,13 +520,15 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Email)?; - self.email_set(*req, access_token, session).await?.into() + self.email_set(*req, access_token, session, using) + .await? + .into() } SetRequestMethod::Mailbox(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Mailbox)?; - self.mailbox_set(*req, access_token).await?.into() + self.mailbox_set(*req, access_token, using).await?.into() } SetRequestMethod::Identity(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -504,7 +552,7 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_is_member(req.account_id)?; - self.sieve_script_set(*req, access_token, session) + self.sieve_script_set(*req, access_token, session, using) .await? .into() } @@ -518,7 +566,7 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::AddressBook)?; - self.address_book_set(*req, access_token, session) + self.address_book_set(*req, access_token, session, using) .await? .into() } @@ -526,7 +574,7 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::ContactCard)?; - self.contact_card_set(*req, access_token, session) + self.contact_card_set(*req, access_token, session, using) .await? .into() } @@ -534,7 +582,7 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::FileNode)?; - self.file_node_set(*req, access_token, session) + self.file_node_set(*req, access_token, session, using) .await? .into() } @@ -548,19 +596,21 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Calendar)?; - self.calendar_set(*req, access_token, session).await?.into() + self.calendar_set(*req, access_token, session, using) + .await? + .into() } SetRequestMethod::CalendarEvent(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; - self.calendar_event_set(*req, access_token, session) + self.calendar_event_set(*req, access_token, session, using) .await? .into() } SetRequestMethod::CalendarEventNotification(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - access_token.assert_is_member(req.account_id)?; + access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_notification_set(*req, access_token, session) .await? @@ -568,9 +618,11 @@ impl RequestHandler for Server { } SetRequestMethod::ParticipantIdentity(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - access_token.assert_is_member(req.account_id)?; + access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; - self.participant_identity_set(*req).await?.into() + self.participant_identity_set(*req, access_token) + .await? + .into() } SetRequestMethod::Registry(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -589,62 +641,86 @@ impl RequestHandler for Server { RequestMethod::Changes(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - self.changes(*req, method_name.obj, access_token) + self.changes(*req, method_name.obj, access_token, using) .await? .into_method_response() } RequestMethod::Copy(req) => match req { CopyRequestMethod::Email(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; - resolve_account_id(&mut req.from_account_id, method_name.obj, access_token)?; + resolve_from_account_id( + &mut req.from_account_id, + method_name.obj, + access_token, + )?; access_token .assert_has_access(req.account_id, Collection::Email)? - .assert_has_access(req.from_account_id, Collection::Email)?; + .assert_has_source_access(req.from_account_id, Collection::Email)?; - self.email_copy(*req, access_token, next_call, session) + self.email_copy(*req, access_token, next_call, session, using) .await? .into() } CopyRequestMethod::Blob(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; + resolve_from_account_id( + &mut req.from_account_id, + method_name.obj, + access_token, + )?; access_token.assert_is_member(req.account_id)?; + if !access_token.has_account_access(req.from_account_id.document_id()) { + return Err(trc::JmapEvent::FromAccountNotFound.into_err()); + } self.blob_copy(*req, access_token).await?.into() } CopyRequestMethod::ContactCard(mut req) => { - resolve_account_id(&mut req.from_account_id, method_name.obj, access_token)?; + resolve_from_account_id( + &mut req.from_account_id, + method_name.obj, + access_token, + )?; resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token .assert_has_access(req.account_id, Collection::ContactCard)? - .assert_has_access(req.from_account_id, Collection::ContactCard)?; + .assert_has_source_access(req.from_account_id, Collection::ContactCard)?; - self.contact_card_copy(*req, access_token, next_call, session) + self.contact_card_copy(*req, access_token, next_call, session, using) .await? .into() } CopyRequestMethod::CalendarEvent(mut req) => { - resolve_account_id(&mut req.from_account_id, method_name.obj, access_token)?; + resolve_from_account_id( + &mut req.from_account_id, + method_name.obj, + access_token, + )?; resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token .assert_has_access(req.account_id, Collection::CalendarEvent)? - .assert_has_access(req.from_account_id, Collection::CalendarEvent)?; + .assert_has_source_access(req.from_account_id, Collection::CalendarEvent)?; - self.calendar_event_copy(*req, access_token, next_call, session) + self.calendar_event_copy(*req, access_token, next_call, session, using) .await? .into() } CopyRequestMethod::FileNode(mut req) => { - resolve_account_id(&mut req.from_account_id, method_name.obj, access_token)?; + resolve_from_account_id( + &mut req.from_account_id, + method_name.obj, + access_token, + )?; resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token .assert_has_access(req.account_id, Collection::FileNode)? - .assert_has_access(req.from_account_id, Collection::FileNode)?; + .assert_has_source_access(req.from_account_id, Collection::FileNode)?; - self.file_node_copy(*req, access_token, next_call, session) + self.file_node_copy(*req, access_token, next_call, session, using) .await? .into() } @@ -653,7 +729,9 @@ impl RequestHandler for Server { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Email)?; - self.email_import(*req, access_token, session).await?.into() + self.email_import(*req, access_token, session, using) + .await? + .into() } RequestMethod::Parse(req) => match req { ParseRequestMethod::Email(mut req) => { @@ -675,7 +753,9 @@ impl RequestHandler for Server { self.calendar_event_parse(*req, access_token).await?.into() } }, - RequestMethod::QueryChanges(req) => self.query_changes(req, access_token).await?.into(), + RequestMethod::QueryChanges(req) => { + self.query_changes(req, access_token, using).await?.into() + } RequestMethod::SearchSnippet(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; access_token.assert_has_access(req.account_id, Collection::Email)?; @@ -706,7 +786,7 @@ impl RequestHandler for Server { trc::event!( Jmap(JmapEvent::MethodCall), - Id = method_name.as_str(), + Command = method_name.as_str(), SpanId = session.session_id, AccountId = access_token.account_id(), Elapsed = op_start.elapsed(), @@ -716,6 +796,20 @@ impl RequestHandler for Server { } } +pub(crate) fn resolve_from_account_id( + account_id: &mut Id, + obj: MethodObject, + access_token: &AccessToken, +) -> trc::Result<()> { + resolve_account_id(account_id, obj, access_token).map_err(|err| { + if err.matches(trc::EventType::Jmap(JmapEvent::AccountNotFound)) { + trc::JmapEvent::FromAccountNotFound.into_err() + } else { + err + } + }) +} + pub(crate) fn resolve_account_id( account_id: &mut Id, obj: MethodObject, diff --git a/crates/jmap/src/api/session.rs b/crates/jmap/src/api/session.rs index 2c0fe96e..f65b30c3 100644 --- a/crates/jmap/src/api/session.rs +++ b/crates/jmap/src/api/session.rs @@ -4,9 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; +use common::{ + Server, + auth::{AccessToken, oauth::scope::OAuthScopes}, +}; use jmap_proto::request::capability::{ - Account, Capabilities, Capability, EmptyCapabilities, Session, + Account, BaseCapabilities, Capabilities, Capability, CapabilityIds, EmptyCapabilities, + PrincipalOwnerCapabilities, Session, }; use registry::schema::enums::Permission; use std::future::Future; @@ -22,6 +26,79 @@ pub trait SessionHandler: Sync + Send { ) -> impl Future> + Send; } +pub trait JmapAccount { + fn jmap_account(&self, access_token: &AccessToken, account_id: u32, name: String) -> Account; + + fn jmap_account_capabilities( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> impl Iterator; +} + +impl JmapAccount for Server { + fn jmap_account(&self, access_token: &AccessToken, account_id: u32, name: String) -> Account { + let mut account_capabilities = + VecMap::with_capacity(self.core.jmap.capabilities.account.len() + 1); + account_capabilities.extend(self.jmap_account_capabilities(access_token, account_id)); + account_capabilities.append( + Capability::PrincipalsOwner, + Capabilities::PrincipalsOwner(PrincipalOwnerCapabilities { + account_id_for_principal: Id::from(access_token.account_id()), + principal_id: Id::from(account_id), + }), + ); + Account { + name, + is_personal: account_id == access_token.account_id(), + is_read_only: access_token.is_read_only(account_id), + account_capabilities, + } + } + + fn jmap_account_capabilities( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> impl Iterator { + let base_capabilities = &self.core.jmap.capabilities; + let is_owner = access_token.is_member(account_id); + let current_user_principal_id = Some(Id::from(access_token.account_id())); + let supports_private = access_token.has_permission(Permission::JmapMetadataPrivate); + let capability_ids = access_token + .account_capabilities(base_capabilities) + .filter(|capability| { + is_owner + || !matches!( + capability, + Capability::Submission + | Capability::VacationResponse + | Capability::Sieve + | Capability::Quota + | Capability::EmailPush + ) + }) + .collect::(); + + Capability::all_capabilities() + .iter() + .copied() + .filter(move |capability| capability_ids.contains(*capability)) + .filter_map(move |capability| { + let capabilities = match base_capabilities.account.get(&capability) { + Some(Capabilities::Metadata(metadata)) => Capabilities::Metadata( + metadata.to_account_capabilities(capability_ids, supports_private)?, + ), + Some(capabilities) => { + capabilities.to_account_capabilities(current_user_principal_id, is_owner) + } + None => Capabilities::Empty(EmptyCapabilities::default()), + }; + Some((capability, capabilities)) + }) + } +} + impl SessionHandler for Server { async fn handle_session_resource( &self, @@ -30,7 +107,13 @@ impl SessionHandler for Server { ) -> trc::Result { let mut session = Session::new(base_url, &self.core.jmap.capabilities); session.set_state(access_token.state()); - let account_capabilities = &self.core.jmap.capabilities.account; + if access_token.oauth_scopes().is_some() { + for capability in Capability::all_capabilities() { + if !access_token.allows_oauth_scopes(capability_scopes(*capability)) { + session.capabilities.remove(capability); + } + } + } // Set primary account let account = self @@ -39,27 +122,24 @@ impl SessionHandler for Server { .caused_by(trc::location!())?; session.username = account.name().to_string(); let account_id = Id::from(access_token.account_id()); - let mut account = Account { - name: account.name().to_string(), - is_personal: true, - is_read_only: false, - account_capabilities: VecMap::with_capacity(account_capabilities.len()), - }; - for capability in access_token.account_capabilities() { - session.primary_accounts.append(capability, account_id); - account.account_capabilities.append( - capability, - account_capabilities - .get(&capability) - .map(|v| v.to_account_capabilities(account_id.into(), true)) - .unwrap_or_else(|| Capabilities::Empty(EmptyCapabilities::default())), - ); + let primary_account = self.jmap_account( + access_token, + access_token.account_id(), + account.name().to_string(), + ); + for capability in access_token.account_capabilities(&self.core.jmap.capabilities) { + if capability != Capability::Metadata + || primary_account + .account_capabilities + .contains_key(&Capability::Metadata) + { + session.primary_accounts.append(capability, account_id); + } } - session.accounts.append(account_id, account); + session.accounts.append(account_id, primary_account); // Add secondary accounts for &account_id in access_token.secondary_ids() { - let is_owner = access_token.is_member(account_id); let Some(account) = self .try_account(account_id) .await @@ -72,24 +152,18 @@ impl SessionHandler for Server { ); continue; }; + session.accounts.append( + Id::from(account_id), + self.jmap_account(access_token, account_id, account.name().to_string()), + ); + } - let account_id = Id::from(account_id); - let mut account = Account { - name: account.name().to_string(), - is_personal: false, - is_read_only: false, - account_capabilities: VecMap::with_capacity(account_capabilities.len()), - }; - for capability in access_token.account_capabilities() { - account.account_capabilities.append( - capability, - account_capabilities - .get(&capability) - .map(|v| v.to_account_capabilities(account_id.into(), is_owner)) - .unwrap_or_else(|| Capabilities::Empty(EmptyCapabilities::default())), - ); - } - session.accounts.append(account_id, account); + if !session.accounts.values().any(|account| { + account + .account_capabilities + .contains_key(&Capability::Metadata) + }) { + session.capabilities.remove(&Capability::Metadata); } Ok(session) @@ -97,11 +171,17 @@ impl SessionHandler for Server { } trait AccountCapabilities { - fn account_capabilities(&self) -> impl Iterator; + fn account_capabilities( + &self, + base_capabilities: &BaseCapabilities, + ) -> impl Iterator; } impl AccountCapabilities for AccessToken { - fn account_capabilities(&self) -> impl Iterator { + fn account_capabilities( + &self, + base_capabilities: &BaseCapabilities, + ) -> impl Iterator { Capability::all_capabilities() .iter() .filter(move |capability| { @@ -119,16 +199,66 @@ impl AccountCapabilities for AccessToken { Capability::Blob => Permission::JmapBlobGet, Capability::Quota => Permission::JmapQuotaGet, Capability::FileNode => Permission::JmapFileNodeGet, + Capability::Metadata + if base_capabilities + .account + .contains_key(&Capability::Metadata) => + { + Permission::JmapMetadataGet + } Capability::WebSocket | Capability::Principals | Capability::PrincipalsAvailability | Capability::Stalwart => return true, - Capability::Core | Capability::PrincipalsOwner | Capability::WebPushVapid => { + Capability::Core + | Capability::PrincipalsOwner + | Capability::WebPushVapid + | Capability::Metadata => { return false; } }; self.has_permission(permission) }) + .filter(|capability| self.allows_oauth_scopes(capability_scopes(**capability))) .copied() } } + +pub fn capability_scopes(capability: Capability) -> OAuthScopes { + match capability { + Capability::Core + | Capability::Blob + | Capability::Quota + | Capability::Principals + | Capability::PrincipalsOwner + | Capability::WebSocket + | Capability::WebPushVapid + | Capability::Metadata => OAuthScopes::INTEROPERABLE, + Capability::Mail + | Capability::Submission + | Capability::VacationResponse + | Capability::Sieve + | Capability::MailShare + | Capability::EmailPush => OAuthScopes::MAIL, + Capability::Contacts | Capability::ContactsParse => OAuthScopes::CONTACTS, + Capability::Calendars | Capability::CalendarsParse | Capability::PrincipalsAvailability => { + OAuthScopes::CALENDARS + } + Capability::FileNode => OAuthScopes::FILES, + Capability::Stalwart => OAuthScopes::NONE, + } +} + +pub fn allowed_capabilities(access_token: &AccessToken, using: CapabilityIds) -> CapabilityIds { + if access_token.oauth_scopes().is_none() { + return using; + } + Capability::all_capabilities() + .iter() + .copied() + .filter(|capability| { + using.contains(*capability) + && access_token.allows_oauth_scopes(capability_scopes(*capability)) + }) + .collect() +} diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 0d79d093..252775d2 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -4,16 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::download::BlobDownload; +use super::download::{BlobAccess, BlobDownload}; +use super::embedded::EmbeddedBlobs; use common::{Server, auth::AccessToken}; use jmap_proto::{ - error::set::{SetError, SetErrorType}, + error::set::SetError, method::copy::{CopyBlobRequest, CopyBlobResponse}, + object::blob::BlobProperty, request::MaybeInvalid, }; use registry::schema::enums::Permission; use std::future::Future; -use store::write::{BatchBuilder, BlobLink, BlobOp, now}; +use store::write::{BatchBuilder, BlobLink, BlobOp, blob::BlobLinkValue, now}; use trc::AddContext; use types::blob::{BlobClass, BlobId}; use utils::map::vec_map::VecMap; @@ -44,16 +46,13 @@ impl BlobCopy for Server { let blob_id = match blob_id { MaybeInvalid::Value(blob_id) => blob_id, invalid => { - response.not_copied.append( - invalid, - SetError::new(SetErrorType::BlobNotFound).with_description( - "blobId does not exist or not enough permissions to access it.", - ), - ); + response.not_copied.append(invalid, not_found()); continue; } }; - if self.has_access_blob(&blob_id, access_token).await? { + if let BlobAccess::Allowed { size } = + self.has_access_blob(&blob_id, access_token).await? + { // Enforce quota if !access_token.has_permission(Permission::UnlimitedUploads) && !self @@ -73,6 +72,26 @@ impl BlobCopy for Server { continue; } + if matches!(blob_id.class, BlobClass::Embedded { .. }) { + match self + .embedded_blob(&blob_id) + .await + .caused_by(trc::location!())? + { + Some(data) => { + let dest_blob_id = self + .put_jmap_blob(account_id, &data) + .await + .caused_by(trc::location!())?; + response.copied.append(blob_id, dest_blob_id); + } + None => { + response.not_copied.append(blob_id, not_found()); + } + } + continue; + } + let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; batch.with_account_id(account_id).set( @@ -80,10 +99,10 @@ impl BlobCopy for Server { hash: blob_id.hash.clone(), to: BlobLink::Temporary { until }, }, - vec![], + BlobLinkValue::new(size.unwrap_or_default()), ); self.store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; @@ -98,15 +117,15 @@ impl BlobCopy for Server { response.copied.append(blob_id, dest_blob_id); } else { - response.not_copied.append( - blob_id, - SetError::new(SetErrorType::BlobNotFound).with_description( - "blobId does not exist or not enough permissions to access it.", - ), - ); + response.not_copied.append(blob_id, not_found()); } } Ok(response) } } + +fn not_found() -> SetError { + SetError::not_found() + .with_description("blobId does not exist or not enough permissions to access it.") +} diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 741aa9aa..5b817deb 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -4,146 +4,327 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; +use super::embedded::EmbeddedBlobs; +use common::{ + GroupwareResourceRef, GroupwareResources, Server, auth::AccessToken, sharing::EffectiveAcl, + storage::blob::FetchedBlob, +}; use email::cache::MessageCacheFetch; use email::cache::email::MessageCacheAccess; -use email::message::metadata::MessageMetadata; +use email::cache::mailbox::MailboxCacheAccess; +use email::message::metadata::MetadataRow; use groupware::cache::GroupwareCache; +use hyper::body::Bytes; use registry::schema::enums::Permission; use std::future::Future; use store::ValueKey; -use store::write::{AlignedBytes, Archive}; use trc::AddContext; use types::acl::Acl; use types::blob::{BlobClass, BlobId}; use types::collection::{Collection, SyncCollection}; use types::field::EmailField; -use utils::chained_bytes::ChainedBytes; + +pub struct DownloadedBlob { + pub bytes: FetchedBlob, + pub extra_headers_len: usize, +} + +pub enum BlobContents { + Plain(FetchedBlob), + Message { + headers: Vec, + body_offset: usize, + blob: FetchedBlob, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlobAccess { + Denied, + Allowed { size: Option }, +} + +impl BlobAccess { + pub fn is_allowed(&self) -> bool { + matches!(self, BlobAccess::Allowed { .. }) + } +} + +impl BlobContents { + pub fn into_downloaded(self) -> trc::Result { + match self { + BlobContents::Plain(bytes) => Ok(DownloadedBlob { + bytes, + extra_headers_len: 0, + }), + BlobContents::Message { + headers, + body_offset, + mut blob, + } => { + blob.replace_prefix(body_offset, &headers) + .caused_by(trc::location!())?; + Ok(DownloadedBlob { + bytes: blob, + extra_headers_len: headers.len().saturating_sub(body_offset), + }) + } + } + } + + pub fn into_http_parts(self) -> trc::Result<(Option, Bytes)> { + match self { + BlobContents::Plain(bytes) => Ok((None, bytes.into())), + BlobContents::Message { + headers, + body_offset, + mut blob, + } => { + blob.try_reserve_extra(headers.capacity()) + .caused_by(trc::location!())?; + let body = Bytes::from(blob); + let body = body.slice(body_offset.min(body.len())..); + Ok((Some(headers.into()), body)) + } + } + } +} + +impl DownloadedBlob { + pub fn stored_len(&self, blob_id: &BlobId) -> Option { + (blob_id.section.is_none() && !matches!(blob_id.class, BlobClass::Embedded { .. })) + .then(|| self.bytes.len().saturating_sub(self.extra_headers_len)) + } +} pub trait BlobDownload: Sync + Send { fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, - ) -> impl Future>>> + Send; + ) -> impl Future>> + Send; + + fn blob_download_with_extra( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> impl Future>> + Send; + + fn blob_contents( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> impl Future>> + Send; fn has_access_blob( &self, blob_id: &BlobId, access_token: &AccessToken, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } impl BlobDownload for Server { - #[allow(clippy::blocks_in_conditions)] async fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, - ) -> trc::Result>> { - if self.has_access_blob(blob_id, access_token).await? { - if let Some(section) = &blob_id.section { - self.get_blob_section(&blob_id.hash, section) - .await - .caused_by(trc::location!()) - } else { - let blob = self - .blob_store() - .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) - .await - .caused_by(trc::location!()); - match (&blob_id.class, blob) { - ( - BlobClass::Linked { - account_id, - collection, - document_id, - }, - Ok(Some(data)), - ) if *collection == Collection::Email as u8 => { - let Some(archive) = self - .store() - .get_value::>(ValueKey::property( - *account_id, - Collection::Email, - *document_id, - EmailField::Metadata, - )) - .await - .caused_by(trc::location!())? - else { - return Ok(Some(data)); - }; - let metadata = archive - .to_unarchived::() - .caused_by(trc::location!())?; - let body_offset = metadata.inner.blob_body_offset.to_native(); - if metadata.inner.root_part().offset_body.to_native() != body_offset { - let raw_message = ChainedBytes::new( - metadata.inner.raw_headers.as_ref(), - ) - .with_last(data.get(body_offset as usize..).unwrap_or_default()); - Ok(Some(raw_message.to_bytes())) - } else { - Ok(Some(data)) - } - } - (_, blob) => blob, - } - } - } else { - Ok(None) + ) -> trc::Result> { + self.blob_download_with_extra(blob_id, access_token) + .await + .map(|blob| blob.map(|blob| blob.bytes)) + } + + async fn blob_download_with_extra( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> trc::Result> { + self.blob_contents(blob_id, access_token) + .await? + .map(BlobContents::into_downloaded) + .transpose() + } + + async fn blob_contents( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> trc::Result> { + let BlobAccess::Allowed { size } = self.has_access_blob(blob_id, access_token).await? + else { + return Ok(None); + }; + + if matches!(blob_id.class, BlobClass::Embedded { .. }) { + return match self + .embedded_blob(blob_id) + .await + .caused_by(trc::location!())? + { + Some(bytes) => self + .account_blob(bytes) + .map(|blob| Some(BlobContents::Plain(blob))), + None => Ok(None), + }; + } else if let Some(section) = &blob_id.section { + return self + .fetch_blob_section(&blob_id.hash, section, size) + .await + .caused_by(trc::location!()) + .map(|blob| blob.map(BlobContents::Plain)); } + + let Some(blob) = self + .fetch_blob(&blob_id.hash, 0..usize::MAX, size) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + if let BlobClass::Linked { + account_id, + collection, + document_id, + } = &blob_id.class + && *collection == Collection::Email as u8 + && let Some(row) = self + .store() + .get_value::(ValueKey::immutable( + *account_id, + Collection::Email, + *document_id, + EmailField::Metadata, + )) + .await + .caused_by(trc::location!())? + { + let metadata = row.unarchive().caused_by(trc::location!())?; + let body_offset = metadata.blob_body_offset(); + if metadata.headers_len() != body_offset { + let headers = row.raw_headers().caused_by(trc::location!())?; + return Ok(Some(BlobContents::Message { + headers: headers.into_owned(), + body_offset, + blob, + })); + } + } + + Ok(Some(BlobContents::Plain(blob))) } async fn has_access_blob( &self, blob_id: &BlobId, access_token: &AccessToken, - ) -> trc::Result { - Ok( - (blob_id.class.is_superuser() && access_token.has_permission(Permission::FetchAnyBlob)) - || (self - .store() - .blob_has_access(&blob_id.hash, &blob_id.class) - .await - .caused_by(trc::location!())? - && match &blob_id.class { - BlobClass::Linked { - account_id, - collection, - document_id, - } => { - if access_token.is_member(*account_id) { - true - } else { - match Collection::from(*collection) { - Collection::Email => self - .get_cached_messages(*account_id) - .await - .caused_by(trc::location!())? - .shared_messages(access_token, Acl::ReadItems) - .contains(*document_id), - collection @ (Collection::FileNode - | Collection::ContactCard - | Collection::CalendarEvent) => self - .fetch_dav_resources( - access_token.account_id(), - *account_id, - SyncCollection::from(collection), - ) - .await - .caused_by(trc::location!())? - .shared_items(access_token, [Acl::ReadItems], true) - .contains(*document_id), - _ => false, - } - } + ) -> trc::Result { + if let BlobClass::Embedded { + account_id, + collection, + document_id, + } = &blob_id.class + { + return self + .has_access_embedded_blob(access_token, *account_id, *collection, *document_id) + .await + .map(|allowed| { + if allowed { + BlobAccess::Allowed { size: None } + } else { + BlobAccess::Denied + } + }); + } + + if blob_id.class.is_superuser() && access_token.has_permission(Permission::FetchAnyBlob) { + return Ok(BlobAccess::Allowed { size: None }); + } + + let Some(size) = self + .store() + .blob_link_size(&blob_id.hash, &blob_id.class) + .await + .caused_by(trc::location!())? + else { + return Ok(BlobAccess::Denied); + }; + + let allowed = match &blob_id.class { + BlobClass::Linked { + account_id, + collection, + document_id, + } => { + if access_token.is_member(*account_id) { + true + } else { + match Collection::from(*collection) { + Collection::Email => { + let cache = self + .get_cached_messages(*account_id) + .await + .caused_by(trc::location!())?; + cache.email_by_id(document_id).is_some_and(|message| { + message.mailboxes().iter().any(|mailbox| { + cache.mailbox_by_id(&mailbox.mailbox_id).is_some_and( + |mailbox| { + mailbox + .acls + .as_slice() + .effective_acl(access_token) + .contains(Acl::ReadItems) + }, + ) + }) + }) } - BlobClass::Reserved { account_id, .. } => { - access_token.is_member(*account_id) + Collection::FileNode => self + .fetch_groupware_resources( + access_token.account_id(), + *account_id, + SyncCollection::FileNode, + ) + .await + .caused_by(trc::location!())? + .file_acl(access_token, *document_id) + .contains(Acl::ReadItems), + collection @ (Collection::ContactCard | Collection::CalendarEvent) => { + let resources = self + .fetch_groupware_resources( + access_token.account_id(), + *account_id, + SyncCollection::from(collection), + ) + .await + .caused_by(trc::location!())?; + resources.item_by_id(*document_id).is_some_and(|item| { + is_shared_item_readable(&resources, item, access_token) + }) } - }), - ) + _ => false, + } + } + } + BlobClass::Reserved { account_id, .. } => access_token.is_member(*account_id), + BlobClass::Embedded { .. } => false, + }; + + Ok(if allowed { + BlobAccess::Allowed { + size: Some(size).filter(|size| *size != 0), + } + } else { + BlobAccess::Denied + }) } } + +pub(crate) fn is_shared_item_readable( + resources: &GroupwareResources, + item: GroupwareResourceRef<'_>, + access_token: &AccessToken, +) -> bool { + item.child_names() + .iter() + .any(|name| resources.has_access_to_container(access_token, name.parent_id, Acl::ReadItems)) +} diff --git a/crates/jmap/src/blob/embedded.rs b/crates/jmap/src/blob/embedded.rs new file mode 100644 index 00000000..ff2e6f5a --- /dev/null +++ b/crates/jmap/src/blob/embedded.rs @@ -0,0 +1,604 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::download::{BlobDownload, is_shared_item_readable}; +use calcard::{ + common::{ + blob::{BlobIdGenerator, BlobIdOutcome}, + export::{ExportError, ImportError}, + }, + icalendar::{ + ICalendar, ICalendarComponentType, ICalendarEntry, ICalendarParameterName, + ICalendarProperty, ICalendarValue, + }, + jscalendar::{ + JSCalendar, JSCalendarProperty, JSCalendarValue, + export::ExportOptions as CalendarExportOptions, ext::JSCalendarPatch, + }, + jscontact::{JSContact, JSContactProperty, export::ExportOptions as ContactExportOptions}, + vcard::VCard, +}; +use common::{PROD_ID, Server, auth::AccessToken, network::memory::MemoryReservation}; +use encodify::base64::LENIENT; +use groupware::{ + cache::GroupwareCache, + calendar::{CalendarEventContent, privacy::EventPrivacy}, + contact::ContactCardContent, +}; +use jmap_proto::error::set::{InvalidProperty, SetError}; +use jmap_tools::{Key, Map, Property, Value}; +use std::{borrow::Cow, future::Future}; +use store::{ + ValueKey, + ahash::{AHashMap, AHashSet}, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{ + blob::{BlobClass, BlobId}, + blob_hash::BlobHash, + collection::{Collection, SyncCollection}, + field::{CalendarEventField, ContactField}, + id::Id, +}; + +#[derive(Debug, Clone, Copy)] +pub struct EmbeddedBlobIds { + account_id: u32, + collection: Collection, + document_id: u32, +} + +pub struct ResolvedBlobs { + blobs: AHashMap>, + _memory: Vec, +} + +pub enum EmbeddingError { + NotFound(Vec), + TooLarge { max_size: usize }, +} + +const MAX_EVENT_TIME_ZONES: usize = 64; + +type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; + +struct InstanceCopies<'a, 'x> { + event: &'a EventMap<'x>, + instances: usize, +} + +pub trait EmbeddedBlobs: Sync + Send { + fn embedded_blob( + &self, + blob_id: &BlobId, + ) -> impl Future>>> + Send; + + fn resolve_blobs<'x>( + &self, + access_token: &AccessToken, + blob_ids: impl Iterator + Send, + max_size: usize, + ) -> impl Future>> + Send; + + fn has_access_embedded_blob( + &self, + access_token: &AccessToken, + account_id: u32, + collection: u8, + document_id: u32, + ) -> impl Future> + Send; +} + +pub trait EmbeddedExport: Sync + Send { + fn export_icalendar( + &self, + access_token: &AccessToken, + js_calendar: JSCalendar<'_, Id, BlobId>, + ) -> impl Future>>>> + Send; + + fn export_vcard( + &self, + access_token: &AccessToken, + js_contact: JSContact<'_, Id, BlobId>, + ) -> impl Future>>>> + Send; +} + +impl EmbeddedBlobIds { + pub fn new(account_id: u32, collection: Collection, document_id: u32) -> Self { + EmbeddedBlobIds { + account_id, + collection, + document_id, + } + } + + fn blob_id(&self, data: &[u8]) -> BlobId { + BlobId::new( + BlobHash::generate(data), + BlobClass::Embedded { + account_id: self.account_id, + collection: self.collection.into(), + document_id: self.document_id, + }, + ) + } +} + +impl BlobIdGenerator for EmbeddedBlobIds { + fn blob_id(&mut self, data: Vec, _: Option<&str>) -> BlobIdOutcome { + BlobIdOutcome::Generated(EmbeddedBlobIds::blob_id(self, &data)) + } +} + +impl ResolvedBlobs { + pub fn take(&mut self, blob_id: &BlobId) -> Option> { + self.blobs.remove(blob_id) + } + + fn size<'x>(&self, blob_ids: impl Iterator) -> usize { + blob_ids + .filter_map(|blob_id| self.blobs.get(blob_id)) + .map(Vec::len) + .fold(0, usize::saturating_add) + } + + fn value_size(&self, value: &EventValue<'_>) -> usize { + match value { + Value::Element(JSCalendarValue::BlobId(blob_id)) => { + self.blobs.get(blob_id).map_or(0, Vec::len) + } + Value::Str(text) => data_uri_size(text), + Value::Array(values) => values + .iter() + .map(|value| self.value_size(value)) + .fold(0, usize::saturating_add), + Value::Object(object) => object + .values() + .map(|value| self.value_size(value)) + .fold(0, usize::saturating_add), + _ => 0, + } + } +} + +impl<'a, 'x> InstanceCopies<'a, 'x> { + fn of(js_calendar: &'a JSCalendar<'x, Id, BlobId>) -> Vec { + js_calendar + .0 + .as_object_and_get(&Key::Property(JSCalendarProperty::Entries)) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_object) + .filter_map(|event| { + let instances = event + .get(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .and_then(Value::as_object)? + .values() + .filter(|patch| patch.is_instance_patch()) + .count(); + (instances > 0).then_some(InstanceCopies { event, instances }) + }) + .collect() + } + + fn size(&self, resolved: &ResolvedBlobs) -> usize { + self.event + .iter() + .filter(|(key, _)| { + !matches!(key, Key::Property(JSCalendarProperty::RecurrenceOverrides)) + }) + .map(|(_, value)| resolved.value_size(value)) + .fold(0, usize::saturating_add) + .saturating_mul(self.instances) + } +} + +impl EmbeddedBlobs for Server { + async fn embedded_blob(&self, blob_id: &BlobId) -> trc::Result>> { + let BlobClass::Embedded { + account_id, + collection, + document_id, + } = blob_id.class + else { + return Ok(None); + }; + + match Collection::from(collection) { + Collection::CalendarEvent => { + let Some(archive) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + let content = archive + .unarchive::() + .caused_by(trc::location!())?; + + Ok(matching_binary( + content.data.event.blob_binaries(), + &blob_id.hash, + )) + } + Collection::ContactCard => { + let Some(archive) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::ContactCard, + document_id, + ContactField::Content, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + let content = archive + .unarchive::() + .caused_by(trc::location!())?; + + Ok(matching_binary(content.card.blob_binaries(), &blob_id.hash)) + } + _ => Ok(None), + } + } + + async fn resolve_blobs<'x>( + &self, + access_token: &AccessToken, + blob_ids: impl Iterator + Send, + max_size: usize, + ) -> trc::Result> { + let mut resolved = AHashMap::new(); + let mut resolved_memory = Vec::new(); + let mut resolved_size = 0usize; + let mut not_found = Vec::new(); + for blob_id in blob_ids { + if resolved.contains_key(blob_id) || not_found.contains(blob_id) { + continue; + } + match self.blob_download(blob_id, access_token).await? { + Some(data) if !data.is_empty() => { + resolved_size = resolved_size.saturating_add(data.len()); + if resolved_size > max_size { + return Ok(Err(EmbeddingError::TooLarge { max_size })); + } + let (data, memory) = data.into_parts(); + resolved.insert(blob_id.clone(), data); + resolved_memory.push(memory); + } + _ => not_found.push(blob_id.clone()), + } + } + + Ok(if not_found.is_empty() { + Ok(ResolvedBlobs { + blobs: resolved, + _memory: resolved_memory, + }) + } else { + Err(EmbeddingError::NotFound(not_found)) + }) + } + + async fn has_access_embedded_blob( + &self, + access_token: &AccessToken, + account_id: u32, + collection: u8, + document_id: u32, + ) -> trc::Result { + let collection @ (Collection::CalendarEvent | Collection::ContactCard) = + Collection::from(collection) + else { + return Ok(false); + }; + let resources = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::from(collection), + ) + .await + .caused_by(trc::location!())?; + let Some(resource) = resources.item_by_id(document_id) else { + return Ok(false); + }; + + Ok(if access_token.is_member(account_id) { + true + } else { + is_shared_item_readable(&resources, resource, access_token) + && resource + .event_flags() + .is_none_or(|flags| EventPrivacy::from_flags(flags).is_public()) + }) + } +} + +impl EmbeddedExport for Server { + async fn export_icalendar( + &self, + access_token: &AccessToken, + js_calendar: JSCalendar<'_, Id, BlobId>, + ) -> trc::Result>>> { + let max_instances = self.core.groupware.max_ical_instances; + let instance_copies = InstanceCopies::of(&js_calendar); + if instance_copies + .iter() + .map(|copies| copies.instances) + .sum::() + > max_instances + { + return Ok(Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::RecurrenceOverrides) + .with_description(format!( + "An event cannot have more than {max_instances} recurrence overrides." + )))); + } + + let max_size = self.core.groupware.max_ical_size; + let mut resolved = match self + .resolve_blobs(access_token, js_calendar.blob_ids(), max_size) + .await? + { + Ok(resolved) => resolved, + Err(err) => return Ok(Err(err.into())), + }; + if instance_copies + .iter() + .map(|copies| copies.size(&resolved)) + .fold(resolved.size(js_calendar.blob_ids()), usize::saturating_add) + > max_size + { + return Ok(Err(EmbeddingError::TooLarge { max_size }.into())); + } + + let event_prod_id = js_calendar + .0 + .as_object_and_get(&Key::Property(JSCalendarProperty::Entries)) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|event| event.as_object_and_get(&Key::Property(JSCalendarProperty::ProdId))) + .find_map(Value::as_str) + .map(Cow::into_owned); + let mut ical = match js_calendar.into_icalendar_with_report( + CalendarExportOptions::new() + .max_expansions(max_instances) + .max_embedded_size(max_size) + .with_blob_resolver(|blob_id: &BlobId| resolved.take(blob_id)), + ) { + Ok((ical, rejected_patches)) => match rejected_patches.into_iter().next() { + Some(rejected) => { + return Ok(Err(export_error( + ExportError::InvalidPatch { + recurrence_id: rejected.recurrence_id, + pointer: rejected.pointer, + }, + "calendar event", + "iCalendar", + ))); + } + None => ical, + }, + Err(err) => return Ok(Err(export_error(err, "calendar event", "iCalendar"))), + }; + let mut tz_ids = AHashSet::with_capacity(4); + if ical + .components + .iter() + .filter(|component| { + !matches!( + component.component_type, + ICalendarComponentType::VTimezone + | ICalendarComponentType::Standard + | ICalendarComponentType::Daylight + ) + }) + .flat_map(|component| &component.entries) + .flat_map(|entry| &entry.params) + .filter(|param| param.name == ICalendarParameterName::Tzid) + .filter_map(|param| param.value.as_text()) + .any(|tz_id| tz_ids.insert(tz_id) && tz_ids.len() > MAX_EVENT_TIME_ZONES) + { + return Ok(Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::TimeZone) + .with_description(format!( + "An event cannot reference more than {MAX_EVENT_TIME_ZONES} time zones." + )))); + } + if let Some(root) = ical + .components + .first_mut() + .filter(|root| root.component_type == ICalendarComponentType::VCalendar) + { + match root.property_mut(&ICalendarProperty::Prodid) { + Some(entry) => { + if let Some(prod_id) = event_prod_id { + entry.values = [ICalendarValue::Text(prod_id)].into(); + } + } + None => root.entries.push( + ICalendarEntry::new(ICalendarProperty::Prodid) + .with_value(event_prod_id.unwrap_or_else(|| PROD_ID.to_string())), + ), + } + } + ical.add_missing_timezones(); + + Ok(assert_embedded_size( + ical.embedded_size(), + ical, + self.core.groupware.max_attachments_size, + "attachments", + )) + } + + async fn export_vcard( + &self, + access_token: &AccessToken, + js_contact: JSContact<'_, Id, BlobId>, + ) -> trc::Result>>> { + let max_size = self.core.groupware.max_vcard_size; + let mut resolved = match self + .resolve_blobs(access_token, js_contact.blob_ids(), max_size) + .await? + { + Ok(resolved) => resolved, + Err(err) => return Ok(Err(err.into())), + }; + if resolved.size(js_contact.blob_ids()) > max_size { + return Ok(Err(EmbeddingError::TooLarge { max_size }.into())); + } + + let card = match js_contact.into_vcard_with( + ContactExportOptions::new() + .max_embedded_size(max_size) + .with_blob_resolver(|blob_id: &BlobId| resolved.take(blob_id)), + ) { + Ok(card) => card, + Err(err) => return Ok(Err(export_error(err, "contact", "vCard"))), + }; + + Ok(assert_embedded_size( + card.embedded_size(), + card, + self.core.groupware.max_media_size, + "media", + )) + } +} + +impl From for SetError

{ + fn from(err: EmbeddingError) -> Self { + match err { + EmbeddingError::NotFound(not_found) => SetError::invalid_properties().with_description( + unresolved_description(not_found.iter().map(BlobId::to_string)), + ), + EmbeddingError::TooLarge { max_size } => { + SetError::too_large().with_description(format!( + "The embedded binaries exceed the maximum object size of {max_size} bytes." + )) + } + } + } +} + +fn export_error(err: ExportError, object: &str, format: &str) -> SetError

{ + match err { + ExportError::EmbeddedSizeExceeded { max } => SetError::too_large().with_description( + format!("The embedded binaries exceed the maximum object size of {max} bytes."), + ), + ExportError::UnresolvedBlob { blob_id } => { + SetError::invalid_properties().with_description(unresolved_description([blob_id])) + } + ExportError::InvalidPatch { + recurrence_id, + pointer, + } => SetError::invalid_properties() + .with_property(InvalidProperty::Path(vec![ + Key::Owned("recurrenceOverrides".to_string()), + Key::Owned(escape_pointer_token(recurrence_id)), + Key::Owned(escape_pointer_token(pointer)), + ])) + .with_description("The patch object could not be applied to the recurrence override."), + err => SetError::invalid_properties() + .with_description(format!("Failed to convert {object} to {format}: {err}.")), + } +} + +fn unresolved_description(blob_ids: impl IntoIterator) -> String { + let mut description = + String::from("The following blobIds do not exist or are not accessible: "); + for (pos, blob_id) in blob_ids.into_iter().enumerate() { + if pos > 0 { + description.push_str(", "); + } + description.push_str(&blob_id); + } + description.push('.'); + description +} + +fn matching_binary<'x>( + mut binaries: impl Iterator, + hash: &BlobHash, +) -> Option> { + binaries + .find(|data| &BlobHash::generate(data) == hash) + .map(<[u8]>::to_vec) +} + +pub fn import_error(err: ImportError) -> trc::Error { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + .details("Failed to generate a blob id for an embedded binary.") +} + +fn escape_pointer_token(token: String) -> String { + if !token.contains(['~', '/']) { + return token; + } + let mut escaped = String::with_capacity(token.len() + 8); + for ch in token.chars() { + match ch { + '~' => escaped.push_str("~0"), + '/' => escaped.push_str("~1"), + ch => escaped.push(ch), + } + } + escaped +} + +fn data_uri_size(text: &str) -> usize { + let Some((header, data)) = text.split_once(',').filter(|(header, _)| { + header + .as_bytes() + .get(..5) + .is_some_and(|scheme| scheme.eq_ignore_ascii_case(b"data:")) + }) else { + return 0; + }; + + if header + .rsplit_once(';') + .is_some_and(|(_, encoding)| encoding.eq_ignore_ascii_case("base64")) + { + LENIENT.decoded_len(data).unwrap_or_default() + } else { + data.len() + } +} + +fn assert_embedded_size( + size: usize, + object: T, + max_size: usize, + kind: &str, +) -> Result> { + if max_size == 0 || size <= max_size { + Ok(object) + } else { + Err(SetError::too_large().with_description(format!( + "The size of the embedded {kind} ({size} bytes) exceeds the maximum of {max_size} bytes." + ))) + } +} + diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index aaa60818..dcefc1f1 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -4,9 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::download::BlobDownload; +use super::{download::BlobDownload, embedded::EmbeddedBlobs}; use common::{Server, auth::AccessToken}; -use email::message::metadata::MessageData; +use email::message::messagedata::MessageData; +use encodify::base64::STANDARD; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, @@ -16,17 +17,13 @@ use jmap_proto::{ request::{IntoValid, MaybeInvalid}, }; use jmap_tools::{Map, Value}; -use mail_builder::encoders::Base64Encoder; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; use std::future::Future; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; +use store::ValueKey; use trc::AddContext; use types::{blob::BlobClass, collection::Collection, id::Id, type_state::DataType}; -use utils::map::vec_map::VecMap; +use utils::{map::vec_map::VecMap, text::validate_utf8}; pub trait BlobOperations: Sync + Send { fn blob_get( @@ -92,63 +89,38 @@ impl BlobOperations for Server { DigestProperty::Sha => { let mut hasher = Sha1::new(); hasher.update(bytes_range); - String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() + STANDARD.encode(hasher.finalize()) } DigestProperty::Sha256 => { let mut hasher = Sha256::new(); hasher.update(bytes_range); - String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() + STANDARD.encode(hasher.finalize()) } DigestProperty::Sha512 => { let mut hasher = Sha512::new(); hasher.update(bytes_range); - String::from_utf8( - Base64Encoder::new() - .encode(&hasher.finalize()[..]) - .unwrap_or_default(), - ) - .unwrap() + STANDARD.encode(hasher.finalize()) } } .into(), BlobProperty::Data(data) => match data { - DataProperty::AsText => match std::str::from_utf8(bytes_range) { - Ok(text) => text.to_string().into(), - Err(_) => { + DataProperty::AsText => match validate_utf8(bytes_range) { + Some(text) => text.to_string().into(), + None => { blob.insert_unchecked(BlobProperty::IsEncodingProblem, true); Value::Null } }, - DataProperty::AsBase64 => String::from_utf8( - Base64Encoder::new().encode(bytes_range).unwrap_or_default(), - ) - .unwrap() - .into(), - DataProperty::Default => match std::str::from_utf8(bytes_range) { - Ok(text) => { + DataProperty::AsBase64 => STANDARD.encode(bytes_range).into(), + DataProperty::Default => match validate_utf8(bytes_range) { + Some(text) => { property = BlobProperty::Data(DataProperty::AsText); text.to_string().into() } - Err(_) => { + None => { property = BlobProperty::Data(DataProperty::AsBase64); blob.insert_unchecked(BlobProperty::IsEncodingProblem, true); - String::from_utf8( - Base64Encoder::new() - .encode(bytes_range) - .unwrap_or_default(), - ) - .unwrap() - .into() + STANDARD.encode(bytes_range).into() } }, }, @@ -168,33 +140,15 @@ impl BlobOperations for Server { } async fn blob_lookup(&self, request: BlobLookupRequest) -> trc::Result { - let mut include_email = false; - let mut include_mailbox = false; - let mut include_thread = false; - - let type_names = request - .type_names - .into_iter() - .map(|tn| match tn { - MaybeInvalid::Value(value) => { - match &value { - DataType::Email => { - include_email = true; - } - DataType::Mailbox => { - include_mailbox = true; - } - DataType::Thread => { - include_thread = true; - } - _ => (), - } - - Ok(value) - } - MaybeInvalid::Invalid(_) => Err(trc::JmapEvent::UnknownDataType.into_err()), - }) - .collect::, _>>()?; + let mut type_names = Vec::with_capacity(request.type_names.len()); + for type_name in request.type_names { + let MaybeInvalid::Value(type_name) = type_name else { + return Err(trc::JmapEvent::UnknownDataType.into_err()); + }; + if !type_names.contains(&type_name) { + type_names.push(type_name); + } + } let req_account_id = request.account_id.document_id(); let mut response = BlobLookupResponse { account_id: request.account_id, @@ -203,62 +157,75 @@ impl BlobOperations for Server { }; for id in request.ids.into_valid() { - let mut matched_ids = VecMap::new(); + let mut matched_ids = type_names + .iter() + .map(|type_name| (*type_name, Vec::new())) + .collect::>(); match &id.class { BlobClass::Linked { account_id, collection, document_id, - } if *account_id == req_account_id => { - let collection = Collection::from(*collection); - if collection == Collection::Email { - if let Some(data_) = self - .store() - .get_value::>(ValueKey::archive( - req_account_id, - Collection::Email, - *document_id, - )) - .await? - { - let data = data_ - .unarchive::() - .caused_by(trc::location!())?; - if include_email { - matched_ids.append( - DataType::Email, - vec![Id::from_parts(u32::from(data.thread_id), *document_id)], - ); - } - if include_thread { - matched_ids.append( - DataType::Thread, - vec![Id::from(u32::from(data.thread_id))], - ); - } - if include_mailbox { - matched_ids.append( - DataType::Mailbox, - data.mailboxes - .iter() - .map(|m| { - debug_assert!(m.uid != 0); - Id::from(u32::from(m.mailbox_id)) - }) - .collect::>(), - ); + } if *account_id == req_account_id + && self + .store() + .blob_link_size(&id.hash, &id.class) + .await + .caused_by(trc::location!())? + .is_some() => + { + match Collection::from(*collection) { + Collection::Email => { + if let Some(data) = self + .store() + .get_value::(ValueKey::archive( + req_account_id, + Collection::Email, + *document_id, + )) + .await? + { + if let Some(ids) = matched_ids.get_mut(&DataType::Email) { + ids.push(Id::from_parts(data.thread_id, *document_id)); + } + if let Some(ids) = matched_ids.get_mut(&DataType::Thread) { + ids.push(Id::from(data.thread_id)); + } + if let Some(ids) = matched_ids.get_mut(&DataType::Mailbox) { + ids.extend(data.mailboxes.iter().map(|m| { + debug_assert!(m.uid != 0); + Id::from(m.mailbox_id) + })); + } } } - } else { - match DataType::try_from(collection) { - Ok(data_type) if type_names.contains(&data_type) => { - matched_ids.append(data_type, vec![Id::from(*document_id)]); + collection => { + if let Ok(data_type) = DataType::try_from(collection) + && let Some(ids) = matched_ids.get_mut(&data_type) + { + ids.push(Id::from(*document_id)); } - _ => (), } } } + BlobClass::Embedded { + account_id, + collection, + document_id, + } if *account_id == req_account_id + && self + .embedded_blob(&id) + .await + .caused_by(trc::location!())? + .is_some() => + { + if let Ok(data_type) = DataType::try_from(Collection::from(*collection)) + && let Some(ids) = matched_ids.get_mut(&data_type) + { + ids.push(Id::from(*document_id)); + } + } BlobClass::Reserved { account_id, .. } if *account_id == req_account_id => {} _ => { response.not_found.push(id); diff --git a/crates/jmap/src/blob/mod.rs b/crates/jmap/src/blob/mod.rs index d409834a..98bd6370 100644 --- a/crates/jmap/src/blob/mod.rs +++ b/crates/jmap/src/blob/mod.rs @@ -8,6 +8,7 @@ use types::{blob::BlobId, id::Id}; pub mod copy; pub mod download; +pub mod embedded; pub mod get; pub mod upload; diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 8411e6cf..bf979145 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -4,7 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{UploadResponse, download::BlobDownload}; +use super::{ + UploadResponse, + download::{BlobAccess, BlobDownload}, + embedded::EmbeddedBlobs, +}; use common::{Server, auth::AccessToken}; use jmap_proto::{ error::set::SetError, @@ -16,9 +20,9 @@ use jmap_proto::{ use registry::schema::enums::Permission; use std::future::Future; use trc::AddContext; -use types::id::Id; +use types::{blob::BlobClass, id::Id}; -#[cfg(feature = "test_mode")] +#[cfg(stalwart_tests)] pub static DISABLE_UPLOAD_QUOTA: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); @@ -57,6 +61,7 @@ impl BlobUpload for Server { 'outer: for (create_id, upload_object) in request.create { let mut data = Vec::new(); + let mut data_memory = Vec::new(); for data_source in upload_object.data { let bytes = match data_source { @@ -86,7 +91,9 @@ impl BlobUpload for Server { } }; - if !self.has_access_blob(&id, access_token).await? { + let BlobAccess::Allowed { size } = + self.has_access_blob(&id, access_token).await? + else { response.not_created.append( create_id, SetError::forbidden().with_description(format!( @@ -94,29 +101,43 @@ impl BlobUpload for Server { )), ); continue 'outer; - } + }; let offset = offset.unwrap_or(0); let length = length .map(|length| length.saturating_add(offset)) .unwrap_or(usize::MAX); - let bytes = if let Some(section) = &id.section { - self.get_blob_section(&id.hash, section) + let window = |bytes: Vec| { + if offset == 0 && length == usize::MAX { + bytes + } else { + bytes + .get(offset..std::cmp::min(length, bytes.len())) + .unwrap_or_default() + .to_vec() + } + }; + let bytes = if matches!(id.class, BlobClass::Embedded { .. }) { + self.embedded_blob(&id) + .await + .caused_by(trc::location!())? + .map(window) + } else if let Some(section) = &id.section { + self.fetch_blob_section(&id.hash, section, size) .await? - .map(|bytes| { - if offset == 0 && length == usize::MAX { - bytes - } else { - bytes - .get(offset..std::cmp::min(length, bytes.len())) - .unwrap_or_default() - .to_vec() - } + .map(|blob| { + let (bytes, memory) = blob.into_parts(); + data_memory.push(memory); + window(bytes) }) } else { - self.blob_store() - .get_blob(id.hash.as_slice(), offset..length) + self.fetch_blob(&id.hash, offset..length, size) .await? + .map(|blob| { + let (bytes, memory) = blob.into_parts(); + data_memory.push(memory); + bytes + }) }; if let Some(bytes) = bytes { bytes @@ -208,7 +229,7 @@ impl BlobUpload for Server { .is_upload_allowed(access_token) .caused_by(trc::location!())?; - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { // Used for concurrent upload tests if data == b"sleep" { @@ -229,12 +250,12 @@ impl BlobUpload for Server { .ctx(trc::Key::Total, self.core.jmap.upload_tmp_quota_amount) .ctx(trc::Key::Expires, status.expires_in)); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if !DISABLE_UPLOAD_QUOTA.load(std::sync::atomic::Ordering::Relaxed) { return err; } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] return err; } } diff --git a/crates/jmap/src/calendar/get.rs b/crates/jmap/src/calendar/get.rs index 76c36c96..0a10a3f6 100644 --- a/crates/jmap/src/calendar/get.rs +++ b/crates/jmap/src/calendar/get.rs @@ -4,8 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::acl::JmapRights, calendar::Availability, changes::state::JmapCacheState}; -use calcard::jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}; +use crate::{ + api::{ + acl::JmapRights, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + calendar::Availability, + changes::state::JmapCacheState, +}; +use calcard::{ + icalendar::ICalendarDuration, + jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}, +}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{ cache::GroupwareCache, @@ -15,14 +25,15 @@ use groupware::{ }, }; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::calendar::{self, CalendarProperty, CalendarValue, IncludeInAvailability}, + request::capability::CapabilityIds, }; use jmap_tools::{Key, Map, Value}; use store::{ ValueKey, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, ValueClass}, + write::{Archive, ArchiveBytes, ValueClass}, }; use trc::AddContext; use types::{ @@ -36,6 +47,7 @@ pub trait CalendarGet: Sync + Send { &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -44,28 +56,39 @@ impl CalendarGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - CalendarProperty::Id, - CalendarProperty::Name, - CalendarProperty::Description, - CalendarProperty::Color, - CalendarProperty::SortOrder, - CalendarProperty::IsSubscribed, - CalendarProperty::IsVisible, - CalendarProperty::IsDefault, - CalendarProperty::IncludeInAvailability, - CalendarProperty::DefaultAlertsWithTime, - CalendarProperty::DefaultAlertsWithoutTime, - CalendarProperty::TimeZone, - CalendarProperty::ShareWith, - CalendarProperty::MyRights, - ]); + let (properties, selection) = select_properties( + &mut request, + &[ + CalendarProperty::Id, + CalendarProperty::Name, + CalendarProperty::Description, + CalendarProperty::Color, + CalendarProperty::SortOrder, + CalendarProperty::IsSubscribed, + CalendarProperty::IsVisible, + CalendarProperty::IsDefault, + CalendarProperty::IncludeInAvailability, + CalendarProperty::DefaultAlertsWithTime, + CalendarProperty::DefaultAlertsWithoutTime, + CalendarProperty::TimeZone, + CalendarProperty::ShareWith, + CalendarProperty::MyRights, + ], + using, + )?; + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Calendar); + let metadata_get = metadata.get(selection); + let metadata_slots = metadata_get.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let personal_id = access_token.personal_id(account_id, Collection::Calendar); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, @@ -77,30 +100,51 @@ impl CalendarGet for Server { } else { cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) }; - let default_calendar_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), - }) - .await - .caused_by(trc::location!())? - .or_else(|| cache.document_ids(true).min()); + let default_calendar_id = if properties.contains(&CalendarProperty::IsDefault) { + self.store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), + }) + .await + .caused_by(trc::location!())? + .or_else(|| cache.document_ids(true).min()) + } else { + None + }; let ids = if let Some(ids) = ids { ids } else { - calendar_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + calendar_ids.iter().map(Into::into), + self.core.jmap.get_max_objects, + )? + }; + let mut metadata_values = match &metadata_get { + Some(get) => { + let mut documents = MetadataDocuments::default(); + for document_id in ids + .iter() + .map(|id| id.document_id()) + .filter(|document_id| calendar_ids.contains(*document_id)) + { + documents.insert( + document_id, + cache + .container_resource_by_id(document_id) + .map_or_else(Default::default, |resource| resource.metadata_kinds()), + ); + } + Some(metadata.load(self, account_id, get, &documents).await?) + } + None => None, }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(true).into(), + state: sampled.state(cache.get_state(true)).into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; @@ -114,7 +158,7 @@ impl CalendarGet for Server { } let _calendar = if let Some(calendar) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, @@ -129,7 +173,9 @@ impl CalendarGet for Server { let calendar = _calendar .unarchive::() .caused_by(trc::location!())?; - let mut result = Map::with_capacity(properties.len()); + let personal_preferences = calendar.personal_preferences(personal_id); + let personal_flags = calendar.personal_flags(personal_id, is_owner); + let mut result = Map::with_capacity(properties.len() + metadata_slots); for property in &properties { match property { CalendarProperty::Id => { @@ -138,7 +184,11 @@ impl CalendarGet for Server { CalendarProperty::Name => { result.insert_unchecked( CalendarProperty::Name, - calendar.preferences(personal_id).name.to_string(), + calendar + .preferences(personal_id) + .map_or_else(String::new, |preferences| { + preferences.name.to_string() + }), ); } CalendarProperty::Description => { @@ -146,15 +196,14 @@ impl CalendarGet for Server { CalendarProperty::Description, calendar .preferences(personal_id) - .description - .as_ref() - .map(|v| v.to_string()), + .and_then(|preferences| preferences.description.as_ref()) + .map(|description| description.to_string()), ); } CalendarProperty::SortOrder => { result.insert_unchecked( CalendarProperty::SortOrder, - calendar.preferences(personal_id).sort_order.to_native(), + personal_preferences.map_or(0, |p| p.sort_order.to_native()), ); } CalendarProperty::IsDefault => { @@ -166,9 +215,7 @@ impl CalendarGet for Server { CalendarProperty::IsSubscribed => { result.insert_unchecked( CalendarProperty::IsSubscribed, - Value::Bool( - calendar.preferences(personal_id).flags & CALENDAR_SUBSCRIBED != 0, - ), + Value::Bool(personal_flags & CALENDAR_SUBSCRIBED != 0), ); } CalendarProperty::Color => { @@ -176,31 +223,27 @@ impl CalendarGet for Server { CalendarProperty::Color, calendar .preferences(personal_id) - .color - .as_ref() - .map(|c| c.to_string()), + .and_then(|preferences| preferences.color.as_ref()) + .map(|color| color.to_string()), ); } CalendarProperty::IsVisible => { result.insert_unchecked( CalendarProperty::IsVisible, - Value::Bool( - calendar.preferences(personal_id).flags & CALENDAR_INVISIBLE == 0, - ), + Value::Bool(personal_flags & CALENDAR_INVISIBLE == 0), ); } CalendarProperty::IncludeInAvailability => { result.insert_unchecked( CalendarProperty::IncludeInAvailability, Value::Element(CalendarValue::IncludeInAvailability( - IncludeInAvailability::from_flags( - calendar.preferences(personal_id).flags.to_native(), - ) - .unwrap_or(if is_owner { - IncludeInAvailability::All - } else { - IncludeInAvailability::None - }), + IncludeInAvailability::from_flags(personal_flags).unwrap_or( + if is_owner { + IncludeInAvailability::All + } else { + IncludeInAvailability::None + }, + ), )), ); } @@ -229,8 +272,7 @@ impl CalendarGet for Server { CalendarProperty::TimeZone, calendar .preferences(personal_id) - .time_zone - .tz() + .and_then(|preferences| preferences.time_zone.tz()) .map(|tz| Value::Element(CalendarValue::Timezone(tz))) .unwrap_or(Value::Null), ); @@ -262,6 +304,9 @@ impl CalendarGet for Server { } } } + if let Some(values) = &mut metadata_values { + values.insert_into(document_id, &mut result); + } response.list.push(result.into()); } @@ -274,9 +319,24 @@ fn default_alarm_to_value( ) -> ( Key<'static, CalendarProperty>, Value<'static, CalendarProperty, CalendarValue>, +) { + default_alert_value( + alarm.id.as_str(), + alarm.offset.to_native(), + alarm.flags.to_native(), + ) +} + +pub(crate) fn default_alert_value( + id: &str, + offset: ICalendarDuration, + flags: u16, +) -> ( + Key<'static, CalendarProperty>, + Value<'static, CalendarProperty, CalendarValue>, ) { ( - Key::Owned(alarm.id.to_string()), + Key::Owned(id.to_string()), Value::Object(Map::from(vec![ ( Key::Property(CalendarProperty::Type), @@ -284,7 +344,7 @@ fn default_alarm_to_value( ), ( Key::Property(CalendarProperty::Action), - Value::Element(CalendarValue::Action(if alarm.flags & ALERT_EMAIL != 0 { + Value::Element(CalendarValue::Action(if flags & ALERT_EMAIL != 0 { JSCalendarAlertAction::Email } else { JSCalendarAlertAction::Display @@ -299,12 +359,12 @@ fn default_alarm_to_value( ), ( Key::Property(CalendarProperty::Offset), - Value::Element(CalendarValue::Duration(alarm.offset.to_native())), + Value::Element(CalendarValue::Duration(offset)), ), ( Key::Property(CalendarProperty::RelativeTo), Value::Element(CalendarValue::RelativeTo( - if alarm.flags & ALERT_RELATIVE_TO_END != 0 { + if flags & ALERT_RELATIVE_TO_END != 0 { JSCalendarRelativeTo::End } else { JSCalendarRelativeTo::Start diff --git a/crates/jmap/src/calendar/mod.rs b/crates/jmap/src/calendar/mod.rs index 10517a28..0701495a 100644 --- a/crates/jmap/src/calendar/mod.rs +++ b/crates/jmap/src/calendar/mod.rs @@ -4,9 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use groupware::calendar::{ - CALENDAR_AVAILABILITY_ALL, CALENDAR_AVAILABILITY_ATTENDING, CALENDAR_AVAILABILITY_NONE, -}; +use groupware::calendar::availability::AvailabilityPreference; use jmap_proto::object::calendar::IncludeInAvailability; pub mod get; @@ -18,14 +16,10 @@ pub(crate) trait Availability: Sized { impl Availability for IncludeInAvailability { fn from_flags(flags: u16) -> Option { - if flags & CALENDAR_AVAILABILITY_ALL != 0 { - Some(IncludeInAvailability::All) - } else if flags & CALENDAR_AVAILABILITY_ATTENDING != 0 { - Some(IncludeInAvailability::Attending) - } else if flags & CALENDAR_AVAILABILITY_NONE != 0 { - Some(IncludeInAvailability::None) - } else { - None - } + AvailabilityPreference::from_flags(flags).map(|preference| match preference { + AvailabilityPreference::All => IncludeInAvailability::All, + AvailabilityPreference::Attending => IncludeInAvailability::Attending, + AvailabilityPreference::None => IncludeInAvailability::None, + }) } } diff --git a/crates/jmap/src/calendar/set.rs b/crates/jmap/src/calendar/set.rs index f41f10b0..69ebf3b2 100644 --- a/crates/jmap/src/calendar/set.rs +++ b/crates/jmap/src/calendar/set.rs @@ -5,16 +5,34 @@ */ use crate::api::acl::{JmapAcl, JmapRights}; +use crate::api::metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, ObjectMetadata, + PreparedMetadata, is_empty_update, reject_uncommitted, +}; +use crate::api::pending_creates::PendingCreates; +use crate::calendar::{Availability, get::default_alert_value}; use crate::changes::state::JmapCacheState; use calcard::jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use common::{ + GroupwareResources, Server, + auth::AccessToken, + sharing::{EffectiveAcl, grants::ShareUpdate}, + storage::{batch::MAX_BATCH_DOCUMENTS, index::PresenceFlags, quota::ObjectQuotaUsage}, +}; use groupware::{ DestroyArchive, cache::GroupwareCache, calendar::{ ALERT_EMAIL, ALERT_RELATIVE_TO_END, ALERT_WITH_TIME, CALENDAR_AVAILABILITY_ALL, CALENDAR_AVAILABILITY_ATTENDING, CALENDAR_AVAILABILITY_NONE, CALENDAR_INVISIBLE, - CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, + CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, + EVENT_HAS_ALARMS, MAX_USER_ALERTS, Timezone, + alerts::{CalendarAlarmsReschedule, CalendarSettings, DefaultAlertsResolver}, + color::CssColor, + notification::CalendarNotificationReap, + privacy::EventPrivacy, + schedule::{AlarmTasks, EventAlarmScheduler}, + storage::{DirectChangeNotification, NotificationQuota}, }, }; use http_proto::HttpSessionData; @@ -22,21 +40,22 @@ use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::calendar::{self, CalendarProperty, CalendarValue, IncludeInAvailability}, - request::{MaybeInvalid, reference::MaybeIdReference}, + request::{MaybeInvalid, capability::CapabilityIds, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Map, Value}; use rand::{RngExt, distr::Alphanumeric}; +use registry::schema::enums::StorageQuota; use store::{ - SerializeInfallible, ValueKey, + ValueKey, ahash::AHashSet, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, + write::{Archive, ArchiveBytes, BatchBuilder, PendingId, ValueClass}, }; use trc::AddContext; use types::{ - acl::Acl, + acl::{Acl, AclGrant}, collection::{Collection, SyncCollection}, - field::PrincipalField, + field::{CalendarEventField, PrincipalField}, id::Id, }; @@ -46,6 +65,7 @@ pub trait CalendarSet: Sync + Send { request: SetRequest<'_, calendar::Calendar>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -55,24 +75,53 @@ impl CalendarSet for Server { mut request: SetRequest<'_, calendar::Calendar>, access_token: &AccessToken, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Calendar); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(true, &request.if_in_state)?); + .with_state(sampled.assert_state(cache.get_state(true), &request.if_in_state)?); + let mut metadata_writer = MetadataWriter::new(metadata, account_id); let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); let is_shared = access_token.is_shared(account_id); - let mut set_default = None; + let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let mut set_default: Option = None; + let mut created_slots = PendingCreates::new(); + let mut created_server_set = Vec::new(); + let mut request_alerts = RequestDefaultAlerts::default(); + + // Obtain quota + let quota = if request.has_creates() && !is_shared { + let account = self.account(account_id).await.caused_by(trc::location!())?; + self.object_quota_usage(&account, StorageQuota::MaxCalendars, || { + cache.resources.count(true) + }) + } else { + ObjectQuotaUsage::unlimited() + }; // Process creates let mut batch = BatchBuilder::new(); - 'create: for (id, object) in request.unwrap_create() { + 'create: for (id, mut object) in request.unwrap_create() { + if !quota.has_room(created_slots.len()) { + response.not_created.append( + id, + SetError::over_quota().with_description(concat!( + "There are too many calendars, ", + "please delete some before adding a new one." + )), + ); + continue 'create; + } + if is_shared { response.not_created.append( id, @@ -97,7 +146,44 @@ impl CalendarSet for Server { }; // Process changes - if let Err(err) = update_calendar(None, object, &mut calendar, access_token, account_id) + let metadata_patches = + match metadata_writer.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + let client_properties = object + .as_object() + .map(|object| { + object + .keys() + .filter_map(|key| key.as_property().cloned()) + .collect::>() + }) + .unwrap_or_default(); + calendar.preferences_mut(personal_id).flags |= CALENDAR_SUBSCRIBED; + let changes = + match update_calendar(None, object, &mut calendar, access_token, account_id) { + Ok(changes) => changes, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + calendar.sync_owner_preferences(account_id, personal_id); + if changes.default_alerts + && let Err(err) = self + .assert_unique_default_alerts( + &cache, + account_id, + personal_id, + None, + &calendar, + &request_alerts, + ) + .await? { response.not_created.append(id, err); continue 'create; @@ -105,25 +191,58 @@ impl CalendarSet for Server { // Validate ACLs if !calendar.acls.is_empty() { - if let Err(err) = self.acl_validate(&calendar.acls).await { + if let Err(err) = JmapRights::validate_shares::( + ShareUpdate { + collection: Collection::Calendar, + owner_id: account_id, + actor: None, + current: &[] as &[AclGrant], + max_shares: self.core.groupware.max_shares_per_item, + }, + &calendar.acls, + ) { + response.not_created.append(id, err); + continue 'create; + } + if let Err(err) = self + .acl_validate(access_token, account_id, &calendar.acls, &[] as &[AclGrant]) + .await + { response.not_created.append(id, err.into()); continue 'create; } - - self.refresh_acls(&calendar.acls, None) - .await - .caused_by(trc::location!())?; } + let prepared_metadata = match metadata_patches { + Some(patches) => match metadata_writer + .prepare_for(self, patches, MetadataAccess::FULL, MetadataTarget::Create) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }, + None => None, + }; + // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Calendar, 1) - .await - .caused_by(trc::location!())?; + let server_set = calendar.server_set_values(personal_id, &client_properties); + request_alerts.accept(None, &calendar, personal_id); + let document_id = batch.reserve_document_id(account_id, Collection::Calendar); + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + calendar.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, &mut batch)?; + } calendar .insert( - access_token.account_tenant_ids(), + access_token.changed_by(), account_id, document_id, &mut batch, @@ -134,14 +253,22 @@ impl CalendarSet for Server { &request.arguments.on_success_set_is_default && id_ref == &id { - set_default = Some(document_id); + set_default = Some(PendingId::Slot(document_id)); } - response.created(id, document_id); + created_server_set.push((id.clone(), document_id, server_set)); + created_slots.push(id, document_id); } // Process updates - 'update: for (id, object) in request.unwrap_update() { + metadata_writer + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .container_resource_by_id(document_id) + .map(|resource| resource.metadata_kinds()) + }) + .await?; + 'update: for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -154,12 +281,20 @@ impl CalendarSet for Server { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } + let metadata_patches = + match metadata_writer.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }; // Obtain calendar let document_id = id.document_id(); let calendar_ = if let Some(calendar_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, @@ -178,47 +313,158 @@ impl CalendarSet for Server { .deserialize::() .caused_by(trc::location!())?; + let acl = calendar.inner.acls.effective_acl(access_token); + if is_shared && !acl.contains_any([Acl::Read, Acl::ReadItems].into_iter()) { + response.not_updated.append(id, SetError::not_found()); + continue 'update; + } + let metadata_access = MetadataAccess { + may_write_shared: !is_shared || acl.contains(Acl::ModifyMetadata), + may_read: true, + }; + let mut metadata_patches = metadata_patches; + if object + .as_object() + .is_some_and(|object| is_empty_update::(object, id)) + && let Some(patches) = metadata_patches.take() + { + match metadata_writer + .write_metadata_only::( + self, + patches, + metadata_access, + &calendar, + document_id, + &mut batch, + ) + .await? + { + Ok(()) => response.updated.append(id, None), + Err(err) => response.not_updated.append(id, err), + } + continue 'update; + } + // Apply changes - let has_acl_changes = match update_calendar( + if !is_shared { + new_calendar.inherit_owner_preferences(account_id, personal_id); + } + let changes = match update_calendar( Some(id), object, &mut new_calendar, access_token, account_id, ) { - Ok(has_acl_changes_) => has_acl_changes_, + Ok(changes) => changes, Err(err) => { response.not_updated.append(id, err); continue 'update; } }; + if !is_shared { + new_calendar.sync_owner_preferences(account_id, personal_id); + } // Validate ACL - if is_shared { - let acl = calendar.inner.acls.effective_acl(access_token); - if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { - response.not_updated.append( - id, - SetError::forbidden() - .with_description("You are not allowed to modify this calendar."), - ); + if is_shared + && ((changes.shared_properties && !acl.contains(Acl::Modify)) + || (changes.acls && !acl.contains(Acl::Share))) + { + response.not_updated.append( + id, + SetError::forbidden() + .with_description("You are not allowed to modify this calendar."), + ); + continue 'update; + } + if changes.default_alerts + && let Err(err) = self + .assert_unique_default_alerts( + &cache, + account_id, + personal_id, + Some(document_id), + &new_calendar, + &request_alerts, + ) + .await? + { + response.not_updated.append(id, err); + continue 'update; + } + if changes.acls { + if let Err(err) = JmapRights::validate_shares::( + ShareUpdate { + collection: Collection::Calendar, + owner_id: account_id, + actor: is_shared.then_some(acl), + current: &calendar.inner.acls, + max_shares: self.core.groupware.max_shares_per_item, + }, + &new_calendar.acls, + ) { + response.not_updated.append(id, err); continue 'update; } - } - if has_acl_changes { - if let Err(err) = self.acl_validate(&new_calendar.acls).await { + if let Err(err) = self + .acl_validate( + access_token, + account_id, + &new_calendar.acls, + &calendar.inner.acls, + ) + .await + { response.not_updated.append(id, err.into()); continue 'update; } - self.refresh_archived_acls(&new_calendar.acls, calendar.inner.acls.as_slice()) - .await - .caused_by(trc::location!())?; } + let prepared_metadata = match metadata_patches { + Some(patches) => match metadata_writer + .prepare_for( + self, + patches, + metadata_access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }, + None => None, + }; + + self.reschedule_calendar_alarms( + &cache, + account_id, + document_id, + CalendarSettings::from(calendar.inner), + CalendarSettings::from(&new_calendar), + &mut batch, + ) + .await + .caused_by(trc::location!())?; + request_alerts.accept(Some(document_id), &new_calendar, personal_id); + // Update record + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + new_calendar.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, &mut batch)?; + } new_calendar .update( - access_token.account_tenant_ids(), + access_token.changed_by(), calendar, account_id, document_id, @@ -229,22 +475,29 @@ impl CalendarSet for Server { } // Process deletions - let mut reset_default_calendar = false; + let mut destroyed_calendars: Vec = Vec::new(); if !will_destroy.is_empty() { let mut destroy_children = AHashSet::new(); let mut destroy_parents = AHashSet::new(); - let default_calendar_id = self - .store() - .get_value::(ValueKey { - account_id, - collection: Collection::Principal.into(), - document_id: 0, - class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), - }) - .await - .caused_by(trc::location!())?; let on_destroy_remove_events = request.arguments.on_destroy_remove_events.unwrap_or(false); + let calendar_cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::Calendar, + &will_destroy + .iter() + .map(|id| id.document_id()) + .filter(|&document_id| { + cache + .container_resource_by_id(document_id) + .is_some_and(|calendar| !calendar.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; for id in will_destroy { let document_id = id.document_id(); @@ -255,7 +508,7 @@ impl CalendarSet for Server { let Some(calendar_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, @@ -277,7 +530,7 @@ impl CalendarSet for Server { .inner .acls .effective_acl(access_token) - .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) + .contains(Acl::Delete) { response.not_destroyed.append( id, @@ -288,14 +541,39 @@ impl CalendarSet for Server { } // Obtain children ids - let children_ids = cache.children_ids(document_id).collect::>(); - if !children_ids.is_empty() && !on_destroy_remove_events { + let mut has_visible_children = false; + let mut has_restricted_children = false; + for flags in cache + .children(document_id) + .map(|child| child.resource.event_flags().unwrap_or_default()) + { + match EventPrivacy::from_flags(flags) { + EventPrivacy::Public => has_visible_children = true, + EventPrivacy::Private => { + has_visible_children = true; + has_restricted_children = true; + } + EventPrivacy::Secret => { + has_visible_children |= !is_shared; + has_restricted_children = true; + } + } + } + if has_visible_children && !on_destroy_remove_events { response .not_destroyed .append(id, SetError::calendar_has_event()); continue; } - destroy_children.extend(children_ids.iter().copied()); + if is_shared && has_restricted_children { + response.not_destroyed.append( + id, + SetError::forbidden() + .with_description("You are not allowed to delete this calendar."), + ); + continue; + } + destroy_children.extend(cache.children_ids(document_id)); destroy_parents.insert(document_id); // Delete record @@ -303,71 +581,163 @@ impl CalendarSet for Server { .container_resource_path_by_id(document_id) .map(|resource| cache.format_resource(resource)); DestroyArchive(calendar) - .delete( - access_token.account_tenant_ids(), + .delete_with_cleanup( + self, + access_token.changed_by(), account_id, document_id, delete_path, + &calendar_cleanup, &mut batch, ) + .await .caused_by(trc::location!())?; - if default_calendar_id == Some(document_id) { - reset_default_calendar = true; - } + destroyed_calendars.push(document_id); response.destroyed.push(id); } + if !destroy_parents.is_empty() { + let destroyed_calendar_ids = destroy_parents.iter().copied().collect::>(); + self.reap_calendar_notifications( + access_token, + account_id, + &destroyed_calendar_ids, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + // Delete children if !destroy_children.is_empty() { let account_info = self .account_info(access_token.account_id()) .await .caused_by(trc::location!())?; - for document_id in destroy_children { - if let Some(event_) = self - .store() - .get_value::>(ValueKey::archive( + let mut resolver = DefaultAlertsResolver::with_resources(account_id, cache.clone()); + let mut quota = NotificationQuota::default(); + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::CalendarEvent, + &destroy_children + .iter() + .copied() + .filter(|&document_id| { + cache + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; + let notify_children = !access_token.is_member(account_id); + let alarm_ids = cache.event_ids_with_flags(EVENT_HAS_ALARMS); + let mut destroy_children = destroy_children.into_iter().collect::>(); + destroy_children.sort_unstable(); + destroy_children.dedup(); + let mut content_ids = Vec::new(); + for chunk in destroy_children.chunks(MAX_BATCH_DOCUMENTS) { + content_ids.clear(); + content_ids.extend( + chunk + .iter() + .copied() + .filter(|document_id| alarm_ids.contains(*document_id)), + ); + let mut archives = self + .archive_chunk( account_id, Collection::CalendarEvent, - document_id, - )) - .await? - { - let event = event_ - .to_unarchived::() - .caused_by(trc::location!())?; - - if event - .inner - .names - .iter() - .all(|n| destroy_parents.contains(&n.parent_id.to_native())) - { - // Event only belongs to calendars being deleted, delete it - DestroyArchive(event).delete_all( - &account_info, - account_id, - document_id, - false, - &mut batch, - )?; - } else { - // Unlink calendar id from event - let mut new_event = event - .deserialize::() + chunk, + Some((CalendarEventField::Content.field(), &content_ids)), + ) + .await + .caused_by(trc::location!())?; + let mut tasks = + AlarmTasks::for_chunk(account_id, chunk, &alarm_ids, &archives)?; + for &document_id in chunk { + if let Some(event_) = archives.take_archive(document_id) { + let event = event_ + .to_unarchived::() .caused_by(trc::location!())?; - new_event + + for calendar_id in event + .inner + .calendar_ids() + .filter(|_| notify_children) + .filter(|calendar_id| destroy_parents.contains(calendar_id)) + { + self.notify_calendar_removal( + access_token, + account_id, + document_id, + calendar_id, + event.inner.flags.to_native(), + &mut quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + + if event + .inner .names - .retain(|n| !destroy_parents.contains(&n.parent_id)); - new_event.update( - access_token.account_tenant_ids(), - event, - account_id, - document_id, - &mut batch, - )?; + .iter() + .all(|n| destroy_parents.contains(&n.parent_id.to_native())) + { + // Event only belongs to calendars being deleted, delete it + DestroyArchive(event) + .remove_all( + self, + &account_info, + account_id, + document_id, + archives.take_value(document_id), + &mut tasks, + false, + &cleanup, + &mut batch, + ) + .await?; + } else { + // Unlink calendar id from event + let mut new_event = event + .deserialize::() + .caused_by(trc::location!())?; + new_event + .names + .retain(|n| !destroy_parents.contains(&n.parent_id)); + self.reschedule_event_alarms( + account_id, + document_id, + event.inner, + archives.take_value(document_id), + &new_event + .names + .iter() + .map(|name| name.parent_id) + .collect::>(), + &mut tasks, + &mut resolver, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + new_event.update_meta( + access_token.changed_by(), + event, + account_id, + document_id, + None, + &mut batch, + )?; + } } } } @@ -376,55 +746,258 @@ impl CalendarSet for Server { // Set default calendar if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_set_is_default { - set_default = Some(id.document_id()); + let document_id = id.document_id(); + let is_visible = if is_shared { + cache + .shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) + .contains(document_id) + } else { + cache.has_container_id(&document_id) + }; + if is_visible && !destroyed_calendars.contains(&document_id) { + set_default = Some(PendingId::Assigned(document_id)); + } } - if let Some(default_calendar_id) = set_default { + let mut changed_default = None; + if let Some(default_calendar_id) = set_default.filter(|_| !is_shared) { if response.not_created.is_empty() && response.not_updated.is_empty() && response.not_destroyed.is_empty() { - batch - .with_account_id(account_id) - .with_collection(Collection::Principal) - .with_document(0) - .set( - PrincipalField::DefaultCalendarId, - default_calendar_id.serialize(), - ); + let previous_default_id = self + .store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), + }) + .await + .caused_by(trc::location!())? + .or_else(|| cache.document_ids(true).min()); + if !matches!(default_calendar_id, PendingId::Assigned(id) if Some(id) == previous_default_id) + { + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .set(PrincipalField::DefaultCalendarId, default_calendar_id); + changed_default = Some((default_calendar_id, previous_default_id)); + } } - } else if reset_default_calendar { + } else if !destroyed_calendars.is_empty() { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::DefaultCalendarId); + .with_document(0); + for &document_id in &destroyed_calendars { + batch.clear_if_equals(PrincipalField::DefaultCalendarId, document_id); + } } // Write changes - if !batch.is_empty() - && let Ok(change_id) = self - .commit_batch(batch) - .await - .caused_by(trc::location!())? - .last_change_id(account_id) - { - self.notify_task_queue(); - response.new_state = State::Exact(change_id).into(); + if !batch.is_empty() { + let assigned_ids = match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut response, + created_slots.into_create_ids(), + "Another process modified this calendar, please try again.", + ); + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; + + created_slots.resolve(&mut response, &assigned_ids); + let default_calendar_id = if created_server_set.is_empty() { + None + } else { + self.store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), + }) + .await + .caused_by(trc::location!())? + .or_else(|| { + cache + .document_ids(true) + .filter(|id| !destroyed_calendars.contains(id)) + .chain( + created_server_set + .iter() + .map(|(_, slot, _)| assigned_ids.slot(*slot)), + ) + .min() + }) + }; + for (create_id, slot, mut server_set) in created_server_set { + server_set.push(( + CalendarProperty::IsDefault, + Value::Bool(default_calendar_id == Some(assigned_ids.slot(slot))), + )); + response.add_created_properties(&create_id, server_set); + } + + if let Some((default_calendar_id, previous_default_id)) = changed_default { + let default_calendar_id = match default_calendar_id { + PendingId::Assigned(id) => id, + PendingId::Slot(slot) => assigned_ids.slot(slot), + }; + response.add_server_set_property( + Id::from(default_calendar_id), + CalendarProperty::IsDefault, + true, + ); + if let Some(previous_default_id) = previous_default_id + .filter(|id| *id != default_calendar_id && !destroyed_calendars.contains(id)) + { + response.add_server_set_property( + Id::from(previous_default_id), + CalendarProperty::IsDefault, + false, + ); + } + } + + if let Some(change_id) = assigned_ids.change_id(account_id, SyncCollection::Calendar) { + response.new_state = State::Exact(change_id).into(); + } } Ok(response) } } +#[derive(Debug, Default)] +struct CalendarChanges { + acls: bool, + shared_properties: bool, + default_alerts: bool, +} + +#[derive(Debug, Default)] +struct RequestDefaultAlerts { + updated_calendars: AHashSet, + alert_ids: AHashSet, +} + +impl RequestDefaultAlerts { + fn accept(&mut self, document_id: Option, calendar: &Calendar, personal_id: u32) { + if let Some(document_id) = document_id { + self.updated_calendars.insert(document_id); + } + self.alert_ids.extend( + calendar + .preferences + .iter() + .filter(|preferences| preferences.account_id == personal_id) + .flat_map(|preferences| preferences.default_alerts.iter()) + .map(|alert| alert.id.clone()), + ); + } +} + +trait DefaultAlertValidator { + fn assert_unique_default_alerts( + &self, + cache: &GroupwareResources, + account_id: u32, + personal_id: u32, + document_id: Option, + calendar: &Calendar, + request_alerts: &RequestDefaultAlerts, + ) -> impl Future>>> + Send; +} + +impl DefaultAlertValidator for Server { + async fn assert_unique_default_alerts( + &self, + cache: &GroupwareResources, + account_id: u32, + personal_id: u32, + document_id: Option, + calendar: &Calendar, + request_alerts: &RequestDefaultAlerts, + ) -> trc::Result>> { + let alerts = &calendar.preferences(personal_id).default_alerts; + let mut alert_ids = AHashSet::with_capacity(alerts.len()); + if !alerts.iter().all(|alert| { + alert_ids.insert(alert.id.as_str()) && !request_alerts.alert_ids.contains(&alert.id) + }) { + return Ok(Err(duplicate_default_alert_id())); + } + if alert_ids.is_empty() { + return Ok(Ok(())); + } + + for other_id in cache.document_ids(true).filter(|other_id| { + Some(*other_id) != document_id + && !request_alerts.updated_calendars.contains(other_id) + && cache + .container_resource_by_id(*other_id) + .is_some_and(|calendar| { + calendar + .personal_calendar_preferences(personal_id) + .is_some() + }) + }) { + let Some(other) = self + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + other_id, + )) + .await + .caused_by(trc::location!())? + else { + continue; + }; + if other + .unarchive::() + .caused_by(trc::location!())? + .personal_preferences(personal_id) + .is_some_and(|preferences| { + preferences + .default_alerts + .iter() + .any(|alert| alert_ids.contains(alert.id.as_str())) + }) + { + return Ok(Err(duplicate_default_alert_id())); + } + } + + Ok(Ok(())) + } +} + +fn duplicate_default_alert_id() -> SetError { + SetError::invalid_properties() + .with_properties([ + CalendarProperty::DefaultAlertsWithTime, + CalendarProperty::DefaultAlertsWithoutTime, + ]) + .with_description("Default alert ids must be unique across all calendars in the account.") +} + fn update_calendar( expected_id: Option, updates: Value<'_, CalendarProperty, CalendarValue>, calendar: &mut Calendar, access_token: &AccessToken, account_id: u32, -) -> Result> { +) -> Result> { let personal_id = access_token.personal_id(account_id, Collection::Calendar); - let mut has_acl_changes = false; + if access_token.is_member(account_id) { + calendar.subscribe_member(personal_id); + } + let mut changes = CalendarChanges::default(); for (property, value) in updates.into_expanded_object() { let Key::Property(property) = property else { @@ -433,6 +1006,24 @@ fn update_calendar( .with_description("Invalid property.")); }; + if !matches!( + property, + CalendarProperty::Id + | CalendarProperty::Name + | CalendarProperty::Color + | CalendarProperty::SortOrder + | CalendarProperty::IsSubscribed + | CalendarProperty::IsVisible + | CalendarProperty::TimeZone + | CalendarProperty::IncludeInAvailability + | CalendarProperty::DefaultAlertsWithTime + | CalendarProperty::DefaultAlertsWithoutTime + | CalendarProperty::ShareWith + | CalendarProperty::Pointer(_) + ) { + changes.shared_properties = true; + } + match (property, value) { (CalendarProperty::Name, Value::Str(value)) if (1..=255).contains(&value.len()) => { calendar.preferences_mut(personal_id).name = value.into_owned(); @@ -443,7 +1034,9 @@ fn update_calendar( (CalendarProperty::Description, Value::Null) => { calendar.preferences_mut(personal_id).description = None; } - (CalendarProperty::Color, Value::Str(value)) if value.len() < 16 => { + (CalendarProperty::Color, Value::Str(value)) + if CssColor::parse(&value).is_some_and(|color| color.is_standard()) => + { calendar.preferences_mut(personal_id).color = value.into_owned().into(); } (CalendarProperty::Color, Value::Null) => { @@ -455,7 +1048,9 @@ fn update_calendar( (CalendarProperty::TimeZone, Value::Null) => { calendar.preferences_mut(personal_id).time_zone = Timezone::Default; } - (CalendarProperty::SortOrder, Value::Number(value)) => { + (CalendarProperty::SortOrder, Value::Number(value)) + if value.is_u64() && value.cast_to_u64() < 1 << 31 => + { calendar.preferences_mut(personal_id).sort_order = value.cast_to_u64() as u32; } (CalendarProperty::IsSubscribed, Value::Bool(subscribe)) => { @@ -496,26 +1091,29 @@ fn update_calendar( ( property @ (CalendarProperty::DefaultAlertsWithTime | CalendarProperty::DefaultAlertsWithoutTime), - Value::Object(value), + value @ (Value::Object(_) | Value::Null), ) => { let with_time = matches!(property, CalendarProperty::DefaultAlertsWithTime); let alerts = &mut calendar.preferences_mut(personal_id).default_alerts; + changes.default_alerts = true; alerts.retain(|alert| (alert.flags & ALERT_WITH_TIME != 0) != with_time); - for (key, value) in value.into_vec() { - if let Value::Object(value) = value { - alerts.push(value_to_default_alert( - key.to_string().into_owned(), - value, - with_time, - )?); + if let Value::Object(value) = value { + for (key, value) in value.into_vec() { + if let Value::Object(value) = value { + alerts.push(value_to_default_alert( + key.to_string().into_owned(), + value, + with_time, + )?); + } } } } (CalendarProperty::ShareWith, value) => { calendar.acls = JmapRights::acl_set::(value)?; - has_acl_changes = true; + changes.acls = true; } (CalendarProperty::Pointer(pointer), value) => { let mut ptr_iter = pointer.iter(); @@ -527,7 +1125,7 @@ fn update_calendar( ptr_iter, value, )?; - has_acl_changes = true; + changes.acls = true; } Some(JsonPointerItem::Key(Key::Property( property @ (CalendarProperty::DefaultAlertsWithTime @@ -545,6 +1143,7 @@ fn update_calendar( let with_time = matches!(property, CalendarProperty::DefaultAlertsWithTime); let alerts = &mut calendar.preferences_mut(personal_id).default_alerts; + changes.default_alerts = true; alerts.retain(|alert| { (alert.flags & ALERT_WITH_TIME != 0) != with_time || alert.id != id }); @@ -554,13 +1153,13 @@ fn update_calendar( } } _ => { - return Err(SetError::invalid_properties() + return Err(SetError::invalid_patch() .with_property(CalendarProperty::Pointer(pointer)) .with_description("Field could not be patched.")); } }, _ => { - return Err(SetError::invalid_properties() + return Err(SetError::invalid_patch() .with_property(CalendarProperty::Pointer(pointer)) .with_description("Field could not be patched.")); } @@ -582,13 +1181,37 @@ fn update_calendar( } // Validate name - if calendar.preferences(personal_id).name.is_empty() { + let preferences = calendar.preferences(personal_id); + if preferences.name.is_empty() { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Name) .with_description("Missing name.")); } - Ok(has_acl_changes) + if changes.default_alerts { + let with_time = preferences + .default_alerts + .iter() + .filter(|alert| alert.flags & ALERT_WITH_TIME != 0) + .count(); + for (property, count) in [ + (CalendarProperty::DefaultAlertsWithTime, with_time), + ( + CalendarProperty::DefaultAlertsWithoutTime, + preferences.default_alerts.len() - with_time, + ), + ] { + if count > MAX_USER_ALERTS { + return Err(SetError::invalid_properties() + .with_property(property) + .with_description(format!( + "A calendar cannot have more than {MAX_USER_ALERTS} default alerts of each kind." + ))); + } + } + } + + Ok(changes) } fn value_to_default_alert( @@ -641,12 +1264,12 @@ fn value_to_default_alert( alert.offset = value; has_offset = true; } - (CalendarProperty::Offset, Value::Element(CalendarValue::Type(value))) + (CalendarProperty::Type, Value::Element(CalendarValue::Type(value))) if value != JSCalendarType::OffsetTrigger => { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Trigger) - .with_description("Invalid alert trigger type.")); + .with_description("Default alerts must use an OffsetTrigger.")); } _ => {} } @@ -668,3 +1291,72 @@ fn value_to_default_alert( .with_description("Missing alert offset.")) } } + +trait CalendarServerSet { + fn server_set_values( + &self, + personal_id: u32, + client_properties: &[CalendarProperty], + ) -> Vec<( + CalendarProperty, + Value<'static, CalendarProperty, CalendarValue>, + )>; +} + +impl CalendarServerSet for Calendar { + fn server_set_values( + &self, + personal_id: u32, + client_properties: &[CalendarProperty], + ) -> Vec<( + CalendarProperty, + Value<'static, CalendarProperty, CalendarValue>, + )> { + let preferences = self.preferences(personal_id); + let default_alerts = |with_time: bool| { + Value::Object(Map::from_iter( + preferences + .default_alerts + .iter() + .filter(|alert| (alert.flags & ALERT_WITH_TIME != 0) == with_time) + .map(|alert| default_alert_value(&alert.id, alert.offset.clone(), alert.flags)), + )) + }; + [ + ( + CalendarProperty::SortOrder, + Value::Number(preferences.sort_order.into()), + ), + ( + CalendarProperty::IsSubscribed, + Value::Bool(preferences.flags & CALENDAR_SUBSCRIBED != 0), + ), + ( + CalendarProperty::IsVisible, + Value::Bool(preferences.flags & CALENDAR_INVISIBLE == 0), + ), + ( + CalendarProperty::IncludeInAvailability, + Value::Element(CalendarValue::IncludeInAvailability( + IncludeInAvailability::from_flags(preferences.flags) + .unwrap_or(IncludeInAvailability::All), + )), + ), + ( + CalendarProperty::DefaultAlertsWithTime, + default_alerts(true), + ), + ( + CalendarProperty::DefaultAlertsWithoutTime, + default_alerts(false), + ), + ( + CalendarProperty::MyRights, + JmapRights::all_rights::(), + ), + ] + .into_iter() + .filter(|(property, _)| !client_properties.contains(property)) + .collect() + } +} diff --git a/crates/jmap/src/calendar_event/copy.rs b/crates/jmap/src/calendar_event/copy.rs index 0502c796..03db9dc4 100644 --- a/crates/jmap/src/calendar_event/copy.rs +++ b/crates/jmap/src/calendar_event/copy.rs @@ -5,36 +5,62 @@ */ use crate::{ - calendar_event::{CalendarSyntheticId, set::CalendarEventSet}, + api::metadata::{ + MetadataPatches, MetadataPreload, MetadataType, MetadataWriter, NewMetadata, ObjectMetadata, + }, + calendar_event::{ + CalendarSyntheticId, JSCalendarEntries, UidIndex, detach_iana_timezones, + privacy::assert_privacy_access, + server_set::{PendingBlobIds, ServerSetValues}, + set::{CalendarEventSet, CopiedEvent, EventSetContext, EventSource, too_many_events}, + }, changes::state::JmapCacheState, }; -use calcard::jscalendar::JSCalendarProperty; -use common::{Server, auth::AccessToken}; -use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; +use calcard::{icalendar::ICalendar, jscalendar::JSCalendarProperty}; +use common::{Server, auth::AccessToken, storage::dav::CachedUid}; +use groupware::{ + cache::GroupwareCache, + calendar::{ + CalendarEventContent, CalendarEventData, EVENT_HIDE_ATTENDEES, EVENT_SECRET, + EventPreferences, + alerts::DefaultAlertsResolver, + identity::{CalendarAddresses, EventOwnership, ParticipantIdentityAddresses}, + participants::VisibleParticipants, + privacy::{EventPrivacy, EventViewer}, + rights::EventAcl, + storage::NotificationQuota, + user::UserDataView, + }, +}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ - copy::{CopyRequest, CopyResponse}, + copy::{CopyRequest, CopyResponse, CopySourceId}, set::SetRequest, }, object::calendar_event, request::{ - Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, + Call, MaybeInvalid, RequestMethod, SetRequestMethod, + capability::CapabilityIds, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, types::state::State, }; +use jmap_tools::{Key, Value}; +use registry::schema::enums::StorageQuota; +use std::borrow::Cow; use store::{ ValueKey, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::CalendarEventField, + metadata::MetadataKinds, }; use utils::map::vec_map::VecMap; @@ -45,6 +71,7 @@ pub trait JmapCalendarEventCopy: Sync + Send { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -55,6 +82,7 @@ impl JmapCalendarEventCopy for Server { access_token: &AccessToken, next_call: &mut Option>>, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); @@ -64,15 +92,17 @@ impl JmapCalendarEventCopy for Server { .into_err() .details("From accountId is equal to fromAccountId")); } + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::CalendarEvent); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await .caused_by(trc::location!())?; - let old_state = cache.assert_state(false, &request.if_in_state)?; + let old_state = sampled.assert_state(cache.get_state(false), &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, @@ -80,113 +110,284 @@ impl JmapCalendarEventCopy for Server { old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), + destroyed: None, }; + let from_sampled = metadata.viewer_change_id(self, from_account_id).await?; let from_cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::Calendar, ) .await .caused_by(trc::location!())?; - let from_calendar_event_ids = if access_token.is_member(from_account_id) { - from_cache.document_ids(false).collect::() - } else { - from_cache.shared_items(access_token, [Acl::ReadItems], true) + from_sampled.assert_state(from_cache.get_state(false), &request.if_from_in_state)?; + let is_from_owner = access_token.is_member(from_account_id); + let from_personal_id = access_token.personal_id(from_account_id, Collection::Calendar); + let shared_from_event_ids = (!is_from_owner).then(|| { + let mut shared_ids = from_cache.shared_items(access_token, [Acl::ReadItems], true); + shared_ids -= from_cache.event_ids_with_flags(EVENT_SECRET); + shared_ids + }); + let is_readable_source = |document_id: u32| match &shared_from_event_ids { + Some(ids) => ids.contains(document_id), + None => from_cache.has_item_id(&document_id), }; - let can_add_calendars = if access_token.is_shared(account_id) { - cache - .shared_containers(access_token, [Acl::AddItems], true) - .into() - } else { - None - }; + let is_shared = access_token.is_shared(account_id); + let can_add_calendars = + is_shared.then(|| cache.shared_containers(access_token, [Acl::AddItems], true)); let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); + let mut created_slots: Vec<(String, Slot, ServerSetValues)> = Vec::new(); // Obtain account info let account_info = self - .account_info(access_token.account_id()) + .account_info(account_id) .await .caused_by(trc::location!())?; + let identities = if is_shared { + self.account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())? + } else { + CalendarAddresses::default() + }; + let mut viewer_identities = None; + let viewer = metadata.viewer(); + let mut metadata_writer = MetadataWriter::new(metadata, account_id); + let mut preload = MetadataPreload::default(); + let mut creates = Vec::with_capacity(request.create.len()); + for (create_id, mut create) in request.create { + let source_id = create.take_source_id(JSCalendarProperty::Id); + let patches = metadata_writer.extract(MetadataPatches::for_create(), &mut create); + if let (Ok(source_id), Ok(_)) = (&source_id, &patches) + && !source_id.is_synthetic() + && is_readable_source(source_id.document_id()) + { + let document_id = source_id.document_id(); + let kinds = from_cache + .item_by_id(document_id) + .map_or(MetadataKinds::NONE, |resource| resource.metadata_kinds()); + if viewer.is_some() || !kinds.is_empty() { + preload.insert_source(document_id, kinds); + } + } + creates.push((create_id, source_id, create, patches)); + } + metadata_writer + .preload(self, from_account_id, preload) + .await?; + let uid_index = UidIndex::new( + &cache, + creates.iter().filter_map(|(_, source_id, create, _)| { + match create.as_object_and_get(&Key::Property(JSCalendarProperty::Uid)) { + Some(Value::Str(uid)) => Some(uid.cached_uid()), + _ => source_id.as_ref().ok().and_then(|source_id| { + from_cache + .resources + .find(source_id.document_id(), false) + .and_then(|resource| resource.uid()) + .map(Cow::Borrowed) + }), + } + }), + ); + + // Obtain quota + let account = self.account(account_id).await.caused_by(trc::location!())?; + let quota = self.object_quota_usage(&account, StorageQuota::MaxCalendarEvents, || { + cache.resources.count(false) + }); // Prepare batch let mut batch = BatchBuilder::new(); + let mut context = EventSetContext { + cache: &cache, + access_token, + account_id, + account_info: &account_info, + send_scheduling_messages: false, + can_add_calendars: can_add_calendars.as_ref(), + identities: &identities, + uid_index, + will_destroy: Vec::new(), + default_alerts: DefaultAlertsResolver::with_resources(account_id, cache.clone()), + notification_quota: NotificationQuota::default(), + disk_quota: self + .disk_quota_budget(&account) + .await + .caused_by(trc::location!())?, + metadata: metadata_writer, + }; - 'create: for (id, create) in request.create.into_valid() { - let from_calendar_event_id = id.document_id(); - if !from_calendar_event_ids.contains(from_calendar_event_id) { + 'create: for (create_id, source_id, create, patches) in creates { + if !quota.has_room(created_slots.len()) { + response.not_created.append(create_id, too_many_events()); + continue; + } + let source_id = match source_id { + Ok(source_id) => source_id, + Err(err) => { + response.not_created.append(create_id, err); + continue; + } + }; + let patches = match patches { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(create_id, err); + continue; + } + }; + + let from_calendar_event_id = source_id.document_id(); + if !is_readable_source(from_calendar_event_id) { response.not_created.append( - id, + create_id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + source_id, response.from_account_id )), ); continue; } - if id.is_synthetic() { + if source_id.is_synthetic() { response.not_created.append( - id, + create_id, SetError::invalid_properties() .with_property(JSCalendarProperty::Id) .with_description(format!( - "Item {} is a synthetic id and cannot be copied.", - id + "Item {source_id} is a synthetic id and cannot be copied." )), ); continue; } - let Some(_calendar_event) = self + let source_resource = from_cache.resources.find(from_calendar_event_id, false); + let Some((flags, source_resource)) = source_resource + .as_ref() + .and_then(|resource| resource.event_flags().map(|flags| (flags, resource))) + else { + response.not_created.append( + create_id, + SetError::not_found().with_description(format!( + "Item {} not found in account {}.", + source_id, response.from_account_id + )), + ); + continue; + }; + if let Err(err) = assert_privacy_access( + EventPrivacy::from_flags(flags), + EventViewer::new(is_from_owner), + ) { + response.not_created.append(create_id, err); + continue; + } + let Some(content) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::property( from_account_id, Collection::CalendarEvent, from_calendar_event_id, + CalendarEventField::Content, )) .await? else { response.not_created.append( - id, + create_id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + source_id, response.from_account_id )), ); continue; }; - let calendar_event = _calendar_event - .deserialize::() + let content = content + .unarchive::() .caused_by(trc::location!())?; + let preferences = content.preferences(from_personal_id); + let use_default_alerts = + preferences.is_some_and(|preferences| preferences.use_default_alerts()); + let mut event = if is_from_owner { + rkyv_deserialize::<_, ICalendar>(&content.data.event) + } else { + preferences + .map(rkyv_deserialize::<_, EventPreferences>) + .transpose() + .and_then(|preferences| { + rkyv_deserialize::<_, CalendarEventData>(&content.data).map(|mut data| { + data.apply_user_data(preferences.as_ref(), UserDataView::Full); + data.event + }) + }) + } + .caused_by(trc::location!())?; + detach_iana_timezones(&mut event); + let hidden_acl = (!is_from_owner && flags & EVENT_HIDE_ATTENDEES != 0) + .then(|| { + EventAcl::for_calendars( + &from_cache, + access_token, + source_resource + .child_names() + .iter() + .map(|calendar| calendar.parent_id), + ) + }) + .filter(|acl| !acl.may_write(EventOwnership::NotOwner)); + let participant_filter = match hidden_acl { + Some(acl) => { + if !is_shared && viewer_identities.is_none() { + viewer_identities = Some( + self.account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())?, + ); + } + let viewer = viewer_identities.as_ref().unwrap_or(&identities); + (!acl.may_write(viewer.event_ownership(&event))).then_some(viewer) + } + None => None, + }; + let mut event = event.into_jscalendar(); + if let Some(identities) = participant_filter { + for entry in event.entries_mut() { + entry.retain_visible_participants(identities); + } + } + let source = EventSource::Copy(CopiedEvent { + event, + flags, + use_default_alerts, + }); match self .create_calendar_event( - &cache, + &mut context, &mut batch, - access_token, - account_id, - &account_info, - false, - &can_add_calendars, - calendar_event.data.event.into_jscalendar(), + source, create, + Some(NewMetadata::Copy { + patches, + source_id: from_calendar_event_id, + }), ) .await? { - Ok(document_id) => { - response.created(id, document_id); + Ok((slot, server_set)) => { + created_slots.push((create_id, slot, server_set)); // Add to destroy list if on_success_delete { - destroy_ids.push(MaybeInvalid::Value(id)); + destroy_ids.push(MaybeInvalid::Value(source_id)); } } Err(err) => { - response.not_created.append(id, err); + response.not_created.append(create_id, err); continue 'create; } } @@ -194,14 +395,30 @@ impl JmapCalendarEventCopy for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; - self.notify_task_queue(); + let assigned_ids = match context.metadata.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + for (create_id, _, _) in created_slots { + response.not_created.append( + create_id, + SetError::forbidden().with_description( + "Another process modified this calendar event, please try again.", + ), + ); + } + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; - response.new_state = State::Exact(change_id); + for (create_id, slot, mut server_set) in created_slots { + let document_id = assigned_ids.slot(slot); + server_set.resolve_document_id(document_id); + response.created_with_properties(create_id, document_id, server_set); + } + + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::Calendar)); } // Destroy ids diff --git a/crates/jmap/src/calendar_event/get.rs b/crates/jmap/src/calendar_event/get.rs index ae2cdabb..68963192 100644 --- a/crates/jmap/src/calendar_event/get.rs +++ b/crates/jmap/src/calendar_event/get.rs @@ -4,45 +4,60 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{calendar_event::CalendarSyntheticId, changes::state::JmapCacheState}; +use crate::{ + api::{ + budget::GetBudget, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + blob::embedded::{EmbeddedBlobIds, import_error}, + calendar_event::{CalendarSyntheticId, EventMap, EventValue, is_origin}, + changes::state::JmapCacheState, +}; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ - ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarParameter, - ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationRole, - ICalendarProperty, ICalendarValue, + ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, + ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, + ICalendarValue, ICalendarValueType, }, jscalendar::{ - JSCalendarDateTime, JSCalendarProperty, JSCalendarValue, import::ConversionOptions, + JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue, import::ImportOptions, }, }; -use chrono::DateTime; -use common::{Server, auth::AccessToken}; +use common::{ + CachedName, GroupwareResources, Server, + auth::{AccessToken, AccountInfo}, + network::memory::MemoryReservation, + storage::batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, +}; use groupware::{ cache::GroupwareCache, calendar::{ - CalendarEvent, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, - PREF_USE_DEFAULT_ALERTS, expand::CalendarEventExpansion, + ArchivedCalendarEventContent, CalendarEventContent, CalendarEventData, EVENT_DRAFT, + EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, EVENT_SECRET, + alerts::{DefaultAlertsResolver, DefaultAlertsView, ICalendarDefaultAlerts}, + expand::{CalendarEventExpansion, SECONDS_PER_DAY, resolve_local}, + identity::{CalendarAddresses, EventOwnership, ParticipantIdentityAddresses}, + participants::VisibleParticipants, + privacy::{EventPrivacy, ICalendarPrivacy}, + rights::EventAcl, + user::{ICalendarUserData, UserDataEntry, UserDataView}, }, }; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, - object::{JmapObjectId, calendar_event}, - request::IntoValid, + method::get::{GetRequest, GetResponse, all_ids}, + object::calendar_event, + request::{IntoValid, capability::CapabilityIds}, }; use jmap_tools::{Key, Map, Value}; -use std::{borrow::Cow, str::FromStr}; -use store::{ - ValueKey, - ahash::{AHashMap, AHashSet}, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive}, -}; +use std::{ops::Range, str::FromStr}; +use store::{ahash::AHashSet, write::serialize::rkyv_deserialize}; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, + field::CalendarEventField, id::Id, }; @@ -51,6 +66,8 @@ pub trait CalendarEventGet: Sync + Send { &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> impl Future>> + Send; } @@ -59,58 +76,279 @@ impl CalendarEventGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> trc::Result> { let return_all_properties = request.properties.is_none(); - let properties = request.unwrap_properties(&[]); + let (properties, selection) = select_properties(&mut request, &[], using)?; + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::CalendarEvent); + let metadata_get = metadata.get(selection); let account_id = request.account_id.document_id(); - let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let sampled = metadata.viewer_change_id(self, account_id).await?; + let default_tz = request.arguments.resolved_time_zone()?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await?; - let calendar_event_ids = if access_token.is_member(account_id) { - cache.document_ids(false).collect::() - } else { - cache.shared_items(access_token, [Acl::ReadItems], true) + let is_account_member = access_token.is_member(account_id); + let shared_event_ids = (!is_account_member).then(|| { + let mut shared_ids = cache.shared_items(access_token, [Acl::ReadItems], true); + shared_ids -= cache.event_ids_with_flags(EVENT_SECRET); + shared_ids + }); + let readable_event = |document_id: u32| { + shared_event_ids + .as_ref() + .is_none_or(|ids| ids.contains(document_id)) + .then(|| cache.item_by_id(document_id)) + .flatten() }; - let (mut ids, has_synthetic_ids) = if let Some(rr) = request.ids.take() { + let mut ids = if let Some(rr) = request.ids.take() { let rr = rr.unwrap(); if rr.len() > self.core.jmap.get_max_objects { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } - let mut ids = Vec::with_capacity(rr.len()); - let mut has_synthetic_ids = false; - - for id in rr.into_valid() { - has_synthetic_ids |= id.is_synthetic(); - ids.push(id); - } - - (ids, has_synthetic_ids) - } else { - ( - calendar_event_ids + rr.into_valid() + .enumerate() + .map(|(index, id)| (id, index)) + .collect::>() + } else if let Some(shared_event_ids) = &shared_event_ids { + all_ids( + shared_event_ids .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>(), - false, - ) + .map(Id::from) + .enumerate() + .map(|(index, id)| (id, index)), + self.core.jmap.get_max_objects, + )? + } else { + all_ids( + cache + .document_ids(false) + .map(Id::from) + .enumerate() + .map(|(index, id)| (id, index)), + self.core.jmap.get_max_objects, + )? + }; + ids.sort_unstable_by_key(|(id, index)| (id.document_id(), u64::from(*id), *index)); + ids.dedup_by_key(|(id, _)| *id); + let mut metadata_values = match &metadata_get { + Some(get) => { + let mut documents = MetadataDocuments::default(); + for (id, _) in &ids { + let document_id = id.document_id(); + if let Some(resource) = readable_event(document_id).filter(|resource| { + EventPrivacy::from_flags(resource.event_flags().unwrap_or_default()) + .private_view(is_account_member) + == Some(false) + }) { + documents.insert(document_id, resource.metadata_kinds()); + } + } + Some(metadata.load(self, account_id, get, &documents).await?) + } + None => None, }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(false).into(), + state: sampled.state(cache.get_state(false)).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; - let mut return_converted_props = !return_all_properties; + let override_range = if request.arguments.recurrence_overrides_after.is_some() + || request.arguments.recurrence_overrides_before.is_some() + { + let after = request + .arguments + .recurrence_overrides_after + .map(|v| v.timestamp) + .unwrap_or(i64::MIN); + let before = request + .arguments + .recurrence_overrides_before + .map(|v| v.timestamp) + .unwrap_or(i64::MAX); + Some(after..before) + } else { + None + }; + let projection = ProjectionRequest::new( + (!return_all_properties).then_some(properties), + request.arguments.reduce_participants.unwrap_or(false), + override_range, + )?; + let mut projector = + EventProjector::new(self, access_token, account_id, &projection).await?; + let mut results = ProjectionResults::with_capacity(ids.len()); + + let mut budget = GetBudget::new(reservation); + let mut batch = DocumentBatch::new(MAX_BATCH_BYTES); + let mut remaining = ids.as_slice(); + while !remaining.is_empty() { + let chunk_len = remaining + .chunk_by(same_document) + .take(MAX_BATCH_DOCUMENTS) + .map(<[_]>::len) + .sum(); + let Some((chunk, rest)) = remaining.split_at_checked(chunk_len) else { + break; + }; + remaining = rest; + + batch.reset(); + for group in chunk.chunk_by(same_document) { + if let Some((first_id, _)) = group.first() + && projector.needs_archive(group) + && readable_event(first_id.document_id()).is_some() + { + batch.request(first_id.document_id()); + } + } + + loop { + self.read_archive_batch( + account_id, + Collection::CalendarEvent, + CalendarEventField::Content.into(), + &mut batch, + ) + .await?; + + for group in chunk.chunk_by(same_document) { + let Some((first_id, _)) = group.first() else { + continue; + }; + let document_id = first_id.document_id(); + let Some(resource) = readable_event(document_id) else { + if batch.is_first_round() { + results.push_not_found(group); + } + continue; + }; + let archive = if projector.needs_archive(group) { + match batch.lookup(document_id) { + Lookup::Found(archive) => { + budget.add_archive(archive.as_bytes().len(), group.len())?; + Some(archive) + } + Lookup::Missing => { + results.push_not_found(group); + continue; + } + Lookup::Skip => continue, + } + } else if batch.is_first_round() { + None + } else { + continue; + }; + let flags = resource.event_flags().unwrap_or_default(); + let projected_from = results.found.len(); + projector + .project( + self, + access_token, + ProjectedEvent { + cache: &cache, + document_id, + content: archive + .map(|archive| archive.unarchive::()) + .transpose() + .caused_by(trc::location!())?, + flags, + privacy: EventPrivacy::from_flags(flags), + calendar_ids: resource.child_names(), + time_zone: default_tz, + ids: group, + }, + &mut results, + ) + .await?; + if let Some(values) = &mut metadata_values { + for (_, result) in results.found.iter_mut().skip(projected_from) { + values.insert_into(document_id, result); + } + } + } + + if !batch.next_round() { + break; + } + } + } + + for id in results.not_found { + response.push_not_found(id); + } + let mut results = results.found; + results.sort_unstable_by_key(|(index, _)| *index); + response + .list + .extend(results.into_iter().map(|(_, result)| result.into())); + + budget.commit(); + Ok(response) + } +} + +fn same_document((a, _): &(Id, usize), (b, _): &(Id, usize)) -> bool { + a.document_id() == b.document_id() +} + +pub(crate) struct ProjectionRequest { + return_all_properties: bool, + jmap_properties: Vec>, + jscal_properties: Vec>, + return_utc_dates: bool, + return_is_origin: bool, + needs_conversion: bool, + needs_event: bool, + needs_content: bool, + reduce_participants: bool, + override_range: Option>, +} + +pub(crate) struct EventProjector<'x> { + request: &'x ProjectionRequest, + account_id: u32, + personal_id: u32, + is_account_member: bool, + account_info: Option, + identities: Option, + default_alerts: DefaultAlertsResolver, +} + +pub(crate) struct ProjectedEvent<'x> { + pub cache: &'x GroupwareResources, + pub document_id: u32, + pub content: Option<&'x ArchivedCalendarEventContent>, + pub flags: u16, + pub privacy: EventPrivacy, + pub calendar_ids: &'x [CachedName], + pub time_zone: Tz, + pub ids: &'x [(Id, usize)], +} + +#[derive(Default)] +pub(crate) struct ProjectionResults { + pub found: Vec<(usize, EventMap<'static>)>, + pub not_found: Vec, +} + +impl ProjectionRequest { + pub(crate) fn new( + properties: Option>>, + reduce_participants: bool, + override_range: Option>, + ) -> trc::Result { + let return_all_properties = properties.is_none(); let mut return_is_origin = false; let mut return_utc_dates = false; - - let (jmap_properties, jscal_properties) = if !return_all_properties { + let (jmap_properties, jscal_properties) = if let Some(properties) = properties { let mut jmap_properties = Vec::with_capacity(4); let mut jscal_properties = Vec::with_capacity(properties.len()); @@ -135,14 +373,17 @@ impl CalendarEventGet for Server { jmap_properties.push(property); } _ => { - if matches!(property, JSCalendarProperty::ICalendar) { - return_converted_props = true; - } - jscal_properties.push(property); } } } + if return_utc_dates + && jscal_properties.contains(&JSCalendarProperty::RecurrenceOverrides) + { + return Err(trc::JmapEvent::InvalidArguments.into_err().details( + "recurrenceOverrides cannot be requested together with utcStart or utcEnd", + )); + } (jmap_properties, jscal_properties) } else { return_is_origin = true; @@ -152,309 +393,370 @@ impl CalendarEventGet for Server { JSCalendarProperty::CalendarIds, JSCalendarProperty::IsDraft, JSCalendarProperty::IsOrigin, + JSCalendarProperty::UseDefaultAlerts, + JSCalendarProperty::MayInviteSelf, + JSCalendarProperty::MayInviteOthers, + JSCalendarProperty::HideAttendees, ], vec![], ) }; - let current_account_info = self - .account_info(access_token.account_id()) - .await - .caused_by(trc::location!())?; - let return_is_origin = if return_is_origin { - if account_id == access_token.account_id() { - Some(Cow::Borrowed(¤t_account_info)) - } else { + let needs_conversion = return_all_properties || !jscal_properties.is_empty(); + let needs_event = needs_conversion || return_utc_dates || return_is_origin; + let needs_content = + needs_event || jmap_properties.contains(&JSCalendarProperty::UseDefaultAlerts); + + Ok(ProjectionRequest { + return_all_properties, + jmap_properties, + jscal_properties, + return_utc_dates, + return_is_origin, + needs_conversion, + needs_event, + needs_content, + reduce_participants, + override_range, + }) + } + + fn requested(&self) -> RequestedProperties<'_> { + RequestedProperties { + return_all: self.return_all_properties, + properties: &self.jscal_properties, + } + } +} + +impl ProjectionResults { + fn with_capacity(capacity: usize) -> Self { + ProjectionResults { + found: Vec::with_capacity(capacity), + not_found: Vec::new(), + } + } + + fn push_not_found(&mut self, ids: &[(Id, usize)]) { + self.not_found.extend(ids.iter().map(|(id, _)| *id)); + } + + fn push_metadata( + &mut self, + metadata: &EventMetadata<'_>, + ids: &[(Id, usize)], + properties: &[JSCalendarProperty], + ) { + self.found.extend( + ids.iter() + .map(|(id, index)| (*index, metadata.to_object(*id, properties))), + ); + } +} + +impl<'x> EventProjector<'x> { + pub(crate) async fn new( + server: &Server, + access_token: &AccessToken, + account_id: u32, + request: &'x ProjectionRequest, + ) -> trc::Result { + Ok(EventProjector { + request, + account_id, + personal_id: access_token.personal_id(account_id, Collection::Calendar), + is_account_member: access_token.is_member(account_id), + account_info: if request.return_is_origin { Some( - self.account_info(account_id) + server + .account_info(account_id) .await - .map(Cow::Owned) .caused_by(trc::location!())?, ) - } - } else { - None - }; - - // Sort by baseId - let mut original_order: Option> = None; - if has_synthetic_ids { - original_order = Some(ids.iter().enumerate().map(|(i, id)| (*id, i)).collect()); - ids.sort_unstable_by_key(|id| id.document_id()); - } - let mut ids = ids.into_iter().peekable(); - - // Process arguments - let override_range = if request.arguments.recurrence_overrides_after.is_some() - || request.arguments.recurrence_overrides_before.is_some() - { - let after = request - .arguments - .recurrence_overrides_after - .map(|v| v.timestamp) - .unwrap_or(i64::MIN); - let before = request - .arguments - .recurrence_overrides_before - .map(|v| v.timestamp) - .unwrap_or(i64::MAX); - if after < before { - Some(after..before) } else { None + }, + identities: None, + default_alerts: DefaultAlertsResolver::default(), + }) + } + + pub(crate) fn needs_archive(&self, ids: &[(Id, usize)]) -> bool { + self.request.needs_content || ids.iter().any(|(id, _)| id.is_synthetic()) + } + + pub(crate) async fn project( + &mut self, + server: &Server, + access_token: &AccessToken, + document: ProjectedEvent<'_>, + results: &mut ProjectionResults, + ) -> trc::Result<()> { + let request = self.request; + let requested = request.requested(); + let jmap_properties = &request.jmap_properties; + let jscal_properties = &request.jscal_properties; + let return_all_properties = request.return_all_properties; + let return_utc_dates = request.return_utc_dates; + let needs_conversion = request.needs_conversion; + let needs_event = request.needs_event; + let reduce_participants = request.reduce_participants; + let default_tz = document.time_zone; + let expansion_tz = if return_utc_dates { + default_tz + } else { + Tz::Floating + }; + let override_range = request.override_range.as_ref(); + let account_id = self.account_id; + let personal_id = self.personal_id; + let is_account_member = self.is_account_member; + let ProjectedEvent { + document_id, + content, + flags, + privacy, + calendar_ids, + ids: group, + cache, + .. + } = document; + + let mut recurrence_keys = Vec::with_capacity(group.len()); + let mut base_index = None; + for (id, index) in group { + match id.recurrence_key() { + Some(recurrence_key) => recurrence_keys.push((recurrence_key, *id, *index)), + None => base_index = Some(*index), } + } + + let Some(is_private_view) = privacy.private_view(is_account_member) else { + results.push_not_found(group); + return Ok(()); + }; + let mut metadata = EventMetadata { + flags, + calendar_ids, + is_private_view, + use_default_alerts: false, + is_origin: false, + utc_range: None, + }; + + let Some(archived_content) = content else { + results.push_metadata(&metadata, group, jmap_properties); + return Ok(()); + }; + let archived_preferences = archived_content.preferences(personal_id); + metadata.use_default_alerts = + archived_preferences.is_some_and(|preferences| preferences.use_default_alerts()); + if !needs_event && recurrence_keys.is_empty() { + results.push_metadata(&metadata, group, jmap_properties); + return Ok(()); + } + let mut content = CalendarEventContent { + data: rkyv_deserialize(&archived_content.data).caused_by(trc::location!())?, + preferences: archived_preferences + .map(rkyv_deserialize) + .transpose() + .caused_by(trc::location!())? + .into_iter() + .collect(), + }; + let Some(is_private_view) = privacy + .max(content.data.event.privacy()) + .private_view(is_account_member) + else { + results.push_not_found(group); + return Ok(()); + }; + + let personal_updated = if !is_account_member { + let preferences = content + .preferences + .iter() + .find(|p| p.account_id == personal_id); + if requested.any(PERSONAL_PROPERTIES) { + content + .data + .apply_user_data(preferences, UserDataView::Full); + } + preferences + .map(|p| p.updated) + .filter(|updated| *updated > 0) } else { None }; - let default_tz = request.arguments.time_zone.unwrap_or(Tz::UTC); - let reduce_participants = request.arguments.reduce_participants.unwrap_or(false); - while let Some(id) = ids.next() { - // Obtain the calendar_event object - let document_id = id.document_id(); - if !calendar_event_ids.contains(document_id) { - response.push_not_found(id); - continue; - } + let private_is_origin = if is_private_view { + let is_origin = self + .account_info + .as_ref() + .is_some_and(|account| is_origin(&content.data.event, account.addresses())); + content.data.event = std::mem::take(&mut content.data.event).into_private_view(); + Some(is_origin) + } else { + None + }; - let Some(_calendar_event) = self - .store() - .get_value::>(ValueKey::archive( + if !is_private_view && requested.any(ALERT_PROPERTIES) { + let default_alerts = self + .default_alerts + .resolve_for_content( + server, account_id, - Collection::CalendarEvent, - document_id, - )) - .await? - else { - response.push_not_found(id); - continue; - }; - let mut calendar_event = _calendar_event - .deserialize::() + personal_id, + &content, + calendar_ids.iter().map(CachedName::calendar_id), + ) + .await .caused_by(trc::location!())?; + content + .data + .event + .apply_default_alerts(&default_alerts, DefaultAlertsView::Replace); + } - // Extract recurrence keys from synthetic ids - let mut recurrence_keys = AHashSet::new(); - let mut include_base_event = false; - if let Some(recurrence_key) = id.recurrence_key() { - recurrence_keys.insert(recurrence_key); + let hidden_acl = (!is_account_member + && flags & EVENT_HIDE_ATTENDEES != 0 + && requested.any(PARTICIPANT_PROPERTIES)) + .then(|| { + EventAcl::for_calendars( + cache, + access_token, + calendar_ids.iter().map(CachedName::calendar_id), + ) + }) + .filter(|acl| !acl.may_write(EventOwnership::NotOwner)); + let participant_filter = if requested.any(PARTICIPANT_PROPERTIES) + && (reduce_participants || hidden_acl.is_some()) + { + if self.identities.is_none() { + let account_info = server + .account_info(access_token.account_id()) + .await + .caused_by(trc::location!())?; + self.identities = server + .identity_addresses(access_token.account_id(), &account_info) + .await + .caused_by(trc::location!())? + .into(); + } + self.identities.as_ref().filter(|identities| { + reduce_participants + || hidden_acl.is_some_and(|acl| { + !acl.may_write(identities.event_ownership(&content.data.event)) + }) + }) + } else { + None + }; + let event_is_origin = private_is_origin.unwrap_or_else(|| { + self.account_info + .as_ref() + .is_some_and(|account| is_origin(&content.data.event, account.addresses())) + }); + + let needs_binaries = requested.any(BLOB_PROPERTIES); + let binaries = if needs_binaries { + InstanceBinaries::Keep + } else { + InstanceBinaries::Skip + }; + let convert = |ical: ICalendar| { + let options = ImportOptions::new() + .include_ical_components(!return_all_properties && !is_private_view) + .return_first(true); + if needs_binaries { + ical.into_jscalendar_with::(options.with_blob_id_generator( + EmbeddedBlobIds::new(account_id, Collection::CalendarEvent, document_id), + )) } else { - include_base_event = true; + ical.into_jscalendar_with::(options) } - while let Some(next_id) = ids.peek() { - if next_id.document_id() == document_id { - if let Some(recurrence_key) = next_id.recurrence_key() { - recurrence_keys.insert(recurrence_key); - } else { - include_base_event = true; - } - ids.next(); - } else { - break; - } - } - - // Reduce participants - if reduce_participants { - for component in &mut calendar_event.data.event.components { - if component.component_type.is_scheduling_object() { - component.entries.retain(|entry| match &entry.name { - ICalendarProperty::Attendee => { - entry.parameters(&ICalendarParameterName::Role).any(|role| { - matches!( - role, - ICalendarParameterValue::Role( - ICalendarParticipationRole::Owner, - ), - ) - }) || entry.calendar_address().is_some_and(|addr| { - current_account_info - .addresses() - .iter() - .any(|a| a.eq_ignore_ascii_case(addr)) - }) - } - _ => true, - }); - } - } - } - - // Expand synthetic ids - let mut results = Vec::with_capacity(recurrence_keys.len() + 1); - if !recurrence_keys.is_empty() { - let ical = &calendar_event.data.event; - if let Some(expansions) = calendar_event - .data - .expand_from_ids(&mut recurrence_keys, default_tz) - { - for expansion in expansions { - let Some(recurrence_key) = expansion.recurrence_key() else { - continue; - }; - let component = &ical.components[expansion.comp_id as usize]; - let source_component = component; - let is_recurrent = component.is_recurrent(); - let is_recurrent_or_override = - is_recurrent || component.is_recurrence_override(); - let mut has_duration = false; - let component_ids = &component.component_ids; - let mut tz = None; - let mut component = ICalendarComponent { - component_type: component.component_type.clone(), - component_ids: Vec::new(), - entries: component - .entries - .iter() - .filter(|entry| match &entry.name { - ICalendarProperty::Dtstart - | ICalendarProperty::Dtend - | ICalendarProperty::Exdate - | ICalendarProperty::Exrule - | ICalendarProperty::Rdate - | ICalendarProperty::Rrule - | ICalendarProperty::RecurrenceId => { - if let Some(new_tz) = entry - .tz_id() - .and_then(|id| Tz::from_str(id).ok()) - .filter(|tz| *tz != Tz::UTC) - { - tz = Some(new_tz); - } - false - } - ICalendarProperty::Due - | ICalendarProperty::Completed - | ICalendarProperty::Created => is_recurrent, - ICalendarProperty::Duration => { - has_duration = true; - true - } - _ => true, - }) - .cloned() - .collect::>(), - }; - - let tz = tz.unwrap_or(default_tz); - let tz_name = tz.name().unwrap_or_default().to_string(); - - let start_timestamp = DateTime::from_timestamp(expansion.start, 0) - .map(|dt| dt.with_timezone(&tz)) - .map(|dt| dt.naive_local()) - .map(|dt| dt.and_utc().timestamp()) - .unwrap_or(expansion.start); - - let end_timestamp = DateTime::from_timestamp(expansion.end, 0) - .map(|dt| dt.with_timezone(&tz)) - .map(|dt| dt.naive_local()) - .map(|dt| dt.and_utc().timestamp()) - .unwrap_or(expansion.end); - - component.entries.push(ICalendarEntry { - name: ICalendarProperty::Dtstart, - params: vec![ICalendarParameter::tzid(tz_name.clone())], - values: vec![ICalendarValue::PartialDateTime(Box::new( - PartialDateTime::from_naive_timestamp(start_timestamp), - ))], - }); - - if is_recurrent_or_override { - component.entries.push( - source_component - .property(&ICalendarProperty::RecurrenceId) - .filter(|entry| { - entry - .parameters(&ICalendarParameterName::Range) - .next() - .is_none() - || calendar_event - .data - .expand_single(expansion.comp_id, default_tz) - .is_some_and(|first| { - first.start_naive == expansion.start_naive - }) - }) - .cloned() - .unwrap_or_else(|| ICalendarEntry { - name: ICalendarProperty::RecurrenceId, - params: vec![ICalendarParameter::tzid(tz_name.clone())], - values: vec![ICalendarValue::PartialDateTime(Box::new( - PartialDateTime::from_naive_timestamp(start_timestamp), - ))], - }), - ); - } - - if !has_duration { - component.entries.push(ICalendarEntry { - name: ICalendarProperty::Dtend, - params: vec![ICalendarParameter::tzid(tz_name)], - values: vec![ICalendarValue::PartialDateTime(Box::new( - PartialDateTime::from_naive_timestamp(end_timestamp), - ))], - }); - } - - let mut expanded_ical = ICalendar { - components: vec![ - ICalendarComponent { - component_type: ICalendarComponentType::VCalendar, - entries: vec![], - component_ids: vec![1], - }, - component, - ], - }; - - if !component_ids.is_empty() { - for component_id in component_ids { - let mut sub_component = - ical.components[*component_id as usize].clone(); - sub_component.component_ids.clear(); - let component_id = expanded_ical.components.len() as u32; - expanded_ical.components.push(sub_component); - expanded_ical.components[1].component_ids.push(component_id); - } - } - - results.push(( - ::new(recurrence_key, document_id), - expanded_ical, - expansion, - )); - } - } - - for recurrence_key in recurrence_keys { - response.push_not_found(::new( - recurrence_key, - document_id, - )); - } - } - - if include_base_event { - let mut event = std::mem::take(&mut calendar_event.data.event); - - // Obtain UTC start/end if requested - let expansion = if return_utc_dates - && let Some(expansion) = event - .components + .map(JSCalendar::into_inner) + .map_err(import_error) + }; + let mut instances = Vec::with_capacity(recurrence_keys.len() + 1); + let mut templates: Vec<(u32, EventValue<'static>)> = Vec::new(); + if !recurrence_keys.is_empty() { + let mut keys = recurrence_keys + .iter() + .map(|(recurrence_key, _, _)| *recurrence_key) + .collect::>(); + for expansion in content + .data + .expand_from_ids(&mut keys, expansion_tz) + .unwrap_or_default() + { + let Some(position) = expansion.recurrence_key().and_then(|recurrence_key| { + recurrence_keys .iter() - .position(|c| { - c.component_type.is_scheduling_object() && !c.is_recurrence_override() - }) - .and_then(|comp_id| { - calendar_event - .data - .expand_single(comp_id as u32, default_tz) - }) { - expansion - } else { - CalendarEventExpansion::default() + .position(|(key, _, _)| *key == recurrence_key) + }) else { + continue; }; + let (recurrence_key, id, index) = recurrence_keys.swap_remove(position); + let jscal = if needs_conversion { + let user_data = (!is_account_member) + .then(|| { + content + .data + .event + .user_data_override(expansion.comp_id, recurrence_key) + }) + .flatten(); + let is_shared_component = user_data.is_none() + && content.data.has_stable_instance_duration(expansion.comp_id); + let cached = is_shared_component + .then(|| { + templates + .iter() + .position(|(comp_id, _)| *comp_id == expansion.comp_id) + }) + .flatten(); + match cached { + Some(position) => { + let mut jscal = templates[position].1.clone(); + jscal.set_instance_dates(&expansion); + Some(jscal) + } + None => { + let Some(ical) = content.data.instance(&expansion, user_data, binaries) + else { + results.not_found.push(id); + continue; + }; + let jscal = convert(ical)?; + if is_shared_component { + templates.push((expansion.comp_id, jscal.clone())); + } + Some(jscal) + } + } + } else { + None + }; + instances.push(EventInstance { + id, + index, + jscal, + utc_range: Some((expansion.start, expansion.end)), + }); + } - // Remove recurrence ids + for (_, id, _) in recurrence_keys { + results.not_found.push(id); + } + } + + if let Some(index) = base_index { + let utc_range = return_utc_dates + .then(|| content.data.expand_base(default_tz)) + .flatten() + .map(|expansion| (expansion.start, expansion.end)); + let ical = needs_conversion.then(|| { + let mut event = std::mem::take(&mut content.data.event); if let Some(range) = &override_range { let remove_ids = event .components @@ -467,7 +769,14 @@ impl CalendarEventGet for Server { .and_then(|p| p.values.first()) .and_then(|v| v.as_partial_date_time()) .and_then(|v| v.to_date_time()) - .and_then(|v| v.to_date_time_with_tz(default_tz)) + .and_then(|v| { + v.to_date_time_with_tz( + c.property(&ICalendarProperty::RecurrenceId) + .and_then(|entry| entry.tz_id()) + .and_then(|tz| Tz::from_str(tz).ok()) + .unwrap_or(default_tz), + ) + }) .map(|v| v.timestamp()) && !range.contains(×tamp) { @@ -485,172 +794,664 @@ impl CalendarEventGet for Server { } } } - - results.push((Id::from(document_id), event, expansion)); - } - - for (id, ical, expansion) in results { - let is_origin = return_is_origin.as_ref().is_some_and(|account| { - ical.components - .iter() - .find(|c| c.component_type.is_scheduling_object()) - .and_then(|c| c.property(&ICalendarProperty::Organizer)) - .and_then(|v| v.calendar_address()) - .is_none_or(|v| { - account - .addresses() - .iter() - .any(|a| a.eq_ignore_ascii_case(v)) - }) - }); - - let jscal = ical - .into_jscalendar_with_opt::( - ConversionOptions::default() - .include_ical_components(return_converted_props) - .return_first(true), - ) - .into_inner(); - let mut result = if return_all_properties { - jscal.into_object().unwrap() - } else { - let is_synthetic = id.is_synthetic(); - let is_null_for_synthetic = |property: &JSCalendarProperty| { - is_synthetic - && matches!( - property, - JSCalendarProperty::RecurrenceRule - | JSCalendarProperty::RecurrenceOverrides - ) - }; - let mut result = - Map::from_iter(jscal.into_expanded_object().filter(|(k, _)| { - k.as_property().is_some_and(|p| { - jscal_properties.contains(p) && !is_null_for_synthetic(p) - }) - })); - for property in jscal_properties - .iter() - .filter(|property| is_null_for_synthetic(property)) - { - result.insert_unchecked(property.clone(), Value::Null); - } - result - }; - - for property in &jmap_properties { - match property { - JSCalendarProperty::Id => { - result.insert_unchecked( - JSCalendarProperty::Id, - Value::Element(JSCalendarValue::Id(id)), - ); - } - JSCalendarProperty::BaseEventId => { - result.insert_unchecked( - JSCalendarProperty::BaseEventId, - if id.is_synthetic() { - Value::Element(JSCalendarValue::Id(id.document_id().into())) - } else { - Value::Null - }, - ); - } - JSCalendarProperty::CalendarIds => { - let mut obj = Map::with_capacity(calendar_event.names.len()); - for id in calendar_event.names.iter() { - obj.insert_unchecked( - JSCalendarProperty::IdValue(Id::from(id.parent_id)), - true, - ); - } - result.insert_unchecked( - JSCalendarProperty::CalendarIds, - Value::Object(obj), - ); - } - JSCalendarProperty::IsDraft => { - result.insert_unchecked( - JSCalendarProperty::IsDraft, - Value::Bool(calendar_event.flags & EVENT_DRAFT != 0), - ); - } - JSCalendarProperty::IsOrigin => { - result.insert_unchecked( - JSCalendarProperty::IsOrigin, - Value::Bool(is_origin), - ); - } - JSCalendarProperty::MayInviteSelf => { - result.insert_unchecked( - JSCalendarProperty::MayInviteSelf, - Value::Bool(calendar_event.flags & EVENT_INVITE_SELF != 0), - ); - } - JSCalendarProperty::MayInviteOthers => { - result.insert_unchecked( - JSCalendarProperty::MayInviteOthers, - Value::Bool(calendar_event.flags & EVENT_INVITE_OTHERS != 0), - ); - } - JSCalendarProperty::HideAttendees => { - result.insert_unchecked( - JSCalendarProperty::HideAttendees, - Value::Bool(calendar_event.flags & EVENT_HIDE_ATTENDEES != 0), - ); - } - - JSCalendarProperty::UtcStart => { - result.insert_unchecked( - JSCalendarProperty::UtcStart, - Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( - expansion.start, - false, - ))), - ); - } - JSCalendarProperty::UtcEnd => { - result.insert_unchecked( - JSCalendarProperty::UtcEnd, - Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( - expansion.end, - false, - ))), - ); - } - JSCalendarProperty::UseDefaultAlerts => { - result.insert_unchecked( - JSCalendarProperty::UseDefaultAlerts, - Value::Bool( - calendar_event - .preferences(personal_id) - .is_some_and(|v| v.flags & PREF_USE_DEFAULT_ALERTS != 0), - ), - ); - } - - _ => {} - } + if binaries == InstanceBinaries::Skip { + event.strip_binary_entries(); } - - response.list.push(result.into()); - } - } - - // Restore original order - if let Some(original_order) = original_order { - response.list.sort_by_key(|obj| { - obj.as_object() - .unwrap() - .get(&Key::Property(JSCalendarProperty::::Id)) - .and_then(|v| v.as_element()) - .and_then(|v: &JSCalendarValue| v.as_id()) - .and_then(|id| original_order.get(&id)) - .cloned() - .unwrap_or(usize::MAX) + event + }); + instances.push(EventInstance { + id: Id::from(document_id), + index, + jscal: ical.map(convert).transpose()?, + utc_range, }); } - Ok(response) + metadata.is_private_view = is_private_view; + metadata.is_origin = event_is_origin; + for instance in instances { + let mut result = match instance.jscal { + Some(mut jscal) => { + if let Some(identities) = participant_filter { + jscal.retain_visible_participants(identities); + } + if let Some(range) = override_range.filter(|_| !instance.id.is_synthetic()) { + jscal.retain_overrides_in(range, default_tz); + } + + if return_all_properties { + jscal.into_object().unwrap_or_else(Map::new) + } else { + let is_synthetic = instance.id.is_synthetic(); + let is_null_for_synthetic = |property: &JSCalendarProperty| { + is_synthetic + && matches!( + property, + JSCalendarProperty::RecurrenceRule + | JSCalendarProperty::RecurrenceOverrides + ) + }; + let mut result = + Map::from_iter(jscal.into_expanded_object().filter(|(k, _)| { + k.as_property().is_some_and(|p| { + jscal_properties.contains(p) && !is_null_for_synthetic(p) + }) + })); + for property in jscal_properties + .iter() + .filter(|property| is_null_for_synthetic(property)) + { + result.insert_unchecked(property.clone(), Value::Null); + } + result + } + } + None => Map::with_capacity(jmap_properties.len()), + }; + + if let Some(personal_updated) = personal_updated + && let Some(updated) = result.get_mut(&Key::Property(JSCalendarProperty::Updated)) + && matches!( + updated, + Value::Element(JSCalendarValue::DateTime(current)) + if current.timestamp < personal_updated + ) + { + *updated = Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + personal_updated, + false, + ))); + } + + metadata.utc_range = instance.utc_range; + metadata.insert_properties(&mut result, instance.id, jmap_properties); + results.found.push((instance.index, result)); + } + + Ok(()) } } + +const PERSONAL_PROPERTIES: &[JSCalendarProperty] = &[ + JSCalendarProperty::Keywords, + JSCalendarProperty::Color, + JSCalendarProperty::FreeBusyStatus, + JSCalendarProperty::Alerts, + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::ICalendar, +]; +const ALERT_PROPERTIES: &[JSCalendarProperty] = &[ + JSCalendarProperty::Alerts, + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::ICalendar, +]; +const PARTICIPANT_PROPERTIES: &[JSCalendarProperty] = &[ + JSCalendarProperty::Participants, + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::ICalendar, +]; +const BLOB_PROPERTIES: &[JSCalendarProperty] = &[ + JSCalendarProperty::Links, + JSCalendarProperty::Locations, + JSCalendarProperty::Participants, + JSCalendarProperty::RecurrenceOverrides, + JSCalendarProperty::ICalendar, +]; + +struct RequestedProperties<'x> { + return_all: bool, + properties: &'x [JSCalendarProperty], +} + +impl RequestedProperties<'_> { + fn any(&self, properties: &[JSCalendarProperty]) -> bool { + self.return_all + || self + .properties + .iter() + .any(|property| properties.contains(property)) + } +} + +struct EventInstance { + id: Id, + index: usize, + jscal: Option>, + utc_range: Option<(i64, i64)>, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum InstanceBinaries { + Keep, + Skip, +} + +trait InstanceDates { + fn set_instance_dates(&mut self, expansion: &CalendarEventExpansion); +} + +impl InstanceDates for EventValue<'static> { + fn set_instance_dates(&mut self, expansion: &CalendarEventExpansion) { + let Some(event) = self.as_object_mut() else { + return; + }; + for (property, timestamp) in [ + (JSCalendarProperty::Start, expansion.start_naive), + ( + JSCalendarProperty::RecurrenceId, + expansion.recurrence_id().naive, + ), + ] { + if let Some(Value::Element(JSCalendarValue::DateTime(date_time))) = + event.get_mut(&Key::Property(property)) + { + date_time.timestamp = timestamp; + } + } + } +} + +struct EventMetadata<'x> { + flags: u16, + calendar_ids: &'x [CachedName], + is_private_view: bool, + use_default_alerts: bool, + is_origin: bool, + utc_range: Option<(i64, i64)>, +} + +impl EventMetadata<'_> { + fn to_object( + &self, + id: Id, + properties: &[JSCalendarProperty], + ) -> Map<'static, JSCalendarProperty, JSCalendarValue> { + let mut result = Map::with_capacity(properties.len()); + self.insert_properties(&mut result, id, properties); + result + } + + fn insert_properties( + &self, + result: &mut Map<'_, JSCalendarProperty, JSCalendarValue>, + id: Id, + properties: &[JSCalendarProperty], + ) { + for property in properties + .iter() + .filter(|property| !self.is_private_view || property.is_private_view_property()) + { + let value = match property { + JSCalendarProperty::Id => Value::Element(JSCalendarValue::Id(id)), + JSCalendarProperty::BaseEventId if id.is_synthetic() => { + Value::Element(JSCalendarValue::Id(id.document_id().into())) + } + JSCalendarProperty::BaseEventId => Value::Null, + JSCalendarProperty::CalendarIds => { + Value::Object(Map::from_iter(self.calendar_ids.iter().map(|name| { + ( + Key::Property(JSCalendarProperty::IdValue(Id::from(name.parent_id))), + Value::Bool(true), + ) + }))) + } + JSCalendarProperty::IsDraft => Value::Bool(self.flags & EVENT_DRAFT != 0), + JSCalendarProperty::IsOrigin => Value::Bool(self.is_origin), + JSCalendarProperty::MayInviteSelf => { + Value::Bool(self.flags & EVENT_INVITE_SELF != 0) + } + JSCalendarProperty::MayInviteOthers => { + Value::Bool(self.flags & EVENT_INVITE_OTHERS != 0) + } + JSCalendarProperty::HideAttendees => { + Value::Bool(self.flags & EVENT_HIDE_ATTENDEES != 0) + } + JSCalendarProperty::UseDefaultAlerts => Value::Bool(self.use_default_alerts), + JSCalendarProperty::UtcStart => match self.utc_range { + Some((start, _)) => Value::Element(JSCalendarValue::DateTime( + JSCalendarDateTime::new(start, false), + )), + None => continue, + }, + JSCalendarProperty::UtcEnd => match self.utc_range { + Some((_, end)) => Value::Element(JSCalendarValue::DateTime( + JSCalendarDateTime::new(end, false), + )), + None => continue, + }, + _ => continue, + }; + result.insert_unchecked(property.clone(), value); + } + } +} + +trait CalendarId { + fn calendar_id(&self) -> u32; +} + +impl CalendarId for CachedName { + fn calendar_id(&self) -> u32 { + self.parent_id + } +} + +trait EventPrivacyView { + fn private_view(self, is_account_member: bool) -> Option; +} + +impl EventPrivacyView for EventPrivacy { + fn private_view(self, is_account_member: bool) -> Option { + match self { + EventPrivacy::Public => Some(false), + EventPrivacy::Private => Some(!is_account_member), + EventPrivacy::Secret => is_account_member.then_some(false), + } + } +} + +trait OverrideRange { + fn retain_overrides_in(&mut self, range: &Range, default_tz: Tz); +} + +impl OverrideRange for Value<'_, JSCalendarProperty, JSCalendarValue> { + fn retain_overrides_in(&mut self, range: &Range, default_tz: Tz) { + let Some(event) = self.as_object_mut() else { + return; + }; + let tz = event + .get(&Key::Property(JSCalendarProperty::TimeZone)) + .and_then(Value::as_str) + .and_then(|tz| Tz::from_str(&tz).ok()) + .unwrap_or(default_tz); + let key = Key::Property(JSCalendarProperty::RecurrenceOverrides); + let Some(overrides) = event.get_mut(&key).and_then(Value::as_object_mut) else { + return; + }; + overrides.as_mut_vec().retain(|(key, _)| match key { + Key::Property(JSCalendarProperty::DateTime(date_time)) => { + resolve_local(tz, date_time.timestamp) + .is_some_and(|timestamp| range.contains(×tamp)) + } + _ => true, + }); + if overrides.is_empty() { + event.remove(&key); + } + } +} + +#[derive(Clone, Copy)] +enum InstanceParameters { + All, + TimeZone, +} + +trait InstanceDateTime { + fn with_date_time( + &self, + name: ICalendarProperty, + naive: i64, + fallback_tz: Option, + parameters: InstanceParameters, + ) -> ICalendarEntry; + + fn with_date_time_in( + &self, + name: ICalendarProperty, + naive: i64, + tz: Option, + ) -> ICalendarEntry; +} + +impl InstanceDateTime for ICalendarEntry { + fn with_date_time_in( + &self, + name: ICalendarProperty, + naive: i64, + tz: Option, + ) -> ICalendarEntry { + let is_date = self + .values + .first() + .and_then(ICalendarValue::as_partial_date_time) + .is_some_and(|value| !value.has_time()); + let (params, value) = match tz { + _ if is_date => ( + vec![ICalendarParameter::value(ICalendarValueType::Date)], + PartialDateTime::from_date_timestamp(naive), + ), + Some(tz) if tz.is_utc() => (vec![], PartialDateTime::from_utc_timestamp(naive)), + tz => ( + tz.and_then(|tz| tz.name()) + .map(|tz_name| ICalendarParameter::tzid(tz_name.into_owned())) + .into_iter() + .collect(), + PartialDateTime::from_naive_timestamp(naive), + ), + }; + ICalendarEntry { + name, + params, + values: [ICalendarValue::PartialDateTime(value)].into(), + } + } + + fn with_date_time( + &self, + name: ICalendarProperty, + naive: i64, + fallback_tz: Option, + parameters: InstanceParameters, + ) -> ICalendarEntry { + let value = match self + .values + .first() + .and_then(ICalendarValue::as_partial_date_time) + { + Some(value) if !value.has_time() => PartialDateTime::from_date_timestamp(naive), + Some(value) if value.has_zone() => PartialDateTime::from_utc_timestamp(naive), + _ => PartialDateTime::from_naive_timestamp(naive), + }; + + ICalendarEntry { + name, + params: self + .params + .iter() + .filter(|param| match parameters { + InstanceParameters::All => param.name != ICalendarParameterName::Range, + InstanceParameters::TimeZone => matches!( + param.name, + ICalendarParameterName::Tzid | ICalendarParameterName::Value + ), + }) + .map(|param| match (¶m.name, ¶m.value) { + (ICalendarParameterName::Tzid, ICalendarParameterValue::Text(tz_id)) => { + Tz::from_str(tz_id) + .ok() + .or(fallback_tz) + .and_then(|tz| tz.name()) + .map_or_else( + || param.clone(), + |tz_name| ICalendarParameter::tzid(tz_name.into_owned()), + ) + } + _ => param.clone(), + }) + .collect(), + values: [ICalendarValue::PartialDateTime(value)].into(), + } + } +} + +pub(super) trait StableInstanceDuration { + fn has_stable_instance_duration(&self, comp_id: u32) -> bool; +} + +impl StableInstanceDuration for CalendarEventData { + fn has_stable_instance_duration(&self, comp_id: u32) -> bool { + let Some(component) = self.event.components.get(comp_id as usize) else { + return false; + }; + if component.property(&ICalendarProperty::Duration).is_some() + || component.property(&ICalendarProperty::Dtend).is_none() + { + return true; + } + + self.time_ranges + .iter() + .find(|range| range.id as u32 == comp_id) + .is_some_and(|range| { + [range.start_tz, range.end_tz] + .into_iter() + .all(|tz_id| Tz::from_id(tz_id).is_some_and(|tz| tz.has_fixed_offset())) + }) + } +} + +pub(super) trait EventInstanceBuilder { + fn instance( + &self, + expansion: &CalendarEventExpansion, + user_data: Option<&ICalendarComponent>, + binaries: InstanceBinaries, + ) -> Option; +} + +impl EventInstanceBuilder for CalendarEventData { + fn instance( + &self, + expansion: &CalendarEventExpansion, + user_data: Option<&ICalendarComponent>, + binaries: InstanceBinaries, + ) -> Option { + let source = self.event.components.get(expansion.comp_id as usize)?; + let is_recurrent = source.is_recurrent(); + let range = self + .time_ranges + .iter() + .find(|range| range.id as u32 == expansion.comp_id); + let start_tz = range.and_then(|range| Tz::from_id(range.start_tz)); + let end_tz = range.and_then(|range| Tz::from_id(range.end_tz)); + let mut dtstart = None; + let mut dtend = None; + let mut recurrence_id = None; + let mut has_duration = false; + let mut entries = source + .entries + .iter() + .filter(|entry| user_data.is_none() || !entry.is_user_data()) + .chain(user_data.into_iter().flat_map(|component| { + component + .entries + .iter() + .filter(|entry| entry.is_user_data()) + })) + .filter(|entry| binaries == InstanceBinaries::Keep || !entry.has_binary_value()) + .filter(|entry| match &entry.name { + ICalendarProperty::Dtstart => { + dtstart = Some(*entry); + false + } + ICalendarProperty::Dtend => { + dtend = Some(*entry); + false + } + ICalendarProperty::RecurrenceId => { + recurrence_id = Some(*entry); + false + } + ICalendarProperty::Exdate + | ICalendarProperty::Exrule + | ICalendarProperty::Rdate + | ICalendarProperty::Rrule => false, + ICalendarProperty::Due + | ICalendarProperty::Completed + | ICalendarProperty::Created => is_recurrent, + ICalendarProperty::Duration => { + has_duration = true; + true + } + _ => true, + }) + .cloned() + .collect::>(); + + entries.push(match (dtstart, recurrence_id) { + (Some(dtstart), _) => dtstart.with_date_time( + ICalendarProperty::Dtstart, + expansion.start_naive, + start_tz, + InstanceParameters::All, + ), + (None, Some(recurrence_id)) => recurrence_id.with_date_time_in( + ICalendarProperty::Dtstart, + expansion.start_naive, + start_tz, + ), + (None, None) => ICalendarEntry { + name: ICalendarProperty::Dtstart, + params: vec![], + values: [ICalendarValue::PartialDateTime( + PartialDateTime::from_naive_timestamp(expansion.start_naive), + )] + .into(), + }, + }); + + if is_recurrent || recurrence_id.is_some() { + let instance_id = expansion.recurrence_id(); + if let Some(entry) = recurrence_id + .map(|entry| (entry, InstanceParameters::All)) + .or_else(|| dtstart.map(|entry| (entry, InstanceParameters::TimeZone))) + { + entries.push(entry.0.with_date_time( + ICalendarProperty::RecurrenceId, + instance_id.naive, + start_tz, + entry.1, + )); + } + } + + let rebased_end = if !has_duration + && let Some(dtend) = dtend + && let Some(end) = dtend + .values + .first() + .and_then(ICalendarValue::as_partial_date_time) + .and_then(|value| value.to_date_time()) + && let Some(start) = dtstart + .and_then(|dtstart| dtstart.values.first()) + .and_then(ICalendarValue::as_partial_date_time) + .and_then(|value| value.to_date_time()) + { + Some(dtend.with_date_time( + ICalendarProperty::Dtend, + expansion.start_naive + end.date_time.duration_since(start.date_time).as_secs(), + end_tz, + InstanceParameters::All, + )) + } else { + None + }; + + if let Some(rebased_end) = rebased_end { + entries.push(rebased_end); + } else if !has_duration + && expansion.end > expansion.start + && !entries + .iter() + .any(|entry| entry.name == ICalendarProperty::Due) + { + let is_date = dtstart + .or(recurrence_id) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_partial_date_time) + .is_some_and(|value| !value.has_time()); + entries.push(ICalendarEntry { + name: ICalendarProperty::Duration, + params: vec![], + values: [ICalendarValue::Duration(if is_date { + ICalendarDuration::from_days( + (expansion.end_naive - expansion.start_naive) / SECONDS_PER_DAY, + ) + } else { + ICalendarDuration::from_seconds(expansion.end - expansion.start) + })] + .into(), + }); + } + + let mut components = Vec::with_capacity(2 + source.component_ids.len()); + components.push(ICalendarComponent { + component_type: ICalendarComponentType::VCalendar, + entries: vec![], + component_ids: vec![1], + }); + components.push(ICalendarComponent { + component_type: source.component_type.clone(), + entries, + component_ids: vec![], + }); + components.extend( + user_data + .unwrap_or(source) + .component_ids + .iter() + .filter_map(|component_id| self.event.components.get(*component_id as usize)) + .map(|component| ICalendarComponent { + component_type: component.component_type.clone(), + entries: if binaries == InstanceBinaries::Keep + || component.component_type == ICalendarComponentType::VAlarm + { + component.entries.clone() + } else { + component + .entries + .iter() + .filter(|entry| !entry.has_binary_value()) + .cloned() + .collect() + }, + component_ids: vec![], + }), + ); + let child_ids = (2..components.len() as u32).collect(); + if let Some(instance) = components.get_mut(1) { + instance.component_ids = child_ids; + } + + Some(ICalendar { components }) + } +} + +trait BinaryEntry { + fn has_binary_value(&self) -> bool; +} + +impl BinaryEntry for ICalendarEntry { + fn has_binary_value(&self) -> bool { + self.values + .iter() + .any(|value| matches!(value, ICalendarValue::Binary(_))) + } +} + +trait StripBinaryEntries { + fn strip_binary_entries(&mut self); +} + +impl StripBinaryEntries for ICalendar { + fn strip_binary_entries(&mut self) { + for component in self + .components + .iter_mut() + .filter(|component| component.component_type != ICalendarComponentType::VAlarm) + { + component.entries.retain(|entry| !entry.has_binary_value()); + } + } +} + +trait PrivateViewProperty { + fn is_private_view_property(&self) -> bool; +} + +impl PrivateViewProperty for JSCalendarProperty { + fn is_private_view_property(&self) -> bool { + matches!( + self, + JSCalendarProperty::Id + | JSCalendarProperty::BaseEventId + | JSCalendarProperty::CalendarIds + | JSCalendarProperty::IsDraft + | JSCalendarProperty::IsOrigin + | JSCalendarProperty::UtcStart + | JSCalendarProperty::UtcEnd + ) + } +} + diff --git a/crates/jmap/src/calendar_event/hidden.rs b/crates/jmap/src/calendar_event/hidden.rs new file mode 100644 index 00000000..84a26f59 --- /dev/null +++ b/crates/jmap/src/calendar_event/hidden.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + blob::embedded::EmbeddedBlobIds, + calendar_event::{EventValue, JSCalendarEntries}, +}; +use calcard::{ + icalendar::ICalendar, + jscalendar::{JSCalendar, JSCalendarProperty, import::ImportOptions}, +}; +use groupware::calendar::{ + hidden::{HiddenAdditions, HiddenParticipantsMerge}, + identity::CalendarAddresses, + participants::VisibleParticipants, +}; +use jmap_tools::{Key, Value}; +use types::{blob::BlobId, collection::Collection, id::Id}; + +pub(super) struct HiddenParticipants<'x> { + full: JSCalendar<'x, Id, BlobId>, +} + +impl<'x> HiddenParticipants<'x> { + pub fn hide(group: &mut JSCalendar<'x, Id, BlobId>, identities: &CalendarAddresses) -> Self { + let full = group.clone(); + for entry in group.entries_mut() { + entry.retain_visible_participants(identities); + } + HiddenParticipants { full } + } + + pub fn merge( + mut self, + before: &JSCalendar<'x, Id, BlobId>, + after: JSCalendar<'x, Id, BlobId>, + ) -> (JSCalendar<'x, Id, BlobId>, HiddenAdditions) { + let mut additions = HiddenAdditions::default(); + let before_entries = before + .0 + .as_object_and_get(&Key::Property(JSCalendarProperty::Entries)) + .and_then(Value::as_array) + .unwrap_or_default(); + let after_entries = after + .0 + .into_object() + .and_then(|mut group| group.remove(&Key::Property(JSCalendarProperty::Entries))) + .and_then(Value::into_array) + .unwrap_or_default(); + for ((full_entry, before_entry), after_entry) in self + .full + .entries_mut() + .iter_mut() + .zip(before_entries) + .zip(after_entries) + { + let entry_additions = full_entry.merge_visible_changes(before_entry, after_entry); + additions.base |= entry_additions.base; + additions.overrides |= entry_additions.overrides; + additions.ignored += entry_additions.ignored; + } + (self.full, additions) + } +} + +pub(super) fn visible_overrides( + view: ICalendar, + identities: &CalendarAddresses, + account_id: u32, + document_id: u32, +) -> EventValue<'static> { + view.into_jscalendar_with::( + ImportOptions::new() + .include_ical_components(false) + .return_first(true) + .with_blob_id_generator(EmbeddedBlobIds::new( + account_id, + Collection::CalendarEvent, + document_id, + )), + ) + .ok() + .and_then(|event| { + let mut event = event.into_inner(); + event.retain_visible_participants(identities); + event.into_object() + }) + .and_then(|mut event| event.remove(&Key::Property(JSCalendarProperty::RecurrenceOverrides))) + .unwrap_or(Value::Null) +} diff --git a/crates/jmap/src/calendar_event/mod.rs b/crates/jmap/src/calendar_event/mod.rs index ff4ec864..6bdb7efa 100644 --- a/crates/jmap/src/calendar_event/mod.rs +++ b/crates/jmap/src/calendar_event/mod.rs @@ -4,33 +4,44 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use calcard::jscalendar::JSCalendarProperty; -use common::Server; -use groupware::calendar::expand::RecurrenceKey; +use calcard::{ + common::timezone::Tz, + icalendar::{ + ArchivedICalendar, ArchivedICalendarComponent, ICalendar, ICalendarComponent, + ICalendarComponentType, ICalendarProperty, ICalendarValue, + }, + jscalendar::{JSCalendar, JSCalendarProperty, JSCalendarType, JSCalendarValue}, +}; +use common::{DavName, GroupwareResources, Server, storage::dav::CachedUid}; +use groupware::calendar::{ + CalendarEventContent, + expand::{ComponentRecurrenceId, RecurrenceKey}, + index::ICalendarObjectUid, +}; use jmap_proto::error::set::SetError; +use jmap_tools::{Key, Map, Value}; +use std::borrow::Cow; +use store::{ + ValueKey, + ahash::{AHashMap, AHashSet}, + write::{Archive, ArchiveBytes}, +}; use trc::AddContext; -use types::{collection::Collection, field::CalendarEventField, id::Id}; +use types::{blob::BlobId, collection::Collection, field::CalendarEventField, id::Id}; + +pub(super) type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +pub(super) type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; pub mod copy; pub mod get; +pub mod hidden; pub mod parse; +pub mod privacy; pub mod query; +pub mod server_set; pub mod set; - -/* - -TODO: Not yet implemented: - -- CalendarEvent - - Per-user properties (However, the database schema is ready to support this) - - mayInviteSelf, mayInviteOthers and hideAttendees (stored but not enforced) - -- Principal/getAvailability - - If there are overlapping BusyPeriod time ranges with different "busyStatus" properties - the server MUST choose the value in the following order: confirmed > unavailable > tentative. - - Return event properties - -*/ +pub mod user; +pub mod validate; pub trait CalendarSyntheticId { fn new(key: RecurrenceKey, document_id: u32) -> Self; @@ -54,28 +65,316 @@ impl CalendarSyntheticId for Id { } } -pub(super) async fn assert_is_unique_uid( - server: &Server, - account_id: u32, - uid: Option<&str>, -) -> trc::Result>>> { - if let Some(uid) = uid - && server - .document_exists( - account_id, - Collection::CalendarEvent, - CalendarEventField::Uid, - uid.as_bytes(), - ) - .await - .caused_by(trc::location!())? - { - Ok(Err(SetError::invalid_properties() - .with_property(JSCalendarProperty::Uid) - .with_description(format!( - "An event with UID {uid} already exists.", - )))) - } else { +enum UidEntry { + Stored { + document_id: u32, + calendar_ids: Vec, + }, + Created { + calendar_ids: Vec, + recurrence_ids: Vec, + }, +} + +#[derive(Default)] +pub struct UidIndex<'x>(AHashMap, Vec>); + +impl<'x> UidIndex<'x> { + pub fn new<'y>( + resources: &'x GroupwareResources, + cached_uids: impl IntoIterator>, + ) -> Self { + let uids = cached_uids.into_iter().collect::>(); + let mut index = AHashMap::with_capacity(uids.len()); + if !uids.is_empty() { + for resource in resources.resources.iter_run(false) { + if let Some(uid) = resource.uid().filter(|uid| uids.contains(*uid)) { + index + .entry(Cow::Borrowed(uid)) + .or_insert_with(Vec::new) + .push(UidEntry::Stored { + document_id: resource.document_id(), + calendar_ids: resource + .child_names() + .iter() + .map(|name| name.parent_id) + .collect(), + }); + } + } + } + UidIndex(index) + } + + fn documents(&self, uid: &str) -> &[UidEntry] { + self.0 + .get(uid.cached_uid().as_ref()) + .map_or(&[], Vec::as_slice) + } + + pub fn record(&mut self, ical: &ICalendar, calendar_ids: &[u32]) { + let Some(uid) = ical.object_uid().map(CachedUid::cached_uid) else { + return; + }; + let entry = UidEntry::Created { + calendar_ids: calendar_ids.to_vec(), + recurrence_ids: ical.instance_recurrence_ids(), + }; + match self.0.get_mut(uid.as_ref()) { + Some(entries) => entries.push(entry), + None => { + self.0.insert(Cow::Owned(uid.into_owned()), vec![entry]); + } + } + } + + pub async fn assert_is_unique( + &self, + server: &Server, + account_id: u32, + ical: &ICalendar, + names: &[DavName], + document_id: Option, + will_destroy: &[Id], + ) -> trc::Result>>> { + let Some(uid) = ical.object_uid() else { + return Ok(Ok(())); + }; + let instances = ical.instance_recurrence_ids(); + let is_distinct_instance = |recurrence_ids: &[i64]| { + !instances.is_empty() + && !recurrence_ids.is_empty() + && !recurrence_ids + .iter() + .any(|recurrence_id| instances.contains(recurrence_id)) + }; + + for entry in self.documents(uid) { + let (calendar_ids, is_distinct) = match entry { + UidEntry::Created { + calendar_ids, + recurrence_ids, + } => (calendar_ids, is_distinct_instance(recurrence_ids)), + UidEntry::Stored { + document_id: stored_id, + calendar_ids, + } => { + if document_id == Some(*stored_id) + || will_destroy + .iter() + .any(|id| !id.is_synthetic() && id.document_id() == *stored_id) + { + continue; + } + let stored_content = server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + *stored_id, + CalendarEventField::Content, + )) + .await?; + let recurrence_ids = stored_content + .as_ref() + .map(|content| content.unarchive::()) + .transpose() + .caused_by(trc::location!())? + .map(|content| content.data.event.instance_recurrence_ids()) + .unwrap_or_default(); + (calendar_ids, is_distinct_instance(&recurrence_ids)) + } + }; + + if !is_distinct { + return Ok(Err(duplicate_uid(uid))); + } + + if let Some(calendar_id) = names + .iter() + .map(DavName::parent_id) + .find(|calendar_id| calendar_ids.contains(calendar_id)) + { + return Ok(Err(shared_uid_calendar(uid, calendar_id))); + } + } + Ok(Ok(())) } } + +fn duplicate_uid(uid: &str) -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::Uid) + .with_description(format!("An event with UID {uid} already exists.")) +} + +fn shared_uid_calendar(uid: &str, calendar_id: u32) -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::CalendarIds) + .with_description(format!( + concat!( + "Another occurrence of UID {} is already stored in calendar {}. ", + "Events that share a UID have to be in different calendars." + ), + uid, + Id::from(calendar_id) + )) +} + +pub(super) trait JSCalendarEntries<'x> { + fn new_event() -> Self; + + fn first_entry(&self) -> Option<&EventMap<'x>>; + + fn first_entry_mut(&mut self) -> Option<&mut EventValue<'x>>; + + fn entries_mut(&mut self) -> &mut [EventValue<'x>]; +} + +impl<'x> JSCalendarEntries<'x> for JSCalendar<'x, Id, BlobId> { + fn new_event() -> Self { + JSCalendar(Value::Object(Map::from(vec![ + ( + Key::Property(JSCalendarProperty::Type), + Value::Element(JSCalendarValue::Type(JSCalendarType::Group)), + ), + ( + Key::Property(JSCalendarProperty::Entries), + Value::Array(vec![Value::Object(Map::new())]), + ), + ]))) + } + + fn first_entry(&self) -> Option<&EventMap<'x>> { + self.0 + .as_object_and_get(&Key::Property(JSCalendarProperty::Entries)) + .and_then(Value::as_array) + .and_then(|entries| entries.first()) + .and_then(Value::as_object) + } + + fn first_entry_mut(&mut self) -> Option<&mut EventValue<'x>> { + self.0 + .as_object_mut()? + .get_mut(&Key::Property(JSCalendarProperty::Entries))? + .as_array_mut()? + .first_mut() + .filter(|entry| entry.as_object().is_some()) + } + + fn entries_mut(&mut self) -> &mut [EventValue<'x>] { + self.0 + .as_object_mut() + .and_then(|group| group.get_mut(&Key::Property(JSCalendarProperty::Entries))) + .and_then(Value::as_array_mut) + .map_or(&mut [], Vec::as_mut_slice) + } +} + +pub(super) fn is_origin(ical: &ICalendar, addresses: &[String]) -> bool { + ical.main_component() + .and_then(|component| component.property(&ICalendarProperty::Organizer)) + .and_then(|entry| entry.calendar_address()) + .is_none_or(|organizer| { + addresses + .iter() + .any(|address| address.eq_ignore_ascii_case(organizer)) + }) +} + +pub(super) trait EventMainComponent { + type Component: ComponentRecurrenceId; + + fn scheduling_components(&self) -> impl Iterator; + + fn is_override(component: &Self::Component) -> bool; + + fn main_component(&self) -> Option<&Self::Component> { + let mut components = self.scheduling_components(); + let first = components.next()?; + if Self::is_override(first) { + Some( + components + .find(|component| !Self::is_override(component)) + .unwrap_or(first), + ) + } else { + Some(first) + } + } + + fn instance_recurrence_ids(&self) -> Vec { + let mut recurrence_ids = Vec::new(); + for component in self.scheduling_components() { + if !Self::is_override(component) { + return Vec::new(); + } else if let Some(recurrence_id) = component.recurrence_id(Tz::Floating) { + recurrence_ids.push(recurrence_id.utc); + } + } + recurrence_ids + } + + fn instance_component(&self, recurrence_id: i64) -> Option<&Self::Component> { + self.scheduling_components() + .find(|component| { + Self::is_override(component) + && component + .recurrence_id(Tz::Floating) + .is_some_and(|id| id.utc == recurrence_id) + }) + .or_else(|| self.main_component()) + } +} + +impl EventMainComponent for ICalendar { + type Component = ICalendarComponent; + + fn scheduling_components(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + } + + fn is_override(component: &ICalendarComponent) -> bool { + component.is_recurrence_override() + } +} + +impl EventMainComponent for ArchivedICalendar { + type Component = ArchivedICalendarComponent; + + fn scheduling_components(&self) -> impl Iterator { + self.components + .iter() + .filter(|component| component.component_type.is_scheduling_object()) + } + + fn is_override(component: &ArchivedICalendarComponent) -> bool { + component.is_recurrence_override() + } +} + +pub(super) fn detach_iana_timezones(ical: &mut ICalendar) { + let Some((root, components)) = ical.components.split_first_mut() else { + return; + }; + if root.component_type != ICalendarComponentType::VCalendar { + return; + } + root.component_ids.retain(|component_id| { + component_id + .checked_sub(1) + .and_then(|component_id| components.get(component_id as usize)) + .is_none_or(|component| { + component.component_type != ICalendarComponentType::VTimezone + || !component + .property(&ICalendarProperty::Tzid) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_text) + .is_some_and(|tz_id| Tz::iana(tz_id).is_some()) + }) + }); +} + diff --git a/crates/jmap/src/calendar_event/parse.rs b/crates/jmap/src/calendar_event/parse.rs index b6507aa0..e28a016c 100644 --- a/crates/jmap/src/calendar_event/parse.rs +++ b/crates/jmap/src/calendar_event/parse.rs @@ -7,7 +7,7 @@ use crate::blob::download::BlobDownload; use calcard::{ icalendar::ICalendar, - jscalendar::{JSCalendarProperty, import::ConversionOptions}, + jscalendar::{JSCALENDAR_VERSION, JSCalendar, JSCalendarProperty, import::ImportOptions}, }; use common::{Server, auth::AccessToken}; use jmap_proto::{ @@ -36,11 +36,12 @@ impl CalendarEventParse for Server { if request.blob_ids.len() > self.core.jmap.calendar_parse_max_items { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } - let return_all_properties = request.properties.is_none(); let properties = request .properties - .map(|v| v.into_valid().collect::>()) - .unwrap_or_default(); + .map(|v| v.into_valid().collect::>()); + let return_version = properties + .as_ref() + .is_none_or(|properties| properties.contains(&JSCalendarProperty::Version)); let mut response = ParseResponse { account_id: request.account_id, @@ -51,36 +52,57 @@ impl CalendarEventParse for Server { for blob_id in request.blob_ids.into_valid() { // Fetch raw message to parse - let raw_vcard = match self.blob_download(&blob_id, access_token).await? { - Some(raw_vcard) => raw_vcard, + let raw_ical = match self.blob_download(&blob_id, access_token).await? { + Some(raw_ical) => raw_ical, None => { response.not_found.push(MaybeInvalid::Value(blob_id)); continue; } }; - let Ok(vcard) = ICalendar::parse(std::str::from_utf8(&raw_vcard).unwrap_or_default()) + let Ok(ical) = ICalendar::parse(std::str::from_utf8(&raw_ical).unwrap_or_default()) + else { + response.not_parsable.push(blob_id); + continue; + }; + let Some(Value::Array(mut js_calendar_entries)) = ical + .into_jscalendar_with::(ImportOptions::new()) + .ok() + .map(JSCalendar::into_inner) + .and_then(Value::into_object) + .and_then(|mut group| group.remove(&Key::Property(JSCalendarProperty::Entries))) + .filter(|entries| { + entries.as_array().is_some_and(|entries| { + !entries.is_empty() + && entries.iter().all(|entry| entry.as_object().is_some()) + }) + }) else { response.not_parsable.push(blob_id); continue; }; - let mut js_calendar_entries = vcard - .into_jscalendar_with_opt::(ConversionOptions::default()) - .into_inner() - .into_object() - .unwrap() - .remove(&Key::Property(JSCalendarProperty::Entries)) - .unwrap() - .into_array() - .unwrap(); - if !return_all_properties { - for entry in &mut js_calendar_entries { + for entry in js_calendar_entries + .iter_mut() + .filter_map(Value::as_object_mut) + { + if let Some(properties) = &properties { entry - .as_object_mut() - .unwrap() .as_mut_vec() .retain(|(k, _)| k.as_property().is_some_and(|k| properties.contains(k))); } + if return_version { + entry.insert( + Key::Property(JSCalendarProperty::Version), + Value::Str(JSCALENDAR_VERSION.into()), + ); + } + for property in METADATA_PROPERTIES.iter().filter(|property| { + properties + .as_ref() + .is_none_or(|properties| properties.contains(property)) + }) { + entry.insert(Key::Property(property.clone()), Value::Null); + } } response @@ -91,3 +113,11 @@ impl CalendarEventParse for Server { Ok(response) } } + +const METADATA_PROPERTIES: [JSCalendarProperty; 5] = [ + JSCalendarProperty::Id, + JSCalendarProperty::BaseEventId, + JSCalendarProperty::CalendarIds, + JSCalendarProperty::IsDraft, + JSCalendarProperty::IsOrigin, +]; diff --git a/crates/jmap/src/calendar_event/privacy.rs b/crates/jmap/src/calendar_event/privacy.rs new file mode 100644 index 00000000..4f1c1196 --- /dev/null +++ b/crates/jmap/src/calendar_event/privacy.rs @@ -0,0 +1,173 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::calendar_event::CalendarSyntheticId; +use calcard::{ + icalendar::ICalendar, + jscalendar::{JSCalendarPrivacy, JSCalendarProperty, JSCalendarValue}, +}; +use common::{GroupwareResources, storage::dav::CachedUid}; +use groupware::calendar::privacy::{EventPrivacy, EventViewer, ICalendarPrivacy, PrivacyDenied}; +use jmap_proto::{error::set::SetError, request::MaybeInvalid}; +use jmap_tools::{Key, Value}; +use std::borrow::Cow; +use store::{ + ahash::{AHashMap, AHashSet}, + roaring::RoaringBitmap, +}; +use types::{blob::BlobId, id::Id}; +use utils::map::vec_map::VecMap; + +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; + +pub(super) fn assert_privacy_access( + privacy: EventPrivacy, + viewer: EventViewer, +) -> Result<(), SetError>> { + privacy.check_access(viewer).map_err(|denied| match denied { + PrivacyDenied::Forbidden => SetError::forbidden() + .with_description("Private events can only be modified by the calendar owner."), + PrivacyDenied::NotFound => SetError::not_found(), + }) +} + +pub(super) fn assert_privacy_allowed( + ical: &ICalendar, + viewer: EventViewer, +) -> Result<(), SetError>> { + if ical.privacy().may_be_set_by(viewer) { + Ok(()) + } else { + Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::Privacy) + .with_description("Only the calendar owner can set a privacy other than public.")) + } +} + +pub(super) fn uid_privacy_conflict() -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::Privacy) + .with_description("All events with the same UID must have the same privacy.") +} + +#[derive(Default)] +pub(super) struct UidPrivacyConflicts { + pub creates: AHashSet, + pub updates: RoaringBitmap, +} + +impl UidPrivacyConflicts { + pub fn new( + cache: &GroupwareResources, + creates: Option<&VecMap>>, + updates: Option<&VecMap, EventValue<'_>>>, + will_destroy: &[Id], + ) -> Self { + let mut touched: AHashMap, UidEvents<'_>> = AHashMap::new(); + + for (create_id, object) in creates.into_iter().flat_map(|creates| creates.iter()) { + if let Some(Value::Str(uid)) = object_property(object, JSCalendarProperty::Uid) { + let events = touched.entry(uid.cached_uid()).or_default(); + events.create_ids.push(create_id); + events + .create_privacies + .push(patched_privacy(object).unwrap_or_default()); + } + } + + let mut patched: AHashMap = AHashMap::new(); + for (id, object) in updates.into_iter().flat_map(|updates| updates.iter()) { + if let MaybeInvalid::Value(id) = id + && !id.is_synthetic() + && let Some(privacy) = patched_privacy(object) + && let Some(uid) = cache + .resources + .find(id.document_id(), false) + .and_then(|resource| resource.uid()) + { + patched.insert(id.document_id(), privacy); + touched.entry(Cow::Borrowed(uid)).or_default(); + } + } + + let mut conflicts = UidPrivacyConflicts::default(); + if touched.is_empty() { + return conflicts; + } + for resource in cache.resources.iter_run(false) { + if let Some(events) = resource.uid().and_then(|uid| touched.get_mut(uid)) { + events.documents.push(( + resource.document_id(), + resource + .event_flags() + .map(EventPrivacy::from_flags) + .unwrap_or_default(), + )); + } + } + + for events in touched.into_values() { + let mut privacies = events.create_privacies.iter().copied().chain( + events + .documents + .iter() + .filter(|(document_id, _)| { + !will_destroy + .iter() + .any(|id| !id.is_synthetic() && id.document_id() == *document_id) + }) + .map(|(document_id, privacy)| { + patched.get(document_id).copied().unwrap_or(*privacy) + }), + ); + if privacies + .next() + .is_some_and(|first| privacies.any(|privacy| privacy != first)) + { + conflicts + .creates + .extend(events.create_ids.into_iter().cloned()); + conflicts.updates.extend( + events + .documents + .iter() + .map(|(document_id, _)| *document_id) + .filter(|document_id| patched.contains_key(document_id)), + ); + } + } + + conflicts + } +} + +#[derive(Default)] +struct UidEvents<'x> { + create_ids: Vec<&'x String>, + create_privacies: Vec, + documents: Vec<(u32, EventPrivacy)>, +} + +fn object_property<'x, 'y>( + object: &'y EventValue<'x>, + property: JSCalendarProperty, +) -> Option<&'y EventValue<'x>> { + object + .as_object() + .and_then(|object| object.get(&Key::Property(property))) +} + +fn patched_privacy(object: &EventValue<'_>) -> Option { + match object_property(object, JSCalendarProperty::Privacy)? { + Value::Element(JSCalendarValue::Privacy(privacy)) => Some(match privacy { + JSCalendarPrivacy::Public => EventPrivacy::Public, + JSCalendarPrivacy::Private => EventPrivacy::Private, + JSCalendarPrivacy::Secret => EventPrivacy::Secret, + }), + Value::Null => Some(EventPrivacy::Public), + _ => None, + } +} diff --git a/crates/jmap/src/calendar_event/query.rs b/crates/jmap/src/calendar_event/query.rs index 8de01ddb..799515c2 100644 --- a/crates/jmap/src/calendar_event/query.rs +++ b/crates/jmap/src/calendar_event/query.rs @@ -4,36 +4,53 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; -use calcard::{common::timezone::Tz, jscalendar::JSCalendarDateTime}; -use chrono::offset::TimeZone; -use common::{Server, auth::AccessToken}; +use crate::{ + api::{ + metadata::{ + MetadataType, ObjectMetadata, ResourceScope, filter_containers, flagged_documents, + }, + query::QueryResponseBuilder, + }, + changes::state::JmapCacheState, +}; +use calcard::{common::timezone::Tz, icalendar::ArchivedICalendar, jscalendar::JSCalendarDateTime}; +use common::{ + GroupwareResourceRef, GroupwareResources, Server, auth::AccessToken, + storage::batch::MAX_BATCH_DOCUMENTS, +}; use groupware::{ cache::GroupwareCache, - calendar::{CalendarEvent, expand::RecurrenceKey}, + calendar::{ + CalendarEventContent, EVENT_FLOATING_START, EVENT_HIDE_ATTENDEES, EVENT_PRIVATE, + EVENT_SECRET, + expand::{MAX_UTC_OFFSET, resolve_local}, + identity::{CalendarAddresses, ParticipantIdentityAddresses}, + instance_filter::{AttendeeSearch, InstanceTextFilter}, + }, }; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::{ - calendar, + calendar::{self, CalendarFilter}, calendar_event::{self, CalendarEventComparator, CalendarEventFilter}, }, - request::MaybeInvalid, - types::state::State, + request::{MaybeInvalid, capability::CapabilityIds}, }; use nlp::language::Language; -use std::{cmp::Ordering, sync::Arc}; +use std::cmp::Ordering; use store::{ ValueKey, + ahash::AHashMap, roaring::RoaringBitmap, - search::{CalendarSearchField, SearchComparator, SearchFilter, SearchQuery}, - write::{AlignedBytes, Archive, SearchIndex}, + search::{CalendarSearchField, QueryResults, SearchComparator, SearchFilter, SearchQuery}, + write::{Archive, ArchiveBytes, SearchIndex}, }; use trc::AddContext; use types::{ - TimeRange, + OverlapRule, TimeRange, acl::Acl, collection::{Collection, SyncCollection}, + field::CalendarEventField, }; pub trait CalendarEventQuery: Sync + Send { @@ -41,12 +58,14 @@ pub trait CalendarEventQuery: Sync + Send { &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; fn calendar_query( &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -55,18 +74,156 @@ impl CalendarEventQuery for Server { &self, mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::CalendarEvent); + let sampled = metadata.viewer_change_id(self, account_id).await?; + let metadata_query = + metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(CalendarEventFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; let mut filters = Vec::with_capacity(request.filter.len()); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await?; - let default_tz = request.arguments.time_zone.unwrap_or(Tz::UTC); + let default_tz = request.arguments.resolved_time_zone()?; + let default_language = self.core.email.default_language; + let is_member = access_token.is_member(account_id); + let expand_recurrences = request.arguments.expand_recurrences.unwrap_or(false); let mut filter: Option = None; + let mut instance_filter = InstanceTextFilter::default(); + + if expand_recurrences + && (request + .filter + .iter() + .any(|cond| matches!(cond, Filter::Or | Filter::Not)) + || request + .filter + .iter() + .filter(|cond| matches!(cond, Filter::And)) + .count() + > 1) + { + return Err(trc::JmapEvent::InvalidArguments.into_err().details( + "The filter must be a single FilterCondition when expanding recurrences", + )); + } + + if expand_recurrences + && let Some(unsupported) = + request + .sort + .iter() + .flatten() + .find_map(|comparator| match &comparator.property { + CalendarEventComparator::_T(property) => Some(property), + _ => None, + }) + { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(unsupported.clone())); + } + + let is_shared = access_token.is_shared(account_id); + let hides_text = !is_member + && request.filter.iter().any(|cond| { + matches!( + cond, + Filter::Property( + CalendarEventFilter::Text(_) + | CalendarEventFilter::Title(_) + | CalendarEventFilter::Description(_) + | CalendarEventFilter::Location(_) + | CalendarEventFilter::Owner(_) + | CalendarEventFilter::Participant(_) + ) + ) + }); + let [secret_ids, private_ids, hidden_attendee_ids] = if is_shared || hides_text { + cache.event_ids_with_flag_masks([ + EVENT_SECRET, + EVENT_PRIVATE | EVENT_SECRET, + EVENT_HIDE_ATTENDEES, + ]) + } else { + std::array::from_fn(|_| RoaringBitmap::new()) + }; + let mask = if is_shared { + let mut shared_ids = cache.shared_items(access_token, [Acl::ReadItems], true); + shared_ids -= secret_ids; + shared_ids + } else { + cache.document_ids(false).collect() + }; + let metadata_hidden = + (is_shared && !metadata_query.is_empty()).then(|| private_ids.clone()); + let searches_attendees = !is_member + && request.filter.iter().any(|cond| { + matches!( + cond, + Filter::Property( + CalendarEventFilter::Text(_) | CalendarEventFilter::Participant(_) + ) + ) + }); + let hidden_attendees = searches_attendees + .then(|| HiddenAttendees::new(&cache, access_token, &hidden_attendee_ids)) + .flatten(); + let identities = match &hidden_attendees { + Some(_) => Some( + self.account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())?, + ), + None => None, + }; + let visibility = if hides_text { + TextVisibility::new( + &cache, + private_ids, + hidden_attendees + .as_ref() + .filter(|_| !expand_recurrences) + .map(|hidden| hidden.filtered.clone()) + .unwrap_or_default(), + ) + } else { + TextVisibility::default() + }; + let attendee_matches = match (&hidden_attendees, &identities) { + (Some(hidden), Some(identities)) if !expand_recurrences => Some(VisibleAttendees { + hidden, + identities, + account_id, + mask: &mask, + default_language, + }), + _ => None, + }; + let mut metadata_leaves = metadata + .evaluate(self, account_id, &metadata_query, || { + let mut flagged = + flagged_documents(&cache, ResourceScope::Items, is_shared.then_some(&mask)); + if let Some(hidden) = &metadata_hidden { + flagged -= hidden; + } + flagged + }) + .await? + .into_iter(); + let candidates = if expand_recurrences { + BoundCandidates::Widened + } else { + BoundCandidates::Exact + }; // Extract from/to arguments for cond in &request.filter { @@ -89,105 +246,111 @@ impl CalendarEventQuery for Server { cache.children_ids(id.document_id()), ))) } - CalendarEventFilter::Uid(uid) => { - filters.push(SearchFilter::eq(CalendarSearchField::Uid, uid)); + CalendarEventFilter::Uid(value) => { + filters.push(SearchFilter::is_in_set(cache.uid_matches(&value))); } CalendarEventFilter::Text(value) => { - let (text, language) = - Language::detect(value, self.core.email.default_language); - filters.push(SearchFilter::Or); - filters.push(SearchFilter::has_text( - CalendarSearchField::Title, - text.clone(), - language, - )); - filters.push(SearchFilter::has_text( - CalendarSearchField::Description, - text.clone(), - language, - )); - filters.push(SearchFilter::has_text( - CalendarSearchField::Location, - text.clone(), - language, - )); - filters.push(SearchFilter::has_text( - CalendarSearchField::Owner, - text.clone(), - language, - )); - filters.push(SearchFilter::has_text( - CalendarSearchField::Attendee, - text, - language, - )); - filters.push(SearchFilter::End); + let (text, language) = Language::detect(value, default_language); + let filter = + SearchFilter::has_text(CalendarSearchField::Title, text, language); + if expand_recurrences { + instance_filter.add(TEXT_FIELDS, &filter); + } + let verified = match &attendee_matches { + Some(matches) => Some(matches.of(self, &filter).await?), + None => None, + }; + visibility.push_any_field(&mut filters, filter, TEXT_FIELDS, verified); } CalendarEventFilter::Title(title) => { - filters.push(SearchFilter::has_text_detect( + let filter = SearchFilter::has_text_detect( CalendarSearchField::Title, title, - self.core.email.default_language, - )); + default_language, + ); + if expand_recurrences { + instance_filter.add(&[CalendarSearchField::Title], &filter); + } + visibility.push(&mut filters, filter); } CalendarEventFilter::Description(description) => { - filters.push(SearchFilter::has_text_detect( + let filter = SearchFilter::has_text_detect( CalendarSearchField::Description, description, - self.core.email.default_language, - )); + default_language, + ); + if expand_recurrences { + instance_filter.add(&[CalendarSearchField::Description], &filter); + } + visibility.push(&mut filters, filter); } CalendarEventFilter::Location(location) => { - filters.push(SearchFilter::has_text_detect( + let filter = SearchFilter::has_text_detect( CalendarSearchField::Location, location, - self.core.email.default_language, - )); + default_language, + ); + if expand_recurrences { + instance_filter.add(&[CalendarSearchField::Location], &filter); + } + visibility.push(&mut filters, filter); } CalendarEventFilter::Owner(owner) => { - filters.push(SearchFilter::has_text( + let filter = SearchFilter::has_text( CalendarSearchField::Owner, owner, Language::None, - )); + ); + if expand_recurrences { + instance_filter.add(&[CalendarSearchField::Owner], &filter); + } + visibility.push(&mut filters, filter); } - CalendarEventFilter::Attendee(attendee) => { - filters.push(SearchFilter::has_text( - CalendarSearchField::Attendee, - attendee, + CalendarEventFilter::Participant(participant) => { + let filter = SearchFilter::has_text( + CalendarSearchField::Owner, + participant, Language::None, - )); + ); + if expand_recurrences { + instance_filter.add(PARTICIPANT_FIELDS, &filter); + } + let verified = match &attendee_matches { + Some(matches) => Some(matches.of(self, &filter).await?), + None => None, + }; + visibility.push_any_field( + &mut filters, + filter, + PARTICIPANT_FIELDS, + verified, + ); } CalendarEventFilter::After(after) => { - /* - The end of the event, or any recurrence of the event, in the time zone given - as the "timeZone" argument, must be after this date to match the condition. - */ if let Some(after) = local_timestamp(&after, default_tz) { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - r.event_time_range() - .and_then(|(_, end)| (after < end).then_some(r.document_id)) - }), - ))); + filters.push(SearchFilter::is_in_set( + TimeBound::After(after) + .event_ids(self, &cache, account_id, default_tz, candidates) + .await?, + )); } } CalendarEventFilter::Before(before) => { - /* - The start of the event, or any recurrence of the event, in the time zone given - as the "timeZone" argument, must be before this date to match the condition. - */ - if let Some(before) = local_timestamp(&before, default_tz) { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - r.event_time_range().and_then(|(start, _)| { - (before > start).then_some(r.document_id) - }) - }), - ))); + filters.push(SearchFilter::is_in_set( + TimeBound::Before(before) + .event_ids(self, &cache, account_id, default_tz, candidates) + .await?, + )); } } + CalendarEventFilter::Metadata(_) => { + let mut matches = metadata_leaves.next().unwrap_or_default(); + if let Some(hidden) = &metadata_hidden { + matches -= hidden; + } + filters.push(SearchFilter::is_in_set(matches)); + } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() @@ -209,32 +372,42 @@ impl CalendarEventQuery for Server { } } - let expand_recurrences = request.arguments.expand_recurrences.unwrap_or(false); - let comparators = if !expand_recurrences { + let expand_range = if expand_recurrences { + let Some(time_range) = filter.filter(|f| f.start != i64::MIN && f.end != i64::MAX) + else { + return Err(trc::JmapEvent::InvalidArguments.into_err().details( + "Both 'after' and 'before' filters are required when expanding recurrences", + )); + }; + if time_range.end.saturating_sub(time_range.start) + > self.core.groupware.max_expanded_query_duration as i64 + { + return Err(trc::JmapEvent::ExpandDurationTooLarge.into_err().details( + "The time span between 'after' and 'before' exceeds maxExpandedQueryDuration", + )); + } + Some(time_range) + } else { + None + }; + let sort = if !expand_recurrences { request .sort .take() .unwrap_or_default() .into_iter() - .map(|comparator| match comparator.property { - CalendarEventComparator::Start | CalendarEventComparator::RecurrenceId => { - Ok(SearchComparator::field( - CalendarSearchField::Start, - comparator.is_ascending, - )) - } - CalendarEventComparator::Uid => Ok(SearchComparator::field( - CalendarSearchField::Uid, - comparator.is_ascending, - )), - CalendarEventComparator::Created | CalendarEventComparator::Updated => { - Err(trc::JmapEvent::UnsupportedSort - .into_err() - .details(comparator.property.into_string().into_owned())) - } - CalendarEventComparator::_T(other) => Err(trc::JmapEvent::UnsupportedSort - .into_err() - .details(other.to_string())), + .map(|comparator| { + let key = match comparator.property { + CalendarEventComparator::Id => EventSortKey::Id, + CalendarEventComparator::Start => EventSortKey::Start, + CalendarEventComparator::Uid => EventSortKey::Uid, + CalendarEventComparator::Created => EventSortKey::Created, + CalendarEventComparator::Updated => EventSortKey::Updated, + CalendarEventComparator::_T(other) => { + return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); + } + }; + Ok((key, comparator.is_ascending)) }) .collect::, _>>()? } else { @@ -243,18 +416,54 @@ impl CalendarEventQuery for Server { let results = self .search_store() - .query_account( + .filter_account( SearchQuery::new(SearchIndex::Calendar) .with_filters(filters) - .with_comparators(comparators) .with_account_id(account_id) - .with_mask(if access_token.is_shared(account_id) { - cache.shared_items(access_token, [Acl::ReadItems], true) - } else { - cache.document_ids(false).collect() - }), + .with_mask(mask), ) .await?; + let sort = EventSortKey::until_unique(sort); + let results = if results.len() > 1 && !sort.is_empty() { + if let [(EventSortKey::Id, is_ascending)] = sort.as_slice() { + let mut results = results.into_iter().collect::>(); + if !is_ascending { + results.reverse(); + } + results + } else { + let items = matched_items(&cache, &results); + let comparators = sort + .into_iter() + .map(|(key, is_ascending)| match key { + EventSortKey::Id => { + rank_comparator(&items, |item| Some(item.document_id()), is_ascending) + } + EventSortKey::Start => { + let floating_tz = (!default_tz.is_utc()).then_some(default_tz); + rank_comparator( + &items, + |item| item.event_start(floating_tz), + is_ascending, + ) + } + EventSortKey::Uid => { + rank_comparator(&items, |item| item.uid(), is_ascending) + } + EventSortKey::Created => { + rank_comparator(&items, |item| item.created_at(), is_ascending) + } + EventSortKey::Updated => { + rank_comparator(&items, |item| item.modified_at(), is_ascending) + } + }) + .collect(); + QueryResults::new(results, comparators).into_sorted() + } + } else { + results.into_iter().collect() + }; + let query_state = sampled.state(cache.get_state(false)); // Extract comparators let comparators = request @@ -263,104 +472,125 @@ impl CalendarEventQuery for Server { .filter(|s| !s.is_empty()) .unwrap_or_default(); - if expand_recurrences && !results.is_empty() { - let Some(time_range) = filter.filter(|f| f.start != i64::MIN && f.end != i64::MAX) - else { - return Err(trc::JmapEvent::InvalidArguments.into_err().details( - "Both 'after' and 'before' filters are required when expanding recurrences", - )); - }; + if let Some(time_range) = expand_range { let max_instances = self.core.groupware.max_ical_instances; let mut expanded_results = Vec::with_capacity(results.len() as usize); - let has_uid_comparator = comparators - .iter() - .any(|c| matches!(c.property, CalendarEventComparator::Uid)); for document_id in results { - let Some(_calendar_event) = self + let Some(resource) = cache.item_by_id(document_id) else { + continue; + }; + let Some(_content) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::property( account_id, Collection::CalendarEvent, document_id, + CalendarEventField::Content, )) .await? else { continue; }; - let calendar_event = _calendar_event - .unarchive::() + let content = _content + .unarchive::() .caused_by(trc::location!())?; - - // Expand recurrences - let uid = if has_uid_comparator { - Arc::new( - calendar_event - .data - .event - .uids() - .next() - .unwrap_or_default() - .to_string(), - ) - } else { - Arc::new(String::new()) + let event = &content.data.event; + let mut scheduling_components = event + .components + .iter() + .filter(|component| component.component_type.is_scheduling_object()); + let is_recurring = + match (scheduling_components.next(), scheduling_components.next()) { + (Some(component), None) => component.is_recurrent(), + _ => true, + }; + let attendees = match (&hidden_attendees, &identities) { + (Some(hidden), Some(identities)) + if hidden.is_filtered(document_id, event, identities) => + { + AttendeeSearch::VisibleTo(identities) + } + _ => AttendeeSearch::All, }; - for expansion in calendar_event + let event_filter = (!instance_filter.is_empty()) + .then(|| instance_filter.for_event(event, default_language, attendees)); + let mut component_matches: Vec<(u32, bool)> = Vec::new(); + let created = resource.created_at().unwrap_or_default(); + let updated = resource.modified_at().unwrap_or_default(); + let uid = resource.uid().unwrap_or_default(); + + for expansion in content .data - .expand(default_tz, time_range) + .expand(default_tz, time_range, OverlapRule::Jmap) .unwrap_or_default() { - let Some(recurrence_key) = expansion.recurrence_key() else { - continue; + let prefix = match (is_recurring, expansion.recurrence_key()) { + (false, _) => 0, + (true, Some(recurrence_key)) => recurrence_key.prefix(), + (true, None) => continue, }; - if expanded_results.len() < max_instances { - expanded_results.push(SearchResult { - created: calendar_event.created.to_native().to_be_bytes(), - updated: calendar_event.modified.to_native().to_be_bytes(), - start: expansion.start.to_be_bytes(), - uid: uid.clone(), - document_id, - recurrence_key, - }); - } else { - return Err(trc::JmapEvent::InvalidArguments.into_err().details( - "The number of expanded recurrences exceeds the server limit", - )); + if let Some(event_filter) = &event_filter { + let is_match = match component_matches + .iter() + .find(|(comp_id, _)| *comp_id == expansion.comp_id) + { + Some((_, is_match)) => *is_match, + None => { + let is_match = event + .components + .get(expansion.comp_id as usize) + .is_some_and(|component| event_filter.matches(component)); + component_matches.push((expansion.comp_id, is_match)); + is_match + } + }; + if !is_match { + continue; + } } + if expanded_results.len() >= max_instances { + return Err(trc::JmapEvent::CannotCalculateOccurrences + .into_err() + .details( + "The number of expanded recurrences exceeds the server limit", + )); + } + expanded_results.push(ExpandedResult { + prefix, + document_id, + start: expansion.start, + created, + updated, + uid, + }); } } let mut response = QueryResponseBuilder::new( expanded_results.len(), self.core.jmap.query_max_results, - cache.get_state(false), + query_state, &request, ); - // Sort results - if !expanded_results.is_empty() { - expanded_results.sort_by(|a, b| { - for comparator in comparators { - let ordering = if comparator.is_ascending { - a.get_property(&comparator.property) - .cmp(b.get_property(&comparator.property)) + response.response.can_calculate_changes = false; + expanded_results.sort_by(|a, b| { + comparators + .iter() + .map(|comparator| { + let ordering = a.cmp_by(b, &comparator.property); + if comparator.is_ascending { + ordering } else { - b.get_property(&comparator.property) - .cmp(a.get_property(&comparator.property)) - }; - - if ordering != Ordering::Equal { - return ordering; + ordering.reverse() } - } - Ordering::Equal - }); - - // Add results - for result in expanded_results { - if !response.add(result.recurrence_key.prefix(), result.document_id) { - break; - } + }) + .find(|ordering| ordering.is_ne()) + .unwrap_or(Ordering::Equal) + }); + for result in expanded_results { + if !response.add(result.prefix, result.document_id) { + break; } } response.build() @@ -368,7 +598,7 @@ impl CalendarEventQuery for Server { let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, - cache.get_state(false), + query_state, &request, ); for document_id in results { @@ -384,22 +614,56 @@ impl CalendarEventQuery for Server { &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Calendar); + let mut metadata_filters = Vec::new(); + for filter in &request.filter { + match filter { + Filter::Property(CalendarFilter::Metadata(filter)) => { + metadata_filters.push(filter); + } + Filter::Property(CalendarFilter::_T(other)) => { + return Err(trc::JmapEvent::UnsupportedFilter + .into_err() + .details(other.clone())); + } + Filter::And | Filter::Or | Filter::Not | Filter::Close => {} + } + } + let metadata_query = metadata.query(metadata_filters)?; + let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await?; - let results = cache.document_ids(true).collect::>(); + let is_member = access_token.is_member(account_id); + let readable = if is_member { + cache.document_ids(true).collect::() + } else { + cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) + }; + let leaves = metadata + .evaluate(self, account_id, &metadata_query, || { + flagged_documents( + &cache, + ResourceScope::Containers, + (!is_member).then_some(&readable), + ) + }) + .await?; + let results = filter_containers(&request.filter, leaves, readable); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, - State::Initial, + sampled.state(cache.get_state(true)), &request, ); @@ -413,32 +677,459 @@ impl CalendarEventQuery for Server { } } +const ITEM_LOOKUP_RATIO: u64 = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EventSortKey { + Id, + Start, + Uid, + Created, + Updated, +} + +impl EventSortKey { + fn until_unique(mut sort: Vec<(EventSortKey, bool)>) -> Vec<(EventSortKey, bool)> { + if let Some(position) = sort.iter().position(|(key, _)| *key == EventSortKey::Id) { + sort.truncate(position + 1); + if position > 0 && sort.last().is_some_and(|(_, is_ascending)| *is_ascending) { + sort.pop(); + } + } + sort + } +} + +trait EventStart { + fn event_start(&self, floating_tz: Option) -> Option; +} + +impl EventStart for GroupwareResourceRef<'_> { + fn event_start(&self, floating_tz: Option) -> Option { + let (start, _) = self.event_time_range()?; + Some( + match floating_tz.filter(|_| { + self.event_flags() + .is_some_and(|flags| flags & EVENT_FLOATING_START != 0) + }) { + Some(tz) => resolve_local(tz, start).unwrap_or(start), + None => start, + }, + ) + } +} + +pub(crate) fn matched_items<'x>( + cache: &'x GroupwareResources, + results: &RoaringBitmap, +) -> Vec> { + let mut items = Vec::with_capacity(results.len() as usize); + if results.len().saturating_mul(ITEM_LOOKUP_RATIO) < cache.resources.len() as u64 { + items.extend( + results + .iter() + .filter_map(|document_id| cache.item_by_id(document_id)), + ); + } else { + let mut ids = results.iter().peekable(); + for resource in cache.resources.iter_run(false) { + let document_id = resource.document_id(); + while ids.next_if(|id| *id < document_id).is_some() {} + if ids.next_if_eq(&document_id).is_some() && !resource.is_container() { + items.push(resource); + } + if ids.peek().is_none() { + break; + } + } + } + items +} + +pub(crate) fn rank_comparator<'x, K: Ord>( + items: &[GroupwareResourceRef<'x>], + key: impl Fn(&GroupwareResourceRef<'x>) -> Option, + is_ascending: bool, +) -> SearchComparator { + let mut keys = items + .iter() + .filter_map(|item| key(item).map(|key| (key, item.document_id()))) + .collect::>(); + keys.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + let mut ranks = AHashMap::with_capacity(keys.len()); + let mut rank = 0u32; + let mut previous = None; + for (key, document_id) in &keys { + if previous.is_some_and(|previous| previous != key) { + rank += 1; + } + previous = Some(key); + ranks.insert(*document_id, rank); + } + SearchComparator::sorted_set(ranks, is_ascending) +} + fn local_timestamp(dt: &JSCalendarDateTime, tz: Tz) -> Option { - tz.from_local_datetime(&dt.to_naive_date_time()?) - .single() + tz.from_local(dt.to_naive_date_time()?) .map(|dt| dt.timestamp()) } #[derive(Debug)] -struct SearchResult { - recurrence_key: RecurrenceKey, +struct ExpandedResult<'x> { + prefix: u32, document_id: u32, - start: [u8; std::mem::size_of::()], - created: [u8; std::mem::size_of::()], - updated: [u8; std::mem::size_of::()], - uid: Arc, + start: i64, + created: i64, + updated: i64, + uid: &'x str, } -impl SearchResult { - fn get_property(&self, comparator: &CalendarEventComparator) -> &[u8] { +impl ExpandedResult<'_> { + fn cmp_by(&self, other: &Self, comparator: &CalendarEventComparator) -> Ordering { match comparator { - CalendarEventComparator::Uid => self.uid.as_bytes(), - CalendarEventComparator::Start | CalendarEventComparator::RecurrenceId => { - self.start.as_ref() + CalendarEventComparator::Id => { + (self.prefix, self.document_id).cmp(&(other.prefix, other.document_id)) } - CalendarEventComparator::Created => self.created.as_ref(), - CalendarEventComparator::Updated => self.updated.as_ref(), - CalendarEventComparator::_T(_) => &[], + CalendarEventComparator::Start => self.start.cmp(&other.start), + CalendarEventComparator::Uid => self.uid.cmp(other.uid), + CalendarEventComparator::Created => self.created.cmp(&other.created), + CalendarEventComparator::Updated => self.updated.cmp(&other.updated), + CalendarEventComparator::_T(_) => Ordering::Equal, } } } + +#[derive(Debug, Default)] +struct TextVisibility { + public_ids: Option, + attendee_ids: Option, +} + +impl TextVisibility { + fn new( + cache: &GroupwareResources, + private_ids: RoaringBitmap, + hidden_attendee_ids: RoaringBitmap, + ) -> Self { + if private_ids.is_empty() && hidden_attendee_ids.is_empty() { + return TextVisibility::default(); + } + let all_ids = cache.document_ids(false).collect::(); + let visible_ids = + |hidden_ids: RoaringBitmap| (!hidden_ids.is_empty()).then(|| &all_ids - hidden_ids); + + TextVisibility { + public_ids: visible_ids(private_ids), + attendee_ids: visible_ids(hidden_attendee_ids), + } + } + + fn push(&self, filters: &mut Vec, filter: SearchFilter) { + match &self.public_ids { + Some(public_ids) => { + filters.push(SearchFilter::And); + filters.push(filter); + filters.push(SearchFilter::is_in_set(public_ids.clone())); + filters.push(SearchFilter::End); + } + None => filters.push(filter), + } + } + + fn push_any_field( + &self, + filters: &mut Vec, + filter: SearchFilter, + fields: &[CalendarSearchField], + visible_attendees: Option, + ) { + let SearchFilter::Text { + op, + value, + language, + .. + } = filter + else { + return; + }; + if self.public_ids.is_some() { + filters.push(SearchFilter::And); + } + filters.push(SearchFilter::Or); + for field in fields + .iter() + .filter(|field| **field != CalendarSearchField::Attendee) + { + filters.push(SearchFilter::Text { + field: (*field).into(), + op, + value: value.clone(), + language, + }); + } + let attendee = SearchFilter::Text { + field: CalendarSearchField::Attendee.into(), + op, + value, + language, + }; + match &self.attendee_ids { + Some(attendee_ids) => { + filters.push(SearchFilter::And); + filters.push(attendee); + filters.push(SearchFilter::is_in_set(attendee_ids.clone())); + filters.push(SearchFilter::End); + } + None => filters.push(attendee), + } + if let Some(visible_attendees) = visible_attendees.filter(|ids| !ids.is_empty()) { + filters.push(SearchFilter::is_in_set(visible_attendees)); + } + filters.push(SearchFilter::End); + if let Some(public_ids) = &self.public_ids { + filters.push(SearchFilter::is_in_set(public_ids.clone())); + filters.push(SearchFilter::End); + } + } +} + +struct HiddenAttendees { + filtered: RoaringBitmap, + writable: RoaringBitmap, +} + +struct VisibleAttendees<'x> { + hidden: &'x HiddenAttendees, + identities: &'x CalendarAddresses, + account_id: u32, + mask: &'x RoaringBitmap, + default_language: Language, +} + +impl HiddenAttendees { + fn new( + cache: &GroupwareResources, + access_token: &AccessToken, + hidden_ids: &RoaringBitmap, + ) -> Option { + if hidden_ids.is_empty() { + return None; + } + let write_all = cache.shared_containers(access_token, [Acl::ModifyItems], true); + let write_own = + cache.shared_containers(access_token, [Acl::ModifyItems, Acl::ModifyItemsOwn], true); + let mut filtered = RoaringBitmap::new(); + let mut writable = RoaringBitmap::new(); + for document_id in hidden_ids { + let Some(resource) = cache.item_by_id(document_id) else { + continue; + }; + let calendar_ids = resource.child_names(); + if !calendar_ids + .iter() + .all(|name| write_all.contains(name.parent_id)) + { + filtered.insert(document_id); + if calendar_ids + .iter() + .all(|name| write_own.contains(name.parent_id)) + { + writable.insert(document_id); + } + } + } + (!filtered.is_empty()).then_some(HiddenAttendees { filtered, writable }) + } + + fn is_filtered( + &self, + document_id: u32, + event: &ArchivedICalendar, + identities: &CalendarAddresses, + ) -> bool { + self.filtered.contains(document_id) + && !(self.writable.contains(document_id) + && identities.archived_event_ownership(event).may_write_own()) + } +} + +impl VisibleAttendees<'_> { + async fn of(&self, server: &Server, filter: &SearchFilter) -> trc::Result { + let mut candidates = &self.hidden.filtered & self.mask; + let SearchFilter::Text { + op, + value, + language, + .. + } = filter + else { + return Ok(RoaringBitmap::new()); + }; + if candidates.is_empty() { + return Ok(candidates); + } + let attendee = SearchFilter::Text { + field: CalendarSearchField::Attendee.into(), + op: *op, + value: value.clone(), + language: *language, + }; + let mut text_filter = InstanceTextFilter::default(); + text_filter.add(&[CalendarSearchField::Attendee], &attendee); + candidates = server + .search_store() + .filter_account( + SearchQuery::new(SearchIndex::Calendar) + .with_filters(vec![attendee]) + .with_account_id(self.account_id) + .with_mask(candidates), + ) + .await?; + + let mut matches = RoaringBitmap::new(); + let mut chunk = Vec::new(); + let mut candidates = candidates.iter(); + loop { + chunk.clear(); + chunk.extend(candidates.by_ref().take(MAX_BATCH_DOCUMENTS)); + if chunk.is_empty() { + break; + } + server + .archives( + self.account_id, + Collection::CalendarEvent, + CalendarEventField::Content.into(), + &chunk, + |document_id, archive| { + let event = &archive + .unarchive::() + .caused_by(trc::location!())? + .data + .event; + let is_match = + !self.hidden.is_filtered(document_id, event, self.identities) || { + let filter = text_filter.for_event( + event, + self.default_language, + AttendeeSearch::VisibleTo(self.identities), + ); + event + .components + .iter() + .filter(|component| { + component.component_type.is_scheduling_object() + }) + .any(|component| filter.matches(component)) + }; + if is_match { + matches.insert(document_id); + } + Ok(true) + }, + ) + .await?; + } + Ok(matches) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BoundCandidates { + Widened, + Exact, +} + +#[derive(Debug, Clone, Copy)] +enum TimeBound { + After(i64), + Before(i64), +} + +impl TimeBound { + fn matches(self, (start, end): (i64, i64), margin: i64) -> bool { + match self { + TimeBound::After(after) => after.saturating_sub(margin) < end, + TimeBound::Before(before) => before.saturating_add(margin) > start, + } + } + + fn time_range(self) -> TimeRange { + match self { + TimeBound::After(after) => TimeRange::new(after, i64::MAX), + TimeBound::Before(before) => TimeRange::new(i64::MIN, before), + } + } + + async fn event_ids( + self, + server: &Server, + cache: &GroupwareResources, + account_id: u32, + default_tz: Tz, + candidates: BoundCandidates, + ) -> trc::Result { + let margin = if default_tz.is_utc() { + 0 + } else { + MAX_UTC_OFFSET + }; + let mut event_ids = RoaringBitmap::new(); + + for resource in cache.resources.iter_run(false) { + let Some(range) = resource.event_time_range() else { + continue; + }; + let document_id = resource.document_id(); + if self.matches(range, -margin) + || (self.matches(range, margin) + && (candidates == BoundCandidates::Widened + || self + .contains_instance(server, account_id, document_id, default_tz) + .await?)) + { + event_ids.insert(document_id); + } + } + + Ok(event_ids) + } + + async fn contains_instance( + self, + server: &Server, + account_id: u32, + document_id: u32, + default_tz: Tz, + ) -> trc::Result { + let Some(archive) = server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await? + else { + return Ok(false); + }; + + Ok(archive + .unarchive::() + .caused_by(trc::location!())? + .data + .expand(default_tz, self.time_range(), OverlapRule::Jmap) + .is_some_and(|instances| !instances.is_empty())) + } +} + +const TEXT_FIELDS: &[CalendarSearchField] = &[ + CalendarSearchField::Title, + CalendarSearchField::Description, + CalendarSearchField::Location, + CalendarSearchField::Owner, + CalendarSearchField::Attendee, +]; +const PARTICIPANT_FIELDS: &[CalendarSearchField] = + &[CalendarSearchField::Owner, CalendarSearchField::Attendee]; diff --git a/crates/jmap/src/calendar_event/server_set.rs b/crates/jmap/src/calendar_event/server_set.rs new file mode 100644 index 00000000..cb1e7d5a --- /dev/null +++ b/crates/jmap/src/calendar_event/server_set.rs @@ -0,0 +1,345 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::blob::embedded::EmbeddedBlobIds; +use crate::calendar_event::{EventMap, JSCalendarEntries}; +use calcard::{ + icalendar::{ICalendar, ICalendarComponent, ICalendarProperty, ICalendarValue}, + jscalendar::{ + JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue, import::ImportOptions, + }, +}; +use groupware::calendar::{identity::CalendarAddresses, participants::VisibleParticipants}; +use jmap_tools::{JsonPointerItem, Key, Value}; +use types::{ + blob::{BlobClass, BlobId}, + collection::Collection, + id::Id, +}; + +type EventValue = Value<'static, JSCalendarProperty, JSCalendarValue>; +pub type ServerSetValues = Vec<(JSCalendarProperty, EventValue)>; + +pub const PENDING_DOCUMENT_ID: u32 = u32::MAX; + +const TRACKED_PROPERTIES: [JSCalendarProperty; 9] = [ + JSCalendarProperty::Uid, + JSCalendarProperty::Type, + JSCalendarProperty::Created, + JSCalendarProperty::Updated, + JSCalendarProperty::Sequence, + JSCalendarProperty::OrganizerCalendarAddress, + JSCalendarProperty::TimeZone, + JSCalendarProperty::Start, + JSCalendarProperty::Duration, +]; + +const DATE_PROPERTIES: [JSCalendarProperty; 3] = [ + JSCalendarProperty::TimeZone, + JSCalendarProperty::Start, + JSCalendarProperty::Duration, +]; + +const COMPONENT_PROPERTIES: [(JSCalendarProperty, ICalendarProperty); 5] = [ + (JSCalendarProperty::Uid, ICalendarProperty::Uid), + (JSCalendarProperty::Created, ICalendarProperty::Created), + (JSCalendarProperty::Updated, ICalendarProperty::Dtstamp), + (JSCalendarProperty::Sequence, ICalendarProperty::Sequence), + ( + JSCalendarProperty::OrganizerCalendarAddress, + ICalendarProperty::Organizer, + ), +]; + +#[derive(Debug, Default)] +pub struct TrackedValues { + values: Vec<(JSCalendarProperty, Option)>, + is_origin: Option, +} + +impl TrackedValues { + pub fn from_patch( + patch: &Value<'_, JSCalendarProperty, JSCalendarValue>, + ) -> Self { + TrackedValues { + values: patch + .as_object() + .into_iter() + .flat_map(|object| object.iter()) + .filter_map(|(key, value)| match key { + Key::Property(property) if TRACKED_PROPERTIES.contains(property) => { + Some((property.clone(), Some(value.clone().into_owned()))) + } + _ => None, + }) + .collect(), + is_origin: None, + } + } + + pub fn from_event(group: &JSCalendar<'_, Id, BlobId>, is_origin: Option) -> Self { + TrackedValues::from_map(group.first_entry(), &TRACKED_PROPERTIES, is_origin) + } + + pub fn from_dates(group: &JSCalendar<'_, Id, BlobId>) -> Self { + TrackedValues::from_map(group.first_entry(), &DATE_PROPERTIES, None) + } + + fn from_map( + event: Option<&EventMap<'_>>, + properties: &[JSCalendarProperty], + is_origin: Option, + ) -> Self { + TrackedValues { + values: properties + .iter() + .map(|property| { + ( + property.clone(), + event + .and_then(|event| event.get(&Key::Property(property.clone()))) + .map(|value| value.clone().into_owned()), + ) + }) + .collect(), + is_origin, + } + } + + pub fn with_component(mut self, component: Option<&ICalendarComponent>) -> Self { + self.values.reserve(COMPONENT_PROPERTIES.len() + 1); + self.values.push(( + JSCalendarProperty::Type, + component + .and_then(|component| component.component_type.to_jscalendar_type()) + .map(|typ| Value::Element(JSCalendarValue::Type(typ))), + )); + for (property, ical_property) in COMPONENT_PROPERTIES { + let value = component + .and_then(|component| component.property(&ical_property)) + .and_then(|entry| entry.values.first()) + .and_then(|value| match value { + ICalendarValue::Text(text) => Some(Value::Str(text.clone().into())), + ICalendarValue::Uri(uri) => Some(Value::Str(uri.to_unwrapped_string().into())), + ICalendarValue::Integer(number) => Some(Value::Number((*number).into())), + ICalendarValue::PartialDateTime(date_time) if date_time.has_date_and_time() => { + date_time.to_timestamp().map(|timestamp| { + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + timestamp, false, + ))) + }) + } + _ => None, + }); + self.values.push((property, value)); + } + self + } + + pub fn with_origin(mut self, is_origin: bool) -> Self { + self.is_origin = Some(is_origin); + self + } + + pub fn server_set( + self, + client: &TrackedValues, + before: Option<&TrackedValues>, + ) -> ServerSetValues { + let mut server_set = Vec::new(); + for (property, after) in self.values { + let before_value = before.map(|before| before.get(&property)); + if before_value.is_some_and(|before| before == after.as_ref()) + || client.get(&property).is_some_and(|client| match &after { + Some(after) => client == after, + None => client.is_null(), + }) + { + continue; + } + match after { + Some(after) => server_set.push((property, after)), + None if before_value.flatten().is_some() => { + server_set.push((property, Value::Null)); + } + None => {} + } + } + if let Some(is_origin) = self.is_origin + && before.is_none_or(|before| before.is_origin != Some(is_origin)) + { + server_set.push((JSCalendarProperty::IsOrigin, Value::Bool(is_origin))); + } + server_set + } + + fn get(&self, property: &JSCalendarProperty) -> Option<&EventValue> { + self.values + .iter() + .find(|(tracked, _)| tracked == property) + .and_then(|(_, value)| value.as_ref()) + } +} + +#[derive(Debug, Default)] +pub struct BlobProperties(Vec>); + +pub struct ImportedValues<'a> { + pub participants: bool, + pub blob_properties: &'a BlobProperties, + pub every_blob_property: bool, + pub visible_to: Option<&'a CalendarAddresses>, +} + +impl BlobProperties { + pub fn from_patch( + patch: &Value<'_, JSCalendarProperty, JSCalendarValue>, + is_changed: impl Fn(&BlobId) -> bool, + ) -> Self { + let mut properties = Vec::new(); + for (key, value) in patch + .as_object() + .into_iter() + .flat_map(|object| object.iter()) + { + let property = match key { + Key::Property(JSCalendarProperty::Pointer(pointer)) => match pointer.first() { + Some(JsonPointerItem::Key(Key::Property(property))) => property, + _ => continue, + }, + Key::Property(property) => property, + _ => continue, + }; + if !properties.contains(property) && value.any_blob_id(&is_changed) { + properties.push(property.clone()); + } + } + BlobProperties(properties) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl ImportedValues<'_> { + pub fn is_requested(&self) -> bool { + self.participants || self.every_blob_property || !self.blob_properties.is_empty() + } + + pub fn values( + &self, + ical: &ICalendar, + account_id: u32, + document_id: u32, + ) -> impl Iterator, EventValue)> { + ical.clone() + .into_jscalendar_with::( + ImportOptions::new() + .include_ical_components(false) + .return_first(true) + .with_blob_id_generator(EmbeddedBlobIds::new( + account_id, + Collection::CalendarEvent, + document_id, + )), + ) + .map(|event| { + let mut event = event.into_inner(); + if let Some(identities) = self.visible_to { + event.retain_visible_participants(identities); + } + event + }) + .into_iter() + .flat_map(EventValue::into_expanded_object) + .filter_map(move |(key, value)| match key { + Key::Property(property) + if (self.participants && property == JSCalendarProperty::Participants) + || ((self.every_blob_property + || self.blob_properties.0.contains(&property)) + && value.any_blob_id(&|_: &BlobId| true)) => + { + Some((property, value)) + } + _ => None, + }) + } +} + +pub trait EmbeddedBlobId { + fn is_embedded_in(&self, account_id: u32, document_id: u32) -> bool; +} + +impl EmbeddedBlobId for BlobId { + fn is_embedded_in(&self, account_id: u32, document_id: u32) -> bool { + matches!( + self.class, + BlobClass::Embedded { + account_id: blob_account_id, + collection, + document_id: blob_document_id, + } if blob_account_id == account_id + && collection == u8::from(Collection::CalendarEvent) + && blob_document_id == document_id + ) + } +} + +pub trait PendingBlobIds { + fn resolve_document_id(&mut self, document_id: u32); +} + +impl PendingBlobIds for ServerSetValues { + fn resolve_document_id(&mut self, document_id: u32) { + for (_, value) in self.iter_mut() { + value.update_blob_ids(&|blob_id| { + if let BlobClass::Embedded { + document_id: pending @ PENDING_DOCUMENT_ID, + .. + } = &mut blob_id.class + { + *pending = document_id; + } + }); + } + } +} + +trait BlobIdValues { + fn any_blob_id(&self, predicate: &impl Fn(&BlobId) -> bool) -> bool; + + fn update_blob_ids(&mut self, update: &impl Fn(&mut BlobId)); +} + +impl BlobIdValues for Value<'_, JSCalendarProperty, JSCalendarValue> { + fn any_blob_id(&self, predicate: &impl Fn(&BlobId) -> bool) -> bool { + match self { + Value::Element(JSCalendarValue::BlobId(blob_id)) => predicate(blob_id), + Value::Array(values) => values.iter().any(|value| value.any_blob_id(predicate)), + Value::Object(object) => object.values().any(|value| value.any_blob_id(predicate)), + _ => false, + } + } + + fn update_blob_ids(&mut self, update: &impl Fn(&mut BlobId)) { + match self { + Value::Element(JSCalendarValue::BlobId(blob_id)) => update(blob_id), + Value::Array(values) => { + for value in values { + value.update_blob_ids(update); + } + } + Value::Object(object) => { + for (_, value) in object.as_mut_vec() { + value.update_blob_ids(update); + } + } + _ => {} + } + } +} + diff --git a/crates/jmap/src/calendar_event/set.rs b/crates/jmap/src/calendar_event/set.rs index 0a8264cb..7b7bec75 100644 --- a/crates/jmap/src/calendar_event/set.rs +++ b/crates/jmap/src/calendar_event/set.rs @@ -4,61 +4,105 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::calendar_event::{CalendarSyntheticId, assert_is_unique_uid}; +use crate::api::metadata::{ + MetadataAccess, MetadataPatches, MetadataPreload, MetadataTarget, MetadataType, MetadataWriter, + NewMetadata, ObjectMetadata, PreparedMetadata, is_empty_update, reject_uncommitted, +}; +use crate::api::pending_creates::PendingCreates; +use crate::blob::embedded::EmbeddedExport; +use crate::calendar_event::{ + CalendarSyntheticId, EventMainComponent, EventValue, JSCalendarEntries, UidIndex, + detach_iana_timezones, + get::{EventInstanceBuilder, InstanceBinaries}, + hidden::{HiddenParticipants, visible_overrides}, + is_origin, + privacy::{ + UidPrivacyConflicts, assert_privacy_access, assert_privacy_allowed, uid_privacy_conflict, + }, + server_set::{ + BlobProperties, EmbeddedBlobId, ImportedValues, PENDING_DOCUMENT_ID, PendingBlobIds, + ServerSetValues, TrackedValues, + }, + user::user_data_error, + validate::{ + BooleanValue, InstanceMetadata, InstanceView, OverridePatch, OverridePointer, PropertyName, + }, +}; use crate::changes::state::JmapCacheState; use calcard::{ - common::{PartialDateTime, timezone::Tz}, + common::{ + PartialDateTime, + timezone::{NominalDuration, Tz, ZonedDateTime}, + }, icalendar::{ - ICalendar, ICalendarAction, ICalendarComponent, ICalendarComponentType, ICalendarDuration, - ICalendarEntry, ICalendarParameter, ICalendarParameterValue, ICalendarProperty, - ICalendarRelated, ICalendarValue, + ICalendar, ICalendarComponent, ICalendarDuration, ICalendarProperty, ICalendarValue, + }, + jscalendar::{ + JSCALENDAR_VERSION, JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue, }, - jscalendar::{JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue}, }; -use chrono::DateTime; use common::{ - DavName, DavResources, Server, + ArchivedDavName, DavName, GroupwareResources, Server, auth::{AccessToken, AccountInfo}, + storage::{ + dav::CachedUid, + quota::{DiskQuotaBudget, ObjectQuotaUsage}, + }, }; +use compact_str::ToCompactString; use groupware::{ - DestroyArchive, + DestroyArchive, SizeWriter, cache::GroupwareCache, calendar::{ - ALERT_EMAIL, ALERT_RELATIVE_TO_END, ArchivedDefaultAlert, Calendar, CalendarEvent, - CalendarEventData, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, - EVENT_INVITE_SELF, PREF_USE_DEFAULT_ALERTS, + CalendarEvent, CalendarEventContent, CalendarEventData, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, + EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, + alerts::{DefaultAlertsResolver, DefaultAlertsView, ICalendarDefaultAlerts}, expand::{CalendarEventExpansion, ComponentRecurrenceId, RecurrenceKey, resolve_local}, + identity::{CalendarAddresses, ParticipantIdentityAddresses}, + index::ICalendarObjectUid, itip::ItipSendStatus, + notification::{NotificationAudience, may_have_viewers}, + privacy::{EventPrivacy, EventViewer}, + rights::{EventAcl, EventChanges}, + schedule::{AlarmTasks, EventAlarmScheduler, EventAlarmUsers}, + sequence::{ICalendarSequence, SCHEDULING_EVENT_FLAGS}, + storage::{DirectChange, DirectChangeNotification, NotificationQuota}, + user::{ICalendarUserData, UpdatedPolicy, UserDataSplit, UserDataUpdate, UserDataView}, }, scheduling::{ - ItipMessages, - event_create::itip_create, + ItipError, ItipMessages, + event_create::{itip_attendee_create, itip_create}, event_update::itip_update, itip::{itip_assign_organizer, itip_unreachable_recipient}, + recipient::RecipientPolicy, }, }; use http_proto::HttpSessionData; +use jiff::Timestamp; use jmap_proto::{ - error::set::SetError, + error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::calendar_event, - request::MaybeInvalid, + request::{MaybeInvalid, capability::CapabilityIds}, types::state::State, }; -use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Map, Value}; +use jmap_tools::{Element, JsonPointer, JsonPointerHandler, JsonPointerItem, Key, Map, Value}; +use registry::schema::enums::StorageQuota; use std::{borrow::Cow, str::FromStr}; use store::{ ValueKey, - ahash::AHashSet, + ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, now, serialize::rkyv_deserialize}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot, now, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection, VanishedCollection}, + field::CalendarEventField, id::Id, + metadata::MetadataKinds, }; pub trait CalendarEventSet: Sync + Send { @@ -67,84 +111,202 @@ pub trait CalendarEventSet: Sync + Send { request: SetRequest<'_, calendar_event::CalendarEvent>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; - #[allow(clippy::too_many_arguments)] fn create_calendar_event( &self, - cache: &DavResources, + context: &mut EventSetContext<'_>, batch: &mut BatchBuilder, - access_token: &AccessToken, - account_id: u32, - account_info: &AccountInfo, - send_scheduling_messages: bool, - can_add_calendars: &Option, - js_calendar_event: JSCalendar<'_, Id, BlobId>, + source: EventSource<'_>, updates: Value<'_, JSCalendarProperty, JSCalendarValue>, - ) -> impl Future>>>>; + metadata: Option, + ) -> impl Future>; } +pub(crate) type CreatedEvent = Result<(Slot, ServerSetValues), SetError>>; + +pub struct EventSetContext<'x> { + pub cache: &'x GroupwareResources, + pub access_token: &'x AccessToken, + pub account_id: u32, + pub account_info: &'x AccountInfo, + pub send_scheduling_messages: bool, + pub can_add_calendars: Option<&'x RoaringBitmap>, + pub identities: &'x CalendarAddresses, + pub uid_index: UidIndex<'x>, + pub will_destroy: Vec, + pub default_alerts: DefaultAlertsResolver, + pub notification_quota: NotificationQuota, + pub disk_quota: DiskQuotaBudget, + pub metadata: MetadataWriter, +} + +pub enum EventSource<'x> { + Create, + Copy(CopiedEvent<'x>), +} + +pub struct CopiedEvent<'x> { + pub event: JSCalendar<'x, Id, BlobId>, + pub flags: u16, + pub use_default_alerts: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EventWrite { + Create, + Copy, + Update, +} + +#[derive(Debug, Clone, Copy)] +struct PatchTarget { + write: EventWrite, + id: Option, + is_origin: Option, +} + +const EVENT_METADATA_FLAGS: u16 = + EVENT_DRAFT | EVENT_INVITE_SELF | EVENT_INVITE_OTHERS | EVENT_HIDE_ATTENDEES; + impl CalendarEventSet for Server { async fn calendar_event_set( &self, mut request: SetRequest<'_, calendar_event::CalendarEvent>, access_token: &AccessToken, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::CalendarEvent); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await?; let account_info = self - .scheduling_account_info(access_token.account_id(), account_id) + .account_info(account_id) .await .caused_by(trc::location!())?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(false, &request.if_in_state)?); + .with_state(sampled.assert_state(cache.get_state(false), &request.if_in_state)?); let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + let is_owner = access_token.is_member(account_id); + let identities = if is_owner { + CalendarAddresses::default() + } else { + self.account_identity_addresses(access_token.account_id()) + .await + .caused_by(trc::location!())? + }; + let uid_privacy_conflicts = UidPrivacyConflicts::new( + &cache, + request.create.as_ref(), + request.update.as_ref(), + &will_destroy, + ); // Obtain calendarIds - let (can_add_calendars, can_delete_calendars, can_modify_calendars) = - if access_token.is_shared(account_id) { - ( - cache - .shared_containers(access_token, [Acl::AddItems], true) - .into(), - cache - .shared_containers(access_token, [Acl::RemoveItems], true) - .into(), - cache - .shared_containers(access_token, [Acl::ModifyItems], true) - .into(), - ) - } else { - (None, None, None) - }; + let is_shared = access_token.is_shared(account_id); + let (can_add_calendars, can_delete_calendars) = if is_shared { + ( + cache + .shared_containers(access_token, [Acl::AddItems], true) + .into(), + cache + .shared_containers(access_token, [Acl::RemoveItems], true) + .into(), + ) + } else { + (None, None) + }; + + // Obtain quota + let quota = if request.has_creates() { + let account = self.account(account_id).await.caused_by(trc::location!())?; + self.object_quota_usage(&account, StorageQuota::MaxCalendarEvents, || { + cache.resources.count(false) + }) + } else { + ObjectQuotaUsage::unlimited() + }; // Process creates let mut batch = BatchBuilder::new(); let send_scheduling_messages = request.arguments.send_scheduling_messages.unwrap_or(false); - 'create: for (id, object) in request.unwrap_create() { + let mut created_slots = PendingCreates::new(); + let mut created_server_set = Vec::new(); + let uid_index = UidIndex::new( + &cache, + request + .create + .iter() + .flat_map(|create| create.values()) + .filter_map(|object| { + match object.as_object_and_get(&Key::Property(JSCalendarProperty::Uid)) { + Some(Value::Str(uid)) => Some(uid.cached_uid()), + _ => None, + } + }), + ); + let mut context = EventSetContext { + cache: &cache, + access_token, + account_id, + account_info: &account_info, + send_scheduling_messages, + can_add_calendars: can_add_calendars.as_ref(), + identities: &identities, + uid_index, + will_destroy, + default_alerts: DefaultAlertsResolver::with_resources(account_id, cache.clone()), + notification_quota: NotificationQuota::default(), + disk_quota: if request.has_creates() || request.has_updates() { + self.disk_quota_budget(account_info.account()) + .await + .caused_by(trc::location!())? + } else { + DiskQuotaBudget::unlimited() + }, + metadata: MetadataWriter::new(metadata, account_id), + }; + 'create: for (id, mut object) in request.unwrap_create() { + if !quota.has_room(created_slots.len()) { + response.not_created.append(id, too_many_events()); + continue 'create; + } + if uid_privacy_conflicts.creates.contains(&id) { + response.not_created.append(id, uid_privacy_conflict()); + continue 'create; + } + let new_metadata = match context + .metadata + .extract(MetadataPatches::for_create(), &mut object) + { + Ok(patches) => patches.map(NewMetadata::Create), + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + match self .create_calendar_event( - &cache, + &mut context, &mut batch, - access_token, - account_id, - &account_info, - send_scheduling_messages, - &can_add_calendars, - JSCalendar::default(), + EventSource::Create, object, + new_metadata, ) .await? { - Ok(document_id) => { - response.created(id, document_id); + Ok((slot, server_set)) => { + created_server_set.push((id.clone(), slot, server_set)); + created_slots.push(id, slot); } Err(err) => { response.not_created.append(id, err); @@ -154,26 +316,29 @@ impl CalendarEventSet for Server { } // Group updates and instance removals by event - let has_synthetic_ids = will_destroy.iter().any(|id| id.is_synthetic()) + let has_synthetic_ids = context.will_destroy.iter().any(|id| id.is_synthetic()) || request.update.as_ref().is_some_and(|update| { update .iter() .any(|(id, _)| matches!(id, MaybeInvalid::Value(id) if id.is_synthetic())) }); let is_destroyed_event = |document_id: u32| { - will_destroy + context + .will_destroy .iter() .any(|id| !id.is_synthetic() && id.document_id() == document_id) }; let will_be_destroyed = |id: Id| { - will_destroy.iter().any(|destroy_id| { + context.will_destroy.iter().any(|destroy_id| { *destroy_id == id || (!destroy_id.is_synthetic() && destroy_id.document_id() == id.document_id()) }) }; - let mut updates: Vec = - Vec::with_capacity(request.update.as_ref().map_or(0, |update| update.len())); - for (id, object) in request.unwrap_update() { + let mut updates = + EventUpdates::with_capacity(request.update.as_ref().map_or(0, |update| update.len())); + let mut metadata_patches = AHashMap::new(); + let mut preload = MetadataPreload::default(); + for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -181,74 +346,100 @@ impl CalendarEventSet for Server { continue; } }; - let document_id = id.document_id(); if will_be_destroyed(id) { response.not_updated.append(id, SetError::will_destroy()); continue; } - let update = EventUpdate::for_document(&mut updates, document_id, has_synthetic_ids); - if let Some(recurrence_key) = id.recurrence_key() { - update.instances.push(InstanceOp { + match context + .metadata + .extract(MetadataPatches::for_update(), &mut object) + { + Ok(Some(patches)) if id.is_synthetic() => { + response.not_updated.append(id, instance_metadata(&patches)); + continue; + } + Ok(Some(patches)) => { + let document_id = id.document_id(); + preload.insert( + document_id, + &patches, + cache + .item_by_id(document_id) + .map_or(MetadataKinds::NONE, |resource| resource.metadata_kinds()), + ); + metadata_patches.insert(document_id, patches); + } + Ok(None) => {} + Err(err) => { + response.not_updated.append(id, err); + continue; + } + } + let op = match id.recurrence_key() { + Some(recurrence_key) => EventOp::Instance(InstanceOp { id, recurrence_key, - patch: Some(object), target: None, - is_destroy: false, - }); - } else if update.base_id.is_none() { - update.base_id = Some(id); - update.base_patch = Some(object); - } else { - response.not_updated.append( - id, - SetError::invalid_properties() - .with_property(JSCalendarProperty::Id) - .with_description("Duplicate event id."), - ); - } + action: InstanceAction::Update { + patch: object, + server_set: None, + }, + }), + None => EventOp::Base { id, patch: object }, + }; + updates.push(has_synthetic_ids, op, &mut response); } - for id in will_destroy.iter().copied() { + for id in context.will_destroy.iter().copied() { let Some(recurrence_key) = id.recurrence_key() else { continue; }; - let document_id = id.document_id(); - if is_destroyed_event(document_id) { + if is_destroyed_event(id.document_id()) { response.not_destroyed.append(id, SetError::will_destroy()); continue; } - EventUpdate::for_document(&mut updates, document_id, has_synthetic_ids) - .instances - .push(InstanceOp { + updates.push( + has_synthetic_ids, + EventOp::Instance(InstanceOp { id, recurrence_key, - patch: None, target: None, - is_destroy: true, - }); + action: InstanceAction::Destroy, + }), + &mut response, + ); } - let mut destroy_events = will_destroy; + context.metadata.preload(self, account_id, preload).await?; + let mut destroy_events = std::mem::take(&mut context.will_destroy); if has_synthetic_ids { destroy_events.retain(|id| !id.is_synthetic()); } + let moved_uid_index = if updates + .pending + .iter() + .any(EventUpdate::patches_calendar_ids) + { + UidIndex::new( + &cache, + updates.pending.iter().filter_map(|update| { + cache + .item_by_id(update.document_id) + .and_then(|resource| resource.uid()) + .map(Cow::Borrowed) + }), + ) + } else { + UidIndex::default() + }; // Process updates - 'update: for mut update in updates { + 'update: for mut update in updates.pending { let document_id = update.document_id; - if update.base_id.is_some() && !update.instances.is_empty() { - update.fail( - &mut response, - SetError::invalid_properties() - .with_property(JSCalendarProperty::Id) - .with_description(concat!( - "A base event and its instances cannot be modified ", - "in the same request." - )), - ); - continue 'update; - } + let mut update_metadata = update + .base_id() + .and_then(|_| metadata_patches.remove(&document_id)); let calendar_event_ = if let Some(calendar_event_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, @@ -263,14 +454,79 @@ impl CalendarEventSet for Server { let calendar_event = calendar_event_ .to_unarchived::() .caused_by(trc::location!())?; + if let Err(err) = assert_privacy_access( + EventPrivacy::from_flags(calendar_event.inner.flags.to_native()), + EventViewer::new(is_owner), + ) { + update.fail(&mut response, err); + continue 'update; + } + if update.base_id().is_some() && uid_privacy_conflicts.updates.contains(document_id) { + update.fail(&mut response, uid_privacy_conflict()); + continue 'update; + } + let metadata_access = if is_shared && update_metadata.is_some() { + MetadataAccess::from_container_acls( + calendar_event + .inner + .names + .iter() + .map(|name| cache.container_acl(access_token, name.parent_id())), + ) + } else { + MetadataAccess::FULL + }; + if !metadata_access.may_read { + update.fail(&mut response, SetError::not_found()); + continue 'update; + } + if update.is_empty_base_update() + && let Some(patches) = update_metadata.take() + { + match context + .metadata + .write_metadata_only::( + self, + patches, + metadata_access, + &calendar_event, + document_id, + &mut batch, + ) + .await? + { + Ok(()) => update.succeed(&mut response), + Err(err) => update.fail(&mut response, err), + } + continue 'update; + } + let Some(content_) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await? + else { + update.fail(&mut response, SetError::not_found()); + continue 'update; + }; + let content = content_ + .to_unarchived::() + .caused_by(trc::location!())?; let mut new_calendar_event = calendar_event .deserialize::() .caused_by(trc::location!())?; + let mut new_content = content + .deserialize::() + .caused_by(trc::location!())?; // Resolve synthetic ids into recurrence instances let mut has_instances = false; - if !update.instances.is_empty() { - match update.plan_instances(&new_calendar_event.data, &mut response) { + if update.has_instances() { + match update.plan_instances(&new_content.data, &mut response) { InstancePlan::Instances => { has_instances = true; } @@ -285,59 +541,116 @@ impl CalendarEventSet for Server { } } - let mut js_calendar_group = - std::mem::take(&mut new_calendar_event.data.event).into_jscalendar::(); + let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let ownership = identities.event_ownership(&new_content.data.event); + let was_origin = is_origin(&new_content.data.event, account_info.addresses()); + let view_default_alerts = context + .default_alerts + .resolve_for_content( + self, + account_id, + personal_id, + &new_content, + new_calendar_event.names.iter().map(DavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + let mut view = if is_owner { + new_content.data.event.clone() + } else { + new_content + .data + .user_data_view(new_content.preferences(personal_id), UserDataView::Full) + }; + let mut previous_event = std::mem::take(&mut new_content.data.event); + let previous_preferences = (!is_owner) + .then(|| new_content.preferences(personal_id).cloned()) + .flatten(); + view.apply_default_alerts(&view_default_alerts, DefaultAlertsView::Merge); + detach_iana_timezones(&mut view); + let mut js_calendar_group = view.into_jscalendar::(); + let tracked_before = update.base_patch().map(|patch| { + ( + TrackedValues::from_patch(patch), + TrackedValues::from_event(&js_calendar_group, Some(was_origin)), + ) + }); + let blob_properties = update + .base_patch() + .map(|patch| { + BlobProperties::from_patch(patch, |blob_id| { + !blob_id.is_embedded_in(account_id, document_id) + }) + }) + .unwrap_or_default(); + update.track_instance_values(&previous_event, |blob_id| { + !blob_id.is_embedded_in(account_id, document_id) + }); + let is_restricted = is_shared + && !EventAcl::for_calendars( + &cache, + access_token, + calendar_event + .inner + .names + .iter() + .map(ArchivedDavName::parent_id), + ) + .may_write(ownership); + let hidden_participants = (is_restricted + && calendar_event.inner.flags.to_native() & EVENT_HIDE_ATTENDEES != 0) + .then(|| HiddenParticipants::hide(&mut js_calendar_group, &identities)); + let per_user_changes = hidden_participants + .is_none() + .then(|| update.per_user_patch_changes()) + .flatten(); + let before_group = per_user_changes + .is_none() + .then(|| js_calendar_group.clone()); // Apply per-instance changes to the recurrence overrides of the base event - if has_instances && !update.apply_instances(&mut js_calendar_group, &mut response) { - continue 'update; + if has_instances { + let metadata = InstanceMetadata { + is_origin: was_origin, + flags: new_calendar_event.flags, + calendar_ids: &new_calendar_event.names, + use_default_alerts: new_content + .preferences(personal_id) + .is_some_and(|preferences| preferences.use_default_alerts()), + }; + if !update.apply_instances(&mut js_calendar_group, &metadata, &mut response) { + continue 'update; + } } // Process changes - if let Err(err) = update_calendar_event( - access_token.personal_id(account_id, Collection::Calendar), - update.base_id, - update.base_patch.take().unwrap_or_default(), + let is_use_default_alerts_changed = match update_calendar_event( + &cache, + personal_id, + PatchTarget { + write: EventWrite::Update, + id: update.base_id(), + is_origin: Some(was_origin), + }, + update.take_base_patch(), &mut new_calendar_event, + &mut new_content, &mut js_calendar_group, ) { - update.fail(&mut response, err); - continue 'update; - } - - // Convert JSCalendar to iCalendar - let Some(ical) = js_calendar_group.into_icalendar() else { - update.fail( - &mut response, - SetError::invalid_properties() - .with_description("Failed to convert calendar event to iCalendar."), - ); - continue 'update; - }; - new_calendar_event.data.event = ical; - stamp_updated(&mut new_calendar_event.data.event, now() as i64); - - // Assign an organizer when participants were added to an event that had none - if let Some(organizer_address) = account_info.addresses().first() { - itip_assign_organizer(&mut new_calendar_event.data.event, organizer_address); - } - - // Validate UID - match ( - new_calendar_event.data.event.uids().next(), - calendar_event.inner.data.event.uids().next(), - ) { - (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} - (None, None) | (None, Some(_)) => {} - _ => { - update.fail( - &mut response, - SetError::invalid_properties() - .with_property(JSCalendarProperty::Uid) - .with_description("You cannot change the UID of a calendar event."), - ); + Ok(is_changed) => is_changed, + Err(err) => { + update.fail(&mut response, err); continue 'update; } + }; + + // Validate calendarIds limit + let max_calendars = self.core.groupware.max_calendars_per_event; + if new_calendar_event.names.len() > max_calendars + && new_calendar_event.names.len() > calendar_event.inner.names.len() + { + update.fail(&mut response, too_many_calendars(max_calendars)); + continue 'update; } // Validate new calendarIds @@ -353,17 +666,13 @@ impl CalendarEventSet for Server { )), ); continue 'update; - } else if can_add_calendars - .as_ref() - .is_some_and(|ids| !ids.contains(calendar_id)) - { - update.fail( - &mut response, - SetError::forbidden().with_description(format!( - "You are not allowed to add calendar events to calendar {}.", - Id::from(calendar_id) - )), - ); + } else if can_add_calendars.as_ref().is_some_and(|ids| { + !ids.contains(calendar_id) + && !(EventAcl::for_calendar(&cache, access_token, calendar_id) + .may_manage_own_items() + && ownership.may_write_own()) + }) { + update.fail(&mut response, add_calendar_forbidden(calendar_id)); continue 'update; } } @@ -371,7 +680,11 @@ impl CalendarEventSet for Server { // Validate deleted calendarIds if let Some(can_delete_calendars) = &can_delete_calendars { for calendar_id in new_calendar_event.removed_calendar_ids(calendar_event.inner) { - if !can_delete_calendars.contains(calendar_id) { + if !can_delete_calendars.contains(calendar_id) + && !(EventAcl::for_calendar(&cache, access_token, calendar_id) + .may_manage_own_items() + && ownership.may_write_own()) + { update.fail( &mut response, SetError::forbidden().with_description(format!( @@ -383,84 +696,393 @@ impl CalendarEventSet for Server { } } } + let is_calendar_ids_changed = new_calendar_event + .added_calendar_ids(calendar_event.inner) + .next() + .is_some() + || new_calendar_event + .removed_calendar_ids(calendar_event.inner) + .next() + .is_some(); + if is_calendar_ids_changed + && let Err(err) = moved_uid_index + .assert_is_unique( + self, + account_id, + &previous_event, + &new_calendar_event.names, + Some(document_id), + &destroy_events, + ) + .await? + { + update.fail(&mut response, err); + continue 'update; + } + let is_metadata_changed = is_calendar_ids_changed + || (calendar_event.inner.flags.to_native() ^ new_calendar_event.flags) + & EVENT_METADATA_FLAGS + != 0; - // Validate changed calendarIds - if let Some(can_modify_calendars) = &can_modify_calendars { - for calendar_id in new_calendar_event.unchanged_calendar_ids(calendar_event.inner) { - if !can_modify_calendars.contains(calendar_id) { - update.fail( - &mut response, - SetError::forbidden().with_description(format!( - "You are not allowed to modify calendar {}.", - Id::from(calendar_id) - )), - ); + let mut changes = match &before_group { + Some(before_group) => EventChanges::between( + before_group, + &js_calendar_group, + calendar_event.inner.flags.to_native(), + new_calendar_event.flags, + &identities, + ), + None => per_user_changes.unwrap_or_default(), + }; + let hidden_additions = match (hidden_participants, &before_group) { + (Some(hidden_participants), Some(before_group)) => { + let visible_group = + std::mem::replace(&mut js_calendar_group, JSCalendar(Value::Null)); + let (full_group, additions) = + hidden_participants.merge(before_group, visible_group); + js_calendar_group = full_group; + Some(additions) + } + _ => None, + }; + if is_use_default_alerts_changed { + changes.insert_per_user(); + } + let tracked = tracked_before.map(|(client, before)| { + ( + client, + before, + TrackedValues::from_dates(&js_calendar_group), + ) + }); + + // Convert JSCalendar to iCalendar + let ical = match self + .export_icalendar(access_token, js_calendar_group) + .await? + { + Ok(ical) => ical, + Err(err) => { + update.fail( + &mut response, + if hidden_additions.is_some() { + SetError::invalid_properties() + .with_description("The calendar event could not be converted.") + } else { + err + }, + ); + continue 'update; + } + }; + new_content.data.event = ical; + changes.classify_recurrence_sets(&previous_event, &new_content.data.event); + let is_per_user_change = changes.is_per_user_only(); + if let Err(err) = + assert_privacy_allowed(&new_content.data.event, EventViewer::new(is_owner)) + { + update.fail(&mut response, err); + continue 'update; + } + let new_default_alerts = context + .default_alerts + .resolve_for_content( + self, + account_id, + personal_id, + &new_content, + new_calendar_event.names.iter().map(DavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + for defaults in [&view_default_alerts, &new_default_alerts] { + new_content.data.event.strip_default_alerts(defaults); + } + if let Err(err) = new_content + .data + .event + .validate_user_data_changes(&previous_event) + { + update.fail(&mut response, user_data_error(err)); + continue 'update; + } + + let mut is_user_data_only = false; + let mut is_stored_event = false; + let mut personal_updated = None; + let mut dropped_overrides = Vec::new(); + if !is_owner { + let user_data = UserDataUpdate { + account_id: personal_id, + view: UserDataView::Full, + previous: previous_preferences.as_ref(), + current: new_content + .preferences + .iter() + .find(|preferences| preferences.account_id == personal_id), + updated: if was_origin { + UpdatedPolicy::Server(now() as i64) + } else { + UpdatedPolicy::Client + }, + }; + let split = if is_per_user_change { + new_content + .data + .event + .extract_user_preferences(&previous_event, user_data) + .map(|preferences| (preferences, UserDataSplit::UserDataOnly)) + } else { + new_content + .data + .event + .split_user_data(&previous_event, user_data) + }; + match split { + Ok((preferences, split)) => { + if was_origin + && split == UserDataSplit::UserDataOnly + && previous_preferences + .as_ref() + .map_or(0, |previous| previous.updated) + != preferences.updated + { + personal_updated = Some(preferences.updated); + } + new_content.set_preferences(preferences); + if split == UserDataSplit::UserDataOnly { + is_stored_event = true; + if is_metadata_changed { + new_content.data.event = previous_event.clone(); + } else { + new_content.data.event = std::mem::take(&mut previous_event); + is_user_data_only = true; + } + } + } + Err(err) => { + update.fail(&mut response, user_data_error(err)); continue 'update; } } + } else if is_per_user_change { + let mut event = if is_metadata_changed { + previous_event.clone() + } else { + std::mem::take(&mut previous_event) + }; + dropped_overrides = event.copy_user_data(&new_content.data.event); + new_content.data.event = event; + is_user_data_only = !is_metadata_changed; + } + + // Assign an organizer when participants were added to an event that had none + let is_organizer_assigned = !is_user_data_only + && account_info + .addresses() + .first() + .is_some_and(|organizer_address| { + itip_assign_organizer(&mut new_content.data.event, organizer_address) + }); + let has_scheduling_changes = !is_user_data_only + && new_content + .data + .event + .has_scheduling_changes(&previous_event); + if is_owner + && !is_metadata_changed + && !is_organizer_assigned + && !has_scheduling_changes + && new_content.data.event.has_same_sequences(&previous_event) + { + is_user_data_only = true; + } + let is_origin_event = is_origin(&new_content.data.event, account_info.addresses()); + if !is_user_data_only && is_origin_event { + if has_scheduling_changes + || (calendar_event.inner.flags.to_native() ^ new_calendar_event.flags) + & SCHEDULING_EVENT_FLAGS + != 0 + { + new_content.data.event.increment_sequence(&previous_event); + } + stamp_updated(&mut new_content.data.event, now() as i64); + } + + if let Err(err) = assert_participants_limit( + &new_content.data.event, + self.core + .groupware + .max_ical_attendees_per_instance + .saturating_sub(hidden_additions.map_or(0, |additions| additions.ignored)), + ) { + update.fail(&mut response, err); + continue 'update; + } + + // Validate UID + if !is_user_data_only { + let description = match ( + new_content.data.event.object_uid(), + previous_event.object_uid(), + ) { + (Some(new_uid), Some(old_uid)) if new_uid == old_uid => None, + (None, None) => None, + (None, Some(_)) => Some("The UID of a calendar event cannot be removed."), + _ => Some("You cannot change the UID of a calendar event."), + }; + if let Some(description) = description { + update.fail( + &mut response, + SetError::invalid_properties() + .with_property(JSCalendarProperty::Uid) + .with_description(description), + ); + continue 'update; + } + } + + if is_restricted { + let mut unchanged = new_calendar_event + .unchanged_calendar_ids(calendar_event.inner) + .peekable(); + let acl = if unchanged.peek().is_none() { + EventAcl::for_calendars( + &cache, + access_token, + new_calendar_event.added_calendar_ids(calendar_event.inner), + ) + } else { + EventAcl::for_calendars(&cache, access_token, unchanged) + }; + if let Err(err) = + changes.assert_allowed(&acl, ownership, calendar_event.inner.flags.to_native()) + { + update.fail( + &mut response, + SetError::forbidden().with_description(err.description()), + ); + continue 'update; + } } // Check size and quota - new_calendar_event.size = new_calendar_event.data.event.size() as u32; + new_calendar_event.size = SizeWriter::ical(&new_content.data.event) as u32; if new_calendar_event.size as usize > self.core.groupware.max_ical_size { update.fail( &mut response, - SetError::invalid_properties().with_description(format!( - "Event size {} exceeds the maximum allowed size of {} bytes.", - new_calendar_event.size, self.core.groupware.max_ical_size - )), + too_large_event( + new_calendar_event.size as usize, + self.core.groupware.max_ical_size, + ), ); continue 'update; } // Obtain previous alarm let now = now() as i64; - let prev_email_alarm = calendar_event.inner.data.next_alarm(now, Tz::Floating); - + let previous_calendar_ids = calendar_event + .inner + .names + .iter() + .map(ArchivedDavName::parent_id) + .collect::>(); + let prev_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, content.inner)? + .with_event_flags(calendar_event.inner.flags.to_native()), + &content.inner.data, + &previous_calendar_ids, + now, + &mut context.default_alerts, + ) + .await + .caused_by(trc::location!())?; // Build event - let mut next_email_alarm = None; - new_calendar_event.data = CalendarEventData::new( - new_calendar_event.data.event, - Tz::Floating, - self.core.groupware.max_ical_instances, - &mut next_email_alarm, - ); + if !is_stored_event || is_calendar_ids_changed { + let owner_default_alerts = context + .default_alerts + .resolve_for_content( + self, + account_id, + account_id, + &new_content, + new_calendar_event.names.iter().map(DavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + new_content.data = CalendarEventData::new_with_default_alerts( + new_content.data.event, + Tz::Floating, + self.core.groupware.max_ical_instances, + &owner_default_alerts, + ); + } + let next_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &new_content)? + .with_event_flags(new_calendar_event.flags), + &new_content.data, + &new_calendar_event + .names + .iter() + .map(DavName::parent_id) + .collect::>(), + now, + &mut context.default_alerts, + ) + .await + .caused_by(trc::location!())?; // Scheduling let mut itip_messages = None; - let itip_status = if send_scheduling_messages { + let itip_status = if send_scheduling_messages + && !is_user_data_only + && new_calendar_event.flags & EVENT_DRAFT == 0 + { ItipSendStatus::resolve( self, access_token, &account_info, - new_calendar_event.data.event_range_end(), + new_content.data.event_range_end(), ) } else { ItipSendStatus::NotRequested }; if itip_status.is_send() { - if let Some(calendar_address) = itip_unreachable_recipient( - &new_calendar_event.data.event, - account_info.addresses(), - ) { + if let Some(calendar_address) = + itip_unreachable_recipient(&new_content.data.event, account_info.addresses()) + { update.fail( &mut response, - SetError::no_supported_schedule_methods(calendar_address), + if hidden_additions.is_some() && !identities.contains(calendar_address) { + SetError::new(SetErrorType::NoSupportedScheduleMethods) + .with_description( + "No supported scheduling method for at least one recipient.", + ) + } else { + SetError::no_supported_schedule_methods(calendar_address) + }, ); continue 'update; } + let policy = RecipientPolicy::new(&self.core.groupware, new_calendar_event.flags); let result = if new_calendar_event.schedule_tag.is_some() { - let old_ical = rkyv_deserialize(&calendar_event.inner.data.event) - .caused_by(trc::location!())?; - itip_update( - &mut new_calendar_event.data.event, - &old_ical, + &mut new_content.data.event, + &previous_event, account_info.addresses(), + policy, ) } else { - itip_create(&mut new_calendar_event.data.event, account_info.addresses()) + itip_create( + &mut new_content.data.event, + account_info.addresses(), + policy, + ) }; match result { @@ -504,7 +1126,11 @@ impl CalendarEventSet for Server { &mut response, SetError::invalid_properties() .with_property(JSCalendarProperty::Participants) - .with_description(err.to_string()), + .with_description(if hidden_additions.is_some() { + Cow::Borrowed("The scheduling messages could not be built.") + } else { + Cow::Owned(err.to_string()) + }), ); continue 'update; } @@ -513,7 +1139,7 @@ impl CalendarEventSet for Server { Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, DocumentId = document_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); // Event changed, but there are no iTIP messages to send @@ -542,18 +1168,146 @@ impl CalendarEventSet for Server { // Validate quota let extra_bytes = (new_calendar_event.size as u64) .saturating_sub(u32::from(calendar_event.inner.size) as u64); - if extra_bytes > 0 { - match self - .has_available_quota(account_info.account(), extra_bytes) - .await - { - Ok(_) => {} - Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { - update.fail(&mut response, SetError::over_quota()); - continue 'update; - } - Err(err) => return Err(err.caused_by(trc::location!())), + if context.disk_quota.check(extra_bytes).is_err() { + update.fail(&mut response, SetError::over_quota()); + continue 'update; + } + + let visible_to = hidden_additions.is_some().then_some(&identities); + let added_overrides = hidden_additions.is_some_and(|additions| additions.overrides); + let server_set = tracked.map(|(client, before, dates)| { + let mut server_set = dates + .with_component(new_content.data.event.main_component()) + .with_origin(is_origin_event) + .server_set(&client, Some(&before)); + let imported = ImportedValues { + participants: is_organizer_assigned + || hidden_additions.is_some_and(|additions| additions.base), + blob_properties: &blob_properties, + every_blob_property: false, + visible_to, + }; + if imported.is_requested() { + server_set.extend( + imported + .values(&new_content.data.event, account_id, document_id) + .filter(|(property, _)| { + !added_overrides + || *property != JSCalendarProperty::RecurrenceOverrides + }), + ); } + if let Some(identities) = visible_to.filter(|_| added_overrides) { + let mut view = new_content + .data + .user_data_view(new_content.preferences(personal_id), UserDataView::Full); + view.apply_default_alerts(&new_default_alerts, DefaultAlertsView::Replace); + server_set.push(( + JSCalendarProperty::RecurrenceOverrides, + visible_overrides(view, identities, account_id, document_id), + )); + } + server_set + }); + let mut instance_server_set = Vec::with_capacity(update.instances_mut().len()); + for instance in update.instances_mut() { + let (id, recurrence_key) = (instance.id, instance.recurrence_key); + let Some(tracked) = instance.take_server_set() else { + continue; + }; + let mut values = TrackedValues::default() + .with_component( + new_content + .data + .event + .instance_component(tracked.recurrence_id), + ) + .server_set(&tracked.client, Some(&tracked.before)); + let imported = ImportedValues { + participants: added_overrides, + blob_properties: &tracked.blob_properties, + every_blob_property: false, + visible_to, + }; + if imported.is_requested() { + values.extend(instance_imported_values( + &new_content.data, + recurrence_key, + &imported, + account_id, + document_id, + )); + } + instance_server_set.push((id, values)); + } + + let prepared_metadata = match update_metadata { + Some(patches) => { + let access = MetadataAccess { + may_write_shared: metadata_access.may_write_shared + && (!is_shared + || new_calendar_event + .added_calendar_ids(calendar_event.inner) + .all(|calendar_id| { + cache + .container_acl(access_token, calendar_id) + .contains(Acl::ModifyMetadata) + })), + ..metadata_access + }; + match context + .metadata + .prepare_for( + self, + patches, + access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + update.fail(&mut response, err); + continue 'update; + } + } + } + None => None, + }; + context.disk_quota.reserve(extra_bytes); + + let notify_calendar_ids = if is_user_data_only { + Vec::new() + } else { + new_calendar_event.all_calendar_ids(calendar_event.inner) + }; + let notify_event_flags = + new_calendar_event.flags | calendar_event.inner.flags.to_native(); + if !is_user_data_only + && may_have_viewers( + access_token, + account_id, + &cache, + ¬ify_calendar_ids, + NotificationAudience::of_event(notify_event_flags), + ) + { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Updated { + event_id: document_id, + previous: previous_event, + current: new_content.data.event.clone(), + calendar_ids: notify_calendar_ids, + event_flags: notify_event_flags, + }, + Some(&cache), + &mut context.notification_quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; } // Update record @@ -563,26 +1317,40 @@ impl CalendarEventSet for Server { cache.format_resource_path_by_parent(document_id, calendar_id) }) .collect::>(); + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + new_calendar_event.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + context.metadata.write(prepared, document_id, &mut batch)?; + } new_calendar_event - .update( - access_token.account_tenant_ids(), + .update_full( + new_content, + access_token.changed_by(), calendar_event, + content.inner, account_id, document_id, + None, &mut batch, ) .caused_by(trc::location!())?; for path in vanished_paths { batch.log_vanished_item(VanishedCollection::Calendar, path); } - if prev_email_alarm != next_email_alarm { - if let Some(prev_alarm) = prev_email_alarm { - prev_alarm.delete_task(&mut batch); - } - if let Some(next_alarm) = next_email_alarm { - next_alarm.write_task(&mut batch); - } - } + self.replace_event_alarms( + account_id, + document_id, + prev_email_alarms, + next_email_alarms, + &mut AlarmTasks::None, + &mut batch, + ) + .await + .caused_by(trc::location!())?; if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) @@ -590,9 +1358,59 @@ impl CalendarEventSet for Server { } update.succeed(&mut response); + if let (Some(base_id), Some(server_set)) = (update.base_id(), server_set) { + response.add_server_set_properties(base_id, server_set); + } + if let Some(base_id) = update.base_id().filter(|_| !dropped_overrides.is_empty()) { + response.add_server_set_properties( + base_id, + dropped_overrides.into_iter().map(|recurrence_id| { + ( + JSCalendarProperty::Pointer(JsonPointer::new(vec![ + JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceOverrides, + )), + JsonPointerItem::Key(Key::Property(JSCalendarProperty::DateTime( + JSCalendarDateTime::new(recurrence_id, true), + ))), + ])), + Value::Null, + ) + }), + ); + } + if let (Some(base_id), Some(updated)) = (update.base_id(), personal_updated) { + response.add_server_set_property( + base_id, + JSCalendarProperty::Updated, + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + updated, false, + ))), + ); + } + for (id, server_set) in instance_server_set { + response.add_server_set_properties(id, server_set); + } } // Process deletions + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::CalendarEvent, + &destroy_events + .iter() + .map(|id| id.document_id()) + .filter(|&document_id| { + cache + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; 'destroy: for id in destroy_events { let document_id = id.document_id(); @@ -603,7 +1421,7 @@ impl CalendarEventSet for Server { let Some(calendar_event_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, @@ -620,19 +1438,67 @@ impl CalendarEventSet for Server { .caused_by(trc::location!())?; // Validate ACLs + if let Err(err) = assert_privacy_access( + EventPrivacy::from_flags(calendar_event.inner.flags.to_native()), + EventViewer::new(is_owner), + ) { + response.not_destroyed.append(id, err); + continue 'destroy; + } + let notify_calendar_ids = calendar_event.inner.calendar_ids().collect::>(); + let notify_event_flags = calendar_event.inner.flags.to_native(); + let may_notify = may_have_viewers( + access_token, + account_id, + &cache, + ¬ify_calendar_ids, + NotificationAudience::of_event(notify_event_flags), + ); + let stored_content = if is_owner && !may_notify { + None + } else { + self.store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())? + }; + let stored_event = stored_content + .as_ref() + .map(|content| { + content + .unarchive::() + .and_then(|content| rkyv_deserialize::<_, ICalendar>(&content.data.event)) + }) + .transpose() + .caused_by(trc::location!())?; if let Some(can_delete_calendars) = &can_delete_calendars { + let mut may_write_own = None; for name in calendar_event.inner.names.iter() { let parent_id = name.parent_id.to_native(); - if !can_delete_calendars.contains(parent_id) { - response.not_destroyed.append( - id, - SetError::forbidden().with_description(format!( - "You are not allowed to remove events from calendar {}.", - Id::from(parent_id) - )), - ); - continue 'destroy; + if can_delete_calendars.contains(parent_id) + || (EventAcl::for_calendar(&cache, access_token, parent_id) + .may_manage_own_items() + && *may_write_own.get_or_insert_with(|| { + stored_event.as_ref().is_some_and(|event| { + identities.event_ownership(event).may_write_own() + }) + })) + { + continue; } + response.not_destroyed.append( + id, + SetError::forbidden().with_description(format!( + "You are not allowed to remove events from calendar {}.", + Id::from(parent_id) + )), + ); + continue 'destroy; } } @@ -642,7 +1508,7 @@ impl CalendarEventSet for Server { self, access_token, &account_info, - calendar_event.inner.data.event_range_end(), + calendar_event.inner.event_range_end(), ) } else { ItipSendStatus::NotRequested @@ -664,14 +1530,36 @@ impl CalendarEventSet for Server { } // Delete event + if let Some(previous) = stored_event.filter(|_| may_notify) { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Destroyed { + event_id: document_id, + previous, + calendar_ids: notify_calendar_ids, + event_flags: notify_event_flags, + }, + Some(&cache), + &mut context.notification_quota, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } DestroyArchive(calendar_event) - .delete_all( + .remove_all( + self, &account_info, account_id, document_id, + stored_content, + &mut AlarmTasks::None, itip_status.is_send(), + &cleanup, &mut batch, ) + .await .caused_by(trc::location!())?; for path in cache.format_resource_paths_by_id(document_id) { @@ -683,14 +1571,28 @@ impl CalendarEventSet for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; - self.notify_task_queue(); + let assigned_ids = match context.metadata.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut response, + created_slots.into_create_ids(), + "Another process modified this calendar event, please try again.", + ); + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; - response.new_state = State::Exact(change_id).into(); + created_slots.resolve(&mut response, &assigned_ids); + for (create_id, slot, mut server_set) in created_server_set { + server_set.resolve_document_id(assigned_ids.slot(slot)); + response.add_created_properties(&create_id, server_set); + } + + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::Calendar)) + .into(); } Ok(response) @@ -698,41 +1600,105 @@ impl CalendarEventSet for Server { async fn create_calendar_event( &self, - cache: &DavResources, + context: &mut EventSetContext<'_>, batch: &mut BatchBuilder, - access_token: &AccessToken, - account_id: u32, - account_info: &AccountInfo, - send_scheduling_messages: bool, - can_add_calendars: &Option, - mut js_calendar_group: JSCalendar<'_, Id, BlobId>, + source: EventSource<'_>, updates: Value<'_, JSCalendarProperty, JSCalendarValue>, - ) -> trc::Result>>> { + metadata: Option, + ) -> trc::Result { + let cache = context.cache; + let access_token = context.access_token; + let account_id = context.account_id; + let account_info = context.account_info; + let send_scheduling_messages = context.send_scheduling_messages; + let can_add_calendars = context.can_add_calendars; + let identities = context.identities; + let personal_id = access_token.personal_id(account_id, Collection::Calendar); + let is_member = access_token.is_member(account_id); + // Process changes + let client_values = TrackedValues::from_patch(&updates); + let blob_properties = BlobProperties::from_patch(&updates, |_| true); + let has_version = matches!( + updates.as_object_and_get(&Key::Property(JSCalendarProperty::Version)), + Some(Value::Str(_)) + ); let mut event = CalendarEvent::default(); - let use_default_alerts = match update_calendar_event( - access_token.personal_id(account_id, Collection::Calendar), - None, - updates, - &mut event, - &mut js_calendar_group, - ) { - Ok(use_default_alerts) => use_default_alerts, - Err(err) => { - return Ok(Err(err)); + let mut content = CalendarEventContent::default(); + let (write, mut js_calendar_group, before) = match source { + EventSource::Create => (EventWrite::Create, JSCalendar::new_event(), None), + EventSource::Copy(copied) => { + event.flags = copied.flags & EVENT_METADATA_FLAGS; + if copied.use_default_alerts + && let Err(err) = content + .preferences_mut(personal_id) + .set_use_default_alerts(true) + { + return Ok(Err(user_data_error(err))); + } + let before = TrackedValues::from_event(&copied.event, None); + (EventWrite::Copy, copied.event, Some(before)) } }; + if let Err(err) = update_calendar_event( + cache, + personal_id, + PatchTarget { + write, + id: None, + is_origin: None, + }, + updates, + &mut event, + &mut content, + &mut js_calendar_group, + ) { + return Ok(Err(err)); + } + if event.names.len() > self.core.groupware.max_calendars_per_event { + return Ok(Err(too_many_calendars( + self.core.groupware.max_calendars_per_event, + ))); + } + + // Verify that the calendar ids valid + if let Some(name) = event + .names + .iter() + .find(|name| !cache.has_container_id(&name.parent_id)) + { + return Ok(Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::CalendarIds) + .with_description(format!( + "calendarId {} does not exist.", + Id::from(name.parent_id) + )))); + } + if let Some(ids) = can_add_calendars + && let Some(name) = event.names.iter().find(|name| { + !ids.contains(name.parent_id) + && !EventAcl::for_calendar(cache, access_token, name.parent_id) + .may_manage_own_items() + }) + { + return Ok(Err(add_calendar_forbidden(name.parent_id))); + } + let dates = TrackedValues::from_dates(&js_calendar_group); // Convert JSCalendar to iCalendar - let Some(mut ical) = js_calendar_group.into_icalendar() else { - return Ok(Err(SetError::invalid_properties().with_description( - "Failed to convert calendar event to iCalendar.", - ))); + let mut ical = match self + .export_icalendar(access_token, js_calendar_group) + .await? + { + Ok(ical) => ical, + Err(err) => return Ok(Err(err)), }; - stamp_updated(&mut ical, now() as i64); + if let Err(err) = assert_privacy_allowed(&ical, EventViewer::new(is_member)) { + return Ok(Err(err)); + } // Generate a UID when the client omitted one - if ical.uids().next().is_none() { + if ical.object_uid().is_none() { let uid = generate_uid(); for component in &mut ical.components { if component.component_type.is_event_or_todo() { @@ -741,112 +1707,171 @@ impl CalendarEventSet for Server { } } - // Verify that the calendar ids valid - let default_alert_comp_id = ical.components.len(); - for name in &event.names { - if !cache.has_container_id(&name.parent_id) { - return Ok(Err(SetError::invalid_properties() - .with_property(JSCalendarProperty::CalendarIds) - .with_description(format!( - "calendarId {} does not exist.", - Id::from(name.parent_id) - )))); - } else if can_add_calendars - .as_ref() - .is_some_and(|ids| !ids.contains(name.parent_id)) - { - return Ok(Err(SetError::forbidden().with_description(format!( - "You are not allowed to add calendar events to calendar {}.", - Id::from(name.parent_id) - )))); - } else if let Some(show_without_time) = use_default_alerts - && let Some(_calendar) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Calendar, - name.parent_id, - )) - .await? - { - ical.components.extend( - _calendar - .unarchive::() - .caused_by(trc::location!())? - .default_alerts( - access_token.personal_id(account_id, Collection::Calendar), - !show_without_time, - ) - .map(default_alert_to_ical), - ); - } - } + content.data.event = ical; + let user_default_alerts = context + .default_alerts + .resolve_for_content( + self, + account_id, + personal_id, + &content, + event.names.iter().map(DavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + let mut ical = std::mem::take(&mut content.data.event); + ical.strip_default_alerts(&user_default_alerts); - // Add default alarms - if ical.components.len() > default_alert_comp_id { - let component_ids = default_alert_comp_id as u32..ical.components.len() as u32; - for component in &mut ical.components { - if component.component_type.is_event_or_todo() - && !component.is_recurrence_override() - { - component.component_ids.extend(component_ids.clone()); + if let Err(err) = ical.validate_user_data() { + return Ok(Err(user_data_error(err))); + } + if !is_member { + match ical + .extract_user_data(personal_id) + .and_then(|mut preferences| { + preferences.inherit_default_alerts(content.preferences(personal_id))?; + Ok(preferences) + }) { + Ok(preferences) => { + content.set_preferences(preferences); + ical.apply_user_data(None, UserDataView::Full); + } + Err(err) => { + return Ok(Err(user_data_error(err))); } } } // Assign an organizer when the event has participants but none was provided - if let Some(organizer_address) = account_info.addresses().first() { - itip_assign_organizer(&mut ical, organizer_address); + let is_organizer_assigned = account_info + .addresses() + .first() + .is_some_and(|organizer_address| itip_assign_organizer(&mut ical, organizer_address)); + if let Some(ids) = can_add_calendars + && let Some(name) = event + .names + .iter() + .find(|name| !ids.contains(name.parent_id)) + && !identities.event_ownership(&ical).may_write_own() + { + return Ok(Err(add_calendar_forbidden(name.parent_id))); + } + let metadata_access = if metadata.is_some() && access_token.is_shared(account_id) { + MetadataAccess::from_container_acls( + event + .names + .iter() + .map(|name| cache.container_acl(access_token, name.parent_id)), + ) + } else { + MetadataAccess::FULL + }; + let now = now() as i64; + stamp_created(&mut ical, now); + if is_origin(&ical, account_info.addresses()) { + stamp_updated(&mut ical, now); + clamp_created(&mut ical, now); + } + + if let Err(err) = + assert_participants_limit(&ical, self.core.groupware.max_ical_attendees_per_instance) + { + return Ok(Err(err)); } // Validate UID - if let Err(err) = assert_is_unique_uid(self, account_id, ical.uids().next()).await? { + if let Err(err) = context + .uid_index + .assert_is_unique( + self, + account_id, + &ical, + &event.names, + None, + &context.will_destroy, + ) + .await? + { return Ok(Err(err)); } // Check size and quota - let size = ical.size(); + let size = SizeWriter::ical(&ical); if size > self.core.groupware.max_ical_size { - return Ok(Err(SetError::invalid_properties().with_description( - format!( - "Event size {} exceeds the maximum allowed size of {} bytes.", - size, self.core.groupware.max_ical_size - ), + return Ok(Err(too_large_event( + size, + self.core.groupware.max_ical_size, ))); } // Build event - let mut next_email_alarm = None; - event.data = CalendarEventData::new( - ical, + content.data.event = ical; + let owner_default_alerts = context + .default_alerts + .resolve_for_content( + self, + account_id, + account_id, + &content, + event.names.iter().map(DavName::parent_id), + ) + .await + .caused_by(trc::location!())?; + content.data = CalendarEventData::new_with_default_alerts( + std::mem::take(&mut content.data.event), Tz::Floating, self.core.groupware.max_ical_instances, - &mut next_email_alarm, + &owner_default_alerts, ); - event.size = size as u32; + let next_email_alarms = self + .next_event_alarms( + account_id, + &EventAlarmUsers::new(account_id, &content)?.with_event_flags(event.flags), + &content.data, + &event + .names + .iter() + .map(DavName::parent_id) + .collect::>(), + now, + &mut context.default_alerts, + ) + .await + .caused_by(trc::location!())?; // Scheduling let mut itip_messages = None; - let itip_status = if send_scheduling_messages { + let itip_status = if send_scheduling_messages && event.flags & EVENT_DRAFT == 0 { ItipSendStatus::resolve( self, access_token, account_info, - event.data.event_range_end(), + content.data.event_range_end(), ) } else { ItipSendStatus::NotRequested }; if itip_status.is_send() { if let Some(calendar_address) = - itip_unreachable_recipient(&event.data.event, account_info.addresses()) + itip_unreachable_recipient(&content.data.event, account_info.addresses()) { return Ok(Err(SetError::no_supported_schedule_methods( calendar_address, ))); } - match itip_create(&mut event.data.event, account_info.addresses()) { + match itip_create( + &mut content.data.event, + account_info.addresses(), + RecipientPolicy::new(&self.core.groupware, event.flags), + ) + .or_else(|err| { + if matches!(err, ItipError::NotOrganizer) { + itip_attendee_create(&mut content.data.event, account_info.addresses()) + } else { + Err(err) + } + }) { Ok(messages) => { if messages.iter().map(|r| r.to.len()).sum::() < self.core.groupware.itip_outbound_max_recipients @@ -872,7 +1897,7 @@ impl CalendarEventSet for Server { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -889,29 +1914,91 @@ impl CalendarEventSet for Server { } // Validate quota - match self - .has_available_quota(account_info.account(), size as u64) - .await + if context.disk_quota.check(size as u64).is_err() { + return Ok(Err(SetError::over_quota())); + } + + let mut server_set = dates + .with_component(content.data.event.main_component()) + .with_origin(is_origin(&content.data.event, account_info.addresses())) + .server_set(&client_values, before.as_ref()); + if write == EventWrite::Create && !has_version { + server_set.push(( + JSCalendarProperty::Version, + Value::Str(JSCALENDAR_VERSION.into()), + )); + } + let imported = ImportedValues { + participants: is_organizer_assigned, + blob_properties: &blob_properties, + every_blob_property: write == EventWrite::Copy + && content.data.event.embedded_size() > 0, + visible_to: None, + }; + if imported.is_requested() { + server_set.extend(imported.values( + &content.data.event, + account_id, + PENDING_DOCUMENT_ID, + )); + } + + let prepared_metadata = match context + .metadata + .prepare_new(self, metadata, metadata_access) + .await? { - Ok(_) => {} - Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { - return Ok(Err(SetError::over_quota())); - } - Err(err) => return Err(err.caused_by(trc::location!())), + Ok(prepared) => prepared, + Err(err) => return Ok(Err(err)), + }; + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + event.set_metadata_kinds(kinds); } // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::CalendarEvent, 1) + context.disk_quota.reserve(size as u64); + let document_id = batch.reserve_document_id(account_id, Collection::CalendarEvent); + let notify_calendar_ids = event.calendar_ids().collect::>(); + context + .uid_index + .record(&content.data.event, ¬ify_calendar_ids); + if may_have_viewers( + access_token, + account_id, + cache, + ¬ify_calendar_ids, + NotificationAudience::of_event(event.flags), + ) { + self.notify_direct_change( + access_token, + account_id, + DirectChange::Created { + event_id: document_id.into(), + current: content.data.event.clone(), + calendar_ids: notify_calendar_ids, + event_flags: event.flags, + }, + Some(cache), + &mut context.notification_quota, + batch, + ) .await .caused_by(trc::location!())?; + } + if let Some(prepared) = prepared_metadata { + context.metadata.write(prepared, document_id, batch)?; + } event .insert( - access_token.account_tenant_ids(), + content, + access_token.changed_by(), account_id, document_id, - next_email_alarm, + None, + next_email_alarms, batch, ) .caused_by(trc::location!())?; @@ -920,7 +2007,7 @@ impl CalendarEventSet for Server { itip_messages.queue(batch).caused_by(trc::location!())?; } - Ok(Ok(document_id)) + Ok(Ok((document_id, server_set))) } } @@ -935,90 +2022,207 @@ fn stamp_updated(ical: &mut ICalendar, timestamp: i64) { .iter_mut() .find(|entry| entry.name == ICalendarProperty::Dtstamp) { - entry.values = vec![ICalendarValue::PartialDateTime(Box::new(dtstamp.clone()))]; + entry.values = [ICalendarValue::PartialDateTime(dtstamp.clone())].into(); } else { component.add_dtstamp(dtstamp.clone()); } } } +fn stamp_created(ical: &mut ICalendar, timestamp: i64) { + for component in &mut ical.components { + if component.component_type.is_event_or_todo() + && component.property(&ICalendarProperty::Created).is_none() + { + component.add_property( + ICalendarProperty::Created, + ICalendarValue::PartialDateTime(PartialDateTime::from_utc_timestamp(timestamp)), + ); + } + } +} + +fn clamp_created(ical: &mut ICalendar, timestamp: i64) { + for entry in ical + .components + .iter_mut() + .filter(|component| component.component_type.is_event_or_todo()) + .flat_map(|component| component.entries.iter_mut()) + .filter(|entry| entry.name == ICalendarProperty::Created) + { + if entry + .values + .first() + .and_then(|value| value.as_partial_date_time()) + .and_then(|dt| dt.to_timestamp()) + .is_none_or(|created| created > timestamp) + { + entry.values = [ICalendarValue::PartialDateTime( + PartialDateTime::from_utc_timestamp(timestamp), + )] + .into(); + } + } +} + +fn assert_participants_limit( + ical: &ICalendar, + max_participants: usize, +) -> Result<(), SetError>> { + if ical.components.iter().any(|component| { + component + .entries + .iter() + .filter(|entry| entry.name == ICalendarProperty::Attendee) + .count() + > max_participants + }) { + Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::Participants) + .with_description(format!( + "An event cannot have more than {max_participants} participants." + ))) + } else { + Ok(()) + } +} + +fn instance_metadata(patches: &MetadataPatches) -> SetError> { + SetError::invalid_properties() + .with_property(if patches.has_shared() { + JSCalendarProperty::Metadata + } else { + JSCalendarProperty::PrivateMetadata + }) + .with_description("Metadata belongs to the base event and cannot be set on an instance.") +} + +pub(crate) fn too_many_events() -> SetError> { + SetError::over_quota().with_description(concat!( + "There are too many calendar events, ", + "please delete some before adding a new one." + )) +} + +fn too_many_calendars(max: usize) -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::CalendarIds) + .with_description(format!( + "A calendar event cannot belong to more than {max} calendars." + )) +} + +fn too_large_event(size: usize, max_size: usize) -> SetError> { + SetError::too_large().with_description(format!( + "Event size {size} exceeds the maximum allowed size of {max_size} bytes." + )) +} + +fn add_calendar_forbidden(calendar_id: u32) -> SetError> { + SetError::forbidden().with_description(format!( + "You are not allowed to add calendar events to calendar {}.", + Id::from(calendar_id) + )) +} + +fn misplaced_excluded() -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::Excluded) + .with_description("excluded can only be set in a recurrenceOverrides patch.") +} + +fn immutable_property(property: JSCalendarProperty) -> SetError> { + SetError::invalid_properties() + .with_property(property) + .with_description("This property is immutable.") +} + fn update_calendar_event<'x>( + cache: &GroupwareResources, personal_id: u32, - expected_id: Option, + target: PatchTarget, updates: Value<'x, JSCalendarProperty, JSCalendarValue>, event: &mut CalendarEvent, + content: &mut CalendarEventContent, js_calendar_group: &mut JSCalendar<'x, Id, BlobId>, -) -> Result, SetError>> { - // Extract event - let js_calendar_events = js_calendar_group - .0 - .as_object_mut() - .unwrap() - .get_mut(&Key::Property(JSCalendarProperty::Entries)) - .unwrap() - .as_array_mut() - .unwrap(); - - let js_calendar_event = if let Some(js_calendar_event) = js_calendar_events.first_mut() { - js_calendar_event - } else { - js_calendar_events.push(Value::Object(Map::new())); - js_calendar_events.first_mut().unwrap() +) -> Result>> { + let Some(js_calendar_event) = js_calendar_group.first_entry_mut() else { + return Err(SetError::invalid_properties() + .with_description("Failed to convert calendar event to JSCalendar.")); }; + let previous_use_default_alerts = content + .preferences(personal_id) + .is_some_and(|preferences| preferences.use_default_alerts()); + let mut use_default_alerts = previous_use_default_alerts; let mut utc_start = None; let mut utc_end = None; - let mut use_default_alerts = false; - let mut show_without_time = false; - let mut entries = js_calendar_event.as_object_mut().unwrap(); + let mut has_start = false; + let mut has_duration = false; + let mut has_recurrence_changes = false; - for (property, value) in updates.into_expanded_object() { - let Key::Property(property) = property else { - return Err(SetError::invalid_properties() - .with_property(property.to_owned()) - .with_description("Invalid property.")); + for (key, value) in updates.into_expanded_object() { + if target.write == EventWrite::Create && value.is_null() { + continue; + } + + let property = match key { + Key::Property(property) => property, + key => { + if !key.to_string().is_valid_property_name() { + return Err(SetError::invalid_properties() + .with_property(key.to_owned()) + .with_description("Invalid property.")); + } + if let Value::Object(entries) = js_calendar_event { + if value.is_null() { + entries.remove(&key); + } else { + entries.insert(key, value); + } + } + continue; + } }; match (property, value) { - (JSCalendarProperty::IsDraft, Value::Bool(set)) => { - if set { + (JSCalendarProperty::IsDraft, value @ (Value::Bool(_) | Value::Null)) => { + if matches!(value, Value::Bool(true)) { + if target.write == EventWrite::Update && event.flags & EVENT_DRAFT == 0 { + return Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::IsDraft) + .with_description( + "isDraft cannot be set to true on an event that is not a draft.", + )); + } event.flags |= EVENT_DRAFT; } else { event.flags &= !EVENT_DRAFT; } } - (JSCalendarProperty::MayInviteSelf, Value::Bool(set)) => { - if set { - event.flags |= EVENT_INVITE_SELF; + ( + property @ (JSCalendarProperty::MayInviteSelf + | JSCalendarProperty::MayInviteOthers + | JSCalendarProperty::HideAttendees), + value @ (Value::Bool(_) | Value::Null), + ) => { + let flag = match property { + JSCalendarProperty::MayInviteSelf => EVENT_INVITE_SELF, + JSCalendarProperty::MayInviteOthers => EVENT_INVITE_OTHERS, + _ => EVENT_HIDE_ATTENDEES, + }; + if value.is_same_boolean(true) { + event.flags |= flag; } else { - event.flags &= !EVENT_INVITE_SELF; + event.flags &= !flag; } } - (JSCalendarProperty::MayInviteOthers, Value::Bool(set)) => { - if set { - event.flags |= EVENT_INVITE_OTHERS; - } else { - event.flags &= !EVENT_INVITE_OTHERS; - } - } - (JSCalendarProperty::HideAttendees, Value::Bool(set)) => { - if set { - event.flags |= EVENT_HIDE_ATTENDEES; - } else { - event.flags &= !EVENT_HIDE_ATTENDEES; - } - } - (JSCalendarProperty::UseDefaultAlerts, Value::Bool(set)) => { - use_default_alerts = set; - if set { - event.preferences_mut(personal_id).flags |= PREF_USE_DEFAULT_ALERTS; - } else if let Some(preferences) = event - .preferences - .iter_mut() - .find(|p| p.account_id == personal_id) - { - preferences.flags &= !PREF_USE_DEFAULT_ALERTS; - } + (JSCalendarProperty::UseDefaultAlerts, value @ (Value::Bool(_) | Value::Null)) => { + use_default_alerts = value.is_same_boolean(true); + content + .preferences_mut(personal_id) + .set_use_default_alerts(use_default_alerts) + .map_err(user_data_error)?; } (JSCalendarProperty::UtcStart, Value::Element(JSCalendarValue::DateTime(start))) => { utc_start = Some(start.timestamp); @@ -1029,40 +2233,82 @@ fn update_calendar_event<'x>( (JSCalendarProperty::CalendarIds, value) => { patch_parent_ids(&mut event.names, None, value)?; } - (JSCalendarProperty::Pointer(pointer), value) => { - if matches!( - pointer.first(), - Some(JsonPointerItem::Key(Key::Property( - JSCalendarProperty::CalendarIds - ))) - ) { - let mut pointer = pointer.iter(); - pointer.next(); - patch_parent_ids(&mut event.names, pointer.next(), value)?; - } else if !js_calendar_event.patch_jptr(pointer.iter(), value) { + (JSCalendarProperty::Version, value) => { + if !matches!(&value, Value::Str(version) if version == JSCALENDAR_VERSION) { return Err(SetError::invalid_properties() - .with_property(JSCalendarProperty::Pointer(pointer)) - .with_description("Patch operation failed.")); + .with_property(JSCalendarProperty::Version) + .with_description(concat!( + "Unsupported JSCalendar version, only version ", + "2.0 is supported." + ))); + } + } + (JSCalendarProperty::Excluded, _) => { + return Err(misplaced_excluded()); + } + (JSCalendarProperty::Pointer(pointer), value) => { + let (pointer, value) = match pointer.first() { + Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::Excluded))) => { + return Err(misplaced_excluded()); + } + Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::Version))) => { + return Err(SetError::invalid_patch() + .with_property(JSCalendarProperty::Pointer(pointer)) + .with_description("The version property cannot be patched.")); + } + Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::CalendarIds))) => { + patch_parent_ids(&mut event.names, pointer.as_slice().get(1), value)?; + continue; + } + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceOverrides, + ))) => { + has_recurrence_changes = true; + pointer.validate_override_pointer(&value)?; + match js_calendar_event.acknowledged_alert_pointer(&pointer) { + Some(alert_pointer) => (alert_pointer, value.into_acknowledged()), + None => (pointer, value), + } + } + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::RecurrenceRule, + ))) => { + has_recurrence_changes = true; + (pointer, value) + } + _ => (pointer, value), + }; + if !js_calendar_event.patch_jptr(pointer.iter(), value) { + return Err(if target.write == EventWrite::Update { + SetError::invalid_patch() + } else { + SetError::invalid_properties() + } + .with_property(JSCalendarProperty::Pointer(pointer)) + .with_description("Patch operation failed.")); } - entries = js_calendar_event.as_object_mut().unwrap(); } (JSCalendarProperty::Id, value) => { - if !expected_id.is_some_and(|expected| crate::matches_id(&value, expected)) { - return Err(SetError::invalid_properties() - .with_property(JSCalendarProperty::Id) - .with_description("This property is immutable.")); + if !target + .id + .is_some_and(|expected| crate::matches_id(&value, expected)) + { + return Err(immutable_property(JSCalendarProperty::Id)); } } - ( - property @ (JSCalendarProperty::BaseEventId - | JSCalendarProperty::IsOrigin - | JSCalendarProperty::Method), - value, - ) => { - if entries.get(&Key::Property(property.clone())) != Some(&value) { - return Err(SetError::invalid_properties() - .with_property(property) - .with_description("This property is immutable.")); + (JSCalendarProperty::IsOrigin, value) => { + if !value.is_null() + && !matches!( + (value.as_bool(), target.is_origin), + (Some(value), Some(is_origin)) if value == is_origin + ) + { + return Err(immutable_property(JSCalendarProperty::IsOrigin)); + } + } + (property @ (JSCalendarProperty::BaseEventId | JSCalendarProperty::Method), value) => { + if !value.is_null() { + return Err(immutable_property(property)); } } ( @@ -1079,86 +2325,150 @@ fn update_calendar_event<'x>( .with_property(property) .with_description("Invalid value.")); } - ( - property @ (JSCalendarProperty::Locations | JSCalendarProperty::Participants), - Value::Object(values), - ) => { - for (_, value) in values.iter() { - if let Some(values) = value - .as_object_and_get(&Key::Property(JSCalendarProperty::Links)) - .and_then(|v| v.as_object()) - { - for (_, value) in values.iter() { - if value.as_object().is_some_and(|v| { - v.keys() - .any(|k| matches!(k, Key::Property(JSCalendarProperty::BlobId))) - }) { + (property, value) => { + match &property { + JSCalendarProperty::Start => { + has_start = true; + } + JSCalendarProperty::Duration => { + has_duration = true; + } + JSCalendarProperty::RecurrenceOverrides => { + has_recurrence_changes = true; + match &value { + Value::Object(overrides) => overrides + .values() + .try_for_each(OverridePatch::validate_override_patch)?, + Value::Null => {} + _ => { return Err(SetError::invalid_properties() - .with_property(property) - .with_description("blobIds in links is not supported.")); + .with_property(JSCalendarProperty::RecurrenceOverrides) + .with_description("Invalid value.")); } } } - } - entries.insert(property, Value::Object(values)); - } - (property, value) => { - if let (JSCalendarProperty::ShowWithoutTime, Value::Bool(set)) = (&property, &value) - { - show_without_time = *set; + JSCalendarProperty::RecurrenceId | JSCalendarProperty::RecurrenceRule => { + has_recurrence_changes = true; + } + _ => {} } - entries.insert(property, value); + if let Value::Object(entries) = js_calendar_event { + entries.insert(property, value); + } } } } - // Validate UTC start/end - if let (Some(mut start), Some(mut end)) = (utc_start, utc_end) { - if start >= end { - return Err(SetError::invalid_properties() - .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd]) - .with_description("utcStart must be before utcEnd.")); - } + let Value::Object(entries) = js_calendar_event else { + return Err(SetError::invalid_properties() + .with_description("Failed to convert calendar event to JSCalendar.")); + }; - if let Some(timezone) = entries + if has_recurrence_changes + && entries + .get(&Key::Property(JSCalendarProperty::RecurrenceId)) + .is_some_and(|value| !value.is_null()) + { + let recurrence_rule = entries + .get(&Key::Property(JSCalendarProperty::RecurrenceRule)) + .is_some_and(|value| !value.is_null()); + let recurrence_overrides = entries + .get(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .is_some_and(|value| { + value + .as_object() + .is_some_and(|overrides| !overrides.is_empty()) + }); + if recurrence_rule || recurrence_overrides { + return Err(SetError::invalid_properties() + .with_properties([ + JSCalendarProperty::RecurrenceId, + if recurrence_rule { + JSCalendarProperty::RecurrenceRule + } else { + JSCalendarProperty::RecurrenceOverrides + }, + ]) + .with_description( + "recurrenceId cannot be set together with recurrenceRule or recurrenceOverrides.", + )); + } + } + + // Validate UTC start/end + if utc_start.is_some() && has_start { + return Err(SetError::invalid_properties() + .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::Start]) + .with_description("utcStart cannot be set together with start.")); + } + if utc_end.is_some() && has_duration { + return Err(SetError::invalid_properties() + .with_properties([JSCalendarProperty::UtcEnd, JSCalendarProperty::Duration]) + .with_description("utcEnd cannot be set together with duration.")); + } + if utc_start.is_some() || utc_end.is_some() { + let tz = match entries .get(&Key::Property(JSCalendarProperty::TimeZone)) .and_then(|v| v.as_str()) .and_then(|tz| Tz::from_str(tz.as_ref()).ok()) { - if let Some(dt_start) = - DateTime::from_timestamp(start, 0).map(|dt| dt.with_timezone(&timezone)) - { - start = dt_start.naive_local().and_utc().timestamp(); + Some(tz) => tz, + None => { + let tz = calendars_time_zone(cache, personal_id, &event.names); + entries.insert( + Key::Property(JSCalendarProperty::TimeZone), + Value::Str(tz.name().unwrap_or(Cow::Borrowed("Etc/UTC"))), + ); + tz } - if let Some(dt_end) = - DateTime::from_timestamp(end, 0).map(|dt| dt.with_timezone(&timezone)) - { - end = dt_end.naive_local().and_utc().timestamp(); - } - } else { + }; + + if let Some(start) = utc_start { + let local_start = Timestamp::from_second(start) + .map(|start| tz.from_timestamp(start).naive_timestamp()) + .map_err(|_| { + SetError::invalid_properties() + .with_property(JSCalendarProperty::UtcStart) + .with_description("Invalid utcStart value.") + })?; entries.insert( - Key::Property(JSCalendarProperty::TimeZone), - Value::Str(Cow::Borrowed("Etc/UTC")), + Key::Property(JSCalendarProperty::Start), + Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( + local_start, + true, + ))), ); } - entries.insert( - Key::Property(JSCalendarProperty::Start), - Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( - start, true, - ))), - ); - entries.insert( - Key::Property(JSCalendarProperty::Duration), - Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_seconds( - end - start, - ))), - ); - } else if utc_start.is_some() || utc_end.is_some() { - return Err(SetError::invalid_properties() - .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd]) - .with_description("Both utcStart and utcEnd must be provided.")); + if let Some(end) = utc_end { + let start = match utc_start { + Some(start) => start, + None => entries + .get(&Key::Property(JSCalendarProperty::Start)) + .and_then(|v| v.as_element()) + .and_then(|v| match v { + JSCalendarValue::DateTime(start) => resolve_local(tz, start.timestamp), + _ => None, + }) + .ok_or_else(|| { + SetError::invalid_properties() + .with_property(JSCalendarProperty::UtcEnd) + .with_description("utcEnd requires the event to have a start.") + })?, + }; + if end < start { + return Err(SetError::invalid_properties() + .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd]) + .with_description("utcEnd cannot be before the start of the event.")); + } + entries.insert( + Key::Property(JSCalendarProperty::Duration), + Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_seconds( + end - start, + ))), + ); + } } // Make sure the calendar_event belongs to at least one calendar @@ -1168,22 +2478,65 @@ fn update_calendar_event<'x>( .with_description("Event has to belong to at least one calendar.")); } - Ok(use_default_alerts.then_some(show_without_time)) + Ok(use_default_alerts != previous_use_default_alerts) +} + +fn calendars_time_zone(cache: &GroupwareResources, personal_id: u32, names: &[DavName]) -> Tz { + let mut time_zones = names.iter().map(|name| { + cache + .container_resource_by_id(name.parent_id) + .and_then(|resource| { + resource + .calendar_preferences(personal_id) + .map(|preferences| preferences.tz) + }) + }); + let first = time_zones.next().flatten(); + first + .filter(|tz| time_zones.all(|other| other == Some(*tz))) + .unwrap_or(Tz::UTC) +} + +struct EventUpdates<'x> { + pending: Vec>, + failed: Vec, } struct EventUpdate<'x> { document_id: u32, - base_id: Option, - base_patch: Option, JSCalendarValue>>, - instances: Vec>, + ops: EventOps<'x>, +} + +enum EventOps<'x> { + Base { id: Id, patch: EventValue<'x> }, + Instances(Vec>), +} + +enum EventOp<'x> { + Base { id: Id, patch: EventValue<'x> }, + Instance(InstanceOp<'x>), } struct InstanceOp<'x> { id: Id, recurrence_key: RecurrenceKey, - patch: Option, JSCalendarValue>>, target: Option, - is_destroy: bool, + action: InstanceAction<'x>, +} + +enum InstanceAction<'x> { + Update { + patch: EventValue<'x>, + server_set: Option, + }, + Destroy, +} + +struct InstanceServerSet { + recurrence_id: i64, + client: TrackedValues, + before: TrackedValues, + blob_properties: BlobProperties, } struct InstanceTarget { @@ -1191,6 +2544,7 @@ struct InstanceTarget { recurrence_id: i64, recurrence_id_naive: i64, start_naive: i64, + start: Option, duration: i64, } @@ -1208,48 +2562,260 @@ enum InstanceResolution { NotFound, } -trait JSCalendarEvent<'x> { - fn event_mut( - &mut self, - ) -> Option<&mut Value<'x, JSCalendarProperty, JSCalendarValue>>; +trait AlertAcknowledgement { + fn acknowledged_alert_pointer( + &self, + pointer: &JsonPointer>, + ) -> Option>>; + + fn overrides_alerts(&self, recurrence_id: &Key<'_, JSCalendarProperty>) -> Option; + + fn into_acknowledged(self) -> Self; } -impl<'x> JSCalendarEvent<'x> for JSCalendar<'x, Id, BlobId> { - fn event_mut( +impl AlertAcknowledgement for Value<'_, JSCalendarProperty, JSCalendarValue> { + fn acknowledged_alert_pointer( + &self, + pointer: &JsonPointer>, + ) -> Option>> { + match pointer.as_slice() { + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::RecurrenceOverrides)), + JsonPointerItem::Key(recurrence_id), + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Pointer(alert_pointer))), + ] if alert_pointer.is_acknowledgement() + && matches!( + recurrence_id, + Key::Property(JSCalendarProperty::DateTime(_)) + ) + && self.overrides_alerts(recurrence_id) != Some(true) => + { + Some(alert_pointer.clone()) + } + _ => None, + } + } + + fn overrides_alerts(&self, recurrence_id: &Key<'_, JSCalendarProperty>) -> Option { + self.as_object_and_get(&Key::Property(JSCalendarProperty::RecurrenceOverrides)) + .and_then(|overrides| overrides.as_object_and_get(recurrence_id)) + .and_then(Value::as_object) + .map(|instance| { + instance.keys().any(|property| match property { + Key::Property(JSCalendarProperty::Alerts) => true, + Key::Property(JSCalendarProperty::Pointer(pointer)) => matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::Alerts + ))) + ), + _ => false, + }) + }) + } + + fn into_acknowledged(self) -> Self { + if let Value::Str(text) = &self + && let Some(acknowledged) = as Element>::try_parse::< + JSCalendarProperty, + >( + &Key::Property(JSCalendarProperty::Acknowledged), text + ) + { + Value::Element(acknowledged) + } else { + self + } + } +} + +trait ComponentSequences { + fn has_same_sequences(&self, other: &ICalendar) -> bool; +} + +impl ComponentSequences for ICalendar { + fn has_same_sequences(&self, other: &ICalendar) -> bool { + let sequence = |component: &ICalendarComponent| { + component + .property(&ICalendarProperty::Sequence) + .and_then(|entry| entry.values.first()) + .and_then(ICalendarValue::as_integer) + }; + self.scheduling_components() + .map(sequence) + .eq(other.scheduling_components().map(sequence)) + } +} + +trait AcknowledgementPointer { + fn is_acknowledgement(&self) -> bool; +} + +impl AcknowledgementPointer for JsonPointer> { + fn is_acknowledgement(&self) -> bool { + matches!( + self.as_slice(), + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Alerts)), + JsonPointerItem::Key(_), + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Acknowledged)), + ] + ) + } +} + +impl<'x> EventUpdates<'x> { + fn with_capacity(capacity: usize) -> Self { + EventUpdates { + pending: Vec::with_capacity(capacity), + failed: Vec::new(), + } + } + + fn push( &mut self, - ) -> Option<&mut Value<'x, JSCalendarProperty, JSCalendarValue>> { - self.0 - .as_object_mut()? - .get_mut(&Key::Property(JSCalendarProperty::Entries))? - .as_array_mut()? - .first_mut() + has_synthetic_ids: bool, + op: EventOp<'x>, + response: &mut SetResponse, + ) { + let document_id = op.id().document_id(); + if self.failed.contains(&document_id) { + op.fail(response, mixed_base_and_instances()); + return; + } + let index = has_synthetic_ids + .then(|| { + self.pending + .iter() + .position(|update| update.document_id == document_id) + }) + .flatten(); + let Some(index) = index else { + self.pending.push(EventUpdate { + document_id, + ops: op.into_ops(), + }); + return; + }; + + let is_base_op = matches!(op, EventOp::Base { .. }); + if matches!(self.pending[index].ops, EventOps::Base { .. }) != is_base_op { + self.failed.push(document_id); + self.pending + .remove(index) + .fail(response, mixed_base_and_instances()); + op.fail(response, mixed_base_and_instances()); + } else if is_base_op { + response.not_updated.append( + op.id(), + SetError::invalid_properties() + .with_property(JSCalendarProperty::Id) + .with_description("Duplicate event id."), + ); + } else if let (EventOps::Instances(instances), EventOp::Instance(instance)) = + (&mut self.pending[index].ops, op) + { + instances.push(instance); + } } } impl<'x> EventUpdate<'x> { - fn for_document<'y>( - updates: &'y mut Vec>, - document_id: u32, - has_synthetic_ids: bool, - ) -> &'y mut EventUpdate<'x> { - let index = if has_synthetic_ids { - updates - .iter() - .position(|update| update.document_id == document_id) - } else { - None - }; + fn base_id(&self) -> Option { + match &self.ops { + EventOps::Base { id, .. } => Some(*id), + EventOps::Instances(_) => None, + } + } - match index { - Some(index) => &mut updates[index], - None => { - updates.push(EventUpdate { - document_id, - base_id: None, - base_patch: None, - instances: Vec::new(), + fn base_patch(&self) -> Option<&EventValue<'x>> { + match &self.ops { + EventOps::Base { patch, .. } => Some(patch), + EventOps::Instances(_) => None, + } + } + + fn is_empty_base_update(&self) -> bool { + match &self.ops { + EventOps::Base { id, patch } => patch + .as_object() + .is_some_and(|patch| is_empty_update::(patch, *id)), + EventOps::Instances(_) => false, + } + } + + fn take_base_patch(&mut self) -> EventValue<'x> { + match &mut self.ops { + EventOps::Base { patch, .. } => std::mem::take(patch), + EventOps::Instances(_) => EventValue::Null, + } + } + + fn has_instances(&self) -> bool { + matches!(&self.ops, EventOps::Instances(instances) if !instances.is_empty()) + } + + fn patches_calendar_ids(&self) -> bool { + let EventOps::Base { patch, .. } = &self.ops else { + return false; + }; + patch.as_object().is_some_and(|patch| { + patch.keys().any(|key| match key { + Key::Property(JSCalendarProperty::CalendarIds) => true, + Key::Property(JSCalendarProperty::Pointer(pointer)) => matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property( + JSCalendarProperty::CalendarIds + ))) + ), + _ => false, + }) + }) + } + + fn per_user_patch_changes(&self) -> Option { + match &self.ops { + EventOps::Base { patch, .. } => EventChanges::from_per_user_patch(patch), + EventOps::Instances(instances) => { + let mut changes = EventChanges::default(); + for instance in instances { + let InstanceAction::Update { patch, .. } = &instance.action else { + return None; + }; + if !EventChanges::from_per_user_patch(patch)?.is_empty() { + changes.insert_per_user(); + } + } + Some(changes) + } + } + } + + fn instances_mut(&mut self) -> &mut [InstanceOp<'x>] { + match &mut self.ops { + EventOps::Instances(instances) => instances.as_mut_slice(), + EventOps::Base { .. } => &mut [], + } + } + + fn track_instance_values( + &mut self, + previous_event: &ICalendar, + is_stored_blob_id: impl Fn(&BlobId) -> bool, + ) { + for instance in self.instances_mut() { + let Some(recurrence_id) = instance.target.as_ref().map(|target| target.recurrence_id) + else { + continue; + }; + if let InstanceAction::Update { patch, server_set } = &mut instance.action { + *server_set = Some(InstanceServerSet { + recurrence_id, + client: TrackedValues::from_patch(patch), + before: TrackedValues::default() + .with_component(previous_event.instance_component(recurrence_id)), + blob_properties: BlobProperties::from_patch(patch, &is_stored_blob_id), }); - updates.last_mut().unwrap() } } } @@ -1259,23 +2825,26 @@ impl<'x> EventUpdate<'x> { response: &mut SetResponse, err: SetError>, ) { - if let Some(id) = self.base_id { - response.not_updated.append(id, err.clone()); - } - for instance in &self.instances { - instance.fail(response, err.clone()); + match &self.ops { + EventOps::Base { id, .. } => response.not_updated.append(*id, err), + EventOps::Instances(instances) => { + for instance in instances { + instance.fail(response, err.clone()); + } + } } } fn succeed(&self, response: &mut SetResponse) { - if let Some(id) = self.base_id { - response.updated.append(id, None); - } - for instance in &self.instances { - if instance.is_destroy { - response.destroyed.push(instance.id); - } else { - response.updated.append(instance.id, None); + match &self.ops { + EventOps::Base { id, .. } => response.updated.append(*id, None), + EventOps::Instances(instances) => { + for instance in instances { + match instance.action { + InstanceAction::Destroy => response.destroyed.push(instance.id), + InstanceAction::Update { .. } => response.updated.append(instance.id, None), + } + } } } } @@ -1285,18 +2854,20 @@ impl<'x> EventUpdate<'x> { data: &CalendarEventData, response: &mut SetResponse, ) -> InstancePlan { - let mut recurrence_keys = self - .instances + let EventOps::Instances(instances) = &mut self.ops else { + return InstancePlan::BaseEvent; + }; + let mut recurrence_keys = instances .iter() .map(|instance| instance.recurrence_key) .collect::>(); let expansions = data .expand_from_ids(&mut recurrence_keys, Tz::UTC) .unwrap_or_default(); - let uid = data.event.uids().next(); + let uid = data.event.object_uid(); let mut has_base_event = false; - self.instances.retain_mut(|instance| { + instances.retain_mut(|instance| { let mut matches = expansions .iter() .filter(|expansion| expansion.recurrence_key() == Some(instance.recurrence_key)); @@ -1334,30 +2905,27 @@ impl<'x> EventUpdate<'x> { }); if has_base_event { - if self.instances.len() > 1 { - self.fail( - response, - SetError::invalid_properties() - .with_property(JSCalendarProperty::Id) - .with_description(concat!( - "A base event and its instances cannot be modified ", - "in the same request." - )), - ); + if instances.len() > 1 { + self.fail(response, mixed_base_and_instances()); return InstancePlan::Nothing; } - let instance = self.instances.pop().unwrap(); - return if instance.is_destroy { - InstancePlan::DestroyEvent(instance.id) - } else { - self.base_id = Some(instance.id); - self.base_patch = instance.patch; - InstancePlan::BaseEvent + let Some(instance) = instances.pop() else { + return InstancePlan::Nothing; + }; + return match instance.action { + InstanceAction::Destroy => InstancePlan::DestroyEvent(instance.id), + InstanceAction::Update { patch, .. } => { + self.ops = EventOps::Base { + id: instance.id, + patch, + }; + InstancePlan::BaseEvent + } }; } - if self.instances.is_empty() { + if instances.is_empty() { InstancePlan::Nothing } else { InstancePlan::Instances @@ -1367,14 +2935,20 @@ impl<'x> EventUpdate<'x> { fn apply_instances( &mut self, js_calendar_group: &mut JSCalendar<'x, Id, BlobId>, + metadata: &InstanceMetadata<'_>, response: &mut SetResponse, ) -> bool { - let Some(js_calendar_event) = js_calendar_group.event_mut() else { - self.fail( - response, - SetError::invalid_properties() - .with_description("Failed to convert calendar event to JSCalendar."), - ); + let EventOps::Instances(instances) = &mut self.ops else { + return false; + }; + let Some(js_calendar_event) = js_calendar_group.first_entry_mut() else { + for instance in instances.iter() { + instance.fail( + response, + SetError::invalid_properties() + .with_description("Failed to convert calendar event to JSCalendar."), + ); + } return false; }; let tz = js_calendar_event @@ -1382,11 +2956,16 @@ impl<'x> EventUpdate<'x> { .and_then(|tz| tz.as_str()) .and_then(|tz| Tz::from_str(tz.as_ref()).ok()) .unwrap_or(Tz::UTC); - let duration = js_calendar_event + let base_duration = js_calendar_event .as_object_and_get(&Key::Property(JSCalendarProperty::Duration)) - .cloned(); + .and_then(Value::as_element) + .and_then(|duration| match duration { + JSCalendarValue::Duration(duration) => duration.to_nominal(), + _ => None, + }) + .unwrap_or(NominalDuration::new(0, 0)); - self.instances.retain_mut(|instance| { + instances.retain_mut(|instance| { let Some(target) = instance.target.take() else { return false; }; @@ -1411,8 +2990,9 @@ impl<'x> EventUpdate<'x> { match target.apply( js_calendar_event, key, - duration.as_ref(), - instance.patch.take(), + base_duration, + metadata, + instance.take_patch(), instance.id, ) { Ok(_) => true, @@ -1423,24 +3003,94 @@ impl<'x> EventUpdate<'x> { } }); - !self.instances.is_empty() + !instances.is_empty() } } -impl InstanceOp<'_> { +impl<'x> EventOp<'x> { + fn id(&self) -> Id { + match self { + EventOp::Base { id, .. } => *id, + EventOp::Instance(instance) => instance.id, + } + } + + fn into_ops(self) -> EventOps<'x> { + match self { + EventOp::Base { id, patch } => EventOps::Base { id, patch }, + EventOp::Instance(instance) => EventOps::Instances(vec![instance]), + } + } + fn fail( &self, response: &mut SetResponse, err: SetError>, ) { - if self.is_destroy { - response.not_destroyed.append(self.id, err); - } else { - response.not_updated.append(self.id, err); + match self { + EventOp::Base { id, .. } => response.not_updated.append(*id, err), + EventOp::Instance(instance) => instance.fail(response, err), } } } +impl<'x> InstanceOp<'x> { + fn fail( + &self, + response: &mut SetResponse, + err: SetError>, + ) { + match self.action { + InstanceAction::Destroy => response.not_destroyed.append(self.id, err), + InstanceAction::Update { .. } => response.not_updated.append(self.id, err), + } + } + + fn take_patch(&mut self) -> Option> { + match &mut self.action { + InstanceAction::Update { patch, .. } => Some(std::mem::take(patch)), + InstanceAction::Destroy => None, + } + } + + fn take_server_set(&mut self) -> Option { + match &mut self.action { + InstanceAction::Update { server_set, .. } => server_set.take(), + InstanceAction::Destroy => None, + } + } +} + +fn instance_imported_values( + data: &CalendarEventData, + recurrence_key: RecurrenceKey, + imported: &ImportedValues<'_>, + account_id: u32, + document_id: u32, +) -> ServerSetValues { + let mut recurrence_keys = AHashSet::from_iter([recurrence_key]); + let Some(ical) = data + .expand_from_ids(&mut recurrence_keys, Tz::Floating) + .unwrap_or_default() + .iter() + .find(|expansion| expansion.recurrence_key() == Some(recurrence_key)) + .and_then(|expansion| data.instance(expansion, None, InstanceBinaries::Keep)) + else { + return ServerSetValues::new(); + }; + + imported.values(&ical, account_id, document_id).collect() +} + +fn mixed_base_and_instances() -> SetError> { + SetError::invalid_properties() + .with_property(JSCalendarProperty::Id) + .with_description(concat!( + "A base event and its instances cannot be modified ", + "in the same request." + )) +} + impl InstanceTarget { fn resolve( expansion: &CalendarEventExpansion, @@ -1482,6 +3132,9 @@ impl InstanceTarget { recurrence_id: recurrence_id.utc, recurrence_id_naive: recurrence_id.naive, start_naive: expansion.start_naive, + start: expansion + .flags + .resolve_start(expansion.start_tz, expansion.start_naive), duration: expansion.end - expansion.start, }) } @@ -1510,12 +3163,14 @@ impl InstanceTarget { &self, js_calendar_event: &mut Value<'x, JSCalendarProperty, JSCalendarValue>, key: JSCalendarDateTime, - duration: Option<&Value<'x, JSCalendarProperty, JSCalendarValue>>, + base_duration: NominalDuration, + metadata: &InstanceMetadata<'_>, patch: Option, JSCalendarValue>>, id: Id, ) -> Result<(), SetError>> { let invalid_event = || SetError::invalid_properties().with_description("Failed to parse stored event."); + let key_timestamp = key.timestamp; let key = Key::Property(JSCalendarProperty::DateTime(key)); let patch = match patch { @@ -1546,8 +3201,38 @@ impl InstanceTarget { } }; - for (property, value) in patch.iter() { - Self::validate(property, value, id)?; + let redirects_acknowledgements = js_calendar_event.overrides_alerts(&key) != Some(true); + let view = InstanceView { + event: js_calendar_event.as_object().ok_or_else(invalid_event)?, + metadata, + id, + recurrence_ids: [self.recurrence_id_naive, key_timestamp], + }; + let mut overrides = Vec::with_capacity(patch.len()); + let mut acknowledgements = Vec::new(); + for (property, value) in patch.into_vec() { + if !view.is_override_key(&property, &value)? { + continue; + } + match property { + Key::Property(JSCalendarProperty::Pointer(pointer)) + if redirects_acknowledgements && pointer.is_acknowledgement() => + { + acknowledgements.push((pointer, value)); + } + property => overrides.push((property, value)), + } + } + + for (pointer, value) in acknowledgements { + if !js_calendar_event.patch_jptr(pointer.iter(), value) { + return Err(SetError::invalid_patch() + .with_property(JSCalendarProperty::Pointer(pointer)) + .with_description("Patch operation failed.")); + } + } + if overrides.is_empty() { + return Ok(()); } let instance = js_calendar_event @@ -1563,7 +3248,9 @@ impl InstanceTarget { .as_object_mut() .ok_or_else(invalid_event)?; - if !instance.contains_key(&Key::Property(JSCalendarProperty::Start)) { + if self.start_naive != key_timestamp + && !instance.contains_key(&Key::Property(JSCalendarProperty::Start)) + { instance.insert( Key::Property(JSCalendarProperty::Start), Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( @@ -1572,111 +3259,28 @@ impl InstanceTarget { ))), ); } - if !instance.contains_key(&Key::Property(JSCalendarProperty::Duration)) { + let spans_base_duration = self.start.is_some_and(|start| { + start + .checked_add_nominal(base_duration) + .is_some_and(|end| end.signed_duration_since(start).as_secs() == self.duration) + }); + if !spans_base_duration + && !instance.contains_key(&Key::Property(JSCalendarProperty::Duration)) + { instance.insert( Key::Property(JSCalendarProperty::Duration), - duration.cloned().unwrap_or_else(|| { - Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_seconds( - self.duration, - ))) - }), + Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_seconds( + self.duration, + ))), ); } - for (property, value) in patch.into_vec() { - if matches!(Self::validate(&property, &value, id), Ok(true)) { - instance.insert(property, value); - } + for (property, value) in overrides { + instance.insert(property, value); } Ok(()) } - - fn validate( - property: &Key<'_, JSCalendarProperty>, - value: &Value<'_, JSCalendarProperty, JSCalendarValue>, - id: Id, - ) -> Result>> { - let Key::Property(property) = property else { - return Err(SetError::invalid_properties() - .with_property(property.to_owned()) - .with_description("Invalid property.")); - }; - let rejected = SetError::invalid_properties() - .with_property(property.clone()) - .with_description("This property cannot be modified on a single occurrence."); - - match property { - JSCalendarProperty::Id => { - if crate::matches_id(value, id) { - Ok(false) - } else { - Err(SetError::invalid_properties() - .with_property(JSCalendarProperty::Id) - .with_description("This property is immutable.")) - } - } - JSCalendarProperty::Pointer(pointer) => { - let mut tokens = pointer.iter(); - let (Some(JsonPointerItem::Key(Key::Property(first))), third) = - (tokens.next(), tokens.nth(1)) - else { - return Err(rejected); - }; - - if Self::is_event_property(first) { - Err(rejected) - } else { - Ok(!Self::is_inherited_property(first) - && !matches!( - (first, third), - ( - JSCalendarProperty::Participants, - Some(JsonPointerItem::Key(Key::Property( - JSCalendarProperty::CalendarAddress - ))) - ) - )) - } - } - property if Self::is_event_property(property) => Err(rejected), - property => Ok(!Self::is_inherited_property(property)), - } - } - - fn is_event_property(property: &JSCalendarProperty) -> bool { - matches!( - property, - JSCalendarProperty::BaseEventId - | JSCalendarProperty::CalendarIds - | JSCalendarProperty::IsDraft - | JSCalendarProperty::IsOrigin - | JSCalendarProperty::UtcStart - | JSCalendarProperty::UtcEnd - | JSCalendarProperty::UseDefaultAlerts - | JSCalendarProperty::MayInviteSelf - | JSCalendarProperty::MayInviteOthers - | JSCalendarProperty::HideAttendees - ) - } - - fn is_inherited_property(property: &JSCalendarProperty) -> bool { - matches!( - property, - JSCalendarProperty::Type - | JSCalendarProperty::Method - | JSCalendarProperty::OrganizerCalendarAddress - | JSCalendarProperty::Privacy - | JSCalendarProperty::ProdId - | JSCalendarProperty::RecurrenceId - | JSCalendarProperty::RecurrenceIdTimeZone - | JSCalendarProperty::SentBy - | JSCalendarProperty::Uid - | JSCalendarProperty::RecurrenceOverrides - | JSCalendarProperty::RecurrenceRule - | JSCalendarProperty::RelatedTo - ) - } } fn patch_parent_ids( @@ -1729,30 +3333,6 @@ fn patch_parent_ids( } } -fn default_alert_to_ical(alert: &ArchivedDefaultAlert) -> ICalendarComponent { - let flags = alert.flags.to_native(); - ICalendarComponent { - component_type: ICalendarComponentType::VAlarm, - entries: vec![ - ICalendarEntry::new(ICalendarProperty::Action).with_value( - if flags & ALERT_EMAIL != 0 { - ICalendarValue::Action(ICalendarAction::Email) - } else { - ICalendarValue::Action(ICalendarAction::Display) - }, - ), - ICalendarEntry::new(ICalendarProperty::Trigger) - .with_param_opt((flags & ALERT_RELATIVE_TO_END != 0).then_some( - ICalendarParameter::related(ICalendarParameterValue::Related( - ICalendarRelated::End, - )), - )) - .with_value(ICalendarValue::Duration(alert.offset.to_native())), - ], - component_ids: vec![], - } -} - fn generate_uid() -> String { let mut bytes = rand::random::<[u8; 16]>(); bytes[6] = (bytes[6] & 0x0f) | 0x40; diff --git a/crates/jmap/src/calendar_event/user.rs b/crates/jmap/src/calendar_event/user.rs new file mode 100644 index 00000000..7a9b809a --- /dev/null +++ b/crates/jmap/src/calendar_event/user.rs @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use calcard::jscalendar::JSCalendarProperty; +use groupware::calendar::{ + MAX_USER_ALERTS, MAX_USER_INSTANCES, MAX_USER_KEYWORD_LEN, MAX_USER_KEYWORDS, + user::UserDataError, +}; +use jmap_proto::error::set::SetError; +use types::id::Id; + +pub(super) fn user_data_error(err: UserDataError) -> SetError> { + let (property, description) = match err { + UserDataError::TooManyKeywords => ( + JSCalendarProperty::Keywords, + format!("An event cannot have more than {MAX_USER_KEYWORDS} keywords."), + ), + UserDataError::KeywordTooLong => ( + JSCalendarProperty::Keywords, + format!("Keywords cannot be longer than {MAX_USER_KEYWORD_LEN} bytes."), + ), + UserDataError::TooManyAlerts => ( + JSCalendarProperty::Alerts, + format!("An event cannot have more than {MAX_USER_ALERTS} alerts."), + ), + UserDataError::TooManyInstances => ( + JSCalendarProperty::RecurrenceOverrides, + format!( + "Personal properties cannot be set on more than {MAX_USER_INSTANCES} occurrences." + ), + ), + UserDataError::InvalidColor => ( + JSCalendarProperty::Color, + "The color is not a valid CSS color.".to_string(), + ), + }; + + SetError::invalid_properties() + .with_property(property) + .with_description(description) +} diff --git a/crates/jmap/src/calendar_event/validate.rs b/crates/jmap/src/calendar_event/validate.rs new file mode 100644 index 00000000..aab60f5a --- /dev/null +++ b/crates/jmap/src/calendar_event/validate.rs @@ -0,0 +1,328 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use calcard::jscalendar::{JSCALENDAR_VERSION, JSCalendarProperty, JSCalendarValue}; +use common::DavName; +use groupware::calendar::{ + EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, +}; +use jmap_proto::error::set::SetError; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; +use std::borrow::Cow; +use types::{blob::BlobId, id::Id}; + +type EventValue<'x> = Value<'x, JSCalendarProperty, JSCalendarValue>; +type EventMap<'x> = Map<'x, JSCalendarProperty, JSCalendarValue>; +type EventKey<'x> = Key<'x, JSCalendarProperty>; +type EventError = SetError>; + +pub(super) trait EventProperty { + fn is_event_metadata(&self) -> bool; + + fn is_series_property(&self) -> bool; +} + +impl EventProperty for JSCalendarProperty { + fn is_event_metadata(&self) -> bool { + matches!( + self, + JSCalendarProperty::Id + | JSCalendarProperty::BaseEventId + | JSCalendarProperty::CalendarIds + | JSCalendarProperty::IsDraft + | JSCalendarProperty::IsOrigin + | JSCalendarProperty::UtcStart + | JSCalendarProperty::UtcEnd + | JSCalendarProperty::UseDefaultAlerts + | JSCalendarProperty::MayInviteSelf + | JSCalendarProperty::MayInviteOthers + | JSCalendarProperty::HideAttendees + | JSCalendarProperty::Metadata + | JSCalendarProperty::PrivateMetadata + ) + } + + fn is_series_property(&self) -> bool { + matches!( + self, + JSCalendarProperty::Type + | JSCalendarProperty::Method + | JSCalendarProperty::OrganizerCalendarAddress + | JSCalendarProperty::Privacy + | JSCalendarProperty::ProdId + | JSCalendarProperty::RecurrenceId + | JSCalendarProperty::RecurrenceIdTimeZone + | JSCalendarProperty::SentBy + | JSCalendarProperty::Uid + | JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::RecurrenceRule + ) + } +} + +pub(super) trait OverridePatch { + fn validate_override_patch(&self) -> Result<(), EventError>; +} + +impl OverridePatch for EventValue<'_> { + fn validate_override_patch(&self) -> Result<(), EventError> { + match self { + Value::Object(patch) => patch + .keys() + .try_for_each(OverridePatch::validate_override_patch), + _ => Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::RecurrenceOverrides) + .with_description("Expected a patch object.")), + } + } +} + +impl OverridePatch for EventKey<'_> { + fn validate_override_patch(&self) -> Result<(), EventError> { + let rejected = match self { + Key::Property(JSCalendarProperty::Pointer(pointer)) => { + pointer.rejected_override_token() + } + Key::Property(property) + if property.is_event_metadata() || property.is_series_property() => + { + Some(property) + } + _ => None, + }; + + match rejected { + Some(property) => Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::RecurrenceOverrides) + .with_description(match property { + JSCalendarProperty::UtcStart | JSCalendarProperty::UtcEnd => Cow::Borrowed( + "utcStart and utcEnd cannot be set inside recurrenceOverrides.", + ), + property => Cow::Owned(format!( + "{} cannot be set inside recurrenceOverrides.", + property.to_string() + )), + })), + None => Ok(()), + } + } +} + +pub(super) trait OverridePointer { + fn validate_override_pointer(&self, value: &EventValue<'_>) -> Result<(), EventError>; + + fn rejected_override_token(&self) -> Option<&JSCalendarProperty>; +} + +impl OverridePointer for JsonPointer> { + fn validate_override_pointer(&self, value: &EventValue<'_>) -> Result<(), EventError> { + match self.as_slice() { + [_, _] if value.is_null() => Ok(()), + [_, _] => value.validate_override_patch(), + [_, _, JsonPointerItem::Key(key), ..] => key.validate_override_patch(), + _ => Ok(()), + } + } + + fn rejected_override_token(&self) -> Option<&JSCalendarProperty> { + match self.as_slice() { + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + _, + JsonPointerItem::Key(Key::Property(property @ JSCalendarProperty::CalendarAddress)), + ] => Some(property), + [JsonPointerItem::Key(Key::Property(property))] if property.is_series_property() => { + Some(property) + } + [JsonPointerItem::Key(Key::Property(property)), ..] + if property.is_event_metadata() + || matches!( + property, + JSCalendarProperty::RecurrenceOverrides + | JSCalendarProperty::RecurrenceRule + ) => + { + Some(property) + } + _ => None, + } + } +} + +pub(super) trait BooleanValue { + fn is_same_boolean(&self, current: bool) -> bool; +} + +impl BooleanValue for EventValue<'_> { + fn is_same_boolean(&self, current: bool) -> bool { + match self { + Value::Bool(value) => *value == current, + Value::Null => !current, + _ => false, + } + } +} + +pub(super) trait PropertyName { + fn is_valid_property_name(&self) -> bool; +} + +impl PropertyName for str { + fn is_valid_property_name(&self) -> bool { + match self.split_once(':') { + Some((prefix, name)) => { + !prefix.is_empty() + && prefix.split('.').all(|label| { + !label.is_empty() + && !label.starts_with('-') + && !label.ends_with('-') + && label + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || !ch.is_ascii()) + }) + && !name.is_empty() + && name.chars().all(|ch| { + matches!(ch, ' ' | '\t' | '!' | '#'..='.' | '0'..='}') || !ch.is_ascii() + }) + } + None => { + !self.is_empty() + && self + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'@') + } + } + } +} + +pub(super) struct InstanceMetadata<'a> { + pub is_origin: bool, + pub flags: u16, + pub calendar_ids: &'a [DavName], + pub use_default_alerts: bool, +} + +pub(super) struct InstanceView<'a, 'x> { + pub event: &'a EventMap<'x>, + pub metadata: &'a InstanceMetadata<'a>, + pub id: Id, + pub recurrence_ids: [i64; 2], +} + +impl InstanceView<'_, '_> { + pub fn is_override_key( + &self, + key: &EventKey<'_>, + value: &EventValue<'_>, + ) -> Result { + let property = match key { + Key::Property(property) => property, + Key::Borrowed(name) if name.is_valid_property_name() => return Ok(true), + Key::Owned(name) if name.is_valid_property_name() => return Ok(true), + key => { + return Err(SetError::invalid_properties() + .with_property(key.to_owned()) + .with_description("Invalid property.")); + } + }; + let is_inherited = match property { + JSCalendarProperty::Id => { + return if crate::matches_id(value, self.id) { + Ok(false) + } else { + Err(SetError::invalid_properties() + .with_property(JSCalendarProperty::Id) + .with_description("This property is immutable.")) + }; + } + JSCalendarProperty::BaseEventId => { + crate::matches_id(value, Id::from(self.id.document_id())) + } + JSCalendarProperty::IsOrigin => value.as_bool() == Some(self.metadata.is_origin), + JSCalendarProperty::CalendarIds => self.is_same_calendar_ids(value), + JSCalendarProperty::IsDraft => self.is_same_flag(value, EVENT_DRAFT), + JSCalendarProperty::MayInviteSelf => self.is_same_flag(value, EVENT_INVITE_SELF), + JSCalendarProperty::MayInviteOthers => self.is_same_flag(value, EVENT_INVITE_OTHERS), + JSCalendarProperty::HideAttendees => self.is_same_flag(value, EVENT_HIDE_ATTENDEES), + JSCalendarProperty::UseDefaultAlerts => { + value.is_same_boolean(self.metadata.use_default_alerts) + } + JSCalendarProperty::RecurrenceId => matches!( + value, + Value::Element(JSCalendarValue::DateTime(date_time)) + if self.recurrence_ids.contains(&date_time.timestamp) + ), + JSCalendarProperty::RecurrenceIdTimeZone => { + self.is_base_value(&JSCalendarProperty::TimeZone, value) + } + JSCalendarProperty::RecurrenceOverrides | JSCalendarProperty::RecurrenceRule => { + value.is_null() + } + JSCalendarProperty::Version => { + matches!(value, Value::Str(version) if version == JSCALENDAR_VERSION) + } + JSCalendarProperty::Pointer(pointer) => match pointer.as_slice() { + [ + JsonPointerItem::Key(Key::Property(JSCalendarProperty::Participants)), + JsonPointerItem::Key(participant_id), + JsonPointerItem::Key(Key::Property(JSCalendarProperty::CalendarAddress)), + ] => self + .event + .get(&Key::Property(JSCalendarProperty::Participants)) + .and_then(|participants| participants.as_object_and_get(participant_id)) + .and_then(|participant| { + participant + .as_object_and_get(&Key::Property(JSCalendarProperty::CalendarAddress)) + }) + .is_some_and(|address| address == value), + [JsonPointerItem::Key(Key::Property(first)), ..] + if first.is_event_metadata() || first.is_series_property() => + { + false + } + _ => return Ok(true), + }, + JSCalendarProperty::UtcStart | JSCalendarProperty::UtcEnd => false, + property if property.is_series_property() => self.is_base_value(property, value), + _ => return Ok(true), + }; + + if is_inherited { + Ok(false) + } else { + Err(SetError::invalid_properties() + .with_property(property.clone()) + .with_description("This property cannot be modified on a single occurrence.")) + } + } + + fn is_base_value(&self, property: &JSCalendarProperty, value: &EventValue<'_>) -> bool { + match self.event.get(&Key::Property(property.clone())) { + Some(base) => base == value, + None => value.is_null(), + } + } + + fn is_same_flag(&self, value: &EventValue<'_>, flag: u16) -> bool { + value.is_same_boolean(self.metadata.flags & flag != 0) + } + + fn is_same_calendar_ids(&self, value: &EventValue<'_>) -> bool { + value.as_object().is_some_and(|calendar_ids| { + calendar_ids.len() == self.metadata.calendar_ids.len() + && calendar_ids.iter().all(|(key, value)| { + matches!(value, Value::Bool(true)) + && matches!(key, Key::Property(JSCalendarProperty::IdValue(id)) + if self + .metadata + .calendar_ids + .iter() + .any(|name| name.parent_id == id.document_id())) + }) + }) + } +} + diff --git a/crates/jmap/src/calendar_event_notification/get.rs b/crates/jmap/src/calendar_event_notification/get.rs index e8a12e24..b550b8ce 100644 --- a/crates/jmap/src/calendar_event_notification/get.rs +++ b/crates/jmap/src/calendar_event_notification/get.rs @@ -4,21 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::blob::embedded::import_error; +use crate::calendar_event_notification::{NotificationTypeFlags, patch::JSCalendarPatch}; use crate::changes::state::JmapCacheState; use calcard::{ - icalendar::{ArchivedICalendarProperty, ICalendar}, - jscalendar::import::ConversionOptions, + icalendar::{ArchivedICalendar, ArchivedICalendarProperty, ICalendar}, + jscalendar::{JSCalendar, import::ImportOptions}, }; use common::{Server, auth::AccessToken}; use groupware::{ cache::GroupwareCache, calendar::{ - ArchivedChangedBy, CalendarEventNotification, EVENT_NOTIFICATION_IS_CHANGE, - EVENT_NOTIFICATION_IS_DRAFT, + ArchivedChangedBy, CalendarEventNotification, CalendarEventNotificationContent, + EVENT_NOTIFICATION_IS_CHANGE, EVENT_NOTIFICATION_IS_DRAFT, + notification::CalendarNotificationViewers, }, }; use jmap_proto::{ - method::get::GetRequest, + method::get::{GetRequest, all_ids}, object::calendar_event_notification::{ self, CalendarEventNotificationGetResponse, CalendarEventNotificationObject, CalendarEventNotificationProperty, CalendarEventNotificationType, PersonObject, @@ -27,12 +30,14 @@ use jmap_proto::{ }; use store::{ ValueKey, - write::{AlignedBytes, Archive, serialize::rkyv_deserialize}, + ahash::AHashMap, + write::{Archive, ArchiveBytes, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ blob::BlobId, collection::{Collection, SyncCollection}, + field::CalendarNotificationField, id::Id, }; @@ -54,12 +59,17 @@ impl CalendarEventNotificationGet for Server { let properties = request.unwrap_properties(&[ CalendarEventNotificationProperty::Id, CalendarEventNotificationProperty::Created, - CalendarEventNotificationProperty::Type, CalendarEventNotificationProperty::ChangedBy, + CalendarEventNotificationProperty::Comment, + CalendarEventNotificationProperty::Type, + CalendarEventNotificationProperty::CalendarEventId, + CalendarEventNotificationProperty::IsDraft, + CalendarEventNotificationProperty::Event, + CalendarEventNotificationProperty::EventPatch, ]); let account_id = request.account_id.document_id(); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::CalendarEventNotification, @@ -67,14 +77,20 @@ impl CalendarEventNotificationGet for Server { .await .caused_by(trc::location!())?; + let viewer = self + .notification_viewer(access_token, account_id) + .await + .caused_by(trc::location!())?; let ids = if let Some(ids) = ids { ids } else { - cache - .document_ids(false) - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + viewer + .visible_notifications(&cache) + .into_iter() + .map(Into::into), + self.core.jmap.get_max_objects, + )? }; let mut response = CalendarEventNotificationGetResponse { account_id: request.account_id.into(), @@ -83,111 +99,206 @@ impl CalendarEventNotificationGet for Server { not_found: not_found_ids, }; + let mut changed_by_cache: AHashMap = AHashMap::new(); + let mut needs_meta = false; + let mut needs_content = false; + let mut wants_event = false; + let mut wants_patch = false; + for property in &properties { + match property { + CalendarEventNotificationProperty::Id + | CalendarEventNotificationProperty::Created + | CalendarEventNotificationProperty::CalendarEventId => (), + CalendarEventNotificationProperty::Type + | CalendarEventNotificationProperty::IsDraft + | CalendarEventNotificationProperty::ChangedBy => { + needs_meta = true; + } + _ => { + needs_meta = true; + needs_content = true; + wants_event |= *property == CalendarEventNotificationProperty::Event; + wants_patch |= *property == CalendarEventNotificationProperty::EventPatch; + } + } + } + for id in ids { // Obtain the event object let document_id = id.document_id(); - let _event = if let Some(event) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEventNotification, - document_id, - )) - .await? - { - event - } else { + let Some(resource) = cache.item_by_id(document_id).filter(|resource| { + resource + .notification() + .is_some_and(|notification| viewer.can_view(¬ification)) + }) else { response.push_not_found(id); continue; }; - let event = _event - .unarchive::() - .caused_by(trc::location!())?; + + let _event; + let event = if needs_meta { + _event = match self + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEventNotification, + document_id, + )) + .await? + { + Some(event) => event, + None => { + response.push_not_found(id); + continue; + } + }; + Some( + _event + .unarchive::() + .caused_by(trc::location!())?, + ) + } else { + None + }; + + let _content; + let content = if needs_content { + _content = match self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEventNotification, + document_id, + CalendarNotificationField::Content, + )) + .await? + { + Some(content) => content, + None => { + response.push_not_found(id); + continue; + } + }; + Some( + _content + .unarchive::() + .caused_by(trc::location!())?, + ) + } else { + None + }; let mut result = CalendarEventNotificationObject { id, ..Default::default() }; for property in &properties { match property { - CalendarEventNotificationProperty::Id => {} + CalendarEventNotificationProperty::Id + | CalendarEventNotificationProperty::Event + | CalendarEventNotificationProperty::EventPatch => {} CalendarEventNotificationProperty::Created => { - result.created = Some(UTCDate::from_timestamp(event.created.to_native())); + result.created = resource.created_at().map(UTCDate::from_timestamp).into(); } CalendarEventNotificationProperty::CalendarEventId => { - result.calendar_event_id = - event.event_id.as_ref().map(|id| id.to_native().into()); + result.calendar_event_id = resource + .event_id() + .filter(|id| *id != u32::MAX) + .map(Id::from) + .into(); } CalendarEventNotificationProperty::ChangedBy => { - let mut changed_by = PersonObject::default(); - - match &event.changed_by { - ArchivedChangedBy::PrincipalId(id) => { - if let Ok(account) = self.account(id.to_native()).await { - changed_by.name = - account.description().unwrap_or(account.name()).to_string(); - changed_by.email = account.name().to_string().into(); - } - changed_by.principal_id = Some(id.to_native().into()); + let changed_by = match event.map(|event| &event.changed_by) { + Some(ArchivedChangedBy::PrincipalId(id)) => { + let principal_id = id.to_native(); + let changed_by = match changed_by_cache.get(&principal_id) { + Some(changed_by) => changed_by.clone(), + None => { + let mut changed_by = PersonObject { + principal_id: Some(principal_id.into()), + ..Default::default() + }; + if let Ok(account) = self.account_info(principal_id).await { + changed_by.name = account + .description() + .unwrap_or(account.name()) + .to_string(); + changed_by.email = account.addresses().first().cloned(); + changed_by.calendar_address = changed_by + .email + .as_ref() + .map(|email| format!("mailto:{email}")); + } + changed_by_cache.insert(principal_id, changed_by.clone()); + changed_by + } + }; + Some(changed_by) } - ArchivedChangedBy::CalendarAddress(email) => { - changed_by.email = Some(email.to_string()); - changed_by.calendar_address = Some(format!("mailto:{email}")); - } - } - - result.changed_by = Some(changed_by); + Some(ArchivedChangedBy::CalendarAddress(email)) => Some(PersonObject { + name: email.to_string(), + email: Some(email.to_string()), + principal_id: None, + calendar_address: Some(format!("mailto:{email}")), + }), + None => None, + }; + result.changed_by = changed_by.into(); } CalendarEventNotificationProperty::Comment => { - result.comment = event - .event - .components - .iter() - .filter(|c| c.component_type.is_scheduling_object()) - .flat_map(|c| c.entries.iter()) - .find(|e| matches!(e.name, ArchivedICalendarProperty::Comment)) - .and_then(|e| e.values.first().and_then(|v| v.as_text())) - .map(|v| v.to_string()); + result.comment = content + .and_then(|content| content.itip_message()) + .and_then(|message| { + message + .components + .iter() + .filter(|c| c.component_type.is_scheduling_object()) + .flat_map(|c| c.entries.iter()) + .find(|e| matches!(e.name, ArchivedICalendarProperty::Comment)) + .and_then(|e| e.values.first().and_then(|v| v.as_text())) + }) + .map(|v| v.to_string()) + .into(); } CalendarEventNotificationProperty::Type => { - result.notification_type = - Some(if event.flags & EVENT_NOTIFICATION_IS_CHANGE != 0 { - CalendarEventNotificationType::Updated - } else if !event.event.components.is_empty() { - CalendarEventNotificationType::Created - } else { - CalendarEventNotificationType::Destroyed - }); + result.notification_type = event + .map(|event| { + CalendarEventNotificationType::from_flags(event.flags.to_native()) + }) + .into(); } CalendarEventNotificationProperty::IsDraft => { - result.is_draft = Some(event.flags & EVENT_NOTIFICATION_IS_DRAFT != 0); + result.is_draft = event + .map(|event| event.flags & EVENT_NOTIFICATION_IS_DRAFT != 0) + .into(); } - CalendarEventNotificationProperty::Event => { - if event.flags & EVENT_NOTIFICATION_IS_CHANGE == 0 && result.event.is_none() - { - let js_event = rkyv_deserialize::<_, ICalendar>(&event.event) - .caused_by(trc::location!())? - .into_jscalendar_with_opt::( - ConversionOptions::default() - .include_ical_components(false) - .return_first(true), - ); - result.event = js_event.into(); - } - } - CalendarEventNotificationProperty::EventPatch => { - if event.flags & EVENT_NOTIFICATION_IS_CHANGE != 0 - && result.event_patch.is_none() - { - let js_event = rkyv_deserialize::<_, ICalendar>(&event.event) - .caused_by(trc::location!())? - .into_jscalendar_with_opt::( - ConversionOptions::default() - .include_ical_components(false) - .return_first(true), - ); - result.event_patch = js_event.into(); - } + } + } + + if wants_event || wants_patch { + let (snapshot, patch) = match event.zip(content) { + Some((event, content)) => { + let is_change = event.flags.to_native() & EVENT_NOTIFICATION_IS_CHANGE != 0; + let snapshot = content.previous().or_else(|| content.current()); + let snapshot = (wants_event || is_change) + .then_some(snapshot) + .flatten() + .map(to_jscalendar) + .transpose()?; + let patch = match (&snapshot, content.current()) { + (Some(previous), Some(current)) if wants_patch && is_change => { + Some(previous.patch_to(&to_jscalendar(current)?)) + } + _ => None, + }; + (snapshot, patch) } + None => (None, None), + }; + if wants_event { + result.event = snapshot.into(); + } + if wants_patch { + result.event_patch = patch.into(); } } response.list.push(result); @@ -196,3 +307,14 @@ impl CalendarEventNotificationGet for Server { Ok(response) } } + +fn to_jscalendar(ical: &ArchivedICalendar) -> trc::Result> { + rkyv_deserialize::<_, ICalendar>(ical) + .caused_by(trc::location!())? + .into_jscalendar_with::( + ImportOptions::new() + .include_ical_components(false) + .return_first(true), + ) + .map_err(import_error) +} diff --git a/crates/jmap/src/calendar_event_notification/mod.rs b/crates/jmap/src/calendar_event_notification/mod.rs index c036acc9..8c2a1650 100644 --- a/crates/jmap/src/calendar_event_notification/mod.rs +++ b/crates/jmap/src/calendar_event_notification/mod.rs @@ -5,5 +5,25 @@ */ pub mod get; +pub mod patch; pub mod query; pub mod set; + +use groupware::calendar::{EVENT_NOTIFICATION_IS_CHANGE, EVENT_NOTIFICATION_IS_DESTROY}; +use jmap_proto::object::calendar_event_notification::CalendarEventNotificationType; + +pub(crate) trait NotificationTypeFlags { + fn from_flags(flags: u16) -> Self; +} + +impl NotificationTypeFlags for CalendarEventNotificationType { + fn from_flags(flags: u16) -> Self { + if flags & EVENT_NOTIFICATION_IS_CHANGE != 0 { + CalendarEventNotificationType::Updated + } else if flags & EVENT_NOTIFICATION_IS_DESTROY != 0 { + CalendarEventNotificationType::Destroyed + } else { + CalendarEventNotificationType::Created + } + } +} diff --git a/crates/jmap/src/calendar_event_notification/patch.rs b/crates/jmap/src/calendar_event_notification/patch.rs new file mode 100644 index 00000000..82712fad --- /dev/null +++ b/crates/jmap/src/calendar_event_notification/patch.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use calcard::jscalendar::{JSCalendar, JSCalendarProperty, JSCalendarValue}; +use jmap_tools::{Key, Map, Value}; +use types::{blob::BlobId, id::Id}; + +type EventMap = Map<'static, JSCalendarProperty, JSCalendarValue>; + +pub trait JSCalendarPatch { + fn patch_to(&self, after: &Self) -> Self; +} + +impl JSCalendarPatch for JSCalendar<'static, Id, BlobId> { + fn patch_to(&self, after: &Self) -> Self { + let empty = EventMap::new(); + let mut patch = EventMap::new(); + diff_objects( + &mut String::new(), + self.0.as_object().unwrap_or(&empty), + after.0.as_object().unwrap_or(&empty), + &mut patch, + ); + JSCalendar(Value::Object(patch)) + } +} + +fn diff_objects(prefix: &mut String, before: &EventMap, after: &EventMap, patch: &mut EventMap) { + for (key, before_value) in before.iter() { + match after.get(key) { + Some(after_value) if after_value == before_value => {} + Some(Value::Object(after_object)) + if let Value::Object(before_object) = before_value + && !after_object.adds_null(before_object) => + { + let len = prefix.len(); + prefix.push_segment(&key.to_string()); + prefix.push('/'); + diff_objects(prefix, before_object, after_object, patch); + prefix.truncate(len); + } + Some(after_value) => { + patch.insert_unchecked(prefix.patch_key(key), after_value.clone()); + } + None => { + patch.insert_unchecked(prefix.patch_key(key), Value::Null); + } + } + } + for (key, after_value) in after.iter().filter(|(key, _)| !before.contains_key(key)) { + patch.insert_unchecked(prefix.patch_key(key), after_value.clone()); + } +} + +trait NullMembers { + fn adds_null(&self, before: &Self) -> bool; +} + +impl NullMembers for EventMap { + fn adds_null(&self, before: &Self) -> bool { + self.iter().any(|(key, value)| { + matches!(value, Value::Null) && !matches!(before.get(key), Some(Value::Null)) + }) + } +} + +trait PatchPointer { + fn patch_key( + &self, + key: &Key<'static, JSCalendarProperty>, + ) -> Key<'static, JSCalendarProperty>; + + fn push_segment(&mut self, segment: &str); +} + +impl PatchPointer for String { + fn patch_key( + &self, + key: &Key<'static, JSCalendarProperty>, + ) -> Key<'static, JSCalendarProperty> { + let segment = key.to_string(); + if self.is_empty() && !segment.contains(['~', '/']) { + key.clone() + } else { + let mut pointer = String::with_capacity(self.len() + segment.len() + 2); + pointer.push_str(self); + pointer.push_segment(&segment); + Key::Owned(pointer) + } + } + + fn push_segment(&mut self, segment: &str) { + for ch in segment.chars() { + match ch { + '~' => self.push_str("~0"), + '/' => self.push_str("~1"), + ch => self.push(ch), + } + } + } +} + diff --git a/crates/jmap/src/calendar_event_notification/query.rs b/crates/jmap/src/calendar_event_notification/query.rs index bc5ec2f3..21e60539 100644 --- a/crates/jmap/src/calendar_event_notification/query.rs +++ b/crates/jmap/src/calendar_event_notification/query.rs @@ -4,29 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; +use crate::{ + api::query::QueryResponseBuilder, calendar_event_notification::NotificationTypeFlags, + changes::state::JmapCacheState, +}; use common::{Server, auth::AccessToken}; -use groupware::cache::GroupwareCache; +use groupware::{cache::GroupwareCache, calendar::notification::CalendarNotificationViewers}; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::calendar_event_notification::{ CalendarEventNotification, CalendarEventNotificationComparator, - CalendarEventNotificationFilter, + CalendarEventNotificationFilter, CalendarEventNotificationType as NotificationType, }, request::IntoValid, }; use store::{ - IterateParams, U32_LEN, U64_LEN, ValueKey, ahash::AHashSet, roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, - write::{IndexPropertyClass, SearchIndex, ValueClass, key::DeserializeBigEndian}, + write::SearchIndex, }; use trc::AddContext; -use types::{ - collection::{Collection, SyncCollection}, - field::CalendarNotificationField, -}; +use types::collection::SyncCollection; pub trait CalendarEventNotificationQuery: Sync + Send { fn calendar_event_notification_query( @@ -36,12 +35,6 @@ pub trait CalendarEventNotificationQuery: Sync + Send { ) -> impl Future> + Send; } -struct Notification { - document_id: u32, - created: u64, - event_id: u32, -} - impl CalendarEventNotificationQuery for Server { async fn calendar_event_notification_query( &self, @@ -51,81 +44,67 @@ impl CalendarEventNotificationQuery for Server { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::CalendarEventNotification, ) .await?; - let mut notifications = Vec::with_capacity(16); - let mut document_ids = RoaringBitmap::new(); - self.store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::CalendarEventNotification.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::CalendarEventNotification.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: CalendarNotificationField::CreatedToId.into(), - value: u64::MAX, - }), - }, - ) - .ascending(), - |key, value| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - notifications.push(Notification { - document_id, - created: key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, - event_id: value.deserialize_be_u32(0)?, - }); - document_ids.insert(document_id); - - Ok(true) - }, - ) + let visible_ids = self + .notification_viewer(access_token, account_id) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .visible_notifications(&cache); for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { CalendarEventNotificationFilter::Before(before) => { - let before = before.timestamp() as u64; + let before = before.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - notifications - .iter() - .filter_map(|n| (n.created < before).then_some(n.document_id)), + cache.resources.iter().filter_map(|r| { + r.created_at() + .filter(|created_at| !r.is_container() && *created_at < before) + .map(|_| r.document_id()) + }), ))) } CalendarEventNotificationFilter::After(after) => { - let after = after.timestamp() as u64; + let after = after.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - notifications - .iter() - .filter_map(|n| (n.created > after).then_some(n.document_id)), + cache.resources.iter().filter_map(|r| { + r.created_at() + .filter(|created_at| !r.is_container() && *created_at >= after) + .map(|_| r.document_id()) + }), ))) } + CalendarEventNotificationFilter::Type(notification_type) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.resources.iter().filter_map(|resource| { + resource + .notification() + .filter(|notification| { + !resource.is_container() + && NotificationType::from_flags(notification.flags) + == notification_type + }) + .map(|_| resource.document_id()) + }), + ))); + } CalendarEventNotificationFilter::CalendarEventIds(ids) => { let ids = ids .into_valid() .map(|id| id.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - notifications - .iter() - .filter_map(|n| ids.contains(&n.event_id).then_some(n.document_id)), + cache.resources.iter().filter_map(|r| { + r.event_id() + .filter(|event_id| !r.is_container() && ids.contains(event_id)) + .map(|_| r.document_id()) + }), ))) } unsupported => { @@ -163,13 +142,9 @@ impl CalendarEventNotificationQuery for Server { } }; } - if !is_ascending { - notifications.reverse(); - } - let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) - .with_mask(document_ids) + .with_mask(visible_ids) .filter() .into_bitmap(); @@ -181,11 +156,23 @@ impl CalendarEventNotificationQuery for Server { ); if !results.is_empty() { - let results = results.into_iter().collect::>(); - for notification in notifications { - if results.contains(¬ification.document_id) - && !response.add(0, notification.document_id) - { + let mut notifications = results + .iter() + .filter_map(|document_id| { + cache + .item_by_id(document_id) + .and_then(|resource| resource.created_at()) + .map(|created_at| (document_id, created_at)) + }) + .collect::>(); + notifications + .sort_unstable_by_key(|(document_id, created_at)| (*created_at, *document_id)); + if !is_ascending { + notifications.reverse(); + } + + for (document_id, _) in notifications { + if !response.add(0, document_id) { break; } } diff --git a/crates/jmap/src/calendar_event_notification/set.rs b/crates/jmap/src/calendar_event_notification/set.rs index e6bdddfe..6f566aef 100644 --- a/crates/jmap/src/calendar_event_notification/set.rs +++ b/crates/jmap/src/calendar_event_notification/set.rs @@ -4,8 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::changes::state::JmapCacheState; use common::{Server, auth::AccessToken}; -use groupware::{DestroyArchive, cache::GroupwareCache, calendar::CalendarEventNotification}; +use groupware::{ + cache::GroupwareCache, + calendar::{ + CalendarEventNotification, + notification::{CalendarNotificationDismiss, CalendarNotificationViewers}, + }, +}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, @@ -16,7 +23,7 @@ use jmap_proto::{ }; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; use trc::AddContext; use types::collection::{Collection, SyncCollection}; @@ -41,13 +48,19 @@ impl CalendarEventNotificationSet for Server { ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::CalendarEventNotification, ) .await?; - let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? + .with_state(cache.assert_state(false, &request.if_in_state)?); + + let viewer = self + .notification_viewer(access_token, account_id) + .await + .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for (id, _) in request.unwrap_create() { @@ -69,14 +82,18 @@ impl CalendarEventNotificationSet for Server { for id in request.unwrap_destroy().into_valid() { let document_id = id.document_id(); - if !cache.has_item_id(&document_id) { + if !cache.item_by_id(document_id).is_some_and(|resource| { + resource + .notification() + .is_some_and(|notification| viewer.can_view(¬ification)) + }) { response.not_destroyed.append(id, SetError::not_found()); continue; }; let _event = if let Some(event) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, @@ -92,13 +109,8 @@ impl CalendarEventNotificationSet for Server { .to_unarchived::() .caused_by(trc::location!())?; - DestroyArchive(event) - .delete( - access_token.account_tenant_ids(), - account_id, - document_id, - &mut batch, - ) + self.dismiss_notification(access_token, account_id, document_id, event, &mut batch) + .await .caused_by(trc::location!())?; response.destroyed.push(id); @@ -109,7 +121,9 @@ impl CalendarEventNotificationSet for Server { let change_id = self .commit_batch(batch) .await - .and_then(|ids| ids.last_change_id(account_id)) + .map(|ids| { + ids.last_change_id(account_id, SyncCollection::CalendarEventNotification) + }) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 821b1e20..f5a5953a 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -4,26 +4,55 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::auth::JmapAuthorization, changes::state::JmapCacheState}; -use common::{Server, auth::AccessToken}; +use crate::{ + api::{ + auth::JmapAuthorization, + metadata::{MetadataType, ObjectMetadata}, + }, + changes::{ + page::{ + Coverage, LogRead, MetadataChanges, PageChange, PartialProperties, UpdatedProperties, + ViewerChange, ViewerChanges, fill_page, private_query, + }, + state::{JmapCacheState, max_state}, + }, + participant_identity::changes::ParticipantIdentityChanges, +}; +use calcard::{jscalendar::JSCalendarProperty, jscontact::JSContactProperty}; +use common::{ + GroupwareResources, MessageStoreCache, Server, auth::AccessToken, + storage::metadata::MetadataViewer, +}; use email::cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}; -use groupware::cache::GroupwareCache; +use groupware::{ + cache::GroupwareCache, + calendar::{EVENT_SECRET, notification::CalendarNotificationViewers}, +}; use jmap_proto::{ - method::changes::{ChangesRequest, ChangesResponse}, - object::{JmapObject, NullObject, mailbox::MailboxProperty}, - request::method::MethodObject, + method::{ + PropertyWrapper, + changes::{ChangesRequest, ChangesResponse}, + }, + object::{ + JmapObject, NullObject, addressbook::AddressBookProperty, calendar::CalendarProperty, + email::EmailProperty, file_node::FileNodeProperty, mailbox::MailboxProperty, + }, + request::{capability::CapabilityIds, method::MethodObject}, response::{ChangesResponseMethod, ResponseMethod}, types::state::State, }; -use std::future::Future; +use jmap_tools::Property; +use std::{future::Future, sync::Arc}; use store::{ query::log::{Change, Query}, roaring::RoaringBitmap, + write::LogCollection, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + id::Id, }; pub trait ChangesLookup: Sync + Send { @@ -32,13 +61,86 @@ pub trait ChangesLookup: Sync + Send { request: ChangesRequest, object: MethodObject, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } pub struct IntermediateChangesResponse { pub response: ChangesResponse, pub object: MethodObject, - pub only_container_changes: bool, + pub updated_properties: Option, +} + +#[derive(Clone, Copy)] +struct ChangesScope { + account_id: u32, + collection: SyncCollection, + viewer: Option, + read_private: bool, +} + +enum Snapshot { + Messages(Arc), + Resources(Arc), +} + +impl Snapshot { + fn state(&self, is_container: bool) -> State { + match self { + Snapshot::Messages(cache) => cache.get_state(is_container), + Snapshot::Resources(cache) => cache.get_state(is_container), + } + } + + fn contains(&self, object: MethodObject, is_container: bool, document_id: u32) -> bool { + match (self, object) { + (Snapshot::Messages(cache), MethodObject::Mailbox) => { + cache.mailbox_by_id(&document_id).is_some() + } + (Snapshot::Messages(cache), _) => cache.email_by_id(&document_id).is_some(), + (Snapshot::Resources(cache), MethodObject::FileNode) => { + cache.has_item_id(&document_id) || cache.has_container_id(&document_id) + } + (Snapshot::Resources(cache), _) if is_container => cache.has_container_id(&document_id), + (Snapshot::Resources(cache), _) => cache.has_item_id(&document_id), + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum HiddenChanges { + Drop, + ReportAsDestroyed, +} + +impl HiddenChanges { + fn for_object(object: MethodObject) -> Self { + match object { + MethodObject::CalendarEvent + | MethodObject::Calendar + | MethodObject::AddressBook + | MethodObject::ContactCard + | MethodObject::FileNode + | MethodObject::Mailbox + | MethodObject::CalendarEventNotification => HiddenChanges::ReportAsDestroyed, + _ => HiddenChanges::Drop, + } + } + + fn rewrite(self, change: Change, id: u64) -> Option { + match self { + HiddenChanges::Drop => None, + HiddenChanges::ReportAsDestroyed => match change { + Change::DeleteItem(_) | Change::DeleteContainer(_) => Some(change), + Change::UpdateItem(_) => Some(Change::DeleteItem(id)), + Change::UpdateContainer(_) => Some(Change::DeleteContainer(id)), + Change::InsertItem(_) + | Change::InsertContainer(_) + | Change::UpdateContainerPartial(..) + | Change::UpdateItemMetadata(_) => None, + }, + } + } } impl ChangesLookup for Server { @@ -47,6 +149,7 @@ impl ChangesLookup for Server { request: ChangesRequest, object: MethodObject, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { // Map collection and validate ACLs let (collection, is_container) = match object { @@ -100,7 +203,7 @@ impl ChangesLookup for Server { (SyncCollection::Calendar, false) } MethodObject::CalendarEventNotification => { - access_token.assert_is_member(request.account_id)?; + access_token.assert_has_access(request.account_id, Collection::CalendarEvent)?; (SyncCollection::CalendarEventNotification, false) } @@ -109,17 +212,31 @@ impl ChangesLookup for Server { (SyncCollection::ShareNotification, false) } + MethodObject::ParticipantIdentity => { + access_token.assert_has_access(request.account_id, Collection::CalendarEvent)?; + + return self + .participant_identity_changes(request, access_token) + .await; + } + MethodObject::Quota => { + access_token.assert_is_member(request.account_id)?; + + return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); + } _ => { return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } }; - let max_changes = std::cmp::min( - request - .max_changes - .filter(|n| *n != 0) - .unwrap_or(usize::MAX), - self.core.jmap.changes_max_results, - ); + let max_changes = match request.max_changes { + Some(0) => { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details("maxChanges must be greater than 0.")); + } + Some(max_changes) => max_changes.min(self.core.jmap.changes_max_results), + None => self.core.jmap.changes_max_results, + }; let mut response: ChangesResponse = ChangesResponse { account_id: request.account_id, old_state: request.since_state.clone(), @@ -132,7 +249,179 @@ impl ChangesLookup for Server { }; let account_id = request.account_id.document_id(); - let allowed_ids: Option = if access_token.is_member(account_id) { + let object_metadata = metadata_type(object) + .map(|metadata_type| ObjectMetadata::new(self, access_token, using, metadata_type)); + let metadata = match &object_metadata { + Some(object_metadata) if object_metadata.support().is_some() => { + if request.ignore_metadata_only_changes == Some(true) { + MetadataChanges::Ignored + } else { + MetadataChanges::Reported + } + } + _ => MetadataChanges::Unsupported, + }; + let viewer = object_metadata.as_ref().and_then(ObjectMetadata::viewer); + let state_viewer = object_metadata + .as_ref() + .and_then(ObjectMetadata::state_viewer); + let viewer_change_id = match (state_viewer, &object_metadata) { + (Some(viewer), Some(object_metadata)) => { + self.metadata_viewer_state(viewer, account_id, object_metadata.collection()) + .await + .caused_by(trc::location!())? + .change_id + } + _ => 0, + }; + let scope = ChangesScope { + account_id, + collection, + viewer, + read_private: viewer.is_some() && metadata == MetadataChanges::Reported, + }; + + let mut snapshot = None; + let (items_sent, log, coverage) = match &request.since_state { + State::Initial => { + let log = read_changes( + self, + scope, + Query::All, + true, + scope.read_private && viewer_change_id > 0, + viewer_change_id, + ) + .await?; + if log.is_empty() { + response.new_state = max_state(State::Initial, viewer_change_id); + return Ok(IntermediateChangesResponse { + response, + object, + updated_properties: None, + }); + } + + (0, log, Coverage::Latest { viewer_change_id }) + } + State::Exact(change_id) => { + snapshot = match collection { + SyncCollection::Calendar + | SyncCollection::AddressBook + | SyncCollection::FileNode => Some(Snapshot::Resources( + self.fetch_groupware_resources( + access_token.account_id(), + account_id, + collection, + ) + .await + .caused_by(trc::location!())?, + )), + SyncCollection::Email => Some(Snapshot::Messages( + self.get_cached_messages(account_id).await?, + )), + _ => None, + }; + + let mut read_shared = true; + let mut shared_change_id = 0; + if let Some(last_state) = snapshot + .as_ref() + .map(|snapshot| snapshot.state(is_container)) + { + if let State::Exact(shared) = last_state { + shared_change_id = shared; + } + response.new_state = max_state(last_state, viewer_change_id); + + if response.new_state == State::Exact(*change_id) { + return Ok(IntermediateChangesResponse { + response, + object, + updated_properties: None, + }); + } + + read_shared = state_viewer.is_none() + || *change_id < shared_change_id + || *change_id > viewer_change_id; + } + + ( + 0, + read_changes( + self, + scope, + Query::Since(*change_id), + read_shared, + scope.read_private && viewer_change_id > *change_id, + viewer_change_id.max(shared_change_id), + ) + .await?, + Coverage::Latest { viewer_change_id }, + ) + } + State::Intermediate(intermediate_state) => { + let log = read_changes( + self, + scope, + Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), + true, + scope.read_private && viewer_change_id >= intermediate_state.from_id, + viewer_change_id, + ) + .await?; + let total = log.total(is_container); + if intermediate_state.items_sent >= total { + ( + 0, + read_changes( + self, + scope, + Query::Since(intermediate_state.to_id), + true, + scope.read_private && viewer_change_id >= intermediate_state.to_id, + viewer_change_id, + ) + .await?, + Coverage::Latest { viewer_change_id }, + ) + } else { + (intermediate_state.items_sent, log, Coverage::Range) + } + } + }; + + if request.since_state != State::Initial { + if log.is_truncated() { + return Err(trc::JmapEvent::CannotCalculateChanges + .into_err() + .details("Change log is truncated")); + } else if !log.is_valid_since() { + return Err(trc::JmapEvent::CannotCalculateChanges + .into_err() + .details("Since state is invalid")); + } + } + + let allowed_ids: Option = if object + == MethodObject::CalendarEventNotification + { + let cache = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::CalendarEventNotification, + ) + .await + .caused_by(trc::location!())?; + Some( + self.notification_viewer(access_token, account_id) + .await + .caused_by(trc::location!())? + .visible_notifications(&cache), + ) + } else if access_token.is_member(account_id) { None } else { Some(match object { @@ -148,15 +437,15 @@ impl ChangesLookup for Server { let cache = self.get_cached_messages(account_id).await?; let shared = cache.shared_messages(access_token, Acl::ReadItems); let mut threads = RoaringBitmap::new(); - for item in &cache.emails.items { - if shared.contains(item.document_id) { - threads.insert(item.thread_id); + for item in cache.emails.iter() { + if shared.contains(item.document_id()) { + threads.insert(item.thread_id()); } } threads } MethodObject::AddressBook => self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -164,7 +453,7 @@ impl ChangesLookup for Server { .await? .shared_containers(access_token, [Acl::Read, Acl::ReadItems], true), MethodObject::ContactCard => self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, @@ -172,240 +461,269 @@ impl ChangesLookup for Server { .await? .shared_items(access_token, [Acl::ReadItems], true), MethodObject::Calendar => self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await? .shared_containers(access_token, [Acl::Read, Acl::ReadItems], true), - MethodObject::CalendarEvent => self - .fetch_dav_resources( - access_token.account_id(), - account_id, - SyncCollection::Calendar, - ) - .await? - .shared_items(access_token, [Acl::ReadItems], true), - MethodObject::FileNode => self - .fetch_dav_resources( + MethodObject::CalendarEvent => { + let cache = self + .fetch_groupware_resources( + access_token.account_id(), + account_id, + SyncCollection::Calendar, + ) + .await?; + let mut shared_ids = cache.shared_items(access_token, [Acl::ReadItems], true); + shared_ids -= cache.event_ids_with_flags(EVENT_SECRET); + shared_ids + } + MethodObject::FileNode => { + self.fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await? - .shared_documents(access_token, [Acl::Read, Acl::ReadItems], true), + .file_access(access_token) + .discoverable + } _ => RoaringBitmap::new(), }) }; - let (items_sent, changelog) = match &request.since_state { - State::Initial => { - let changelog = self - .store() - .changes(account_id, collection.into(), Query::All) - .await?; - if changelog.changes.is_empty() && changelog.from_change_id == 0 { - return Ok(IntermediateChangesResponse { - response, - object, - only_container_changes: false, - }); - } + let hidden_changes = HiddenChanges::for_object(object); + let visible = |change: Change| { + let Some(allowed) = allowed_ids.as_ref() else { + return Some(change); + }; + let id = if is_container { + change.container_id() + } else { + change.item_id() + }?; - (0, changelog) - } - State::Exact(change_id) => { - let last_state = match collection { - SyncCollection::Calendar | SyncCollection::AddressBook => self - .fetch_dav_resources(access_token.account_id(), account_id, collection) - .await - .caused_by(trc::location!())? - .get_state(is_container) - .into(), - SyncCollection::Email => self - .get_cached_messages(account_id) - .await? - .get_state(is_container) - .into(), - _ => None, - }; - - if let Some(last_state) = last_state { - response.new_state = last_state; - - if response.new_state == State::Exact(*change_id) { - return Ok(IntermediateChangesResponse { - response, - object, - only_container_changes: false, - }); - } - } - - ( - 0, - self.store() - .changes(account_id, collection.into(), Query::Since(*change_id)) - .await?, - ) - } - State::Intermediate(intermediate_state) => { - let changelog = self - .store() - .changes( - account_id, - collection.into(), - Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), - ) - .await?; - if (is_container - && intermediate_state.items_sent >= changelog.total_container_changes()) - || (!is_container - && intermediate_state.items_sent >= changelog.total_item_changes()) - { - ( - 0, - self.store() - .changes( - account_id, - collection.into(), - Query::Since(intermediate_state.to_id), - ) - .await?, - ) - } else { - (intermediate_state.items_sent, changelog) - } + if allowed.contains(id as u32) { + Some(change) + } else { + hidden_changes.rewrite(change, id) } }; - - if (changelog.is_truncated || changelog.from_change_id == 0) - && request.since_state != State::Initial - { - return Err(trc::JmapEvent::CannotCalculateChanges.into_err().details( - if changelog.is_truncated { - "Change log is truncated" - } else { - "Since state is invalid" - }, - )); - } - - let mut changes = changelog - .changes - .into_iter() - .filter(|change| { - (is_container && change.is_container_change()) - || (!is_container && change.is_item_change()) - }) - .filter(|change| { - allowed_ids.as_ref().is_none_or(|allowed| { - let id = if is_container { - change.container_id() - } else { - change.item_id() - }; - id.is_some_and(|id| allowed.contains(id as u32)) - }) - }) - .skip(items_sent) - .peekable(); - - let mut items_changed = false; - for change in (&mut changes).take(max_changes) { - match change { - Change::InsertContainer(item) | Change::InsertItem(item) => { - response.created.push(item.into()); - } - Change::UpdateContainer(item) | Change::UpdateItem(item) => { - response.updated.push(item.into()); - items_changed = true; - } - Change::DeleteContainer(item) | Change::DeleteItem(item) => { - response.destroyed.push(item.into()); - } - Change::UpdateContainerProperty(item) => { - response.updated.push(item.into()); - } - }; - } - - let change_id = (if is_container { - changelog.container_change_id + let partial = if object == MethodObject::Mailbox { + PartialProperties::Counts } else { - changelog.item_change_id - }) - .unwrap_or(changelog.to_change_id); + PartialProperties::Unknown + }; - response.has_more_changes = changes.peek().is_some(); - if response.has_more_changes { - response.new_state = State::new_intermediate( - changelog.from_change_id, - change_id, - items_sent + max_changes, - ); - } else if response.new_state == State::Initial { - response.new_state = State::new_exact(change_id) - } + let exists = |change: &ViewerChange| match (change, &snapshot) { + (ViewerChange::Private(change), Some(snapshot)) => { + let id = if is_container { + change.container_id() + } else { + change.item_id() + }; + id.is_some_and(|id| snapshot.contains(object, is_container, id as u32)) + } + _ => true, + }; + + let bounds = log.bounds(is_container); + let page = match log.into_changes(is_container) { + ViewerChanges::Shared(changes) => fill_page( + changes + .into_iter() + .filter(|change| { + (is_container && change.is_container_change()) + || (!is_container && change.is_item_change()) + }) + .filter_map(visible), + items_sent, + max_changes, + metadata, + partial, + &mut response, + ), + ViewerChanges::Merged(changes) => fill_page( + changes.into_iter().filter(exists).filter_map(|change| { + let inner = change.change(); + visible(inner).map(|visible| { + if visible == inner { + change + } else { + ViewerChange::Shared(visible) + } + }) + }), + items_sent, + max_changes, + metadata, + partial, + &mut response, + ), + }; + + response.has_more_changes = page.has_more; + response.new_state = bounds.new_state( + response.new_state, + page.has_more.then_some(items_sent + max_changes), + coverage, + ); Ok(IntermediateChangesResponse { - only_container_changes: is_container && !response.updated.is_empty() && !items_changed, response, object, + updated_properties: page.updated_properties, }) } } +async fn read_changes( + server: &Server, + scope: ChangesScope, + query: Query, + read_shared: bool, + read_private: bool, + known_change_id: u64, +) -> trc::Result { + let store = server.store(); + let shared = if read_shared { + Some( + store + .changes(scope.account_id, scope.collection.into(), query) + .await?, + ) + } else { + None + }; + let private = match scope.viewer.filter(|_| read_private).and_then(|viewer| { + private_query(query, shared.as_ref(), known_change_id).map(|query| (viewer, query)) + }) { + Some((viewer, query)) => Some( + store + .changes( + scope.account_id, + LogCollection::Private { + collection: scope.collection, + viewer: viewer.account_id(), + }, + query, + ) + .await?, + ), + None => None, + }; + Ok(LogRead { shared, private }) +} + +fn metadata_type(object: MethodObject) -> Option { + match object { + MethodObject::Email => Some(MetadataType::Email), + MethodObject::Mailbox => Some(MetadataType::Mailbox), + MethodObject::Calendar => Some(MetadataType::Calendar), + MethodObject::CalendarEvent => Some(MetadataType::CalendarEvent), + MethodObject::AddressBook => Some(MetadataType::AddressBook), + MethodObject::ContactCard => Some(MetadataType::ContactCard), + MethodObject::FileNode => Some(MetadataType::FileNode), + _ => None, + } +} + impl IntermediateChangesResponse { pub fn into_method_response(self) -> ResponseMethod<'static> { + let properties = self.updated_properties; + let response = self.response; ResponseMethod::Changes(match self.object { - MethodObject::Email => ChangesResponseMethod::Email(transmute_response(self.response)), - MethodObject::Mailbox => { - let mut response = transmute_response(self.response); - if self.only_container_changes { - response.updated_properties = vec![ - MailboxProperty::TotalEmails.into(), - MailboxProperty::UnreadEmails.into(), - MailboxProperty::TotalThreads.into(), - MailboxProperty::UnreadThreads.into(), - ] - .into(); - } - ChangesResponseMethod::Mailbox(response) - } + MethodObject::Email => ChangesResponseMethod::Email(transmute_response( + response, + updated_properties( + properties, + EmailProperty::Metadata, + EmailProperty::PrivateMetadata, + &[], + ), + )), + MethodObject::Mailbox => ChangesResponseMethod::Mailbox(transmute_response( + response, + updated_properties( + properties, + MailboxProperty::Metadata, + MailboxProperty::PrivateMetadata, + &[ + MailboxProperty::TotalEmails, + MailboxProperty::UnreadEmails, + MailboxProperty::TotalThreads, + MailboxProperty::UnreadThreads, + ], + ), + )), MethodObject::Thread => { - ChangesResponseMethod::Thread(transmute_response(self.response)) + ChangesResponseMethod::Thread(transmute_response(response, None)) } MethodObject::Identity => { - ChangesResponseMethod::Identity(transmute_response(self.response)) + ChangesResponseMethod::Identity(transmute_response(response, None)) } MethodObject::EmailSubmission => { - ChangesResponseMethod::EmailSubmission(transmute_response(self.response)) - } - MethodObject::AddressBook => { - ChangesResponseMethod::AddressBook(transmute_response(self.response)) - } - MethodObject::ContactCard => { - ChangesResponseMethod::ContactCard(transmute_response(self.response)) - } - MethodObject::FileNode => { - ChangesResponseMethod::FileNode(transmute_response(self.response)) - } - MethodObject::Calendar => { - ChangesResponseMethod::Calendar(transmute_response(self.response)) + ChangesResponseMethod::EmailSubmission(transmute_response(response, None)) } + MethodObject::AddressBook => ChangesResponseMethod::AddressBook(transmute_response( + response, + updated_properties( + properties, + AddressBookProperty::Metadata, + AddressBookProperty::PrivateMetadata, + &[], + ), + )), + MethodObject::ContactCard => ChangesResponseMethod::ContactCard(transmute_response( + response, + updated_properties( + properties, + JSContactProperty::::Metadata, + JSContactProperty::::PrivateMetadata, + &[], + ), + )), + MethodObject::FileNode => ChangesResponseMethod::FileNode(transmute_response( + response, + updated_properties( + properties, + FileNodeProperty::Metadata, + FileNodeProperty::PrivateMetadata, + &[], + ), + )), + MethodObject::Calendar => ChangesResponseMethod::Calendar(transmute_response( + response, + updated_properties( + properties, + CalendarProperty::Metadata, + CalendarProperty::PrivateMetadata, + &[], + ), + )), MethodObject::CalendarEvent => { - ChangesResponseMethod::CalendarEvent(transmute_response(self.response)) + ChangesResponseMethod::CalendarEvent(transmute_response( + response, + updated_properties( + properties, + JSCalendarProperty::::Metadata, + JSCalendarProperty::::PrivateMetadata, + &[], + ), + )) } MethodObject::CalendarEventNotification => { - ChangesResponseMethod::CalendarEventNotification(transmute_response(self.response)) + ChangesResponseMethod::CalendarEventNotification(transmute_response(response, None)) } MethodObject::ShareNotification => { - ChangesResponseMethod::ShareNotification(transmute_response(self.response)) + ChangesResponseMethod::ShareNotification(transmute_response(response, None)) } - MethodObject::ParticipantIdentity - | MethodObject::Core + MethodObject::ParticipantIdentity => { + ChangesResponseMethod::ParticipantIdentity(transmute_response(response, None)) + } + MethodObject::Core | MethodObject::Blob | MethodObject::PushSubscription | MethodObject::SearchSnippet @@ -418,8 +736,29 @@ impl IntermediateChangesResponse { } } +fn updated_properties( + properties: Option, + metadata: P, + private_metadata: P, + counts: &[P], +) -> Option>> { + let properties = properties?; + let mut names = Vec::with_capacity(counts.len() + 2); + if properties.has_counts() { + names.extend(counts.iter().cloned().map(PropertyWrapper::from)); + } + if properties.has_metadata() { + names.push(metadata.into()); + } + if properties.has_private_metadata() { + names.push(private_metadata.into()); + } + Some(names) +} + fn transmute_response( response: ChangesResponse, + updated_properties: Option>>, ) -> Box> { Box::new(ChangesResponse { account_id: response.account_id, @@ -429,6 +768,6 @@ fn transmute_response( created: response.created, updated: response.updated, destroyed: response.destroyed, - updated_properties: None, + updated_properties, }) } diff --git a/crates/jmap/src/changes/mod.rs b/crates/jmap/src/changes/mod.rs index 39a5a4e1..6444c489 100644 --- a/crates/jmap/src/changes/mod.rs +++ b/crates/jmap/src/changes/mod.rs @@ -5,5 +5,7 @@ */ pub mod get; +pub mod page; pub mod query; pub mod state; + diff --git a/crates/jmap/src/changes/page.rs b/crates/jmap/src/changes/page.rs new file mode 100644 index 00000000..0bfdc821 --- /dev/null +++ b/crates/jmap/src/changes/page.rs @@ -0,0 +1,410 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::state::max_state; +use jmap_proto::{method::changes::ChangesResponse, object::NullObject, types::state::State}; +use store::{ + ahash::{AHashMap, AHashSet}, + query::log::{Change, Changes, Query}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ViewerChange { + Shared(Change), + SharedAndPrivate(Change), + Private(Change), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MetadataChanges { + Unsupported, + Ignored, + Reported, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PartialProperties { + Counts, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct UpdatedProperties(u8); + +#[derive(Debug, Default)] +pub(crate) struct LogRead { + pub shared: Option, + pub private: Option, +} + +#[derive(Debug)] +pub(crate) enum ViewerChanges { + Shared(Vec), + Merged(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Page { + pub has_more: bool, + pub updated_properties: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LogBounds { + pub first_change_id: u64, + pub last_change_id: u64, + pub private_change_id: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Coverage { + Latest { viewer_change_id: u64 }, + Range, +} + +impl UpdatedProperties { + pub const COUNTS: UpdatedProperties = UpdatedProperties(1); + pub const METADATA: UpdatedProperties = UpdatedProperties(1 << 1); + pub const PRIVATE_METADATA: UpdatedProperties = UpdatedProperties(1 << 2); + + pub fn has_counts(self) -> bool { + self.0 & Self::COUNTS.0 != 0 + } + + pub fn has_metadata(self) -> bool { + self.0 & Self::METADATA.0 != 0 + } + + pub fn has_private_metadata(self) -> bool { + self.0 & Self::PRIVATE_METADATA.0 != 0 + } + + pub(crate) fn union(self, other: UpdatedProperties) -> Self { + UpdatedProperties(self.0 | other.0) + } + + fn intersection(self, other: UpdatedProperties) -> Self { + UpdatedProperties(self.0 & other.0) + } + + fn is_empty(self) -> bool { + self.0 == 0 + } +} + +pub(crate) trait PageChange: Copy { + fn change(self) -> Change; + fn is_metadata_only(self) -> bool; + fn properties(self, partial: PartialProperties) -> Option; +} + +impl PageChange for Change { + fn change(self) -> Change { + self + } + + fn is_metadata_only(self) -> bool { + Change::is_metadata_only(&self) + } + + fn properties(self, partial: PartialProperties) -> Option { + match self { + Change::UpdateItemMetadata(_) => Some(UpdatedProperties::METADATA), + Change::UpdateContainerPartial(_, changed) => { + let mut properties = UpdatedProperties::default(); + if changed.has_properties() { + match partial { + PartialProperties::Counts => { + properties = properties.union(UpdatedProperties::COUNTS); + } + PartialProperties::Unknown => return None, + } + } + if changed.has_metadata() { + properties = properties.union(UpdatedProperties::METADATA); + } + Some(properties) + } + _ => None, + } + } +} + +impl PageChange for ViewerChange { + fn change(self) -> Change { + match self { + ViewerChange::Shared(change) + | ViewerChange::SharedAndPrivate(change) + | ViewerChange::Private(change) => change, + } + } + + fn is_metadata_only(self) -> bool { + match self { + ViewerChange::Shared(change) | ViewerChange::SharedAndPrivate(change) => { + Change::is_metadata_only(&change) + } + ViewerChange::Private(_) => true, + } + } + + fn properties(self, partial: PartialProperties) -> Option { + match self { + ViewerChange::Shared(change) => change.properties(partial), + ViewerChange::SharedAndPrivate(change) => change + .properties(partial) + .map(|shared| shared.union(UpdatedProperties::PRIVATE_METADATA)), + ViewerChange::Private(_) => Some(UpdatedProperties::PRIVATE_METADATA), + } + } +} + +fn change_id(change: Change, is_container: bool) -> Option { + if is_container { + change.container_id() + } else { + change.item_id() + } +} + +fn is_kind(change: &Change, is_container: bool) -> bool { + if is_container { + change.is_container_change() + } else { + change.is_item_change() + } +} + +impl LogRead { + pub(crate) fn is_truncated(&self) -> bool { + [self.shared.as_ref(), self.private.as_ref()] + .into_iter() + .flatten() + .any(|changes| changes.is_truncated) + } + + pub(crate) fn is_valid_since(&self) -> bool { + let mut logs = [self.shared.as_ref(), self.private.as_ref()] + .into_iter() + .flatten() + .peekable(); + logs.peek().is_none() || logs.any(|changes| changes.from_change_id != 0) + } + + pub(crate) fn is_empty(&self) -> bool { + [self.shared.as_ref(), self.private.as_ref()] + .into_iter() + .flatten() + .all(|changes| changes.changes.is_empty() && changes.from_change_id == 0) + } + + pub(crate) fn bounds(&self, is_container: bool) -> LogBounds { + let logs = || { + [self.shared.as_ref(), self.private.as_ref()] + .into_iter() + .flatten() + }; + LogBounds { + first_change_id: logs() + .map(|changes| changes.from_change_id) + .filter(|change_id| *change_id != 0) + .min() + .unwrap_or_default(), + last_change_id: logs() + .map(|changes| { + kind_change_id(changes, is_container).unwrap_or(changes.to_change_id) + }) + .max() + .unwrap_or_default(), + private_change_id: self + .private + .as_ref() + .and_then(|changes| kind_change_id(changes, is_container)) + .unwrap_or_default(), + } + } + + pub(crate) fn total(&self, is_container: bool) -> usize { + let shared = self + .shared + .as_ref() + .map(|changes| changes.changes.as_slice()) + .unwrap_or_default(); + let mut total = shared + .iter() + .filter(|change| is_kind(change, is_container)) + .count(); + if let Some(private) = &self.private { + let shared_ids = shared + .iter() + .filter_map(|change| change_id(*change, is_container)) + .collect::>(); + total += private + .changes + .iter() + .filter(|change| is_kind(change, is_container) && change.is_metadata_only()) + .filter_map(|change| change_id(*change, is_container)) + .filter(|id| !shared_ids.contains(id)) + .count(); + } + total + } + + pub(crate) fn into_changes(self, is_container: bool) -> ViewerChanges { + let shared = self + .shared + .map(|changes| changes.changes) + .unwrap_or_default(); + match self.private { + Some(private) => { + ViewerChanges::Merged(merge_private(shared, private.changes, is_container)) + } + None => ViewerChanges::Shared(shared), + } + } +} + +impl LogBounds { + pub(crate) fn new_state( + self, + current: State, + next_items_sent: Option, + coverage: Coverage, + ) -> State { + match (next_items_sent, current, coverage) { + (Some(items_sent), _, _) => { + State::new_intermediate(self.first_change_id, self.last_change_id, items_sent) + } + (None, State::Initial, Coverage::Latest { viewer_change_id }) => { + State::new_exact(self.last_change_id.max(viewer_change_id)) + } + (None, State::Initial, Coverage::Range) => State::new_exact(self.last_change_id), + (None, current, _) => max_state(current, self.private_change_id), + } + } +} + +pub(crate) fn private_query( + query: Query, + shared: Option<&Changes>, + known_change_id: u64, +) -> Option { + let bound = shared + .map_or(0, |changes| changes.to_change_id) + .max(known_change_id); + match query { + Query::All => Some(Query::RangeInclusive(0, bound)), + Query::Since(change_id) => change_id + .checked_add(1) + .filter(|from| *from <= bound) + .map(|from| Query::RangeInclusive(from, bound)), + Query::SinceInclusive(change_id) => { + (change_id <= bound).then_some(Query::RangeInclusive(change_id, bound)) + } + Query::RangeInclusive(..) | Query::Range(..) => Some(query), + } +} + +fn kind_change_id(changes: &Changes, is_container: bool) -> Option { + if is_container { + changes.container_change_id + } else { + changes.item_change_id + } +} + +pub(crate) fn merge_private( + shared: Vec, + private: Vec, + is_container: bool, +) -> Vec { + let mut merged = shared + .into_iter() + .filter(|change| is_kind(change, is_container)) + .map(ViewerChange::Shared) + .collect::>(); + let positions = merged + .iter() + .enumerate() + .filter_map(|(position, change)| { + Some((change_id(change.change(), is_container)?, position)) + }) + .collect::>(); + + for change in private + .into_iter() + .filter(|change| is_kind(change, is_container) && change.is_metadata_only()) + { + let Some(id) = change_id(change, is_container) else { + continue; + }; + match positions + .get(&id) + .and_then(|position| merged.get_mut(*position)) + { + Some(entry) => { + if let ViewerChange::Shared(shared) = *entry + && matches!( + shared, + Change::UpdateItemMetadata(_) | Change::UpdateContainerPartial(..) + ) + { + *entry = ViewerChange::SharedAndPrivate(shared); + } + } + None => merged.push(ViewerChange::Private(change)), + } + } + + merged +} + +pub(crate) fn fill_page( + changes: impl Iterator, + items_sent: usize, + max_changes: usize, + metadata: MetadataChanges, + partial: PartialProperties, + response: &mut ChangesResponse, +) -> Page { + let mut changes = changes + .filter(|change| metadata == MetadataChanges::Reported || !change.is_metadata_only()) + .skip(items_sent) + .peekable(); + let mask = match metadata { + MetadataChanges::Unsupported | MetadataChanges::Ignored => UpdatedProperties::COUNTS, + MetadataChanges::Reported => UpdatedProperties(u8::MAX), + }; + let mut updated_properties = Some(UpdatedProperties::default()); + + for change in (&mut changes).take(max_changes) { + match change.change() { + Change::InsertContainer(id) | Change::InsertItem(id) => { + response.created.push(id.into()); + } + Change::DeleteContainer(id) | Change::DeleteItem(id) => { + response.destroyed.push(id.into()); + } + Change::UpdateContainer(id) + | Change::UpdateItem(id) + | Change::UpdateItemMetadata(id) + | Change::UpdateContainerPartial(id, _) => { + response.updated.push(id.into()); + updated_properties = updated_properties.and_then(|properties| { + change + .properties(partial) + .map(|changed| properties.union(changed.intersection(mask))) + }); + } + } + } + + Page { + has_more: changes.peek().is_some(), + updated_properties: updated_properties + .filter(|properties| !response.updated.is_empty() && !properties.is_empty()), + } +} diff --git a/crates/jmap/src/changes/query.rs b/crates/jmap/src/changes/query.rs index 77e391f1..6310a695 100644 --- a/crates/jmap/src/changes/query.rs +++ b/crates/jmap/src/changes/query.rs @@ -6,10 +6,14 @@ use super::get::ChangesLookup; use crate::{ - api::request::resolve_account_id, calendar_event::query::CalendarEventQuery, + api::{auth::JmapAuthorization, request::resolve_account_id}, + calendar_event::query::CalendarEventQuery, calendar_event_notification::query::CalendarEventNotificationQuery, - contact::query::ContactCardQuery, email::query::EmailQuery, file::query::FileNodeQuery, - mailbox::query::MailboxQuery, share_notification::query::ShareNotificationQuery, + contact::query::ContactCardQuery, + email::query::EmailQuery, + file::query::FileNodeQuery, + mailbox::query::MailboxQuery, + share_notification::query::ShareNotificationQuery, submission::query::EmailSubmissionQuery, }; use common::{Server, auth::AccessToken}; @@ -19,15 +23,18 @@ use jmap_proto::{ query_changes::{AddedItem, QueryChangesRequest, QueryChangesResponse}, }, object::{JmapObject, NullObject}, - request::{QueryChangesRequestMethod, method::MethodObject}, + request::{QueryChangesRequestMethod, capability::CapabilityIds, method::MethodObject}, }; use std::future::Future; +use store::ahash::AHashSet; +use types::id::Id; pub trait QueryChanges: Sync + Send { fn query_changes( &self, request: QueryChangesRequestMethod, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -36,6 +43,7 @@ impl QueryChanges for Server { &self, request: QueryChangesRequestMethod, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let mut response; let mut is_mutable = true; @@ -53,6 +61,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::Email, access_token, + using, ) .await? .response; @@ -71,7 +80,9 @@ impl QueryChanges for Server { .as_ref() .is_some_and(|sort| sort.iter().any(|s| !s.is_immutable())); - results = self.email_query((*request).into(), access_token).await?; + results = self + .email_query((*request).into(), access_token, using) + .await?; } QueryChangesRequestMethod::Mailbox(mut request) => { // Query changes @@ -81,6 +92,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::Mailbox, access_token, + using, ) .await? .response; @@ -93,7 +105,9 @@ impl QueryChanges for Server { } up_to_id = request.up_to_id; - results = self.mailbox_query((*request).into(), access_token).await?; + results = self + .mailbox_query((*request).into(), access_token, using) + .await?; } QueryChangesRequestMethod::EmailSubmission(mut request) => { // Query changes @@ -107,6 +121,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::EmailSubmission, access_token, + using, ) .await? .response; @@ -133,6 +148,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::ContactCard, access_token, + using, ) .await? .response; @@ -146,7 +162,7 @@ impl QueryChanges for Server { up_to_id = request.up_to_id; results = self - .contact_card_query((*request).into(), access_token) + .contact_card_query((*request).into(), access_token, using) .await?; } QueryChangesRequestMethod::FileNode(mut request) => { @@ -161,6 +177,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::FileNode, access_token, + using, ) .await? .response; @@ -174,10 +191,14 @@ impl QueryChanges for Server { up_to_id = request.up_to_id; results = self - .file_node_query((*request).into(), access_token) + .file_node_query((*request).into(), access_token, using) .await?; } QueryChangesRequestMethod::CalendarEvent(mut request) => { + if request.arguments.expand_recurrences.unwrap_or(false) { + return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); + } + // Query changes resolve_account_id( &mut request.account_id, @@ -189,6 +210,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::CalendarEvent, access_token, + using, ) .await? .response; @@ -202,7 +224,7 @@ impl QueryChanges for Server { up_to_id = request.up_to_id; results = self - .calendar_event_query((*request).into(), access_token) + .calendar_event_query((*request).into(), access_token, using) .await?; } QueryChangesRequestMethod::CalendarEventNotification(mut request) => { @@ -217,6 +239,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::CalendarEventNotification, access_token, + using, ) .await? .response; @@ -245,6 +268,7 @@ impl QueryChanges for Server { build_changes_request(&request), MethodObject::ShareNotification, access_token, + using, ) .await? .response; @@ -262,23 +286,32 @@ impl QueryChanges for Server { QueryChangesRequestMethod::Principal(_) => { return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } - QueryChangesRequestMethod::Quota(_) => { + QueryChangesRequestMethod::Quota(mut request) => { + resolve_account_id(&mut request.account_id, MethodObject::Quota, access_token)?; + access_token.assert_is_member(request.account_id)?; return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } } if has_changes { if is_mutable { + let changed = changes + .created + .iter() + .chain(changes.updated.iter()) + .copied() + .collect::>(); for (index, id) in results.ids.into_iter().enumerate() { - if changes.created.contains(&id) || changes.updated.contains(&id) { + if changed.contains(&id) { response.added.push(AddedItem::new(id, index)); } } response.removed = changes.updated; } else { + let created = changes.created.iter().copied().collect::>(); for (index, id) in results.ids.into_iter().enumerate() { - if changes.created.contains(&id) { + if created.contains(&id) { response.added.push(AddedItem::new(id, index)); } if matches!(up_to_id, Some(up_to_id) if up_to_id == id) { @@ -302,6 +335,7 @@ fn build_changes_request(req: &QueryChangesRequest) -> Changes account_id: req.account_id, since_state: req.since_query_state.clone(), max_changes: req.max_changes, + ignore_metadata_only_changes: Some(false), } } diff --git a/crates/jmap/src/changes/state.rs b/crates/jmap/src/changes/state.rs index 076fda71..f69c6892 100644 --- a/crates/jmap/src/changes/state.rs +++ b/crates/jmap/src/changes/state.rs @@ -4,11 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{DavResources, MessageStoreCache, Server}; +use common::{GroupwareResources, MessageStoreCache, Server, storage::metadata::MetadataViewer}; use jmap_proto::types::state::State; use std::future::Future; use trc::AddContext; -use types::{ChangeId, collection::SyncCollection}; +use types::{ + ChangeId, + collection::{Collection, SyncCollection}, +}; pub trait StateManager: Sync + Send { fn get_state( @@ -25,6 +28,18 @@ pub trait StateManager: Sync + Send { ) -> impl Future> + Send; } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ViewerChangeId(ChangeId); + +pub trait MetadataStateManager: Sync + Send { + fn viewer_change_id( + &self, + viewer: Option, + account_id: u32, + collection: Collection, + ) -> impl Future> + Send; +} + pub trait JmapCacheState: Sync + Send { fn get_state(&self, is_container: bool) -> State; @@ -67,6 +82,48 @@ impl StateManager for Server { } } +impl ViewerChangeId { + pub fn state(self, shared: State) -> State { + max_state(shared, self.0) + } + + pub fn assert_state(self, shared: State, if_in_state: &Option) -> trc::Result { + let old_state = self.state(shared); + if let Some(if_in_state) = if_in_state + && &old_state != if_in_state + { + return Err(trc::JmapEvent::StateMismatch.into_err()); + } + Ok(old_state) + } +} + +impl MetadataStateManager for Server { + async fn viewer_change_id( + &self, + viewer: Option, + account_id: u32, + collection: Collection, + ) -> trc::Result { + match viewer { + Some(viewer) => self + .metadata_viewer_state(viewer, account_id, collection) + .await + .caused_by(trc::location!()) + .map(|state| ViewerChangeId(state.change_id)), + None => Ok(ViewerChangeId::default()), + } + } +} + +pub(crate) fn max_state(state: State, change_id: ChangeId) -> State { + match state { + State::Initial if change_id != 0 => State::Exact(change_id), + State::Exact(current) => State::Exact(current.max(change_id)), + state => state, + } +} + #[inline(always)] fn cache_state(change_id: ChangeId) -> State { (change_id != 0).then_some(change_id).into() @@ -82,7 +139,7 @@ impl JmapCacheState for MessageStoreCache { } } -impl JmapCacheState for DavResources { +impl JmapCacheState for GroupwareResources { fn get_state(&self, is_container: bool) -> State { cache_state(if is_container { self.container_change_id diff --git a/crates/jmap/src/contact/copy.rs b/crates/jmap/src/contact/copy.rs index a6e68b41..a6a93200 100644 --- a/crates/jmap/src/contact/copy.rs +++ b/crates/jmap/src/contact/copy.rs @@ -4,33 +4,46 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{changes::state::JmapCacheState, contact::set::ContactCardSet}; +use crate::{ + api::metadata::{ + MetadataPatches, MetadataPreload, MetadataType, MetadataWriter, NewMetadata, ObjectMetadata, + }, + changes::state::JmapCacheState, + contact::{ + CreatedUids, + set::{ContactCardSet, too_many_contacts}, + }, +}; +use calcard::jscontact::JSContactProperty; use common::{Server, auth::AccessToken}; -use groupware::{cache::GroupwareCache, contact::ContactCard}; +use groupware::{cache::GroupwareCache, contact::ContactCardContent}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ - copy::{CopyRequest, CopyResponse}, + copy::{CopyRequest, CopyResponse, CopySourceId}, set::SetRequest, }, object::contact, request::{ - Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, + Call, MaybeInvalid, RequestMethod, SetRequestMethod, + capability::CapabilityIds, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, types::state::State, }; +use registry::schema::enums::StorageQuota; use store::{ ValueKey, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, + field::ContactField, + metadata::MetadataKinds, }; use utils::map::vec_map::VecMap; @@ -41,6 +54,7 @@ pub trait JmapContactCardCopy: Sync + Send { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -51,6 +65,7 @@ impl JmapContactCardCopy for Server { access_token: &AccessToken, next_call: &mut Option>>, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); @@ -61,15 +76,17 @@ impl JmapContactCardCopy for Server { .into_err() .details("From accountId is equal to fromAccountId")); } + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::ContactCard); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await .caused_by(trc::location!())?; - let old_state = cache.assert_state(false, &request.if_in_state)?; + let old_state = sampled.assert_state(cache.get_state(false), &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, @@ -77,20 +94,22 @@ impl JmapContactCardCopy for Server { old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), + destroyed: None, }; let from_cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::AddressBook, ) .await .caused_by(trc::location!())?; - let from_contact_ids = if access_token.is_member(from_account_id) { - from_cache.document_ids(false).collect::() - } else { - from_cache.shared_items(access_token, [Acl::ReadItems], true) + let shared_from_contact_ids = (!access_token.is_member(from_account_id)) + .then(|| from_cache.shared_items(access_token, [Acl::ReadItems], true)); + let is_readable_source = |document_id: u32| match &shared_from_contact_ids { + Some(ids) => ids.contains(document_id), + None => from_cache.has_item_id(&document_id), }; let can_add_address_books = if access_token.is_shared(account_id) { @@ -102,18 +121,67 @@ impl JmapContactCardCopy for Server { }; let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); + let mut created_slots: Vec<(String, Slot)> = Vec::new(); // Obtain quota + let quota = self.object_quota_usage(&account, StorageQuota::MaxContactCards, || { + cache.resources.count(false) + }); + let mut disk_quota = self + .disk_quota_budget(&account) + .await + .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); + let viewer = metadata.viewer(); + let mut metadata_writer = MetadataWriter::new(metadata, account_id); + let mut preload = MetadataPreload::default(); + for document_id in request + .create + .values() + .filter_map(|create| create.source_id(JSContactProperty::Id)) + .map(|source_id| source_id.document_id()) + .filter(|document_id| is_readable_source(*document_id)) + { + let kinds = from_cache + .item_by_id(document_id) + .map_or(MetadataKinds::NONE, |resource| resource.metadata_kinds()); + if viewer.is_some() || !kinds.is_empty() { + preload.insert_source(document_id, kinds); + } + } + metadata_writer + .preload(self, from_account_id, preload) + .await?; + let mut created_uids = CreatedUids::default(); - 'create: for (id, create) in request.create.into_valid() { - let from_contact_id = id.document_id(); - if !from_contact_ids.contains(from_contact_id) { + 'create: for (create_id, mut create) in request.create { + if !quota.has_room(created_slots.len()) { + response.not_created.append(create_id, too_many_contacts()); + continue; + } + + let source_id = match create.take_source_id(JSContactProperty::Id) { + Ok(source_id) => source_id, + Err(err) => { + response.not_created.append(create_id, err); + continue; + } + }; + let patches = match metadata_writer.extract(MetadataPatches::for_create(), &mut create) + { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(create_id, err); + continue; + } + }; + let from_contact_id = source_id.document_id(); + if !is_readable_source(from_contact_id) { response.not_created.append( - id, + create_id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + source_id, response.from_account_id )), ); continue; @@ -121,25 +189,26 @@ impl JmapContactCardCopy for Server { let Some(_contact) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::property( from_account_id, Collection::ContactCard, from_contact_id, + ContactField::Content, )) .await? else { response.not_created.append( - id, + create_id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + source_id, response.from_account_id )), ); continue; }; let contact = _contact - .deserialize::() + .deserialize::() .caused_by(trc::location!())?; match self @@ -147,24 +216,30 @@ impl JmapContactCardCopy for Server { &cache, &mut batch, access_token, - &account, + &mut disk_quota, account_id, &can_add_address_books, + &mut created_uids, contact.card.into_jscontact(), create, + &mut metadata_writer, + Some(NewMetadata::Copy { + patches, + source_id: from_contact_id, + }), ) .await? { Ok(document_id) => { - response.created(id, document_id); + created_slots.push((create_id, document_id)); // Add to destroy list if on_success_delete { - destroy_ids.push(MaybeInvalid::Value(id)); + destroy_ids.push(MaybeInvalid::Value(source_id)); } } Err(err) => { - response.not_created.append(id, err); + response.not_created.append(create_id, err); continue 'create; } } @@ -172,13 +247,28 @@ impl JmapContactCardCopy for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; + let assigned_ids = match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + for (create_id, _) in created_slots { + response.not_created.append( + create_id, + SetError::forbidden().with_description( + "Another process modified this contact card, please try again.", + ), + ); + } + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; - response.new_state = State::Exact(change_id); + for (create_id, slot) in created_slots { + response.created(create_id, assigned_ids.slot(slot)); + } + + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::AddressBook)); } // Destroy ids diff --git a/crates/jmap/src/contact/get.rs b/crates/jmap/src/contact/get.rs index c0baf257..2229c94d 100644 --- a/crates/jmap/src/contact/get.rs +++ b/crates/jmap/src/contact/get.rs @@ -4,25 +4,34 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::changes::state::JmapCacheState; -use calcard::jscontact::{JSContactProperty, JSContactValue, import::ConversionOptions}; -use common::{Server, auth::AccessToken}; -use groupware::{cache::GroupwareCache, contact::ContactCard}; +use crate::{ + api::{ + budget::GetBudget, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + blob::embedded::{EmbeddedBlobIds, import_error}, + changes::state::JmapCacheState, +}; +use calcard::jscontact::{JSContactProperty, JSContactValue, import::ImportOptions}; +use common::{ + Server, + auth::AccessToken, + network::memory::MemoryReservation, + storage::batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, +}; +use groupware::{cache::GroupwareCache, contact::ContactCardContent}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::contact, + request::capability::CapabilityIds, }; use jmap_tools::{Map, Value}; -use store::{ - ValueKey, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive}, -}; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, + field::ContactField, id::Id, }; @@ -31,6 +40,8 @@ pub trait ContactCardGet: Sync + Send { &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> impl Future>> + Send; } @@ -39,42 +50,73 @@ impl ContactCardGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; let return_all_properties = request.properties.is_none(); - let properties = - request.unwrap_properties(&[JSContactProperty::Id, JSContactProperty::AddressBookIds]); + let (properties, selection) = select_properties( + &mut request, + &[JSContactProperty::Id, JSContactProperty::AddressBookIds], + using, + )?; + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::ContactCard); + let metadata_get = metadata.get(selection); let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await?; - let contact_ids = if access_token.is_member(account_id) { - cache.document_ids(false).collect::() - } else { - cache.shared_items(access_token, [Acl::ReadItems], true) + let shared_contact_ids = (!access_token.is_member(account_id)) + .then(|| cache.shared_items(access_token, [Acl::ReadItems], true)); + let readable_contact = |document_id: u32| { + shared_contact_ids + .as_ref() + .is_none_or(|ids| ids.contains(document_id)) + .then(|| cache.item_by_id(document_id)) + .flatten() }; let ids = if let Some(ids) = ids { ids + } else if let Some(shared_contact_ids) = &shared_contact_ids { + all_ids( + shared_contact_ids.iter().map(Into::into), + self.core.jmap.get_max_objects, + )? } else { - contact_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + cache.document_ids(false).map(Into::into), + self.core.jmap.get_max_objects, + )? + }; + let mut metadata_values = match &metadata_get { + Some(get) => { + let mut documents = MetadataDocuments::default(); + for id in &ids { + let document_id = id.document_id(); + if let Some(resource) = readable_contact(document_id) { + documents.insert(document_id, resource.metadata_kinds()); + } + } + Some(metadata.load(self, account_id, get, &documents).await?) + } + None => None, }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(false).into(), + state: sampled.state(cache.get_state(false)).into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; let mut return_id = return_all_properties; let mut return_address_book_ids = return_all_properties; let mut return_converted_props = !return_all_properties; + let mut return_blob_ids = return_all_properties; + let mut needs_content = return_all_properties; if !return_all_properties { for property in &properties { @@ -87,73 +129,128 @@ impl ContactCardGet for Server { } JSContactProperty::VCard => { return_converted_props = true; + return_blob_ids = true; + needs_content = true; + } + JSContactProperty::Media | JSContactProperty::Localizations => { + return_blob_ids = true; + needs_content = true; + } + _ => { + needs_content = true; } - _ => {} } } } - for id in ids { - // Obtain the contact object - let document_id = id.document_id(); - if !contact_ids.contains(document_id) { - response.push_not_found(id); - continue; + let mut budget = GetBudget::new(reservation); + let mut batch = DocumentBatch::new(MAX_BATCH_BYTES); + for chunk in ids.chunks(MAX_BATCH_DOCUMENTS) { + batch.reset(); + if needs_content { + for id in chunk { + if readable_contact(id.document_id()).is_some() { + batch.request(id.document_id()); + } + } } - let _contact = if let Some(contact) = self - .store() - .get_value::>(ValueKey::archive( + loop { + self.read_archive_batch( account_id, Collection::ContactCard, - document_id, - )) - .await? - { - contact - } else { - response.push_not_found(id); - continue; - }; - - let contact = _contact - .deserialize::() - .caused_by(trc::location!())?; - - let jscontact = contact - .card - .into_jscontact_with_options::( - ConversionOptions::default().include_vcard_parameters(return_converted_props), + ContactField::Content.into(), + &mut batch, ) - .into_inner(); - let mut result = if return_all_properties { - jscontact.into_object().unwrap() - } else { - Map::from_iter( - jscontact - .into_expanded_object() - .filter(|(k, _)| k.as_property().is_some_and(|p| properties.contains(p))), - ) - }; + .await?; - if return_id { - result.insert_unchecked( - JSContactProperty::Id, - Value::Element(JSContactValue::Id(id)), - ); - } + for &id in chunk { + let document_id = id.document_id(); + let Some(resource) = readable_contact(document_id) else { + if batch.is_first_round() { + response.push_not_found(id); + } + continue; + }; - if return_address_book_ids { - let mut obj = Map::with_capacity(contact.names.len()); - for id in contact.names.iter() { - obj.insert_unchecked(JSContactProperty::IdValue(Id::from(id.parent_id)), true); + let mut result = if needs_content { + let content = match batch.lookup(document_id) { + Lookup::Found(content) => content, + Lookup::Missing => { + response.push_not_found(id); + continue; + } + Lookup::Skip => continue, + }; + budget.add_archive(content.as_bytes().len(), 1)?; + let content = content + .deserialize::() + .caused_by(trc::location!())?; + + let options = + ImportOptions::new().include_vcard_parameters(return_converted_props); + let jscontact = if return_blob_ids { + content.card.into_jscontact_with::( + options.with_blob_id_generator(EmbeddedBlobIds::new( + account_id, + Collection::ContactCard, + document_id, + )), + ) + } else { + content.card.into_jscontact_with::(options) + } + .map_err(import_error)? + .into_inner(); + + if return_all_properties { + jscontact.into_object().unwrap() + } else { + Map::from_iter(jscontact.into_expanded_object().filter(|(k, _)| { + k.as_property().is_some_and(|p| properties.contains(p)) + })) + } + } else if batch.is_first_round() { + Map::with_capacity(2) + } else { + continue; + }; + + if return_id { + result.insert_unchecked( + JSContactProperty::Id, + Value::Element(JSContactValue::Id(id)), + ); + } + + if return_address_book_ids { + let names = resource.child_names(); + let mut obj = Map::with_capacity(names.len()); + for name in names { + obj.insert_unchecked( + JSContactProperty::IdValue(Id::from(name.parent_id)), + true, + ); + } + result.insert_unchecked( + JSContactProperty::AddressBookIds, + Value::Object(obj), + ); + } + if let Some(values) = &mut metadata_values { + values.insert_into(document_id, &mut result); + } + + response.list.push(result.into()); } - result.insert_unchecked(JSContactProperty::AddressBookIds, Value::Object(obj)); - } - response.list.push(result.into()); + if !batch.next_round() { + break; + } + } } + budget.commit(); Ok(response) } } diff --git a/crates/jmap/src/contact/mod.rs b/crates/jmap/src/contact/mod.rs index 0f5cb5fa..4cb201a8 100644 --- a/crates/jmap/src/contact/mod.rs +++ b/crates/jmap/src/contact/mod.rs @@ -5,10 +5,10 @@ */ use calcard::jscontact::JSContactProperty; -use common::{DavName, DavResources, Server}; +use common::{DavName, GroupwareResources, NO_ID}; use jmap_proto::error::set::SetError; -use trc::AddContext; -use types::{collection::Collection, field::ContactField, id::Id}; +use store::ahash::AHashMap; +use types::id::Id; pub mod copy; pub mod get; @@ -16,32 +16,51 @@ pub mod parse; pub mod query; pub mod set; -pub(super) async fn assert_is_unique_uid( - server: &Server, - resources: &DavResources, - account_id: u32, +#[derive(Default)] +pub struct CreatedUids(AHashMap>); + +impl CreatedUids { + pub fn record(&mut self, address_books: &[DavName], uid: Option<&str>) { + let Some(uid) = uid else { + return; + }; + let address_book_ids = address_books.iter().map(|name| name.parent_id); + match self.0.get_mut(uid) { + Some(ids) => ids.extend(address_book_ids), + None => { + self.0.insert(uid.to_string(), address_book_ids.collect()); + } + } + } + + fn address_book_with(&self, addressbook_ids: &[DavName], uid: &str) -> Option { + self.0.get(uid)?.iter().copied().find(|address_book_id| { + addressbook_ids + .iter() + .any(|name| name.parent_id == *address_book_id) + }) + } +} + +pub(super) fn assert_is_unique_uid( + resources: &GroupwareResources, + created_uids: &CreatedUids, addressbook_ids: &[DavName], uid: Option<&str>, ) -> trc::Result>>> { if let Some(uid) = uid { - let hits = server - .document_ids_matching( - account_id, - Collection::ContactCard, - ContactField::Uid, - uid.as_bytes(), - ) - .await - .caused_by(trc::location!())?; + let hits = resources.uid_matches(uid); if !hits.is_empty() { for document_id in resources .paths .iter() - .filter(move |item| { - item.parent_id - .is_some_and(|id| addressbook_ids.iter().any(|ab| ab.parent_id == id)) + .filter(move |(_, path)| { + path.parent_id != NO_ID + && addressbook_ids + .iter() + .any(|ab| ab.parent_id == path.parent_id) }) - .map(|path| resources.resources[path.resource_idx].document_id) + .map(|(_, path)| path.document_id) { if hits.contains(document_id) { return Ok(Err(SetError::invalid_properties() @@ -53,6 +72,15 @@ pub(super) async fn assert_is_unique_uid( } } } + + if let Some(address_book_id) = created_uids.address_book_with(addressbook_ids, uid) { + return Ok(Err(SetError::invalid_properties() + .with_property(JSContactProperty::Uid) + .with_description(format!( + "Contact with UID {uid} is already created in address book {}.", + Id::from(address_book_id) + )))); + } } Ok(Ok(())) diff --git a/crates/jmap/src/contact/query.rs b/crates/jmap/src/contact/query.rs index 840629a5..6be0d574 100644 --- a/crates/jmap/src/contact/query.rs +++ b/crates/jmap/src/contact/query.rs @@ -4,30 +4,32 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; +use crate::{ + api::{ + metadata::{ + MetadataType, ObjectMetadata, ResourceScope, filter_containers, flagged_documents, + }, + query::QueryResponseBuilder, + }, + calendar_event::query::{matched_items, rank_comparator}, + changes::state::JmapCacheState, +}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::{ - addressbook::AddressBook, + addressbook::{AddressBook, AddressBookFilter}, contact::{ContactCard, ContactCardComparator, ContactCardFilter}, }, - request::MaybeInvalid, - types::state::State, + request::{MaybeInvalid, capability::CapabilityIds}, }; use store::{ - IterateParams, U32_LEN, U64_LEN, ValueKey, roaring::RoaringBitmap, - search::{ContactSearchField, SearchComparator, SearchFilter, SearchQuery}, - write::{IndexPropertyClass, SearchIndex, ValueClass, key::DeserializeBigEndian}, -}; -use trc::AddContext; -use types::{ - acl::Acl, - collection::{Collection, SyncCollection}, - field::ContactField, + search::{ContactSearchField, QueryResults, SearchFilter, SearchQuery}, + write::SearchIndex, }; +use types::{acl::Acl, collection::SyncCollection}; use utils::sanitize_email; pub trait ContactCardQuery: Sync + Send { @@ -35,93 +37,52 @@ pub trait ContactCardQuery: Sync + Send { &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; fn address_book_query( &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } -#[derive(Clone)] -struct CreatedUpdated { - document_id: u32, - created: u64, - updated: u64, -} - impl ContactCardQuery for Server { async fn contact_card_query( &self, mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::ContactCard); + let sampled = metadata.viewer_change_id(self, account_id).await?; + let metadata_query = + metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(ContactCardFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await?; - let mut created_to_updated = Vec::new(); - - if request.filter.iter().any(|cond| { - matches!( - cond, - Filter::Property( - ContactCardFilter::CreatedBefore(_) - | ContactCardFilter::CreatedAfter(_) - | ContactCardFilter::UpdatedBefore(_) - | ContactCardFilter::UpdatedAfter(_) - ) - ) - }) || request.sort.as_ref().is_some_and(|v| { - v.iter().any(|sort| { - matches!( - sort.property, - ContactCardComparator::Created | ContactCardComparator::Updated - ) + let is_shared = access_token.is_shared(account_id); + let mask = if is_shared { + cache.shared_items(access_token, [Acl::ReadItems], true) + } else { + cache.document_ids(false).collect() + }; + let mut metadata_leaves = metadata + .evaluate(self, account_id, &metadata_query, || { + flagged_documents(&cache, ResourceScope::Items, is_shared.then_some(&mask)) }) - }) { - self.store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::ContactCard.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: ContactField::CreatedToUpdated.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::ContactCard.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: ContactField::CreatedToUpdated.into(), - value: u64::MAX, - }), - }, - ) - .ascending(), - |key, value| { - created_to_updated.push(CreatedUpdated { - document_id: key.deserialize_be_u32(key.len() - U32_LEN)?, - created: key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, - updated: value.deserialize_be_u64(0)?, - }); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - } + .await? + .into_iter(); for cond in std::mem::take(&mut request.filter) { match cond { @@ -175,10 +136,10 @@ impl ContactCardQuery for Server { filters.push(SearchFilter::has_keyword(ContactSearchField::Member, value)); } ContactCardFilter::Kind(value) => { - filters.push(SearchFilter::eq(ContactSearchField::Kind, value)); + filters.push(SearchFilter::text_eq(ContactSearchField::Kind, value)); } - ContactCardFilter::Uid(value) => { - filters.push(SearchFilter::eq(ContactSearchField::Uid, value)) + ContactCardFilter::Uid(uid) => { + filters.push(SearchFilter::is_in_set(cache.uid_matches(&uid))); } ContactCardFilter::Email(email) => filters.push(SearchFilter::has_keyword( ContactSearchField::Email, @@ -222,37 +183,50 @@ impl ContactCardQuery for Server { filters.push(SearchFilter::End); } ContactCardFilter::CreatedBefore(before) => { - let before = before.timestamp() as u64; + let before = before.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - created_to_updated - .iter() - .filter_map(|cu| (cu.created < before).then_some(cu.document_id)), + cache.resources.iter().filter_map(|r| { + r.created_at() + .filter(|created_at| *created_at < before) + .map(|_| r.document_id()) + }), ))); } ContactCardFilter::CreatedAfter(after) => { - let after = after.timestamp() as u64; + let after = after.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - created_to_updated - .iter() - .filter_map(|cu| (cu.created > after).then_some(cu.document_id)), + cache.resources.iter().filter_map(|r| { + r.created_at() + .filter(|created_at| *created_at > after) + .map(|_| r.document_id()) + }), ))); } ContactCardFilter::UpdatedBefore(before) => { - let before = before.timestamp() as u64; + let before = before.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - created_to_updated - .iter() - .filter_map(|cu| (cu.updated < before).then_some(cu.document_id)), + cache.resources.iter().filter_map(|r| { + r.modified_at() + .filter(|modified_at| *modified_at < before) + .map(|_| r.document_id()) + }), ))); } ContactCardFilter::UpdatedAfter(after) => { - let after = after.timestamp() as u64; + let after = after.timestamp(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - created_to_updated - .iter() - .filter_map(|cu| (cu.updated > after).then_some(cu.document_id)), + cache.resources.iter().filter_map(|r| { + r.modified_at() + .filter(|modified_at| *modified_at > after) + .map(|_| r.document_id()) + }), ))); } + ContactCardFilter::Metadata(_) => { + filters.push(SearchFilter::is_in_set( + metadata_leaves.next().unwrap_or_default(), + )); + } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() @@ -274,31 +248,17 @@ impl ContactCardQuery for Server { } } - let comparators = request + let sort = request .sort .take() .unwrap_or_default() .into_iter() .map(|comparator| match comparator.property { - ContactCardComparator::Created => Ok(SearchComparator::sorted_set( - created_to_updated - .iter() - .enumerate() - .map(|(idx, u)| (u.document_id, idx as u32)) - .collect(), - comparator.is_ascending, - )), + ContactCardComparator::Created => { + Ok((ContactSortKey::Created, comparator.is_ascending)) + } ContactCardComparator::Updated => { - let mut updated = created_to_updated.clone(); - updated.sort_by_key(|a| a.updated); - Ok(SearchComparator::sorted_set( - updated - .iter() - .enumerate() - .map(|(idx, u)| (u.document_id, idx as u32)) - .collect(), - comparator.is_ascending, - )) + Ok((ContactSortKey::Updated, comparator.is_ascending)) } other => Err(trc::JmapEvent::UnsupportedSort .into_err() @@ -308,23 +268,35 @@ impl ContactCardQuery for Server { let results = self .search_store() - .query_account( + .filter_account( SearchQuery::new(SearchIndex::Contacts) .with_filters(filters) - .with_comparators(comparators) .with_account_id(account_id) - .with_mask(if access_token.is_shared(account_id) { - cache.shared_items(access_token, [Acl::ReadItems], true) - } else { - cache.document_ids(false).collect() - }), + .with_mask(mask), ) .await?; + let results = if results.len() > 1 && !sort.is_empty() { + let items = matched_items(&cache, &results); + let comparators = sort + .into_iter() + .map(|(key, is_ascending)| match key { + ContactSortKey::Created => { + rank_comparator(&items, |item| item.created_at(), is_ascending) + } + ContactSortKey::Updated => { + rank_comparator(&items, |item| item.modified_at(), is_ascending) + } + }) + .collect(); + QueryResults::new(results, comparators).into_sorted() + } else { + results.into_iter().collect() + }; let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, - cache.get_state(false), + sampled.state(cache.get_state(false)), &request, ); @@ -341,22 +313,56 @@ impl ContactCardQuery for Server { &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::AddressBook); + let mut metadata_filters = Vec::new(); + for filter in &request.filter { + match filter { + Filter::Property(AddressBookFilter::Metadata(filter)) => { + metadata_filters.push(filter); + } + Filter::Property(AddressBookFilter::_T(other)) => { + return Err(trc::JmapEvent::UnsupportedFilter + .into_err() + .details(other.clone())); + } + Filter::And | Filter::Or | Filter::Not | Filter::Close => {} + } + } + let metadata_query = metadata.query(metadata_filters)?; + let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await?; - let results = cache.document_ids(true).collect::>(); + let is_member = access_token.is_member(account_id); + let readable = if is_member { + cache.document_ids(true).collect::() + } else { + cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) + }; + let leaves = metadata + .evaluate(self, account_id, &metadata_query, || { + flagged_documents( + &cache, + ResourceScope::Containers, + (!is_member).then_some(&readable), + ) + }) + .await?; + let results = filter_containers(&request.filter, leaves, readable); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, - State::Initial, + sampled.state(cache.get_state(true)), &request, ); @@ -369,3 +375,9 @@ impl ContactCardQuery for Server { response.build() } } + +#[derive(Debug, Clone, Copy)] +enum ContactSortKey { + Created, + Updated, +} diff --git a/crates/jmap/src/contact/set.rs b/crates/jmap/src/contact/set.rs index b67b13a0..4e707955 100644 --- a/crates/jmap/src/contact/set.rs +++ b/crates/jmap/src/contact/set.rs @@ -4,34 +4,47 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::api::metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, NewMetadata, + ObjectMetadata, PreparedMetadata, is_empty_update, reject_uncommitted, +}; +use crate::api::pending_creates::PendingCreates; +use crate::blob::embedded::EmbeddedExport; use crate::changes::state::JmapCacheState; -use crate::contact::assert_is_unique_uid; +use crate::contact::{CreatedUids, assert_is_unique_uid}; use calcard::jscontact::{JSContact, JSContactProperty, JSContactValue}; use common::{ - DavName, DavResources, Server, - auth::{AccessToken, AccountCache}, + DavName, GroupwareResources, Server, + auth::AccessToken, + storage::quota::{DiskQuotaBudget, ObjectQuotaUsage}, +}; +use groupware::{ + DestroyArchive, SizeWriter, + cache::GroupwareCache, + contact::{ContactCard, ContactCardContent}, }; -use groupware::{DestroyArchive, cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::contact, - request::MaybeInvalid, + request::{MaybeInvalid, capability::CapabilityIds}, types::state::State, }; use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Value}; +use registry::schema::enums::StorageQuota; use store::{ ValueKey, ahash::AHashSet, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot}, }; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection, VanishedCollection}, + field::ContactField, id::Id, }; @@ -41,20 +54,24 @@ pub trait ContactCardSet: Sync + Send { request: SetRequest<'_, contact::ContactCard>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; #[allow(clippy::too_many_arguments)] fn create_contact_card( &self, - cache: &DavResources, + cache: &GroupwareResources, batch: &mut BatchBuilder, access_token: &AccessToken, - account: &AccountCache, + disk_quota: &mut DiskQuotaBudget, account_id: u32, can_add_address_books: &Option, + created_uids: &mut CreatedUids, js_contact: JSContact<'_, Id, BlobId>, updates: Value<'_, JSContactProperty, JSContactValue>, - ) -> impl Future>>>>; + metadata_writer: &mut MetadataWriter, + new_metadata: Option, + ) -> impl Future>>>>; } impl ContactCardSet for Server { @@ -63,23 +80,28 @@ impl ContactCardSet for Server { mut request: SetRequest<'_, contact::ContactCard>, access_token: &AccessToken, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::ContactCard); + let sampled = metadata.viewer_change_id(self, account_id).await?; let account = self.account(account_id).await.caused_by(trc::location!())?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::AddressBook, ) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(false, &request.if_in_state)?); + .with_state(sampled.assert_state(cache.get_state(false), &request.if_in_state)?); + let mut metadata_writer = MetadataWriter::new(metadata, account_id); let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); // Obtain addressBookIds + let is_shared = access_token.is_shared(account_id); let (can_add_address_books, can_delete_address_books, can_modify_address_books) = - if access_token.is_shared(account_id) { + if is_shared { ( cache .shared_containers(access_token, [Acl::AddItems], true) @@ -95,24 +117,58 @@ impl ContactCardSet for Server { (None, None, None) }; + // Obtain quota + let quota = if request.has_creates() { + self.object_quota_usage(&account, StorageQuota::MaxContactCards, || { + cache.resources.count(false) + }) + } else { + ObjectQuotaUsage::unlimited() + }; + let mut disk_quota = if request.has_creates() || request.has_updates() { + self.disk_quota_budget(&account) + .await + .caused_by(trc::location!())? + } else { + DiskQuotaBudget::unlimited() + }; + // Process creates let mut batch = BatchBuilder::new(); - 'create: for (id, object) in request.unwrap_create() { + let mut created_slots = PendingCreates::new(); + let mut created_uids = CreatedUids::default(); + 'create: for (id, mut object) in request.unwrap_create() { + if !quota.has_room(created_slots.len()) { + response.not_created.append(id, too_many_contacts()); + continue 'create; + } + let new_metadata = + match metadata_writer.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches.map(NewMetadata::Create), + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + match self .create_contact_card( &cache, &mut batch, access_token, - &account, + &mut disk_quota, account_id, &can_add_address_books, + &mut created_uids, JSContact::default(), object, + &mut metadata_writer, + new_metadata, ) .await? { Ok(document_id) => { - response.created(id, document_id); + created_slots.push(id, document_id); } Err(err) => { response.not_created.append(id, err); @@ -122,7 +178,14 @@ impl ContactCardSet for Server { } // Process updates - 'update: for (id, object) in request.unwrap_update() { + metadata_writer + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .item_by_id(document_id) + .map(|resource| resource.metadata_kinds()) + }) + .await?; + 'update: for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -135,12 +198,20 @@ impl ContactCardSet for Server { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } + let mut metadata_patches = + match metadata_writer.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }; // Obtain contact card let document_id = id.document_id(); let contact_card_ = if let Some(contact_card_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, @@ -155,10 +226,64 @@ impl ContactCardSet for Server { let contact_card = contact_card_ .to_unarchived::() .caused_by(trc::location!())?; + let metadata_access = + if is_shared && metadata_patches.is_some() { + MetadataAccess::from_container_acls( + contact_card.inner.names.iter().map(|name| { + cache.container_acl(access_token, name.parent_id.to_native()) + }), + ) + } else { + MetadataAccess::FULL + }; + if !metadata_access.may_read { + response.not_updated.append(id, SetError::not_found()); + continue 'update; + } + if object + .as_object() + .is_some_and(|object| is_empty_update::(object, id)) + && let Some(patches) = metadata_patches.take() + { + match metadata_writer + .write_metadata_only::( + self, + patches, + metadata_access, + &contact_card, + document_id, + &mut batch, + ) + .await? + { + Ok(()) => response.updated.append(id, None), + Err(err) => response.not_updated.append(id, err), + } + continue 'update; + } + let Some(content_) = self + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::ContactCard, + document_id, + ContactField::Content, + )) + .await? + else { + response.not_updated.append(id, SetError::not_found()); + continue 'update; + }; + let content = content_ + .to_unarchived::() + .caused_by(trc::location!())?; let mut new_contact_card = contact_card .deserialize::() .caused_by(trc::location!())?; - let mut js_contact = new_contact_card.card.into_jscontact(); + let mut new_content = content + .deserialize::() + .caused_by(trc::location!())?; + let mut js_contact = new_content.card.into_jscontact(); // Process changes if let Err(err) = update_contact_card( @@ -172,22 +297,32 @@ impl ContactCardSet for Server { } // Convert JSContact to vCard - if let Some(vcard) = js_contact.into_vcard() { - new_contact_card.size = vcard.size() as u32; - new_contact_card.card = vcard; - } else { - response.not_updated.append( - id, - SetError::invalid_properties() - .with_description("Failed to convert contact to vCard."), - ); - continue 'update; + match self.export_vcard(access_token, js_contact).await? { + Ok(vcard) => { + new_content.card = vcard; + } + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } } // Validate UID - match (new_contact_card.card.uid(), contact_card.inner.card.uid()) { - (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} - (None, None) | (None, Some(_)) => {} + let mut assigned_uid = None; + match ( + content.inner.card.uid().filter(|uid| !uid.is_empty()), + new_content.card.uid().filter(|uid| !uid.is_empty()), + ) { + (Some(stored_uid), Some(uid)) if stored_uid == uid => {} + (None, uid) => { + if let Err(err) = + assert_is_unique_uid(&cache, &created_uids, &new_contact_card.names, uid)? + { + response.not_updated.append(id, err); + continue 'update; + } + assigned_uid = uid.map(str::to_string); + } _ => { response.not_updated.append( id, @@ -199,6 +334,17 @@ impl ContactCardSet for Server { } } + // Validate addressBookIds limit + let max_address_books = self.core.groupware.max_address_books_per_card; + if new_contact_card.names.len() > max_address_books + && new_contact_card.names.len() > contact_card.inner.names.len() + { + response + .not_updated + .append(id, too_many_address_books(max_address_books)); + continue 'update; + } + // Validate new addressBookIds for addressbook_id in new_contact_card.added_addressbook_ids(contact_card.inner) { if !cache.has_container_id(&addressbook_id) { @@ -261,42 +407,79 @@ impl ContactCardSet for Server { } // Check size and quota - if new_contact_card.size as usize > self.core.groupware.max_vcard_size { + let size = SizeWriter::vcard(&new_content.card, self.core.groupware.vcard_version); + if size > self.core.groupware.max_vcard_size { response.not_updated.append( id, - SetError::invalid_properties().with_description(format!( - "Contact size {} exceeds the maximum allowed size of {} bytes.", - new_contact_card.size, self.core.groupware.max_vcard_size - )), + contact_too_large(size, self.core.groupware.max_vcard_size), ); continue 'update; } - let extra_bytes = (new_contact_card.size as u64) - .saturating_sub(u32::from(contact_card.inner.size) as u64); - if extra_bytes > 0 { - match self.has_available_quota(&account, extra_bytes).await { - Ok(_) => {} - Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { - response.not_updated.append(id, SetError::over_quota()); - continue 'update; - } - Err(err) => return Err(err.caused_by(trc::location!())), - } + let extra_bytes = + (size as u64).saturating_sub(u32::from(contact_card.inner.size) as u64); + if disk_quota.check(extra_bytes).is_err() { + response.not_updated.append(id, SetError::over_quota()); + continue 'update; } + let prepared_metadata = match metadata_patches { + Some(patches) => match metadata_writer + .prepare_for( + self, + patches, + MetadataAccess { + may_write_shared: metadata_access.may_write_shared + && (!is_shared + || new_contact_card + .added_addressbook_ids(contact_card.inner) + .all(|addressbook_id| { + cache + .container_acl(access_token, addressbook_id) + .contains(Acl::ModifyMetadata) + })), + ..metadata_access + }, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => Some(prepared), + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }, + None => None, + }; + // Update record + disk_quota.reserve(extra_bytes); let vanished_paths = new_contact_card .removed_addressbook_ids(contact_card.inner) .filter_map(|addressbook_id| { cache.format_resource_path_by_parent(document_id, addressbook_id) }) .collect::>(); + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + new_contact_card.set_metadata_kinds(kinds); + } + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, &mut batch)?; + } + created_uids.record(&new_contact_card.names, assigned_uid.as_deref()); new_contact_card - .update( - access_token.account_tenant_ids(), + .update_full( + new_content, + self.core.groupware.vcard_version, + access_token.changed_by(), contact_card, + content.inner, account_id, document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -307,6 +490,23 @@ impl ContactCardSet for Server { } // Process deletions + let cleanup = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::ContactCard, + &will_destroy + .iter() + .map(|id| id.document_id()) + .filter(|&document_id| { + cache + .item_by_id(document_id) + .is_some_and(|item| !item.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; 'destroy: for id in will_destroy { let document_id = id.document_id(); @@ -317,7 +517,7 @@ impl ContactCardSet for Server { let Some(contact_card_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, @@ -352,12 +552,16 @@ impl ContactCardSet for Server { // Delete record DestroyArchive(contact_card) - .delete_all( - access_token.account_tenant_ids(), + .remove_all( + self, + access_token.changed_by(), account_id, document_id, + None, + &cleanup, &mut batch, ) + .await .caused_by(trc::location!())?; for path in cache.format_resource_paths_by_id(document_id) { @@ -369,15 +573,24 @@ impl ContactCardSet for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; + let assigned_ids = match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut response, + created_slots.into_create_ids(), + "Another process modified this contact card, please try again.", + ); + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; - self.notify_task_queue(); + created_slots.resolve(&mut response, &assigned_ids); - response.new_state = State::Exact(change_id).into(); + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::AddressBook)) + .into(); } Ok(response) @@ -385,20 +598,28 @@ impl ContactCardSet for Server { async fn create_contact_card( &self, - cache: &DavResources, + cache: &GroupwareResources, batch: &mut BatchBuilder, access_token: &AccessToken, - account: &AccountCache, + disk_quota: &mut DiskQuotaBudget, account_id: u32, can_add_address_books: &Option, + created_uids: &mut CreatedUids, mut js_contact: JSContact<'_, Id, BlobId>, updates: Value<'_, JSContactProperty, JSContactValue>, - ) -> trc::Result>>> { + metadata_writer: &mut MetadataWriter, + new_metadata: Option, + ) -> trc::Result>>> { // Process changes let mut names = Vec::new(); if let Err(err) = update_contact_card(None, updates, &mut names, &mut js_contact) { return Ok(Err(err)); } + if names.len() > self.core.groupware.max_address_books_per_card { + return Ok(Err(too_many_address_books( + self.core.groupware.max_address_books_per_card, + ))); + } // Verify that the address book ids valid for name in &names { @@ -421,57 +642,97 @@ impl ContactCardSet for Server { } // Convert JSContact to vCard - let Some(card) = js_contact.into_vcard() else { - return Ok(Err(SetError::invalid_properties() - .with_description("Failed to convert contact to vCard."))); + let card = match self.export_vcard(access_token, js_contact).await? { + Ok(card) => card, + Err(err) => return Ok(Err(err)), }; // Validate UID - if let Err(err) = assert_is_unique_uid(self, cache, account_id, &names, card.uid()).await? { + if let Err(err) = assert_is_unique_uid(cache, created_uids, &names, card.uid())? { return Ok(Err(err)); } // Check size and quota - let size = card.size(); + let size = SizeWriter::vcard(&card, self.core.groupware.vcard_version); if size > self.core.groupware.max_vcard_size { - return Ok(Err(SetError::invalid_properties().with_description( - format!( - "Contact size {} exceeds the maximum allowed size of {} bytes.", - size, self.core.groupware.max_vcard_size - ), + return Ok(Err(contact_too_large( + size, + self.core.groupware.max_vcard_size, ))); } - match self.has_available_quota(account, size as u64).await { - Ok(_) => {} - Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { - return Ok(Err(SetError::over_quota())); - } - Err(err) => return Err(err.caused_by(trc::location!())), + if disk_quota.check(size as u64).is_err() { + return Ok(Err(SetError::over_quota())); } + let metadata_access = match (&new_metadata, can_add_address_books) { + (Some(_), Some(_)) => MetadataAccess::from_container_acls( + names + .iter() + .map(|name| cache.container_acl(access_token, name.parent_id)), + ), + _ => MetadataAccess::FULL, + }; + let prepared_metadata = match metadata_writer + .prepare_new(self, new_metadata, metadata_access) + .await? + { + Ok(prepared) => prepared, + Err(err) => return Ok(Err(err)), + }; + // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::ContactCard, 1) - .await - .caused_by(trc::location!())?; - ContactCard { + disk_quota.reserve(size as u64); + let document_id = batch.reserve_document_id(account_id, Collection::ContactCard); + let mut contact_card = ContactCard { names, - size: size as u32, - card, ..Default::default() + }; + if let Some(kinds) = prepared_metadata + .as_ref() + .and_then(PreparedMetadata::shared_kinds) + { + contact_card.set_metadata_kinds(kinds); } - .insert( - access_token.account_tenant_ids(), - account_id, - document_id, - batch, - ) - .caused_by(trc::location!()) - .map(|_| Ok(document_id)) + if let Some(prepared) = prepared_metadata { + metadata_writer.write(prepared, document_id, batch)?; + } + created_uids.record(&contact_card.names, card.uid()); + contact_card + .insert( + ContactCardContent { card }, + self.core.groupware.vcard_version, + access_token.changed_by(), + account_id, + document_id, + None, + batch, + ) + .caused_by(trc::location!())?; + Ok(Ok(document_id)) } } +pub(crate) fn too_many_contacts() -> SetError> { + SetError::over_quota().with_description(concat!( + "There are too many contact cards, ", + "please delete some before adding a new one." + )) +} + +fn contact_too_large(size: usize, max_size: usize) -> SetError> { + SetError::too_large().with_description(format!( + "Contact size {size} exceeds the maximum allowed size of {max_size} bytes." + )) +} + +fn too_many_address_books(max: usize) -> SetError> { + SetError::invalid_properties() + .with_property(JSContactProperty::AddressBookIds) + .with_description(format!( + "A contact card cannot belong to more than {max} address books." + )) +} + fn update_contact_card<'x>( expected_id: Option, updates: Value<'x, JSContactProperty, JSContactValue>, @@ -502,25 +763,17 @@ fn update_contact_card<'x>( pointer.next(); patch_parent_ids(addressbooks, pointer.next(), value)?; } else if !js_contact.0.patch_jptr(pointer.iter(), value) { - return Err(SetError::invalid_properties() + let error = if expected_id.is_some() { + SetError::invalid_patch() + } else { + SetError::invalid_properties() + }; + return Err(error .with_property(JSContactProperty::Pointer(pointer)) .with_description("Patch operation failed.")); } entries = js_contact.0.as_object_mut().unwrap(); } - (JSContactProperty::Media, Value::Object(media)) => { - for (_, value) in media.iter() { - if value.as_object().is_some_and(|v| { - v.keys() - .any(|k| matches!(k, Key::Property(JSContactProperty::BlobId))) - }) { - return Err(SetError::invalid_properties() - .with_property(JSContactProperty::Media) - .with_description("blobIds in media is not supported.")); - } - } - entries.insert(JSContactProperty::Media, Value::Object(media)); - } (JSContactProperty::Id, value) => { if !expected_id.is_some_and(|expected| crate::matches_id(&value, expected)) { return Err(SetError::invalid_properties() diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 1ac288fb..86deedfb 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -5,35 +5,46 @@ */ use crate::{ + api::metadata::{ + MetadataAccess, MetadataPatches, MetadataPreload, MetadataType, MetadataWriter, + NewMetadata, ObjectMetadata, + }, changes::state::JmapCacheState, - email::{PatchResult, handle_email_patch, ingested_into_object}, + email::{PatchResult, handle_email_patch, ingested_into_object, parse_keywords}, +}; +use common::{ + MAX_RECEIVED_AT, Server, auth::AccessToken, ipc::PushNotification, sharing::EffectiveAcl, }; -use common::{Server, auth::AccessToken, ipc::PushNotification}; use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + cache::{ + MessageCacheFetch, + email::{MessageAccess, MessageCacheAccess}, + mailbox::MailboxCacheAccess, + }, mailbox::JUNK_ID, message::{ - copy::{CopyMessageError, EmailCopy}, - ingest::EmailIngest, + copy::{CopyMessageError, CopySources, EmailCopy}, + ingest::{EmailIngest, SpamSample, SpamSamples}, }, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ - copy::{CopyRequest, CopyResponse}, + copy::{CopyRequest, CopyResponse, CopySourceId}, set::SetRequest, }, object::email::{Email, EmailProperty, EmailValue}, request::{ - Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, + Call, MaybeInvalid, RequestMethod, SetRequestMethod, + capability::CapabilityIds, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, }; use jmap_tools::{Key, Value}; use std::future::Future; -use store::write::BatchBuilder; +use store::write::{BatchBuilder, now}; use trc::AddContext; use types::{ acl::Acl, @@ -48,6 +59,7 @@ pub trait JmapEmailCopy: Sync + Send { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -58,6 +70,7 @@ impl JmapEmailCopy for Server { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); @@ -67,8 +80,10 @@ impl JmapEmailCopy for Server { .into_err() .details("From accountId is equal to fromAccountId")); } + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Email); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self.get_cached_messages(account_id).await?; - let old_state = cache.assert_state(false, &request.if_in_state)?; + let old_state = sampled.assert_state(cache.get_state(false), &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, @@ -76,16 +91,17 @@ impl JmapEmailCopy for Server { old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), + destroyed: None, }; let from_cache = self .get_cached_messages(from_account_id) .await .caused_by(trc::location!())?; - let from_message_ids = if access_token.is_member(from_account_id) { - from_cache.email_document_ids() + let from_access = if access_token.is_member(from_account_id) { + MessageAccess::All } else { - from_cache.shared_messages(access_token, Acl::ReadItems) + MessageAccess::Mailboxes(from_cache.shared_mailboxes(access_token, Acl::ReadItems)) }; let can_add_mailbox_ids = if access_token.is_shared(account_id) { @@ -93,24 +109,62 @@ impl JmapEmailCopy for Server { } else { None }; + let is_shared = access_token.is_shared(account_id); + let viewer = metadata.viewer(); let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); - let mut train_batch = BatchBuilder::new(); - let mut did_train = false; + let mut spam_samples = SpamSamples::default(); let mut last_change_id = None; - train_batch.with_account_id(from_account_id); - 'create: for (id, create) in request.create.into_valid() { - let mut from_message_id = None; + let mut metadata_writer = MetadataWriter::new(metadata, account_id); + let mut preload = MetadataPreload::default(); + let mut source_ids = Vec::with_capacity(request.create.len()); + for document_id in request + .create + .values() + .filter_map(|create| create.source_id(EmailProperty::Id)) + .map(|source_id| source_id.document_id()) + { + source_ids.push(document_id); + let Some(email) = from_cache + .email_by_id(&document_id) + .filter(|email| from_access.allows(*email)) + else { + continue; + }; + let kinds = from_cache.metadata_kinds(email); + if viewer.is_some() || !kinds.is_empty() { + preload.insert_source(document_id, kinds); + } + } + metadata_writer + .preload(self, from_account_id, preload) + .await?; + let mut sources = CopySources::new(from_account_id, source_ids); + let account = self.account(account_id).await.caused_by(trc::location!())?; + + 'create: for (id, mut create) in request.create { + let from_message_id = match create.take_source_id(EmailProperty::Id) { + Ok(source_id) => source_id, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + let metadata_patches = + match metadata.extract(MetadataPatches::for_create(), &mut create) { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; let mut mailboxes = Vec::new(); let mut keywords = Vec::new(); let mut received_at = None; for (property, value) in create.into_expanded_object() { match (property, value) { - (Key::Property(EmailProperty::Id), Value::Element(EmailValue::Id(src))) => { - from_message_id = Some(src); - } (Key::Property(EmailProperty::MailboxIds), Value::Object(ids)) => { mailboxes = ids .into_expanded_boolean_set() @@ -119,11 +173,14 @@ impl JmapEmailCopy for Server { }) .collect(); } - (Key::Property(EmailProperty::Keywords), Value::Object(keywords_)) => { - keywords = keywords_ - .into_expanded_boolean_set() - .filter_map(|id| id.try_into_property()?.try_into_keyword()) - .collect(); + (Key::Property(EmailProperty::Keywords), keywords_ @ Value::Object(_)) => { + match parse_keywords(keywords_) { + Ok(keywords_) => keywords = keywords_, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + } } (Key::Property(EmailProperty::Pointer(pointer)), value) => { match handle_email_patch(&pointer, value) { @@ -144,7 +201,7 @@ impl JmapEmailCopy for Server { mailboxes.retain(|mid| mid != &id); } PatchResult::Invalid(set_error) => { - response.not_created.append(id, set_error); + response.not_created.append(id.clone(), set_error); continue 'create; } } @@ -153,11 +210,12 @@ impl JmapEmailCopy for Server { Key::Property(EmailProperty::ReceivedAt), Value::Element(EmailValue::Date(value)), ) => { - received_at = value.into(); + received_at = + (value.timestamp().clamp(0, MAX_RECEIVED_AT as i64) as u64).into(); } (property, _) => { response.not_created.append( - id, + id.clone(), SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Invalid property or value.".to_string()), @@ -167,21 +225,15 @@ impl JmapEmailCopy for Server { } } - let Some(from_message_id) = from_message_id else { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(EmailProperty::Id) - .with_description("Missing or invalid \"id\" property."), - ); - continue 'create; - }; - if !from_message_ids.contains(from_message_id.document_id()) { + if !from_cache + .email_by_id(&from_message_id.document_id()) + .is_some_and(|email| from_access.allows(email)) + { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + from_message_id, response.from_account_id )), ); continue 'create; @@ -190,7 +242,7 @@ impl JmapEmailCopy for Server { // Make sure message belongs to at least one mailbox if mailboxes.is_empty() { response.not_created.append( - id, + id.clone(), SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description("Message has to belong to at least one mailbox."), @@ -202,7 +254,7 @@ impl JmapEmailCopy for Server { for mailbox_id in &mailboxes { if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( - id, + id.clone(), SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description(format!("mailboxId {mailbox_id} does not exist.")), @@ -210,7 +262,7 @@ impl JmapEmailCopy for Server { continue 'create; } else if matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(*mailbox_id)) { response.not_created.append( - id, + id.clone(), SetError::forbidden().with_description(format!( "You are not allowed to add messages to mailbox {mailbox_id}." )), @@ -219,46 +271,99 @@ impl JmapEmailCopy for Server { } } + // Validate per-email limits + if let Err(err) = self + .core + .email + .limits + .validate_email(mailboxes.len(), &keywords) + { + response.not_created.append(id.clone(), err.into()); + continue 'create; + } + + let access = if is_shared && (metadata_patches.is_some() || viewer.is_some()) { + MetadataAccess::from_container_acls(mailboxes.iter().map(|mailbox_id| { + cache + .mailbox_by_id(mailbox_id) + .map(|mailbox| mailbox.acls.as_slice().effective_acl(access_token)) + .unwrap_or_default() + })) + } else { + MetadataAccess::FULL + }; + let ingest_metadata = match metadata_writer + .prepare_ingest( + self, + NewMetadata::Copy { + patches: metadata_patches, + source_id: from_message_id.document_id(), + }, + access, + ) + .await? + { + Ok(metadata) => metadata, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; + // Add response - let train_spam = mailboxes.contains(&JUNK_ID); + let Some(source) = sources + .take(self, from_message_id.document_id()) + .await + .caused_by(trc::location!())? + else { + response.not_created.append( + id, + SetError::not_found().with_description("Message not found in account."), + ); + continue 'create; + }; + let sample = if mailboxes.contains(&JUNK_ID) && self.core.spam.classifier.is_some() { + Some( + SpamSample::from_metadata(source.metadata(), true) + .caused_by(trc::location!())?, + ) + } else { + None + }; match self .copy_message( - from_account_id, - from_message_id.document_id(), - account_id, + source, + &account, mailboxes, keywords, - received_at.map(|dt| dt.timestamp() as u64), + received_at.unwrap_or_else(|| { + from_cache + .email_by_id(&from_message_id.document_id()) + .map(|v| v.received_at()) + .unwrap_or_else(now) + }), + ingest_metadata, session.session_id, ) .await? { Ok(email) => { last_change_id = Some(email.change_id); - if train_spam { - self.add_account_spam_sample( - &mut train_batch, - from_account_id, - from_message_id.document_id(), - true, - session.session_id, - ) - .await - .caused_by(trc::location!())?; - train_batch.commit_point(); - did_train = true; + if let Some(sample) = sample { + spam_samples.add_sample(sample); } response .created .append(id, ingested_into_object(email).into()); + if on_success_delete { + destroy_ids.push(MaybeInvalid::Value(from_message_id)); + } } Err(err) => { response.not_created.append( id, match err { - CopyMessageError::NotFound => SetError::not_found() - .with_description("Message not found in account."), CopyMessageError::OverQuota => SetError::over_quota(), CopyMessageError::AlreadyExists(existing) => SetError::already_exists() .with_existing_id(types::id::Id::from(existing)), @@ -266,17 +371,23 @@ impl JmapEmailCopy for Server { ); } } - - // Add to destroy list - if on_success_delete { - destroy_ids.push(MaybeInvalid::Value(from_message_id)); - } } - if did_train { - self.commit_batch(train_batch) - .await - .caused_by(trc::location!())?; + if !spam_samples.is_empty() { + let mut train_batch = BatchBuilder::new(); + self.add_account_spam_samples( + &mut train_batch, + from_account_id, + spam_samples, + session.session_id, + ) + .await + .caused_by(trc::location!())?; + if !train_batch.is_empty() { + self.commit_batch(train_batch) + .await + .caused_by(trc::location!())?; + } } // Update state @@ -290,7 +401,9 @@ impl JmapEmailCopy for Server { .with_change(DataType::Thread), )) .await; - response.new_state = self.get_cached_messages(account_id).await?.get_state(false); + let sampled = metadata.viewer_change_id(self, account_id).await?; + response.new_state = + sampled.state(self.get_cached_messages(account_id).await?.get_state(false)); } // Destroy ids diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index cf4804da..e00962bf 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -4,48 +4,58 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::changes::state::JmapCacheState; -use common::{Server, auth::AccessToken}; +use crate::{ + api::{ + budget::GetBudget, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + changes::state::{JmapCacheState, MetadataStateManager}, +}; +use common::{ + Server, + auth::AccessToken, + network::memory::MemoryReservation, + storage::batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, +}; +use compact_str::format_compact; use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess}, + cache::{ + MessageCacheFetch, + email::{MessageAccess, MessageCacheAccess}, + mailbox::MailboxCacheAccess, + }, message::{ - body::{ToBodyPart, truncate_html, truncate_plain}, - headers::{HeaderToValue, IntoForm}, - metadata::{ - ArchivedMetadataPartType, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, - MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM, + jmap::{ + BodyValueOptions, EmailNeeds, EmailRender, + estimate::{EmailCost, EmailEstimate}, }, + messagedata::{EmailMessageData, MetadataForm}, }, }; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, - object::email::{Email, EmailProperty, EmailValue, HeaderForm}, - request::IntoValid, + method::get::{GetRequest, GetResponse, all_ids, unique_properties}, + object::email::{Email, EmailProperty, EmailValue}, + request::{IntoValid, capability::CapabilityIds}, types::date::UTCDate, }; -use jmap_tools::{Key, Map, Value}; -use mail_parser::HeaderValue; +use jmap_tools::{Map, Value}; use std::future::Future; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; use trc::{AddContext, StoreEvent}; use types::{ acl::Acl, blob::{BlobClass, BlobId}, - blob_hash::BlobHash, collection::Collection, - field::EmailField, id::Id, + keyword::HASATTACHMENT, }; -use utils::chained_bytes::ChainedBytes; pub trait EmailGet: Sync + Send { fn email_get( &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> impl Future>> + Send; } @@ -54,38 +64,54 @@ impl EmailGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, + reservation: &mut MemoryReservation, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - EmailProperty::Id, - EmailProperty::BlobId, - EmailProperty::ThreadId, - EmailProperty::MailboxIds, - EmailProperty::Keywords, - EmailProperty::Size, - EmailProperty::ReceivedAt, - EmailProperty::MessageId, - EmailProperty::InReplyTo, - EmailProperty::References, - EmailProperty::Sender, - EmailProperty::From, - EmailProperty::To, - EmailProperty::Cc, - EmailProperty::Bcc, - EmailProperty::ReplyTo, - EmailProperty::Subject, - EmailProperty::SentAt, - EmailProperty::HasAttachment, - EmailProperty::Preview, - EmailProperty::BodyValues, - EmailProperty::TextBody, - EmailProperty::HtmlBody, - EmailProperty::Attachments, - ]); + let is_default = request.properties.is_none(); + let (properties, selection) = select_properties( + &mut request, + &[ + EmailProperty::Id, + EmailProperty::BlobId, + EmailProperty::ThreadId, + EmailProperty::MailboxIds, + EmailProperty::Keywords, + EmailProperty::Size, + EmailProperty::ReceivedAt, + EmailProperty::MessageId, + EmailProperty::InReplyTo, + EmailProperty::References, + EmailProperty::Sender, + EmailProperty::From, + EmailProperty::To, + EmailProperty::Cc, + EmailProperty::Bcc, + EmailProperty::ReplyTo, + EmailProperty::Subject, + EmailProperty::SentAt, + EmailProperty::HasAttachment, + EmailProperty::Preview, + EmailProperty::BodyValues, + EmailProperty::TextBody, + EmailProperty::HtmlBody, + EmailProperty::Attachments, + ], + using, + )?; + let object_metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Email); + let metadata = if is_default { + None + } else { + object_metadata.get(selection) + }; + let metadata_slots = metadata.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); let body_properties = request .arguments .body_properties - .map(|v| v.into_valid().collect()) + .map(|v| unique_properties(v.into_valid())) .unwrap_or_else(|| { vec![ EmailProperty::PartId, @@ -100,350 +126,282 @@ impl EmailGet for Server { EmailProperty::Location, ] }); - let fetch_text_body_values = request.arguments.fetch_text_body_values.unwrap_or(false); - let fetch_html_body_values = request.arguments.fetch_html_body_values.unwrap_or(false); - let fetch_all_body_values = request.arguments.fetch_all_body_values.unwrap_or(false); - let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); + let options = BodyValueOptions { + fetch_text: request.arguments.fetch_text_body_values.unwrap_or(false), + fetch_html: request.arguments.fetch_html_body_values.unwrap_or(false), + fetch_all: request.arguments.fetch_all_body_values.unwrap_or(false), + max_bytes: request.arguments.max_body_value_bytes.unwrap_or(0), + }; + let mut needs_metadata = false; + for property in &properties { + let is_cached = matches!( + property, + EmailProperty::Id + | EmailProperty::ThreadId + | EmailProperty::MailboxIds + | EmailProperty::Keywords + | EmailProperty::Size + | EmailProperty::ReceivedAt + | EmailProperty::HasAttachment + ); + if !is_cached && (!EmailRender::renders(property) || !property.is_allowed_form()) { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format_compact!("Invalid property {property:?}"))); + } + needs_metadata |= !is_cached; + } + if let Some(property) = body_properties + .iter() + .find(|property| !property.is_allowed_form()) + { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format_compact!("Invalid property {property:?}"))); + } + let needs = EmailNeeds::new(&properties, &body_properties, &options); + let estimate = EmailEstimate::new(&properties, &body_properties, &options); + let mut budget = GetBudget::new(reservation); let account_id = request.account_id.document_id(); + let sampled = self + .viewer_change_id( + object_metadata.state_viewer(), + account_id, + Collection::Email, + ) + .await?; let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - let message_ids = if access_token.is_member(account_id) { - cache.email_document_ids() + let access = if access_token.is_member(account_id) { + MessageAccess::All } else { - cache.shared_messages(access_token, Acl::ReadItems) + MessageAccess::Mailboxes(cache.shared_mailboxes(access_token, Acl::ReadItems)) }; let ids = if let Some(ids) = ids { ids } else { - cache - .emails - .items - .iter() - .take(self.core.jmap.get_max_objects) - .map(|item| Id::from_parts(item.thread_id, item.document_id)) - .collect() + all_ids( + cache + .emails + .iter() + .filter(|item| access.allows(*item)) + .map(|item| Id::from_parts(item.thread_id(), item.document_id())), + self.core.jmap.get_max_objects, + )? }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(false).into(), + state: sampled.state(cache.get_state(false)).into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; - - // Check if we need to fetch the raw headers or body - let mut needs_body = false; - for property in &properties { - if matches!( - property, - EmailProperty::BodyValues - | EmailProperty::TextBody - | EmailProperty::HtmlBody - | EmailProperty::Attachments - | EmailProperty::BodyStructure - ) { - needs_body = true; - break; - } - } - - for id in ids { - // Obtain the email object - if !message_ids.contains(id.document_id()) { - response.push_not_found(id); - continue; - } - let metadata_ = match self - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - id.document_id(), - EmailField::Metadata, - )) - .await? - { - Some(metadata) => metadata, - None => { - response.push_not_found(id); - continue; + let mut metadata_values = match &metadata { + Some(metadata) => { + let mut documents = MetadataDocuments::default(); + for document_id in ids.iter().map(Id::document_id) { + if let Some(email) = cache + .email_by_id(&document_id) + .filter(|email| access.allows(*email)) + { + documents.insert(document_id, cache.metadata_kinds(email)); + } } - }; - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; + Some( + object_metadata + .load::(self, account_id, metadata, &documents) + .await?, + ) + } + None => None, + }; - // Obtain message data - let data = match cache.email_by_id(&id.document_id()) { - Some(data) => data, - None => { - response.push_not_found(id); - continue; - } - }; - - // Retrieve raw message if needed - let blob_hash = BlobHash::from(&metadata.blob_hash); - let raw_body; - let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); - if needs_body { - raw_body = self - .blob_store() - .get_blob(blob_hash.as_slice(), 0..usize::MAX) - .await?; - - if let Some(raw_body) = &raw_body { - raw_message.append( - raw_body - .get(metadata.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - } else { - trc::event!( - Store(StoreEvent::NotFound), - AccountId = account_id, - DocumentId = id.document_id(), - Collection = Collection::Email, - BlobId = blob_hash.to_hex(), - Details = "Blob not found.", - CausedBy = trc::location!(), - ); - - response.push_not_found(id); - continue; + let form = if needs.headers { + MetadataForm::Row + } else { + MetadataForm::Structure + }; + let mut batch = DocumentBatch::new(MAX_BATCH_BYTES); + for chunk in ids.chunks(MAX_BATCH_DOCUMENTS) { + batch.reset(); + if needs_metadata { + for id in chunk { + if cache + .email_by_id(&id.document_id()) + .is_some_and(|data| access.allows(data)) + { + batch.request(id.document_id()); + } } } - let blob_id = BlobId { - hash: blob_hash, - class: BlobClass::Linked { - account_id, - collection: Collection::Email.into(), - document_id: id.document_id(), - }, - section: None, - }; - // Prepare response - let mut email: Map<'_, EmailProperty, EmailValue> = - Map::with_capacity(properties.len()); - let contents = &metadata.contents[0]; - let root_part = &contents.parts[0]; - let blob_body_offset = metadata.blob_body_offset.to_native() as isize - - root_part.offset_body.to_native() as isize; - for property in &properties { - match property { - EmailProperty::Id => { - email.insert_unchecked(EmailProperty::Id, Id::from(*id)); - } - EmailProperty::ThreadId => { - email.insert_unchecked(EmailProperty::ThreadId, Id::from(id.prefix_id())); - } - EmailProperty::BlobId => { - email.insert_unchecked(EmailProperty::BlobId, blob_id.clone()); - } - EmailProperty::MailboxIds => { - let mut obj = Map::with_capacity(data.mailboxes.len()); - for id in data.mailboxes.iter() { - debug_assert!(id.uid != 0); - obj.insert_unchecked( - EmailProperty::IdValue(Id::from(id.mailbox_id)), - true, - ); + loop { + self.read_metadata_batch(account_id, form, &mut batch) + .await + .caused_by(trc::location!())?; + + for &id in chunk { + let Some(data) = cache + .email_by_id(&id.document_id()) + .filter(|data| access.allows(*data)) + else { + if batch.is_first_round() { + response.push_not_found(id); } - - email.insert_unchecked(property.clone(), Value::Object(obj)); - } - EmailProperty::Keywords => { - let mut obj = Map::with_capacity(2); - for keyword in cache.expand_keywords(data) { - obj.insert_unchecked(EmailProperty::Keyword(keyword), true); + continue; + }; + let metadata = if needs_metadata { + match batch.lookup(id.document_id()) { + Lookup::Found(metadata) => Some(metadata), + Lookup::Missing => { + response.push_not_found(id); + continue; + } + Lookup::Skip => continue, } - email.insert_unchecked(property.clone(), Value::Object(obj)); - } - EmailProperty::Size => { - email.insert_unchecked(EmailProperty::Size, data.size); - } - EmailProperty::ReceivedAt => { - email.insert_unchecked( - EmailProperty::ReceivedAt, - EmailValue::Date(UTCDate::from_timestamp( - (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, - )), - ); - } - EmailProperty::Preview => { - if !metadata.preview.is_empty() { - email.insert_unchecked( - EmailProperty::Preview, - metadata.preview.to_string(), - ); + } else if batch.is_first_round() { + None + } else { + continue; + }; + + let raw_headers; + let mut blob = None; + let mut cost = EmailCost::default(); + let render = if let Some(metadata) = metadata { + raw_headers = metadata.raw_headers().caused_by(trc::location!())?; + let metadata = metadata.unarchive().caused_by(trc::location!())?; + cost = estimate.cost(metadata); + budget.add(cost.bytes)?; + budget.add_light(cost.light)?; + let blob_hash = metadata.blob_hash(); + if needs.blob { + let Some(value) = self + .fetch_blob(&blob_hash, 0..usize::MAX, Some(metadata.size() as u64)) + .await? + else { + trc::event!( + Store(StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id.document_id(), + Collection = Collection::Email, + BlobId = blob_hash.to_hex(), + Details = "Blob not found.", + CausedBy = trc::location!(), + ); + + response.push_not_found(id); + continue; + }; + blob = Some(value); } - } - EmailProperty::HasAttachment => { - email.insert_unchecked( - EmailProperty::HasAttachment, - (metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0, - ); - } - EmailProperty::Subject => { - email.insert_unchecked( - EmailProperty::Subject, - root_part - .header_value(&MetadataHeaderName::Subject) - .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Text)) - .unwrap_or_default(), - ); - } - EmailProperty::SentAt => { - email.insert_unchecked( - EmailProperty::SentAt, - root_part - .header_value(&MetadataHeaderName::Date) - .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Date)) - .unwrap_or_default(), - ); - } - EmailProperty::MessageId - | EmailProperty::InReplyTo - | EmailProperty::References => { - email.insert_unchecked( - property.clone(), - root_part - .header_value(&match property { - EmailProperty::MessageId => MetadataHeaderName::MessageId, - EmailProperty::InReplyTo => MetadataHeaderName::InReplyTo, - EmailProperty::References => MetadataHeaderName::References, - _ => unreachable!(), - }) - .map(|value| { - HeaderValue::from(value).into_form(&HeaderForm::MessageIds) - }) - .unwrap_or_default(), - ); - } + Some(( + metadata, + raw_headers.as_deref(), + BlobId { + hash: blob_hash, + class: BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id: id.document_id(), + }, + section: None, + }, + )) + } else { + None + }; + let render = render.as_ref().map(|(metadata, headers, blob_id)| { + EmailRender::new( + metadata, + *headers, + blob.as_deref(), + blob_id, + &body_properties, + &options, + ) + }); - EmailProperty::Sender - | EmailProperty::From - | EmailProperty::To - | EmailProperty::Cc - | EmailProperty::Bcc - | EmailProperty::ReplyTo => { - email.insert_unchecked( - property.clone(), - root_part - .header_value(&match property { - EmailProperty::Sender => MetadataHeaderName::Sender, - EmailProperty::From => MetadataHeaderName::From, - EmailProperty::To => MetadataHeaderName::To, - EmailProperty::Cc => MetadataHeaderName::Cc, - EmailProperty::Bcc => MetadataHeaderName::Bcc, - EmailProperty::ReplyTo => MetadataHeaderName::ReplyTo, - _ => unreachable!(), - }) - .map(|value| { - HeaderValue::from(value).into_form(&HeaderForm::Addresses) - }) - .unwrap_or_default(), - ); - } - EmailProperty::Header(_) => { - email.insert_unchecked( - property.clone(), - root_part.header_to_value(property, &raw_message), - ); - } - EmailProperty::Headers => { - email.insert_unchecked( - EmailProperty::Headers, - root_part.headers_to_value(&raw_message), - ); - } - EmailProperty::TextBody - | EmailProperty::HtmlBody - | EmailProperty::Attachments => { - let list = match property { - EmailProperty::TextBody => &contents.text_body, - EmailProperty::HtmlBody => &contents.html_body, - EmailProperty::Attachments => &contents.attachments, - _ => unreachable!(), - } - .iter(); - email.insert_unchecked( - property.clone(), - list.map(|part_id| { - contents.to_body_part( - u16::from(part_id) as u32, - &body_properties, - &raw_message, - &blob_id, - blob_body_offset, - ) - }) - .collect::>(), - ); - } - EmailProperty::BodyStructure => { - email.insert_unchecked( - EmailProperty::BodyStructure, - contents.to_body_part( - 0, - &body_properties, - &raw_message, - &blob_id, - blob_body_offset, - ), - ); - } - EmailProperty::BodyValues => { - let mut body_values = Map::with_capacity(contents.parts.len()); - for (part_id, part) in contents.parts.iter().enumerate() { - if part.is_text_mime_type() - && (fetch_all_body_values - || (fetch_html_body_values - && contents.is_html_part(part_id as u16)) - || (fetch_text_body_values - && contents.is_text_part(part_id as u16))) - { - let contents = part.decode_contents(&raw_message); - - let (is_truncated, value) = match &part.body { - ArchivedMetadataPartType::Text => { - truncate_plain(contents.as_str(), max_body_value_bytes) - } - ArchivedMetadataPartType::Html => { - truncate_html(contents.as_str(), max_body_value_bytes) - } - _ => unreachable!(), - }; - - body_values.insert_unchecked( - Key::Owned(part_id.to_string()), - Map::with_capacity(3) - .with_key_value( - EmailProperty::IsEncodingProblem, - (part.flags & PART_ENCODING_PROBLEM) != 0, - ) - .with_key_value(EmailProperty::IsTruncated, is_truncated) - .with_key_value(EmailProperty::Value, value), + // Prepare response + let mut email: Map<'_, EmailProperty, EmailValue> = + Map::with_capacity(properties.len() + metadata_slots); + for property in &properties { + match property { + EmailProperty::Id => { + email.insert_unchecked(EmailProperty::Id, Id::from(*id)); + } + EmailProperty::ThreadId => { + email.insert_unchecked( + EmailProperty::ThreadId, + Id::from(id.prefix_id()), ); } - } - email.insert_unchecked(EmailProperty::BodyValues, body_values); - } + EmailProperty::MailboxIds => { + let mut obj = Map::with_capacity(data.mailboxes().len()); + for id in data + .mailboxes() + .iter() + .filter(|id| access.allows_mailbox(id.mailbox_id)) + { + debug_assert!(id.uid != 0); + obj.insert_unchecked( + EmailProperty::IdValue(Id::from(id.mailbox_id)), + true, + ); + } - _ => { - return Err(trc::JmapEvent::InvalidArguments - .into_err() - .details(format!("Invalid property {property:?}"))); + email.insert_unchecked(property.clone(), Value::Object(obj)); + } + EmailProperty::Keywords => { + let mut obj = Map::with_capacity(2); + for keyword in cache.expand_keywords(data) { + obj.insert_unchecked(EmailProperty::Keyword(keyword), true); + } + email.insert_unchecked(property.clone(), Value::Object(obj)); + } + EmailProperty::Size => { + email.insert_unchecked(EmailProperty::Size, data.size()); + } + EmailProperty::ReceivedAt => { + email.insert_unchecked( + EmailProperty::ReceivedAt, + EmailValue::Date(UTCDate::from_timestamp( + data.received_at() as i64 + )), + ); + } + EmailProperty::HasAttachment => { + email.insert_unchecked( + EmailProperty::HasAttachment, + (data.keywords() & 1 << HASATTACHMENT) != 0, + ); + } + _ => { + if let Some(value) = + render.as_ref().and_then(|render| render.value(property)) + { + email.insert_unchecked(property.clone(), value); + } + } + } } + if let Some(metadata_values) = &mut metadata_values { + metadata_values.insert_into(id.document_id(), &mut email); + } + budget.add(estimate.overrun(&email, &cost))?; + response.list.push(Value::Object(email).into_owned()); + } + + if !batch.next_round() { + break; } } - response.list.push(email.into()); } + budget.commit(); Ok(response) } } diff --git a/crates/jmap/src/email/grouped.rs b/crates/jmap/src/email/grouped.rs new file mode 100644 index 00000000..a18ed1e1 --- /dev/null +++ b/crates/jmap/src/email/grouped.rs @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Server, auth::AccessToken}; +use email::message::ingest::{ + EmailIngest, IngestEmail, IngestSource, IngestedEmail, + group::{IngestGroup, ThreadKey}, +}; +use mail_parser::{Message, MessageParser}; +use types::{blob_hash::BlobHash, keyword::Keyword}; + +pub(super) struct QueuedEmail { + pub id: String, + pub blob_hash: Option, + pub mailbox_ids: Vec, + pub keywords: Vec, + pub received_at: Option, + pub train_classifier: bool, +} + +pub(super) struct GroupedIngest<'a, R> { + access_token: &'a AccessToken, + session_id: u64, + group: IngestGroup, + queued: Vec<(QueuedEmail, R)>, + pub results: Vec<(String, trc::Result)>, +} + +impl QueuedEmail { + pub fn params<'x>( + &'x self, + raw_message: &'x [u8], + message: Option>, + access_token: &'x AccessToken, + session_id: u64, + ) -> IngestEmail<'x> { + IngestEmail { + raw_message, + message, + blob_hash: self.blob_hash.as_ref(), + access_token, + source: IngestSource::Jmap { + train_classifier: self.train_classifier, + }, + mailbox_ids: self.mailbox_ids.clone(), + keywords: self.keywords.clone(), + received_at: self.received_at, + metadata: None, + session_id, + } + } +} + +impl<'a, R: AsRef<[u8]>> GroupedIngest<'a, R> { + pub fn new(access_token: &'a AccessToken, session_id: u64) -> Self { + GroupedIngest { + access_token, + session_id, + group: IngestGroup::default(), + queued: Vec::new(), + results: Vec::new(), + } + } + + pub fn is_empty(&self) -> bool { + self.queued.is_empty() + } + + pub fn must_flush_before(&self, size: usize) -> bool { + self.group.must_flush_before(size) + } + + pub fn ingest<'x>( + &'x mut self, + server: &'x Server, + email: &'x QueuedEmail, + raw_message: &'x [u8], + message: Option>, + key: ThreadKey, + ) -> impl Future> + 'x { + self.group.ingest( + server, + email.params(raw_message, message, self.access_token, self.session_id), + key, + ) + } + + pub fn queue(&mut self, email: QueuedEmail, raw_message: R, result: trc::Result<()>) { + match result { + Ok(()) => self.queued.push((email, raw_message)), + Err(err) => self.results.push((email.id, Err(err))), + } + } + + pub async fn flush(&mut self, server: &Server) -> trc::Result<()> { + match self.group.flush(server).await { + Ok(emails) => { + self.results.extend( + self.queued + .drain(..) + .map(|(email, _)| email.id) + .zip(emails.into_iter().map(Ok)), + ); + Ok(()) + } + Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { + if self.queued.len() == 1 { + if let Some((email, _)) = self.queued.pop() { + self.results.push((email.id, Err(err))); + } + } else { + for (email, raw_message) in self.queued.drain(..) { + let raw_message = raw_message.as_ref(); + let message = MessageParser::new().parse(raw_message); + let result = Box::pin(server.email_ingest(email.params( + raw_message, + message, + self.access_token, + self.session_id, + ))) + .await; + self.results.push((email.id, result)); + } + } + Ok(()) + } + Err(err) => { + self.queued.clear(); + Err(err) + } + } + } +} diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 3a6a41db..ea6312f0 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -5,23 +5,30 @@ */ use crate::{ - blob::download::BlobDownload, changes::state::JmapCacheState, email::ingested_into_object, + api::metadata::{MetadataType, ObjectMetadata}, + blob::download::BlobDownload, + changes::state::JmapCacheState, + email::{ + grouped::{GroupedIngest, QueuedEmail}, + ingested_into_object, invalid_keywords, + }, }; -use common::{Server, auth::AccessToken, ipc::PushNotification}; +use common::storage::blob::FetchedBlob; +use common::{MAX_RECEIVED_AT, Server, auth::AccessToken, ipc::PushNotification}; use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::JUNK_ID, - message::ingest::{EmailIngest, IngestEmail, IngestSource}, + message::ingest::{group::ThreadKey, recheck::ThreadRecheck}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, object::email::EmailProperty, - request::reference::MaybeIdReference, + request::{capability::CapabilityIds, reference::MaybeIdReference}, types::state::State, }; -use mail_parser::{HeaderName, MessageParser}; +use mail_parser::MessageParser; use std::future::Future; use types::{ acl::Acl, @@ -37,6 +44,7 @@ pub trait EmailImport: Sync + Send { request: ImportEmailRequest, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -46,11 +54,15 @@ impl EmailImport for Server { request: ImportEmailRequest, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result { // Validate state let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Email); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self.get_cached_messages(account_id).await?; - let old_state: State = cache.assert_state(false, &request.if_in_state)?; + let old_state: State = + sampled.assert_state(cache.get_state(false), &request.if_in_state)?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { cache.shared_mailboxes(access_token, Acl::AddItems).into() } else { @@ -59,12 +71,12 @@ impl EmailImport for Server { // Obtain import access token let import_access_token = if account_id != access_token.account_id() { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { AccessToken::from_id_maybe_invalid(account_id).into() } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { use common::auth::BuildAccessToken; use trc::AddContext; @@ -86,7 +98,13 @@ impl EmailImport for Server { not_created: VecMap::new(), }; + let mut grouped = GroupedIngest::new( + import_access_token.as_ref().unwrap_or(access_token), + session.session_id, + ); let mut last_change_id = None; + let mut recheck = ThreadRecheck::default(); + let grouping = request.emails.len() > 1; 'outer: for (id, email) in request.emails { // Validate mailboxIds let mailbox_ids = email @@ -128,6 +146,22 @@ impl EmailImport for Server { } } + if !email.keywords.iter().all(Keyword::is_jmap_valid) { + response.not_created.append(id, invalid_keywords()); + continue; + } + + // Validate per-email limits + if let Err(err) = self + .core + .email + .limits + .validate_email(mailbox_ids.len(), &email.keywords) + { + response.not_created.append(id, err.into()); + continue; + } + let MaybeIdReference::Id(blob_id) = email.blob_id else { response.not_created.append( id, @@ -139,8 +173,11 @@ impl EmailImport for Server { }; // Fetch raw message to import - let raw_message = match self.blob_download(&blob_id, access_token).await? { - Some(raw_message) => raw_message, + let blob = match self + .blob_download_with_extra(&blob_id, access_token) + .await? + { + Some(blob) => blob, None => { response.not_created.append( id, @@ -151,73 +188,87 @@ impl EmailImport for Server { } }; + let is_stored = blob.stored_len(&blob_id) == Some(blob.bytes.len()); + let raw_message = blob.bytes; + // Import message let parsed = MessageParser::new().parse(&raw_message); - let is_valid_message = parsed.as_ref().is_some_and(|message| { - message - .headers() - .iter() - .any(|header| !matches!(header.name, HeaderName::Other(_))) - }); - if !is_valid_message { + let Some(message) = parsed.filter(|message| message.headers().has_known()) else { response.not_created.append( id, SetError::new(SetErrorType::InvalidEmail) .with_description("Blob does not contain a valid RFC 5322 message."), ); continue; + }; + if let Err(err) = self.core.email.limits.validate_header_section(&message) { + response.not_created.append(id, err.into()); + continue; } - match self - .email_ingest(IngestEmail { - raw_message: &raw_message, - message: parsed, - blob_hash: Some(&blob_id.hash), - access_token: import_access_token.as_ref().unwrap_or(access_token), - source: IngestSource::Jmap { - train_classifier: email - .keywords - .iter() - .any(|k| matches!(k, Keyword::Junk | Keyword::NotJunk)) - || mailbox_ids.contains(&JUNK_ID), - }, - mailbox_ids, - keywords: email.keywords, - received_at: email.received_at.map(|r| r.into()), - session_id: session.session_id, - }) + let received_at = email + .received_at + .map(|date| date.timestamp().clamp(0, MAX_RECEIVED_AT as i64) as u64); + + let key = if grouping { + ThreadKey::new(&message) + } else { + ThreadKey::default() + }; + if grouped.must_flush_before(raw_message.len()) + && let Err(err) = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) .await { - Ok(email) => { - last_change_id = Some(email.change_id); - response - .created - .append(id, ingested_into_object(email).into()); - } - Err(mut err) => match err.as_ref() { - trc::EventType::Limit(trc::LimitEvent::Quota) => { - response.not_created.append( - id, - SetError::new(SetErrorType::OverQuota) - .with_description("You have exceeded your disk quota."), - ); - } - trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => { - response.not_created.append( - id, - SetError::new(SetErrorType::InvalidEmail).with_description( - err.take_value(trc::Key::Reason) - .and_then(|v| v.into_string()) - .unwrap(), - ), - ); - } - _ => { - return Err(err); - } - }, + recheck.finish(self, session.session_id).await; + return Err(err); + } + + let email = QueuedEmail { + id, + blob_hash: is_stored.then_some(blob_id.hash), + train_classifier: email + .keywords + .iter() + .any(|k| matches!(k, Keyword::Junk | Keyword::NotJunk)) + || mailbox_ids.contains(&JUNK_ID), + mailbox_ids, + keywords: email.keywords, + received_at, + }; + let result = grouped + .ingest(self, &email, &raw_message, Some(message), key) + .await; + grouped.queue(email, raw_message, result); + if let Err(err) = record( + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) { + recheck.finish(self, session.session_id).await; + return Err(err); } } + let flushed = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) + .await; + recheck.finish(self, session.session_id).await; + flushed?; + // Update state if let Some(change_id) = last_change_id { self.broadcast_push_notification(PushNotification::StateChange( @@ -230,9 +281,64 @@ impl EmailImport for Server { )) .await; - response.new_state = self.get_cached_messages(account_id).await?.get_state(false); + let sampled = metadata.viewer_change_id(self, account_id).await?; + response.new_state = + sampled.state(self.get_cached_messages(account_id).await?.get_state(false)); } Ok(response) } } + +async fn flush( + server: &Server, + grouped: &mut GroupedIngest<'_, FetchedBlob>, + response: &mut ImportEmailResponse, + recheck: &mut ThreadRecheck, + last_change_id: &mut Option, + account_id: u32, +) -> trc::Result<()> { + grouped.flush(server).await?; + record(grouped, response, recheck, last_change_id, account_id) +} + +fn record( + grouped: &mut GroupedIngest<'_, FetchedBlob>, + response: &mut ImportEmailResponse, + recheck: &mut ThreadRecheck, + last_change_id: &mut Option, + account_id: u32, +) -> trc::Result<()> { + for (id, result) in grouped.results.drain(..) { + match result { + Ok(mut email) => { + recheck.push(account_id, &mut email); + *last_change_id = Some(email.change_id); + response + .created + .append(id, ingested_into_object(email).into()); + } + Err(mut err) => match err.as_ref() { + trc::EventType::Limit(trc::LimitEvent::Quota) => { + response.not_created.append( + id, + SetError::new(SetErrorType::OverQuota) + .with_description("You have exceeded your account quota."), + ); + } + trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => { + response.not_created.append( + id, + SetError::new(SetErrorType::InvalidEmail).with_description( + err.take_value(trc::Key::Reason) + .and_then(|v| v.into_string()) + .unwrap_or_default(), + ), + ); + } + _ => return Err(err), + }, + } + } + Ok(()) +} diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index ccd4b559..df95989e 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -14,6 +14,7 @@ use types::{id::Id, keyword::Keyword}; pub mod copy; pub mod get; +mod grouped; pub mod import; pub mod parse; pub mod query; @@ -39,6 +40,22 @@ pub(crate) enum PatchResult<'x> { Invalid(SetError), } +pub(crate) fn parse_keywords<'x>( + keywords: Value<'x, EmailProperty, EmailValue>, +) -> Result, SetError> { + keywords + .into_expanded_boolean_set() + .map(|keyword| keyword.try_into_property()?.try_into_keyword()) + .collect::>>() + .ok_or_else(invalid_keywords) +} + +pub(crate) fn invalid_keywords() -> SetError { + SetError::invalid_properties() + .with_property(EmailProperty::Keywords) + .with_description("Invalid keyword.") +} + pub(crate) fn handle_email_patch<'x>( pointer: &'x JsonPointer, value: Value<'_, EmailProperty, EmailValue>, diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 8c89bdcc..b3b4a3f8 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -6,23 +6,23 @@ use crate::blob::download::BlobDownload; use common::{Server, auth::AccessToken}; -use email::message::index::PREVIEW_LENGTH; +use compact_str::format_compact; use email::message::{ - body::{ToBodyPart, TruncateBody}, - headers::HeaderToValue, + jmap::{BodyValueOptions, EmailRender}, + metadata::{ArchivedMessageMetadata, ExtraHeaders, MessageMetadata}, }; use jmap_proto::{ - method::parse::{ParseRequest, ParseResponse}, + method::{ + get::unique_properties, + parse::{ParseRequest, ParseResponse}, + }, object::email::{Email, EmailProperty}, request::{IntoValid, MaybeInvalid, reference::MaybeIdReference}, }; -use jmap_tools::{Key, Map, Value}; -use mail_parser::{ - HeaderName, MessageParser, MimeHeaders, PartType, decoders::html::html_to_text, - parsers::preview::preview_text, -}; +use jmap_tools::{Map, Value}; +use mail_parser::MessageParser; use std::future::Future; -use utils::{chained_bytes::ChainedBytes, map::vec_map::VecMap}; +use utils::map::vec_map::VecMap; pub trait EmailParse: Sync + Send { fn email_parse( @@ -43,7 +43,7 @@ impl EmailParse for Server { } let properties = request .properties - .map(|v| v.into_valid().collect()) + .map(|v| unique_properties(v.into_valid())) .unwrap_or_else(|| { vec![ EmailProperty::BlobId, @@ -71,7 +71,7 @@ impl EmailParse for Server { let body_properties = request .arguments .body_properties - .map(|v| v.into_valid().collect()) + .map(|v| unique_properties(v.into_valid())) .unwrap_or_else(|| { vec![ EmailProperty::PartId, @@ -86,10 +86,38 @@ impl EmailParse for Server { EmailProperty::Location, ] }); - let fetch_text_body_values = request.arguments.fetch_text_body_values.unwrap_or(false); - let fetch_html_body_values = request.arguments.fetch_html_body_values.unwrap_or(false); - let fetch_all_body_values = request.arguments.fetch_all_body_values.unwrap_or(false); - let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); + let options = BodyValueOptions { + fetch_text: request.arguments.fetch_text_body_values.unwrap_or(false), + fetch_html: request.arguments.fetch_html_body_values.unwrap_or(false), + fetch_all: request.arguments.fetch_all_body_values.unwrap_or(false), + max_bytes: request.arguments.max_body_value_bytes.unwrap_or(0), + }; + if let Some(property) = properties + .iter() + .find(|property| { + !property.is_allowed_form() + || !(EmailRender::renders(property) + || matches!( + property, + EmailProperty::Size + | EmailProperty::HasAttachment + | EmailProperty::Id + | EmailProperty::ThreadId + | EmailProperty::Keywords + | EmailProperty::MailboxIds + | EmailProperty::ReceivedAt + )) + }) + .or_else(|| { + body_properties + .iter() + .find(|property| !property.is_allowed_form()) + }) + { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format_compact!("Invalid property {property:?}"))); + } let mut response = ParseResponse { account_id: request.account_id, @@ -107,186 +135,67 @@ impl EmailParse for Server { } }; // Fetch raw message to parse - let raw_message = match self.blob_download(&blob_id, access_token).await? { - Some(raw_message) => raw_message, + let (raw_message, extra_headers_len) = match self + .blob_download_with_extra(&blob_id, access_token) + .await? + { + Some(blob) => (blob.bytes, blob.extra_headers_len), None => { response.not_found.push(MaybeInvalid::Value(blob_id)); continue; } }; - let message = match MessageParser::new().parse(&raw_message).filter(|message| { - message - .root_part() - .headers() - .iter() - .any(|header| !matches!(header.name, HeaderName::Other(_))) - }) { + let message = match MessageParser::new() + .parse(&raw_message) + .filter(|message| message.headers().has_known()) + { Some(message) => message, None => { response.not_parsable.push(blob_id); continue; } }; - let raw_message = ChainedBytes::new(&raw_message); + let built = + MessageMetadata::build(&message, &ExtraHeaders::default(), blob_id.hash.clone()); + let archive = + rkyv::api::high::to_bytes_in::<_, rkyv::rancor::Error>(&built.metadata, Vec::new()) + .map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + })?; + let metadata = rkyv::access::(&archive) + .map_err(|err| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .reason(err) + })?; + let render = EmailRender::new( + metadata, + Some(&built.raw_headers), + Some(&raw_message), + &blob_id, + &body_properties, + &options, + ) + .with_blob_prefix(extra_headers_len); - // Prepare response let mut email = Map::with_capacity(properties.len()); for property in &properties { - match property { - EmailProperty::BlobId => { - email.insert_unchecked(EmailProperty::BlobId, blob_id.clone()); - } - - EmailProperty::Size => { - email.insert_unchecked( - EmailProperty::Size, - Value::Number(raw_message.len().into()), - ); - } - EmailProperty::HasAttachment => { - email.insert_unchecked( - EmailProperty::HasAttachment, - Value::Bool(message.parts.iter().enumerate().any(|(part_id, part)| { - let part_id = part_id as u32; - match &part.body { - PartType::Html(_) | PartType::Text(_) => { - !message.text_body.contains(&part_id) - && !message.html_body.contains(&part_id) - } - PartType::Binary(_) | PartType::Message(_) => true, - _ => false, - } - })), - ); - } - EmailProperty::Preview => { - email.insert_unchecked( - EmailProperty::Preview, - match message - .text_body - .first() - .or_else(|| message.html_body.first()) - .and_then(|idx| message.parts.get(*idx as usize)) - .map(|part| &part.body) - { - Some(PartType::Text(text)) => { - preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH) - .into() - } - Some(PartType::Html(html)) => preview_text( - html_to_text(html).replace('\r', "").into(), - PREVIEW_LENGTH, - ) - .into(), - _ => Value::Null, - }, - ); - } - EmailProperty::MessageId - | EmailProperty::InReplyTo - | EmailProperty::References - | EmailProperty::Sender - | EmailProperty::From - | EmailProperty::To - | EmailProperty::Cc - | EmailProperty::Bcc - | EmailProperty::ReplyTo - | EmailProperty::Subject - | EmailProperty::SentAt - | EmailProperty::Header(_) => { - email.insert_unchecked( - property.clone(), - message.parts[0] - .headers - .header_to_value(property, &raw_message), - ); - } - EmailProperty::Headers => { - email.insert_unchecked( - EmailProperty::Headers, - message.parts[0].headers.headers_to_value(&raw_message), - ); - } - EmailProperty::TextBody - | EmailProperty::HtmlBody - | EmailProperty::Attachments => { - let list = match property { - EmailProperty::TextBody => &message.text_body, - EmailProperty::HtmlBody => &message.html_body, - EmailProperty::Attachments => &message.attachments, - _ => unreachable!(), - } - .iter(); - email.insert_unchecked( - property.clone(), - list.map(|part_id| { - message.parts.to_body_part( - *part_id, - &body_properties, - &raw_message, - &blob_id, - 0, - ) - }) - .collect::>(), - ); - } - EmailProperty::BodyStructure => { - email.insert_unchecked( - EmailProperty::BodyStructure, - message.parts.to_body_part( - 0, - &body_properties, - &raw_message, - &blob_id, - 0, - ), - ); - } - EmailProperty::BodyValues => { - let mut body_values = Map::with_capacity(message.parts.len()); - for (part_id, part) in message.parts.iter().enumerate() { - let part_id = part_id as u32; - if part.is_text() - && part - .content_type() - .is_none_or(|ct| ct.ctype().eq_ignore_ascii_case("text")) - && (fetch_all_body_values - || (fetch_html_body_values - && message.html_body.contains(&part_id)) - || (fetch_text_body_values - && message.text_body.contains(&part_id))) - { - let (is_truncated, value) = - part.body.truncate(max_body_value_bytes); - body_values.insert_unchecked( - Key::Owned(part_id.to_string()), - Map::with_capacity(3) - .with_key_value( - EmailProperty::IsEncodingProblem, - part.is_encoding_problem, - ) - .with_key_value(EmailProperty::IsTruncated, is_truncated) - .with_key_value(EmailProperty::Value, value), - ); - } - } - email.insert_unchecked(EmailProperty::BodyValues, body_values); - } + let value = match property { + EmailProperty::Size => Value::Number(raw_message.len().into()), + EmailProperty::HasAttachment => Value::Bool(built.has_attachments), EmailProperty::Id | EmailProperty::ThreadId | EmailProperty::Keywords | EmailProperty::MailboxIds - | EmailProperty::ReceivedAt => { - email.insert_unchecked(property.clone(), Value::Null); - } - - _ => { - return Err(trc::JmapEvent::InvalidArguments - .into_err() - .details(format!("Invalid property {property:?}"))); - } - } + | EmailProperty::ReceivedAt => Value::Null, + _ => match render.value(property) { + Some(value) => value.into_owned(), + None => continue, + }, + }; + email.insert_unchecked(property.clone(), value); } response.parsed.append(blob_id, email.into()); } diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index a5cedbb1..b0097609 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,438 +4,667 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; -use common::{MessageStoreCache, Server, auth::AccessToken}; -use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; +mod filter; +mod text; +mod window; + + +use crate::{ + api::{ + metadata::{MetadataType, ObjectMetadata}, + query::QueryResponseBuilder, + }, + changes::state::JmapCacheState, +}; +use common::{ + MessageStoreCache, MessagesCache, Server, auth::AccessToken, cache::email::MessageRef, +}; +use compact_str::ToCompactString; +use email::{ + cache::{ + MessageCacheFetch, + email::{MessageAccess, MessageCacheAccess, thread_keywords}, + mailbox::MailboxCacheAccess, + }, + message::sortkeys::{ + CacheSorter, EmailSortKeys, MessageCacheField, MessageComparator, MessageSortField, + }, +}; +use filter::{QueryFilter, RowFilter}; use jmap_proto::{ - method::query::{Filter, QueryRequest, QueryResponse}, - object::email::{Email, EmailComparator, EmailFilter}, + method::query::{Comparator, Filter, QueryRequest, QueryResponse}, + object::email::{Email, EmailComparator, EmailQueryFilter}, + request::capability::CapabilityIds, + types::state::State, }; -use mail_parser::HeaderName; -use nlp::language::Language; use std::future::Future; use store::{ - ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, - search::{ - EmailSearchField, SearchComparator, SearchFilter, SearchOperator, SearchQuery, SearchValue, - }, + search::{SearchComparator, SearchQuery}, write::SearchIndex, }; use trc::AddContext; -use types::{acl::Acl, keyword::Keyword}; -use utils::map::vec_map::VecMap; +use types::acl::Acl; +use window::{TextSource, TextWindow, WindowedWalk}; + +const SPARSE_RESIDUAL_RATIO: u64 = 8; pub trait EmailQuery: Sync + Send { fn email_query( &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WalkOrder { + Ascending, + Descending, +} + +enum Plan { + Walk(WalkOrder), + TopK(CacheSorter), + Sorted(Vec), +} + +enum Candidates { + All, + Positions(Vec), +} + +enum Pending { + Ready(Page), + Sort { + page: Page, + matches: RoaringBitmap, + comparators: Vec, + }, + Window { + page: Page, + candidates: Candidates, + order: WalkOrder, + text: TextWindow, + }, +} + +struct Page { + builder: QueryResponseBuilder, + threads: Option, + total: usize, +} + impl EmailQuery for Server { async fn email_query( &self, mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); - let mut filters = Vec::with_capacity(request.filter.len()); - let cached_messages = self + let object_metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Email); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; + let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - - for filter in std::mem::take(&mut request.filter) { - match filter { - Filter::Property(cond) => match cond { - EmailFilter::Text(text) => { - let (text, language) = - Language::detect(text, self.core.email.default_language); - - filters.push(SearchFilter::Or); - filters.push(SearchFilter::has_text( - EmailSearchField::From, - &text, - Language::None, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::To, - &text, - Language::None, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::Cc, - &text, - Language::None, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::Bcc, - &text, - Language::None, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::Subject, - &text, - language, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::Body, - &text, - language, - )); - filters.push(SearchFilter::has_text( - EmailSearchField::Attachment, - text, - language, - )); - filters.push(SearchFilter::End); - } - EmailFilter::From(text) => filters.push(SearchFilter::has_text( - EmailSearchField::From, - text, - Language::None, - )), - EmailFilter::To(text) => filters.push(SearchFilter::has_text( - EmailSearchField::To, - text, - Language::None, - )), - EmailFilter::Cc(text) => filters.push(SearchFilter::has_text( - EmailSearchField::Cc, - text, - Language::None, - )), - EmailFilter::Bcc(text) => filters.push(SearchFilter::has_text( - EmailSearchField::Bcc, - text, - Language::None, - )), - EmailFilter::Subject(text) => filters.push(SearchFilter::has_text_detect( - EmailSearchField::Subject, - text, - self.core.email.default_language, - )), - EmailFilter::Body(text) => filters.push(SearchFilter::has_text_detect( - EmailSearchField::Body, - text, - self.core.email.default_language, - )), - EmailFilter::Header(header) => { - let mut header = header.into_iter(); - let header_name = header.next().ok_or_else(|| { - trc::JmapEvent::InvalidArguments - .into_err() - .details("Header name is missing.".to_string()) - })?; - - if let Some(header_name) = HeaderName::parse(header_name) { - let value = header.next(); - let op = if matches!( - header_name, - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId - ) || value.is_none() - { - SearchOperator::Equal - } else { - SearchOperator::Contains - }; - - filters.push(SearchFilter::cond( - EmailSearchField::Headers, - op, - SearchValue::KeyValues(VecMap::with_capacity(1).with_append( - header_name.as_str().to_lowercase(), - value.unwrap_or_default(), - )), - )); - } - } - EmailFilter::InMailbox(mailbox) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .in_mailbox(mailbox.document_id()) - .map(|item| item.document_id), - ))) - } - EmailFilter::InMailboxOtherThan(mailboxes) => { - let mailboxes = mailboxes - .into_iter() - .map(|m| m.document_id()) - .collect::>(); - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cached_messages.emails.items.iter().filter_map(|item| { - if item - .mailboxes - .iter() - .any(|mb| !mailboxes.contains(&mb.mailbox_id)) - { - Some(item.document_id) - } else { - None - } - }), - ))); - } - EmailFilter::Before(date) => filters.push(SearchFilter::lt( - EmailSearchField::ReceivedAt, - date.timestamp(), - )), - EmailFilter::After(date) => filters.push(SearchFilter::gt( - EmailSearchField::ReceivedAt, - date.timestamp(), - )), - EmailFilter::MinSize(size) => { - filters.push(SearchFilter::ge(EmailSearchField::Size, size)) - } - EmailFilter::MaxSize(size) => { - filters.push(SearchFilter::lt(EmailSearchField::Size, size)) - } - EmailFilter::AllInThreadHaveKeyword(keyword) => filters.push( - SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, true)), - ), - EmailFilter::SomeInThreadHaveKeyword(keyword) => filters.push( - SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, false)), - ), - EmailFilter::NoneInThreadHaveKeyword(keyword) => { - filters.push(SearchFilter::Not); - filters.push(SearchFilter::is_in_set(thread_keywords( - &cached_messages, - keyword, - false, - ))); - filters.push(SearchFilter::End); - } - EmailFilter::HasKeyword(keyword) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .with_keyword(&keyword) - .map(|item| item.document_id), - ))); - } - EmailFilter::NotKeyword(keyword) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .without_keyword(&keyword) - .map(|item| item.document_id), - ))); - } - EmailFilter::HasAttachment(has_attach) => { - filters.push(SearchFilter::eq( - EmailSearchField::HasAttachment, - has_attach, - )); - } - - // Non-standard - EmailFilter::Id(ids) => { - let mut set = RoaringBitmap::new(); - for id in ids { - set.insert(id.document_id()); - } - filters.push(SearchFilter::is_in_set(set)); - } - EmailFilter::SentBefore(date) => { - filters.push(SearchFilter::lt(EmailSearchField::SentAt, date.timestamp())) - } - EmailFilter::SentAfter(date) => { - filters.push(SearchFilter::gt(EmailSearchField::SentAt, date.timestamp())) - } - EmailFilter::InThread(id) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .in_thread(id.document_id()) - .map(|item| item.document_id), - ))) - } - other => { - return Err(trc::JmapEvent::UnsupportedFilter - .into_err() - .details(other.to_string())); - } - }, - Filter::And => { - filters.push(SearchFilter::And); - } - Filter::Or => { - filters.push(SearchFilter::Or); - } - Filter::Not => { - filters.push(SearchFilter::Not); - } - Filter::Close => { - filters.push(SearchFilter::End); - } - } - } - - // Parse sort criteria - let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); - for comparator in request - .sort - .take() - .filter(|s| !s.is_empty()) - .unwrap_or_default() - { - comparators.push(match comparator.property { - EmailComparator::ReceivedAt => { - SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending) - } - EmailComparator::Size => { - SearchComparator::field(EmailSearchField::Size, comparator.is_ascending) - } - EmailComparator::From => { - SearchComparator::field(EmailSearchField::From, comparator.is_ascending) - } - EmailComparator::To => { - SearchComparator::field(EmailSearchField::To, comparator.is_ascending) - } - EmailComparator::Subject => { - SearchComparator::field(EmailSearchField::Subject, comparator.is_ascending) - } - EmailComparator::SentAt => { - SearchComparator::field(EmailSearchField::SentAt, comparator.is_ascending) - } - EmailComparator::HasKeyword(keyword) => SearchComparator::set( - RoaringBitmap::from_iter( - cached_messages - .with_keyword(&keyword) - .map(|item| item.document_id), - ), - comparator.is_ascending, - ), - EmailComparator::AllInThreadHaveKeyword(keyword) => SearchComparator::set( - thread_keywords(&cached_messages, keyword, true), - comparator.is_ascending, - ), - EmailComparator::SomeInThreadHaveKeyword(keyword) => SearchComparator::set( - thread_keywords(&cached_messages, keyword, false), - comparator.is_ascending, - ), - // Non-standard - EmailComparator::Cc => { - SearchComparator::field(EmailSearchField::Cc, comparator.is_ascending) - } - - other => { - return Err(trc::JmapEvent::UnsupportedSort - .into_err() - .details(other.to_string())); - } - }); - } - - let results = self - .search_store() - .query_account( - SearchQuery::new(SearchIndex::Email) - .with_filters(filters) - .with_comparators(comparators) - .with_account_id(account_id) - .with_mask(if access_token.is_shared(account_id) { - cached_messages.shared_messages(access_token, Acl::ReadItems) - } else { - cached_messages - .emails - .items - .iter() - .map(|item| item.document_id) - .collect() - }), - ) - .await?; - - let collapse_threads = request.arguments.collapse_threads.unwrap_or(false); - let total_results = if collapse_threads { - let mut seen_thread_ids = AHashSet::new(); - results - .iter() - .filter_map(|document_id| { - cached_messages - .email_by_id(document_id) - .map(|email| email.thread_id) - }) - .filter(|thread_id| seen_thread_ids.insert(*thread_id)) - .count() + let access = if access_token.is_shared(account_id) { + MessageAccess::Mailboxes(cache.shared_mailboxes(access_token, Acl::ReadItems)) } else { - results.len() + MessageAccess::All + }; + let metadata = + object_metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(EmailQueryFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; + let metadata_matches = object_metadata + .evaluate(self, account_id, &metadata, || { + cache + .with_metadata() + .filter(|message| access.allows(*message)) + .map(|message| message.document_id()) + .collect::() + }) + .await?; + let filter = QueryFilter::parse( + std::mem::take(&mut request.filter), + &cache, + access, + metadata_matches.into_iter(), + self.core.email.default_language, + )?; + let mut comparators = sort_comparators(request.sort.take())?; + let query_state = sampled.state(cache.get_state(false)); + let max_results = self.core.jmap.query_max_results; + + let Some(mut filter) = filter.filter(|query| !query.row.access.is_empty()) else { + return Page::new(&request, max_results, query_state).finish(); }; - let mut response = QueryResponseBuilder::new( - total_results, - self.core.jmap.query_max_results, - cached_messages.get_state(false), - &request, - ); - - if !results.is_empty() { - let mut seen_thread_ids = AHashSet::new(); - - for document_id in results { - let Some(thread_id) = cached_messages - .email_by_id(&document_id) - .map(|email| email.thread_id) - else { - continue; - }; - if collapse_threads && !seen_thread_ids.insert(thread_id) { - continue; - } - - if !response.add(thread_id, document_id) { - break; - } + restrict_thread_keywords(&mut comparators, &cache, &mut filter); + let windowed = if self.search_store().supports_windowed_walk() { + let resolve_rows = self + .search_store() + .account_query_rows(cache.emails.len() as u64); + WindowedWalk::of( + &request, + &comparators, + &mut filter, + account_id, + resolve_rows, + ) + } else { + None + }; + let residual_filters = filter.residual_filters(); + if !residual_filters.is_empty() { + let residual = self + .search_store() + .filter_account( + SearchQuery::new(SearchIndex::Email) + .with_filters(residual_filters) + .with_account_id(account_id) + .with_mask(filter.readable(&cache)), + ) + .await + .caused_by(trc::location!())?; + if residual.is_empty() { + return Page::new(&request, max_results, query_state).finish(); } + filter.row.residual = Some(residual); } - response.build() + let mut row = filter.into_row(); + match Pending::new( + &cache, + &mut row, + comparators, + request, + max_results, + query_state, + windowed, + ) { + Pending::Ready(page) => page.finish(), + Pending::Window { + mut page, + candidates, + order, + text, + } => { + page.walk_windowed( + TextSource::Store(self), + &cache, + &mut row, + &candidates, + order, + text, + ) + .await?; + page.finish() + } + Pending::Sort { + mut page, + matches, + comparators, + } => { + let results = self + .query_emails( + account_id, + &cache, + SearchQuery::new(SearchIndex::Email) + .with_account_id(account_id) + .with_mask(matches), + comparators, + ) + .await?; + page.feed(&cache, results); + page.finish() + } + } } } -fn thread_keywords(cache: &MessageStoreCache, keyword: Keyword, match_all: bool) -> RoaringBitmap { - let keyword_doc_ids = - RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|item| item.document_id)); - if keyword_doc_ids.is_empty() { - return keyword_doc_ids; - } - let mut not_matched_ids = RoaringBitmap::new(); - let mut matched_ids = RoaringBitmap::new(); - - let mut thread_map: AHashMap = AHashMap::new(); - - for item in &cache.emails.items { - thread_map - .entry(item.thread_id) - .or_default() - .insert(item.document_id); - } - - for item in &cache.emails.items { - let keyword_doc_id = item.document_id; - if !keyword_doc_ids.contains(keyword_doc_id) - || matched_ids.contains(keyword_doc_id) - || not_matched_ids.contains(keyword_doc_id) +impl Pending { + fn new( + cache: &MessageStoreCache, + filter: &mut RowFilter, + comparators: Vec, + mut request: QueryRequest, + max_results: usize, + query_state: State, + windowed: Option, + ) -> Self { + let candidates = Candidates::new(&cache.emails, filter); + if let Some(WindowedWalk { + order, + query, + resolve_rows, + }) = windowed { - continue; + let page = Page::new(&request, max_results, query_state); + let wanted = usize::try_from(request.position.unwrap_or(0)) + .unwrap_or(0) + .saturating_add(page.builder.limit); + if wanted == 0 { + return Pending::Ready(page); + } + let count = match (&candidates, &filter.residual) { + (Candidates::All, Some(residual)) => { + usize::try_from(residual.len()).unwrap_or(usize::MAX) + } + (candidates, _) => candidates.len(&cache.emails), + }; + return Pending::Window { + text: TextWindow::new(query, wanted, count, resolve_rows), + page, + candidates, + order, + }; + } + let plan = Plan::new(&request, cache, comparators); + if request.anchor.is_none() + && let Some(position) = request.position.filter(|position| *position < 0) + && !matches!(plan, Plan::Sorted(_)) + { + let total = candidates.count(&cache.emails, filter, collapse_threads(&request)); + let start = i64::try_from(total) + .unwrap_or(i64::MAX) + .saturating_add(i64::from(position)) + .max(0); + request.position = Some(i32::try_from(start).unwrap_or(i32::MAX)); } - if let Some(thread_doc_ids) = thread_map.get(&item.thread_id) { - let mut thread_tag_intersection = thread_doc_ids.clone(); - thread_tag_intersection &= &keyword_doc_ids; + let mut page = Page::new(&request, max_results, query_state); + match plan { + Plan::Walk(order) => { + page.walk(&cache.emails, filter, &candidates, order); + Pending::Ready(page) + } + Plan::TopK(sorter) => { + let start = usize::try_from(request.position.unwrap_or(0)).unwrap_or(0); + page.top(cache, filter, &candidates, &sorter, start); + Pending::Ready(page) + } + Plan::Sorted(comparators) => Pending::Sort { + page, + matches: candidates.matches(&cache.emails, filter), + comparators, + }, + } + } +} - if (match_all && &thread_tag_intersection == thread_doc_ids) - || (!match_all && !thread_tag_intersection.is_empty()) +fn restrict_thread_keywords( + comparators: &mut [MessageComparator], + cache: &MessageStoreCache, + filter: &mut QueryFilter, +) { + for comparator in comparators { + if let MessageComparator::Cache { + field: MessageCacheField::ThreadKeyword { keyword, match_all }, + ascending, + } = comparator + && let Some(visible) = filter.visible(cache) + { + let documents = thread_keywords(cache, keyword, *match_all, None, Some(visible)); + *comparator = MessageComparator::Search(SearchComparator::set(documents, *ascending)); + } + } +} + +fn collapse_threads(request: &QueryRequest) -> bool { + request.arguments.collapse_threads.unwrap_or(false) +} + +impl Plan { + fn walk_order(comparators: &[MessageComparator]) -> Option { + match comparators { + [] => Some(WalkOrder::Ascending), + [ + MessageComparator::Cache { + field: MessageCacheField::ReceivedAt, + ascending, + }, + ] => Some(if *ascending { + WalkOrder::Ascending + } else { + WalkOrder::Descending + }), + _ => None, + } + } + + fn new( + request: &QueryRequest, + cache: &MessageStoreCache, + comparators: Vec, + ) -> Self { + if let Some(order) = Plan::walk_order(&comparators) { + return Plan::Walk(order); + } + match comparators.as_slice() { + _ if !collapse_threads(request) && request.anchor.is_none() => { + match CacheSorter::new(cache, comparators) { + Ok(sorter) => Plan::TopK(sorter), + Err(comparators) => Plan::Sorted(comparators), + } + } + _ => Plan::Sorted(comparators), + } + } +} + +impl WindowedWalk { + fn of( + request: &QueryRequest, + comparators: &[MessageComparator], + filter: &mut QueryFilter, + account_id: u32, + resolve_rows: Option, + ) -> Option { + let order = Plan::walk_order(comparators)?; + if request.anchor.is_some() + || request.calculate_total.unwrap_or(false) + || request.position.is_some_and(|position| position < 0) + { + return None; + } + filter.take_text().map(|filters| WindowedWalk { + order, + query: SearchQuery::new(SearchIndex::Email) + .with_filters(filters) + .with_account_id(account_id), + resolve_rows, + }) + } +} + +impl Candidates { + fn new(cache: &MessagesCache, filter: &mut RowFilter) -> Self { + match filter.residual.take() { + Some(residual) + if residual.len().saturating_mul(SPARSE_RESIDUAL_RATIO) < cache.len() as u64 => { - matched_ids |= thread_doc_ids; - } else if !thread_tag_intersection.is_empty() { - not_matched_ids |= &thread_tag_intersection; + let mut positions = residual + .iter() + .filter_map(|document_id| cache.position(document_id)) + .collect::>(); + positions.sort_unstable(); + Candidates::Positions(positions) + } + residual => { + filter.residual = residual; + Candidates::All } } } - matched_ids + fn len(&self, cache: &MessagesCache) -> usize { + match self { + Candidates::All => cache.len(), + Candidates::Positions(positions) => positions.len(), + } + } + + fn count(&self, cache: &MessagesCache, filter: &RowFilter, collapse_threads: bool) -> usize { + match self { + Candidates::All => count_matches(cache.iter(), filter, collapse_threads), + Candidates::Positions(positions) => count_matches( + positions.iter().filter_map(|position| cache.at(*position)), + filter, + collapse_threads, + ), + } + } + + fn matches(&self, cache: &MessagesCache, filter: &RowFilter) -> RoaringBitmap { + match self { + Candidates::All => matching_ids(cache.iter(), filter), + Candidates::Positions(positions) => matching_ids( + positions.iter().filter_map(|position| cache.at(*position)), + filter, + ), + } + } +} + +fn count_matches<'x>( + messages: impl Iterator>, + filter: &RowFilter, + collapse_threads: bool, +) -> usize { + if collapse_threads { + let mut threads = RoaringBitmap::new(); + messages + .filter(|message| filter.matches(*message) && threads.insert(message.thread_id())) + .count() + } else { + messages.filter(|message| filter.matches(*message)).count() + } +} + +fn matching_ids<'x>( + messages: impl Iterator>, + filter: &RowFilter, +) -> RoaringBitmap { + messages + .filter(|message| filter.matches(*message)) + .map(|message| message.document_id()) + .collect() +} + +impl Page { + fn new(request: &QueryRequest, max_results: usize, query_state: State) -> Self { + Page { + builder: QueryResponseBuilder::new_streaming(max_results, query_state, request), + threads: collapse_threads(request).then(RoaringBitmap::new), + total: 0, + } + } + + fn emit(&mut self, message: MessageRef<'_>) -> bool { + if self + .threads + .as_mut() + .is_some_and(|threads| !threads.insert(message.thread_id())) + { + return true; + } + self.total += 1; + self.builder.add(message.thread_id(), message.document_id()) + } + + fn count(&mut self, message: MessageRef<'_>) { + if self + .threads + .as_mut() + .is_none_or(|threads| threads.insert(message.thread_id())) + { + self.total += 1; + } + } + + fn fill<'x>( + &mut self, + filter: &RowFilter, + messages: &mut impl Iterator>, + ) -> bool { + for message in messages { + if filter.matches(message) && !self.emit(message) { + return true; + } + } + false + } + + fn count_rest<'x>( + &mut self, + filter: &RowFilter, + messages: impl Iterator>, + ) { + for message in messages { + if filter.matches(message) { + self.count(message); + } + } + } + + fn walk( + &mut self, + cache: &MessagesCache, + filter: &RowFilter, + candidates: &Candidates, + order: WalkOrder, + ) { + let counts = self.builder.wants_total(); + match (candidates, order) { + (Candidates::All, WalkOrder::Ascending) => { + let mut messages = cache.iter(); + if self.fill(filter, &mut messages) && counts { + self.count_rest(filter, messages); + } + } + (Candidates::All, WalkOrder::Descending) => { + let mut messages = cache.iter().rev(); + if self.fill(filter, &mut messages) && counts { + self.count_rest(filter, messages); + } + } + (Candidates::Positions(positions), WalkOrder::Ascending) => { + let mut messages = positions.iter().filter_map(|position| cache.at(*position)); + if self.fill(filter, &mut messages) && counts { + self.count_rest(filter, messages); + } + } + (Candidates::Positions(positions), WalkOrder::Descending) => { + let mut messages = positions + .iter() + .rev() + .filter_map(|position| cache.at(*position)); + if self.fill(filter, &mut messages) && counts { + self.count_rest(filter, messages); + } + } + } + } + + fn top( + &mut self, + cache: &MessageStoreCache, + filter: &RowFilter, + candidates: &Candidates, + sorter: &CacheSorter, + start: usize, + ) { + let count = start.saturating_add(self.builder.limit); + let (messages, total) = match candidates { + Candidates::All => sorter.top( + cache, + cache + .emails + .iter() + .filter(|message| filter.matches(*message)), + count, + ), + Candidates::Positions(positions) => sorter.top( + cache, + positions + .iter() + .filter_map(|position| cache.emails.at(*position)) + .filter(|message| filter.matches(*message)), + count, + ), + }; + self.total = total; + for (document_id, thread_id) in messages { + if !self.builder.add(thread_id, document_id) { + break; + } + } + } + + fn feed(&mut self, cache: &MessageStoreCache, results: Vec) { + let counts = self.builder.wants_total(); + let mut results = results + .into_iter() + .filter_map(|document_id| cache.email_by_id(&document_id)); + for message in results.by_ref() { + if !self.emit(message) { + break; + } + } + if counts { + for message in results { + self.count(message); + } + } + } + + fn finish(self) -> trc::Result { + let total = self.builder.wants_total().then_some(self.total); + self.builder.finish(total) + } +} + +fn sort_comparators( + sort: Option>>, +) -> trc::Result> { + let sort = sort.filter(|sort| !sort.is_empty()).unwrap_or_default(); + let mut comparators = Vec::with_capacity(sort.len()); + for comparator in sort { + let ascending = comparator.is_ascending; + comparators.push(match comparator.property { + EmailComparator::ReceivedAt => MessageComparator::Cache { + field: MessageCacheField::ReceivedAt, + ascending, + }, + EmailComparator::Size => MessageComparator::Cache { + field: MessageCacheField::Size, + ascending, + }, + EmailComparator::From => MessageComparator::SortKey { + field: MessageSortField::From, + ascending, + }, + EmailComparator::To => MessageComparator::SortKey { + field: MessageSortField::To, + ascending, + }, + EmailComparator::Subject => MessageComparator::SortKey { + field: MessageSortField::Subject, + ascending, + }, + EmailComparator::SentAt => MessageComparator::Cache { + field: MessageCacheField::SentAt, + ascending, + }, + EmailComparator::HasKeyword(keyword) => MessageComparator::Cache { + field: MessageCacheField::Keyword(keyword), + ascending, + }, + EmailComparator::AllInThreadHaveKeyword(keyword) => MessageComparator::Cache { + field: MessageCacheField::ThreadKeyword { + keyword, + match_all: true, + }, + ascending, + }, + EmailComparator::SomeInThreadHaveKeyword(keyword) => MessageComparator::Cache { + field: MessageCacheField::ThreadKeyword { + keyword, + match_all: false, + }, + ascending, + }, + other => { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(other.to_compact_string())); + } + }); + } + Ok(comparators) } diff --git a/crates/jmap/src/email/query/filter.rs b/crates/jmap/src/email/query/filter.rs new file mode 100644 index 00000000..9581e03f --- /dev/null +++ b/crates/jmap/src/email/query/filter.rs @@ -0,0 +1,707 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::text::{push_text, strip_subject_prefix}; +use common::{MessageStoreCache, cache::email::MessageRef}; +use compact_str::ToCompactString; +use email::cache::email::{MessageAccess, MessageCacheAccess, SearchOperator, thread_keywords}; +use jmap_proto::{ + method::query::Filter, + object::email::{EmailFilter, EmailQueryFilter}, +}; +use mail_parser::HeaderName; +use nlp::language::Language; +use std::ops::RangeInclusive; +use store::{ + ahash::AHashSet, + roaring::RoaringBitmap, + search::{EmailSearchField, KeyValueMatch, SearchFilter}, +}; +use types::keyword::{HASATTACHMENT, HASNOATTACHMENT, Keyword}; + +pub(super) struct RowFilter { + pub access: MessageAccess, + mailboxes: Mailboxes, + not_in_mailboxes: Option>, + thread_id: Option, + all_keywords: u32, + no_keywords: u32, + received: Option>, + sent: Option>, + size: Option>, + pub residual: Option, +} + +enum Mailboxes { + Any, + One(u32), + All(Vec), +} + +pub(super) struct QueryFilter { + pub row: RowFilter, + pub residual: Vec, + pub text: Vec, + visible: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Absorbed { + Yes, + Contradiction, +} + +impl RowFilter { + pub fn new(access: MessageAccess) -> Self { + RowFilter { + access, + mailboxes: Mailboxes::Any, + not_in_mailboxes: None, + thread_id: None, + all_keywords: 0, + no_keywords: 0, + received: None, + sent: None, + size: None, + residual: None, + } + } + + #[inline(always)] + pub fn matches(&self, message: MessageRef<'_>) -> bool { + (match &self.mailboxes { + Mailboxes::Any => true, + Mailboxes::One(mailbox_id) => message.has_mailbox_id(*mailbox_id), + Mailboxes::All(mailbox_ids) => mailbox_ids + .iter() + .all(|mailbox_id| message.has_mailbox_id(*mailbox_id)), + }) && (self.all_keywords | self.no_keywords == 0 || { + let keywords = message.keywords(); + keywords & self.all_keywords == self.all_keywords && keywords & self.no_keywords == 0 + }) && self + .thread_id + .is_none_or(|thread_id| message.thread_id() == thread_id) + && self + .received + .as_ref() + .is_none_or(|range| range.contains(&(message.received_at() as i64))) + && self + .sent + .as_ref() + .is_none_or(|range| range.contains(&message.sent_at_absolute())) + && self + .size + .as_ref() + .is_none_or(|range| range.contains(&i64::from(message.size()))) + && self.not_in_mailboxes.as_deref().is_none_or(|mailboxes| { + message + .mailboxes() + .iter() + .any(|uid| mailboxes.binary_search(&uid.mailbox_id).is_err()) + }) + && self + .residual + .as_ref() + .is_none_or(|documents| documents.contains(message.document_id())) + && self.access.allows(message) + } + + fn absorb(&mut self, condition: EmailFilter) -> Result { + match condition { + EmailFilter::InMailbox(mailbox_id) => { + let mailbox_id = mailbox_id.document_id(); + match &mut self.mailboxes { + Mailboxes::Any => self.mailboxes = Mailboxes::One(mailbox_id), + Mailboxes::One(current) if *current == mailbox_id => (), + Mailboxes::One(current) => { + self.mailboxes = Mailboxes::All(vec![*current, mailbox_id]); + } + Mailboxes::All(current) => current.push(mailbox_id), + } + Ok(Absorbed::Yes) + } + EmailFilter::InThread(thread_id) => { + Ok(narrow_id(&mut self.thread_id, thread_id.document_id())) + } + EmailFilter::InMailboxOtherThan(mailboxes) if self.not_in_mailboxes.is_none() => { + let mut mailboxes = mailboxes + .iter() + .map(|mailbox_id| mailbox_id.document_id()) + .collect::>(); + mailboxes.sort_unstable(); + self.not_in_mailboxes = Some(mailboxes); + Ok(Absorbed::Yes) + } + EmailFilter::HasKeyword(keyword) => match keyword.id() { + Ok(id) => { + self.all_keywords |= 1 << id; + Ok(Absorbed::Yes) + } + Err(_) => Err(EmailFilter::HasKeyword(keyword)), + }, + EmailFilter::NotKeyword(keyword) => match keyword.id() { + Ok(id) => { + self.no_keywords |= 1 << id; + Ok(Absorbed::Yes) + } + Err(_) => Err(EmailFilter::NotKeyword(keyword)), + }, + EmailFilter::HasAttachment(has_attachment) => { + self.all_keywords |= if has_attachment { + 1 << HASATTACHMENT + } else { + 1 << HASNOATTACHMENT + }; + Ok(Absorbed::Yes) + } + EmailFilter::Before(date) => narrow_range( + &mut self.received, + i64::MIN, + date.timestamp().saturating_sub(1), + ), + EmailFilter::After(date) => { + narrow_range(&mut self.received, date.timestamp(), i64::MAX) + } + EmailFilter::SentBefore(date) => { + narrow_range(&mut self.sent, i64::MIN, date.timestamp().saturating_sub(1)) + } + EmailFilter::SentAfter(date) => { + narrow_range(&mut self.sent, date.timestamp().saturating_add(1), i64::MAX) + } + EmailFilter::MinSize(size) => narrow_range(&mut self.size, i64::from(size), i64::MAX), + EmailFilter::MaxSize(size) => { + narrow_range(&mut self.size, i64::MIN, i64::from(size) - 1) + } + condition => Err(condition), + } + } +} + +fn narrow_id(current: &mut Option, id: u32) -> Absorbed { + match current { + Some(current) if *current != id => Absorbed::Contradiction, + _ => { + *current = Some(id); + Absorbed::Yes + } + } +} + +fn narrow_range( + current: &mut Option>, + from: i64, + to: i64, +) -> Result { + let range = match current.take() { + Some(range) => (*range.start()).max(from)..=(*range.end()).min(to), + None => from..=to, + }; + let absorbed = if range.is_empty() { + Absorbed::Contradiction + } else { + Absorbed::Yes + }; + *current = Some(range); + Ok(absorbed) +} + +impl QueryFilter { + pub fn parse( + filters: Vec>, + cache: &MessageStoreCache, + access: MessageAccess, + mut metadata_matches: impl Iterator, + default_language: Language, + ) -> trc::Result> { + let mut row = RowFilter::new(access); + let mut residual = Vec::new(); + let mut text = Vec::new(); + let mut subtrees = 0usize; + let mut is_empty = false; + let mut visible = None; + let verbatim = has_empty_group(&filters); + + for filter in filters { + let top_level = !verbatim && subtrees == 0; + match filter { + Filter::Property(EmailQueryFilter::Metadata(_)) => residual.push( + SearchFilter::is_in_set(metadata_matches.next().unwrap_or_default()), + ), + Filter::Property(EmailQueryFilter::Email(condition)) if top_level => { + match row.absorb(condition) { + Ok(Absorbed::Yes) => (), + Ok(Absorbed::Contradiction) => is_empty = true, + Err(condition) if is_text(&condition) => push_condition( + &mut text, + condition, + cache, + &row.access, + &mut visible, + default_language, + )?, + Err(condition) => push_condition( + &mut residual, + condition, + cache, + &row.access, + &mut visible, + default_language, + )?, + } + } + Filter::Property(EmailQueryFilter::Email(condition)) => push_condition( + &mut residual, + condition, + cache, + &row.access, + &mut visible, + default_language, + )?, + Filter::And if top_level => (), + Filter::And => { + subtrees += 1; + residual.push(SearchFilter::And); + } + Filter::Or => { + subtrees += 1; + residual.push(SearchFilter::Or); + } + Filter::Not => { + subtrees += 1; + residual.push(SearchFilter::Not); + } + Filter::Close if subtrees > 0 || verbatim => { + subtrees = subtrees.saturating_sub(1); + residual.push(SearchFilter::End); + } + Filter::Close => (), + } + } + + if let Mailboxes::All(mailbox_ids) = &mut row.mailboxes { + mailbox_ids.sort_unstable(); + mailbox_ids.dedup(); + } + + Ok((!is_empty).then_some(QueryFilter { + row, + residual, + text, + visible, + })) + } + + pub fn visible(&mut self, cache: &MessageStoreCache) -> Option<&RoaringBitmap> { + visible_ids(cache, &self.row.access, &mut self.visible) + } + + pub fn readable(&mut self, cache: &MessageStoreCache) -> RoaringBitmap { + match &self.row.access { + MessageAccess::All => cache.email_document_ids(), + access => self + .visible + .take() + .unwrap_or_else(|| readable_ids(cache, access)), + } + } + + pub fn into_row(self) -> RowFilter { + let QueryFilter { + mut row, visible, .. + } = self; + if matches!(row.access, MessageAccess::Mailboxes(_)) { + row.residual = row.residual.or(visible); + if row.residual.is_some() { + row.access = MessageAccess::All; + } + } + row + } + + pub fn residual_filters(&mut self) -> Vec { + let mut filters = std::mem::take(&mut self.residual); + filters.append(&mut self.text); + filters + } + + pub fn take_text(&mut self) -> Option> { + if self.text.is_empty() { + return None; + } + let mut external = Vec::new(); + let mut depth = 0usize; + let mut start = 0; + let mut locality = Locality::None; + for (end, filter) in self.residual.iter().enumerate() { + locality = locality.with(filter); + match filter { + SearchFilter::And | SearchFilter::Or | SearchFilter::Not => depth += 1, + SearchFilter::End => depth = depth.saturating_sub(1), + _ => (), + } + if depth == 0 { + match locality { + Locality::Mixed => return None, + Locality::External => external.push(start..end + 1), + Locality::None | Locality::Local => (), + } + start = end + 1; + locality = Locality::None; + } + } + + let mut text = std::mem::take(&mut self.text); + if !external.is_empty() { + let mut external = external.into_iter().peekable(); + let mut local = Vec::with_capacity(self.residual.len()); + for (pos, filter) in std::mem::take(&mut self.residual).into_iter().enumerate() { + while external.next_if(|range| range.end <= pos).is_some() {} + if external.peek().is_some_and(|range| range.contains(&pos)) { + text.push(filter); + } else { + local.push(filter); + } + } + self.residual = local; + } + Some(text) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Locality { + None, + Local, + External, + Mixed, +} + +impl Locality { + fn with(self, filter: &SearchFilter) -> Self { + let other = match filter { + SearchFilter::DocumentSet(_) => Locality::Local, + SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } + | SearchFilter::Integer { .. } => Locality::External, + SearchFilter::And | SearchFilter::Or | SearchFilter::Not | SearchFilter::End => { + return self; + } + }; + match self { + Locality::None => other, + current if current == other => current, + _ => Locality::Mixed, + } + } +} + +fn readable_ids(cache: &MessageStoreCache, access: &MessageAccess) -> RoaringBitmap { + cache + .emails + .iter() + .filter(|message| access.allows(*message)) + .map(|message| message.document_id()) + .collect() +} + +fn visible_ids<'x>( + cache: &MessageStoreCache, + access: &MessageAccess, + visible: &'x mut Option, +) -> Option<&'x RoaringBitmap> { + match access { + MessageAccess::All => None, + access => Some(visible.get_or_insert_with(|| readable_ids(cache, access))), + } +} + +fn has_empty_group(filters: &[Filter]) -> bool { + let mut leaves: Vec = Vec::new(); + for filter in filters { + match filter { + Filter::And | Filter::Or | Filter::Not => leaves.push(0), + Filter::Close => match leaves.pop() { + Some(0) => return true, + Some(count) => { + if let Some(parent) = leaves.last_mut() { + *parent += count; + } + } + None => (), + }, + Filter::Property(EmailQueryFilter::Email(EmailFilter::Header(header))) + if header + .first() + .is_some_and(|name| HeaderName::parse(name.as_str()).is_none()) => {} + Filter::Property(_) => { + if let Some(count) = leaves.last_mut() { + *count += 1; + } + } + } + } + leaves.contains(&0) +} + +fn is_text(condition: &EmailFilter) -> bool { + matches!( + condition, + EmailFilter::Text(_) + | EmailFilter::From(_) + | EmailFilter::To(_) + | EmailFilter::Cc(_) + | EmailFilter::Bcc(_) + | EmailFilter::Subject(_) + | EmailFilter::Body(_) + | EmailFilter::Header(_) + ) +} + +pub(super) fn push_condition( + filters: &mut Vec, + condition: EmailFilter, + cache: &MessageStoreCache, + access: &MessageAccess, + visible: &mut Option, + default_language: Language, +) -> trc::Result<()> { + match condition { + EmailFilter::Text(text) => { + let (text, language) = Language::detect(text, default_language); + push_text(filters, &strip_subject_prefix(&text), language); + } + EmailFilter::From(text) => filters.push(SearchFilter::has_text( + EmailSearchField::From, + text, + Language::None, + )), + EmailFilter::To(text) => filters.push(SearchFilter::has_text( + EmailSearchField::To, + text, + Language::None, + )), + EmailFilter::Cc(text) => filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + text, + Language::None, + )), + EmailFilter::Bcc(text) => filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + text, + Language::None, + )), + EmailFilter::Subject(text) => filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + strip_subject_prefix(&text), + default_language, + )), + EmailFilter::Body(text) => filters.push(SearchFilter::has_text_detect( + EmailSearchField::Body, + text, + default_language, + )), + EmailFilter::Header(header) => { + let mut header = header.into_iter(); + let header_name = header.next().ok_or_else(|| { + trc::JmapEvent::InvalidArguments + .into_err() + .details("Header name is missing.") + })?; + + match (HeaderName::parse(header_name), header.next()) { + (Some(HeaderName::Subject), Some(value)) => { + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + strip_subject_prefix(&value), + default_language, + )) + } + (Some(HeaderName::From), Some(value)) => filters.push(SearchFilter::has_text( + EmailSearchField::From, + value, + Language::None, + )), + (Some(HeaderName::To), Some(value)) => filters.push(SearchFilter::has_text( + EmailSearchField::To, + value, + Language::None, + )), + (Some(HeaderName::Cc), Some(value)) => filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + value, + Language::None, + )), + (Some(HeaderName::Bcc), Some(value)) => filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + value, + Language::None, + )), + (Some(header_name), value) => { + let op = match value { + Some(value) + if matches!( + header_name, + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId + ) => + { + KeyValueMatch::Equals(value) + } + Some(value) => KeyValueMatch::Contains(value), + None => KeyValueMatch::Exists, + }; + filters.push(SearchFilter::KeyValue { + field: EmailSearchField::Headers.into(), + key: header_name.as_str().to_lowercase(), + op, + }); + } + (None, _) => {} + } + } + EmailFilter::InMailbox(mailbox) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .in_mailbox(mailbox.document_id()) + .map(|item| item.document_id()), + ))) + } + EmailFilter::InMailboxOtherThan(mailboxes) => { + let mailboxes = mailboxes + .into_iter() + .map(|m| m.document_id()) + .collect::>(); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.emails.iter().filter_map(|item| { + if item + .mailboxes() + .iter() + .any(|mb| !mailboxes.contains(&mb.mailbox_id)) + { + Some(item.document_id()) + } else { + None + } + }), + ))); + } + EmailFilter::Before(date) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received(date.timestamp(), SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::After(date) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .received(date.timestamp(), SearchOperator::GreaterEqualThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::MinSize(size) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .size(size, SearchOperator::GreaterEqualThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::MaxSize(size) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .size(size, SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::AllInThreadHaveKeyword(keyword) => { + filters.push(SearchFilter::is_in_set(thread_keywords( + cache, + &keyword, + true, + None, + visible_ids(cache, access, visible), + ))) + } + EmailFilter::SomeInThreadHaveKeyword(keyword) => { + filters.push(SearchFilter::is_in_set(thread_keywords( + cache, + &keyword, + false, + None, + visible_ids(cache, access, visible), + ))) + } + EmailFilter::NoneInThreadHaveKeyword(keyword) => { + filters.push(SearchFilter::Not); + filters.push(SearchFilter::is_in_set(thread_keywords( + cache, + &keyword, + false, + None, + visible_ids(cache, access, visible), + ))); + filters.push(SearchFilter::End); + } + EmailFilter::HasKeyword(keyword) => filters.push(SearchFilter::is_in_set( + RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|item| item.document_id())), + )), + EmailFilter::NotKeyword(keyword) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&keyword) + .map(|item| item.document_id()), + ))) + } + EmailFilter::HasAttachment(has_attach) => { + let keyword = if has_attach { + Keyword::HasAttachment + } else { + Keyword::HasNoAttachment + }; + + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&keyword).map(|item| item.document_id()), + ))); + } + EmailFilter::Id(ids) => { + let mut set = RoaringBitmap::new(); + for id in ids { + set.insert(id.document_id()); + } + filters.push(SearchFilter::is_in_set(set)); + } + EmailFilter::SentBefore(date) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .sent(date.timestamp(), SearchOperator::LowerThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::SentAfter(date) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .sent(date.timestamp(), SearchOperator::GreaterThan) + .map(|m| m.document_id()), + ))) + } + EmailFilter::InThread(id) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .in_thread(id.document_id()) + .map(|item| item.document_id()), + ))) + } + other => { + return Err(trc::JmapEvent::UnsupportedFilter + .into_err() + .details(other.to_compact_string())); + } + } + + Ok(()) +} diff --git a/crates/jmap/src/email/query/text.rs b/crates/jmap/src/email/query/text.rs new file mode 100644 index 00000000..9cc42544 --- /dev/null +++ b/crates/jmap/src/email/query/text.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use email::message::index::subject::{search_subject, search_subject_query}; +use nlp::language::Language; +use std::borrow::Cow; +use store::search::{EmailSearchField, SearchFilter}; + +const MAX_TEXT_TOKENS: usize = 8; + +pub(super) fn strip_subject_prefix(text: &str) -> Cow<'_, str> { + let trimmed = text.trim(); + let quoted = trimmed + .chars() + .next() + .filter(|quote| matches!(quote, '"' | '\'')) + .and_then(|quote| { + trimmed + .strip_prefix(quote)? + .strip_suffix(quote) + .map(|inner| (quote, inner)) + }); + match quoted { + Some((quote, inner)) => match search_subject(inner) { + Cow::Borrowed(name) if name.is_empty() || name.len() == inner.len() => { + Cow::Borrowed(text) + } + name => Cow::Owned(format!("{quote}{name}{quote}")), + }, + None => search_subject_query(trimmed), + } +} + +pub(super) fn push_text(filters: &mut Vec, text: &str, language: Language) { + let mut tokens = TextTokens { rest: text }; + let Some(first) = tokens.next() else { + push_text_fields(filters, text, language); + return; + }; + if tokens.remainder().is_empty() { + push_text_fields(filters, first, language); + return; + } + + filters.push(SearchFilter::And); + push_text_fields(filters, first, language); + for token in tokens.take(MAX_TEXT_TOKENS - 1) { + push_text_fields(filters, token, language); + } + filters.push(SearchFilter::End); +} + +fn push_text_fields(filters: &mut Vec, text: &str, language: Language) { + filters.extend([ + SearchFilter::Or, + SearchFilter::has_text(EmailSearchField::From, text, Language::None), + SearchFilter::has_text(EmailSearchField::To, text, Language::None), + SearchFilter::has_text(EmailSearchField::Cc, text, Language::None), + SearchFilter::has_text(EmailSearchField::Bcc, text, Language::None), + SearchFilter::has_text(EmailSearchField::Subject, text, language), + SearchFilter::has_text(EmailSearchField::Body, text, language), + SearchFilter::has_text(EmailSearchField::Attachment, text, language), + SearchFilter::End, + ]); +} + +struct TextTokens<'x> { + rest: &'x str, +} + +impl<'x> TextTokens<'x> { + fn remainder(&self) -> &'x str { + self.rest.trim() + } +} + +impl<'x> Iterator for TextTokens<'x> { + type Item = &'x str; + + fn next(&mut self) -> Option<&'x str> { + let text = self.rest.trim_start(); + let quote = text.chars().next()?; + let (token, rest) = matches!(quote, '"' | '\'') + .then(|| split_phrase(text, quote)) + .flatten() + .unwrap_or_else(|| text.split_once(char::is_whitespace).unwrap_or((text, ""))); + self.rest = rest; + Some(token) + } +} + +fn split_phrase(text: &str, quote: char) -> Option<(&str, &str)> { + let body = text.strip_prefix(quote)?; + body.match_indices(quote).find_map(|(pos, _)| { + let (phrase, rest) = text.split_at_checked(pos + 2)?; + rest.chars() + .next() + .is_none_or(char::is_whitespace) + .then_some((phrase, rest)) + }) +} + diff --git a/crates/jmap/src/email/query/window.rs b/crates/jmap/src/email/query/window.rs new file mode 100644 index 00000000..4d835f96 --- /dev/null +++ b/crates/jmap/src/email/query/window.rs @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Candidates, Page, SPARSE_RESIDUAL_RATIO, WalkOrder, filter::RowFilter}; +use common::{MessageStoreCache, MessagesCache, Server, cache::email::MessageRef}; +use email::cache::email::MessageCacheAccess; +use store::{dispatch::search::MAX_FILTER_CANDIDATES, roaring::RoaringBitmap, search::SearchQuery}; +use trc::AddContext; + +const MIN_FIRST_WINDOW: usize = 32; +const MAX_FIRST_WINDOW: usize = 128; +const WINDOW_GROWTH: usize = 4; +const MAX_WINDOWS: u32 = 6; +const MAX_MISSED_WINDOWS: u32 = 2; +const PRIOR_HITS: f64 = 0.5; +const RESOLVED_ROW_COST: f64 = 0.35; +const NEEDED_ROWS_MARGIN: f64 = 1.25; + +const _: () = + assert!(MAX_FIRST_WINDOW * WINDOW_GROWTH.pow(MAX_WINDOWS - 3) < MAX_FILTER_CANDIDATES); + +pub(super) struct WindowedWalk { + pub order: WalkOrder, + pub query: SearchQuery, + pub resolve_rows: Option, +} + +pub(super) enum TextSource<'x> { + Store(&'x Server), + #[cfg(all(test, stalwart_tests))] + Fake(&'x super::tests::FakeText), +} + +pub(super) struct TextWindow { + query: SearchQuery, + ids: Vec, + window: usize, + growth: usize, + wanted: u64, + candidates: u64, + resolve_rows: f64, + examined: u64, + matched: u64, + last_examined: u64, + last_matched: u64, + windows: u32, + missed: u32, + #[cfg(all(test, stalwart_tests))] + resolve_after: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum WindowStep { + Next, + Done, + Resolve, +} + +impl TextSource<'_> { + async fn filter_candidates( + &self, + query: &SearchQuery, + candidates: &[u32], + ) -> trc::Result { + match self { + TextSource::Store(server) => server + .search_store() + .filter_candidates(query, candidates) + .await + .caused_by(trc::location!()), + #[cfg(all(test, stalwart_tests))] + TextSource::Fake(text) => Ok(text.filter_candidates(query, candidates)), + } + } + + async fn filter_account(&self, query: SearchQuery) -> trc::Result { + match self { + TextSource::Store(server) => server + .search_store() + .filter_account(query) + .await + .caused_by(trc::location!()), + #[cfg(all(test, stalwart_tests))] + TextSource::Fake(text) => Ok(text.filter_account(&query)), + } + } +} + +impl TextWindow { + pub fn new( + query: SearchQuery, + wanted: usize, + candidates: usize, + resolve_rows: Option, + ) -> Self { + let wanted = wanted.max(1); + TextWindow { + query, + ids: Vec::new(), + window: wanted + .saturating_mul(2) + .clamp(MIN_FIRST_WINDOW, MAX_FIRST_WINDOW), + growth: WINDOW_GROWTH, + wanted: wanted as u64, + candidates: candidates as u64, + resolve_rows: resolve_rows.unwrap_or(f64::INFINITY), + examined: 0, + matched: 0, + last_examined: 0, + last_matched: 0, + windows: 0, + missed: 0, + #[cfg(all(test, stalwart_tests))] + resolve_after: None, + } + } + + #[cfg(all(test, stalwart_tests))] + pub fn reschedule(&mut self, first_window: usize, growth: usize, resolve_after: u32) { + self.window = first_window.max(1); + self.growth = growth.max(1); + self.resolve_after = Some(resolve_after); + } + + pub fn next_window(&self) -> usize { + self.window.min(MAX_FILTER_CANDIDATES) + } + + pub fn record(&mut self, examined: usize, matched: u64) { + self.last_examined = examined as u64; + self.last_matched = matched; + self.examined += self.last_examined; + self.matched += matched; + self.windows += 1; + self.missed = if matched == 0 { self.missed + 1 } else { 0 }; + } + + pub fn plan(&mut self, emitted: usize) -> WindowStep { + if emitted as u64 >= self.wanted { + return WindowStep::Done; + } + #[cfg(all(test, stalwart_tests))] + if let Some(windows) = self.resolve_after { + self.window = self.window.saturating_mul(self.growth); + return if self.windows >= windows { + WindowStep::Resolve + } else { + WindowStep::Next + }; + } + if self.windows > MAX_WINDOWS || self.missed >= MAX_MISSED_WINDOWS { + return WindowStep::Resolve; + } + let rate = (self.last_matched as f64 + PRIOR_HITS) / self.last_examined.max(1) as f64; + let left = self.candidates.saturating_sub(self.examined) as f64; + let remaining = self.wanted.saturating_sub(emitted as u64).max(1) as f64; + let window_rows = (remaining / rate).min(left); + let resolve_rows = + (RESOLVED_ROW_COST * (self.matched as f64 + rate * left)).min(self.resolve_rows); + if window_rows > resolve_rows || window_rows > self.window_capacity() { + return WindowStep::Resolve; + } + let grown = self.window.saturating_mul(self.growth); + self.window = if self.last_matched > 0 { + let needed = (window_rows * NEEDED_ROWS_MARGIN).ceil() as usize; + grown.min(needed.max(MIN_FIRST_WINDOW)) + } else { + grown + }; + WindowStep::Next + } + + fn window_capacity(&self) -> f64 { + let mut window = self.window; + let mut rows = 0usize; + for _ in self.windows..=MAX_WINDOWS { + window = window + .saturating_mul(self.growth) + .min(MAX_FILTER_CANDIDATES); + rows = rows.saturating_add(window); + } + rows as f64 + } + + fn into_resolved(self, mask: RoaringBitmap) -> SearchQuery { + self.query.with_mask(mask) + } +} + +impl Page { + fn fill_window<'x>( + &self, + filter: &RowFilter, + messages: &mut impl Iterator>, + text: &mut TextWindow, + window: &mut Vec>, + ) -> bool { + window.clear(); + window.extend( + messages + .by_ref() + .filter(|message| { + filter.matches(*message) + && self + .threads + .as_ref() + .is_none_or(|threads| !threads.contains(message.thread_id())) + }) + .take(text.next_window()), + ); + text.ids.clear(); + text.ids + .extend(window.iter().map(|message| message.document_id())); + !window.is_empty() + } + + fn emit_window( + &mut self, + window: &[MessageRef<'_>], + hits: &RoaringBitmap, + text: &mut TextWindow, + ) -> WindowStep { + let requested = text.next_window(); + text.record(window.len(), hits.len()); + + for message in window { + if hits.contains(message.document_id()) && !self.emit(*message) { + return WindowStep::Done; + } + } + + if window.len() < requested { + WindowStep::Done + } else { + text.plan(self.total) + } + } + + pub(super) async fn walk_windowed( + &mut self, + source: TextSource<'_>, + cache: &MessageStoreCache, + filter: &mut RowFilter, + candidates: &Candidates, + order: WalkOrder, + text: TextWindow, + ) -> trc::Result<()> { + let messages = &cache.emails; + match (candidates, order) { + (Candidates::All, WalkOrder::Ascending) => { + self.fill_windowed( + &source, + cache, + filter, + candidates, + &mut messages.iter(), + order, + text, + ) + .await + } + (Candidates::All, WalkOrder::Descending) => { + self.fill_windowed( + &source, + cache, + filter, + candidates, + &mut messages.iter().rev(), + order, + text, + ) + .await + } + (Candidates::Positions(positions), WalkOrder::Ascending) => { + self.fill_windowed( + &source, + cache, + filter, + candidates, + &mut positions + .iter() + .filter_map(|position| messages.at(*position)), + order, + text, + ) + .await + } + (Candidates::Positions(positions), WalkOrder::Descending) => { + self.fill_windowed( + &source, + cache, + filter, + candidates, + &mut positions + .iter() + .rev() + .filter_map(|position| messages.at(*position)), + order, + text, + ) + .await + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn fill_windowed<'x>( + &mut self, + source: &TextSource<'_>, + cache: &'x MessageStoreCache, + filter: &mut RowFilter, + candidates: &Candidates, + messages: &mut (impl Iterator> + Send), + order: WalkOrder, + mut text: TextWindow, + ) -> trc::Result<()> { + let mut window = Vec::with_capacity(text.next_window()); + while self.fill_window(filter, messages, &mut text, &mut window) { + let hits = source.filter_candidates(&text.query, &text.ids).await?; + match self.emit_window(&window, &hits, &mut text) { + WindowStep::Next => (), + WindowStep::Done => return Ok(()), + WindowStep::Resolve => { + let last = window.last().map(|message| message.position()); + let resolved = source + .filter_account(text.into_resolved(cache.email_document_ids())) + .await?; + let matches = match filter.residual.take() { + Some(residual) => residual & resolved, + None => resolved, + }; + if let Some(last) = last.filter(|_| !matches.is_empty()) { + let cache = &cache.emails; + match remaining_positions(cache, candidates, &matches, order, last) { + Some(positions) => self.fill( + filter, + &mut positions.iter().filter_map(|position| cache.at(*position)), + ), + None => { + filter.residual = Some(matches); + self.fill(filter, messages) + } + }; + } + return Ok(()); + } + } + } + Ok(()) + } +} + +fn remaining_positions( + cache: &MessagesCache, + candidates: &Candidates, + matches: &RoaringBitmap, + order: WalkOrder, + last: u32, +) -> Option> { + if matches.len().saturating_mul(SPARSE_RESIDUAL_RATIO) >= candidates.len(cache) as u64 { + return None; + } + let mut positions = matches + .iter() + .filter_map(|document_id| cache.position(document_id)) + .filter(|position| { + (match order { + WalkOrder::Ascending => *position > last, + WalkOrder::Descending => *position < last, + }) && match candidates { + Candidates::All => true, + Candidates::Positions(positions) => positions.binary_search(position).is_ok(), + } + }) + .collect::>(); + match order { + WalkOrder::Ascending => positions.sort_unstable(), + WalkOrder::Descending => positions.sort_unstable_by(|a, b| b.cmp(a)), + } + Some(positions) +} diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 0752a5a4..d0b3e01a 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -5,30 +5,47 @@ */ use crate::{ - blob::download::BlobDownload, + api::metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, NewMetadata, + ObjectMetadata, + }, + blob::download::{BlobAccess, BlobDownload}, changes::state::JmapCacheState, - email::{PatchResult, handle_email_patch, ingested_into_object}, + email::{ + PatchResult, + grouped::{GroupedIngest, QueuedEmail}, + handle_email_patch, ingested_into_object, parse_keywords, + }, }; use common::{ - Server, auth::AccessToken, ipc::PushNotification, storage::index::ObjectIndexBuilder, + MAX_RECEIVED_AT, MessageUid, Server, auth::AccessToken, ipc::PushNotification, + network::memory::MemoryReservation, storage::index::ObjectIndexBuilder, }; use email::message::headers::{BuildHeader, ValueToHeader}; use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, - mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, + cache::{ + MessageCacheFetch, + email::{MessageAccess, MessageCacheAccess}, + mailbox::MailboxCacheAccess, + }, + mailbox::{JUNK_ID, TRASH_ID}, message::{ delete::EmailDeletion, - ingest::{EmailIngest, IngestEmail, IngestSource}, - metadata::MessageData, + ingest::{ + EmailIngest, IngestEmail, IngestSource, SpamSamples, group::ThreadKey, + recheck::ThreadRecheck, + }, + messagedata::{KeywordDiff, PendingMessageData, merge_keywords}, }, + presence::update_email_presence, }; use http_proto::HttpSessionData; use jmap_proto::{ - error::set::{SetError, SetErrorType}, + error::set::{InvalidProperty, SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::email::{Email, EmailProperty, EmailValue}, references::resolve::ResolveCreatedReference, - request::MaybeInvalid, + request::{MaybeInvalid, capability::CapabilityIds}, types::state::State, }; use jmap_tools::{Key, Value}; @@ -44,17 +61,16 @@ use mail_parser::MessageParser; use std::future::Future; use std::{borrow::Cow, collections::HashMap}; use store::{ - ValueKey, ahash::AHashMap, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{BatchBuilder, PendingId, SlotRange}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, id::Id, - keyword::{ArchivedKeyword, Keyword}, + keyword::Keyword, type_state::{DataType, StateChange}, }; @@ -64,6 +80,7 @@ pub trait EmailSet: Sync + Send { request: SetRequest<'_, Email>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -73,37 +90,53 @@ impl EmailSet for Server { mut request: SetRequest<'_, Email>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { // Prepare response let account_id = request.account_id.document_id(); + let mut metadata_writer = MetadataWriter::new( + ObjectMetadata::new(self, access_token, using, MetadataType::Email), + account_id, + ); + let sampled = metadata_writer + .metadata() + .viewer_change_id(self, account_id) + .await?; let cache = self.get_cached_messages(account_id).await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(false, &request.if_in_state)?); + .with_state(sampled.assert_state(cache.get_state(false), &request.if_in_state)?); // Obtain mailboxIds - let (can_add_mailbox_ids, can_delete_mailbox_ids, can_modify_mailbox_ids) = - if access_token.is_shared(account_id) { - ( - cache.shared_mailboxes(access_token, Acl::AddItems).into(), - cache - .shared_mailboxes(access_token, Acl::RemoveItems) - .into(), - cache - .shared_mailboxes(access_token, Acl::ModifyItems) - .into(), - ) - } else { - (None, None, None) - }; + let is_shared = access_token.is_shared(account_id); + let ( + can_read_mailbox_ids, + can_add_mailbox_ids, + can_delete_mailbox_ids, + can_modify_mailbox_ids, + ) = if is_shared { + ( + cache.shared_mailboxes(access_token, Acl::ReadItems).into(), + cache.shared_mailboxes(access_token, Acl::AddItems).into(), + cache + .shared_mailboxes(access_token, Acl::RemoveItems) + .into(), + cache + .shared_mailboxes(access_token, Acl::ModifyItems) + .into(), + ) + } else { + (None, None, None, None) + }; + let mut can_set_metadata_mailbox_ids = None; // Obtain import access token let import_access_token = if account_id != access_token.account_id() { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { AccessToken::from_id_maybe_invalid(account_id).into() } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { use common::auth::BuildAccessToken; self.access_token(account_id) @@ -120,7 +153,22 @@ impl EmailSet for Server { let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); // Process creates - 'create: for (id, object) in request.unwrap_create() { + let mut recheck = ThreadRecheck::default(); + let mut grouped = GroupedIngest::new( + import_access_token.as_ref().unwrap_or(access_token), + session.session_id, + ); + let creates = request.unwrap_create(); + let grouping = creates.len() > 1; + 'create: for (id, mut object) in creates { + let metadata_patches = + match metadata_writer.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + }; let Value::Object(mut object) = object else { continue; }; @@ -128,11 +176,13 @@ impl EmailSet for Server { let has_body_structure = object.contains_key(&Key::Property(EmailProperty::BodyStructure)); let mut builder = MessageBuilder::new(); + let mut attachment_memory = Vec::new(); let mut mailboxes = Vec::new(); let mut keywords = Vec::new(); let mut received_at = None; // Parse body values + let mut invalid_body_values = Vec::new(); let body_values = object .remove(&Key::Property(EmailProperty::BodyValues)) .and_then(|obj| obj.into_object()) @@ -140,25 +190,73 @@ impl EmailSet for Server { let mut values = HashMap::with_capacity(obj.len()); for (key, value) in obj.into_vec() { let id = key.into_string(); - if let Value::Object(mut bv) = value { - values.insert( - id, - bv.remove(&Key::Property(EmailProperty::Value))? - .into_string()?, - ); - } else { + let Value::Object(bv) = value else { return None; + }; + let mut contents = None; + for (property, value) in bv.into_vec() { + match (property, value) { + (Key::Property(EmailProperty::Value), value) => { + contents = value.into_string(); + } + ( + Key::Property( + flag @ (EmailProperty::IsTruncated + | EmailProperty::IsEncodingProblem), + ), + Value::Bool(true), + ) => { + invalid_body_values.push(InvalidProperty::Path(vec![ + Key::Property(EmailProperty::BodyValues), + Key::Owned(id.replace('~', "~0").replace('/', "~1")), + Key::Property(flag), + ])); + } + _ => {} + } } + values.insert(id, contents?); } Some(values) }); + if !invalid_body_values.is_empty() { + response.not_created.append( + id, + SetError::invalid_properties() + .with_properties(invalid_body_values) + .with_description( + "isTruncated and isEncodingProblem must be false or omitted.", + ), + ); + continue 'create; + } let mut size_attachments = 0; + let mut missing_blobs = Vec::new(); // Parse properties for (property, mut value) in object.into_vec() { if let Err(err) = response.resolve_self_references(&mut value, 0, false) { - response.not_created.append(id, err); - continue 'create; + if grouped.is_empty() { + response.not_created.append(id, err); + continue 'create; + } + if let Err(err) = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) + .await + { + recheck.finish(self, session.session_id).await; + return Err(err); + } + if let Err(err) = response.resolve_self_references(&mut value, 0, false) { + response.not_created.append(id, err); + continue 'create; + } }; let Key::Property(property) = property else { response.invalid_property_create(id, property.into_owned()); @@ -174,11 +272,14 @@ impl EmailSet for Server { }) .collect(); } - (EmailProperty::Keywords, Value::Object(keywords_)) => { - keywords = keywords_ - .into_expanded_boolean_set() - .filter_map(|id| id.try_into_property()?.try_into_keyword()) - .collect(); + (EmailProperty::Keywords, keywords_ @ Value::Object(_)) => { + match parse_keywords(keywords_) { + Ok(keywords_) => keywords = keywords_, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + } } (EmailProperty::Pointer(pointer), value) => { match handle_email_patch(&pointer, value) { @@ -243,7 +344,8 @@ impl EmailSet for Server { } (EmailProperty::ReceivedAt, Value::Element(EmailValue::Date(value))) => { - received_at = (value.timestamp() as u64).into(); + received_at = + (value.timestamp().clamp(0, MAX_RECEIVED_AT as i64) as u64).into(); } (EmailProperty::SentAt, Value::Element(EmailValue::Date(value))) => { @@ -332,7 +434,13 @@ impl EmailSet for Server { EmailProperty::BlobId, Value::Element(EmailValue::BlobId(value)), ) => { - blob_id = value.into(); + blob_id = Some(MaybeInvalid::Value(value)); + } + (EmailProperty::BlobId, Value::Str(value)) + if is_jmap_id(&value) => + { + blob_id = + Some(MaybeInvalid::Invalid(value.into_owned())); } (EmailProperty::Disposition, Value::Str(value)) => { content_disposition = value.into_owned().into(); @@ -513,6 +621,10 @@ impl EmailSet for Server { } // Set Content-Type and Content-Disposition + let is_text = content_type + .as_bytes() + .get(..5) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"text/")); let mut content_type = ContentType::new(content_type); if !is_multipart { if let Some(charset) = charset { @@ -531,7 +643,7 @@ impl EmailSet for Server { ); continue 'create; } - } else if part_id.is_some() { + } else if part_id.is_some() && is_text { content_type .attributes .push(("charset".into(), "utf-8".into())); @@ -562,7 +674,7 @@ impl EmailSet for Server { headers.push(("Content-Type".into(), content_type.into())); // In test, sort headers to avoid randomness - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { headers.sort_unstable_by(|a, b| match a.0.cmp(&b.0) { std::cmp::Ordering::Equal => a.1.cmp(&b.1), @@ -574,18 +686,41 @@ impl EmailSet for Server { headers, contents: if !is_multipart { if let Some(blob_id) = blob_id { - match self.blob_download(&blob_id, access_token).await? { - Some(contents) => { - BodyPart::Binary(contents.into()) + match blob_id { + MaybeInvalid::Value(blob_id) + if missing_blobs.is_empty() => + { + match self + .blob_download(&blob_id, access_token) + .await? + { + Some(contents) => { + let (contents, memory) = + contents.into_parts(); + attachment_memory.push(memory); + BodyPart::Binary(contents.into()) + } + None => { + missing_blobs + .push(MaybeInvalid::Value(blob_id)); + BodyPart::Binary(Cow::Borrowed(&[])) + } + } } - None => { - response.not_created.append( - id, - SetError::new(SetErrorType::BlobNotFound).with_description( - format!("blobId {blob_id} does not exist on this server.") - ), - ); - continue 'create; + MaybeInvalid::Value(blob_id) => { + if !matches!( + self.has_access_blob(&blob_id, access_token) + .await?, + BlobAccess::Allowed { .. } + ) { + missing_blobs + .push(MaybeInvalid::Value(blob_id)); + } + BodyPart::Binary(Cow::Borrowed(&[])) + } + invalid => { + missing_blobs.push(invalid); + BodyPart::Binary(Cow::Borrowed(&[])) } } } else if let Some(part_id) = part_id { @@ -664,6 +799,23 @@ impl EmailSet for Server { } } + (EmailProperty::Header(header), _) + if header + .header + .as_bytes() + .get(..8) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"content-")) => + { + response.not_created.append( + id, + SetError::invalid_properties() + .with_property(EmailProperty::Header(header)) + .with_description( + "Content-* headers can only be set on body parts.", + ), + ); + continue 'create; + } (EmailProperty::Header(header), value) => { match builder.build_header(header, value) { Ok(builder_) => { @@ -685,6 +837,16 @@ impl EmailSet for Server { } } + if !missing_blobs.is_empty() { + response.not_created.append( + id, + SetError::new(SetErrorType::BlobNotFound) + .with_description("One or more blobIds do not exist on this server.") + .with_not_found(missing_blobs), + ); + continue 'create; + } + // Make sure message belongs to at least one mailbox if mailboxes.is_empty() { response.not_created.append( @@ -724,21 +886,44 @@ impl EmailSet for Server { } } - // Make sure the message is not empty - if builder.headers.is_empty() - && builder.body.is_none() - && builder.html_body.is_none() - && builder.text_body.is_none() - && builder.attachments.is_none() + // Validate per-email limits + if let Err(err) = self + .core + .email + .limits + .validate_email(mailboxes.len(), &keywords) { - response.not_created.append( - id, - SetError::invalid_properties() - .with_description("Message has to have at least one header or body part."), - ); + response.not_created.append(id, err.into()); continue 'create; } + let ingest_metadata = match metadata_patches { + Some(metadata_patches) => { + let access = MetadataAccess { + may_write_shared: !is_shared || { + let ids = can_set_metadata_mailbox_ids.get_or_insert_with(|| { + cache.shared_mailboxes(access_token, Acl::ModifyMetadata) + }); + mailboxes.iter().all(|id| ids.contains(*id)) + }, + may_read: can_read_mailbox_ids + .as_ref() + .is_none_or(|ids| mailboxes.iter().any(|id| ids.contains(*id))), + }; + match metadata_writer + .prepare_ingest(self, NewMetadata::Create(metadata_patches), access) + .await? + { + Ok(metadata) => metadata, + Err(err) => { + response.not_created.append(id, err); + continue 'create; + } + } + } + None => None, + }; + match builder .headers .iter() @@ -753,7 +938,7 @@ impl EmailSet for Server { } // In test, sort headers to avoid randomness - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { builder .headers @@ -767,44 +952,130 @@ impl EmailSet for Server { let mut raw_message = Vec::with_capacity((4 * size_attachments / 3) + 1024); builder.write_to(&mut raw_message).unwrap_or_default(); + let message = match MessageParser::new().parse(&raw_message) { + Some(message) => { + if let Err(err) = self.core.email.limits.validate_header_section(&message) { + response.not_created.append(id, err.into()); + continue 'create; + } + Some(message) + } + None => None, + }; + // Ingest message - match self - .email_ingest(IngestEmail { - raw_message: &raw_message, - message: MessageParser::new().parse(&raw_message), + if let (Some(parsed), None) = (&message, &ingest_metadata) { + let key = if grouping { + ThreadKey::new(parsed) + } else { + ThreadKey::default() + }; + if grouped.must_flush_before(raw_message.len()) + && let Err(err) = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) + .await + { + recheck.finish(self, session.session_id).await; + return Err(err); + } + let email = QueuedEmail { + id, blob_hash: None, - access_token: import_access_token.as_ref().unwrap_or(access_token), mailbox_ids: mailboxes, keywords, received_at, - source: IngestSource::Jmap { - train_classifier: true, + train_classifier: true, + }; + let result = grouped + .ingest(self, &email, &raw_message, message, key) + .await; + grouped.queue( + email, + HeldMessage { + raw: raw_message, + _memory: attachment_memory, }, - session_id: session.session_id, - }) + result, + ); + } else { + if let Err(err) = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) .await - { - Ok(message) => { - last_change_id = message.change_id.into(); - response - .created - .insert(id, ingested_into_object(message).into()); + { + recheck.finish(self, session.session_id).await; + return Err(err); } - Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { - response.not_created.append( - id, - SetError::new(SetErrorType::OverQuota) - .with_description("You have exceeded your disk quota."), - ); - } - Err(err) => return Err(err), + let result = self + .email_ingest(IngestEmail { + raw_message: &raw_message, + message, + blob_hash: None, + access_token: import_access_token.as_ref().unwrap_or(access_token), + mailbox_ids: mailboxes, + keywords, + received_at, + source: IngestSource::Jmap { + train_classifier: true, + }, + metadata: ingest_metadata, + session_id: session.session_id, + }) + .await; + grouped.results.push((id, result)); + } + if let Err(err) = record( + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) { + recheck.finish(self, session.session_id).await; + return Err(err); } } + if let Err(err) = flush( + self, + &mut grouped, + &mut response, + &mut recheck, + &mut last_change_id, + account_id, + ) + .await + { + recheck.finish(self, session.session_id).await; + return Err(err); + } + recheck.finish(self, session.session_id).await; // Process updates let mut batch = BatchBuilder::new(); + let mut spam_samples = SpamSamples::default(); let mut changed_mailboxes: AHashMap> = AHashMap::new(); + let mut update_mailboxes: Vec<(u32, Option)> = Vec::new(); let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); + let mut has_shared_updates = false; + let mut shared_updates_committed = false; + metadata_writer + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .email_by_id(&document_id) + .map(|email| cache.metadata_kinds(email)) + }) + .await?; 'update: for (id, object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, @@ -818,309 +1089,430 @@ impl EmailSet for Server { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } - - // Obtain message data let document_id = id.document_id(); - let data_ = match self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - document_id, - )) - .await? - { - Some(data) => data, - None => { + let (current_kinds, metadata_writes, has_mailbox_changes, train_spam) = { + let mut object = object; + let metadata_patches = + match metadata_writer.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + }; + + // Obtain message data + let Some(data) = cache.message_data(document_id).filter(|data| { + can_read_mailbox_ids.as_ref().is_none_or(|ids| { + data.mailboxes + .iter() + .any(|mailbox| ids.contains(mailbox.mailbox_id)) + }) + }) else { response.not_updated.append(id, SetError::not_found()); continue 'update; - } - }; - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_data = data.inner.to_builder(); - - for (property, mut value) in object.into_expanded_object() { - if let Err(err) = response.resolve_self_references(&mut value, 0, false) { - response.not_updated.append(id, err); - continue 'update; }; + let mut new_data = data.clone(); + let mut has_keyword_replace = false; - match (property, value) { - (Key::Property(EmailProperty::MailboxIds), Value::Object(ids)) => { - new_data.set_mailboxes( - ids.into_expanded_boolean_set() - .filter_map(|id| { - let mailbox_id = - id.try_into_property()?.try_into_id()?.document_id(); - UidMailbox::new( - mailbox_id, - data.inner.message_uid(mailbox_id).unwrap_or(0), - ) - .into() - }) - .collect(), - ); - } - (Key::Property(EmailProperty::Keywords), Value::Object(keywords_)) => { - new_data.set_keywords( - keywords_ - .into_expanded_boolean_set() - .filter_map(|keyword| { - keyword.try_into_property()?.try_into_keyword() - }) - .collect(), - ); - } - (Key::Property(EmailProperty::Pointer(pointer)), value) => { - match handle_email_patch(&pointer, value) { - PatchResult::SetKeyword(keyword) => { - new_data.add_keyword(keyword.clone()); + for (property, mut value) in object.into_expanded_object() { + if let Err(err) = response.resolve_self_references(&mut value, 0, false) { + response.not_updated.append(id, err); + continue 'update; + }; + + match (property, value) { + (Key::Property(EmailProperty::MailboxIds), Value::Object(ids)) => { + new_data.set_mailboxes( + ids.into_expanded_boolean_set() + .filter_map(|id| { + let mailbox_id = + id.try_into_property()?.try_into_id()?.document_id(); + MessageUid::new( + mailbox_id, + data.message_uid(mailbox_id).unwrap_or(0), + ) + .into() + }) + .collect(), + ); + } + (Key::Property(EmailProperty::Keywords), keywords @ Value::Object(_)) => { + has_keyword_replace = true; + match parse_keywords(keywords) { + Ok(keywords) => new_data.set_keywords(keywords), + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } } - PatchResult::RemoveKeyword(keyword) => { - new_data.remove_keyword(keyword); + } + (Key::Property(EmailProperty::Pointer(pointer)), value) => { + match handle_email_patch(&pointer, value) { + PatchResult::SetKeyword(keyword) => { + new_data.add_keyword(keyword.clone()); + } + PatchResult::RemoveKeyword(keyword) => { + new_data.remove_keyword(keyword); + } + PatchResult::AddMailbox(id) => { + new_data.add_mailbox(MessageUid::new_unassigned(id)); + } + PatchResult::RemoveMailbox(id) => { + new_data.remove_mailbox(id); + } + PatchResult::Invalid(set_error) => { + response.not_updated.append(id, set_error); + continue 'update; + } } - PatchResult::AddMailbox(id) => { - new_data.add_mailbox(UidMailbox::new_unassigned(id)); - } - PatchResult::RemoveMailbox(id) => { - new_data.remove_mailbox(id); - } - PatchResult::Invalid(set_error) => { - response.not_updated.append(id, set_error); + } + (Key::Property(EmailProperty::Id), value) => { + if !crate::matches_id(&value, id) { + response.not_updated.append( + id, + SetError::invalid_properties() + .with_property(EmailProperty::Id) + .with_description("The id property is immutable."), + ); continue 'update; } } - } - (Key::Property(EmailProperty::Id), value) => { - if !crate::matches_id(&value, id) { - response.not_updated.append( - id, - SetError::invalid_properties() - .with_property(EmailProperty::Id) - .with_description("The id property is immutable."), - ); + (property, _) => { + response.invalid_property_update(id, property.into_owned()); continue 'update; } } - (property, _) => { - response.invalid_property_update(id, property.into_owned()); + } + + let current_kinds = data.metadata_kinds(); + let mut shared_kinds = None; + let metadata_writes = if let Some(metadata_patches) = metadata_patches { + let access = MetadataAccess { + may_write_shared: !is_shared || { + let ids = can_set_metadata_mailbox_ids.get_or_insert_with(|| { + cache.shared_mailboxes(access_token, Acl::ModifyMetadata) + }); + data.mailboxes + .iter() + .chain(new_data.added_mailboxes(&data)) + .all(|mailbox| ids.contains(mailbox.mailbox_id)) + }, + may_read: true, + }; + match metadata_writer + .prepare_for( + self, + metadata_patches, + access, + MetadataTarget::UpdateInThread { + document_id, + thread_id: data.thread_id, + }, + ) + .await? + { + Ok(prepared) if prepared.is_empty() => None, + Ok(prepared) => { + shared_kinds = prepared.shared_kinds(); + Some(prepared) + } + Err(err) => { + response.not_updated.append(id, err); + continue 'update; + } + } + } else { + None + }; + + let has_keyword_changes = new_data.has_keyword_changes(&data); + let has_mailbox_changes = new_data.has_mailbox_changes(&data); + if !has_keyword_changes && !has_mailbox_changes { + match metadata_writes { + Some(prepared) => { + has_shared_updates |= !prepared.is_private_only(); + if let Some(presence) = metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())? + { + update_email_presence( + &mut batch, + account_id, + document_id, + current_kinds, + presence, + ); + } + batch.commit_point(); + will_update.push(id); + } + None => { + response.updated.append(id, None); + } + } + continue 'update; + } + + // Validate per-email limits + if let Err(err) = new_data.validate_limits(&data, &self.core.email.limits) { + response.not_updated.append(id, err.into()); + continue 'update; + } + + // Process keywords + let mut train_spam = None; + update_mailboxes.clear(); + if has_keyword_changes { + // Verify permissions on shared accounts + if can_modify_mailbox_ids.as_ref().is_some_and(|ids| { + !data + .mailboxes + .iter() + .chain(new_data.added_mailboxes(&data)) + .all(|mb| ids.contains(mb.mailbox_id)) + }) { + response.not_updated.append( + id, + SetError::forbidden() + .with_description("You are not allowed to modify keywords."), + ); continue 'update; } - } - } - - let has_keyword_changes = new_data.has_keyword_changes(data.inner); - let has_mailbox_changes = new_data.has_mailbox_changes(data.inner); - if !has_keyword_changes && !has_mailbox_changes { - response.updated.append(id, None); - continue 'update; - } - - // Process keywords - let mut train_spam = None; - if has_keyword_changes { - // Verify permissions on shared accounts - if can_modify_mailbox_ids.as_ref().is_some_and(|ids| { - !new_data - .mailboxes - .iter() - .any(|mb| ids.contains(mb.mailbox_id)) - }) { - response.not_updated.append( - id, - SetError::forbidden() - .with_description("You are not allowed to modify keywords."), - ); - continue 'update; - } - - // Process keyword changes - let mut changed_seen = false; - for keyword in new_data.added_keywords(data.inner) { - match keyword { - Keyword::Seen => { - changed_seen = true; - } - Keyword::Junk => { - train_spam = Some(true); - } - Keyword::NotJunk => { - train_spam = Some(false); - } - _ => {} + if can_delete_mailbox_ids.as_ref().is_some_and(|ids| { + new_data + .added_keywords(&data) + .chain(new_data.removed_keywords(&data)) + .any(|keyword| keyword == Keyword::Deleted) + && !data + .mailboxes + .iter() + .chain(new_data.added_mailboxes(&data)) + .all(|mb| ids.contains(mb.mailbox_id)) + }) { + response.not_updated.append( + id, + SetError::forbidden().with_description( + "You are not allowed to change the $deleted keyword.", + ), + ); + continue 'update; } - } - for keyword in new_data.removed_keywords(data.inner) { - match keyword { - ArchivedKeyword::Seen => { - changed_seen = true; - } - ArchivedKeyword::Junk if train_spam.is_none() => { - train_spam = Some(false); - } - _ => {} - } - } - // Set all current mailboxes as changed if the Seen tag changed - if changed_seen { - for mailbox_id in new_data.mailboxes.iter() { - changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); - } - } - } - - // Process mailboxes - if has_mailbox_changes { - // Make sure the message is at least in one mailbox - if new_data.mailboxes.is_empty() { - response.not_updated.append( - id, - SetError::invalid_properties() - .with_property(EmailProperty::MailboxIds) - .with_description("Message has to belong to at least one mailbox."), - ); - continue 'update; - } - - // Make sure all new mailboxIds are valid - for mailbox_id in new_data.added_mailboxes(data.inner) { - if cache.has_mailbox_id(&mailbox_id.mailbox_id) { - // Verify permissions on shared accounts - if can_add_mailbox_ids - .as_ref() - .is_none_or(|ids| ids.contains(mailbox_id.mailbox_id)) - { - if mailbox_id.mailbox_id == JUNK_ID { + // Process keyword changes + let mut changed_seen = false; + for keyword in new_data.added_keywords(&data) { + match keyword { + Keyword::Seen => { + changed_seen = true; + } + Keyword::Junk => { train_spam = Some(true); } - - changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); - } else { - response.not_updated.append( - id, - SetError::forbidden().with_description(format!( - "You are not allowed to add messages to mailbox {}.", - Id::from(mailbox_id.mailbox_id) - )), - ); - continue 'update; + Keyword::NotJunk => { + train_spam = Some(false); + } + _ => {} } - } else { + } + for keyword in new_data.removed_keywords(&data) { + match keyword { + Keyword::Seen => { + changed_seen = true; + } + Keyword::Junk if train_spam.is_none() => { + train_spam = Some(false); + } + _ => {} + } + } + + // Set all current mailboxes as changed if the Seen tag changed + if changed_seen { + for mailbox_id in new_data.mailboxes.iter() { + update_mailboxes.push((mailbox_id.mailbox_id, None)); + } + } + } + + // Process mailboxes + let mut uid_slots = SlotRange::default(); + if has_mailbox_changes { + // Make sure the message is at least in one mailbox + if new_data.mailboxes.is_empty() { response.not_updated.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) - .with_description(format!( - "mailboxId {} does not exist.", - Id::from(mailbox_id.mailbox_id) - )), + .with_description("Message has to belong to at least one mailbox."), ); continue 'update; } - } - // Add all removed mailboxes to change list - for mailbox_id in new_data.removed_mailboxes(data.inner) { - // Verify permissions on shared accounts - if can_delete_mailbox_ids - .as_ref() - .is_none_or(|ids| ids.contains(u32::from(mailbox_id.mailbox_id))) - { - if mailbox_id.mailbox_id == JUNK_ID - && !new_data - .mailboxes - .iter() - .any(|mb| mb.mailbox_id == TRASH_ID) - { - train_spam = Some(false); + // Make sure all new mailboxIds are valid + for mailbox_id in new_data.added_mailboxes(&data) { + if cache.has_mailbox_id(&mailbox_id.mailbox_id) { + // Verify permissions on shared accounts + if can_add_mailbox_ids + .as_ref() + .is_none_or(|ids| ids.contains(mailbox_id.mailbox_id)) + { + if mailbox_id.mailbox_id == JUNK_ID { + train_spam = Some(true); + } + + update_mailboxes.push((mailbox_id.mailbox_id, None)); + } else { + response.not_updated.append( + id, + SetError::forbidden().with_description(format!( + "You are not allowed to add messages to mailbox {}.", + Id::from(mailbox_id.mailbox_id) + )), + ); + continue 'update; + } + } else { + response.not_updated.append( + id, + SetError::invalid_properties() + .with_property(EmailProperty::MailboxIds) + .with_description(format!( + "mailboxId {} does not exist.", + Id::from(mailbox_id.mailbox_id) + )), + ); + continue 'update; } - - changed_mailboxes - .entry(mailbox_id.mailbox_id.to_native()) - .or_default() - .push(mailbox_id.uid.to_native()); - } else { - response.not_updated.append( - id, - SetError::forbidden().with_description(format!( - "You are not allowed to delete messages from mailbox {}.", - mailbox_id.mailbox_id - )), - ); - continue 'update; } - } - // Obtain IMAP UIDs for added mailboxes - let ids = self - .assign_email_ids( + // Add all removed mailboxes to change list + for mailbox_id in new_data.removed_mailboxes(&data) { + // Verify permissions on shared accounts + if can_delete_mailbox_ids + .as_ref() + .is_none_or(|ids| ids.contains(mailbox_id.mailbox_id)) + { + if mailbox_id.mailbox_id == JUNK_ID + && !new_data + .mailboxes + .iter() + .any(|mb| mb.mailbox_id == TRASH_ID) + { + train_spam = Some(false); + } + + update_mailboxes.push((mailbox_id.mailbox_id, Some(mailbox_id.uid))); + } else { + response.not_updated.append( + id, + SetError::forbidden().with_description(format!( + "You are not allowed to delete messages from mailbox {}.", + mailbox_id.mailbox_id + )), + ); + continue 'update; + } + } + + // Reserve IMAP UIDs for added mailboxes + uid_slots = batch.reserve_uids( account_id, new_data .mailboxes .iter() .filter(|m| m.uid == 0) .map(|m| m.mailbox_id), - false, - ) - .await - .caused_by(trc::location!())?; - for (uid_mailbox, uid) in new_data - .mailboxes - .iter_mut() - .filter(|m| m.uid == 0) - .zip(ids) - { - uid_mailbox.uid = uid; + ); } - } - // Write changes - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(document_id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data.seal()), + // Write changes + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(document_id); + if has_mailbox_changes { + if let Some(kinds) = shared_kinds { + new_data.set_metadata_kinds(kinds); + } + batch + .custom(ObjectIndexBuilder::new().with_current(data).with_changes( + PendingMessageData { + data: new_data, + uid_slots, + thread_slot: None, + change_id: None, + }, + )) + .caused_by(trc::location!())?; + } else { + merge_keywords( + &mut batch, + data.thread_id, + if has_keyword_replace { + KeywordDiff::replace(new_data.keywords().collect()) + } else { + new_data.keyword_diff(&data) + }, + ); + } + + ( + current_kinds, + metadata_writes, + has_mailbox_changes, + train_spam, ) - .caused_by(trc::location!())?; + }; if let Some(train_spam) = train_spam { - self.add_account_spam_sample( - &mut batch, - account_id, - document_id, - train_spam, - session.session_id, - ) - .await - .caused_by(trc::location!())?; + spam_samples.add(document_id, train_spam); + } + + if let Some(prepared) = metadata_writes + && let Some(presence) = metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())? + && !has_mailbox_changes + { + update_email_presence(&mut batch, account_id, document_id, current_kinds, presence); } batch.commit_point(); + for (mailbox_id, uid) in update_mailboxes.drain(..) { + changed_mailboxes.entry(mailbox_id).or_default().extend(uid); + } + has_shared_updates = true; will_update.push(id); } + self.add_account_spam_samples(&mut batch, account_id, spam_samples, session.session_id) + .await + .caused_by(trc::location!())?; + + let mut has_private_changes = false; if !batch.is_empty() { // Log mailbox changes for (parent_id, deleted_uids) in changed_mailboxes { - batch.log_container_property_change(SyncCollection::Email, parent_id); + batch.log_container_property_change( + SyncCollection::Email, + PendingId::Assigned(parent_id), + ); for deleted_uid in deleted_uids { batch.log_vanished_item(VanishedCollection::Email, (parent_id, deleted_uid)); } } - match self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - { - Ok(change_id) => { - last_change_id = change_id.into(); + match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => { + if has_shared_updates { + shared_updates_committed = true; + last_change_id = assigned_ids + .last_change_id(account_id, SyncCollection::Email) + .into(); + } else { + has_private_changes = true; + } // Add to updated list for id in will_update { @@ -1145,32 +1537,37 @@ impl EmailSet for Server { // Process deletions if !will_destroy.is_empty() { - let email_ids = cache.email_document_ids(); - let can_destroy_message_ids = if access_token.is_shared(account_id) { - cache.shared_messages(access_token, Acl::RemoveItems).into() - } else { - None + let (can_read, can_destroy) = match (can_read_mailbox_ids, can_delete_mailbox_ids) { + (Some(can_read), Some(can_delete)) => ( + MessageAccess::Mailboxes(can_read), + MessageAccess::Mailboxes(can_delete), + ), + _ => (MessageAccess::All, MessageAccess::All), }; let mut destroy_ids = RoaringBitmap::new(); for destroy_id in will_destroy { let document_id = destroy_id.document_id(); - if email_ids.contains(document_id) { - if !matches!(&can_destroy_message_ids, Some(ids) if !ids.contains(document_id)) - { + match cache + .email_by_id(&document_id) + .filter(|message| can_read.allows(*message)) + { + Some(message) if can_destroy.allows(message) => { destroy_ids.insert(document_id); response.destroyed.push(destroy_id); - } else { + } + Some(_) => { response.not_destroyed.append( destroy_id, SetError::forbidden() .with_description("You are not allowed to delete this message."), ); } - } else { - response - .not_destroyed - .append(destroy_id, SetError::not_found()); + None => { + response + .not_destroyed + .append(destroy_id, SetError::not_found()); + } } } @@ -1180,19 +1577,32 @@ impl EmailSet for Server { let not_destroyed = self .emails_delete( account_id, - access_token.tenant_id(), + access_token.changed_by(), &mut batch, destroy_ids, ) .await?; if !batch.is_empty() { - last_change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())? - .into(); - self.notify_task_queue(); + match self.commit_batch(batch).await { + Ok(assigned_ids) => { + last_change_id = assigned_ids + .last_change_id(account_id, SyncCollection::Email) + .into(); + } + Err(err) if err.is_assertion_failure() => { + for destroy_id in response.destroyed.drain(..) { + response.not_destroyed.append( + destroy_id, + SetError::forbidden().with_description( + "Another process modified this message, please try again.", + ), + ); + } + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } } // Mark messages that were not found as not destroyed (this should not occur in practice) @@ -1216,7 +1626,7 @@ impl EmailSet for Server { // Update state if let Some(change_id) = last_change_id { - if response.updated.is_empty() && response.destroyed.is_empty() { + if !shared_updates_committed && response.destroyed.is_empty() { // Message ingest does not broadcast state changes let mut state_change = StateChange::new(account_id) .with_change_id(change_id) @@ -1236,10 +1646,75 @@ impl EmailSet for Server { )) .await; } - - response.new_state = State::Exact(change_id).into(); + } + if last_change_id.is_some() || has_private_changes { + response.new_state = metadata_writer + .metadata() + .viewer_change_id(self, account_id) + .await? + .state(last_change_id.map_or_else(|| cache.get_state(false), State::Exact)) + .into(); } Ok(response) } } + +async fn flush( + server: &Server, + grouped: &mut GroupedIngest<'_, HeldMessage>, + response: &mut SetResponse, + recheck: &mut ThreadRecheck, + last_change_id: &mut Option, + account_id: u32, +) -> trc::Result<()> { + grouped.flush(server).await?; + record(grouped, response, recheck, last_change_id, account_id) +} + +fn record( + grouped: &mut GroupedIngest<'_, HeldMessage>, + response: &mut SetResponse, + recheck: &mut ThreadRecheck, + last_change_id: &mut Option, + account_id: u32, +) -> trc::Result<()> { + for (id, result) in grouped.results.drain(..) { + match result { + Ok(mut message) => { + recheck.push(account_id, &mut message); + *last_change_id = message.change_id.into(); + response + .created + .insert(id, ingested_into_object(message).into()); + } + Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { + response.not_created.append( + id, + SetError::new(SetErrorType::OverQuota) + .with_description("You have exceeded your account quota."), + ); + } + Err(err) => return Err(err), + } + } + Ok(()) +} + +struct HeldMessage { + raw: Vec, + _memory: Vec, +} + +impl AsRef<[u8]> for HeldMessage { + fn as_ref(&self) -> &[u8] { + &self.raw + } +} + +fn is_jmap_id(value: &str) -> bool { + (1..=255).contains(&value.len()) + && value + .bytes() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'-' | b'_')) +} diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 0127abe7..eecbfbd2 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -4,11 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; +use common::{ + Server, + auth::AccessToken, + storage::batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, +}; use email::{ - cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::metadata::{ - ArchivedMetadataPartType, DecodedPartContent, MessageMetadata, MetadataHeaderName, + cache::{ + MessageCacheFetch, + email::{MessageAccess, MessageCacheAccess}, + mailbox::MailboxCacheAccess, + }, + message::{ + messagedata::{EmailMessageData, MetadataForm}, + metadata::{PartKind, PartSource}, }, }; use jmap_proto::{ @@ -19,17 +28,15 @@ use jmap_proto::{ object::email::EmailFilter, request::MaybeInvalid, }; -use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use std::future::Future; use store::{ - ValueKey, backend::MAX_TOKEN_LENGTH, - write::{AlignedBytes, Archive}, + search::{MAX_PLANNED_TOKENS, MAX_QUERY_LEAVES, MAX_QUERY_PLANNED_TOKENS}, }; use trc::AddContext; -use types::{acl::Acl, collection::Collection, field::EmailField}; -use utils::chained_bytes::ChainedBytes; +use types::{acl::Acl, collection::Collection}; +use utils::cheeky_hash::{CheekyHash, CheekyHashSet}; pub trait EmailSearchSnippet: Sync + Send { fn email_search_snippet( @@ -48,6 +55,9 @@ impl EmailSearchSnippet for Server { let mut filter_stack = vec![]; let mut include_term = true; let mut terms = vec![]; + let mut seen = CheekyHashSet::default(); + let mut planned = 0usize; + let mut conditions = 0usize; let mut is_exact = false; let mut language = self.core.email.default_language; @@ -59,23 +69,46 @@ impl EmailSearchSnippet for Server { | EmailFilter::Body(text) = cond && include_term { + conditions += 1; + if conditions > MAX_QUERY_LEAVES { + return Err(too_many_terms()); + } let (text, language_) = Language::detect(text, self.core.email.default_language); language = language_; if (text.starts_with('"') && text.ends_with('"')) || (text.starts_with('\'') && text.ends_with('\'')) { - for token in language.tokenize_text(&text, MAX_TOKEN_LENGTH) { - terms.push(token.word.into_owned()); - } + let before = terms.len(); + terms.extend( + language + .tokenize_text(&text, MAX_TOKEN_LENGTH) + .take(MAX_PLANNED_TOKENS) + .map(|token| token.word.into_owned()), + ); + planned += terms.len() - before; is_exact = true; } else { + let mut condition_words = 0; for token in Stemmer::new(&text, language, MAX_TOKEN_LENGTH) { + if !seen.insert(CheekyHash::new(token.word.as_bytes())) { + continue; + } terms.push(token.word.into_owned()); - if let Some(stemmed_word) = token.stemmed_word { + if let Some(stemmed_word) = token.stemmed_word + && seen.insert(CheekyHash::new(stemmed_word.as_bytes())) + { terms.push(stemmed_word.into_owned()); } + condition_words += 1; + if condition_words == MAX_PLANNED_TOKENS { + break; + } } + planned += condition_words; + } + if planned > MAX_QUERY_PLANNED_TOKENS { + return Err(too_many_terms()); } } } @@ -93,165 +126,159 @@ impl EmailSearchSnippet for Server { } } } + let email_ids = request.email_ids.unwrap(); + if email_ids.len() > self.core.jmap.snippet_max_results { + return Err(trc::JmapEvent::RequestTooLarge.into_err()); + } + let account_id = request.account_id.document_id(); let cached_messages = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - let document_ids = if access_token.is_member(account_id) { - cached_messages.email_document_ids() + let access = if access_token.is_member(account_id) { + MessageAccess::All } else { - cached_messages.shared_messages(access_token, Acl::ReadItems) + MessageAccess::Mailboxes(cached_messages.shared_mailboxes(access_token, Acl::ReadItems)) }; - let email_ids = request.email_ids.unwrap(); let mut response = GetSearchSnippetResponse { account_id: request.account_id, list: Vec::with_capacity(email_ids.len()), not_found: None, }; let mut not_found = Vec::new(); + let mut batch = DocumentBatch::new(MAX_BATCH_BYTES); - if email_ids.len() > self.core.jmap.snippet_max_results { - return Err(trc::JmapEvent::RequestTooLarge.into_err()); - } - - for email_id in email_ids { - let email_id = match email_id { - MaybeInvalid::Value(email_id) => email_id, - invalid => { - not_found.push(invalid); - continue; + for chunk in email_ids.chunks(MAX_BATCH_DOCUMENTS) { + batch.reset(); + if !terms.is_empty() { + for email_id in chunk { + if let MaybeInvalid::Value(email_id) = email_id + && cached_messages + .email_by_id(&email_id.document_id()) + .is_some_and(|message| access.allows(message)) + { + batch.request(email_id.document_id()); + } } - }; - let document_id = email_id.document_id(); - let mut snippet = SearchSnippet { - email_id, - subject: None, - preview: None, - }; - if !document_ids.contains(document_id) { - not_found.push(MaybeInvalid::Value(email_id)); - continue; - } else if terms.is_empty() { - response.list.push(snippet); - continue; - } - let metadata_ = match self - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await? - { - Some(metadata) => metadata, - None => { - not_found.push(MaybeInvalid::Value(email_id)); - continue; - } - }; - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; - - // Add subject snippet - let contents = &metadata.contents[0]; - if let Some(subject) = contents - .root_part() - .header_value(&MetadataHeaderName::Subject) - .and_then(|v| v.as_text()) - .and_then(|v| generate_snippet(v, &terms, language, is_exact)) - { - snippet.subject = subject.into(); } - // Download message - let raw_body = if let Some(raw_body) = self - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) - .await? - { - raw_body - } else { - trc::event!( - Store(trc::StoreEvent::NotFound), - AccountId = account_id, - DocumentId = email_id.document_id(), - Collection = Collection::Email, - BlobId = metadata.blob_hash.0.as_slice(), - Details = "Blob not found.", - CausedBy = trc::location!(), - ); + loop { + self.read_metadata_batch(account_id, MetadataForm::Structure, &mut batch) + .await + .caused_by(trc::location!())?; - not_found.push(MaybeInvalid::Value(email_id)); - continue; - }; - let raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()).with_last( - raw_body - .get(metadata.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - - // Find a matching part - 'outer: for part in contents.parts.iter() { - match &part.body { - ArchivedMetadataPartType::Text => { - let text = match part.decode_contents(&raw_message) { - DecodedPartContent::Text(text) => text, - _ => unreachable!(), - }; - - if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { - snippet.preview = body.into(); - break; + for email_id in chunk { + let email_id = match email_id { + MaybeInvalid::Value(email_id) => *email_id, + invalid => { + if batch.is_first_round() { + not_found.push(invalid.clone()); + } + continue; } - } - ArchivedMetadataPartType::Html => { - let text = match part.decode_contents(&raw_message) { - DecodedPartContent::Text(html) => html_to_text(&html), - _ => unreachable!(), - }; - - if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { - snippet.preview = body.into(); - break; + }; + let document_id = email_id.document_id(); + let mut snippet = SearchSnippet { + email_id, + subject: None, + preview: None, + }; + if !cached_messages + .email_by_id(&document_id) + .is_some_and(|message| access.allows(message)) + { + if batch.is_first_round() { + not_found.push(MaybeInvalid::Value(email_id)); } + continue; + } else if terms.is_empty() { + response.list.push(snippet); + continue; } - ArchivedMetadataPartType::Message(message) => { - for part in metadata.contents[u16::from(message) as usize].parts.iter() { - if let ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html = - part.body - { - let text = match (part.decode_contents(&raw_message), &part.body) { - ( - DecodedPartContent::Text(text), - ArchivedMetadataPartType::Text, - ) => text, - ( - DecodedPartContent::Text(html), - ArchivedMetadataPartType::Html, - ) => html_to_text(&html).into(), - _ => unreachable!(), - }; + let structure = match batch.lookup(document_id) { + Lookup::Found(structure) => structure, + Lookup::Missing => { + not_found.push(MaybeInvalid::Value(email_id)); + continue; + } + Lookup::Skip => continue, + }; + let metadata = structure.unarchive().caused_by(trc::location!())?; + let root = metadata.root(); - if let Some(body) = + // Add subject snippet + if let Some(subject) = root + .envelope() + .subject() + .and_then(|subject| generate_snippet(subject, &terms, language, is_exact)) + { + snippet.subject = subject.into(); + } + + // Download message + let blob_hash = metadata.blob_hash(); + let Some(raw_body) = self + .fetch_blob(&blob_hash, 0..usize::MAX, Some(metadata.size() as u64)) + .await? + else { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = email_id.document_id(), + Collection = Collection::Email, + BlobId = blob_hash.as_slice(), + Details = "Blob not found.", + CausedBy = trc::location!(), + ); + + not_found.push(MaybeInvalid::Value(email_id)); + continue; + }; + let raw = metadata.raw_message(None, &raw_body); + let source = PartSource::Raw(raw); + + // Find a matching part + 'outer: for part in root.parts() { + match part.kind() { + PartKind::Text | PartKind::Html => { + if let Some(body) = part.plain_text(&source).and_then(|text| { generate_snippet(&text, &terms, language, is_exact) - { + }) { snippet.preview = body.into(); - break 'outer; + break; } } + PartKind::Message => { + let Some(nested) = part.nested() else { + continue; + }; + let Some(nested_source) = metadata.source(nested, raw) else { + continue; + }; + for part in nested.parts().filter(|part| part.is_text()) { + if let Some(body) = + part.plain_text(&nested_source).and_then(|text| { + generate_snippet(&text, &terms, language, is_exact) + }) + { + snippet.preview = body.into(); + break 'outer; + } + } + } + _ => (), } } - _ => (), + + response.list.push(snippet); + } + + if !batch.next_round() { + break; } } - //} - - response.list.push(snippet); } if !not_found.is_empty() { @@ -261,3 +288,10 @@ impl EmailSearchSnippet for Server { Ok(response) } } + +#[cold] +fn too_many_terms() -> trc::Error { + trc::LimitEvent::SizeRequest + .into_err() + .details("Too many search terms in one query") +} diff --git a/crates/jmap/src/file/copy.rs b/crates/jmap/src/file/copy.rs index e1dd5111..24aa0f53 100644 --- a/crates/jmap/src/file/copy.rs +++ b/crates/jmap/src/file/copy.rs @@ -4,43 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - api::acl::JmapAcl, - blob::download::BlobDownload, - changes::state::JmapCacheState, - file::set::{ - Collision, NoResolver, fetch_existing_modified, find_sibling_collision, pick_unique_rename, - update_file_node, validate_file_node_hierarchy, - }, +use super::{ + node::NoResolver, + set::create_node, + writer::{FileNodeWriter, WriteOptions, fetch_archive}, }; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use crate::{ + api::metadata::{MetadataPreload, MetadataType, ObjectMetadata}, + changes::state::JmapCacheState, +}; +use common::{Server, auth::AccessToken}; use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ - copy::{CopyRequest, CopyResponse}, + copy::{CopyRequest, CopyResponse, CopySourceId}, set::SetRequest, }, - object::file_node::{self, FileNodeProperty, OnExists}, + object::file_node::{self, FileNodeProperty, FileNodeValue}, request::{ - Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, + Call, MaybeInvalid, RequestMethod, SetRequestMethod, + capability::CapabilityIds, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, types::state::State, }; -use store::{ - ValueKey, - ahash::{AHashMap, AHashSet}, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, now}, -}; +use jmap_tools::{Key, Value}; use trc::AddContext; -use types::{ - acl::Acl, - collection::{Collection, SyncCollection}, -}; +use types::{collection::SyncCollection, id::Id}; use utils::map::vec_map::VecMap; pub trait FileNodeCopy: Sync + Send { @@ -50,6 +43,7 @@ pub trait FileNodeCopy: Sync + Send { access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -60,6 +54,7 @@ impl FileNodeCopy for Server { access_token: &AccessToken, next_call: &mut Option>>, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); @@ -70,15 +65,17 @@ impl FileNodeCopy for Server { .details("From accountId is equal to fromAccountId")); } + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::FileNode); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; - let old_state = cache.assert_state(false, &request.if_in_state)?; + let old_state = sampled.assert_state(cache.get_state(false), &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, @@ -86,366 +83,170 @@ impl FileNodeCopy for Server { old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), + destroyed: Some(None), }; let from_cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), from_account_id, SyncCollection::FileNode, ) .await .caused_by(trc::location!())?; - let from_node_ids = if access_token.is_member(from_account_id) { - from_cache - .resources - .iter() - .map(|r| r.document_id) - .collect::() - } else { - let mut readable = - from_cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true); - readable |= from_cache.shared_items(access_token, [Acl::ReadItems], true); - readable - }; + let from_readable = (!access_token.is_member(from_account_id)) + .then(|| from_cache.file_access(access_token).readable); - let is_shared = access_token.is_shared(account_id); - let can_add_to = if is_shared { - Some(cache.shared_containers(access_token, [Acl::AddItems], true)) - } else { - None + let options = WriteOptions { + on_exists: request.arguments.on_exists, + remove_children: request + .arguments + .on_destroy_remove_children + .unwrap_or(false), + case_insensitive: request + .arguments + .compare_case_insensitively + .unwrap_or(false), }; - let on_exists = request.arguments.on_exists; - let case_insensitive = request - .arguments - .compare_case_insensitively - .unwrap_or(false); - let on_destroy_remove_children = request - .arguments - .on_destroy_remove_children - .unwrap_or(false); let on_success_delete = request.on_success_destroy_original.unwrap_or(false); - - let mut batch = BatchBuilder::new(); - let mut pending_names: AHashMap<(u32, String), Option> = AHashMap::new(); - let mut implicit_destroys: AHashSet = AHashSet::new(); - let mut created_folders = AHashMap::new(); + let viewer = metadata.viewer(); + let mut writer = + FileNodeWriter::new(self, access_token, account_id, &cache, options, metadata) + .with_quota() + .await?; + let mut created = Vec::with_capacity(request.create.len()); + let mut awaiting_existing = Vec::new(); let mut destroy_ids = Vec::new(); + let is_readable = |document_id: u32| { + from_cache.resources.find_any(document_id).is_some() + && from_readable + .as_ref() + .is_none_or(|readable| readable.contains(document_id)) + }; - 'create: for (id, create) in request.create.into_valid() { - let from_document_id = id.document_id(); - if !from_node_ids.contains(from_document_id) { - response.not_created.append( - id, - SetError::not_found().with_description(format!( - "Item {} not found in account {}.", - id, response.from_account_id - )), - ); - continue; + let mut preload = MetadataPreload::default(); + for document_id in request + .create + .values() + .filter_map(|create| create.source_id(FileNodeProperty::Id)) + .map(|source_id| source_id.document_id()) + { + if let Some(resource) = from_cache.resources.find_any(document_id) + && from_readable + .as_ref() + .is_none_or(|readable| readable.contains(document_id)) + { + let kinds = resource.metadata_kinds(); + if viewer.is_some() || !kinds.is_empty() { + preload.insert_source(document_id, kinds); + } } + } + writer + .metadata + .preload(self, from_account_id, preload) + .await?; - let Some(source) = self - .store() - .get_value::>(ValueKey::archive( - from_account_id, - Collection::FileNode, - from_document_id, - )) - .await - .caused_by(trc::location!())? - else { + for (create_id, mut create) in request.create { + let source_id = match create.take_source_id(FileNodeProperty::Id) { + Ok(source_id) => source_id, + Err(err) => { + response.not_created.append(create_id, err); + continue; + } + }; + let from_document_id = source_id.document_id(); + let source = if is_readable(from_document_id) { + fetch_archive(self, from_account_id, from_document_id).await? + } else { + None + }; + let Some(source) = source else { response.not_created.append( - id, + create_id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", - id, response.from_account_id + source_id, response.from_account_id )), ); continue; }; - let mut file_node = source + let mut node = source .deserialize::() .caused_by(trc::location!())?; - // ACLs are account-scoped; do not carry the source account's grants over. - file_node.acls.clear(); + node.acls.clear(); + node.unsubscribed.clear(); + node.parent_id = 0; - let has_acl_changes = - match update_file_node(None, create, &mut file_node, true, &NoResolver) { - Ok(result) => { - if let Some(blob_id) = result.blob_id { - let file_details = file_node.file.get_or_insert_default(); - if !self.has_access_blob(&blob_id, access_token).await? { - response.not_created.append( - id, - SetError::forbidden().with_description(format!( - "You do not have access to blobId {blob_id}." - )), - ); - continue 'create; - } else if let Some(blob_contents) = self - .blob_store() - .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) - .await? - { - file_details.size = blob_contents.len() as u32; - } else { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(FileNodeProperty::BlobId) - .with_description("Blob could not be found."), - ); - continue 'create; - } - file_details.blob_hash = blob_id.hash; - } - - if file_node - .file - .as_ref() - .is_some_and(|f| f.blob_hash.is_empty()) - { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(FileNodeProperty::BlobId) - .with_description("Missing blob id."), - ); - continue 'create; - } - - result.has_acl_changes - } - Err(err) => { - response.not_created.append(id, err); - continue 'create; - } - }; - - if let Err(err) = - validate_file_node_hierarchy(None, &file_node, is_shared, &cache, &created_folders) + match create_node( + &mut writer, + String::new(), + node, + create, + &NoResolver, + Some(from_document_id), + ) + .await? { - response.not_created.append(id, err); - continue 'create; - } - - if file_node.modified == 0 { - file_node.modified = now() as i64; - } - - let renamed = match find_sibling_collision( - None, - &file_node, - &cache, - &pending_names, - case_insensitive, - ) { - Collision::None => false, - Collision::Existing(existing) => { - let effective = match on_exists { - OnExists::Newest => { - let existing_modified = - fetch_existing_modified(self.store(), account_id, existing).await?; - if file_node.modified > existing_modified { - OnExists::Replace - } else { - response.not_created.append( - id, - SetError::already_exists() - .with_existing_id(types::id::Id::from(existing)), - ); - continue 'create; - } - } - other => other, - }; - match effective { - OnExists::Reject => { - response.not_created.append( - id, - SetError::already_exists() - .with_existing_id(types::id::Id::from(existing)), - ); - continue 'create; - } - OnExists::Rename => { - file_node.name = pick_unique_rename( - &file_node.name, - None, - file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - OnExists::Replace => { - if let Some(target) = cache.any_resource_path_by_id(existing) { - let subtree_len = cache.subtree(target.path()).count(); - if subtree_len > 1 && !on_destroy_remove_children { - response - .not_created - .append(id, SetError::node_has_children()); - continue 'create; - } - } - implicit_destroys.insert(existing); - false - } - OnExists::Newest => unreachable!(), + Ok(node) => { + created.push((create_id, node)); + if on_success_delete { + destroy_ids.push(MaybeInvalid::Value(source_id)); } } - Collision::Pending => match on_exists { - OnExists::Rename => { - file_node.name = pick_unique_rename( - &file_node.name, - None, - file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - OnExists::Reject | OnExists::Replace | OnExists::Newest => { - let key = crate::file::set::pending_key(&file_node, case_insensitive); - let mut err = SetError::already_exists(); - if let Some(Some(doc_id)) = pending_names.get(&key) { - err = err.with_existing_id(types::id::Id::from(*doc_id)); - } - response.not_created.append(id, err); - continue 'create; - } + Err((_, rejection)) => match rejection.existing { + Some(slot) => awaiting_existing.push((create_id, rejection.error, slot)), + None => response.not_created.append(create_id, rejection.error), }, - }; + } + } - // Permission and ACL inheritance for the destination parent - if file_node.parent_id > 0 { - let parent_id = file_node.parent_id - 1; + let (destroyed, _) = writer.execute_destroys(Vec::new()).await?; - // The user must be allowed to add children to the destination parent - if let Some(allowed) = &can_add_to - && !created_folders.contains_key(&parent_id) - && !allowed.contains(parent_id) - { - response.not_created.append( - id, - SetError::forbidden().with_description( - "You are not allowed to create file nodes in this folder.", - ), - ); - continue 'create; - } - - let parent_acls = created_folders.get(&parent_id).cloned().or_else(|| { - cache - .container_resource_by_id(parent_id) - .and_then(|r| r.acls()) - .map(|a| a.to_vec()) - }); - if !has_acl_changes { - if let Some(parent_acls) = parent_acls { - file_node.acls = parent_acls; + if !writer.batch.is_empty() { + let assigned_ids = match writer.metadata.commit(self, writer.batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + for (create_id, _) in created { + response.not_created.append( + create_id, + SetError::forbidden().with_description( + "Another process modified this file node, please try again.", + ), + ); } - } else if is_shared - && parent_acls - .is_none_or(|acls| !acls.effective_acl(access_token).contains(Acl::Share)) - { - response.not_created.append( - id, - SetError::forbidden() - .with_description("You are not allowed to share this file node."), - ); - continue 'create; + for (id, err, _) in awaiting_existing { + response.not_created.append(id, err); + } + return Ok(response); } - } else if is_shared { - response.not_created.append( - id, - SetError::forbidden() - .with_description("Cannot create top-level folder in a shared account."), - ); - continue 'create; - } - - if !file_node.acls.is_empty() { - if let Err(err) = self.acl_validate(&file_node.acls).await { - response.not_created.append(id, err.into()); - continue 'create; - } - self.refresh_acls(&file_node.acls, None) - .await - .caused_by(trc::location!())?; - } - - let document_id = self - .store() - .assign_document_ids(account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; - if file_node.file.is_none() { - created_folders.insert(document_id, file_node.acls.clone()); - } - pending_names.insert( - crate::file::set::pending_key(&file_node, case_insensitive), - None, - ); - let final_name = file_node.name.clone(); - let set_created = file_node.created == 0; - let set_modified = file_node.modified == 0; - file_node - .insert( - access_token.account_tenant_ids(), - account_id, - document_id, - set_created, - set_modified, - &mut batch, - ) - .caused_by(trc::location!())?; - response.created(id, document_id); - if renamed - && let Some(value) = response.created.get_mut(&id) - && let jmap_tools::Value::Object(map) = value - { - map.insert_unchecked( - jmap_tools::Key::Property(FileNodeProperty::Name), - jmap_tools::Value::Str(std::borrow::Cow::Owned(final_name)), - ); - } - - if on_success_delete { - destroy_ids.push(MaybeInvalid::Value(id)); - } - } - - for did in &implicit_destroys { - let Some(node) = cache.any_resource_path_by_id(*did) else { - continue; + Err(err) => return Err(err.caused_by(trc::location!())), }; - let mut ids = cache.subtree(node.path()).collect::>(); - ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); - let sorted = ids.into_iter().map(|a| a.document_id()).collect::>(); - groupware::DestroyArchive(sorted) - .delete_batch( - self, - access_token.account_tenant_ids(), - account_id, - cache.format_resource(node).into(), - &mut batch, - ) - .await - .caused_by(trc::location!())?; - } - if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; - response.new_state = State::Exact(change_id); + for (id, err, slot) in awaiting_existing { + response + .not_created + .append(id, err.with_existing_id(Id::from(assigned_ids.slot(slot)))); + } + for (create_id, node) in created { + let mut values = node.values; + values.insert_unchecked( + Key::Property(FileNodeProperty::Id), + Value::Element(FileNodeValue::Id(Id::from(assigned_ids.slot(node.slot)))), + ); + response.created.append(create_id, Value::Object(values)); + } + + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::FileNode)); + if !destroyed.is_empty() { + response.destroyed = Some(Some(destroyed)); + } + } else { + for (id, err, _) in awaiting_existing { + response.not_created.append(id, err); + } } if on_success_delete && !destroy_ids.is_empty() { diff --git a/crates/jmap/src/file/get.rs b/crates/jmap/src/file/get.rs index 6846b803..50e85401 100644 --- a/crates/jmap/src/file/get.rs +++ b/crates/jmap/src/file/get.rs @@ -4,33 +4,51 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::acl::JmapRights, changes::state::JmapCacheState}; -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; -use groupware::{cache::GroupwareCache, file::FileNode}; +use super::node::{default_media_type, node_type_from_id, target_value}; +use crate::{ + api::{ + acl::JmapRights, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + changes::state::JmapCacheState, +}; +use common::{ + GroupwareResourceRef, Server, + auth::AccessToken, + storage::{ + batch::{DocumentBatch, Lookup, MAX_BATCH_BYTES, MAX_BATCH_DOCUMENTS}, + dav::{FILE_KIND_DIRECTORY, FILE_KIND_FILE, FILE_KIND_SYMLINK}, + }, +}; +use groupware::{ + cache::GroupwareCache, + file::{FileNode, FileNodeRole}, +}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, - object::file_node::{self, FileNodeNodeType, FileNodeProperty, FileNodeValue}, + method::get::{GetRequest, GetResponse, all_ids}, + object::file_node::{self, FileNodeProperty, FileNodeValue}, + request::capability::CapabilityIds, types::date::UTCDate, }; use jmap_tools::{Map, Value}; -use store::{ - ValueKey, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, now}, -}; +use store::roaring::RoaringBitmap; use trc::AddContext; use types::{ - acl::{Acl, AclGrant}, + acl::Acl, blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, + field::Field, + id::Id, }; +use utils::map::bitmap::Bitmap; pub trait FileNodeGet: Sync + Send { fn file_node_get( &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -39,271 +57,402 @@ impl FileNodeGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - FileNodeProperty::Id, - FileNodeProperty::ParentId, - FileNodeProperty::NodeType, - FileNodeProperty::BlobId, - FileNodeProperty::Target, - FileNodeProperty::Size, - FileNodeProperty::Name, - FileNodeProperty::Type, - FileNodeProperty::Created, - FileNodeProperty::Modified, - FileNodeProperty::Accessed, - FileNodeProperty::Changed, - FileNodeProperty::Executable, - FileNodeProperty::IsSubscribed, - FileNodeProperty::MyRights, - FileNodeProperty::ShareWith, - FileNodeProperty::Role, - ]); + let (mut properties, selection) = select_properties( + &mut request, + &[ + FileNodeProperty::Id, + FileNodeProperty::ParentId, + FileNodeProperty::Type, + FileNodeProperty::BlobId, + FileNodeProperty::Target, + FileNodeProperty::Size, + FileNodeProperty::Name, + FileNodeProperty::MediaType, + FileNodeProperty::Created, + FileNodeProperty::Modified, + FileNodeProperty::Accessed, + FileNodeProperty::Changed, + FileNodeProperty::Executable, + FileNodeProperty::IsSubscribed, + FileNodeProperty::MyRights, + FileNodeProperty::ShareWith, + FileNodeProperty::Role, + ], + using, + )?; + if !properties.contains(&FileNodeProperty::Type) { + properties.push(FileNodeProperty::Type); + } + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::FileNode); + let metadata_get = metadata.get(selection); + let metadata_slots = metadata_get.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); let account_id = request.account_id.document_id(); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await?; - // TODO: draft-14 section 5 case 2 - ancestors of shared nodes should be discoverable with mayRead=false - let file_node_ids = if access_token.is_member(account_id) { - cache - .resources - .iter() - .map(|r| r.document_id) - .collect::() - } else { - cache.shared_documents(access_token, [Acl::Read, Acl::ReadItems], true) + let is_owner = access_token.is_member(account_id); + let access = (!is_owner).then(|| cache.file_access(access_token)); + let is_visible = |document_id: u32| { + access + .as_ref() + .is_none_or(|access| access.discoverable.contains(document_id)) }; - let mut ids = if let Some(ids) = ids { - ids - } else { - file_node_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + let mut ids = match ids { + Some(ids) => ids, + None => match &access { + Some(access) => all_ids( + access.discoverable.iter().map(Id::from), + self.core.jmap.get_max_objects, + )?, + None => all_ids( + cache + .resources + .iter() + .map(|resource| Id::from(resource.document_id())), + self.core.jmap.get_max_objects, + )?, + }, }; if request.arguments.fetch_parents.unwrap_or(false) { - let mut seen: RoaringBitmap = ids.iter().map(|i| i.document_id()).collect(); - let mut extra: Vec = Vec::new(); - for id in &ids { + let mut seen = ids + .iter() + .map(|id| id.document_id()) + .collect::(); + let mut ancestors = Vec::new(); + for id in ids.iter().filter(|id| is_visible(id.document_id())) { let mut current = cache - .any_resource_path_by_id(id.document_id()) - .and_then(|r| r.parent_id()); + .resources + .find_any(id.document_id()) + .and_then(|resource| resource.parent_id()); while let Some(parent_id) = current { if !seen.insert(parent_id) { break; } - if file_node_ids.contains(parent_id) { - extra.push(parent_id.into()); + if is_visible(parent_id) { + ancestors.push(Id::from(parent_id)); } current = cache - .container_resource_by_id(parent_id) - .and_then(|r| r.parent_id()); + .resources + .find_any(parent_id) + .and_then(|resource| resource.parent_id()); } } - ids.extend(extra); + ids.extend(ancestors); } + + let needs_node_archive = properties.iter().any(|property| { + matches!( + property, + FileNodeProperty::BlobId + | FileNodeProperty::Target + | FileNodeProperty::Accessed + | FileNodeProperty::Changed + ) + }); + let needs_directory_archive = + needs_node_archive || properties.contains(&FileNodeProperty::IsSubscribed); + let needs_archive = |resource: &GroupwareResourceRef<'_>| { + if resource.is_container() { + needs_directory_archive + } else { + needs_node_archive + } + }; + let rights_access = access.as_ref().filter(|_| { + properties.iter().any(|property| { + matches!( + property, + FileNodeProperty::MyRights | FileNodeProperty::ShareWith + ) + }) + }); + let personal_id = access_token.personal_id(account_id, Collection::FileNode); + let is_readable = |document_id: u32| { + access + .as_ref() + .is_none_or(|access| access.readable.contains(document_id)) + }; + let mut metadata_values = match &metadata_get { + Some(get) => { + let mut documents = MetadataDocuments::default(); + for id in &ids { + let document_id = id.document_id(); + if let Some(resource) = cache + .resources + .find_any(document_id) + .filter(|_| is_visible(document_id) && is_readable(document_id)) + { + documents.insert(document_id, resource.metadata_kinds()); + } + } + Some(metadata.load(self, account_id, get, &documents).await?) + } + None => None, + }; let mut response = GetResponse { account_id: request.account_id.into(), - state: cache.get_state(false).into(), + state: sampled.state(cache.get_state(false)).into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; - for id in ids { - // Obtain the file_node object - let document_id = id.document_id(); - if !file_node_ids.contains(document_id) { - response.push_not_found(id); - continue; - } - let _file_node = if let Some(file_node) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::FileNode, - document_id, - )) - .await? - { - file_node - } else { - response.push_not_found(id); - continue; - }; - let file_node = _file_node - .unarchive::() - .caused_by(trc::location!())?; - let mut result = Map::with_capacity(properties.len()); - for property in &properties { - match property { - FileNodeProperty::Id => { - result.insert_unchecked(FileNodeProperty::Id, FileNodeValue::Id(id)); - } - FileNodeProperty::Name => { - result.insert_unchecked(FileNodeProperty::Name, file_node.name.to_string()); - } - FileNodeProperty::ShareWith => { - result.insert_unchecked( - FileNodeProperty::ShareWith, - JmapRights::share_with::( - account_id, - access_token, - &file_node - .acls - .iter() - .map(AclGrant::from) - .collect::>(), - ), - ); - } - FileNodeProperty::MyRights => { - result.insert_unchecked( - FileNodeProperty::MyRights, - if access_token.is_shared(account_id) { - JmapRights::rights::( - file_node.acls.effective_acl(access_token), - ) - } else { - JmapRights::all_rights::() - }, - ); - } - FileNodeProperty::ParentId => { - let parent_id = file_node.parent_id.to_native(); - - result.insert_unchecked( - FileNodeProperty::ParentId, - if parent_id > 0 { - Value::Element(FileNodeValue::Id((parent_id - 1).into())) - } else { - Value::Null - }, - ); - } - FileNodeProperty::BlobId => { - result.insert_unchecked( - FileNodeProperty::BlobId, - if let Some(file) = file_node.file.as_ref() { - Value::Element(FileNodeValue::BlobId(BlobId::new( - BlobHash::from(&file.blob_hash), - BlobClass::Linked { - account_id, - collection: Collection::FileNode.into(), - document_id: id.document_id(), - }, - ))) - } else { - Value::Null - }, - ); - } - FileNodeProperty::Size => { - result.insert_unchecked( - FileNodeProperty::Size, - if let Some(file) = file_node.file.as_ref() { - Value::Number(file.size.to_native().into()) - } else { - Value::Null - }, - ); - } - FileNodeProperty::Type => { - result.insert_unchecked( - FileNodeProperty::Type, - if let Some(file) = file_node.file.as_ref() { - Value::Str( - file.media_type - .as_ref() - .map(|t| t.to_string()) - .unwrap_or_else(|| "application/octet-stream".to_string()) - .into(), - ) - } else { - Value::Null - }, - ); - } - FileNodeProperty::Executable => { - result.insert_unchecked( - FileNodeProperty::Executable, - if let Some(file) = file_node.file.as_ref() { - Value::Bool(file.executable) - } else { - Value::Null - }, - ); - } - FileNodeProperty::Created => { - result.insert_unchecked( - FileNodeProperty::Created, - Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( - file_node.created.to_native(), - ))), - ); - } - FileNodeProperty::Modified => { - result.insert_unchecked( - FileNodeProperty::Modified, - Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( - file_node.modified.to_native(), - ))), - ); - } - FileNodeProperty::Accessed => { - // TODO: needs serialization change (per-user accessed timestamp); returns now() as a placeholder - result.insert_unchecked( - FileNodeProperty::Accessed, - Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( - now() as i64 - ))), - ); - } - FileNodeProperty::Changed => { - // TODO: needs serialization change (dedicated server-set changed timestamp); returns modified as a placeholder - result.insert_unchecked( - FileNodeProperty::Changed, - Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( - file_node.modified.to_native(), - ))), - ); - } - FileNodeProperty::NodeType => { - let node_type = if file_node.file.is_some() { - FileNodeNodeType::File - } else { - FileNodeNodeType::Directory - }; - result.insert_unchecked( - FileNodeProperty::NodeType, - Value::Str(node_type.as_str().into()), - ); - } - FileNodeProperty::Target => { - result.insert_unchecked(FileNodeProperty::Target, Value::Null); - } - FileNodeProperty::Role => { - result.insert_unchecked(FileNodeProperty::Role, Value::Null); - } - FileNodeProperty::IsSubscribed => { - // TODO: needs serialization change (per-user subscription state); always true for now - result.insert_unchecked(FileNodeProperty::IsSubscribed, Value::Bool(true)); - } - property => { - result.insert_unchecked(property.clone(), Value::Null); + let mut batch = DocumentBatch::new(MAX_BATCH_BYTES); + for chunk in ids.chunks(MAX_BATCH_DOCUMENTS) { + batch.reset(); + if needs_directory_archive { + for id in chunk { + let document_id = id.document_id(); + if is_visible(document_id) + && is_readable(document_id) + && cache + .resources + .find_any(document_id) + .is_some_and(|resource| needs_archive(&resource)) + { + batch.request(document_id); } } } - response.list.push(result.into()); + + loop { + self.read_archive_batch( + account_id, + Collection::FileNode, + Field::ARCHIVE, + &mut batch, + ) + .await?; + + for &id in chunk { + let document_id = id.document_id(); + let Some(resource) = cache + .resources + .find_any(document_id) + .filter(|_| is_visible(document_id)) + else { + if batch.is_first_round() { + response.push_not_found(id); + } + continue; + }; + if !is_readable(document_id) { + if batch.is_first_round() { + let mut result = discoverable_only( + &properties, + id, + &resource, + rights_access.map(|access| { + access.node_rights(document_id, resource.parent_id()) + }), + ); + if let Some(values) = &mut metadata_values { + values.insert_into(document_id, &mut result); + } + response.list.push(result.into()); + } + continue; + } + let archive = if needs_archive(&resource) { + match batch.lookup(document_id) { + Lookup::Found(archive) => Some(archive), + Lookup::Missing => { + response.push_not_found(id); + continue; + } + Lookup::Skip => continue, + } + } else if batch.is_first_round() { + None + } else { + continue; + }; + let node = archive + .map(|archive| archive.unarchive::()) + .transpose() + .caused_by(trc::location!())?; + let flags = resource.file_flags().unwrap_or_default(); + let kind = flags.kind(); + let is_file = kind == FILE_KIND_FILE; + let is_directory = kind == FILE_KIND_DIRECTORY; + let rights = rights_access + .filter(|_| is_directory) + .map(|access| access.node_rights(document_id, resource.parent_id())); + + let mut result = Map::with_capacity(properties.len() + metadata_slots); + for property in &properties { + let value = match property { + FileNodeProperty::Id => Value::Element(FileNodeValue::Id(id)), + FileNodeProperty::Name => Value::Str( + resource + .container_name() + .unwrap_or_default() + .to_string() + .into(), + ), + FileNodeProperty::ParentId => { + resource.parent_id().map_or(Value::Null, |parent_id| { + Value::Element(FileNodeValue::Id(parent_id.into())) + }) + } + FileNodeProperty::Type => { + Value::Str(node_type_from_id(kind).as_str().into()) + } + FileNodeProperty::BlobId if is_file => node + .and_then(|node| node.file()) + .map_or(Value::Null, |file| { + Value::Element(FileNodeValue::BlobId(BlobId::new( + BlobHash::from(&file.blob_hash), + BlobClass::Linked { + account_id, + collection: Collection::FileNode.into(), + document_id, + }, + ))) + }), + FileNodeProperty::Target if kind == FILE_KIND_SYMLINK => node + .and_then(|node| node.symlink_target()) + .map_or(Value::Null, target_value), + FileNodeProperty::Size if is_file => resource + .size() + .map_or(Value::Null, |size| Value::Number(size.into())), + FileNodeProperty::MediaType if is_file => Value::Str( + resource + .media_type() + .unwrap_or(default_media_type()) + .to_string() + .into(), + ), + FileNodeProperty::Executable if is_file => { + Value::Bool(flags.is_executable()) + } + FileNodeProperty::BlobId + | FileNodeProperty::Target + | FileNodeProperty::Size + | FileNodeProperty::MediaType + | FileNodeProperty::Executable => continue, + FileNodeProperty::Created => date_value(resource.created_at()), + FileNodeProperty::Modified => date_value(resource.modified_at()), + FileNodeProperty::Accessed => { + date_value(node.map(|node| node.accessed.to_native())) + } + FileNodeProperty::Changed => { + date_value(node.map(|node| node.changed.to_native())) + } + FileNodeProperty::IsSubscribed if is_directory => { + Value::Bool(node.is_none_or(|node| node.is_subscribed(personal_id))) + } + FileNodeProperty::Role if is_directory => { + FileNodeRole::from_id(flags.role()) + .map_or(Value::Null, |role| Value::Str(role.as_str().into())) + } + FileNodeProperty::MyRights if is_directory => my_rights(rights), + FileNodeProperty::ShareWith if is_directory => { + share_with(&resource, rights) + } + FileNodeProperty::IsSubscribed + | FileNodeProperty::Role + | FileNodeProperty::MyRights + | FileNodeProperty::ShareWith => continue, + property => { + result.insert_unchecked(property.clone(), Value::Null); + continue; + } + }; + result.insert_unchecked(property.clone(), value); + } + if let Some(values) = &mut metadata_values { + values.insert_into(document_id, &mut result); + } + response.list.push(result.into()); + } + + if !batch.next_round() { + break; + } + } } Ok(response) } } + +fn discoverable_only( + properties: &[FileNodeProperty], + id: Id, + resource: &GroupwareResourceRef<'_>, + rights: Option>, +) -> Map<'static, FileNodeProperty, FileNodeValue> { + let mut result = Map::with_capacity(properties.len()); + for property in properties { + let value = match property { + FileNodeProperty::Id => Value::Element(FileNodeValue::Id(id)), + FileNodeProperty::Name => Value::Str( + resource + .container_name() + .unwrap_or_default() + .to_string() + .into(), + ), + FileNodeProperty::ParentId => resource.parent_id().map_or(Value::Null, |parent_id| { + Value::Element(FileNodeValue::Id(parent_id.into())) + }), + FileNodeProperty::Type => Value::Str( + node_type_from_id(resource.file_kind().unwrap_or_default()) + .as_str() + .into(), + ), + FileNodeProperty::MyRights => my_rights(rights), + FileNodeProperty::IsSubscribed => Value::Bool(true), + FileNodeProperty::ShareWith => share_with(resource, rights), + FileNodeProperty::BlobId + | FileNodeProperty::Target + | FileNodeProperty::Size + | FileNodeProperty::MediaType + | FileNodeProperty::Executable => continue, + _ => Value::Null, + }; + result.insert_unchecked(property.clone(), value); + } + result +} + +fn share_with( + resource: &GroupwareResourceRef<'_>, + rights: Option>, +) -> Value<'static, FileNodeProperty, FileNodeValue> { + let acls = resource.acls(); + if !acls.is_empty() && rights.is_none_or(|rights| rights.contains(Acl::Share)) { + JmapRights::grants_value::(acls) + } else { + Value::Null + } +} + +pub(super) fn my_rights( + rights: Option>, +) -> Value<'static, FileNodeProperty, FileNodeValue> { + match rights { + Some(rights) => JmapRights::rights::(rights), + None => JmapRights::all_rights::(), + } +} + +pub(super) fn date_value( + timestamp: Option, +) -> Value<'static, FileNodeProperty, FileNodeValue> { + timestamp.map_or(Value::Null, |timestamp| { + Value::Element(FileNodeValue::Date(UTCDate::from_timestamp(timestamp))) + }) +} diff --git a/crates/jmap/src/file/mod.rs b/crates/jmap/src/file/mod.rs index a8757a97..9f8635a3 100644 --- a/crates/jmap/src/file/mod.rs +++ b/crates/jmap/src/file/mod.rs @@ -4,7 +4,120 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::{ + GroupwareResources, Server, + auth::AccountCache, + storage::quota::{DiskQuotaBudget, ObjectQuotaUsage}, +}; +use jmap_proto::{error::set::SetError, object::file_node::FileNodeProperty}; +use registry::schema::enums::StorageQuota; + pub mod copy; pub mod get; +pub mod node; pub mod query; pub mod set; +pub mod writer; + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct ObjectCounts { + pub files: usize, + pub folders: usize, +} + +pub(crate) struct FileNodeQuota { + files: ObjectQuotaUsage, + folders: ObjectQuotaUsage, + disk: DiskQuotaBudget, +} + +impl FileNodeQuota { + pub(crate) fn new( + server: &Server, + account: &AccountCache, + cache: &GroupwareResources, + disk: DiskQuotaBudget, + ) -> Self { + let files_limit = server.object_quota_limit(account, StorageQuota::MaxFiles); + let folders_limit = server.object_quota_limit(account, StorageQuota::MaxFolders); + if files_limit.is_none() && folders_limit.is_none() { + return Self { + disk, + ..Self::unlimited() + }; + } + + let folders = cache.resources.count(true); + let files = cache.resources.len().saturating_sub(folders); + let usage = |limit: Option, used| { + limit.map_or_else(ObjectQuotaUsage::unlimited, |limit| ObjectQuotaUsage { + used, + limit, + }) + }; + Self { + files: usage(files_limit, files), + folders: usage(folders_limit, folders), + disk, + } + } + + pub(crate) fn unlimited() -> Self { + Self { + files: ObjectQuotaUsage::unlimited(), + folders: ObjectQuotaUsage::unlimited(), + disk: DiskQuotaBudget::unlimited(), + } + } + + pub(crate) fn validate( + &self, + is_folder: bool, + created: ObjectCounts, + destroyed: ObjectCounts, + ) -> Result<(), SetError> { + let (usage, pending, released, description) = if is_folder { + ( + &self.folders, + created.folders, + destroyed.folders, + "There are too many folders, please delete some before adding a new one.", + ) + } else { + ( + &self.files, + created.files, + destroyed.files, + "There are too many files, please delete some before adding a new one.", + ) + }; + if (ObjectQuotaUsage { + used: usage.used.saturating_sub(released), + limit: usage.limit, + }) + .has_room(pending) + { + Ok(()) + } else { + Err(SetError::over_quota().with_description(description)) + } + } + + pub(crate) fn validate_disk(&self, bytes: u64) -> Result<(), SetError> { + self.disk.check(bytes).map_err(|event| { + SetError::over_quota().with_description(match event { + _ => "The disk quota has been exceeded.", + }) + }) + } + + #[inline(always)] + pub(crate) fn reserve_disk(&mut self, bytes: u64) { + self.disk.reserve(bytes); + } + + #[inline(always)] + pub(crate) fn release_disk(&mut self, bytes: u64) { + self.disk.release(bytes); + } +} diff --git a/crates/jmap/src/file/node.rs b/crates/jmap/src/file/node.rs new file mode 100644 index 00000000..8c505c44 --- /dev/null +++ b/crates/jmap/src/file/node.rs @@ -0,0 +1,575 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::api::{acl::JmapRights, parent_ref::ParentRef}; +use common::storage::dav::{ + FILE_KIND_DIRECTORY, FILE_KIND_FILE, FILE_KIND_SYMLINK, FORBIDDEN_FILE_NAME_CHARS, + MAX_DAV_FILE_NAME_LEN, +}; +use groupware::file::{ + FileNode, FileNodeContent, FileNodeRole, FileProperties, + symlink::{SymlinkTargetBuilder, symlink_target_elements}, +}; +use jmap_proto::{ + error::set::SetError, + object::{ + AnyId, + file_node::{self, FileNodeNodeType, FileNodeProperty, FileNodeValue}, + }, + references::resolve::ResolveCreatedReference, +}; +use jmap_tools::{JsonPointerItem, Key, Value}; +use std::borrow::Cow; +use store::write::now; +use types::{acl::AclGrant, blob::BlobId, blob_hash::BlobHash, id::Id, media_type}; + +pub(crate) const MAX_NAME_LEN: usize = MAX_DAV_FILE_NAME_LEN; +const MEDIA_TYPE_OCTET_STREAM: &str = "application/octet-stream"; + +#[derive(Debug, Default)] +pub(super) struct NodePatch { + pub name: Option, + pub parent: Option, + pub kind: Option, + pub blob_id: Option>, + pub target: Option>, + pub size: Option, + pub media_type: Option>, + pub executable: Option>, + pub created: Option, + pub modified: Option, + pub accessed: Option, + pub role: Option>, + pub subscribed: Option>, + pub acls: Option>>, + pub my_rights: Option>, + pub server_created: bool, + pub server_modified: bool, + pub server_accessed: bool, +} + +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct PatchEffects { + pub renamed: bool, + pub moved: bool, + pub content: bool, + pub acls: bool, + pub role: bool, + pub subscription: bool, +} + +impl PatchEffects { + pub fn changes_node(&self) -> bool { + self.renamed || self.moved || self.content || self.acls || self.role + } +} + +pub(super) struct ResolvedBlob { + pub hash: BlobHash, + pub size: u32, +} + +impl NodePatch { + pub fn parse>( + expected_id: Option, + updates: Value<'_, FileNodeProperty, FileNodeValue>, + node: &FileNode, + is_create: bool, + resolver: &R, + ) -> Result> { + let mut patch = NodePatch::default(); + + for (property, mut value) in updates.into_expanded_object() { + let Key::Property(property) = property else { + return Err(SetError::invalid_properties() + .with_property(property.to_owned()) + .with_description("Invalid property.")); + }; + + resolver.resolve_self_references(&mut value, 0, false)?; + + match (property, value) { + (FileNodeProperty::Name, Value::Str(value)) => { + validate_name(&value)?; + patch.name = Some(value.into_owned()); + } + (FileNodeProperty::ParentId, Value::Element(FileNodeValue::Id(value))) => { + patch.parent = Some(ParentRef::from_id(value)); + } + (FileNodeProperty::ParentId, Value::Null) => { + patch.parent = Some(ParentRef::ROOT); + } + (FileNodeProperty::Type, Value::Str(value)) => { + let kind = FileNodeNodeType::parse(&value) + .map(node_type_id) + .ok_or_else(|| invalid(FileNodeProperty::Type, "Unsupported @type."))?; + if !is_create && kind != node.kind_id() { + return Err(invalid( + FileNodeProperty::Type, + "@type is immutable after creation.", + )); + } + patch.kind = Some(kind); + } + (FileNodeProperty::BlobId, Value::Element(FileNodeValue::BlobId(value))) => { + patch.blob_id = Some(Some(value)); + } + (FileNodeProperty::BlobId, Value::Null) => { + patch.blob_id = Some(None); + } + (FileNodeProperty::Target, Value::Array(elements)) => { + let mut builder = SymlinkTargetBuilder::with_capacity( + elements + .iter() + .map(|e| e.as_str().map_or(0, |s| s.len() + 1)) + .sum(), + ); + for element in elements { + let Value::Str(element) = element else { + return Err(invalid( + FileNodeProperty::Target, + "target elements must be strings.", + )); + }; + builder + .push(&element) + .map_err(|err| invalid(FileNodeProperty::Target, err.description()))?; + } + patch.target = + Some(Some(builder.build().map_err(|err| { + invalid(FileNodeProperty::Target, err.description()) + })?)); + } + (FileNodeProperty::Target, Value::Null) => { + patch.target = Some(None); + } + (FileNodeProperty::Size, Value::Number(value)) + if let Some(size) = value.as_u64() => + { + patch.size = Some(size); + } + (FileNodeProperty::Size, Value::Null) => {} + (FileNodeProperty::MediaType, Value::Str(value)) => { + if !media_type::is_valid_media_type(&value) { + return Err(invalid( + FileNodeProperty::MediaType, + "mediaType is not a valid media type.", + )); + } + patch.media_type = Some(Some(value.into_owned())); + } + (FileNodeProperty::MediaType, Value::Null) => { + patch.media_type = Some(None); + } + (FileNodeProperty::Executable, Value::Bool(value)) => { + patch.executable = Some(Some(value)); + } + (FileNodeProperty::Executable, Value::Null) => { + patch.executable = Some(None); + } + (FileNodeProperty::Created, Value::Element(FileNodeValue::Date(value))) => { + patch.created = Some(value.timestamp()); + } + (FileNodeProperty::Created, Value::Null) => { + patch.created = Some(now() as i64); + patch.server_created = true; + } + (FileNodeProperty::Modified, Value::Element(FileNodeValue::Date(value))) => { + patch.modified = Some(value.timestamp()); + } + (FileNodeProperty::Modified, Value::Null) => { + patch.modified = Some(now() as i64); + patch.server_modified = true; + } + (FileNodeProperty::Accessed, Value::Element(FileNodeValue::Date(value))) => { + patch.accessed = Some(value.timestamp()); + } + (FileNodeProperty::Accessed, Value::Null) => { + patch.accessed = Some(now() as i64); + patch.server_accessed = true; + } + (FileNodeProperty::Role, Value::Str(value)) => { + patch.role = + Some(Some(FileNodeRole::parse(&value).ok_or_else(|| { + invalid(FileNodeProperty::Role, "Unsupported role.") + })?)); + } + (FileNodeProperty::Role, Value::Null) => { + patch.role = Some(None); + } + (FileNodeProperty::IsSubscribed, Value::Bool(value)) => { + patch.subscribed = Some(Some(value)); + } + (FileNodeProperty::IsSubscribed, Value::Null) => { + patch.subscribed = Some(None); + } + (FileNodeProperty::ShareWith, Value::Null) => { + patch.acls = Some(None); + } + (FileNodeProperty::ShareWith, value) => { + patch.acls = Some(Some(JmapRights::acl_set::(value)?)); + } + (FileNodeProperty::Pointer(pointer), value) + if matches!( + pointer.first(), + Some(JsonPointerItem::Key(Key::Property( + FileNodeProperty::ShareWith + ))) + ) => + { + let mut pointer = pointer.iter(); + pointer.next(); + let current = match patch.acls.take() { + Some(acls) => acls.unwrap_or_default(), + None => node.acls.clone(), + }; + patch.acls = Some(Some(JmapRights::acl_patch::( + current, pointer, value, + )?)); + } + (FileNodeProperty::Id, value) => { + if !expected_id.is_some_and(|expected| crate::matches_id(&value, expected)) { + return Err(invalid( + FileNodeProperty::Id, + "The id property is immutable.", + )); + } + } + (FileNodeProperty::Changed, Value::Element(FileNodeValue::Date(value))) + if !is_create && value.timestamp() == node.changed => {} + (FileNodeProperty::Changed, _) => { + return Err(invalid( + FileNodeProperty::Changed, + "changed is server-set and not settable by clients.", + )); + } + (FileNodeProperty::MyRights, value @ Value::Object(_)) if !is_create => { + patch.my_rights = Some(value.into_owned()); + } + (FileNodeProperty::MyRights, Value::Null) => { + patch.my_rights = Some(Value::Null); + } + (property, _) => { + return Err(invalid(property, "Field could not be set.")); + } + } + } + + Ok(patch) + } + + pub fn echoes_type(&self) -> bool { + match &self.media_type { + Some(Some(media_type)) => { + media_type::media_type_essence(media_type).as_deref() != Some(media_type.as_str()) + } + Some(None) | None => true, + } + } + + pub fn blob_to_resolve(&self, node: &FileNode) -> Option<&BlobId> { + self.blob_id + .as_ref() + .and_then(Option::as_ref) + .filter(|blob_id| { + self.kind.unwrap_or_else(|| node.kind_id()) == FILE_KIND_FILE + && (blob_id.section.is_some() + || node + .file() + .is_none_or(|file| file.blob_hash != blob_id.hash)) + }) + } + + pub fn apply( + self, + node: &mut FileNode, + blob: Option, + personal_id: u32, + ) -> Result> { + let mut effects = PatchEffects::default(); + let kind = self.kind.unwrap_or_else(|| node.kind_id()); + + if kind != FILE_KIND_FILE { + if matches!(self.blob_id, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::BlobId, + "blobId is only defined for File nodes.", + )); + } else if self.size.is_some() { + return Err(invalid( + FileNodeProperty::Size, + "size is only defined for File nodes.", + )); + } else if matches!(self.media_type, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::MediaType, + "mediaType is only defined for File nodes.", + )); + } else if matches!(self.executable, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::Executable, + "executable is only defined for File nodes.", + )); + } + } + if kind != FILE_KIND_SYMLINK && matches!(self.target, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::Target, + "target is only defined for SymLink nodes.", + )); + } + if kind != FILE_KIND_DIRECTORY { + if matches!(self.role, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::Role, + "role is only defined for Directory nodes.", + )); + } else if matches!(self.subscribed, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::IsSubscribed, + "isSubscribed is only defined for Directory nodes.", + )); + } else if matches!(self.acls, Some(Some(_))) { + return Err(invalid( + FileNodeProperty::ShareWith, + "shareWith is only defined for Directory nodes.", + )); + } else if self + .my_rights + .as_ref() + .is_some_and(|value| !matches!(value, Value::Null)) + { + return Err(invalid( + FileNodeProperty::MyRights, + "myRights is only defined for Directory nodes.", + )); + } + } else if matches!(self.my_rights, Some(Value::Null)) { + return Err(invalid( + FileNodeProperty::MyRights, + "myRights is server-set and not settable by clients.", + )); + } + + match kind { + FILE_KIND_FILE => { + if matches!(self.blob_id, Some(None)) { + return Err(invalid( + FileNodeProperty::BlobId, + "blobId cannot be null for File nodes.", + )); + } + let mut file = match std::mem::take(&mut node.content) { + FileNodeContent::File(file) => file, + _ => FileProperties::default(), + }; + if let Some(blob) = blob { + file.blob_hash = blob.hash; + file.size = blob.size; + effects.content = true; + } else if file.blob_hash.is_empty() { + return Err(invalid( + FileNodeProperty::BlobId, + "blobId is required for File nodes.", + )); + } + if let Some(size) = self.size + && size != file.size as u64 + { + return Err(invalid( + FileNodeProperty::Size, + "size does not match the size of the blob.", + )); + } + if let Some(media_type) = self.media_type { + effects.content |= file.media_type != media_type; + file.media_type = media_type; + } + if let Some(executable) = self.executable { + let executable = executable.unwrap_or(false); + effects.content |= file.executable != executable; + file.executable = executable; + } + node.content = FileNodeContent::File(file); + } + FILE_KIND_SYMLINK => match self.target { + Some(Some(target)) => { + effects.content |= node.symlink_target() != Some(target.as_str()); + node.content = FileNodeContent::Symlink(target); + } + Some(None) => { + return Err(invalid( + FileNodeProperty::Target, + "target cannot be null for SymLink nodes.", + )); + } + None if node.symlink_target().is_none() => { + return Err(invalid( + FileNodeProperty::Target, + "target is required for SymLink nodes.", + )); + } + None => {} + }, + _ => { + node.content = FileNodeContent::Directory; + if let Some(role) = self.role { + effects.role = node.role() != role; + node.set_role(role); + } + } + } + + if let Some(name) = self.name + && name != node.name + { + node.name = name; + effects.renamed = true; + } + if let Some(parent) = self.parent { + effects.moved = parent != ParentRef::from_stored(node.parent_id) || parent.is_pending(); + node.parent_id = parent.as_stored(); + } + if let Some(created) = self.created { + effects.content |= node.created != created; + node.created = created; + } + if let Some(modified) = self.modified { + effects.content |= node.modified != modified; + node.modified = modified; + } + if let Some(accessed) = self.accessed { + effects.content |= node.accessed != accessed; + node.accessed = accessed; + } + if kind == FILE_KIND_DIRECTORY { + if let Some(subscribed) = self.subscribed.map(|subscribed| subscribed.unwrap_or(true)) + && node.is_subscribed(personal_id) != subscribed + { + node.set_subscribed(personal_id, subscribed); + effects.subscription = true; + } + if let Some(acls) = self.acls { + let acls = acls.unwrap_or_default(); + effects.acls = node.acls != acls; + node.acls = acls; + } + } + + if node.name.is_empty() { + return Err(invalid(FileNodeProperty::Name, "Missing name.")); + } + + Ok(effects) + } +} + +pub(crate) fn validate_name(name: &str) -> Result<(), SetError> { + if !(1..=MAX_NAME_LEN).contains(&name.len()) { + Err(invalid( + FileNodeProperty::Name, + "Name must be between 1 and 255 octets.", + )) + } else if name.contains(|c: char| FORBIDDEN_FILE_NAME_CHARS.contains(c) || c.is_control()) { + Err(invalid( + FileNodeProperty::Name, + "Name contains a forbidden character.", + )) + } else if is_reserved_name(name) { + Err(invalid( + FileNodeProperty::Name, + "Name is reserved and cannot be used.", + )) + } else { + Ok(()) + } +} + +fn is_reserved_name(name: &str) -> bool { + let stem = match name.split_once('.') { + Some((stem, _)) if !stem.is_empty() => stem, + _ => name, + }; + is_reserved_stem(name) || is_reserved_stem(stem) +} + +fn is_reserved_stem(name: &str) -> bool { + hashify::set_ignore_case!( + name.as_bytes(), + ".", + "..", + "CON", + "PRN", + "AUX", + "NUL", + "COM0", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT0", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", + ) +} + +pub(super) fn node_type_id(node_type: FileNodeNodeType) -> u8 { + match node_type { + FileNodeNodeType::File => FILE_KIND_FILE, + FileNodeNodeType::Directory => FILE_KIND_DIRECTORY, + FileNodeNodeType::SymLink => FILE_KIND_SYMLINK, + } +} + +pub(super) fn node_type_from_id(kind: u8) -> FileNodeNodeType { + match kind { + FILE_KIND_FILE => FileNodeNodeType::File, + FILE_KIND_SYMLINK => FileNodeNodeType::SymLink, + _ => FileNodeNodeType::Directory, + } +} + +pub(super) fn target_value(target: &str) -> Value<'static, FileNodeProperty, FileNodeValue> { + Value::Array( + symlink_target_elements(target) + .map(|element| Value::Str(Cow::Owned(element.to_string()))) + .collect(), + ) +} + +pub(super) fn default_media_type() -> &'static str { + MEDIA_TYPE_OCTET_STREAM +} + +pub(super) fn invalid( + property: FileNodeProperty, + description: &'static str, +) -> SetError { + SetError::invalid_properties() + .with_property(property) + .with_description(description) +} + +pub(super) struct NoResolver; + +impl ResolveCreatedReference for NoResolver { + fn get_created_id(&self, _: &str) -> Option { + None + } +} + diff --git a/crates/jmap/src/file/query.rs b/crates/jmap/src/file/query.rs index bff5cc15..2b17acc8 100644 --- a/crates/jmap/src/file/query.rs +++ b/crates/jmap/src/file/query.rs @@ -4,272 +4,388 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; -use common::{Server, auth::AccessToken}; -use groupware::cache::GroupwareCache; +use super::node::node_type_id; +use crate::{ + api::{ + metadata::{MetadataType, ObjectMetadata, ResourceScope, flagged_documents}, + query::QueryResponseBuilder, + }, + changes::state::JmapCacheState, +}; +use common::{ + GroupwareResourceRef, GroupwareResources, Server, + auth::AccessToken, + storage::dav::{FILE_KIND_DIRECTORY, FILE_KIND_FILE}, +}; +use groupware::{ + cache::GroupwareCache, + file::{FileNode, FileNodeRole}, +}; use jmap_proto::{ - method::query::{Filter, QueryRequest, QueryResponse}, - object::file_node::{FileNode, FileNodeComparator, FileNodeFilter}, - request::MaybeInvalid, + method::query::{Comparator, Filter, QueryRequest, QueryResponse}, + object::file_node::{ + FileNode as FileNodeObject, FileNodeComparator, FileNodeFilter, FileNodeNodeType, + }, + request::{MaybeInvalid, capability::CapabilityIds}, }; +use std::cmp::Ordering; +use store::IterateParams; use store::{ - ahash::AHashMap, + U32_LEN, ValueKey, roaring::RoaringBitmap, - search::{SearchFilter, SearchQuery}, - write::SearchIndex, + search::{FileSearchField, SearchFilter, SearchQuery}, + write::{BlobLink, BlobOp, SearchIndex, ValueClass, key::DeserializeBigEndian}, }; -use types::{acl::Acl, collection::SyncCollection}; +use trc::AddContext; +use types::{ + blob_hash::BLOB_HASH_LEN, + collection::{Collection, SyncCollection}, + field::Field, + id::Id, + media_type::MediaTypeId, +}; +use utils::glob::GlobPattern; + +const MAX_TREE_DEPTH: usize = 256; pub trait FileNodeQuery: Sync + Send { fn file_node_query( &self, - request: QueryRequest, + request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } impl FileNodeQuery for Server { async fn file_node_query( &self, - mut request: QueryRequest, + mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); - let mut filters = Vec::with_capacity(request.filter.len()); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::FileNode); + let sampled = metadata.viewer_change_id(self, account_id).await?; + let metadata_query = + metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(FileNodeFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await?; + let depth = request.arguments.depth.unwrap_or(0) as usize; + let fts_enabled = self + .core + .email + .index_fields + .contains_key(&SearchIndex::File); + let mut uses_fts = false; + let mut filters = Vec::with_capacity(request.filter.len()); + let (mask, hidden) = if access_token.is_member(account_id) { + (all_ids(&cache), None) + } else { + let access = cache.file_access(access_token); + let hidden = &access.discoverable - &access.readable; + (access.discoverable, Some(hidden)) + }; + let is_visible = |id: Id| mask.contains(id.document_id()); + let readable_only = |set: RoaringBitmap| match &hidden { + Some(hidden) => set - hidden, + None => set, + }; + let mut metadata_leaves = metadata + .evaluate(self, account_id, &metadata_query, || { + let readable = hidden.as_ref().map(|hidden| &mask - hidden); + flagged_documents(&cache, ResourceScope::All, readable.as_ref()) + }) + .await? + .into_iter(); for cond in std::mem::take(&mut request.filter) { match cond { - Filter::Property(cond) => match cond { - FileNodeFilter::AncestorId(MaybeInvalid::Value(id)) => { - if let Some(resource) = - cache.container_resource_path_by_id(id.document_id()) - { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.subtree(resource.path()).map(|r| r.document_id()), - ))) - } else { - filters.push(SearchFilter::is_in_set(RoaringBitmap::new())); - } - } - FileNodeFilter::DescendantId(MaybeInvalid::Value(id)) => { - let mut ancestors = RoaringBitmap::new(); - let mut current = cache - .any_resource_path_by_id(id.document_id()) - .and_then(|r| r.parent_id()); - while let Some(parent_id) = current { - if !ancestors.insert(parent_id) { - break; + Filter::Property(cond) => { + let set = match cond { + FileNodeFilter::IsTopLevel(is_top_level) => { + let within = if depth == 0 { + matching(&cache, |r| r.parent_id().is_none()) + } else { + cache + .tree_with_depth(depth) + .map(|resource| resource.document_id()) + .collect() + }; + if is_top_level { + within + } else { + all_ids(&cache) - within } - current = cache - .container_resource_by_id(parent_id) + } + FileNodeFilter::ParentId(MaybeInvalid::Value(id)) if is_visible(id) => { + descendants(&cache, id.document_id(), depth + 1) + } + FileNodeFilter::AncestorId(MaybeInvalid::Value(id)) if is_visible(id) => { + descendants(&cache, id.document_id(), usize::MAX) + } + FileNodeFilter::DescendantId(MaybeInvalid::Value(id)) if is_visible(id) => { + let mut ancestors = RoaringBitmap::new(); + let mut current = cache + .resources + .find_any(id.document_id()) .and_then(|r| r.parent_id()); - } - filters.push(SearchFilter::is_in_set(ancestors)); - } - FileNodeFilter::ParentId(MaybeInvalid::Value(id)) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.children_ids(id.document_id()), - ))); - } - FileNodeFilter::IsTopLevel(is_top_level) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - if is_top_level == r.parent_id().is_none() { - Some(r.document_id) - } else { - None + while let Some(parent_id) = current { + if !ancestors.insert(parent_id) { + break; } - }), - ))); - } - FileNodeFilter::NodeType(node_type) => { - let want_container = match node_type.as_str() { - "directory" => Some(true), - "file" => Some(false), - _ => None, - }; - let set = match want_container { - Some(is_container) => { - RoaringBitmap::from_iter(cache.resources.iter().filter_map(|r| { - if r.is_container() == is_container { - Some(r.document_id) - } else { - None - } - })) + current = cache + .resources + .find_any(parent_id) + .and_then(|r| r.parent_id()); } - // TODO: support symlink nodeType once target storage exists + ancestors + } + FileNodeFilter::NodeType(node_type) => { + match FileNodeNodeType::parse(&node_type).map(node_type_id) { + Some(kind) => matching(&cache, |r| r.file_kind() == Some(kind)), + None => RoaringBitmap::new(), + } + } + FileNodeFilter::Role(role) => match FileNodeRole::parse(&role) { + Some(role) => readable_only(matching(&cache, |r| { + is_directory(r) && r.file_role() == role.id() + })), None => RoaringBitmap::new(), - }; - filters.push(SearchFilter::is_in_set(set)); - } - FileNodeFilter::Name(name) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - if r.container_name().is_some_and(|n| n == name) { - Some(r.document_id) - } else { - None - } - }), - ))); - } - FileNodeFilter::NameMatch(name) => { - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - if r.container_name().is_some_and(|n| name.matches(n)) { - Some(r.document_id) - } else { - None - } - }), - ))); - } - FileNodeFilter::MinSize(size) => { - let size = size as u32; - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - if r.size().is_some_and(|s| s >= size) { - Some(r.document_id) - } else { - None - } - }), - ))); - } - FileNodeFilter::MaxSize(size) => { - let size = size as u32; - filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( - cache.resources.iter().filter_map(|r| { - if r.size().is_some_and(|s| s <= size) { - Some(r.document_id) - } else { - None - } - }), - ))); - } - // TODO: filters below require fetching archives or new indexes; ignore for now - FileNodeFilter::Role(_) - | FileNodeFilter::HasAnyRole(_) - | FileNodeFilter::BlobId(_) - | FileNodeFilter::IsExecutable(_) - | FileNodeFilter::CreatedBefore(_) - | FileNodeFilter::CreatedAfter(_) - | FileNodeFilter::ModifiedBefore(_) - | FileNodeFilter::ModifiedAfter(_) - | FileNodeFilter::AccessedBefore(_) - | FileNodeFilter::AccessedAfter(_) - | FileNodeFilter::Type(_) - | FileNodeFilter::TypeMatch(_) - | FileNodeFilter::Text(_) - | FileNodeFilter::Body(_) - | FileNodeFilter::AncestorId(_) - | FileNodeFilter::DescendantId(_) - | FileNodeFilter::ParentId(_) - | FileNodeFilter::_T(_) => {} - }, - Filter::And => { - filters.push(SearchFilter::And); - } - Filter::Or => { - filters.push(SearchFilter::Or); - } - Filter::Not => { - filters.push(SearchFilter::Not); - } - Filter::Close => { - filters.push(SearchFilter::End); + }, + FileNodeFilter::HasAnyRole(true) => readable_only(matching(&cache, |r| { + is_directory(r) && r.file_role() != 0 + })), + FileNodeFilter::HasAnyRole(false) => readable_only(matching(&cache, |r| { + is_directory(r) && r.file_role() == 0 + })), + FileNodeFilter::BlobId(MaybeInvalid::Value(blob_id)) => { + let mut ids = RoaringBitmap::new(); + let collection = u8::from(Collection::FileNode); + let key = |document_id| ValueKey { + account_id, + collection, + document_id, + class: ValueClass::Blob(BlobOp::Link { + hash: blob_id.hash.clone(), + to: BlobLink::Document, + }), + }; + self.store() + .iterate( + IterateParams::new(key(0), key(u32::MAX)).no_values(), + |key, _| { + ids.insert( + key.deserialize_be_u32(BLOB_HASH_LEN + U32_LEN + 1)?, + ); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + readable_only(ids) + } + FileNodeFilter::ParentId(_) + | FileNodeFilter::AncestorId(_) + | FileNodeFilter::DescendantId(_) + | FileNodeFilter::BlobId(_) => RoaringBitmap::new(), + FileNodeFilter::IsExecutable(true) => { + readable_only(matching(&cache, |r| is_file(r) && r.is_executable())) + } + FileNodeFilter::IsExecutable(false) => { + matching(&cache, |r| is_file(r) && !r.is_executable()) + } + FileNodeFilter::CreatedBefore(date) => { + let date = date.timestamp(); + readable_only(matching(&cache, |r| { + r.created_at().is_some_and(|c| c < date) + })) + } + FileNodeFilter::CreatedAfter(date) => { + let date = date.timestamp(); + readable_only(matching(&cache, |r| { + r.created_at().is_some_and(|c| c >= date) + })) + } + FileNodeFilter::ModifiedBefore(date) => { + let date = date.timestamp(); + readable_only(matching(&cache, |r| { + r.modified_at().is_some_and(|m| m < date) + })) + } + FileNodeFilter::ModifiedAfter(date) => { + let date = date.timestamp(); + readable_only(matching(&cache, |r| { + r.modified_at().is_some_and(|m| m >= date) + })) + } + FileNodeFilter::AccessedBefore(date) => { + let date = date.timestamp(); + readable_only( + accessed_matching(self, account_id, |accessed| accessed < date) + .await?, + ) + } + FileNodeFilter::AccessedAfter(date) => { + let date = date.timestamp(); + readable_only( + accessed_matching(self, account_id, |accessed| accessed >= date) + .await?, + ) + } + FileNodeFilter::MinSize(size) => readable_only(matching(&cache, |r| { + r.size().is_some_and(|s| s as u64 >= size) + })), + FileNodeFilter::MaxSize(size) => readable_only(matching(&cache, |r| { + r.size().is_some_and(|s| (s as u64) < size) + })), + FileNodeFilter::Name(name) => { + matching(&cache, |r| r.container_name() == Some(name.as_str())) + } + FileNodeFilter::NameMatch(pattern) => name_matching(&cache, &pattern), + FileNodeFilter::MediaType(media_type) => { + readable_only(matching(&cache, |r| { + file_media_type(r) == Some(media_type.as_str()) + })) + } + FileNodeFilter::MediaTypeMatch(pattern) => { + readable_only(type_matching(&cache, &pattern)) + } + FileNodeFilter::Text(text) => { + let pattern = GlobPattern::compile(&text, true); + let set = name_matching(&cache, &pattern) + | readable_only(type_matching(&cache, &pattern)); + if fts_enabled { + uses_fts = true; + filters.push(SearchFilter::Or); + filters.extend(readable_text_filter( + SearchFilter::has_text_detect( + FileSearchField::Content, + text, + self.core.email.default_language, + ), + hidden.as_ref().map(|hidden| &mask - hidden), + )); + filters.extend([SearchFilter::is_in_set(set), SearchFilter::End]); + continue; + } + set + } + FileNodeFilter::Body(text) if fts_enabled => { + uses_fts = true; + filters.extend(readable_text_filter( + SearchFilter::has_text_detect( + FileSearchField::Content, + text, + self.core.email.default_language, + ), + hidden.as_ref().map(|hidden| &mask - hidden), + )); + continue; + } + FileNodeFilter::Metadata(_) => { + readable_only(metadata_leaves.next().unwrap_or_default()) + } + unsupported @ (FileNodeFilter::Body(_) | FileNodeFilter::_T(_)) => { + return Err(trc::JmapEvent::UnsupportedFilter + .into_err() + .details(unsupported.into_string())); + } + }; + filters.push(SearchFilter::is_in_set(set)); } + Filter::And => filters.push(SearchFilter::And), + Filter::Or => filters.push(SearchFilter::Or), + Filter::Not => filters.push(SearchFilter::Not), + Filter::Close => filters.push(SearchFilter::End), } } - let results = SearchQuery::new(SearchIndex::InMemory) - .with_filters(filters) - .with_mask(if access_token.is_shared(account_id) { - cache.shared_documents(access_token, [Acl::Read, Acl::ReadItems], true) - } else { - cache.resources.iter().map(|r| r.document_id).collect() - }) - .filter() - .into_bitmap(); + let results = if uses_fts { + self.search_store() + .filter_account( + SearchQuery::new(SearchIndex::File) + .with_filters(filters) + .with_account_id(account_id) + .with_mask(mask), + ) + .await? + } else { + SearchQuery::new(SearchIndex::InMemory) + .with_filters(filters) + .with_mask(mask) + .filter() + .into_bitmap() + }; let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, - cache.get_state(false), + sampled.state(cache.get_state(false)), &request, ); - // Only name, size and nodeType can be sorted from the cache. - // TODO: created/modified/type/tree sorts require archive or hierarchy traversal - let sortable = request - .sort - .as_deref() - .unwrap_or_default() - .iter() - .filter(|c| { - matches!( - c.property, - FileNodeComparator::Name - | FileNodeComparator::Size - | FileNodeComparator::NodeType - ) - }) - .collect::>(); + let comparators = request.sort.take().unwrap_or_default(); + let mut collations = Vec::with_capacity(comparators.len()); + for comparator in &comparators { + if matches!(comparator.property, FileNodeComparator::_T(_)) { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(comparator.property.as_str().to_string())); + } + match Collation::parse(comparator.collation.as_deref()) { + Some(collation) => collations.push(collation), + None => { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(comparator.collation.clone().unwrap_or_default())); + } + } + } - if sortable.is_empty() { + if comparators.is_empty() { for document_id in results { if !response.add(0, document_id) { break; } } } else { - let by_id = cache - .resources + let has_tree = comparators .iter() - .map(|r| (r.document_id, r)) - .collect::>(); - let mut ids = results.iter().collect::>(); - ids.sort_unstable_by(|a, b| { - for cmp in &sortable { - let ra = by_id.get(a); - let rb = by_id.get(b); - let ordering = match cmp.property { - FileNodeComparator::Name => ra - .and_then(|r| r.container_name()) - .cmp(&rb.and_then(|r| r.container_name())), - FileNodeComparator::Size => { - ra.and_then(|r| r.size()).cmp(&rb.and_then(|r| r.size())) - } - FileNodeComparator::NodeType => { - // Directories sort before files - let a_dir = ra.map(|r| r.is_container()).unwrap_or(false); - let b_dir = rb.map(|r| r.is_container()).unwrap_or(false); - b_dir.cmp(&a_dir) - } - _ => std::cmp::Ordering::Equal, - }; - let ordering = if cmp.is_ascending { - ordering + .any(|comparator| matches!(comparator.property, FileNodeComparator::Tree)); + let mut entries = results + .iter() + .filter_map(|document_id| cache.resources.find_any(document_id)) + .map(|resource| SortEntry { + readable: hidden + .as_ref() + .is_none_or(|hidden| !hidden.contains(resource.document_id())), + tree: if has_tree { + tree_path(&cache, &resource) } else { - ordering.reverse() - }; - if ordering != std::cmp::Ordering::Equal { - return ordering; - } - } - a.cmp(b) + Vec::new() + }, + resource, + }) + .collect::>(); + entries.sort_unstable_by(|a, b| { + comparators + .iter() + .zip(collations.iter()) + .map(|(comparator, collation)| compare(comparator, *collation, a, b)) + .find(|ordering| *ordering != Ordering::Equal) + .unwrap_or_else(|| a.resource.document_id().cmp(&b.resource.document_id())) }); - for document_id in ids { - if !response.add(0, document_id) { + for entry in entries { + if !response.add(0, entry.resource.document_id()) { break; } } @@ -278,3 +394,284 @@ impl FileNodeQuery for Server { response.build() } } + +struct SortEntry<'x> { + resource: GroupwareResourceRef<'x>, + readable: bool, + tree: Vec<&'x str>, +} + +impl<'x> SortEntry<'x> { + fn readable(&self, value: impl FnOnce(&GroupwareResourceRef<'x>) -> Option) -> Option { + self.readable.then(|| value(&self.resource)).flatten() + } +} + +fn readable_text_filter( + text: SearchFilter, + readable: Option, +) -> impl Iterator { + match readable { + Some(readable) => vec![ + SearchFilter::And, + text, + SearchFilter::is_in_set(readable), + SearchFilter::End, + ], + None => vec![text], + } + .into_iter() +} + +async fn accessed_matching( + server: &Server, + account_id: u32, + predicate: impl Fn(i64) -> bool + Send + Sync, +) -> trc::Result { + let mut ids = RoaringBitmap::new(); + server + .archives( + account_id, + Collection::FileNode, + Field::ARCHIVE, + &(), + |document_id, archive| { + if predicate(archive.unarchive::()?.accessed.to_native()) { + ids.insert(document_id); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(ids) +} + +fn compare( + comparator: &Comparator, + collation: Collation, + a: &SortEntry<'_>, + b: &SortEntry<'_>, +) -> Ordering { + let (ra, rb) = (&a.resource, &b.resource); + let ordering = match comparator.property { + FileNodeComparator::Name => collation.compare( + ra.container_name().unwrap_or_default(), + rb.container_name().unwrap_or_default(), + ), + FileNodeComparator::Size => a + .readable(GroupwareResourceRef::size) + .cmp(&b.readable(GroupwareResourceRef::size)), + FileNodeComparator::Created => a + .readable(GroupwareResourceRef::created_at) + .cmp(&b.readable(GroupwareResourceRef::created_at)), + FileNodeComparator::Modified => a + .readable(GroupwareResourceRef::modified_at) + .cmp(&b.readable(GroupwareResourceRef::modified_at)), + FileNodeComparator::NodeType => kind_rank(ra).cmp(&kind_rank(rb)), + FileNodeComparator::MediaType => a + .readable(file_media_type) + .cmp(&b.readable(file_media_type)), + FileNodeComparator::Tree => { + return a + .tree + .iter() + .zip(b.tree.iter()) + .map(|(x, y)| collation.compare(x, y).then_with(|| x.cmp(y))) + .find(|ordering| *ordering != Ordering::Equal) + .map(|ordering| { + if comparator.is_ascending { + ordering + } else { + ordering.reverse() + } + }) + .unwrap_or_else(|| a.tree.len().cmp(&b.tree.len())); + } + FileNodeComparator::_T(_) => Ordering::Equal, + }; + if comparator.is_ascending { + ordering + } else { + ordering.reverse() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Collation { + Octet, + AsciiCasemap, + UnicodeCasemap, + AsciiNumeric, +} + +impl Collation { + fn parse(collation: Option<&str>) -> Option { + match collation { + None => Some(Collation::UnicodeCasemap), + Some(collation) => hashify::map!(collation.as_bytes(), Collation, + "i;octet" => Collation::Octet, + "i;ascii-casemap" => Collation::AsciiCasemap, + "i;unicode-casemap" => Collation::UnicodeCasemap, + "i;ascii-numeric" => Collation::AsciiNumeric, + ) + .copied(), + } + } + + fn compare(self, a: &str, b: &str) -> Ordering { + match self { + Collation::Octet => a.cmp(b), + Collation::AsciiCasemap => a + .bytes() + .map(|c| c.to_ascii_lowercase()) + .cmp(b.bytes().map(|c| c.to_ascii_lowercase())), + Collation::UnicodeCasemap if a.is_ascii() && b.is_ascii() => { + Collation::AsciiCasemap.compare(a, b) + } + Collation::UnicodeCasemap => a + .chars() + .flat_map(char::to_lowercase) + .cmp(b.chars().flat_map(char::to_lowercase)), + Collation::AsciiNumeric => match (leading_number(a), leading_number(b)) { + (Some(a), Some(b)) => a.len().cmp(&b.len()).then_with(|| a.cmp(b)), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + }, + } + } +} + +fn leading_number(value: &str) -> Option<&str> { + let digits = value + .split(|c: char| !c.is_ascii_digit()) + .next() + .filter(|digits| !digits.is_empty())?; + Some(digits.trim_start_matches('0')) +} + +fn kind_rank(resource: &GroupwareResourceRef<'_>) -> u8 { + match resource.file_kind() { + Some(FILE_KIND_DIRECTORY) => 0, + Some(FILE_KIND_FILE) => 2, + _ => 1, + } +} + +fn tree_path<'x>( + cache: &'x GroupwareResources, + resource: &GroupwareResourceRef<'x>, +) -> Vec<&'x str> { + let mut path = Vec::with_capacity(8); + let mut current = Some(*resource); + while let Some(node) = current { + if path.len() >= MAX_TREE_DEPTH { + break; + } + path.push(node.container_name().unwrap_or_default()); + current = node + .parent_id() + .and_then(|parent_id| cache.resources.find_any(parent_id)); + } + path.reverse(); + path +} + +fn is_file(resource: &GroupwareResourceRef<'_>) -> bool { + resource.file_kind() == Some(FILE_KIND_FILE) +} + +fn is_directory(resource: &GroupwareResourceRef<'_>) -> bool { + resource.file_kind() == Some(FILE_KIND_DIRECTORY) +} + +fn file_media_type<'x>(resource: &GroupwareResourceRef<'x>) -> Option<&'x str> { + if is_file(resource) { + resource.media_type() + } else { + None + } +} + +fn all_ids(cache: &GroupwareResources) -> RoaringBitmap { + cache.resources.iter().map(|r| r.document_id()).collect() +} + +fn matching( + cache: &GroupwareResources, + predicate: impl Fn(&GroupwareResourceRef<'_>) -> bool, +) -> RoaringBitmap { + cache + .resources + .iter() + .filter(|r| predicate(r)) + .map(|r| r.document_id()) + .collect() +} + +fn name_matching(cache: &GroupwareResources, pattern: &GlobPattern) -> RoaringBitmap { + matching(cache, |r| { + r.container_name().is_some_and(|n| pattern.matches(n)) + }) +} + +fn type_matching(cache: &GroupwareResources, pattern: &GlobPattern) -> RoaringBitmap { + const UNKNOWN: u8 = 0; + const MATCHES: u8 = 1; + const DIFFERS: u8 = 2; + + let mut states = vec![UNKNOWN; MediaTypeId::UNCATALOGUED.raw() as usize]; + let mut ids = RoaringBitmap::new(); + for resource in cache.resources.iter() { + let Some(flags) = resource + .file_flags() + .filter(|flags| flags.kind() == FILE_KIND_FILE) + else { + continue; + }; + let id = flags.media_type_id(); + let is_match = match states.get_mut(id.raw() as usize) { + Some(state) => { + if *state == UNKNOWN { + *state = if resource + .media_type() + .is_some_and(|name| pattern.matches(name)) + { + MATCHES + } else { + DIFFERS + }; + } + *state == MATCHES + } + None => resource + .media_type() + .is_some_and(|name| pattern.matches(name)), + }; + if is_match { + ids.insert(resource.document_id()); + } + } + ids +} + +fn descendants(cache: &GroupwareResources, document_id: u32, max_depth: usize) -> RoaringBitmap { + let Some(root) = cache.container_resource_path_by_id(document_id) else { + return RoaringBitmap::new(); + }; + let path = root.path(); + let iter: Box> = if max_depth == usize::MAX { + Box::new(cache.subtree(path).map(|r| r.document_id())) + } else { + Box::new( + cache + .subtree_with_depth(path, max_depth) + .map(|r| r.document_id()), + ) + }; + let mut ids = iter.collect::(); + ids.remove(document_id); + ids +} + diff --git a/crates/jmap/src/file/set.rs b/crates/jmap/src/file/set.rs index 0b3d2827..f95e3df6 100644 --- a/crates/jmap/src/file/set.rs +++ b/crates/jmap/src/file/set.rs @@ -4,45 +4,53 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{ + get::{date_value, my_rights}, + node::{NodePatch, PatchEffects, default_media_type, invalid}, + writer::{Claim, FileNodeWriter, NodeRef, Placement, Rejection, WriteOptions, fetch_archive}, +}; use crate::{ - api::acl::{JmapAcl, JmapRights}, - blob::download::BlobDownload, + api::{ + metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, NewMetadata, + ObjectMetadata, PreparedMetadata, is_empty_update, reject_uncommitted, + }, + parent_ref::{CreateResolver, ParentRef}, + }, changes::state::JmapCacheState, }; -use common::{DavResourceMetadata, DavResources, Server, auth::AccessToken, sharing::EffectiveAcl}; -use groupware::{DestroyArchive, cache::GroupwareCache, file::FileNode}; +use common::{ + Server, + auth::AccessToken, + storage::dav::{FILE_KIND_DIRECTORY, FilePresence}, +}; +use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::{ AnyId, - file_node::{self, FileNodeProperty, FileNodeValue, OnExists}, + file_node::{self, FileNodeProperty, FileNodeValue}, }, references::resolve::ResolveCreatedReference, - request::MaybeInvalid, + request::{MaybeInvalid, capability::CapabilityIds}, types::state::State, }; -use jmap_tools::{JsonPointerItem, Key, Value}; +use jmap_tools::{Key, Map, Value}; +use std::borrow::Cow; use store::{ - ValueKey, ahash::{AHashMap, AHashSet}, - write::{AlignedBytes, Archive, BatchBuilder, now}, + write::{Archive, ArchiveBytes, Slot}, }; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, - blob::BlobId, collection::{Collection, SyncCollection}, id::Id, + media_type::media_type_essence, }; - -const FORBIDDEN_NAME_CHARS: &str = "/<>:\"\\|?*"; -const FORBIDDEN_NODE_NAMES: &[&str] = &[ - ".", "..", "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", - "COM7", "COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", - "LPT9", -]; +use utils::map::bitmap::Bitmap; pub trait FileNodeSet: Sync + Send { fn file_node_set( @@ -50,982 +58,1037 @@ pub trait FileNodeSet: Sync + Send { request: SetRequest<'_, file_node::FileNode>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; } +pub(super) type NodeValues = Map<'static, FileNodeProperty, FileNodeValue>; + +pub(super) struct PreparedCreate { + create_id: String, + slot: Slot, + node: FileNode, + parent: ParentRef, + omitted: Omitted, + metadata: Option, +} + +struct PreparedUpdate { + id: Id, + archive: Archive, + node: FileNode, + parent: ParentRef, + previous: (ParentRef, String), + effects: PatchEffects, + growth: u64, + report_size: bool, + echo: Omitted, + metadata: Option, +} + +enum Planned { + Create(PreparedCreate), + Update(PreparedUpdate), +} + +#[derive(Debug, Default, Clone, Copy)] +struct Omitted { + media_type: bool, + executable: bool, + created: bool, + modified: bool, + accessed: bool, + subscribed: bool, +} + +impl Omitted { + fn is_empty(&self) -> bool { + !(self.media_type + || self.executable + || self.created + || self.modified + || self.accessed + || self.subscribed) + } + + fn insert_into(&self, node: &FileNode, values: &mut NodeValues) { + if let Some(file) = node.file() { + if self.media_type { + values.insert_unchecked( + Key::Property(FileNodeProperty::MediaType), + Value::Str(normalized_media_type(file.media_type.as_deref())), + ); + } + if self.executable { + values.insert_unchecked( + Key::Property(FileNodeProperty::Executable), + Value::Bool(file.executable), + ); + } + } + for (is_echoed, property, timestamp) in [ + (self.created, FileNodeProperty::Created, node.created), + (self.modified, FileNodeProperty::Modified, node.modified), + (self.accessed, FileNodeProperty::Accessed, node.accessed), + ] { + if is_echoed { + values.insert_unchecked(Key::Property(property), date_value(Some(timestamp))); + } + } + if self.subscribed { + values.insert_unchecked( + Key::Property(FileNodeProperty::IsSubscribed), + Value::Bool(true), + ); + } + } +} + +pub(super) struct CreatedNode { + pub create_id: String, + pub slot: Slot, + pub values: NodeValues, +} + +enum RejectedKey { + Create(String), + Update(Id), +} + impl FileNodeSet for Server { async fn file_node_set( &self, mut request: SetRequest<'_, file_node::FileNode>, access_token: &AccessToken, _session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let metadata = ObjectMetadata::new(self, access_token, using, MetadataType::FileNode); + let sampled = metadata.viewer_change_id(self, account_id).await?; let cache = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::FileNode, ) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(false, &request.if_in_state)?); - let mut will_destroy = response.collect_will_destroy(request.unwrap_destroy()); - let is_shared = access_token.is_shared(account_id); - let on_destroy_remove_children = request - .arguments - .on_destroy_remove_children - .unwrap_or(false); - let on_exists = request.arguments.on_exists; - let case_insensitive = request - .arguments - .compare_case_insensitively - .unwrap_or(false); - let mut pending_names: AHashMap<(u32, String), Option> = AHashMap::new(); - let mut implicit_destroys: AHashSet = AHashSet::new(); + .with_state(sampled.assert_state(cache.get_state(false), &request.if_in_state)?); + let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + let options = WriteOptions { + on_exists: request.arguments.on_exists, + remove_children: request + .arguments + .on_destroy_remove_children + .unwrap_or(false), + case_insensitive: request + .arguments + .compare_case_insensitively + .unwrap_or(false), + }; + let mut writer = + FileNodeWriter::new(self, access_token, account_id, &cache, options, metadata); + if request.has_creates() + || request.update.as_ref().is_some_and(|updates| { + updates.values().any(|object| { + object.as_object().is_some_and(|object| { + object.contains_key(&Key::Property(FileNodeProperty::BlobId)) + }) + }) + }) + { + writer = writer.with_quota().await?; + } - // Process creates - let mut batch = BatchBuilder::new(); - let mut created_folders = AHashMap::new(); - 'create: for (id, object) in request.unwrap_create() { - let mut file_node = FileNode::default(); - - // Process changes - let has_acl_changes = - match update_file_node(None, object, &mut file_node, true, &response) { - Ok(result) => { - if let Some(blob_id) = result.blob_id { - let file_details = file_node.file.get_or_insert_default(); - if !self.has_access_blob(&blob_id, access_token).await? { - response.not_created.append( - id, - SetError::forbidden().with_description(format!( - "You do not have access to blobId {blob_id}." - )), - ); - continue; - } else if let Some(blob_contents) = self - .blob_store() - .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) - .await? - { - file_details.size = blob_contents.len() as u32; - } else { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(FileNodeProperty::BlobId) - .with_description("Blob could not be found."), - ); - continue 'create; - } - - file_details.blob_hash = blob_id.hash; - } - - // Validate blob hash - if file_node - .file - .as_ref() - .is_some_and(|f| f.blob_hash.is_empty()) + let moving = request + .update + .as_ref() + .map(|updates| { + updates + .iter() + .filter_map(|(id, object)| match id { + MaybeInvalid::Value(id) + if object.as_object().is_some_and(|object| { + object.contains_key(&Key::Property(FileNodeProperty::ParentId)) + }) => { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(FileNodeProperty::BlobId) - .with_description("Missing blob id."), - ); - continue 'create; + Some(id.document_id()) } + _ => None, + }) + .collect::>() + }) + .unwrap_or_default(); + let (destroy_groups, not_destroyed) = writer.plan_destroys(will_destroy, &moving); + for (id, err) in not_destroyed { + response.not_destroyed.append(id, err); + } - result.has_acl_changes - } - Err(err) => { - response.not_created.append(id, err); - continue 'create; - } - }; - - // Validate hierarchy - if let Err(err) = - validate_file_node_hierarchy(None, &file_node, is_shared, &cache, &created_folders) - { - response.not_created.append(id, err); - continue 'create; - } - - if file_node.modified == 0 { - file_node.modified = now() as i64; - } - - let renamed = match find_sibling_collision( + let mut planned = Vec::new(); + let mut pending_creates: AHashMap = AHashMap::new(); + for (create_id, object) in request.unwrap_create() { + let resolver = CreateResolver::new(&pending_creates); + match prepare_create( + &mut writer, + create_id, + FileNode::default(), + object, + &resolver, None, - &file_node, - &cache, - &pending_names, - case_insensitive, - ) { - Collision::None => false, - Collision::Existing(existing) => { - let effective = match on_exists { - OnExists::Newest => { - let existing_modified = - fetch_existing_modified(self.store(), account_id, existing).await?; - if file_node.modified > existing_modified { - OnExists::Replace - } else { - response.not_created.append( - id, - SetError::already_exists().with_existing_id(Id::from(existing)), - ); - continue 'create; - } - } - other => other, - }; - match effective { - OnExists::Reject => { - response.not_created.append( - id, - SetError::already_exists().with_existing_id(Id::from(existing)), - ); - continue 'create; - } - OnExists::Rename => { - file_node.name = pick_unique_rename( - &file_node.name, - None, - file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - OnExists::Replace => { - if let Some(target) = cache.any_resource_path_by_id(existing) { - let subtree_len = cache.subtree(target.path()).count(); - if subtree_len > 1 && !on_destroy_remove_children { - response - .not_created - .append(id, SetError::node_has_children()); - continue 'create; - } - } - implicit_destroys.insert(existing); - false - } - OnExists::Newest => unreachable!(), - } + ) + .await? + { + Ok(create) => { + pending_creates.insert(create.create_id.clone(), create.slot); + planned.push(Planned::Create(create)); } - Collision::Pending => match on_exists { - OnExists::Reject | OnExists::Replace | OnExists::Newest => { - let key = pending_key(&file_node, case_insensitive); - let mut err = SetError::already_exists(); - if let Some(Some(doc_id)) = pending_names.get(&key) { - err = err.with_existing_id(Id::from(*doc_id)); - } - response.not_created.append(id, err); - continue 'create; - } - OnExists::Rename => { - file_node.name = pick_unique_rename( - &file_node.name, - None, - file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - }, - }; - - // Inherit ACLs from parent - if file_node.parent_id > 0 { - let parent_id = file_node.parent_id - 1; - let parent_acls = created_folders.get(&parent_id).cloned().or_else(|| { - cache - .container_resource_by_id(parent_id) - .and_then(|r| r.acls()) - .map(|a| a.to_vec()) - }); - if !has_acl_changes { - if let Some(parent_acls) = parent_acls { - file_node.acls = parent_acls; - } - } else if is_shared - && parent_acls - .is_none_or(|acls| !acls.effective_acl(access_token).contains(Acl::Share)) - { - response.not_created.append( - id, - SetError::forbidden() - .with_description("You are not allowed to share this file node."), - ); - continue 'create; - } - } - - // Validate ACLs - if !file_node.acls.is_empty() { - if let Err(err) = self.acl_validate(&file_node.acls).await { - response.not_created.append(id, err.into()); - continue 'create; - } - - self.refresh_acls(&file_node.acls, None) - .await - .caused_by(trc::location!())?; - } - - // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::FileNode, 1) - .await - .caused_by(trc::location!())?; - if file_node.file.is_none() { - created_folders.insert(document_id, file_node.acls.clone()); - } - let final_name = file_node.name.clone(); - pending_names.insert(pending_key(&file_node, case_insensitive), None); - let set_created = file_node.created == 0; - let set_modified = file_node.modified == 0; - file_node - .insert( - access_token.account_tenant_ids(), - account_id, - document_id, - set_created, - set_modified, - &mut batch, - ) - .caused_by(trc::location!())?; - let create_id = id.clone(); - response.created(id, document_id); - if renamed && let Some(Value::Object(map)) = response.created.get_mut(&create_id) { - map.insert_unchecked( - Key::Property(FileNodeProperty::Name), - Value::Str(std::borrow::Cow::Owned(final_name)), - ); + Err((create_id, err)) => response.not_created.append(create_id, err), } } - // Process updates - 'update: for (id, object) in request.unwrap_update() { + writer + .metadata + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .resources + .find_any(document_id) + .map(|resource| resource.metadata_kinds()) + }) + .await?; + for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { response.not_updated.append(invalid, SetError::not_found()); - continue 'update; + continue; } }; - // Make sure id won't be destroyed - if will_destroy.contains(&id) || implicit_destroys.contains(&id.document_id()) { - response.not_updated.append(id, SetError::will_destroy()); - continue 'update; - } - - // Obtain file node - let document_id = id.document_id(); - let file_node_ = if let Some(file_node_) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::FileNode, - document_id, - )) - .await? + let patches = match writer + .metadata + .extract(MetadataPatches::for_update(), &mut object) { - file_node_ - } else { - response.not_updated.append(id, SetError::not_found()); - continue 'update; - }; - let file_node = file_node_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_file_node = file_node - .deserialize::() - .caused_by(trc::location!())?; - - // Apply changes - let (has_acl_changes, modified_set) = - match update_file_node(Some(id), object, &mut new_file_node, false, &response) { - Ok(result) => { - let modified_set = result.modified_set; - if let Some(blob_id) = result.blob_id { - let file_details = new_file_node.file.get_or_insert_default(); - if !self.has_access_blob(&blob_id, access_token).await? { - response.not_updated.append( - id, - SetError::forbidden().with_description(format!( - "You do not have access to blobId {blob_id}." - )), - ); - continue; - } else if let Some(blob_contents) = self - .blob_store() - .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) - .await? - { - file_details.size = blob_contents.len() as u32; - } else { - response.not_updated.append( - id, - SetError::invalid_properties() - .with_property(FileNodeProperty::BlobId) - .with_description("Blob could not be found."), - ); - continue 'update; - } - - file_details.blob_hash = blob_id.hash; - } - - (result.has_acl_changes, modified_set) - } - Err(err) => { - response.not_updated.append(id, err); - continue 'update; - } - }; - - // Validate hierarchy - if let Err(err) = validate_file_node_hierarchy( - Some(document_id), - &new_file_node, - is_shared, - &cache, - &created_folders, - ) { - response.not_updated.append(id, err); - continue 'update; - } - - let renamed = match find_sibling_collision( - Some(document_id), - &new_file_node, - &cache, - &pending_names, - case_insensitive, - ) { - Collision::None => false, - Collision::Existing(existing) => { - let effective = match on_exists { - OnExists::Newest => { - let existing_modified = - fetch_existing_modified(self.store(), account_id, existing).await?; - if new_file_node.modified > existing_modified { - OnExists::Replace - } else { - response.not_updated.append( - id, - SetError::already_exists().with_existing_id(Id::from(existing)), - ); - continue 'update; - } - } - other => other, - }; - match effective { - OnExists::Reject => { - response.not_updated.append( - id, - SetError::already_exists().with_existing_id(Id::from(existing)), - ); - continue 'update; - } - OnExists::Rename => { - new_file_node.name = pick_unique_rename( - &new_file_node.name, - Some(document_id), - new_file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - OnExists::Replace => { - if let Some(target) = cache.any_resource_path_by_id(existing) { - let subtree_len = cache.subtree(target.path()).count(); - if subtree_len > 1 && !on_destroy_remove_children { - response - .not_updated - .append(id, SetError::node_has_children()); - continue 'update; - } - } - implicit_destroys.insert(existing); - false - } - OnExists::Newest => unreachable!(), - } + Ok(patches) => patches, + Err(err) => { + response.not_updated.append(id, err); + continue; } - Collision::Pending => match on_exists { - OnExists::Reject | OnExists::Replace | OnExists::Newest => { - let key = pending_key(&new_file_node, case_insensitive); - let mut err = SetError::already_exists(); - if let Some(Some(doc_id)) = pending_names.get(&key) { - err = err.with_existing_id(Id::from(*doc_id)); - } - response.not_updated.append(id, err); - continue 'update; - } - OnExists::Rename => { - new_file_node.name = pick_unique_rename( - &new_file_node.name, - Some(document_id), - new_file_node.parent_id, - &cache, - &pending_names, - case_insensitive, - ); - true - } - }, }; - - // Validate ACL - if is_shared { - let acl = file_node.inner.acls.effective_acl(access_token); - if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { - response.not_updated.append( - id, - SetError::forbidden() - .with_description("You are not allowed to modify this file node."), - ); - continue 'update; - } + let resolver = CreateResolver::new(&pending_creates); + match prepare_update(&mut writer, id, object, patches, &resolver).await? { + Ok(update) => planned.push(Planned::Update(update)), + Err(err) => response.not_updated.append(id, err), } - if has_acl_changes { - if let Err(err) = self.acl_validate(&new_file_node.acls).await { - response.not_updated.append(id, err.into()); - continue 'update; - } - self.refresh_acls( - &new_file_node.acls, - Some( - file_node - .inner - .acls - .iter() - .map(AclGrant::from) - .collect::>() - .as_slice(), - ), - ) - .await - .caused_by(trc::location!())?; - } - - let final_name = new_file_node.name.clone(); - pending_names.insert( - pending_key(&new_file_node, case_insensitive), - Some(document_id), - ); - // Update record. Bump modified to now() unless the client supplied a value. - new_file_node - .update( - access_token.account_tenant_ids(), - file_node, - account_id, - document_id, - !modified_set, - &mut batch, - ) - .caused_by(trc::location!())?; - let updated_value = if renamed { - let mut map = jmap_tools::Map::with_capacity(1); - map.insert_unchecked( - Key::Property(FileNodeProperty::Name), - Value::Str(std::borrow::Cow::Owned(final_name)), - ); - Some(Value::Object(map)) - } else { - None - }; - response.updated.append(id, updated_value); } - // Process deletions - for did in &implicit_destroys { - let id = Id::from(*did); - if !will_destroy.contains(&id) { - will_destroy.push(id); - } - } - let mut destroy_ids = AHashSet::with_capacity(will_destroy.len()); - 'destroy: for id in will_destroy { - let document_id = id.document_id(); - - let Some(file_node) = cache.any_resource_path_by_id(document_id) else { - response.not_destroyed.append(id, SetError::not_found()); - continue 'destroy; - }; - - // Find ids to delete - let mut ids = cache.subtree(file_node.path()).collect::>(); - if ids.is_empty() { - debug_assert!(false, "Resource found in cache but not in subtree"); - continue 'destroy; - } - - // Sort ids descending from the deepest to the root - ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); - let mut sorted_ids = Vec::with_capacity(ids.len()); - sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); - - // Validate not already deleted - for child_id in &sorted_ids { - if !destroy_ids.insert(*child_id) { - response.not_destroyed.append( - id, - SetError::will_destroy().with_description( - "File node or one of its children is already marked for deletion.", - ), - ); - continue 'destroy; + let mut created = Vec::new(); + let mut awaiting_existing = Vec::new(); + let mut unprocessed = planned + .iter() + .map(Planned::node_ref) + .collect::>(); + let mut allow_defer = true; + while !planned.is_empty() { + let pending = planned.len(); + let mut deferred = Vec::new(); + for mut item in planned { + let node_ref = item.node_ref(); + match item.place(&mut writer, allow_defer.then_some(&unprocessed)) { + Claim::Deferred => deferred.push(item), + Claim::Rejected(rejection) => { + unprocessed.remove(&node_ref); + let key = item.reject(&mut writer); + match rejection.existing { + Some(slot) => awaiting_existing.push((key, rejection.error, slot)), + None => key.append(&mut response, rejection.error), + } + } + Claim::Accepted { renamed } => { + unprocessed.remove(&node_ref); + match item { + Planned::Create(create) => { + created.push(commit_create(&mut writer, create, renamed)?); + } + Planned::Update(update) => { + let id = update.id; + let values = commit_update(&mut writer, update, renamed)?; + response.updated.append(id, values); + } + } + } } } - - // Validate ACLs - if !access_token.is_member(account_id) { - let permissions = cache.shared_containers(access_token, [Acl::Delete], false); - if permissions.len() < sorted_ids.len() as u64 - || !sorted_ids.iter().all(|id| permissions.contains(*id)) + if deferred.len() == pending { + for index in resolve_name_cycles(&mut writer, &deferred) + .into_iter() + .rev() { - response.not_destroyed.append( - id, - SetError::forbidden() - .with_description("You are not allowed to delete this file node."), - ); - continue 'destroy; + if let Planned::Update(update) = deferred.remove(index) { + unprocessed.remove(&NodeRef::Node(update.id.document_id())); + let id = update.id; + let values = commit_update(&mut writer, update, false)?; + response.updated.append(id, values); + } } } - - // Obtain children ids - if sorted_ids.len() > 1 && !on_destroy_remove_children { - response - .not_destroyed - .append(id, SetError::node_has_children()); - continue 'destroy; - } - - // Delete record - response - .destroyed - .extend(sorted_ids.iter().copied().map(Id::from)); - - DestroyArchive(sorted_ids) - .delete_batch( - self, - access_token.account_tenant_ids(), - account_id, - cache.format_resource(file_node).into(), - &mut batch, - ) - .await?; + allow_defer = deferred.len() < pending; + planned = deferred; } - // Write changes - if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; + let (destroyed, not_destroyed) = writer.execute_destroys(destroy_groups).await?; + response.destroyed.extend(destroyed); + for (id, err) in not_destroyed { + response.not_destroyed.append(id, err); + } + writer.finish(); - response.new_state = State::Exact(change_id).into(); + if !writer.batch.is_empty() { + let assigned_ids = match writer.metadata.commit(self, writer.batch).await { + Ok(assigned_ids) => assigned_ids, + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut response, + created.into_iter().map(|create| create.create_id), + "Another process modified this file node, please try again.", + ); + for (key, err, _) in awaiting_existing { + key.append(&mut response, err); + } + return Ok(response); + } + Err(err) => return Err(err.caused_by(trc::location!())), + }; + + for (key, err, slot) in awaiting_existing { + key.append( + &mut response, + err.with_existing_id(Id::from(assigned_ids.slot(slot))), + ); + } + for create in created { + let mut values = create.values; + values.insert_unchecked( + Key::Property(FileNodeProperty::Id), + Value::Element(FileNodeValue::Id(Id::from(assigned_ids.slot(create.slot)))), + ); + response + .created + .insert(create.create_id, Value::Object(values)); + } + + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::FileNode)) + .into(); + } else { + for (key, err, _) in awaiting_existing { + key.append(&mut response, err); + } } Ok(response) } } -pub(super) struct UpdateResult { - pub(super) has_acl_changes: bool, - pub(super) blob_id: Option, - pub(super) modified_set: bool, -} +impl Planned { + fn node_ref(&self) -> NodeRef { + match self { + Planned::Create(create) => NodeRef::Pending(create.slot), + Planned::Update(update) => NodeRef::Node(update.id.document_id()), + } + } -pub(super) struct NoResolver; + fn place( + &mut self, + writer: &mut FileNodeWriter<'_>, + unprocessed: Option<&AHashSet>, + ) -> Claim { + match self { + Planned::Create(create) => place_create(writer, create, unprocessed), + Planned::Update(update) => place_update(writer, update, unprocessed), + } + } -impl ResolveCreatedReference for NoResolver { - fn get_created_id(&self, _: &str) -> Option { - None + fn reject(self, writer: &mut FileNodeWriter<'_>) -> RejectedKey { + match self { + Planned::Create(create) => { + writer.discard_create(create.slot); + RejectedKey::Create(create.create_id) + } + Planned::Update(update) => RejectedKey::Update(update.id), + } } } -pub(super) fn update_file_node>( - expected_id: Option, - updates: Value<'_, FileNodeProperty, FileNodeValue>, - file_node: &mut FileNode, - is_create: bool, +impl RejectedKey { + fn append( + self, + response: &mut SetResponse, + err: SetError, + ) { + match self { + RejectedKey::Create(create_id) => response.not_created.append(create_id, err), + RejectedKey::Update(id) => response.not_updated.append(id, err), + } + } +} + +pub(super) async fn create_node>( + writer: &mut FileNodeWriter<'_>, + create_id: String, + node: FileNode, + object: Value<'_, FileNodeProperty, FileNodeValue>, resolver: &R, -) -> Result> { - let mut has_acl_changes = false; - let mut blob_id = None; - let mut pending_size: Option = None; - let mut pending_type: Option> = None; - let mut pending_executable: Option = None; - let mut modified_set = false; - - for (property, mut value) in updates.into_expanded_object() { - let Key::Property(property) = property else { - return Err(SetError::invalid_properties() - .with_property(property.to_owned()) - .with_description("Invalid property.")); + copy_source: Option, +) -> trc::Result> { + let mut create = + match prepare_create(writer, create_id, node, object, resolver, copy_source).await? { + Ok(create) => create, + Err((create_id, err)) => return Ok(Err((create_id, err.into()))), }; - - resolver.resolve_self_references(&mut value, 0, false)?; - - match (property, value) { - (FileNodeProperty::Name, Value::Str(value)) => { - if !(1..=255).contains(&value.len()) { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Name) - .with_description("Name must be between 1 and 255 octets.")); - } else if value.contains(|c: char| FORBIDDEN_NAME_CHARS.contains(c)) { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Name) - .with_description("Name contains a forbidden character.")); - } else if FORBIDDEN_NODE_NAMES - .iter() - .any(|n| n.eq_ignore_ascii_case(value.as_ref())) - { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Name) - .with_description("Name is reserved and cannot be used.")); - } - file_node.name = value.into_owned(); - } - (FileNodeProperty::ParentId, Value::Element(FileNodeValue::Id(value))) => { - file_node.parent_id = value.document_id() + 1; - } - (FileNodeProperty::ParentId, Value::Null) => { - file_node.parent_id = 0; - } - (FileNodeProperty::BlobId, Value::Element(FileNodeValue::BlobId(value))) => { - if file_node - .file - .as_ref() - .is_none_or(|f| f.blob_hash != value.hash) - { - blob_id = Some(value); - } - } - (FileNodeProperty::BlobId, Value::Null) => {} - (FileNodeProperty::Size, Value::Number(value)) => { - let value = value.cast_to_u64(); - if value > u32::MAX as u64 { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Size) - .with_description("size is too large.")); - } - pending_size = Some(value as u32); - } - (FileNodeProperty::Size, Value::Null) => { - pending_size = Some(0); - } - (FileNodeProperty::Type, Value::Str(value)) - if (1..=256).contains(&value.len()) && value.contains('/') => - { - // TODO: validate full RFC 6838 Section 4.2 ABNF for media types - pending_type = Some(Some(value.into_owned())); - } - (FileNodeProperty::Type, Value::Null) => { - pending_type = Some(None); - } - (FileNodeProperty::Executable, Value::Bool(value)) => { - pending_executable = Some(value); - } - (FileNodeProperty::Executable, Value::Null) => { - pending_executable = Some(false); - } - (FileNodeProperty::Created, Value::Element(FileNodeValue::Date(value))) - if is_create => - { - file_node.created = value.timestamp(); - } - (FileNodeProperty::Created, _) => { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Created) - .with_description("created is immutable after creation.")); - } - (FileNodeProperty::Modified, Value::Element(FileNodeValue::Date(value))) => { - file_node.modified = value.timestamp(); - modified_set = true; - } - (FileNodeProperty::Modified, Value::Null) => { - file_node.modified = now() as i64; - modified_set = true; - } - // TODO: persist accessed per-user (draft-13 section 3.1) - (FileNodeProperty::Accessed, _) => {} - (FileNodeProperty::NodeType, _) if is_create => {} - (FileNodeProperty::NodeType, _) => { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::NodeType) - .with_description("nodeType is immutable after creation.")); - } - // TODO: implement symlink target storage and resolution - (FileNodeProperty::Target, _) => {} - (FileNodeProperty::Changed, _) => { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Changed) - .with_description("changed is server-set and not settable by clients.")); - } - // TODO: store and validate FileNode role for directories - (FileNodeProperty::Role, _) => {} - (FileNodeProperty::ShareWith, value) => { - file_node.acls = JmapRights::acl_set::(value)?; - has_acl_changes = true; - } - (FileNodeProperty::Pointer(pointer), value) - if matches!( - pointer.first(), - Some(JsonPointerItem::Key(Key::Property( - FileNodeProperty::ShareWith - ))) - ) => - { - let mut pointer = pointer.iter(); - pointer.next(); - - file_node.acls = JmapRights::acl_patch::( - std::mem::take(&mut file_node.acls), - pointer, - value, - )?; - has_acl_changes = true; - } - (FileNodeProperty::Id, value) => { - if !expected_id.is_some_and(|expected| crate::matches_id(&value, expected)) { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Id) - .with_description("The id property is immutable.")); - } - } - (property, _) => { - return Err(SetError::invalid_properties() - .with_property(property.clone()) - .with_description("Field could not be set.")); - } + match place_create(writer, &mut create, None) { + Claim::Accepted { renamed } => commit_create(writer, create, renamed).map(Ok), + Claim::Rejected(rejection) => { + writer.discard_create(create.slot); + Ok(Err((create.create_id, rejection))) } - } - - let will_be_file = file_node.file.is_some() || blob_id.is_some(); - if will_be_file { - let file = file_node.file.get_or_insert_default(); - if let Some(size) = pending_size { - file.size = size; + Claim::Deferred => { + writer.discard_create(create.slot); + Ok(Err((create.create_id, SetError::already_exists().into()))) } - if let Some(media_type) = pending_type { - file.media_type = media_type; - } - if let Some(executable) = pending_executable { - file.executable = executable; - } - } else { - let sets_non_null = matches!(pending_type, Some(Some(_))) - || matches!(pending_size, Some(s) if s != 0) - || matches!(pending_executable, Some(true)); - if sets_non_null { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Type) - .with_description("size, type and executable may only be set on file nodes.")); - } - } - - // Validate name - if file_node.name.is_empty() { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::Name) - .with_description("Missing name.")); - } - - Ok(UpdateResult { - has_acl_changes, - blob_id, - modified_set, - }) -} - -pub(super) fn validate_file_node_hierarchy( - document_id: Option, - node: &FileNode, - is_shared: bool, - cache: &DavResources, - created_folders: &AHashMap>, -) -> Result<(), SetError> { - if node.parent_id == 0 { - if is_shared && document_id.is_none() { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::ParentId) - .with_description("Cannot create top-level folder in a shared account.")); - } - } else { - let parent_id = node.parent_id - 1; - - if let Some(document_id) = document_id { - if document_id == parent_id { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::ParentId) - .with_description("A file node cannot be its own parent.")); - } - - // Validate circular references - if let Some(file) = cache.container_resource_path_by_id(document_id) - && cache - .subtree(file.path()) - .any(|r| r.document_id() == parent_id) - { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::ParentId) - .with_description("Circular reference in parent ids.")); - } - } - - // Make sure the parent is a container - if !created_folders.contains_key(&parent_id) - && cache.container_resource_by_id(parent_id).is_none() - { - return Err(SetError::invalid_properties() - .with_property(FileNodeProperty::ParentId) - .with_description("Parent ID does not exist or is not a folder.")); - } - } - - Ok(()) -} - -#[derive(Copy, Clone, PartialEq, Eq)] -pub(super) enum Collision { - None, - Existing(u32), - Pending, -} - -pub(super) async fn fetch_existing_modified( - store: &store::Store, - account_id: u32, - document_id: u32, -) -> trc::Result { - Ok(store - .get_value::>(ValueKey::archive( - account_id, - Collection::FileNode, - document_id, - )) - .await? - .map(|arch| { - arch.unarchive::() - .map(|node| node.modified.to_native()) - .unwrap_or(0) - }) - .unwrap_or(0)) -} - -pub(super) fn names_equal(a: &str, b: &str, case_insensitive: bool) -> bool { - if case_insensitive { - a.eq_ignore_ascii_case(b) - } else { - a == b } } -pub(super) fn pending_key(node: &FileNode, case_insensitive: bool) -> (u32, String) { - ( - node.parent_id, - if case_insensitive { - node.name.to_lowercase() - } else { - node.name.clone() +async fn prepare_create>( + writer: &mut FileNodeWriter<'_>, + create_id: String, + mut node: FileNode, + mut object: Value<'_, FileNodeProperty, FileNodeValue>, + resolver: &R, + copy_source: Option, +) -> trc::Result)>> { + let new_metadata = match writer + .metadata + .extract(MetadataPatches::for_create(), &mut object) + { + Ok(patches) => match copy_source { + Some(source_id) => Some(NewMetadata::Copy { patches, source_id }), + None => patches.map(NewMetadata::Create), }, + Err(err) => return Ok(Err((create_id, err))), + }; + let patch = match NodePatch::parse(None, object, &node, true, resolver) { + Ok(patch) => patch, + Err(err) => return Ok(Err((create_id, err))), + }; + match (patch.kind, copy_source) { + (None, None) => { + return Ok(Err(( + create_id, + invalid(FileNodeProperty::Type, "@type is required on creation."), + ))); + } + (Some(kind), Some(_)) if kind != node.kind_id() => { + return Ok(Err(( + create_id, + invalid( + FileNodeProperty::Type, + "@type must match the @type of the source node.", + ), + ))); + } + _ => {} + } + let parent = patch.parent.unwrap_or(ParentRef::ROOT); + if parent.is_root() && !writer.is_owner() { + return Ok(Err(( + create_id, + SetError::forbidden() + .with_description("Cannot create top-level folder in a shared account."), + ))); + } else if !writer.can_add_to(parent) { + return Ok(Err(( + create_id, + SetError::forbidden() + .with_description("You are not allowed to create file nodes in this folder."), + ))); + } + let blob = match patch.blob_to_resolve(&node) { + Some(blob_id) => match writer.resolve_blob(blob_id).await? { + Ok(blob) => Some(blob), + Err(err) => return Ok(Err((create_id, err))), + }, + None => None, + }; + let omitted = Omitted { + media_type: patch.echoes_type(), + executable: !matches!(patch.executable, Some(Some(_))), + created: patch.created.is_none() || patch.server_created, + modified: patch.modified.is_none() || patch.server_modified, + accessed: patch.accessed.is_none() || patch.server_accessed, + subscribed: patch.kind.unwrap_or_else(|| node.kind_id()) == FILE_KIND_DIRECTORY + && !matches!(patch.subscribed, Some(Some(_))), + }; + let effects = match patch.apply(&mut node, blob, writer.personal_id) { + Ok(effects) => effects, + Err(err) => return Ok(Err((create_id, err))), + }; + if effects.acls + && writer + .inherited_rights(parent) + .is_some_and(|rights| !rights.contains(Acl::Share)) + { + return Ok(Err(( + create_id, + SetError::forbidden().with_description("You are not allowed to share this file node."), + ))); + } + if !node.acls.is_empty() + && let Err(err) = writer + .validate_acls( + &node.acls, + writer.inherited_rights(parent), + &[] as &[AclGrant], + ) + .await + { + return Ok(Err((create_id, err))); + } + let metadata_access = match (&new_metadata, writer.inherited_rights(parent)) { + (Some(_), Some(rights)) => MetadataAccess { + may_write_shared: rights.contains(Acl::ModifyMetadata), + may_read: rights.contains(Acl::Read) || rights.contains(Acl::ReadItems), + }, + _ => MetadataAccess::FULL, + }; + let metadata = match writer + .metadata + .prepare_new(writer.server, new_metadata, metadata_access) + .await? + { + Ok(metadata) => metadata, + Err(err) => return Ok(Err((create_id, err))), + }; + + let slot = writer + .batch + .reserve_document_id(writer.account_id, Collection::FileNode); + writer.stage_create(slot, parent); + Ok(Ok(PreparedCreate { + create_id, + slot, + node, + parent, + omitted, + metadata, + })) +} + +fn place_create( + writer: &mut FileNodeWriter<'_>, + create: &mut PreparedCreate, + unprocessed: Option<&AHashSet>, +) -> Claim { + match writer.validate_placement(None, create.parent, true, unprocessed) { + Ok(Placement::Ready) => {} + Ok(Placement::Deferred) => return Claim::Deferred, + Err(err) => return Claim::Rejected(err.into()), + } + if let Err(err) = writer + .validate_quota(create.node.is_directory()) + .and_then(|_| writer.validate_disk_quota(create.node.size() as u64)) + { + return Claim::Rejected(err.into()); + } + writer.claim_name( + None, + Some(create.slot), + None, + create.parent, + &mut create.node, + unprocessed, ) } -pub(super) fn find_sibling_collision( - document_id: Option, +fn commit_create( + writer: &mut FileNodeWriter<'_>, + create: PreparedCreate, + renamed: bool, +) -> trc::Result { + let PreparedCreate { + create_id, + slot, + mut node, + parent, + omitted, + metadata, + } = create; + writer.register_created(slot, parent, node.is_directory()); + node.stamp_insert( + omitted.created && node.created == 0, + omitted.modified && node.modified == 0, + omitted.accessed && node.accessed == 0, + ); + node.parent_id = parent.as_stored(); + node.set_presence( + metadata + .as_ref() + .and_then(PreparedMetadata::file_presence) + .unwrap_or(FilePresence::NONE), + ); + writer.reserve_disk_quota(node.size() as u64); + let values = created_values(&node, &omitted, writer.inherited_rights(parent), renamed); + if let Some(metadata) = metadata { + writer.metadata.write(metadata, slot, &mut writer.batch)?; + } + node.insert_stamped( + writer.access_token.changed_by(), + writer.account_id, + slot, + parent.slot(), + &mut writer.batch, + ) + .caused_by(trc::location!())?; + Ok(CreatedNode { + create_id, + slot, + values, + }) +} + +async fn prepare_update( + writer: &mut FileNodeWriter<'_>, + id: Id, + object: Value<'_, FileNodeProperty, FileNodeValue>, + metadata_patches: Option, + resolver: &CreateResolver<'_>, +) -> trc::Result>> { + let is_metadata_only = metadata_patches.is_some() + && object + .as_object() + .is_some_and(|object| is_empty_update::(object, id)); + let document_id = id.document_id(); + if writer.is_destroyed(document_id) { + return Ok(Err(SetError::will_destroy())); + } else if !writer.is_visible(document_id) { + return Ok(Err(SetError::not_found())); + } else if let Err(err) = + writer.validate_sent_properties(document_id, &object, metadata_patches.as_ref()) + { + return Ok(Err(err)); + } + let Some(archive) = fetch_archive(writer.server, writer.account_id, document_id).await? else { + return Ok(Err(SetError::not_found())); + }; + let current = archive + .to_unarchived::() + .caused_by(trc::location!())?; + let mut node = current + .deserialize::() + .caused_by(trc::location!())?; + let old_parent = ParentRef::from_stored(node.parent_id); + + let patch = match NodePatch::parse(Some(id), object, &node, false, resolver) { + Ok(patch) => patch, + Err(err) => return Ok(Err(err)), + }; + let blob = match patch.blob_to_resolve(&node) { + Some(_) if !writer.has_right(document_id, Acl::ModifyItems) => { + return Ok(Err(SetError::forbidden().with_description( + "You are not allowed to modify the content of this file node.", + ))); + } + Some(blob_id) => match writer.resolve_blob(blob_id).await? { + Ok(blob) => Some(blob), + Err(err) => return Ok(Err(err)), + }, + None => None, + }; + if node.is_directory() + && let Some(requested @ Value::Object(_)) = patch.my_rights.as_ref() + && !same_object(requested, &my_rights(writer.node_rights_of(document_id))) + { + return Ok(Err(invalid( + FileNodeProperty::MyRights, + "myRights is server-set and not settable by clients.", + ))); + } + let report_size = blob.is_some() && patch.size.is_none(); + let replaces_content = blob.is_some(); + let echo = Omitted { + media_type: patch.media_type.is_some() && patch.echoes_type(), + executable: matches!(patch.executable, Some(None)), + created: patch.server_created, + modified: patch.server_modified, + accessed: patch.server_accessed, + subscribed: node.is_directory() && matches!(patch.subscribed, Some(None)), + }; + let parent = patch.parent.unwrap_or(old_parent); + let previous = (old_parent, node.name.clone()); + let effects = match patch.apply(&mut node, blob, writer.personal_id) { + Ok(effects) => effects, + Err(err) => return Ok(Err(err)), + }; + let growth = if replaces_content { + (node.size() as u64).saturating_sub(current.inner.size() as u64) + } else { + 0 + }; + if !is_metadata_only + && let Err(err) = writer.validate_update_rights(document_id, old_parent, parent, &effects) + { + return Ok(Err(err)); + } + if effects.acls + && let Err(err) = writer + .validate_acls( + &node.acls, + writer.rights_of(document_id), + ¤t.inner.acls, + ) + .await + { + return Ok(Err(err)); + } + let metadata = match metadata_patches { + Some(patches) => { + let access = MetadataAccess { + may_write_shared: writer.has_right(document_id, Acl::ModifyMetadata), + may_read: writer.is_readable(document_id), + }; + match writer + .metadata + .prepare_for( + writer.server, + patches, + access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(metadata) => Some(metadata), + Err(err) => return Ok(Err(err)), + } + } + None => None, + }; + + Ok(Ok(PreparedUpdate { + id, + archive, + node, + parent, + previous, + effects, + growth, + report_size, + echo, + metadata, + })) +} + +fn place_update( + writer: &mut FileNodeWriter<'_>, + update: &mut PreparedUpdate, + unprocessed: Option<&AHashSet>, +) -> Claim { + let document_id = update.id.document_id(); + if writer.is_destroyed(document_id) { + return Claim::Rejected(SetError::will_destroy().into()); + } + if let Err(err) = writer.validate_disk_quota(update.growth) { + return Claim::Rejected(err.into()); + } + if let Some(unprocessed) = unprocessed + && (update.effects.renamed || update.effects.moved) + && writer + .holder_of(update.parent, &update.node.name) + .is_some_and(|holder| { + holder != NodeRef::Node(document_id) && unprocessed.contains(&holder) + }) + { + return Claim::Deferred; + } + match writer.validate_placement( + Some(document_id), + update.parent, + update.effects.moved, + unprocessed, + ) { + Ok(Placement::Ready) => {} + Ok(Placement::Deferred) => return Claim::Deferred, + Err(err) => return Claim::Rejected(err.into()), + } + if !update.effects.renamed && !update.effects.moved { + return Claim::Accepted { renamed: false }; + } + writer.claim_name( + Some(document_id), + None, + Some((update.previous.0, update.previous.1.as_str())), + update.parent, + &mut update.node, + unprocessed, + ) +} + +fn commit_update( + writer: &mut FileNodeWriter<'_>, + update: PreparedUpdate, + renamed: bool, +) -> trc::Result>> { + let document_id = update.id.document_id(); + let metadata = update.metadata; + if !update.effects.changes_node() && !update.effects.subscription { + if let Some(metadata) = metadata { + if let Some(presence) = metadata.file_presence() { + FileNode::rewrite_presence( + &update + .archive + .to_unarchived::() + .caused_by(trc::location!())?, + presence, + writer.account_id, + document_id, + &mut writer.batch, + ) + .caused_by(trc::location!())?; + } + writer + .metadata + .write(metadata, document_id, &mut writer.batch)?; + writer.batch.commit_point(); + } + if update.echo.is_empty() { + return Ok(None); + } + let mut values = Map::with_capacity(2); + update.echo.insert_into(&update.node, &mut values); + return Ok(Some(Value::Object(values))); + } + let current = update + .archive + .to_unarchived::() + .caused_by(trc::location!())?; + let mut node = update.node; + node.parent_id = update.parent.as_stored(); + node.stamp_update(false); + if let Some(presence) = metadata.as_ref().and_then(PreparedMetadata::file_presence) { + node.set_presence(presence); + } + writer.register_update( + document_id, + update.parent, + &update.effects, + node.is_directory(), + ); + + let mut values = Map::with_capacity(3); + if writer.is_readable(document_id) { + values.insert_unchecked( + Key::Property(FileNodeProperty::Changed), + date_value(Some(node.changed)), + ); + } + if renamed { + values.insert_unchecked( + Key::Property(FileNodeProperty::Name), + Value::Str(Cow::Owned(node.name.clone())), + ); + } + if update.report_size + && let Some(file) = node.file() + { + values.insert_unchecked( + Key::Property(FileNodeProperty::Size), + Value::Number(file.size.into()), + ); + } + update.echo.insert_into(&node, &mut values); + + if let Some(metadata) = metadata { + writer + .metadata + .write(metadata, document_id, &mut writer.batch)?; + } + writer.reserve_disk_quota(update.growth); + node.update_stamped( + writer.access_token.changed_by(), + current, + writer.account_id, + document_id, + update.parent.slot(), + &mut writer.batch, + ) + .caused_by(trc::location!())?; + if update.effects.renamed || update.effects.moved { + writer.log_vanished_href(document_id); + } + Ok(Some(Value::Object(values))) +} + +fn resolve_name_cycles(writer: &mut FileNodeWriter<'_>, deferred: &[Planned]) -> Vec { + const UNVISITED: u8 = 0; + const VISITING: u8 = 1; + const VISITED: u8 = 2; + + let updates = deferred + .iter() + .enumerate() + .filter_map(|(index, item)| match item { + Planned::Update(update) => Some((update.id.document_id(), index)), + Planned::Create(_) => None, + }) + .collect::>(); + let waits_on = deferred + .iter() + .map(|item| match item { + Planned::Update(update) => match writer.holder_of(update.parent, &update.node.name) { + Some(NodeRef::Node(holder_id)) => updates.get(&holder_id).copied(), + _ => None, + }, + Planned::Create(_) => None, + }) + .collect::>(); + + let mut state = vec![UNVISITED; deferred.len()]; + let mut path = Vec::new(); + let mut committed = Vec::new(); + for start in 0..deferred.len() { + path.clear(); + let mut current = Some(start); + while let Some(index) = current { + match state.get(index).copied() { + Some(UNVISITED) => { + state[index] = VISITING; + path.push(index); + current = waits_on.get(index).copied().flatten(); + } + Some(VISITING) => { + let cycle = path + .iter() + .copied() + .skip_while(|member| *member != index) + .collect::>(); + if claim_cycle(writer, deferred, &cycle) { + committed.extend(cycle); + } + current = None; + } + _ => current = None, + } + } + for index in &path { + state[*index] = VISITED; + } + } + committed.sort_unstable(); + committed +} + +fn claim_cycle(writer: &mut FileNodeWriter<'_>, deferred: &[Planned], cycle: &[usize]) -> bool { + let members = cycle + .iter() + .filter_map(|index| match deferred.get(*index) { + Some(Planned::Update(update)) => Some(update), + _ => None, + }) + .collect::>(); + let mut staged = Vec::with_capacity(members.len()); + let mut valid = writer + .validate_disk_quota(members.iter().map(|update| update.growth).sum()) + .is_ok(); + for update in &members { + let document_id = update.id.document_id(); + if !valid + || writer.is_destroyed(document_id) + || !matches!( + writer.validate_placement( + Some(document_id), + update.parent, + update.effects.moved, + None + ), + Ok(Placement::Ready) + ) + { + valid = false; + break; + } + if update.effects.moved { + writer.stage_move(document_id, update.parent); + staged.push(document_id); + } + } + let claimed = valid + && writer.claim_cycle(members.iter().map(|update| { + ( + update.id.document_id(), + update.previous.0, + update.previous.1.as_str(), + update.parent, + update.node.name.as_str(), + ) + })); + if !claimed { + for document_id in staged { + writer.unstage_move(document_id); + } + } + claimed +} + +fn created_values( node: &FileNode, - cache: &DavResources, - pending: &AHashMap<(u32, String), Option>, - case_insensitive: bool, -) -> Collision { - let node_parent_id = if node.parent_id == 0 { - None - } else { - Some(node.parent_id - 1) - }; - for resource in &cache.resources { - if let DavResourceMetadata::File { - name, parent_id, .. - } = &resource.data - && document_id.is_none_or(|id| id != resource.document_id) - && node_parent_id == *parent_id - && names_equal(&node.name, name, case_insensitive) - { - return Collision::Existing(resource.document_id); - } + omitted: &Omitted, + rights: Option>, + renamed: bool, +) -> NodeValues { + let mut values = Map::with_capacity(10); + if renamed { + values.insert_unchecked( + Key::Property(FileNodeProperty::Name), + Value::Str(Cow::Owned(node.name.clone())), + ); } - if pending.contains_key(&pending_key(node, case_insensitive)) { - return Collision::Pending; + if let Some(file) = node.file() { + values.insert_unchecked( + Key::Property(FileNodeProperty::Size), + Value::Number(file.size.into()), + ); } - Collision::None + omitted.insert_into(node, &mut values); + values.insert_unchecked( + Key::Property(FileNodeProperty::Changed), + date_value(Some(node.changed)), + ); + if node.is_directory() { + values.insert_unchecked(Key::Property(FileNodeProperty::MyRights), my_rights(rights)); + } + values } -pub(super) fn pick_unique_rename( - base: &str, - document_id: Option, - parent_id: u32, - cache: &DavResources, - pending: &AHashMap<(u32, String), Option>, - case_insensitive: bool, -) -> String { - let (stem, ext) = match base.rfind('.') { - Some(i) if i > 0 && i < base.len() - 1 => (&base[..i], &base[i..]), - _ => (base, ""), - }; - let fold = |s: &str| { - if case_insensitive { - s.to_lowercase() - } else { - s.to_string() - } - }; - let node_parent_id = if parent_id == 0 { - None - } else { - Some(parent_id - 1) - }; +fn normalized_media_type(media_type: Option<&str>) -> Cow<'static, str> { + media_type + .and_then(media_type_essence) + .map_or(Cow::Borrowed(default_media_type()), |essence| { + Cow::Owned(essence.into_owned()) + }) +} - // Collect all sibling names once, instead of rescanning per probe. - let mut taken: AHashSet = AHashSet::new(); - for resource in &cache.resources { - if let DavResourceMetadata::File { - name, parent_id, .. - } = &resource.data - && document_id.is_none_or(|id| id != resource.document_id) - && node_parent_id == *parent_id - { - taken.insert(fold(name)); - } - } - for ((pending_parent, pending_name), _) in pending { - if *pending_parent == parent_id { - taken.insert(fold(pending_name)); +fn same_object( + a: &Value<'_, FileNodeProperty, FileNodeValue>, + b: &Value<'_, FileNodeProperty, FileNodeValue>, +) -> bool { + match (a.as_object(), b.as_object()) { + (Some(a), Some(b)) => { + a.len() == b.len() && a.iter().all(|(key, value)| b.get(key) == Some(value)) } + _ => false, } +} - for n in 2u32.. { - let candidate = format!("{stem} ({n}){ext}"); - if !taken.contains(&fold(&candidate)) { - return candidate; - } +impl ResolveCreatedReference for CreateResolver<'_> { + fn get_created_id(&self, id_ref: &str) -> Option { + self.created_id(id_ref) } - unreachable!() } diff --git a/crates/jmap/src/file/writer.rs b/crates/jmap/src/file/writer.rs new file mode 100644 index 00000000..fc6c3d87 --- /dev/null +++ b/crates/jmap/src/file/writer.rs @@ -0,0 +1,1346 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + FileNodeQuota, ObjectCounts, + node::{PatchEffects, ResolvedBlob, invalid, validate_name}, +}; +use crate::{ + api::acl::{JmapAcl, JmapRights}, + api::metadata::{MetadataPatches, MetadataWriter, ObjectMetadata}, + api::parent_ref::ParentRef, + blob::{ + download::{BlobAccess, BlobDownload}, + embedded::EmbeddedBlobs, + }, +}; +use common::{ + GroupwareResources, Server, + auth::AccessToken, + sharing::{ + file::FileNodeAccess, + grants::{AclGrantLookup, ShareUpdate}, + }, + storage::dav::{FILE_KIND_SYMLINK, MAX_FILE_NODE_DEPTH}, +}; +use groupware::{DestroyArchive, file::FileNode}; +use jmap_proto::{ + error::set::SetError, + object::file_node::{self, FileNodeProperty, FileNodeValue, OnExists}, +}; +use jmap_tools::{JsonPointerItem, Key, Value}; +use std::borrow::Cow; +use store::{ + ValueKey, + ahash::{AHashMap, AHashSet}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot, now}, +}; +use trc::AddContext; +use types::{ + acl::{Acl, AclGrant}, + blob::{BlobClass, BlobId}, + blob_hash::BlobHash, + collection::{Collection, SyncCollection, VanishedCollection}, + id::Id, +}; +use utils::map::bitmap::Bitmap; + +const RENAME_ATTEMPTS: u32 = 10_000; +const MAX_ANCESTOR_WALK: usize = MAX_FILE_NODE_DEPTH * 4; + +#[derive(Debug, Clone, Copy)] +pub(super) struct WriteOptions { + pub on_exists: OnExists, + pub remove_children: bool, + pub case_insensitive: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(super) enum NodeRef { + Node(u32), + Pending(Slot), +} + +pub(super) enum Claim { + Accepted { renamed: bool }, + Deferred, + Rejected(Rejection), +} + +pub(super) enum Placement { + Ready, + Deferred, +} + +pub(super) struct Rejection { + pub error: SetError, + pub existing: Option, +} + +pub(super) struct DestroyGroup { + pub id: Id, + pub ids: Vec, + pub deferred: Vec, + pub path: Option, + pub requires_empty: bool, +} + +pub(super) struct FileNodeWriter<'x> { + pub server: &'x Server, + pub access_token: &'x AccessToken, + pub account_id: u32, + pub cache: &'x GroupwareResources, + pub personal_id: u32, + pub batch: BatchBuilder, + pub metadata: MetadataWriter, + pub implicit_destroys: Vec, + access: Option, + options: WriteOptions, + siblings: AHashMap, NodeRef>>, + destroyed: AHashSet, + parents: AHashMap, + accepted_folders: AHashSet, + touched: AHashSet, + received: AHashSet, + logged_subtrees: Vec, + cached_heights: AHashMap, + rename_suffixes: AHashMap<(ParentRef, String), u32>, + quota: FileNodeQuota, + created: usize, + destroyed_counts: ObjectCounts, +} + +impl NodeRef { + #[inline(always)] + pub fn from_parent(parent: ParentRef) -> Option { + match parent { + ParentRef::Root => None, + ParentRef::Node(document_id) => Some(NodeRef::Node(document_id)), + ParentRef::Pending(slot) => Some(NodeRef::Pending(slot)), + } + } +} + +impl From> for Rejection { + fn from(error: SetError) -> Self { + Rejection { + error, + existing: None, + } + } +} + +impl<'x> FileNodeWriter<'x> { + pub fn new( + server: &'x Server, + access_token: &'x AccessToken, + account_id: u32, + cache: &'x GroupwareResources, + options: WriteOptions, + metadata: ObjectMetadata, + ) -> Self { + let access = (!access_token.is_member(account_id)).then(|| cache.file_access(access_token)); + FileNodeWriter { + server, + access_token, + account_id, + cache, + personal_id: access_token.personal_id(account_id, Collection::FileNode), + batch: BatchBuilder::new(), + metadata: MetadataWriter::new(metadata, account_id), + implicit_destroys: Vec::new(), + access, + options, + siblings: AHashMap::new(), + destroyed: AHashSet::new(), + parents: AHashMap::new(), + accepted_folders: AHashSet::new(), + touched: AHashSet::new(), + received: AHashSet::new(), + logged_subtrees: Vec::new(), + cached_heights: AHashMap::new(), + rename_suffixes: AHashMap::new(), + quota: FileNodeQuota::unlimited(), + created: 0, + destroyed_counts: ObjectCounts::default(), + } + } + + pub async fn with_quota(mut self) -> trc::Result { + let account = self + .server + .account(self.account_id) + .await + .caused_by(trc::location!())?; + let disk = self + .server + .disk_quota_budget(&account) + .await + .caused_by(trc::location!())?; + self.quota = FileNodeQuota::new(self.server, &account, self.cache, disk); + Ok(self) + } + + #[inline(always)] + pub fn validate_disk_quota(&self, bytes: u64) -> Result<(), SetError> { + self.quota.validate_disk(bytes) + } + + #[inline(always)] + pub fn reserve_disk_quota(&mut self, bytes: u64) { + self.quota.reserve_disk(bytes); + } + + pub fn validate_quota(&self, is_folder: bool) -> Result<(), SetError> { + let folders = self.accepted_folders.len(); + self.quota.validate( + is_folder, + ObjectCounts { + files: self.created.saturating_sub(folders), + folders, + }, + self.destroyed_counts, + ) + } + + #[inline(always)] + pub fn is_owner(&self) -> bool { + self.access.is_none() + } + + #[inline(always)] + pub fn has_right(&self, document_id: u32, acl: Acl) -> bool { + self.access + .as_ref() + .is_none_or(|access| access.has_acl(document_id, acl)) + } + + pub fn node_rights_of(&self, document_id: u32) -> Option> { + self.access + .as_ref() + .map(|access| access.node_rights(document_id, self.stored_parent(document_id))) + } + + pub fn rights_of(&self, document_id: u32) -> Option> { + self.access.as_ref().map(|access| access.acl(document_id)) + } + + #[inline(always)] + pub fn is_readable(&self, document_id: u32) -> bool { + self.access + .as_ref() + .is_none_or(|access| access.readable.contains(document_id)) + } + + #[inline(always)] + pub fn is_visible(&self, document_id: u32) -> bool { + self.access + .as_ref() + .is_none_or(|access| access.discoverable.contains(document_id)) + } + + #[inline(always)] + pub fn is_destroyed(&self, document_id: u32) -> bool { + self.destroyed.contains(&document_id) + } + + pub fn inherited_rights(&self, parent: ParentRef) -> Option> { + let access = self.access.as_ref()?; + Some( + self.ancestors(NodeRef::from_parent(parent)) + .find_map(|node| match node { + NodeRef::Node(document_id) => Some(access.acl(document_id)), + NodeRef::Pending(_) => None, + }) + .unwrap_or_default(), + ) + } + + pub fn plan_destroys( + &mut self, + ids: Vec, + moving: &AHashSet, + ) -> (Vec, Vec<(Id, SetError)>) { + let mut failed = Vec::new(); + let mut requested = AHashMap::with_capacity(ids.len()); + for id in ids { + let document_id = id.document_id(); + if self.cache.resources.find_any(document_id).is_some() && self.is_visible(document_id) + { + requested.insert(document_id, id); + } else { + failed.push((id, SetError::not_found())); + } + } + + let remove_children = self.options.remove_children; + let mut subtrees: AHashMap, Vec)> = + AHashMap::with_capacity(requested.len()); + let mut rejected: AHashMap> = AHashMap::new(); + let mut order = requested + .keys() + .map(|&document_id| (self.depth_of(ParentRef::Node(document_id)), document_id)) + .collect::>(); + order.sort_unstable(); + let mut covered = AHashSet::new(); + for (_, document_id) in order { + if covered.contains(&document_id) { + continue; + } + let nodes = if remove_children { + self.subtree_nodes(document_id) + } else { + vec![(document_id, self.stored_parent(document_id))] + }; + let allowed = self.may_destroy(&nodes); + let ids = nodes.into_iter().map(|(id, _)| id).collect::>(); + let planned = if !allowed { + rejected.insert(document_id, not_permitted()); + (ids, Vec::new()) + } else if remove_children { + let mut moved_out = AHashSet::new(); + for &id in ids.iter().rev() { + if id != document_id && moving.contains(&id) && !moved_out.contains(&id) { + moved_out.extend(self.subtree_ids(id)); + } + } + let (ids, deferred): (Vec, Vec) = if moved_out.is_empty() { + (ids, Vec::new()) + } else { + ids.into_iter().partition(|id| !moved_out.contains(id)) + }; + covered.extend(ids.iter().copied()); + (ids, deferred) + } else { + (ids, Vec::new()) + }; + subtrees.insert(document_id, planned); + } + + if !remove_children { + let children = requested + .keys() + .filter_map(|&document_id| { + let children = self.cache.children_ids(document_id).collect::>(); + (!children.is_empty()).then_some((document_id, children)) + }) + .collect::>(); + loop { + let mut changed = false; + for (document_id, children) in &children { + if !rejected.contains_key(document_id) + && children.iter().any(|child| { + (!requested.contains_key(child) && !moving.contains(child)) + || rejected.contains_key(child) + }) + { + rejected.insert(*document_id, SetError::node_has_children()); + changed = true; + } + } + if !changed { + break; + } + } + } + + let mut groups = Vec::with_capacity(requested.len()); + let mut roots = requested + .into_iter() + .filter_map(|(document_id, id)| match rejected.remove(&document_id) { + Some(err) => { + failed.push((id, err)); + None + } + None => Some((document_id, id)), + }) + .collect::>(); + roots.sort_by_cached_key(|(document_id, _)| self.depth_of(ParentRef::Node(*document_id))); + for (document_id, id) in roots { + if self.destroyed.contains(&document_id) { + continue; + } + let (ids, deferred) = subtrees.remove(&document_id).unwrap_or_default(); + self.mark_destroyed(&ids); + groups.push(DestroyGroup { + id, + path: self.vanished_href(document_id), + ids, + deferred, + requires_empty: !remove_children, + }); + } + + (groups, failed) + } + + pub async fn execute_destroys( + &mut self, + mut groups: Vec, + ) -> trc::Result<(Vec, Vec<(Id, SetError)>)> { + for group in &mut groups { + for id in std::mem::take(&mut group.deferred).into_iter().rev() { + if !self.destroyed.contains(&id) + && let Some(NodeRef::Node(parent_id)) = self.parent_of(NodeRef::Node(id)) + && self.destroyed.contains(&parent_id) + { + self.mark_destroyed(&[id]); + group.ids.push(id); + } + } + } + let mut destroyed = Vec::with_capacity(groups.len()); + let mut failed = Vec::new(); + let mut kept = AHashSet::new(); + let implicit_destroys = std::mem::take(&mut self.implicit_destroys); + let cache = self.cache; + let cleanup = self + .server + .preload_container_cleanup( + self.access_token.changed_by(), + self.account_id, + Collection::FileNode, + &groups + .iter() + .chain(&implicit_destroys) + .flat_map(|group| group.ids.iter().copied()) + .filter(|&document_id| { + cache + .resources + .find_any(document_id) + .is_some_and(|resource| !resource.metadata_kinds().is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; + for group in groups.into_iter().rev().chain(implicit_destroys) { + let root_id = group.id.document_id(); + if group.requires_empty + && self.cache.children_ids(root_id).any(|child| { + kept.contains(&child) + || (!self.destroyed.contains(&child) + && self.parent_of(NodeRef::Node(child)) == Some(NodeRef::Node(root_id))) + }) + { + kept.insert(root_id); + failed.push((group.id, SetError::node_has_children())); + continue; + } + if group.ids.len() > 1 { + destroyed.extend( + group + .ids + .iter() + .filter(|id| **id != group.id.document_id()) + .map(|id| Id::from(*id)), + ); + } + destroyed.push(group.id); + DestroyArchive(group.ids) + .delete_batch( + self.server, + self.access_token.changed_by(), + self.account_id, + &cleanup, + group.path, + &mut self.batch, + ) + .await + .caused_by(trc::location!())?; + } + Ok((destroyed, failed)) + } + + pub fn validate_placement( + &mut self, + document_id: Option, + parent: ParentRef, + moved: bool, + unprocessed: Option<&AHashSet>, + ) -> Result> { + match parent { + ParentRef::Root => { + if !self.is_owner() && moved { + return Err(SetError::forbidden() + .with_description("Cannot create top-level folder in a shared account.")); + } + } + ParentRef::Pending(slot) => { + if !self.accepted_folders.contains(&slot) { + return match unprocessed { + Some(unprocessed) if unprocessed.contains(&NodeRef::Pending(slot)) => { + Ok(Placement::Deferred) + } + _ => Err(invalid( + FileNodeProperty::ParentId, + "Parent ID does not exist or is not a folder.", + )), + }; + } + } + ParentRef::Node(parent_id) => { + if self.destroyed.contains(&parent_id) { + return Err(invalid( + FileNodeProperty::ParentId, + "Parent is being destroyed in this request.", + )); + } else if !self + .cache + .container_resource_by_id(parent_id) + .is_some_and(|_| self.is_visible(parent_id)) + { + return Err(invalid( + FileNodeProperty::ParentId, + "Parent ID does not exist or is not a folder.", + )); + } + } + } + if !moved || parent.is_root() { + return Ok(Placement::Ready); + } + if let Some(document_id) = document_id + && self.is_ancestor_or_self(document_id, parent) + { + return Err(invalid( + FileNodeProperty::ParentId, + "Circular reference in parent ids.", + )); + } + let height = document_id.map_or(1, |id| self.subtree_height(id)); + if self.depth_of(parent) + height > MAX_FILE_NODE_DEPTH { + return Err(invalid( + FileNodeProperty::ParentId, + "Maximum folder depth exceeded.", + )); + } + Ok(Placement::Ready) + } + + pub fn can_add_to(&self, parent: ParentRef) -> bool { + match parent { + _ if self.is_owner() => true, + ParentRef::Root => false, + ParentRef::Pending(slot) => self.parents.contains_key(&NodeRef::Pending(slot)), + ParentRef::Node(parent_id) => self.has_right(parent_id, Acl::AddItems), + } + } + + pub fn validate_sent_properties( + &self, + document_id: u32, + update: &Value<'_, FileNodeProperty, FileNodeValue>, + metadata: Option<&MetadataPatches>, + ) -> Result<(), SetError> { + if self.is_owner() { + return Ok(()); + } + let is_readable = self.is_readable(document_id); + if !is_readable && metadata.is_some() { + return Err(not_permitted()); + } + let Some(object) = update.as_object() else { + return Ok(()); + }; + for (key, _) in object.iter() { + let Key::Property(property) = key else { + continue; + }; + let property = match property { + FileNodeProperty::Pointer(pointer) => match pointer.first() { + Some(JsonPointerItem::Key(Key::Property(head))) => head, + _ => property, + }, + property => property, + }; + let (allowed, description) = match property { + FileNodeProperty::Id | FileNodeProperty::MyRights => continue, + FileNodeProperty::Name if !is_readable => { + if self.has_right(document_id, Acl::Modify) { + continue; + } + return Err(not_permitted()); + } + FileNodeProperty::ShareWith if !is_readable => { + if self.has_right(document_id, Acl::Share) { + continue; + } + return Err(not_permitted()); + } + _ if !is_readable => return Err(not_permitted()), + FileNodeProperty::IsSubscribed + | FileNodeProperty::Metadata + | FileNodeProperty::PrivateMetadata + | FileNodeProperty::IdValue(_) + | FileNodeProperty::Rights(_) + | FileNodeProperty::Pointer(_) => continue, + FileNodeProperty::Name | FileNodeProperty::ParentId | FileNodeProperty::Role => ( + self.has_right(document_id, Acl::Modify), + "You are not allowed to rename or move this file node.", + ), + FileNodeProperty::BlobId + | FileNodeProperty::Target + | FileNodeProperty::Size + | FileNodeProperty::MediaType + | FileNodeProperty::Executable + | FileNodeProperty::Created + | FileNodeProperty::Modified + | FileNodeProperty::Accessed => ( + self.has_right(document_id, Acl::ModifyItems), + "You are not allowed to modify the content of this file node.", + ), + FileNodeProperty::ShareWith => ( + self.has_right(document_id, Acl::Share) + || self + .cache + .resources + .find_any(document_id) + .is_some_and(|resource| !resource.is_container()), + "You are not allowed to share this file node.", + ), + FileNodeProperty::Type | FileNodeProperty::Changed => ( + self.has_right(document_id, Acl::Modify) + || self.has_right(document_id, Acl::ModifyItems), + "You are not allowed to modify this file node.", + ), + }; + if !allowed { + return Err(SetError::forbidden().with_description(description)); + } + } + Ok(()) + } + + pub fn validate_update_rights( + &self, + document_id: u32, + old_parent: ParentRef, + new_parent: ParentRef, + effects: &PatchEffects, + ) -> Result<(), SetError> { + if self.is_owner() { + return Ok(()); + } + let forbidden = + |description: &'static str| Err(SetError::forbidden().with_description(description)); + if effects.moved + && (!self.can_add_to(new_parent) + || old_parent + .document_id() + .is_none_or(|parent_id| !self.has_right(parent_id, Acl::RemoveItems))) + { + forbidden("You are not allowed to move this file node to the requested folder.") + } else if effects.moved && self.moves_unreadable(document_id) { + Err(not_permitted()) + } else if effects.subscription + && !self.has_right(document_id, Acl::Read) + && !self.has_right(document_id, Acl::ReadItems) + { + forbidden("You are not allowed to subscribe to this file node.") + } else { + Ok(()) + } + } + + fn moves_unreadable(&self, document_id: u32) -> bool { + let Some(access) = &self.access else { + return false; + }; + let root = NodeRef::Node(document_id); + let is_hidden_inside = |id: u32| { + !access.readable.contains(id) + && !self.destroyed.contains(&id) + && self + .ancestors(Some(NodeRef::Node(id))) + .any(|node| node == root) + }; + let stored = match self.cache.any_resource_path_by_id(document_id) { + Some(path) => self + .cache + .subtree(path.path()) + .any(|resource| is_hidden_inside(resource.document_id())), + None => is_hidden_inside(document_id), + }; + stored + || self.parents.keys().any(|node| match node { + NodeRef::Node(id) => is_hidden_inside(*id), + NodeRef::Pending(_) => false, + }) + } + + pub async fn validate_acls( + &self, + acls: &[AclGrant], + actor: Option>, + current: &G, + ) -> Result<(), SetError> { + JmapRights::validate_shares::( + ShareUpdate { + collection: Collection::FileNode, + owner_id: self.account_id, + actor, + current, + max_shares: self.server.core.groupware.max_shares_per_item, + }, + acls, + )?; + self.server + .acl_validate(self.access_token, self.account_id, acls, current) + .await + .map_err(Into::into) + } + + pub async fn resolve_blob( + &mut self, + blob_id: &BlobId, + ) -> trc::Result>> { + let BlobAccess::Allowed { size: known_size } = self + .server + .has_access_blob(blob_id, self.access_token) + .await? + else { + return Ok(Err(SetError::forbidden().with_description(format!( + "You do not have access to blobId {blob_id}." + )))); + }; + + let (hash, size) = if matches!(blob_id.class, BlobClass::Embedded { .. }) { + let Some(bytes) = self + .server + .embedded_blob(blob_id) + .await + .caused_by(trc::location!())? + else { + return Ok(Err(blob_not_found())); + }; + let (hash, hold) = self + .server + .hold_blob(self.account_id, blob_id.hash.clone(), &bytes, 60) + .await + .caused_by(trc::location!())?; + hold.commit(&mut self.batch); + (hash, bytes.len() as u64) + } else if let Some(section) = &blob_id.section { + let Some(bytes) = self + .server + .fetch_blob_section(&blob_id.hash, section, known_size) + .await + .caused_by(trc::location!())? + else { + return Ok(Err(blob_not_found())); + }; + let (hash, hold) = self + .server + .hold_blob(self.account_id, BlobHash::generate(&bytes), &bytes, 60) + .await + .caused_by(trc::location!())?; + hold.commit(&mut self.batch); + (hash, bytes.len() as u64) + } else { + match known_size { + Some(size) => (blob_id.hash.clone(), size), + None => match self + .server + .fetch_blob(&blob_id.hash, 0..usize::MAX, None) + .await + .caused_by(trc::location!())? + { + Some(bytes) => (blob_id.hash.clone(), bytes.len() as u64), + None => return Ok(Err(blob_not_found())), + }, + } + }; + + Ok(u32::try_from(size) + .map(|size| ResolvedBlob { hash, size }) + .map_err(|_| SetError::too_large().with_description("File is too large."))) + } + + pub fn stage_create(&mut self, slot: Slot, parent: ParentRef) { + self.parents.insert(NodeRef::Pending(slot), parent); + } + + pub fn discard_create(&mut self, slot: Slot) { + self.parents.remove(&NodeRef::Pending(slot)); + } + + pub fn stage_move(&mut self, document_id: u32, parent: ParentRef) { + self.parents.insert(NodeRef::Node(document_id), parent); + } + + pub fn unstage_move(&mut self, document_id: u32) { + if !self.touched.contains(&document_id) { + self.parents.remove(&NodeRef::Node(document_id)); + } + } + + pub fn register_created(&mut self, slot: Slot, parent: ParentRef, is_folder: bool) { + self.created += 1; + if is_folder { + self.accepted_folders.insert(slot); + } + if let ParentRef::Node(parent_id) = parent { + self.received.insert(parent_id); + } + } + + pub fn register_update( + &mut self, + document_id: u32, + parent: ParentRef, + effects: &PatchEffects, + is_directory: bool, + ) { + self.touched.insert(document_id); + if effects.moved { + self.parents.insert(NodeRef::Node(document_id), parent); + if let ParentRef::Node(parent_id) = parent { + self.received.insert(parent_id); + } + } + if is_directory && (effects.moved || effects.renamed || effects.acls) { + self.logged_subtrees.push(document_id); + } + } + + pub fn log_vanished_href(&mut self, document_id: u32) { + if let Some(href) = self.vanished_href(document_id) { + self.batch + .with_account_id(self.account_id) + .log_vanished_item(VanishedCollection::FileNode, href); + } + } + + pub fn holder_of(&mut self, parent: ParentRef, name: &str) -> Option { + self.load_siblings(parent); + self.siblings + .get(&parent) + .and_then(|names| names.get(fold_name(name, self.options.case_insensitive).as_ref())) + .copied() + } + + pub fn claim_name( + &mut self, + document_id: Option, + slot: Option, + previous: Option<(ParentRef, &str)>, + parent: ParentRef, + node: &mut FileNode, + unprocessed: Option<&AHashSet>, + ) -> Claim { + let holder = self + .holder_of(parent, &node.name) + .filter(|holder| document_id.is_none_or(|id| *holder != NodeRef::Node(id))); + + let mut renamed = false; + if let Some(holder) = holder { + if unprocessed.is_some_and(|unprocessed| unprocessed.contains(&holder)) { + return Claim::Deferred; + } + + if let NodeRef::Node(holder_id) = holder + && !self.is_visible(holder_id) + { + return match self.options.on_exists { + OnExists::Rename => match self.unique_name(parent, &node.name) { + Some(name) => { + node.name = name; + self.register_name(document_id, slot, previous, parent, node); + Claim::Accepted { renamed: true } + } + None => Claim::Rejected(not_permitted().into()), + }, + _ => Claim::Rejected(not_permitted().into()), + }; + } + + let on_exists = match (self.options.on_exists, holder) { + (OnExists::Replace, NodeRef::Node(holder_id)) if !self.may_delete(holder_id) => { + return Claim::Rejected(not_permitted().into()); + } + (OnExists::Newest, NodeRef::Node(holder_id)) + if !self.may_delete(holder_id) + || !self.is_readable(holder_id) + || document_id.is_some_and(|id| !self.is_readable(id)) => + { + return Claim::Rejected(not_permitted().into()); + } + (OnExists::Newest, NodeRef::Node(holder_id)) => { + let incoming = if node.modified == 0 { + now() as i64 + } else { + node.modified + }; + if self + .cache + .resources + .find_any(holder_id) + .and_then(|resource| resource.modified_at()) + .is_some_and(|existing| incoming > existing) + { + OnExists::Replace + } else { + OnExists::Reject + } + } + (OnExists::Newest, _) => OnExists::Reject, + (on_exists, _) => on_exists, + }; + + match (on_exists, holder) { + (OnExists::Rename, _) => match self.unique_name(parent, &node.name) { + Some(name) => { + node.name = name; + renamed = true; + } + None => return Claim::Rejected(already_exists(holder)), + }, + (OnExists::Replace, NodeRef::Node(holder_id)) + if self.cache.resources.find_any(holder_id).is_some() => + { + if let Err(err) = self.replace(holder_id, document_id) { + return Claim::Rejected(err.into()); + } + } + (_, holder) => return Claim::Rejected(already_exists(holder)), + } + } + + self.register_name(document_id, slot, previous, parent, node); + Claim::Accepted { renamed } + } + + fn register_name( + &mut self, + document_id: Option, + slot: Option, + previous: Option<(ParentRef, &str)>, + parent: ParentRef, + node: &FileNode, + ) { + if let Some((previous_parent, previous_name)) = previous { + self.load_siblings(previous_parent); + let previous_key = fold_name(previous_name, self.options.case_insensitive); + if let Some(names) = self.siblings.get_mut(&previous_parent) + && document_id + .is_some_and(|id| names.get(previous_key.as_ref()) == Some(&NodeRef::Node(id))) + { + names.remove(previous_key.as_ref()); + } + } + + let holder = match (document_id, slot) { + (Some(document_id), _) => NodeRef::Node(document_id), + (None, Some(slot)) => NodeRef::Pending(slot), + (None, None) => return, + }; + let key = self.fold(&node.name).into_owned(); + self.siblings + .entry(parent) + .or_default() + .insert(Cow::Owned(key), holder); + } + + pub fn claim_cycle<'y>( + &mut self, + entries: impl Iterator + Clone, + ) -> bool { + for (_, previous_parent, _, parent, _) in entries.clone() { + self.load_siblings(previous_parent); + self.load_siblings(parent); + } + let moving = entries + .clone() + .map(|(document_id, ..)| document_id) + .collect::>(); + let case_insensitive = self.options.case_insensitive; + let mut targets = AHashSet::new(); + for (document_id, _, _, parent, name) in entries.clone() { + let key = fold_name(name, case_insensitive); + let holder = self + .siblings + .get(&parent) + .and_then(|names| names.get(key.as_ref())) + .copied(); + let is_free = match holder { + None => true, + Some(NodeRef::Node(holder_id)) => { + holder_id == document_id || moving.contains(&holder_id) + } + Some(NodeRef::Pending(_)) => false, + }; + if !is_free || !targets.insert((parent, key)) { + return false; + } + } + for (document_id, previous_parent, previous_name, ..) in entries.clone() { + let key = fold_name(previous_name, case_insensitive); + if let Some(names) = self.siblings.get_mut(&previous_parent) + && names.get(key.as_ref()) == Some(&NodeRef::Node(document_id)) + { + names.remove(key.as_ref()); + } + } + for (document_id, _, _, parent, name) in entries { + let key = fold_name(name, case_insensitive).into_owned(); + self.siblings + .entry(parent) + .or_default() + .insert(Cow::Owned(key), NodeRef::Node(document_id)); + } + true + } + + pub fn finish(&mut self) { + let subtrees = std::mem::take(&mut self.logged_subtrees); + if subtrees.is_empty() { + return; + } + let mut logged = AHashSet::new(); + self.batch + .with_account_id(self.account_id) + .with_collection(Collection::FileNode); + for document_id in subtrees { + for child in self.subtree_ids(document_id) { + if child != document_id + && !self.destroyed.contains(&child) + && !self.touched.contains(&child) + && logged.insert(child) + { + self.batch + .with_document(child) + .log_item_update(SyncCollection::FileNode, None); + } + } + } + if !logged.is_empty() { + self.batch.commit_point(); + } + } + + fn replace( + &mut self, + holder_id: u32, + claimant: Option, + ) -> Result<(), SetError> { + let mut nodes = self.subtree_nodes(holder_id); + nodes.retain(|(id, _)| !self.destroyed.contains(id)); + if nodes.iter().any(|(id, _)| { + claimant == Some(*id) || self.touched.contains(id) || self.received.contains(id) + }) { + return Err(SetError::already_exists() + .with_existing_id(Id::from(holder_id)) + .with_description( + "The existing file node is modified in this request and cannot be replaced.", + )); + } + if nodes.len() > 1 { + if !self.options.remove_children { + return Err(SetError::node_has_children()); + } else if !self.may_destroy(&nodes) { + return Err(not_permitted()); + } + } + let ids = nodes.into_iter().map(|(id, _)| id).collect::>(); + if let Some(holder) = self.cache.resources.find_any(holder_id) + && let Some(name) = holder.container_name() + && let Some(names) = self + .siblings + .get_mut(&holder.parent_id().map_or(ParentRef::ROOT, ParentRef::Node)) + { + names.remove(fold_name(name, self.options.case_insensitive).as_ref()); + } + self.mark_destroyed(&ids); + self.implicit_destroys.push(DestroyGroup { + id: Id::from(holder_id), + path: self.vanished_href(holder_id), + ids, + deferred: Vec::new(), + requires_empty: false, + }); + Ok(()) + } + + fn may_delete(&self, document_id: u32) -> bool { + self.access + .as_ref() + .is_none_or(|access| access.may_delete(document_id, self.stored_parent(document_id))) + } + + fn may_destroy(&self, nodes: &[(u32, Option)]) -> bool { + self.access + .as_ref() + .is_none_or(|access| access.may_destroy(nodes.iter().copied())) + } + + fn stored_parent(&self, document_id: u32) -> Option { + self.cache + .resources + .find_any(document_id) + .and_then(|resource| resource.parent_id()) + } + + fn mark_destroyed(&mut self, ids: &[u32]) { + for &id in ids { + if self.destroyed.insert(id) { + if self.cache.container_resource_by_id(id).is_some() { + self.destroyed_counts.folders += 1; + } else { + self.destroyed_counts.files += 1; + } + if let Some(resource) = self.cache.resources.find_any(id) { + self.quota.release_disk( + (size_of::() + + resource.container_name().map_or(0, str::len) + + resource.size().unwrap_or_default() as usize) + as u64, + ); + } + } + } + } + + fn unique_name(&mut self, parent: ParentRef, base: &str) -> Option { + use std::fmt::Write; + let (stem, ext) = match base.rsplit_once('.') { + Some((stem, ext)) if !stem.is_empty() && !ext.is_empty() => (stem, ext), + _ => (base, ""), + }; + let names = self.siblings.get(&parent); + let mut candidate = String::with_capacity(base.len() + 8); + let hint = (parent, base.to_string()); + let first = self.rename_suffixes.get(&hint).copied().unwrap_or(2); + for n in first..RENAME_ATTEMPTS { + let suffix_len = + if ext.is_empty() { 0 } else { ext.len() + 1 } + n.ilog10() as usize + 4; + let max_stem = super::node::MAX_NAME_LEN.checked_sub(suffix_len)?; + let stem = stem + .char_indices() + .map(|(start, ch)| start + ch.len_utf8()) + .take_while(|end| *end <= max_stem) + .last() + .and_then(|end| stem.get(..end)) + .unwrap_or_default(); + candidate.clear(); + let _ = if ext.is_empty() { + write!(candidate, "{stem} ({n})") + } else { + write!(candidate, "{stem} ({n}).{ext}") + }; + if names.is_none_or(|names| !names.contains_key(self.fold(&candidate).as_ref())) + && validate_name(&candidate).is_ok() + { + self.rename_suffixes.insert(hint, n + 1); + return Some(candidate); + } + } + None + } + + fn load_siblings(&mut self, parent: ParentRef) { + if self.siblings.contains_key(&parent) { + return; + } + let cache = self.cache; + let mut names = AHashMap::new(); + let push = + |names: &mut AHashMap, NodeRef>, document_id: u32, name: &'x str| { + if !self.destroyed.contains(&document_id) { + names.insert( + fold_name(name, self.options.case_insensitive), + NodeRef::Node(document_id), + ); + } + }; + match parent { + ParentRef::Root => { + for resource in cache.resources.iter() { + if resource.parent_id().is_none() + && let Some(name) = resource.container_name() + { + push(&mut names, resource.document_id(), name); + } + } + } + ParentRef::Node(parent_id) => { + for child in cache.children(parent_id) { + if let Some(name) = child.resource.container_name() { + push(&mut names, child.document_id(), name); + } + } + } + ParentRef::Pending(_) => {} + } + self.siblings.insert(parent, names); + } + + #[inline(always)] + fn fold<'y>(&self, name: &'y str) -> Cow<'y, str> { + fold_name(name, self.options.case_insensitive) + } + + fn subtree_ids(&self, document_id: u32) -> Vec { + self.subtree_nodes(document_id) + .into_iter() + .map(|(id, _)| id) + .collect() + } + + fn subtree_nodes(&self, document_id: u32) -> Vec<(u32, Option)> { + match self.cache.any_resource_path_by_id(document_id) { + Some(path) => { + let mut nodes = self + .cache + .subtree(path.path()) + .map(|resource| { + ( + resource.hierarchy_seq(), + resource.document_id(), + resource.parent_id(), + ) + }) + .collect::>(); + nodes.sort_unstable_by(|a, b| b.cmp(a)); + nodes.dedup_by_key(|(_, id, _)| *id); + nodes + .into_iter() + .map(|(_, id, parent_id)| (id, parent_id)) + .collect() + } + None => vec![(document_id, self.stored_parent(document_id))], + } + } + + fn vanished_href(&self, document_id: u32) -> Option { + self.cache + .any_resource_path_by_id(document_id) + .filter(|path| path.resource.file_kind() != Some(FILE_KIND_SYMLINK)) + .map(|path| self.cache.format_resource(path)) + } + + fn parent_of(&self, node: NodeRef) -> Option { + match self.parents.get(&node) { + Some(parent) => NodeRef::from_parent(*parent), + None => match node { + NodeRef::Node(document_id) => self + .cache + .resources + .find_any(document_id) + .and_then(|resource| resource.parent_id()) + .map(NodeRef::Node), + NodeRef::Pending(_) => None, + }, + } + } + + fn ancestors(&self, start: Option) -> impl Iterator + '_ { + std::iter::successors(start, |node| self.parent_of(*node)).take(MAX_ANCESTOR_WALK) + } + + fn depth_of(&self, parent: ParentRef) -> usize { + self.ancestors(NodeRef::from_parent(parent)).count() + } + + fn is_ancestor_or_self(&self, ancestor: u32, parent: ParentRef) -> bool { + self.ancestors(NodeRef::from_parent(parent)) + .any(|node| node == NodeRef::Node(ancestor)) + } + + fn subtree_height(&mut self, document_id: u32) -> usize { + if self.cache.container_resource_by_id(document_id).is_none() { + return 1; + } + let target = NodeRef::Node(document_id); + let nodes = self + .parents + .keys() + .copied() + .filter(|node| *node != target) + .collect::>(); + let mut height = self.cached_height(document_id); + for node in nodes { + let distance = self + .ancestors(self.parent_of(node)) + .position(|ancestor| ancestor == target); + if let Some(distance) = distance { + let node_height = match node { + NodeRef::Node(id) => self.cached_height(id), + NodeRef::Pending(_) => 1, + }; + height = height.max(distance + 1 + node_height); + } + } + height + } + + fn cached_height(&mut self, document_id: u32) -> usize { + if let Some(height) = self.cached_heights.get(&document_id) { + return *height; + } + let height = match self.cache.any_resource_path_by_id(document_id) { + Some(root) if root.is_container() => { + let base = root.path().bytes().filter(|b| *b == b'/').count(); + self.cache + .subtree(root.path()) + .map(|resource| { + resource.path().bytes().filter(|b| *b == b'/').count() - base + 1 + }) + .max() + .unwrap_or(1) + } + _ => 1, + }; + self.cached_heights.insert(document_id, height); + height + } +} + +fn already_exists(holder: NodeRef) -> Rejection { + match holder { + NodeRef::Node(holder_id) => SetError::already_exists() + .with_existing_id(Id::from(holder_id)) + .into(), + NodeRef::Pending(slot) => Rejection { + error: SetError::already_exists(), + existing: Some(slot), + }, + } +} + +pub(super) fn fold_name(name: &str, case_insensitive: bool) -> Cow<'_, str> { + if case_insensitive && name.chars().any(|c| c.is_uppercase()) { + Cow::Owned(name.to_lowercase()) + } else { + Cow::Borrowed(name) + } +} + +fn not_permitted() -> SetError { + SetError::forbidden().with_description("You do not have permission to perform this operation.") +} + +fn blob_not_found() -> SetError { + SetError::invalid_properties() + .with_property(FileNodeProperty::BlobId) + .with_description("Blob could not be found.") +} + +pub(super) async fn fetch_archive( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result>> { + server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) + .await + .caused_by(trc::location!()) +} diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 2a196035..08467045 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -4,11 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::StoredIdentities; use crate::changes::state::StateManager; use common::{Server, storage::index::ObjectIndexBuilder}; use email::identity::{ArchivedEmailAddress, Identity}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::identity::{self, IdentityProperty, IdentityValue}, }; use jmap_tools::{Map, Value}; @@ -16,14 +17,13 @@ use std::{collections::BTreeSet, future::Future}; use store::{ SerializeInfallible, ValueKey, rkyv::{option::ArchivedOption, vec::ArchivedVec}, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, assert::AssertValue}, + write::{BatchBuilder, assert::AssertValue}, xxhash_rust::xxh3::Xxh3, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, - field::{Field, IdentityField, PrincipalField}, + field::{Field, PrincipalField}, }; pub trait IdentityGet: Sync + Send { @@ -35,7 +35,12 @@ pub trait IdentityGet: Sync + Send { fn identity_get_or_create( &self, account_id: u32, - ) -> impl Future> + Send; + ) -> impl Future> + Send; + + fn identities( + &self, + account_id: u32, + ) -> impl Future> + Send; } impl IdentityGet for Server { @@ -55,15 +60,14 @@ impl IdentityGet for Server { IdentityProperty::MayDelete, ]); let account_id = request.account_id.document_id(); - let identity_ids = self.identity_get_or_create(account_id).await?; + let identities = self.identity_get_or_create(account_id).await?; let ids = if let Some(ids) = ids { ids } else { - identity_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + identities.iter().map(|(document_id, _)| document_id.into()), + self.core.jmap.get_max_objects, + )? }; let mut response = GetResponse { account_id: request.account_id.into(), @@ -77,26 +81,11 @@ impl IdentityGet for Server { for id in ids { // Obtain the identity object - let document_id = id.document_id(); - if !identity_ids.contains(document_id) { - response.push_not_found(id); - continue; - } - let _identity = if let Some(identity) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Identity, - document_id, - )) - .await? - { - identity - } else { + let Some(identity) = identities.get(id.document_id()) else { response.push_not_found(id); continue; }; - let identity = _identity + let identity = identity .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); @@ -148,7 +137,7 @@ impl IdentityGet for Server { Ok(response) } - async fn identity_get_or_create(&self, account_id: u32) -> trc::Result { + async fn identity_get_or_create(&self, account_id: u32) -> trc::Result { // Obtain account info let account_info = self .account_info(account_id) @@ -176,45 +165,31 @@ impl IdentityGet for Server { )) .await .caused_by(trc::location!())?; + let identities = self.identities(account_id).await?; - let mut identity_ids = self - .document_ids(account_id, Collection::Identity, IdentityField::DocumentId) - .await?; if stored_hash == Some(addresses_hash) { - return Ok(identity_ids); + return Ok(identities); } // Determine which addresses are missing and which identities are no longer valid let member_of = &account_info.account().id_member_of; let mut missing_addresses = addresses.clone(); - let mut obsolete_ids = Vec::new(); - for document_id in &identity_ids { - if let Some(identity) = self - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Identity, - document_id, - )) + let mut obsolete = Vec::new(); + for (document_id, archive) in identities.iter() { + let identity = archive + .to_unarchived::() + .caused_by(trc::location!())?; + let email = identity.inner.email.as_str(); + + if addresses.contains(email) { + missing_addresses.remove(email); + } else if !self + .account_id_from_email(email, true) .await .caused_by(trc::location!())? + .is_some_and(|id| id == account_id || member_of.contains(&id)) { - let email = identity - .unarchive::() - .caused_by(trc::location!())? - .email - .as_str(); - - if addresses.contains(email) { - missing_addresses.remove(email); - } else if !self - .account_id_from_email(email, true) - .await - .caused_by(trc::location!())? - .is_some_and(|id| id == account_id || member_of.contains(&id)) - { - obsolete_ids.push(document_id); - } + obsolete.push((document_id, identity)); } } @@ -226,26 +201,19 @@ impl IdentityGet for Server { // Create identities for the new addresses if !missing_addresses.is_empty() { let name = account_info.description().unwrap_or(account_info.name()); - let mut next_document_id = self - .store() - .assign_document_ids( - account_id, - Collection::Identity, - missing_addresses.len() as u64, - ) - .await - .caused_by(trc::location!())?; - for email in missing_addresses { + let slots = batch.reserve_document_ids( + account_id, + Collection::Identity, + missing_addresses.len() as u32, + ); + for (email, slot) in missing_addresses.into_iter().zip(slots.iter()) { let name = if name.is_empty() { email.to_string() } else { name.to_string() }; - let document_id = next_document_id; - next_document_id -= 1; batch - .with_document(document_id) - .tag(IdentityField::DocumentId) + .create_document(slot) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Identity { name, email: email.to_string(), @@ -253,19 +221,16 @@ impl IdentityGet for Server { })) .caused_by(trc::location!())? .commit_point(); - identity_ids.insert(document_id); } } // Delete identities whose address no longer belongs to this account - for document_id in obsolete_ids { + for (document_id, identity) in obsolete { batch .with_document(document_id) - .untag(IdentityField::DocumentId) - .clear(Field::ARCHIVE) - .log_item_delete(SyncCollection::Identity, None) + .custom(ObjectIndexBuilder::<_, ()>::new().with_current(identity)) + .caused_by(trc::location!())? .commit_point(); - identity_ids.remove(document_id); } batch @@ -281,13 +246,29 @@ impl IdentityGet for Server { ); match self.commit_batch(batch).await { - Ok(_) => Ok(identity_ids), - Err(err) if err.is_assertion_failure() => self - .document_ids(account_id, Collection::Identity, IdentityField::DocumentId) - .await - .caused_by(trc::location!()), - Err(err) => Err(err.caused_by(trc::location!())), + Ok(_) => {} + Err(err) if err.is_assertion_failure() => {} + Err(err) => return Err(err.caused_by(trc::location!())), } + + self.identities(account_id).await + } + + async fn identities(&self, account_id: u32) -> trc::Result { + let mut identities = StoredIdentities::default(); + self.archives( + account_id, + Collection::Identity, + Field::ARCHIVE, + &(), + |document_id, archive| { + identities.push(document_id, archive); + Ok(true) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| identities) } } diff --git a/crates/jmap/src/identity/mod.rs b/crates/jmap/src/identity/mod.rs index f460abaa..f3241c62 100644 --- a/crates/jmap/src/identity/mod.rs +++ b/crates/jmap/src/identity/mod.rs @@ -4,5 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use store::write::{Archive, ArchiveBytes}; + pub mod get; pub mod set; + +#[derive(Default)] +pub struct StoredIdentities(Vec<(u32, Archive)>); + +impl StoredIdentities { + pub fn get(&self, document_id: u32) -> Option<&Archive> { + self.0 + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| self.0.get(idx)) + .map(|(_, archive)| archive) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> impl Iterator)> { + self.0.iter().map(|(id, archive)| (*id, archive)) + } + + fn push(&mut self, document_id: u32, archive: Archive) { + self.0.push((document_id, archive)); + } +} diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index f52cd714..07c7f3ac 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{StoredIdentities, get::IdentityGet}; +use crate::api::pending_creates::PendingCreates; use common::{Server, storage::index::ObjectIndexBuilder}; use email::identity::{EmailAddress, Identity}; use jmap_proto::{ @@ -19,12 +21,11 @@ use registry::schema::enums::StorageQuota; use std::future::Future; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, - field::{Field, IdentityField}, id::Id, }; use utils::sanitize_email; @@ -42,11 +43,14 @@ impl IdentitySet for Server { mut request: SetRequest<'_, identity::Identity>, ) -> trc::Result> { let account_id = request.account_id.document_id(); - let identity_ids = self - .document_ids(account_id, Collection::Identity, IdentityField::DocumentId) - .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + let identities = if request.has_creates() || !will_destroy.is_empty() { + self.identities(account_id).await? + } else { + StoredIdentities::default() + }; + let mut identity_count = identities.len() as u64; let account_info = self .account_info(account_id) .await @@ -54,6 +58,7 @@ impl IdentitySet for Server { // Process creates let mut batch = BatchBuilder::new(); + let mut pending_creates = PendingCreates::new(); 'create: for (id, object) in request.unwrap_create() { let mut identity = Identity::default(); @@ -97,7 +102,7 @@ impl IdentitySet for Server { } // Validate quota - if identity_ids.len() + if identity_count >= self.object_quota( account_info.object_quotas(), StorageQuota::MaxEmailIdentities, @@ -114,20 +119,16 @@ impl IdentitySet for Server { } // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Identity, 1) - .await - .caused_by(trc::location!())?; + let slot = batch.reserve_document_id(account_id, Collection::Identity); batch .with_account_id(account_id) .with_collection(Collection::Identity) - .with_document(document_id) - .tag(IdentityField::DocumentId) + .create_document(slot) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(identity)) .caused_by(trc::location!())? .commit_point(); - response.created(id, document_id); + identity_count += 1; + pending_creates.push(id, slot); } // Process updates @@ -149,7 +150,7 @@ impl IdentitySet for Server { let document_id = id.document_id(); let identity_ = if let Some(identity_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Identity, document_id, @@ -204,15 +205,19 @@ impl IdentitySet for Server { // Process deletions for id in will_destroy { let document_id = id.document_id(); - if identity_ids.contains(document_id) { - // Update record + if let Some(identity) = identities.get(document_id) { batch .with_account_id(account_id) .with_collection(Collection::Identity) .with_document(document_id) - .untag(IdentityField::DocumentId) - .clear(Field::ARCHIVE) - .log_item_delete(SyncCollection::Identity, None) + .custom( + ObjectIndexBuilder::<_, ()>::new().with_current( + identity + .to_unarchived::() + .caused_by(trc::location!())?, + ), + ) + .caused_by(trc::location!())? .commit_point(); response.destroyed.push(id); } else { @@ -222,13 +227,13 @@ impl IdentitySet for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; + let assigned_ids = self.commit_batch(batch).await.caused_by(trc::location!())?; - response.new_state = State::Exact(change_id).into(); + response.new_state = + State::Exact(assigned_ids.last_change_id(account_id, SyncCollection::Identity)) + .into(); + + pending_creates.resolve(&mut response, &assigned_ids); } Ok(response) diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 085de809..f32e5750 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -5,6 +5,7 @@ */ #![warn(clippy::large_futures)] +#![recursion_limit = "256"] use jmap_proto::object::JmapObjectId; use jmap_tools::{Element, Property, Value}; diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index c5e4062f..df5a38a8 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -7,21 +7,29 @@ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use email::cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::mailbox::{Mailbox, MailboxProperty, MailboxValue}, + request::capability::CapabilityIds, }; use jmap_tools::{Map, Value}; use std::future::Future; -use store::ahash::AHashSet; -use types::{acl::Acl, collection::Collection, keyword::Keyword, special_use::SpecialUse}; +use store::ahash::AHashMap; +use types::{acl::Acl, collection::Collection, id::Id, special_use::SpecialUse}; -use crate::{api::acl::JmapRights, changes::state::JmapCacheState}; +use crate::{ + api::{ + acl::JmapRights, + metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + }, + changes::state::JmapCacheState, +}; pub trait MailboxGet: Sync + Send { fn mailbox_get( &self, request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -30,23 +38,34 @@ impl MailboxGet for Server { &self, mut request: GetRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - MailboxProperty::Id, - MailboxProperty::Name, - MailboxProperty::ParentId, - MailboxProperty::Role, - MailboxProperty::SortOrder, - MailboxProperty::IsSubscribed, - MailboxProperty::TotalEmails, - MailboxProperty::UnreadEmails, - MailboxProperty::TotalThreads, - MailboxProperty::UnreadThreads, - MailboxProperty::MyRights, - ]); + let (properties, selection) = select_properties( + &mut request, + &[ + MailboxProperty::Id, + MailboxProperty::Name, + MailboxProperty::ParentId, + MailboxProperty::Role, + MailboxProperty::SortOrder, + MailboxProperty::IsSubscribed, + MailboxProperty::TotalEmails, + MailboxProperty::UnreadEmails, + MailboxProperty::TotalThreads, + MailboxProperty::UnreadThreads, + MailboxProperty::MyRights, + ], + using, + )?; + let object_metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Mailbox); + let metadata = object_metadata.get(selection); + let metadata_slots = metadata.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); let account_id = request.account_id.document_id(); let personal_id = access_token.personal_id(account_id, Collection::Mailbox); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let cache = self.get_cached_messages(account_id).await?; let shared_ids = if access_token.is_shared(account_id) { cache.shared_mailboxes(access_token, Acl::Read).into() @@ -56,22 +75,70 @@ impl MailboxGet for Server { let ids = if let Some(ids) = ids { ids } else { - cache - .mailboxes - .index - .keys() - .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) - .copied() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + cache + .mailboxes + .index + .keys() + .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) + .copied() + .map(Into::into), + self.core.jmap.get_max_objects, + )? }; let mut response = GetResponse { account_id: request.account_id.into(), - state: Some(cache.get_state(true)), + state: Some(sampled.state(cache.get_state(true))), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, }; + let mut metadata_values = match &metadata { + Some(metadata) => { + let mut documents = MetadataDocuments::default(); + for document_id in ids.iter().map(Id::document_id) { + if let Some(mailbox) = cache.mailbox_by_id(&document_id).filter(|_| { + shared_ids + .as_ref() + .is_none_or(|ids| ids.contains(document_id)) + }) { + documents.insert(document_id, mailbox.metadata_kinds); + } + } + Some( + object_metadata + .load::( + self, account_id, metadata, &documents, + ) + .await?, + ) + } + None => None, + }; + let count_threads = properties.iter().any(|property| { + matches!( + property, + MailboxProperty::TotalThreads | MailboxProperty::UnreadThreads + ) + }); + let counters = if count_threads + || properties.iter().any(|property| { + matches!( + property, + MailboxProperty::TotalEmails | MailboxProperty::UnreadEmails + ) + }) { + cache.mailbox_counters( + ids.iter().map(Id::document_id).filter(|document_id| { + cache.mailbox_by_id(document_id).is_some() + && shared_ids + .as_ref() + .is_none_or(|ids| ids.contains(*document_id)) + }), + count_threads, + ) + } else { + AHashMap::new() + }; for id in ids { // Obtain the mailbox object @@ -88,7 +155,8 @@ impl MailboxGet for Server { continue; }; - let mut mailbox = Map::with_capacity(properties.len()); + let mut mailbox = Map::with_capacity(properties.len() + metadata_slots); + let counter = counters.get(&document_id); for property in &properties { let value = match property { @@ -109,30 +177,17 @@ impl MailboxGet for Server { } } MailboxProperty::TotalEmails => { - Value::Number(cache.in_mailbox(document_id).count().into()) + Value::Number(counter.map_or(0, |c| c.total).into()) + } + MailboxProperty::UnreadEmails => { + Value::Number(counter.map_or(0, |c| c.unread).into()) + } + MailboxProperty::TotalThreads => { + Value::Number(counter.map_or(0, |c| c.threads.len()).into()) + } + MailboxProperty::UnreadThreads => { + Value::Number(counter.map_or(0, |c| c.unread_threads.len()).into()) } - MailboxProperty::UnreadEmails => Value::Number( - cache - .in_mailbox_without_keyword(document_id, &Keyword::Seen) - .count() - .into(), - ), - MailboxProperty::TotalThreads => Value::Number( - cache - .in_mailbox(document_id) - .map(|m| m.thread_id) - .collect::>() - .len() - .into(), - ), - MailboxProperty::UnreadThreads => Value::Number( - cache - .in_mailbox_without_keyword(document_id, &Keyword::Seen) - .map(|m| m.thread_id) - .collect::>() - .len() - .into(), - ), MailboxProperty::MyRights => { if access_token.is_shared(account_id) { JmapRights::rights::( @@ -156,7 +211,9 @@ impl MailboxGet for Server { mailbox.insert_unchecked(property.clone(), value); } - // Add result to response + if let Some(metadata_values) = &mut metadata_values { + metadata_values.insert_into(document_id, &mut mailbox); + } response.list.push(mailbox.into()); } Ok(response) diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index ef917b5b..d82a59a0 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,12 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; +use crate::{ + api::{ + metadata::{MetadataType, ObjectMetadata}, + query::QueryResponseBuilder, + }, + changes::state::JmapCacheState, +}; use common::{Server, auth::AccessToken}; use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse}, object::mailbox::{Mailbox, MailboxComparator, MailboxFilter}, + request::capability::CapabilityIds, }; use std::{collections::BTreeMap, future::Future}; use store::{ @@ -18,13 +25,15 @@ use store::{ search::{SearchComparator, SearchFilter, SearchQuery}, write::SearchIndex, }; -use types::{acl::Acl, collection::Collection, special_use::SpecialUse}; +use types::{acl::Acl, collection::Collection, metadata::MetadataKinds, special_use::SpecialUse}; +use utils::text::IgnoreCaseNeedle; pub trait MailboxQuery: Sync + Send { fn mailbox_query( &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -33,13 +42,46 @@ impl MailboxQuery for Server { &self, mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); let personal_id = access_token.personal_id(account_id, Collection::Mailbox); let sort_as_tree = request.arguments.sort_as_tree.unwrap_or(false); let filter_as_tree = request.arguments.filter_as_tree.unwrap_or(false); let mut filters = Vec::with_capacity(request.filter.len()); + let object_metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Mailbox); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let mailboxes = self.get_cached_messages(account_id).await?; + let readable: RoaringBitmap = if access_token.is_shared(account_id) { + mailboxes.shared_mailboxes(access_token, Acl::Read) + } else { + mailboxes + .mailboxes + .items + .iter() + .map(|m| m.document_id) + .collect() + }; + let metadata = + object_metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(MailboxFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; + let mut metadata_matches = object_metadata + .evaluate(self, account_id, &metadata, || { + mailboxes + .mailboxes + .items + .iter() + .filter(|mailbox| { + mailbox.metadata_kinds.contains(MetadataKinds::JMAP) + && readable.contains(mailbox.document_id) + }) + .map(|mailbox| mailbox.document_id) + .collect::() + }) + .await? + .into_iter(); for cond in std::mem::take(&mut request.filter) { match cond { @@ -60,20 +102,20 @@ impl MailboxQuery for Server { )); } MailboxFilter::Name(name) => { - #[cfg(any(feature = "dev_mode", feature = "test_mode"))] + #[cfg(any(feature = "dev_mode", stalwart_tests))] { // Used for concurrent requests tests if name == "__sleep" { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - let name = name.to_lowercase(); + let name = IgnoreCaseNeedle::new(&name); filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() - .filter(|mailbox| mailbox.name.to_lowercase().contains(&name)) + .filter(|mailbox| name.contains(&mailbox.name)) .map(|m| m.document_id) .collect::(), )); @@ -127,6 +169,9 @@ impl MailboxQuery for Server { .collect::(), )); } + MailboxFilter::Metadata(_) => filters.push(SearchFilter::is_in_set( + metadata_matches.next().unwrap_or_default(), + )), MailboxFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() @@ -157,7 +202,7 @@ impl MailboxQuery for Server { .mailboxes .items .iter() - .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) + .map(|mailbox| (&*mailbox.path, mailbox.document_id)) .collect::>(); comparators.push(SearchComparator::sorted_set( sorted_set @@ -182,7 +227,7 @@ impl MailboxQuery for Server { .mailboxes .items .iter() - .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) + .map(|mailbox| (&*mailbox.name, mailbox.document_id)) .collect::>(); SearchComparator::sorted_set( @@ -229,16 +274,7 @@ impl MailboxQuery for Server { let mut results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_comparators(comparators) - .with_mask(if access_token.is_shared(account_id) { - mailboxes.shared_mailboxes(access_token, Acl::Read) - } else { - mailboxes - .mailboxes - .items - .iter() - .map(|m| m.document_id) - .collect() - }) + .with_mask(readable) .filter(); // Filter as tree @@ -248,15 +284,17 @@ impl MailboxQuery for Server { for document_id in results.results() { let mut check_id = document_id; for _ in 0..self.core.email.mailbox_max_depth { - if let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) { - if let Some(parent_id) = mailbox.parent_id() { - if results.results().contains(parent_id) { - check_id = parent_id; - } else { - break; - } - } else { + let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) else { + break; + }; + match mailbox.parent_id() { + Some(parent_id) if results.results().contains(parent_id) => { + check_id = parent_id; + } + Some(_) => break, + None => { new_results.insert(document_id); + break; } } } @@ -268,7 +306,7 @@ impl MailboxQuery for Server { let mut response = QueryResponseBuilder::new( results.results().len() as usize, self.core.jmap.query_max_results, - mailboxes.get_state(true), + sampled.state(mailboxes.get_state(true)), &request, ); diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 793fdc8b..6929a6df 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -5,27 +5,44 @@ */ use crate::{ - api::acl::{JmapAcl, JmapRights}, + api::{ + acl::{JmapAcl, JmapRights}, + metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, + ObjectMetadata, is_empty_update, + }, + parent_ref::{CreateResolver, ParentRef}, + }, changes::state::JmapCacheState, }; use common::{ - Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, + Server, + auth::AccessToken, + sharing::EffectiveAcl, + storage::index::{ObjectIndexBuilder, PresenceFlags, RewritePresence}, }; -#[allow(unused_imports)] -use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID, UidMailbox}; use email::{ - cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, + cache::{ + MessageCacheFetch, + email::{MailboxCounters, MessageCacheAccess}, + mailbox::MailboxCacheAccess, + }, mailbox::{ Mailbox, destroy::{MailboxDestroy, MailboxDestroyError}, + merge_subscription, + role::RoleChange, }, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, - object::mailbox::{self, MailboxProperty, MailboxValue}, + object::{ + AnyId, + mailbox::{self, MailboxProperty, MailboxValue}, + }, references::resolve::ResolveCreatedReference, - request::MaybeInvalid, + request::{MaybeInvalid, capability::CapabilityIds}, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Map, Value}; @@ -33,14 +50,21 @@ use registry::schema::enums::StorageQuota; use std::future::Future; use store::{ ValueKey, + ahash::AHashMap, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, assert::AssertValue}, + write::{Archive, ArchiveBytes, BatchBuilder, Slot, assert::AssertValue}, }; use trc::AddContext; use types::{ - acl::Acl, collection::Collection, field::MailboxField, id::Id, special_use::SpecialUse, + acl::{Acl, AclGrant}, + collection::{Collection, SyncCollection}, + field::MailboxField, + id::Id, + special_use::SpecialUse, }; +const SERVER_SET_MISMATCH: &str = "Server-set property does not match the current value."; + pub struct SetContext<'x> { account_id: u32, access_token: &'x AccessToken, @@ -48,6 +72,7 @@ pub struct SetContext<'x> { response: SetResponse, mailbox_ids: RoaringBitmap, will_destroy: Vec, + counters: AHashMap, } pub trait MailboxSet: Sync + Send { @@ -55,34 +80,87 @@ pub trait MailboxSet: Sync + Send { &self, request: SetRequest<'_, mailbox::Mailbox>, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; + #[allow(clippy::type_complexity)] fn mailbox_set_item( &self, changes_: Map<'_, MailboxProperty, MailboxValue>, update: Option<(u32, Archive)>, ctx: &SetContext, + resolver: Option<&CreateResolver<'_>>, ) -> impl Future< Output = trc::Result< - Result, SetError>, + Result< + (ObjectIndexBuilder, Mailbox>, ParentRef), + SetError, + >, >, > + Send; } +impl ResolveCreatedReference for CreateResolver<'_> { + fn get_created_id(&self, id_ref: &str) -> Option { + self.created_id(id_ref) + } +} + impl MailboxSet for Server { #[allow(clippy::blocks_in_conditions)] async fn mailbox_set( &self, mut request: SetRequest<'_, mailbox::Mailbox>, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { // Prepare response let account_id = request.account_id.document_id(); let on_destroy_remove_emails = request.arguments.on_destroy_remove_emails.unwrap_or(false); + let object_metadata = ObjectMetadata::new(self, access_token, using, MetadataType::Mailbox); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let cache = self.get_cached_messages(account_id).await?; + let shared_state = cache.get_state(true); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state(cache.assert_state(true, &request.if_in_state)?); + .with_state(sampled.assert_state(shared_state.clone(), &request.if_in_state)?); + let account_info = self.account(account_id).await?; + let mut metadata_writer = + MetadataWriter::new(object_metadata, account_id).with_owner(account_info.clone()); + metadata_writer + .preload_updates(self, request.update.as_ref(), |document_id| { + cache + .mailbox_by_id(&document_id) + .map(|mailbox| mailbox.metadata_kinds) + }) + .await?; let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + let mut count_threads = false; + let mut counter_ids = Vec::new(); + for (id, object) in request.update.iter().flat_map(|update| update.iter()) { + if let (MaybeInvalid::Value(id), Value::Object(object)) = (id, object) { + let mut has_counters = false; + for key in object.keys() { + match key { + Key::Property( + MailboxProperty::TotalEmails | MailboxProperty::UnreadEmails, + ) => { + has_counters = true; + } + Key::Property( + MailboxProperty::TotalThreads | MailboxProperty::UnreadThreads, + ) => { + has_counters = true; + count_threads = true; + } + _ => {} + } + } + if has_counters { + counter_ids.push(id.document_id()); + } + } + } + let counters = cache.mailbox_counters(counter_ids, count_threads); let mut ctx = SetContext { account_id, is_shared: access_token.is_shared(account_id), @@ -90,19 +168,28 @@ impl MailboxSet for Server { response, mailbox_ids: RoaringBitmap::from_iter(cache.mailboxes.index.keys()), will_destroy, + counters, }; let mut change_id = None; - let account_info = self.account(account_id).await?; // Process creates let mut batch = BatchBuilder::new(); - 'create: for (id, object) in request.unwrap_create() { + let mut pending_creates: AHashMap = AHashMap::new(); + 'create: for (id, mut object) in request.unwrap_create() { + let metadata_patches = + match object_metadata.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches, + Err(err) => { + ctx.response.not_created.append(id, err); + continue 'create; + } + }; let Some(object) = object.into_object() else { continue; }; // Validate quota - if ctx.mailbox_ids.len() + if ctx.mailbox_ids.len() + pending_creates.len() as u64 >= self.object_quota(account_info.object_quotas(), StorageQuota::MaxMailboxes) as u64 { @@ -116,33 +203,69 @@ impl MailboxSet for Server { continue 'create; } - match self.mailbox_set_item(object, None, &ctx).await? { - Ok(builder) => { + match self + .mailbox_set_item( + object, + None, + &ctx, + Some(&CreateResolver::new(&pending_creates)), + ) + .await? + { + Ok((mut builder, parent)) => { + let metadata_writes = if let Some(metadata_patches) = metadata_patches { + let access = if ctx.is_shared { + MetadataAccess { + may_write_shared: false, + may_read: false, + } + } else { + MetadataAccess::FULL + }; + match metadata_writer + .prepare_for(self, metadata_patches, access, MetadataTarget::Create) + .await? + { + Ok(prepared) => { + if let Some((kinds, mailbox)) = + prepared.shared_kinds().zip(builder.changes_mut()) + { + mailbox.set_metadata_kinds(kinds); + } + Some(prepared) + } + Err(err) => { + ctx.response.not_created.append(id, err); + continue 'create; + } + } + } else { + None + }; + batch .with_account_id(account_id) .with_collection(Collection::Mailbox); - let parent_id = builder.changes().unwrap().parent_id; - if parent_id > 0 { + if let Some(parent_document_id) = parent.document_id() { batch - .with_document(parent_id - 1) + .with_document(parent_document_id) .assert_value(MailboxField::Archive, AssertValue::Some); } - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Mailbox, 1) - .await - .caused_by(trc::location!())?; - + let slot = batch.reserve_document_id(account_id, Collection::Mailbox); batch - .with_document(document_id) - .custom(builder) - .caused_by(trc::location!())? - .commit_point(); + .create_document(slot) + .custom(builder.with_pending_id_opt(parent.slot())) + .caused_by(trc::location!())?; + if let Some(prepared) = metadata_writes { + metadata_writer + .write(prepared, slot, &mut batch) + .caused_by(trc::location!())?; + } + batch.commit_point(); - ctx.mailbox_ids.insert(document_id); - ctx.response.created(id, document_id); + pending_creates.insert(id, slot); } Err(err) => { ctx.response.not_created.append(id, err); @@ -152,18 +275,25 @@ impl MailboxSet for Server { } if !batch.is_empty() { - change_id = self - .commit_batch(batch) + let assigned_ids = metadata_writer + .commit(self, batch) .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())? + .caused_by(trc::location!())?; + change_id = assigned_ids + .last_change_id(account_id, SyncCollection::Email) .into(); + + for (id, slot) in pending_creates { + let document_id = assigned_ids.slot(slot); + ctx.mailbox_ids.insert(document_id); + ctx.response.created(id, document_id); + } } // Process updates let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); let mut batch = BatchBuilder::new(); - 'update: for (id, object) in request.unwrap_update() { + 'update: for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -180,15 +310,23 @@ impl MailboxSet for Server { .append(id, SetError::will_destroy()); continue 'update; } + let metadata_patches = + match object_metadata.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + ctx.response.not_updated.append(id, err); + continue 'update; + } + }; let Some(object) = object.into_object() else { continue 'update; }; // Obtain mailbox let document_id = id.document_id(); - if let Some(mailbox) = self + if let Some(mailbox_archive) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, document_id, @@ -196,18 +334,31 @@ impl MailboxSet for Server { .await? { // Validate ACL - let mailbox = mailbox + let mailbox = mailbox_archive .into_deserialized::() .caused_by(trc::location!())?; - if ctx.is_shared { - let acl = mailbox.inner.acls.effective_acl(access_token); - let subscription_only = object.keys().all(|key| { - matches!( - key, - Key::Property(MailboxProperty::IsSubscribed | MailboxProperty::Id) + let subscription_only = object.keys().all(|key| { + matches!( + key, + Key::Property( + MailboxProperty::IsSubscribed + | MailboxProperty::Id + | MailboxProperty::MyRights + | MailboxProperty::TotalEmails + | MailboxProperty::UnreadEmails + | MailboxProperty::TotalThreads + | MailboxProperty::UnreadThreads ) - }); - if subscription_only { + ) + }); + let acl = ctx + .is_shared + .then(|| mailbox.inner.acls.effective_acl(access_token)); + if let Some(acl) = acl { + if metadata_patches.is_some() && !acl.contains(Acl::Read) { + ctx.response.not_updated.append(id, SetError::not_found()); + continue 'update; + } else if subscription_only { if !acl.contains(Acl::Read) { ctx.response.not_updated.append( id, @@ -237,27 +388,128 @@ impl MailboxSet for Server { } } + let metadata_writes = if let Some(metadata_patches) = metadata_patches { + let access = MetadataAccess { + may_write_shared: acl.is_none_or(|acl| acl.contains(Acl::ModifyMetadata)), + may_read: acl.is_none_or(|acl| acl.contains(Acl::Read)), + }; + match metadata_writer + .prepare_for( + self, + metadata_patches, + access, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(prepared) => (!prepared.is_empty()).then_some(prepared), + Err(err) => { + ctx.response.not_updated.append(id, err); + continue 'update; + } + } + } else { + None + }; + + if is_empty_update::(&object, id) { + match metadata_writes { + Some(prepared) => { + if let Some(presence) = metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())? + { + Mailbox::rewrite_presence( + &mailbox_archive + .to_unarchived::() + .caused_by(trc::location!())?, + presence.after, + account_id, + document_id, + &mut batch, + ) + .caused_by(trc::location!())?; + } + batch.commit_point(); + will_update.push(id); + } + None => { + ctx.response.updated.append(id, None); + } + } + continue 'update; + } + match self - .mailbox_set_item(object, (document_id, mailbox).into(), &ctx) + .mailbox_set_item(object, (document_id, mailbox).into(), &ctx, None) .await? { - Ok(builder) => { + Ok((mut builder, parent)) => { batch .with_account_id(account_id) .with_collection(Collection::Mailbox); - let parent_id = builder.changes().unwrap().parent_id; - if parent_id > 0 { - batch - .with_document(parent_id - 1) - .assert_value(MailboxField::Archive, AssertValue::Some); - } + if subscription_only { + let subscriber = access_token.account_id(); + let subscribe = builder.changes().unwrap().is_subscribed(subscriber); + let subscription_changed = builder.current().is_some_and(|current| { + current.inner.is_subscribed(subscriber) != subscribe + }); - batch - .with_document(document_id) - .custom(builder) - .caused_by(trc::location!())? - .commit_point(); + if !subscription_changed && metadata_writes.is_none() { + ctx.response.updated.append(id, None); + continue 'update; + } + + if let Some(prepared) = metadata_writes + && let Some(presence) = metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())? + { + Mailbox::rewrite_presence( + &mailbox_archive + .to_unarchived::() + .caused_by(trc::location!())?, + presence.after, + account_id, + document_id, + &mut batch, + ) + .caused_by(trc::location!())?; + } + if subscription_changed { + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox) + .with_document(document_id); + merge_subscription(&mut batch, subscriber, subscribe); + } + batch.commit_point(); + } else { + if let Some(kinds) = metadata_writes + .as_ref() + .and_then(|prepared| prepared.shared_kinds()) + && let Some(mailbox) = builder.changes_mut() + { + mailbox.set_metadata_kinds(kinds); + } + if let Some(parent_document_id) = parent.document_id() { + batch + .with_document(parent_document_id) + .assert_value(MailboxField::Archive, AssertValue::Some); + } + + batch + .with_document(document_id) + .custom(builder) + .caused_by(trc::location!())?; + if let Some(prepared) = metadata_writes { + metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())?; + } + batch.commit_point(); + } will_update.push(id); } Err(err) => { @@ -271,13 +523,10 @@ impl MailboxSet for Server { } if !batch.is_empty() { - match self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - { - Ok(change_id_) => { - change_id = Some(change_id_); + match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => { + change_id = + Some(assigned_ids.last_change_id(account_id, SyncCollection::Email)); for id in will_update { ctx.response.updated.append(id, None); } @@ -299,6 +548,23 @@ impl MailboxSet for Server { } // Process deletions + let destroy_containers = self + .preload_container_cleanup( + access_token.changed_by(), + account_id, + Collection::Mailbox, + &ctx.will_destroy + .iter() + .map(|id| id.document_id()) + .filter(|document_id| { + cache + .mailbox_by_id(document_id) + .is_some_and(|mailbox| !mailbox.metadata_kinds.is_empty()) + }) + .collect(), + ) + .await + .caused_by(trc::location!())?; for id in ctx.will_destroy { match self .mailbox_destroy( @@ -306,6 +572,7 @@ impl MailboxSet for Server { id.document_id(), ctx.access_token, on_destroy_remove_emails, + Some(&destroy_containers), ) .await? { @@ -346,8 +613,12 @@ impl MailboxSet for Server { } // Write changes - if let Some(change_id) = change_id { - ctx.response.new_state = State::Exact(change_id).into(); + if change_id.is_some() { + ctx.response.new_state = object_metadata + .viewer_change_id(self, account_id) + .await? + .state(change_id.map_or(shared_state, State::Exact)) + .into(); } Ok(ctx.response) @@ -359,15 +630,25 @@ impl MailboxSet for Server { changes_: Map<'_, MailboxProperty, MailboxValue>, update: Option<(u32, Archive)>, ctx: &SetContext<'_>, - ) -> trc::Result, SetError>> { + resolver: Option<&CreateResolver<'_>>, + ) -> trc::Result< + Result< + (ObjectIndexBuilder, Mailbox>, ParentRef), + SetError, + >, + > { // Parse properties let mut changes = update .as_ref() .map(|(_, obj)| obj.inner.clone()) .unwrap_or_else(|| Mailbox::new(String::new())); + let mut parent = ParentRef::from_stored(changes.parent_id); let mut has_acl_changes = false; for (property, mut value) in changes_.into_vec() { - if let Err(err) = ctx.response.resolve_self_references(&mut value, 0, false) { + if let Err(err) = match resolver { + Some(resolver) => resolver.resolve_self_references(&mut value, 0, false), + None => ctx.response.resolve_self_references(&mut value, 0, false), + } { return Ok(Err(err)); }; match (&property, value) { @@ -392,18 +673,23 @@ impl MailboxSet for Server { Key::Property(MailboxProperty::ParentId), Value::Element(MailboxValue::Id(value)), ) => { - let parent_id = value.document_id(); - if ctx.will_destroy.contains(&value) { - return Ok(Err(SetError::will_destroy() - .with_description("Parent ID will be destroyed."))); - } else if !ctx.mailbox_ids.contains(parent_id) { + let Some(parent_ref) = ParentRef::from_client_id(value, resolver) else { return Ok(Err(SetError::invalid_properties() .with_description("Parent ID does not exist."))); + }; + if let Some(parent_id) = parent_ref.document_id() { + if ctx.will_destroy.contains(&value) { + return Ok(Err(SetError::will_destroy() + .with_description("Parent ID will be destroyed."))); + } else if !ctx.mailbox_ids.contains(parent_id) { + return Ok(Err(SetError::invalid_properties() + .with_description("Parent ID does not exist."))); + } } - changes.parent_id = parent_id + 1; + parent = parent_ref; } (Key::Property(MailboxProperty::ParentId), Value::Null) => { - changes.parent_id = 0; + parent = ParentRef::ROOT; } (Key::Property(MailboxProperty::IsSubscribed), Value::Bool(subscribe)) => { let account_id = ctx @@ -421,13 +707,18 @@ impl MailboxSet for Server { Key::Property(MailboxProperty::Role), Value::Element(MailboxValue::Role(role)), ) => { - changes.role = role; + changes.set_role(role); } (Key::Property(MailboxProperty::Role), Value::Null) => { - changes.role = SpecialUse::None; + changes.set_role(SpecialUse::None); } - (Key::Property(MailboxProperty::SortOrder), Value::Number(value)) => { - changes.sort_order = Some(value.cast_to_u64() as u32); + (Key::Property(MailboxProperty::SortOrder), Value::Number(value)) + if let Some(sort_order) = value + .as_u64() + .and_then(|sort_order| u32::try_from(sort_order).ok()) + .filter(|sort_order| *sort_order < 1 << 31) => + { + changes.sort_order = Some(sort_order); } (Key::Property(MailboxProperty::ShareWith), value) => { match JmapRights::acl_set::(value) { @@ -475,6 +766,53 @@ impl MailboxSet for Server { .with_description("The id property is immutable.".to_string()))); } } + (Key::Property(MailboxProperty::MyRights), value) + if let Some((_, current)) = &update => + { + let rights = if ctx.is_shared { + JmapRights::rights::( + current.inner.acls.effective_acl(ctx.access_token), + ) + } else { + JmapRights::all_rights::() + }; + let is_current = match (&value, &rights) { + (Value::Object(requested), Value::Object(rights)) => { + requested.len() == rights.len() + && requested + .iter() + .all(|(right, value)| rights.get(right) == Some(value)) + } + _ => false, + }; + if !is_current { + return Ok(Err(SetError::invalid_properties() + .with_property(MailboxProperty::MyRights) + .with_description(SERVER_SET_MISMATCH))); + } + } + ( + Key::Property( + counter @ (MailboxProperty::TotalEmails + | MailboxProperty::UnreadEmails + | MailboxProperty::TotalThreads + | MailboxProperty::UnreadThreads), + ), + Value::Number(value), + ) if let Some((document_id, _)) = &update => { + let counters = ctx.counters.get(document_id); + let current = match counter { + MailboxProperty::TotalEmails => counters.map_or(0, |c| c.total), + MailboxProperty::UnreadEmails => counters.map_or(0, |c| c.unread), + MailboxProperty::TotalThreads => counters.map_or(0, |c| c.threads.len()), + _ => counters.map_or(0, |c| c.unread_threads.len()), + }; + if value.as_u64() != u64::try_from(current).ok() { + return Ok(Err(SetError::invalid_properties() + .with_property(counter.clone()) + .with_description(SERVER_SET_MISMATCH))); + } + } _ => { return Ok(Err(SetError::invalid_properties() .with_property(property.into_owned()) @@ -483,10 +821,15 @@ impl MailboxSet for Server { } } - // Validate depth and circular parent-child relationship - if update - .as_ref() - .is_none_or(|(_, m)| m.inner.parent_id != changes.parent_id) + changes.parent_id = parent.as_stored(); + let cached_mailboxes = self.get_cached_messages(ctx.account_id).await?; + + // Validate depth and circular parent-child relationship. A parent created within + // this request has no ancestors in storage yet, so there is nothing to walk. + if !parent.is_pending() + && update + .as_ref() + .is_none_or(|(_, m)| m.inner.parent_id != changes.parent_id) { let mut mailbox_parent_id = changes.parent_id; let current_mailbox_id = update @@ -508,22 +851,12 @@ impl MailboxSet for Server { } let parent_document_id = mailbox_parent_id - 1; - if let Some(mailbox_) = self - .store() - .get_value::>(ValueKey::archive( - ctx.account_id, - Collection::Mailbox, - parent_document_id, - )) - .await? - { - let mailbox = mailbox_ - .unarchive::() - .caused_by(trc::location!())?; + if let Some(mailbox) = cached_mailboxes.mailbox_by_id(&parent_document_id) { if depth == 0 && ctx.is_shared && !mailbox .acls + .as_slice() .effective_acl(ctx.access_token) .contains(Acl::CreateChild) { @@ -532,7 +865,7 @@ impl MailboxSet for Server { ))); } - mailbox_parent_id = mailbox.parent_id.into(); + mailbox_parent_id = mailbox.parent_id().map_or(0, |parent_id| parent_id + 1); } else if ctx.mailbox_ids.contains(parent_document_id) { // Parent mailbox is probably created within the same request success = true; @@ -553,34 +886,20 @@ impl MailboxSet for Server { } } - let cached_mailboxes = self.get_cached_messages(ctx.account_id).await?; - - // Verify that the mailbox role is unique. - if update - .as_ref() - .is_none_or(|(_, m)| m.inner.role != changes.role) - { - if !matches!(changes.role, SpecialUse::None) - && cached_mailboxes.mailbox_by_role(&changes.role).is_some() - { - return Ok(Err(SetError::invalid_properties() - .with_property(MailboxProperty::Role) - .with_description(format!( - "A mailbox with role '{}' already exists.", - changes.role.as_str().unwrap_or_default() - )))); - } - - // Role of internal folders cannot be modified - if update.as_ref().is_some_and(|(document_id, _)| { - *document_id == INBOX_ID || *document_id == TRASH_ID || *document_id == JUNK_ID - }) { - return Ok(Err(SetError::invalid_properties() - .with_property(MailboxProperty::Role) - .with_description( - "You are not allowed to change the role of Inbox, Junk or Trash folders.", - ))); - } + let role_change = match &update { + Some((document_id, current)) => RoleChange::Update { + document_id: *document_id, + current: current.inner.role(), + role: changes.role(), + }, + None => RoleChange::Create { + role: changes.role(), + }, + }; + if let Err(err) = role_change.validate(&cached_mailboxes) { + return Ok(Err(SetError::invalid_properties() + .with_property(MailboxProperty::Role) + .with_description(err.to_string()))); } // Verify that the mailbox name is unique. @@ -590,11 +909,13 @@ impl MailboxSet for Server { let current_id = update.as_ref().map(|(document_id, _)| *document_id); if update.as_ref().is_none_or(|(_, m)| { m.inner.name != changes.name || m.inner.parent_id != changes.parent_id - }) && let Some(existing) = cached_mailboxes.mailboxes.items.iter().find(|m| { - Some(m.document_id) != current_id - && m.parent_id().map_or(0, |id| id + 1) == changes.parent_id - && m.name.to_lowercase() == lower_name - }) { + }) && let Some(parent_cache_id) = parent.cache_id() + && let Some(existing) = cached_mailboxes.mailboxes.items.iter().find(|m| { + Some(m.document_id) != current_id + && m.parent_id() == parent_cache_id + && m.name.to_lowercase() == lower_name + }) + { return Ok(Err(SetError::already_exists() .with_existing_id(Id::from(existing.document_id)) .with_description(format!( @@ -608,26 +929,30 @@ impl MailboxSet for Server { .with_description("Mailbox name cannot be empty."))); } - // Refresh ACLs + // Validate ACLs let current = update.map(|(_, current)| current); - if has_acl_changes { - if !changes.acls.is_empty() - && let Err(err) = self.acl_validate(&changes.acls).await - { - return Ok(Err(err.into())); - } - - self.refresh_acls( - &changes.acls, - current.as_ref().map(|m| m.inner.acls.as_slice()), - ) - .await - .caused_by(trc::location!())?; + if has_acl_changes + && !changes.acls.is_empty() + && let Err(err) = self + .acl_validate( + ctx.access_token, + ctx.account_id, + &changes.acls, + current + .as_ref() + .map_or(&[] as &[AclGrant], |current| current.inner.acls.as_slice()), + ) + .await + { + return Ok(Err(err.into())); } // Validate - Ok(Ok(ObjectIndexBuilder::new() - .with_changes(changes) - .with_current_opt(current))) + Ok(Ok(( + ObjectIndexBuilder::new() + .with_changes(changes) + .with_current_opt(current), + parent, + ))) } } diff --git a/crates/jmap/src/participant_identity/changes.rs b/crates/jmap/src/participant_identity/changes.rs new file mode 100644 index 00000000..3a9821d6 --- /dev/null +++ b/crates/jmap/src/participant_identity/changes.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{changes::get::IntermediateChangesResponse, participant_identity::set::identity_state}; +use common::{Server, auth::AccessToken}; +use groupware::calendar::ParticipantIdentities; +use jmap_proto::{ + method::changes::{ChangesRequest, ChangesResponse}, + request::method::MethodObject, + types::state::State, +}; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{collection::Collection, field::PrincipalField, id::Id}; + +pub trait ParticipantIdentityChanges: Sync + Send { + fn participant_identity_changes( + &self, + request: ChangesRequest, + access_token: &AccessToken, + ) -> impl Future> + Send; +} + +impl ParticipantIdentityChanges for Server { + async fn participant_identity_changes( + &self, + request: ChangesRequest, + access_token: &AccessToken, + ) -> trc::Result { + let identities = self + .store() + .get_value::>(ValueKey::property( + access_token.account_id(), + Collection::Principal, + 0, + PrincipalField::ParticipantIdentities, + )) + .await + .caused_by(trc::location!())? + .map(|archive| archive.deserialize::()) + .transpose() + .caused_by(trc::location!())? + .unwrap_or_default(); + let since = match &request.since_state { + State::Initial => 0, + State::Exact(change_id) => u32::try_from(*change_id) + .map_err(|_| trc::JmapEvent::CannotCalculateChanges.into_err())?, + State::Intermediate(_) => { + return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); + } + }; + let max_changes = match request.max_changes { + Some(0) => { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details("maxChanges must be greater than 0.")); + } + Some(max_changes) => max_changes.min(self.core.jmap.changes_max_results), + None => self.core.jmap.changes_max_results, + }; + + let changes = identities + .changes_since(since, max_changes) + .ok_or_else(|| trc::JmapEvent::CannotCalculateChanges.into_err())?; + + Ok(IntermediateChangesResponse { + response: ChangesResponse { + account_id: request.account_id, + old_state: request.since_state, + new_state: identity_state(changes.new_change_id), + has_more_changes: changes.has_more_changes, + created: changes.created.into_iter().map(Id::from).collect(), + updated: changes.updated.into_iter().map(Id::from).collect(), + destroyed: changes.destroyed.into_iter().map(Id::from).collect(), + updated_properties: None, + }, + object: MethodObject::ParticipantIdentity, + updated_properties: None, + }) + } +} diff --git a/crates/jmap/src/participant_identity/get.rs b/crates/jmap/src/participant_identity/get.rs index 00eb39a4..8f871fb3 100644 --- a/crates/jmap/src/participant_identity/get.rs +++ b/crates/jmap/src/participant_identity/get.rs @@ -4,16 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; +use crate::participant_identity::set::identity_state; +use common::{Server, auth::AccessToken}; use groupware::calendar::{ParticipantIdentities, ParticipantIdentity}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::participant_identity::{self, ParticipantIdentityProperty, ParticipantIdentityValue}, + types::state::State, }; use jmap_tools::{Map, Value}; use store::{ Serialize, ValueKey, - write::{AlignedBytes, Archive, Archiver, BatchBuilder}, + write::{Archive, ArchiveBytes, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField, id::Id}; @@ -22,18 +24,20 @@ pub trait ParticipantIdentityGet: Sync + Send { fn participant_identity_get( &self, request: GetRequest, + access_token: &AccessToken, ) -> impl Future>> + Send; fn participant_identity_get_or_create( &self, account_id: u32, - ) -> impl Future>>> + Send; + ) -> impl Future>>> + Send; } impl ParticipantIdentityGet for Server { async fn participant_identity_get( &self, mut request: GetRequest, + access_token: &AccessToken, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ @@ -42,12 +46,13 @@ impl ParticipantIdentityGet for Server { ParticipantIdentityProperty::CalendarAddress, ParticipantIdentityProperty::IsDefault, ]); - let account_id = request.account_id.document_id(); - let identities = self.participant_identity_get_or_create(account_id).await?; + let identities = self + .participant_identity_get_or_create(access_token.account_id()) + .await?; let mut response = GetResponse { account_id: request.account_id.into(), - state: None, + state: Some(State::Initial), list: Vec::new(), not_found: not_found_ids, }; @@ -62,16 +67,18 @@ impl ParticipantIdentityGet for Server { let identities = identities .unarchive::() .caused_by(trc::location!())?; + response.state = Some(identity_state(identities.change_id.to_native())); let ids = if let Some(ids) = ids { ids } else { - identities - .identities - .iter() - .take(self.core.jmap.get_max_objects) - .map(|i| Id::from(i.id.to_native())) - .collect::>() + all_ids( + identities + .identities + .iter() + .map(|i| Id::from(i.id.to_native())), + self.core.jmap.get_max_objects, + )? }; for id in ids { @@ -115,10 +122,10 @@ impl ParticipantIdentityGet for Server { async fn participant_identity_get_or_create( &self, account_id: u32, - ) -> trc::Result>> { + ) -> trc::Result>> { if let Some(identities) = self .store() - .get_value::>(ValueKey::property( + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, @@ -150,6 +157,7 @@ impl ParticipantIdentityGet for Server { .collect(), default: 0, default_name: name.to_string(), + ..Default::default() }; let mut batch = BatchBuilder::new(); @@ -167,7 +175,7 @@ impl ParticipantIdentityGet for Server { self.commit_batch(batch).await.caused_by(trc::location!())?; self.store() - .get_value::>(ValueKey::property( + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, diff --git a/crates/jmap/src/participant_identity/mod.rs b/crates/jmap/src/participant_identity/mod.rs index f460abaa..93448a2b 100644 --- a/crates/jmap/src/participant_identity/mod.rs +++ b/crates/jmap/src/participant_identity/mod.rs @@ -4,5 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod changes; pub mod get; pub mod set; diff --git a/crates/jmap/src/participant_identity/set.rs b/crates/jmap/src/participant_identity/set.rs index e43041db..be7ae60b 100644 --- a/crates/jmap/src/participant_identity/set.rs +++ b/crates/jmap/src/participant_identity/set.rs @@ -5,9 +5,9 @@ */ use crate::participant_identity::get::ParticipantIdentityGet; -use common::Server; +use common::{Server, auth::AccessToken, ipc::PushNotification}; use groupware::{ - calendar::{ParticipantIdentities, ParticipantIdentity}, + calendar::{ParticipantIdentities, ParticipantIdentity, ParticipantIdentityChangeType}, strip_mailto_scheme, }; use jmap_proto::{ @@ -15,22 +15,29 @@ use jmap_proto::{ method::set::{SetRequest, SetResponse}, object::participant_identity::{self, ParticipantIdentityProperty, ParticipantIdentityValue}, request::{MaybeInvalid, reference::MaybeIdReference}, + types::state::State, }; use jmap_tools::{Key, Value}; use registry::schema::prelude::StorageQuota; use store::{ - Serialize, + Serialize, ValueKey, ahash::AHashSet, - write::{Archiver, BatchBuilder}, + write::{Archive, ArchiveBytes, Archiver, BatchBuilder}, }; use trc::AddContext; -use types::{collection::Collection, field::PrincipalField, id::Id}; +use types::{ + collection::Collection, + field::PrincipalField, + id::Id, + type_state::{DataType, StateChange}, +}; use utils::sanitize_email; pub trait ParticipantIdentitySet: Sync + Send { fn participant_identity_set( &self, request: SetRequest<'_, participant_identity::ParticipantIdentity>, + access_token: &AccessToken, ) -> impl Future>> + Send; } @@ -38,10 +45,15 @@ impl ParticipantIdentitySet for Server { async fn participant_identity_set( &self, mut request: SetRequest<'_, participant_identity::ParticipantIdentity>, + access_token: &AccessToken, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + if !access_token.is_account_id(account_id) { + return reject_identity_changes(self, request, response, will_destroy, access_token) + .await; + } let (identity_archive, mut identities) = match self.participant_identity_get_or_create(account_id).await? { Some(archive) => { @@ -54,6 +66,16 @@ impl ParticipantIdentitySet for Server { None => (None, ParticipantIdentities::default()), }; + let old_state = identity_state(identities.change_id); + if request + .if_in_state + .as_ref() + .is_some_and(|if_in_state| if_in_state != &old_state) + { + return Err(trc::JmapEvent::StateMismatch.into_err()); + } + response = response.with_state(old_state); + let account_info = self .account_info(account_id) .await @@ -68,8 +90,18 @@ impl ParticipantIdentitySet for Server { // Process creates let mut has_changes = false; + let mut new_default = None; + let previous_default = identities.default; + let mut created_ids = Vec::new(); 'create: for (id, object) in request.unwrap_create() { let mut identity = ParticipantIdentity::default(); + let client_name = object + .as_object_and_get(&Key::Property(ParticipantIdentityProperty::Name)) + .is_some(); + let client_address = object + .as_object_and_get(&Key::Property(ParticipantIdentityProperty::CalendarAddress)) + .and_then(|value| value.as_str()) + .map(|value| value.into_owned()); if let Err(err) = validate_identity_value(None, object, &mut identity, &allowed_emails) { @@ -82,12 +114,7 @@ impl ParticipantIdentitySet for Server { .iter() .any(|i| i.calendar_address == identity.calendar_address) { - response.not_created.append( - id, - SetError::invalid_properties() - .with_property(ParticipantIdentityProperty::CalendarAddress) - .with_description("Calendar address already in use.".to_string()), - ); + response.not_created.append(id, address_in_use()); continue 'create; } @@ -122,11 +149,36 @@ impl ParticipantIdentitySet for Server { &request.arguments.on_success_set_is_default && id_ref == &id { - identities.default = document_id; + new_default = Some(document_id); } has_changes = true; - response.created(id, document_id); + let calendar_address = identities + .identities + .last() + .map(|identity| identity.calendar_address.clone()) + .unwrap_or_default(); + response.created(id.clone(), document_id); + response.add_created_properties( + &id, + (!client_name) + .then(|| { + ( + ParticipantIdentityProperty::Name, + Value::Str(identities.default_name.clone().into()), + ) + }) + .into_iter() + .chain( + (client_address.as_deref() != Some(calendar_address.as_str())).then(|| { + ( + ParticipantIdentityProperty::CalendarAddress, + Value::Str(calendar_address.into()), + ) + }), + ), + ); + created_ids.push(document_id); } // Process updates @@ -144,22 +196,53 @@ impl ParticipantIdentitySet for Server { continue 'update; } + let document_id = id.document_id(); let Some(identity) = identities .identities .iter_mut() - .find(|i| i.id == id.document_id()) + .find(|i| i.id == document_id) else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; - if let Err(err) = validate_identity_value(Some(id), object, identity, &allowed_emails) { + let mut updated_identity = identity.clone(); + let client_address = object + .as_object_and_get(&Key::Property(ParticipantIdentityProperty::CalendarAddress)) + .and_then(|value| value.as_str()) + .map(|value| value.into_owned()); + if let Err(err) = + validate_identity_value(Some(id), object, &mut updated_identity, &allowed_emails) + { response.not_updated.append(id, err); continue 'update; } + if identities.identities.iter().any(|i| { + i.id != document_id && i.calendar_address == updated_identity.calendar_address + }) { + response.not_updated.append(id, address_in_use()); + continue 'update; + } + let normalized_address = client_address + .is_some_and(|address| address != updated_identity.calendar_address) + .then(|| updated_identity.calendar_address.clone()); + if let Some(identity) = identities + .identities + .iter_mut() + .find(|i| i.id == document_id) + { + *identity = updated_identity; + } has_changes = true; response.updated.append(id, None); + if let Some(address) = normalized_address { + response.add_server_set_property( + id, + ParticipantIdentityProperty::CalendarAddress, + Value::Str(address.into()), + ); + } } // Process deletions @@ -179,15 +262,90 @@ impl ParticipantIdentitySet for Server { } if let Some(MaybeIdReference::Id(id)) = request.arguments.on_success_set_is_default { - let id = id.document_id(); - if identities.identities.iter().any(|i| i.id == id) { - identities.default = id; - has_changes = true; + new_default = Some(id.document_id()); + } + if let Some(default_id) = new_default.filter(|default_id| { + response.not_created.is_empty() + && response.not_updated.is_empty() + && response.not_destroyed.is_empty() + && identities.identities.iter().any(|i| i.id == *default_id) + }) { + identities.default = default_id; + } else if !identities + .identities + .iter() + .any(|i| i.id == identities.default) + && let Some(first) = identities.identities.first() + { + identities.default = first.id; + } + for document_id in created_ids + .iter() + .filter(|document_id| **document_id != identities.default) + { + response.add_server_set_property( + Id::from(*document_id), + ParticipantIdentityProperty::IsDefault, + false, + ); + } + if identities.default != previous_default { + has_changes = true; + response.add_server_set_property( + Id::from(identities.default), + ParticipantIdentityProperty::IsDefault, + true, + ); + if identities + .identities + .iter() + .any(|i| i.id == previous_default) + { + response.add_server_set_property( + Id::from(previous_default), + ParticipantIdentityProperty::IsDefault, + false, + ); } } // Write changes if has_changes { + identities.begin_changes(); + let changed_defaults = [identities.default, previous_default] + .into_iter() + .filter(|_| identities.default != previous_default); + let updated_ids = response + .updated + .keys() + .map(|id| id.document_id()) + .chain(changed_defaults) + .filter(|id| { + !created_ids.contains(id) + && !response + .destroyed + .iter() + .any(|destroyed| destroyed.document_id() == *id) + }) + .collect::>(); + for (id, change_type) in created_ids + .iter() + .map(|id| (*id, ParticipantIdentityChangeType::Created)) + .chain( + updated_ids + .into_iter() + .map(|id| (id, ParticipantIdentityChangeType::Updated)), + ) + .chain( + response + .destroyed + .iter() + .map(|id| (id.document_id(), ParticipantIdentityChangeType::Destroyed)), + ) + { + identities.record_change(id, change_type); + } + let change_id = identities.change_id; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) @@ -204,12 +362,70 @@ impl ParticipantIdentitySet for Server { ); self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = Some(identity_state(change_id)); + self.broadcast_push_notification(PushNotification::StateChange( + StateChange::new(account_id) + .with_change_id(change_id as u64) + .with_change(DataType::ParticipantIdentity), + )) + .await; } Ok(response) } } +async fn reject_identity_changes( + server: &Server, + mut request: SetRequest<'_, participant_identity::ParticipantIdentity>, + response: SetResponse, + will_destroy: Vec, + access_token: &AccessToken, +) -> trc::Result> { + let change_id = match server + .store() + .get_value::>(ValueKey::property( + access_token.account_id(), + Collection::Principal, + 0, + PrincipalField::ParticipantIdentities, + )) + .await + .caused_by(trc::location!())? + { + Some(archive) => archive + .unarchive::() + .caused_by(trc::location!())? + .change_id + .to_native(), + None => 0, + }; + let state = identity_state(change_id); + if request + .if_in_state + .as_ref() + .is_some_and(|if_in_state| if_in_state != &state) + { + return Err(trc::JmapEvent::StateMismatch.into_err()); + } + let mut response = response.with_state(state); + for (id, _) in request.unwrap_create() { + response.not_created.append(id, identities_not_shared()); + } + for (id, _) in request.unwrap_update() { + response.not_updated.append(id, identities_not_shared()); + } + for id in will_destroy { + response.not_destroyed.append(id, identities_not_shared()); + } + Ok(response) +} + +fn identities_not_shared() -> SetError { + SetError::forbidden() + .with_description("Participant identities can only be changed in your own account.") +} + fn validate_identity_value( expected_id: Option, update: Value<'_, ParticipantIdentityProperty, ParticipantIdentityValue>, @@ -235,11 +451,9 @@ fn validate_identity_value( if allowed_emails.iter().any(|e| e == &email) { identity.calendar_address = format!("mailto:{email}"); } else { - return Err(SetError::invalid_properties() - .with_property(ParticipantIdentityProperty::CalendarAddress) - .with_description( - "Calendar address not configured for this account.".to_string(), - )); + return Err(SetError::forbidden().with_description( + "Calendar address not configured for this account.", + )); } } else { return Err(SetError::invalid_properties() @@ -272,3 +486,11 @@ fn validate_identity_value( .with_description("Missing calendar address.")) } } + +fn address_in_use() -> SetError { + SetError::forbidden().with_description("Calendar address already in use.") +} + +pub(crate) fn identity_state(change_id: u32) -> State { + State::from((change_id != 0).then_some(change_id as u64)) +} diff --git a/crates/jmap/src/principal/availability.rs b/crates/jmap/src/principal/availability.rs index e24010f2..85fc1851 100644 --- a/crates/jmap/src/principal/availability.rs +++ b/crates/jmap/src/principal/availability.rs @@ -4,49 +4,49 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{calendar::Availability, calendar_event::CalendarSyntheticId}; -use calcard::{ - common::timezone::Tz, - icalendar::{ - ArchivedICalendarClassification, ArchivedICalendarParameterValue, - ArchivedICalendarParticipationStatus, ArchivedICalendarProperty, ArchivedICalendarStatus, - ArchivedICalendarTransparency, ArchivedICalendarValue, ICalendarParameterName, - }, - jscalendar::{JSCalendar, JSCalendarProperty, JSCalendarValue}, +use crate::calendar_event::{ + CalendarSyntheticId, + get::{EventProjector, ProjectedEvent, ProjectionRequest, ProjectionResults}, }; +use calcard::jscalendar::JSCalendar; use common::{ - Server, TinyCalendarPreferences, + KV_RATE_LIMIT_AVAILABILITY, Server, auth::{AccessToken, BuildAccessToken}, }; use groupware::{ cache::GroupwareCache, - calendar::{CALENDAR_SUBSCRIBED, CalendarEvent, expand::RecurrenceKey}, - strip_mailto_scheme, + calendar::{ + CalendarEventContent, + availability::{AvailabilityCalendars, BusyLevel, ComponentBusy, PrincipalRole}, + expand::{MAX_UTC_OFFSET, RecurrenceKey}, + identity::ParticipantIdentityAddresses, + privacy::EventPrivacy, + user::BASE_INSTANCE, + }, }; use jmap_proto::{ method::availability::{ BusyPeriod, BusyStatus, GetAvailabilityRequest, GetAvailabilityResponse, }, - object::calendar::IncludeInAvailability, - request::IntoValid, + request::MaybeInvalid, types::date::UTCDate, }; -use jmap_tools::{Key, Map, Value}; +use jmap_tools::Value; use registry::schema::enums::Permission; -use std::{collections::hash_map::Entry, future::Future}; +use std::future::Future; use store::{ ValueKey, ahash::AHashMap, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ - TimeRange, - acl::Acl, + OverlapRule, TimeRange, + blob::BlobId, collection::{Collection, SyncCollection}, + field::CalendarEventField, id::Id, }; -use utils::sanitize_email; pub trait PrincipalGetAvailability: Sync + Send { fn principal_get_availability( @@ -62,39 +62,74 @@ impl PrincipalGetAvailability for Server { request: GetAvailabilityRequest, access_token: &AccessToken, ) -> trc::Result { - if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::JmapPrincipalGetAvailability) + let account_id = request.account_id.document_id(); + if !access_token.has_account_access(account_id) + || self + .try_account(account_id) + .await + .caused_by(trc::location!())? + .is_none() { - return Err(trc::JmapEvent::Forbidden + return Err(trc::JmapEvent::AccountNotFound .into_err() - .details("The administrator has disabled directory queries.".to_string())); + .details("Account not found.")); } - // Process parameters if !request.id.is_valid() { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("Missing principal id")); } - let properties = request - .event_properties - .map(|props| props.into_valid().collect::>()) - .unwrap_or_default(); - if properties - .iter() - .any(|p| !matches!(p, JSCalendarProperty::Id | JSCalendarProperty::BaseEventId)) - { - return Err(trc::JmapEvent::InvalidArguments + let filter = request.time_range().ok_or_else(|| { + trc::JmapEvent::InvalidArguments.into_err().details( + "utcStart and utcEnd must be valid UTCDate values, with utcEnd after utcStart", + ) + })?; + if filter.end - filter.start > self.core.groupware.max_availability_duration as i64 { + return Err(trc::JmapEvent::TooLarge .into_err() - .details("Only 'id' and 'baseEventId' properties are supported in results")); + .details("The requested time span exceeds maxAvailabilityDuration")); + } + if let Some(rate) = &self.core.groupware.availability_rate + && !access_token.has_permission(Permission::UnlimitedRequests) + && self + .core + .storage + .memory + .is_rate_allowed( + KV_RATE_LIMIT_AVAILABILITY, + &access_token.account_id().to_be_bytes(), + rate, + ) + .await + .caused_by(trc::location!())? + .is_some() + { + return Err(trc::JmapEvent::RateLimit + .into_err() + .details("Too many availability requests, please try again later")); } - let return_event_details = !properties.is_empty(); - let max_instances = self.core.groupware.max_ical_instances; - let filter = TimeRange { - start: request.utc_start.timestamp(), - end: request.utc_end.timestamp(), - }; let principal_id = request.id.document_id(); + if self + .try_account(principal_id) + .await + .caused_by(trc::location!())? + .is_none_or(|account| { + !account.is_same_tenant(access_token) + && !access_token.has_permission(Permission::SysAccountQuery) + }) + { + return Err(trc::JmapEvent::ObjectNotFound + .into_err() + .details("Principal not found.")); + } + let show_details = + request.show_details && access_token.has_permission(Permission::JmapCalendarEventGet); + let max_instances = self.core.groupware.max_ical_instances; + let candidates = TimeRange::new( + filter.start.saturating_sub(MAX_UTC_OFFSET), + filter.end.saturating_add(MAX_UTC_OFFSET), + ); let principal = self .access_token(principal_id) .await @@ -104,102 +139,72 @@ impl PrincipalGetAvailability for Server { .account_info(principal_id) .await .caused_by(trc::location!())?; - let mut periods = Vec::new(); + let identities = self + .identity_addresses(principal_id, &principal_account) + .await + .caused_by(trc::location!())?; + let mut intervals = Vec::new(); + let mut components = Vec::new(); + let mut events = AHashMap::new(); + let projection = show_details + .then(|| { + ProjectionRequest::new( + request.event_properties.map(|properties| { + properties + .into_iter() + .filter_map(MaybeInvalid::try_unwrap) + .collect() + }), + false, + None, + ) + }) + .transpose()?; for account_id in principal.all_ids_by_collection(Collection::Calendar) { let resources = self - .fetch_dav_resources( + .fetch_groupware_resources( access_token.account_id(), account_id, SyncCollection::Calendar, ) .await .caused_by(trc::location!())?; - - // Obtain shared ids - let is_account_owner = principal_id == account_id; - let shared_ids = if !access_token.is_member(account_id) { - // Condition: The user has the "mayReadFreeBusy" permission for the calendar. - let shared_ids = resources.shared_items( - access_token, - [Acl::ReadItems, Acl::SchedulingReadFreeBusy], - true, - ); - if shared_ids.is_empty() { - continue; + let Some(mut calendars) = + AvailabilityCalendars::new(&resources, account_id, access_token, &principal) + else { + continue; + }; + let mut projector = match &projection { + Some(projection) => { + Some(EventProjector::new(self, access_token, account_id, projection).await?) } - - shared_ids.into() - } else { - None + None => None, }; - // Condition: The event finishes after the "utcStart" argument and starts before the "utcEnd" argument. - let mut preferences_cache: AHashMap> = - AHashMap::default(); - 'next_event: for resource in resources.resources.iter().filter(|r| { - r.event_time_range().is_some_and(|(start, end)| { - shared_ids - .as_ref() - .is_none_or(|ids| ids.contains(r.document_id)) - && filter.is_in_range(false, start, end) - }) - }) { - // Obtain calendar settings - let mut include_in_availability = None; - let mut default_tz = Tz::UTC; - let mut is_subscribed = is_account_owner; - for calendar_id in resource - .child_names() - .unwrap_or_default() - .iter() - .map(|n| n.parent_id) - { - match preferences_cache.entry(calendar_id) { - Entry::Occupied(e) => { - if let Some(prefs) = e.get() { - default_tz = prefs.tz; - is_subscribed |= prefs.flags & CALENDAR_SUBSCRIBED != 0; - include_in_availability = - IncludeInAvailability::from_flags(prefs.flags); - } - } - Entry::Vacant(e) => { - if let Some(prefs) = resources - .container_resource_by_id(calendar_id) - .and_then(|r| r.calendar_preferences(principal_id)) - { - default_tz = prefs.tz; - is_subscribed |= prefs.flags & CALENDAR_SUBSCRIBED != 0; - include_in_availability = - IncludeInAvailability::from_flags(prefs.flags); - e.insert(Some(prefs)); - } else { - e.insert(None); - } - } - } + for resource in resources.resources.iter() { + let (Some((start, end)), Some(flags)) = + (resource.event_time_range(), resource.event_flags()) + else { + continue; + }; + let privacy = EventPrivacy::from_flags(flags); + if privacy == EventPrivacy::Secret || !candidates.overlaps(start, end) { + continue; } - let include_in_availability = include_in_availability.unwrap_or({ - if is_account_owner { - IncludeInAvailability::All - } else { - IncludeInAvailability::None - } - }); + let names = resource.child_names(); + let Some(calendar) = calendars.relevant_calendar(names) else { + continue; + }; - if !is_subscribed || include_in_availability == IncludeInAvailability::None { - continue 'next_event; - } - - // Fetch event - let document_id = resource.document_id; + let document_id = resource.document_id(); let Some(archive) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::property( account_id, Collection::CalendarEvent, document_id, + CalendarEventField::Content, )) .await .caused_by(trc::location!())? @@ -207,221 +212,257 @@ impl PrincipalGetAvailability for Server { continue; }; let event = archive - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; + let preferences = match calendars.role() { + PrincipalRole::Owner => None, + PrincipalRole::Sharee => event.preferences(principal_id), + }; - // Find the component ids that match the criteria - let mut matching_component_ids = AHashMap::new(); - 'next_component: for (component_id, component) in - event.data.event.components.iter().enumerate() + components.clear(); + components.extend(event.data.event.components.iter().map(|component| { + ComponentBusy::parse(component, &identities, calendars.role()) + .filter(|busy| calendar.includes(busy)) + })); + if components.iter().all(Option::is_none) { + continue; + } + + let may_read_details = show_details + && privacy == EventPrivacy::Public + && calendars.may_read_items(names); + let first_interval = intervals.len(); + for expansion in event + .data + .expand(calendar.tz, filter, OverlapRule::Jmap) + .unwrap_or_default() { - if !component.component_type.is_event_or_todo() { - continue 'next_component; - } - - let mut is_cancelled = false; - let mut is_main_event = true; - let mut busy_status = None; - - for entry in component.entries.iter() { - match (&entry.name, entry.values.first()) { - ( - ArchivedICalendarProperty::Status, - Some(ArchivedICalendarValue::Status( - ArchivedICalendarStatus::Cancelled, - )), - ) => { - // The "status" property of the event is not "cancelled". - is_cancelled = true; - } - (ArchivedICalendarProperty::RecurrenceId, _) => { - is_main_event = false; - } - ( - ArchivedICalendarProperty::Class, - Some(ArchivedICalendarValue::Classification( - ArchivedICalendarClassification::Confidential, - )), - ) => { - // Condition: The event's "privacy" property is not "secret". - continue 'next_component; - } - ( - ArchivedICalendarProperty::Transp, - Some(ArchivedICalendarValue::Transparency( - ArchivedICalendarTransparency::Transparent, - )), - ) => { - // Condition: The "freeBusyStatus" property of the event is "busy" (or omitted, as this is the default). - continue 'next_component; - } - (ArchivedICalendarProperty::Attendee, Some(value)) - if include_in_availability == IncludeInAvailability::Attending => - { - if let Some(attendee) = value.as_text().and_then(|attendee| { - sanitize_email(strip_mailto_scheme(attendee)) - }) { - // Condition: the Principal is a participant of the event, and has a "participationStatus" of "accepted" or "tentative". - if principal_account.addresses().contains(&attendee) { - busy_status = Some( - entry - .parameters(&ICalendarParameterName::Partstat) - .next() - .map(|v| { - match v { - ArchivedICalendarParameterValue::Partstat( - ArchivedICalendarParticipationStatus::Accepted, - ) => BusyStatus::Confirmed, - ArchivedICalendarParameterValue::Partstat( - ArchivedICalendarParticipationStatus::Tentative, - ) => BusyStatus::Tentative, - ArchivedICalendarParameterValue::Partstat( - ArchivedICalendarParticipationStatus::Declined, - ) => { - is_cancelled = true; - BusyStatus::Unavailable - } - _ => BusyStatus::Unavailable, - } - }) - .unwrap_or(BusyStatus::Unavailable), - ); - } - } - } - _ => (), - } - } - - if is_cancelled { - if is_main_event { - continue 'next_event; - } else { - continue 'next_component; - } - } - - let busy_status = if let Some(busy_status) = busy_status { - busy_status - } else if include_in_availability == IncludeInAvailability::All { - BusyStatus::Confirmed - } else { - continue 'next_component; - }; - - matching_component_ids.insert(component_id as u32, busy_status); - } - - if matching_component_ids.is_empty() { - // No events matched the criteria - continue 'next_event; - } - - for expansion in event.data.expand(default_tz, filter).unwrap_or_default() { - let Some(busy_status) = matching_component_ids.get(&expansion.comp_id) else { + let Some(Some(busy)) = components.get(expansion.comp_id as usize) else { continue; }; - let Some(recurrence_key) = expansion.recurrence_key() else { + if expansion.end <= expansion.start { continue; - }; - if periods.len() < max_instances { - periods.push(FreeBusyResult { - utc_start: expansion.start, - utc_end: expansion.end, - busy_status: *busy_status, - recurrence_key, - document_id, - }); - } else { - return Err(trc::JmapEvent::RequestTooLarge + } + let recurrence_key = (preferences.is_some() + || (may_read_details && busy.is_recurrent_or_override())) + .then(|| expansion.recurrence_key()) + .flatten(); + if preferences + .and_then(|preferences| { + preferences.is_free( + recurrence_key.map_or(BASE_INSTANCE, RecurrenceKey::prefix), + ) + }) + .unwrap_or(false) + { + continue; + } + if intervals.len() == max_instances { + return Err(trc::JmapEvent::TooLarge .into_err() .details("The number of expanded instances exceeds the server limit")); } + let event_id = match (may_read_details, busy.is_recurrent_or_override()) { + (false, _) => None, + (true, true) => recurrence_key + .map(|key| ::new(key, document_id)), + (true, false) => Some(Id::from(document_id)), + }; + intervals.push(BusyInterval { + start: expansion.start, + end: expansion.end, + status: busy_status(busy), + event: event_id.map(|id| (account_id, id)), + }); + } + + if let Some(projector) = projector.as_mut() { + let mut detail_ids: Vec<(Id, usize)> = Vec::new(); + for (_, id) in intervals[first_interval..] + .iter() + .filter_map(|interval| interval.event) + { + if !detail_ids.iter().any(|(existing, _)| *existing == id) { + detail_ids.push((id, detail_ids.len())); + } + } + if !detail_ids.is_empty() { + let mut projected = ProjectionResults::default(); + projector + .project( + self, + access_token, + ProjectedEvent { + cache: &resources, + document_id, + content: Some(event), + flags, + privacy, + calendar_ids: names, + time_zone: calendar.tz, + ids: &detail_ids, + }, + &mut projected, + ) + .await?; + events.extend(projected.found.into_iter().filter_map(|(index, object)| { + detail_ids.get(index).map(|(id, _)| { + ((account_id, *id), JSCalendar(Value::Object(object))) + }) + })); + } } } } - let mut result = GetAvailabilityResponse { - list: Vec::with_capacity(periods.len()), - }; - - if periods.is_empty() { - return Ok(result); - } - - // Sort by busy status and start time - periods.sort_unstable_by(|a, b| { - a.busy_status - .cmp(&b.busy_status) - .then_with(|| a.utc_start.cmp(&b.utc_start)) - }); - - if return_event_details { - for period in periods { - result.list.push(period.into()); + let mut busy_periods = Vec::with_capacity(intervals.len()); + let mut timeline = BusyTimeline::default(); + for interval in intervals { + match interval + .event + .and_then(|key| events.remove(&key).map(|event| (key.0, event))) + { + Some(event) => busy_periods.push(BusyInterval { + start: interval.start, + end: interval.end, + status: interval.status, + event: Some(event), + }), + None => timeline.insert(interval.start, interval.end, interval.status), } - } else { - // Merge intervals with same busy status - let mut start_time = periods[0].utc_start; - let mut end_time = periods[0].utc_end; - let mut current_status = periods[0].busy_status; + } + timeline.merge_into(&mut busy_periods); + busy_periods.sort_by_key(|period| (period.start, period.end, period.status)); - for curr in periods.iter().skip(1) { - if curr.utc_start <= end_time && curr.busy_status == current_status { - end_time = end_time.max(curr.utc_end); - } else { - result.list.push(BusyPeriod { - utc_start: UTCDate::from_timestamp(start_time), - utc_end: UTCDate::from_timestamp(end_time), - busy_status: Some(current_status), + Ok(GetAvailabilityResponse { + list: busy_periods.into_iter().map(BusyPeriod::from).collect(), + }) + } +} + +type EventDetails = (u32, JSCalendar<'static, Id, BlobId>); + +struct BusyInterval { + start: i64, + end: i64, + status: BusyStatus, + event: Option, +} + +impl From> for BusyPeriod { + fn from(interval: BusyInterval) -> Self { + let (account_id, event) = interval.event.unzip(); + BusyPeriod { + utc_start: UTCDate::from_timestamp(interval.start), + utc_end: UTCDate::from_timestamp(interval.end), + busy_status: Some(interval.status), + event, + account_id: account_id.map(Id::from), + } + } +} + +fn busy_status(busy: &ComponentBusy) -> BusyStatus { + match busy.busy_level() { + BusyLevel::Confirmed => BusyStatus::Confirmed, + BusyLevel::Tentative => BusyStatus::Tentative, + BusyLevel::Unavailable => BusyStatus::Unavailable, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Edge { + Start, + End, +} + +#[derive(Debug, Clone, Copy)] +struct Boundary { + time: i64, + edge: Edge, + status: BusyStatus, +} + +#[derive(Debug, Default)] +struct BusyTimeline { + boundaries: Vec, +} + +#[derive(Debug, Default)] +struct ActiveStatuses { + tentative: u32, + unavailable: u32, + confirmed: u32, +} + +impl BusyTimeline { + fn insert(&mut self, start: i64, end: i64, status: BusyStatus) { + self.boundaries.extend([ + Boundary { + time: start, + edge: Edge::Start, + status, + }, + Boundary { + time: end, + edge: Edge::End, + status, + }, + ]); + } + + fn merge_into(mut self, intervals: &mut Vec>) { + self.boundaries + .sort_unstable_by_key(|boundary| boundary.time); + let mut active = ActiveStatuses::default(); + let mut current: Option<(i64, BusyStatus)> = None; + let mut boundaries = self.boundaries.into_iter().peekable(); + + while let Some(boundary) = boundaries.next() { + let time = boundary.time; + active.apply(boundary); + while let Some(boundary) = boundaries.next_if(|next| next.time == time) { + active.apply(boundary); + } + + let status = active.top(); + if current.map(|(_, status)| status) != status { + if let Some((start, status)) = current { + intervals.push(BusyInterval { + start, + end: time, + status, event: None, }); - start_time = curr.utc_start; - end_time = curr.utc_end; - current_status = curr.busy_status; } + current = status.map(|status| (time, status)); } - - result.list.push(BusyPeriod { - utc_start: UTCDate::from_timestamp(start_time), - utc_end: UTCDate::from_timestamp(end_time), - busy_status: Some(current_status), - event: None, - }); - } - - Ok(result) - } -} - -struct FreeBusyResult { - utc_start: i64, - utc_end: i64, - busy_status: BusyStatus, - recurrence_key: RecurrenceKey, - document_id: u32, -} - -impl From for BusyPeriod { - fn from(value: FreeBusyResult) -> Self { - BusyPeriod { - utc_start: UTCDate::from_timestamp(value.utc_start), - utc_end: UTCDate::from_timestamp(value.utc_end), - busy_status: Some(value.busy_status), - event: JSCalendar(Value::Object(Map::from(vec![ - ( - Key::Property(JSCalendarProperty::Id), - Value::Element(JSCalendarValue::Id(::new( - value.recurrence_key, - value.document_id, - ))), - ), - ( - Key::Property(JSCalendarProperty::BaseEventId), - Value::Element(JSCalendarValue::Id(Id::from(value.document_id))), - ), - ]))) - .into(), } } } + +impl ActiveStatuses { + fn apply(&mut self, boundary: Boundary) { + let count = match boundary.status { + BusyStatus::Tentative => &mut self.tentative, + BusyStatus::Unavailable => &mut self.unavailable, + BusyStatus::Confirmed => &mut self.confirmed, + }; + match boundary.edge { + Edge::Start => *count += 1, + Edge::End => *count -= 1, + } + } + + fn top(&self) -> Option { + [ + (self.confirmed, BusyStatus::Confirmed), + (self.unavailable, BusyStatus::Unavailable), + (self.tentative, BusyStatus::Tentative), + ] + .into_iter() + .find(|(count, _)| *count > 0) + .map(|(_, status)| status) + } +} + diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index 0071e3da..18ec3973 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -4,18 +4,26 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; +use crate::api::session::JmapAccount; +use common::{ + Server, + auth::{AccessToken, AccountCache}, +}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::principal::{Principal, PrincipalProperty, PrincipalType, PrincipalValue}, request::capability::Capability, types::state::State, }; use jmap_tools::{Key, Map, Value}; use registry::schema::prelude::{ObjectType, Permission}; -use std::future::Future; +use std::{future::Future, sync::Arc}; use store::{registry::RegistryQuery, roaring::RoaringBitmap}; use trc::AddContext; +use types::id::Id; + +type PrincipalObject = Value<'static, PrincipalProperty, PrincipalValue>; +type PrincipalMap = Map<'static, PrincipalProperty, PrincipalValue>; pub trait PrincipalGet: Sync + Send { fn principal_get( @@ -23,6 +31,12 @@ pub trait PrincipalGet: Sync + Send { request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; + + fn visible_principal( + &self, + access_token: &AccessToken, + principal_id: u32, + ) -> impl Future>>> + Send; } impl PrincipalGet for Server { @@ -31,14 +45,8 @@ impl PrincipalGet for Server { mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { - if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::JmapPrincipalGet) - { - return Err(trc::JmapEvent::Forbidden - .into_err() - .details("The administrator has disabled directory queries.".to_string())); - } - + let can_query_accounts = access_token.has_permission(Permission::SysAccountQuery); + let allow_directory_query = self.core.groupware.allow_directory_query || can_query_accounts; let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ PrincipalProperty::Id, @@ -46,26 +54,34 @@ impl PrincipalGet for Server { PrincipalProperty::Name, PrincipalProperty::Description, PrincipalProperty::Email, + PrincipalProperty::Timezone, + PrincipalProperty::Capabilities, + PrincipalProperty::Accounts, ]); // Return all principals - let principal_ids = self - .registry() - .query::( - RegistryQuery::new(ObjectType::Account).with_tenant(access_token.tenant_id()), - ) - .await - .caused_by(trc::location!())?; - - let ids = if let Some(ids) = ids { - ids - } else { - principal_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + let query = RegistryQuery::new(ObjectType::Account); + let ids = match ids { + Some(ids) => ids, + None if allow_directory_query => all_ids( + self.registry() + .query::( + query.with_limit(self.core.jmap.get_max_objects.saturating_add(1)), + ) + .await + .caused_by(trc::location!())? + .iter() + .map(Into::into), + self.core.jmap.get_max_objects, + )?, + None => all_ids( + access_token.all_ids().map(Into::into), + self.core.jmap.get_max_objects, + )?, }; + let has_availability_permission = + access_token.has_permission(Permission::JmapPrincipalGetAvailability); + let mut accounts = AccountObjects::default(); let mut response = GetResponse { account_id: request.account_id.into(), state: State::Initial.into(), @@ -76,14 +92,14 @@ impl PrincipalGet for Server { for id in ids { // Obtain the principal let document_id = id.document_id(); - if !principal_ids.contains(document_id) { + if !allow_directory_query && !access_token.has_account_access(document_id) { + response.push_not_found(id); + continue; + } + let Some(principal) = self.visible_principal(access_token, document_id).await? else { response.push_not_found(id); continue; }; - let principal = self - .account(document_id) - .await - .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { @@ -102,66 +118,22 @@ impl PrincipalGet for Server { .map(|v| Value::Str(v.to_string().into())) .unwrap_or(Value::Null), PrincipalProperty::Email => Value::Str(principal.name().to_string().into()), - PrincipalProperty::Accounts => Value::Object(Map::from(vec![( - Key::Property(PrincipalProperty::IdValue(id)), - Value::Object(Map::from_iter( - [ - Capability::Mail, - Capability::Contacts, - Capability::Calendars, - Capability::FileNode, - Capability::Principals, - ] - .iter() - .map(|cap| { - ( - Key::Property(PrincipalProperty::Capability(*cap)), - Value::Object(Map::new()), - ) - }) - .chain([ - ( - Key::Property(PrincipalProperty::Capability( - Capability::PrincipalsOwner, - )), - Value::Object(Map::from(vec![ - ( - Key::Borrowed("accountIdForPrincipal"), - Value::Element(PrincipalValue::Id(id)), - ), - ( - Key::Borrowed("principalId"), - Value::Element(PrincipalValue::Id(id)), - ), - ])), - ), - ( - Key::Property(PrincipalProperty::Capability( - Capability::Calendars, - )), - Value::Object(Map::from(vec![ - ( - Key::Borrowed("accountId"), - Value::Element(PrincipalValue::Id(id)), - ), - (Key::Borrowed("mayGetAvailability"), Value::Bool(true)), - (Key::Borrowed("mayShareWith"), Value::Bool(true)), - ( - Key::Borrowed("calendarAddress"), - Value::Str( - format!("mailto:{}", principal.name()).into(), - ), - ), - ])), - ), - ]), - )), - )])), + PrincipalProperty::Accounts => { + if access_token.is_member(document_id) + || access_token.is_shared(document_id) + { + Value::Object(Map::from(vec![( + Key::Property(PrincipalProperty::IdValue(id)), + accounts.account(self, access_token, id, principal.name())?, + )])) + } else { + Value::Null + } + } PrincipalProperty::Capabilities => Value::Object(Map::from_iter( [ Capability::Mail, Capability::Contacts, - Capability::Calendars, Capability::FileNode, Capability::Principals, ] @@ -171,7 +143,42 @@ impl PrincipalGet for Server { Key::Property(PrincipalProperty::Capability(*cap)), Value::Object(Map::new()), ) - }), + }) + .chain([( + Key::Property(PrincipalProperty::Capability(Capability::Calendars)), + Value::Object(Map::from(vec![ + ( + Key::Borrowed("accountId"), + if access_token.is_member(document_id) + || access_token.is_shared(document_id) + { + Value::Element(PrincipalValue::Id(id)) + } else { + Value::Null + }, + ), + ( + Key::Borrowed("mayGetAvailability"), + Value::Bool( + has_availability_permission + && (can_query_accounts + || principal.is_same_tenant(access_token)), + ), + ), + ( + Key::Borrowed("mayShareWith"), + Value::Bool(document_id != access_token.account_id()), + ), + ( + Key::Borrowed("calendarAddress"), + if principal.addresses.is_empty() { + Value::Null + } else { + Value::Str(format!("mailto:{}", principal.name()).into()) + }, + ), + ])), + )]), )), _ => Value::Null, }; @@ -183,4 +190,142 @@ impl PrincipalGet for Server { Ok(response) } + + #[cfg_attr( + not(feature = "enterprise"), + allow(unused_variables, clippy::let_and_return) + )] + async fn visible_principal( + &self, + access_token: &AccessToken, + principal_id: u32, + ) -> trc::Result>> { + let account = self + .try_account(principal_id) + .await + .caused_by(trc::location!()); + account + } +} + +#[derive(Default)] +struct AccountObjects { + member_capabilities: Option, + shared_capabilities: Option, +} + +impl AccountObjects { + fn account( + &mut self, + server: &Server, + access_token: &AccessToken, + id: Id, + name: &str, + ) -> trc::Result { + let account_id = id.document_id(); + let template = if access_token.is_member(account_id) { + &mut self.member_capabilities + } else { + &mut self.shared_capabilities + }; + let template = match template { + Some(template) => template, + None => template.insert(Self::capabilities(server, access_token, account_id)?), + }; + let mut capabilities = PrincipalMap::with_capacity(template.len() + 1); + capabilities.extend( + template + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); + capabilities.insert_unchecked( + Key::Property(PrincipalProperty::Capability(Capability::PrincipalsOwner)), + Value::Object(Map::from(vec![ + ( + Key::Borrowed("accountIdForPrincipal"), + Value::Element(PrincipalValue::Id(Id::from(access_token.account_id()))), + ), + ( + Key::Borrowed("principalId"), + Value::Element(PrincipalValue::Id(id)), + ), + ])), + ); + + Ok(Value::Object(Map::from(vec![ + (Key::Borrowed("name"), Value::Str(name.to_string().into())), + ( + Key::Borrowed("isPersonal"), + Value::Bool(account_id == access_token.account_id()), + ), + ( + Key::Borrowed("isReadOnly"), + Value::Bool(access_token.is_read_only(account_id)), + ), + ( + Key::Borrowed("accountCapabilities"), + Value::Object(capabilities), + ), + ]))) + } + + fn capabilities( + server: &Server, + access_token: &AccessToken, + account_id: u32, + ) -> trc::Result { + server + .jmap_account_capabilities(access_token, account_id) + .map(|(capability, capabilities)| { + serde_json::to_value(capabilities).map(|value| { + ( + Key::Property(PrincipalProperty::Capability(capability)), + value.into_principal_value(), + ) + }) + }) + .collect::>() + .map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + }) + } +} + +trait IntoPrincipalValue { + fn into_principal_value(self) -> PrincipalObject; +} + +impl IntoPrincipalValue for serde_json::Value { + fn into_principal_value(self) -> PrincipalObject { + match self { + serde_json::Value::Null => Value::Null, + serde_json::Value::Bool(value) => Value::Bool(value), + serde_json::Value::Number(number) => { + if let Some(number) = number.as_u64() { + number.into() + } else if let Some(number) = number.as_i64() { + number.into() + } else if let Some(number) = number.as_f64() { + number.into() + } else { + Value::Null + } + } + serde_json::Value::String(value) => Value::Str(value.into()), + serde_json::Value::Array(values) => Value::Array( + values + .into_iter() + .map(IntoPrincipalValue::into_principal_value) + .collect(), + ), + serde_json::Value::Object(values) => Value::Object( + values + .into_iter() + .map(|(key, value)| (Key::Owned(key), value.into_principal_value())) + .collect(), + ), + } + } } diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index e38de82d..e6fe52c1 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -4,11 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::api::query::QueryResponseBuilder; +use crate::{api::query::QueryResponseBuilder, principal::get::PrincipalGet}; use common::{Server, auth::AccessToken}; +use compact_str::ToCompactString; +use groupware::decode_mailto_address; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::principal::{Principal, PrincipalFilter, PrincipalType}, + request::capability::{Capability, CapabilityIds}, types::state::State, }; use registry::{ @@ -26,12 +29,14 @@ use store::{ write::SearchIndex, }; use trc::AddContext; +use utils::sanitize_email; pub trait PrincipalQuery: Sync + Send { fn principal_query( &self, request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -40,59 +45,73 @@ impl PrincipalQuery for Server { &self, mut request: QueryRequest, access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { if !self.core.groupware.allow_directory_query - && !access_token.has_permission(Permission::JmapPrincipalQuery) + && !access_token.has_permission(Permission::SysAccountQuery) { return Err(trc::JmapEvent::Forbidden .into_err() - .details("The administrator has disabled directory queries.".to_string())); + .details("The administrator has disabled directory queries.")); } - let principal_ids = self - .registry() - .query::( - RegistryQuery::new(ObjectType::Account).with_tenant(access_token.tenant_id()), + let query = RegistryQuery::new(ObjectType::Account); + let principal_ids = if request.filter.is_empty() + || request.filter.iter().any(|cond| { + matches!( + cond, + Filter::Not | Filter::Property(PrincipalFilter::AccountIds(_)) + ) + }) { + Some( + self.registry() + .query::(query) + .await + .caused_by(trc::location!())?, ) - .await - .caused_by(trc::location!())?; + } else { + None + }; let mut filters = Vec::with_capacity(request.filter.len()); for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { PrincipalFilter::Name(name) | PrincipalFilter::Email(name) => { + let account_id = self.account_id_from_email(&name, false).await?; filters.push(SearchFilter::is_in_set( - match self.account_id_from_email(&name, false).await? { - Some(account_id) => { - RoaringBitmap::from_sorted_iter([account_id]).unwrap() + visible_principal_ids(self, access_token, account_id).await?, + )); + } + PrincipalFilter::CalendarAddress(address) + if using.contains(Capability::PrincipalsAvailability) => + { + let account_id = + match sanitize_email(decode_mailto_address(&address).as_ref()) { + Some(address) => { + self.account_id_from_email(&address, false).await? } - None => RoaringBitmap::new(), - }, + None => None, + }; + filters.push(SearchFilter::is_in_set( + visible_principal_ids(self, access_token, account_id).await?, )); } PrincipalFilter::AccountIds(ids) => { filters.push(SearchFilter::is_in_set( ids.into_iter() - .filter_map(|id| { - let id = id.document_id(); - if principal_ids.contains(id) { - Some(id) - } else { - None - } + .map(|id| id.document_id()) + .filter(|id| { + principal_ids.as_ref().is_some_and(|ids| ids.contains(*id)) }) .collect::(), )); } PrincipalFilter::Text(text) => { + let query = RegistryQuery::new(ObjectType::Account); filters.push(SearchFilter::is_in_set( self.registry() - .query::( - RegistryQuery::new(ObjectType::Account) - .with_tenant(access_token.tenant_id()) - .text(Property::Text, text), - ) + .query::(query.text(Property::Text, text)) .await .caused_by(trc::location!())?, )); @@ -107,13 +126,10 @@ impl PrincipalQuery for Server { } }; + let query = RegistryQuery::new(ObjectType::Account); filters.push(SearchFilter::is_in_set( self.registry() - .query::( - RegistryQuery::new(ObjectType::Account) - .equal(Property::Type, typ.to_id()) - .with_tenant(access_token.tenant_id()), - ) + .query::(query.equal(Property::Type, typ.to_id())) .await .caused_by(trc::location!())?, )); @@ -121,7 +137,7 @@ impl PrincipalQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())); + .details(other.to_compact_string())); } }, Filter::And => { @@ -139,9 +155,22 @@ impl PrincipalQuery for Server { } } + let mask = principal_ids.unwrap_or_else(|| { + filters + .iter() + .filter_map(|filter| match filter { + SearchFilter::DocumentSet(set) => Some(set), + _ => None, + }) + .fold(RoaringBitmap::new(), |mut mask, set| { + mask |= set; + mask + }) + }); + let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) - .with_mask(principal_ids) + .with_mask(mask) .filter() .into_bitmap(); @@ -161,3 +190,21 @@ impl PrincipalQuery for Server { response.build() } } + +async fn visible_principal_ids( + server: &Server, + access_token: &AccessToken, + account_id: Option, +) -> trc::Result { + Ok(match account_id { + Some(account_id) + if server + .visible_principal(access_token, account_id) + .await? + .is_some() => + { + RoaringBitmap::from_iter([account_id]) + } + _ => RoaringBitmap::new(), + }) +} diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index db7ffb4e..190542f5 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -8,7 +8,7 @@ use common::{Server, auth::AccessToken, ipc::PushEvent}; use email::push::{PushSubscriptions, Urgency}; use jmap_proto::{ method::{ - get::{GetRequest, GetResponse}, + get::{GetRequest, GetResponse, all_ids}, query::ArchivedFilter, }, object::{ @@ -21,7 +21,7 @@ use jmap_tools::{Key, Map, Property, Value}; use std::future::Future; use store::{ Serialize, ValueKey, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, + write::{Archive, ArchiveBytes, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; @@ -61,7 +61,7 @@ impl PushSubscriptionFetch for Server { let Some(subscriptions_) = self .store() - .get_value::>(ValueKey::property( + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, @@ -81,13 +81,14 @@ impl PushSubscriptionFetch for Server { let ids = if let Some(ids) = ids { ids } else { - subscriptions - .inner - .subscriptions - .iter() - .take(self.core.jmap.get_max_objects) - .map(|s| Id::from(s.id.to_native())) - .collect::>() + all_ids( + subscriptions + .inner + .subscriptions + .iter() + .map(|s| Id::from(s.id.to_native())), + self.core.jmap.get_max_objects, + )? }; for id in ids { @@ -110,9 +111,9 @@ impl PushSubscriptionFetch for Server { result.insert_unchecked(PushSubscriptionProperty::Id, id); } PushSubscriptionProperty::Url | PushSubscriptionProperty::Keys => { - return Err(trc::JmapEvent::Forbidden.into_err().details( - "The 'url' and 'keys' properties are not readable".to_string(), - )); + return Err(trc::JmapEvent::Forbidden + .into_err() + .details("The 'url' and 'keys' properties are not readable")); } PushSubscriptionProperty::DeviceClientId => { result.insert_unchecked( @@ -205,8 +206,8 @@ impl PushSubscriptionFetch for Server { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) - .with_account_id(account_id) - .tag(PrincipalField::PushSubscriptions); + .with_document(account_id) + .untag(PrincipalField::PushSubscriptions); } batch @@ -233,7 +234,6 @@ impl PushSubscriptionFetch for Server { .inner .ipc .push_tx - .clone() .send(PushEvent::PushServerUpdate { account_id, broadcast: true, @@ -356,62 +356,3 @@ fn archived_condition_to_value( Value::Object(Map::with_capacity(1).with_key_value(Key::Borrowed(key), value)) } -#[cfg(test)] -mod tests { - use super::*; - use jmap_proto::method::query::{Filter, FilterWrapper}; - use serde::Deserialize; - - fn parse_filter(json: &str) -> Vec> { - let value: Value = - serde_json::from_str(json).expect("valid filter json"); - FilterWrapper::::deserialize(&value) - .expect("parseable filter") - .0 - } - - fn store_and_serialize( - filter: &[Filter], - ) -> Value<'static, PushSubscriptionProperty, PushSubscriptionValue> { - let bytes = rkyv::to_bytes::(&filter.to_vec()).expect("serialize"); - let archived = rkyv::access::< - > as rkyv::Archive>::Archived, - rkyv::rancor::Error, - >(&bytes) - .expect("access archived"); - archived_filter_node(&mut archived.iter().peekable()).unwrap_or(Value::Null) - } - - #[test] - fn email_push_filter_round_trips() { - let mailbox_a = Id::from_parts(0, 10).to_string(); - let mailbox_b = Id::from_parts(0, 20).to_string(); - - for json in [ - format!(r#"{{"inMailbox":"{mailbox_a}"}}"#), - format!(r#"{{"inMailbox":"{mailbox_a}","hasKeyword":"$seen"}}"#), - format!( - r#"{{"operator":"OR","conditions":[{{"inMailbox":"{mailbox_a}"}},{{"hasKeyword":"$notify"}}]}}"# - ), - format!( - r#"{{"operator":"AND","conditions":[{{"inMailbox":"{mailbox_a}"}},{{"operator":"NOT","conditions":[{{"hasKeyword":"$junk"}}]}}]}}"# - ), - format!( - r#"{{"operator":"OR","conditions":[{{"subject":"hello"}},{{"operator":"AND","conditions":[{{"from":"alice@example.com"}},{{"inMailboxOtherThan":["{mailbox_a}","{mailbox_b}"]}},{{"minSize":1024}},{{"hasAttachment":true}}]}}]}}"# - ), - ] { - let parsed = parse_filter(&json); - let serialized = store_and_serialize(&parsed); - let round_tripped = FilterWrapper::::deserialize(&serialized) - .expect("reparseable filter") - .0; - - assert_eq!(parsed, round_tripped, "filter did not round-trip: {json}"); - } - } - - #[test] - fn empty_filter_serializes_to_null() { - assert!(matches!(store_and_serialize(&[]), Value::Null)); - } -} diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index b7640834..98ea437e 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -4,12 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{ - Engine, alphabet, - engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}, -}; -use common::{Server, auth::AccessToken, ipc::PushEvent, network::is_global_ip}; +use common::{Server, auth::AccessToken, ipc::PushEvent}; use email::push::{EmailPush, Keys, PushSubscription, PushSubscriptions, Urgency}; +use encodify::base64::{Base64, Padding, URL_SAFE}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::{ @@ -29,25 +26,23 @@ use jmap_proto::{ use jmap_tools::{Key, Map, Property, Value}; use rand::distr::Alphanumeric; use registry::schema::enums::StorageQuota; -use reqwest::Url; use std::future::Future; -use std::net::IpAddr; use std::str::FromStr; use store::{ Serialize, ValueKey, rand::{RngExt, rng}, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, + write::{Archive, ArchiveBytes, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; -use utils::map::bitmap::Bitmap; +use utils::{ + map::bitmap::Bitmap, + outbound::{OutboundError, OutboundPolicy}, +}; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days const VERIFICATION_CODE_LEN: usize = 32; -const URL_SAFE_INDIFFERENT: GeneralPurpose = GeneralPurpose::new( - &alphabet::URL_SAFE, - GeneralPurposeConfig::new().with_decode_padding_mode(DecodePaddingMode::Indifferent), -); +const URL_SAFE_INDIFFERENT: Base64 = URL_SAFE.with_padding(Padding::Optional); pub trait PushSubscriptionSet: Sync + Send { fn push_subscription_set( @@ -67,7 +62,7 @@ impl PushSubscriptionSet for Server { let account_id = access_token.account_id(); let subscriptions_archive = self .store() - .get_value::>(ValueKey::property( + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, @@ -123,6 +118,8 @@ impl PushSubscriptionSet for Server { } } + access_token.restrict_data_types(&mut push.types); + if push.device_client_id.is_empty() || push.url.is_empty() { response.not_created.append( id, @@ -277,7 +274,6 @@ impl PushSubscriptionSet for Server { .inner .ipc .push_tx - .clone() .send(PushEvent::PushServerUpdate { account_id, broadcast: true, @@ -318,10 +314,10 @@ fn validate_push_value( push.device_client_id = value.into_owned(); } (PushSubscriptionProperty::Url, Value::Str(value)) if is_create && value.len() < 512 => { - validate_push_url(value.as_ref()).map_err(|description| { + validate_push_url(value.as_ref()).map_err(|err| { SetError::invalid_properties() .with_property(property.clone()) - .with_description(description) + .with_description(err.as_str()) })?; push.url = value.into_owned(); } @@ -330,11 +326,11 @@ fn validate_push_value( value .get(&Key::Property(PushSubscriptionProperty::Auth)) .and_then(|v| v.as_str()) - .and_then(|v| URL_SAFE_INDIFFERENT.decode(v.as_ref()).ok()), + .and_then(|v| URL_SAFE_INDIFFERENT.decode(v.as_bytes()).ok()), value .get(&Key::Property(PushSubscriptionProperty::P256dh)) .and_then(|v| v.as_str()) - .and_then(|v| URL_SAFE_INDIFFERENT.decode(v.as_ref()).ok()), + .and_then(|v| URL_SAFE_INDIFFERENT.decode(v.as_bytes()).ok()), ) { if p256::PublicKey::from_sec1_bytes(&p256dh).is_err() { return Err(SetError::invalid_properties() @@ -377,6 +373,10 @@ fn validate_push_value( .with_description("Invalid data type.")); } } + + if !is_create { + access_token.restrict_data_types(&mut push.types); + } } (PushSubscriptionProperty::VerificationCode, Value::Str(value)) if !is_create => { if push.verification_code == value { @@ -392,6 +392,9 @@ fn validate_push_value( } (PushSubscriptionProperty::Types, Value::Null) => { push.types = Bitmap::all(); + if !is_create { + access_token.restrict_data_types(&mut push.types); + } } (PushSubscriptionProperty::VerificationCode, Value::Null) => {} (PushSubscriptionProperty::EmailPush, Value::Null) => { @@ -421,38 +424,8 @@ fn validate_push_value( Ok(()) } -fn validate_push_url(url: &str) -> Result<(), &'static str> { - let url = Url::parse(url).map_err(|_| "Invalid push subscription URL.")?; - - if url.scheme() != "https" { - return Err("Push subscription URLs must use the https scheme."); - } - - if !url.username().is_empty() || url.password().is_some() { - return Err("Push subscription URLs must not contain credentials."); - } - - let Some(host) = url.host_str() else { - return Err("Push subscription URLs must contain a host."); - }; - - let host = host - .strip_prefix('[') - .and_then(|host| host.strip_suffix(']')) - .unwrap_or(host); - - if let Ok(ip) = host.parse::() { - #[cfg(feature = "test_mode")] - if ip.is_loopback() { - return Ok(()); - } - - if !is_global_ip(&ip) { - return Err("Push subscription URLs must not point to a local or reserved IP address."); - } - } - - Ok(()) +fn validate_push_url(url: &str) -> Result<(), OutboundError> { + OutboundPolicy::PublicOnly.parse_url(url).map(|_| ()) } fn parse_email_push( @@ -520,12 +493,13 @@ fn parse_urgency( value .as_str() .and_then(|value| { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), Urgency, "very-low" => Urgency::VeryLow, "low" => Urgency::Low, "normal" => Urgency::Normal, "high" => Urgency::High, ) + .copied() }) .ok_or_else(|| email_push_error("Invalid urgency value.")) } @@ -536,61 +510,3 @@ fn email_push_error(description: &'static str) -> SetError, - access_token: &AccessToken, + _access_token: &AccessToken, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ @@ -61,12 +61,6 @@ impl QuotaGet for Server { not_found: not_found_ids, }; - let account = if account_id == access_token.account_id() { - Cow::Borrowed(&account) - } else { - Cow::Owned(self.account(account_id).await.caused_by(trc::location!())?) - }; - for id in ids { // Obtain the sieve script object let document_id = id.document_id(); @@ -83,15 +77,12 @@ impl QuotaGet for Server { QuotaProperty::Used => { (self.get_used_quota_account(account_id).await?.max(0) as u64).into() } - QuotaProperty::HardLimit => account.as_ref().disk_quota().into(), + QuotaProperty::HardLimit => account.disk_quota().into(), QuotaProperty::Scope => "account".to_string().into(), - QuotaProperty::Name => account.as_ref().name().to_string().into(), - QuotaProperty::Description => account - .as_ref() - .description - .as_ref() - .map(|s| s.to_string()) - .into(), + QuotaProperty::Name => account.name().to_string().into(), + QuotaProperty::Description => { + account.description.as_ref().map(|s| s.to_string()).into() + } QuotaProperty::Types => vec![ Value::Element(QuotaValue::Types(DataType::Email)), Value::Element(QuotaValue::Types(DataType::SieveScript)), diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index 074eefa1..aa73493d 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -8,8 +8,9 @@ use crate::registry::{ EnterpriseRegistry, mapping::{ RegistryGetResponse, account::account_get, bootstrap::bootstrap_get, - cluster::cluster_node_get, log::log_get, queued_message::queued_message_get, - report::report_get, spam_sample::spam_sample_get, task::task_get, + cluster::cluster_node_get, index_queue::index_queue_status_get, log::log_get, + queued_message::queued_message_get, report::report_get, spam_sample::spam_sample_get, + task::task_get, }, }; use common::{Server, auth::AccessToken, network::dkim::generate_dkim_public_key}; @@ -19,13 +20,10 @@ use jmap_proto::{ }; use jmap_tools::Key; use registry::{ - jmap::{IntoValue, JmapValue, RegistryValue}, + jmap::{IntoValue, JmapValue, RegistryValue, ValueOptions}, schema::{ enums::Permission, - prelude::{ - OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, - Property, - }, + prelude::{OBJ_FILTER_ACCOUNT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, Property}, structs::Account, }, types::id::ObjectId, @@ -61,14 +59,11 @@ impl RegistryGet for Server { self.assert_enterprise_object(object_type)?; let object_flags = object_type.flags(); - let is_tenant_filtered = - (object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); let is_account_filtered = (object_flags & OBJ_FILTER_ACCOUNT) != 0 && !access_token.has_permission(Permission::Impersonate); let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; let has_properties = request.properties.is_some(); let mut get = RegistryGetResponse { - access_token, server: self, account_id: request.account_id.document_id(), object_type, @@ -88,7 +83,6 @@ impl RegistryGet for Server { not_found: not_found_ids, }, object_flags, - is_tenant_filtered, is_account_filtered, }; if has_properties { @@ -98,8 +92,6 @@ impl RegistryGet for Server { match object_type { ObjectType::AcmeProvider | ObjectType::AddressBook - | ObjectType::AiModel - | ObjectType::Alert | ObjectType::AllowedIp | ObjectType::Application | ObjectType::Asn @@ -120,7 +112,6 @@ impl RegistryGet for Server { | ObjectType::DnsResolver | ObjectType::DnsServer | ObjectType::Email - | ObjectType::Enterprise | ObjectType::EventTracingLevel | ObjectType::FileStorage | ObjectType::Http @@ -132,8 +123,8 @@ impl RegistryGet for Server { | ObjectType::SystemSettings | ObjectType::MemoryLookupKey | ObjectType::MemoryLookupKeyValue + | ObjectType::Metadata | ObjectType::Metrics - | ObjectType::MetricsStore | ObjectType::MtaConnectionStrategy | ObjectType::MtaDeliverySchedule | ObjectType::MtaExtensions @@ -171,7 +162,6 @@ impl RegistryGet for Server { | ObjectType::SpamDnsblServer | ObjectType::SpamDnsblSettings | ObjectType::SpamFileExtension - | ObjectType::SpamLlm | ObjectType::SpamPyzor | ObjectType::SpamRule | ObjectType::SpamSettings @@ -181,166 +171,23 @@ impl RegistryGet for Server { | ObjectType::TaskManager | ObjectType::TlsReportSettings | ObjectType::Tracer - | ObjectType::TracingStore | ObjectType::WebDav | ObjectType::WebHook | ObjectType::Account | ObjectType::DsnReportSettings | ObjectType::MailingList | ObjectType::OAuthClient + | ObjectType::OAuthGrant | ObjectType::Role - | ObjectType::Tenant - | ObjectType::MaskedEmail | ObjectType::PublicKey | ObjectType::DkimSignature - | ObjectType::Domain => { - let is_singleton = (get.object_flags & OBJ_SINGLETON) != 0; - - let ids = if let Some(ids) = get.ids.take() { - ids - } else { - self.registry() - .query::>( - RegistryQuery::new(object_type) - .with_tenant(access_token.tenant_id()) - .with_account_opt(is_account_filtered.then_some(get.account_id)) - .with_limit(self.core.jmap.get_max_objects), - ) - .await - .caused_by(trc::location!())? - }; - get.response.list.reserve(ids.len()); - - for id in ids { - let object = if let Some(object) = self - .registry() - .get(ObjectId::new(object_type, id)) - .await - .caused_by(trc::location!())? - { - if (is_tenant_filtered - && access_token.tenant_id().map(Id::from) - != object.inner.member_tenant_id()) - || (is_account_filtered - && object.inner.account_id() != Some(Id::from(get.account_id))) - { - get.not_found(id); - continue; - } - object - } else if id.is_singleton() && is_singleton { - Object::from(object_type) - } else { - get.not_found(id); - continue; - }; - - let mut extra_properties: VecMap = VecMap::new(); - match &object.inner { - ObjectInner::DkimSignature(obj) - if get.properties.is_empty() - || get.properties.contains(&Property::PublicKey) => - { - if let Ok(public_key) = generate_dkim_public_key(obj).await { - extra_properties - .append(Property::PublicKey, JmapValue::Str(public_key.into())); - } - } - ObjectInner::Account(obj) => { - if get.properties.is_empty() - || get.properties.contains(&Property::UsedDiskQuota) - { - let quota = self.get_used_quota_account(id.document_id()).await?; - extra_properties.append( - Property::UsedDiskQuota, - JmapValue::Number(quota.into()), - ); - } - if get.properties.is_empty() - || get.properties.contains(&Property::EmailAddress) - { - let (name, domain_id) = match &obj { - Account::User(obj) => (obj.name.as_str(), obj.domain_id), - Account::Group(obj) => (obj.name.as_str(), obj.domain_id), - }; - let domain = self.domain_by_id(domain_id.document_id()).await?; - let email = format!( - "{}@{}", - name, - domain.as_ref().map(|d| d.name()).unwrap_or_default() - ); - extra_properties - .append(Property::EmailAddress, JmapValue::Str(email.into())); - } - } - ObjectInner::MailingList(obj) - if get.properties.is_empty() - || get.properties.contains(&Property::EmailAddress) => - { - let domain = self.domain_by_id(obj.domain_id.document_id()).await?; - let email = format!( - "{}@{}", - obj.name, - domain.as_ref().map(|d| d.name()).unwrap_or_default() - ); - extra_properties - .append(Property::EmailAddress, JmapValue::Str(email.into())); - } - ObjectInner::Tenant(obj) - if get.properties.is_empty() - || get.properties.contains(&Property::UsedDiskQuota) => - { - let quota = self.get_used_quota_tenant(id.document_id()).await?; - extra_properties - .append(Property::UsedDiskQuota, JmapValue::Number(quota.into())); - } - ObjectInner::Domain(obj) - if get.properties.is_empty() - || get.properties.contains(&Property::DnsZoneFile) => - { - extra_properties.append( - Property::DnsZoneFile, - JmapValue::Str(self.build_bind_dns_records(id, obj).await?.into()), - ); - } - ObjectInner::AcmeProvider(obj) - if get.properties.is_empty() - || get.properties.contains(&Property::Description) => - { - let mut description = obj.directory.clone(); - let account = obj - .account_uri - .rsplit('/') - .find(|segment| !segment.is_empty()) - .unwrap_or(obj.account_uri.as_str()); - if !account.is_empty() { - description.push_str(" ("); - description.push_str(account); - description.push(')'); - } - extra_properties - .append(Property::Description, JmapValue::Str(description.into())); - } - _ => {} - } - - let mut object = object.into_value(); - if !extra_properties.is_empty() - && let JmapValue::Object(obj) = &mut object - { - for (key, value) in extra_properties { - obj.insert_unchecked(key, value); - } - } - - get.insert(id, object); - } - - Ok(get.into_response()) - } + | ObjectType::Domain => registry_object_get(get).await, ObjectType::QueuedMessage => { queued_message_get(get).await.map(|get| get.into_response()) } + ObjectType::IndexQueueStatus => index_queue_status_get(get) + .await + .map(|get| get.into_response()), ObjectType::Task => task_get(get).await.map(|get| get.into_response()), ObjectType::ClusterNode => cluster_node_get(get).await.map(|get| get.into_response()), ObjectType::ArfExternalReport @@ -359,19 +206,135 @@ impl RegistryGet for Server { | ObjectType::AccountPassword | ObjectType::AppPassword => account_get(get).await.map(|get| get.into_response()), ObjectType::Action => Ok(get.not_found_any().into_response()), - #[cfg(not(feature = "enterprise"))] - _ => Ok(get.not_found_any().into_response()), } } } +async fn registry_object_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let server = get.server; + let value_options = ValueOptions { + include_secrets: server.core.network.include_secrets, + }; + let object_type = get.object_type; + let is_account_filtered = get.is_account_filtered; + let is_singleton = (get.object_flags & OBJ_SINGLETON) != 0; + + let query = RegistryQuery::new(object_type); + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + server + .registry() + .query::>( + query + .with_account_opt(is_account_filtered.then_some(get.account_id)) + .with_limit(server.core.jmap.get_max_objects), + ) + .await + .caused_by(trc::location!())? + }; + get.response.list.reserve(ids.len()); + + for id in ids { + let object = if let Some(object) = server + .registry() + .get(ObjectId::new(object_type, id)) + .await + .caused_by(trc::location!())? + { + #[cfg(not(feature = "enterprise"))] + let is_tenant_mismatch = false; + if is_tenant_mismatch + || (is_account_filtered + && object.inner.account_id() != Some(Id::from(get.account_id))) + { + get.not_found(id); + continue; + } + object + } else if id.is_singleton() && is_singleton { + Object::from(object_type) + } else { + get.not_found(id); + continue; + }; + + let mut extra_properties: VecMap = VecMap::new(); + match &object.inner { + ObjectInner::DkimSignature(obj) + if get.properties.is_empty() || get.properties.contains(&Property::PublicKey) => + { + if let Ok(public_key) = generate_dkim_public_key(obj).await { + extra_properties.append(Property::PublicKey, JmapValue::Str(public_key.into())); + } + } + ObjectInner::Account(obj) => { + if get.properties.is_empty() || get.properties.contains(&Property::UsedDiskQuota) { + let quota = server.get_used_quota_account(id.document_id()).await?; + extra_properties + .append(Property::UsedDiskQuota, JmapValue::Number(quota.into())); + } + if get.properties.is_empty() || get.properties.contains(&Property::EmailAddress) { + let (name, domain_id) = match obj.as_ref() { + Account::User(obj) => (obj.name.as_str(), obj.domain_id), + Account::Group(obj) => (obj.name.as_str(), obj.domain_id), + }; + let domain = server.domain_by_id(domain_id.document_id()).await?; + let email = format!( + "{}@{}", + name, + domain.as_ref().map(|d| d.name()).unwrap_or_default() + ); + extra_properties.append(Property::EmailAddress, JmapValue::Str(email.into())); + } + } + ObjectInner::MailingList(obj) + if get.properties.is_empty() + || get.properties.contains(&Property::EmailAddress) => + { + let domain = server.domain_by_id(obj.domain_id.document_id()).await?; + let email = format!( + "{}@{}", + obj.name, + domain.as_ref().map(|d| d.name()).unwrap_or_default() + ); + extra_properties.append(Property::EmailAddress, JmapValue::Str(email.into())); + } + ObjectInner::Domain(obj) + if get.properties.is_empty() || get.properties.contains(&Property::DnsZoneFile) => + { + extra_properties.append( + Property::DnsZoneFile, + JmapValue::Str(server.build_bind_dns_records(id, obj).await?.into()), + ); + } + _ => {} + } + + let mut object = object.into_value_with(value_options); + if !extra_properties.is_empty() + && let JmapValue::Object(obj) = &mut object + { + for (key, value) in extra_properties { + obj.insert_unchecked(key, value); + } + } + + get.insert(id, object); + } + + Ok(get.into_response()) +} + impl RegistryGetResponse<'_> { pub fn insert(&mut self, id: Id, mut object: JmapValue<'static>) { let object_map = object.as_object_mut().unwrap(); - if self.is_tenant_filtered && self.access_token.tenant_id().is_some() { - object_map.remove(&Key::Property(Property::MemberTenantId)); - } else if self.is_account_filtered { + #[cfg(not(feature = "enterprise"))] + let is_tenant_removed = false; + if !is_tenant_removed && self.is_account_filtered { object_map.remove(&Key::Property(Property::AccountId)); } diff --git a/crates/jmap/src/registry/mapping/account.rs b/crates/jmap/src/registry/mapping/account.rs index 4a67652b..9b8489a1 100644 --- a/crates/jmap/src/registry/mapping/account.rs +++ b/crates/jmap/src/registry/mapping/account.rs @@ -26,11 +26,14 @@ use common::{ ipc::CacheInvalidation, storage::encryption::{EncryptionMethod, parse_public_key}, }; +use compact_str::{CompactString, format_compact}; use directory::core::secret::{SecretVerificationResult, hash_secret, verify_mfa_secret_hash}; use jmap_proto::{error::set::SetError, request::MaybeInvalid, types::state::State}; use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; use registry::{ - jmap::{IntoValue, JsonPointerPatch, MaybeUnpatched, RegistryJsonPatch, RegistryValue}, + jmap::{ + IntoValue, JsonPointerPatch, MaybeUnpatched, RegistryJsonPatch, RegistryValue, ValueOptions, + }, schema::{ enums::{CredentialType, StorageQuota}, prelude::{MASKED_PASSWORD, Object, ObjectInner, ObjectType, Property}, @@ -67,7 +70,9 @@ pub(crate) async fn account_set( return Ok(set); }; let revision = object.revision; - let old_account = if let ObjectInner::Account(Account::User(account)) = object.inner { + let old_account = if let ObjectInner::Account(account) = object.inner + && let Account::User(account) = *account + { account } else { set.fail_all(SetError::not_found()); @@ -144,7 +149,7 @@ pub(crate) async fn account_set( current_secret: None, otp_auth: OtpAuth { otp_code: None, - otp_url: if old_credential.otp_auth.is_some() { + otp_url: if old_credential.otp_url.is_some() { Some(MASKED_PASSWORD.to_string()) } else { None @@ -186,7 +191,7 @@ pub(crate) async fn account_set( let is_empty_otp = account_pass.otp_auth.otp_url.as_deref() == Some(MASKED_PASSWORD) || (account_pass.otp_auth.otp_url.is_none() - && old_credential.otp_auth.is_none()); + && old_credential.otp_url.is_none()); if !is_empty_secret || !is_empty_otp { let user_provided_secret = if !is_empty_secret { account_pass.secret.as_ref().unwrap() @@ -194,12 +199,12 @@ pub(crate) async fn account_set( old_credential.secret.as_str() }; if is_empty_otp { - account_pass.otp_auth.otp_url = old_credential.otp_auth.clone(); + account_pass.otp_auth.otp_url = old_credential.otp_url.clone(); } // Password changes are not supported when using external directories if (user_provided_secret != old_credential.secret - || account_pass.otp_auth.otp_url != old_credential.otp_auth) + || account_pass.otp_auth.otp_url != old_credential.otp_url) && set .server .domain_by_id(account.domain_id.document_id()) @@ -217,7 +222,7 @@ pub(crate) async fn account_set( } if user_provided_secret != old_credential.secret - || account_pass.otp_auth.otp_url != old_credential.otp_auth + || account_pass.otp_auth.otp_url != old_credential.otp_url { if old_credential.secret.is_empty() { set.response.not_updated.append( @@ -232,7 +237,8 @@ pub(crate) async fn account_set( let current_otp_code = account_pass.otp_auth.otp_code; if let Some(current_secret) = account_pass.current_secret { match verify_mfa_secret_hash( - old_credential.otp_auth.as_deref(), + &set.server.inner.data.password_verifier, + old_credential.otp_url.as_deref(), current_otp_code.as_deref(), &old_credential.secret, current_secret.as_ref(), @@ -259,7 +265,7 @@ pub(crate) async fn account_set( .ctx(trc::Key::RemoteIp, set.remote_ip) .ctx( trc::Key::AccountName, - account.name().to_string(), + CompactString::from(account.name()), )); } else { set.response.not_updated.append( @@ -310,6 +316,7 @@ pub(crate) async fn account_set( } old_credential.secret = hash_secret( + &set.server.inner.data.password_verifier, set.server.core.network.security.password_hash_algorithm, user_provided_secret.as_bytes().to_vec(), ) @@ -317,8 +324,8 @@ pub(crate) async fn account_set( .caused_by(trc::location!())?; } - if account_pass.otp_auth.otp_url != old_credential.otp_auth { - old_credential.otp_auth = account_pass.otp_auth.otp_url; + if account_pass.otp_auth.otp_url != old_credential.otp_url { + old_credential.otp_url = account_pass.otp_auth.otp_url; } } else { set.response.not_updated.append( @@ -435,6 +442,7 @@ pub(crate) async fn account_set( // Generate App password and hash secret let app_pass = AppPassword::new(last_credential_id as u32); credential.secret = hash_secret( + &set.server.inner.data.password_verifier, set.server.core.network.security.password_hash_algorithm, app_pass.secret.to_vec(), ) @@ -488,6 +496,7 @@ pub(crate) async fn account_set( // Generate API key and hash secret let api_key = ApiKey::new(set.account_id, last_credential_id as u32); credential.secret = hash_secret( + &set.server.inner.data.password_verifier, set.server.core.network.security.password_hash_algorithm, api_key.secret.to_vec(), ) @@ -641,13 +650,15 @@ pub(crate) async fn account_set( } if account.credentials != old_account.credentials { cache_invalidator.invalidate(CacheInvalidation::AccessToken(set.account_id)); + cache_invalidator.process_credentials_update( + set.account_id, + &old_account.credentials, + &account.credentials, + ); } - let object = Object::new(ObjectInner::Account(Account::User(account))); - let old_object = Object::with_revision( - ObjectInner::Account(Account::User(old_account.clone())), - revision, - ); + let object = Object::new(Account::User(account).into()); + let old_object = Object::with_revision(Account::User(old_account.clone()).into(), revision); match set .server @@ -708,6 +719,9 @@ pub(crate) async fn account_get( else { return Ok(get.not_found_any()); }; + let value_options = ValueOptions { + include_secrets: get.server.core.network.include_secrets, + }; match get.object_type { ObjectType::AccountSettings => { @@ -743,16 +757,17 @@ pub(crate) async fn account_get( .take() .unwrap_or_else(|| vec![Id::singleton()]) .into_iter(); + let mut password = account.credentials.into_iter().find_map(|pass| { + if let Credential::Password(pass) = pass { + Some(pass) + } else { + None + } + }); for id in ids.by_ref() { if id == Id::singleton() - && let Some(pass) = account.credentials.iter().find_map(|pass| { - if let Credential::Password(pass) = pass { - Some(pass) - } else { - None - } - }) + && let Some(pass) = password.take() { get.insert( id, @@ -760,15 +775,11 @@ pub(crate) async fn account_get( current_secret: None, otp_auth: OtpAuth { otp_code: None, - otp_url: if pass.otp_auth.is_some() { - MASKED_PASSWORD.to_string().into() - } else { - None - }, + otp_url: pass.otp_url, }, - secret: MASKED_PASSWORD.to_string().into(), + secret: pass.secret.into(), } - .into_value(), + .into_value_with(value_options), ); break; } else { @@ -796,7 +807,7 @@ pub(crate) async fn account_get( if ids.contains(&pass.credential_id) => { let id = pass.credential_id; - let mut credential = pass.into_value(); + let mut credential = pass.into_value_with(value_options); credential .as_object_mut() .unwrap() @@ -902,10 +913,12 @@ pub(crate) async fn credential_query( } } property => { - return Err(trc::JmapEvent::UnsupportedSort.into_err().details(format!( - "Property {} is not supported for sorting", - property - ))); + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(format_compact!( + "Property {} is not supported for sorting", + property + ))); } } diff --git a/crates/jmap/src/registry/mapping/action.rs b/crates/jmap/src/registry/mapping/action.rs index b43ab9d4..b72f6c89 100644 --- a/crates/jmap/src/registry/mapping/action.rs +++ b/crates/jmap/src/registry/mapping/action.rs @@ -13,28 +13,29 @@ use common::{ ipc::{BroadcastEvent, QueueEvent, RegistryChange}, }; use jmap_proto::error::set::{SetError, SetErrorType}; -use jmap_tools::{JsonPointer, Key}; +use jmap_tools::{JsonPointer, Key, Value}; use mail_auth::{ AuthenticatedMessage, Dkim2Result, DkimResult, DmarcResult, dkim2::Envelope as Dkim2Envelope, dmarc::verify::DmarcParameters, spf::verify::SpfParameters, }; use mail_parser::MessageParser; use registry::{ - jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, + jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch, RegistryValue}, schema::{ enums::{SpamClassifyParameters, SpamClassifyResult, SpamClassifyTagDisposition}, prelude::{ObjectType, Property}, - structs::{Action, DmarcTroubleshoot, SpamClassify, SpamClassifyTag}, + structs::{Action, DmarcTroubleshoot, ReloadSettingsResult, SpamClassify, SpamClassifyTag}, }, - types::{EnumImpl, ObjectImpl}, + types::{EnumImpl, ObjectImpl, map::Map}, }; use smtp_proto::{MAIL_BODY_7BIT, MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; use spam_filter::{ - SpamFilterInput, + MessageTexts, SpamFilterInput, analysis::{init::SpamFilterInit, score::SpamFilterAnalyzeScore}, }; use std::time::Instant; use store::{registry::bootstrap::Bootstrap, write::now}; +use types::id::Id; use utils::map::vec_map::VecMap; pub(crate) async fn action_set( @@ -77,12 +78,12 @@ pub(crate) async fn action_set( } match action { - Action::ReloadSettings + Action::ReloadSettings(_) | Action::ReloadTlsCertificates | Action::ReloadLookupStores | Action::ReloadBlockedIps => { let object = match action { - Action::ReloadSettings => ObjectType::DataStore, + Action::ReloadSettings(_) => ObjectType::DataStore, Action::ReloadTlsCertificates => ObjectType::Certificate, Action::ReloadLookupStores => ObjectType::StoreLookup, Action::ReloadBlockedIps => ObjectType::BlockedIp, @@ -90,11 +91,14 @@ pub(crate) async fn action_set( }; #[cfg(target_os = "linux")] ServiceState::Reloading.notify(); - let result = - Box::pin(set.server.reload_registry(RegistryChange::Reload(object))).await; + let result = set + .server + .reload_registry(RegistryChange::Reload(object)) + .await; #[cfg(target_os = "linux")] ServiceState::Ready.notify(); let result = result?; + result.log(); if !result.has_errors() { set.server @@ -102,7 +106,32 @@ pub(crate) async fn action_set( object, ))) .await; - set.response.created(id, now()); + if matches!(action, Action::ReloadSettings(_)) { + let warnings = result + .warnings + .into_iter() + .map(|warning| { + format!( + "{}: {}", + warning.object_id.object().as_str(), + warning.message + ) + }) + .collect(); + let mut result = ReloadSettingsResult { + warnings: Map::new(warnings), + } + .into_value(); + if let Some(object) = result.as_object_mut() { + object.as_mut_vec().push(( + Key::Property(Property::Id), + Value::Element(RegistryValue::Id(Id::from(now()))), + )); + } + set.response.created.insert(id, result); + } else { + set.response.created(id, now()); + } } else { set.response .not_created @@ -164,19 +193,18 @@ pub(crate) async fn action_set( | Property::RemoteIp | Property::EhloDomain | Property::MailFrom - | Property::To + | Property::RcptTo ) ) }); set.response.created.insert(id, result); } else { + let error = SetError::invalid_properties(); set.response.not_created.append( id, - SetError::invalid_properties() - .with_property(Property::Body) - .with_description( - "Failed to parse the message for DMARC troubleshooting".to_string(), - ), + error.with_description( + "Failed to parse the message for DMARC troubleshooting".to_string(), + ), ); } } @@ -196,9 +224,9 @@ pub(crate) async fn action_set( | Property::EhloDomain | Property::AuthenticatedAs | Property::IsTls - | Property::EnvFrom - | Property::EnvFromParameters - | Property::EnvRcptTo + | Property::MailFrom + | Property::MailFromParameters + | Property::RcptTo ) ) }); @@ -247,11 +275,11 @@ async fn classify_spam(server: &Server, mut request: SpamClassify) -> Option Option Option Option Option Option MAIL_BODY_7BIT, Some(SpamClassifyParameters::Bit8Mime8BitMIMEMessageContent) => MAIL_BODY_BINARYMIME, Some(SpamClassifyParameters::BinaryMime) => MAIL_BODY_8BITMIME, Some(SpamClassifyParameters::SmtpUtf8) => MAIL_SMTPUTF8, None => 0, }, - env_rcpt_orig_to: request.env_rcpt_to.iter().map(String::as_str).collect(), - env_rcpt_rewritten_to: request.env_rcpt_to.iter().map(String::as_str).collect(), + env_rcpt_orig_to: request.rcpt_to.iter().map(String::as_str).collect(), + env_rcpt_rewritten_to: request.rcpt_to.iter().map(String::as_str).collect(), is_test: true, is_train: false, }; @@ -409,6 +435,7 @@ async fn classify_spam(server: &Server, mut request: SpamClassify) -> Option SpamClassifyResult::Reject, }; + request.tags = VecMap::with_capacity(ctx.result.tags.len()); for tag in ctx.result.tags { let (score, disposition) = match server.core.spam.lists.scores.get(&tag) { @@ -417,7 +444,7 @@ async fn classify_spam(server: &Server, mut request: SpamClassify) -> Option (0.0, SpamClassifyTagDisposition::Reject), }; request.tags.append( - tag, + tag.into_string(), SpamClassifyTag { disposition, score: (score as f64).into(), @@ -449,11 +476,7 @@ async fn dmarc_troubleshoot( server .inner .cache - .build_auth_parameters(SpfParameters::verify_ehlo( - remote_ip, - &ehlo_domain, - local_host, - )), + .build_auth_parameters(SpfParameters::helo(remote_ip, &ehlo_domain, local_host)), ) .await; @@ -520,7 +543,7 @@ async fn dmarc_troubleshoot( server.inner.cache.build_auth_parameters(&auth_message), Dkim2Envelope { mail_from: &mail_from, - rcpt_to: request.to.iter(), + rcpt_to: request.rcpt_to.iter(), }, ) .await; @@ -543,7 +566,7 @@ async fn dmarc_troubleshoot( message: &auth_message, dkim_output: &dkim_output, dkim2_output: Some(&dkim2_output), - rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), + mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), spf_output: &mail_spf_output, })) .await; diff --git a/crates/jmap/src/registry/mapping/bootstrap.rs b/crates/jmap/src/registry/mapping/bootstrap.rs index 2314c490..39de890e 100644 --- a/crates/jmap/src/registry/mapping/bootstrap.rs +++ b/crates/jmap/src/registry/mapping/bootstrap.rs @@ -5,7 +5,10 @@ */ use crate::registry::{ - mapping::{RegistryGetResponse, RegistrySetResponse, map_bootstrap_error}, + mapping::{ + RegistryGetResponse, RegistrySetResponse, cluster::validate_in_memory_store, + map_bootstrap_error, + }, set::map_write_error, }; use common::{ @@ -20,12 +23,12 @@ use jmap_proto::{ use jmap_tools::{JsonPointer, JsonPointerItem, Key}; use rand::{RngExt, distr::Alphanumeric, rng}; use registry::{ - jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch}, + jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch, ValueOptions}, schema::{ enums::{AcmeChallengeType, DnsRecordType}, prelude::{Object, Property}, structs::{ - Account, AcmeProvider, BlobStore, Bootstrap, CertificateManagement, + Account, AcmeProvider, Authentication, BlobStore, Bootstrap, CertificateManagement, CertificateManagementProperties, Credential, DataStore, Directory, DirectoryBootstrap, DkimManagement, DkimManagementProperties, DnsManagement, DnsManagementProperties, DnsServer, DnsServerBootstrap, Domain, InMemoryStore, PasswordCredential, RocksDbStore, @@ -37,9 +40,9 @@ use registry::{ }; use std::time::Duration; use store::{ - RegistryStore, SUBSPACE_PROPERTY, Store, + RegistryStore, Store, Subspace, registry::write::{RegistryWrite, RegistryWriteResult}, - write::{AnyKey, BatchBuilder}, + write::{AnyKey, BatchBuilder, key::SystemKind}, }; use types::id::Id; use utils::{DomainPart, is_valid_domain}; @@ -62,7 +65,9 @@ pub(crate) async fn bootstrap_get( if id == Id::singleton() { get.insert( Id::singleton(), - build_default_bootstrap(get.server).into_value(), + build_default_bootstrap(get.server).into_value_with(ValueOptions { + include_secrets: get.server.core.network.include_secrets, + }), ); break; } else { @@ -125,7 +130,7 @@ pub(crate) async fn bootstrap_set( // Validate domain name and hostname let server_hostname = bootstrap - .server_hostname + .default_hostname .trim() .to_lowercase() .to_ascii_domain() @@ -142,7 +147,7 @@ pub(crate) async fn bootstrap_set( set.response.not_updated.append( id, SetError::invalid_properties() - .with_property(Property::ServerHostname) + .with_property(Property::DefaultHostname) .with_description("Invalid server hostname"), ); break; @@ -157,6 +162,13 @@ pub(crate) async fn bootstrap_set( break; } + if let Err(err) = validate_in_memory_store(set.server, &bootstrap.in_memory_store).await? { + set.response + .not_updated + .append(id, err.with_property(Property::InMemoryStore)); + break; + } + // Build store let store = match Store::build(bootstrap.data_store.clone()).await { Ok(store) => store, @@ -184,8 +196,8 @@ pub(crate) async fn bootstrap_set( // Make sure this is blank deployment let probe = store.get_value::(AnyKey { - subspace: SUBSPACE_PROPERTY, - key: vec![0u8], + subspace: Subspace::System, + key: vec![SystemKind::SchemaVersion as u8], }); match tokio::time::timeout(Duration::from_secs(30), probe).await { Ok(Ok(None)) => {} @@ -204,9 +216,8 @@ pub(crate) async fn bootstrap_set( SetError::invalid_properties() .with_property(Property::DataStore) .with_description(concat!( - "The selected data store contains information from an older version. ", - "Please follow the upgrade instructions at ", - "https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md" + "The selected data store was created by an older version ", + "or a different edition of Stalwart and cannot be used." )), ); break; @@ -339,6 +350,14 @@ pub(crate) async fn bootstrap_set( } } } + if let Some(directory_id) = directory_id + && let Err(err) = set_default_directory(®istry, directory_id).await + { + set.response + .not_updated + .append(id, err.with_property(Property::Directory)); + break 'outer; + } if let Some(dns_server) = map_dns_server(&bootstrap.dns_server) { match write_object(®istry, &dns_server.into()).await { Ok(id) => { @@ -367,6 +386,10 @@ pub(crate) async fn bootstrap_set( directory: "https://acme-v02.api.letsencrypt.org/directory".to_string(), #[cfg(feature = "dev_mode")] directory: "https://localhost:14000/dir".to_string(), + #[cfg(not(feature = "dev_mode"))] + description: "Let's Encrypt".to_string(), + #[cfg(feature = "dev_mode")] + description: "Pebble".to_string(), ..Default::default() }; if let Err(err) = acme_create_account(&mut acme_provider, None).await { @@ -400,7 +423,7 @@ pub(crate) async fn bootstrap_set( ]); let domain = Domain { name: domain_name.clone(), - is_enabled: true, + enable: true, certificate_management: if let Some(acme_provider_id) = acme_provider_id { CertificateManagement::Automatic(CertificateManagementProperties { acme_provider_id, @@ -423,7 +446,6 @@ pub(crate) async fn bootstrap_set( } else { DnsManagement::Manual }, - directory_id, ..Default::default() }; let domain_id = match write_object(®istry, &domain.into()).await { @@ -438,7 +460,7 @@ pub(crate) async fn bootstrap_set( // Write system settings let system_settings = SystemSettings { - default_hostname: bootstrap.server_hostname, + default_hostname: bootstrap.default_hostname, default_domain_id: domain_id, ..Default::default() }; @@ -474,7 +496,7 @@ pub(crate) async fn bootstrap_set( })); } if !batch.is_empty() { - match registry.store().write(batch.build_all()).await { + match registry.store().write_batch(&mut batch).await { Ok(_) => {} Err(err) => { trc::error!(err.caused_by(trc::location!())); @@ -496,6 +518,7 @@ pub(crate) async fn bootstrap_set( credentials: List::from_iter([Credential::Password(PasswordCredential { credential_id: Id::new(0), secret: hash_secret( + &set.server.inner.data.password_verifier, set.server.core.network.security.password_hash_algorithm, secret.clone().into_bytes(), ) @@ -537,7 +560,47 @@ pub(crate) async fn bootstrap_set( } async fn write_object(registry: &RegistryStore, object: &Object) -> Result> { - match registry.write(RegistryWrite::insert(object)).await { + apply_write(registry, RegistryWrite::insert(object)).await +} + +async fn set_default_directory( + registry: &RegistryStore, + directory_id: Id, +) -> Result> { + match registry.get(Authentication::OBJECT.singleton()).await { + Ok(Some(old_object)) => { + let mut authentication = Authentication::from(old_object.clone()); + authentication.directory_id = Some(directory_id); + apply_write( + registry, + RegistryWrite::update(Id::singleton(), &authentication.into(), &old_object), + ) + .await + } + Ok(None) => { + write_object( + registry, + &Authentication { + directory_id: Some(directory_id), + ..Default::default() + } + .into(), + ) + .await + } + Err(err) => { + let details = format!("Failed to save settings: {err}"); + trc::error!(err.caused_by(trc::location!())); + Err(SetError::invalid_properties().with_description(details)) + } + } +} + +async fn apply_write( + registry: &RegistryStore, + write: RegistryWrite<'_>, +) -> Result> { + match registry.write(write).await { Ok(RegistryWriteResult::Success(id)) => Ok(id), Ok(err) => Err(map_write_error(err)), Err(err) => { @@ -559,7 +622,7 @@ fn map_directory(directory: &DirectoryBootstrap) -> Option { fn map_dns_server(dns_server: &DnsServerBootstrap) -> Option { match dns_server { - DnsServerBootstrap::Manual | DnsServerBootstrap::Deprecated1 => None, + DnsServerBootstrap::Manual => None, DnsServerBootstrap::Tsig(dns_server_tsig) => { DnsServer::Tsig(dns_server_tsig.clone()).into() } @@ -678,12 +741,12 @@ fn build_default_bootstrap(server: &Server) -> Bootstrap { directory: DirectoryBootstrap::Internal, tracer: Tracer::Log(TracerLog { path: "/var/log/inbuxa/".to_string(), - prefix: "inbuxa".to_string(), + prefix: "stalwart.log".to_string(), ansi: true, enable: true, ..Default::default() }), - server_hostname, + default_hostname: server_hostname, default_domain, request_tls_certificate: true, generate_dkim_keys: true, diff --git a/crates/jmap/src/registry/mapping/cluster.rs b/crates/jmap/src/registry/mapping/cluster.rs index bfe05edf..42564b3b 100644 --- a/crates/jmap/src/registry/mapping/cluster.rs +++ b/crates/jmap/src/registry/mapping/cluster.rs @@ -4,13 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap_proto::{object::registry::RegistryComparator, types::state::State}; -use registry::{jmap::IntoValue, schema::prelude::Property}; +use common::Server; +use jmap_proto::{error::set::SetError, object::registry::RegistryComparator, types::state::State}; +use registry::{ + jmap::IntoValue, + schema::{ + prelude::Property, + structs::{Coordinator, InMemoryStore}, + }, +}; use store::ahash::AHashSet; +use types::id::Id; use crate::{ api::query::QueryResponseBuilder, - registry::mapping::{RegistryGetResponse, RegistryQueryResponse}, + registry::mapping::{ + ObjectResponse, RegistryGetResponse, RegistryQueryResponse, ValidationResult, + }, }; pub(crate) async fn cluster_node_get( @@ -55,7 +65,7 @@ pub(crate) async fn cluster_node_query( { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details("Only sorting by 'nodeId' is supported for cluster nodes".to_string())); + .details("Only sorting by 'nodeId' is supported for cluster nodes")); } let nodes = req.server.registry().cluster_node_list().await?; @@ -76,3 +86,51 @@ pub(crate) async fn cluster_node_query( Ok(response) } + +pub(crate) async fn validate_in_memory_store( + server: &Server, + store: &InMemoryStore, +) -> ValidationResult { + if matches!(store, InMemoryStore::Default) + && !matches!( + server + .registry() + .object::(Id::singleton()) + .await? + .unwrap_or_default(), + Coordinator::Disabled + ) + { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Type) + .with_description( + "The local in-memory store cannot be used while a cluster coordinator is configured, disable the coordinator first.", + ))); + } + + Ok(Ok(ObjectResponse::default())) +} + +pub(crate) async fn validate_coordinator( + server: &Server, + coordinator: &Coordinator, +) -> ValidationResult { + if !matches!(coordinator, Coordinator::Disabled) + && matches!( + server + .registry() + .object::(Id::singleton()) + .await? + .unwrap_or_default(), + InMemoryStore::Default + ) + { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Type) + .with_description( + "A cluster coordinator requires a Redis in-memory store, configure one first.", + ))); + } + + Ok(Ok(ObjectResponse::default())) +} diff --git a/crates/jmap/src/registry/mapping/dkim.rs b/crates/jmap/src/registry/mapping/dkim.rs index 3126bcf9..376dfbdc 100644 --- a/crates/jmap/src/registry/mapping/dkim.rs +++ b/crates/jmap/src/registry/mapping/dkim.rs @@ -4,34 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::{ - ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, -}; +use crate::registry::mapping::{ObjectResponse, ValidationResult}; use common::config::smtp::auth::DkimSigners; use jmap_proto::error::set::SetError; -use registry::schema::{enums::TenantStorageQuota, structs::DkimSignature}; +use registry::schema::structs::DkimSignature; pub(crate) async fn validate_dkim_signature( - set: &RegistrySetResponse<'_>, key: &mut DkimSignature, old_key: Option<&DkimSignature>, ) -> ValidationResult { - let response = if old_key.is_none() { - match validate_tenant_quota( - set.server, - set.access_token, - TenantStorageQuota::MaxDkimKeys, - ) - .await? - { - Ok(response) => response, - Err(err) => { - return Ok(Err(err)); - } - } - } else { - ObjectResponse::default() - }; + let response = ObjectResponse::default(); if old_key.is_none_or(|old_key| old_key.private_key() != key.private_key()) && let Err(err) = DkimSigners::default() diff --git a/crates/jmap/src/registry/mapping/domain.rs b/crates/jmap/src/registry/mapping/domain.rs index 1d703881..ee20c8d7 100644 --- a/crates/jmap/src/registry/mapping/domain.rs +++ b/crates/jmap/src/registry/mapping/domain.rs @@ -4,14 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::{ - ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, -}; +use crate::registry::mapping::{ObjectResponse, RegistrySetResponse, ValidationResult}; use common::network::{dkim::generate_dkim_selector, dns::update::DnsUpdater}; use jmap_proto::error::set::{SetError, SetErrorType}; use registry::{ schema::{ - enums::{AcmeChallengeType, DkimSignatureType, DnsRecordType, TenantStorageQuota}, + enums::{AcmeChallengeType, DkimSignatureType, DnsRecordType}, prelude::{ObjectType, Property}, structs::{ AcmeProvider, CertificateManagement, DkimManagement, DkimManagementProperties, @@ -21,26 +19,20 @@ use registry::{ }, types::map::Map, }; +use std::borrow::Cow; use types::id::Id; +use utils::outbound::{OutboundError, OutboundPolicy}; +#[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] pub(crate) async fn validate_domain( set: &RegistrySetResponse<'_>, domain: &mut Domain, old_domain: Option<&Domain>, + outbound_policy: OutboundPolicy, tasks: &mut Vec, ) -> ValidationResult { - let response = if old_domain.is_none() { - match validate_tenant_quota(set.server, set.access_token, TenantStorageQuota::MaxDomains) - .await? - { - Ok(response) => response, - Err(err) => { - return Ok(Err(err)); - } - } - } else { - ObjectResponse::default() - }; + + let response = ObjectResponse::default(); // Validate DKIM selector template if let DkimManagement::Automatic(DkimManagementProperties { @@ -63,45 +55,38 @@ pub(crate) async fn validate_domain( .with_description(err))); } - // Validate that names and aliases do not collide with another domain - let registry = set.server.registry(); - if old_domain.is_none_or(|old| old.name != domain.name) - && let Some(existing) = registry - .primary_key( - ObjectType::Domain.into(), - Property::Aliases, - domain.name.as_bytes().to_vec(), - ) + let check_name = + old_domain.is_none_or(|old| old.name != domain.name && !old.aliases.contains(&domain.name)); + let mut lookups = Vec::with_capacity(usize::from(check_name) + domain.aliases.len() * 2); + if check_name { + lookups.push((Property::Aliases, domain.name.as_bytes())); + } + for alias in domain.aliases.iter().filter(|alias| { + *alias != &domain.name + && old_domain.is_none_or(|old| !old.aliases.contains(*alias) && &old.name != *alias) + }) { + lookups.push((Property::Name, alias.as_bytes())); + lookups.push((Property::Aliases, alias.as_bytes())); + } + if !lookups.is_empty() + && let Some((position, existing)) = set + .server + .registry() + .first_primary_key(ObjectType::Domain.into(), &lookups) .await? { return Ok(Err(SetError::new(SetErrorType::PrimaryKeyViolation) - .with_property(Property::Name) + .with_property(if check_name && position == 0 { + Property::Name + } else { + Property::Aliases + }) .with_object_id(existing))); } - for alias in domain.aliases.iter() { - if alias == &domain.name - || old_domain.is_some_and(|old| old.aliases.contains(alias) || &old.name == alias) - { - continue; - } - - for index in [Property::Name, Property::Aliases] { - if let Some(existing) = registry - .primary_key(ObjectType::Domain.into(), index, alias.as_bytes().to_vec()) - .await? - { - return Ok(Err(SetError::new(SetErrorType::PrimaryKeyViolation) - .with_property(Property::Aliases) - .with_object_id(existing))); - } - } - } - // Schedule DNS update task let will_trigger_dkim = matches!(domain.dkim_management, DkimManagement::Automatic(_)) - && old_domain - .is_none_or(|old| !matches!(old.dkim_management, DkimManagement::Automatic(_))); + && old_domain.is_none_or(|old| old.dkim_management != domain.dkim_management); let will_schedule_dkim = !will_trigger_dkim && matches!(domain.dkim_management, DkimManagement::Automatic(_)) && publishes_dkim(domain) @@ -151,12 +136,7 @@ pub(crate) async fn validate_domain( // Schedule ACME renewal task if needed if !will_trigger_acme && let CertificateManagement::Automatic(details) = &domain.certificate_management - && old_domain.is_none_or(|old| { - !matches!( - old.certificate_management, - CertificateManagement::Automatic(_) - ) - }) + && old_domain.is_none_or(|old| old.certificate_management != domain.certificate_management) { let Some(provider) = set .server @@ -189,33 +169,25 @@ pub(crate) async fn validate_domain( } fn publishes_dkim(domain: &Domain) -> bool { - matches!( - &domain.dns_management, - DnsManagement::Automatic(details) if details.publish_records.contains(&DnsRecordType::Dkim) - ) + domain.dns_management.publishes(DnsRecordType::Dkim) } pub(crate) async fn validate_dns_server( set: &RegistrySetResponse<'_>, dns: &mut DnsServer, old_dns: Option<&DnsServer>, + outbound_policy: OutboundPolicy, ) -> ValidationResult { - let response = if old_dns.is_none() { - match validate_tenant_quota( - set.server, - set.access_token, - TenantStorageQuota::MaxDnsServers, - ) - .await? - { - Ok(response) => response, - Err(err) => { - return Ok(Err(err)); - } - } - } else { - ObjectResponse::default() - }; + if outbound_policy == OutboundPolicy::PublicOnly + && old_dns.is_none_or(|old_dns| old_dns != dns) + && let Err((property, err)) = validate_dns_server_destination(dns).await + { + return Ok(Err(SetError::invalid_properties() + .with_property(property) + .with_description(err.as_str()))); + } + + let response = ObjectResponse::default(); if old_dns.is_none_or(|old_dns| old_dns != dns) && let Err(err) = DnsUpdater::build(dns.clone(), set.server.core.clone()).await @@ -226,3 +198,71 @@ pub(crate) async fn validate_dns_server( Ok(Ok(response)) } + +async fn validate_dns_server_destination(dns: &DnsServer) -> Result<(), (Property, OutboundError)> { + let policy = OutboundPolicy::PublicOnly; + let check_url = |url: &str| { + policy + .parse_url(url) + .map(|_| ()) + .map_err(|err| (Property::Url, err)) + }; + let check_host = |host: &str| { + if host.contains("://") { + policy.parse_url(host) + } else { + policy.parse_url(&format!("https://{host}")) + } + .map(|_| ()) + .map_err(|err| (Property::Host, err)) + }; + + match dns { + DnsServer::Tsig(server) => policy + .check_ip(server.host.into_inner()) + .map_err(|err| (Property::Host, err)), + DnsServer::BluecatV2(server) => check_url(&server.url), + DnsServer::Cpanel(server) => check_url(&server.url), + DnsServer::Plesk(server) => check_url(&server.url), + DnsServer::PowerDns(server) => server + .url + .as_deref() + .ok_or((Property::Url, OutboundError::MissingHost)) + .and_then(check_url), + DnsServer::UltraDns(server) => server.url.as_deref().map_or(Ok(()), check_url), + DnsServer::EdgeDns(server) => check_host(&server.host), + DnsServer::Infoblox(server) => check_host(&server.host), + DnsServer::Volcengine(server) => { + if server + .scheme + .as_deref() + .is_some_and(|scheme| !scheme.eq_ignore_ascii_case("https")) + { + Err((Property::Scheme, OutboundError::InsecureScheme)) + } else { + server.host.as_deref().map_or(Ok(()), check_host) + } + } + DnsServer::GoogleCloudDns(server) => { + let Ok(service_account) = server.service_account_json.secret().await else { + return Ok(()); + }; + serde_json::from_str::>(&service_account) + .ok() + .and_then(|account| account.token_uri) + .map_or(Ok(()), |token_uri| { + policy + .parse_url(&token_uri) + .map(|_| ()) + .map_err(|err| (Property::ServiceAccountJson, err)) + }) + } + _ => Ok(()), + } +} + +#[derive(serde::Deserialize)] +struct ServiceAccountTokenUri<'x> { + #[serde(borrow)] + token_uri: Option>, +} diff --git a/crates/jmap/src/registry/mapping/index_queue.rs b/crates/jmap/src/registry/mapping/index_queue.rs new file mode 100644 index 00000000..c99bbe55 --- /dev/null +++ b/crates/jmap/src/registry/mapping/index_queue.rs @@ -0,0 +1,483 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + api::query::QueryResponseBuilder, + registry::{ + mapping::{RegistryGetResponse, RegistryQueryResponse, RegistrySetResponse}, + query::RegistryQueryFilters, + }, +}; +use common::{Server, storage::search::SearchIndexStatus}; +use jmap_proto::{error::set::SetError, object::registry::RegistryComparator, types::state::State}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use registry::{ + jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, + schema::{ + enums::{IndexStatusType, IndexType}, + prelude::Property, + structs::{IndexQueueStatus, IndexStatus, IndexStatusFailed}, + }, + types::{EnumImpl, datetime::UTCDateTime}, +}; +use store::{ + IterateParams, U32_LEN, ValueKey, + registry::RegistryFilterOp, + write::{ + BatchBuilder, QueueNotify, SearchIndex, SearchIndexClass, ValueClass, + key::DeserializeBigEndian, + }, +}; +use trc::AddContext; +use types::id::Id; + +const MAX_QUEUED_ITEMS: u64 = 10000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct QueuePartition { + index: SearchIndex, + partition: u32, +} + +pub(crate) async fn index_queue_status_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + scan_partitions( + get.server, + None, + None, + None, + get.server.core.jmap.get_max_objects, + true, + ) + .await? + }; + + // Counting the queued documents of every partition is bounded by a single budget + // shared by the whole request + let mut budget = MAX_QUEUED_ITEMS; + + for id in ids { + match QueuePartition::from_id(id) { + Some(partition) => match build_status(get.server, partition, &mut budget).await? { + Some(status) => get.insert(id, status.into_value()), + None => get.not_found(id), + }, + None => get.not_found(id), + } + } + + Ok(get) +} + +pub(crate) async fn index_queue_status_query( + mut req: RegistryQueryResponse<'_>, +) -> trc::Result { + let mut index = None; + let mut partition = None; + let mut status = None; + + req.request + .extract_filters(|property, op, value| match property { + Property::Index => { + if matches!(op, RegistryFilterOp::Equal) + && let Some(index_) = value + .as_str() + .and_then(IndexType::parse) + .and_then(search_index) + { + index = Some(index_); + true + } else { + false + } + } + Property::Partition => { + if matches!(op, RegistryFilterOp::Equal) + && let Some(partition_) = value.as_str().and_then(|v| v.parse::().ok()) + { + partition = Some(partition_); + true + } else { + false + } + } + Property::Status => { + if matches!(op, RegistryFilterOp::Equal) + && let Some(status_) = value.as_str().and_then(IndexStatusType::parse) + { + status = Some(status_); + true + } else { + false + } + } + _ => false, + })?; + + if req + .request + .sort + .as_ref() + .and_then(|sort| sort.first()) + .is_some_and(|comp| !matches!(comp.property, RegistryComparator::Property(Property::Id))) + { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details("Only sorting by 'id' is supported for index queues")); + } + + // Paging by anchor is resolved by the response builder, which needs the full result set + let has_anchor = req.request.anchor.is_some(); + let params = req + .request + .extract_parameters(req.server.core.jmap.query_max_results, None)?; + let mut response = QueryResponseBuilder::new( + req.server.core.jmap.query_max_results + 1, + req.server.core.jmap.query_max_results, + State::Initial, + &req.request, + ); + + let max_partitions = match req.request.position { + _ if has_anchor || response.response.total.is_some() => usize::MAX, + Some(position) if position < 0 => usize::MAX, + position => usize::try_from(position.unwrap_or(0)) + .unwrap_or(usize::MAX) + .saturating_add(response.limit) + .saturating_add(1), + }; + let partitions = scan_partitions( + req.server, + index, + partition, + status, + max_partitions, + params.sort_ascending, + ) + .await?; + + let mut total = 0; + for id in partitions { + total += 1; + if response.response.total.is_some() { + if !response.is_full() { + response.add_id(id); + } + } else if !response.add_id(id) { + break; + } + } + + if response.response.total.is_some() { + response.response.total = Some(total); + } + + if let Some(limit) = response.response.limit + && total < limit + { + response.response.limit = None; + } + + Ok(response) +} + +pub(crate) async fn index_queue_status_set( + mut set: RegistrySetResponse<'_>, +) -> trc::Result> { + set.fail_all_create("Index queues cannot be created"); + set.fail_all_destroy("Index queues cannot be deleted"); + + let mut batch = BatchBuilder::new(); + let mut has_changes = false; + + 'outer: for (id, value) in set.update.drain(..) { + let Some(partition) = QueuePartition::from_id(id) else { + set.response.not_updated.append(id, SetError::not_found()); + continue; + }; + + let stored = set + .server + .search_index_status(partition.index, partition.partition) + .await + .caused_by(trc::location!())?; + let attempts = stored.as_ref().map_or(0, |stored| stored.attempts); + let Some(mut status) = status_object(partition, 0, 0, stored) else { + set.response.not_updated.append(id, SetError::not_found()); + continue; + }; + let previous = status.status.clone(); + + for (key, value) in value.into_expanded_object() { + let ptr = match key { + Key::Property(prop) => { + JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]) + } + Key::Borrowed(other) => JsonPointer::parse(other), + Key::Owned(other) => JsonPointer::parse(&other), + }; + + if let Err(err) = status.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { + set.response.not_updated.append(id, err.into()); + continue 'outer; + } + } + + if status.status != previous { + match &status.status { + IndexStatus::Running => { + batch.clear(partition.status_class()); + } + IndexStatus::Failed(failed) => { + batch.set( + partition.status_class(), + SearchIndexStatus { + next_retry: failed.next_retry.timestamp().max(0) as u64, + attempts, + reason: failed.reason.clone(), + } + .serialize(), + ); + } + } + + batch.commit_point(); + has_changes = true; + } + + set.response.updated.append(id, None); + } + + if has_changes { + set.server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; + set.server + .notify_queues(QueueNotify { + search_index: true, + ..Default::default() + }) + .await; + } + + Ok(set) +} + +async fn build_status( + server: &Server, + partition: QueuePartition, + budget: &mut u64, +) -> trc::Result> { + let stored = server + .search_index_status(partition.index, partition.partition) + .await + .caused_by(trc::location!())?; + let (queued_updates, queued_deletions) = queued_items(server, partition, budget) + .await + .caused_by(trc::location!())?; + + if stored.is_none() && queued_updates == 0 && queued_deletions == 0 { + return Ok(None); + } + + Ok(status_object( + partition, + queued_updates, + queued_deletions, + stored, + )) +} + +fn status_object( + partition: QueuePartition, + queued_updates: u64, + queued_deletions: u64, + stored: Option, +) -> Option { + Some(IndexQueueStatus { + index: index_type(partition.index)?, + partition: partition.partition as u64, + queued_updates, + queued_deletions, + status: match stored { + Some(stored) => IndexStatus::Failed(IndexStatusFailed { + reason: stored.reason, + next_retry: UTCDateTime::from_timestamp(stored.next_retry as i64), + }), + None => IndexStatus::Running, + }, + }) +} + +async fn queued_items( + server: &Server, + partition: QueuePartition, + budget: &mut u64, +) -> trc::Result<(u64, u64)> { + if *budget == 0 { + return Ok((0, 0)); + } + + let (from_class, to_class) = + SearchIndexClass::queue_range(partition.index, partition.partition); + let mut queued_updates = 0; + let mut queued_deletions = 0; + + server + .store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::SearchIndex(from_class)), + ValueKey::from(ValueClass::SearchIndex(to_class)), + ) + .limit(usize::try_from(*budget).unwrap_or(usize::MAX)), + |_, value| { + if value.first().is_some_and(|flag| *flag != 0) { + queued_updates += 1; + } else { + queued_deletions += 1; + } + + Ok(queued_updates + queued_deletions < *budget) + }, + ) + .await?; + + *budget -= queued_updates + queued_deletions; + + Ok((queued_updates, queued_deletions)) +} + +async fn scan_partitions( + server: &Server, + index: Option, + partition: Option, + status: Option, + max_results: usize, + ascending: bool, +) -> trc::Result> { + let (from_key, to_key) = SearchIndexClass::control_range( + (index.unwrap_or(SearchIndex::Email), partition.unwrap_or(0)), + index.map(|index| (index, partition.unwrap_or(u32::MAX))), + ); + + // Every partition is stored as a queue index key optionally followed by a status key, + // so a partition is only emitted once the next key reveals whether it has a status + let mut ids: Vec = Vec::new(); + let mut pending: Option<(QueuePartition, bool)> = None; + let matches_status = |has_status: bool| match status { + Some(IndexStatusType::Failed) => has_status, + Some(IndexStatusType::Running) => !has_status, + None => true, + }; + + server + .store() + .iterate( + IterateParams::new(from_key, to_key) + .set_ascending(ascending) + .no_values() + .expect_rows(max_results.saturating_mul(2).saturating_add(2)), + |key, _| { + if key.len() != U32_LEN + 3 { + return Ok(true); + } + let Some(index) = SearchIndex::try_from_u8(key[1]) else { + return Ok(true); + }; + let queue_partition = QueuePartition { + index, + partition: key.deserialize_be_u32(2)?, + }; + if partition.is_some_and(|partition| partition != queue_partition.partition) { + return Ok(true); + } + let is_status = key[U32_LEN + 2] == SearchIndexClass::CONTROL_STATUS; + + match &mut pending { + Some((last, has_status)) if *last == queue_partition => { + *has_status |= is_status; + return Ok(true); + } + _ => (), + } + + if let Some((last, has_status)) = pending.replace((queue_partition, is_status)) + && matches_status(has_status) + { + ids.push(last.to_id()); + if ids.len() >= max_results { + pending = None; + return Ok(false); + } + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if let Some((last, has_status)) = pending + && matches_status(has_status) + && ids.len() < max_results + { + ids.push(last.to_id()); + } + + Ok(ids) +} + +impl QueuePartition { + fn from_id(id: Id) -> Option { + let id = id.id(); + + if id >> 40 != 0 { + return None; + } + + SearchIndex::try_from_u8((id >> 32) as u8).map(|index| QueuePartition { + index, + partition: id as u32, + }) + } + + fn to_id(self) -> Id { + Id::from(((self.index.to_u8() as u64) << 32) | self.partition as u64) + } + + fn status_class(&self) -> ValueClass { + ValueClass::SearchIndex(SearchIndexClass::QueueStatus { + index: self.index, + partition: self.partition, + }) + } +} + +fn index_type(index: SearchIndex) -> Option { + match index { + SearchIndex::Email => Some(IndexType::Email), + SearchIndex::Calendar => Some(IndexType::Calendar), + SearchIndex::Contacts => Some(IndexType::Contacts), + SearchIndex::File => Some(IndexType::File), + #[cfg(not(feature = "enterprise"))] + SearchIndex::Tracing | SearchIndex::InMemory => None, + } +} + +fn search_index(index: IndexType) -> Option { + match index { + IndexType::Email => Some(SearchIndex::Email), + IndexType::Calendar => Some(SearchIndex::Calendar), + IndexType::Contacts => Some(SearchIndex::Contacts), + IndexType::File => Some(SearchIndex::File), + } +} diff --git a/crates/jmap/src/registry/mapping/log.rs b/crates/jmap/src/registry/mapping/log.rs index 9dc8bc95..e7196438 100644 --- a/crates/jmap/src/registry/mapping/log.rs +++ b/crates/jmap/src/registry/mapping/log.rs @@ -11,8 +11,9 @@ use crate::{ query::RegistryQueryFilters, }, }; -use chrono::DateTime; +use jiff::Timestamp; use jmap_proto::types::state::State; +use memchr::memrchr; use registry::{ jmap::IntoValue, schema::{enums::TracingLevel, prelude::Property, structs::Log}, @@ -281,19 +282,19 @@ fn is_log_header(line: &str) -> bool { let Some((timestamp, _)) = line.split_once(' ') else { return false; }; - DateTime::parse_from_rfc3339(timestamp).is_ok() + timestamp.parse::().is_ok() } fn log_from_line(line: &str) -> Option { let line = strip_ansi(line); let (timestamp, rest) = line.split_once(' ')?; - let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?; + let timestamp = timestamp.parse::().ok()?; let (level, rest) = rest.trim().split_once(' ')?; let (_, rest) = rest.trim().split_once(" (")?; let (event_id, details) = rest.split_once(")")?; Some(Log { - timestamp: UTCDateTime::from_timestamp(timestamp.timestamp()), + timestamp: UTCDateTime::from_timestamp(timestamp.as_second()), level: TracingLevel::parse(&level.to_ascii_lowercase()).unwrap_or(TracingLevel::Info), event: EventType::parse(event_id)?, details: details.trim().to_string(), @@ -451,21 +452,26 @@ impl RawRevLines { } } - let buffer_length = self.buffer_end; - - for ch in self.buffer[..self.buffer_end].iter().rev() { - self.buffer_end -= 1; - // Found a new line character to break on - if *ch == LF_BYTE { - result.push(self.buffer[self.buffer_end + 1..buffer_length].to_vec()); + let buffer = self.buffer.get(..self.buffer_end).unwrap_or_default(); + match memrchr(LF_BYTE, buffer) { + Some(pos) => { + result.push(buffer.get(pos + 1..).unwrap_or_default().to_vec()); + self.buffer_end = pos; break 'outer; } + None => { + result.push(buffer.to_vec()); + self.buffer_end = 0; + } } - - result.push(self.buffer[..buffer_length].to_vec()); } - Ok(Some(result.into_iter().rev().flatten().collect())) + result.reverse(); + Ok(Some(if result.len() == 1 { + result.swap_remove(0) + } else { + result.concat() + })) } } @@ -508,3 +514,4 @@ impl Iterator for RevLines { ) } } + diff --git a/crates/jmap/src/registry/mapping/mod.rs b/crates/jmap/src/registry/mapping/mod.rs index 3a34a28b..74c72906 100644 --- a/crates/jmap/src/registry/mapping/mod.rs +++ b/crates/jmap/src/registry/mapping/mod.rs @@ -27,7 +27,9 @@ pub mod bootstrap; pub mod cluster; pub mod dkim; pub mod domain; +pub mod index_queue; pub mod log; +pub mod oauth_client; pub mod principal; pub mod public_key; pub mod queued_message; @@ -40,14 +42,12 @@ pub mod tls; pub(crate) struct RegistryGetResponse<'x> { pub server: &'x Server, - pub access_token: &'x AccessToken, pub account_id: u32, pub ids: Option>, pub properties: AHashSet, pub response: GetResponse, pub object_type: ObjectType, pub object_flags: u64, - pub is_tenant_filtered: bool, pub is_account_filtered: bool, } @@ -61,7 +61,6 @@ pub(crate) struct RegistrySetResponse<'x> { pub destroy: Vec, pub response: SetResponse, pub object_type: ObjectType, - pub is_tenant_filtered: bool, pub is_account_filtered: bool, } diff --git a/crates/jmap/src/registry/mapping/oauth_client.rs b/crates/jmap/src/registry/mapping/oauth_client.rs new file mode 100644 index 00000000..209d9a29 --- /dev/null +++ b/crates/jmap/src/registry/mapping/oauth_client.rs @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ObjectResponse, ValidationResult}; +use common::{ + Server, + auth::{AccessToken, oauth::registration::validate_oauth_client}, +}; +use directory::core::secret::{hash_secret, is_password_hash}; +use jmap_proto::error::set::SetError; +use registry::schema::{enums::Permission, structs::OAuthClient}; +use trc::AddContext; + +pub fn is_superuser(access_token: &AccessToken) -> bool { + #[cfg(not(feature = "enterprise"))] + let is_tenant_member = false; + !is_tenant_member && access_token.has_permission(Permission::Impersonate) +} + +#[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] +pub async fn validate_oauth_client_object( + server: &Server, + access_token: &AccessToken, + client: &mut OAuthClient, +) -> ValidationResult { + if let Err((property, err)) = validate_oauth_client(client, is_superuser(access_token)) { + return Ok(Err(SetError::invalid_properties() + .with_property(property) + .with_description(err.error_description.unwrap_or(err.error)))); + } + + if let Some(secret) = client.secret.as_mut() + && !secret.is_empty() + && !(matches!(secret.as_bytes().first(), Some(&b'$' | &b'{')) && is_password_hash(secret)) + { + *secret = hash_secret( + &server.inner.data.password_verifier, + server.core.network.security.password_hash_algorithm, + std::mem::take(secret).into_bytes(), + ) + .await + .caused_by(trc::location!())?; + } + + Ok(Ok(ObjectResponse::default())) +} diff --git a/crates/jmap/src/registry/mapping/principal.rs b/crates/jmap/src/registry/mapping/principal.rs index 46829389..ded2f52c 100644 --- a/crates/jmap/src/registry/mapping/principal.rs +++ b/crates/jmap/src/registry/mapping/principal.rs @@ -14,19 +14,17 @@ use jmap_proto::error::set::SetError; use registry::{schema::structs::TaskStatus, types::datetime::UTCDateTime}; use registry::{ schema::{ - enums::{AccountType, Permission, TenantStorageQuota}, + enums::{AccountType, Permission}, prelude::{MASKED_PASSWORD, ObjectType, Property}, structs::{Account, Credential, Role, Task, TaskDestroyAccount}, }, types::EnumImpl, }; -use store::{ - registry::{RegistryObjectCounter, RegistryQuery}, - write::{BatchBuilder, RegistryClass, ValueClass, now}, -}; +use store::write::{BatchBuilder, RegistryClass, ValueClass, now}; use trc::AddContext; use types::id::Id; + #[derive(Clone, Copy)] pub enum AccountUpdate<'x> { Update(&'x Account), @@ -60,6 +58,12 @@ pub async fn validate_account( None }; + if let AccountUpdate::Update(old_account) = old_account + && let Err(err) = validate_account_access(server, access_token, old_account).await? + { + return Ok(Err(err)); + } + let validate_permissions = match (&mut account, old_account) { (Account::User(account), AccountUpdate::Update(Account::User(old_account))) => { // Validate credentials @@ -97,11 +101,11 @@ pub async fn validate_account( credential.secret = old_credential.secret.clone(); } if credential - .otp_auth + .otp_url .as_ref() - .is_some_and(|otp_auth| otp_auth == MASKED_PASSWORD) + .is_some_and(|otp_url| otp_url == MASKED_PASSWORD) { - credential.otp_auth = old_credential.otp_auth.clone(); + credential.otp_url = old_credential.otp_url.clone(); } if credential.secret != old_credential.secret { @@ -131,6 +135,7 @@ pub async fn validate_account( } credential.secret = hash_secret( + &server.inner.data.password_verifier, server.core.network.security.password_hash_algorithm, std::mem::take(&mut credential.secret).into_bytes(), ) @@ -194,13 +199,6 @@ pub async fn validate_account( account.permissions != old_account.permissions || account.roles != old_account.roles } (Account::User(account), AccountUpdate::Create(_)) => { - // Validate tenant quotas - if let Err(err) = - validate_tenant_quota(server, access_token, TenantStorageQuota::MaxAccounts).await? - { - return Ok(Err(err)); - } - // Validate credentials for (index, credential) in account.credentials.values_mut().enumerate() { if let Err(err) = validate_credential_creation( @@ -218,16 +216,7 @@ pub async fn validate_account( true } - (Account::Group(_), AccountUpdate::Create(_)) => { - // Validate tenant quotas - if let Err(err) = - validate_tenant_quota(server, access_token, TenantStorageQuota::MaxGroups).await? - { - return Ok(Err(err)); - } - - true - } + (Account::Group(_), AccountUpdate::Create(_)) => true, (Account::User(_), AccountUpdate::Update(Account::Group(_))) | (Account::Group(_), AccountUpdate::Update(Account::User(_))) => { return Ok(Err(SetError::invalid_properties() @@ -258,6 +247,17 @@ pub async fn validate_account( result } +pub async fn validate_account_access( + server: &Server, + access_token: &AccessToken, + account: &Account, +) -> trc::Result>> { + server + .can_set_permissions(access_token, account) + .await + .map(|result| result.map_err(build_set_error)) +} + async fn validate_credential_creation( server: &Server, credential: &mut Credential, @@ -293,6 +293,7 @@ async fn validate_credential_creation( .with_description(err))) } else { credential.secret = hash_secret( + &server.inner.data.password_verifier, server.core.network.security.password_hash_algorithm, std::mem::take(&mut credential.secret).into_bytes(), ) @@ -311,21 +312,12 @@ async fn validate_credential_creation( } } +#[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] pub(crate) async fn validate_role( - server: &Server, access_token: &AccessToken, role: &mut Role, old_role: Option<&Role>, ) -> ValidationResult { - if old_role.is_none() { - // Validate tenant quotas - if let Err(err) = - validate_tenant_quota(server, access_token, TenantStorageQuota::MaxRoles).await? - { - return Ok(Err(err)); - } - } - if old_role.is_none_or(|old_role| { old_role.enabled_permissions != role.enabled_permissions || old_role.disabled_permissions != role.disabled_permissions @@ -348,15 +340,6 @@ pub(crate) async fn validate_role( } -#[cfg(not(feature = "enterprise"))] -pub async fn validate_tenant_quota( - _server: &Server, - _access_token: &AccessToken, - _quota: TenantStorageQuota, -) -> ValidationResult { - ValidationResult::Ok(Ok(ObjectResponse::default())) -} - pub async fn schedule_account_destruction( server: &Server, account_id: Id, @@ -366,9 +349,19 @@ pub async fn schedule_account_destruction( #[cfg(not(feature = "enterprise"))] let status = TaskStatus::now(); - let (account_domain_id, account_name, account_type) = match account { - Account::User(account) => (account.domain_id, account.name.clone(), AccountType::User), - Account::Group(account) => (account.domain_id, account.name.clone(), AccountType::Group), + let (account_domain_id, account_name, account_type, member_group_ids) = match account { + Account::User(account) => ( + account.domain_id, + account.name.clone(), + AccountType::User, + account.member_group_ids.clone(), + ), + Account::Group(account) => ( + account.domain_id, + account.name.clone(), + AccountType::Group, + Default::default(), + ), }; let mut batch = BatchBuilder::new(); @@ -377,11 +370,11 @@ pub async fn schedule_account_destruction( account_id, account_name, account_type, + member_group_ids, status, })); - server.store().write(batch.build_all()).await?; - server.notify_task_queue(); + server.commit_batch(batch).await?; Ok(()) } @@ -424,9 +417,9 @@ async fn restore_account_id(server: &Server, id: u32) -> trc::Result<()> { ); let last_id = server .store() - .write(id_batch.build_all()) - .await - .and_then(|v| v.last_counter_id())?; + .write_batch(&mut id_batch) + .await? + .last_counter_id(); if last_id < id as i64 { return Err(trc::StoreEvent::UnexpectedError diff --git a/crates/jmap/src/registry/mapping/queued_message.rs b/crates/jmap/src/registry/mapping/queued_message.rs index 599eaf75..2359b27c 100644 --- a/crates/jmap/src/registry/mapping/queued_message.rs +++ b/crates/jmap/src/registry/mapping/queued_message.rs @@ -7,14 +7,16 @@ use crate::{ api::query::QueryResponseBuilder, registry::{ - mapping::{RegistryGetResponse, RegistryQueryResponse, RegistrySetResponse}, + mapping::{ + RegistryGetResponse, RegistryQueryResponse, RegistrySetResponse, task::query_page, + }, query::RegistryQueryFilters, }, }; use common::{ Server, config::smtp::queue::{ArchivedQueueExpiry, QueueName}, - ipc::QueueEvent, + ipc::{QueueEvent, QueueNames, QueueRefresh}, }; use jmap_proto::{error::set::SetError, object::registry::RegistryComparator, types::state::State}; use jmap_tools::{JsonPointer, JsonPointerItem, Key}; @@ -22,7 +24,7 @@ use registry::{ jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, schema::{ enums::{DeliveryErrorType, MessageFlag, RecipientFlag}, - prelude::{ObjectType, Property}, + prelude::Property, structs::{ DeliveryError, QueueExpiry, QueueExpiryAttempts, QueueExpiryTtl, QueuedMessage, QueuedRecipient, RecipientStatus, ServerResponse, @@ -34,19 +36,24 @@ use smtp::queue::{ self, ArchivedError, ArchivedErrorDetails, ArchivedMessage, ArchivedStatus, ErrorDetails, FROM_AUTHENTICATED, FROM_AUTOGENERATED, FROM_DSN, FROM_REPORT, FROM_UNAUTHENTICATED, FROM_UNAUTHENTICATED_DMARC, Message, MessageWrapper, RCPT_DSN_SENT, Schedule, Status, - rcpt_spam_percentage, spool::SmtpSpool, + rcpt_spam_percentage, + spool::SmtpSpool, + walk::{QueueWalk, WalkStep, walk_queues}, }; use std::str::FromStr; + +const LISTING_READ_LIMIT: usize = 4096; use store::{ - Deserialize, IterateParams, U64_LEN, ValueKey, + Deserialize, IterateParams, ValueKey, ahash::AHashSet, - registry::{RegistryFilterOp, RegistryQuery}, - write::{AlignedBytes, Archive, QueueClass, ValueClass, key::DeserializeBigEndian, now}, + registry::RegistryFilterOp, + write::{Archive, ArchiveBytes, QueueClass, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; use types::{blob::BlobId, blob_hash::BlobHash, id::Id}; use utils::{DomainPart, map::vec_map::VecMap}; + pub(crate) async fn queued_message_set( mut set: RegistrySetResponse<'_>, ) -> trc::Result> { @@ -54,14 +61,11 @@ pub(crate) async fn queued_message_set( set.fail_all_create("Queued messages cannot be created"); // Obtain tenant domains - let tenant_domains = if let Some(tenant_id) = set.access_token.tenant_id() { - Some(tenant_domains(set.server, tenant_id).await?) - } else { - None - }; + #[cfg(not(feature = "enterprise"))] + let tenant_domains: Option> = None; // Process update operations - let mut refresh_queue = false; + let mut refresh_queues = QueueNames::default(); 'outer: for (id, value) in set.update.drain(..) { let queue_id = id.id(); let Some(archive) = set.server.read_message_archive(queue_id).await? else { @@ -117,10 +121,10 @@ pub(crate) async fn queued_message_set( .iter() .any(|(address, _)| address.as_str() == rcpt.address.as_ref()) { - rcpt.status = Status::PermanentFailure(ErrorDetails { + rcpt.status = Status::PermanentFailure(Box::new(ErrorDetails { entity: "localhost".into(), details: queue::Error::Io("Delivery canceled.".into()), - }); + })); has_changes = true; modified_rcpts.insert(idx); } @@ -199,7 +203,13 @@ pub(crate) async fn queued_message_set( if has_changes { // Delete message if there are no pending deliveries let message = MessageWrapper::new(queued_message, queue_id, QueueName::default()); - let is_success = if message.message.recipients.iter().any(|recipient| { + let queues = message + .message + .recipients + .iter() + .map(|rcpt| rcpt.queue) + .collect::(); + let status = if message.message.recipients.iter().any(|recipient| { matches!( recipient.status, Status::TemporaryFailure(_) | Status::Scheduled @@ -209,36 +219,41 @@ pub(crate) async fn queued_message_set( .save_registry_changes(set.server, prev_events, modified_rcpts) .await } else { - message.remove_registry(set.server, prev_events).await + message + .remove_registry(set.server, prev_events, archive.version) + .await }; - if !is_success { - set.response.not_updated.append( - id, - SetError::forbidden().with_description("Queue update operation failed"), - ); + if let Some(description) = status.failure("Queue update operation failed") { + set.response + .not_updated + .append(id, SetError::forbidden().with_description(description)); continue; } - refresh_queue = true; + refresh_queues.merge(&queues); } set.response.updated.append(id, None); } - if refresh_queue { + if !refresh_queues.is_empty() { let _ = set .server .inner .ipc .queue_tx - .send(QueueEvent::Refresh) + .send(QueueEvent::Refresh(QueueRefresh::Queues(refresh_queues))) .await; } // Process destroy operations for id in set.destroy.drain(..) { - let Some(message) = set.server.read_message(id.id(), QueueName::default()).await else { + let Some((message, version)) = set + .server + .read_message_versioned(id.id(), QueueName::default()) + .await + else { set.response.not_destroyed.append(id, SetError::not_found()); continue; }; @@ -250,13 +265,17 @@ pub(crate) async fn queued_message_set( .try_domain_part() .is_some_and(|domain| domains.contains(domain)) }) { - if message.remove(set.server, None).await { - set.response.destroyed.push(id); - } else { - set.response.not_destroyed.append( - id, - SetError::forbidden().with_description("Queue delete operation failed"), - ); + let prev_events = message.message.next_events(); + match message + .remove_registry(set.server, prev_events, version) + .await + .failure("Queue delete operation failed") + { + None => set.response.destroyed.push(id), + Some(description) => set + .response + .not_destroyed + .append(id, SetError::forbidden().with_description(description)), } } else { set.response.not_destroyed.append(id, SetError::not_found()); @@ -281,11 +300,8 @@ pub(crate) async fn queued_message_get( }; // Obtain tenant domains - let tenant_domains = if let Some(tenant_id) = get.access_token.tenant_id() { - Some(tenant_domains(get.server, tenant_id).await?) - } else { - None - }; + #[cfg(not(feature = "enterprise"))] + let tenant_domains: Option> = None; for id in ids { let Some(message_archive) = get.server.read_message_archive(id.id()).await? else { @@ -320,12 +336,8 @@ pub(crate) async fn queued_message_query( let mut filter_from = None; let mut filter_to = None; - // Obtain tenant domains - let tenant_domains = if let Some(tenant_id) = req.access_token.tenant_id() { - Some(tenant_domains(req.server, tenant_id).await?) - } else { - None - }; + #[cfg(not(feature = "enterprise"))] + let tenant_id: Option = None; req.request .extract_filters(|property, op, value| match property { @@ -394,7 +406,7 @@ pub(crate) async fn queued_message_query( { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details("Only sorting by 'due' is supported for queued messages".to_string())); + .details("Only sorting by 'due' is supported for queued messages")); } let params = req @@ -402,62 +414,60 @@ pub(crate) async fn queued_message_query( .extract_parameters(req.server.core.jmap.query_max_results, None)?; let has_filters = filter_text.is_some() || filter_from.is_some() || filter_to.is_some(); - if has_filters || tenant_domains.is_some() { - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); - + if has_filters { let mut results = Vec::with_capacity(8); - req.server - .core - .storage - .data - .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let message_ = as Deserialize>::deserialize(value) - .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; - let message = message_ - .unarchive::() - .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; + let mut collect = |key: &[u8], value: &[u8]| { + let message_ = as Deserialize>::deserialize(value) + .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; + let message = message_ + .unarchive::() + .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; - if let Some(due) = message.next_delivery_event(queue_name) - && tenant_domains + if let Some(due) = message.next_delivery_event(queue_name) + && (due_from..=due_to).contains(&due) + && queue_name + .as_ref() + .is_none_or(|q| message.recipients.iter().any(|r| &r.queue == q)) + && filter_text + .as_ref() + .map(|text| { + message.return_path.contains(text) + || message + .recipients + .iter() + .any(|r| r.address().contains(text)) + }) + .unwrap_or_else(|| { + filter_from .as_ref() - .is_none_or(|domains| message.has_domain(domains)) - && (due_from..=due_to).contains(&due) - && queue_name - .as_ref() - .is_none_or(|q| message.recipients.iter().any(|r| &r.queue == q)) - && (!has_filters - || (filter_text - .as_ref() - .map(|text| { - message.return_path.contains(text) - || message - .recipients - .iter() - .any(|r| r.address().contains(text)) - }) - .unwrap_or_else(|| { - filter_from - .as_ref() - .is_none_or(|from| message.return_path.contains(from)) - && filter_to.as_ref().is_none_or(|to| { - message - .recipients - .iter() - .any(|r| r.address().contains(to)) - }) - }))) - { - results.push((key.deserialize_be_u64(0)?, due)); - } + .is_none_or(|from| message.return_path.contains(from)) + && filter_to.as_ref().is_none_or(|to| { + message.recipients.iter().any(|r| r.address().contains(to)) + }) + }) + { + results.push((key.deserialize_be_u64(0)?, due)); + } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; + Ok(true) + }; + + match tenant_id { + _ => { + req.server + .store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), + ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), + ) + .ascending(), + &mut collect, + ) + .await + .caused_by(trc::location!())?; + } + } // Build response let mut response = QueryResponseBuilder::new( @@ -494,7 +504,7 @@ pub(crate) async fn queued_message_query( let anchor_id = anchor.id(); if let Some(archive) = req.server.read_message_archive(anchor_id).await? && let Ok(archived) = archive.unarchive::() - && let Some(anchor_due) = archived.next_delivery_event(queue_name) + && let Some(anchor_due) = archived.next_event(queue_name) && anchor_due >= due_from && anchor_due <= due_to { @@ -506,50 +516,90 @@ pub(crate) async fn queued_message_query( } } - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: due_from, - queue_id: 0, - queue_name: [0; 8], - }, - ))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: due_to, - queue_id: u64::MAX, - queue_name: [u8::MAX; 8], - }, - ))); - - let mut seen_ids = AHashSet::with_capacity(8); - req.server - .store() - .iterate( - IterateParams::new(from_key, to_key) + let page = req + .request + .anchor + .is_none() + .then(|| query_page(&req.request, &response)) + .flatten(); + let count_total = response.response.total.is_some(); + let mut entries = Vec::new(); + match tenant_id { + _ => { + if let Some(queue_name) = queue_name { + let mut iterate = IterateParams::new( + queue_event_key(queue_name, due_from, 0), + queue_event_key(queue_name, due_to, u64::MAX), + ) .set_ascending(params.sort_ascending) - .no_values(), - |key, _| { - let id = key.deserialize_be_u64(U64_LEN)?; - if queue_name.is_none_or(|queue_name| { - queue_name.as_slice() == key.get(U64_LEN * 2..).unwrap_or_default() - }) && seen_ids.insert(id) + .no_values(); + if let Some(page) = page + && !count_total { - total += 1; - if response.response.total.is_some() { - if !response.is_full() { - response.add_id(id.into()); - } - Ok(true) - } else { - Ok(response.add_id(id.into())) - } - } else { - Ok(true) + iterate = iterate.limit(page); } - }, - ) - .await - .caused_by(trc::location!())?; + + req.server + .store() + .iterate(iterate, |key, _| { + let event = store::write::QueueEvent::deserialize(key)?; + entries.push((event.due, event.queue_id)); + Ok(count_total || page.is_none_or(|page| entries.len() < page)) + }) + .await + .caused_by(trc::location!())?; + } else { + let read_limit = match page { + Some(page) if !count_total => page.saturating_add(1), + _ => LISTING_READ_LIMIT, + }; + let mut current = None; + let mut taken = 0; + walk_queues( + req.server.store(), + QueueWalk { + due_from, + due_to, + ascending: params.sort_ascending, + read_limit, + }, + |event| { + if current != Some(event.queue_name) { + current = Some(event.queue_name); + taken = 0; + } + entries.push((event.due, event.queue_id)); + taken += 1; + if count_total || page.is_none_or(|page| taken < page) { + WalkStep::Continue + } else { + WalkStep::NextQueue + } + }, + ) + .await?; + } + } + } + + if params.sort_ascending { + entries.sort_unstable(); + } else { + entries.sort_unstable_by(|a, b| b.cmp(a)); + } + let mut seen_ids = AHashSet::with_capacity(entries.len()); + for (_, id) in entries { + if seen_ids.insert(id) { + total += 1; + if count_total { + if !response.is_full() { + response.add_id(id.into()); + } + } else if !response.add_id(id.into()) { + break; + } + } + } if response.response.total.is_some() { response.response.total = Some(total); @@ -565,12 +615,17 @@ pub(crate) async fn queued_message_query( } } - -#[cfg(not(feature = "enterprise"))] -async fn tenant_domains(_server: &Server, _tenant_id: u32) -> trc::Result> { - Ok(AHashSet::new()) +fn queue_event_key(queue_name: QueueName, due: u64, queue_id: u64) -> ValueKey { + ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due, + queue_id, + queue_name: queue_name.into_inner(), + }, + ))) } + fn map_message(message_in: &ArchivedMessage) -> QueuedMessage { let mut message_out = QueuedMessage { blob_id: BlobId::new(BlobHash::from(&message_in.blob_hash), Default::default()), @@ -658,10 +713,8 @@ fn map_message(message_in: &ArchivedMessage) -> QueuedMessage { // Parse recipient flags let rcpt_flags = rcpt_in.flags.to_native(); - for (bit, flag) in [(RCPT_DSN_SENT, RecipientFlag::DsnSent)] { - if rcpt_flags & bit != 0 { - rcpt_out.flags.push(flag); - } + if rcpt_flags & RCPT_DSN_SENT != 0 { + rcpt_out.flags.push(RecipientFlag::DsnSent); } if rcpt_spam_percentage(rcpt_flags).is_some_and(|percentage| percentage >= 50) { rcpt_out.flags.push(RecipientFlag::SpamPayload); @@ -730,33 +783,22 @@ fn build_enhanced_code(esc: &[u8; 3]) -> String { async fn queued_ids(server: &Server, max_results: usize) -> trc::Result> { let mut events = AHashSet::with_capacity(8); + if max_results == 0 { + return Ok(events); + } - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: 0, - queue_id: 0, - queue_name: [0; 8], + walk_queues( + server.store(), + QueueWalk::ascending(max_results.saturating_add(1)), + |event| { + events.insert(event.queue_id); + if events.len() < max_results { + WalkStep::Continue + } else { + WalkStep::Stop + } }, - ))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: u64::MAX, - queue_id: u64::MAX, - queue_name: [u8::MAX; 8], - }, - ))); - - server - .store() - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - events.insert(key.deserialize_be_u64(U64_LEN)?); - - Ok(events.len() < max_results) - }, - ) - .await - .caused_by(trc::location!()) - .map(|_| events) + ) + .await + .map(|_| events) } diff --git a/crates/jmap/src/registry/mapping/report.rs b/crates/jmap/src/registry/mapping/report.rs index da01a22a..0e2948f6 100644 --- a/crates/jmap/src/registry/mapping/report.rs +++ b/crates/jmap/src/registry/mapping/report.rs @@ -8,9 +8,10 @@ use crate::{ api::query::QueryResponseBuilder, registry::{ mapping::{RegistryGetResponse, RegistryQueryResponse, RegistrySetResponse}, - query::RegistryQueryFilters, + query::{RegistryQueryFilters, with_page_position}, }, }; +use compact_str::format_compact; use jmap_proto::{error::set::SetError, types::state::State}; use jmap_tools::{Key, Value}; use registry::{ @@ -18,12 +19,15 @@ use registry::{ schema::prelude::{Object, ObjectInner, ObjectType, Property}, types::{EnumImpl, datetime::UTCDateTime}, }; -use smtp::reporting::index::{ExternalReportIndex, InternalReportIndex}; +use smtp::reporting::{ + aggregate::{ReportItem, read_dmarc_records, read_tls_policies}, + index::{ExternalReportIndex, InternalReportIndex}, +}; use std::str::FromStr; use store::{ U64_LEN, ValueKey, registry::{RegistryFilter, RegistryFilterValue, RegistryQuery}, - write::{BatchBuilder, RegistryClass, ValueClass, key::KeySerializer}, + write::{BatchBuilder, RegistryClass, ValueClass, assert::AssertValue, key::KeySerializer}, }; use trc::AddContext; use types::id::Id; @@ -111,19 +115,16 @@ pub(crate) async fn report_set( } // Process reports to destroy - let tenant_id = set.access_token.tenant_id().map(Id::from); + let mut closed = Vec::new(); for id in set.destroy.drain(..) { let item_id = id.id(); let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id }); - if let Some(report) = set + let report = set .server .store() - .get_value::(ValueKey::from(key)) - .await? - .filter(|report| { - !set.is_tenant_filtered || report.inner.member_tenant_id() == tenant_id - }) - { + .get_value::(ValueKey::from(key.clone())) + .await?; + if let Some(report) = report { match &report.inner { ObjectInner::DmarcExternalReport(report) => { report.write_ops(&mut batch, item_id, false); @@ -134,11 +135,19 @@ pub(crate) async fn report_set( ObjectInner::ArfExternalReport(report) => { report.write_ops(&mut batch, item_id, false); } - ObjectInner::DmarcInternalReport(report) => { - report.write_ops(&mut batch, item_id, false); + ObjectInner::DmarcInternalReport(inner) => { + let item = ReportItem::new(ObjectType::DmarcInternalReport, item_id); + batch.assert_value(key, AssertValue::Hash(report.revision)); + inner.write_ops(&mut batch, item_id, false); + item.clear(&mut batch); + closed.push(item); } - ObjectInner::TlsInternalReport(report) => { - report.write_ops(&mut batch, item_id, false); + ObjectInner::TlsInternalReport(inner) => { + let item = ReportItem::new(ObjectType::TlsInternalReport, item_id); + batch.assert_value(key, AssertValue::Hash(report.revision)); + inner.write_ops(&mut batch, item_id, false); + item.clear(&mut batch); + closed.push(item); } _ => {} } @@ -152,11 +161,15 @@ pub(crate) async fn report_set( if !batch.is_empty() { set.server - .store() - .write(batch.build_all()) + .commit_batch(batch) .await .caused_by(trc::location!())?; } + for item in closed { + if let Err(err) = item.clear_orphans(set.server.store()).await { + trc::error!(err.details("Failed to clear orphaned report rows")); + } + } Ok(set) } @@ -165,6 +178,9 @@ pub(crate) async fn report_get( mut get: RegistryGetResponse<'_>, ) -> trc::Result> { let object_id = get.object_type.to_id(); + let external_query = RegistryQuery::new(get.object_type); + #[cfg(not(feature = "enterprise"))] + let external_query = external_query.greater_than(Property::ExpiresAt, 0u64); let ids = if let Some(ids) = get.ids.take() { ids } else if matches!( @@ -173,20 +189,10 @@ pub(crate) async fn report_get( | ObjectType::TlsExternalReport | ObjectType::ArfExternalReport ) { - if get.is_tenant_filtered { - get.server.registry().query::>( - RegistryQuery::new(get.object_type) - .with_tenant(get.access_token.tenant_id()) - .with_limit(get.server.core.jmap.get_max_objects), - ) - } else { - get.server.registry().query::>( - RegistryQuery::new(get.object_type) - .greater_than(Property::ExpiresAt, 0u64) - .with_limit(get.server.core.jmap.get_max_objects), - ) - } - .await? + get.server + .registry() + .query::>(external_query.with_limit(get.server.core.jmap.get_max_objects)) + .await? } else { get.server .registry() @@ -202,20 +208,25 @@ pub(crate) async fn report_get( .await? }; - let tenant_id = get.access_token.tenant_id().map(Id::from); for id in ids { - if let Some(report) = get + let report = get .server .store() .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { object_id, item_id: id.id(), }))) - .await? - .filter(|report| { - !get.is_tenant_filtered || report.inner.member_tenant_id() == tenant_id - }) - { + .await?; + if let Some(mut report) = report { + match &mut report.inner { + ObjectInner::DmarcInternalReport(report) => { + read_dmarc_records(get.server.store(), id.id(), report, usize::MAX).await?; + } + ObjectInner::TlsInternalReport(report) => { + read_tls_policies(get.server.store(), id.id(), report, usize::MAX).await?; + } + _ => {} + } get.insert(id, report.into_value()); } else { get.not_found(id); @@ -228,8 +239,7 @@ pub(crate) async fn report_get( pub(crate) async fn report_query( mut req: RegistryQueryResponse<'_>, ) -> trc::Result { - let mut query = store::registry::RegistryQuery::new(req.object_type) - .with_tenant(req.access_token.tenant_id()); + let mut query = store::registry::RegistryQuery::new(req.object_type); let is_internal = matches!( req.object_type, ObjectType::DmarcInternalReport | ObjectType::TlsInternalReport @@ -237,41 +247,25 @@ pub(crate) async fn report_query( req.request .extract_filters(|property, op, value| match property { - Property::Domain => { + Property::Domain if is_internal => { if let serde_json::Value::String(value) = value { - match req.object_type { - ObjectType::DmarcInternalReport => { - query.filters.push(RegistryFilter::greater_than_or_equal( - property, - RegistryFilterValue::Bytes( - KeySerializer::new(value.len() + U64_LEN) - .write(value.as_str()) - .write(0u64) - .finalize(), - ), - true, - )); - query.filters.push(RegistryFilter::less_than_or_equal( - property, - RegistryFilterValue::Bytes( - KeySerializer::new(value.len() + U64_LEN) - .write(value.as_str()) - .write(u64::MAX) - .finalize(), - ), - true, - )); - - true - } - ObjectType::TlsInternalReport => { - query - .filters - .push(RegistryFilter::equal(property, value, true)); - true - } - _ => false, - } + let from = KeySerializer::new(value.len() + 1 + (U64_LEN * 2)) + .write(value.as_str()) + .write(0u8) + .finalize(); + let mut to = from.clone(); + to.extend_from_slice(&[u8::MAX; U64_LEN * 2]); + query.filters.push(RegistryFilter::greater_than_or_equal( + property, + RegistryFilterValue::Bytes(from), + true, + )); + query.filters.push(RegistryFilter::less_than_or_equal( + property, + RegistryFilterValue::Bytes(to), + true, + )); + true } else { false } @@ -284,18 +278,6 @@ pub(crate) async fn report_query( false } } - Property::MemberTenantId if !is_internal => { - if req.access_token.tenant_id().is_none() - && let Some(id) = value.as_str().and_then(|s| Id::from_str(s).ok()) - { - query - .filters - .push(RegistryFilter::equal(property, id.id(), false)); - true - } else { - false - } - } Property::TotalFailedSessions | Property::TotalSuccessfulSessions if !is_internal => { if let Some(value) = value.as_u64() { query.filters.push(store::registry::RegistryFilter { @@ -347,20 +329,17 @@ pub(crate) async fn report_query( )); } } - if let Some(limit) = params.limit { - query = query.with_limit(limit); - if let Some(anchor) = params.anchor { - query = query.with_anchor(anchor); - } else if let Some(position) = params.position { - query = query.with_index_start(position); - } - } - - let matches = req.server.registry().query::>(query).await?; + let page = req + .server + .registry() + .query_page::>(params.paged(query)) + .await?; + let position = params.page_position(page.position)?; + let matches = page.ids; let results = match params.sort_by { Property::Id => { let mut results = matches; - if !params.sort_ascending { + if params.needs_reversal() { results.sort_unstable_by(|a, b| b.cmp(a)); } results @@ -396,19 +375,24 @@ pub(crate) async fn report_query( } } property => { - return Err(trc::JmapEvent::UnsupportedSort.into_err().details(format!( - "Property {} is not supported for sorting", - property - ))); + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(format_compact!( + "Property {} is not supported for sorting", + property + ))); } }; // Build response - let mut response = QueryResponseBuilder::new( - results.len(), - req.server.core.jmap.query_max_results, - State::Initial, - &req.request, + let mut response = with_page_position( + QueryResponseBuilder::new( + results.len(), + req.server.core.jmap.query_max_results, + State::Initial, + &req.request, + ), + position, ); for id in results { diff --git a/crates/jmap/src/registry/mapping/spam_sample.rs b/crates/jmap/src/registry/mapping/spam_sample.rs index f78df665..a9d5cbc1 100644 --- a/crates/jmap/src/registry/mapping/spam_sample.rs +++ b/crates/jmap/src/registry/mapping/spam_sample.rs @@ -9,12 +9,13 @@ use crate::{ blob::download::BlobDownload, registry::{ mapping::{RegistryGetResponse, RegistryQueryResponse, RegistrySetResponse}, - query::RegistryQueryFilters, + query::{RegistryQueryFilters, with_page_position}, }, }; +use compact_str::format_compact; use jmap_proto::{error::set::SetError, types::state::State}; use jmap_tools::JsonPointer; -use mail_parser::{MessageParser, parsers::fields::thread::thread_name}; +use mail_parser::{MessageParser, thread_name}; use registry::{ jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, schema::{ @@ -28,7 +29,7 @@ use std::str::FromStr; use store::{ SerializeInfallible, ValueKey, registry::RegistryQuery, - write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, now}, + write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, blob::BlobLinkValue, now}, }; use trc::AddContext; use types::{blob::BlobClass, id::Id}; @@ -81,9 +82,21 @@ pub(crate) async fn spam_sample_set( continue; } - let Some(bytes) = set + if sample.blob_id.section.is_some() + || matches!(sample.blob_id.class, BlobClass::Embedded { .. }) + { + set.response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::BlobId) + .with_description("blobId must refer to a stored email message"), + ); + continue; + } + + let Some(blob) = set .server - .blob_download(&sample.blob_id, set.access_token) + .blob_download_with_extra(&sample.blob_id, set.access_token) .await? else { set.response.not_created.append( @@ -95,7 +108,7 @@ pub(crate) async fn spam_sample_set( continue; }; - if bytes.len() > set.server.core.email.mail_max_size { + if blob.bytes.len() > set.server.core.email.mail_max_size { set.response.not_created.append( id, SetError::invalid_properties() @@ -108,7 +121,7 @@ pub(crate) async fn spam_sample_set( continue; } - let Some(message) = MessageParser::new().parse(&bytes) else { + let Some(message) = MessageParser::new().parse(&blob.bytes) else { set.response.not_created.append( id, SetError::invalid_properties() @@ -142,24 +155,29 @@ pub(crate) async fn spam_sample_set( // Write sample to store let item_id = set.server.registry().assign_id(); + let account_id = sample + .account_id + .map(|id| id.document_id()) + .unwrap_or(u32::MAX); + batch.with_account_id(account_id); + let blob_size = blob.bytes.len().saturating_sub(blob.extra_headers_len); batch .set( BlobOp::Link { hash: sample.blob_id.hash.clone(), to: BlobLink::Temporary { until: expires_at }, }, - ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), + BlobLinkValue::new(blob_size).with_object_id(ObjectId::new( + ObjectType::SpamTrainingSample, + item_id.into(), + )), ) .set( ValueClass::Registry(RegistryClass::Index { index_id: Property::AccountId.to_id(), object_id, item_id, - key: sample - .account_id - .map(|id| id.id()) - .unwrap_or(u32::MAX as u64) - .serialize(), + key: (account_id as u64).serialize(), }), vec![], ) @@ -225,8 +243,7 @@ pub(crate) async fn spam_sample_set( if !batch.is_empty() { set.server - .store() - .write(batch.build_all()) + .commit_batch(batch) .await .caused_by(trc::location!())?; } @@ -305,7 +322,7 @@ pub(crate) async fn spam_sample_query( _ => false, })?; - let mut query = if let Some(account_id) = account_id { + let query = if let Some(account_id) = account_id { RegistryQuery::new(req.object_type).with_account(account_id.document_id()) } else if !can_impersonate { RegistryQuery::new(req.object_type).with_account(req.request.account_id.document_id()) @@ -317,37 +334,39 @@ pub(crate) async fn spam_sample_query( .request .extract_parameters(req.server.core.jmap.query_max_results, Some(Property::Id))?; - if let Some(limit) = params.limit { - query = query.with_limit(limit); - if let Some(anchor) = params.anchor { - query = query.with_anchor(anchor); - } else if let Some(position) = params.position { - query = query.with_index_start(position); - } - } - - let mut results = req.server.registry().query::>(query).await?; + let page = req + .server + .registry() + .query_page::>(params.paged(query)) + .await?; + let position = params.page_position(page.position)?; + let mut results = page.ids; match params.sort_by { Property::Id => { - if !params.sort_ascending { + if params.needs_reversal() { results.sort_unstable_by(|a, b| b.cmp(a)); } } property => { - return Err(trc::JmapEvent::UnsupportedSort.into_err().details(format!( - "Property {} is not supported for sorting", - property - ))); + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(format_compact!( + "Property {} is not supported for sorting", + property + ))); } } // Build response - let mut response = QueryResponseBuilder::new( - results.len(), - req.server.core.jmap.query_max_results, - State::Initial, - &req.request, + let mut response = with_page_position( + QueryResponseBuilder::new( + results.len(), + req.server.core.jmap.query_max_results, + State::Initial, + &req.request, + ), + position, ); for id in results { diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs index 9669c6c1..d4484863 100644 --- a/crates/jmap/src/registry/mapping/task.rs +++ b/crates/jmap/src/registry/mapping/task.rs @@ -14,7 +14,8 @@ use crate::{ use common::Server; use jmap_proto::{ error::set::{SetError, SetErrorType}, - object::registry::RegistryComparator, + method::query::QueryRequest, + object::{JmapObject, registry::RegistryComparator}, types::state::State, }; use jmap_tools::{JsonPointer, JsonPointerItem, Key}; @@ -23,7 +24,7 @@ use registry::{ schema::{ enums::{TaskStatusType, TaskType}, prelude::Property, - structs::Task, + structs::{CertificateManagement, DkimManagement, Domain, Task}, }, types::{ EnumImpl, ObjectImpl, @@ -31,12 +32,13 @@ use registry::{ index::{IndexBuilder, IndexKey}, }, }; -use services::task_manager::lock::TaskLockManager; -use std::str::FromStr; +use services::task_manager::{lock::TaskLockManager, reconcile::schedule_replacing}; +use std::{cmp::Ordering, str::FromStr}; use store::{ - IterateParams, SerializeInfallible, U64_LEN, ValueKey, + IterateParams, ValueKey, + ahash::AHashSet, registry::RegistryFilterOp, - write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian}, + write::{BatchBuilder, RegistryClass, TaskDue, TaskId, TaskQueueClass, ValueClass}, }; use trc::AddContext; use types::id::Id; @@ -82,67 +84,75 @@ pub(crate) async fn task_set( } let task_type = task.object_type(); - match task_type { - TaskType::IndexDocument - | TaskType::UnindexDocument - | TaskType::IndexTrace - | TaskType::AccountMaintenance - | TaskType::TenantMaintenance + let is_client_task = match task_type { + TaskType::AccountMaintenance | TaskType::StoreMaintenance | TaskType::SpamFilterMaintenance | TaskType::AcmeRenewal | TaskType::DkimManagement - | TaskType::DnsManagement => { - let mut index = IndexBuilder::default(); - task.index(&mut index); - - // Validate foreign keys - for key in index.keys { - if let IndexKey::ForeignKey { - object_id: foreign_id, - .. - } = key - && !set - .server - .store() - .key_exists(ValueKey::from(ValueClass::Registry( - RegistryClass::IndexId { - object_id: foreign_id.object().to_id(), - item_id: foreign_id.id().id(), - }, - ))) - .await - .caused_by(trc::location!())? - { - set.response.not_created.append( - id, - SetError::new(SetErrorType::InvalidForeignKey) - .with_object_id(foreign_id), - ); - continue 'outer; - } - } - - let task_id = set.server.registry().assign_id(); - batch.schedule_task_with_id(task_id, task).commit_point(); - set.response.created(id, task_id); - } + | TaskType::DnsManagement => true, TaskType::CalendarAlarmEmail | TaskType::CalendarAlarmNotification | TaskType::CalendarItipMessage | TaskType::MergeThreads | TaskType::DmarcReport | TaskType::TlsReport + | TaskType::DmarcReportWindow + | TaskType::TlsReportWindow | TaskType::DestroyAccount - | TaskType::RestoreArchivedItem => { - set.response.not_created.append( - id, - SetError::forbidden().with_description(format!( - "{} is an internal task type that cannot be created by clients", - task_type.as_str() - )), - ); + | TaskType::ShareRevokeCleanup => false, + }; + if is_client_task { + let mut index = IndexBuilder::default(); + task.index(&mut index); + + // Validate foreign keys + for key in index.keys { + if let IndexKey::ForeignKey { + object_id: foreign_id, + .. + } = key + && !set + .server + .store() + .key_exists(ValueKey::from(ValueClass::Registry( + RegistryClass::IndexId { + object_id: foreign_id.object().to_id(), + item_id: foreign_id.id().id(), + }, + ))) + .await + .caused_by(trc::location!())? + { + set.response.not_created.append( + id, + SetError::new(SetErrorType::InvalidForeignKey).with_object_id(foreign_id), + ); + continue 'outer; + } } + + let task_id = match TaskId::singleton_of(&task) { + Some(task_id) => { + schedule_replacing(set.server, &mut batch, task).await?; + batch.commit_point(); + task_id + } + None => { + let task_id = set.server.registry().assign_id(); + batch.schedule_task_with_id(task_id, task).commit_point(); + task_id + } + }; + set.response.created(id, task_id); + } else { + set.response.not_created.append( + id, + SetError::forbidden().with_description(format!( + "{} is an internal task type that cannot be created by clients", + task_type.as_str() + )), + ); } } @@ -153,7 +163,9 @@ pub(crate) async fn task_set( .server .store() .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: task_id }, + TaskQueueClass::Task { + id: TaskId::Assigned(task_id), + }, ))) .await? else { @@ -172,7 +184,7 @@ pub(crate) async fn task_set( continue 'outer; } - if !set.server.try_lock_task(task_id).await { + let Some(lock) = set.server.lock_task(task_id).await else { set.response.not_updated.append( id, SetError::forbidden().with_description( @@ -180,8 +192,8 @@ pub(crate) async fn task_set( ), ); continue; - } - locked_tasks.push(task_id); + }; + locked_tasks.push(lock); let old_timestamp = task.due_timestamp(); let old_status = task.status().clone(); @@ -206,25 +218,26 @@ pub(crate) async fn task_set( } if task.status() != &old_status { - let timestamp = task.due_timestamp(); - if timestamp != old_timestamp { - batch - .clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id: task_id, - due: old_timestamp, - })) - .set( - ValueClass::TaskQueue(TaskQueueClass::Due { - id: task_id, - due: timestamp, - }), - task.object_type().to_id().serialize(), - ); - } + let id = TaskId::Assigned(task_id); + let typ = task.object_type(); + batch + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + typ, + due: old_timestamp, + })) + .set_task_due_marked( + id, + typ, + task.due_timestamp(), + TaskQueueClass::due_marker(task.status()), + ); batch .set( - ValueClass::TaskQueue(TaskQueueClass::Task { id: task_id }), + ValueClass::TaskQueue(TaskQueueClass::Task { + id: TaskId::Assigned(task_id), + }), task.to_pickled_vec(), ) .commit_point(); @@ -240,7 +253,9 @@ pub(crate) async fn task_set( .server .store() .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: task_id }, + TaskQueueClass::Task { + id: TaskId::Assigned(task_id), + }, ))) .await? else { @@ -259,9 +274,28 @@ pub(crate) async fn task_set( continue; } - locked_tasks.push(task_id); + if let Some(reason) = managed_domain_task(set.server, &task).await? { + set.response + .not_destroyed + .append(id, SetError::forbidden().with_description(reason)); + continue; + } + + if matches!(task, Task::DestroyAccount(_)) { + let Some(lock) = set.server.lock_task(task_id).await else { + set.response.not_destroyed.append( + id, + SetError::forbidden().with_description( + "Task is currently being processed and cannot be destroyed".to_string(), + ), + ); + continue; + }; + locked_tasks.push(lock); + } let due = task.due_timestamp(); + let typ = task.object_type(); #[cfg(not(feature = "enterprise"))] @@ -276,9 +310,12 @@ pub(crate) async fn task_set( } batch - .clear(ValueClass::TaskQueue(TaskQueueClass::Task { id: task_id })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Task { + id: TaskId::Assigned(task_id), + })) .clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id: task_id, + id: TaskId::Assigned(task_id), + typ, due, })) .commit_point(); @@ -286,26 +323,57 @@ pub(crate) async fn task_set( set.response.destroyed.push(id); } - let has_changes = !batch.is_empty(); - if has_changes { + if !batch.is_empty() { set.server - .store() - .write(batch.build_all()) + .commit_batch(batch) .await .caused_by(trc::location!())?; } - for task_id in locked_tasks { - set.server.remove_index_lock(task_id).await; - } - - if has_changes { - set.server.notify_task_queue(); + for lock in locked_tasks { + set.server.unlock_task(lock).await; } Ok(set) } +async fn managed_domain_task(server: &Server, task: &Task) -> trc::Result> { + match task { + Task::AcmeRenewal(task) => server + .registry() + .object::(task.domain_id) + .await + .caused_by(trc::location!()) + .map(|domain| { + domain + .is_some_and(|domain| { + matches!( + domain.certificate_management, + CertificateManagement::Automatic(_) + ) + }) + .then_some( + "The domain uses automatic certificate management; switch it to manual certificate management before destroying its ACME renewal task", + ) + }), + Task::DkimManagement(task) => server + .registry() + .object::(task.domain_id) + .await + .caused_by(trc::location!()) + .map(|domain| { + domain + .is_some_and(|domain| { + matches!(domain.dkim_management, DkimManagement::Automatic(_)) + }) + .then_some( + "The domain uses automatic DKIM management; switch it to manual DKIM management before destroying its DKIM management task", + ) + }), + _ => Ok(None), + } +} + pub(crate) async fn task_get( mut get: RegistryGetResponse<'_>, ) -> trc::Result> { @@ -321,7 +389,9 @@ pub(crate) async fn task_get( .server .store() .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: id.id() }, + TaskQueueClass::Task { + id: TaskId::Assigned(id.id()), + }, ))) .await? { @@ -346,9 +416,10 @@ pub(crate) async fn task_get( pub(crate) async fn task_query( mut req: RegistryQueryResponse<'_>, ) -> trc::Result { - let mut due_from = 1u64; + let mut due_from = 0u64; let mut due_to = u64::MAX; let mut typ = None; + let mut status_marker = None; req.request .extract_filters(|property, op, value| match property { @@ -375,11 +446,19 @@ pub(crate) async fn task_query( } Property::Status => { if let Some(typ) = value.as_str().and_then(TaskStatusType::parse) { - if typ == TaskStatusType::Failed { - due_from = u64::MAX; - due_to = u64::MAX; - } - true + let marker = match typ { + TaskStatusType::Failed => { + due_from = u64::MAX; + due_to = u64::MAX; + return true; + } + TaskStatusType::Pending => None, + TaskStatusType::Retry => Some(TaskQueueClass::DUE_RETRY), + TaskStatusType::Stalled => Some(TaskQueueClass::DUE_STALLED), + }; + due_to = due_to.min(u64::MAX - 1); + status_marker = Some(marker); + due_from <= due_to } else { false } @@ -405,7 +484,7 @@ pub(crate) async fn task_query( { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details("Only sorting by 'due' is supported for tasks".to_string())); + .details("Only sorting by 'due' is supported for tasks")); } let params = req @@ -419,7 +498,9 @@ pub(crate) async fn task_query( .server .store() .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: anchor_id }, + TaskQueueClass::Task { + id: TaskId::Assigned(anchor_id), + }, ))) .await .caused_by(trc::location!())? @@ -444,51 +525,73 @@ pub(crate) async fn task_query( &req.request, ); + let page = query_page(&req.request, &response); + let count_total = response.response.total.is_some(); + let (first_type, type_count) = match typ { + Some(typ) => (typ.to_id(), 1), + None => (0, TaskType::COUNT as u16), + }; let mut total = 0; - let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { - id: from_id, - due: due_from, - })); - let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { - id: to_id, - due: due_to, - })); + let mut entries = Vec::new(); + let mut seen = AHashSet::new(); + for typ in (first_type..first_type + type_count).filter_map(TaskType::from_id) { + let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(from_id), + typ, + due: due_from, + })); + let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(to_id), + typ, + due: due_to, + })); + let mut iterate = IterateParams::new(from_key, to_key).set_ascending(params.sort_ascending); + if status_marker.is_none() { + iterate = iterate.no_values(); + if let Some(page) = page + && !count_total + { + iterate = iterate.limit(page); + } + } - req.server - .store() - .iterate( - IterateParams::new(from_key, to_key) - .set_ascending(params.sort_ascending) - .set_values(typ.is_some()), - |key, value| { - if let Some(typ) = typ { - let task_type = - TaskType::from_id(value.deserialize_be_u16(0)?).ok_or_else(|| { - trc::StoreEvent::DataCorruption - .into_err() - .ctx(trc::Key::Key, key.to_vec()) - .ctx(trc::Key::Value, value.to_vec()) - .caused_by(trc::location!()) - })?; - if task_type != typ { - return Ok(true); - } + let mut taken = 0; + req.server + .store() + .iterate(iterate, |key, value| { + if status_marker + .is_some_and(|marker| marker != TaskQueueClass::due_marker_of(value)) + { + return Ok(true); + } + let TaskDue { due, id, .. } = TaskDue::deserialize(key)?; + if !seen.insert(id) { + return Ok(true); } - - let id = key.deserialize_be_u64(U64_LEN)?; total += 1; - if response.response.total.is_some() { - if !response.is_full() { - response.add_id(id.into()); - } - Ok(true) - } else { - Ok(response.add_id(id.into())) + if page.is_none_or(|page| taken < page) { + entries.push((due, id)); + taken += 1; } - }, - ) - .await - .caused_by(trc::location!())?; + Ok(count_total || page.is_none_or(|page| taken < page)) + }) + .await + .caused_by(trc::location!())?; + + if let Some(page) = page + && entries.len() > page + { + entries.select_nth_unstable_by(page, |a, b| due_order(a, b, params.sort_ascending)); + entries.truncate(page); + } + } + + entries.sort_unstable_by(|a, b| due_order(a, b, params.sort_ascending)); + for (_, id) in entries { + if !response.add_id(id.into()) { + break; + } + } if response.response.total.is_some() { response.response.total = Some(total); @@ -505,23 +608,63 @@ pub(crate) async fn task_query( async fn task_ids(server: &Server, max_results: usize) -> trc::Result> { let mut tasks = Vec::with_capacity(8); - let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 1 })); - let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { - id: u64::MAX, - due: u64::MAX, - })); + let mut seen = AHashSet::new(); - server - .store() - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - tasks.push(key.deserialize_be_u64(U64_LEN)?.into()); + for typ in (0..TaskType::COUNT as u16).filter_map(TaskType::from_id) { + if tasks.len() >= max_results { + break; + } + let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(0), + typ, + due: 0, + })); + let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(u64::MAX), + typ, + due: u64::MAX, + })); - Ok(tasks.len() < max_results) - }, - ) - .await - .caused_by(trc::location!()) - .map(|_| tasks) + server + .store() + .iterate( + IterateParams::new(from_key, to_key) + .ascending() + .no_values() + .limit(max_results - tasks.len()), + |key, _| { + let id = TaskDue::deserialize(key)?.id; + if seen.insert(id) { + tasks.push(id.into()); + } + Ok(tasks.len() < max_results) + }, + ) + .await + .caused_by(trc::location!())?; + } + + Ok(tasks) +} + +fn due_order(a: &(u64, u64), b: &(u64, u64), ascending: bool) -> Ordering { + if ascending { a.cmp(b) } else { b.cmp(a) } +} + +pub(super) fn query_page( + request: &QueryRequest, + response: &QueryResponseBuilder, +) -> Option { + let limit = response.limit; + if limit == 0 { + None + } else if request.anchor.is_some() { + usize::try_from(request.anchor_offset.unwrap_or(0).max(0)) + .ok() + .map(|offset| offset + limit + 1) + } else { + usize::try_from(request.position.unwrap_or(0)) + .ok() + .map(|position| position + limit) + } } diff --git a/crates/jmap/src/registry/mapping/tls.rs b/crates/jmap/src/registry/mapping/tls.rs index 632845fe..f28888fa 100644 --- a/crates/jmap/src/registry/mapping/tls.rs +++ b/crates/jmap/src/registry/mapping/tls.rs @@ -4,9 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::{ - ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, -}; +use crate::registry::mapping::{ObjectResponse, ValidationResult}; use common::network::acme::{ ParsedCert, account::{EabSettings, acme_create_account}, @@ -15,31 +13,23 @@ use jmap_proto::error::set::SetError; use registry::{ jmap::JmapValue, schema::{ - enums::TenantStorageQuota, prelude::Property, structs::{AcmeProvider, Certificate}, }, - types::{datetime::UTCDateTime, map::Map}, + types::map::Map, }; -use utils::map::vec_map::VecMap; +use utils::{map::vec_map::VecMap, outbound::OutboundPolicy}; pub(crate) async fn validate_acme_provider( - set: &RegistrySetResponse<'_>, provider: &mut AcmeProvider, unpatched_properties: VecMap>, + outbound_policy: OutboundPolicy, ) -> ValidationResult { - let response = match validate_tenant_quota( - set.server, - set.access_token, - TenantStorageQuota::MaxAcmeProviders, - ) - .await? - { - Ok(response) => response, - Err(err) => { - return Ok(Err(err)); - } - }; + if let Err(err) = validate_acme_directory(provider, outbound_policy) { + return Ok(Err(err)); + } + + let response = ObjectResponse::default(); // Obtain EAB credentials let mut eab_key_id = None; @@ -89,10 +79,8 @@ pub(crate) async fn validate_certificate( match cert.certificate.value().await { Ok(pem) => match ParsedCert::parse(pem.as_ref()) { Ok(parsed) => { - cert.not_valid_after = - UTCDateTime::from_timestamp(parsed.valid_not_after.timestamp()); - cert.not_valid_before = - UTCDateTime::from_timestamp(parsed.valid_not_before.timestamp()); + cert.not_valid_after = parsed.valid_not_after; + cert.not_valid_before = parsed.valid_not_before; cert.issuer = parsed.issuer; cert.subject_alternative_names = Map::new(parsed.sans); @@ -110,3 +98,17 @@ pub(crate) async fn validate_certificate( Ok(Ok(ObjectResponse::default())) } } + +pub(crate) fn validate_acme_directory( + provider: &AcmeProvider, + outbound_policy: OutboundPolicy, +) -> Result<(), SetError> { + outbound_policy + .parse_url(&provider.directory) + .map(|_| ()) + .map_err(|err| { + SetError::invalid_properties() + .with_property(Property::Directory) + .with_description(err.as_str()) + }) +} diff --git a/crates/jmap/src/registry/mod.rs b/crates/jmap/src/registry/mod.rs index 6bb01bda..aa36e01d 100644 --- a/crates/jmap/src/registry/mod.rs +++ b/crates/jmap/src/registry/mod.rs @@ -17,21 +17,9 @@ pub trait EnterpriseRegistry { } impl EnterpriseRegistry for Server { - fn assert_enterprise_object(&self, object_type: ObjectType) -> trc::Result<()> { - if !matches!( - object_type, - ObjectType::MaskedEmail - | ObjectType::ArchivedItem - | ObjectType::Metric - | ObjectType::Trace - ) { - return Ok(()); - } - - Err(trc::JmapEvent::Forbidden.into_err().details(concat!( - "This feature is only available in the Enterprise edition. ", - "Obtain your trial license at https://license.stalw.art/trial." - ))) + #[cfg(not(feature = "enterprise"))] + fn assert_enterprise_object(&self, _object_type: ObjectType) -> trc::Result<()> { + Ok(()) } } diff --git a/crates/jmap/src/registry/query.rs b/crates/jmap/src/registry/query.rs index 22ce60fc..f34383e5 100644 --- a/crates/jmap/src/registry/query.rs +++ b/crates/jmap/src/registry/query.rs @@ -10,12 +10,14 @@ use crate::{ EnterpriseRegistry, mapping::{ RegistryQueryResponse, account::credential_query, cluster::cluster_node_query, - log::log_query, queued_message::queued_message_query, report::report_query, + index_queue::index_queue_status_query, log::log_query, + queued_message::queued_message_query, report::report_query, spam_sample::spam_sample_query, task::task_query, }, }, }; use common::{Server, auth::AccessToken}; +use compact_str::format_compact; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse}, object::registry::{Registry, RegistryComparator, RegistryFilter, RegistryFilterOperator}, @@ -113,6 +115,15 @@ impl RegistryQuery for Server { .and_then(|response| response.build()) } + ObjectType::IndexQueueStatus => index_queue_status_query(RegistryQueryResponse { + server: self, + access_token, + object_type, + request, + }) + .await + .and_then(|response| response.build()), + ObjectType::Task => task_query(RegistryQueryResponse { server: self, access_token, @@ -136,15 +147,13 @@ impl RegistryQuery for Server { .details("Actions cannot be queried")), _ => { - let mut query = store::registry::RegistryQuery::new(object_type) - .with_tenant(access_token.tenant_id()); + let mut query = store::registry::RegistryQuery::new(object_type); let can_impersonate = access_token.has_permission(Permission::Impersonate); if !can_impersonate { query = query.with_account(request.account_id.document_id()); } let indexes = object_type.indexes(); request.extract_filters(|property, op, value| match property { - Property::MemberTenantId if access_token.tenant_id().is_some() => true, Property::AccountId if !can_impersonate => true, property => { let Some(index) = indexes.iter().find(|i| i.prop == property) else { @@ -205,27 +214,27 @@ impl RegistryQuery for Server { let params = request .extract_parameters(self.core.jmap.query_max_results, Some(Property::Id))?; - if let Some(limit) = params.limit { - query = query.with_limit(limit); - if let Some(anchor) = params.anchor { - query = query.with_anchor(anchor); - } else if let Some(position) = params.position { - query = query.with_index_start(position); - } - } + let mut position = None; let matches = if query.has_filters() || params.sort_by == Property::Id { - let matches = self.registry().query::>(query).await?; - if matches.is_empty() { - return QueryResponseBuilder::new( - 0, - self.core.jmap.query_max_results, - State::Initial, - &request, + let page = self + .registry() + .query_page::>(params.paged(query)) + .await?; + position = params.page_position(page.position)?; + if page.ids.is_empty() { + return with_page_position( + QueryResponseBuilder::new( + 0, + self.core.jmap.query_max_results, + State::Initial, + &request, + ), + position, ) .build(); } - matches.into() + page.ids.into() } else { None }; @@ -233,7 +242,7 @@ impl RegistryQuery for Server { let results = match params.sort_by { Property::Id => { let mut results = matches.unwrap(); - if !params.sort_ascending { + if params.needs_reversal() { results.sort_unstable_by(|a, b| b.cmp(a)); } results @@ -244,7 +253,10 @@ impl RegistryQuery for Server { .find(|i| i.prop == property && i.value != IndexSchemaValueType::Text) else { return Err(trc::JmapEvent::UnsupportedSort.into_err().details( - format!("Property {} is not supported for sorting", property), + format_compact!( + "Property {} is not supported for sorting", + property + ), )); }; @@ -266,11 +278,14 @@ impl RegistryQuery for Server { }; // Build response - let mut response = QueryResponseBuilder::new( - results.len(), - self.core.jmap.query_max_results, - State::Initial, - &request, + let mut response = with_page_position( + QueryResponseBuilder::new( + results.len(), + self.core.jmap.query_max_results, + State::Initial, + &request, + ), + position, ); for id in results { @@ -306,6 +321,47 @@ pub(crate) struct RegistryQueryParameters { pub limit: Option, } +impl RegistryQueryParameters { + pub fn paged(&self, query: store::registry::RegistryQuery) -> store::registry::RegistryQuery { + let Some(limit) = self.limit else { + return query; + }; + let query = query.with_limit(limit); + let query = if self.sort_ascending { + query.with_ascending() + } else { + query.with_descending() + }; + match (self.anchor, self.position) { + (Some(anchor), _) => query.with_anchor(anchor), + (None, Some(position)) => query.with_index_start(position), + (None, None) => query, + } + } + + pub fn page_position(&self, position: Option) -> trc::Result> { + match (self.limit, position) { + (None, _) => Ok(None), + (Some(_), Some(position)) => Ok(Some(position)), + (Some(_), None) => Err(trc::JmapEvent::AnchorNotFound.into_err()), + } + } + + pub fn needs_reversal(&self) -> bool { + !self.sort_ascending && self.limit.is_none() + } +} + +pub(crate) fn with_page_position( + response: QueryResponseBuilder, + position: Option, +) -> QueryResponseBuilder { + match position { + Some(position) => response.with_position(position), + None => response, + } +} + impl RegistryQueryFilters for QueryRequest { fn extract_filters( &mut self, @@ -332,7 +388,7 @@ impl RegistryQueryFilters for QueryRequest { }; if !cb(property, operator, value) { return Err(trc::JmapEvent::UnsupportedFilter.into_err().details( - format!( + format_compact!( "Filter on property {} is not supported or invalid", property ), @@ -340,16 +396,14 @@ impl RegistryQueryFilters for QueryRequest { } } RegistryFilter::_T(other) => { - return Err(trc::JmapEvent::UnsupportedFilter - .into_err() - .details(other.to_string())); + return Err(trc::JmapEvent::UnsupportedFilter.into_err().details(other)); } }, Filter::And | Filter::Close => {} Filter::Or | Filter::Not => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details("Only AND is supported in filters".to_string())); + .details("Only AND is supported in filters")); } } } @@ -362,15 +416,6 @@ impl RegistryQueryFilters for QueryRequest { max_results: usize, external_filter: Option, ) -> trc::Result { - #[cfg(feature = "test_mode")] - let comparator = self - .sort - .take() - .unwrap_or_default() - .into_iter() - .next() - .unwrap_or_else(|| Comparator::ascending(RegistryComparator::Property(Property::Id))); - #[cfg(not(feature = "test_mode"))] let comparator = self .sort .take() @@ -384,7 +429,7 @@ impl RegistryQueryFilters for QueryRequest { if external_filter.is_some_and(|f| f == property) && !self.calculate_total.unwrap_or(false) && self.anchor_offset.is_none_or(|offset| offset == 0) - && self.position.is_none_or(|pos| pos > 0) + && self.position.is_none_or(|pos| pos >= 0) { Ok(RegistryQueryParameters { sort_by: property, @@ -410,7 +455,10 @@ impl RegistryQueryFilters for QueryRequest { } RegistryComparator::_T(other) => Err(trc::JmapEvent::UnsupportedSort .into_err() - .details(format!("Property {} is not supported for sorting", other))), + .details(format_compact!( + "Property {} is not supported for sorting", + other + ))), } } } diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index cc904548..73b997d0 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -11,12 +11,15 @@ use crate::registry::{ account::account_set, action::action_set, bootstrap::bootstrap_set, + cluster::{validate_coordinator, validate_in_memory_store}, dkim::validate_dkim_signature, domain::{validate_dns_server, validate_domain}, + index_queue::index_queue_status_set, map_bootstrap_error, + oauth_client::validate_oauth_client_object, principal::{ - AccountUpdate, schedule_account_destruction, validate_account, validate_role, - validate_tenant_quota, + AccountUpdate, schedule_account_destruction, validate_account, validate_account_access, + validate_role, }, public_key::validate_public_key, queued_message::queued_message_set, @@ -24,15 +27,16 @@ use crate::registry::{ sieve::validate_sieve_script, spam_sample::spam_sample_set, task::task_set, - tls::{validate_acme_provider, validate_certificate}, + tls::{validate_acme_directory, validate_acme_provider, validate_certificate}, }, }; use common::{ - Server, auth::AccessToken, cache::invalidate::CacheInvalidationBuilder, - expr::if_block::BootstrapExprExt, ipc::CacheInvalidation, - manager::application::WebApplicationManager, + Server, + auth::AccessToken, + cache::invalidate::CacheInvalidationBuilder, + expr::if_block::BootstrapExprExt, + manager::{app_client::validate_url_prefix, application::WebApplicationManager}, }; -use directory::core::secret::{hash_secret, is_password_hash}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -45,18 +49,14 @@ use jmap_tools::{JsonPointer, JsonPointerItem, Key}; use registry::{ jmap::{JmapValue, JsonPointerPatch, MaybeUnpatched, RegistryValue}, schema::{ - enums::{Permission, TenantStorageQuota}, - prelude::{ - OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, - Property, - }, - structs::{ - Certificate, DkimSignature, DnsServer, Domain, PublicKey, Role, SieveSystemScript, - SieveUserScript, Task, - }, + enums::Permission, + prelude::{OBJ_FILTER_ACCOUNT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, Property}, + structs::{Certificate, DkimSignature, DnsServer, Domain, PublicKey, Role, Task}, }, types::id::ObjectId, }; +use services::task_manager::reconcile::schedule_replacing; +use services::task_manager::tasks::rebuild_shares::RevokeAccountGrants; use std::borrow::Cow; use store::{ registry::{ @@ -67,7 +67,7 @@ use store::{ }; use trc::AddContext; use types::id::Id; -use utils::map::vec_map::VecMap; +use utils::{map::vec_map::VecMap, outbound::OutboundPolicy}; pub trait RegistrySet: Sync + Send { fn registry_set( @@ -79,6 +79,449 @@ pub trait RegistrySet: Sync + Send { ) -> impl Future>> + Send; } +async fn registry_object_set( + mut set: RegistrySetResponse<'_>, +) -> trc::Result> { + let server = set.server; + let access_token = set.access_token; + let object_type = set.object_type; + let object_flags = object_type.flags(); + let is_singleton = (object_flags & OBJ_SINGLETON) != 0; + let has_account_id = (object_flags & OBJ_FILTER_ACCOUNT) != 0; + let can_set_account = access_token.has_permission(Permission::Impersonate); + let is_account_filtered = set.is_account_filtered; + // Bundle modifications together + let mut modifications = Vec::with_capacity(set.create.len() + set.update.len()); + for (id, value) in set.create.drain() { + if is_singleton + && let Some(object) = server + .registry() + .get(ObjectId::new(object_type, Id::singleton())) + .await + .caused_by(trc::location!())? + { + modifications.push(( + Modification::Create { + client_id: id, + object: Some(object), + }, + value, + Object::from(set.object_type), + )); + } else { + modifications.push(( + Modification::Create { + client_id: id, + object: None, + }, + value, + Object::from(set.object_type), + )); + } + } + for (id, value) in set.update.drain(..) { + if let Some(object) = server + .registry() + .get(ObjectId::new(object_type, id)) + .await + .caused_by(trc::location!())? + { + #[cfg(not(feature = "enterprise"))] + let is_tenant_mismatch = false; + if is_tenant_mismatch + || (is_account_filtered + && object.inner.account_id() != Some(Id::from(set.account_id))) + { + set.response.not_updated.append(id, SetError::not_found()); + continue; + } + + modifications.push(( + Modification::Update { + id, + object: object.clone(), + }, + value, + object, + )); + } else if is_singleton { + modifications.push(( + Modification::Update { + id, + object: Object::from(set.object_type), + }, + value, + Object::from(set.object_type), + )); + } else { + set.response.not_updated.append(id, SetError::not_found()); + } + } + + // Process modifications + let mut cache_invalidator = CacheInvalidationBuilder::default(); + 'outer: for (modification, mut value, mut new_object) in modifications { + // Initial validations + let is_create = matches!(modification, Modification::Create { .. }); + let mut unpatched_properties = VecMap::new(); + + if let Err(err) = set.response.resolve_self_references(&mut value, 0, true) { + set.failed(modification, err); + continue 'outer; + }; + + if is_create + || value + .as_object() + .unwrap() + .get(&Key::Property(Property::Type)) + .and_then(|v| v.as_str()) + .is_some_and(|t| new_object.object_variant().is_some_and(|v| v != t)) + { + // Patch object + let ptr = JsonPointer::new(vec![]); + let patch = JsonPointerPatch::new(&ptr).with_create(true); + match new_object.patch(patch.with_can_set_account(can_set_account), value) { + Ok(MaybeUnpatched::Patched) => {} + Ok(MaybeUnpatched::Unpatched { property, value }) => { + unpatched_properties.append(property, value); + } + Ok(MaybeUnpatched::UnpatchedMany { properties }) => { + unpatched_properties = properties; + } + Err(err) => { + set.failed(modification, err.into()); + continue 'outer; + } + } + + // Add tenantId for tenant filtered objects + + // Add accountId + if has_account_id { + new_object.inner.set_account_id(set.account_id.into()); + } + } else { + for (key, value) in value.into_expanded_object() { + let ptr = match key { + Key::Property(Property::Type) => { + continue; + } + Key::Property(prop) => { + JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]) + } + Key::Borrowed(other) => JsonPointer::parse(other), + Key::Owned(other) => JsonPointer::parse(&other), + }; + + // Patch object + let patch = JsonPointerPatch::new(&ptr).with_create(false); + match new_object.patch(patch.with_can_set_account(can_set_account), value) { + Ok(MaybeUnpatched::Patched) => {} + Ok(MaybeUnpatched::Unpatched { property, value }) => { + unpatched_properties.append(property, value); + } + Ok(MaybeUnpatched::UnpatchedMany { properties }) => { + if unpatched_properties.is_empty() { + unpatched_properties = properties; + } else { + unpatched_properties.extend(properties); + } + } + Err(err) => { + set.failed(modification, err.into()); + continue 'outer; + } + } + } + } + + // Validate objects + let mut tasks = Vec::new(); + #[cfg(not(feature = "enterprise"))] + let outbound_policy = OutboundPolicy::Trusted; + let result = match &mut new_object.inner { + ObjectInner::Account(account) => { + validate_account(server, access_token, account, modification.as_account()).await? + } + ObjectInner::Role(role) => { + validate_role(access_token, role, modification.as_role()).await? + } + ObjectInner::PublicKey(key) => { + validate_public_key(&set, key, modification.as_public_key()).await? + } + ObjectInner::DkimSignature(key) => { + validate_dkim_signature(key, modification.as_dkim_signature()).await? + } + ObjectInner::Domain(domain) => { + validate_domain( + &set, + domain, + modification.as_domain(), + outbound_policy, + &mut tasks, + ) + .await? + } + ObjectInner::DnsServer(dns) => { + validate_dns_server(&set, dns, modification.as_dns_server(), outbound_policy) + .await? + } + ObjectInner::InMemoryStore(store) => validate_in_memory_store(server, store).await?, + ObjectInner::Coordinator(coordinator) => { + validate_coordinator(server, coordinator).await? + } + ObjectInner::OAuthClient(client) => { + validate_oauth_client_object(server, access_token, client).await? + } + ObjectInner::Application(app) => app + .url_prefix + .iter() + .find_map(|prefix| validate_url_prefix(prefix).err()) + .map_or_else( + || Ok(ObjectResponse::default()), + |err| { + Err(SetError::invalid_properties() + .with_property(Property::UrlPrefix) + .with_description(err)) + }, + ), + ObjectInner::AcmeProvider(provider) if is_create => { + validate_acme_provider(provider, unpatched_properties, outbound_policy).await? + } + ObjectInner::AcmeProvider(provider) => { + validate_acme_directory(provider, outbound_policy) + .map(|_| ObjectResponse::default()) + } + ObjectInner::Certificate(cert) => { + validate_certificate(cert, modification.as_certificate()).await? + } + ObjectInner::SieveUserScript(script) => { + validate_sieve_script( + set.server, + &script.contents, + modification.as_sieve_script(), + false, + ) + .await? + } + ObjectInner::SieveSystemScript(script) => { + validate_sieve_script( + set.server, + &script.contents, + modification.as_sieve_script(), + true, + ) + .await? + } + _ => Ok(ObjectResponse::default()), + }; + + let mut response = match result { + Ok(response) => response, + Err(err) => { + set.failed(modification, err); + continue 'outer; + } + }; + + // Validate expressions + if let Some(expressions) = new_object.inner.expression_ctxs() { + let mut bp = Bootstrap::new_uninitialized(server.registry().clone()); + + for expression in expressions { + bp.compile_expr(ObjectId::new(object_type, 0u64.into()), &expression); + if !bp.errors.is_empty() { + set.failed( + modification, + map_bootstrap_error(bp.errors) + .with_object_id_opt(None) + .with_property(expression.property), + ); + continue 'outer; + } + } + } + + // Save object + let result = match &modification { + Modification::Create { client_id, object } => { + if let Some(object) = object { + if object.inner != new_object.inner { + server + .registry() + .write(RegistryWrite::update(Id::singleton(), &new_object, object)) + .await? + } else { + set.response.created(client_id.to_string(), Id::singleton()); + continue; + } + } else { + server + .registry() + .write(RegistryWrite::Insert { + object: &new_object, + id: response.id, + }) + .await? + } + } + Modification::Update { id, object } => { + if object.inner != new_object.inner { + if !(is_singleton && object.revision == 0) { + server + .registry() + .write(RegistryWrite::update(*id, &new_object, object)) + .await? + } else { + server + .registry() + .write(RegistryWrite::insert(&new_object)) + .await? + } + } else { + set.response.updated.append(*id, None); + continue; + } + } + }; + + let object_id = match (modification, result) { + (Modification::Update { id, object }, RegistryWriteResult::Success(_)) => { + cache_invalidator.process_update(id, &object, &new_object); + if let (ObjectInner::Application(previous), ObjectInner::Application(updated)) = + (&object.inner, &new_object.inner) + && previous.resource_url != updated.resource_url + && let Err(err) = WebApplicationManager::delete_bundle(server, id).await + { + trc::error!(err.details("Failed to delete cached application bundle")); + } + set.response.updated.append( + id, + if !response.object.is_empty() { + Some(JmapValue::Object(response.object)) + } else { + None + }, + ); + Some(id) + } + (Modification::Create { client_id, .. }, RegistryWriteResult::Success(id)) => { + cache_invalidator.process_create(&new_object); + response.object.insert(Property::Id, RegistryValue::Id(id)); + set.response + .created + .insert(client_id, JmapValue::Object(response.object)); + Some(id) + } + (Modification::Update { id, .. }, err) => { + set.response.not_updated.append(id, map_write_error(err)); + None + } + (Modification::Create { client_id, .. }, err) => { + set.response + .not_created + .append(client_id, map_write_error(err)); + None + } + }; + + // Dispatch tasks + if !tasks.is_empty() + && let Some(object_id) = object_id + { + let mut batch = BatchBuilder::new(); + for mut task in tasks.drain(..) { + match &mut task { + Task::AcmeRenewal(task) => task.domain_id = object_id, + Task::DkimManagement(task) => task.domain_id = object_id, + Task::DnsManagement(task) => task.domain_id = object_id, + _ => unreachable!(), + } + schedule_replacing(set.server, &mut batch, task).await?; + } + set.server.commit_batch(batch).await?; + } + } + + #[cfg(not(feature = "enterprise"))] + const ACCOUNT_ORPHAN_TYPES: &[ObjectType] = &[ObjectType::PublicKey, ObjectType::OAuthGrant]; + + // Process destroy + for id in set.destroy.drain(..) { + let object_id = ObjectId::new(object_type, id); + if let Some(object) = server + .registry() + .get(object_id) + .await + .caused_by(trc::location!())? + .filter(|object| { + #[cfg(not(feature = "enterprise"))] + let is_tenant_mismatch = false; + !(is_tenant_mismatch + || (is_account_filtered + && object.inner.account_id() != Some(Id::from(set.account_id)))) + }) + { + if let ObjectInner::Account(account) = &object.inner + && let Err(err) = validate_account_access(server, access_token, account).await? + { + set.response.not_destroyed.append(id, err); + continue; + } + + match server + .registry() + .write(RegistryWrite::Delete { + object_id, + object: Some(&object), + allowed_orphan_types: if object_type == ObjectType::Account { + ACCOUNT_ORPHAN_TYPES + } else { + &[] + }, + }) + .await? + { + RegistryWriteResult::Success(_) => { + if let ObjectInner::Account(account) = &object.inner { + for sharee_id in server + .revoke_account_grants(id.document_id()) + .await + .caused_by(trc::location!())? + { + server + .invalidate_grantee(&mut cache_invalidator, sharee_id) + .await; + } + + schedule_account_destruction(set.server, id, account).await?; + } + + if matches!(object.inner, ObjectInner::Application(_)) + && let Err(err) = WebApplicationManager::delete_bundle(server, id).await + { + trc::error!(err.details("Failed to delete cached application bundle")); + } + + cache_invalidator.process_delete(id, &object); + set.response.destroyed.push(id); + } + err => { + set.response.not_destroyed.append(id, map_write_error(err)); + } + } + } else { + set.response.not_destroyed.append(id, SetError::not_found()); + } + } + + // Finalize cache invalidation + server.invalidate_caches(cache_invalidator).await?; + + Ok(set.into_response()) +} + #[allow(clippy::large_enum_variant)] enum Modification { Create { @@ -111,9 +554,6 @@ impl RegistrySet for Server { let object_flags = object_type.flags(); let is_singleton = (object_flags & OBJ_SINGLETON) != 0; let has_account_id = (object_flags & OBJ_FILTER_ACCOUNT) != 0; - let is_tenant_filtered = - (object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); - let can_set_tenant = access_token.tenant_id().is_none(); let can_set_account = access_token.has_permission(Permission::Impersonate); let is_account_filtered = has_account_id && !can_set_account; @@ -161,7 +601,6 @@ impl RegistrySet for Server { account_id: request.account_id.document_id(), object_type, response, - is_tenant_filtered, is_account_filtered, create, update, @@ -183,7 +622,6 @@ impl RegistrySet for Server { | ObjectType::DmarcReportSettings | ObjectType::DnsResolver | ObjectType::Email - | ObjectType::Enterprise | ObjectType::FileStorage | ObjectType::Http | ObjectType::HttpForm @@ -191,8 +629,8 @@ impl RegistrySet for Server { | ObjectType::InMemoryStore | ObjectType::Jmap | ObjectType::SystemSettings + | ObjectType::Metadata | ObjectType::Metrics - | ObjectType::MetricsStore | ObjectType::MtaConnectionStrategy | ObjectType::MtaExtensions | ObjectType::MtaInboundSession @@ -216,18 +654,14 @@ impl RegistrySet for Server { | ObjectType::SieveUserInterpreter | ObjectType::SpamClassifier | ObjectType::SpamDnsblSettings - | ObjectType::SpamLlm | ObjectType::SpamPyzor | ObjectType::SpamSettings | ObjectType::SpfReportSettings | ObjectType::TaskManager | ObjectType::TlsReportSettings - | ObjectType::TracingStore | ObjectType::WebDav | ObjectType::DsnReportSettings | ObjectType::AcmeProvider - | ObjectType::AiModel - | ObjectType::Alert | ObjectType::AllowedIp | ObjectType::Application | ObjectType::BlockedIp @@ -259,471 +693,12 @@ impl RegistrySet for Server { | ObjectType::WebHook | ObjectType::PublicKey | ObjectType::DkimSignature - | ObjectType::MaskedEmail | ObjectType::Account | ObjectType::MailingList | ObjectType::OAuthClient + | ObjectType::OAuthGrant | ObjectType::Role - | ObjectType::Tenant - | ObjectType::Domain => { - // Bundle modifications together - let mut modifications = Vec::with_capacity(set.create.len() + set.update.len()); - for (id, value) in set.create.drain() { - if is_singleton - && let Some(object) = self - .registry() - .get(ObjectId::new(object_type, Id::singleton())) - .await - .caused_by(trc::location!())? - { - modifications.push(( - Modification::Create { - client_id: id, - object: Some(object), - }, - value, - Object::from(set.object_type), - )); - } else { - modifications.push(( - Modification::Create { - client_id: id, - object: None, - }, - value, - Object::from(set.object_type), - )); - } - } - for (id, value) in set.update.drain(..) { - if let Some(object) = self - .registry() - .get(ObjectId::new(object_type, id)) - .await - .caused_by(trc::location!())? - { - if (is_tenant_filtered - && access_token.tenant_id().map(Id::from) - != object.inner.member_tenant_id()) - || (is_account_filtered - && object.inner.account_id() != Some(Id::from(set.account_id))) - { - set.response.not_updated.append(id, SetError::not_found()); - continue; - } - - modifications.push(( - Modification::Update { - id, - object: object.clone(), - }, - value, - object, - )); - } else if is_singleton { - modifications.push(( - Modification::Update { - id, - object: Object::from(set.object_type), - }, - value, - Object::from(set.object_type), - )); - } else { - set.response.not_updated.append(id, SetError::not_found()); - } - } - - // Process modifications - let mut cache_invalidator = CacheInvalidationBuilder::default(); - 'outer: for (modification, mut value, mut new_object) in modifications { - // Initial validations - let is_create = matches!(modification, Modification::Create { .. }); - let mut unpatched_properties = VecMap::new(); - - if let Err(err) = set.response.resolve_self_references(&mut value, 0, true) { - set.failed(modification, err); - continue 'outer; - }; - - if is_create - || value - .as_object() - .unwrap() - .get(&Key::Property(Property::Type)) - .and_then(|v| v.as_str()) - .is_some_and(|t| new_object.object_variant().is_some_and(|v| v != t)) - { - // Patch object - match new_object.patch( - JsonPointerPatch::new(&JsonPointer::new(vec![])) - .with_create(true) - .with_can_set_tenant(can_set_tenant) - .with_can_set_account(can_set_account), - value, - ) { - Ok(MaybeUnpatched::Patched) => {} - Ok(MaybeUnpatched::Unpatched { property, value }) => { - unpatched_properties.append(property, value); - } - Ok(MaybeUnpatched::UnpatchedMany { properties }) => { - unpatched_properties = properties; - } - Err(err) => { - set.failed(modification, err.into()); - continue 'outer; - } - } - - // Add tenantId for tenant filtered objects - if is_tenant_filtered && let Some(tenant_id) = set.access_token.tenant_id() - { - new_object.inner.set_member_tenant_id(tenant_id.into()); - } - - // Add accountId - if has_account_id { - new_object.inner.set_account_id(set.account_id.into()); - } - } else { - for (key, value) in value.into_expanded_object() { - let ptr = match key { - Key::Property(Property::Type) => { - continue; - } - Key::Property(prop) => { - JsonPointer::new(vec![JsonPointerItem::Key(Key::Property( - prop, - ))]) - } - Key::Borrowed(other) => JsonPointer::parse(other), - Key::Owned(other) => JsonPointer::parse(&other), - }; - - // Patch object - match new_object.patch( - JsonPointerPatch::new(&ptr) - .with_create(false) - .with_can_set_tenant(can_set_tenant) - .with_can_set_account(can_set_account), - value, - ) { - Ok(MaybeUnpatched::Patched) => {} - Ok(MaybeUnpatched::Unpatched { property, value }) => { - unpatched_properties.append(property, value); - } - Ok(MaybeUnpatched::UnpatchedMany { properties }) => { - if unpatched_properties.is_empty() { - unpatched_properties = properties; - } else { - unpatched_properties.extend(properties); - } - } - Err(err) => { - set.failed(modification, err.into()); - continue 'outer; - } - } - } - } - - // Validate objects - let mut tasks = Vec::new(); - let result = match &mut new_object.inner { - ObjectInner::Account(account) => { - validate_account(self, access_token, account, modification.as_account()) - .await? - } - ObjectInner::Role(role) => { - validate_role(self, access_token, role, modification.as_role()).await? - } - ObjectInner::PublicKey(key) => { - validate_public_key(&set, key, modification.as_public_key()).await? - } - ObjectInner::DkimSignature(key) => { - validate_dkim_signature(&set, key, modification.as_dkim_signature()) - .await? - } - ObjectInner::Domain(domain) => { - validate_domain(&set, domain, modification.as_domain(), &mut tasks) - .await? - } - ObjectInner::DnsServer(dns) => { - validate_dns_server(&set, dns, modification.as_dns_server()).await? - } - ObjectInner::MailingList(_) if is_create => { - validate_tenant_quota( - self, - access_token, - TenantStorageQuota::MaxMailingLists, - ) - .await? - } - ObjectInner::OAuthClient(client) => { - if let Some(secret) = client.secret.as_mut() - && !secret.is_empty() - && !(matches!(secret.as_bytes().first(), Some(&b'$' | &b'{')) - && is_password_hash(secret)) - { - *secret = hash_secret( - set.server.core.network.security.password_hash_algorithm, - std::mem::take(secret).into_bytes(), - ) - .await - .caused_by(trc::location!())?; - } - if is_create { - validate_tenant_quota( - self, - access_token, - TenantStorageQuota::MaxOauthClients, - ) - .await? - } else { - Ok(ObjectResponse::default()) - } - } - ObjectInner::Directory(_) if is_create => { - validate_tenant_quota( - self, - access_token, - TenantStorageQuota::MaxDirectories, - ) - .await? - } - ObjectInner::AcmeProvider(provider) if is_create => { - validate_acme_provider(&set, provider, unpatched_properties).await? - } - ObjectInner::Certificate(cert) => { - validate_certificate(cert, modification.as_certificate()).await? - } - ObjectInner::SieveUserScript(SieveUserScript { contents, .. }) => { - validate_sieve_script( - set.server, - contents, - modification.as_sieve_script(), - false, - ) - .await? - } - ObjectInner::SieveSystemScript(SieveSystemScript { contents, .. }) => { - validate_sieve_script( - set.server, - contents, - modification.as_sieve_script(), - true, - ) - .await? - } - _ => Ok(ObjectResponse::default()), - }; - - let mut response = match result { - Ok(response) => response, - Err(err) => { - set.failed(modification, err); - continue 'outer; - } - }; - - // Validate expressions - if let Some(expressions) = new_object.inner.expression_ctxs() { - let mut bp = Bootstrap::new_uninitialized(self.registry().clone()); - - for expression in expressions { - bp.compile_expr(ObjectId::new(object_type, 0u64.into()), &expression); - if !bp.errors.is_empty() { - set.failed( - modification, - map_bootstrap_error(bp.errors) - .with_object_id_opt(None) - .with_property(expression.property), - ); - continue 'outer; - } - } - } - - // Save object - let result = match &modification { - Modification::Create { client_id, object } => { - if let Some(object) = object { - if object.inner != new_object.inner { - self.registry() - .write(RegistryWrite::update( - Id::singleton(), - &new_object, - object, - )) - .await? - } else { - set.response.created(client_id.to_string(), Id::singleton()); - continue; - } - } else { - self.registry() - .write(RegistryWrite::Insert { - object: &new_object, - id: response.id, - }) - .await? - } - } - Modification::Update { id, object } => { - if object.inner != new_object.inner { - if !(is_singleton && object.revision == 0) { - self.registry() - .write(RegistryWrite::update(*id, &new_object, object)) - .await? - } else { - self.registry() - .write(RegistryWrite::insert(&new_object)) - .await? - } - } else { - set.response.updated.append(*id, None); - continue; - } - } - }; - - let object_id = match (modification, result) { - (Modification::Update { id, object }, RegistryWriteResult::Success(_)) => { - cache_invalidator.process_update(id, &object, &new_object); - if let ( - ObjectInner::Application(previous), - ObjectInner::Application(updated), - ) = (&object.inner, &new_object.inner) - && previous.resource_url != updated.resource_url - && let Err(err) = - WebApplicationManager::delete_bundle(self, id).await - { - trc::error!( - err.details("Failed to delete cached application bundle") - ); - } - set.response.updated.append( - id, - if !response.object.is_empty() { - Some(JmapValue::Object(response.object)) - } else { - None - }, - ); - Some(id) - } - ( - Modification::Create { client_id, .. }, - RegistryWriteResult::Success(id), - ) => { - cache_invalidator.process_create(&new_object); - response.object.insert(Property::Id, RegistryValue::Id(id)); - set.response - .created - .insert(client_id, JmapValue::Object(response.object)); - Some(id) - } - (Modification::Update { id, .. }, err) => { - set.response.not_updated.append(id, map_write_error(err)); - None - } - (Modification::Create { client_id, .. }, err) => { - set.response - .not_created - .append(client_id, map_write_error(err)); - None - } - }; - - // Dispatch tasks - if !tasks.is_empty() - && let Some(object_id) = object_id - { - let mut batch = BatchBuilder::new(); - for mut task in tasks.drain(..) { - match &mut task { - Task::AcmeRenewal(task) => task.domain_id = object_id, - Task::DkimManagement(task) => task.domain_id = object_id, - Task::DnsManagement(task) => task.domain_id = object_id, - _ => unreachable!(), - } - batch.schedule_task(task); - } - set.server.store().write(batch.build_all()).await?; - set.server.notify_task_queue(); - } - } - - // Process destroy - for id in set.destroy.drain(..) { - let object_id = ObjectId::new(object_type, id); - if let Some(object) = self - .registry() - .get(object_id) - .await - .caused_by(trc::location!())? - .filter(|object| { - !((is_tenant_filtered - && access_token.tenant_id().map(Id::from) - != object.inner.member_tenant_id()) - || (is_account_filtered - && object.inner.account_id() != Some(Id::from(set.account_id)))) - }) - { - match self - .registry() - .write(RegistryWrite::Delete { - object_id, - object: Some(&object), - allowed_orphan_types: if object_type == ObjectType::Account { - &[ObjectType::PublicKey, ObjectType::MaskedEmail] - } else { - &[] - }, - }) - .await? - { - RegistryWriteResult::Success(_) => { - if let ObjectInner::Account(account) = &object.inner { - for sharee_id in self - .store() - .acl_revoke_all(id.document_id()) - .await - .caused_by(trc::location!())? - { - cache_invalidator - .invalidate(CacheInvalidation::AccessToken(sharee_id)); - } - - schedule_account_destruction(set.server, id, account).await?; - } - - if matches!(object.inner, ObjectInner::Application(_)) - && let Err(err) = - WebApplicationManager::delete_bundle(self, id).await - { - trc::error!( - err.details("Failed to delete cached application bundle") - ); - } - - cache_invalidator.process_delete(id, &object); - set.response.destroyed.push(id); - } - err => { - set.response.not_destroyed.append(id, map_write_error(err)); - } - } - } else { - set.response.not_destroyed.append(id, SetError::not_found()); - } - } - - // Finalize cache invalidation - self.invalidate_caches(cache_invalidator).await?; - - Ok(set.into_response()) - } + | ObjectType::Domain => registry_object_set(set).await, ObjectType::ArfExternalReport | ObjectType::DmarcExternalReport | ObjectType::TlsExternalReport @@ -737,37 +712,30 @@ impl RegistrySet for Server { ObjectType::AccountSettings | ObjectType::ApiKey | ObjectType::AccountPassword - | ObjectType::AppPassword => Box::pin(account_set(set)) - .await - .map(|set| set.into_response()), + | ObjectType::AppPassword => account_set(set).await.map(|set| set.into_response()), ObjectType::QueuedMessage => { queued_message_set(set).await.map(|set| set.into_response()) } - ObjectType::Task => task_set(set).await.map(|set| set.into_response()), - - ObjectType::Action => Box::pin(action_set(set)) + ObjectType::IndexQueueStatus => index_queue_status_set(set) .await .map(|set| set.into_response()), + ObjectType::Task => task_set(set).await.map(|set| set.into_response()), + + ObjectType::Action => action_set(set).await.map(|set| set.into_response()), + ObjectType::Bootstrap => Box::pin(bootstrap_set(set)) .await .map(|set| set.into_response()), - ObjectType::Log | ObjectType::Metric | ObjectType::Trace | ObjectType::ClusterNode => { + ObjectType::Log | ObjectType::ClusterNode => { set.fail_all_create("Telemetry objects cannot be created"); set.fail_all_update("Telemetry objects cannot be modified"); set.fail_all_destroy("Telemetry objects cannot be deleted"); Ok(set.into_response()) } - #[cfg(not(feature = "enterprise"))] - _ => { - set.fail_all_create("Enterprise objects cannot be created"); - set.fail_all_update("Enterprise objects cannot be modified"); - set.fail_all_destroy("Enterprise objects cannot be deleted"); - Ok(set.into_response()) - } } } } @@ -878,6 +846,7 @@ impl Modification { } } + fn as_dns_server(&self) -> Option<&DnsServer> { match self { Modification::Create { .. } => None, @@ -902,10 +871,8 @@ impl Modification { match self { Modification::Create { .. } => None, Modification::Update { object, .. } => match &object.inner { - ObjectInner::SieveUserScript(SieveUserScript { contents, .. }) - | ObjectInner::SieveSystemScript(SieveSystemScript { contents, .. }) => { - Some(contents.as_str()) - } + ObjectInner::SieveUserScript(script) => Some(script.contents.as_str()), + ObjectInner::SieveSystemScript(script) => Some(script.contents.as_str()), _ => None, }, } diff --git a/crates/jmap/src/share_notification/get.rs b/crates/jmap/src/share_notification/get.rs index a9da1f24..801cb731 100644 --- a/crates/jmap/src/share_notification/get.rs +++ b/crates/jmap/src/share_notification/get.rs @@ -65,6 +65,7 @@ impl ShareNotificationGet for Server { let mut account_cache: AHashMap> = AHashMap::new(); + let has_ids = request.ids.is_some(); let mut ids = if let Some(ids) = request.ids.take() { let ids = ids.unwrap(); if ids.len() <= self.core.jmap.get_max_objects { @@ -86,7 +87,6 @@ impl ShareNotificationGet for Server { } else { AHashSet::new() }; - let has_ids = !ids.is_empty(); if min_id == u64::MAX { min_id = SnowflakeIdGenerator::from_duration( @@ -109,22 +109,27 @@ impl ShareNotificationGet for Server { not_found: vec![], }; let mut notifications = Vec::new(); + let params = IterateParams::new( + LogKey { + account_id, + collection: SyncCollection::ShareNotification.into(), + change_id: min_id, + }, + LogKey { + account_id, + collection: SyncCollection::ShareNotification.into(), + change_id: max_id.saturating_add(1), + }, + ) + .descending(); self.store() .iterate( - IterateParams::new( - LogKey { - account_id, - collection: SyncCollection::ShareNotification.into(), - change_id: min_id, - }, - LogKey { - account_id, - collection: SyncCollection::ShareNotification.into(), - change_id: max_id.saturating_add(1), - }, - ) - .descending(), + match (has_ids, ids.is_empty()) { + (false, _) => params.limit(self.core.jmap.get_max_objects.saturating_add(1)), + (true, true) => params.only_first(), + (true, false) => params, + }, |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if response.state.is_none() { @@ -139,11 +144,14 @@ impl ShareNotificationGet for Server { } Ok((!has_ids || !ids.is_empty()) - && notifications.len() < self.core.jmap.get_max_objects) + && notifications.len() <= self.core.jmap.get_max_objects) }, ) .await .caused_by(trc::location!())?; + if notifications.len() > self.core.jmap.get_max_objects { + return Err(trc::JmapEvent::RequestTooLarge.into_err()); + } for (change_id, notification) in notifications { let changed_by_account = @@ -252,7 +260,7 @@ fn map_rights( for right in CalendarRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), - Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), + Value::Bool(right.is_granted(&rights)), ); } } @@ -260,7 +268,7 @@ fn map_rights( for right in AddressBookRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), - Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), + Value::Bool(right.is_granted(&rights)), ); } } @@ -268,7 +276,7 @@ fn map_rights( for right in FileNodeRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), - Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), + Value::Bool(right.is_granted(&rights)), ); } } @@ -276,7 +284,7 @@ fn map_rights( for right in MailboxRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), - Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), + Value::Bool(right.is_granted(&rights)), ); } } diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 7abda923..dc2bda70 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -4,18 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::changes::state::StateManager; -use common::Server; +use crate::{ + api::metadata::{MetadataDocuments, MetadataType, ObjectMetadata, select_properties}, + changes::state::StateManager, +}; +use common::{Server, auth::AccessToken}; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::sieve::{Sieve, SieveProperty, SieveValue}, + request::capability::CapabilityIds, }; use jmap_tools::{Map, Value}; use std::future::Future; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -28,6 +32,8 @@ pub trait SieveScriptGet: Sync + Send { fn sieve_script_get( &self, request: GetRequest, + access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future>> + Send; } @@ -35,32 +41,48 @@ impl SieveScriptGet for Server { async fn sieve_script_get( &self, mut request: GetRequest, + access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result> { let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let properties = request.unwrap_properties(&[ - SieveProperty::Id, - SieveProperty::Name, - SieveProperty::BlobId, - SieveProperty::IsActive, - ]); + let (properties, selection) = select_properties( + &mut request, + &[ + SieveProperty::Id, + SieveProperty::Name, + SieveProperty::BlobId, + SieveProperty::IsActive, + ], + using, + )?; + let object_metadata = + ObjectMetadata::new(self, access_token, using, MetadataType::SieveScript); + let metadata = object_metadata.get(selection); + let metadata_slots = metadata.as_ref().map_or(0, |get| { + usize::from(get.wants_shared()) + usize::from(get.wants_private()) + }); + let mut documents = MetadataDocuments::default(); + let mut listed = Vec::new(); let account_id = request.account_id.document_id(); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let script_ids = self .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await?; let ids = if let Some(ids) = ids { ids } else { - script_ids - .iter() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect::>() + all_ids( + script_ids.iter().map(Into::into), + self.core.jmap.get_max_objects, + )? }; let mut response = GetResponse { account_id: request.account_id.into(), - state: self - .get_state(account_id, SyncCollection::SieveScript) - .await? + state: sampled + .state( + self.get_state(account_id, SyncCollection::SieveScript) + .await?, + ) .into(), list: Vec::with_capacity(ids.len()), not_found: not_found_ids, @@ -76,7 +98,7 @@ impl SieveScriptGet for Server { } let sieve_ = if let Some(sieve) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -91,7 +113,7 @@ impl SieveScriptGet for Server { let sieve = sieve_ .unarchive::() .caused_by(trc::location!())?; - let mut result = Map::with_capacity(properties.len()); + let mut result = Map::with_capacity(properties.len() + metadata_slots); for property in &properties { match property { SieveProperty::Id => { @@ -114,11 +136,7 @@ impl SieveScriptGet for Server { collection: Collection::SieveScript.into(), document_id, }, - section: BlobSection { - size: u32::from(sieve.size) as usize, - ..Default::default() - } - .into(), + section: BlobSection::new(0, u32::from(sieve.size) as usize, 0).into(), }; result.insert_unchecked( @@ -126,11 +144,29 @@ impl SieveScriptGet for Server { Value::Element(SieveValue::BlobId(blob_id)), ); } + SieveProperty::Metadata + | SieveProperty::PrivateMetadata + | SieveProperty::Pointer(_) => unreachable!(), } } + if metadata.is_some() { + documents.insert(document_id, sieve.metadata_kinds()); + listed.push(document_id); + } response.list.push(result.into()); } + if let Some(metadata) = &metadata { + let mut values = object_metadata + .load::(self, account_id, metadata, &documents) + .await?; + for (value, document_id) in response.list.iter_mut().zip(listed) { + if let Value::Object(object) = value { + values.insert_into(document_id, object); + } + } + } + Ok(response) } } diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index 3e319d05..1a029e7c 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -4,12 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{api::query::QueryResponseBuilder, changes::state::StateManager}; -use common::Server; +use crate::{ + api::{ + metadata::{MetadataType, ObjectMetadata}, + query::QueryResponseBuilder, + }, + changes::state::StateManager, +}; +use common::{Server, auth::AccessToken}; use email::sieve::ingest::SieveScriptIngest; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::sieve::{Sieve, SieveComparator, SieveFilter}, + request::capability::CapabilityIds, }; use std::future::Future; use store::{ @@ -28,6 +35,8 @@ pub trait SieveScriptQuery: Sync + Send { fn sieve_script_query( &self, request: QueryRequest, + access_token: &AccessToken, + using: CapabilityIds, ) -> impl Future> + Send; } @@ -35,8 +44,13 @@ impl SieveScriptQuery for Server { async fn sieve_script_query( &self, mut request: QueryRequest, + access_token: &AccessToken, + using: CapabilityIds, ) -> trc::Result { let account_id = request.account_id.document_id(); + let object_metadata = + ObjectMetadata::new(self, access_token, using, MetadataType::SieveScript); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let mut filters = Vec::with_capacity(request.filter.len()); let active_script_id = if request .filter @@ -87,6 +101,16 @@ impl SieveScriptQuery for Server { .await .caused_by(trc::location!())?; + let metadata = + object_metadata.query(request.filter.iter().filter_map(|filter| match filter { + Filter::Property(SieveFilter::Metadata(filter)) => Some(filter), + _ => None, + }))?; + let mut metadata_matches = object_metadata + .evaluate(self, account_id, &metadata, || &document_ids) + .await? + .into_iter(); + for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { @@ -118,6 +142,9 @@ impl SieveScriptQuery for Server { filters.push(SearchFilter::is_in_set(inactive_set)); } } + SieveFilter::Metadata(_) => filters.push(SearchFilter::is_in_set( + metadata_matches.next().unwrap_or_default(), + )), SieveFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter.into_err().details(other)); } @@ -169,8 +196,10 @@ impl SieveScriptQuery for Server { let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, - self.get_state(account_id, SyncCollection::SieveScript) - .await?, + sampled.state( + self.get_state(account_id, SyncCollection::SieveScript) + .await?, + ), &request, ); diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index c947aed1..231d9f62 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -4,11 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{blob::download::BlobDownload, changes::state::StateManager}; +use crate::{ + api::metadata::{ + MetadataAccess, MetadataPatches, MetadataTarget, MetadataType, MetadataWriter, + ObjectMetadata, PreparedMetadata, is_empty_update, reject_uncommitted, + }, + blob::download::BlobDownload, + changes::state::StateManager, +}; use common::{ Server, - auth::{AccessToken, AccountCache}, - storage::index::ObjectIndexBuilder, + auth::AccessToken, + ipc::CacheInvalidation, + storage::{ + index::{ObjectIndexBuilder, PresenceFlags, RewritePresence}, + quota::DiskQuotaBudget, + }, }; use email::sieve::{ ArchivedSieveScript, SieveScript, delete::SieveScriptDelete, ingest::SieveScriptIngest, @@ -19,7 +30,7 @@ use jmap_proto::{ method::set::{SetRequest, SetResponse}, object::sieve::{Sieve, SieveProperty, SieveValue}, references::resolve::ResolveCreatedReference, - request::{MaybeInvalid, reference::MaybeIdReference}, + request::{MaybeInvalid, capability::CapabilityIds, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{Key, Map, Value}; @@ -28,22 +39,39 @@ use registry::schema::enums::StorageQuota; use sieve::compiler::ErrorType; use std::future::Future; use store::{ - Serialize, SerializeInfallible, ValueKey, + ValueKey, rand::{RngExt, rng}, - write::{AlignedBytes, Archive, Archiver, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder, PendingId, Slot}, }; use trc::AddContext; use types::{ blob::{BlobClass, BlobId, BlobSection}, + blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::{PrincipalField, SieveField}, id::Id, + metadata::MetadataKinds, }; +#[derive(Debug, Clone, Copy)] +enum Deactivate { + No, + Always, + IfActive(u32), +} + +struct PendingSieveScript { + id: String, + slot: Slot, + blob_hash: BlobHash, + blob_size: usize, + is_active: bool, +} + pub struct SetContext<'x> { account_id: u32, access_token: &'x AccessToken, - account_cache: &'x AccountCache, + disk_quota: DiskQuotaBudget, response: SetResponse, } @@ -53,6 +81,7 @@ pub trait SieveScriptSet: Sync + Send { request: SetRequest<'_, Sieve>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> impl Future>> + Send; #[allow(clippy::type_complexity)] @@ -71,26 +100,37 @@ impl SieveScriptSet for Server { mut request: SetRequest<'_, Sieve>, access_token: &AccessToken, session: &HttpSessionData, + using: CapabilityIds, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let object_metadata = + ObjectMetadata::new(self, access_token, using, MetadataType::SieveScript); + let sampled = object_metadata.viewer_change_id(self, account_id).await?; let sieve_ids = self .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await?; let account = self.account(account_id).await.caused_by(trc::location!())?; + let shared_state = self + .get_state(account_id, SyncCollection::SieveScript) + .await?; let mut ctx = SetContext { account_id, access_token, - account_cache: &account, + disk_quota: if request.has_creates() || request.has_updates() { + self.disk_quota_budget(&account) + .await + .caused_by(trc::location!())? + } else { + DiskQuotaBudget::unlimited() + }, response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? - .with_state( - self.assert_state( - account_id, - SyncCollection::SieveScript, - &request.if_in_state, - ) - .await?, - ), + .with_state(sampled.assert_state(shared_state.clone(), &request.if_in_state)?), }; + let mut metadata_writer = + MetadataWriter::new(object_metadata, account_id).with_owner(account.clone()); + metadata_writer + .preload_updates(self, request.update.as_ref(), |_| Some(MetadataKinds::JMAP)) + .await?; let will_destroy = ctx.response.collect_will_destroy(request.unwrap_destroy()); // Validate active script id @@ -102,84 +142,102 @@ impl SieveScriptSet for Server { // Process creates let mut batch = BatchBuilder::new(); - let mut activations = Vec::new(); - for (id, object) in request.unwrap_create() { + let mut activations: Vec<(PendingId, bool)> = Vec::new(); + let mut activate_script = match &request.arguments.on_success_activate_script { + Some(MaybeIdReference::Id(id)) => Some(PendingId::Assigned(id.document_id())), + _ => None, + }; + let mut pending_creates: Vec = Vec::new(); + 'create: for (id, mut object) in request.unwrap_create() { if sieve_ids.len() < self.object_quota(account.object_quotas(), StorageQuota::MaxSieveScripts) as u64 { + let metadata_patches = + match object_metadata.extract(MetadataPatches::for_create(), &mut object) { + Ok(patches) => patches, + Err(err) => { + ctx.response.not_created.append(id, err); + continue 'create; + } + }; match self .sieve_set_item(object, None, &ctx, session.session_id) .await? { Ok(mut result) => { + let metadata_writes = match sieve_metadata_writes( + self, + &mut metadata_writer, + metadata_patches, + result.builder.changes_mut(), + MetadataTarget::Create, + ) + .await? + { + Ok(writes) => writes, + Err(err) => { + ctx.response.not_created.append(id, err); + continue 'create; + } + }; // Store blob let sieve = &mut result.builder.changes_mut().unwrap(); + let blob = result.blob_update.as_deref().ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Created script carries no blob.") + })?; let (blob_hash, blob_hold) = self - .put_temporary_blob( - account_id, - result.blob_update.as_ref().unwrap(), - 60, - ) + .hold_blob(account_id, BlobHash::generate(blob), blob, 60) .await?; sieve.blob_hash = blob_hash; let blob_size = sieve.size as usize; + ctx.disk_quota.reserve(blob_size as u64); let blob_hash = sieve.blob_hash.clone(); // Write record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::SieveScript, 1) - .await - .caused_by(trc::location!())?; + let slot = batch.reserve_document_id(account_id, Collection::SieveScript); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .with_document(document_id) + .create_document(slot) .custom( result .builder - .with_changed_by(ctx.access_token.account_tenant_ids()), + .with_changed_by(ctx.access_token.changed_by()), ) - .caused_by(trc::location!())? - .clear(blob_hold) - .commit_point(); + .caused_by(trc::location!())?; + blob_hold.release(&mut batch); + if let Some(prepared) = metadata_writes { + metadata_writer + .write(prepared, slot, &mut batch) + .caused_by(trc::location!())?; + } + batch.commit_point(); // Set isActive if needed if let Some(set_item) = result.set_item { - activations.push((document_id, set_item)); + activations.push((PendingId::Slot(slot), set_item)); } - let mut result = Map::with_capacity(1) - .with_key_value(SieveProperty::Id, SieveValue::Id(document_id.into())) - .with_key_value( - SieveProperty::BlobId, - SieveValue::BlobId(BlobId { - hash: blob_hash, - class: BlobClass::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }, - section: BlobSection { - size: blob_size, - ..Default::default() - } - .into(), - }), - ); - // Update active script if needed - if let Some(MaybeIdReference::Reference(id_ref)) = + let is_active = if let Some(MaybeIdReference::Reference(id_ref)) = &request.arguments.on_success_activate_script && id_ref == &id { - request.arguments.on_success_activate_script = - Some(MaybeIdReference::Id(Id::from(document_id))); - result.insert_unchecked(SieveProperty::IsActive, true); - } + activate_script = Some(PendingId::Slot(slot)); + true + } else { + false + }; - // Add result with updated blobId - ctx.response.created.insert(id, result.into()); + pending_creates.push(PendingSieveScript { + id, + slot, + blob_hash, + blob_size, + is_active, + }); } Err(err) => { ctx.response.not_created.append(id, err); @@ -197,7 +255,7 @@ impl SieveScriptSet for Server { } // Process updates - 'update: for (id, object) in request.unwrap_update() { + 'update: for (id, mut object) in request.unwrap_update() { let id = match id { MaybeInvalid::Value(id) => id, invalid => { @@ -214,12 +272,20 @@ impl SieveScriptSet for Server { .append(id, SetError::will_destroy()); continue 'update; } + let metadata_patches = + match object_metadata.extract(MetadataPatches::for_update(), &mut object) { + Ok(patches) => patches, + Err(err) => { + ctx.response.not_updated.append(id, err); + continue 'update; + } + }; // Obtain sieve script let document_id = id.document_id(); if let Some(sieve_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -229,6 +295,59 @@ impl SieveScriptSet for Server { let sieve = sieve_ .to_unarchived::() .caused_by(trc::location!())?; + let is_metadata_only = metadata_patches.is_some() + && object + .as_object() + .is_some_and(|object| is_empty_update::(object, id)); + let is_activated = matches!( + &request.arguments.on_success_activate_script, + Some(MaybeIdReference::Id(active_id)) if active_id.document_id() == document_id + ); + + if is_metadata_only { + let metadata_writes = match sieve_metadata_writes( + self, + &mut metadata_writer, + metadata_patches, + None, + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(writes) => writes, + Err(err) => { + ctx.response.not_updated.append(id, err); + continue 'update; + } + }; + if let Some(prepared) = metadata_writes { + if let Some(presence) = metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())? + { + SieveScript::rewrite_presence( + &sieve, + presence.after, + account_id, + document_id, + &mut batch, + ) + .caused_by(trc::location!())?; + } + batch.commit_point(); + } + let result = is_activated.then(|| { + Value::Object( + Map::with_capacity(1).with_key_value(SieveProperty::IsActive, true), + ) + }); + ctx.response.updated.append(id, result); + continue 'update; + } + if sieve.inner.name.eq_ignore_ascii_case("vacation") { + ctx.response.not_updated.append(id, vacation_forbidden()); + continue 'update; + } match self .sieve_set_item( @@ -240,6 +359,21 @@ impl SieveScriptSet for Server { .await? { Ok(mut result) => { + let metadata_writes = match sieve_metadata_writes( + self, + &mut metadata_writer, + metadata_patches, + result.builder.changes_mut(), + MetadataTarget::Update { document_id }, + ) + .await? + { + Ok(writes) => writes, + Err(err) => { + ctx.response.not_updated.append(id, err); + continue 'update; + } + }; // Prepare write batch batch .with_account_id(account_id) @@ -249,10 +383,12 @@ impl SieveScriptSet for Server { let blob_id = if let Some(blob) = result.blob_update.take() { // Store blob let sieve = &mut result.builder.changes_mut().unwrap(); - let (blob_hash, blob_hold) = - self.put_temporary_blob(account_id, &blob, 60).await?; + let (blob_hash, blob_hold) = self + .hold_blob(account_id, BlobHash::generate(&blob), &blob, 60) + .await?; sieve.blob_hash = blob_hash; - batch.clear(blob_hold); + blob_hold.release(&mut batch); + ctx.disk_quota.reserve(blob.len() as u64); BlobId { hash: sieve.blob_hash.clone(), @@ -261,11 +397,7 @@ impl SieveScriptSet for Server { collection: Collection::SieveScript.into(), document_id, }, - section: BlobSection { - size: sieve.size as usize, - ..Default::default() - } - .into(), + section: BlobSection::new(0, sieve.size as usize, 0).into(), } .into() } else { @@ -274,7 +406,7 @@ impl SieveScriptSet for Server { // Set isActive if needed if let Some(set_item) = result.set_item { - activations.push((document_id, set_item)); + activations.push((PendingId::Assigned(document_id), set_item)); } // Write record @@ -282,10 +414,15 @@ impl SieveScriptSet for Server { .custom( result .builder - .with_changed_by(ctx.access_token.account_tenant_ids()), + .with_changed_by(ctx.access_token.changed_by()), ) - .caused_by(trc::location!())? - .commit_point(); + .caused_by(trc::location!())?; + if let Some(prepared) = metadata_writes { + metadata_writer + .write(prepared, document_id, &mut batch) + .caused_by(trc::location!())?; + } + batch.commit_point(); // Update blobId property if needed let mut result = Map::with_capacity(1); @@ -297,10 +434,7 @@ impl SieveScriptSet for Server { } // Add active script property if needed - if let Some(MaybeIdReference::Id(id)) = - &request.arguments.on_success_activate_script - && document_id == id.document_id() - { + if is_activated { result.insert_unchecked(SieveProperty::IsActive, true); } @@ -351,54 +485,118 @@ impl SieveScriptSet for Server { } // Non-standard script activation handling - let mut on_success_deactivate_script = request + let mut deactivate = if request .arguments .on_success_deactivate_script - .unwrap_or(false); - if activations.len() == 1 { - let (document_id, set_item) = activations[0]; - let is_active = active_script_id.is_some_and(|active_id| active_id == document_id); + .unwrap_or(false) + { + Deactivate::Always + } else { + Deactivate::No + }; + if let [(document_id, set_item)] = activations[..] { + let active_id = document_id + .assigned() + .filter(|document_id| active_script_id == Some(*document_id)); if set_item { - if request.arguments.on_success_activate_script.is_none() && !is_active { - request.arguments.on_success_activate_script = - Some(MaybeIdReference::Id(document_id.into())); + if request.arguments.on_success_activate_script.is_none() && active_id.is_none() { + activate_script = Some(document_id); } - } else if !on_success_deactivate_script && is_active { - on_success_deactivate_script = true; + } else if let (Deactivate::No, Some(active_id)) = (deactivate, active_id) { + deactivate = Deactivate::IfActive(active_id); } } // Activate / deactivate scripts + let mut active_script_changed = false; if ctx.response.not_created.is_empty() && ctx.response.not_updated.is_empty() && ctx.response.not_destroyed.is_empty() - && (request.arguments.on_success_activate_script.is_some() - || on_success_deactivate_script) { - if let Some(MaybeIdReference::Id(id)) = request.arguments.on_success_activate_script { - batch - .with_account_id(account_id) - .with_collection(Collection::Principal) - .with_document(0) - .set(PrincipalField::ActiveScriptId, id.document_id().serialize()); - } else if on_success_deactivate_script { - batch - .with_account_id(account_id) - .with_collection(Collection::Principal) - .with_document(0) - .clear(PrincipalField::ActiveScriptId); + active_script_changed = + activate_script.is_some() || !matches!(deactivate, Deactivate::No); + match (activate_script, deactivate) { + (Some(document_id), _) => { + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .set(PrincipalField::ActiveScriptId, document_id); + } + (None, Deactivate::IfActive(document_id)) => { + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear_if_equals(PrincipalField::ActiveScriptId, document_id); + } + (None, Deactivate::Always) => { + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .with_document(0) + .clear(PrincipalField::ActiveScriptId); + } + (None, Deactivate::No) => {} } } // Write changes - if !batch.is_empty() - && let Ok(change_id) = self - .commit_batch(batch) - .await - .caused_by(trc::location!())? - .last_change_id(account_id) - { - ctx.response.new_state = State::Exact(change_id).into(); + let mut shared_change_id = None; + if !batch.is_empty() { + match metadata_writer.commit(self, batch).await { + Ok(assigned_ids) => { + shared_change_id = + assigned_ids.change_id(account_id, SyncCollection::SieveScript); + if active_script_changed { + self.invalidate_caches( + CacheInvalidation::ActiveSieveScript(account_id).into(), + ) + .await + .caused_by(trc::location!())?; + } + + for create in pending_creates { + let document_id = assigned_ids.slot(create.slot); + let mut result = Map::with_capacity(1) + .with_key_value(SieveProperty::Id, SieveValue::Id(document_id.into())) + .with_key_value( + SieveProperty::BlobId, + SieveValue::BlobId(BlobId { + hash: create.blob_hash, + class: BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + section: BlobSection::new(0, create.blob_size, 0).into(), + }), + ); + if create.is_active { + result.insert_unchecked(SieveProperty::IsActive, true); + } + ctx.response.created.insert(create.id, result.into()); + } + } + Err(err) if err.is_assertion_failure() => { + reject_uncommitted( + &mut ctx.response, + pending_creates.into_iter().map(|create| create.id), + "Another process modified this script, please try again.", + ); + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + } + + if shared_change_id.is_some() { + ctx.response.new_state = object_metadata + .viewer_change_id(self, account_id) + .await? + .state(shared_change_id.map_or(shared_state, State::Exact)) + .into(); } Ok(ctx.response) @@ -412,17 +610,6 @@ impl SieveScriptSet for Server { ctx: &SetContext<'_>, session_id: u64, ) -> trc::Result, SetError>> { - // Vacation script cannot be modified - if update - .as_ref() - .is_some_and(|(_, obj)| obj.inner.name.eq_ignore_ascii_case("vacation")) - { - return Ok(Err(SetError::forbidden().with_description(concat!( - "The 'vacation' script cannot be modified, ", - "use VacationResponse/set instead." - )))); - } - // Parse properties let mut set_item = None; let mut changes = update @@ -454,10 +641,15 @@ impl SieveScriptSet for Server { ctx.account_id, Collection::SieveScript, SieveField::Name, - value.as_bytes(), + value.to_lowercase().as_bytes(), ) .await? - .min() + .iter() + .find(|id| { + update + .as_ref() + .is_none_or(|(document_id, _)| id != document_id) + }) { return Ok(Err(SetError::already_exists() .with_existing_id(id.into()) @@ -518,31 +710,24 @@ impl SieveScriptSet for Server { !matches!(blob_id.class, BlobClass::Linked { account_id, collection, document_id: d } if account_id == ctx.account_id && collection == u8::from(Collection::SieveScript) && *document_id == d) }) { // Check access - if let Some(mut bytes) = self.blob_download(&blob_id, ctx.access_token).await? { + if let Some(bytes) = self.blob_download(&blob_id, ctx.access_token).await? { // Check quota - match self - .has_available_quota(ctx.account_cache, bytes.len() as u64) - .await - { - Ok(_) => (), - Err(err) => { - if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) - || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) - { - trc::error!(err.account_id(ctx.account_id).span_id(session_id)); - return Ok(Err(SetError::over_quota())); - } else { - return Err(err); - } - } + if let Err(event) = ctx.disk_quota.check(bytes.len() as u64) { + trc::error!( + event + .into_err() + .account_id(ctx.account_id) + .span_id(session_id) + ); + return Ok(Err(SetError::over_quota())); } // Compile script match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { changes.size = bytes.len() as u32; - bytes.extend(Archiver::new(script).untrusted().serialize().caused_by(trc::location!())?); - bytes.into() + changes.set_script(&script); + Some(bytes.into_bytes()) } Err(err) => { return Ok(Err(SetError::new( @@ -582,8 +767,39 @@ impl SieveScriptSet for Server { } } +async fn sieve_metadata_writes( + server: &Server, + writer: &mut MetadataWriter, + patches: Option, + script: Option<&mut SieveScript>, + target: MetadataTarget, +) -> trc::Result, SetError>> { + let Some(patches) = patches else { + return Ok(Ok(None)); + }; + let prepared = match writer + .prepare_for(server, patches, MetadataAccess::FULL, target) + .await? + { + Ok(prepared) if prepared.is_empty() => return Ok(Ok(None)), + Ok(prepared) => prepared, + Err(err) => return Ok(Err(err)), + }; + if let Some((kinds, script)) = prepared.shared_kinds().zip(script) { + script.set_metadata_kinds(kinds); + } + Ok(Ok(Some(prepared))) +} + pub struct SetItemResponse<'x> { - builder: ObjectIndexBuilder<&'x ArchivedSieveScript, SieveScript>, + builder: ObjectIndexBuilder, SieveScript>, blob_update: Option>, set_item: Option, } + +fn vacation_forbidden() -> SetError { + SetError::forbidden().with_description(concat!( + "The 'vacation' script cannot be modified, ", + "use VacationResponse/set instead." + )) +} diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index fff7f8a0..f430c551 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -7,8 +7,8 @@ use crate::changes::state::StateManager; use common::Server; use email::submission::{ - ArchivedAddress, ArchivedEnvelope, ArchivedUndoStatus, Delivered, DeliveryStatus, - EmailSubmission, + ArchivedAddress, ArchivedEnvelope, ArchivedUndoStatus, Delivered, DeliveryResults, + DeliveryStatus, EmailSubmission, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, @@ -16,15 +16,12 @@ use jmap_proto::{ types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; -use smtp::queue::{ArchivedError, ArchivedErrorDetails, ArchivedStatus, Message, spool::SmtpSpool}; -use smtp_proto::ArchivedResponse; +use smtp::queue::{Message, spool::SmtpSpool}; use std::future::Future; use store::{ IterateParams, U32_LEN, ValueKey, rkyv::option::ArchivedOption, - write::{ - AlignedBytes, Archive, IndexPropertyClass, ValueClass, key::DeserializeBigEndian, now, - }, + write::{Archive, ArchiveBytes, IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ @@ -39,6 +36,13 @@ pub trait EmailSubmissionGet: Sync + Send { &self, request: GetRequest, ) -> impl Future>> + Send; + + fn email_submission_delivery_results( + &self, + account_id: u32, + queue_id: u64, + delivery_status: &mut VecMap, + ) -> impl Future> + Send; } impl EmailSubmissionGet for Server { @@ -74,7 +78,7 @@ impl EmailSubmissionGet for Server { document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), - value: now() - (3 * 86400), + value: 0, }), }, ValueKey { @@ -88,15 +92,19 @@ impl EmailSubmissionGet for Server { }, ) .ascending() - .no_values(), + .no_values() + .limit(self.core.jmap.get_max_objects.saturating_add(1)), |key, _| { ids.push(Id::from(key.deserialize_be_u32(key.len() - U32_LEN)?)); - Ok(ids.len() < self.core.jmap.get_max_objects) + Ok(ids.len() <= self.core.jmap.get_max_objects) }, ) .await .caused_by(trc::location!())?; + if ids.len() > self.core.jmap.get_max_objects { + return Err(trc::JmapEvent::RequestTooLarge.into_err()); + } ids }; @@ -115,7 +123,7 @@ impl EmailSubmissionGet for Server { let document_id = id.document_id(); let submission_ = if let Some(submission) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, @@ -138,8 +146,18 @@ impl EmailSubmissionGet for Server { .map(|(k, v)| (k.to_string(), DeliveryStatus::from(v))) .collect::>(); let mut queued_status = None; - if let Some(queue_id) = submission.queue_id.as_ref().map(u64::from) { - if let Some(queued_message_) = self + if let Some(queue_id) = submission + .queue_id + .as_ref() + .filter(|_| !matches!(submission.undo_status, ArchivedUndoStatus::Canceled)) + .map(u64::from) + { + if self + .email_submission_delivery_results(account_id, queue_id, &mut delivery_status) + .await? + { + queued_status = email_submission::UndoStatus::Final.into(); + } else if let Some(queued_message_) = self .read_message_archive(queue_id) .await .caused_by(trc::location!())? @@ -149,28 +167,16 @@ impl EmailSubmissionGet for Server { .caused_by(trc::location!())?; for rcpt in queued_message.recipients.iter() { *delivery_status.get_mut_or_insert(rcpt.address().to_string()) = - DeliveryStatus { - smtp_reply: match &rcpt.status { - ArchivedStatus::Completed(reply) => { - format_archived_response(&reply.response) - } - ArchivedStatus::TemporaryFailure(reply) - | ArchivedStatus::PermanentFailure(reply) => { - format_archived_error_details(reply) - } - ArchivedStatus::Scheduled => "250 2.1.5 Queued".to_string(), - }, - delivered: match &rcpt.status { - ArchivedStatus::Scheduled - | ArchivedStatus::TemporaryFailure(_) => Delivered::Queued, - ArchivedStatus::Completed(_) => Delivered::Yes, - ArchivedStatus::PermanentFailure(_) => Delivered::No, - }, - displayed: false, - }; + rcpt.delivery_status(); } queued_status = email_submission::UndoStatus::Pending.into(); } else { + self.email_submission_delivery_results( + account_id, + queue_id, + &mut delivery_status, + ) + .await?; queued_status = email_submission::UndoStatus::Final.into(); } } @@ -258,6 +264,31 @@ impl EmailSubmissionGet for Server { Ok(response) } + + async fn email_submission_delivery_results( + &self, + account_id: u32, + queue_id: u64, + delivery_status: &mut VecMap, + ) -> trc::Result { + let Some(results) = self + .store() + .get_value::>(DeliveryResults::key(account_id, queue_id)) + .await? + else { + return Ok(false); + }; + for result in results + .unarchive::() + .caused_by(trc::location!())? + .0 + .iter() + { + *delivery_status.get_mut_or_insert(result.address.to_string()) = + DeliveryStatus::from(result); + } + Ok(true) + } } fn build_envelope( @@ -297,28 +328,3 @@ fn build_address( ) .into() } - -fn format_archived_response(response: &ArchivedResponse>) -> String { - format!( - "Code: {}, Enhanced code: {}.{}.{}, Message: {}", - response.code, - response.esc[0], - response.esc[1], - response.esc[2], - response.message.replace('\n', " "), - ) -} - -fn format_archived_error_details(response: &ArchivedErrorDetails) -> String { - match &response.details { - ArchivedError::UnexpectedResponse(response) => format_archived_response(&response.response), - ArchivedError::DnsError(details) - | ArchivedError::Io(details) - | ArchivedError::ConnectionError(details) - | ArchivedError::TlsError(details) - | ArchivedError::DaneError(details) - | ArchivedError::MtaStsError(details) => details.to_string(), - ArchivedError::RateLimited => "Rate limited".to_string(), - ArchivedError::ConcurrencyLimited => "Concurrency limited".to_string(), - } -} diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index 280a225e..73c6de42 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -18,9 +18,7 @@ use store::{ ahash::AHashSet, roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, - write::{ - IndexPropertyClass, QueueClass, SearchIndex, ValueClass, key::DeserializeBigEndian, now, - }, + write::{IndexPropertyClass, QueueClass, SearchIndex, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ @@ -64,7 +62,7 @@ impl EmailSubmissionQuery for Server { document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), - value: now() - (3 * 86400), + value: 0, }), }, ValueKey { @@ -87,12 +85,10 @@ impl EmailSubmissionQuery for Server { email_id: value.deserialize_be_u32(0)?, thread_id: value.deserialize_be_u32(U32_LEN)?, identity_id: value.deserialize_be_u32(U32_LEN + U32_LEN)?, - queue_id: if value.len() == (U32_LEN * 3) + U64_LEN + 1 { - value.deserialize_be_u64(U32_LEN * 3)? - } else { - 0 - }, - undo_status: value.last().copied().unwrap(), + queue_id: value.deserialize_be_u64(U32_LEN * 3)?, + undo_status: value.get(U32_LEN * 3 + U64_LEN).copied().ok_or_else( + || trc::Error::corrupted_key(key, value.into(), trc::location!()), + )?, }); document_ids.insert(document_id); diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index f6985158..6ba016c6 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -9,11 +9,15 @@ use common::{ config::smtp::queue::QueueName, network::{ServerInstance, stream::NullIo}, storage::index::ObjectIndexBuilder, + storage::quota::ObjectQuotaUsage, }; use email::{ identity::Identity, - message::metadata::{ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, MessageMetadata}, - submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, + message::metadata::{HeaderId, MetadataRow}, + submission::{ + Address, Delivered, DeliveryResults, DeliveryStatus, EmailSubmission, QUEUED_REPLY, + UndoStatus, + }, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -28,8 +32,10 @@ use jmap_proto::{ types::{date::UTCDate, state::State}, }; use jmap_tools::{Key, Map, Value}; +use registry::schema::enums::StorageQuota; use smtp::{ core::{Session, SessionData}, + inbound::data::MessageOrigin, queue::spool::SmtpSpool, }; use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; @@ -37,10 +43,14 @@ use std::{borrow::Cow, future::Future}; use std::{collections::HashMap, sync::Arc, time::Duration}; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder, now}, + write::{Archive, ArchiveBytes, BatchBuilder, now}, }; use trc::AddContext; -use types::{collection::Collection, field::EmailField, id::Id}; +use types::{ + collection::{Collection, SyncCollection}, + field::EmailField, + id::Id, +}; use utils::{map::vec_map::VecMap, sanitize_email}; pub trait EmailSubmissionSet: Sync + Send { @@ -73,11 +83,40 @@ impl EmailSubmissionSet for Server { let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = response.collect_will_destroy(request.unwrap_destroy()); + // Obtain submission quota + let limit = if request.has_creates() { + let account = self.account(account_id).await.caused_by(trc::location!())?; + self.object_quota_limit(&account, StorageQuota::MaxEmailSubmissions) + } else { + None + }; + let quota = match limit { + Some(limit) => ObjectQuotaUsage { + used: self + .object_count(account_id, StorageQuota::MaxEmailSubmissions) + .await?, + limit, + }, + None => ObjectQuotaUsage::unlimited(), + }; + // Process creates let mut success_email_ids = HashMap::new(); let mut changed_email_ids = HashMap::new(); let mut batch = BatchBuilder::new(); + let mut pending_creates = Vec::new(); for (id, object) in request.unwrap_create() { + if !quota.has_room(pending_creates.len()) { + response.not_created.append( + id, + SetError::new(SetErrorType::OverQuota).with_description(concat!( + "There are too many email submissions, ", + "please delete some before adding a new one." + )), + ); + continue; + } + match self .send_message(account_id, &response, instance, object) .await? @@ -97,39 +136,16 @@ impl EmailSubmissionSet for Server { }; // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::EmailSubmission, 1) - .await - .caused_by(trc::location!())?; + let slot = batch.reserve_document_id(account_id, Collection::EmailSubmission); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .with_document(document_id) + .create_document(slot) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(submission)) .caused_by(trc::location!())? .commit_point(); - response.created.insert( - id, - Value::Object( - Map::with_capacity(3) - .with_key_value( - EmailSubmissionProperty::Id, - Value::Element(Id::from(document_id).into()), - ) - .with_key_value( - EmailSubmissionProperty::SendAt, - Value::Element(EmailSubmissionValue::Date( - UTCDate::from_timestamp(send_at as i64), - )), - ) - .with_key_value( - EmailSubmissionProperty::UndoStatus, - Value::Element(EmailSubmissionValue::UndoStatus(undo_status)), - ), - ), - ); + pending_creates.push((id, slot, send_at, undo_status)); } Err(err) => { response.not_created.append(id, err); @@ -156,7 +172,7 @@ impl EmailSubmissionSet for Server { let document_id = id.document_id(); let submission = if let Some(submission) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, @@ -214,11 +230,24 @@ impl EmailSubmissionSet for Server { match undo_status { Some(email_submission::UndoStatus::Canceled) => { - if let Some(queue_message) = - self.read_message(queue_id, QueueName::default()).await + if let Some((queue_message, version)) = self + .read_message_versioned(queue_id, QueueName::default()) + .await { // Delete message from queue - queue_message.remove(self, None).await; + let prev_events = queue_message.message.next_events(); + if let Some(description) = queue_message + .remove_registry(self, prev_events, version) + .await + .failure("The message could not be removed from the queue.") + { + response.not_updated.append( + id, + SetError::new(SetErrorType::CannotUnsend) + .with_description(description), + ); + continue 'update; + } // Update record let mut new_submission = submission.inner.clone(); @@ -270,7 +299,7 @@ impl EmailSubmissionSet for Server { let document_id = id.document_id(); if let Some(submission) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, @@ -285,13 +314,17 @@ impl EmailSubmissionSet for Server { u32::from(submission.inner.thread_id), u32::from(submission.inner.email_id), ); + let queue_id = submission.inner.queue_id.as_ref().map(u64::from); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .with_document(document_id) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(submission)) - .caused_by(trc::location!())? - .commit_point(); + .caused_by(trc::location!())?; + if let Some(queue_id) = queue_id { + DeliveryResults::clear(&mut batch, account_id, queue_id); + } + batch.commit_point(); changed_email_ids.insert(id, email_id); response.destroyed.push(id); } else { @@ -301,12 +334,34 @@ impl EmailSubmissionSet for Server { // Write changes if !batch.is_empty() { - let change_id = self - .commit_batch(batch) - .await - .and_then(|ids| ids.last_change_id(account_id)) - .caused_by(trc::location!())?; - response.new_state = State::Exact(change_id).into(); + let assigned_ids = self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = State::Exact( + assigned_ids.last_change_id(account_id, SyncCollection::EmailSubmission), + ) + .into(); + + for (id, slot, send_at, undo_status) in pending_creates { + response.created.insert( + id, + Value::Object( + Map::with_capacity(3) + .with_key_value( + EmailSubmissionProperty::Id, + Value::Element(Id::from(assigned_ids.slot(slot)).into()), + ) + .with_key_value( + EmailSubmissionProperty::SendAt, + Value::Element(EmailSubmissionValue::Date( + UTCDate::from_timestamp(send_at as i64), + )), + ) + .with_key_value( + EmailSubmissionProperty::UndoStatus, + Value::Element(EmailSubmissionValue::UndoStatus(undo_status)), + ), + ), + ); + } } // On success @@ -508,7 +563,7 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Identity, submission.identity_id, @@ -547,83 +602,49 @@ impl EmailSubmissionSet for Server { }; // Obtain message metadata - let metadata_ = if let Some(metadata) = self + let Some(row) = self .store() - .get_value::>(ValueKey::property( + .get_value::(ValueKey::immutable( account_id, Collection::Email, submission.email_id, EmailField::Metadata, )) .await? - { - metadata - } else { + else { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::EmailId) .with_description("Email not found."))); }; - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; + let metadata = row.unarchive().caused_by(trc::location!())?; // Add recipients to envelope if missing - let mut bcc_header = None; if rcpt_to.is_empty() { - for header in metadata.contents[0].parts[0].headers.iter() { - if matches!( - header.name, - ArchivedMetadataHeaderName::To - | ArchivedMetadataHeaderName::Cc - | ArchivedMetadataHeaderName::Bcc - ) { - if matches!(header.name, ArchivedMetadataHeaderName::Bcc) { - bcc_header = Some(header); - } - match &header.value { - ArchivedMetadataHeaderValue::AddressList(addr) => { - for address in addr.iter() { - if let Some(address) = address - .address - .as_ref() - .map(|v| v.as_ref()) - .and_then(sanitize_email) - && !rcpt_to.iter().any(|rcpt| rcpt.address == address) - { - submission.envelope.rcpt_to.push(Address { - email: address.to_string(), - parameters: None, - }); - rcpt_to.push(RcptTo { - address: Cow::Owned(address), - ..Default::default() - }); - } - } - } - ArchivedMetadataHeaderValue::AddressGroup(groups) => { - for group in groups.iter() { - for address in group.addresses.iter() { - if let Some(address) = address - .address - .as_ref() - .map(|v| v.as_ref()) - .and_then(sanitize_email) - && !rcpt_to.iter().any(|rcpt| rcpt.address == address) - { - submission.envelope.rcpt_to.push(Address { - email: address.to_string(), - parameters: None, - }); - rcpt_to.push(RcptTo { - address: Cow::Owned(address), - ..Default::default() - }); - } - } - } - } - _ => {} + let headers = row.raw_headers().caused_by(trc::location!())?; + for header in metadata.root().root_part().headers().iter() { + if !matches!(header.id(), HeaderId::TO | HeaderId::CC | HeaderId::BCC) { + continue; + } + let Some(raw) = header.raw_value(&headers) else { + continue; + }; + let parsed = mail_parser::HeaderForm::Addresses.parse(raw); + let Some(list) = parsed.value().as_address() else { + continue; + }; + for address in list + .mailboxes() + .filter_map(|mailbox| mailbox.address().and_then(sanitize_email)) + { + if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { + submission.envelope.rcpt_to.push(Address { + email: address.to_string(), + parameters: None, + }); + rcpt_to.push(RcptTo { + address: Cow::Owned(address), + ..Default::default() + }); } } } @@ -632,11 +653,6 @@ impl EmailSubmissionSet for Server { return Ok(Err(SetError::new(SetErrorType::NoRecipients) .with_description("No recipients found in email."))); } - } else { - bcc_header = metadata.contents[0].parts[0] - .headers - .iter() - .find(|header| matches!(header.name, ArchivedMetadataHeaderName::Bcc)); } // Update sendAt @@ -649,36 +665,36 @@ impl EmailSubmissionSet for Server { }; // Obtain raw message - let mut message = if let Some(message) = self - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) + let Some(blob) = self + .fetch_blob( + &metadata.blob_hash(), + 0..usize::MAX, + Some(metadata.size() as u64), + ) .await? - { - if message.len() > self.core.email.mail_max_size { - return Ok(Err(SetError::new(SetErrorType::InvalidEmail) - .with_description(format!( - "Message exceeds maximum size of {} bytes.", - self.core.email.mail_max_size - )))); - } - - message - } else { + else { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::EmailId) .with_description("Blob for email not found."))); }; - - // Remove BCC header if present - if let Some(bcc_header) = bcc_header { - let mut new_message = Vec::with_capacity(message.len()); - let range = bcc_header.name_value_range(); - new_message.extend_from_slice(&message[..range.start]); - new_message.extend_from_slice(&message[range.end..]); - message = new_message; + if blob.len() > self.core.email.mail_max_size { + return Ok(Err(SetError::new(SetErrorType::InvalidEmail) + .with_description(format!( + "Message exceeds maximum size of {} bytes.", + self.core.email.mail_max_size + )))); } + // Remove BCC header if present + let stripped = match metadata.strip_root_fields(&blob, HeaderId::BCC) { + Cow::Owned(stripped) => Some(stripped), + Cow::Borrowed(_) => None, + }; + let (raw_message, message_memory) = blob.into_parts(); + let message = stripped.unwrap_or(raw_message); + // Begin local SMTP session + let origin = MessageOrigin::Submission { account_id }; let mut session = Session::::local( self.clone(), instance.clone(), @@ -719,7 +735,8 @@ impl EmailSubmissionSet for Server { // DATA if has_success { session.data.message = message; - let response = session.queue_message().await; + session.data.message_memory = Some(message_memory); + let response = session.queue_message(origin).await; if let smtp::core::State::Accepted(queue_id) = session.state { Ok((responses, Some(queue_id))) } else { @@ -761,7 +778,7 @@ impl EmailSubmissionSet for Server { }, smtp_reply: response .map(|s| s.to_string()) - .unwrap_or_else(|| "250 2.1.5 Queued".to_string()), + .unwrap_or_else(|| QUEUED_REPLY.to_string()), displayed: false, }, ) diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 654b1b1a..a860c9a3 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -6,20 +6,15 @@ use crate::changes::state::StateManager; use common::{Server, auth::AccessToken}; -use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; +use email::cache::{MessageCacheFetch, email::MessageAccess, mailbox::MailboxCacheAccess}; use jmap_proto::{ - method::get::{GetRequest, GetResponse}, + method::get::{GetRequest, GetResponse, all_ids}, object::thread::{Thread, ThreadProperty, ThreadValue}, request::MaybeInvalid, }; use jmap_tools::Map; use std::future::Future; -use store::{ - ahash::AHashMap, - roaring::RoaringBitmap, - search::{EmailSearchField, SearchComparator, SearchField, SearchQuery}, - write::SearchIndex, -}; +use store::ahash::AHashMap; use trc::AddContext; use types::{acl::Acl, collection::SyncCollection, id::Id}; @@ -42,37 +37,41 @@ impl ThreadGet for Server { .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - let shared_ids = if access_token.is_shared(account_id) { - Some(cache.shared_messages(access_token, Acl::ReadItems)) + let access = if access_token.is_shared(account_id) { + MessageAccess::Mailboxes(cache.shared_mailboxes(access_token, Acl::ReadItems)) } else { - None + MessageAccess::All }; - let mut thread_map: AHashMap = AHashMap::with_capacity(32); - let mut all_ids = RoaringBitmap::new(); - for item in &cache.emails.items { - if shared_ids - .as_ref() - .is_some_and(|ids| !ids.contains(item.document_id)) - { - continue; - } - thread_map - .entry(item.thread_id) - .or_default() - .insert(item.document_id); - all_ids.insert(item.document_id); - } let (ids, not_found_ids) = request.unwrap_ids(self.core.jmap.get_max_objects)?; + let mut thread_map: AHashMap>; let ids = if let Some(ids) = ids { + thread_map = ids + .iter() + .map(|id| (id.document_id(), Vec::new())) + .collect(); + if !access.is_empty() { + for item in cache.emails.iter() { + if let Some(document_ids) = thread_map.get_mut(&item.thread_id()) + && access.allows(item) + { + document_ids.push(item.document_id()); + } + } + } ids } else { - thread_map - .keys() - .copied() - .take(self.core.jmap.get_max_objects) - .map(Into::into) - .collect() + thread_map = AHashMap::with_capacity(32); + for item in cache.emails.iter().filter(|item| access.allows(*item)) { + thread_map + .entry(item.thread_id()) + .or_default() + .push(item.document_id()); + } + all_ids( + thread_map.keys().copied().map(Into::into), + self.core.jmap.get_max_objects, + )? }; let add_email_ids = request.properties.is_none_or(|p| { p.unwrap() @@ -88,41 +87,22 @@ impl ThreadGet for Server { not_found: not_found_ids, }; - let ordered_ids = if add_email_ids && !all_ids.is_empty() { - Some( - self.search_store() - .query_account( - SearchQuery::new(SearchIndex::Email) - .with_account_id(account_id) - .with_mask(all_ids) - .with_comparator(SearchComparator::Field { - field: SearchField::Email(EmailSearchField::ReceivedAt), - ascending: true, - }), - ) - .await?, - ) - } else { - None - }; - for id in ids { let thread_id = id.document_id(); - if let Some(mut document_ids) = thread_map.remove(&thread_id) { + if let Some(document_ids) = thread_map + .remove(&thread_id) + .filter(|document_ids| !document_ids.is_empty()) + { let mut thread: Map<'_, ThreadProperty, ThreadValue> = Map::with_capacity(2).with_key_value(ThreadProperty::Id, id); - if let Some(ordered_ids) = &ordered_ids { - let mut ids = Vec::with_capacity(document_ids.len() as usize); - for &id in ordered_ids.iter() { - if document_ids.remove(id) { - ids.push(Id::from_parts(thread_id, id)); - } - } - for id in document_ids.iter() { - ids.push(Id::from_parts(thread_id, id)); - } - - thread.insert_unchecked(ThreadProperty::EmailIds, ids); + if add_email_ids { + thread.insert_unchecked( + ThreadProperty::EmailIds, + document_ids + .into_iter() + .map(|document_id| Id::from_parts(thread_id, document_id)) + .collect::>(), + ); } response.list.push(thread.into()); } else { diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index ab2b8ef0..8a095613 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -22,7 +22,7 @@ use jmap_tools::{Map, Value}; use std::future::Future; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{ @@ -92,7 +92,7 @@ impl VacationResponseGet for Server { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? && let Some(sieve_) = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 44524906..f4e45f4f 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -6,7 +6,9 @@ use super::get::VacationResponseGet; use crate::changes::state::StateManager; -use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; +use common::{ + Server, auth::AccessToken, ipc::CacheInvalidation, storage::index::ObjectIndexBuilder, +}; use email::sieve::{SieveScript, VacationResponse, ingest::SieveScriptIngest}; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -22,11 +24,12 @@ use mail_parser::decoders::html::html_to_text; use std::borrow::Cow; use std::future::Future; use store::{ - Serialize, SerializeInfallible, ValueKey, - write::{AlignedBytes, Archive, Archiver, BatchBuilder}, + ValueKey, + write::{Archive, ArchiveBytes, BatchBuilder, PendingId}, }; use trc::AddContext; use types::{ + blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::PrincipalField, id::Id, @@ -109,7 +112,7 @@ impl VacationResponseSet for Server { let (mut sieve, prev_sieve) = if let Some(document_id) = document_id { let prev_sieve = self .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -132,9 +135,8 @@ impl VacationResponseSet for Server { ( SieveScript { name: "vacation".into(), - blob_hash: Default::default(), - size: 0, vacation_response: VacationResponse::default().into(), + ..Default::default() }, None, ) @@ -247,51 +249,52 @@ impl VacationResponseSet for Server { let mut obj = ObjectIndexBuilder::new() .with_current_opt(prev_sieve) .with_changes(sieve) - .with_changed_by(access_token.account_tenant_ids()); + .with_changed_by(access_token.changed_by()); // Update id - let document_id = if let Some(document_id) = document_id { - batch.with_document(document_id); - document_id - } else { - let document_id = self - .store() - .assign_document_ids(account_id, Collection::SieveScript, 1) - .await - .caused_by(trc::location!())?; - batch.with_document(document_id); - document_id + let document_id = match document_id { + Some(document_id) => { + batch.with_document(document_id); + PendingId::Assigned(document_id) + } + None => { + let slot = batch.reserve_document_id(account_id, Collection::SieveScript); + batch.create_document(slot); + PendingId::Slot(slot) + } }; // Create sieve script only if there are changes if build_script { // Upload new blob + let script = self.build_script(obj.changes_mut().unwrap())?; let (blob_hash, blob_hold) = self - .put_temporary_blob( - account_id, - &self.build_script(obj.changes_mut().unwrap())?, - 60, - ) + .hold_blob(account_id, BlobHash::generate(&script), &script, 60) .await?; obj.changes_mut().unwrap().blob_hash = blob_hash; - batch.clear(blob_hold); + blob_hold.release(&mut batch); }; batch.custom(obj).caused_by(trc::location!())?; // Deactivate other sieve scripts - let was_active = active_script_id == Some(document_id); + let was_active = document_id + .assigned() + .filter(|document_id| active_script_id == Some(*document_id)); + let mut active_script_changed = false; if is_active { - if !was_active { + if was_active.is_none() { batch .with_collection(Collection::Principal) - .with_document(0) - .set(PrincipalField::ActiveScriptId, document_id.serialize()); + .with_document(0); + batch.set(PrincipalField::ActiveScriptId, document_id); + active_script_changed = true; } - } else if was_active { + } else if let Some(active_id) = was_active { batch .with_collection(Collection::Principal) .with_document(0) - .clear(PrincipalField::ActiveScriptId); + .clear_if_equals(PrincipalField::ActiveScriptId, active_id); + active_script_changed = true; } // Write changes @@ -299,9 +302,14 @@ impl VacationResponseSet for Server { response.new_state = Some(State::Exact( self.commit_batch(batch) .await - .and_then(|ids| ids.last_change_id(account_id)) + .map(|ids| ids.last_change_id(account_id, SyncCollection::SieveScript)) .caused_by(trc::location!())?, )); + if active_script_changed { + self.invalidate_caches(CacheInvalidation::ActiveSieveScript(account_id).into()) + .await + .caused_by(trc::location!())?; + } } // Add result @@ -421,16 +429,8 @@ impl VacationResponseSet for Server { match self.core.sieve.untrusted_compiler.compile(&script) { Ok(compiled_script) => { - // Update blob length obj.size = script.len() as u32; - - // Serialize script - script.extend( - Archiver::new(compiled_script) - .untrusted() - .serialize() - .caused_by(trc::location!())?, - ); + obj.set_script(&compiled_script); Ok(script) } diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 234de7d7..a1dc19a7 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -4,8 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::api::{ToRequestError, notifications_into_push_objects, request::RequestHandler}; -use common::{Server, auth::AccessToken, ipc::PushNotification}; +use crate::api::{ + ToRequestError, notifications_into_push_objects, request::RequestHandler, + session::allowed_capabilities, +}; +use common::{ + BuildServer, Server, + auth::AccessToken, + cache::http_auth::RevocationEpoch, + ipc::{PushNotification, ViewerStateChange}, + network::memory::MemoryClass, +}; +use compact_str::ToCompactString; use futures_util::{SinkExt, StreamExt}; use http_proto::HttpSessionData; use hyper::upgrade::Upgraded; @@ -20,7 +30,10 @@ use std::future::Future; use std::time::Instant; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; -use tungstenite::Message; +use tungstenite::{ + Message, + protocol::{CloseFrame, frame::coding::CloseCode}, +}; use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; @@ -29,6 +42,7 @@ pub trait WebSocketHandler: Sync + Send { &self, stream: WebSocketStream>, access_token: AccessToken, + epoch: RevocationEpoch, session: HttpSessionData, ) -> impl Future + Send; } @@ -37,7 +51,8 @@ impl WebSocketHandler for Server { async fn handle_websocket_stream( &self, mut stream: WebSocketStream>, - access_token: AccessToken, + mut access_token: AccessToken, + epoch: RevocationEpoch, session: HttpSessionData, ) { trc::event!( @@ -56,29 +71,13 @@ impl WebSocketHandler for Server { let mut next_event = heartbeat; // Register with push manager - let mut push_rx = match self - .subscribe_push_manager(&access_token, Bitmap::all()) - .await - { - Ok(push_rx) => push_rx, - Err(err) => { - trc::error!( - err.details("Failed to subscribe to push manager") - .span_id(session.session_id) - ); - - let _ = stream - .send(Message::Text( - WebSocketRequestError::from(RequestError::internal_server_error()) - .to_json() - .into(), - )) - .await; - return; - } - }; + let mut allowed_types = Bitmap::all(); + access_token.restrict_data_types(&mut allowed_types); + let mut push_rx = self.subscribe_push_manager(&access_token, allowed_types); + let mut revocation_epoch = epoch; let mut notifications = Vec::new(); + let mut received = Vec::new(); let mut change_types: Bitmap = Bitmap::new(); loop { @@ -93,16 +92,82 @@ impl WebSocketHandler for Server { self.core.jmap.request_max_calls, self.core.jmap.request_max_size, ) { - Ok(WebSocketMessage::Request(request)) => { - let response = self - .handle_jmap_request( - request.request, - &access_token, - &session, - ) - .await; - WebSocketResponse::from_response(response, request.id) - .to_json() + Ok(WebSocketMessage::Request(mut request)) => { + let server = self.inner.build_server(); + let renewal = match access_token + .grant_id() + .map(|_| server.inner.cache.grant_revocations.epoch()) + { + Some(epoch) if epoch != revocation_epoch => server + .renew_access_token(&access_token, session.remote_ip) + .await + .map(|renewed| { + revocation_epoch = epoch; + Some(renewed) + }), + _ => server.refresh_access_token(&access_token, session.remote_ip).await, + }; + let renewal = match renewal { + Ok(renewed) => { + if let Some(renewed) = renewed { + access_token = renewed; + } + Ok(()) + } + Err(err) if matches!(err.as_ref(), trc::EventType::Auth(_) | trc::EventType::Security(_)) => { + trc::event!( + Jmap(JmapEvent::WebsocketStop), + SpanId = session.session_id, + Reason = "Authorization revoked" + ); + let _ = stream.close(Some(CloseFrame { + code: CloseCode::Policy, + reason: "Authorization revoked".into(), + })).await; + break; + } + Err(err) => Err(err), + }; + match renewal { + Ok(()) => match server + .is_http_authenticated_request_allowed(&access_token, session.remote_ip) + .await + { + Ok(_in_flight) => { + request.request.using = allowed_capabilities(&access_token, request.request.using); + let mut reservation = server.memory_reservation(MemoryClass::Read); + let response = server + .handle_jmap_request( + request.request, + &access_token, + &session, + &mut reservation, + ) + .await; + match WebSocketResponse::from_response(response, request.id).to_json() { + Ok(response) => response, + Err(err) => { + trc::event!( + Jmap(JmapEvent::WebsocketError), + Details = "Failed to serialize response", + SpanId = session.session_id, + Reason = err.to_compact_string() + ); + WebSocketRequestError::from(RequestError::internal_server_error()).to_json() + } + } + } + Err(err) => { + let response = WebSocketRequestError::from_error(err.to_request_error(), request.id).to_json(); + trc::error!(err.span_id(session.session_id)); + response + } + }, + Err(err) => { + trc::error!(err.details("Failed to renew WebSocket access token").span_id(session.session_id)); + WebSocketRequestError::from_error(RequestError::internal_server_error(), request.id).to_json() + } + } } Ok(WebSocketMessage::PushEnable(push_enable)) => { change_types = if !push_enable.data_types.is_empty() { @@ -110,6 +175,7 @@ impl WebSocketHandler for Server { } else { Bitmap::all() }; + change_types.intersection(&allowed_types); continue; } Ok(WebSocketMessage::PushDisable) => { @@ -126,7 +192,7 @@ impl WebSocketHandler for Server { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Failed to send text message", SpanId = session.session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); } } @@ -135,7 +201,7 @@ impl WebSocketHandler for Server { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Failed to send pong message", SpanId = session.session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); } } @@ -153,7 +219,7 @@ impl WebSocketHandler for Server { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Websocket error", SpanId = session.session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); break; } @@ -172,47 +238,50 @@ impl WebSocketHandler for Server { } } } - push_notification = push_rx.recv() => { - if let Some(push_notification) = push_notification { - match push_notification { - PushNotification::StateChange(state_change) => { - let mut types = state_change.types; - types.intersection(&change_types); + is_open = push_rx.recv(&mut received) => { + if is_open { + for push_notification in received.drain(..) { + match push_notification { + PushNotification::StateChange(state_change) + | PushNotification::ViewerStateChange(ViewerStateChange { + change: state_change, + .. + }) => { + let mut types = state_change.types; + types.intersection(&change_types); - if !types.is_empty() { - notifications.push(PushNotification::StateChange( - StateChange { - account_id: state_change.account_id, - types, - change_id: state_change.change_id, - } - )); - } - }, - PushNotification::EmailPush(email_push) => { - let state_change = email_push.to_state_change(); - let mut types = state_change.types; - types.intersection(&change_types); + if !types.is_empty() { + notifications.push(PushNotification::StateChange( + StateChange { + types, + ..state_change + } + )); + } + }, + PushNotification::EmailPush(email_push) => { + let state_change = email_push.to_state_change(); + let mut types = state_change.types; + types.intersection(&change_types); - if !types.is_empty() { - notifications.push(PushNotification::StateChange( - StateChange { - account_id: state_change.account_id, - types, - change_id: state_change.change_id, - } - )); - } - }, - PushNotification::CalendarAlert(calendar_alert) => { - if change_types.contains(DataType::CalendarAlert) { - notifications.push(PushNotification::CalendarAlert( - calendar_alert - )); - } - }, + if !types.is_empty() { + notifications.push(PushNotification::StateChange( + StateChange { + types, + ..state_change + } + )); + } + }, + PushNotification::CalendarAlert(calendar_alert) => { + if change_types.contains(DataType::CalendarAlert) { + notifications.push(PushNotification::CalendarAlert( + calendar_alert + )); + } + }, + } } - } else { trc::event!( Jmap(JmapEvent::WebsocketStop), @@ -241,7 +310,7 @@ impl WebSocketHandler for Server { Jmap(JmapEvent::WebsocketError), Details = "Failed to send state change message.", SpanId = session.session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); } } @@ -257,7 +326,7 @@ impl WebSocketHandler for Server { Jmap(JmapEvent::WebsocketError), Details = "Failed to send ping message.", SpanId = session.session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); break; } diff --git a/crates/jmap/src/websocket/upgrade.rs b/crates/jmap/src/websocket/upgrade.rs index 945b58e0..23300700 100644 --- a/crates/jmap/src/websocket/upgrade.rs +++ b/crates/jmap/src/websocket/upgrade.rs @@ -5,7 +5,8 @@ */ use super::stream::WebSocketHandler; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, cache::http_auth::RevocationEpoch}; +use compact_str::ToCompactString; use http_proto::*; use hyper::StatusCode; use hyper_util::rt::TokioIo; @@ -19,6 +20,7 @@ pub trait WebSocketUpgrade: Sync + Send { &self, req: HttpRequest, access_token: AccessToken, + epoch: RevocationEpoch, session: HttpSessionData, ) -> impl Future> + Send; } @@ -28,6 +30,7 @@ impl WebSocketUpgrade for Server { &self, req: HttpRequest, access_token: AccessToken, + epoch: RevocationEpoch, session: HttpSessionData, ) -> trc::Result { let headers = req.headers(); @@ -79,17 +82,16 @@ impl WebSocketUpgrade for Server { let session_id = session.session_id; match hyper::upgrade::on(req).await { Ok(upgraded) => { - Box::pin( - jmap.handle_websocket_stream( - WebSocketStream::from_raw_socket( - TokioIo::new(upgraded), - Role::Server, - None, - ) - .await, - access_token, - session, - ), + jmap.handle_websocket_stream( + WebSocketStream::from_raw_socket( + TokioIo::new(upgraded), + Role::Server, + None, + ) + .await, + access_token, + epoch, + session, ) .await; } @@ -98,7 +100,7 @@ impl WebSocketUpgrade for Server { Jmap(JmapEvent::WebsocketError), Details = "Websocket upgrade failed", SpanId = session_id, - Reason = err.to_string() + Reason = err.to_compact_string() ); } } diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 9d948f03..0ddd3f13 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "webdav", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.16.27" +version = "1.0.0" edition = "2024" [[bin]] @@ -20,7 +20,7 @@ coordinator = { path = "../coordinator" } jmap = { path = "../jmap" } types = { path = "../types" } smtp = { path = "../smtp" } -smtp-proto = { version = "0.2", features = ["rkyv", "serde"] } +smtp-proto = { path = "../smtp-proto", features = ["rkyv", "serde"] } imap = { path = "../imap" } pop3 = { path = "../pop3" } spam-filter = { path = "../spam-filter" } @@ -37,16 +37,18 @@ trc = { path = "../trc" } utils = { path = "../utils" } registry = { path = "../registry" } http_proto = { path = "../http-proto" } -migration = { path = "../migration" } -tokio = { version = "1.53", features = ["full"] } -rustls = { version = "0.23.43", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +tokio = { version = "1.54", features = ["full"] } +rustls = { version = "0.23.46", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } [target.'cfg(all(target_arch = "x86_64", any(target_os = "linux", target_os = "freebsd")))'.dependencies] -libc = "0.2.189" +libc = "0.2.191" [target.'cfg(not(any(target_env = "msvc", target_os = "freebsd")))'.dependencies] tikv-jemallocator = "0.7" +[target.'cfg(all(target_os = "linux", target_env = "gnu"))'.dependencies] +tikv-jemallocator = { version = "0.7", features = ["unprefixed_malloc_on_supported_platforms"] } + [features] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "azure", "nats", "enterprise", "zenoh", "kafka"] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "azure", "nats", "enterprise"] @@ -57,7 +59,7 @@ postgres = ["store/postgres", "directory/postgres"] mysql = ["store/mysql", "directory/mysql"] rocks = ["store/rocks"] s3 = ["store/s3"] -redis = ["store/redis", "coordinator/redis"] +redis = ["store/redis", "coordinator/redis", "common/redis"] azure = ["store/azure"] nats = ["coordinator/nats"] zenoh = ["coordinator/zenoh"] @@ -75,8 +77,13 @@ enterprise = [ "jmap/enterprise", "dav/enterprise", "groupware/enterprise", "trc/enterprise", + "registry/enterprise", "services/enterprise", - "migration/enterprise" ] + "coordinator/enterprise", + "imap/enterprise", + "pop3/enterprise", + "utils/enterprise", + "http_proto/enterprise" ] dev_mode = [ "common/dev_mode", "trc/dev_mode", "dav/dev_mode", diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index 92f792ed..e9acb5be 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -18,9 +18,12 @@ use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use services::{StartServices, broadcast::subscriber::spawn_broadcast_subscriber}; use smtp::{StartQueueManager, core::SmtpSessionManager}; -use std::time::Duration; +use std::{process::ExitCode, time::Duration}; use trc::Collector; -use utils::wait_for_shutdown; +use utils::{ + runtime::{exit, run_server}, + wait_for_shutdown, +}; #[cfg(all( target_arch = "x86_64", @@ -38,8 +41,11 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -#[tokio::main] -async fn main() -> std::io::Result<()> { +fn main() -> std::io::Result { + run_server(async_main()) +} + +async fn async_main() -> ExitCode { // Install AWS-LC-RS as the default Rustls crypto provider rustls::crypto::aws_lc_rs::default_provider() .install_default() @@ -48,17 +54,21 @@ async fn main() -> std::io::Result<()> { // Build the shared outbound TLS configurations utils::http::init_shared_tls_configs(); - // Load config and apply macros - let mut init = Box::pin(BootManager::init()).await; + // Prepare the archive compression dictionaries + store::write::compress::init(); - // Migrate database - if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { + // Load config and apply macros + let mut init = BootManager::init().await; + + // Verify database schema version + if let Err(err) = init.inner.build_server().verify_schema_version().await { + eprintln!("{err}"); trc::event!( Server(trc::ServerEvent::StartupError), - Details = "Failed to migrate database, aborting startup.", + Details = "Unsupported database schema version, aborting startup.", Reason = err, ); - return Ok(()); + return ExitCode::FAILURE; } // Init services @@ -74,6 +84,7 @@ async fn main() -> std::io::Result<()> { if std::env::var("INSERT_TEST_DATA").is_ok() { let server = init.inner.build_server(); test_data::insert_test_data(&server).await; + #[cfg(feature = "enterprise")] server.insert_test_metrics().await; } @@ -114,6 +125,7 @@ async fn main() -> std::io::Result<()> { }); // Start broadcast subscriber + let inner = init.inner.clone(); spawn_broadcast_subscriber(init.inner, shutdown_rx); #[cfg(target_os = "linux")] @@ -125,14 +137,17 @@ async fn main() -> std::io::Result<()> { #[cfg(target_os = "linux")] ServiceState::Stopping.notify(); - // Shutdown collector - Collector::shutdown(); - // Stop services let _ = shutdown_tx.send(true); + // Persist the newest cache snapshots before the process exits + inner.cache.swap.stop().await; + + // Shutdown collector + Collector::shutdown(); + // Wait for services to finish tokio::time::sleep(Duration::from_secs(1)).await; - Ok(()) + exit(0); } diff --git a/crates/main/src/test_data.rs b/crates/main/src/test_data.rs index d2f73a1a..6291a875 100644 --- a/crates/main/src/test_data.rs +++ b/crates/main/src/test_data.rs @@ -71,7 +71,7 @@ pub async fn insert_test_data(server: &Server) { ValueClass::Registry(RegistryClass::Item { object_id, item_id }), report_bytes, ); - server.store().write(batch.build_all()).await.unwrap(); + server.store().write_batch(&mut batch).await.unwrap(); } for report in sample_dmarc_internal_reports() { @@ -85,28 +85,28 @@ pub async fn insert_test_data(server: &Server) { ValueClass::Registry(RegistryClass::Item { object_id, item_id }), report_bytes, ); - server.store().write(batch.build_all()).await.unwrap(); + server.store().write_batch(&mut batch).await.unwrap(); } for report in sample_tls_external_reports() { let mut batch = BatchBuilder::new(); let item_id = server.inner.data.queue_id_gen.generate(); report.write_ops(&mut batch, item_id, true); - server.store().write(batch.build_all()).await.unwrap(); + server.store().write_batch(&mut batch).await.unwrap(); } for report in sample_dmarc_external_reports() { let mut batch = BatchBuilder::new(); let item_id = server.inner.data.queue_id_gen.generate(); report.write_ops(&mut batch, item_id, true); - server.store().write(batch.build_all()).await.unwrap(); + server.store().write_batch(&mut batch).await.unwrap(); } for report in sample_arf_external_reports() { let mut batch = BatchBuilder::new(); let item_id = server.inner.data.queue_id_gen.generate(); report.write_ops(&mut batch, item_id, true); - server.store().write(batch.build_all()).await.unwrap(); + server.store().write_batch(&mut batch).await.unwrap(); } } @@ -154,14 +154,14 @@ fn sample_queued_messages(blob_hashes: Vec) -> Vec { }, expires: QueueExpiry::Ttl(365 * 24 * 3600), queue: Default::default(), - status: Status::Completed(HostResponse { + status: Status::Completed(Box::new(HostResponse { hostname: "mx.example.org".into(), response: Response { code: 250, esc: [2, 1, 5], message: "OK".into(), }, - }), + })), flags: RCPT_DSN_SENT, orcpt: Some("bob@example.org".into()), }, @@ -191,17 +191,17 @@ fn sample_queued_messages(blob_hashes: Vec) -> Vec { }, expires: QueueExpiry::Ttl(365 * 24 * 3600), queue: Default::default(), - status: Status::TemporaryFailure(ErrorDetails { + status: Status::TemporaryFailure(Box::new(ErrorDetails { entity: "mx.remote.net".into(), - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: "RCPT TO".into(), response: Response { code: 450, esc: [4, 2, 1], message: "Mailbox temporarily unavailable".into(), }, - }), - }), + })), + })), flags: 0, orcpt: None, }], @@ -230,10 +230,10 @@ fn sample_queued_messages(blob_hashes: Vec) -> Vec { }, expires: QueueExpiry::Ttl(365 * 24 * 3600), queue: Default::default(), - status: Status::TemporaryFailure(ErrorDetails { + status: Status::TemporaryFailure(Box::new(ErrorDetails { entity: "mx.bigcorp.com".into(), details: Error::ConnectionError("Rejected by policy".into()), - }), + })), flags: rcpt_spam_flag(72), orcpt: None, }], @@ -465,7 +465,7 @@ fn sample_dmarc_internal_reports() -> Vec { domain: "trusted-sender.com".to_string(), policy_identifier: 100, report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "dmarc@myserver.com".to_string(), errors: Map::new(vec![]), @@ -521,7 +521,7 @@ fn sample_dmarc_internal_reports() -> Vec { domain: "strict-domain.org".to_string(), policy_identifier: 200, report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "dmarc@myserver.com".to_string(), errors: Map::new(vec!["DKIM signature verification failed".to_string()]), @@ -577,7 +577,7 @@ fn sample_dmarc_internal_reports() -> Vec { domain: "new-policy.io".to_string(), policy_identifier: 300, report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "dmarc@myserver.com".to_string(), errors: Map::new(vec![]), @@ -666,7 +666,6 @@ fn sample_tls_external_reports() -> Vec { TlsExternalReport { expires_at: future, from: "tls-reports@bigprovider.com".to_string(), - member_tenant_id: None, received_at: now(), report: TlsReport { contact_info: Some("postmaster@bigprovider.com".to_string()), @@ -732,7 +731,6 @@ fn sample_tls_external_reports() -> Vec { TlsExternalReport { expires_at: future, from: "noreply@securemail.org".to_string(), - member_tenant_id: None, received_at: now(), report: TlsReport { contact_info: None, @@ -822,7 +820,6 @@ fn sample_tls_external_reports() -> Vec { TlsExternalReport { expires_at: future, from: "reports@mailhoster.net".to_string(), - member_tenant_id: None, received_at: now(), report: TlsReport { contact_info: Some("abuse@mailhoster.net".to_string()), @@ -918,10 +915,9 @@ fn sample_dmarc_external_reports() -> Vec { DmarcExternalReport { expires_at: future, from: "noreply-dmarc@google.com".to_string(), - member_tenant_id: None, received_at: now(), report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "noreply-dmarc@google.com".to_string(), errors: Map::new(vec![]), @@ -1027,10 +1023,9 @@ fn sample_dmarc_external_reports() -> Vec { DmarcExternalReport { expires_at: future, from: "dmarc@yahoo.com".to_string(), - member_tenant_id: None, received_at: now(), report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "dmarc@yahoo.com".to_string(), errors: Map::new(vec![]), @@ -1148,10 +1143,9 @@ fn sample_dmarc_external_reports() -> Vec { DmarcExternalReport { expires_at: future, from: "dmarc-noreply@microsoft.com".to_string(), - member_tenant_id: None, received_at: now(), report: DmarcReport { - date_range_begin: days_ago(1), + date_range_start: days_ago(1), date_range_end: now(), email: "dmarc-noreply@microsoft.com".to_string(), errors: Map::new(vec![]), @@ -1266,7 +1260,6 @@ fn sample_arf_external_reports() -> Vec { ArfExternalReport { expires_at: future, from: "fbl@isp-provider.com".to_string(), - member_tenant_id: None, received_at: now(), report: ArfFeedbackReport { arrival_date: Some(days_ago(1)), @@ -1310,7 +1303,6 @@ fn sample_arf_external_reports() -> Vec { ArfExternalReport { expires_at: future, from: "authfail@receiver.org".to_string(), - member_tenant_id: None, received_at: now(), report: ArfFeedbackReport { arrival_date: Some(days_ago(2)), @@ -1355,7 +1347,6 @@ fn sample_arf_external_reports() -> Vec { ArfExternalReport { expires_at: future, from: "reports@antiphish.net".to_string(), - member_tenant_id: None, received_at: now(), report: ArfFeedbackReport { arrival_date: Some(days_ago(3)), diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 817477e6..09a130f7 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -14,15 +14,14 @@ store = { path = "../store" } email = { path = "../email" } trc = { path = "../trc" } registry = { path = "../registry" } -mail-parser = { version = "0.11", features = ["full_encoding"] } -sieve-rs = { version = "0.7", features = ["rkyv"] } -tokio = { version = "1.53", features = ["full"] } +encodify = "1.0" +sieve-rs = { path = "../sieve" } +tokio = { version = "1.54", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } compact_str = "0.10.0" [features] -test_mode = [] -enterprise = [] +enterprise = ["imap/enterprise", "jmap_proto/enterprise", "directory/enterprise", "common/enterprise", "store/enterprise", "email/enterprise", "trc/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index cd130857..dc7dc7b8 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -5,15 +5,16 @@ */ use super::{Command, ResponseCode, SerializeResponse, Session, State}; -use common::{ - KV_RATE_LIMIT_IMAP, - network::{SessionResult, SessionStream}, -}; +use common::network::{SessionResult, SessionStream}; +use compact_str::format_compact; use imap_proto::receiver::{self, Request}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use trc::{AddContext, SecurityEvent}; use types::{collection::Collection, field::SieveField}; +const UNAUTHENTICATED_LITERAL_BYE: &[u8] = + b"BYE \"Literal exceeds the maximum size allowed before authentication.\"\r\n"; + impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { let mut bytes = bytes.iter(); @@ -22,6 +23,22 @@ impl Session { loop { match self.receiver.parse(&mut bytes) { + Ok(_) + if matches!( + self.state, + State::NotAuthenticated { + sasl_failure: Some(_), + .. + } + ) => + { + if let Some(failure) = self.state.take_sasl_failure() + && let Err(err) = self.write_error(failure).await + { + trc::error!(err.span_id(self.session_id)); + return SessionResult::Close; + } + } Ok(request) => match self.validate_request(request).await { Ok(request) => { requests.push(request); @@ -47,6 +64,35 @@ impl Session { break; } Err(receiver::Error::Error { response }) => { + if self.receiver.is_discarding() { + if matches!(self.state, State::NotAuthenticated { .. }) { + trc::error!(response.span_id(self.session_id)); + self.write(UNAUTHENTICATED_LITERAL_BYE).await.ok(); + return SessionResult::Close; + } + + let refusal = trc::ManageSieveEvent::Error + .into_err() + .details(format_compact!( + "Literal exceeds the maximum request size of {} bytes.", + self.receiver.max_request_size + )) + .code(ResponseCode::QuotaMaxSize); + if let Err(err) = self.write_error(refusal).await { + trc::error!(err.span_id(self.session_id)); + return SessionResult::Close; + } + continue; + } + + if let Some(failure) = self.state.take_sasl_failure() { + if let Err(err) = self.write_error(failure).await { + trc::error!(err.span_id(self.session_id)); + return SessionResult::Close; + } + break; + } + // Check for port scanners if matches!( (&self.state, response.key(trc::Key::Code)), @@ -97,7 +143,7 @@ impl Session { Command::CheckScript => self.handle_checkscript(request).await, Command::HaveSpace => self.handle_havespace(request).await, Command::Capability => self.handle_capability("").await, - Command::Authenticate => Box::pin(self.handle_authenticate(request)).await, + Command::Authenticate => self.handle_authenticate(request).await, Command::StartTls => self.handle_start_tls().await, Command::Logout => self.handle_logout().await, Command::Noop => self.handle_noop(request).await, @@ -180,29 +226,10 @@ impl Session { | Command::CheckScript | Command::Unauthenticate => { if let State::Authenticated { access_token, .. } = &self.state { - if let Some(rate) = &self.server.core.imap.rate_requests { - if self - .server - .in_memory_store() - .is_rate_allowed( - KV_RATE_LIMIT_IMAP, - &access_token.account_id().to_be_bytes(), - rate, - true, - ) - .await - .caused_by(trc::location!())? - .is_none() - { - Ok(command) - } else { - Err(trc::LimitEvent::TooManyRequests - .into_err() - .code(ResponseCode::TryLater)) - } - } else { - Ok(command) - } + self.server + .is_imap_request_allowed(access_token) + .await + .map(|_| command) } else { Err(trc::ManageSieveEvent::Error .into_err() diff --git a/crates/managesieve/src/core/mod.rs b/crates/managesieve/src/core/mod.rs index ce1239c7..31b47d7e 100644 --- a/crates/managesieve/src/core/mod.rs +++ b/crates/managesieve/src/core/mod.rs @@ -33,6 +33,7 @@ pub struct Session { pub enum State { NotAuthenticated { auth_failures: u32, + sasl_failure: Option, }, Authenticated { access_token: AccessToken, @@ -41,6 +42,13 @@ pub enum State { } impl State { + pub fn take_sasl_failure(&mut self) -> Option { + match self { + State::NotAuthenticated { sasl_failure, .. } => sasl_failure.take(), + _ => None, + } + } + pub fn access_token(&self) -> &AccessToken { match self { State::Authenticated { access_token, .. } => access_token, @@ -133,7 +141,7 @@ pub enum ResponseCode { Active, NonExistent, AlreadyExists, - Tag(String), + Tag(CompactString), Warnings, } @@ -268,7 +276,7 @@ pub trait SerializeResponse { impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(64); - buf.extend_from_slice(self.value_as_str(trc::Key::Type).unwrap_or("NO").as_bytes()); + buf.extend_from_slice(b"NO"); if let Some(code) = self .value_as_str(trc::Key::Code) .or_else(|| match self.as_ref() { diff --git a/crates/managesieve/src/core/session.rs b/crates/managesieve/src/core/session.rs index 4de2f932..c803f426 100644 --- a/crates/managesieve/src/core/session.rs +++ b/crates/managesieve/src/core/session.rs @@ -10,7 +10,7 @@ use common::{ BuildServer, network::{SessionData, SessionManager, SessionResult, SessionStream}, }; -use imap_proto::receiver::{self, Receiver}; +use imap_proto::receiver::{self, MAX_UNAUTHENTICATED_REQUEST_SIZE, Receiver}; use tokio_rustls::server::TlsStream; impl SessionManager for ManageSieveSessionManager { @@ -23,11 +23,14 @@ impl SessionManager for ManageSieveSessionManager { // Create session let server = self.inner.build_server(); let mut session = Session { - receiver: Receiver::with_max_request_size(server.core.imap.max_request_size) + receiver: Receiver::with_max_request_size(MAX_UNAUTHENTICATED_REQUEST_SIZE) .with_start_state(receiver::State::Command { is_uid: false }), server, instance: session.instance, - state: State::NotAuthenticated { auth_failures: 0 }, + state: State::NotAuthenticated { + auth_failures: 0, + sasl_failure: None, + }, session_id: session.session_id, stream: session.stream, in_flight: session.in_flight, @@ -130,8 +133,9 @@ impl Session { false } - pub async fn into_tls(self) -> Result>, ()> { - let receiver = Receiver::with_max_request_size(self.server.core.imap.max_request_size) + pub async fn into_tls(mut self) -> Result>, ()> { + self.state.take_sasl_failure(); + let receiver = Receiver::with_max_request_size(self.receiver.max_request_size) .with_start_state(receiver::State::Command { is_uid: false }); Ok(Session { diff --git a/crates/managesieve/src/lib.rs b/crates/managesieve/src/lib.rs index 0237a695..ac5ac0f6 100644 --- a/crates/managesieve/src/lib.rs +++ b/crates/managesieve/src/lib.rs @@ -11,134 +11,3 @@ pub mod op; static SERVER_GREETING: &str = "Stalwart ManageSieve at your service."; -#[cfg(test)] -mod tests { - use imap_proto::receiver::{Error, Receiver, Request, State, Token}; - - use crate::core::Command; - - #[test] - fn receiver_parse_managesieve() { - let mut receiver = Receiver::new().with_start_state(State::Command { is_uid: false }); - - for (frames, expected_requests) in [ - ( - vec!["Authenticate \"DIGEST-MD5\"\r\n"], - vec![Request { - tag: "".into(), - command: Command::Authenticate, - tokens: vec![Token::Argument(b"DIGEST-MD5".to_vec())], - }], - ), - ( - vec![ - " AUTHENTICATE \"GSSAPI\" {56+}\r\n", - "cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA==\r\n", - ], - vec![Request { - tag: "".into(), - command: Command::Authenticate, - tokens: vec![ - Token::Argument(b"GSSAPI".to_vec()), - Token::Argument( - b"cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA==".to_vec(), - ), - ], - }], - ), - ( - vec!["Authenticate \"PLAIN\" \"QJIrweAPyo6Q1T9xu\"\r\n"], - vec![Request { - tag: "".into(), - command: Command::Authenticate, - tokens: vec![ - Token::Argument(b"PLAIN".to_vec()), - Token::Argument(b"QJIrweAPyo6Q1T9xu".to_vec()), - ], - }], - ), - ( - vec!["StartTls\r\n"], - vec![Request { - tag: "".into(), - command: Command::StartTls, - tokens: vec![], - }], - ), - ( - vec!["HAVESPACE \"myscript\" 999999\r\n"], - vec![Request { - tag: "".into(), - command: Command::HaveSpace, - tokens: vec![ - Token::Argument(b"myscript".to_vec()), - Token::Argument(b"999999".to_vec()), - ], - }], - ), - ( - vec![ - "Putscript \"foo\" {31+}\r\n", - "#comment\r\n", - "InvalidSieveCommand\r\n\r\n", - ], - vec![Request { - tag: "".into(), - command: Command::PutScript, - tokens: vec![ - Token::Argument(b"foo".to_vec()), - Token::Argument(b"#comment\r\nInvalidSieveCommand\r\n".to_vec()), - ], - }], - ), - ( - vec!["Listscripts\r\n"], - vec![Request { - tag: "".into(), - command: Command::ListScripts, - tokens: vec![], - }], - ), - ( - vec!["Setactive \"baz\"\r\n"], - vec![Request { - tag: "".into(), - command: Command::SetActive, - tokens: vec![Token::Argument(b"baz".to_vec())], - }], - ), - ( - vec!["Renamescript \"foo\" \"bar\"\r\n"], - vec![Request { - tag: "".into(), - command: Command::RenameScript, - tokens: vec![ - Token::Argument(b"foo".to_vec()), - Token::Argument(b"bar".to_vec()), - ], - }], - ), - ( - vec!["NOOP \"STARTTLS-SYNC-42\"\r\n"], - vec![Request { - tag: "".into(), - command: Command::Noop, - tokens: vec![Token::Argument(b"STARTTLS-SYNC-42".to_vec())], - }], - ), - ] { - let mut requests = Vec::new(); - for frame in &frames { - let mut bytes = frame.as_bytes().iter(); - loop { - match receiver.parse(&mut bytes) { - Ok(request) => requests.push(request), - Err(Error::NeedsMoreData | Error::NeedsLiteral { .. }) => break, - Err(err) => panic!("{:?} for frames {:#?}", err, frames), - } - } - } - assert_eq!(requests, expected_requests, "{:#?}", frames); - } - } -} diff --git a/crates/managesieve/src/op/authenticate.rs b/crates/managesieve/src/op/authenticate.rs index 3f117cf9..47d3ba9f 100644 --- a/crates/managesieve/src/op/authenticate.rs +++ b/crates/managesieve/src/op/authenticate.rs @@ -3,18 +3,23 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; -use crate::core::{Command, Session, State, StatusResponse}; +use crate::core::{Command, ResponseCode, Session, State, StatusResponse}; use common::{ - auth::AuthRequest, + auth::{ + AuthRequest, + oauth::scope::OAuthScopes, + sasl::{BearerFailure, BearerMechanism, SASL_CANCEL}, + }, network::{SessionStream, limiter::LimiterResult}, }; use directory::Credentials; +use encodify::base64::LENIENT; use imap_proto::{ protocol::authenticate::Mechanism, - receiver::{self, Request}, + receiver::{self, MAX_UNAUTHENTICATED_REQUEST_SIZE, Request}, }; -use mail_parser::decoders::base64::base64_decode; use registry::schema::enums::Permission; impl Session { @@ -28,36 +33,14 @@ impl Session { let mut tokens = request.tokens.into_iter(); let mechanism = Mechanism::parse(&tokens.next().unwrap().unwrap_bytes()) .map_err(|err| trc::AuthEvent::Error.into_err().details(err))?; - let mut params: Vec = tokens + let mut params: Vec = tokens .filter_map(|token| token.unwrap_string().ok()) .collect(); - let credentials = match mechanism { - Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { - if !params.is_empty() { - base64_decode(params.pop().unwrap().as_bytes()) - .and_then(|challenge| { - if mechanism == Mechanism::Plain { - Credentials::decode_sasl_challenge_plain(&challenge) - } else { - Credentials::decode_sasl_challenge_oauth(&challenge) - } - }) - .ok_or_else(|| { - trc::AuthEvent::Error - .into_err() - .details("Failed to decode challenge.") - })? - } else { - self.receiver.request = receiver::Request { - tag: "".into(), - command: Command::Authenticate, - tokens: vec![receiver::Token::Argument(mechanism.into_bytes())], - }; - self.receiver.state = receiver::State::Argument { last_ch: b' ' }; - return Ok(b"{0}\r\n".to_vec()); - } - } + let bearer = match mechanism { + Mechanism::Plain => None, + Mechanism::OAuthBearer => Some(BearerMechanism::OAuthBearer), + Mechanism::XOauth2 => Some(BearerMechanism::XOAuth2), _ => { return Err(trc::AuthEvent::Error .into_err() @@ -65,34 +48,88 @@ impl Session { } }; + if bearer.is_some() && !self.server.accepts_bearer_tokens(self.stream.is_tls()) { + return Err(trc::ManageSieveEvent::Error + .into_err() + .code(ResponseCode::EncryptNeeded) + .details("OAUTHBEARER and XOAUTH2 require a TLS connection.")); + } + + let param = match params.pop() { + Some(param) if param == SASL_CANCEL => { + return Err(trc::ManageSieveEvent::Error + .into_err() + .details("Authentication cancelled.")); + } + Some(param) if !param.is_empty() => param, + _ => { + self.expect_sasl_response(&mechanism); + return Ok(b"{0}\r\n".to_vec()); + } + }; + let credentials = LENIENT.decode(param.as_bytes()).ok().map(|challenge| { + if bearer.is_none() { + Credentials::decode_sasl_challenge_plain(&challenge) + } else { + Credentials::decode_sasl_challenge_oauth(&challenge) + } + }); + let credentials = match (credentials, bearer) { + (Some(Some(credentials)), _) => credentials, + (Some(None), Some(bearer)) => { + let err = self.malformed_credentials().await; + return match BearerFailure::from_error(&err) { + Some(failure) => Ok(self + .bearer_challenge(bearer, failure, None, err, &mechanism) + .await), + None => Err(err), + }; + } + _ => return Err(self.malformed_credentials().await), + }; + // Authenticate - let access_token = self - .server - .authenticate(&AuthRequest::from_credentials( - credentials, - self.session_id, - self.remote_addr, - )) - .await - .map_err(|err| { - if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { - match &self.state { - State::NotAuthenticated { auth_failures } - if *auth_failures < self.server.core.imap.max_auth_failures => - { - self.state = State::NotAuthenticated { - auth_failures: auth_failures + 1, - }; - } - _ => { - return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); + let request = AuthRequest::from_credentials(credentials, self.session_id, self.remote_addr); + let access_token = match self.server.authenticate(&request).await { + Ok(authenticated) => match bearer { + Some(bearer) => { + match authenticated + .access_token + .assert_allows_oauth_scopes(OAuthScopes::MAIL) + { + Ok(access_token) => access_token, + Err(err) => { + return Ok(self + .bearer_challenge( + bearer, + BearerFailure::InsufficientScope, + Some(&request.credentials), + BearerFailure::insufficient_scope(err), + &mechanism, + ) + .await); } } } - - err - }) - .and_then(|token| token.assert_has_permission(Permission::SieveAuthenticate))?; + None => authenticated.access_token, + } + .assert_has_permission(Permission::SieveAuthenticate)?, + Err(err) => { + let err = self.register_auth_failure(err); + return match (bearer, BearerFailure::from_error(&err)) { + (Some(bearer), Some(failure)) => Ok(self + .bearer_challenge( + bearer, + failure, + Some(&request.credentials), + err, + &mechanism, + ) + .await), + _ => Err(err), + }; + } + }; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { @@ -108,12 +145,78 @@ impl Session { access_token, in_flight, }; + self.receiver.max_request_size = self.server.core.imap.max_request_size; Ok(StatusResponse::ok("Authentication successful").into_bytes()) } + async fn bearer_challenge( + &mut self, + bearer: BearerMechanism, + failure: BearerFailure, + credentials: Option<&Credentials>, + err: trc::Error, + mechanism: &Mechanism, + ) -> Vec { + let challenge = self + .server + .bearer_failure_challenge(bearer, failure, credentials) + .await; + if let State::NotAuthenticated { sasl_failure, .. } = &mut self.state { + *sasl_failure = Some(err); + } + self.expect_sasl_response(mechanism); + format!("\"{challenge}\"\r\n").into_bytes() + } + + fn expect_sasl_response(&mut self, mechanism: &Mechanism) { + self.receiver.request = receiver::Request { + tag: "".into(), + command: Command::Authenticate, + tokens: vec![receiver::Token::Argument( + receiver::ArgumentBytes::from_slice(mechanism.as_bytes()), + )], + }; + self.receiver.state = receiver::State::Argument { last_ch: b' ' }; + } + + async fn malformed_credentials(&mut self) -> trc::Error { + let err = self + .server + .auth_failure( + trc::AuthEvent::Failed + .into_err() + .details("Invalid SASL response."), + self.remote_addr, + None, + ) + .await; + self.register_auth_failure(err) + } + + fn register_auth_failure(&mut self, err: trc::Error) -> trc::Error { + if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { + match &mut self.state { + State::NotAuthenticated { auth_failures, .. } + if *auth_failures < self.server.core.imap.max_auth_failures => + { + *auth_failures += 1; + } + _ => { + return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); + } + } + } + + err + } + pub async fn handle_unauthenticate(&mut self) -> trc::Result> { - self.state = State::NotAuthenticated { auth_failures: 0 }; + self.state = State::NotAuthenticated { + auth_failures: 0, + sasl_failure: None, + }; + self.receiver.max_request_size = MAX_UNAUTHENTICATED_REQUEST_SIZE; trc::event!( ManageSieve(trc::ManageSieveEvent::Unauthenticate), diff --git a/crates/managesieve/src/op/capability.rs b/crates/managesieve/src/op/capability.rs index c0ceff6a..4d4f6c05 100644 --- a/crates/managesieve/src/op/capability.rs +++ b/crates/managesieve/src/op/capability.rs @@ -19,11 +19,14 @@ impl Session { if !self.stream.is_tls() { response.extend_from_slice(b"\"STARTTLS\"\r\n"); } - if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { + let is_tls = self.stream.is_tls(); + if !is_tls && !self.server.core.imap.allow_plain_auth { + response.extend_from_slice(b"\"SASL\" \"\"\r\n"); + } else if self.server.accepts_bearer_tokens(is_tls) { response.extend_from_slice(b"\"SASL\" \"PLAIN OAUTHBEARER XOAUTH2\"\r\n"); } else { - response.extend_from_slice(b"\"SASL\" \"OAUTHBEARER XOAUTH2\"\r\n"); - }; + response.extend_from_slice(b"\"SASL\" \"PLAIN\"\r\n"); + } if let Some(sieve) = self.server .core diff --git a/crates/managesieve/src/op/checkscript.rs b/crates/managesieve/src/op/checkscript.rs index a07b453d..3773023c 100644 --- a/crates/managesieve/src/op/checkscript.rs +++ b/crates/managesieve/src/op/checkscript.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::ToCompactString; use std::time::Instant; use common::network::SessionStream; @@ -44,7 +45,7 @@ impl Session { .map_err(|err| { trc::ManageSieveEvent::Error .into_err() - .details(err.to_string()) + .details(err.to_compact_string()) }) } } diff --git a/crates/managesieve/src/op/deletescript.rs b/crates/managesieve/src/op/deletescript.rs index 5b720ae2..75609edd 100644 --- a/crates/managesieve/src/op/deletescript.rs +++ b/crates/managesieve/src/op/deletescript.rs @@ -54,7 +54,7 @@ impl Session { trc::event!( ManageSieve(trc::ManageSieveEvent::DeleteScript), SpanId = self.session_id, - Id = name, + Script = name, DocumentId = document_id, Elapsed = op_start.elapsed() ); diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 5895f5cb..7402bfb6 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -12,10 +12,10 @@ use registry::schema::enums::Permission; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; -use types::{blob::BlobSection, blob_hash::BlobHash, collection::Collection}; +use types::collection::Collection; impl Session { pub async fn handle_getscript(&mut self, request: Request) -> trc::Result> { @@ -38,7 +38,7 @@ impl Session { let sieve_ = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -54,16 +54,10 @@ impl Session { let sieve = sieve_ .unarchive::() .caused_by(trc::location!())?; - let blob_size = u32::from(sieve.size) as usize; let script = self .server - .get_blob_section( - &BlobHash::from(&sieve.blob_hash), - &BlobSection { - size: blob_size, - ..Default::default() - }, - ) + .blob_store() + .get_blob(sieve.blob_hash.0.as_ref(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -72,11 +66,10 @@ impl Session { .details("Script blob not found") .code(ResponseCode::NonExistent) })?; - debug_assert_eq!(script.len(), blob_size); let mut response = Vec::with_capacity(script.len() + 32); response.push(b'{'); - response.extend_from_slice(blob_size.to_string().as_bytes()); + response.extend_from_slice(script.len().to_string().as_bytes()); response.extend_from_slice(b"}\r\n"); response.extend(script); response.extend_from_slice(b"\r\n"); @@ -84,7 +77,7 @@ impl Session { trc::event!( ManageSieve(trc::ManageSieveEvent::GetScript), SpanId = self.session_id, - Id = name, + Script = name, DocumentId = document_id, Elapsed = op_start.elapsed() ); diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 4d09f29f..0ec0d4c0 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -11,7 +11,7 @@ use registry::schema::enums::Permission; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AddContext; use types::{collection::Collection, field::SieveField}; @@ -41,7 +41,7 @@ impl Session { if let Some(script_) = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 4752c232..e6a70730 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -6,17 +6,18 @@ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; +use compact_str::ToCompactString; use email::sieve::SieveScript; use imap_proto::receiver::Request; use registry::schema::enums::{Permission, StorageQuota}; use sieve::compiler::ErrorType; use std::time::Instant; use store::{ - Serialize, ValueKey, - write::{AlignedBytes, Archive, Archiver, BatchBuilder}, + ValueKey, + write::{Archive, ArchiveBytes, BatchBuilder}, }; use trc::AddContext; -use types::{collection::Collection, field::SieveField}; +use types::{blob_hash::BlobHash, collection::Collection, field::SieveField}; impl Session { pub async fn handle_putscript(&mut self, request: Request) -> trc::Result> { @@ -35,7 +36,7 @@ impl Session { })? .trim() .to_string(); - let mut script_bytes = tokens + let script_bytes = tokens .next() .ok_or_else(|| { trc::ManageSieveEvent::Error @@ -49,10 +50,6 @@ impl Session { let access_token = self.state.access_token(); let account_id = access_token.account_id(); let account = self.server.account(account_id).await?; - self.server - .has_available_quota(&account, script_bytes.len() as u64) - .await - .caused_by(trc::location!())?; if self .server @@ -72,34 +69,27 @@ impl Session { } // Compile script - match self + let compiled_script = match self .server .core .sieve .untrusted_compiler .compile(&script_bytes) { - Ok(compiled_script) => { - script_bytes.extend( - Archiver::new(compiled_script) - .untrusted() - .serialize() - .caused_by(trc::location!())?, - ); - } + Ok(compiled_script) => compiled_script, Err(err) => { return Err(if let ErrorType::ScriptTooLong = &err.error_type() { trc::ManageSieveEvent::Error .into_err() - .details(err.to_string()) + .details(err.to_compact_string()) .code(ResponseCode::QuotaMaxSize) } else { trc::ManageSieveEvent::Error .into_err() - .details(err.to_string()) + .details(err.to_compact_string()) }); } - } + }; // Validate name if let Some(document_id) = self.validate_name(account_id, &name).await? { @@ -107,7 +97,7 @@ impl Session { let script_ = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -127,13 +117,19 @@ impl Session { // Write script blob let (blob_hash, blob_hold) = self .server - .put_temporary_blob(account_id, &script_bytes, 60) + .hold_blob( + account_id, + BlobHash::generate(&script_bytes), + &script_bytes, + 60, + ) .await?; // Write record let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + self.server.limit_disk_quota(&mut batch, &account); batch - .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .custom( @@ -143,13 +139,14 @@ impl Session { .deserialize() .caused_by(trc::location!())? .with_size(script_size as u32) - .with_blob_hash(blob_hash.clone()), + .with_blob_hash(blob_hash.clone()) + .with_script(&compiled_script), ) .with_current(script) - .with_changed_by(account.account_tenant_ids()), + .with_changed_by(account.changed_by()), ) - .caused_by(trc::location!())? - .clear(blob_hold); + .caused_by(trc::location!())?; + blob_hold.release(&mut batch); self.server .commit_batch(batch) @@ -159,7 +156,7 @@ impl Session { trc::event!( ManageSieve(trc::ManageSieveEvent::UpdateScript), SpanId = self.session_id, - Id = name.to_string(), + Script = name, DocumentId = document_id, Size = script_size, Elapsed = op_start.elapsed(), @@ -168,33 +165,36 @@ impl Session { // Write script blob let (blob_hash, blob_hold) = self .server - .put_temporary_blob(account_id, &script_bytes, 60) + .hold_blob( + account_id, + BlobHash::generate(&script_bytes), + &script_bytes, + 60, + ) .await?; // Write record let mut batch = BatchBuilder::new(); - let document_id = self - .server - .store() - .assign_document_ids(account_id, Collection::SieveScript, 1) - .await - .caused_by(trc::location!())?; + let slot = batch.reserve_document_id(account_id, Collection::SieveScript); + batch.with_account_id(account_id); + self.server.limit_disk_quota(&mut batch, &account); batch - .with_account_id(account_id) .with_collection(Collection::SieveScript) - .with_document(document_id) + .create_document(slot) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes( SieveScript::new(name.clone(), blob_hash.clone()) - .with_size(script_size as u32), + .with_size(script_size as u32) + .with_script(&compiled_script), ) - .with_changed_by(account.account_tenant_ids()), + .with_changed_by(account.changed_by()), ) - .caused_by(trc::location!())? - .clear(blob_hold); + .caused_by(trc::location!())?; + blob_hold.release(&mut batch); - self.server + let assigned_ids = self + .server .commit_batch(batch) .await .caused_by(trc::location!())?; @@ -202,8 +202,8 @@ impl Session { trc::event!( ManageSieve(trc::ManageSieveEvent::CreateScript), SpanId = self.session_id, - Id = name, - DocumentId = document_id, + Script = name, + DocumentId = assigned_ids.slot(slot), Elapsed = op_start.elapsed() ); } diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index fef3ce17..ea0f0af4 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -6,13 +6,14 @@ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::{network::SessionStream, storage::index::ObjectIndexBuilder}; +use compact_str::format_compact; use email::sieve::SieveScript; use imap_proto::receiver::Request; use registry::schema::enums::Permission; use std::time::Instant; use store::{ ValueKey, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{Archive, ArchiveBytes, BatchBuilder}, }; use trc::AddContext; use types::collection::Collection; @@ -54,7 +55,9 @@ impl Session { if self.validate_name(account_id, &new_name).await?.is_some() { return Err(trc::ManageSieveEvent::Error .into_err() - .details(format!("A sieve script with name '{name}' already exists.",)) + .details(format_compact!( + "A sieve script with name '{name}' already exists.", + )) .code(ResponseCode::AlreadyExists)); } @@ -62,7 +65,7 @@ impl Session { let script = self .server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, @@ -100,7 +103,7 @@ impl Session { trc::event!( ManageSieve(trc::ManageSieveEvent::RenameScript), SpanId = self.session_id, - Id = new_name, + Script = new_name, DocumentId = document_id, Elapsed = op_start.elapsed() ); diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index 1cb74cd1..fdbf7771 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -6,7 +6,7 @@ use std::time::Instant; -use common::network::SessionStream; +use common::{ipc::CacheInvalidation, network::SessionStream}; use imap_proto::receiver::Request; use registry::schema::enums::Permission; use store::{SerializeInfallible, write::BatchBuilder}; @@ -53,11 +53,15 @@ impl Session { .commit_batch(batch) .await .caused_by(trc::location!())?; + self.server + .invalidate_caches(CacheInvalidation::ActiveSieveScript(account_id).into()) + .await + .caused_by(trc::location!())?; trc::event!( ManageSieve(trc::ManageSieveEvent::SetActive), SpanId = self.session_id, - Id = name, + Script = name, Elapsed = op_start.elapsed() ); diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml deleted file mode 100644 index 30b37951..00000000 --- a/crates/migration/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "migration" -version = "0.16.27" -edition = "2024" - -[dependencies] -store = { path = "../store" } -trc = { path = "../trc" } -types = { path = "../types" } -common = { path = "../common" } -email = { path = "../email" } -spam-filter = { path = "../spam-filter" } -registry = { path = "../registry" } -tokio = { version = "1.53", features = ["net", "macros"] } - -[features] -test_mode = [] -enterprise = [] - -[dev-dependencies] -tokio = { version = "1.53", features = ["full"] } - -[lints] -workspace = true diff --git a/crates/migration/src/destroy.rs b/crates/migration/src/destroy.rs deleted file mode 100644 index 2477d145..00000000 --- a/crates/migration/src/destroy.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use store::{ - Store, U64_LEN, - write::{AnyKey, key::KeySerializer}, -}; -use trc::AddContext; - -pub async fn destroy_subspace(store: &Store, subspace: u8) -> trc::Result<()> { - store - .delete_range( - AnyKey { - subspace, - key: KeySerializer::new(U64_LEN).write(0u8).finalize(), - }, - AnyKey { - subspace, - key: KeySerializer::new(U64_LEN) - .write(&[u8::MAX; 64][..]) - .finalize(), - }, - ) - .await - .caused_by(trc::location!()) -} diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs deleted file mode 100644 index 3066afb8..00000000 --- a/crates/migration/src/lib.rs +++ /dev/null @@ -1,136 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -#![warn(clippy::large_futures)] - -use crate::v016::migrate_v0_16; -use common::{DATABASE_SCHEMA_VERSION, Server}; -use store::{ - IterateParams, SUBSPACE_PROPERTY, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_REPORT_IN, - SUBSPACE_REPORT_OUT, SerializeInfallible, - write::{AnyClass, AnyKey, BatchBuilder, ValueClass}, -}; -use trc::AddContext; - -pub mod destroy; -pub mod v016; - -pub async fn try_migrate(server: &Server) -> trc::Result<()> { - match server - .store() - .get_value::(AnyKey { - subspace: SUBSPACE_PROPERTY, - key: vec![0u8], - }) - .await - .caused_by(trc::location!())? - { - Some(DATABASE_SCHEMA_VERSION) => { - if !std::env::var("DANGER_FORCE_MIGRATE").is_ok_and(|v| v == "1") { - return Ok(()); - } - } - Some(0..=4) => { - abort(concat!( - "You must first upgrade to version 0.15, please read ", - "https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md" - )); - } - Some(5) => { - if !server.registry().is_recovery_mode() { - abort(concat!( - "Upgrading to version 0.16 is a multi-step process, please read ", - "https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md" - )); - } - } - - Some(version) => { - panic!( - "Unknown database schema version, expected {} or below, found {}", - DATABASE_SCHEMA_VERSION, version - ); - } - _ => { - if is_new_install(server).await.caused_by(trc::location!())? { - write_schema_version(server).await?; - return Ok(()); - } else { - abort(concat!( - "You must first upgrade to version 0.15, please read ", - "https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md" - )); - } - } - } - - migrate_v0_16(server).await?; - write_schema_version(server).await -} - -async fn write_schema_version(server: &Server) -> trc::Result<()> { - let mut batch = BatchBuilder::new(); - batch.set( - ValueClass::Any(AnyClass { - subspace: SUBSPACE_PROPERTY, - key: vec![0u8], - }), - DATABASE_SCHEMA_VERSION.serialize(), - ); - - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - - Ok(()) -} - -fn abort(message: &str) -> ! { - eprintln!("Migration aborted: {message}"); - panic!("Migration aborted: {message}"); -} - -async fn is_new_install(server: &Server) -> trc::Result { - for subspace in [ - SUBSPACE_QUEUE_MESSAGE, - SUBSPACE_REPORT_IN, - SUBSPACE_REPORT_OUT, - SUBSPACE_PROPERTY, - ] { - let mut has_data = false; - - server - .store() - .iterate( - IterateParams::new( - AnyKey { - subspace, - key: vec![0u8], - }, - AnyKey { - subspace, - key: vec![u8::MAX; 16], - }, - ) - .no_values(), - |_, _| { - has_data = true; - - Ok(false) - }, - ) - .await - .caused_by(trc::location!())?; - - if has_data { - return Ok(false); - } - } - - Ok(true) -} diff --git a/crates/migration/src/v016.rs b/crates/migration/src/v016.rs deleted file mode 100644 index 8614d23e..00000000 --- a/crates/migration/src/v016.rs +++ /dev/null @@ -1,299 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::destroy::destroy_subspace; -use common::{Server, manager::SPAM_TRAINER_KEY}; -use registry::{ - schema::{ - prelude::{ObjectType, Property}, - structs::{ArchivedEmail, ArchivedItem}, - }, - types::{EnumImpl, ObjectImpl, datetime::UTCDateTime, id::ObjectId}, -}; -use spam_filter::modules::classifier::SpamTrainer; -use store::{ - Deserialize, IterateParams, SUBSPACE_BLOB_LINK, SUBSPACE_DIRECTORY, SUBSPACE_QUOTA, - SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_TELEMETRY_SPAN, Serialize, SerializeInfallible, U32_LEN, U64_LEN, - search::{SearchField, SearchFilter, SearchQuery}, - write::{ - AlignedBytes, AnyClass, AnyKey, Archive, Archiver, BatchBuilder, RegistryClass, - SearchIndex, ValueClass, key::DeserializeBigEndian, now, - }, -}; -use trc::AddContext; -use types::{blob::BlobId, blob_hash::BLOB_HASH_LEN}; - -const LEGACY_SUBSPACE_BLOB_EXTRA: u8 = b'j'; // Now SUBSPACE_DELETED_ITEMS -const LEGACY_SUBSPACE_BITMAP_ID: u8 = b'b'; // Now SUBSPACE_REGISTRY_IDX -const LEGACY_SUBSPACE_SETTINGS: u8 = b's'; // Now SUBSPACE_REGISTRY -const LEGACY_SUBSPACE_FTS_INDEX: u8 = b'g'; // Now SUBSPACE_REGISTRY_PK -const LEGACY_SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; // Now SUBSPACE_SPAM_SAMPLES - -pub async fn migrate_v0_16(server: &Server) -> trc::Result<()> { - // Delete tracing index - server - .search_store() - .unindex( - SearchQuery::new(SearchIndex::Tracing) - .with_filter(SearchFilter::lt(SearchField::Id, u64::MAX)), - ) - .await - .caused_by(trc::location!())?; - - // Delete old quotas - server - .store() - .delete_range( - AnyKey { - subspace: SUBSPACE_QUOTA, - key: vec![0x04], - }, - AnyKey { - subspace: SUBSPACE_QUOTA, - key: vec![0x05], - }, - ) - .await - .caused_by(trc::location!())?; - - // Destroy old and incompatible subspaces - for namespace in [ - LEGACY_SUBSPACE_BLOB_EXTRA, - LEGACY_SUBSPACE_TELEMETRY_INDEX, - LEGACY_SUBSPACE_SETTINGS, - LEGACY_SUBSPACE_BITMAP_ID, - LEGACY_SUBSPACE_FTS_INDEX, - SUBSPACE_REPORT_IN, - SUBSPACE_REPORT_OUT, - SUBSPACE_DIRECTORY, - SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_TELEMETRY_SPAN, - SUBSPACE_TASK_QUEUE, - ] { - destroy_subspace(server.store(), namespace) - .await - .caused_by(trc::location!())?; - } - - // Migrate blob links - migrate_blob_links(server).await?; - - // Migrate spam model - migrate_spam_model(server).await?; - - Ok(()) -} - -async fn migrate_spam_model(server: &Server) -> trc::Result<()> { - let Some(mut trainer) = server - .blob_store() - .get_blob(SPAM_TRAINER_KEY, 0..usize::MAX) - .await - .and_then(|archive| match archive { - Some(archive) => as Deserialize>::deserialize(&archive) - .and_then(|archive| archive.deserialize_untrusted::()) - .map(Some), - None => Ok(None), - }) - .caused_by(trc::location!())? - else { - return Ok(()); - }; - - if trainer.last_id == 0 { - return Ok(()); - } - - if let Some(config) = &server.core.spam.classifier { - trainer.reservoir.ham.total_seen = std::cmp::min( - config.min_ham_samples + config.reservoir_capacity as u64, - trainer.reservoir.ham.total_seen, - ); - trainer.reservoir.spam.total_seen = std::cmp::min( - config.min_spam_samples + config.reservoir_capacity as u64, - trainer.reservoir.spam.total_seen, - ); - } else { - trainer.reservoir.ham.total_seen = std::cmp::min( - trainer.reservoir.ham.buffer.len() as u64, - trainer.reservoir.ham.total_seen, - ); - trainer.reservoir.spam.total_seen = std::cmp::min( - trainer.reservoir.spam.buffer.len() as u64, - trainer.reservoir.spam.total_seen, - ); - } - - trainer.reservoir.ham.buffer.clear(); - trainer.reservoir.spam.buffer.clear(); - trainer.last_id = 0; - - server - .blob_store() - .put_blob( - SPAM_TRAINER_KEY, - &Archiver::new(trainer) - .serialize() - .caused_by(trc::location!())?, - server.core.email.compression, - ) - .await - .caused_by(trc::location!())?; - - Ok(()) -} - -async fn migrate_blob_links(server: &Server) -> trc::Result<()> { - let mut delete_keys = Vec::new(); - let mut archived_items = Vec::new(); - let now = now(); - - server - .store() - .iterate( - IterateParams::new( - AnyKey { - subspace: SUBSPACE_BLOB_LINK, - key: vec![0u8], - }, - AnyKey { - subspace: SUBSPACE_BLOB_LINK, - key: vec![u8::MAX; 32], - }, - ), - |key, value| { - const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; - - const QUOTA_LINK: u8 = 0; - const UNDELETE_LINK: u8 = 1; - const SPAM_SAMPLE_LINK: u8 = 2; - - if key.len() == TEMP_LINK && value.len() == 1 { - let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; - if until > now { - let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; - let hash = types::blob_hash::BlobHash::try_from_hash_slice( - key.get(0..BLOB_HASH_LEN).ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?, - ) - .unwrap(); - - match value.first().copied() { - Some(UNDELETE_LINK) => { - archived_items.push((key.to_vec(), account_id, hash, until)); - } - Some(SPAM_SAMPLE_LINK | QUOTA_LINK) => { - delete_keys.push(key.to_vec()); - } - _ => {} - } - } - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - // Delete spam samples and quota links - let mut batch = BatchBuilder::new(); - for key in delete_keys { - batch.clear(ValueClass::Any(AnyClass { - subspace: SUBSPACE_BLOB_LINK, - key, - })); - - if batch.is_large_batch() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - } - if !batch.is_empty() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - - // Migrate spam samples - let mut batch = BatchBuilder::new(); - let id_gen = &server.inner.data.registry_id_gen; - let mut last_id = 0; - for (key, account_id, blob_hash, until) in archived_items { - let item = ArchivedItem::Email(ArchivedEmail { - account_id: account_id.into(), - blob_id: BlobId::new(blob_hash.clone(), Default::default()), - archived_until: UTCDateTime::from_timestamp(until as i64), - archived_at: UTCDateTime::now(), - from: "Unavailable".to_string(), - received_at: UTCDateTime::now(), - subject: "...".to_string(), - size: 0, - }) - .to_pickled_vec(); - let object_id = ObjectType::ArchivedItem.to_id(); - - loop { - let new_id = id_gen.generate(); - if new_id != last_id { - last_id = new_id; - break; - } else { - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - } - let item_id = last_id; - - batch - .set( - ValueClass::Any(AnyClass { - subspace: SUBSPACE_BLOB_LINK, - key, - }), - ObjectId::new(ObjectType::ArchivedItem, item_id.into()).serialize(), - ) - .set( - ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId.to_id(), - object_id, - item_id, - key: (account_id as u64).serialize(), - }), - vec![], - ) - .set( - ValueClass::Registry(RegistryClass::Item { object_id, item_id }), - item, - ); - - if batch.is_large_batch() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - } - - if !batch.is_empty() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - - Ok(()) -} diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index 63a6f9e6..0fce93e2 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,23 +1,22 @@ [package] name = "nlp" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] -xxhash-rust = { version = "0.8.18", features = ["xxh3"] } +xxhash-rust = { version = "0.8.19", features = ["xxh3"] } serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.12", features = ["serde"] } whatlang = "0.18" # Language detection rust-stemmers = "1.2" # Stemmers -jieba-rs = "0.10" # Chinese stemmer +jieba-rs = "0.11" # Chinese stemmer psl = "2" -maplit = "1.0.2" -hashify = "0.2.9" -rand = "0.10.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } +memchr = "2.8" +hashify = "0.3" +rand = "0.10.3" +rkyv.workspace = true [features] -test_mode = [] [dev-dependencies] diff --git a/crates/nlp/build.rs b/crates/nlp/build.rs new file mode 100644 index 00000000..3f921c77 --- /dev/null +++ b/crates/nlp/build.rs @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{env, fmt::Write, fs, io, path::Path}; + +const BMP_WORDS: u32 = 0x10000 / 64; + +fn write_table(out: &mut String, name: &str, property: fn(char) -> bool) { + let _ = write!(out, "static {name}: [u64; {BMP_WORDS}] = ["); + for index in 0..BMP_WORDS { + let word = (0..64).fold(0u64, |word, bit| { + if char::from_u32(index * 64 + bit).is_some_and(property) { + word | (1 << bit) + } else { + word + } + }); + let _ = write!(out, "{word:#x},"); + } + out.push_str("];\n"); +} + +fn main() -> io::Result<()> { + let mut tables = String::with_capacity(96 * 1024); + write_table(&mut tables, "ALPHABETIC_BMP", char::is_alphabetic); + write_table(&mut tables, "ALPHANUMERIC_BMP", char::is_alphanumeric); + write_table(&mut tables, "UPPERCASE_BMP", char::is_uppercase); + + let out_dir = env::var_os("OUT_DIR").ok_or_else(|| io::Error::other("OUT_DIR is not set"))?; + fs::write(Path::new(&out_dir).join("unicode_bmp.rs"), tables)?; + + println!("cargo::rustc-check-cfg=cfg(swar_alnum)"); + if matches!( + env::var("CARGO_CFG_TARGET_ARCH").as_deref(), + Ok("x86_64" | "aarch64") + ) { + println!("cargo::rustc-cfg=swar_alnum"); + } + println!("cargo::rerun-if-changed=build.rs"); + Ok(()) +} diff --git a/crates/nlp/src/classifier/feature.rs b/crates/nlp/src/classifier/feature.rs index dfd9863a..b15d620b 100644 --- a/crates/nlp/src/classifier/feature.rs +++ b/crates/nlp/src/classifier/feature.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::collections::HashMap; +use std::{collections::HashMap, hash::BuildHasher}; use xxhash_rust::xxh3::xxh3_64_with_seed; #[derive(Debug)] @@ -113,16 +113,58 @@ pub trait FeatureBuilder { fn build_feature(&self, bytes: &[u8], weight: f32) -> Self::Feature; - fn scale(&self, features: &mut HashMap) { + fn scale(&self, features: &mut HashMap) { // Log frequency scaling for x in features.values_mut() { *x = x.ln_1p(); } } - fn build( + fn build_part( &self, - features_in: &HashMap, + features_in: &HashMap, + account_id: Option, + features_out: &mut Vec, + ) { + features_out.clear(); + features_out.reserve(features_in.len()); + let mut buf = Vec::with_capacity(2 + 4 + 63); + for (feature, count) in features_in { + buf.clear(); + buf.extend_from_slice(&feature.prefix().to_be_bytes()); + buf.extend_from_slice(feature.value()); + if let Some(account_id) = account_id { + buf.extend_from_slice(&account_id.to_be_bytes()); + } + features_out.push(self.build_feature(&buf, *count)); + } + } + + fn build_base_and_account( + &self, + features_in: &HashMap, + account_id: u32, + base_out: &mut Vec, + account_out: &mut Vec, + ) { + base_out.clear(); + base_out.reserve(features_in.len()); + account_out.clear(); + account_out.reserve(features_in.len()); + let mut buf = Vec::with_capacity(2 + 4 + 63); + for (feature, count) in features_in { + buf.clear(); + buf.extend_from_slice(&feature.prefix().to_be_bytes()); + buf.extend_from_slice(feature.value()); + base_out.push(self.build_feature(&buf, *count)); + buf.extend_from_slice(&account_id.to_be_bytes()); + account_out.push(self.build_feature(&buf, *count)); + } + } + + fn build( + &self, + features_in: &HashMap, account_id: Option, l2_normalize: bool, ) -> Vec { diff --git a/crates/nlp/src/classifier/model.rs b/crates/nlp/src/classifier/model.rs index 414357bb..bac22573 100644 --- a/crates/nlp/src/classifier/model.rs +++ b/crates/nlp/src/classifier/model.rs @@ -9,6 +9,29 @@ use crate::classifier::{ sigmoid, }; +#[derive(Debug, Clone, Copy, Default)] +pub struct Score { + z: f32, + squares: f64, +} + +impl Score { + pub fn combine(self, other: Score) -> Score { + Score { + z: self.z + other.z, + squares: self.squares + other.squares, + } + } + + fn scaled(self, l2_normalize: bool) -> f32 { + if l2_normalize && self.squares > 0.0 { + self.z / self.squares.sqrt() as f32 + } else { + self.z + } + } +} + #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct FhClassifier { pub(crate) parameters: Vec, @@ -33,6 +56,19 @@ impl FhClassifier { sigmoid(z + self.bias) } + pub fn score(&self, features: &[FhFeature]) -> Score { + let mut score = Score::default(); + for f in features { + score.z += self.parameters[f.idx] * f.weight; + score.squares += f64::from(f.weight) * f64::from(f.weight); + } + score + } + + pub fn predict_proba_score(&self, score: Score, l2_normalize: bool) -> f32 { + sigmoid(score.scaled(l2_normalize) + self.bias) + } + pub fn predict(&self, features: &[FhFeature]) -> f32 { if self.predict_proba_sample(features) > 0.7 { 1.0 @@ -78,6 +114,22 @@ impl CcfhClassifier { sigmoid(z + self.bias) } + pub fn score(&self, features: &[CcfhFeature]) -> Score { + let mut score = Score::default(); + for f in features { + let q = self.indicators[f.idx_i]; + let v1 = self.parameters[f.idx_w1]; + let v2 = self.parameters[f.idx_w2]; + score.z += (q * v1 + (1.0 - q) * v2) * f.weight; + score.squares += f64::from(f.weight) * f64::from(f.weight); + } + score + } + + pub fn predict_proba_score(&self, score: Score, l2_normalize: bool) -> f32 { + sigmoid(score.scaled(l2_normalize) + self.bias) + } + pub fn predict(&self, features: &[CcfhFeature]) -> f32 { if self.predict_proba_sample(features) >= 0.5 { 1.0 @@ -107,3 +159,4 @@ impl CcfhClassifier { !self.parameters.is_empty() } } + diff --git a/crates/nlp/src/classifier/reservoir.rs b/crates/nlp/src/classifier/reservoir.rs index 21827243..8ad118ef 100644 --- a/crates/nlp/src/classifier/reservoir.rs +++ b/crates/nlp/src/classifier/reservoir.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use rand::{RngExt, seq::IndexedRandom}; +use rand::{Rng, RngExt, seq::IndexedRandom}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub struct SampleReservoir { @@ -19,7 +19,13 @@ pub struct SampleReservoirClass { } impl SampleReservoir { - pub fn update_reservoir(&mut self, item: &T, is_spam: bool, capacity: usize) { + pub fn update_reservoir( + &mut self, + item: &T, + is_spam: bool, + capacity: usize, + rng: &mut impl Rng, + ) { let class = if is_spam { &mut self.spam } else { @@ -32,7 +38,7 @@ impl SampleReservoir { class.buffer.push(item.clone()); } else if let Some(buf) = class .buffer - .get_mut(rand::rng().random_range(0..class.total_seen as usize)) + .get_mut(rng.random_range(0..class.total_seen as usize)) { *buf = item.clone(); } @@ -52,6 +58,7 @@ impl SampleReservoir { &mut self, count_needed: usize, is_spam: bool, + rng: &mut impl Rng, ) -> impl Iterator { (if is_spam { &mut self.spam @@ -59,7 +66,7 @@ impl SampleReservoir { &mut self.ham }) .buffer - .sample(&mut rand::rng(), count_needed) + .sample(rng, count_needed) } pub fn remove_sample(&mut self, item: &T, is_spam: bool) { diff --git a/crates/nlp/src/classifier/sgd.rs b/crates/nlp/src/classifier/sgd.rs index 08c04642..fde75ad3 100644 --- a/crates/nlp/src/classifier/sgd.rs +++ b/crates/nlp/src/classifier/sgd.rs @@ -120,332 +120,3 @@ impl Optimizer for Sgd { } } -#[cfg(test)] -pub mod tests { - use crate::classifier::{ - Optimizer, - adam::Adam, - feature::{ - CcfhFeature, CcfhFeatureBuilder, FeatureBuilder, FhFeature, FhFeatureBuilder, Sample, - UnprocessedFeature, - }, - ftrl::Ftrl, - train::{CcfhTrainer, FhTrainer}, - }; - use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; - use std::{ - collections::HashMap, - fs::File, - io::{BufRead, BufReader}, - time::Instant, - }; - - #[ignore] - #[test] - fn text_classifier() { - let reader = BufReader::new( - File::open("/Users/me/code/playground/phishing_email.csv") - .expect("Could not open file"), - ); - let mut samples = Vec::with_capacity(1024); - - let time = Instant::now(); - - for line in reader.lines().skip(1) { - let line = line.unwrap(); - let (text, class) = line.trim().rsplit_once(',').unwrap(); - //let (class, text) = line.trim().split_once(',').unwrap(); - let text = text.trim_start_matches('"').trim_end_matches('"'); - - samples.push((text.to_string(), class == "1")); - } - - println!("Loaded {} samples in {:?}", samples.len(), time.elapsed()); - - samples.shuffle(&mut StdRng::seed_from_u64(42)); - - let (train_samples, test_samples) = train_test_split(&samples, 0.2); - - println!( - "Training samples: {}, Testing samples: {}", - train_samples.len(), - test_samples.len() - ); - - const FH_SIZE: usize = 16; - const CCFH_SIZE: usize = FH_SIZE - 2; - let mut rng = StdRng::seed_from_u64(42); - - let fh_builder = FhFeatureBuilder { - weight_mask: (1 << FH_SIZE) - 1, - }; - let mut fh_train_samples = build_fh_samples(train_samples.as_slice(), &fh_builder); - fh_train_samples.shuffle(&mut rng); - let fh_test_samples = build_fh_samples(test_samples.as_slice(), &fh_builder); - let ccfh_builder = CcfhFeatureBuilder { - weight_mask: (1 << FH_SIZE) - 1, - indicator_mask: (1 << CCFH_SIZE) - 1, - }; - let mut ccfh_train_samples = build_ccfh_samples(train_samples.as_slice(), &ccfh_builder); - ccfh_train_samples.shuffle(&mut rng); - let ccfh_test_samples = build_ccfh_samples(test_samples.as_slice(), &ccfh_builder); - - fh_model_stats( - "FTRL", - FhTrainer::new(Ftrl::new(1 << FH_SIZE)), - &fh_train_samples, - &fh_test_samples, - ); - - ccfh_model_stats( - "FTRL + FTRL", - CcfhTrainer::new( - Ftrl::new(1 << FH_SIZE), - Ftrl::new(1 << CCFH_SIZE).with_initial_weights(0.5), - ), - &ccfh_train_samples, - &ccfh_test_samples, - ); - - fh_model_stats( - "Adam", - FhTrainer::new(Adam::new(1 << FH_SIZE, 0.01)), - &fh_train_samples, - &fh_test_samples, - ); - - ccfh_model_stats( - "Adam + Adam", - CcfhTrainer::new( - Adam::new(1 << FH_SIZE, 0.01), - Adam::new(1 << CCFH_SIZE, 0.01).with_initial_weights(0.5), - ), - &ccfh_train_samples, - &ccfh_test_samples, - ); - - /*fh_model_stats( - "SGD", - FhTrainer::new(Sgd::new(1 << FH_SIZE, 0.0001, 0.0, 0.0001)), - &fh_train_samples, - &fh_test_samples, - ); - - ccfh_model_stats( - "FTRL + SGD", - CcfhTrainer::new( - Ftrl::new(1 << FH_SIZE), - Sgd::new(1 << CCFH_SIZE, 0.0001, 0.0, 0.0001).with_initial_parameters(0.5), - ), - &ccfh_train_samples, - &ccfh_test_samples, - );*/ - } - - fn fh_model_stats( - name: &str, - mut model: FhTrainer, - train_samples: &[Sample], - test_samples: &[Sample], - ) { - print!("⏳ Training {}... ", name); - let time = Instant::now(); - let mut batch = Vec::new(); - for sample in train_samples { - batch.push(sample); - if batch.len() == 128 { - model.fit(&mut batch, 5); - batch.clear(); - } - } - if !batch.is_empty() { - model.fit(&mut batch, 5); - } - println!(" trained in {:?}", time.elapsed()); - let y_pred = model - .build_classifier() - .predict_batch(test_samples.iter().map(|s| &s.features)); - let y_train: Vec = test_samples.iter().map(|s| s.class).collect(); - println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred)); - println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0)); - println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0)); - println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0)); - } - - fn ccfh_model_stats( - name: &str, - mut model: CcfhTrainer, - train_samples: &[Sample], - test_samples: &[Sample], - ) { - print!("⏳ Training {}... ", name); - let time = Instant::now(); - let mut batch = Vec::new(); - for sample in train_samples { - batch.push(sample); - if batch.len() == 128 { - model.fit(&mut batch, 5); - batch.clear(); - } - } - if !batch.is_empty() { - model.fit(&mut batch, 5); - } - println!(" trained in {:?}", time.elapsed()); - let y_pred = model - .build_classifier() - .predict_batch(test_samples.iter().map(|s| &s.features)); - let y_train: Vec = test_samples.iter().map(|s| s.class).collect(); - println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred)); - println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0)); - println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0)); - println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0)); - } - - fn accuracy_score(y_true: &[f32], y_pred: &[f32]) -> f32 { - y_true - .iter() - .zip(y_pred.iter()) - .filter(|(true_val, pred_val)| **true_val == **pred_val) - .count() as f32 - / y_true.len() as f32 - } - - fn precision_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { - let true_positives = y_true - .iter() - .zip(y_pred.iter()) - .filter(|(true_val, pred_val)| { - **pred_val == positive_class && **true_val == positive_class - }) - .count() as f32; - - let predicted_positives = y_pred - .iter() - .filter(|pred_val| **pred_val == positive_class) - .count() as f32; - - if predicted_positives == 0.0 { - 0.0 - } else { - true_positives / predicted_positives - } - } - - fn recall_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { - let true_positives = y_true - .iter() - .zip(y_pred.iter()) - .filter(|(true_val, pred_val)| { - **pred_val == positive_class && **true_val == positive_class - }) - .count() as f32; - - let actual_positives = y_true - .iter() - .filter(|true_val| **true_val == positive_class) - .count() as f32; - - if actual_positives == 0.0 { - 0.0 - } else { - true_positives / actual_positives - } - } - - fn f1_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { - let precision = precision_score(y_true, y_pred, positive_class); - let recall = recall_score(y_true, y_pred, positive_class); - - if precision + recall == 0.0 { - 0.0 - } else { - 2.0 * (precision * recall) / (precision + recall) - } - } - - #[allow(clippy::type_complexity)] - pub fn train_test_split( - data: &[(String, bool)], - test_size: f32, - ) -> (Vec<(&String, bool)>, Vec<(&String, bool)>) { - let mut class_0: Vec<(&String, bool)> = Vec::new(); - let mut class_1: Vec<(&String, bool)> = Vec::new(); - - for (sample, class) in data { - if !*class { - class_0.push((sample, *class)); - } else { - class_1.push((sample, *class)); - } - } - - let test_count_0 = (class_0.len() as f32 * test_size).round() as usize; - let test_count_1 = (class_1.len() as f32 * test_size).round() as usize; - - let (test_0, train_0) = class_0.split_at(test_count_0); - let (test_1, train_1) = class_1.split_at(test_count_1); - - let mut train = Vec::new(); - let mut test = Vec::new(); - - train.extend_from_slice(train_0); - train.extend_from_slice(train_1); - test.extend_from_slice(test_0); - test.extend_from_slice(test_1); - - (train, test) - } - - pub fn build_fh_samples( - data: &[(&String, bool)], - builder: &FhFeatureBuilder, - ) -> Vec> { - let mut samples = Vec::with_capacity(data.len()); - - for (text, class) in data { - let mut sample: HashMap = HashMap::new(); - for word in text.split_whitespace() { - *sample.entry(word.to_string()).or_default() += 1.0; - } - builder.scale(&mut sample); - samples.push(Sample { - features: builder.build(&sample, 12345.into(), true), - class: if *class { 1.0 } else { 0.0 }, - }); - } - - samples - } - - pub fn build_ccfh_samples( - data: &[(&String, bool)], - builder: &CcfhFeatureBuilder, - ) -> Vec> { - let mut samples = Vec::with_capacity(data.len()); - - for (text, class) in data { - let mut sample: HashMap = HashMap::new(); - for word in text.split_whitespace() { - *sample.entry(word.to_string()).or_default() += 1.0; - } - builder.scale(&mut sample); - samples.push(Sample { - features: builder.build(&sample, 12345.into(), true), - class: if *class { 1.0 } else { 0.0 }, - }); - } - - samples - } - - impl UnprocessedFeature for String { - fn prefix(&self) -> u16 { - 0 - } - - fn value(&self) -> &[u8] { - self.as_bytes() - } - } -} diff --git a/crates/nlp/src/language/detect.rs b/crates/nlp/src/language/detect.rs index 657f806a..234cb4dc 100644 --- a/crates/nlp/src/language/detect.rs +++ b/crates/nlp/src/language/detect.rs @@ -6,9 +6,15 @@ use super::Language; use ahash::AHashMap; -use whatlang::{Lang, detect}; +use whatlang::{Lang, Script, detect, detect_script}; pub const MIN_LANGUAGE_SCORE: f64 = 0.6; +pub const MAX_LANGUAGE_SAMPLE: usize = 4 << 10; +const SAMPLE_WINDOWS: usize = 3; +const SAMPLE_WINDOW: usize = MAX_LANGUAGE_SAMPLE / SAMPLE_WINDOWS; + +#[cfg(stalwart_tests)] +pub static DETECT_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); #[derive(Debug)] struct WeightedAverage { @@ -71,166 +77,112 @@ impl LanguageDetector { } pub fn detect_single(text: &str) -> Option<(Language, f64)> { - detect(text).map(|info| { - ( - match info.lang() { - Lang::Epo => Language::Esperanto, - Lang::Eng => Language::English, - Lang::Rus => Language::Russian, - Lang::Cmn => Language::Mandarin, - Lang::Spa => Language::Spanish, - Lang::Por => Language::Portuguese, - Lang::Ita => Language::Italian, - Lang::Ben => Language::Bengali, - Lang::Fra => Language::French, - Lang::Deu => Language::German, - Lang::Ukr => Language::Ukrainian, - Lang::Kat => Language::Georgian, - Lang::Ara => Language::Arabic, - Lang::Hin => Language::Hindi, - Lang::Jpn => Language::Japanese, - Lang::Heb => Language::Hebrew, - Lang::Yid => Language::Yiddish, - Lang::Pol => Language::Polish, - Lang::Amh => Language::Amharic, - Lang::Jav => Language::Javanese, - Lang::Kor => Language::Korean, - Lang::Nob => Language::Bokmal, - Lang::Dan => Language::Danish, - Lang::Swe => Language::Swedish, - Lang::Fin => Language::Finnish, - Lang::Tur => Language::Turkish, - Lang::Nld => Language::Dutch, - Lang::Hun => Language::Hungarian, - Lang::Ces => Language::Czech, - Lang::Ell => Language::Greek, - Lang::Bul => Language::Bulgarian, - Lang::Bel => Language::Belarusian, - Lang::Mar => Language::Marathi, - Lang::Kan => Language::Kannada, - Lang::Ron => Language::Romanian, - Lang::Slv => Language::Slovene, - Lang::Hrv => Language::Croatian, - Lang::Srp => Language::Serbian, - Lang::Mkd => Language::Macedonian, - Lang::Lit => Language::Lithuanian, - Lang::Lav => Language::Latvian, - Lang::Est => Language::Estonian, - Lang::Tam => Language::Tamil, - Lang::Vie => Language::Vietnamese, - Lang::Urd => Language::Urdu, - Lang::Tha => Language::Thai, - Lang::Guj => Language::Gujarati, - Lang::Uzb => Language::Uzbek, - Lang::Pan => Language::Punjabi, - Lang::Aze => Language::Azerbaijani, - Lang::Ind => Language::Indonesian, - Lang::Tel => Language::Telugu, - Lang::Pes => Language::Persian, - Lang::Mal => Language::Malayalam, - Lang::Ori => Language::Oriya, - Lang::Mya => Language::Burmese, - Lang::Nep => Language::Nepali, - Lang::Sin => Language::Sinhalese, - Lang::Khm => Language::Khmer, - Lang::Tuk => Language::Turkmen, - Lang::Aka => Language::Akan, - Lang::Zul => Language::Zulu, - Lang::Sna => Language::Shona, - Lang::Afr => Language::Afrikaans, - Lang::Lat => Language::Latin, - Lang::Slk => Language::Slovak, - Lang::Cat => Language::Catalan, - Lang::Tgl => Language::Tagalog, - Lang::Hye => Language::Armenian, - _ => Language::Unknown, - }, - info.confidence(), - ) - }) + #[cfg(stalwart_tests)] + DETECT_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut sample = String::new(); + if text.len() > MAX_LANGUAGE_SAMPLE { + sample.reserve(MAX_LANGUAGE_SAMPLE + SAMPLE_WINDOWS); + let tail = text.len() - SAMPLE_WINDOW; + for start in [0, tail / 2, tail] { + let start = text.floor_char_boundary(start); + if let Some(window) = + text.get(start..text.floor_char_boundary(start + SAMPLE_WINDOW)) + { + sample.push_str(window); + sample.push('\n'); + } + } + } + detect(if sample.is_empty() { text } else { &sample }) + .map(|info| (Language::from(info.lang()), info.confidence())) + } + + pub(crate) fn detect_by_script(text: &str) -> Option { + match detect_script(text)? { + Script::Mandarin => detect(text).map(|info| info.lang()), + script => match script.langs() { + [lang] => Some(*lang), + _ => None, + }, + } + .map(Language::from) } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn detect_languages() { - let inputs = [ - ( - "The quick brown fox jumps over the lazy dog", - Language::English, - ), - ( - "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!", - Language::Spanish, - ), - ( - "Ma la volpe col suo balzo ha raggiunto il quieto Fido", - Language::Italian, - ), - ( - "Jaz em prisão bota que vexa dez cegonhas felizes", - Language::Portuguese, - ), - ( - "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich", - Language::German, - ), - ("עטלף אבק נס דרך מזגן שהתפוצץ כי חם", Language::Hebrew), - ( - "Съешь ещё этих мягких французских булок, да выпей же чаю", - Language::Russian, - ), - ( - "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!", - Language::Ukrainian, - ), - ( - "Љубазни фењерџија чађавог лица хоће да ми покаже штос", - Language::Serbian, - ), - ( - "Pijamalı hasta yağız şoföre çabucak güvendi", - Language::Turkish, - ), - ("己所不欲,勿施于人。", Language::Mandarin), - ("井の中の蛙大海を知らず", Language::Japanese), - ("시작이 반이다", Language::Korean), - ]; - - let mut detector = LanguageDetector::new(); - - for input in inputs.iter() { - assert_eq!(detector.detect(input.0, 0.0), input.1); +impl From for Language { + fn from(lang: Lang) -> Self { + match lang { + Lang::Epo => Language::Esperanto, + Lang::Eng => Language::English, + Lang::Rus => Language::Russian, + Lang::Cmn => Language::Mandarin, + Lang::Spa => Language::Spanish, + Lang::Por => Language::Portuguese, + Lang::Ita => Language::Italian, + Lang::Ben => Language::Bengali, + Lang::Fra => Language::French, + Lang::Deu => Language::German, + Lang::Ukr => Language::Ukrainian, + Lang::Kat => Language::Georgian, + Lang::Ara => Language::Arabic, + Lang::Hin => Language::Hindi, + Lang::Jpn => Language::Japanese, + Lang::Heb => Language::Hebrew, + Lang::Yid => Language::Yiddish, + Lang::Pol => Language::Polish, + Lang::Amh => Language::Amharic, + Lang::Jav => Language::Javanese, + Lang::Kor => Language::Korean, + Lang::Nob => Language::Bokmal, + Lang::Dan => Language::Danish, + Lang::Swe => Language::Swedish, + Lang::Fin => Language::Finnish, + Lang::Tur => Language::Turkish, + Lang::Nld => Language::Dutch, + Lang::Hun => Language::Hungarian, + Lang::Ces => Language::Czech, + Lang::Ell => Language::Greek, + Lang::Bul => Language::Bulgarian, + Lang::Bel => Language::Belarusian, + Lang::Mar => Language::Marathi, + Lang::Kan => Language::Kannada, + Lang::Ron => Language::Romanian, + Lang::Slv => Language::Slovene, + Lang::Hrv => Language::Croatian, + Lang::Srp => Language::Serbian, + Lang::Mkd => Language::Macedonian, + Lang::Lit => Language::Lithuanian, + Lang::Lav => Language::Latvian, + Lang::Est => Language::Estonian, + Lang::Tam => Language::Tamil, + Lang::Vie => Language::Vietnamese, + Lang::Urd => Language::Urdu, + Lang::Tha => Language::Thai, + Lang::Guj => Language::Gujarati, + Lang::Uzb => Language::Uzbek, + Lang::Pan => Language::Punjabi, + Lang::Aze => Language::Azerbaijani, + Lang::Ind => Language::Indonesian, + Lang::Tel => Language::Telugu, + Lang::Pes => Language::Persian, + Lang::Mal => Language::Malayalam, + Lang::Ori => Language::Oriya, + Lang::Mya => Language::Burmese, + Lang::Nep => Language::Nepali, + Lang::Sin => Language::Sinhalese, + Lang::Khm => Language::Khmer, + Lang::Tuk => Language::Turkmen, + Lang::Aka => Language::Akan, + Lang::Zul => Language::Zulu, + Lang::Sna => Language::Shona, + Lang::Afr => Language::Afrikaans, + Lang::Lat => Language::Latin, + Lang::Slk => Language::Slovak, + Lang::Cat => Language::Catalan, + Lang::Tgl => Language::Tagalog, + Lang::Hye => Language::Armenian, + _ => Language::Unknown, } } - - #[test] - fn weighted_language() { - let mut detector = LanguageDetector::new(); - for lang in [ - (Language::Spanish, 0.5, 70), - (Language::Japanese, 0.2, 100), - (Language::Japanese, 0.3, 100), - (Language::Japanese, 0.4, 200), - (Language::English, 0.7, 50), - ] - .iter() - { - let w = detector - .lang_detected - .entry(lang.0) - .or_insert_with(|| WeightedAverage { - weight: 0, - confidence: 0.0, - occurrences: 0, - }); - w.occurrences += 1; - w.weight += lang.2; - w.confidence += lang.1 * lang.2 as f64; - } - assert_eq!(detector.most_frequent_language(), Some(Language::Japanese)); - } } + diff --git a/crates/nlp/src/language/mod.rs b/crates/nlp/src/language/mod.rs index 7c1b7869..8c5a801d 100644 --- a/crates/nlp/src/language/mod.rs +++ b/crates/nlp/src/language/mod.rs @@ -16,6 +16,10 @@ use crate::tokenizers::{ }; use std::borrow::Cow; +const MIN_QUERY_LANGUAGE_SCORE: f64 = 0.3; +const MAX_SCRIPT_DETECTION_WORDS: usize = 3; +const MAX_SCRIPT_DETECTION_BYTES: usize = 256; + pub type LanguageTokenizer<'x> = Box>> + 'x + Sync + Send>; impl Language { @@ -36,7 +40,7 @@ impl Language { Language::None => { Box::new( SpaceTokenizer::new(text, max_token_length).map(|word| Token { - word: word.into(), + word, from: 0, to: 0, }), @@ -210,62 +214,43 @@ impl Language { } impl Language { + pub fn split_prefix(text: &str) -> Option<(Language, &str)> { + let (code, text) = text.split_once(':')?; + Some((Language::from_iso_639(code)?, text)) + } + pub fn detect(text: String, default: Language) -> (String, Language) { - if let Some((l, t)) = text - .split_once(':') - .and_then(|(l, t)| (Language::from_iso_639(l)?, t).into()) - { + if let Some((l, t)) = Language::split_prefix(&text) { (t.to_string(), l) } else { - let l = LanguageDetector::detect_single(&text) - .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) - .unwrap_or(default); + let l = Language::detect_text(&text, default); (text, l) } } -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn iso_639_codes() { - for (code, expected) in [ - ("nl", Language::Dutch), - ("fi", Language::Finnish), - ("hu", Language::Hungarian), - ("tr", Language::Turkish), - ("ro", Language::Romanian), - ("nb", Language::Bokmal), - ("no", Language::Bokmal), - ("yi", Language::Yiddish), - ("eo", Language::Esperanto), - ("uk", Language::Ukrainian), - ("cs", Language::Czech), - ("sk", Language::Slovak), - ("bg", Language::Bulgarian), - ("be", Language::Belarusian), - ("th", Language::Thai), - ("mr", Language::Marathi), - ("kn", Language::Kannada), - ("jv", Language::Javanese), - ("en", Language::English), - ("pl", Language::Polish), - ] { - assert_eq!(Language::from_iso_639(code), Some(expected), "{code}"); + pub fn detect_non_ascii(text: String, default: Language) -> (String, Language) { + if let Some((l, t)) = Language::split_prefix(&text) { + (t.to_string(), l) + } else if text.is_ascii() { + (text, default) + } else if text.len() > MAX_SCRIPT_DETECTION_BYTES + || text + .split_whitespace() + .nth(MAX_SCRIPT_DETECTION_WORDS) + .is_some() + { + let l = Language::detect_text(&text, default); + (text, l) + } else { + let l = LanguageDetector::detect_by_script(&text).unwrap_or(default); + (text, l) } } - #[test] - fn iso_639_locale_suffixes() { - for code in ["nl_NL", "nl_BE", "nl_NL@euro", "nl-NL"] { - assert_eq!( - Language::from_iso_639(code), - Some(Language::Dutch), - "{code}" - ); - } - assert_eq!(Language::from_iso_639("xx_XX"), None); + fn detect_text(text: &str, default: Language) -> Language { + LanguageDetector::detect_single(text) + .and_then(|(l, c)| (c > MIN_QUERY_LANGUAGE_SCORE).then_some(l)) + .unwrap_or(default) } } + diff --git a/crates/nlp/src/language/search_snippet.rs b/crates/nlp/src/language/search_snippet.rs index c4ddfe6a..d5a9b633 100644 --- a/crates/nlp/src/language/search_snippet.rs +++ b/crates/nlp/src/language/search_snippet.rs @@ -5,6 +5,7 @@ */ use super::Language; +use std::collections::VecDeque; fn escape_char(c: char, string: &mut String) { match c { @@ -28,244 +29,219 @@ fn escape_char_len(c: char) -> usize { } } -pub struct Term { +const MAX_SNIPPET_LEN: usize = 255; +const MARK_LEN: usize = "".len() * 2; +const LEADING_CONTEXT_THRESHOLD: usize = 240; +const LEADING_CONTEXT_CHARS: usize = 40; +const LEADING_CONTEXT_WORDS: usize = 3; +const MAX_TOKEN_LEN: usize = 200; +const MAX_WINDOW_CAPACITY: usize = 64; + +struct Term { offset: usize, len: usize, } +struct SnippetWriter<'x> { + text: &'x str, + snippet: String, + last_end: Option, + stop_offset: usize, +} + +enum Flow { + Continue, + Full, +} + pub fn generate_snippet( text: &str, needles: &[impl AsRef], language: Language, is_exact: bool, ) -> Option { - let mut terms = Vec::new(); + let mut writer = SnippetWriter { + text, + snippet: String::new(), + last_end: None, + stop_offset: usize::MAX, + }; + let tokens = language.tokenize_text(text, MAX_TOKEN_LEN); + if is_exact { - let tokens = language.tokenize_text(text, 200).collect::>(); - for tokens in tokens.windows(needles.len()) { + if needles.is_empty() { + return None; + } + let mut window = VecDeque::with_capacity(needles.len().min(MAX_WINDOW_CAPACITY)); + for token in tokens { + if window.len() == needles.len() { + window.pop_front(); + } + window.push_back(token); + if window.len() < needles.len() { + continue; + } + if window + .front() + .is_some_and(|first| first.from >= writer.stop_offset) + { + break; + } if needles .iter() - .zip(tokens) + .zip(&window) .all(|(needle, token)| needle.as_ref() == token.word.as_ref()) { - for token in tokens { - terms.push(Term { + for token in &window { + if let Flow::Full = writer.push(Term { offset: token.from, len: token.to - token.from, - }); + })? { + return Some(writer.snippet); + } } } } } else { - for token in language.tokenize_text(text, 200) { + for token in tokens { + if token.from >= writer.stop_offset { + break; + } if needles.iter().any(|needle| { let needle = needle.as_ref(); needle == token.word.as_ref() || needle.len() > 2 && token.word.contains(needle) - }) { - terms.push(Term { - offset: token.from, - len: token.to - token.from, - }); + }) && let Flow::Full = writer.push(Term { + offset: token.from, + len: token.to - token.from, + })? { + return Some(writer.snippet); } } } - if terms.is_empty() { - return None; + + writer.finish() +} + +impl SnippetWriter<'_> { + fn push(&mut self, term: Term) -> Option { + match self.last_end { + Some(end) => { + if let Flow::Full = self.write_gap(end, term.offset)? { + return Some(Flow::Full); + } + } + None => self.write_leading_context(term.offset)?, + } + + if self.snippet.len() + MARK_LEN + term.len + 1 > MAX_SNIPPET_LEN { + return Some(Flow::Full); + } + + let end = term.offset + term.len; + self.snippet.push_str(""); + self.snippet.push_str(self.text.get(term.offset..end)?); + self.snippet.push_str(""); + self.last_end = Some(end); + self.stop_offset = self.overflow_offset(end); + + Some(Flow::Continue) } - let mut snippet = String::with_capacity(text.len()); - let start_offset = terms.first()?.offset; + fn finish(mut self) -> Option { + let end = self.last_end?; + self.write_gap(end, self.text.len())?; + Some(self.snippet) + } - if start_offset > 0 { - let mut word_count = 0; - let mut from_offset = 0; - let mut last_is_space = false; + fn write_leading_context(&mut self, start_offset: usize) -> Option<()> { + self.snippet + .reserve(self.text.len().min(MAX_SNIPPET_LEN + 1)); - if text.len() > 240 { - for (pos, char) in text.get(0..start_offset)?.char_indices().rev() { - // Add up to 2 words or 40 characters of context - if char.is_whitespace() { - if !last_is_space { - word_count += 1; - if word_count == 3 { - break; + if start_offset > 0 { + let mut word_count = 0; + let mut from_offset = 0; + let mut last_is_space = false; + + if self.text.len() > LEADING_CONTEXT_THRESHOLD { + for (pos, char) in self.text.get(0..start_offset)?.char_indices().rev() { + if char.is_whitespace() { + if !last_is_space { + word_count += 1; + if word_count == LEADING_CONTEXT_WORDS { + break; + } + last_is_space = true; } - last_is_space = true; + } else { + last_is_space = false; } - } else { + from_offset = pos; + if start_offset - from_offset >= LEADING_CONTEXT_CHARS { + break; + } + } + } + + last_is_space = false; + for char in self.text.get(from_offset..start_offset)?.chars() { + if !char.is_whitespace() { last_is_space = false; - } - from_offset = pos; - if start_offset - from_offset >= 40 { - break; - } - } - } - - last_is_space = false; - for char in text.get(from_offset..start_offset)?.chars() { - if !char.is_whitespace() { - last_is_space = false; - } else { - if last_is_space { - continue; - } - last_is_space = true; - } - escape_char(char, &mut snippet); - } - } - - let mut terms = terms.iter().peekable(); - - 'outer: while let Some(term) = terms.next() { - if snippet.len() + ("".len() * 2) + term.len + 1 > 255 { - break; - } - - snippet.push_str(""); - snippet.push_str(text.get(term.offset..term.offset + term.len)?); - snippet.push_str(""); - - let next_offset = if let Some(next_term) = terms.peek() { - next_term.offset - } else { - text.len() - }; - - let mut last_is_space = false; - for char in text.get(term.offset + term.len..next_offset)?.chars() { - if !char.is_whitespace() { - last_is_space = false; - } else { - if last_is_space { - continue; - } - last_is_space = true; - } - - if snippet.len() + escape_char_len(char) <= 255 { - escape_char(char, &mut snippet); - } else { - break 'outer; - } - } - } - - Some(snippet) -} - -#[cfg(test)] -mod tests { - use crate::language::{Language, search_snippet::generate_snippet}; - - #[test] - fn search_snippets() { - let inputs = [ - ( - vec![ - "Help a friend from Abidjan Côte d'Ivoire", - concat!( - "When my mother died when she was given birth to me, my father took me so ", - "special because I am motherless. Before the death of my late father on 22nd June ", - "2013 in a private hospital here in Abidjan Côte d'Ivoire. He secretly called me on his ", - "bedside and told me that he has a sum of $7.5M (Seven Million five Hundred ", - "Thousand Dollars) left in a suspense account in a local bank here in Abidjan Côte ", - "d'Ivoire, that he used my name as his only daughter for the next of kin in deposit of ", - "the fund. ", - "I am 24year old. Dear I am honorably seeking your assistance in the following ways. ", - "1) To provide any bank account where this money would be transferred into. ", - "2) To serve as the guardian of this fund. ", - "3) To make arrangement for me to come over to your country to further my ", - "education and to secure a residential permit for me in your country. ", - "Moreover, I am willing to offer you 30 percent of the total sum as compensation for ", - "your effort input after the successful transfer of this fund to your nominated ", - "account overseas." - ), - ], - vec![ - ( - vec!["côte"], - vec![ - "Help a friend from Abidjan Côte d'Ivoire", - concat!( - "in Abidjan Côte d'Ivoire. He secretly called me on his bedside ", - "and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand ", - "Dollars) left in a suspense account in a local bank here in Abidjan ", - "Côte d'Ivoire, that " - ), - ], - ), - ( - vec!["your", "country"], - vec![concat!( - "honorably seeking your assistance in the following ways. ", - "1) To provide any bank account where this money would be transferred into. 2) ", - "To serve as the guardian of this fund. 3) To make arrangement for me to come ", - "over to your " - )], - ), - ( - vec!["overseas"], - vec!["nominated account overseas."], - ), - ], - ), - ( - vec![ - "孫子兵法", - concat!( - "<\"孫子兵法:\">", - "孫子曰:兵者,國之大事,死生之地,存亡之道,不可不察也。", - "孫子曰:凡用兵之法,馳車千駟,革車千乘,帶甲十萬;千里饋糧,則內外之費賓客之用,膠漆之材,", - "車甲之奉,日費千金,然後十萬之師舉矣。", - "孫子曰:凡用兵之法,全國為上,破國次之;全旅為上,破旅次之;全卒為上,破卒次之;全伍為上,破伍次之。", - "是故百戰百勝,非善之善者也;不戰而屈人之兵,善之善者也。", - "孫子曰:昔之善戰者,先為不可勝,以待敵之可勝,不可勝在己,可勝在敵。故善戰者,能為不可勝,不能使敵必可勝。", - "故曰:勝可知,而不可為。", - "兵者,詭道也。故能而示之不能,用而示之不用,近而示之遠,遠而示之近。利而誘之,亂而取之,實而備之,強而避之,", - "怒而撓之,卑而驕之,佚而勞之,親而離之。攻其無備,出其不意,此兵家之勝,不可先傳也。", - "夫未戰而廟算勝者,得算多也;未戰而廟算不勝者,得算少也;多算勝,少算不勝,而況於無算乎?吾以此觀之,勝負見矣。", - "孫子曰:凡治眾如治寡,分數是也。鬥眾如鬥寡,形名是也。三軍之眾,可使必受敵而無敗者,奇正是也。兵之所加,", - "如以碬投卵者,虛實是也。", - ), - ], - vec![ - ( - vec!["孫子兵法"], - vec![ - "孫子兵法", - concat!( - "<"孫子兵法:">孫子曰:兵者,國之大事,死生之地,存亡之道,", - "不可不察也。孫子曰:凡用兵之法,馳車千駟,革車千乘,帶甲十萬;千里饋糧,則內外之費賓客之用,膠" - ), - ], - ), - ( - vec!["孫子曰"], - vec![concat!( - "<"孫子兵法:">孫子曰:兵者,國之大事,死生之地,存亡之道,", - "不可不察也。孫子曰:凡用兵之法,馳車千駟,革車千乘,帶甲十萬;千里饋糧,則內外之費賓", - )], - ), - ], - ), - ]; - - for (parts, tests) in inputs { - for (needles, snippets) in tests { - let mut results = Vec::new(); - - for part in &parts { - if let Some(matched) = - generate_snippet(part, &needles, Language::English, false) - { - results.push(matched); + } else { + if last_is_space { + continue; } + last_is_space = true; } - - assert_eq!(snippets, results); + escape_char(char, &mut self.snippet); } } + + Some(()) + } + + fn write_gap(&mut self, from: usize, to: usize) -> Option { + let mut last_is_space = false; + for char in self.text.get(from..to)?.chars() { + if !char.is_whitespace() { + last_is_space = false; + } else { + if last_is_space { + continue; + } + last_is_space = true; + } + + if self.snippet.len() + escape_char_len(char) <= MAX_SNIPPET_LEN { + escape_char(char, &mut self.snippet); + } else { + return Some(Flow::Full); + } + } + + Some(Flow::Continue) + } + + fn overflow_offset(&self, from: usize) -> usize { + let Some(rest) = self.text.get(from..) else { + return usize::MAX; + }; + let mut len = self.snippet.len(); + let mut last_is_space = false; + for (pos, char) in rest.char_indices() { + if !char.is_whitespace() { + last_is_space = false; + } else { + if last_is_space { + continue; + } + last_is_space = true; + } + len += escape_char_len(char); + if len > MAX_SNIPPET_LEN { + return from + pos; + } + } + usize::MAX } } + diff --git a/crates/nlp/src/language/stemmer.rs b/crates/nlp/src/language/stemmer.rs index e69cd4c2..d1283a26 100644 --- a/crates/nlp/src/language/stemmer.rs +++ b/crates/nlp/src/language/stemmer.rs @@ -127,26 +127,3 @@ pub static STEMMER_MAP: &[Option] = &[ None, // None = 70, ]; -#[cfg(test)] -mod tests { - - use super::*; - - #[test] - fn stemmer() { - let inputs = [ - ( - "love loving lovingly loved lovely", - Language::English, - "love", - ), - ("querer queremos quer", Language::Spanish, "quer"), - ]; - - for (input, language, result) in inputs { - for token in Stemmer::new(input, language, 40) { - assert_eq!(token.stemmed_word.unwrap_or(token.word), result); - } - } - } -} diff --git a/crates/nlp/src/tokenizers/chinese.rs b/crates/nlp/src/tokenizers/chinese.rs index 8c4a607b..e465e2e2 100644 --- a/crates/nlp/src/tokenizers/chinese.rs +++ b/crates/nlp/src/tokenizers/chinese.rs @@ -93,110 +93,3 @@ where } } -#[cfg(test)] -mod tests { - use crate::tokenizers::{Token, chinese::ChineseTokenizer, word::WordTokenizer}; - - #[test] - fn chinese_tokenizer() { - assert_eq!( - ChineseTokenizer::new(WordTokenizer::new( - "孫子曰:兵者,國之大事,死生之地,存亡之道,不可不察也。", - 40 - ),) - .collect::>(), - vec![ - Token { - word: "孫".into(), - from: 0, - to: 3 - }, - Token { - word: "子".into(), - from: 3, - to: 6 - }, - Token { - word: "曰".into(), - from: 6, - to: 9 - }, - Token { - word: "兵".into(), - from: 12, - to: 15 - }, - Token { - word: "者".into(), - from: 15, - to: 18 - }, - Token { - word: "國".into(), - from: 21, - to: 24 - }, - Token { - word: "之".into(), - from: 24, - to: 27 - }, - Token { - word: "大事".into(), - from: 27, - to: 33 - }, - Token { - word: "死".into(), - from: 36, - to: 39 - }, - Token { - word: "生".into(), - from: 39, - to: 42 - }, - Token { - word: "之".into(), - from: 42, - to: 45 - }, - Token { - word: "地".into(), - from: 45, - to: 48 - }, - Token { - word: "存亡".into(), - from: 51, - to: 57 - }, - Token { - word: "之".into(), - from: 57, - to: 60 - }, - Token { - word: "道".into(), - from: 60, - to: 63 - }, - Token { - word: "不可不".into(), - from: 66, - to: 75 - }, - Token { - word: "察".into(), - from: 75, - to: 78 - }, - Token { - word: "也".into(), - from: 78, - to: 81 - } - ] - ); - } -} diff --git a/crates/nlp/src/tokenizers/japanese.rs b/crates/nlp/src/tokenizers/japanese.rs index 123f536d..47738d0e 100644 --- a/crates/nlp/src/tokenizers/japanese.rs +++ b/crates/nlp/src/tokenizers/japanese.rs @@ -5,11 +5,16 @@ */ use super::{InnerToken, Token}; -use maplit::hashmap; -use std::collections::HashMap; +use ahash::AHashMap; use std::vec::IntoIter; use std::{hash::Hash, sync::LazyLock}; +macro_rules! score_map { + ($($key:expr => $value:expr),* $(,)?) => { + [$(($key, $value)),*].into_iter().collect::>() + }; +} + pub struct JapaneseTokenizer<'x, T, I> where T: Iterator>, @@ -55,7 +60,7 @@ where let token_from = token_to; token_to += word.len(); Token { - word: I::new_alphabetic(word.to_string()), + word: I::new_alphabetic(word), from: token_from, to: token_to, } @@ -74,8 +79,8 @@ where const BIAS: i32 = -332; -fn get_score(d: &HashMap, s: &T) -> i32 { - d.get(s).cloned().unwrap_or(0) +fn get_score(d: &AHashMap, s: &T) -> i32 { + d.get(s).copied().unwrap_or(0) } fn get_ctype(c: char) -> char { @@ -167,13 +172,12 @@ pub fn tokenize(s: &str) -> Vec { p.push(if score < 0 { 'O' } else { 'B' }); if 0 < score { - result.push(word.clone()); - word.clear(); + result.push(std::mem::take(&mut word)); } word.push(segments[index]); } - result.push(word.clone()); + result.push(word); result } @@ -184,224 +188,132 @@ const E1: char = '\u{F0003}'; const E2: char = '\u{F0004}'; const E3: char = '\u{F0005}'; -static BC1: LazyLock> = LazyLock::new(|| { - hashmap! { ('H', 'H') => 6, ('I', 'I') => 2461, ('K', 'H') => 406, ('O', 'H') => -1378, } +static BC1: LazyLock> = LazyLock::new(|| { + score_map! { ('H', 'H') => 6, ('I', 'I') => 2461, ('K', 'H') => 406, ('O', 'H') => -1378, } }); -static BC2: LazyLock> = LazyLock::new(|| { - hashmap! { ('A', 'A') => -3267, ('A', 'I') => 2744, ('A', 'N') => -878, ('H', 'H') => -4070, ('H', 'M') => -1711, ('H', 'N') => 4012, ('H', 'O') => 3761, ('I', 'A') => 1327, ('I', 'H') => -1184, ('I', 'I') => -1332, ('I', 'K') => 1721, ('I', 'O') => 5492, ('K', 'I') => 3831, ('K', 'K') => -8741, ('M', 'H') => -3132, ('M', 'K') => 3334, ('O', 'O') => -2920, } +static BC2: LazyLock> = LazyLock::new(|| { + score_map! { ('A', 'A') => -3267, ('A', 'I') => 2744, ('A', 'N') => -878, ('H', 'H') => -4070, ('H', 'M') => -1711, ('H', 'N') => 4012, ('H', 'O') => 3761, ('I', 'A') => 1327, ('I', 'H') => -1184, ('I', 'I') => -1332, ('I', 'K') => 1721, ('I', 'O') => 5492, ('K', 'I') => 3831, ('K', 'K') => -8741, ('M', 'H') => -3132, ('M', 'K') => 3334, ('O', 'O') => -2920, } }); -static BC3: LazyLock> = LazyLock::new(|| { - hashmap! { ('H', 'H') => 996, ('H', 'I') => 626, ('H', 'K') => -721, ('H', 'N') => -1307, ('H', 'O') => -836, ('I', 'H') => -301, ('K', 'K') => 2762, ('M', 'K') => 1079, ('M', 'M') => 4034, ('O', 'A') => -1652, ('O', 'H') => 266, } +static BC3: LazyLock> = LazyLock::new(|| { + score_map! { ('H', 'H') => 996, ('H', 'I') => 626, ('H', 'K') => -721, ('H', 'N') => -1307, ('H', 'O') => -836, ('I', 'H') => -301, ('K', 'K') => 2762, ('M', 'K') => 1079, ('M', 'M') => 4034, ('O', 'A') => -1652, ('O', 'H') => 266, } }); -static BP1: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'B') => 295, ('O', 'B') => 304, ('O', 'O') => -125, ('U', 'B') => 352, } +static BP1: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'B') => 295, ('O', 'B') => 304, ('O', 'O') => -125, ('U', 'B') => 352, } }); -static BP2: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'O') => 60, ('O', 'O') => -1762, } +static BP2: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'O') => 60, ('O', 'O') => -1762, } }); -static BQ1: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H') => 1150, ('B', 'H', 'M') => 1521, ('B', 'I', 'I') => -1158, ('B', 'I', 'M') => 886, ('B', 'M', 'H') => 1208, ('B', 'N', 'H') => 449, ('B', 'O', 'H') => -91, ('B', 'O', 'O') => -2597, ('O', 'H', 'I') => 451, ('O', 'I', 'H') => -296, ('O', 'K', 'A') => 1851, ('O', 'K', 'H') => -1020, ('O', 'K', 'K') => 904, ('O', 'O', 'O') => 2965, } +static BQ1: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H') => 1150, ('B', 'H', 'M') => 1521, ('B', 'I', 'I') => -1158, ('B', 'I', 'M') => 886, ('B', 'M', 'H') => 1208, ('B', 'N', 'H') => 449, ('B', 'O', 'H') => -91, ('B', 'O', 'O') => -2597, ('O', 'H', 'I') => 451, ('O', 'I', 'H') => -296, ('O', 'K', 'A') => 1851, ('O', 'K', 'H') => -1020, ('O', 'K', 'K') => 904, ('O', 'O', 'O') => 2965, } }); -static BQ2: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H') => 118, ('B', 'H', 'I') => -1159, ('B', 'H', 'M') => 466, ('B', 'I', 'H') => -919, ('B', 'K', 'K') => -1720, ('B', 'K', 'O') => 864, ('O', 'H', 'H') => -1139, ('O', 'H', 'M') => -181, ('O', 'I', 'H') => 153, ('U', 'H', 'I') => -1146, } +static BQ2: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H') => 118, ('B', 'H', 'I') => -1159, ('B', 'H', 'M') => 466, ('B', 'I', 'H') => -919, ('B', 'K', 'K') => -1720, ('B', 'K', 'O') => 864, ('O', 'H', 'H') => -1139, ('O', 'H', 'M') => -181, ('O', 'I', 'H') => 153, ('U', 'H', 'I') => -1146, } }); -static BQ3: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H') => -792, ('B', 'H', 'I') => 2664, ('B', 'I', 'I') => -299, ('B', 'K', 'I') => 419, ('B', 'M', 'H') => 937, ('B', 'M', 'M') => 8335, ('B', 'N', 'N') => 998, ('B', 'O', 'H') => 775, ('O', 'H', 'H') => 2174, ('O', 'H', 'M') => 439, ('O', 'I', 'I') => 280, ('O', 'K', 'H') => 1798, ('O', 'K', 'I') => -793, ('O', 'K', 'O') => -2242, ('O', 'M', 'H') => -2402, ('O', 'O', 'O') => 11699, } +static BQ3: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H') => -792, ('B', 'H', 'I') => 2664, ('B', 'I', 'I') => -299, ('B', 'K', 'I') => 419, ('B', 'M', 'H') => 937, ('B', 'M', 'M') => 8335, ('B', 'N', 'N') => 998, ('B', 'O', 'H') => 775, ('O', 'H', 'H') => 2174, ('O', 'H', 'M') => 439, ('O', 'I', 'I') => 280, ('O', 'K', 'H') => 1798, ('O', 'K', 'I') => -793, ('O', 'K', 'O') => -2242, ('O', 'M', 'H') => -2402, ('O', 'O', 'O') => 11699, } }); -static BQ4: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H') => -3895, ('B', 'I', 'H') => 3761, ('B', 'I', 'I') => -4654, ('B', 'I', 'K') => 1348, ('B', 'K', 'K') => -1806, ('B', 'M', 'I') => -3385, ('B', 'O', 'O') => -12396, ('O', 'A', 'H') => 926, ('O', 'H', 'H') => 266, ('O', 'H', 'K') => -2036, ('O', 'N', 'N') => -973, } +static BQ4: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H') => -3895, ('B', 'I', 'H') => 3761, ('B', 'I', 'I') => -4654, ('B', 'I', 'K') => 1348, ('B', 'K', 'K') => -1806, ('B', 'M', 'I') => -3385, ('B', 'O', 'O') => -12396, ('O', 'A', 'H') => 926, ('O', 'H', 'H') => 266, ('O', 'H', 'K') => -2036, ('O', 'N', 'N') => -973, } }); -static BW1: LazyLock> = LazyLock::new(|| { - hashmap! { (',', 'と') => 660, (',', '同') => 727, (B1, 'あ') => 1404, (B1, '同') => 542, ('、', 'と') => 660, ('、', '同') => 727, ('」', 'と') => 1682, ('あ', 'っ') => 1505, ('い', 'う') => 1743, ('い', 'っ') => -2055, ('い', 'る') => 672, ('う', 'し') => -4817, ('う', 'ん') => 665, ('か', 'ら') => 3472, ('が', 'ら') => 600, ('こ', 'う') => -790, ('こ', 'と') => 2083, ('こ', 'ん') => -1262, ('さ', 'ら') => -4143, ('さ', 'ん') => 4573, ('し', 'た') => 2641, ('し', 'て') => 1104, ('す', 'で') => -3399, ('そ', 'こ') => 1977, ('そ', 'れ') => -871, ('た', 'ち') => 1122, ('た', 'め') => 601, ('っ', 'た') => 3463, ('つ', 'い') => -802, ('て', 'い') => 805, ('て', 'き') => 1249, ('で', 'き') => 1127, ('で', 'す') => 3445, ('で', 'は') => 844, ('と', 'い') => -4915, ('と', 'み') => 1922, ('ど', 'こ') => 3887, ('な', 'い') => 5713, ('な', 'っ') => 3015, ('な', 'ど') => 7379, ('な', 'ん') => -1113, ('に', 'し') => 2468, ('に', 'は') => 1498, ('に', 'も') => 1671, ('に', '対') => -912, ('の', '一') => -501, ('の', '中') => 741, ('ま', 'せ') => 2448, ('ま', 'で') => 1711, ('ま', 'ま') => 2600, ('ま', 'る') => -2155, ('や', 'む') => -1947, ('よ', 'っ') => -2565, ('れ', 'た') => 2369, ('れ', 'で') => -913, ('を', 'し') => 1860, ('を', '見') => 731, ('亡', 'く') => -1886, ('京', '都') => 2558, ('取', 'り') => -2784, ('大', 'き') => -2604, ('大', '阪') => 1497, ('平', '方') => -2314, ('引', 'き') => -1336, ('日', '本') => -195, ('本', '当') => -2423, ('毎', '日') => -2113, ('目', '指') => -724, ('」', 'と') => 1682, } +static BW1: LazyLock> = LazyLock::new(|| { + score_map! { (',', 'と') => 660, (',', '同') => 727, (B1, 'あ') => 1404, (B1, '同') => 542, ('、', 'と') => 660, ('、', '同') => 727, ('」', 'と') => 1682, ('あ', 'っ') => 1505, ('い', 'う') => 1743, ('い', 'っ') => -2055, ('い', 'る') => 672, ('う', 'し') => -4817, ('う', 'ん') => 665, ('か', 'ら') => 3472, ('が', 'ら') => 600, ('こ', 'う') => -790, ('こ', 'と') => 2083, ('こ', 'ん') => -1262, ('さ', 'ら') => -4143, ('さ', 'ん') => 4573, ('し', 'た') => 2641, ('し', 'て') => 1104, ('す', 'で') => -3399, ('そ', 'こ') => 1977, ('そ', 'れ') => -871, ('た', 'ち') => 1122, ('た', 'め') => 601, ('っ', 'た') => 3463, ('つ', 'い') => -802, ('て', 'い') => 805, ('て', 'き') => 1249, ('で', 'き') => 1127, ('で', 'す') => 3445, ('で', 'は') => 844, ('と', 'い') => -4915, ('と', 'み') => 1922, ('ど', 'こ') => 3887, ('な', 'い') => 5713, ('な', 'っ') => 3015, ('な', 'ど') => 7379, ('な', 'ん') => -1113, ('に', 'し') => 2468, ('に', 'は') => 1498, ('に', 'も') => 1671, ('に', '対') => -912, ('の', '一') => -501, ('の', '中') => 741, ('ま', 'せ') => 2448, ('ま', 'で') => 1711, ('ま', 'ま') => 2600, ('ま', 'る') => -2155, ('や', 'む') => -1947, ('よ', 'っ') => -2565, ('れ', 'た') => 2369, ('れ', 'で') => -913, ('を', 'し') => 1860, ('を', '見') => 731, ('亡', 'く') => -1886, ('京', '都') => 2558, ('取', 'り') => -2784, ('大', 'き') => -2604, ('大', '阪') => 1497, ('平', '方') => -2314, ('引', 'き') => -1336, ('日', '本') => -195, ('本', '当') => -2423, ('毎', '日') => -2113, ('目', '指') => -724, ('」', 'と') => 1682, } }); -static BW2: LazyLock> = LazyLock::new(|| { - hashmap! { ('.', '.') => -11822, ('1', '1') => -669, ('―', '―') => -5730, ('−', '−') => -13175, ('い', 'う') => -1609, ('う', 'か') => 2490, ('か', 'し') => -1350, ('か', 'も') => -602, ('か', 'ら') => -7194, ('か', 'れ') => 4612, ('が', 'い') => 853, ('が', 'ら') => -3198, ('き', 'た') => 1941, ('く', 'な') => -1597, ('こ', 'と') => -8392, ('こ', 'の') => -4193, ('さ', 'せ') => 4533, ('さ', 'れ') => 13168, ('さ', 'ん') => -3977, ('し', 'い') => -1819, ('し', 'か') => -545, ('し', 'た') => 5078, ('し', 'て') => 972, ('し', 'な') => 939, ('そ', 'の') => -3744, ('た', 'い') => -1253, ('た', 'た') => -662, ('た', 'だ') => -3857, ('た', 'ち') => -786, ('た', 'と') => 1224, ('た', 'は') => -939, ('っ', 'た') => 4589, ('っ', 'て') => 1647, ('っ', 'と') => -2094, ('て', 'い') => 6144, ('て', 'き') => 3640, ('て', 'く') => 2551, ('て', 'は') => -3110, ('て', 'も') => -3065, ('で', 'い') => 2666, ('で', 'き') => -1528, ('で', 'し') => -3828, ('で', 'す') => -4761, ('で', 'も') => -4203, ('と', 'い') => 1890, ('と', 'こ') => -1746, ('と', 'と') => -2279, ('と', 'の') => 720, ('と', 'み') => 5168, ('と', 'も') => -3941, ('な', 'い') => -2488, ('な', 'が') => -1313, ('な', 'ど') => -6509, ('な', 'の') => 2614, ('な', 'ん') => 3099, ('に', 'お') => -1615, ('に', 'し') => 2748, ('に', 'な') => 2454, ('に', 'よ') => -7236, ('に', '対') => -14943, ('に', '従') => -4688, ('に', '関') => -11388, ('の', 'か') => 2093, ('の', 'で') => -7059, ('の', 'に') => -6041, ('の', 'の') => -6125, ('は', 'い') => 1073, ('は', 'が') => -1033, ('は', 'ず') => -2532, ('ば', 'れ') => 1813, ('ま', 'し') => -1316, ('ま', 'で') => -6621, ('ま', 'れ') => 5409, ('め', 'て') => -3153, ('も', 'い') => 2230, ('も', 'の') => -10713, ('ら', 'か') => -944, ('ら', 'し') => -1611, ('ら', 'に') => -1897, ('り', 'し') => 651, ('り', 'ま') => 1620, ('れ', 'た') => 4270, ('れ', 'て') => 849, ('れ', 'ば') => 4114, ('ろ', 'う') => 6067, ('わ', 'れ') => 7901, ('を', '通') => -11877, ('ん', 'だ') => 728, ('ん', 'な') => -4115, ('一', '人') => 602, ('一', '方') => -1375, ('一', '日') => 970, ('一', '部') => -1051, ('上', 'が') => -4479, ('会', '社') => -1116, ('出', 'て') => 2163, ('分', 'の') => -7758, ('同', '党') => 970, ('同', '日') => -913, ('大', '阪') => -2471, ('委', '員') => -1250, ('少', 'な') => -1050, ('年', '度') => -8669, ('年', '間') => -1626, ('府', '県') => -2363, ('手', '権') => -1982, ('新', '聞') => -4066, ('日', '新') => -722, ('日', '本') => -7068, ('日', '米') => 3372, ('曜', '日') => -601, ('朝', '鮮') => -2355, ('本', '人') => -2697, ('東', '京') => -1543, ('然', 'と') => -1384, ('社', '会') => -1276, ('立', 'て') => -990, ('第', 'に') => -1612, ('米', '国') => -4268, ('1', '1') => -669, ('ク', '゙') => 1319,} +static BW2: LazyLock> = LazyLock::new(|| { + score_map! { ('.', '.') => -11822, ('1', '1') => -669, ('―', '―') => -5730, ('−', '−') => -13175, ('い', 'う') => -1609, ('う', 'か') => 2490, ('か', 'し') => -1350, ('か', 'も') => -602, ('か', 'ら') => -7194, ('か', 'れ') => 4612, ('が', 'い') => 853, ('が', 'ら') => -3198, ('き', 'た') => 1941, ('く', 'な') => -1597, ('こ', 'と') => -8392, ('こ', 'の') => -4193, ('さ', 'せ') => 4533, ('さ', 'れ') => 13168, ('さ', 'ん') => -3977, ('し', 'い') => -1819, ('し', 'か') => -545, ('し', 'た') => 5078, ('し', 'て') => 972, ('し', 'な') => 939, ('そ', 'の') => -3744, ('た', 'い') => -1253, ('た', 'た') => -662, ('た', 'だ') => -3857, ('た', 'ち') => -786, ('た', 'と') => 1224, ('た', 'は') => -939, ('っ', 'た') => 4589, ('っ', 'て') => 1647, ('っ', 'と') => -2094, ('て', 'い') => 6144, ('て', 'き') => 3640, ('て', 'く') => 2551, ('て', 'は') => -3110, ('て', 'も') => -3065, ('で', 'い') => 2666, ('で', 'き') => -1528, ('で', 'し') => -3828, ('で', 'す') => -4761, ('で', 'も') => -4203, ('と', 'い') => 1890, ('と', 'こ') => -1746, ('と', 'と') => -2279, ('と', 'の') => 720, ('と', 'み') => 5168, ('と', 'も') => -3941, ('な', 'い') => -2488, ('な', 'が') => -1313, ('な', 'ど') => -6509, ('な', 'の') => 2614, ('な', 'ん') => 3099, ('に', 'お') => -1615, ('に', 'し') => 2748, ('に', 'な') => 2454, ('に', 'よ') => -7236, ('に', '対') => -14943, ('に', '従') => -4688, ('に', '関') => -11388, ('の', 'か') => 2093, ('の', 'で') => -7059, ('の', 'に') => -6041, ('の', 'の') => -6125, ('は', 'い') => 1073, ('は', 'が') => -1033, ('は', 'ず') => -2532, ('ば', 'れ') => 1813, ('ま', 'し') => -1316, ('ま', 'で') => -6621, ('ま', 'れ') => 5409, ('め', 'て') => -3153, ('も', 'い') => 2230, ('も', 'の') => -10713, ('ら', 'か') => -944, ('ら', 'し') => -1611, ('ら', 'に') => -1897, ('り', 'し') => 651, ('り', 'ま') => 1620, ('れ', 'た') => 4270, ('れ', 'て') => 849, ('れ', 'ば') => 4114, ('ろ', 'う') => 6067, ('わ', 'れ') => 7901, ('を', '通') => -11877, ('ん', 'だ') => 728, ('ん', 'な') => -4115, ('一', '人') => 602, ('一', '方') => -1375, ('一', '日') => 970, ('一', '部') => -1051, ('上', 'が') => -4479, ('会', '社') => -1116, ('出', 'て') => 2163, ('分', 'の') => -7758, ('同', '党') => 970, ('同', '日') => -913, ('大', '阪') => -2471, ('委', '員') => -1250, ('少', 'な') => -1050, ('年', '度') => -8669, ('年', '間') => -1626, ('府', '県') => -2363, ('手', '権') => -1982, ('新', '聞') => -4066, ('日', '新') => -722, ('日', '本') => -7068, ('日', '米') => 3372, ('曜', '日') => -601, ('朝', '鮮') => -2355, ('本', '人') => -2697, ('東', '京') => -1543, ('然', 'と') => -1384, ('社', '会') => -1276, ('立', 'て') => -990, ('第', 'に') => -1612, ('米', '国') => -4268, ('1', '1') => -669, ('ク', '゙') => 1319,} }); -static BW3: LazyLock> = LazyLock::new(|| { - hashmap! { ('あ', 'た') => -2194, ('あ', 'り') => 719, ('あ', 'る') => 3846, ('い', '.') => -1185, ('い', '。') => -1185, ('い', 'い') => 5308, ('い', 'え') => 2079, ('い', 'く') => 3029, ('い', 'た') => 2056, ('い', 'っ') => 1883, ('い', 'る') => 5600, ('い', 'わ') => 1527, ('う', 'ち') => 1117, ('う', 'と') => 4798, ('え', 'と') => 1454, ('か', '.') => 2857, ('か', '。') => 2857, ('か', 'け') => -743, ('か', 'っ') => -4098, ('か', 'に') => -669, ('か', 'ら') => 6520, ('か', 'り') => -2670, ('が', ',') => 1816, ('が', '、') => 1816, ('が', 'き') => -4855, ('が', 'け') => -1127, ('が', 'っ') => -913, ('が', 'ら') => -4977, ('が', 'り') => -2064, ('き', 'た') => 1645, ('け', 'ど') => 1374, ('こ', 'と') => 7397, ('こ', 'の') => 1542, ('こ', 'ろ') => -2757, ('さ', 'い') => -714, ('さ', 'を') => 976, ('し', ',') => 1557, ('し', '、') => 1557, ('し', 'い') => -3714, ('し', 'た') => 3562, ('し', 'て') => 1449, ('し', 'な') => 2608, ('し', 'ま') => 1200, ('す', '.') => -1310, ('す', '。') => -1310, ('す', 'る') => 6521, ('ず', ',') => 3426, ('ず', '、') => 3426, ('ず', 'に') => 841, ('そ', 'う') => 428, ('た', '.') => 8875, ('た', '。') => 8875, ('た', 'い') => -594, ('た', 'の') => 812, ('た', 'り') => -1183, ('た', 'る') => -853, ('だ', '.') => 4098, ('だ', '。') => 4098, ('だ', 'っ') => 1004, ('っ', 'た') => -4748, ('っ', 'て') => 300, ('て', 'い') => 6240, ('て', 'お') => 855, ('て', 'も') => 302, ('で', 'す') => 1437, ('で', 'に') => -1482, ('で', 'は') => 2295, ('と', 'う') => -1387, ('と', 'し') => 2266, ('と', 'の') => 541, ('と', 'も') => -3543, ('ど', 'う') => 4664, ('な', 'い') => 1796, ('な', 'く') => -903, ('な', 'ど') => 2135, ('に', ',') => -1021, ('に', '、') => -1021, ('に', 'し') => 1771, ('に', 'な') => 1906, ('に', 'は') => 2644, ('の', ',') => -724, ('の', '、') => -724, ('の', '子') => -1000, ('は', ',') => 1337, ('は', '、') => 1337, ('べ', 'き') => 2181, ('ま', 'し') => 1113, ('ま', 'す') => 6943, ('ま', 'っ') => -1549, ('ま', 'で') => 6154, ('ま', 'れ') => -793, ('ら', 'し') => 1479, ('ら', 'れ') => 6820, ('る', 'る') => 3818, ('れ', ',') => 854, ('れ', '、') => 854, ('れ', 'た') => 1850, ('れ', 'て') => 1375, ('れ', 'ば') => -3246, ('れ', 'る') => 1091, ('わ', 'れ') => -605, ('ん', 'だ') => 606, ('ん', 'で') => 798, ('カ', '月') => 990, ('会', '議') => 860, ('入', 'り') => 1232, ('大', '会') => 2217, ('始', 'め') => 1681, ('市', ' ') => 965, ('新', '聞') => -5055, ('日', ',') => 974, ('日', '、') => 974, ('社', '会') => 2024, ('カ', '月') => 990, } +static BW3: LazyLock> = LazyLock::new(|| { + score_map! { ('あ', 'た') => -2194, ('あ', 'り') => 719, ('あ', 'る') => 3846, ('い', '.') => -1185, ('い', '。') => -1185, ('い', 'い') => 5308, ('い', 'え') => 2079, ('い', 'く') => 3029, ('い', 'た') => 2056, ('い', 'っ') => 1883, ('い', 'る') => 5600, ('い', 'わ') => 1527, ('う', 'ち') => 1117, ('う', 'と') => 4798, ('え', 'と') => 1454, ('か', '.') => 2857, ('か', '。') => 2857, ('か', 'け') => -743, ('か', 'っ') => -4098, ('か', 'に') => -669, ('か', 'ら') => 6520, ('か', 'り') => -2670, ('が', ',') => 1816, ('が', '、') => 1816, ('が', 'き') => -4855, ('が', 'け') => -1127, ('が', 'っ') => -913, ('が', 'ら') => -4977, ('が', 'り') => -2064, ('き', 'た') => 1645, ('け', 'ど') => 1374, ('こ', 'と') => 7397, ('こ', 'の') => 1542, ('こ', 'ろ') => -2757, ('さ', 'い') => -714, ('さ', 'を') => 976, ('し', ',') => 1557, ('し', '、') => 1557, ('し', 'い') => -3714, ('し', 'た') => 3562, ('し', 'て') => 1449, ('し', 'な') => 2608, ('し', 'ま') => 1200, ('す', '.') => -1310, ('す', '。') => -1310, ('す', 'る') => 6521, ('ず', ',') => 3426, ('ず', '、') => 3426, ('ず', 'に') => 841, ('そ', 'う') => 428, ('た', '.') => 8875, ('た', '。') => 8875, ('た', 'い') => -594, ('た', 'の') => 812, ('た', 'り') => -1183, ('た', 'る') => -853, ('だ', '.') => 4098, ('だ', '。') => 4098, ('だ', 'っ') => 1004, ('っ', 'た') => -4748, ('っ', 'て') => 300, ('て', 'い') => 6240, ('て', 'お') => 855, ('て', 'も') => 302, ('で', 'す') => 1437, ('で', 'に') => -1482, ('で', 'は') => 2295, ('と', 'う') => -1387, ('と', 'し') => 2266, ('と', 'の') => 541, ('と', 'も') => -3543, ('ど', 'う') => 4664, ('な', 'い') => 1796, ('な', 'く') => -903, ('な', 'ど') => 2135, ('に', ',') => -1021, ('に', '、') => -1021, ('に', 'し') => 1771, ('に', 'な') => 1906, ('に', 'は') => 2644, ('の', ',') => -724, ('の', '、') => -724, ('の', '子') => -1000, ('は', ',') => 1337, ('は', '、') => 1337, ('べ', 'き') => 2181, ('ま', 'し') => 1113, ('ま', 'す') => 6943, ('ま', 'っ') => -1549, ('ま', 'で') => 6154, ('ま', 'れ') => -793, ('ら', 'し') => 1479, ('ら', 'れ') => 6820, ('る', 'る') => 3818, ('れ', ',') => 854, ('れ', '、') => 854, ('れ', 'た') => 1850, ('れ', 'て') => 1375, ('れ', 'ば') => -3246, ('れ', 'る') => 1091, ('わ', 'れ') => -605, ('ん', 'だ') => 606, ('ん', 'で') => 798, ('カ', '月') => 990, ('会', '議') => 860, ('入', 'り') => 1232, ('大', '会') => 2217, ('始', 'め') => 1681, ('市', ' ') => 965, ('新', '聞') => -5055, ('日', ',') => 974, ('日', '、') => 974, ('社', '会') => 2024, ('カ', '月') => 990, } }); -static TC1: LazyLock> = LazyLock::new(|| { - hashmap! { ('A', 'A', 'A') => 1093, ('H', 'H', 'H') => 1029, ('H', 'H', 'M') => 580, ('H', 'I', 'I') => 998, ('H', 'O', 'H') => -390, ('H', 'O', 'M') => -331, ('I', 'H', 'I') => 1169, ('I', 'O', 'H') => -142, ('I', 'O', 'I') => -1015, ('I', 'O', 'M') => 467, ('M', 'M', 'H') => 187, ('O', 'O', 'I') => -1832, } +static TC1: LazyLock> = LazyLock::new(|| { + score_map! { ('A', 'A', 'A') => 1093, ('H', 'H', 'H') => 1029, ('H', 'H', 'M') => 580, ('H', 'I', 'I') => 998, ('H', 'O', 'H') => -390, ('H', 'O', 'M') => -331, ('I', 'H', 'I') => 1169, ('I', 'O', 'H') => -142, ('I', 'O', 'I') => -1015, ('I', 'O', 'M') => 467, ('M', 'M', 'H') => 187, ('O', 'O', 'I') => -1832, } }); -static TC2: LazyLock> = LazyLock::new(|| { - hashmap! { ('H', 'H', 'O') => 2088, ('H', 'I', 'I') => -1023, ('H', 'M', 'M') => -1154, ('I', 'H', 'I') => -1965, ('K', 'K', 'H') => 703, ('O', 'I', 'I') => -2649, } +static TC2: LazyLock> = LazyLock::new(|| { + score_map! { ('H', 'H', 'O') => 2088, ('H', 'I', 'I') => -1023, ('H', 'M', 'M') => -1154, ('I', 'H', 'I') => -1965, ('K', 'K', 'H') => 703, ('O', 'I', 'I') => -2649, } }); -static TC3: LazyLock> = LazyLock::new(|| { - hashmap! { ('A', 'A', 'A') => -294, ('H', 'H', 'H') => 346, ('H', 'H', 'I') => -341, ('H', 'I', 'I') => -1088, ('H', 'I', 'K') => 731, ('H', 'O', 'H') => -1486, ('I', 'H', 'H') => 128, ('I', 'H', 'I') => -3041, ('I', 'H', 'O') => -1935, ('I', 'I', 'H') => -825, ('I', 'I', 'M') => -1035, ('I', 'O', 'I') => -542, ('K', 'H', 'H') => -1216, ('K', 'K', 'A') => 491, ('K', 'K', 'H') => -1217, ('K', 'O', 'K') => -1009, ('M', 'H', 'H') => -2694, ('M', 'H', 'M') => -457, ('M', 'H', 'O') => 123, ('M', 'M', 'H') => -471, ('N', 'N', 'H') => -1689, ('N', 'N', 'O') => 662, ('O', 'H', 'O') => -3393, } +static TC3: LazyLock> = LazyLock::new(|| { + score_map! { ('A', 'A', 'A') => -294, ('H', 'H', 'H') => 346, ('H', 'H', 'I') => -341, ('H', 'I', 'I') => -1088, ('H', 'I', 'K') => 731, ('H', 'O', 'H') => -1486, ('I', 'H', 'H') => 128, ('I', 'H', 'I') => -3041, ('I', 'H', 'O') => -1935, ('I', 'I', 'H') => -825, ('I', 'I', 'M') => -1035, ('I', 'O', 'I') => -542, ('K', 'H', 'H') => -1216, ('K', 'K', 'A') => 491, ('K', 'K', 'H') => -1217, ('K', 'O', 'K') => -1009, ('M', 'H', 'H') => -2694, ('M', 'H', 'M') => -457, ('M', 'H', 'O') => 123, ('M', 'M', 'H') => -471, ('N', 'N', 'H') => -1689, ('N', 'N', 'O') => 662, ('O', 'H', 'O') => -3393, } }); -static TC4: LazyLock> = LazyLock::new(|| { - hashmap! { ('H', 'H', 'H') => -203, ('H', 'H', 'I') => 1344, ('H', 'H', 'K') => 365, ('H', 'H', 'M') => -122, ('H', 'H', 'N') => 182, ('H', 'H', 'O') => 669, ('H', 'I', 'H') => 804, ('H', 'I', 'I') => 679, ('H', 'O', 'H') => 446, ('I', 'H', 'H') => 695, ('I', 'H', 'O') => -2324, ('I', 'I', 'H') => 321, ('I', 'I', 'I') => 1497, ('I', 'I', 'O') => 656, ('I', 'O', 'O') => 54, ('K', 'A', 'K') => 4845, ('K', 'K', 'A') => 3386, ('K', 'K', 'K') => 3065, ('M', 'H', 'H') => -405, ('M', 'H', 'I') => 201, ('M', 'M', 'H') => -241, ('M', 'M', 'M') => 661, ('M', 'O', 'M') => 841, } +static TC4: LazyLock> = LazyLock::new(|| { + score_map! { ('H', 'H', 'H') => -203, ('H', 'H', 'I') => 1344, ('H', 'H', 'K') => 365, ('H', 'H', 'M') => -122, ('H', 'H', 'N') => 182, ('H', 'H', 'O') => 669, ('H', 'I', 'H') => 804, ('H', 'I', 'I') => 679, ('H', 'O', 'H') => 446, ('I', 'H', 'H') => 695, ('I', 'H', 'O') => -2324, ('I', 'I', 'H') => 321, ('I', 'I', 'I') => 1497, ('I', 'I', 'O') => 656, ('I', 'O', 'O') => 54, ('K', 'A', 'K') => 4845, ('K', 'K', 'A') => 3386, ('K', 'K', 'K') => 3065, ('M', 'H', 'H') => -405, ('M', 'H', 'I') => 201, ('M', 'M', 'H') => -241, ('M', 'M', 'M') => 661, ('M', 'O', 'M') => 841, } }); -static TQ1: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H', 'H') => -227, ('B', 'H', 'H', 'I') => 316, ('B', 'H', 'I', 'H') => -132, ('B', 'I', 'H', 'H') => 60, ('B', 'I', 'I', 'I') => 1595, ('B', 'N', 'H', 'H') => -744, ('B', 'O', 'H', 'H') => 225, ('B', 'O', 'O', 'O') => -908, ('O', 'A', 'K', 'K') => 482, ('O', 'H', 'H', 'H') => 281, ('O', 'H', 'I', 'H') => 249, ('O', 'I', 'H', 'I') => 200, ('O', 'I', 'I', 'H') => -68, } +static TQ1: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H', 'H') => -227, ('B', 'H', 'H', 'I') => 316, ('B', 'H', 'I', 'H') => -132, ('B', 'I', 'H', 'H') => 60, ('B', 'I', 'I', 'I') => 1595, ('B', 'N', 'H', 'H') => -744, ('B', 'O', 'H', 'H') => 225, ('B', 'O', 'O', 'O') => -908, ('O', 'A', 'K', 'K') => 482, ('O', 'H', 'H', 'H') => 281, ('O', 'H', 'I', 'H') => 249, ('O', 'I', 'H', 'I') => 200, ('O', 'I', 'I', 'H') => -68, } }); -static TQ2: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'I', 'H', 'H') => -1401, ('B', 'I', 'I', 'I') => -1033, ('B', 'K', 'A', 'K') => -543, ('B', 'O', 'O', 'O') => -5591, } +static TQ2: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'I', 'H', 'H') => -1401, ('B', 'I', 'I', 'I') => -1033, ('B', 'K', 'A', 'K') => -543, ('B', 'O', 'O', 'O') => -5591, } }); -static TQ3: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H', 'H') => 478, ('B', 'H', 'H', 'M') => -1073, ('B', 'H', 'I', 'H') => 222, ('B', 'H', 'I', 'I') => -504, ('B', 'I', 'I', 'H') => -116, ('B', 'I', 'I', 'I') => -105, ('B', 'M', 'H', 'I') => -863, ('B', 'M', 'H', 'M') => -464, ('B', 'O', 'M', 'H') => 620, ('O', 'H', 'H', 'H') => 346, ('O', 'H', 'H', 'I') => 1729, ('O', 'H', 'I', 'I') => 997, ('O', 'H', 'M', 'H') => 481, ('O', 'I', 'H', 'H') => 623, ('O', 'I', 'I', 'H') => 1344, ('O', 'K', 'A', 'K') => 2792, ('O', 'K', 'H', 'H') => 587, ('O', 'K', 'K', 'A') => 679, ('O', 'O', 'H', 'H') => 110, ('O', 'O', 'I', 'I') => -685, } +static TQ3: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H', 'H') => 478, ('B', 'H', 'H', 'M') => -1073, ('B', 'H', 'I', 'H') => 222, ('B', 'H', 'I', 'I') => -504, ('B', 'I', 'I', 'H') => -116, ('B', 'I', 'I', 'I') => -105, ('B', 'M', 'H', 'I') => -863, ('B', 'M', 'H', 'M') => -464, ('B', 'O', 'M', 'H') => 620, ('O', 'H', 'H', 'H') => 346, ('O', 'H', 'H', 'I') => 1729, ('O', 'H', 'I', 'I') => 997, ('O', 'H', 'M', 'H') => 481, ('O', 'I', 'H', 'H') => 623, ('O', 'I', 'I', 'H') => 1344, ('O', 'K', 'A', 'K') => 2792, ('O', 'K', 'H', 'H') => 587, ('O', 'K', 'K', 'A') => 679, ('O', 'O', 'H', 'H') => 110, ('O', 'O', 'I', 'I') => -685, } }); -static TQ4: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H', 'H', 'H') => -721, ('B', 'H', 'H', 'M') => -3604, ('B', 'H', 'I', 'I') => -966, ('B', 'I', 'I', 'H') => -607, ('B', 'I', 'I', 'I') => -2181, ('O', 'A', 'A', 'A') => -2763, ('O', 'A', 'K', 'K') => 180, ('O', 'H', 'H', 'H') => -294, ('O', 'H', 'H', 'I') => 2446, ('O', 'H', 'H', 'O') => 480, ('O', 'H', 'I', 'H') => -1573, ('O', 'I', 'H', 'H') => 1935, ('O', 'I', 'H', 'I') => -493, ('O', 'I', 'I', 'H') => 626, ('O', 'I', 'I', 'I') => -4007, ('O', 'K', 'A', 'K') => -8156, } +static TQ4: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H', 'H', 'H') => -721, ('B', 'H', 'H', 'M') => -3604, ('B', 'H', 'I', 'I') => -966, ('B', 'I', 'I', 'H') => -607, ('B', 'I', 'I', 'I') => -2181, ('O', 'A', 'A', 'A') => -2763, ('O', 'A', 'K', 'K') => 180, ('O', 'H', 'H', 'H') => -294, ('O', 'H', 'H', 'I') => 2446, ('O', 'H', 'H', 'O') => 480, ('O', 'H', 'I', 'H') => -1573, ('O', 'I', 'H', 'H') => 1935, ('O', 'I', 'H', 'I') => -493, ('O', 'I', 'I', 'H') => 626, ('O', 'I', 'I', 'I') => -4007, ('O', 'K', 'A', 'K') => -8156, } }); -static TW1: LazyLock> = LazyLock::new(|| { - hashmap! { ('に', 'つ', 'い') => -4681, ('東', '京', '都') => 2026, } +static TW1: LazyLock> = LazyLock::new(|| { + score_map! { ('に', 'つ', 'い') => -4681, ('東', '京', '都') => 2026, } }); -static TW2: LazyLock> = LazyLock::new(|| { - hashmap! { ('あ', 'る', '程') => -2049, ('い', 'っ', 'た') => -1256, ('こ', 'ろ', 'が') => -2434, ('し', 'ょ', 'う') => 3873, ('そ', 'の', '後') => -4430, ('だ', 'っ', 'て') => -1049, ('て', 'い', 'た') => 1833, ('と', 'し', 'て') => -4657, ('と', 'も', 'に') => -4517, ('も', 'の', 'で') => 1882, ('一', '気', 'に') => -792, ('初', 'め', 'て') => -1512, ('同', '時', 'に') => -8097, ('大', 'き', 'な') => -1255, ('対', 'し', 'て') => -2721, ('社', '会', '党') => -3216, } +static TW2: LazyLock> = LazyLock::new(|| { + score_map! { ('あ', 'る', '程') => -2049, ('い', 'っ', 'た') => -1256, ('こ', 'ろ', 'が') => -2434, ('し', 'ょ', 'う') => 3873, ('そ', 'の', '後') => -4430, ('だ', 'っ', 'て') => -1049, ('て', 'い', 'た') => 1833, ('と', 'し', 'て') => -4657, ('と', 'も', 'に') => -4517, ('も', 'の', 'で') => 1882, ('一', '気', 'に') => -792, ('初', 'め', 'て') => -1512, ('同', '時', 'に') => -8097, ('大', 'き', 'な') => -1255, ('対', 'し', 'て') => -2721, ('社', '会', '党') => -3216, } }); -static TW3: LazyLock> = LazyLock::new(|| { - hashmap! { ('い', 'た', 'だ') => -1734, ('し', 'て', 'い') => 1314, ('と', 'し', 'て') => -4314, ('に', 'つ', 'い') => -5483, ('に', 'と', 'っ') => -5989, ('に', '当', 'た') => -6247, ('の', 'で', ',') => -727, ('の', 'で', '、') => -727, ('の', 'も', 'の') => -600, ('れ', 'か', 'ら') => -3752, ('十', '二', '月') => -2287, } +static TW3: LazyLock> = LazyLock::new(|| { + score_map! { ('い', 'た', 'だ') => -1734, ('し', 'て', 'い') => 1314, ('と', 'し', 'て') => -4314, ('に', 'つ', 'い') => -5483, ('に', 'と', 'っ') => -5989, ('に', '当', 'た') => -6247, ('の', 'で', ',') => -727, ('の', 'で', '、') => -727, ('の', 'も', 'の') => -600, ('れ', 'か', 'ら') => -3752, ('十', '二', '月') => -2287, } }); -static TW4: LazyLock> = LazyLock::new(|| { - hashmap! { ('い', 'う', '.') => 8576, ('い', 'う', '。') => 8576, ('か', 'ら', 'な') => -2348, ('し', 'て', 'い') => 2958, ('た', 'が', ',') => 1516, ('た', 'が', '、') => 1516, ('て', 'い', 'る') => 1538, ('と', 'い', 'う') => 1349, ('ま', 'し', 'た') => 5543, ('ま', 'せ', 'ん') => 1097, ('よ', 'う', 'と') => -4258, ('よ', 'る', 'と') => 5865, } +static TW4: LazyLock> = LazyLock::new(|| { + score_map! { ('い', 'う', '.') => 8576, ('い', 'う', '。') => 8576, ('か', 'ら', 'な') => -2348, ('し', 'て', 'い') => 2958, ('た', 'が', ',') => 1516, ('た', 'が', '、') => 1516, ('て', 'い', 'る') => 1538, ('と', 'い', 'う') => 1349, ('ま', 'し', 'た') => 5543, ('ま', 'せ', 'ん') => 1097, ('よ', 'う', 'と') => -4258, ('よ', 'る', 'と') => 5865, } }); -static UC1: LazyLock> = LazyLock::new(|| { - hashmap! { 'A' => 484, 'K' => 93, 'M' => 645, 'O' => -505, } +static UC1: LazyLock> = LazyLock::new(|| { + score_map! { 'A' => 484, 'K' => 93, 'M' => 645, 'O' => -505, } }); -static UC2: LazyLock> = LazyLock::new(|| { - hashmap! { 'A' => 819, 'H' => 1059, 'I' => 409, 'M' => 3987, 'N' => 5775, 'O' => 646, } +static UC2: LazyLock> = LazyLock::new(|| { + score_map! { 'A' => 819, 'H' => 1059, 'I' => 409, 'M' => 3987, 'N' => 5775, 'O' => 646, } }); -static UC3: LazyLock> = LazyLock::new(|| { - hashmap! { 'A' => -1370, 'I' => 2311, } +static UC3: LazyLock> = LazyLock::new(|| { + score_map! { 'A' => -1370, 'I' => 2311, } }); -static UC4: LazyLock> = LazyLock::new(|| { - hashmap! { 'A' => -2643, 'H' => 1809, 'I' => -1032, 'K' => -3450, 'M' => 3565, 'N' => 3876, 'O' => 6646, } +static UC4: LazyLock> = LazyLock::new(|| { + score_map! { 'A' => -2643, 'H' => 1809, 'I' => -1032, 'K' => -3450, 'M' => 3565, 'N' => 3876, 'O' => 6646, } }); -static UC5: LazyLock> = LazyLock::new(|| { - hashmap! { 'H' => 313, 'I' => -1238, 'K' => -799, 'M' => 539, 'O' => -831, } +static UC5: LazyLock> = LazyLock::new(|| { + score_map! { 'H' => 313, 'I' => -1238, 'K' => -799, 'M' => 539, 'O' => -831, } }); -static UC6: LazyLock> = LazyLock::new(|| { - hashmap! { 'H' => -506, 'I' => -253, 'K' => 87, 'M' => 247, 'O' => -387, } +static UC6: LazyLock> = LazyLock::new(|| { + score_map! { 'H' => -506, 'I' => -253, 'K' => 87, 'M' => 247, 'O' => -387, } }); -static UP1: LazyLock> = LazyLock::new(|| { - hashmap! { 'O' => -214, } +static UP1: LazyLock> = LazyLock::new(|| { + score_map! { 'O' => -214, } }); -static UP2: LazyLock> = LazyLock::new(|| { - hashmap! { 'B' => 69, 'O' => 935, } +static UP2: LazyLock> = LazyLock::new(|| { + score_map! { 'B' => 69, 'O' => 935, } }); -static UP3: LazyLock> = LazyLock::new(|| { - hashmap! { 'B' => 189, } +static UP3: LazyLock> = LazyLock::new(|| { + score_map! { 'B' => 189, } }); -static UQ1: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H') => 21, ('B', 'I') => -12, ('B', 'K') => -99, ('B', 'N') => 142, ('B', 'O') => -56, ('O', 'H') => -95, ('O', 'I') => 477, ('O', 'K') => 410, ('O', 'O') => -2422, } +static UQ1: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H') => 21, ('B', 'I') => -12, ('B', 'K') => -99, ('B', 'N') => 142, ('B', 'O') => -56, ('O', 'H') => -95, ('O', 'I') => 477, ('O', 'K') => 410, ('O', 'O') => -2422, } }); -static UQ2: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'H') => 216, ('B', 'I') => 113, ('O', 'K') => 1759, } +static UQ2: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'H') => 216, ('B', 'I') => 113, ('O', 'K') => 1759, } }); -static UQ3: LazyLock> = LazyLock::new(|| { - hashmap! { ('B', 'A') => -479, ('B', 'H') => 42, ('B', 'I') => 1913, ('B', 'K') => -7198, ('B', 'M') => 3160, ('B', 'N') => 6427, ('B', 'O') => 14761, ('O', 'I') => -827, ('O', 'N') => -3212, } +static UQ3: LazyLock> = LazyLock::new(|| { + score_map! { ('B', 'A') => -479, ('B', 'H') => 42, ('B', 'I') => 1913, ('B', 'K') => -7198, ('B', 'M') => 3160, ('B', 'N') => 6427, ('B', 'O') => 14761, ('O', 'I') => -827, ('O', 'N') => -3212, } }); -static UW1: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => 156, '、' => 156, '「' => -463, 'あ' => -941, 'う' => -127, 'が' => -553, 'き' => 121, 'こ' => 505, 'で' => -201, 'と' => -547, 'ど' => -123, 'に' => -789, 'の' => -185, 'は' => -847, 'も' => -466, 'や' => -470, 'よ' => 182, 'ら' => -292, 'り' => 208, 'れ' => 169, 'を' => -446, 'ん' => -137, '・' => -135, '主' => -402, '京' => -268, '区' => -912, '午' => 871, '国' => -460, '大' => 561, '委' => 729, '市' => -411, '日' => -141, '理' => 361, '生' => -408, '県' => -386, '都' => -718, '「' => -463, '・' => -135, } +static UW1: LazyLock> = LazyLock::new(|| { + score_map! { ',' => 156, '、' => 156, '「' => -463, 'あ' => -941, 'う' => -127, 'が' => -553, 'き' => 121, 'こ' => 505, 'で' => -201, 'と' => -547, 'ど' => -123, 'に' => -789, 'の' => -185, 'は' => -847, 'も' => -466, 'や' => -470, 'よ' => 182, 'ら' => -292, 'り' => 208, 'れ' => 169, 'を' => -446, 'ん' => -137, '・' => -135, '主' => -402, '京' => -268, '区' => -912, '午' => 871, '国' => -460, '大' => 561, '委' => 729, '市' => -411, '日' => -141, '理' => 361, '生' => -408, '県' => -386, '都' => -718, '「' => -463, '・' => -135, } }); -static UW2: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => -829, '、' => -829, '〇' => 892, '「' => -645, '」' => 3145, 'あ' => -538, 'い' => 505, 'う' => 134, 'お' => -502, 'か' => 1454, 'が' => -856, 'く' => -412, 'こ' => 1141, 'さ' => 878, 'ざ' => 540, 'し' => 1529, 'す' => -675, 'せ' => 300, 'そ' => -1011, 'た' => 188, 'だ' => 1837, 'つ' => -949, 'て' => -291, 'で' => -268, 'と' => -981, 'ど' => 1273, 'な' => 1063, 'に' => -1764, 'の' => 130, 'は' => -409, 'ひ' => -1273, 'べ' => 1261, 'ま' => 600, 'も' => -1263, 'や' => -402, 'よ' => 1639, 'り' => -579, 'る' => -694, 'れ' => 571, 'を' => -2516, 'ん' => 2095, 'ア' => -587, 'カ' => 306, 'キ' => 568, 'ッ' => 831, '三' => -758, '不' => -2150, '世' => -302, '中' => -968, '主' => -861, '事' => 492, '人' => -123, '会' => 978, '保' => 362, '入' => 548, '初' => -3025, '副' => -1566, '北' => -3414, '区' => -422, '大' => -1769, '天' => -865, '太' => -483, '子' => -1519, '学' => 760, '実' => 1023, '小' => -2009, '市' => -813, '年' => -1060, '強' => 1067, '手' => -1519, '揺' => -1033, '政' => 1522, '文' => -1355, '新' => -1682, '日' => -1815, '明' => -1462, '最' => -630, '朝' => -1843, '本' => -1650, '東' => -931, '果' => -665, '次' => -2378, '民' => -180, '気' => -1740, '理' => 752, '発' => 529, '目' => -1584, '相' => -242, '県' => -1165, '立' => -763, '第' => 810, '米' => 509, '自' => -1353, '行' => 838, '西' => -744, '見' => -3874, '調' => 1010, '議' => 1198, '込' => 3041, '開' => 1758, '間' => -1257, '「' => -645, '」' => 3145, 'ッ' => 831, 'ア' => -587, 'カ' => 306, 'キ' => 568, } +static UW2: LazyLock> = LazyLock::new(|| { + score_map! { ',' => -829, '、' => -829, '〇' => 892, '「' => -645, '」' => 3145, 'あ' => -538, 'い' => 505, 'う' => 134, 'お' => -502, 'か' => 1454, 'が' => -856, 'く' => -412, 'こ' => 1141, 'さ' => 878, 'ざ' => 540, 'し' => 1529, 'す' => -675, 'せ' => 300, 'そ' => -1011, 'た' => 188, 'だ' => 1837, 'つ' => -949, 'て' => -291, 'で' => -268, 'と' => -981, 'ど' => 1273, 'な' => 1063, 'に' => -1764, 'の' => 130, 'は' => -409, 'ひ' => -1273, 'べ' => 1261, 'ま' => 600, 'も' => -1263, 'や' => -402, 'よ' => 1639, 'り' => -579, 'る' => -694, 'れ' => 571, 'を' => -2516, 'ん' => 2095, 'ア' => -587, 'カ' => 306, 'キ' => 568, 'ッ' => 831, '三' => -758, '不' => -2150, '世' => -302, '中' => -968, '主' => -861, '事' => 492, '人' => -123, '会' => 978, '保' => 362, '入' => 548, '初' => -3025, '副' => -1566, '北' => -3414, '区' => -422, '大' => -1769, '天' => -865, '太' => -483, '子' => -1519, '学' => 760, '実' => 1023, '小' => -2009, '市' => -813, '年' => -1060, '強' => 1067, '手' => -1519, '揺' => -1033, '政' => 1522, '文' => -1355, '新' => -1682, '日' => -1815, '明' => -1462, '最' => -630, '朝' => -1843, '本' => -1650, '東' => -931, '果' => -665, '次' => -2378, '民' => -180, '気' => -1740, '理' => 752, '発' => 529, '目' => -1584, '相' => -242, '県' => -1165, '立' => -763, '第' => 810, '米' => 509, '自' => -1353, '行' => 838, '西' => -744, '見' => -3874, '調' => 1010, '議' => 1198, '込' => 3041, '開' => 1758, '間' => -1257, '「' => -645, '」' => 3145, 'ッ' => 831, 'ア' => -587, 'カ' => 306, 'キ' => 568, } }); -static UW3: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => 4889, '1' => -800, '−' => -1723, '、' => 4889, '々' => -2311, '〇' => 5827, '」' => 2670, '〓' => -3573, 'あ' => -2696, 'い' => 1006, 'う' => 2342, 'え' => 1983, 'お' => -4864, 'か' => -1163, 'が' => 3271, 'く' => 1004, 'け' => 388, 'げ' => 401, 'こ' => -3552, 'ご' => -3116, 'さ' => -1058, 'し' => -395, 'す' => 584, 'せ' => 3685, 'そ' => -5228, 'た' => 842, 'ち' => -521, 'っ' => -1444, 'つ' => -1081, 'て' => 6167, 'で' => 2318, 'と' => 1691, 'ど' => -899, 'な' => -2788, 'に' => 2745, 'の' => 4056, 'は' => 4555, 'ひ' => -2171, 'ふ' => -1798, 'へ' => 1199, 'ほ' => -5516, 'ま' => -4384, 'み' => -120, 'め' => 1205, 'も' => 2323, 'や' => -788, 'よ' => -202, 'ら' => 727, 'り' => 649, 'る' => 5905, 'れ' => 2773, 'わ' => -1207, 'を' => 6620, 'ん' => -518, 'ア' => 551, 'グ' => 1319, 'ス' => 874, 'ッ' => -1350, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, '・' => -3794, '一' => -1619, '下' => -1759, '世' => -2087, '両' => 3815, '中' => 653, '主' => -758, '予' => -1193, '二' => 974, '人' => 2742, '今' => 792, '他' => 1889, '以' => -1368, '低' => 811, '何' => 4265, '作' => -361, '保' => -2439, '元' => 4858, '党' => 3593, '全' => 1574, '公' => -3030, '六' => 755, '共' => -1880, '円' => 5807, '再' => 3095, '分' => 457, '初' => 2475, '別' => 1129, '前' => 2286, '副' => 4437, '力' => 365, '動' => -949, '務' => -1872, '化' => 1327, '北' => -1038, '区' => 4646, '千' => -2309, '午' => -783, '協' => -1006, '口' => 483, '右' => 1233, '各' => 3588, '合' => -241, '同' => 3906, '和' => -837, '員' => 4513, '国' => 642, '型' => 1389, '場' => 1219, '外' => -241, '妻' => 2016, '学' => -1356, '安' => -423, '実' => -1008, '家' => 1078, '小' => -513, '少' => -3102, '州' => 1155, '市' => 3197, '平' => -1804, '年' => 2416, '広' => -1030, '府' => 1605, '度' => 1452, '建' => -2352, '当' => -3885, '得' => 1905, '思' => -1291, '性' => 1822, '戸' => -488, '指' => -3973, '政' => -2013, '教' => -1479, '数' => 3222, '文' => -1489, '新' => 1764, '日' => 2099, '旧' => 5792, '昨' => -661, '時' => -1248, '曜' => -951, '最' => -937, '月' => 4125, '期' => 360, '李' => 3094, '村' => 364, '東' => -805, '核' => 5156, '森' => 2438, '業' => 484, '氏' => 2613, '民' => -1694, '決' => -1073, '法' => 1868, '海' => -495, '無' => 979, '物' => 461, '特' => -3850, '生' => -273, '用' => 914, '町' => 1215, '的' => 7313, '直' => -1835, '省' => 792, '県' => 6293, '知' => -1528, '私' => 4231, '税' => 401, '立' => -960, '第' => 1201, '米' => 7767, '系' => 3066, '約' => 3663, '級' => 1384, '統' => -4229, '総' => 1163, '線' => 1255, '者' => 6457, '能' => 725, '自' => -2869, '英' => 785, '見' => 1044, '調' => -562, '財' => -733, '費' => 1777, '車' => 1835, '軍' => 1375, '込' => -1504, '通' => -1136, '選' => -681, '郎' => 1026, '郡' => 4404, '部' => 1200, '金' => 2163, '長' => 421, '開' => -1432, '間' => 1302, '関' => -1282, '雨' => 2009, '電' => -1045, '非' => 2066, '駅' => 1620, '1' => -800, '」' => 2670, '・' => -3794, 'ッ' => -1350, 'ア' => 551, 'ス' => 874, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, } +static UW3: LazyLock> = LazyLock::new(|| { + score_map! { ',' => 4889, '1' => -800, '−' => -1723, '、' => 4889, '々' => -2311, '〇' => 5827, '」' => 2670, '〓' => -3573, 'あ' => -2696, 'い' => 1006, 'う' => 2342, 'え' => 1983, 'お' => -4864, 'か' => -1163, 'が' => 3271, 'く' => 1004, 'け' => 388, 'げ' => 401, 'こ' => -3552, 'ご' => -3116, 'さ' => -1058, 'し' => -395, 'す' => 584, 'せ' => 3685, 'そ' => -5228, 'た' => 842, 'ち' => -521, 'っ' => -1444, 'つ' => -1081, 'て' => 6167, 'で' => 2318, 'と' => 1691, 'ど' => -899, 'な' => -2788, 'に' => 2745, 'の' => 4056, 'は' => 4555, 'ひ' => -2171, 'ふ' => -1798, 'へ' => 1199, 'ほ' => -5516, 'ま' => -4384, 'み' => -120, 'め' => 1205, 'も' => 2323, 'や' => -788, 'よ' => -202, 'ら' => 727, 'り' => 649, 'る' => 5905, 'れ' => 2773, 'わ' => -1207, 'を' => 6620, 'ん' => -518, 'ア' => 551, 'グ' => 1319, 'ス' => 874, 'ッ' => -1350, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, '・' => -3794, '一' => -1619, '下' => -1759, '世' => -2087, '両' => 3815, '中' => 653, '主' => -758, '予' => -1193, '二' => 974, '人' => 2742, '今' => 792, '他' => 1889, '以' => -1368, '低' => 811, '何' => 4265, '作' => -361, '保' => -2439, '元' => 4858, '党' => 3593, '全' => 1574, '公' => -3030, '六' => 755, '共' => -1880, '円' => 5807, '再' => 3095, '分' => 457, '初' => 2475, '別' => 1129, '前' => 2286, '副' => 4437, '力' => 365, '動' => -949, '務' => -1872, '化' => 1327, '北' => -1038, '区' => 4646, '千' => -2309, '午' => -783, '協' => -1006, '口' => 483, '右' => 1233, '各' => 3588, '合' => -241, '同' => 3906, '和' => -837, '員' => 4513, '国' => 642, '型' => 1389, '場' => 1219, '外' => -241, '妻' => 2016, '学' => -1356, '安' => -423, '実' => -1008, '家' => 1078, '小' => -513, '少' => -3102, '州' => 1155, '市' => 3197, '平' => -1804, '年' => 2416, '広' => -1030, '府' => 1605, '度' => 1452, '建' => -2352, '当' => -3885, '得' => 1905, '思' => -1291, '性' => 1822, '戸' => -488, '指' => -3973, '政' => -2013, '教' => -1479, '数' => 3222, '文' => -1489, '新' => 1764, '日' => 2099, '旧' => 5792, '昨' => -661, '時' => -1248, '曜' => -951, '最' => -937, '月' => 4125, '期' => 360, '李' => 3094, '村' => 364, '東' => -805, '核' => 5156, '森' => 2438, '業' => 484, '氏' => 2613, '民' => -1694, '決' => -1073, '法' => 1868, '海' => -495, '無' => 979, '物' => 461, '特' => -3850, '生' => -273, '用' => 914, '町' => 1215, '的' => 7313, '直' => -1835, '省' => 792, '県' => 6293, '知' => -1528, '私' => 4231, '税' => 401, '立' => -960, '第' => 1201, '米' => 7767, '系' => 3066, '約' => 3663, '級' => 1384, '統' => -4229, '総' => 1163, '線' => 1255, '者' => 6457, '能' => 725, '自' => -2869, '英' => 785, '見' => 1044, '調' => -562, '財' => -733, '費' => 1777, '車' => 1835, '軍' => 1375, '込' => -1504, '通' => -1136, '選' => -681, '郎' => 1026, '郡' => 4404, '部' => 1200, '金' => 2163, '長' => 421, '開' => -1432, '間' => 1302, '関' => -1282, '雨' => 2009, '電' => -1045, '非' => 2066, '駅' => 1620, '1' => -800, '」' => 2670, '・' => -3794, 'ッ' => -1350, 'ア' => 551, 'ス' => 874, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, } }); -static UW4: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => 3930, '.' => 3508, '―' => -4841, '、' => 3930, '。' => 3508, '〇' => 4999, '「' => 1895, '」' => 3798, '〓' => -5156, 'あ' => 4752, 'い' => -3435, 'う' => -640, 'え' => -2514, 'お' => 2405, 'か' => 530, 'が' => 6006, 'き' => -4482, 'ぎ' => -3821, 'く' => -3788, 'け' => -4376, 'げ' => -4734, 'こ' => 2255, 'ご' => 1979, 'さ' => 2864, 'し' => -843, 'じ' => -2506, 'す' => -731, 'ず' => 1251, 'せ' => 181, 'そ' => 4091, 'た' => 5034, 'だ' => 5408, 'ち' => -3654, 'っ' => -5882, 'つ' => -1659, 'て' => 3994, 'で' => 7410, 'と' => 4547, 'な' => 5433, 'に' => 6499, 'ぬ' => 1853, 'ね' => 1413, 'の' => 7396, 'は' => 8578, 'ば' => 1940, 'ひ' => 4249, 'び' => -4134, 'ふ' => 1345, 'へ' => 6665, 'べ' => -744, 'ほ' => 1464, 'ま' => 1051, 'み' => -2082, 'む' => -882, 'め' => -5046, 'も' => 4169, 'ゃ' => -2666, 'や' => 2795, 'ょ' => -1544, 'よ' => 3351, 'ら' => -2922, 'り' => -9726, 'る' => -14896, 'れ' => -2613, 'ろ' => -4570, 'わ' => -1783, 'を' => 13150, 'ん' => -2352, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ッ' => -724, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, '・' => -4371, 'ー' => -11870, '一' => -2069, '中' => 2210, '予' => 782, '事' => -190, '井' => -1768, '人' => 1036, '以' => 544, '会' => 950, '体' => -1286, '作' => 530, '側' => 4292, '先' => 601, '党' => -2006, '共' => -1212, '内' => 584, '円' => 788, '初' => 1347, '前' => 1623, '副' => 3879, '力' => -302, '動' => -740, '務' => -2715, '化' => 776, '区' => 4517, '協' => 1013, '参' => 1555, '合' => -1834, '和' => -681, '員' => -910, '器' => -851, '回' => 1500, '国' => -619, '園' => -1200, '地' => 866, '場' => -1410, '塁' => -2094, '士' => -1413, '多' => 1067, '大' => 571, '子' => -4802, '学' => -1397, '定' => -1057, '寺' => -809, '小' => 1910, '屋' => -1328, '山' => -1500, '島' => -2056, '川' => -2667, '市' => 2771, '年' => 374, '庁' => -4556, '後' => 456, '性' => 553, '感' => 916, '所' => -1566, '支' => 856, '改' => 787, '政' => 2182, '教' => 704, '文' => 522, '方' => -856, '日' => 1798, '時' => 1829, '最' => 845, '月' => -9066, '木' => -485, '来' => -442, '校' => -360, '業' => -1043, '氏' => 5388, '民' => -2716, '気' => -910, '沢' => -939, '済' => -543, '物' => -735, '率' => 672, '球' => -1267, '生' => -1286, '産' => -1101, '田' => -2900, '町' => 1826, '的' => 2586, '目' => 922, '省' => -3485, '県' => 2997, '空' => -867, '立' => -2112, '第' => 788, '米' => 2937, '系' => 786, '約' => 2171, '経' => 1146, '統' => -1169, '総' => 940, '線' => -994, '署' => 749, '者' => 2145, '能' => -730, '般' => -852, '行' => -792, '規' => 792, '警' => -1184, '議' => -244, '谷' => -1000, '賞' => 730, '車' => -1481, '軍' => 1158, '輪' => -1433, '込' => -3370, '近' => 929, '道' => -1291, '選' => 2596, '郎' => -4866, '都' => 1192, '野' => -1100, '銀' => -2213, '長' => 357, '間' => -2344, '院' => -2297, '際' => -2604, '電' => -878, '領' => -1659, '題' => -792, '館' => -1984, '首' => 1749, '高' => 2120, '「' => 1895, '」' => 3798, '・' => -4371, 'ッ' => -724, 'ー' => -11870, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, } +static UW4: LazyLock> = LazyLock::new(|| { + score_map! { ',' => 3930, '.' => 3508, '―' => -4841, '、' => 3930, '。' => 3508, '〇' => 4999, '「' => 1895, '」' => 3798, '〓' => -5156, 'あ' => 4752, 'い' => -3435, 'う' => -640, 'え' => -2514, 'お' => 2405, 'か' => 530, 'が' => 6006, 'き' => -4482, 'ぎ' => -3821, 'く' => -3788, 'け' => -4376, 'げ' => -4734, 'こ' => 2255, 'ご' => 1979, 'さ' => 2864, 'し' => -843, 'じ' => -2506, 'す' => -731, 'ず' => 1251, 'せ' => 181, 'そ' => 4091, 'た' => 5034, 'だ' => 5408, 'ち' => -3654, 'っ' => -5882, 'つ' => -1659, 'て' => 3994, 'で' => 7410, 'と' => 4547, 'な' => 5433, 'に' => 6499, 'ぬ' => 1853, 'ね' => 1413, 'の' => 7396, 'は' => 8578, 'ば' => 1940, 'ひ' => 4249, 'び' => -4134, 'ふ' => 1345, 'へ' => 6665, 'べ' => -744, 'ほ' => 1464, 'ま' => 1051, 'み' => -2082, 'む' => -882, 'め' => -5046, 'も' => 4169, 'ゃ' => -2666, 'や' => 2795, 'ょ' => -1544, 'よ' => 3351, 'ら' => -2922, 'り' => -9726, 'る' => -14896, 'れ' => -2613, 'ろ' => -4570, 'わ' => -1783, 'を' => 13150, 'ん' => -2352, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ッ' => -724, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, '・' => -4371, 'ー' => -11870, '一' => -2069, '中' => 2210, '予' => 782, '事' => -190, '井' => -1768, '人' => 1036, '以' => 544, '会' => 950, '体' => -1286, '作' => 530, '側' => 4292, '先' => 601, '党' => -2006, '共' => -1212, '内' => 584, '円' => 788, '初' => 1347, '前' => 1623, '副' => 3879, '力' => -302, '動' => -740, '務' => -2715, '化' => 776, '区' => 4517, '協' => 1013, '参' => 1555, '合' => -1834, '和' => -681, '員' => -910, '器' => -851, '回' => 1500, '国' => -619, '園' => -1200, '地' => 866, '場' => -1410, '塁' => -2094, '士' => -1413, '多' => 1067, '大' => 571, '子' => -4802, '学' => -1397, '定' => -1057, '寺' => -809, '小' => 1910, '屋' => -1328, '山' => -1500, '島' => -2056, '川' => -2667, '市' => 2771, '年' => 374, '庁' => -4556, '後' => 456, '性' => 553, '感' => 916, '所' => -1566, '支' => 856, '改' => 787, '政' => 2182, '教' => 704, '文' => 522, '方' => -856, '日' => 1798, '時' => 1829, '最' => 845, '月' => -9066, '木' => -485, '来' => -442, '校' => -360, '業' => -1043, '氏' => 5388, '民' => -2716, '気' => -910, '沢' => -939, '済' => -543, '物' => -735, '率' => 672, '球' => -1267, '生' => -1286, '産' => -1101, '田' => -2900, '町' => 1826, '的' => 2586, '目' => 922, '省' => -3485, '県' => 2997, '空' => -867, '立' => -2112, '第' => 788, '米' => 2937, '系' => 786, '約' => 2171, '経' => 1146, '統' => -1169, '総' => 940, '線' => -994, '署' => 749, '者' => 2145, '能' => -730, '般' => -852, '行' => -792, '規' => 792, '警' => -1184, '議' => -244, '谷' => -1000, '賞' => 730, '車' => -1481, '軍' => 1158, '輪' => -1433, '込' => -3370, '近' => 929, '道' => -1291, '選' => 2596, '郎' => -4866, '都' => 1192, '野' => -1100, '銀' => -2213, '長' => 357, '間' => -2344, '院' => -2297, '際' => -2604, '電' => -878, '領' => -1659, '題' => -792, '館' => -1984, '首' => 1749, '高' => 2120, '「' => 1895, '」' => 3798, '・' => -4371, 'ッ' => -724, 'ー' => -11870, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, } }); -static UW5: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => 465, '.' => -299, '1' => -514, E2 => -32768, ']' => -2762, '、' => 465, '。' => -299, '「' => 363, 'あ' => 1655, 'い' => 331, 'う' => -503, 'え' => 1199, 'お' => 527, 'か' => 647, 'が' => -421, 'き' => 1624, 'ぎ' => 1971, 'く' => 312, 'げ' => -983, 'さ' => -1537, 'し' => -1371, 'す' => -852, 'だ' => -1186, 'ち' => 1093, 'っ' => 52, 'つ' => 921, 'て' => -18, 'で' => -850, 'と' => -127, 'ど' => 1682, 'な' => -787, 'に' => -1224, 'の' => -635, 'は' => -578, 'べ' => 1001, 'み' => 502, 'め' => 865, 'ゃ' => 3350, 'ょ' => 854, 'り' => -208, 'る' => 429, 'れ' => 504, 'わ' => 419, 'を' => -1264, 'ん' => 327, 'イ' => 241, 'ル' => 451, 'ン' => -343, '中' => -871, '京' => 722, '会' => -1153, '党' => -654, '務' => 3519, '区' => -901, '告' => 848, '員' => 2104, '大' => -1296, '学' => -548, '定' => 1785, '嵐' => -1304, '市' => -2991, '席' => 921, '年' => 1763, '思' => 872, '所' => -814, '挙' => 1618, '新' => -1682, '日' => 218, '月' => -4353, '査' => 932, '格' => 1356, '機' => -1508, '氏' => -1347, '田' => 240, '町' => -3912, '的' => -3149, '相' => 1319, '省' => -1052, '県' => -4003, '研' => -997, '社' => -278, '空' => -813, '統' => 1955, '者' => -2233, '表' => 663, '語' => -1073, '議' => 1219, '選' => -1018, '郎' => -368, '長' => 786, '間' => 1191, '題' => 2368, '館' => -689, '1' => -514, '「' => 363, 'イ' => 241, 'ル' => 451, 'ン' => -343, } +static UW5: LazyLock> = LazyLock::new(|| { + score_map! { ',' => 465, '.' => -299, '1' => -514, E2 => -32768, ']' => -2762, '、' => 465, '。' => -299, '「' => 363, 'あ' => 1655, 'い' => 331, 'う' => -503, 'え' => 1199, 'お' => 527, 'か' => 647, 'が' => -421, 'き' => 1624, 'ぎ' => 1971, 'く' => 312, 'げ' => -983, 'さ' => -1537, 'し' => -1371, 'す' => -852, 'だ' => -1186, 'ち' => 1093, 'っ' => 52, 'つ' => 921, 'て' => -18, 'で' => -850, 'と' => -127, 'ど' => 1682, 'な' => -787, 'に' => -1224, 'の' => -635, 'は' => -578, 'べ' => 1001, 'み' => 502, 'め' => 865, 'ゃ' => 3350, 'ょ' => 854, 'り' => -208, 'る' => 429, 'れ' => 504, 'わ' => 419, 'を' => -1264, 'ん' => 327, 'イ' => 241, 'ル' => 451, 'ン' => -343, '中' => -871, '京' => 722, '会' => -1153, '党' => -654, '務' => 3519, '区' => -901, '告' => 848, '員' => 2104, '大' => -1296, '学' => -548, '定' => 1785, '嵐' => -1304, '市' => -2991, '席' => 921, '年' => 1763, '思' => 872, '所' => -814, '挙' => 1618, '新' => -1682, '日' => 218, '月' => -4353, '査' => 932, '格' => 1356, '機' => -1508, '氏' => -1347, '田' => 240, '町' => -3912, '的' => -3149, '相' => 1319, '省' => -1052, '県' => -4003, '研' => -997, '社' => -278, '空' => -813, '統' => 1955, '者' => -2233, '表' => 663, '語' => -1073, '議' => 1219, '選' => -1018, '郎' => -368, '長' => 786, '間' => 1191, '題' => 2368, '館' => -689, '1' => -514, '「' => 363, 'イ' => 241, 'ル' => 451, 'ン' => -343, } }); -static UW6: LazyLock> = LazyLock::new(|| { - hashmap! { ',' => 227, '.' => 808, '1' => -270, E1 => 306, '、' => 227, '。' => 808, 'あ' => -307, 'う' => 189, 'か' => 241, 'が' => -73, 'く' => -121, 'こ' => -200, 'じ' => 1782, 'す' => 383, 'た' => -428, 'っ' => 573, 'て' => -1014, 'で' => 101, 'と' => -105, 'な' => -253, 'に' => -149, 'の' => -417, 'は' => -236, 'も' => -206, 'り' => 187, 'る' => -135, 'を' => 195, 'ル' => -673, 'ン' => -496, '一' => -277, '中' => 201, '件' => -800, '会' => 624, '前' => 302, '区' => 1792, '員' => -1212, '委' => 798, '学' => -960, '市' => 887, '広' => -695, '後' => 535, '業' => -697, '相' => 753, '社' => -507, '福' => 974, '空' => -822, '者' => 1811, '連' => 463, '郎' => 1082, '1' => -270, 'ル' => -673, 'ン' => -496, } +static UW6: LazyLock> = LazyLock::new(|| { + score_map! { ',' => 227, '.' => 808, '1' => -270, E1 => 306, '、' => 227, '。' => 808, 'あ' => -307, 'う' => 189, 'か' => 241, 'が' => -73, 'く' => -121, 'こ' => -200, 'じ' => 1782, 'す' => 383, 'た' => -428, 'っ' => 573, 'て' => -1014, 'で' => 101, 'と' => -105, 'な' => -253, 'に' => -149, 'の' => -417, 'は' => -236, 'も' => -206, 'り' => 187, 'る' => -135, 'を' => 195, 'ル' => -673, 'ン' => -496, '一' => -277, '中' => 201, '件' => -800, '会' => 624, '前' => 302, '区' => 1792, '員' => -1212, '委' => 798, '学' => -960, '市' => 887, '広' => -695, '後' => 535, '業' => -697, '相' => 753, '社' => -507, '福' => 974, '空' => -822, '者' => 1811, '連' => 463, '郎' => 1082, '1' => -270, 'ル' => -673, 'ン' => -496, } }); -#[cfg(test)] -mod tests { - use crate::tokenizers::{Token, japanese::JapaneseTokenizer, word::WordTokenizer}; - - #[test] - fn japanese_tokenizer() { - assert_eq!( - JapaneseTokenizer::new(WordTokenizer::new( - "お先に失礼します あなたの名前は何ですか 123 abc-872", - 40 - )) - .collect::>(), - vec![ - Token { - word: "お先".into(), - from: 0, - to: 6 - }, - Token { - word: "に".into(), - from: 6, - to: 9 - }, - Token { - word: "失礼".into(), - from: 9, - to: 15 - }, - Token { - word: "し".into(), - from: 15, - to: 18 - }, - Token { - word: "ます".into(), - from: 18, - to: 24 - }, - Token { - word: "あなた".into(), - from: 25, - to: 34 - }, - Token { - word: "の".into(), - from: 34, - to: 37 - }, - Token { - word: "名前".into(), - from: 37, - to: 43 - }, - Token { - word: "は".into(), - from: 43, - to: 46 - }, - Token { - word: "何".into(), - from: 46, - to: 49 - }, - Token { - word: "です".into(), - from: 49, - to: 55 - }, - Token { - word: "か".into(), - from: 55, - to: 58 - }, - Token { - word: "123".into(), - from: 59, - to: 62 - }, - Token { - word: "abc".into(), - from: 63, - to: 66 - }, - Token { - word: "872".into(), - from: 67, - to: 70 - } - ] - ); - } -} diff --git a/crates/nlp/src/tokenizers/mod.rs b/crates/nlp/src/tokenizers/mod.rs index 9682eea9..bd974b52 100644 --- a/crates/nlp/src/tokenizers/mod.rs +++ b/crates/nlp/src/tokenizers/mod.rs @@ -9,6 +9,7 @@ pub mod japanese; pub mod space; pub mod stream; pub mod types; +mod unicode; pub mod word; use std::borrow::Cow; diff --git a/crates/nlp/src/tokenizers/space.rs b/crates/nlp/src/tokenizers/space.rs index 9dbec5ee..bad6dfac 100644 --- a/crates/nlp/src/tokenizers/space.rs +++ b/crates/nlp/src/tokenizers/space.rs @@ -4,54 +4,73 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::str::Chars; +use std::borrow::Cow; + +use super::{unicode::is_uppercase, word::next_alnum_run}; pub struct SpaceTokenizer<'x> { - iterator: Chars<'x>, - token: String, + text: &'x str, + pos: usize, max_token_length: usize, } impl SpaceTokenizer<'_> { pub fn new(text: &'_ str, max_token_length: usize) -> SpaceTokenizer<'_> { SpaceTokenizer { - iterator: text.chars(), - token: String::new(), + text, + pos: 0, max_token_length, } } } -impl Iterator for SpaceTokenizer<'_> { - type Item = String; +#[inline(never)] +fn lowercase_ascii(word: &str) -> String { + let mut token = String::with_capacity(word.len()); + for &byte in word.as_bytes() { + token.push(byte.to_ascii_lowercase() as char); + } + token +} + +#[inline(never)] +fn lowercase_chars(word: &str) -> String { + let mut token = String::with_capacity(word.len()); + for ch in word.chars() { + if is_uppercase(ch) { + token.extend(ch.to_lowercase()); + } else { + token.push(ch); + } + } + token +} + +impl<'x> Iterator for SpaceTokenizer<'x> { + type Item = Cow<'x, str>; fn next(&mut self) -> Option { - for ch in self.iterator.by_ref() { - if ch.is_alphanumeric() { - if ch.is_uppercase() { - for ch in ch.to_lowercase() { - self.token.push(ch); - } - } else { - self.token.push(ch); - } - } else if !self.token.is_empty() { - if self.token.len() < self.max_token_length { - return Some(std::mem::take(&mut self.token)); - } else { - self.token.clear(); - } - } - } + let text = self.text; + loop { + let run = next_alnum_run(text, self.pos)?; + self.pos = run.resume; + let word = &text[run.start..run.end]; - if !self.token.is_empty() { - if self.token.len() < self.max_token_length { - return Some(std::mem::take(&mut self.token)); + if !run.has_upper { + if word.len() < self.max_token_length { + return Some(Cow::Borrowed(word)); + } + } else if run.is_ascii { + if word.len() < self.max_token_length { + return Some(Cow::Owned(lowercase_ascii(word))); + } } else { - self.token.clear(); + let token = lowercase_chars(word); + if token.len() < self.max_token_length { + return Some(Cow::Owned(token)); + } } } - - None } } + diff --git a/crates/nlp/src/tokenizers/stream.rs b/crates/nlp/src/tokenizers/stream.rs index 40f02f7e..416424c4 100644 --- a/crates/nlp/src/tokenizers/stream.rs +++ b/crates/nlp/src/tokenizers/stream.rs @@ -73,108 +73,3 @@ impl WordStemTokenizer { } } -#[cfg(test)] -pub mod tests { - use crate::tokenizers::{ - stream::WordStemTokenizer, - types::{TokenType, TypesTokenizer}, - }; - - #[test] - fn stream_tokenizer() { - let inputs = [ - ( - "The quick brown fox jumps over the lazy dog", - vec!["quick", "brown", "fox", "jump", "lazi", "dog"], - ), - ( - "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!", - vec!["jovencill", "emponzoñ", "whisky", "figurot", "exhib"], - ), - ( - "Ma la volpe col suo balzo ha raggiunto il quieto Fido", - vec!["volp", "balz", "raggiunt", "quiet", "fid"], - ), - ( - "Jaz em prisão bota que vexa dez cegonhas felizes", - vec!["jaz", "prisã", "bot", "vex", "dez", "cegonh", "feliz"], - ), - ( - "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich", - vec![ - "zwolf", "boxkampf", "jagt", "victor", "quer", "gross", "sylt", "deich", - ], - ), - ( - "עטלף אבק נס דרך מזגן שהתפוצץ כי חם", - vec!["עטלף", "אבק", "נס", "דרך", "מזגן", "שהתפוצץ", "כי", "חם"], - ), - ( - "Съешь ещё этих мягких французских булок, да выпей же чаю", - vec![ - "съеш", - "ещё", - "эт", - "мягк", - "французск", - "булок", - "вып", - "ча", - ], - ), - ( - "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!", - vec![ - "чуєш", - "їх", - "доцю", - "га", - "кумедна", - "ж", - "ти", - "прощайся", - "без", - "ґольфів", - ], - ), - ( - "Љубазни фењерџија чађавог лица хоће да ми покаже штос", - vec![ - "љубазни", - "фењерџија", - "чађавог", - "лица", - "хоће", - "да", - "ми", - "покаже", - "штос", - ], - ), - ( - "Pijamalı hasta yağız şoföre çabucak güvendi", - vec!["pijamalı", "hasta", "yağız", "şoför", "çabucak", "güvendi"], - ), - ("己所不欲,勿施于人。", vec!["己所不欲", "勿施于人"]), - ( - "井の中の蛙大海を知らず", - vec!["井", "の", "中", "の", "蛙大", "海", "を", "知ら", "ず"], - ), - ("시작이 반이다", vec!["시작이", "반이다"]), - ]; - - for (input, expect) in inputs.iter() { - let tokenizer = WordStemTokenizer::new(input); - let mut result = Vec::new(); - for token in TypesTokenizer::new(&input.to_lowercase()) { - if let TokenType::Alphabetic(word) = token.word { - tokenizer.tokenize(word, |t| { - result.push(t.into_owned()); - }); - } - } - - assert_eq!(&result, expect,); - } - } -} diff --git a/crates/nlp/src/tokenizers/types.rs b/crates/nlp/src/tokenizers/types.rs index 7818d3b6..37239af1 100644 --- a/crates/nlp/src/tokenizers/types.rs +++ b/crates/nlp/src/tokenizers/types.rs @@ -4,18 +4,72 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::str::CharIndices; +use super::{Token, unicode::is_alphabetic}; -use super::Token; +mod scan; + +const ASCII_SPACE: [bool; 256] = { + let mut table = [false; 256]; + let mut byte = 0x09u8; + while byte <= 0x0d { + table[byte as usize] = true; + byte += 1; + } + table[0x20] = true; + table +}; + +#[cfg(stalwart_tests)] +pub static TOKENIZED_BYTES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +const QUEUE_COMPACTION_HEAD: usize = 1024; +const CLASS_ALPHA: u8 = 1; +const CLASS_DIGIT: u8 = 2; + +const ASCII_ALNUM: [u8; 256] = { + let mut table = [0u8; 256]; + let mut byte = b'A'; + while byte <= b'Z' { + table[byte as usize] = CLASS_ALPHA; + table[(byte + 32) as usize] = CLASS_ALPHA; + byte += 1; + } + let mut byte = b'0'; + while byte <= b'9' { + table[byte as usize] = CLASS_DIGIT; + byte += 1; + } + table +}; + +type StopChar<'x> = (TokenType<&'x str, &'x str, &'x str, &'x str>, usize, bool); + +#[inline(always)] +fn char_at(text: &str, pos: usize) -> Option { + text.get(pos..).and_then(|rest| rest.chars().next()) +} + +#[cfg(all(test, stalwart_tests))] +pub(super) fn ascii_alnum_class(byte: u8) -> u8 { + ASCII_ALNUM[byte as usize] +} + +#[cfg(all(test, stalwart_tests))] +pub(super) fn ascii_space_class(byte: u8) -> bool { + ASCII_SPACE[byte as usize] +} #[derive(Debug)] pub struct TypesTokenizer<'x> { text: &'x str, - iter: CharIndices<'x>, + pos: usize, tokens: Vec>>, + head: usize, peek_pos: usize, last_ch_is_space: bool, last_token_is_dot: bool, + next_at: NextByte, + next_dot: NextByte, eof: bool, tokenize_urls: bool, tokenize_urls_without_scheme: bool, @@ -53,10 +107,10 @@ impl<'x> Iterator for TypesTokenizer<'x> { // Try parsing URL with scheme if self.tokenize_urls - && matches!( - token.word, - TokenType::Alphabetic(t) | TokenType::Alphanumeric(t) - if t.len() <= 8 && t.is_ascii()) + && let TokenType::Alphabetic(scheme) | TokenType::Alphanumeric(scheme) = token.word + && scheme_may_follow(self.text, token.to) + && scheme.len() <= 8 + && scheme.is_ascii() && self.try_skip_url_scheme() { if let Some(url) = self.try_parse_url(token.into()) { @@ -68,7 +122,13 @@ impl<'x> Iterator for TypesTokenizer<'x> { } // Try parsing email - if self.tokenize_emails && token.word.is_email_atom() { + if self.tokenize_emails + && token.word.is_email_atom() + && email_may_follow(self.text, token.to) + && self + .next_at + .reachable(self.text, token.from, token.to, FOLLOW_EMAILISH) + { self.peek_rewind(); if let Some(email) = self.try_parse_email() { self.peek_advance(); @@ -78,7 +138,13 @@ impl<'x> Iterator for TypesTokenizer<'x> { } // Try parsing URL without scheme - if self.tokenize_urls_without_scheme && token.word.is_domain_atom(true) { + if self.tokenize_urls_without_scheme + && token.word.is_domain_atom(true) + && host_may_follow(self.text, token.to) + && self + .next_dot + .reachable(self.text, token.from, token.to, FOLLOW_HOSTISH) + { self.peek_rewind(); if let Some(url) = self.try_parse_url(None) { self.peek_advance(); @@ -103,14 +169,19 @@ impl<'x> Iterator for TypesTokenizer<'x> { impl<'x> TypesTokenizer<'x> { pub fn new(text: &'x str) -> Self { + #[cfg(stalwart_tests)] + TOKENIZED_BYTES.fetch_add(text.len() as u64, std::sync::atomic::Ordering::Relaxed); Self { text, - iter: text.char_indices(), + pos: 0, tokens: Vec::new(), eof: false, + head: 0, peek_pos: 0, last_ch_is_space: false, last_token_is_dot: false, + next_at: NextByte::new(b'@'), + next_dot: NextByte::new(b'.'), tokenize_urls: true, tokenize_urls_without_scheme: true, tokenize_emails: true, @@ -139,55 +210,60 @@ impl<'x> TypesTokenizer<'x> { } fn consume(&mut self) -> bool { - let mut has_alpha = false; - let mut has_number = false; - - let mut start_pos = usize::MAX; - let mut end_pos = usize::MAX; - - let mut stop_char = None; - - for (pos, ch) in self.iter.by_ref() { - if ch.is_alphabetic() { - has_alpha = true; - } else if ch.is_ascii_digit() { - has_number = true; - } else { - let last_was_space = self.last_ch_is_space; - self.last_ch_is_space = ch.is_whitespace(); - stop_char = Token { - word: if self.last_ch_is_space { - if last_was_space { - continue; - } else { - TokenType::Space - } - } else if ch.is_ascii() { - TokenType::Punctuation(ch) - } else { - TokenType::Other(ch) - }, - from: pos, - to: pos + ch.len_utf8(), - } - .into(); - break; - } - self.last_ch_is_space = false; - - if start_pos == usize::MAX { - start_pos = pos; - } - end_pos = pos + ch.len_utf8(); + if self.head == self.tokens.len() { + self.tokens.clear(); + self.head = 0; + } else if self.head >= QUEUE_COMPACTION_HEAD { + self.tokens.drain(..self.head); + self.head = 0; } - if start_pos != usize::MAX { + let bytes = self.text.as_bytes(); + let mut pos = self.pos; + + if self.last_ch_is_space { + while let Some(&byte) = bytes.get(pos) { + if byte < 0x80 { + if !ASCII_SPACE[byte as usize] { + break; + } + pos += 1; + } else { + match char_at(self.text, pos) { + Some(ch) if ch.is_whitespace() => pos += ch.len_utf8(), + _ => break, + } + } + } + } + + let start_pos = pos; + let (end_pos, classes, stop_char) = scan::scan_word(self.text, pos); + pos = end_pos; + + if end_pos != start_pos { + self.last_ch_is_space = false; + } + + let stop_char = stop_char.map(|(word, len, is_space)| { + let from = pos; + pos += len; + self.last_ch_is_space = is_space; + Token { + word, + from, + to: from + len, + } + }); + self.pos = pos; + + if end_pos != start_pos { let text = &self.text[start_pos..end_pos]; self.tokens.push(Token { - word: if has_alpha && has_number { + word: if classes == CLASS_ALPHA | CLASS_DIGIT { TokenType::Alphanumeric(text) - } else if has_alpha { + } else if classes == CLASS_ALPHA { TokenType::Alphabetic(text) } else { TokenType::Integer(text) @@ -209,31 +285,36 @@ impl<'x> TypesTokenizer<'x> { } fn next_(&mut self) -> Option>> { - if self.tokens.is_empty() && !self.eof { + if self.head == self.tokens.len() && !self.eof { self.consume(); } - if !self.tokens.is_empty() { - Some(self.tokens.remove(0)) - } else { - None + match self.tokens.get(self.head) { + Some(&token) => { + self.head += 1; + Some(token) + } + None => None, } } fn peek(&mut self) -> Option>> { - while self.tokens.len() <= self.peek_pos && !self.eof { + while self.tokens.len() - self.head <= self.peek_pos && !self.eof { self.consume(); } - self.tokens.get(self.peek_pos).map(|t| { - self.peek_pos += 1; - *t - }) + debug_assert!(self.head + self.peek_pos <= self.tokens.len()); + match self.tokens.get(self.head + self.peek_pos) { + Some(&token) => { + self.peek_pos += 1; + Some(token) + } + None => None, + } } fn peek_advance(&mut self) { - if self.peek_pos > 0 { - self.tokens.drain(..self.peek_pos); - self.peek_pos = 0; - } + debug_assert!(self.head + self.peek_pos <= self.tokens.len()); + self.head += self.peek_pos; + self.peek_pos = 0; } #[inline(always)] @@ -288,18 +369,12 @@ impl<'x> TypesTokenizer<'x> { let mut dot_count = 0; let mut is_ipv6 = false; - let mut last_label_is_tld = false; + let mut last_label = None; while let Some(token) = self.peek() { match token.word { TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) => { - last_label_is_tld = text.len() >= 2 - && psl::Psl::find( - &psl::List, - [text.to_ascii_lowercase().as_bytes()].into_iter(), - ) - .typ - .is_some(); + last_label = Some(text); text_count += 1; } TokenType::Integer(text) => { @@ -344,8 +419,10 @@ impl<'x> TypesTokenizer<'x> { self.peek_pos = restore_pos; let is_ip = is_ipv6 || (int_count == 4 && dot_count == 3 && text_count == 0); if end_pos != usize::MAX { - is_valid_host = - (last_label_is_tld && dot_count >= 1 && (text_count + int_count) >= 2) || is_ip; + is_valid_host = is_ip + || (dot_count >= 1 + && (text_count + int_count) >= 2 + && last_label_is_tld(last_label)); (start_pos, end_pos, is_ip) } else { return None; @@ -502,7 +579,7 @@ impl<'x> TypesTokenizer<'x> { let mut last_ch = u8::MAX; let mut has_int = false; let mut has_alpha = false; - let mut last_label_is_tld = false; + let mut last_label = None; let mut dot_count = 0; let mut start_pos = usize::MAX; @@ -526,26 +603,20 @@ impl<'x> TypesTokenizer<'x> { .map(|(from, to)| (from, to, true)); } TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) if text.len() <= 63 => { - last_label_is_tld = text.len() >= 2 - && psl::Psl::find( - &psl::List, - [text.to_ascii_lowercase().as_bytes()].into_iter(), - ) - .typ - .is_some(); + last_label = Some(text); has_alpha = true; last_ch = 0; } TokenType::Other(_) => { has_alpha = true; - last_label_is_tld = false; + last_label = None; last_ch = 0; } TokenType::Integer(text) => { if text.len() <= 3 { has_int = true; } - last_label_is_tld = false; + last_label = None; last_ch = 0; } _ => { @@ -570,7 +641,7 @@ impl<'x> TypesTokenizer<'x> { } let is_ipv4 = has_int && !has_alpha && dot_count == 3; - if end_pos != usize::MAX && dot_count >= 1 && (last_label_is_tld || is_ipv4) { + if end_pos != usize::MAX && dot_count >= 1 && (is_ipv4 || last_label_is_tld(last_label)) { (start_pos, end_pos, is_ipv4).into() } else { None @@ -605,6 +676,15 @@ impl<'x> TypesTokenizer<'x> { fn try_parse_number(&mut self) -> Option>> { self.peek_rewind(); + if let Some(token) = self.tokens.get(self.head) + && !matches!( + token.word, + TokenType::Integer(_) | TokenType::Punctuation('-') + ) + { + return None; + } + let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; let mut restore_pos = self.peek_pos; @@ -658,6 +738,13 @@ impl<'x> TypesTokenizer<'x> { } fn try_skip_url_scheme(&mut self) -> bool { + if let Some(token) = self.tokens.get(self.head + self.peek_pos) + && !matches!(token.word, TokenType::Punctuation(':' | '+')) + { + self.peek_pos = 0; + return false; + } + enum State { None, PlusAlpha, @@ -686,6 +773,196 @@ impl<'x> TypesTokenizer<'x> { } } +const FOLLOW_RUN: u8 = 1 << 0; +const FOLLOW_EMAIL_PUNCT: u8 = 1 << 1; +const FOLLOW_DOT: u8 = 1 << 2; +const FOLLOW_AT: u8 = 1 << 3; +const FOLLOW_HYPHEN: u8 = 1 << 4; +const FOLLOW_NON_ASCII: u8 = 1 << 5; + +const FOLLOW_CLASS: [u8; 256] = { + let mut table = [0u8; 256]; + let mut index = 0; + while index < 256 { + let byte = index as u8; + let mut class = 0; + if byte.is_ascii_alphanumeric() { + class |= FOLLOW_RUN; + } + if matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'/' + | b'=' + | b'?' + | b'^' + | b'_' + | b'`' + | b'{' + | b'|' + | b'}' + | b'~' + ) { + class |= FOLLOW_EMAIL_PUNCT; + } + if byte == b'.' { + class |= FOLLOW_DOT; + } + if byte == b'@' { + class |= FOLLOW_AT; + } + if byte == b'-' { + class |= FOLLOW_HYPHEN; + } + if byte >= 0x80 { + class |= FOLLOW_NON_ASCII; + } + table[index] = class; + index += 1; + } + table +}; + +#[inline(always)] +fn follow_class(text: &str, at: usize) -> u8 { + match text.as_bytes().get(at) { + Some(&byte) => FOLLOW_CLASS[byte as usize], + None => 0, + } +} + +const FOLLOW_EMAILISH: u8 = FOLLOW_RUN | FOLLOW_EMAIL_PUNCT | FOLLOW_DOT | FOLLOW_NON_ASCII; +const FOLLOW_HOSTISH: u8 = FOLLOW_RUN | FOLLOW_DOT | FOLLOW_HYPHEN | FOLLOW_NON_ASCII; + +const REACH_SCAN: usize = 8; +const REACH_LIMIT: usize = 255; + +#[derive(Debug, Clone, Copy)] +struct NextByte { + byte: u8, + searched_from: usize, + found: usize, +} + +impl NextByte { + const fn new(byte: u8) -> Self { + NextByte { + byte, + searched_from: usize::MAX, + found: 0, + } + } + + #[inline(always)] + fn reachable(&mut self, text: &str, from: usize, at: usize, allowed: u8) -> bool { + let bytes = text.as_bytes(); + let limit = bytes.len().min(from.saturating_add(REACH_LIMIT)); + let Some(window) = bytes.get(at..limit) else { + return false; + }; + for &byte in window.iter().take(REACH_SCAN) { + if byte == self.byte { + return true; + } + if FOLLOW_CLASS[byte as usize] & allowed == 0 { + return false; + } + } + window.len() > REACH_SCAN + && self.reachable_beyond_scan(bytes, at + REACH_SCAN, limit, allowed) + } + + #[inline(never)] + fn reachable_beyond_scan( + &mut self, + bytes: &[u8], + at: usize, + limit: usize, + allowed: u8, + ) -> bool { + if at < self.searched_from || at > self.found { + self.searched_from = at; + self.found = bytes + .get(at..) + .and_then(|rest| memchr::memchr(self.byte, rest)) + .map_or(bytes.len(), |offset| at + offset); + } + self.found < limit + && bytes.get(at..self.found).is_some_and(|between| { + between + .iter() + .all(|&byte| FOLLOW_CLASS[byte as usize] & allowed != 0) + }) + } +} + +#[inline(always)] +fn scheme_may_follow(text: &str, at: usize) -> bool { + matches!(text.as_bytes().get(at), Some(b':' | b'+')) +} + +#[inline(always)] +fn email_may_follow(text: &str, at: usize) -> bool { + let class = follow_class(text, at); + if class & (FOLLOW_RUN | FOLLOW_EMAIL_PUNCT | FOLLOW_AT | FOLLOW_NON_ASCII) != 0 { + true + } else { + class & FOLLOW_DOT != 0 + && follow_class(text, at + 1) & (FOLLOW_RUN | FOLLOW_EMAIL_PUNCT | FOLLOW_NON_ASCII) + != 0 + } +} + +#[inline(always)] +fn host_may_follow(text: &str, at: usize) -> bool { + let class = follow_class(text, at); + if class & (FOLLOW_RUN | FOLLOW_HYPHEN | FOLLOW_NON_ASCII) != 0 { + true + } else { + class & FOLLOW_DOT != 0 && follow_class(text, at + 1) & (FOLLOW_RUN | FOLLOW_NON_ASCII) != 0 + } +} + +#[inline(always)] +fn last_label_is_tld(label: Option<&str>) -> bool { + match label { + Some(text) if text.len() >= 2 => is_public_suffix(text), + _ => false, + } +} + +#[inline(never)] +fn is_public_suffix(text: &str) -> bool { + let bytes = text.as_bytes(); + if !bytes.iter().any(u8::is_ascii_uppercase) { + return has_suffix_type(bytes); + } + let mut buf = [0u8; 64]; + match buf.get_mut(..bytes.len()) { + Some(lowered) => { + for (dst, src) in lowered.iter_mut().zip(bytes) { + *dst = src.to_ascii_lowercase(); + } + has_suffix_type(lowered) + } + None => has_suffix_type(text.to_ascii_lowercase().as_bytes()), + } +} + +#[inline(always)] +fn has_suffix_type(label: &[u8]) -> bool { + psl::Psl::find(&psl::List, [label].into_iter()) + .typ + .is_some() +} + impl TokenType { fn is_email_atom(&self) -> bool { matches!( @@ -728,2150 +1005,3 @@ impl TokenType { } } -#[cfg(test)] -mod test { - - use super::{TokenType, TypesTokenizer}; - - #[test] - fn type_tokenizer() { - // Credits: test suite from linkify crate - for (text, expected) in [ - ("", vec![]), - ("foo", vec![TokenType::Alphabetic("foo")]), - (":", vec![TokenType::Punctuation(':')]), - ( - "://", - vec![ - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - ], - ), - ( - ":::", - vec![ - TokenType::Punctuation(':'), - TokenType::Punctuation(':'), - TokenType::Punctuation(':'), - ], - ), - ( - "://foo", - vec![ - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ( - "1://foo", - vec![ - TokenType::Integer("1"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ( - "123://foo", - vec![ - TokenType::Integer("123"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ( - "+://foo", - vec![ - TokenType::Punctuation('+'), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ( - "-://foo", - vec![ - TokenType::Punctuation('-'), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ( - ".://foo", - vec![ - TokenType::Punctuation('.'), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("foo"), - ], - ), - ("1abc://foo", vec![TokenType::UrlNoHost("1abc://foo")]), - ("a://foo", vec![TokenType::UrlNoHost("a://foo")]), - ("a123://foo", vec![TokenType::UrlNoHost("a123://foo")]), - ("a123b://foo", vec![TokenType::UrlNoHost("a123b://foo")]), - ("a+b://foo", vec![TokenType::UrlNoHost("a+b://foo")]), - ( - "a-b://foo", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('-'), - TokenType::UrlNoHost("b://foo"), - ], - ), - ( - "a.b://foo", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('.'), - TokenType::UrlNoHost("b://foo"), - ], - ), - ("ABC://foo", vec![TokenType::UrlNoHost("ABC://foo")]), - ( - ".http://example.org/", - vec![ - TokenType::Punctuation('.'), - TokenType::Url("http://example.org/"), - ], - ), - ( - "1.http://example.org/", - vec![ - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::Url("http://example.org/"), - ], - ), - ( - "ab://", - vec![ - TokenType::Alphabetic("ab"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - ], - ), - ( - "file://", - vec![ - TokenType::Alphabetic("file"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - ], - ), - ( - "file:// ", - vec![ - TokenType::Alphabetic("file"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Space, - ], - ), - ( - "\"file://\"", - vec![ - TokenType::Punctuation('"'), - TokenType::Alphabetic("file"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Punctuation('"'), - ], - ), - ( - "\"file://...\", ", - vec![ - TokenType::Punctuation('"'), - TokenType::Alphabetic("file"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('"'), - TokenType::Punctuation(','), - TokenType::Space, - ], - ), - ( - "file://somefile", - vec![TokenType::UrlNoHost("file://somefile")], - ), - ( - "file://../relative", - vec![TokenType::UrlNoHost("file://../relative")], - ), - ( - "http://a.", - vec![ - TokenType::UrlNoHost("http://a"), - TokenType::Punctuation('.'), - ], - ), - ("http://127.0.0.1", vec![TokenType::Url("http://127.0.0.1")]), - ( - "http://127.0.0.1/", - vec![TokenType::Url("http://127.0.0.1/")], - ), - ("ab://c", vec![TokenType::UrlNoHost("ab://c")]), - ( - "http://example.org/", - vec![TokenType::Url("http://example.org/")], - ), - ( - "http://example.org/123", - vec![TokenType::Url("http://example.org/123")], - ), - ( - "http://example.org/?foo=test&bar=123", - vec![TokenType::Url("http://example.org/?foo=test&bar=123")], - ), - ( - "http://example.org/?foo=%20", - vec![TokenType::Url("http://example.org/?foo=%20")], - ), - ( - "http://example.org/%3C", - vec![TokenType::Url("http://example.org/%3C")], - ), - ("example.org/", vec![TokenType::UrlNoScheme("example.org/")]), - ( - "example.org/123", - vec![TokenType::UrlNoScheme("example.org/123")], - ), - ( - "example.org/?foo=test&bar=123", - vec![TokenType::UrlNoScheme("example.org/?foo=test&bar=123")], - ), - ( - "example.org/?foo=%20", - vec![TokenType::UrlNoScheme("example.org/?foo=%20")], - ), - ( - "example.org/%3C", - vec![TokenType::UrlNoScheme("example.org/%3C")], - ), - ( - "foo http://example.org/", - vec![ - TokenType::Alphabetic("foo"), - TokenType::Space, - TokenType::Url("http://example.org/"), - ], - ), - ( - "http://example.org/ bar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/\tbar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/\nbar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/\u{b}bar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/\u{c}bar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/\rbar", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "foo example.org/", - vec![ - TokenType::Alphabetic("foo"), - TokenType::Space, - TokenType::UrlNoScheme("example.org/"), - ], - ), - ( - "example.org/ bar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/\tbar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/\nbar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/\u{b}bar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/\u{c}bar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/\rbar", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Space, - TokenType::Alphabetic("bar"), - ], - ), - ( - "http://example.org/<", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('<'), - ], - ), - ( - "http://example.org/>", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org/<>", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('<'), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org/\0", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('\0'), - ], - ), - ( - "http://example.org/\u{e}", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('\u{e}'), - ], - ), - ( - "http://example.org/\u{7f}", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('\u{7f}'), - ], - ), - ( - "http://example.org/\u{9f}", - vec![TokenType::Url("http://example.org/\u{9f}")], - ), - ( - "http://example.org/foo|bar", - vec![ - TokenType::Url("http://example.org/foo"), - TokenType::Punctuation('|'), - TokenType::Alphabetic("bar"), - ], - ), - ( - "example.org/<", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('<'), - ], - ), - ( - "example.org/>", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org/<>", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('<'), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org/\0", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('\0'), - ], - ), - ( - "example.org/\u{e}", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('\u{e}'), - ], - ), - ( - "example.org/\u{7f}", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('\u{7f}'), - ], - ), - ( - "example.org/\u{9f}", - vec![TokenType::UrlNoScheme("example.org/\u{9f}")], - ), - ( - "http://example.org/.", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('.'), - ], - ), - ( - "http://example.org/..", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - ], - ), - ( - "http://example.org/,", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation(','), - ], - ), - ( - "http://example.org/:", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation(':'), - ], - ), - ( - "http://example.org/?", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('?'), - ], - ), - ( - "http://example.org/!", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('!'), - ], - ), - ( - "http://example.org/;", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation(';'), - ], - ), - ( - "example.org/.", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('.'), - ], - ), - ( - "example.org/..", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - ], - ), - ( - "example.org/,", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(','), - ], - ), - ( - "example.org/:", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(':'), - ], - ), - ( - "example.org/?", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('?'), - ], - ), - ( - "example.org/!", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('!'), - ], - ), - ( - "example.org/;", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(';'), - ], - ), - ( - "http://example.org/a(b)", - vec![TokenType::Url("http://example.org/a(b)")], - ), - ( - "http://example.org/a[b]", - vec![TokenType::Url("http://example.org/a[b]")], - ), - ( - "http://example.org/a{b}", - vec![TokenType::Url("http://example.org/a{b}")], - ), - ( - "http://example.org/a'b'", - vec![TokenType::Url("http://example.org/a'b'")], - ), - ( - "(http://example.org/)", - vec![ - TokenType::Punctuation('('), - TokenType::Url("http://example.org/"), - TokenType::Punctuation(')'), - ], - ), - ( - "[http://example.org/]", - vec![ - TokenType::Punctuation('['), - TokenType::Url("http://example.org/"), - TokenType::Punctuation(']'), - ], - ), - ( - "{http://example.org/}", - vec![ - TokenType::Punctuation('{'), - TokenType::Url("http://example.org/"), - TokenType::Punctuation('}'), - ], - ), - ( - "\"http://example.org/\"", - vec![ - TokenType::Punctuation('"'), - TokenType::Url("http://example.org/"), - TokenType::Punctuation('"'), - ], - ), - ( - "'http://example.org/'", - vec![ - TokenType::Punctuation('\''), - TokenType::Url("http://example.org/"), - TokenType::Punctuation('\''), - ], - ), - ( - "example.org/a(b)", - vec![TokenType::UrlNoScheme("example.org/a(b)")], - ), - ( - "example.org/a[b]", - vec![TokenType::UrlNoScheme("example.org/a[b]")], - ), - ( - "example.org/a{b}", - vec![TokenType::UrlNoScheme("example.org/a{b}")], - ), - ( - "example.org/a'b'", - vec![TokenType::UrlNoScheme("example.org/a'b'")], - ), - ( - "(example.org/)", - vec![ - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(')'), - ], - ), - ( - "[example.org/]", - vec![ - TokenType::Punctuation('['), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(']'), - ], - ), - ( - "{example.org/}", - vec![ - TokenType::Punctuation('{'), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('}'), - ], - ), - ( - "\"example.org/\"", - vec![ - TokenType::Punctuation('"'), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('"'), - ], - ), - ( - "'example.org/'", - vec![ - TokenType::Punctuation('\''), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('\''), - ], - ), - ( - "((http://example.org/))", - vec![ - TokenType::Punctuation('('), - TokenType::Punctuation('('), - TokenType::Url("http://example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation(')'), - ], - ), - ( - "((http://example.org/a(b)))", - vec![ - TokenType::Punctuation('('), - TokenType::Punctuation('('), - TokenType::Url("http://example.org/a(b)"), - TokenType::Punctuation(')'), - TokenType::Punctuation(')'), - ], - ), - ( - "[(http://example.org/)]", - vec![ - TokenType::Punctuation('['), - TokenType::Punctuation('('), - TokenType::Url("http://example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation(']'), - ], - ), - ( - "(http://example.org/).", - vec![ - TokenType::Punctuation('('), - TokenType::Url("http://example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation('.'), - ], - ), - ( - "(http://example.org/.)", - vec![ - TokenType::Punctuation('('), - TokenType::Url("http://example.org/"), - TokenType::Punctuation('.'), - TokenType::Punctuation(')'), - ], - ), - ( - "http://example.org/>", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org/(", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('('), - ], - ), - ( - "http://example.org/(.", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('('), - TokenType::Punctuation('.'), - ], - ), - ( - "http://example.org/]()", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation(']'), - TokenType::Punctuation('('), - TokenType::Punctuation(')'), - ], - ), - ( - "((example.org/))", - vec![ - TokenType::Punctuation('('), - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation(')'), - ], - ), - ( - "((example.org/a(b)))", - vec![ - TokenType::Punctuation('('), - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/a(b)"), - TokenType::Punctuation(')'), - TokenType::Punctuation(')'), - ], - ), - ( - "[(example.org/)]", - vec![ - TokenType::Punctuation('['), - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation(']'), - ], - ), - ( - "(example.org/).", - vec![ - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation('.'), - ], - ), - ( - "(example.org/.)", - vec![ - TokenType::Punctuation('('), - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('.'), - TokenType::Punctuation(')'), - ], - ), - ( - "example.org/>", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org/(", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('('), - ], - ), - ( - "example.org/(.", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('('), - TokenType::Punctuation('.'), - ], - ), - ( - "example.org/]()", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation(']'), - TokenType::Punctuation('('), - TokenType::Punctuation(')'), - ], - ), - ( - "'https://example.org'", - vec![ - TokenType::Punctuation('\''), - TokenType::Url("https://example.org"), - TokenType::Punctuation('\''), - ], - ), - ( - "\"https://example.org\"", - vec![ - TokenType::Punctuation('"'), - TokenType::Url("https://example.org"), - TokenType::Punctuation('"'), - ], - ), - ( - "''https://example.org''", - vec![ - TokenType::Punctuation('\''), - TokenType::Punctuation('\''), - TokenType::Url("https://example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('\''), - ], - ), - ( - "'https://example.org''", - vec![ - TokenType::Punctuation('\''), - TokenType::Url("https://example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('\''), - ], - ), - ( - "'https://example.org", - vec![ - TokenType::Punctuation('\''), - TokenType::Url("https://example.org"), - ], - ), - ( - "http://example.org/'_(foo)", - vec![TokenType::Url("http://example.org/'_(foo)")], - ), - ( - "http://example.org/'_(foo)'", - vec![TokenType::Url("http://example.org/'_(foo)'")], - ), - ( - "http://example.org/''", - vec![TokenType::Url("http://example.org/''")], - ), - ( - "http://example.org/'''", - vec![ - TokenType::Url("http://example.org/''"), - TokenType::Punctuation('\''), - ], - ), - ( - "http://example.org/'.", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('\''), - TokenType::Punctuation('.'), - ], - ), - ( - "http://example.org/'a", - vec![TokenType::Url("http://example.org/'a")], - ), - ( - "http://example.org/it's", - vec![TokenType::Url("http://example.org/it's")], - ), - ( - "example.org/'_(foo)", - vec![TokenType::UrlNoScheme("example.org/'_(foo)")], - ), - ( - "example.org/'_(foo)'", - vec![TokenType::UrlNoScheme("example.org/'_(foo)'")], - ), - ( - "example.org/''", - vec![TokenType::UrlNoScheme("example.org/''")], - ), - ( - "example.org/'''", - vec![ - TokenType::UrlNoScheme("example.org/''"), - TokenType::Punctuation('\''), - ], - ), - ( - "example.org/'.", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('\''), - TokenType::Punctuation('.'), - ], - ), - ( - "example.org/'a", - vec![TokenType::UrlNoScheme("example.org/'a")], - ), - ( - "example.org/it's", - vec![TokenType::UrlNoScheme("example.org/it's")], - ), - ( - "http://example.org/\"a", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('"'), - TokenType::Alphabetic("a"), - ], - ), - ( - "http://example.org/\"a\"", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('"'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('"'), - ], - ), - ( - "http://example.org/`a", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('`'), - TokenType::Alphabetic("a"), - ], - ), - ( - "http://example.org/`a`", - vec![ - TokenType::Url("http://example.org/"), - TokenType::Punctuation('`'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('`'), - ], - ), - ( - "https://example.org*", - vec![ - TokenType::Url("https://example.org"), - TokenType::Punctuation('*'), - ], - ), - ( - "https://example.org/*", - vec![ - TokenType::Url("https://example.org/"), - TokenType::Punctuation('*'), - ], - ), - ( - "https://example.org/**", - vec![ - TokenType::Url("https://example.org/"), - TokenType::Punctuation('*'), - TokenType::Punctuation('*'), - ], - ), - ( - "https://example.org/*/a", - vec![TokenType::Url("https://example.org/*/a")], - ), - ( - "example.org/`a", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('`'), - TokenType::Alphabetic("a"), - ], - ), - ( - "example.org/`a`", - vec![ - TokenType::UrlNoScheme("example.org/"), - TokenType::Punctuation('`'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('`'), - ], - ), - ( - "http://example.org\">", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org'>", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org\"/>", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation('/'), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org'/>", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('/'), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org

", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('<'), - TokenType::Alphabetic("p"), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org

", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("p"), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org\">", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org'>", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org\"/>", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation('/'), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org'/>", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation('/'), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org

", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('<'), - TokenType::Alphabetic("p"), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org

", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("p"), - TokenType::Punctuation('>'), - ], - ), - ( - "http://example.org\");", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation(')'), - TokenType::Punctuation(';'), - ], - ), - ( - "http://example.org');", - vec![ - TokenType::Url("http://example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation(')'), - TokenType::Punctuation(';'), - ], - ), - ( - "", - vec![ - TokenType::Punctuation('<'), - TokenType::Alphabetic("img"), - TokenType::Space, - TokenType::Alphabetic("src"), - TokenType::Punctuation('='), - TokenType::Punctuation('"'), - TokenType::Url("http://example.org/test.svg"), - TokenType::Punctuation('"'), - TokenType::Punctuation('>'), - ], - ), - ( - "
", - vec![ - TokenType::Punctuation('<'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Alphabetic("a"), - TokenType::Space, - TokenType::Alphabetic("href"), - TokenType::Punctuation('='), - TokenType::Punctuation('"'), - TokenType::Url("http://example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - ], - ), - ( - "
", - vec![ - TokenType::Punctuation('<'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Alphabetic("a"), - TokenType::Space, - TokenType::Alphabetic("href"), - TokenType::Punctuation('='), - TokenType::Punctuation('"'), - TokenType::Url("http://example.org"), - TokenType::Punctuation('"'), - TokenType::Space, - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('>'), - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - ], - ), - ( - "
\n \n
", - vec![ - TokenType::Punctuation('<'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - TokenType::Space, - TokenType::Punctuation('<'), - TokenType::Alphabetic("img"), - TokenType::Space, - TokenType::Alphabetic("src"), - TokenType::Punctuation('='), - TokenType::Punctuation('"'), - TokenType::Url("http://example.org/test3.jpg"), - TokenType::Punctuation('"'), - TokenType::Space, - TokenType::Punctuation('/'), - TokenType::Punctuation('>'), - TokenType::Space, - TokenType::Punctuation('<'), - TokenType::Punctuation('/'), - TokenType::Alphabetic("div"), - TokenType::Punctuation('>'), - ], - ), - ( - "example.org\");", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('"'), - TokenType::Punctuation(')'), - TokenType::Punctuation(';'), - ], - ), - ( - "example.org');", - vec![ - TokenType::UrlNoScheme("example.org"), - TokenType::Punctuation('\''), - TokenType::Punctuation(')'), - TokenType::Punctuation(';'), - ], - ), - ( - "http://example.org/", - vec![TokenType::Url("http://example.org/")], - ), - ( - "http://example.org/a/", - vec![TokenType::Url("http://example.org/a/")], - ), - ( - "http://example.org//", - vec![TokenType::Url("http://example.org//")], - ), - ("example.org/", vec![TokenType::UrlNoScheme("example.org/")]), - ( - "example.org/a/", - vec![TokenType::UrlNoScheme("example.org/a/")], - ), - ( - "example.org//", - vec![TokenType::UrlNoScheme("example.org//")], - ), - ( - "http://one.org/ http://two.org/", - vec![ - TokenType::Url("http://one.org/"), - TokenType::Space, - TokenType::Url("http://two.org/"), - ], - ), - ( - "http://one.org/ : http://two.org/", - vec![ - TokenType::Url("http://one.org/"), - TokenType::Space, - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::Url("http://two.org/"), - ], - ), - ( - "(http://one.org/)(http://two.org/)", - vec![ - TokenType::Punctuation('('), - TokenType::Url("http://one.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation('('), - TokenType::Url("http://two.org/"), - TokenType::Punctuation(')'), - ], - ), - ( - "one.org/ two.org/", - vec![ - TokenType::UrlNoScheme("one.org/"), - TokenType::Space, - TokenType::UrlNoScheme("two.org/"), - ], - ), - ( - "one.org/ : two.org/", - vec![ - TokenType::UrlNoScheme("one.org/"), - TokenType::Space, - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::UrlNoScheme("two.org/"), - ], - ), - ( - "(one.org/)(two.org/)", - vec![ - TokenType::Punctuation('('), - TokenType::UrlNoScheme("one.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation('('), - TokenType::UrlNoScheme("two.org/"), - TokenType::Punctuation(')'), - ], - ), - ( - "http://one.org/ two.org/", - vec![ - TokenType::Url("http://one.org/"), - TokenType::Space, - TokenType::UrlNoScheme("two.org/"), - ], - ), - ( - "one.org/ : http://two.org/", - vec![ - TokenType::UrlNoScheme("one.org/"), - TokenType::Space, - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::Url("http://two.org/"), - ], - ), - ( - "(http://one.org/)(two.org/)", - vec![ - TokenType::Punctuation('('), - TokenType::Url("http://one.org/"), - TokenType::Punctuation(')'), - TokenType::Punctuation('('), - TokenType::UrlNoScheme("two.org/"), - TokenType::Punctuation(')'), - ], - ), - ( - "http://üñîçøðé.com", - vec![TokenType::Url("http://üñîçøðé.com")], - ), - ( - "http://üñîçøðé.com/ä", - vec![TokenType::Url("http://üñîçøðé.com/ä")], - ), - ( - "http://example.org/¡", - vec![TokenType::Url("http://example.org/¡")], - ), - ( - "http://example.org/¢", - vec![TokenType::Url("http://example.org/¢")], - ), - ( - "http://example.org/😀", - vec![TokenType::Url("http://example.org/😀")], - ), - ( - "http://example.org/¢/", - vec![TokenType::Url("http://example.org/¢/")], - ), - ( - "http://xn--c1h.example.com/", - vec![TokenType::Url("http://xn--c1h.example.com/")], - ), - ("üñîçøðé.com", vec![TokenType::UrlNoScheme("üñîçøðé.com")]), - ( - "üñîçøðé.com/ä", - vec![TokenType::UrlNoScheme("üñîçøðé.com/ä")], - ), - ( - "example.org/¡", - vec![TokenType::UrlNoScheme("example.org/¡")], - ), - ( - "example.org/¢", - vec![TokenType::UrlNoScheme("example.org/¢")], - ), - ( - "example.org/😀", - vec![TokenType::UrlNoScheme("example.org/😀")], - ), - ( - "example.org/¢/", - vec![TokenType::UrlNoScheme("example.org/¢/")], - ), - ( - "xn--c1h.example.com/", - vec![TokenType::UrlNoScheme("xn--c1h.example.com/")], - ), - ( - "example.", - vec![ - TokenType::Alphabetic("example"), - TokenType::Punctuation('.'), - ], - ), - ( - "example./", - vec![ - TokenType::Alphabetic("example"), - TokenType::Punctuation('.'), - TokenType::Punctuation('/'), - ], - ), - ( - "foo.com.", - vec![ - TokenType::UrlNoScheme("foo.com"), - TokenType::Punctuation('.'), - ], - ), - ( - "example.c", - vec![ - TokenType::Alphabetic("example"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("c"), - ], - ), - ("example.co", vec![TokenType::UrlNoScheme("example.co")]), - ("example.com", vec![TokenType::UrlNoScheme("example.com")]), - ("e.com", vec![TokenType::UrlNoScheme("e.com")]), - ( - "exampl.e.c", - vec![ - TokenType::Alphabetic("exampl"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("e"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("c"), - ], - ), - ("exampl.e.co", vec![TokenType::UrlNoScheme("exampl.e.co")]), - ( - "e.xample.c", - vec![ - TokenType::Alphabetic("e"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("xample"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("c"), - ], - ), - ("e.xample.co", vec![TokenType::UrlNoScheme("e.xample.co")]), - ( - "v1.1.1", - vec![ - TokenType::Alphanumeric("v1"), - TokenType::Punctuation('.'), - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::Integer("1"), - ], - ), - ( - "foo.bar@example.org", - vec![TokenType::Email("foo.bar@example.org")], - ), - ( - "example.com@example.com", - vec![TokenType::Email("example.com@example.com")], - ), - ( - "Look, no scheme: example.org/foo email@foo.com", - vec![ - TokenType::Alphabetic("Look"), - TokenType::Punctuation(','), - TokenType::Space, - TokenType::Alphabetic("no"), - TokenType::Space, - TokenType::Alphabetic("scheme"), - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::UrlNoScheme("example.org/foo"), - TokenType::Space, - TokenType::Email("email@foo.com"), - ], - ), - ( - "Web:\nwww.foobar.co\nE-Mail:\n bar@foobar.co (bla bla bla)", - vec![ - TokenType::Alphabetic("Web"), - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::UrlNoScheme("www.foobar.co"), - TokenType::Space, - TokenType::Alphabetic("E"), - TokenType::Punctuation('-'), - TokenType::Alphabetic("Mail"), - TokenType::Punctuation(':'), - TokenType::Space, - TokenType::Email("bar@foobar.co"), - TokenType::Space, - TokenType::Punctuation('('), - TokenType::Alphabetic("bla"), - TokenType::Space, - TokenType::Alphabetic("bla"), - TokenType::Space, - TokenType::Alphabetic("bla"), - TokenType::Punctuation(')'), - ], - ), - ( - "upi://pay?pa=XXXXXXX&pn=XXXXX", - vec![TokenType::UrlNoHost("upi://pay?pa=XXXXXXX&pn=XXXXX")], - ), - ( - "https://example.org?pa=XXXXXXX&pn=XXXXX", - vec![TokenType::Url("https://example.org?pa=XXXXXXX&pn=XXXXX")], - ), - ( - "website https://domain.com", - vec![ - TokenType::Alphabetic("website"), - TokenType::Space, - TokenType::Url("https://domain.com"), - ], - ), - ("a12.b-c.com", vec![TokenType::UrlNoScheme("a12.b-c.com")]), - ( - "v1.2.3", - vec![ - TokenType::Alphanumeric("v1"), - TokenType::Punctuation('.'), - TokenType::Integer("2"), - TokenType::Punctuation('.'), - TokenType::Integer("3"), - ], - ), - ( - "https://12-7.0.0.1/", - vec![TokenType::UrlNoHost("https://12-7.0.0.1/")], - ), - ( - "https://user:pass@example.com/", - vec![TokenType::Url("https://user:pass@example.com/")], - ), - ( - "https://user:-.!$@example.com/", - vec![TokenType::Url("https://user:-.!$@example.com/")], - ), - ( - "https://user:!$&'()*+,;=@example.com/", - vec![TokenType::Url("https://user:!$&'()*+,;=@example.com/")], - ), - ( - "https://user:pass@ex@mple.com/", - vec![ - TokenType::UrlNoHost("https://user:pass@ex"), - TokenType::Punctuation('@'), - TokenType::UrlNoScheme("mple.com/"), - ], - ), - ( - "https://localhost:8080!", - vec![ - TokenType::UrlNoHost("https://localhost:8080"), - TokenType::Punctuation('!'), - ], - ), - ( - "https://localhost:8080/", - vec![TokenType::UrlNoHost("https://localhost:8080/")], - ), - ( - "https://user:pass@example.com:8080/hi", - vec![TokenType::Url("https://user:pass@example.com:8080/hi")], - ), - ( - "https://127.0.0.1/", - vec![TokenType::Url("https://127.0.0.1/")], - ), - ("1.0.0.0", vec![TokenType::IpAddr("1.0.0.0")]), - ( - "1.0.0.0/foo/bar", - vec![TokenType::UrlNoScheme("1.0.0.0/foo/bar")], - ), - ("1.0 ", vec![TokenType::Float("1.0"), TokenType::Space]), - ( - "1.0.0", - vec![ - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::Integer("0"), - TokenType::Punctuation('.'), - TokenType::Integer("0"), - ], - ), - ( - "1.0.0.0.0", - vec![ - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::IpAddr("0.0.0.0"), - ], - ), - ( - "1.0.0.", - vec![ - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::Integer("0"), - TokenType::Punctuation('.'), - TokenType::Integer("0"), - TokenType::Punctuation('.'), - ], - ), - ( - "https://example.com.:8080/test", - vec![TokenType::Url("https://example.com.:8080/test")], - ), - ( - "https://example.org'", - vec![ - TokenType::Url("https://example.org"), - TokenType::Punctuation('\''), - ], - ), - ( - "https://example.org'a@example.com", - vec![TokenType::Url("https://example.org'a@example.com")], - ), - ( - "https://a.com'https://b.com", - vec![ - TokenType::UrlNoHost("https://a.com'https"), - TokenType::Punctuation(':'), - TokenType::Punctuation('/'), - TokenType::Punctuation('/'), - TokenType::UrlNoScheme("b.com"), - ], - ), - ( - "https://example.com...", - vec![ - TokenType::Url("https://example.com"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - ], - ), - ( - "www.example..com", - vec![ - TokenType::Alphabetic("www"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("example"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Alphabetic("com"), - ], - ), - ( - "https://.www.example.com", - vec![TokenType::Url("https://.www.example.com")], - ), - ( - "-a.com", - vec![TokenType::Punctuation('-'), TokenType::UrlNoScheme("a.com")], - ), - ("https://a.-b.com", vec![TokenType::Url("https://a.-b.com")]), - ( - "a-.com", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('-'), - TokenType::Punctuation('.'), - TokenType::Alphabetic("com"), - ], - ), - ( - "a.b-.com", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("b"), - TokenType::Punctuation('-'), - TokenType::Punctuation('.'), - TokenType::Alphabetic("com"), - ], - ), - ("https://a.b-.com", vec![TokenType::Url("https://a.b-.com")]), - ( - "https://example.com-/", - vec![ - TokenType::Url("https://example.com"), - TokenType::Punctuation('-'), - TokenType::Punctuation('/'), - ], - ), - ( - "https://example.org-", - vec![ - TokenType::Url("https://example.org"), - TokenType::Punctuation('-'), - ], - ), - ( - "example.com@about", - vec![ - TokenType::UrlNoScheme("example.com"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("about"), - ], - ), - ( - "example.com/@about", - vec![TokenType::UrlNoScheme("example.com/@about")], - ), - ( - "https://example.com/@about", - vec![TokenType::Url("https://example.com/@about")], - ), - ( - "info@v1.1.1", - vec![ - TokenType::Alphabetic("info"), - TokenType::Punctuation('@'), - TokenType::Alphanumeric("v1"), - TokenType::Punctuation('.'), - TokenType::Integer("1"), - TokenType::Punctuation('.'), - TokenType::Integer("1"), - ], - ), - ("file:///", vec![TokenType::UrlNoHost("file:///")]), - ( - "file:///home/foo", - vec![TokenType::UrlNoHost("file:///home/foo")], - ), - ( - "file://localhost/home/foo", - vec![TokenType::UrlNoHost("file://localhost/home/foo")], - ), - ( - "facetime://+19995551234", - vec![TokenType::UrlNoHost("facetime://+19995551234")], - ), - ( - "test://123'456!!!", - vec![ - TokenType::UrlNoHost("test://123'456"), - TokenType::Punctuation('!'), - TokenType::Punctuation('!'), - TokenType::Punctuation('!'), - ], - ), - ( - "test://123'456...", - vec![ - TokenType::UrlNoHost("test://123'456"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - ], - ), - ( - "test://123'456!!!/", - vec![ - TokenType::UrlNoHost("test://123'456"), - TokenType::Punctuation('!'), - TokenType::Punctuation('!'), - TokenType::Punctuation('!'), - TokenType::Punctuation('/'), - ], - ), - ( - "test://123'456.../", - vec![ - TokenType::UrlNoHost("test://123'456"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Punctuation('/'), - ], - ), - ( - "1abc://example.com", - vec![TokenType::Url("1abc://example.com")], - ), - ( - "¡¢example.com", - vec![TokenType::UrlNoScheme("¡¢example.com")], - ), - ("foo", vec![TokenType::Alphabetic("foo")]), - ("@", vec![TokenType::Punctuation('@')]), - ( - "a@", - vec![TokenType::Alphabetic("a"), TokenType::Punctuation('@')], - ), - ( - "@a", - vec![TokenType::Punctuation('@'), TokenType::Alphabetic("a")], - ), - ( - "@@@", - vec![ - TokenType::Punctuation('@'), - TokenType::Punctuation('@'), - TokenType::Punctuation('@'), - ], - ), - ("foo@example.com", vec![TokenType::Email("foo@example.com")]), - ( - "foo.bar@example.com", - vec![TokenType::Email("foo.bar@example.com")], - ), - ( - "#!$%&'*+-/=?^_`{}|~@example.org", - vec![TokenType::Email("#!$%&'*+-/=?^_`{}|~@example.org")], - ), - ( - "foo a@b.com", - vec![ - TokenType::Alphabetic("foo"), - TokenType::Space, - TokenType::Email("a@b.com"), - ], - ), - ( - "a@b.com foo", - vec![ - TokenType::Email("a@b.com"), - TokenType::Space, - TokenType::Alphabetic("foo"), - ], - ), - ( - "\na@b.com", - vec![TokenType::Space, TokenType::Email("a@b.com")], - ), - ( - "a@b.com\n", - vec![TokenType::Email("a@b.com"), TokenType::Space], - ), - ( - "(a@example.com)", - vec![ - TokenType::Punctuation('('), - TokenType::Email("a@example.com"), - TokenType::Punctuation(')'), - ], - ), - ( - "\"a@example.com\"", - vec![ - TokenType::Punctuation('"'), - TokenType::Email("a@example.com"), - TokenType::Punctuation('"'), - ], - ), - ( - "\"a@example.com\"", - vec![ - TokenType::Punctuation('"'), - TokenType::Email("a@example.com"), - TokenType::Punctuation('"'), - ], - ), - ( - ",a@example.com,", - vec![ - TokenType::Punctuation(','), - TokenType::Email("a@example.com"), - TokenType::Punctuation(','), - ], - ), - ( - ":a@example.com:", - vec![ - TokenType::Punctuation(':'), - TokenType::Email("a@example.com"), - TokenType::Punctuation(':'), - ], - ), - ( - ";a@example.com;", - vec![ - TokenType::Punctuation(';'), - TokenType::Email("a@example.com"), - TokenType::Punctuation(';'), - ], - ), - ( - ".@example.com", - vec![ - TokenType::Punctuation('.'), - TokenType::Punctuation('@'), - TokenType::UrlNoScheme("example.com"), - ], - ), - ( - "foo.@example.com", - vec![ - TokenType::Alphabetic("foo"), - TokenType::Punctuation('.'), - TokenType::Punctuation('@'), - TokenType::UrlNoScheme("example.com"), - ], - ), - ( - ".foo@example.com", - vec![ - TokenType::Punctuation('.'), - TokenType::Email("foo@example.com"), - ], - ), - ( - ".foo@example.com", - vec![ - TokenType::Punctuation('.'), - TokenType::Email("foo@example.com"), - ], - ), - ( - "a..b@example.com", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('.'), - TokenType::Punctuation('.'), - TokenType::Email("b@example.com"), - ], - ), - ( - "a@example.com.", - vec![ - TokenType::Email("a@example.com"), - TokenType::Punctuation('.'), - ], - ), - ( - "a@b", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("b"), - ], - ), - ( - "a@b.", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("b"), - TokenType::Punctuation('.'), - ], - ), - ( - "a@b.com.", - vec![TokenType::Email("a@b.com"), TokenType::Punctuation('.')], - ), - ( - "a@example.com-", - vec![ - TokenType::Email("a@example.com"), - TokenType::Punctuation('-'), - ], - ), - ("a@foo-bar.com", vec![TokenType::Email("a@foo-bar.com")]), - ( - "a@-foo.com", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Punctuation('-'), - TokenType::UrlNoScheme("foo.com"), - ], - ), - ( - "a@b-.", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("b"), - TokenType::Punctuation('-'), - TokenType::Punctuation('.'), - ], - ), - ( - "a@b", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("b"), - ], - ), - ( - "a@b.", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("b"), - TokenType::Punctuation('.'), - ], - ), - ( - "a@example.com b@example.com", - vec![ - TokenType::Email("a@example.com"), - TokenType::Space, - TokenType::Email("b@example.com"), - ], - ), - ( - "a@example.com @ b@example.com", - vec![ - TokenType::Email("a@example.com"), - TokenType::Space, - TokenType::Punctuation('@'), - TokenType::Space, - TokenType::Email("b@example.com"), - ], - ), - ( - "a@xy.com;b@xy.com,c@xy.com", - vec![ - TokenType::Email("a@xy.com"), - TokenType::Punctuation(';'), - TokenType::Email("b@xy.com"), - TokenType::Punctuation(','), - TokenType::Email("c@xy.com"), - ], - ), - ( - "üñîçøðé@example.com", - vec![TokenType::Email("üñîçøðé@example.com")], - ), - ( - "üñîçøðé@üñîçøðé.com", - vec![TokenType::Email("üñîçøðé@üñîçøðé.com")], - ), - ("www@example.com", vec![TokenType::Email("www@example.com")]), - ( - "a@a.xyϸ", - vec![ - TokenType::Alphabetic("a"), - TokenType::Punctuation('@'), - TokenType::Alphabetic("a"), - TokenType::Punctuation('.'), - TokenType::Alphabetic("xyϸ"), - ], - ), - ( - "100 -100 100.00 -100.00 $100 $100.00", - vec![ - TokenType::Integer("100"), - TokenType::Space, - TokenType::Integer("-100"), - TokenType::Space, - TokenType::Float("100.00"), - TokenType::Space, - TokenType::Float("-100.00"), - TokenType::Space, - TokenType::Punctuation('$'), - TokenType::Integer("100"), - TokenType::Space, - TokenType::Punctuation('$'), - TokenType::Float("100.00"), - ], - ), - ( - " - 100 100 . 00", - vec![ - TokenType::Space, - TokenType::Punctuation('-'), - TokenType::Space, - TokenType::Integer("100"), - TokenType::Space, - TokenType::Integer("100"), - TokenType::Space, - TokenType::Punctuation('.'), - TokenType::Space, - TokenType::Integer("00"), - ], - ), - ( - "send $100.00 to user@domain.com or visit domain.com/pay-me!", - vec![ - TokenType::Alphabetic("send"), - TokenType::Space, - TokenType::Punctuation('$'), - TokenType::Float("100.00"), - TokenType::Space, - TokenType::Alphabetic("to"), - TokenType::Space, - TokenType::Email("user@domain.com"), - TokenType::Space, - TokenType::Alphabetic("or"), - TokenType::Space, - TokenType::Alphabetic("visit"), - TokenType::Space, - TokenType::UrlNoScheme("domain.com/pay-me"), - TokenType::Punctuation('!'), - ], - ), - ( - "vEⓡ𝔂 𝔽𝕌Ňℕy ţ乇𝕏𝓣 wWiIiIIttHh l133t5p3/-\\|<", - vec![ - TokenType::Alphabetic("vEⓡ𝔂"), - TokenType::Space, - TokenType::Alphabetic("𝔽𝕌Ňℕy"), - TokenType::Space, - TokenType::Alphabetic("ţ乇𝕏𝓣"), - TokenType::Space, - TokenType::Alphabetic("wWiIiIIttHh"), - TokenType::Space, - TokenType::Alphanumeric("l133t5p3"), - TokenType::Punctuation('/'), - TokenType::Punctuation('-'), - TokenType::Punctuation('\\'), - TokenType::Punctuation('|'), - TokenType::Punctuation('<'), - ], - ), - ] { - let result = TypesTokenizer::new(text) - .map(|t| t.word) - .collect::>(); - - assert_eq!(result, expected, "text: {:?}", text); - - /*print!("({text:?}, "); - print!("vec!["); - for (pos, item) in result.into_iter().enumerate() { - if pos > 0 { - print!(", "); - } - print!("TokenType::{:?}", item); - } - println!("]),");*/ - } - } -} diff --git a/crates/nlp/src/tokenizers/types/scan.rs b/crates/nlp/src/tokenizers/types/scan.rs new file mode 100644 index 00000000..f4a391a0 --- /dev/null +++ b/crates/nlp/src/tokenizers/types/scan.rs @@ -0,0 +1,218 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ASCII_ALNUM, ASCII_SPACE, StopChar, TokenType}; + +cfg_select! { + swar_alnum => { + pub(super) use dispatch::scan_word; + } + _ => { + pub(super) use sequential::scan_word; + } +} + +#[inline(always)] +fn scalar_alnum_run(bytes: &[u8], mut pos: usize) -> (usize, u8) { + let mut classes = 0; + while let Some(&byte) = bytes.get(pos) { + let class = ASCII_ALNUM[byte as usize]; + if class == 0 { + break; + } + classes |= class; + pos += 1; + } + (pos, classes) +} + +#[inline(always)] +fn ascii_stop<'x>(byte: u8) -> StopChar<'x> { + let is_space = ASCII_SPACE[byte as usize]; + ( + if is_space { + TokenType::Space + } else { + TokenType::Punctuation(byte as char) + }, + 1, + is_space, + ) +} + +#[inline(always)] +fn non_ascii_stop<'x>(ch: char) -> StopChar<'x> { + let is_space = ch.is_whitespace(); + ( + if is_space { + TokenType::Space + } else { + TokenType::Other(ch) + }, + ch.len_utf8(), + is_space, + ) +} + +#[cfg(any(all(test, stalwart_tests), not(swar_alnum)))] +mod sequential { + use super::{ + super::{CLASS_ALPHA, StopChar, char_at, is_alphabetic}, + ascii_stop, non_ascii_stop, scalar_alnum_run, + }; + + #[inline(always)] + pub(in super::super) fn scan_word( + text: &str, + mut pos: usize, + ) -> (usize, u8, Option>) { + let bytes = text.as_bytes(); + let mut classes = 0; + let stop = loop { + let (end, run_classes) = scalar_alnum_run(bytes, pos); + pos = end; + classes |= run_classes; + match bytes.get(pos) { + None => break None, + Some(&byte) if byte < 0x80 => break Some(ascii_stop(byte)), + Some(_) => match char_at(text, pos) { + Some(ch) if is_alphabetic(ch as u32) => { + classes |= CLASS_ALPHA; + pos += ch.len_utf8(); + } + Some(ch) => break Some(non_ascii_stop(ch)), + None => break None, + }, + } + }; + (pos, classes, stop) + } +} + +#[cfg(any(all(test, stalwart_tests), swar_alnum))] +mod dispatch { + use super::{ + super::{ASCII_ALNUM, CLASS_ALPHA, StopChar, is_alphabetic}, + ascii_stop, non_ascii_stop, swar, + }; + + #[inline(always)] + pub(in super::super) fn scan_word( + text: &str, + mut pos: usize, + ) -> (usize, u8, Option>) { + let bytes = text.as_bytes(); + let mut classes = 0; + let stop = loop { + match bytes.get(pos) { + None => break None, + Some(&byte) if ASCII_ALNUM[byte as usize] != 0 => { + let (end, run_classes) = swar::ascii_alnum_run(bytes, pos); + pos = end; + classes |= run_classes; + } + Some(&byte) if byte < 0x80 => break Some(ascii_stop(byte)), + Some(_) => { + let Some((code, len)) = decode_non_ascii(bytes, pos) else { + break None; + }; + if !is_alphabetic(code) { + break char::from_u32(code).map(non_ascii_stop); + } + classes |= CLASS_ALPHA; + pos += len; + if bytes.get(pos).is_some_and(|&byte| byte >= 0x80) { + pos = non_ascii_alphabetic_run(bytes, pos); + } + } + } + }; + (pos, classes, stop) + } + + #[inline(always)] + pub(super) fn decode_non_ascii(bytes: &[u8], pos: usize) -> Option<(u32, usize)> { + match *bytes.get(pos..)? { + [lead @ 0xC0..=0xDF, b1, ..] => { + Some(((u32::from(lead & 0x1F) << 6) | u32::from(b1 & 0x3F), 2)) + } + [lead @ 0xE0..=0xEF, b1, b2, ..] => Some(( + (u32::from(lead & 0x0F) << 12) | (u32::from(b1 & 0x3F) << 6) | u32::from(b2 & 0x3F), + 3, + )), + [lead @ 0xF0..=0xF7, b1, b2, b3, ..] => Some(( + (u32::from(lead & 0x07) << 18) + | (u32::from(b1 & 0x3F) << 12) + | (u32::from(b2 & 0x3F) << 6) + | u32::from(b3 & 0x3F), + 4, + )), + _ => None, + } + } + + #[inline(never)] + fn non_ascii_alphabetic_run(bytes: &[u8], mut pos: usize) -> usize { + while let Some((code, len)) = decode_non_ascii(bytes, pos) { + if !is_alphabetic(code) { + break; + } + pos += len; + } + pos + } +} + +#[cfg(any(all(test, stalwart_tests), swar_alnum))] +mod swar { + use super::{ + super::{CLASS_ALPHA, CLASS_DIGIT}, + scalar_alnum_run, + }; + + const LANES: u64 = 0x0101_0101_0101_0101; + const HIGH: u64 = LANES * 0x80; + const LOW: u64 = LANES * 0x7f; + const CASE: u64 = LANES * 0x20; + const WORD: usize = size_of::(); + + #[inline(always)] + fn in_range(low: u64, first: u8, last: u8) -> u64 { + (low + LANES * (0x80 - first as u64)) & !(low + LANES * (0x7f - last as u64)) + } + + #[inline(always)] + fn run_classes(digits: u64, alphas: u64) -> u8 { + (u8::from(digits != 0) * CLASS_DIGIT) | (u8::from(alphas != 0) * CLASS_ALPHA) + } + + #[inline(always)] + pub(super) fn ascii_alnum_run(bytes: &[u8], mut pos: usize) -> (usize, u8) { + let mut digits = 0; + let mut alphas = 0; + while let Some(chunk) = bytes.get(pos..).and_then(<[u8]>::first_chunk::) { + let word = u64::from_le_bytes(*chunk); + let low = word & LOW; + let ascii = !word & HIGH; + let digit = in_range(low, b'0', b'9') & ascii; + let alpha = in_range(low | CASE, b'a', b'z') & ascii; + let stop = !(digit | alpha) & HIGH; + let run = stop.wrapping_sub(1) & !stop; + digits |= digit & run; + alphas |= alpha & run; + if stop != 0 { + return ( + pos + (stop.trailing_zeros() / u8::BITS) as usize, + run_classes(digits, alphas), + ); + } + pos += WORD; + } + let (end, classes) = scalar_alnum_run(bytes, pos); + (end, classes | run_classes(digits, alphas)) + } +} + diff --git a/crates/nlp/src/tokenizers/unicode.rs b/crates/nlp/src/tokenizers/unicode.rs new file mode 100644 index 00000000..6a6a9ece --- /dev/null +++ b/crates/nlp/src/tokenizers/unicode.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +include!(concat!(env!("OUT_DIR"), "/unicode_bmp.rs")); + +#[inline(always)] +fn bmp_bit(table: &[u64], code: u32) -> Option { + table + .get(code as usize / 64) + .map(|word| (word >> (code % 64)) & 1 != 0) +} + +#[inline(always)] +pub(super) fn is_alphabetic(code: u32) -> bool { + bmp_bit(&ALPHABETIC_BMP, code) + .unwrap_or_else(|| char::from_u32(code).is_some_and(char::is_alphabetic)) +} + +#[inline(always)] +pub(super) fn is_alphanumeric(ch: char) -> bool { + bmp_bit(&ALPHANUMERIC_BMP, ch as u32).unwrap_or_else(|| ch.is_alphanumeric()) +} + +#[inline(always)] +pub(super) fn is_uppercase(ch: char) -> bool { + bmp_bit(&UPPERCASE_BMP, ch as u32).unwrap_or_else(|| ch.is_uppercase()) +} + diff --git a/crates/nlp/src/tokenizers/word.rs b/crates/nlp/src/tokenizers/word.rs index 18ef22b6..8721d09c 100644 --- a/crates/nlp/src/tokenizers/word.rs +++ b/crates/nlp/src/tokenizers/word.rs @@ -4,14 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, str::CharIndices}; +use std::borrow::Cow; -use super::Token; +use super::{ + Token, + unicode::{is_alphanumeric, is_uppercase}, +}; pub struct WordTokenizer<'x> { max_token_length: usize, text: &'x str, - iterator: CharIndices<'x>, + pos: usize, } impl WordTokenizer<'_> { @@ -19,132 +22,166 @@ impl WordTokenizer<'_> { WordTokenizer { max_token_length, text, - iterator: text.char_indices(), + pos: 0, } } } +pub(super) struct AlnumRun { + pub start: usize, + pub end: usize, + pub resume: usize, + pub has_upper: bool, + pub is_ascii: bool, +} + +const ASCII_CASE_BIT: u8 = 0x20; + +struct WideStop { + pos: usize, + len: usize, + has_upper: bool, +} + +#[inline(never)] +fn skip_wide(text: &str, from: usize) -> WideStop { + for (offset, ch) in text[from..].char_indices() { + if ch.is_ascii() { + return WideStop { + pos: from + offset, + len: 0, + has_upper: false, + }; + } + if is_alphanumeric(ch) { + return WideStop { + pos: from + offset, + len: ch.len_utf8(), + has_upper: is_uppercase(ch), + }; + } + } + WideStop { + pos: text.len(), + len: 0, + has_upper: false, + } +} + +#[inline(never)] +fn extend_wide(text: &str, from: usize) -> WideStop { + let mut has_upper = false; + for (offset, ch) in text[from..].char_indices() { + if ch.is_ascii() { + return WideStop { + pos: from + offset, + len: 0, + has_upper, + }; + } + if !is_alphanumeric(ch) { + return WideStop { + pos: from + offset, + len: ch.len_utf8(), + has_upper, + }; + } + has_upper = has_upper || is_uppercase(ch); + } + WideStop { + pos: text.len(), + len: 0, + has_upper, + } +} + +#[inline(always)] +pub(super) fn next_alnum_run(text: &str, from: usize) -> Option { + let bytes = text.as_bytes(); + let mut pos = from; + + let (start, first_len, first_upper) = 'find: loop { + match bytes.get(pos) { + Some(&byte) if byte < 0x80 => { + if byte.is_ascii_alphanumeric() { + break 'find (pos, 0, false); + } + pos += 1; + } + Some(_) => { + let hit = skip_wide(text, pos); + pos = hit.pos; + if hit.len != 0 { + break 'find (pos, hit.len, hit.has_upper); + } + if pos == bytes.len() { + return None; + } + } + None => return None, + } + }; + + let mut case_bits = u8::MAX; + let mut has_upper = first_upper; + let mut is_ascii = first_len == 0; + let mut end = start + first_len; + let resume = 'run: loop { + while let Some(&byte) = bytes.get(end) { + if byte >= 0x80 { + let hit = extend_wide(text, end); + if hit.pos != end { + has_upper |= hit.has_upper; + is_ascii = false; + end = hit.pos; + } + if hit.len != 0 { + break 'run end + hit.len; + } + continue 'run; + } + if !byte.is_ascii_alphanumeric() { + break 'run end + 1; + } + case_bits &= byte; + end += 1; + } + break 'run end; + }; + + Some(AlnumRun { + start, + end, + resume, + has_upper: has_upper || case_bits & ASCII_CASE_BIT == 0, + is_ascii, + }) +} + /// Parses indo-european text into lowercase tokens. impl<'x> Iterator for WordTokenizer<'x> { type Item = Token>; fn next(&mut self) -> Option { - while let Some((token_start, ch)) = self.iterator.next() { - if ch.is_alphanumeric() { - let mut is_uppercase = ch.is_uppercase(); - let token_end = (&mut self.iterator) - .filter_map(|(pos, ch)| { - if ch.is_alphanumeric() { - if !is_uppercase && ch.is_uppercase() { - is_uppercase = true; - } - None - } else { - pos.into() - } - }) - .next() - .unwrap_or(self.text.len()); + let text = self.text; + loop { + let run = next_alnum_run(text, self.pos)?; + self.pos = run.resume; - let token_len = token_end - token_start; - if token_end > token_start && token_len <= self.max_token_length { - return Token::new( - token_start, - token_len, - if is_uppercase { - self.text[token_start..token_end].to_lowercase().into() - } else { - self.text[token_start..token_end].into() - }, - ) - .into(); - } - } - } - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn indo_european_tokenizer() { - let inputs = [ - ( - "The quick brown fox jumps over the lazy dog", - vec![ - Token::new(0, 3, "the".into()), - Token::new(4, 5, "quick".into()), - Token::new(10, 5, "brown".into()), - Token::new(16, 3, "fox".into()), - Token::new(20, 5, "jumps".into()), - Token::new(26, 4, "over".into()), - Token::new(31, 3, "the".into()), - Token::new(35, 4, "lazy".into()), - Token::new(40, 3, "dog".into()), - ], - ), - ( - "Jovencillo EMPONZOÑADO de whisky: ¡qué figurota exhibe!", - vec![ - Token::new(0, 10, "jovencillo".into()), - Token::new(11, 12, "emponzoñado".into()), - Token::new(24, 2, "de".into()), - Token::new(27, 6, "whisky".into()), - Token::new(37, 4, "qué".into()), - Token::new(42, 8, "figurota".into()), - Token::new(51, 6, "exhibe".into()), - ], - ), - ( - "ZWÖLF Boxkämpfer jagten Victor quer über den großen Sylter Deich", - vec![ - Token::new(0, 6, "zwölf".into()), - Token::new(7, 11, "boxkämpfer".into()), - Token::new(19, 6, "jagten".into()), - Token::new(26, 6, "victor".into()), - Token::new(33, 4, "quer".into()), - Token::new(38, 5, "über".into()), - Token::new(44, 3, "den".into()), - Token::new(48, 7, "großen".into()), - Token::new(56, 6, "sylter".into()), - Token::new(63, 5, "deich".into()), - ], - ), - ( - "Съешь ещё этих мягких французских булок, да выпей же чаю", - vec![ - Token::new(0, 10, "съешь".into()), - Token::new(11, 6, "ещё".into()), - Token::new(18, 8, "этих".into()), - Token::new(27, 12, "мягких".into()), - Token::new(40, 22, "французских".into()), - Token::new(63, 10, "булок".into()), - Token::new(75, 4, "да".into()), - Token::new(80, 10, "выпей".into()), - Token::new(91, 4, "же".into()), - Token::new(96, 6, "чаю".into()), - ], - ), - ( - "Pijamalı hasta yağız şoföre çabucak güvendi", - vec![ - Token::new(0, 9, "pijamalı".into()), - Token::new(10, 5, "hasta".into()), - Token::new(16, 7, "yağız".into()), - Token::new(24, 8, "şoföre".into()), - Token::new(33, 8, "çabucak".into()), - Token::new(42, 8, "güvendi".into()), - ], - ), - ]; - - for (input, tokens) in inputs.iter() { - for (pos, token) in WordTokenizer::new(input, 40).enumerate() { - assert_eq!(token, tokens[pos]); + let token_len = run.end - run.start; + if token_len <= self.max_token_length { + let word = &text[run.start..run.end]; + return Some(Token::new( + run.start, + token_len, + if run.has_upper { + Cow::Owned(word.to_lowercase()) + } else { + Cow::Borrowed(word) + }, + )); } } } } + diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index cbdfe250..3d32a418 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pop3" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -13,12 +13,15 @@ trc = { path = "../trc" } types = { path = "../types" } email = { path = "../email" } registry = { path = "../registry" } -mail-parser = { version = "0.11", features = ["full_encoding"] } -tokio = { version = "1.53", features = ["full"] } +encodify = "1.0" +tokio = { version = "1.54", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } +compact_str = "0.10.0" +memchr = "2.8" +itoa = "1" [features] -test_mode = [] +enterprise = ["store/enterprise", "common/enterprise", "directory/enterprise", "imap/enterprise", "utils/enterprise", "trc/enterprise", "email/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/pop3/src/client.rs b/crates/pop3/src/client.rs index ab171bd4..247617f9 100644 --- a/crates/pop3/src/client.rs +++ b/crates/pop3/src/client.rs @@ -8,12 +8,9 @@ use crate::{ Session, State, protocol::{Command, Mechanism, request::Error}, }; -use common::{ - KV_RATE_LIMIT_IMAP, - network::{SessionResult, SessionStream}, -}; +use common::network::{SessionResult, SessionStream}; use directory::Credentials; -use trc::{AddContext, SecurityEvent}; +use trc::SecurityEvent; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { @@ -29,6 +26,21 @@ impl Session { loop { match self.receiver.parse(&mut bytes) { + Ok(_) | Err(Error::Parse(_)) + if matches!( + self.state, + State::NotAuthenticated { + sasl_failure: Some(_), + .. + } + ) => + { + if let Some(failure) = self.state.take_sasl_failure() + && !self.write_err(failure).await + { + return SessionResult::Close; + } + } Ok(request) => { // Group delete requests when possible match (request, requests.last_mut()) { @@ -100,11 +112,14 @@ impl Session { } else { unreachable!() }; - Box::pin(self.handle_auth(Credentials::Basic { - username, - secret: string, - mfa_token: None, - })) + self.authenticate( + Credentials::Basic { + username, + secret: string, + mfa_token: None, + }, + None, + ) .await .map(|_| SessionResult::Continue) } @@ -147,11 +162,10 @@ impl Session { self.handle_stls().await.map(|_| SessionResult::UpgradeTls) } Command::Utf8 => self.handle_utf8().await.map(|_| SessionResult::Continue), - Command::Auth { mechanism, params } => { - Box::pin(self.handle_sasl(mechanism, params)) - .await - .map(|_| SessionResult::Continue) - } + Command::Auth { mechanism, params } => self + .handle_sasl(mechanism, params) + .await + .map(|_| SessionResult::Continue), Command::Apop { .. } => Err(trc::Pop3Event::Error .into_err() .details("APOP not supported.")), @@ -181,10 +195,7 @@ impl Session { ) -> trc::Result> { match &command { Command::Capa | Command::Quit | Command::Noop => Ok(command), - Command::Auth { - mechanism: Mechanism::Plain, - .. - } + Command::Auth { .. } | Command::User { .. } | Command::Pass { .. } | Command::Apop { .. } => { @@ -208,15 +219,6 @@ impl Session { .details("Already authenticated.")) } } - Command::Auth { .. } => { - if let State::NotAuthenticated { .. } = &self.state { - Ok(command) - } else { - Err(trc::Pop3Event::Error - .into_err() - .details("Already authenticated.")) - } - } Command::Stls => { if !self.stream.is_tls() { Ok(command) @@ -236,28 +238,11 @@ impl Session { | Command::Utf8 | Command::Stat | Command::Rset => { - if let State::Authenticated { mailbox, .. } = &self.state { - if let Some(rate) = &self.server.core.imap.rate_requests { - if self - .server - .in_memory_store() - .is_rate_allowed( - KV_RATE_LIMIT_IMAP, - &mailbox.account_id.to_be_bytes(), - rate, - true, - ) - .await - .caused_by(trc::location!())? - .is_none() - { - Ok(command) - } else { - Err(trc::LimitEvent::TooManyRequests.into_err()) - } - } else { - Ok(command) - } + if let State::Authenticated { access_token, .. } = &self.state { + self.server + .is_imap_request_allowed(access_token) + .await + .map(|_| command) } else { Err(trc::Pop3Event::Error .into_err() diff --git a/crates/pop3/src/lib.rs b/crates/pop3/src/lib.rs index d800fd7e..6f42df22 100644 --- a/crates/pop3/src/lib.rs +++ b/crates/pop3/src/lib.rs @@ -50,6 +50,7 @@ pub enum State { NotAuthenticated { auth_failures: u32, username: Option, + sasl_failure: Option, }, Authenticated { mailbox: Mailbox, @@ -59,6 +60,13 @@ pub enum State { } impl State { + pub fn take_sasl_failure(&mut self) -> Option { + match self { + State::NotAuthenticated { sasl_failure, .. } => sasl_failure.take(), + _ => None, + } + } + pub fn mailbox(&self) -> &Mailbox { match self { State::Authenticated { mailbox, .. } => mailbox, diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 17473557..9ea6cd9f 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -10,7 +10,6 @@ use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::INBOX_ID, }; -use std::collections::BTreeMap; use trc::AddContext; use types::special_use::SpecialUse; @@ -39,7 +38,7 @@ impl Session { .await .caused_by(trc::location!())?; - if cache.emails.items.is_empty() { + if cache.emails.is_empty() { return Ok(Mailbox::default()); } @@ -49,37 +48,33 @@ impl Session { .unwrap_or_default(); // Sort by UID - let message_map = cache + let mut messages = cache .emails - .items .iter() .filter_map(|message| { message - .mailboxes + .mailboxes() .iter() .find(|m| m.mailbox_id == INBOX_ID) - .map(|m| (m.uid, (message.document_id, message.size))) + .map(|m| Message { + id: message.document_id(), + uid: m.uid, + size: message.size(), + deleted: false, + }) }) - .collect::>(); + .collect::>(); + messages.sort_unstable_by_key(|message| message.uid); + messages.dedup_by_key(|message| message.uid); - // Create mailbox - let mut mailbox = Mailbox { - messages: Vec::with_capacity(message_map.len()), + Ok(Mailbox { + total: messages.len() as u32, + size: messages + .iter() + .fold(0u32, |size, message| size.wrapping_add(message.size)), + messages, uid_validity, account_id, - ..Default::default() - }; - for (uid, (id, size)) in message_map { - mailbox.messages.push(Message { - id, - uid, - size, - deleted: false, - }); - mailbox.total += 1; - mailbox.size += size; - } - - Ok(mailbox) + }) } } diff --git a/crates/pop3/src/op/authenticate.rs b/crates/pop3/src/op/authenticate.rs index cf423e34..86790b1e 100644 --- a/crates/pop3/src/op/authenticate.rs +++ b/crates/pop3/src/op/authenticate.rs @@ -9,11 +9,15 @@ use crate::{ protocol::{Command, Mechanism, request}, }; use common::{ - auth::AuthRequest, + auth::{ + AuthRequest, + oauth::scope::OAuthScopes, + sasl::{BearerFailure, BearerMechanism, SASL_CANCEL, SASL_EMPTY}, + }, network::{SessionStream, limiter::LimiterResult}, }; use directory::Credentials; -use mail_parser::decoders::base64::base64_decode; +use encodify::base64::LENIENT; use registry::schema::enums::Permission; impl Session { @@ -22,75 +26,129 @@ impl Session { mechanism: Mechanism, mut params: Vec, ) -> trc::Result<()> { - match mechanism { - Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { - if !params.is_empty() { - let credentials = base64_decode(params.pop().unwrap().as_bytes()) - .and_then(|challenge| { - if mechanism == Mechanism::Plain { - Credentials::decode_sasl_challenge_plain(&challenge) - } else { - Credentials::decode_sasl_challenge_oauth(&challenge) + let bearer = match mechanism { + Mechanism::Plain => None, + Mechanism::OAuthBearer => Some(BearerMechanism::OAuthBearer), + Mechanism::XOauth2 => Some(BearerMechanism::XOAuth2), + _ => { + return Err(trc::AuthEvent::Error + .into_err() + .details("Authentication mechanism not supported.")); + } + }; + + if bearer.is_some() && !self.server.accepts_bearer_tokens(self.stream.is_tls()) { + return Err(trc::Pop3Event::Error + .into_err() + .details("OAUTHBEARER and XOAUTH2 require a TLS connection.")); + } + + match params.pop() { + Some(param) if param == SASL_CANCEL => Err(trc::Pop3Event::Error + .into_err() + .details("Authentication cancelled.")), + Some(param) if param != SASL_EMPTY => { + let credentials = LENIENT.decode(param.as_bytes()).ok().map(|challenge| { + if bearer.is_none() { + Credentials::decode_sasl_challenge_plain(&challenge) + } else { + Credentials::decode_sasl_challenge_oauth(&challenge) + } + }); + match (credentials, bearer) { + (Some(Some(credentials)), _) => self.authenticate(credentials, bearer).await, + (Some(None), Some(mechanism)) => { + let err = self.malformed_credentials().await; + match BearerFailure::from_error(&err) { + Some(failure) => { + self.bearer_challenge(mechanism, failure, None, err).await } - }) - .ok_or_else(|| { - trc::AuthEvent::Error - .into_err() - .details("Invalid SASL challenge") - })?; - - Box::pin(self.handle_auth(credentials)).await - } else { - // TODO: This hack is temporary until the SASL library is developed - self.receiver.state = request::State::Argument { - request: Command::Auth { - mechanism: mechanism.as_str().as_bytes().to_vec(), - params: vec![], - }, - num: 1, - last_is_space: true, - }; - - self.write_bytes("+\r\n").await + None => Err(err), + } + } + _ => Err(self.malformed_credentials().await), } } - _ => Err(trc::AuthEvent::Error - .into_err() - .details("Authentication mechanism not supported.")), + _ => { + self.expect_sasl_response(&mechanism); + self.write_bytes("+\r\n").await + } } } - pub async fn handle_auth(&mut self, credentials: Credentials) -> trc::Result<()> { - // Authenticate - let access_token = self + async fn bearer_challenge( + &mut self, + mechanism: BearerMechanism, + failure: BearerFailure, + credentials: Option<&Credentials>, + err: trc::Error, + ) -> trc::Result<()> { + let challenge = self .server - .authenticate(&AuthRequest::from_credentials( - credentials, - self.session_id, - self.remote_addr, - )) - .await - .map_err(|err| { - if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { - match &self.state { - State::NotAuthenticated { - auth_failures, - username, - } if *auth_failures < self.server.core.imap.max_auth_failures => { - self.state = State::NotAuthenticated { - auth_failures: auth_failures + 1, - username: username.clone(), - }; - } - _ => { - return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); + .bearer_failure_challenge(mechanism, failure, credentials) + .await; + if let State::NotAuthenticated { sasl_failure, .. } = &mut self.state { + *sasl_failure = Some(err); + } + self.expect_sasl_response(match mechanism { + BearerMechanism::OAuthBearer => &Mechanism::OAuthBearer, + BearerMechanism::XOAuth2 => &Mechanism::XOauth2, + }); + self.write_bytes(format!("+ {challenge}\r\n")).await + } + + fn expect_sasl_response(&mut self, mechanism: &Mechanism) { + self.receiver.state = request::State::Argument { + request: Command::Auth { + mechanism: mechanism.as_str().as_bytes().to_vec(), + params: vec![], + }, + num: 1, + last_is_space: true, + }; + } + + pub async fn authenticate( + &mut self, + credentials: Credentials, + bearer: Option, + ) -> trc::Result<()> { + // Authenticate + let request = AuthRequest::from_credentials(credentials, self.session_id, self.remote_addr); + let access_token = match self.server.authenticate(&request).await { + Ok(authenticated) => match bearer { + Some(mechanism) => { + match authenticated + .access_token + .assert_allows_oauth_scopes(OAuthScopes::MAIL) + { + Ok(access_token) => access_token, + Err(err) => { + return self + .bearer_challenge( + mechanism, + BearerFailure::InsufficientScope, + Some(&request.credentials), + BearerFailure::insufficient_scope(err), + ) + .await; } } } - - err - }) - .and_then(|token| token.assert_has_permission(Permission::Pop3Authenticate))?; + None => authenticated.access_token, + } + .assert_has_permission(Permission::Pop3Authenticate)?, + Err(err) => { + let err = self.register_auth_failure(err); + return match (bearer, BearerFailure::from_error(&err)) { + (Some(mechanism), Some(failure)) => { + self.bearer_challenge(mechanism, failure, Some(&request.credentials), err) + .await + } + _ => Err(err), + }; + } + }; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { @@ -112,4 +170,35 @@ impl Session { }; self.write_ok("Authentication successful").await } + + async fn malformed_credentials(&mut self) -> trc::Error { + let err = self + .server + .auth_failure( + trc::AuthEvent::Failed + .into_err() + .details("Invalid SASL response."), + self.remote_addr, + None, + ) + .await; + self.register_auth_failure(err) + } + + fn register_auth_failure(&mut self, err: trc::Error) -> trc::Error { + if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { + match &mut self.state { + State::NotAuthenticated { auth_failures, .. } + if *auth_failures < self.server.core.imap.max_auth_failures => + { + *auth_failures += 1; + } + _ => { + return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); + } + } + } + + err + } } diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index 20ddc83a..3d3762d8 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -91,7 +91,7 @@ impl Session { .server .emails_delete( mailbox.account_id, - self.state.access_token().tenant_id(), + self.state.access_token().changed_by(), &mut batch, deleted, ) @@ -103,7 +103,6 @@ impl Session { .commit_batch(batch) .await .caused_by(trc::location!())?; - self.server.notify_task_queue(); } if not_deleted.is_empty() { self.write_ok(format!( diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index f1f576db..30c5762c 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -4,19 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Session, protocol::response::Response}; +use crate::{ + Session, + protocol::response::{MessageWriter, Response}, +}; use common::network::SessionStream; -use email::message::metadata::MessageMetadata; +use email::message::metadata::MetadataRow; use registry::schema::enums::Permission; use std::time::Instant; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; +use store::ValueKey; use trc::AddContext; use types::{collection::Collection, field::EmailField}; use utils::chained_bytes::ChainedBytes; +const MESSAGE_CHUNK_LEN: usize = 64 * 1024; + impl Session { pub async fn handle_fetch(&mut self, msg: u32, lines: Option) -> trc::Result<()> { // Validate access @@ -27,10 +29,10 @@ impl Session { let op_start = Instant::now(); let mailbox = self.state.mailbox(); if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { - if let Some(metadata_) = self + if let Some(row) = self .server .store() - .get_value::>(ValueKey::property( + .get_value::(ValueKey::immutable( mailbox.account_id, Collection::Email, message.id, @@ -39,13 +41,15 @@ impl Session { .await .caused_by(trc::location!())? { - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; + let metadata = row.unarchive().caused_by(trc::location!())?; + let headers = row.raw_headers().caused_by(trc::location!())?; if let Some(bytes) = self .server - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) + .fetch_blob( + &metadata.blob_hash(), + 0..usize::MAX, + Some(metadata.size() as u64), + ) .await .caused_by(trc::location!())? { @@ -56,16 +60,23 @@ impl Session { Elapsed = op_start.elapsed() ); - let bytes = ChainedBytes::new(metadata.raw_headers.as_ref()) - .with_last( - bytes - .get(metadata.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ) - .get_full_range(); - - self.write_bytes(Response::Message:: { bytes, lines }.serialize()) - .await + match Response::::message( + ChainedBytes::from_blob(&headers, &bytes, metadata.blob_body_offset()), + metadata.headers_len(), + lines, + ) { + Response::Message(bytes, end) => { + let mut buf = Vec::new(); + let mut writer = + MessageWriter::new(bytes, end, &mut buf, MESSAGE_CHUNK_LEN); + while !writer.fill(&mut buf) { + self.write_bytes(&buf).await?; + buf.clear(); + } + self.write_bytes(&buf).await + } + response => self.write_bytes(response.serialize()).await, + } } else { Err(trc::Pop3Event::Error .into_err() diff --git a/crates/pop3/src/op/list.rs b/crates/pop3/src/op/list.rs index 07232172..61545698 100644 --- a/crates/pop3/src/op/list.rs +++ b/crates/pop3/src/op/list.rs @@ -9,7 +9,10 @@ use std::time::Instant; use common::network::SessionStream; use registry::schema::enums::Permission; -use crate::{Session, protocol::response::Response}; +use crate::{ + Session, + protocol::response::{Response, UniqueId}, +}; impl Session { pub async fn handle_list(&mut self, msg: Option) -> trc::Result<()> { @@ -93,7 +96,10 @@ impl Session { mailbox .messages .iter() - .map(|m| format!("{}{}", mailbox.uid_validity, m.uid)) + .map(|m| UniqueId { + uid_validity: mailbox.uid_validity, + uid: m.uid, + }) .collect::>(), ) .serialize(), diff --git a/crates/pop3/src/op/mod.rs b/crates/pop3/src/op/mod.rs index 6086f169..754197c7 100644 --- a/crates/pop3/src/op/mod.rs +++ b/crates/pop3/src/op/mod.rs @@ -18,10 +18,13 @@ pub mod list; impl Session { pub async fn handle_capa(&mut self) -> trc::Result<()> { - let mechanisms = if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { + let is_tls = self.stream.is_tls(); + let mechanisms = if !is_tls && !self.server.core.imap.allow_plain_auth { + vec![] + } else if self.server.accepts_bearer_tokens(is_tls) { vec![Mechanism::Plain, Mechanism::OAuthBearer, Mechanism::XOauth2] } else { - vec![Mechanism::OAuthBearer, Mechanism::XOauth2] + vec![Mechanism::Plain] }; trc::event!( diff --git a/crates/pop3/src/protocol/dot_stuffer.rs b/crates/pop3/src/protocol/dot_stuffer.rs new file mode 100644 index 00000000..9d67a7b5 --- /dev/null +++ b/crates/pop3/src/protocol/dot_stuffer.rs @@ -0,0 +1,134 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::response::MessageEnd; +use memchr::memchr_iter; +use utils::chained_bytes::ChainedBytes; + +const CRLF: &[u8] = b"\r\n"; + +const DENSE_LINE_LEN: usize = 8; +const DENSE_STREAK: u32 = 8; +const SPARSE_LINE_LEN: usize = 64; + +pub struct DotStuffer { + last: u8, +} + +impl Default for DotStuffer { + fn default() -> Self { + Self { last: b'\n' } + } +} + +impl DotStuffer { + pub fn push(&mut self, buf: &mut Vec, bytes: &[u8]) { + let mut offset = 0; + while let Some(rest) = bytes.get(offset..).filter(|rest| !rest.is_empty()) { + offset += self.copy_sparse(buf, rest); + offset += self.copy_dense(buf, bytes.get(offset..).unwrap_or_default()); + } + } + + pub fn finish(self, buf: &mut Vec, end: MessageEnd) { + if self.last != b'\n' { + buf.extend_from_slice(b"\r\n"); + } + if end == MessageEnd::BlankLine { + buf.extend_from_slice(b"\r\n"); + } + buf.extend_from_slice(b".\r\n"); + } + + pub fn wire_len(bytes: &ChainedBytes<'_>) -> usize { + let mut last = b'\n'; + let mut bare_lf = 0; + for segment in bytes.segments() { + for pos in memchr_iter(b'\n', segment) { + let prev = pos + .checked_sub(1) + .and_then(|prev| segment.get(prev)) + .copied() + .unwrap_or(last); + if prev != b'\r' { + bare_lf += 1; + } + } + if let Some(&byte) = segment.last() { + last = byte; + } + } + let final_crlf = if last != b'\n' { CRLF.len() } else { 0 }; + bytes.len() + bare_lf + final_crlf + } + + fn copy_sparse(&mut self, buf: &mut Vec, bytes: &[u8]) -> usize { + if self.last == b'\n' && bytes.first() == Some(&b'.') { + buf.push(b'.'); + } + let mut run_start = 0; + let mut short_lines = 0; + for pos in memchr_iter(b'\n', bytes) { + let prev = pos + .checked_sub(1) + .and_then(|prev| bytes.get(prev)) + .copied() + .unwrap_or(self.last); + let is_bare = prev != b'\r'; + let is_dot = bytes.get(pos + 1) == Some(&b'.'); + if is_bare || is_dot { + buf.extend_from_slice(bytes.get(run_start..pos).unwrap_or_default()); + if pos - run_start < DENSE_LINE_LEN { + short_lines += 1; + if short_lines == DENSE_STREAK { + self.last = prev; + return pos; + } + } else { + short_lines = 0; + } + if is_bare { + buf.push(b'\r'); + } + buf.push(b'\n'); + if is_dot { + buf.push(b'.'); + } + run_start = pos + 1; + } + } + buf.extend_from_slice(bytes.get(run_start..).unwrap_or_default()); + if let Some(&last) = bytes.last() { + self.last = last; + } + bytes.len() + } + + fn copy_dense(&mut self, buf: &mut Vec, bytes: &[u8]) -> usize { + let mut line_start = 0; + for (pos, &byte) in bytes.iter().enumerate() { + if byte == b'\n' { + if self.last != b'\r' { + buf.push(b'\r'); + } + buf.push(b'\n'); + self.last = b'\n'; + if pos - line_start >= SPARSE_LINE_LEN { + return pos + 1; + } + line_start = pos + 1; + } else { + if byte == b'.' && self.last == b'\n' { + buf.push(b'.'); + } + buf.push(byte); + self.last = byte; + } + } + bytes.len() + } +} + diff --git a/crates/pop3/src/protocol/mod.rs b/crates/pop3/src/protocol/mod.rs index 07d207c4..b409ea81 100644 --- a/crates/pop3/src/protocol/mod.rs +++ b/crates/pop3/src/protocol/mod.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod dot_stuffer; pub mod request; pub mod response; diff --git a/crates/pop3/src/protocol/request.rs b/crates/pop3/src/protocol/request.rs index bc440fa3..41ab7ab1 100644 --- a/crates/pop3/src/protocol/request.rs +++ b/crates/pop3/src/protocol/request.rs @@ -321,135 +321,3 @@ impl Mechanism { } } -#[cfg(test)] -mod tests { - use crate::protocol::{Command, Mechanism, request::Error}; - - use super::Parser; - - #[test] - fn parse_command() { - let mut parser = Parser::default(); - let mut chunked = String::new(); - let mut chunked_expected = Vec::new(); - - for (cmd, request) in [ - ("QuiT", Command::Quit), - (" \r\n NOOP ", Command::Noop), - ("STAT ", Command::Stat), - ("LIST ", Command::List { msg: None }), - (" list 100 ", Command::List { msg: 100.into() }), - ("retr 55", Command::Retr { msg: 55 }), - ("DELE 99", Command::Dele { msg: 99 }), - (" rset ", Command::Rset), - ("top 8000 1234", Command::Top { msg: 8000, n: 1234 }), - ("uidl", Command::Uidl { msg: None }), - ("uidl 000099999", Command::Uidl { msg: 99999.into() }), - ( - "USER test", - Command::User { - name: "test".to_string(), - }, - ), - ( - "PASS secret", - Command::Pass { - string: "secret".to_string(), - }, - ), - ( - "APOP mrose c4c9334bac560ecc979e58001b3e22fb", - Command::Apop { - name: "mrose".to_string(), - digest: "c4c9334bac560ecc979e58001b3e22fb".to_string(), - }, - ), - ("utf8", Command::Utf8), - ("capa", Command::Capa), - ( - "AUTH GSSAPI", - Command::Auth { - mechanism: Mechanism::Gssapi, - params: vec![], - }, - ), - ( - "AUTH PLAIN dGVzdAB0ZXN0AHRlc3Q=", - Command::Auth { - mechanism: Mechanism::Plain, - params: vec!["dGVzdAB0ZXN0AHRlc3Q=".to_string()], - }, - ), - ] { - assert_eq!( - parser.parse(&mut cmd.as_bytes().iter()), - Err(Error::NeedsMoreData) - ); - assert_eq!( - parser.parse(&mut b"\r\n".iter()), - Ok(request.clone()), - "{:?}", - cmd - ); - chunked.push_str(cmd); - chunked.push_str("\r\n"); - chunked_expected.push(request); - } - - for chunk_size in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] { - let mut parser = Parser::default(); - let mut requests = Vec::new(); - - for chunk in chunked.as_bytes().chunks(chunk_size) { - let mut chunk = chunk.iter(); - loop { - match parser.parse(&mut chunk) { - Ok(request) => { - requests.push(request); - } - Err(Error::NeedsMoreData) => break, - Err(err) => { - panic!("Unexpected error on chunk size {chunk_size}: {err:?}"); - } - } - } - } - - assert_eq!(requests, chunked_expected, "Chunk size: {}", chunk_size); - } - - for cmd in [ - "user", - "pass", - "user a b", - "pass c d", - "apop", - "apop a", - "apop a b c", - "quit 1", - "stat 1", - "list 1 2", - "retr", - "retr 1 2", - "dele", - "dele 1 2", - "noop 1", - "rset 1", - "top", - "top 1 2 3", - "uidl 1 2 3", - "capa 1", - "stls 1", - "utf8 1", - "auth", - "auth unknown", - ] { - assert_eq!( - parser.parse(&mut cmd.as_bytes().iter()), - Err(Error::NeedsMoreData) - ); - let result = parser.parse(&mut b"\r\n".iter()); - assert!(result.is_err(), "{:?}", result); - } - } -} diff --git a/crates/pop3/src/protocol/response.rs b/crates/pop3/src/protocol/response.rs index b8731885..83326929 100644 --- a/crates/pop3/src/protocol/response.rs +++ b/crates/pop3/src/protocol/response.rs @@ -4,25 +4,169 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::Mechanism; -use std::{borrow::Cow, fmt::Display}; -use utils::chained_bytes::SliceRange; +use super::{Mechanism, dot_stuffer::DotStuffer}; +use std::borrow::Cow; +use utils::chained_bytes::ChainedBytes; + +const STATUS_LINE_LEN: usize = 32; +const STUFFING_RESERVE_DIVISOR: usize = 64; +const BLANK_LINE_TAIL_LEN: usize = 3; +const CRLF_LEN: usize = 2; +const LIST_NUMBER_LEN: usize = 6; +const ERR_PREFIX: &[u8] = b"-ERR "; +const SYS_TEMP_CODE: &[u8] = b"[SYS/TEMP] "; + +pub trait ListValue { + const LEN_HINT: usize; + + fn write_value(&self, buf: &mut Vec); +} + +impl ListValue for u32 { + const LEN_HINT: usize = 8; + + fn write_value(&self, buf: &mut Vec) { + buf.extend_from_slice(itoa::Buffer::new().format(*self).as_bytes()); + } +} + +pub struct UniqueId { + pub uid_validity: u32, + pub uid: u32, +} + +impl ListValue for UniqueId { + const LEN_HINT: usize = 20; + + fn write_value(&self, buf: &mut Vec) { + self.uid_validity.write_value(buf); + self.uid.write_value(buf); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MessageEnd { + AsIs, + BlankLine, +} pub enum Response<'x, T> { Ok(Cow<'static, str>), Err(Cow<'static, str>), List(Vec), - Message { - bytes: SliceRange<'x>, - lines: Option, - }, + Message(ChainedBytes<'x>, MessageEnd), Capability { mechanisms: Vec, stls: bool, }, } -impl<'x, T: Display> Response<'x, T> { +impl<'x, T> Response<'x, T> { + pub fn message(raw_message: ChainedBytes<'x>, body_offset: usize, lines: Option) -> Self { + let Some(lines) = lines else { + return Response::Message(raw_message, MessageEnd::AsIs); + }; + let len = raw_message.len(); + let body_offset = body_offset.min(len); + let end = if body_offset == len && !ends_with_blank_line(&raw_message) { + MessageEnd::BlankLine + } else { + MessageEnd::AsIs + }; + let body_lines = raw_message + .view(body_offset..len) + .unwrap_or_default() + .line_prefix_len(lines as usize); + Response::Message( + raw_message + .view(0..body_offset + body_lines) + .unwrap_or(raw_message), + end, + ) + } +} + +pub struct MessageWriter<'x> { + current: &'x [u8], + next: &'x [u8], + stuffer: Option, + end: MessageEnd, + chunk_len: usize, +} + +impl<'x> MessageWriter<'x> { + pub fn new( + bytes: ChainedBytes<'x>, + end: MessageEnd, + buf: &mut Vec, + chunk_len: usize, + ) -> Self { + let mut octets = DotStuffer::wire_len(&bytes); + if end == MessageEnd::BlankLine { + octets += CRLF_LEN; + } + buf.reserve( + octets.min(chunk_len) + + bytes.len().min(chunk_len) / STUFFING_RESERVE_DIVISOR + + STATUS_LINE_LEN, + ); + buf.extend_from_slice(b"+OK "); + buf.extend_from_slice(itoa::Buffer::new().format(octets).as_bytes()); + buf.extend_from_slice(b" octets\r\n"); + let [current, next] = bytes.segments(); + MessageWriter { + current, + next, + stuffer: Some(DotStuffer::default()), + end, + chunk_len, + } + } + + pub fn fill(&mut self, buf: &mut Vec) -> bool { + loop { + let Some(stuffer) = &mut self.stuffer else { + return true; + }; + if buf.len() >= self.chunk_len { + return false; + } + if self.current.is_empty() { + if self.next.is_empty() { + if let Some(stuffer) = self.stuffer.take() { + stuffer.finish(buf, self.end); + } + return true; + } + self.current = std::mem::take(&mut self.next); + } + let take = self + .current + .len() + .min(self.chunk_len.saturating_sub(buf.len())); + let (slice, rest) = self + .current + .split_at_checked(take) + .unwrap_or((self.current, &[])); + stuffer.push(buf, slice); + self.current = rest; + } + } +} + +fn ends_with_blank_line(bytes: &ChainedBytes<'_>) -> bool { + let len = bytes.len(); + bytes + .get(len.saturating_sub(BLANK_LINE_TAIL_LEN)..len) + .is_some_and(|tail| { + matches!( + tail.as_ref(), + [b'\n'] | [b'\r', b'\n'] | [.., b'\n', b'\n'] | [.., b'\n', b'\r', b'\n'] + ) + }) +} + +impl Response<'_, T> { pub fn serialize(&self) -> Vec { match self { Response::Ok(message) => { @@ -32,85 +176,27 @@ impl<'x, T: Display> Response<'x, T> { buf.extend_from_slice(b"\r\n"); buf } - Response::Err(message) => { - let mut buf = Vec::with_capacity(message.len() + 6); - buf.extend_from_slice(b"-ERR "); - buf.extend_from_slice(message.as_bytes()); - buf.extend_from_slice(b"\r\n"); - buf - } - Response::List(octets) => { - let mut buf = Vec::with_capacity(octets.len() * 8 + 10); - buf.extend_from_slice(format!("+OK {} messages\r\n", octets.len()).as_bytes()); - for (num, octet) in octets.iter().enumerate() { - buf.extend_from_slice((num + 1).to_string().as_bytes()); - buf.extend_from_slice(b" "); - buf.extend_from_slice(octet.to_string().as_bytes()); + Response::Err(message) => serialize_err(b"", message), + Response::List(values) => { + let mut buf = Vec::with_capacity( + values.len() * (LIST_NUMBER_LEN + T::LEN_HINT + CRLF_LEN) + STATUS_LINE_LEN, + ); + let mut number = itoa::Buffer::new(); + buf.extend_from_slice(b"+OK "); + buf.extend_from_slice(number.format(values.len()).as_bytes()); + buf.extend_from_slice(b" messages\r\n"); + for (num, value) in values.iter().enumerate() { + buf.extend_from_slice(number.format(num + 1).as_bytes()); + buf.push(b' '); + value.write_value(&mut buf); buf.extend_from_slice(b"\r\n"); } buf.extend_from_slice(b".\r\n"); buf } - Response::Message { bytes, lines } => { - let lines = *lines; - let mut message = Vec::with_capacity(bytes.len() + 16); - let mut octets = 0; - let mut last_byte = b'\n'; - let mut in_headers = lines.is_some(); - let mut is_blank_line = true; - let mut body_lines = 0; - - // Transparency procedure - for &byte in bytes.into_iter() { - // POP3 requires that lines end with CRLF, do this check to ensure that - if byte == b'\n' && last_byte != b'\r' { - message.push(b'\r'); - octets += 1; - } - - if byte == b'.' && last_byte == b'\n' { - message.push(b'.'); - } - message.push(byte); - octets += 1; - last_byte = byte; - - match byte { - b'\n' => { - if in_headers { - in_headers = !is_blank_line; - } else { - body_lines += 1; - } - if !in_headers && lines.is_some_and(|lines| body_lines >= lines) { - break; - } - is_blank_line = true; - } - b'\r' => {} - _ => { - is_blank_line = false; - } - } - } - - if last_byte != b'\n' { - message.extend_from_slice(b"\r\n"); - octets += 2; - } - - if in_headers { - message.extend_from_slice(b"\r\n"); - octets += 2; - } - - message.extend_from_slice(b".\r\n"); - - let mut buf = Vec::with_capacity(message.len() + 24); - buf.extend_from_slice(b"+OK "); - buf.extend_from_slice(octets.to_string().as_bytes()); - buf.extend_from_slice(b" octets\r\n"); - buf.extend_from_slice(&message); + Response::Message(bytes, end) => { + let mut buf = Vec::new(); + MessageWriter::new(*bytes, *end, &mut buf, usize::MAX).fill(&mut buf); buf } Response::Capability { mechanisms, stls } => { @@ -177,108 +263,28 @@ pub trait SerializeResponse { impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { - let message = self - .value_as_str(trc::Key::Details) - .unwrap_or_else(|| self.as_ref().message()); - let mut buf = Vec::with_capacity(message.len() + 6); - buf.extend_from_slice(b"-ERR "); - buf.extend_from_slice(message.as_bytes()); - buf.extend_from_slice(b"\r\n"); - buf + let code: &[u8] = match self.as_ref() { + trc::EventType::Limit( + trc::LimitEvent::TooManyRequests + | trc::LimitEvent::ConcurrentRequest + | trc::LimitEvent::MemoryBudget, + ) => SYS_TEMP_CODE, + _ => b"", + }; + serialize_err( + code, + self.value_as_str(trc::Key::Details) + .unwrap_or_else(|| self.as_ref().message()), + ) } } -#[cfg(test)] -mod tests { - use super::Response; - use crate::protocol::Mechanism; - use utils::chained_bytes::SliceRange; - - #[test] - fn serialize_response() { - for (cmd, expected) in [ - ( - Response::Ok("message 1 deleted".into()), - "+OK message 1 deleted\r\n", - ), - ( - Response::Err("permission denied".into()), - "-ERR permission denied\r\n", - ), - ( - Response::List(vec![100, 200, 300]), - "+OK 3 messages\r\n1 100\r\n2 200\r\n3 300\r\n.\r\n", - ), - ( - Response::Capability { - mechanisms: vec![Mechanism::Plain, Mechanism::CramMd5], - stls: true, - }, - concat!( - "+OK Capability list follows\r\n", - "USER\r\n", - "SASL PLAIN CRAM-MD5\r\n", - "STLS\r\n", - "TOP\r\n", - "RESP-CODES\r\n", - "PIPELINING\r\n", - "EXPIRE NEVER\r\n", - "UIDL\r\n", - "UTF8\r\n", - "IMPLEMENTATION Stalwart Server\r\n.\r\n" - ), - ), - ( - Response::Message { - bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), - lines: None, - }, - "+OK 37 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), - lines: Some(0), - }, - "+OK 17 octets\r\nSubject: test\r\n\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), - lines: Some(2), - }, - "+OK 27 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), - lines: Some(100), - }, - "+OK 37 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Single(b"Subject: test\n\nbody\n"), - lines: None, - }, - "+OK 23 octets\r\nSubject: test\r\n\r\nbody\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Single(b"Subject: test\n\n.leading dot\n"), - lines: Some(1), - }, - "+OK 31 octets\r\nSubject: test\r\n\r\n..leading dot\r\n.\r\n", - ), - ( - Response::Message { - bytes: SliceRange::Single(b".dot\r\nSubject: test\r\n"), - lines: Some(3), - }, - "+OK 23 octets\r\n..dot\r\nSubject: test\r\n\r\n.\r\n", - ), - ] { - assert_eq!(expected, String::from_utf8(cmd.serialize()).unwrap()); - } - } +fn serialize_err(code: &[u8], message: &str) -> Vec { + let mut buf = Vec::with_capacity(ERR_PREFIX.len() + code.len() + message.len() + CRLF_LEN); + buf.extend_from_slice(ERR_PREFIX); + buf.extend_from_slice(code); + buf.extend_from_slice(message.as_bytes()); + buf.extend_from_slice(b"\r\n"); + buf } + diff --git a/crates/pop3/src/session.rs b/crates/pop3/src/session.rs index 63c8eb0f..9c98d7b6 100644 --- a/crates/pop3/src/session.rs +++ b/crates/pop3/src/session.rs @@ -15,6 +15,7 @@ use common::{ BuildServer, network::{SessionData, SessionManager, SessionResult, SessionStream}, }; +use compact_str::ToCompactString; use std::borrow::Cow; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::server::TlsStream; @@ -33,6 +34,7 @@ impl SessionManager for Pop3SessionManager { state: State::NotAuthenticated { auth_failures: 0, username: None, + sasl_failure: None, }, stream: session.stream, in_flight: session.in_flight, @@ -98,7 +100,7 @@ impl Session { trc::event!( Network(trc::NetworkEvent::ReadError), SpanId = self.session_id, - Reason = err.to_string() , + Reason = err.to_compact_string() , CausedBy = trc::location!() ); break; @@ -132,7 +134,8 @@ impl Session { false } - pub async fn into_tls(self) -> Result>, ()> { + pub async fn into_tls(mut self) -> Result>, ()> { + self.state.take_sasl_failure(); Ok(Session { stream: self .instance diff --git a/crates/registry/Cargo.toml b/crates/registry/Cargo.toml index a34fe4d8..ccdcb560 100644 --- a/crates/registry/Cargo.toml +++ b/crates/registry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "registry" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -8,16 +8,17 @@ utils = { path = "../utils" } trc = { path = "../trc" } types = { path = "../types" } serde = { version = "1.0", features = ["derive"]} -hashify = "0.2.9" +hashify = "0.3" ahash = { version = "0.8" } -jmap-tools = { version = "0.1" } -mail-auth = { version = "0.13" } -tokio = { version = "1.53", features = ["fs"] } +jmap-tools = { path = "../jmap-tools" } +mail-auth = { version = "0.14" } +smtp-report = { path = "../smtp-report" } +tokio = { version = "1.54", features = ["fs"] } lz4_flex = { version = "0.14", features = ["alloc"], default-features = false } +compact_str = "0.10.0" [features] -test_mode = [] -enterprise = [] +enterprise = ["trc/enterprise", "utils/enterprise"] [lints] workspace = true diff --git a/crates/registry/src/jmap/mod.rs b/crates/registry/src/jmap/mod.rs index a1d31cbb..14d6cc2b 100644 --- a/crates/registry/src/jmap/mod.rs +++ b/crates/registry/src/jmap/mod.rs @@ -45,7 +45,6 @@ pub struct JsonPointerPatch<'x> { pos: usize, validators: &'x [StringValidator], is_create: bool, - can_set_tenant: bool, can_set_account: bool, } @@ -66,6 +65,18 @@ pub trait RegistryJsonEnumPatch: Debug { -> PatchResult<'x>; } +#[derive(Clone, Copy, Default, Debug)] +pub struct ValueOptions { + pub include_secrets: bool, +} + pub trait IntoValue { fn into_value(self) -> JmapValue<'static>; + + fn into_value_with(self, _opts: ValueOptions) -> JmapValue<'static> + where + Self: Sized, + { + self.into_value() + } } diff --git a/crates/registry/src/jmap/patch.rs b/crates/registry/src/jmap/patch.rs index b3c92399..48b6911a 100644 --- a/crates/registry/src/jmap/patch.rs +++ b/crates/registry/src/jmap/patch.rs @@ -27,7 +27,6 @@ impl<'x> JsonPointerPatch<'x> { pos: 0, validators: &[], is_create: false, - can_set_tenant: false, can_set_account: false, } } @@ -38,7 +37,6 @@ impl<'x> JsonPointerPatch<'x> { pos: 0, validators: &[], is_create: self.is_create, - can_set_tenant: self.can_set_tenant, can_set_account: self.can_set_account, } } @@ -49,7 +47,6 @@ impl<'x> JsonPointerPatch<'x> { pos: 0, validators: &[], is_create: self.is_create, - can_set_tenant: self.can_set_tenant, can_set_account: self.can_set_account, } } @@ -59,10 +56,6 @@ impl<'x> JsonPointerPatch<'x> { self } - pub fn with_can_set_tenant(mut self, can_set_tenant: bool) -> Self { - self.can_set_tenant = can_set_tenant; - self - } pub fn with_can_set_account(mut self, can_set_account: bool) -> Self { self.can_set_account = can_set_account; @@ -127,16 +120,6 @@ impl<'x> JsonPointerPatch<'x> { )) } - pub fn assert_can_set_tenant(self) -> Result { - if self.can_set_tenant { - Ok(self) - } else { - Err(PatchError::new( - self.cloned(), - "Cannot modify memberTenantId property", - )) - } - } pub fn assert_can_set_account(self) -> Result { if self.can_set_account { diff --git a/crates/registry/src/jmap/properties.rs b/crates/registry/src/jmap/properties.rs index df900a46..afcb9c31 100644 --- a/crates/registry/src/jmap/properties.rs +++ b/crates/registry/src/jmap/properties.rs @@ -6,8 +6,9 @@ use crate::{jmap::RegistryValue, schema::prelude::Property, types::EnumImpl}; use jmap_tools::Key; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{blob::BlobId, id::Id, text::Text}; impl jmap_tools::Property for Property { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { @@ -17,6 +18,10 @@ impl jmap_tools::Property for Property { fn to_cow(&self) -> Cow<'static, str> { self.as_str().into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl FromStr for Property { @@ -35,7 +40,6 @@ impl jmap_tools::Element for RegistryValue { match prop { Property::Id | Property::MemberGroupIds - | Property::MemberTenantId | Property::RoleIds | Property::DnsServerId | Property::DirectoryId @@ -45,12 +49,10 @@ impl jmap_tools::Element for RegistryValue { | Property::DefaultCertificateId | Property::DefaultUserRoleIds | Property::DefaultGroupRoleIds - | Property::DefaultTenantRoleIds | Property::DefaultAdminRoleIds | Property::ListenerIds | Property::PublicKey | Property::QueueId - | Property::ModelId | Property::AcmeProviderId => { if let Some(reference) = value.strip_prefix('#') { Some(RegistryValue::IdReference(reference.to_string())) @@ -73,10 +75,20 @@ impl jmap_tools::Element for RegistryValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl RegistryValue { + pub fn text(&self) -> Text<'_> { match self { - RegistryValue::Id(id) => id.to_string().into(), - RegistryValue::BlobId(blob_id) => blob_id.to_string().into(), - RegistryValue::IdReference(r) => format!("#{r}").into(), + RegistryValue::Id(id) => Text::Id(*id), + RegistryValue::BlobId(blob_id) => Text::Display(blob_id), + RegistryValue::IdReference(r) => Text::Reference(r), } } } diff --git a/crates/registry/src/jmap/ser.rs b/crates/registry/src/jmap/ser.rs index 3cdb64d0..40023324 100644 --- a/crates/registry/src/jmap/ser.rs +++ b/crates/registry/src/jmap/ser.rs @@ -5,7 +5,7 @@ */ use crate::{ - jmap::{IntoValue, JmapValue}, + jmap::{IntoValue, JmapValue, ValueOptions}, schema::prelude::Property, types::EnumImpl, }; @@ -15,8 +15,12 @@ use utils::map::vec_map::VecMap; impl IntoValue for Option { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { - Some(value) => value.into_value(), + Some(value) => value.into_value_with(opts), None => JmapValue::Null, } } @@ -76,9 +80,13 @@ impl MapKey for T { impl IntoValue for VecMap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(self.len()); for (k, v) in self { - map.insert_unchecked(k.to_key(), v.into_value()); + map.insert_unchecked(k.to_key(), v.into_value_with(opts)); } JmapValue::Object(map) } diff --git a/crates/registry/src/pickle.rs b/crates/registry/src/pickle.rs index dcfcc9f3..bd63d3cf 100644 --- a/crates/registry/src/pickle.rs +++ b/crates/registry/src/pickle.rs @@ -6,13 +6,15 @@ use crate::types::EnumImpl; use std::{borrow::Cow, collections::HashMap}; -use utils::{ - codec::leb128::{Leb128_, Leb128Reader, Leb128Writer}, - map::vec_map::VecMap, -}; +use utils::map::vec_map::VecMap; const COMPRESS_MARKER: u8 = 1 << 7; const COMPRESS_WATERMARK: usize = 8192; +const COMPRESS_LEN_SIZE: usize = std::mem::size_of::(); +const BLOCK_PLACEHOLDER_LEN: usize = 2; +const BLOCK_SHORT_LIMIT: usize = 1 << 14; +const SMALL_HEADER_LIMIT: u64 = 1 << 7; +pub const MAX_BITMAP_FLAGS: u64 = 64; pub trait Pickle: Sized { fn pickle(&self, out: &mut Vec); @@ -22,86 +24,166 @@ pub trait Pickle: Sized { pub struct PickledStream<'x> { data: Cow<'x, [u8]>, pos: usize, + end: usize, version: u8, } -pub(crate) fn maybe_compress_pickle(input: Vec) -> Vec { - let input_len = input.len() - 1; // Exclude the version byte - if input_len > COMPRESS_WATERMARK { - let (version, input) = input.split_first().unwrap(); - let mut bytes: Vec = vec![ - version | COMPRESS_MARKER; - lz4_flex::block::get_maximum_output_size(input_len) - + 1 - + std::mem::size_of::() - ]; +#[derive(Debug, Clone, Copy)] +pub struct Block { + parent_end: usize, + fields: u64, + flags: u64, + bitmap: u64, +} - // Compress the data - let compressed_len = - lz4_flex::compress_into(input, &mut bytes[std::mem::size_of::() + 1..]).unwrap(); - if compressed_len < input_len { - // Prepend the length of the uncompressed data - bytes[1..(std::mem::size_of::() + 1)] - .copy_from_slice(&(input_len as u32).to_le_bytes()); +impl Block { + #[inline(always)] + pub fn has(&self, field: u64) -> bool { + field < self.fields + } - // Truncate to the actual size - bytes.truncate(compressed_len + std::mem::size_of::() + 1); - return bytes; + #[inline(always)] + pub fn flag(&self, flag: u64) -> bool { + flag < self.flags + && u32::try_from(flag) + .ok() + .and_then(|shift| self.bitmap.checked_shr(shift)) + .is_some_and(|word| word & 1 != 0) + } +} + +pub(crate) fn maybe_compress_pickle(input: Vec) -> Vec { + let Some((&version, body)) = input.split_first() else { + return input; + }; + let Ok(body_len) = u32::try_from(body.len()) else { + return input; + }; + if body.len() <= COMPRESS_WATERMARK { + return input; + } + let mut bytes = + vec![0u8; 1 + COMPRESS_LEN_SIZE + lz4_flex::block::get_maximum_output_size(body.len())]; + let Some((header, rest)) = bytes.split_first_mut() else { + return input; + }; + *header = version | COMPRESS_MARKER; + let Some((len_slot, compressed)) = rest.split_first_chunk_mut::() else { + return input; + }; + *len_slot = body_len.to_le_bytes(); + match lz4_flex::compress_into(body, compressed) { + Ok(compressed_len) if compressed_len < body.len() => { + bytes.truncate(1 + COMPRESS_LEN_SIZE + compressed_len); + bytes } + _ => input, } - input } impl<'x> PickledStream<'x> { pub fn new(data: &'x [u8]) -> Option { let (marker, data) = data.split_first()?; let version = marker & !COMPRESS_MARKER; - if marker & COMPRESS_MARKER != 0 { - lz4_flex::block::decompress_size_prepended(data) - .ok() - .map(|data| PickledStream { - data: Cow::Owned(data), - pos: 0, - version, - }) + let data = if marker & COMPRESS_MARKER != 0 { + Cow::Owned(lz4_flex::block::decompress_size_prepended(data).ok()?) } else { - PickledStream { - data: Cow::Borrowed(data), - pos: 0, - version, - } - .into() - } - } - - pub fn read(&mut self) -> Option { - self.data.get(self.pos).copied().inspect(|_| self.pos += 1) - } - - pub fn read_leb128(&mut self) -> Option { - self.data - .get(self.pos..) - .and_then(|bytes| bytes.read_leb128()) - .map(|(value, read_bytes)| { - self.pos += read_bytes; - value - }) - } - - #[inline(always)] - pub fn read_bytes(&mut self, len: usize) -> Option<&'_ [u8]> { - self.data.get(self.pos..self.pos + len).inspect(|_| { - self.pos += len; + Cow::Borrowed(data) + }; + Some(PickledStream { + end: data.len(), + data, + pos: 0, + version, }) } #[inline(always)] - pub fn eof(&self) -> bool { - self.pos >= self.data.len() + pub fn remaining_len(&self) -> usize { + self.end.saturating_sub(self.pos) } #[inline(always)] - pub fn bytes(&self) -> &'_ [u8] { + fn remaining(&self) -> &[u8] { + self.data.get(self.pos..self.end).unwrap_or_default() + } + + #[inline(always)] + pub fn read(&mut self) -> Option { + let byte = *self.remaining().first()?; + self.pos += 1; + Some(byte) + } + + #[inline(always)] + pub fn read_bytes(&mut self, len: usize) -> Option<&[u8]> { + let start = self.pos; + let stop = start.checked_add(len).filter(|stop| *stop <= self.end)?; + self.pos = stop; + self.data.get(start..stop) + } + + #[inline(always)] + pub fn read_array(&mut self) -> Option<[u8; N]> { + let bytes = *self.remaining().first_chunk::()?; + self.pos += N; + Some(bytes) + } + + #[inline(always)] + pub fn read_varint(&mut self) -> Option { + let mut buf = self.remaining(); + let available = buf.len(); + let value = varint::read(&mut buf)?; + self.pos += available - buf.len(); + Some(value) + } + + #[inline(always)] + pub fn read_bitmap(&mut self, flags: u64) -> Option { + match usize::try_from(flags.div_ceil(8)).ok()? { + 0 => Some(0), + 1 => self.read().map(u64::from), + len => { + let bytes = self.read_bytes(len)?; + let mut word = [0u8; 8]; + word.iter_mut() + .zip(bytes) + .for_each(|(dst, src)| *dst = *src); + Some(u64::from_le_bytes(word)) + } + } + } + + #[inline(always)] + pub fn open_block(&mut self) -> Option { + let len = usize::try_from(self.read_varint()?).ok()?; + let block_end = self.pos.checked_add(len).filter(|end| *end <= self.end)?; + let parent_end = std::mem::replace(&mut self.end, block_end); + let fields = self.read_varint()?; + let flags = self.read_varint()?; + let bitmap = self.read_bitmap(flags)?; + Some(Block { + parent_end, + fields, + flags, + bitmap, + }) + } + + #[inline(always)] + pub fn close_block(&mut self, block: Block) { + self.pos = self.end; + self.end = block.parent_end; + } + + #[inline(always)] + pub fn eof(&self) -> bool { + self.pos >= self.end + } + + #[inline(always)] + pub fn bytes(&self) -> &[u8] { self.data.as_ref() } @@ -111,51 +193,201 @@ impl<'x> PickledStream<'x> { } } -impl Pickle for u16 { - fn pickle(&self, out: &mut Vec) { - let _ = out.write_leb128(*self); +#[inline(always)] +pub fn write_bitmap(out: &mut Vec, bitmap: u64, flags: u64) { + match flags.div_ceil(8) { + 0 => {} + 1 => out.push(bitmap as u8), + len => out.extend_from_slice(bitmap.to_le_bytes().get(..len as usize).unwrap_or_default()), + } +} + +#[inline(always)] +pub fn begin_block(out: &mut Vec, fields: u64, flags: u64, bitmap: u64) -> usize { + let start = out.len(); + if fields < SMALL_HEADER_LIMIT && flags < SMALL_HEADER_LIMIT { + out.extend_from_slice(&[0, 0, ((fields as u8) << 1) | 1, ((flags as u8) << 1) | 1]); + } else { + out.extend_from_slice(&[0; BLOCK_PLACEHOLDER_LEN]); + varint::write(out, fields); + varint::write(out, flags); + } + write_bitmap(out, bitmap, flags); + start +} + +#[inline(always)] +pub fn end_block(out: &mut Vec, start: usize) { + let len = out.len() - start - BLOCK_PLACEHOLDER_LEN; + if len < BLOCK_SHORT_LIMIT { + if let Some(slot) = out.get_mut(start..start + BLOCK_PLACEHOLDER_LEN) { + slot.copy_from_slice(&(((len << 2) | 0b10) as u16).to_le_bytes()); + } + } else { + end_long_block(out, start, len); + } +} + +#[cold] +fn end_long_block(out: &mut Vec, start: usize, len: usize) { + let mut header = Vec::with_capacity(9); + varint::write(&mut header, len as u64); + out.splice(start..start + BLOCK_PLACEHOLDER_LEN, header); +} + +pub mod varint { + #[inline(always)] + pub fn write(out: &mut Vec, value: u64) { + if value < 1 << 7 { + out.push(((value as u8) << 1) | 1); + } else if value < 1 << 14 { + out.extend_from_slice(&(((value << 2) | 0b10) as u16).to_le_bytes()); + } else if value < 1 << 21 { + let [a, b, c, _] = (((value << 3) | 0b100) as u32).to_le_bytes(); + out.extend_from_slice(&[a, b, c]); + } else if value < 1 << 28 { + out.extend_from_slice(&(((value << 4) | 0b1000) as u32).to_le_bytes()); + } else { + write_wide(out, value); + } } - fn unpickle(stream: &mut PickledStream<'_>) -> Option { - stream.read_leb128() + #[inline(always)] + fn write_wide(out: &mut Vec, value: u64) { + if value < 1 << 56 { + let bits = u64::BITS - value.leading_zeros(); + let len = bits.div_ceil(7); + let word = (value << len) | (1 << (len - 1)); + let start = out.len(); + out.extend_from_slice(&word.to_le_bytes()); + out.truncate(start + len as usize); + } else { + out.push(0); + out.extend_from_slice(&value.to_le_bytes()); + } + } + + #[inline(always)] + pub fn read(buf: &mut &[u8]) -> Option { + let (&byte, rest) = buf.split_first()?; + if byte & 1 != 0 { + *buf = rest; + return Some((byte >> 1) as u64); + } + if let Some(chunk) = buf.first_chunk::<8>() { + let word = u64::from_le_bytes(*chunk); + match byte.trailing_zeros() { + 1 => { + *buf = buf.get(2..)?; + return Some((word >> 2) & 0x3FFF); + } + 2 => { + *buf = buf.get(3..)?; + return Some((word >> 3) & 0x1F_FFFF); + } + 3 => { + *buf = buf.get(4..)?; + return Some((word >> 4) & 0x0FFF_FFFF); + } + 4 => { + *buf = buf.get(5..)?; + return Some((word >> 5) & 0x7_FFFF_FFFF); + } + zeros @ 5..=7 => { + let len = zeros + 1; + *buf = buf.get(len as usize..)?; + return Some((word >> len) & (u64::MAX >> (64 - 7 * len))); + } + _ => {} + } + } + read_tail(buf) + } + + #[inline] + fn read_tail(buf: &mut &[u8]) -> Option { + let (&first, rest) = buf.split_first()?; + if first == 0 { + let (word, rest) = rest.split_first_chunk::<8>()?; + *buf = rest; + return Some(u64::from_le_bytes(*word)); + } + let len = first.trailing_zeros() + 1; + let (bytes, rest) = buf.split_at_checked(len as usize)?; + let mut word = [0u8; 8]; + word.iter_mut() + .zip(bytes) + .for_each(|(dst, src)| *dst = *src); + *buf = rest; + Some(u64::from_le_bytes(word) >> len) + } + + #[inline(always)] + pub fn zigzag(value: i64) -> u64 { + ((value << 1) ^ (value >> 63)) as u64 + } + + #[inline(always)] + pub fn unzigzag(value: u64) -> i64 { + ((value >> 1) as i64) ^ -((value & 1) as i64) } } impl Pickle for u64 { + #[inline(always)] fn pickle(&self, out: &mut Vec) { - let _ = out.write_leb128(*self); + varint::write(out, *self); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { - stream.read_leb128() + stream.read_varint() } } impl Pickle for u32 { + #[inline(always)] fn pickle(&self, out: &mut Vec) { - let _ = out.write_leb128(*self); + varint::write(out, u64::from(*self)); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { - stream.read_leb128() + u32::try_from(stream.read_varint()?).ok() + } +} + +impl Pickle for u16 { + #[inline(always)] + fn pickle(&self, out: &mut Vec) { + varint::write(out, u64::from(*self)); + } + + #[inline(always)] + fn unpickle(stream: &mut PickledStream<'_>) -> Option { + u16::try_from(stream.read_varint()?).ok() } } impl Pickle for i64 { + #[inline(always)] fn pickle(&self, out: &mut Vec) { - let _ = out.write_leb128(*self as u64); + varint::write(out, varint::zigzag(*self)); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { - stream.read_leb128::().map(|v| v as i64) + stream.read_varint().map(varint::unzigzag) } } impl Pickle for bool { + #[inline(always)] fn pickle(&self, out: &mut Vec) { - out.push(if *self { 1 } else { 0 }); + out.push(u8::from(*self)); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { match stream.read()? { 0 => Some(false), @@ -166,44 +398,46 @@ impl Pickle for bool { } impl Pickle for String { + #[inline(always)] fn pickle(&self, out: &mut Vec) { - (self.len() as u32).pickle(out); + varint::write(out, self.len() as u64); out.extend_from_slice(self.as_bytes()); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { - u32::unpickle(stream) - .and_then(|len| stream.read_bytes(len as usize)) - .and_then(|bytes| String::from_utf8(bytes.to_vec()).ok()) + let len = usize::try_from(stream.read_varint()?).ok()?; + std::str::from_utf8(stream.read_bytes(len)?) + .ok() + .map(str::to_owned) } } impl Pickle for T { + #[inline(always)] fn pickle(&self, out: &mut Vec) { self.to_id().pickle(out); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { u16::unpickle(stream).and_then(Self::from_id) } } -impl Pickle for Option -where - T: Pickle, -{ +impl Pickle for Option { + #[inline(always)] fn pickle(&self, out: &mut Vec) { match self { Some(value) => { out.push(1); value.pickle(out); } - None => { - out.push(0); - } + None => out.push(0), } } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { match stream.read()? { 0 => Some(None), @@ -220,7 +454,7 @@ where S: std::hash::BuildHasher + Default, { fn pickle(&self, out: &mut Vec) { - (self.len() as u32).pickle(out); + varint::write(out, self.len() as u64); for (key, value) in self { key.pickle(out); value.pickle(out); @@ -228,8 +462,9 @@ where } fn unpickle(stream: &mut PickledStream<'_>) -> Option { - let len = u32::unpickle(stream)? as usize; - let mut map = HashMap::with_capacity_and_hasher(len, S::default()); + let len = usize::try_from(stream.read_varint()?).ok()?; + let mut map = + HashMap::with_capacity_and_hasher(len.min(stream.remaining().len()), S::default()); for _ in 0..len { let key = K::unpickle(stream)?; let value = V::unpickle(stream)?; @@ -245,7 +480,7 @@ where V: Pickle, { fn pickle(&self, out: &mut Vec) { - (self.len() as u32).pickle(out); + varint::write(out, self.len() as u64); for (key, value) in self { key.pickle(out); value.pickle(out); @@ -253,8 +488,8 @@ where } fn unpickle(stream: &mut PickledStream<'_>) -> Option { - let len = u32::unpickle(stream)? as usize; - let mut map = VecMap::with_capacity(len); + let len = usize::try_from(stream.read_varint()?).ok()?; + let mut map = VecMap::with_capacity(len.min(stream.remaining().len())); for _ in 0..len { let key = K::unpickle(stream)?; let value = V::unpickle(stream)?; @@ -265,11 +500,14 @@ where } impl Pickle for trc::Key { + #[inline(always)] fn pickle(&self, out: &mut Vec) { self.to_id().pickle(out); } + #[inline(always)] fn unpickle(stream: &mut PickledStream<'_>) -> Option { u16::unpickle(stream).and_then(Self::from_id) } } + diff --git a/crates/registry/src/schema/enums.rs b/crates/registry/src/schema/enums.rs index ab3d16ca..2c524c61 100644 --- a/crates/registry/src/schema/enums.rs +++ b/crates/registry/src/schema/enums.rs @@ -51,49 +51,6 @@ pub enum ActionType { ResumeMtaQueue = 10, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum AiModelType { - #[default] - Chat = 0, - Text = 1, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum AlertEmailType { - #[default] - Disabled = 0, - Enabled = 1, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum AlertEventType { - #[default] - Disabled = 0, - Enabled = 1, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum ArchivedItemStatus { - #[default] - Archived = 0, - RequestRestore = 1, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum ArchivedItemType { - #[default] - Email = 0, - FileNode = 1, - CalendarEvent = 2, - ContactCard = 3, - SieveScript = 4, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ArfAuthFailureType { @@ -151,6 +108,17 @@ pub enum AsnType { Dns = 2, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum AzureAuthType { + #[default] + SharedKey = 0, + SasToken = 1, + ClientSecret = 2, + ManagedIdentity = 3, + WorkloadIdentity = 4, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum AzureEnvironment { @@ -160,24 +128,11 @@ pub enum AzureEnvironment { UsGovernment = 2, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum BlobStoreBaseType { - #[default] - S3 = 0, - Azure = 1, - FileSystem = 2, - FoundationDb = 3, - PostgreSql = 4, - MySql = 5, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum BlobStoreType { #[default] Default = 0, - Sharded = 1, S3 = 2, Azure = 3, FileSystem = 4, @@ -186,6 +141,19 @@ pub enum BlobStoreType { MySql = 7, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum BlobSwapStoreType { + #[default] + Default = 0, + S3 = 1, + Azure = 2, + FileSystem = 3, + FoundationDb = 4, + PostgreSql = 5, + MySql = 6, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum BlockReason { @@ -198,6 +166,16 @@ pub enum BlockReason { Other = 5, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum CacheSwapType { + #[default] + Disabled = 0, + LocalFile = 1, + Redis = 2, + BlobStore = 3, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum CertificateManagementType { @@ -206,6 +184,15 @@ pub enum CertificateManagementType { Automatic = 1, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum ClientIpSource { + #[default] + Peer = 0, + XForwardedFor = 1, + Forwarded = 2, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ClusterListenerGroupType { @@ -349,9 +336,15 @@ pub enum DirectoryType { #[repr(u16)] pub enum Dkim2Flag { #[default] - Donotmodify = 0, - Donotexplode = 1, - Feedback = 2, + Feedback = 0, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum Dkim2HashAlgorithm { + #[default] + Sha256 = 0, + Sha512 = 1, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -411,6 +404,7 @@ pub enum DkimSignatureType { Dkim1RsaSha256 = 1, Dkim2Ed25519Sha256 = 2, Dkim2RsaSha256 = 3, + Dkim2MlDsa44Ed25519Sha512 = 4, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -549,84 +543,6 @@ pub enum DnsServerBootstrapType { #[default] Manual = 0, Tsig = 1, - Deprecated1 = 2, - Cloudflare = 3, - DigitalOcean = 4, - DeSEC = 5, - Ovh = 6, - Bunny = 7, - Porkbun = 8, - Dnsimple = 9, - Spaceship = 10, - Route53 = 11, - GoogleCloudDns = 12, - Alidns = 13, - ArvanCloud = 14, - Autodns = 15, - AzureDns = 16, - BaiduCloud = 17, - BluecatV2 = 18, - ClouDns = 19, - Constellix = 20, - Cpanel = 21, - Ddnss = 22, - DnsMadeEasy = 23, - Domeneshop = 24, - Dreamhost = 25, - DuckDns = 26, - Dynu = 27, - EasyDns = 28, - EdgeDns = 29, - Exoscale = 30, - FreeMyIp = 31, - GandiV5 = 32, - Gcore = 33, - Glesys = 34, - Godaddy = 35, - Hetzner = 36, - HostingDe = 37, - Hostinger = 38, - HuaweiCloud = 39, - Hurricane = 40, - IbmCloud = 41, - Infoblox = 42, - Infomaniak = 43, - Inwx = 44, - Ionos = 45, - Ipv64 = 46, - Joker = 47, - Lightsail = 48, - Linode = 49, - LuaDns = 50, - MythicBeasts = 51, - Namecheap = 52, - NameDotCom = 53, - NameSilo = 54, - Netcup = 55, - Netlify = 56, - Nifcloud = 57, - Ns1 = 58, - OracleCloud = 59, - Plesk = 60, - Safedns = 61, - Scaleway = 62, - TencentCloud = 63, - Transip = 64, - UltraDns = 65, - Vercel = 66, - Volcengine = 67, - Vultr = 68, - WebSupport = 69, - YandexCloud = 70, - PowerDns = 71, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum DnsServerType { - #[default] - Tsig = 0, - Deprecated1 = 1, Cloudflare = 2, DigitalOcean = 3, DeSEC = 4, @@ -698,6 +614,82 @@ pub enum DnsServerType { PowerDns = 70, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum DnsServerType { + #[default] + Tsig = 0, + Cloudflare = 1, + DigitalOcean = 2, + DeSEC = 3, + Ovh = 4, + Bunny = 5, + Porkbun = 6, + Dnsimple = 7, + Spaceship = 8, + Route53 = 9, + GoogleCloudDns = 10, + Alidns = 11, + ArvanCloud = 12, + Autodns = 13, + AzureDns = 14, + BaiduCloud = 15, + BluecatV2 = 16, + ClouDns = 17, + Constellix = 18, + Cpanel = 19, + Ddnss = 20, + DnsMadeEasy = 21, + Domeneshop = 22, + Dreamhost = 23, + DuckDns = 24, + Dynu = 25, + EasyDns = 26, + EdgeDns = 27, + Exoscale = 28, + FreeMyIp = 29, + GandiV5 = 30, + Gcore = 31, + Glesys = 32, + Godaddy = 33, + Hetzner = 34, + HostingDe = 35, + Hostinger = 36, + HuaweiCloud = 37, + Hurricane = 38, + IbmCloud = 39, + Infoblox = 40, + Infomaniak = 41, + Inwx = 42, + Ionos = 43, + Ipv64 = 44, + Joker = 45, + Lightsail = 46, + Linode = 47, + LuaDns = 48, + MythicBeasts = 49, + Namecheap = 50, + NameDotCom = 51, + NameSilo = 52, + Netcup = 53, + Netlify = 54, + Nifcloud = 55, + Ns1 = 56, + OracleCloud = 57, + Plesk = 58, + Safedns = 59, + Scaleway = 60, + TencentCloud = 61, + Transip = 62, + UltraDns = 63, + Vercel = 64, + Volcengine = 65, + Vultr = 66, + WebSupport = 67, + YandexCloud = 68, + PowerDns = 69, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum EncryptionAtRestType { @@ -724,22 +716,16 @@ pub enum ExpressionConstant { Relaxed = 0, Strict = 1, Disable = 2, - Optional = 3, - Require = 4, - Ipv4Only = 5, - Ipv6Only = 6, - Ipv6ThenIpv4 = 7, - Ipv4ThenIpv6 = 8, - Hourly = 9, - Daily = 10, - Weekly = 11, - Login = 12, - Plain = 13, - Xoauth2 = 14, - Oauthbearer = 15, - Mixer = 16, - Stanag4406 = 17, - Nsep = 18, + Hourly = 3, + Daily = 4, + Weekly = 5, + Login = 6, + Plain = 7, + Xoauth2 = 8, + Oauthbearer = 9, + Mixer = 10, + Stanag4406 = 11, + Nsep = 12, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -820,24 +806,26 @@ pub enum ExpressionVariable { ReplyToLocal = 71, ReplyToName = 72, RetryNum = 73, - ReverseIp = 74, - Scheme = 75, - Sender = 76, - SenderDomain = 77, - Size = 78, - Sld = 79, - Source = 80, - Subject = 81, - SubjectThread = 82, - SubjectWords = 83, - To = 84, - ToDomain = 85, - ToLocal = 86, - ToName = 87, - Url = 88, - UrlOriginal = 89, - Value = 90, - ValueLower = 91, + Scheme = 74, + Sender = 75, + SenderDomain = 76, + Size = 77, + Sld = 78, + Source = 79, + Subject = 80, + SubjectThread = 81, + SubjectWords = 82, + To = 83, + ToDomain = 84, + ToLocal = 85, + ToName = 86, + Url = 87, + UrlOriginal = 88, + Value = 89, + ValueLower = 90, + SpamScore = 91, + SpamProbability = 92, + SpamThreshold = 93, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -867,21 +855,11 @@ pub enum HttpLookupFormatType { List = 1, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum InMemoryStoreBaseType { - #[default] - Redis = 0, - RedisCluster = 1, - RedisSentinel = 2, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum InMemoryStoreType { #[default] Default = 0, - Sharded = 1, Redis = 2, RedisCluster = 3, RedisSentinel = 4, @@ -889,12 +867,20 @@ pub enum InMemoryStoreType { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] -pub enum IndexDocumentType { +pub enum IndexStatusType { #[default] - Email = 0, - Calendar = 1, - Contacts = 2, - File = 3, + Running = 0, + Failed = 1, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum IndexType { + #[cfg_attr(not(feature = "enterprise"), default)] + Email = 1, + Calendar = 2, + Contacts = 3, + File = 4, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -1004,258 +990,258 @@ pub enum Locale { CyGB = 67, DaDK = 68, DeAT = 69, - DeBE = 71, - En001 = 72, - DeCH = 73, - DeDE = 74, - DeIT = 76, - DeLI = 77, - DeLU = 78, - Ar001 = 79, - DoiIN = 80, - DsbDE = 81, - DvMV = 82, - DzBT = 83, - ElCY = 84, - ElGR = 85, - EnAG = 87, - EnAU = 88, - EnBW = 89, - EnCA = 90, - EnDK = 91, - EnGB = 92, - EnHK = 93, - EnIE = 94, - EnIL = 96, - EnIN = 97, - EnNG = 98, - EnNZ = 99, - EnPH = 100, - EnSC = 101, - EnSG = 102, - EnZA = 104, - EnZM = 105, - EnZW = 106, - Eo = 107, - EsAR = 108, - EsBO = 109, - EsCL = 110, - EsCO = 111, - EsCR = 112, - EsCU = 113, - EsDO = 114, - EsEC = 115, - EsES = 116, - EsGT = 118, - EsHN = 119, - EsMX = 120, - EsNI = 121, - EsPA = 122, - EsPE = 123, - EsPR = 124, - EsPY = 125, - EsSV = 126, - EsUS = 127, - EsUY = 128, - EsVE = 129, - EtEE = 130, - EuES = 131, - ZhHans = 132, - FaIR = 133, - FfSN = 134, - FiFI = 135, - FilPH = 137, - FoFO = 138, - FrBE = 139, - FrCA = 141, - FrCH = 142, - FrFR = 143, - FrLU = 145, - ZhHant = 146, - FurIT = 147, - FyDE = 148, - FyNL = 149, - GaIE = 150, - ZhHantHK = 151, - GdGB = 152, - GezER = 153, - SrLatnBA = 154, - GezET = 155, - SrLatnME = 156, - GlES = 157, - AzLatnAZ = 158, - GuIN = 159, - GvGB = 160, - HaNG = 161, - HakTW = 162, - HeIL = 163, - HiIN = 164, - HifFJ = 165, - HneIN = 166, - HrHR = 167, - HsbDE = 168, - HtHT = 169, - HuHU = 170, - HyAM = 171, - IaFR = 172, - IdID = 173, - IgNG = 174, - IkCA = 175, - IsIS = 176, - ItCH = 177, - ItIT = 178, - IuCA = 180, - JaJP = 181, - KaGE = 182, - KabDZ = 183, - KkKZ = 184, - KlGL = 185, - KmKH = 186, - KnIN = 187, - KoKR = 188, - KokIN = 189, - KsIN = 190, - KsDevaIN = 191, - KuTR = 192, - KwGB = 193, - KyKG = 194, - LbLU = 195, - LgUG = 196, - LiBE = 197, - LiNL = 198, - LijIT = 199, - LnCD = 200, - LoLA = 201, - LtLT = 202, - LvLV = 203, - LzhTW = 204, - MagIN = 205, - MaiIN = 206, - MaiNP = 207, - MfeMU = 208, - MgMG = 209, - MhrRU = 210, - MiNZ = 211, - MiqNI = 212, - MjwIN = 213, - MkMK = 214, - MlIN = 215, - MnMN = 216, - MniIN = 217, - MnwMM = 218, - MrIN = 219, - MsMY = 220, - MtMT = 221, - MyMM = 222, - NanTW = 223, - NanLatnTW = 224, - NbNO = 225, - NdsDE = 226, - NdsNL = 227, - NeNP = 228, - NhnMX = 229, - NiuNU = 230, - NiuNZ = 231, - NlAW = 232, - NlBE = 233, - NlNL = 235, - NnNO = 237, - NrZA = 238, - NsoZA = 239, - OcFR = 240, - OmET = 241, - OmKE = 242, - OrIN = 243, - OsRU = 244, - PaIN = 245, - PaPK = 246, - PapAW = 247, - PapCW = 248, - PlPL = 249, - PsAF = 250, - PtBR = 251, - PtPT = 252, - QuzPE = 254, - RajIN = 255, - RoRO = 256, - RuRU = 257, - RuUA = 258, - RwRW = 259, - SaIN = 260, - SahRU = 261, - SatIN = 262, - ScIT = 263, - SdIN = 264, - SdDevaIN = 265, - SeNO = 266, - SgsLT = 267, - ShnMM = 268, - ShsCA = 269, - SiLK = 270, - SidET = 271, - SkSK = 272, - SlSI = 273, - SmWS = 274, - SoDJ = 275, - SoET = 276, - SoKE = 277, - SoSO = 278, - SqAL = 279, - SqMK = 280, - SrME = 281, - SrRS = 282, - SrLatnRS = 283, - SsZA = 284, - StZA = 285, - SvFI = 286, - SvSE = 288, - SwKE = 289, - SwTZ = 290, - SzlPL = 291, - TaIN = 292, - TaLK = 293, - TcyIN = 294, - TeIN = 295, - TgTJ = 296, - ThTH = 297, - TheNP = 298, - TiER = 299, - TiET = 300, - TigER = 301, - TkTM = 302, - TlPH = 303, - TnZA = 304, - ToTO = 305, - TpiPG = 306, - TrCY = 307, - TrTR = 308, - TsZA = 309, - TtRU = 310, - TtLatnRU = 311, - UgCN = 312, - UkUA = 313, - UnmUS = 314, - UrIN = 315, - UrPK = 316, - UzUZ = 317, - UzCyrlUZ = 318, - VeZA = 319, - ViVN = 320, - WaBE = 321, - UzLatnUZ = 322, - WaeCH = 323, - WalET = 324, - WoSN = 325, - XhZA = 326, - YiUS = 327, - YoNG = 328, - YueHK = 329, - YuwPG = 330, - ZhCN = 331, - ZhHK = 332, - ZhSG = 333, - ZhTW = 334, - ZuZA = 335, + DeBE = 70, + En001 = 71, + DeCH = 72, + DeDE = 73, + DeIT = 74, + DeLI = 75, + DeLU = 76, + Ar001 = 77, + DoiIN = 78, + DsbDE = 79, + DvMV = 80, + DzBT = 81, + ElCY = 82, + ElGR = 83, + EnAG = 84, + EnAU = 85, + EnBW = 86, + EnCA = 87, + EnDK = 88, + EnGB = 89, + EnHK = 90, + EnIE = 91, + EnIL = 92, + EnIN = 93, + EnNG = 94, + EnNZ = 95, + EnPH = 96, + EnSC = 97, + EnSG = 98, + EnZA = 99, + EnZM = 100, + EnZW = 101, + Eo = 102, + EsAR = 103, + EsBO = 104, + EsCL = 105, + EsCO = 106, + EsCR = 107, + EsCU = 108, + EsDO = 109, + EsEC = 110, + EsES = 111, + EsGT = 112, + EsHN = 113, + EsMX = 114, + EsNI = 115, + EsPA = 116, + EsPE = 117, + EsPR = 118, + EsPY = 119, + EsSV = 120, + EsUS = 121, + EsUY = 122, + EsVE = 123, + EtEE = 124, + EuES = 125, + ZhHans = 126, + FaIR = 127, + FfSN = 128, + FiFI = 129, + FilPH = 130, + FoFO = 131, + FrBE = 132, + FrCA = 133, + FrCH = 134, + FrFR = 135, + FrLU = 136, + ZhHant = 137, + FurIT = 138, + FyDE = 139, + FyNL = 140, + GaIE = 141, + ZhHantHK = 142, + GdGB = 143, + GezER = 144, + SrLatnBA = 145, + GezET = 146, + SrLatnME = 147, + GlES = 148, + AzLatnAZ = 149, + GuIN = 150, + GvGB = 151, + HaNG = 152, + HakTW = 153, + HeIL = 154, + HiIN = 155, + HifFJ = 156, + HneIN = 157, + HrHR = 158, + HsbDE = 159, + HtHT = 160, + HuHU = 161, + HyAM = 162, + IaFR = 163, + IdID = 164, + IgNG = 165, + IkCA = 166, + IsIS = 167, + ItCH = 168, + ItIT = 169, + IuCA = 170, + JaJP = 171, + KaGE = 172, + KabDZ = 173, + KkKZ = 174, + KlGL = 175, + KmKH = 176, + KnIN = 177, + KoKR = 178, + KokIN = 179, + KsIN = 180, + KsDevaIN = 181, + KuTR = 182, + KwGB = 183, + KyKG = 184, + LbLU = 185, + LgUG = 186, + LiBE = 187, + LiNL = 188, + LijIT = 189, + LnCD = 190, + LoLA = 191, + LtLT = 192, + LvLV = 193, + LzhTW = 194, + MagIN = 195, + MaiIN = 196, + MaiNP = 197, + MfeMU = 198, + MgMG = 199, + MhrRU = 200, + MiNZ = 201, + MiqNI = 202, + MjwIN = 203, + MkMK = 204, + MlIN = 205, + MnMN = 206, + MniIN = 207, + MnwMM = 208, + MrIN = 209, + MsMY = 210, + MtMT = 211, + MyMM = 212, + NanTW = 213, + NanLatnTW = 214, + NbNO = 215, + NdsDE = 216, + NdsNL = 217, + NeNP = 218, + NhnMX = 219, + NiuNU = 220, + NiuNZ = 221, + NlAW = 222, + NlBE = 223, + NlNL = 224, + NnNO = 225, + NrZA = 226, + NsoZA = 227, + OcFR = 228, + OmET = 229, + OmKE = 230, + OrIN = 231, + OsRU = 232, + PaIN = 233, + PaPK = 234, + PapAW = 235, + PapCW = 236, + PlPL = 237, + PsAF = 238, + PtBR = 239, + PtPT = 240, + QuzPE = 241, + RajIN = 242, + RoRO = 243, + RuRU = 244, + RuUA = 245, + RwRW = 246, + SaIN = 247, + SahRU = 248, + SatIN = 249, + ScIT = 250, + SdIN = 251, + SdDevaIN = 252, + SeNO = 253, + SgsLT = 254, + ShnMM = 255, + ShsCA = 256, + SiLK = 257, + SidET = 258, + SkSK = 259, + SlSI = 260, + SmWS = 261, + SoDJ = 262, + SoET = 263, + SoKE = 264, + SoSO = 265, + SqAL = 266, + SqMK = 267, + SrME = 268, + SrRS = 269, + SrLatnRS = 270, + SsZA = 271, + StZA = 272, + SvFI = 273, + SvSE = 274, + SwKE = 275, + SwTZ = 276, + SzlPL = 277, + TaIN = 278, + TaLK = 279, + TcyIN = 280, + TeIN = 281, + TgTJ = 282, + ThTH = 283, + TheNP = 284, + TiER = 285, + TiET = 286, + TigER = 287, + TkTM = 288, + TlPH = 289, + TnZA = 290, + ToTO = 291, + TpiPG = 292, + TrCY = 293, + TrTR = 294, + TsZA = 295, + TtRU = 296, + TtLatnRU = 297, + UgCN = 298, + UkUA = 299, + UnmUS = 300, + UrIN = 301, + UrPK = 302, + UzUZ = 303, + UzCyrlUZ = 304, + VeZA = 305, + ViVN = 306, + WaBE = 307, + UzLatnUZ = 308, + WaeCH = 309, + WalET = 310, + WoSN = 311, + XhZA = 312, + YiUS = 313, + YoNG = 314, + YueHK = 315, + YuwPG = 316, + ZhCN = 317, + ZhHK = 318, + ZhSG = 319, + ZhTW = 320, + ZuZA = 321, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -1275,7 +1261,6 @@ pub enum LookupStoreType { PostgreSql = 0, MySql = 1, Sqlite = 2, - Sharded = 3, Redis = 4, RedisCluster = 5, RedisSentinel = 6, @@ -1295,11 +1280,16 @@ pub enum MessageFlag { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] -pub enum MetricType { +pub enum MetadataDataType { #[default] - Counter = 0, - Gauge = 1, - Histogram = 2, + Email = 0, + Mailbox = 1, + SieveScript = 2, + Calendar = 3, + CalendarEvent = 4, + AddressBook = 5, + ContactCard = 6, + FileNode = 7, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -1319,17 +1309,6 @@ pub enum MetricsPrometheusType { Enabled = 1, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum MetricsStoreType { - #[default] - Disabled = 0, - Default = 1, - FoundationDb = 2, - PostgreSql = 3, - MySql = 4, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MilterVersion { @@ -1470,6 +1449,71 @@ pub enum NetworkListenerProtocol { ManageSieve = 5, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OAuthClientKind { + #[default] + Registered = 0, + Dynamic = 1, + MetadataDocument = 2, + FirstParty = 3, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OAuthClientMetadataPolicy { + #[default] + Disabled = 0, + SameDomain = 1, + Any = 2, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OAuthGrantType { + #[default] + AuthorizationCode = 0, + RefreshToken = 1, + DeviceCode = 2, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OAuthScope { + #[default] + Mail = 0, + Contacts = 1, + Calendars = 2, + Files = 3, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OidcIntrospectionAuth { + #[default] + ClientSecretBasic = 0, + ClientSecretPost = 1, + None = 2, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OidcOpaqueTokenValidation { + #[default] + None = 0, + UserInfo = 1, + Introspection = 2, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum OidcUnscopedTokenPolicy { + #[default] + Reject = 0, + Interoperable = 1, + Full = 2, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum OvhEndpoint { @@ -1509,7 +1553,6 @@ pub enum Permission { #[default] Authenticate = 0, AuthenticateWithAlias = 1, - InteractAi = 2, Impersonate = 3, UnlimitedRequests = 4, UnlimitedUploads = 5, @@ -1570,604 +1613,545 @@ pub enum Permission { JmapPrincipalChanges = 60, JmapPrincipalQueryChanges = 61, JmapPrincipalGetAvailability = 62, - JmapPrincipalCreate = 63, - JmapPrincipalUpdate = 64, - JmapPrincipalDestroy = 65, - JmapQuotaGet = 66, - JmapQuotaChanges = 67, - JmapQuotaQuery = 68, - JmapQuotaQueryChanges = 69, - JmapBlobGet = 70, - JmapBlobCopy = 71, - JmapBlobLookup = 72, - JmapBlobUpload = 73, - JmapAddressBookGet = 74, - JmapAddressBookChanges = 75, - JmapAddressBookCreate = 76, - JmapAddressBookUpdate = 77, - JmapAddressBookDestroy = 78, - JmapContactCardGet = 79, - JmapContactCardChanges = 80, - JmapContactCardQuery = 81, - JmapContactCardQueryChanges = 82, - JmapContactCardCreate = 83, - JmapContactCardUpdate = 84, - JmapContactCardDestroy = 85, - JmapContactCardCopy = 86, - JmapContactCardParse = 87, - JmapFileNodeGet = 88, - JmapFileNodeChanges = 89, - JmapFileNodeQuery = 90, - JmapFileNodeQueryChanges = 91, - JmapFileNodeCreate = 92, - JmapFileNodeUpdate = 93, - JmapFileNodeDestroy = 94, - JmapFileNodeCopy = 659, - JmapShareNotificationGet = 95, - JmapShareNotificationChanges = 96, - JmapShareNotificationQuery = 97, - JmapShareNotificationQueryChanges = 98, - JmapShareNotificationCreate = 99, - JmapShareNotificationUpdate = 100, - JmapShareNotificationDestroy = 101, - JmapCalendarGet = 102, - JmapCalendarChanges = 103, - JmapCalendarCreate = 104, - JmapCalendarUpdate = 105, - JmapCalendarDestroy = 106, - JmapCalendarEventGet = 107, - JmapCalendarEventChanges = 108, - JmapCalendarEventQuery = 109, - JmapCalendarEventQueryChanges = 110, - JmapCalendarEventCreate = 111, - JmapCalendarEventUpdate = 112, - JmapCalendarEventDestroy = 113, - JmapCalendarEventCopy = 114, - JmapCalendarEventParse = 115, - JmapCalendarEventNotificationGet = 116, - JmapCalendarEventNotificationChanges = 117, - JmapCalendarEventNotificationQuery = 118, - JmapCalendarEventNotificationQueryChanges = 119, - JmapCalendarEventNotificationCreate = 120, - JmapCalendarEventNotificationUpdate = 121, - JmapCalendarEventNotificationDestroy = 122, - JmapParticipantIdentityGet = 123, - JmapParticipantIdentityChanges = 124, - JmapParticipantIdentityCreate = 125, - JmapParticipantIdentityUpdate = 126, - JmapParticipantIdentityDestroy = 127, - JmapCoreEcho = 128, - ImapAuthenticate = 129, - ImapAclGet = 130, - ImapAclSet = 131, - ImapMyRights = 132, - ImapListRights = 133, - ImapAppend = 134, - ImapCapability = 135, - ImapId = 136, - ImapCopy = 137, - ImapMove = 138, - ImapCreate = 139, - ImapDelete = 140, - ImapEnable = 141, - ImapExpunge = 142, - ImapFetch = 143, - ImapIdle = 144, - ImapList = 145, - ImapLsub = 146, - ImapNamespace = 147, - ImapRename = 148, - ImapSearch = 149, - ImapSort = 150, - ImapSelect = 151, - ImapExamine = 152, - ImapStatus = 153, - ImapStore = 154, - ImapSubscribe = 155, - ImapThread = 156, - Pop3Authenticate = 157, - Pop3List = 158, - Pop3Uidl = 159, - Pop3Stat = 160, - Pop3Retr = 161, - Pop3Dele = 162, - SieveAuthenticate = 163, - SieveListScripts = 164, - SieveSetActive = 165, - SieveGetScript = 166, - SievePutScript = 167, - SieveDeleteScript = 168, - SieveRenameScript = 169, - SieveCheckScript = 170, - SieveHaveSpace = 171, - DavSyncCollection = 172, - DavExpandProperty = 173, - DavPrincipalAcl = 174, - DavPrincipalList = 175, - DavPrincipalMatch = 176, - DavPrincipalSearch = 177, - DavPrincipalSearchPropSet = 178, - DavFilePropFind = 179, - DavFilePropPatch = 180, - DavFileGet = 181, - DavFileMkCol = 182, - DavFileDelete = 183, - DavFilePut = 184, - DavFileCopy = 185, - DavFileMove = 186, - DavFileLock = 187, - DavFileAcl = 188, - DavCardPropFind = 189, - DavCardPropPatch = 190, - DavCardGet = 191, - DavCardMkCol = 192, - DavCardDelete = 193, - DavCardPut = 194, - DavCardCopy = 195, - DavCardMove = 196, - DavCardLock = 197, - DavCardAcl = 198, - DavCardQuery = 199, - DavCardMultiGet = 200, - DavCalPropFind = 201, - DavCalPropPatch = 202, - DavCalGet = 203, - DavCalMkCol = 204, - DavCalDelete = 205, - DavCalPut = 206, - DavCalCopy = 207, - DavCalMove = 208, - DavCalLock = 209, - DavCalAcl = 210, - DavCalQuery = 211, - DavCalMultiGet = 212, - DavCalFreeBusyQuery = 213, - OAuthClientRegistration = 214, - OAuthClientOverride = 215, - LiveTracing = 216, - LiveMetrics = 217, - LiveDeliveryTest = 218, - ScimAccess = 660, - SysAccountGet = 219, - SysAccountCreate = 220, - SysAccountUpdate = 221, - SysAccountDestroy = 222, - SysAccountQuery = 223, - SysAccountPasswordGet = 224, - SysAccountPasswordUpdate = 225, - SysAccountSettingsGet = 226, - SysAccountSettingsUpdate = 227, - SysAcmeProviderGet = 228, - SysAcmeProviderCreate = 229, - SysAcmeProviderUpdate = 230, - SysAcmeProviderDestroy = 231, - SysAcmeProviderQuery = 232, - ActionReloadSettings = 233, - ActionReloadTlsCertificates = 234, - ActionReloadLookupStores = 235, - ActionReloadBlockedIps = 236, - ActionUpdateApps = 237, - ActionTroubleshootDmarc = 238, - ActionClassifySpam = 239, - ActionInvalidateCaches = 240, - ActionInvalidateNegativeCaches = 241, - ActionPauseMtaQueue = 242, - ActionResumeMtaQueue = 243, - SysActionGet = 244, - SysActionCreate = 245, - SysActionUpdate = 246, - SysActionDestroy = 247, - SysActionQuery = 248, - SysAddressBookGet = 249, - SysAddressBookUpdate = 250, - SysAiModelGet = 251, - SysAiModelCreate = 252, - SysAiModelUpdate = 253, - SysAiModelDestroy = 254, - SysAiModelQuery = 255, - SysAlertGet = 256, - SysAlertCreate = 257, - SysAlertUpdate = 258, - SysAlertDestroy = 259, - SysAlertQuery = 260, - SysAllowedIpGet = 261, - SysAllowedIpCreate = 262, - SysAllowedIpUpdate = 263, - SysAllowedIpDestroy = 264, - SysAllowedIpQuery = 265, - SysApiKeyGet = 266, - SysApiKeyCreate = 267, - SysApiKeyUpdate = 268, - SysApiKeyDestroy = 269, - SysApiKeyQuery = 270, - SysAppPasswordGet = 271, - SysAppPasswordCreate = 272, - SysAppPasswordUpdate = 273, - SysAppPasswordDestroy = 274, - SysAppPasswordQuery = 275, - SysApplicationGet = 276, - SysApplicationCreate = 277, - SysApplicationUpdate = 278, - SysApplicationDestroy = 279, - SysApplicationQuery = 280, - SysArchivedItemGet = 281, - SysArchivedItemCreate = 282, - SysArchivedItemUpdate = 283, - SysArchivedItemDestroy = 284, - SysArchivedItemQuery = 285, - SysArfExternalReportGet = 286, - SysArfExternalReportCreate = 287, - SysArfExternalReportUpdate = 288, - SysArfExternalReportDestroy = 289, - SysArfExternalReportQuery = 290, - SysAsnGet = 291, - SysAsnUpdate = 292, - SysAuthenticationGet = 293, - SysAuthenticationUpdate = 294, - SysBlobStoreGet = 295, - SysBlobStoreUpdate = 296, - SysBlockedIpGet = 297, - SysBlockedIpCreate = 298, - SysBlockedIpUpdate = 299, - SysBlockedIpDestroy = 300, - SysBlockedIpQuery = 301, - SysBootstrapGet = 302, - SysBootstrapUpdate = 303, - SysCacheGet = 304, - SysCacheUpdate = 305, - SysCalendarGet = 306, - SysCalendarUpdate = 307, - SysCalendarAlarmGet = 308, - SysCalendarAlarmUpdate = 309, - SysCalendarSchedulingGet = 310, - SysCalendarSchedulingUpdate = 311, - SysCertificateGet = 312, - SysCertificateCreate = 313, - SysCertificateUpdate = 314, - SysCertificateDestroy = 315, - SysCertificateQuery = 316, - SysClusterNodeGet = 317, - SysClusterNodeCreate = 318, - SysClusterNodeUpdate = 319, - SysClusterNodeDestroy = 320, - SysClusterNodeQuery = 321, - SysClusterRoleGet = 322, - SysClusterRoleCreate = 323, - SysClusterRoleUpdate = 324, - SysClusterRoleDestroy = 325, - SysClusterRoleQuery = 326, - SysCoordinatorGet = 327, - SysCoordinatorUpdate = 328, - SysDataRetentionGet = 329, - SysDataRetentionUpdate = 330, - SysDataStoreGet = 331, - SysDataStoreUpdate = 332, - SysDirectoryGet = 333, - SysDirectoryCreate = 334, - SysDirectoryUpdate = 335, - SysDirectoryDestroy = 336, - SysDirectoryQuery = 337, - SysDkimReportSettingsGet = 338, - SysDkimReportSettingsUpdate = 339, - SysDkimSignatureGet = 340, - SysDkimSignatureCreate = 341, - SysDkimSignatureUpdate = 342, - SysDkimSignatureDestroy = 343, - SysDkimSignatureQuery = 344, - SysDmarcExternalReportGet = 345, - SysDmarcExternalReportCreate = 346, - SysDmarcExternalReportUpdate = 347, - SysDmarcExternalReportDestroy = 348, - SysDmarcExternalReportQuery = 349, - SysDmarcInternalReportGet = 350, - SysDmarcInternalReportCreate = 351, - SysDmarcInternalReportUpdate = 352, - SysDmarcInternalReportDestroy = 353, - SysDmarcInternalReportQuery = 354, - SysDmarcReportSettingsGet = 355, - SysDmarcReportSettingsUpdate = 356, - SysDnsResolverGet = 357, - SysDnsResolverUpdate = 358, - SysDnsServerGet = 359, - SysDnsServerCreate = 360, - SysDnsServerUpdate = 361, - SysDnsServerDestroy = 362, - SysDnsServerQuery = 363, - SysDomainGet = 364, - SysDomainCreate = 365, - SysDomainUpdate = 366, - SysDomainDestroy = 367, - SysDomainQuery = 368, - SysDsnReportSettingsGet = 369, - SysDsnReportSettingsUpdate = 370, - SysEmailGet = 371, - SysEmailUpdate = 372, - SysEnterpriseGet = 373, - SysEnterpriseUpdate = 374, - SysEventTracingLevelGet = 375, - SysEventTracingLevelCreate = 376, - SysEventTracingLevelUpdate = 377, - SysEventTracingLevelDestroy = 378, - SysEventTracingLevelQuery = 379, - SysFileStorageGet = 380, - SysFileStorageUpdate = 381, - SysHttpGet = 382, - SysHttpUpdate = 383, - SysHttpFormGet = 384, - SysHttpFormUpdate = 385, - SysHttpLookupGet = 386, - SysHttpLookupCreate = 387, - SysHttpLookupUpdate = 388, - SysHttpLookupDestroy = 389, - SysHttpLookupQuery = 390, - SysImapGet = 391, - SysImapUpdate = 392, - SysInMemoryStoreGet = 393, - SysInMemoryStoreUpdate = 394, - SysJmapGet = 395, - SysJmapUpdate = 396, - SysLogGet = 397, - SysLogCreate = 398, - SysLogUpdate = 399, - SysLogDestroy = 400, - SysLogQuery = 401, - SysMailingListGet = 402, - SysMailingListCreate = 403, - SysMailingListUpdate = 404, - SysMailingListDestroy = 405, - SysMailingListQuery = 406, - SysMaskedEmailGet = 407, - SysMaskedEmailCreate = 408, - SysMaskedEmailUpdate = 409, - SysMaskedEmailDestroy = 410, - SysMaskedEmailQuery = 411, - SysMemoryLookupKeyGet = 412, - SysMemoryLookupKeyCreate = 413, - SysMemoryLookupKeyUpdate = 414, - SysMemoryLookupKeyDestroy = 415, - SysMemoryLookupKeyQuery = 416, - SysMemoryLookupKeyValueGet = 417, - SysMemoryLookupKeyValueCreate = 418, - SysMemoryLookupKeyValueUpdate = 419, - SysMemoryLookupKeyValueDestroy = 420, - SysMemoryLookupKeyValueQuery = 421, - SysMetricGet = 422, - SysMetricCreate = 423, - SysMetricUpdate = 424, - SysMetricDestroy = 425, - SysMetricQuery = 426, - SysMetricsGet = 427, - SysMetricsUpdate = 428, - SysMetricsStoreGet = 429, - SysMetricsStoreUpdate = 430, - SysMtaConnectionStrategyGet = 431, - SysMtaConnectionStrategyCreate = 432, - SysMtaConnectionStrategyUpdate = 433, - SysMtaConnectionStrategyDestroy = 434, - SysMtaConnectionStrategyQuery = 435, - SysMtaDeliveryScheduleGet = 436, - SysMtaDeliveryScheduleCreate = 437, - SysMtaDeliveryScheduleUpdate = 438, - SysMtaDeliveryScheduleDestroy = 439, - SysMtaDeliveryScheduleQuery = 440, - SysMtaExtensionsGet = 441, - SysMtaExtensionsUpdate = 442, - SysMtaHookGet = 443, - SysMtaHookCreate = 444, - SysMtaHookUpdate = 445, - SysMtaHookDestroy = 446, - SysMtaHookQuery = 447, - SysMtaInboundSessionGet = 448, - SysMtaInboundSessionUpdate = 449, - SysMtaInboundThrottleGet = 450, - SysMtaInboundThrottleCreate = 451, - SysMtaInboundThrottleUpdate = 452, - SysMtaInboundThrottleDestroy = 453, - SysMtaInboundThrottleQuery = 454, - SysMtaMilterGet = 455, - SysMtaMilterCreate = 456, - SysMtaMilterUpdate = 457, - SysMtaMilterDestroy = 458, - SysMtaMilterQuery = 459, - SysMtaOutboundStrategyGet = 460, - SysMtaOutboundStrategyUpdate = 461, - SysMtaOutboundThrottleGet = 462, - SysMtaOutboundThrottleCreate = 463, - SysMtaOutboundThrottleUpdate = 464, - SysMtaOutboundThrottleDestroy = 465, - SysMtaOutboundThrottleQuery = 466, - SysMtaQueueQuotaGet = 467, - SysMtaQueueQuotaCreate = 468, - SysMtaQueueQuotaUpdate = 469, - SysMtaQueueQuotaDestroy = 470, - SysMtaQueueQuotaQuery = 471, - SysMtaRouteGet = 472, - SysMtaRouteCreate = 473, - SysMtaRouteUpdate = 474, - SysMtaRouteDestroy = 475, - SysMtaRouteQuery = 476, - SysMtaStageAuthGet = 477, - SysMtaStageAuthUpdate = 478, - SysMtaStageConnectGet = 479, - SysMtaStageConnectUpdate = 480, - SysMtaStageDataGet = 481, - SysMtaStageDataUpdate = 482, - SysMtaStageEhloGet = 483, - SysMtaStageEhloUpdate = 484, - SysMtaStageMailGet = 485, - SysMtaStageMailUpdate = 486, - SysMtaStageRcptGet = 487, - SysMtaStageRcptUpdate = 488, - SysMtaStsGet = 489, - SysMtaStsUpdate = 490, - SysMtaTlsStrategyGet = 491, - SysMtaTlsStrategyCreate = 492, - SysMtaTlsStrategyUpdate = 493, - SysMtaTlsStrategyDestroy = 494, - SysMtaTlsStrategyQuery = 495, - SysMtaVirtualQueueGet = 496, - SysMtaVirtualQueueCreate = 497, - SysMtaVirtualQueueUpdate = 498, - SysMtaVirtualQueueDestroy = 499, - SysMtaVirtualQueueQuery = 500, - SysNetworkListenerGet = 501, - SysNetworkListenerCreate = 502, - SysNetworkListenerUpdate = 503, - SysNetworkListenerDestroy = 504, - SysNetworkListenerQuery = 505, - SysOAuthClientGet = 506, - SysOAuthClientCreate = 507, - SysOAuthClientUpdate = 508, - SysOAuthClientDestroy = 509, - SysOAuthClientQuery = 510, - SysOidcProviderGet = 511, - SysOidcProviderUpdate = 512, - SysPublicKeyGet = 513, - SysPublicKeyCreate = 514, - SysPublicKeyUpdate = 515, - SysPublicKeyDestroy = 516, - SysPublicKeyQuery = 517, - SysQueuedMessageGet = 518, - SysQueuedMessageCreate = 519, - SysQueuedMessageUpdate = 520, - SysQueuedMessageDestroy = 521, - SysQueuedMessageQuery = 522, - SysReportSettingsGet = 523, - SysReportSettingsUpdate = 524, - SysRoleGet = 525, - SysRoleCreate = 526, - SysRoleUpdate = 527, - SysRoleDestroy = 528, - SysRoleQuery = 529, - SysSearchGet = 530, - SysSearchUpdate = 531, - SysSearchStoreGet = 532, - SysSearchStoreUpdate = 533, - SysSecurityGet = 534, - SysSecurityUpdate = 535, - SysSenderAuthGet = 536, - SysSenderAuthUpdate = 537, - SysSharingGet = 538, - SysSharingUpdate = 539, - SysSieveSystemInterpreterGet = 540, - SysSieveSystemInterpreterUpdate = 541, - SysSieveSystemScriptGet = 542, - SysSieveSystemScriptCreate = 543, - SysSieveSystemScriptUpdate = 544, - SysSieveSystemScriptDestroy = 545, - SysSieveSystemScriptQuery = 546, - SysSieveUserInterpreterGet = 547, - SysSieveUserInterpreterUpdate = 548, - SysSieveUserScriptGet = 549, - SysSieveUserScriptCreate = 550, - SysSieveUserScriptUpdate = 551, - SysSieveUserScriptDestroy = 552, - SysSieveUserScriptQuery = 553, - SysSpamClassifierGet = 554, - SysSpamClassifierUpdate = 555, - SysSpamDnsblServerGet = 556, - SysSpamDnsblServerCreate = 557, - SysSpamDnsblServerUpdate = 558, - SysSpamDnsblServerDestroy = 559, - SysSpamDnsblServerQuery = 560, - SysSpamDnsblSettingsGet = 561, - SysSpamDnsblSettingsUpdate = 562, - SysSpamFileExtensionGet = 563, - SysSpamFileExtensionCreate = 564, - SysSpamFileExtensionUpdate = 565, - SysSpamFileExtensionDestroy = 566, - SysSpamFileExtensionQuery = 567, - SysSpamLlmGet = 568, - SysSpamLlmUpdate = 569, - SysSpamPyzorGet = 570, - SysSpamPyzorUpdate = 571, - SysSpamRuleGet = 572, - SysSpamRuleCreate = 573, - SysSpamRuleUpdate = 574, - SysSpamRuleDestroy = 575, - SysSpamRuleQuery = 576, - SysSpamSettingsGet = 577, - SysSpamSettingsUpdate = 578, - SysSpamTagGet = 579, - SysSpamTagCreate = 580, - SysSpamTagUpdate = 581, - SysSpamTagDestroy = 582, - SysSpamTagQuery = 583, - SysSpamTrainingSampleGet = 584, - SysSpamTrainingSampleCreate = 585, - SysSpamTrainingSampleUpdate = 586, - SysSpamTrainingSampleDestroy = 587, - SysSpamTrainingSampleQuery = 588, - SysSpfReportSettingsGet = 589, - SysSpfReportSettingsUpdate = 590, - SysStoreLookupGet = 591, - SysStoreLookupCreate = 592, - SysStoreLookupUpdate = 593, - SysStoreLookupDestroy = 594, - SysStoreLookupQuery = 595, - SysSystemSettingsGet = 596, - SysSystemSettingsUpdate = 597, - TaskIndexDocument = 598, - TaskUnindexDocument = 599, - TaskIndexTrace = 600, - TaskCalendarAlarmEmail = 601, - TaskCalendarAlarmNotification = 602, - TaskCalendarItipMessage = 603, - TaskMergeThreads = 604, - TaskDmarcReport = 605, - TaskTlsReport = 606, - TaskRestoreArchivedItem = 607, - TaskDestroyAccount = 608, - TaskAccountMaintenance = 609, - TaskTenantMaintenance = 610, - TaskStoreMaintenance = 611, - TaskSpamFilterMaintenance = 612, - TaskAcmeRenewal = 613, - TaskDkimManagement = 614, - TaskDnsManagement = 615, - SysTaskGet = 616, - SysTaskCreate = 617, - SysTaskUpdate = 618, - SysTaskDestroy = 619, - SysTaskQuery = 620, - SysTaskManagerGet = 621, - SysTaskManagerUpdate = 622, - SysTenantGet = 623, - SysTenantCreate = 624, - SysTenantUpdate = 625, - SysTenantDestroy = 626, - SysTenantQuery = 627, - SysTlsExternalReportGet = 628, - SysTlsExternalReportCreate = 629, - SysTlsExternalReportUpdate = 630, - SysTlsExternalReportDestroy = 631, - SysTlsExternalReportQuery = 632, - SysTlsInternalReportGet = 633, - SysTlsInternalReportCreate = 634, - SysTlsInternalReportUpdate = 635, - SysTlsInternalReportDestroy = 636, - SysTlsInternalReportQuery = 637, - SysTlsReportSettingsGet = 638, - SysTlsReportSettingsUpdate = 639, - SysTraceGet = 640, - SysTraceCreate = 641, - SysTraceUpdate = 642, - SysTraceDestroy = 643, - SysTraceQuery = 644, - SysTracerGet = 645, - SysTracerCreate = 646, - SysTracerUpdate = 647, - SysTracerDestroy = 648, - SysTracerQuery = 649, - SysTracingStoreGet = 650, - SysTracingStoreUpdate = 651, - SysWebDavGet = 652, - SysWebDavUpdate = 653, - SysWebHookGet = 654, - SysWebHookCreate = 655, - SysWebHookUpdate = 656, - SysWebHookDestroy = 657, - SysWebHookQuery = 658, + JmapQuotaGet = 63, + JmapQuotaChanges = 64, + JmapQuotaQuery = 65, + JmapQuotaQueryChanges = 66, + JmapBlobGet = 67, + JmapBlobCopy = 68, + JmapBlobLookup = 69, + JmapBlobUpload = 70, + JmapAddressBookGet = 71, + JmapAddressBookChanges = 72, + JmapAddressBookCreate = 73, + JmapAddressBookUpdate = 74, + JmapAddressBookDestroy = 75, + JmapContactCardGet = 76, + JmapContactCardChanges = 77, + JmapContactCardQuery = 78, + JmapContactCardQueryChanges = 79, + JmapContactCardCreate = 80, + JmapContactCardUpdate = 81, + JmapContactCardDestroy = 82, + JmapContactCardCopy = 83, + JmapContactCardParse = 84, + JmapFileNodeGet = 85, + JmapFileNodeChanges = 86, + JmapFileNodeQuery = 87, + JmapFileNodeQueryChanges = 88, + JmapFileNodeCreate = 89, + JmapFileNodeUpdate = 90, + JmapFileNodeDestroy = 91, + JmapFileNodeCopy = 92, + JmapShareNotificationGet = 93, + JmapShareNotificationChanges = 94, + JmapShareNotificationQuery = 95, + JmapShareNotificationQueryChanges = 96, + JmapShareNotificationCreate = 97, + JmapShareNotificationUpdate = 98, + JmapShareNotificationDestroy = 99, + JmapCalendarGet = 100, + JmapCalendarChanges = 101, + JmapCalendarCreate = 102, + JmapCalendarUpdate = 103, + JmapCalendarDestroy = 104, + JmapCalendarEventGet = 105, + JmapCalendarEventChanges = 106, + JmapCalendarEventQuery = 107, + JmapCalendarEventQueryChanges = 108, + JmapCalendarEventCreate = 109, + JmapCalendarEventUpdate = 110, + JmapCalendarEventDestroy = 111, + JmapCalendarEventCopy = 112, + JmapCalendarEventParse = 113, + JmapCalendarEventNotificationGet = 114, + JmapCalendarEventNotificationChanges = 115, + JmapCalendarEventNotificationQuery = 116, + JmapCalendarEventNotificationQueryChanges = 117, + JmapCalendarEventNotificationCreate = 118, + JmapCalendarEventNotificationUpdate = 119, + JmapCalendarEventNotificationDestroy = 120, + JmapParticipantIdentityGet = 121, + JmapParticipantIdentityChanges = 122, + JmapParticipantIdentityCreate = 123, + JmapParticipantIdentityUpdate = 124, + JmapParticipantIdentityDestroy = 125, + JmapCoreEcho = 126, + JmapMetadataGet = 127, + JmapMetadataSet = 128, + JmapMetadataPrivate = 129, + ImapAuthenticate = 130, + ImapAclGet = 131, + ImapAclSet = 132, + ImapMyRights = 133, + ImapListRights = 134, + ImapAppend = 135, + ImapCapability = 136, + ImapId = 137, + ImapCopy = 138, + ImapMove = 139, + ImapCreate = 140, + ImapDelete = 141, + ImapEnable = 142, + ImapExpunge = 143, + ImapFetch = 144, + ImapIdle = 145, + ImapList = 146, + ImapLsub = 147, + ImapNamespace = 148, + ImapRename = 149, + ImapSearch = 150, + ImapSort = 151, + ImapSelect = 152, + ImapExamine = 153, + ImapStatus = 154, + ImapStore = 155, + ImapSubscribe = 156, + ImapThread = 157, + ImapMetadataGet = 158, + ImapMetadataSet = 159, + ImapMetadataPrivate = 160, + Pop3Authenticate = 161, + Pop3List = 162, + Pop3Uidl = 163, + Pop3Stat = 164, + Pop3Retr = 165, + Pop3Dele = 166, + SieveAuthenticate = 167, + SieveListScripts = 168, + SieveSetActive = 169, + SieveGetScript = 170, + SievePutScript = 171, + SieveDeleteScript = 172, + SieveRenameScript = 173, + SieveCheckScript = 174, + SieveHaveSpace = 175, + DavSyncCollection = 176, + DavExpandProperty = 177, + DavPrincipalAcl = 178, + DavPrincipalList = 179, + DavPrincipalMatch = 180, + DavPrincipalSearch = 181, + DavPrincipalSearchPropSet = 182, + DavFilePropFind = 183, + DavFilePropPatch = 184, + DavFileGet = 185, + DavFileMkCol = 186, + DavFileDelete = 187, + DavFilePut = 188, + DavFileCopy = 189, + DavFileMove = 190, + DavFileLock = 191, + DavFileAcl = 192, + DavCardPropFind = 193, + DavCardPropPatch = 194, + DavCardGet = 195, + DavCardMkCol = 196, + DavCardDelete = 197, + DavCardPut = 198, + DavCardCopy = 199, + DavCardMove = 200, + DavCardLock = 201, + DavCardAcl = 202, + DavCardQuery = 203, + DavCardMultiGet = 204, + DavCalPropFind = 205, + DavCalPropPatch = 206, + DavCalGet = 207, + DavCalMkCol = 208, + DavCalDelete = 209, + DavCalPut = 210, + DavCalCopy = 211, + DavCalMove = 212, + DavCalLock = 213, + DavCalAcl = 214, + DavCalQuery = 215, + DavCalMultiGet = 216, + DavCalFreeBusyQuery = 217, + LiveDeliveryTest = 220, + SysAccountGet = 222, + SysAccountCreate = 223, + SysAccountUpdate = 224, + SysAccountDestroy = 225, + SysAccountQuery = 226, + SysAccountPasswordGet = 227, + SysAccountPasswordUpdate = 228, + SysAccountSettingsGet = 229, + SysAccountSettingsUpdate = 230, + SysAcmeProviderGet = 231, + SysAcmeProviderCreate = 232, + SysAcmeProviderUpdate = 233, + SysAcmeProviderDestroy = 234, + SysAcmeProviderQuery = 235, + ActionReloadSettings = 236, + ActionReloadTlsCertificates = 237, + ActionReloadLookupStores = 238, + ActionReloadBlockedIps = 239, + ActionUpdateApps = 240, + ActionTroubleshootDmarc = 241, + ActionClassifySpam = 242, + ActionInvalidateCaches = 243, + ActionInvalidateNegativeCaches = 244, + ActionPauseMtaQueue = 245, + ActionResumeMtaQueue = 246, + SysActionCreate = 247, + SysAddressBookGet = 248, + SysAddressBookUpdate = 249, + SysAllowedIpGet = 260, + SysAllowedIpCreate = 261, + SysAllowedIpUpdate = 262, + SysAllowedIpDestroy = 263, + SysAllowedIpQuery = 264, + SysApiKeyGet = 265, + SysApiKeyCreate = 266, + SysApiKeyUpdate = 267, + SysApiKeyDestroy = 268, + SysApiKeyQuery = 269, + SysAppPasswordGet = 270, + SysAppPasswordCreate = 271, + SysAppPasswordUpdate = 272, + SysAppPasswordDestroy = 273, + SysAppPasswordQuery = 274, + SysApplicationGet = 275, + SysApplicationCreate = 276, + SysApplicationUpdate = 277, + SysApplicationDestroy = 278, + SysApplicationQuery = 279, + SysArfExternalReportGet = 284, + SysArfExternalReportDestroy = 285, + SysArfExternalReportQuery = 286, + SysAsnGet = 287, + SysAsnUpdate = 288, + SysAuthenticationGet = 289, + SysAuthenticationUpdate = 290, + SysBlobStoreGet = 291, + SysBlobStoreUpdate = 292, + SysBlockedIpGet = 293, + SysBlockedIpCreate = 294, + SysBlockedIpUpdate = 295, + SysBlockedIpDestroy = 296, + SysBlockedIpQuery = 297, + SysBootstrapGet = 298, + SysBootstrapUpdate = 299, + SysCacheGet = 300, + SysCacheUpdate = 301, + SysCalendarGet = 302, + SysCalendarUpdate = 303, + SysCalendarAlarmGet = 304, + SysCalendarAlarmUpdate = 305, + SysCalendarSchedulingGet = 306, + SysCalendarSchedulingUpdate = 307, + SysCertificateGet = 308, + SysCertificateCreate = 309, + SysCertificateUpdate = 310, + SysCertificateDestroy = 311, + SysCertificateQuery = 312, + SysClusterNodeGet = 313, + SysClusterNodeQuery = 314, + SysClusterRoleGet = 315, + SysClusterRoleCreate = 316, + SysClusterRoleUpdate = 317, + SysClusterRoleDestroy = 318, + SysClusterRoleQuery = 319, + SysCoordinatorGet = 320, + SysCoordinatorUpdate = 321, + SysDataRetentionGet = 322, + SysDataRetentionUpdate = 323, + SysDataStoreGet = 324, + SysDataStoreUpdate = 325, + SysDirectoryGet = 326, + SysDirectoryCreate = 327, + SysDirectoryUpdate = 328, + SysDirectoryDestroy = 329, + SysDirectoryQuery = 330, + SysDkimReportSettingsGet = 331, + SysDkimReportSettingsUpdate = 332, + SysDkimSignatureGet = 333, + SysDkimSignatureCreate = 334, + SysDkimSignatureUpdate = 335, + SysDkimSignatureDestroy = 336, + SysDkimSignatureQuery = 337, + SysDmarcExternalReportGet = 338, + SysDmarcExternalReportDestroy = 339, + SysDmarcExternalReportQuery = 340, + SysDmarcInternalReportGet = 341, + SysDmarcInternalReportUpdate = 342, + SysDmarcInternalReportDestroy = 343, + SysDmarcInternalReportQuery = 344, + SysDmarcReportSettingsGet = 345, + SysDmarcReportSettingsUpdate = 346, + SysDnsResolverGet = 347, + SysDnsResolverUpdate = 348, + SysDnsServerGet = 349, + SysDnsServerCreate = 350, + SysDnsServerUpdate = 351, + SysDnsServerDestroy = 352, + SysDnsServerQuery = 353, + SysDomainGet = 354, + SysDomainCreate = 355, + SysDomainUpdate = 356, + SysDomainDestroy = 357, + SysDomainQuery = 358, + SysDsnReportSettingsGet = 359, + SysDsnReportSettingsUpdate = 360, + SysEmailGet = 361, + SysEmailUpdate = 362, + SysEventTracingLevelGet = 365, + SysEventTracingLevelCreate = 366, + SysEventTracingLevelUpdate = 367, + SysEventTracingLevelDestroy = 368, + SysEventTracingLevelQuery = 369, + SysFileStorageGet = 370, + SysFileStorageUpdate = 371, + SysHttpGet = 372, + SysHttpUpdate = 373, + SysHttpFormGet = 374, + SysHttpFormUpdate = 375, + SysHttpLookupGet = 376, + SysHttpLookupCreate = 377, + SysHttpLookupUpdate = 378, + SysHttpLookupDestroy = 379, + SysHttpLookupQuery = 380, + SysImapGet = 381, + SysImapUpdate = 382, + SysInMemoryStoreGet = 383, + SysInMemoryStoreUpdate = 384, + SysIndexQueueStatusGet = 385, + SysIndexQueueStatusUpdate = 386, + SysIndexQueueStatusQuery = 387, + SysJmapGet = 388, + SysJmapUpdate = 389, + SysLogGet = 390, + SysLogQuery = 391, + SysMailingListGet = 392, + SysMailingListCreate = 393, + SysMailingListUpdate = 394, + SysMailingListDestroy = 395, + SysMailingListQuery = 396, + SysMemoryLookupKeyGet = 402, + SysMemoryLookupKeyCreate = 403, + SysMemoryLookupKeyUpdate = 404, + SysMemoryLookupKeyDestroy = 405, + SysMemoryLookupKeyQuery = 406, + SysMemoryLookupKeyValueGet = 407, + SysMemoryLookupKeyValueCreate = 408, + SysMemoryLookupKeyValueUpdate = 409, + SysMemoryLookupKeyValueDestroy = 410, + SysMemoryLookupKeyValueQuery = 411, + SysMetadataGet = 412, + SysMetadataUpdate = 413, + SysMetricsGet = 416, + SysMetricsUpdate = 417, + SysMtaConnectionStrategyGet = 420, + SysMtaConnectionStrategyCreate = 421, + SysMtaConnectionStrategyUpdate = 422, + SysMtaConnectionStrategyDestroy = 423, + SysMtaConnectionStrategyQuery = 424, + SysMtaDeliveryScheduleGet = 425, + SysMtaDeliveryScheduleCreate = 426, + SysMtaDeliveryScheduleUpdate = 427, + SysMtaDeliveryScheduleDestroy = 428, + SysMtaDeliveryScheduleQuery = 429, + SysMtaExtensionsGet = 430, + SysMtaExtensionsUpdate = 431, + SysMtaHookGet = 432, + SysMtaHookCreate = 433, + SysMtaHookUpdate = 434, + SysMtaHookDestroy = 435, + SysMtaHookQuery = 436, + SysMtaInboundSessionGet = 437, + SysMtaInboundSessionUpdate = 438, + SysMtaInboundThrottleGet = 439, + SysMtaInboundThrottleCreate = 440, + SysMtaInboundThrottleUpdate = 441, + SysMtaInboundThrottleDestroy = 442, + SysMtaInboundThrottleQuery = 443, + SysMtaMilterGet = 444, + SysMtaMilterCreate = 445, + SysMtaMilterUpdate = 446, + SysMtaMilterDestroy = 447, + SysMtaMilterQuery = 448, + SysMtaOutboundStrategyGet = 449, + SysMtaOutboundStrategyUpdate = 450, + SysMtaOutboundThrottleGet = 451, + SysMtaOutboundThrottleCreate = 452, + SysMtaOutboundThrottleUpdate = 453, + SysMtaOutboundThrottleDestroy = 454, + SysMtaOutboundThrottleQuery = 455, + SysMtaQueueQuotaGet = 456, + SysMtaQueueQuotaCreate = 457, + SysMtaQueueQuotaUpdate = 458, + SysMtaQueueQuotaDestroy = 459, + SysMtaQueueQuotaQuery = 460, + SysMtaRouteGet = 461, + SysMtaRouteCreate = 462, + SysMtaRouteUpdate = 463, + SysMtaRouteDestroy = 464, + SysMtaRouteQuery = 465, + SysMtaStageAuthGet = 466, + SysMtaStageAuthUpdate = 467, + SysMtaStageConnectGet = 468, + SysMtaStageConnectUpdate = 469, + SysMtaStageDataGet = 470, + SysMtaStageDataUpdate = 471, + SysMtaStageEhloGet = 472, + SysMtaStageEhloUpdate = 473, + SysMtaStageMailGet = 474, + SysMtaStageMailUpdate = 475, + SysMtaStageRcptGet = 476, + SysMtaStageRcptUpdate = 477, + SysMtaStsGet = 478, + SysMtaStsUpdate = 479, + SysMtaTlsStrategyGet = 480, + SysMtaTlsStrategyCreate = 481, + SysMtaTlsStrategyUpdate = 482, + SysMtaTlsStrategyDestroy = 483, + SysMtaTlsStrategyQuery = 484, + SysMtaVirtualQueueGet = 485, + SysMtaVirtualQueueCreate = 486, + SysMtaVirtualQueueUpdate = 487, + SysMtaVirtualQueueDestroy = 488, + SysMtaVirtualQueueQuery = 489, + SysNetworkListenerGet = 490, + SysNetworkListenerCreate = 491, + SysNetworkListenerUpdate = 492, + SysNetworkListenerDestroy = 493, + SysNetworkListenerQuery = 494, + SysOAuthClientGet = 495, + SysOAuthClientCreate = 496, + SysOAuthClientUpdate = 497, + SysOAuthClientDestroy = 498, + SysOAuthClientQuery = 499, + SysOAuthGrantGet = 500, + SysOAuthGrantDestroy = 501, + SysOAuthGrantQuery = 502, + SysOidcProviderGet = 503, + SysOidcProviderUpdate = 504, + SysPublicKeyGet = 505, + SysPublicKeyCreate = 506, + SysPublicKeyUpdate = 507, + SysPublicKeyDestroy = 508, + SysPublicKeyQuery = 509, + SysQueuedMessageGet = 510, + SysQueuedMessageUpdate = 511, + SysQueuedMessageDestroy = 512, + SysQueuedMessageQuery = 513, + SysReportSettingsGet = 514, + SysReportSettingsUpdate = 515, + SysRoleGet = 516, + SysRoleCreate = 517, + SysRoleUpdate = 518, + SysRoleDestroy = 519, + SysRoleQuery = 520, + SysSearchGet = 521, + SysSearchUpdate = 522, + SysSearchStoreGet = 523, + SysSearchStoreUpdate = 524, + SysSecurityGet = 525, + SysSecurityUpdate = 526, + SysSenderAuthGet = 527, + SysSenderAuthUpdate = 528, + SysSharingGet = 529, + SysSharingUpdate = 530, + SysSieveSystemInterpreterGet = 531, + SysSieveSystemInterpreterUpdate = 532, + SysSieveSystemScriptGet = 533, + SysSieveSystemScriptCreate = 534, + SysSieveSystemScriptUpdate = 535, + SysSieveSystemScriptDestroy = 536, + SysSieveSystemScriptQuery = 537, + SysSieveUserInterpreterGet = 538, + SysSieveUserInterpreterUpdate = 539, + SysSieveUserScriptGet = 540, + SysSieveUserScriptCreate = 541, + SysSieveUserScriptUpdate = 542, + SysSieveUserScriptDestroy = 543, + SysSieveUserScriptQuery = 544, + SysSpamClassifierGet = 547, + SysSpamClassifierUpdate = 548, + SysSpamDnsblServerGet = 549, + SysSpamDnsblServerCreate = 550, + SysSpamDnsblServerUpdate = 551, + SysSpamDnsblServerDestroy = 552, + SysSpamDnsblServerQuery = 553, + SysSpamDnsblSettingsGet = 554, + SysSpamDnsblSettingsUpdate = 555, + SysSpamFileExtensionGet = 556, + SysSpamFileExtensionCreate = 557, + SysSpamFileExtensionUpdate = 558, + SysSpamFileExtensionDestroy = 559, + SysSpamFileExtensionQuery = 560, + SysSpamPyzorGet = 561, + SysSpamPyzorUpdate = 562, + SysSpamRuleGet = 563, + SysSpamRuleCreate = 564, + SysSpamRuleUpdate = 565, + SysSpamRuleDestroy = 566, + SysSpamRuleQuery = 567, + SysSpamSettingsGet = 568, + SysSpamSettingsUpdate = 569, + SysSpamTagGet = 570, + SysSpamTagCreate = 571, + SysSpamTagUpdate = 572, + SysSpamTagDestroy = 573, + SysSpamTagQuery = 574, + SysSpamTrainingSampleGet = 575, + SysSpamTrainingSampleCreate = 576, + SysSpamTrainingSampleDestroy = 577, + SysSpamTrainingSampleQuery = 578, + SysSpfReportSettingsGet = 579, + SysSpfReportSettingsUpdate = 580, + SysStoreLookupGet = 581, + SysStoreLookupCreate = 582, + SysStoreLookupUpdate = 583, + SysStoreLookupDestroy = 584, + SysStoreLookupQuery = 585, + SysSystemSettingsGet = 586, + SysSystemSettingsUpdate = 587, + TaskCalendarAlarmEmail = 588, + TaskCalendarAlarmNotification = 589, + TaskCalendarItipMessage = 590, + TaskMergeThreads = 591, + TaskDmarcReport = 592, + TaskTlsReport = 593, + TaskDestroyAccount = 595, + TaskAccountMaintenance = 596, + TaskStoreMaintenance = 598, + TaskSpamFilterMaintenance = 599, + TaskAcmeRenewal = 600, + TaskDkimManagement = 601, + TaskDnsManagement = 602, + TaskShareRevokeCleanup = 603, + TaskDmarcReportWindow = 604, + TaskTlsReportWindow = 605, + SysTaskGet = 606, + SysTaskCreate = 607, + SysTaskUpdate = 608, + SysTaskDestroy = 609, + SysTaskQuery = 610, + SysTaskManagerGet = 611, + SysTaskManagerUpdate = 612, + SysTlsExternalReportGet = 618, + SysTlsExternalReportDestroy = 619, + SysTlsExternalReportQuery = 620, + SysTlsInternalReportGet = 621, + SysTlsInternalReportUpdate = 622, + SysTlsInternalReportDestroy = 623, + SysTlsInternalReportQuery = 624, + SysTlsReportSettingsGet = 625, + SysTlsReportSettingsUpdate = 626, + SysTracerGet = 629, + SysTracerCreate = 630, + SysTracerUpdate = 631, + SysTracerDestroy = 632, + SysTracerQuery = 633, + SysWebDavGet = 636, + SysWebDavUpdate = 637, + SysWebHookGet = 638, + SysWebHookCreate = 639, + SysWebHookUpdate = 640, + SysWebHookDestroy = 641, + SysWebHookQuery = 642, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2264,6 +2248,16 @@ pub enum RedisProtocol { Resp3 = 1, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum RedisSwapStoreType { + #[default] + Default = 0, + Redis = 1, + RedisCluster = 2, + RedisSentinel = 3, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum RolesType { @@ -2330,8 +2324,6 @@ pub enum SearchCalendarField { Location = 2, Owner = 3, Attendee = 4, - Start = 5, - Uid = 6, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2348,7 +2340,6 @@ pub enum SearchContactField { OnlineService = 7, Address = 8, Note = 9, - Uid = 10, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2362,11 +2353,7 @@ pub enum SearchEmailField { Subject = 4, Body = 5, Attachment = 6, - ReceivedAt = 7, - SentAt = 8, - Size = 9, - HasAttachment = 10, - Headers = 11, + Headers = 7, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2389,15 +2376,6 @@ pub enum SearchStoreType { MySql = 5, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum SearchTracingField { - #[default] - EventType = 0, - QueueId = 1, - Keywords = 2, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SecretKeyOptionalType { @@ -2450,6 +2428,16 @@ pub enum ServiceProtocol { Managesieve = 7, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum SharedObjectType { + #[default] + Mailbox = 0, + Calendar = 1, + AddressBook = 2, + FileNode = 3, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SieveCapability { @@ -2565,14 +2553,6 @@ pub enum SpamDnsblServerType { Body = 6, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum SpamLlmType { - #[default] - Disable = 0, - Enable = 1, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SpamRuleType { @@ -2662,7 +2642,6 @@ pub enum StorageQuota { MaxContactCards = 11, MaxFiles = 12, MaxFolders = 13, - MaxMaskedAddresses = 14, MaxAppPasswords = 15, MaxApiKeys = 16, MaxPublicKeys = 17, @@ -2686,6 +2665,7 @@ pub enum TaskAccountMaintenanceType { Reindex = 1, RecalculateImapUid = 2, RecalculateQuota = 3, + RebuildShares = 4, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2714,6 +2694,7 @@ pub enum TaskStatusType { Pending = 0, Retry = 1, Failed = 2, + Stalled = 3, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -2721,68 +2702,39 @@ pub enum TaskStatusType { pub enum TaskStoreMaintenanceType { #[default] ReindexAccounts = 0, - ReindexTelemetry = 1, PurgeAccounts = 2, PurgeData = 3, PurgeBlob = 4, ResetRateLimiters = 5, ResetUserQuotas = 6, - ResetTenantQuotas = 7, ResetBlobQuotas = 8, RemoveAuthTokens = 9, RemoveLockQueueMessage = 10, RemoveLockTask = 11, RemoveLockDav = 12, - RemoveSieveId = 13, - RemoveGreylist = 14, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum TaskTenantMaintenanceType { - #[default] - RecalculateQuota = 0, + RemoveGreylist = 13, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TaskType { #[default] - IndexDocument = 0, - UnindexDocument = 1, - IndexTrace = 2, - CalendarAlarmEmail = 3, - CalendarAlarmNotification = 4, - CalendarItipMessage = 5, - MergeThreads = 6, - DmarcReport = 7, - TlsReport = 8, - RestoreArchivedItem = 9, - DestroyAccount = 10, - AccountMaintenance = 11, - TenantMaintenance = 12, - StoreMaintenance = 13, - SpamFilterMaintenance = 14, - AcmeRenewal = 15, - DkimManagement = 16, - DnsManagement = 17, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum TenantStorageQuota { - #[default] - MaxAccounts = 0, - MaxGroups = 1, - MaxDomains = 2, - MaxMailingLists = 3, - MaxRoles = 4, - MaxOauthClients = 5, - MaxDkimKeys = 6, - MaxDnsServers = 7, - MaxDirectories = 8, - MaxAcmeProviders = 9, - MaxDiskQuota = 10, + CalendarAlarmEmail = 0, + CalendarAlarmNotification = 1, + CalendarItipMessage = 2, + MergeThreads = 3, + DmarcReport = 4, + TlsReport = 5, + DestroyAccount = 7, + AccountMaintenance = 8, + StoreMaintenance = 10, + SpamFilterMaintenance = 11, + AcmeRenewal = 12, + DkimManagement = 13, + DnsManagement = 14, + ShareRevokeCleanup = 15, + DmarcReportWindow = 16, + TlsReportWindow = 17, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] @@ -3440,23 +3392,6 @@ pub enum TlsVersion { Tls13 = 1, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum TraceValueType { - #[default] - String = 0, - UnsignedInt = 1, - Integer = 2, - Boolean = 3, - Float = 4, - UTCDateTime = 5, - Duration = 6, - IpAddr = 7, - List = 8, - Event = 9, - Null = 10, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TracerType { @@ -3491,17 +3426,6 @@ pub enum TracingLevelOpt { Trace = 5, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum TracingStoreType { - #[default] - Disabled = 0, - Default = 1, - FoundationDb = 2, - PostgreSql = 3, - MySql = 4, -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TsigAlgorithm { @@ -3636,11 +3560,7 @@ pub static MTA_QUEUE_SENDER_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::Sender, ExpressionVariable::SenderDomain, ExpressionVariable::Priority, - ExpressionVariable::RetryNum, - ExpressionVariable::NotifyNum, - ExpressionVariable::ExpiresIn, - ExpressionVariable::LastStatus, - ExpressionVariable::LastError, + ExpressionVariable::Recipients, ]; pub static MTA_RCPT_DOMAIN_VARIABLE: &[ExpressionVariable] = &[ExpressionVariable::RcptDomain]; @@ -3667,6 +3587,52 @@ pub static MTA_RCPT_TO_VARIABLE: &[ExpressionVariable] = &[ pub static MTA_RCPT_VARIABLE: &[ExpressionVariable] = &[ExpressionVariable::Rcpt]; +pub static SPAM_AI_VARIABLE: &[ExpressionVariable] = &[ + ExpressionVariable::SpamScore, + ExpressionVariable::SpamThreshold, + ExpressionVariable::RemoteIp, + ExpressionVariable::RemoteIpPtr, + ExpressionVariable::HeloDomain, + ExpressionVariable::AuthenticatedAs, + ExpressionVariable::Asn, + ExpressionVariable::Country, + ExpressionVariable::IsTls, + ExpressionVariable::EnvFrom, + ExpressionVariable::EnvFromLocal, + ExpressionVariable::EnvFromDomain, + ExpressionVariable::EnvTo, + ExpressionVariable::From, + ExpressionVariable::FromName, + ExpressionVariable::FromLocal, + ExpressionVariable::FromDomain, + ExpressionVariable::ReplyTo, + ExpressionVariable::ReplyToName, + ExpressionVariable::ReplyToLocal, + ExpressionVariable::ReplyToDomain, + ExpressionVariable::To, + ExpressionVariable::ToName, + ExpressionVariable::ToLocal, + ExpressionVariable::ToDomain, + ExpressionVariable::Cc, + ExpressionVariable::CcName, + ExpressionVariable::CcLocal, + ExpressionVariable::CcDomain, + ExpressionVariable::Bcc, + ExpressionVariable::BccName, + ExpressionVariable::BccLocal, + ExpressionVariable::BccDomain, + ExpressionVariable::Body, + ExpressionVariable::BodyText, + ExpressionVariable::BodyHtml, + ExpressionVariable::BodyWords, + ExpressionVariable::BodyRaw, + ExpressionVariable::Subject, + ExpressionVariable::SubjectThread, + ExpressionVariable::SubjectWords, + ExpressionVariable::Location, + ExpressionVariable::SpamProbability, +]; + pub static SPAM_DEFAULT_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::RemoteIp, ExpressionVariable::RemoteIpPtr, @@ -3708,6 +3674,7 @@ pub static SPAM_DEFAULT_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static SPAM_EMAIL_VARIABLE: &[ExpressionVariable] = &[ @@ -3757,6 +3724,7 @@ pub static SPAM_EMAIL_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static SPAM_GENERIC_VARIABLE: &[ExpressionVariable] = &[ @@ -3801,6 +3769,7 @@ pub static SPAM_GENERIC_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static SPAM_HEADER_VARIABLE: &[ExpressionVariable] = &[ @@ -3851,12 +3820,12 @@ pub static SPAM_HEADER_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static SPAM_IP_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::Ip, ExpressionVariable::Value, - ExpressionVariable::ReverseIp, ExpressionVariable::IpReverse, ExpressionVariable::Octets, ExpressionVariable::IsV4, @@ -3901,6 +3870,7 @@ pub static SPAM_IP_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static SPAM_URL_VARIABLE: &[ExpressionVariable] = &[ @@ -3955,6 +3925,7 @@ pub static SPAM_URL_VARIABLE: &[ExpressionVariable] = &[ ExpressionVariable::SubjectThread, ExpressionVariable::SubjectWords, ExpressionVariable::Location, + ExpressionVariable::SpamProbability, ]; pub static MTA_AGGREGATE_CONSTANT: &[ExpressionConstant] = &[ @@ -3971,25 +3942,12 @@ pub static MTA_AUTH_TYPE_CONSTANT: &[ExpressionConstant] = &[ ExpressionConstant::Oauthbearer, ]; -pub static MTA_IP_STRATEGY_CONSTANT: &[ExpressionConstant] = &[ - ExpressionConstant::Ipv4Only, - ExpressionConstant::Ipv6Only, - ExpressionConstant::Ipv6ThenIpv4, - ExpressionConstant::Ipv4ThenIpv6, -]; - pub static MTA_PRIORITY_CONSTANT: &[ExpressionConstant] = &[ ExpressionConstant::Mixer, ExpressionConstant::Stanag4406, ExpressionConstant::Nsep, ]; -pub static MTA_REQUIRE_CONSTANT: &[ExpressionConstant] = &[ - ExpressionConstant::Optional, - ExpressionConstant::Require, - ExpressionConstant::Disable, -]; - pub static MTA_VERIFY_CONSTANT: &[ExpressionConstant] = &[ ExpressionConstant::Relaxed, ExpressionConstant::Strict, diff --git a/crates/registry/src/schema/enums_impl.rs b/crates/registry/src/schema/enums_impl.rs index 8dcc752f..ee62dea3 100644 --- a/crates/registry/src/schema/enums_impl.rs +++ b/crates/registry/src/schema/enums_impl.rs @@ -10,11 +10,13 @@ use crate::schema::prelude::*; impl EnumImpl for AccountType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + AccountType, b"User" => AccountType::User, b"Group" => AccountType::Group, } + .copied() } fn as_str(&self) -> &'static str { @@ -60,13 +62,15 @@ impl<'de> serde::Deserialize<'de> for AccountType { impl EnumImpl for AcmeChallengeType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + AcmeChallengeType, b"TlsAlpn01" => AcmeChallengeType::TlsAlpn01, b"DnsPersist01" => AcmeChallengeType::DnsPersist01, b"Dns01" => AcmeChallengeType::Dns01, b"Http01" => AcmeChallengeType::Http01, } + .copied() } fn as_str(&self) -> &'static str { @@ -116,13 +120,15 @@ impl<'de> serde::Deserialize<'de> for AcmeChallengeType { impl EnumImpl for AcmeRenewBefore { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + AcmeRenewBefore, b"R12" => AcmeRenewBefore::R12, b"R23" => AcmeRenewBefore::R23, b"R34" => AcmeRenewBefore::R34, b"R45" => AcmeRenewBefore::R45, } + .copied() } fn as_str(&self) -> &'static str { @@ -172,8 +178,9 @@ impl<'de> serde::Deserialize<'de> for AcmeRenewBefore { impl EnumImpl for ActionType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ActionType, b"ReloadSettings" => ActionType::ReloadSettings, b"ReloadTlsCertificates" => ActionType::ReloadTlsCertificates, b"ReloadLookupStores" => ActionType::ReloadLookupStores, @@ -186,6 +193,7 @@ impl EnumImpl for ActionType { b"PauseMtaQueue" => ActionType::PauseMtaQueue, b"ResumeMtaQueue" => ActionType::ResumeMtaQueue, } + .copied() } fn as_str(&self) -> &'static str { @@ -247,269 +255,11 @@ impl<'de> serde::Deserialize<'de> for ActionType { } } -impl EnumImpl for AiModelType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Chat" => AiModelType::Chat, - b"Text" => AiModelType::Text, - } - } - - fn as_str(&self) -> &'static str { - match self { - AiModelType::Chat => "Chat", - AiModelType::Text => "Text", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(AiModelType::Chat), - 1 => Some(AiModelType::Text), - _ => None, - } - } - - const COUNT: usize = 2; -} - -impl serde::Serialize for AiModelType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for AiModelType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - -impl EnumImpl for AlertEmailType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Disabled" => AlertEmailType::Disabled, - b"Enabled" => AlertEmailType::Enabled, - } - } - - fn as_str(&self) -> &'static str { - match self { - AlertEmailType::Disabled => "Disabled", - AlertEmailType::Enabled => "Enabled", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(AlertEmailType::Disabled), - 1 => Some(AlertEmailType::Enabled), - _ => None, - } - } - - const COUNT: usize = 2; -} - -impl serde::Serialize for AlertEmailType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for AlertEmailType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - -impl EnumImpl for AlertEventType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Disabled" => AlertEventType::Disabled, - b"Enabled" => AlertEventType::Enabled, - } - } - - fn as_str(&self) -> &'static str { - match self { - AlertEventType::Disabled => "Disabled", - AlertEventType::Enabled => "Enabled", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(AlertEventType::Disabled), - 1 => Some(AlertEventType::Enabled), - _ => None, - } - } - - const COUNT: usize = 2; -} - -impl serde::Serialize for AlertEventType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for AlertEventType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - -impl EnumImpl for ArchivedItemStatus { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"archived" => ArchivedItemStatus::Archived, - b"requestRestore" => ArchivedItemStatus::RequestRestore, - } - } - - fn as_str(&self) -> &'static str { - match self { - ArchivedItemStatus::Archived => "archived", - ArchivedItemStatus::RequestRestore => "requestRestore", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(ArchivedItemStatus::Archived), - 1 => Some(ArchivedItemStatus::RequestRestore), - _ => None, - } - } - - const COUNT: usize = 2; -} - -impl serde::Serialize for ArchivedItemStatus { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for ArchivedItemStatus { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - -impl EnumImpl for ArchivedItemType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Email" => ArchivedItemType::Email, - b"FileNode" => ArchivedItemType::FileNode, - b"CalendarEvent" => ArchivedItemType::CalendarEvent, - b"ContactCard" => ArchivedItemType::ContactCard, - b"SieveScript" => ArchivedItemType::SieveScript, - } - } - - fn as_str(&self) -> &'static str { - match self { - ArchivedItemType::Email => "Email", - ArchivedItemType::FileNode => "FileNode", - ArchivedItemType::CalendarEvent => "CalendarEvent", - ArchivedItemType::ContactCard => "ContactCard", - ArchivedItemType::SieveScript => "SieveScript", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(ArchivedItemType::Email), - 1 => Some(ArchivedItemType::FileNode), - 2 => Some(ArchivedItemType::CalendarEvent), - 3 => Some(ArchivedItemType::ContactCard), - 4 => Some(ArchivedItemType::SieveScript), - _ => None, - } - } - - const COUNT: usize = 5; -} - -impl serde::Serialize for ArchivedItemType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for ArchivedItemType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for ArfAuthFailureType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ArfAuthFailureType, b"adsp" => ArfAuthFailureType::Adsp, b"bodyHash" => ArfAuthFailureType::BodyHash, b"revoked" => ArfAuthFailureType::Revoked, @@ -518,6 +268,7 @@ impl EnumImpl for ArfAuthFailureType { b"dmarc" => ArfAuthFailureType::Dmarc, b"unspecified" => ArfAuthFailureType::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -573,8 +324,9 @@ impl<'de> serde::Deserialize<'de> for ArfAuthFailureType { impl EnumImpl for ArfDeliveryResult { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ArfDeliveryResult, b"delivered" => ArfDeliveryResult::Delivered, b"spam" => ArfDeliveryResult::Spam, b"policy" => ArfDeliveryResult::Policy, @@ -582,6 +334,7 @@ impl EnumImpl for ArfDeliveryResult { b"other" => ArfDeliveryResult::Other, b"unspecified" => ArfDeliveryResult::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -635,8 +388,9 @@ impl<'de> serde::Deserialize<'de> for ArfDeliveryResult { impl EnumImpl for ArfFeedbackType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ArfFeedbackType, b"abuse" => ArfFeedbackType::Abuse, b"authFailure" => ArfFeedbackType::AuthFailure, b"fraud" => ArfFeedbackType::Fraud, @@ -644,6 +398,7 @@ impl EnumImpl for ArfFeedbackType { b"virus" => ArfFeedbackType::Virus, b"other" => ArfFeedbackType::Other, } + .copied() } fn as_str(&self) -> &'static str { @@ -697,14 +452,16 @@ impl<'de> serde::Deserialize<'de> for ArfFeedbackType { impl EnumImpl for ArfIdentityAlignment { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ArfIdentityAlignment, b"none" => ArfIdentityAlignment::None, b"spf" => ArfIdentityAlignment::Spf, b"dkim" => ArfIdentityAlignment::Dkim, b"dkimSpf" => ArfIdentityAlignment::DkimSpf, b"unspecified" => ArfIdentityAlignment::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -756,12 +513,14 @@ impl<'de> serde::Deserialize<'de> for ArfIdentityAlignment { impl EnumImpl for AsnType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + AsnType, b"Disabled" => AsnType::Disabled, b"Resource" => AsnType::Resource, b"Dns" => AsnType::Dns, } + .copied() } fn as_str(&self) -> &'static str { @@ -807,14 +566,77 @@ impl<'de> serde::Deserialize<'de> for AsnType { } } +impl EnumImpl for AzureAuthType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + AzureAuthType, + b"SharedKey" => AzureAuthType::SharedKey, + b"SasToken" => AzureAuthType::SasToken, + b"ClientSecret" => AzureAuthType::ClientSecret, + b"ManagedIdentity" => AzureAuthType::ManagedIdentity, + b"WorkloadIdentity" => AzureAuthType::WorkloadIdentity, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + AzureAuthType::SharedKey => "SharedKey", + AzureAuthType::SasToken => "SasToken", + AzureAuthType::ClientSecret => "ClientSecret", + AzureAuthType::ManagedIdentity => "ManagedIdentity", + AzureAuthType::WorkloadIdentity => "WorkloadIdentity", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(AzureAuthType::SharedKey), + 1 => Some(AzureAuthType::SasToken), + 2 => Some(AzureAuthType::ClientSecret), + 3 => Some(AzureAuthType::ManagedIdentity), + 4 => Some(AzureAuthType::WorkloadIdentity), + _ => None, + } + } + + const COUNT: usize = 5; +} + +impl serde::Serialize for AzureAuthType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for AzureAuthType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for AzureEnvironment { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + AzureEnvironment, b"public" => AzureEnvironment::Public, b"china" => AzureEnvironment::China, b"us-government" => AzureEnvironment::UsGovernment, } + .copied() } fn as_str(&self) -> &'static str { @@ -860,74 +682,13 @@ impl<'de> serde::Deserialize<'de> for AzureEnvironment { } } -impl EnumImpl for BlobStoreBaseType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"S3" => BlobStoreBaseType::S3, - b"Azure" => BlobStoreBaseType::Azure, - b"FileSystem" => BlobStoreBaseType::FileSystem, - b"FoundationDb" => BlobStoreBaseType::FoundationDb, - b"PostgreSql" => BlobStoreBaseType::PostgreSql, - b"MySql" => BlobStoreBaseType::MySql, - } - } - - fn as_str(&self) -> &'static str { - match self { - BlobStoreBaseType::S3 => "S3", - BlobStoreBaseType::Azure => "Azure", - BlobStoreBaseType::FileSystem => "FileSystem", - BlobStoreBaseType::FoundationDb => "FoundationDb", - BlobStoreBaseType::PostgreSql => "PostgreSql", - BlobStoreBaseType::MySql => "MySql", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(BlobStoreBaseType::S3), - 1 => Some(BlobStoreBaseType::Azure), - 2 => Some(BlobStoreBaseType::FileSystem), - 3 => Some(BlobStoreBaseType::FoundationDb), - 4 => Some(BlobStoreBaseType::PostgreSql), - 5 => Some(BlobStoreBaseType::MySql), - _ => None, - } - } - - const COUNT: usize = 6; -} - -impl serde::Serialize for BlobStoreBaseType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for BlobStoreBaseType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for BlobStoreType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + BlobStoreType, b"Default" => BlobStoreType::Default, - b"Sharded" => BlobStoreType::Sharded, b"S3" => BlobStoreType::S3, b"Azure" => BlobStoreType::Azure, b"FileSystem" => BlobStoreType::FileSystem, @@ -935,12 +696,12 @@ impl EnumImpl for BlobStoreType { b"PostgreSql" => BlobStoreType::PostgreSql, b"MySql" => BlobStoreType::MySql, } + .copied() } fn as_str(&self) -> &'static str { match self { BlobStoreType::Default => "Default", - BlobStoreType::Sharded => "Sharded", BlobStoreType::S3 => "S3", BlobStoreType::Azure => "Azure", BlobStoreType::FileSystem => "FileSystem", @@ -957,7 +718,6 @@ impl EnumImpl for BlobStoreType { fn from_id(id: u16) -> Option { match id { 0 => Some(BlobStoreType::Default), - 1 => Some(BlobStoreType::Sharded), 2 => Some(BlobStoreType::S3), 3 => Some(BlobStoreType::Azure), 4 => Some(BlobStoreType::FileSystem), @@ -990,10 +750,78 @@ impl<'de> serde::Deserialize<'de> for BlobStoreType { } } +impl EnumImpl for BlobSwapStoreType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + BlobSwapStoreType, + b"Default" => BlobSwapStoreType::Default, + b"S3" => BlobSwapStoreType::S3, + b"Azure" => BlobSwapStoreType::Azure, + b"FileSystem" => BlobSwapStoreType::FileSystem, + b"FoundationDb" => BlobSwapStoreType::FoundationDb, + b"PostgreSql" => BlobSwapStoreType::PostgreSql, + b"MySql" => BlobSwapStoreType::MySql, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + BlobSwapStoreType::Default => "Default", + BlobSwapStoreType::S3 => "S3", + BlobSwapStoreType::Azure => "Azure", + BlobSwapStoreType::FileSystem => "FileSystem", + BlobSwapStoreType::FoundationDb => "FoundationDb", + BlobSwapStoreType::PostgreSql => "PostgreSql", + BlobSwapStoreType::MySql => "MySql", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(BlobSwapStoreType::Default), + 1 => Some(BlobSwapStoreType::S3), + 2 => Some(BlobSwapStoreType::Azure), + 3 => Some(BlobSwapStoreType::FileSystem), + 4 => Some(BlobSwapStoreType::FoundationDb), + 5 => Some(BlobSwapStoreType::PostgreSql), + 6 => Some(BlobSwapStoreType::MySql), + _ => None, + } + } + + const COUNT: usize = 7; +} + +impl serde::Serialize for BlobSwapStoreType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for BlobSwapStoreType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for BlockReason { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + BlockReason, b"rcptToFailure" => BlockReason::RcptToFailure, b"authFailure" => BlockReason::AuthFailure, b"loitering" => BlockReason::Loitering, @@ -1001,6 +829,7 @@ impl EnumImpl for BlockReason { b"manual" => BlockReason::Manual, b"other" => BlockReason::Other, } + .copied() } fn as_str(&self) -> &'static str { @@ -1052,13 +881,73 @@ impl<'de> serde::Deserialize<'de> for BlockReason { } } +impl EnumImpl for CacheSwapType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + CacheSwapType, + b"Disabled" => CacheSwapType::Disabled, + b"LocalFile" => CacheSwapType::LocalFile, + b"Redis" => CacheSwapType::Redis, + b"BlobStore" => CacheSwapType::BlobStore, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + CacheSwapType::Disabled => "Disabled", + CacheSwapType::LocalFile => "LocalFile", + CacheSwapType::Redis => "Redis", + CacheSwapType::BlobStore => "BlobStore", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(CacheSwapType::Disabled), + 1 => Some(CacheSwapType::LocalFile), + 2 => Some(CacheSwapType::Redis), + 3 => Some(CacheSwapType::BlobStore), + _ => None, + } + } + + const COUNT: usize = 4; +} + +impl serde::Serialize for CacheSwapType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for CacheSwapType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for CertificateManagementType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CertificateManagementType, b"Manual" => CertificateManagementType::Manual, b"Automatic" => CertificateManagementType::Automatic, } + .copied() } fn as_str(&self) -> &'static str { @@ -1102,15 +991,72 @@ impl<'de> serde::Deserialize<'de> for CertificateManagementType { } } +impl EnumImpl for ClientIpSource { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + ClientIpSource, + b"peer" => ClientIpSource::Peer, + b"xForwardedFor" => ClientIpSource::XForwardedFor, + b"forwarded" => ClientIpSource::Forwarded, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + ClientIpSource::Peer => "peer", + ClientIpSource::XForwardedFor => "xForwardedFor", + ClientIpSource::Forwarded => "forwarded", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(ClientIpSource::Peer), + 1 => Some(ClientIpSource::XForwardedFor), + 2 => Some(ClientIpSource::Forwarded), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for ClientIpSource { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for ClientIpSource { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for ClusterListenerGroupType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ClusterListenerGroupType, b"EnableAll" => ClusterListenerGroupType::EnableAll, b"DisableAll" => ClusterListenerGroupType::DisableAll, b"EnableSome" => ClusterListenerGroupType::EnableSome, b"DisableSome" => ClusterListenerGroupType::DisableSome, } + .copied() } fn as_str(&self) -> &'static str { @@ -1160,12 +1106,14 @@ impl<'de> serde::Deserialize<'de> for ClusterListenerGroupType { impl EnumImpl for ClusterNodeStatus { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ClusterNodeStatus, b"active" => ClusterNodeStatus::Active, b"stale" => ClusterNodeStatus::Stale, b"inactive" => ClusterNodeStatus::Inactive, } + .copied() } fn as_str(&self) -> &'static str { @@ -1213,13 +1161,15 @@ impl<'de> serde::Deserialize<'de> for ClusterNodeStatus { impl EnumImpl for ClusterTaskGroupType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ClusterTaskGroupType, b"EnableAll" => ClusterTaskGroupType::EnableAll, b"DisableAll" => ClusterTaskGroupType::DisableAll, b"EnableSome" => ClusterTaskGroupType::EnableSome, b"DisableSome" => ClusterTaskGroupType::DisableSome, } + .copied() } fn as_str(&self) -> &'static str { @@ -1269,8 +1219,9 @@ impl<'de> serde::Deserialize<'de> for ClusterTaskGroupType { impl EnumImpl for ClusterTaskType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ClusterTaskType, b"storeMaintenance" => ClusterTaskType::StoreMaintenance, b"accountMaintenance" => ClusterTaskType::AccountMaintenance, b"metricsCalculate" => ClusterTaskType::MetricsCalculate, @@ -1282,6 +1233,7 @@ impl EnumImpl for ClusterTaskType { b"taskQueueProcessing" => ClusterTaskType::TaskQueueProcessing, b"taskScheduler" => ClusterTaskType::TaskScheduler, } + .copied() } fn as_str(&self) -> &'static str { @@ -1343,11 +1295,13 @@ impl<'de> serde::Deserialize<'de> for ClusterTaskType { impl EnumImpl for CompressionAlgo { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CompressionAlgo, b"lz4" => CompressionAlgo::Lz4, b"none" => CompressionAlgo::None, } + .copied() } fn as_str(&self) -> &'static str { @@ -1393,8 +1347,9 @@ impl<'de> serde::Deserialize<'de> for CompressionAlgo { impl EnumImpl for CoordinatorType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CoordinatorType, b"Disabled" => CoordinatorType::Disabled, b"Default" => CoordinatorType::Default, b"Kafka" => CoordinatorType::Kafka, @@ -1404,6 +1359,7 @@ impl EnumImpl for CoordinatorType { b"RedisCluster" => CoordinatorType::RedisCluster, b"RedisSentinel" => CoordinatorType::RedisSentinel, } + .copied() } fn as_str(&self) -> &'static str { @@ -1461,12 +1417,14 @@ impl<'de> serde::Deserialize<'de> for CoordinatorType { impl EnumImpl for CredentialPermissionsType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CredentialPermissionsType, b"Inherit" => CredentialPermissionsType::Inherit, b"Disable" => CredentialPermissionsType::Disable, b"Replace" => CredentialPermissionsType::Replace, } + .copied() } fn as_str(&self) -> &'static str { @@ -1514,12 +1472,14 @@ impl<'de> serde::Deserialize<'de> for CredentialPermissionsType { impl EnumImpl for CredentialType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CredentialType, b"Password" => CredentialType::Password, b"AppPassword" => CredentialType::AppPassword, b"ApiKey" => CredentialType::ApiKey, } + .copied() } fn as_str(&self) -> &'static str { @@ -1567,12 +1527,14 @@ impl<'de> serde::Deserialize<'de> for CredentialType { impl EnumImpl for CronType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + CronType, b"Daily" => CronType::Daily, b"Weekly" => CronType::Weekly, b"Hourly" => CronType::Hourly, } + .copied() } fn as_str(&self) -> &'static str { @@ -1620,14 +1582,16 @@ impl<'de> serde::Deserialize<'de> for CronType { impl EnumImpl for DataStoreType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DataStoreType, b"RocksDb" => DataStoreType::RocksDb, b"Sqlite" => DataStoreType::Sqlite, b"FoundationDb" => DataStoreType::FoundationDb, b"PostgreSql" => DataStoreType::PostgreSql, b"MySql" => DataStoreType::MySql, } + .copied() } fn as_str(&self) -> &'static str { @@ -1679,8 +1643,9 @@ impl<'de> serde::Deserialize<'de> for DataStoreType { impl EnumImpl for DeliveryErrorType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DeliveryErrorType, b"dnsError" => DeliveryErrorType::DnsError, b"unexpectedResponse" => DeliveryErrorType::UnexpectedResponse, b"connectionError" => DeliveryErrorType::ConnectionError, @@ -1691,6 +1656,7 @@ impl EnumImpl for DeliveryErrorType { b"concurrencyLimited" => DeliveryErrorType::ConcurrencyLimited, b"io" => DeliveryErrorType::Io, } + .copied() } fn as_str(&self) -> &'static str { @@ -1750,13 +1716,15 @@ impl<'de> serde::Deserialize<'de> for DeliveryErrorType { impl EnumImpl for DirectoryBootstrapType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DirectoryBootstrapType, b"Internal" => DirectoryBootstrapType::Internal, b"Ldap" => DirectoryBootstrapType::Ldap, b"Sql" => DirectoryBootstrapType::Sql, b"Oidc" => DirectoryBootstrapType::Oidc, } + .copied() } fn as_str(&self) -> &'static str { @@ -1806,12 +1774,14 @@ impl<'de> serde::Deserialize<'de> for DirectoryBootstrapType { impl EnumImpl for DirectoryType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DirectoryType, b"Ldap" => DirectoryType::Ldap, b"Sql" => DirectoryType::Sql, b"Oidc" => DirectoryType::Oidc, } + .copied() } fn as_str(&self) -> &'static str { @@ -1859,18 +1829,16 @@ impl<'de> serde::Deserialize<'de> for DirectoryType { impl EnumImpl for Dkim2Flag { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), - b"donotmodify" => Dkim2Flag::Donotmodify, - b"donotexplode" => Dkim2Flag::Donotexplode, + Dkim2Flag, b"feedback" => Dkim2Flag::Feedback, } + .copied() } fn as_str(&self) -> &'static str { match self { - Dkim2Flag::Donotmodify => "donotmodify", - Dkim2Flag::Donotexplode => "donotexplode", Dkim2Flag::Feedback => "feedback", } } @@ -1881,14 +1849,12 @@ impl EnumImpl for Dkim2Flag { fn from_id(id: u16) -> Option { match id { - 0 => Some(Dkim2Flag::Donotmodify), - 1 => Some(Dkim2Flag::Donotexplode), - 2 => Some(Dkim2Flag::Feedback), + 0 => Some(Dkim2Flag::Feedback), _ => None, } } - const COUNT: usize = 3; + const COUNT: usize = 1; } impl serde::Serialize for Dkim2Flag { @@ -1910,10 +1876,63 @@ impl<'de> serde::Deserialize<'de> for Dkim2Flag { } } +impl EnumImpl for Dkim2HashAlgorithm { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + Dkim2HashAlgorithm, + b"sha256" => Dkim2HashAlgorithm::Sha256, + b"sha512" => Dkim2HashAlgorithm::Sha512, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + Dkim2HashAlgorithm::Sha256 => "sha256", + Dkim2HashAlgorithm::Sha512 => "sha512", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(Dkim2HashAlgorithm::Sha256), + 1 => Some(Dkim2HashAlgorithm::Sha512), + _ => None, + } + } + + const COUNT: usize = 2; +} + +impl serde::Serialize for Dkim2HashAlgorithm { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for Dkim2HashAlgorithm { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for DkimAuthResult { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimAuthResult, b"none" => DkimAuthResult::None, b"pass" => DkimAuthResult::Pass, b"fail" => DkimAuthResult::Fail, @@ -1922,6 +1941,7 @@ impl EnumImpl for DkimAuthResult { b"tempError" => DkimAuthResult::TempError, b"permError" => DkimAuthResult::PermError, } + .copied() } fn as_str(&self) -> &'static str { @@ -1977,13 +1997,15 @@ impl<'de> serde::Deserialize<'de> for DkimAuthResult { impl EnumImpl for DkimCanonicalization { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimCanonicalization, b"relaxed/relaxed" => DkimCanonicalization::RelaxedRelaxed, b"simple/simple" => DkimCanonicalization::SimpleSimple, b"relaxed/simple" => DkimCanonicalization::RelaxedSimple, b"simple/relaxed" => DkimCanonicalization::SimpleRelaxed, } + .copied() } fn as_str(&self) -> &'static str { @@ -2033,11 +2055,13 @@ impl<'de> serde::Deserialize<'de> for DkimCanonicalization { impl EnumImpl for DkimHash { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimHash, b"sha256" => DkimHash::Sha256, b"sha1" => DkimHash::Sha1, } + .copied() } fn as_str(&self) -> &'static str { @@ -2083,11 +2107,13 @@ impl<'de> serde::Deserialize<'de> for DkimHash { impl EnumImpl for DkimManagementType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimManagementType, b"Automatic" => DkimManagementType::Automatic, b"Manual" => DkimManagementType::Manual, } + .copied() } fn as_str(&self) -> &'static str { @@ -2133,13 +2159,15 @@ impl<'de> serde::Deserialize<'de> for DkimManagementType { impl EnumImpl for DkimRotationStage { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimRotationStage, b"active" => DkimRotationStage::Active, b"pending" => DkimRotationStage::Pending, b"retiring" => DkimRotationStage::Retiring, b"retired" => DkimRotationStage::Retired, } + .copied() } fn as_str(&self) -> &'static str { @@ -2189,13 +2217,16 @@ impl<'de> serde::Deserialize<'de> for DkimRotationStage { impl EnumImpl for DkimSignatureType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DkimSignatureType, b"Dkim1Ed25519Sha256" => DkimSignatureType::Dkim1Ed25519Sha256, b"Dkim1RsaSha256" => DkimSignatureType::Dkim1RsaSha256, b"Dkim2Ed25519Sha256" => DkimSignatureType::Dkim2Ed25519Sha256, b"Dkim2RsaSha256" => DkimSignatureType::Dkim2RsaSha256, + b"Dkim2MlDsa44Ed25519Sha512" => DkimSignatureType::Dkim2MlDsa44Ed25519Sha512, } + .copied() } fn as_str(&self) -> &'static str { @@ -2204,6 +2235,7 @@ impl EnumImpl for DkimSignatureType { DkimSignatureType::Dkim1RsaSha256 => "Dkim1RsaSha256", DkimSignatureType::Dkim2Ed25519Sha256 => "Dkim2Ed25519Sha256", DkimSignatureType::Dkim2RsaSha256 => "Dkim2RsaSha256", + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => "Dkim2MlDsa44Ed25519Sha512", } } @@ -2217,11 +2249,12 @@ impl EnumImpl for DkimSignatureType { 1 => Some(DkimSignatureType::Dkim1RsaSha256), 2 => Some(DkimSignatureType::Dkim2Ed25519Sha256), 3 => Some(DkimSignatureType::Dkim2RsaSha256), + 4 => Some(DkimSignatureType::Dkim2MlDsa44Ed25519Sha512), _ => None, } } - const COUNT: usize = 4; + const COUNT: usize = 5; } impl serde::Serialize for DkimSignatureType { @@ -2245,14 +2278,16 @@ impl<'de> serde::Deserialize<'de> for DkimSignatureType { impl EnumImpl for DmarcActionDisposition { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcActionDisposition, b"none" => DmarcActionDisposition::None, b"pass" => DmarcActionDisposition::Pass, b"quarantine" => DmarcActionDisposition::Quarantine, b"reject" => DmarcActionDisposition::Reject, b"unspecified" => DmarcActionDisposition::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -2304,12 +2339,14 @@ impl<'de> serde::Deserialize<'de> for DmarcActionDisposition { impl EnumImpl for DmarcAlignment { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcAlignment, b"relaxed" => DmarcAlignment::Relaxed, b"strict" => DmarcAlignment::Strict, b"unspecified" => DmarcAlignment::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -2357,12 +2394,14 @@ impl<'de> serde::Deserialize<'de> for DmarcAlignment { impl EnumImpl for DmarcDiscovery { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcDiscovery, b"psl" => DmarcDiscovery::Psl, b"treewalk" => DmarcDiscovery::Treewalk, b"unspecified" => DmarcDiscovery::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -2410,13 +2449,15 @@ impl<'de> serde::Deserialize<'de> for DmarcDiscovery { impl EnumImpl for DmarcDisposition { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcDisposition, b"none" => DmarcDisposition::None, b"quarantine" => DmarcDisposition::Quarantine, b"reject" => DmarcDisposition::Reject, b"unspecified" => DmarcDisposition::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -2466,8 +2507,9 @@ impl<'de> serde::Deserialize<'de> for DmarcDisposition { impl EnumImpl for DmarcPolicyOverride { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcPolicyOverride, b"Forwarded" => DmarcPolicyOverride::Forwarded, b"SampledOut" => DmarcPolicyOverride::SampledOut, b"TrustedForwarder" => DmarcPolicyOverride::TrustedForwarder, @@ -2476,6 +2518,7 @@ impl EnumImpl for DmarcPolicyOverride { b"Other" => DmarcPolicyOverride::Other, b"PolicyTestMode" => DmarcPolicyOverride::PolicyTestMode, } + .copied() } fn as_str(&self) -> &'static str { @@ -2531,12 +2574,14 @@ impl<'de> serde::Deserialize<'de> for DmarcPolicyOverride { impl EnumImpl for DmarcResult { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcResult, b"pass" => DmarcResult::Pass, b"fail" => DmarcResult::Fail, b"unspecified" => DmarcResult::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -2584,8 +2629,9 @@ impl<'de> serde::Deserialize<'de> for DmarcResult { impl EnumImpl for DmarcTroubleshootAuthResultType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DmarcTroubleshootAuthResultType, b"Pass" => DmarcTroubleshootAuthResultType::Pass, b"Fail" => DmarcTroubleshootAuthResultType::Fail, b"SoftFail" => DmarcTroubleshootAuthResultType::SoftFail, @@ -2594,6 +2640,7 @@ impl EnumImpl for DmarcTroubleshootAuthResultType { b"Neutral" => DmarcTroubleshootAuthResultType::Neutral, b"None" => DmarcTroubleshootAuthResultType::None, } + .copied() } fn as_str(&self) -> &'static str { @@ -2649,11 +2696,13 @@ impl<'de> serde::Deserialize<'de> for DmarcTroubleshootAuthResultType { impl EnumImpl for DnsManagementType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DnsManagementType, b"Manual" => DnsManagementType::Manual, b"Automatic" => DnsManagementType::Automatic, } + .copied() } fn as_str(&self) -> &'static str { @@ -2699,13 +2748,15 @@ impl<'de> serde::Deserialize<'de> for DnsManagementType { impl EnumImpl for DnsPublishStatus { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DnsPublishStatus, b"synced" => DnsPublishStatus::Synced, b"pending" => DnsPublishStatus::Pending, b"failed" => DnsPublishStatus::Failed, b"unknown" => DnsPublishStatus::Unknown, } + .copied() } fn as_str(&self) -> &'static str { @@ -2755,8 +2806,9 @@ impl<'de> serde::Deserialize<'de> for DnsPublishStatus { impl EnumImpl for DnsRecordType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DnsRecordType, b"dkim" => DnsRecordType::Dkim, b"tlsa" => DnsRecordType::Tlsa, b"spf" => DnsRecordType::Spf, @@ -2770,6 +2822,7 @@ impl EnumImpl for DnsRecordType { b"autoConfigLegacy" => DnsRecordType::AutoConfigLegacy, b"autoDiscover" => DnsRecordType::AutoDiscover, } + .copied() } fn as_str(&self) -> &'static str { @@ -2835,12 +2888,14 @@ impl<'de> serde::Deserialize<'de> for DnsRecordType { impl EnumImpl for DnsResolverProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DnsResolverProtocol, b"tls" => DnsResolverProtocol::Tls, b"udp" => DnsResolverProtocol::Udp, b"tcp" => DnsResolverProtocol::Tcp, } + .copied() } fn as_str(&self) -> &'static str { @@ -2888,14 +2943,16 @@ impl<'de> serde::Deserialize<'de> for DnsResolverProtocol { impl EnumImpl for DnsResolverType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + DnsResolverType, b"System" => DnsResolverType::System, b"Custom" => DnsResolverType::Custom, b"Cloudflare" => DnsResolverType::Cloudflare, b"Quad9" => DnsResolverType::Quad9, b"Google" => DnsResolverType::Google, } + .copied() } fn as_str(&self) -> &'static str { @@ -2952,7 +3009,6 @@ impl EnumImpl for DnsServerBootstrapType { DnsServerBootstrapType, b"Manual" => DnsServerBootstrapType::Manual, b"Tsig" => DnsServerBootstrapType::Tsig, - b"Deprecated1" => DnsServerBootstrapType::Deprecated1, b"Cloudflare" => DnsServerBootstrapType::Cloudflare, b"DigitalOcean" => DnsServerBootstrapType::DigitalOcean, b"DeSEC" => DnsServerBootstrapType::DeSEC, @@ -3030,7 +3086,6 @@ impl EnumImpl for DnsServerBootstrapType { match self { DnsServerBootstrapType::Manual => "Manual", DnsServerBootstrapType::Tsig => "Tsig", - DnsServerBootstrapType::Deprecated1 => "Deprecated1", DnsServerBootstrapType::Cloudflare => "Cloudflare", DnsServerBootstrapType::DigitalOcean => "DigitalOcean", DnsServerBootstrapType::DeSEC => "DeSEC", @@ -3111,81 +3166,80 @@ impl EnumImpl for DnsServerBootstrapType { match id { 0 => Some(DnsServerBootstrapType::Manual), 1 => Some(DnsServerBootstrapType::Tsig), - 2 => Some(DnsServerBootstrapType::Deprecated1), - 3 => Some(DnsServerBootstrapType::Cloudflare), - 4 => Some(DnsServerBootstrapType::DigitalOcean), - 5 => Some(DnsServerBootstrapType::DeSEC), - 6 => Some(DnsServerBootstrapType::Ovh), - 7 => Some(DnsServerBootstrapType::Bunny), - 8 => Some(DnsServerBootstrapType::Porkbun), - 9 => Some(DnsServerBootstrapType::Dnsimple), - 10 => Some(DnsServerBootstrapType::Spaceship), - 11 => Some(DnsServerBootstrapType::Route53), - 12 => Some(DnsServerBootstrapType::GoogleCloudDns), - 13 => Some(DnsServerBootstrapType::Alidns), - 14 => Some(DnsServerBootstrapType::ArvanCloud), - 15 => Some(DnsServerBootstrapType::Autodns), - 16 => Some(DnsServerBootstrapType::AzureDns), - 17 => Some(DnsServerBootstrapType::BaiduCloud), - 18 => Some(DnsServerBootstrapType::BluecatV2), - 19 => Some(DnsServerBootstrapType::ClouDns), - 20 => Some(DnsServerBootstrapType::Constellix), - 21 => Some(DnsServerBootstrapType::Cpanel), - 22 => Some(DnsServerBootstrapType::Ddnss), - 23 => Some(DnsServerBootstrapType::DnsMadeEasy), - 24 => Some(DnsServerBootstrapType::Domeneshop), - 25 => Some(DnsServerBootstrapType::Dreamhost), - 26 => Some(DnsServerBootstrapType::DuckDns), - 27 => Some(DnsServerBootstrapType::Dynu), - 28 => Some(DnsServerBootstrapType::EasyDns), - 29 => Some(DnsServerBootstrapType::EdgeDns), - 30 => Some(DnsServerBootstrapType::Exoscale), - 31 => Some(DnsServerBootstrapType::FreeMyIp), - 32 => Some(DnsServerBootstrapType::GandiV5), - 33 => Some(DnsServerBootstrapType::Gcore), - 34 => Some(DnsServerBootstrapType::Glesys), - 35 => Some(DnsServerBootstrapType::Godaddy), - 36 => Some(DnsServerBootstrapType::Hetzner), - 37 => Some(DnsServerBootstrapType::HostingDe), - 38 => Some(DnsServerBootstrapType::Hostinger), - 39 => Some(DnsServerBootstrapType::HuaweiCloud), - 40 => Some(DnsServerBootstrapType::Hurricane), - 41 => Some(DnsServerBootstrapType::IbmCloud), - 42 => Some(DnsServerBootstrapType::Infoblox), - 43 => Some(DnsServerBootstrapType::Infomaniak), - 44 => Some(DnsServerBootstrapType::Inwx), - 45 => Some(DnsServerBootstrapType::Ionos), - 46 => Some(DnsServerBootstrapType::Ipv64), - 47 => Some(DnsServerBootstrapType::Joker), - 48 => Some(DnsServerBootstrapType::Lightsail), - 49 => Some(DnsServerBootstrapType::Linode), - 50 => Some(DnsServerBootstrapType::LuaDns), - 51 => Some(DnsServerBootstrapType::MythicBeasts), - 52 => Some(DnsServerBootstrapType::Namecheap), - 53 => Some(DnsServerBootstrapType::NameDotCom), - 54 => Some(DnsServerBootstrapType::NameSilo), - 55 => Some(DnsServerBootstrapType::Netcup), - 56 => Some(DnsServerBootstrapType::Netlify), - 57 => Some(DnsServerBootstrapType::Nifcloud), - 58 => Some(DnsServerBootstrapType::Ns1), - 59 => Some(DnsServerBootstrapType::OracleCloud), - 60 => Some(DnsServerBootstrapType::Plesk), - 61 => Some(DnsServerBootstrapType::Safedns), - 62 => Some(DnsServerBootstrapType::Scaleway), - 63 => Some(DnsServerBootstrapType::TencentCloud), - 64 => Some(DnsServerBootstrapType::Transip), - 65 => Some(DnsServerBootstrapType::UltraDns), - 66 => Some(DnsServerBootstrapType::Vercel), - 67 => Some(DnsServerBootstrapType::Volcengine), - 68 => Some(DnsServerBootstrapType::Vultr), - 69 => Some(DnsServerBootstrapType::WebSupport), - 70 => Some(DnsServerBootstrapType::YandexCloud), - 71 => Some(DnsServerBootstrapType::PowerDns), + 2 => Some(DnsServerBootstrapType::Cloudflare), + 3 => Some(DnsServerBootstrapType::DigitalOcean), + 4 => Some(DnsServerBootstrapType::DeSEC), + 5 => Some(DnsServerBootstrapType::Ovh), + 6 => Some(DnsServerBootstrapType::Bunny), + 7 => Some(DnsServerBootstrapType::Porkbun), + 8 => Some(DnsServerBootstrapType::Dnsimple), + 9 => Some(DnsServerBootstrapType::Spaceship), + 10 => Some(DnsServerBootstrapType::Route53), + 11 => Some(DnsServerBootstrapType::GoogleCloudDns), + 12 => Some(DnsServerBootstrapType::Alidns), + 13 => Some(DnsServerBootstrapType::ArvanCloud), + 14 => Some(DnsServerBootstrapType::Autodns), + 15 => Some(DnsServerBootstrapType::AzureDns), + 16 => Some(DnsServerBootstrapType::BaiduCloud), + 17 => Some(DnsServerBootstrapType::BluecatV2), + 18 => Some(DnsServerBootstrapType::ClouDns), + 19 => Some(DnsServerBootstrapType::Constellix), + 20 => Some(DnsServerBootstrapType::Cpanel), + 21 => Some(DnsServerBootstrapType::Ddnss), + 22 => Some(DnsServerBootstrapType::DnsMadeEasy), + 23 => Some(DnsServerBootstrapType::Domeneshop), + 24 => Some(DnsServerBootstrapType::Dreamhost), + 25 => Some(DnsServerBootstrapType::DuckDns), + 26 => Some(DnsServerBootstrapType::Dynu), + 27 => Some(DnsServerBootstrapType::EasyDns), + 28 => Some(DnsServerBootstrapType::EdgeDns), + 29 => Some(DnsServerBootstrapType::Exoscale), + 30 => Some(DnsServerBootstrapType::FreeMyIp), + 31 => Some(DnsServerBootstrapType::GandiV5), + 32 => Some(DnsServerBootstrapType::Gcore), + 33 => Some(DnsServerBootstrapType::Glesys), + 34 => Some(DnsServerBootstrapType::Godaddy), + 35 => Some(DnsServerBootstrapType::Hetzner), + 36 => Some(DnsServerBootstrapType::HostingDe), + 37 => Some(DnsServerBootstrapType::Hostinger), + 38 => Some(DnsServerBootstrapType::HuaweiCloud), + 39 => Some(DnsServerBootstrapType::Hurricane), + 40 => Some(DnsServerBootstrapType::IbmCloud), + 41 => Some(DnsServerBootstrapType::Infoblox), + 42 => Some(DnsServerBootstrapType::Infomaniak), + 43 => Some(DnsServerBootstrapType::Inwx), + 44 => Some(DnsServerBootstrapType::Ionos), + 45 => Some(DnsServerBootstrapType::Ipv64), + 46 => Some(DnsServerBootstrapType::Joker), + 47 => Some(DnsServerBootstrapType::Lightsail), + 48 => Some(DnsServerBootstrapType::Linode), + 49 => Some(DnsServerBootstrapType::LuaDns), + 50 => Some(DnsServerBootstrapType::MythicBeasts), + 51 => Some(DnsServerBootstrapType::Namecheap), + 52 => Some(DnsServerBootstrapType::NameDotCom), + 53 => Some(DnsServerBootstrapType::NameSilo), + 54 => Some(DnsServerBootstrapType::Netcup), + 55 => Some(DnsServerBootstrapType::Netlify), + 56 => Some(DnsServerBootstrapType::Nifcloud), + 57 => Some(DnsServerBootstrapType::Ns1), + 58 => Some(DnsServerBootstrapType::OracleCloud), + 59 => Some(DnsServerBootstrapType::Plesk), + 60 => Some(DnsServerBootstrapType::Safedns), + 61 => Some(DnsServerBootstrapType::Scaleway), + 62 => Some(DnsServerBootstrapType::TencentCloud), + 63 => Some(DnsServerBootstrapType::Transip), + 64 => Some(DnsServerBootstrapType::UltraDns), + 65 => Some(DnsServerBootstrapType::Vercel), + 66 => Some(DnsServerBootstrapType::Volcengine), + 67 => Some(DnsServerBootstrapType::Vultr), + 68 => Some(DnsServerBootstrapType::WebSupport), + 69 => Some(DnsServerBootstrapType::YandexCloud), + 70 => Some(DnsServerBootstrapType::PowerDns), _ => None, } } - const COUNT: usize = 72; + const COUNT: usize = 71; } impl serde::Serialize for DnsServerBootstrapType { @@ -3213,7 +3267,6 @@ impl EnumImpl for DnsServerType { value.as_bytes(), DnsServerType, b"Tsig" => DnsServerType::Tsig, - b"Deprecated1" => DnsServerType::Deprecated1, b"Cloudflare" => DnsServerType::Cloudflare, b"DigitalOcean" => DnsServerType::DigitalOcean, b"DeSEC" => DnsServerType::DeSEC, @@ -3290,7 +3343,6 @@ impl EnumImpl for DnsServerType { fn as_str(&self) -> &'static str { match self { DnsServerType::Tsig => "Tsig", - DnsServerType::Deprecated1 => "Deprecated1", DnsServerType::Cloudflare => "Cloudflare", DnsServerType::DigitalOcean => "DigitalOcean", DnsServerType::DeSEC => "DeSEC", @@ -3370,81 +3422,80 @@ impl EnumImpl for DnsServerType { fn from_id(id: u16) -> Option { match id { 0 => Some(DnsServerType::Tsig), - 1 => Some(DnsServerType::Deprecated1), - 2 => Some(DnsServerType::Cloudflare), - 3 => Some(DnsServerType::DigitalOcean), - 4 => Some(DnsServerType::DeSEC), - 5 => Some(DnsServerType::Ovh), - 6 => Some(DnsServerType::Bunny), - 7 => Some(DnsServerType::Porkbun), - 8 => Some(DnsServerType::Dnsimple), - 9 => Some(DnsServerType::Spaceship), - 10 => Some(DnsServerType::Route53), - 11 => Some(DnsServerType::GoogleCloudDns), - 12 => Some(DnsServerType::Alidns), - 13 => Some(DnsServerType::ArvanCloud), - 14 => Some(DnsServerType::Autodns), - 15 => Some(DnsServerType::AzureDns), - 16 => Some(DnsServerType::BaiduCloud), - 17 => Some(DnsServerType::BluecatV2), - 18 => Some(DnsServerType::ClouDns), - 19 => Some(DnsServerType::Constellix), - 20 => Some(DnsServerType::Cpanel), - 21 => Some(DnsServerType::Ddnss), - 22 => Some(DnsServerType::DnsMadeEasy), - 23 => Some(DnsServerType::Domeneshop), - 24 => Some(DnsServerType::Dreamhost), - 25 => Some(DnsServerType::DuckDns), - 26 => Some(DnsServerType::Dynu), - 27 => Some(DnsServerType::EasyDns), - 28 => Some(DnsServerType::EdgeDns), - 29 => Some(DnsServerType::Exoscale), - 30 => Some(DnsServerType::FreeMyIp), - 31 => Some(DnsServerType::GandiV5), - 32 => Some(DnsServerType::Gcore), - 33 => Some(DnsServerType::Glesys), - 34 => Some(DnsServerType::Godaddy), - 35 => Some(DnsServerType::Hetzner), - 36 => Some(DnsServerType::HostingDe), - 37 => Some(DnsServerType::Hostinger), - 38 => Some(DnsServerType::HuaweiCloud), - 39 => Some(DnsServerType::Hurricane), - 40 => Some(DnsServerType::IbmCloud), - 41 => Some(DnsServerType::Infoblox), - 42 => Some(DnsServerType::Infomaniak), - 43 => Some(DnsServerType::Inwx), - 44 => Some(DnsServerType::Ionos), - 45 => Some(DnsServerType::Ipv64), - 46 => Some(DnsServerType::Joker), - 47 => Some(DnsServerType::Lightsail), - 48 => Some(DnsServerType::Linode), - 49 => Some(DnsServerType::LuaDns), - 50 => Some(DnsServerType::MythicBeasts), - 51 => Some(DnsServerType::Namecheap), - 52 => Some(DnsServerType::NameDotCom), - 53 => Some(DnsServerType::NameSilo), - 54 => Some(DnsServerType::Netcup), - 55 => Some(DnsServerType::Netlify), - 56 => Some(DnsServerType::Nifcloud), - 57 => Some(DnsServerType::Ns1), - 58 => Some(DnsServerType::OracleCloud), - 59 => Some(DnsServerType::Plesk), - 60 => Some(DnsServerType::Safedns), - 61 => Some(DnsServerType::Scaleway), - 62 => Some(DnsServerType::TencentCloud), - 63 => Some(DnsServerType::Transip), - 64 => Some(DnsServerType::UltraDns), - 65 => Some(DnsServerType::Vercel), - 66 => Some(DnsServerType::Volcengine), - 67 => Some(DnsServerType::Vultr), - 68 => Some(DnsServerType::WebSupport), - 69 => Some(DnsServerType::YandexCloud), - 70 => Some(DnsServerType::PowerDns), + 1 => Some(DnsServerType::Cloudflare), + 2 => Some(DnsServerType::DigitalOcean), + 3 => Some(DnsServerType::DeSEC), + 4 => Some(DnsServerType::Ovh), + 5 => Some(DnsServerType::Bunny), + 6 => Some(DnsServerType::Porkbun), + 7 => Some(DnsServerType::Dnsimple), + 8 => Some(DnsServerType::Spaceship), + 9 => Some(DnsServerType::Route53), + 10 => Some(DnsServerType::GoogleCloudDns), + 11 => Some(DnsServerType::Alidns), + 12 => Some(DnsServerType::ArvanCloud), + 13 => Some(DnsServerType::Autodns), + 14 => Some(DnsServerType::AzureDns), + 15 => Some(DnsServerType::BaiduCloud), + 16 => Some(DnsServerType::BluecatV2), + 17 => Some(DnsServerType::ClouDns), + 18 => Some(DnsServerType::Constellix), + 19 => Some(DnsServerType::Cpanel), + 20 => Some(DnsServerType::Ddnss), + 21 => Some(DnsServerType::DnsMadeEasy), + 22 => Some(DnsServerType::Domeneshop), + 23 => Some(DnsServerType::Dreamhost), + 24 => Some(DnsServerType::DuckDns), + 25 => Some(DnsServerType::Dynu), + 26 => Some(DnsServerType::EasyDns), + 27 => Some(DnsServerType::EdgeDns), + 28 => Some(DnsServerType::Exoscale), + 29 => Some(DnsServerType::FreeMyIp), + 30 => Some(DnsServerType::GandiV5), + 31 => Some(DnsServerType::Gcore), + 32 => Some(DnsServerType::Glesys), + 33 => Some(DnsServerType::Godaddy), + 34 => Some(DnsServerType::Hetzner), + 35 => Some(DnsServerType::HostingDe), + 36 => Some(DnsServerType::Hostinger), + 37 => Some(DnsServerType::HuaweiCloud), + 38 => Some(DnsServerType::Hurricane), + 39 => Some(DnsServerType::IbmCloud), + 40 => Some(DnsServerType::Infoblox), + 41 => Some(DnsServerType::Infomaniak), + 42 => Some(DnsServerType::Inwx), + 43 => Some(DnsServerType::Ionos), + 44 => Some(DnsServerType::Ipv64), + 45 => Some(DnsServerType::Joker), + 46 => Some(DnsServerType::Lightsail), + 47 => Some(DnsServerType::Linode), + 48 => Some(DnsServerType::LuaDns), + 49 => Some(DnsServerType::MythicBeasts), + 50 => Some(DnsServerType::Namecheap), + 51 => Some(DnsServerType::NameDotCom), + 52 => Some(DnsServerType::NameSilo), + 53 => Some(DnsServerType::Netcup), + 54 => Some(DnsServerType::Netlify), + 55 => Some(DnsServerType::Nifcloud), + 56 => Some(DnsServerType::Ns1), + 57 => Some(DnsServerType::OracleCloud), + 58 => Some(DnsServerType::Plesk), + 59 => Some(DnsServerType::Safedns), + 60 => Some(DnsServerType::Scaleway), + 61 => Some(DnsServerType::TencentCloud), + 62 => Some(DnsServerType::Transip), + 63 => Some(DnsServerType::UltraDns), + 64 => Some(DnsServerType::Vercel), + 65 => Some(DnsServerType::Volcengine), + 66 => Some(DnsServerType::Vultr), + 67 => Some(DnsServerType::WebSupport), + 68 => Some(DnsServerType::YandexCloud), + 69 => Some(DnsServerType::PowerDns), _ => None, } } - const COUNT: usize = 71; + const COUNT: usize = 70; } impl serde::Serialize for DnsServerType { @@ -3468,14 +3519,16 @@ impl<'de> serde::Deserialize<'de> for DnsServerType { impl EnumImpl for EncryptionAtRestType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + EncryptionAtRestType, b"Disabled" => EncryptionAtRestType::Disabled, b"Aes128" => EncryptionAtRestType::Aes128, b"Aes256" => EncryptionAtRestType::Aes256, b"Aes256Gcm" => EncryptionAtRestType::Aes256Gcm, b"ChaCha20Poly1305" => EncryptionAtRestType::ChaCha20Poly1305, } + .copied() } fn as_str(&self) -> &'static str { @@ -3527,11 +3580,13 @@ impl<'de> serde::Deserialize<'de> for EncryptionAtRestType { impl EnumImpl for EventPolicy { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + EventPolicy, b"include" => EventPolicy::Include, b"exclude" => EventPolicy::Exclude, } + .copied() } fn as_str(&self) -> &'static str { @@ -3577,17 +3632,12 @@ impl<'de> serde::Deserialize<'de> for EventPolicy { impl EnumImpl for ExpressionConstant { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ExpressionConstant, b"relaxed" => ExpressionConstant::Relaxed, b"strict" => ExpressionConstant::Strict, b"disable" => ExpressionConstant::Disable, - b"optional" => ExpressionConstant::Optional, - b"require" => ExpressionConstant::Require, - b"ipv4_only" => ExpressionConstant::Ipv4Only, - b"ipv6_only" => ExpressionConstant::Ipv6Only, - b"ipv6_then_ipv4" => ExpressionConstant::Ipv6ThenIpv4, - b"ipv4_then_ipv6" => ExpressionConstant::Ipv4ThenIpv6, b"hourly" => ExpressionConstant::Hourly, b"daily" => ExpressionConstant::Daily, b"weekly" => ExpressionConstant::Weekly, @@ -3599,6 +3649,7 @@ impl EnumImpl for ExpressionConstant { b"stanag4406" => ExpressionConstant::Stanag4406, b"nsep" => ExpressionConstant::Nsep, } + .copied() } fn as_str(&self) -> &'static str { @@ -3606,12 +3657,6 @@ impl EnumImpl for ExpressionConstant { ExpressionConstant::Relaxed => "relaxed", ExpressionConstant::Strict => "strict", ExpressionConstant::Disable => "disable", - ExpressionConstant::Optional => "optional", - ExpressionConstant::Require => "require", - ExpressionConstant::Ipv4Only => "ipv4_only", - ExpressionConstant::Ipv6Only => "ipv6_only", - ExpressionConstant::Ipv6ThenIpv4 => "ipv6_then_ipv4", - ExpressionConstant::Ipv4ThenIpv6 => "ipv4_then_ipv6", ExpressionConstant::Hourly => "hourly", ExpressionConstant::Daily => "daily", ExpressionConstant::Weekly => "weekly", @@ -3634,27 +3679,21 @@ impl EnumImpl for ExpressionConstant { 0 => Some(ExpressionConstant::Relaxed), 1 => Some(ExpressionConstant::Strict), 2 => Some(ExpressionConstant::Disable), - 3 => Some(ExpressionConstant::Optional), - 4 => Some(ExpressionConstant::Require), - 5 => Some(ExpressionConstant::Ipv4Only), - 6 => Some(ExpressionConstant::Ipv6Only), - 7 => Some(ExpressionConstant::Ipv6ThenIpv4), - 8 => Some(ExpressionConstant::Ipv4ThenIpv6), - 9 => Some(ExpressionConstant::Hourly), - 10 => Some(ExpressionConstant::Daily), - 11 => Some(ExpressionConstant::Weekly), - 12 => Some(ExpressionConstant::Login), - 13 => Some(ExpressionConstant::Plain), - 14 => Some(ExpressionConstant::Xoauth2), - 15 => Some(ExpressionConstant::Oauthbearer), - 16 => Some(ExpressionConstant::Mixer), - 17 => Some(ExpressionConstant::Stanag4406), - 18 => Some(ExpressionConstant::Nsep), + 3 => Some(ExpressionConstant::Hourly), + 4 => Some(ExpressionConstant::Daily), + 5 => Some(ExpressionConstant::Weekly), + 6 => Some(ExpressionConstant::Login), + 7 => Some(ExpressionConstant::Plain), + 8 => Some(ExpressionConstant::Xoauth2), + 9 => Some(ExpressionConstant::Oauthbearer), + 10 => Some(ExpressionConstant::Mixer), + 11 => Some(ExpressionConstant::Stanag4406), + 12 => Some(ExpressionConstant::Nsep), _ => None, } } - const COUNT: usize = 19; + const COUNT: usize = 13; } impl serde::Serialize for ExpressionConstant { @@ -3755,7 +3794,6 @@ impl EnumImpl for ExpressionVariable { b"reply_to.local" => ExpressionVariable::ReplyToLocal, b"reply_to.name" => ExpressionVariable::ReplyToName, b"retry_num" => ExpressionVariable::RetryNum, - b"reverse_ip" => ExpressionVariable::ReverseIp, b"scheme" => ExpressionVariable::Scheme, b"sender" => ExpressionVariable::Sender, b"sender_domain" => ExpressionVariable::SenderDomain, @@ -3773,6 +3811,9 @@ impl EnumImpl for ExpressionVariable { b"url_original" => ExpressionVariable::UrlOriginal, b"value" => ExpressionVariable::Value, b"value_lower" => ExpressionVariable::ValueLower, + b"spam_score" => ExpressionVariable::SpamScore, + b"spam_probability" => ExpressionVariable::SpamProbability, + b"spam_threshold" => ExpressionVariable::SpamThreshold, } .copied() } @@ -3853,7 +3894,6 @@ impl EnumImpl for ExpressionVariable { ExpressionVariable::ReplyToLocal => "reply_to.local", ExpressionVariable::ReplyToName => "reply_to.name", ExpressionVariable::RetryNum => "retry_num", - ExpressionVariable::ReverseIp => "reverse_ip", ExpressionVariable::Scheme => "scheme", ExpressionVariable::Sender => "sender", ExpressionVariable::SenderDomain => "sender_domain", @@ -3871,6 +3911,9 @@ impl EnumImpl for ExpressionVariable { ExpressionVariable::UrlOriginal => "url_original", ExpressionVariable::Value => "value", ExpressionVariable::ValueLower => "value_lower", + ExpressionVariable::SpamScore => "spam_score", + ExpressionVariable::SpamProbability => "spam_probability", + ExpressionVariable::SpamThreshold => "spam_threshold", } } @@ -3954,29 +3997,31 @@ impl EnumImpl for ExpressionVariable { 71 => Some(ExpressionVariable::ReplyToLocal), 72 => Some(ExpressionVariable::ReplyToName), 73 => Some(ExpressionVariable::RetryNum), - 74 => Some(ExpressionVariable::ReverseIp), - 75 => Some(ExpressionVariable::Scheme), - 76 => Some(ExpressionVariable::Sender), - 77 => Some(ExpressionVariable::SenderDomain), - 78 => Some(ExpressionVariable::Size), - 79 => Some(ExpressionVariable::Sld), - 80 => Some(ExpressionVariable::Source), - 81 => Some(ExpressionVariable::Subject), - 82 => Some(ExpressionVariable::SubjectThread), - 83 => Some(ExpressionVariable::SubjectWords), - 84 => Some(ExpressionVariable::To), - 85 => Some(ExpressionVariable::ToDomain), - 86 => Some(ExpressionVariable::ToLocal), - 87 => Some(ExpressionVariable::ToName), - 88 => Some(ExpressionVariable::Url), - 89 => Some(ExpressionVariable::UrlOriginal), - 90 => Some(ExpressionVariable::Value), - 91 => Some(ExpressionVariable::ValueLower), + 74 => Some(ExpressionVariable::Scheme), + 75 => Some(ExpressionVariable::Sender), + 76 => Some(ExpressionVariable::SenderDomain), + 77 => Some(ExpressionVariable::Size), + 78 => Some(ExpressionVariable::Sld), + 79 => Some(ExpressionVariable::Source), + 80 => Some(ExpressionVariable::Subject), + 81 => Some(ExpressionVariable::SubjectThread), + 82 => Some(ExpressionVariable::SubjectWords), + 83 => Some(ExpressionVariable::To), + 84 => Some(ExpressionVariable::ToDomain), + 85 => Some(ExpressionVariable::ToLocal), + 86 => Some(ExpressionVariable::ToName), + 87 => Some(ExpressionVariable::Url), + 88 => Some(ExpressionVariable::UrlOriginal), + 89 => Some(ExpressionVariable::Value), + 90 => Some(ExpressionVariable::ValueLower), + 91 => Some(ExpressionVariable::SpamScore), + 92 => Some(ExpressionVariable::SpamProbability), + 93 => Some(ExpressionVariable::SpamThreshold), _ => None, } } - const COUNT: usize = 92; + const COUNT: usize = 94; } impl serde::Serialize for ExpressionVariable { @@ -4000,13 +4045,15 @@ impl<'de> serde::Deserialize<'de> for ExpressionVariable { impl EnumImpl for FailureReportingOption { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + FailureReportingOption, b"all" => FailureReportingOption::All, b"any" => FailureReportingOption::Any, b"dkimFailure" => FailureReportingOption::DkimFailure, b"spfFailure" => FailureReportingOption::SpfFailure, } + .copied() } fn as_str(&self) -> &'static str { @@ -4056,12 +4103,14 @@ impl<'de> serde::Deserialize<'de> for FailureReportingOption { impl EnumImpl for HttpAuthType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + HttpAuthType, b"Unauthenticated" => HttpAuthType::Unauthenticated, b"Basic" => HttpAuthType::Basic, b"Bearer" => HttpAuthType::Bearer, } + .copied() } fn as_str(&self) -> &'static str { @@ -4109,11 +4158,13 @@ impl<'de> serde::Deserialize<'de> for HttpAuthType { impl EnumImpl for HttpLookupFormatType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + HttpLookupFormatType, b"Csv" => HttpLookupFormatType::Csv, b"List" => HttpLookupFormatType::List, } + .copied() } fn as_str(&self) -> &'static str { @@ -4157,75 +4208,23 @@ impl<'de> serde::Deserialize<'de> for HttpLookupFormatType { } } -impl EnumImpl for InMemoryStoreBaseType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Redis" => InMemoryStoreBaseType::Redis, - b"RedisCluster" => InMemoryStoreBaseType::RedisCluster, - b"RedisSentinel" => InMemoryStoreBaseType::RedisSentinel, - } - } - - fn as_str(&self) -> &'static str { - match self { - InMemoryStoreBaseType::Redis => "Redis", - InMemoryStoreBaseType::RedisCluster => "RedisCluster", - InMemoryStoreBaseType::RedisSentinel => "RedisSentinel", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(InMemoryStoreBaseType::Redis), - 1 => Some(InMemoryStoreBaseType::RedisCluster), - 2 => Some(InMemoryStoreBaseType::RedisSentinel), - _ => None, - } - } - - const COUNT: usize = 3; -} - -impl serde::Serialize for InMemoryStoreBaseType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for InMemoryStoreBaseType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for InMemoryStoreType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + InMemoryStoreType, b"Default" => InMemoryStoreType::Default, - b"Sharded" => InMemoryStoreType::Sharded, b"Redis" => InMemoryStoreType::Redis, b"RedisCluster" => InMemoryStoreType::RedisCluster, b"RedisSentinel" => InMemoryStoreType::RedisSentinel, } + .copied() } fn as_str(&self) -> &'static str { match self { InMemoryStoreType::Default => "Default", - InMemoryStoreType::Sharded => "Sharded", InMemoryStoreType::Redis => "Redis", InMemoryStoreType::RedisCluster => "RedisCluster", InMemoryStoreType::RedisSentinel => "RedisSentinel", @@ -4239,7 +4238,6 @@ impl EnumImpl for InMemoryStoreType { fn from_id(id: u16) -> Option { match id { 0 => Some(InMemoryStoreType::Default), - 1 => Some(InMemoryStoreType::Sharded), 2 => Some(InMemoryStoreType::Redis), 3 => Some(InMemoryStoreType::RedisCluster), 4 => Some(InMemoryStoreType::RedisSentinel), @@ -4269,23 +4267,21 @@ impl<'de> serde::Deserialize<'de> for InMemoryStoreType { } } -impl EnumImpl for IndexDocumentType { +impl EnumImpl for IndexStatusType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), - b"email" => IndexDocumentType::Email, - b"calendar" => IndexDocumentType::Calendar, - b"contacts" => IndexDocumentType::Contacts, - b"file" => IndexDocumentType::File, + IndexStatusType, + b"Running" => IndexStatusType::Running, + b"Failed" => IndexStatusType::Failed, } + .copied() } fn as_str(&self) -> &'static str { match self { - IndexDocumentType::Email => "email", - IndexDocumentType::Calendar => "calendar", - IndexDocumentType::Contacts => "contacts", - IndexDocumentType::File => "file", + IndexStatusType::Running => "Running", + IndexStatusType::Failed => "Failed", } } @@ -4295,18 +4291,16 @@ impl EnumImpl for IndexDocumentType { fn from_id(id: u16) -> Option { match id { - 0 => Some(IndexDocumentType::Email), - 1 => Some(IndexDocumentType::Calendar), - 2 => Some(IndexDocumentType::Contacts), - 3 => Some(IndexDocumentType::File), + 0 => Some(IndexStatusType::Running), + 1 => Some(IndexStatusType::Failed), _ => None, } } - const COUNT: usize = 4; + const COUNT: usize = 2; } -impl serde::Serialize for IndexDocumentType { +impl serde::Serialize for IndexStatusType { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, @@ -4315,7 +4309,66 @@ impl serde::Serialize for IndexDocumentType { } } -impl<'de> serde::Deserialize<'de> for IndexDocumentType { +impl<'de> serde::Deserialize<'de> for IndexStatusType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for IndexType { + #[cfg(not(feature = "enterprise"))] + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + IndexType, + b"email" => IndexType::Email, + b"calendar" => IndexType::Calendar, + b"contacts" => IndexType::Contacts, + b"file" => IndexType::File, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + IndexType::Email => "email", + IndexType::Calendar => "calendar", + IndexType::Contacts => "contacts", + IndexType::File => "file", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 1 => Some(IndexType::Email), + 2 => Some(IndexType::Calendar), + 3 => Some(IndexType::Contacts), + 4 => Some(IndexType::File), + _ => None, + } + } + + const COUNT: usize = 5; +} + +impl serde::Serialize for IndexType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for IndexType { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -4327,11 +4380,13 @@ impl<'de> serde::Deserialize<'de> for IndexDocumentType { impl EnumImpl for IpProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + IpProtocol, b"udp" => IpProtocol::Udp, b"tcp" => IpProtocol::Tcp, } + .copied() } fn as_str(&self) -> &'static str { @@ -4377,11 +4432,13 @@ impl<'de> serde::Deserialize<'de> for IpProtocol { impl EnumImpl for JokerAuthType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + JokerAuthType, b"ApiKey" => JokerAuthType::ApiKey, b"UsernamePassword" => JokerAuthType::UsernamePassword, } + .copied() } fn as_str(&self) -> &'static str { @@ -4427,8 +4484,9 @@ impl<'de> serde::Deserialize<'de> for JokerAuthType { impl EnumImpl for JwtSignatureAlgorithm { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + JwtSignatureAlgorithm, b"es256" => JwtSignatureAlgorithm::Es256, b"es384" => JwtSignatureAlgorithm::Es384, b"ps256" => JwtSignatureAlgorithm::Ps256, @@ -4441,6 +4499,7 @@ impl EnumImpl for JwtSignatureAlgorithm { b"hs384" => JwtSignatureAlgorithm::Hs384, b"hs512" => JwtSignatureAlgorithm::Hs512, } + .copied() } fn as_str(&self) -> &'static str { @@ -5236,266 +5295,263 @@ impl EnumImpl for Locale { 67 => Some(Locale::CyGB), 68 => Some(Locale::DaDK), 69 => Some(Locale::DeAT), - 71 => Some(Locale::DeBE), - 72 => Some(Locale::En001), - 73 => Some(Locale::DeCH), - 74 => Some(Locale::DeDE), - 76 => Some(Locale::DeIT), - 77 => Some(Locale::DeLI), - 78 => Some(Locale::DeLU), - 79 => Some(Locale::Ar001), - 80 => Some(Locale::DoiIN), - 81 => Some(Locale::DsbDE), - 82 => Some(Locale::DvMV), - 83 => Some(Locale::DzBT), - 84 => Some(Locale::ElCY), - 85 => Some(Locale::ElGR), - 87 => Some(Locale::EnAG), - 88 => Some(Locale::EnAU), - 89 => Some(Locale::EnBW), - 90 => Some(Locale::EnCA), - 91 => Some(Locale::EnDK), - 92 => Some(Locale::EnGB), - 93 => Some(Locale::EnHK), - 94 => Some(Locale::EnIE), - 96 => Some(Locale::EnIL), - 97 => Some(Locale::EnIN), - 98 => Some(Locale::EnNG), - 99 => Some(Locale::EnNZ), - 100 => Some(Locale::EnPH), - 101 => Some(Locale::EnSC), - 102 => Some(Locale::EnSG), - 104 => Some(Locale::EnZA), - 105 => Some(Locale::EnZM), - 106 => Some(Locale::EnZW), - 107 => Some(Locale::Eo), - 108 => Some(Locale::EsAR), - 109 => Some(Locale::EsBO), - 110 => Some(Locale::EsCL), - 111 => Some(Locale::EsCO), - 112 => Some(Locale::EsCR), - 113 => Some(Locale::EsCU), - 114 => Some(Locale::EsDO), - 115 => Some(Locale::EsEC), - 116 => Some(Locale::EsES), - 118 => Some(Locale::EsGT), - 119 => Some(Locale::EsHN), - 120 => Some(Locale::EsMX), - 121 => Some(Locale::EsNI), - 122 => Some(Locale::EsPA), - 123 => Some(Locale::EsPE), - 124 => Some(Locale::EsPR), - 125 => Some(Locale::EsPY), - 126 => Some(Locale::EsSV), - 127 => Some(Locale::EsUS), - 128 => Some(Locale::EsUY), - 129 => Some(Locale::EsVE), - 130 => Some(Locale::EtEE), - 131 => Some(Locale::EuES), - 132 => Some(Locale::ZhHans), - 133 => Some(Locale::FaIR), - 134 => Some(Locale::FfSN), - 135 => Some(Locale::FiFI), - 137 => Some(Locale::FilPH), - 138 => Some(Locale::FoFO), - 139 => Some(Locale::FrBE), - 141 => Some(Locale::FrCA), - 142 => Some(Locale::FrCH), - 143 => Some(Locale::FrFR), - 145 => Some(Locale::FrLU), - 146 => Some(Locale::ZhHant), - 147 => Some(Locale::FurIT), - 148 => Some(Locale::FyDE), - 149 => Some(Locale::FyNL), - 150 => Some(Locale::GaIE), - 151 => Some(Locale::ZhHantHK), - 152 => Some(Locale::GdGB), - 153 => Some(Locale::GezER), - 154 => Some(Locale::SrLatnBA), - 155 => Some(Locale::GezET), - 156 => Some(Locale::SrLatnME), - 157 => Some(Locale::GlES), - 158 => Some(Locale::AzLatnAZ), - 159 => Some(Locale::GuIN), - 160 => Some(Locale::GvGB), - 161 => Some(Locale::HaNG), - 162 => Some(Locale::HakTW), - 163 => Some(Locale::HeIL), - 164 => Some(Locale::HiIN), - 165 => Some(Locale::HifFJ), - 166 => Some(Locale::HneIN), - 167 => Some(Locale::HrHR), - 168 => Some(Locale::HsbDE), - 169 => Some(Locale::HtHT), - 170 => Some(Locale::HuHU), - 171 => Some(Locale::HyAM), - 172 => Some(Locale::IaFR), - 173 => Some(Locale::IdID), - 174 => Some(Locale::IgNG), - 175 => Some(Locale::IkCA), - 176 => Some(Locale::IsIS), - 177 => Some(Locale::ItCH), - 178 => Some(Locale::ItIT), - 180 => Some(Locale::IuCA), - 181 => Some(Locale::JaJP), - 182 => Some(Locale::KaGE), - 183 => Some(Locale::KabDZ), - 184 => Some(Locale::KkKZ), - 185 => Some(Locale::KlGL), - 186 => Some(Locale::KmKH), - 187 => Some(Locale::KnIN), - 188 => Some(Locale::KoKR), - 189 => Some(Locale::KokIN), - 190 => Some(Locale::KsIN), - 191 => Some(Locale::KsDevaIN), - 192 => Some(Locale::KuTR), - 193 => Some(Locale::KwGB), - 194 => Some(Locale::KyKG), - 195 => Some(Locale::LbLU), - 196 => Some(Locale::LgUG), - 197 => Some(Locale::LiBE), - 198 => Some(Locale::LiNL), - 199 => Some(Locale::LijIT), - 200 => Some(Locale::LnCD), - 201 => Some(Locale::LoLA), - 202 => Some(Locale::LtLT), - 203 => Some(Locale::LvLV), - 204 => Some(Locale::LzhTW), - 205 => Some(Locale::MagIN), - 206 => Some(Locale::MaiIN), - 207 => Some(Locale::MaiNP), - 208 => Some(Locale::MfeMU), - 209 => Some(Locale::MgMG), - 210 => Some(Locale::MhrRU), - 211 => Some(Locale::MiNZ), - 212 => Some(Locale::MiqNI), - 213 => Some(Locale::MjwIN), - 214 => Some(Locale::MkMK), - 215 => Some(Locale::MlIN), - 216 => Some(Locale::MnMN), - 217 => Some(Locale::MniIN), - 218 => Some(Locale::MnwMM), - 219 => Some(Locale::MrIN), - 220 => Some(Locale::MsMY), - 221 => Some(Locale::MtMT), - 222 => Some(Locale::MyMM), - 223 => Some(Locale::NanTW), - 224 => Some(Locale::NanLatnTW), - 225 => Some(Locale::NbNO), - 226 => Some(Locale::NdsDE), - 227 => Some(Locale::NdsNL), - 228 => Some(Locale::NeNP), - 229 => Some(Locale::NhnMX), - 230 => Some(Locale::NiuNU), - 231 => Some(Locale::NiuNZ), - 232 => Some(Locale::NlAW), - 233 => Some(Locale::NlBE), - 235 => Some(Locale::NlNL), - 237 => Some(Locale::NnNO), - 238 => Some(Locale::NrZA), - 239 => Some(Locale::NsoZA), - 240 => Some(Locale::OcFR), - 241 => Some(Locale::OmET), - 242 => Some(Locale::OmKE), - 243 => Some(Locale::OrIN), - 244 => Some(Locale::OsRU), - 245 => Some(Locale::PaIN), - 246 => Some(Locale::PaPK), - 247 => Some(Locale::PapAW), - 248 => Some(Locale::PapCW), - 249 => Some(Locale::PlPL), - 250 => Some(Locale::PsAF), - 251 => Some(Locale::PtBR), - 252 => Some(Locale::PtPT), - 254 => Some(Locale::QuzPE), - 255 => Some(Locale::RajIN), - 256 => Some(Locale::RoRO), - 257 => Some(Locale::RuRU), - 258 => Some(Locale::RuUA), - 259 => Some(Locale::RwRW), - 260 => Some(Locale::SaIN), - 261 => Some(Locale::SahRU), - 262 => Some(Locale::SatIN), - 263 => Some(Locale::ScIT), - 264 => Some(Locale::SdIN), - 265 => Some(Locale::SdDevaIN), - 266 => Some(Locale::SeNO), - 267 => Some(Locale::SgsLT), - 268 => Some(Locale::ShnMM), - 269 => Some(Locale::ShsCA), - 270 => Some(Locale::SiLK), - 271 => Some(Locale::SidET), - 272 => Some(Locale::SkSK), - 273 => Some(Locale::SlSI), - 274 => Some(Locale::SmWS), - 275 => Some(Locale::SoDJ), - 276 => Some(Locale::SoET), - 277 => Some(Locale::SoKE), - 278 => Some(Locale::SoSO), - 279 => Some(Locale::SqAL), - 280 => Some(Locale::SqMK), - 281 => Some(Locale::SrME), - 282 => Some(Locale::SrRS), - 283 => Some(Locale::SrLatnRS), - 284 => Some(Locale::SsZA), - 285 => Some(Locale::StZA), - 286 => Some(Locale::SvFI), - 288 => Some(Locale::SvSE), - 289 => Some(Locale::SwKE), - 290 => Some(Locale::SwTZ), - 291 => Some(Locale::SzlPL), - 292 => Some(Locale::TaIN), - 293 => Some(Locale::TaLK), - 294 => Some(Locale::TcyIN), - 295 => Some(Locale::TeIN), - 296 => Some(Locale::TgTJ), - 297 => Some(Locale::ThTH), - 298 => Some(Locale::TheNP), - 299 => Some(Locale::TiER), - 300 => Some(Locale::TiET), - 301 => Some(Locale::TigER), - 302 => Some(Locale::TkTM), - 303 => Some(Locale::TlPH), - 304 => Some(Locale::TnZA), - 305 => Some(Locale::ToTO), - 306 => Some(Locale::TpiPG), - 307 => Some(Locale::TrCY), - 308 => Some(Locale::TrTR), - 309 => Some(Locale::TsZA), - 310 => Some(Locale::TtRU), - 311 => Some(Locale::TtLatnRU), - 312 => Some(Locale::UgCN), - 313 => Some(Locale::UkUA), - 314 => Some(Locale::UnmUS), - 315 => Some(Locale::UrIN), - 316 => Some(Locale::UrPK), - 317 => Some(Locale::UzUZ), - 318 => Some(Locale::UzCyrlUZ), - 319 => Some(Locale::VeZA), - 320 => Some(Locale::ViVN), - 321 => Some(Locale::WaBE), - 322 => Some(Locale::UzLatnUZ), - 323 => Some(Locale::WaeCH), - 324 => Some(Locale::WalET), - 325 => Some(Locale::WoSN), - 326 => Some(Locale::XhZA), - 327 => Some(Locale::YiUS), - 328 => Some(Locale::YoNG), - 329 => Some(Locale::YueHK), - 330 => Some(Locale::YuwPG), - 331 => Some(Locale::ZhCN), - 332 => Some(Locale::ZhHK), - 333 => Some(Locale::ZhSG), - 334 => Some(Locale::ZhTW), - 335 => Some(Locale::ZuZA), - 70 | 75 | 86 | 95 | 103 | 117 | 136 | 140 | 144 | 179 | 234 | 236 | 253 | 287 => { - Some(Locale::EnUS) - } + 70 => Some(Locale::DeBE), + 71 => Some(Locale::En001), + 72 => Some(Locale::DeCH), + 73 => Some(Locale::DeDE), + 74 => Some(Locale::DeIT), + 75 => Some(Locale::DeLI), + 76 => Some(Locale::DeLU), + 77 => Some(Locale::Ar001), + 78 => Some(Locale::DoiIN), + 79 => Some(Locale::DsbDE), + 80 => Some(Locale::DvMV), + 81 => Some(Locale::DzBT), + 82 => Some(Locale::ElCY), + 83 => Some(Locale::ElGR), + 84 => Some(Locale::EnAG), + 85 => Some(Locale::EnAU), + 86 => Some(Locale::EnBW), + 87 => Some(Locale::EnCA), + 88 => Some(Locale::EnDK), + 89 => Some(Locale::EnGB), + 90 => Some(Locale::EnHK), + 91 => Some(Locale::EnIE), + 92 => Some(Locale::EnIL), + 93 => Some(Locale::EnIN), + 94 => Some(Locale::EnNG), + 95 => Some(Locale::EnNZ), + 96 => Some(Locale::EnPH), + 97 => Some(Locale::EnSC), + 98 => Some(Locale::EnSG), + 99 => Some(Locale::EnZA), + 100 => Some(Locale::EnZM), + 101 => Some(Locale::EnZW), + 102 => Some(Locale::Eo), + 103 => Some(Locale::EsAR), + 104 => Some(Locale::EsBO), + 105 => Some(Locale::EsCL), + 106 => Some(Locale::EsCO), + 107 => Some(Locale::EsCR), + 108 => Some(Locale::EsCU), + 109 => Some(Locale::EsDO), + 110 => Some(Locale::EsEC), + 111 => Some(Locale::EsES), + 112 => Some(Locale::EsGT), + 113 => Some(Locale::EsHN), + 114 => Some(Locale::EsMX), + 115 => Some(Locale::EsNI), + 116 => Some(Locale::EsPA), + 117 => Some(Locale::EsPE), + 118 => Some(Locale::EsPR), + 119 => Some(Locale::EsPY), + 120 => Some(Locale::EsSV), + 121 => Some(Locale::EsUS), + 122 => Some(Locale::EsUY), + 123 => Some(Locale::EsVE), + 124 => Some(Locale::EtEE), + 125 => Some(Locale::EuES), + 126 => Some(Locale::ZhHans), + 127 => Some(Locale::FaIR), + 128 => Some(Locale::FfSN), + 129 => Some(Locale::FiFI), + 130 => Some(Locale::FilPH), + 131 => Some(Locale::FoFO), + 132 => Some(Locale::FrBE), + 133 => Some(Locale::FrCA), + 134 => Some(Locale::FrCH), + 135 => Some(Locale::FrFR), + 136 => Some(Locale::FrLU), + 137 => Some(Locale::ZhHant), + 138 => Some(Locale::FurIT), + 139 => Some(Locale::FyDE), + 140 => Some(Locale::FyNL), + 141 => Some(Locale::GaIE), + 142 => Some(Locale::ZhHantHK), + 143 => Some(Locale::GdGB), + 144 => Some(Locale::GezER), + 145 => Some(Locale::SrLatnBA), + 146 => Some(Locale::GezET), + 147 => Some(Locale::SrLatnME), + 148 => Some(Locale::GlES), + 149 => Some(Locale::AzLatnAZ), + 150 => Some(Locale::GuIN), + 151 => Some(Locale::GvGB), + 152 => Some(Locale::HaNG), + 153 => Some(Locale::HakTW), + 154 => Some(Locale::HeIL), + 155 => Some(Locale::HiIN), + 156 => Some(Locale::HifFJ), + 157 => Some(Locale::HneIN), + 158 => Some(Locale::HrHR), + 159 => Some(Locale::HsbDE), + 160 => Some(Locale::HtHT), + 161 => Some(Locale::HuHU), + 162 => Some(Locale::HyAM), + 163 => Some(Locale::IaFR), + 164 => Some(Locale::IdID), + 165 => Some(Locale::IgNG), + 166 => Some(Locale::IkCA), + 167 => Some(Locale::IsIS), + 168 => Some(Locale::ItCH), + 169 => Some(Locale::ItIT), + 170 => Some(Locale::IuCA), + 171 => Some(Locale::JaJP), + 172 => Some(Locale::KaGE), + 173 => Some(Locale::KabDZ), + 174 => Some(Locale::KkKZ), + 175 => Some(Locale::KlGL), + 176 => Some(Locale::KmKH), + 177 => Some(Locale::KnIN), + 178 => Some(Locale::KoKR), + 179 => Some(Locale::KokIN), + 180 => Some(Locale::KsIN), + 181 => Some(Locale::KsDevaIN), + 182 => Some(Locale::KuTR), + 183 => Some(Locale::KwGB), + 184 => Some(Locale::KyKG), + 185 => Some(Locale::LbLU), + 186 => Some(Locale::LgUG), + 187 => Some(Locale::LiBE), + 188 => Some(Locale::LiNL), + 189 => Some(Locale::LijIT), + 190 => Some(Locale::LnCD), + 191 => Some(Locale::LoLA), + 192 => Some(Locale::LtLT), + 193 => Some(Locale::LvLV), + 194 => Some(Locale::LzhTW), + 195 => Some(Locale::MagIN), + 196 => Some(Locale::MaiIN), + 197 => Some(Locale::MaiNP), + 198 => Some(Locale::MfeMU), + 199 => Some(Locale::MgMG), + 200 => Some(Locale::MhrRU), + 201 => Some(Locale::MiNZ), + 202 => Some(Locale::MiqNI), + 203 => Some(Locale::MjwIN), + 204 => Some(Locale::MkMK), + 205 => Some(Locale::MlIN), + 206 => Some(Locale::MnMN), + 207 => Some(Locale::MniIN), + 208 => Some(Locale::MnwMM), + 209 => Some(Locale::MrIN), + 210 => Some(Locale::MsMY), + 211 => Some(Locale::MtMT), + 212 => Some(Locale::MyMM), + 213 => Some(Locale::NanTW), + 214 => Some(Locale::NanLatnTW), + 215 => Some(Locale::NbNO), + 216 => Some(Locale::NdsDE), + 217 => Some(Locale::NdsNL), + 218 => Some(Locale::NeNP), + 219 => Some(Locale::NhnMX), + 220 => Some(Locale::NiuNU), + 221 => Some(Locale::NiuNZ), + 222 => Some(Locale::NlAW), + 223 => Some(Locale::NlBE), + 224 => Some(Locale::NlNL), + 225 => Some(Locale::NnNO), + 226 => Some(Locale::NrZA), + 227 => Some(Locale::NsoZA), + 228 => Some(Locale::OcFR), + 229 => Some(Locale::OmET), + 230 => Some(Locale::OmKE), + 231 => Some(Locale::OrIN), + 232 => Some(Locale::OsRU), + 233 => Some(Locale::PaIN), + 234 => Some(Locale::PaPK), + 235 => Some(Locale::PapAW), + 236 => Some(Locale::PapCW), + 237 => Some(Locale::PlPL), + 238 => Some(Locale::PsAF), + 239 => Some(Locale::PtBR), + 240 => Some(Locale::PtPT), + 241 => Some(Locale::QuzPE), + 242 => Some(Locale::RajIN), + 243 => Some(Locale::RoRO), + 244 => Some(Locale::RuRU), + 245 => Some(Locale::RuUA), + 246 => Some(Locale::RwRW), + 247 => Some(Locale::SaIN), + 248 => Some(Locale::SahRU), + 249 => Some(Locale::SatIN), + 250 => Some(Locale::ScIT), + 251 => Some(Locale::SdIN), + 252 => Some(Locale::SdDevaIN), + 253 => Some(Locale::SeNO), + 254 => Some(Locale::SgsLT), + 255 => Some(Locale::ShnMM), + 256 => Some(Locale::ShsCA), + 257 => Some(Locale::SiLK), + 258 => Some(Locale::SidET), + 259 => Some(Locale::SkSK), + 260 => Some(Locale::SlSI), + 261 => Some(Locale::SmWS), + 262 => Some(Locale::SoDJ), + 263 => Some(Locale::SoET), + 264 => Some(Locale::SoKE), + 265 => Some(Locale::SoSO), + 266 => Some(Locale::SqAL), + 267 => Some(Locale::SqMK), + 268 => Some(Locale::SrME), + 269 => Some(Locale::SrRS), + 270 => Some(Locale::SrLatnRS), + 271 => Some(Locale::SsZA), + 272 => Some(Locale::StZA), + 273 => Some(Locale::SvFI), + 274 => Some(Locale::SvSE), + 275 => Some(Locale::SwKE), + 276 => Some(Locale::SwTZ), + 277 => Some(Locale::SzlPL), + 278 => Some(Locale::TaIN), + 279 => Some(Locale::TaLK), + 280 => Some(Locale::TcyIN), + 281 => Some(Locale::TeIN), + 282 => Some(Locale::TgTJ), + 283 => Some(Locale::ThTH), + 284 => Some(Locale::TheNP), + 285 => Some(Locale::TiER), + 286 => Some(Locale::TiET), + 287 => Some(Locale::TigER), + 288 => Some(Locale::TkTM), + 289 => Some(Locale::TlPH), + 290 => Some(Locale::TnZA), + 291 => Some(Locale::ToTO), + 292 => Some(Locale::TpiPG), + 293 => Some(Locale::TrCY), + 294 => Some(Locale::TrTR), + 295 => Some(Locale::TsZA), + 296 => Some(Locale::TtRU), + 297 => Some(Locale::TtLatnRU), + 298 => Some(Locale::UgCN), + 299 => Some(Locale::UkUA), + 300 => Some(Locale::UnmUS), + 301 => Some(Locale::UrIN), + 302 => Some(Locale::UrPK), + 303 => Some(Locale::UzUZ), + 304 => Some(Locale::UzCyrlUZ), + 305 => Some(Locale::VeZA), + 306 => Some(Locale::ViVN), + 307 => Some(Locale::WaBE), + 308 => Some(Locale::UzLatnUZ), + 309 => Some(Locale::WaeCH), + 310 => Some(Locale::WalET), + 311 => Some(Locale::WoSN), + 312 => Some(Locale::XhZA), + 313 => Some(Locale::YiUS), + 314 => Some(Locale::YoNG), + 315 => Some(Locale::YueHK), + 316 => Some(Locale::YuwPG), + 317 => Some(Locale::ZhCN), + 318 => Some(Locale::ZhHK), + 319 => Some(Locale::ZhSG), + 320 => Some(Locale::ZhTW), + 321 => Some(Locale::ZuZA), _ => None, } } - const COUNT: usize = 336; + const COUNT: usize = 322; } impl serde::Serialize for Locale { @@ -5519,13 +5575,15 @@ impl<'de> serde::Deserialize<'de> for Locale { impl EnumImpl for LogRotateFrequency { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + LogRotateFrequency, b"daily" => LogRotateFrequency::Daily, b"hourly" => LogRotateFrequency::Hourly, b"minutely" => LogRotateFrequency::Minutely, b"never" => LogRotateFrequency::Never, } + .copied() } fn as_str(&self) -> &'static str { @@ -5574,17 +5632,19 @@ impl<'de> serde::Deserialize<'de> for LogRotateFrequency { } impl EnumImpl for LookupStoreType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + LookupStoreType, b"PostgreSql" => LookupStoreType::PostgreSql, b"MySql" => LookupStoreType::MySql, b"Sqlite" => LookupStoreType::Sqlite, - b"Sharded" => LookupStoreType::Sharded, b"Redis" => LookupStoreType::Redis, b"RedisCluster" => LookupStoreType::RedisCluster, b"RedisSentinel" => LookupStoreType::RedisSentinel, } + .copied() } fn as_str(&self) -> &'static str { @@ -5592,7 +5652,6 @@ impl EnumImpl for LookupStoreType { LookupStoreType::PostgreSql => "PostgreSql", LookupStoreType::MySql => "MySql", LookupStoreType::Sqlite => "Sqlite", - LookupStoreType::Sharded => "Sharded", LookupStoreType::Redis => "Redis", LookupStoreType::RedisCluster => "RedisCluster", LookupStoreType::RedisSentinel => "RedisSentinel", @@ -5608,7 +5667,6 @@ impl EnumImpl for LookupStoreType { 0 => Some(LookupStoreType::PostgreSql), 1 => Some(LookupStoreType::MySql), 2 => Some(LookupStoreType::Sqlite), - 3 => Some(LookupStoreType::Sharded), 4 => Some(LookupStoreType::Redis), 5 => Some(LookupStoreType::RedisCluster), 6 => Some(LookupStoreType::RedisSentinel), @@ -5640,8 +5698,9 @@ impl<'de> serde::Deserialize<'de> for LookupStoreType { impl EnumImpl for MessageFlag { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MessageFlag, b"authenticated" => MessageFlag::Authenticated, b"unauthenticated" => MessageFlag::Unauthenticated, b"unauthenticatedDmarc" => MessageFlag::UnauthenticatedDmarc, @@ -5649,6 +5708,7 @@ impl EnumImpl for MessageFlag { b"report" => MessageFlag::Report, b"autogenerated" => MessageFlag::Autogenerated, } + .copied() } fn as_str(&self) -> &'static str { @@ -5700,21 +5760,33 @@ impl<'de> serde::Deserialize<'de> for MessageFlag { } } -impl EnumImpl for MetricType { +impl EnumImpl for MetadataDataType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), - b"Counter" => MetricType::Counter, - b"Gauge" => MetricType::Gauge, - b"Histogram" => MetricType::Histogram, + MetadataDataType, + b"Email" => MetadataDataType::Email, + b"Mailbox" => MetadataDataType::Mailbox, + b"SieveScript" => MetadataDataType::SieveScript, + b"Calendar" => MetadataDataType::Calendar, + b"CalendarEvent" => MetadataDataType::CalendarEvent, + b"AddressBook" => MetadataDataType::AddressBook, + b"ContactCard" => MetadataDataType::ContactCard, + b"FileNode" => MetadataDataType::FileNode, } + .copied() } fn as_str(&self) -> &'static str { match self { - MetricType::Counter => "Counter", - MetricType::Gauge => "Gauge", - MetricType::Histogram => "Histogram", + MetadataDataType::Email => "Email", + MetadataDataType::Mailbox => "Mailbox", + MetadataDataType::SieveScript => "SieveScript", + MetadataDataType::Calendar => "Calendar", + MetadataDataType::CalendarEvent => "CalendarEvent", + MetadataDataType::AddressBook => "AddressBook", + MetadataDataType::ContactCard => "ContactCard", + MetadataDataType::FileNode => "FileNode", } } @@ -5724,17 +5796,22 @@ impl EnumImpl for MetricType { fn from_id(id: u16) -> Option { match id { - 0 => Some(MetricType::Counter), - 1 => Some(MetricType::Gauge), - 2 => Some(MetricType::Histogram), + 0 => Some(MetadataDataType::Email), + 1 => Some(MetadataDataType::Mailbox), + 2 => Some(MetadataDataType::SieveScript), + 3 => Some(MetadataDataType::Calendar), + 4 => Some(MetadataDataType::CalendarEvent), + 5 => Some(MetadataDataType::AddressBook), + 6 => Some(MetadataDataType::ContactCard), + 7 => Some(MetadataDataType::FileNode), _ => None, } } - const COUNT: usize = 3; + const COUNT: usize = 8; } -impl serde::Serialize for MetricType { +impl serde::Serialize for MetadataDataType { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, @@ -5743,7 +5820,7 @@ impl serde::Serialize for MetricType { } } -impl<'de> serde::Deserialize<'de> for MetricType { +impl<'de> serde::Deserialize<'de> for MetadataDataType { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -5755,12 +5832,14 @@ impl<'de> serde::Deserialize<'de> for MetricType { impl EnumImpl for MetricsOtelType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MetricsOtelType, b"Disabled" => MetricsOtelType::Disabled, b"Http" => MetricsOtelType::Http, b"Grpc" => MetricsOtelType::Grpc, } + .copied() } fn as_str(&self) -> &'static str { @@ -5808,11 +5887,13 @@ impl<'de> serde::Deserialize<'de> for MetricsOtelType { impl EnumImpl for MetricsPrometheusType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MetricsPrometheusType, b"Disabled" => MetricsPrometheusType::Disabled, b"Enabled" => MetricsPrometheusType::Enabled, } + .copied() } fn as_str(&self) -> &'static str { @@ -5856,72 +5937,15 @@ impl<'de> serde::Deserialize<'de> for MetricsPrometheusType { } } -impl EnumImpl for MetricsStoreType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Disabled" => MetricsStoreType::Disabled, - b"Default" => MetricsStoreType::Default, - b"FoundationDb" => MetricsStoreType::FoundationDb, - b"PostgreSql" => MetricsStoreType::PostgreSql, - b"MySql" => MetricsStoreType::MySql, - } - } - - fn as_str(&self) -> &'static str { - match self { - MetricsStoreType::Disabled => "Disabled", - MetricsStoreType::Default => "Default", - MetricsStoreType::FoundationDb => "FoundationDb", - MetricsStoreType::PostgreSql => "PostgreSql", - MetricsStoreType::MySql => "MySql", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(MetricsStoreType::Disabled), - 1 => Some(MetricsStoreType::Default), - 2 => Some(MetricsStoreType::FoundationDb), - 3 => Some(MetricsStoreType::PostgreSql), - 4 => Some(MetricsStoreType::MySql), - _ => None, - } - } - - const COUNT: usize = 5; -} - -impl serde::Serialize for MetricsStoreType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for MetricsStoreType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for MilterVersion { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MilterVersion, b"v2" => MilterVersion::V2, b"v6" => MilterVersion::V6, } + .copied() } fn as_str(&self) -> &'static str { @@ -5967,8 +5991,9 @@ impl<'de> serde::Deserialize<'de> for MilterVersion { impl EnumImpl for ModelSize { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ModelSize, b"16" => ModelSize::V16, b"17" => ModelSize::V17, b"18" => ModelSize::V18, @@ -5983,6 +6008,7 @@ impl EnumImpl for ModelSize { b"27" => ModelSize::V27, b"28" => ModelSize::V28, } + .copied() } fn as_str(&self) -> &'static str { @@ -6050,11 +6076,13 @@ impl<'de> serde::Deserialize<'de> for ModelSize { impl EnumImpl for MtaDeliveryExpirationType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaDeliveryExpirationType, b"Ttl" => MtaDeliveryExpirationType::Ttl, b"Attempts" => MtaDeliveryExpirationType::Attempts, } + .copied() } fn as_str(&self) -> &'static str { @@ -6100,11 +6128,13 @@ impl<'de> serde::Deserialize<'de> for MtaDeliveryExpirationType { impl EnumImpl for MtaDeliveryScheduleIntervalsOrDefaultType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaDeliveryScheduleIntervalsOrDefaultType, b"Default" => MtaDeliveryScheduleIntervalsOrDefaultType::Default, b"Custom" => MtaDeliveryScheduleIntervalsOrDefaultType::Custom, } + .copied() } fn as_str(&self) -> &'static str { @@ -6150,8 +6180,9 @@ impl<'de> serde::Deserialize<'de> for MtaDeliveryScheduleIntervalsOrDefaultType impl EnumImpl for MtaInboundThrottleKey { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaInboundThrottleKey, b"listener" => MtaInboundThrottleKey::Listener, b"remoteIp" => MtaInboundThrottleKey::RemoteIp, b"localIp" => MtaInboundThrottleKey::LocalIp, @@ -6162,6 +6193,7 @@ impl EnumImpl for MtaInboundThrottleKey { b"rcpt" => MtaInboundThrottleKey::Rcpt, b"rcptDomain" => MtaInboundThrottleKey::RcptDomain, } + .copied() } fn as_str(&self) -> &'static str { @@ -6221,13 +6253,15 @@ impl<'de> serde::Deserialize<'de> for MtaInboundThrottleKey { impl EnumImpl for MtaIpStrategy { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaIpStrategy, b"v4ThenV6" => MtaIpStrategy::V4ThenV6, b"v6ThenV4" => MtaIpStrategy::V6ThenV4, b"v4Only" => MtaIpStrategy::V4Only, b"v6Only" => MtaIpStrategy::V6Only, } + .copied() } fn as_str(&self) -> &'static str { @@ -6277,8 +6311,9 @@ impl<'de> serde::Deserialize<'de> for MtaIpStrategy { impl EnumImpl for MtaOutboundThrottleKey { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaOutboundThrottleKey, b"mx" => MtaOutboundThrottleKey::Mx, b"remoteIp" => MtaOutboundThrottleKey::RemoteIp, b"localIp" => MtaOutboundThrottleKey::LocalIp, @@ -6286,6 +6321,7 @@ impl EnumImpl for MtaOutboundThrottleKey { b"senderDomain" => MtaOutboundThrottleKey::SenderDomain, b"rcptDomain" => MtaOutboundThrottleKey::RcptDomain, } + .copied() } fn as_str(&self) -> &'static str { @@ -6339,11 +6375,13 @@ impl<'de> serde::Deserialize<'de> for MtaOutboundThrottleKey { impl EnumImpl for MtaProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaProtocol, b"smtp" => MtaProtocol::Smtp, b"lmtp" => MtaProtocol::Lmtp, } + .copied() } fn as_str(&self) -> &'static str { @@ -6389,13 +6427,15 @@ impl<'de> serde::Deserialize<'de> for MtaProtocol { impl EnumImpl for MtaQueueQuotaKey { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaQueueQuotaKey, b"sender" => MtaQueueQuotaKey::Sender, b"senderDomain" => MtaQueueQuotaKey::SenderDomain, b"rcpt" => MtaQueueQuotaKey::Rcpt, b"rcptDomain" => MtaQueueQuotaKey::RcptDomain, } + .copied() } fn as_str(&self) -> &'static str { @@ -6445,12 +6485,14 @@ impl<'de> serde::Deserialize<'de> for MtaQueueQuotaKey { impl EnumImpl for MtaRequiredOrOptional { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaRequiredOrOptional, b"optional" => MtaRequiredOrOptional::Optional, b"require" => MtaRequiredOrOptional::Require, b"disable" => MtaRequiredOrOptional::Disable, } + .copied() } fn as_str(&self) -> &'static str { @@ -6498,12 +6540,14 @@ impl<'de> serde::Deserialize<'de> for MtaRequiredOrOptional { impl EnumImpl for MtaRouteType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaRouteType, b"Mx" => MtaRouteType::Mx, b"Relay" => MtaRouteType::Relay, b"Local" => MtaRouteType::Local, } + .copied() } fn as_str(&self) -> &'static str { @@ -6551,8 +6595,9 @@ impl<'de> serde::Deserialize<'de> for MtaRouteType { impl EnumImpl for MtaStage { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + MtaStage, b"connect" => MtaStage::Connect, b"ehlo" => MtaStage::Ehlo, b"auth" => MtaStage::Auth, @@ -6560,6 +6605,7 @@ impl EnumImpl for MtaStage { b"rcpt" => MtaStage::Rcpt, b"data" => MtaStage::Data, } + .copied() } fn as_str(&self) -> &'static str { @@ -6613,8 +6659,9 @@ impl<'de> serde::Deserialize<'de> for MtaStage { impl EnumImpl for NetworkListenerProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + NetworkListenerProtocol, b"smtp" => NetworkListenerProtocol::Smtp, b"lmtp" => NetworkListenerProtocol::Lmtp, b"http" => NetworkListenerProtocol::Http, @@ -6622,6 +6669,7 @@ impl EnumImpl for NetworkListenerProtocol { b"pop3" => NetworkListenerProtocol::Pop3, b"manageSieve" => NetworkListenerProtocol::ManageSieve, } + .copied() } fn as_str(&self) -> &'static str { @@ -6673,10 +6721,402 @@ impl<'de> serde::Deserialize<'de> for NetworkListenerProtocol { } } +impl EnumImpl for OAuthClientKind { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OAuthClientKind, + b"registered" => OAuthClientKind::Registered, + b"dynamic" => OAuthClientKind::Dynamic, + b"metadataDocument" => OAuthClientKind::MetadataDocument, + b"firstParty" => OAuthClientKind::FirstParty, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OAuthClientKind::Registered => "registered", + OAuthClientKind::Dynamic => "dynamic", + OAuthClientKind::MetadataDocument => "metadataDocument", + OAuthClientKind::FirstParty => "firstParty", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OAuthClientKind::Registered), + 1 => Some(OAuthClientKind::Dynamic), + 2 => Some(OAuthClientKind::MetadataDocument), + 3 => Some(OAuthClientKind::FirstParty), + _ => None, + } + } + + const COUNT: usize = 4; +} + +impl serde::Serialize for OAuthClientKind { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OAuthClientKind { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OAuthClientMetadataPolicy { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OAuthClientMetadataPolicy, + b"disabled" => OAuthClientMetadataPolicy::Disabled, + b"sameDomain" => OAuthClientMetadataPolicy::SameDomain, + b"any" => OAuthClientMetadataPolicy::Any, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OAuthClientMetadataPolicy::Disabled => "disabled", + OAuthClientMetadataPolicy::SameDomain => "sameDomain", + OAuthClientMetadataPolicy::Any => "any", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OAuthClientMetadataPolicy::Disabled), + 1 => Some(OAuthClientMetadataPolicy::SameDomain), + 2 => Some(OAuthClientMetadataPolicy::Any), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for OAuthClientMetadataPolicy { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OAuthClientMetadataPolicy { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OAuthGrantType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OAuthGrantType, + b"authorizationCode" => OAuthGrantType::AuthorizationCode, + b"refreshToken" => OAuthGrantType::RefreshToken, + b"deviceCode" => OAuthGrantType::DeviceCode, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OAuthGrantType::AuthorizationCode => "authorizationCode", + OAuthGrantType::RefreshToken => "refreshToken", + OAuthGrantType::DeviceCode => "deviceCode", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OAuthGrantType::AuthorizationCode), + 1 => Some(OAuthGrantType::RefreshToken), + 2 => Some(OAuthGrantType::DeviceCode), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for OAuthGrantType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OAuthGrantType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OAuthScope { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OAuthScope, + b"mail" => OAuthScope::Mail, + b"contacts" => OAuthScope::Contacts, + b"calendars" => OAuthScope::Calendars, + b"files" => OAuthScope::Files, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OAuthScope::Mail => "mail", + OAuthScope::Contacts => "contacts", + OAuthScope::Calendars => "calendars", + OAuthScope::Files => "files", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OAuthScope::Mail), + 1 => Some(OAuthScope::Contacts), + 2 => Some(OAuthScope::Calendars), + 3 => Some(OAuthScope::Files), + _ => None, + } + } + + const COUNT: usize = 4; +} + +impl serde::Serialize for OAuthScope { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OAuthScope { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OidcIntrospectionAuth { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OidcIntrospectionAuth, + b"clientSecretBasic" => OidcIntrospectionAuth::ClientSecretBasic, + b"clientSecretPost" => OidcIntrospectionAuth::ClientSecretPost, + b"none" => OidcIntrospectionAuth::None, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OidcIntrospectionAuth::ClientSecretBasic => "clientSecretBasic", + OidcIntrospectionAuth::ClientSecretPost => "clientSecretPost", + OidcIntrospectionAuth::None => "none", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OidcIntrospectionAuth::ClientSecretBasic), + 1 => Some(OidcIntrospectionAuth::ClientSecretPost), + 2 => Some(OidcIntrospectionAuth::None), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for OidcIntrospectionAuth { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OidcIntrospectionAuth { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OidcOpaqueTokenValidation { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OidcOpaqueTokenValidation, + b"none" => OidcOpaqueTokenValidation::None, + b"userInfo" => OidcOpaqueTokenValidation::UserInfo, + b"introspection" => OidcOpaqueTokenValidation::Introspection, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OidcOpaqueTokenValidation::None => "none", + OidcOpaqueTokenValidation::UserInfo => "userInfo", + OidcOpaqueTokenValidation::Introspection => "introspection", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OidcOpaqueTokenValidation::None), + 1 => Some(OidcOpaqueTokenValidation::UserInfo), + 2 => Some(OidcOpaqueTokenValidation::Introspection), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for OidcOpaqueTokenValidation { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OidcOpaqueTokenValidation { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + +impl EnumImpl for OidcUnscopedTokenPolicy { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + OidcUnscopedTokenPolicy, + b"reject" => OidcUnscopedTokenPolicy::Reject, + b"interoperable" => OidcUnscopedTokenPolicy::Interoperable, + b"full" => OidcUnscopedTokenPolicy::Full, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + OidcUnscopedTokenPolicy::Reject => "reject", + OidcUnscopedTokenPolicy::Interoperable => "interoperable", + OidcUnscopedTokenPolicy::Full => "full", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(OidcUnscopedTokenPolicy::Reject), + 1 => Some(OidcUnscopedTokenPolicy::Interoperable), + 2 => Some(OidcUnscopedTokenPolicy::Full), + _ => None, + } + } + + const COUNT: usize = 3; +} + +impl serde::Serialize for OidcUnscopedTokenPolicy { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for OidcUnscopedTokenPolicy { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for OvhEndpoint { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + OvhEndpoint, b"ovh-eu" => OvhEndpoint::OvhEu, b"ovh-ca" => OvhEndpoint::OvhCa, b"kimsufi-eu" => OvhEndpoint::KimsufiEu, @@ -6684,6 +7124,7 @@ impl EnumImpl for OvhEndpoint { b"soyoustart-eu" => OvhEndpoint::SoyoustartEu, b"soyoustart-ca" => OvhEndpoint::SoyoustartCa, } + .copied() } fn as_str(&self) -> &'static str { @@ -6737,13 +7178,15 @@ impl<'de> serde::Deserialize<'de> for OvhEndpoint { impl EnumImpl for PasswordHashAlgorithm { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PasswordHashAlgorithm, b"argon2id" => PasswordHashAlgorithm::Argon2id, b"bcrypt" => PasswordHashAlgorithm::Bcrypt, b"scrypt" => PasswordHashAlgorithm::Scrypt, b"pbkdf2" => PasswordHashAlgorithm::Pbkdf2, } + .copied() } fn as_str(&self) -> &'static str { @@ -6793,14 +7236,16 @@ impl<'de> serde::Deserialize<'de> for PasswordHashAlgorithm { impl EnumImpl for PasswordStrength { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PasswordStrength, b"zero" => PasswordStrength::Zero, b"one" => PasswordStrength::One, b"two" => PasswordStrength::Two, b"three" => PasswordStrength::Three, b"four" => PasswordStrength::Four, } + .copied() } fn as_str(&self) -> &'static str { @@ -6851,13 +7296,13 @@ impl<'de> serde::Deserialize<'de> for PasswordStrength { } impl EnumImpl for Permission { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { hashify::map! { value.as_bytes(), Permission, b"authenticate" => Permission::Authenticate, b"authenticateWithAlias" => Permission::AuthenticateWithAlias, - b"interactAi" => Permission::InteractAi, b"impersonate" => Permission::Impersonate, b"unlimitedRequests" => Permission::UnlimitedRequests, b"unlimitedUploads" => Permission::UnlimitedUploads, @@ -6918,9 +7363,6 @@ impl EnumImpl for Permission { b"jmapPrincipalChanges" => Permission::JmapPrincipalChanges, b"jmapPrincipalQueryChanges" => Permission::JmapPrincipalQueryChanges, b"jmapPrincipalGetAvailability" => Permission::JmapPrincipalGetAvailability, - b"jmapPrincipalCreate" => Permission::JmapPrincipalCreate, - b"jmapPrincipalUpdate" => Permission::JmapPrincipalUpdate, - b"jmapPrincipalDestroy" => Permission::JmapPrincipalDestroy, b"jmapQuotaGet" => Permission::JmapQuotaGet, b"jmapQuotaChanges" => Permission::JmapQuotaChanges, b"jmapQuotaQuery" => Permission::JmapQuotaQuery, @@ -6985,6 +7427,9 @@ impl EnumImpl for Permission { b"jmapParticipantIdentityUpdate" => Permission::JmapParticipantIdentityUpdate, b"jmapParticipantIdentityDestroy" => Permission::JmapParticipantIdentityDestroy, b"jmapCoreEcho" => Permission::JmapCoreEcho, + b"jmapMetadataGet" => Permission::JmapMetadataGet, + b"jmapMetadataSet" => Permission::JmapMetadataSet, + b"jmapMetadataPrivate" => Permission::JmapMetadataPrivate, b"imapAuthenticate" => Permission::ImapAuthenticate, b"imapAclGet" => Permission::ImapAclGet, b"imapAclSet" => Permission::ImapAclSet, @@ -7013,6 +7458,9 @@ impl EnumImpl for Permission { b"imapStore" => Permission::ImapStore, b"imapSubscribe" => Permission::ImapSubscribe, b"imapThread" => Permission::ImapThread, + b"imapMetadataGet" => Permission::ImapMetadataGet, + b"imapMetadataSet" => Permission::ImapMetadataSet, + b"imapMetadataPrivate" => Permission::ImapMetadataPrivate, b"pop3Authenticate" => Permission::Pop3Authenticate, b"pop3List" => Permission::Pop3List, b"pop3Uidl" => Permission::Pop3Uidl, @@ -7070,12 +7518,7 @@ impl EnumImpl for Permission { b"davCalQuery" => Permission::DavCalQuery, b"davCalMultiGet" => Permission::DavCalMultiGet, b"davCalFreeBusyQuery" => Permission::DavCalFreeBusyQuery, - b"oAuthClientRegistration" => Permission::OAuthClientRegistration, - b"oAuthClientOverride" => Permission::OAuthClientOverride, - b"liveTracing" => Permission::LiveTracing, - b"liveMetrics" => Permission::LiveMetrics, b"liveDeliveryTest" => Permission::LiveDeliveryTest, - b"scimAccess" => Permission::ScimAccess, b"sysAccountGet" => Permission::SysAccountGet, b"sysAccountCreate" => Permission::SysAccountCreate, b"sysAccountUpdate" => Permission::SysAccountUpdate, @@ -7101,23 +7544,9 @@ impl EnumImpl for Permission { b"actionInvalidateNegativeCaches" => Permission::ActionInvalidateNegativeCaches, b"actionPauseMtaQueue" => Permission::ActionPauseMtaQueue, b"actionResumeMtaQueue" => Permission::ActionResumeMtaQueue, - b"sysActionGet" => Permission::SysActionGet, b"sysActionCreate" => Permission::SysActionCreate, - b"sysActionUpdate" => Permission::SysActionUpdate, - b"sysActionDestroy" => Permission::SysActionDestroy, - b"sysActionQuery" => Permission::SysActionQuery, b"sysAddressBookGet" => Permission::SysAddressBookGet, b"sysAddressBookUpdate" => Permission::SysAddressBookUpdate, - b"sysAiModelGet" => Permission::SysAiModelGet, - b"sysAiModelCreate" => Permission::SysAiModelCreate, - b"sysAiModelUpdate" => Permission::SysAiModelUpdate, - b"sysAiModelDestroy" => Permission::SysAiModelDestroy, - b"sysAiModelQuery" => Permission::SysAiModelQuery, - b"sysAlertGet" => Permission::SysAlertGet, - b"sysAlertCreate" => Permission::SysAlertCreate, - b"sysAlertUpdate" => Permission::SysAlertUpdate, - b"sysAlertDestroy" => Permission::SysAlertDestroy, - b"sysAlertQuery" => Permission::SysAlertQuery, b"sysAllowedIpGet" => Permission::SysAllowedIpGet, b"sysAllowedIpCreate" => Permission::SysAllowedIpCreate, b"sysAllowedIpUpdate" => Permission::SysAllowedIpUpdate, @@ -7138,14 +7567,7 @@ impl EnumImpl for Permission { b"sysApplicationUpdate" => Permission::SysApplicationUpdate, b"sysApplicationDestroy" => Permission::SysApplicationDestroy, b"sysApplicationQuery" => Permission::SysApplicationQuery, - b"sysArchivedItemGet" => Permission::SysArchivedItemGet, - b"sysArchivedItemCreate" => Permission::SysArchivedItemCreate, - b"sysArchivedItemUpdate" => Permission::SysArchivedItemUpdate, - b"sysArchivedItemDestroy" => Permission::SysArchivedItemDestroy, - b"sysArchivedItemQuery" => Permission::SysArchivedItemQuery, b"sysArfExternalReportGet" => Permission::SysArfExternalReportGet, - b"sysArfExternalReportCreate" => Permission::SysArfExternalReportCreate, - b"sysArfExternalReportUpdate" => Permission::SysArfExternalReportUpdate, b"sysArfExternalReportDestroy" => Permission::SysArfExternalReportDestroy, b"sysArfExternalReportQuery" => Permission::SysArfExternalReportQuery, b"sysAsnGet" => Permission::SysAsnGet, @@ -7175,9 +7597,6 @@ impl EnumImpl for Permission { b"sysCertificateDestroy" => Permission::SysCertificateDestroy, b"sysCertificateQuery" => Permission::SysCertificateQuery, b"sysClusterNodeGet" => Permission::SysClusterNodeGet, - b"sysClusterNodeCreate" => Permission::SysClusterNodeCreate, - b"sysClusterNodeUpdate" => Permission::SysClusterNodeUpdate, - b"sysClusterNodeDestroy" => Permission::SysClusterNodeDestroy, b"sysClusterNodeQuery" => Permission::SysClusterNodeQuery, b"sysClusterRoleGet" => Permission::SysClusterRoleGet, b"sysClusterRoleCreate" => Permission::SysClusterRoleCreate, @@ -7203,12 +7622,9 @@ impl EnumImpl for Permission { b"sysDkimSignatureDestroy" => Permission::SysDkimSignatureDestroy, b"sysDkimSignatureQuery" => Permission::SysDkimSignatureQuery, b"sysDmarcExternalReportGet" => Permission::SysDmarcExternalReportGet, - b"sysDmarcExternalReportCreate" => Permission::SysDmarcExternalReportCreate, - b"sysDmarcExternalReportUpdate" => Permission::SysDmarcExternalReportUpdate, b"sysDmarcExternalReportDestroy" => Permission::SysDmarcExternalReportDestroy, b"sysDmarcExternalReportQuery" => Permission::SysDmarcExternalReportQuery, b"sysDmarcInternalReportGet" => Permission::SysDmarcInternalReportGet, - b"sysDmarcInternalReportCreate" => Permission::SysDmarcInternalReportCreate, b"sysDmarcInternalReportUpdate" => Permission::SysDmarcInternalReportUpdate, b"sysDmarcInternalReportDestroy" => Permission::SysDmarcInternalReportDestroy, b"sysDmarcInternalReportQuery" => Permission::SysDmarcInternalReportQuery, @@ -7230,8 +7646,6 @@ impl EnumImpl for Permission { b"sysDsnReportSettingsUpdate" => Permission::SysDsnReportSettingsUpdate, b"sysEmailGet" => Permission::SysEmailGet, b"sysEmailUpdate" => Permission::SysEmailUpdate, - b"sysEnterpriseGet" => Permission::SysEnterpriseGet, - b"sysEnterpriseUpdate" => Permission::SysEnterpriseUpdate, b"sysEventTracingLevelGet" => Permission::SysEventTracingLevelGet, b"sysEventTracingLevelCreate" => Permission::SysEventTracingLevelCreate, b"sysEventTracingLevelUpdate" => Permission::SysEventTracingLevelUpdate, @@ -7252,23 +7666,18 @@ impl EnumImpl for Permission { b"sysImapUpdate" => Permission::SysImapUpdate, b"sysInMemoryStoreGet" => Permission::SysInMemoryStoreGet, b"sysInMemoryStoreUpdate" => Permission::SysInMemoryStoreUpdate, + b"sysIndexQueueStatusGet" => Permission::SysIndexQueueStatusGet, + b"sysIndexQueueStatusUpdate" => Permission::SysIndexQueueStatusUpdate, + b"sysIndexQueueStatusQuery" => Permission::SysIndexQueueStatusQuery, b"sysJmapGet" => Permission::SysJmapGet, b"sysJmapUpdate" => Permission::SysJmapUpdate, b"sysLogGet" => Permission::SysLogGet, - b"sysLogCreate" => Permission::SysLogCreate, - b"sysLogUpdate" => Permission::SysLogUpdate, - b"sysLogDestroy" => Permission::SysLogDestroy, b"sysLogQuery" => Permission::SysLogQuery, b"sysMailingListGet" => Permission::SysMailingListGet, b"sysMailingListCreate" => Permission::SysMailingListCreate, b"sysMailingListUpdate" => Permission::SysMailingListUpdate, b"sysMailingListDestroy" => Permission::SysMailingListDestroy, b"sysMailingListQuery" => Permission::SysMailingListQuery, - b"sysMaskedEmailGet" => Permission::SysMaskedEmailGet, - b"sysMaskedEmailCreate" => Permission::SysMaskedEmailCreate, - b"sysMaskedEmailUpdate" => Permission::SysMaskedEmailUpdate, - b"sysMaskedEmailDestroy" => Permission::SysMaskedEmailDestroy, - b"sysMaskedEmailQuery" => Permission::SysMaskedEmailQuery, b"sysMemoryLookupKeyGet" => Permission::SysMemoryLookupKeyGet, b"sysMemoryLookupKeyCreate" => Permission::SysMemoryLookupKeyCreate, b"sysMemoryLookupKeyUpdate" => Permission::SysMemoryLookupKeyUpdate, @@ -7279,15 +7688,10 @@ impl EnumImpl for Permission { b"sysMemoryLookupKeyValueUpdate" => Permission::SysMemoryLookupKeyValueUpdate, b"sysMemoryLookupKeyValueDestroy" => Permission::SysMemoryLookupKeyValueDestroy, b"sysMemoryLookupKeyValueQuery" => Permission::SysMemoryLookupKeyValueQuery, - b"sysMetricGet" => Permission::SysMetricGet, - b"sysMetricCreate" => Permission::SysMetricCreate, - b"sysMetricUpdate" => Permission::SysMetricUpdate, - b"sysMetricDestroy" => Permission::SysMetricDestroy, - b"sysMetricQuery" => Permission::SysMetricQuery, + b"sysMetadataGet" => Permission::SysMetadataGet, + b"sysMetadataUpdate" => Permission::SysMetadataUpdate, b"sysMetricsGet" => Permission::SysMetricsGet, b"sysMetricsUpdate" => Permission::SysMetricsUpdate, - b"sysMetricsStoreGet" => Permission::SysMetricsStoreGet, - b"sysMetricsStoreUpdate" => Permission::SysMetricsStoreUpdate, b"sysMtaConnectionStrategyGet" => Permission::SysMtaConnectionStrategyGet, b"sysMtaConnectionStrategyCreate" => Permission::SysMtaConnectionStrategyCreate, b"sysMtaConnectionStrategyUpdate" => Permission::SysMtaConnectionStrategyUpdate, @@ -7368,6 +7772,9 @@ impl EnumImpl for Permission { b"sysOAuthClientUpdate" => Permission::SysOAuthClientUpdate, b"sysOAuthClientDestroy" => Permission::SysOAuthClientDestroy, b"sysOAuthClientQuery" => Permission::SysOAuthClientQuery, + b"sysOAuthGrantGet" => Permission::SysOAuthGrantGet, + b"sysOAuthGrantDestroy" => Permission::SysOAuthGrantDestroy, + b"sysOAuthGrantQuery" => Permission::SysOAuthGrantQuery, b"sysOidcProviderGet" => Permission::SysOidcProviderGet, b"sysOidcProviderUpdate" => Permission::SysOidcProviderUpdate, b"sysPublicKeyGet" => Permission::SysPublicKeyGet, @@ -7376,7 +7783,6 @@ impl EnumImpl for Permission { b"sysPublicKeyDestroy" => Permission::SysPublicKeyDestroy, b"sysPublicKeyQuery" => Permission::SysPublicKeyQuery, b"sysQueuedMessageGet" => Permission::SysQueuedMessageGet, - b"sysQueuedMessageCreate" => Permission::SysQueuedMessageCreate, b"sysQueuedMessageUpdate" => Permission::SysQueuedMessageUpdate, b"sysQueuedMessageDestroy" => Permission::SysQueuedMessageDestroy, b"sysQueuedMessageQuery" => Permission::SysQueuedMessageQuery, @@ -7425,8 +7831,6 @@ impl EnumImpl for Permission { b"sysSpamFileExtensionUpdate" => Permission::SysSpamFileExtensionUpdate, b"sysSpamFileExtensionDestroy" => Permission::SysSpamFileExtensionDestroy, b"sysSpamFileExtensionQuery" => Permission::SysSpamFileExtensionQuery, - b"sysSpamLlmGet" => Permission::SysSpamLlmGet, - b"sysSpamLlmUpdate" => Permission::SysSpamLlmUpdate, b"sysSpamPyzorGet" => Permission::SysSpamPyzorGet, b"sysSpamPyzorUpdate" => Permission::SysSpamPyzorUpdate, b"sysSpamRuleGet" => Permission::SysSpamRuleGet, @@ -7443,7 +7847,6 @@ impl EnumImpl for Permission { b"sysSpamTagQuery" => Permission::SysSpamTagQuery, b"sysSpamTrainingSampleGet" => Permission::SysSpamTrainingSampleGet, b"sysSpamTrainingSampleCreate" => Permission::SysSpamTrainingSampleCreate, - b"sysSpamTrainingSampleUpdate" => Permission::SysSpamTrainingSampleUpdate, b"sysSpamTrainingSampleDestroy" => Permission::SysSpamTrainingSampleDestroy, b"sysSpamTrainingSampleQuery" => Permission::SysSpamTrainingSampleQuery, b"sysSpfReportSettingsGet" => Permission::SysSpfReportSettingsGet, @@ -7455,24 +7858,22 @@ impl EnumImpl for Permission { b"sysStoreLookupQuery" => Permission::SysStoreLookupQuery, b"sysSystemSettingsGet" => Permission::SysSystemSettingsGet, b"sysSystemSettingsUpdate" => Permission::SysSystemSettingsUpdate, - b"taskIndexDocument" => Permission::TaskIndexDocument, - b"taskUnindexDocument" => Permission::TaskUnindexDocument, - b"taskIndexTrace" => Permission::TaskIndexTrace, b"taskCalendarAlarmEmail" => Permission::TaskCalendarAlarmEmail, b"taskCalendarAlarmNotification" => Permission::TaskCalendarAlarmNotification, b"taskCalendarItipMessage" => Permission::TaskCalendarItipMessage, b"taskMergeThreads" => Permission::TaskMergeThreads, b"taskDmarcReport" => Permission::TaskDmarcReport, b"taskTlsReport" => Permission::TaskTlsReport, - b"taskRestoreArchivedItem" => Permission::TaskRestoreArchivedItem, b"taskDestroyAccount" => Permission::TaskDestroyAccount, b"taskAccountMaintenance" => Permission::TaskAccountMaintenance, - b"taskTenantMaintenance" => Permission::TaskTenantMaintenance, b"taskStoreMaintenance" => Permission::TaskStoreMaintenance, b"taskSpamFilterMaintenance" => Permission::TaskSpamFilterMaintenance, b"taskAcmeRenewal" => Permission::TaskAcmeRenewal, b"taskDkimManagement" => Permission::TaskDkimManagement, b"taskDnsManagement" => Permission::TaskDnsManagement, + b"taskShareRevokeCleanup" => Permission::TaskShareRevokeCleanup, + b"taskDmarcReportWindow" => Permission::TaskDmarcReportWindow, + b"taskTlsReportWindow" => Permission::TaskTlsReportWindow, b"sysTaskGet" => Permission::SysTaskGet, b"sysTaskCreate" => Permission::SysTaskCreate, b"sysTaskUpdate" => Permission::SysTaskUpdate, @@ -7480,35 +7881,20 @@ impl EnumImpl for Permission { b"sysTaskQuery" => Permission::SysTaskQuery, b"sysTaskManagerGet" => Permission::SysTaskManagerGet, b"sysTaskManagerUpdate" => Permission::SysTaskManagerUpdate, - b"sysTenantGet" => Permission::SysTenantGet, - b"sysTenantCreate" => Permission::SysTenantCreate, - b"sysTenantUpdate" => Permission::SysTenantUpdate, - b"sysTenantDestroy" => Permission::SysTenantDestroy, - b"sysTenantQuery" => Permission::SysTenantQuery, b"sysTlsExternalReportGet" => Permission::SysTlsExternalReportGet, - b"sysTlsExternalReportCreate" => Permission::SysTlsExternalReportCreate, - b"sysTlsExternalReportUpdate" => Permission::SysTlsExternalReportUpdate, b"sysTlsExternalReportDestroy" => Permission::SysTlsExternalReportDestroy, b"sysTlsExternalReportQuery" => Permission::SysTlsExternalReportQuery, b"sysTlsInternalReportGet" => Permission::SysTlsInternalReportGet, - b"sysTlsInternalReportCreate" => Permission::SysTlsInternalReportCreate, b"sysTlsInternalReportUpdate" => Permission::SysTlsInternalReportUpdate, b"sysTlsInternalReportDestroy" => Permission::SysTlsInternalReportDestroy, b"sysTlsInternalReportQuery" => Permission::SysTlsInternalReportQuery, b"sysTlsReportSettingsGet" => Permission::SysTlsReportSettingsGet, b"sysTlsReportSettingsUpdate" => Permission::SysTlsReportSettingsUpdate, - b"sysTraceGet" => Permission::SysTraceGet, - b"sysTraceCreate" => Permission::SysTraceCreate, - b"sysTraceUpdate" => Permission::SysTraceUpdate, - b"sysTraceDestroy" => Permission::SysTraceDestroy, - b"sysTraceQuery" => Permission::SysTraceQuery, b"sysTracerGet" => Permission::SysTracerGet, b"sysTracerCreate" => Permission::SysTracerCreate, b"sysTracerUpdate" => Permission::SysTracerUpdate, b"sysTracerDestroy" => Permission::SysTracerDestroy, b"sysTracerQuery" => Permission::SysTracerQuery, - b"sysTracingStoreGet" => Permission::SysTracingStoreGet, - b"sysTracingStoreUpdate" => Permission::SysTracingStoreUpdate, b"sysWebDavGet" => Permission::SysWebDavGet, b"sysWebDavUpdate" => Permission::SysWebDavUpdate, b"sysWebHookGet" => Permission::SysWebHookGet, @@ -7524,7 +7910,6 @@ impl EnumImpl for Permission { match self { Permission::Authenticate => "authenticate", Permission::AuthenticateWithAlias => "authenticateWithAlias", - Permission::InteractAi => "interactAi", Permission::Impersonate => "impersonate", Permission::UnlimitedRequests => "unlimitedRequests", Permission::UnlimitedUploads => "unlimitedUploads", @@ -7585,9 +7970,6 @@ impl EnumImpl for Permission { Permission::JmapPrincipalChanges => "jmapPrincipalChanges", Permission::JmapPrincipalQueryChanges => "jmapPrincipalQueryChanges", Permission::JmapPrincipalGetAvailability => "jmapPrincipalGetAvailability", - Permission::JmapPrincipalCreate => "jmapPrincipalCreate", - Permission::JmapPrincipalUpdate => "jmapPrincipalUpdate", - Permission::JmapPrincipalDestroy => "jmapPrincipalDestroy", Permission::JmapQuotaGet => "jmapQuotaGet", Permission::JmapQuotaChanges => "jmapQuotaChanges", Permission::JmapQuotaQuery => "jmapQuotaQuery", @@ -7662,6 +8044,9 @@ impl EnumImpl for Permission { Permission::JmapParticipantIdentityUpdate => "jmapParticipantIdentityUpdate", Permission::JmapParticipantIdentityDestroy => "jmapParticipantIdentityDestroy", Permission::JmapCoreEcho => "jmapCoreEcho", + Permission::JmapMetadataGet => "jmapMetadataGet", + Permission::JmapMetadataSet => "jmapMetadataSet", + Permission::JmapMetadataPrivate => "jmapMetadataPrivate", Permission::ImapAuthenticate => "imapAuthenticate", Permission::ImapAclGet => "imapAclGet", Permission::ImapAclSet => "imapAclSet", @@ -7690,6 +8075,9 @@ impl EnumImpl for Permission { Permission::ImapStore => "imapStore", Permission::ImapSubscribe => "imapSubscribe", Permission::ImapThread => "imapThread", + Permission::ImapMetadataGet => "imapMetadataGet", + Permission::ImapMetadataSet => "imapMetadataSet", + Permission::ImapMetadataPrivate => "imapMetadataPrivate", Permission::Pop3Authenticate => "pop3Authenticate", Permission::Pop3List => "pop3List", Permission::Pop3Uidl => "pop3Uidl", @@ -7747,12 +8135,7 @@ impl EnumImpl for Permission { Permission::DavCalQuery => "davCalQuery", Permission::DavCalMultiGet => "davCalMultiGet", Permission::DavCalFreeBusyQuery => "davCalFreeBusyQuery", - Permission::OAuthClientRegistration => "oAuthClientRegistration", - Permission::OAuthClientOverride => "oAuthClientOverride", - Permission::LiveTracing => "liveTracing", - Permission::LiveMetrics => "liveMetrics", Permission::LiveDeliveryTest => "liveDeliveryTest", - Permission::ScimAccess => "scimAccess", Permission::SysAccountGet => "sysAccountGet", Permission::SysAccountCreate => "sysAccountCreate", Permission::SysAccountUpdate => "sysAccountUpdate", @@ -7778,23 +8161,9 @@ impl EnumImpl for Permission { Permission::ActionInvalidateNegativeCaches => "actionInvalidateNegativeCaches", Permission::ActionPauseMtaQueue => "actionPauseMtaQueue", Permission::ActionResumeMtaQueue => "actionResumeMtaQueue", - Permission::SysActionGet => "sysActionGet", Permission::SysActionCreate => "sysActionCreate", - Permission::SysActionUpdate => "sysActionUpdate", - Permission::SysActionDestroy => "sysActionDestroy", - Permission::SysActionQuery => "sysActionQuery", Permission::SysAddressBookGet => "sysAddressBookGet", Permission::SysAddressBookUpdate => "sysAddressBookUpdate", - Permission::SysAiModelGet => "sysAiModelGet", - Permission::SysAiModelCreate => "sysAiModelCreate", - Permission::SysAiModelUpdate => "sysAiModelUpdate", - Permission::SysAiModelDestroy => "sysAiModelDestroy", - Permission::SysAiModelQuery => "sysAiModelQuery", - Permission::SysAlertGet => "sysAlertGet", - Permission::SysAlertCreate => "sysAlertCreate", - Permission::SysAlertUpdate => "sysAlertUpdate", - Permission::SysAlertDestroy => "sysAlertDestroy", - Permission::SysAlertQuery => "sysAlertQuery", Permission::SysAllowedIpGet => "sysAllowedIpGet", Permission::SysAllowedIpCreate => "sysAllowedIpCreate", Permission::SysAllowedIpUpdate => "sysAllowedIpUpdate", @@ -7815,14 +8184,7 @@ impl EnumImpl for Permission { Permission::SysApplicationUpdate => "sysApplicationUpdate", Permission::SysApplicationDestroy => "sysApplicationDestroy", Permission::SysApplicationQuery => "sysApplicationQuery", - Permission::SysArchivedItemGet => "sysArchivedItemGet", - Permission::SysArchivedItemCreate => "sysArchivedItemCreate", - Permission::SysArchivedItemUpdate => "sysArchivedItemUpdate", - Permission::SysArchivedItemDestroy => "sysArchivedItemDestroy", - Permission::SysArchivedItemQuery => "sysArchivedItemQuery", Permission::SysArfExternalReportGet => "sysArfExternalReportGet", - Permission::SysArfExternalReportCreate => "sysArfExternalReportCreate", - Permission::SysArfExternalReportUpdate => "sysArfExternalReportUpdate", Permission::SysArfExternalReportDestroy => "sysArfExternalReportDestroy", Permission::SysArfExternalReportQuery => "sysArfExternalReportQuery", Permission::SysAsnGet => "sysAsnGet", @@ -7852,9 +8214,6 @@ impl EnumImpl for Permission { Permission::SysCertificateDestroy => "sysCertificateDestroy", Permission::SysCertificateQuery => "sysCertificateQuery", Permission::SysClusterNodeGet => "sysClusterNodeGet", - Permission::SysClusterNodeCreate => "sysClusterNodeCreate", - Permission::SysClusterNodeUpdate => "sysClusterNodeUpdate", - Permission::SysClusterNodeDestroy => "sysClusterNodeDestroy", Permission::SysClusterNodeQuery => "sysClusterNodeQuery", Permission::SysClusterRoleGet => "sysClusterRoleGet", Permission::SysClusterRoleCreate => "sysClusterRoleCreate", @@ -7880,12 +8239,9 @@ impl EnumImpl for Permission { Permission::SysDkimSignatureDestroy => "sysDkimSignatureDestroy", Permission::SysDkimSignatureQuery => "sysDkimSignatureQuery", Permission::SysDmarcExternalReportGet => "sysDmarcExternalReportGet", - Permission::SysDmarcExternalReportCreate => "sysDmarcExternalReportCreate", - Permission::SysDmarcExternalReportUpdate => "sysDmarcExternalReportUpdate", Permission::SysDmarcExternalReportDestroy => "sysDmarcExternalReportDestroy", Permission::SysDmarcExternalReportQuery => "sysDmarcExternalReportQuery", Permission::SysDmarcInternalReportGet => "sysDmarcInternalReportGet", - Permission::SysDmarcInternalReportCreate => "sysDmarcInternalReportCreate", Permission::SysDmarcInternalReportUpdate => "sysDmarcInternalReportUpdate", Permission::SysDmarcInternalReportDestroy => "sysDmarcInternalReportDestroy", Permission::SysDmarcInternalReportQuery => "sysDmarcInternalReportQuery", @@ -7907,8 +8263,6 @@ impl EnumImpl for Permission { Permission::SysDsnReportSettingsUpdate => "sysDsnReportSettingsUpdate", Permission::SysEmailGet => "sysEmailGet", Permission::SysEmailUpdate => "sysEmailUpdate", - Permission::SysEnterpriseGet => "sysEnterpriseGet", - Permission::SysEnterpriseUpdate => "sysEnterpriseUpdate", Permission::SysEventTracingLevelGet => "sysEventTracingLevelGet", Permission::SysEventTracingLevelCreate => "sysEventTracingLevelCreate", Permission::SysEventTracingLevelUpdate => "sysEventTracingLevelUpdate", @@ -7929,23 +8283,18 @@ impl EnumImpl for Permission { Permission::SysImapUpdate => "sysImapUpdate", Permission::SysInMemoryStoreGet => "sysInMemoryStoreGet", Permission::SysInMemoryStoreUpdate => "sysInMemoryStoreUpdate", + Permission::SysIndexQueueStatusGet => "sysIndexQueueStatusGet", + Permission::SysIndexQueueStatusUpdate => "sysIndexQueueStatusUpdate", + Permission::SysIndexQueueStatusQuery => "sysIndexQueueStatusQuery", Permission::SysJmapGet => "sysJmapGet", Permission::SysJmapUpdate => "sysJmapUpdate", Permission::SysLogGet => "sysLogGet", - Permission::SysLogCreate => "sysLogCreate", - Permission::SysLogUpdate => "sysLogUpdate", - Permission::SysLogDestroy => "sysLogDestroy", Permission::SysLogQuery => "sysLogQuery", Permission::SysMailingListGet => "sysMailingListGet", Permission::SysMailingListCreate => "sysMailingListCreate", Permission::SysMailingListUpdate => "sysMailingListUpdate", Permission::SysMailingListDestroy => "sysMailingListDestroy", Permission::SysMailingListQuery => "sysMailingListQuery", - Permission::SysMaskedEmailGet => "sysMaskedEmailGet", - Permission::SysMaskedEmailCreate => "sysMaskedEmailCreate", - Permission::SysMaskedEmailUpdate => "sysMaskedEmailUpdate", - Permission::SysMaskedEmailDestroy => "sysMaskedEmailDestroy", - Permission::SysMaskedEmailQuery => "sysMaskedEmailQuery", Permission::SysMemoryLookupKeyGet => "sysMemoryLookupKeyGet", Permission::SysMemoryLookupKeyCreate => "sysMemoryLookupKeyCreate", Permission::SysMemoryLookupKeyUpdate => "sysMemoryLookupKeyUpdate", @@ -7956,15 +8305,10 @@ impl EnumImpl for Permission { Permission::SysMemoryLookupKeyValueUpdate => "sysMemoryLookupKeyValueUpdate", Permission::SysMemoryLookupKeyValueDestroy => "sysMemoryLookupKeyValueDestroy", Permission::SysMemoryLookupKeyValueQuery => "sysMemoryLookupKeyValueQuery", - Permission::SysMetricGet => "sysMetricGet", - Permission::SysMetricCreate => "sysMetricCreate", - Permission::SysMetricUpdate => "sysMetricUpdate", - Permission::SysMetricDestroy => "sysMetricDestroy", - Permission::SysMetricQuery => "sysMetricQuery", + Permission::SysMetadataGet => "sysMetadataGet", + Permission::SysMetadataUpdate => "sysMetadataUpdate", Permission::SysMetricsGet => "sysMetricsGet", Permission::SysMetricsUpdate => "sysMetricsUpdate", - Permission::SysMetricsStoreGet => "sysMetricsStoreGet", - Permission::SysMetricsStoreUpdate => "sysMetricsStoreUpdate", Permission::SysMtaConnectionStrategyGet => "sysMtaConnectionStrategyGet", Permission::SysMtaConnectionStrategyCreate => "sysMtaConnectionStrategyCreate", Permission::SysMtaConnectionStrategyUpdate => "sysMtaConnectionStrategyUpdate", @@ -8045,6 +8389,9 @@ impl EnumImpl for Permission { Permission::SysOAuthClientUpdate => "sysOAuthClientUpdate", Permission::SysOAuthClientDestroy => "sysOAuthClientDestroy", Permission::SysOAuthClientQuery => "sysOAuthClientQuery", + Permission::SysOAuthGrantGet => "sysOAuthGrantGet", + Permission::SysOAuthGrantDestroy => "sysOAuthGrantDestroy", + Permission::SysOAuthGrantQuery => "sysOAuthGrantQuery", Permission::SysOidcProviderGet => "sysOidcProviderGet", Permission::SysOidcProviderUpdate => "sysOidcProviderUpdate", Permission::SysPublicKeyGet => "sysPublicKeyGet", @@ -8053,7 +8400,6 @@ impl EnumImpl for Permission { Permission::SysPublicKeyDestroy => "sysPublicKeyDestroy", Permission::SysPublicKeyQuery => "sysPublicKeyQuery", Permission::SysQueuedMessageGet => "sysQueuedMessageGet", - Permission::SysQueuedMessageCreate => "sysQueuedMessageCreate", Permission::SysQueuedMessageUpdate => "sysQueuedMessageUpdate", Permission::SysQueuedMessageDestroy => "sysQueuedMessageDestroy", Permission::SysQueuedMessageQuery => "sysQueuedMessageQuery", @@ -8102,8 +8448,6 @@ impl EnumImpl for Permission { Permission::SysSpamFileExtensionUpdate => "sysSpamFileExtensionUpdate", Permission::SysSpamFileExtensionDestroy => "sysSpamFileExtensionDestroy", Permission::SysSpamFileExtensionQuery => "sysSpamFileExtensionQuery", - Permission::SysSpamLlmGet => "sysSpamLlmGet", - Permission::SysSpamLlmUpdate => "sysSpamLlmUpdate", Permission::SysSpamPyzorGet => "sysSpamPyzorGet", Permission::SysSpamPyzorUpdate => "sysSpamPyzorUpdate", Permission::SysSpamRuleGet => "sysSpamRuleGet", @@ -8120,7 +8464,6 @@ impl EnumImpl for Permission { Permission::SysSpamTagQuery => "sysSpamTagQuery", Permission::SysSpamTrainingSampleGet => "sysSpamTrainingSampleGet", Permission::SysSpamTrainingSampleCreate => "sysSpamTrainingSampleCreate", - Permission::SysSpamTrainingSampleUpdate => "sysSpamTrainingSampleUpdate", Permission::SysSpamTrainingSampleDestroy => "sysSpamTrainingSampleDestroy", Permission::SysSpamTrainingSampleQuery => "sysSpamTrainingSampleQuery", Permission::SysSpfReportSettingsGet => "sysSpfReportSettingsGet", @@ -8132,24 +8475,22 @@ impl EnumImpl for Permission { Permission::SysStoreLookupQuery => "sysStoreLookupQuery", Permission::SysSystemSettingsGet => "sysSystemSettingsGet", Permission::SysSystemSettingsUpdate => "sysSystemSettingsUpdate", - Permission::TaskIndexDocument => "taskIndexDocument", - Permission::TaskUnindexDocument => "taskUnindexDocument", - Permission::TaskIndexTrace => "taskIndexTrace", Permission::TaskCalendarAlarmEmail => "taskCalendarAlarmEmail", Permission::TaskCalendarAlarmNotification => "taskCalendarAlarmNotification", Permission::TaskCalendarItipMessage => "taskCalendarItipMessage", Permission::TaskMergeThreads => "taskMergeThreads", Permission::TaskDmarcReport => "taskDmarcReport", Permission::TaskTlsReport => "taskTlsReport", - Permission::TaskRestoreArchivedItem => "taskRestoreArchivedItem", Permission::TaskDestroyAccount => "taskDestroyAccount", Permission::TaskAccountMaintenance => "taskAccountMaintenance", - Permission::TaskTenantMaintenance => "taskTenantMaintenance", Permission::TaskStoreMaintenance => "taskStoreMaintenance", Permission::TaskSpamFilterMaintenance => "taskSpamFilterMaintenance", Permission::TaskAcmeRenewal => "taskAcmeRenewal", Permission::TaskDkimManagement => "taskDkimManagement", Permission::TaskDnsManagement => "taskDnsManagement", + Permission::TaskShareRevokeCleanup => "taskShareRevokeCleanup", + Permission::TaskDmarcReportWindow => "taskDmarcReportWindow", + Permission::TaskTlsReportWindow => "taskTlsReportWindow", Permission::SysTaskGet => "sysTaskGet", Permission::SysTaskCreate => "sysTaskCreate", Permission::SysTaskUpdate => "sysTaskUpdate", @@ -8157,35 +8498,20 @@ impl EnumImpl for Permission { Permission::SysTaskQuery => "sysTaskQuery", Permission::SysTaskManagerGet => "sysTaskManagerGet", Permission::SysTaskManagerUpdate => "sysTaskManagerUpdate", - Permission::SysTenantGet => "sysTenantGet", - Permission::SysTenantCreate => "sysTenantCreate", - Permission::SysTenantUpdate => "sysTenantUpdate", - Permission::SysTenantDestroy => "sysTenantDestroy", - Permission::SysTenantQuery => "sysTenantQuery", Permission::SysTlsExternalReportGet => "sysTlsExternalReportGet", - Permission::SysTlsExternalReportCreate => "sysTlsExternalReportCreate", - Permission::SysTlsExternalReportUpdate => "sysTlsExternalReportUpdate", Permission::SysTlsExternalReportDestroy => "sysTlsExternalReportDestroy", Permission::SysTlsExternalReportQuery => "sysTlsExternalReportQuery", Permission::SysTlsInternalReportGet => "sysTlsInternalReportGet", - Permission::SysTlsInternalReportCreate => "sysTlsInternalReportCreate", Permission::SysTlsInternalReportUpdate => "sysTlsInternalReportUpdate", Permission::SysTlsInternalReportDestroy => "sysTlsInternalReportDestroy", Permission::SysTlsInternalReportQuery => "sysTlsInternalReportQuery", Permission::SysTlsReportSettingsGet => "sysTlsReportSettingsGet", Permission::SysTlsReportSettingsUpdate => "sysTlsReportSettingsUpdate", - Permission::SysTraceGet => "sysTraceGet", - Permission::SysTraceCreate => "sysTraceCreate", - Permission::SysTraceUpdate => "sysTraceUpdate", - Permission::SysTraceDestroy => "sysTraceDestroy", - Permission::SysTraceQuery => "sysTraceQuery", Permission::SysTracerGet => "sysTracerGet", Permission::SysTracerCreate => "sysTracerCreate", Permission::SysTracerUpdate => "sysTracerUpdate", Permission::SysTracerDestroy => "sysTracerDestroy", Permission::SysTracerQuery => "sysTracerQuery", - Permission::SysTracingStoreGet => "sysTracingStoreGet", - Permission::SysTracingStoreUpdate => "sysTracingStoreUpdate", Permission::SysWebDavGet => "sysWebDavGet", Permission::SysWebDavUpdate => "sysWebDavUpdate", Permission::SysWebHookGet => "sysWebHookGet", @@ -8204,7 +8530,6 @@ impl EnumImpl for Permission { match id { 0 => Some(Permission::Authenticate), 1 => Some(Permission::AuthenticateWithAlias), - 2 => Some(Permission::InteractAi), 3 => Some(Permission::Impersonate), 4 => Some(Permission::UnlimitedRequests), 5 => Some(Permission::UnlimitedUploads), @@ -8265,609 +8590,1156 @@ impl EnumImpl for Permission { 60 => Some(Permission::JmapPrincipalChanges), 61 => Some(Permission::JmapPrincipalQueryChanges), 62 => Some(Permission::JmapPrincipalGetAvailability), - 63 => Some(Permission::JmapPrincipalCreate), - 64 => Some(Permission::JmapPrincipalUpdate), - 65 => Some(Permission::JmapPrincipalDestroy), - 66 => Some(Permission::JmapQuotaGet), - 67 => Some(Permission::JmapQuotaChanges), - 68 => Some(Permission::JmapQuotaQuery), - 69 => Some(Permission::JmapQuotaQueryChanges), - 70 => Some(Permission::JmapBlobGet), - 71 => Some(Permission::JmapBlobCopy), - 72 => Some(Permission::JmapBlobLookup), - 73 => Some(Permission::JmapBlobUpload), - 74 => Some(Permission::JmapAddressBookGet), - 75 => Some(Permission::JmapAddressBookChanges), - 76 => Some(Permission::JmapAddressBookCreate), - 77 => Some(Permission::JmapAddressBookUpdate), - 78 => Some(Permission::JmapAddressBookDestroy), - 79 => Some(Permission::JmapContactCardGet), - 80 => Some(Permission::JmapContactCardChanges), - 81 => Some(Permission::JmapContactCardQuery), - 82 => Some(Permission::JmapContactCardQueryChanges), - 83 => Some(Permission::JmapContactCardCreate), - 84 => Some(Permission::JmapContactCardUpdate), - 85 => Some(Permission::JmapContactCardDestroy), - 86 => Some(Permission::JmapContactCardCopy), - 87 => Some(Permission::JmapContactCardParse), - 88 => Some(Permission::JmapFileNodeGet), - 89 => Some(Permission::JmapFileNodeChanges), - 90 => Some(Permission::JmapFileNodeQuery), - 91 => Some(Permission::JmapFileNodeQueryChanges), - 92 => Some(Permission::JmapFileNodeCreate), - 93 => Some(Permission::JmapFileNodeUpdate), - 94 => Some(Permission::JmapFileNodeDestroy), - 659 => Some(Permission::JmapFileNodeCopy), - 95 => Some(Permission::JmapShareNotificationGet), - 96 => Some(Permission::JmapShareNotificationChanges), - 97 => Some(Permission::JmapShareNotificationQuery), - 98 => Some(Permission::JmapShareNotificationQueryChanges), - 99 => Some(Permission::JmapShareNotificationCreate), - 100 => Some(Permission::JmapShareNotificationUpdate), - 101 => Some(Permission::JmapShareNotificationDestroy), - 102 => Some(Permission::JmapCalendarGet), - 103 => Some(Permission::JmapCalendarChanges), - 104 => Some(Permission::JmapCalendarCreate), - 105 => Some(Permission::JmapCalendarUpdate), - 106 => Some(Permission::JmapCalendarDestroy), - 107 => Some(Permission::JmapCalendarEventGet), - 108 => Some(Permission::JmapCalendarEventChanges), - 109 => Some(Permission::JmapCalendarEventQuery), - 110 => Some(Permission::JmapCalendarEventQueryChanges), - 111 => Some(Permission::JmapCalendarEventCreate), - 112 => Some(Permission::JmapCalendarEventUpdate), - 113 => Some(Permission::JmapCalendarEventDestroy), - 114 => Some(Permission::JmapCalendarEventCopy), - 115 => Some(Permission::JmapCalendarEventParse), - 116 => Some(Permission::JmapCalendarEventNotificationGet), - 117 => Some(Permission::JmapCalendarEventNotificationChanges), - 118 => Some(Permission::JmapCalendarEventNotificationQuery), - 119 => Some(Permission::JmapCalendarEventNotificationQueryChanges), - 120 => Some(Permission::JmapCalendarEventNotificationCreate), - 121 => Some(Permission::JmapCalendarEventNotificationUpdate), - 122 => Some(Permission::JmapCalendarEventNotificationDestroy), - 123 => Some(Permission::JmapParticipantIdentityGet), - 124 => Some(Permission::JmapParticipantIdentityChanges), - 125 => Some(Permission::JmapParticipantIdentityCreate), - 126 => Some(Permission::JmapParticipantIdentityUpdate), - 127 => Some(Permission::JmapParticipantIdentityDestroy), - 128 => Some(Permission::JmapCoreEcho), - 129 => Some(Permission::ImapAuthenticate), - 130 => Some(Permission::ImapAclGet), - 131 => Some(Permission::ImapAclSet), - 132 => Some(Permission::ImapMyRights), - 133 => Some(Permission::ImapListRights), - 134 => Some(Permission::ImapAppend), - 135 => Some(Permission::ImapCapability), - 136 => Some(Permission::ImapId), - 137 => Some(Permission::ImapCopy), - 138 => Some(Permission::ImapMove), - 139 => Some(Permission::ImapCreate), - 140 => Some(Permission::ImapDelete), - 141 => Some(Permission::ImapEnable), - 142 => Some(Permission::ImapExpunge), - 143 => Some(Permission::ImapFetch), - 144 => Some(Permission::ImapIdle), - 145 => Some(Permission::ImapList), - 146 => Some(Permission::ImapLsub), - 147 => Some(Permission::ImapNamespace), - 148 => Some(Permission::ImapRename), - 149 => Some(Permission::ImapSearch), - 150 => Some(Permission::ImapSort), - 151 => Some(Permission::ImapSelect), - 152 => Some(Permission::ImapExamine), - 153 => Some(Permission::ImapStatus), - 154 => Some(Permission::ImapStore), - 155 => Some(Permission::ImapSubscribe), - 156 => Some(Permission::ImapThread), - 157 => Some(Permission::Pop3Authenticate), - 158 => Some(Permission::Pop3List), - 159 => Some(Permission::Pop3Uidl), - 160 => Some(Permission::Pop3Stat), - 161 => Some(Permission::Pop3Retr), - 162 => Some(Permission::Pop3Dele), - 163 => Some(Permission::SieveAuthenticate), - 164 => Some(Permission::SieveListScripts), - 165 => Some(Permission::SieveSetActive), - 166 => Some(Permission::SieveGetScript), - 167 => Some(Permission::SievePutScript), - 168 => Some(Permission::SieveDeleteScript), - 169 => Some(Permission::SieveRenameScript), - 170 => Some(Permission::SieveCheckScript), - 171 => Some(Permission::SieveHaveSpace), - 172 => Some(Permission::DavSyncCollection), - 173 => Some(Permission::DavExpandProperty), - 174 => Some(Permission::DavPrincipalAcl), - 175 => Some(Permission::DavPrincipalList), - 176 => Some(Permission::DavPrincipalMatch), - 177 => Some(Permission::DavPrincipalSearch), - 178 => Some(Permission::DavPrincipalSearchPropSet), - 179 => Some(Permission::DavFilePropFind), - 180 => Some(Permission::DavFilePropPatch), - 181 => Some(Permission::DavFileGet), - 182 => Some(Permission::DavFileMkCol), - 183 => Some(Permission::DavFileDelete), - 184 => Some(Permission::DavFilePut), - 185 => Some(Permission::DavFileCopy), - 186 => Some(Permission::DavFileMove), - 187 => Some(Permission::DavFileLock), - 188 => Some(Permission::DavFileAcl), - 189 => Some(Permission::DavCardPropFind), - 190 => Some(Permission::DavCardPropPatch), - 191 => Some(Permission::DavCardGet), - 192 => Some(Permission::DavCardMkCol), - 193 => Some(Permission::DavCardDelete), - 194 => Some(Permission::DavCardPut), - 195 => Some(Permission::DavCardCopy), - 196 => Some(Permission::DavCardMove), - 197 => Some(Permission::DavCardLock), - 198 => Some(Permission::DavCardAcl), - 199 => Some(Permission::DavCardQuery), - 200 => Some(Permission::DavCardMultiGet), - 201 => Some(Permission::DavCalPropFind), - 202 => Some(Permission::DavCalPropPatch), - 203 => Some(Permission::DavCalGet), - 204 => Some(Permission::DavCalMkCol), - 205 => Some(Permission::DavCalDelete), - 206 => Some(Permission::DavCalPut), - 207 => Some(Permission::DavCalCopy), - 208 => Some(Permission::DavCalMove), - 209 => Some(Permission::DavCalLock), - 210 => Some(Permission::DavCalAcl), - 211 => Some(Permission::DavCalQuery), - 212 => Some(Permission::DavCalMultiGet), - 213 => Some(Permission::DavCalFreeBusyQuery), - 214 => Some(Permission::OAuthClientRegistration), - 215 => Some(Permission::OAuthClientOverride), - 216 => Some(Permission::LiveTracing), - 217 => Some(Permission::LiveMetrics), - 218 => Some(Permission::LiveDeliveryTest), - 660 => Some(Permission::ScimAccess), - 219 => Some(Permission::SysAccountGet), - 220 => Some(Permission::SysAccountCreate), - 221 => Some(Permission::SysAccountUpdate), - 222 => Some(Permission::SysAccountDestroy), - 223 => Some(Permission::SysAccountQuery), - 224 => Some(Permission::SysAccountPasswordGet), - 225 => Some(Permission::SysAccountPasswordUpdate), - 226 => Some(Permission::SysAccountSettingsGet), - 227 => Some(Permission::SysAccountSettingsUpdate), - 228 => Some(Permission::SysAcmeProviderGet), - 229 => Some(Permission::SysAcmeProviderCreate), - 230 => Some(Permission::SysAcmeProviderUpdate), - 231 => Some(Permission::SysAcmeProviderDestroy), - 232 => Some(Permission::SysAcmeProviderQuery), - 233 => Some(Permission::ActionReloadSettings), - 234 => Some(Permission::ActionReloadTlsCertificates), - 235 => Some(Permission::ActionReloadLookupStores), - 236 => Some(Permission::ActionReloadBlockedIps), - 237 => Some(Permission::ActionUpdateApps), - 238 => Some(Permission::ActionTroubleshootDmarc), - 239 => Some(Permission::ActionClassifySpam), - 240 => Some(Permission::ActionInvalidateCaches), - 241 => Some(Permission::ActionInvalidateNegativeCaches), - 242 => Some(Permission::ActionPauseMtaQueue), - 243 => Some(Permission::ActionResumeMtaQueue), - 244 => Some(Permission::SysActionGet), - 245 => Some(Permission::SysActionCreate), - 246 => Some(Permission::SysActionUpdate), - 247 => Some(Permission::SysActionDestroy), - 248 => Some(Permission::SysActionQuery), - 249 => Some(Permission::SysAddressBookGet), - 250 => Some(Permission::SysAddressBookUpdate), - 251 => Some(Permission::SysAiModelGet), - 252 => Some(Permission::SysAiModelCreate), - 253 => Some(Permission::SysAiModelUpdate), - 254 => Some(Permission::SysAiModelDestroy), - 255 => Some(Permission::SysAiModelQuery), - 256 => Some(Permission::SysAlertGet), - 257 => Some(Permission::SysAlertCreate), - 258 => Some(Permission::SysAlertUpdate), - 259 => Some(Permission::SysAlertDestroy), - 260 => Some(Permission::SysAlertQuery), - 261 => Some(Permission::SysAllowedIpGet), - 262 => Some(Permission::SysAllowedIpCreate), - 263 => Some(Permission::SysAllowedIpUpdate), - 264 => Some(Permission::SysAllowedIpDestroy), - 265 => Some(Permission::SysAllowedIpQuery), - 266 => Some(Permission::SysApiKeyGet), - 267 => Some(Permission::SysApiKeyCreate), - 268 => Some(Permission::SysApiKeyUpdate), - 269 => Some(Permission::SysApiKeyDestroy), - 270 => Some(Permission::SysApiKeyQuery), - 271 => Some(Permission::SysAppPasswordGet), - 272 => Some(Permission::SysAppPasswordCreate), - 273 => Some(Permission::SysAppPasswordUpdate), - 274 => Some(Permission::SysAppPasswordDestroy), - 275 => Some(Permission::SysAppPasswordQuery), - 276 => Some(Permission::SysApplicationGet), - 277 => Some(Permission::SysApplicationCreate), - 278 => Some(Permission::SysApplicationUpdate), - 279 => Some(Permission::SysApplicationDestroy), - 280 => Some(Permission::SysApplicationQuery), - 281 => Some(Permission::SysArchivedItemGet), - 282 => Some(Permission::SysArchivedItemCreate), - 283 => Some(Permission::SysArchivedItemUpdate), - 284 => Some(Permission::SysArchivedItemDestroy), - 285 => Some(Permission::SysArchivedItemQuery), - 286 => Some(Permission::SysArfExternalReportGet), - 287 => Some(Permission::SysArfExternalReportCreate), - 288 => Some(Permission::SysArfExternalReportUpdate), - 289 => Some(Permission::SysArfExternalReportDestroy), - 290 => Some(Permission::SysArfExternalReportQuery), - 291 => Some(Permission::SysAsnGet), - 292 => Some(Permission::SysAsnUpdate), - 293 => Some(Permission::SysAuthenticationGet), - 294 => Some(Permission::SysAuthenticationUpdate), - 295 => Some(Permission::SysBlobStoreGet), - 296 => Some(Permission::SysBlobStoreUpdate), - 297 => Some(Permission::SysBlockedIpGet), - 298 => Some(Permission::SysBlockedIpCreate), - 299 => Some(Permission::SysBlockedIpUpdate), - 300 => Some(Permission::SysBlockedIpDestroy), - 301 => Some(Permission::SysBlockedIpQuery), - 302 => Some(Permission::SysBootstrapGet), - 303 => Some(Permission::SysBootstrapUpdate), - 304 => Some(Permission::SysCacheGet), - 305 => Some(Permission::SysCacheUpdate), - 306 => Some(Permission::SysCalendarGet), - 307 => Some(Permission::SysCalendarUpdate), - 308 => Some(Permission::SysCalendarAlarmGet), - 309 => Some(Permission::SysCalendarAlarmUpdate), - 310 => Some(Permission::SysCalendarSchedulingGet), - 311 => Some(Permission::SysCalendarSchedulingUpdate), - 312 => Some(Permission::SysCertificateGet), - 313 => Some(Permission::SysCertificateCreate), - 314 => Some(Permission::SysCertificateUpdate), - 315 => Some(Permission::SysCertificateDestroy), - 316 => Some(Permission::SysCertificateQuery), - 317 => Some(Permission::SysClusterNodeGet), - 318 => Some(Permission::SysClusterNodeCreate), - 319 => Some(Permission::SysClusterNodeUpdate), - 320 => Some(Permission::SysClusterNodeDestroy), - 321 => Some(Permission::SysClusterNodeQuery), - 322 => Some(Permission::SysClusterRoleGet), - 323 => Some(Permission::SysClusterRoleCreate), - 324 => Some(Permission::SysClusterRoleUpdate), - 325 => Some(Permission::SysClusterRoleDestroy), - 326 => Some(Permission::SysClusterRoleQuery), - 327 => Some(Permission::SysCoordinatorGet), - 328 => Some(Permission::SysCoordinatorUpdate), - 329 => Some(Permission::SysDataRetentionGet), - 330 => Some(Permission::SysDataRetentionUpdate), - 331 => Some(Permission::SysDataStoreGet), - 332 => Some(Permission::SysDataStoreUpdate), - 333 => Some(Permission::SysDirectoryGet), - 334 => Some(Permission::SysDirectoryCreate), - 335 => Some(Permission::SysDirectoryUpdate), - 336 => Some(Permission::SysDirectoryDestroy), - 337 => Some(Permission::SysDirectoryQuery), - 338 => Some(Permission::SysDkimReportSettingsGet), - 339 => Some(Permission::SysDkimReportSettingsUpdate), - 340 => Some(Permission::SysDkimSignatureGet), - 341 => Some(Permission::SysDkimSignatureCreate), - 342 => Some(Permission::SysDkimSignatureUpdate), - 343 => Some(Permission::SysDkimSignatureDestroy), - 344 => Some(Permission::SysDkimSignatureQuery), - 345 => Some(Permission::SysDmarcExternalReportGet), - 346 => Some(Permission::SysDmarcExternalReportCreate), - 347 => Some(Permission::SysDmarcExternalReportUpdate), - 348 => Some(Permission::SysDmarcExternalReportDestroy), - 349 => Some(Permission::SysDmarcExternalReportQuery), - 350 => Some(Permission::SysDmarcInternalReportGet), - 351 => Some(Permission::SysDmarcInternalReportCreate), - 352 => Some(Permission::SysDmarcInternalReportUpdate), - 353 => Some(Permission::SysDmarcInternalReportDestroy), - 354 => Some(Permission::SysDmarcInternalReportQuery), - 355 => Some(Permission::SysDmarcReportSettingsGet), - 356 => Some(Permission::SysDmarcReportSettingsUpdate), - 357 => Some(Permission::SysDnsResolverGet), - 358 => Some(Permission::SysDnsResolverUpdate), - 359 => Some(Permission::SysDnsServerGet), - 360 => Some(Permission::SysDnsServerCreate), - 361 => Some(Permission::SysDnsServerUpdate), - 362 => Some(Permission::SysDnsServerDestroy), - 363 => Some(Permission::SysDnsServerQuery), - 364 => Some(Permission::SysDomainGet), - 365 => Some(Permission::SysDomainCreate), - 366 => Some(Permission::SysDomainUpdate), - 367 => Some(Permission::SysDomainDestroy), - 368 => Some(Permission::SysDomainQuery), - 369 => Some(Permission::SysDsnReportSettingsGet), - 370 => Some(Permission::SysDsnReportSettingsUpdate), - 371 => Some(Permission::SysEmailGet), - 372 => Some(Permission::SysEmailUpdate), - 373 => Some(Permission::SysEnterpriseGet), - 374 => Some(Permission::SysEnterpriseUpdate), - 375 => Some(Permission::SysEventTracingLevelGet), - 376 => Some(Permission::SysEventTracingLevelCreate), - 377 => Some(Permission::SysEventTracingLevelUpdate), - 378 => Some(Permission::SysEventTracingLevelDestroy), - 379 => Some(Permission::SysEventTracingLevelQuery), - 380 => Some(Permission::SysFileStorageGet), - 381 => Some(Permission::SysFileStorageUpdate), - 382 => Some(Permission::SysHttpGet), - 383 => Some(Permission::SysHttpUpdate), - 384 => Some(Permission::SysHttpFormGet), - 385 => Some(Permission::SysHttpFormUpdate), - 386 => Some(Permission::SysHttpLookupGet), - 387 => Some(Permission::SysHttpLookupCreate), - 388 => Some(Permission::SysHttpLookupUpdate), - 389 => Some(Permission::SysHttpLookupDestroy), - 390 => Some(Permission::SysHttpLookupQuery), - 391 => Some(Permission::SysImapGet), - 392 => Some(Permission::SysImapUpdate), - 393 => Some(Permission::SysInMemoryStoreGet), - 394 => Some(Permission::SysInMemoryStoreUpdate), - 395 => Some(Permission::SysJmapGet), - 396 => Some(Permission::SysJmapUpdate), - 397 => Some(Permission::SysLogGet), - 398 => Some(Permission::SysLogCreate), - 399 => Some(Permission::SysLogUpdate), - 400 => Some(Permission::SysLogDestroy), - 401 => Some(Permission::SysLogQuery), - 402 => Some(Permission::SysMailingListGet), - 403 => Some(Permission::SysMailingListCreate), - 404 => Some(Permission::SysMailingListUpdate), - 405 => Some(Permission::SysMailingListDestroy), - 406 => Some(Permission::SysMailingListQuery), - 407 => Some(Permission::SysMaskedEmailGet), - 408 => Some(Permission::SysMaskedEmailCreate), - 409 => Some(Permission::SysMaskedEmailUpdate), - 410 => Some(Permission::SysMaskedEmailDestroy), - 411 => Some(Permission::SysMaskedEmailQuery), - 412 => Some(Permission::SysMemoryLookupKeyGet), - 413 => Some(Permission::SysMemoryLookupKeyCreate), - 414 => Some(Permission::SysMemoryLookupKeyUpdate), - 415 => Some(Permission::SysMemoryLookupKeyDestroy), - 416 => Some(Permission::SysMemoryLookupKeyQuery), - 417 => Some(Permission::SysMemoryLookupKeyValueGet), - 418 => Some(Permission::SysMemoryLookupKeyValueCreate), - 419 => Some(Permission::SysMemoryLookupKeyValueUpdate), - 420 => Some(Permission::SysMemoryLookupKeyValueDestroy), - 421 => Some(Permission::SysMemoryLookupKeyValueQuery), - 422 => Some(Permission::SysMetricGet), - 423 => Some(Permission::SysMetricCreate), - 424 => Some(Permission::SysMetricUpdate), - 425 => Some(Permission::SysMetricDestroy), - 426 => Some(Permission::SysMetricQuery), - 427 => Some(Permission::SysMetricsGet), - 428 => Some(Permission::SysMetricsUpdate), - 429 => Some(Permission::SysMetricsStoreGet), - 430 => Some(Permission::SysMetricsStoreUpdate), - 431 => Some(Permission::SysMtaConnectionStrategyGet), - 432 => Some(Permission::SysMtaConnectionStrategyCreate), - 433 => Some(Permission::SysMtaConnectionStrategyUpdate), - 434 => Some(Permission::SysMtaConnectionStrategyDestroy), - 435 => Some(Permission::SysMtaConnectionStrategyQuery), - 436 => Some(Permission::SysMtaDeliveryScheduleGet), - 437 => Some(Permission::SysMtaDeliveryScheduleCreate), - 438 => Some(Permission::SysMtaDeliveryScheduleUpdate), - 439 => Some(Permission::SysMtaDeliveryScheduleDestroy), - 440 => Some(Permission::SysMtaDeliveryScheduleQuery), - 441 => Some(Permission::SysMtaExtensionsGet), - 442 => Some(Permission::SysMtaExtensionsUpdate), - 443 => Some(Permission::SysMtaHookGet), - 444 => Some(Permission::SysMtaHookCreate), - 445 => Some(Permission::SysMtaHookUpdate), - 446 => Some(Permission::SysMtaHookDestroy), - 447 => Some(Permission::SysMtaHookQuery), - 448 => Some(Permission::SysMtaInboundSessionGet), - 449 => Some(Permission::SysMtaInboundSessionUpdate), - 450 => Some(Permission::SysMtaInboundThrottleGet), - 451 => Some(Permission::SysMtaInboundThrottleCreate), - 452 => Some(Permission::SysMtaInboundThrottleUpdate), - 453 => Some(Permission::SysMtaInboundThrottleDestroy), - 454 => Some(Permission::SysMtaInboundThrottleQuery), - 455 => Some(Permission::SysMtaMilterGet), - 456 => Some(Permission::SysMtaMilterCreate), - 457 => Some(Permission::SysMtaMilterUpdate), - 458 => Some(Permission::SysMtaMilterDestroy), - 459 => Some(Permission::SysMtaMilterQuery), - 460 => Some(Permission::SysMtaOutboundStrategyGet), - 461 => Some(Permission::SysMtaOutboundStrategyUpdate), - 462 => Some(Permission::SysMtaOutboundThrottleGet), - 463 => Some(Permission::SysMtaOutboundThrottleCreate), - 464 => Some(Permission::SysMtaOutboundThrottleUpdate), - 465 => Some(Permission::SysMtaOutboundThrottleDestroy), - 466 => Some(Permission::SysMtaOutboundThrottleQuery), - 467 => Some(Permission::SysMtaQueueQuotaGet), - 468 => Some(Permission::SysMtaQueueQuotaCreate), - 469 => Some(Permission::SysMtaQueueQuotaUpdate), - 470 => Some(Permission::SysMtaQueueQuotaDestroy), - 471 => Some(Permission::SysMtaQueueQuotaQuery), - 472 => Some(Permission::SysMtaRouteGet), - 473 => Some(Permission::SysMtaRouteCreate), - 474 => Some(Permission::SysMtaRouteUpdate), - 475 => Some(Permission::SysMtaRouteDestroy), - 476 => Some(Permission::SysMtaRouteQuery), - 477 => Some(Permission::SysMtaStageAuthGet), - 478 => Some(Permission::SysMtaStageAuthUpdate), - 479 => Some(Permission::SysMtaStageConnectGet), - 480 => Some(Permission::SysMtaStageConnectUpdate), - 481 => Some(Permission::SysMtaStageDataGet), - 482 => Some(Permission::SysMtaStageDataUpdate), - 483 => Some(Permission::SysMtaStageEhloGet), - 484 => Some(Permission::SysMtaStageEhloUpdate), - 485 => Some(Permission::SysMtaStageMailGet), - 486 => Some(Permission::SysMtaStageMailUpdate), - 487 => Some(Permission::SysMtaStageRcptGet), - 488 => Some(Permission::SysMtaStageRcptUpdate), - 489 => Some(Permission::SysMtaStsGet), - 490 => Some(Permission::SysMtaStsUpdate), - 491 => Some(Permission::SysMtaTlsStrategyGet), - 492 => Some(Permission::SysMtaTlsStrategyCreate), - 493 => Some(Permission::SysMtaTlsStrategyUpdate), - 494 => Some(Permission::SysMtaTlsStrategyDestroy), - 495 => Some(Permission::SysMtaTlsStrategyQuery), - 496 => Some(Permission::SysMtaVirtualQueueGet), - 497 => Some(Permission::SysMtaVirtualQueueCreate), - 498 => Some(Permission::SysMtaVirtualQueueUpdate), - 499 => Some(Permission::SysMtaVirtualQueueDestroy), - 500 => Some(Permission::SysMtaVirtualQueueQuery), - 501 => Some(Permission::SysNetworkListenerGet), - 502 => Some(Permission::SysNetworkListenerCreate), - 503 => Some(Permission::SysNetworkListenerUpdate), - 504 => Some(Permission::SysNetworkListenerDestroy), - 505 => Some(Permission::SysNetworkListenerQuery), - 506 => Some(Permission::SysOAuthClientGet), - 507 => Some(Permission::SysOAuthClientCreate), - 508 => Some(Permission::SysOAuthClientUpdate), - 509 => Some(Permission::SysOAuthClientDestroy), - 510 => Some(Permission::SysOAuthClientQuery), - 511 => Some(Permission::SysOidcProviderGet), - 512 => Some(Permission::SysOidcProviderUpdate), - 513 => Some(Permission::SysPublicKeyGet), - 514 => Some(Permission::SysPublicKeyCreate), - 515 => Some(Permission::SysPublicKeyUpdate), - 516 => Some(Permission::SysPublicKeyDestroy), - 517 => Some(Permission::SysPublicKeyQuery), - 518 => Some(Permission::SysQueuedMessageGet), - 519 => Some(Permission::SysQueuedMessageCreate), - 520 => Some(Permission::SysQueuedMessageUpdate), - 521 => Some(Permission::SysQueuedMessageDestroy), - 522 => Some(Permission::SysQueuedMessageQuery), - 523 => Some(Permission::SysReportSettingsGet), - 524 => Some(Permission::SysReportSettingsUpdate), - 525 => Some(Permission::SysRoleGet), - 526 => Some(Permission::SysRoleCreate), - 527 => Some(Permission::SysRoleUpdate), - 528 => Some(Permission::SysRoleDestroy), - 529 => Some(Permission::SysRoleQuery), - 530 => Some(Permission::SysSearchGet), - 531 => Some(Permission::SysSearchUpdate), - 532 => Some(Permission::SysSearchStoreGet), - 533 => Some(Permission::SysSearchStoreUpdate), - 534 => Some(Permission::SysSecurityGet), - 535 => Some(Permission::SysSecurityUpdate), - 536 => Some(Permission::SysSenderAuthGet), - 537 => Some(Permission::SysSenderAuthUpdate), - 538 => Some(Permission::SysSharingGet), - 539 => Some(Permission::SysSharingUpdate), - 540 => Some(Permission::SysSieveSystemInterpreterGet), - 541 => Some(Permission::SysSieveSystemInterpreterUpdate), - 542 => Some(Permission::SysSieveSystemScriptGet), - 543 => Some(Permission::SysSieveSystemScriptCreate), - 544 => Some(Permission::SysSieveSystemScriptUpdate), - 545 => Some(Permission::SysSieveSystemScriptDestroy), - 546 => Some(Permission::SysSieveSystemScriptQuery), - 547 => Some(Permission::SysSieveUserInterpreterGet), - 548 => Some(Permission::SysSieveUserInterpreterUpdate), - 549 => Some(Permission::SysSieveUserScriptGet), - 550 => Some(Permission::SysSieveUserScriptCreate), - 551 => Some(Permission::SysSieveUserScriptUpdate), - 552 => Some(Permission::SysSieveUserScriptDestroy), - 553 => Some(Permission::SysSieveUserScriptQuery), - 554 => Some(Permission::SysSpamClassifierGet), - 555 => Some(Permission::SysSpamClassifierUpdate), - 556 => Some(Permission::SysSpamDnsblServerGet), - 557 => Some(Permission::SysSpamDnsblServerCreate), - 558 => Some(Permission::SysSpamDnsblServerUpdate), - 559 => Some(Permission::SysSpamDnsblServerDestroy), - 560 => Some(Permission::SysSpamDnsblServerQuery), - 561 => Some(Permission::SysSpamDnsblSettingsGet), - 562 => Some(Permission::SysSpamDnsblSettingsUpdate), - 563 => Some(Permission::SysSpamFileExtensionGet), - 564 => Some(Permission::SysSpamFileExtensionCreate), - 565 => Some(Permission::SysSpamFileExtensionUpdate), - 566 => Some(Permission::SysSpamFileExtensionDestroy), - 567 => Some(Permission::SysSpamFileExtensionQuery), - 568 => Some(Permission::SysSpamLlmGet), - 569 => Some(Permission::SysSpamLlmUpdate), - 570 => Some(Permission::SysSpamPyzorGet), - 571 => Some(Permission::SysSpamPyzorUpdate), - 572 => Some(Permission::SysSpamRuleGet), - 573 => Some(Permission::SysSpamRuleCreate), - 574 => Some(Permission::SysSpamRuleUpdate), - 575 => Some(Permission::SysSpamRuleDestroy), - 576 => Some(Permission::SysSpamRuleQuery), - 577 => Some(Permission::SysSpamSettingsGet), - 578 => Some(Permission::SysSpamSettingsUpdate), - 579 => Some(Permission::SysSpamTagGet), - 580 => Some(Permission::SysSpamTagCreate), - 581 => Some(Permission::SysSpamTagUpdate), - 582 => Some(Permission::SysSpamTagDestroy), - 583 => Some(Permission::SysSpamTagQuery), - 584 => Some(Permission::SysSpamTrainingSampleGet), - 585 => Some(Permission::SysSpamTrainingSampleCreate), - 586 => Some(Permission::SysSpamTrainingSampleUpdate), - 587 => Some(Permission::SysSpamTrainingSampleDestroy), - 588 => Some(Permission::SysSpamTrainingSampleQuery), - 589 => Some(Permission::SysSpfReportSettingsGet), - 590 => Some(Permission::SysSpfReportSettingsUpdate), - 591 => Some(Permission::SysStoreLookupGet), - 592 => Some(Permission::SysStoreLookupCreate), - 593 => Some(Permission::SysStoreLookupUpdate), - 594 => Some(Permission::SysStoreLookupDestroy), - 595 => Some(Permission::SysStoreLookupQuery), - 596 => Some(Permission::SysSystemSettingsGet), - 597 => Some(Permission::SysSystemSettingsUpdate), - 598 => Some(Permission::TaskIndexDocument), - 599 => Some(Permission::TaskUnindexDocument), - 600 => Some(Permission::TaskIndexTrace), - 601 => Some(Permission::TaskCalendarAlarmEmail), - 602 => Some(Permission::TaskCalendarAlarmNotification), - 603 => Some(Permission::TaskCalendarItipMessage), - 604 => Some(Permission::TaskMergeThreads), - 605 => Some(Permission::TaskDmarcReport), - 606 => Some(Permission::TaskTlsReport), - 607 => Some(Permission::TaskRestoreArchivedItem), - 608 => Some(Permission::TaskDestroyAccount), - 609 => Some(Permission::TaskAccountMaintenance), - 610 => Some(Permission::TaskTenantMaintenance), - 611 => Some(Permission::TaskStoreMaintenance), - 612 => Some(Permission::TaskSpamFilterMaintenance), - 613 => Some(Permission::TaskAcmeRenewal), - 614 => Some(Permission::TaskDkimManagement), - 615 => Some(Permission::TaskDnsManagement), - 616 => Some(Permission::SysTaskGet), - 617 => Some(Permission::SysTaskCreate), - 618 => Some(Permission::SysTaskUpdate), - 619 => Some(Permission::SysTaskDestroy), - 620 => Some(Permission::SysTaskQuery), - 621 => Some(Permission::SysTaskManagerGet), - 622 => Some(Permission::SysTaskManagerUpdate), - 623 => Some(Permission::SysTenantGet), - 624 => Some(Permission::SysTenantCreate), - 625 => Some(Permission::SysTenantUpdate), - 626 => Some(Permission::SysTenantDestroy), - 627 => Some(Permission::SysTenantQuery), - 628 => Some(Permission::SysTlsExternalReportGet), - 629 => Some(Permission::SysTlsExternalReportCreate), - 630 => Some(Permission::SysTlsExternalReportUpdate), - 631 => Some(Permission::SysTlsExternalReportDestroy), - 632 => Some(Permission::SysTlsExternalReportQuery), - 633 => Some(Permission::SysTlsInternalReportGet), - 634 => Some(Permission::SysTlsInternalReportCreate), - 635 => Some(Permission::SysTlsInternalReportUpdate), - 636 => Some(Permission::SysTlsInternalReportDestroy), - 637 => Some(Permission::SysTlsInternalReportQuery), - 638 => Some(Permission::SysTlsReportSettingsGet), - 639 => Some(Permission::SysTlsReportSettingsUpdate), - 640 => Some(Permission::SysTraceGet), - 641 => Some(Permission::SysTraceCreate), - 642 => Some(Permission::SysTraceUpdate), - 643 => Some(Permission::SysTraceDestroy), - 644 => Some(Permission::SysTraceQuery), - 645 => Some(Permission::SysTracerGet), - 646 => Some(Permission::SysTracerCreate), - 647 => Some(Permission::SysTracerUpdate), - 648 => Some(Permission::SysTracerDestroy), - 649 => Some(Permission::SysTracerQuery), - 650 => Some(Permission::SysTracingStoreGet), - 651 => Some(Permission::SysTracingStoreUpdate), - 652 => Some(Permission::SysWebDavGet), - 653 => Some(Permission::SysWebDavUpdate), - 654 => Some(Permission::SysWebHookGet), - 655 => Some(Permission::SysWebHookCreate), - 656 => Some(Permission::SysWebHookUpdate), - 657 => Some(Permission::SysWebHookDestroy), - 658 => Some(Permission::SysWebHookQuery), + 63 => Some(Permission::JmapQuotaGet), + 64 => Some(Permission::JmapQuotaChanges), + 65 => Some(Permission::JmapQuotaQuery), + 66 => Some(Permission::JmapQuotaQueryChanges), + 67 => Some(Permission::JmapBlobGet), + 68 => Some(Permission::JmapBlobCopy), + 69 => Some(Permission::JmapBlobLookup), + 70 => Some(Permission::JmapBlobUpload), + 71 => Some(Permission::JmapAddressBookGet), + 72 => Some(Permission::JmapAddressBookChanges), + 73 => Some(Permission::JmapAddressBookCreate), + 74 => Some(Permission::JmapAddressBookUpdate), + 75 => Some(Permission::JmapAddressBookDestroy), + 76 => Some(Permission::JmapContactCardGet), + 77 => Some(Permission::JmapContactCardChanges), + 78 => Some(Permission::JmapContactCardQuery), + 79 => Some(Permission::JmapContactCardQueryChanges), + 80 => Some(Permission::JmapContactCardCreate), + 81 => Some(Permission::JmapContactCardUpdate), + 82 => Some(Permission::JmapContactCardDestroy), + 83 => Some(Permission::JmapContactCardCopy), + 84 => Some(Permission::JmapContactCardParse), + 85 => Some(Permission::JmapFileNodeGet), + 86 => Some(Permission::JmapFileNodeChanges), + 87 => Some(Permission::JmapFileNodeQuery), + 88 => Some(Permission::JmapFileNodeQueryChanges), + 89 => Some(Permission::JmapFileNodeCreate), + 90 => Some(Permission::JmapFileNodeUpdate), + 91 => Some(Permission::JmapFileNodeDestroy), + 92 => Some(Permission::JmapFileNodeCopy), + 93 => Some(Permission::JmapShareNotificationGet), + 94 => Some(Permission::JmapShareNotificationChanges), + 95 => Some(Permission::JmapShareNotificationQuery), + 96 => Some(Permission::JmapShareNotificationQueryChanges), + 97 => Some(Permission::JmapShareNotificationCreate), + 98 => Some(Permission::JmapShareNotificationUpdate), + 99 => Some(Permission::JmapShareNotificationDestroy), + 100 => Some(Permission::JmapCalendarGet), + 101 => Some(Permission::JmapCalendarChanges), + 102 => Some(Permission::JmapCalendarCreate), + 103 => Some(Permission::JmapCalendarUpdate), + 104 => Some(Permission::JmapCalendarDestroy), + 105 => Some(Permission::JmapCalendarEventGet), + 106 => Some(Permission::JmapCalendarEventChanges), + 107 => Some(Permission::JmapCalendarEventQuery), + 108 => Some(Permission::JmapCalendarEventQueryChanges), + 109 => Some(Permission::JmapCalendarEventCreate), + 110 => Some(Permission::JmapCalendarEventUpdate), + 111 => Some(Permission::JmapCalendarEventDestroy), + 112 => Some(Permission::JmapCalendarEventCopy), + 113 => Some(Permission::JmapCalendarEventParse), + 114 => Some(Permission::JmapCalendarEventNotificationGet), + 115 => Some(Permission::JmapCalendarEventNotificationChanges), + 116 => Some(Permission::JmapCalendarEventNotificationQuery), + 117 => Some(Permission::JmapCalendarEventNotificationQueryChanges), + 118 => Some(Permission::JmapCalendarEventNotificationCreate), + 119 => Some(Permission::JmapCalendarEventNotificationUpdate), + 120 => Some(Permission::JmapCalendarEventNotificationDestroy), + 121 => Some(Permission::JmapParticipantIdentityGet), + 122 => Some(Permission::JmapParticipantIdentityChanges), + 123 => Some(Permission::JmapParticipantIdentityCreate), + 124 => Some(Permission::JmapParticipantIdentityUpdate), + 125 => Some(Permission::JmapParticipantIdentityDestroy), + 126 => Some(Permission::JmapCoreEcho), + 127 => Some(Permission::JmapMetadataGet), + 128 => Some(Permission::JmapMetadataSet), + 129 => Some(Permission::JmapMetadataPrivate), + 130 => Some(Permission::ImapAuthenticate), + 131 => Some(Permission::ImapAclGet), + 132 => Some(Permission::ImapAclSet), + 133 => Some(Permission::ImapMyRights), + 134 => Some(Permission::ImapListRights), + 135 => Some(Permission::ImapAppend), + 136 => Some(Permission::ImapCapability), + 137 => Some(Permission::ImapId), + 138 => Some(Permission::ImapCopy), + 139 => Some(Permission::ImapMove), + 140 => Some(Permission::ImapCreate), + 141 => Some(Permission::ImapDelete), + 142 => Some(Permission::ImapEnable), + 143 => Some(Permission::ImapExpunge), + 144 => Some(Permission::ImapFetch), + 145 => Some(Permission::ImapIdle), + 146 => Some(Permission::ImapList), + 147 => Some(Permission::ImapLsub), + 148 => Some(Permission::ImapNamespace), + 149 => Some(Permission::ImapRename), + 150 => Some(Permission::ImapSearch), + 151 => Some(Permission::ImapSort), + 152 => Some(Permission::ImapSelect), + 153 => Some(Permission::ImapExamine), + 154 => Some(Permission::ImapStatus), + 155 => Some(Permission::ImapStore), + 156 => Some(Permission::ImapSubscribe), + 157 => Some(Permission::ImapThread), + 158 => Some(Permission::ImapMetadataGet), + 159 => Some(Permission::ImapMetadataSet), + 160 => Some(Permission::ImapMetadataPrivate), + 161 => Some(Permission::Pop3Authenticate), + 162 => Some(Permission::Pop3List), + 163 => Some(Permission::Pop3Uidl), + 164 => Some(Permission::Pop3Stat), + 165 => Some(Permission::Pop3Retr), + 166 => Some(Permission::Pop3Dele), + 167 => Some(Permission::SieveAuthenticate), + 168 => Some(Permission::SieveListScripts), + 169 => Some(Permission::SieveSetActive), + 170 => Some(Permission::SieveGetScript), + 171 => Some(Permission::SievePutScript), + 172 => Some(Permission::SieveDeleteScript), + 173 => Some(Permission::SieveRenameScript), + 174 => Some(Permission::SieveCheckScript), + 175 => Some(Permission::SieveHaveSpace), + 176 => Some(Permission::DavSyncCollection), + 177 => Some(Permission::DavExpandProperty), + 178 => Some(Permission::DavPrincipalAcl), + 179 => Some(Permission::DavPrincipalList), + 180 => Some(Permission::DavPrincipalMatch), + 181 => Some(Permission::DavPrincipalSearch), + 182 => Some(Permission::DavPrincipalSearchPropSet), + 183 => Some(Permission::DavFilePropFind), + 184 => Some(Permission::DavFilePropPatch), + 185 => Some(Permission::DavFileGet), + 186 => Some(Permission::DavFileMkCol), + 187 => Some(Permission::DavFileDelete), + 188 => Some(Permission::DavFilePut), + 189 => Some(Permission::DavFileCopy), + 190 => Some(Permission::DavFileMove), + 191 => Some(Permission::DavFileLock), + 192 => Some(Permission::DavFileAcl), + 193 => Some(Permission::DavCardPropFind), + 194 => Some(Permission::DavCardPropPatch), + 195 => Some(Permission::DavCardGet), + 196 => Some(Permission::DavCardMkCol), + 197 => Some(Permission::DavCardDelete), + 198 => Some(Permission::DavCardPut), + 199 => Some(Permission::DavCardCopy), + 200 => Some(Permission::DavCardMove), + 201 => Some(Permission::DavCardLock), + 202 => Some(Permission::DavCardAcl), + 203 => Some(Permission::DavCardQuery), + 204 => Some(Permission::DavCardMultiGet), + 205 => Some(Permission::DavCalPropFind), + 206 => Some(Permission::DavCalPropPatch), + 207 => Some(Permission::DavCalGet), + 208 => Some(Permission::DavCalMkCol), + 209 => Some(Permission::DavCalDelete), + 210 => Some(Permission::DavCalPut), + 211 => Some(Permission::DavCalCopy), + 212 => Some(Permission::DavCalMove), + 213 => Some(Permission::DavCalLock), + 214 => Some(Permission::DavCalAcl), + 215 => Some(Permission::DavCalQuery), + 216 => Some(Permission::DavCalMultiGet), + 217 => Some(Permission::DavCalFreeBusyQuery), + 220 => Some(Permission::LiveDeliveryTest), + 222 => Some(Permission::SysAccountGet), + 223 => Some(Permission::SysAccountCreate), + 224 => Some(Permission::SysAccountUpdate), + 225 => Some(Permission::SysAccountDestroy), + 226 => Some(Permission::SysAccountQuery), + 227 => Some(Permission::SysAccountPasswordGet), + 228 => Some(Permission::SysAccountPasswordUpdate), + 229 => Some(Permission::SysAccountSettingsGet), + 230 => Some(Permission::SysAccountSettingsUpdate), + 231 => Some(Permission::SysAcmeProviderGet), + 232 => Some(Permission::SysAcmeProviderCreate), + 233 => Some(Permission::SysAcmeProviderUpdate), + 234 => Some(Permission::SysAcmeProviderDestroy), + 235 => Some(Permission::SysAcmeProviderQuery), + 236 => Some(Permission::ActionReloadSettings), + 237 => Some(Permission::ActionReloadTlsCertificates), + 238 => Some(Permission::ActionReloadLookupStores), + 239 => Some(Permission::ActionReloadBlockedIps), + 240 => Some(Permission::ActionUpdateApps), + 241 => Some(Permission::ActionTroubleshootDmarc), + 242 => Some(Permission::ActionClassifySpam), + 243 => Some(Permission::ActionInvalidateCaches), + 244 => Some(Permission::ActionInvalidateNegativeCaches), + 245 => Some(Permission::ActionPauseMtaQueue), + 246 => Some(Permission::ActionResumeMtaQueue), + 247 => Some(Permission::SysActionCreate), + 248 => Some(Permission::SysAddressBookGet), + 249 => Some(Permission::SysAddressBookUpdate), + 260 => Some(Permission::SysAllowedIpGet), + 261 => Some(Permission::SysAllowedIpCreate), + 262 => Some(Permission::SysAllowedIpUpdate), + 263 => Some(Permission::SysAllowedIpDestroy), + 264 => Some(Permission::SysAllowedIpQuery), + 265 => Some(Permission::SysApiKeyGet), + 266 => Some(Permission::SysApiKeyCreate), + 267 => Some(Permission::SysApiKeyUpdate), + 268 => Some(Permission::SysApiKeyDestroy), + 269 => Some(Permission::SysApiKeyQuery), + 270 => Some(Permission::SysAppPasswordGet), + 271 => Some(Permission::SysAppPasswordCreate), + 272 => Some(Permission::SysAppPasswordUpdate), + 273 => Some(Permission::SysAppPasswordDestroy), + 274 => Some(Permission::SysAppPasswordQuery), + 275 => Some(Permission::SysApplicationGet), + 276 => Some(Permission::SysApplicationCreate), + 277 => Some(Permission::SysApplicationUpdate), + 278 => Some(Permission::SysApplicationDestroy), + 279 => Some(Permission::SysApplicationQuery), + 284 => Some(Permission::SysArfExternalReportGet), + 285 => Some(Permission::SysArfExternalReportDestroy), + 286 => Some(Permission::SysArfExternalReportQuery), + 287 => Some(Permission::SysAsnGet), + 288 => Some(Permission::SysAsnUpdate), + 289 => Some(Permission::SysAuthenticationGet), + 290 => Some(Permission::SysAuthenticationUpdate), + 291 => Some(Permission::SysBlobStoreGet), + 292 => Some(Permission::SysBlobStoreUpdate), + 293 => Some(Permission::SysBlockedIpGet), + 294 => Some(Permission::SysBlockedIpCreate), + 295 => Some(Permission::SysBlockedIpUpdate), + 296 => Some(Permission::SysBlockedIpDestroy), + 297 => Some(Permission::SysBlockedIpQuery), + 298 => Some(Permission::SysBootstrapGet), + 299 => Some(Permission::SysBootstrapUpdate), + 300 => Some(Permission::SysCacheGet), + 301 => Some(Permission::SysCacheUpdate), + 302 => Some(Permission::SysCalendarGet), + 303 => Some(Permission::SysCalendarUpdate), + 304 => Some(Permission::SysCalendarAlarmGet), + 305 => Some(Permission::SysCalendarAlarmUpdate), + 306 => Some(Permission::SysCalendarSchedulingGet), + 307 => Some(Permission::SysCalendarSchedulingUpdate), + 308 => Some(Permission::SysCertificateGet), + 309 => Some(Permission::SysCertificateCreate), + 310 => Some(Permission::SysCertificateUpdate), + 311 => Some(Permission::SysCertificateDestroy), + 312 => Some(Permission::SysCertificateQuery), + 313 => Some(Permission::SysClusterNodeGet), + 314 => Some(Permission::SysClusterNodeQuery), + 315 => Some(Permission::SysClusterRoleGet), + 316 => Some(Permission::SysClusterRoleCreate), + 317 => Some(Permission::SysClusterRoleUpdate), + 318 => Some(Permission::SysClusterRoleDestroy), + 319 => Some(Permission::SysClusterRoleQuery), + 320 => Some(Permission::SysCoordinatorGet), + 321 => Some(Permission::SysCoordinatorUpdate), + 322 => Some(Permission::SysDataRetentionGet), + 323 => Some(Permission::SysDataRetentionUpdate), + 324 => Some(Permission::SysDataStoreGet), + 325 => Some(Permission::SysDataStoreUpdate), + 326 => Some(Permission::SysDirectoryGet), + 327 => Some(Permission::SysDirectoryCreate), + 328 => Some(Permission::SysDirectoryUpdate), + 329 => Some(Permission::SysDirectoryDestroy), + 330 => Some(Permission::SysDirectoryQuery), + 331 => Some(Permission::SysDkimReportSettingsGet), + 332 => Some(Permission::SysDkimReportSettingsUpdate), + 333 => Some(Permission::SysDkimSignatureGet), + 334 => Some(Permission::SysDkimSignatureCreate), + 335 => Some(Permission::SysDkimSignatureUpdate), + 336 => Some(Permission::SysDkimSignatureDestroy), + 337 => Some(Permission::SysDkimSignatureQuery), + 338 => Some(Permission::SysDmarcExternalReportGet), + 339 => Some(Permission::SysDmarcExternalReportDestroy), + 340 => Some(Permission::SysDmarcExternalReportQuery), + 341 => Some(Permission::SysDmarcInternalReportGet), + 342 => Some(Permission::SysDmarcInternalReportUpdate), + 343 => Some(Permission::SysDmarcInternalReportDestroy), + 344 => Some(Permission::SysDmarcInternalReportQuery), + 345 => Some(Permission::SysDmarcReportSettingsGet), + 346 => Some(Permission::SysDmarcReportSettingsUpdate), + 347 => Some(Permission::SysDnsResolverGet), + 348 => Some(Permission::SysDnsResolverUpdate), + 349 => Some(Permission::SysDnsServerGet), + 350 => Some(Permission::SysDnsServerCreate), + 351 => Some(Permission::SysDnsServerUpdate), + 352 => Some(Permission::SysDnsServerDestroy), + 353 => Some(Permission::SysDnsServerQuery), + 354 => Some(Permission::SysDomainGet), + 355 => Some(Permission::SysDomainCreate), + 356 => Some(Permission::SysDomainUpdate), + 357 => Some(Permission::SysDomainDestroy), + 358 => Some(Permission::SysDomainQuery), + 359 => Some(Permission::SysDsnReportSettingsGet), + 360 => Some(Permission::SysDsnReportSettingsUpdate), + 361 => Some(Permission::SysEmailGet), + 362 => Some(Permission::SysEmailUpdate), + 365 => Some(Permission::SysEventTracingLevelGet), + 366 => Some(Permission::SysEventTracingLevelCreate), + 367 => Some(Permission::SysEventTracingLevelUpdate), + 368 => Some(Permission::SysEventTracingLevelDestroy), + 369 => Some(Permission::SysEventTracingLevelQuery), + 370 => Some(Permission::SysFileStorageGet), + 371 => Some(Permission::SysFileStorageUpdate), + 372 => Some(Permission::SysHttpGet), + 373 => Some(Permission::SysHttpUpdate), + 374 => Some(Permission::SysHttpFormGet), + 375 => Some(Permission::SysHttpFormUpdate), + 376 => Some(Permission::SysHttpLookupGet), + 377 => Some(Permission::SysHttpLookupCreate), + 378 => Some(Permission::SysHttpLookupUpdate), + 379 => Some(Permission::SysHttpLookupDestroy), + 380 => Some(Permission::SysHttpLookupQuery), + 381 => Some(Permission::SysImapGet), + 382 => Some(Permission::SysImapUpdate), + 383 => Some(Permission::SysInMemoryStoreGet), + 384 => Some(Permission::SysInMemoryStoreUpdate), + 385 => Some(Permission::SysIndexQueueStatusGet), + 386 => Some(Permission::SysIndexQueueStatusUpdate), + 387 => Some(Permission::SysIndexQueueStatusQuery), + 388 => Some(Permission::SysJmapGet), + 389 => Some(Permission::SysJmapUpdate), + 390 => Some(Permission::SysLogGet), + 391 => Some(Permission::SysLogQuery), + 392 => Some(Permission::SysMailingListGet), + 393 => Some(Permission::SysMailingListCreate), + 394 => Some(Permission::SysMailingListUpdate), + 395 => Some(Permission::SysMailingListDestroy), + 396 => Some(Permission::SysMailingListQuery), + 402 => Some(Permission::SysMemoryLookupKeyGet), + 403 => Some(Permission::SysMemoryLookupKeyCreate), + 404 => Some(Permission::SysMemoryLookupKeyUpdate), + 405 => Some(Permission::SysMemoryLookupKeyDestroy), + 406 => Some(Permission::SysMemoryLookupKeyQuery), + 407 => Some(Permission::SysMemoryLookupKeyValueGet), + 408 => Some(Permission::SysMemoryLookupKeyValueCreate), + 409 => Some(Permission::SysMemoryLookupKeyValueUpdate), + 410 => Some(Permission::SysMemoryLookupKeyValueDestroy), + 411 => Some(Permission::SysMemoryLookupKeyValueQuery), + 412 => Some(Permission::SysMetadataGet), + 413 => Some(Permission::SysMetadataUpdate), + 416 => Some(Permission::SysMetricsGet), + 417 => Some(Permission::SysMetricsUpdate), + 420 => Some(Permission::SysMtaConnectionStrategyGet), + 421 => Some(Permission::SysMtaConnectionStrategyCreate), + 422 => Some(Permission::SysMtaConnectionStrategyUpdate), + 423 => Some(Permission::SysMtaConnectionStrategyDestroy), + 424 => Some(Permission::SysMtaConnectionStrategyQuery), + 425 => Some(Permission::SysMtaDeliveryScheduleGet), + 426 => Some(Permission::SysMtaDeliveryScheduleCreate), + 427 => Some(Permission::SysMtaDeliveryScheduleUpdate), + 428 => Some(Permission::SysMtaDeliveryScheduleDestroy), + 429 => Some(Permission::SysMtaDeliveryScheduleQuery), + 430 => Some(Permission::SysMtaExtensionsGet), + 431 => Some(Permission::SysMtaExtensionsUpdate), + 432 => Some(Permission::SysMtaHookGet), + 433 => Some(Permission::SysMtaHookCreate), + 434 => Some(Permission::SysMtaHookUpdate), + 435 => Some(Permission::SysMtaHookDestroy), + 436 => Some(Permission::SysMtaHookQuery), + 437 => Some(Permission::SysMtaInboundSessionGet), + 438 => Some(Permission::SysMtaInboundSessionUpdate), + 439 => Some(Permission::SysMtaInboundThrottleGet), + 440 => Some(Permission::SysMtaInboundThrottleCreate), + 441 => Some(Permission::SysMtaInboundThrottleUpdate), + 442 => Some(Permission::SysMtaInboundThrottleDestroy), + 443 => Some(Permission::SysMtaInboundThrottleQuery), + 444 => Some(Permission::SysMtaMilterGet), + 445 => Some(Permission::SysMtaMilterCreate), + 446 => Some(Permission::SysMtaMilterUpdate), + 447 => Some(Permission::SysMtaMilterDestroy), + 448 => Some(Permission::SysMtaMilterQuery), + 449 => Some(Permission::SysMtaOutboundStrategyGet), + 450 => Some(Permission::SysMtaOutboundStrategyUpdate), + 451 => Some(Permission::SysMtaOutboundThrottleGet), + 452 => Some(Permission::SysMtaOutboundThrottleCreate), + 453 => Some(Permission::SysMtaOutboundThrottleUpdate), + 454 => Some(Permission::SysMtaOutboundThrottleDestroy), + 455 => Some(Permission::SysMtaOutboundThrottleQuery), + 456 => Some(Permission::SysMtaQueueQuotaGet), + 457 => Some(Permission::SysMtaQueueQuotaCreate), + 458 => Some(Permission::SysMtaQueueQuotaUpdate), + 459 => Some(Permission::SysMtaQueueQuotaDestroy), + 460 => Some(Permission::SysMtaQueueQuotaQuery), + 461 => Some(Permission::SysMtaRouteGet), + 462 => Some(Permission::SysMtaRouteCreate), + 463 => Some(Permission::SysMtaRouteUpdate), + 464 => Some(Permission::SysMtaRouteDestroy), + 465 => Some(Permission::SysMtaRouteQuery), + 466 => Some(Permission::SysMtaStageAuthGet), + 467 => Some(Permission::SysMtaStageAuthUpdate), + 468 => Some(Permission::SysMtaStageConnectGet), + 469 => Some(Permission::SysMtaStageConnectUpdate), + 470 => Some(Permission::SysMtaStageDataGet), + 471 => Some(Permission::SysMtaStageDataUpdate), + 472 => Some(Permission::SysMtaStageEhloGet), + 473 => Some(Permission::SysMtaStageEhloUpdate), + 474 => Some(Permission::SysMtaStageMailGet), + 475 => Some(Permission::SysMtaStageMailUpdate), + 476 => Some(Permission::SysMtaStageRcptGet), + 477 => Some(Permission::SysMtaStageRcptUpdate), + 478 => Some(Permission::SysMtaStsGet), + 479 => Some(Permission::SysMtaStsUpdate), + 480 => Some(Permission::SysMtaTlsStrategyGet), + 481 => Some(Permission::SysMtaTlsStrategyCreate), + 482 => Some(Permission::SysMtaTlsStrategyUpdate), + 483 => Some(Permission::SysMtaTlsStrategyDestroy), + 484 => Some(Permission::SysMtaTlsStrategyQuery), + 485 => Some(Permission::SysMtaVirtualQueueGet), + 486 => Some(Permission::SysMtaVirtualQueueCreate), + 487 => Some(Permission::SysMtaVirtualQueueUpdate), + 488 => Some(Permission::SysMtaVirtualQueueDestroy), + 489 => Some(Permission::SysMtaVirtualQueueQuery), + 490 => Some(Permission::SysNetworkListenerGet), + 491 => Some(Permission::SysNetworkListenerCreate), + 492 => Some(Permission::SysNetworkListenerUpdate), + 493 => Some(Permission::SysNetworkListenerDestroy), + 494 => Some(Permission::SysNetworkListenerQuery), + 495 => Some(Permission::SysOAuthClientGet), + 496 => Some(Permission::SysOAuthClientCreate), + 497 => Some(Permission::SysOAuthClientUpdate), + 498 => Some(Permission::SysOAuthClientDestroy), + 499 => Some(Permission::SysOAuthClientQuery), + 500 => Some(Permission::SysOAuthGrantGet), + 501 => Some(Permission::SysOAuthGrantDestroy), + 502 => Some(Permission::SysOAuthGrantQuery), + 503 => Some(Permission::SysOidcProviderGet), + 504 => Some(Permission::SysOidcProviderUpdate), + 505 => Some(Permission::SysPublicKeyGet), + 506 => Some(Permission::SysPublicKeyCreate), + 507 => Some(Permission::SysPublicKeyUpdate), + 508 => Some(Permission::SysPublicKeyDestroy), + 509 => Some(Permission::SysPublicKeyQuery), + 510 => Some(Permission::SysQueuedMessageGet), + 511 => Some(Permission::SysQueuedMessageUpdate), + 512 => Some(Permission::SysQueuedMessageDestroy), + 513 => Some(Permission::SysQueuedMessageQuery), + 514 => Some(Permission::SysReportSettingsGet), + 515 => Some(Permission::SysReportSettingsUpdate), + 516 => Some(Permission::SysRoleGet), + 517 => Some(Permission::SysRoleCreate), + 518 => Some(Permission::SysRoleUpdate), + 519 => Some(Permission::SysRoleDestroy), + 520 => Some(Permission::SysRoleQuery), + 521 => Some(Permission::SysSearchGet), + 522 => Some(Permission::SysSearchUpdate), + 523 => Some(Permission::SysSearchStoreGet), + 524 => Some(Permission::SysSearchStoreUpdate), + 525 => Some(Permission::SysSecurityGet), + 526 => Some(Permission::SysSecurityUpdate), + 527 => Some(Permission::SysSenderAuthGet), + 528 => Some(Permission::SysSenderAuthUpdate), + 529 => Some(Permission::SysSharingGet), + 530 => Some(Permission::SysSharingUpdate), + 531 => Some(Permission::SysSieveSystemInterpreterGet), + 532 => Some(Permission::SysSieveSystemInterpreterUpdate), + 533 => Some(Permission::SysSieveSystemScriptGet), + 534 => Some(Permission::SysSieveSystemScriptCreate), + 535 => Some(Permission::SysSieveSystemScriptUpdate), + 536 => Some(Permission::SysSieveSystemScriptDestroy), + 537 => Some(Permission::SysSieveSystemScriptQuery), + 538 => Some(Permission::SysSieveUserInterpreterGet), + 539 => Some(Permission::SysSieveUserInterpreterUpdate), + 540 => Some(Permission::SysSieveUserScriptGet), + 541 => Some(Permission::SysSieveUserScriptCreate), + 542 => Some(Permission::SysSieveUserScriptUpdate), + 543 => Some(Permission::SysSieveUserScriptDestroy), + 544 => Some(Permission::SysSieveUserScriptQuery), + 547 => Some(Permission::SysSpamClassifierGet), + 548 => Some(Permission::SysSpamClassifierUpdate), + 549 => Some(Permission::SysSpamDnsblServerGet), + 550 => Some(Permission::SysSpamDnsblServerCreate), + 551 => Some(Permission::SysSpamDnsblServerUpdate), + 552 => Some(Permission::SysSpamDnsblServerDestroy), + 553 => Some(Permission::SysSpamDnsblServerQuery), + 554 => Some(Permission::SysSpamDnsblSettingsGet), + 555 => Some(Permission::SysSpamDnsblSettingsUpdate), + 556 => Some(Permission::SysSpamFileExtensionGet), + 557 => Some(Permission::SysSpamFileExtensionCreate), + 558 => Some(Permission::SysSpamFileExtensionUpdate), + 559 => Some(Permission::SysSpamFileExtensionDestroy), + 560 => Some(Permission::SysSpamFileExtensionQuery), + 561 => Some(Permission::SysSpamPyzorGet), + 562 => Some(Permission::SysSpamPyzorUpdate), + 563 => Some(Permission::SysSpamRuleGet), + 564 => Some(Permission::SysSpamRuleCreate), + 565 => Some(Permission::SysSpamRuleUpdate), + 566 => Some(Permission::SysSpamRuleDestroy), + 567 => Some(Permission::SysSpamRuleQuery), + 568 => Some(Permission::SysSpamSettingsGet), + 569 => Some(Permission::SysSpamSettingsUpdate), + 570 => Some(Permission::SysSpamTagGet), + 571 => Some(Permission::SysSpamTagCreate), + 572 => Some(Permission::SysSpamTagUpdate), + 573 => Some(Permission::SysSpamTagDestroy), + 574 => Some(Permission::SysSpamTagQuery), + 575 => Some(Permission::SysSpamTrainingSampleGet), + 576 => Some(Permission::SysSpamTrainingSampleCreate), + 577 => Some(Permission::SysSpamTrainingSampleDestroy), + 578 => Some(Permission::SysSpamTrainingSampleQuery), + 579 => Some(Permission::SysSpfReportSettingsGet), + 580 => Some(Permission::SysSpfReportSettingsUpdate), + 581 => Some(Permission::SysStoreLookupGet), + 582 => Some(Permission::SysStoreLookupCreate), + 583 => Some(Permission::SysStoreLookupUpdate), + 584 => Some(Permission::SysStoreLookupDestroy), + 585 => Some(Permission::SysStoreLookupQuery), + 586 => Some(Permission::SysSystemSettingsGet), + 587 => Some(Permission::SysSystemSettingsUpdate), + 588 => Some(Permission::TaskCalendarAlarmEmail), + 589 => Some(Permission::TaskCalendarAlarmNotification), + 590 => Some(Permission::TaskCalendarItipMessage), + 591 => Some(Permission::TaskMergeThreads), + 592 => Some(Permission::TaskDmarcReport), + 593 => Some(Permission::TaskTlsReport), + 595 => Some(Permission::TaskDestroyAccount), + 596 => Some(Permission::TaskAccountMaintenance), + 598 => Some(Permission::TaskStoreMaintenance), + 599 => Some(Permission::TaskSpamFilterMaintenance), + 600 => Some(Permission::TaskAcmeRenewal), + 601 => Some(Permission::TaskDkimManagement), + 602 => Some(Permission::TaskDnsManagement), + 603 => Some(Permission::TaskShareRevokeCleanup), + 604 => Some(Permission::TaskDmarcReportWindow), + 605 => Some(Permission::TaskTlsReportWindow), + 606 => Some(Permission::SysTaskGet), + 607 => Some(Permission::SysTaskCreate), + 608 => Some(Permission::SysTaskUpdate), + 609 => Some(Permission::SysTaskDestroy), + 610 => Some(Permission::SysTaskQuery), + 611 => Some(Permission::SysTaskManagerGet), + 612 => Some(Permission::SysTaskManagerUpdate), + 618 => Some(Permission::SysTlsExternalReportGet), + 619 => Some(Permission::SysTlsExternalReportDestroy), + 620 => Some(Permission::SysTlsExternalReportQuery), + 621 => Some(Permission::SysTlsInternalReportGet), + 622 => Some(Permission::SysTlsInternalReportUpdate), + 623 => Some(Permission::SysTlsInternalReportDestroy), + 624 => Some(Permission::SysTlsInternalReportQuery), + 625 => Some(Permission::SysTlsReportSettingsGet), + 626 => Some(Permission::SysTlsReportSettingsUpdate), + 629 => Some(Permission::SysTracerGet), + 630 => Some(Permission::SysTracerCreate), + 631 => Some(Permission::SysTracerUpdate), + 632 => Some(Permission::SysTracerDestroy), + 633 => Some(Permission::SysTracerQuery), + 636 => Some(Permission::SysWebDavGet), + 637 => Some(Permission::SysWebDavUpdate), + 638 => Some(Permission::SysWebHookGet), + 639 => Some(Permission::SysWebHookCreate), + 640 => Some(Permission::SysWebHookUpdate), + 641 => Some(Permission::SysWebHookDestroy), + 642 => Some(Permission::SysWebHookQuery), _ => None, } } - const COUNT: usize = 661; + const COUNT: usize = 643; +} + +impl Permission { + pub const ALL: &[Permission] = &[ + Permission::Authenticate, + Permission::AuthenticateWithAlias, + Permission::Impersonate, + Permission::UnlimitedRequests, + Permission::UnlimitedUploads, + Permission::FetchAnyBlob, + Permission::EmailSend, + Permission::EmailReceive, + Permission::CalendarAlarmsSend, + Permission::CalendarSchedulingSend, + Permission::CalendarSchedulingReceive, + Permission::JmapPushSubscriptionGet, + Permission::JmapPushSubscriptionCreate, + Permission::JmapPushSubscriptionUpdate, + Permission::JmapPushSubscriptionDestroy, + Permission::JmapMailboxGet, + Permission::JmapMailboxChanges, + Permission::JmapMailboxQuery, + Permission::JmapMailboxQueryChanges, + Permission::JmapMailboxCreate, + Permission::JmapMailboxUpdate, + Permission::JmapMailboxDestroy, + Permission::JmapThreadGet, + Permission::JmapThreadChanges, + Permission::JmapEmailGet, + Permission::JmapEmailChanges, + Permission::JmapEmailQuery, + Permission::JmapEmailQueryChanges, + Permission::JmapEmailCreate, + Permission::JmapEmailUpdate, + Permission::JmapEmailDestroy, + Permission::JmapEmailCopy, + Permission::JmapEmailImport, + Permission::JmapEmailParse, + Permission::JmapSearchSnippetGet, + Permission::JmapIdentityGet, + Permission::JmapIdentityChanges, + Permission::JmapIdentityCreate, + Permission::JmapIdentityUpdate, + Permission::JmapIdentityDestroy, + Permission::JmapEmailSubmissionGet, + Permission::JmapEmailSubmissionChanges, + Permission::JmapEmailSubmissionQuery, + Permission::JmapEmailSubmissionQueryChanges, + Permission::JmapEmailSubmissionCreate, + Permission::JmapEmailSubmissionUpdate, + Permission::JmapEmailSubmissionDestroy, + Permission::JmapVacationResponseGet, + Permission::JmapVacationResponseCreate, + Permission::JmapVacationResponseUpdate, + Permission::JmapVacationResponseDestroy, + Permission::JmapSieveScriptGet, + Permission::JmapSieveScriptQuery, + Permission::JmapSieveScriptValidate, + Permission::JmapSieveScriptCreate, + Permission::JmapSieveScriptUpdate, + Permission::JmapSieveScriptDestroy, + Permission::JmapPrincipalGet, + Permission::JmapPrincipalQuery, + Permission::JmapPrincipalChanges, + Permission::JmapPrincipalQueryChanges, + Permission::JmapPrincipalGetAvailability, + Permission::JmapQuotaGet, + Permission::JmapQuotaChanges, + Permission::JmapQuotaQuery, + Permission::JmapQuotaQueryChanges, + Permission::JmapBlobGet, + Permission::JmapBlobCopy, + Permission::JmapBlobLookup, + Permission::JmapBlobUpload, + Permission::JmapAddressBookGet, + Permission::JmapAddressBookChanges, + Permission::JmapAddressBookCreate, + Permission::JmapAddressBookUpdate, + Permission::JmapAddressBookDestroy, + Permission::JmapContactCardGet, + Permission::JmapContactCardChanges, + Permission::JmapContactCardQuery, + Permission::JmapContactCardQueryChanges, + Permission::JmapContactCardCreate, + Permission::JmapContactCardUpdate, + Permission::JmapContactCardDestroy, + Permission::JmapContactCardCopy, + Permission::JmapContactCardParse, + Permission::JmapFileNodeGet, + Permission::JmapFileNodeChanges, + Permission::JmapFileNodeQuery, + Permission::JmapFileNodeQueryChanges, + Permission::JmapFileNodeCreate, + Permission::JmapFileNodeUpdate, + Permission::JmapFileNodeDestroy, + Permission::JmapFileNodeCopy, + Permission::JmapShareNotificationGet, + Permission::JmapShareNotificationChanges, + Permission::JmapShareNotificationQuery, + Permission::JmapShareNotificationQueryChanges, + Permission::JmapShareNotificationCreate, + Permission::JmapShareNotificationUpdate, + Permission::JmapShareNotificationDestroy, + Permission::JmapCalendarGet, + Permission::JmapCalendarChanges, + Permission::JmapCalendarCreate, + Permission::JmapCalendarUpdate, + Permission::JmapCalendarDestroy, + Permission::JmapCalendarEventGet, + Permission::JmapCalendarEventChanges, + Permission::JmapCalendarEventQuery, + Permission::JmapCalendarEventQueryChanges, + Permission::JmapCalendarEventCreate, + Permission::JmapCalendarEventUpdate, + Permission::JmapCalendarEventDestroy, + Permission::JmapCalendarEventCopy, + Permission::JmapCalendarEventParse, + Permission::JmapCalendarEventNotificationGet, + Permission::JmapCalendarEventNotificationChanges, + Permission::JmapCalendarEventNotificationQuery, + Permission::JmapCalendarEventNotificationQueryChanges, + Permission::JmapCalendarEventNotificationCreate, + Permission::JmapCalendarEventNotificationUpdate, + Permission::JmapCalendarEventNotificationDestroy, + Permission::JmapParticipantIdentityGet, + Permission::JmapParticipantIdentityChanges, + Permission::JmapParticipantIdentityCreate, + Permission::JmapParticipantIdentityUpdate, + Permission::JmapParticipantIdentityDestroy, + Permission::JmapCoreEcho, + Permission::JmapMetadataGet, + Permission::JmapMetadataSet, + Permission::JmapMetadataPrivate, + Permission::ImapAuthenticate, + Permission::ImapAclGet, + Permission::ImapAclSet, + Permission::ImapMyRights, + Permission::ImapListRights, + Permission::ImapAppend, + Permission::ImapCapability, + Permission::ImapId, + Permission::ImapCopy, + Permission::ImapMove, + Permission::ImapCreate, + Permission::ImapDelete, + Permission::ImapEnable, + Permission::ImapExpunge, + Permission::ImapFetch, + Permission::ImapIdle, + Permission::ImapList, + Permission::ImapLsub, + Permission::ImapNamespace, + Permission::ImapRename, + Permission::ImapSearch, + Permission::ImapSort, + Permission::ImapSelect, + Permission::ImapExamine, + Permission::ImapStatus, + Permission::ImapStore, + Permission::ImapSubscribe, + Permission::ImapThread, + Permission::ImapMetadataGet, + Permission::ImapMetadataSet, + Permission::ImapMetadataPrivate, + Permission::Pop3Authenticate, + Permission::Pop3List, + Permission::Pop3Uidl, + Permission::Pop3Stat, + Permission::Pop3Retr, + Permission::Pop3Dele, + Permission::SieveAuthenticate, + Permission::SieveListScripts, + Permission::SieveSetActive, + Permission::SieveGetScript, + Permission::SievePutScript, + Permission::SieveDeleteScript, + Permission::SieveRenameScript, + Permission::SieveCheckScript, + Permission::SieveHaveSpace, + Permission::DavSyncCollection, + Permission::DavExpandProperty, + Permission::DavPrincipalAcl, + Permission::DavPrincipalList, + Permission::DavPrincipalMatch, + Permission::DavPrincipalSearch, + Permission::DavPrincipalSearchPropSet, + Permission::DavFilePropFind, + Permission::DavFilePropPatch, + Permission::DavFileGet, + Permission::DavFileMkCol, + Permission::DavFileDelete, + Permission::DavFilePut, + Permission::DavFileCopy, + Permission::DavFileMove, + Permission::DavFileLock, + Permission::DavFileAcl, + Permission::DavCardPropFind, + Permission::DavCardPropPatch, + Permission::DavCardGet, + Permission::DavCardMkCol, + Permission::DavCardDelete, + Permission::DavCardPut, + Permission::DavCardCopy, + Permission::DavCardMove, + Permission::DavCardLock, + Permission::DavCardAcl, + Permission::DavCardQuery, + Permission::DavCardMultiGet, + Permission::DavCalPropFind, + Permission::DavCalPropPatch, + Permission::DavCalGet, + Permission::DavCalMkCol, + Permission::DavCalDelete, + Permission::DavCalPut, + Permission::DavCalCopy, + Permission::DavCalMove, + Permission::DavCalLock, + Permission::DavCalAcl, + Permission::DavCalQuery, + Permission::DavCalMultiGet, + Permission::DavCalFreeBusyQuery, + Permission::LiveDeliveryTest, + Permission::SysAccountGet, + Permission::SysAccountCreate, + Permission::SysAccountUpdate, + Permission::SysAccountDestroy, + Permission::SysAccountQuery, + Permission::SysAccountPasswordGet, + Permission::SysAccountPasswordUpdate, + Permission::SysAccountSettingsGet, + Permission::SysAccountSettingsUpdate, + Permission::SysAcmeProviderGet, + Permission::SysAcmeProviderCreate, + Permission::SysAcmeProviderUpdate, + Permission::SysAcmeProviderDestroy, + Permission::SysAcmeProviderQuery, + Permission::ActionReloadSettings, + Permission::ActionReloadTlsCertificates, + Permission::ActionReloadLookupStores, + Permission::ActionReloadBlockedIps, + Permission::ActionUpdateApps, + Permission::ActionTroubleshootDmarc, + Permission::ActionClassifySpam, + Permission::ActionInvalidateCaches, + Permission::ActionInvalidateNegativeCaches, + Permission::ActionPauseMtaQueue, + Permission::ActionResumeMtaQueue, + Permission::SysActionCreate, + Permission::SysAddressBookGet, + Permission::SysAddressBookUpdate, + Permission::SysAllowedIpGet, + Permission::SysAllowedIpCreate, + Permission::SysAllowedIpUpdate, + Permission::SysAllowedIpDestroy, + Permission::SysAllowedIpQuery, + Permission::SysApiKeyGet, + Permission::SysApiKeyCreate, + Permission::SysApiKeyUpdate, + Permission::SysApiKeyDestroy, + Permission::SysApiKeyQuery, + Permission::SysAppPasswordGet, + Permission::SysAppPasswordCreate, + Permission::SysAppPasswordUpdate, + Permission::SysAppPasswordDestroy, + Permission::SysAppPasswordQuery, + Permission::SysApplicationGet, + Permission::SysApplicationCreate, + Permission::SysApplicationUpdate, + Permission::SysApplicationDestroy, + Permission::SysApplicationQuery, + Permission::SysArfExternalReportGet, + Permission::SysArfExternalReportDestroy, + Permission::SysArfExternalReportQuery, + Permission::SysAsnGet, + Permission::SysAsnUpdate, + Permission::SysAuthenticationGet, + Permission::SysAuthenticationUpdate, + Permission::SysBlobStoreGet, + Permission::SysBlobStoreUpdate, + Permission::SysBlockedIpGet, + Permission::SysBlockedIpCreate, + Permission::SysBlockedIpUpdate, + Permission::SysBlockedIpDestroy, + Permission::SysBlockedIpQuery, + Permission::SysBootstrapGet, + Permission::SysBootstrapUpdate, + Permission::SysCacheGet, + Permission::SysCacheUpdate, + Permission::SysCalendarGet, + Permission::SysCalendarUpdate, + Permission::SysCalendarAlarmGet, + Permission::SysCalendarAlarmUpdate, + Permission::SysCalendarSchedulingGet, + Permission::SysCalendarSchedulingUpdate, + Permission::SysCertificateGet, + Permission::SysCertificateCreate, + Permission::SysCertificateUpdate, + Permission::SysCertificateDestroy, + Permission::SysCertificateQuery, + Permission::SysClusterNodeGet, + Permission::SysClusterNodeQuery, + Permission::SysClusterRoleGet, + Permission::SysClusterRoleCreate, + Permission::SysClusterRoleUpdate, + Permission::SysClusterRoleDestroy, + Permission::SysClusterRoleQuery, + Permission::SysCoordinatorGet, + Permission::SysCoordinatorUpdate, + Permission::SysDataRetentionGet, + Permission::SysDataRetentionUpdate, + Permission::SysDataStoreGet, + Permission::SysDataStoreUpdate, + Permission::SysDirectoryGet, + Permission::SysDirectoryCreate, + Permission::SysDirectoryUpdate, + Permission::SysDirectoryDestroy, + Permission::SysDirectoryQuery, + Permission::SysDkimReportSettingsGet, + Permission::SysDkimReportSettingsUpdate, + Permission::SysDkimSignatureGet, + Permission::SysDkimSignatureCreate, + Permission::SysDkimSignatureUpdate, + Permission::SysDkimSignatureDestroy, + Permission::SysDkimSignatureQuery, + Permission::SysDmarcExternalReportGet, + Permission::SysDmarcExternalReportDestroy, + Permission::SysDmarcExternalReportQuery, + Permission::SysDmarcInternalReportGet, + Permission::SysDmarcInternalReportUpdate, + Permission::SysDmarcInternalReportDestroy, + Permission::SysDmarcInternalReportQuery, + Permission::SysDmarcReportSettingsGet, + Permission::SysDmarcReportSettingsUpdate, + Permission::SysDnsResolverGet, + Permission::SysDnsResolverUpdate, + Permission::SysDnsServerGet, + Permission::SysDnsServerCreate, + Permission::SysDnsServerUpdate, + Permission::SysDnsServerDestroy, + Permission::SysDnsServerQuery, + Permission::SysDomainGet, + Permission::SysDomainCreate, + Permission::SysDomainUpdate, + Permission::SysDomainDestroy, + Permission::SysDomainQuery, + Permission::SysDsnReportSettingsGet, + Permission::SysDsnReportSettingsUpdate, + Permission::SysEmailGet, + Permission::SysEmailUpdate, + Permission::SysEventTracingLevelGet, + Permission::SysEventTracingLevelCreate, + Permission::SysEventTracingLevelUpdate, + Permission::SysEventTracingLevelDestroy, + Permission::SysEventTracingLevelQuery, + Permission::SysFileStorageGet, + Permission::SysFileStorageUpdate, + Permission::SysHttpGet, + Permission::SysHttpUpdate, + Permission::SysHttpFormGet, + Permission::SysHttpFormUpdate, + Permission::SysHttpLookupGet, + Permission::SysHttpLookupCreate, + Permission::SysHttpLookupUpdate, + Permission::SysHttpLookupDestroy, + Permission::SysHttpLookupQuery, + Permission::SysImapGet, + Permission::SysImapUpdate, + Permission::SysInMemoryStoreGet, + Permission::SysInMemoryStoreUpdate, + Permission::SysIndexQueueStatusGet, + Permission::SysIndexQueueStatusUpdate, + Permission::SysIndexQueueStatusQuery, + Permission::SysJmapGet, + Permission::SysJmapUpdate, + Permission::SysLogGet, + Permission::SysLogQuery, + Permission::SysMailingListGet, + Permission::SysMailingListCreate, + Permission::SysMailingListUpdate, + Permission::SysMailingListDestroy, + Permission::SysMailingListQuery, + Permission::SysMemoryLookupKeyGet, + Permission::SysMemoryLookupKeyCreate, + Permission::SysMemoryLookupKeyUpdate, + Permission::SysMemoryLookupKeyDestroy, + Permission::SysMemoryLookupKeyQuery, + Permission::SysMemoryLookupKeyValueGet, + Permission::SysMemoryLookupKeyValueCreate, + Permission::SysMemoryLookupKeyValueUpdate, + Permission::SysMemoryLookupKeyValueDestroy, + Permission::SysMemoryLookupKeyValueQuery, + Permission::SysMetadataGet, + Permission::SysMetadataUpdate, + Permission::SysMetricsGet, + Permission::SysMetricsUpdate, + Permission::SysMtaConnectionStrategyGet, + Permission::SysMtaConnectionStrategyCreate, + Permission::SysMtaConnectionStrategyUpdate, + Permission::SysMtaConnectionStrategyDestroy, + Permission::SysMtaConnectionStrategyQuery, + Permission::SysMtaDeliveryScheduleGet, + Permission::SysMtaDeliveryScheduleCreate, + Permission::SysMtaDeliveryScheduleUpdate, + Permission::SysMtaDeliveryScheduleDestroy, + Permission::SysMtaDeliveryScheduleQuery, + Permission::SysMtaExtensionsGet, + Permission::SysMtaExtensionsUpdate, + Permission::SysMtaHookGet, + Permission::SysMtaHookCreate, + Permission::SysMtaHookUpdate, + Permission::SysMtaHookDestroy, + Permission::SysMtaHookQuery, + Permission::SysMtaInboundSessionGet, + Permission::SysMtaInboundSessionUpdate, + Permission::SysMtaInboundThrottleGet, + Permission::SysMtaInboundThrottleCreate, + Permission::SysMtaInboundThrottleUpdate, + Permission::SysMtaInboundThrottleDestroy, + Permission::SysMtaInboundThrottleQuery, + Permission::SysMtaMilterGet, + Permission::SysMtaMilterCreate, + Permission::SysMtaMilterUpdate, + Permission::SysMtaMilterDestroy, + Permission::SysMtaMilterQuery, + Permission::SysMtaOutboundStrategyGet, + Permission::SysMtaOutboundStrategyUpdate, + Permission::SysMtaOutboundThrottleGet, + Permission::SysMtaOutboundThrottleCreate, + Permission::SysMtaOutboundThrottleUpdate, + Permission::SysMtaOutboundThrottleDestroy, + Permission::SysMtaOutboundThrottleQuery, + Permission::SysMtaQueueQuotaGet, + Permission::SysMtaQueueQuotaCreate, + Permission::SysMtaQueueQuotaUpdate, + Permission::SysMtaQueueQuotaDestroy, + Permission::SysMtaQueueQuotaQuery, + Permission::SysMtaRouteGet, + Permission::SysMtaRouteCreate, + Permission::SysMtaRouteUpdate, + Permission::SysMtaRouteDestroy, + Permission::SysMtaRouteQuery, + Permission::SysMtaStageAuthGet, + Permission::SysMtaStageAuthUpdate, + Permission::SysMtaStageConnectGet, + Permission::SysMtaStageConnectUpdate, + Permission::SysMtaStageDataGet, + Permission::SysMtaStageDataUpdate, + Permission::SysMtaStageEhloGet, + Permission::SysMtaStageEhloUpdate, + Permission::SysMtaStageMailGet, + Permission::SysMtaStageMailUpdate, + Permission::SysMtaStageRcptGet, + Permission::SysMtaStageRcptUpdate, + Permission::SysMtaStsGet, + Permission::SysMtaStsUpdate, + Permission::SysMtaTlsStrategyGet, + Permission::SysMtaTlsStrategyCreate, + Permission::SysMtaTlsStrategyUpdate, + Permission::SysMtaTlsStrategyDestroy, + Permission::SysMtaTlsStrategyQuery, + Permission::SysMtaVirtualQueueGet, + Permission::SysMtaVirtualQueueCreate, + Permission::SysMtaVirtualQueueUpdate, + Permission::SysMtaVirtualQueueDestroy, + Permission::SysMtaVirtualQueueQuery, + Permission::SysNetworkListenerGet, + Permission::SysNetworkListenerCreate, + Permission::SysNetworkListenerUpdate, + Permission::SysNetworkListenerDestroy, + Permission::SysNetworkListenerQuery, + Permission::SysOAuthClientGet, + Permission::SysOAuthClientCreate, + Permission::SysOAuthClientUpdate, + Permission::SysOAuthClientDestroy, + Permission::SysOAuthClientQuery, + Permission::SysOAuthGrantGet, + Permission::SysOAuthGrantDestroy, + Permission::SysOAuthGrantQuery, + Permission::SysOidcProviderGet, + Permission::SysOidcProviderUpdate, + Permission::SysPublicKeyGet, + Permission::SysPublicKeyCreate, + Permission::SysPublicKeyUpdate, + Permission::SysPublicKeyDestroy, + Permission::SysPublicKeyQuery, + Permission::SysQueuedMessageGet, + Permission::SysQueuedMessageUpdate, + Permission::SysQueuedMessageDestroy, + Permission::SysQueuedMessageQuery, + Permission::SysReportSettingsGet, + Permission::SysReportSettingsUpdate, + Permission::SysRoleGet, + Permission::SysRoleCreate, + Permission::SysRoleUpdate, + Permission::SysRoleDestroy, + Permission::SysRoleQuery, + Permission::SysSearchGet, + Permission::SysSearchUpdate, + Permission::SysSearchStoreGet, + Permission::SysSearchStoreUpdate, + Permission::SysSecurityGet, + Permission::SysSecurityUpdate, + Permission::SysSenderAuthGet, + Permission::SysSenderAuthUpdate, + Permission::SysSharingGet, + Permission::SysSharingUpdate, + Permission::SysSieveSystemInterpreterGet, + Permission::SysSieveSystemInterpreterUpdate, + Permission::SysSieveSystemScriptGet, + Permission::SysSieveSystemScriptCreate, + Permission::SysSieveSystemScriptUpdate, + Permission::SysSieveSystemScriptDestroy, + Permission::SysSieveSystemScriptQuery, + Permission::SysSieveUserInterpreterGet, + Permission::SysSieveUserInterpreterUpdate, + Permission::SysSieveUserScriptGet, + Permission::SysSieveUserScriptCreate, + Permission::SysSieveUserScriptUpdate, + Permission::SysSieveUserScriptDestroy, + Permission::SysSieveUserScriptQuery, + Permission::SysSpamClassifierGet, + Permission::SysSpamClassifierUpdate, + Permission::SysSpamDnsblServerGet, + Permission::SysSpamDnsblServerCreate, + Permission::SysSpamDnsblServerUpdate, + Permission::SysSpamDnsblServerDestroy, + Permission::SysSpamDnsblServerQuery, + Permission::SysSpamDnsblSettingsGet, + Permission::SysSpamDnsblSettingsUpdate, + Permission::SysSpamFileExtensionGet, + Permission::SysSpamFileExtensionCreate, + Permission::SysSpamFileExtensionUpdate, + Permission::SysSpamFileExtensionDestroy, + Permission::SysSpamFileExtensionQuery, + Permission::SysSpamPyzorGet, + Permission::SysSpamPyzorUpdate, + Permission::SysSpamRuleGet, + Permission::SysSpamRuleCreate, + Permission::SysSpamRuleUpdate, + Permission::SysSpamRuleDestroy, + Permission::SysSpamRuleQuery, + Permission::SysSpamSettingsGet, + Permission::SysSpamSettingsUpdate, + Permission::SysSpamTagGet, + Permission::SysSpamTagCreate, + Permission::SysSpamTagUpdate, + Permission::SysSpamTagDestroy, + Permission::SysSpamTagQuery, + Permission::SysSpamTrainingSampleGet, + Permission::SysSpamTrainingSampleCreate, + Permission::SysSpamTrainingSampleDestroy, + Permission::SysSpamTrainingSampleQuery, + Permission::SysSpfReportSettingsGet, + Permission::SysSpfReportSettingsUpdate, + Permission::SysStoreLookupGet, + Permission::SysStoreLookupCreate, + Permission::SysStoreLookupUpdate, + Permission::SysStoreLookupDestroy, + Permission::SysStoreLookupQuery, + Permission::SysSystemSettingsGet, + Permission::SysSystemSettingsUpdate, + Permission::TaskCalendarAlarmEmail, + Permission::TaskCalendarAlarmNotification, + Permission::TaskCalendarItipMessage, + Permission::TaskMergeThreads, + Permission::TaskDmarcReport, + Permission::TaskTlsReport, + Permission::TaskDestroyAccount, + Permission::TaskAccountMaintenance, + Permission::TaskStoreMaintenance, + Permission::TaskSpamFilterMaintenance, + Permission::TaskAcmeRenewal, + Permission::TaskDkimManagement, + Permission::TaskDnsManagement, + Permission::TaskShareRevokeCleanup, + Permission::TaskDmarcReportWindow, + Permission::TaskTlsReportWindow, + Permission::SysTaskGet, + Permission::SysTaskCreate, + Permission::SysTaskUpdate, + Permission::SysTaskDestroy, + Permission::SysTaskQuery, + Permission::SysTaskManagerGet, + Permission::SysTaskManagerUpdate, + Permission::SysTlsExternalReportGet, + Permission::SysTlsExternalReportDestroy, + Permission::SysTlsExternalReportQuery, + Permission::SysTlsInternalReportGet, + Permission::SysTlsInternalReportUpdate, + Permission::SysTlsInternalReportDestroy, + Permission::SysTlsInternalReportQuery, + Permission::SysTlsReportSettingsGet, + Permission::SysTlsReportSettingsUpdate, + Permission::SysTracerGet, + Permission::SysTracerCreate, + Permission::SysTracerUpdate, + Permission::SysTracerDestroy, + Permission::SysTracerQuery, + Permission::SysWebDavGet, + Permission::SysWebDavUpdate, + Permission::SysWebHookGet, + Permission::SysWebHookCreate, + Permission::SysWebHookUpdate, + Permission::SysWebHookDestroy, + Permission::SysWebHookQuery, + ]; } impl serde::Serialize for Permission { @@ -8891,12 +9763,14 @@ impl<'de> serde::Deserialize<'de> for Permission { impl EnumImpl for PermissionsType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PermissionsType, b"Inherit" => PermissionsType::Inherit, b"Merge" => PermissionsType::Merge, b"Replace" => PermissionsType::Replace, } + .copied() } fn as_str(&self) -> &'static str { @@ -8944,12 +9818,14 @@ impl<'de> serde::Deserialize<'de> for PermissionsType { impl EnumImpl for PolicyEnforcement { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PolicyEnforcement, b"enforce" => PolicyEnforcement::Enforce, b"testing" => PolicyEnforcement::Testing, b"disable" => PolicyEnforcement::Disable, } + .copied() } fn as_str(&self) -> &'static str { @@ -8997,12 +9873,14 @@ impl<'de> serde::Deserialize<'de> for PolicyEnforcement { impl EnumImpl for PostgreSqlRecyclingMethod { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PostgreSqlRecyclingMethod, b"fast" => PostgreSqlRecyclingMethod::Fast, b"verified" => PostgreSqlRecyclingMethod::Verified, b"clean" => PostgreSqlRecyclingMethod::Clean, } + .copied() } fn as_str(&self) -> &'static str { @@ -9050,8 +9928,9 @@ impl<'de> serde::Deserialize<'de> for PostgreSqlRecyclingMethod { impl EnumImpl for ProviderInfo { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ProviderInfo, b"providerName" => ProviderInfo::ProviderName, b"providerShortName" => ProviderInfo::ProviderShortName, b"userDocumentation" => ProviderInfo::UserDocumentation, @@ -9061,6 +9940,7 @@ impl EnumImpl for ProviderInfo { b"logoWidth" => ProviderInfo::LogoWidth, b"logoHeight" => ProviderInfo::LogoHeight, } + .copied() } fn as_str(&self) -> &'static str { @@ -9118,13 +9998,15 @@ impl<'de> serde::Deserialize<'de> for ProviderInfo { impl EnumImpl for PublicStringOptionalType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PublicStringOptionalType, b"None" => PublicStringOptionalType::None, b"Value" => PublicStringOptionalType::Value, b"EnvironmentVariable" => PublicStringOptionalType::EnvironmentVariable, b"File" => PublicStringOptionalType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -9174,12 +10056,14 @@ impl<'de> serde::Deserialize<'de> for PublicStringOptionalType { impl EnumImpl for PublicTextType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + PublicTextType, b"Text" => PublicTextType::Text, b"EnvironmentVariable" => PublicTextType::EnvironmentVariable, b"File" => PublicTextType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -9227,11 +10111,13 @@ impl<'de> serde::Deserialize<'de> for PublicTextType { impl EnumImpl for QueueExpiryType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + QueueExpiryType, b"Ttl" => QueueExpiryType::Ttl, b"Attempts" => QueueExpiryType::Attempts, } + .copied() } fn as_str(&self) -> &'static str { @@ -9277,11 +10163,13 @@ impl<'de> serde::Deserialize<'de> for QueueExpiryType { impl EnumImpl for RecipientFlag { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + RecipientFlag, b"dsnSent" => RecipientFlag::DsnSent, b"spamPayload" => RecipientFlag::SpamPayload, } + .copied() } fn as_str(&self) -> &'static str { @@ -9327,13 +10215,15 @@ impl<'de> serde::Deserialize<'de> for RecipientFlag { impl EnumImpl for RecipientStatusType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + RecipientStatusType, b"Scheduled" => RecipientStatusType::Scheduled, b"Completed" => RecipientStatusType::Completed, b"TemporaryFailure" => RecipientStatusType::TemporaryFailure, b"PermanentFailure" => RecipientStatusType::PermanentFailure, } + .copied() } fn as_str(&self) -> &'static str { @@ -9383,11 +10273,13 @@ impl<'de> serde::Deserialize<'de> for RecipientStatusType { impl EnumImpl for RedisProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + RedisProtocol, b"resp2" => RedisProtocol::Resp2, b"resp3" => RedisProtocol::Resp3, } + .copied() } fn as_str(&self) -> &'static str { @@ -9431,13 +10323,73 @@ impl<'de> serde::Deserialize<'de> for RedisProtocol { } } +impl EnumImpl for RedisSwapStoreType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + RedisSwapStoreType, + b"Default" => RedisSwapStoreType::Default, + b"Redis" => RedisSwapStoreType::Redis, + b"RedisCluster" => RedisSwapStoreType::RedisCluster, + b"RedisSentinel" => RedisSwapStoreType::RedisSentinel, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + RedisSwapStoreType::Default => "Default", + RedisSwapStoreType::Redis => "Redis", + RedisSwapStoreType::RedisCluster => "RedisCluster", + RedisSwapStoreType::RedisSentinel => "RedisSentinel", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(RedisSwapStoreType::Default), + 1 => Some(RedisSwapStoreType::Redis), + 2 => Some(RedisSwapStoreType::RedisCluster), + 3 => Some(RedisSwapStoreType::RedisSentinel), + _ => None, + } + } + + const COUNT: usize = 4; +} + +impl serde::Serialize for RedisSwapStoreType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for RedisSwapStoreType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for RolesType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + RolesType, b"Default" => RolesType::Default, b"Custom" => RolesType::Custom, } + .copied() } fn as_str(&self) -> &'static str { @@ -9483,8 +10435,9 @@ impl<'de> serde::Deserialize<'de> for RolesType { impl EnumImpl for S3StoreRegionType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + S3StoreRegionType, b"UsEast1" => S3StoreRegionType::UsEast1, b"UsEast2" => S3StoreRegionType::UsEast2, b"UsWest1" => S3StoreRegionType::UsWest1, @@ -9529,6 +10482,7 @@ impl EnumImpl for S3StoreRegionType { b"WaApSoutheast2" => S3StoreRegionType::WaApSoutheast2, b"Custom" => S3StoreRegionType::Custom, } + .copied() } fn as_str(&self) -> &'static str { @@ -9656,16 +10610,16 @@ impl<'de> serde::Deserialize<'de> for S3StoreRegionType { impl EnumImpl for SearchCalendarField { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SearchCalendarField, b"title" => SearchCalendarField::Title, b"description" => SearchCalendarField::Description, b"location" => SearchCalendarField::Location, b"owner" => SearchCalendarField::Owner, b"attendee" => SearchCalendarField::Attendee, - b"start" => SearchCalendarField::Start, - b"uid" => SearchCalendarField::Uid, } + .copied() } fn as_str(&self) -> &'static str { @@ -9675,8 +10629,6 @@ impl EnumImpl for SearchCalendarField { SearchCalendarField::Location => "location", SearchCalendarField::Owner => "owner", SearchCalendarField::Attendee => "attendee", - SearchCalendarField::Start => "start", - SearchCalendarField::Uid => "uid", } } @@ -9691,13 +10643,11 @@ impl EnumImpl for SearchCalendarField { 2 => Some(SearchCalendarField::Location), 3 => Some(SearchCalendarField::Owner), 4 => Some(SearchCalendarField::Attendee), - 5 => Some(SearchCalendarField::Start), - 6 => Some(SearchCalendarField::Uid), _ => None, } } - const COUNT: usize = 7; + const COUNT: usize = 5; } impl serde::Serialize for SearchCalendarField { @@ -9721,8 +10671,9 @@ impl<'de> serde::Deserialize<'de> for SearchCalendarField { impl EnumImpl for SearchContactField { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SearchContactField, b"member" => SearchContactField::Member, b"kind" => SearchContactField::Kind, b"name" => SearchContactField::Name, @@ -9733,8 +10684,8 @@ impl EnumImpl for SearchContactField { b"onlineService" => SearchContactField::OnlineService, b"address" => SearchContactField::Address, b"note" => SearchContactField::Note, - b"uid" => SearchContactField::Uid, } + .copied() } fn as_str(&self) -> &'static str { @@ -9749,7 +10700,6 @@ impl EnumImpl for SearchContactField { SearchContactField::OnlineService => "onlineService", SearchContactField::Address => "address", SearchContactField::Note => "note", - SearchContactField::Uid => "uid", } } @@ -9769,12 +10719,11 @@ impl EnumImpl for SearchContactField { 7 => Some(SearchContactField::OnlineService), 8 => Some(SearchContactField::Address), 9 => Some(SearchContactField::Note), - 10 => Some(SearchContactField::Uid), _ => None, } } - const COUNT: usize = 11; + const COUNT: usize = 10; } impl serde::Serialize for SearchContactField { @@ -9798,8 +10747,9 @@ impl<'de> serde::Deserialize<'de> for SearchContactField { impl EnumImpl for SearchEmailField { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SearchEmailField, b"from" => SearchEmailField::From, b"to" => SearchEmailField::To, b"cc" => SearchEmailField::Cc, @@ -9807,12 +10757,9 @@ impl EnumImpl for SearchEmailField { b"subject" => SearchEmailField::Subject, b"body" => SearchEmailField::Body, b"attachment" => SearchEmailField::Attachment, - b"receivedAt" => SearchEmailField::ReceivedAt, - b"sentAt" => SearchEmailField::SentAt, - b"size" => SearchEmailField::Size, - b"hasAttachment" => SearchEmailField::HasAttachment, b"headers" => SearchEmailField::Headers, } + .copied() } fn as_str(&self) -> &'static str { @@ -9824,10 +10771,6 @@ impl EnumImpl for SearchEmailField { SearchEmailField::Subject => "subject", SearchEmailField::Body => "body", SearchEmailField::Attachment => "attachment", - SearchEmailField::ReceivedAt => "receivedAt", - SearchEmailField::SentAt => "sentAt", - SearchEmailField::Size => "size", - SearchEmailField::HasAttachment => "hasAttachment", SearchEmailField::Headers => "headers", } } @@ -9845,16 +10788,12 @@ impl EnumImpl for SearchEmailField { 4 => Some(SearchEmailField::Subject), 5 => Some(SearchEmailField::Body), 6 => Some(SearchEmailField::Attachment), - 7 => Some(SearchEmailField::ReceivedAt), - 8 => Some(SearchEmailField::SentAt), - 9 => Some(SearchEmailField::Size), - 10 => Some(SearchEmailField::HasAttachment), - 11 => Some(SearchEmailField::Headers), + 7 => Some(SearchEmailField::Headers), _ => None, } } - const COUNT: usize = 12; + const COUNT: usize = 8; } impl serde::Serialize for SearchEmailField { @@ -9878,11 +10817,13 @@ impl<'de> serde::Deserialize<'de> for SearchEmailField { impl EnumImpl for SearchFileField { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SearchFileField, b"name" => SearchFileField::Name, b"content" => SearchFileField::Content, } + .copied() } fn as_str(&self) -> &'static str { @@ -9928,8 +10869,9 @@ impl<'de> serde::Deserialize<'de> for SearchFileField { impl EnumImpl for SearchStoreType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SearchStoreType, b"Default" => SearchStoreType::Default, b"ElasticSearch" => SearchStoreType::ElasticSearch, b"Meilisearch" => SearchStoreType::Meilisearch, @@ -9937,6 +10879,7 @@ impl EnumImpl for SearchStoreType { b"PostgreSql" => SearchStoreType::PostgreSql, b"MySql" => SearchStoreType::MySql, } + .copied() } fn as_str(&self) -> &'static str { @@ -9988,68 +10931,17 @@ impl<'de> serde::Deserialize<'de> for SearchStoreType { } } -impl EnumImpl for SearchTracingField { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"eventType" => SearchTracingField::EventType, - b"queueId" => SearchTracingField::QueueId, - b"keywords" => SearchTracingField::Keywords, - } - } - - fn as_str(&self) -> &'static str { - match self { - SearchTracingField::EventType => "eventType", - SearchTracingField::QueueId => "queueId", - SearchTracingField::Keywords => "keywords", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(SearchTracingField::EventType), - 1 => Some(SearchTracingField::QueueId), - 2 => Some(SearchTracingField::Keywords), - _ => None, - } - } - - const COUNT: usize = 3; -} - -impl serde::Serialize for SearchTracingField { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for SearchTracingField { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for SecretKeyOptionalType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SecretKeyOptionalType, b"None" => SecretKeyOptionalType::None, b"Value" => SecretKeyOptionalType::Value, b"EnvironmentVariable" => SecretKeyOptionalType::EnvironmentVariable, b"File" => SecretKeyOptionalType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -10099,12 +10991,14 @@ impl<'de> serde::Deserialize<'de> for SecretKeyOptionalType { impl EnumImpl for SecretKeyType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SecretKeyType, b"Value" => SecretKeyType::Value, b"EnvironmentVariable" => SecretKeyType::EnvironmentVariable, b"File" => SecretKeyType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -10152,13 +11046,15 @@ impl<'de> serde::Deserialize<'de> for SecretKeyType { impl EnumImpl for SecretTextOptionalType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SecretTextOptionalType, b"None" => SecretTextOptionalType::None, b"Text" => SecretTextOptionalType::Text, b"EnvironmentVariable" => SecretTextOptionalType::EnvironmentVariable, b"File" => SecretTextOptionalType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -10208,12 +11104,14 @@ impl<'de> serde::Deserialize<'de> for SecretTextOptionalType { impl EnumImpl for SecretTextType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SecretTextType, b"Text" => SecretTextType::Text, b"EnvironmentVariable" => SecretTextType::EnvironmentVariable, b"File" => SecretTextType::File, } + .copied() } fn as_str(&self) -> &'static str { @@ -10261,8 +11159,9 @@ impl<'de> serde::Deserialize<'de> for SecretTextType { impl EnumImpl for ServiceProtocol { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ServiceProtocol, b"jmap" => ServiceProtocol::Jmap, b"imap" => ServiceProtocol::Imap, b"pop3" => ServiceProtocol::Pop3, @@ -10272,6 +11171,7 @@ impl EnumImpl for ServiceProtocol { b"webdav" => ServiceProtocol::Webdav, b"managesieve" => ServiceProtocol::Managesieve, } + .copied() } fn as_str(&self) -> &'static str { @@ -10327,10 +11227,69 @@ impl<'de> serde::Deserialize<'de> for ServiceProtocol { } } +impl EnumImpl for SharedObjectType { + fn parse(value: &str) -> Option { + hashify::map! { + value.as_bytes(), + SharedObjectType, + b"mailbox" => SharedObjectType::Mailbox, + b"calendar" => SharedObjectType::Calendar, + b"addressBook" => SharedObjectType::AddressBook, + b"fileNode" => SharedObjectType::FileNode, + } + .copied() + } + + fn as_str(&self) -> &'static str { + match self { + SharedObjectType::Mailbox => "mailbox", + SharedObjectType::Calendar => "calendar", + SharedObjectType::AddressBook => "addressBook", + SharedObjectType::FileNode => "fileNode", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(SharedObjectType::Mailbox), + 1 => Some(SharedObjectType::Calendar), + 2 => Some(SharedObjectType::AddressBook), + 3 => Some(SharedObjectType::FileNode), + _ => None, + } + } + + const COUNT: usize = 4; +} + +impl serde::Serialize for SharedObjectType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for SharedObjectType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for SieveCapability { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SieveCapability, b"envelope" => SieveCapability::Envelope, b"envelope-dsn" => SieveCapability::EnvelopeDsn, b"envelope-deliverby" => SieveCapability::EnvelopeDeliverby, @@ -10381,6 +11340,7 @@ impl EnumImpl for SieveCapability { b"vnd.inbuxa.while" => SieveCapability::VndStalwartWhile, b"vnd.inbuxa.expressions" => SieveCapability::VndStalwartExpressions, } + .copied() } fn as_str(&self) -> &'static str { @@ -10520,12 +11480,14 @@ impl<'de> serde::Deserialize<'de> for SieveCapability { impl EnumImpl for Sig0Algorithm { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + Sig0Algorithm, b"ecdsa-p256-sha256" => Sig0Algorithm::EcdsaP256Sha256, b"ecdsa-p384-sha384" => Sig0Algorithm::EcdsaP384Sha384, b"ed25519" => Sig0Algorithm::Ed25519, } + .copied() } fn as_str(&self) -> &'static str { @@ -10573,12 +11535,14 @@ impl<'de> serde::Deserialize<'de> for Sig0Algorithm { impl EnumImpl for SpamClassifierModelType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamClassifierModelType, b"FtrlFh" => SpamClassifierModelType::FtrlFh, b"FtrlCcfh" => SpamClassifierModelType::FtrlCcfh, b"Disabled" => SpamClassifierModelType::Disabled, } + .copied() } fn as_str(&self) -> &'static str { @@ -10626,13 +11590,15 @@ impl<'de> serde::Deserialize<'de> for SpamClassifierModelType { impl EnumImpl for SpamClassifyParameters { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamClassifyParameters, b"bit7" => SpamClassifyParameters::Bit7, b"bit8Mime - 8-bit MIME message content" => SpamClassifyParameters::Bit8Mime8BitMIMEMessageContent, b"binaryMime" => SpamClassifyParameters::BinaryMime, b"smtpUtf8" => SpamClassifyParameters::SmtpUtf8, } + .copied() } fn as_str(&self) -> &'static str { @@ -10684,13 +11650,15 @@ impl<'de> serde::Deserialize<'de> for SpamClassifyParameters { impl EnumImpl for SpamClassifyResult { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamClassifyResult, b"spam" => SpamClassifyResult::Spam, b"ham" => SpamClassifyResult::Ham, b"reject" => SpamClassifyResult::Reject, b"discard" => SpamClassifyResult::Discard, } + .copied() } fn as_str(&self) -> &'static str { @@ -10740,12 +11708,14 @@ impl<'de> serde::Deserialize<'de> for SpamClassifyResult { impl EnumImpl for SpamClassifyTagDisposition { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamClassifyTagDisposition, b"score" => SpamClassifyTagDisposition::Score, b"reject" => SpamClassifyTagDisposition::Reject, b"discard" => SpamClassifyTagDisposition::Discard, } + .copied() } fn as_str(&self) -> &'static str { @@ -10793,8 +11763,9 @@ impl<'de> serde::Deserialize<'de> for SpamClassifyTagDisposition { impl EnumImpl for SpamDnsblServerType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamDnsblServerType, b"Any" => SpamDnsblServerType::Any, b"Url" => SpamDnsblServerType::Url, b"Domain" => SpamDnsblServerType::Domain, @@ -10803,6 +11774,7 @@ impl EnumImpl for SpamDnsblServerType { b"Header" => SpamDnsblServerType::Header, b"Body" => SpamDnsblServerType::Body, } + .copied() } fn as_str(&self) -> &'static str { @@ -10856,60 +11828,11 @@ impl<'de> serde::Deserialize<'de> for SpamDnsblServerType { } } -impl EnumImpl for SpamLlmType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Disable" => SpamLlmType::Disable, - b"Enable" => SpamLlmType::Enable, - } - } - - fn as_str(&self) -> &'static str { - match self { - SpamLlmType::Disable => "Disable", - SpamLlmType::Enable => "Enable", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(SpamLlmType::Disable), - 1 => Some(SpamLlmType::Enable), - _ => None, - } - } - - const COUNT: usize = 2; -} - -impl serde::Serialize for SpamLlmType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for SpamLlmType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for SpamRuleType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamRuleType, b"Any" => SpamRuleType::Any, b"Url" => SpamRuleType::Url, b"Domain" => SpamRuleType::Domain, @@ -10918,6 +11841,7 @@ impl EnumImpl for SpamRuleType { b"Header" => SpamRuleType::Header, b"Body" => SpamRuleType::Body, } + .copied() } fn as_str(&self) -> &'static str { @@ -10973,12 +11897,14 @@ impl<'de> serde::Deserialize<'de> for SpamRuleType { impl EnumImpl for SpamTagType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpamTagType, b"Score" => SpamTagType::Score, b"Discard" => SpamTagType::Discard, b"Reject" => SpamTagType::Reject, } + .copied() } fn as_str(&self) -> &'static str { @@ -11026,8 +11952,9 @@ impl<'de> serde::Deserialize<'de> for SpamTagType { impl EnumImpl for SpecialUse { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpecialUse, b"inbox" => SpecialUse::Inbox, b"trash" => SpecialUse::Trash, b"junk" => SpecialUse::Junk, @@ -11040,6 +11967,7 @@ impl EnumImpl for SpecialUse { b"scheduled" => SpecialUse::Scheduled, b"snoozed" => SpecialUse::Snoozed, } + .copied() } fn as_str(&self) -> &'static str { @@ -11103,8 +12031,9 @@ impl<'de> serde::Deserialize<'de> for SpecialUse { impl EnumImpl for SpfAuthResult { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpfAuthResult, b"none" => SpfAuthResult::None, b"neutral" => SpfAuthResult::Neutral, b"pass" => SpfAuthResult::Pass, @@ -11113,6 +12042,7 @@ impl EnumImpl for SpfAuthResult { b"tempError" => SpfAuthResult::TempError, b"permError" => SpfAuthResult::PermError, } + .copied() } fn as_str(&self) -> &'static str { @@ -11168,12 +12098,14 @@ impl<'de> serde::Deserialize<'de> for SpfAuthResult { impl EnumImpl for SpfDomainScope { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SpfDomainScope, b"helo" => SpfDomainScope::Helo, b"mailFrom" => SpfDomainScope::MailFrom, b"unspecified" => SpfDomainScope::Unspecified, } + .copied() } fn as_str(&self) -> &'static str { @@ -11221,13 +12153,15 @@ impl<'de> serde::Deserialize<'de> for SpfDomainScope { impl EnumImpl for SqlAuthStoreType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SqlAuthStoreType, b"Default" => SqlAuthStoreType::Default, b"PostgreSql" => SqlAuthStoreType::PostgreSql, b"MySql" => SqlAuthStoreType::MySql, b"Sqlite" => SqlAuthStoreType::Sqlite, } + .copied() } fn as_str(&self) -> &'static str { @@ -11276,9 +12210,11 @@ impl<'de> serde::Deserialize<'de> for SqlAuthStoreType { } impl EnumImpl for StorageQuota { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + StorageQuota, b"maxEmails" => StorageQuota::MaxEmails, b"maxMailboxes" => StorageQuota::MaxMailboxes, b"maxEmailSubmissions" => StorageQuota::MaxEmailSubmissions, @@ -11293,12 +12229,12 @@ impl EnumImpl for StorageQuota { b"maxContactCards" => StorageQuota::MaxContactCards, b"maxFiles" => StorageQuota::MaxFiles, b"maxFolders" => StorageQuota::MaxFolders, - b"maxMaskedAddresses" => StorageQuota::MaxMaskedAddresses, b"maxAppPasswords" => StorageQuota::MaxAppPasswords, b"maxApiKeys" => StorageQuota::MaxApiKeys, b"maxPublicKeys" => StorageQuota::MaxPublicKeys, b"maxDiskQuota" => StorageQuota::MaxDiskQuota, } + .copied() } fn as_str(&self) -> &'static str { @@ -11317,7 +12253,6 @@ impl EnumImpl for StorageQuota { StorageQuota::MaxContactCards => "maxContactCards", StorageQuota::MaxFiles => "maxFiles", StorageQuota::MaxFolders => "maxFolders", - StorageQuota::MaxMaskedAddresses => "maxMaskedAddresses", StorageQuota::MaxAppPasswords => "maxAppPasswords", StorageQuota::MaxApiKeys => "maxApiKeys", StorageQuota::MaxPublicKeys => "maxPublicKeys", @@ -11345,7 +12280,6 @@ impl EnumImpl for StorageQuota { 11 => Some(StorageQuota::MaxContactCards), 12 => Some(StorageQuota::MaxFiles), 13 => Some(StorageQuota::MaxFolders), - 14 => Some(StorageQuota::MaxMaskedAddresses), 15 => Some(StorageQuota::MaxAppPasswords), 16 => Some(StorageQuota::MaxApiKeys), 17 => Some(StorageQuota::MaxPublicKeys), @@ -11378,12 +12312,14 @@ impl<'de> serde::Deserialize<'de> for StorageQuota { impl EnumImpl for SubAddressingType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + SubAddressingType, b"Enabled" => SubAddressingType::Enabled, b"Custom" => SubAddressingType::Custom, b"Disabled" => SubAddressingType::Disabled, } + .copied() } fn as_str(&self) -> &'static str { @@ -11431,13 +12367,16 @@ impl<'de> serde::Deserialize<'de> for SubAddressingType { impl EnumImpl for TaskAccountMaintenanceType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TaskAccountMaintenanceType, b"purge" => TaskAccountMaintenanceType::Purge, b"reindex" => TaskAccountMaintenanceType::Reindex, b"recalculateImapUid" => TaskAccountMaintenanceType::RecalculateImapUid, b"recalculateQuota" => TaskAccountMaintenanceType::RecalculateQuota, + b"rebuildShares" => TaskAccountMaintenanceType::RebuildShares, } + .copied() } fn as_str(&self) -> &'static str { @@ -11446,6 +12385,7 @@ impl EnumImpl for TaskAccountMaintenanceType { TaskAccountMaintenanceType::Reindex => "reindex", TaskAccountMaintenanceType::RecalculateImapUid => "recalculateImapUid", TaskAccountMaintenanceType::RecalculateQuota => "recalculateQuota", + TaskAccountMaintenanceType::RebuildShares => "rebuildShares", } } @@ -11459,11 +12399,12 @@ impl EnumImpl for TaskAccountMaintenanceType { 1 => Some(TaskAccountMaintenanceType::Reindex), 2 => Some(TaskAccountMaintenanceType::RecalculateImapUid), 3 => Some(TaskAccountMaintenanceType::RecalculateQuota), + 4 => Some(TaskAccountMaintenanceType::RebuildShares), _ => None, } } - const COUNT: usize = 4; + const COUNT: usize = 5; } impl serde::Serialize for TaskAccountMaintenanceType { @@ -11487,11 +12428,13 @@ impl<'de> serde::Deserialize<'de> for TaskAccountMaintenanceType { impl EnumImpl for TaskRetryStrategyType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TaskRetryStrategyType, b"ExponentialBackoff" => TaskRetryStrategyType::ExponentialBackoff, b"FixedDelay" => TaskRetryStrategyType::FixedDelay, } + .copied() } fn as_str(&self) -> &'static str { @@ -11537,14 +12480,16 @@ impl<'de> serde::Deserialize<'de> for TaskRetryStrategyType { impl EnumImpl for TaskSpamFilterMaintenanceType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TaskSpamFilterMaintenanceType, b"train" => TaskSpamFilterMaintenanceType::Train, b"retrain" => TaskSpamFilterMaintenanceType::Retrain, b"abort" => TaskSpamFilterMaintenanceType::Abort, b"reset" => TaskSpamFilterMaintenanceType::Reset, b"updateRules" => TaskSpamFilterMaintenanceType::UpdateRules, } + .copied() } fn as_str(&self) -> &'static str { @@ -11596,12 +12541,15 @@ impl<'de> serde::Deserialize<'de> for TaskSpamFilterMaintenanceType { impl EnumImpl for TaskStatusType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TaskStatusType, b"Pending" => TaskStatusType::Pending, b"Retry" => TaskStatusType::Retry, b"Failed" => TaskStatusType::Failed, + b"Stalled" => TaskStatusType::Stalled, } + .copied() } fn as_str(&self) -> &'static str { @@ -11609,6 +12557,7 @@ impl EnumImpl for TaskStatusType { TaskStatusType::Pending => "Pending", TaskStatusType::Retry => "Retry", TaskStatusType::Failed => "Failed", + TaskStatusType::Stalled => "Stalled", } } @@ -11621,11 +12570,12 @@ impl EnumImpl for TaskStatusType { 0 => Some(TaskStatusType::Pending), 1 => Some(TaskStatusType::Retry), 2 => Some(TaskStatusType::Failed), + 3 => Some(TaskStatusType::Stalled), _ => None, } } - const COUNT: usize = 3; + const COUNT: usize = 4; } impl serde::Serialize for TaskStatusType { @@ -11648,43 +12598,40 @@ impl<'de> serde::Deserialize<'de> for TaskStatusType { } impl EnumImpl for TaskStoreMaintenanceType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TaskStoreMaintenanceType, b"reindexAccounts" => TaskStoreMaintenanceType::ReindexAccounts, - b"reindexTelemetry" => TaskStoreMaintenanceType::ReindexTelemetry, b"purgeAccounts" => TaskStoreMaintenanceType::PurgeAccounts, b"purgeData" => TaskStoreMaintenanceType::PurgeData, b"purgeBlob" => TaskStoreMaintenanceType::PurgeBlob, b"resetRateLimiters" => TaskStoreMaintenanceType::ResetRateLimiters, b"resetUserQuotas" => TaskStoreMaintenanceType::ResetUserQuotas, - b"resetTenantQuotas" => TaskStoreMaintenanceType::ResetTenantQuotas, b"resetBlobQuotas" => TaskStoreMaintenanceType::ResetBlobQuotas, b"removeAuthTokens" => TaskStoreMaintenanceType::RemoveAuthTokens, b"removeLockQueueMessage" => TaskStoreMaintenanceType::RemoveLockQueueMessage, b"removeLockTask" => TaskStoreMaintenanceType::RemoveLockTask, b"removeLockDav" => TaskStoreMaintenanceType::RemoveLockDav, - b"removeSieveId" => TaskStoreMaintenanceType::RemoveSieveId, b"removeGreylist" => TaskStoreMaintenanceType::RemoveGreylist, } + .copied() } fn as_str(&self) -> &'static str { match self { TaskStoreMaintenanceType::ReindexAccounts => "reindexAccounts", - TaskStoreMaintenanceType::ReindexTelemetry => "reindexTelemetry", TaskStoreMaintenanceType::PurgeAccounts => "purgeAccounts", TaskStoreMaintenanceType::PurgeData => "purgeData", TaskStoreMaintenanceType::PurgeBlob => "purgeBlob", TaskStoreMaintenanceType::ResetRateLimiters => "resetRateLimiters", TaskStoreMaintenanceType::ResetUserQuotas => "resetUserQuotas", - TaskStoreMaintenanceType::ResetTenantQuotas => "resetTenantQuotas", TaskStoreMaintenanceType::ResetBlobQuotas => "resetBlobQuotas", TaskStoreMaintenanceType::RemoveAuthTokens => "removeAuthTokens", TaskStoreMaintenanceType::RemoveLockQueueMessage => "removeLockQueueMessage", TaskStoreMaintenanceType::RemoveLockTask => "removeLockTask", TaskStoreMaintenanceType::RemoveLockDav => "removeLockDav", - TaskStoreMaintenanceType::RemoveSieveId => "removeSieveId", TaskStoreMaintenanceType::RemoveGreylist => "removeGreylist", } } @@ -11696,25 +12643,22 @@ impl EnumImpl for TaskStoreMaintenanceType { fn from_id(id: u16) -> Option { match id { 0 => Some(TaskStoreMaintenanceType::ReindexAccounts), - 1 => Some(TaskStoreMaintenanceType::ReindexTelemetry), 2 => Some(TaskStoreMaintenanceType::PurgeAccounts), 3 => Some(TaskStoreMaintenanceType::PurgeData), 4 => Some(TaskStoreMaintenanceType::PurgeBlob), 5 => Some(TaskStoreMaintenanceType::ResetRateLimiters), 6 => Some(TaskStoreMaintenanceType::ResetUserQuotas), - 7 => Some(TaskStoreMaintenanceType::ResetTenantQuotas), 8 => Some(TaskStoreMaintenanceType::ResetBlobQuotas), 9 => Some(TaskStoreMaintenanceType::RemoveAuthTokens), 10 => Some(TaskStoreMaintenanceType::RemoveLockQueueMessage), 11 => Some(TaskStoreMaintenanceType::RemoveLockTask), 12 => Some(TaskStoreMaintenanceType::RemoveLockDav), - 13 => Some(TaskStoreMaintenanceType::RemoveSieveId), - 14 => Some(TaskStoreMaintenanceType::RemoveGreylist), + 13 => Some(TaskStoreMaintenanceType::RemoveGreylist), _ => None, } } - const COUNT: usize = 15; + const COUNT: usize = 14; } impl serde::Serialize for TaskStoreMaintenanceType { @@ -11736,98 +12680,50 @@ impl<'de> serde::Deserialize<'de> for TaskStoreMaintenanceType { } } -impl EnumImpl for TaskTenantMaintenanceType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"recalculateQuota" => TaskTenantMaintenanceType::RecalculateQuota, - } - } - - fn as_str(&self) -> &'static str { - match self { - TaskTenantMaintenanceType::RecalculateQuota => "recalculateQuota", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(TaskTenantMaintenanceType::RecalculateQuota), - _ => None, - } - } - - const COUNT: usize = 1; -} - -impl serde::Serialize for TaskTenantMaintenanceType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for TaskTenantMaintenanceType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for TaskType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), - b"IndexDocument" => TaskType::IndexDocument, - b"UnindexDocument" => TaskType::UnindexDocument, - b"IndexTrace" => TaskType::IndexTrace, + TaskType, b"CalendarAlarmEmail" => TaskType::CalendarAlarmEmail, b"CalendarAlarmNotification" => TaskType::CalendarAlarmNotification, b"CalendarItipMessage" => TaskType::CalendarItipMessage, b"MergeThreads" => TaskType::MergeThreads, b"DmarcReport" => TaskType::DmarcReport, b"TlsReport" => TaskType::TlsReport, - b"RestoreArchivedItem" => TaskType::RestoreArchivedItem, b"DestroyAccount" => TaskType::DestroyAccount, b"AccountMaintenance" => TaskType::AccountMaintenance, - b"TenantMaintenance" => TaskType::TenantMaintenance, b"StoreMaintenance" => TaskType::StoreMaintenance, b"SpamFilterMaintenance" => TaskType::SpamFilterMaintenance, b"AcmeRenewal" => TaskType::AcmeRenewal, b"DkimManagement" => TaskType::DkimManagement, b"DnsManagement" => TaskType::DnsManagement, + b"ShareRevokeCleanup" => TaskType::ShareRevokeCleanup, + b"DmarcReportWindow" => TaskType::DmarcReportWindow, + b"TlsReportWindow" => TaskType::TlsReportWindow, } + .copied() } fn as_str(&self) -> &'static str { match self { - TaskType::IndexDocument => "IndexDocument", - TaskType::UnindexDocument => "UnindexDocument", - TaskType::IndexTrace => "IndexTrace", TaskType::CalendarAlarmEmail => "CalendarAlarmEmail", TaskType::CalendarAlarmNotification => "CalendarAlarmNotification", TaskType::CalendarItipMessage => "CalendarItipMessage", TaskType::MergeThreads => "MergeThreads", TaskType::DmarcReport => "DmarcReport", TaskType::TlsReport => "TlsReport", - TaskType::RestoreArchivedItem => "RestoreArchivedItem", TaskType::DestroyAccount => "DestroyAccount", TaskType::AccountMaintenance => "AccountMaintenance", - TaskType::TenantMaintenance => "TenantMaintenance", TaskType::StoreMaintenance => "StoreMaintenance", TaskType::SpamFilterMaintenance => "SpamFilterMaintenance", TaskType::AcmeRenewal => "AcmeRenewal", TaskType::DkimManagement => "DkimManagement", TaskType::DnsManagement => "DnsManagement", + TaskType::ShareRevokeCleanup => "ShareRevokeCleanup", + TaskType::DmarcReportWindow => "DmarcReportWindow", + TaskType::TlsReportWindow => "TlsReportWindow", } } @@ -11837,24 +12733,22 @@ impl EnumImpl for TaskType { fn from_id(id: u16) -> Option { match id { - 0 => Some(TaskType::IndexDocument), - 1 => Some(TaskType::UnindexDocument), - 2 => Some(TaskType::IndexTrace), - 3 => Some(TaskType::CalendarAlarmEmail), - 4 => Some(TaskType::CalendarAlarmNotification), - 5 => Some(TaskType::CalendarItipMessage), - 6 => Some(TaskType::MergeThreads), - 7 => Some(TaskType::DmarcReport), - 8 => Some(TaskType::TlsReport), - 9 => Some(TaskType::RestoreArchivedItem), - 10 => Some(TaskType::DestroyAccount), - 11 => Some(TaskType::AccountMaintenance), - 12 => Some(TaskType::TenantMaintenance), - 13 => Some(TaskType::StoreMaintenance), - 14 => Some(TaskType::SpamFilterMaintenance), - 15 => Some(TaskType::AcmeRenewal), - 16 => Some(TaskType::DkimManagement), - 17 => Some(TaskType::DnsManagement), + 0 => Some(TaskType::CalendarAlarmEmail), + 1 => Some(TaskType::CalendarAlarmNotification), + 2 => Some(TaskType::CalendarItipMessage), + 3 => Some(TaskType::MergeThreads), + 4 => Some(TaskType::DmarcReport), + 5 => Some(TaskType::TlsReport), + 7 => Some(TaskType::DestroyAccount), + 8 => Some(TaskType::AccountMaintenance), + 10 => Some(TaskType::StoreMaintenance), + 11 => Some(TaskType::SpamFilterMaintenance), + 12 => Some(TaskType::AcmeRenewal), + 13 => Some(TaskType::DkimManagement), + 14 => Some(TaskType::DnsManagement), + 15 => Some(TaskType::ShareRevokeCleanup), + 16 => Some(TaskType::DmarcReportWindow), + 17 => Some(TaskType::TlsReportWindow), _ => None, } } @@ -11862,6 +12756,27 @@ impl EnumImpl for TaskType { const COUNT: usize = 18; } +impl TaskType { + pub const ALL: &[TaskType] = &[ + TaskType::CalendarAlarmEmail, + TaskType::CalendarAlarmNotification, + TaskType::CalendarItipMessage, + TaskType::MergeThreads, + TaskType::DmarcReport, + TaskType::TlsReport, + TaskType::DestroyAccount, + TaskType::AccountMaintenance, + TaskType::StoreMaintenance, + TaskType::SpamFilterMaintenance, + TaskType::AcmeRenewal, + TaskType::DkimManagement, + TaskType::DnsManagement, + TaskType::ShareRevokeCleanup, + TaskType::DmarcReportWindow, + TaskType::TlsReportWindow, + ]; +} + impl serde::Serialize for TaskType { fn serialize(&self, serializer: S) -> Result where @@ -11881,83 +12796,6 @@ impl<'de> serde::Deserialize<'de> for TaskType { } } -impl EnumImpl for TenantStorageQuota { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"maxAccounts" => TenantStorageQuota::MaxAccounts, - b"maxGroups" => TenantStorageQuota::MaxGroups, - b"maxDomains" => TenantStorageQuota::MaxDomains, - b"maxMailingLists" => TenantStorageQuota::MaxMailingLists, - b"maxRoles" => TenantStorageQuota::MaxRoles, - b"maxOauthClients" => TenantStorageQuota::MaxOauthClients, - b"maxDkimKeys" => TenantStorageQuota::MaxDkimKeys, - b"maxDnsServers" => TenantStorageQuota::MaxDnsServers, - b"maxDirectories" => TenantStorageQuota::MaxDirectories, - b"maxAcmeProviders" => TenantStorageQuota::MaxAcmeProviders, - b"maxDiskQuota" => TenantStorageQuota::MaxDiskQuota, - } - } - - fn as_str(&self) -> &'static str { - match self { - TenantStorageQuota::MaxAccounts => "maxAccounts", - TenantStorageQuota::MaxGroups => "maxGroups", - TenantStorageQuota::MaxDomains => "maxDomains", - TenantStorageQuota::MaxMailingLists => "maxMailingLists", - TenantStorageQuota::MaxRoles => "maxRoles", - TenantStorageQuota::MaxOauthClients => "maxOauthClients", - TenantStorageQuota::MaxDkimKeys => "maxDkimKeys", - TenantStorageQuota::MaxDnsServers => "maxDnsServers", - TenantStorageQuota::MaxDirectories => "maxDirectories", - TenantStorageQuota::MaxAcmeProviders => "maxAcmeProviders", - TenantStorageQuota::MaxDiskQuota => "maxDiskQuota", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(TenantStorageQuota::MaxAccounts), - 1 => Some(TenantStorageQuota::MaxGroups), - 2 => Some(TenantStorageQuota::MaxDomains), - 3 => Some(TenantStorageQuota::MaxMailingLists), - 4 => Some(TenantStorageQuota::MaxRoles), - 5 => Some(TenantStorageQuota::MaxOauthClients), - 6 => Some(TenantStorageQuota::MaxDkimKeys), - 7 => Some(TenantStorageQuota::MaxDnsServers), - 8 => Some(TenantStorageQuota::MaxDirectories), - 9 => Some(TenantStorageQuota::MaxAcmeProviders), - 10 => Some(TenantStorageQuota::MaxDiskQuota), - _ => None, - } - } - - const COUNT: usize = 11; -} - -impl serde::Serialize for TenantStorageQuota { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for TenantStorageQuota { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for TimeZone { fn parse(value: &str) -> Option { hashify::map! { @@ -13800,8 +14638,9 @@ impl<'de> serde::Deserialize<'de> for TimeZone { impl EnumImpl for TlsCipherSuite { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TlsCipherSuite, b"tls13-aes-256-gcm-sha384" => TlsCipherSuite::Tls13Aes256GcmSha384, b"tls13-aes-128-gcm-sha256" => TlsCipherSuite::Tls13Aes128GcmSha256, b"tls13-chacha20-poly1305-sha256" => TlsCipherSuite::Tls13Chacha20Poly1305Sha256, @@ -13812,6 +14651,7 @@ impl EnumImpl for TlsCipherSuite { b"tls-ecdhe-rsa-with-aes-128-gcm-sha256" => TlsCipherSuite::TlsEcdheRsaWithAes128GcmSha256, b"tls-ecdhe-rsa-with-chacha20-poly1305-sha256" => TlsCipherSuite::TlsEcdheRsaWithChacha20Poly1305Sha256, } + .copied() } fn as_str(&self) -> &'static str { @@ -13883,13 +14723,15 @@ impl<'de> serde::Deserialize<'de> for TlsCipherSuite { impl EnumImpl for TlsPolicyType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TlsPolicyType, b"tlsa" => TlsPolicyType::Tlsa, b"sts" => TlsPolicyType::Sts, b"noPolicyFound" => TlsPolicyType::NoPolicyFound, b"other" => TlsPolicyType::Other, } + .copied() } fn as_str(&self) -> &'static str { @@ -13939,8 +14781,9 @@ impl<'de> serde::Deserialize<'de> for TlsPolicyType { impl EnumImpl for TlsResultType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TlsResultType, b"startTlsNotSupported" => TlsResultType::StartTlsNotSupported, b"certificateHostMismatch" => TlsResultType::CertificateHostMismatch, b"certificateExpired" => TlsResultType::CertificateExpired, @@ -13954,6 +14797,7 @@ impl EnumImpl for TlsResultType { b"stsWebpkiInvalid" => TlsResultType::StsWebpkiInvalid, b"other" => TlsResultType::Other, } + .copied() } fn as_str(&self) -> &'static str { @@ -14019,11 +14863,13 @@ impl<'de> serde::Deserialize<'de> for TlsResultType { impl EnumImpl for TlsVersion { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TlsVersion, b"tls12" => TlsVersion::Tls12, b"tls13" => TlsVersion::Tls13, } + .copied() } fn as_str(&self) -> &'static str { @@ -14067,93 +14913,18 @@ impl<'de> serde::Deserialize<'de> for TlsVersion { } } -impl EnumImpl for TraceValueType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"String" => TraceValueType::String, - b"UnsignedInt" => TraceValueType::UnsignedInt, - b"Integer" => TraceValueType::Integer, - b"Boolean" => TraceValueType::Boolean, - b"Float" => TraceValueType::Float, - b"UTCDateTime" => TraceValueType::UTCDateTime, - b"Duration" => TraceValueType::Duration, - b"IpAddr" => TraceValueType::IpAddr, - b"List" => TraceValueType::List, - b"Event" => TraceValueType::Event, - b"Null" => TraceValueType::Null, - } - } - - fn as_str(&self) -> &'static str { - match self { - TraceValueType::String => "String", - TraceValueType::UnsignedInt => "UnsignedInt", - TraceValueType::Integer => "Integer", - TraceValueType::Boolean => "Boolean", - TraceValueType::Float => "Float", - TraceValueType::UTCDateTime => "UTCDateTime", - TraceValueType::Duration => "Duration", - TraceValueType::IpAddr => "IpAddr", - TraceValueType::List => "List", - TraceValueType::Event => "Event", - TraceValueType::Null => "Null", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(TraceValueType::String), - 1 => Some(TraceValueType::UnsignedInt), - 2 => Some(TraceValueType::Integer), - 3 => Some(TraceValueType::Boolean), - 4 => Some(TraceValueType::Float), - 5 => Some(TraceValueType::UTCDateTime), - 6 => Some(TraceValueType::Duration), - 7 => Some(TraceValueType::IpAddr), - 8 => Some(TraceValueType::List), - 9 => Some(TraceValueType::Event), - 10 => Some(TraceValueType::Null), - _ => None, - } - } - - const COUNT: usize = 11; -} - -impl serde::Serialize for TraceValueType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for TraceValueType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for TracerType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TracerType, b"Log" => TracerType::Log, b"Stdout" => TracerType::Stdout, b"Journal" => TracerType::Journal, b"OtelHttp" => TracerType::OtelHttp, b"OtelGrpc" => TracerType::OtelGrpc, } + .copied() } fn as_str(&self) -> &'static str { @@ -14205,14 +14976,16 @@ impl<'de> serde::Deserialize<'de> for TracerType { impl EnumImpl for TracingLevel { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TracingLevel, b"error" => TracingLevel::Error, b"warn" => TracingLevel::Warn, b"info" => TracingLevel::Info, b"debug" => TracingLevel::Debug, b"trace" => TracingLevel::Trace, } + .copied() } fn as_str(&self) -> &'static str { @@ -14264,8 +15037,9 @@ impl<'de> serde::Deserialize<'de> for TracingLevel { impl EnumImpl for TracingLevelOpt { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TracingLevelOpt, b"disable" => TracingLevelOpt::Disable, b"error" => TracingLevelOpt::Error, b"warn" => TracingLevelOpt::Warn, @@ -14273,6 +15047,7 @@ impl EnumImpl for TracingLevelOpt { b"debug" => TracingLevelOpt::Debug, b"trace" => TracingLevelOpt::Trace, } + .copied() } fn as_str(&self) -> &'static str { @@ -14324,69 +15099,11 @@ impl<'de> serde::Deserialize<'de> for TracingLevelOpt { } } -impl EnumImpl for TracingStoreType { - fn parse(value: &str) -> Option { - hashify::tiny_map! { - value.as_bytes(), - b"Disabled" => TracingStoreType::Disabled, - b"Default" => TracingStoreType::Default, - b"FoundationDb" => TracingStoreType::FoundationDb, - b"PostgreSql" => TracingStoreType::PostgreSql, - b"MySql" => TracingStoreType::MySql, - } - } - - fn as_str(&self) -> &'static str { - match self { - TracingStoreType::Disabled => "Disabled", - TracingStoreType::Default => "Default", - TracingStoreType::FoundationDb => "FoundationDb", - TracingStoreType::PostgreSql => "PostgreSql", - TracingStoreType::MySql => "MySql", - } - } - - fn to_id(&self) -> u16 { - *self as u16 - } - - fn from_id(id: u16) -> Option { - match id { - 0 => Some(TracingStoreType::Disabled), - 1 => Some(TracingStoreType::Default), - 2 => Some(TracingStoreType::FoundationDb), - 3 => Some(TracingStoreType::PostgreSql), - 4 => Some(TracingStoreType::MySql), - _ => None, - } - } - - const COUNT: usize = 5; -} - -impl serde::Serialize for TracingStoreType { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for TracingStoreType { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) - } -} - impl EnumImpl for TsigAlgorithm { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + TsigAlgorithm, b"hmac-md5" => TsigAlgorithm::HmacMd5, b"gss" => TsigAlgorithm::Gss, b"hmac-sha1" => TsigAlgorithm::HmacSha1, @@ -14398,6 +15115,7 @@ impl EnumImpl for TsigAlgorithm { b"hmac-sha512" => TsigAlgorithm::HmacSha512, b"hmac-sha512-256" => TsigAlgorithm::HmacSha512256, } + .copied() } fn as_str(&self) -> &'static str { @@ -14459,12 +15177,14 @@ impl<'de> serde::Deserialize<'de> for TsigAlgorithm { impl EnumImpl for UserRolesType { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + UserRolesType, b"User" => UserRolesType::User, b"Admin" => UserRolesType::Admin, b"Custom" => UserRolesType::Custom, } + .copied() } fn as_str(&self) -> &'static str { @@ -14512,11 +15232,13 @@ impl<'de> serde::Deserialize<'de> for UserRolesType { impl EnumImpl for VCardVersion { fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + VCardVersion, b"v4" => VCardVersion::V4, b"v3" => VCardVersion::V3, } + .copied() } fn as_str(&self) -> &'static str { diff --git a/crates/registry/src/schema/mod.rs b/crates/registry/src/schema/mod.rs index 4ee88b4e..0e83736e 100644 --- a/crates/registry/src/schema/mod.rs +++ b/crates/registry/src/schema/mod.rs @@ -137,3 +137,4 @@ impl From for String { value.as_str().to_string() } } + diff --git a/crates/registry/src/schema/prelude.rs b/crates/registry/src/schema/prelude.rs index 322ced78..84de7deb 100644 --- a/crates/registry/src/schema/prelude.rs +++ b/crates/registry/src/schema/prelude.rs @@ -8,6 +8,7 @@ pub use crate::jmap::JmapValue; pub use crate::jmap::MaybeUnpatched; pub use crate::jmap::PatchResult; pub use crate::jmap::RegistryJsonEnumPatch; +pub use crate::jmap::ValueOptions; pub use crate::jmap::{ JsonPointerPatch, RegistryJsonPatch, RegistryJsonPropertyPatch, patch::object_type, }; @@ -21,7 +22,9 @@ pub use crate::types::datetime::UTCDateTime; pub use crate::types::duration::Duration; pub use crate::types::error::*; pub use crate::types::float::Float; -pub use crate::types::index::{IndexBuilder, IndexSchema, IndexSchemaType, IndexSchemaValueType}; +pub use crate::types::index::{ + IndexBuilder, IndexSchema, IndexSchemaType, IndexSchemaValueType, IndexValue, +}; pub use crate::types::ipaddr::IpAddr; pub use crate::types::ipmask::IpAddrOrMask; pub use crate::types::list::List; @@ -31,7 +34,6 @@ pub use crate::types::string::StringValidator; pub use serde::{Deserialize, Serialize}; pub use std::borrow::Cow; pub use std::str::FromStr; -use std::sync::LazyLock; pub use types::blob::BlobId; pub use types::id::Id; pub use utils::map::vec_map::VecMap; @@ -55,19 +57,11 @@ pub const OBJ_SINGLETON: u64 = 1; pub const OBJ_SEQ_ID: u64 = 1 << 1; pub const OBJ_FILTER_ACCOUNT: u64 = 1 << 2; pub const OBJ_FILTER_TENANT: u64 = 1 << 3; +pub const OBJ_NOT_STORED: u64 = 1 << 4; +pub const OBJ_NO_GET: u64 = 1 << 5; +pub const OBJ_NO_CREATE: u64 = 1 << 6; +pub const OBJ_NO_UPDATE: u64 = 1 << 7; +pub const OBJ_NO_DESTROY: u64 = 1 << 8; +pub const OBJ_NO_QUERY: u64 = 1 << 9; pub const MASKED_PASSWORD: &str = "****"; - -// Temporary migration aid, will be removed by v1. -pub static INCLUDE_SECRETS: LazyLock = LazyLock::new(|| { - std::env::var("STALWART_INCLUDE_SECRETS") - .is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) -}); - -pub fn secret_value(value: impl IntoValue) -> JmapValue<'static> { - if *INCLUDE_SECRETS { - value.into_value() - } else { - JmapValue::Str(MASKED_PASSWORD.into()) - } -} diff --git a/crates/registry/src/schema/properties.rs b/crates/registry/src/schema/properties.rs index e7469dfa..4fcc9147 100644 --- a/crates/registry/src/schema/properties.rs +++ b/crates/registry/src/schema/properties.rs @@ -10,123 +10,115 @@ use crate::schema::prelude::*; #[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(untagged)] pub enum ObjectInner { - Account(Account), - AccountPassword(AccountPassword), - AccountSettings(AccountSettings), - AcmeProvider(AcmeProvider), - Action(Action), - AddressBook(AddressBook), - AiModel(AiModel), - Alert(Alert), - AllowedIp(AllowedIp), - ApiKey(ApiKey), - AppPassword(AppPassword), - Application(Application), - ArchivedItem(ArchivedItem), - ArfExternalReport(ArfExternalReport), - Asn(Asn), - Authentication(Authentication), - BlobStore(BlobStore), - BlockedIp(BlockedIp), - Bootstrap(Bootstrap), - Cache(Cache), - Calendar(Calendar), - CalendarAlarm(CalendarAlarm), - CalendarScheduling(CalendarScheduling), - Certificate(Certificate), - ClusterNode(ClusterNode), - ClusterRole(ClusterRole), - Coordinator(Coordinator), - DataRetention(DataRetention), - DataStore(DataStore), - Directory(Directory), - DkimReportSettings(DkimReportSettings), - DkimSignature(DkimSignature), - DmarcExternalReport(DmarcExternalReport), - DmarcInternalReport(DmarcInternalReport), - DmarcReportSettings(DmarcReportSettings), - DnsResolver(DnsResolver), - DnsServer(DnsServer), - Domain(Domain), - DsnReportSettings(DsnReportSettings), - Email(Email), - Enterprise(Enterprise), - EventTracingLevel(EventTracingLevel), - FileStorage(FileStorage), - Http(Http), - HttpForm(HttpForm), - HttpLookup(HttpLookup), - Imap(Imap), - InMemoryStore(InMemoryStore), - Jmap(Jmap), - Log(Log), - MailingList(MailingList), - MaskedEmail(MaskedEmail), - MemoryLookupKey(MemoryLookupKey), - MemoryLookupKeyValue(MemoryLookupKeyValue), - Metric(Metric), - Metrics(Metrics), - MetricsStore(MetricsStore), - MtaConnectionStrategy(MtaConnectionStrategy), - MtaDeliverySchedule(MtaDeliverySchedule), - MtaExtensions(MtaExtensions), - MtaHook(MtaHook), - MtaInboundSession(MtaInboundSession), - MtaInboundThrottle(MtaInboundThrottle), - MtaMilter(MtaMilter), - MtaOutboundStrategy(MtaOutboundStrategy), - MtaOutboundThrottle(MtaOutboundThrottle), - MtaQueueQuota(MtaQueueQuota), - MtaRoute(MtaRoute), - MtaStageAuth(MtaStageAuth), - MtaStageConnect(MtaStageConnect), - MtaStageData(MtaStageData), - MtaStageEhlo(MtaStageEhlo), - MtaStageMail(MtaStageMail), - MtaStageRcpt(MtaStageRcpt), - MtaSts(MtaSts), - MtaTlsStrategy(MtaTlsStrategy), - MtaVirtualQueue(MtaVirtualQueue), - NetworkListener(NetworkListener), - OAuthClient(OAuthClient), - OidcProvider(OidcProvider), - PublicKey(PublicKey), - QueuedMessage(QueuedMessage), - ReportSettings(ReportSettings), - Role(Role), - Search(Search), - SearchStore(SearchStore), - Security(Security), - SenderAuth(SenderAuth), - Sharing(Sharing), - SieveSystemInterpreter(SieveSystemInterpreter), - SieveSystemScript(SieveSystemScript), - SieveUserInterpreter(SieveUserInterpreter), - SieveUserScript(SieveUserScript), - SpamClassifier(SpamClassifier), - SpamDnsblServer(SpamDnsblServer), - SpamDnsblSettings(SpamDnsblSettings), - SpamFileExtension(SpamFileExtension), - SpamLlm(SpamLlm), - SpamPyzor(SpamPyzor), - SpamRule(SpamRule), - SpamSettings(SpamSettings), - SpamTag(SpamTag), - SpamTrainingSample(SpamTrainingSample), - SpfReportSettings(SpfReportSettings), - StoreLookup(StoreLookup), - SystemSettings(SystemSettings), - Task(Task), - TaskManager(TaskManager), - Tenant(Tenant), - TlsExternalReport(TlsExternalReport), - TlsInternalReport(TlsInternalReport), - TlsReportSettings(TlsReportSettings), - Trace(Trace), - Tracer(Tracer), - TracingStore(TracingStore), - WebDav(WebDav), - WebHook(WebHook), + Account(Box), + AccountPassword(Box), + AccountSettings(Box), + AcmeProvider(Box), + Action(Box), + AddressBook(Box), + AllowedIp(Box), + ApiKey(Box), + AppPassword(Box), + Application(Box), + ArfExternalReport(Box), + Asn(Box), + Authentication(Box), + BlobStore(Box), + BlockedIp(Box), + Bootstrap(Box), + Cache(Box), + Calendar(Box), + CalendarAlarm(Box), + CalendarScheduling(Box), + Certificate(Box), + ClusterNode(Box), + ClusterRole(Box), + Coordinator(Box), + DataRetention(Box), + DataStore(Box), + Directory(Box), + DkimReportSettings(Box), + DkimSignature(Box), + DmarcExternalReport(Box), + DmarcInternalReport(Box), + DmarcReportSettings(Box), + DnsResolver(Box), + DnsServer(Box), + Domain(Box), + DsnReportSettings(Box), + Email(Box), + EventTracingLevel(Box), + FileStorage(Box), + Http(Box), + HttpForm(Box), + HttpLookup(Box), + Imap(Box), + InMemoryStore(Box), + IndexQueueStatus(Box), + Jmap(Box), + Log(Box), + MailingList(Box), + MemoryLookupKey(Box), + MemoryLookupKeyValue(Box), + Metadata(Box), + Metrics(Box), + MtaConnectionStrategy(Box), + MtaDeliverySchedule(Box), + MtaExtensions(Box), + MtaHook(Box), + MtaInboundSession(Box), + MtaInboundThrottle(Box), + MtaMilter(Box), + MtaOutboundStrategy(Box), + MtaOutboundThrottle(Box), + MtaQueueQuota(Box), + MtaRoute(Box), + MtaStageAuth(Box), + MtaStageConnect(Box), + MtaStageData(Box), + MtaStageEhlo(Box), + MtaStageMail(Box), + MtaStageRcpt(Box), + MtaSts(Box), + MtaTlsStrategy(Box), + MtaVirtualQueue(Box), + NetworkListener(Box), + OAuthClient(Box), + OAuthGrant(Box), + OidcProvider(Box), + PublicKey(Box), + QueuedMessage(Box), + ReportSettings(Box), + Role(Box), + Search(Box), + SearchStore(Box), + Security(Box), + SenderAuth(Box), + Sharing(Box), + SieveSystemInterpreter(Box), + SieveSystemScript(Box), + SieveUserInterpreter(Box), + SieveUserScript(Box), + SpamClassifier(Box), + SpamDnsblServer(Box), + SpamDnsblSettings(Box), + SpamFileExtension(Box), + SpamPyzor(Box), + SpamRule(Box), + SpamSettings(Box), + SpamTag(Box), + SpamTrainingSample(Box), + SpfReportSettings(Box), + StoreLookup(Box), + SystemSettings(Box), + Task(Box), + TaskManager(Box), + TlsExternalReport(Box), + TlsInternalReport(Box), + TlsReportSettings(Box), + Tracer(Box), + WebDav(Box), + WebHook(Box), } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -138,13 +130,10 @@ pub enum ObjectType { AcmeProvider = 3, Action = 4, AddressBook = 5, - AiModel = 6, - Alert = 7, AllowedIp = 8, ApiKey = 9, AppPassword = 10, Application = 11, - ArchivedItem = 12, ArfExternalReport = 13, Asn = 14, Authentication = 15, @@ -172,7 +161,6 @@ pub enum ObjectType { Domain = 37, DsnReportSettings = 38, Email = 39, - Enterprise = 40, EventTracingLevel = 41, FileStorage = 42, Http = 43, @@ -180,1013 +168,1011 @@ pub enum ObjectType { HttpLookup = 45, Imap = 46, InMemoryStore = 47, - Jmap = 48, - Log = 49, - MailingList = 50, - MaskedEmail = 51, - MemoryLookupKey = 52, - MemoryLookupKeyValue = 53, - Metric = 54, - Metrics = 55, - MetricsStore = 56, - MtaConnectionStrategy = 57, - MtaDeliverySchedule = 58, - MtaExtensions = 59, - MtaHook = 60, - MtaInboundSession = 61, - MtaInboundThrottle = 62, - MtaMilter = 63, - MtaOutboundStrategy = 64, - MtaOutboundThrottle = 65, - MtaQueueQuota = 66, - MtaRoute = 67, - MtaStageAuth = 68, - MtaStageConnect = 69, - MtaStageData = 70, - MtaStageEhlo = 71, - MtaStageMail = 72, - MtaStageRcpt = 73, - MtaSts = 74, - MtaTlsStrategy = 75, - MtaVirtualQueue = 76, - NetworkListener = 77, - OAuthClient = 78, - OidcProvider = 79, - PublicKey = 80, - QueuedMessage = 81, - ReportSettings = 82, - Role = 83, - Search = 84, - SearchStore = 85, - Security = 86, - SenderAuth = 87, - Sharing = 88, - SieveSystemInterpreter = 89, - SieveSystemScript = 90, - SieveUserInterpreter = 91, - SieveUserScript = 92, - SpamClassifier = 93, - SpamDnsblServer = 94, - SpamDnsblSettings = 95, - SpamFileExtension = 96, - SpamLlm = 97, - SpamPyzor = 98, - SpamRule = 99, - SpamSettings = 100, - SpamTag = 101, - SpamTrainingSample = 102, - SpfReportSettings = 103, - StoreLookup = 104, - SystemSettings = 105, - Task = 106, - TaskManager = 107, - Tenant = 108, - TlsExternalReport = 109, - TlsInternalReport = 110, - TlsReportSettings = 111, - Trace = 112, - Tracer = 113, - TracingStore = 114, - WebDav = 115, - WebHook = 116, + IndexQueueStatus = 48, + Jmap = 49, + Log = 50, + MailingList = 51, + MemoryLookupKey = 53, + MemoryLookupKeyValue = 54, + Metadata = 55, + Metrics = 57, + MtaConnectionStrategy = 59, + MtaDeliverySchedule = 60, + MtaExtensions = 61, + MtaHook = 62, + MtaInboundSession = 63, + MtaInboundThrottle = 64, + MtaMilter = 65, + MtaOutboundStrategy = 66, + MtaOutboundThrottle = 67, + MtaQueueQuota = 68, + MtaRoute = 69, + MtaStageAuth = 70, + MtaStageConnect = 71, + MtaStageData = 72, + MtaStageEhlo = 73, + MtaStageMail = 74, + MtaStageRcpt = 75, + MtaSts = 76, + MtaTlsStrategy = 77, + MtaVirtualQueue = 78, + NetworkListener = 79, + OAuthClient = 80, + OAuthGrant = 81, + OidcProvider = 82, + PublicKey = 83, + QueuedMessage = 84, + ReportSettings = 85, + Role = 86, + Search = 87, + SearchStore = 88, + Security = 89, + SenderAuth = 90, + Sharing = 91, + SieveSystemInterpreter = 92, + SieveSystemScript = 93, + SieveUserInterpreter = 94, + SieveUserScript = 95, + SpamClassifier = 97, + SpamDnsblServer = 98, + SpamDnsblSettings = 99, + SpamFileExtension = 100, + SpamPyzor = 101, + SpamRule = 102, + SpamSettings = 103, + SpamTag = 104, + SpamTrainingSample = 105, + SpfReportSettings = 106, + StoreLookup = 107, + SystemSettings = 108, + Task = 109, + TaskManager = 110, + TlsExternalReport = 112, + TlsInternalReport = 113, + TlsReportSettings = 114, + Tracer = 116, + WebDav = 118, + WebHook = 119, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum Property { Type = 0, - AbuseBanPeriod = 678, - AbuseBanRate = 677, - AccessKey = 118, - AccessKeyId = 327, - AccessToken = 890, - AccessTokenExpiry = 619, - AccessTokens = 132, - AccountDomainId = 810, - AccountId = 57, - AccountIdentifier = 315, + AbuseBanPeriod = 792, + AbuseBanRate = 791, + AcceptedAudiences = 710, + AccessKeyId = 354, + AccessToken = 378, + AccessTokenExpiry = 721, + AccessTokens = 159, + AccountDomainId = 951, + AccountHashPrefix = 17, + AccountId = 71, + AccountIdentifier = 374, AccountKey = 15, - AccountName = 809, - AccountSwitchKey = 891, - AccountType = 811, + AccountName = 950, + AccountSwitchKey = 379, + AccountType = 952, AccountUri = 16, - Accounts = 151, - AcmeProviderId = 182, - AddAuthResultsHeader = 554, - AddDateHeader = 555, - AddDeliveredToHeader = 556, - AddMessageIdHeader = 557, - AddReceivedHeader = 558, - AddReceivedSpfHeader = 559, - AddReturnPathHeader = 560, - AdditionalInformation = 838, - Address = 44, - Addresses = 579, - AggregateContactInfo = 268, - AggregateDkimSignDomain = 274, - AggregateFromAddress = 269, - AggregateFromName = 270, - AggregateMaxReportSize = 271, - AggregateOrgName = 272, - AggregateSendFrequency = 273, - AggregateSubject = 275, - AlarmId = 798, - Algorithms = 225, - Aliases = 339, - AllowCount = 768, - AllowDirectoryQueries = 695, - AllowExternalRcpts = 164, - AllowInvalidCerts = 26, - AllowPlainTextAuth = 424, - AllowRelaying = 348, - AllowScimProvisioning = 932, - AllowSpamTraining = 369, - AllowedEndpoints = 398, - AllowedIps = 49, - AllowedNotifyUris = 712, - Alpha = 388, - AnonymousClientRegistration = 614, - Ansi = 858, - ApiKey = 325, - ApiUser = 892, - ApplicationKey = 321, - ApplicationSecret = 322, - ArcResult = 292, - ArcVerify = 690, - ArchiveDeletedAccountsFor = 203, - ArchiveDeletedItemsFor = 202, - ArchivedAt = 58, - ArchivedItemType = 820, - ArchivedUntil = 59, - ArrivalDate = 68, - AsnUrls = 102, - AttemptNumber = 829, - Attempts = 303, - AttrClass = 470, - AttrDescription = 471, - AttrEmail = 472, - AttrEmailAlias = 473, - AttrMemberOf = 474, - AttrSecret = 475, - AttrSecretChanged = 476, - Auid = 215, - Auth = 897, - AuthBanPeriod = 680, - AuthBanRate = 679, - AuthCodeExpiry = 616, - AuthCodeMaxAttempts = 613, - AuthFailure = 81, - AuthId = 886, - AuthSecret = 501, - AuthToken = 314, - AuthUsername = 502, - AuthenticatedAs = 740, - AuthenticationResults = 69, - AutoAddInvitations = 171, - AutoUpdateFrequency = 53, - BaseDn = 463, - BaseUrl = 882, - BearerToken = 403, - Beta = 389, - Bind = 589, - BindAuthentication = 466, - BindDn = 464, - BindSecret = 465, - BlobCleanupSchedule = 200, - BlobId = 60, - BlobSize = 655, - BlobStore = 126, - BlockCount = 766, - Body = 38, - Brokers = 459, - Bucket = 658, - BufferSize = 656, - Buffered = 863, - CacheSize = 929, - Canonicalization = 216, - CapacityClient = 584, - CapacityReadBuffer = 585, - CapacitySubscription = 586, - CatchAllAddress = 346, - Categories = 759, - Certificate = 176, - CertificateManagement = 342, + Accounts = 182, + AcmeProviderId = 220, + AddAuthResultsHeader = 649, + AddDateHeader = 650, + AddDeliveredToHeader = 651, + AddMessageIdHeader = 652, + AddReceivedHeader = 653, + AddReceivedSpfHeader = 654, + AddReturnPathHeader = 655, + AdditionalInformation = 980, + Address = 58, + AggregateContactInfo = 311, + AggregateDkimSignDomain = 317, + AggregateFromAddress = 312, + AggregateFromName = 313, + AggregateOrgName = 315, + AggregateSendFrequency = 316, + AlarmId = 938, + AlertAfter = 957, + Algorithms = 265, + Aliases = 411, + AllowCount = 903, + AllowDirectoryQueries = 809, + AllowExternalRcpts = 203, + AllowIdTokenClaims = 717, + AllowInvalidCerts = 41, + AllowPlainTextAuth = 502, + AllowRelaying = 419, + AllowSpamTraining = 446, + AllowedEndpoints = 475, + AllowedIps = 63, + AllowedNotifyUris = 824, + Alpha = 464, + AnonymousTimeout = 506, + Ansi = 999, + ApiKey = 39, + ApiKeyId = 372, + ApiUser = 390, + ApplicationKey = 399, + ApplicationSecret = 400, + ArcResult = 337, + ArcVerify = 804, + ArrivalDate = 82, + AsnUrls = 116, + AttemptNumber = 971, + Attempts = 348, + AttrClass = 559, + AttrDescription = 560, + AttrEmail = 561, + AttrEmailAlias = 562, + AttrMemberOf = 563, + AttrSecret = 564, + AttrSecretChanged = 565, + Auid = 253, + Auth = 139, + AuthBanPeriod = 794, + AuthBanRate = 793, + AuthCodeExpiry = 719, + AuthFailure = 95, + AuthId = 370, + AuthSecret = 597, + AuthUsername = 598, + AuthenticatedAs = 882, + AuthenticatedTimeout = 507, + AuthenticationResults = 83, + AutoAddInvitations = 209, + AutoUpdateInterval = 66, + AvailabilityRateLimit = 201, + BaseDn = 552, + BearerToken = 482, + Beta = 465, + Bind = 681, + BindAuthentication = 555, + BindDn = 553, + BindSecret = 554, + BlobCleanupSchedule = 238, + BlobId = 74, + BlobSize = 763, + BlobStore = 153, + BlockCount = 901, + Bucket = 767, + BufferSize = 764, + Buffered = 1004, + CaaIdentities = 19, + CacheSize = 766, + Canonicalization = 254, + CatchAllAddress = 417, + Certificate = 214, + CertificateManagement = 413, ChallengeType = 10, - ChangesMaxResults = 435, - Chunking = 517, - ClaimGroups = 612, - ClaimName = 611, - ClaimUsername = 609, - Cleartext = 693, - ClientId = 604, - ClientIp = 898, - ClientSecret = 878, - ClientToken = 889, - ClusterFile = 382, - ColumnClass = 781, - ColumnDescription = 782, - ColumnEmail = 779, - ColumnSecret = 780, - Comment = 240, - CompartmentOcid = 905, - CompressionAlgorithm = 359, - Concurrency = 304, - Condition = 34, - Confidence = 760, - Config = 873, - ConfigName = 883, - ConnectTimeout = 505, - Connection = 539, - ConsumerKey = 323, + ChunkSize = 756, + Chunking = 614, + ClaimGroups = 709, + ClaimName = 708, + ClaimUsername = 706, + Cleartext = 807, + ClientCapacity = 677, + ClientId = 132, + ClientIp = 391, + ClientIpSource = 479, + ClientKind = 701, + ClientMetadataDocuments = 727, + ClientName = 700, + ClientSecret = 133, + ClientToken = 377, + ClusterFile = 458, + ColumnClass = 921, + ColumnDescription = 922, + ColumnEmail = 919, + ColumnSecret = 920, + CommandTimeout = 631, + Comment = 280, + CompartmentOcid = 398, + CompressionAlgorithm = 431, + Condition = 48, + Config = 1011, + ConfigName = 367, + ConnectTimeout = 602, + Connection = 634, + ConsumerKey = 401, Contact = 11, - ContactInfo = 844, - Contacts = 133, - Container = 117, - Content = 65, - ContentTypes = 758, - Contents = 708, - Context = 877, - Count = 258, - Create = 367, - CreatedAt = 46, - CreatedBy = 486, - CredentialId = 627, - Credentials = 588, + ContactInfo = 986, + Contacts = 160, + Container = 137, + ContentTypes = 900, + Contents = 79, + Context = 364, + Count = 301, + Create = 444, + CreatedAt = 60, + CredentialId = 730, + Credentials = 385, CurrentSecret = 4, - CustomEndpoint = 662, - CustomRegion = 663, - CustomRule = 787, - CustomerNumber = 899, - Dane = 569, - DataCleanupSchedule = 199, - DataStore = 125, - DataTimeout = 506, - Database = 575, - DatacenterId = 383, - DateRangeBegin = 245, - DateRangeEnd = 246, - DateRangeStart = 845, - Day = 192, - DeadPropertyMaxSize = 868, - DefaultAdminRoleIds = 108, - DefaultCertificateId = 790, - DefaultDisplayName = 20, - DefaultDomain = 122, - DefaultDomainId = 789, - DefaultExpiryDuplicate = 709, - DefaultExpiryVacation = 710, - DefaultFolders = 360, - DefaultFromAddress = 405, - DefaultFromName = 697, - DefaultGroupRoleIds = 106, - DefaultHostname = 788, - DefaultHrefName = 21, - DefaultLanguage = 665, - DefaultName = 408, - DefaultReturnPath = 701, - DefaultSubject = 411, - DefaultSubjectPrefix = 714, - DefaultTenantRoleIds = 107, - DefaultUserRoleIds = 105, - Definition = 235, - Delay = 825, - DeleteAfter = 229, - DeleteAfterUse = 777, - DeliverAt = 238, - DeliverBy = 518, - DeliverTo = 404, - DeliveryResult = 82, - Depth = 381, + CustomRule = 927, + CustomerNumber = 392, + Dane = 664, + DataCleanupSchedule = 237, + DataStore = 152, + DataTimeout = 603, + DataTypes = 582, + Database = 670, + DatacenterId = 459, + DateRangeEnd = 286, + DateRangeStart = 285, + Day = 230, + DefaultAdminRoleIds = 123, + DefaultCertificateId = 929, + DefaultDisplayName = 24, + DefaultDomain = 149, + DefaultDomainId = 928, + DefaultExpiryDuplicate = 821, + DefaultExpiryVacation = 822, + DefaultFolders = 432, + DefaultFromAddress = 484, + DefaultFromName = 811, + DefaultGroupRoleIds = 121, + DefaultHostname = 148, + DefaultHrefName = 25, + DefaultLanguage = 770, + DefaultName = 487, + DefaultReturnPath = 814, + DefaultSubject = 489, + DefaultSubjectPrefix = 826, + DefaultUserRoleIds = 120, + Definition = 275, + Delay = 966, + DeleteAfter = 269, + DeleteAfterUse = 918, + DeliverAt = 278, + DeliverBy = 615, + DeliverTo = 483, + DeliveryResult = 96, + Depth = 457, Description = 6, - Details = 297, + Details = 342, Directory = 12, - DirectoryId = 104, - DisableCapabilities = 711, - DisabledPermissions = 629, - DiscardAfter = 872, - Disposition = 747, - Dkim2Pass = 917, - Dkim2Result = 916, - DkimAdspDns = 83, - DkimCanonicalizedBody = 84, - DkimCanonicalizedHeader = 85, - DkimDomain = 86, - DkimIdentity = 87, - DkimManagement = 343, - DkimPass = 291, - DkimResults = 266, - DkimSelector = 88, - DkimSelectorDns = 89, - DkimSignDomain = 231, - DkimSignatures = 155, - DkimStrict = 686, - DkimVerify = 687, - DmarcPass = 294, - DmarcPolicy = 295, - DmarcResult = 293, - DmarcVerify = 691, - DnsIpv4 = 134, - DnsIpv6 = 135, - DnsManagement = 344, - DnsMtaSts = 136, - DnsMx = 137, - DnsPtr = 138, - DnsRbl = 139, - DnsServer = 130, - DnsServerId = 300, - DnsTlsa = 140, - DnsTxt = 141, - DnsView = 894, - DnsZoneFile = 345, - DocumentId = 804, - DocumentType = 814, - Domain = 232, - DomainId = 221, - DomainLimit = 750, - DomainNames = 147, - DomainNamesNegative = 148, - Domains = 146, - Dsn = 519, - Due = 797, - DuplicateExpiry = 699, - Duration = 515, + DirectoryId = 119, + DirectoryRecipients = 188, + DirectoryRecipientsTtl = 189, + DisableCapabilities = 823, + DisabledPermissions = 732, + DiscardAfter = 1010, + Disposition = 890, + Dkim2Pass = 336, + Dkim2Result = 335, + DkimAdspDns = 97, + DkimCanonicalizedBody = 98, + DkimCanonicalizedHeader = 99, + DkimDomain = 100, + DkimIdentity = 101, + DkimManagement = 414, + DkimPass = 334, + DkimResults = 309, + DkimSelector = 102, + DkimSelectorDns = 103, + DkimSignDomain = 271, + DkimSignatures = 186, + DkimStrict = 800, + DkimVerify = 801, + DmarcPass = 339, + DmarcPolicy = 340, + DmarcResult = 338, + DmarcVerify = 805, + DnsIpv4 = 161, + DnsIpv6 = 162, + DnsManagement = 415, + DnsMtaSts = 163, + DnsMx = 164, + DnsPtr = 165, + DnsServer = 157, + DnsServerId = 345, + DnsTlsa = 167, + DnsTxt = 168, + DnsView = 387, + DnsZoneFile = 416, + DocumentId = 946, + Domain = 272, + DomainId = 259, + DomainLimit = 892, + DomainNames = 178, + DomainNamesNegative = 179, + Domains = 177, + Dsn = 616, + Due = 937, + Duration = 612, EabHmacKey = 13, EabKeyId = 14, - EhloDomain = 283, - EhloHostname = 503, - EhloTimeout = 507, - Elapsed = 296, - Else = 375, - Email = 242, - EmailAddress = 393, - EmailAddresses = 149, - EmailAddressesNegative = 150, - EmailAlert = 35, - EmailDomain = 488, - EmailLimit = 751, - EmailPrefix = 487, - EmailTemplate = 174, - Enable = 37, - EnableAssistedDiscovery = 865, - EnableEdns = 305, - EnableHsts = 399, - EnableLogExporter = 860, - EnableSpamFilter = 562, - EnableSpanExporter = 861, - Enabled = 50, - EnabledPermissions = 628, - EncryptAtRest = 358, - EncryptOnAppend = 357, + EhloDomain = 325, + EhloHostname = 599, + EhloTimeout = 604, + Elapsed = 341, + Else = 451, + Email = 282, + EmailAddress = 469, + EmailAddresses = 180, + EmailAddressesNegative = 181, + EmailLimit = 893, + Enable = 51, + EnableAssistedDiscovery = 1005, + EnableEdns = 349, + EnableHsts = 476, + EnableLogExporter = 1001, + EnableSpamFilter = 657, + EnableSpanExporter = 1002, + EnabledPermissions = 731, + EncryptAtRest = 430, + EncryptOnAppend = 429, EncryptionAtRest = 9, - EncryptionKey = 622, - Endpoint = 499, - EnvFrom = 742, - EnvFromParameters = 743, - EnvId = 639, - EnvRcptTo = 744, - EnvelopeFrom = 264, - EnvelopeTo = 263, - Environment = 881, - ErrorCommand = 210, - ErrorMessage = 209, - ErrorType = 208, - Errors = 247, - EvaluatedDisposition = 259, - EvaluatedDkim = 260, - EvaluatedSpf = 261, - Event = 372, - EventAlert = 36, - EventEnd = 801, - EventEndTz = 803, - EventId = 799, - EventMessage = 43, - EventSourceThrottle = 447, - EventStart = 800, - EventStartTz = 802, - Events = 142, - EventsPolicy = 855, - Expire = 217, - Expires = 100, - ExpiresAt = 47, - ExpiresAttempts = 632, - Expiry = 512, - Expn = 520, - ExpungeSchedule = 198, - ExpungeSchedulingInboxAfter = 197, - ExpungeShareNotifyAfter = 196, - ExpungeSubmissionsAfter = 195, - ExpungeTrashAfter = 194, - Extension = 754, - Extensions = 257, - ExternalId = 933, - ExtraContactInfo = 243, - Factor = 821, - FailOnTimeout = 490, - FailedAt = 826, - FailedAttemptNumber = 827, - FailedSessionCount = 837, - FailureDetails = 851, - FailureDkimSignDomain = 279, - FailureFromAddress = 276, - FailureFromName = 277, - FailureReason = 828, - FailureReasonCode = 839, - FailureSendFrequency = 278, - FailureSubject = 280, - FeatureL2Normalize = 738, - FeatureLogScale = 739, - FeedbackType = 67, - FieldEmail = 406, - FieldHoneyPot = 407, - FieldName = 409, - FieldSubject = 412, - FilePath = 676, - Files = 144, - FilterLogin = 467, - FilterMailbox = 468, - FilterMemberOf = 469, - Fingerprint = 902, - Flags = 638, - FlagsAction = 537, - FlagsProtocol = 538, - FolderId = 909, - ForDomain = 485, - Format = 415, - From = 62, - FromAddress = 39, - FromEmail = 165, - FromName = 40, - FutureRelease = 521, - GenerateDkimKeys = 124, - Generator = 918, - GeoUrls = 103, - GetMaxResults = 436, - GreetingTimeout = 508, - GreylistFor = 770, - GroupClass = 477, - GroupId = 460, - HeaderFrom = 265, - Headers = 93, - HoldMetricsFor = 206, - HoldMtaReportsFor = 204, - HoldSamplesFor = 730, - HoldTracesFor = 205, - Host = 333, - HostedZoneId = 331, - Hostname = 185, - Hour = 190, - HttpAuth = 32, - HttpHeaders = 33, - HttpRsvpEnable = 168, - HttpRsvpLinkExpiry = 169, - HttpRsvpTemplate = 175, - HttpRsvpUrl = 170, - HttpRua = 842, - HumanResult = 234, - ICalendarData = 807, + EncryptionKey = 724, + EnvId = 740, + EnvelopeFrom = 307, + EnvelopeTo = 306, + Environment = 138, + ErrorCommand = 248, + ErrorMessage = 247, + ErrorType = 246, + Errors = 287, + EvaluatedDisposition = 302, + EvaluatedDkim = 303, + EvaluatedSpf = 304, + Event = 448, + EventEnd = 941, + EventEndTz = 943, + EventId = 939, + EventSourceThrottle = 532, + EventStart = 940, + EventStartTz = 942, + Events = 169, + EventsPolicy = 996, + Expire = 255, + Expires = 745, + ExpiresAt = 61, + ExpiresAttempts = 735, + Expiry = 609, + Expn = 617, + ExpungeSchedule = 236, + ExpungeSchedulingInboxAfter = 235, + ExpungeShareNotifyAfter = 234, + ExpungeSubmissionsAfter = 233, + ExpungeTrashAfter = 232, + Extension = 896, + Extensions = 297, + ExtraContactInfo = 283, + Factor = 962, + FailOnTimeout = 579, + FailedAt = 968, + FailedAttemptNumber = 969, + FailedSessionCount = 979, + FailureDetails = 992, + FailureDkimSignDomain = 321, + FailureFromAddress = 318, + FailureFromName = 319, + FailureReason = 970, + FailureReasonCode = 981, + FailureSendFrequency = 320, + FailureSubject = 322, + FeatureL2Normalize = 880, + FeatureLogScale = 881, + FeedbackType = 81, + FieldEmail = 485, + FieldHoneyPot = 486, + FieldName = 488, + FieldSubject = 490, + FilePath = 790, + Files = 171, + FilterLogin = 556, + FilterMemberOf = 558, + FilterRecipient = 557, + Fingerprint = 395, + FirstFailedAt = 972, + FirstParty = 697, + Flags = 263, + FlagsAction = 632, + FlagsProtocol = 633, + FlushChanges = 144, + FlushInterval = 145, + FolderId = 410, + Format = 493, + ForwardedHops = 480, + From = 76, + FromAddress = 53, + FromName = 54, + FutureRelease = 618, + GenerateDkimKeys = 151, + Generation = 703, + Generator = 298, + GeoUrls = 117, + GrantTypes = 699, + GreetingTimeout = 605, + GreylistFor = 905, + GroupClass = 566, + GroupId = 548, + GroupwareLarge = 175, + GroupwareLargeTtl = 176, + HeaderFrom = 308, + Headers = 107, + HoldMtaReportsFor = 242, + HoldSamplesFor = 872, + Host = 376, + HostedZoneId = 404, + Hostname = 223, + Hosts = 547, + Hour = 228, + HttpAuth = 118, + HttpHeaders = 40, + HttpRsvpEnable = 206, + HttpRsvpLinkExpiry = 207, + HttpRsvpUrl = 208, + HttpRua = 984, + HumanResult = 274, + ICalendarData = 948, Id = 1, - IdTokenExpiry = 621, - IdentityAlignment = 91, - If = 376, - ImpersonateServiceAccount = 320, - ImplicitTls = 546, - InMemoryStore = 128, - InboundReportAddresses = 651, - InboundReportForwarding = 652, - InboundReportMaxSize = 928, - Incidents = 70, - IncludeSource = 352, - IndexAsn = 94, - IndexAsnName = 95, - IndexBatchSize = 664, - IndexCalendar = 667, - IndexCalendarFields = 668, - IndexContactFields = 670, - IndexContacts = 669, - IndexCountry = 96, - IndexEmail = 671, - IndexEmailFields = 672, - IndexKey = 421, - IndexTelemetry = 673, - IndexTracingFields = 674, - IndexValue = 422, - IndicatorParameters = 736, - InitialDelay = 822, - Interval = 500, - Intervals = 516, - IpLimit = 752, - IpLookupStrategy = 543, - IpRevPtr = 290, - IpRevResult = 289, - IsActive = 707, - IsArchive = 755, - IsBad = 756, - IsEnabled = 340, - IsFromOrganizer = 806, - IsGlobPattern = 491, - IsGzipped = 416, - IsNz = 757, - IsSenderAllowed = 564, - IsSpam = 776, - IsTls = 741, - Issuer = 181, - IssuerUrl = 606, - ItipMaxSize = 172, - Jitter = 824, - Key = 334, - KeyName = 337, - KeyPrefix = 120, - KeyValues = 853, - L1Ratio = 391, - L2Ratio = 392, - LastRenewal = 186, - LearnHamFromCard = 727, - LearnHamFromReply = 735, - LearnSpamFromRblHits = 728, - LearnSpamFromTraps = 729, - Level = 373, - LicenseKey = 370, - Line = 875, - ListenerIds = 183, - Listeners = 188, - LivePropertyMaxSize = 869, + IdTokenExpiry = 723, + IdentityAlignment = 105, + IdleTimeout = 508, + If = 452, + ImapServerAdmin = 589, + ImapServerComment = 588, + ImpersonateServiceAccount = 384, + ImplicitTls = 641, + InMemoryStore = 155, + InboundReportAddresses = 758, + InboundReportForwarding = 759, + Incidents = 84, + IncludeSecrets = 935, + IncludeSource = 424, + Index = 513, + IndexAsn = 108, + IndexAsnName = 109, + IndexBatchSize = 769, + IndexCalendar = 772, + IndexCalendarFields = 773, + IndexContactFields = 775, + IndexContacts = 774, + IndexCountry = 110, + IndexEmail = 776, + IndexEmailFields = 777, + IndexFiles = 780, + IndexKey = 498, + IndexValue = 499, + IndicatorParameters = 878, + InitialDelay = 963, + InstanceHashes = 264, + Interval = 596, + Intervals = 613, + IntrospectionAuthMethod = 714, + IntrospectionClientId = 712, + IntrospectionClientSecret = 713, + IpLimit = 894, + IpLookupStrategy = 638, + IpRevPtr = 333, + IpRevResult = 332, + IsActive = 820, + IsArchive = 897, + IsBad = 898, + IsFromOrganizer = 947, + IsGlobPattern = 580, + IsGzipped = 494, + IsNz = 899, + IsSenderAllowed = 659, + IsSpam = 917, + IsTls = 883, + Issuer = 219, + IssuerDomainNames = 18, + IssuerUrl = 705, + Jitter = 965, + Key = 375, + KeyName = 406, + KeyPrefix = 141, + L1Ratio = 467, + L2Ratio = 468, + LastRenewal = 224, + LastUsedAt = 702, + LearnHamFromCard = 869, + LearnHamFromReply = 877, + LearnSpamFromRblHits = 870, + LearnSpamFromTraps = 871, + LenientIssuerMatch = 718, + Level = 449, + Line = 358, + ListenerIds = 221, + Listeners = 226, + LocalIp = 600, + LocalIps = 601, Locale = 7, - Logo = 341, - LogoUrl = 371, - LoiterBanPeriod = 682, - LoiterBanRate = 681, - Lossy = 854, - MachineId = 384, - MailExchangers = 793, - MailFrom = 284, - MailFromTimeout = 509, - MailRua = 841, - MailingLists = 154, - MaintenanceType = 796, - ManagedZone = 318, - Match = 374, - MaxAddressBooks = 23, - MaxAge = 566, - MaxAllowedPacket = 576, - MaxApiKeys = 115, - MaxAppPasswords = 114, - MaxAttachmentSize = 353, - MaxAttempts = 511, - MaxAttendees = 157, - MaxAuthFailures = 425, - MaxCalendars = 160, - MaxChangesHistory = 201, - MaxConcurrent = 426, - MaxConcurrentRequests = 439, - MaxConcurrentUploads = 442, - MaxConnections = 603, - MaxContacts = 24, - MaxCpuCycles = 702, - MaxDelay = 823, - MaxDuration = 530, - MaxEntries = 417, - MaxEntrySize = 418, - MaxEventNotifications = 163, - MaxEvents = 161, - MaxFailures = 547, - MaxFiles = 378, - MaxFolders = 379, - MaxHeaderSize = 715, - MaxICalendarSize = 159, - MaxIdentities = 363, - MaxIncludes = 716, - MaxLocalVars = 717, - MaxLockTimeout = 866, - MaxLocks = 867, - MaxMailboxDepth = 355, - MaxMailboxNameLength = 356, - MaxMailboxes = 364, - MaxMaskedAddresses = 365, - MaxMatchVars = 718, - MaxMessageSize = 354, - MaxMessages = 361, - MaxMessagesPerCommand = 924, - MaxMessagesPerSave = 927, - MaxMethodCalls = 438, - MaxMultihomed = 544, - MaxMxHosts = 545, - MaxNestedBlocks = 720, - MaxNestedForEvery = 721, - MaxNestedIncludes = 703, - MaxNestedTests = 722, - MaxOutMessages = 704, - MaxParticipantIdentities = 162, - MaxPublicKeys = 366, - MaxPushSize = 923, - MaxReceivedHeaders = 561, - MaxRecipients = 173, - MaxReconnects = 580, - MaxRecurrenceExpansions = 158, - MaxRedirects = 705, - MaxReportSize = 852, - MaxRequestRate = 427, - MaxRequestSize = 428, - MaxResponseSize = 527, - MaxResults = 871, - MaxRetries = 18, - MaxRetryWait = 648, - MaxScriptNameLength = 719, - MaxScriptSize = 723, - MaxScripts = 726, - MaxShares = 696, - MaxSize = 101, - MaxStringLength = 724, - MaxSubmissions = 362, - MaxSubscriptions = 458, - MaxUidBatches = 926, - MaxUploadCount = 444, - MaxUploadSize = 443, - MaxVCardSize = 22, - MaxVarNameLength = 725, - MaxVarSize = 706, - MemberGroupIds = 864, - MemberTenantId = 19, - Message = 92, - MessageIdHostname = 698, - MessageIds = 819, - Messages = 145, - Metric = 493, - Metrics = 497, - MetricsCollectionInterval = 207, - MetricsPolicy = 498, - MinHamSamples = 731, - MinRetryWait = 649, - MinSpamSamples = 732, - MinTriggerInterval = 166, - MinUidBatchSize = 925, - Minute = 191, - Mode = 567, - Model = 28, - ModelId = 764, - ModelType = 30, - MtPriority = 522, - MtaSts = 570, - MtaStsTimeout = 572, - Multiline = 859, - MustMatchSender = 550, - MxHosts = 568, - Name = 25, - Namespace = 414, - NegativeTtl = 156, - NextNotify = 634, - NextRetry = 633, - NextTransitionAt = 223, - NoCapabilityCheck = 700, - NoEcho = 587, - NoSoliciting = 523, - NodeId = 184, - NotValidAfter = 179, - NotValidBefore = 180, - Notify = 513, - NotifyCount = 642, - NotifyDue = 643, - NumFeatures = 390, - NumReplicas = 350, - NumShards = 351, - OauthClientId = 930, - OnSuccessRenewCertificate = 813, - OpenTelemetry = 495, - Options = 630, - Orcpt = 645, - OrgName = 241, - OrganizationName = 843, - Origin = 301, - OriginalEnvelopeId = 71, - OriginalMailFrom = 72, - OriginalRcptTo = 73, + Logo = 412, + LoiterBanPeriod = 796, + LoiterBanRate = 795, + Lossy = 995, + MachineId = 460, + MailExchangers = 932, + MailFrom = 326, + MailFromParameters = 884, + MailFromTimeout = 606, + MailRua = 983, + MailingLists = 185, + MaintenanceType = 936, + ManagedZone = 382, + Match = 450, + MaxAccountSize = 146, + MaxAddressBooks = 27, + MaxAddressBooksPerCard = 30, + MaxAge = 661, + MaxAggregateReportSize = 314, + MaxAllowedPacket = 671, + MaxApiKeys = 130, + MaxAppPasswords = 129, + MaxAttachmentSize = 425, + MaxAttachmentText = 912, + MaxAttachmentsSize = 202, + MaxAttempts = 608, + MaxAttendees = 191, + MaxAuthFailures = 503, + MaxAvailabilityDuration = 200, + MaxBodyText = 911, + MaxBufferedMemory = 930, + MaxCalendars = 194, + MaxCalendarsPerEvent = 198, + MaxChangesHistory = 239, + MaxChangesResults = 521, + MaxConcurrentConnections = 504, + MaxConcurrentIndexTasks = 779, + MaxConcurrentRequests = 43, + MaxConcurrentUploads = 527, + MaxConnections = 694, + MaxContacts = 28, + MaxCpuCycles = 815, + MaxDelay = 964, + MaxDepth = 585, + MaxDuration = 627, + MaxEntries = 495, + MaxEntrySize = 496, + MaxEventNotifications = 197, + MaxEvents = 195, + MaxExpandedQueryDuration = 199, + MaxExtractArchiveEntries = 786, + MaxExtractDecompressedSize = 783, + MaxExtractDocumentSize = 781, + MaxExtractPartSize = 784, + MaxExtractParts = 785, + MaxExtractPdfObjects = 787, + MaxExtractRtfDepth = 788, + MaxExtractTextSize = 782, + MaxFailures = 642, + MaxFiles = 454, + MaxFlagLength = 441, + MaxFlagsPerEmail = 440, + MaxFolders = 455, + MaxGetResults = 522, + MaxHeaderCount = 443, + MaxHeaderSize = 442, + MaxICalendarSize = 193, + MaxIdentities = 435, + MaxInboundReportSize = 762, + MaxIncludes = 827, + MaxItipSize = 210, + MaxLivePropertySize = 1008, + MaxLocalVars = 828, + MaxLockTimeout = 1006, + MaxLocks = 1007, + MaxMailboxDepth = 427, + MaxMailboxNameLength = 428, + MaxMailboxes = 436, + MaxMailboxesPerEmail = 439, + MaxMatchVars = 829, + MaxMediaSize = 31, + MaxMessageSize = 426, + MaxMessageText = 913, + MaxMessages = 433, + MaxMessagesPerCommand = 509, + MaxMessagesPerSave = 512, + MaxMethodCalls = 524, + MaxMultihomed = 639, + MaxMxHosts = 640, + MaxNestedBlocks = 831, + MaxNestedForEvery = 832, + MaxNestedIncludes = 816, + MaxNestedTests = 833, + MaxOutMessages = 817, + MaxParticipantIdentities = 196, + MaxPasswordLength = 126, + MaxPolls = 578, + MaxPrivateSize = 586, + MaxPublicKeys = 438, + MaxPushAttempts = 534, + MaxPushSize = 546, + MaxQueryResults = 523, + MaxQueryScan = 587, + MaxReceivedHeaders = 656, + MaxRecipients = 211, + MaxReconnects = 674, + MaxRecurrenceExpansions = 192, + MaxRedirectUrls = 914, + MaxRedirects = 818, + MaxReportSize = 993, + MaxRequestSize = 505, + MaxResponseSize = 624, + MaxResults = 1009, + MaxRetries = 140, + MaxRetryWait = 750, + MaxScriptNameLength = 830, + MaxScriptSize = 834, + MaxScripts = 837, + MaxSetObjects = 525, + MaxShares = 810, + MaxSize = 115, + MaxSnippetResults = 526, + MaxStringLength = 835, + MaxSubmissions = 434, + MaxSubscriptions = 543, + MaxUidBatches = 511, + MaxUploadCount = 529, + MaxUploadSize = 528, + MaxVCardSize = 26, + MaxVarNameLength = 836, + MaxVarSize = 819, + MemberGroupIds = 953, + Message = 106, + MessageIdHostname = 812, + MessageIds = 959, + MessageTimeout = 549, + Messages = 172, + MessagesLarge = 173, + MessagesLargeTtl = 174, + Metrics = 594, + MetricsPolicy = 595, + MinAccountSize = 147, + MinHamSamples = 873, + MinPasswordLength = 125, + MinPasswordStrength = 127, + MinRetryWait = 751, + MinSpamSamples = 874, + MinTriggerInterval = 204, + MinUidBatchSize = 510, + Minute = 229, + Mode = 662, + Model = 37, + MtPriority = 619, + MtaSts = 665, + MtaStsTimeout = 667, + Multiline = 1000, + MustMatchSender = 645, + MxHosts = 663, + Name = 36, + Namespace = 492, + NegativeTtl = 187, + NextNotify = 736, + NextRetry = 517, + NextTransitionAt = 261, + NoCapabilityCheck = 813, + NoEcho = 680, + NoSoliciting = 620, + NodeId = 222, + NotValidAfter = 217, + NotValidBefore = 218, + Notify = 610, + NotifyCount = 743, + NotifyDue = 744, + NumFeatures = 466, + NumReplicas = 422, + NumShards = 423, + OauthClientId = 68, + ObjectType = 967, + OnSuccessRenewCertificate = 955, + OpaqueTokenValidation = 711, + OpenTelemetry = 592, + Options = 733, + Orcpt = 747, + OrgName = 281, + OrganizationName = 985, + Origin = 346, + OriginalEnvelopeId = 85, + OriginalMailFrom = 86, + OriginalRcptTo = 87, OtpAuth = 5, - OtpCode = 625, - OtpUrl = 626, - OutboundReportDomain = 653, - OutboundReportSubmitter = 654, - OverrideProxyTrustedNetworks = 590, - OverrideType = 239, - OvhEndpoint = 324, - Parameters = 737, - ParseLimitContact = 433, - ParseLimitEmail = 434, - ParseLimitEvent = 432, - Password = 876, - PasswordDefaultExpiry = 113, - PasswordHashAlgorithm = 109, - PasswordMaxLength = 111, - PasswordMinLength = 110, - PasswordMinStrength = 112, - Path = 380, - Period = 646, - Permissions = 48, - PingInterval = 583, - Pipelining = 524, - Policies = 846, - PolicyAdkim = 250, - PolicyAspf = 251, - PolicyDiscoveryMethod = 920, - PolicyDisposition = 252, - PolicyDomain = 248, - PolicyFailureReportingOptions = 255, - PolicyIdentifier = 237, - PolicyIdentifiers = 840, - PolicyNp = 919, - PolicyOverrideReasons = 262, - PolicyStrings = 848, - PolicySubdomainDisposition = 253, - PolicyTestingMode = 254, - PolicyType = 847, - PolicyVersion = 249, - PollInterval = 489, - PollingInterval = 311, - PoolMaxConnections = 478, - PoolMinConnections = 577, - PoolRecyclingMethod = 631, - PoolTimeoutCreate = 479, - PoolTimeoutRecycle = 480, - PoolTimeoutWait = 481, - PoolWorkers = 657, - Port = 299, - PreferredChain = 910, - Prefix = 856, - PreserveIntermediates = 306, - Priority = 483, - PrivateKey = 177, - PrivateKeyPassword = 904, - PrivateKeyPem = 903, - PrivateZone = 319, - PrivateZoneOnly = 332, - Profile = 661, - ProjectId = 317, - Prometheus = 496, - Prompt = 765, - PropagationDelay = 313, - PropagationTimeout = 312, - ProtectedHeaders = 713, - Protocol = 298, - ProtocolVersion = 533, - ProviderInfo = 795, - ProxyTrustedNetworks = 792, - PublicKey = 218, - PublishRecords = 302, - PushAttemptWait = 448, - PushMaxAttempts = 449, - PushRequestTimeout = 452, - PushRetryWait = 450, - PushShardsTotal = 454, - PushThrottle = 451, - PushVerifyTimeout = 453, - QueryEmailAliases = 786, - QueryLogin = 783, - QueryMaxResults = 437, - QueryMemberOf = 785, - QueryRecipient = 784, - QueueId = 514, - QueueName = 644, - Quotas = 394, - Rate = 532, - RateLimit = 410, - RateLimitAnonymous = 397, - RateLimitAuthenticated = 396, - Ratio = 767, - RcptToTimeout = 510, - ReadFromReplicas = 650, - ReadReplicas = 578, - Reason = 45, - ReceivedAt = 63, - ReceivedFromIp = 636, - ReceivedViaPort = 637, - ReceivingIp = 836, - ReceivingMxHelo = 835, - ReceivingMxHostname = 834, - Recipients = 484, - Records = 256, - RecurrenceId = 805, - RedirectRoot = 911, - RedirectUris = 605, - Refresh = 419, - RefreshTokenExpiry = 617, - RefreshTokenRenewal = 618, - Region = 330, - RejectNonFqdn = 563, - RemoteIp = 282, - RenewBefore = 17, - Report = 66, - ReportAddressUri = 349, - ReportId = 244, - ReportedDomains = 74, - ReportedUris = 75, - ReportingMta = 76, - RequestMaxSize = 870, - RequestTlsCertificate = 123, - Require = 551, - RequireAudience = 607, - RequireClientRegistration = 615, - RequireScopes = 608, - RequireTls = 525, - ReservoirCapacity = 733, - ResourceGroup = 880, - ResourceUrl = 51, - ResponseCode = 212, - ResponseEnhanced = 213, - ResponseHeaders = 401, - ResponseHostname = 211, - ResponseMessage = 214, - ResponsePosCategory = 761, - ResponsePosConfidence = 762, - ResponsePosExplanation = 763, - Result = 233, - ResultType = 832, - RetireAfter = 228, - Retry = 420, - RetryCount = 640, - RetryDue = 641, - ReturnPath = 635, - ReuseKey = 912, - ReverseIpVerify = 692, - Rewrite = 565, - RoleIds = 193, - Roles = 152, - Rotate = 857, - RotateAfter = 227, - Route = 540, - Rua = 236, - Sandbox = 896, - SasToken = 119, - SaslMechanisms = 549, - ScanBanPaths = 683, - ScanBanPeriod = 685, - ScanBanRate = 684, - Schedule = 541, - Scheduling = 143, - Scheme = 908, - Scope = 281, - Score = 745, - ScoreDiscard = 771, - ScoreReject = 772, - ScoreSpam = 773, - Script = 553, - SearchStore = 127, + OtpCode = 728, + OtpUrl = 729, + OutboundReportDomain = 760, + OutboundReportSubmitter = 761, + OverrideProxyTrustedNetworks = 682, + OverrideType = 279, + OvhEndpoint = 402, + Parameters = 879, + ParseLimitContact = 519, + ParseLimitEmail = 520, + ParseLimitEvent = 518, + Partition = 514, + Password = 363, + PasswordDefaultExpiry = 128, + PasswordHashAlgorithm = 124, + Path = 456, + Pending = 704, + Period = 748, + Permissions = 62, + PingInterval = 676, + Pipelining = 621, + Policies = 987, + PolicyAdkim = 290, + PolicyAspf = 291, + PolicyDiscoveryMethod = 300, + PolicyDisposition = 292, + PolicyDomain = 288, + PolicyFailureReportingOptions = 295, + PolicyIdentifier = 277, + PolicyIdentifiers = 982, + PolicyNp = 299, + PolicyOverrideReasons = 305, + PolicyStrings = 989, + PolicySubdomainDisposition = 293, + PolicyTestingMode = 294, + PolicyType = 988, + PolicyVersion = 289, + PollInterval = 360, + PoolMaxConnections = 567, + PoolMinConnections = 672, + PoolRecyclingMethod = 734, + PoolTimeoutCreate = 568, + PoolTimeoutRecycle = 569, + PoolTimeoutWait = 570, + PoolWorkers = 765, + Port = 344, + PostLogoutRedirectUris = 696, + PreferredChain = 22, + Prefix = 997, + PreserveIntermediates = 350, + Priority = 572, + PrivateKey = 215, + PrivateKeyPassword = 397, + PrivateKeyPem = 396, + PrivateMetadata = 584, + PrivateZone = 383, + PrivateZoneOnly = 405, + Profile = 47, + ProjectId = 381, + Prometheus = 593, + PropagationDelay = 362, + PropagationTimeout = 361, + ProtectedHeaders = 825, + Protocol = 343, + ProtocolVersion = 630, + ProviderInfo = 934, + ProxyTrustedNetworks = 931, + PublicKey = 256, + PublishRecords = 347, + PushAttemptWait = 533, + PushRequestTimeout = 537, + PushRetryWait = 535, + PushShardsTotal = 539, + PushThrottle = 536, + PushVerifyTimeout = 538, + QueryEmailAliases = 926, + QueryLogin = 923, + QueryMemberOf = 925, + QueryRecipient = 924, + QueueId = 611, + QueueName = 746, + QueuedDeletions = 516, + QueuedUpdates = 515, + Quotas = 470, + Rate = 629, + RateLimit = 44, + RateLimitAnonymous = 474, + RateLimitAuthenticated = 473, + Ratio = 902, + RcptTo = 327, + RcptToTimeout = 607, + ReadBufferCapacity = 678, + ReadFromReplicas = 752, + Reason = 59, + ReceivedAt = 77, + ReceivedFromIp = 738, + ReceivedViaPort = 739, + ReceivingIp = 978, + ReceivingMxHelo = 977, + ReceivingMxHostname = 976, + Recipients = 573, + Records = 296, + RecurrenceId = 945, + RedirectCacheTtl = 916, + RedirectRoot = 481, + RedirectTimeout = 915, + RedirectUris = 695, + RefreshInterval = 114, + RefreshTokenExpiry = 720, + Region = 356, + RejectNonFqdn = 658, + RemoteIp = 324, + RenewBefore = 20, + Report = 80, + ReportAddressUri = 420, + ReportId = 284, + ReportedDomains = 88, + ReportedUris = 89, + ReportingMta = 90, + RequestTimeout = 675, + RequestTlsCertificate = 150, + Require = 646, + RequireTls = 622, + ReservoirCapacity = 875, + ResourceGroup = 366, + ResourceUrl = 64, + ResponseCode = 250, + ResponseEnhanced = 251, + ResponseHeaders = 478, + ResponseHostname = 249, + ResponseMessage = 252, + Result = 273, + ResultType = 974, + Retention = 143, + RetireAfter = 268, + Retry = 497, + RetryCount = 741, + RetryDue = 742, + ReturnPath = 737, + ReuseKey = 23, + ReverseIpVerify = 806, + Rewrite = 660, + RoleIds = 231, + Roles = 183, + Rotate = 998, + RotateAfter = 267, + Route = 635, + Rua = 276, + Sandbox = 389, + SasToken = 134, + SaslMechanisms = 644, + ScanBanPaths = 797, + ScanBanPeriod = 799, + ScanBanRate = 798, + Schedule = 636, + Scheduling = 170, + Scheme = 409, + Scope = 323, + Scopes = 698, + Score = 885, + ScoreDiscard = 906, + ScoreReject = 907, + ScoreSpam = 908, + Script = 648, + SearchStore = 154, Secret = 3, - SecretAccessKey = 328, - SecretApiKey = 326, - SecretId = 906, - SecretKey = 659, - SecurityToken = 660, - Selector = 222, - SelectorTemplate = 226, - SendFrequency = 230, - SendingMtaIp = 833, - SentinelSecret = 915, - SentinelUsername = 914, - Separator = 97, - ServerHostname = 121, - ServerId = 934, - Servers = 308, - ServiceAccountJson = 316, - ServiceName = 913, - Services = 794, - SessionToken = 329, - SetMaxObjects = 440, - ShardIndex = 830, - SharedSecret = 895, - Sig0Algorithm = 336, - SignatureAlgorithm = 623, - SignatureKey = 624, - SignerName = 335, - Size = 64, - SkipDeploy = 885, - SkipFirst = 423, - SmtpGreeting = 552, - SnippetMaxResults = 441, - SocketBacklog = 591, - SocketNoDelay = 592, - SocketReceiveBufferSize = 593, - SocketReuseAddress = 594, - SocketReusePort = 595, - SocketSendBufferSize = 596, - SocketTosV4 = 597, - SocketTtl = 598, - SourceIp = 77, - SourceIps = 504, - SourcePort = 78, - SpamFilterRulesUrl = 775, - SpfDns = 90, - SpfEhloDomain = 285, - SpfEhloResult = 286, - SpfEhloVerify = 688, - SpfFromVerify = 689, - SpfMailFromDomain = 287, - SpfMailFromResult = 288, - SpfResults = 267, - Stage = 224, - Stages = 529, - StartTime = 56, - StartTls = 571, - Status = 61, - StorageAccount = 116, - Store = 778, - Stores = 694, - Strategy = 816, - SubAddressing = 347, - SubAuthId = 887, - Subject = 41, - SubjectAlternativeNames = 178, - Subscribe = 368, - SubscriptionId = 879, - Sum = 494, - Summary = 808, - SupportedLanguages = 666, - Tag = 748, - Tags = 746, - TaskTypes = 189, - Tasks = 187, - TcpOnError = 307, - TeamId = 907, - TempFailOnError = 528, - Temperature = 27, - Template = 167, - TenancyOcid = 900, - TenantId = 831, - Tenants = 153, + SecretKey = 355, + Selector = 260, + SelectorTemplate = 266, + SendFrequency = 270, + SendingMtaIp = 975, + SentinelSecret = 755, + SentinelUsername = 754, + Separator = 111, + ServerId = 403, + Servers = 352, + ServiceAccountJson = 380, + ServiceName = 753, + Services = 933, + SessionTimeout = 550, + SessionToken = 357, + ShardIndex = 973, + SharedSecret = 388, + SignatureAlgorithm = 725, + SignatureKey = 726, + Size = 78, + SkipAudienceValidation = 716, + SkipDeploy = 369, + SkipFirst = 500, + SmtpGreeting = 647, + SocketBacklog = 683, + SocketNoDelay = 684, + SocketReceiveBufferSize = 685, + SocketReuseAddress = 686, + SocketReusePort = 687, + SocketSendBufferSize = 688, + SocketTosV4 = 689, + SocketTtl = 690, + SourceIp = 91, + SourcePort = 92, + SpamFilterRulesUrl = 910, + SpamLookup = 166, + SpfDns = 104, + SpfEhloDomain = 328, + SpfEhloResult = 329, + SpfEhloVerify = 802, + SpfFromVerify = 803, + SpfMailFromDomain = 330, + SpfMailFromResult = 331, + SpfResults = 310, + Stage = 262, + Stages = 626, + StartTls = 666, + Status = 75, + StorageAccount = 136, + Store = 142, + Strategy = 956, + SubAddressing = 418, + SubAuthId = 371, + Subject = 55, + SubjectAlternativeNames = 216, + Subscribe = 445, + SubscriptionCapacity = 679, + SubscriptionId = 365, + Summary = 949, + SupportedLanguages = 771, + Swap = 190, + Tag = 891, + Tags = 886, + TargetAccountId = 944, + TaskTypes = 227, + Tasks = 225, + TcpOnError = 351, + TeamId = 408, + TempFailOnError = 625, + TenancyOcid = 393, + TenantId = 131, Text = 2, - Then = 377, - ThirdParty = 219, - ThirdPartyHash = 220, - ThreadName = 818, - ThreadPoolSize = 791, - ThreadsPerNode = 574, - Throttle = 862, + Then = 453, + ThirdParty = 257, + ThirdPartyHash = 258, + ThreadName = 958, + ThreadsPerNode = 669, + Throttle = 1003, TimeZone = 8, - Timeout = 29, - TimeoutAnonymous = 429, - TimeoutAuthenticated = 430, - TimeoutCommand = 534, - TimeoutConnect = 535, - TimeoutConnection = 581, - TimeoutData = 536, - TimeoutIdle = 431, - TimeoutMessage = 461, - TimeoutRequest = 582, - TimeoutSession = 462, - Timestamp = 482, - Title = 55, - Tls = 542, - TlsDisableCipherSuites = 599, - TlsDisableProtocols = 600, - TlsIgnoreClientOrder = 601, - TlsImplicit = 602, - TlsTimeout = 573, - To = 42, - Token = 888, - TotalDeadline = 817, - TotalFailedSessions = 850, - TotalSuccessfulSessions = 849, - TraceId = 815, - Tracer = 129, - TrainFrequency = 734, - TransactionRetryDelay = 385, - TransactionRetryLimit = 386, - TransactionTimeout = 387, - TransferLimit = 531, - TrustContacts = 769, - TrustReplies = 774, - TsigAlgorithm = 338, - Ttl = 310, - UnpackDirectory = 54, - UpdateRecords = 812, - UploadQuota = 445, - UploadTtl = 446, - Url = 31, - UrlLimit = 753, - UrlPrefix = 52, - Urls = 647, - UsePermissiveCors = 400, - UseTls = 309, - UseXForwarded = 402, - UsedDiskQuota = 395, - UserAgent = 79, - UserCodeExpiry = 620, - UserOcid = 901, - Username = 131, - UsernameDomain = 610, - VCardVersion = 931, - ValidateDomain = 413, - Value = 492, - VariableName = 675, - VerifyAfterWrite = 874, - Version = 80, - ViewName = 884, - Vrfy = 526, - WaitOnFail = 548, - WapiVersion = 893, - WebPushContact = 922, - WebPushKey = 921, - WebsocketHeartbeat = 455, - WebsocketThrottle = 456, - WebsocketTimeout = 457, - Zone = 749, - ZoneIpV4 = 98, - ZoneIpV6 = 99, + Timeout = 42, + Timestamp = 571, + Tls = 637, + TlsDisableCipherSuites = 691, + TlsDisableProtocols = 692, + TlsIgnoreClientOrder = 693, + TlsTimeout = 668, + To = 56, + Token = 373, + TokenFile = 135, + TotalFailedSessions = 991, + TotalSuccessfulSessions = 990, + Tracer = 156, + TrainInterval = 876, + TransactionRetryDelay = 461, + TransactionRetryLimit = 462, + TransactionTimeout = 463, + TransferLimit = 628, + TrustContacts = 904, + TrustReplies = 909, + TsigAlgorithm = 407, + Ttl = 359, + UnpackDirectory = 67, + UnscopedTokens = 715, + UpdateRecords = 954, + UploadQuota = 530, + UploadTtl = 531, + Url = 38, + UrlLimit = 895, + UrlPrefix = 65, + Urls = 749, + UsePermissiveCors = 477, + UseTls = 353, + UsedDiskQuota = 471, + UserAgent = 93, + UserCodeExpiry = 722, + UserOcid = 394, + Username = 158, + UsernameDomain = 707, + VCardVersion = 29, + ValidateDomain = 491, + Value = 581, + VariableName = 789, + VendorNamespaces = 583, + VerifyAfterWrite = 768, + Version = 94, + ViewName = 368, + Vrfy = 623, + WaitOnFail = 643, + WapiVersion = 386, + Warnings = 757, + WebPushContact = 545, + WebPushKey = 544, + WebsocketHeartbeat = 540, + WebsocketThrottle = 541, + WebsocketTimeout = 542, + WindowEnd = 960, + Zone = 501, + ZoneIpV4 = 112, + ZoneIpV6 = 113, } diff --git a/crates/registry/src/schema/properties_impl.rs b/crates/registry/src/schema/properties_impl.rs index 461418df..db7f96b7 100644 --- a/crates/registry/src/schema/properties_impl.rs +++ b/crates/registry/src/schema/properties_impl.rs @@ -9,22 +9,21 @@ use crate::schema::prelude::*; impl EnumImpl for ObjectType { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + ObjectType, b"Account" => ObjectType::Account, b"AccountPassword" => ObjectType::AccountPassword, b"AccountSettings" => ObjectType::AccountSettings, b"AcmeProvider" => ObjectType::AcmeProvider, b"Action" => ObjectType::Action, b"AddressBook" => ObjectType::AddressBook, - b"AiModel" => ObjectType::AiModel, - b"Alert" => ObjectType::Alert, b"AllowedIp" => ObjectType::AllowedIp, b"ApiKey" => ObjectType::ApiKey, b"AppPassword" => ObjectType::AppPassword, b"Application" => ObjectType::Application, - b"ArchivedItem" => ObjectType::ArchivedItem, b"ArfExternalReport" => ObjectType::ArfExternalReport, b"Asn" => ObjectType::Asn, b"Authentication" => ObjectType::Authentication, @@ -52,7 +51,6 @@ impl EnumImpl for ObjectType { b"Domain" => ObjectType::Domain, b"DsnReportSettings" => ObjectType::DsnReportSettings, b"Email" => ObjectType::Email, - b"Enterprise" => ObjectType::Enterprise, b"EventTracingLevel" => ObjectType::EventTracingLevel, b"FileStorage" => ObjectType::FileStorage, b"Http" => ObjectType::Http, @@ -60,15 +58,14 @@ impl EnumImpl for ObjectType { b"HttpLookup" => ObjectType::HttpLookup, b"Imap" => ObjectType::Imap, b"InMemoryStore" => ObjectType::InMemoryStore, + b"IndexQueueStatus" => ObjectType::IndexQueueStatus, b"Jmap" => ObjectType::Jmap, b"Log" => ObjectType::Log, b"MailingList" => ObjectType::MailingList, - b"MaskedEmail" => ObjectType::MaskedEmail, b"MemoryLookupKey" => ObjectType::MemoryLookupKey, b"MemoryLookupKeyValue" => ObjectType::MemoryLookupKeyValue, - b"Metric" => ObjectType::Metric, + b"Metadata" => ObjectType::Metadata, b"Metrics" => ObjectType::Metrics, - b"MetricsStore" => ObjectType::MetricsStore, b"MtaConnectionStrategy" => ObjectType::MtaConnectionStrategy, b"MtaDeliverySchedule" => ObjectType::MtaDeliverySchedule, b"MtaExtensions" => ObjectType::MtaExtensions, @@ -91,6 +88,7 @@ impl EnumImpl for ObjectType { b"MtaVirtualQueue" => ObjectType::MtaVirtualQueue, b"NetworkListener" => ObjectType::NetworkListener, b"OAuthClient" => ObjectType::OAuthClient, + b"OAuthGrant" => ObjectType::OAuthGrant, b"OidcProvider" => ObjectType::OidcProvider, b"PublicKey" => ObjectType::PublicKey, b"QueuedMessage" => ObjectType::QueuedMessage, @@ -109,7 +107,6 @@ impl EnumImpl for ObjectType { b"SpamDnsblServer" => ObjectType::SpamDnsblServer, b"SpamDnsblSettings" => ObjectType::SpamDnsblSettings, b"SpamFileExtension" => ObjectType::SpamFileExtension, - b"SpamLlm" => ObjectType::SpamLlm, b"SpamPyzor" => ObjectType::SpamPyzor, b"SpamRule" => ObjectType::SpamRule, b"SpamSettings" => ObjectType::SpamSettings, @@ -120,16 +117,14 @@ impl EnumImpl for ObjectType { b"SystemSettings" => ObjectType::SystemSettings, b"Task" => ObjectType::Task, b"TaskManager" => ObjectType::TaskManager, - b"Tenant" => ObjectType::Tenant, b"TlsExternalReport" => ObjectType::TlsExternalReport, b"TlsInternalReport" => ObjectType::TlsInternalReport, b"TlsReportSettings" => ObjectType::TlsReportSettings, - b"Trace" => ObjectType::Trace, b"Tracer" => ObjectType::Tracer, - b"TracingStore" => ObjectType::TracingStore, b"WebDav" => ObjectType::WebDav, b"WebHook" => ObjectType::WebHook, } + .copied() } fn as_str(&self) -> &'static str { @@ -140,13 +135,10 @@ impl EnumImpl for ObjectType { ObjectType::AcmeProvider => "AcmeProvider", ObjectType::Action => "Action", ObjectType::AddressBook => "AddressBook", - ObjectType::AiModel => "AiModel", - ObjectType::Alert => "Alert", ObjectType::AllowedIp => "AllowedIp", ObjectType::ApiKey => "ApiKey", ObjectType::AppPassword => "AppPassword", ObjectType::Application => "Application", - ObjectType::ArchivedItem => "ArchivedItem", ObjectType::ArfExternalReport => "ArfExternalReport", ObjectType::Asn => "Asn", ObjectType::Authentication => "Authentication", @@ -174,7 +166,6 @@ impl EnumImpl for ObjectType { ObjectType::Domain => "Domain", ObjectType::DsnReportSettings => "DsnReportSettings", ObjectType::Email => "Email", - ObjectType::Enterprise => "Enterprise", ObjectType::EventTracingLevel => "EventTracingLevel", ObjectType::FileStorage => "FileStorage", ObjectType::Http => "Http", @@ -182,15 +173,14 @@ impl EnumImpl for ObjectType { ObjectType::HttpLookup => "HttpLookup", ObjectType::Imap => "Imap", ObjectType::InMemoryStore => "InMemoryStore", + ObjectType::IndexQueueStatus => "IndexQueueStatus", ObjectType::Jmap => "Jmap", ObjectType::Log => "Log", ObjectType::MailingList => "MailingList", - ObjectType::MaskedEmail => "MaskedEmail", ObjectType::MemoryLookupKey => "MemoryLookupKey", ObjectType::MemoryLookupKeyValue => "MemoryLookupKeyValue", - ObjectType::Metric => "Metric", + ObjectType::Metadata => "Metadata", ObjectType::Metrics => "Metrics", - ObjectType::MetricsStore => "MetricsStore", ObjectType::MtaConnectionStrategy => "MtaConnectionStrategy", ObjectType::MtaDeliverySchedule => "MtaDeliverySchedule", ObjectType::MtaExtensions => "MtaExtensions", @@ -213,6 +203,7 @@ impl EnumImpl for ObjectType { ObjectType::MtaVirtualQueue => "MtaVirtualQueue", ObjectType::NetworkListener => "NetworkListener", ObjectType::OAuthClient => "OAuthClient", + ObjectType::OAuthGrant => "OAuthGrant", ObjectType::OidcProvider => "OidcProvider", ObjectType::PublicKey => "PublicKey", ObjectType::QueuedMessage => "QueuedMessage", @@ -231,7 +222,6 @@ impl EnumImpl for ObjectType { ObjectType::SpamDnsblServer => "SpamDnsblServer", ObjectType::SpamDnsblSettings => "SpamDnsblSettings", ObjectType::SpamFileExtension => "SpamFileExtension", - ObjectType::SpamLlm => "SpamLlm", ObjectType::SpamPyzor => "SpamPyzor", ObjectType::SpamRule => "SpamRule", ObjectType::SpamSettings => "SpamSettings", @@ -242,13 +232,10 @@ impl EnumImpl for ObjectType { ObjectType::SystemSettings => "SystemSettings", ObjectType::Task => "Task", ObjectType::TaskManager => "TaskManager", - ObjectType::Tenant => "Tenant", ObjectType::TlsExternalReport => "TlsExternalReport", ObjectType::TlsInternalReport => "TlsInternalReport", ObjectType::TlsReportSettings => "TlsReportSettings", - ObjectType::Trace => "Trace", ObjectType::Tracer => "Tracer", - ObjectType::TracingStore => "TracingStore", ObjectType::WebDav => "WebDav", ObjectType::WebHook => "WebHook", } @@ -266,13 +253,10 @@ impl EnumImpl for ObjectType { 3 => Some(ObjectType::AcmeProvider), 4 => Some(ObjectType::Action), 5 => Some(ObjectType::AddressBook), - 6 => Some(ObjectType::AiModel), - 7 => Some(ObjectType::Alert), 8 => Some(ObjectType::AllowedIp), 9 => Some(ObjectType::ApiKey), 10 => Some(ObjectType::AppPassword), 11 => Some(ObjectType::Application), - 12 => Some(ObjectType::ArchivedItem), 13 => Some(ObjectType::ArfExternalReport), 14 => Some(ObjectType::Asn), 15 => Some(ObjectType::Authentication), @@ -300,7 +284,6 @@ impl EnumImpl for ObjectType { 37 => Some(ObjectType::Domain), 38 => Some(ObjectType::DsnReportSettings), 39 => Some(ObjectType::Email), - 40 => Some(ObjectType::Enterprise), 41 => Some(ObjectType::EventTracingLevel), 42 => Some(ObjectType::FileStorage), 43 => Some(ObjectType::Http), @@ -308,80 +291,76 @@ impl EnumImpl for ObjectType { 45 => Some(ObjectType::HttpLookup), 46 => Some(ObjectType::Imap), 47 => Some(ObjectType::InMemoryStore), - 48 => Some(ObjectType::Jmap), - 49 => Some(ObjectType::Log), - 50 => Some(ObjectType::MailingList), - 51 => Some(ObjectType::MaskedEmail), - 52 => Some(ObjectType::MemoryLookupKey), - 53 => Some(ObjectType::MemoryLookupKeyValue), - 54 => Some(ObjectType::Metric), - 55 => Some(ObjectType::Metrics), - 56 => Some(ObjectType::MetricsStore), - 57 => Some(ObjectType::MtaConnectionStrategy), - 58 => Some(ObjectType::MtaDeliverySchedule), - 59 => Some(ObjectType::MtaExtensions), - 60 => Some(ObjectType::MtaHook), - 61 => Some(ObjectType::MtaInboundSession), - 62 => Some(ObjectType::MtaInboundThrottle), - 63 => Some(ObjectType::MtaMilter), - 64 => Some(ObjectType::MtaOutboundStrategy), - 65 => Some(ObjectType::MtaOutboundThrottle), - 66 => Some(ObjectType::MtaQueueQuota), - 67 => Some(ObjectType::MtaRoute), - 68 => Some(ObjectType::MtaStageAuth), - 69 => Some(ObjectType::MtaStageConnect), - 70 => Some(ObjectType::MtaStageData), - 71 => Some(ObjectType::MtaStageEhlo), - 72 => Some(ObjectType::MtaStageMail), - 73 => Some(ObjectType::MtaStageRcpt), - 74 => Some(ObjectType::MtaSts), - 75 => Some(ObjectType::MtaTlsStrategy), - 76 => Some(ObjectType::MtaVirtualQueue), - 77 => Some(ObjectType::NetworkListener), - 78 => Some(ObjectType::OAuthClient), - 79 => Some(ObjectType::OidcProvider), - 80 => Some(ObjectType::PublicKey), - 81 => Some(ObjectType::QueuedMessage), - 82 => Some(ObjectType::ReportSettings), - 83 => Some(ObjectType::Role), - 84 => Some(ObjectType::Search), - 85 => Some(ObjectType::SearchStore), - 86 => Some(ObjectType::Security), - 87 => Some(ObjectType::SenderAuth), - 88 => Some(ObjectType::Sharing), - 89 => Some(ObjectType::SieveSystemInterpreter), - 90 => Some(ObjectType::SieveSystemScript), - 91 => Some(ObjectType::SieveUserInterpreter), - 92 => Some(ObjectType::SieveUserScript), - 93 => Some(ObjectType::SpamClassifier), - 94 => Some(ObjectType::SpamDnsblServer), - 95 => Some(ObjectType::SpamDnsblSettings), - 96 => Some(ObjectType::SpamFileExtension), - 97 => Some(ObjectType::SpamLlm), - 98 => Some(ObjectType::SpamPyzor), - 99 => Some(ObjectType::SpamRule), - 100 => Some(ObjectType::SpamSettings), - 101 => Some(ObjectType::SpamTag), - 102 => Some(ObjectType::SpamTrainingSample), - 103 => Some(ObjectType::SpfReportSettings), - 104 => Some(ObjectType::StoreLookup), - 105 => Some(ObjectType::SystemSettings), - 106 => Some(ObjectType::Task), - 107 => Some(ObjectType::TaskManager), - 108 => Some(ObjectType::Tenant), - 109 => Some(ObjectType::TlsExternalReport), - 110 => Some(ObjectType::TlsInternalReport), - 111 => Some(ObjectType::TlsReportSettings), - 112 => Some(ObjectType::Trace), - 113 => Some(ObjectType::Tracer), - 114 => Some(ObjectType::TracingStore), - 115 => Some(ObjectType::WebDav), - 116 => Some(ObjectType::WebHook), + 48 => Some(ObjectType::IndexQueueStatus), + 49 => Some(ObjectType::Jmap), + 50 => Some(ObjectType::Log), + 51 => Some(ObjectType::MailingList), + 53 => Some(ObjectType::MemoryLookupKey), + 54 => Some(ObjectType::MemoryLookupKeyValue), + 55 => Some(ObjectType::Metadata), + 57 => Some(ObjectType::Metrics), + 59 => Some(ObjectType::MtaConnectionStrategy), + 60 => Some(ObjectType::MtaDeliverySchedule), + 61 => Some(ObjectType::MtaExtensions), + 62 => Some(ObjectType::MtaHook), + 63 => Some(ObjectType::MtaInboundSession), + 64 => Some(ObjectType::MtaInboundThrottle), + 65 => Some(ObjectType::MtaMilter), + 66 => Some(ObjectType::MtaOutboundStrategy), + 67 => Some(ObjectType::MtaOutboundThrottle), + 68 => Some(ObjectType::MtaQueueQuota), + 69 => Some(ObjectType::MtaRoute), + 70 => Some(ObjectType::MtaStageAuth), + 71 => Some(ObjectType::MtaStageConnect), + 72 => Some(ObjectType::MtaStageData), + 73 => Some(ObjectType::MtaStageEhlo), + 74 => Some(ObjectType::MtaStageMail), + 75 => Some(ObjectType::MtaStageRcpt), + 76 => Some(ObjectType::MtaSts), + 77 => Some(ObjectType::MtaTlsStrategy), + 78 => Some(ObjectType::MtaVirtualQueue), + 79 => Some(ObjectType::NetworkListener), + 80 => Some(ObjectType::OAuthClient), + 81 => Some(ObjectType::OAuthGrant), + 82 => Some(ObjectType::OidcProvider), + 83 => Some(ObjectType::PublicKey), + 84 => Some(ObjectType::QueuedMessage), + 85 => Some(ObjectType::ReportSettings), + 86 => Some(ObjectType::Role), + 87 => Some(ObjectType::Search), + 88 => Some(ObjectType::SearchStore), + 89 => Some(ObjectType::Security), + 90 => Some(ObjectType::SenderAuth), + 91 => Some(ObjectType::Sharing), + 92 => Some(ObjectType::SieveSystemInterpreter), + 93 => Some(ObjectType::SieveSystemScript), + 94 => Some(ObjectType::SieveUserInterpreter), + 95 => Some(ObjectType::SieveUserScript), + 97 => Some(ObjectType::SpamClassifier), + 98 => Some(ObjectType::SpamDnsblServer), + 99 => Some(ObjectType::SpamDnsblSettings), + 100 => Some(ObjectType::SpamFileExtension), + 101 => Some(ObjectType::SpamPyzor), + 102 => Some(ObjectType::SpamRule), + 103 => Some(ObjectType::SpamSettings), + 104 => Some(ObjectType::SpamTag), + 105 => Some(ObjectType::SpamTrainingSample), + 106 => Some(ObjectType::SpfReportSettings), + 107 => Some(ObjectType::StoreLookup), + 108 => Some(ObjectType::SystemSettings), + 109 => Some(ObjectType::Task), + 110 => Some(ObjectType::TaskManager), + 112 => Some(ObjectType::TlsExternalReport), + 113 => Some(ObjectType::TlsInternalReport), + 114 => Some(ObjectType::TlsReportSettings), + 116 => Some(ObjectType::Tracer), + 118 => Some(ObjectType::WebDav), + 119 => Some(ObjectType::WebHook), _ => None, } } - const COUNT: usize = 117; + const COUNT: usize = 120; } impl serde::Serialize for ObjectType { @@ -404,18 +383,21 @@ impl<'de> serde::Deserialize<'de> for ObjectType { } impl EnumImpl for Property { + #[cfg(not(feature = "enterprise"))] fn parse(value: &str) -> Option { - hashify::tiny_map! { + hashify::map! { value.as_bytes(), + Property, b"@type" => Property::Type, b"abuseBanPeriod" => Property::AbuseBanPeriod, b"abuseBanRate" => Property::AbuseBanRate, - b"accessKey" => Property::AccessKey, + b"acceptedAudiences" => Property::AcceptedAudiences, b"accessKeyId" => Property::AccessKeyId, b"accessToken" => Property::AccessToken, b"accessTokenExpiry" => Property::AccessTokenExpiry, b"accessTokens" => Property::AccessTokens, b"accountDomainId" => Property::AccountDomainId, + b"accountHashPrefix" => Property::AccountHashPrefix, b"accountId" => Property::AccountId, b"accountIdentifier" => Property::AccountIdentifier, b"accountKey" => Property::AccountKey, @@ -434,43 +416,37 @@ impl EnumImpl for Property { b"addReturnPathHeader" => Property::AddReturnPathHeader, b"additionalInformation" => Property::AdditionalInformation, b"address" => Property::Address, - b"addresses" => Property::Addresses, b"aggregateContactInfo" => Property::AggregateContactInfo, b"aggregateDkimSignDomain" => Property::AggregateDkimSignDomain, b"aggregateFromAddress" => Property::AggregateFromAddress, b"aggregateFromName" => Property::AggregateFromName, - b"aggregateMaxReportSize" => Property::AggregateMaxReportSize, b"aggregateOrgName" => Property::AggregateOrgName, b"aggregateSendFrequency" => Property::AggregateSendFrequency, - b"aggregateSubject" => Property::AggregateSubject, b"alarmId" => Property::AlarmId, + b"alertAfter" => Property::AlertAfter, b"algorithms" => Property::Algorithms, b"aliases" => Property::Aliases, b"allowCount" => Property::AllowCount, b"allowDirectoryQueries" => Property::AllowDirectoryQueries, b"allowExternalRcpts" => Property::AllowExternalRcpts, + b"allowIdTokenClaims" => Property::AllowIdTokenClaims, b"allowInvalidCerts" => Property::AllowInvalidCerts, b"allowPlainTextAuth" => Property::AllowPlainTextAuth, b"allowRelaying" => Property::AllowRelaying, - b"allowScimProvisioning" => Property::AllowScimProvisioning, b"allowSpamTraining" => Property::AllowSpamTraining, b"allowedEndpoints" => Property::AllowedEndpoints, b"allowedIps" => Property::AllowedIps, b"allowedNotifyUris" => Property::AllowedNotifyUris, b"alpha" => Property::Alpha, - b"anonymousClientRegistration" => Property::AnonymousClientRegistration, + b"anonymousTimeout" => Property::AnonymousTimeout, b"ansi" => Property::Ansi, b"apiKey" => Property::ApiKey, + b"apiKeyId" => Property::ApiKeyId, b"apiUser" => Property::ApiUser, b"applicationKey" => Property::ApplicationKey, b"applicationSecret" => Property::ApplicationSecret, b"arcResult" => Property::ArcResult, b"arcVerify" => Property::ArcVerify, - b"archiveDeletedAccountsFor" => Property::ArchiveDeletedAccountsFor, - b"archiveDeletedItemsFor" => Property::ArchiveDeletedItemsFor, - b"archivedAt" => Property::ArchivedAt, - b"archivedItemType" => Property::ArchivedItemType, - b"archivedUntil" => Property::ArchivedUntil, b"arrivalDate" => Property::ArrivalDate, b"asnUrls" => Property::AsnUrls, b"attemptNumber" => Property::AttemptNumber, @@ -487,18 +463,17 @@ impl EnumImpl for Property { b"authBanPeriod" => Property::AuthBanPeriod, b"authBanRate" => Property::AuthBanRate, b"authCodeExpiry" => Property::AuthCodeExpiry, - b"authCodeMaxAttempts" => Property::AuthCodeMaxAttempts, b"authFailure" => Property::AuthFailure, b"authId" => Property::AuthId, b"authSecret" => Property::AuthSecret, - b"authToken" => Property::AuthToken, b"authUsername" => Property::AuthUsername, b"authenticatedAs" => Property::AuthenticatedAs, + b"authenticatedTimeout" => Property::AuthenticatedTimeout, b"authenticationResults" => Property::AuthenticationResults, b"autoAddInvitations" => Property::AutoAddInvitations, - b"autoUpdateFrequency" => Property::AutoUpdateFrequency, + b"autoUpdateInterval" => Property::AutoUpdateInterval, + b"availabilityRateLimit" => Property::AvailabilityRateLimit, b"baseDn" => Property::BaseDn, - b"baseUrl" => Property::BaseUrl, b"bearerToken" => Property::BearerToken, b"beta" => Property::Beta, b"bind" => Property::Bind, @@ -510,29 +485,29 @@ impl EnumImpl for Property { b"blobSize" => Property::BlobSize, b"blobStore" => Property::BlobStore, b"blockCount" => Property::BlockCount, - b"body" => Property::Body, - b"brokers" => Property::Brokers, b"bucket" => Property::Bucket, b"bufferSize" => Property::BufferSize, b"buffered" => Property::Buffered, + b"caaIdentities" => Property::CaaIdentities, b"cacheSize" => Property::CacheSize, b"canonicalization" => Property::Canonicalization, - b"capacityClient" => Property::CapacityClient, - b"capacityReadBuffer" => Property::CapacityReadBuffer, - b"capacitySubscription" => Property::CapacitySubscription, b"catchAllAddress" => Property::CatchAllAddress, - b"categories" => Property::Categories, b"certificate" => Property::Certificate, b"certificateManagement" => Property::CertificateManagement, b"challengeType" => Property::ChallengeType, - b"changesMaxResults" => Property::ChangesMaxResults, + b"chunkSize" => Property::ChunkSize, b"chunking" => Property::Chunking, b"claimGroups" => Property::ClaimGroups, b"claimName" => Property::ClaimName, b"claimUsername" => Property::ClaimUsername, b"cleartext" => Property::Cleartext, + b"clientCapacity" => Property::ClientCapacity, b"clientId" => Property::ClientId, b"clientIp" => Property::ClientIp, + b"clientIpSource" => Property::ClientIpSource, + b"clientKind" => Property::ClientKind, + b"clientMetadataDocuments" => Property::ClientMetadataDocuments, + b"clientName" => Property::ClientName, b"clientSecret" => Property::ClientSecret, b"clientToken" => Property::ClientToken, b"clusterFile" => Property::ClusterFile, @@ -540,12 +515,11 @@ impl EnumImpl for Property { b"columnDescription" => Property::ColumnDescription, b"columnEmail" => Property::ColumnEmail, b"columnSecret" => Property::ColumnSecret, + b"commandTimeout" => Property::CommandTimeout, b"comment" => Property::Comment, b"compartmentOcid" => Property::CompartmentOcid, b"compressionAlgorithm" => Property::CompressionAlgorithm, - b"concurrency" => Property::Concurrency, b"condition" => Property::Condition, - b"confidence" => Property::Confidence, b"config" => Property::Config, b"configName" => Property::ConfigName, b"connectTimeout" => Property::ConnectTimeout, @@ -555,32 +529,27 @@ impl EnumImpl for Property { b"contactInfo" => Property::ContactInfo, b"contacts" => Property::Contacts, b"container" => Property::Container, - b"content" => Property::Content, b"contentTypes" => Property::ContentTypes, b"contents" => Property::Contents, b"context" => Property::Context, b"count" => Property::Count, b"create" => Property::Create, b"createdAt" => Property::CreatedAt, - b"createdBy" => Property::CreatedBy, b"credentialId" => Property::CredentialId, b"credentials" => Property::Credentials, b"currentSecret" => Property::CurrentSecret, - b"customEndpoint" => Property::CustomEndpoint, - b"customRegion" => Property::CustomRegion, b"customRule" => Property::CustomRule, b"customerNumber" => Property::CustomerNumber, b"dane" => Property::Dane, b"dataCleanupSchedule" => Property::DataCleanupSchedule, b"dataStore" => Property::DataStore, b"dataTimeout" => Property::DataTimeout, + b"dataTypes" => Property::DataTypes, b"database" => Property::Database, b"datacenterId" => Property::DatacenterId, - b"dateRangeBegin" => Property::DateRangeBegin, b"dateRangeEnd" => Property::DateRangeEnd, b"dateRangeStart" => Property::DateRangeStart, b"day" => Property::Day, - b"deadPropertyMaxSize" => Property::DeadPropertyMaxSize, b"defaultAdminRoleIds" => Property::DefaultAdminRoleIds, b"defaultCertificateId" => Property::DefaultCertificateId, b"defaultDisplayName" => Property::DefaultDisplayName, @@ -599,7 +568,6 @@ impl EnumImpl for Property { b"defaultReturnPath" => Property::DefaultReturnPath, b"defaultSubject" => Property::DefaultSubject, b"defaultSubjectPrefix" => Property::DefaultSubjectPrefix, - b"defaultTenantRoleIds" => Property::DefaultTenantRoleIds, b"defaultUserRoleIds" => Property::DefaultUserRoleIds, b"definition" => Property::Definition, b"delay" => Property::Delay, @@ -614,6 +582,8 @@ impl EnumImpl for Property { b"details" => Property::Details, b"directory" => Property::Directory, b"directoryId" => Property::DirectoryId, + b"directoryRecipients" => Property::DirectoryRecipients, + b"directoryRecipientsTtl" => Property::DirectoryRecipientsTtl, b"disableCapabilities" => Property::DisableCapabilities, b"disabledPermissions" => Property::DisabledPermissions, b"discardAfter" => Property::DiscardAfter, @@ -644,7 +614,6 @@ impl EnumImpl for Property { b"dnsMtaSts" => Property::DnsMtaSts, b"dnsMx" => Property::DnsMx, b"dnsPtr" => Property::DnsPtr, - b"dnsRbl" => Property::DnsRbl, b"dnsServer" => Property::DnsServer, b"dnsServerId" => Property::DnsServerId, b"dnsTlsa" => Property::DnsTlsa, @@ -652,7 +621,6 @@ impl EnumImpl for Property { b"dnsView" => Property::DnsView, b"dnsZoneFile" => Property::DnsZoneFile, b"documentId" => Property::DocumentId, - b"documentType" => Property::DocumentType, b"domain" => Property::Domain, b"domainId" => Property::DomainId, b"domainLimit" => Property::DomainLimit, @@ -661,7 +629,6 @@ impl EnumImpl for Property { b"domains" => Property::Domains, b"dsn" => Property::Dsn, b"due" => Property::Due, - b"duplicateExpiry" => Property::DuplicateExpiry, b"duration" => Property::Duration, b"eabHmacKey" => Property::EabHmacKey, b"eabKeyId" => Property::EabKeyId, @@ -674,11 +641,7 @@ impl EnumImpl for Property { b"emailAddress" => Property::EmailAddress, b"emailAddresses" => Property::EmailAddresses, b"emailAddressesNegative" => Property::EmailAddressesNegative, - b"emailAlert" => Property::EmailAlert, - b"emailDomain" => Property::EmailDomain, b"emailLimit" => Property::EmailLimit, - b"emailPrefix" => Property::EmailPrefix, - b"emailTemplate" => Property::EmailTemplate, b"enable" => Property::Enable, b"enableAssistedDiscovery" => Property::EnableAssistedDiscovery, b"enableEdns" => Property::EnableEdns, @@ -686,17 +649,12 @@ impl EnumImpl for Property { b"enableLogExporter" => Property::EnableLogExporter, b"enableSpamFilter" => Property::EnableSpamFilter, b"enableSpanExporter" => Property::EnableSpanExporter, - b"enabled" => Property::Enabled, b"enabledPermissions" => Property::EnabledPermissions, b"encryptAtRest" => Property::EncryptAtRest, b"encryptOnAppend" => Property::EncryptOnAppend, b"encryptionAtRest" => Property::EncryptionAtRest, b"encryptionKey" => Property::EncryptionKey, - b"endpoint" => Property::Endpoint, - b"envFrom" => Property::EnvFrom, - b"envFromParameters" => Property::EnvFromParameters, b"envId" => Property::EnvId, - b"envRcptTo" => Property::EnvRcptTo, b"envelopeFrom" => Property::EnvelopeFrom, b"envelopeTo" => Property::EnvelopeTo, b"environment" => Property::Environment, @@ -708,11 +666,9 @@ impl EnumImpl for Property { b"evaluatedDkim" => Property::EvaluatedDkim, b"evaluatedSpf" => Property::EvaluatedSpf, b"event" => Property::Event, - b"eventAlert" => Property::EventAlert, b"eventEnd" => Property::EventEnd, b"eventEndTz" => Property::EventEndTz, b"eventId" => Property::EventId, - b"eventMessage" => Property::EventMessage, b"eventSourceThrottle" => Property::EventSourceThrottle, b"eventStart" => Property::EventStart, b"eventStartTz" => Property::EventStartTz, @@ -731,7 +687,6 @@ impl EnumImpl for Property { b"expungeTrashAfter" => Property::ExpungeTrashAfter, b"extension" => Property::Extension, b"extensions" => Property::Extensions, - b"externalId" => Property::ExternalId, b"extraContactInfo" => Property::ExtraContactInfo, b"factor" => Property::Factor, b"failOnTimeout" => Property::FailOnTimeout, @@ -756,43 +711,47 @@ impl EnumImpl for Property { b"filePath" => Property::FilePath, b"files" => Property::Files, b"filterLogin" => Property::FilterLogin, - b"filterMailbox" => Property::FilterMailbox, b"filterMemberOf" => Property::FilterMemberOf, + b"filterRecipient" => Property::FilterRecipient, b"fingerprint" => Property::Fingerprint, + b"firstFailedAt" => Property::FirstFailedAt, + b"firstParty" => Property::FirstParty, b"flags" => Property::Flags, b"flagsAction" => Property::FlagsAction, b"flagsProtocol" => Property::FlagsProtocol, + b"flushChanges" => Property::FlushChanges, + b"flushInterval" => Property::FlushInterval, b"folderId" => Property::FolderId, - b"forDomain" => Property::ForDomain, b"format" => Property::Format, + b"forwardedHops" => Property::ForwardedHops, b"from" => Property::From, b"fromAddress" => Property::FromAddress, - b"fromEmail" => Property::FromEmail, b"fromName" => Property::FromName, b"futureRelease" => Property::FutureRelease, b"generateDkimKeys" => Property::GenerateDkimKeys, + b"generation" => Property::Generation, b"generator" => Property::Generator, b"geoUrls" => Property::GeoUrls, - b"getMaxResults" => Property::GetMaxResults, + b"grantTypes" => Property::GrantTypes, b"greetingTimeout" => Property::GreetingTimeout, b"greylistFor" => Property::GreylistFor, b"groupClass" => Property::GroupClass, b"groupId" => Property::GroupId, + b"groupwareLarge" => Property::GroupwareLarge, + b"groupwareLargeTtl" => Property::GroupwareLargeTtl, b"headerFrom" => Property::HeaderFrom, b"headers" => Property::Headers, - b"holdMetricsFor" => Property::HoldMetricsFor, b"holdMtaReportsFor" => Property::HoldMtaReportsFor, b"holdSamplesFor" => Property::HoldSamplesFor, - b"holdTracesFor" => Property::HoldTracesFor, b"host" => Property::Host, b"hostedZoneId" => Property::HostedZoneId, b"hostname" => Property::Hostname, + b"hosts" => Property::Hosts, b"hour" => Property::Hour, b"httpAuth" => Property::HttpAuth, b"httpHeaders" => Property::HttpHeaders, b"httpRsvpEnable" => Property::HttpRsvpEnable, b"httpRsvpLinkExpiry" => Property::HttpRsvpLinkExpiry, - b"httpRsvpTemplate" => Property::HttpRsvpTemplate, b"httpRsvpUrl" => Property::HttpRsvpUrl, b"httpRua" => Property::HttpRua, b"humanResult" => Property::HumanResult, @@ -800,15 +759,19 @@ impl EnumImpl for Property { b"id" => Property::Id, b"idTokenExpiry" => Property::IdTokenExpiry, b"identityAlignment" => Property::IdentityAlignment, + b"idleTimeout" => Property::IdleTimeout, b"if" => Property::If, + b"imapServerAdmin" => Property::ImapServerAdmin, + b"imapServerComment" => Property::ImapServerComment, b"impersonateServiceAccount" => Property::ImpersonateServiceAccount, b"implicitTls" => Property::ImplicitTls, b"inMemoryStore" => Property::InMemoryStore, b"inboundReportAddresses" => Property::InboundReportAddresses, b"inboundReportForwarding" => Property::InboundReportForwarding, - b"inboundReportMaxSize" => Property::InboundReportMaxSize, b"incidents" => Property::Incidents, + b"includeSecrets" => Property::IncludeSecrets, b"includeSource" => Property::IncludeSource, + b"index" => Property::Index, b"indexAsn" => Property::IndexAsn, b"indexAsnName" => Property::IndexAsnName, b"indexBatchSize" => Property::IndexBatchSize, @@ -819,14 +782,17 @@ impl EnumImpl for Property { b"indexCountry" => Property::IndexCountry, b"indexEmail" => Property::IndexEmail, b"indexEmailFields" => Property::IndexEmailFields, + b"indexFiles" => Property::IndexFiles, b"indexKey" => Property::IndexKey, - b"indexTelemetry" => Property::IndexTelemetry, - b"indexTracingFields" => Property::IndexTracingFields, b"indexValue" => Property::IndexValue, b"indicatorParameters" => Property::IndicatorParameters, b"initialDelay" => Property::InitialDelay, + b"instanceHashes" => Property::InstanceHashes, b"interval" => Property::Interval, b"intervals" => Property::Intervals, + b"introspectionAuthMethod" => Property::IntrospectionAuthMethod, + b"introspectionClientId" => Property::IntrospectionClientId, + b"introspectionClientSecret" => Property::IntrospectionClientSecret, b"ipLimit" => Property::IpLimit, b"ipLookupStrategy" => Property::IpLookupStrategy, b"ipRevPtr" => Property::IpRevPtr, @@ -834,7 +800,6 @@ impl EnumImpl for Property { b"isActive" => Property::IsActive, b"isArchive" => Property::IsArchive, b"isBad" => Property::IsBad, - b"isEnabled" => Property::IsEnabled, b"isFromOrganizer" => Property::IsFromOrganizer, b"isGlobPattern" => Property::IsGlobPattern, b"isGzipped" => Property::IsGzipped, @@ -843,80 +808,111 @@ impl EnumImpl for Property { b"isSpam" => Property::IsSpam, b"isTls" => Property::IsTls, b"issuer" => Property::Issuer, + b"issuerDomainNames" => Property::IssuerDomainNames, b"issuerUrl" => Property::IssuerUrl, - b"itipMaxSize" => Property::ItipMaxSize, b"jitter" => Property::Jitter, b"key" => Property::Key, b"keyName" => Property::KeyName, b"keyPrefix" => Property::KeyPrefix, - b"keyValues" => Property::KeyValues, b"l1Ratio" => Property::L1Ratio, b"l2Ratio" => Property::L2Ratio, b"lastRenewal" => Property::LastRenewal, + b"lastUsedAt" => Property::LastUsedAt, b"learnHamFromCard" => Property::LearnHamFromCard, b"learnHamFromReply" => Property::LearnHamFromReply, b"learnSpamFromRblHits" => Property::LearnSpamFromRblHits, b"learnSpamFromTraps" => Property::LearnSpamFromTraps, + b"lenientIssuerMatch" => Property::LenientIssuerMatch, b"level" => Property::Level, - b"licenseKey" => Property::LicenseKey, b"line" => Property::Line, b"listenerIds" => Property::ListenerIds, b"listeners" => Property::Listeners, - b"livePropertyMaxSize" => Property::LivePropertyMaxSize, + b"localIp" => Property::LocalIp, + b"localIps" => Property::LocalIps, b"locale" => Property::Locale, b"logo" => Property::Logo, - b"logoUrl" => Property::LogoUrl, b"loiterBanPeriod" => Property::LoiterBanPeriod, b"loiterBanRate" => Property::LoiterBanRate, b"lossy" => Property::Lossy, b"machineId" => Property::MachineId, b"mailExchangers" => Property::MailExchangers, b"mailFrom" => Property::MailFrom, + b"mailFromParameters" => Property::MailFromParameters, b"mailFromTimeout" => Property::MailFromTimeout, b"mailRua" => Property::MailRua, b"mailingLists" => Property::MailingLists, b"maintenanceType" => Property::MaintenanceType, b"managedZone" => Property::ManagedZone, b"match" => Property::Match, + b"maxAccountSize" => Property::MaxAccountSize, b"maxAddressBooks" => Property::MaxAddressBooks, + b"maxAddressBooksPerCard" => Property::MaxAddressBooksPerCard, b"maxAge" => Property::MaxAge, + b"maxAggregateReportSize" => Property::MaxAggregateReportSize, b"maxAllowedPacket" => Property::MaxAllowedPacket, b"maxApiKeys" => Property::MaxApiKeys, b"maxAppPasswords" => Property::MaxAppPasswords, b"maxAttachmentSize" => Property::MaxAttachmentSize, + b"maxAttachmentText" => Property::MaxAttachmentText, + b"maxAttachmentsSize" => Property::MaxAttachmentsSize, b"maxAttempts" => Property::MaxAttempts, b"maxAttendees" => Property::MaxAttendees, b"maxAuthFailures" => Property::MaxAuthFailures, + b"maxAvailabilityDuration" => Property::MaxAvailabilityDuration, + b"maxBodyText" => Property::MaxBodyText, + b"maxBufferedMemory" => Property::MaxBufferedMemory, b"maxCalendars" => Property::MaxCalendars, + b"maxCalendarsPerEvent" => Property::MaxCalendarsPerEvent, b"maxChangesHistory" => Property::MaxChangesHistory, - b"maxConcurrent" => Property::MaxConcurrent, + b"maxChangesResults" => Property::MaxChangesResults, + b"maxConcurrentConnections" => Property::MaxConcurrentConnections, + b"maxConcurrentIndexTasks" => Property::MaxConcurrentIndexTasks, b"maxConcurrentRequests" => Property::MaxConcurrentRequests, b"maxConcurrentUploads" => Property::MaxConcurrentUploads, b"maxConnections" => Property::MaxConnections, b"maxContacts" => Property::MaxContacts, b"maxCpuCycles" => Property::MaxCpuCycles, b"maxDelay" => Property::MaxDelay, + b"maxDepth" => Property::MaxDepth, b"maxDuration" => Property::MaxDuration, b"maxEntries" => Property::MaxEntries, b"maxEntrySize" => Property::MaxEntrySize, b"maxEventNotifications" => Property::MaxEventNotifications, b"maxEvents" => Property::MaxEvents, + b"maxExpandedQueryDuration" => Property::MaxExpandedQueryDuration, + b"maxExtractArchiveEntries" => Property::MaxExtractArchiveEntries, + b"maxExtractDecompressedSize" => Property::MaxExtractDecompressedSize, + b"maxExtractDocumentSize" => Property::MaxExtractDocumentSize, + b"maxExtractPartSize" => Property::MaxExtractPartSize, + b"maxExtractParts" => Property::MaxExtractParts, + b"maxExtractPdfObjects" => Property::MaxExtractPdfObjects, + b"maxExtractRtfDepth" => Property::MaxExtractRtfDepth, + b"maxExtractTextSize" => Property::MaxExtractTextSize, b"maxFailures" => Property::MaxFailures, b"maxFiles" => Property::MaxFiles, + b"maxFlagLength" => Property::MaxFlagLength, + b"maxFlagsPerEmail" => Property::MaxFlagsPerEmail, b"maxFolders" => Property::MaxFolders, + b"maxGetResults" => Property::MaxGetResults, + b"maxHeaderCount" => Property::MaxHeaderCount, b"maxHeaderSize" => Property::MaxHeaderSize, b"maxICalendarSize" => Property::MaxICalendarSize, b"maxIdentities" => Property::MaxIdentities, + b"maxInboundReportSize" => Property::MaxInboundReportSize, b"maxIncludes" => Property::MaxIncludes, + b"maxItipSize" => Property::MaxItipSize, + b"maxLivePropertySize" => Property::MaxLivePropertySize, b"maxLocalVars" => Property::MaxLocalVars, b"maxLockTimeout" => Property::MaxLockTimeout, b"maxLocks" => Property::MaxLocks, b"maxMailboxDepth" => Property::MaxMailboxDepth, b"maxMailboxNameLength" => Property::MaxMailboxNameLength, b"maxMailboxes" => Property::MaxMailboxes, - b"maxMaskedAddresses" => Property::MaxMaskedAddresses, + b"maxMailboxesPerEmail" => Property::MaxMailboxesPerEmail, b"maxMatchVars" => Property::MaxMatchVars, + b"maxMediaSize" => Property::MaxMediaSize, b"maxMessageSize" => Property::MaxMessageSize, + b"maxMessageText" => Property::MaxMessageText, b"maxMessages" => Property::MaxMessages, b"maxMessagesPerCommand" => Property::MaxMessagesPerCommand, b"maxMessagesPerSave" => Property::MaxMessagesPerSave, @@ -929,15 +925,21 @@ impl EnumImpl for Property { b"maxNestedTests" => Property::MaxNestedTests, b"maxOutMessages" => Property::MaxOutMessages, b"maxParticipantIdentities" => Property::MaxParticipantIdentities, + b"maxPasswordLength" => Property::MaxPasswordLength, + b"maxPolls" => Property::MaxPolls, + b"maxPrivateSize" => Property::MaxPrivateSize, b"maxPublicKeys" => Property::MaxPublicKeys, + b"maxPushAttempts" => Property::MaxPushAttempts, b"maxPushSize" => Property::MaxPushSize, + b"maxQueryResults" => Property::MaxQueryResults, + b"maxQueryScan" => Property::MaxQueryScan, b"maxReceivedHeaders" => Property::MaxReceivedHeaders, b"maxRecipients" => Property::MaxRecipients, b"maxReconnects" => Property::MaxReconnects, b"maxRecurrenceExpansions" => Property::MaxRecurrenceExpansions, + b"maxRedirectUrls" => Property::MaxRedirectUrls, b"maxRedirects" => Property::MaxRedirects, b"maxReportSize" => Property::MaxReportSize, - b"maxRequestRate" => Property::MaxRequestRate, b"maxRequestSize" => Property::MaxRequestSize, b"maxResponseSize" => Property::MaxResponseSize, b"maxResults" => Property::MaxResults, @@ -946,8 +948,10 @@ impl EnumImpl for Property { b"maxScriptNameLength" => Property::MaxScriptNameLength, b"maxScriptSize" => Property::MaxScriptSize, b"maxScripts" => Property::MaxScripts, + b"maxSetObjects" => Property::MaxSetObjects, b"maxShares" => Property::MaxShares, b"maxSize" => Property::MaxSize, + b"maxSnippetResults" => Property::MaxSnippetResults, b"maxStringLength" => Property::MaxStringLength, b"maxSubmissions" => Property::MaxSubmissions, b"maxSubscriptions" => Property::MaxSubscriptions, @@ -958,16 +962,19 @@ impl EnumImpl for Property { b"maxVarNameLength" => Property::MaxVarNameLength, b"maxVarSize" => Property::MaxVarSize, b"memberGroupIds" => Property::MemberGroupIds, - b"memberTenantId" => Property::MemberTenantId, b"message" => Property::Message, b"messageIdHostname" => Property::MessageIdHostname, b"messageIds" => Property::MessageIds, + b"messageTimeout" => Property::MessageTimeout, b"messages" => Property::Messages, - b"metric" => Property::Metric, + b"messagesLarge" => Property::MessagesLarge, + b"messagesLargeTtl" => Property::MessagesLargeTtl, b"metrics" => Property::Metrics, - b"metricsCollectionInterval" => Property::MetricsCollectionInterval, b"metricsPolicy" => Property::MetricsPolicy, + b"minAccountSize" => Property::MinAccountSize, b"minHamSamples" => Property::MinHamSamples, + b"minPasswordLength" => Property::MinPasswordLength, + b"minPasswordStrength" => Property::MinPasswordStrength, b"minRetryWait" => Property::MinRetryWait, b"minSpamSamples" => Property::MinSpamSamples, b"minTriggerInterval" => Property::MinTriggerInterval, @@ -975,8 +982,6 @@ impl EnumImpl for Property { b"minute" => Property::Minute, b"mode" => Property::Mode, b"model" => Property::Model, - b"modelId" => Property::ModelId, - b"modelType" => Property::ModelType, b"mtPriority" => Property::MtPriority, b"mtaSts" => Property::MtaSts, b"mtaStsTimeout" => Property::MtaStsTimeout, @@ -1002,7 +1007,9 @@ impl EnumImpl for Property { b"numReplicas" => Property::NumReplicas, b"numShards" => Property::NumShards, b"oauthClientId" => Property::OauthClientId, + b"objectType" => Property::ObjectType, b"onSuccessRenewCertificate" => Property::OnSuccessRenewCertificate, + b"opaqueTokenValidation" => Property::OpaqueTokenValidation, b"openTelemetry" => Property::OpenTelemetry, b"options" => Property::Options, b"orcpt" => Property::Orcpt, @@ -1024,13 +1031,12 @@ impl EnumImpl for Property { b"parseLimitContact" => Property::ParseLimitContact, b"parseLimitEmail" => Property::ParseLimitEmail, b"parseLimitEvent" => Property::ParseLimitEvent, + b"partition" => Property::Partition, b"password" => Property::Password, b"passwordDefaultExpiry" => Property::PasswordDefaultExpiry, b"passwordHashAlgorithm" => Property::PasswordHashAlgorithm, - b"passwordMaxLength" => Property::PasswordMaxLength, - b"passwordMinLength" => Property::PasswordMinLength, - b"passwordMinStrength" => Property::PasswordMinStrength, b"path" => Property::Path, + b"pending" => Property::Pending, b"period" => Property::Period, b"permissions" => Property::Permissions, b"pingInterval" => Property::PingInterval, @@ -1052,7 +1058,6 @@ impl EnumImpl for Property { b"policyType" => Property::PolicyType, b"policyVersion" => Property::PolicyVersion, b"pollInterval" => Property::PollInterval, - b"pollingInterval" => Property::PollingInterval, b"poolMaxConnections" => Property::PoolMaxConnections, b"poolMinConnections" => Property::PoolMinConnections, b"poolRecyclingMethod" => Property::PoolRecyclingMethod, @@ -1061,6 +1066,7 @@ impl EnumImpl for Property { b"poolTimeoutWait" => Property::PoolTimeoutWait, b"poolWorkers" => Property::PoolWorkers, b"port" => Property::Port, + b"postLogoutRedirectUris" => Property::PostLogoutRedirectUris, b"preferredChain" => Property::PreferredChain, b"prefix" => Property::Prefix, b"preserveIntermediates" => Property::PreserveIntermediates, @@ -1068,12 +1074,12 @@ impl EnumImpl for Property { b"privateKey" => Property::PrivateKey, b"privateKeyPassword" => Property::PrivateKeyPassword, b"privateKeyPem" => Property::PrivateKeyPem, + b"privateMetadata" => Property::PrivateMetadata, b"privateZone" => Property::PrivateZone, b"privateZoneOnly" => Property::PrivateZoneOnly, b"profile" => Property::Profile, b"projectId" => Property::ProjectId, b"prometheus" => Property::Prometheus, - b"prompt" => Property::Prompt, b"propagationDelay" => Property::PropagationDelay, b"propagationTimeout" => Property::PropagationTimeout, b"protectedHeaders" => Property::ProtectedHeaders, @@ -1084,7 +1090,6 @@ impl EnumImpl for Property { b"publicKey" => Property::PublicKey, b"publishRecords" => Property::PublishRecords, b"pushAttemptWait" => Property::PushAttemptWait, - b"pushMaxAttempts" => Property::PushMaxAttempts, b"pushRequestTimeout" => Property::PushRequestTimeout, b"pushRetryWait" => Property::PushRetryWait, b"pushShardsTotal" => Property::PushShardsTotal, @@ -1092,20 +1097,22 @@ impl EnumImpl for Property { b"pushVerifyTimeout" => Property::PushVerifyTimeout, b"queryEmailAliases" => Property::QueryEmailAliases, b"queryLogin" => Property::QueryLogin, - b"queryMaxResults" => Property::QueryMaxResults, b"queryMemberOf" => Property::QueryMemberOf, b"queryRecipient" => Property::QueryRecipient, b"queueId" => Property::QueueId, b"queueName" => Property::QueueName, + b"queuedDeletions" => Property::QueuedDeletions, + b"queuedUpdates" => Property::QueuedUpdates, b"quotas" => Property::Quotas, b"rate" => Property::Rate, b"rateLimit" => Property::RateLimit, b"rateLimitAnonymous" => Property::RateLimitAnonymous, b"rateLimitAuthenticated" => Property::RateLimitAuthenticated, b"ratio" => Property::Ratio, + b"rcptTo" => Property::RcptTo, b"rcptToTimeout" => Property::RcptToTimeout, + b"readBufferCapacity" => Property::ReadBufferCapacity, b"readFromReplicas" => Property::ReadFromReplicas, - b"readReplicas" => Property::ReadReplicas, b"reason" => Property::Reason, b"receivedAt" => Property::ReceivedAt, b"receivedFromIp" => Property::ReceivedFromIp, @@ -1116,11 +1123,12 @@ impl EnumImpl for Property { b"recipients" => Property::Recipients, b"records" => Property::Records, b"recurrenceId" => Property::RecurrenceId, + b"redirectCacheTtl" => Property::RedirectCacheTtl, b"redirectRoot" => Property::RedirectRoot, + b"redirectTimeout" => Property::RedirectTimeout, b"redirectUris" => Property::RedirectUris, - b"refresh" => Property::Refresh, + b"refreshInterval" => Property::RefreshInterval, b"refreshTokenExpiry" => Property::RefreshTokenExpiry, - b"refreshTokenRenewal" => Property::RefreshTokenRenewal, b"region" => Property::Region, b"rejectNonFqdn" => Property::RejectNonFqdn, b"remoteIp" => Property::RemoteIp, @@ -1131,12 +1139,9 @@ impl EnumImpl for Property { b"reportedDomains" => Property::ReportedDomains, b"reportedUris" => Property::ReportedUris, b"reportingMta" => Property::ReportingMta, - b"requestMaxSize" => Property::RequestMaxSize, + b"requestTimeout" => Property::RequestTimeout, b"requestTlsCertificate" => Property::RequestTlsCertificate, b"require" => Property::Require, - b"requireAudience" => Property::RequireAudience, - b"requireClientRegistration" => Property::RequireClientRegistration, - b"requireScopes" => Property::RequireScopes, b"requireTls" => Property::RequireTls, b"reservoirCapacity" => Property::ReservoirCapacity, b"resourceGroup" => Property::ResourceGroup, @@ -1146,11 +1151,9 @@ impl EnumImpl for Property { b"responseHeaders" => Property::ResponseHeaders, b"responseHostname" => Property::ResponseHostname, b"responseMessage" => Property::ResponseMessage, - b"responsePosCategory" => Property::ResponsePosCategory, - b"responsePosConfidence" => Property::ResponsePosConfidence, - b"responsePosExplanation" => Property::ResponsePosExplanation, b"result" => Property::Result, b"resultType" => Property::ResultType, + b"retention" => Property::Retention, b"retireAfter" => Property::RetireAfter, b"retry" => Property::Retry, b"retryCount" => Property::RetryCount, @@ -1175,6 +1178,7 @@ impl EnumImpl for Property { b"scheduling" => Property::Scheduling, b"scheme" => Property::Scheme, b"scope" => Property::Scope, + b"scopes" => Property::Scopes, b"score" => Property::Score, b"scoreDiscard" => Property::ScoreDiscard, b"scoreReject" => Property::ScoreReject, @@ -1182,11 +1186,7 @@ impl EnumImpl for Property { b"script" => Property::Script, b"searchStore" => Property::SearchStore, b"secret" => Property::Secret, - b"secretAccessKey" => Property::SecretAccessKey, - b"secretApiKey" => Property::SecretApiKey, - b"secretId" => Property::SecretId, b"secretKey" => Property::SecretKey, - b"securityToken" => Property::SecurityToken, b"selector" => Property::Selector, b"selectorTemplate" => Property::SelectorTemplate, b"sendFrequency" => Property::SendFrequency, @@ -1194,25 +1194,22 @@ impl EnumImpl for Property { b"sentinelSecret" => Property::SentinelSecret, b"sentinelUsername" => Property::SentinelUsername, b"separator" => Property::Separator, - b"serverHostname" => Property::ServerHostname, b"serverId" => Property::ServerId, b"servers" => Property::Servers, b"serviceAccountJson" => Property::ServiceAccountJson, b"serviceName" => Property::ServiceName, b"services" => Property::Services, + b"sessionTimeout" => Property::SessionTimeout, b"sessionToken" => Property::SessionToken, - b"setMaxObjects" => Property::SetMaxObjects, b"shardIndex" => Property::ShardIndex, b"sharedSecret" => Property::SharedSecret, - b"sig0Algorithm" => Property::Sig0Algorithm, b"signatureAlgorithm" => Property::SignatureAlgorithm, b"signatureKey" => Property::SignatureKey, - b"signerName" => Property::SignerName, b"size" => Property::Size, + b"skipAudienceValidation" => Property::SkipAudienceValidation, b"skipDeploy" => Property::SkipDeploy, b"skipFirst" => Property::SkipFirst, b"smtpGreeting" => Property::SmtpGreeting, - b"snippetMaxResults" => Property::SnippetMaxResults, b"socketBacklog" => Property::SocketBacklog, b"socketNoDelay" => Property::SocketNoDelay, b"socketReceiveBufferSize" => Property::SocketReceiveBufferSize, @@ -1222,9 +1219,9 @@ impl EnumImpl for Property { b"socketTosV4" => Property::SocketTosV4, b"socketTtl" => Property::SocketTtl, b"sourceIp" => Property::SourceIp, - b"sourceIps" => Property::SourceIps, b"sourcePort" => Property::SourcePort, b"spamFilterRulesUrl" => Property::SpamFilterRulesUrl, + b"spamLookup" => Property::SpamLookup, b"spfDns" => Property::SpfDns, b"spfEhloDomain" => Property::SpfEhloDomain, b"spfEhloResult" => Property::SpfEhloResult, @@ -1235,70 +1232,53 @@ impl EnumImpl for Property { b"spfResults" => Property::SpfResults, b"stage" => Property::Stage, b"stages" => Property::Stages, - b"startTime" => Property::StartTime, b"startTls" => Property::StartTls, b"status" => Property::Status, b"storageAccount" => Property::StorageAccount, b"store" => Property::Store, - b"stores" => Property::Stores, b"strategy" => Property::Strategy, b"subAddressing" => Property::SubAddressing, b"subAuthId" => Property::SubAuthId, b"subject" => Property::Subject, b"subjectAlternativeNames" => Property::SubjectAlternativeNames, b"subscribe" => Property::Subscribe, + b"subscriptionCapacity" => Property::SubscriptionCapacity, b"subscriptionId" => Property::SubscriptionId, - b"sum" => Property::Sum, b"summary" => Property::Summary, b"supportedLanguages" => Property::SupportedLanguages, + b"swap" => Property::Swap, b"tag" => Property::Tag, b"tags" => Property::Tags, + b"targetAccountId" => Property::TargetAccountId, b"taskTypes" => Property::TaskTypes, b"tasks" => Property::Tasks, b"tcpOnError" => Property::TcpOnError, b"teamId" => Property::TeamId, b"tempFailOnError" => Property::TempFailOnError, - b"temperature" => Property::Temperature, - b"template" => Property::Template, b"tenancyOcid" => Property::TenancyOcid, b"tenantId" => Property::TenantId, - b"tenants" => Property::Tenants, b"text" => Property::Text, b"then" => Property::Then, b"thirdParty" => Property::ThirdParty, b"thirdPartyHash" => Property::ThirdPartyHash, b"threadName" => Property::ThreadName, - b"threadPoolSize" => Property::ThreadPoolSize, b"threadsPerNode" => Property::ThreadsPerNode, b"throttle" => Property::Throttle, b"timeZone" => Property::TimeZone, b"timeout" => Property::Timeout, - b"timeoutAnonymous" => Property::TimeoutAnonymous, - b"timeoutAuthenticated" => Property::TimeoutAuthenticated, - b"timeoutCommand" => Property::TimeoutCommand, - b"timeoutConnect" => Property::TimeoutConnect, - b"timeoutConnection" => Property::TimeoutConnection, - b"timeoutData" => Property::TimeoutData, - b"timeoutIdle" => Property::TimeoutIdle, - b"timeoutMessage" => Property::TimeoutMessage, - b"timeoutRequest" => Property::TimeoutRequest, - b"timeoutSession" => Property::TimeoutSession, b"timestamp" => Property::Timestamp, - b"title" => Property::Title, b"tls" => Property::Tls, b"tlsDisableCipherSuites" => Property::TlsDisableCipherSuites, b"tlsDisableProtocols" => Property::TlsDisableProtocols, b"tlsIgnoreClientOrder" => Property::TlsIgnoreClientOrder, - b"tlsImplicit" => Property::TlsImplicit, b"tlsTimeout" => Property::TlsTimeout, b"to" => Property::To, b"token" => Property::Token, - b"totalDeadline" => Property::TotalDeadline, + b"tokenFile" => Property::TokenFile, b"totalFailedSessions" => Property::TotalFailedSessions, b"totalSuccessfulSessions" => Property::TotalSuccessfulSessions, - b"traceId" => Property::TraceId, b"tracer" => Property::Tracer, - b"trainFrequency" => Property::TrainFrequency, + b"trainInterval" => Property::TrainInterval, b"transactionRetryDelay" => Property::TransactionRetryDelay, b"transactionRetryLimit" => Property::TransactionRetryLimit, b"transactionTimeout" => Property::TransactionTimeout, @@ -1308,6 +1288,7 @@ impl EnumImpl for Property { b"tsigAlgorithm" => Property::TsigAlgorithm, b"ttl" => Property::Ttl, b"unpackDirectory" => Property::UnpackDirectory, + b"unscopedTokens" => Property::UnscopedTokens, b"updateRecords" => Property::UpdateRecords, b"uploadQuota" => Property::UploadQuota, b"uploadTtl" => Property::UploadTtl, @@ -1317,7 +1298,6 @@ impl EnumImpl for Property { b"urls" => Property::Urls, b"usePermissiveCors" => Property::UsePermissiveCors, b"useTls" => Property::UseTls, - b"useXForwarded" => Property::UseXForwarded, b"usedDiskQuota" => Property::UsedDiskQuota, b"userAgent" => Property::UserAgent, b"userCodeExpiry" => Property::UserCodeExpiry, @@ -1328,21 +1308,25 @@ impl EnumImpl for Property { b"validateDomain" => Property::ValidateDomain, b"value" => Property::Value, b"variableName" => Property::VariableName, + b"vendorNamespaces" => Property::VendorNamespaces, b"verifyAfterWrite" => Property::VerifyAfterWrite, b"version" => Property::Version, b"viewName" => Property::ViewName, b"vrfy" => Property::Vrfy, b"waitOnFail" => Property::WaitOnFail, b"wapiVersion" => Property::WapiVersion, + b"warnings" => Property::Warnings, b"webPushContact" => Property::WebPushContact, b"webPushKey" => Property::WebPushKey, b"websocketHeartbeat" => Property::WebsocketHeartbeat, b"websocketThrottle" => Property::WebsocketThrottle, b"websocketTimeout" => Property::WebsocketTimeout, + b"windowEnd" => Property::WindowEnd, b"zone" => Property::Zone, b"zoneIpV4" => Property::ZoneIpV4, b"zoneIpV6" => Property::ZoneIpV6, } + .copied() } fn as_str(&self) -> &'static str { @@ -1350,12 +1334,13 @@ impl EnumImpl for Property { Property::Type => "@type", Property::AbuseBanPeriod => "abuseBanPeriod", Property::AbuseBanRate => "abuseBanRate", - Property::AccessKey => "accessKey", + Property::AcceptedAudiences => "acceptedAudiences", Property::AccessKeyId => "accessKeyId", Property::AccessToken => "accessToken", Property::AccessTokenExpiry => "accessTokenExpiry", Property::AccessTokens => "accessTokens", Property::AccountDomainId => "accountDomainId", + Property::AccountHashPrefix => "accountHashPrefix", Property::AccountId => "accountId", Property::AccountIdentifier => "accountIdentifier", Property::AccountKey => "accountKey", @@ -1374,43 +1359,37 @@ impl EnumImpl for Property { Property::AddReturnPathHeader => "addReturnPathHeader", Property::AdditionalInformation => "additionalInformation", Property::Address => "address", - Property::Addresses => "addresses", Property::AggregateContactInfo => "aggregateContactInfo", Property::AggregateDkimSignDomain => "aggregateDkimSignDomain", Property::AggregateFromAddress => "aggregateFromAddress", Property::AggregateFromName => "aggregateFromName", - Property::AggregateMaxReportSize => "aggregateMaxReportSize", Property::AggregateOrgName => "aggregateOrgName", Property::AggregateSendFrequency => "aggregateSendFrequency", - Property::AggregateSubject => "aggregateSubject", Property::AlarmId => "alarmId", + Property::AlertAfter => "alertAfter", Property::Algorithms => "algorithms", Property::Aliases => "aliases", Property::AllowCount => "allowCount", Property::AllowDirectoryQueries => "allowDirectoryQueries", Property::AllowExternalRcpts => "allowExternalRcpts", + Property::AllowIdTokenClaims => "allowIdTokenClaims", Property::AllowInvalidCerts => "allowInvalidCerts", Property::AllowPlainTextAuth => "allowPlainTextAuth", Property::AllowRelaying => "allowRelaying", - Property::AllowScimProvisioning => "allowScimProvisioning", Property::AllowSpamTraining => "allowSpamTraining", Property::AllowedEndpoints => "allowedEndpoints", Property::AllowedIps => "allowedIps", Property::AllowedNotifyUris => "allowedNotifyUris", Property::Alpha => "alpha", - Property::AnonymousClientRegistration => "anonymousClientRegistration", + Property::AnonymousTimeout => "anonymousTimeout", Property::Ansi => "ansi", Property::ApiKey => "apiKey", + Property::ApiKeyId => "apiKeyId", Property::ApiUser => "apiUser", Property::ApplicationKey => "applicationKey", Property::ApplicationSecret => "applicationSecret", Property::ArcResult => "arcResult", Property::ArcVerify => "arcVerify", - Property::ArchiveDeletedAccountsFor => "archiveDeletedAccountsFor", - Property::ArchiveDeletedItemsFor => "archiveDeletedItemsFor", - Property::ArchivedAt => "archivedAt", - Property::ArchivedItemType => "archivedItemType", - Property::ArchivedUntil => "archivedUntil", Property::ArrivalDate => "arrivalDate", Property::AsnUrls => "asnUrls", Property::AttemptNumber => "attemptNumber", @@ -1427,18 +1406,17 @@ impl EnumImpl for Property { Property::AuthBanPeriod => "authBanPeriod", Property::AuthBanRate => "authBanRate", Property::AuthCodeExpiry => "authCodeExpiry", - Property::AuthCodeMaxAttempts => "authCodeMaxAttempts", Property::AuthFailure => "authFailure", Property::AuthId => "authId", Property::AuthSecret => "authSecret", - Property::AuthToken => "authToken", Property::AuthUsername => "authUsername", Property::AuthenticatedAs => "authenticatedAs", + Property::AuthenticatedTimeout => "authenticatedTimeout", Property::AuthenticationResults => "authenticationResults", Property::AutoAddInvitations => "autoAddInvitations", - Property::AutoUpdateFrequency => "autoUpdateFrequency", + Property::AutoUpdateInterval => "autoUpdateInterval", + Property::AvailabilityRateLimit => "availabilityRateLimit", Property::BaseDn => "baseDn", - Property::BaseUrl => "baseUrl", Property::BearerToken => "bearerToken", Property::Beta => "beta", Property::Bind => "bind", @@ -1450,29 +1428,29 @@ impl EnumImpl for Property { Property::BlobSize => "blobSize", Property::BlobStore => "blobStore", Property::BlockCount => "blockCount", - Property::Body => "body", - Property::Brokers => "brokers", Property::Bucket => "bucket", Property::BufferSize => "bufferSize", Property::Buffered => "buffered", + Property::CaaIdentities => "caaIdentities", Property::CacheSize => "cacheSize", Property::Canonicalization => "canonicalization", - Property::CapacityClient => "capacityClient", - Property::CapacityReadBuffer => "capacityReadBuffer", - Property::CapacitySubscription => "capacitySubscription", Property::CatchAllAddress => "catchAllAddress", - Property::Categories => "categories", Property::Certificate => "certificate", Property::CertificateManagement => "certificateManagement", Property::ChallengeType => "challengeType", - Property::ChangesMaxResults => "changesMaxResults", + Property::ChunkSize => "chunkSize", Property::Chunking => "chunking", Property::ClaimGroups => "claimGroups", Property::ClaimName => "claimName", Property::ClaimUsername => "claimUsername", Property::Cleartext => "cleartext", + Property::ClientCapacity => "clientCapacity", Property::ClientId => "clientId", Property::ClientIp => "clientIp", + Property::ClientIpSource => "clientIpSource", + Property::ClientKind => "clientKind", + Property::ClientMetadataDocuments => "clientMetadataDocuments", + Property::ClientName => "clientName", Property::ClientSecret => "clientSecret", Property::ClientToken => "clientToken", Property::ClusterFile => "clusterFile", @@ -1480,12 +1458,11 @@ impl EnumImpl for Property { Property::ColumnDescription => "columnDescription", Property::ColumnEmail => "columnEmail", Property::ColumnSecret => "columnSecret", + Property::CommandTimeout => "commandTimeout", Property::Comment => "comment", Property::CompartmentOcid => "compartmentOcid", Property::CompressionAlgorithm => "compressionAlgorithm", - Property::Concurrency => "concurrency", Property::Condition => "condition", - Property::Confidence => "confidence", Property::Config => "config", Property::ConfigName => "configName", Property::ConnectTimeout => "connectTimeout", @@ -1495,32 +1472,27 @@ impl EnumImpl for Property { Property::ContactInfo => "contactInfo", Property::Contacts => "contacts", Property::Container => "container", - Property::Content => "content", Property::ContentTypes => "contentTypes", Property::Contents => "contents", Property::Context => "context", Property::Count => "count", Property::Create => "create", Property::CreatedAt => "createdAt", - Property::CreatedBy => "createdBy", Property::CredentialId => "credentialId", Property::Credentials => "credentials", Property::CurrentSecret => "currentSecret", - Property::CustomEndpoint => "customEndpoint", - Property::CustomRegion => "customRegion", Property::CustomRule => "customRule", Property::CustomerNumber => "customerNumber", Property::Dane => "dane", Property::DataCleanupSchedule => "dataCleanupSchedule", Property::DataStore => "dataStore", Property::DataTimeout => "dataTimeout", + Property::DataTypes => "dataTypes", Property::Database => "database", Property::DatacenterId => "datacenterId", - Property::DateRangeBegin => "dateRangeBegin", Property::DateRangeEnd => "dateRangeEnd", Property::DateRangeStart => "dateRangeStart", Property::Day => "day", - Property::DeadPropertyMaxSize => "deadPropertyMaxSize", Property::DefaultAdminRoleIds => "defaultAdminRoleIds", Property::DefaultCertificateId => "defaultCertificateId", Property::DefaultDisplayName => "defaultDisplayName", @@ -1539,7 +1511,6 @@ impl EnumImpl for Property { Property::DefaultReturnPath => "defaultReturnPath", Property::DefaultSubject => "defaultSubject", Property::DefaultSubjectPrefix => "defaultSubjectPrefix", - Property::DefaultTenantRoleIds => "defaultTenantRoleIds", Property::DefaultUserRoleIds => "defaultUserRoleIds", Property::Definition => "definition", Property::Delay => "delay", @@ -1554,6 +1525,8 @@ impl EnumImpl for Property { Property::Details => "details", Property::Directory => "directory", Property::DirectoryId => "directoryId", + Property::DirectoryRecipients => "directoryRecipients", + Property::DirectoryRecipientsTtl => "directoryRecipientsTtl", Property::DisableCapabilities => "disableCapabilities", Property::DisabledPermissions => "disabledPermissions", Property::DiscardAfter => "discardAfter", @@ -1584,7 +1557,6 @@ impl EnumImpl for Property { Property::DnsMtaSts => "dnsMtaSts", Property::DnsMx => "dnsMx", Property::DnsPtr => "dnsPtr", - Property::DnsRbl => "dnsRbl", Property::DnsServer => "dnsServer", Property::DnsServerId => "dnsServerId", Property::DnsTlsa => "dnsTlsa", @@ -1592,7 +1564,6 @@ impl EnumImpl for Property { Property::DnsView => "dnsView", Property::DnsZoneFile => "dnsZoneFile", Property::DocumentId => "documentId", - Property::DocumentType => "documentType", Property::Domain => "domain", Property::DomainId => "domainId", Property::DomainLimit => "domainLimit", @@ -1601,7 +1572,6 @@ impl EnumImpl for Property { Property::Domains => "domains", Property::Dsn => "dsn", Property::Due => "due", - Property::DuplicateExpiry => "duplicateExpiry", Property::Duration => "duration", Property::EabHmacKey => "eabHmacKey", Property::EabKeyId => "eabKeyId", @@ -1614,11 +1584,7 @@ impl EnumImpl for Property { Property::EmailAddress => "emailAddress", Property::EmailAddresses => "emailAddresses", Property::EmailAddressesNegative => "emailAddressesNegative", - Property::EmailAlert => "emailAlert", - Property::EmailDomain => "emailDomain", Property::EmailLimit => "emailLimit", - Property::EmailPrefix => "emailPrefix", - Property::EmailTemplate => "emailTemplate", Property::Enable => "enable", Property::EnableAssistedDiscovery => "enableAssistedDiscovery", Property::EnableEdns => "enableEdns", @@ -1626,17 +1592,12 @@ impl EnumImpl for Property { Property::EnableLogExporter => "enableLogExporter", Property::EnableSpamFilter => "enableSpamFilter", Property::EnableSpanExporter => "enableSpanExporter", - Property::Enabled => "enabled", Property::EnabledPermissions => "enabledPermissions", Property::EncryptAtRest => "encryptAtRest", Property::EncryptOnAppend => "encryptOnAppend", Property::EncryptionAtRest => "encryptionAtRest", Property::EncryptionKey => "encryptionKey", - Property::Endpoint => "endpoint", - Property::EnvFrom => "envFrom", - Property::EnvFromParameters => "envFromParameters", Property::EnvId => "envId", - Property::EnvRcptTo => "envRcptTo", Property::EnvelopeFrom => "envelopeFrom", Property::EnvelopeTo => "envelopeTo", Property::Environment => "environment", @@ -1648,11 +1609,9 @@ impl EnumImpl for Property { Property::EvaluatedDkim => "evaluatedDkim", Property::EvaluatedSpf => "evaluatedSpf", Property::Event => "event", - Property::EventAlert => "eventAlert", Property::EventEnd => "eventEnd", Property::EventEndTz => "eventEndTz", Property::EventId => "eventId", - Property::EventMessage => "eventMessage", Property::EventSourceThrottle => "eventSourceThrottle", Property::EventStart => "eventStart", Property::EventStartTz => "eventStartTz", @@ -1671,7 +1630,6 @@ impl EnumImpl for Property { Property::ExpungeTrashAfter => "expungeTrashAfter", Property::Extension => "extension", Property::Extensions => "extensions", - Property::ExternalId => "externalId", Property::ExtraContactInfo => "extraContactInfo", Property::Factor => "factor", Property::FailOnTimeout => "failOnTimeout", @@ -1696,43 +1654,47 @@ impl EnumImpl for Property { Property::FilePath => "filePath", Property::Files => "files", Property::FilterLogin => "filterLogin", - Property::FilterMailbox => "filterMailbox", Property::FilterMemberOf => "filterMemberOf", + Property::FilterRecipient => "filterRecipient", Property::Fingerprint => "fingerprint", + Property::FirstFailedAt => "firstFailedAt", + Property::FirstParty => "firstParty", Property::Flags => "flags", Property::FlagsAction => "flagsAction", Property::FlagsProtocol => "flagsProtocol", + Property::FlushChanges => "flushChanges", + Property::FlushInterval => "flushInterval", Property::FolderId => "folderId", - Property::ForDomain => "forDomain", Property::Format => "format", + Property::ForwardedHops => "forwardedHops", Property::From => "from", Property::FromAddress => "fromAddress", - Property::FromEmail => "fromEmail", Property::FromName => "fromName", Property::FutureRelease => "futureRelease", Property::GenerateDkimKeys => "generateDkimKeys", + Property::Generation => "generation", Property::Generator => "generator", Property::GeoUrls => "geoUrls", - Property::GetMaxResults => "getMaxResults", + Property::GrantTypes => "grantTypes", Property::GreetingTimeout => "greetingTimeout", Property::GreylistFor => "greylistFor", Property::GroupClass => "groupClass", Property::GroupId => "groupId", + Property::GroupwareLarge => "groupwareLarge", + Property::GroupwareLargeTtl => "groupwareLargeTtl", Property::HeaderFrom => "headerFrom", Property::Headers => "headers", - Property::HoldMetricsFor => "holdMetricsFor", Property::HoldMtaReportsFor => "holdMtaReportsFor", Property::HoldSamplesFor => "holdSamplesFor", - Property::HoldTracesFor => "holdTracesFor", Property::Host => "host", Property::HostedZoneId => "hostedZoneId", Property::Hostname => "hostname", + Property::Hosts => "hosts", Property::Hour => "hour", Property::HttpAuth => "httpAuth", Property::HttpHeaders => "httpHeaders", Property::HttpRsvpEnable => "httpRsvpEnable", Property::HttpRsvpLinkExpiry => "httpRsvpLinkExpiry", - Property::HttpRsvpTemplate => "httpRsvpTemplate", Property::HttpRsvpUrl => "httpRsvpUrl", Property::HttpRua => "httpRua", Property::HumanResult => "humanResult", @@ -1740,15 +1702,19 @@ impl EnumImpl for Property { Property::Id => "id", Property::IdTokenExpiry => "idTokenExpiry", Property::IdentityAlignment => "identityAlignment", + Property::IdleTimeout => "idleTimeout", Property::If => "if", + Property::ImapServerAdmin => "imapServerAdmin", + Property::ImapServerComment => "imapServerComment", Property::ImpersonateServiceAccount => "impersonateServiceAccount", Property::ImplicitTls => "implicitTls", Property::InMemoryStore => "inMemoryStore", Property::InboundReportAddresses => "inboundReportAddresses", Property::InboundReportForwarding => "inboundReportForwarding", - Property::InboundReportMaxSize => "inboundReportMaxSize", Property::Incidents => "incidents", + Property::IncludeSecrets => "includeSecrets", Property::IncludeSource => "includeSource", + Property::Index => "index", Property::IndexAsn => "indexAsn", Property::IndexAsnName => "indexAsnName", Property::IndexBatchSize => "indexBatchSize", @@ -1759,14 +1725,17 @@ impl EnumImpl for Property { Property::IndexCountry => "indexCountry", Property::IndexEmail => "indexEmail", Property::IndexEmailFields => "indexEmailFields", + Property::IndexFiles => "indexFiles", Property::IndexKey => "indexKey", - Property::IndexTelemetry => "indexTelemetry", - Property::IndexTracingFields => "indexTracingFields", Property::IndexValue => "indexValue", Property::IndicatorParameters => "indicatorParameters", Property::InitialDelay => "initialDelay", + Property::InstanceHashes => "instanceHashes", Property::Interval => "interval", Property::Intervals => "intervals", + Property::IntrospectionAuthMethod => "introspectionAuthMethod", + Property::IntrospectionClientId => "introspectionClientId", + Property::IntrospectionClientSecret => "introspectionClientSecret", Property::IpLimit => "ipLimit", Property::IpLookupStrategy => "ipLookupStrategy", Property::IpRevPtr => "ipRevPtr", @@ -1774,7 +1743,6 @@ impl EnumImpl for Property { Property::IsActive => "isActive", Property::IsArchive => "isArchive", Property::IsBad => "isBad", - Property::IsEnabled => "isEnabled", Property::IsFromOrganizer => "isFromOrganizer", Property::IsGlobPattern => "isGlobPattern", Property::IsGzipped => "isGzipped", @@ -1783,80 +1751,111 @@ impl EnumImpl for Property { Property::IsSpam => "isSpam", Property::IsTls => "isTls", Property::Issuer => "issuer", + Property::IssuerDomainNames => "issuerDomainNames", Property::IssuerUrl => "issuerUrl", - Property::ItipMaxSize => "itipMaxSize", Property::Jitter => "jitter", Property::Key => "key", Property::KeyName => "keyName", Property::KeyPrefix => "keyPrefix", - Property::KeyValues => "keyValues", Property::L1Ratio => "l1Ratio", Property::L2Ratio => "l2Ratio", Property::LastRenewal => "lastRenewal", + Property::LastUsedAt => "lastUsedAt", Property::LearnHamFromCard => "learnHamFromCard", Property::LearnHamFromReply => "learnHamFromReply", Property::LearnSpamFromRblHits => "learnSpamFromRblHits", Property::LearnSpamFromTraps => "learnSpamFromTraps", + Property::LenientIssuerMatch => "lenientIssuerMatch", Property::Level => "level", - Property::LicenseKey => "licenseKey", Property::Line => "line", Property::ListenerIds => "listenerIds", Property::Listeners => "listeners", - Property::LivePropertyMaxSize => "livePropertyMaxSize", + Property::LocalIp => "localIp", + Property::LocalIps => "localIps", Property::Locale => "locale", Property::Logo => "logo", - Property::LogoUrl => "logoUrl", Property::LoiterBanPeriod => "loiterBanPeriod", Property::LoiterBanRate => "loiterBanRate", Property::Lossy => "lossy", Property::MachineId => "machineId", Property::MailExchangers => "mailExchangers", Property::MailFrom => "mailFrom", + Property::MailFromParameters => "mailFromParameters", Property::MailFromTimeout => "mailFromTimeout", Property::MailRua => "mailRua", Property::MailingLists => "mailingLists", Property::MaintenanceType => "maintenanceType", Property::ManagedZone => "managedZone", Property::Match => "match", + Property::MaxAccountSize => "maxAccountSize", Property::MaxAddressBooks => "maxAddressBooks", + Property::MaxAddressBooksPerCard => "maxAddressBooksPerCard", Property::MaxAge => "maxAge", + Property::MaxAggregateReportSize => "maxAggregateReportSize", Property::MaxAllowedPacket => "maxAllowedPacket", Property::MaxApiKeys => "maxApiKeys", Property::MaxAppPasswords => "maxAppPasswords", Property::MaxAttachmentSize => "maxAttachmentSize", + Property::MaxAttachmentText => "maxAttachmentText", + Property::MaxAttachmentsSize => "maxAttachmentsSize", Property::MaxAttempts => "maxAttempts", Property::MaxAttendees => "maxAttendees", Property::MaxAuthFailures => "maxAuthFailures", + Property::MaxAvailabilityDuration => "maxAvailabilityDuration", + Property::MaxBodyText => "maxBodyText", + Property::MaxBufferedMemory => "maxBufferedMemory", Property::MaxCalendars => "maxCalendars", + Property::MaxCalendarsPerEvent => "maxCalendarsPerEvent", Property::MaxChangesHistory => "maxChangesHistory", - Property::MaxConcurrent => "maxConcurrent", + Property::MaxChangesResults => "maxChangesResults", + Property::MaxConcurrentConnections => "maxConcurrentConnections", + Property::MaxConcurrentIndexTasks => "maxConcurrentIndexTasks", Property::MaxConcurrentRequests => "maxConcurrentRequests", Property::MaxConcurrentUploads => "maxConcurrentUploads", Property::MaxConnections => "maxConnections", Property::MaxContacts => "maxContacts", Property::MaxCpuCycles => "maxCpuCycles", Property::MaxDelay => "maxDelay", + Property::MaxDepth => "maxDepth", Property::MaxDuration => "maxDuration", Property::MaxEntries => "maxEntries", Property::MaxEntrySize => "maxEntrySize", Property::MaxEventNotifications => "maxEventNotifications", Property::MaxEvents => "maxEvents", + Property::MaxExpandedQueryDuration => "maxExpandedQueryDuration", + Property::MaxExtractArchiveEntries => "maxExtractArchiveEntries", + Property::MaxExtractDecompressedSize => "maxExtractDecompressedSize", + Property::MaxExtractDocumentSize => "maxExtractDocumentSize", + Property::MaxExtractPartSize => "maxExtractPartSize", + Property::MaxExtractParts => "maxExtractParts", + Property::MaxExtractPdfObjects => "maxExtractPdfObjects", + Property::MaxExtractRtfDepth => "maxExtractRtfDepth", + Property::MaxExtractTextSize => "maxExtractTextSize", Property::MaxFailures => "maxFailures", Property::MaxFiles => "maxFiles", + Property::MaxFlagLength => "maxFlagLength", + Property::MaxFlagsPerEmail => "maxFlagsPerEmail", Property::MaxFolders => "maxFolders", + Property::MaxGetResults => "maxGetResults", + Property::MaxHeaderCount => "maxHeaderCount", Property::MaxHeaderSize => "maxHeaderSize", Property::MaxICalendarSize => "maxICalendarSize", Property::MaxIdentities => "maxIdentities", + Property::MaxInboundReportSize => "maxInboundReportSize", Property::MaxIncludes => "maxIncludes", + Property::MaxItipSize => "maxItipSize", + Property::MaxLivePropertySize => "maxLivePropertySize", Property::MaxLocalVars => "maxLocalVars", Property::MaxLockTimeout => "maxLockTimeout", Property::MaxLocks => "maxLocks", Property::MaxMailboxDepth => "maxMailboxDepth", Property::MaxMailboxNameLength => "maxMailboxNameLength", Property::MaxMailboxes => "maxMailboxes", - Property::MaxMaskedAddresses => "maxMaskedAddresses", + Property::MaxMailboxesPerEmail => "maxMailboxesPerEmail", Property::MaxMatchVars => "maxMatchVars", + Property::MaxMediaSize => "maxMediaSize", Property::MaxMessageSize => "maxMessageSize", + Property::MaxMessageText => "maxMessageText", Property::MaxMessages => "maxMessages", Property::MaxMessagesPerCommand => "maxMessagesPerCommand", Property::MaxMessagesPerSave => "maxMessagesPerSave", @@ -1869,15 +1868,21 @@ impl EnumImpl for Property { Property::MaxNestedTests => "maxNestedTests", Property::MaxOutMessages => "maxOutMessages", Property::MaxParticipantIdentities => "maxParticipantIdentities", + Property::MaxPasswordLength => "maxPasswordLength", + Property::MaxPolls => "maxPolls", + Property::MaxPrivateSize => "maxPrivateSize", Property::MaxPublicKeys => "maxPublicKeys", + Property::MaxPushAttempts => "maxPushAttempts", Property::MaxPushSize => "maxPushSize", + Property::MaxQueryResults => "maxQueryResults", + Property::MaxQueryScan => "maxQueryScan", Property::MaxReceivedHeaders => "maxReceivedHeaders", Property::MaxRecipients => "maxRecipients", Property::MaxReconnects => "maxReconnects", Property::MaxRecurrenceExpansions => "maxRecurrenceExpansions", + Property::MaxRedirectUrls => "maxRedirectUrls", Property::MaxRedirects => "maxRedirects", Property::MaxReportSize => "maxReportSize", - Property::MaxRequestRate => "maxRequestRate", Property::MaxRequestSize => "maxRequestSize", Property::MaxResponseSize => "maxResponseSize", Property::MaxResults => "maxResults", @@ -1886,8 +1891,10 @@ impl EnumImpl for Property { Property::MaxScriptNameLength => "maxScriptNameLength", Property::MaxScriptSize => "maxScriptSize", Property::MaxScripts => "maxScripts", + Property::MaxSetObjects => "maxSetObjects", Property::MaxShares => "maxShares", Property::MaxSize => "maxSize", + Property::MaxSnippetResults => "maxSnippetResults", Property::MaxStringLength => "maxStringLength", Property::MaxSubmissions => "maxSubmissions", Property::MaxSubscriptions => "maxSubscriptions", @@ -1898,16 +1905,19 @@ impl EnumImpl for Property { Property::MaxVarNameLength => "maxVarNameLength", Property::MaxVarSize => "maxVarSize", Property::MemberGroupIds => "memberGroupIds", - Property::MemberTenantId => "memberTenantId", Property::Message => "message", Property::MessageIdHostname => "messageIdHostname", Property::MessageIds => "messageIds", + Property::MessageTimeout => "messageTimeout", Property::Messages => "messages", - Property::Metric => "metric", + Property::MessagesLarge => "messagesLarge", + Property::MessagesLargeTtl => "messagesLargeTtl", Property::Metrics => "metrics", - Property::MetricsCollectionInterval => "metricsCollectionInterval", Property::MetricsPolicy => "metricsPolicy", + Property::MinAccountSize => "minAccountSize", Property::MinHamSamples => "minHamSamples", + Property::MinPasswordLength => "minPasswordLength", + Property::MinPasswordStrength => "minPasswordStrength", Property::MinRetryWait => "minRetryWait", Property::MinSpamSamples => "minSpamSamples", Property::MinTriggerInterval => "minTriggerInterval", @@ -1915,8 +1925,6 @@ impl EnumImpl for Property { Property::Minute => "minute", Property::Mode => "mode", Property::Model => "model", - Property::ModelId => "modelId", - Property::ModelType => "modelType", Property::MtPriority => "mtPriority", Property::MtaSts => "mtaSts", Property::MtaStsTimeout => "mtaStsTimeout", @@ -1942,7 +1950,9 @@ impl EnumImpl for Property { Property::NumReplicas => "numReplicas", Property::NumShards => "numShards", Property::OauthClientId => "oauthClientId", + Property::ObjectType => "objectType", Property::OnSuccessRenewCertificate => "onSuccessRenewCertificate", + Property::OpaqueTokenValidation => "opaqueTokenValidation", Property::OpenTelemetry => "openTelemetry", Property::Options => "options", Property::Orcpt => "orcpt", @@ -1964,13 +1974,12 @@ impl EnumImpl for Property { Property::ParseLimitContact => "parseLimitContact", Property::ParseLimitEmail => "parseLimitEmail", Property::ParseLimitEvent => "parseLimitEvent", + Property::Partition => "partition", Property::Password => "password", Property::PasswordDefaultExpiry => "passwordDefaultExpiry", Property::PasswordHashAlgorithm => "passwordHashAlgorithm", - Property::PasswordMaxLength => "passwordMaxLength", - Property::PasswordMinLength => "passwordMinLength", - Property::PasswordMinStrength => "passwordMinStrength", Property::Path => "path", + Property::Pending => "pending", Property::Period => "period", Property::Permissions => "permissions", Property::PingInterval => "pingInterval", @@ -1992,7 +2001,6 @@ impl EnumImpl for Property { Property::PolicyType => "policyType", Property::PolicyVersion => "policyVersion", Property::PollInterval => "pollInterval", - Property::PollingInterval => "pollingInterval", Property::PoolMaxConnections => "poolMaxConnections", Property::PoolMinConnections => "poolMinConnections", Property::PoolRecyclingMethod => "poolRecyclingMethod", @@ -2001,6 +2009,7 @@ impl EnumImpl for Property { Property::PoolTimeoutWait => "poolTimeoutWait", Property::PoolWorkers => "poolWorkers", Property::Port => "port", + Property::PostLogoutRedirectUris => "postLogoutRedirectUris", Property::PreferredChain => "preferredChain", Property::Prefix => "prefix", Property::PreserveIntermediates => "preserveIntermediates", @@ -2008,12 +2017,12 @@ impl EnumImpl for Property { Property::PrivateKey => "privateKey", Property::PrivateKeyPassword => "privateKeyPassword", Property::PrivateKeyPem => "privateKeyPem", + Property::PrivateMetadata => "privateMetadata", Property::PrivateZone => "privateZone", Property::PrivateZoneOnly => "privateZoneOnly", Property::Profile => "profile", Property::ProjectId => "projectId", Property::Prometheus => "prometheus", - Property::Prompt => "prompt", Property::PropagationDelay => "propagationDelay", Property::PropagationTimeout => "propagationTimeout", Property::ProtectedHeaders => "protectedHeaders", @@ -2024,7 +2033,6 @@ impl EnumImpl for Property { Property::PublicKey => "publicKey", Property::PublishRecords => "publishRecords", Property::PushAttemptWait => "pushAttemptWait", - Property::PushMaxAttempts => "pushMaxAttempts", Property::PushRequestTimeout => "pushRequestTimeout", Property::PushRetryWait => "pushRetryWait", Property::PushShardsTotal => "pushShardsTotal", @@ -2032,20 +2040,22 @@ impl EnumImpl for Property { Property::PushVerifyTimeout => "pushVerifyTimeout", Property::QueryEmailAliases => "queryEmailAliases", Property::QueryLogin => "queryLogin", - Property::QueryMaxResults => "queryMaxResults", Property::QueryMemberOf => "queryMemberOf", Property::QueryRecipient => "queryRecipient", Property::QueueId => "queueId", Property::QueueName => "queueName", + Property::QueuedDeletions => "queuedDeletions", + Property::QueuedUpdates => "queuedUpdates", Property::Quotas => "quotas", Property::Rate => "rate", Property::RateLimit => "rateLimit", Property::RateLimitAnonymous => "rateLimitAnonymous", Property::RateLimitAuthenticated => "rateLimitAuthenticated", Property::Ratio => "ratio", + Property::RcptTo => "rcptTo", Property::RcptToTimeout => "rcptToTimeout", + Property::ReadBufferCapacity => "readBufferCapacity", Property::ReadFromReplicas => "readFromReplicas", - Property::ReadReplicas => "readReplicas", Property::Reason => "reason", Property::ReceivedAt => "receivedAt", Property::ReceivedFromIp => "receivedFromIp", @@ -2056,11 +2066,12 @@ impl EnumImpl for Property { Property::Recipients => "recipients", Property::Records => "records", Property::RecurrenceId => "recurrenceId", + Property::RedirectCacheTtl => "redirectCacheTtl", Property::RedirectRoot => "redirectRoot", + Property::RedirectTimeout => "redirectTimeout", Property::RedirectUris => "redirectUris", - Property::Refresh => "refresh", + Property::RefreshInterval => "refreshInterval", Property::RefreshTokenExpiry => "refreshTokenExpiry", - Property::RefreshTokenRenewal => "refreshTokenRenewal", Property::Region => "region", Property::RejectNonFqdn => "rejectNonFqdn", Property::RemoteIp => "remoteIp", @@ -2071,12 +2082,9 @@ impl EnumImpl for Property { Property::ReportedDomains => "reportedDomains", Property::ReportedUris => "reportedUris", Property::ReportingMta => "reportingMta", - Property::RequestMaxSize => "requestMaxSize", + Property::RequestTimeout => "requestTimeout", Property::RequestTlsCertificate => "requestTlsCertificate", Property::Require => "require", - Property::RequireAudience => "requireAudience", - Property::RequireClientRegistration => "requireClientRegistration", - Property::RequireScopes => "requireScopes", Property::RequireTls => "requireTls", Property::ReservoirCapacity => "reservoirCapacity", Property::ResourceGroup => "resourceGroup", @@ -2086,11 +2094,9 @@ impl EnumImpl for Property { Property::ResponseHeaders => "responseHeaders", Property::ResponseHostname => "responseHostname", Property::ResponseMessage => "responseMessage", - Property::ResponsePosCategory => "responsePosCategory", - Property::ResponsePosConfidence => "responsePosConfidence", - Property::ResponsePosExplanation => "responsePosExplanation", Property::Result => "result", Property::ResultType => "resultType", + Property::Retention => "retention", Property::RetireAfter => "retireAfter", Property::Retry => "retry", Property::RetryCount => "retryCount", @@ -2115,6 +2121,7 @@ impl EnumImpl for Property { Property::Scheduling => "scheduling", Property::Scheme => "scheme", Property::Scope => "scope", + Property::Scopes => "scopes", Property::Score => "score", Property::ScoreDiscard => "scoreDiscard", Property::ScoreReject => "scoreReject", @@ -2122,11 +2129,7 @@ impl EnumImpl for Property { Property::Script => "script", Property::SearchStore => "searchStore", Property::Secret => "secret", - Property::SecretAccessKey => "secretAccessKey", - Property::SecretApiKey => "secretApiKey", - Property::SecretId => "secretId", Property::SecretKey => "secretKey", - Property::SecurityToken => "securityToken", Property::Selector => "selector", Property::SelectorTemplate => "selectorTemplate", Property::SendFrequency => "sendFrequency", @@ -2134,25 +2137,22 @@ impl EnumImpl for Property { Property::SentinelSecret => "sentinelSecret", Property::SentinelUsername => "sentinelUsername", Property::Separator => "separator", - Property::ServerHostname => "serverHostname", Property::ServerId => "serverId", Property::Servers => "servers", Property::ServiceAccountJson => "serviceAccountJson", Property::ServiceName => "serviceName", Property::Services => "services", + Property::SessionTimeout => "sessionTimeout", Property::SessionToken => "sessionToken", - Property::SetMaxObjects => "setMaxObjects", Property::ShardIndex => "shardIndex", Property::SharedSecret => "sharedSecret", - Property::Sig0Algorithm => "sig0Algorithm", Property::SignatureAlgorithm => "signatureAlgorithm", Property::SignatureKey => "signatureKey", - Property::SignerName => "signerName", Property::Size => "size", + Property::SkipAudienceValidation => "skipAudienceValidation", Property::SkipDeploy => "skipDeploy", Property::SkipFirst => "skipFirst", Property::SmtpGreeting => "smtpGreeting", - Property::SnippetMaxResults => "snippetMaxResults", Property::SocketBacklog => "socketBacklog", Property::SocketNoDelay => "socketNoDelay", Property::SocketReceiveBufferSize => "socketReceiveBufferSize", @@ -2162,9 +2162,9 @@ impl EnumImpl for Property { Property::SocketTosV4 => "socketTosV4", Property::SocketTtl => "socketTtl", Property::SourceIp => "sourceIp", - Property::SourceIps => "sourceIps", Property::SourcePort => "sourcePort", Property::SpamFilterRulesUrl => "spamFilterRulesUrl", + Property::SpamLookup => "spamLookup", Property::SpfDns => "spfDns", Property::SpfEhloDomain => "spfEhloDomain", Property::SpfEhloResult => "spfEhloResult", @@ -2175,70 +2175,53 @@ impl EnumImpl for Property { Property::SpfResults => "spfResults", Property::Stage => "stage", Property::Stages => "stages", - Property::StartTime => "startTime", Property::StartTls => "startTls", Property::Status => "status", Property::StorageAccount => "storageAccount", Property::Store => "store", - Property::Stores => "stores", Property::Strategy => "strategy", Property::SubAddressing => "subAddressing", Property::SubAuthId => "subAuthId", Property::Subject => "subject", Property::SubjectAlternativeNames => "subjectAlternativeNames", Property::Subscribe => "subscribe", + Property::SubscriptionCapacity => "subscriptionCapacity", Property::SubscriptionId => "subscriptionId", - Property::Sum => "sum", Property::Summary => "summary", Property::SupportedLanguages => "supportedLanguages", + Property::Swap => "swap", Property::Tag => "tag", Property::Tags => "tags", + Property::TargetAccountId => "targetAccountId", Property::TaskTypes => "taskTypes", Property::Tasks => "tasks", Property::TcpOnError => "tcpOnError", Property::TeamId => "teamId", Property::TempFailOnError => "tempFailOnError", - Property::Temperature => "temperature", - Property::Template => "template", Property::TenancyOcid => "tenancyOcid", Property::TenantId => "tenantId", - Property::Tenants => "tenants", Property::Text => "text", Property::Then => "then", Property::ThirdParty => "thirdParty", Property::ThirdPartyHash => "thirdPartyHash", Property::ThreadName => "threadName", - Property::ThreadPoolSize => "threadPoolSize", Property::ThreadsPerNode => "threadsPerNode", Property::Throttle => "throttle", Property::TimeZone => "timeZone", Property::Timeout => "timeout", - Property::TimeoutAnonymous => "timeoutAnonymous", - Property::TimeoutAuthenticated => "timeoutAuthenticated", - Property::TimeoutCommand => "timeoutCommand", - Property::TimeoutConnect => "timeoutConnect", - Property::TimeoutConnection => "timeoutConnection", - Property::TimeoutData => "timeoutData", - Property::TimeoutIdle => "timeoutIdle", - Property::TimeoutMessage => "timeoutMessage", - Property::TimeoutRequest => "timeoutRequest", - Property::TimeoutSession => "timeoutSession", Property::Timestamp => "timestamp", - Property::Title => "title", Property::Tls => "tls", Property::TlsDisableCipherSuites => "tlsDisableCipherSuites", Property::TlsDisableProtocols => "tlsDisableProtocols", Property::TlsIgnoreClientOrder => "tlsIgnoreClientOrder", - Property::TlsImplicit => "tlsImplicit", Property::TlsTimeout => "tlsTimeout", Property::To => "to", Property::Token => "token", - Property::TotalDeadline => "totalDeadline", + Property::TokenFile => "tokenFile", Property::TotalFailedSessions => "totalFailedSessions", Property::TotalSuccessfulSessions => "totalSuccessfulSessions", - Property::TraceId => "traceId", Property::Tracer => "tracer", - Property::TrainFrequency => "trainFrequency", + Property::TrainInterval => "trainInterval", Property::TransactionRetryDelay => "transactionRetryDelay", Property::TransactionRetryLimit => "transactionRetryLimit", Property::TransactionTimeout => "transactionTimeout", @@ -2248,6 +2231,7 @@ impl EnumImpl for Property { Property::TsigAlgorithm => "tsigAlgorithm", Property::Ttl => "ttl", Property::UnpackDirectory => "unpackDirectory", + Property::UnscopedTokens => "unscopedTokens", Property::UpdateRecords => "updateRecords", Property::UploadQuota => "uploadQuota", Property::UploadTtl => "uploadTtl", @@ -2257,7 +2241,6 @@ impl EnumImpl for Property { Property::Urls => "urls", Property::UsePermissiveCors => "usePermissiveCors", Property::UseTls => "useTls", - Property::UseXForwarded => "useXForwarded", Property::UsedDiskQuota => "usedDiskQuota", Property::UserAgent => "userAgent", Property::UserCodeExpiry => "userCodeExpiry", @@ -2268,17 +2251,20 @@ impl EnumImpl for Property { Property::ValidateDomain => "validateDomain", Property::Value => "value", Property::VariableName => "variableName", + Property::VendorNamespaces => "vendorNamespaces", Property::VerifyAfterWrite => "verifyAfterWrite", Property::Version => "version", Property::ViewName => "viewName", Property::Vrfy => "vrfy", Property::WaitOnFail => "waitOnFail", Property::WapiVersion => "wapiVersion", + Property::Warnings => "warnings", Property::WebPushContact => "webPushContact", Property::WebPushKey => "webPushKey", Property::WebsocketHeartbeat => "websocketHeartbeat", Property::WebsocketThrottle => "websocketThrottle", Property::WebsocketTimeout => "websocketTimeout", + Property::WindowEnd => "windowEnd", Property::Zone => "zone", Property::ZoneIpV4 => "zoneIpV4", Property::ZoneIpV6 => "zoneIpV6", @@ -2292,945 +2278,947 @@ impl EnumImpl for Property { fn from_id(id: u16) -> Option { match id { 0 => Some(Property::Type), - 678 => Some(Property::AbuseBanPeriod), - 677 => Some(Property::AbuseBanRate), - 118 => Some(Property::AccessKey), - 327 => Some(Property::AccessKeyId), - 890 => Some(Property::AccessToken), - 619 => Some(Property::AccessTokenExpiry), - 132 => Some(Property::AccessTokens), - 810 => Some(Property::AccountDomainId), - 57 => Some(Property::AccountId), - 315 => Some(Property::AccountIdentifier), + 792 => Some(Property::AbuseBanPeriod), + 791 => Some(Property::AbuseBanRate), + 710 => Some(Property::AcceptedAudiences), + 354 => Some(Property::AccessKeyId), + 378 => Some(Property::AccessToken), + 721 => Some(Property::AccessTokenExpiry), + 159 => Some(Property::AccessTokens), + 951 => Some(Property::AccountDomainId), + 17 => Some(Property::AccountHashPrefix), + 71 => Some(Property::AccountId), + 374 => Some(Property::AccountIdentifier), 15 => Some(Property::AccountKey), - 809 => Some(Property::AccountName), - 891 => Some(Property::AccountSwitchKey), - 811 => Some(Property::AccountType), + 950 => Some(Property::AccountName), + 379 => Some(Property::AccountSwitchKey), + 952 => Some(Property::AccountType), 16 => Some(Property::AccountUri), - 151 => Some(Property::Accounts), - 182 => Some(Property::AcmeProviderId), - 554 => Some(Property::AddAuthResultsHeader), - 555 => Some(Property::AddDateHeader), - 556 => Some(Property::AddDeliveredToHeader), - 557 => Some(Property::AddMessageIdHeader), - 558 => Some(Property::AddReceivedHeader), - 559 => Some(Property::AddReceivedSpfHeader), - 560 => Some(Property::AddReturnPathHeader), - 838 => Some(Property::AdditionalInformation), - 44 => Some(Property::Address), - 579 => Some(Property::Addresses), - 268 => Some(Property::AggregateContactInfo), - 274 => Some(Property::AggregateDkimSignDomain), - 269 => Some(Property::AggregateFromAddress), - 270 => Some(Property::AggregateFromName), - 271 => Some(Property::AggregateMaxReportSize), - 272 => Some(Property::AggregateOrgName), - 273 => Some(Property::AggregateSendFrequency), - 275 => Some(Property::AggregateSubject), - 798 => Some(Property::AlarmId), - 225 => Some(Property::Algorithms), - 339 => Some(Property::Aliases), - 768 => Some(Property::AllowCount), - 695 => Some(Property::AllowDirectoryQueries), - 164 => Some(Property::AllowExternalRcpts), - 26 => Some(Property::AllowInvalidCerts), - 424 => Some(Property::AllowPlainTextAuth), - 348 => Some(Property::AllowRelaying), - 932 => Some(Property::AllowScimProvisioning), - 369 => Some(Property::AllowSpamTraining), - 398 => Some(Property::AllowedEndpoints), - 49 => Some(Property::AllowedIps), - 712 => Some(Property::AllowedNotifyUris), - 388 => Some(Property::Alpha), - 614 => Some(Property::AnonymousClientRegistration), - 858 => Some(Property::Ansi), - 325 => Some(Property::ApiKey), - 892 => Some(Property::ApiUser), - 321 => Some(Property::ApplicationKey), - 322 => Some(Property::ApplicationSecret), - 292 => Some(Property::ArcResult), - 690 => Some(Property::ArcVerify), - 203 => Some(Property::ArchiveDeletedAccountsFor), - 202 => Some(Property::ArchiveDeletedItemsFor), - 58 => Some(Property::ArchivedAt), - 820 => Some(Property::ArchivedItemType), - 59 => Some(Property::ArchivedUntil), - 68 => Some(Property::ArrivalDate), - 102 => Some(Property::AsnUrls), - 829 => Some(Property::AttemptNumber), - 303 => Some(Property::Attempts), - 470 => Some(Property::AttrClass), - 471 => Some(Property::AttrDescription), - 472 => Some(Property::AttrEmail), - 473 => Some(Property::AttrEmailAlias), - 474 => Some(Property::AttrMemberOf), - 475 => Some(Property::AttrSecret), - 476 => Some(Property::AttrSecretChanged), - 215 => Some(Property::Auid), - 897 => Some(Property::Auth), - 680 => Some(Property::AuthBanPeriod), - 679 => Some(Property::AuthBanRate), - 616 => Some(Property::AuthCodeExpiry), - 613 => Some(Property::AuthCodeMaxAttempts), - 81 => Some(Property::AuthFailure), - 886 => Some(Property::AuthId), - 501 => Some(Property::AuthSecret), - 314 => Some(Property::AuthToken), - 502 => Some(Property::AuthUsername), - 740 => Some(Property::AuthenticatedAs), - 69 => Some(Property::AuthenticationResults), - 171 => Some(Property::AutoAddInvitations), - 53 => Some(Property::AutoUpdateFrequency), - 463 => Some(Property::BaseDn), - 882 => Some(Property::BaseUrl), - 403 => Some(Property::BearerToken), - 389 => Some(Property::Beta), - 589 => Some(Property::Bind), - 466 => Some(Property::BindAuthentication), - 464 => Some(Property::BindDn), - 465 => Some(Property::BindSecret), - 200 => Some(Property::BlobCleanupSchedule), - 60 => Some(Property::BlobId), - 655 => Some(Property::BlobSize), - 126 => Some(Property::BlobStore), - 766 => Some(Property::BlockCount), - 38 => Some(Property::Body), - 459 => Some(Property::Brokers), - 658 => Some(Property::Bucket), - 656 => Some(Property::BufferSize), - 863 => Some(Property::Buffered), - 929 => Some(Property::CacheSize), - 216 => Some(Property::Canonicalization), - 584 => Some(Property::CapacityClient), - 585 => Some(Property::CapacityReadBuffer), - 586 => Some(Property::CapacitySubscription), - 346 => Some(Property::CatchAllAddress), - 759 => Some(Property::Categories), - 176 => Some(Property::Certificate), - 342 => Some(Property::CertificateManagement), + 182 => Some(Property::Accounts), + 220 => Some(Property::AcmeProviderId), + 649 => Some(Property::AddAuthResultsHeader), + 650 => Some(Property::AddDateHeader), + 651 => Some(Property::AddDeliveredToHeader), + 652 => Some(Property::AddMessageIdHeader), + 653 => Some(Property::AddReceivedHeader), + 654 => Some(Property::AddReceivedSpfHeader), + 655 => Some(Property::AddReturnPathHeader), + 980 => Some(Property::AdditionalInformation), + 58 => Some(Property::Address), + 311 => Some(Property::AggregateContactInfo), + 317 => Some(Property::AggregateDkimSignDomain), + 312 => Some(Property::AggregateFromAddress), + 313 => Some(Property::AggregateFromName), + 315 => Some(Property::AggregateOrgName), + 316 => Some(Property::AggregateSendFrequency), + 938 => Some(Property::AlarmId), + 957 => Some(Property::AlertAfter), + 265 => Some(Property::Algorithms), + 411 => Some(Property::Aliases), + 903 => Some(Property::AllowCount), + 809 => Some(Property::AllowDirectoryQueries), + 203 => Some(Property::AllowExternalRcpts), + 717 => Some(Property::AllowIdTokenClaims), + 41 => Some(Property::AllowInvalidCerts), + 502 => Some(Property::AllowPlainTextAuth), + 419 => Some(Property::AllowRelaying), + 446 => Some(Property::AllowSpamTraining), + 475 => Some(Property::AllowedEndpoints), + 63 => Some(Property::AllowedIps), + 824 => Some(Property::AllowedNotifyUris), + 464 => Some(Property::Alpha), + 506 => Some(Property::AnonymousTimeout), + 999 => Some(Property::Ansi), + 39 => Some(Property::ApiKey), + 372 => Some(Property::ApiKeyId), + 390 => Some(Property::ApiUser), + 399 => Some(Property::ApplicationKey), + 400 => Some(Property::ApplicationSecret), + 337 => Some(Property::ArcResult), + 804 => Some(Property::ArcVerify), + 82 => Some(Property::ArrivalDate), + 116 => Some(Property::AsnUrls), + 971 => Some(Property::AttemptNumber), + 348 => Some(Property::Attempts), + 559 => Some(Property::AttrClass), + 560 => Some(Property::AttrDescription), + 561 => Some(Property::AttrEmail), + 562 => Some(Property::AttrEmailAlias), + 563 => Some(Property::AttrMemberOf), + 564 => Some(Property::AttrSecret), + 565 => Some(Property::AttrSecretChanged), + 253 => Some(Property::Auid), + 139 => Some(Property::Auth), + 794 => Some(Property::AuthBanPeriod), + 793 => Some(Property::AuthBanRate), + 719 => Some(Property::AuthCodeExpiry), + 95 => Some(Property::AuthFailure), + 370 => Some(Property::AuthId), + 597 => Some(Property::AuthSecret), + 598 => Some(Property::AuthUsername), + 882 => Some(Property::AuthenticatedAs), + 507 => Some(Property::AuthenticatedTimeout), + 83 => Some(Property::AuthenticationResults), + 209 => Some(Property::AutoAddInvitations), + 66 => Some(Property::AutoUpdateInterval), + 201 => Some(Property::AvailabilityRateLimit), + 552 => Some(Property::BaseDn), + 482 => Some(Property::BearerToken), + 465 => Some(Property::Beta), + 681 => Some(Property::Bind), + 555 => Some(Property::BindAuthentication), + 553 => Some(Property::BindDn), + 554 => Some(Property::BindSecret), + 238 => Some(Property::BlobCleanupSchedule), + 74 => Some(Property::BlobId), + 763 => Some(Property::BlobSize), + 153 => Some(Property::BlobStore), + 901 => Some(Property::BlockCount), + 767 => Some(Property::Bucket), + 764 => Some(Property::BufferSize), + 1004 => Some(Property::Buffered), + 19 => Some(Property::CaaIdentities), + 766 => Some(Property::CacheSize), + 254 => Some(Property::Canonicalization), + 417 => Some(Property::CatchAllAddress), + 214 => Some(Property::Certificate), + 413 => Some(Property::CertificateManagement), 10 => Some(Property::ChallengeType), - 435 => Some(Property::ChangesMaxResults), - 517 => Some(Property::Chunking), - 612 => Some(Property::ClaimGroups), - 611 => Some(Property::ClaimName), - 609 => Some(Property::ClaimUsername), - 693 => Some(Property::Cleartext), - 604 => Some(Property::ClientId), - 898 => Some(Property::ClientIp), - 878 => Some(Property::ClientSecret), - 889 => Some(Property::ClientToken), - 382 => Some(Property::ClusterFile), - 781 => Some(Property::ColumnClass), - 782 => Some(Property::ColumnDescription), - 779 => Some(Property::ColumnEmail), - 780 => Some(Property::ColumnSecret), - 240 => Some(Property::Comment), - 905 => Some(Property::CompartmentOcid), - 359 => Some(Property::CompressionAlgorithm), - 304 => Some(Property::Concurrency), - 34 => Some(Property::Condition), - 760 => Some(Property::Confidence), - 873 => Some(Property::Config), - 883 => Some(Property::ConfigName), - 505 => Some(Property::ConnectTimeout), - 539 => Some(Property::Connection), - 323 => Some(Property::ConsumerKey), + 756 => Some(Property::ChunkSize), + 614 => Some(Property::Chunking), + 709 => Some(Property::ClaimGroups), + 708 => Some(Property::ClaimName), + 706 => Some(Property::ClaimUsername), + 807 => Some(Property::Cleartext), + 677 => Some(Property::ClientCapacity), + 132 => Some(Property::ClientId), + 391 => Some(Property::ClientIp), + 479 => Some(Property::ClientIpSource), + 701 => Some(Property::ClientKind), + 727 => Some(Property::ClientMetadataDocuments), + 700 => Some(Property::ClientName), + 133 => Some(Property::ClientSecret), + 377 => Some(Property::ClientToken), + 458 => Some(Property::ClusterFile), + 921 => Some(Property::ColumnClass), + 922 => Some(Property::ColumnDescription), + 919 => Some(Property::ColumnEmail), + 920 => Some(Property::ColumnSecret), + 631 => Some(Property::CommandTimeout), + 280 => Some(Property::Comment), + 398 => Some(Property::CompartmentOcid), + 431 => Some(Property::CompressionAlgorithm), + 48 => Some(Property::Condition), + 1011 => Some(Property::Config), + 367 => Some(Property::ConfigName), + 602 => Some(Property::ConnectTimeout), + 634 => Some(Property::Connection), + 401 => Some(Property::ConsumerKey), 11 => Some(Property::Contact), - 844 => Some(Property::ContactInfo), - 133 => Some(Property::Contacts), - 117 => Some(Property::Container), - 65 => Some(Property::Content), - 758 => Some(Property::ContentTypes), - 708 => Some(Property::Contents), - 877 => Some(Property::Context), - 258 => Some(Property::Count), - 367 => Some(Property::Create), - 46 => Some(Property::CreatedAt), - 486 => Some(Property::CreatedBy), - 627 => Some(Property::CredentialId), - 588 => Some(Property::Credentials), + 986 => Some(Property::ContactInfo), + 160 => Some(Property::Contacts), + 137 => Some(Property::Container), + 900 => Some(Property::ContentTypes), + 79 => Some(Property::Contents), + 364 => Some(Property::Context), + 301 => Some(Property::Count), + 444 => Some(Property::Create), + 60 => Some(Property::CreatedAt), + 730 => Some(Property::CredentialId), + 385 => Some(Property::Credentials), 4 => Some(Property::CurrentSecret), - 662 => Some(Property::CustomEndpoint), - 663 => Some(Property::CustomRegion), - 787 => Some(Property::CustomRule), - 899 => Some(Property::CustomerNumber), - 569 => Some(Property::Dane), - 199 => Some(Property::DataCleanupSchedule), - 125 => Some(Property::DataStore), - 506 => Some(Property::DataTimeout), - 575 => Some(Property::Database), - 383 => Some(Property::DatacenterId), - 245 => Some(Property::DateRangeBegin), - 246 => Some(Property::DateRangeEnd), - 845 => Some(Property::DateRangeStart), - 192 => Some(Property::Day), - 868 => Some(Property::DeadPropertyMaxSize), - 108 => Some(Property::DefaultAdminRoleIds), - 790 => Some(Property::DefaultCertificateId), - 20 => Some(Property::DefaultDisplayName), - 122 => Some(Property::DefaultDomain), - 789 => Some(Property::DefaultDomainId), - 709 => Some(Property::DefaultExpiryDuplicate), - 710 => Some(Property::DefaultExpiryVacation), - 360 => Some(Property::DefaultFolders), - 405 => Some(Property::DefaultFromAddress), - 697 => Some(Property::DefaultFromName), - 106 => Some(Property::DefaultGroupRoleIds), - 788 => Some(Property::DefaultHostname), - 21 => Some(Property::DefaultHrefName), - 665 => Some(Property::DefaultLanguage), - 408 => Some(Property::DefaultName), - 701 => Some(Property::DefaultReturnPath), - 411 => Some(Property::DefaultSubject), - 714 => Some(Property::DefaultSubjectPrefix), - 107 => Some(Property::DefaultTenantRoleIds), - 105 => Some(Property::DefaultUserRoleIds), - 235 => Some(Property::Definition), - 825 => Some(Property::Delay), - 229 => Some(Property::DeleteAfter), - 777 => Some(Property::DeleteAfterUse), - 238 => Some(Property::DeliverAt), - 518 => Some(Property::DeliverBy), - 404 => Some(Property::DeliverTo), - 82 => Some(Property::DeliveryResult), - 381 => Some(Property::Depth), + 927 => Some(Property::CustomRule), + 392 => Some(Property::CustomerNumber), + 664 => Some(Property::Dane), + 237 => Some(Property::DataCleanupSchedule), + 152 => Some(Property::DataStore), + 603 => Some(Property::DataTimeout), + 582 => Some(Property::DataTypes), + 670 => Some(Property::Database), + 459 => Some(Property::DatacenterId), + 286 => Some(Property::DateRangeEnd), + 285 => Some(Property::DateRangeStart), + 230 => Some(Property::Day), + 123 => Some(Property::DefaultAdminRoleIds), + 929 => Some(Property::DefaultCertificateId), + 24 => Some(Property::DefaultDisplayName), + 149 => Some(Property::DefaultDomain), + 928 => Some(Property::DefaultDomainId), + 821 => Some(Property::DefaultExpiryDuplicate), + 822 => Some(Property::DefaultExpiryVacation), + 432 => Some(Property::DefaultFolders), + 484 => Some(Property::DefaultFromAddress), + 811 => Some(Property::DefaultFromName), + 121 => Some(Property::DefaultGroupRoleIds), + 148 => Some(Property::DefaultHostname), + 25 => Some(Property::DefaultHrefName), + 770 => Some(Property::DefaultLanguage), + 487 => Some(Property::DefaultName), + 814 => Some(Property::DefaultReturnPath), + 489 => Some(Property::DefaultSubject), + 826 => Some(Property::DefaultSubjectPrefix), + 120 => Some(Property::DefaultUserRoleIds), + 275 => Some(Property::Definition), + 966 => Some(Property::Delay), + 269 => Some(Property::DeleteAfter), + 918 => Some(Property::DeleteAfterUse), + 278 => Some(Property::DeliverAt), + 615 => Some(Property::DeliverBy), + 483 => Some(Property::DeliverTo), + 96 => Some(Property::DeliveryResult), + 457 => Some(Property::Depth), 6 => Some(Property::Description), - 297 => Some(Property::Details), + 342 => Some(Property::Details), 12 => Some(Property::Directory), - 104 => Some(Property::DirectoryId), - 711 => Some(Property::DisableCapabilities), - 629 => Some(Property::DisabledPermissions), - 872 => Some(Property::DiscardAfter), - 747 => Some(Property::Disposition), - 917 => Some(Property::Dkim2Pass), - 916 => Some(Property::Dkim2Result), - 83 => Some(Property::DkimAdspDns), - 84 => Some(Property::DkimCanonicalizedBody), - 85 => Some(Property::DkimCanonicalizedHeader), - 86 => Some(Property::DkimDomain), - 87 => Some(Property::DkimIdentity), - 343 => Some(Property::DkimManagement), - 291 => Some(Property::DkimPass), - 266 => Some(Property::DkimResults), - 88 => Some(Property::DkimSelector), - 89 => Some(Property::DkimSelectorDns), - 231 => Some(Property::DkimSignDomain), - 155 => Some(Property::DkimSignatures), - 686 => Some(Property::DkimStrict), - 687 => Some(Property::DkimVerify), - 294 => Some(Property::DmarcPass), - 295 => Some(Property::DmarcPolicy), - 293 => Some(Property::DmarcResult), - 691 => Some(Property::DmarcVerify), - 134 => Some(Property::DnsIpv4), - 135 => Some(Property::DnsIpv6), - 344 => Some(Property::DnsManagement), - 136 => Some(Property::DnsMtaSts), - 137 => Some(Property::DnsMx), - 138 => Some(Property::DnsPtr), - 139 => Some(Property::DnsRbl), - 130 => Some(Property::DnsServer), - 300 => Some(Property::DnsServerId), - 140 => Some(Property::DnsTlsa), - 141 => Some(Property::DnsTxt), - 894 => Some(Property::DnsView), - 345 => Some(Property::DnsZoneFile), - 804 => Some(Property::DocumentId), - 814 => Some(Property::DocumentType), - 232 => Some(Property::Domain), - 221 => Some(Property::DomainId), - 750 => Some(Property::DomainLimit), - 147 => Some(Property::DomainNames), - 148 => Some(Property::DomainNamesNegative), - 146 => Some(Property::Domains), - 519 => Some(Property::Dsn), - 797 => Some(Property::Due), - 699 => Some(Property::DuplicateExpiry), - 515 => Some(Property::Duration), + 119 => Some(Property::DirectoryId), + 188 => Some(Property::DirectoryRecipients), + 189 => Some(Property::DirectoryRecipientsTtl), + 823 => Some(Property::DisableCapabilities), + 732 => Some(Property::DisabledPermissions), + 1010 => Some(Property::DiscardAfter), + 890 => Some(Property::Disposition), + 336 => Some(Property::Dkim2Pass), + 335 => Some(Property::Dkim2Result), + 97 => Some(Property::DkimAdspDns), + 98 => Some(Property::DkimCanonicalizedBody), + 99 => Some(Property::DkimCanonicalizedHeader), + 100 => Some(Property::DkimDomain), + 101 => Some(Property::DkimIdentity), + 414 => Some(Property::DkimManagement), + 334 => Some(Property::DkimPass), + 309 => Some(Property::DkimResults), + 102 => Some(Property::DkimSelector), + 103 => Some(Property::DkimSelectorDns), + 271 => Some(Property::DkimSignDomain), + 186 => Some(Property::DkimSignatures), + 800 => Some(Property::DkimStrict), + 801 => Some(Property::DkimVerify), + 339 => Some(Property::DmarcPass), + 340 => Some(Property::DmarcPolicy), + 338 => Some(Property::DmarcResult), + 805 => Some(Property::DmarcVerify), + 161 => Some(Property::DnsIpv4), + 162 => Some(Property::DnsIpv6), + 415 => Some(Property::DnsManagement), + 163 => Some(Property::DnsMtaSts), + 164 => Some(Property::DnsMx), + 165 => Some(Property::DnsPtr), + 157 => Some(Property::DnsServer), + 345 => Some(Property::DnsServerId), + 167 => Some(Property::DnsTlsa), + 168 => Some(Property::DnsTxt), + 387 => Some(Property::DnsView), + 416 => Some(Property::DnsZoneFile), + 946 => Some(Property::DocumentId), + 272 => Some(Property::Domain), + 259 => Some(Property::DomainId), + 892 => Some(Property::DomainLimit), + 178 => Some(Property::DomainNames), + 179 => Some(Property::DomainNamesNegative), + 177 => Some(Property::Domains), + 616 => Some(Property::Dsn), + 937 => Some(Property::Due), + 612 => Some(Property::Duration), 13 => Some(Property::EabHmacKey), 14 => Some(Property::EabKeyId), - 283 => Some(Property::EhloDomain), - 503 => Some(Property::EhloHostname), - 507 => Some(Property::EhloTimeout), - 296 => Some(Property::Elapsed), - 375 => Some(Property::Else), - 242 => Some(Property::Email), - 393 => Some(Property::EmailAddress), - 149 => Some(Property::EmailAddresses), - 150 => Some(Property::EmailAddressesNegative), - 35 => Some(Property::EmailAlert), - 488 => Some(Property::EmailDomain), - 751 => Some(Property::EmailLimit), - 487 => Some(Property::EmailPrefix), - 174 => Some(Property::EmailTemplate), - 37 => Some(Property::Enable), - 865 => Some(Property::EnableAssistedDiscovery), - 305 => Some(Property::EnableEdns), - 399 => Some(Property::EnableHsts), - 860 => Some(Property::EnableLogExporter), - 562 => Some(Property::EnableSpamFilter), - 861 => Some(Property::EnableSpanExporter), - 50 => Some(Property::Enabled), - 628 => Some(Property::EnabledPermissions), - 358 => Some(Property::EncryptAtRest), - 357 => Some(Property::EncryptOnAppend), + 325 => Some(Property::EhloDomain), + 599 => Some(Property::EhloHostname), + 604 => Some(Property::EhloTimeout), + 341 => Some(Property::Elapsed), + 451 => Some(Property::Else), + 282 => Some(Property::Email), + 469 => Some(Property::EmailAddress), + 180 => Some(Property::EmailAddresses), + 181 => Some(Property::EmailAddressesNegative), + 893 => Some(Property::EmailLimit), + 51 => Some(Property::Enable), + 1005 => Some(Property::EnableAssistedDiscovery), + 349 => Some(Property::EnableEdns), + 476 => Some(Property::EnableHsts), + 1001 => Some(Property::EnableLogExporter), + 657 => Some(Property::EnableSpamFilter), + 1002 => Some(Property::EnableSpanExporter), + 731 => Some(Property::EnabledPermissions), + 430 => Some(Property::EncryptAtRest), + 429 => Some(Property::EncryptOnAppend), 9 => Some(Property::EncryptionAtRest), - 622 => Some(Property::EncryptionKey), - 499 => Some(Property::Endpoint), - 742 => Some(Property::EnvFrom), - 743 => Some(Property::EnvFromParameters), - 639 => Some(Property::EnvId), - 744 => Some(Property::EnvRcptTo), - 264 => Some(Property::EnvelopeFrom), - 263 => Some(Property::EnvelopeTo), - 881 => Some(Property::Environment), - 210 => Some(Property::ErrorCommand), - 209 => Some(Property::ErrorMessage), - 208 => Some(Property::ErrorType), - 247 => Some(Property::Errors), - 259 => Some(Property::EvaluatedDisposition), - 260 => Some(Property::EvaluatedDkim), - 261 => Some(Property::EvaluatedSpf), - 372 => Some(Property::Event), - 36 => Some(Property::EventAlert), - 801 => Some(Property::EventEnd), - 803 => Some(Property::EventEndTz), - 799 => Some(Property::EventId), - 43 => Some(Property::EventMessage), - 447 => Some(Property::EventSourceThrottle), - 800 => Some(Property::EventStart), - 802 => Some(Property::EventStartTz), - 142 => Some(Property::Events), - 855 => Some(Property::EventsPolicy), - 217 => Some(Property::Expire), - 100 => Some(Property::Expires), - 47 => Some(Property::ExpiresAt), - 632 => Some(Property::ExpiresAttempts), - 512 => Some(Property::Expiry), - 520 => Some(Property::Expn), - 198 => Some(Property::ExpungeSchedule), - 197 => Some(Property::ExpungeSchedulingInboxAfter), - 196 => Some(Property::ExpungeShareNotifyAfter), - 195 => Some(Property::ExpungeSubmissionsAfter), - 194 => Some(Property::ExpungeTrashAfter), - 754 => Some(Property::Extension), - 257 => Some(Property::Extensions), - 933 => Some(Property::ExternalId), - 243 => Some(Property::ExtraContactInfo), - 821 => Some(Property::Factor), - 490 => Some(Property::FailOnTimeout), - 826 => Some(Property::FailedAt), - 827 => Some(Property::FailedAttemptNumber), - 837 => Some(Property::FailedSessionCount), - 851 => Some(Property::FailureDetails), - 279 => Some(Property::FailureDkimSignDomain), - 276 => Some(Property::FailureFromAddress), - 277 => Some(Property::FailureFromName), - 828 => Some(Property::FailureReason), - 839 => Some(Property::FailureReasonCode), - 278 => Some(Property::FailureSendFrequency), - 280 => Some(Property::FailureSubject), - 738 => Some(Property::FeatureL2Normalize), - 739 => Some(Property::FeatureLogScale), - 67 => Some(Property::FeedbackType), - 406 => Some(Property::FieldEmail), - 407 => Some(Property::FieldHoneyPot), - 409 => Some(Property::FieldName), - 412 => Some(Property::FieldSubject), - 676 => Some(Property::FilePath), - 144 => Some(Property::Files), - 467 => Some(Property::FilterLogin), - 468 => Some(Property::FilterMailbox), - 469 => Some(Property::FilterMemberOf), - 902 => Some(Property::Fingerprint), - 638 => Some(Property::Flags), - 537 => Some(Property::FlagsAction), - 538 => Some(Property::FlagsProtocol), - 909 => Some(Property::FolderId), - 485 => Some(Property::ForDomain), - 415 => Some(Property::Format), - 62 => Some(Property::From), - 39 => Some(Property::FromAddress), - 165 => Some(Property::FromEmail), - 40 => Some(Property::FromName), - 521 => Some(Property::FutureRelease), - 124 => Some(Property::GenerateDkimKeys), - 918 => Some(Property::Generator), - 103 => Some(Property::GeoUrls), - 436 => Some(Property::GetMaxResults), - 508 => Some(Property::GreetingTimeout), - 770 => Some(Property::GreylistFor), - 477 => Some(Property::GroupClass), - 460 => Some(Property::GroupId), - 265 => Some(Property::HeaderFrom), - 93 => Some(Property::Headers), - 206 => Some(Property::HoldMetricsFor), - 204 => Some(Property::HoldMtaReportsFor), - 730 => Some(Property::HoldSamplesFor), - 205 => Some(Property::HoldTracesFor), - 333 => Some(Property::Host), - 331 => Some(Property::HostedZoneId), - 185 => Some(Property::Hostname), - 190 => Some(Property::Hour), - 32 => Some(Property::HttpAuth), - 33 => Some(Property::HttpHeaders), - 168 => Some(Property::HttpRsvpEnable), - 169 => Some(Property::HttpRsvpLinkExpiry), - 175 => Some(Property::HttpRsvpTemplate), - 170 => Some(Property::HttpRsvpUrl), - 842 => Some(Property::HttpRua), - 234 => Some(Property::HumanResult), - 807 => Some(Property::ICalendarData), + 724 => Some(Property::EncryptionKey), + 740 => Some(Property::EnvId), + 307 => Some(Property::EnvelopeFrom), + 306 => Some(Property::EnvelopeTo), + 138 => Some(Property::Environment), + 248 => Some(Property::ErrorCommand), + 247 => Some(Property::ErrorMessage), + 246 => Some(Property::ErrorType), + 287 => Some(Property::Errors), + 302 => Some(Property::EvaluatedDisposition), + 303 => Some(Property::EvaluatedDkim), + 304 => Some(Property::EvaluatedSpf), + 448 => Some(Property::Event), + 941 => Some(Property::EventEnd), + 943 => Some(Property::EventEndTz), + 939 => Some(Property::EventId), + 532 => Some(Property::EventSourceThrottle), + 940 => Some(Property::EventStart), + 942 => Some(Property::EventStartTz), + 169 => Some(Property::Events), + 996 => Some(Property::EventsPolicy), + 255 => Some(Property::Expire), + 745 => Some(Property::Expires), + 61 => Some(Property::ExpiresAt), + 735 => Some(Property::ExpiresAttempts), + 609 => Some(Property::Expiry), + 617 => Some(Property::Expn), + 236 => Some(Property::ExpungeSchedule), + 235 => Some(Property::ExpungeSchedulingInboxAfter), + 234 => Some(Property::ExpungeShareNotifyAfter), + 233 => Some(Property::ExpungeSubmissionsAfter), + 232 => Some(Property::ExpungeTrashAfter), + 896 => Some(Property::Extension), + 297 => Some(Property::Extensions), + 283 => Some(Property::ExtraContactInfo), + 962 => Some(Property::Factor), + 579 => Some(Property::FailOnTimeout), + 968 => Some(Property::FailedAt), + 969 => Some(Property::FailedAttemptNumber), + 979 => Some(Property::FailedSessionCount), + 992 => Some(Property::FailureDetails), + 321 => Some(Property::FailureDkimSignDomain), + 318 => Some(Property::FailureFromAddress), + 319 => Some(Property::FailureFromName), + 970 => Some(Property::FailureReason), + 981 => Some(Property::FailureReasonCode), + 320 => Some(Property::FailureSendFrequency), + 322 => Some(Property::FailureSubject), + 880 => Some(Property::FeatureL2Normalize), + 881 => Some(Property::FeatureLogScale), + 81 => Some(Property::FeedbackType), + 485 => Some(Property::FieldEmail), + 486 => Some(Property::FieldHoneyPot), + 488 => Some(Property::FieldName), + 490 => Some(Property::FieldSubject), + 790 => Some(Property::FilePath), + 171 => Some(Property::Files), + 556 => Some(Property::FilterLogin), + 558 => Some(Property::FilterMemberOf), + 557 => Some(Property::FilterRecipient), + 395 => Some(Property::Fingerprint), + 972 => Some(Property::FirstFailedAt), + 697 => Some(Property::FirstParty), + 263 => Some(Property::Flags), + 632 => Some(Property::FlagsAction), + 633 => Some(Property::FlagsProtocol), + 144 => Some(Property::FlushChanges), + 145 => Some(Property::FlushInterval), + 410 => Some(Property::FolderId), + 493 => Some(Property::Format), + 480 => Some(Property::ForwardedHops), + 76 => Some(Property::From), + 53 => Some(Property::FromAddress), + 54 => Some(Property::FromName), + 618 => Some(Property::FutureRelease), + 151 => Some(Property::GenerateDkimKeys), + 703 => Some(Property::Generation), + 298 => Some(Property::Generator), + 117 => Some(Property::GeoUrls), + 699 => Some(Property::GrantTypes), + 605 => Some(Property::GreetingTimeout), + 905 => Some(Property::GreylistFor), + 566 => Some(Property::GroupClass), + 548 => Some(Property::GroupId), + 175 => Some(Property::GroupwareLarge), + 176 => Some(Property::GroupwareLargeTtl), + 308 => Some(Property::HeaderFrom), + 107 => Some(Property::Headers), + 242 => Some(Property::HoldMtaReportsFor), + 872 => Some(Property::HoldSamplesFor), + 376 => Some(Property::Host), + 404 => Some(Property::HostedZoneId), + 223 => Some(Property::Hostname), + 547 => Some(Property::Hosts), + 228 => Some(Property::Hour), + 118 => Some(Property::HttpAuth), + 40 => Some(Property::HttpHeaders), + 206 => Some(Property::HttpRsvpEnable), + 207 => Some(Property::HttpRsvpLinkExpiry), + 208 => Some(Property::HttpRsvpUrl), + 984 => Some(Property::HttpRua), + 274 => Some(Property::HumanResult), + 948 => Some(Property::ICalendarData), 1 => Some(Property::Id), - 621 => Some(Property::IdTokenExpiry), - 91 => Some(Property::IdentityAlignment), - 376 => Some(Property::If), - 320 => Some(Property::ImpersonateServiceAccount), - 546 => Some(Property::ImplicitTls), - 128 => Some(Property::InMemoryStore), - 651 => Some(Property::InboundReportAddresses), - 652 => Some(Property::InboundReportForwarding), - 928 => Some(Property::InboundReportMaxSize), - 70 => Some(Property::Incidents), - 352 => Some(Property::IncludeSource), - 94 => Some(Property::IndexAsn), - 95 => Some(Property::IndexAsnName), - 664 => Some(Property::IndexBatchSize), - 667 => Some(Property::IndexCalendar), - 668 => Some(Property::IndexCalendarFields), - 670 => Some(Property::IndexContactFields), - 669 => Some(Property::IndexContacts), - 96 => Some(Property::IndexCountry), - 671 => Some(Property::IndexEmail), - 672 => Some(Property::IndexEmailFields), - 421 => Some(Property::IndexKey), - 673 => Some(Property::IndexTelemetry), - 674 => Some(Property::IndexTracingFields), - 422 => Some(Property::IndexValue), - 736 => Some(Property::IndicatorParameters), - 822 => Some(Property::InitialDelay), - 500 => Some(Property::Interval), - 516 => Some(Property::Intervals), - 752 => Some(Property::IpLimit), - 543 => Some(Property::IpLookupStrategy), - 290 => Some(Property::IpRevPtr), - 289 => Some(Property::IpRevResult), - 707 => Some(Property::IsActive), - 755 => Some(Property::IsArchive), - 756 => Some(Property::IsBad), - 340 => Some(Property::IsEnabled), - 806 => Some(Property::IsFromOrganizer), - 491 => Some(Property::IsGlobPattern), - 416 => Some(Property::IsGzipped), - 757 => Some(Property::IsNz), - 564 => Some(Property::IsSenderAllowed), - 776 => Some(Property::IsSpam), - 741 => Some(Property::IsTls), - 181 => Some(Property::Issuer), - 606 => Some(Property::IssuerUrl), - 172 => Some(Property::ItipMaxSize), - 824 => Some(Property::Jitter), - 334 => Some(Property::Key), - 337 => Some(Property::KeyName), - 120 => Some(Property::KeyPrefix), - 853 => Some(Property::KeyValues), - 391 => Some(Property::L1Ratio), - 392 => Some(Property::L2Ratio), - 186 => Some(Property::LastRenewal), - 727 => Some(Property::LearnHamFromCard), - 735 => Some(Property::LearnHamFromReply), - 728 => Some(Property::LearnSpamFromRblHits), - 729 => Some(Property::LearnSpamFromTraps), - 373 => Some(Property::Level), - 370 => Some(Property::LicenseKey), - 875 => Some(Property::Line), - 183 => Some(Property::ListenerIds), - 188 => Some(Property::Listeners), - 869 => Some(Property::LivePropertyMaxSize), + 723 => Some(Property::IdTokenExpiry), + 105 => Some(Property::IdentityAlignment), + 508 => Some(Property::IdleTimeout), + 452 => Some(Property::If), + 589 => Some(Property::ImapServerAdmin), + 588 => Some(Property::ImapServerComment), + 384 => Some(Property::ImpersonateServiceAccount), + 641 => Some(Property::ImplicitTls), + 155 => Some(Property::InMemoryStore), + 758 => Some(Property::InboundReportAddresses), + 759 => Some(Property::InboundReportForwarding), + 84 => Some(Property::Incidents), + 935 => Some(Property::IncludeSecrets), + 424 => Some(Property::IncludeSource), + 513 => Some(Property::Index), + 108 => Some(Property::IndexAsn), + 109 => Some(Property::IndexAsnName), + 769 => Some(Property::IndexBatchSize), + 772 => Some(Property::IndexCalendar), + 773 => Some(Property::IndexCalendarFields), + 775 => Some(Property::IndexContactFields), + 774 => Some(Property::IndexContacts), + 110 => Some(Property::IndexCountry), + 776 => Some(Property::IndexEmail), + 777 => Some(Property::IndexEmailFields), + 780 => Some(Property::IndexFiles), + 498 => Some(Property::IndexKey), + 499 => Some(Property::IndexValue), + 878 => Some(Property::IndicatorParameters), + 963 => Some(Property::InitialDelay), + 264 => Some(Property::InstanceHashes), + 596 => Some(Property::Interval), + 613 => Some(Property::Intervals), + 714 => Some(Property::IntrospectionAuthMethod), + 712 => Some(Property::IntrospectionClientId), + 713 => Some(Property::IntrospectionClientSecret), + 894 => Some(Property::IpLimit), + 638 => Some(Property::IpLookupStrategy), + 333 => Some(Property::IpRevPtr), + 332 => Some(Property::IpRevResult), + 820 => Some(Property::IsActive), + 897 => Some(Property::IsArchive), + 898 => Some(Property::IsBad), + 947 => Some(Property::IsFromOrganizer), + 580 => Some(Property::IsGlobPattern), + 494 => Some(Property::IsGzipped), + 899 => Some(Property::IsNz), + 659 => Some(Property::IsSenderAllowed), + 917 => Some(Property::IsSpam), + 883 => Some(Property::IsTls), + 219 => Some(Property::Issuer), + 18 => Some(Property::IssuerDomainNames), + 705 => Some(Property::IssuerUrl), + 965 => Some(Property::Jitter), + 375 => Some(Property::Key), + 406 => Some(Property::KeyName), + 141 => Some(Property::KeyPrefix), + 467 => Some(Property::L1Ratio), + 468 => Some(Property::L2Ratio), + 224 => Some(Property::LastRenewal), + 702 => Some(Property::LastUsedAt), + 869 => Some(Property::LearnHamFromCard), + 877 => Some(Property::LearnHamFromReply), + 870 => Some(Property::LearnSpamFromRblHits), + 871 => Some(Property::LearnSpamFromTraps), + 718 => Some(Property::LenientIssuerMatch), + 449 => Some(Property::Level), + 358 => Some(Property::Line), + 221 => Some(Property::ListenerIds), + 226 => Some(Property::Listeners), + 600 => Some(Property::LocalIp), + 601 => Some(Property::LocalIps), 7 => Some(Property::Locale), - 341 => Some(Property::Logo), - 371 => Some(Property::LogoUrl), - 682 => Some(Property::LoiterBanPeriod), - 681 => Some(Property::LoiterBanRate), - 854 => Some(Property::Lossy), - 384 => Some(Property::MachineId), - 793 => Some(Property::MailExchangers), - 284 => Some(Property::MailFrom), - 509 => Some(Property::MailFromTimeout), - 841 => Some(Property::MailRua), - 154 => Some(Property::MailingLists), - 796 => Some(Property::MaintenanceType), - 318 => Some(Property::ManagedZone), - 374 => Some(Property::Match), - 23 => Some(Property::MaxAddressBooks), - 566 => Some(Property::MaxAge), - 576 => Some(Property::MaxAllowedPacket), - 115 => Some(Property::MaxApiKeys), - 114 => Some(Property::MaxAppPasswords), - 353 => Some(Property::MaxAttachmentSize), - 511 => Some(Property::MaxAttempts), - 157 => Some(Property::MaxAttendees), - 425 => Some(Property::MaxAuthFailures), - 160 => Some(Property::MaxCalendars), - 201 => Some(Property::MaxChangesHistory), - 426 => Some(Property::MaxConcurrent), - 439 => Some(Property::MaxConcurrentRequests), - 442 => Some(Property::MaxConcurrentUploads), - 603 => Some(Property::MaxConnections), - 24 => Some(Property::MaxContacts), - 702 => Some(Property::MaxCpuCycles), - 823 => Some(Property::MaxDelay), - 530 => Some(Property::MaxDuration), - 417 => Some(Property::MaxEntries), - 418 => Some(Property::MaxEntrySize), - 163 => Some(Property::MaxEventNotifications), - 161 => Some(Property::MaxEvents), - 547 => Some(Property::MaxFailures), - 378 => Some(Property::MaxFiles), - 379 => Some(Property::MaxFolders), - 715 => Some(Property::MaxHeaderSize), - 159 => Some(Property::MaxICalendarSize), - 363 => Some(Property::MaxIdentities), - 716 => Some(Property::MaxIncludes), - 717 => Some(Property::MaxLocalVars), - 866 => Some(Property::MaxLockTimeout), - 867 => Some(Property::MaxLocks), - 355 => Some(Property::MaxMailboxDepth), - 356 => Some(Property::MaxMailboxNameLength), - 364 => Some(Property::MaxMailboxes), - 365 => Some(Property::MaxMaskedAddresses), - 718 => Some(Property::MaxMatchVars), - 354 => Some(Property::MaxMessageSize), - 361 => Some(Property::MaxMessages), - 924 => Some(Property::MaxMessagesPerCommand), - 927 => Some(Property::MaxMessagesPerSave), - 438 => Some(Property::MaxMethodCalls), - 544 => Some(Property::MaxMultihomed), - 545 => Some(Property::MaxMxHosts), - 720 => Some(Property::MaxNestedBlocks), - 721 => Some(Property::MaxNestedForEvery), - 703 => Some(Property::MaxNestedIncludes), - 722 => Some(Property::MaxNestedTests), - 704 => Some(Property::MaxOutMessages), - 162 => Some(Property::MaxParticipantIdentities), - 366 => Some(Property::MaxPublicKeys), - 923 => Some(Property::MaxPushSize), - 561 => Some(Property::MaxReceivedHeaders), - 173 => Some(Property::MaxRecipients), - 580 => Some(Property::MaxReconnects), - 158 => Some(Property::MaxRecurrenceExpansions), - 705 => Some(Property::MaxRedirects), - 852 => Some(Property::MaxReportSize), - 427 => Some(Property::MaxRequestRate), - 428 => Some(Property::MaxRequestSize), - 527 => Some(Property::MaxResponseSize), - 871 => Some(Property::MaxResults), - 18 => Some(Property::MaxRetries), - 648 => Some(Property::MaxRetryWait), - 719 => Some(Property::MaxScriptNameLength), - 723 => Some(Property::MaxScriptSize), - 726 => Some(Property::MaxScripts), - 696 => Some(Property::MaxShares), - 101 => Some(Property::MaxSize), - 724 => Some(Property::MaxStringLength), - 362 => Some(Property::MaxSubmissions), - 458 => Some(Property::MaxSubscriptions), - 926 => Some(Property::MaxUidBatches), - 444 => Some(Property::MaxUploadCount), - 443 => Some(Property::MaxUploadSize), - 22 => Some(Property::MaxVCardSize), - 725 => Some(Property::MaxVarNameLength), - 706 => Some(Property::MaxVarSize), - 864 => Some(Property::MemberGroupIds), - 19 => Some(Property::MemberTenantId), - 92 => Some(Property::Message), - 698 => Some(Property::MessageIdHostname), - 819 => Some(Property::MessageIds), - 145 => Some(Property::Messages), - 493 => Some(Property::Metric), - 497 => Some(Property::Metrics), - 207 => Some(Property::MetricsCollectionInterval), - 498 => Some(Property::MetricsPolicy), - 731 => Some(Property::MinHamSamples), - 649 => Some(Property::MinRetryWait), - 732 => Some(Property::MinSpamSamples), - 166 => Some(Property::MinTriggerInterval), - 925 => Some(Property::MinUidBatchSize), - 191 => Some(Property::Minute), - 567 => Some(Property::Mode), - 28 => Some(Property::Model), - 764 => Some(Property::ModelId), - 30 => Some(Property::ModelType), - 522 => Some(Property::MtPriority), - 570 => Some(Property::MtaSts), - 572 => Some(Property::MtaStsTimeout), - 859 => Some(Property::Multiline), - 550 => Some(Property::MustMatchSender), - 568 => Some(Property::MxHosts), - 25 => Some(Property::Name), - 414 => Some(Property::Namespace), - 156 => Some(Property::NegativeTtl), - 634 => Some(Property::NextNotify), - 633 => Some(Property::NextRetry), - 223 => Some(Property::NextTransitionAt), - 700 => Some(Property::NoCapabilityCheck), - 587 => Some(Property::NoEcho), - 523 => Some(Property::NoSoliciting), - 184 => Some(Property::NodeId), - 179 => Some(Property::NotValidAfter), - 180 => Some(Property::NotValidBefore), - 513 => Some(Property::Notify), - 642 => Some(Property::NotifyCount), - 643 => Some(Property::NotifyDue), - 390 => Some(Property::NumFeatures), - 350 => Some(Property::NumReplicas), - 351 => Some(Property::NumShards), - 930 => Some(Property::OauthClientId), - 813 => Some(Property::OnSuccessRenewCertificate), - 495 => Some(Property::OpenTelemetry), - 630 => Some(Property::Options), - 645 => Some(Property::Orcpt), - 241 => Some(Property::OrgName), - 843 => Some(Property::OrganizationName), - 301 => Some(Property::Origin), - 71 => Some(Property::OriginalEnvelopeId), - 72 => Some(Property::OriginalMailFrom), - 73 => Some(Property::OriginalRcptTo), + 412 => Some(Property::Logo), + 796 => Some(Property::LoiterBanPeriod), + 795 => Some(Property::LoiterBanRate), + 995 => Some(Property::Lossy), + 460 => Some(Property::MachineId), + 932 => Some(Property::MailExchangers), + 326 => Some(Property::MailFrom), + 884 => Some(Property::MailFromParameters), + 606 => Some(Property::MailFromTimeout), + 983 => Some(Property::MailRua), + 185 => Some(Property::MailingLists), + 936 => Some(Property::MaintenanceType), + 382 => Some(Property::ManagedZone), + 450 => Some(Property::Match), + 146 => Some(Property::MaxAccountSize), + 27 => Some(Property::MaxAddressBooks), + 30 => Some(Property::MaxAddressBooksPerCard), + 661 => Some(Property::MaxAge), + 314 => Some(Property::MaxAggregateReportSize), + 671 => Some(Property::MaxAllowedPacket), + 130 => Some(Property::MaxApiKeys), + 129 => Some(Property::MaxAppPasswords), + 425 => Some(Property::MaxAttachmentSize), + 912 => Some(Property::MaxAttachmentText), + 202 => Some(Property::MaxAttachmentsSize), + 608 => Some(Property::MaxAttempts), + 191 => Some(Property::MaxAttendees), + 503 => Some(Property::MaxAuthFailures), + 200 => Some(Property::MaxAvailabilityDuration), + 911 => Some(Property::MaxBodyText), + 930 => Some(Property::MaxBufferedMemory), + 194 => Some(Property::MaxCalendars), + 198 => Some(Property::MaxCalendarsPerEvent), + 239 => Some(Property::MaxChangesHistory), + 521 => Some(Property::MaxChangesResults), + 504 => Some(Property::MaxConcurrentConnections), + 779 => Some(Property::MaxConcurrentIndexTasks), + 43 => Some(Property::MaxConcurrentRequests), + 527 => Some(Property::MaxConcurrentUploads), + 694 => Some(Property::MaxConnections), + 28 => Some(Property::MaxContacts), + 815 => Some(Property::MaxCpuCycles), + 964 => Some(Property::MaxDelay), + 585 => Some(Property::MaxDepth), + 627 => Some(Property::MaxDuration), + 495 => Some(Property::MaxEntries), + 496 => Some(Property::MaxEntrySize), + 197 => Some(Property::MaxEventNotifications), + 195 => Some(Property::MaxEvents), + 199 => Some(Property::MaxExpandedQueryDuration), + 786 => Some(Property::MaxExtractArchiveEntries), + 783 => Some(Property::MaxExtractDecompressedSize), + 781 => Some(Property::MaxExtractDocumentSize), + 784 => Some(Property::MaxExtractPartSize), + 785 => Some(Property::MaxExtractParts), + 787 => Some(Property::MaxExtractPdfObjects), + 788 => Some(Property::MaxExtractRtfDepth), + 782 => Some(Property::MaxExtractTextSize), + 642 => Some(Property::MaxFailures), + 454 => Some(Property::MaxFiles), + 441 => Some(Property::MaxFlagLength), + 440 => Some(Property::MaxFlagsPerEmail), + 455 => Some(Property::MaxFolders), + 522 => Some(Property::MaxGetResults), + 443 => Some(Property::MaxHeaderCount), + 442 => Some(Property::MaxHeaderSize), + 193 => Some(Property::MaxICalendarSize), + 435 => Some(Property::MaxIdentities), + 762 => Some(Property::MaxInboundReportSize), + 827 => Some(Property::MaxIncludes), + 210 => Some(Property::MaxItipSize), + 1008 => Some(Property::MaxLivePropertySize), + 828 => Some(Property::MaxLocalVars), + 1006 => Some(Property::MaxLockTimeout), + 1007 => Some(Property::MaxLocks), + 427 => Some(Property::MaxMailboxDepth), + 428 => Some(Property::MaxMailboxNameLength), + 436 => Some(Property::MaxMailboxes), + 439 => Some(Property::MaxMailboxesPerEmail), + 829 => Some(Property::MaxMatchVars), + 31 => Some(Property::MaxMediaSize), + 426 => Some(Property::MaxMessageSize), + 913 => Some(Property::MaxMessageText), + 433 => Some(Property::MaxMessages), + 509 => Some(Property::MaxMessagesPerCommand), + 512 => Some(Property::MaxMessagesPerSave), + 524 => Some(Property::MaxMethodCalls), + 639 => Some(Property::MaxMultihomed), + 640 => Some(Property::MaxMxHosts), + 831 => Some(Property::MaxNestedBlocks), + 832 => Some(Property::MaxNestedForEvery), + 816 => Some(Property::MaxNestedIncludes), + 833 => Some(Property::MaxNestedTests), + 817 => Some(Property::MaxOutMessages), + 196 => Some(Property::MaxParticipantIdentities), + 126 => Some(Property::MaxPasswordLength), + 578 => Some(Property::MaxPolls), + 586 => Some(Property::MaxPrivateSize), + 438 => Some(Property::MaxPublicKeys), + 534 => Some(Property::MaxPushAttempts), + 546 => Some(Property::MaxPushSize), + 523 => Some(Property::MaxQueryResults), + 587 => Some(Property::MaxQueryScan), + 656 => Some(Property::MaxReceivedHeaders), + 211 => Some(Property::MaxRecipients), + 674 => Some(Property::MaxReconnects), + 192 => Some(Property::MaxRecurrenceExpansions), + 914 => Some(Property::MaxRedirectUrls), + 818 => Some(Property::MaxRedirects), + 993 => Some(Property::MaxReportSize), + 505 => Some(Property::MaxRequestSize), + 624 => Some(Property::MaxResponseSize), + 1009 => Some(Property::MaxResults), + 140 => Some(Property::MaxRetries), + 750 => Some(Property::MaxRetryWait), + 830 => Some(Property::MaxScriptNameLength), + 834 => Some(Property::MaxScriptSize), + 837 => Some(Property::MaxScripts), + 525 => Some(Property::MaxSetObjects), + 810 => Some(Property::MaxShares), + 115 => Some(Property::MaxSize), + 526 => Some(Property::MaxSnippetResults), + 835 => Some(Property::MaxStringLength), + 434 => Some(Property::MaxSubmissions), + 543 => Some(Property::MaxSubscriptions), + 511 => Some(Property::MaxUidBatches), + 529 => Some(Property::MaxUploadCount), + 528 => Some(Property::MaxUploadSize), + 26 => Some(Property::MaxVCardSize), + 836 => Some(Property::MaxVarNameLength), + 819 => Some(Property::MaxVarSize), + 953 => Some(Property::MemberGroupIds), + 106 => Some(Property::Message), + 812 => Some(Property::MessageIdHostname), + 959 => Some(Property::MessageIds), + 549 => Some(Property::MessageTimeout), + 172 => Some(Property::Messages), + 173 => Some(Property::MessagesLarge), + 174 => Some(Property::MessagesLargeTtl), + 594 => Some(Property::Metrics), + 595 => Some(Property::MetricsPolicy), + 147 => Some(Property::MinAccountSize), + 873 => Some(Property::MinHamSamples), + 125 => Some(Property::MinPasswordLength), + 127 => Some(Property::MinPasswordStrength), + 751 => Some(Property::MinRetryWait), + 874 => Some(Property::MinSpamSamples), + 204 => Some(Property::MinTriggerInterval), + 510 => Some(Property::MinUidBatchSize), + 229 => Some(Property::Minute), + 662 => Some(Property::Mode), + 37 => Some(Property::Model), + 619 => Some(Property::MtPriority), + 665 => Some(Property::MtaSts), + 667 => Some(Property::MtaStsTimeout), + 1000 => Some(Property::Multiline), + 645 => Some(Property::MustMatchSender), + 663 => Some(Property::MxHosts), + 36 => Some(Property::Name), + 492 => Some(Property::Namespace), + 187 => Some(Property::NegativeTtl), + 736 => Some(Property::NextNotify), + 517 => Some(Property::NextRetry), + 261 => Some(Property::NextTransitionAt), + 813 => Some(Property::NoCapabilityCheck), + 680 => Some(Property::NoEcho), + 620 => Some(Property::NoSoliciting), + 222 => Some(Property::NodeId), + 217 => Some(Property::NotValidAfter), + 218 => Some(Property::NotValidBefore), + 610 => Some(Property::Notify), + 743 => Some(Property::NotifyCount), + 744 => Some(Property::NotifyDue), + 466 => Some(Property::NumFeatures), + 422 => Some(Property::NumReplicas), + 423 => Some(Property::NumShards), + 68 => Some(Property::OauthClientId), + 967 => Some(Property::ObjectType), + 955 => Some(Property::OnSuccessRenewCertificate), + 711 => Some(Property::OpaqueTokenValidation), + 592 => Some(Property::OpenTelemetry), + 733 => Some(Property::Options), + 747 => Some(Property::Orcpt), + 281 => Some(Property::OrgName), + 985 => Some(Property::OrganizationName), + 346 => Some(Property::Origin), + 85 => Some(Property::OriginalEnvelopeId), + 86 => Some(Property::OriginalMailFrom), + 87 => Some(Property::OriginalRcptTo), 5 => Some(Property::OtpAuth), - 625 => Some(Property::OtpCode), - 626 => Some(Property::OtpUrl), - 653 => Some(Property::OutboundReportDomain), - 654 => Some(Property::OutboundReportSubmitter), - 590 => Some(Property::OverrideProxyTrustedNetworks), - 239 => Some(Property::OverrideType), - 324 => Some(Property::OvhEndpoint), - 737 => Some(Property::Parameters), - 433 => Some(Property::ParseLimitContact), - 434 => Some(Property::ParseLimitEmail), - 432 => Some(Property::ParseLimitEvent), - 876 => Some(Property::Password), - 113 => Some(Property::PasswordDefaultExpiry), - 109 => Some(Property::PasswordHashAlgorithm), - 111 => Some(Property::PasswordMaxLength), - 110 => Some(Property::PasswordMinLength), - 112 => Some(Property::PasswordMinStrength), - 380 => Some(Property::Path), - 646 => Some(Property::Period), - 48 => Some(Property::Permissions), - 583 => Some(Property::PingInterval), - 524 => Some(Property::Pipelining), - 846 => Some(Property::Policies), - 250 => Some(Property::PolicyAdkim), - 251 => Some(Property::PolicyAspf), - 920 => Some(Property::PolicyDiscoveryMethod), - 252 => Some(Property::PolicyDisposition), - 248 => Some(Property::PolicyDomain), - 255 => Some(Property::PolicyFailureReportingOptions), - 237 => Some(Property::PolicyIdentifier), - 840 => Some(Property::PolicyIdentifiers), - 919 => Some(Property::PolicyNp), - 262 => Some(Property::PolicyOverrideReasons), - 848 => Some(Property::PolicyStrings), - 253 => Some(Property::PolicySubdomainDisposition), - 254 => Some(Property::PolicyTestingMode), - 847 => Some(Property::PolicyType), - 249 => Some(Property::PolicyVersion), - 489 => Some(Property::PollInterval), - 311 => Some(Property::PollingInterval), - 478 => Some(Property::PoolMaxConnections), - 577 => Some(Property::PoolMinConnections), - 631 => Some(Property::PoolRecyclingMethod), - 479 => Some(Property::PoolTimeoutCreate), - 480 => Some(Property::PoolTimeoutRecycle), - 481 => Some(Property::PoolTimeoutWait), - 657 => Some(Property::PoolWorkers), - 299 => Some(Property::Port), - 910 => Some(Property::PreferredChain), - 856 => Some(Property::Prefix), - 306 => Some(Property::PreserveIntermediates), - 483 => Some(Property::Priority), - 177 => Some(Property::PrivateKey), - 904 => Some(Property::PrivateKeyPassword), - 903 => Some(Property::PrivateKeyPem), - 319 => Some(Property::PrivateZone), - 332 => Some(Property::PrivateZoneOnly), - 661 => Some(Property::Profile), - 317 => Some(Property::ProjectId), - 496 => Some(Property::Prometheus), - 765 => Some(Property::Prompt), - 313 => Some(Property::PropagationDelay), - 312 => Some(Property::PropagationTimeout), - 713 => Some(Property::ProtectedHeaders), - 298 => Some(Property::Protocol), - 533 => Some(Property::ProtocolVersion), - 795 => Some(Property::ProviderInfo), - 792 => Some(Property::ProxyTrustedNetworks), - 218 => Some(Property::PublicKey), - 302 => Some(Property::PublishRecords), - 448 => Some(Property::PushAttemptWait), - 449 => Some(Property::PushMaxAttempts), - 452 => Some(Property::PushRequestTimeout), - 450 => Some(Property::PushRetryWait), - 454 => Some(Property::PushShardsTotal), - 451 => Some(Property::PushThrottle), - 453 => Some(Property::PushVerifyTimeout), - 786 => Some(Property::QueryEmailAliases), - 783 => Some(Property::QueryLogin), - 437 => Some(Property::QueryMaxResults), - 785 => Some(Property::QueryMemberOf), - 784 => Some(Property::QueryRecipient), - 514 => Some(Property::QueueId), - 644 => Some(Property::QueueName), - 394 => Some(Property::Quotas), - 532 => Some(Property::Rate), - 410 => Some(Property::RateLimit), - 397 => Some(Property::RateLimitAnonymous), - 396 => Some(Property::RateLimitAuthenticated), - 767 => Some(Property::Ratio), - 510 => Some(Property::RcptToTimeout), - 650 => Some(Property::ReadFromReplicas), - 578 => Some(Property::ReadReplicas), - 45 => Some(Property::Reason), - 63 => Some(Property::ReceivedAt), - 636 => Some(Property::ReceivedFromIp), - 637 => Some(Property::ReceivedViaPort), - 836 => Some(Property::ReceivingIp), - 835 => Some(Property::ReceivingMxHelo), - 834 => Some(Property::ReceivingMxHostname), - 484 => Some(Property::Recipients), - 256 => Some(Property::Records), - 805 => Some(Property::RecurrenceId), - 911 => Some(Property::RedirectRoot), - 605 => Some(Property::RedirectUris), - 419 => Some(Property::Refresh), - 617 => Some(Property::RefreshTokenExpiry), - 618 => Some(Property::RefreshTokenRenewal), - 330 => Some(Property::Region), - 563 => Some(Property::RejectNonFqdn), - 282 => Some(Property::RemoteIp), - 17 => Some(Property::RenewBefore), - 66 => Some(Property::Report), - 349 => Some(Property::ReportAddressUri), - 244 => Some(Property::ReportId), - 74 => Some(Property::ReportedDomains), - 75 => Some(Property::ReportedUris), - 76 => Some(Property::ReportingMta), - 870 => Some(Property::RequestMaxSize), - 123 => Some(Property::RequestTlsCertificate), - 551 => Some(Property::Require), - 607 => Some(Property::RequireAudience), - 615 => Some(Property::RequireClientRegistration), - 608 => Some(Property::RequireScopes), - 525 => Some(Property::RequireTls), - 733 => Some(Property::ReservoirCapacity), - 880 => Some(Property::ResourceGroup), - 51 => Some(Property::ResourceUrl), - 212 => Some(Property::ResponseCode), - 213 => Some(Property::ResponseEnhanced), - 401 => Some(Property::ResponseHeaders), - 211 => Some(Property::ResponseHostname), - 214 => Some(Property::ResponseMessage), - 761 => Some(Property::ResponsePosCategory), - 762 => Some(Property::ResponsePosConfidence), - 763 => Some(Property::ResponsePosExplanation), - 233 => Some(Property::Result), - 832 => Some(Property::ResultType), - 228 => Some(Property::RetireAfter), - 420 => Some(Property::Retry), - 640 => Some(Property::RetryCount), - 641 => Some(Property::RetryDue), - 635 => Some(Property::ReturnPath), - 912 => Some(Property::ReuseKey), - 692 => Some(Property::ReverseIpVerify), - 565 => Some(Property::Rewrite), - 193 => Some(Property::RoleIds), - 152 => Some(Property::Roles), - 857 => Some(Property::Rotate), - 227 => Some(Property::RotateAfter), - 540 => Some(Property::Route), - 236 => Some(Property::Rua), - 896 => Some(Property::Sandbox), - 119 => Some(Property::SasToken), - 549 => Some(Property::SaslMechanisms), - 683 => Some(Property::ScanBanPaths), - 685 => Some(Property::ScanBanPeriod), - 684 => Some(Property::ScanBanRate), - 541 => Some(Property::Schedule), - 143 => Some(Property::Scheduling), - 908 => Some(Property::Scheme), - 281 => Some(Property::Scope), - 745 => Some(Property::Score), - 771 => Some(Property::ScoreDiscard), - 772 => Some(Property::ScoreReject), - 773 => Some(Property::ScoreSpam), - 553 => Some(Property::Script), - 127 => Some(Property::SearchStore), + 728 => Some(Property::OtpCode), + 729 => Some(Property::OtpUrl), + 760 => Some(Property::OutboundReportDomain), + 761 => Some(Property::OutboundReportSubmitter), + 682 => Some(Property::OverrideProxyTrustedNetworks), + 279 => Some(Property::OverrideType), + 402 => Some(Property::OvhEndpoint), + 879 => Some(Property::Parameters), + 519 => Some(Property::ParseLimitContact), + 520 => Some(Property::ParseLimitEmail), + 518 => Some(Property::ParseLimitEvent), + 514 => Some(Property::Partition), + 363 => Some(Property::Password), + 128 => Some(Property::PasswordDefaultExpiry), + 124 => Some(Property::PasswordHashAlgorithm), + 456 => Some(Property::Path), + 704 => Some(Property::Pending), + 748 => Some(Property::Period), + 62 => Some(Property::Permissions), + 676 => Some(Property::PingInterval), + 621 => Some(Property::Pipelining), + 987 => Some(Property::Policies), + 290 => Some(Property::PolicyAdkim), + 291 => Some(Property::PolicyAspf), + 300 => Some(Property::PolicyDiscoveryMethod), + 292 => Some(Property::PolicyDisposition), + 288 => Some(Property::PolicyDomain), + 295 => Some(Property::PolicyFailureReportingOptions), + 277 => Some(Property::PolicyIdentifier), + 982 => Some(Property::PolicyIdentifiers), + 299 => Some(Property::PolicyNp), + 305 => Some(Property::PolicyOverrideReasons), + 989 => Some(Property::PolicyStrings), + 293 => Some(Property::PolicySubdomainDisposition), + 294 => Some(Property::PolicyTestingMode), + 988 => Some(Property::PolicyType), + 289 => Some(Property::PolicyVersion), + 360 => Some(Property::PollInterval), + 567 => Some(Property::PoolMaxConnections), + 672 => Some(Property::PoolMinConnections), + 734 => Some(Property::PoolRecyclingMethod), + 568 => Some(Property::PoolTimeoutCreate), + 569 => Some(Property::PoolTimeoutRecycle), + 570 => Some(Property::PoolTimeoutWait), + 765 => Some(Property::PoolWorkers), + 344 => Some(Property::Port), + 696 => Some(Property::PostLogoutRedirectUris), + 22 => Some(Property::PreferredChain), + 997 => Some(Property::Prefix), + 350 => Some(Property::PreserveIntermediates), + 572 => Some(Property::Priority), + 215 => Some(Property::PrivateKey), + 397 => Some(Property::PrivateKeyPassword), + 396 => Some(Property::PrivateKeyPem), + 584 => Some(Property::PrivateMetadata), + 383 => Some(Property::PrivateZone), + 405 => Some(Property::PrivateZoneOnly), + 47 => Some(Property::Profile), + 381 => Some(Property::ProjectId), + 593 => Some(Property::Prometheus), + 362 => Some(Property::PropagationDelay), + 361 => Some(Property::PropagationTimeout), + 825 => Some(Property::ProtectedHeaders), + 343 => Some(Property::Protocol), + 630 => Some(Property::ProtocolVersion), + 934 => Some(Property::ProviderInfo), + 931 => Some(Property::ProxyTrustedNetworks), + 256 => Some(Property::PublicKey), + 347 => Some(Property::PublishRecords), + 533 => Some(Property::PushAttemptWait), + 537 => Some(Property::PushRequestTimeout), + 535 => Some(Property::PushRetryWait), + 539 => Some(Property::PushShardsTotal), + 536 => Some(Property::PushThrottle), + 538 => Some(Property::PushVerifyTimeout), + 926 => Some(Property::QueryEmailAliases), + 923 => Some(Property::QueryLogin), + 925 => Some(Property::QueryMemberOf), + 924 => Some(Property::QueryRecipient), + 611 => Some(Property::QueueId), + 746 => Some(Property::QueueName), + 516 => Some(Property::QueuedDeletions), + 515 => Some(Property::QueuedUpdates), + 470 => Some(Property::Quotas), + 629 => Some(Property::Rate), + 44 => Some(Property::RateLimit), + 474 => Some(Property::RateLimitAnonymous), + 473 => Some(Property::RateLimitAuthenticated), + 902 => Some(Property::Ratio), + 327 => Some(Property::RcptTo), + 607 => Some(Property::RcptToTimeout), + 678 => Some(Property::ReadBufferCapacity), + 752 => Some(Property::ReadFromReplicas), + 59 => Some(Property::Reason), + 77 => Some(Property::ReceivedAt), + 738 => Some(Property::ReceivedFromIp), + 739 => Some(Property::ReceivedViaPort), + 978 => Some(Property::ReceivingIp), + 977 => Some(Property::ReceivingMxHelo), + 976 => Some(Property::ReceivingMxHostname), + 573 => Some(Property::Recipients), + 296 => Some(Property::Records), + 945 => Some(Property::RecurrenceId), + 916 => Some(Property::RedirectCacheTtl), + 481 => Some(Property::RedirectRoot), + 915 => Some(Property::RedirectTimeout), + 695 => Some(Property::RedirectUris), + 114 => Some(Property::RefreshInterval), + 720 => Some(Property::RefreshTokenExpiry), + 356 => Some(Property::Region), + 658 => Some(Property::RejectNonFqdn), + 324 => Some(Property::RemoteIp), + 20 => Some(Property::RenewBefore), + 80 => Some(Property::Report), + 420 => Some(Property::ReportAddressUri), + 284 => Some(Property::ReportId), + 88 => Some(Property::ReportedDomains), + 89 => Some(Property::ReportedUris), + 90 => Some(Property::ReportingMta), + 675 => Some(Property::RequestTimeout), + 150 => Some(Property::RequestTlsCertificate), + 646 => Some(Property::Require), + 622 => Some(Property::RequireTls), + 875 => Some(Property::ReservoirCapacity), + 366 => Some(Property::ResourceGroup), + 64 => Some(Property::ResourceUrl), + 250 => Some(Property::ResponseCode), + 251 => Some(Property::ResponseEnhanced), + 478 => Some(Property::ResponseHeaders), + 249 => Some(Property::ResponseHostname), + 252 => Some(Property::ResponseMessage), + 273 => Some(Property::Result), + 974 => Some(Property::ResultType), + 143 => Some(Property::Retention), + 268 => Some(Property::RetireAfter), + 497 => Some(Property::Retry), + 741 => Some(Property::RetryCount), + 742 => Some(Property::RetryDue), + 737 => Some(Property::ReturnPath), + 23 => Some(Property::ReuseKey), + 806 => Some(Property::ReverseIpVerify), + 660 => Some(Property::Rewrite), + 231 => Some(Property::RoleIds), + 183 => Some(Property::Roles), + 998 => Some(Property::Rotate), + 267 => Some(Property::RotateAfter), + 635 => Some(Property::Route), + 276 => Some(Property::Rua), + 389 => Some(Property::Sandbox), + 134 => Some(Property::SasToken), + 644 => Some(Property::SaslMechanisms), + 797 => Some(Property::ScanBanPaths), + 799 => Some(Property::ScanBanPeriod), + 798 => Some(Property::ScanBanRate), + 636 => Some(Property::Schedule), + 170 => Some(Property::Scheduling), + 409 => Some(Property::Scheme), + 323 => Some(Property::Scope), + 698 => Some(Property::Scopes), + 885 => Some(Property::Score), + 906 => Some(Property::ScoreDiscard), + 907 => Some(Property::ScoreReject), + 908 => Some(Property::ScoreSpam), + 648 => Some(Property::Script), + 154 => Some(Property::SearchStore), 3 => Some(Property::Secret), - 328 => Some(Property::SecretAccessKey), - 326 => Some(Property::SecretApiKey), - 906 => Some(Property::SecretId), - 659 => Some(Property::SecretKey), - 660 => Some(Property::SecurityToken), - 222 => Some(Property::Selector), - 226 => Some(Property::SelectorTemplate), - 230 => Some(Property::SendFrequency), - 833 => Some(Property::SendingMtaIp), - 915 => Some(Property::SentinelSecret), - 914 => Some(Property::SentinelUsername), - 97 => Some(Property::Separator), - 121 => Some(Property::ServerHostname), - 934 => Some(Property::ServerId), - 308 => Some(Property::Servers), - 316 => Some(Property::ServiceAccountJson), - 913 => Some(Property::ServiceName), - 794 => Some(Property::Services), - 329 => Some(Property::SessionToken), - 440 => Some(Property::SetMaxObjects), - 830 => Some(Property::ShardIndex), - 895 => Some(Property::SharedSecret), - 336 => Some(Property::Sig0Algorithm), - 623 => Some(Property::SignatureAlgorithm), - 624 => Some(Property::SignatureKey), - 335 => Some(Property::SignerName), - 64 => Some(Property::Size), - 885 => Some(Property::SkipDeploy), - 423 => Some(Property::SkipFirst), - 552 => Some(Property::SmtpGreeting), - 441 => Some(Property::SnippetMaxResults), - 591 => Some(Property::SocketBacklog), - 592 => Some(Property::SocketNoDelay), - 593 => Some(Property::SocketReceiveBufferSize), - 594 => Some(Property::SocketReuseAddress), - 595 => Some(Property::SocketReusePort), - 596 => Some(Property::SocketSendBufferSize), - 597 => Some(Property::SocketTosV4), - 598 => Some(Property::SocketTtl), - 77 => Some(Property::SourceIp), - 504 => Some(Property::SourceIps), - 78 => Some(Property::SourcePort), - 775 => Some(Property::SpamFilterRulesUrl), - 90 => Some(Property::SpfDns), - 285 => Some(Property::SpfEhloDomain), - 286 => Some(Property::SpfEhloResult), - 688 => Some(Property::SpfEhloVerify), - 689 => Some(Property::SpfFromVerify), - 287 => Some(Property::SpfMailFromDomain), - 288 => Some(Property::SpfMailFromResult), - 267 => Some(Property::SpfResults), - 224 => Some(Property::Stage), - 529 => Some(Property::Stages), - 56 => Some(Property::StartTime), - 571 => Some(Property::StartTls), - 61 => Some(Property::Status), - 116 => Some(Property::StorageAccount), - 778 => Some(Property::Store), - 694 => Some(Property::Stores), - 816 => Some(Property::Strategy), - 347 => Some(Property::SubAddressing), - 887 => Some(Property::SubAuthId), - 41 => Some(Property::Subject), - 178 => Some(Property::SubjectAlternativeNames), - 368 => Some(Property::Subscribe), - 879 => Some(Property::SubscriptionId), - 494 => Some(Property::Sum), - 808 => Some(Property::Summary), - 666 => Some(Property::SupportedLanguages), - 748 => Some(Property::Tag), - 746 => Some(Property::Tags), - 189 => Some(Property::TaskTypes), - 187 => Some(Property::Tasks), - 307 => Some(Property::TcpOnError), - 907 => Some(Property::TeamId), - 528 => Some(Property::TempFailOnError), - 27 => Some(Property::Temperature), - 167 => Some(Property::Template), - 900 => Some(Property::TenancyOcid), - 831 => Some(Property::TenantId), - 153 => Some(Property::Tenants), + 355 => Some(Property::SecretKey), + 260 => Some(Property::Selector), + 266 => Some(Property::SelectorTemplate), + 270 => Some(Property::SendFrequency), + 975 => Some(Property::SendingMtaIp), + 755 => Some(Property::SentinelSecret), + 754 => Some(Property::SentinelUsername), + 111 => Some(Property::Separator), + 403 => Some(Property::ServerId), + 352 => Some(Property::Servers), + 380 => Some(Property::ServiceAccountJson), + 753 => Some(Property::ServiceName), + 933 => Some(Property::Services), + 550 => Some(Property::SessionTimeout), + 357 => Some(Property::SessionToken), + 973 => Some(Property::ShardIndex), + 388 => Some(Property::SharedSecret), + 725 => Some(Property::SignatureAlgorithm), + 726 => Some(Property::SignatureKey), + 78 => Some(Property::Size), + 716 => Some(Property::SkipAudienceValidation), + 369 => Some(Property::SkipDeploy), + 500 => Some(Property::SkipFirst), + 647 => Some(Property::SmtpGreeting), + 683 => Some(Property::SocketBacklog), + 684 => Some(Property::SocketNoDelay), + 685 => Some(Property::SocketReceiveBufferSize), + 686 => Some(Property::SocketReuseAddress), + 687 => Some(Property::SocketReusePort), + 688 => Some(Property::SocketSendBufferSize), + 689 => Some(Property::SocketTosV4), + 690 => Some(Property::SocketTtl), + 91 => Some(Property::SourceIp), + 92 => Some(Property::SourcePort), + 910 => Some(Property::SpamFilterRulesUrl), + 166 => Some(Property::SpamLookup), + 104 => Some(Property::SpfDns), + 328 => Some(Property::SpfEhloDomain), + 329 => Some(Property::SpfEhloResult), + 802 => Some(Property::SpfEhloVerify), + 803 => Some(Property::SpfFromVerify), + 330 => Some(Property::SpfMailFromDomain), + 331 => Some(Property::SpfMailFromResult), + 310 => Some(Property::SpfResults), + 262 => Some(Property::Stage), + 626 => Some(Property::Stages), + 666 => Some(Property::StartTls), + 75 => Some(Property::Status), + 136 => Some(Property::StorageAccount), + 142 => Some(Property::Store), + 956 => Some(Property::Strategy), + 418 => Some(Property::SubAddressing), + 371 => Some(Property::SubAuthId), + 55 => Some(Property::Subject), + 216 => Some(Property::SubjectAlternativeNames), + 445 => Some(Property::Subscribe), + 679 => Some(Property::SubscriptionCapacity), + 365 => Some(Property::SubscriptionId), + 949 => Some(Property::Summary), + 771 => Some(Property::SupportedLanguages), + 190 => Some(Property::Swap), + 891 => Some(Property::Tag), + 886 => Some(Property::Tags), + 944 => Some(Property::TargetAccountId), + 227 => Some(Property::TaskTypes), + 225 => Some(Property::Tasks), + 351 => Some(Property::TcpOnError), + 408 => Some(Property::TeamId), + 625 => Some(Property::TempFailOnError), + 393 => Some(Property::TenancyOcid), + 131 => Some(Property::TenantId), 2 => Some(Property::Text), - 377 => Some(Property::Then), - 219 => Some(Property::ThirdParty), - 220 => Some(Property::ThirdPartyHash), - 818 => Some(Property::ThreadName), - 791 => Some(Property::ThreadPoolSize), - 574 => Some(Property::ThreadsPerNode), - 862 => Some(Property::Throttle), + 453 => Some(Property::Then), + 257 => Some(Property::ThirdParty), + 258 => Some(Property::ThirdPartyHash), + 958 => Some(Property::ThreadName), + 669 => Some(Property::ThreadsPerNode), + 1003 => Some(Property::Throttle), 8 => Some(Property::TimeZone), - 29 => Some(Property::Timeout), - 429 => Some(Property::TimeoutAnonymous), - 430 => Some(Property::TimeoutAuthenticated), - 534 => Some(Property::TimeoutCommand), - 535 => Some(Property::TimeoutConnect), - 581 => Some(Property::TimeoutConnection), - 536 => Some(Property::TimeoutData), - 431 => Some(Property::TimeoutIdle), - 461 => Some(Property::TimeoutMessage), - 582 => Some(Property::TimeoutRequest), - 462 => Some(Property::TimeoutSession), - 482 => Some(Property::Timestamp), - 55 => Some(Property::Title), - 542 => Some(Property::Tls), - 599 => Some(Property::TlsDisableCipherSuites), - 600 => Some(Property::TlsDisableProtocols), - 601 => Some(Property::TlsIgnoreClientOrder), - 602 => Some(Property::TlsImplicit), - 573 => Some(Property::TlsTimeout), - 42 => Some(Property::To), - 888 => Some(Property::Token), - 817 => Some(Property::TotalDeadline), - 850 => Some(Property::TotalFailedSessions), - 849 => Some(Property::TotalSuccessfulSessions), - 815 => Some(Property::TraceId), - 129 => Some(Property::Tracer), - 734 => Some(Property::TrainFrequency), - 385 => Some(Property::TransactionRetryDelay), - 386 => Some(Property::TransactionRetryLimit), - 387 => Some(Property::TransactionTimeout), - 531 => Some(Property::TransferLimit), - 769 => Some(Property::TrustContacts), - 774 => Some(Property::TrustReplies), - 338 => Some(Property::TsigAlgorithm), - 310 => Some(Property::Ttl), - 54 => Some(Property::UnpackDirectory), - 812 => Some(Property::UpdateRecords), - 445 => Some(Property::UploadQuota), - 446 => Some(Property::UploadTtl), - 31 => Some(Property::Url), - 753 => Some(Property::UrlLimit), - 52 => Some(Property::UrlPrefix), - 647 => Some(Property::Urls), - 400 => Some(Property::UsePermissiveCors), - 309 => Some(Property::UseTls), - 402 => Some(Property::UseXForwarded), - 395 => Some(Property::UsedDiskQuota), - 79 => Some(Property::UserAgent), - 620 => Some(Property::UserCodeExpiry), - 901 => Some(Property::UserOcid), - 131 => Some(Property::Username), - 610 => Some(Property::UsernameDomain), - 931 => Some(Property::VCardVersion), - 413 => Some(Property::ValidateDomain), - 492 => Some(Property::Value), - 675 => Some(Property::VariableName), - 874 => Some(Property::VerifyAfterWrite), - 80 => Some(Property::Version), - 884 => Some(Property::ViewName), - 526 => Some(Property::Vrfy), - 548 => Some(Property::WaitOnFail), - 893 => Some(Property::WapiVersion), - 922 => Some(Property::WebPushContact), - 921 => Some(Property::WebPushKey), - 455 => Some(Property::WebsocketHeartbeat), - 456 => Some(Property::WebsocketThrottle), - 457 => Some(Property::WebsocketTimeout), - 749 => Some(Property::Zone), - 98 => Some(Property::ZoneIpV4), - 99 => Some(Property::ZoneIpV6), + 42 => Some(Property::Timeout), + 571 => Some(Property::Timestamp), + 637 => Some(Property::Tls), + 691 => Some(Property::TlsDisableCipherSuites), + 692 => Some(Property::TlsDisableProtocols), + 693 => Some(Property::TlsIgnoreClientOrder), + 668 => Some(Property::TlsTimeout), + 56 => Some(Property::To), + 373 => Some(Property::Token), + 135 => Some(Property::TokenFile), + 991 => Some(Property::TotalFailedSessions), + 990 => Some(Property::TotalSuccessfulSessions), + 156 => Some(Property::Tracer), + 876 => Some(Property::TrainInterval), + 461 => Some(Property::TransactionRetryDelay), + 462 => Some(Property::TransactionRetryLimit), + 463 => Some(Property::TransactionTimeout), + 628 => Some(Property::TransferLimit), + 904 => Some(Property::TrustContacts), + 909 => Some(Property::TrustReplies), + 407 => Some(Property::TsigAlgorithm), + 359 => Some(Property::Ttl), + 67 => Some(Property::UnpackDirectory), + 715 => Some(Property::UnscopedTokens), + 954 => Some(Property::UpdateRecords), + 530 => Some(Property::UploadQuota), + 531 => Some(Property::UploadTtl), + 38 => Some(Property::Url), + 895 => Some(Property::UrlLimit), + 65 => Some(Property::UrlPrefix), + 749 => Some(Property::Urls), + 477 => Some(Property::UsePermissiveCors), + 353 => Some(Property::UseTls), + 471 => Some(Property::UsedDiskQuota), + 93 => Some(Property::UserAgent), + 722 => Some(Property::UserCodeExpiry), + 394 => Some(Property::UserOcid), + 158 => Some(Property::Username), + 707 => Some(Property::UsernameDomain), + 29 => Some(Property::VCardVersion), + 491 => Some(Property::ValidateDomain), + 581 => Some(Property::Value), + 789 => Some(Property::VariableName), + 583 => Some(Property::VendorNamespaces), + 768 => Some(Property::VerifyAfterWrite), + 94 => Some(Property::Version), + 368 => Some(Property::ViewName), + 623 => Some(Property::Vrfy), + 643 => Some(Property::WaitOnFail), + 386 => Some(Property::WapiVersion), + 757 => Some(Property::Warnings), + 545 => Some(Property::WebPushContact), + 544 => Some(Property::WebPushKey), + 540 => Some(Property::WebsocketHeartbeat), + 541 => Some(Property::WebsocketThrottle), + 542 => Some(Property::WebsocketTimeout), + 960 => Some(Property::WindowEnd), + 501 => Some(Property::Zone), + 112 => Some(Property::ZoneIpV4), + 113 => Some(Property::ZoneIpV6), _ => None, } } - const COUNT: usize = 935; + const COUNT: usize = 1012; } impl serde::Serialize for Property { @@ -3261,13 +3249,10 @@ impl ObjectType { ObjectType::AcmeProvider => AcmeProvider::FLAGS, ObjectType::Action => Action::FLAGS, ObjectType::AddressBook => AddressBook::FLAGS, - ObjectType::AiModel => AiModel::FLAGS, - ObjectType::Alert => Alert::FLAGS, ObjectType::AllowedIp => AllowedIp::FLAGS, ObjectType::ApiKey => ApiKey::FLAGS, ObjectType::AppPassword => AppPassword::FLAGS, ObjectType::Application => Application::FLAGS, - ObjectType::ArchivedItem => ArchivedItem::FLAGS, ObjectType::ArfExternalReport => ArfExternalReport::FLAGS, ObjectType::Asn => Asn::FLAGS, ObjectType::Authentication => Authentication::FLAGS, @@ -3295,7 +3280,6 @@ impl ObjectType { ObjectType::Domain => Domain::FLAGS, ObjectType::DsnReportSettings => DsnReportSettings::FLAGS, ObjectType::Email => Email::FLAGS, - ObjectType::Enterprise => Enterprise::FLAGS, ObjectType::EventTracingLevel => EventTracingLevel::FLAGS, ObjectType::FileStorage => FileStorage::FLAGS, ObjectType::Http => Http::FLAGS, @@ -3303,15 +3287,14 @@ impl ObjectType { ObjectType::HttpLookup => HttpLookup::FLAGS, ObjectType::Imap => Imap::FLAGS, ObjectType::InMemoryStore => InMemoryStore::FLAGS, + ObjectType::IndexQueueStatus => IndexQueueStatus::FLAGS, ObjectType::Jmap => Jmap::FLAGS, ObjectType::Log => Log::FLAGS, ObjectType::MailingList => MailingList::FLAGS, - ObjectType::MaskedEmail => MaskedEmail::FLAGS, ObjectType::MemoryLookupKey => MemoryLookupKey::FLAGS, ObjectType::MemoryLookupKeyValue => MemoryLookupKeyValue::FLAGS, - ObjectType::Metric => Metric::FLAGS, + ObjectType::Metadata => Metadata::FLAGS, ObjectType::Metrics => Metrics::FLAGS, - ObjectType::MetricsStore => MetricsStore::FLAGS, ObjectType::MtaConnectionStrategy => MtaConnectionStrategy::FLAGS, ObjectType::MtaDeliverySchedule => MtaDeliverySchedule::FLAGS, ObjectType::MtaExtensions => MtaExtensions::FLAGS, @@ -3334,6 +3317,7 @@ impl ObjectType { ObjectType::MtaVirtualQueue => MtaVirtualQueue::FLAGS, ObjectType::NetworkListener => NetworkListener::FLAGS, ObjectType::OAuthClient => OAuthClient::FLAGS, + ObjectType::OAuthGrant => OAuthGrant::FLAGS, ObjectType::OidcProvider => OidcProvider::FLAGS, ObjectType::PublicKey => PublicKey::FLAGS, ObjectType::QueuedMessage => QueuedMessage::FLAGS, @@ -3352,7 +3336,6 @@ impl ObjectType { ObjectType::SpamDnsblServer => SpamDnsblServer::FLAGS, ObjectType::SpamDnsblSettings => SpamDnsblSettings::FLAGS, ObjectType::SpamFileExtension => SpamFileExtension::FLAGS, - ObjectType::SpamLlm => SpamLlm::FLAGS, ObjectType::SpamPyzor => SpamPyzor::FLAGS, ObjectType::SpamRule => SpamRule::FLAGS, ObjectType::SpamSettings => SpamSettings::FLAGS, @@ -3363,13 +3346,10 @@ impl ObjectType { ObjectType::SystemSettings => SystemSettings::FLAGS, ObjectType::Task => Task::FLAGS, ObjectType::TaskManager => TaskManager::FLAGS, - ObjectType::Tenant => Tenant::FLAGS, ObjectType::TlsExternalReport => TlsExternalReport::FLAGS, ObjectType::TlsInternalReport => TlsInternalReport::FLAGS, ObjectType::TlsReportSettings => TlsReportSettings::FLAGS, - ObjectType::Trace => Trace::FLAGS, ObjectType::Tracer => Tracer::FLAGS, - ObjectType::TracingStore => TracingStore::FLAGS, ObjectType::WebDav => WebDav::FLAGS, ObjectType::WebHook => WebHook::FLAGS, } @@ -3393,21 +3373,11 @@ impl ObjectType { IndexSchemaType::Search, IndexSchemaValueType::Id, ), - IndexSchema::new( - Property::ExternalId, - IndexSchemaType::Search, - IndexSchemaValueType::Keyword, - ), IndexSchema::new( Property::MemberGroupIds, IndexSchemaType::Search, IndexSchemaValueType::Id, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), IndexSchema::new( Property::Name, IndexSchemaType::Search, @@ -3420,22 +3390,12 @@ impl ObjectType { IndexSchemaType::Search, IndexSchemaValueType::Text, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), ], ObjectType::AllowedIp => vec![IndexSchema::new( Property::Address, IndexSchemaType::Unique, IndexSchemaValueType::IpMask, )], - ObjectType::ArchivedItem => vec![IndexSchema::new( - Property::AccountId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - )], ObjectType::BlockedIp => vec![IndexSchema::new( Property::Address, IndexSchemaType::Unique, @@ -3451,28 +3411,17 @@ impl ObjectType { IndexSchemaType::Unique, IndexSchemaValueType::Keyword, )], - ObjectType::Directory => vec![IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - )], + ObjectType::Directory => vec![ + ], ObjectType::DkimSignature => vec![ IndexSchema::new( Property::DomainId, IndexSchemaType::Search, IndexSchemaValueType::Id, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), ], - ObjectType::DnsServer => vec![IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - )], + ObjectType::DnsServer => vec![ + ], ObjectType::Domain => vec![ IndexSchema::new( Property::Text, @@ -3484,11 +3433,6 @@ impl ObjectType { IndexSchemaType::Unique, IndexSchemaValueType::Keyword, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), IndexSchema::new( Property::Name, IndexSchemaType::Unique, @@ -3501,17 +3445,7 @@ impl ObjectType { IndexSchemaType::Search, IndexSchemaValueType::Text, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), ], - ObjectType::MaskedEmail => vec![IndexSchema::new( - Property::AccountId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - )], ObjectType::MemoryLookupKey => vec![IndexSchema::new( Property::Namespace, IndexSchemaType::Search, @@ -3563,12 +3497,12 @@ impl ObjectType { IndexSchemaType::Unique, IndexSchemaValueType::Keyword, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), ], + ObjectType::OAuthGrant => vec![IndexSchema::new( + Property::AccountId, + IndexSchemaType::Search, + IndexSchemaValueType::Id, + )], ObjectType::PublicKey => vec![IndexSchema::new( Property::AccountId, IndexSchemaType::Search, @@ -3580,11 +3514,6 @@ impl ObjectType { IndexSchemaType::Search, IndexSchemaValueType::Text, ), - IndexSchema::new( - Property::MemberTenantId, - IndexSchemaType::Search, - IndexSchemaValueType::Id, - ), ], ObjectType::SieveSystemScript => vec![IndexSchema::new( Property::Name, @@ -3621,11 +3550,6 @@ impl ObjectType { IndexSchemaType::Search, IndexSchemaValueType::Id, )], - ObjectType::Tenant => vec![IndexSchema::new( - Property::Text, - IndexSchemaType::Search, - IndexSchemaValueType::Text, - )], _ => vec![], } } @@ -3636,15 +3560,12 @@ impl ObjectType { ObjectType::AccountPassword => Permission::SysAccountPasswordGet, ObjectType::AccountSettings => Permission::SysAccountSettingsGet, ObjectType::AcmeProvider => Permission::SysAcmeProviderGet, - ObjectType::Action => Permission::SysActionGet, + ObjectType::Action => unreachable!(), ObjectType::AddressBook => Permission::SysAddressBookGet, - ObjectType::AiModel => Permission::SysAiModelGet, - ObjectType::Alert => Permission::SysAlertGet, ObjectType::AllowedIp => Permission::SysAllowedIpGet, ObjectType::ApiKey => Permission::SysApiKeyGet, ObjectType::AppPassword => Permission::SysAppPasswordGet, ObjectType::Application => Permission::SysApplicationGet, - ObjectType::ArchivedItem => Permission::SysArchivedItemGet, ObjectType::ArfExternalReport => Permission::SysArfExternalReportGet, ObjectType::Asn => Permission::SysAsnGet, ObjectType::Authentication => Permission::SysAuthenticationGet, @@ -3672,7 +3593,6 @@ impl ObjectType { ObjectType::Domain => Permission::SysDomainGet, ObjectType::DsnReportSettings => Permission::SysDsnReportSettingsGet, ObjectType::Email => Permission::SysEmailGet, - ObjectType::Enterprise => Permission::SysEnterpriseGet, ObjectType::EventTracingLevel => Permission::SysEventTracingLevelGet, ObjectType::FileStorage => Permission::SysFileStorageGet, ObjectType::Http => Permission::SysHttpGet, @@ -3680,15 +3600,14 @@ impl ObjectType { ObjectType::HttpLookup => Permission::SysHttpLookupGet, ObjectType::Imap => Permission::SysImapGet, ObjectType::InMemoryStore => Permission::SysInMemoryStoreGet, + ObjectType::IndexQueueStatus => Permission::SysIndexQueueStatusGet, ObjectType::Jmap => Permission::SysJmapGet, ObjectType::Log => Permission::SysLogGet, ObjectType::MailingList => Permission::SysMailingListGet, - ObjectType::MaskedEmail => Permission::SysMaskedEmailGet, ObjectType::MemoryLookupKey => Permission::SysMemoryLookupKeyGet, ObjectType::MemoryLookupKeyValue => Permission::SysMemoryLookupKeyValueGet, - ObjectType::Metric => Permission::SysMetricGet, + ObjectType::Metadata => Permission::SysMetadataGet, ObjectType::Metrics => Permission::SysMetricsGet, - ObjectType::MetricsStore => Permission::SysMetricsStoreGet, ObjectType::MtaConnectionStrategy => Permission::SysMtaConnectionStrategyGet, ObjectType::MtaDeliverySchedule => Permission::SysMtaDeliveryScheduleGet, ObjectType::MtaExtensions => Permission::SysMtaExtensionsGet, @@ -3711,6 +3630,7 @@ impl ObjectType { ObjectType::MtaVirtualQueue => Permission::SysMtaVirtualQueueGet, ObjectType::NetworkListener => Permission::SysNetworkListenerGet, ObjectType::OAuthClient => Permission::SysOAuthClientGet, + ObjectType::OAuthGrant => Permission::SysOAuthGrantGet, ObjectType::OidcProvider => Permission::SysOidcProviderGet, ObjectType::PublicKey => Permission::SysPublicKeyGet, ObjectType::QueuedMessage => Permission::SysQueuedMessageGet, @@ -3729,7 +3649,6 @@ impl ObjectType { ObjectType::SpamDnsblServer => Permission::SysSpamDnsblServerGet, ObjectType::SpamDnsblSettings => Permission::SysSpamDnsblSettingsGet, ObjectType::SpamFileExtension => Permission::SysSpamFileExtensionGet, - ObjectType::SpamLlm => Permission::SysSpamLlmGet, ObjectType::SpamPyzor => Permission::SysSpamPyzorGet, ObjectType::SpamRule => Permission::SysSpamRuleGet, ObjectType::SpamSettings => Permission::SysSpamSettingsGet, @@ -3740,13 +3659,10 @@ impl ObjectType { ObjectType::SystemSettings => Permission::SysSystemSettingsGet, ObjectType::Task => Permission::SysTaskGet, ObjectType::TaskManager => Permission::SysTaskManagerGet, - ObjectType::Tenant => Permission::SysTenantGet, ObjectType::TlsExternalReport => Permission::SysTlsExternalReportGet, ObjectType::TlsInternalReport => Permission::SysTlsInternalReportGet, ObjectType::TlsReportSettings => Permission::SysTlsReportSettingsGet, - ObjectType::Trace => Permission::SysTraceGet, ObjectType::Tracer => Permission::SysTracerGet, - ObjectType::TracingStore => Permission::SysTracingStoreGet, ObjectType::WebDav => Permission::SysWebDavGet, ObjectType::WebHook => Permission::SysWebHookGet, } @@ -3756,14 +3672,10 @@ impl ObjectType { match self { ObjectType::Account => Permission::SysAccountQuery, ObjectType::AcmeProvider => Permission::SysAcmeProviderQuery, - ObjectType::Action => Permission::SysActionQuery, - ObjectType::AiModel => Permission::SysAiModelQuery, - ObjectType::Alert => Permission::SysAlertQuery, ObjectType::AllowedIp => Permission::SysAllowedIpQuery, ObjectType::ApiKey => Permission::SysApiKeyQuery, ObjectType::AppPassword => Permission::SysAppPasswordQuery, ObjectType::Application => Permission::SysApplicationQuery, - ObjectType::ArchivedItem => Permission::SysArchivedItemQuery, ObjectType::ArfExternalReport => Permission::SysArfExternalReportQuery, ObjectType::BlockedIp => Permission::SysBlockedIpQuery, ObjectType::Certificate => Permission::SysCertificateQuery, @@ -3777,12 +3689,11 @@ impl ObjectType { ObjectType::Domain => Permission::SysDomainQuery, ObjectType::EventTracingLevel => Permission::SysEventTracingLevelQuery, ObjectType::HttpLookup => Permission::SysHttpLookupQuery, + ObjectType::IndexQueueStatus => Permission::SysIndexQueueStatusQuery, ObjectType::Log => Permission::SysLogQuery, ObjectType::MailingList => Permission::SysMailingListQuery, - ObjectType::MaskedEmail => Permission::SysMaskedEmailQuery, ObjectType::MemoryLookupKey => Permission::SysMemoryLookupKeyQuery, ObjectType::MemoryLookupKeyValue => Permission::SysMemoryLookupKeyValueQuery, - ObjectType::Metric => Permission::SysMetricQuery, ObjectType::MtaConnectionStrategy => Permission::SysMtaConnectionStrategyQuery, ObjectType::MtaDeliverySchedule => Permission::SysMtaDeliveryScheduleQuery, ObjectType::MtaHook => Permission::SysMtaHookQuery, @@ -3795,6 +3706,7 @@ impl ObjectType { ObjectType::MtaVirtualQueue => Permission::SysMtaVirtualQueueQuery, ObjectType::NetworkListener => Permission::SysNetworkListenerQuery, ObjectType::OAuthClient => Permission::SysOAuthClientQuery, + ObjectType::OAuthGrant => Permission::SysOAuthGrantQuery, ObjectType::PublicKey => Permission::SysPublicKeyQuery, ObjectType::QueuedMessage => Permission::SysQueuedMessageQuery, ObjectType::Role => Permission::SysRoleQuery, @@ -3807,850 +3719,392 @@ impl ObjectType { ObjectType::SpamTrainingSample => Permission::SysSpamTrainingSampleQuery, ObjectType::StoreLookup => Permission::SysStoreLookupQuery, ObjectType::Task => Permission::SysTaskQuery, - ObjectType::Tenant => Permission::SysTenantQuery, ObjectType::TlsExternalReport => Permission::SysTlsExternalReportQuery, ObjectType::TlsInternalReport => Permission::SysTlsInternalReportQuery, - ObjectType::Trace => Permission::SysTraceQuery, ObjectType::Tracer => Permission::SysTracerQuery, ObjectType::WebHook => Permission::SysWebHookQuery, _ => unreachable!(), } } - pub fn set_permission(&self) -> [Permission; 3] { + pub fn create_permission(&self) -> Permission { match self { - ObjectType::Account => [ - Permission::SysAccountCreate, - Permission::SysAccountUpdate, - Permission::SysAccountDestroy, - ], - ObjectType::AccountPassword => [ - Permission::SysAccountPasswordUpdate, - Permission::SysAccountPasswordUpdate, - Permission::SysAccountPasswordUpdate, - ], - ObjectType::AccountSettings => [ - Permission::SysAccountSettingsUpdate, - Permission::SysAccountSettingsUpdate, - Permission::SysAccountSettingsUpdate, - ], - ObjectType::AcmeProvider => [ - Permission::SysAcmeProviderCreate, - Permission::SysAcmeProviderUpdate, - Permission::SysAcmeProviderDestroy, - ], - ObjectType::Action => [ - Permission::SysActionCreate, - Permission::SysActionUpdate, - Permission::SysActionDestroy, - ], - ObjectType::AddressBook => [ - Permission::SysAddressBookUpdate, - Permission::SysAddressBookUpdate, - Permission::SysAddressBookUpdate, - ], - ObjectType::AiModel => [ - Permission::SysAiModelCreate, - Permission::SysAiModelUpdate, - Permission::SysAiModelDestroy, - ], - ObjectType::Alert => [ - Permission::SysAlertCreate, - Permission::SysAlertUpdate, - Permission::SysAlertDestroy, - ], - ObjectType::AllowedIp => [ - Permission::SysAllowedIpCreate, - Permission::SysAllowedIpUpdate, - Permission::SysAllowedIpDestroy, - ], - ObjectType::ApiKey => [ - Permission::SysApiKeyCreate, - Permission::SysApiKeyUpdate, - Permission::SysApiKeyDestroy, - ], - ObjectType::AppPassword => [ - Permission::SysAppPasswordCreate, - Permission::SysAppPasswordUpdate, - Permission::SysAppPasswordDestroy, - ], - ObjectType::Application => [ - Permission::SysApplicationCreate, - Permission::SysApplicationUpdate, - Permission::SysApplicationDestroy, - ], - ObjectType::ArchivedItem => [ - Permission::SysArchivedItemCreate, - Permission::SysArchivedItemUpdate, - Permission::SysArchivedItemDestroy, - ], - ObjectType::ArfExternalReport => [ - Permission::SysArfExternalReportCreate, - Permission::SysArfExternalReportUpdate, - Permission::SysArfExternalReportDestroy, - ], - ObjectType::Asn => [ - Permission::SysAsnUpdate, - Permission::SysAsnUpdate, - Permission::SysAsnUpdate, - ], - ObjectType::Authentication => [ - Permission::SysAuthenticationUpdate, - Permission::SysAuthenticationUpdate, - Permission::SysAuthenticationUpdate, - ], - ObjectType::BlobStore => [ - Permission::SysBlobStoreUpdate, - Permission::SysBlobStoreUpdate, - Permission::SysBlobStoreUpdate, - ], - ObjectType::BlockedIp => [ - Permission::SysBlockedIpCreate, - Permission::SysBlockedIpUpdate, - Permission::SysBlockedIpDestroy, - ], - ObjectType::Bootstrap => [ - Permission::SysBootstrapUpdate, - Permission::SysBootstrapUpdate, - Permission::SysBootstrapUpdate, - ], - ObjectType::Cache => [ - Permission::SysCacheUpdate, - Permission::SysCacheUpdate, - Permission::SysCacheUpdate, - ], - ObjectType::Calendar => [ - Permission::SysCalendarUpdate, - Permission::SysCalendarUpdate, - Permission::SysCalendarUpdate, - ], - ObjectType::CalendarAlarm => [ - Permission::SysCalendarAlarmUpdate, - Permission::SysCalendarAlarmUpdate, - Permission::SysCalendarAlarmUpdate, - ], - ObjectType::CalendarScheduling => [ - Permission::SysCalendarSchedulingUpdate, - Permission::SysCalendarSchedulingUpdate, - Permission::SysCalendarSchedulingUpdate, - ], - ObjectType::Certificate => [ - Permission::SysCertificateCreate, - Permission::SysCertificateUpdate, - Permission::SysCertificateDestroy, - ], - ObjectType::ClusterNode => [ - Permission::SysClusterNodeCreate, - Permission::SysClusterNodeUpdate, - Permission::SysClusterNodeDestroy, - ], - ObjectType::ClusterRole => [ - Permission::SysClusterRoleCreate, - Permission::SysClusterRoleUpdate, - Permission::SysClusterRoleDestroy, - ], - ObjectType::Coordinator => [ - Permission::SysCoordinatorUpdate, - Permission::SysCoordinatorUpdate, - Permission::SysCoordinatorUpdate, - ], - ObjectType::DataRetention => [ - Permission::SysDataRetentionUpdate, - Permission::SysDataRetentionUpdate, - Permission::SysDataRetentionUpdate, - ], - ObjectType::DataStore => [ - Permission::SysDataStoreUpdate, - Permission::SysDataStoreUpdate, - Permission::SysDataStoreUpdate, - ], - ObjectType::Directory => [ - Permission::SysDirectoryCreate, - Permission::SysDirectoryUpdate, - Permission::SysDirectoryDestroy, - ], - ObjectType::DkimReportSettings => [ - Permission::SysDkimReportSettingsUpdate, - Permission::SysDkimReportSettingsUpdate, - Permission::SysDkimReportSettingsUpdate, - ], - ObjectType::DkimSignature => [ - Permission::SysDkimSignatureCreate, - Permission::SysDkimSignatureUpdate, - Permission::SysDkimSignatureDestroy, - ], - ObjectType::DmarcExternalReport => [ - Permission::SysDmarcExternalReportCreate, - Permission::SysDmarcExternalReportUpdate, - Permission::SysDmarcExternalReportDestroy, - ], - ObjectType::DmarcInternalReport => [ - Permission::SysDmarcInternalReportCreate, - Permission::SysDmarcInternalReportUpdate, - Permission::SysDmarcInternalReportDestroy, - ], - ObjectType::DmarcReportSettings => [ - Permission::SysDmarcReportSettingsUpdate, - Permission::SysDmarcReportSettingsUpdate, - Permission::SysDmarcReportSettingsUpdate, - ], - ObjectType::DnsResolver => [ - Permission::SysDnsResolverUpdate, - Permission::SysDnsResolverUpdate, - Permission::SysDnsResolverUpdate, - ], - ObjectType::DnsServer => [ - Permission::SysDnsServerCreate, - Permission::SysDnsServerUpdate, - Permission::SysDnsServerDestroy, - ], - ObjectType::Domain => [ - Permission::SysDomainCreate, - Permission::SysDomainUpdate, - Permission::SysDomainDestroy, - ], - ObjectType::DsnReportSettings => [ - Permission::SysDsnReportSettingsUpdate, - Permission::SysDsnReportSettingsUpdate, - Permission::SysDsnReportSettingsUpdate, - ], - ObjectType::Email => [ - Permission::SysEmailUpdate, - Permission::SysEmailUpdate, - Permission::SysEmailUpdate, - ], - ObjectType::Enterprise => [ - Permission::SysEnterpriseUpdate, - Permission::SysEnterpriseUpdate, - Permission::SysEnterpriseUpdate, - ], - ObjectType::EventTracingLevel => [ - Permission::SysEventTracingLevelCreate, - Permission::SysEventTracingLevelUpdate, - Permission::SysEventTracingLevelDestroy, - ], - ObjectType::FileStorage => [ - Permission::SysFileStorageUpdate, - Permission::SysFileStorageUpdate, - Permission::SysFileStorageUpdate, - ], - ObjectType::Http => [ - Permission::SysHttpUpdate, - Permission::SysHttpUpdate, - Permission::SysHttpUpdate, - ], - ObjectType::HttpForm => [ - Permission::SysHttpFormUpdate, - Permission::SysHttpFormUpdate, - Permission::SysHttpFormUpdate, - ], - ObjectType::HttpLookup => [ - Permission::SysHttpLookupCreate, - Permission::SysHttpLookupUpdate, - Permission::SysHttpLookupDestroy, - ], - ObjectType::Imap => [ - Permission::SysImapUpdate, - Permission::SysImapUpdate, - Permission::SysImapUpdate, - ], - ObjectType::InMemoryStore => [ - Permission::SysInMemoryStoreUpdate, - Permission::SysInMemoryStoreUpdate, - Permission::SysInMemoryStoreUpdate, - ], - ObjectType::Jmap => [ - Permission::SysJmapUpdate, - Permission::SysJmapUpdate, - Permission::SysJmapUpdate, - ], - ObjectType::Log => [ - Permission::SysLogCreate, - Permission::SysLogUpdate, - Permission::SysLogDestroy, - ], - ObjectType::MailingList => [ - Permission::SysMailingListCreate, - Permission::SysMailingListUpdate, - Permission::SysMailingListDestroy, - ], - ObjectType::MaskedEmail => [ - Permission::SysMaskedEmailCreate, - Permission::SysMaskedEmailUpdate, - Permission::SysMaskedEmailDestroy, - ], - ObjectType::MemoryLookupKey => [ - Permission::SysMemoryLookupKeyCreate, - Permission::SysMemoryLookupKeyUpdate, - Permission::SysMemoryLookupKeyDestroy, - ], - ObjectType::MemoryLookupKeyValue => [ - Permission::SysMemoryLookupKeyValueCreate, - Permission::SysMemoryLookupKeyValueUpdate, - Permission::SysMemoryLookupKeyValueDestroy, - ], - ObjectType::Metric => [ - Permission::SysMetricCreate, - Permission::SysMetricUpdate, - Permission::SysMetricDestroy, - ], - ObjectType::Metrics => [ - Permission::SysMetricsUpdate, - Permission::SysMetricsUpdate, - Permission::SysMetricsUpdate, - ], - ObjectType::MetricsStore => [ - Permission::SysMetricsStoreUpdate, - Permission::SysMetricsStoreUpdate, - Permission::SysMetricsStoreUpdate, - ], - ObjectType::MtaConnectionStrategy => [ - Permission::SysMtaConnectionStrategyCreate, - Permission::SysMtaConnectionStrategyUpdate, - Permission::SysMtaConnectionStrategyDestroy, - ], - ObjectType::MtaDeliverySchedule => [ - Permission::SysMtaDeliveryScheduleCreate, - Permission::SysMtaDeliveryScheduleUpdate, - Permission::SysMtaDeliveryScheduleDestroy, - ], - ObjectType::MtaExtensions => [ - Permission::SysMtaExtensionsUpdate, - Permission::SysMtaExtensionsUpdate, - Permission::SysMtaExtensionsUpdate, - ], - ObjectType::MtaHook => [ - Permission::SysMtaHookCreate, - Permission::SysMtaHookUpdate, - Permission::SysMtaHookDestroy, - ], - ObjectType::MtaInboundSession => [ - Permission::SysMtaInboundSessionUpdate, - Permission::SysMtaInboundSessionUpdate, - Permission::SysMtaInboundSessionUpdate, - ], - ObjectType::MtaInboundThrottle => [ - Permission::SysMtaInboundThrottleCreate, - Permission::SysMtaInboundThrottleUpdate, - Permission::SysMtaInboundThrottleDestroy, - ], - ObjectType::MtaMilter => [ - Permission::SysMtaMilterCreate, - Permission::SysMtaMilterUpdate, - Permission::SysMtaMilterDestroy, - ], - ObjectType::MtaOutboundStrategy => [ - Permission::SysMtaOutboundStrategyUpdate, - Permission::SysMtaOutboundStrategyUpdate, - Permission::SysMtaOutboundStrategyUpdate, - ], - ObjectType::MtaOutboundThrottle => [ - Permission::SysMtaOutboundThrottleCreate, - Permission::SysMtaOutboundThrottleUpdate, - Permission::SysMtaOutboundThrottleDestroy, - ], - ObjectType::MtaQueueQuota => [ - Permission::SysMtaQueueQuotaCreate, - Permission::SysMtaQueueQuotaUpdate, - Permission::SysMtaQueueQuotaDestroy, - ], - ObjectType::MtaRoute => [ - Permission::SysMtaRouteCreate, - Permission::SysMtaRouteUpdate, - Permission::SysMtaRouteDestroy, - ], - ObjectType::MtaStageAuth => [ - Permission::SysMtaStageAuthUpdate, - Permission::SysMtaStageAuthUpdate, - Permission::SysMtaStageAuthUpdate, - ], - ObjectType::MtaStageConnect => [ - Permission::SysMtaStageConnectUpdate, - Permission::SysMtaStageConnectUpdate, - Permission::SysMtaStageConnectUpdate, - ], - ObjectType::MtaStageData => [ - Permission::SysMtaStageDataUpdate, - Permission::SysMtaStageDataUpdate, - Permission::SysMtaStageDataUpdate, - ], - ObjectType::MtaStageEhlo => [ - Permission::SysMtaStageEhloUpdate, - Permission::SysMtaStageEhloUpdate, - Permission::SysMtaStageEhloUpdate, - ], - ObjectType::MtaStageMail => [ - Permission::SysMtaStageMailUpdate, - Permission::SysMtaStageMailUpdate, - Permission::SysMtaStageMailUpdate, - ], - ObjectType::MtaStageRcpt => [ - Permission::SysMtaStageRcptUpdate, - Permission::SysMtaStageRcptUpdate, - Permission::SysMtaStageRcptUpdate, - ], - ObjectType::MtaSts => [ - Permission::SysMtaStsUpdate, - Permission::SysMtaStsUpdate, - Permission::SysMtaStsUpdate, - ], - ObjectType::MtaTlsStrategy => [ - Permission::SysMtaTlsStrategyCreate, - Permission::SysMtaTlsStrategyUpdate, - Permission::SysMtaTlsStrategyDestroy, - ], - ObjectType::MtaVirtualQueue => [ - Permission::SysMtaVirtualQueueCreate, - Permission::SysMtaVirtualQueueUpdate, - Permission::SysMtaVirtualQueueDestroy, - ], - ObjectType::NetworkListener => [ - Permission::SysNetworkListenerCreate, - Permission::SysNetworkListenerUpdate, - Permission::SysNetworkListenerDestroy, - ], - ObjectType::OAuthClient => [ - Permission::SysOAuthClientCreate, - Permission::SysOAuthClientUpdate, - Permission::SysOAuthClientDestroy, - ], - ObjectType::OidcProvider => [ - Permission::SysOidcProviderUpdate, - Permission::SysOidcProviderUpdate, - Permission::SysOidcProviderUpdate, - ], - ObjectType::PublicKey => [ - Permission::SysPublicKeyCreate, - Permission::SysPublicKeyUpdate, - Permission::SysPublicKeyDestroy, - ], - ObjectType::QueuedMessage => [ - Permission::SysQueuedMessageCreate, - Permission::SysQueuedMessageUpdate, - Permission::SysQueuedMessageDestroy, - ], - ObjectType::ReportSettings => [ - Permission::SysReportSettingsUpdate, - Permission::SysReportSettingsUpdate, - Permission::SysReportSettingsUpdate, - ], - ObjectType::Role => [ - Permission::SysRoleCreate, - Permission::SysRoleUpdate, - Permission::SysRoleDestroy, - ], - ObjectType::Search => [ - Permission::SysSearchUpdate, - Permission::SysSearchUpdate, - Permission::SysSearchUpdate, - ], - ObjectType::SearchStore => [ - Permission::SysSearchStoreUpdate, - Permission::SysSearchStoreUpdate, - Permission::SysSearchStoreUpdate, - ], - ObjectType::Security => [ - Permission::SysSecurityUpdate, - Permission::SysSecurityUpdate, - Permission::SysSecurityUpdate, - ], - ObjectType::SenderAuth => [ - Permission::SysSenderAuthUpdate, - Permission::SysSenderAuthUpdate, - Permission::SysSenderAuthUpdate, - ], - ObjectType::Sharing => [ - Permission::SysSharingUpdate, - Permission::SysSharingUpdate, - Permission::SysSharingUpdate, - ], - ObjectType::SieveSystemInterpreter => [ - Permission::SysSieveSystemInterpreterUpdate, - Permission::SysSieveSystemInterpreterUpdate, - Permission::SysSieveSystemInterpreterUpdate, - ], - ObjectType::SieveSystemScript => [ - Permission::SysSieveSystemScriptCreate, - Permission::SysSieveSystemScriptUpdate, - Permission::SysSieveSystemScriptDestroy, - ], - ObjectType::SieveUserInterpreter => [ - Permission::SysSieveUserInterpreterUpdate, - Permission::SysSieveUserInterpreterUpdate, - Permission::SysSieveUserInterpreterUpdate, - ], - ObjectType::SieveUserScript => [ - Permission::SysSieveUserScriptCreate, - Permission::SysSieveUserScriptUpdate, - Permission::SysSieveUserScriptDestroy, - ], - ObjectType::SpamClassifier => [ - Permission::SysSpamClassifierUpdate, - Permission::SysSpamClassifierUpdate, - Permission::SysSpamClassifierUpdate, - ], - ObjectType::SpamDnsblServer => [ - Permission::SysSpamDnsblServerCreate, - Permission::SysSpamDnsblServerUpdate, - Permission::SysSpamDnsblServerDestroy, - ], - ObjectType::SpamDnsblSettings => [ - Permission::SysSpamDnsblSettingsUpdate, - Permission::SysSpamDnsblSettingsUpdate, - Permission::SysSpamDnsblSettingsUpdate, - ], - ObjectType::SpamFileExtension => [ - Permission::SysSpamFileExtensionCreate, - Permission::SysSpamFileExtensionUpdate, - Permission::SysSpamFileExtensionDestroy, - ], - ObjectType::SpamLlm => [ - Permission::SysSpamLlmUpdate, - Permission::SysSpamLlmUpdate, - Permission::SysSpamLlmUpdate, - ], - ObjectType::SpamPyzor => [ - Permission::SysSpamPyzorUpdate, - Permission::SysSpamPyzorUpdate, - Permission::SysSpamPyzorUpdate, - ], - ObjectType::SpamRule => [ - Permission::SysSpamRuleCreate, - Permission::SysSpamRuleUpdate, - Permission::SysSpamRuleDestroy, - ], - ObjectType::SpamSettings => [ - Permission::SysSpamSettingsUpdate, - Permission::SysSpamSettingsUpdate, - Permission::SysSpamSettingsUpdate, - ], - ObjectType::SpamTag => [ - Permission::SysSpamTagCreate, - Permission::SysSpamTagUpdate, - Permission::SysSpamTagDestroy, - ], - ObjectType::SpamTrainingSample => [ - Permission::SysSpamTrainingSampleCreate, - Permission::SysSpamTrainingSampleUpdate, - Permission::SysSpamTrainingSampleDestroy, - ], - ObjectType::SpfReportSettings => [ - Permission::SysSpfReportSettingsUpdate, - Permission::SysSpfReportSettingsUpdate, - Permission::SysSpfReportSettingsUpdate, - ], - ObjectType::StoreLookup => [ - Permission::SysStoreLookupCreate, - Permission::SysStoreLookupUpdate, - Permission::SysStoreLookupDestroy, - ], - ObjectType::SystemSettings => [ - Permission::SysSystemSettingsUpdate, - Permission::SysSystemSettingsUpdate, - Permission::SysSystemSettingsUpdate, - ], - ObjectType::Task => [ - Permission::SysTaskCreate, - Permission::SysTaskUpdate, - Permission::SysTaskDestroy, - ], - ObjectType::TaskManager => [ - Permission::SysTaskManagerUpdate, - Permission::SysTaskManagerUpdate, - Permission::SysTaskManagerUpdate, - ], - ObjectType::Tenant => [ - Permission::SysTenantCreate, - Permission::SysTenantUpdate, - Permission::SysTenantDestroy, - ], - ObjectType::TlsExternalReport => [ - Permission::SysTlsExternalReportCreate, - Permission::SysTlsExternalReportUpdate, - Permission::SysTlsExternalReportDestroy, - ], - ObjectType::TlsInternalReport => [ - Permission::SysTlsInternalReportCreate, - Permission::SysTlsInternalReportUpdate, - Permission::SysTlsInternalReportDestroy, - ], - ObjectType::TlsReportSettings => [ - Permission::SysTlsReportSettingsUpdate, - Permission::SysTlsReportSettingsUpdate, - Permission::SysTlsReportSettingsUpdate, - ], - ObjectType::Trace => [ - Permission::SysTraceCreate, - Permission::SysTraceUpdate, - Permission::SysTraceDestroy, - ], - ObjectType::Tracer => [ - Permission::SysTracerCreate, - Permission::SysTracerUpdate, - Permission::SysTracerDestroy, - ], - ObjectType::TracingStore => [ - Permission::SysTracingStoreUpdate, - Permission::SysTracingStoreUpdate, - Permission::SysTracingStoreUpdate, - ], - ObjectType::WebDav => [ - Permission::SysWebDavUpdate, - Permission::SysWebDavUpdate, - Permission::SysWebDavUpdate, - ], - ObjectType::WebHook => [ - Permission::SysWebHookCreate, - Permission::SysWebHookUpdate, - Permission::SysWebHookDestroy, - ], + ObjectType::Account => Permission::SysAccountCreate, + ObjectType::AccountPassword => Permission::SysAccountPasswordUpdate, + ObjectType::AccountSettings => Permission::SysAccountSettingsUpdate, + ObjectType::AcmeProvider => Permission::SysAcmeProviderCreate, + ObjectType::Action => Permission::SysActionCreate, + ObjectType::AddressBook => Permission::SysAddressBookUpdate, + ObjectType::AllowedIp => Permission::SysAllowedIpCreate, + ObjectType::ApiKey => Permission::SysApiKeyCreate, + ObjectType::AppPassword => Permission::SysAppPasswordCreate, + ObjectType::Application => Permission::SysApplicationCreate, + ObjectType::ArfExternalReport => unreachable!(), + ObjectType::Asn => Permission::SysAsnUpdate, + ObjectType::Authentication => Permission::SysAuthenticationUpdate, + ObjectType::BlobStore => Permission::SysBlobStoreUpdate, + ObjectType::BlockedIp => Permission::SysBlockedIpCreate, + ObjectType::Bootstrap => Permission::SysBootstrapUpdate, + ObjectType::Cache => Permission::SysCacheUpdate, + ObjectType::Calendar => Permission::SysCalendarUpdate, + ObjectType::CalendarAlarm => Permission::SysCalendarAlarmUpdate, + ObjectType::CalendarScheduling => Permission::SysCalendarSchedulingUpdate, + ObjectType::Certificate => Permission::SysCertificateCreate, + ObjectType::ClusterNode => unreachable!(), + ObjectType::ClusterRole => Permission::SysClusterRoleCreate, + ObjectType::Coordinator => Permission::SysCoordinatorUpdate, + ObjectType::DataRetention => Permission::SysDataRetentionUpdate, + ObjectType::DataStore => Permission::SysDataStoreUpdate, + ObjectType::Directory => Permission::SysDirectoryCreate, + ObjectType::DkimReportSettings => Permission::SysDkimReportSettingsUpdate, + ObjectType::DkimSignature => Permission::SysDkimSignatureCreate, + ObjectType::DmarcExternalReport => unreachable!(), + ObjectType::DmarcInternalReport => unreachable!(), + ObjectType::DmarcReportSettings => Permission::SysDmarcReportSettingsUpdate, + ObjectType::DnsResolver => Permission::SysDnsResolverUpdate, + ObjectType::DnsServer => Permission::SysDnsServerCreate, + ObjectType::Domain => Permission::SysDomainCreate, + ObjectType::DsnReportSettings => Permission::SysDsnReportSettingsUpdate, + ObjectType::Email => Permission::SysEmailUpdate, + ObjectType::EventTracingLevel => Permission::SysEventTracingLevelCreate, + ObjectType::FileStorage => Permission::SysFileStorageUpdate, + ObjectType::Http => Permission::SysHttpUpdate, + ObjectType::HttpForm => Permission::SysHttpFormUpdate, + ObjectType::HttpLookup => Permission::SysHttpLookupCreate, + ObjectType::Imap => Permission::SysImapUpdate, + ObjectType::InMemoryStore => Permission::SysInMemoryStoreUpdate, + ObjectType::IndexQueueStatus => unreachable!(), + ObjectType::Jmap => Permission::SysJmapUpdate, + ObjectType::Log => unreachable!(), + ObjectType::MailingList => Permission::SysMailingListCreate, + ObjectType::MemoryLookupKey => Permission::SysMemoryLookupKeyCreate, + ObjectType::MemoryLookupKeyValue => Permission::SysMemoryLookupKeyValueCreate, + ObjectType::Metadata => Permission::SysMetadataUpdate, + ObjectType::Metrics => Permission::SysMetricsUpdate, + ObjectType::MtaConnectionStrategy => Permission::SysMtaConnectionStrategyCreate, + ObjectType::MtaDeliverySchedule => Permission::SysMtaDeliveryScheduleCreate, + ObjectType::MtaExtensions => Permission::SysMtaExtensionsUpdate, + ObjectType::MtaHook => Permission::SysMtaHookCreate, + ObjectType::MtaInboundSession => Permission::SysMtaInboundSessionUpdate, + ObjectType::MtaInboundThrottle => Permission::SysMtaInboundThrottleCreate, + ObjectType::MtaMilter => Permission::SysMtaMilterCreate, + ObjectType::MtaOutboundStrategy => Permission::SysMtaOutboundStrategyUpdate, + ObjectType::MtaOutboundThrottle => Permission::SysMtaOutboundThrottleCreate, + ObjectType::MtaQueueQuota => Permission::SysMtaQueueQuotaCreate, + ObjectType::MtaRoute => Permission::SysMtaRouteCreate, + ObjectType::MtaStageAuth => Permission::SysMtaStageAuthUpdate, + ObjectType::MtaStageConnect => Permission::SysMtaStageConnectUpdate, + ObjectType::MtaStageData => Permission::SysMtaStageDataUpdate, + ObjectType::MtaStageEhlo => Permission::SysMtaStageEhloUpdate, + ObjectType::MtaStageMail => Permission::SysMtaStageMailUpdate, + ObjectType::MtaStageRcpt => Permission::SysMtaStageRcptUpdate, + ObjectType::MtaSts => Permission::SysMtaStsUpdate, + ObjectType::MtaTlsStrategy => Permission::SysMtaTlsStrategyCreate, + ObjectType::MtaVirtualQueue => Permission::SysMtaVirtualQueueCreate, + ObjectType::NetworkListener => Permission::SysNetworkListenerCreate, + ObjectType::OAuthClient => Permission::SysOAuthClientCreate, + ObjectType::OAuthGrant => unreachable!(), + ObjectType::OidcProvider => Permission::SysOidcProviderUpdate, + ObjectType::PublicKey => Permission::SysPublicKeyCreate, + ObjectType::QueuedMessage => unreachable!(), + ObjectType::ReportSettings => Permission::SysReportSettingsUpdate, + ObjectType::Role => Permission::SysRoleCreate, + ObjectType::Search => Permission::SysSearchUpdate, + ObjectType::SearchStore => Permission::SysSearchStoreUpdate, + ObjectType::Security => Permission::SysSecurityUpdate, + ObjectType::SenderAuth => Permission::SysSenderAuthUpdate, + ObjectType::Sharing => Permission::SysSharingUpdate, + ObjectType::SieveSystemInterpreter => Permission::SysSieveSystemInterpreterUpdate, + ObjectType::SieveSystemScript => Permission::SysSieveSystemScriptCreate, + ObjectType::SieveUserInterpreter => Permission::SysSieveUserInterpreterUpdate, + ObjectType::SieveUserScript => Permission::SysSieveUserScriptCreate, + ObjectType::SpamClassifier => Permission::SysSpamClassifierUpdate, + ObjectType::SpamDnsblServer => Permission::SysSpamDnsblServerCreate, + ObjectType::SpamDnsblSettings => Permission::SysSpamDnsblSettingsUpdate, + ObjectType::SpamFileExtension => Permission::SysSpamFileExtensionCreate, + ObjectType::SpamPyzor => Permission::SysSpamPyzorUpdate, + ObjectType::SpamRule => Permission::SysSpamRuleCreate, + ObjectType::SpamSettings => Permission::SysSpamSettingsUpdate, + ObjectType::SpamTag => Permission::SysSpamTagCreate, + ObjectType::SpamTrainingSample => Permission::SysSpamTrainingSampleCreate, + ObjectType::SpfReportSettings => Permission::SysSpfReportSettingsUpdate, + ObjectType::StoreLookup => Permission::SysStoreLookupCreate, + ObjectType::SystemSettings => Permission::SysSystemSettingsUpdate, + ObjectType::Task => Permission::SysTaskCreate, + ObjectType::TaskManager => Permission::SysTaskManagerUpdate, + ObjectType::TlsExternalReport => unreachable!(), + ObjectType::TlsInternalReport => unreachable!(), + ObjectType::TlsReportSettings => Permission::SysTlsReportSettingsUpdate, + ObjectType::Tracer => Permission::SysTracerCreate, + ObjectType::WebDav => Permission::SysWebDavUpdate, + ObjectType::WebHook => Permission::SysWebHookCreate, + } + } + + pub fn update_permission(&self) -> Permission { + match self { + ObjectType::Account => Permission::SysAccountUpdate, + ObjectType::AccountPassword => Permission::SysAccountPasswordUpdate, + ObjectType::AccountSettings => Permission::SysAccountSettingsUpdate, + ObjectType::AcmeProvider => Permission::SysAcmeProviderUpdate, + ObjectType::Action => unreachable!(), + ObjectType::AddressBook => Permission::SysAddressBookUpdate, + ObjectType::AllowedIp => Permission::SysAllowedIpUpdate, + ObjectType::ApiKey => Permission::SysApiKeyUpdate, + ObjectType::AppPassword => Permission::SysAppPasswordUpdate, + ObjectType::Application => Permission::SysApplicationUpdate, + ObjectType::ArfExternalReport => unreachable!(), + ObjectType::Asn => Permission::SysAsnUpdate, + ObjectType::Authentication => Permission::SysAuthenticationUpdate, + ObjectType::BlobStore => Permission::SysBlobStoreUpdate, + ObjectType::BlockedIp => Permission::SysBlockedIpUpdate, + ObjectType::Bootstrap => Permission::SysBootstrapUpdate, + ObjectType::Cache => Permission::SysCacheUpdate, + ObjectType::Calendar => Permission::SysCalendarUpdate, + ObjectType::CalendarAlarm => Permission::SysCalendarAlarmUpdate, + ObjectType::CalendarScheduling => Permission::SysCalendarSchedulingUpdate, + ObjectType::Certificate => Permission::SysCertificateUpdate, + ObjectType::ClusterNode => unreachable!(), + ObjectType::ClusterRole => Permission::SysClusterRoleUpdate, + ObjectType::Coordinator => Permission::SysCoordinatorUpdate, + ObjectType::DataRetention => Permission::SysDataRetentionUpdate, + ObjectType::DataStore => Permission::SysDataStoreUpdate, + ObjectType::Directory => Permission::SysDirectoryUpdate, + ObjectType::DkimReportSettings => Permission::SysDkimReportSettingsUpdate, + ObjectType::DkimSignature => Permission::SysDkimSignatureUpdate, + ObjectType::DmarcExternalReport => unreachable!(), + ObjectType::DmarcInternalReport => Permission::SysDmarcInternalReportUpdate, + ObjectType::DmarcReportSettings => Permission::SysDmarcReportSettingsUpdate, + ObjectType::DnsResolver => Permission::SysDnsResolverUpdate, + ObjectType::DnsServer => Permission::SysDnsServerUpdate, + ObjectType::Domain => Permission::SysDomainUpdate, + ObjectType::DsnReportSettings => Permission::SysDsnReportSettingsUpdate, + ObjectType::Email => Permission::SysEmailUpdate, + ObjectType::EventTracingLevel => Permission::SysEventTracingLevelUpdate, + ObjectType::FileStorage => Permission::SysFileStorageUpdate, + ObjectType::Http => Permission::SysHttpUpdate, + ObjectType::HttpForm => Permission::SysHttpFormUpdate, + ObjectType::HttpLookup => Permission::SysHttpLookupUpdate, + ObjectType::Imap => Permission::SysImapUpdate, + ObjectType::InMemoryStore => Permission::SysInMemoryStoreUpdate, + ObjectType::IndexQueueStatus => Permission::SysIndexQueueStatusUpdate, + ObjectType::Jmap => Permission::SysJmapUpdate, + ObjectType::Log => unreachable!(), + ObjectType::MailingList => Permission::SysMailingListUpdate, + ObjectType::MemoryLookupKey => Permission::SysMemoryLookupKeyUpdate, + ObjectType::MemoryLookupKeyValue => Permission::SysMemoryLookupKeyValueUpdate, + ObjectType::Metadata => Permission::SysMetadataUpdate, + ObjectType::Metrics => Permission::SysMetricsUpdate, + ObjectType::MtaConnectionStrategy => Permission::SysMtaConnectionStrategyUpdate, + ObjectType::MtaDeliverySchedule => Permission::SysMtaDeliveryScheduleUpdate, + ObjectType::MtaExtensions => Permission::SysMtaExtensionsUpdate, + ObjectType::MtaHook => Permission::SysMtaHookUpdate, + ObjectType::MtaInboundSession => Permission::SysMtaInboundSessionUpdate, + ObjectType::MtaInboundThrottle => Permission::SysMtaInboundThrottleUpdate, + ObjectType::MtaMilter => Permission::SysMtaMilterUpdate, + ObjectType::MtaOutboundStrategy => Permission::SysMtaOutboundStrategyUpdate, + ObjectType::MtaOutboundThrottle => Permission::SysMtaOutboundThrottleUpdate, + ObjectType::MtaQueueQuota => Permission::SysMtaQueueQuotaUpdate, + ObjectType::MtaRoute => Permission::SysMtaRouteUpdate, + ObjectType::MtaStageAuth => Permission::SysMtaStageAuthUpdate, + ObjectType::MtaStageConnect => Permission::SysMtaStageConnectUpdate, + ObjectType::MtaStageData => Permission::SysMtaStageDataUpdate, + ObjectType::MtaStageEhlo => Permission::SysMtaStageEhloUpdate, + ObjectType::MtaStageMail => Permission::SysMtaStageMailUpdate, + ObjectType::MtaStageRcpt => Permission::SysMtaStageRcptUpdate, + ObjectType::MtaSts => Permission::SysMtaStsUpdate, + ObjectType::MtaTlsStrategy => Permission::SysMtaTlsStrategyUpdate, + ObjectType::MtaVirtualQueue => Permission::SysMtaVirtualQueueUpdate, + ObjectType::NetworkListener => Permission::SysNetworkListenerUpdate, + ObjectType::OAuthClient => Permission::SysOAuthClientUpdate, + ObjectType::OAuthGrant => unreachable!(), + ObjectType::OidcProvider => Permission::SysOidcProviderUpdate, + ObjectType::PublicKey => Permission::SysPublicKeyUpdate, + ObjectType::QueuedMessage => Permission::SysQueuedMessageUpdate, + ObjectType::ReportSettings => Permission::SysReportSettingsUpdate, + ObjectType::Role => Permission::SysRoleUpdate, + ObjectType::Search => Permission::SysSearchUpdate, + ObjectType::SearchStore => Permission::SysSearchStoreUpdate, + ObjectType::Security => Permission::SysSecurityUpdate, + ObjectType::SenderAuth => Permission::SysSenderAuthUpdate, + ObjectType::Sharing => Permission::SysSharingUpdate, + ObjectType::SieveSystemInterpreter => Permission::SysSieveSystemInterpreterUpdate, + ObjectType::SieveSystemScript => Permission::SysSieveSystemScriptUpdate, + ObjectType::SieveUserInterpreter => Permission::SysSieveUserInterpreterUpdate, + ObjectType::SieveUserScript => Permission::SysSieveUserScriptUpdate, + ObjectType::SpamClassifier => Permission::SysSpamClassifierUpdate, + ObjectType::SpamDnsblServer => Permission::SysSpamDnsblServerUpdate, + ObjectType::SpamDnsblSettings => Permission::SysSpamDnsblSettingsUpdate, + ObjectType::SpamFileExtension => Permission::SysSpamFileExtensionUpdate, + ObjectType::SpamPyzor => Permission::SysSpamPyzorUpdate, + ObjectType::SpamRule => Permission::SysSpamRuleUpdate, + ObjectType::SpamSettings => Permission::SysSpamSettingsUpdate, + ObjectType::SpamTag => Permission::SysSpamTagUpdate, + ObjectType::SpamTrainingSample => unreachable!(), + ObjectType::SpfReportSettings => Permission::SysSpfReportSettingsUpdate, + ObjectType::StoreLookup => Permission::SysStoreLookupUpdate, + ObjectType::SystemSettings => Permission::SysSystemSettingsUpdate, + ObjectType::Task => Permission::SysTaskUpdate, + ObjectType::TaskManager => Permission::SysTaskManagerUpdate, + ObjectType::TlsExternalReport => unreachable!(), + ObjectType::TlsInternalReport => Permission::SysTlsInternalReportUpdate, + ObjectType::TlsReportSettings => Permission::SysTlsReportSettingsUpdate, + ObjectType::Tracer => Permission::SysTracerUpdate, + ObjectType::WebDav => Permission::SysWebDavUpdate, + ObjectType::WebHook => Permission::SysWebHookUpdate, + } + } + + pub fn destroy_permission(&self) -> Permission { + match self { + ObjectType::Account => Permission::SysAccountDestroy, + ObjectType::AccountPassword => Permission::SysAccountPasswordUpdate, + ObjectType::AccountSettings => Permission::SysAccountSettingsUpdate, + ObjectType::AcmeProvider => Permission::SysAcmeProviderDestroy, + ObjectType::Action => unreachable!(), + ObjectType::AddressBook => Permission::SysAddressBookUpdate, + ObjectType::AllowedIp => Permission::SysAllowedIpDestroy, + ObjectType::ApiKey => Permission::SysApiKeyDestroy, + ObjectType::AppPassword => Permission::SysAppPasswordDestroy, + ObjectType::Application => Permission::SysApplicationDestroy, + ObjectType::ArfExternalReport => Permission::SysArfExternalReportDestroy, + ObjectType::Asn => Permission::SysAsnUpdate, + ObjectType::Authentication => Permission::SysAuthenticationUpdate, + ObjectType::BlobStore => Permission::SysBlobStoreUpdate, + ObjectType::BlockedIp => Permission::SysBlockedIpDestroy, + ObjectType::Bootstrap => Permission::SysBootstrapUpdate, + ObjectType::Cache => Permission::SysCacheUpdate, + ObjectType::Calendar => Permission::SysCalendarUpdate, + ObjectType::CalendarAlarm => Permission::SysCalendarAlarmUpdate, + ObjectType::CalendarScheduling => Permission::SysCalendarSchedulingUpdate, + ObjectType::Certificate => Permission::SysCertificateDestroy, + ObjectType::ClusterNode => unreachable!(), + ObjectType::ClusterRole => Permission::SysClusterRoleDestroy, + ObjectType::Coordinator => Permission::SysCoordinatorUpdate, + ObjectType::DataRetention => Permission::SysDataRetentionUpdate, + ObjectType::DataStore => Permission::SysDataStoreUpdate, + ObjectType::Directory => Permission::SysDirectoryDestroy, + ObjectType::DkimReportSettings => Permission::SysDkimReportSettingsUpdate, + ObjectType::DkimSignature => Permission::SysDkimSignatureDestroy, + ObjectType::DmarcExternalReport => Permission::SysDmarcExternalReportDestroy, + ObjectType::DmarcInternalReport => Permission::SysDmarcInternalReportDestroy, + ObjectType::DmarcReportSettings => Permission::SysDmarcReportSettingsUpdate, + ObjectType::DnsResolver => Permission::SysDnsResolverUpdate, + ObjectType::DnsServer => Permission::SysDnsServerDestroy, + ObjectType::Domain => Permission::SysDomainDestroy, + ObjectType::DsnReportSettings => Permission::SysDsnReportSettingsUpdate, + ObjectType::Email => Permission::SysEmailUpdate, + ObjectType::EventTracingLevel => Permission::SysEventTracingLevelDestroy, + ObjectType::FileStorage => Permission::SysFileStorageUpdate, + ObjectType::Http => Permission::SysHttpUpdate, + ObjectType::HttpForm => Permission::SysHttpFormUpdate, + ObjectType::HttpLookup => Permission::SysHttpLookupDestroy, + ObjectType::Imap => Permission::SysImapUpdate, + ObjectType::InMemoryStore => Permission::SysInMemoryStoreUpdate, + ObjectType::IndexQueueStatus => unreachable!(), + ObjectType::Jmap => Permission::SysJmapUpdate, + ObjectType::Log => unreachable!(), + ObjectType::MailingList => Permission::SysMailingListDestroy, + ObjectType::MemoryLookupKey => Permission::SysMemoryLookupKeyDestroy, + ObjectType::MemoryLookupKeyValue => Permission::SysMemoryLookupKeyValueDestroy, + ObjectType::Metadata => Permission::SysMetadataUpdate, + ObjectType::Metrics => Permission::SysMetricsUpdate, + ObjectType::MtaConnectionStrategy => Permission::SysMtaConnectionStrategyDestroy, + ObjectType::MtaDeliverySchedule => Permission::SysMtaDeliveryScheduleDestroy, + ObjectType::MtaExtensions => Permission::SysMtaExtensionsUpdate, + ObjectType::MtaHook => Permission::SysMtaHookDestroy, + ObjectType::MtaInboundSession => Permission::SysMtaInboundSessionUpdate, + ObjectType::MtaInboundThrottle => Permission::SysMtaInboundThrottleDestroy, + ObjectType::MtaMilter => Permission::SysMtaMilterDestroy, + ObjectType::MtaOutboundStrategy => Permission::SysMtaOutboundStrategyUpdate, + ObjectType::MtaOutboundThrottle => Permission::SysMtaOutboundThrottleDestroy, + ObjectType::MtaQueueQuota => Permission::SysMtaQueueQuotaDestroy, + ObjectType::MtaRoute => Permission::SysMtaRouteDestroy, + ObjectType::MtaStageAuth => Permission::SysMtaStageAuthUpdate, + ObjectType::MtaStageConnect => Permission::SysMtaStageConnectUpdate, + ObjectType::MtaStageData => Permission::SysMtaStageDataUpdate, + ObjectType::MtaStageEhlo => Permission::SysMtaStageEhloUpdate, + ObjectType::MtaStageMail => Permission::SysMtaStageMailUpdate, + ObjectType::MtaStageRcpt => Permission::SysMtaStageRcptUpdate, + ObjectType::MtaSts => Permission::SysMtaStsUpdate, + ObjectType::MtaTlsStrategy => Permission::SysMtaTlsStrategyDestroy, + ObjectType::MtaVirtualQueue => Permission::SysMtaVirtualQueueDestroy, + ObjectType::NetworkListener => Permission::SysNetworkListenerDestroy, + ObjectType::OAuthClient => Permission::SysOAuthClientDestroy, + ObjectType::OAuthGrant => Permission::SysOAuthGrantDestroy, + ObjectType::OidcProvider => Permission::SysOidcProviderUpdate, + ObjectType::PublicKey => Permission::SysPublicKeyDestroy, + ObjectType::QueuedMessage => Permission::SysQueuedMessageDestroy, + ObjectType::ReportSettings => Permission::SysReportSettingsUpdate, + ObjectType::Role => Permission::SysRoleDestroy, + ObjectType::Search => Permission::SysSearchUpdate, + ObjectType::SearchStore => Permission::SysSearchStoreUpdate, + ObjectType::Security => Permission::SysSecurityUpdate, + ObjectType::SenderAuth => Permission::SysSenderAuthUpdate, + ObjectType::Sharing => Permission::SysSharingUpdate, + ObjectType::SieveSystemInterpreter => Permission::SysSieveSystemInterpreterUpdate, + ObjectType::SieveSystemScript => Permission::SysSieveSystemScriptDestroy, + ObjectType::SieveUserInterpreter => Permission::SysSieveUserInterpreterUpdate, + ObjectType::SieveUserScript => Permission::SysSieveUserScriptDestroy, + ObjectType::SpamClassifier => Permission::SysSpamClassifierUpdate, + ObjectType::SpamDnsblServer => Permission::SysSpamDnsblServerDestroy, + ObjectType::SpamDnsblSettings => Permission::SysSpamDnsblSettingsUpdate, + ObjectType::SpamFileExtension => Permission::SysSpamFileExtensionDestroy, + ObjectType::SpamPyzor => Permission::SysSpamPyzorUpdate, + ObjectType::SpamRule => Permission::SysSpamRuleDestroy, + ObjectType::SpamSettings => Permission::SysSpamSettingsUpdate, + ObjectType::SpamTag => Permission::SysSpamTagDestroy, + ObjectType::SpamTrainingSample => Permission::SysSpamTrainingSampleDestroy, + ObjectType::SpfReportSettings => Permission::SysSpfReportSettingsUpdate, + ObjectType::StoreLookup => Permission::SysStoreLookupDestroy, + ObjectType::SystemSettings => Permission::SysSystemSettingsUpdate, + ObjectType::Task => Permission::SysTaskDestroy, + ObjectType::TaskManager => Permission::SysTaskManagerUpdate, + ObjectType::TlsExternalReport => Permission::SysTlsExternalReportDestroy, + ObjectType::TlsInternalReport => Permission::SysTlsInternalReportDestroy, + ObjectType::TlsReportSettings => Permission::SysTlsReportSettingsUpdate, + ObjectType::Tracer => Permission::SysTracerDestroy, + ObjectType::WebDav => Permission::SysWebDavUpdate, + ObjectType::WebHook => Permission::SysWebHookDestroy, } } } impl ObjectInner { - pub fn member_tenant_id(&self) -> Option { - match self { - ObjectInner::Account(Account::User(obj)) => obj.member_tenant_id, - ObjectInner::Account(Account::Group(obj)) => obj.member_tenant_id, - ObjectInner::AcmeProvider(obj) => obj.member_tenant_id, - ObjectInner::ArfExternalReport(obj) => obj.member_tenant_id, - ObjectInner::Directory(Directory::Ldap(obj)) => obj.member_tenant_id, - ObjectInner::Directory(Directory::Sql(obj)) => obj.member_tenant_id, - ObjectInner::Directory(Directory::Oidc(obj)) => obj.member_tenant_id, - ObjectInner::DkimSignature(DkimSignature::Dkim1Ed25519Sha256(obj)) => { - obj.member_tenant_id - } - ObjectInner::DkimSignature(DkimSignature::Dkim1RsaSha256(obj)) => obj.member_tenant_id, - ObjectInner::DkimSignature(DkimSignature::Dkim2Ed25519Sha256(obj)) => { - obj.member_tenant_id - } - ObjectInner::DkimSignature(DkimSignature::Dkim2RsaSha256(obj)) => obj.member_tenant_id, - ObjectInner::DmarcExternalReport(obj) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Tsig(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Cloudflare(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::DigitalOcean(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::DeSEC(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Ovh(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Bunny(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Porkbun(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Dnsimple(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Spaceship(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Route53(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::GoogleCloudDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Alidns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::ArvanCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Autodns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::AzureDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::BaiduCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::BluecatV2(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::ClouDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Constellix(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Cpanel(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Ddnss(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::DnsMadeEasy(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Domeneshop(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Dreamhost(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::DuckDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Dynu(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::EasyDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::EdgeDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Exoscale(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::FreeMyIp(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::GandiV5(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Gcore(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Glesys(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Godaddy(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Hetzner(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::HostingDe(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Hostinger(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::HuaweiCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Hurricane(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::IbmCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Infoblox(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Infomaniak(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Inwx(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Ionos(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Ipv64(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Joker(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Lightsail(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Linode(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::LuaDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::MythicBeasts(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Namecheap(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::NameDotCom(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::NameSilo(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Netcup(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Netlify(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Nifcloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Ns1(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::OracleCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Plesk(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Safedns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Scaleway(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::TencentCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Transip(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::UltraDns(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Vercel(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Volcengine(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::Vultr(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::WebSupport(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::YandexCloud(obj)) => obj.member_tenant_id, - ObjectInner::DnsServer(DnsServer::PowerDns(obj)) => obj.member_tenant_id, - ObjectInner::Domain(obj) => obj.member_tenant_id, - ObjectInner::MailingList(obj) => obj.member_tenant_id, - ObjectInner::OAuthClient(obj) => obj.member_tenant_id, - ObjectInner::Role(obj) => obj.member_tenant_id, - ObjectInner::TlsExternalReport(obj) => obj.member_tenant_id, - _ => None, - } - } - - pub fn set_member_tenant_id(&mut self, id: Id) { - match self { - ObjectInner::Account(Account::User(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::Account(Account::Group(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::AcmeProvider(obj) => obj.member_tenant_id = Some(id), - ObjectInner::ArfExternalReport(obj) => obj.member_tenant_id = Some(id), - ObjectInner::Directory(Directory::Ldap(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::Directory(Directory::Sql(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::Directory(Directory::Oidc(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DkimSignature(DkimSignature::Dkim1Ed25519Sha256(obj)) => { - obj.member_tenant_id = Some(id) - } - ObjectInner::DkimSignature(DkimSignature::Dkim1RsaSha256(obj)) => { - obj.member_tenant_id = Some(id) - } - ObjectInner::DkimSignature(DkimSignature::Dkim2Ed25519Sha256(obj)) => { - obj.member_tenant_id = Some(id) - } - ObjectInner::DkimSignature(DkimSignature::Dkim2RsaSha256(obj)) => { - obj.member_tenant_id = Some(id) - } - ObjectInner::DmarcExternalReport(obj) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Tsig(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Cloudflare(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::DigitalOcean(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::DeSEC(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Ovh(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Bunny(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Porkbun(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Dnsimple(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Spaceship(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Route53(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::GoogleCloudDns(obj)) => { - obj.member_tenant_id = Some(id) - } - ObjectInner::DnsServer(DnsServer::Alidns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::ArvanCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Autodns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::AzureDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::BaiduCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::BluecatV2(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::ClouDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Constellix(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Cpanel(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Ddnss(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::DnsMadeEasy(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Domeneshop(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Dreamhost(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::DuckDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Dynu(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::EasyDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::EdgeDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Exoscale(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::FreeMyIp(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::GandiV5(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Gcore(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Glesys(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Godaddy(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Hetzner(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::HostingDe(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Hostinger(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::HuaweiCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Hurricane(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::IbmCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Infoblox(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Infomaniak(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Inwx(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Ionos(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Ipv64(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Joker(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Lightsail(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Linode(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::LuaDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::MythicBeasts(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Namecheap(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::NameDotCom(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::NameSilo(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Netcup(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Netlify(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Nifcloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Ns1(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::OracleCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Plesk(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Safedns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Scaleway(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::TencentCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Transip(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::UltraDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Vercel(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Volcengine(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::Vultr(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::WebSupport(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::YandexCloud(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::DnsServer(DnsServer::PowerDns(obj)) => obj.member_tenant_id = Some(id), - ObjectInner::Domain(obj) => obj.member_tenant_id = Some(id), - ObjectInner::MailingList(obj) => obj.member_tenant_id = Some(id), - ObjectInner::OAuthClient(obj) => obj.member_tenant_id = Some(id), - ObjectInner::Role(obj) => obj.member_tenant_id = Some(id), - ObjectInner::TlsExternalReport(obj) => obj.member_tenant_id = Some(id), - _ => {} - } - } - pub fn account_id(&self) -> Option { match self { - ObjectInner::ArchivedItem(ArchivedItem::Email(obj)) => Some(obj.account_id), - ObjectInner::ArchivedItem(ArchivedItem::FileNode(obj)) => Some(obj.account_id), - ObjectInner::ArchivedItem(ArchivedItem::CalendarEvent(obj)) => Some(obj.account_id), - ObjectInner::ArchivedItem(ArchivedItem::ContactCard(obj)) => Some(obj.account_id), - ObjectInner::ArchivedItem(ArchivedItem::SieveScript(obj)) => Some(obj.account_id), - ObjectInner::MaskedEmail(obj) => Some(obj.account_id), + ObjectInner::OAuthGrant(obj) => Some(obj.account_id), ObjectInner::PublicKey(obj) => Some(obj.account_id), ObjectInner::SpamTrainingSample(obj) => obj.account_id, - ObjectInner::Task(Task::IndexDocument(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::UnindexDocument(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::CalendarAlarmEmail(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::CalendarAlarmNotification(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::CalendarItipMessage(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::MergeThreads(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::RestoreArchivedItem(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::DestroyAccount(obj)) => Some(obj.account_id), - ObjectInner::Task(Task::AccountMaintenance(obj)) => Some(obj.account_id), + ObjectInner::Task(inner) => match inner.as_ref() { + Task::CalendarAlarmEmail(obj) => Some(obj.account_id), + Task::CalendarAlarmNotification(obj) => Some(obj.account_id), + Task::CalendarItipMessage(obj) => Some(obj.account_id), + Task::MergeThreads(obj) => Some(obj.account_id), + Task::DestroyAccount(obj) => Some(obj.account_id), + Task::AccountMaintenance(obj) => Some(obj.account_id), + Task::ShareRevokeCleanup(obj) => Some(obj.account_id), + _ => None, + }, _ => None, } } pub fn set_account_id(&mut self, id: Id) { match self { - ObjectInner::ArchivedItem(ArchivedItem::Email(obj)) => obj.account_id = id, - ObjectInner::ArchivedItem(ArchivedItem::FileNode(obj)) => obj.account_id = id, - ObjectInner::ArchivedItem(ArchivedItem::CalendarEvent(obj)) => obj.account_id = id, - ObjectInner::ArchivedItem(ArchivedItem::ContactCard(obj)) => obj.account_id = id, - ObjectInner::ArchivedItem(ArchivedItem::SieveScript(obj)) => obj.account_id = id, - ObjectInner::MaskedEmail(obj) => obj.account_id = id, + ObjectInner::OAuthGrant(obj) => obj.account_id = id, ObjectInner::PublicKey(obj) => obj.account_id = id, ObjectInner::SpamTrainingSample(obj) => obj.account_id = Some(id), - ObjectInner::Task(Task::IndexDocument(obj)) => obj.account_id = id, - ObjectInner::Task(Task::UnindexDocument(obj)) => obj.account_id = id, - ObjectInner::Task(Task::CalendarAlarmEmail(obj)) => obj.account_id = id, - ObjectInner::Task(Task::CalendarAlarmNotification(obj)) => obj.account_id = id, - ObjectInner::Task(Task::CalendarItipMessage(obj)) => obj.account_id = id, - ObjectInner::Task(Task::MergeThreads(obj)) => obj.account_id = id, - ObjectInner::Task(Task::RestoreArchivedItem(obj)) => obj.account_id = id, - ObjectInner::Task(Task::DestroyAccount(obj)) => obj.account_id = id, - ObjectInner::Task(Task::AccountMaintenance(obj)) => obj.account_id = id, + ObjectInner::Task(inner) => match inner.as_mut() { + Task::CalendarAlarmEmail(obj) => obj.account_id = id, + Task::CalendarAlarmNotification(obj) => obj.account_id = id, + Task::CalendarItipMessage(obj) => obj.account_id = id, + Task::MergeThreads(obj) => obj.account_id = id, + Task::DestroyAccount(obj) => obj.account_id = id, + Task::AccountMaintenance(obj) => obj.account_id = id, + Task::ShareRevokeCleanup(obj) => obj.account_id = id, + _ => {} + }, _ => {} } } @@ -4658,34 +4112,31 @@ impl ObjectInner { pub fn to_pickled_vec(&self) -> Vec { match &self { ObjectInner::Account(obj) => obj.to_pickled_vec(), - ObjectInner::AccountPassword(obj) => obj.to_pickled_vec(), - ObjectInner::AccountSettings(obj) => obj.to_pickled_vec(), + ObjectInner::AccountPassword(_) => unimplemented!("AccountPassword is not stored"), + ObjectInner::AccountSettings(_) => unimplemented!("AccountSettings is not stored"), ObjectInner::AcmeProvider(obj) => obj.to_pickled_vec(), - ObjectInner::Action(obj) => obj.to_pickled_vec(), + ObjectInner::Action(_) => unimplemented!("Action is not stored"), ObjectInner::AddressBook(obj) => obj.to_pickled_vec(), - ObjectInner::AiModel(obj) => obj.to_pickled_vec(), - ObjectInner::Alert(obj) => obj.to_pickled_vec(), ObjectInner::AllowedIp(obj) => obj.to_pickled_vec(), - ObjectInner::ApiKey(obj) => obj.to_pickled_vec(), - ObjectInner::AppPassword(obj) => obj.to_pickled_vec(), + ObjectInner::ApiKey(_) => unimplemented!("ApiKey is not stored"), + ObjectInner::AppPassword(_) => unimplemented!("AppPassword is not stored"), ObjectInner::Application(obj) => obj.to_pickled_vec(), - ObjectInner::ArchivedItem(obj) => obj.to_pickled_vec(), ObjectInner::ArfExternalReport(obj) => obj.to_pickled_vec(), ObjectInner::Asn(obj) => obj.to_pickled_vec(), ObjectInner::Authentication(obj) => obj.to_pickled_vec(), ObjectInner::BlobStore(obj) => obj.to_pickled_vec(), ObjectInner::BlockedIp(obj) => obj.to_pickled_vec(), - ObjectInner::Bootstrap(obj) => obj.to_pickled_vec(), + ObjectInner::Bootstrap(_) => unimplemented!("Bootstrap is not stored"), ObjectInner::Cache(obj) => obj.to_pickled_vec(), ObjectInner::Calendar(obj) => obj.to_pickled_vec(), ObjectInner::CalendarAlarm(obj) => obj.to_pickled_vec(), ObjectInner::CalendarScheduling(obj) => obj.to_pickled_vec(), ObjectInner::Certificate(obj) => obj.to_pickled_vec(), - ObjectInner::ClusterNode(obj) => obj.to_pickled_vec(), + ObjectInner::ClusterNode(_) => unimplemented!("ClusterNode is not stored"), ObjectInner::ClusterRole(obj) => obj.to_pickled_vec(), ObjectInner::Coordinator(obj) => obj.to_pickled_vec(), ObjectInner::DataRetention(obj) => obj.to_pickled_vec(), - ObjectInner::DataStore(obj) => obj.to_pickled_vec(), + ObjectInner::DataStore(_) => unimplemented!("DataStore is not stored"), ObjectInner::Directory(obj) => obj.to_pickled_vec(), ObjectInner::DkimReportSettings(obj) => obj.to_pickled_vec(), ObjectInner::DkimSignature(obj) => obj.to_pickled_vec(), @@ -4697,7 +4148,6 @@ impl ObjectInner { ObjectInner::Domain(obj) => obj.to_pickled_vec(), ObjectInner::DsnReportSettings(obj) => obj.to_pickled_vec(), ObjectInner::Email(obj) => obj.to_pickled_vec(), - ObjectInner::Enterprise(obj) => obj.to_pickled_vec(), ObjectInner::EventTracingLevel(obj) => obj.to_pickled_vec(), ObjectInner::FileStorage(obj) => obj.to_pickled_vec(), ObjectInner::Http(obj) => obj.to_pickled_vec(), @@ -4705,15 +4155,14 @@ impl ObjectInner { ObjectInner::HttpLookup(obj) => obj.to_pickled_vec(), ObjectInner::Imap(obj) => obj.to_pickled_vec(), ObjectInner::InMemoryStore(obj) => obj.to_pickled_vec(), + ObjectInner::IndexQueueStatus(_) => unimplemented!("IndexQueueStatus is not stored"), ObjectInner::Jmap(obj) => obj.to_pickled_vec(), - ObjectInner::Log(obj) => obj.to_pickled_vec(), + ObjectInner::Log(_) => unimplemented!("Log is not stored"), ObjectInner::MailingList(obj) => obj.to_pickled_vec(), - ObjectInner::MaskedEmail(obj) => obj.to_pickled_vec(), ObjectInner::MemoryLookupKey(obj) => obj.to_pickled_vec(), ObjectInner::MemoryLookupKeyValue(obj) => obj.to_pickled_vec(), - ObjectInner::Metric(obj) => obj.to_pickled_vec(), + ObjectInner::Metadata(obj) => obj.to_pickled_vec(), ObjectInner::Metrics(obj) => obj.to_pickled_vec(), - ObjectInner::MetricsStore(obj) => obj.to_pickled_vec(), ObjectInner::MtaConnectionStrategy(obj) => obj.to_pickled_vec(), ObjectInner::MtaDeliverySchedule(obj) => obj.to_pickled_vec(), ObjectInner::MtaExtensions(obj) => obj.to_pickled_vec(), @@ -4736,9 +4185,10 @@ impl ObjectInner { ObjectInner::MtaVirtualQueue(obj) => obj.to_pickled_vec(), ObjectInner::NetworkListener(obj) => obj.to_pickled_vec(), ObjectInner::OAuthClient(obj) => obj.to_pickled_vec(), + ObjectInner::OAuthGrant(obj) => obj.to_pickled_vec(), ObjectInner::OidcProvider(obj) => obj.to_pickled_vec(), ObjectInner::PublicKey(obj) => obj.to_pickled_vec(), - ObjectInner::QueuedMessage(obj) => obj.to_pickled_vec(), + ObjectInner::QueuedMessage(_) => unimplemented!("QueuedMessage is not stored"), ObjectInner::ReportSettings(obj) => obj.to_pickled_vec(), ObjectInner::Role(obj) => obj.to_pickled_vec(), ObjectInner::Search(obj) => obj.to_pickled_vec(), @@ -4754,7 +4204,6 @@ impl ObjectInner { ObjectInner::SpamDnsblServer(obj) => obj.to_pickled_vec(), ObjectInner::SpamDnsblSettings(obj) => obj.to_pickled_vec(), ObjectInner::SpamFileExtension(obj) => obj.to_pickled_vec(), - ObjectInner::SpamLlm(obj) => obj.to_pickled_vec(), ObjectInner::SpamPyzor(obj) => obj.to_pickled_vec(), ObjectInner::SpamRule(obj) => obj.to_pickled_vec(), ObjectInner::SpamSettings(obj) => obj.to_pickled_vec(), @@ -4765,13 +4214,10 @@ impl ObjectInner { ObjectInner::SystemSettings(obj) => obj.to_pickled_vec(), ObjectInner::Task(obj) => obj.to_pickled_vec(), ObjectInner::TaskManager(obj) => obj.to_pickled_vec(), - ObjectInner::Tenant(obj) => obj.to_pickled_vec(), ObjectInner::TlsExternalReport(obj) => obj.to_pickled_vec(), ObjectInner::TlsInternalReport(obj) => obj.to_pickled_vec(), ObjectInner::TlsReportSettings(obj) => obj.to_pickled_vec(), - ObjectInner::Trace(obj) => obj.to_pickled_vec(), ObjectInner::Tracer(obj) => obj.to_pickled_vec(), - ObjectInner::TracingStore(obj) => obj.to_pickled_vec(), ObjectInner::WebDav(obj) => obj.to_pickled_vec(), ObjectInner::WebHook(obj) => obj.to_pickled_vec(), } @@ -4782,189 +4228,307 @@ impl ObjectInner { stream: &mut crate::pickle::PickledStream<'_>, ) -> Option { match object { - ObjectType::Account => Pickle::unpickle(stream).map(ObjectInner::Account), - ObjectType::AccountPassword => { - Pickle::unpickle(stream).map(ObjectInner::AccountPassword) + ObjectType::Account => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Account(Box::new(obj))) } - ObjectType::AccountSettings => { - Pickle::unpickle(stream).map(ObjectInner::AccountSettings) + ObjectType::AccountPassword => unimplemented!("AccountPassword is not stored"), + ObjectType::AccountSettings => unimplemented!("AccountSettings is not stored"), + ObjectType::AcmeProvider => { + Pickle::unpickle(stream).map(|obj| ObjectInner::AcmeProvider(Box::new(obj))) + } + ObjectType::Action => unimplemented!("Action is not stored"), + ObjectType::AddressBook => { + Pickle::unpickle(stream).map(|obj| ObjectInner::AddressBook(Box::new(obj))) + } + ObjectType::AllowedIp => { + Pickle::unpickle(stream).map(|obj| ObjectInner::AllowedIp(Box::new(obj))) + } + ObjectType::ApiKey => unimplemented!("ApiKey is not stored"), + ObjectType::AppPassword => unimplemented!("AppPassword is not stored"), + ObjectType::Application => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Application(Box::new(obj))) } - ObjectType::AcmeProvider => Pickle::unpickle(stream).map(ObjectInner::AcmeProvider), - ObjectType::Action => Pickle::unpickle(stream).map(ObjectInner::Action), - ObjectType::AddressBook => Pickle::unpickle(stream).map(ObjectInner::AddressBook), - ObjectType::AiModel => Pickle::unpickle(stream).map(ObjectInner::AiModel), - ObjectType::Alert => Pickle::unpickle(stream).map(ObjectInner::Alert), - ObjectType::AllowedIp => Pickle::unpickle(stream).map(ObjectInner::AllowedIp), - ObjectType::ApiKey => Pickle::unpickle(stream).map(ObjectInner::ApiKey), - ObjectType::AppPassword => Pickle::unpickle(stream).map(ObjectInner::AppPassword), - ObjectType::Application => Pickle::unpickle(stream).map(ObjectInner::Application), - ObjectType::ArchivedItem => Pickle::unpickle(stream).map(ObjectInner::ArchivedItem), ObjectType::ArfExternalReport => { - Pickle::unpickle(stream).map(ObjectInner::ArfExternalReport) + Pickle::unpickle(stream).map(|obj| ObjectInner::ArfExternalReport(Box::new(obj))) + } + ObjectType::Asn => Pickle::unpickle(stream).map(|obj| ObjectInner::Asn(Box::new(obj))), + ObjectType::Authentication => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Authentication(Box::new(obj))) + } + ObjectType::BlobStore => { + Pickle::unpickle(stream).map(|obj| ObjectInner::BlobStore(Box::new(obj))) + } + ObjectType::BlockedIp => { + Pickle::unpickle(stream).map(|obj| ObjectInner::BlockedIp(Box::new(obj))) + } + ObjectType::Bootstrap => unimplemented!("Bootstrap is not stored"), + ObjectType::Cache => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Cache(Box::new(obj))) + } + ObjectType::Calendar => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Calendar(Box::new(obj))) + } + ObjectType::CalendarAlarm => { + Pickle::unpickle(stream).map(|obj| ObjectInner::CalendarAlarm(Box::new(obj))) } - ObjectType::Asn => Pickle::unpickle(stream).map(ObjectInner::Asn), - ObjectType::Authentication => Pickle::unpickle(stream).map(ObjectInner::Authentication), - ObjectType::BlobStore => Pickle::unpickle(stream).map(ObjectInner::BlobStore), - ObjectType::BlockedIp => Pickle::unpickle(stream).map(ObjectInner::BlockedIp), - ObjectType::Bootstrap => Pickle::unpickle(stream).map(ObjectInner::Bootstrap), - ObjectType::Cache => Pickle::unpickle(stream).map(ObjectInner::Cache), - ObjectType::Calendar => Pickle::unpickle(stream).map(ObjectInner::Calendar), - ObjectType::CalendarAlarm => Pickle::unpickle(stream).map(ObjectInner::CalendarAlarm), ObjectType::CalendarScheduling => { - Pickle::unpickle(stream).map(ObjectInner::CalendarScheduling) + Pickle::unpickle(stream).map(|obj| ObjectInner::CalendarScheduling(Box::new(obj))) + } + ObjectType::Certificate => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Certificate(Box::new(obj))) + } + ObjectType::ClusterNode => unimplemented!("ClusterNode is not stored"), + ObjectType::ClusterRole => { + Pickle::unpickle(stream).map(|obj| ObjectInner::ClusterRole(Box::new(obj))) + } + ObjectType::Coordinator => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Coordinator(Box::new(obj))) + } + ObjectType::DataRetention => { + Pickle::unpickle(stream).map(|obj| ObjectInner::DataRetention(Box::new(obj))) + } + ObjectType::DataStore => unimplemented!("DataStore is not stored"), + ObjectType::Directory => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Directory(Box::new(obj))) } - ObjectType::Certificate => Pickle::unpickle(stream).map(ObjectInner::Certificate), - ObjectType::ClusterNode => Pickle::unpickle(stream).map(ObjectInner::ClusterNode), - ObjectType::ClusterRole => Pickle::unpickle(stream).map(ObjectInner::ClusterRole), - ObjectType::Coordinator => Pickle::unpickle(stream).map(ObjectInner::Coordinator), - ObjectType::DataRetention => Pickle::unpickle(stream).map(ObjectInner::DataRetention), - ObjectType::DataStore => Pickle::unpickle(stream).map(ObjectInner::DataStore), - ObjectType::Directory => Pickle::unpickle(stream).map(ObjectInner::Directory), ObjectType::DkimReportSettings => { - Pickle::unpickle(stream).map(ObjectInner::DkimReportSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::DkimReportSettings(Box::new(obj))) + } + ObjectType::DkimSignature => { + Pickle::unpickle(stream).map(|obj| ObjectInner::DkimSignature(Box::new(obj))) } - ObjectType::DkimSignature => Pickle::unpickle(stream).map(ObjectInner::DkimSignature), ObjectType::DmarcExternalReport => { - Pickle::unpickle(stream).map(ObjectInner::DmarcExternalReport) + Pickle::unpickle(stream).map(|obj| ObjectInner::DmarcExternalReport(Box::new(obj))) } ObjectType::DmarcInternalReport => { - Pickle::unpickle(stream).map(ObjectInner::DmarcInternalReport) + Pickle::unpickle(stream).map(|obj| ObjectInner::DmarcInternalReport(Box::new(obj))) } ObjectType::DmarcReportSettings => { - Pickle::unpickle(stream).map(ObjectInner::DmarcReportSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::DmarcReportSettings(Box::new(obj))) + } + ObjectType::DnsResolver => { + Pickle::unpickle(stream).map(|obj| ObjectInner::DnsResolver(Box::new(obj))) + } + ObjectType::DnsServer => { + Pickle::unpickle(stream).map(|obj| ObjectInner::DnsServer(Box::new(obj))) + } + ObjectType::Domain => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Domain(Box::new(obj))) } - ObjectType::DnsResolver => Pickle::unpickle(stream).map(ObjectInner::DnsResolver), - ObjectType::DnsServer => Pickle::unpickle(stream).map(ObjectInner::DnsServer), - ObjectType::Domain => Pickle::unpickle(stream).map(ObjectInner::Domain), ObjectType::DsnReportSettings => { - Pickle::unpickle(stream).map(ObjectInner::DsnReportSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::DsnReportSettings(Box::new(obj))) + } + ObjectType::Email => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Email(Box::new(obj))) } - ObjectType::Email => Pickle::unpickle(stream).map(ObjectInner::Email), - ObjectType::Enterprise => Pickle::unpickle(stream).map(ObjectInner::Enterprise), ObjectType::EventTracingLevel => { - Pickle::unpickle(stream).map(ObjectInner::EventTracingLevel) + Pickle::unpickle(stream).map(|obj| ObjectInner::EventTracingLevel(Box::new(obj))) + } + ObjectType::FileStorage => { + Pickle::unpickle(stream).map(|obj| ObjectInner::FileStorage(Box::new(obj))) + } + ObjectType::Http => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Http(Box::new(obj))) + } + ObjectType::HttpForm => { + Pickle::unpickle(stream).map(|obj| ObjectInner::HttpForm(Box::new(obj))) + } + ObjectType::HttpLookup => { + Pickle::unpickle(stream).map(|obj| ObjectInner::HttpLookup(Box::new(obj))) + } + ObjectType::Imap => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Imap(Box::new(obj))) + } + ObjectType::InMemoryStore => { + Pickle::unpickle(stream).map(|obj| ObjectInner::InMemoryStore(Box::new(obj))) + } + ObjectType::IndexQueueStatus => unimplemented!("IndexQueueStatus is not stored"), + ObjectType::Jmap => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Jmap(Box::new(obj))) + } + ObjectType::Log => unimplemented!("Log is not stored"), + ObjectType::MailingList => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MailingList(Box::new(obj))) } - ObjectType::FileStorage => Pickle::unpickle(stream).map(ObjectInner::FileStorage), - ObjectType::Http => Pickle::unpickle(stream).map(ObjectInner::Http), - ObjectType::HttpForm => Pickle::unpickle(stream).map(ObjectInner::HttpForm), - ObjectType::HttpLookup => Pickle::unpickle(stream).map(ObjectInner::HttpLookup), - ObjectType::Imap => Pickle::unpickle(stream).map(ObjectInner::Imap), - ObjectType::InMemoryStore => Pickle::unpickle(stream).map(ObjectInner::InMemoryStore), - ObjectType::Jmap => Pickle::unpickle(stream).map(ObjectInner::Jmap), - ObjectType::Log => Pickle::unpickle(stream).map(ObjectInner::Log), - ObjectType::MailingList => Pickle::unpickle(stream).map(ObjectInner::MailingList), - ObjectType::MaskedEmail => Pickle::unpickle(stream).map(ObjectInner::MaskedEmail), ObjectType::MemoryLookupKey => { - Pickle::unpickle(stream).map(ObjectInner::MemoryLookupKey) + Pickle::unpickle(stream).map(|obj| ObjectInner::MemoryLookupKey(Box::new(obj))) } ObjectType::MemoryLookupKeyValue => { - Pickle::unpickle(stream).map(ObjectInner::MemoryLookupKeyValue) + Pickle::unpickle(stream).map(|obj| ObjectInner::MemoryLookupKeyValue(Box::new(obj))) } - ObjectType::Metric => Pickle::unpickle(stream).map(ObjectInner::Metric), - ObjectType::Metrics => Pickle::unpickle(stream).map(ObjectInner::Metrics), - ObjectType::MetricsStore => Pickle::unpickle(stream).map(ObjectInner::MetricsStore), - ObjectType::MtaConnectionStrategy => { - Pickle::unpickle(stream).map(ObjectInner::MtaConnectionStrategy) + ObjectType::Metadata => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Metadata(Box::new(obj))) } + ObjectType::Metrics => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Metrics(Box::new(obj))) + } + ObjectType::MtaConnectionStrategy => Pickle::unpickle(stream) + .map(|obj| ObjectInner::MtaConnectionStrategy(Box::new(obj))), ObjectType::MtaDeliverySchedule => { - Pickle::unpickle(stream).map(ObjectInner::MtaDeliverySchedule) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaDeliverySchedule(Box::new(obj))) + } + ObjectType::MtaExtensions => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaExtensions(Box::new(obj))) + } + ObjectType::MtaHook => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaHook(Box::new(obj))) } - ObjectType::MtaExtensions => Pickle::unpickle(stream).map(ObjectInner::MtaExtensions), - ObjectType::MtaHook => Pickle::unpickle(stream).map(ObjectInner::MtaHook), ObjectType::MtaInboundSession => { - Pickle::unpickle(stream).map(ObjectInner::MtaInboundSession) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaInboundSession(Box::new(obj))) } ObjectType::MtaInboundThrottle => { - Pickle::unpickle(stream).map(ObjectInner::MtaInboundThrottle) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaInboundThrottle(Box::new(obj))) + } + ObjectType::MtaMilter => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaMilter(Box::new(obj))) } - ObjectType::MtaMilter => Pickle::unpickle(stream).map(ObjectInner::MtaMilter), ObjectType::MtaOutboundStrategy => { - Pickle::unpickle(stream).map(ObjectInner::MtaOutboundStrategy) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaOutboundStrategy(Box::new(obj))) } ObjectType::MtaOutboundThrottle => { - Pickle::unpickle(stream).map(ObjectInner::MtaOutboundThrottle) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaOutboundThrottle(Box::new(obj))) + } + ObjectType::MtaQueueQuota => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaQueueQuota(Box::new(obj))) + } + ObjectType::MtaRoute => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaRoute(Box::new(obj))) + } + ObjectType::MtaStageAuth => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageAuth(Box::new(obj))) } - ObjectType::MtaQueueQuota => Pickle::unpickle(stream).map(ObjectInner::MtaQueueQuota), - ObjectType::MtaRoute => Pickle::unpickle(stream).map(ObjectInner::MtaRoute), - ObjectType::MtaStageAuth => Pickle::unpickle(stream).map(ObjectInner::MtaStageAuth), ObjectType::MtaStageConnect => { - Pickle::unpickle(stream).map(ObjectInner::MtaStageConnect) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageConnect(Box::new(obj))) + } + ObjectType::MtaStageData => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageData(Box::new(obj))) + } + ObjectType::MtaStageEhlo => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageEhlo(Box::new(obj))) + } + ObjectType::MtaStageMail => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageMail(Box::new(obj))) + } + ObjectType::MtaStageRcpt => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaStageRcpt(Box::new(obj))) + } + ObjectType::MtaSts => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaSts(Box::new(obj))) + } + ObjectType::MtaTlsStrategy => { + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaTlsStrategy(Box::new(obj))) } - ObjectType::MtaStageData => Pickle::unpickle(stream).map(ObjectInner::MtaStageData), - ObjectType::MtaStageEhlo => Pickle::unpickle(stream).map(ObjectInner::MtaStageEhlo), - ObjectType::MtaStageMail => Pickle::unpickle(stream).map(ObjectInner::MtaStageMail), - ObjectType::MtaStageRcpt => Pickle::unpickle(stream).map(ObjectInner::MtaStageRcpt), - ObjectType::MtaSts => Pickle::unpickle(stream).map(ObjectInner::MtaSts), - ObjectType::MtaTlsStrategy => Pickle::unpickle(stream).map(ObjectInner::MtaTlsStrategy), ObjectType::MtaVirtualQueue => { - Pickle::unpickle(stream).map(ObjectInner::MtaVirtualQueue) + Pickle::unpickle(stream).map(|obj| ObjectInner::MtaVirtualQueue(Box::new(obj))) } ObjectType::NetworkListener => { - Pickle::unpickle(stream).map(ObjectInner::NetworkListener) + Pickle::unpickle(stream).map(|obj| ObjectInner::NetworkListener(Box::new(obj))) } - ObjectType::OAuthClient => Pickle::unpickle(stream).map(ObjectInner::OAuthClient), - ObjectType::OidcProvider => Pickle::unpickle(stream).map(ObjectInner::OidcProvider), - ObjectType::PublicKey => Pickle::unpickle(stream).map(ObjectInner::PublicKey), - ObjectType::QueuedMessage => Pickle::unpickle(stream).map(ObjectInner::QueuedMessage), - ObjectType::ReportSettings => Pickle::unpickle(stream).map(ObjectInner::ReportSettings), - ObjectType::Role => Pickle::unpickle(stream).map(ObjectInner::Role), - ObjectType::Search => Pickle::unpickle(stream).map(ObjectInner::Search), - ObjectType::SearchStore => Pickle::unpickle(stream).map(ObjectInner::SearchStore), - ObjectType::Security => Pickle::unpickle(stream).map(ObjectInner::Security), - ObjectType::SenderAuth => Pickle::unpickle(stream).map(ObjectInner::SenderAuth), - ObjectType::Sharing => Pickle::unpickle(stream).map(ObjectInner::Sharing), - ObjectType::SieveSystemInterpreter => { - Pickle::unpickle(stream).map(ObjectInner::SieveSystemInterpreter) + ObjectType::OAuthClient => { + Pickle::unpickle(stream).map(|obj| ObjectInner::OAuthClient(Box::new(obj))) } + ObjectType::OAuthGrant => { + Pickle::unpickle(stream).map(|obj| ObjectInner::OAuthGrant(Box::new(obj))) + } + ObjectType::OidcProvider => { + Pickle::unpickle(stream).map(|obj| ObjectInner::OidcProvider(Box::new(obj))) + } + ObjectType::PublicKey => { + Pickle::unpickle(stream).map(|obj| ObjectInner::PublicKey(Box::new(obj))) + } + ObjectType::QueuedMessage => unimplemented!("QueuedMessage is not stored"), + ObjectType::ReportSettings => { + Pickle::unpickle(stream).map(|obj| ObjectInner::ReportSettings(Box::new(obj))) + } + ObjectType::Role => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Role(Box::new(obj))) + } + ObjectType::Search => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Search(Box::new(obj))) + } + ObjectType::SearchStore => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SearchStore(Box::new(obj))) + } + ObjectType::Security => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Security(Box::new(obj))) + } + ObjectType::SenderAuth => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SenderAuth(Box::new(obj))) + } + ObjectType::Sharing => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Sharing(Box::new(obj))) + } + ObjectType::SieveSystemInterpreter => Pickle::unpickle(stream) + .map(|obj| ObjectInner::SieveSystemInterpreter(Box::new(obj))), ObjectType::SieveSystemScript => { - Pickle::unpickle(stream).map(ObjectInner::SieveSystemScript) + Pickle::unpickle(stream).map(|obj| ObjectInner::SieveSystemScript(Box::new(obj))) } ObjectType::SieveUserInterpreter => { - Pickle::unpickle(stream).map(ObjectInner::SieveUserInterpreter) + Pickle::unpickle(stream).map(|obj| ObjectInner::SieveUserInterpreter(Box::new(obj))) } ObjectType::SieveUserScript => { - Pickle::unpickle(stream).map(ObjectInner::SieveUserScript) + Pickle::unpickle(stream).map(|obj| ObjectInner::SieveUserScript(Box::new(obj))) + } + ObjectType::SpamClassifier => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamClassifier(Box::new(obj))) } - ObjectType::SpamClassifier => Pickle::unpickle(stream).map(ObjectInner::SpamClassifier), ObjectType::SpamDnsblServer => { - Pickle::unpickle(stream).map(ObjectInner::SpamDnsblServer) + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamDnsblServer(Box::new(obj))) } ObjectType::SpamDnsblSettings => { - Pickle::unpickle(stream).map(ObjectInner::SpamDnsblSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamDnsblSettings(Box::new(obj))) } ObjectType::SpamFileExtension => { - Pickle::unpickle(stream).map(ObjectInner::SpamFileExtension) + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamFileExtension(Box::new(obj))) + } + ObjectType::SpamPyzor => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamPyzor(Box::new(obj))) + } + ObjectType::SpamRule => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamRule(Box::new(obj))) + } + ObjectType::SpamSettings => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamSettings(Box::new(obj))) + } + ObjectType::SpamTag => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamTag(Box::new(obj))) } - ObjectType::SpamLlm => Pickle::unpickle(stream).map(ObjectInner::SpamLlm), - ObjectType::SpamPyzor => Pickle::unpickle(stream).map(ObjectInner::SpamPyzor), - ObjectType::SpamRule => Pickle::unpickle(stream).map(ObjectInner::SpamRule), - ObjectType::SpamSettings => Pickle::unpickle(stream).map(ObjectInner::SpamSettings), - ObjectType::SpamTag => Pickle::unpickle(stream).map(ObjectInner::SpamTag), ObjectType::SpamTrainingSample => { - Pickle::unpickle(stream).map(ObjectInner::SpamTrainingSample) + Pickle::unpickle(stream).map(|obj| ObjectInner::SpamTrainingSample(Box::new(obj))) } ObjectType::SpfReportSettings => { - Pickle::unpickle(stream).map(ObjectInner::SpfReportSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::SpfReportSettings(Box::new(obj))) + } + ObjectType::StoreLookup => { + Pickle::unpickle(stream).map(|obj| ObjectInner::StoreLookup(Box::new(obj))) + } + ObjectType::SystemSettings => { + Pickle::unpickle(stream).map(|obj| ObjectInner::SystemSettings(Box::new(obj))) + } + ObjectType::Task => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Task(Box::new(obj))) + } + ObjectType::TaskManager => { + Pickle::unpickle(stream).map(|obj| ObjectInner::TaskManager(Box::new(obj))) } - ObjectType::StoreLookup => Pickle::unpickle(stream).map(ObjectInner::StoreLookup), - ObjectType::SystemSettings => Pickle::unpickle(stream).map(ObjectInner::SystemSettings), - ObjectType::Task => Pickle::unpickle(stream).map(ObjectInner::Task), - ObjectType::TaskManager => Pickle::unpickle(stream).map(ObjectInner::TaskManager), - ObjectType::Tenant => Pickle::unpickle(stream).map(ObjectInner::Tenant), ObjectType::TlsExternalReport => { - Pickle::unpickle(stream).map(ObjectInner::TlsExternalReport) + Pickle::unpickle(stream).map(|obj| ObjectInner::TlsExternalReport(Box::new(obj))) } ObjectType::TlsInternalReport => { - Pickle::unpickle(stream).map(ObjectInner::TlsInternalReport) + Pickle::unpickle(stream).map(|obj| ObjectInner::TlsInternalReport(Box::new(obj))) } ObjectType::TlsReportSettings => { - Pickle::unpickle(stream).map(ObjectInner::TlsReportSettings) + Pickle::unpickle(stream).map(|obj| ObjectInner::TlsReportSettings(Box::new(obj))) + } + ObjectType::Tracer => { + Pickle::unpickle(stream).map(|obj| ObjectInner::Tracer(Box::new(obj))) + } + ObjectType::WebDav => { + Pickle::unpickle(stream).map(|obj| ObjectInner::WebDav(Box::new(obj))) + } + ObjectType::WebHook => { + Pickle::unpickle(stream).map(|obj| ObjectInner::WebHook(Box::new(obj))) } - ObjectType::Trace => Pickle::unpickle(stream).map(ObjectInner::Trace), - ObjectType::Tracer => Pickle::unpickle(stream).map(ObjectInner::Tracer), - ObjectType::TracingStore => Pickle::unpickle(stream).map(ObjectInner::TracingStore), - ObjectType::WebDav => Pickle::unpickle(stream).map(ObjectInner::WebDav), - ObjectType::WebHook => Pickle::unpickle(stream).map(ObjectInner::WebHook), } } @@ -4973,293 +4537,257 @@ impl ObjectInner { D: serde::Deserializer<'de>, { match object { - ObjectType::Account => Account::deserialize(deserializer).map(ObjectInner::Account), - ObjectType::AccountPassword => { - AccountPassword::deserialize(deserializer).map(ObjectInner::AccountPassword) + ObjectType::Account => { + Account::deserialize(deserializer).map(|obj| ObjectInner::Account(Box::new(obj))) } - ObjectType::AccountSettings => { - AccountSettings::deserialize(deserializer).map(ObjectInner::AccountSettings) + ObjectType::AccountPassword => AccountPassword::deserialize(deserializer) + .map(|obj| ObjectInner::AccountPassword(Box::new(obj))), + ObjectType::AccountSettings => AccountSettings::deserialize(deserializer) + .map(|obj| ObjectInner::AccountSettings(Box::new(obj))), + ObjectType::AcmeProvider => AcmeProvider::deserialize(deserializer) + .map(|obj| ObjectInner::AcmeProvider(Box::new(obj))), + ObjectType::Action => { + Action::deserialize(deserializer).map(|obj| ObjectInner::Action(Box::new(obj))) } - ObjectType::AcmeProvider => { - AcmeProvider::deserialize(deserializer).map(ObjectInner::AcmeProvider) + ObjectType::AddressBook => AddressBook::deserialize(deserializer) + .map(|obj| ObjectInner::AddressBook(Box::new(obj))), + ObjectType::AllowedIp => AllowedIp::deserialize(deserializer) + .map(|obj| ObjectInner::AllowedIp(Box::new(obj))), + ObjectType::ApiKey => { + ApiKey::deserialize(deserializer).map(|obj| ObjectInner::ApiKey(Box::new(obj))) } - ObjectType::Action => Action::deserialize(deserializer).map(ObjectInner::Action), - ObjectType::AddressBook => { - AddressBook::deserialize(deserializer).map(ObjectInner::AddressBook) + ObjectType::AppPassword => AppPassword::deserialize(deserializer) + .map(|obj| ObjectInner::AppPassword(Box::new(obj))), + ObjectType::Application => Application::deserialize(deserializer) + .map(|obj| ObjectInner::Application(Box::new(obj))), + ObjectType::ArfExternalReport => ArfExternalReport::deserialize(deserializer) + .map(|obj| ObjectInner::ArfExternalReport(Box::new(obj))), + ObjectType::Asn => { + Asn::deserialize(deserializer).map(|obj| ObjectInner::Asn(Box::new(obj))) } - ObjectType::AiModel => AiModel::deserialize(deserializer).map(ObjectInner::AiModel), - ObjectType::Alert => Alert::deserialize(deserializer).map(ObjectInner::Alert), - ObjectType::AllowedIp => { - AllowedIp::deserialize(deserializer).map(ObjectInner::AllowedIp) + ObjectType::Authentication => Authentication::deserialize(deserializer) + .map(|obj| ObjectInner::Authentication(Box::new(obj))), + ObjectType::BlobStore => BlobStore::deserialize(deserializer) + .map(|obj| ObjectInner::BlobStore(Box::new(obj))), + ObjectType::BlockedIp => BlockedIp::deserialize(deserializer) + .map(|obj| ObjectInner::BlockedIp(Box::new(obj))), + ObjectType::Bootstrap => Bootstrap::deserialize(deserializer) + .map(|obj| ObjectInner::Bootstrap(Box::new(obj))), + ObjectType::Cache => { + Cache::deserialize(deserializer).map(|obj| ObjectInner::Cache(Box::new(obj))) } - ObjectType::ApiKey => ApiKey::deserialize(deserializer).map(ObjectInner::ApiKey), - ObjectType::AppPassword => { - AppPassword::deserialize(deserializer).map(ObjectInner::AppPassword) + ObjectType::Calendar => { + Calendar::deserialize(deserializer).map(|obj| ObjectInner::Calendar(Box::new(obj))) } - ObjectType::Application => { - Application::deserialize(deserializer).map(ObjectInner::Application) + ObjectType::CalendarAlarm => CalendarAlarm::deserialize(deserializer) + .map(|obj| ObjectInner::CalendarAlarm(Box::new(obj))), + ObjectType::CalendarScheduling => CalendarScheduling::deserialize(deserializer) + .map(|obj| ObjectInner::CalendarScheduling(Box::new(obj))), + ObjectType::Certificate => Certificate::deserialize(deserializer) + .map(|obj| ObjectInner::Certificate(Box::new(obj))), + ObjectType::ClusterNode => ClusterNode::deserialize(deserializer) + .map(|obj| ObjectInner::ClusterNode(Box::new(obj))), + ObjectType::ClusterRole => ClusterRole::deserialize(deserializer) + .map(|obj| ObjectInner::ClusterRole(Box::new(obj))), + ObjectType::Coordinator => Coordinator::deserialize(deserializer) + .map(|obj| ObjectInner::Coordinator(Box::new(obj))), + ObjectType::DataRetention => DataRetention::deserialize(deserializer) + .map(|obj| ObjectInner::DataRetention(Box::new(obj))), + ObjectType::DataStore => DataStore::deserialize(deserializer) + .map(|obj| ObjectInner::DataStore(Box::new(obj))), + ObjectType::Directory => Directory::deserialize(deserializer) + .map(|obj| ObjectInner::Directory(Box::new(obj))), + ObjectType::DkimReportSettings => DkimReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::DkimReportSettings(Box::new(obj))), + ObjectType::DkimSignature => DkimSignature::deserialize(deserializer) + .map(|obj| ObjectInner::DkimSignature(Box::new(obj))), + ObjectType::DmarcExternalReport => DmarcExternalReport::deserialize(deserializer) + .map(|obj| ObjectInner::DmarcExternalReport(Box::new(obj))), + ObjectType::DmarcInternalReport => DmarcInternalReport::deserialize(deserializer) + .map(|obj| ObjectInner::DmarcInternalReport(Box::new(obj))), + ObjectType::DmarcReportSettings => DmarcReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::DmarcReportSettings(Box::new(obj))), + ObjectType::DnsResolver => DnsResolver::deserialize(deserializer) + .map(|obj| ObjectInner::DnsResolver(Box::new(obj))), + ObjectType::DnsServer => DnsServer::deserialize(deserializer) + .map(|obj| ObjectInner::DnsServer(Box::new(obj))), + ObjectType::Domain => { + Domain::deserialize(deserializer).map(|obj| ObjectInner::Domain(Box::new(obj))) } - ObjectType::ArchivedItem => { - ArchivedItem::deserialize(deserializer).map(ObjectInner::ArchivedItem) + ObjectType::DsnReportSettings => DsnReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::DsnReportSettings(Box::new(obj))), + ObjectType::Email => { + Email::deserialize(deserializer).map(|obj| ObjectInner::Email(Box::new(obj))) } - ObjectType::ArfExternalReport => { - ArfExternalReport::deserialize(deserializer).map(ObjectInner::ArfExternalReport) + ObjectType::EventTracingLevel => EventTracingLevel::deserialize(deserializer) + .map(|obj| ObjectInner::EventTracingLevel(Box::new(obj))), + ObjectType::FileStorage => FileStorage::deserialize(deserializer) + .map(|obj| ObjectInner::FileStorage(Box::new(obj))), + ObjectType::Http => { + Http::deserialize(deserializer).map(|obj| ObjectInner::Http(Box::new(obj))) } - ObjectType::Asn => Asn::deserialize(deserializer).map(ObjectInner::Asn), - ObjectType::Authentication => { - Authentication::deserialize(deserializer).map(ObjectInner::Authentication) + ObjectType::HttpForm => { + HttpForm::deserialize(deserializer).map(|obj| ObjectInner::HttpForm(Box::new(obj))) } - ObjectType::BlobStore => { - BlobStore::deserialize(deserializer).map(ObjectInner::BlobStore) + ObjectType::HttpLookup => HttpLookup::deserialize(deserializer) + .map(|obj| ObjectInner::HttpLookup(Box::new(obj))), + ObjectType::Imap => { + Imap::deserialize(deserializer).map(|obj| ObjectInner::Imap(Box::new(obj))) } - ObjectType::BlockedIp => { - BlockedIp::deserialize(deserializer).map(ObjectInner::BlockedIp) + ObjectType::InMemoryStore => InMemoryStore::deserialize(deserializer) + .map(|obj| ObjectInner::InMemoryStore(Box::new(obj))), + ObjectType::IndexQueueStatus => IndexQueueStatus::deserialize(deserializer) + .map(|obj| ObjectInner::IndexQueueStatus(Box::new(obj))), + ObjectType::Jmap => { + Jmap::deserialize(deserializer).map(|obj| ObjectInner::Jmap(Box::new(obj))) } - ObjectType::Bootstrap => { - Bootstrap::deserialize(deserializer).map(ObjectInner::Bootstrap) - } - ObjectType::Cache => Cache::deserialize(deserializer).map(ObjectInner::Cache), - ObjectType::Calendar => Calendar::deserialize(deserializer).map(ObjectInner::Calendar), - ObjectType::CalendarAlarm => { - CalendarAlarm::deserialize(deserializer).map(ObjectInner::CalendarAlarm) - } - ObjectType::CalendarScheduling => { - CalendarScheduling::deserialize(deserializer).map(ObjectInner::CalendarScheduling) - } - ObjectType::Certificate => { - Certificate::deserialize(deserializer).map(ObjectInner::Certificate) - } - ObjectType::ClusterNode => { - ClusterNode::deserialize(deserializer).map(ObjectInner::ClusterNode) - } - ObjectType::ClusterRole => { - ClusterRole::deserialize(deserializer).map(ObjectInner::ClusterRole) - } - ObjectType::Coordinator => { - Coordinator::deserialize(deserializer).map(ObjectInner::Coordinator) - } - ObjectType::DataRetention => { - DataRetention::deserialize(deserializer).map(ObjectInner::DataRetention) - } - ObjectType::DataStore => { - DataStore::deserialize(deserializer).map(ObjectInner::DataStore) - } - ObjectType::Directory => { - Directory::deserialize(deserializer).map(ObjectInner::Directory) - } - ObjectType::DkimReportSettings => { - DkimReportSettings::deserialize(deserializer).map(ObjectInner::DkimReportSettings) - } - ObjectType::DkimSignature => { - DkimSignature::deserialize(deserializer).map(ObjectInner::DkimSignature) - } - ObjectType::DmarcExternalReport => { - DmarcExternalReport::deserialize(deserializer).map(ObjectInner::DmarcExternalReport) - } - ObjectType::DmarcInternalReport => { - DmarcInternalReport::deserialize(deserializer).map(ObjectInner::DmarcInternalReport) - } - ObjectType::DmarcReportSettings => { - DmarcReportSettings::deserialize(deserializer).map(ObjectInner::DmarcReportSettings) - } - ObjectType::DnsResolver => { - DnsResolver::deserialize(deserializer).map(ObjectInner::DnsResolver) - } - ObjectType::DnsServer => { - DnsServer::deserialize(deserializer).map(ObjectInner::DnsServer) - } - ObjectType::Domain => Domain::deserialize(deserializer).map(ObjectInner::Domain), - ObjectType::DsnReportSettings => { - DsnReportSettings::deserialize(deserializer).map(ObjectInner::DsnReportSettings) - } - ObjectType::Email => Email::deserialize(deserializer).map(ObjectInner::Email), - ObjectType::Enterprise => { - Enterprise::deserialize(deserializer).map(ObjectInner::Enterprise) - } - ObjectType::EventTracingLevel => { - EventTracingLevel::deserialize(deserializer).map(ObjectInner::EventTracingLevel) - } - ObjectType::FileStorage => { - FileStorage::deserialize(deserializer).map(ObjectInner::FileStorage) - } - ObjectType::Http => Http::deserialize(deserializer).map(ObjectInner::Http), - ObjectType::HttpForm => HttpForm::deserialize(deserializer).map(ObjectInner::HttpForm), - ObjectType::HttpLookup => { - HttpLookup::deserialize(deserializer).map(ObjectInner::HttpLookup) - } - ObjectType::Imap => Imap::deserialize(deserializer).map(ObjectInner::Imap), - ObjectType::InMemoryStore => { - InMemoryStore::deserialize(deserializer).map(ObjectInner::InMemoryStore) - } - ObjectType::Jmap => Jmap::deserialize(deserializer).map(ObjectInner::Jmap), - ObjectType::Log => Log::deserialize(deserializer).map(ObjectInner::Log), - ObjectType::MailingList => { - MailingList::deserialize(deserializer).map(ObjectInner::MailingList) - } - ObjectType::MaskedEmail => { - MaskedEmail::deserialize(deserializer).map(ObjectInner::MaskedEmail) - } - ObjectType::MemoryLookupKey => { - MemoryLookupKey::deserialize(deserializer).map(ObjectInner::MemoryLookupKey) + ObjectType::Log => { + Log::deserialize(deserializer).map(|obj| ObjectInner::Log(Box::new(obj))) } + ObjectType::MailingList => MailingList::deserialize(deserializer) + .map(|obj| ObjectInner::MailingList(Box::new(obj))), + ObjectType::MemoryLookupKey => MemoryLookupKey::deserialize(deserializer) + .map(|obj| ObjectInner::MemoryLookupKey(Box::new(obj))), ObjectType::MemoryLookupKeyValue => MemoryLookupKeyValue::deserialize(deserializer) - .map(ObjectInner::MemoryLookupKeyValue), - ObjectType::Metric => Metric::deserialize(deserializer).map(ObjectInner::Metric), - ObjectType::Metrics => Metrics::deserialize(deserializer).map(ObjectInner::Metrics), - ObjectType::MetricsStore => { - MetricsStore::deserialize(deserializer).map(ObjectInner::MetricsStore) + .map(|obj| ObjectInner::MemoryLookupKeyValue(Box::new(obj))), + ObjectType::Metadata => { + Metadata::deserialize(deserializer).map(|obj| ObjectInner::Metadata(Box::new(obj))) + } + ObjectType::Metrics => { + Metrics::deserialize(deserializer).map(|obj| ObjectInner::Metrics(Box::new(obj))) } ObjectType::MtaConnectionStrategy => MtaConnectionStrategy::deserialize(deserializer) - .map(ObjectInner::MtaConnectionStrategy), - ObjectType::MtaDeliverySchedule => { - MtaDeliverySchedule::deserialize(deserializer).map(ObjectInner::MtaDeliverySchedule) + .map(|obj| ObjectInner::MtaConnectionStrategy(Box::new(obj))), + ObjectType::MtaDeliverySchedule => MtaDeliverySchedule::deserialize(deserializer) + .map(|obj| ObjectInner::MtaDeliverySchedule(Box::new(obj))), + ObjectType::MtaExtensions => MtaExtensions::deserialize(deserializer) + .map(|obj| ObjectInner::MtaExtensions(Box::new(obj))), + ObjectType::MtaHook => { + MtaHook::deserialize(deserializer).map(|obj| ObjectInner::MtaHook(Box::new(obj))) } - ObjectType::MtaExtensions => { - MtaExtensions::deserialize(deserializer).map(ObjectInner::MtaExtensions) + ObjectType::MtaInboundSession => MtaInboundSession::deserialize(deserializer) + .map(|obj| ObjectInner::MtaInboundSession(Box::new(obj))), + ObjectType::MtaInboundThrottle => MtaInboundThrottle::deserialize(deserializer) + .map(|obj| ObjectInner::MtaInboundThrottle(Box::new(obj))), + ObjectType::MtaMilter => MtaMilter::deserialize(deserializer) + .map(|obj| ObjectInner::MtaMilter(Box::new(obj))), + ObjectType::MtaOutboundStrategy => MtaOutboundStrategy::deserialize(deserializer) + .map(|obj| ObjectInner::MtaOutboundStrategy(Box::new(obj))), + ObjectType::MtaOutboundThrottle => MtaOutboundThrottle::deserialize(deserializer) + .map(|obj| ObjectInner::MtaOutboundThrottle(Box::new(obj))), + ObjectType::MtaQueueQuota => MtaQueueQuota::deserialize(deserializer) + .map(|obj| ObjectInner::MtaQueueQuota(Box::new(obj))), + ObjectType::MtaRoute => { + MtaRoute::deserialize(deserializer).map(|obj| ObjectInner::MtaRoute(Box::new(obj))) } - ObjectType::MtaHook => MtaHook::deserialize(deserializer).map(ObjectInner::MtaHook), - ObjectType::MtaInboundSession => { - MtaInboundSession::deserialize(deserializer).map(ObjectInner::MtaInboundSession) + ObjectType::MtaStageAuth => MtaStageAuth::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageAuth(Box::new(obj))), + ObjectType::MtaStageConnect => MtaStageConnect::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageConnect(Box::new(obj))), + ObjectType::MtaStageData => MtaStageData::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageData(Box::new(obj))), + ObjectType::MtaStageEhlo => MtaStageEhlo::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageEhlo(Box::new(obj))), + ObjectType::MtaStageMail => MtaStageMail::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageMail(Box::new(obj))), + ObjectType::MtaStageRcpt => MtaStageRcpt::deserialize(deserializer) + .map(|obj| ObjectInner::MtaStageRcpt(Box::new(obj))), + ObjectType::MtaSts => { + MtaSts::deserialize(deserializer).map(|obj| ObjectInner::MtaSts(Box::new(obj))) } - ObjectType::MtaInboundThrottle => { - MtaInboundThrottle::deserialize(deserializer).map(ObjectInner::MtaInboundThrottle) + ObjectType::MtaTlsStrategy => MtaTlsStrategy::deserialize(deserializer) + .map(|obj| ObjectInner::MtaTlsStrategy(Box::new(obj))), + ObjectType::MtaVirtualQueue => MtaVirtualQueue::deserialize(deserializer) + .map(|obj| ObjectInner::MtaVirtualQueue(Box::new(obj))), + ObjectType::NetworkListener => NetworkListener::deserialize(deserializer) + .map(|obj| ObjectInner::NetworkListener(Box::new(obj))), + ObjectType::OAuthClient => OAuthClient::deserialize(deserializer) + .map(|obj| ObjectInner::OAuthClient(Box::new(obj))), + ObjectType::OAuthGrant => OAuthGrant::deserialize(deserializer) + .map(|obj| ObjectInner::OAuthGrant(Box::new(obj))), + ObjectType::OidcProvider => OidcProvider::deserialize(deserializer) + .map(|obj| ObjectInner::OidcProvider(Box::new(obj))), + ObjectType::PublicKey => PublicKey::deserialize(deserializer) + .map(|obj| ObjectInner::PublicKey(Box::new(obj))), + ObjectType::QueuedMessage => QueuedMessage::deserialize(deserializer) + .map(|obj| ObjectInner::QueuedMessage(Box::new(obj))), + ObjectType::ReportSettings => ReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::ReportSettings(Box::new(obj))), + ObjectType::Role => { + Role::deserialize(deserializer).map(|obj| ObjectInner::Role(Box::new(obj))) } - ObjectType::MtaMilter => { - MtaMilter::deserialize(deserializer).map(ObjectInner::MtaMilter) + ObjectType::Search => { + Search::deserialize(deserializer).map(|obj| ObjectInner::Search(Box::new(obj))) } - ObjectType::MtaOutboundStrategy => { - MtaOutboundStrategy::deserialize(deserializer).map(ObjectInner::MtaOutboundStrategy) + ObjectType::SearchStore => SearchStore::deserialize(deserializer) + .map(|obj| ObjectInner::SearchStore(Box::new(obj))), + ObjectType::Security => { + Security::deserialize(deserializer).map(|obj| ObjectInner::Security(Box::new(obj))) } - ObjectType::MtaOutboundThrottle => { - MtaOutboundThrottle::deserialize(deserializer).map(ObjectInner::MtaOutboundThrottle) + ObjectType::SenderAuth => SenderAuth::deserialize(deserializer) + .map(|obj| ObjectInner::SenderAuth(Box::new(obj))), + ObjectType::Sharing => { + Sharing::deserialize(deserializer).map(|obj| ObjectInner::Sharing(Box::new(obj))) } - ObjectType::MtaQueueQuota => { - MtaQueueQuota::deserialize(deserializer).map(ObjectInner::MtaQueueQuota) - } - ObjectType::MtaRoute => MtaRoute::deserialize(deserializer).map(ObjectInner::MtaRoute), - ObjectType::MtaStageAuth => { - MtaStageAuth::deserialize(deserializer).map(ObjectInner::MtaStageAuth) - } - ObjectType::MtaStageConnect => { - MtaStageConnect::deserialize(deserializer).map(ObjectInner::MtaStageConnect) - } - ObjectType::MtaStageData => { - MtaStageData::deserialize(deserializer).map(ObjectInner::MtaStageData) - } - ObjectType::MtaStageEhlo => { - MtaStageEhlo::deserialize(deserializer).map(ObjectInner::MtaStageEhlo) - } - ObjectType::MtaStageMail => { - MtaStageMail::deserialize(deserializer).map(ObjectInner::MtaStageMail) - } - ObjectType::MtaStageRcpt => { - MtaStageRcpt::deserialize(deserializer).map(ObjectInner::MtaStageRcpt) - } - ObjectType::MtaSts => MtaSts::deserialize(deserializer).map(ObjectInner::MtaSts), - ObjectType::MtaTlsStrategy => { - MtaTlsStrategy::deserialize(deserializer).map(ObjectInner::MtaTlsStrategy) - } - ObjectType::MtaVirtualQueue => { - MtaVirtualQueue::deserialize(deserializer).map(ObjectInner::MtaVirtualQueue) - } - ObjectType::NetworkListener => { - NetworkListener::deserialize(deserializer).map(ObjectInner::NetworkListener) - } - ObjectType::OAuthClient => { - OAuthClient::deserialize(deserializer).map(ObjectInner::OAuthClient) - } - ObjectType::OidcProvider => { - OidcProvider::deserialize(deserializer).map(ObjectInner::OidcProvider) - } - ObjectType::PublicKey => { - PublicKey::deserialize(deserializer).map(ObjectInner::PublicKey) - } - ObjectType::QueuedMessage => { - QueuedMessage::deserialize(deserializer).map(ObjectInner::QueuedMessage) - } - ObjectType::ReportSettings => { - ReportSettings::deserialize(deserializer).map(ObjectInner::ReportSettings) - } - ObjectType::Role => Role::deserialize(deserializer).map(ObjectInner::Role), - ObjectType::Search => Search::deserialize(deserializer).map(ObjectInner::Search), - ObjectType::SearchStore => { - SearchStore::deserialize(deserializer).map(ObjectInner::SearchStore) - } - ObjectType::Security => Security::deserialize(deserializer).map(ObjectInner::Security), - ObjectType::SenderAuth => { - SenderAuth::deserialize(deserializer).map(ObjectInner::SenderAuth) - } - ObjectType::Sharing => Sharing::deserialize(deserializer).map(ObjectInner::Sharing), ObjectType::SieveSystemInterpreter => SieveSystemInterpreter::deserialize(deserializer) - .map(ObjectInner::SieveSystemInterpreter), - ObjectType::SieveSystemScript => { - SieveSystemScript::deserialize(deserializer).map(ObjectInner::SieveSystemScript) - } + .map(|obj| ObjectInner::SieveSystemInterpreter(Box::new(obj))), + ObjectType::SieveSystemScript => SieveSystemScript::deserialize(deserializer) + .map(|obj| ObjectInner::SieveSystemScript(Box::new(obj))), ObjectType::SieveUserInterpreter => SieveUserInterpreter::deserialize(deserializer) - .map(ObjectInner::SieveUserInterpreter), - ObjectType::SieveUserScript => { - SieveUserScript::deserialize(deserializer).map(ObjectInner::SieveUserScript) + .map(|obj| ObjectInner::SieveUserInterpreter(Box::new(obj))), + ObjectType::SieveUserScript => SieveUserScript::deserialize(deserializer) + .map(|obj| ObjectInner::SieveUserScript(Box::new(obj))), + ObjectType::SpamClassifier => SpamClassifier::deserialize(deserializer) + .map(|obj| ObjectInner::SpamClassifier(Box::new(obj))), + ObjectType::SpamDnsblServer => SpamDnsblServer::deserialize(deserializer) + .map(|obj| ObjectInner::SpamDnsblServer(Box::new(obj))), + ObjectType::SpamDnsblSettings => SpamDnsblSettings::deserialize(deserializer) + .map(|obj| ObjectInner::SpamDnsblSettings(Box::new(obj))), + ObjectType::SpamFileExtension => SpamFileExtension::deserialize(deserializer) + .map(|obj| ObjectInner::SpamFileExtension(Box::new(obj))), + ObjectType::SpamPyzor => SpamPyzor::deserialize(deserializer) + .map(|obj| ObjectInner::SpamPyzor(Box::new(obj))), + ObjectType::SpamRule => { + SpamRule::deserialize(deserializer).map(|obj| ObjectInner::SpamRule(Box::new(obj))) } - ObjectType::SpamClassifier => { - SpamClassifier::deserialize(deserializer).map(ObjectInner::SpamClassifier) + ObjectType::SpamSettings => SpamSettings::deserialize(deserializer) + .map(|obj| ObjectInner::SpamSettings(Box::new(obj))), + ObjectType::SpamTag => { + SpamTag::deserialize(deserializer).map(|obj| ObjectInner::SpamTag(Box::new(obj))) } - ObjectType::SpamDnsblServer => { - SpamDnsblServer::deserialize(deserializer).map(ObjectInner::SpamDnsblServer) + ObjectType::SpamTrainingSample => SpamTrainingSample::deserialize(deserializer) + .map(|obj| ObjectInner::SpamTrainingSample(Box::new(obj))), + ObjectType::SpfReportSettings => SpfReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::SpfReportSettings(Box::new(obj))), + ObjectType::StoreLookup => StoreLookup::deserialize(deserializer) + .map(|obj| ObjectInner::StoreLookup(Box::new(obj))), + ObjectType::SystemSettings => SystemSettings::deserialize(deserializer) + .map(|obj| ObjectInner::SystemSettings(Box::new(obj))), + ObjectType::Task => { + Task::deserialize(deserializer).map(|obj| ObjectInner::Task(Box::new(obj))) } - ObjectType::SpamDnsblSettings => { - SpamDnsblSettings::deserialize(deserializer).map(ObjectInner::SpamDnsblSettings) + ObjectType::TaskManager => TaskManager::deserialize(deserializer) + .map(|obj| ObjectInner::TaskManager(Box::new(obj))), + ObjectType::TlsExternalReport => TlsExternalReport::deserialize(deserializer) + .map(|obj| ObjectInner::TlsExternalReport(Box::new(obj))), + ObjectType::TlsInternalReport => TlsInternalReport::deserialize(deserializer) + .map(|obj| ObjectInner::TlsInternalReport(Box::new(obj))), + ObjectType::TlsReportSettings => TlsReportSettings::deserialize(deserializer) + .map(|obj| ObjectInner::TlsReportSettings(Box::new(obj))), + ObjectType::Tracer => { + Tracer::deserialize(deserializer).map(|obj| ObjectInner::Tracer(Box::new(obj))) } - ObjectType::SpamFileExtension => { - SpamFileExtension::deserialize(deserializer).map(ObjectInner::SpamFileExtension) + ObjectType::WebDav => { + WebDav::deserialize(deserializer).map(|obj| ObjectInner::WebDav(Box::new(obj))) } - ObjectType::SpamLlm => SpamLlm::deserialize(deserializer).map(ObjectInner::SpamLlm), - ObjectType::SpamPyzor => { - SpamPyzor::deserialize(deserializer).map(ObjectInner::SpamPyzor) + ObjectType::WebHook => { + WebHook::deserialize(deserializer).map(|obj| ObjectInner::WebHook(Box::new(obj))) } - ObjectType::SpamRule => SpamRule::deserialize(deserializer).map(ObjectInner::SpamRule), - ObjectType::SpamSettings => { - SpamSettings::deserialize(deserializer).map(ObjectInner::SpamSettings) - } - ObjectType::SpamTag => SpamTag::deserialize(deserializer).map(ObjectInner::SpamTag), - ObjectType::SpamTrainingSample => { - SpamTrainingSample::deserialize(deserializer).map(ObjectInner::SpamTrainingSample) - } - ObjectType::SpfReportSettings => { - SpfReportSettings::deserialize(deserializer).map(ObjectInner::SpfReportSettings) - } - ObjectType::StoreLookup => { - StoreLookup::deserialize(deserializer).map(ObjectInner::StoreLookup) - } - ObjectType::SystemSettings => { - SystemSettings::deserialize(deserializer).map(ObjectInner::SystemSettings) - } - ObjectType::Task => Task::deserialize(deserializer).map(ObjectInner::Task), - ObjectType::TaskManager => { - TaskManager::deserialize(deserializer).map(ObjectInner::TaskManager) - } - ObjectType::Tenant => Tenant::deserialize(deserializer).map(ObjectInner::Tenant), - ObjectType::TlsExternalReport => { - TlsExternalReport::deserialize(deserializer).map(ObjectInner::TlsExternalReport) - } - ObjectType::TlsInternalReport => { - TlsInternalReport::deserialize(deserializer).map(ObjectInner::TlsInternalReport) - } - ObjectType::TlsReportSettings => { - TlsReportSettings::deserialize(deserializer).map(ObjectInner::TlsReportSettings) - } - ObjectType::Trace => Trace::deserialize(deserializer).map(ObjectInner::Trace), - ObjectType::Tracer => Tracer::deserialize(deserializer).map(ObjectInner::Tracer), - ObjectType::TracingStore => { - TracingStore::deserialize(deserializer).map(ObjectInner::TracingStore) - } - ObjectType::WebDav => WebDav::deserialize(deserializer).map(ObjectInner::WebDav), - ObjectType::WebHook => WebHook::deserialize(deserializer).map(ObjectInner::WebHook), } } pub fn expression_ctxs(&self) -> Option>> { match &self { - ObjectInner::Alert(obj) => Some(obj.expression_ctxs()), ObjectInner::DkimReportSettings(obj) => Some(obj.expression_ctxs()), ObjectInner::DmarcReportSettings(obj) => Some(obj.expression_ctxs()), ObjectInner::DsnReportSettings(obj) => Some(obj.expression_ctxs()), @@ -5300,13 +4828,10 @@ impl Object { ObjectInner::AcmeProvider(_) => AcmeProvider::FLAGS, ObjectInner::Action(_) => Action::FLAGS, ObjectInner::AddressBook(_) => AddressBook::FLAGS, - ObjectInner::AiModel(_) => AiModel::FLAGS, - ObjectInner::Alert(_) => Alert::FLAGS, ObjectInner::AllowedIp(_) => AllowedIp::FLAGS, ObjectInner::ApiKey(_) => ApiKey::FLAGS, ObjectInner::AppPassword(_) => AppPassword::FLAGS, ObjectInner::Application(_) => Application::FLAGS, - ObjectInner::ArchivedItem(_) => ArchivedItem::FLAGS, ObjectInner::ArfExternalReport(_) => ArfExternalReport::FLAGS, ObjectInner::Asn(_) => Asn::FLAGS, ObjectInner::Authentication(_) => Authentication::FLAGS, @@ -5334,7 +4859,6 @@ impl Object { ObjectInner::Domain(_) => Domain::FLAGS, ObjectInner::DsnReportSettings(_) => DsnReportSettings::FLAGS, ObjectInner::Email(_) => Email::FLAGS, - ObjectInner::Enterprise(_) => Enterprise::FLAGS, ObjectInner::EventTracingLevel(_) => EventTracingLevel::FLAGS, ObjectInner::FileStorage(_) => FileStorage::FLAGS, ObjectInner::Http(_) => Http::FLAGS, @@ -5342,15 +4866,14 @@ impl Object { ObjectInner::HttpLookup(_) => HttpLookup::FLAGS, ObjectInner::Imap(_) => Imap::FLAGS, ObjectInner::InMemoryStore(_) => InMemoryStore::FLAGS, + ObjectInner::IndexQueueStatus(_) => IndexQueueStatus::FLAGS, ObjectInner::Jmap(_) => Jmap::FLAGS, ObjectInner::Log(_) => Log::FLAGS, ObjectInner::MailingList(_) => MailingList::FLAGS, - ObjectInner::MaskedEmail(_) => MaskedEmail::FLAGS, ObjectInner::MemoryLookupKey(_) => MemoryLookupKey::FLAGS, ObjectInner::MemoryLookupKeyValue(_) => MemoryLookupKeyValue::FLAGS, - ObjectInner::Metric(_) => Metric::FLAGS, + ObjectInner::Metadata(_) => Metadata::FLAGS, ObjectInner::Metrics(_) => Metrics::FLAGS, - ObjectInner::MetricsStore(_) => MetricsStore::FLAGS, ObjectInner::MtaConnectionStrategy(_) => MtaConnectionStrategy::FLAGS, ObjectInner::MtaDeliverySchedule(_) => MtaDeliverySchedule::FLAGS, ObjectInner::MtaExtensions(_) => MtaExtensions::FLAGS, @@ -5373,6 +4896,7 @@ impl Object { ObjectInner::MtaVirtualQueue(_) => MtaVirtualQueue::FLAGS, ObjectInner::NetworkListener(_) => NetworkListener::FLAGS, ObjectInner::OAuthClient(_) => OAuthClient::FLAGS, + ObjectInner::OAuthGrant(_) => OAuthGrant::FLAGS, ObjectInner::OidcProvider(_) => OidcProvider::FLAGS, ObjectInner::PublicKey(_) => PublicKey::FLAGS, ObjectInner::QueuedMessage(_) => QueuedMessage::FLAGS, @@ -5391,7 +4915,6 @@ impl Object { ObjectInner::SpamDnsblServer(_) => SpamDnsblServer::FLAGS, ObjectInner::SpamDnsblSettings(_) => SpamDnsblSettings::FLAGS, ObjectInner::SpamFileExtension(_) => SpamFileExtension::FLAGS, - ObjectInner::SpamLlm(_) => SpamLlm::FLAGS, ObjectInner::SpamPyzor(_) => SpamPyzor::FLAGS, ObjectInner::SpamRule(_) => SpamRule::FLAGS, ObjectInner::SpamSettings(_) => SpamSettings::FLAGS, @@ -5402,13 +4925,10 @@ impl Object { ObjectInner::SystemSettings(_) => SystemSettings::FLAGS, ObjectInner::Task(_) => Task::FLAGS, ObjectInner::TaskManager(_) => TaskManager::FLAGS, - ObjectInner::Tenant(_) => Tenant::FLAGS, ObjectInner::TlsExternalReport(_) => TlsExternalReport::FLAGS, ObjectInner::TlsInternalReport(_) => TlsInternalReport::FLAGS, ObjectInner::TlsReportSettings(_) => TlsReportSettings::FLAGS, - ObjectInner::Trace(_) => Trace::FLAGS, ObjectInner::Tracer(_) => Tracer::FLAGS, - ObjectInner::TracingStore(_) => TracingStore::FLAGS, ObjectInner::WebDav(_) => WebDav::FLAGS, ObjectInner::WebHook(_) => WebHook::FLAGS, } @@ -5422,13 +4942,10 @@ impl Object { ObjectInner::AcmeProvider(_) => ObjectType::AcmeProvider, ObjectInner::Action(_) => ObjectType::Action, ObjectInner::AddressBook(_) => ObjectType::AddressBook, - ObjectInner::AiModel(_) => ObjectType::AiModel, - ObjectInner::Alert(_) => ObjectType::Alert, ObjectInner::AllowedIp(_) => ObjectType::AllowedIp, ObjectInner::ApiKey(_) => ObjectType::ApiKey, ObjectInner::AppPassword(_) => ObjectType::AppPassword, ObjectInner::Application(_) => ObjectType::Application, - ObjectInner::ArchivedItem(_) => ObjectType::ArchivedItem, ObjectInner::ArfExternalReport(_) => ObjectType::ArfExternalReport, ObjectInner::Asn(_) => ObjectType::Asn, ObjectInner::Authentication(_) => ObjectType::Authentication, @@ -5456,7 +4973,6 @@ impl Object { ObjectInner::Domain(_) => ObjectType::Domain, ObjectInner::DsnReportSettings(_) => ObjectType::DsnReportSettings, ObjectInner::Email(_) => ObjectType::Email, - ObjectInner::Enterprise(_) => ObjectType::Enterprise, ObjectInner::EventTracingLevel(_) => ObjectType::EventTracingLevel, ObjectInner::FileStorage(_) => ObjectType::FileStorage, ObjectInner::Http(_) => ObjectType::Http, @@ -5464,15 +4980,14 @@ impl Object { ObjectInner::HttpLookup(_) => ObjectType::HttpLookup, ObjectInner::Imap(_) => ObjectType::Imap, ObjectInner::InMemoryStore(_) => ObjectType::InMemoryStore, + ObjectInner::IndexQueueStatus(_) => ObjectType::IndexQueueStatus, ObjectInner::Jmap(_) => ObjectType::Jmap, ObjectInner::Log(_) => ObjectType::Log, ObjectInner::MailingList(_) => ObjectType::MailingList, - ObjectInner::MaskedEmail(_) => ObjectType::MaskedEmail, ObjectInner::MemoryLookupKey(_) => ObjectType::MemoryLookupKey, ObjectInner::MemoryLookupKeyValue(_) => ObjectType::MemoryLookupKeyValue, - ObjectInner::Metric(_) => ObjectType::Metric, + ObjectInner::Metadata(_) => ObjectType::Metadata, ObjectInner::Metrics(_) => ObjectType::Metrics, - ObjectInner::MetricsStore(_) => ObjectType::MetricsStore, ObjectInner::MtaConnectionStrategy(_) => ObjectType::MtaConnectionStrategy, ObjectInner::MtaDeliverySchedule(_) => ObjectType::MtaDeliverySchedule, ObjectInner::MtaExtensions(_) => ObjectType::MtaExtensions, @@ -5495,6 +5010,7 @@ impl Object { ObjectInner::MtaVirtualQueue(_) => ObjectType::MtaVirtualQueue, ObjectInner::NetworkListener(_) => ObjectType::NetworkListener, ObjectInner::OAuthClient(_) => ObjectType::OAuthClient, + ObjectInner::OAuthGrant(_) => ObjectType::OAuthGrant, ObjectInner::OidcProvider(_) => ObjectType::OidcProvider, ObjectInner::PublicKey(_) => ObjectType::PublicKey, ObjectInner::QueuedMessage(_) => ObjectType::QueuedMessage, @@ -5513,7 +5029,6 @@ impl Object { ObjectInner::SpamDnsblServer(_) => ObjectType::SpamDnsblServer, ObjectInner::SpamDnsblSettings(_) => ObjectType::SpamDnsblSettings, ObjectInner::SpamFileExtension(_) => ObjectType::SpamFileExtension, - ObjectInner::SpamLlm(_) => ObjectType::SpamLlm, ObjectInner::SpamPyzor(_) => ObjectType::SpamPyzor, ObjectInner::SpamRule(_) => ObjectType::SpamRule, ObjectInner::SpamSettings(_) => ObjectType::SpamSettings, @@ -5524,13 +5039,10 @@ impl Object { ObjectInner::SystemSettings(_) => ObjectType::SystemSettings, ObjectInner::Task(_) => ObjectType::Task, ObjectInner::TaskManager(_) => ObjectType::TaskManager, - ObjectInner::Tenant(_) => ObjectType::Tenant, ObjectInner::TlsExternalReport(_) => ObjectType::TlsExternalReport, ObjectInner::TlsInternalReport(_) => ObjectType::TlsInternalReport, ObjectInner::TlsReportSettings(_) => ObjectType::TlsReportSettings, - ObjectInner::Trace(_) => ObjectType::Trace, ObjectInner::Tracer(_) => ObjectType::Tracer, - ObjectInner::TracingStore(_) => ObjectType::TracingStore, ObjectInner::WebDav(_) => ObjectType::WebDav, ObjectInner::WebHook(_) => ObjectType::WebHook, } @@ -5540,7 +5052,6 @@ impl Object { match &self.inner { ObjectInner::Account(obj) => Some(obj.object_type().as_str()), ObjectInner::Action(obj) => Some(obj.object_type().as_str()), - ObjectInner::ArchivedItem(obj) => Some(obj.object_type().as_str()), ObjectInner::Asn(obj) => Some(obj.object_type().as_str()), ObjectInner::BlobStore(obj) => Some(obj.object_type().as_str()), ObjectInner::Coordinator(obj) => Some(obj.object_type().as_str()), @@ -5550,17 +5061,13 @@ impl Object { ObjectInner::DnsResolver(obj) => Some(obj.object_type().as_str()), ObjectInner::DnsServer(obj) => Some(obj.object_type().as_str()), ObjectInner::InMemoryStore(obj) => Some(obj.object_type().as_str()), - ObjectInner::Metric(obj) => Some(obj.object_type().as_str()), - ObjectInner::MetricsStore(obj) => Some(obj.object_type().as_str()), ObjectInner::MtaRoute(obj) => Some(obj.object_type().as_str()), ObjectInner::SearchStore(obj) => Some(obj.object_type().as_str()), ObjectInner::SpamDnsblServer(obj) => Some(obj.object_type().as_str()), - ObjectInner::SpamLlm(obj) => Some(obj.object_type().as_str()), ObjectInner::SpamRule(obj) => Some(obj.object_type().as_str()), ObjectInner::SpamTag(obj) => Some(obj.object_type().as_str()), ObjectInner::Task(obj) => Some(obj.object_type().as_str()), ObjectInner::Tracer(obj) => Some(obj.object_type().as_str()), - ObjectInner::TracingStore(obj) => Some(obj.object_type().as_str()), _ => None, } } @@ -5573,13 +5080,10 @@ impl Object { ObjectInner::AcmeProvider(obj) => obj.validate(errors), ObjectInner::Action(obj) => obj.validate(errors), ObjectInner::AddressBook(obj) => obj.validate(errors), - ObjectInner::AiModel(obj) => obj.validate(errors), - ObjectInner::Alert(obj) => obj.validate(errors), ObjectInner::AllowedIp(obj) => obj.validate(errors), ObjectInner::ApiKey(obj) => obj.validate(errors), ObjectInner::AppPassword(obj) => obj.validate(errors), ObjectInner::Application(obj) => obj.validate(errors), - ObjectInner::ArchivedItem(obj) => obj.validate(errors), ObjectInner::ArfExternalReport(obj) => obj.validate(errors), ObjectInner::Asn(obj) => obj.validate(errors), ObjectInner::Authentication(obj) => obj.validate(errors), @@ -5607,7 +5111,6 @@ impl Object { ObjectInner::Domain(obj) => obj.validate(errors), ObjectInner::DsnReportSettings(obj) => obj.validate(errors), ObjectInner::Email(obj) => obj.validate(errors), - ObjectInner::Enterprise(obj) => obj.validate(errors), ObjectInner::EventTracingLevel(obj) => obj.validate(errors), ObjectInner::FileStorage(obj) => obj.validate(errors), ObjectInner::Http(obj) => obj.validate(errors), @@ -5615,15 +5118,14 @@ impl Object { ObjectInner::HttpLookup(obj) => obj.validate(errors), ObjectInner::Imap(obj) => obj.validate(errors), ObjectInner::InMemoryStore(obj) => obj.validate(errors), + ObjectInner::IndexQueueStatus(obj) => obj.validate(errors), ObjectInner::Jmap(obj) => obj.validate(errors), ObjectInner::Log(obj) => obj.validate(errors), ObjectInner::MailingList(obj) => obj.validate(errors), - ObjectInner::MaskedEmail(obj) => obj.validate(errors), ObjectInner::MemoryLookupKey(obj) => obj.validate(errors), ObjectInner::MemoryLookupKeyValue(obj) => obj.validate(errors), - ObjectInner::Metric(obj) => obj.validate(errors), + ObjectInner::Metadata(obj) => obj.validate(errors), ObjectInner::Metrics(obj) => obj.validate(errors), - ObjectInner::MetricsStore(obj) => obj.validate(errors), ObjectInner::MtaConnectionStrategy(obj) => obj.validate(errors), ObjectInner::MtaDeliverySchedule(obj) => obj.validate(errors), ObjectInner::MtaExtensions(obj) => obj.validate(errors), @@ -5646,6 +5148,7 @@ impl Object { ObjectInner::MtaVirtualQueue(obj) => obj.validate(errors), ObjectInner::NetworkListener(obj) => obj.validate(errors), ObjectInner::OAuthClient(obj) => obj.validate(errors), + ObjectInner::OAuthGrant(obj) => obj.validate(errors), ObjectInner::OidcProvider(obj) => obj.validate(errors), ObjectInner::PublicKey(obj) => obj.validate(errors), ObjectInner::QueuedMessage(obj) => obj.validate(errors), @@ -5664,7 +5167,6 @@ impl Object { ObjectInner::SpamDnsblServer(obj) => obj.validate(errors), ObjectInner::SpamDnsblSettings(obj) => obj.validate(errors), ObjectInner::SpamFileExtension(obj) => obj.validate(errors), - ObjectInner::SpamLlm(obj) => obj.validate(errors), ObjectInner::SpamPyzor(obj) => obj.validate(errors), ObjectInner::SpamRule(obj) => obj.validate(errors), ObjectInner::SpamSettings(obj) => obj.validate(errors), @@ -5675,13 +5177,10 @@ impl Object { ObjectInner::SystemSettings(obj) => obj.validate(errors), ObjectInner::Task(obj) => obj.validate(errors), ObjectInner::TaskManager(obj) => obj.validate(errors), - ObjectInner::Tenant(obj) => obj.validate(errors), ObjectInner::TlsExternalReport(obj) => obj.validate(errors), ObjectInner::TlsInternalReport(obj) => obj.validate(errors), ObjectInner::TlsReportSettings(obj) => obj.validate(errors), - ObjectInner::Trace(obj) => obj.validate(errors), ObjectInner::Tracer(obj) => obj.validate(errors), - ObjectInner::TracingStore(obj) => obj.validate(errors), ObjectInner::WebDav(obj) => obj.validate(errors), ObjectInner::WebHook(obj) => obj.validate(errors), } @@ -5695,13 +5194,10 @@ impl Object { ObjectInner::AcmeProvider(obj) => obj.index(i), ObjectInner::Action(obj) => obj.index(i), ObjectInner::AddressBook(obj) => obj.index(i), - ObjectInner::AiModel(obj) => obj.index(i), - ObjectInner::Alert(obj) => obj.index(i), ObjectInner::AllowedIp(obj) => obj.index(i), ObjectInner::ApiKey(obj) => obj.index(i), ObjectInner::AppPassword(obj) => obj.index(i), ObjectInner::Application(obj) => obj.index(i), - ObjectInner::ArchivedItem(obj) => obj.index(i), ObjectInner::ArfExternalReport(obj) => obj.index(i), ObjectInner::Asn(obj) => obj.index(i), ObjectInner::Authentication(obj) => obj.index(i), @@ -5729,7 +5225,6 @@ impl Object { ObjectInner::Domain(obj) => obj.index(i), ObjectInner::DsnReportSettings(obj) => obj.index(i), ObjectInner::Email(obj) => obj.index(i), - ObjectInner::Enterprise(obj) => obj.index(i), ObjectInner::EventTracingLevel(obj) => obj.index(i), ObjectInner::FileStorage(obj) => obj.index(i), ObjectInner::Http(obj) => obj.index(i), @@ -5737,15 +5232,14 @@ impl Object { ObjectInner::HttpLookup(obj) => obj.index(i), ObjectInner::Imap(obj) => obj.index(i), ObjectInner::InMemoryStore(obj) => obj.index(i), + ObjectInner::IndexQueueStatus(obj) => obj.index(i), ObjectInner::Jmap(obj) => obj.index(i), ObjectInner::Log(obj) => obj.index(i), ObjectInner::MailingList(obj) => obj.index(i), - ObjectInner::MaskedEmail(obj) => obj.index(i), ObjectInner::MemoryLookupKey(obj) => obj.index(i), ObjectInner::MemoryLookupKeyValue(obj) => obj.index(i), - ObjectInner::Metric(obj) => obj.index(i), + ObjectInner::Metadata(obj) => obj.index(i), ObjectInner::Metrics(obj) => obj.index(i), - ObjectInner::MetricsStore(obj) => obj.index(i), ObjectInner::MtaConnectionStrategy(obj) => obj.index(i), ObjectInner::MtaDeliverySchedule(obj) => obj.index(i), ObjectInner::MtaExtensions(obj) => obj.index(i), @@ -5768,6 +5262,7 @@ impl Object { ObjectInner::MtaVirtualQueue(obj) => obj.index(i), ObjectInner::NetworkListener(obj) => obj.index(i), ObjectInner::OAuthClient(obj) => obj.index(i), + ObjectInner::OAuthGrant(obj) => obj.index(i), ObjectInner::OidcProvider(obj) => obj.index(i), ObjectInner::PublicKey(obj) => obj.index(i), ObjectInner::QueuedMessage(obj) => obj.index(i), @@ -5786,7 +5281,6 @@ impl Object { ObjectInner::SpamDnsblServer(obj) => obj.index(i), ObjectInner::SpamDnsblSettings(obj) => obj.index(i), ObjectInner::SpamFileExtension(obj) => obj.index(i), - ObjectInner::SpamLlm(obj) => obj.index(i), ObjectInner::SpamPyzor(obj) => obj.index(i), ObjectInner::SpamRule(obj) => obj.index(i), ObjectInner::SpamSettings(obj) => obj.index(i), @@ -5797,13 +5291,10 @@ impl Object { ObjectInner::SystemSettings(obj) => obj.index(i), ObjectInner::Task(obj) => obj.index(i), ObjectInner::TaskManager(obj) => obj.index(i), - ObjectInner::Tenant(obj) => obj.index(i), ObjectInner::TlsExternalReport(obj) => obj.index(i), ObjectInner::TlsInternalReport(obj) => obj.index(i), ObjectInner::TlsReportSettings(obj) => obj.index(i), - ObjectInner::Trace(obj) => obj.index(i), ObjectInner::Tracer(obj) => obj.index(i), - ObjectInner::TracingStore(obj) => obj.index(i), ObjectInner::WebDav(obj) => obj.index(i), ObjectInner::WebHook(obj) => obj.index(i), } @@ -5821,13 +5312,10 @@ impl Object { ObjectInner::AcmeProvider(obj) => obj.patch(pointer, value), ObjectInner::Action(obj) => obj.patch(pointer, value), ObjectInner::AddressBook(obj) => obj.patch(pointer, value), - ObjectInner::AiModel(obj) => obj.patch(pointer, value), - ObjectInner::Alert(obj) => obj.patch(pointer, value), ObjectInner::AllowedIp(obj) => obj.patch(pointer, value), ObjectInner::ApiKey(obj) => obj.patch(pointer, value), ObjectInner::AppPassword(obj) => obj.patch(pointer, value), ObjectInner::Application(obj) => obj.patch(pointer, value), - ObjectInner::ArchivedItem(obj) => obj.patch(pointer, value), ObjectInner::ArfExternalReport(obj) => obj.patch(pointer, value), ObjectInner::Asn(obj) => obj.patch(pointer, value), ObjectInner::Authentication(obj) => obj.patch(pointer, value), @@ -5855,7 +5343,6 @@ impl Object { ObjectInner::Domain(obj) => obj.patch(pointer, value), ObjectInner::DsnReportSettings(obj) => obj.patch(pointer, value), ObjectInner::Email(obj) => obj.patch(pointer, value), - ObjectInner::Enterprise(obj) => obj.patch(pointer, value), ObjectInner::EventTracingLevel(obj) => obj.patch(pointer, value), ObjectInner::FileStorage(obj) => obj.patch(pointer, value), ObjectInner::Http(obj) => obj.patch(pointer, value), @@ -5863,15 +5350,14 @@ impl Object { ObjectInner::HttpLookup(obj) => obj.patch(pointer, value), ObjectInner::Imap(obj) => obj.patch(pointer, value), ObjectInner::InMemoryStore(obj) => obj.patch(pointer, value), + ObjectInner::IndexQueueStatus(obj) => obj.patch(pointer, value), ObjectInner::Jmap(obj) => obj.patch(pointer, value), ObjectInner::Log(obj) => obj.patch(pointer, value), ObjectInner::MailingList(obj) => obj.patch(pointer, value), - ObjectInner::MaskedEmail(obj) => obj.patch(pointer, value), ObjectInner::MemoryLookupKey(obj) => obj.patch(pointer, value), ObjectInner::MemoryLookupKeyValue(obj) => obj.patch(pointer, value), - ObjectInner::Metric(obj) => obj.patch(pointer, value), + ObjectInner::Metadata(obj) => obj.patch(pointer, value), ObjectInner::Metrics(obj) => obj.patch(pointer, value), - ObjectInner::MetricsStore(obj) => obj.patch(pointer, value), ObjectInner::MtaConnectionStrategy(obj) => obj.patch(pointer, value), ObjectInner::MtaDeliverySchedule(obj) => obj.patch(pointer, value), ObjectInner::MtaExtensions(obj) => obj.patch(pointer, value), @@ -5894,6 +5380,7 @@ impl Object { ObjectInner::MtaVirtualQueue(obj) => obj.patch(pointer, value), ObjectInner::NetworkListener(obj) => obj.patch(pointer, value), ObjectInner::OAuthClient(obj) => obj.patch(pointer, value), + ObjectInner::OAuthGrant(obj) => obj.patch(pointer, value), ObjectInner::OidcProvider(obj) => obj.patch(pointer, value), ObjectInner::PublicKey(obj) => obj.patch(pointer, value), ObjectInner::QueuedMessage(obj) => obj.patch(pointer, value), @@ -5912,7 +5399,6 @@ impl Object { ObjectInner::SpamDnsblServer(obj) => obj.patch(pointer, value), ObjectInner::SpamDnsblSettings(obj) => obj.patch(pointer, value), ObjectInner::SpamFileExtension(obj) => obj.patch(pointer, value), - ObjectInner::SpamLlm(obj) => obj.patch(pointer, value), ObjectInner::SpamPyzor(obj) => obj.patch(pointer, value), ObjectInner::SpamRule(obj) => obj.patch(pointer, value), ObjectInner::SpamSettings(obj) => obj.patch(pointer, value), @@ -5923,13 +5409,10 @@ impl Object { ObjectInner::SystemSettings(obj) => obj.patch(pointer, value), ObjectInner::Task(obj) => obj.patch(pointer, value), ObjectInner::TaskManager(obj) => obj.patch(pointer, value), - ObjectInner::Tenant(obj) => obj.patch(pointer, value), ObjectInner::TlsExternalReport(obj) => obj.patch(pointer, value), ObjectInner::TlsInternalReport(obj) => obj.patch(pointer, value), ObjectInner::TlsReportSettings(obj) => obj.patch(pointer, value), - ObjectInner::Trace(obj) => obj.patch(pointer, value), ObjectInner::Tracer(obj) => obj.patch(pointer, value), - ObjectInner::TracingStore(obj) => obj.patch(pointer, value), ObjectInner::WebDav(obj) => obj.patch(pointer, value), ObjectInner::WebHook(obj) => obj.patch(pointer, value), } @@ -5938,124 +5421,120 @@ impl Object { impl IntoValue for Object { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self.inner { - ObjectInner::Account(obj) => obj.into_value(), - ObjectInner::AccountPassword(obj) => obj.into_value(), - ObjectInner::AccountSettings(obj) => obj.into_value(), - ObjectInner::AcmeProvider(obj) => obj.into_value(), - ObjectInner::Action(obj) => obj.into_value(), - ObjectInner::AddressBook(obj) => obj.into_value(), - ObjectInner::AiModel(obj) => obj.into_value(), - ObjectInner::Alert(obj) => obj.into_value(), - ObjectInner::AllowedIp(obj) => obj.into_value(), - ObjectInner::ApiKey(obj) => obj.into_value(), - ObjectInner::AppPassword(obj) => obj.into_value(), - ObjectInner::Application(obj) => obj.into_value(), - ObjectInner::ArchivedItem(obj) => obj.into_value(), - ObjectInner::ArfExternalReport(obj) => obj.into_value(), - ObjectInner::Asn(obj) => obj.into_value(), - ObjectInner::Authentication(obj) => obj.into_value(), - ObjectInner::BlobStore(obj) => obj.into_value(), - ObjectInner::BlockedIp(obj) => obj.into_value(), - ObjectInner::Bootstrap(obj) => obj.into_value(), - ObjectInner::Cache(obj) => obj.into_value(), - ObjectInner::Calendar(obj) => obj.into_value(), - ObjectInner::CalendarAlarm(obj) => obj.into_value(), - ObjectInner::CalendarScheduling(obj) => obj.into_value(), - ObjectInner::Certificate(obj) => obj.into_value(), - ObjectInner::ClusterNode(obj) => obj.into_value(), - ObjectInner::ClusterRole(obj) => obj.into_value(), - ObjectInner::Coordinator(obj) => obj.into_value(), - ObjectInner::DataRetention(obj) => obj.into_value(), - ObjectInner::DataStore(obj) => obj.into_value(), - ObjectInner::Directory(obj) => obj.into_value(), - ObjectInner::DkimReportSettings(obj) => obj.into_value(), - ObjectInner::DkimSignature(obj) => obj.into_value(), - ObjectInner::DmarcExternalReport(obj) => obj.into_value(), - ObjectInner::DmarcInternalReport(obj) => obj.into_value(), - ObjectInner::DmarcReportSettings(obj) => obj.into_value(), - ObjectInner::DnsResolver(obj) => obj.into_value(), - ObjectInner::DnsServer(obj) => obj.into_value(), - ObjectInner::Domain(obj) => obj.into_value(), - ObjectInner::DsnReportSettings(obj) => obj.into_value(), - ObjectInner::Email(obj) => obj.into_value(), - ObjectInner::Enterprise(obj) => obj.into_value(), - ObjectInner::EventTracingLevel(obj) => obj.into_value(), - ObjectInner::FileStorage(obj) => obj.into_value(), - ObjectInner::Http(obj) => obj.into_value(), - ObjectInner::HttpForm(obj) => obj.into_value(), - ObjectInner::HttpLookup(obj) => obj.into_value(), - ObjectInner::Imap(obj) => obj.into_value(), - ObjectInner::InMemoryStore(obj) => obj.into_value(), - ObjectInner::Jmap(obj) => obj.into_value(), - ObjectInner::Log(obj) => obj.into_value(), - ObjectInner::MailingList(obj) => obj.into_value(), - ObjectInner::MaskedEmail(obj) => obj.into_value(), - ObjectInner::MemoryLookupKey(obj) => obj.into_value(), - ObjectInner::MemoryLookupKeyValue(obj) => obj.into_value(), - ObjectInner::Metric(obj) => obj.into_value(), - ObjectInner::Metrics(obj) => obj.into_value(), - ObjectInner::MetricsStore(obj) => obj.into_value(), - ObjectInner::MtaConnectionStrategy(obj) => obj.into_value(), - ObjectInner::MtaDeliverySchedule(obj) => obj.into_value(), - ObjectInner::MtaExtensions(obj) => obj.into_value(), - ObjectInner::MtaHook(obj) => obj.into_value(), - ObjectInner::MtaInboundSession(obj) => obj.into_value(), - ObjectInner::MtaInboundThrottle(obj) => obj.into_value(), - ObjectInner::MtaMilter(obj) => obj.into_value(), - ObjectInner::MtaOutboundStrategy(obj) => obj.into_value(), - ObjectInner::MtaOutboundThrottle(obj) => obj.into_value(), - ObjectInner::MtaQueueQuota(obj) => obj.into_value(), - ObjectInner::MtaRoute(obj) => obj.into_value(), - ObjectInner::MtaStageAuth(obj) => obj.into_value(), - ObjectInner::MtaStageConnect(obj) => obj.into_value(), - ObjectInner::MtaStageData(obj) => obj.into_value(), - ObjectInner::MtaStageEhlo(obj) => obj.into_value(), - ObjectInner::MtaStageMail(obj) => obj.into_value(), - ObjectInner::MtaStageRcpt(obj) => obj.into_value(), - ObjectInner::MtaSts(obj) => obj.into_value(), - ObjectInner::MtaTlsStrategy(obj) => obj.into_value(), - ObjectInner::MtaVirtualQueue(obj) => obj.into_value(), - ObjectInner::NetworkListener(obj) => obj.into_value(), - ObjectInner::OAuthClient(obj) => obj.into_value(), - ObjectInner::OidcProvider(obj) => obj.into_value(), - ObjectInner::PublicKey(obj) => obj.into_value(), - ObjectInner::QueuedMessage(obj) => obj.into_value(), - ObjectInner::ReportSettings(obj) => obj.into_value(), - ObjectInner::Role(obj) => obj.into_value(), - ObjectInner::Search(obj) => obj.into_value(), - ObjectInner::SearchStore(obj) => obj.into_value(), - ObjectInner::Security(obj) => obj.into_value(), - ObjectInner::SenderAuth(obj) => obj.into_value(), - ObjectInner::Sharing(obj) => obj.into_value(), - ObjectInner::SieveSystemInterpreter(obj) => obj.into_value(), - ObjectInner::SieveSystemScript(obj) => obj.into_value(), - ObjectInner::SieveUserInterpreter(obj) => obj.into_value(), - ObjectInner::SieveUserScript(obj) => obj.into_value(), - ObjectInner::SpamClassifier(obj) => obj.into_value(), - ObjectInner::SpamDnsblServer(obj) => obj.into_value(), - ObjectInner::SpamDnsblSettings(obj) => obj.into_value(), - ObjectInner::SpamFileExtension(obj) => obj.into_value(), - ObjectInner::SpamLlm(obj) => obj.into_value(), - ObjectInner::SpamPyzor(obj) => obj.into_value(), - ObjectInner::SpamRule(obj) => obj.into_value(), - ObjectInner::SpamSettings(obj) => obj.into_value(), - ObjectInner::SpamTag(obj) => obj.into_value(), - ObjectInner::SpamTrainingSample(obj) => obj.into_value(), - ObjectInner::SpfReportSettings(obj) => obj.into_value(), - ObjectInner::StoreLookup(obj) => obj.into_value(), - ObjectInner::SystemSettings(obj) => obj.into_value(), - ObjectInner::Task(obj) => obj.into_value(), - ObjectInner::TaskManager(obj) => obj.into_value(), - ObjectInner::Tenant(obj) => obj.into_value(), - ObjectInner::TlsExternalReport(obj) => obj.into_value(), - ObjectInner::TlsInternalReport(obj) => obj.into_value(), - ObjectInner::TlsReportSettings(obj) => obj.into_value(), - ObjectInner::Trace(obj) => obj.into_value(), - ObjectInner::Tracer(obj) => obj.into_value(), - ObjectInner::TracingStore(obj) => obj.into_value(), - ObjectInner::WebDav(obj) => obj.into_value(), - ObjectInner::WebHook(obj) => obj.into_value(), + ObjectInner::Account(obj) => (*obj).into_value_with(opts), + ObjectInner::AccountPassword(obj) => (*obj).into_value_with(opts), + ObjectInner::AccountSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::AcmeProvider(obj) => (*obj).into_value_with(opts), + ObjectInner::Action(obj) => (*obj).into_value_with(opts), + ObjectInner::AddressBook(obj) => (*obj).into_value_with(opts), + ObjectInner::AllowedIp(obj) => (*obj).into_value_with(opts), + ObjectInner::ApiKey(obj) => (*obj).into_value_with(opts), + ObjectInner::AppPassword(obj) => (*obj).into_value_with(opts), + ObjectInner::Application(obj) => (*obj).into_value_with(opts), + ObjectInner::ArfExternalReport(obj) => (*obj).into_value_with(opts), + ObjectInner::Asn(obj) => (*obj).into_value_with(opts), + ObjectInner::Authentication(obj) => (*obj).into_value_with(opts), + ObjectInner::BlobStore(obj) => (*obj).into_value_with(opts), + ObjectInner::BlockedIp(obj) => (*obj).into_value_with(opts), + ObjectInner::Bootstrap(obj) => (*obj).into_value_with(opts), + ObjectInner::Cache(obj) => (*obj).into_value_with(opts), + ObjectInner::Calendar(obj) => (*obj).into_value_with(opts), + ObjectInner::CalendarAlarm(obj) => (*obj).into_value_with(opts), + ObjectInner::CalendarScheduling(obj) => (*obj).into_value_with(opts), + ObjectInner::Certificate(obj) => (*obj).into_value_with(opts), + ObjectInner::ClusterNode(obj) => (*obj).into_value_with(opts), + ObjectInner::ClusterRole(obj) => (*obj).into_value_with(opts), + ObjectInner::Coordinator(obj) => (*obj).into_value_with(opts), + ObjectInner::DataRetention(obj) => (*obj).into_value_with(opts), + ObjectInner::DataStore(obj) => (*obj).into_value_with(opts), + ObjectInner::Directory(obj) => (*obj).into_value_with(opts), + ObjectInner::DkimReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::DkimSignature(obj) => (*obj).into_value_with(opts), + ObjectInner::DmarcExternalReport(obj) => (*obj).into_value_with(opts), + ObjectInner::DmarcInternalReport(obj) => (*obj).into_value_with(opts), + ObjectInner::DmarcReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::DnsResolver(obj) => (*obj).into_value_with(opts), + ObjectInner::DnsServer(obj) => (*obj).into_value_with(opts), + ObjectInner::Domain(obj) => (*obj).into_value_with(opts), + ObjectInner::DsnReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::Email(obj) => (*obj).into_value_with(opts), + ObjectInner::EventTracingLevel(obj) => (*obj).into_value_with(opts), + ObjectInner::FileStorage(obj) => (*obj).into_value_with(opts), + ObjectInner::Http(obj) => (*obj).into_value_with(opts), + ObjectInner::HttpForm(obj) => (*obj).into_value_with(opts), + ObjectInner::HttpLookup(obj) => (*obj).into_value_with(opts), + ObjectInner::Imap(obj) => (*obj).into_value_with(opts), + ObjectInner::InMemoryStore(obj) => (*obj).into_value_with(opts), + ObjectInner::IndexQueueStatus(obj) => (*obj).into_value_with(opts), + ObjectInner::Jmap(obj) => (*obj).into_value_with(opts), + ObjectInner::Log(obj) => (*obj).into_value_with(opts), + ObjectInner::MailingList(obj) => (*obj).into_value_with(opts), + ObjectInner::MemoryLookupKey(obj) => (*obj).into_value_with(opts), + ObjectInner::MemoryLookupKeyValue(obj) => (*obj).into_value_with(opts), + ObjectInner::Metadata(obj) => (*obj).into_value_with(opts), + ObjectInner::Metrics(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaConnectionStrategy(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaDeliverySchedule(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaExtensions(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaHook(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaInboundSession(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaInboundThrottle(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaMilter(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaOutboundStrategy(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaOutboundThrottle(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaQueueQuota(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaRoute(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageAuth(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageConnect(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageData(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageEhlo(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageMail(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaStageRcpt(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaSts(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaTlsStrategy(obj) => (*obj).into_value_with(opts), + ObjectInner::MtaVirtualQueue(obj) => (*obj).into_value_with(opts), + ObjectInner::NetworkListener(obj) => (*obj).into_value_with(opts), + ObjectInner::OAuthClient(obj) => (*obj).into_value_with(opts), + ObjectInner::OAuthGrant(obj) => (*obj).into_value_with(opts), + ObjectInner::OidcProvider(obj) => (*obj).into_value_with(opts), + ObjectInner::PublicKey(obj) => (*obj).into_value_with(opts), + ObjectInner::QueuedMessage(obj) => (*obj).into_value_with(opts), + ObjectInner::ReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::Role(obj) => (*obj).into_value_with(opts), + ObjectInner::Search(obj) => (*obj).into_value_with(opts), + ObjectInner::SearchStore(obj) => (*obj).into_value_with(opts), + ObjectInner::Security(obj) => (*obj).into_value_with(opts), + ObjectInner::SenderAuth(obj) => (*obj).into_value_with(opts), + ObjectInner::Sharing(obj) => (*obj).into_value_with(opts), + ObjectInner::SieveSystemInterpreter(obj) => (*obj).into_value_with(opts), + ObjectInner::SieveSystemScript(obj) => (*obj).into_value_with(opts), + ObjectInner::SieveUserInterpreter(obj) => (*obj).into_value_with(opts), + ObjectInner::SieveUserScript(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamClassifier(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamDnsblServer(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamDnsblSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamFileExtension(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamPyzor(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamRule(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamTag(obj) => (*obj).into_value_with(opts), + ObjectInner::SpamTrainingSample(obj) => (*obj).into_value_with(opts), + ObjectInner::SpfReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::StoreLookup(obj) => (*obj).into_value_with(opts), + ObjectInner::SystemSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::Task(obj) => (*obj).into_value_with(opts), + ObjectInner::TaskManager(obj) => (*obj).into_value_with(opts), + ObjectInner::TlsExternalReport(obj) => (*obj).into_value_with(opts), + ObjectInner::TlsInternalReport(obj) => (*obj).into_value_with(opts), + ObjectInner::TlsReportSettings(obj) => (*obj).into_value_with(opts), + ObjectInner::Tracer(obj) => (*obj).into_value_with(opts), + ObjectInner::WebDav(obj) => (*obj).into_value_with(opts), + ObjectInner::WebHook(obj) => (*obj).into_value_with(opts), } } } @@ -6063,145 +5542,131 @@ impl IntoValue for Object { impl From for ObjectInner { fn from(obj: ObjectType) -> Self { match obj { - ObjectType::Account => ObjectInner::Account(Default::default()), - ObjectType::AccountPassword => ObjectInner::AccountPassword(Default::default()), - ObjectType::AccountSettings => ObjectInner::AccountSettings(Default::default()), - ObjectType::AcmeProvider => ObjectInner::AcmeProvider(Default::default()), - ObjectType::Action => ObjectInner::Action(Default::default()), - ObjectType::AddressBook => ObjectInner::AddressBook(Default::default()), - ObjectType::AiModel => ObjectInner::AiModel(Default::default()), - ObjectType::Alert => ObjectInner::Alert(Default::default()), - ObjectType::AllowedIp => ObjectInner::AllowedIp(Default::default()), - ObjectType::ApiKey => ObjectInner::ApiKey(Default::default()), - ObjectType::AppPassword => ObjectInner::AppPassword(Default::default()), - ObjectType::Application => ObjectInner::Application(Default::default()), - ObjectType::ArchivedItem => ObjectInner::ArchivedItem(Default::default()), - ObjectType::ArfExternalReport => ObjectInner::ArfExternalReport(Default::default()), - ObjectType::Asn => ObjectInner::Asn(Default::default()), - ObjectType::Authentication => ObjectInner::Authentication(Default::default()), - ObjectType::BlobStore => ObjectInner::BlobStore(Default::default()), - ObjectType::BlockedIp => ObjectInner::BlockedIp(Default::default()), - ObjectType::Bootstrap => ObjectInner::Bootstrap(Default::default()), - ObjectType::Cache => ObjectInner::Cache(Default::default()), - ObjectType::Calendar => ObjectInner::Calendar(Default::default()), - ObjectType::CalendarAlarm => ObjectInner::CalendarAlarm(Default::default()), - ObjectType::CalendarScheduling => ObjectInner::CalendarScheduling(Default::default()), - ObjectType::Certificate => ObjectInner::Certificate(Default::default()), - ObjectType::ClusterNode => ObjectInner::ClusterNode(Default::default()), - ObjectType::ClusterRole => ObjectInner::ClusterRole(Default::default()), - ObjectType::Coordinator => ObjectInner::Coordinator(Default::default()), - ObjectType::DataRetention => ObjectInner::DataRetention(Default::default()), - ObjectType::DataStore => ObjectInner::DataStore(Default::default()), - ObjectType::Directory => ObjectInner::Directory(Default::default()), - ObjectType::DkimReportSettings => ObjectInner::DkimReportSettings(Default::default()), - ObjectType::DkimSignature => ObjectInner::DkimSignature(Default::default()), - ObjectType::DmarcExternalReport => ObjectInner::DmarcExternalReport(Default::default()), - ObjectType::DmarcInternalReport => ObjectInner::DmarcInternalReport(Default::default()), - ObjectType::DmarcReportSettings => ObjectInner::DmarcReportSettings(Default::default()), - ObjectType::DnsResolver => ObjectInner::DnsResolver(Default::default()), - ObjectType::DnsServer => ObjectInner::DnsServer(Default::default()), - ObjectType::Domain => ObjectInner::Domain(Default::default()), - ObjectType::DsnReportSettings => ObjectInner::DsnReportSettings(Default::default()), - ObjectType::Email => ObjectInner::Email(Default::default()), - ObjectType::Enterprise => ObjectInner::Enterprise(Default::default()), - ObjectType::EventTracingLevel => ObjectInner::EventTracingLevel(Default::default()), - ObjectType::FileStorage => ObjectInner::FileStorage(Default::default()), - ObjectType::Http => ObjectInner::Http(Default::default()), - ObjectType::HttpForm => ObjectInner::HttpForm(Default::default()), - ObjectType::HttpLookup => ObjectInner::HttpLookup(Default::default()), - ObjectType::Imap => ObjectInner::Imap(Default::default()), - ObjectType::InMemoryStore => ObjectInner::InMemoryStore(Default::default()), - ObjectType::Jmap => ObjectInner::Jmap(Default::default()), - ObjectType::Log => ObjectInner::Log(Default::default()), - ObjectType::MailingList => ObjectInner::MailingList(Default::default()), - ObjectType::MaskedEmail => ObjectInner::MaskedEmail(Default::default()), - ObjectType::MemoryLookupKey => ObjectInner::MemoryLookupKey(Default::default()), - ObjectType::MemoryLookupKeyValue => { - ObjectInner::MemoryLookupKeyValue(Default::default()) - } - ObjectType::Metric => ObjectInner::Metric(Default::default()), - ObjectType::Metrics => ObjectInner::Metrics(Default::default()), - ObjectType::MetricsStore => ObjectInner::MetricsStore(Default::default()), - ObjectType::MtaConnectionStrategy => { - ObjectInner::MtaConnectionStrategy(Default::default()) - } - ObjectType::MtaDeliverySchedule => ObjectInner::MtaDeliverySchedule(Default::default()), - ObjectType::MtaExtensions => ObjectInner::MtaExtensions(Default::default()), - ObjectType::MtaHook => ObjectInner::MtaHook(Default::default()), - ObjectType::MtaInboundSession => ObjectInner::MtaInboundSession(Default::default()), - ObjectType::MtaInboundThrottle => ObjectInner::MtaInboundThrottle(Default::default()), - ObjectType::MtaMilter => ObjectInner::MtaMilter(Default::default()), - ObjectType::MtaOutboundStrategy => ObjectInner::MtaOutboundStrategy(Default::default()), - ObjectType::MtaOutboundThrottle => ObjectInner::MtaOutboundThrottle(Default::default()), - ObjectType::MtaQueueQuota => ObjectInner::MtaQueueQuota(Default::default()), - ObjectType::MtaRoute => ObjectInner::MtaRoute(Default::default()), - ObjectType::MtaStageAuth => ObjectInner::MtaStageAuth(Default::default()), - ObjectType::MtaStageConnect => ObjectInner::MtaStageConnect(Default::default()), - ObjectType::MtaStageData => ObjectInner::MtaStageData(Default::default()), - ObjectType::MtaStageEhlo => ObjectInner::MtaStageEhlo(Default::default()), - ObjectType::MtaStageMail => ObjectInner::MtaStageMail(Default::default()), - ObjectType::MtaStageRcpt => ObjectInner::MtaStageRcpt(Default::default()), - ObjectType::MtaSts => ObjectInner::MtaSts(Default::default()), - ObjectType::MtaTlsStrategy => ObjectInner::MtaTlsStrategy(Default::default()), - ObjectType::MtaVirtualQueue => ObjectInner::MtaVirtualQueue(Default::default()), - ObjectType::NetworkListener => ObjectInner::NetworkListener(Default::default()), - ObjectType::OAuthClient => ObjectInner::OAuthClient(Default::default()), - ObjectType::OidcProvider => ObjectInner::OidcProvider(Default::default()), - ObjectType::PublicKey => ObjectInner::PublicKey(Default::default()), - ObjectType::QueuedMessage => ObjectInner::QueuedMessage(Default::default()), - ObjectType::ReportSettings => ObjectInner::ReportSettings(Default::default()), - ObjectType::Role => ObjectInner::Role(Default::default()), - ObjectType::Search => ObjectInner::Search(Default::default()), - ObjectType::SearchStore => ObjectInner::SearchStore(Default::default()), - ObjectType::Security => ObjectInner::Security(Default::default()), - ObjectType::SenderAuth => ObjectInner::SenderAuth(Default::default()), - ObjectType::Sharing => ObjectInner::Sharing(Default::default()), + ObjectType::Account => ObjectInner::Account(Box::default()), + ObjectType::AccountPassword => ObjectInner::AccountPassword(Box::default()), + ObjectType::AccountSettings => ObjectInner::AccountSettings(Box::default()), + ObjectType::AcmeProvider => ObjectInner::AcmeProvider(Box::default()), + ObjectType::Action => ObjectInner::Action(Box::default()), + ObjectType::AddressBook => ObjectInner::AddressBook(Box::default()), + ObjectType::AllowedIp => ObjectInner::AllowedIp(Box::default()), + ObjectType::ApiKey => ObjectInner::ApiKey(Box::default()), + ObjectType::AppPassword => ObjectInner::AppPassword(Box::default()), + ObjectType::Application => ObjectInner::Application(Box::default()), + ObjectType::ArfExternalReport => ObjectInner::ArfExternalReport(Box::default()), + ObjectType::Asn => ObjectInner::Asn(Box::default()), + ObjectType::Authentication => ObjectInner::Authentication(Box::default()), + ObjectType::BlobStore => ObjectInner::BlobStore(Box::default()), + ObjectType::BlockedIp => ObjectInner::BlockedIp(Box::default()), + ObjectType::Bootstrap => ObjectInner::Bootstrap(Box::default()), + ObjectType::Cache => ObjectInner::Cache(Box::default()), + ObjectType::Calendar => ObjectInner::Calendar(Box::default()), + ObjectType::CalendarAlarm => ObjectInner::CalendarAlarm(Box::default()), + ObjectType::CalendarScheduling => ObjectInner::CalendarScheduling(Box::default()), + ObjectType::Certificate => ObjectInner::Certificate(Box::default()), + ObjectType::ClusterNode => ObjectInner::ClusterNode(Box::default()), + ObjectType::ClusterRole => ObjectInner::ClusterRole(Box::default()), + ObjectType::Coordinator => ObjectInner::Coordinator(Box::default()), + ObjectType::DataRetention => ObjectInner::DataRetention(Box::default()), + ObjectType::DataStore => ObjectInner::DataStore(Box::default()), + ObjectType::Directory => ObjectInner::Directory(Box::default()), + ObjectType::DkimReportSettings => ObjectInner::DkimReportSettings(Box::default()), + ObjectType::DkimSignature => ObjectInner::DkimSignature(Box::default()), + ObjectType::DmarcExternalReport => ObjectInner::DmarcExternalReport(Box::default()), + ObjectType::DmarcInternalReport => ObjectInner::DmarcInternalReport(Box::default()), + ObjectType::DmarcReportSettings => ObjectInner::DmarcReportSettings(Box::default()), + ObjectType::DnsResolver => ObjectInner::DnsResolver(Box::default()), + ObjectType::DnsServer => ObjectInner::DnsServer(Box::default()), + ObjectType::Domain => ObjectInner::Domain(Box::default()), + ObjectType::DsnReportSettings => ObjectInner::DsnReportSettings(Box::default()), + ObjectType::Email => ObjectInner::Email(Box::default()), + ObjectType::EventTracingLevel => ObjectInner::EventTracingLevel(Box::default()), + ObjectType::FileStorage => ObjectInner::FileStorage(Box::default()), + ObjectType::Http => ObjectInner::Http(Box::default()), + ObjectType::HttpForm => ObjectInner::HttpForm(Box::default()), + ObjectType::HttpLookup => ObjectInner::HttpLookup(Box::default()), + ObjectType::Imap => ObjectInner::Imap(Box::default()), + ObjectType::InMemoryStore => ObjectInner::InMemoryStore(Box::default()), + ObjectType::IndexQueueStatus => ObjectInner::IndexQueueStatus(Box::default()), + ObjectType::Jmap => ObjectInner::Jmap(Box::default()), + ObjectType::Log => ObjectInner::Log(Box::default()), + ObjectType::MailingList => ObjectInner::MailingList(Box::default()), + ObjectType::MemoryLookupKey => ObjectInner::MemoryLookupKey(Box::default()), + ObjectType::MemoryLookupKeyValue => ObjectInner::MemoryLookupKeyValue(Box::default()), + ObjectType::Metadata => ObjectInner::Metadata(Box::default()), + ObjectType::Metrics => ObjectInner::Metrics(Box::default()), + ObjectType::MtaConnectionStrategy => ObjectInner::MtaConnectionStrategy(Box::default()), + ObjectType::MtaDeliverySchedule => ObjectInner::MtaDeliverySchedule(Box::default()), + ObjectType::MtaExtensions => ObjectInner::MtaExtensions(Box::default()), + ObjectType::MtaHook => ObjectInner::MtaHook(Box::default()), + ObjectType::MtaInboundSession => ObjectInner::MtaInboundSession(Box::default()), + ObjectType::MtaInboundThrottle => ObjectInner::MtaInboundThrottle(Box::default()), + ObjectType::MtaMilter => ObjectInner::MtaMilter(Box::default()), + ObjectType::MtaOutboundStrategy => ObjectInner::MtaOutboundStrategy(Box::default()), + ObjectType::MtaOutboundThrottle => ObjectInner::MtaOutboundThrottle(Box::default()), + ObjectType::MtaQueueQuota => ObjectInner::MtaQueueQuota(Box::default()), + ObjectType::MtaRoute => ObjectInner::MtaRoute(Box::default()), + ObjectType::MtaStageAuth => ObjectInner::MtaStageAuth(Box::default()), + ObjectType::MtaStageConnect => ObjectInner::MtaStageConnect(Box::default()), + ObjectType::MtaStageData => ObjectInner::MtaStageData(Box::default()), + ObjectType::MtaStageEhlo => ObjectInner::MtaStageEhlo(Box::default()), + ObjectType::MtaStageMail => ObjectInner::MtaStageMail(Box::default()), + ObjectType::MtaStageRcpt => ObjectInner::MtaStageRcpt(Box::default()), + ObjectType::MtaSts => ObjectInner::MtaSts(Box::default()), + ObjectType::MtaTlsStrategy => ObjectInner::MtaTlsStrategy(Box::default()), + ObjectType::MtaVirtualQueue => ObjectInner::MtaVirtualQueue(Box::default()), + ObjectType::NetworkListener => ObjectInner::NetworkListener(Box::default()), + ObjectType::OAuthClient => ObjectInner::OAuthClient(Box::default()), + ObjectType::OAuthGrant => ObjectInner::OAuthGrant(Box::default()), + ObjectType::OidcProvider => ObjectInner::OidcProvider(Box::default()), + ObjectType::PublicKey => ObjectInner::PublicKey(Box::default()), + ObjectType::QueuedMessage => ObjectInner::QueuedMessage(Box::default()), + ObjectType::ReportSettings => ObjectInner::ReportSettings(Box::default()), + ObjectType::Role => ObjectInner::Role(Box::default()), + ObjectType::Search => ObjectInner::Search(Box::default()), + ObjectType::SearchStore => ObjectInner::SearchStore(Box::default()), + ObjectType::Security => ObjectInner::Security(Box::default()), + ObjectType::SenderAuth => ObjectInner::SenderAuth(Box::default()), + ObjectType::Sharing => ObjectInner::Sharing(Box::default()), ObjectType::SieveSystemInterpreter => { - ObjectInner::SieveSystemInterpreter(Default::default()) + ObjectInner::SieveSystemInterpreter(Box::default()) } - ObjectType::SieveSystemScript => ObjectInner::SieveSystemScript(Default::default()), - ObjectType::SieveUserInterpreter => { - ObjectInner::SieveUserInterpreter(Default::default()) - } - ObjectType::SieveUserScript => ObjectInner::SieveUserScript(Default::default()), - ObjectType::SpamClassifier => ObjectInner::SpamClassifier(Default::default()), - ObjectType::SpamDnsblServer => ObjectInner::SpamDnsblServer(Default::default()), - ObjectType::SpamDnsblSettings => ObjectInner::SpamDnsblSettings(Default::default()), - ObjectType::SpamFileExtension => ObjectInner::SpamFileExtension(Default::default()), - ObjectType::SpamLlm => ObjectInner::SpamLlm(Default::default()), - ObjectType::SpamPyzor => ObjectInner::SpamPyzor(Default::default()), - ObjectType::SpamRule => ObjectInner::SpamRule(Default::default()), - ObjectType::SpamSettings => ObjectInner::SpamSettings(Default::default()), - ObjectType::SpamTag => ObjectInner::SpamTag(Default::default()), - ObjectType::SpamTrainingSample => ObjectInner::SpamTrainingSample(Default::default()), - ObjectType::SpfReportSettings => ObjectInner::SpfReportSettings(Default::default()), - ObjectType::StoreLookup => ObjectInner::StoreLookup(Default::default()), - ObjectType::SystemSettings => ObjectInner::SystemSettings(Default::default()), - ObjectType::Task => ObjectInner::Task(Default::default()), - ObjectType::TaskManager => ObjectInner::TaskManager(Default::default()), - ObjectType::Tenant => ObjectInner::Tenant(Default::default()), - ObjectType::TlsExternalReport => ObjectInner::TlsExternalReport(Default::default()), - ObjectType::TlsInternalReport => ObjectInner::TlsInternalReport(Default::default()), - ObjectType::TlsReportSettings => ObjectInner::TlsReportSettings(Default::default()), - ObjectType::Trace => ObjectInner::Trace(Default::default()), - ObjectType::Tracer => ObjectInner::Tracer(Default::default()), - ObjectType::TracingStore => ObjectInner::TracingStore(Default::default()), - ObjectType::WebDav => ObjectInner::WebDav(Default::default()), - ObjectType::WebHook => ObjectInner::WebHook(Default::default()), + ObjectType::SieveSystemScript => ObjectInner::SieveSystemScript(Box::default()), + ObjectType::SieveUserInterpreter => ObjectInner::SieveUserInterpreter(Box::default()), + ObjectType::SieveUserScript => ObjectInner::SieveUserScript(Box::default()), + ObjectType::SpamClassifier => ObjectInner::SpamClassifier(Box::default()), + ObjectType::SpamDnsblServer => ObjectInner::SpamDnsblServer(Box::default()), + ObjectType::SpamDnsblSettings => ObjectInner::SpamDnsblSettings(Box::default()), + ObjectType::SpamFileExtension => ObjectInner::SpamFileExtension(Box::default()), + ObjectType::SpamPyzor => ObjectInner::SpamPyzor(Box::default()), + ObjectType::SpamRule => ObjectInner::SpamRule(Box::default()), + ObjectType::SpamSettings => ObjectInner::SpamSettings(Box::default()), + ObjectType::SpamTag => ObjectInner::SpamTag(Box::default()), + ObjectType::SpamTrainingSample => ObjectInner::SpamTrainingSample(Box::default()), + ObjectType::SpfReportSettings => ObjectInner::SpfReportSettings(Box::default()), + ObjectType::StoreLookup => ObjectInner::StoreLookup(Box::default()), + ObjectType::SystemSettings => ObjectInner::SystemSettings(Box::default()), + ObjectType::Task => ObjectInner::Task(Box::default()), + ObjectType::TaskManager => ObjectInner::TaskManager(Box::default()), + ObjectType::TlsExternalReport => ObjectInner::TlsExternalReport(Box::default()), + ObjectType::TlsInternalReport => ObjectInner::TlsInternalReport(Box::default()), + ObjectType::TlsReportSettings => ObjectInner::TlsReportSettings(Box::default()), + ObjectType::Tracer => ObjectInner::Tracer(Box::default()), + ObjectType::WebDav => ObjectInner::WebDav(Box::default()), + ObjectType::WebHook => ObjectInner::WebHook(Box::default()), } } } impl From for ObjectInner { fn from(value: Account) -> Self { - ObjectInner::Account(value) + ObjectInner::Account(Box::new(value)) } } impl From for Account { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Account(obj) => obj, + ObjectInner::Account(obj) => *obj, _ => unreachable!(), } } @@ -6209,14 +5674,14 @@ impl From for Account { impl From for ObjectInner { fn from(value: AccountPassword) -> Self { - ObjectInner::AccountPassword(value) + ObjectInner::AccountPassword(Box::new(value)) } } impl From for AccountPassword { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AccountPassword(obj) => obj, + ObjectInner::AccountPassword(obj) => *obj, _ => unreachable!(), } } @@ -6224,14 +5689,14 @@ impl From for AccountPassword { impl From for ObjectInner { fn from(value: AccountSettings) -> Self { - ObjectInner::AccountSettings(value) + ObjectInner::AccountSettings(Box::new(value)) } } impl From for AccountSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AccountSettings(obj) => obj, + ObjectInner::AccountSettings(obj) => *obj, _ => unreachable!(), } } @@ -6239,14 +5704,14 @@ impl From for AccountSettings { impl From for ObjectInner { fn from(value: AcmeProvider) -> Self { - ObjectInner::AcmeProvider(value) + ObjectInner::AcmeProvider(Box::new(value)) } } impl From for AcmeProvider { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AcmeProvider(obj) => obj, + ObjectInner::AcmeProvider(obj) => *obj, _ => unreachable!(), } } @@ -6254,14 +5719,14 @@ impl From for AcmeProvider { impl From for ObjectInner { fn from(value: Action) -> Self { - ObjectInner::Action(value) + ObjectInner::Action(Box::new(value)) } } impl From for Action { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Action(obj) => obj, + ObjectInner::Action(obj) => *obj, _ => unreachable!(), } } @@ -6269,44 +5734,14 @@ impl From for Action { impl From for ObjectInner { fn from(value: AddressBook) -> Self { - ObjectInner::AddressBook(value) + ObjectInner::AddressBook(Box::new(value)) } } impl From for AddressBook { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AddressBook(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: AiModel) -> Self { - ObjectInner::AiModel(value) - } -} - -impl From for AiModel { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::AiModel(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: Alert) -> Self { - ObjectInner::Alert(value) - } -} - -impl From for Alert { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::Alert(obj) => obj, + ObjectInner::AddressBook(obj) => *obj, _ => unreachable!(), } } @@ -6314,14 +5749,14 @@ impl From for Alert { impl From for ObjectInner { fn from(value: AllowedIp) -> Self { - ObjectInner::AllowedIp(value) + ObjectInner::AllowedIp(Box::new(value)) } } impl From for AllowedIp { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AllowedIp(obj) => obj, + ObjectInner::AllowedIp(obj) => *obj, _ => unreachable!(), } } @@ -6329,14 +5764,14 @@ impl From for AllowedIp { impl From for ObjectInner { fn from(value: ApiKey) -> Self { - ObjectInner::ApiKey(value) + ObjectInner::ApiKey(Box::new(value)) } } impl From for ApiKey { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::ApiKey(obj) => obj, + ObjectInner::ApiKey(obj) => *obj, _ => unreachable!(), } } @@ -6344,14 +5779,14 @@ impl From for ApiKey { impl From for ObjectInner { fn from(value: AppPassword) -> Self { - ObjectInner::AppPassword(value) + ObjectInner::AppPassword(Box::new(value)) } } impl From for AppPassword { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::AppPassword(obj) => obj, + ObjectInner::AppPassword(obj) => *obj, _ => unreachable!(), } } @@ -6359,29 +5794,14 @@ impl From for AppPassword { impl From for ObjectInner { fn from(value: Application) -> Self { - ObjectInner::Application(value) + ObjectInner::Application(Box::new(value)) } } impl From for Application { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Application(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: ArchivedItem) -> Self { - ObjectInner::ArchivedItem(value) - } -} - -impl From for ArchivedItem { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::ArchivedItem(obj) => obj, + ObjectInner::Application(obj) => *obj, _ => unreachable!(), } } @@ -6389,14 +5809,14 @@ impl From for ArchivedItem { impl From for ObjectInner { fn from(value: ArfExternalReport) -> Self { - ObjectInner::ArfExternalReport(value) + ObjectInner::ArfExternalReport(Box::new(value)) } } impl From for ArfExternalReport { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::ArfExternalReport(obj) => obj, + ObjectInner::ArfExternalReport(obj) => *obj, _ => unreachable!(), } } @@ -6404,14 +5824,14 @@ impl From for ArfExternalReport { impl From for ObjectInner { fn from(value: Asn) -> Self { - ObjectInner::Asn(value) + ObjectInner::Asn(Box::new(value)) } } impl From for Asn { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Asn(obj) => obj, + ObjectInner::Asn(obj) => *obj, _ => unreachable!(), } } @@ -6419,14 +5839,14 @@ impl From for Asn { impl From for ObjectInner { fn from(value: Authentication) -> Self { - ObjectInner::Authentication(value) + ObjectInner::Authentication(Box::new(value)) } } impl From for Authentication { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Authentication(obj) => obj, + ObjectInner::Authentication(obj) => *obj, _ => unreachable!(), } } @@ -6434,14 +5854,14 @@ impl From for Authentication { impl From for ObjectInner { fn from(value: BlobStore) -> Self { - ObjectInner::BlobStore(value) + ObjectInner::BlobStore(Box::new(value)) } } impl From for BlobStore { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::BlobStore(obj) => obj, + ObjectInner::BlobStore(obj) => *obj, _ => unreachable!(), } } @@ -6449,14 +5869,14 @@ impl From for BlobStore { impl From for ObjectInner { fn from(value: BlockedIp) -> Self { - ObjectInner::BlockedIp(value) + ObjectInner::BlockedIp(Box::new(value)) } } impl From for BlockedIp { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::BlockedIp(obj) => obj, + ObjectInner::BlockedIp(obj) => *obj, _ => unreachable!(), } } @@ -6464,14 +5884,14 @@ impl From for BlockedIp { impl From for ObjectInner { fn from(value: Bootstrap) -> Self { - ObjectInner::Bootstrap(value) + ObjectInner::Bootstrap(Box::new(value)) } } impl From for Bootstrap { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Bootstrap(obj) => obj, + ObjectInner::Bootstrap(obj) => *obj, _ => unreachable!(), } } @@ -6479,14 +5899,14 @@ impl From for Bootstrap { impl From for ObjectInner { fn from(value: Cache) -> Self { - ObjectInner::Cache(value) + ObjectInner::Cache(Box::new(value)) } } impl From for Cache { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Cache(obj) => obj, + ObjectInner::Cache(obj) => *obj, _ => unreachable!(), } } @@ -6494,14 +5914,14 @@ impl From for Cache { impl From for ObjectInner { fn from(value: Calendar) -> Self { - ObjectInner::Calendar(value) + ObjectInner::Calendar(Box::new(value)) } } impl From for Calendar { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Calendar(obj) => obj, + ObjectInner::Calendar(obj) => *obj, _ => unreachable!(), } } @@ -6509,14 +5929,14 @@ impl From for Calendar { impl From for ObjectInner { fn from(value: CalendarAlarm) -> Self { - ObjectInner::CalendarAlarm(value) + ObjectInner::CalendarAlarm(Box::new(value)) } } impl From for CalendarAlarm { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::CalendarAlarm(obj) => obj, + ObjectInner::CalendarAlarm(obj) => *obj, _ => unreachable!(), } } @@ -6524,14 +5944,14 @@ impl From for CalendarAlarm { impl From for ObjectInner { fn from(value: CalendarScheduling) -> Self { - ObjectInner::CalendarScheduling(value) + ObjectInner::CalendarScheduling(Box::new(value)) } } impl From for CalendarScheduling { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::CalendarScheduling(obj) => obj, + ObjectInner::CalendarScheduling(obj) => *obj, _ => unreachable!(), } } @@ -6539,14 +5959,14 @@ impl From for CalendarScheduling { impl From for ObjectInner { fn from(value: Certificate) -> Self { - ObjectInner::Certificate(value) + ObjectInner::Certificate(Box::new(value)) } } impl From for Certificate { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Certificate(obj) => obj, + ObjectInner::Certificate(obj) => *obj, _ => unreachable!(), } } @@ -6554,14 +5974,14 @@ impl From for Certificate { impl From for ObjectInner { fn from(value: ClusterNode) -> Self { - ObjectInner::ClusterNode(value) + ObjectInner::ClusterNode(Box::new(value)) } } impl From for ClusterNode { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::ClusterNode(obj) => obj, + ObjectInner::ClusterNode(obj) => *obj, _ => unreachable!(), } } @@ -6569,14 +5989,14 @@ impl From for ClusterNode { impl From for ObjectInner { fn from(value: ClusterRole) -> Self { - ObjectInner::ClusterRole(value) + ObjectInner::ClusterRole(Box::new(value)) } } impl From for ClusterRole { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::ClusterRole(obj) => obj, + ObjectInner::ClusterRole(obj) => *obj, _ => unreachable!(), } } @@ -6584,14 +6004,14 @@ impl From for ClusterRole { impl From for ObjectInner { fn from(value: Coordinator) -> Self { - ObjectInner::Coordinator(value) + ObjectInner::Coordinator(Box::new(value)) } } impl From for Coordinator { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Coordinator(obj) => obj, + ObjectInner::Coordinator(obj) => *obj, _ => unreachable!(), } } @@ -6599,14 +6019,14 @@ impl From for Coordinator { impl From for ObjectInner { fn from(value: DataRetention) -> Self { - ObjectInner::DataRetention(value) + ObjectInner::DataRetention(Box::new(value)) } } impl From for DataRetention { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DataRetention(obj) => obj, + ObjectInner::DataRetention(obj) => *obj, _ => unreachable!(), } } @@ -6614,14 +6034,14 @@ impl From for DataRetention { impl From for ObjectInner { fn from(value: DataStore) -> Self { - ObjectInner::DataStore(value) + ObjectInner::DataStore(Box::new(value)) } } impl From for DataStore { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DataStore(obj) => obj, + ObjectInner::DataStore(obj) => *obj, _ => unreachable!(), } } @@ -6629,14 +6049,14 @@ impl From for DataStore { impl From for ObjectInner { fn from(value: Directory) -> Self { - ObjectInner::Directory(value) + ObjectInner::Directory(Box::new(value)) } } impl From for Directory { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Directory(obj) => obj, + ObjectInner::Directory(obj) => *obj, _ => unreachable!(), } } @@ -6644,14 +6064,14 @@ impl From for Directory { impl From for ObjectInner { fn from(value: DkimReportSettings) -> Self { - ObjectInner::DkimReportSettings(value) + ObjectInner::DkimReportSettings(Box::new(value)) } } impl From for DkimReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DkimReportSettings(obj) => obj, + ObjectInner::DkimReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -6659,14 +6079,14 @@ impl From for DkimReportSettings { impl From for ObjectInner { fn from(value: DkimSignature) -> Self { - ObjectInner::DkimSignature(value) + ObjectInner::DkimSignature(Box::new(value)) } } impl From for DkimSignature { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DkimSignature(obj) => obj, + ObjectInner::DkimSignature(obj) => *obj, _ => unreachable!(), } } @@ -6674,14 +6094,14 @@ impl From for DkimSignature { impl From for ObjectInner { fn from(value: DmarcExternalReport) -> Self { - ObjectInner::DmarcExternalReport(value) + ObjectInner::DmarcExternalReport(Box::new(value)) } } impl From for DmarcExternalReport { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DmarcExternalReport(obj) => obj, + ObjectInner::DmarcExternalReport(obj) => *obj, _ => unreachable!(), } } @@ -6689,14 +6109,14 @@ impl From for DmarcExternalReport { impl From for ObjectInner { fn from(value: DmarcInternalReport) -> Self { - ObjectInner::DmarcInternalReport(value) + ObjectInner::DmarcInternalReport(Box::new(value)) } } impl From for DmarcInternalReport { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DmarcInternalReport(obj) => obj, + ObjectInner::DmarcInternalReport(obj) => *obj, _ => unreachable!(), } } @@ -6704,14 +6124,14 @@ impl From for DmarcInternalReport { impl From for ObjectInner { fn from(value: DmarcReportSettings) -> Self { - ObjectInner::DmarcReportSettings(value) + ObjectInner::DmarcReportSettings(Box::new(value)) } } impl From for DmarcReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DmarcReportSettings(obj) => obj, + ObjectInner::DmarcReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -6719,14 +6139,14 @@ impl From for DmarcReportSettings { impl From for ObjectInner { fn from(value: DnsResolver) -> Self { - ObjectInner::DnsResolver(value) + ObjectInner::DnsResolver(Box::new(value)) } } impl From for DnsResolver { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DnsResolver(obj) => obj, + ObjectInner::DnsResolver(obj) => *obj, _ => unreachable!(), } } @@ -6734,14 +6154,14 @@ impl From for DnsResolver { impl From for ObjectInner { fn from(value: DnsServer) -> Self { - ObjectInner::DnsServer(value) + ObjectInner::DnsServer(Box::new(value)) } } impl From for DnsServer { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DnsServer(obj) => obj, + ObjectInner::DnsServer(obj) => *obj, _ => unreachable!(), } } @@ -6749,14 +6169,14 @@ impl From for DnsServer { impl From for ObjectInner { fn from(value: Domain) -> Self { - ObjectInner::Domain(value) + ObjectInner::Domain(Box::new(value)) } } impl From for Domain { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Domain(obj) => obj, + ObjectInner::Domain(obj) => *obj, _ => unreachable!(), } } @@ -6764,14 +6184,14 @@ impl From for Domain { impl From for ObjectInner { fn from(value: DsnReportSettings) -> Self { - ObjectInner::DsnReportSettings(value) + ObjectInner::DsnReportSettings(Box::new(value)) } } impl From for DsnReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::DsnReportSettings(obj) => obj, + ObjectInner::DsnReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -6779,29 +6199,14 @@ impl From for DsnReportSettings { impl From for ObjectInner { fn from(value: Email) -> Self { - ObjectInner::Email(value) + ObjectInner::Email(Box::new(value)) } } impl From for Email { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Email(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: Enterprise) -> Self { - ObjectInner::Enterprise(value) - } -} - -impl From for Enterprise { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::Enterprise(obj) => obj, + ObjectInner::Email(obj) => *obj, _ => unreachable!(), } } @@ -6809,14 +6214,14 @@ impl From for Enterprise { impl From for ObjectInner { fn from(value: EventTracingLevel) -> Self { - ObjectInner::EventTracingLevel(value) + ObjectInner::EventTracingLevel(Box::new(value)) } } impl From for EventTracingLevel { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::EventTracingLevel(obj) => obj, + ObjectInner::EventTracingLevel(obj) => *obj, _ => unreachable!(), } } @@ -6824,14 +6229,14 @@ impl From for EventTracingLevel { impl From for ObjectInner { fn from(value: FileStorage) -> Self { - ObjectInner::FileStorage(value) + ObjectInner::FileStorage(Box::new(value)) } } impl From for FileStorage { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::FileStorage(obj) => obj, + ObjectInner::FileStorage(obj) => *obj, _ => unreachable!(), } } @@ -6839,14 +6244,14 @@ impl From for FileStorage { impl From for ObjectInner { fn from(value: Http) -> Self { - ObjectInner::Http(value) + ObjectInner::Http(Box::new(value)) } } impl From for Http { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Http(obj) => obj, + ObjectInner::Http(obj) => *obj, _ => unreachable!(), } } @@ -6854,14 +6259,14 @@ impl From for Http { impl From for ObjectInner { fn from(value: HttpForm) -> Self { - ObjectInner::HttpForm(value) + ObjectInner::HttpForm(Box::new(value)) } } impl From for HttpForm { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::HttpForm(obj) => obj, + ObjectInner::HttpForm(obj) => *obj, _ => unreachable!(), } } @@ -6869,14 +6274,14 @@ impl From for HttpForm { impl From for ObjectInner { fn from(value: HttpLookup) -> Self { - ObjectInner::HttpLookup(value) + ObjectInner::HttpLookup(Box::new(value)) } } impl From for HttpLookup { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::HttpLookup(obj) => obj, + ObjectInner::HttpLookup(obj) => *obj, _ => unreachable!(), } } @@ -6884,14 +6289,14 @@ impl From for HttpLookup { impl From for ObjectInner { fn from(value: Imap) -> Self { - ObjectInner::Imap(value) + ObjectInner::Imap(Box::new(value)) } } impl From for Imap { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Imap(obj) => obj, + ObjectInner::Imap(obj) => *obj, _ => unreachable!(), } } @@ -6899,14 +6304,29 @@ impl From for Imap { impl From for ObjectInner { fn from(value: InMemoryStore) -> Self { - ObjectInner::InMemoryStore(value) + ObjectInner::InMemoryStore(Box::new(value)) } } impl From for InMemoryStore { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::InMemoryStore(obj) => obj, + ObjectInner::InMemoryStore(obj) => *obj, + _ => unreachable!(), + } + } +} + +impl From for ObjectInner { + fn from(value: IndexQueueStatus) -> Self { + ObjectInner::IndexQueueStatus(Box::new(value)) + } +} + +impl From for IndexQueueStatus { + fn from(obj: Object) -> Self { + match obj.inner { + ObjectInner::IndexQueueStatus(obj) => *obj, _ => unreachable!(), } } @@ -6914,14 +6334,14 @@ impl From for InMemoryStore { impl From for ObjectInner { fn from(value: Jmap) -> Self { - ObjectInner::Jmap(value) + ObjectInner::Jmap(Box::new(value)) } } impl From for Jmap { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Jmap(obj) => obj, + ObjectInner::Jmap(obj) => *obj, _ => unreachable!(), } } @@ -6929,14 +6349,14 @@ impl From for Jmap { impl From for ObjectInner { fn from(value: Log) -> Self { - ObjectInner::Log(value) + ObjectInner::Log(Box::new(value)) } } impl From for Log { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Log(obj) => obj, + ObjectInner::Log(obj) => *obj, _ => unreachable!(), } } @@ -6944,29 +6364,14 @@ impl From for Log { impl From for ObjectInner { fn from(value: MailingList) -> Self { - ObjectInner::MailingList(value) + ObjectInner::MailingList(Box::new(value)) } } impl From for MailingList { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MailingList(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: MaskedEmail) -> Self { - ObjectInner::MaskedEmail(value) - } -} - -impl From for MaskedEmail { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::MaskedEmail(obj) => obj, + ObjectInner::MailingList(obj) => *obj, _ => unreachable!(), } } @@ -6974,14 +6379,14 @@ impl From for MaskedEmail { impl From for ObjectInner { fn from(value: MemoryLookupKey) -> Self { - ObjectInner::MemoryLookupKey(value) + ObjectInner::MemoryLookupKey(Box::new(value)) } } impl From for MemoryLookupKey { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MemoryLookupKey(obj) => obj, + ObjectInner::MemoryLookupKey(obj) => *obj, _ => unreachable!(), } } @@ -6989,29 +6394,29 @@ impl From for MemoryLookupKey { impl From for ObjectInner { fn from(value: MemoryLookupKeyValue) -> Self { - ObjectInner::MemoryLookupKeyValue(value) + ObjectInner::MemoryLookupKeyValue(Box::new(value)) } } impl From for MemoryLookupKeyValue { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MemoryLookupKeyValue(obj) => obj, + ObjectInner::MemoryLookupKeyValue(obj) => *obj, _ => unreachable!(), } } } -impl From for ObjectInner { - fn from(value: Metric) -> Self { - ObjectInner::Metric(value) +impl From for ObjectInner { + fn from(value: Metadata) -> Self { + ObjectInner::Metadata(Box::new(value)) } } -impl From for Metric { +impl From for Metadata { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Metric(obj) => obj, + ObjectInner::Metadata(obj) => *obj, _ => unreachable!(), } } @@ -7019,29 +6424,14 @@ impl From for Metric { impl From for ObjectInner { fn from(value: Metrics) -> Self { - ObjectInner::Metrics(value) + ObjectInner::Metrics(Box::new(value)) } } impl From for Metrics { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Metrics(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: MetricsStore) -> Self { - ObjectInner::MetricsStore(value) - } -} - -impl From for MetricsStore { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::MetricsStore(obj) => obj, + ObjectInner::Metrics(obj) => *obj, _ => unreachable!(), } } @@ -7049,14 +6439,14 @@ impl From for MetricsStore { impl From for ObjectInner { fn from(value: MtaConnectionStrategy) -> Self { - ObjectInner::MtaConnectionStrategy(value) + ObjectInner::MtaConnectionStrategy(Box::new(value)) } } impl From for MtaConnectionStrategy { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaConnectionStrategy(obj) => obj, + ObjectInner::MtaConnectionStrategy(obj) => *obj, _ => unreachable!(), } } @@ -7064,14 +6454,14 @@ impl From for MtaConnectionStrategy { impl From for ObjectInner { fn from(value: MtaDeliverySchedule) -> Self { - ObjectInner::MtaDeliverySchedule(value) + ObjectInner::MtaDeliverySchedule(Box::new(value)) } } impl From for MtaDeliverySchedule { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaDeliverySchedule(obj) => obj, + ObjectInner::MtaDeliverySchedule(obj) => *obj, _ => unreachable!(), } } @@ -7079,14 +6469,14 @@ impl From for MtaDeliverySchedule { impl From for ObjectInner { fn from(value: MtaExtensions) -> Self { - ObjectInner::MtaExtensions(value) + ObjectInner::MtaExtensions(Box::new(value)) } } impl From for MtaExtensions { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaExtensions(obj) => obj, + ObjectInner::MtaExtensions(obj) => *obj, _ => unreachable!(), } } @@ -7094,14 +6484,14 @@ impl From for MtaExtensions { impl From for ObjectInner { fn from(value: MtaHook) -> Self { - ObjectInner::MtaHook(value) + ObjectInner::MtaHook(Box::new(value)) } } impl From for MtaHook { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaHook(obj) => obj, + ObjectInner::MtaHook(obj) => *obj, _ => unreachable!(), } } @@ -7109,14 +6499,14 @@ impl From for MtaHook { impl From for ObjectInner { fn from(value: MtaInboundSession) -> Self { - ObjectInner::MtaInboundSession(value) + ObjectInner::MtaInboundSession(Box::new(value)) } } impl From for MtaInboundSession { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaInboundSession(obj) => obj, + ObjectInner::MtaInboundSession(obj) => *obj, _ => unreachable!(), } } @@ -7124,14 +6514,14 @@ impl From for MtaInboundSession { impl From for ObjectInner { fn from(value: MtaInboundThrottle) -> Self { - ObjectInner::MtaInboundThrottle(value) + ObjectInner::MtaInboundThrottle(Box::new(value)) } } impl From for MtaInboundThrottle { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaInboundThrottle(obj) => obj, + ObjectInner::MtaInboundThrottle(obj) => *obj, _ => unreachable!(), } } @@ -7139,14 +6529,14 @@ impl From for MtaInboundThrottle { impl From for ObjectInner { fn from(value: MtaMilter) -> Self { - ObjectInner::MtaMilter(value) + ObjectInner::MtaMilter(Box::new(value)) } } impl From for MtaMilter { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaMilter(obj) => obj, + ObjectInner::MtaMilter(obj) => *obj, _ => unreachable!(), } } @@ -7154,14 +6544,14 @@ impl From for MtaMilter { impl From for ObjectInner { fn from(value: MtaOutboundStrategy) -> Self { - ObjectInner::MtaOutboundStrategy(value) + ObjectInner::MtaOutboundStrategy(Box::new(value)) } } impl From for MtaOutboundStrategy { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaOutboundStrategy(obj) => obj, + ObjectInner::MtaOutboundStrategy(obj) => *obj, _ => unreachable!(), } } @@ -7169,14 +6559,14 @@ impl From for MtaOutboundStrategy { impl From for ObjectInner { fn from(value: MtaOutboundThrottle) -> Self { - ObjectInner::MtaOutboundThrottle(value) + ObjectInner::MtaOutboundThrottle(Box::new(value)) } } impl From for MtaOutboundThrottle { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaOutboundThrottle(obj) => obj, + ObjectInner::MtaOutboundThrottle(obj) => *obj, _ => unreachable!(), } } @@ -7184,14 +6574,14 @@ impl From for MtaOutboundThrottle { impl From for ObjectInner { fn from(value: MtaQueueQuota) -> Self { - ObjectInner::MtaQueueQuota(value) + ObjectInner::MtaQueueQuota(Box::new(value)) } } impl From for MtaQueueQuota { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaQueueQuota(obj) => obj, + ObjectInner::MtaQueueQuota(obj) => *obj, _ => unreachable!(), } } @@ -7199,14 +6589,14 @@ impl From for MtaQueueQuota { impl From for ObjectInner { fn from(value: MtaRoute) -> Self { - ObjectInner::MtaRoute(value) + ObjectInner::MtaRoute(Box::new(value)) } } impl From for MtaRoute { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaRoute(obj) => obj, + ObjectInner::MtaRoute(obj) => *obj, _ => unreachable!(), } } @@ -7214,14 +6604,14 @@ impl From for MtaRoute { impl From for ObjectInner { fn from(value: MtaStageAuth) -> Self { - ObjectInner::MtaStageAuth(value) + ObjectInner::MtaStageAuth(Box::new(value)) } } impl From for MtaStageAuth { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageAuth(obj) => obj, + ObjectInner::MtaStageAuth(obj) => *obj, _ => unreachable!(), } } @@ -7229,14 +6619,14 @@ impl From for MtaStageAuth { impl From for ObjectInner { fn from(value: MtaStageConnect) -> Self { - ObjectInner::MtaStageConnect(value) + ObjectInner::MtaStageConnect(Box::new(value)) } } impl From for MtaStageConnect { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageConnect(obj) => obj, + ObjectInner::MtaStageConnect(obj) => *obj, _ => unreachable!(), } } @@ -7244,14 +6634,14 @@ impl From for MtaStageConnect { impl From for ObjectInner { fn from(value: MtaStageData) -> Self { - ObjectInner::MtaStageData(value) + ObjectInner::MtaStageData(Box::new(value)) } } impl From for MtaStageData { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageData(obj) => obj, + ObjectInner::MtaStageData(obj) => *obj, _ => unreachable!(), } } @@ -7259,14 +6649,14 @@ impl From for MtaStageData { impl From for ObjectInner { fn from(value: MtaStageEhlo) -> Self { - ObjectInner::MtaStageEhlo(value) + ObjectInner::MtaStageEhlo(Box::new(value)) } } impl From for MtaStageEhlo { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageEhlo(obj) => obj, + ObjectInner::MtaStageEhlo(obj) => *obj, _ => unreachable!(), } } @@ -7274,14 +6664,14 @@ impl From for MtaStageEhlo { impl From for ObjectInner { fn from(value: MtaStageMail) -> Self { - ObjectInner::MtaStageMail(value) + ObjectInner::MtaStageMail(Box::new(value)) } } impl From for MtaStageMail { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageMail(obj) => obj, + ObjectInner::MtaStageMail(obj) => *obj, _ => unreachable!(), } } @@ -7289,14 +6679,14 @@ impl From for MtaStageMail { impl From for ObjectInner { fn from(value: MtaStageRcpt) -> Self { - ObjectInner::MtaStageRcpt(value) + ObjectInner::MtaStageRcpt(Box::new(value)) } } impl From for MtaStageRcpt { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaStageRcpt(obj) => obj, + ObjectInner::MtaStageRcpt(obj) => *obj, _ => unreachable!(), } } @@ -7304,14 +6694,14 @@ impl From for MtaStageRcpt { impl From for ObjectInner { fn from(value: MtaSts) -> Self { - ObjectInner::MtaSts(value) + ObjectInner::MtaSts(Box::new(value)) } } impl From for MtaSts { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaSts(obj) => obj, + ObjectInner::MtaSts(obj) => *obj, _ => unreachable!(), } } @@ -7319,14 +6709,14 @@ impl From for MtaSts { impl From for ObjectInner { fn from(value: MtaTlsStrategy) -> Self { - ObjectInner::MtaTlsStrategy(value) + ObjectInner::MtaTlsStrategy(Box::new(value)) } } impl From for MtaTlsStrategy { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaTlsStrategy(obj) => obj, + ObjectInner::MtaTlsStrategy(obj) => *obj, _ => unreachable!(), } } @@ -7334,14 +6724,14 @@ impl From for MtaTlsStrategy { impl From for ObjectInner { fn from(value: MtaVirtualQueue) -> Self { - ObjectInner::MtaVirtualQueue(value) + ObjectInner::MtaVirtualQueue(Box::new(value)) } } impl From for MtaVirtualQueue { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::MtaVirtualQueue(obj) => obj, + ObjectInner::MtaVirtualQueue(obj) => *obj, _ => unreachable!(), } } @@ -7349,14 +6739,14 @@ impl From for MtaVirtualQueue { impl From for ObjectInner { fn from(value: NetworkListener) -> Self { - ObjectInner::NetworkListener(value) + ObjectInner::NetworkListener(Box::new(value)) } } impl From for NetworkListener { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::NetworkListener(obj) => obj, + ObjectInner::NetworkListener(obj) => *obj, _ => unreachable!(), } } @@ -7364,14 +6754,29 @@ impl From for NetworkListener { impl From for ObjectInner { fn from(value: OAuthClient) -> Self { - ObjectInner::OAuthClient(value) + ObjectInner::OAuthClient(Box::new(value)) } } impl From for OAuthClient { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::OAuthClient(obj) => obj, + ObjectInner::OAuthClient(obj) => *obj, + _ => unreachable!(), + } + } +} + +impl From for ObjectInner { + fn from(value: OAuthGrant) -> Self { + ObjectInner::OAuthGrant(Box::new(value)) + } +} + +impl From for OAuthGrant { + fn from(obj: Object) -> Self { + match obj.inner { + ObjectInner::OAuthGrant(obj) => *obj, _ => unreachable!(), } } @@ -7379,14 +6784,14 @@ impl From for OAuthClient { impl From for ObjectInner { fn from(value: OidcProvider) -> Self { - ObjectInner::OidcProvider(value) + ObjectInner::OidcProvider(Box::new(value)) } } impl From for OidcProvider { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::OidcProvider(obj) => obj, + ObjectInner::OidcProvider(obj) => *obj, _ => unreachable!(), } } @@ -7394,14 +6799,14 @@ impl From for OidcProvider { impl From for ObjectInner { fn from(value: PublicKey) -> Self { - ObjectInner::PublicKey(value) + ObjectInner::PublicKey(Box::new(value)) } } impl From for PublicKey { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::PublicKey(obj) => obj, + ObjectInner::PublicKey(obj) => *obj, _ => unreachable!(), } } @@ -7409,14 +6814,14 @@ impl From for PublicKey { impl From for ObjectInner { fn from(value: QueuedMessage) -> Self { - ObjectInner::QueuedMessage(value) + ObjectInner::QueuedMessage(Box::new(value)) } } impl From for QueuedMessage { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::QueuedMessage(obj) => obj, + ObjectInner::QueuedMessage(obj) => *obj, _ => unreachable!(), } } @@ -7424,14 +6829,14 @@ impl From for QueuedMessage { impl From for ObjectInner { fn from(value: ReportSettings) -> Self { - ObjectInner::ReportSettings(value) + ObjectInner::ReportSettings(Box::new(value)) } } impl From for ReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::ReportSettings(obj) => obj, + ObjectInner::ReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -7439,14 +6844,14 @@ impl From for ReportSettings { impl From for ObjectInner { fn from(value: Role) -> Self { - ObjectInner::Role(value) + ObjectInner::Role(Box::new(value)) } } impl From for Role { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Role(obj) => obj, + ObjectInner::Role(obj) => *obj, _ => unreachable!(), } } @@ -7454,14 +6859,14 @@ impl From for Role { impl From for ObjectInner { fn from(value: Search) -> Self { - ObjectInner::Search(value) + ObjectInner::Search(Box::new(value)) } } impl From for Search { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Search(obj) => obj, + ObjectInner::Search(obj) => *obj, _ => unreachable!(), } } @@ -7469,14 +6874,14 @@ impl From for Search { impl From for ObjectInner { fn from(value: SearchStore) -> Self { - ObjectInner::SearchStore(value) + ObjectInner::SearchStore(Box::new(value)) } } impl From for SearchStore { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SearchStore(obj) => obj, + ObjectInner::SearchStore(obj) => *obj, _ => unreachable!(), } } @@ -7484,14 +6889,14 @@ impl From for SearchStore { impl From for ObjectInner { fn from(value: Security) -> Self { - ObjectInner::Security(value) + ObjectInner::Security(Box::new(value)) } } impl From for Security { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Security(obj) => obj, + ObjectInner::Security(obj) => *obj, _ => unreachable!(), } } @@ -7499,14 +6904,14 @@ impl From for Security { impl From for ObjectInner { fn from(value: SenderAuth) -> Self { - ObjectInner::SenderAuth(value) + ObjectInner::SenderAuth(Box::new(value)) } } impl From for SenderAuth { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SenderAuth(obj) => obj, + ObjectInner::SenderAuth(obj) => *obj, _ => unreachable!(), } } @@ -7514,14 +6919,14 @@ impl From for SenderAuth { impl From for ObjectInner { fn from(value: Sharing) -> Self { - ObjectInner::Sharing(value) + ObjectInner::Sharing(Box::new(value)) } } impl From for Sharing { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Sharing(obj) => obj, + ObjectInner::Sharing(obj) => *obj, _ => unreachable!(), } } @@ -7529,14 +6934,14 @@ impl From for Sharing { impl From for ObjectInner { fn from(value: SieveSystemInterpreter) -> Self { - ObjectInner::SieveSystemInterpreter(value) + ObjectInner::SieveSystemInterpreter(Box::new(value)) } } impl From for SieveSystemInterpreter { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SieveSystemInterpreter(obj) => obj, + ObjectInner::SieveSystemInterpreter(obj) => *obj, _ => unreachable!(), } } @@ -7544,14 +6949,14 @@ impl From for SieveSystemInterpreter { impl From for ObjectInner { fn from(value: SieveSystemScript) -> Self { - ObjectInner::SieveSystemScript(value) + ObjectInner::SieveSystemScript(Box::new(value)) } } impl From for SieveSystemScript { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SieveSystemScript(obj) => obj, + ObjectInner::SieveSystemScript(obj) => *obj, _ => unreachable!(), } } @@ -7559,14 +6964,14 @@ impl From for SieveSystemScript { impl From for ObjectInner { fn from(value: SieveUserInterpreter) -> Self { - ObjectInner::SieveUserInterpreter(value) + ObjectInner::SieveUserInterpreter(Box::new(value)) } } impl From for SieveUserInterpreter { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SieveUserInterpreter(obj) => obj, + ObjectInner::SieveUserInterpreter(obj) => *obj, _ => unreachable!(), } } @@ -7574,14 +6979,14 @@ impl From for SieveUserInterpreter { impl From for ObjectInner { fn from(value: SieveUserScript) -> Self { - ObjectInner::SieveUserScript(value) + ObjectInner::SieveUserScript(Box::new(value)) } } impl From for SieveUserScript { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SieveUserScript(obj) => obj, + ObjectInner::SieveUserScript(obj) => *obj, _ => unreachable!(), } } @@ -7589,14 +6994,14 @@ impl From for SieveUserScript { impl From for ObjectInner { fn from(value: SpamClassifier) -> Self { - ObjectInner::SpamClassifier(value) + ObjectInner::SpamClassifier(Box::new(value)) } } impl From for SpamClassifier { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamClassifier(obj) => obj, + ObjectInner::SpamClassifier(obj) => *obj, _ => unreachable!(), } } @@ -7604,14 +7009,14 @@ impl From for SpamClassifier { impl From for ObjectInner { fn from(value: SpamDnsblServer) -> Self { - ObjectInner::SpamDnsblServer(value) + ObjectInner::SpamDnsblServer(Box::new(value)) } } impl From for SpamDnsblServer { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamDnsblServer(obj) => obj, + ObjectInner::SpamDnsblServer(obj) => *obj, _ => unreachable!(), } } @@ -7619,14 +7024,14 @@ impl From for SpamDnsblServer { impl From for ObjectInner { fn from(value: SpamDnsblSettings) -> Self { - ObjectInner::SpamDnsblSettings(value) + ObjectInner::SpamDnsblSettings(Box::new(value)) } } impl From for SpamDnsblSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamDnsblSettings(obj) => obj, + ObjectInner::SpamDnsblSettings(obj) => *obj, _ => unreachable!(), } } @@ -7634,29 +7039,14 @@ impl From for SpamDnsblSettings { impl From for ObjectInner { fn from(value: SpamFileExtension) -> Self { - ObjectInner::SpamFileExtension(value) + ObjectInner::SpamFileExtension(Box::new(value)) } } impl From for SpamFileExtension { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamFileExtension(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: SpamLlm) -> Self { - ObjectInner::SpamLlm(value) - } -} - -impl From for SpamLlm { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::SpamLlm(obj) => obj, + ObjectInner::SpamFileExtension(obj) => *obj, _ => unreachable!(), } } @@ -7664,14 +7054,14 @@ impl From for SpamLlm { impl From for ObjectInner { fn from(value: SpamPyzor) -> Self { - ObjectInner::SpamPyzor(value) + ObjectInner::SpamPyzor(Box::new(value)) } } impl From for SpamPyzor { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamPyzor(obj) => obj, + ObjectInner::SpamPyzor(obj) => *obj, _ => unreachable!(), } } @@ -7679,14 +7069,14 @@ impl From for SpamPyzor { impl From for ObjectInner { fn from(value: SpamRule) -> Self { - ObjectInner::SpamRule(value) + ObjectInner::SpamRule(Box::new(value)) } } impl From for SpamRule { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamRule(obj) => obj, + ObjectInner::SpamRule(obj) => *obj, _ => unreachable!(), } } @@ -7694,14 +7084,14 @@ impl From for SpamRule { impl From for ObjectInner { fn from(value: SpamSettings) -> Self { - ObjectInner::SpamSettings(value) + ObjectInner::SpamSettings(Box::new(value)) } } impl From for SpamSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamSettings(obj) => obj, + ObjectInner::SpamSettings(obj) => *obj, _ => unreachable!(), } } @@ -7709,14 +7099,14 @@ impl From for SpamSettings { impl From for ObjectInner { fn from(value: SpamTag) -> Self { - ObjectInner::SpamTag(value) + ObjectInner::SpamTag(Box::new(value)) } } impl From for SpamTag { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamTag(obj) => obj, + ObjectInner::SpamTag(obj) => *obj, _ => unreachable!(), } } @@ -7724,14 +7114,14 @@ impl From for SpamTag { impl From for ObjectInner { fn from(value: SpamTrainingSample) -> Self { - ObjectInner::SpamTrainingSample(value) + ObjectInner::SpamTrainingSample(Box::new(value)) } } impl From for SpamTrainingSample { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpamTrainingSample(obj) => obj, + ObjectInner::SpamTrainingSample(obj) => *obj, _ => unreachable!(), } } @@ -7739,14 +7129,14 @@ impl From for SpamTrainingSample { impl From for ObjectInner { fn from(value: SpfReportSettings) -> Self { - ObjectInner::SpfReportSettings(value) + ObjectInner::SpfReportSettings(Box::new(value)) } } impl From for SpfReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SpfReportSettings(obj) => obj, + ObjectInner::SpfReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -7754,14 +7144,14 @@ impl From for SpfReportSettings { impl From for ObjectInner { fn from(value: StoreLookup) -> Self { - ObjectInner::StoreLookup(value) + ObjectInner::StoreLookup(Box::new(value)) } } impl From for StoreLookup { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::StoreLookup(obj) => obj, + ObjectInner::StoreLookup(obj) => *obj, _ => unreachable!(), } } @@ -7769,14 +7159,14 @@ impl From for StoreLookup { impl From for ObjectInner { fn from(value: SystemSettings) -> Self { - ObjectInner::SystemSettings(value) + ObjectInner::SystemSettings(Box::new(value)) } } impl From for SystemSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::SystemSettings(obj) => obj, + ObjectInner::SystemSettings(obj) => *obj, _ => unreachable!(), } } @@ -7784,14 +7174,14 @@ impl From for SystemSettings { impl From for ObjectInner { fn from(value: Task) -> Self { - ObjectInner::Task(value) + ObjectInner::Task(Box::new(value)) } } impl From for Task { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Task(obj) => obj, + ObjectInner::Task(obj) => *obj, _ => unreachable!(), } } @@ -7799,29 +7189,14 @@ impl From for Task { impl From for ObjectInner { fn from(value: TaskManager) -> Self { - ObjectInner::TaskManager(value) + ObjectInner::TaskManager(Box::new(value)) } } impl From for TaskManager { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::TaskManager(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: Tenant) -> Self { - ObjectInner::Tenant(value) - } -} - -impl From for Tenant { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::Tenant(obj) => obj, + ObjectInner::TaskManager(obj) => *obj, _ => unreachable!(), } } @@ -7829,14 +7204,14 @@ impl From for Tenant { impl From for ObjectInner { fn from(value: TlsExternalReport) -> Self { - ObjectInner::TlsExternalReport(value) + ObjectInner::TlsExternalReport(Box::new(value)) } } impl From for TlsExternalReport { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::TlsExternalReport(obj) => obj, + ObjectInner::TlsExternalReport(obj) => *obj, _ => unreachable!(), } } @@ -7844,14 +7219,14 @@ impl From for TlsExternalReport { impl From for ObjectInner { fn from(value: TlsInternalReport) -> Self { - ObjectInner::TlsInternalReport(value) + ObjectInner::TlsInternalReport(Box::new(value)) } } impl From for TlsInternalReport { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::TlsInternalReport(obj) => obj, + ObjectInner::TlsInternalReport(obj) => *obj, _ => unreachable!(), } } @@ -7859,29 +7234,14 @@ impl From for TlsInternalReport { impl From for ObjectInner { fn from(value: TlsReportSettings) -> Self { - ObjectInner::TlsReportSettings(value) + ObjectInner::TlsReportSettings(Box::new(value)) } } impl From for TlsReportSettings { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::TlsReportSettings(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: Trace) -> Self { - ObjectInner::Trace(value) - } -} - -impl From for Trace { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::Trace(obj) => obj, + ObjectInner::TlsReportSettings(obj) => *obj, _ => unreachable!(), } } @@ -7889,29 +7249,14 @@ impl From for Trace { impl From for ObjectInner { fn from(value: Tracer) -> Self { - ObjectInner::Tracer(value) + ObjectInner::Tracer(Box::new(value)) } } impl From for Tracer { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::Tracer(obj) => obj, - _ => unreachable!(), - } - } -} - -impl From for ObjectInner { - fn from(value: TracingStore) -> Self { - ObjectInner::TracingStore(value) - } -} - -impl From for TracingStore { - fn from(obj: Object) -> Self { - match obj.inner { - ObjectInner::TracingStore(obj) => obj, + ObjectInner::Tracer(obj) => *obj, _ => unreachable!(), } } @@ -7919,14 +7264,14 @@ impl From for TracingStore { impl From for ObjectInner { fn from(value: WebDav) -> Self { - ObjectInner::WebDav(value) + ObjectInner::WebDav(Box::new(value)) } } impl From for WebDav { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::WebDav(obj) => obj, + ObjectInner::WebDav(obj) => *obj, _ => unreachable!(), } } @@ -7934,14 +7279,14 @@ impl From for WebDav { impl From for ObjectInner { fn from(value: WebHook) -> Self { - ObjectInner::WebHook(value) + ObjectInner::WebHook(Box::new(value)) } } impl From for WebHook { fn from(obj: Object) -> Self { match obj.inner { - ObjectInner::WebHook(obj) => obj, + ObjectInner::WebHook(obj) => *obj, _ => unreachable!(), } } diff --git a/crates/registry/src/schema/structs.rs b/crates/registry/src/schema/structs.rs index 7066a572..24e9bc25 100644 --- a/crates/registry/src/schema/structs.rs +++ b/crates/registry/src/schema/structs.rs @@ -17,7 +17,6 @@ pub enum Account { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AccountPassword { - #[serde(rename = "secret")] pub secret: Option, #[serde(rename = "currentSecret")] pub current_secret: Option, @@ -28,9 +27,7 @@ pub struct AccountPassword { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AccountSettings { - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "locale")] pub locale: Locale, #[serde(rename = "timeZone")] pub time_zone: Option, @@ -43,30 +40,31 @@ pub struct AccountSettings { pub struct AcmeProvider { #[serde(rename = "challengeType")] pub challenge_type: AcmeChallengeType, - #[serde(rename = "contact")] pub contact: Map, - #[serde(rename = "directory")] pub directory: String, #[serde(rename = "accountKey")] pub account_key: String, #[serde(rename = "accountUri")] pub account_uri: String, + #[serde(rename = "accountHashPrefix")] + pub account_hash_prefix: Option, + #[serde(rename = "issuerDomainNames")] + pub issuer_domain_names: Map, + #[serde(rename = "caaIdentities")] + pub caa_identities: Map, #[serde(rename = "renewBefore")] pub renew_before: AcmeRenewBefore, - #[serde(rename = "maxRetries")] - pub max_retries: i64, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, #[serde(rename = "preferredChain")] pub preferred_chain: Option, #[serde(rename = "reuseKey")] pub reuse_key: bool, + pub description: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "@type")] pub enum Action { - ReloadSettings, + ReloadSettings(ReloadSettingsResult), ReloadTlsCertificates, ReloadLookupStores, ReloadBlockedIps, @@ -94,86 +92,16 @@ pub struct AddressBook { pub max_contacts: Option, #[serde(rename = "vCardVersion")] pub v_card_version: VCardVersion, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct AiModel { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "allowInvalidCerts")] - pub allow_invalid_certs: bool, - #[serde(rename = "temperature")] - pub temperature: Float, - #[serde(rename = "model")] - pub model: String, - #[serde(rename = "timeout")] - pub timeout: Duration, - #[serde(rename = "modelType")] - pub model_type: AiModelType, - #[serde(rename = "url")] - pub url: String, - #[serde(rename = "httpAuth")] - pub http_auth: HttpAuth, - #[serde(rename = "httpHeaders")] - pub http_headers: VecMap, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct Alert { - #[serde(rename = "condition")] - pub condition: Expression, - #[serde(rename = "emailAlert")] - pub email_alert: AlertEmail, - #[serde(rename = "eventAlert")] - pub event_alert: AlertEvent, - #[serde(rename = "enable")] - pub enable: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum AlertEmail { - Disabled, - Enabled(AlertEmailProperties), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct AlertEmailProperties { - #[serde(rename = "body")] - pub body: String, - #[serde(rename = "fromAddress")] - pub from_address: String, - #[serde(rename = "fromName")] - pub from_name: Option, - #[serde(rename = "subject")] - pub subject: String, - #[serde(rename = "to")] - pub to: Map, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum AlertEvent { - Disabled, - Enabled(AlertEventProperties), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct AlertEventProperties { - #[serde(rename = "eventMessage")] - pub event_message: Option, + #[serde(rename = "maxAddressBooksPerCard")] + pub max_address_books_per_card: u64, + #[serde(rename = "maxMediaSize")] + pub max_media_size: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AllowedIp { - #[serde(rename = "address")] pub address: IpAddrOrMask, - #[serde(rename = "reason")] pub reason: Option, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, @@ -184,15 +112,12 @@ pub struct AllowedIp { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ApiKey { - #[serde(rename = "description")] pub description: String, - #[serde(rename = "secret")] pub secret: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: Option, - #[serde(rename = "permissions")] pub permissions: CredentialPermissions, #[serde(rename = "allowedIps")] pub allowed_ips: Map, @@ -201,15 +126,12 @@ pub struct ApiKey { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AppPassword { - #[serde(rename = "description")] pub description: String, - #[serde(rename = "secret")] pub secret: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: Option, - #[serde(rename = "permissions")] pub permissions: CredentialPermissions, #[serde(rename = "allowedIps")] pub allowed_ips: Map, @@ -218,142 +140,31 @@ pub struct AppPassword { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Application { - #[serde(rename = "enabled")] - pub enabled: bool, - #[serde(rename = "description")] + pub enable: bool, pub description: String, #[serde(rename = "resourceUrl")] pub resource_url: String, #[serde(rename = "urlPrefix")] pub url_prefix: Map, - #[serde(rename = "autoUpdateFrequency")] - pub auto_update_frequency: Duration, + #[serde(rename = "autoUpdateInterval")] + pub auto_update_interval: Duration, #[serde(rename = "unpackDirectory")] pub unpack_directory: Option, #[serde(rename = "oauthClientId")] pub oauth_client_id: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ArchivedCalendarEvent { - #[serde(rename = "title")] - pub title: String, - #[serde(rename = "startTime")] - pub start_time: Option, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "archivedAt")] - pub archived_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "blobId")] - pub blob_id: BlobId, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ArchivedContactCard { - #[serde(rename = "name")] - pub name: Option, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "archivedAt")] - pub archived_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "blobId")] - pub blob_id: BlobId, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ArchivedEmail { - #[serde(rename = "from")] - pub from: String, - #[serde(rename = "subject")] - pub subject: String, - #[serde(rename = "receivedAt")] - pub received_at: UTCDateTime, - #[serde(rename = "size")] - pub size: u64, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "archivedAt")] - pub archived_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "blobId")] - pub blob_id: BlobId, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ArchivedFileNode { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "archivedAt")] - pub archived_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "blobId")] - pub blob_id: BlobId, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum ArchivedItem { - Email(ArchivedEmail), - FileNode(ArchivedFileNode), - CalendarEvent(ArchivedCalendarEvent), - ContactCard(ArchivedContactCard), - SieveScript(ArchivedSieveScript), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ArchivedSieveScript { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "content")] - pub content: String, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "archivedAt")] - pub archived_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "blobId")] - pub blob_id: BlobId, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ArfExternalReport { - #[serde(rename = "report")] pub report: ArfFeedbackReport, - #[serde(rename = "from")] pub from: String, - #[serde(rename = "subject")] pub subject: String, - #[serde(rename = "to")] pub to: Map, #[serde(rename = "receivedAt")] pub received_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: UTCDateTime, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -365,7 +176,6 @@ pub struct ArfFeedbackReport { pub arrival_date: Option, #[serde(rename = "authenticationResults")] pub authentication_results: Map, - #[serde(rename = "incidents")] pub incidents: u64, #[serde(rename = "originalEnvelopeId")] pub original_envelope_id: Option, @@ -385,7 +195,6 @@ pub struct ArfFeedbackReport { pub source_port: Option, #[serde(rename = "userAgent")] pub user_agent: Option, - #[serde(rename = "version")] pub version: u64, #[serde(rename = "authFailure")] pub auth_failure: ArfAuthFailureType, @@ -409,9 +218,7 @@ pub struct ArfFeedbackReport { pub spf_dns: Option, #[serde(rename = "identityAlignment")] pub identity_alignment: ArfIdentityAlignment, - #[serde(rename = "message")] pub message: Option, - #[serde(rename = "headers")] pub headers: Option, } @@ -432,7 +239,6 @@ pub struct AsnDns { pub index_asn_name: Option, #[serde(rename = "indexCountry")] pub index_country: Option, - #[serde(rename = "separator")] pub separator: String, #[serde(rename = "zoneIpV4")] pub zone_ip_v4: String, @@ -443,16 +249,17 @@ pub struct AsnDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AsnResource { - #[serde(rename = "expires")] - pub expires: Duration, + #[serde(rename = "refreshInterval")] + pub refresh_interval: Duration, #[serde(rename = "maxSize")] pub max_size: u64, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "asnUrls")] pub asn_urls: Map, #[serde(rename = "geoUrls")] pub geo_urls: Map, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] @@ -468,18 +275,16 @@ pub struct Authentication { pub default_user_role_ids: Map, #[serde(rename = "defaultGroupRoleIds")] pub default_group_role_ids: Map, - #[serde(rename = "defaultTenantRoleIds")] - pub default_tenant_role_ids: Map, #[serde(rename = "defaultAdminRoleIds")] pub default_admin_role_ids: Map, #[serde(rename = "passwordHashAlgorithm")] pub password_hash_algorithm: PasswordHashAlgorithm, - #[serde(rename = "passwordMinLength")] - pub password_min_length: u64, - #[serde(rename = "passwordMaxLength")] - pub password_max_length: u64, - #[serde(rename = "passwordMinStrength")] - pub password_min_strength: PasswordStrength, + #[serde(rename = "minPasswordLength")] + pub min_password_length: u64, + #[serde(rename = "maxPasswordLength")] + pub max_password_length: u64, + #[serde(rename = "minPasswordStrength")] + pub min_password_strength: PasswordStrength, #[serde(rename = "passwordDefaultExpiry")] pub password_default_expiry: Option, #[serde(rename = "maxAppPasswords")] @@ -488,18 +293,68 @@ pub struct Authentication { pub max_api_keys: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "@type")] +pub enum AzureAuth { + SharedKey(AzureAuthSharedKey), + SasToken(AzureAuthSasToken), + ClientSecret(AzureAuthClientSecret), + ManagedIdentity(AzureAuthManagedIdentity), + WorkloadIdentity(AzureAuthWorkloadIdentity), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AzureAuthClientSecret { + #[serde(rename = "tenantId")] + pub tenant_id: String, + #[serde(rename = "clientId")] + pub client_id: String, + #[serde(rename = "clientSecret")] + pub client_secret: SecretKey, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AzureAuthManagedIdentity { + #[serde(rename = "clientId")] + pub client_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AzureAuthSasToken { + #[serde(rename = "sasToken")] + pub sas_token: SecretKey, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AzureAuthSharedKey { + #[serde(rename = "accountKey")] + pub account_key: SecretKey, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AzureAuthWorkloadIdentity { + #[serde(rename = "tenantId")] + pub tenant_id: Option, + #[serde(rename = "clientId")] + pub client_id: Option, + #[serde(rename = "tokenFile")] + pub token_file: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct AzureStore { #[serde(rename = "storageAccount")] pub storage_account: String, - #[serde(rename = "container")] pub container: String, - #[serde(rename = "accessKey")] - pub access_key: SecretKeyOptional, - #[serde(rename = "sasToken")] - pub sas_token: SecretKeyOptional, - #[serde(rename = "timeout")] + pub environment: AzureEnvironment, + pub url: Option, + pub auth: AzureAuth, pub timeout: Duration, #[serde(rename = "maxRetries")] pub max_retries: u64, @@ -511,7 +366,6 @@ pub struct AzureStore { #[serde(tag = "@type")] pub enum BlobStore { Default, - Sharded(ShardedBlobStore), S3(S3Store), Azure(AzureStore), FileSystem(FileSystemStore), @@ -520,9 +374,25 @@ pub enum BlobStore { MySql(MySqlStore), } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct BlobStoreSwap { + pub store: BlobSwapStore, + pub retention: Duration, + #[serde(rename = "flushChanges")] + pub flush_changes: u64, + #[serde(rename = "flushInterval")] + pub flush_interval: Duration, + #[serde(rename = "maxAccountSize")] + pub max_account_size: u64, + #[serde(rename = "minAccountSize")] + pub min_account_size: u64, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "@type")] -pub enum BlobStoreBase { +pub enum BlobSwapStore { + Default, S3(S3Store), Azure(AzureStore), FileSystem(FileSystemStore), @@ -534,9 +404,7 @@ pub enum BlobStoreBase { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct BlockedIp { - #[serde(rename = "address")] pub address: IpAddrOrMask, - #[serde(rename = "reason")] pub reason: BlockReason, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, @@ -547,8 +415,8 @@ pub struct BlockedIp { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Bootstrap { - #[serde(rename = "serverHostname")] - pub server_hostname: String, + #[serde(rename = "defaultHostname")] + pub default_hostname: String, #[serde(rename = "defaultDomain")] pub default_domain: String, #[serde(rename = "requestTlsCertificate")] @@ -563,9 +431,7 @@ pub struct Bootstrap { pub search_store: SearchStore, #[serde(rename = "inMemoryStore")] pub in_memory_store: InMemoryStore, - #[serde(rename = "directory")] pub directory: DirectoryBootstrap, - #[serde(rename = "tracer")] pub tracer: Tracer, #[serde(rename = "dnsServer")] pub dns_server: DnsServerBootstrap, @@ -576,7 +442,6 @@ pub struct Bootstrap { pub struct Cache { #[serde(rename = "accessTokens")] pub access_tokens: u64, - #[serde(rename = "contacts")] pub contacts: u64, #[serde(rename = "dnsIpv4")] pub dns_ipv4: u64, @@ -588,23 +453,26 @@ pub struct Cache { pub dns_mx: u64, #[serde(rename = "dnsPtr")] pub dns_ptr: u64, - #[serde(rename = "dnsRbl")] - pub dns_rbl: u64, + #[serde(rename = "spamLookup")] + pub spam_lookup: u64, #[serde(rename = "dnsTlsa")] pub dns_tlsa: u64, #[serde(rename = "dnsTxt")] pub dns_txt: u64, - #[serde(rename = "events")] pub events: u64, - #[serde(rename = "scheduling")] pub scheduling: u64, - #[serde(rename = "files")] pub files: u64, #[serde(rename = "httpAuth")] pub http_auth: u64, - #[serde(rename = "messages")] pub messages: u64, - #[serde(rename = "domains")] + #[serde(rename = "messagesLarge")] + pub messages_large: u64, + #[serde(rename = "messagesLargeTtl")] + pub messages_large_ttl: Duration, + #[serde(rename = "groupwareLarge")] + pub groupware_large: u64, + #[serde(rename = "groupwareLargeTtl")] + pub groupware_large_ttl: Duration, pub domains: u64, #[serde(rename = "domainNames")] pub domain_names: u64, @@ -614,18 +482,28 @@ pub struct Cache { pub email_addresses: u64, #[serde(rename = "emailAddressesNegative")] pub email_addresses_negative: u64, - #[serde(rename = "accounts")] pub accounts: u64, - #[serde(rename = "roles")] pub roles: u64, - #[serde(rename = "tenants")] - pub tenants: u64, #[serde(rename = "mailingLists")] pub mailing_lists: u64, #[serde(rename = "dkimSignatures")] pub dkim_signatures: u64, #[serde(rename = "negativeTtl")] pub negative_ttl: Duration, + #[serde(rename = "directoryRecipients")] + pub directory_recipients: u64, + #[serde(rename = "directoryRecipientsTtl")] + pub directory_recipients_ttl: Duration, + pub swap: CacheSwap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "@type")] +pub enum CacheSwap { + Disabled, + LocalFile(LocalFileSwap), + Redis(RedisSwap), + BlobStore(BlobStoreSwap), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -649,6 +527,16 @@ pub struct Calendar { pub max_participant_identities: Option, #[serde(rename = "maxEventNotifications")] pub max_event_notifications: Option, + #[serde(rename = "maxCalendarsPerEvent")] + pub max_calendars_per_event: u64, + #[serde(rename = "maxExpandedQueryDuration")] + pub max_expanded_query_duration: Duration, + #[serde(rename = "maxAvailabilityDuration")] + pub max_availability_duration: Duration, + #[serde(rename = "availabilityRateLimit")] + pub availability_rate_limit: Option, + #[serde(rename = "maxAttachmentsSize")] + pub max_attachments_size: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -656,22 +544,18 @@ pub struct Calendar { pub struct CalendarAlarm { #[serde(rename = "allowExternalRcpts")] pub allow_external_rcpts: bool, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "fromEmail")] - pub from_email: Option, + #[serde(rename = "fromAddress")] + pub from_address: Option, #[serde(rename = "fromName")] pub from_name: String, #[serde(rename = "minTriggerInterval")] pub min_trigger_interval: Duration, - #[serde(rename = "template")] - pub template: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct CalendarScheduling { - #[serde(rename = "enable")] pub enable: bool, #[serde(rename = "httpRsvpEnable")] pub http_rsvp_enable: bool, @@ -681,20 +565,15 @@ pub struct CalendarScheduling { pub http_rsvp_url: Option, #[serde(rename = "autoAddInvitations")] pub auto_add_invitations: bool, - #[serde(rename = "itipMaxSize")] - pub itip_max_size: u64, + #[serde(rename = "maxItipSize")] + pub max_itip_size: u64, #[serde(rename = "maxRecipients")] pub max_recipients: u64, - #[serde(rename = "emailTemplate")] - pub email_template: Option, - #[serde(rename = "httpRsvpTemplate")] - pub http_rsvp_template: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Certificate { - #[serde(rename = "certificate")] pub certificate: PublicText, #[serde(rename = "privateKey")] pub private_key: SecretText, @@ -704,7 +583,6 @@ pub struct Certificate { pub not_valid_after: UTCDateTime, #[serde(rename = "notValidBefore")] pub not_valid_before: UTCDateTime, - #[serde(rename = "issuer")] pub issuer: String, } @@ -745,24 +623,18 @@ pub struct ClusterListenerGroupProperties { pub struct ClusterNode { #[serde(rename = "nodeId")] pub node_id: u64, - #[serde(rename = "hostname")] pub hostname: String, #[serde(rename = "lastRenewal")] pub last_renewal: UTCDateTime, - #[serde(rename = "status")] pub status: ClusterNodeStatus, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ClusterRole { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "tasks")] pub tasks: ClusterTaskGroup, - #[serde(rename = "listeners")] pub listeners: ClusterListenerGroup, } @@ -814,7 +686,6 @@ pub enum CredentialPermissions { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct CredentialPermissionsList { - #[serde(rename = "permissions")] pub permissions: Map, } @@ -829,27 +700,21 @@ pub enum Cron { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct CronDaily { - #[serde(rename = "hour")] pub hour: u64, - #[serde(rename = "minute")] pub minute: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct CronHourly { - #[serde(rename = "minute")] pub minute: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct CronWeekly { - #[serde(rename = "day")] pub day: u64, - #[serde(rename = "hour")] pub hour: u64, - #[serde(rename = "minute")] pub minute: u64, } @@ -879,18 +744,8 @@ pub struct DataRetention { pub blob_cleanup_schedule: Cron, #[serde(rename = "maxChangesHistory")] pub max_changes_history: Option, - #[serde(rename = "archiveDeletedItemsFor")] - pub archive_deleted_items_for: Option, - #[serde(rename = "archiveDeletedAccountsFor")] - pub archive_deleted_accounts_for: Option, #[serde(rename = "holdMtaReportsFor")] pub hold_mta_reports_for: Option, - #[serde(rename = "holdTracesFor")] - pub hold_traces_for: Option, - #[serde(rename = "holdMetricsFor")] - pub hold_metrics_for: Option, - #[serde(rename = "metricsCollectionInterval")] - pub metrics_collection_interval: Cron, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -942,17 +797,12 @@ pub enum DirectoryBootstrap { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Dkim1Signature { - #[serde(rename = "auid")] pub auid: Option, - #[serde(rename = "canonicalization")] pub canonicalization: DkimCanonicalization, - #[serde(rename = "expire")] pub expire: Option, - #[serde(rename = "headers")] pub headers: Map, #[serde(rename = "privateKey")] pub private_key: SecretText, - #[serde(rename = "report")] pub report: bool, #[serde(rename = "thirdParty")] pub third_party: Option, @@ -960,36 +810,29 @@ pub struct Dkim1Signature { pub third_party_hash: Option, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "selector")] pub selector: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "nextTransitionAt")] pub next_transition_at: Option, - #[serde(rename = "stage")] pub stage: DkimRotationStage, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Dkim2Signature { - #[serde(rename = "flags")] pub flags: Map, + #[serde(rename = "instanceHashes")] + pub instance_hashes: Map, #[serde(rename = "privateKey")] pub private_key: SecretText, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "selector")] pub selector: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "nextTransitionAt")] pub next_transition_at: Option, - #[serde(rename = "stage")] pub stage: DkimRotationStage, } @@ -1003,7 +846,6 @@ pub enum DkimManagement { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DkimManagementProperties { - #[serde(rename = "algorithms")] pub algorithms: Map, #[serde(rename = "selectorTemplate")] pub selector_template: String, @@ -1026,7 +868,6 @@ pub struct DkimReportSettings { pub send_frequency: Expression, #[serde(rename = "dkimSignDomain")] pub dkim_sign_domain: Expression, - #[serde(rename = "subject")] pub subject: Expression, } @@ -1037,16 +878,14 @@ pub enum DkimSignature { Dkim1RsaSha256(Dkim1Signature), Dkim2Ed25519Sha256(Dkim2Signature), Dkim2RsaSha256(Dkim2Signature), + Dkim2MlDsa44Ed25519Sha512(Dkim2Signature), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcDkimResult { - #[serde(rename = "domain")] pub domain: String, - #[serde(rename = "selector")] pub selector: String, - #[serde(rename = "result")] pub result: DkimAuthResult, #[serde(rename = "humanResult")] pub human_result: Option, @@ -1055,41 +894,30 @@ pub struct DmarcDkimResult { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcExtension { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "definition")] pub definition: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcExternalReport { - #[serde(rename = "report")] pub report: DmarcReport, - #[serde(rename = "from")] pub from: String, - #[serde(rename = "subject")] pub subject: String, - #[serde(rename = "to")] pub to: Map, #[serde(rename = "receivedAt")] pub received_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: UTCDateTime, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcInternalReport { - #[serde(rename = "rua")] pub rua: Map, #[serde(rename = "policyIdentifier")] pub policy_identifier: u64, - #[serde(rename = "report")] pub report: DmarcReport, - #[serde(rename = "domain")] pub domain: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, @@ -1102,28 +930,24 @@ pub struct DmarcInternalReport { pub struct DmarcPolicyOverrideReason { #[serde(rename = "overrideType")] pub override_type: DmarcPolicyOverride, - #[serde(rename = "comment")] pub comment: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcReport { - #[serde(rename = "version")] pub version: Float, #[serde(rename = "orgName")] pub org_name: String, - #[serde(rename = "email")] pub email: String, #[serde(rename = "extraContactInfo")] pub extra_contact_info: Option, #[serde(rename = "reportId")] pub report_id: String, - #[serde(rename = "dateRangeBegin")] - pub date_range_begin: UTCDateTime, + #[serde(rename = "dateRangeStart")] + pub date_range_start: UTCDateTime, #[serde(rename = "dateRangeEnd")] pub date_range_end: UTCDateTime, - #[serde(rename = "errors")] pub errors: Map, #[serde(rename = "policyDomain")] pub policy_domain: String, @@ -1141,11 +965,8 @@ pub struct DmarcReport { pub policy_testing_mode: bool, #[serde(rename = "policyFailureReportingOptions")] pub policy_failure_reporting_options: Map, - #[serde(rename = "records")] pub records: List, - #[serde(rename = "extensions")] pub extensions: List, - #[serde(rename = "generator")] pub generator: Option, #[serde(rename = "policyNp")] pub policy_np: DmarcDisposition, @@ -1158,7 +979,6 @@ pub struct DmarcReport { pub struct DmarcReportRecord { #[serde(rename = "sourceIp")] pub source_ip: Option, - #[serde(rename = "count")] pub count: u64, #[serde(rename = "evaluatedDisposition")] pub evaluated_disposition: DmarcActionDisposition, @@ -1178,7 +998,6 @@ pub struct DmarcReportRecord { pub dkim_results: List, #[serde(rename = "spfResults")] pub spf_results: List, - #[serde(rename = "extensions")] pub extensions: List, } @@ -1191,16 +1010,14 @@ pub struct DmarcReportSettings { pub aggregate_from_address: Expression, #[serde(rename = "aggregateFromName")] pub aggregate_from_name: Expression, - #[serde(rename = "aggregateMaxReportSize")] - pub aggregate_max_report_size: Expression, + #[serde(rename = "maxAggregateReportSize")] + pub max_aggregate_report_size: Expression, #[serde(rename = "aggregateOrgName")] pub aggregate_org_name: Expression, #[serde(rename = "aggregateSendFrequency")] pub aggregate_send_frequency: Expression, #[serde(rename = "aggregateDkimSignDomain")] pub aggregate_dkim_sign_domain: Expression, - #[serde(rename = "aggregateSubject")] - pub aggregate_subject: Expression, #[serde(rename = "failureFromAddress")] pub failure_from_address: Expression, #[serde(rename = "failureFromName")] @@ -1216,11 +1033,8 @@ pub struct DmarcReportSettings { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcSpfResult { - #[serde(rename = "domain")] pub domain: String, - #[serde(rename = "scope")] pub scope: SpfDomainScope, - #[serde(rename = "result")] pub result: SpfAuthResult, #[serde(rename = "humanResult")] pub human_result: Option, @@ -1235,9 +1049,8 @@ pub struct DmarcTroubleshoot { pub ehlo_domain: String, #[serde(rename = "mailFrom")] pub mail_from: String, - #[serde(rename = "to")] - pub to: Map, - #[serde(rename = "message")] + #[serde(rename = "rcptTo")] + pub rcpt_to: Map, pub message: Option, #[serde(rename = "spfEhloDomain")] pub spf_ehlo_domain: String, @@ -1267,7 +1080,6 @@ pub struct DmarcTroubleshoot { pub dmarc_pass: bool, #[serde(rename = "dmarcPolicy")] pub dmarc_policy: DmarcDisposition, - #[serde(rename = "elapsed")] pub elapsed: Duration, } @@ -1286,18 +1098,14 @@ pub enum DmarcTroubleshootAuthResult { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DmarcTroubleshootDetails { - #[serde(rename = "details")] pub details: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsCustomResolver { - #[serde(rename = "protocol")] pub protocol: DnsResolverProtocol, - #[serde(rename = "address")] pub address: IpAddr, - #[serde(rename = "port")] pub port: u64, } @@ -1313,7 +1121,6 @@ pub enum DnsManagement { pub struct DnsManagementProperties { #[serde(rename = "dnsServerId")] pub dns_server_id: Id, - #[serde(rename = "origin")] pub origin: Option, #[serde(rename = "publishRecords")] pub publish_records: Map, @@ -1332,15 +1139,13 @@ pub enum DnsResolver { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsResolverCommon { - #[serde(rename = "attempts")] pub attempts: u64, - #[serde(rename = "concurrency")] - pub concurrency: u64, + #[serde(rename = "maxConcurrentRequests")] + pub max_concurrent_requests: u64, #[serde(rename = "enableEdns")] pub enable_edns: bool, #[serde(rename = "preserveIntermediates")] pub preserve_intermediates: bool, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "tcpOnError")] pub tcp_on_error: bool, @@ -1349,17 +1154,14 @@ pub struct DnsResolverCommon { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsResolverCustom { - #[serde(rename = "servers")] pub servers: List, - #[serde(rename = "attempts")] pub attempts: u64, - #[serde(rename = "concurrency")] - pub concurrency: u64, + #[serde(rename = "maxConcurrentRequests")] + pub max_concurrent_requests: u64, #[serde(rename = "enableEdns")] pub enable_edns: bool, #[serde(rename = "preserveIntermediates")] pub preserve_intermediates: bool, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "tcpOnError")] pub tcp_on_error: bool, @@ -1370,15 +1172,13 @@ pub struct DnsResolverCustom { pub struct DnsResolverTls { #[serde(rename = "useTls")] pub use_tls: bool, - #[serde(rename = "attempts")] pub attempts: u64, - #[serde(rename = "concurrency")] - pub concurrency: u64, + #[serde(rename = "maxConcurrentRequests")] + pub max_concurrent_requests: u64, #[serde(rename = "enableEdns")] pub enable_edns: bool, #[serde(rename = "preserveIntermediates")] pub preserve_intermediates: bool, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "tcpOnError")] pub tcp_on_error: bool, @@ -1388,7 +1188,6 @@ pub struct DnsResolverTls { #[serde(tag = "@type")] pub enum DnsServer { Tsig(DnsServerTsig), - Deprecated1, Cloudflare(DnsServerCloudflare), DigitalOcean(DnsServerCloud), DeSEC(DnsServerCloud), @@ -1463,26 +1262,19 @@ pub enum DnsServer { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerAlidns { - #[serde(rename = "accessKey")] - pub access_key: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "region")] pub region: Option, - #[serde(rename = "securityToken")] - pub security_token: SecretKeyOptional, - #[serde(rename = "line")] + #[serde(rename = "sessionToken")] + pub session_token: SecretKeyOptional, pub line: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1492,22 +1284,14 @@ pub struct DnsServerAlidns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerAutodns { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "password")] pub password: SecretKey, - #[serde(rename = "context")] pub context: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1527,18 +1311,12 @@ pub struct DnsServerAzureDns { pub subscription_id: String, #[serde(rename = "resourceGroup")] pub resource_group: String, - #[serde(rename = "environment")] pub environment: AzureEnvironment, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1548,20 +1326,15 @@ pub struct DnsServerAzureDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerBaiduCloud { - #[serde(rename = "accessKey")] - pub access_key: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1571,11 +1344,8 @@ pub struct DnsServerBaiduCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerBluecatV2 { - #[serde(rename = "baseUrl")] - pub base_url: String, - #[serde(rename = "username")] + pub url: String, pub username: String, - #[serde(rename = "password")] pub password: SecretKey, #[serde(rename = "configName")] pub config_name: String, @@ -1583,16 +1353,11 @@ pub struct DnsServerBluecatV2 { pub view_name: String, #[serde(rename = "skipDeploy")] pub skip_deploy: bool, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1604,7 +1369,6 @@ pub struct DnsServerBluecatV2 { pub enum DnsServerBootstrap { Manual, Tsig(DnsServerTsig), - Deprecated1, Cloudflare(DnsServerCloudflare), DigitalOcean(DnsServerCloud), DeSEC(DnsServerCloud), @@ -1683,18 +1447,12 @@ pub struct DnsServerClouDns { pub auth_id: Option, #[serde(rename = "subAuthId")] pub sub_auth_id: Option, - #[serde(rename = "password")] pub password: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1704,18 +1462,12 @@ pub struct DnsServerClouDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerCloud { - #[serde(rename = "secret")] pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1725,18 +1477,12 @@ pub struct DnsServerCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerCloudflare { - #[serde(rename = "secret")] pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1746,20 +1492,15 @@ pub struct DnsServerCloudflare { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerConstellix { - #[serde(rename = "apiKey")] - pub api_key: String, + #[serde(rename = "apiKeyId")] + pub api_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1769,22 +1510,14 @@ pub struct DnsServerConstellix { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerCpanel { - #[serde(rename = "baseUrl")] - pub base_url: String, - #[serde(rename = "username")] + pub url: String, pub username: String, - #[serde(rename = "token")] pub token: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1794,20 +1527,14 @@ pub struct DnsServerCpanel { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerDnsMadeEasy { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secret")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1817,20 +1544,14 @@ pub struct DnsServerDnsMadeEasy { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerDnsimple { - #[serde(rename = "authToken")] - pub auth_token: SecretKey, + pub token: SecretKey, #[serde(rename = "accountIdentifier")] pub account_identifier: String, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1840,20 +1561,13 @@ pub struct DnsServerDnsimple { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerDomeneshop { - #[serde(rename = "authToken")] - pub auth_token: String, - #[serde(rename = "secret")] + pub token: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1863,20 +1577,13 @@ pub struct DnsServerDomeneshop { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerEasyDns { - #[serde(rename = "token")] pub token: String, - #[serde(rename = "key")] pub key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1886,7 +1593,6 @@ pub struct DnsServerEasyDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerEdgeDns { - #[serde(rename = "host")] pub host: String, #[serde(rename = "clientToken")] pub client_token: String, @@ -1896,16 +1602,11 @@ pub struct DnsServerEdgeDns { pub access_token: SecretKey, #[serde(rename = "accountSwitchKey")] pub account_switch_key: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1915,20 +1616,14 @@ pub struct DnsServerEdgeDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerExoscale { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secret")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1938,20 +1633,14 @@ pub struct DnsServerExoscale { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerGlesys { - #[serde(rename = "apiUser")] - pub api_user: String, + pub username: String, #[serde(rename = "apiKey")] pub api_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1961,20 +1650,14 @@ pub struct DnsServerGlesys { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerGodaddy { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secret")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -1994,16 +1677,11 @@ pub struct DnsServerGoogleCloudDns { pub private_zone: bool, #[serde(rename = "impersonateServiceAccount")] pub impersonate_service_account: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2013,22 +1691,16 @@ pub struct DnsServerGoogleCloudDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerHuaweiCloud { - #[serde(rename = "accessKey")] - pub access_key: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "region")] pub region: String, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2038,18 +1710,12 @@ pub struct DnsServerHuaweiCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerHurricane { - #[serde(rename = "credentials")] pub credentials: List, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2059,20 +1725,14 @@ pub struct DnsServerHurricane { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerIbmCloud { - #[serde(rename = "username")] pub username: String, #[serde(rename = "apiKey")] pub api_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2082,28 +1742,19 @@ pub struct DnsServerIbmCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerInfoblox { - #[serde(rename = "host")] pub host: String, - #[serde(rename = "port")] - pub port: Option, - #[serde(rename = "username")] + pub port: Option, pub username: String, - #[serde(rename = "password")] pub password: SecretKey, #[serde(rename = "wapiVersion")] pub wapi_version: Option, #[serde(rename = "dnsView")] pub dns_view: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2113,24 +1764,16 @@ pub struct DnsServerInfoblox { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerInwx { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "password")] pub password: SecretKey, #[serde(rename = "sharedSecret")] pub shared_secret: SecretKeyOptional, - #[serde(rename = "sandbox")] pub sandbox: bool, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2140,18 +1783,12 @@ pub struct DnsServerInwx { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerJoker { - #[serde(rename = "auth")] pub auth: JokerAuth, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2163,24 +1800,17 @@ pub struct DnsServerJoker { pub struct DnsServerLightsail { #[serde(rename = "accessKeyId")] pub access_key_id: String, - #[serde(rename = "secretAccessKey")] - pub secret_access_key: SecretKey, + #[serde(rename = "secretKey")] + pub secret_key: SecretKey, #[serde(rename = "sessionToken")] pub session_token: SecretKeyOptional, - #[serde(rename = "region")] pub region: Option, - #[serde(rename = "domain")] pub domain: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2190,20 +1820,13 @@ pub struct DnsServerLightsail { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerLuaDns { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "authToken")] - pub auth_token: SecretKey, - #[serde(rename = "description")] + pub token: SecretKey, pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2213,20 +1836,13 @@ pub struct DnsServerLuaDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerMythicBeasts { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "password")] pub password: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2236,20 +1852,13 @@ pub struct DnsServerMythicBeasts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerNameDotCom { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "authToken")] - pub auth_token: SecretKey, - #[serde(rename = "description")] + pub token: SecretKey, pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2265,18 +1874,12 @@ pub struct DnsServerNamecheap { pub api_key: SecretKey, #[serde(rename = "clientIp")] pub client_ip: String, - #[serde(rename = "username")] pub username: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2288,20 +1891,14 @@ pub struct DnsServerNamecheap { pub struct DnsServerNetcup { #[serde(rename = "customerNumber")] pub customer_number: String, - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "password")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub password: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2311,20 +1908,15 @@ pub struct DnsServerNetcup { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerNifcloud { - #[serde(rename = "accessKey")] - pub access_key: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2338,26 +1930,19 @@ pub struct DnsServerOracleCloud { pub tenancy_ocid: String, #[serde(rename = "userOcid")] pub user_ocid: String, - #[serde(rename = "fingerprint")] pub fingerprint: String, #[serde(rename = "privateKeyPem")] pub private_key_pem: SecretText, #[serde(rename = "privateKeyPassword")] pub private_key_password: SecretKeyOptional, - #[serde(rename = "region")] pub region: String, #[serde(rename = "compartmentOcid")] pub compartment_ocid: String, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2375,16 +1960,11 @@ pub struct DnsServerOvh { pub consumer_key: SecretKey, #[serde(rename = "ovhEndpoint")] pub ovh_endpoint: OvhEndpoint, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2394,20 +1974,14 @@ pub struct DnsServerOvh { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerPlesk { - #[serde(rename = "baseUrl")] - pub base_url: String, + pub url: String, #[serde(rename = "apiKey")] pub api_key: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2417,20 +1991,14 @@ pub struct DnsServerPlesk { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerPorkbun { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secretApiKey")] - pub secret_api_key: SecretKey, - #[serde(rename = "description")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, + pub secret: SecretKey, pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2442,20 +2010,14 @@ pub struct DnsServerPorkbun { pub struct DnsServerPowerDns { #[serde(rename = "apiKey")] pub api_key: SecretKey, - #[serde(rename = "endpoint")] - pub endpoint: Option, + pub url: Option, #[serde(rename = "serverId")] pub server_id: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2467,26 +2029,20 @@ pub struct DnsServerPowerDns { pub struct DnsServerRoute53 { #[serde(rename = "accessKeyId")] pub access_key_id: String, - #[serde(rename = "secretAccessKey")] - pub secret_access_key: SecretKey, + #[serde(rename = "secretKey")] + pub secret_key: SecretKey, #[serde(rename = "sessionToken")] pub session_token: SecretKeyOptional, - #[serde(rename = "region")] pub region: String, #[serde(rename = "hostedZoneId")] pub hosted_zone_id: Option, #[serde(rename = "privateZoneOnly")] pub private_zone_only: bool, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2496,20 +2052,14 @@ pub struct DnsServerRoute53 { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerSpaceship { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secret")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2519,24 +2069,18 @@ pub struct DnsServerSpaceship { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerTencentCloud { - #[serde(rename = "secretId")] - pub secret_id: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "region")] pub region: Option, #[serde(rename = "sessionToken")] pub session_token: SecretKeyOptional, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2546,20 +2090,14 @@ pub struct DnsServerTencentCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerTransip { - #[serde(rename = "username")] pub username: String, #[serde(rename = "privateKeyPem")] pub private_key_pem: SecretText, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2569,28 +2107,19 @@ pub struct DnsServerTransip { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerTsig { - #[serde(rename = "host")] pub host: IpAddr, - #[serde(rename = "port")] pub port: u64, #[serde(rename = "keyName")] pub key_name: String, - #[serde(rename = "key")] pub key: SecretKey, - #[serde(rename = "protocol")] pub protocol: IpProtocol, #[serde(rename = "tsigAlgorithm")] pub tsig_algorithm: TsigAlgorithm, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2600,22 +2129,14 @@ pub struct DnsServerTsig { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerUltraDns { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "password")] pub password: SecretKey, - #[serde(rename = "endpoint")] - pub endpoint: Option, - #[serde(rename = "description")] + pub url: Option, pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2625,20 +2146,14 @@ pub struct DnsServerUltraDns { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerVercel { - #[serde(rename = "authToken")] - pub auth_token: SecretKey, + pub token: SecretKey, #[serde(rename = "teamId")] pub team_id: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2648,26 +2163,18 @@ pub struct DnsServerVercel { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerVolcengine { - #[serde(rename = "accessKey")] - pub access_key: String, + #[serde(rename = "accessKeyId")] + pub access_key_id: String, #[serde(rename = "secretKey")] pub secret_key: SecretKey, - #[serde(rename = "region")] pub region: Option, - #[serde(rename = "host")] pub host: Option, - #[serde(rename = "scheme")] pub scheme: Option, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2677,20 +2184,14 @@ pub struct DnsServerVolcengine { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct DnsServerWebSupport { - #[serde(rename = "apiKey")] - pub api_key: String, - #[serde(rename = "secret")] + #[serde(rename = "apiKeyId")] + pub api_key_id: String, pub secret: SecretKey, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2704,16 +2205,11 @@ pub struct DnsServerYandexCloud { pub api_key: SecretText, #[serde(rename = "folderId")] pub folder_id: String, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "timeout")] pub timeout: Duration, - #[serde(rename = "ttl")] pub ttl: Duration, - #[serde(rename = "pollingInterval")] - pub polling_interval: Duration, + #[serde(rename = "pollInterval")] + pub poll_interval: Duration, #[serde(rename = "propagationTimeout")] pub propagation_timeout: Duration, #[serde(rename = "propagationDelay")] @@ -2723,28 +2219,18 @@ pub struct DnsServerYandexCloud { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Domain { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "aliases")] pub aliases: Map, - #[serde(rename = "isEnabled")] - pub is_enabled: bool, + pub enable: bool, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "logo")] - pub logo: Option, #[serde(rename = "certificateManagement")] pub certificate_management: CertificateManagement, #[serde(rename = "dkimManagement")] pub dkim_management: DkimManagement, #[serde(rename = "dnsManagement")] pub dns_management: DnsManagement, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "directoryId")] - pub directory_id: Option, #[serde(rename = "catchAllAddress")] pub catch_all_address: Option, #[serde(rename = "subAddressing")] @@ -2753,8 +2239,6 @@ pub struct Domain { pub allow_relaying: bool, #[serde(rename = "reportAddressUri")] pub report_address_uri: Option, - #[serde(rename = "allowScimProvisioning")] - pub allow_scim_provisioning: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -2771,7 +2255,6 @@ pub struct DsnReportSettings { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ElasticSearchStore { - #[serde(rename = "url")] pub url: String, #[serde(rename = "numReplicas")] pub num_replicas: u64, @@ -2779,7 +2262,6 @@ pub struct ElasticSearchStore { pub num_shards: u64, #[serde(rename = "includeSource")] pub include_source: bool, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, @@ -2816,35 +2298,36 @@ pub struct Email { pub max_identities: Option, #[serde(rename = "maxMailboxes")] pub max_mailboxes: Option, - #[serde(rename = "maxMaskedAddresses")] - pub max_masked_addresses: Option, #[serde(rename = "maxPublicKeys")] pub max_public_keys: Option, + #[serde(rename = "maxMailboxesPerEmail")] + pub max_mailboxes_per_email: u64, + #[serde(rename = "maxFlagsPerEmail")] + pub max_flags_per_email: u64, + #[serde(rename = "maxFlagLength")] + pub max_flag_length: u64, + #[serde(rename = "maxHeaderSize")] + pub max_header_size: u64, + #[serde(rename = "maxHeaderCount")] + pub max_header_count: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct EmailAlias { - #[serde(rename = "enabled")] - pub enabled: bool, - #[serde(rename = "name")] + pub enable: bool, pub name: String, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "description")] pub description: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct EmailFolder { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "create")] pub create: bool, - #[serde(rename = "subscribe")] pub subscribe: bool, - #[serde(rename = "aliases")] pub aliases: Map, } @@ -2869,23 +2352,10 @@ pub struct EncryptionSettings { pub allow_spam_training: bool, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct Enterprise { - #[serde(rename = "apiKey")] - pub api_key: SecretKeyOptional, - #[serde(rename = "licenseKey")] - pub license_key: SecretKeyOptional, - #[serde(rename = "logoUrl")] - pub logo_url: Option, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct EventTracingLevel { - #[serde(rename = "event")] pub event: trc::EventType, - #[serde(rename = "level")] pub level: TracingLevelOpt, } @@ -2903,7 +2373,6 @@ pub struct Expression { pub struct ExpressionMatch { #[serde(rename = "if")] pub if_: String, - #[serde(rename = "then")] pub then: String, } @@ -2921,9 +2390,7 @@ pub struct FileStorage { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct FileSystemStore { - #[serde(rename = "path")] pub path: String, - #[serde(rename = "depth")] pub depth: u64, } @@ -2947,9 +2414,7 @@ pub struct FoundationDbStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct FtrlParameters { - #[serde(rename = "alpha")] pub alpha: Float, - #[serde(rename = "beta")] pub beta: Float, #[serde(rename = "numFeatures")] pub num_features: ModelSize, @@ -2962,30 +2427,19 @@ pub struct FtrlParameters { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct GroupAccount { - #[serde(rename = "name")] pub name: String, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "roles")] pub roles: Roles, - #[serde(rename = "quotas")] pub quotas: VecMap, - #[serde(rename = "permissions")] pub permissions: Permissions, - #[serde(rename = "aliases")] pub aliases: List, - #[serde(rename = "locale")] pub locale: Locale, #[serde(rename = "timeZone")] pub time_zone: Option, - #[serde(rename = "externalId")] - pub external_id: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -3003,8 +2457,10 @@ pub struct Http { pub use_permissive_cors: bool, #[serde(rename = "responseHeaders")] pub response_headers: VecMap, - #[serde(rename = "useXForwarded")] - pub use_x_forwarded: bool, + #[serde(rename = "clientIpSource")] + pub client_ip_source: ClientIpSource, + #[serde(rename = "forwardedHops")] + pub forwarded_hops: u64, #[serde(rename = "redirectRoot")] pub redirect_root: Option, } @@ -3020,9 +2476,7 @@ pub enum HttpAuth { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct HttpAuthBasic { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "secret")] pub secret: SecretKey, } @@ -3042,7 +2496,6 @@ pub struct HttpForm { pub default_from_address: String, #[serde(rename = "fieldEmail")] pub field_email: Option, - #[serde(rename = "enable")] pub enable: bool, #[serde(rename = "fieldHoneyPot")] pub field_honey_pot: Option, @@ -3065,11 +2518,8 @@ pub struct HttpForm { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct HttpLookup { - #[serde(rename = "namespace")] pub namespace: String, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "format")] pub format: HttpLookupFormat, #[serde(rename = "isGzipped")] pub is_gzipped: bool, @@ -3079,14 +2529,17 @@ pub struct HttpLookup { pub max_entry_size: u64, #[serde(rename = "maxSize")] pub max_size: u64, - #[serde(rename = "refresh")] - pub refresh: Duration, - #[serde(rename = "retry")] + #[serde(rename = "refreshInterval")] + pub refresh_interval: Duration, pub retry: Duration, - #[serde(rename = "timeout")] - pub timeout: Duration, - #[serde(rename = "url")] pub url: String, + pub timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, + #[serde(rename = "httpAuth")] + pub http_auth: HttpAuth, + #[serde(rename = "httpHeaders")] + pub http_headers: VecMap, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -3096,7 +2549,6 @@ pub struct HttpLookupCsv { pub index_key: u64, #[serde(rename = "indexValue")] pub index_value: Option, - #[serde(rename = "separator")] pub separator: String, #[serde(rename = "skipFirst")] pub skip_first: bool, @@ -3112,9 +2564,7 @@ pub enum HttpLookupFormat { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct HurricaneCredential { - #[serde(rename = "zone")] pub zone: String, - #[serde(rename = "secret")] pub secret: SecretKey, } @@ -3125,18 +2575,18 @@ pub struct Imap { pub allow_plain_text_auth: bool, #[serde(rename = "maxAuthFailures")] pub max_auth_failures: u64, - #[serde(rename = "maxConcurrent")] - pub max_concurrent: Option, - #[serde(rename = "maxRequestRate")] - pub max_request_rate: Option, + #[serde(rename = "maxConcurrentConnections")] + pub max_concurrent_connections: Option, + #[serde(rename = "rateLimit")] + pub rate_limit: Option, #[serde(rename = "maxRequestSize")] pub max_request_size: u64, - #[serde(rename = "timeoutAnonymous")] - pub timeout_anonymous: Duration, - #[serde(rename = "timeoutAuthenticated")] - pub timeout_authenticated: Duration, - #[serde(rename = "timeoutIdle")] - pub timeout_idle: Duration, + #[serde(rename = "anonymousTimeout")] + pub anonymous_timeout: Duration, + #[serde(rename = "authenticatedTimeout")] + pub authenticated_timeout: Duration, + #[serde(rename = "idleTimeout")] + pub idle_timeout: Duration, #[serde(rename = "maxMessagesPerCommand")] pub max_messages_per_command: u64, #[serde(rename = "minUidBatchSize")] @@ -3151,18 +2601,36 @@ pub struct Imap { #[serde(tag = "@type")] pub enum InMemoryStore { Default, - Sharded(ShardedInMemoryStore), Redis(RedisStore), RedisCluster(RedisClusterStore), RedisSentinel(RedisSentinelStore), } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct IndexQueueStatus { + pub index: IndexType, + pub partition: u64, + #[serde(rename = "queuedUpdates")] + pub queued_updates: u64, + #[serde(rename = "queuedDeletions")] + pub queued_deletions: u64, + pub status: IndexStatus, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "@type")] -pub enum InMemoryStoreBase { - Redis(RedisStore), - RedisCluster(RedisClusterStore), - RedisSentinel(RedisSentinelStore), +pub enum IndexStatus { + Running, + Failed(IndexStatusFailed), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct IndexStatusFailed { + pub reason: String, + #[serde(rename = "nextRetry")] + pub next_retry: UTCDateTime, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -3174,22 +2642,22 @@ pub struct Jmap { pub parse_limit_contact: u64, #[serde(rename = "parseLimitEmail")] pub parse_limit_email: u64, - #[serde(rename = "changesMaxResults")] - pub changes_max_results: u64, - #[serde(rename = "getMaxResults")] - pub get_max_results: u64, - #[serde(rename = "queryMaxResults")] - pub query_max_results: u64, + #[serde(rename = "maxChangesResults")] + pub max_changes_results: u64, + #[serde(rename = "maxGetResults")] + pub max_get_results: u64, + #[serde(rename = "maxQueryResults")] + pub max_query_results: u64, #[serde(rename = "maxMethodCalls")] pub max_method_calls: u64, #[serde(rename = "maxConcurrentRequests")] pub max_concurrent_requests: Option, #[serde(rename = "maxRequestSize")] pub max_request_size: u64, - #[serde(rename = "setMaxObjects")] - pub set_max_objects: u64, - #[serde(rename = "snippetMaxResults")] - pub snippet_max_results: u64, + #[serde(rename = "maxSetObjects")] + pub max_set_objects: u64, + #[serde(rename = "maxSnippetResults")] + pub max_snippet_results: u64, #[serde(rename = "maxConcurrentUploads")] pub max_concurrent_uploads: Option, #[serde(rename = "maxUploadSize")] @@ -3204,8 +2672,8 @@ pub struct Jmap { pub event_source_throttle: Duration, #[serde(rename = "pushAttemptWait")] pub push_attempt_wait: Duration, - #[serde(rename = "pushMaxAttempts")] - pub push_max_attempts: u64, + #[serde(rename = "maxPushAttempts")] + pub max_push_attempts: u64, #[serde(rename = "pushRetryWait")] pub push_retry_wait: Duration, #[serde(rename = "pushThrottle")] @@ -3249,33 +2717,31 @@ pub struct JokerAuthApiKey { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct JokerAuthUsernamePassword { - #[serde(rename = "username")] pub username: String, - #[serde(rename = "password")] pub password: SecretKey, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct KafkaCoordinator { - #[serde(rename = "brokers")] - pub brokers: Map, + pub hosts: Map, #[serde(rename = "groupId")] pub group_id: String, - #[serde(rename = "timeoutMessage")] - pub timeout_message: Duration, - #[serde(rename = "timeoutSession")] - pub timeout_session: Duration, + #[serde(rename = "messageTimeout")] + pub message_timeout: Duration, + #[serde(rename = "sessionTimeout")] + pub session_timeout: Duration, + #[serde(rename = "useTls")] + pub use_tls: bool, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct LdapDirectory { - #[serde(rename = "description")] pub description: String, - #[serde(rename = "url")] pub url: String, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, @@ -3291,8 +2757,8 @@ pub struct LdapDirectory { pub bind_authentication: bool, #[serde(rename = "filterLogin")] pub filter_login: String, - #[serde(rename = "filterMailbox")] - pub filter_mailbox: String, + #[serde(rename = "filterRecipient")] + pub filter_recipient: String, #[serde(rename = "filterMemberOf")] pub filter_member_of: Option, #[serde(rename = "attrClass")] @@ -3319,20 +2785,28 @@ pub struct LdapDirectory { pub pool_timeout_recycle: Duration, #[serde(rename = "poolTimeoutWait")] pub pool_timeout_wait: Duration, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct LocalFileSwap { + pub path: String, + #[serde(rename = "flushChanges")] + pub flush_changes: u64, + #[serde(rename = "flushInterval")] + pub flush_interval: Duration, + #[serde(rename = "maxAccountSize")] + pub max_account_size: u64, + #[serde(rename = "minAccountSize")] + pub min_account_size: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Log { - #[serde(rename = "timestamp")] pub timestamp: UTCDateTime, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "event")] pub event: trc::EventType, - #[serde(rename = "details")] pub details: String, } @@ -3342,7 +2816,6 @@ pub enum LookupStore { PostgreSql(PostgreSqlStore), MySql(MySqlStore), Sqlite(SqliteStore), - Sharded(ShardedInMemoryStore), Redis(RedisStore), RedisCluster(RedisClusterStore), RedisSentinel(RedisSentinelStore), @@ -3351,64 +2824,31 @@ pub enum LookupStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MailExchanger { - #[serde(rename = "hostname")] pub hostname: Option, - #[serde(rename = "priority")] pub priority: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MailingList { - #[serde(rename = "name")] pub name: String, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "aliases")] pub aliases: List, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "recipients")] pub recipients: Map, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct MaskedEmail { - #[serde(rename = "enabled")] - pub enabled: bool, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "email")] - pub email: String, - #[serde(rename = "description")] - pub description: Option, - #[serde(rename = "forDomain")] - pub for_domain: Option, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "createdBy")] - pub created_by: Option, - #[serde(rename = "expiresAt")] - pub expires_at: Option, - #[serde(rename = "url")] - pub url: Option, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MeilisearchStore { - #[serde(rename = "url")] pub url: String, #[serde(rename = "pollInterval")] pub poll_interval: Duration, - #[serde(rename = "maxRetries")] - pub max_retries: u64, + #[serde(rename = "maxPolls")] + pub max_polls: u64, #[serde(rename = "failOnTimeout")] pub fail_on_timeout: bool, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, @@ -3421,9 +2861,7 @@ pub struct MeilisearchStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MemoryLookupKey { - #[serde(rename = "namespace")] pub namespace: String, - #[serde(rename = "key")] pub key: String, #[serde(rename = "isGlobPattern")] pub is_glob_pattern: bool, @@ -3432,42 +2870,38 @@ pub struct MemoryLookupKey { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MemoryLookupKeyValue { - #[serde(rename = "namespace")] pub namespace: String, - #[serde(rename = "key")] pub key: String, - #[serde(rename = "value")] pub value: String, #[serde(rename = "isGlobPattern")] pub is_glob_pattern: bool, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum Metric { - Counter(MetricCount), - Gauge(MetricCount), - Histogram(MetricSum), -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] -pub struct MetricCount { - #[serde(rename = "count")] - pub count: u64, - #[serde(rename = "metric")] - pub metric: trc::MetricType, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct MetricSum { - #[serde(rename = "count")] - pub count: u64, - #[serde(rename = "sum")] - pub sum: u64, - #[serde(rename = "metric")] - pub metric: trc::MetricType, +pub struct Metadata { + #[serde(rename = "dataTypes")] + pub data_types: Map, + #[serde(rename = "vendorNamespaces")] + pub vendor_namespaces: bool, + #[serde(rename = "privateMetadata")] + pub private_metadata: bool, + #[serde(rename = "maxDepth")] + pub max_depth: Option, + #[serde(rename = "maxEntrySize")] + pub max_entry_size: u64, + #[serde(rename = "maxSize")] + pub max_size: u64, + #[serde(rename = "maxPrivateSize")] + pub max_private_size: u64, + #[serde(rename = "maxEntries")] + pub max_entries: u64, + #[serde(rename = "maxQueryScan")] + pub max_query_scan: u64, + #[serde(rename = "imapServerComment")] + pub imap_server_comment: Option, + #[serde(rename = "imapServerAdmin")] + pub imap_server_admin: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -3475,9 +2909,7 @@ pub struct MetricSum { pub struct Metrics { #[serde(rename = "openTelemetry")] pub open_telemetry: MetricsOtel, - #[serde(rename = "prometheus")] pub prometheus: MetricsPrometheus, - #[serde(rename = "metrics")] pub metrics: Map, #[serde(rename = "metricsPolicy")] pub metrics_policy: EventPolicy, @@ -3494,23 +2926,19 @@ pub enum MetricsOtel { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MetricsOtelGrpc { - #[serde(rename = "endpoint")] - pub endpoint: Option, - #[serde(rename = "interval")] + pub url: Option, pub interval: Duration, - #[serde(rename = "timeout")] pub timeout: Duration, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MetricsOtelHttp { - #[serde(rename = "endpoint")] - pub endpoint: String, - #[serde(rename = "interval")] + pub url: String, pub interval: Duration, - #[serde(rename = "timeout")] pub timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] @@ -3533,36 +2961,24 @@ pub struct MetricsPrometheusProperties { pub auth_username: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum MetricsStore { - Disabled, - Default, - FoundationDb(FoundationDbStore), - PostgreSql(PostgreSqlStore), - MySql(MySqlStore), -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaConnectionIpHost { #[serde(rename = "ehloHostname")] pub ehlo_hostname: Option, - #[serde(rename = "sourceIp")] - pub source_ip: IpAddr, + #[serde(rename = "localIp")] + pub local_ip: IpAddr, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaConnectionStrategy { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "ehloHostname")] pub ehlo_hostname: Option, - #[serde(rename = "sourceIps")] - pub source_ips: List, + #[serde(rename = "localIps")] + pub local_ips: List, #[serde(rename = "connectTimeout")] pub connect_timeout: Duration, #[serde(rename = "dataTimeout")] @@ -3594,38 +3010,30 @@ pub struct MtaDeliveryExpirationAttempts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaDeliveryExpirationTtl { - #[serde(rename = "expire")] pub expire: Duration, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaDeliverySchedule { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "expiry")] pub expiry: MtaDeliveryExpiration, - #[serde(rename = "notify")] pub notify: MtaDeliveryScheduleIntervalsOrDefault, #[serde(rename = "queueId")] pub queue_id: Id, - #[serde(rename = "retry")] pub retry: MtaDeliveryScheduleIntervalsOrDefault, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaDeliveryScheduleInterval { - #[serde(rename = "duration")] pub duration: Duration, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaDeliveryScheduleIntervals { - #[serde(rename = "intervals")] pub intervals: List, } @@ -3639,13 +3047,10 @@ pub enum MtaDeliveryScheduleIntervalsOrDefault { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaExtensions { - #[serde(rename = "chunking")] pub chunking: Expression, #[serde(rename = "deliverBy")] pub deliver_by: Expression, - #[serde(rename = "dsn")] pub dsn: Expression, - #[serde(rename = "expn")] pub expn: Expression, #[serde(rename = "futureRelease")] pub future_release: Expression, @@ -3653,31 +3058,25 @@ pub struct MtaExtensions { pub mt_priority: Expression, #[serde(rename = "noSoliciting")] pub no_soliciting: Expression, - #[serde(rename = "pipelining")] pub pipelining: Expression, #[serde(rename = "requireTls")] pub require_tls: Expression, - #[serde(rename = "vrfy")] pub vrfy: Expression, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaHook { - #[serde(rename = "allowInvalidCerts")] - pub allow_invalid_certs: bool, - #[serde(rename = "enable")] pub enable: Expression, #[serde(rename = "maxResponseSize")] pub max_response_size: u64, #[serde(rename = "tempFailOnError")] pub temp_fail_on_error: bool, - #[serde(rename = "stages")] pub stages: Map, - #[serde(rename = "timeout")] - pub timeout: Duration, - #[serde(rename = "url")] pub url: String, + pub timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] @@ -3689,8 +3088,8 @@ pub struct MtaHook { pub struct MtaInboundSession { #[serde(rename = "maxDuration")] pub max_duration: Expression, - #[serde(rename = "timeout")] - pub timeout: Expression, + #[serde(rename = "idleTimeout")] + pub idle_timeout: Expression, #[serde(rename = "transferLimit")] pub transfer_limit: Expression, } @@ -3698,15 +3097,11 @@ pub struct MtaInboundSession { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaInboundThrottle { - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "key")] pub key: Map, #[serde(rename = "match")] pub match_: Expression, - #[serde(rename = "rate")] pub rate: Rate, } @@ -3715,26 +3110,22 @@ pub struct MtaInboundThrottle { pub struct MtaMilter { #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, - #[serde(rename = "enable")] pub enable: Expression, - #[serde(rename = "hostname")] - pub hostname: String, + pub host: String, #[serde(rename = "maxResponseSize")] pub max_response_size: u64, #[serde(rename = "tempFailOnError")] pub temp_fail_on_error: bool, #[serde(rename = "protocolVersion")] pub protocol_version: MilterVersion, - #[serde(rename = "port")] pub port: u64, - #[serde(rename = "stages")] pub stages: Map, - #[serde(rename = "timeoutCommand")] - pub timeout_command: Duration, - #[serde(rename = "timeoutConnect")] - pub timeout_connect: Duration, - #[serde(rename = "timeoutData")] - pub timeout_data: Duration, + #[serde(rename = "commandTimeout")] + pub command_timeout: Duration, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Duration, + #[serde(rename = "dataTimeout")] + pub data_timeout: Duration, #[serde(rename = "useTls")] pub use_tls: bool, #[serde(rename = "flagsAction")] @@ -3746,45 +3137,32 @@ pub struct MtaMilter { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaOutboundStrategy { - #[serde(rename = "connection")] pub connection: Expression, - #[serde(rename = "route")] pub route: Expression, - #[serde(rename = "schedule")] pub schedule: Expression, - #[serde(rename = "tls")] pub tls: Expression, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaOutboundThrottle { - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "key")] pub key: Map, #[serde(rename = "match")] pub match_: Expression, - #[serde(rename = "rate")] pub rate: Rate, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaQueueQuota { - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "key")] pub key: Map, #[serde(rename = "match")] pub match_: Expression, - #[serde(rename = "messages")] pub messages: Option, - #[serde(rename = "size")] pub size: Option, } @@ -3799,9 +3177,7 @@ pub enum MtaRoute { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaRouteCommon { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, } @@ -3814,32 +3190,25 @@ pub struct MtaRouteMx { pub max_multihomed: u64, #[serde(rename = "maxMxHosts")] pub max_mx_hosts: u64, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaRouteRelay { - #[serde(rename = "address")] - pub address: String, + pub host: String, #[serde(rename = "authSecret")] pub auth_secret: SecretKeyOptional, #[serde(rename = "authUsername")] pub auth_username: Option, - #[serde(rename = "port")] pub port: u64, - #[serde(rename = "protocol")] pub protocol: MtaProtocol, #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, #[serde(rename = "implicitTls")] pub implicit_tls: bool, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, } @@ -3854,7 +3223,6 @@ pub struct MtaStageAuth { pub sasl_mechanisms: Expression, #[serde(rename = "mustMatchSender")] pub must_match_sender: Expression, - #[serde(rename = "require")] pub require: Expression, } @@ -3863,9 +3231,7 @@ pub struct MtaStageAuth { pub struct MtaStageConnect { #[serde(rename = "smtpGreeting")] pub smtp_greeting: Expression, - #[serde(rename = "hostname")] pub hostname: Expression, - #[serde(rename = "script")] pub script: Expression, } @@ -3892,10 +3258,11 @@ pub struct MtaStageData { pub max_received_headers: Expression, #[serde(rename = "maxMessageSize")] pub max_message_size: Expression, - #[serde(rename = "script")] pub script: Expression, #[serde(rename = "enableSpamFilter")] pub enable_spam_filter: Expression, + #[serde(rename = "dkimSignDomain")] + pub dkim_sign_domain: Expression, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -3903,9 +3270,7 @@ pub struct MtaStageData { pub struct MtaStageEhlo { #[serde(rename = "rejectNonFqdn")] pub reject_non_fqdn: Expression, - #[serde(rename = "require")] pub require: Expression, - #[serde(rename = "script")] pub script: Expression, } @@ -3914,9 +3279,7 @@ pub struct MtaStageEhlo { pub struct MtaStageMail { #[serde(rename = "isSenderAllowed")] pub is_sender_allowed: Expression, - #[serde(rename = "rewrite")] pub rewrite: Expression, - #[serde(rename = "script")] pub script: Expression, } @@ -3931,9 +3294,7 @@ pub struct MtaStageRcpt { pub max_recipients: Expression, #[serde(rename = "allowRelaying")] pub allow_relaying: Expression, - #[serde(rename = "rewrite")] pub rewrite: Expression, - #[serde(rename = "script")] pub script: Expression, } @@ -3942,7 +3303,6 @@ pub struct MtaStageRcpt { pub struct MtaSts { #[serde(rename = "maxAge")] pub max_age: Duration, - #[serde(rename = "mode")] pub mode: PolicyEnforcement, #[serde(rename = "mxHosts")] pub mx_hosts: Map, @@ -3951,13 +3311,10 @@ pub struct MtaSts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaTlsStrategy { - #[serde(rename = "name")] pub name: String, #[serde(rename = "allowInvalidCerts")] pub allow_invalid_certs: bool, - #[serde(rename = "dane")] pub dane: MtaRequiredOrOptional, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "mtaSts")] pub mta_sts: MtaRequiredOrOptional, @@ -3972,34 +3329,17 @@ pub struct MtaTlsStrategy { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MtaVirtualQueue { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "threadsPerNode")] pub threads_per_node: u64, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct MySqlSettings { - #[serde(rename = "host")] - pub host: String, - #[serde(rename = "port")] - pub port: u64, - #[serde(rename = "database")] - pub database: String, - #[serde(rename = "authUsername")] - pub auth_username: Option, - #[serde(rename = "authSecret")] - pub auth_secret: SecretKeyOptional, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct MySqlStore { - #[serde(rename = "timeout")] - pub timeout: Option, + #[serde(rename = "idleTimeout")] + pub idle_timeout: Option, #[serde(rename = "useTls")] pub use_tls: bool, #[serde(rename = "allowInvalidCerts")] @@ -4010,13 +3350,8 @@ pub struct MySqlStore { pub pool_max_connections: Option, #[serde(rename = "poolMinConnections")] pub pool_min_connections: Option, - #[serde(rename = "readReplicas")] - pub read_replicas: List, - #[serde(rename = "host")] pub host: String, - #[serde(rename = "port")] pub port: u64, - #[serde(rename = "database")] pub database: String, #[serde(rename = "authUsername")] pub auth_username: Option, @@ -4027,42 +3362,39 @@ pub struct MySqlStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct NatsCoordinator { - #[serde(rename = "addresses")] - pub addresses: Map, + pub hosts: Map, #[serde(rename = "maxReconnects")] pub max_reconnects: Option, - #[serde(rename = "timeoutConnection")] - pub timeout_connection: Duration, - #[serde(rename = "timeoutRequest")] - pub timeout_request: Duration, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Duration, + #[serde(rename = "requestTimeout")] + pub request_timeout: Duration, #[serde(rename = "pingInterval")] pub ping_interval: Duration, - #[serde(rename = "capacityClient")] - pub capacity_client: u64, - #[serde(rename = "capacityReadBuffer")] - pub capacity_read_buffer: u64, - #[serde(rename = "capacitySubscription")] - pub capacity_subscription: u64, + #[serde(rename = "clientCapacity")] + pub client_capacity: u64, + #[serde(rename = "readBufferCapacity")] + pub read_buffer_capacity: u64, + #[serde(rename = "subscriptionCapacity")] + pub subscription_capacity: u64, #[serde(rename = "noEcho")] pub no_echo: bool, #[serde(rename = "useTls")] pub use_tls: bool, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "authSecret")] pub auth_secret: SecretKeyOptional, #[serde(rename = "authUsername")] pub auth_username: Option, - #[serde(rename = "credentials")] pub credentials: SecretTextOptional, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct NetworkListener { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "bind")] pub bind: Map, - #[serde(rename = "protocol")] pub protocol: NetworkListenerProtocol, #[serde(rename = "overrideProxyTrustedNetworks")] pub override_proxy_trusted_networks: Map, @@ -4090,8 +3422,8 @@ pub struct NetworkListener { pub tls_disable_protocols: Map, #[serde(rename = "tlsIgnoreClientOrder")] pub tls_ignore_client_order: bool, - #[serde(rename = "tlsImplicit")] - pub tls_implicit: bool, + #[serde(rename = "implicitTls")] + pub implicit_tls: bool, #[serde(rename = "tlsTimeout")] pub tls_timeout: Option, #[serde(rename = "maxConnections")] @@ -4103,35 +3435,55 @@ pub struct NetworkListener { pub struct OAuthClient { #[serde(rename = "clientId")] pub client_id: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "contacts")] pub contacts: Map, - #[serde(rename = "secret")] pub secret: Option, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: Option, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, #[serde(rename = "redirectUris")] pub redirect_uris: Map, - #[serde(rename = "logo")] + #[serde(rename = "postLogoutRedirectUris")] + pub post_logout_redirect_uris: Map, pub logo: Option, + #[serde(rename = "firstParty")] + pub first_party: bool, + pub scopes: Map, + #[serde(rename = "grantTypes")] + pub grant_types: Map, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct OAuthGrant { + #[serde(rename = "accountId")] + pub account_id: Id, + #[serde(rename = "clientId")] + pub client_id: String, + #[serde(rename = "clientName")] + pub client_name: Option, + #[serde(rename = "clientKind")] + pub client_kind: OAuthClientKind, + pub scopes: Map, + #[serde(rename = "remoteIp")] + pub remote_ip: Option, + #[serde(rename = "createdAt")] + pub created_at: UTCDateTime, + #[serde(rename = "lastUsedAt")] + pub last_used_at: UTCDateTime, + pub generation: u64, + pub pending: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct OidcDirectory { - #[serde(rename = "description")] pub description: String, #[serde(rename = "issuerUrl")] pub issuer_url: String, - #[serde(rename = "requireAudience")] - pub require_audience: Option, - #[serde(rename = "requireScopes")] - pub require_scopes: Map, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "claimUsername")] pub claim_username: String, #[serde(rename = "usernameDomain")] @@ -4140,25 +3492,33 @@ pub struct OidcDirectory { pub claim_name: Option, #[serde(rename = "claimGroups")] pub claim_groups: Option, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, + #[serde(rename = "acceptedAudiences")] + pub accepted_audiences: Map, + #[serde(rename = "opaqueTokenValidation")] + pub opaque_token_validation: OidcOpaqueTokenValidation, + #[serde(rename = "introspectionClientId")] + pub introspection_client_id: Option, + #[serde(rename = "introspectionClientSecret")] + pub introspection_client_secret: SecretKeyOptional, + #[serde(rename = "introspectionAuthMethod")] + pub introspection_auth_method: OidcIntrospectionAuth, + #[serde(rename = "unscopedTokens")] + pub unscoped_tokens: OidcUnscopedTokenPolicy, + #[serde(rename = "skipAudienceValidation")] + pub skip_audience_validation: bool, + #[serde(rename = "allowIdTokenClaims")] + pub allow_id_token_claims: bool, + #[serde(rename = "lenientIssuerMatch")] + pub lenient_issuer_match: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct OidcProvider { - #[serde(rename = "authCodeMaxAttempts")] - pub auth_code_max_attempts: u64, - #[serde(rename = "anonymousClientRegistration")] - pub anonymous_client_registration: bool, - #[serde(rename = "requireClientRegistration")] - pub require_client_registration: bool, #[serde(rename = "authCodeExpiry")] pub auth_code_expiry: Duration, #[serde(rename = "refreshTokenExpiry")] pub refresh_token_expiry: Duration, - #[serde(rename = "refreshTokenRenewal")] - pub refresh_token_renewal: Duration, #[serde(rename = "accessTokenExpiry")] pub access_token_expiry: Duration, #[serde(rename = "userCodeExpiry")] @@ -4171,6 +3531,8 @@ pub struct OidcProvider { pub signature_algorithm: JwtSignatureAlgorithm, #[serde(rename = "signatureKey")] pub signature_key: SecretText, + #[serde(rename = "clientMetadataDocuments")] + pub client_metadata_documents: OAuthClientMetadataPolicy, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -4187,10 +3549,9 @@ pub struct OtpAuth { pub struct PasswordCredential { #[serde(rename = "credentialId")] pub credential_id: Id, - #[serde(rename = "secret")] pub secret: String, - #[serde(rename = "otpAuth")] - pub otp_auth: Option, + #[serde(rename = "otpUrl")] + pub otp_url: Option, #[serde(rename = "expiresAt")] pub expires_at: Option, #[serde(rename = "allowedIps")] @@ -4214,28 +3575,11 @@ pub struct PermissionsList { pub disabled_permissions: Map, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct PostgreSqlSettings { - #[serde(rename = "host")] - pub host: String, - #[serde(rename = "port")] - pub port: u64, - #[serde(rename = "database")] - pub database: String, - #[serde(rename = "authUsername")] - pub auth_username: Option, - #[serde(rename = "authSecret")] - pub auth_secret: SecretKeyOptional, - #[serde(rename = "options")] - pub options: Option, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct PostgreSqlStore { - #[serde(rename = "timeout")] - pub timeout: Option, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Option, #[serde(rename = "useTls")] pub use_tls: bool, #[serde(rename = "allowInvalidCerts")] @@ -4244,19 +3588,13 @@ pub struct PostgreSqlStore { pub pool_max_connections: Option, #[serde(rename = "poolRecyclingMethod")] pub pool_recycling_method: PostgreSqlRecyclingMethod, - #[serde(rename = "readReplicas")] - pub read_replicas: List, - #[serde(rename = "host")] pub host: String, - #[serde(rename = "port")] pub port: u64, - #[serde(rename = "database")] pub database: String, #[serde(rename = "authUsername")] pub auth_username: Option, #[serde(rename = "authSecret")] pub auth_secret: SecretKeyOptional, - #[serde(rename = "options")] pub options: Option, } @@ -4265,9 +3603,7 @@ pub struct PostgreSqlStore { pub struct PublicKey { #[serde(rename = "accountId")] pub account_id: Id, - #[serde(rename = "key")] pub key: String, - #[serde(rename = "description")] pub description: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, @@ -4289,7 +3625,6 @@ pub enum PublicStringOptional { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct PublicStringValue { - #[serde(rename = "value")] pub value: String, } @@ -4304,7 +3639,6 @@ pub enum PublicText { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct PublicTextValue { - #[serde(rename = "value")] pub value: String, } @@ -4342,19 +3676,15 @@ pub struct QueuedMessage { pub blob_id: BlobId, #[serde(rename = "returnPath")] pub return_path: String, - #[serde(rename = "recipients")] pub recipients: VecMap, #[serde(rename = "receivedFromIp")] pub received_from_ip: IpAddr, #[serde(rename = "receivedViaPort")] pub received_via_port: u64, - #[serde(rename = "flags")] pub flags: Map, #[serde(rename = "envId")] pub env_id: Option, - #[serde(rename = "priority")] pub priority: i64, - #[serde(rename = "size")] pub size: u64, } @@ -4369,24 +3699,18 @@ pub struct QueuedRecipient { pub notify_count: u64, #[serde(rename = "notifyDue")] pub notify_due: UTCDateTime, - #[serde(rename = "expires")] pub expires: QueueExpiry, #[serde(rename = "queueName")] pub queue_name: String, - #[serde(rename = "status")] pub status: RecipientStatus, - #[serde(rename = "flags")] pub flags: Map, - #[serde(rename = "orcpt")] pub orcpt: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Rate { - #[serde(rename = "count")] pub count: u64, - #[serde(rename = "period")] pub period: Duration, } @@ -4402,10 +3726,11 @@ pub enum RecipientStatus { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RedisClusterStore { - #[serde(rename = "urls")] pub urls: Map, - #[serde(rename = "timeout")] - pub timeout: Duration, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "authUsername")] pub auth_username: Option, #[serde(rename = "authSecret")] @@ -4433,12 +3758,13 @@ pub struct RedisClusterStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RedisSentinelStore { - #[serde(rename = "urls")] pub urls: Map, #[serde(rename = "serviceName")] pub service_name: String, - #[serde(rename = "timeout")] - pub timeout: Duration, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "authUsername")] pub auth_username: Option, #[serde(rename = "authSecret")] @@ -4462,10 +3788,11 @@ pub struct RedisSentinelStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RedisStore { - #[serde(rename = "url")] pub url: String, - #[serde(rename = "timeout")] - pub timeout: Duration, + #[serde(rename = "connectTimeout")] + pub connect_timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "poolMaxConnections")] pub pool_max_connections: u64, #[serde(rename = "poolTimeoutCreate")] @@ -4476,6 +3803,38 @@ pub struct RedisStore { pub pool_timeout_recycle: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct RedisSwap { + pub store: RedisSwapStore, + #[serde(rename = "chunkSize")] + pub chunk_size: u64, + pub retention: Duration, + #[serde(rename = "flushChanges")] + pub flush_changes: u64, + #[serde(rename = "flushInterval")] + pub flush_interval: Duration, + #[serde(rename = "maxAccountSize")] + pub max_account_size: u64, + #[serde(rename = "minAccountSize")] + pub min_account_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "@type")] +pub enum RedisSwapStore { + Default, + Redis(RedisStore), + RedisCluster(RedisClusterStore), + RedisSentinel(RedisSentinelStore), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct ReloadSettingsResult { + pub warnings: Map, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ReportSettings { @@ -4487,14 +3846,13 @@ pub struct ReportSettings { pub outbound_report_domain: Option, #[serde(rename = "outboundReportSubmitter")] pub outbound_report_submitter: Expression, - #[serde(rename = "inboundReportMaxSize")] - pub inbound_report_max_size: i64, + #[serde(rename = "maxInboundReportSize")] + pub max_inbound_report_size: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RocksDbStore { - #[serde(rename = "path")] pub path: String, #[serde(rename = "blobSize")] pub blob_size: u64, @@ -4503,16 +3861,13 @@ pub struct RocksDbStore { #[serde(rename = "poolWorkers")] pub pool_workers: Option, #[serde(rename = "cacheSize")] - pub cache_size: u64, + pub cache_size: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Role { - #[serde(rename = "description")] pub description: String, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, #[serde(rename = "roleIds")] pub role_ids: Map, #[serde(rename = "enabledPermissions")] @@ -4531,21 +3886,15 @@ pub enum Roles { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct S3Store { - #[serde(rename = "region")] pub region: S3StoreRegion, - #[serde(rename = "bucket")] pub bucket: String, - #[serde(rename = "accessKey")] - pub access_key: PublicStringOptional, + #[serde(rename = "accessKeyId")] + pub access_key_id: PublicStringOptional, #[serde(rename = "secretKey")] pub secret_key: SecretKeyOptional, - #[serde(rename = "securityToken")] - pub security_token: SecretKeyOptional, #[serde(rename = "sessionToken")] pub session_token: SecretKeyOptional, - #[serde(rename = "profile")] pub profile: Option, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "maxRetries")] pub max_retries: u64, @@ -4560,10 +3909,8 @@ pub struct S3Store { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct S3StoreCustomRegion { - #[serde(rename = "customEndpoint")] - pub custom_endpoint: String, - #[serde(rename = "customRegion")] - pub custom_region: String, + pub url: String, + pub region: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -4635,10 +3982,26 @@ pub struct Search { pub index_email: bool, #[serde(rename = "indexEmailFields")] pub index_email_fields: Map, - #[serde(rename = "indexTelemetry")] - pub index_telemetry: bool, - #[serde(rename = "indexTracingFields")] - pub index_tracing_fields: Map, + #[serde(rename = "maxConcurrentIndexTasks")] + pub max_concurrent_index_tasks: u64, + #[serde(rename = "indexFiles")] + pub index_files: bool, + #[serde(rename = "maxExtractDocumentSize")] + pub max_extract_document_size: u64, + #[serde(rename = "maxExtractTextSize")] + pub max_extract_text_size: u64, + #[serde(rename = "maxExtractDecompressedSize")] + pub max_extract_decompressed_size: u64, + #[serde(rename = "maxExtractPartSize")] + pub max_extract_part_size: u64, + #[serde(rename = "maxExtractParts")] + pub max_extract_parts: u64, + #[serde(rename = "maxExtractArchiveEntries")] + pub max_extract_archive_entries: u64, + #[serde(rename = "maxExtractPdfObjects")] + pub max_extract_pdf_objects: u64, + #[serde(rename = "maxExtractRtfDepth")] + pub max_extract_rtf_depth: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -4657,15 +4020,12 @@ pub enum SearchStore { pub struct SecondaryCredential { #[serde(rename = "credentialId")] pub credential_id: Id, - #[serde(rename = "description")] pub description: String, - #[serde(rename = "secret")] pub secret: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: Option, - #[serde(rename = "permissions")] pub permissions: CredentialPermissions, #[serde(rename = "allowedIps")] pub allowed_ips: Map, @@ -4705,7 +4065,6 @@ pub enum SecretKeyOptional { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SecretKeyValue { - #[serde(rename = "secret")] pub secret: String, } @@ -4729,7 +4088,6 @@ pub enum SecretTextOptional { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SecretTextValue { - #[serde(rename = "secret")] pub secret: String, } @@ -4759,8 +4117,6 @@ pub struct Security { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SenderAuth { - #[serde(rename = "dkimSignDomain")] - pub dkim_sign_domain: Expression, #[serde(rename = "dkimStrict")] pub dkim_strict: bool, #[serde(rename = "dkimVerify")] @@ -4793,26 +4149,10 @@ pub struct ServerResponse { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Service { - #[serde(rename = "hostname")] pub hostname: Option, - #[serde(rename = "cleartext")] pub cleartext: bool, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ShardedBlobStore { - #[serde(rename = "stores")] - pub stores: List, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct ShardedInMemoryStore { - #[serde(rename = "stores")] - pub stores: List, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct Sharing { @@ -4831,8 +4171,6 @@ pub struct SieveSystemInterpreter { pub default_from_name: Expression, #[serde(rename = "messageIdHostname")] pub message_id_hostname: Option, - #[serde(rename = "duplicateExpiry")] - pub duplicate_expiry: Duration, #[serde(rename = "noCapabilityCheck")] pub no_capability_check: bool, #[serde(rename = "defaultReturnPath")] @@ -4856,13 +4194,10 @@ pub struct SieveSystemInterpreter { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SieveSystemScript { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "isActive")] pub is_active: bool, - #[serde(rename = "contents")] pub contents: String, } @@ -4926,20 +4261,16 @@ pub struct SieveUserInterpreter { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SieveUserScript { - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, #[serde(rename = "isActive")] pub is_active: bool, - #[serde(rename = "contents")] pub contents: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamClassifier { - #[serde(rename = "model")] pub model: SpamClassifierModel, #[serde(rename = "learnHamFromCard")] pub learn_ham_from_card: bool, @@ -4955,8 +4286,8 @@ pub struct SpamClassifier { pub min_spam_samples: u64, #[serde(rename = "reservoirCapacity")] pub reservoir_capacity: u64, - #[serde(rename = "trainFrequency")] - pub train_frequency: Option, + #[serde(rename = "trainInterval")] + pub train_interval: Option, #[serde(rename = "learnHamFromReply")] pub learn_ham_from_reply: bool, } @@ -4966,7 +4297,6 @@ pub struct SpamClassifier { pub struct SpamClassifierFtrlCcfh { #[serde(rename = "indicatorParameters")] pub indicator_parameters: FtrlParameters, - #[serde(rename = "parameters")] pub parameters: FtrlParameters, #[serde(rename = "featureL2Normalize")] pub feature_l2_normalize: bool, @@ -4977,7 +4307,6 @@ pub struct SpamClassifierFtrlCcfh { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamClassifierFtrlFh { - #[serde(rename = "parameters")] pub parameters: FtrlParameters, #[serde(rename = "featureL2Normalize")] pub feature_l2_normalize: bool, @@ -4996,7 +4325,6 @@ pub enum SpamClassifierModel { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamClassify { - #[serde(rename = "message")] pub message: String, #[serde(rename = "remoteIp")] pub remote_ip: IpAddr, @@ -5006,26 +4334,21 @@ pub struct SpamClassify { pub authenticated_as: Option, #[serde(rename = "isTls")] pub is_tls: bool, - #[serde(rename = "envFrom")] - pub env_from: String, - #[serde(rename = "envFromParameters")] - pub env_from_parameters: Option, - #[serde(rename = "envRcptTo")] - pub env_rcpt_to: Map, - #[serde(rename = "score")] + #[serde(rename = "mailFrom")] + pub mail_from: String, + #[serde(rename = "mailFromParameters")] + pub mail_from_parameters: Option, + #[serde(rename = "rcptTo")] + pub rcpt_to: Map, pub score: Float, - #[serde(rename = "tags")] pub tags: VecMap, - #[serde(rename = "result")] pub result: SpamClassifyResult, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamClassifyTag { - #[serde(rename = "score")] pub score: Float, - #[serde(rename = "disposition")] pub disposition: SpamClassifyTagDisposition, } @@ -5044,105 +4367,70 @@ pub enum SpamDnsblServer { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerAny { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerBody { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerDomain { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerEmail { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerHeader { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerIp { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamDnsblServerUrl { - #[serde(rename = "tag")] pub tag: Expression, - #[serde(rename = "zone")] pub zone: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, } @@ -5162,7 +4450,6 @@ pub struct SpamDnsblSettings { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamFileExtension { - #[serde(rename = "extension")] pub extension: String, #[serde(rename = "isArchive")] pub is_archive: bool, @@ -5174,50 +4461,15 @@ pub struct SpamFileExtension { pub content_types: Map, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum SpamLlm { - Disable, - Enable(SpamLlmProperties), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct SpamLlmProperties { - #[serde(rename = "categories")] - pub categories: Map, - #[serde(rename = "confidence")] - pub confidence: Map, - #[serde(rename = "responsePosCategory")] - pub response_pos_category: u64, - #[serde(rename = "responsePosConfidence")] - pub response_pos_confidence: Option, - #[serde(rename = "responsePosExplanation")] - pub response_pos_explanation: Option, - #[serde(rename = "modelId")] - pub model_id: Id, - #[serde(rename = "prompt")] - pub prompt: String, - #[serde(rename = "separator")] - pub separator: String, - #[serde(rename = "temperature")] - pub temperature: Float, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamPyzor { #[serde(rename = "blockCount")] pub block_count: u64, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "host")] pub host: String, - #[serde(rename = "port")] pub port: u64, - #[serde(rename = "ratio")] pub ratio: Float, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "allowCount")] pub allow_count: u64, @@ -5238,105 +4490,70 @@ pub enum SpamRule { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleAny { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleBody { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleDomain { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleEmail { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleHeader { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleIp { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamRuleUrl { - #[serde(rename = "condition")] pub condition: Expression, - #[serde(rename = "name")] pub name: String, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "priority")] pub priority: i64, } @@ -5345,7 +4562,6 @@ pub struct SpamRuleUrl { pub struct SpamSettings { #[serde(rename = "trustContacts")] pub trust_contacts: bool, - #[serde(rename = "enable")] pub enable: bool, #[serde(rename = "greylistFor")] pub greylist_for: Option, @@ -5359,6 +4575,18 @@ pub struct SpamSettings { pub trust_replies: bool, #[serde(rename = "spamFilterRulesUrl")] pub spam_filter_rules_url: Option, + #[serde(rename = "maxBodyText")] + pub max_body_text: u64, + #[serde(rename = "maxAttachmentText")] + pub max_attachment_text: u64, + #[serde(rename = "maxMessageText")] + pub max_message_text: u64, + #[serde(rename = "maxRedirectUrls")] + pub max_redirect_urls: u64, + #[serde(rename = "redirectTimeout")] + pub redirect_timeout: Duration, + #[serde(rename = "redirectCacheTtl")] + pub redirect_cache_ttl: Duration, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -5372,25 +4600,20 @@ pub enum SpamTag { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamTagAction { - #[serde(rename = "tag")] pub tag: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamTagScore { - #[serde(rename = "tag")] pub tag: String, - #[serde(rename = "score")] pub score: Float, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SpamTrainingSample { - #[serde(rename = "from")] pub from: String, - #[serde(rename = "subject")] pub subject: String, #[serde(rename = "blobId")] pub blob_id: BlobId, @@ -5415,7 +4638,6 @@ pub struct SpfReportSettings { pub send_frequency: Expression, #[serde(rename = "dkimSignDomain")] pub dkim_sign_domain: Expression, - #[serde(rename = "subject")] pub subject: Expression, } @@ -5431,9 +4653,7 @@ pub enum SqlAuthStore { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SqlDirectory { - #[serde(rename = "description")] pub description: String, - #[serde(rename = "store")] pub store: SqlAuthStore, #[serde(rename = "columnEmail")] pub column_email: String, @@ -5451,27 +4671,22 @@ pub struct SqlDirectory { pub query_member_of: Option, #[serde(rename = "queryEmailAliases")] pub query_email_aliases: Option, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct SqliteStore { - #[serde(rename = "path")] pub path: String, #[serde(rename = "poolWorkers")] pub pool_workers: Option, #[serde(rename = "poolMaxConnections")] - pub pool_max_connections: u64, + pub pool_max_connections: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct StoreLookup { - #[serde(rename = "namespace")] pub namespace: String, - #[serde(rename = "store")] pub store: LookupStore, } @@ -5499,41 +4714,40 @@ pub struct SystemSettings { pub default_domain_id: Id, #[serde(rename = "defaultCertificateId")] pub default_certificate_id: Option, - #[serde(rename = "threadPoolSize")] - pub thread_pool_size: Option, #[serde(rename = "maxConnections")] pub max_connections: u64, + #[serde(rename = "maxBufferedMemory")] + pub max_buffered_memory: Option, #[serde(rename = "proxyTrustedNetworks")] pub proxy_trusted_networks: Map, #[serde(rename = "mailExchangers")] pub mail_exchangers: List, - #[serde(rename = "services")] pub services: VecMap, #[serde(rename = "providerInfo")] pub provider_info: VecMap, + #[serde(rename = "includeSecrets")] + pub include_secrets: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "@type")] pub enum Task { - IndexDocument(TaskIndexDocument), - UnindexDocument(TaskIndexDocument), - IndexTrace(TaskIndexTrace), CalendarAlarmEmail(TaskCalendarAlarmEmail), CalendarAlarmNotification(TaskCalendarAlarmNotification), CalendarItipMessage(TaskCalendarItipMessage), MergeThreads(TaskMergeThreads), DmarcReport(TaskDmarcReport), TlsReport(TaskTlsReport), - RestoreArchivedItem(TaskRestoreArchivedItem), DestroyAccount(TaskDestroyAccount), AccountMaintenance(TaskAccountMaintenance), - TenantMaintenance(TaskTenantMaintenance), StoreMaintenance(TaskStoreMaintenance), SpamFilterMaintenance(TaskSpamFilterMaintenance), AcmeRenewal(TaskDomainManagement), DkimManagement(TaskDomainManagement), DnsManagement(TaskDnsManagement), + ShareRevokeCleanup(TaskShareRevokeCleanup), + DmarcReportWindow(TaskReportWindow), + TlsReportWindow(TaskReportWindow), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -5543,7 +4757,6 @@ pub struct TaskAccountMaintenance { pub account_id: Id, #[serde(rename = "maintenanceType")] pub maintenance_type: TaskAccountMaintenanceType, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5562,11 +4775,14 @@ pub struct TaskCalendarAlarmEmail { pub event_start_tz: u64, #[serde(rename = "eventEndTz")] pub event_end_tz: u64, + #[serde(rename = "targetAccountId")] + pub target_account_id: Option, + #[serde(rename = "recurrenceId")] + pub recurrence_id: Option, #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "documentId")] pub document_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5579,39 +4795,35 @@ pub struct TaskCalendarAlarmNotification { pub event_id: u64, #[serde(rename = "recurrenceId")] pub recurrence_id: Option, + #[serde(rename = "targetAccountId")] + pub target_account_id: Option, #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "documentId")] pub document_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskCalendarItipContents { - #[serde(rename = "from")] pub from: String, - #[serde(rename = "to")] pub to: Map, #[serde(rename = "isFromOrganizer")] pub is_from_organizer: bool, #[serde(rename = "iCalendarData")] pub i_calendar_data: String, - #[serde(rename = "summary")] pub summary: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskCalendarItipMessage { - #[serde(rename = "messages")] pub messages: List, #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "documentId")] pub document_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5626,7 +4838,8 @@ pub struct TaskDestroyAccount { pub account_domain_id: Id, #[serde(rename = "accountType")] pub account_type: AccountType, - #[serde(rename = "status")] + #[serde(rename = "memberGroupIds")] + pub member_group_ids: Map, pub status: TaskStatus, } @@ -5635,7 +4848,6 @@ pub struct TaskDestroyAccount { pub struct TaskDmarcReport { #[serde(rename = "reportId")] pub report_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5648,7 +4860,6 @@ pub struct TaskDnsManagement { pub on_success_renew_certificate: bool, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5657,41 +4868,15 @@ pub struct TaskDnsManagement { pub struct TaskDomainManagement { #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "status")] - pub status: TaskStatus, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TaskIndexDocument { - #[serde(rename = "documentType")] - pub document_type: IndexDocumentType, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "documentId")] - pub document_id: Id, - #[serde(rename = "status")] - pub status: TaskStatus, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TaskIndexTrace { - #[serde(rename = "traceId")] - pub trace_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskManager { - #[serde(rename = "maxAttempts")] - pub max_attempts: u64, - #[serde(rename = "strategy")] pub strategy: TaskRetryStrategy, - #[serde(rename = "totalDeadline")] - pub total_deadline: Duration, + #[serde(rename = "alertAfter")] + pub alert_after: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -5703,24 +4888,14 @@ pub struct TaskMergeThreads { pub thread_name: String, #[serde(rename = "messageIds")] pub message_ids: Map, - #[serde(rename = "status")] pub status: TaskStatus, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] -pub struct TaskRestoreArchivedItem { - #[serde(rename = "blobId")] - pub blob_id: BlobId, - #[serde(rename = "archivedItemType")] - pub archived_item_type: ArchivedItemType, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "archivedUntil")] - pub archived_until: UTCDateTime, - #[serde(rename = "accountId")] - pub account_id: Id, - #[serde(rename = "status")] +pub struct TaskReportWindow { + #[serde(rename = "windowEnd")] + pub window_end: UTCDateTime, pub status: TaskStatus, } @@ -5734,29 +4909,37 @@ pub enum TaskRetryStrategy { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskRetryStrategyBackoff { - #[serde(rename = "factor")] pub factor: Float, #[serde(rename = "initialDelay")] pub initial_delay: Duration, #[serde(rename = "maxDelay")] pub max_delay: Duration, - #[serde(rename = "jitter")] pub jitter: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskRetryStrategyFixed { - #[serde(rename = "delay")] pub delay: Duration, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct TaskShareRevokeCleanup { + #[serde(rename = "objectType")] + pub object_type: SharedObjectType, + #[serde(rename = "accountId")] + pub account_id: Id, + #[serde(rename = "documentId")] + pub document_id: Id, + pub status: TaskStatus, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TaskSpamFilterMaintenance { #[serde(rename = "maintenanceType")] pub maintenance_type: TaskSpamFilterMaintenanceType, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5766,6 +4949,7 @@ pub enum TaskStatus { Pending(TaskStatusPending), Retry(TaskStatusRetry), Failed(TaskStatusFailed), + Stalled(TaskStatusRetry), } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -5786,7 +4970,6 @@ pub struct TaskStatusFailed { pub struct TaskStatusPending { #[serde(rename = "createdAt")] pub created_at: UTCDateTime, - #[serde(rename = "due")] pub due: UTCDateTime, } @@ -5795,10 +4978,11 @@ pub struct TaskStatusPending { pub struct TaskStatusRetry { #[serde(rename = "createdAt")] pub created_at: UTCDateTime, - #[serde(rename = "due")] pub due: UTCDateTime, #[serde(rename = "attemptNumber")] pub attempt_number: u64, + #[serde(rename = "firstFailedAt")] + pub first_failed_at: UTCDateTime, #[serde(rename = "failureReason")] pub failure_reason: String, } @@ -5810,18 +4994,6 @@ pub struct TaskStoreMaintenance { pub maintenance_type: TaskStoreMaintenanceType, #[serde(rename = "shardIndex")] pub shard_index: Option, - #[serde(rename = "status")] - pub status: TaskStatus, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TaskTenantMaintenance { - #[serde(rename = "tenantId")] - pub tenant_id: Id, - #[serde(rename = "maintenanceType")] - pub maintenance_type: TaskTenantMaintenanceType, - #[serde(rename = "status")] pub status: TaskStatus, } @@ -5830,44 +5002,20 @@ pub struct TaskTenantMaintenance { pub struct TaskTlsReport { #[serde(rename = "reportId")] pub report_id: Id, - #[serde(rename = "status")] pub status: TaskStatus, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct Tenant { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "createdAt")] - pub created_at: UTCDateTime, - #[serde(rename = "logo")] - pub logo: Option, - #[serde(rename = "roles")] - pub roles: Roles, - #[serde(rename = "permissions")] - pub permissions: Permissions, - #[serde(rename = "quotas")] - pub quotas: VecMap, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TlsExternalReport { - #[serde(rename = "report")] pub report: TlsReport, - #[serde(rename = "from")] pub from: String, - #[serde(rename = "subject")] pub subject: String, - #[serde(rename = "to")] pub to: Map, #[serde(rename = "receivedAt")] pub received_at: UTCDateTime, #[serde(rename = "expiresAt")] pub expires_at: UTCDateTime, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -5900,9 +5048,7 @@ pub struct TlsInternalReport { pub mail_rua: Map, #[serde(rename = "httpRua")] pub http_rua: Map, - #[serde(rename = "report")] pub report: TlsReport, - #[serde(rename = "domain")] pub domain: String, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, @@ -5923,7 +5069,6 @@ pub struct TlsReport { pub date_range_start: UTCDateTime, #[serde(rename = "dateRangeEnd")] pub date_range_end: UTCDateTime, - #[serde(rename = "policies")] pub policies: List, } @@ -5963,123 +5108,6 @@ pub struct TlsReportSettings { pub send_frequency: Expression, #[serde(rename = "dkimSignDomain")] pub dkim_sign_domain: Expression, - #[serde(rename = "subject")] - pub subject: Expression, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct Trace { - #[serde(rename = "events")] - pub events: List, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceEvent { - #[serde(rename = "event")] - pub event: trc::EventType, - #[serde(rename = "timestamp")] - pub timestamp: UTCDateTime, - #[serde(rename = "keyValues")] - pub key_values: List, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceKeyValue { - #[serde(rename = "key")] - pub key: trc::Key, - #[serde(rename = "value")] - pub value: TraceValue, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum TraceValue { - String(TraceValueString), - UnsignedInt(TraceValueUnsignedInt), - Integer(TraceValueInteger), - Boolean(TraceValueBoolean), - Float(TraceValueFloat), - UTCDateTime(TraceValueUTCDateTime), - Duration(TraceValueDuration), - IpAddr(TraceValueIpAddr), - List(TraceValueList), - Event(TraceValueEvent), - Null, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueBoolean { - #[serde(rename = "value")] - pub value: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueDuration { - #[serde(rename = "value")] - pub value: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueEvent { - #[serde(rename = "event")] - pub event: trc::EventType, - #[serde(rename = "value")] - pub value: List, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueFloat { - #[serde(rename = "value")] - pub value: Float, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueInteger { - #[serde(rename = "value")] - pub value: i64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueIpAddr { - #[serde(rename = "value")] - pub value: IpAddr, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueList { - #[serde(rename = "value")] - pub value: List, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueString { - #[serde(rename = "value")] - pub value: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueUTCDateTime { - #[serde(rename = "value")] - pub value: UTCDateTime, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct TraceValueUnsignedInt { - #[serde(rename = "value")] - pub value: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -6095,13 +5123,9 @@ pub enum Tracer { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TracerCommon { - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "lossy")] pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, @@ -6110,23 +5134,14 @@ pub struct TracerCommon { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TracerLog { - #[serde(rename = "path")] pub path: String, - #[serde(rename = "prefix")] pub prefix: String, - #[serde(rename = "rotate")] pub rotate: LogRotateFrequency, - #[serde(rename = "ansi")] pub ansi: bool, - #[serde(rename = "multiline")] pub multiline: bool, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "lossy")] pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, @@ -6135,27 +5150,20 @@ pub struct TracerLog { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TracerOtelGrpc { - #[serde(rename = "endpoint")] - pub endpoint: Option, + pub url: Option, #[serde(rename = "enableLogExporter")] pub enable_log_exporter: bool, #[serde(rename = "enableSpanExporter")] pub enable_span_exporter: bool, - #[serde(rename = "throttle")] pub throttle: Duration, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] pub http_headers: VecMap, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "lossy")] pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, @@ -6164,27 +5172,22 @@ pub struct TracerOtelGrpc { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TracerOtelHttp { - #[serde(rename = "endpoint")] - pub endpoint: String, + pub url: String, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "enableLogExporter")] pub enable_log_exporter: bool, #[serde(rename = "enableSpanExporter")] pub enable_span_exporter: bool, - #[serde(rename = "throttle")] pub throttle: Duration, - #[serde(rename = "timeout")] pub timeout: Duration, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] pub http_headers: VecMap, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "lossy")] pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, @@ -6193,62 +5196,33 @@ pub struct TracerOtelHttp { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct TracerStdout { - #[serde(rename = "buffered")] pub buffered: bool, - #[serde(rename = "ansi")] pub ansi: bool, - #[serde(rename = "multiline")] pub multiline: bool, - #[serde(rename = "enable")] pub enable: bool, - #[serde(rename = "level")] pub level: TracingLevel, - #[serde(rename = "lossy")] pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "@type")] -pub enum TracingStore { - Disabled, - Default, - FoundationDb(FoundationDbStore), - PostgreSql(PostgreSqlStore), - MySql(MySqlStore), -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct UserAccount { - #[serde(rename = "name")] pub name: String, #[serde(rename = "domainId")] pub domain_id: Id, - #[serde(rename = "credentials")] pub credentials: List, #[serde(rename = "createdAt")] pub created_at: UTCDateTime, #[serde(rename = "memberGroupIds")] pub member_group_ids: Map, - #[serde(rename = "memberTenantId")] - pub member_tenant_id: Option, - #[serde(rename = "roles")] pub roles: UserRoles, - #[serde(rename = "permissions")] pub permissions: Permissions, - #[serde(rename = "quotas")] pub quotas: VecMap, - #[serde(rename = "aliases")] pub aliases: List, - #[serde(rename = "externalId")] - pub external_id: Option, - #[serde(rename = "description")] pub description: Option, - #[serde(rename = "locale")] pub locale: Locale, #[serde(rename = "timeZone")] pub time_zone: Option, @@ -6273,12 +5247,10 @@ pub struct WebDav { pub max_lock_timeout: Duration, #[serde(rename = "maxLocks")] pub max_locks: u64, - #[serde(rename = "deadPropertyMaxSize")] - pub dead_property_max_size: Option, - #[serde(rename = "livePropertyMaxSize")] - pub live_property_max_size: u64, - #[serde(rename = "requestMaxSize")] - pub request_max_size: u64, + #[serde(rename = "maxLivePropertySize")] + pub max_live_property_size: u64, + #[serde(rename = "maxRequestSize")] + pub max_request_size: u64, #[serde(rename = "maxResults")] pub max_results: u64, } @@ -6286,29 +5258,22 @@ pub struct WebDav { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct WebHook { - #[serde(rename = "allowInvalidCerts")] - pub allow_invalid_certs: bool, + pub enable: bool, + pub level: TracingLevel, + pub lossy: bool, #[serde(rename = "signatureKey")] pub signature_key: SecretKeyOptional, - #[serde(rename = "throttle")] pub throttle: Duration, - #[serde(rename = "timeout")] - pub timeout: Duration, #[serde(rename = "discardAfter")] pub discard_after: Duration, - #[serde(rename = "url")] pub url: String, + pub timeout: Duration, + #[serde(rename = "allowInvalidCerts")] + pub allow_invalid_certs: bool, #[serde(rename = "httpAuth")] pub http_auth: HttpAuth, #[serde(rename = "httpHeaders")] pub http_headers: VecMap, - #[serde(rename = "enable")] - pub enable: bool, - #[serde(rename = "level")] - pub level: TracingLevel, - #[serde(rename = "lossy")] - pub lossy: bool, - #[serde(rename = "events")] pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, @@ -6317,6 +5282,5 @@ pub struct WebHook { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct ZenohCoordinator { - #[serde(rename = "config")] pub config: String, } diff --git a/crates/registry/src/schema/structs_impl.rs b/crates/registry/src/schema/structs_impl.rs index ea5ab256..79cf2653 100644 --- a/crates/registry/src/schema/structs_impl.rs +++ b/crates/registry/src/schema/structs_impl.rs @@ -10,7 +10,7 @@ use crate::schema::prelude::*; impl ObjectImpl for Account { const FLAGS: u64 = OBJ_FILTER_TENANT | OBJ_SEQ_ID; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Account; fn validate(&self, errors: &mut Vec) -> bool { @@ -55,6 +55,9 @@ impl Pickle for Account { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(Account::User), 1 => Pickle::unpickle(stream).map(Account::Group), @@ -65,16 +68,20 @@ impl Pickle for Account { impl IntoValue for Account { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Account::User(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("User".into())); obj } Account::Group(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Group".into())); @@ -113,7 +120,7 @@ impl Account { } impl ObjectImpl for AccountPassword { - const FLAGS: u64 = OBJ_SINGLETON; + const FLAGS: u64 = OBJ_SINGLETON | OBJ_NOT_STORED; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::AccountPassword; @@ -137,22 +144,6 @@ impl ObjectImpl for AccountPassword { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for AccountPassword { - fn pickle(&self, out: &mut Vec) { - self.secret.pickle(out); - self.current_secret.pickle(out); - self.otp_auth.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.secret = Pickle::unpickle(stream)?; - this.current_secret = Pickle::unpickle(stream)?; - this.otp_auth = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for AccountPassword { fn default() -> Self { Self { @@ -165,14 +156,32 @@ impl Default for AccountPassword { impl IntoValue for AccountPassword { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); if self.secret.is_some() { - map.insert_unchecked(Property::Secret, secret_value(self.secret)); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } if self.current_secret.is_some() { - map.insert_unchecked(Property::CurrentSecret, secret_value(self.current_secret)); + map.insert_unchecked( + Property::CurrentSecret, + if opts.include_secrets { + self.current_secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } - map.insert_unchecked(Property::OtpAuth, self.otp_auth.into_value()); + map.insert_unchecked(Property::OtpAuth, self.otp_auth.into_value_with(opts)); JmapValue::Object(map) } } @@ -197,7 +206,7 @@ impl RegistryJsonPropertyPatch for AccountPassword { } impl ObjectImpl for AccountSettings { - const FLAGS: u64 = OBJ_SINGLETON; + const FLAGS: u64 = OBJ_SINGLETON | OBJ_NOT_STORED; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::AccountSettings; @@ -221,24 +230,6 @@ impl ObjectImpl for AccountSettings { } } -impl Pickle for AccountSettings { - fn pickle(&self, out: &mut Vec) { - self.description.pickle(out); - self.locale.pickle(out); - self.time_zone.pickle(out); - self.encryption_at_rest.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.locale = Pickle::unpickle(stream)?; - this.time_zone = Pickle::unpickle(stream)?; - this.encryption_at_rest = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for AccountSettings { fn default() -> Self { Self { @@ -252,13 +243,20 @@ impl Default for AccountSettings { impl IntoValue for AccountSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Locale, self.locale.into_value()); - map.insert_unchecked(Property::TimeZone, self.time_zone.into_value()); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Locale, self.locale.into_value_with(opts)); + map.insert_unchecked(Property::TimeZone, self.time_zone.into_value_with(opts)); map.insert_unchecked( Property::EncryptionAtRest, - self.encryption_at_rest.into_value(), + self.encryption_at_rest.into_value_with(opts), ); JmapValue::Object(map) } @@ -286,7 +284,7 @@ impl RegistryJsonPropertyPatch for AccountSettings { impl ObjectImpl for AcmeProvider { const FLAGS: u64 = OBJ_FILTER_TENANT; - const VERSION: u8 = 2; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::AcmeProvider; fn validate(&self, errors: &mut Vec) -> bool { @@ -312,9 +310,21 @@ impl ObjectImpl for AcmeProvider { if value.is_empty() { errors.push(ValidationError::required(Property::AccountUri)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); + if let Some(value) = &self.account_hash_prefix { + if value.is_empty() { + errors.push(ValidationError::required(Property::AccountHashPrefix)); + } + } + let value = &self.issuer_domain_names; + for value in value.iter() { + if value.is_empty() { + errors.push(ValidationError::required(Property::IssuerDomainNames)); + } + } + let value = &self.caa_identities; + for value in value.iter() { + if value.is_empty() { + errors.push(ValidationError::required(Property::CaaIdentities)); } } if let Some(value) = &self.preferred_chain { @@ -322,6 +332,10 @@ impl ObjectImpl for AcmeProvider { errors.push(ValidationError::required(Property::PreferredChain)); } } + let value = &self.description; + if value.is_empty() { + errors.push(ValidationError::required(Property::Description)); + } errors.len() == neb } @@ -330,44 +344,125 @@ impl ObjectImpl for AcmeProvider { i.text(Property::Text, value); } i.text(Property::Text, &self.directory); - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } + i.text(Property::Text, &self.description); } } impl Pickle for AcmeProvider { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.account_hash_prefix.is_some() { + bitmap |= 1 << 0; + } + if self.preferred_chain.is_some() { + bitmap |= 1 << 2; + } + if self.reuse_key { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 13, 4, bitmap); self.challenge_type.pickle(out); self.contact.pickle(out); self.directory.pickle(out); self.account_key.pickle(out); self.account_uri.pickle(out); + if let Some(value) = &self.account_hash_prefix { + value.pickle(out); + } + self.issuer_domain_names.pickle(out); + self.caa_identities.pickle(out); self.renew_before.pickle(out); - self.max_retries.pickle(out); - self.member_tenant_id.pickle(out); - self.preferred_chain.pickle(out); - self.reuse_key.pickle(out); + if let Some(value) = &self.preferred_chain { + value.pickle(out); + } + self.description.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.challenge_type = Pickle::unpickle(stream)?; - this.contact = Pickle::unpickle(stream)?; - this.directory = Pickle::unpickle(stream)?; - this.account_key = Pickle::unpickle(stream)?; - this.account_uri = Pickle::unpickle(stream)?; - this.renew_before = Pickle::unpickle(stream)?; - this.max_retries = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.preferred_chain = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - if stream.version() >= 2 { - this.reuse_key = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: AcmeChallengeType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + AcmeChallengeType::TlsAlpn01 + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + "https://acme-v02.api.letsencrypt.org/directory".to_string() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Map = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Map = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: AcmeRenewBefore = if block.has(8) { + Pickle::unpickle(stream)? + } else { + AcmeRenewBefore::R23 + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field11: bool = if block.has(11) { block.flag(3) } else { false }; + let field12: String = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + challenge_type: field0, + contact: field1, + directory: field2, + account_key: field3, + account_uri: field4, + account_hash_prefix: field5, + issuer_domain_names: field6, + caa_identities: field7, + renew_before: field8, + preferred_chain: field10, + reuse_key: field11, + description: field12, + }) } } @@ -379,28 +474,64 @@ impl Default for AcmeProvider { directory: "https://acme-v02.api.letsencrypt.org/directory".to_string(), account_key: Default::default(), account_uri: Default::default(), + account_hash_prefix: Default::default(), + issuer_domain_names: Default::default(), + caa_identities: Default::default(), renew_before: AcmeRenewBefore::R23, - max_retries: 10i64, - member_tenant_id: Default::default(), preferred_chain: Default::default(), reuse_key: false, + description: Default::default(), } } } impl IntoValue for AcmeProvider { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::ChallengeType, self.challenge_type.into_value()); - map.insert_unchecked(Property::Contact, self.contact.into_value()); - map.insert_unchecked(Property::Directory, self.directory.into_value()); - map.insert_unchecked(Property::AccountKey, secret_value(self.account_key)); - map.insert_unchecked(Property::AccountUri, self.account_uri.into_value()); - map.insert_unchecked(Property::RenewBefore, self.renew_before.into_value()); - map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::PreferredChain, self.preferred_chain.into_value()); - map.insert_unchecked(Property::ReuseKey, self.reuse_key.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(15); + map.insert_unchecked( + Property::ChallengeType, + self.challenge_type.into_value_with(opts), + ); + map.insert_unchecked(Property::Contact, self.contact.into_value_with(opts)); + map.insert_unchecked(Property::Directory, self.directory.into_value_with(opts)); + map.insert_unchecked( + Property::AccountKey, + if opts.include_secrets { + self.account_key.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); + map.insert_unchecked(Property::AccountUri, self.account_uri.into_value_with(opts)); + map.insert_unchecked( + Property::AccountHashPrefix, + self.account_hash_prefix.into_value_with(opts), + ); + map.insert_unchecked( + Property::IssuerDomainNames, + self.issuer_domain_names.into_value_with(opts), + ); + map.insert_unchecked( + Property::CaaIdentities, + self.caa_identities.into_value_with(opts), + ); + map.insert_unchecked( + Property::RenewBefore, + self.renew_before.into_value_with(opts), + ); + map.insert_unchecked( + Property::PreferredChain, + self.preferred_chain.into_value_with(opts), + ); + map.insert_unchecked(Property::ReuseKey, self.reuse_key.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -430,16 +561,17 @@ impl RegistryJsonPropertyPatch for AcmeProvider { } Some(Property::AccountKey) => pointer.assert_server_set(), Some(Property::AccountUri) => pointer.assert_server_set(), + Some(Property::AccountHashPrefix) => pointer.assert_server_set(), + Some(Property::IssuerDomainNames) => pointer.assert_server_set(), + Some(Property::CaaIdentities) => pointer.assert_server_set(), Some(Property::RenewBefore) => self.renew_before.patch(pointer, value), - Some(Property::MaxRetries) => self.max_retries.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::PreferredChain) => self .preferred_chain .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::ReuseKey) => self.reuse_key.patch(pointer, value), - Some(Property::Description) => pointer.assert_server_set(), + Some(Property::Description) => self + .description + .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -450,13 +582,13 @@ impl RegistryJsonPropertyPatch for AcmeProvider { } impl ObjectImpl for Action { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED | OBJ_NO_GET | OBJ_NO_UPDATE | OBJ_NO_DESTROY | OBJ_NO_QUERY; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Action; fn validate(&self, errors: &mut Vec) -> bool { match self { - Action::ReloadSettings => true, + Action::ReloadSettings(inner) => inner.validate(errors), Action::ReloadTlsCertificates => true, Action::ReloadLookupStores => true, Action::ReloadBlockedIps => true, @@ -475,76 +607,23 @@ impl ObjectImpl for Action { impl Default for Action { fn default() -> Self { - Action::ReloadSettings - } -} - -impl Pickle for Action { - fn pickle(&self, out: &mut Vec) { - match self { - Action::ReloadSettings => { - 0u16.pickle(out); - } - Action::ReloadTlsCertificates => { - 1u16.pickle(out); - } - Action::ReloadLookupStores => { - 2u16.pickle(out); - } - Action::ReloadBlockedIps => { - 3u16.pickle(out); - } - Action::UpdateApps => { - 4u16.pickle(out); - } - Action::TroubleshootDmarc(inner) => { - 5u16.pickle(out); - inner.pickle(out); - } - Action::ClassifySpam(inner) => { - 6u16.pickle(out); - inner.pickle(out); - } - Action::InvalidateCaches => { - 7u16.pickle(out); - } - Action::InvalidateNegativeCaches => { - 8u16.pickle(out); - } - Action::PauseMtaQueue => { - 9u16.pickle(out); - } - Action::ResumeMtaQueue => { - 10u16.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(Action::ReloadSettings), - 1 => Some(Action::ReloadTlsCertificates), - 2 => Some(Action::ReloadLookupStores), - 3 => Some(Action::ReloadBlockedIps), - 4 => Some(Action::UpdateApps), - 5 => Pickle::unpickle(stream).map(Action::TroubleshootDmarc), - 6 => Pickle::unpickle(stream).map(Action::ClassifySpam), - 7 => Some(Action::InvalidateCaches), - 8 => Some(Action::InvalidateNegativeCaches), - 9 => Some(Action::PauseMtaQueue), - 10 => Some(Action::ResumeMtaQueue), - _ => None, - } + Action::ReloadSettings(Default::default()) } } impl IntoValue for Action { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { - Action::ReloadSettings => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("ReloadSettings".into())); - JmapValue::Object(obj) + Action::ReloadSettings(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("ReloadSettings".into())); + obj } Action::ReloadTlsCertificates => { let mut obj = jmap_tools::Map::new(); @@ -570,14 +649,14 @@ impl IntoValue for Action { JmapValue::Object(obj) } Action::TroubleshootDmarc(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TroubleshootDmarc".into())); obj } Action::ClassifySpam(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ClassifySpam".into())); @@ -618,7 +697,7 @@ impl RegistryJsonPatch for Action { ) -> PatchResult<'x> { if !pointer.has_next() { match object_type(&pointer, &value)? { - ActionType::ReloadSettings => *self = Action::ReloadSettings, + ActionType::ReloadSettings => *self = Action::ReloadSettings(Default::default()), ActionType::ReloadTlsCertificates => *self = Action::ReloadTlsCertificates, ActionType::ReloadLookupStores => *self = Action::ReloadLookupStores, ActionType::ReloadBlockedIps => *self = Action::ReloadBlockedIps, @@ -634,7 +713,7 @@ impl RegistryJsonPatch for Action { } } match self { - Action::ReloadSettings => pointer.assert_eof(), + Action::ReloadSettings(inner) => inner.patch(pointer, value), Action::ReloadTlsCertificates => pointer.assert_eof(), Action::ReloadLookupStores => pointer.assert_eof(), Action::ReloadBlockedIps => pointer.assert_eof(), @@ -652,7 +731,7 @@ impl RegistryJsonPatch for Action { impl Action { pub fn object_type(&self) -> ActionType { match self { - Action::ReloadSettings => ActionType::ReloadSettings, + Action::ReloadSettings(_) => ActionType::ReloadSettings, Action::ReloadTlsCertificates => ActionType::ReloadTlsCertificates, Action::ReloadLookupStores => ActionType::ReloadLookupStores, Action::ReloadBlockedIps => ActionType::ReloadBlockedIps, @@ -669,7 +748,7 @@ impl Action { impl ObjectImpl for AddressBook { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::AddressBook; fn validate(&self, errors: &mut Vec) -> bool { @@ -694,6 +773,13 @@ impl ObjectImpl for AddressBook { errors.push(ValidationError::min_value(Property::MaxContacts, 1)); } } + let value = &self.max_address_books_per_card; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxAddressBooksPerCard, + 1, + )); + } errors.len() == neb } @@ -702,25 +788,111 @@ impl ObjectImpl for AddressBook { impl Pickle for AddressBook { fn pickle(&self, out: &mut Vec) { - self.default_display_name.pickle(out); - self.default_href_name.pickle(out); + let mut bitmap = 0u64; + if self.default_display_name.is_some() { + bitmap |= 1 << 0; + } + if self.default_href_name.is_some() { + bitmap |= 1 << 1; + } + if self.max_address_books.is_some() { + bitmap |= 1 << 2; + } + if self.max_contacts.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 8, 4, bitmap); + if let Some(value) = &self.default_display_name { + value.pickle(out); + } + if let Some(value) = &self.default_href_name { + value.pickle(out); + } self.max_v_card_size.pickle(out); - self.max_address_books.pickle(out); - self.max_contacts.pickle(out); + if let Some(value) = &self.max_address_books { + value.pickle(out); + } + if let Some(value) = &self.max_contacts { + value.pickle(out); + } self.v_card_version.pickle(out); + self.max_address_books_per_card.pickle(out); + self.max_media_size.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.default_display_name = Pickle::unpickle(stream)?; - this.default_href_name = Pickle::unpickle(stream)?; - this.max_v_card_size = Pickle::unpickle(stream)?; - this.max_address_books = Pickle::unpickle(stream)?; - this.max_contacts = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.v_card_version = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("Stalwart Address Book".to_string()) + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("default".to_string()) + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 524288u64 + }; + let field3: Option = if block.has(3) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(250u64) + }; + let field4: Option = if block.has(4) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: VCardVersion = if block.has(5) { + Pickle::unpickle(stream)? + } else { + VCardVersion::V4 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + stream.close_block(block); + Some(Self { + default_display_name: field0, + default_href_name: field1, + max_v_card_size: field2, + max_address_books: field3, + max_contacts: field4, + v_card_version: field5, + max_address_books_per_card: field6, + max_media_size: field7, + }) } } @@ -733,28 +905,51 @@ impl Default for AddressBook { max_address_books: Some(250u64), max_contacts: Default::default(), v_card_version: VCardVersion::V4, + max_address_books_per_card: 10u64, + max_media_size: 2097152, } } } impl IntoValue for AddressBook { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(10); map.insert_unchecked( Property::DefaultDisplayName, - self.default_display_name.into_value(), + self.default_display_name.into_value_with(opts), ); map.insert_unchecked( Property::DefaultHrefName, - self.default_href_name.into_value(), + self.default_href_name.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxVCardSize, + self.max_v_card_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxVCardSize, self.max_v_card_size.into_value()); map.insert_unchecked( Property::MaxAddressBooks, - self.max_address_books.into_value(), + self.max_address_books.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxContacts, + self.max_contacts.into_value_with(opts), + ); + map.insert_unchecked( + Property::VCardVersion, + self.v_card_version.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAddressBooksPerCard, + self.max_address_books_per_card.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMediaSize, + self.max_media_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxContacts, self.max_contacts.into_value()); - map.insert_unchecked(Property::VCardVersion, self.v_card_version.into_value()); JmapValue::Object(map) } } @@ -776,564 +971,10 @@ impl RegistryJsonPropertyPatch for AddressBook { Some(Property::MaxAddressBooks) => self.max_address_books.patch(pointer, value), Some(Property::MaxContacts) => self.max_contacts.patch(pointer, value), Some(Property::VCardVersion) => self.v_card_version.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ObjectImpl for AiModel { - const FLAGS: u64 = 0; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::AiModel; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.name; - if value.is_empty() { - errors.push(ValidationError::required(Property::Name)); - } - let value = &self.temperature; - if *value > Float::new(1.0) { - errors.push(ValidationError::max_value(Property::Temperature, 1)); - } - if *value < Float::new(0.0) { - errors.push(ValidationError::min_value(Property::Temperature, 0)); - } - let value = &self.model; - if value.is_empty() { - errors.push(ValidationError::required(Property::Model)); - } - let value = &self.url; - if value.is_empty() { - errors.push(ValidationError::required(Property::Url)); - } - let value = &self.http_auth; - value.validate(errors); - let value = &self.http_headers; - for value in value.values() { - if value.is_empty() { - errors.push(ValidationError::required(Property::HttpHeaders)); + Some(Property::MaxAddressBooksPerCard) => { + self.max_address_books_per_card.patch(pointer, value) } - } - errors.len() == neb - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Pickle for AiModel { - fn pickle(&self, out: &mut Vec) { - self.name.pickle(out); - self.allow_invalid_certs.pickle(out); - self.temperature.pickle(out); - self.model.pickle(out); - self.timeout.pickle(out); - self.model_type.pickle(out); - self.url.pickle(out); - self.http_auth.pickle(out); - self.http_headers.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.temperature = Pickle::unpickle(stream)?; - this.model = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.model_type = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for AiModel { - fn default() -> Self { - Self { - name: Default::default(), - allow_invalid_certs: false, - temperature: Float::new(0.7f64), - model: Default::default(), - timeout: Duration::from_millis(120000), - model_type: AiModelType::Chat, - url: Default::default(), - http_auth: Default::default(), - http_headers: Default::default(), - } - } -} - -impl IntoValue for AiModel { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked( - Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), - ); - map.insert_unchecked(Property::Temperature, self.temperature.into_value()); - map.insert_unchecked(Property::Model, self.model.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::ModelType, self.model_type.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for AiModel { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Name) => self - .name - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), - Some(Property::Temperature) => self.temperature.patch(pointer, value), - Some(Property::Model) => self - .model - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), - Some(Property::ModelType) => self.model_type.patch(pointer, value), - Some(Property::Url) => self - .url - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), - Some(Property::HttpHeaders) => self - .http_headers - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ObjectImpl for Alert { - const FLAGS: u64 = 0; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::Alert; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.condition; - value.validate(errors); - let value = &self.email_alert; - value.validate(errors); - let value = &self.event_alert; - value.validate(errors); - errors.len() == neb - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Alert { - pub fn ctx_condition(&self) -> ExpressionContext<'_> { - ExpressionContext { - expr: &self.condition, - default: None, - property: Property::Condition, - allowed_variables: &[], - allowed_constants: &[], - } - } - - pub fn expression_ctxs(&self) -> Vec> { - vec![self.ctx_condition()] - } -} - -impl Pickle for Alert { - fn pickle(&self, out: &mut Vec) { - self.condition.pickle(out); - self.email_alert.pickle(out); - self.event_alert.pickle(out); - self.enable.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.email_alert = Pickle::unpickle(stream)?; - this.event_alert = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for Alert { - fn default() -> Self { - Self { - condition: Default::default(), - email_alert: Default::default(), - event_alert: Default::default(), - enable: true, - } - } -} - -impl IntoValue for Alert { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::EmailAlert, self.email_alert.into_value()); - map.insert_unchecked(Property::EventAlert, self.event_alert.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for Alert { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Condition) => self.condition.patch(pointer, value), - Some(Property::EmailAlert) => self.email_alert.patch(pointer, value), - Some(Property::EventAlert) => self.event_alert.patch(pointer, value), - Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl AlertEmail { - fn validate(&self, errors: &mut Vec) -> bool { - match self { - AlertEmail::Disabled => true, - AlertEmail::Enabled(inner) => inner.validate(errors), - } - } -} - -impl Default for AlertEmail { - fn default() -> Self { - AlertEmail::Disabled - } -} - -impl Pickle for AlertEmail { - fn pickle(&self, out: &mut Vec) { - match self { - AlertEmail::Disabled => { - 0u16.pickle(out); - } - AlertEmail::Enabled(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(AlertEmail::Disabled), - 1 => Pickle::unpickle(stream).map(AlertEmail::Enabled), - _ => None, - } - } -} - -impl IntoValue for AlertEmail { - fn into_value(self) -> JmapValue<'static> { - match self { - AlertEmail::Disabled => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Disabled".into())); - JmapValue::Object(obj) - } - AlertEmail::Enabled(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Enabled".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for AlertEmail { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - AlertEmailType::Disabled => *self = AlertEmail::Disabled, - AlertEmailType::Enabled => *self = AlertEmail::Enabled(Default::default()), - } - } - match self { - AlertEmail::Disabled => pointer.assert_eof(), - AlertEmail::Enabled(inner) => inner.patch(pointer, value), - } - } -} - -impl AlertEmail { - pub fn object_type(&self) -> AlertEmailType { - match self { - AlertEmail::Disabled => AlertEmailType::Disabled, - AlertEmail::Enabled(_) => AlertEmailType::Enabled, - } - } -} - -impl AlertEmailProperties { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.body; - if value.is_empty() { - errors.push(ValidationError::required(Property::Body)); - } - let value = &self.from_address; - if value.is_empty() { - errors.push(ValidationError::required(Property::FromAddress)); - } - if let Some(value) = &self.from_name { - if value.is_empty() { - errors.push(ValidationError::required(Property::FromName)); - } - } - let value = &self.subject; - if value.is_empty() { - errors.push(ValidationError::required(Property::Subject)); - } - let value = &self.to; - for value in value.iter() { - if value.is_empty() { - errors.push(ValidationError::required(Property::To)); - } - } - if value.len() < 1 { - errors.push(ValidationError::min_items(Property::To, 1)); - } - errors.len() == neb - } -} - -impl Pickle for AlertEmailProperties { - fn pickle(&self, out: &mut Vec) { - self.body.pickle(out); - self.from_address.pickle(out); - self.from_name.pickle(out); - self.subject.pickle(out); - self.to.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.body = Pickle::unpickle(stream)?; - this.from_address = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for AlertEmailProperties { - fn default() -> Self { - Self { - body: Default::default(), - from_address: Default::default(), - from_name: Default::default(), - subject: Default::default(), - to: Default::default(), - } - } -} - -impl IntoValue for AlertEmailProperties { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Body, self.body.into_value()); - map.insert_unchecked(Property::FromAddress, self.from_address.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for AlertEmailProperties { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Body) => self.body.patch(pointer, value), - Some(Property::FromAddress) => self - .from_address - .patch(pointer.with_validators(&[StringValidator::Email]), value), - Some(Property::FromName) => self.from_name.patch(pointer, value), - Some(Property::Subject) => self.subject.patch(pointer, value), - Some(Property::To) => self - .to - .patch(pointer.with_validators(&[StringValidator::Email]), value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl AlertEvent { - fn validate(&self, errors: &mut Vec) -> bool { - match self { - AlertEvent::Disabled => true, - AlertEvent::Enabled(inner) => inner.validate(errors), - } - } -} - -impl Default for AlertEvent { - fn default() -> Self { - AlertEvent::Disabled - } -} - -impl Pickle for AlertEvent { - fn pickle(&self, out: &mut Vec) { - match self { - AlertEvent::Disabled => { - 0u16.pickle(out); - } - AlertEvent::Enabled(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(AlertEvent::Disabled), - 1 => Pickle::unpickle(stream).map(AlertEvent::Enabled), - _ => None, - } - } -} - -impl IntoValue for AlertEvent { - fn into_value(self) -> JmapValue<'static> { - match self { - AlertEvent::Disabled => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Disabled".into())); - JmapValue::Object(obj) - } - AlertEvent::Enabled(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Enabled".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for AlertEvent { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - AlertEventType::Disabled => *self = AlertEvent::Disabled, - AlertEventType::Enabled => *self = AlertEvent::Enabled(Default::default()), - } - } - match self { - AlertEvent::Disabled => pointer.assert_eof(), - AlertEvent::Enabled(inner) => inner.patch(pointer, value), - } - } -} - -impl AlertEvent { - pub fn object_type(&self) -> AlertEventType { - match self { - AlertEvent::Disabled => AlertEventType::Disabled, - AlertEvent::Enabled(_) => AlertEventType::Enabled, - } - } -} - -impl AlertEventProperties { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - if let Some(value) = &self.event_message { - if value.is_empty() { - errors.push(ValidationError::required(Property::EventMessage)); - } - } - errors.len() == neb - } -} - -impl Pickle for AlertEventProperties { - fn pickle(&self, out: &mut Vec) { - self.event_message.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.event_message = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for AlertEventProperties { - fn default() -> Self { - Self { - event_message: Default::default(), - } - } -} - -impl IntoValue for AlertEventProperties { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::EventMessage, self.event_message.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for AlertEventProperties { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::EventMessage) => self.event_message.patch(pointer, value), + Some(Property::MaxMediaSize) => self.max_media_size.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -1378,19 +1019,65 @@ impl ObjectImpl for AllowedIp { impl Pickle for AllowedIp { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.reason.is_some() { + bitmap |= 1 << 0; + } + if self.expires_at.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.address.pickle(out); - self.reason.pickle(out); + if let Some(value) = &self.reason { + value.pickle(out); + } self.created_at.pickle(out); - self.expires_at.pickle(out); + if let Some(value) = &self.expires_at { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.address = Pickle::unpickle(stream)?; - this.reason = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: IpAddrOrMask = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: UTCDateTime = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + address: field0, + reason: field1, + created_at: field2, + expires_at: field3, + }) } } @@ -1407,11 +1094,15 @@ impl Default for AllowedIp { impl IntoValue for AllowedIp { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Address, self.address.into_value()); - map.insert_unchecked(Property::Reason, self.reason.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); + map.insert_unchecked(Property::Address, self.address.into_value_with(opts)); + map.insert_unchecked(Property::Reason, self.reason.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -1444,7 +1135,7 @@ impl RegistryJsonPropertyPatch for AllowedIp { } impl ObjectImpl for ApiKey { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::ApiKey; @@ -1481,28 +1172,6 @@ impl ObjectImpl for ApiKey { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for ApiKey { - fn pickle(&self, out: &mut Vec) { - self.description.pickle(out); - self.secret.pickle(out); - self.created_at.pickle(out); - self.expires_at.pickle(out); - self.permissions.pickle(out); - self.allowed_ips.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.allowed_ips = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for ApiKey { fn default() -> Self { Self { @@ -1518,13 +1187,30 @@ impl Default for ApiKey { impl IntoValue for ApiKey { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value()); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); + map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value_with(opts)); JmapValue::Object(map) } } @@ -1552,7 +1238,7 @@ impl RegistryJsonPropertyPatch for ApiKey { } impl ObjectImpl for AppPassword { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::AppPassword; @@ -1589,28 +1275,6 @@ impl ObjectImpl for AppPassword { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for AppPassword { - fn pickle(&self, out: &mut Vec) { - self.description.pickle(out); - self.secret.pickle(out); - self.created_at.pickle(out); - self.expires_at.pickle(out); - self.permissions.pickle(out); - self.allowed_ips.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.allowed_ips = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for AppPassword { fn default() -> Self { Self { @@ -1626,13 +1290,30 @@ impl Default for AppPassword { impl IntoValue for AppPassword { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value()); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); + map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value_with(opts)); JmapValue::Object(map) } } @@ -1661,7 +1342,7 @@ impl RegistryJsonPropertyPatch for AppPassword { impl ObjectImpl for Application { const FLAGS: u64 = 0; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Application; fn validate(&self, errors: &mut Vec) -> bool { @@ -1701,38 +1382,95 @@ impl ObjectImpl for Application { impl Pickle for Application { fn pickle(&self, out: &mut Vec) { - self.enabled.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.unpack_directory.is_some() { + bitmap |= 1 << 1; + } + if self.oauth_client_id.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 7, 3, bitmap); self.description.pickle(out); self.resource_url.pickle(out); self.url_prefix.pickle(out); - self.auto_update_frequency.pickle(out); - self.unpack_directory.pickle(out); - self.oauth_client_id.pickle(out); + self.auto_update_interval.pickle(out); + if let Some(value) = &self.unpack_directory { + value.pickle(out); + } + if let Some(value) = &self.oauth_client_id { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enabled = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.resource_url = Pickle::unpickle(stream)?; - this.url_prefix = Pickle::unpickle(stream)?; - this.auto_update_frequency = Pickle::unpickle(stream)?; - this.unpack_directory = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.oauth_client_id = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(7776000000) + }; + let field5: Option = if block.has(5) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + description: field1, + resource_url: field2, + url_prefix: field3, + auto_update_interval: field4, + unpack_directory: field5, + oauth_client_id: field6, + }) } } impl Default for Application { fn default() -> Self { Self { - enabled: true, + enable: true, description: Default::default(), resource_url: Default::default(), url_prefix: Default::default(), - auto_update_frequency: Duration::from_millis(7776000000), + auto_update_interval: Duration::from_millis(7776000000), unpack_directory: Default::default(), oauth_client_id: Default::default(), } @@ -1741,20 +1479,33 @@ impl Default for Application { impl IntoValue for Application { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Enabled, self.enabled.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::ResourceUrl, self.resource_url.into_value()); - map.insert_unchecked(Property::UrlPrefix, self.url_prefix.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); map.insert_unchecked( - Property::AutoUpdateFrequency, - self.auto_update_frequency.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::ResourceUrl, + self.resource_url.into_value_with(opts), + ); + map.insert_unchecked(Property::UrlPrefix, self.url_prefix.into_value_with(opts)); + map.insert_unchecked( + Property::AutoUpdateInterval, + self.auto_update_interval.into_value_with(opts), ); map.insert_unchecked( Property::UnpackDirectory, - self.unpack_directory.into_value(), + self.unpack_directory.into_value_with(opts), + ); + map.insert_unchecked( + Property::OauthClientId, + self.oauth_client_id.into_value_with(opts), ); - map.insert_unchecked(Property::OauthClientId, self.oauth_client_id.into_value()); JmapValue::Object(map) } } @@ -1766,7 +1517,7 @@ impl RegistryJsonPropertyPatch for Application { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Enabled) => self.enabled.patch(pointer, value), + Some(Property::Enable) => self.enable.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -1776,7 +1527,7 @@ impl RegistryJsonPropertyPatch for Application { Some(Property::UrlPrefix) => self .url_prefix .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AutoUpdateFrequency) => self.auto_update_frequency.patch(pointer, value), + Some(Property::AutoUpdateInterval) => self.auto_update_interval.patch(pointer, value), Some(Property::UnpackDirectory) => self .unpack_directory .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -1792,752 +1543,8 @@ impl RegistryJsonPropertyPatch for Application { } } -impl ArchivedCalendarEvent { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.title; - if value.is_empty() { - errors.push(ValidationError::required(Property::Title)); - } - if let Some(value) = &self.start_time { - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::StartTime, value)); - } - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.archived_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for ArchivedCalendarEvent { - fn pickle(&self, out: &mut Vec) { - self.title.pickle(out); - self.start_time.pickle(out); - self.created_at.pickle(out); - self.account_id.pickle(out); - self.archived_at.pickle(out); - self.archived_until.pickle(out); - self.blob_id.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.title = Pickle::unpickle(stream)?; - this.start_time = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.archived_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ArchivedCalendarEvent { - fn default() -> Self { - Self { - title: Default::default(), - start_time: Default::default(), - created_at: Default::default(), - account_id: Default::default(), - archived_at: Default::default(), - archived_until: Default::default(), - blob_id: Default::default(), - } - } -} - -impl IntoValue for ArchivedCalendarEvent { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Title, self.title.into_value()); - map.insert_unchecked(Property::StartTime, self.start_time.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ArchivedAt, self.archived_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ArchivedCalendarEvent { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Title) => self.title.patch(pointer, value), - Some(Property::StartTime) => self.start_time.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_can_set_account()?, value), - Some(Property::ArchivedAt) => self.archived_at.patch(pointer, value), - Some(Property::ArchivedUntil) => self.archived_until.patch(pointer, value), - Some(Property::BlobId) => self.blob_id.patch(pointer, value), - Some(property @ Property::Status) => Ok(MaybeUnpatched::Unpatched { property, value }), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ArchivedContactCard { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - if let Some(value) = &self.name { - if value.is_empty() { - errors.push(ValidationError::required(Property::Name)); - } - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.archived_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for ArchivedContactCard { - fn pickle(&self, out: &mut Vec) { - self.name.pickle(out); - self.created_at.pickle(out); - self.account_id.pickle(out); - self.archived_at.pickle(out); - self.archived_until.pickle(out); - self.blob_id.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.archived_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ArchivedContactCard { - fn default() -> Self { - Self { - name: Default::default(), - created_at: Default::default(), - account_id: Default::default(), - archived_at: Default::default(), - archived_until: Default::default(), - blob_id: Default::default(), - } - } -} - -impl IntoValue for ArchivedContactCard { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ArchivedAt, self.archived_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ArchivedContactCard { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Name) => self.name.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_can_set_account()?, value), - Some(Property::ArchivedAt) => self.archived_at.patch(pointer, value), - Some(Property::ArchivedUntil) => self.archived_until.patch(pointer, value), - Some(Property::BlobId) => self.blob_id.patch(pointer, value), - Some(property @ Property::Status) => Ok(MaybeUnpatched::Unpatched { property, value }), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ArchivedEmail { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.from; - if value.is_empty() { - errors.push(ValidationError::required(Property::From)); - } - let value = &self.subject; - if value.is_empty() { - errors.push(ValidationError::required(Property::Subject)); - } - let value = &self.received_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ReceivedAt, value)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.archived_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for ArchivedEmail { - fn pickle(&self, out: &mut Vec) { - self.from.pickle(out); - self.subject.pickle(out); - self.received_at.pickle(out); - self.size.pickle(out); - self.account_id.pickle(out); - self.archived_at.pickle(out); - self.archived_until.pickle(out); - self.blob_id.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.received_at = Pickle::unpickle(stream)?; - this.size = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.archived_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ArchivedEmail { - fn default() -> Self { - Self { - from: Default::default(), - subject: Default::default(), - received_at: Default::default(), - size: 0u64, - account_id: Default::default(), - archived_at: Default::default(), - archived_until: Default::default(), - blob_id: Default::default(), - } - } -} - -impl IntoValue for ArchivedEmail { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value()); - map.insert_unchecked(Property::Size, self.size.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ArchivedAt, self.archived_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ArchivedEmail { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::From) => self.from.patch(pointer, value), - Some(Property::Subject) => self.subject.patch(pointer, value), - Some(Property::ReceivedAt) => self.received_at.patch(pointer, value), - Some(Property::Size) => self.size.patch(pointer.assert_read_only()?, value), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_can_set_account()?, value), - Some(Property::ArchivedAt) => self.archived_at.patch(pointer, value), - Some(Property::ArchivedUntil) => self.archived_until.patch(pointer, value), - Some(Property::BlobId) => self.blob_id.patch(pointer, value), - Some(property @ Property::Status) => Ok(MaybeUnpatched::Unpatched { property, value }), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ArchivedFileNode { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.name; - if value.is_empty() { - errors.push(ValidationError::required(Property::Name)); - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.archived_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for ArchivedFileNode { - fn pickle(&self, out: &mut Vec) { - self.name.pickle(out); - self.created_at.pickle(out); - self.account_id.pickle(out); - self.archived_at.pickle(out); - self.archived_until.pickle(out); - self.blob_id.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.archived_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ArchivedFileNode { - fn default() -> Self { - Self { - name: Default::default(), - created_at: Default::default(), - account_id: Default::default(), - archived_at: Default::default(), - archived_until: Default::default(), - blob_id: Default::default(), - } - } -} - -impl IntoValue for ArchivedFileNode { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ArchivedAt, self.archived_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ArchivedFileNode { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Name) => self.name.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_can_set_account()?, value), - Some(Property::ArchivedAt) => self.archived_at.patch(pointer, value), - Some(Property::ArchivedUntil) => self.archived_until.patch(pointer, value), - Some(Property::BlobId) => self.blob_id.patch(pointer, value), - Some(property @ Property::Status) => Ok(MaybeUnpatched::Unpatched { property, value }), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ObjectImpl for ArchivedItem { - const FLAGS: u64 = OBJ_FILTER_ACCOUNT; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::ArchivedItem; - - fn validate(&self, errors: &mut Vec) -> bool { - match self { - ArchivedItem::Email(inner) => inner.validate(errors), - ArchivedItem::FileNode(inner) => inner.validate(errors), - ArchivedItem::CalendarEvent(inner) => inner.validate(errors), - ArchivedItem::ContactCard(inner) => inner.validate(errors), - ArchivedItem::SieveScript(inner) => inner.validate(errors), - } - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - match self { - ArchivedItem::Email(object) => { - object.index(i); - } - ArchivedItem::FileNode(object) => { - object.index(i); - } - ArchivedItem::CalendarEvent(object) => { - object.index(i); - } - ArchivedItem::ContactCard(object) => { - object.index(i); - } - ArchivedItem::SieveScript(object) => { - object.index(i); - } - } - } -} - -impl Default for ArchivedItem { - fn default() -> Self { - ArchivedItem::Email(Default::default()) - } -} - -impl Pickle for ArchivedItem { - fn pickle(&self, out: &mut Vec) { - match self { - ArchivedItem::Email(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - ArchivedItem::FileNode(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - ArchivedItem::CalendarEvent(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - ArchivedItem::ContactCard(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - ArchivedItem::SieveScript(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(ArchivedItem::Email), - 1 => Pickle::unpickle(stream).map(ArchivedItem::FileNode), - 2 => Pickle::unpickle(stream).map(ArchivedItem::CalendarEvent), - 3 => Pickle::unpickle(stream).map(ArchivedItem::ContactCard), - 4 => Pickle::unpickle(stream).map(ArchivedItem::SieveScript), - _ => None, - } - } -} - -impl IntoValue for ArchivedItem { - fn into_value(self) -> JmapValue<'static> { - match self { - ArchivedItem::Email(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Email".into())); - obj - } - ArchivedItem::FileNode(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("FileNode".into())); - obj - } - ArchivedItem::CalendarEvent(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("CalendarEvent".into())); - obj - } - ArchivedItem::ContactCard(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("ContactCard".into())); - obj - } - ArchivedItem::SieveScript(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("SieveScript".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for ArchivedItem { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - ArchivedItemType::Email => *self = ArchivedItem::Email(Default::default()), - ArchivedItemType::FileNode => *self = ArchivedItem::FileNode(Default::default()), - ArchivedItemType::CalendarEvent => { - *self = ArchivedItem::CalendarEvent(Default::default()) - } - ArchivedItemType::ContactCard => { - *self = ArchivedItem::ContactCard(Default::default()) - } - ArchivedItemType::SieveScript => { - *self = ArchivedItem::SieveScript(Default::default()) - } - } - } - match self { - ArchivedItem::Email(inner) => inner.patch(pointer, value), - ArchivedItem::FileNode(inner) => inner.patch(pointer, value), - ArchivedItem::CalendarEvent(inner) => inner.patch(pointer, value), - ArchivedItem::ContactCard(inner) => inner.patch(pointer, value), - ArchivedItem::SieveScript(inner) => inner.patch(pointer, value), - } - } -} - -impl ArchivedItem { - pub fn object_type(&self) -> ArchivedItemType { - match self { - ArchivedItem::Email(_) => ArchivedItemType::Email, - ArchivedItem::FileNode(_) => ArchivedItemType::FileNode, - ArchivedItem::CalendarEvent(_) => ArchivedItemType::CalendarEvent, - ArchivedItem::ContactCard(_) => ArchivedItemType::ContactCard, - ArchivedItem::SieveScript(_) => ArchivedItemType::SieveScript, - } - } -} - -impl ArchivedSieveScript { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.name; - if value.is_empty() { - errors.push(ValidationError::required(Property::Name)); - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - let value = &self.content; - if value.is_empty() { - errors.push(ValidationError::required(Property::Content)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.archived_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for ArchivedSieveScript { - fn pickle(&self, out: &mut Vec) { - self.name.pickle(out); - self.created_at.pickle(out); - self.content.pickle(out); - self.account_id.pickle(out); - self.archived_at.pickle(out); - self.archived_until.pickle(out); - self.blob_id.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.content = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.archived_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ArchivedSieveScript { - fn default() -> Self { - Self { - name: Default::default(), - created_at: Default::default(), - content: Default::default(), - account_id: Default::default(), - archived_at: Default::default(), - archived_until: Default::default(), - blob_id: Default::default(), - } - } -} - -impl IntoValue for ArchivedSieveScript { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::Content, self.content.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ArchivedAt, self.archived_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ArchivedSieveScript { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Name) => self.name.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::Content) => self.content.patch(pointer, value), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_can_set_account()?, value), - Some(Property::ArchivedAt) => self.archived_at.patch(pointer, value), - Some(Property::ArchivedUntil) => self.archived_until.patch(pointer, value), - Some(Property::BlobId) => self.blob_id.patch(pointer, value), - Some(property @ Property::Status) => Ok(MaybeUnpatched::Unpatched { property, value }), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for ArfExternalReport { - const FLAGS: u64 = OBJ_FILTER_TENANT; + const FLAGS: u64 = OBJ_FILTER_TENANT | OBJ_NO_CREATE | OBJ_NO_UPDATE; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::ArfExternalReport; @@ -2567,40 +1574,72 @@ impl ObjectImpl for ArfExternalReport { if !value.is_valid() { errors.push(ValidationError::invalid(Property::ExpiresAt, value)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); } } impl Pickle for ArfExternalReport { fn pickle(&self, out: &mut Vec) { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut bitmap = 0u64; + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.report.pickle(out); self.from.pickle(out); self.subject.pickle(out); self.to.pickle(out); self.received_at.pickle(out); self.expires_at.pickle(out); - self.member_tenant_id.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.report = Pickle::unpickle(stream)?; - this.from = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - this.received_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: ArfFeedbackReport = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + report: field0, + from: field1, + subject: field2, + to: field3, + received_at: field4, + expires_at: field5, + }) } } @@ -2613,21 +1652,23 @@ impl Default for ArfExternalReport { to: Default::default(), received_at: Default::default(), expires_at: Default::default(), - member_tenant_id: Default::default(), } } } impl IntoValue for ArfExternalReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); - map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::From, self.from.into_value_with(opts)); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); + map.insert_unchecked(Property::To, self.to.into_value_with(opts)); + map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -2649,9 +1690,6 @@ impl RegistryJsonPropertyPatch for ArfExternalReport { .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::ReceivedAt) => self.received_at.patch(pointer, value), Some(Property::ExpiresAt) => self.expires_at.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -2781,65 +1819,367 @@ impl ArfFeedbackReport { impl Pickle for ArfFeedbackReport { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.arrival_date.is_some() { + bitmap |= 1 << 0; + } + if self.original_envelope_id.is_some() { + bitmap |= 1 << 1; + } + if self.original_mail_from.is_some() { + bitmap |= 1 << 2; + } + if self.original_rcpt_to.is_some() { + bitmap |= 1 << 3; + } + if self.reporting_mta.is_some() { + bitmap |= 1 << 4; + } + if self.source_ip.is_some() { + bitmap |= 1 << 5; + } + if self.source_port.is_some() { + bitmap |= 1 << 6; + } + if self.user_agent.is_some() { + bitmap |= 1 << 7; + } + if self.dkim_adsp_dns.is_some() { + bitmap |= 1 << 8; + } + if self.dkim_canonicalized_body.is_some() { + bitmap |= 1 << 9; + } + if self.dkim_canonicalized_header.is_some() { + bitmap |= 1 << 10; + } + if self.dkim_domain.is_some() { + bitmap |= 1 << 11; + } + if self.dkim_identity.is_some() { + bitmap |= 1 << 12; + } + if self.dkim_selector.is_some() { + bitmap |= 1 << 13; + } + if self.dkim_selector_dns.is_some() { + bitmap |= 1 << 14; + } + if self.spf_dns.is_some() { + bitmap |= 1 << 15; + } + if self.message.is_some() { + bitmap |= 1 << 16; + } + if self.headers.is_some() { + bitmap |= 1 << 17; + } + let block = crate::pickle::begin_block(out, 27, 18, bitmap); self.feedback_type.pickle(out); - self.arrival_date.pickle(out); + if let Some(value) = &self.arrival_date { + value.pickle(out); + } self.authentication_results.pickle(out); self.incidents.pickle(out); - self.original_envelope_id.pickle(out); - self.original_mail_from.pickle(out); - self.original_rcpt_to.pickle(out); + if let Some(value) = &self.original_envelope_id { + value.pickle(out); + } + if let Some(value) = &self.original_mail_from { + value.pickle(out); + } + if let Some(value) = &self.original_rcpt_to { + value.pickle(out); + } self.reported_domains.pickle(out); self.reported_uris.pickle(out); - self.reporting_mta.pickle(out); - self.source_ip.pickle(out); - self.source_port.pickle(out); - self.user_agent.pickle(out); + if let Some(value) = &self.reporting_mta { + value.pickle(out); + } + if let Some(value) = &self.source_ip { + value.pickle(out); + } + if let Some(value) = &self.source_port { + value.pickle(out); + } + if let Some(value) = &self.user_agent { + value.pickle(out); + } self.version.pickle(out); self.auth_failure.pickle(out); self.delivery_result.pickle(out); - self.dkim_adsp_dns.pickle(out); - self.dkim_canonicalized_body.pickle(out); - self.dkim_canonicalized_header.pickle(out); - self.dkim_domain.pickle(out); - self.dkim_identity.pickle(out); - self.dkim_selector.pickle(out); - self.dkim_selector_dns.pickle(out); - self.spf_dns.pickle(out); + if let Some(value) = &self.dkim_adsp_dns { + value.pickle(out); + } + if let Some(value) = &self.dkim_canonicalized_body { + value.pickle(out); + } + if let Some(value) = &self.dkim_canonicalized_header { + value.pickle(out); + } + if let Some(value) = &self.dkim_domain { + value.pickle(out); + } + if let Some(value) = &self.dkim_identity { + value.pickle(out); + } + if let Some(value) = &self.dkim_selector { + value.pickle(out); + } + if let Some(value) = &self.dkim_selector_dns { + value.pickle(out); + } + if let Some(value) = &self.spf_dns { + value.pickle(out); + } self.identity_alignment.pickle(out); - self.message.pickle(out); - self.headers.pickle(out); + if let Some(value) = &self.message { + value.pickle(out); + } + if let Some(value) = &self.headers { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.feedback_type = Pickle::unpickle(stream)?; - this.arrival_date = Pickle::unpickle(stream)?; - this.authentication_results = Pickle::unpickle(stream)?; - this.incidents = Pickle::unpickle(stream)?; - this.original_envelope_id = Pickle::unpickle(stream)?; - this.original_mail_from = Pickle::unpickle(stream)?; - this.original_rcpt_to = Pickle::unpickle(stream)?; - this.reported_domains = Pickle::unpickle(stream)?; - this.reported_uris = Pickle::unpickle(stream)?; - this.reporting_mta = Pickle::unpickle(stream)?; - this.source_ip = Pickle::unpickle(stream)?; - this.source_port = Pickle::unpickle(stream)?; - this.user_agent = Pickle::unpickle(stream)?; - this.version = Pickle::unpickle(stream)?; - this.auth_failure = Pickle::unpickle(stream)?; - this.delivery_result = Pickle::unpickle(stream)?; - this.dkim_adsp_dns = Pickle::unpickle(stream)?; - this.dkim_canonicalized_body = Pickle::unpickle(stream)?; - this.dkim_canonicalized_header = Pickle::unpickle(stream)?; - this.dkim_domain = Pickle::unpickle(stream)?; - this.dkim_identity = Pickle::unpickle(stream)?; - this.dkim_selector = Pickle::unpickle(stream)?; - this.dkim_selector_dns = Pickle::unpickle(stream)?; - this.spf_dns = Pickle::unpickle(stream)?; - this.identity_alignment = Pickle::unpickle(stream)?; - this.message = Pickle::unpickle(stream)?; - this.headers = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: ArfFeedbackType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Map = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Map = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Option = if block.has(9) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field10: Option = if block.has(10) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field11: Option = if block.has(11) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field12: Option = if block.has(12) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 1u64 + }; + let field14: ArfAuthFailureType = if block.has(14) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field15: ArfDeliveryResult = if block.has(15) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field16: Option = if block.has(16) { + if block.flag(8) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field17: Option = if block.has(17) { + if block.flag(9) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field18: Option = if block.has(18) { + if block.flag(10) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field19: Option = if block.has(19) { + if block.flag(11) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field20: Option = if block.has(20) { + if block.flag(12) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field21: Option = if block.has(21) { + if block.flag(13) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field22: Option = if block.has(22) { + if block.flag(14) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field23: Option = if block.has(23) { + if block.flag(15) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field24: ArfIdentityAlignment = if block.has(24) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field25: Option = if block.has(25) { + if block.flag(16) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field26: Option = if block.has(26) { + if block.flag(17) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + feedback_type: field0, + arrival_date: field1, + authentication_results: field2, + incidents: field3, + original_envelope_id: field4, + original_mail_from: field5, + original_rcpt_to: field6, + reported_domains: field7, + reported_uris: field8, + reporting_mta: field9, + source_ip: field10, + source_port: field11, + user_agent: field12, + version: field13, + auth_failure: field14, + delivery_result: field15, + dkim_adsp_dns: field16, + dkim_canonicalized_body: field17, + dkim_canonicalized_header: field18, + dkim_domain: field19, + dkim_identity: field20, + dkim_selector: field21, + dkim_selector_dns: field22, + spf_dns: field23, + identity_alignment: field24, + message: field25, + headers: field26, + }) } } @@ -2879,58 +2219,92 @@ impl Default for ArfFeedbackReport { impl IntoValue for ArfFeedbackReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(29); - map.insert_unchecked(Property::FeedbackType, self.feedback_type.into_value()); - map.insert_unchecked(Property::ArrivalDate, self.arrival_date.into_value()); + map.insert_unchecked( + Property::FeedbackType, + self.feedback_type.into_value_with(opts), + ); + map.insert_unchecked( + Property::ArrivalDate, + self.arrival_date.into_value_with(opts), + ); map.insert_unchecked( Property::AuthenticationResults, - self.authentication_results.into_value(), + self.authentication_results.into_value_with(opts), ); - map.insert_unchecked(Property::Incidents, self.incidents.into_value()); + map.insert_unchecked(Property::Incidents, self.incidents.into_value_with(opts)); map.insert_unchecked( Property::OriginalEnvelopeId, - self.original_envelope_id.into_value(), + self.original_envelope_id.into_value_with(opts), ); map.insert_unchecked( Property::OriginalMailFrom, - self.original_mail_from.into_value(), + self.original_mail_from.into_value_with(opts), + ); + map.insert_unchecked( + Property::OriginalRcptTo, + self.original_rcpt_to.into_value_with(opts), ); - map.insert_unchecked(Property::OriginalRcptTo, self.original_rcpt_to.into_value()); map.insert_unchecked( Property::ReportedDomains, - self.reported_domains.into_value(), + self.reported_domains.into_value_with(opts), + ); + map.insert_unchecked( + Property::ReportedUris, + self.reported_uris.into_value_with(opts), + ); + map.insert_unchecked( + Property::ReportingMta, + self.reporting_mta.into_value_with(opts), + ); + map.insert_unchecked(Property::SourceIp, self.source_ip.into_value_with(opts)); + map.insert_unchecked(Property::SourcePort, self.source_port.into_value_with(opts)); + map.insert_unchecked(Property::UserAgent, self.user_agent.into_value_with(opts)); + map.insert_unchecked(Property::Version, self.version.into_value_with(opts)); + map.insert_unchecked( + Property::AuthFailure, + self.auth_failure.into_value_with(opts), + ); + map.insert_unchecked( + Property::DeliveryResult, + self.delivery_result.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimAdspDns, + self.dkim_adsp_dns.into_value_with(opts), ); - map.insert_unchecked(Property::ReportedUris, self.reported_uris.into_value()); - map.insert_unchecked(Property::ReportingMta, self.reporting_mta.into_value()); - map.insert_unchecked(Property::SourceIp, self.source_ip.into_value()); - map.insert_unchecked(Property::SourcePort, self.source_port.into_value()); - map.insert_unchecked(Property::UserAgent, self.user_agent.into_value()); - map.insert_unchecked(Property::Version, self.version.into_value()); - map.insert_unchecked(Property::AuthFailure, self.auth_failure.into_value()); - map.insert_unchecked(Property::DeliveryResult, self.delivery_result.into_value()); - map.insert_unchecked(Property::DkimAdspDns, self.dkim_adsp_dns.into_value()); map.insert_unchecked( Property::DkimCanonicalizedBody, - self.dkim_canonicalized_body.into_value(), + self.dkim_canonicalized_body.into_value_with(opts), ); map.insert_unchecked( Property::DkimCanonicalizedHeader, - self.dkim_canonicalized_header.into_value(), + self.dkim_canonicalized_header.into_value_with(opts), + ); + map.insert_unchecked(Property::DkimDomain, self.dkim_domain.into_value_with(opts)); + map.insert_unchecked( + Property::DkimIdentity, + self.dkim_identity.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSelector, + self.dkim_selector.into_value_with(opts), ); - map.insert_unchecked(Property::DkimDomain, self.dkim_domain.into_value()); - map.insert_unchecked(Property::DkimIdentity, self.dkim_identity.into_value()); - map.insert_unchecked(Property::DkimSelector, self.dkim_selector.into_value()); map.insert_unchecked( Property::DkimSelectorDns, - self.dkim_selector_dns.into_value(), + self.dkim_selector_dns.into_value_with(opts), ); - map.insert_unchecked(Property::SpfDns, self.spf_dns.into_value()); + map.insert_unchecked(Property::SpfDns, self.spf_dns.into_value_with(opts)); map.insert_unchecked( Property::IdentityAlignment, - self.identity_alignment.into_value(), + self.identity_alignment.into_value_with(opts), ); - map.insert_unchecked(Property::Message, self.message.into_value()); - map.insert_unchecked(Property::Headers, self.headers.into_value()); + map.insert_unchecked(Property::Message, self.message.into_value_with(opts)); + map.insert_unchecked(Property::Headers, self.headers.into_value_with(opts)); JmapValue::Object(map) } } @@ -3004,6 +2378,7 @@ impl ObjectImpl for Asn { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for Asn { @@ -3030,6 +2405,9 @@ impl Pickle for Asn { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Some(Asn::Disabled), 1 => Pickle::unpickle(stream).map(Asn::Resource), @@ -3041,6 +2419,10 @@ impl Pickle for Asn { impl IntoValue for Asn { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Asn::Disabled => { let mut obj = jmap_tools::Map::new(); @@ -3048,14 +2430,14 @@ impl IntoValue for Asn { JmapValue::Object(obj) } Asn::Resource(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Resource".into())); obj } Asn::Dns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dns".into())); @@ -3117,23 +2499,76 @@ impl AsnDns { impl Pickle for AsnDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.index_asn_name.is_some() { + bitmap |= 1 << 0; + } + if self.index_country.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 6, 2, bitmap); self.index_asn.pickle(out); - self.index_asn_name.pickle(out); - self.index_country.pickle(out); + if let Some(value) = &self.index_asn_name { + value.pickle(out); + } + if let Some(value) = &self.index_country { + value.pickle(out); + } self.separator.pickle(out); self.zone_ip_v4.pickle(out); self.zone_ip_v6.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.index_asn = Pickle::unpickle(stream)?; - this.index_asn_name = Pickle::unpickle(stream)?; - this.index_country = Pickle::unpickle(stream)?; - this.separator = Pickle::unpickle(stream)?; - this.zone_ip_v4 = Pickle::unpickle(stream)?; - this.zone_ip_v6 = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + "|".to_string() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + index_asn: field0, + index_asn_name: field1, + index_country: field2, + separator: field3, + zone_ip_v4: field4, + zone_ip_v6: field5, + }) } } @@ -3152,13 +2587,23 @@ impl Default for AsnDns { impl IntoValue for AsnDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::IndexAsn, self.index_asn.into_value()); - map.insert_unchecked(Property::IndexAsnName, self.index_asn_name.into_value()); - map.insert_unchecked(Property::IndexCountry, self.index_country.into_value()); - map.insert_unchecked(Property::Separator, self.separator.into_value()); - map.insert_unchecked(Property::ZoneIpV4, self.zone_ip_v4.into_value()); - map.insert_unchecked(Property::ZoneIpV6, self.zone_ip_v6.into_value()); + map.insert_unchecked(Property::IndexAsn, self.index_asn.into_value_with(opts)); + map.insert_unchecked( + Property::IndexAsnName, + self.index_asn_name.into_value_with(opts), + ); + map.insert_unchecked( + Property::IndexCountry, + self.index_country.into_value_with(opts), + ); + map.insert_unchecked(Property::Separator, self.separator.into_value_with(opts)); + map.insert_unchecked(Property::ZoneIpV4, self.zone_ip_v4.into_value_with(opts)); + map.insert_unchecked(Property::ZoneIpV6, self.zone_ip_v6.into_value_with(opts)); JmapValue::Object(map) } } @@ -3216,40 +2661,87 @@ impl AsnResource { } errors.len() == neb } + } impl Pickle for AsnResource { fn pickle(&self, out: &mut Vec) { - self.expires.pickle(out); + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 8, 1, bitmap); + self.refresh_interval.pickle(out); self.max_size.pickle(out); self.timeout.pickle(out); self.asn_urls.pickle(out); self.geo_urls.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.expires = Pickle::unpickle(stream)?; - this.max_size = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.asn_urls = Pickle::unpickle(stream)?; - this.geo_urls = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(86400000) + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 104857600 + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: bool = if block.has(5) { block.flag(0) } else { false }; + let field6: HttpAuth = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + refresh_interval: field0, + max_size: field1, + timeout: field2, + asn_urls: field3, + geo_urls: field4, + allow_invalid_certs: field5, + http_auth: field6, + http_headers: field7, + }) } } impl Default for AsnResource { fn default() -> Self { Self { - expires: Duration::from_millis(86400000), + refresh_interval: Duration::from_millis(86400000), max_size: 104857600, timeout: Duration::from_millis(300000), asn_urls: Default::default(), geo_urls: Default::default(), + allow_invalid_certs: false, http_auth: Default::default(), http_headers: Default::default(), } @@ -3258,14 +2750,28 @@ impl Default for AsnResource { impl IntoValue for AsnResource { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Expires, self.expires.into_value()); - map.insert_unchecked(Property::MaxSize, self.max_size.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::AsnUrls, self.asn_urls.into_value()); - map.insert_unchecked(Property::GeoUrls, self.geo_urls.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(10); + map.insert_unchecked( + Property::RefreshInterval, + self.refresh_interval.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxSize, self.max_size.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::AsnUrls, self.asn_urls.into_value_with(opts)); + map.insert_unchecked(Property::GeoUrls, self.geo_urls.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -3277,7 +2783,7 @@ impl RegistryJsonPropertyPatch for AsnResource { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Expires) => self.expires.patch(pointer, value), + Some(Property::RefreshInterval) => self.refresh_interval.patch(pointer, value), Some(Property::MaxSize) => self.max_size.patch(pointer, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::AsnUrls) => self @@ -3286,6 +2792,7 @@ impl RegistryJsonPropertyPatch for AsnResource { Some(Property::GeoUrls) => self .geo_urls .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), Some(Property::HttpHeaders) => self .http_headers @@ -3323,32 +2830,26 @@ impl ObjectImpl for Authentication { errors.push(ValidationError::required(Property::DefaultGroupRoleIds)); } } - let value = &self.default_tenant_role_ids; - for value in value.iter() { - if !value.is_valid() { - errors.push(ValidationError::required(Property::DefaultTenantRoleIds)); - } - } let value = &self.default_admin_role_ids; for value in value.iter() { if !value.is_valid() { errors.push(ValidationError::required(Property::DefaultAdminRoleIds)); } } - let value = &self.password_min_length; + let value = &self.min_password_length; if *value < 1 { - errors.push(ValidationError::min_value(Property::PasswordMinLength, 1)); + errors.push(ValidationError::min_value(Property::MinPasswordLength, 1)); } if *value > 100 { - errors.push(ValidationError::max_value(Property::PasswordMinLength, 100)); + errors.push(ValidationError::max_value(Property::MinPasswordLength, 100)); } - let value = &self.password_max_length; + let value = &self.max_password_length; if *value < 1 { - errors.push(ValidationError::min_value(Property::PasswordMaxLength, 1)); + errors.push(ValidationError::min_value(Property::MaxPasswordLength, 1)); } if *value > 1000 { errors.push(ValidationError::max_value( - Property::PasswordMaxLength, + Property::MaxPasswordLength, 1000, )); } @@ -3373,9 +2874,6 @@ impl ObjectImpl for Authentication { for id in self.default_group_role_ids.iter() { i.foreign_key(ObjectType::Role, Some(*id), None); } - for id in self.default_tenant_role_ids.iter() { - i.foreign_key(ObjectType::Role, Some(*id), None); - } for id in self.default_admin_role_ids.iter() { i.foreign_key(ObjectType::Role, Some(*id), None); } @@ -3384,35 +2882,142 @@ impl ObjectImpl for Authentication { impl Pickle for Authentication { fn pickle(&self, out: &mut Vec) { - self.directory_id.pickle(out); + let mut bitmap = 0u64; + if self.directory_id.is_some() { + bitmap |= 1 << 0; + } + if self.password_default_expiry.is_some() { + bitmap |= 1 << 1; + } + if self.max_app_passwords.is_some() { + bitmap |= 1 << 2; + } + if self.max_api_keys.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 12, 4, bitmap); + if let Some(value) = &self.directory_id { + value.pickle(out); + } self.default_user_role_ids.pickle(out); self.default_group_role_ids.pickle(out); - self.default_tenant_role_ids.pickle(out); + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } self.default_admin_role_ids.pickle(out); self.password_hash_algorithm.pickle(out); - self.password_min_length.pickle(out); - self.password_max_length.pickle(out); - self.password_min_strength.pickle(out); - self.password_default_expiry.pickle(out); - self.max_app_passwords.pickle(out); - self.max_api_keys.pickle(out); + self.min_password_length.pickle(out); + self.max_password_length.pickle(out); + self.min_password_strength.pickle(out); + if let Some(value) = &self.password_default_expiry { + value.pickle(out); + } + if let Some(value) = &self.max_app_passwords { + value.pickle(out); + } + if let Some(value) = &self.max_api_keys { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.directory_id = Pickle::unpickle(stream)?; - this.default_user_role_ids = Pickle::unpickle(stream)?; - this.default_group_role_ids = Pickle::unpickle(stream)?; - this.default_tenant_role_ids = Pickle::unpickle(stream)?; - this.default_admin_role_ids = Pickle::unpickle(stream)?; - this.password_hash_algorithm = Pickle::unpickle(stream)?; - this.password_min_length = Pickle::unpickle(stream)?; - this.password_max_length = Pickle::unpickle(stream)?; - this.password_min_strength = Pickle::unpickle(stream)?; - this.password_default_expiry = Pickle::unpickle(stream)?; - this.max_app_passwords = Pickle::unpickle(stream)?; - this.max_api_keys = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + #[cfg(not(feature = "enterprise"))] + if block.has(3) { + let block = stream.open_block()?; + stream.close_block(block); + } + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: PasswordHashAlgorithm = if block.has(5) { + Pickle::unpickle(stream)? + } else { + PasswordHashAlgorithm::Argon2id + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 8u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 128u64 + }; + let field8: PasswordStrength = if block.has(8) { + Pickle::unpickle(stream)? + } else { + PasswordStrength::Three + }; + let field9: Option = if block.has(9) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(5u64) + }; + let field11: Option = if block.has(11) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(5u64) + }; + stream.close_block(block); + Some(Self { + directory_id: field0, + default_user_role_ids: field1, + default_group_role_ids: field2, + default_admin_role_ids: field4, + password_hash_algorithm: field5, + min_password_length: field6, + max_password_length: field7, + min_password_strength: field8, + password_default_expiry: field9, + max_app_passwords: field10, + max_api_keys: field11, + }) } } @@ -3422,12 +3027,11 @@ impl Default for Authentication { directory_id: Default::default(), default_user_role_ids: Default::default(), default_group_role_ids: Default::default(), - default_tenant_role_ids: Default::default(), default_admin_role_ids: Default::default(), password_hash_algorithm: PasswordHashAlgorithm::Argon2id, - password_min_length: 8u64, - password_max_length: 128u64, - password_min_strength: PasswordStrength::Three, + min_password_length: 8u64, + max_password_length: 128u64, + min_password_strength: PasswordStrength::Three, password_default_expiry: Default::default(), max_app_passwords: Some(5u64), max_api_keys: Some(5u64), @@ -3437,49 +3041,55 @@ impl Default for Authentication { impl IntoValue for Authentication { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::DirectoryId, self.directory_id.into_value()); + map.insert_unchecked( + Property::DirectoryId, + self.directory_id.into_value_with(opts), + ); map.insert_unchecked( Property::DefaultUserRoleIds, - self.default_user_role_ids.into_value(), + self.default_user_role_ids.into_value_with(opts), ); map.insert_unchecked( Property::DefaultGroupRoleIds, - self.default_group_role_ids.into_value(), - ); - map.insert_unchecked( - Property::DefaultTenantRoleIds, - self.default_tenant_role_ids.into_value(), + self.default_group_role_ids.into_value_with(opts), ); map.insert_unchecked( Property::DefaultAdminRoleIds, - self.default_admin_role_ids.into_value(), + self.default_admin_role_ids.into_value_with(opts), ); map.insert_unchecked( Property::PasswordHashAlgorithm, - self.password_hash_algorithm.into_value(), + self.password_hash_algorithm.into_value_with(opts), ); map.insert_unchecked( - Property::PasswordMinLength, - self.password_min_length.into_value(), + Property::MinPasswordLength, + self.min_password_length.into_value_with(opts), ); map.insert_unchecked( - Property::PasswordMaxLength, - self.password_max_length.into_value(), + Property::MaxPasswordLength, + self.max_password_length.into_value_with(opts), ); map.insert_unchecked( - Property::PasswordMinStrength, - self.password_min_strength.into_value(), + Property::MinPasswordStrength, + self.min_password_strength.into_value_with(opts), ); map.insert_unchecked( Property::PasswordDefaultExpiry, - self.password_default_expiry.into_value(), + self.password_default_expiry.into_value_with(opts), ); map.insert_unchecked( Property::MaxAppPasswords, - self.max_app_passwords.into_value(), + self.max_app_passwords.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxApiKeys, + self.max_api_keys.into_value_with(opts), ); - map.insert_unchecked(Property::MaxApiKeys, self.max_api_keys.into_value()); JmapValue::Object(map) } } @@ -3496,18 +3106,15 @@ impl RegistryJsonPropertyPatch for Authentication { Some(Property::DefaultGroupRoleIds) => { self.default_group_role_ids.patch(pointer, value) } - Some(Property::DefaultTenantRoleIds) => { - self.default_tenant_role_ids.patch(pointer, value) - } Some(Property::DefaultAdminRoleIds) => { self.default_admin_role_ids.patch(pointer, value) } Some(Property::PasswordHashAlgorithm) => { self.password_hash_algorithm.patch(pointer, value) } - Some(Property::PasswordMinLength) => self.password_min_length.patch(pointer, value), - Some(Property::PasswordMaxLength) => self.password_max_length.patch(pointer, value), - Some(Property::PasswordMinStrength) => self.password_min_strength.patch(pointer, value), + Some(Property::MinPasswordLength) => self.min_password_length.patch(pointer, value), + Some(Property::MaxPasswordLength) => self.max_password_length.patch(pointer, value), + Some(Property::MinPasswordStrength) => self.min_password_strength.patch(pointer, value), Some(Property::PasswordDefaultExpiry) => { self.password_default_expiry.patch(pointer, value) } @@ -3522,6 +3129,601 @@ impl RegistryJsonPropertyPatch for Authentication { } } +impl AzureAuth { + fn validate(&self, errors: &mut Vec) -> bool { + match self { + AzureAuth::SharedKey(inner) => inner.validate(errors), + AzureAuth::SasToken(inner) => inner.validate(errors), + AzureAuth::ClientSecret(inner) => inner.validate(errors), + AzureAuth::ManagedIdentity(inner) => inner.validate(errors), + AzureAuth::WorkloadIdentity(inner) => inner.validate(errors), + } + } + +} + +impl Default for AzureAuth { + fn default() -> Self { + AzureAuth::SharedKey(Default::default()) + } +} + +impl Pickle for AzureAuth { + fn pickle(&self, out: &mut Vec) { + match self { + AzureAuth::SharedKey(inner) => { + 0u16.pickle(out); + inner.pickle(out); + } + AzureAuth::SasToken(inner) => { + 1u16.pickle(out); + inner.pickle(out); + } + AzureAuth::ClientSecret(inner) => { + 2u16.pickle(out); + inner.pickle(out); + } + AzureAuth::ManagedIdentity(inner) => { + 3u16.pickle(out); + inner.pickle(out); + } + AzureAuth::WorkloadIdentity(inner) => { + 4u16.pickle(out); + inner.pickle(out); + } + } + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + match u16::unpickle(stream)? { + 0 => Pickle::unpickle(stream).map(AzureAuth::SharedKey), + 1 => Pickle::unpickle(stream).map(AzureAuth::SasToken), + 2 => Pickle::unpickle(stream).map(AzureAuth::ClientSecret), + 3 => Pickle::unpickle(stream).map(AzureAuth::ManagedIdentity), + 4 => Pickle::unpickle(stream).map(AzureAuth::WorkloadIdentity), + _ => None, + } + } +} + +impl IntoValue for AzureAuth { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + match self { + AzureAuth::SharedKey(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("SharedKey".into())); + obj + } + AzureAuth::SasToken(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("SasToken".into())); + obj + } + AzureAuth::ClientSecret(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("ClientSecret".into())); + obj + } + AzureAuth::ManagedIdentity(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("ManagedIdentity".into())); + obj + } + AzureAuth::WorkloadIdentity(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("WorkloadIdentity".into())); + obj + } + } + } +} + +impl RegistryJsonPatch for AzureAuth { + fn patch<'x>( + &mut self, + pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + if !pointer.has_next() { + match object_type(&pointer, &value)? { + AzureAuthType::SharedKey => *self = AzureAuth::SharedKey(Default::default()), + AzureAuthType::SasToken => *self = AzureAuth::SasToken(Default::default()), + AzureAuthType::ClientSecret => *self = AzureAuth::ClientSecret(Default::default()), + AzureAuthType::ManagedIdentity => { + *self = AzureAuth::ManagedIdentity(Default::default()) + } + AzureAuthType::WorkloadIdentity => { + *self = AzureAuth::WorkloadIdentity(Default::default()) + } + } + } + match self { + AzureAuth::SharedKey(inner) => inner.patch(pointer, value), + AzureAuth::SasToken(inner) => inner.patch(pointer, value), + AzureAuth::ClientSecret(inner) => inner.patch(pointer, value), + AzureAuth::ManagedIdentity(inner) => inner.patch(pointer, value), + AzureAuth::WorkloadIdentity(inner) => inner.patch(pointer, value), + } + } +} + +impl AzureAuth { + pub fn object_type(&self) -> AzureAuthType { + match self { + AzureAuth::SharedKey(_) => AzureAuthType::SharedKey, + AzureAuth::SasToken(_) => AzureAuthType::SasToken, + AzureAuth::ClientSecret(_) => AzureAuthType::ClientSecret, + AzureAuth::ManagedIdentity(_) => AzureAuthType::ManagedIdentity, + AzureAuth::WorkloadIdentity(_) => AzureAuthType::WorkloadIdentity, + } + } +} + +impl AzureAuthClientSecret { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.tenant_id; + if value.is_empty() { + errors.push(ValidationError::required(Property::TenantId)); + } + let value = &self.client_id; + if value.is_empty() { + errors.push(ValidationError::required(Property::ClientId)); + } + let value = &self.client_secret; + value.validate(errors); + errors.len() == neb + } + +} + +impl Pickle for AzureAuthClientSecret { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); + self.tenant_id.pickle(out); + self.client_id.pickle(out); + self.client_secret.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + tenant_id: field0, + client_id: field1, + client_secret: field2, + }) + } +} + +impl Default for AzureAuthClientSecret { + fn default() -> Self { + Self { + tenant_id: Default::default(), + client_id: Default::default(), + client_secret: Default::default(), + } + } +} + +impl IntoValue for AzureAuthClientSecret { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(5); + map.insert_unchecked(Property::TenantId, self.tenant_id.into_value_with(opts)); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); + map.insert_unchecked( + Property::ClientSecret, + self.client_secret.into_value_with(opts), + ); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for AzureAuthClientSecret { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::TenantId) => self + .tenant_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::ClientId) => self + .client_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::ClientSecret) => self.client_secret.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl AzureAuthManagedIdentity { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + if let Some(value) = &self.client_id { + if value.is_empty() { + errors.push(ValidationError::required(Property::ClientId)); + } + } + errors.len() == neb + } +} + +impl Pickle for AzureAuthManagedIdentity { + fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.client_id.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 1, 1, bitmap); + if let Some(value) = &self.client_id { + value.pickle(out); + } + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { client_id: field0 }) + } +} + +impl Default for AzureAuthManagedIdentity { + fn default() -> Self { + Self { + client_id: Default::default(), + } + } +} + +impl IntoValue for AzureAuthManagedIdentity { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(3); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for AzureAuthManagedIdentity { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::ClientId) => self + .client_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl AzureAuthSasToken { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.sas_token; + value.validate(errors); + errors.len() == neb + } + +} + +impl Pickle for AzureAuthSasToken { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); + self.sas_token.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { sas_token: field0 }) + } +} + +impl Default for AzureAuthSasToken { + fn default() -> Self { + Self { + sas_token: Default::default(), + } + } +} + +impl IntoValue for AzureAuthSasToken { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(3); + map.insert_unchecked(Property::SasToken, self.sas_token.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for AzureAuthSasToken { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::SasToken) => self.sas_token.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl AzureAuthSharedKey { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.account_key; + value.validate(errors); + errors.len() == neb + } + +} + +impl Pickle for AzureAuthSharedKey { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); + self.account_key.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + account_key: field0, + }) + } +} + +impl Default for AzureAuthSharedKey { + fn default() -> Self { + Self { + account_key: Default::default(), + } + } +} + +impl IntoValue for AzureAuthSharedKey { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(3); + map.insert_unchecked(Property::AccountKey, self.account_key.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for AzureAuthSharedKey { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::AccountKey) => self.account_key.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl AzureAuthWorkloadIdentity { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + if let Some(value) = &self.tenant_id { + if value.is_empty() { + errors.push(ValidationError::required(Property::TenantId)); + } + } + if let Some(value) = &self.client_id { + if value.is_empty() { + errors.push(ValidationError::required(Property::ClientId)); + } + } + if let Some(value) = &self.token_file { + if value.is_empty() { + errors.push(ValidationError::required(Property::TokenFile)); + } + } + errors.len() == neb + } +} + +impl Pickle for AzureAuthWorkloadIdentity { + fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.tenant_id.is_some() { + bitmap |= 1 << 0; + } + if self.client_id.is_some() { + bitmap |= 1 << 1; + } + if self.token_file.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 3, 3, bitmap); + if let Some(value) = &self.tenant_id { + value.pickle(out); + } + if let Some(value) = &self.client_id { + value.pickle(out); + } + if let Some(value) = &self.token_file { + value.pickle(out); + } + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + tenant_id: field0, + client_id: field1, + token_file: field2, + }) + } +} + +impl Default for AzureAuthWorkloadIdentity { + fn default() -> Self { + Self { + tenant_id: Default::default(), + client_id: Default::default(), + token_file: Default::default(), + } + } +} + +impl IntoValue for AzureAuthWorkloadIdentity { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(5); + map.insert_unchecked(Property::TenantId, self.tenant_id.into_value_with(opts)); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); + map.insert_unchecked(Property::TokenFile, self.token_file.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for AzureAuthWorkloadIdentity { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::TenantId) => self + .tenant_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::ClientId) => self + .client_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::TokenFile) => self + .token_file + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + impl AzureStore { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -3533,17 +3735,17 @@ impl AzureStore { if value.is_empty() { errors.push(ValidationError::required(Property::Container)); } - let value = &self.access_key; - value.validate(errors); - let value = &self.sas_token; + if let Some(value) = &self.url { + if value.is_empty() { + errors.push(ValidationError::required(Property::Url)); + } + } + let value = &self.auth; value.validate(errors); let value = &self.max_retries; if *value > 10 { errors.push(ValidationError::max_value(Property::MaxRetries, 10)); } - if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxRetries, 1)); - } if let Some(value) = &self.key_prefix { if value.is_empty() { errors.push(ValidationError::required(Property::KeyPrefix)); @@ -3551,29 +3753,95 @@ impl AzureStore { } errors.len() == neb } + } impl Pickle for AzureStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.url.is_some() { + bitmap |= 1 << 0; + } + if self.key_prefix.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.storage_account.pickle(out); self.container.pickle(out); - self.access_key.pickle(out); - self.sas_token.pickle(out); + self.environment.pickle(out); + if let Some(value) = &self.url { + value.pickle(out); + } + self.auth.pickle(out); self.timeout.pickle(out); self.max_retries.pickle(out); - self.key_prefix.pickle(out); + if let Some(value) = &self.key_prefix { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.storage_account = Pickle::unpickle(stream)?; - this.container = Pickle::unpickle(stream)?; - this.access_key = Pickle::unpickle(stream)?; - this.sas_token = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.max_retries = Pickle::unpickle(stream)?; - this.key_prefix = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: AzureEnvironment = if block.has(2) { + Pickle::unpickle(stream)? + } else { + AzureEnvironment::Public + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: AzureAuth = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + storage_account: field0, + container: field1, + environment: field2, + url: field3, + auth: field4, + timeout: field5, + max_retries: field6, + key_prefix: field7, + }) } } @@ -3582,8 +3850,9 @@ impl Default for AzureStore { Self { storage_account: Default::default(), container: Default::default(), - access_key: Default::default(), - sas_token: Default::default(), + environment: AzureEnvironment::Public, + url: Default::default(), + auth: Default::default(), timeout: Duration::from_millis(30000), max_retries: 3u64, key_prefix: Default::default(), @@ -3593,14 +3862,25 @@ impl Default for AzureStore { impl IntoValue for AzureStore { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::StorageAccount, self.storage_account.into_value()); - map.insert_unchecked(Property::Container, self.container.into_value()); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SasToken, self.sas_token.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value()); - map.insert_unchecked(Property::KeyPrefix, self.key_prefix.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(10); + map.insert_unchecked( + Property::StorageAccount, + self.storage_account.into_value_with(opts), + ); + map.insert_unchecked(Property::Container, self.container.into_value_with(opts)); + map.insert_unchecked( + Property::Environment, + self.environment.into_value_with(opts), + ); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Auth, self.auth.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value_with(opts)); + map.insert_unchecked(Property::KeyPrefix, self.key_prefix.into_value_with(opts)); JmapValue::Object(map) } } @@ -3618,8 +3898,11 @@ impl RegistryJsonPropertyPatch for AzureStore { Some(Property::Container) => self .container .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AccessKey) => self.access_key.patch(pointer, value), - Some(Property::SasToken) => self.sas_token.patch(pointer, value), + Some(Property::Environment) => self.environment.patch(pointer, value), + Some(Property::Url) => self + .url + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Auth) => self.auth.patch(pointer, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::MaxRetries) => self.max_retries.patch(pointer, value), Some(Property::KeyPrefix) => self @@ -3636,13 +3919,12 @@ impl RegistryJsonPropertyPatch for AzureStore { impl ObjectImpl for BlobStore { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::BlobStore; fn validate(&self, errors: &mut Vec) -> bool { match self { BlobStore::Default => true, - BlobStore::Sharded(inner) => inner.validate(errors), BlobStore::S3(inner) => inner.validate(errors), BlobStore::Azure(inner) => inner.validate(errors), BlobStore::FileSystem(inner) => inner.validate(errors), @@ -3653,6 +3935,7 @@ impl ObjectImpl for BlobStore { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for BlobStore { @@ -3667,10 +3950,6 @@ impl Pickle for BlobStore { BlobStore::Default => { 0u16.pickle(out); } - BlobStore::Sharded(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } BlobStore::S3(inner) => { 2u16.pickle(out); inner.pickle(out); @@ -3699,9 +3978,11 @@ impl Pickle for BlobStore { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Some(BlobStore::Default), - 1 => Pickle::unpickle(stream).map(BlobStore::Sharded), 2 => Pickle::unpickle(stream).map(BlobStore::S3), 3 => Pickle::unpickle(stream).map(BlobStore::Azure), 4 => Pickle::unpickle(stream).map(BlobStore::FileSystem), @@ -3715,56 +3996,53 @@ impl Pickle for BlobStore { impl IntoValue for BlobStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { BlobStore::Default => { let mut obj = jmap_tools::Map::new(); obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); JmapValue::Object(obj) } - BlobStore::Sharded(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Sharded".into())); - obj - } BlobStore::S3(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("S3".into())); obj } BlobStore::Azure(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Azure".into())); obj } BlobStore::FileSystem(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FileSystem".into())); obj } BlobStore::FoundationDb(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); obj } BlobStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } BlobStore::MySql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); @@ -3783,7 +4061,6 @@ impl RegistryJsonPatch for BlobStore { if !pointer.has_next() { match object_type(&pointer, &value)? { BlobStoreType::Default => *self = BlobStore::Default, - BlobStoreType::Sharded => *self = BlobStore::Sharded(Default::default()), BlobStoreType::S3 => *self = BlobStore::S3(Default::default()), BlobStoreType::Azure => *self = BlobStore::Azure(Default::default()), BlobStoreType::FileSystem => *self = BlobStore::FileSystem(Default::default()), @@ -3794,7 +4071,6 @@ impl RegistryJsonPatch for BlobStore { } match self { BlobStore::Default => pointer.assert_eof(), - BlobStore::Sharded(inner) => inner.patch(pointer, value), BlobStore::S3(inner) => inner.patch(pointer, value), BlobStore::Azure(inner) => inner.patch(pointer, value), BlobStore::FileSystem(inner) => inner.patch(pointer, value), @@ -3809,7 +4085,6 @@ impl BlobStore { pub fn object_type(&self) -> BlobStoreType { match self { BlobStore::Default => BlobStoreType::Default, - BlobStore::Sharded(_) => BlobStoreType::Sharded, BlobStore::S3(_) => BlobStoreType::S3, BlobStore::Azure(_) => BlobStoreType::Azure, BlobStore::FileSystem(_) => BlobStoreType::FileSystem, @@ -3820,108 +4095,261 @@ impl BlobStore { } } -impl BlobStoreBase { +impl BlobStoreSwap { fn validate(&self, errors: &mut Vec) -> bool { - match self { - BlobStoreBase::S3(inner) => inner.validate(errors), - BlobStoreBase::Azure(inner) => inner.validate(errors), - BlobStoreBase::FileSystem(inner) => inner.validate(errors), - BlobStoreBase::FoundationDb(inner) => inner.validate(errors), - BlobStoreBase::PostgreSql(inner) => inner.validate(errors), - BlobStoreBase::MySql(inner) => inner.validate(errors), + let neb = errors.len(); + let value = &self.store; + value.validate(errors); + let value = &self.flush_changes; + if *value < 1 { + errors.push(ValidationError::min_value(Property::FlushChanges, 1)); + } + let value = &self.max_account_size; + if *value < 2048 { + errors.push(ValidationError::min_value(Property::MaxAccountSize, 2048)); + } + errors.len() == neb + } + +} + +impl Pickle for BlobStoreSwap { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 6, 0, 0); + self.store.pickle(out); + self.retention.pickle(out); + self.flush_changes.pickle(out); + self.flush_interval.pickle(out); + self.max_account_size.pickle(out); + self.min_account_size.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: BlobSwapStore = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(604800000) + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 2000u64 + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 1073741824 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + stream.close_block(block); + Some(Self { + store: field0, + retention: field1, + flush_changes: field2, + flush_interval: field3, + max_account_size: field4, + min_account_size: field5, + }) + } +} + +impl Default for BlobStoreSwap { + fn default() -> Self { + Self { + store: Default::default(), + retention: Duration::from_millis(604800000), + flush_changes: 2000u64, + flush_interval: Duration::from_millis(600000), + max_account_size: 1073741824, + min_account_size: 1048576, } } } -impl Default for BlobStoreBase { - fn default() -> Self { - BlobStoreBase::S3(Default::default()) +impl IntoValue for BlobStoreSwap { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(8); + map.insert_unchecked(Property::Store, self.store.into_value_with(opts)); + map.insert_unchecked(Property::Retention, self.retention.into_value_with(opts)); + map.insert_unchecked( + Property::FlushChanges, + self.flush_changes.into_value_with(opts), + ); + map.insert_unchecked( + Property::FlushInterval, + self.flush_interval.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAccountSize, + self.max_account_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinAccountSize, + self.min_account_size.into_value_with(opts), + ); + JmapValue::Object(map) } } -impl Pickle for BlobStoreBase { +impl RegistryJsonPropertyPatch for BlobStoreSwap { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Store) => self.store.patch(pointer, value), + Some(Property::Retention) => self.retention.patch(pointer, value), + Some(Property::FlushChanges) => self.flush_changes.patch(pointer, value), + Some(Property::FlushInterval) => self.flush_interval.patch(pointer, value), + Some(Property::MaxAccountSize) => self.max_account_size.patch(pointer, value), + Some(Property::MinAccountSize) => self.min_account_size.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl BlobSwapStore { + fn validate(&self, errors: &mut Vec) -> bool { + match self { + BlobSwapStore::Default => true, + BlobSwapStore::S3(inner) => inner.validate(errors), + BlobSwapStore::Azure(inner) => inner.validate(errors), + BlobSwapStore::FileSystem(inner) => inner.validate(errors), + BlobSwapStore::FoundationDb(inner) => inner.validate(errors), + BlobSwapStore::PostgreSql(inner) => inner.validate(errors), + BlobSwapStore::MySql(inner) => inner.validate(errors), + } + } + +} + +impl Default for BlobSwapStore { + fn default() -> Self { + BlobSwapStore::Default + } +} + +impl Pickle for BlobSwapStore { fn pickle(&self, out: &mut Vec) { match self { - BlobStoreBase::S3(inner) => { + BlobSwapStore::Default => { 0u16.pickle(out); - inner.pickle(out); } - BlobStoreBase::Azure(inner) => { + BlobSwapStore::S3(inner) => { 1u16.pickle(out); inner.pickle(out); } - BlobStoreBase::FileSystem(inner) => { + BlobSwapStore::Azure(inner) => { 2u16.pickle(out); inner.pickle(out); } - BlobStoreBase::FoundationDb(inner) => { + BlobSwapStore::FileSystem(inner) => { 3u16.pickle(out); inner.pickle(out); } - BlobStoreBase::PostgreSql(inner) => { + BlobSwapStore::FoundationDb(inner) => { 4u16.pickle(out); inner.pickle(out); } - BlobStoreBase::MySql(inner) => { + BlobSwapStore::PostgreSql(inner) => { 5u16.pickle(out); inner.pickle(out); } + BlobSwapStore::MySql(inner) => { + 6u16.pickle(out); + inner.pickle(out); + } } } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(BlobStoreBase::S3), - 1 => Pickle::unpickle(stream).map(BlobStoreBase::Azure), - 2 => Pickle::unpickle(stream).map(BlobStoreBase::FileSystem), - 3 => Pickle::unpickle(stream).map(BlobStoreBase::FoundationDb), - 4 => Pickle::unpickle(stream).map(BlobStoreBase::PostgreSql), - 5 => Pickle::unpickle(stream).map(BlobStoreBase::MySql), + 0 => Some(BlobSwapStore::Default), + 1 => Pickle::unpickle(stream).map(BlobSwapStore::S3), + 2 => Pickle::unpickle(stream).map(BlobSwapStore::Azure), + 3 => Pickle::unpickle(stream).map(BlobSwapStore::FileSystem), + 4 => Pickle::unpickle(stream).map(BlobSwapStore::FoundationDb), + 5 => Pickle::unpickle(stream).map(BlobSwapStore::PostgreSql), + 6 => Pickle::unpickle(stream).map(BlobSwapStore::MySql), _ => None, } } } -impl IntoValue for BlobStoreBase { +impl IntoValue for BlobSwapStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { - BlobStoreBase::S3(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::Default => { + let mut obj = jmap_tools::Map::new(); + obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); + JmapValue::Object(obj) + } + BlobSwapStore::S3(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("S3".into())); obj } - BlobStoreBase::Azure(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::Azure(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Azure".into())); obj } - BlobStoreBase::FileSystem(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::FileSystem(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FileSystem".into())); obj } - BlobStoreBase::FoundationDb(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::FoundationDb(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); obj } - BlobStoreBase::PostgreSql(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::PostgreSql(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } - BlobStoreBase::MySql(obj) => { - let mut obj = obj.into_value(); + BlobSwapStore::MySql(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); @@ -3931,7 +4359,7 @@ impl IntoValue for BlobStoreBase { } } -impl RegistryJsonPatch for BlobStoreBase { +impl RegistryJsonPatch for BlobSwapStore { fn patch<'x>( &mut self, pointer: JsonPointerPatch<'_>, @@ -3939,40 +4367,43 @@ impl RegistryJsonPatch for BlobStoreBase { ) -> PatchResult<'x> { if !pointer.has_next() { match object_type(&pointer, &value)? { - BlobStoreBaseType::S3 => *self = BlobStoreBase::S3(Default::default()), - BlobStoreBaseType::Azure => *self = BlobStoreBase::Azure(Default::default()), - BlobStoreBaseType::FileSystem => { - *self = BlobStoreBase::FileSystem(Default::default()) + BlobSwapStoreType::Default => *self = BlobSwapStore::Default, + BlobSwapStoreType::S3 => *self = BlobSwapStore::S3(Default::default()), + BlobSwapStoreType::Azure => *self = BlobSwapStore::Azure(Default::default()), + BlobSwapStoreType::FileSystem => { + *self = BlobSwapStore::FileSystem(Default::default()) } - BlobStoreBaseType::FoundationDb => { - *self = BlobStoreBase::FoundationDb(Default::default()) + BlobSwapStoreType::FoundationDb => { + *self = BlobSwapStore::FoundationDb(Default::default()) } - BlobStoreBaseType::PostgreSql => { - *self = BlobStoreBase::PostgreSql(Default::default()) + BlobSwapStoreType::PostgreSql => { + *self = BlobSwapStore::PostgreSql(Default::default()) } - BlobStoreBaseType::MySql => *self = BlobStoreBase::MySql(Default::default()), + BlobSwapStoreType::MySql => *self = BlobSwapStore::MySql(Default::default()), } } match self { - BlobStoreBase::S3(inner) => inner.patch(pointer, value), - BlobStoreBase::Azure(inner) => inner.patch(pointer, value), - BlobStoreBase::FileSystem(inner) => inner.patch(pointer, value), - BlobStoreBase::FoundationDb(inner) => inner.patch(pointer, value), - BlobStoreBase::PostgreSql(inner) => inner.patch(pointer, value), - BlobStoreBase::MySql(inner) => inner.patch(pointer, value), + BlobSwapStore::Default => pointer.assert_eof(), + BlobSwapStore::S3(inner) => inner.patch(pointer, value), + BlobSwapStore::Azure(inner) => inner.patch(pointer, value), + BlobSwapStore::FileSystem(inner) => inner.patch(pointer, value), + BlobSwapStore::FoundationDb(inner) => inner.patch(pointer, value), + BlobSwapStore::PostgreSql(inner) => inner.patch(pointer, value), + BlobSwapStore::MySql(inner) => inner.patch(pointer, value), } } } -impl BlobStoreBase { - pub fn object_type(&self) -> BlobStoreBaseType { +impl BlobSwapStore { + pub fn object_type(&self) -> BlobSwapStoreType { match self { - BlobStoreBase::S3(_) => BlobStoreBaseType::S3, - BlobStoreBase::Azure(_) => BlobStoreBaseType::Azure, - BlobStoreBase::FileSystem(_) => BlobStoreBaseType::FileSystem, - BlobStoreBase::FoundationDb(_) => BlobStoreBaseType::FoundationDb, - BlobStoreBase::PostgreSql(_) => BlobStoreBaseType::PostgreSql, - BlobStoreBase::MySql(_) => BlobStoreBaseType::MySql, + BlobSwapStore::Default => BlobSwapStoreType::Default, + BlobSwapStore::S3(_) => BlobSwapStoreType::S3, + BlobSwapStore::Azure(_) => BlobSwapStoreType::Azure, + BlobSwapStore::FileSystem(_) => BlobSwapStoreType::FileSystem, + BlobSwapStore::FoundationDb(_) => BlobSwapStoreType::FoundationDb, + BlobSwapStore::PostgreSql(_) => BlobSwapStoreType::PostgreSql, + BlobSwapStore::MySql(_) => BlobSwapStoreType::MySql, } } } @@ -4007,19 +4438,56 @@ impl ObjectImpl for BlockedIp { impl Pickle for BlockedIp { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.expires_at.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.address.pickle(out); self.reason.pickle(out); self.created_at.pickle(out); - self.expires_at.pickle(out); + if let Some(value) = &self.expires_at { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.address = Pickle::unpickle(stream)?; - this.reason = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: IpAddrOrMask = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: BlockReason = if block.has(1) { + Pickle::unpickle(stream)? + } else { + BlockReason::Manual + }; + let field2: UTCDateTime = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + address: field0, + reason: field1, + created_at: field2, + expires_at: field3, + }) } } @@ -4036,11 +4504,15 @@ impl Default for BlockedIp { impl IntoValue for BlockedIp { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Address, self.address.into_value()); - map.insert_unchecked(Property::Reason, self.reason.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); + map.insert_unchecked(Property::Address, self.address.into_value_with(opts)); + map.insert_unchecked(Property::Reason, self.reason.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -4066,15 +4538,15 @@ impl RegistryJsonPropertyPatch for BlockedIp { } impl ObjectImpl for Bootstrap { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const FLAGS: u64 = OBJ_SINGLETON | OBJ_NOT_STORED; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Bootstrap; fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.server_hostname; + let value = &self.default_hostname; if value.is_empty() { - errors.push(ValidationError::required(Property::ServerHostname)); + errors.push(ValidationError::required(Property::DefaultHostname)); } let value = &self.default_domain; if value.is_empty() { @@ -4098,44 +4570,13 @@ impl ObjectImpl for Bootstrap { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} -impl Pickle for Bootstrap { - fn pickle(&self, out: &mut Vec) { - self.server_hostname.pickle(out); - self.default_domain.pickle(out); - self.request_tls_certificate.pickle(out); - self.generate_dkim_keys.pickle(out); - self.data_store.pickle(out); - self.blob_store.pickle(out); - self.search_store.pickle(out); - self.in_memory_store.pickle(out); - self.directory.pickle(out); - self.tracer.pickle(out); - self.dns_server.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.server_hostname = Pickle::unpickle(stream)?; - this.default_domain = Pickle::unpickle(stream)?; - this.request_tls_certificate = Pickle::unpickle(stream)?; - this.generate_dkim_keys = Pickle::unpickle(stream)?; - this.data_store = Pickle::unpickle(stream)?; - this.blob_store = Pickle::unpickle(stream)?; - this.search_store = Pickle::unpickle(stream)?; - this.in_memory_store = Pickle::unpickle(stream)?; - this.directory = Pickle::unpickle(stream)?; - this.tracer = Pickle::unpickle(stream)?; - this.dns_server = Pickle::unpickle(stream)?; - Some(this) - } } impl Default for Bootstrap { fn default() -> Self { Self { - server_hostname: Default::default(), + default_hostname: Default::default(), default_domain: Default::default(), request_tls_certificate: true, generate_dkim_keys: true, @@ -4158,24 +4599,40 @@ impl Default for Bootstrap { impl IntoValue for Bootstrap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::ServerHostname, self.server_hostname.into_value()); - map.insert_unchecked(Property::DefaultDomain, self.default_domain.into_value()); + map.insert_unchecked( + Property::DefaultHostname, + self.default_hostname.into_value_with(opts), + ); + map.insert_unchecked( + Property::DefaultDomain, + self.default_domain.into_value_with(opts), + ); map.insert_unchecked( Property::RequestTlsCertificate, - self.request_tls_certificate.into_value(), + self.request_tls_certificate.into_value_with(opts), ); map.insert_unchecked( Property::GenerateDkimKeys, - self.generate_dkim_keys.into_value(), + self.generate_dkim_keys.into_value_with(opts), ); - map.insert_unchecked(Property::DataStore, self.data_store.into_value()); - map.insert_unchecked(Property::BlobStore, self.blob_store.into_value()); - map.insert_unchecked(Property::SearchStore, self.search_store.into_value()); - map.insert_unchecked(Property::InMemoryStore, self.in_memory_store.into_value()); - map.insert_unchecked(Property::Directory, self.directory.into_value()); - map.insert_unchecked(Property::Tracer, self.tracer.into_value()); - map.insert_unchecked(Property::DnsServer, self.dns_server.into_value()); + map.insert_unchecked(Property::DataStore, self.data_store.into_value_with(opts)); + map.insert_unchecked(Property::BlobStore, self.blob_store.into_value_with(opts)); + map.insert_unchecked( + Property::SearchStore, + self.search_store.into_value_with(opts), + ); + map.insert_unchecked( + Property::InMemoryStore, + self.in_memory_store.into_value_with(opts), + ); + map.insert_unchecked(Property::Directory, self.directory.into_value_with(opts)); + map.insert_unchecked(Property::Tracer, self.tracer.into_value_with(opts)); + map.insert_unchecked(Property::DnsServer, self.dns_server.into_value_with(opts)); JmapValue::Object(map) } } @@ -4187,7 +4644,7 @@ impl RegistryJsonPropertyPatch for Bootstrap { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ServerHostname) => self.server_hostname.patch(pointer, value), + Some(Property::DefaultHostname) => self.default_hostname.patch(pointer, value), Some(Property::DefaultDomain) => self.default_domain.patch(pointer, value), Some(Property::RequestTlsCertificate) => { self.request_tls_certificate.patch(pointer, value) @@ -4246,9 +4703,9 @@ impl ObjectImpl for Cache { if *value < 2048 { errors.push(ValidationError::min_value(Property::DnsPtr, 2048)); } - let value = &self.dns_rbl; + let value = &self.spam_lookup; if *value < 2048 { - errors.push(ValidationError::min_value(Property::DnsRbl, 2048)); + errors.push(ValidationError::min_value(Property::SpamLookup, 2048)); } let value = &self.dns_tlsa; if *value < 2048 { @@ -4312,10 +4769,6 @@ impl ObjectImpl for Cache { if *value < 2048 { errors.push(ValidationError::min_value(Property::Roles, 2048)); } - let value = &self.tenants; - if *value < 2048 { - errors.push(ValidationError::min_value(Property::Tenants, 2048)); - } let value = &self.mailing_lists; if *value < 2048 { errors.push(ValidationError::min_value(Property::MailingLists, 2048)); @@ -4324,14 +4777,25 @@ impl ObjectImpl for Cache { if *value < 2048 { errors.push(ValidationError::min_value(Property::DkimSignatures, 2048)); } + let value = &self.directory_recipients; + if *value < 2048 { + errors.push(ValidationError::min_value( + Property::DirectoryRecipients, + 2048, + )); + } + let value = &self.swap; + value.validate(errors); errors.len() == neb } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Pickle for Cache { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 33, 0, 0); self.access_tokens.pickle(out); self.contacts.pickle(out); self.dns_ipv4.pickle(out); @@ -4339,7 +4803,7 @@ impl Pickle for Cache { self.dns_mta_sts.pickle(out); self.dns_mx.pickle(out); self.dns_ptr.pickle(out); - self.dns_rbl.pickle(out); + self.spam_lookup.pickle(out); self.dns_tlsa.pickle(out); self.dns_txt.pickle(out); self.events.pickle(out); @@ -4347,6 +4811,10 @@ impl Pickle for Cache { self.files.pickle(out); self.http_auth.pickle(out); self.messages.pickle(out); + self.messages_large.pickle(out); + self.messages_large_ttl.pickle(out); + self.groupware_large.pickle(out); + self.groupware_large_ttl.pickle(out); self.domains.pickle(out); self.domain_names.pickle(out); self.domain_names_negative.pickle(out); @@ -4354,41 +4822,225 @@ impl Pickle for Cache { self.email_addresses_negative.pickle(out); self.accounts.pickle(out); self.roles.pickle(out); - self.tenants.pickle(out); + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } self.mailing_lists.pickle(out); self.dkim_signatures.pickle(out); self.negative_ttl.pickle(out); + self.directory_recipients.pickle(out); + self.directory_recipients_ttl.pickle(out); + self.swap.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_tokens = Pickle::unpickle(stream)?; - this.contacts = Pickle::unpickle(stream)?; - this.dns_ipv4 = Pickle::unpickle(stream)?; - this.dns_ipv6 = Pickle::unpickle(stream)?; - this.dns_mta_sts = Pickle::unpickle(stream)?; - this.dns_mx = Pickle::unpickle(stream)?; - this.dns_ptr = Pickle::unpickle(stream)?; - this.dns_rbl = Pickle::unpickle(stream)?; - this.dns_tlsa = Pickle::unpickle(stream)?; - this.dns_txt = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.scheduling = Pickle::unpickle(stream)?; - this.files = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.messages = Pickle::unpickle(stream)?; - this.domains = Pickle::unpickle(stream)?; - this.domain_names = Pickle::unpickle(stream)?; - this.domain_names_negative = Pickle::unpickle(stream)?; - this.email_addresses = Pickle::unpickle(stream)?; - this.email_addresses_negative = Pickle::unpickle(stream)?; - this.accounts = Pickle::unpickle(stream)?; - this.roles = Pickle::unpickle(stream)?; - this.tenants = Pickle::unpickle(stream)?; - this.mailing_lists = Pickle::unpickle(stream)?; - this.dkim_signatures = Pickle::unpickle(stream)?; - this.negative_ttl = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 10485760 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 33554432 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 6291456 + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 8388608 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 50331648 + }; + let field11: u64 = if block.has(11) { + Pickle::unpickle(stream)? + } else { + 6291456 + }; + let field12: u64 = if block.has(12) { + Pickle::unpickle(stream)? + } else { + 25165824 + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field14: u64 = if block.has(14) { + Pickle::unpickle(stream)? + } else { + 335544320 + }; + let field15: u64 = if block.has(15) { + Pickle::unpickle(stream)? + } else { + 536870912 + }; + let field16: Duration = if block.has(16) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field17: u64 = if block.has(17) { + Pickle::unpickle(stream)? + } else { + 536870912 + }; + let field18: Duration = if block.has(18) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field19: u64 = if block.has(19) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field20: u64 = if block.has(20) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field21: u64 = if block.has(21) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field22: u64 = if block.has(22) { + Pickle::unpickle(stream)? + } else { + 8388608 + }; + let field23: u64 = if block.has(23) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field24: u64 = if block.has(24) { + Pickle::unpickle(stream)? + } else { + 16777216 + }; + let field25: u64 = if block.has(25) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + #[cfg(not(feature = "enterprise"))] + if block.has(26) { + let block = stream.open_block()?; + stream.close_block(block); + } + let field27: u64 = if block.has(27) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field28: u64 = if block.has(28) { + Pickle::unpickle(stream)? + } else { + 8388608 + }; + let field29: Duration = if block.has(29) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field30: u64 = if block.has(30) { + Pickle::unpickle(stream)? + } else { + 2097152 + }; + let field31: Duration = if block.has(31) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field32: CacheSwap = if block.has(32) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_tokens: field0, + contacts: field1, + dns_ipv4: field2, + dns_ipv6: field3, + dns_mta_sts: field4, + dns_mx: field5, + dns_ptr: field6, + spam_lookup: field7, + dns_tlsa: field8, + dns_txt: field9, + events: field10, + scheduling: field11, + files: field12, + http_auth: field13, + messages: field14, + messages_large: field15, + messages_large_ttl: field16, + groupware_large: field17, + groupware_large_ttl: field18, + domains: field19, + domain_names: field20, + domain_names_negative: field21, + email_addresses: field22, + email_addresses_negative: field23, + accounts: field24, + roles: field25, + mailing_lists: field27, + dkim_signatures: field28, + negative_ttl: field29, + directory_recipients: field30, + directory_recipients_ttl: field31, + swap: field32, + }) } } @@ -4396,70 +5048,122 @@ impl Default for Cache { fn default() -> Self { Self { access_tokens: 10485760, - contacts: 10485760, - dns_ipv4: 5242880, - dns_ipv6: 5242880, + contacts: 33554432, + dns_ipv4: 2097152, + dns_ipv6: 1048576, dns_mta_sts: 1048576, - dns_mx: 5242880, + dns_mx: 2097152, dns_ptr: 1048576, - dns_rbl: 5242880, + spam_lookup: 6291456, dns_tlsa: 1048576, - dns_txt: 5242880, - events: 10485760, - scheduling: 1048576, - files: 10485760, - http_auth: 1048576, - messages: 52428800, - domains: 5242880, - domain_names: 10485760, - domain_names_negative: 1048576, - email_addresses: 10485760, + dns_txt: 8388608, + events: 50331648, + scheduling: 6291456, + files: 25165824, + http_auth: 2097152, + messages: 335544320, + messages_large: 536870912, + messages_large_ttl: Duration::from_millis(3600000), + groupware_large: 536870912, + groupware_large_ttl: Duration::from_millis(3600000), + domains: 2097152, + domain_names: 1048576, + domain_names_negative: 2097152, + email_addresses: 8388608, email_addresses_negative: 2097152, - accounts: 20971520, - roles: 5242880, - tenants: 5242880, + accounts: 16777216, + roles: 1048576, mailing_lists: 2097152, - dkim_signatures: 10485760, + dkim_signatures: 8388608, negative_ttl: Duration::from_millis(3600000), + directory_recipients: 2097152, + directory_recipients_ttl: Duration::from_millis(600000), + swap: Default::default(), } } } impl IntoValue for Cache { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(28); - map.insert_unchecked(Property::AccessTokens, self.access_tokens.into_value()); - map.insert_unchecked(Property::Contacts, self.contacts.into_value()); - map.insert_unchecked(Property::DnsIpv4, self.dns_ipv4.into_value()); - map.insert_unchecked(Property::DnsIpv6, self.dns_ipv6.into_value()); - map.insert_unchecked(Property::DnsMtaSts, self.dns_mta_sts.into_value()); - map.insert_unchecked(Property::DnsMx, self.dns_mx.into_value()); - map.insert_unchecked(Property::DnsPtr, self.dns_ptr.into_value()); - map.insert_unchecked(Property::DnsRbl, self.dns_rbl.into_value()); - map.insert_unchecked(Property::DnsTlsa, self.dns_tlsa.into_value()); - map.insert_unchecked(Property::DnsTxt, self.dns_txt.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::Scheduling, self.scheduling.into_value()); - map.insert_unchecked(Property::Files, self.files.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::Messages, self.messages.into_value()); - map.insert_unchecked(Property::Domains, self.domains.into_value()); - map.insert_unchecked(Property::DomainNames, self.domain_names.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(35); + map.insert_unchecked( + Property::AccessTokens, + self.access_tokens.into_value_with(opts), + ); + map.insert_unchecked(Property::Contacts, self.contacts.into_value_with(opts)); + map.insert_unchecked(Property::DnsIpv4, self.dns_ipv4.into_value_with(opts)); + map.insert_unchecked(Property::DnsIpv6, self.dns_ipv6.into_value_with(opts)); + map.insert_unchecked(Property::DnsMtaSts, self.dns_mta_sts.into_value_with(opts)); + map.insert_unchecked(Property::DnsMx, self.dns_mx.into_value_with(opts)); + map.insert_unchecked(Property::DnsPtr, self.dns_ptr.into_value_with(opts)); + map.insert_unchecked(Property::SpamLookup, self.spam_lookup.into_value_with(opts)); + map.insert_unchecked(Property::DnsTlsa, self.dns_tlsa.into_value_with(opts)); + map.insert_unchecked(Property::DnsTxt, self.dns_txt.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked(Property::Scheduling, self.scheduling.into_value_with(opts)); + map.insert_unchecked(Property::Files, self.files.into_value_with(opts)); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked(Property::Messages, self.messages.into_value_with(opts)); + map.insert_unchecked( + Property::MessagesLarge, + self.messages_large.into_value_with(opts), + ); + map.insert_unchecked( + Property::MessagesLargeTtl, + self.messages_large_ttl.into_value_with(opts), + ); + map.insert_unchecked( + Property::GroupwareLarge, + self.groupware_large.into_value_with(opts), + ); + map.insert_unchecked( + Property::GroupwareLargeTtl, + self.groupware_large_ttl.into_value_with(opts), + ); + map.insert_unchecked(Property::Domains, self.domains.into_value_with(opts)); + map.insert_unchecked( + Property::DomainNames, + self.domain_names.into_value_with(opts), + ); map.insert_unchecked( Property::DomainNamesNegative, - self.domain_names_negative.into_value(), + self.domain_names_negative.into_value_with(opts), + ); + map.insert_unchecked( + Property::EmailAddresses, + self.email_addresses.into_value_with(opts), ); - map.insert_unchecked(Property::EmailAddresses, self.email_addresses.into_value()); map.insert_unchecked( Property::EmailAddressesNegative, - self.email_addresses_negative.into_value(), + self.email_addresses_negative.into_value_with(opts), ); - map.insert_unchecked(Property::Accounts, self.accounts.into_value()); - map.insert_unchecked(Property::Roles, self.roles.into_value()); - map.insert_unchecked(Property::Tenants, self.tenants.into_value()); - map.insert_unchecked(Property::MailingLists, self.mailing_lists.into_value()); - map.insert_unchecked(Property::DkimSignatures, self.dkim_signatures.into_value()); - map.insert_unchecked(Property::NegativeTtl, self.negative_ttl.into_value()); + map.insert_unchecked(Property::Accounts, self.accounts.into_value_with(opts)); + map.insert_unchecked(Property::Roles, self.roles.into_value_with(opts)); + map.insert_unchecked( + Property::MailingLists, + self.mailing_lists.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignatures, + self.dkim_signatures.into_value_with(opts), + ); + map.insert_unchecked( + Property::NegativeTtl, + self.negative_ttl.into_value_with(opts), + ); + map.insert_unchecked( + Property::DirectoryRecipients, + self.directory_recipients.into_value_with(opts), + ); + map.insert_unchecked( + Property::DirectoryRecipientsTtl, + self.directory_recipients_ttl.into_value_with(opts), + ); + map.insert_unchecked(Property::Swap, self.swap.into_value_with(opts)); JmapValue::Object(map) } } @@ -4478,7 +5182,7 @@ impl RegistryJsonPropertyPatch for Cache { Some(Property::DnsMtaSts) => self.dns_mta_sts.patch(pointer, value), Some(Property::DnsMx) => self.dns_mx.patch(pointer, value), Some(Property::DnsPtr) => self.dns_ptr.patch(pointer, value), - Some(Property::DnsRbl) => self.dns_rbl.patch(pointer, value), + Some(Property::SpamLookup) => self.spam_lookup.patch(pointer, value), Some(Property::DnsTlsa) => self.dns_tlsa.patch(pointer, value), Some(Property::DnsTxt) => self.dns_txt.patch(pointer, value), Some(Property::Events) => self.events.patch(pointer, value), @@ -4486,6 +5190,10 @@ impl RegistryJsonPropertyPatch for Cache { Some(Property::Files) => self.files.patch(pointer, value), Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), Some(Property::Messages) => self.messages.patch(pointer, value), + Some(Property::MessagesLarge) => self.messages_large.patch(pointer, value), + Some(Property::MessagesLargeTtl) => self.messages_large_ttl.patch(pointer, value), + Some(Property::GroupwareLarge) => self.groupware_large.patch(pointer, value), + Some(Property::GroupwareLargeTtl) => self.groupware_large_ttl.patch(pointer, value), Some(Property::Domains) => self.domains.patch(pointer, value), Some(Property::DomainNames) => self.domain_names.patch(pointer, value), Some(Property::DomainNamesNegative) => self.domain_names_negative.patch(pointer, value), @@ -4495,10 +5203,14 @@ impl RegistryJsonPropertyPatch for Cache { } Some(Property::Accounts) => self.accounts.patch(pointer, value), Some(Property::Roles) => self.roles.patch(pointer, value), - Some(Property::Tenants) => self.tenants.patch(pointer, value), Some(Property::MailingLists) => self.mailing_lists.patch(pointer, value), Some(Property::DkimSignatures) => self.dkim_signatures.patch(pointer, value), Some(Property::NegativeTtl) => self.negative_ttl.patch(pointer, value), + Some(Property::DirectoryRecipients) => self.directory_recipients.patch(pointer, value), + Some(Property::DirectoryRecipientsTtl) => { + self.directory_recipients_ttl.patch(pointer, value) + } + Some(Property::Swap) => self.swap.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -4508,6 +5220,127 @@ impl RegistryJsonPropertyPatch for Cache { } } +impl CacheSwap { + fn validate(&self, errors: &mut Vec) -> bool { + match self { + CacheSwap::Disabled => true, + CacheSwap::LocalFile(inner) => inner.validate(errors), + CacheSwap::Redis(inner) => inner.validate(errors), + CacheSwap::BlobStore(inner) => inner.validate(errors), + } + } + +} + +impl Default for CacheSwap { + fn default() -> Self { + CacheSwap::Disabled + } +} + +impl Pickle for CacheSwap { + fn pickle(&self, out: &mut Vec) { + match self { + CacheSwap::Disabled => { + 0u16.pickle(out); + } + CacheSwap::LocalFile(inner) => { + 1u16.pickle(out); + inner.pickle(out); + } + CacheSwap::Redis(inner) => { + 2u16.pickle(out); + inner.pickle(out); + } + CacheSwap::BlobStore(inner) => { + 3u16.pickle(out); + inner.pickle(out); + } + } + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + match u16::unpickle(stream)? { + 0 => Some(CacheSwap::Disabled), + 1 => Pickle::unpickle(stream).map(CacheSwap::LocalFile), + 2 => Pickle::unpickle(stream).map(CacheSwap::Redis), + 3 => Pickle::unpickle(stream).map(CacheSwap::BlobStore), + _ => None, + } + } +} + +impl IntoValue for CacheSwap { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + match self { + CacheSwap::Disabled => { + let mut obj = jmap_tools::Map::new(); + obj.insert_unchecked(Property::Type, JmapValue::Str("Disabled".into())); + JmapValue::Object(obj) + } + CacheSwap::LocalFile(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("LocalFile".into())); + obj + } + CacheSwap::Redis(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); + obj + } + CacheSwap::BlobStore(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("BlobStore".into())); + obj + } + } + } +} + +impl RegistryJsonPatch for CacheSwap { + fn patch<'x>( + &mut self, + pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + if !pointer.has_next() { + match object_type(&pointer, &value)? { + CacheSwapType::Disabled => *self = CacheSwap::Disabled, + CacheSwapType::LocalFile => *self = CacheSwap::LocalFile(Default::default()), + CacheSwapType::Redis => *self = CacheSwap::Redis(Default::default()), + CacheSwapType::BlobStore => *self = CacheSwap::BlobStore(Default::default()), + } + } + match self { + CacheSwap::Disabled => pointer.assert_eof(), + CacheSwap::LocalFile(inner) => inner.patch(pointer, value), + CacheSwap::Redis(inner) => inner.patch(pointer, value), + CacheSwap::BlobStore(inner) => inner.patch(pointer, value), + } + } +} + +impl CacheSwap { + pub fn object_type(&self) -> CacheSwapType { + match self { + CacheSwap::Disabled => CacheSwapType::Disabled, + CacheSwap::LocalFile(_) => CacheSwapType::LocalFile, + CacheSwap::Redis(_) => CacheSwapType::Redis, + CacheSwap::BlobStore(_) => CacheSwapType::BlobStore, + } + } +} + impl ObjectImpl for Calendar { const FLAGS: u64 = OBJ_SINGLETON; const VERSION: u8 = 0; @@ -4555,6 +5388,16 @@ impl ObjectImpl for Calendar { )); } } + let value = &self.max_calendars_per_event; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxCalendarsPerEvent, + 1, + )); + } + if let Some(value) = &self.availability_rate_limit { + value.validate(errors); + } errors.len() == neb } @@ -4563,29 +5406,183 @@ impl ObjectImpl for Calendar { impl Pickle for Calendar { fn pickle(&self, out: &mut Vec) { - self.default_display_name.pickle(out); - self.default_href_name.pickle(out); + let mut bitmap = 0u64; + if self.default_display_name.is_some() { + bitmap |= 1 << 0; + } + if self.default_href_name.is_some() { + bitmap |= 1 << 1; + } + if self.max_calendars.is_some() { + bitmap |= 1 << 2; + } + if self.max_events.is_some() { + bitmap |= 1 << 3; + } + if self.max_participant_identities.is_some() { + bitmap |= 1 << 4; + } + if self.max_event_notifications.is_some() { + bitmap |= 1 << 5; + } + if self.availability_rate_limit.is_some() { + bitmap |= 1 << 6; + } + let block = crate::pickle::begin_block(out, 14, 7, bitmap); + if let Some(value) = &self.default_display_name { + value.pickle(out); + } + if let Some(value) = &self.default_href_name { + value.pickle(out); + } self.max_attendees.pickle(out); self.max_recurrence_expansions.pickle(out); self.max_i_calendar_size.pickle(out); - self.max_calendars.pickle(out); - self.max_events.pickle(out); - self.max_participant_identities.pickle(out); - self.max_event_notifications.pickle(out); + if let Some(value) = &self.max_calendars { + value.pickle(out); + } + if let Some(value) = &self.max_events { + value.pickle(out); + } + if let Some(value) = &self.max_participant_identities { + value.pickle(out); + } + if let Some(value) = &self.max_event_notifications { + value.pickle(out); + } + self.max_calendars_per_event.pickle(out); + self.max_expanded_query_duration.pickle(out); + self.max_availability_duration.pickle(out); + if let Some(value) = &self.availability_rate_limit { + value.pickle(out); + } + self.max_attachments_size.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.default_display_name = Pickle::unpickle(stream)?; - this.default_href_name = Pickle::unpickle(stream)?; - this.max_attendees = Pickle::unpickle(stream)?; - this.max_recurrence_expansions = Pickle::unpickle(stream)?; - this.max_i_calendar_size = Pickle::unpickle(stream)?; - this.max_calendars = Pickle::unpickle(stream)?; - this.max_events = Pickle::unpickle(stream)?; - this.max_participant_identities = Pickle::unpickle(stream)?; - this.max_event_notifications = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("Stalwart Calendar".to_string()) + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("default".to_string()) + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 20u64 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 3000u64 + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 524288 + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(250u64) + }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(100u64) + }; + let field8: Option = if block.has(8) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(31536000000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(31536000000) + }; + let field12: Option = if block.has(12) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 100u64, + period: Duration::from_millis(60000), + }) + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 5242880 + }; + stream.close_block(block); + Some(Self { + default_display_name: field0, + default_href_name: field1, + max_attendees: field2, + max_recurrence_expansions: field3, + max_i_calendar_size: field4, + max_calendars: field5, + max_events: field6, + max_participant_identities: field7, + max_event_notifications: field8, + max_calendars_per_event: field9, + max_expanded_query_duration: field10, + max_availability_duration: field11, + availability_rate_limit: field12, + max_attachments_size: field13, + }) } } @@ -4601,39 +5598,77 @@ impl Default for Calendar { max_events: Default::default(), max_participant_identities: Some(100u64), max_event_notifications: Default::default(), + max_calendars_per_event: 10u64, + max_expanded_query_duration: Duration::from_millis(31536000000), + max_availability_duration: Duration::from_millis(31536000000), + availability_rate_limit: Some(Rate { + count: 100u64, + period: Duration::from_millis(60000), + }), + max_attachments_size: 5242880, } } } impl IntoValue for Calendar { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); map.insert_unchecked( Property::DefaultDisplayName, - self.default_display_name.into_value(), + self.default_display_name.into_value_with(opts), ); map.insert_unchecked( Property::DefaultHrefName, - self.default_href_name.into_value(), + self.default_href_name.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAttendees, + self.max_attendees.into_value_with(opts), ); - map.insert_unchecked(Property::MaxAttendees, self.max_attendees.into_value()); map.insert_unchecked( Property::MaxRecurrenceExpansions, - self.max_recurrence_expansions.into_value(), + self.max_recurrence_expansions.into_value_with(opts), ); map.insert_unchecked( Property::MaxICalendarSize, - self.max_i_calendar_size.into_value(), + self.max_i_calendar_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxCalendars, self.max_calendars.into_value()); - map.insert_unchecked(Property::MaxEvents, self.max_events.into_value()); + map.insert_unchecked( + Property::MaxCalendars, + self.max_calendars.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxEvents, self.max_events.into_value_with(opts)); map.insert_unchecked( Property::MaxParticipantIdentities, - self.max_participant_identities.into_value(), + self.max_participant_identities.into_value_with(opts), ); map.insert_unchecked( Property::MaxEventNotifications, - self.max_event_notifications.into_value(), + self.max_event_notifications.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxCalendarsPerEvent, + self.max_calendars_per_event.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExpandedQueryDuration, + self.max_expanded_query_duration.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAvailabilityDuration, + self.max_availability_duration.into_value_with(opts), + ); + map.insert_unchecked( + Property::AvailabilityRateLimit, + self.availability_rate_limit.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAttachmentsSize, + self.max_attachments_size.into_value_with(opts), ); JmapValue::Object(map) } @@ -4665,6 +5700,19 @@ impl RegistryJsonPropertyPatch for Calendar { Some(Property::MaxEventNotifications) => { self.max_event_notifications.patch(pointer, value) } + Some(Property::MaxCalendarsPerEvent) => { + self.max_calendars_per_event.patch(pointer, value) + } + Some(Property::MaxExpandedQueryDuration) => { + self.max_expanded_query_duration.patch(pointer, value) + } + Some(Property::MaxAvailabilityDuration) => { + self.max_availability_duration.patch(pointer, value) + } + Some(Property::AvailabilityRateLimit) => { + self.availability_rate_limit.patch(pointer, value) + } + Some(Property::MaxAttachmentsSize) => self.max_attachments_size.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -4681,20 +5729,15 @@ impl ObjectImpl for CalendarAlarm { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - if let Some(value) = &self.from_email { + if let Some(value) = &self.from_address { if value.is_empty() { - errors.push(ValidationError::required(Property::FromEmail)); + errors.push(ValidationError::required(Property::FromAddress)); } } let value = &self.from_name; if value.is_empty() { errors.push(ValidationError::required(Property::FromName)); } - if let Some(value) = &self.template { - if value.is_empty() { - errors.push(ValidationError::required(Property::Template)); - } - } errors.len() == neb } @@ -4703,23 +5746,59 @@ impl ObjectImpl for CalendarAlarm { impl Pickle for CalendarAlarm { fn pickle(&self, out: &mut Vec) { - self.allow_external_rcpts.pickle(out); - self.enable.pickle(out); - self.from_email.pickle(out); + let mut bitmap = 0u64; + if self.allow_external_rcpts { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + if self.from_address.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 6, 4, bitmap); + if let Some(value) = &self.from_address { + value.pickle(out); + } self.from_name.pickle(out); self.min_trigger_interval.pickle(out); - self.template.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_external_rcpts = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.from_email = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.min_trigger_interval = Pickle::unpickle(stream)?; - this.template = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { false }; + let field1: bool = if block.has(1) { block.flag(1) } else { true }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + "Stalwart Calendar".to_string() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + stream.close_block(block); + Some(Self { + allow_external_rcpts: field0, + enable: field1, + from_address: field2, + from_name: field3, + min_trigger_interval: field4, + }) } } @@ -4728,29 +5807,34 @@ impl Default for CalendarAlarm { Self { allow_external_rcpts: false, enable: true, - from_email: Default::default(), + from_address: Default::default(), from_name: "Stalwart Calendar".to_string(), min_trigger_interval: Duration::from_millis(3600000), - template: Default::default(), } } } impl IntoValue for CalendarAlarm { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); map.insert_unchecked( Property::AllowExternalRcpts, - self.allow_external_rcpts.into_value(), + self.allow_external_rcpts.into_value_with(opts), ); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::FromEmail, self.from_email.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::FromAddress, + self.from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FromName, self.from_name.into_value_with(opts)); map.insert_unchecked( Property::MinTriggerInterval, - self.min_trigger_interval.into_value(), + self.min_trigger_interval.into_value_with(opts), ); - map.insert_unchecked(Property::Template, self.template.into_value()); JmapValue::Object(map) } } @@ -4764,14 +5848,13 @@ impl RegistryJsonPropertyPatch for CalendarAlarm { match pointer.next_property() { Some(Property::AllowExternalRcpts) => self.allow_external_rcpts.patch(pointer, value), Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::FromEmail) => self - .from_email + Some(Property::FromAddress) => self + .from_address .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::FromName) => self .from_name .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::MinTriggerInterval) => self.min_trigger_interval.patch(pointer, value), - Some(Property::Template) => self.template.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -4793,24 +5876,14 @@ impl ObjectImpl for CalendarScheduling { errors.push(ValidationError::required(Property::HttpRsvpUrl)); } } - let value = &self.itip_max_size; + let value = &self.max_itip_size; if *value < 100 { - errors.push(ValidationError::min_value(Property::ItipMaxSize, 100)); + errors.push(ValidationError::min_value(Property::MaxItipSize, 100)); } let value = &self.max_recipients; if *value < 1 { errors.push(ValidationError::min_value(Property::MaxRecipients, 1)); } - if let Some(value) = &self.email_template { - if value.is_empty() { - errors.push(ValidationError::required(Property::EmailTemplate)); - } - } - if let Some(value) = &self.http_rsvp_template { - if value.is_empty() { - errors.push(ValidationError::required(Property::HttpRsvpTemplate)); - } - } errors.len() == neb } @@ -4819,29 +5892,71 @@ impl ObjectImpl for CalendarScheduling { impl Pickle for CalendarScheduling { fn pickle(&self, out: &mut Vec) { - self.enable.pickle(out); - self.http_rsvp_enable.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.http_rsvp_enable { + bitmap |= 1 << 1; + } + if self.http_rsvp_url.is_some() { + bitmap |= 1 << 2; + } + if self.auto_add_invitations { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 9, 6, bitmap); self.http_rsvp_link_expiry.pickle(out); - self.http_rsvp_url.pickle(out); - self.auto_add_invitations.pickle(out); - self.itip_max_size.pickle(out); + if let Some(value) = &self.http_rsvp_url { + value.pickle(out); + } + self.max_itip_size.pickle(out); self.max_recipients.pickle(out); - self.email_template.pickle(out); - self.http_rsvp_template.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable = Pickle::unpickle(stream)?; - this.http_rsvp_enable = Pickle::unpickle(stream)?; - this.http_rsvp_link_expiry = Pickle::unpickle(stream)?; - this.http_rsvp_url = Pickle::unpickle(stream)?; - this.auto_add_invitations = Pickle::unpickle(stream)?; - this.itip_max_size = Pickle::unpickle(stream)?; - this.max_recipients = Pickle::unpickle(stream)?; - this.email_template = Pickle::unpickle(stream)?; - this.http_rsvp_template = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: bool = if block.has(1) { block.flag(1) } else { true }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(7776000000) + }; + let field3: Option = if block.has(3) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(3) } else { false }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 524288 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + stream.close_block(block); + Some(Self { + enable: field0, + http_rsvp_enable: field1, + http_rsvp_link_expiry: field2, + http_rsvp_url: field3, + auto_add_invitations: field4, + max_itip_size: field5, + max_recipients: field6, + }) } } @@ -4853,34 +5968,43 @@ impl Default for CalendarScheduling { http_rsvp_link_expiry: Duration::from_millis(7776000000), http_rsvp_url: Default::default(), auto_add_invitations: false, - itip_max_size: 524288, + max_itip_size: 524288, max_recipients: 100u64, - email_template: Default::default(), - http_rsvp_template: Default::default(), } } } impl IntoValue for CalendarScheduling { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::HttpRsvpEnable, self.http_rsvp_enable.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::HttpRsvpEnable, + self.http_rsvp_enable.into_value_with(opts), + ); map.insert_unchecked( Property::HttpRsvpLinkExpiry, - self.http_rsvp_link_expiry.into_value(), + self.http_rsvp_link_expiry.into_value_with(opts), + ); + map.insert_unchecked( + Property::HttpRsvpUrl, + self.http_rsvp_url.into_value_with(opts), ); - map.insert_unchecked(Property::HttpRsvpUrl, self.http_rsvp_url.into_value()); map.insert_unchecked( Property::AutoAddInvitations, - self.auto_add_invitations.into_value(), + self.auto_add_invitations.into_value_with(opts), ); - map.insert_unchecked(Property::ItipMaxSize, self.itip_max_size.into_value()); - map.insert_unchecked(Property::MaxRecipients, self.max_recipients.into_value()); - map.insert_unchecked(Property::EmailTemplate, self.email_template.into_value()); map.insert_unchecked( - Property::HttpRsvpTemplate, - self.http_rsvp_template.into_value(), + Property::MaxItipSize, + self.max_itip_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRecipients, + self.max_recipients.into_value_with(opts), ); JmapValue::Object(map) } @@ -4900,10 +6024,8 @@ impl RegistryJsonPropertyPatch for CalendarScheduling { .http_rsvp_url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::AutoAddInvitations) => self.auto_add_invitations.patch(pointer, value), - Some(Property::ItipMaxSize) => self.itip_max_size.patch(pointer, value), + Some(Property::MaxItipSize) => self.max_itip_size.patch(pointer, value), Some(Property::MaxRecipients) => self.max_recipients.patch(pointer, value), - Some(Property::EmailTemplate) => self.email_template.patch(pointer, value), - Some(Property::HttpRsvpTemplate) => self.http_rsvp_template.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -4954,23 +6076,60 @@ impl ObjectImpl for Certificate { impl Pickle for Certificate { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 6, 0, 0); self.certificate.pickle(out); self.private_key.pickle(out); self.subject_alternative_names.pickle(out); self.not_valid_after.pickle(out); self.not_valid_before.pickle(out); self.issuer.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.certificate = Pickle::unpickle(stream)?; - this.private_key = Pickle::unpickle(stream)?; - this.subject_alternative_names = Pickle::unpickle(stream)?; - this.not_valid_after = Pickle::unpickle(stream)?; - this.not_valid_before = Pickle::unpickle(stream)?; - this.issuer = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: PublicText = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretText = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + certificate: field0, + private_key: field1, + subject_alternative_names: field2, + not_valid_after: field3, + not_valid_before: field4, + issuer: field5, + }) } } @@ -4989,16 +6148,29 @@ impl Default for Certificate { impl IntoValue for Certificate { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Certificate, self.certificate.into_value()); - map.insert_unchecked(Property::PrivateKey, self.private_key.into_value()); + map.insert_unchecked( + Property::Certificate, + self.certificate.into_value_with(opts), + ); + map.insert_unchecked(Property::PrivateKey, self.private_key.into_value_with(opts)); map.insert_unchecked( Property::SubjectAlternativeNames, - self.subject_alternative_names.into_value(), + self.subject_alternative_names.into_value_with(opts), ); - map.insert_unchecked(Property::NotValidAfter, self.not_valid_after.into_value()); - map.insert_unchecked(Property::NotValidBefore, self.not_valid_before.into_value()); - map.insert_unchecked(Property::Issuer, self.issuer.into_value()); + map.insert_unchecked( + Property::NotValidAfter, + self.not_valid_after.into_value_with(opts), + ); + map.insert_unchecked( + Property::NotValidBefore, + self.not_valid_before.into_value_with(opts), + ); + map.insert_unchecked(Property::Issuer, self.issuer.into_value_with(opts)); JmapValue::Object(map) } } @@ -5073,6 +6245,10 @@ impl Pickle for CertificateManagement { impl IntoValue for CertificateManagement { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { CertificateManagement::Manual => { let mut obj = jmap_tools::Map::new(); @@ -5080,7 +6256,7 @@ impl IntoValue for CertificateManagement { JmapValue::Object(obj) } CertificateManagement::Automatic(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Automatic".into())); @@ -5143,15 +6319,29 @@ impl CertificateManagementProperties { impl Pickle for CertificateManagementProperties { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.acme_provider_id.pickle(out); self.subject_alternative_names.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.acme_provider_id = Pickle::unpickle(stream)?; - this.subject_alternative_names = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + acme_provider_id: field0, + subject_alternative_names: field1, + }) } } @@ -5166,11 +6356,18 @@ impl Default for CertificateManagementProperties { impl IntoValue for CertificateManagementProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::AcmeProviderId, self.acme_provider_id.into_value()); + map.insert_unchecked( + Property::AcmeProviderId, + self.acme_provider_id.into_value_with(opts), + ); map.insert_unchecked( Property::SubjectAlternativeNames, - self.subject_alternative_names.into_value(), + self.subject_alternative_names.into_value_with(opts), ); JmapValue::Object(map) } @@ -5259,6 +6456,10 @@ impl Pickle for ClusterListenerGroup { impl IntoValue for ClusterListenerGroup { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { ClusterListenerGroup::EnableAll => { let mut obj = jmap_tools::Map::new(); @@ -5271,14 +6472,14 @@ impl IntoValue for ClusterListenerGroup { JmapValue::Object(obj) } ClusterListenerGroup::EnableSome(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnableSome".into())); obj } ClusterListenerGroup::DisableSome(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DisableSome".into())); @@ -5347,13 +6548,22 @@ impl ClusterListenerGroupProperties { impl Pickle for ClusterListenerGroupProperties { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.listener_ids.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.listener_ids = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + listener_ids: field0, + }) } } @@ -5367,8 +6577,15 @@ impl Default for ClusterListenerGroupProperties { impl IntoValue for ClusterListenerGroupProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::ListenerIds, self.listener_ids.into_value()); + map.insert_unchecked( + Property::ListenerIds, + self.listener_ids.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -5391,7 +6608,7 @@ impl RegistryJsonPropertyPatch for ClusterListenerGroupProperties { } impl ObjectImpl for ClusterNode { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED | OBJ_NO_CREATE | OBJ_NO_UPDATE | OBJ_NO_DESTROY; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::ClusterNode; @@ -5411,24 +6628,6 @@ impl ObjectImpl for ClusterNode { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for ClusterNode { - fn pickle(&self, out: &mut Vec) { - self.node_id.pickle(out); - self.hostname.pickle(out); - self.last_renewal.pickle(out); - self.status.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.node_id = Pickle::unpickle(stream)?; - this.hostname = Pickle::unpickle(stream)?; - this.last_renewal = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for ClusterNode { fn default() -> Self { Self { @@ -5442,11 +6641,18 @@ impl Default for ClusterNode { impl IntoValue for ClusterNode { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::NodeId, self.node_id.into_value()); - map.insert_unchecked(Property::Hostname, self.hostname.into_value()); - map.insert_unchecked(Property::LastRenewal, self.last_renewal.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::NodeId, self.node_id.into_value_with(opts)); + map.insert_unchecked(Property::Hostname, self.hostname.into_value_with(opts)); + map.insert_unchecked( + Property::LastRenewal, + self.last_renewal.into_value_with(opts), + ); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -5502,19 +6708,56 @@ impl ObjectImpl for ClusterRole { impl Pickle for ClusterRole { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.tasks.pickle(out); self.listeners.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.tasks = Pickle::unpickle(stream)?; - this.listeners = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: ClusterTaskGroup = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: ClusterListenerGroup = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + tasks: field2, + listeners: field3, + }) } } @@ -5531,11 +6774,18 @@ impl Default for ClusterRole { impl IntoValue for ClusterRole { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Tasks, self.tasks.into_value()); - map.insert_unchecked(Property::Listeners, self.listeners.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Tasks, self.tasks.into_value_with(opts)); + map.insert_unchecked(Property::Listeners, self.listeners.into_value_with(opts)); JmapValue::Object(map) } } @@ -5610,6 +6860,10 @@ impl Pickle for ClusterTaskGroup { impl IntoValue for ClusterTaskGroup { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { ClusterTaskGroup::EnableAll => { let mut obj = jmap_tools::Map::new(); @@ -5622,14 +6876,14 @@ impl IntoValue for ClusterTaskGroup { JmapValue::Object(obj) } ClusterTaskGroup::EnableSome(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnableSome".into())); obj } ClusterTaskGroup::DisableSome(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DisableSome".into())); @@ -5685,13 +6939,20 @@ impl ClusterTaskGroupProperties { impl Pickle for ClusterTaskGroupProperties { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.task_types.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.task_types = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { task_types: field0 }) } } @@ -5705,8 +6966,12 @@ impl Default for ClusterTaskGroupProperties { impl IntoValue for ClusterTaskGroupProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::TaskTypes, self.task_types.into_value()); + map.insert_unchecked(Property::TaskTypes, self.task_types.into_value_with(opts)); JmapValue::Object(map) } } @@ -5747,6 +7012,7 @@ impl ObjectImpl for Coordinator { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for Coordinator { @@ -5792,6 +7058,9 @@ impl Pickle for Coordinator { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Some(Coordinator::Disabled), 1 => Some(Coordinator::Default), @@ -5808,6 +7077,10 @@ impl Pickle for Coordinator { impl IntoValue for Coordinator { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Coordinator::Disabled => { let mut obj = jmap_tools::Map::new(); @@ -5820,42 +7093,42 @@ impl IntoValue for Coordinator { JmapValue::Object(obj) } Coordinator::Kafka(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Kafka".into())); obj } Coordinator::Nats(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Nats".into())); obj } Coordinator::Zenoh(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Zenoh".into())); obj } Coordinator::Redis(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); obj } Coordinator::RedisCluster(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisCluster".into())); obj } Coordinator::RedisSentinel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisSentinel".into())); @@ -5961,23 +7234,27 @@ impl Pickle for Credential { impl IntoValue for Credential { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Credential::Password(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Password".into())); obj } Credential::AppPassword(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("AppPassword".into())); obj } Credential::ApiKey(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ApiKey".into())); @@ -6063,6 +7340,10 @@ impl Pickle for CredentialPermissions { impl IntoValue for CredentialPermissions { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { CredentialPermissions::Inherit => { let mut obj = jmap_tools::Map::new(); @@ -6070,14 +7351,14 @@ impl IntoValue for CredentialPermissions { JmapValue::Object(obj) } CredentialPermissions::Disable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Disable".into())); obj } CredentialPermissions::Replace(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Replace".into())); @@ -6130,13 +7411,22 @@ impl CredentialPermissionsList { impl Pickle for CredentialPermissionsList { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.permissions.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.permissions = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + permissions: field0, + }) } } @@ -6150,8 +7440,15 @@ impl Default for CredentialPermissionsList { impl IntoValue for CredentialPermissionsList { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -6219,23 +7516,27 @@ impl Pickle for Cron { impl IntoValue for Cron { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Cron::Daily(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Daily".into())); obj } Cron::Weekly(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Weekly".into())); obj } Cron::Hourly(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hourly".into())); @@ -6293,15 +7594,29 @@ impl CronDaily { impl Pickle for CronDaily { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.hour.pickle(out); self.minute.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.hour = Pickle::unpickle(stream)?; - this.minute = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + stream.close_block(block); + Some(Self { + hour: field0, + minute: field1, + }) } } @@ -6316,9 +7631,13 @@ impl Default for CronDaily { impl IntoValue for CronDaily { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Hour, self.hour.into_value()); - map.insert_unchecked(Property::Minute, self.minute.into_value()); + map.insert_unchecked(Property::Hour, self.hour.into_value_with(opts)); + map.insert_unchecked(Property::Minute, self.minute.into_value_with(opts)); JmapValue::Object(map) } } @@ -6354,13 +7673,20 @@ impl CronHourly { impl Pickle for CronHourly { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.minute.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.minute = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + stream.close_block(block); + Some(Self { minute: field0 }) } } @@ -6372,8 +7698,12 @@ impl Default for CronHourly { impl IntoValue for CronHourly { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Minute, self.minute.into_value()); + map.insert_unchecked(Property::Minute, self.minute.into_value_with(opts)); JmapValue::Object(map) } } @@ -6416,17 +7746,36 @@ impl CronWeekly { impl Pickle for CronWeekly { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.day.pickle(out); self.hour.pickle(out); self.minute.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.day = Pickle::unpickle(stream)?; - this.hour = Pickle::unpickle(stream)?; - this.minute = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + stream.close_block(block); + Some(Self { + day: field0, + hour: field1, + minute: field2, + }) } } @@ -6442,10 +7791,14 @@ impl Default for CronWeekly { impl IntoValue for CronWeekly { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Day, self.day.into_value()); - map.insert_unchecked(Property::Hour, self.hour.into_value()); - map.insert_unchecked(Property::Minute, self.minute.into_value()); + map.insert_unchecked(Property::Day, self.day.into_value_with(opts)); + map.insert_unchecked(Property::Hour, self.hour.into_value_with(opts)); + map.insert_unchecked(Property::Minute, self.minute.into_value_with(opts)); JmapValue::Object(map) } } @@ -6490,13 +7843,20 @@ impl CustomRoles { impl Pickle for CustomRoles { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.role_ids.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.role_ids = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { role_ids: field0 }) } } @@ -6510,8 +7870,12 @@ impl Default for CustomRoles { impl IntoValue for CustomRoles { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::RoleIds, self.role_ids.into_value()); + map.insert_unchecked(Property::RoleIds, self.role_ids.into_value_with(opts)); JmapValue::Object(map) } } @@ -6546,8 +7910,6 @@ impl ObjectImpl for DataRetention { value.validate(errors); let value = &self.blob_cleanup_schedule; value.validate(errors); - let value = &self.metrics_collection_interval; - value.validate(errors); errors.len() == neb } @@ -6556,39 +7918,155 @@ impl ObjectImpl for DataRetention { impl Pickle for DataRetention { fn pickle(&self, out: &mut Vec) { - self.expunge_trash_after.pickle(out); - self.expunge_submissions_after.pickle(out); - self.expunge_share_notify_after.pickle(out); - self.expunge_scheduling_inbox_after.pickle(out); + let mut bitmap = 0u64; + if self.expunge_trash_after.is_some() { + bitmap |= 1 << 0; + } + if self.expunge_submissions_after.is_some() { + bitmap |= 1 << 1; + } + if self.expunge_share_notify_after.is_some() { + bitmap |= 1 << 2; + } + if self.expunge_scheduling_inbox_after.is_some() { + bitmap |= 1 << 3; + } + if self.max_changes_history.is_some() { + bitmap |= 1 << 4; + } + if self.hold_mta_reports_for.is_some() { + bitmap |= 1 << 7; + } + let block = crate::pickle::begin_block(out, 14, 10, bitmap); + if let Some(value) = &self.expunge_trash_after { + value.pickle(out); + } + if let Some(value) = &self.expunge_submissions_after { + value.pickle(out); + } + if let Some(value) = &self.expunge_share_notify_after { + value.pickle(out); + } + if let Some(value) = &self.expunge_scheduling_inbox_after { + value.pickle(out); + } self.expunge_schedule.pickle(out); self.data_cleanup_schedule.pickle(out); self.blob_cleanup_schedule.pickle(out); - self.max_changes_history.pickle(out); - self.archive_deleted_items_for.pickle(out); - self.archive_deleted_accounts_for.pickle(out); - self.hold_mta_reports_for.pickle(out); - self.hold_traces_for.pickle(out); - self.hold_metrics_for.pickle(out); - self.metrics_collection_interval.pickle(out); + if let Some(value) = &self.max_changes_history { + value.pickle(out); + } + if let Some(value) = &self.hold_mta_reports_for { + value.pickle(out); + } + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.expunge_trash_after = Pickle::unpickle(stream)?; - this.expunge_submissions_after = Pickle::unpickle(stream)?; - this.expunge_share_notify_after = Pickle::unpickle(stream)?; - this.expunge_scheduling_inbox_after = Pickle::unpickle(stream)?; - this.expunge_schedule = Pickle::unpickle(stream)?; - this.data_cleanup_schedule = Pickle::unpickle(stream)?; - this.blob_cleanup_schedule = Pickle::unpickle(stream)?; - this.max_changes_history = Pickle::unpickle(stream)?; - this.archive_deleted_items_for = Pickle::unpickle(stream)?; - this.archive_deleted_accounts_for = Pickle::unpickle(stream)?; - this.hold_mta_reports_for = Pickle::unpickle(stream)?; - this.hold_traces_for = Pickle::unpickle(stream)?; - this.hold_metrics_for = Pickle::unpickle(stream)?; - this.metrics_collection_interval = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(2592000000)) + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(259200000)) + }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(2592000000)) + }; + let field3: Option = if block.has(3) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(2592000000)) + }; + let field4: Cron = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Cron::Daily(CronDaily { + hour: 0u64, + minute: 0u64, + }) + }; + let field5: Cron = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Cron::Daily(CronDaily { + hour: 2u64, + minute: 0u64, + }) + }; + let field6: Cron = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Cron::Daily(CronDaily { + hour: 4u64, + minute: 0u64, + }) + }; + let field7: Option = if block.has(7) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(10000u64) + }; + let field10: Option = if block.has(10) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(2592000000)) + }; + #[cfg(not(feature = "enterprise"))] + if block.has(13) { + let block = stream.open_block()?; + stream.close_block(block); + } + stream.close_block(block); + Some(Self { + expunge_trash_after: field0, + expunge_submissions_after: field1, + expunge_share_notify_after: field2, + expunge_scheduling_inbox_after: field3, + expunge_schedule: field4, + data_cleanup_schedule: field5, + blob_cleanup_schedule: field6, + max_changes_history: field7, + hold_mta_reports_for: field10, + }) } } @@ -6612,68 +8090,53 @@ impl Default for DataRetention { minute: 0u64, }), max_changes_history: Some(10000u64), - archive_deleted_items_for: Default::default(), - archive_deleted_accounts_for: Default::default(), hold_mta_reports_for: Some(Duration::from_millis(2592000000)), - hold_traces_for: Some(Duration::from_millis(2592000000)), - hold_metrics_for: Some(Duration::from_millis(7776000000)), - metrics_collection_interval: Cron::Hourly(CronHourly { minute: 0u64 }), } } } impl IntoValue for DataRetention { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(16); map.insert_unchecked( Property::ExpungeTrashAfter, - self.expunge_trash_after.into_value(), + self.expunge_trash_after.into_value_with(opts), ); map.insert_unchecked( Property::ExpungeSubmissionsAfter, - self.expunge_submissions_after.into_value(), + self.expunge_submissions_after.into_value_with(opts), ); map.insert_unchecked( Property::ExpungeShareNotifyAfter, - self.expunge_share_notify_after.into_value(), + self.expunge_share_notify_after.into_value_with(opts), ); map.insert_unchecked( Property::ExpungeSchedulingInboxAfter, - self.expunge_scheduling_inbox_after.into_value(), + self.expunge_scheduling_inbox_after.into_value_with(opts), ); map.insert_unchecked( Property::ExpungeSchedule, - self.expunge_schedule.into_value(), + self.expunge_schedule.into_value_with(opts), ); map.insert_unchecked( Property::DataCleanupSchedule, - self.data_cleanup_schedule.into_value(), + self.data_cleanup_schedule.into_value_with(opts), ); map.insert_unchecked( Property::BlobCleanupSchedule, - self.blob_cleanup_schedule.into_value(), + self.blob_cleanup_schedule.into_value_with(opts), ); map.insert_unchecked( Property::MaxChangesHistory, - self.max_changes_history.into_value(), - ); - map.insert_unchecked( - Property::ArchiveDeletedItemsFor, - self.archive_deleted_items_for.into_value(), - ); - map.insert_unchecked( - Property::ArchiveDeletedAccountsFor, - self.archive_deleted_accounts_for.into_value(), + self.max_changes_history.into_value_with(opts), ); map.insert_unchecked( Property::HoldMtaReportsFor, - self.hold_mta_reports_for.into_value(), - ); - map.insert_unchecked(Property::HoldTracesFor, self.hold_traces_for.into_value()); - map.insert_unchecked(Property::HoldMetricsFor, self.hold_metrics_for.into_value()); - map.insert_unchecked( - Property::MetricsCollectionInterval, - self.metrics_collection_interval.into_value(), + self.hold_mta_reports_for.into_value_with(opts), ); JmapValue::Object(map) } @@ -6700,18 +8163,7 @@ impl RegistryJsonPropertyPatch for DataRetention { Some(Property::DataCleanupSchedule) => self.data_cleanup_schedule.patch(pointer, value), Some(Property::BlobCleanupSchedule) => self.blob_cleanup_schedule.patch(pointer, value), Some(Property::MaxChangesHistory) => self.max_changes_history.patch(pointer, value), - Some(Property::ArchiveDeletedItemsFor) => { - self.archive_deleted_items_for.patch(pointer, value) - } - Some(Property::ArchiveDeletedAccountsFor) => { - self.archive_deleted_accounts_for.patch(pointer, value) - } Some(Property::HoldMtaReportsFor) => self.hold_mta_reports_for.patch(pointer, value), - Some(Property::HoldTracesFor) => self.hold_traces_for.patch(pointer, value), - Some(Property::HoldMetricsFor) => self.hold_metrics_for.patch(pointer, value), - Some(Property::MetricsCollectionInterval) => { - self.metrics_collection_interval.patch(pointer, value) - } Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -6722,8 +8174,8 @@ impl RegistryJsonPropertyPatch for DataRetention { } impl ObjectImpl for DataStore { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const FLAGS: u64 = OBJ_SINGLETON | OBJ_NOT_STORED; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::DataStore; fn validate(&self, errors: &mut Vec) -> bool { @@ -6737,6 +8189,7 @@ impl ObjectImpl for DataStore { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for DataStore { @@ -6745,77 +8198,43 @@ impl Default for DataStore { } } -impl Pickle for DataStore { - fn pickle(&self, out: &mut Vec) { - match self { - DataStore::RocksDb(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - DataStore::Sqlite(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - DataStore::FoundationDb(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - DataStore::PostgreSql(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - DataStore::MySql(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(DataStore::RocksDb), - 1 => Pickle::unpickle(stream).map(DataStore::Sqlite), - 2 => Pickle::unpickle(stream).map(DataStore::FoundationDb), - 3 => Pickle::unpickle(stream).map(DataStore::PostgreSql), - 4 => Pickle::unpickle(stream).map(DataStore::MySql), - _ => None, - } - } -} - impl IntoValue for DataStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DataStore::RocksDb(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RocksDb".into())); obj } DataStore::Sqlite(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Sqlite".into())); obj } DataStore::FoundationDb(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); obj } DataStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } DataStore::MySql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); @@ -6902,30 +8321,6 @@ impl DeliveryError { } } -impl Pickle for DeliveryError { - fn pickle(&self, out: &mut Vec) { - self.error_type.pickle(out); - self.error_message.pickle(out); - self.error_command.pickle(out); - self.response_hostname.pickle(out); - self.response_code.pickle(out); - self.response_enhanced.pickle(out); - self.response_message.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.error_type = Pickle::unpickle(stream)?; - this.error_message = Pickle::unpickle(stream)?; - this.error_command = Pickle::unpickle(stream)?; - this.response_hostname = Pickle::unpickle(stream)?; - this.response_code = Pickle::unpickle(stream)?; - this.response_enhanced = Pickle::unpickle(stream)?; - this.response_message = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for DeliveryError { fn default() -> Self { Self { @@ -6942,22 +8337,35 @@ impl Default for DeliveryError { impl IntoValue for DeliveryError { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::ErrorType, self.error_type.into_value()); - map.insert_unchecked(Property::ErrorMessage, self.error_message.into_value()); - map.insert_unchecked(Property::ErrorCommand, self.error_command.into_value()); + map.insert_unchecked(Property::ErrorType, self.error_type.into_value_with(opts)); + map.insert_unchecked( + Property::ErrorMessage, + self.error_message.into_value_with(opts), + ); + map.insert_unchecked( + Property::ErrorCommand, + self.error_command.into_value_with(opts), + ); map.insert_unchecked( Property::ResponseHostname, - self.response_hostname.into_value(), + self.response_hostname.into_value_with(opts), + ); + map.insert_unchecked( + Property::ResponseCode, + self.response_code.into_value_with(opts), ); - map.insert_unchecked(Property::ResponseCode, self.response_code.into_value()); map.insert_unchecked( Property::ResponseEnhanced, - self.response_enhanced.into_value(), + self.response_enhanced.into_value_with(opts), ); map.insert_unchecked( Property::ResponseMessage, - self.response_message.into_value(), + self.response_message.into_value_with(opts), ); JmapValue::Object(map) } @@ -7039,6 +8447,9 @@ impl Pickle for Directory { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(Directory::Ldap), 1 => Pickle::unpickle(stream).map(Directory::Sql), @@ -7050,23 +8461,27 @@ impl Pickle for Directory { impl IntoValue for Directory { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Directory::Ldap(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ldap".into())); obj } Directory::Sql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Sql".into())); obj } Directory::Oidc(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Oidc".into())); @@ -7116,6 +8531,7 @@ impl DirectoryBootstrap { DirectoryBootstrap::Oidc(inner) => inner.validate(errors), } } + } impl Default for DirectoryBootstrap { @@ -7124,40 +8540,12 @@ impl Default for DirectoryBootstrap { } } -impl Pickle for DirectoryBootstrap { - fn pickle(&self, out: &mut Vec) { - match self { - DirectoryBootstrap::Internal => { - 0u16.pickle(out); - } - DirectoryBootstrap::Ldap(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - DirectoryBootstrap::Sql(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - DirectoryBootstrap::Oidc(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(DirectoryBootstrap::Internal), - 1 => Pickle::unpickle(stream).map(DirectoryBootstrap::Ldap), - 2 => Pickle::unpickle(stream).map(DirectoryBootstrap::Sql), - 3 => Pickle::unpickle(stream).map(DirectoryBootstrap::Oidc), - _ => None, - } - } -} - impl IntoValue for DirectoryBootstrap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DirectoryBootstrap::Internal => { let mut obj = jmap_tools::Map::new(); @@ -7165,21 +8553,21 @@ impl IntoValue for DirectoryBootstrap { JmapValue::Object(obj) } DirectoryBootstrap::Ldap(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ldap".into())); obj } DirectoryBootstrap::Sql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Sql".into())); obj } DirectoryBootstrap::Oidc(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Oidc".into())); @@ -7252,11 +8640,6 @@ impl Dkim1Signature { if !value.is_valid() { errors.push(ValidationError::required(Property::DomainId)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.selector; if value.is_empty() { errors.push(ValidationError::required(Property::Selector)); @@ -7276,48 +8659,161 @@ impl Dkim1Signature { fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); i.search(Property::DomainId, &self.domain_id); - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for Dkim1Signature { fn pickle(&self, out: &mut Vec) { - self.auid.pickle(out); + let mut bitmap = 0u64; + if self.auid.is_some() { + bitmap |= 1 << 0; + } + if self.expire.is_some() { + bitmap |= 1 << 1; + } + if self.report { + bitmap |= 1 << 2; + } + if self.third_party.is_some() { + bitmap |= 1 << 3; + } + if self.third_party_hash.is_some() { + bitmap |= 1 << 4; + } + if self.next_transition_at.is_some() { + bitmap |= 1 << 6; + } + let block = crate::pickle::begin_block(out, 14, 7, bitmap); + if let Some(value) = &self.auid { + value.pickle(out); + } self.canonicalization.pickle(out); - self.expire.pickle(out); + if let Some(value) = &self.expire { + value.pickle(out); + } self.headers.pickle(out); self.private_key.pickle(out); - self.report.pickle(out); - self.third_party.pickle(out); - self.third_party_hash.pickle(out); + if let Some(value) = &self.third_party { + value.pickle(out); + } + if let Some(value) = &self.third_party_hash { + value.pickle(out); + } self.domain_id.pickle(out); - self.member_tenant_id.pickle(out); self.selector.pickle(out); self.created_at.pickle(out); - self.next_transition_at.pickle(out); + if let Some(value) = &self.next_transition_at { + value.pickle(out); + } self.stage.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auid = Pickle::unpickle(stream)?; - this.canonicalization = Pickle::unpickle(stream)?; - this.expire = Pickle::unpickle(stream)?; - this.headers = Pickle::unpickle(stream)?; - this.private_key = Pickle::unpickle(stream)?; - this.report = Pickle::unpickle(stream)?; - this.third_party = Pickle::unpickle(stream)?; - this.third_party_hash = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.selector = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.next_transition_at = Pickle::unpickle(stream)?; - this.stage = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: DkimCanonicalization = if block.has(1) { + Pickle::unpickle(stream)? + } else { + DkimCanonicalization::RelaxedRelaxed + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + "From".to_string(), + "To".to_string(), + "Date".to_string(), + "Subject".to_string(), + "Message-ID".to_string(), + ]) + }; + let field4: SecretText = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: bool = if block.has(5) { block.flag(2) } else { true }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: Id = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field10: String = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: UTCDateTime = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: Option = if block.has(12) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field13: DkimRotationStage = if block.has(13) { + Pickle::unpickle(stream)? + } else { + DkimRotationStage::Active + }; + stream.close_block(block); + Some(Self { + auid: field0, + canonicalization: field1, + expire: field2, + headers: field3, + private_key: field4, + report: field5, + third_party: field6, + third_party_hash: field7, + domain_id: field8, + selector: field10, + created_at: field11, + next_transition_at: field12, + stage: field13, + }) } } @@ -7339,7 +8835,6 @@ impl Default for Dkim1Signature { third_party: Default::default(), third_party_hash: Default::default(), domain_id: Default::default(), - member_tenant_id: Default::default(), selector: Default::default(), created_at: Default::default(), next_transition_at: Default::default(), @@ -7350,27 +8845,33 @@ impl Default for Dkim1Signature { impl IntoValue for Dkim1Signature { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(16); - map.insert_unchecked(Property::Auid, self.auid.into_value()); + map.insert_unchecked(Property::Auid, self.auid.into_value_with(opts)); map.insert_unchecked( Property::Canonicalization, - self.canonicalization.into_value(), + self.canonicalization.into_value_with(opts), ); - map.insert_unchecked(Property::Expire, self.expire.into_value()); - map.insert_unchecked(Property::Headers, self.headers.into_value()); - map.insert_unchecked(Property::PrivateKey, self.private_key.into_value()); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::ThirdParty, self.third_party.into_value()); - map.insert_unchecked(Property::ThirdPartyHash, self.third_party_hash.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Selector, self.selector.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); + map.insert_unchecked(Property::Expire, self.expire.into_value_with(opts)); + map.insert_unchecked(Property::Headers, self.headers.into_value_with(opts)); + map.insert_unchecked(Property::PrivateKey, self.private_key.into_value_with(opts)); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::ThirdParty, self.third_party.into_value_with(opts)); + map.insert_unchecked( + Property::ThirdPartyHash, + self.third_party_hash.into_value_with(opts), + ); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked(Property::Selector, self.selector.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); map.insert_unchecked( Property::NextTransitionAt, - self.next_transition_at.into_value(), + self.next_transition_at.into_value_with(opts), ); - map.insert_unchecked(Property::Stage, self.stage.into_value()); + map.insert_unchecked(Property::Stage, self.stage.into_value_with(opts)); JmapValue::Object(map) } } @@ -7398,9 +8899,6 @@ impl RegistryJsonPropertyPatch for Dkim1Signature { .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::ThirdPartyHash) => self.third_party_hash.patch(pointer, value), Some(Property::DomainId) => self.domain_id.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Selector) => self .selector .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -7419,17 +8917,16 @@ impl RegistryJsonPropertyPatch for Dkim1Signature { impl Dkim2Signature { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); + let value = &self.instance_hashes; + if value.len() < 1 { + errors.push(ValidationError::min_items(Property::InstanceHashes, 1)); + } let value = &self.private_key; value.validate(errors); let value = &self.domain_id; if !value.is_valid() { errors.push(ValidationError::required(Property::DomainId)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.selector; if value.is_empty() { errors.push(ValidationError::required(Property::Selector)); @@ -7449,36 +8946,86 @@ impl Dkim2Signature { fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); i.search(Property::DomainId, &self.domain_id); - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for Dkim2Signature { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.next_transition_at.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.flags.pickle(out); + self.instance_hashes.pickle(out); self.private_key.pickle(out); self.domain_id.pickle(out); - self.member_tenant_id.pickle(out); self.selector.pickle(out); self.created_at.pickle(out); - self.next_transition_at.pickle(out); + if let Some(value) = &self.next_transition_at { + value.pickle(out); + } self.stage.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.flags = Pickle::unpickle(stream)?; - this.private_key = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.selector = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.next_transition_at = Pickle::unpickle(stream)?; - this.stage = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![Dkim2HashAlgorithm::Sha256]) + }; + let field2: SecretText = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Id = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: UTCDateTime = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: DkimRotationStage = if block.has(8) { + Pickle::unpickle(stream)? + } else { + DkimRotationStage::Active + }; + stream.close_block(block); + Some(Self { + flags: field0, + instance_hashes: field1, + private_key: field2, + domain_id: field3, + selector: field5, + created_at: field6, + next_transition_at: field7, + stage: field8, + }) } } @@ -7486,9 +9033,9 @@ impl Default for Dkim2Signature { fn default() -> Self { Self { flags: Default::default(), + instance_hashes: Map::new(vec![Dkim2HashAlgorithm::Sha256]), private_key: Default::default(), domain_id: Default::default(), - member_tenant_id: Default::default(), selector: Default::default(), created_at: Default::default(), next_transition_at: Default::default(), @@ -7499,18 +9046,25 @@ impl Default for Dkim2Signature { impl IntoValue for Dkim2Signature { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Flags, self.flags.into_value()); - map.insert_unchecked(Property::PrivateKey, self.private_key.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Selector, self.selector.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(11); + map.insert_unchecked(Property::Flags, self.flags.into_value_with(opts)); + map.insert_unchecked( + Property::InstanceHashes, + self.instance_hashes.into_value_with(opts), + ); + map.insert_unchecked(Property::PrivateKey, self.private_key.into_value_with(opts)); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked(Property::Selector, self.selector.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); map.insert_unchecked( Property::NextTransitionAt, - self.next_transition_at.into_value(), + self.next_transition_at.into_value_with(opts), ); - map.insert_unchecked(Property::Stage, self.stage.into_value()); + map.insert_unchecked(Property::Stage, self.stage.into_value_with(opts)); JmapValue::Object(map) } } @@ -7523,12 +9077,10 @@ impl RegistryJsonPropertyPatch for Dkim2Signature { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::Flags) => self.flags.patch(pointer, value), + Some(Property::InstanceHashes) => self.instance_hashes.patch(pointer, value), Some(Property::PrivateKey) => self.private_key.patch(pointer, value), Some(Property::PublicKey) => pointer.assert_server_set(), Some(Property::DomainId) => self.domain_id.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Selector) => self .selector .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -7583,9 +9135,13 @@ impl Pickle for DkimManagement { impl IntoValue for DkimManagement { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DkimManagement::Automatic(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Automatic".into())); @@ -7643,21 +9199,53 @@ impl DkimManagementProperties { impl Pickle for DkimManagementProperties { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.algorithms.pickle(out); self.selector_template.pickle(out); self.rotate_after.pickle(out); self.retire_after.pickle(out); self.delete_after.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.algorithms = Pickle::unpickle(stream)?; - this.selector_template = Pickle::unpickle(stream)?; - this.rotate_after = Pickle::unpickle(stream)?; - this.retire_after = Pickle::unpickle(stream)?; - this.delete_after = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + DkimSignatureType::Dkim1Ed25519Sha256, + DkimSignatureType::Dkim1RsaSha256, + ]) + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + "v{version}-{algorithm}-{date-%Y%m%d}".to_string() + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(7776000000) + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(604800000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(2592000000) + }; + stream.close_block(block); + Some(Self { + algorithms: field0, + selector_template: field1, + rotate_after: field2, + retire_after: field3, + delete_after: field4, + }) } } @@ -7678,15 +9266,28 @@ impl Default for DkimManagementProperties { impl IntoValue for DkimManagementProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Algorithms, self.algorithms.into_value()); + map.insert_unchecked(Property::Algorithms, self.algorithms.into_value_with(opts)); map.insert_unchecked( Property::SelectorTemplate, - self.selector_template.into_value(), + self.selector_template.into_value_with(opts), + ); + map.insert_unchecked( + Property::RotateAfter, + self.rotate_after.into_value_with(opts), + ); + map.insert_unchecked( + Property::RetireAfter, + self.retire_after.into_value_with(opts), + ); + map.insert_unchecked( + Property::DeleteAfter, + self.delete_after.into_value_with(opts), ); - map.insert_unchecked(Property::RotateAfter, self.rotate_after.into_value()); - map.insert_unchecked(Property::RetireAfter, self.retire_after.into_value()); - map.insert_unchecked(Property::DeleteAfter, self.delete_after.into_value()); JmapValue::Object(map) } } @@ -7742,7 +9343,7 @@ impl DkimReportSettings { ExpressionContext { expr: &self.from_address, default: Some(Expression { - else_: "'noreply-dkim@' + system('domain')".to_string(), + else_: "'noreply-dkim@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::FromAddress, @@ -7781,7 +9382,7 @@ impl DkimReportSettings { ExpressionContext { expr: &self.dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::DkimSignDomain, @@ -7816,21 +9417,68 @@ impl DkimReportSettings { impl Pickle for DkimReportSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.from_address.pickle(out); self.from_name.pickle(out); self.send_frequency.pickle(out); self.dkim_sign_domain.pickle(out); self.subject.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from_address = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.send_frequency = Pickle::unpickle(stream)?; - this.dkim_sign_domain = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'noreply-dkim@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Report Subsystem'".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "[1, 1d]".to_string(), + ..Default::default() + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'DKIM Authentication Failure Report'".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + from_address: field0, + from_name: field1, + send_frequency: field2, + dkim_sign_domain: field3, + subject: field4, + }) } } @@ -7838,7 +9486,7 @@ impl Default for DkimReportSettings { fn default() -> Self { Self { from_address: Expression { - else_: "'noreply-dkim@' + system('domain')".to_string(), + else_: "'noreply-dkim@' + system('report_domain')".to_string(), ..Default::default() }, from_name: Expression { @@ -7850,7 +9498,7 @@ impl Default for DkimReportSettings { ..Default::default() }, dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, subject: Expression { @@ -7863,12 +9511,25 @@ impl Default for DkimReportSettings { impl IntoValue for DkimReportSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::FromAddress, self.from_address.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); - map.insert_unchecked(Property::SendFrequency, self.send_frequency.into_value()); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); + map.insert_unchecked( + Property::FromAddress, + self.from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FromName, self.from_name.into_value_with(opts)); + map.insert_unchecked( + Property::SendFrequency, + self.send_frequency.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), + ); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); JmapValue::Object(map) } } @@ -7905,6 +9566,7 @@ impl ObjectImpl for DkimSignature { DkimSignature::Dkim1RsaSha256(inner) => inner.validate(errors), DkimSignature::Dkim2Ed25519Sha256(inner) => inner.validate(errors), DkimSignature::Dkim2RsaSha256(inner) => inner.validate(errors), + DkimSignature::Dkim2MlDsa44Ed25519Sha512(inner) => inner.validate(errors), } } @@ -7922,6 +9584,9 @@ impl ObjectImpl for DkimSignature { DkimSignature::Dkim2RsaSha256(object) => { object.index(i); } + DkimSignature::Dkim2MlDsa44Ed25519Sha512(object) => { + object.index(i); + } } } } @@ -7951,15 +9616,23 @@ impl Pickle for DkimSignature { 3u16.pickle(out); inner.pickle(out); } + DkimSignature::Dkim2MlDsa44Ed25519Sha512(inner) => { + 4u16.pickle(out); + inner.pickle(out); + } } } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(DkimSignature::Dkim1Ed25519Sha256), 1 => Pickle::unpickle(stream).map(DkimSignature::Dkim1RsaSha256), 2 => Pickle::unpickle(stream).map(DkimSignature::Dkim2Ed25519Sha256), 3 => Pickle::unpickle(stream).map(DkimSignature::Dkim2RsaSha256), + 4 => Pickle::unpickle(stream).map(DkimSignature::Dkim2MlDsa44Ed25519Sha512), _ => None, } } @@ -7967,35 +9640,47 @@ impl Pickle for DkimSignature { impl IntoValue for DkimSignature { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DkimSignature::Dkim1Ed25519Sha256(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dkim1Ed25519Sha256".into())); obj } DkimSignature::Dkim1RsaSha256(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dkim1RsaSha256".into())); obj } DkimSignature::Dkim2Ed25519Sha256(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dkim2Ed25519Sha256".into())); obj } DkimSignature::Dkim2RsaSha256(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dkim2RsaSha256".into())); obj } + DkimSignature::Dkim2MlDsa44Ed25519Sha512(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut().unwrap().insert_unchecked( + Property::Type, + JmapValue::Str("Dkim2MlDsa44Ed25519Sha512".into()), + ); + obj + } } } } @@ -8020,6 +9705,9 @@ impl RegistryJsonPatch for DkimSignature { DkimSignatureType::Dkim2RsaSha256 => { *self = DkimSignature::Dkim2RsaSha256(Default::default()) } + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => { + *self = DkimSignature::Dkim2MlDsa44Ed25519Sha512(Default::default()) + } } } match self { @@ -8027,6 +9715,7 @@ impl RegistryJsonPatch for DkimSignature { DkimSignature::Dkim1RsaSha256(inner) => inner.patch(pointer, value), DkimSignature::Dkim2Ed25519Sha256(inner) => inner.patch(pointer, value), DkimSignature::Dkim2RsaSha256(inner) => inner.patch(pointer, value), + DkimSignature::Dkim2MlDsa44Ed25519Sha512(inner) => inner.patch(pointer, value), } } } @@ -8038,6 +9727,9 @@ impl DkimSignature { DkimSignature::Dkim1RsaSha256(_) => DkimSignatureType::Dkim1RsaSha256, DkimSignature::Dkim2Ed25519Sha256(_) => DkimSignatureType::Dkim2Ed25519Sha256, DkimSignature::Dkim2RsaSha256(_) => DkimSignatureType::Dkim2RsaSha256, + DkimSignature::Dkim2MlDsa44Ed25519Sha512(_) => { + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 + } } } } @@ -8064,19 +9756,53 @@ impl DmarcDkimResult { impl Pickle for DmarcDkimResult { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.human_result.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.domain.pickle(out); self.selector.pickle(out); self.result.pickle(out); - self.human_result.pickle(out); + if let Some(value) = &self.human_result { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.domain = Pickle::unpickle(stream)?; - this.selector = Pickle::unpickle(stream)?; - this.result = Pickle::unpickle(stream)?; - this.human_result = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: DkimAuthResult = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + domain: field0, + selector: field1, + result: field2, + human_result: field3, + }) } } @@ -8093,11 +9819,18 @@ impl Default for DmarcDkimResult { impl IntoValue for DmarcDkimResult { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Domain, self.domain.into_value()); - map.insert_unchecked(Property::Selector, self.selector.into_value()); - map.insert_unchecked(Property::Result, self.result.into_value()); - map.insert_unchecked(Property::HumanResult, self.human_result.into_value()); + map.insert_unchecked(Property::Domain, self.domain.into_value_with(opts)); + map.insert_unchecked(Property::Selector, self.selector.into_value_with(opts)); + map.insert_unchecked(Property::Result, self.result.into_value_with(opts)); + map.insert_unchecked( + Property::HumanResult, + self.human_result.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -8141,15 +9874,29 @@ impl DmarcExtension { impl Pickle for DmarcExtension { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.name.pickle(out); self.definition.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.definition = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + definition: field1, + }) } } @@ -8164,9 +9911,13 @@ impl Default for DmarcExtension { impl IntoValue for DmarcExtension { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Definition, self.definition.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::Definition, self.definition.into_value_with(opts)); JmapValue::Object(map) } } @@ -8190,8 +9941,8 @@ impl RegistryJsonPropertyPatch for DmarcExtension { } impl ObjectImpl for DmarcExternalReport { - const FLAGS: u64 = OBJ_FILTER_TENANT; - const VERSION: u8 = 1; + const FLAGS: u64 = OBJ_FILTER_TENANT | OBJ_NO_CREATE | OBJ_NO_UPDATE; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::DmarcExternalReport; fn validate(&self, errors: &mut Vec) -> bool { @@ -8220,40 +9971,72 @@ impl ObjectImpl for DmarcExternalReport { if !value.is_valid() { errors.push(ValidationError::invalid(Property::ExpiresAt, value)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); } } impl Pickle for DmarcExternalReport { fn pickle(&self, out: &mut Vec) { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut bitmap = 0u64; + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.report.pickle(out); self.from.pickle(out); self.subject.pickle(out); self.to.pickle(out); self.received_at.pickle(out); self.expires_at.pickle(out); - self.member_tenant_id.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.report = Pickle::unpickle(stream)?; - this.from = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - this.received_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: DmarcReport = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + report: field0, + from: field1, + subject: field2, + to: field3, + received_at: field4, + expires_at: field5, + }) } } @@ -8266,21 +10049,23 @@ impl Default for DmarcExternalReport { to: Default::default(), received_at: Default::default(), expires_at: Default::default(), - member_tenant_id: Default::default(), } } } impl IntoValue for DmarcExternalReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); - map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::From, self.from.into_value_with(opts)); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); + map.insert_unchecked(Property::To, self.to.into_value_with(opts)); + map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -8302,9 +10087,6 @@ impl RegistryJsonPropertyPatch for DmarcExternalReport { .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::ReceivedAt) => self.received_at.patch(pointer, value), Some(Property::ExpiresAt) => self.expires_at.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -8315,8 +10097,8 @@ impl RegistryJsonPropertyPatch for DmarcExternalReport { } impl ObjectImpl for DmarcInternalReport { - const FLAGS: u64 = 0; - const VERSION: u8 = 1; + const FLAGS: u64 = OBJ_NO_CREATE; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::DmarcInternalReport; fn validate(&self, errors: &mut Vec) -> bool { @@ -8349,23 +10131,60 @@ impl ObjectImpl for DmarcInternalReport { impl Pickle for DmarcInternalReport { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 6, 0, 0); self.rua.pickle(out); self.policy_identifier.pickle(out); self.report.pickle(out); self.domain.pickle(out); self.created_at.pickle(out); self.deliver_at.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.rua = Pickle::unpickle(stream)?; - this.policy_identifier = Pickle::unpickle(stream)?; - this.report = Pickle::unpickle(stream)?; - this.domain = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.deliver_at = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: DmarcReport = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + rua: field0, + policy_identifier: field1, + report: field2, + domain: field3, + created_at: field4, + deliver_at: field5, + }) } } @@ -8384,16 +10203,20 @@ impl Default for DmarcInternalReport { impl IntoValue for DmarcInternalReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Rua, self.rua.into_value()); + map.insert_unchecked(Property::Rua, self.rua.into_value_with(opts)); map.insert_unchecked( Property::PolicyIdentifier, - self.policy_identifier.into_value(), + self.policy_identifier.into_value_with(opts), ); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::Domain, self.domain.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::DeliverAt, self.deliver_at.into_value()); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::Domain, self.domain.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::DeliverAt, self.deliver_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -8438,15 +10261,39 @@ impl DmarcPolicyOverrideReason { impl Pickle for DmarcPolicyOverrideReason { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.comment.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); self.override_type.pickle(out); - self.comment.pickle(out); + if let Some(value) = &self.comment { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.override_type = Pickle::unpickle(stream)?; - this.comment = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: DmarcPolicyOverride = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + override_type: field0, + comment: field1, + }) } } @@ -8461,9 +10308,16 @@ impl Default for DmarcPolicyOverrideReason { impl IntoValue for DmarcPolicyOverrideReason { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::OverrideType, self.override_type.into_value()); - map.insert_unchecked(Property::Comment, self.comment.into_value()); + map.insert_unchecked( + Property::OverrideType, + self.override_type.into_value_with(opts), + ); + map.insert_unchecked(Property::Comment, self.comment.into_value_with(opts)); JmapValue::Object(map) } } @@ -8506,9 +10360,9 @@ impl DmarcReport { if value.is_empty() { errors.push(ValidationError::required(Property::ReportId)); } - let value = &self.date_range_begin; + let value = &self.date_range_start; if !value.is_valid() { - errors.push(ValidationError::invalid(Property::DateRangeBegin, value)); + errors.push(ValidationError::invalid(Property::DateRangeStart, value)); } let value = &self.date_range_end; if !value.is_valid() { @@ -8548,59 +10402,188 @@ impl DmarcReport { impl Pickle for DmarcReport { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.extra_contact_info.is_some() { + bitmap |= 1 << 0; + } + if self.policy_version.is_some() { + bitmap |= 1 << 1; + } + if self.policy_testing_mode { + bitmap |= 1 << 2; + } + if self.generator.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 21, 4, bitmap); self.version.pickle(out); self.org_name.pickle(out); self.email.pickle(out); - self.extra_contact_info.pickle(out); + if let Some(value) = &self.extra_contact_info { + value.pickle(out); + } self.report_id.pickle(out); - self.date_range_begin.pickle(out); + self.date_range_start.pickle(out); self.date_range_end.pickle(out); self.errors.pickle(out); self.policy_domain.pickle(out); - self.policy_version.pickle(out); + if let Some(value) = &self.policy_version { + value.pickle(out); + } self.policy_adkim.pickle(out); self.policy_aspf.pickle(out); self.policy_disposition.pickle(out); self.policy_subdomain_disposition.pickle(out); - self.policy_testing_mode.pickle(out); self.policy_failure_reporting_options.pickle(out); self.records.pickle(out); self.extensions.pickle(out); - self.generator.pickle(out); + if let Some(value) = &self.generator { + value.pickle(out); + } self.policy_np.pickle(out); self.policy_discovery_method.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.version = Pickle::unpickle(stream)?; - this.org_name = Pickle::unpickle(stream)?; - this.email = Pickle::unpickle(stream)?; - this.extra_contact_info = Pickle::unpickle(stream)?; - this.report_id = Pickle::unpickle(stream)?; - this.date_range_begin = Pickle::unpickle(stream)?; - this.date_range_end = Pickle::unpickle(stream)?; - this.errors = Pickle::unpickle(stream)?; - this.policy_domain = Pickle::unpickle(stream)?; - this.policy_version = Pickle::unpickle(stream)?; - this.policy_adkim = Pickle::unpickle(stream)?; - this.policy_aspf = Pickle::unpickle(stream)?; - this.policy_disposition = Pickle::unpickle(stream)?; - this.policy_subdomain_disposition = Pickle::unpickle(stream)?; - this.policy_testing_mode = Pickle::unpickle(stream)?; - this.policy_failure_reporting_options = Pickle::unpickle(stream)?; - this.records = Pickle::unpickle(stream)?; - this.extensions = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.generator = Pickle::unpickle(stream)?; - } - if stream.version() >= 1 { - this.policy_np = Pickle::unpickle(stream)?; - } - if stream.version() >= 1 { - this.policy_discovery_method = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: Float = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Float::new(1.0f64) + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: UTCDateTime = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Map = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: String = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Option = if block.has(9) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field10: DmarcAlignment = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: DmarcAlignment = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: DmarcDisposition = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field13: DmarcDisposition = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field14: bool = if block.has(14) { block.flag(2) } else { false }; + let field15: Map = if block.has(15) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field16: List = if block.has(16) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field17: List = if block.has(17) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field18: Option = if block.has(18) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field19: DmarcDisposition = if block.has(19) { + Pickle::unpickle(stream)? + } else { + DmarcDisposition::Unspecified + }; + let field20: DmarcDiscovery = if block.has(20) { + Pickle::unpickle(stream)? + } else { + DmarcDiscovery::Unspecified + }; + stream.close_block(block); + Some(Self { + version: field0, + org_name: field1, + email: field2, + extra_contact_info: field3, + report_id: field4, + date_range_start: field5, + date_range_end: field6, + errors: field7, + policy_domain: field8, + policy_version: field9, + policy_adkim: field10, + policy_aspf: field11, + policy_disposition: field12, + policy_subdomain_disposition: field13, + policy_testing_mode: field14, + policy_failure_reporting_options: field15, + records: field16, + extensions: field17, + generator: field18, + policy_np: field19, + policy_discovery_method: field20, + }) } } @@ -8612,7 +10595,7 @@ impl Default for DmarcReport { email: Default::default(), extra_contact_info: Default::default(), report_id: Default::default(), - date_range_begin: Default::default(), + date_range_start: Default::default(), date_range_end: Default::default(), errors: Default::default(), policy_domain: Default::default(), @@ -8634,45 +10617,64 @@ impl Default for DmarcReport { impl IntoValue for DmarcReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(23); - map.insert_unchecked(Property::Version, self.version.into_value()); - map.insert_unchecked(Property::OrgName, self.org_name.into_value()); - map.insert_unchecked(Property::Email, self.email.into_value()); + map.insert_unchecked(Property::Version, self.version.into_value_with(opts)); + map.insert_unchecked(Property::OrgName, self.org_name.into_value_with(opts)); + map.insert_unchecked(Property::Email, self.email.into_value_with(opts)); map.insert_unchecked( Property::ExtraContactInfo, - self.extra_contact_info.into_value(), + self.extra_contact_info.into_value_with(opts), ); - map.insert_unchecked(Property::ReportId, self.report_id.into_value()); - map.insert_unchecked(Property::DateRangeBegin, self.date_range_begin.into_value()); - map.insert_unchecked(Property::DateRangeEnd, self.date_range_end.into_value()); - map.insert_unchecked(Property::Errors, self.errors.into_value()); - map.insert_unchecked(Property::PolicyDomain, self.policy_domain.into_value()); - map.insert_unchecked(Property::PolicyVersion, self.policy_version.into_value()); - map.insert_unchecked(Property::PolicyAdkim, self.policy_adkim.into_value()); - map.insert_unchecked(Property::PolicyAspf, self.policy_aspf.into_value()); + map.insert_unchecked(Property::ReportId, self.report_id.into_value_with(opts)); + map.insert_unchecked( + Property::DateRangeStart, + self.date_range_start.into_value_with(opts), + ); + map.insert_unchecked( + Property::DateRangeEnd, + self.date_range_end.into_value_with(opts), + ); + map.insert_unchecked(Property::Errors, self.errors.into_value_with(opts)); + map.insert_unchecked( + Property::PolicyDomain, + self.policy_domain.into_value_with(opts), + ); + map.insert_unchecked( + Property::PolicyVersion, + self.policy_version.into_value_with(opts), + ); + map.insert_unchecked( + Property::PolicyAdkim, + self.policy_adkim.into_value_with(opts), + ); + map.insert_unchecked(Property::PolicyAspf, self.policy_aspf.into_value_with(opts)); map.insert_unchecked( Property::PolicyDisposition, - self.policy_disposition.into_value(), + self.policy_disposition.into_value_with(opts), ); map.insert_unchecked( Property::PolicySubdomainDisposition, - self.policy_subdomain_disposition.into_value(), + self.policy_subdomain_disposition.into_value_with(opts), ); map.insert_unchecked( Property::PolicyTestingMode, - self.policy_testing_mode.into_value(), + self.policy_testing_mode.into_value_with(opts), ); map.insert_unchecked( Property::PolicyFailureReportingOptions, - self.policy_failure_reporting_options.into_value(), + self.policy_failure_reporting_options.into_value_with(opts), ); - map.insert_unchecked(Property::Records, self.records.into_value()); - map.insert_unchecked(Property::Extensions, self.extensions.into_value()); - map.insert_unchecked(Property::Generator, self.generator.into_value()); - map.insert_unchecked(Property::PolicyNp, self.policy_np.into_value()); + map.insert_unchecked(Property::Records, self.records.into_value_with(opts)); + map.insert_unchecked(Property::Extensions, self.extensions.into_value_with(opts)); + map.insert_unchecked(Property::Generator, self.generator.into_value_with(opts)); + map.insert_unchecked(Property::PolicyNp, self.policy_np.into_value_with(opts)); map.insert_unchecked( Property::PolicyDiscoveryMethod, - self.policy_discovery_method.into_value(), + self.policy_discovery_method.into_value_with(opts), ); JmapValue::Object(map) } @@ -8692,7 +10694,7 @@ impl RegistryJsonPropertyPatch for DmarcReport { .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::ExtraContactInfo) => self.extra_contact_info.patch(pointer, value), Some(Property::ReportId) => self.report_id.patch(pointer, value), - Some(Property::DateRangeBegin) => self.date_range_begin.patch(pointer, value), + Some(Property::DateRangeStart) => self.date_range_start.patch(pointer, value), Some(Property::DateRangeEnd) => self.date_range_end.patch(pointer, value), Some(Property::Errors) => self.errors.patch(pointer, value), Some(Property::PolicyDomain) => self.policy_domain.patch(pointer, value), @@ -8766,35 +10768,118 @@ impl DmarcReportRecord { impl Pickle for DmarcReportRecord { fn pickle(&self, out: &mut Vec) { - self.source_ip.pickle(out); + let mut bitmap = 0u64; + if self.source_ip.is_some() { + bitmap |= 1 << 0; + } + if self.envelope_to.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 12, 2, bitmap); + if let Some(value) = &self.source_ip { + value.pickle(out); + } self.count.pickle(out); self.evaluated_disposition.pickle(out); self.evaluated_dkim.pickle(out); self.evaluated_spf.pickle(out); self.policy_override_reasons.pickle(out); - self.envelope_to.pickle(out); + if let Some(value) = &self.envelope_to { + value.pickle(out); + } self.envelope_from.pickle(out); self.header_from.pickle(out); self.dkim_results.pickle(out); self.spf_results.pickle(out); self.extensions.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.source_ip = Pickle::unpickle(stream)?; - this.count = Pickle::unpickle(stream)?; - this.evaluated_disposition = Pickle::unpickle(stream)?; - this.evaluated_dkim = Pickle::unpickle(stream)?; - this.evaluated_spf = Pickle::unpickle(stream)?; - this.policy_override_reasons = Pickle::unpickle(stream)?; - this.envelope_to = Pickle::unpickle(stream)?; - this.envelope_from = Pickle::unpickle(stream)?; - this.header_from = Pickle::unpickle(stream)?; - this.dkim_results = Pickle::unpickle(stream)?; - this.spf_results = Pickle::unpickle(stream)?; - this.extensions = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: DmarcActionDisposition = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: DmarcResult = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: DmarcResult = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: List = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: String = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: String = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: List = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field10: List = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: List = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + source_ip: field0, + count: field1, + evaluated_disposition: field2, + evaluated_dkim: field3, + evaluated_spf: field4, + policy_override_reasons: field5, + envelope_to: field6, + envelope_from: field7, + header_from: field8, + dkim_results: field9, + spf_results: field10, + extensions: field11, + }) } } @@ -8819,25 +10904,41 @@ impl Default for DmarcReportRecord { impl IntoValue for DmarcReportRecord { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::SourceIp, self.source_ip.into_value()); - map.insert_unchecked(Property::Count, self.count.into_value()); + map.insert_unchecked(Property::SourceIp, self.source_ip.into_value_with(opts)); + map.insert_unchecked(Property::Count, self.count.into_value_with(opts)); map.insert_unchecked( Property::EvaluatedDisposition, - self.evaluated_disposition.into_value(), + self.evaluated_disposition.into_value_with(opts), + ); + map.insert_unchecked( + Property::EvaluatedDkim, + self.evaluated_dkim.into_value_with(opts), + ); + map.insert_unchecked( + Property::EvaluatedSpf, + self.evaluated_spf.into_value_with(opts), ); - map.insert_unchecked(Property::EvaluatedDkim, self.evaluated_dkim.into_value()); - map.insert_unchecked(Property::EvaluatedSpf, self.evaluated_spf.into_value()); map.insert_unchecked( Property::PolicyOverrideReasons, - self.policy_override_reasons.into_value(), + self.policy_override_reasons.into_value_with(opts), ); - map.insert_unchecked(Property::EnvelopeTo, self.envelope_to.into_value()); - map.insert_unchecked(Property::EnvelopeFrom, self.envelope_from.into_value()); - map.insert_unchecked(Property::HeaderFrom, self.header_from.into_value()); - map.insert_unchecked(Property::DkimResults, self.dkim_results.into_value()); - map.insert_unchecked(Property::SpfResults, self.spf_results.into_value()); - map.insert_unchecked(Property::Extensions, self.extensions.into_value()); + map.insert_unchecked(Property::EnvelopeTo, self.envelope_to.into_value_with(opts)); + map.insert_unchecked( + Property::EnvelopeFrom, + self.envelope_from.into_value_with(opts), + ); + map.insert_unchecked(Property::HeaderFrom, self.header_from.into_value_with(opts)); + map.insert_unchecked( + Property::DkimResults, + self.dkim_results.into_value_with(opts), + ); + map.insert_unchecked(Property::SpfResults, self.spf_results.into_value_with(opts)); + map.insert_unchecked(Property::Extensions, self.extensions.into_value_with(opts)); JmapValue::Object(map) } } @@ -8887,7 +10988,7 @@ impl ObjectImpl for DmarcReportSettings { value.validate(errors); let value = &self.aggregate_from_name; value.validate(errors); - let value = &self.aggregate_max_report_size; + let value = &self.max_aggregate_report_size; value.validate(errors); let value = &self.aggregate_org_name; value.validate(errors); @@ -8895,8 +10996,6 @@ impl ObjectImpl for DmarcReportSettings { value.validate(errors); let value = &self.aggregate_dkim_sign_domain; value.validate(errors); - let value = &self.aggregate_subject; - value.validate(errors); let value = &self.failure_from_address; value.validate(errors); let value = &self.failure_from_name; @@ -8931,7 +11030,7 @@ impl DmarcReportSettings { ExpressionContext { expr: &self.aggregate_from_address, default: Some(Expression { - else_: "'noreply-dmarc@' + system('domain')".to_string(), + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::AggregateFromAddress, @@ -8953,14 +11052,14 @@ impl DmarcReportSettings { } } - pub fn ctx_aggregate_max_report_size(&self) -> ExpressionContext<'_> { + pub fn ctx_max_aggregate_report_size(&self) -> ExpressionContext<'_> { ExpressionContext { - expr: &self.aggregate_max_report_size, + expr: &self.max_aggregate_report_size, default: Some(Expression { else_: "5242880".to_string(), ..Default::default() }), - property: Property::AggregateMaxReportSize, + property: Property::MaxAggregateReportSize, allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } @@ -8970,7 +11069,7 @@ impl DmarcReportSettings { ExpressionContext { expr: &self.aggregate_org_name, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::AggregateOrgName, @@ -8996,24 +11095,11 @@ impl DmarcReportSettings { ExpressionContext { expr: &self.aggregate_dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::AggregateDkimSignDomain, - allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, - allowed_constants: &[], - } - } - - pub fn ctx_aggregate_subject(&self) -> ExpressionContext<'_> { - ExpressionContext { - expr: &self.aggregate_subject, - default: Some(Expression { - else_: "'DMARC Aggregate Report'".to_string(), - ..Default::default() - }), - property: Property::AggregateSubject, - allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, + allowed_variables: MTA_QUEUE_SENDER_VARIABLE, allowed_constants: &[], } } @@ -9022,7 +11108,7 @@ impl DmarcReportSettings { ExpressionContext { expr: &self.failure_from_address, default: Some(Expression { - else_: "'noreply-dmarc@' + system('domain')".to_string(), + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::FailureFromAddress, @@ -9061,7 +11147,7 @@ impl DmarcReportSettings { ExpressionContext { expr: &self.failure_dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::FailureDkimSignDomain, @@ -9088,11 +11174,10 @@ impl DmarcReportSettings { self.ctx_aggregate_contact_info(), self.ctx_aggregate_from_address(), self.ctx_aggregate_from_name(), - self.ctx_aggregate_max_report_size(), + self.ctx_max_aggregate_report_size(), self.ctx_aggregate_org_name(), self.ctx_aggregate_send_frequency(), self.ctx_aggregate_dkim_sign_domain(), - self.ctx_aggregate_subject(), self.ctx_failure_from_address(), self.ctx_failure_from_name(), self.ctx_failure_send_frequency(), @@ -9104,37 +11189,138 @@ impl DmarcReportSettings { impl Pickle for DmarcReportSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 12, 0, 0); self.aggregate_contact_info.pickle(out); self.aggregate_from_address.pickle(out); self.aggregate_from_name.pickle(out); - self.aggregate_max_report_size.pickle(out); + self.max_aggregate_report_size.pickle(out); self.aggregate_org_name.pickle(out); self.aggregate_send_frequency.pickle(out); self.aggregate_dkim_sign_domain.pickle(out); - self.aggregate_subject.pickle(out); self.failure_from_address.pickle(out); self.failure_from_name.pickle(out); self.failure_send_frequency.pickle(out); self.failure_dkim_sign_domain.pickle(out); self.failure_subject.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.aggregate_contact_info = Pickle::unpickle(stream)?; - this.aggregate_from_address = Pickle::unpickle(stream)?; - this.aggregate_from_name = Pickle::unpickle(stream)?; - this.aggregate_max_report_size = Pickle::unpickle(stream)?; - this.aggregate_org_name = Pickle::unpickle(stream)?; - this.aggregate_send_frequency = Pickle::unpickle(stream)?; - this.aggregate_dkim_sign_domain = Pickle::unpickle(stream)?; - this.aggregate_subject = Pickle::unpickle(stream)?; - this.failure_from_address = Pickle::unpickle(stream)?; - this.failure_from_name = Pickle::unpickle(stream)?; - this.failure_send_frequency = Pickle::unpickle(stream)?; - this.failure_dkim_sign_domain = Pickle::unpickle(stream)?; - this.failure_subject = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Report Subsystem'".to_string(), + ..Default::default() + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5242880".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "daily".to_string(), + ..Default::default() + } + }; + let field6: Expression = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field7: Expression = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field8: Expression = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Report Subsystem'".to_string(), + ..Default::default() + } + }; + let field9: Expression = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "[1, 1d]".to_string(), + ..Default::default() + } + }; + let field10: Expression = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field11: Expression = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'DMARC Authentication Failure Report'".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + aggregate_contact_info: field0, + aggregate_from_address: field1, + aggregate_from_name: field2, + max_aggregate_report_size: field3, + aggregate_org_name: field4, + aggregate_send_frequency: field5, + aggregate_dkim_sign_domain: field6, + failure_from_address: field7, + failure_from_name: field8, + failure_send_frequency: field9, + failure_dkim_sign_domain: field10, + failure_subject: field11, + }) } } @@ -9146,19 +11332,19 @@ impl Default for DmarcReportSettings { ..Default::default() }, aggregate_from_address: Expression { - else_: "'noreply-dmarc@' + system('domain')".to_string(), + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), ..Default::default() }, aggregate_from_name: Expression { else_: "'Report Subsystem'".to_string(), ..Default::default() }, - aggregate_max_report_size: Expression { + max_aggregate_report_size: Expression { else_: "5242880".to_string(), ..Default::default() }, aggregate_org_name: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, aggregate_send_frequency: Expression { @@ -9166,15 +11352,11 @@ impl Default for DmarcReportSettings { ..Default::default() }, aggregate_dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), - ..Default::default() - }, - aggregate_subject: Expression { - else_: "'DMARC Aggregate Report'".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, failure_from_address: Expression { - else_: "'noreply-dmarc@' + system('domain')".to_string(), + else_: "'noreply-dmarc@' + system('report_domain')".to_string(), ..Default::default() }, failure_from_name: Expression { @@ -9186,7 +11368,7 @@ impl Default for DmarcReportSettings { ..Default::default() }, failure_dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, failure_subject: Expression { @@ -9199,56 +11381,59 @@ impl Default for DmarcReportSettings { impl IntoValue for DmarcReportSettings { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(15); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(14); map.insert_unchecked( Property::AggregateContactInfo, - self.aggregate_contact_info.into_value(), + self.aggregate_contact_info.into_value_with(opts), ); map.insert_unchecked( Property::AggregateFromAddress, - self.aggregate_from_address.into_value(), + self.aggregate_from_address.into_value_with(opts), ); map.insert_unchecked( Property::AggregateFromName, - self.aggregate_from_name.into_value(), + self.aggregate_from_name.into_value_with(opts), ); map.insert_unchecked( - Property::AggregateMaxReportSize, - self.aggregate_max_report_size.into_value(), + Property::MaxAggregateReportSize, + self.max_aggregate_report_size.into_value_with(opts), ); map.insert_unchecked( Property::AggregateOrgName, - self.aggregate_org_name.into_value(), + self.aggregate_org_name.into_value_with(opts), ); map.insert_unchecked( Property::AggregateSendFrequency, - self.aggregate_send_frequency.into_value(), + self.aggregate_send_frequency.into_value_with(opts), ); map.insert_unchecked( Property::AggregateDkimSignDomain, - self.aggregate_dkim_sign_domain.into_value(), - ); - map.insert_unchecked( - Property::AggregateSubject, - self.aggregate_subject.into_value(), + self.aggregate_dkim_sign_domain.into_value_with(opts), ); map.insert_unchecked( Property::FailureFromAddress, - self.failure_from_address.into_value(), + self.failure_from_address.into_value_with(opts), ); map.insert_unchecked( Property::FailureFromName, - self.failure_from_name.into_value(), + self.failure_from_name.into_value_with(opts), ); map.insert_unchecked( Property::FailureSendFrequency, - self.failure_send_frequency.into_value(), + self.failure_send_frequency.into_value_with(opts), ); map.insert_unchecked( Property::FailureDkimSignDomain, - self.failure_dkim_sign_domain.into_value(), + self.failure_dkim_sign_domain.into_value_with(opts), + ); + map.insert_unchecked( + Property::FailureSubject, + self.failure_subject.into_value_with(opts), ); - map.insert_unchecked(Property::FailureSubject, self.failure_subject.into_value()); JmapValue::Object(map) } } @@ -9267,8 +11452,8 @@ impl RegistryJsonPropertyPatch for DmarcReportSettings { self.aggregate_from_address.patch(pointer, value) } Some(Property::AggregateFromName) => self.aggregate_from_name.patch(pointer, value), - Some(Property::AggregateMaxReportSize) => { - self.aggregate_max_report_size.patch(pointer, value) + Some(Property::MaxAggregateReportSize) => { + self.max_aggregate_report_size.patch(pointer, value) } Some(Property::AggregateOrgName) => self.aggregate_org_name.patch(pointer, value), Some(Property::AggregateSendFrequency) => { @@ -9277,7 +11462,6 @@ impl RegistryJsonPropertyPatch for DmarcReportSettings { Some(Property::AggregateDkimSignDomain) => { self.aggregate_dkim_sign_domain.patch(pointer, value) } - Some(Property::AggregateSubject) => self.aggregate_subject.patch(pointer, value), Some(Property::FailureFromAddress) => self.failure_from_address.patch(pointer, value), Some(Property::FailureFromName) => self.failure_from_name.patch(pointer, value), Some(Property::FailureSendFrequency) => { @@ -9314,19 +11498,53 @@ impl DmarcSpfResult { impl Pickle for DmarcSpfResult { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.human_result.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.domain.pickle(out); self.scope.pickle(out); self.result.pickle(out); - self.human_result.pickle(out); + if let Some(value) = &self.human_result { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.domain = Pickle::unpickle(stream)?; - this.scope = Pickle::unpickle(stream)?; - this.result = Pickle::unpickle(stream)?; - this.human_result = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SpfDomainScope = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SpfAuthResult = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + domain: field0, + scope: field1, + result: field2, + human_result: field3, + }) } } @@ -9343,11 +11561,18 @@ impl Default for DmarcSpfResult { impl IntoValue for DmarcSpfResult { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Domain, self.domain.into_value()); - map.insert_unchecked(Property::Scope, self.scope.into_value()); - map.insert_unchecked(Property::Result, self.result.into_value()); - map.insert_unchecked(Property::HumanResult, self.human_result.into_value()); + map.insert_unchecked(Property::Domain, self.domain.into_value_with(opts)); + map.insert_unchecked(Property::Scope, self.scope.into_value_with(opts)); + map.insert_unchecked(Property::Result, self.result.into_value_with(opts)); + map.insert_unchecked( + Property::HumanResult, + self.human_result.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -9389,10 +11614,10 @@ impl DmarcTroubleshoot { if value.is_empty() { errors.push(ValidationError::required(Property::MailFrom)); } - let value = &self.to; + let value = &self.rcpt_to; for value in value.iter() { if value.is_empty() { - errors.push(ValidationError::required(Property::To)); + errors.push(ValidationError::required(Property::RcptTo)); } } if let Some(value) = &self.message { @@ -9434,63 +11659,13 @@ impl DmarcTroubleshoot { } } -impl Pickle for DmarcTroubleshoot { - fn pickle(&self, out: &mut Vec) { - self.remote_ip.pickle(out); - self.ehlo_domain.pickle(out); - self.mail_from.pickle(out); - self.to.pickle(out); - self.message.pickle(out); - self.spf_ehlo_domain.pickle(out); - self.spf_ehlo_result.pickle(out); - self.spf_mail_from_domain.pickle(out); - self.spf_mail_from_result.pickle(out); - self.ip_rev_result.pickle(out); - self.ip_rev_ptr.pickle(out); - self.dkim_results.pickle(out); - self.dkim_pass.pickle(out); - self.dkim2_result.pickle(out); - self.dkim2_pass.pickle(out); - self.arc_result.pickle(out); - self.dmarc_result.pickle(out); - self.dmarc_pass.pickle(out); - self.dmarc_policy.pickle(out); - self.elapsed.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.remote_ip = Pickle::unpickle(stream)?; - this.ehlo_domain = Pickle::unpickle(stream)?; - this.mail_from = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - this.message = Pickle::unpickle(stream)?; - this.spf_ehlo_domain = Pickle::unpickle(stream)?; - this.spf_ehlo_result = Pickle::unpickle(stream)?; - this.spf_mail_from_domain = Pickle::unpickle(stream)?; - this.spf_mail_from_result = Pickle::unpickle(stream)?; - this.ip_rev_result = Pickle::unpickle(stream)?; - this.ip_rev_ptr = Pickle::unpickle(stream)?; - this.dkim_results = Pickle::unpickle(stream)?; - this.dkim_pass = Pickle::unpickle(stream)?; - this.dkim2_result = Pickle::unpickle(stream)?; - this.dkim2_pass = Pickle::unpickle(stream)?; - this.arc_result = Pickle::unpickle(stream)?; - this.dmarc_result = Pickle::unpickle(stream)?; - this.dmarc_pass = Pickle::unpickle(stream)?; - this.dmarc_policy = Pickle::unpickle(stream)?; - this.elapsed = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for DmarcTroubleshoot { fn default() -> Self { Self { remote_ip: Default::default(), ehlo_domain: Default::default(), mail_from: Default::default(), - to: Default::default(), + rcpt_to: Default::default(), message: Default::default(), spf_ehlo_domain: Default::default(), spf_ehlo_result: Default::default(), @@ -9513,33 +11688,58 @@ impl Default for DmarcTroubleshoot { impl IntoValue for DmarcTroubleshoot { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(22); - map.insert_unchecked(Property::RemoteIp, self.remote_ip.into_value()); - map.insert_unchecked(Property::EhloDomain, self.ehlo_domain.into_value()); - map.insert_unchecked(Property::MailFrom, self.mail_from.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); - map.insert_unchecked(Property::Message, self.message.into_value()); - map.insert_unchecked(Property::SpfEhloDomain, self.spf_ehlo_domain.into_value()); - map.insert_unchecked(Property::SpfEhloResult, self.spf_ehlo_result.into_value()); + map.insert_unchecked(Property::RemoteIp, self.remote_ip.into_value_with(opts)); + map.insert_unchecked(Property::EhloDomain, self.ehlo_domain.into_value_with(opts)); + map.insert_unchecked(Property::MailFrom, self.mail_from.into_value_with(opts)); + map.insert_unchecked(Property::RcptTo, self.rcpt_to.into_value_with(opts)); + map.insert_unchecked(Property::Message, self.message.into_value_with(opts)); + map.insert_unchecked( + Property::SpfEhloDomain, + self.spf_ehlo_domain.into_value_with(opts), + ); + map.insert_unchecked( + Property::SpfEhloResult, + self.spf_ehlo_result.into_value_with(opts), + ); map.insert_unchecked( Property::SpfMailFromDomain, - self.spf_mail_from_domain.into_value(), + self.spf_mail_from_domain.into_value_with(opts), ); map.insert_unchecked( Property::SpfMailFromResult, - self.spf_mail_from_result.into_value(), + self.spf_mail_from_result.into_value_with(opts), ); - map.insert_unchecked(Property::IpRevResult, self.ip_rev_result.into_value()); - map.insert_unchecked(Property::IpRevPtr, self.ip_rev_ptr.into_value()); - map.insert_unchecked(Property::DkimResults, self.dkim_results.into_value()); - map.insert_unchecked(Property::DkimPass, self.dkim_pass.into_value()); - map.insert_unchecked(Property::Dkim2Result, self.dkim2_result.into_value()); - map.insert_unchecked(Property::Dkim2Pass, self.dkim2_pass.into_value()); - map.insert_unchecked(Property::ArcResult, self.arc_result.into_value()); - map.insert_unchecked(Property::DmarcResult, self.dmarc_result.into_value()); - map.insert_unchecked(Property::DmarcPass, self.dmarc_pass.into_value()); - map.insert_unchecked(Property::DmarcPolicy, self.dmarc_policy.into_value()); - map.insert_unchecked(Property::Elapsed, self.elapsed.into_value()); + map.insert_unchecked( + Property::IpRevResult, + self.ip_rev_result.into_value_with(opts), + ); + map.insert_unchecked(Property::IpRevPtr, self.ip_rev_ptr.into_value_with(opts)); + map.insert_unchecked( + Property::DkimResults, + self.dkim_results.into_value_with(opts), + ); + map.insert_unchecked(Property::DkimPass, self.dkim_pass.into_value_with(opts)); + map.insert_unchecked( + Property::Dkim2Result, + self.dkim2_result.into_value_with(opts), + ); + map.insert_unchecked(Property::Dkim2Pass, self.dkim2_pass.into_value_with(opts)); + map.insert_unchecked(Property::ArcResult, self.arc_result.into_value_with(opts)); + map.insert_unchecked( + Property::DmarcResult, + self.dmarc_result.into_value_with(opts), + ); + map.insert_unchecked(Property::DmarcPass, self.dmarc_pass.into_value_with(opts)); + map.insert_unchecked( + Property::DmarcPolicy, + self.dmarc_policy.into_value_with(opts), + ); + map.insert_unchecked(Property::Elapsed, self.elapsed.into_value_with(opts)); JmapValue::Object(map) } } @@ -9556,8 +11756,8 @@ impl RegistryJsonPropertyPatch for DmarcTroubleshoot { Some(Property::MailFrom) => self .mail_from .patch(pointer.with_validators(&[StringValidator::Email]), value), - Some(Property::To) => self - .to + Some(Property::RcptTo) => self + .rcpt_to .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::Message) => self.message.patch(pointer, value), Some(Property::SpfEhloDomain) => self.spf_ehlo_domain.patch(pointer, value), @@ -9604,54 +11804,12 @@ impl Default for DmarcTroubleshootAuthResult { } } -impl Pickle for DmarcTroubleshootAuthResult { - fn pickle(&self, out: &mut Vec) { - match self { - DmarcTroubleshootAuthResult::Pass => { - 0u16.pickle(out); - } - DmarcTroubleshootAuthResult::Fail(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - DmarcTroubleshootAuthResult::SoftFail(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - DmarcTroubleshootAuthResult::TempError(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - DmarcTroubleshootAuthResult::PermError(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - DmarcTroubleshootAuthResult::Neutral(inner) => { - 5u16.pickle(out); - inner.pickle(out); - } - DmarcTroubleshootAuthResult::None => { - 6u16.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(DmarcTroubleshootAuthResult::Pass), - 1 => Pickle::unpickle(stream).map(DmarcTroubleshootAuthResult::Fail), - 2 => Pickle::unpickle(stream).map(DmarcTroubleshootAuthResult::SoftFail), - 3 => Pickle::unpickle(stream).map(DmarcTroubleshootAuthResult::TempError), - 4 => Pickle::unpickle(stream).map(DmarcTroubleshootAuthResult::PermError), - 5 => Pickle::unpickle(stream).map(DmarcTroubleshootAuthResult::Neutral), - 6 => Some(DmarcTroubleshootAuthResult::None), - _ => None, - } - } -} - impl IntoValue for DmarcTroubleshootAuthResult { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DmarcTroubleshootAuthResult::Pass => { let mut obj = jmap_tools::Map::new(); @@ -9659,35 +11817,35 @@ impl IntoValue for DmarcTroubleshootAuthResult { JmapValue::Object(obj) } DmarcTroubleshootAuthResult::Fail(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Fail".into())); obj } DmarcTroubleshootAuthResult::SoftFail(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("SoftFail".into())); obj } DmarcTroubleshootAuthResult::TempError(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TempError".into())); obj } DmarcTroubleshootAuthResult::PermError(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PermError".into())); obj } DmarcTroubleshootAuthResult::Neutral(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Neutral".into())); @@ -9767,18 +11925,6 @@ impl DmarcTroubleshootDetails { } } -impl Pickle for DmarcTroubleshootDetails { - fn pickle(&self, out: &mut Vec) { - self.details.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.details = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for DmarcTroubleshootDetails { fn default() -> Self { Self { @@ -9789,8 +11935,12 @@ impl Default for DmarcTroubleshootDetails { impl IntoValue for DmarcTroubleshootDetails { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Details, self.details.into_value()); + map.insert_unchecked(Property::Details, self.details.into_value_with(opts)); JmapValue::Object(map) } } @@ -9832,17 +11982,36 @@ impl DnsCustomResolver { impl Pickle for DnsCustomResolver { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.protocol.pickle(out); self.address.pickle(out); self.port.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.protocol = Pickle::unpickle(stream)?; - this.address = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: DnsResolverProtocol = if block.has(0) { + Pickle::unpickle(stream)? + } else { + DnsResolverProtocol::Udp + }; + let field1: IpAddr = if block.has(1) { + Pickle::unpickle(stream)? + } else { + IpAddr::from_str("127.0.0.1").unwrap() + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 53u64 + }; + stream.close_block(block); + Some(Self { + protocol: field0, + address: field1, + port: field2, + }) } } @@ -9858,10 +12027,14 @@ impl Default for DnsCustomResolver { impl IntoValue for DnsCustomResolver { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Protocol, self.protocol.into_value()); - map.insert_unchecked(Property::Address, self.address.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); + map.insert_unchecked(Property::Protocol, self.protocol.into_value_with(opts)); + map.insert_unchecked(Property::Address, self.address.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); JmapValue::Object(map) } } @@ -9935,6 +12108,10 @@ impl Pickle for DnsManagement { impl IntoValue for DnsManagement { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DnsManagement::Manual => { let mut obj = jmap_tools::Map::new(); @@ -9942,7 +12119,7 @@ impl IntoValue for DnsManagement { JmapValue::Object(obj) } DnsManagement::Automatic(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Automatic".into())); @@ -10008,17 +12185,58 @@ impl DnsManagementProperties { impl Pickle for DnsManagementProperties { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.origin.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 3, 1, bitmap); self.dns_server_id.pickle(out); - self.origin.pickle(out); + if let Some(value) = &self.origin { + value.pickle(out); + } self.publish_records.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.dns_server_id = Pickle::unpickle(stream)?; - this.origin = Pickle::unpickle(stream)?; - this.publish_records = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + DnsRecordType::Dkim, + DnsRecordType::Spf, + DnsRecordType::Mx, + DnsRecordType::Dmarc, + DnsRecordType::Srv, + DnsRecordType::MtaSts, + DnsRecordType::TlsRpt, + DnsRecordType::Caa, + DnsRecordType::AutoConfig, + DnsRecordType::AutoConfigLegacy, + DnsRecordType::AutoDiscover, + ]) + }; + stream.close_block(block); + Some(Self { + dns_server_id: field0, + origin: field1, + publish_records: field2, + }) } } @@ -10046,10 +12264,20 @@ impl Default for DnsManagementProperties { impl IntoValue for DnsManagementProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::DnsServerId, self.dns_server_id.into_value()); - map.insert_unchecked(Property::Origin, self.origin.into_value()); - map.insert_unchecked(Property::PublishRecords, self.publish_records.into_value()); + map.insert_unchecked( + Property::DnsServerId, + self.dns_server_id.into_value_with(opts), + ); + map.insert_unchecked(Property::Origin, self.origin.into_value_with(opts)); + map.insert_unchecked( + Property::PublishRecords, + self.publish_records.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -10126,6 +12354,9 @@ impl Pickle for DnsResolver { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(DnsResolver::System), 1 => Pickle::unpickle(stream).map(DnsResolver::Custom), @@ -10139,37 +12370,41 @@ impl Pickle for DnsResolver { impl IntoValue for DnsResolver { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DnsResolver::System(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("System".into())); obj } DnsResolver::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); obj } DnsResolver::Cloudflare(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Cloudflare".into())); obj } DnsResolver::Quad9(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Quad9".into())); obj } DnsResolver::Google(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Google".into())); @@ -10224,23 +12459,52 @@ impl DnsResolverCommon { impl Pickle for DnsResolverCommon { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.enable_edns { + bitmap |= 1 << 0; + } + if self.preserve_intermediates { + bitmap |= 1 << 1; + } + if self.tcp_on_error { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 6, 3, bitmap); self.attempts.pickle(out); - self.concurrency.pickle(out); - self.enable_edns.pickle(out); - self.preserve_intermediates.pickle(out); + self.max_concurrent_requests.pickle(out); self.timeout.pickle(out); - self.tcp_on_error.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.attempts = Pickle::unpickle(stream)?; - this.concurrency = Pickle::unpickle(stream)?; - this.enable_edns = Pickle::unpickle(stream)?; - this.preserve_intermediates = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.tcp_on_error = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field2: bool = if block.has(2) { block.flag(0) } else { true }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field5: bool = if block.has(5) { block.flag(2) } else { true }; + stream.close_block(block); + Some(Self { + attempts: field0, + max_concurrent_requests: field1, + enable_edns: field2, + preserve_intermediates: field3, + timeout: field4, + tcp_on_error: field5, + }) } } @@ -10248,7 +12512,7 @@ impl Default for DnsResolverCommon { fn default() -> Self { Self { attempts: 2u64, - concurrency: 2u64, + max_concurrent_requests: 2u64, enable_edns: true, preserve_intermediates: true, timeout: Duration::from_millis(5000), @@ -10259,16 +12523,26 @@ impl Default for DnsResolverCommon { impl IntoValue for DnsResolverCommon { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Attempts, self.attempts.into_value()); - map.insert_unchecked(Property::Concurrency, self.concurrency.into_value()); - map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value()); + map.insert_unchecked(Property::Attempts, self.attempts.into_value_with(opts)); + map.insert_unchecked( + Property::MaxConcurrentRequests, + self.max_concurrent_requests.into_value_with(opts), + ); + map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value_with(opts)); map.insert_unchecked( Property::PreserveIntermediates, - self.preserve_intermediates.into_value(), + self.preserve_intermediates.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::TcpOnError, + self.tcp_on_error.into_value_with(opts), ); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::TcpOnError, self.tcp_on_error.into_value()); JmapValue::Object(map) } } @@ -10281,7 +12555,9 @@ impl RegistryJsonPropertyPatch for DnsResolverCommon { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::Attempts) => self.attempts.patch(pointer, value), - Some(Property::Concurrency) => self.concurrency.patch(pointer, value), + Some(Property::MaxConcurrentRequests) => { + self.max_concurrent_requests.patch(pointer, value) + } Some(Property::EnableEdns) => self.enable_edns.patch(pointer, value), Some(Property::PreserveIntermediates) => { self.preserve_intermediates.patch(pointer, value) @@ -10313,25 +12589,59 @@ impl DnsResolverCustom { impl Pickle for DnsResolverCustom { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.enable_edns { + bitmap |= 1 << 0; + } + if self.preserve_intermediates { + bitmap |= 1 << 1; + } + if self.tcp_on_error { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 7, 3, bitmap); self.servers.pickle(out); self.attempts.pickle(out); - self.concurrency.pickle(out); - self.enable_edns.pickle(out); - self.preserve_intermediates.pickle(out); + self.max_concurrent_requests.pickle(out); self.timeout.pickle(out); - self.tcp_on_error.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.servers = Pickle::unpickle(stream)?; - this.attempts = Pickle::unpickle(stream)?; - this.concurrency = Pickle::unpickle(stream)?; - this.enable_edns = Pickle::unpickle(stream)?; - this.preserve_intermediates = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.tcp_on_error = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: List = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field3: bool = if block.has(3) { block.flag(0) } else { true }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field6: bool = if block.has(6) { block.flag(2) } else { true }; + stream.close_block(block); + Some(Self { + servers: field0, + attempts: field1, + max_concurrent_requests: field2, + enable_edns: field3, + preserve_intermediates: field4, + timeout: field5, + tcp_on_error: field6, + }) } } @@ -10340,7 +12650,7 @@ impl Default for DnsResolverCustom { Self { servers: Default::default(), attempts: 2u64, - concurrency: 2u64, + max_concurrent_requests: 2u64, enable_edns: true, preserve_intermediates: true, timeout: Duration::from_millis(5000), @@ -10351,17 +12661,27 @@ impl Default for DnsResolverCustom { impl IntoValue for DnsResolverCustom { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Servers, self.servers.into_value()); - map.insert_unchecked(Property::Attempts, self.attempts.into_value()); - map.insert_unchecked(Property::Concurrency, self.concurrency.into_value()); - map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value()); + map.insert_unchecked(Property::Servers, self.servers.into_value_with(opts)); + map.insert_unchecked(Property::Attempts, self.attempts.into_value_with(opts)); + map.insert_unchecked( + Property::MaxConcurrentRequests, + self.max_concurrent_requests.into_value_with(opts), + ); + map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value_with(opts)); map.insert_unchecked( Property::PreserveIntermediates, - self.preserve_intermediates.into_value(), + self.preserve_intermediates.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::TcpOnError, + self.tcp_on_error.into_value_with(opts), ); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::TcpOnError, self.tcp_on_error.into_value()); JmapValue::Object(map) } } @@ -10375,7 +12695,9 @@ impl RegistryJsonPropertyPatch for DnsResolverCustom { match pointer.next_property() { Some(Property::Servers) => self.servers.patch(pointer, value), Some(Property::Attempts) => self.attempts.patch(pointer, value), - Some(Property::Concurrency) => self.concurrency.patch(pointer, value), + Some(Property::MaxConcurrentRequests) => { + self.max_concurrent_requests.patch(pointer, value) + } Some(Property::EnableEdns) => self.enable_edns.patch(pointer, value), Some(Property::PreserveIntermediates) => { self.preserve_intermediates.patch(pointer, value) @@ -10399,25 +12721,57 @@ impl DnsResolverTls { impl Pickle for DnsResolverTls { fn pickle(&self, out: &mut Vec) { - self.use_tls.pickle(out); + let mut bitmap = 0u64; + if self.use_tls { + bitmap |= 1 << 0; + } + if self.enable_edns { + bitmap |= 1 << 1; + } + if self.preserve_intermediates { + bitmap |= 1 << 2; + } + if self.tcp_on_error { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 7, 4, bitmap); self.attempts.pickle(out); - self.concurrency.pickle(out); - self.enable_edns.pickle(out); - self.preserve_intermediates.pickle(out); + self.max_concurrent_requests.pickle(out); self.timeout.pickle(out); - self.tcp_on_error.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.use_tls = Pickle::unpickle(stream)?; - this.attempts = Pickle::unpickle(stream)?; - this.concurrency = Pickle::unpickle(stream)?; - this.enable_edns = Pickle::unpickle(stream)?; - this.preserve_intermediates = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.tcp_on_error = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: bool = if block.has(4) { block.flag(2) } else { true }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field6: bool = if block.has(6) { block.flag(3) } else { true }; + stream.close_block(block); + Some(Self { + use_tls: field0, + attempts: field1, + max_concurrent_requests: field2, + enable_edns: field3, + preserve_intermediates: field4, + timeout: field5, + tcp_on_error: field6, + }) } } @@ -10426,7 +12780,7 @@ impl Default for DnsResolverTls { Self { use_tls: true, attempts: 2u64, - concurrency: 2u64, + max_concurrent_requests: 2u64, enable_edns: true, preserve_intermediates: true, timeout: Duration::from_millis(5000), @@ -10437,17 +12791,27 @@ impl Default for DnsResolverTls { impl IntoValue for DnsResolverTls { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); - map.insert_unchecked(Property::Attempts, self.attempts.into_value()); - map.insert_unchecked(Property::Concurrency, self.concurrency.into_value()); - map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value()); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); + map.insert_unchecked(Property::Attempts, self.attempts.into_value_with(opts)); + map.insert_unchecked( + Property::MaxConcurrentRequests, + self.max_concurrent_requests.into_value_with(opts), + ); + map.insert_unchecked(Property::EnableEdns, self.enable_edns.into_value_with(opts)); map.insert_unchecked( Property::PreserveIntermediates, - self.preserve_intermediates.into_value(), + self.preserve_intermediates.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::TcpOnError, + self.tcp_on_error.into_value_with(opts), ); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::TcpOnError, self.tcp_on_error.into_value()); JmapValue::Object(map) } } @@ -10461,7 +12825,9 @@ impl RegistryJsonPropertyPatch for DnsResolverTls { match pointer.next_property() { Some(Property::UseTls) => self.use_tls.patch(pointer, value), Some(Property::Attempts) => self.attempts.patch(pointer, value), - Some(Property::Concurrency) => self.concurrency.patch(pointer, value), + Some(Property::MaxConcurrentRequests) => { + self.max_concurrent_requests.patch(pointer, value) + } Some(Property::EnableEdns) => self.enable_edns.patch(pointer, value), Some(Property::PreserveIntermediates) => { self.preserve_intermediates.patch(pointer, value) @@ -10479,13 +12845,12 @@ impl RegistryJsonPropertyPatch for DnsResolverTls { impl ObjectImpl for DnsServer { const FLAGS: u64 = OBJ_FILTER_TENANT; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::DnsServer; fn validate(&self, errors: &mut Vec) -> bool { match self { DnsServer::Tsig(inner) => inner.validate(errors), - DnsServer::Deprecated1 => true, DnsServer::Cloudflare(inner) => inner.validate(errors), DnsServer::DigitalOcean(inner) => inner.validate(errors), DnsServer::DeSEC(inner) => inner.validate(errors), @@ -10563,7 +12928,6 @@ impl ObjectImpl for DnsServer { DnsServer::Tsig(object) => { object.index(i); } - DnsServer::Deprecated1 => {} DnsServer::Cloudflare(object) => { object.index(i); } @@ -10788,361 +13152,360 @@ impl Pickle for DnsServer { 0u16.pickle(out); inner.pickle(out); } - DnsServer::Deprecated1 => { - 1u16.pickle(out); - } DnsServer::Cloudflare(inner) => { - 2u16.pickle(out); + 1u16.pickle(out); inner.pickle(out); } DnsServer::DigitalOcean(inner) => { - 3u16.pickle(out); + 2u16.pickle(out); inner.pickle(out); } DnsServer::DeSEC(inner) => { - 4u16.pickle(out); + 3u16.pickle(out); inner.pickle(out); } DnsServer::Ovh(inner) => { - 5u16.pickle(out); + 4u16.pickle(out); inner.pickle(out); } DnsServer::Bunny(inner) => { - 6u16.pickle(out); + 5u16.pickle(out); inner.pickle(out); } DnsServer::Porkbun(inner) => { - 7u16.pickle(out); + 6u16.pickle(out); inner.pickle(out); } DnsServer::Dnsimple(inner) => { - 8u16.pickle(out); + 7u16.pickle(out); inner.pickle(out); } DnsServer::Spaceship(inner) => { - 9u16.pickle(out); + 8u16.pickle(out); inner.pickle(out); } DnsServer::Route53(inner) => { - 10u16.pickle(out); + 9u16.pickle(out); inner.pickle(out); } DnsServer::GoogleCloudDns(inner) => { - 11u16.pickle(out); + 10u16.pickle(out); inner.pickle(out); } DnsServer::Alidns(inner) => { - 12u16.pickle(out); + 11u16.pickle(out); inner.pickle(out); } DnsServer::ArvanCloud(inner) => { - 13u16.pickle(out); + 12u16.pickle(out); inner.pickle(out); } DnsServer::Autodns(inner) => { - 14u16.pickle(out); + 13u16.pickle(out); inner.pickle(out); } DnsServer::AzureDns(inner) => { - 15u16.pickle(out); + 14u16.pickle(out); inner.pickle(out); } DnsServer::BaiduCloud(inner) => { - 16u16.pickle(out); + 15u16.pickle(out); inner.pickle(out); } DnsServer::BluecatV2(inner) => { - 17u16.pickle(out); + 16u16.pickle(out); inner.pickle(out); } DnsServer::ClouDns(inner) => { - 18u16.pickle(out); + 17u16.pickle(out); inner.pickle(out); } DnsServer::Constellix(inner) => { - 19u16.pickle(out); + 18u16.pickle(out); inner.pickle(out); } DnsServer::Cpanel(inner) => { - 20u16.pickle(out); + 19u16.pickle(out); inner.pickle(out); } DnsServer::Ddnss(inner) => { - 21u16.pickle(out); + 20u16.pickle(out); inner.pickle(out); } DnsServer::DnsMadeEasy(inner) => { - 22u16.pickle(out); + 21u16.pickle(out); inner.pickle(out); } DnsServer::Domeneshop(inner) => { - 23u16.pickle(out); + 22u16.pickle(out); inner.pickle(out); } DnsServer::Dreamhost(inner) => { - 24u16.pickle(out); + 23u16.pickle(out); inner.pickle(out); } DnsServer::DuckDns(inner) => { - 25u16.pickle(out); + 24u16.pickle(out); inner.pickle(out); } DnsServer::Dynu(inner) => { - 26u16.pickle(out); + 25u16.pickle(out); inner.pickle(out); } DnsServer::EasyDns(inner) => { - 27u16.pickle(out); + 26u16.pickle(out); inner.pickle(out); } DnsServer::EdgeDns(inner) => { - 28u16.pickle(out); + 27u16.pickle(out); inner.pickle(out); } DnsServer::Exoscale(inner) => { - 29u16.pickle(out); + 28u16.pickle(out); inner.pickle(out); } DnsServer::FreeMyIp(inner) => { - 30u16.pickle(out); + 29u16.pickle(out); inner.pickle(out); } DnsServer::GandiV5(inner) => { - 31u16.pickle(out); + 30u16.pickle(out); inner.pickle(out); } DnsServer::Gcore(inner) => { - 32u16.pickle(out); + 31u16.pickle(out); inner.pickle(out); } DnsServer::Glesys(inner) => { - 33u16.pickle(out); + 32u16.pickle(out); inner.pickle(out); } DnsServer::Godaddy(inner) => { - 34u16.pickle(out); + 33u16.pickle(out); inner.pickle(out); } DnsServer::Hetzner(inner) => { - 35u16.pickle(out); + 34u16.pickle(out); inner.pickle(out); } DnsServer::HostingDe(inner) => { - 36u16.pickle(out); + 35u16.pickle(out); inner.pickle(out); } DnsServer::Hostinger(inner) => { - 37u16.pickle(out); + 36u16.pickle(out); inner.pickle(out); } DnsServer::HuaweiCloud(inner) => { - 38u16.pickle(out); + 37u16.pickle(out); inner.pickle(out); } DnsServer::Hurricane(inner) => { - 39u16.pickle(out); + 38u16.pickle(out); inner.pickle(out); } DnsServer::IbmCloud(inner) => { - 40u16.pickle(out); + 39u16.pickle(out); inner.pickle(out); } DnsServer::Infoblox(inner) => { - 41u16.pickle(out); + 40u16.pickle(out); inner.pickle(out); } DnsServer::Infomaniak(inner) => { - 42u16.pickle(out); + 41u16.pickle(out); inner.pickle(out); } DnsServer::Inwx(inner) => { - 43u16.pickle(out); + 42u16.pickle(out); inner.pickle(out); } DnsServer::Ionos(inner) => { - 44u16.pickle(out); + 43u16.pickle(out); inner.pickle(out); } DnsServer::Ipv64(inner) => { - 45u16.pickle(out); + 44u16.pickle(out); inner.pickle(out); } DnsServer::Joker(inner) => { - 46u16.pickle(out); + 45u16.pickle(out); inner.pickle(out); } DnsServer::Lightsail(inner) => { - 47u16.pickle(out); + 46u16.pickle(out); inner.pickle(out); } DnsServer::Linode(inner) => { - 48u16.pickle(out); + 47u16.pickle(out); inner.pickle(out); } DnsServer::LuaDns(inner) => { - 49u16.pickle(out); + 48u16.pickle(out); inner.pickle(out); } DnsServer::MythicBeasts(inner) => { - 50u16.pickle(out); + 49u16.pickle(out); inner.pickle(out); } DnsServer::Namecheap(inner) => { - 51u16.pickle(out); + 50u16.pickle(out); inner.pickle(out); } DnsServer::NameDotCom(inner) => { - 52u16.pickle(out); + 51u16.pickle(out); inner.pickle(out); } DnsServer::NameSilo(inner) => { - 53u16.pickle(out); + 52u16.pickle(out); inner.pickle(out); } DnsServer::Netcup(inner) => { - 54u16.pickle(out); + 53u16.pickle(out); inner.pickle(out); } DnsServer::Netlify(inner) => { - 55u16.pickle(out); + 54u16.pickle(out); inner.pickle(out); } DnsServer::Nifcloud(inner) => { - 56u16.pickle(out); + 55u16.pickle(out); inner.pickle(out); } DnsServer::Ns1(inner) => { - 57u16.pickle(out); + 56u16.pickle(out); inner.pickle(out); } DnsServer::OracleCloud(inner) => { - 58u16.pickle(out); + 57u16.pickle(out); inner.pickle(out); } DnsServer::Plesk(inner) => { - 59u16.pickle(out); + 58u16.pickle(out); inner.pickle(out); } DnsServer::Safedns(inner) => { - 60u16.pickle(out); + 59u16.pickle(out); inner.pickle(out); } DnsServer::Scaleway(inner) => { - 61u16.pickle(out); + 60u16.pickle(out); inner.pickle(out); } DnsServer::TencentCloud(inner) => { - 62u16.pickle(out); + 61u16.pickle(out); inner.pickle(out); } DnsServer::Transip(inner) => { - 63u16.pickle(out); + 62u16.pickle(out); inner.pickle(out); } DnsServer::UltraDns(inner) => { - 64u16.pickle(out); + 63u16.pickle(out); inner.pickle(out); } DnsServer::Vercel(inner) => { - 65u16.pickle(out); + 64u16.pickle(out); inner.pickle(out); } DnsServer::Volcengine(inner) => { - 66u16.pickle(out); + 65u16.pickle(out); inner.pickle(out); } DnsServer::Vultr(inner) => { - 67u16.pickle(out); + 66u16.pickle(out); inner.pickle(out); } DnsServer::WebSupport(inner) => { - 68u16.pickle(out); + 67u16.pickle(out); inner.pickle(out); } DnsServer::YandexCloud(inner) => { - 69u16.pickle(out); + 68u16.pickle(out); inner.pickle(out); } DnsServer::PowerDns(inner) => { - 70u16.pickle(out); + 69u16.pickle(out); inner.pickle(out); } } } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(DnsServer::Tsig), - 1 => Some(DnsServer::Deprecated1), - 2 => Pickle::unpickle(stream).map(DnsServer::Cloudflare), - 3 => Pickle::unpickle(stream).map(DnsServer::DigitalOcean), - 4 => Pickle::unpickle(stream).map(DnsServer::DeSEC), - 5 => Pickle::unpickle(stream).map(DnsServer::Ovh), - 6 => Pickle::unpickle(stream).map(DnsServer::Bunny), - 7 => Pickle::unpickle(stream).map(DnsServer::Porkbun), - 8 => Pickle::unpickle(stream).map(DnsServer::Dnsimple), - 9 => Pickle::unpickle(stream).map(DnsServer::Spaceship), - 10 => Pickle::unpickle(stream).map(DnsServer::Route53), - 11 => Pickle::unpickle(stream).map(DnsServer::GoogleCloudDns), - 12 => Pickle::unpickle(stream).map(DnsServer::Alidns), - 13 => Pickle::unpickle(stream).map(DnsServer::ArvanCloud), - 14 => Pickle::unpickle(stream).map(DnsServer::Autodns), - 15 => Pickle::unpickle(stream).map(DnsServer::AzureDns), - 16 => Pickle::unpickle(stream).map(DnsServer::BaiduCloud), - 17 => Pickle::unpickle(stream).map(DnsServer::BluecatV2), - 18 => Pickle::unpickle(stream).map(DnsServer::ClouDns), - 19 => Pickle::unpickle(stream).map(DnsServer::Constellix), - 20 => Pickle::unpickle(stream).map(DnsServer::Cpanel), - 21 => Pickle::unpickle(stream).map(DnsServer::Ddnss), - 22 => Pickle::unpickle(stream).map(DnsServer::DnsMadeEasy), - 23 => Pickle::unpickle(stream).map(DnsServer::Domeneshop), - 24 => Pickle::unpickle(stream).map(DnsServer::Dreamhost), - 25 => Pickle::unpickle(stream).map(DnsServer::DuckDns), - 26 => Pickle::unpickle(stream).map(DnsServer::Dynu), - 27 => Pickle::unpickle(stream).map(DnsServer::EasyDns), - 28 => Pickle::unpickle(stream).map(DnsServer::EdgeDns), - 29 => Pickle::unpickle(stream).map(DnsServer::Exoscale), - 30 => Pickle::unpickle(stream).map(DnsServer::FreeMyIp), - 31 => Pickle::unpickle(stream).map(DnsServer::GandiV5), - 32 => Pickle::unpickle(stream).map(DnsServer::Gcore), - 33 => Pickle::unpickle(stream).map(DnsServer::Glesys), - 34 => Pickle::unpickle(stream).map(DnsServer::Godaddy), - 35 => Pickle::unpickle(stream).map(DnsServer::Hetzner), - 36 => Pickle::unpickle(stream).map(DnsServer::HostingDe), - 37 => Pickle::unpickle(stream).map(DnsServer::Hostinger), - 38 => Pickle::unpickle(stream).map(DnsServer::HuaweiCloud), - 39 => Pickle::unpickle(stream).map(DnsServer::Hurricane), - 40 => Pickle::unpickle(stream).map(DnsServer::IbmCloud), - 41 => Pickle::unpickle(stream).map(DnsServer::Infoblox), - 42 => Pickle::unpickle(stream).map(DnsServer::Infomaniak), - 43 => Pickle::unpickle(stream).map(DnsServer::Inwx), - 44 => Pickle::unpickle(stream).map(DnsServer::Ionos), - 45 => Pickle::unpickle(stream).map(DnsServer::Ipv64), - 46 => Pickle::unpickle(stream).map(DnsServer::Joker), - 47 => Pickle::unpickle(stream).map(DnsServer::Lightsail), - 48 => Pickle::unpickle(stream).map(DnsServer::Linode), - 49 => Pickle::unpickle(stream).map(DnsServer::LuaDns), - 50 => Pickle::unpickle(stream).map(DnsServer::MythicBeasts), - 51 => Pickle::unpickle(stream).map(DnsServer::Namecheap), - 52 => Pickle::unpickle(stream).map(DnsServer::NameDotCom), - 53 => Pickle::unpickle(stream).map(DnsServer::NameSilo), - 54 => Pickle::unpickle(stream).map(DnsServer::Netcup), - 55 => Pickle::unpickle(stream).map(DnsServer::Netlify), - 56 => Pickle::unpickle(stream).map(DnsServer::Nifcloud), - 57 => Pickle::unpickle(stream).map(DnsServer::Ns1), - 58 => Pickle::unpickle(stream).map(DnsServer::OracleCloud), - 59 => Pickle::unpickle(stream).map(DnsServer::Plesk), - 60 => Pickle::unpickle(stream).map(DnsServer::Safedns), - 61 => Pickle::unpickle(stream).map(DnsServer::Scaleway), - 62 => Pickle::unpickle(stream).map(DnsServer::TencentCloud), - 63 => Pickle::unpickle(stream).map(DnsServer::Transip), - 64 => Pickle::unpickle(stream).map(DnsServer::UltraDns), - 65 => Pickle::unpickle(stream).map(DnsServer::Vercel), - 66 => Pickle::unpickle(stream).map(DnsServer::Volcengine), - 67 => Pickle::unpickle(stream).map(DnsServer::Vultr), - 68 => Pickle::unpickle(stream).map(DnsServer::WebSupport), - 69 => Pickle::unpickle(stream).map(DnsServer::YandexCloud), - 70 => Pickle::unpickle(stream).map(DnsServer::PowerDns), + 1 => Pickle::unpickle(stream).map(DnsServer::Cloudflare), + 2 => Pickle::unpickle(stream).map(DnsServer::DigitalOcean), + 3 => Pickle::unpickle(stream).map(DnsServer::DeSEC), + 4 => Pickle::unpickle(stream).map(DnsServer::Ovh), + 5 => Pickle::unpickle(stream).map(DnsServer::Bunny), + 6 => Pickle::unpickle(stream).map(DnsServer::Porkbun), + 7 => Pickle::unpickle(stream).map(DnsServer::Dnsimple), + 8 => Pickle::unpickle(stream).map(DnsServer::Spaceship), + 9 => Pickle::unpickle(stream).map(DnsServer::Route53), + 10 => Pickle::unpickle(stream).map(DnsServer::GoogleCloudDns), + 11 => Pickle::unpickle(stream).map(DnsServer::Alidns), + 12 => Pickle::unpickle(stream).map(DnsServer::ArvanCloud), + 13 => Pickle::unpickle(stream).map(DnsServer::Autodns), + 14 => Pickle::unpickle(stream).map(DnsServer::AzureDns), + 15 => Pickle::unpickle(stream).map(DnsServer::BaiduCloud), + 16 => Pickle::unpickle(stream).map(DnsServer::BluecatV2), + 17 => Pickle::unpickle(stream).map(DnsServer::ClouDns), + 18 => Pickle::unpickle(stream).map(DnsServer::Constellix), + 19 => Pickle::unpickle(stream).map(DnsServer::Cpanel), + 20 => Pickle::unpickle(stream).map(DnsServer::Ddnss), + 21 => Pickle::unpickle(stream).map(DnsServer::DnsMadeEasy), + 22 => Pickle::unpickle(stream).map(DnsServer::Domeneshop), + 23 => Pickle::unpickle(stream).map(DnsServer::Dreamhost), + 24 => Pickle::unpickle(stream).map(DnsServer::DuckDns), + 25 => Pickle::unpickle(stream).map(DnsServer::Dynu), + 26 => Pickle::unpickle(stream).map(DnsServer::EasyDns), + 27 => Pickle::unpickle(stream).map(DnsServer::EdgeDns), + 28 => Pickle::unpickle(stream).map(DnsServer::Exoscale), + 29 => Pickle::unpickle(stream).map(DnsServer::FreeMyIp), + 30 => Pickle::unpickle(stream).map(DnsServer::GandiV5), + 31 => Pickle::unpickle(stream).map(DnsServer::Gcore), + 32 => Pickle::unpickle(stream).map(DnsServer::Glesys), + 33 => Pickle::unpickle(stream).map(DnsServer::Godaddy), + 34 => Pickle::unpickle(stream).map(DnsServer::Hetzner), + 35 => Pickle::unpickle(stream).map(DnsServer::HostingDe), + 36 => Pickle::unpickle(stream).map(DnsServer::Hostinger), + 37 => Pickle::unpickle(stream).map(DnsServer::HuaweiCloud), + 38 => Pickle::unpickle(stream).map(DnsServer::Hurricane), + 39 => Pickle::unpickle(stream).map(DnsServer::IbmCloud), + 40 => Pickle::unpickle(stream).map(DnsServer::Infoblox), + 41 => Pickle::unpickle(stream).map(DnsServer::Infomaniak), + 42 => Pickle::unpickle(stream).map(DnsServer::Inwx), + 43 => Pickle::unpickle(stream).map(DnsServer::Ionos), + 44 => Pickle::unpickle(stream).map(DnsServer::Ipv64), + 45 => Pickle::unpickle(stream).map(DnsServer::Joker), + 46 => Pickle::unpickle(stream).map(DnsServer::Lightsail), + 47 => Pickle::unpickle(stream).map(DnsServer::Linode), + 48 => Pickle::unpickle(stream).map(DnsServer::LuaDns), + 49 => Pickle::unpickle(stream).map(DnsServer::MythicBeasts), + 50 => Pickle::unpickle(stream).map(DnsServer::Namecheap), + 51 => Pickle::unpickle(stream).map(DnsServer::NameDotCom), + 52 => Pickle::unpickle(stream).map(DnsServer::NameSilo), + 53 => Pickle::unpickle(stream).map(DnsServer::Netcup), + 54 => Pickle::unpickle(stream).map(DnsServer::Netlify), + 55 => Pickle::unpickle(stream).map(DnsServer::Nifcloud), + 56 => Pickle::unpickle(stream).map(DnsServer::Ns1), + 57 => Pickle::unpickle(stream).map(DnsServer::OracleCloud), + 58 => Pickle::unpickle(stream).map(DnsServer::Plesk), + 59 => Pickle::unpickle(stream).map(DnsServer::Safedns), + 60 => Pickle::unpickle(stream).map(DnsServer::Scaleway), + 61 => Pickle::unpickle(stream).map(DnsServer::TencentCloud), + 62 => Pickle::unpickle(stream).map(DnsServer::Transip), + 63 => Pickle::unpickle(stream).map(DnsServer::UltraDns), + 64 => Pickle::unpickle(stream).map(DnsServer::Vercel), + 65 => Pickle::unpickle(stream).map(DnsServer::Volcengine), + 66 => Pickle::unpickle(stream).map(DnsServer::Vultr), + 67 => Pickle::unpickle(stream).map(DnsServer::WebSupport), + 68 => Pickle::unpickle(stream).map(DnsServer::YandexCloud), + 69 => Pickle::unpickle(stream).map(DnsServer::PowerDns), _ => None, } } @@ -11150,497 +13513,496 @@ impl Pickle for DnsServer { impl IntoValue for DnsServer { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DnsServer::Tsig(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Tsig".into())); obj } - DnsServer::Deprecated1 => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Deprecated1".into())); - JmapValue::Object(obj) - } DnsServer::Cloudflare(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Cloudflare".into())); obj } DnsServer::DigitalOcean(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DigitalOcean".into())); obj } DnsServer::DeSEC(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DeSEC".into())); obj } DnsServer::Ovh(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ovh".into())); obj } DnsServer::Bunny(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Bunny".into())); obj } DnsServer::Porkbun(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Porkbun".into())); obj } DnsServer::Dnsimple(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dnsimple".into())); obj } DnsServer::Spaceship(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Spaceship".into())); obj } DnsServer::Route53(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Route53".into())); obj } DnsServer::GoogleCloudDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("GoogleCloudDns".into())); obj } DnsServer::Alidns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Alidns".into())); obj } DnsServer::ArvanCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ArvanCloud".into())); obj } DnsServer::Autodns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Autodns".into())); obj } DnsServer::AzureDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("AzureDns".into())); obj } DnsServer::BaiduCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("BaiduCloud".into())); obj } DnsServer::BluecatV2(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("BluecatV2".into())); obj } DnsServer::ClouDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ClouDns".into())); obj } DnsServer::Constellix(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Constellix".into())); obj } DnsServer::Cpanel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Cpanel".into())); obj } DnsServer::Ddnss(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ddnss".into())); obj } DnsServer::DnsMadeEasy(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DnsMadeEasy".into())); obj } DnsServer::Domeneshop(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Domeneshop".into())); obj } DnsServer::Dreamhost(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dreamhost".into())); obj } DnsServer::DuckDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DuckDns".into())); obj } DnsServer::Dynu(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dynu".into())); obj } DnsServer::EasyDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EasyDns".into())); obj } DnsServer::EdgeDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EdgeDns".into())); obj } DnsServer::Exoscale(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Exoscale".into())); obj } DnsServer::FreeMyIp(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FreeMyIp".into())); obj } DnsServer::GandiV5(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("GandiV5".into())); obj } DnsServer::Gcore(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Gcore".into())); obj } DnsServer::Glesys(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Glesys".into())); obj } DnsServer::Godaddy(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Godaddy".into())); obj } DnsServer::Hetzner(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hetzner".into())); obj } DnsServer::HostingDe(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("HostingDe".into())); obj } DnsServer::Hostinger(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hostinger".into())); obj } DnsServer::HuaweiCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("HuaweiCloud".into())); obj } DnsServer::Hurricane(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hurricane".into())); obj } DnsServer::IbmCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("IbmCloud".into())); obj } DnsServer::Infoblox(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Infoblox".into())); obj } DnsServer::Infomaniak(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Infomaniak".into())); obj } DnsServer::Inwx(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Inwx".into())); obj } DnsServer::Ionos(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ionos".into())); obj } DnsServer::Ipv64(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ipv64".into())); obj } DnsServer::Joker(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Joker".into())); obj } DnsServer::Lightsail(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Lightsail".into())); obj } DnsServer::Linode(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Linode".into())); obj } DnsServer::LuaDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("LuaDns".into())); obj } DnsServer::MythicBeasts(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MythicBeasts".into())); obj } DnsServer::Namecheap(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Namecheap".into())); obj } DnsServer::NameDotCom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("NameDotCom".into())); obj } DnsServer::NameSilo(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("NameSilo".into())); obj } DnsServer::Netcup(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Netcup".into())); obj } DnsServer::Netlify(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Netlify".into())); obj } DnsServer::Nifcloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Nifcloud".into())); obj } DnsServer::Ns1(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ns1".into())); obj } DnsServer::OracleCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("OracleCloud".into())); obj } DnsServer::Plesk(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Plesk".into())); obj } DnsServer::Safedns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Safedns".into())); obj } DnsServer::Scaleway(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Scaleway".into())); obj } DnsServer::TencentCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TencentCloud".into())); obj } DnsServer::Transip(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Transip".into())); obj } DnsServer::UltraDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("UltraDns".into())); obj } DnsServer::Vercel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Vercel".into())); obj } DnsServer::Volcengine(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Volcengine".into())); obj } DnsServer::Vultr(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Vultr".into())); obj } DnsServer::WebSupport(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("WebSupport".into())); obj } DnsServer::YandexCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("YandexCloud".into())); obj } DnsServer::PowerDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PowerDns".into())); @@ -11659,7 +14021,6 @@ impl RegistryJsonPatch for DnsServer { if !pointer.has_next() { match object_type(&pointer, &value)? { DnsServerType::Tsig => *self = DnsServer::Tsig(Default::default()), - DnsServerType::Deprecated1 => *self = DnsServer::Deprecated1, DnsServerType::Cloudflare => *self = DnsServer::Cloudflare(Default::default()), DnsServerType::DigitalOcean => *self = DnsServer::DigitalOcean(Default::default()), DnsServerType::DeSEC => *self = DnsServer::DeSEC(Default::default()), @@ -11735,7 +14096,6 @@ impl RegistryJsonPatch for DnsServer { } match self { DnsServer::Tsig(inner) => inner.patch(pointer, value), - DnsServer::Deprecated1 => pointer.assert_eof(), DnsServer::Cloudflare(inner) => inner.patch(pointer, value), DnsServer::DigitalOcean(inner) => inner.patch(pointer, value), DnsServer::DeSEC(inner) => inner.patch(pointer, value), @@ -11813,7 +14173,6 @@ impl DnsServer { pub fn object_type(&self) -> DnsServerType { match self { DnsServer::Tsig(_) => DnsServerType::Tsig, - DnsServer::Deprecated1 => DnsServerType::Deprecated1, DnsServer::Cloudflare(_) => DnsServerType::Cloudflare, DnsServer::DigitalOcean(_) => DnsServerType::DigitalOcean, DnsServer::DeSEC(_) => DnsServerType::DeSEC, @@ -11890,9 +14249,9 @@ impl DnsServer { impl DnsServerAlidns { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.access_key; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::AccessKey)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -11901,7 +14260,7 @@ impl DnsServerAlidns { errors.push(ValidationError::required(Property::Region)); } } - let value = &self.security_token; + let value = &self.session_token; value.validate(errors); if let Some(value) = &self.line { if value.is_empty() { @@ -11912,69 +14271,145 @@ impl DnsServerAlidns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerAlidns { fn pickle(&self, out: &mut Vec) { - self.access_key.pickle(out); + let mut bitmap = 0u64; + if self.region.is_some() { + bitmap |= 1 << 0; + } + if self.line.is_some() { + bitmap |= 1 << 1; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 12, 4, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); - self.region.pickle(out); - self.security_token.pickle(out); - self.line.pickle(out); + if let Some(value) = &self.region { + value.pickle(out); + } + self.session_token.pickle(out); + if let Some(value) = &self.line { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.security_token = Pickle::unpickle(stream)?; - this.line = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: SecretKeyOptional = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: Option = if block.has(11) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + region: field2, + session_token: field3, + line: field4, + description: field5, + timeout: field7, + ttl: field8, + poll_interval: field9, + propagation_timeout: field10, + propagation_delay: field11, + }) } } impl Default for DnsServerAlidns { fn default() -> Self { Self { - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), region: Default::default(), - security_token: Default::default(), + session_token: Default::default(), line: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -11983,27 +14418,39 @@ impl Default for DnsServerAlidns { impl IntoValue for DnsServerAlidns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::SecurityToken, self.security_token.into_value()); - map.insert_unchecked(Property::Line, self.line.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked( + Property::SessionToken, + self.session_token.into_value_with(opts), + ); + map.insert_unchecked(Property::Line, self.line.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -12016,26 +14463,23 @@ impl RegistryJsonPropertyPatch for DnsServerAlidns { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AccessKey) => self - .access_key + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Region) => self .region .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::SecurityToken) => self.security_token.patch(pointer, value), + Some(Property::SessionToken) => self.session_token.patch(pointer, value), Some(Property::Line) => self .line .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -12060,49 +14504,107 @@ impl DnsServerAutodns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerAutodns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.context.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 10, 3, bitmap); self.username.pickle(out); self.password.pickle(out); - self.context.pickle(out); + if let Some(value) = &self.context { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.context = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + password: field1, + context: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } @@ -12113,10 +14615,9 @@ impl Default for DnsServerAutodns { password: Default::default(), context: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -12125,25 +14626,31 @@ impl Default for DnsServerAutodns { impl IntoValue for DnsServerAutodns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::Context, self.context.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); + map.insert_unchecked(Property::Context, self.context.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -12164,12 +14671,9 @@ impl RegistryJsonPropertyPatch for DnsServerAutodns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -12206,24 +14710,21 @@ impl DnsServerAzureDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerAzureDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 13, 2, bitmap); self.tenant_id.pickle(out); self.client_id.pickle(out); self.client_secret.pickle(out); @@ -12231,30 +14732,97 @@ impl Pickle for DnsServerAzureDns { self.resource_group.pickle(out); self.environment.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tenant_id = Pickle::unpickle(stream)?; - this.client_id = Pickle::unpickle(stream)?; - this.client_secret = Pickle::unpickle(stream)?; - this.subscription_id = Pickle::unpickle(stream)?; - this.resource_group = Pickle::unpickle(stream)?; - this.environment = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: AzureEnvironment = if block.has(5) { + Pickle::unpickle(stream)? + } else { + AzureEnvironment::Public + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field12: Option = if block.has(12) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + tenant_id: field0, + client_id: field1, + client_secret: field2, + subscription_id: field3, + resource_group: field4, + environment: field5, + description: field6, + timeout: field8, + ttl: field9, + poll_interval: field10, + propagation_timeout: field11, + propagation_delay: field12, + }) } } @@ -12268,10 +14836,9 @@ impl Default for DnsServerAzureDns { resource_group: Default::default(), environment: AzureEnvironment::Public, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -12280,28 +14847,46 @@ impl Default for DnsServerAzureDns { impl IntoValue for DnsServerAzureDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::TenantId, self.tenant_id.into_value()); - map.insert_unchecked(Property::ClientId, self.client_id.into_value()); - map.insert_unchecked(Property::ClientSecret, self.client_secret.into_value()); - map.insert_unchecked(Property::SubscriptionId, self.subscription_id.into_value()); - map.insert_unchecked(Property::ResourceGroup, self.resource_group.into_value()); - map.insert_unchecked(Property::Environment, self.environment.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::TenantId, self.tenant_id.into_value_with(opts)); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::ClientSecret, + self.client_secret.into_value_with(opts), + ); + map.insert_unchecked( + Property::SubscriptionId, + self.subscription_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::ResourceGroup, + self.resource_group.into_value_with(opts), + ); + map.insert_unchecked( + Property::Environment, + self.environment.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -12331,12 +14916,9 @@ impl RegistryJsonPropertyPatch for DnsServerAzureDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -12351,9 +14933,9 @@ impl RegistryJsonPropertyPatch for DnsServerAzureDns { impl DnsServerBaiduCloud { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.access_key; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::AccessKey)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -12361,60 +14943,103 @@ impl DnsServerBaiduCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerBaiduCloud { fn pickle(&self, out: &mut Vec) { - self.access_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerBaiduCloud { fn default() -> Self { Self { - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -12423,24 +15048,33 @@ impl Default for DnsServerBaiduCloud { impl IntoValue for DnsServerBaiduCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -12453,19 +15087,16 @@ impl RegistryJsonPropertyPatch for DnsServerBaiduCloud { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AccessKey) => self - .access_key + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -12480,9 +15111,9 @@ impl RegistryJsonPropertyPatch for DnsServerBaiduCloud { impl DnsServerBluecatV2 { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.base_url; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::BaseUrl)); + errors.push(ValidationError::required(Property::Url)); } let value = &self.username; if value.is_empty() { @@ -12502,72 +15133,133 @@ impl DnsServerBluecatV2 { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerBluecatV2 { fn pickle(&self, out: &mut Vec) { - self.base_url.pickle(out); + let mut bitmap = 0u64; + if self.skip_deploy { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 13, 3, bitmap); + self.url.pickle(out); self.username.pickle(out); self.password.pickle(out); self.config_name.pickle(out); self.view_name.pickle(out); - self.skip_deploy.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.base_url = Pickle::unpickle(stream)?; - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.config_name = Pickle::unpickle(stream)?; - this.view_name = Pickle::unpickle(stream)?; - this.skip_deploy = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: bool = if block.has(5) { block.flag(0) } else { false }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field12: Option = if block.has(12) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + username: field1, + password: field2, + config_name: field3, + view_name: field4, + skip_deploy: field5, + description: field6, + timeout: field8, + ttl: field9, + poll_interval: field10, + propagation_timeout: field11, + propagation_delay: field12, + }) } } impl Default for DnsServerBluecatV2 { fn default() -> Self { Self { - base_url: Default::default(), + url: Default::default(), username: Default::default(), password: Default::default(), config_name: Default::default(), view_name: Default::default(), skip_deploy: false, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -12576,28 +15268,34 @@ impl Default for DnsServerBluecatV2 { impl IntoValue for DnsServerBluecatV2 { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::BaseUrl, self.base_url.into_value()); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::ConfigName, self.config_name.into_value()); - map.insert_unchecked(Property::ViewName, self.view_name.into_value()); - map.insert_unchecked(Property::SkipDeploy, self.skip_deploy.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); + map.insert_unchecked(Property::ConfigName, self.config_name.into_value_with(opts)); + map.insert_unchecked(Property::ViewName, self.view_name.into_value_with(opts)); + map.insert_unchecked(Property::SkipDeploy, self.skip_deploy.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -12610,8 +15308,8 @@ impl RegistryJsonPropertyPatch for DnsServerBluecatV2 { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::BaseUrl) => self - .base_url + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Username) => self .username @@ -12627,12 +15325,9 @@ impl RegistryJsonPropertyPatch for DnsServerBluecatV2 { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -12649,7 +15344,6 @@ impl DnsServerBootstrap { match self { DnsServerBootstrap::Manual => true, DnsServerBootstrap::Tsig(inner) => inner.validate(errors), - DnsServerBootstrap::Deprecated1 => true, DnsServerBootstrap::Cloudflare(inner) => inner.validate(errors), DnsServerBootstrap::DigitalOcean(inner) => inner.validate(errors), DnsServerBootstrap::DeSEC(inner) => inner.validate(errors), @@ -12721,6 +15415,7 @@ impl DnsServerBootstrap { DnsServerBootstrap::PowerDns(inner) => inner.validate(errors), } } + } impl Default for DnsServerBootstrap { @@ -12729,379 +15424,12 @@ impl Default for DnsServerBootstrap { } } -impl Pickle for DnsServerBootstrap { - fn pickle(&self, out: &mut Vec) { - match self { - DnsServerBootstrap::Manual => { - 0u16.pickle(out); - } - DnsServerBootstrap::Tsig(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Deprecated1 => { - 2u16.pickle(out); - } - DnsServerBootstrap::Cloudflare(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::DigitalOcean(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::DeSEC(inner) => { - 5u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Ovh(inner) => { - 6u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Bunny(inner) => { - 7u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Porkbun(inner) => { - 8u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Dnsimple(inner) => { - 9u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Spaceship(inner) => { - 10u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Route53(inner) => { - 11u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::GoogleCloudDns(inner) => { - 12u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Alidns(inner) => { - 13u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::ArvanCloud(inner) => { - 14u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Autodns(inner) => { - 15u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::AzureDns(inner) => { - 16u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::BaiduCloud(inner) => { - 17u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::BluecatV2(inner) => { - 18u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::ClouDns(inner) => { - 19u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Constellix(inner) => { - 20u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Cpanel(inner) => { - 21u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Ddnss(inner) => { - 22u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::DnsMadeEasy(inner) => { - 23u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Domeneshop(inner) => { - 24u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Dreamhost(inner) => { - 25u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::DuckDns(inner) => { - 26u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Dynu(inner) => { - 27u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::EasyDns(inner) => { - 28u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::EdgeDns(inner) => { - 29u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Exoscale(inner) => { - 30u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::FreeMyIp(inner) => { - 31u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::GandiV5(inner) => { - 32u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Gcore(inner) => { - 33u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Glesys(inner) => { - 34u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Godaddy(inner) => { - 35u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Hetzner(inner) => { - 36u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::HostingDe(inner) => { - 37u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Hostinger(inner) => { - 38u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::HuaweiCloud(inner) => { - 39u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Hurricane(inner) => { - 40u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::IbmCloud(inner) => { - 41u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Infoblox(inner) => { - 42u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Infomaniak(inner) => { - 43u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Inwx(inner) => { - 44u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Ionos(inner) => { - 45u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Ipv64(inner) => { - 46u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Joker(inner) => { - 47u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Lightsail(inner) => { - 48u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Linode(inner) => { - 49u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::LuaDns(inner) => { - 50u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::MythicBeasts(inner) => { - 51u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Namecheap(inner) => { - 52u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::NameDotCom(inner) => { - 53u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::NameSilo(inner) => { - 54u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Netcup(inner) => { - 55u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Netlify(inner) => { - 56u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Nifcloud(inner) => { - 57u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Ns1(inner) => { - 58u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::OracleCloud(inner) => { - 59u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Plesk(inner) => { - 60u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Safedns(inner) => { - 61u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Scaleway(inner) => { - 62u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::TencentCloud(inner) => { - 63u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Transip(inner) => { - 64u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::UltraDns(inner) => { - 65u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Vercel(inner) => { - 66u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Volcengine(inner) => { - 67u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::Vultr(inner) => { - 68u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::WebSupport(inner) => { - 69u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::YandexCloud(inner) => { - 70u16.pickle(out); - inner.pickle(out); - } - DnsServerBootstrap::PowerDns(inner) => { - 71u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(DnsServerBootstrap::Manual), - 1 => Pickle::unpickle(stream).map(DnsServerBootstrap::Tsig), - 2 => Some(DnsServerBootstrap::Deprecated1), - 3 => Pickle::unpickle(stream).map(DnsServerBootstrap::Cloudflare), - 4 => Pickle::unpickle(stream).map(DnsServerBootstrap::DigitalOcean), - 5 => Pickle::unpickle(stream).map(DnsServerBootstrap::DeSEC), - 6 => Pickle::unpickle(stream).map(DnsServerBootstrap::Ovh), - 7 => Pickle::unpickle(stream).map(DnsServerBootstrap::Bunny), - 8 => Pickle::unpickle(stream).map(DnsServerBootstrap::Porkbun), - 9 => Pickle::unpickle(stream).map(DnsServerBootstrap::Dnsimple), - 10 => Pickle::unpickle(stream).map(DnsServerBootstrap::Spaceship), - 11 => Pickle::unpickle(stream).map(DnsServerBootstrap::Route53), - 12 => Pickle::unpickle(stream).map(DnsServerBootstrap::GoogleCloudDns), - 13 => Pickle::unpickle(stream).map(DnsServerBootstrap::Alidns), - 14 => Pickle::unpickle(stream).map(DnsServerBootstrap::ArvanCloud), - 15 => Pickle::unpickle(stream).map(DnsServerBootstrap::Autodns), - 16 => Pickle::unpickle(stream).map(DnsServerBootstrap::AzureDns), - 17 => Pickle::unpickle(stream).map(DnsServerBootstrap::BaiduCloud), - 18 => Pickle::unpickle(stream).map(DnsServerBootstrap::BluecatV2), - 19 => Pickle::unpickle(stream).map(DnsServerBootstrap::ClouDns), - 20 => Pickle::unpickle(stream).map(DnsServerBootstrap::Constellix), - 21 => Pickle::unpickle(stream).map(DnsServerBootstrap::Cpanel), - 22 => Pickle::unpickle(stream).map(DnsServerBootstrap::Ddnss), - 23 => Pickle::unpickle(stream).map(DnsServerBootstrap::DnsMadeEasy), - 24 => Pickle::unpickle(stream).map(DnsServerBootstrap::Domeneshop), - 25 => Pickle::unpickle(stream).map(DnsServerBootstrap::Dreamhost), - 26 => Pickle::unpickle(stream).map(DnsServerBootstrap::DuckDns), - 27 => Pickle::unpickle(stream).map(DnsServerBootstrap::Dynu), - 28 => Pickle::unpickle(stream).map(DnsServerBootstrap::EasyDns), - 29 => Pickle::unpickle(stream).map(DnsServerBootstrap::EdgeDns), - 30 => Pickle::unpickle(stream).map(DnsServerBootstrap::Exoscale), - 31 => Pickle::unpickle(stream).map(DnsServerBootstrap::FreeMyIp), - 32 => Pickle::unpickle(stream).map(DnsServerBootstrap::GandiV5), - 33 => Pickle::unpickle(stream).map(DnsServerBootstrap::Gcore), - 34 => Pickle::unpickle(stream).map(DnsServerBootstrap::Glesys), - 35 => Pickle::unpickle(stream).map(DnsServerBootstrap::Godaddy), - 36 => Pickle::unpickle(stream).map(DnsServerBootstrap::Hetzner), - 37 => Pickle::unpickle(stream).map(DnsServerBootstrap::HostingDe), - 38 => Pickle::unpickle(stream).map(DnsServerBootstrap::Hostinger), - 39 => Pickle::unpickle(stream).map(DnsServerBootstrap::HuaweiCloud), - 40 => Pickle::unpickle(stream).map(DnsServerBootstrap::Hurricane), - 41 => Pickle::unpickle(stream).map(DnsServerBootstrap::IbmCloud), - 42 => Pickle::unpickle(stream).map(DnsServerBootstrap::Infoblox), - 43 => Pickle::unpickle(stream).map(DnsServerBootstrap::Infomaniak), - 44 => Pickle::unpickle(stream).map(DnsServerBootstrap::Inwx), - 45 => Pickle::unpickle(stream).map(DnsServerBootstrap::Ionos), - 46 => Pickle::unpickle(stream).map(DnsServerBootstrap::Ipv64), - 47 => Pickle::unpickle(stream).map(DnsServerBootstrap::Joker), - 48 => Pickle::unpickle(stream).map(DnsServerBootstrap::Lightsail), - 49 => Pickle::unpickle(stream).map(DnsServerBootstrap::Linode), - 50 => Pickle::unpickle(stream).map(DnsServerBootstrap::LuaDns), - 51 => Pickle::unpickle(stream).map(DnsServerBootstrap::MythicBeasts), - 52 => Pickle::unpickle(stream).map(DnsServerBootstrap::Namecheap), - 53 => Pickle::unpickle(stream).map(DnsServerBootstrap::NameDotCom), - 54 => Pickle::unpickle(stream).map(DnsServerBootstrap::NameSilo), - 55 => Pickle::unpickle(stream).map(DnsServerBootstrap::Netcup), - 56 => Pickle::unpickle(stream).map(DnsServerBootstrap::Netlify), - 57 => Pickle::unpickle(stream).map(DnsServerBootstrap::Nifcloud), - 58 => Pickle::unpickle(stream).map(DnsServerBootstrap::Ns1), - 59 => Pickle::unpickle(stream).map(DnsServerBootstrap::OracleCloud), - 60 => Pickle::unpickle(stream).map(DnsServerBootstrap::Plesk), - 61 => Pickle::unpickle(stream).map(DnsServerBootstrap::Safedns), - 62 => Pickle::unpickle(stream).map(DnsServerBootstrap::Scaleway), - 63 => Pickle::unpickle(stream).map(DnsServerBootstrap::TencentCloud), - 64 => Pickle::unpickle(stream).map(DnsServerBootstrap::Transip), - 65 => Pickle::unpickle(stream).map(DnsServerBootstrap::UltraDns), - 66 => Pickle::unpickle(stream).map(DnsServerBootstrap::Vercel), - 67 => Pickle::unpickle(stream).map(DnsServerBootstrap::Volcengine), - 68 => Pickle::unpickle(stream).map(DnsServerBootstrap::Vultr), - 69 => Pickle::unpickle(stream).map(DnsServerBootstrap::WebSupport), - 70 => Pickle::unpickle(stream).map(DnsServerBootstrap::YandexCloud), - 71 => Pickle::unpickle(stream).map(DnsServerBootstrap::PowerDns), - _ => None, - } - } -} - impl IntoValue for DnsServerBootstrap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { DnsServerBootstrap::Manual => { let mut obj = jmap_tools::Map::new(); @@ -13109,495 +15437,490 @@ impl IntoValue for DnsServerBootstrap { JmapValue::Object(obj) } DnsServerBootstrap::Tsig(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Tsig".into())); obj } - DnsServerBootstrap::Deprecated1 => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Deprecated1".into())); - JmapValue::Object(obj) - } DnsServerBootstrap::Cloudflare(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Cloudflare".into())); obj } DnsServerBootstrap::DigitalOcean(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DigitalOcean".into())); obj } DnsServerBootstrap::DeSEC(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DeSEC".into())); obj } DnsServerBootstrap::Ovh(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ovh".into())); obj } DnsServerBootstrap::Bunny(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Bunny".into())); obj } DnsServerBootstrap::Porkbun(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Porkbun".into())); obj } DnsServerBootstrap::Dnsimple(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dnsimple".into())); obj } DnsServerBootstrap::Spaceship(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Spaceship".into())); obj } DnsServerBootstrap::Route53(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Route53".into())); obj } DnsServerBootstrap::GoogleCloudDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("GoogleCloudDns".into())); obj } DnsServerBootstrap::Alidns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Alidns".into())); obj } DnsServerBootstrap::ArvanCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ArvanCloud".into())); obj } DnsServerBootstrap::Autodns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Autodns".into())); obj } DnsServerBootstrap::AzureDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("AzureDns".into())); obj } DnsServerBootstrap::BaiduCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("BaiduCloud".into())); obj } DnsServerBootstrap::BluecatV2(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("BluecatV2".into())); obj } DnsServerBootstrap::ClouDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ClouDns".into())); obj } DnsServerBootstrap::Constellix(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Constellix".into())); obj } DnsServerBootstrap::Cpanel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Cpanel".into())); obj } DnsServerBootstrap::Ddnss(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ddnss".into())); obj } DnsServerBootstrap::DnsMadeEasy(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DnsMadeEasy".into())); obj } DnsServerBootstrap::Domeneshop(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Domeneshop".into())); obj } DnsServerBootstrap::Dreamhost(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dreamhost".into())); obj } DnsServerBootstrap::DuckDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DuckDns".into())); obj } DnsServerBootstrap::Dynu(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Dynu".into())); obj } DnsServerBootstrap::EasyDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EasyDns".into())); obj } DnsServerBootstrap::EdgeDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EdgeDns".into())); obj } DnsServerBootstrap::Exoscale(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Exoscale".into())); obj } DnsServerBootstrap::FreeMyIp(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FreeMyIp".into())); obj } DnsServerBootstrap::GandiV5(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("GandiV5".into())); obj } DnsServerBootstrap::Gcore(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Gcore".into())); obj } DnsServerBootstrap::Glesys(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Glesys".into())); obj } DnsServerBootstrap::Godaddy(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Godaddy".into())); obj } DnsServerBootstrap::Hetzner(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hetzner".into())); obj } DnsServerBootstrap::HostingDe(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("HostingDe".into())); obj } DnsServerBootstrap::Hostinger(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hostinger".into())); obj } DnsServerBootstrap::HuaweiCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("HuaweiCloud".into())); obj } DnsServerBootstrap::Hurricane(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Hurricane".into())); obj } DnsServerBootstrap::IbmCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("IbmCloud".into())); obj } DnsServerBootstrap::Infoblox(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Infoblox".into())); obj } DnsServerBootstrap::Infomaniak(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Infomaniak".into())); obj } DnsServerBootstrap::Inwx(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Inwx".into())); obj } DnsServerBootstrap::Ionos(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ionos".into())); obj } DnsServerBootstrap::Ipv64(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ipv64".into())); obj } DnsServerBootstrap::Joker(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Joker".into())); obj } DnsServerBootstrap::Lightsail(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Lightsail".into())); obj } DnsServerBootstrap::Linode(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Linode".into())); obj } DnsServerBootstrap::LuaDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("LuaDns".into())); obj } DnsServerBootstrap::MythicBeasts(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MythicBeasts".into())); obj } DnsServerBootstrap::Namecheap(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Namecheap".into())); obj } DnsServerBootstrap::NameDotCom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("NameDotCom".into())); obj } DnsServerBootstrap::NameSilo(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("NameSilo".into())); obj } DnsServerBootstrap::Netcup(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Netcup".into())); obj } DnsServerBootstrap::Netlify(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Netlify".into())); obj } DnsServerBootstrap::Nifcloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Nifcloud".into())); obj } DnsServerBootstrap::Ns1(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ns1".into())); obj } DnsServerBootstrap::OracleCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("OracleCloud".into())); obj } DnsServerBootstrap::Plesk(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Plesk".into())); obj } DnsServerBootstrap::Safedns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Safedns".into())); obj } DnsServerBootstrap::Scaleway(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Scaleway".into())); obj } DnsServerBootstrap::TencentCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TencentCloud".into())); obj } DnsServerBootstrap::Transip(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Transip".into())); obj } DnsServerBootstrap::UltraDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("UltraDns".into())); obj } DnsServerBootstrap::Vercel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Vercel".into())); obj } DnsServerBootstrap::Volcengine(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Volcengine".into())); obj } DnsServerBootstrap::Vultr(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Vultr".into())); obj } DnsServerBootstrap::WebSupport(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("WebSupport".into())); obj } DnsServerBootstrap::YandexCloud(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("YandexCloud".into())); obj } DnsServerBootstrap::PowerDns(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PowerDns".into())); @@ -13619,7 +15942,6 @@ impl RegistryJsonPatch for DnsServerBootstrap { DnsServerBootstrapType::Tsig => { *self = DnsServerBootstrap::Tsig(Default::default()) } - DnsServerBootstrapType::Deprecated1 => *self = DnsServerBootstrap::Deprecated1, DnsServerBootstrapType::Cloudflare => { *self = DnsServerBootstrap::Cloudflare(Default::default()) } @@ -13828,7 +16150,6 @@ impl RegistryJsonPatch for DnsServerBootstrap { match self { DnsServerBootstrap::Manual => pointer.assert_eof(), DnsServerBootstrap::Tsig(inner) => inner.patch(pointer, value), - DnsServerBootstrap::Deprecated1 => pointer.assert_eof(), DnsServerBootstrap::Cloudflare(inner) => inner.patch(pointer, value), DnsServerBootstrap::DigitalOcean(inner) => inner.patch(pointer, value), DnsServerBootstrap::DeSEC(inner) => inner.patch(pointer, value), @@ -13907,7 +16228,6 @@ impl DnsServerBootstrap { match self { DnsServerBootstrap::Manual => DnsServerBootstrapType::Manual, DnsServerBootstrap::Tsig(_) => DnsServerBootstrapType::Tsig, - DnsServerBootstrap::Deprecated1 => DnsServerBootstrapType::Deprecated1, DnsServerBootstrap::Cloudflare(_) => DnsServerBootstrapType::Cloudflare, DnsServerBootstrap::DigitalOcean(_) => DnsServerBootstrapType::DigitalOcean, DnsServerBootstrap::DeSEC(_) => DnsServerBootstrapType::DeSEC, @@ -14000,49 +16320,116 @@ impl DnsServerClouDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerClouDns { fn pickle(&self, out: &mut Vec) { - self.auth_id.pickle(out); - self.sub_auth_id.pickle(out); + let mut bitmap = 0u64; + if self.auth_id.is_some() { + bitmap |= 1 << 0; + } + if self.sub_auth_id.is_some() { + bitmap |= 1 << 1; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 10, 4, bitmap); + if let Some(value) = &self.auth_id { + value.pickle(out); + } + if let Some(value) = &self.sub_auth_id { + value.pickle(out); + } self.password.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_id = Pickle::unpickle(stream)?; - this.sub_auth_id = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + auth_id: field0, + sub_auth_id: field1, + password: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } @@ -14053,10 +16440,9 @@ impl Default for DnsServerClouDns { sub_auth_id: Default::default(), password: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14065,25 +16451,31 @@ impl Default for DnsServerClouDns { impl IntoValue for DnsServerClouDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::AuthId, self.auth_id.into_value()); - map.insert_unchecked(Property::SubAuthId, self.sub_auth_id.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::AuthId, self.auth_id.into_value_with(opts)); + map.insert_unchecked(Property::SubAuthId, self.sub_auth_id.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14106,12 +16498,9 @@ impl RegistryJsonPropertyPatch for DnsServerClouDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14132,45 +16521,84 @@ impl DnsServerCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerCloud { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + secret: field0, + description: field1, + timeout: field3, + ttl: field4, + poll_interval: field5, + propagation_timeout: field6, + propagation_delay: field7, + }) } } @@ -14179,10 +16607,9 @@ impl Default for DnsServerCloud { Self { secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14191,23 +16618,29 @@ impl Default for DnsServerCloud { impl IntoValue for DnsServerCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14224,12 +16657,9 @@ impl RegistryJsonPropertyPatch for DnsServerCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14250,48 +16680,84 @@ impl DnsServerCloudflare { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerCloudflare { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - if stream.version() < 1 { - let _: Option = Pickle::unpickle(stream)?; - } - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + secret: field0, + description: field1, + timeout: field3, + ttl: field4, + poll_interval: field5, + propagation_timeout: field6, + propagation_delay: field7, + }) } } @@ -14300,10 +16766,9 @@ impl Default for DnsServerCloudflare { Self { secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14312,23 +16777,29 @@ impl Default for DnsServerCloudflare { impl IntoValue for DnsServerCloudflare { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14341,17 +16812,13 @@ impl RegistryJsonPropertyPatch for DnsServerCloudflare { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(property @ Property::Email) => Ok(MaybeUnpatched::Unpatched { property, value }), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14366,9 +16833,9 @@ impl RegistryJsonPropertyPatch for DnsServerCloudflare { impl DnsServerConstellix { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -14376,60 +16843,103 @@ impl DnsServerConstellix { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerConstellix { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerConstellix { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14438,24 +16948,30 @@ impl Default for DnsServerConstellix { impl IntoValue for DnsServerConstellix { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14468,19 +16984,16 @@ impl RegistryJsonPropertyPatch for DnsServerConstellix { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14495,9 +17008,9 @@ impl RegistryJsonPropertyPatch for DnsServerConstellix { impl DnsServerCpanel { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.base_url; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::BaseUrl)); + errors.push(ValidationError::required(Property::Url)); } let value = &self.username; if value.is_empty() { @@ -14509,63 +17022,111 @@ impl DnsServerCpanel { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerCpanel { fn pickle(&self, out: &mut Vec) { - self.base_url.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 10, 2, bitmap); + self.url.pickle(out); self.username.pickle(out); self.token.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.base_url = Pickle::unpickle(stream)?; - this.username = Pickle::unpickle(stream)?; - this.token = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + username: field1, + token: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } impl Default for DnsServerCpanel { fn default() -> Self { Self { - base_url: Default::default(), + url: Default::default(), username: Default::default(), token: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14574,25 +17135,31 @@ impl Default for DnsServerCpanel { impl IntoValue for DnsServerCpanel { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::BaseUrl, self.base_url.into_value()); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Token, self.token.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14605,8 +17172,8 @@ impl RegistryJsonPropertyPatch for DnsServerCpanel { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::BaseUrl) => self - .base_url + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Username) => self .username @@ -14615,12 +17182,9 @@ impl RegistryJsonPropertyPatch for DnsServerCpanel { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14635,9 +17199,9 @@ impl RegistryJsonPropertyPatch for DnsServerCpanel { impl DnsServerDnsMadeEasy { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret; value.validate(errors); @@ -14645,60 +17209,103 @@ impl DnsServerDnsMadeEasy { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerDnsMadeEasy { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerDnsMadeEasy { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14707,24 +17314,30 @@ impl Default for DnsServerDnsMadeEasy { impl IntoValue for DnsServerDnsMadeEasy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14737,19 +17350,16 @@ impl RegistryJsonPropertyPatch for DnsServerDnsMadeEasy { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14764,7 +17374,7 @@ impl RegistryJsonPropertyPatch for DnsServerDnsMadeEasy { impl DnsServerDnsimple { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.auth_token; + let value = &self.token; value.validate(errors); let value = &self.account_identifier; if value.is_empty() { @@ -14774,63 +17384,103 @@ impl DnsServerDnsimple { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerDnsimple { fn pickle(&self, out: &mut Vec) { - self.auth_token.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.token.pickle(out); self.account_identifier.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_token = Pickle::unpickle(stream)?; - this.account_identifier = Pickle::unpickle(stream)?; - if stream.version() < 1 { - let _: SecretKey = Pickle::unpickle(stream)?; - } - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + token: field0, + account_identifier: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerDnsimple { fn default() -> Self { Self { - auth_token: Default::default(), + token: Default::default(), account_identifier: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14839,27 +17489,33 @@ impl Default for DnsServerDnsimple { impl IntoValue for DnsServerDnsimple { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AuthToken, self.auth_token.into_value()); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); map.insert_unchecked( Property::AccountIdentifier, - self.account_identifier.into_value(), + self.account_identifier.into_value_with(opts), ); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -14872,20 +17528,16 @@ impl RegistryJsonPropertyPatch for DnsServerDnsimple { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AuthToken) => self.auth_token.patch(pointer, value), + Some(Property::Token) => self.token.patch(pointer, value), Some(Property::AccountIdentifier) => self .account_identifier .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(property @ Property::Secret) => Ok(MaybeUnpatched::Unpatched { property, value }), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -14900,9 +17552,9 @@ impl RegistryJsonPropertyPatch for DnsServerDnsimple { impl DnsServerDomeneshop { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.auth_token; + let value = &self.token; if value.is_empty() { - errors.push(ValidationError::required(Property::AuthToken)); + errors.push(ValidationError::required(Property::Token)); } let value = &self.secret; value.validate(errors); @@ -14910,60 +17562,103 @@ impl DnsServerDomeneshop { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerDomeneshop { fn pickle(&self, out: &mut Vec) { - self.auth_token.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.token.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_token = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + token: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerDomeneshop { fn default() -> Self { Self { - auth_token: Default::default(), + token: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -14972,24 +17667,30 @@ impl Default for DnsServerDomeneshop { impl IntoValue for DnsServerDomeneshop { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AuthToken, self.auth_token.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15002,19 +17703,16 @@ impl RegistryJsonPropertyPatch for DnsServerDomeneshop { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AuthToken) => self - .auth_token + Some(Property::Token) => self + .token .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15039,47 +17737,91 @@ impl DnsServerEasyDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerEasyDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.token.pickle(out); self.key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.token = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + token: field0, + key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -15089,10 +17831,9 @@ impl Default for DnsServerEasyDns { token: Default::default(), key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15101,24 +17842,30 @@ impl Default for DnsServerEasyDns { impl IntoValue for DnsServerEasyDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Token, self.token.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15138,12 +17885,9 @@ impl RegistryJsonPropertyPatch for DnsServerEasyDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15179,53 +17923,121 @@ impl DnsServerEdgeDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerEdgeDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.account_switch_key.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 12, 3, bitmap); self.host.pickle(out); self.client_token.pickle(out); self.client_secret.pickle(out); self.access_token.pickle(out); - self.account_switch_key.pickle(out); + if let Some(value) = &self.account_switch_key { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.host = Pickle::unpickle(stream)?; - this.client_token = Pickle::unpickle(stream)?; - this.client_secret = Pickle::unpickle(stream)?; - this.access_token = Pickle::unpickle(stream)?; - this.account_switch_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: SecretKey = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: Option = if block.has(11) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + host: field0, + client_token: field1, + client_secret: field2, + access_token: field3, + account_switch_key: field4, + description: field5, + timeout: field7, + ttl: field8, + poll_interval: field9, + propagation_timeout: field10, + propagation_delay: field11, + }) } } @@ -15238,10 +18050,9 @@ impl Default for DnsServerEdgeDns { access_token: Default::default(), account_switch_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15250,30 +18061,45 @@ impl Default for DnsServerEdgeDns { impl IntoValue for DnsServerEdgeDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::ClientToken, self.client_token.into_value()); - map.insert_unchecked(Property::ClientSecret, self.client_secret.into_value()); - map.insert_unchecked(Property::AccessToken, self.access_token.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked( + Property::ClientToken, + self.client_token.into_value_with(opts), + ); + map.insert_unchecked( + Property::ClientSecret, + self.client_secret.into_value_with(opts), + ); + map.insert_unchecked( + Property::AccessToken, + self.access_token.into_value_with(opts), + ); map.insert_unchecked( Property::AccountSwitchKey, - self.account_switch_key.into_value(), + self.account_switch_key.into_value_with(opts), ); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15300,12 +18126,9 @@ impl RegistryJsonPropertyPatch for DnsServerEdgeDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15320,9 +18143,9 @@ impl RegistryJsonPropertyPatch for DnsServerEdgeDns { impl DnsServerExoscale { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret; value.validate(errors); @@ -15330,60 +18153,103 @@ impl DnsServerExoscale { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerExoscale { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerExoscale { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15392,24 +18258,30 @@ impl Default for DnsServerExoscale { impl IntoValue for DnsServerExoscale { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15422,19 +18294,16 @@ impl RegistryJsonPropertyPatch for DnsServerExoscale { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15449,9 +18318,9 @@ impl RegistryJsonPropertyPatch for DnsServerExoscale { impl DnsServerGlesys { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_user; + let value = &self.username; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiUser)); + errors.push(ValidationError::required(Property::Username)); } let value = &self.api_key; value.validate(errors); @@ -15459,60 +18328,103 @@ impl DnsServerGlesys { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerGlesys { fn pickle(&self, out: &mut Vec) { - self.api_user.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.username.pickle(out); self.api_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_user = Pickle::unpickle(stream)?; - this.api_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + api_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerGlesys { fn default() -> Self { Self { - api_user: Default::default(), + username: Default::default(), api_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15521,24 +18433,30 @@ impl Default for DnsServerGlesys { impl IntoValue for DnsServerGlesys { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiUser, self.api_user.into_value()); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15551,19 +18469,16 @@ impl RegistryJsonPropertyPatch for DnsServerGlesys { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiUser) => self - .api_user + Some(Property::Username) => self + .username .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::ApiKey) => self.api_key.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15578,9 +18493,9 @@ impl RegistryJsonPropertyPatch for DnsServerGlesys { impl DnsServerGodaddy { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret; value.validate(errors); @@ -15588,60 +18503,103 @@ impl DnsServerGodaddy { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerGodaddy { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerGodaddy { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15650,24 +18608,30 @@ impl Default for DnsServerGodaddy { impl IntoValue for DnsServerGodaddy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15680,19 +18644,16 @@ impl RegistryJsonPropertyPatch for DnsServerGodaddy { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15729,53 +18690,128 @@ impl DnsServerGoogleCloudDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerGoogleCloudDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.managed_zone.is_some() { + bitmap |= 1 << 0; + } + if self.private_zone { + bitmap |= 1 << 1; + } + if self.impersonate_service_account.is_some() { + bitmap |= 1 << 2; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 12, 5, bitmap); self.service_account_json.pickle(out); self.project_id.pickle(out); - self.managed_zone.pickle(out); - self.private_zone.pickle(out); - self.impersonate_service_account.pickle(out); + if let Some(value) = &self.managed_zone { + value.pickle(out); + } + if let Some(value) = &self.impersonate_service_account { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.service_account_json = Pickle::unpickle(stream)?; - this.project_id = Pickle::unpickle(stream)?; - this.managed_zone = Pickle::unpickle(stream)?; - this.private_zone = Pickle::unpickle(stream)?; - this.impersonate_service_account = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretText = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { false }; + let field4: Option = if block.has(4) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: Option = if block.has(11) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + service_account_json: field0, + project_id: field1, + managed_zone: field2, + private_zone: field3, + impersonate_service_account: field4, + description: field5, + timeout: field7, + ttl: field8, + poll_interval: field9, + propagation_timeout: field10, + propagation_delay: field11, + }) } } @@ -15788,10 +18824,9 @@ impl Default for DnsServerGoogleCloudDns { private_zone: false, impersonate_service_account: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15800,33 +18835,45 @@ impl Default for DnsServerGoogleCloudDns { impl IntoValue for DnsServerGoogleCloudDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); map.insert_unchecked( Property::ServiceAccountJson, - self.service_account_json.into_value(), + self.service_account_json.into_value_with(opts), + ); + map.insert_unchecked(Property::ProjectId, self.project_id.into_value_with(opts)); + map.insert_unchecked( + Property::ManagedZone, + self.managed_zone.into_value_with(opts), + ); + map.insert_unchecked( + Property::PrivateZone, + self.private_zone.into_value_with(opts), ); - map.insert_unchecked(Property::ProjectId, self.project_id.into_value()); - map.insert_unchecked(Property::ManagedZone, self.managed_zone.into_value()); - map.insert_unchecked(Property::PrivateZone, self.private_zone.into_value()); map.insert_unchecked( Property::ImpersonateServiceAccount, - self.impersonate_service_account.into_value(), + self.impersonate_service_account.into_value_with(opts), ); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15853,12 +18900,9 @@ impl RegistryJsonPropertyPatch for DnsServerGoogleCloudDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -15873,9 +18917,9 @@ impl RegistryJsonPropertyPatch for DnsServerGoogleCloudDns { impl DnsServerHuaweiCloud { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.access_key; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::AccessKey)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -15887,63 +18931,111 @@ impl DnsServerHuaweiCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerHuaweiCloud { fn pickle(&self, out: &mut Vec) { - self.access_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 10, 2, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); self.region.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + "ap-southeast-1".to_string() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + region: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } impl Default for DnsServerHuaweiCloud { fn default() -> Self { Self { - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), region: "ap-southeast-1".to_string(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -15952,25 +19044,34 @@ impl Default for DnsServerHuaweiCloud { impl IntoValue for DnsServerHuaweiCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -15983,8 +19084,8 @@ impl RegistryJsonPropertyPatch for DnsServerHuaweiCloud { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AccessKey) => self - .access_key + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Region) => self @@ -15993,12 +19094,9 @@ impl RegistryJsonPropertyPatch for DnsServerHuaweiCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16024,45 +19122,84 @@ impl DnsServerHurricane { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerHurricane { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.credentials.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.credentials = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: List = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + credentials: field0, + description: field1, + timeout: field3, + ttl: field4, + poll_interval: field5, + propagation_timeout: field6, + propagation_delay: field7, + }) } } @@ -16071,10 +19208,9 @@ impl Default for DnsServerHurricane { Self { credentials: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16083,23 +19219,32 @@ impl Default for DnsServerHurricane { impl IntoValue for DnsServerHurricane { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Credentials, self.credentials.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Credentials, + self.credentials.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16116,12 +19261,9 @@ impl RegistryJsonPropertyPatch for DnsServerHurricane { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16146,47 +19288,91 @@ impl DnsServerIbmCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerIbmCloud { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.username.pickle(out); self.api_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.api_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + api_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -16196,10 +19382,9 @@ impl Default for DnsServerIbmCloud { username: Default::default(), api_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16208,24 +19393,30 @@ impl Default for DnsServerIbmCloud { impl IntoValue for DnsServerIbmCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16245,12 +19436,9 @@ impl RegistryJsonPropertyPatch for DnsServerIbmCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16270,8 +19458,11 @@ impl DnsServerInfoblox { errors.push(ValidationError::required(Property::Host)); } if let Some(value) = &self.port { - if value.is_empty() { - errors.push(ValidationError::required(Property::Port)); + if *value > 65535 { + errors.push(ValidationError::max_value(Property::Port, 65535)); + } + if *value < 1 { + errors.push(ValidationError::min_value(Property::Port, 1)); } } let value = &self.username; @@ -16294,55 +19485,146 @@ impl DnsServerInfoblox { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerInfoblox { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.port.is_some() { + bitmap |= 1 << 0; + } + if self.wapi_version.is_some() { + bitmap |= 1 << 1; + } + if self.dns_view.is_some() { + bitmap |= 1 << 2; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 13, 5, bitmap); self.host.pickle(out); - self.port.pickle(out); + if let Some(value) = &self.port { + value.pickle(out); + } self.username.pickle(out); self.password.pickle(out); - self.wapi_version.pickle(out); - self.dns_view.pickle(out); + if let Some(value) = &self.wapi_version { + value.pickle(out); + } + if let Some(value) = &self.dns_view { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.wapi_version = Pickle::unpickle(stream)?; - this.dns_view = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: SecretKey = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field12: Option = if block.has(12) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + host: field0, + port: field1, + username: field2, + password: field3, + wapi_version: field4, + dns_view: field5, + description: field6, + timeout: field8, + ttl: field9, + poll_interval: field10, + propagation_timeout: field11, + propagation_delay: field12, + }) } } @@ -16356,10 +19638,9 @@ impl Default for DnsServerInfoblox { wapi_version: Default::default(), dns_view: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16368,28 +19649,37 @@ impl Default for DnsServerInfoblox { impl IntoValue for DnsServerInfoblox { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::WapiVersion, self.wapi_version.into_value()); - map.insert_unchecked(Property::DnsView, self.dns_view.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::WapiVersion, + self.wapi_version.into_value_with(opts), + ); + map.insert_unchecked(Property::DnsView, self.dns_view.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16405,9 +19695,7 @@ impl RegistryJsonPropertyPatch for DnsServerInfoblox { Some(Property::Host) => self .host .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Port) => self - .port - .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Port) => self.port.patch(pointer, value), Some(Property::Username) => self .username .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -16421,12 +19709,9 @@ impl RegistryJsonPropertyPatch for DnsServerInfoblox { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16453,51 +19738,103 @@ impl DnsServerInwx { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerInwx { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.sandbox { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 11, 3, bitmap); self.username.pickle(out); self.password.pickle(out); self.shared_secret.pickle(out); - self.sandbox.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.shared_secret = Pickle::unpickle(stream)?; - this.sandbox = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKeyOptional = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + password: field1, + shared_secret: field2, + sandbox: field3, + description: field4, + timeout: field6, + ttl: field7, + poll_interval: field8, + propagation_timeout: field9, + propagation_delay: field10, + }) } } @@ -16509,10 +19846,9 @@ impl Default for DnsServerInwx { shared_secret: Default::default(), sandbox: false, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16521,26 +19857,35 @@ impl Default for DnsServerInwx { impl IntoValue for DnsServerInwx { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::SharedSecret, self.shared_secret.into_value()); - map.insert_unchecked(Property::Sandbox, self.sandbox.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::SharedSecret, + self.shared_secret.into_value_with(opts), + ); + map.insert_unchecked(Property::Sandbox, self.sandbox.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16562,12 +19907,9 @@ impl RegistryJsonPropertyPatch for DnsServerInwx { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16588,45 +19930,84 @@ impl DnsServerJoker { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerJoker { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.auth.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: JokerAuth = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + auth: field0, + description: field1, + timeout: field3, + ttl: field4, + poll_interval: field5, + propagation_timeout: field6, + propagation_delay: field7, + }) } } @@ -16635,10 +20016,9 @@ impl Default for DnsServerJoker { Self { auth: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16647,23 +20027,29 @@ impl Default for DnsServerJoker { impl IntoValue for DnsServerJoker { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Auth, self.auth.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Auth, self.auth.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16680,12 +20066,9 @@ impl RegistryJsonPropertyPatch for DnsServerJoker { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16704,7 +20087,7 @@ impl DnsServerLightsail { if value.is_empty() { errors.push(ValidationError::required(Property::AccessKeyId)); } - let value = &self.secret_access_key; + let value = &self.secret_key; value.validate(errors); let value = &self.session_token; value.validate(errors); @@ -16722,53 +20105,130 @@ impl DnsServerLightsail { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerLightsail { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.region.is_some() { + bitmap |= 1 << 0; + } + if self.domain.is_some() { + bitmap |= 1 << 1; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 12, 4, bitmap); self.access_key_id.pickle(out); - self.secret_access_key.pickle(out); + self.secret_key.pickle(out); self.session_token.pickle(out); - self.region.pickle(out); - self.domain.pickle(out); + if let Some(value) = &self.region { + value.pickle(out); + } + if let Some(value) = &self.domain { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key_id = Pickle::unpickle(stream)?; - this.secret_access_key = Pickle::unpickle(stream)?; - this.session_token = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.domain = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKeyOptional = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: Option = if block.has(11) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + session_token: field2, + region: field3, + domain: field4, + description: field5, + timeout: field7, + ttl: field8, + poll_interval: field9, + propagation_timeout: field10, + propagation_delay: field11, + }) } } @@ -16776,15 +20236,14 @@ impl Default for DnsServerLightsail { fn default() -> Self { Self { access_key_id: Default::default(), - secret_access_key: Default::default(), + secret_key: Default::default(), session_token: Default::default(), region: Default::default(), domain: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16793,30 +20252,39 @@ impl Default for DnsServerLightsail { impl IntoValue for DnsServerLightsail { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::AccessKeyId, self.access_key_id.into_value()); map.insert_unchecked( - Property::SecretAccessKey, - self.secret_access_key.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), ); - map.insert_unchecked(Property::SessionToken, self.session_token.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::Domain, self.domain.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::SessionToken, + self.session_token.into_value_with(opts), + ); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked(Property::Domain, self.domain.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16832,7 +20300,7 @@ impl RegistryJsonPropertyPatch for DnsServerLightsail { Some(Property::AccessKeyId) => self .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::SecretAccessKey) => self.secret_access_key.patch(pointer, value), + Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::SessionToken) => self.session_token.patch(pointer, value), Some(Property::Region) => self .region @@ -16843,12 +20311,9 @@ impl RegistryJsonPropertyPatch for DnsServerLightsail { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -16867,53 +20332,97 @@ impl DnsServerLuaDns { if value.is_empty() { errors.push(ValidationError::required(Property::Username)); } - let value = &self.auth_token; + let value = &self.token; value.validate(errors); let value = &self.description; if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerLuaDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.username.pickle(out); - self.auth_token.pickle(out); + self.token.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.auth_token = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + token: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -16921,12 +20430,11 @@ impl Default for DnsServerLuaDns { fn default() -> Self { Self { username: Default::default(), - auth_token: Default::default(), + token: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -16935,24 +20443,30 @@ impl Default for DnsServerLuaDns { impl IntoValue for DnsServerLuaDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::AuthToken, self.auth_token.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -16968,16 +20482,13 @@ impl RegistryJsonPropertyPatch for DnsServerLuaDns { Some(Property::Username) => self .username .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthToken) => self.auth_token.patch(pointer, value), + Some(Property::Token) => self.token.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17002,47 +20513,91 @@ impl DnsServerMythicBeasts { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerMythicBeasts { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.username.pickle(out); self.password.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + password: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -17052,10 +20607,9 @@ impl Default for DnsServerMythicBeasts { username: Default::default(), password: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17064,24 +20618,30 @@ impl Default for DnsServerMythicBeasts { impl IntoValue for DnsServerMythicBeasts { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17101,12 +20661,9 @@ impl RegistryJsonPropertyPatch for DnsServerMythicBeasts { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17125,53 +20682,97 @@ impl DnsServerNameDotCom { if value.is_empty() { errors.push(ValidationError::required(Property::Username)); } - let value = &self.auth_token; + let value = &self.token; value.validate(errors); let value = &self.description; if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerNameDotCom { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.username.pickle(out); - self.auth_token.pickle(out); + self.token.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.auth_token = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + token: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -17179,12 +20780,11 @@ impl Default for DnsServerNameDotCom { fn default() -> Self { Self { username: Default::default(), - auth_token: Default::default(), + token: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17193,24 +20793,30 @@ impl Default for DnsServerNameDotCom { impl IntoValue for DnsServerNameDotCom { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::AuthToken, self.auth_token.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17226,16 +20832,13 @@ impl RegistryJsonPropertyPatch for DnsServerNameDotCom { Some(Property::Username) => self .username .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthToken) => self.auth_token.patch(pointer, value), + Some(Property::Token) => self.token.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17269,51 +20872,114 @@ impl DnsServerNamecheap { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerNamecheap { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.username.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 11, 3, bitmap); self.api_user.pickle(out); self.api_key.pickle(out); self.client_ip.pickle(out); - self.username.pickle(out); + if let Some(value) = &self.username { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_user = Pickle::unpickle(stream)?; - this.api_key = Pickle::unpickle(stream)?; - this.client_ip = Pickle::unpickle(stream)?; - this.username = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_user: field0, + api_key: field1, + client_ip: field2, + username: field3, + description: field4, + timeout: field6, + ttl: field7, + poll_interval: field8, + propagation_timeout: field9, + propagation_delay: field10, + }) } } @@ -17325,10 +20991,9 @@ impl Default for DnsServerNamecheap { client_ip: Default::default(), username: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17337,26 +21002,32 @@ impl Default for DnsServerNamecheap { impl IntoValue for DnsServerNamecheap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::ApiUser, self.api_user.into_value()); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::ClientIp, self.client_ip.into_value()); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiUser, self.api_user.into_value_with(opts)); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); + map.insert_unchecked(Property::ClientIp, self.client_ip.into_value_with(opts)); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17382,12 +21053,9 @@ impl RegistryJsonPropertyPatch for DnsServerNamecheap { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17406,9 +21074,9 @@ impl DnsServerNetcup { if value.is_empty() { errors.push(ValidationError::required(Property::CustomerNumber)); } - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.password; value.validate(errors); @@ -17416,49 +21084,98 @@ impl DnsServerNetcup { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerNetcup { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 10, 2, bitmap); self.customer_number.pickle(out); - self.api_key.pickle(out); + self.api_key_id.pickle(out); self.password.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.customer_number = Pickle::unpickle(stream)?; - this.api_key = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + customer_number: field0, + api_key_id: field1, + password: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } @@ -17466,13 +21183,12 @@ impl Default for DnsServerNetcup { fn default() -> Self { Self { customer_number: Default::default(), - api_key: Default::default(), + api_key_id: Default::default(), password: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17481,25 +21197,34 @@ impl Default for DnsServerNetcup { impl IntoValue for DnsServerNetcup { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::CustomerNumber, self.customer_number.into_value()); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::CustomerNumber, + self.customer_number.into_value_with(opts), + ); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17515,19 +21240,16 @@ impl RegistryJsonPropertyPatch for DnsServerNetcup { Some(Property::CustomerNumber) => self .customer_number .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Password) => self.password.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17542,9 +21264,9 @@ impl RegistryJsonPropertyPatch for DnsServerNetcup { impl DnsServerNifcloud { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.access_key; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::AccessKey)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -17552,60 +21274,103 @@ impl DnsServerNifcloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerNifcloud { fn pickle(&self, out: &mut Vec) { - self.access_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerNifcloud { fn default() -> Self { Self { - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17614,24 +21379,33 @@ impl Default for DnsServerNifcloud { impl IntoValue for DnsServerNifcloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17644,19 +21418,16 @@ impl RegistryJsonPropertyPatch for DnsServerNifcloud { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AccessKey) => self - .access_key + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17699,24 +21470,21 @@ impl DnsServerOracleCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerOracleCloud { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 14, 2, bitmap); self.tenancy_ocid.pickle(out); self.user_ocid.pickle(out); self.fingerprint.pickle(out); @@ -17725,31 +21493,103 @@ impl Pickle for DnsServerOracleCloud { self.region.pickle(out); self.compartment_ocid.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tenancy_ocid = Pickle::unpickle(stream)?; - this.user_ocid = Pickle::unpickle(stream)?; - this.fingerprint = Pickle::unpickle(stream)?; - this.private_key_pem = Pickle::unpickle(stream)?; - this.private_key_password = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.compartment_ocid = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: SecretText = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: SecretKeyOptional = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: String = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field12: Duration = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field13: Option = if block.has(13) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + tenancy_ocid: field0, + user_ocid: field1, + fingerprint: field2, + private_key_pem: field3, + private_key_password: field4, + region: field5, + compartment_ocid: field6, + description: field7, + timeout: field9, + ttl: field10, + poll_interval: field11, + propagation_timeout: field12, + propagation_delay: field13, + }) } } @@ -17764,10 +21604,9 @@ impl Default for DnsServerOracleCloud { region: Default::default(), compartment_ocid: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17776,35 +21615,50 @@ impl Default for DnsServerOracleCloud { impl IntoValue for DnsServerOracleCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(16); - map.insert_unchecked(Property::TenancyOcid, self.tenancy_ocid.into_value()); - map.insert_unchecked(Property::UserOcid, self.user_ocid.into_value()); - map.insert_unchecked(Property::Fingerprint, self.fingerprint.into_value()); - map.insert_unchecked(Property::PrivateKeyPem, self.private_key_pem.into_value()); + map.insert_unchecked( + Property::TenancyOcid, + self.tenancy_ocid.into_value_with(opts), + ); + map.insert_unchecked(Property::UserOcid, self.user_ocid.into_value_with(opts)); + map.insert_unchecked( + Property::Fingerprint, + self.fingerprint.into_value_with(opts), + ); + map.insert_unchecked( + Property::PrivateKeyPem, + self.private_key_pem.into_value_with(opts), + ); map.insert_unchecked( Property::PrivateKeyPassword, - self.private_key_password.into_value(), + self.private_key_password.into_value_with(opts), ); - map.insert_unchecked(Property::Region, self.region.into_value()); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); map.insert_unchecked( Property::CompartmentOcid, - self.compartment_ocid.into_value(), + self.compartment_ocid.into_value_with(opts), ); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17837,12 +21691,9 @@ impl RegistryJsonPropertyPatch for DnsServerOracleCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -17869,51 +21720,105 @@ impl DnsServerOvh { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerOvh { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 11, 2, bitmap); self.application_key.pickle(out); self.application_secret.pickle(out); self.consumer_key.pickle(out); self.ovh_endpoint.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.application_key = Pickle::unpickle(stream)?; - this.application_secret = Pickle::unpickle(stream)?; - this.consumer_key = Pickle::unpickle(stream)?; - this.ovh_endpoint = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKey = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: OvhEndpoint = if block.has(3) { + Pickle::unpickle(stream)? + } else { + OvhEndpoint::OvhEu + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field10: Option = if block.has(10) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + application_key: field0, + application_secret: field1, + consumer_key: field2, + ovh_endpoint: field3, + description: field4, + timeout: field6, + ttl: field7, + poll_interval: field8, + propagation_timeout: field9, + propagation_delay: field10, + }) } } @@ -17925,10 +21830,9 @@ impl Default for DnsServerOvh { consumer_key: Default::default(), ovh_endpoint: OvhEndpoint::OvhEu, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -17937,29 +21841,44 @@ impl Default for DnsServerOvh { impl IntoValue for DnsServerOvh { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::ApplicationKey, self.application_key.into_value()); + map.insert_unchecked( + Property::ApplicationKey, + self.application_key.into_value_with(opts), + ); map.insert_unchecked( Property::ApplicationSecret, - self.application_secret.into_value(), + self.application_secret.into_value_with(opts), ); - map.insert_unchecked(Property::ConsumerKey, self.consumer_key.into_value()); - map.insert_unchecked(Property::OvhEndpoint, self.ovh_endpoint.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::ConsumerKey, + self.consumer_key.into_value_with(opts), + ); + map.insert_unchecked( + Property::OvhEndpoint, + self.ovh_endpoint.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -17981,12 +21900,9 @@ impl RegistryJsonPropertyPatch for DnsServerOvh { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18001,9 +21917,9 @@ impl RegistryJsonPropertyPatch for DnsServerOvh { impl DnsServerPlesk { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.base_url; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::BaseUrl)); + errors.push(ValidationError::required(Property::Url)); } let value = &self.api_key; value.validate(errors); @@ -18011,60 +21927,103 @@ impl DnsServerPlesk { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerPlesk { fn pickle(&self, out: &mut Vec) { - self.base_url.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.url.pickle(out); self.api_key.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.base_url = Pickle::unpickle(stream)?; - this.api_key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + api_key: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerPlesk { fn default() -> Self { Self { - base_url: Default::default(), + url: Default::default(), api_key: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18073,24 +22032,30 @@ impl Default for DnsServerPlesk { impl IntoValue for DnsServerPlesk { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::BaseUrl, self.base_url.into_value()); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18103,19 +22068,16 @@ impl RegistryJsonPropertyPatch for DnsServerPlesk { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::BaseUrl) => self - .base_url + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::ApiKey) => self.api_key.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18130,73 +22092,113 @@ impl RegistryJsonPropertyPatch for DnsServerPlesk { impl DnsServerPorkbun { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } - let value = &self.secret_api_key; + let value = &self.secret; value.validate(errors); let value = &self.description; if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerPorkbun { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); - self.secret_api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); + self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret_api_key = Pickle::unpickle(stream)?; - if stream.version() < 1 { - let _: SecretKey = Pickle::unpickle(stream)?; - } - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerPorkbun { fn default() -> Self { Self { - api_key: Default::default(), - secret_api_key: Default::default(), + api_key_id: Default::default(), + secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18205,24 +22207,30 @@ impl Default for DnsServerPorkbun { impl IntoValue for DnsServerPorkbun { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::SecretApiKey, self.secret_api_key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18235,20 +22243,16 @@ impl RegistryJsonPropertyPatch for DnsServerPorkbun { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::SecretApiKey) => self.secret_api_key.patch(pointer, value), - Some(property @ Property::Secret) => Ok(MaybeUnpatched::Unpatched { property, value }), + Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18265,9 +22269,9 @@ impl DnsServerPowerDns { let neb = errors.len(); let value = &self.api_key; value.validate(errors); - if let Some(value) = &self.endpoint { + if let Some(value) = &self.url { if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } } if let Some(value) = &self.server_id { @@ -18279,49 +22283,116 @@ impl DnsServerPowerDns { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerPowerDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.url.is_some() { + bitmap |= 1 << 0; + } + if self.server_id.is_some() { + bitmap |= 1 << 1; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 10, 4, bitmap); self.api_key.pickle(out); - self.endpoint.pickle(out); - self.server_id.pickle(out); + if let Some(value) = &self.url { + value.pickle(out); + } + if let Some(value) = &self.server_id { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.endpoint = Pickle::unpickle(stream)?; - this.server_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key: field0, + url: field1, + server_id: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } @@ -18329,13 +22400,12 @@ impl Default for DnsServerPowerDns { fn default() -> Self { Self { api_key: Default::default(), - endpoint: Default::default(), + url: Default::default(), server_id: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18344,25 +22414,31 @@ impl Default for DnsServerPowerDns { impl IntoValue for DnsServerPowerDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); - map.insert_unchecked(Property::ServerId, self.server_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::ServerId, self.server_id.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18376,8 +22452,8 @@ impl RegistryJsonPropertyPatch for DnsServerPowerDns { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::ApiKey) => self.api_key.patch(pointer, value), - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::ServerId) => self .server_id @@ -18385,12 +22461,9 @@ impl RegistryJsonPropertyPatch for DnsServerPowerDns { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18409,7 +22482,7 @@ impl DnsServerRoute53 { if value.is_empty() { errors.push(ValidationError::required(Property::AccessKeyId)); } - let value = &self.secret_access_key; + let value = &self.secret_key; value.validate(errors); let value = &self.session_token; value.validate(errors); @@ -18426,55 +22499,126 @@ impl DnsServerRoute53 { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerRoute53 { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.hosted_zone_id.is_some() { + bitmap |= 1 << 0; + } + if self.private_zone_only { + bitmap |= 1 << 1; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 13, 4, bitmap); self.access_key_id.pickle(out); - self.secret_access_key.pickle(out); + self.secret_key.pickle(out); self.session_token.pickle(out); self.region.pickle(out); - self.hosted_zone_id.pickle(out); - self.private_zone_only.pickle(out); + if let Some(value) = &self.hosted_zone_id { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key_id = Pickle::unpickle(stream)?; - this.secret_access_key = Pickle::unpickle(stream)?; - this.session_token = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.hosted_zone_id = Pickle::unpickle(stream)?; - this.private_zone_only = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: SecretKeyOptional = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + "us-east-1".to_string() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field12: Option = if block.has(12) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + session_token: field2, + region: field3, + hosted_zone_id: field4, + private_zone_only: field5, + description: field6, + timeout: field8, + ttl: field9, + poll_interval: field10, + propagation_timeout: field11, + propagation_delay: field12, + }) } } @@ -18482,16 +22626,15 @@ impl Default for DnsServerRoute53 { fn default() -> Self { Self { access_key_id: Default::default(), - secret_access_key: Default::default(), + secret_key: Default::default(), session_token: Default::default(), region: "us-east-1".to_string(), hosted_zone_id: Default::default(), private_zone_only: false, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18500,34 +22643,46 @@ impl Default for DnsServerRoute53 { impl IntoValue for DnsServerRoute53 { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::AccessKeyId, self.access_key_id.into_value()); map.insert_unchecked( - Property::SecretAccessKey, - self.secret_access_key.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked( + Property::SessionToken, + self.session_token.into_value_with(opts), + ); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked( + Property::HostedZoneId, + self.hosted_zone_id.into_value_with(opts), ); - map.insert_unchecked(Property::SessionToken, self.session_token.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::HostedZoneId, self.hosted_zone_id.into_value()); map.insert_unchecked( Property::PrivateZoneOnly, - self.private_zone_only.into_value(), + self.private_zone_only.into_value_with(opts), ); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18543,7 +22698,7 @@ impl RegistryJsonPropertyPatch for DnsServerRoute53 { Some(Property::AccessKeyId) => self .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::SecretAccessKey) => self.secret_access_key.patch(pointer, value), + Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::SessionToken) => self.session_token.patch(pointer, value), Some(Property::Region) => self .region @@ -18555,12 +22710,9 @@ impl RegistryJsonPropertyPatch for DnsServerRoute53 { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18575,9 +22727,9 @@ impl RegistryJsonPropertyPatch for DnsServerRoute53 { impl DnsServerSpaceship { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret; value.validate(errors); @@ -18585,60 +22737,103 @@ impl DnsServerSpaceship { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerSpaceship { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerSpaceship { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18647,24 +22842,30 @@ impl Default for DnsServerSpaceship { impl IntoValue for DnsServerSpaceship { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18677,19 +22878,16 @@ impl RegistryJsonPropertyPatch for DnsServerSpaceship { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18704,9 +22902,9 @@ impl RegistryJsonPropertyPatch for DnsServerSpaceship { impl DnsServerTencentCloud { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.secret_id; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::SecretId)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -18721,66 +22919,128 @@ impl DnsServerTencentCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerTencentCloud { fn pickle(&self, out: &mut Vec) { - self.secret_id.pickle(out); + let mut bitmap = 0u64; + if self.region.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 11, 3, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); - self.region.pickle(out); + if let Some(value) = &self.region { + value.pickle(out); + } self.session_token.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.secret_id = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.session_token = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: SecretKeyOptional = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + region: field2, + session_token: field3, + description: field4, + timeout: field6, + ttl: field7, + poll_interval: field8, + propagation_timeout: field9, + propagation_delay: field10, + }) } } impl Default for DnsServerTencentCloud { fn default() -> Self { Self { - secret_id: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), region: Default::default(), session_token: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18789,26 +23049,38 @@ impl Default for DnsServerTencentCloud { impl IntoValue for DnsServerTencentCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::SecretId, self.secret_id.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::SessionToken, self.session_token.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked( + Property::SessionToken, + self.session_token.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18821,8 +23093,8 @@ impl RegistryJsonPropertyPatch for DnsServerTencentCloud { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::SecretId) => self - .secret_id + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Region) => self @@ -18832,12 +23104,9 @@ impl RegistryJsonPropertyPatch for DnsServerTencentCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -18862,47 +23131,91 @@ impl DnsServerTransip { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerTransip { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.username.pickle(out); self.private_key_pem.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.private_key_pem = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretText = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + private_key_pem: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -18912,10 +23225,9 @@ impl Default for DnsServerTransip { username: Default::default(), private_key_pem: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -18924,24 +23236,33 @@ impl Default for DnsServerTransip { impl IntoValue for DnsServerTransip { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::PrivateKeyPem, self.private_key_pem.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::PrivateKeyPem, + self.private_key_pem.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -18961,12 +23282,9 @@ impl RegistryJsonPropertyPatch for DnsServerTransip { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19002,24 +23320,21 @@ impl DnsServerTsig { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerTsig { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 13, 2, bitmap); self.host.pickle(out); self.port.pickle(out); self.key_name.pickle(out); @@ -19027,30 +23342,97 @@ impl Pickle for DnsServerTsig { self.protocol.pickle(out); self.tsig_algorithm.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.key_name = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.protocol = Pickle::unpickle(stream)?; - this.tsig_algorithm = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: IpAddr = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 53u64 + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: SecretKey = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: IpProtocol = if block.has(4) { + Pickle::unpickle(stream)? + } else { + IpProtocol::Udp + }; + let field5: TsigAlgorithm = if block.has(5) { + Pickle::unpickle(stream)? + } else { + TsigAlgorithm::HmacSha512 + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field11: Duration = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field12: Option = if block.has(12) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + host: field0, + port: field1, + key_name: field2, + key: field3, + protocol: field4, + tsig_algorithm: field5, + description: field6, + timeout: field8, + ttl: field9, + poll_interval: field10, + propagation_timeout: field11, + propagation_delay: field12, + }) } } @@ -19064,10 +23446,9 @@ impl Default for DnsServerTsig { protocol: IpProtocol::Udp, tsig_algorithm: TsigAlgorithm::HmacSha512, description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19076,28 +23457,37 @@ impl Default for DnsServerTsig { impl IntoValue for DnsServerTsig { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::KeyName, self.key_name.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Protocol, self.protocol.into_value()); - map.insert_unchecked(Property::TsigAlgorithm, self.tsig_algorithm.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::KeyName, self.key_name.into_value_with(opts)); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked(Property::Protocol, self.protocol.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::TsigAlgorithm, + self.tsig_algorithm.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19123,12 +23513,9 @@ impl RegistryJsonPropertyPatch for DnsServerTsig { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19149,58 +23536,116 @@ impl DnsServerUltraDns { } let value = &self.password; value.validate(errors); - if let Some(value) = &self.endpoint { + if let Some(value) = &self.url { if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } } let value = &self.description; if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerUltraDns { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.url.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 10, 3, bitmap); self.username.pickle(out); self.password.pickle(out); - self.endpoint.pickle(out); + if let Some(value) = &self.url { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - this.endpoint = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field9: Option = if block.has(9) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + password: field1, + url: field2, + description: field3, + timeout: field5, + ttl: field6, + poll_interval: field7, + propagation_timeout: field8, + propagation_delay: field9, + }) } } @@ -19209,12 +23654,11 @@ impl Default for DnsServerUltraDns { Self { username: Default::default(), password: Default::default(), - endpoint: Default::default(), + url: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19223,25 +23667,31 @@ impl Default for DnsServerUltraDns { impl IntoValue for DnsServerUltraDns { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19258,18 +23708,15 @@ impl RegistryJsonPropertyPatch for DnsServerUltraDns { .username .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Password) => self.password.patch(pointer, value), - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19284,7 +23731,7 @@ impl RegistryJsonPropertyPatch for DnsServerUltraDns { impl DnsServerVercel { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.auth_token; + let value = &self.token; value.validate(errors); if let Some(value) = &self.team_id { if value.is_empty() { @@ -19295,60 +23742,112 @@ impl DnsServerVercel { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerVercel { fn pickle(&self, out: &mut Vec) { - self.auth_token.pickle(out); - self.team_id.pickle(out); + let mut bitmap = 0u64; + if self.team_id.is_some() { + bitmap |= 1 << 0; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 9, 3, bitmap); + self.token.pickle(out); + if let Some(value) = &self.team_id { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_token = Pickle::unpickle(stream)?; - this.team_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + token: field0, + team_id: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerVercel { fn default() -> Self { Self { - auth_token: Default::default(), + token: Default::default(), team_id: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19357,24 +23856,30 @@ impl Default for DnsServerVercel { impl IntoValue for DnsServerVercel { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AuthToken, self.auth_token.into_value()); - map.insert_unchecked(Property::TeamId, self.team_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::Token, self.token.into_value_with(opts)); + map.insert_unchecked(Property::TeamId, self.team_id.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19387,19 +23892,16 @@ impl RegistryJsonPropertyPatch for DnsServerVercel { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AuthToken) => self.auth_token.patch(pointer, value), + Some(Property::Token) => self.token.patch(pointer, value), Some(Property::TeamId) => self .team_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19414,9 +23916,9 @@ impl RegistryJsonPropertyPatch for DnsServerVercel { impl DnsServerVolcengine { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.access_key; + let value = &self.access_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::AccessKey)); + errors.push(ValidationError::required(Property::AccessKeyId)); } let value = &self.secret_key; value.validate(errors); @@ -19439,69 +23941,154 @@ impl DnsServerVolcengine { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerVolcengine { fn pickle(&self, out: &mut Vec) { - self.access_key.pickle(out); + let mut bitmap = 0u64; + if self.region.is_some() { + bitmap |= 1 << 0; + } + if self.host.is_some() { + bitmap |= 1 << 1; + } + if self.scheme.is_some() { + bitmap |= 1 << 2; + } + if self.propagation_delay.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 12, 5, bitmap); + self.access_key_id.pickle(out); self.secret_key.pickle(out); - self.region.pickle(out); - self.host.pickle(out); - self.scheme.pickle(out); + if let Some(value) = &self.region { + value.pickle(out); + } + if let Some(value) = &self.host { + value.pickle(out); + } + if let Some(value) = &self.scheme { + value.pickle(out); + } self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.region = Pickle::unpickle(stream)?; - this.host = Pickle::unpickle(stream)?; - this.scheme = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: Option = if block.has(11) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + access_key_id: field0, + secret_key: field1, + region: field2, + host: field3, + scheme: field4, + description: field5, + timeout: field7, + ttl: field8, + poll_interval: field9, + propagation_timeout: field10, + propagation_delay: field11, + }) } } impl Default for DnsServerVolcengine { fn default() -> Self { Self { - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), region: Default::default(), host: Default::default(), scheme: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19510,27 +24097,36 @@ impl Default for DnsServerVolcengine { impl IntoValue for DnsServerVolcengine { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Scheme, self.scheme.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Scheme, self.scheme.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19543,8 +24139,8 @@ impl RegistryJsonPropertyPatch for DnsServerVolcengine { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AccessKey) => self - .access_key + Some(Property::AccessKeyId) => self + .access_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), Some(Property::Region) => self @@ -19559,12 +24155,9 @@ impl RegistryJsonPropertyPatch for DnsServerVolcengine { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19579,9 +24172,9 @@ impl RegistryJsonPropertyPatch for DnsServerVolcengine { impl DnsServerWebSupport { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.api_key; + let value = &self.api_key_id; if value.is_empty() { - errors.push(ValidationError::required(Property::ApiKey)); + errors.push(ValidationError::required(Property::ApiKeyId)); } let value = &self.secret; value.validate(errors); @@ -19589,60 +24182,103 @@ impl DnsServerWebSupport { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerWebSupport { fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); + self.api_key_id.pickle(out); self.secret.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key_id: field0, + secret: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } impl Default for DnsServerWebSupport { fn default() -> Self { Self { - api_key: Default::default(), + api_key_id: Default::default(), secret: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19651,24 +24287,30 @@ impl Default for DnsServerWebSupport { impl IntoValue for DnsServerWebSupport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKeyId, self.api_key_id.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19681,19 +24323,16 @@ impl RegistryJsonPropertyPatch for DnsServerWebSupport { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::ApiKey) => self - .api_key + Some(Property::ApiKeyId) => self + .api_key_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19718,47 +24357,91 @@ impl DnsServerYandexCloud { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for DnsServerYandexCloud { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.propagation_delay.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.api_key.pickle(out); self.folder_id.pickle(out); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.timeout.pickle(out); self.ttl.pickle(out); - self.polling_interval.pickle(out); + self.poll_interval.pickle(out); self.propagation_timeout.pickle(out); - self.propagation_delay.pickle(out); + if let Some(value) = &self.propagation_delay { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.folder_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.ttl = Pickle::unpickle(stream)?; - this.polling_interval = Pickle::unpickle(stream)?; - this.propagation_timeout = Pickle::unpickle(stream)?; - this.propagation_delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretText = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + api_key: field0, + folder_id: field1, + description: field2, + timeout: field4, + ttl: field5, + poll_interval: field6, + propagation_timeout: field7, + propagation_delay: field8, + }) } } @@ -19768,10 +24451,9 @@ impl Default for DnsServerYandexCloud { api_key: Default::default(), folder_id: Default::default(), description: Default::default(), - member_tenant_id: Default::default(), timeout: Duration::from_millis(30000), ttl: Duration::from_millis(300000), - polling_interval: Duration::from_millis(15000), + poll_interval: Duration::from_millis(15000), propagation_timeout: Duration::from_millis(60000), propagation_delay: Default::default(), } @@ -19780,24 +24462,30 @@ impl Default for DnsServerYandexCloud { impl IntoValue for DnsServerYandexCloud { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::FolderId, self.folder_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Ttl, self.ttl.into_value()); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); + map.insert_unchecked(Property::FolderId, self.folder_id.into_value_with(opts)); map.insert_unchecked( - Property::PollingInterval, - self.polling_interval.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::Ttl, self.ttl.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), ); map.insert_unchecked( Property::PropagationTimeout, - self.propagation_timeout.into_value(), + self.propagation_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PropagationDelay, - self.propagation_delay.into_value(), + self.propagation_delay.into_value_with(opts), ); JmapValue::Object(map) } @@ -19817,12 +24505,9 @@ impl RegistryJsonPropertyPatch for DnsServerYandexCloud { Some(Property::Description) => self .description .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Ttl) => self.ttl.patch(pointer, value), - Some(Property::PollingInterval) => self.polling_interval.patch(pointer, value), + Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), Some(Property::PropagationTimeout) => self.propagation_timeout.patch(pointer, value), Some(Property::PropagationDelay) => self.propagation_delay.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -19836,7 +24521,7 @@ impl RegistryJsonPropertyPatch for DnsServerYandexCloud { impl ObjectImpl for Domain { const FLAGS: u64 = OBJ_FILTER_TENANT | OBJ_SEQ_ID; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Domain; fn validate(&self, errors: &mut Vec) -> bool { @@ -19860,27 +24545,12 @@ impl ObjectImpl for Domain { errors.push(ValidationError::required(Property::Description)); } } - if let Some(value) = &self.logo { - if value.is_empty() { - errors.push(ValidationError::required(Property::Logo)); - } - } let value = &self.certificate_management; value.validate(errors); let value = &self.dkim_management; value.validate(errors); let value = &self.dns_management; value.validate(errors); - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } - if let Some(value) = &self.directory_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::DirectoryId)); - } - } if let Some(value) = &self.catch_all_address { if value.is_empty() { errors.push(ValidationError::required(Property::CatchAllAddress)); @@ -19910,55 +24580,131 @@ impl ObjectImpl for Domain { } self.certificate_management.index(i); self.dns_management.index(i); - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } - i.foreign_key(ObjectType::Directory, self.directory_id, None); } } impl Pickle for Domain { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.description.is_some() { + bitmap |= 1 << 1; + } + if self.catch_all_address.is_some() { + bitmap |= 1 << 5; + } + if self.allow_relaying { + bitmap |= 1 << 6; + } + if self.report_address_uri.is_some() { + bitmap |= 1 << 7; + } + let block = crate::pickle::begin_block(out, 16, 9, bitmap); self.name.pickle(out); self.aliases.pickle(out); - self.is_enabled.pickle(out); self.created_at.pickle(out); - self.description.pickle(out); - self.logo.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.certificate_management.pickle(out); self.dkim_management.pickle(out); self.dns_management.pickle(out); - self.member_tenant_id.pickle(out); - self.directory_id.pickle(out); - self.catch_all_address.pickle(out); + if let Some(value) = &self.catch_all_address { + value.pickle(out); + } self.sub_addressing.pickle(out); - self.allow_relaying.pickle(out); - self.report_address_uri.pickle(out); - self.allow_scim_provisioning.pickle(out); + if let Some(value) = &self.report_address_uri { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.aliases = Pickle::unpickle(stream)?; - this.is_enabled = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.logo = Pickle::unpickle(stream)?; - this.certificate_management = Pickle::unpickle(stream)?; - this.dkim_management = Pickle::unpickle(stream)?; - this.dns_management = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.directory_id = Pickle::unpickle(stream)?; - this.catch_all_address = Pickle::unpickle(stream)?; - this.sub_addressing = Pickle::unpickle(stream)?; - this.allow_relaying = Pickle::unpickle(stream)?; - this.report_address_uri = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.allow_scim_provisioning = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(0) } else { true }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: CertificateManagement = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: DkimManagement = if block.has(7) { + Pickle::unpickle(stream)? + } else { + DkimManagement::Automatic(Default::default()) + }; + let field8: DnsManagement = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: Option = if block.has(11) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field12: SubAddressing = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field13: bool = if block.has(13) { block.flag(6) } else { false }; + let field14: Option = if block.has(14) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("mailto:postmaster".to_string()) + }; + stream.close_block(block); + Some(Self { + name: field0, + aliases: field1, + enable: field2, + created_at: field3, + description: field4, + certificate_management: field6, + dkim_management: field7, + dns_management: field8, + catch_all_address: field11, + sub_addressing: field12, + allow_relaying: field13, + report_address_uri: field14, + }) } } @@ -19967,54 +24713,62 @@ impl Default for Domain { Self { name: Default::default(), aliases: Default::default(), - is_enabled: true, + enable: true, created_at: Default::default(), description: Default::default(), - logo: Default::default(), certificate_management: Default::default(), - dkim_management: Default::default(), + dkim_management: DkimManagement::Automatic(Default::default()), dns_management: Default::default(), - member_tenant_id: Default::default(), - directory_id: Default::default(), catch_all_address: Default::default(), sub_addressing: Default::default(), allow_relaying: false, report_address_uri: Some("mailto:postmaster".to_string()), - allow_scim_provisioning: false, } } } impl IntoValue for Domain { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(18); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Aliases, self.aliases.into_value()); - map.insert_unchecked(Property::IsEnabled, self.is_enabled.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Logo, self.logo.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::Aliases, self.aliases.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); map.insert_unchecked( Property::CertificateManagement, - self.certificate_management.into_value(), + self.certificate_management.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimManagement, + self.dkim_management.into_value_with(opts), + ); + map.insert_unchecked( + Property::DnsManagement, + self.dns_management.into_value_with(opts), ); - map.insert_unchecked(Property::DkimManagement, self.dkim_management.into_value()); - map.insert_unchecked(Property::DnsManagement, self.dns_management.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::DirectoryId, self.directory_id.into_value()); map.insert_unchecked( Property::CatchAllAddress, - self.catch_all_address.into_value(), + self.catch_all_address.into_value_with(opts), + ); + map.insert_unchecked( + Property::SubAddressing, + self.sub_addressing.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowRelaying, + self.allow_relaying.into_value_with(opts), ); - map.insert_unchecked(Property::SubAddressing, self.sub_addressing.into_value()); - map.insert_unchecked(Property::AllowRelaying, self.allow_relaying.into_value()); map.insert_unchecked( Property::ReportAddressUri, - self.report_address_uri.into_value(), - ); - map.insert_unchecked( - Property::AllowScimProvisioning, - self.allow_scim_provisioning.into_value(), + self.report_address_uri.into_value_with(opts), ); JmapValue::Object(map) } @@ -20033,20 +24787,15 @@ impl RegistryJsonPropertyPatch for Domain { Some(Property::Aliases) => self .aliases .patch(pointer.with_validators(&[StringValidator::Domain]), value), - Some(Property::IsEnabled) => self.is_enabled.patch(pointer, value), + Some(Property::Enable) => self.enable.patch(pointer, value), Some(Property::CreatedAt) => pointer.assert_server_set(), Some(Property::Description) => self.description.patch(pointer, value), - Some(Property::Logo) => self.logo.patch(pointer, value), Some(Property::CertificateManagement) => { self.certificate_management.patch(pointer, value) } Some(Property::DkimManagement) => self.dkim_management.patch(pointer, value), Some(Property::DnsManagement) => self.dns_management.patch(pointer, value), Some(Property::DnsZoneFile) => pointer.assert_server_set(), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), - Some(Property::DirectoryId) => self.directory_id.patch(pointer, value), Some(Property::CatchAllAddress) => self .catch_all_address .patch(pointer.with_validators(&[StringValidator::Email]), value), @@ -20055,9 +24804,6 @@ impl RegistryJsonPropertyPatch for Domain { Some(Property::ReportAddressUri) => self .report_address_uri .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AllowScimProvisioning) => { - self.allow_scim_provisioning.patch(pointer, value) - } Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -20091,7 +24837,7 @@ impl DsnReportSettings { ExpressionContext { expr: &self.from_address, default: Some(Expression { - else_: "'MAILER-DAEMON@' + system('domain')".to_string(), + else_: "'MAILER-DAEMON@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::FromAddress, @@ -20117,7 +24863,7 @@ impl DsnReportSettings { ExpressionContext { expr: &self.dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::DkimSignDomain, @@ -20137,17 +24883,48 @@ impl DsnReportSettings { impl Pickle for DsnReportSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.from_address.pickle(out); self.from_name.pickle(out); self.dkim_sign_domain.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from_address = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.dkim_sign_domain = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'MAILER-DAEMON@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Mail Delivery Subsystem'".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + from_address: field0, + from_name: field1, + dkim_sign_domain: field2, + }) } } @@ -20155,7 +24932,7 @@ impl Default for DsnReportSettings { fn default() -> Self { Self { from_address: Expression { - else_: "'MAILER-DAEMON@' + system('domain')".to_string(), + else_: "'MAILER-DAEMON@' + system('report_domain')".to_string(), ..Default::default() }, from_name: Expression { @@ -20163,7 +24940,7 @@ impl Default for DsnReportSettings { ..Default::default() }, dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, } @@ -20172,10 +24949,20 @@ impl Default for DsnReportSettings { impl IntoValue for DsnReportSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::FromAddress, self.from_address.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); + map.insert_unchecked( + Property::FromAddress, + self.from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FromName, self.from_name.into_value_with(opts)); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -20227,31 +25014,73 @@ impl ElasticSearchStore { } errors.len() == neb } + } impl Pickle for ElasticSearchStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.include_source { + bitmap |= 1 << 0; + } + if self.allow_invalid_certs { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.url.pickle(out); self.num_replicas.pickle(out); self.num_shards.pickle(out); - self.include_source.pickle(out); self.timeout.pickle(out); - self.allow_invalid_certs.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.url = Pickle::unpickle(stream)?; - this.num_replicas = Pickle::unpickle(stream)?; - this.num_shards = Pickle::unpickle(stream)?; - this.include_source = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + let field6: HttpAuth = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + num_replicas: field1, + num_shards: field2, + include_source: field3, + timeout: field4, + allow_invalid_certs: field5, + http_auth: field6, + http_headers: field7, + }) } } @@ -20272,18 +25101,31 @@ impl Default for ElasticSearchStore { impl IntoValue for ElasticSearchStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::NumReplicas, self.num_replicas.into_value()); - map.insert_unchecked(Property::NumShards, self.num_shards.into_value()); - map.insert_unchecked(Property::IncludeSource, self.include_source.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked( + Property::NumReplicas, + self.num_replicas.into_value_with(opts), + ); + map.insert_unchecked(Property::NumShards, self.num_shards.into_value_with(opts)); + map.insert_unchecked( + Property::IncludeSource, + self.include_source.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), ); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); JmapValue::Object(map) } } @@ -20366,16 +25208,37 @@ impl ObjectImpl for Email { errors.push(ValidationError::min_value(Property::MaxMailboxes, 1)); } } - if let Some(value) = &self.max_masked_addresses { - if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxMaskedAddresses, 1)); - } - } if let Some(value) = &self.max_public_keys { if *value < 1 { errors.push(ValidationError::min_value(Property::MaxPublicKeys, 1)); } } + let value = &self.max_mailboxes_per_email; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxMailboxesPerEmail, + 1, + )); + } + let value = &self.max_flags_per_email; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxFlagsPerEmail, 1)); + } + let value = &self.max_flag_length; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxFlagLength, 1)); + } + if *value > 1024 { + errors.push(ValidationError::max_value(Property::MaxFlagLength, 1024)); + } + let value = &self.max_header_size; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxHeaderSize, 1024)); + } + let value = &self.max_header_count; + if *value < 100 { + errors.push(ValidationError::min_value(Property::MaxHeaderCount, 100)); + } errors.len() == neb } @@ -20384,39 +25247,186 @@ impl ObjectImpl for Email { impl Pickle for Email { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.encrypt_on_append { + bitmap |= 1 << 0; + } + if self.encrypt_at_rest { + bitmap |= 1 << 1; + } + if self.max_messages.is_some() { + bitmap |= 1 << 2; + } + if self.max_submissions.is_some() { + bitmap |= 1 << 3; + } + if self.max_identities.is_some() { + bitmap |= 1 << 4; + } + if self.max_mailboxes.is_some() { + bitmap |= 1 << 5; + } + if self.max_public_keys.is_some() { + bitmap |= 1 << 7; + } + let block = crate::pickle::begin_block(out, 19, 8, bitmap); self.max_attachment_size.pickle(out); self.max_message_size.pickle(out); self.max_mailbox_depth.pickle(out); self.max_mailbox_name_length.pickle(out); - self.encrypt_on_append.pickle(out); - self.encrypt_at_rest.pickle(out); self.compression_algorithm.pickle(out); self.default_folders.pickle(out); - self.max_messages.pickle(out); - self.max_submissions.pickle(out); - self.max_identities.pickle(out); - self.max_mailboxes.pickle(out); - self.max_masked_addresses.pickle(out); - self.max_public_keys.pickle(out); + if let Some(value) = &self.max_messages { + value.pickle(out); + } + if let Some(value) = &self.max_submissions { + value.pickle(out); + } + if let Some(value) = &self.max_identities { + value.pickle(out); + } + if let Some(value) = &self.max_mailboxes { + value.pickle(out); + } + if let Some(value) = &self.max_public_keys { + value.pickle(out); + } + self.max_mailboxes_per_email.pickle(out); + self.max_flags_per_email.pickle(out); + self.max_flag_length.pickle(out); + self.max_header_size.pickle(out); + self.max_header_count.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_attachment_size = Pickle::unpickle(stream)?; - this.max_message_size = Pickle::unpickle(stream)?; - this.max_mailbox_depth = Pickle::unpickle(stream)?; - this.max_mailbox_name_length = Pickle::unpickle(stream)?; - this.encrypt_on_append = Pickle::unpickle(stream)?; - this.encrypt_at_rest = Pickle::unpickle(stream)?; - this.compression_algorithm = Pickle::unpickle(stream)?; - this.default_folders = Pickle::unpickle(stream)?; - this.max_messages = Pickle::unpickle(stream)?; - this.max_submissions = Pickle::unpickle(stream)?; - this.max_identities = Pickle::unpickle(stream)?; - this.max_mailboxes = Pickle::unpickle(stream)?; - this.max_masked_addresses = Pickle::unpickle(stream)?; - this.max_public_keys = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 50000000u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 75000000u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 255u64 + }; + let field4: bool = if block.has(4) { block.flag(0) } else { false }; + let field5: bool = if block.has(5) { block.flag(1) } else { true }; + let field6: CompressionAlgo = if block.has(6) { + Pickle::unpickle(stream)? + } else { + CompressionAlgo::Lz4 + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Option = if block.has(8) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field9: Option = if block.has(9) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(500u64) + }; + let field10: Option = if block.has(10) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(20u64) + }; + let field11: Option = if block.has(11) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(250u64) + }; + let field13: Option = if block.has(13) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(5u64) + }; + let field14: u64 = if block.has(14) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field15: u64 = if block.has(15) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field16: u64 = if block.has(16) { + Pickle::unpickle(stream)? + } else { + 128u64 + }; + let field17: u64 = if block.has(17) { + Pickle::unpickle(stream)? + } else { + 524288u64 + }; + let field18: u64 = if block.has(18) { + Pickle::unpickle(stream)? + } else { + 5000u64 + }; + stream.close_block(block); + Some(Self { + max_attachment_size: field0, + max_message_size: field1, + max_mailbox_depth: field2, + max_mailbox_name_length: field3, + encrypt_on_append: field4, + encrypt_at_rest: field5, + compression_algorithm: field6, + default_folders: field7, + max_messages: field8, + max_submissions: field9, + max_identities: field10, + max_mailboxes: field11, + max_public_keys: field13, + max_mailboxes_per_email: field14, + max_flags_per_email: field15, + max_flag_length: field16, + max_header_size: field17, + max_header_count: field18, + }) } } @@ -20435,47 +25445,95 @@ impl Default for Email { max_submissions: Some(500u64), max_identities: Some(20u64), max_mailboxes: Some(250u64), - max_masked_addresses: Some(5u64), max_public_keys: Some(5u64), + max_mailboxes_per_email: 100u64, + max_flags_per_email: 100u64, + max_flag_length: 128u64, + max_header_size: 524288u64, + max_header_count: 5000u64, } } } impl IntoValue for Email { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(16); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(21); map.insert_unchecked( Property::MaxAttachmentSize, - self.max_attachment_size.into_value(), + self.max_attachment_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMessageSize, + self.max_message_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxMessageSize, self.max_message_size.into_value()); map.insert_unchecked( Property::MaxMailboxDepth, - self.max_mailbox_depth.into_value(), + self.max_mailbox_depth.into_value_with(opts), ); map.insert_unchecked( Property::MaxMailboxNameLength, - self.max_mailbox_name_length.into_value(), + self.max_mailbox_name_length.into_value_with(opts), ); map.insert_unchecked( Property::EncryptOnAppend, - self.encrypt_on_append.into_value(), + self.encrypt_on_append.into_value_with(opts), + ); + map.insert_unchecked( + Property::EncryptAtRest, + self.encrypt_at_rest.into_value_with(opts), ); - map.insert_unchecked(Property::EncryptAtRest, self.encrypt_at_rest.into_value()); map.insert_unchecked( Property::CompressionAlgorithm, - self.compression_algorithm.into_value(), + self.compression_algorithm.into_value_with(opts), ); - map.insert_unchecked(Property::DefaultFolders, self.default_folders.into_value()); - map.insert_unchecked(Property::MaxMessages, self.max_messages.into_value()); - map.insert_unchecked(Property::MaxSubmissions, self.max_submissions.into_value()); - map.insert_unchecked(Property::MaxIdentities, self.max_identities.into_value()); - map.insert_unchecked(Property::MaxMailboxes, self.max_mailboxes.into_value()); map.insert_unchecked( - Property::MaxMaskedAddresses, - self.max_masked_addresses.into_value(), + Property::DefaultFolders, + self.default_folders.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMessages, + self.max_messages.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxSubmissions, + self.max_submissions.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxIdentities, + self.max_identities.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMailboxes, + self.max_mailboxes.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxPublicKeys, + self.max_public_keys.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMailboxesPerEmail, + self.max_mailboxes_per_email.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxFlagsPerEmail, + self.max_flags_per_email.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxFlagLength, + self.max_flag_length.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxHeaderSize, + self.max_header_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxHeaderCount, + self.max_header_count.into_value_with(opts), ); - map.insert_unchecked(Property::MaxPublicKeys, self.max_public_keys.into_value()); JmapValue::Object(map) } } @@ -20503,8 +25561,14 @@ impl RegistryJsonPropertyPatch for Email { Some(Property::MaxSubmissions) => self.max_submissions.patch(pointer, value), Some(Property::MaxIdentities) => self.max_identities.patch(pointer, value), Some(Property::MaxMailboxes) => self.max_mailboxes.patch(pointer, value), - Some(Property::MaxMaskedAddresses) => self.max_masked_addresses.patch(pointer, value), Some(Property::MaxPublicKeys) => self.max_public_keys.patch(pointer, value), + Some(Property::MaxMailboxesPerEmail) => { + self.max_mailboxes_per_email.patch(pointer, value) + } + Some(Property::MaxFlagsPerEmail) => self.max_flags_per_email.patch(pointer, value), + Some(Property::MaxFlagLength) => self.max_flag_length.patch(pointer, value), + Some(Property::MaxHeaderSize) => self.max_header_size.patch(pointer, value), + Some(Property::MaxHeaderCount) => self.max_header_count.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -20534,7 +25598,11 @@ impl EmailAlias { } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.unique_global_composite(Property::Email, &self.name, &self.domain_id); + i.unique_global_composite( + Property::Email, + &self.name, + IndexValue::U32(self.domain_id.document_id()), + ); i.text(Property::Text, &self.name); i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); } @@ -20542,26 +25610,58 @@ impl EmailAlias { impl Pickle for EmailAlias { fn pickle(&self, out: &mut Vec) { - self.enabled.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.description.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.name.pickle(out); self.domain_id.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enabled = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Id = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + name: field1, + domain_id: field2, + description: field3, + }) } } impl Default for EmailAlias { fn default() -> Self { Self { - enabled: true, + enable: true, name: Default::default(), domain_id: Default::default(), description: Default::default(), @@ -20571,11 +25671,18 @@ impl Default for EmailAlias { impl IntoValue for EmailAlias { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Enabled, self.enabled.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -20587,7 +25694,7 @@ impl RegistryJsonPropertyPatch for EmailAlias { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Enabled) => self.enabled.patch(pointer, value), + Some(Property::Enable) => self.enable.patch(pointer, value), Some(Property::Name) => self.name.patch( pointer.with_validators(&[StringValidator::EmailLocalPart]), value, @@ -20622,19 +25729,40 @@ impl EmailFolder { impl Pickle for EmailFolder { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.create { + bitmap |= 1 << 0; + } + if self.subscribe { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.name.pickle(out); - self.create.pickle(out); - self.subscribe.pickle(out); self.aliases.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.create = Pickle::unpickle(stream)?; - this.subscribe = Pickle::unpickle(stream)?; - this.aliases = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: bool = if block.has(2) { block.flag(1) } else { true }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + create: field1, + subscribe: field2, + aliases: field3, + }) } } @@ -20651,11 +25779,15 @@ impl Default for EmailFolder { impl IntoValue for EmailFolder { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Create, self.create.into_value()); - map.insert_unchecked(Property::Subscribe, self.subscribe.into_value()); - map.insert_unchecked(Property::Aliases, self.aliases.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::Create, self.create.into_value_with(opts)); + map.insert_unchecked(Property::Subscribe, self.subscribe.into_value_with(opts)); + map.insert_unchecked(Property::Aliases, self.aliases.into_value_with(opts)); JmapValue::Object(map) } } @@ -20759,6 +25891,10 @@ impl Pickle for EncryptionAtRest { impl IntoValue for EncryptionAtRest { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { EncryptionAtRest::Disabled => { let mut obj = jmap_tools::Map::new(); @@ -20766,28 +25902,28 @@ impl IntoValue for EncryptionAtRest { JmapValue::Object(obj) } EncryptionAtRest::Aes128(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Aes128".into())); obj } EncryptionAtRest::Aes256(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Aes256".into())); obj } EncryptionAtRest::Aes256Gcm(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Aes256Gcm".into())); obj } EncryptionAtRest::ChaCha20Poly1305(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ChaCha20Poly1305".into())); @@ -20859,17 +25995,33 @@ impl EncryptionSettings { impl Pickle for EncryptionSettings { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.encrypt_on_append { + bitmap |= 1 << 0; + } + if self.allow_spam_training { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 3, 2, bitmap); self.public_key.pickle(out); - self.encrypt_on_append.pickle(out); - self.allow_spam_training.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.public_key = Pickle::unpickle(stream)?; - this.encrypt_on_append = Pickle::unpickle(stream)?; - this.allow_spam_training = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + stream.close_block(block); + Some(Self { + public_key: field0, + encrypt_on_append: field1, + allow_spam_training: field2, + }) } } @@ -20885,15 +26037,19 @@ impl Default for EncryptionSettings { impl IntoValue for EncryptionSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::PublicKey, self.public_key.into_value()); + map.insert_unchecked(Property::PublicKey, self.public_key.into_value_with(opts)); map.insert_unchecked( Property::EncryptOnAppend, - self.encrypt_on_append.into_value(), + self.encrypt_on_append.into_value_with(opts), ); map.insert_unchecked( Property::AllowSpamTraining, - self.allow_spam_training.into_value(), + self.allow_spam_training.into_value_with(opts), ); JmapValue::Object(map) } @@ -20918,85 +26074,6 @@ impl RegistryJsonPropertyPatch for EncryptionSettings { } } -impl ObjectImpl for Enterprise { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::Enterprise; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.api_key; - value.validate(errors); - let value = &self.license_key; - value.validate(errors); - if let Some(value) = &self.logo_url { - if value.is_empty() { - errors.push(ValidationError::required(Property::LogoUrl)); - } - } - errors.len() == neb - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Pickle for Enterprise { - fn pickle(&self, out: &mut Vec) { - self.api_key.pickle(out); - self.license_key.pickle(out); - self.logo_url.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - this.license_key = Pickle::unpickle(stream)?; - this.logo_url = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for Enterprise { - fn default() -> Self { - Self { - api_key: Default::default(), - license_key: Default::default(), - logo_url: Default::default(), - } - } -} - -impl IntoValue for Enterprise { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); - map.insert_unchecked(Property::LicenseKey, self.license_key.into_value()); - map.insert_unchecked(Property::LogoUrl, self.logo_url.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for Enterprise { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::ApiKey) => self.api_key.patch(pointer, value), - Some(Property::LicenseKey) => self.license_key.patch(pointer, value), - Some(Property::LogoUrl) => self - .logo_url - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for EventTracingLevel { const FLAGS: u64 = 0; const VERSION: u8 = 0; @@ -21013,15 +26090,32 @@ impl ObjectImpl for EventTracingLevel { impl Pickle for EventTracingLevel { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.event.pickle(out); self.level.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.event = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: trc::EventType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TracingLevelOpt = if block.has(1) { + Pickle::unpickle(stream)? + } else { + TracingLevelOpt::Info + }; + stream.close_block(block); + Some(Self { + event: field0, + level: field1, + }) } } @@ -21036,9 +26130,13 @@ impl Default for EventTracingLevel { impl IntoValue for EventTracingLevel { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Event, self.event.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); + map.insert_unchecked(Property::Event, self.event.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); JmapValue::Object(map) } } @@ -21078,15 +26176,29 @@ impl Expression { impl Pickle for Expression { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.match_.pickle(out); self.else_.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.match_ = Pickle::unpickle(stream)?; - this.else_ = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: List = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + match_: field0, + else_: field1, + }) } } @@ -21101,9 +26213,13 @@ impl Default for Expression { impl IntoValue for Expression { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Match, self.match_.into_value()); - map.insert_unchecked(Property::Else, self.else_.into_value()); + map.insert_unchecked(Property::Match, self.match_.into_value_with(opts)); + map.insert_unchecked(Property::Else, self.else_.into_value_with(opts)); JmapValue::Object(map) } } @@ -21143,15 +26259,29 @@ impl ExpressionMatch { impl Pickle for ExpressionMatch { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.if_.pickle(out); self.then.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.if_ = Pickle::unpickle(stream)?; - this.then = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + if_: field0, + then: field1, + }) } } @@ -21166,9 +26296,13 @@ impl Default for ExpressionMatch { impl IntoValue for ExpressionMatch { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::If, self.if_.into_value()); - map.insert_unchecked(Property::Then, self.then.into_value()); + map.insert_unchecked(Property::If, self.if_.into_value_with(opts)); + map.insert_unchecked(Property::Then, self.then.into_value_with(opts)); JmapValue::Object(map) } } @@ -21216,17 +26350,58 @@ impl ObjectImpl for FileStorage { impl Pickle for FileStorage { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.max_files.is_some() { + bitmap |= 1 << 0; + } + if self.max_folders.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 3, 2, bitmap); self.max_size.pickle(out); - self.max_files.pickle(out); - self.max_folders.pickle(out); + if let Some(value) = &self.max_files { + value.pickle(out); + } + if let Some(value) = &self.max_folders { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_size = Pickle::unpickle(stream)?; - this.max_files = Pickle::unpickle(stream)?; - this.max_folders = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 26214400 + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + max_size: field0, + max_files: field1, + max_folders: field2, + }) } } @@ -21242,10 +26417,14 @@ impl Default for FileStorage { impl IntoValue for FileStorage { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::MaxSize, self.max_size.into_value()); - map.insert_unchecked(Property::MaxFiles, self.max_files.into_value()); - map.insert_unchecked(Property::MaxFolders, self.max_folders.into_value()); + map.insert_unchecked(Property::MaxSize, self.max_size.into_value_with(opts)); + map.insert_unchecked(Property::MaxFiles, self.max_files.into_value_with(opts)); + map.insert_unchecked(Property::MaxFolders, self.max_folders.into_value_with(opts)); JmapValue::Object(map) } } @@ -21286,15 +26465,29 @@ impl FileSystemStore { impl Pickle for FileSystemStore { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.path.pickle(out); self.depth.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.path = Pickle::unpickle(stream)?; - this.depth = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + stream.close_block(block); + Some(Self { + path: field0, + depth: field1, + }) } } @@ -21309,9 +26502,13 @@ impl Default for FileSystemStore { impl IntoValue for FileSystemStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Path, self.path.into_value()); - map.insert_unchecked(Property::Depth, self.depth.into_value()); + map.insert_unchecked(Property::Path, self.path.into_value_with(opts)); + map.insert_unchecked(Property::Depth, self.depth.into_value_with(opts)); JmapValue::Object(map) } } @@ -21374,23 +26571,112 @@ impl FoundationDbStore { impl Pickle for FoundationDbStore { fn pickle(&self, out: &mut Vec) { - self.cluster_file.pickle(out); - self.datacenter_id.pickle(out); - self.machine_id.pickle(out); - self.transaction_retry_delay.pickle(out); - self.transaction_retry_limit.pickle(out); - self.transaction_timeout.pickle(out); + let mut bitmap = 0u64; + if self.cluster_file.is_some() { + bitmap |= 1 << 0; + } + if self.datacenter_id.is_some() { + bitmap |= 1 << 1; + } + if self.machine_id.is_some() { + bitmap |= 1 << 2; + } + if self.transaction_retry_delay.is_some() { + bitmap |= 1 << 3; + } + if self.transaction_retry_limit.is_some() { + bitmap |= 1 << 4; + } + if self.transaction_timeout.is_some() { + bitmap |= 1 << 5; + } + let block = crate::pickle::begin_block(out, 6, 6, bitmap); + if let Some(value) = &self.cluster_file { + value.pickle(out); + } + if let Some(value) = &self.datacenter_id { + value.pickle(out); + } + if let Some(value) = &self.machine_id { + value.pickle(out); + } + if let Some(value) = &self.transaction_retry_delay { + value.pickle(out); + } + if let Some(value) = &self.transaction_retry_limit { + value.pickle(out); + } + if let Some(value) = &self.transaction_timeout { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.cluster_file = Pickle::unpickle(stream)?; - this.datacenter_id = Pickle::unpickle(stream)?; - this.machine_id = Pickle::unpickle(stream)?; - this.transaction_retry_delay = Pickle::unpickle(stream)?; - this.transaction_retry_limit = Pickle::unpickle(stream)?; - this.transaction_timeout = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + cluster_file: field0, + datacenter_id: field1, + machine_id: field2, + transaction_retry_delay: field3, + transaction_retry_limit: field4, + transaction_timeout: field5, + }) } } @@ -21409,21 +26695,31 @@ impl Default for FoundationDbStore { impl IntoValue for FoundationDbStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::ClusterFile, self.cluster_file.into_value()); - map.insert_unchecked(Property::DatacenterId, self.datacenter_id.into_value()); - map.insert_unchecked(Property::MachineId, self.machine_id.into_value()); + map.insert_unchecked( + Property::ClusterFile, + self.cluster_file.into_value_with(opts), + ); + map.insert_unchecked( + Property::DatacenterId, + self.datacenter_id.into_value_with(opts), + ); + map.insert_unchecked(Property::MachineId, self.machine_id.into_value_with(opts)); map.insert_unchecked( Property::TransactionRetryDelay, - self.transaction_retry_delay.into_value(), + self.transaction_retry_delay.into_value_with(opts), ); map.insert_unchecked( Property::TransactionRetryLimit, - self.transaction_retry_limit.into_value(), + self.transaction_retry_limit.into_value_with(opts), ); map.insert_unchecked( Property::TransactionTimeout, - self.transaction_timeout.into_value(), + self.transaction_timeout.into_value_with(opts), ); JmapValue::Object(map) } @@ -21486,21 +26782,50 @@ impl FtrlParameters { impl Pickle for FtrlParameters { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.alpha.pickle(out); self.beta.pickle(out); self.num_features.pickle(out); self.l1_ratio.pickle(out); self.l2_ratio.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.alpha = Pickle::unpickle(stream)?; - this.beta = Pickle::unpickle(stream)?; - this.num_features = Pickle::unpickle(stream)?; - this.l1_ratio = Pickle::unpickle(stream)?; - this.l2_ratio = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Float = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Float::new(2.0f64) + }; + let field1: Float = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Float::new(1.0f64) + }; + let field2: ModelSize = if block.has(2) { + Pickle::unpickle(stream)? + } else { + ModelSize::V20 + }; + let field3: Float = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Float::new(0.001f64) + }; + let field4: Float = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Float::new(0.0001f64) + }; + stream.close_block(block); + Some(Self { + alpha: field0, + beta: field1, + num_features: field2, + l1_ratio: field3, + l2_ratio: field4, + }) } } @@ -21518,12 +26843,19 @@ impl Default for FtrlParameters { impl IntoValue for FtrlParameters { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Alpha, self.alpha.into_value()); - map.insert_unchecked(Property::Beta, self.beta.into_value()); - map.insert_unchecked(Property::NumFeatures, self.num_features.into_value()); - map.insert_unchecked(Property::L1Ratio, self.l1_ratio.into_value()); - map.insert_unchecked(Property::L2Ratio, self.l2_ratio.into_value()); + map.insert_unchecked(Property::Alpha, self.alpha.into_value_with(opts)); + map.insert_unchecked(Property::Beta, self.beta.into_value_with(opts)); + map.insert_unchecked( + Property::NumFeatures, + self.num_features.into_value_with(opts), + ); + map.insert_unchecked(Property::L1Ratio, self.l1_ratio.into_value_with(opts)); + map.insert_unchecked(Property::L2Ratio, self.l2_ratio.into_value_with(opts)); JmapValue::Object(map) } } @@ -21569,11 +26901,6 @@ impl GroupAccount { if !value.is_valid() { errors.push(ValidationError::invalid(Property::CreatedAt, value)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.roles; value.validate(errors); let value = &self.permissions; @@ -21582,16 +26909,15 @@ impl GroupAccount { for value in value.values() { value.validate(errors); } - if let Some(value) = &self.external_id { - if value.is_empty() { - errors.push(ValidationError::required(Property::ExternalId)); - } - } errors.len() == neb } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.unique_global_composite(Property::Email, &self.name, &self.domain_id); + i.unique_global_composite( + Property::Email, + &self.name, + IndexValue::U32(self.domain_id.document_id()), + ); i.text(Property::Text, &self.name); i.search(Property::Name, &self.name); i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); @@ -21599,53 +26925,113 @@ impl GroupAccount { if let Some(value) = &self.description { i.text(Property::Text, value); } - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } self.roles.index(i); for item in self.aliases.values() { item.index(i); } - if let Some(value) = &self.external_id { - i.search(Property::ExternalId, value); - } } } impl Pickle for GroupAccount { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.time_zone.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 12, 4, bitmap); self.name.pickle(out); self.domain_id.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.created_at.pickle(out); - self.member_tenant_id.pickle(out); self.roles.pickle(out); self.quotas.pickle(out); self.permissions.pickle(out); self.aliases.pickle(out); self.locale.pickle(out); - self.time_zone.pickle(out); - self.external_id.pickle(out); + if let Some(value) = &self.time_zone { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.roles = Pickle::unpickle(stream)?; - this.quotas = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.aliases = Pickle::unpickle(stream)?; - this.locale = Pickle::unpickle(stream)?; - this.time_zone = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.external_id = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Roles = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: VecMap = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Permissions = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: List = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Locale = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Locale::EnUS + }; + let field10: Option = if block.has(10) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + domain_id: field1, + description: field2, + created_at: field3, + roles: field5, + quotas: field6, + permissions: field7, + aliases: field8, + locale: field9, + time_zone: field10, + }) } } @@ -21656,33 +27042,39 @@ impl Default for GroupAccount { domain_id: Default::default(), description: Default::default(), created_at: Default::default(), - member_tenant_id: Default::default(), roles: Default::default(), quotas: Default::default(), permissions: Default::default(), aliases: Default::default(), locale: Locale::EnUS, time_zone: Default::default(), - external_id: Default::default(), } } } impl IntoValue for GroupAccount { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Roles, self.roles.into_value()); - map.insert_unchecked(Property::Quotas, self.quotas.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::Aliases, self.aliases.into_value()); - map.insert_unchecked(Property::Locale, self.locale.into_value()); - map.insert_unchecked(Property::TimeZone, self.time_zone.into_value()); - map.insert_unchecked(Property::ExternalId, self.external_id.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::Roles, self.roles.into_value_with(opts)); + map.insert_unchecked(Property::Quotas, self.quotas.into_value_with(opts)); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); + map.insert_unchecked(Property::Aliases, self.aliases.into_value_with(opts)); + map.insert_unchecked(Property::Locale, self.locale.into_value_with(opts)); + map.insert_unchecked(Property::TimeZone, self.time_zone.into_value_with(opts)); JmapValue::Object(map) } } @@ -21702,9 +27094,6 @@ impl RegistryJsonPropertyPatch for GroupAccount { Some(Property::EmailAddress) => pointer.assert_server_set(), Some(Property::Description) => self.description.patch(pointer, value), Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Roles) => self.roles.patch(pointer, value), Some(Property::Quotas) => self.quotas.patch(pointer, value), Some(Property::UsedDiskQuota) => pointer.assert_server_set(), @@ -21712,7 +27101,6 @@ impl RegistryJsonPropertyPatch for GroupAccount { Some(Property::Aliases) => self.aliases.patch(pointer, value), Some(Property::Locale) => self.locale.patch(pointer, value), Some(Property::TimeZone) => self.time_zone.patch(pointer, value), - Some(Property::ExternalId) => self.external_id.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -21724,7 +27112,7 @@ impl RegistryJsonPropertyPatch for GroupAccount { impl ObjectImpl for Http { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Http; fn validate(&self, errors: &mut Vec) -> bool { @@ -21743,6 +27131,13 @@ impl ObjectImpl for Http { errors.push(ValidationError::required(Property::ResponseHeaders)); } } + let value = &self.forwarded_hops; + if *value < 1 { + errors.push(ValidationError::min_value(Property::ForwardedHops, 1)); + } + if *value > 16 { + errors.push(ValidationError::max_value(Property::ForwardedHops, 16)); + } if let Some(value) = &self.redirect_root { if value.is_empty() { errors.push(ValidationError::required(Property::RedirectRoot)); @@ -21775,29 +27170,114 @@ impl Http { impl Pickle for Http { fn pickle(&self, out: &mut Vec) { - self.rate_limit_authenticated.pickle(out); - self.rate_limit_anonymous.pickle(out); + let mut bitmap = 0u64; + if self.rate_limit_authenticated.is_some() { + bitmap |= 1 << 0; + } + if self.rate_limit_anonymous.is_some() { + bitmap |= 1 << 1; + } + if self.enable_hsts { + bitmap |= 1 << 2; + } + if self.use_permissive_cors { + bitmap |= 1 << 3; + } + if self.redirect_root.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 9, 5, bitmap); + if let Some(value) = &self.rate_limit_authenticated { + value.pickle(out); + } + if let Some(value) = &self.rate_limit_anonymous { + value.pickle(out); + } self.allowed_endpoints.pickle(out); - self.enable_hsts.pickle(out); - self.use_permissive_cors.pickle(out); self.response_headers.pickle(out); - self.use_x_forwarded.pickle(out); - self.redirect_root.pickle(out); + self.client_ip_source.pickle(out); + self.forwarded_hops.pickle(out); + if let Some(value) = &self.redirect_root { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.rate_limit_authenticated = Pickle::unpickle(stream)?; - this.rate_limit_anonymous = Pickle::unpickle(stream)?; - this.allowed_endpoints = Pickle::unpickle(stream)?; - this.enable_hsts = Pickle::unpickle(stream)?; - this.use_permissive_cors = Pickle::unpickle(stream)?; - this.response_headers = Pickle::unpickle(stream)?; - this.use_x_forwarded = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.redirect_root = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 1000u64, + period: Duration::from_millis(60000), + }) + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 100u64, + period: Duration::from_millis(60000), + }) + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "200".to_string(), + ..Default::default() + } + }; + let field3: bool = if block.has(3) { block.flag(2) } else { false }; + let field4: bool = if block.has(4) { block.flag(3) } else { false }; + let field5: VecMap = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: ClientIpSource = if block.has(6) { + Pickle::unpickle(stream)? + } else { + ClientIpSource::Peer + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 1u64 + }; + let field8: Option = if block.has(8) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("/account".to_string()) + }; + stream.close_block(block); + Some(Self { + rate_limit_authenticated: field0, + rate_limit_anonymous: field1, + allowed_endpoints: field2, + enable_hsts: field3, + use_permissive_cors: field4, + response_headers: field5, + client_ip_source: field6, + forwarded_hops: field7, + redirect_root: field8, + }) } } @@ -21819,7 +27299,8 @@ impl Default for Http { enable_hsts: false, use_permissive_cors: false, response_headers: Default::default(), - use_x_forwarded: false, + client_ip_source: ClientIpSource::Peer, + forwarded_hops: 1u64, redirect_root: Some("/account".to_string()), } } @@ -21827,30 +27308,44 @@ impl Default for Http { impl IntoValue for Http { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(11); map.insert_unchecked( Property::RateLimitAuthenticated, - self.rate_limit_authenticated.into_value(), + self.rate_limit_authenticated.into_value_with(opts), ); map.insert_unchecked( Property::RateLimitAnonymous, - self.rate_limit_anonymous.into_value(), + self.rate_limit_anonymous.into_value_with(opts), ); map.insert_unchecked( Property::AllowedEndpoints, - self.allowed_endpoints.into_value(), + self.allowed_endpoints.into_value_with(opts), ); - map.insert_unchecked(Property::EnableHsts, self.enable_hsts.into_value()); + map.insert_unchecked(Property::EnableHsts, self.enable_hsts.into_value_with(opts)); map.insert_unchecked( Property::UsePermissiveCors, - self.use_permissive_cors.into_value(), + self.use_permissive_cors.into_value_with(opts), ); map.insert_unchecked( Property::ResponseHeaders, - self.response_headers.into_value(), + self.response_headers.into_value_with(opts), + ); + map.insert_unchecked( + Property::ClientIpSource, + self.client_ip_source.into_value_with(opts), + ); + map.insert_unchecked( + Property::ForwardedHops, + self.forwarded_hops.into_value_with(opts), + ); + map.insert_unchecked( + Property::RedirectRoot, + self.redirect_root.into_value_with(opts), ); - map.insert_unchecked(Property::UseXForwarded, self.use_x_forwarded.into_value()); - map.insert_unchecked(Property::RedirectRoot, self.redirect_root.into_value()); JmapValue::Object(map) } } @@ -21872,7 +27367,8 @@ impl RegistryJsonPropertyPatch for Http { Some(Property::ResponseHeaders) => self .response_headers .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::UseXForwarded) => self.use_x_forwarded.patch(pointer, value), + Some(Property::ClientIpSource) => self.client_ip_source.patch(pointer, value), + Some(Property::ForwardedHops) => self.forwarded_hops.patch(pointer, value), Some(Property::RedirectRoot) => self .redirect_root .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -21893,6 +27389,7 @@ impl HttpAuth { HttpAuth::Bearer(inner) => inner.validate(errors), } } + } impl Default for HttpAuth { @@ -21930,6 +27427,10 @@ impl Pickle for HttpAuth { impl IntoValue for HttpAuth { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { HttpAuth::Unauthenticated => { let mut obj = jmap_tools::Map::new(); @@ -21937,14 +27438,14 @@ impl IntoValue for HttpAuth { JmapValue::Object(obj) } HttpAuth::Basic(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Basic".into())); obj } HttpAuth::Bearer(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Bearer".into())); @@ -21996,19 +27497,34 @@ impl HttpAuthBasic { value.validate(errors); errors.len() == neb } + } impl Pickle for HttpAuthBasic { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.username.pickle(out); self.secret.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + secret: field1, + }) } } @@ -22023,9 +27539,13 @@ impl Default for HttpAuthBasic { impl IntoValue for HttpAuthBasic { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); JmapValue::Object(map) } } @@ -22057,17 +27577,27 @@ impl HttpAuthBearer { value.validate(errors); errors.len() == neb } + } impl Pickle for HttpAuthBearer { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.bearer_token.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.bearer_token = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + bearer_token: field0, + }) } } @@ -22081,8 +27611,15 @@ impl Default for HttpAuthBearer { impl IntoValue for HttpAuthBearer { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::BearerToken, self.bearer_token.into_value()); + map.insert_unchecked( + Property::BearerToken, + self.bearer_token.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -22160,35 +27697,147 @@ impl ObjectImpl for HttpForm { impl Pickle for HttpForm { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.field_email.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + if self.field_honey_pot.is_some() { + bitmap |= 1 << 2; + } + if self.field_name.is_some() { + bitmap |= 1 << 3; + } + if self.rate_limit.is_some() { + bitmap |= 1 << 4; + } + if self.field_subject.is_some() { + bitmap |= 1 << 5; + } + if self.validate_domain { + bitmap |= 1 << 6; + } + let block = crate::pickle::begin_block(out, 12, 7, bitmap); self.deliver_to.pickle(out); self.default_from_address.pickle(out); - self.field_email.pickle(out); - self.enable.pickle(out); - self.field_honey_pot.pickle(out); + if let Some(value) = &self.field_email { + value.pickle(out); + } + if let Some(value) = &self.field_honey_pot { + value.pickle(out); + } self.max_size.pickle(out); self.default_name.pickle(out); - self.field_name.pickle(out); - self.rate_limit.pickle(out); + if let Some(value) = &self.field_name { + value.pickle(out); + } + if let Some(value) = &self.rate_limit { + value.pickle(out); + } self.default_subject.pickle(out); - self.field_subject.pickle(out); - self.validate_domain.pickle(out); + if let Some(value) = &self.field_subject { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.deliver_to = Pickle::unpickle(stream)?; - this.default_from_address = Pickle::unpickle(stream)?; - this.field_email = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.field_honey_pot = Pickle::unpickle(stream)?; - this.max_size = Pickle::unpickle(stream)?; - this.default_name = Pickle::unpickle(stream)?; - this.field_name = Pickle::unpickle(stream)?; - this.rate_limit = Pickle::unpickle(stream)?; - this.default_subject = Pickle::unpickle(stream)?; - this.field_subject = Pickle::unpickle(stream)?; - this.validate_domain = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + "postmaster@localhost".to_string() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { false }; + let field4: Option = if block.has(4) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 102400 + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + "Anonymous".to_string() + }; + let field7: Option = if block.has(7) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: Option = if block.has(8) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 5u64, + period: Duration::from_millis(3600000), + }) + }; + let field9: String = if block.has(9) { + Pickle::unpickle(stream)? + } else { + "Contact form submission".to_string() + }; + let field10: Option = if block.has(10) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field11: bool = if block.has(11) { block.flag(6) } else { true }; + stream.close_block(block); + Some(Self { + deliver_to: field0, + default_from_address: field1, + field_email: field2, + enable: field3, + field_honey_pot: field4, + max_size: field5, + default_name: field6, + field_name: field7, + rate_limit: field8, + default_subject: field9, + field_subject: field10, + validate_domain: field11, + }) } } @@ -22216,22 +27865,41 @@ impl Default for HttpForm { impl IntoValue for HttpForm { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::DeliverTo, self.deliver_to.into_value()); + map.insert_unchecked(Property::DeliverTo, self.deliver_to.into_value_with(opts)); map.insert_unchecked( Property::DefaultFromAddress, - self.default_from_address.into_value(), + self.default_from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FieldEmail, self.field_email.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::FieldHoneyPot, + self.field_honey_pot.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxSize, self.max_size.into_value_with(opts)); + map.insert_unchecked( + Property::DefaultName, + self.default_name.into_value_with(opts), + ); + map.insert_unchecked(Property::FieldName, self.field_name.into_value_with(opts)); + map.insert_unchecked(Property::RateLimit, self.rate_limit.into_value_with(opts)); + map.insert_unchecked( + Property::DefaultSubject, + self.default_subject.into_value_with(opts), + ); + map.insert_unchecked( + Property::FieldSubject, + self.field_subject.into_value_with(opts), + ); + map.insert_unchecked( + Property::ValidateDomain, + self.validate_domain.into_value_with(opts), ); - map.insert_unchecked(Property::FieldEmail, self.field_email.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::FieldHoneyPot, self.field_honey_pot.into_value()); - map.insert_unchecked(Property::MaxSize, self.max_size.into_value()); - map.insert_unchecked(Property::DefaultName, self.default_name.into_value()); - map.insert_unchecked(Property::FieldName, self.field_name.into_value()); - map.insert_unchecked(Property::RateLimit, self.rate_limit.into_value()); - map.insert_unchecked(Property::DefaultSubject, self.default_subject.into_value()); - map.insert_unchecked(Property::FieldSubject, self.field_subject.into_value()); - map.insert_unchecked(Property::ValidateDomain, self.validate_domain.into_value()); JmapValue::Object(map) } } @@ -22318,6 +27986,14 @@ impl ObjectImpl for HttpLookup { if value.is_empty() { errors.push(ValidationError::required(Property::Url)); } + let value = &self.http_auth; + value.validate(errors); + let value = &self.http_headers; + for value in value.values() { + if value.is_empty() { + errors.push(ValidationError::required(Property::HttpHeaders)); + } + } errors.len() == neb } @@ -22328,33 +28004,111 @@ impl ObjectImpl for HttpLookup { impl Pickle for HttpLookup { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.is_gzipped { + bitmap |= 1 << 1; + } + if self.allow_invalid_certs { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 14, 3, bitmap); self.namespace.pickle(out); - self.enable.pickle(out); self.format.pickle(out); - self.is_gzipped.pickle(out); self.max_entries.pickle(out); self.max_entry_size.pickle(out); self.max_size.pickle(out); - self.refresh.pickle(out); + self.refresh_interval.pickle(out); self.retry.pickle(out); - self.timeout.pickle(out); self.url.pickle(out); + self.timeout.pickle(out); + self.http_auth.pickle(out); + self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.namespace = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.format = Pickle::unpickle(stream)?; - this.is_gzipped = Pickle::unpickle(stream)?; - this.max_entries = Pickle::unpickle(stream)?; - this.max_entry_size = Pickle::unpickle(stream)?; - this.max_size = Pickle::unpickle(stream)?; - this.refresh = Pickle::unpickle(stream)?; - this.retry = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: HttpLookupFormat = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { false }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 100000u64 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 512u64 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 104857600 + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(43200000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field9: String = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field11: bool = if block.has(11) { block.flag(2) } else { false }; + let field12: HttpAuth = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field13: VecMap = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + namespace: field0, + enable: field1, + format: field2, + is_gzipped: field3, + max_entries: field4, + max_entry_size: field5, + max_size: field6, + refresh_interval: field7, + retry: field8, + url: field9, + timeout: field10, + allow_invalid_certs: field11, + http_auth: field12, + http_headers: field13, + }) } } @@ -22368,28 +28122,50 @@ impl Default for HttpLookup { max_entries: 100000u64, max_entry_size: 512u64, max_size: 104857600, - refresh: Duration::from_millis(43200000), + refresh_interval: Duration::from_millis(43200000), retry: Duration::from_millis(3600000), - timeout: Duration::from_millis(30000), url: Default::default(), + timeout: Duration::from_millis(30000), + allow_invalid_certs: false, + http_auth: Default::default(), + http_headers: Default::default(), } } } impl IntoValue for HttpLookup { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::Namespace, self.namespace.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Format, self.format.into_value()); - map.insert_unchecked(Property::IsGzipped, self.is_gzipped.into_value()); - map.insert_unchecked(Property::MaxEntries, self.max_entries.into_value()); - map.insert_unchecked(Property::MaxEntrySize, self.max_entry_size.into_value()); - map.insert_unchecked(Property::MaxSize, self.max_size.into_value()); - map.insert_unchecked(Property::Refresh, self.refresh.into_value()); - map.insert_unchecked(Property::Retry, self.retry.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); + map.insert_unchecked(Property::Namespace, self.namespace.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Format, self.format.into_value_with(opts)); + map.insert_unchecked(Property::IsGzipped, self.is_gzipped.into_value_with(opts)); + map.insert_unchecked(Property::MaxEntries, self.max_entries.into_value_with(opts)); + map.insert_unchecked( + Property::MaxEntrySize, + self.max_entry_size.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxSize, self.max_size.into_value_with(opts)); + map.insert_unchecked( + Property::RefreshInterval, + self.refresh_interval.into_value_with(opts), + ); + map.insert_unchecked(Property::Retry, self.retry.into_value_with(opts)); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -22413,12 +28189,17 @@ impl RegistryJsonPropertyPatch for HttpLookup { Some(Property::MaxEntries) => self.max_entries.patch(pointer, value), Some(Property::MaxEntrySize) => self.max_entry_size.patch(pointer, value), Some(Property::MaxSize) => self.max_size.patch(pointer, value), - Some(Property::Refresh) => self.refresh.patch(pointer, value), + Some(Property::RefreshInterval) => self.refresh_interval.patch(pointer, value), Some(Property::Retry) => self.retry.patch(pointer, value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Url) => self .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), + Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), + Some(Property::HttpHeaders) => self + .http_headers + .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -22441,19 +28222,51 @@ impl HttpLookupCsv { impl Pickle for HttpLookupCsv { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.index_value.is_some() { + bitmap |= 1 << 0; + } + if self.skip_first { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.index_key.pickle(out); - self.index_value.pickle(out); + if let Some(value) = &self.index_value { + value.pickle(out); + } self.separator.pickle(out); - self.skip_first.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.index_key = Pickle::unpickle(stream)?; - this.index_value = Pickle::unpickle(stream)?; - this.separator = Pickle::unpickle(stream)?; - this.skip_first = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + ",".to_string() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { false }; + stream.close_block(block); + Some(Self { + index_key: field0, + index_value: field1, + separator: field2, + skip_first: field3, + }) } } @@ -22470,11 +28283,15 @@ impl Default for HttpLookupCsv { impl IntoValue for HttpLookupCsv { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::IndexKey, self.index_key.into_value()); - map.insert_unchecked(Property::IndexValue, self.index_value.into_value()); - map.insert_unchecked(Property::Separator, self.separator.into_value()); - map.insert_unchecked(Property::SkipFirst, self.skip_first.into_value()); + map.insert_unchecked(Property::IndexKey, self.index_key.into_value_with(opts)); + map.insert_unchecked(Property::IndexValue, self.index_value.into_value_with(opts)); + map.insert_unchecked(Property::Separator, self.separator.into_value_with(opts)); + map.insert_unchecked(Property::SkipFirst, self.skip_first.into_value_with(opts)); JmapValue::Object(map) } } @@ -22540,9 +28357,13 @@ impl Pickle for HttpLookupFormat { impl IntoValue for HttpLookupFormat { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { HttpLookupFormat::Csv(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Csv".into())); @@ -22596,19 +28417,34 @@ impl HurricaneCredential { value.validate(errors); errors.len() == neb } + } impl Pickle for HurricaneCredential { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.zone.pickle(out); self.secret.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.zone = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + zone: field0, + secret: field1, + }) } } @@ -22623,9 +28459,13 @@ impl Default for HurricaneCredential { impl IntoValue for HurricaneCredential { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Secret, self.secret.into_value()); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Secret, self.secret.into_value_with(opts)); JmapValue::Object(map) } } @@ -22653,7 +28493,7 @@ impl RegistryJsonPropertyPatch for HurricaneCredential { impl ObjectImpl for Imap { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Imap; fn validate(&self, errors: &mut Vec) -> bool { @@ -22662,12 +28502,15 @@ impl ObjectImpl for Imap { if *value < 1 { errors.push(ValidationError::min_value(Property::MaxAuthFailures, 1)); } - if let Some(value) = &self.max_concurrent { + if let Some(value) = &self.max_concurrent_connections { if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxConcurrent, 1)); + errors.push(ValidationError::min_value( + Property::MaxConcurrentConnections, + 1, + )); } } - if let Some(value) = &self.max_request_rate { + if let Some(value) = &self.rate_limit { value.validate(errors); } let value = &self.max_messages_per_command; @@ -22703,43 +28546,122 @@ impl ObjectImpl for Imap { impl Pickle for Imap { fn pickle(&self, out: &mut Vec) { - self.allow_plain_text_auth.pickle(out); + let mut bitmap = 0u64; + if self.allow_plain_text_auth { + bitmap |= 1 << 0; + } + if self.max_concurrent_connections.is_some() { + bitmap |= 1 << 1; + } + if self.rate_limit.is_some() { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 12, 3, bitmap); self.max_auth_failures.pickle(out); - self.max_concurrent.pickle(out); - self.max_request_rate.pickle(out); + if let Some(value) = &self.max_concurrent_connections { + value.pickle(out); + } + if let Some(value) = &self.rate_limit { + value.pickle(out); + } self.max_request_size.pickle(out); - self.timeout_anonymous.pickle(out); - self.timeout_authenticated.pickle(out); - self.timeout_idle.pickle(out); + self.anonymous_timeout.pickle(out); + self.authenticated_timeout.pickle(out); + self.idle_timeout.pickle(out); self.max_messages_per_command.pickle(out); self.min_uid_batch_size.pickle(out); self.max_uid_batches.pickle(out); self.max_messages_per_save.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_plain_text_auth = Pickle::unpickle(stream)?; - this.max_auth_failures = Pickle::unpickle(stream)?; - this.max_concurrent = Pickle::unpickle(stream)?; - this.max_request_rate = Pickle::unpickle(stream)?; - this.max_request_size = Pickle::unpickle(stream)?; - this.timeout_anonymous = Pickle::unpickle(stream)?; - this.timeout_authenticated = Pickle::unpickle(stream)?; - this.timeout_idle = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.max_messages_per_command = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - if stream.version() >= 1 { - this.min_uid_batch_size = Pickle::unpickle(stream)?; - } - if stream.version() >= 1 { - this.max_uid_batches = Pickle::unpickle(stream)?; - } - if stream.version() >= 1 { - this.max_messages_per_save = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { false }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(16u64) + }; + let field3: Option = if block.has(3) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 2000u64, + period: Duration::from_millis(60000), + }) + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 52428800 + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1800000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1800000) + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 1000000u64 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 500u64 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 10000u64 + }; + let field11: u64 = if block.has(11) { + Pickle::unpickle(stream)? + } else { + 1000000u64 + }; + stream.close_block(block); + Some(Self { + allow_plain_text_auth: field0, + max_auth_failures: field1, + max_concurrent_connections: field2, + rate_limit: field3, + max_request_size: field4, + anonymous_timeout: field5, + authenticated_timeout: field6, + idle_timeout: field7, + max_messages_per_command: field8, + min_uid_batch_size: field9, + max_uid_batches: field10, + max_messages_per_save: field11, + }) } } @@ -22748,15 +28670,15 @@ impl Default for Imap { Self { allow_plain_text_auth: false, max_auth_failures: 3u64, - max_concurrent: Some(16u64), - max_request_rate: Some(Rate { + max_concurrent_connections: Some(16u64), + rate_limit: Some(Rate { count: 2000u64, period: Duration::from_millis(60000), }), max_request_size: 52428800, - timeout_anonymous: Duration::from_millis(60000), - timeout_authenticated: Duration::from_millis(1800000), - timeout_idle: Duration::from_millis(1800000), + anonymous_timeout: Duration::from_millis(60000), + authenticated_timeout: Duration::from_millis(1800000), + idle_timeout: Duration::from_millis(1800000), max_messages_per_command: 1000000u64, min_uid_batch_size: 500u64, max_uid_batches: 10000u64, @@ -22767,39 +28689,55 @@ impl Default for Imap { impl IntoValue for Imap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); map.insert_unchecked( Property::AllowPlainTextAuth, - self.allow_plain_text_auth.into_value(), + self.allow_plain_text_auth.into_value_with(opts), ); map.insert_unchecked( Property::MaxAuthFailures, - self.max_auth_failures.into_value(), - ); - map.insert_unchecked(Property::MaxConcurrent, self.max_concurrent.into_value()); - map.insert_unchecked(Property::MaxRequestRate, self.max_request_rate.into_value()); - map.insert_unchecked(Property::MaxRequestSize, self.max_request_size.into_value()); - map.insert_unchecked( - Property::TimeoutAnonymous, - self.timeout_anonymous.into_value(), + self.max_auth_failures.into_value_with(opts), ); map.insert_unchecked( - Property::TimeoutAuthenticated, - self.timeout_authenticated.into_value(), + Property::MaxConcurrentConnections, + self.max_concurrent_connections.into_value_with(opts), + ); + map.insert_unchecked(Property::RateLimit, self.rate_limit.into_value_with(opts)); + map.insert_unchecked( + Property::MaxRequestSize, + self.max_request_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::AnonymousTimeout, + self.anonymous_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::AuthenticatedTimeout, + self.authenticated_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::IdleTimeout, + self.idle_timeout.into_value_with(opts), ); - map.insert_unchecked(Property::TimeoutIdle, self.timeout_idle.into_value()); map.insert_unchecked( Property::MaxMessagesPerCommand, - self.max_messages_per_command.into_value(), + self.max_messages_per_command.into_value_with(opts), ); map.insert_unchecked( Property::MinUidBatchSize, - self.min_uid_batch_size.into_value(), + self.min_uid_batch_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxUidBatches, + self.max_uid_batches.into_value_with(opts), ); - map.insert_unchecked(Property::MaxUidBatches, self.max_uid_batches.into_value()); map.insert_unchecked( Property::MaxMessagesPerSave, - self.max_messages_per_save.into_value(), + self.max_messages_per_save.into_value_with(opts), ); JmapValue::Object(map) } @@ -22814,14 +28752,16 @@ impl RegistryJsonPropertyPatch for Imap { match pointer.next_property() { Some(Property::AllowPlainTextAuth) => self.allow_plain_text_auth.patch(pointer, value), Some(Property::MaxAuthFailures) => self.max_auth_failures.patch(pointer, value), - Some(Property::MaxConcurrent) => self.max_concurrent.patch(pointer, value), - Some(Property::MaxRequestRate) => self.max_request_rate.patch(pointer, value), - Some(Property::MaxRequestSize) => self.max_request_size.patch(pointer, value), - Some(Property::TimeoutAnonymous) => self.timeout_anonymous.patch(pointer, value), - Some(Property::TimeoutAuthenticated) => { - self.timeout_authenticated.patch(pointer, value) + Some(Property::MaxConcurrentConnections) => { + self.max_concurrent_connections.patch(pointer, value) } - Some(Property::TimeoutIdle) => self.timeout_idle.patch(pointer, value), + Some(Property::RateLimit) => self.rate_limit.patch(pointer, value), + Some(Property::MaxRequestSize) => self.max_request_size.patch(pointer, value), + Some(Property::AnonymousTimeout) => self.anonymous_timeout.patch(pointer, value), + Some(Property::AuthenticatedTimeout) => { + self.authenticated_timeout.patch(pointer, value) + } + Some(Property::IdleTimeout) => self.idle_timeout.patch(pointer, value), Some(Property::MaxMessagesPerCommand) => { self.max_messages_per_command.patch(pointer, value) } @@ -22845,7 +28785,6 @@ impl ObjectImpl for InMemoryStore { fn validate(&self, errors: &mut Vec) -> bool { match self { InMemoryStore::Default => true, - InMemoryStore::Sharded(inner) => inner.validate(errors), InMemoryStore::Redis(inner) => inner.validate(errors), InMemoryStore::RedisCluster(inner) => inner.validate(errors), InMemoryStore::RedisSentinel(inner) => inner.validate(errors), @@ -22853,6 +28792,7 @@ impl ObjectImpl for InMemoryStore { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for InMemoryStore { @@ -22867,10 +28807,6 @@ impl Pickle for InMemoryStore { InMemoryStore::Default => { 0u16.pickle(out); } - InMemoryStore::Sharded(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } InMemoryStore::Redis(inner) => { 2u16.pickle(out); inner.pickle(out); @@ -22887,9 +28823,11 @@ impl Pickle for InMemoryStore { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Some(InMemoryStore::Default), - 1 => Pickle::unpickle(stream).map(InMemoryStore::Sharded), 2 => Pickle::unpickle(stream).map(InMemoryStore::Redis), 3 => Pickle::unpickle(stream).map(InMemoryStore::RedisCluster), 4 => Pickle::unpickle(stream).map(InMemoryStore::RedisSentinel), @@ -22900,35 +28838,32 @@ impl Pickle for InMemoryStore { impl IntoValue for InMemoryStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { InMemoryStore::Default => { let mut obj = jmap_tools::Map::new(); obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); JmapValue::Object(obj) } - InMemoryStore::Sharded(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Sharded".into())); - obj - } InMemoryStore::Redis(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); obj } InMemoryStore::RedisCluster(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisCluster".into())); obj } InMemoryStore::RedisSentinel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisSentinel".into())); @@ -22947,7 +28882,6 @@ impl RegistryJsonPatch for InMemoryStore { if !pointer.has_next() { match object_type(&pointer, &value)? { InMemoryStoreType::Default => *self = InMemoryStore::Default, - InMemoryStoreType::Sharded => *self = InMemoryStore::Sharded(Default::default()), InMemoryStoreType::Redis => *self = InMemoryStore::Redis(Default::default()), InMemoryStoreType::RedisCluster => { *self = InMemoryStore::RedisCluster(Default::default()) @@ -22959,7 +28893,6 @@ impl RegistryJsonPatch for InMemoryStore { } match self { InMemoryStore::Default => pointer.assert_eof(), - InMemoryStore::Sharded(inner) => inner.patch(pointer, value), InMemoryStore::Redis(inner) => inner.patch(pointer, value), InMemoryStore::RedisCluster(inner) => inner.patch(pointer, value), InMemoryStore::RedisSentinel(inner) => inner.patch(pointer, value), @@ -22971,7 +28904,6 @@ impl InMemoryStore { pub fn object_type(&self) -> InMemoryStoreType { match self { InMemoryStore::Default => InMemoryStoreType::Default, - InMemoryStore::Sharded(_) => InMemoryStoreType::Sharded, InMemoryStore::Redis(_) => InMemoryStoreType::Redis, InMemoryStore::RedisCluster(_) => InMemoryStoreType::RedisCluster, InMemoryStore::RedisSentinel(_) => InMemoryStoreType::RedisSentinel, @@ -22979,79 +28911,115 @@ impl InMemoryStore { } } -impl InMemoryStoreBase { +impl ObjectImpl for IndexQueueStatus { + const FLAGS: u64 = OBJ_NOT_STORED | OBJ_NO_CREATE | OBJ_NO_DESTROY; + const VERSION: u8 = 0; + const OBJECT: ObjectType = ObjectType::IndexQueueStatus; + + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.status; + value.validate(errors); + errors.len() == neb + } + + fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} +} + +impl Default for IndexQueueStatus { + fn default() -> Self { + Self { + index: Default::default(), + partition: 0u64, + queued_updates: 0u64, + queued_deletions: 0u64, + status: Default::default(), + } + } +} + +impl IntoValue for IndexQueueStatus { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(7); + map.insert_unchecked(Property::Index, self.index.into_value_with(opts)); + map.insert_unchecked(Property::Partition, self.partition.into_value_with(opts)); + map.insert_unchecked( + Property::QueuedUpdates, + self.queued_updates.into_value_with(opts), + ); + map.insert_unchecked( + Property::QueuedDeletions, + self.queued_deletions.into_value_with(opts), + ); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for IndexQueueStatus { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Index) => pointer.assert_server_set(), + Some(Property::Partition) => pointer.assert_server_set(), + Some(Property::QueuedUpdates) => pointer.assert_server_set(), + Some(Property::QueuedDeletions) => pointer.assert_server_set(), + Some(Property::Status) => self.status.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl IndexStatus { fn validate(&self, errors: &mut Vec) -> bool { match self { - InMemoryStoreBase::Redis(inner) => inner.validate(errors), - InMemoryStoreBase::RedisCluster(inner) => inner.validate(errors), - InMemoryStoreBase::RedisSentinel(inner) => inner.validate(errors), + IndexStatus::Running => true, + IndexStatus::Failed(inner) => inner.validate(errors), } } } -impl Default for InMemoryStoreBase { +impl Default for IndexStatus { fn default() -> Self { - InMemoryStoreBase::Redis(Default::default()) + IndexStatus::Running } } -impl Pickle for InMemoryStoreBase { - fn pickle(&self, out: &mut Vec) { - match self { - InMemoryStoreBase::Redis(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - InMemoryStoreBase::RedisCluster(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - InMemoryStoreBase::RedisSentinel(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(InMemoryStoreBase::Redis), - 1 => Pickle::unpickle(stream).map(InMemoryStoreBase::RedisCluster), - 2 => Pickle::unpickle(stream).map(InMemoryStoreBase::RedisSentinel), - _ => None, - } - } -} - -impl IntoValue for InMemoryStoreBase { +impl IntoValue for IndexStatus { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { - InMemoryStoreBase::Redis(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); - obj + IndexStatus::Running => { + let mut obj = jmap_tools::Map::new(); + obj.insert_unchecked(Property::Type, JmapValue::Str("Running".into())); + JmapValue::Object(obj) } - InMemoryStoreBase::RedisCluster(obj) => { - let mut obj = obj.into_value(); + IndexStatus::Failed(obj) => { + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("RedisCluster".into())); - obj - } - InMemoryStoreBase::RedisSentinel(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("RedisSentinel".into())); + .insert_unchecked(Property::Type, JmapValue::Str("Failed".into())); obj } } } } -impl RegistryJsonPatch for InMemoryStoreBase { +impl RegistryJsonPatch for IndexStatus { fn patch<'x>( &mut self, pointer: JsonPointerPatch<'_>, @@ -23059,38 +29027,84 @@ impl RegistryJsonPatch for InMemoryStoreBase { ) -> PatchResult<'x> { if !pointer.has_next() { match object_type(&pointer, &value)? { - InMemoryStoreBaseType::Redis => { - *self = InMemoryStoreBase::Redis(Default::default()) - } - InMemoryStoreBaseType::RedisCluster => { - *self = InMemoryStoreBase::RedisCluster(Default::default()) - } - InMemoryStoreBaseType::RedisSentinel => { - *self = InMemoryStoreBase::RedisSentinel(Default::default()) - } + IndexStatusType::Running => *self = IndexStatus::Running, + IndexStatusType::Failed => *self = IndexStatus::Failed(Default::default()), } } match self { - InMemoryStoreBase::Redis(inner) => inner.patch(pointer, value), - InMemoryStoreBase::RedisCluster(inner) => inner.patch(pointer, value), - InMemoryStoreBase::RedisSentinel(inner) => inner.patch(pointer, value), + IndexStatus::Running => pointer.assert_eof(), + IndexStatus::Failed(inner) => inner.patch(pointer, value), } } } -impl InMemoryStoreBase { - pub fn object_type(&self) -> InMemoryStoreBaseType { +impl IndexStatus { + pub fn object_type(&self) -> IndexStatusType { match self { - InMemoryStoreBase::Redis(_) => InMemoryStoreBaseType::Redis, - InMemoryStoreBase::RedisCluster(_) => InMemoryStoreBaseType::RedisCluster, - InMemoryStoreBase::RedisSentinel(_) => InMemoryStoreBaseType::RedisSentinel, + IndexStatus::Running => IndexStatusType::Running, + IndexStatus::Failed(_) => IndexStatusType::Failed, + } + } +} + +impl IndexStatusFailed { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.reason; + if value.is_empty() { + errors.push(ValidationError::required(Property::Reason)); + } + let value = &self.next_retry; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::NextRetry, value)); + } + errors.len() == neb + } +} + +impl Default for IndexStatusFailed { + fn default() -> Self { + Self { + reason: Default::default(), + next_retry: Default::default(), + } + } +} + +impl IntoValue for IndexStatusFailed { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(4); + map.insert_unchecked(Property::Reason, self.reason.into_value_with(opts)); + map.insert_unchecked(Property::NextRetry, self.next_retry.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for IndexStatusFailed { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Reason) => self.reason.patch(pointer, value), + Some(Property::NextRetry) => self.next_retry.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), } } } impl ObjectImpl for Jmap { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 2; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Jmap; fn validate(&self, errors: &mut Vec) -> bool { @@ -23107,17 +29121,17 @@ impl ObjectImpl for Jmap { if *value < 1 { errors.push(ValidationError::min_value(Property::ParseLimitEmail, 1)); } - let value = &self.changes_max_results; + let value = &self.max_changes_results; if *value < 1 { - errors.push(ValidationError::min_value(Property::ChangesMaxResults, 1)); + errors.push(ValidationError::min_value(Property::MaxChangesResults, 1)); } - let value = &self.get_max_results; + let value = &self.max_get_results; if *value < 1 { - errors.push(ValidationError::min_value(Property::GetMaxResults, 1)); + errors.push(ValidationError::min_value(Property::MaxGetResults, 1)); } - let value = &self.query_max_results; + let value = &self.max_query_results; if *value < 1 { - errors.push(ValidationError::min_value(Property::QueryMaxResults, 1)); + errors.push(ValidationError::min_value(Property::MaxQueryResults, 1)); } let value = &self.max_method_calls; if *value < 1 { @@ -23135,13 +29149,13 @@ impl ObjectImpl for Jmap { if *value < 1 { errors.push(ValidationError::min_value(Property::MaxRequestSize, 1)); } - let value = &self.set_max_objects; + let value = &self.max_set_objects; if *value < 1 { - errors.push(ValidationError::min_value(Property::SetMaxObjects, 1)); + errors.push(ValidationError::min_value(Property::MaxSetObjects, 1)); } - let value = &self.snippet_max_results; + let value = &self.max_snippet_results; if *value < 1 { - errors.push(ValidationError::min_value(Property::SnippetMaxResults, 1)); + errors.push(ValidationError::min_value(Property::MaxSnippetResults, 1)); } if let Some(value) = &self.max_concurrent_uploads { if *value < 1 { @@ -23170,9 +29184,9 @@ impl ObjectImpl for Jmap { if *value < Duration::from_millis(1000) { errors.push(ValidationError::min_value(Property::UploadTtl, 1000)); } - let value = &self.push_max_attempts; + let value = &self.max_push_attempts; if *value < 1 { - errors.push(ValidationError::min_value(Property::PushMaxAttempts, 1)); + errors.push(ValidationError::min_value(Property::MaxPushAttempts, 1)); } let value = &self.push_shards_total; if *value < 1 { @@ -23198,29 +29212,48 @@ impl ObjectImpl for Jmap { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Pickle for Jmap { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.max_concurrent_requests.is_some() { + bitmap |= 1 << 0; + } + if self.max_concurrent_uploads.is_some() { + bitmap |= 1 << 1; + } + if self.max_subscriptions.is_some() { + bitmap |= 1 << 2; + } + if self.web_push_contact.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 31, 4, bitmap); self.parse_limit_event.pickle(out); self.parse_limit_contact.pickle(out); self.parse_limit_email.pickle(out); - self.changes_max_results.pickle(out); - self.get_max_results.pickle(out); - self.query_max_results.pickle(out); + self.max_changes_results.pickle(out); + self.max_get_results.pickle(out); + self.max_query_results.pickle(out); self.max_method_calls.pickle(out); - self.max_concurrent_requests.pickle(out); + if let Some(value) = &self.max_concurrent_requests { + value.pickle(out); + } self.max_request_size.pickle(out); - self.set_max_objects.pickle(out); - self.snippet_max_results.pickle(out); - self.max_concurrent_uploads.pickle(out); + self.max_set_objects.pickle(out); + self.max_snippet_results.pickle(out); + if let Some(value) = &self.max_concurrent_uploads { + value.pickle(out); + } self.max_upload_size.pickle(out); self.max_upload_count.pickle(out); self.upload_quota.pickle(out); self.upload_ttl.pickle(out); self.event_source_throttle.pickle(out); self.push_attempt_wait.pickle(out); - self.push_max_attempts.pickle(out); + self.max_push_attempts.pickle(out); self.push_retry_wait.pickle(out); self.push_throttle.pickle(out); self.push_request_timeout.pickle(out); @@ -23229,52 +29262,227 @@ impl Pickle for Jmap { self.websocket_heartbeat.pickle(out); self.websocket_throttle.pickle(out); self.websocket_timeout.pickle(out); - self.max_subscriptions.pickle(out); + if let Some(value) = &self.max_subscriptions { + value.pickle(out); + } self.web_push_key.pickle(out); - self.web_push_contact.pickle(out); + if let Some(value) = &self.web_push_contact { + value.pickle(out); + } self.max_push_size.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.parse_limit_event = Pickle::unpickle(stream)?; - this.parse_limit_contact = Pickle::unpickle(stream)?; - this.parse_limit_email = Pickle::unpickle(stream)?; - this.changes_max_results = Pickle::unpickle(stream)?; - this.get_max_results = Pickle::unpickle(stream)?; - this.query_max_results = Pickle::unpickle(stream)?; - this.max_method_calls = Pickle::unpickle(stream)?; - this.max_concurrent_requests = Pickle::unpickle(stream)?; - this.max_request_size = Pickle::unpickle(stream)?; - this.set_max_objects = Pickle::unpickle(stream)?; - this.snippet_max_results = Pickle::unpickle(stream)?; - this.max_concurrent_uploads = Pickle::unpickle(stream)?; - this.max_upload_size = Pickle::unpickle(stream)?; - this.max_upload_count = Pickle::unpickle(stream)?; - this.upload_quota = Pickle::unpickle(stream)?; - this.upload_ttl = Pickle::unpickle(stream)?; - this.event_source_throttle = Pickle::unpickle(stream)?; - this.push_attempt_wait = Pickle::unpickle(stream)?; - this.push_max_attempts = Pickle::unpickle(stream)?; - this.push_retry_wait = Pickle::unpickle(stream)?; - this.push_throttle = Pickle::unpickle(stream)?; - this.push_request_timeout = Pickle::unpickle(stream)?; - this.push_verify_timeout = Pickle::unpickle(stream)?; - this.push_shards_total = Pickle::unpickle(stream)?; - this.websocket_heartbeat = Pickle::unpickle(stream)?; - this.websocket_throttle = Pickle::unpickle(stream)?; - this.websocket_timeout = Pickle::unpickle(stream)?; - this.max_subscriptions = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.web_push_key = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - if stream.version() >= 1 { - this.web_push_contact = Pickle::unpickle(stream)?; - } - if stream.version() >= 2 { - this.max_push_size = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 5000u64 + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500u64 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 5000u64 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 16u64 + }; + let field7: Option = if block.has(7) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(4u64) + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 10000000u64 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 500u64 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field11: Option = if block.has(11) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(4u64) + }; + let field12: u64 = if block.has(12) { + Pickle::unpickle(stream)? + } else { + 50000000u64 + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 1000u64 + }; + let field14: u64 = if block.has(14) { + Pickle::unpickle(stream)? + } else { + 50000000u64 + }; + let field15: Duration = if block.has(15) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field16: Duration = if block.has(16) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field17: Duration = if block.has(17) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field18: u64 = if block.has(18) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field19: Duration = if block.has(19) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field20: Duration = if block.has(20) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field21: Duration = if block.has(21) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field22: Duration = if block.has(22) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field23: u64 = if block.has(23) { + Pickle::unpickle(stream)? + } else { + 1u64 + }; + let field24: Duration = if block.has(24) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field25: Duration = if block.has(25) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field26: Duration = if block.has(26) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field27: Option = if block.has(27) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(15u64) + }; + let field28: SecretTextOptional = if block.has(28) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field29: Option = if block.has(29) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field30: u64 = if block.has(30) { + Pickle::unpickle(stream)? + } else { + 4096u64 + }; + stream.close_block(block); + Some(Self { + parse_limit_event: field0, + parse_limit_contact: field1, + parse_limit_email: field2, + max_changes_results: field3, + max_get_results: field4, + max_query_results: field5, + max_method_calls: field6, + max_concurrent_requests: field7, + max_request_size: field8, + max_set_objects: field9, + max_snippet_results: field10, + max_concurrent_uploads: field11, + max_upload_size: field12, + max_upload_count: field13, + upload_quota: field14, + upload_ttl: field15, + event_source_throttle: field16, + push_attempt_wait: field17, + max_push_attempts: field18, + push_retry_wait: field19, + push_throttle: field20, + push_request_timeout: field21, + push_verify_timeout: field22, + push_shards_total: field23, + websocket_heartbeat: field24, + websocket_throttle: field25, + websocket_timeout: field26, + max_subscriptions: field27, + web_push_key: field28, + web_push_contact: field29, + max_push_size: field30, + }) } } @@ -23284,14 +29492,14 @@ impl Default for Jmap { parse_limit_event: 10u64, parse_limit_contact: 10u64, parse_limit_email: 10u64, - changes_max_results: 5000u64, - get_max_results: 500u64, - query_max_results: 5000u64, + max_changes_results: 5000u64, + max_get_results: 500u64, + max_query_results: 5000u64, max_method_calls: 16u64, max_concurrent_requests: Some(4u64), max_request_size: 10000000u64, - set_max_objects: 500u64, - snippet_max_results: 100u64, + max_set_objects: 500u64, + max_snippet_results: 100u64, max_concurrent_uploads: Some(4u64), max_upload_size: 50000000u64, max_upload_count: 1000u64, @@ -23299,7 +29507,7 @@ impl Default for Jmap { upload_ttl: Duration::from_millis(3600000), event_source_throttle: Duration::from_millis(1000), push_attempt_wait: Duration::from_millis(60000), - push_max_attempts: 3u64, + max_push_attempts: 3u64, push_retry_wait: Duration::from_millis(1000), push_throttle: Duration::from_millis(1000), push_request_timeout: Duration::from_millis(10000), @@ -23318,92 +29526,132 @@ impl Default for Jmap { impl IntoValue for Jmap { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(33); map.insert_unchecked( Property::ParseLimitEvent, - self.parse_limit_event.into_value(), + self.parse_limit_event.into_value_with(opts), ); map.insert_unchecked( Property::ParseLimitContact, - self.parse_limit_contact.into_value(), + self.parse_limit_contact.into_value_with(opts), ); map.insert_unchecked( Property::ParseLimitEmail, - self.parse_limit_email.into_value(), + self.parse_limit_email.into_value_with(opts), ); map.insert_unchecked( - Property::ChangesMaxResults, - self.changes_max_results.into_value(), + Property::MaxChangesResults, + self.max_changes_results.into_value_with(opts), ); - map.insert_unchecked(Property::GetMaxResults, self.get_max_results.into_value()); map.insert_unchecked( - Property::QueryMaxResults, - self.query_max_results.into_value(), + Property::MaxGetResults, + self.max_get_results.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxQueryResults, + self.max_query_results.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMethodCalls, + self.max_method_calls.into_value_with(opts), ); - map.insert_unchecked(Property::MaxMethodCalls, self.max_method_calls.into_value()); map.insert_unchecked( Property::MaxConcurrentRequests, - self.max_concurrent_requests.into_value(), + self.max_concurrent_requests.into_value_with(opts), ); - map.insert_unchecked(Property::MaxRequestSize, self.max_request_size.into_value()); - map.insert_unchecked(Property::SetMaxObjects, self.set_max_objects.into_value()); map.insert_unchecked( - Property::SnippetMaxResults, - self.snippet_max_results.into_value(), + Property::MaxRequestSize, + self.max_request_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxSetObjects, + self.max_set_objects.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxSnippetResults, + self.max_snippet_results.into_value_with(opts), ); map.insert_unchecked( Property::MaxConcurrentUploads, - self.max_concurrent_uploads.into_value(), + self.max_concurrent_uploads.into_value_with(opts), ); - map.insert_unchecked(Property::MaxUploadSize, self.max_upload_size.into_value()); - map.insert_unchecked(Property::MaxUploadCount, self.max_upload_count.into_value()); - map.insert_unchecked(Property::UploadQuota, self.upload_quota.into_value()); - map.insert_unchecked(Property::UploadTtl, self.upload_ttl.into_value()); + map.insert_unchecked( + Property::MaxUploadSize, + self.max_upload_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxUploadCount, + self.max_upload_count.into_value_with(opts), + ); + map.insert_unchecked( + Property::UploadQuota, + self.upload_quota.into_value_with(opts), + ); + map.insert_unchecked(Property::UploadTtl, self.upload_ttl.into_value_with(opts)); map.insert_unchecked( Property::EventSourceThrottle, - self.event_source_throttle.into_value(), + self.event_source_throttle.into_value_with(opts), ); map.insert_unchecked( Property::PushAttemptWait, - self.push_attempt_wait.into_value(), + self.push_attempt_wait.into_value_with(opts), ); map.insert_unchecked( - Property::PushMaxAttempts, - self.push_max_attempts.into_value(), + Property::MaxPushAttempts, + self.max_push_attempts.into_value_with(opts), + ); + map.insert_unchecked( + Property::PushRetryWait, + self.push_retry_wait.into_value_with(opts), + ); + map.insert_unchecked( + Property::PushThrottle, + self.push_throttle.into_value_with(opts), ); - map.insert_unchecked(Property::PushRetryWait, self.push_retry_wait.into_value()); - map.insert_unchecked(Property::PushThrottle, self.push_throttle.into_value()); map.insert_unchecked( Property::PushRequestTimeout, - self.push_request_timeout.into_value(), + self.push_request_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PushVerifyTimeout, - self.push_verify_timeout.into_value(), + self.push_verify_timeout.into_value_with(opts), ); map.insert_unchecked( Property::PushShardsTotal, - self.push_shards_total.into_value(), + self.push_shards_total.into_value_with(opts), ); map.insert_unchecked( Property::WebsocketHeartbeat, - self.websocket_heartbeat.into_value(), + self.websocket_heartbeat.into_value_with(opts), ); map.insert_unchecked( Property::WebsocketThrottle, - self.websocket_throttle.into_value(), + self.websocket_throttle.into_value_with(opts), ); map.insert_unchecked( Property::WebsocketTimeout, - self.websocket_timeout.into_value(), + self.websocket_timeout.into_value_with(opts), ); map.insert_unchecked( Property::MaxSubscriptions, - self.max_subscriptions.into_value(), + self.max_subscriptions.into_value_with(opts), + ); + map.insert_unchecked( + Property::WebPushKey, + self.web_push_key.into_value_with(opts), + ); + map.insert_unchecked( + Property::WebPushContact, + self.web_push_contact.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxPushSize, + self.max_push_size.into_value_with(opts), ); - map.insert_unchecked(Property::WebPushKey, self.web_push_key.into_value()); - map.insert_unchecked(Property::WebPushContact, self.web_push_contact.into_value()); - map.insert_unchecked(Property::MaxPushSize, self.max_push_size.into_value()); JmapValue::Object(map) } } @@ -23418,16 +29666,16 @@ impl RegistryJsonPropertyPatch for Jmap { Some(Property::ParseLimitEvent) => self.parse_limit_event.patch(pointer, value), Some(Property::ParseLimitContact) => self.parse_limit_contact.patch(pointer, value), Some(Property::ParseLimitEmail) => self.parse_limit_email.patch(pointer, value), - Some(Property::ChangesMaxResults) => self.changes_max_results.patch(pointer, value), - Some(Property::GetMaxResults) => self.get_max_results.patch(pointer, value), - Some(Property::QueryMaxResults) => self.query_max_results.patch(pointer, value), + Some(Property::MaxChangesResults) => self.max_changes_results.patch(pointer, value), + Some(Property::MaxGetResults) => self.max_get_results.patch(pointer, value), + Some(Property::MaxQueryResults) => self.max_query_results.patch(pointer, value), Some(Property::MaxMethodCalls) => self.max_method_calls.patch(pointer, value), Some(Property::MaxConcurrentRequests) => { self.max_concurrent_requests.patch(pointer, value) } Some(Property::MaxRequestSize) => self.max_request_size.patch(pointer, value), - Some(Property::SetMaxObjects) => self.set_max_objects.patch(pointer, value), - Some(Property::SnippetMaxResults) => self.snippet_max_results.patch(pointer, value), + Some(Property::MaxSetObjects) => self.max_set_objects.patch(pointer, value), + Some(Property::MaxSnippetResults) => self.max_snippet_results.patch(pointer, value), Some(Property::MaxConcurrentUploads) => { self.max_concurrent_uploads.patch(pointer, value) } @@ -23437,7 +29685,7 @@ impl RegistryJsonPropertyPatch for Jmap { Some(Property::UploadTtl) => self.upload_ttl.patch(pointer, value), Some(Property::EventSourceThrottle) => self.event_source_throttle.patch(pointer, value), Some(Property::PushAttemptWait) => self.push_attempt_wait.patch(pointer, value), - Some(Property::PushMaxAttempts) => self.push_max_attempts.patch(pointer, value), + Some(Property::MaxPushAttempts) => self.max_push_attempts.patch(pointer, value), Some(Property::PushRetryWait) => self.push_retry_wait.patch(pointer, value), Some(Property::PushThrottle) => self.push_throttle.patch(pointer, value), Some(Property::PushRequestTimeout) => self.push_request_timeout.patch(pointer, value), @@ -23468,6 +29716,7 @@ impl JokerAuth { JokerAuth::UsernamePassword(inner) => inner.validate(errors), } } + } impl Default for JokerAuth { @@ -23501,16 +29750,20 @@ impl Pickle for JokerAuth { impl IntoValue for JokerAuth { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { JokerAuth::ApiKey(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ApiKey".into())); obj } JokerAuth::UsernamePassword(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("UsernamePassword".into())); @@ -23557,17 +29810,25 @@ impl JokerAuthApiKey { value.validate(errors); errors.len() == neb } + } impl Pickle for JokerAuthApiKey { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.api_key.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.api_key = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKey = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { api_key: field0 }) } } @@ -23581,8 +29842,12 @@ impl Default for JokerAuthApiKey { impl IntoValue for JokerAuthApiKey { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::ApiKey, self.api_key.into_value()); + map.insert_unchecked(Property::ApiKey, self.api_key.into_value_with(opts)); JmapValue::Object(map) } } @@ -23615,19 +29880,34 @@ impl JokerAuthUsernamePassword { value.validate(errors); errors.len() == neb } + } impl Pickle for JokerAuthUsernamePassword { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.username.pickle(out); self.password.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.username = Pickle::unpickle(stream)?; - this.password = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKey = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + username: field0, + password: field1, + }) } } @@ -23642,9 +29922,13 @@ impl Default for JokerAuthUsernamePassword { impl IntoValue for JokerAuthUsernamePassword { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Username, self.username.into_value()); - map.insert_unchecked(Property::Password, self.password.into_value()); + map.insert_unchecked(Property::Username, self.username.into_value_with(opts)); + map.insert_unchecked(Property::Password, self.password.into_value_with(opts)); JmapValue::Object(map) } } @@ -23672,14 +29956,14 @@ impl RegistryJsonPropertyPatch for JokerAuthUsernamePassword { impl KafkaCoordinator { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.brokers; + let value = &self.hosts; for value in value.iter() { if value.is_empty() { - errors.push(ValidationError::required(Property::Brokers)); + errors.push(ValidationError::required(Property::Hosts)); } } if value.len() < 1 { - errors.push(ValidationError::min_items(Property::Brokers, 1)); + errors.push(ValidationError::min_items(Property::Hosts, 1)); } let value = &self.group_id; if value.is_empty() { @@ -23691,40 +29975,92 @@ impl KafkaCoordinator { impl Pickle for KafkaCoordinator { fn pickle(&self, out: &mut Vec) { - self.brokers.pickle(out); + let mut bitmap = 0u64; + if self.use_tls { + bitmap |= 1 << 0; + } + if self.allow_invalid_certs { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 6, 2, bitmap); + self.hosts.pickle(out); self.group_id.pickle(out); - self.timeout_message.pickle(out); - self.timeout_session.pickle(out); + self.message_timeout.pickle(out); + self.session_timeout.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.brokers = Pickle::unpickle(stream)?; - this.group_id = Pickle::unpickle(stream)?; - this.timeout_message = Pickle::unpickle(stream)?; - this.timeout_session = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field4: bool = if block.has(4) { block.flag(0) } else { false }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + stream.close_block(block); + Some(Self { + hosts: field0, + group_id: field1, + message_timeout: field2, + session_timeout: field3, + use_tls: field4, + allow_invalid_certs: field5, + }) } } impl Default for KafkaCoordinator { fn default() -> Self { Self { - brokers: Default::default(), + hosts: Default::default(), group_id: Default::default(), - timeout_message: Duration::from_millis(5000), - timeout_session: Duration::from_millis(5000), + message_timeout: Duration::from_millis(5000), + session_timeout: Duration::from_millis(5000), + use_tls: false, + allow_invalid_certs: false, } } } impl IntoValue for KafkaCoordinator { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Brokers, self.brokers.into_value()); - map.insert_unchecked(Property::GroupId, self.group_id.into_value()); - map.insert_unchecked(Property::TimeoutMessage, self.timeout_message.into_value()); - map.insert_unchecked(Property::TimeoutSession, self.timeout_session.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(8); + map.insert_unchecked(Property::Hosts, self.hosts.into_value_with(opts)); + map.insert_unchecked(Property::GroupId, self.group_id.into_value_with(opts)); + map.insert_unchecked( + Property::MessageTimeout, + self.message_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::SessionTimeout, + self.session_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -23736,12 +30072,14 @@ impl RegistryJsonPropertyPatch for KafkaCoordinator { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Brokers) => self.brokers.patch(pointer, value), + Some(Property::Hosts) => self.hosts.patch(pointer, value), Some(Property::GroupId) => self .group_id .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::TimeoutMessage) => self.timeout_message.patch(pointer, value), - Some(Property::TimeoutSession) => self.timeout_session.patch(pointer, value), + Some(Property::MessageTimeout) => self.message_timeout.patch(pointer, value), + Some(Property::SessionTimeout) => self.session_timeout.patch(pointer, value), + Some(Property::UseTls) => self.use_tls.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -23777,9 +30115,9 @@ impl LdapDirectory { if value.is_empty() { errors.push(ValidationError::required(Property::FilterLogin)); } - let value = &self.filter_mailbox; + let value = &self.filter_recipient; if value.is_empty() { - errors.push(ValidationError::required(Property::FilterMailbox)); + errors.push(ValidationError::required(Property::FilterRecipient)); } if let Some(value) = &self.filter_member_of { if value.is_empty() { @@ -23839,36 +30177,51 @@ impl LdapDirectory { 8192, )); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for LdapDirectory { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.use_tls { + bitmap |= 1 << 1; + } + if self.bind_dn.is_some() { + bitmap |= 1 << 2; + } + if self.bind_authentication { + bitmap |= 1 << 3; + } + if self.filter_member_of.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 26, 6, bitmap); self.description.pickle(out); self.url.pickle(out); + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } self.timeout.pickle(out); - self.allow_invalid_certs.pickle(out); - self.use_tls.pickle(out); self.base_dn.pickle(out); - self.bind_dn.pickle(out); + if let Some(value) = &self.bind_dn { + value.pickle(out); + } self.bind_secret.pickle(out); - self.bind_authentication.pickle(out); self.filter_login.pickle(out); - self.filter_mailbox.pickle(out); - self.filter_member_of.pickle(out); + self.filter_recipient.pickle(out); + if let Some(value) = &self.filter_member_of { + value.pickle(out); + } self.attr_class.pickle(out); self.attr_description.pickle(out); self.attr_email.pickle(out); @@ -23881,37 +30234,159 @@ impl Pickle for LdapDirectory { self.pool_timeout_create.pickle(out); self.pool_timeout_recycle.pickle(out); self.pool_timeout_wait.pickle(out); - self.member_tenant_id.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.base_dn = Pickle::unpickle(stream)?; - this.bind_dn = Pickle::unpickle(stream)?; - this.bind_secret = Pickle::unpickle(stream)?; - this.bind_authentication = Pickle::unpickle(stream)?; - this.filter_login = Pickle::unpickle(stream)?; - this.filter_mailbox = Pickle::unpickle(stream)?; - this.filter_member_of = Pickle::unpickle(stream)?; - this.attr_class = Pickle::unpickle(stream)?; - this.attr_description = Pickle::unpickle(stream)?; - this.attr_email = Pickle::unpickle(stream)?; - this.attr_email_alias = Pickle::unpickle(stream)?; - this.attr_member_of = Pickle::unpickle(stream)?; - this.attr_secret = Pickle::unpickle(stream)?; - this.attr_secret_changed = Pickle::unpickle(stream)?; - this.group_class = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_timeout_create = Pickle::unpickle(stream)?; - this.pool_timeout_recycle = Pickle::unpickle(stream)?; - this.pool_timeout_wait = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + "ldap://localhost:389".to_string() + }; + #[cfg(not(feature = "enterprise"))] + if block.has(2) { + let block = stream.open_block()?; + stream.close_block(block); + } + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field4: bool = if block.has(4) { block.flag(0) } else { false }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: SecretKeyOptional = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: bool = if block.has(9) { block.flag(3) } else { true }; + let field10: String = if block.has(10) { + Pickle::unpickle(stream)? + } else { + "(&(objectClass=inetOrgPerson)(mail=?))".to_string() + }; + let field11: String = if block.has(11) { + Pickle::unpickle(stream)? + } else { + "(|(&(objectClass=inetOrgPerson)(|(mail=?)(mailAlias=?)))(&(objectClass=groupOfNames)(|(mail=?)(mailAlias=?))))".to_string() + }; + let field12: Option = if block.has(12) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("(&(objectClass=groupOfNames)(member=?))".to_string()) + }; + let field13: Map = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["objectClass".to_string()]) + }; + let field14: Map = if block.has(14) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["description".to_string()]) + }; + let field15: Map = if block.has(15) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["mail".to_string()]) + }; + let field16: Map = if block.has(16) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["mailAlias".to_string()]) + }; + let field17: Map = if block.has(17) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["memberOf".to_string()]) + }; + let field18: Map = if block.has(18) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["userPassword".to_string()]) + }; + let field19: Map = if block.has(19) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["pwdChangeTime".to_string()]) + }; + let field20: String = if block.has(20) { + Pickle::unpickle(stream)? + } else { + "groupOfNames".to_string() + }; + let field21: u64 = if block.has(21) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field22: Duration = if block.has(22) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field23: Duration = if block.has(23) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field24: Duration = if block.has(24) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + stream.close_block(block); + Some(Self { + description: field0, + url: field1, + timeout: field3, + allow_invalid_certs: field4, + use_tls: field5, + base_dn: field6, + bind_dn: field7, + bind_secret: field8, + bind_authentication: field9, + filter_login: field10, + filter_recipient: field11, + filter_member_of: field12, + attr_class: field13, + attr_description: field14, + attr_email: field15, + attr_email_alias: field16, + attr_member_of: field17, + attr_secret: field18, + attr_secret_changed: field19, + group_class: field20, + pool_max_connections: field21, + pool_timeout_create: field22, + pool_timeout_recycle: field23, + pool_timeout_wait: field24, + }) } } @@ -23928,7 +30403,7 @@ impl Default for LdapDirectory { bind_secret: Default::default(), bind_authentication: true, filter_login: "(&(objectClass=inetOrgPerson)(mail=?))".to_string(), - filter_mailbox: "(|(&(objectClass=inetOrgPerson)(|(mail=?)(mailAlias=?)))(&(objectClass=groupOfNames)(|(mail=?)(mailAlias=?))))".to_string(), + filter_recipient: "(|(&(objectClass=inetOrgPerson)(|(mail=?)(mailAlias=?)))(&(objectClass=groupOfNames)(|(mail=?)(mailAlias=?))))".to_string(), filter_member_of: Some("(&(objectClass=groupOfNames)(member=?))".to_string()), attr_class: Map::new(vec!["objectClass".to_string()]), attr_description: Map::new(vec!["description".to_string()]), @@ -23942,63 +30417,83 @@ impl Default for LdapDirectory { pool_timeout_create: Duration::from_millis(30000), pool_timeout_recycle: Duration::from_millis(30000), pool_timeout_wait: Duration::from_millis(30000), - member_tenant_id: Default::default(), } } } impl IntoValue for LdapDirectory { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(27); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(28); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); - map.insert_unchecked(Property::BaseDn, self.base_dn.into_value()); - map.insert_unchecked(Property::BindDn, self.bind_dn.into_value()); - map.insert_unchecked(Property::BindSecret, self.bind_secret.into_value()); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); + map.insert_unchecked(Property::BaseDn, self.base_dn.into_value_with(opts)); + map.insert_unchecked(Property::BindDn, self.bind_dn.into_value_with(opts)); + map.insert_unchecked(Property::BindSecret, self.bind_secret.into_value_with(opts)); map.insert_unchecked( Property::BindAuthentication, - self.bind_authentication.into_value(), + self.bind_authentication.into_value_with(opts), ); - map.insert_unchecked(Property::FilterLogin, self.filter_login.into_value()); - map.insert_unchecked(Property::FilterMailbox, self.filter_mailbox.into_value()); - map.insert_unchecked(Property::FilterMemberOf, self.filter_member_of.into_value()); - map.insert_unchecked(Property::AttrClass, self.attr_class.into_value()); + map.insert_unchecked( + Property::FilterLogin, + self.filter_login.into_value_with(opts), + ); + map.insert_unchecked( + Property::FilterRecipient, + self.filter_recipient.into_value_with(opts), + ); + map.insert_unchecked( + Property::FilterMemberOf, + self.filter_member_of.into_value_with(opts), + ); + map.insert_unchecked(Property::AttrClass, self.attr_class.into_value_with(opts)); map.insert_unchecked( Property::AttrDescription, - self.attr_description.into_value(), + self.attr_description.into_value_with(opts), ); - map.insert_unchecked(Property::AttrEmail, self.attr_email.into_value()); - map.insert_unchecked(Property::AttrEmailAlias, self.attr_email_alias.into_value()); - map.insert_unchecked(Property::AttrMemberOf, self.attr_member_of.into_value()); - map.insert_unchecked(Property::AttrSecret, self.attr_secret.into_value()); + map.insert_unchecked(Property::AttrEmail, self.attr_email.into_value_with(opts)); + map.insert_unchecked( + Property::AttrEmailAlias, + self.attr_email_alias.into_value_with(opts), + ); + map.insert_unchecked( + Property::AttrMemberOf, + self.attr_member_of.into_value_with(opts), + ); + map.insert_unchecked(Property::AttrSecret, self.attr_secret.into_value_with(opts)); map.insert_unchecked( Property::AttrSecretChanged, - self.attr_secret_changed.into_value(), + self.attr_secret_changed.into_value_with(opts), ); - map.insert_unchecked(Property::GroupClass, self.group_class.into_value()); + map.insert_unchecked(Property::GroupClass, self.group_class.into_value_with(opts)); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutCreate, - self.pool_timeout_create.into_value(), + self.pool_timeout_create.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutRecycle, - self.pool_timeout_recycle.into_value(), + self.pool_timeout_recycle.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutWait, - self.pool_timeout_wait.into_value(), + self.pool_timeout_wait.into_value_with(opts), ); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); JmapValue::Object(map) } } @@ -24030,8 +30525,8 @@ impl RegistryJsonPropertyPatch for LdapDirectory { Some(Property::FilterLogin) => self .filter_login .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::FilterMailbox) => self - .filter_mailbox + Some(Property::FilterRecipient) => self + .filter_recipient .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::FilterMemberOf) => self .filter_member_of @@ -24064,9 +30559,137 @@ impl RegistryJsonPropertyPatch for LdapDirectory { Some(Property::PoolTimeoutCreate) => self.pool_timeout_create.patch(pointer, value), Some(Property::PoolTimeoutRecycle) => self.pool_timeout_recycle.patch(pointer, value), Some(Property::PoolTimeoutWait) => self.pool_timeout_wait.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl LocalFileSwap { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.path; + if value.is_empty() { + errors.push(ValidationError::required(Property::Path)); + } + let value = &self.flush_changes; + if *value < 1 { + errors.push(ValidationError::min_value(Property::FlushChanges, 1)); + } + let value = &self.max_account_size; + if *value < 2048 { + errors.push(ValidationError::min_value(Property::MaxAccountSize, 2048)); + } + errors.len() == neb + } +} + +impl Pickle for LocalFileSwap { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); + self.path.pickle(out); + self.flush_changes.pickle(out); + self.flush_interval.pickle(out); + self.max_account_size.pickle(out); + self.min_account_size.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2000u64 + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 1073741824 + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + stream.close_block(block); + Some(Self { + path: field0, + flush_changes: field1, + flush_interval: field2, + max_account_size: field3, + min_account_size: field4, + }) + } +} + +impl Default for LocalFileSwap { + fn default() -> Self { + Self { + path: Default::default(), + flush_changes: 2000u64, + flush_interval: Duration::from_millis(600000), + max_account_size: 1073741824, + min_account_size: 1048576, + } + } +} + +impl IntoValue for LocalFileSwap { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(7); + map.insert_unchecked(Property::Path, self.path.into_value_with(opts)); + map.insert_unchecked( + Property::FlushChanges, + self.flush_changes.into_value_with(opts), + ); + map.insert_unchecked( + Property::FlushInterval, + self.flush_interval.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAccountSize, + self.max_account_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinAccountSize, + self.min_account_size.into_value_with(opts), + ); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for LocalFileSwap { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Path) => self + .path + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::FlushChanges) => self.flush_changes.patch(pointer, value), + Some(Property::FlushInterval) => self.flush_interval.patch(pointer, value), + Some(Property::MaxAccountSize) => self.max_account_size.patch(pointer, value), + Some(Property::MinAccountSize) => self.min_account_size.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -24077,7 +30700,7 @@ impl RegistryJsonPropertyPatch for LdapDirectory { } impl ObjectImpl for Log { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED | OBJ_NO_CREATE | OBJ_NO_UPDATE | OBJ_NO_DESTROY; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::Log; @@ -24097,24 +30720,6 @@ impl ObjectImpl for Log { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for Log { - fn pickle(&self, out: &mut Vec) { - self.timestamp.pickle(out); - self.level.pickle(out); - self.event.pickle(out); - self.details.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.timestamp = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.event = Pickle::unpickle(stream)?; - this.details = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for Log { fn default() -> Self { Self { @@ -24128,11 +30733,15 @@ impl Default for Log { impl IntoValue for Log { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Timestamp, self.timestamp.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Event, self.event.into_value()); - map.insert_unchecked(Property::Details, self.details.into_value()); + map.insert_unchecked(Property::Timestamp, self.timestamp.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Event, self.event.into_value_with(opts)); + map.insert_unchecked(Property::Details, self.details.into_value_with(opts)); JmapValue::Object(map) } } @@ -24163,12 +30772,12 @@ impl LookupStore { LookupStore::PostgreSql(inner) => inner.validate(errors), LookupStore::MySql(inner) => inner.validate(errors), LookupStore::Sqlite(inner) => inner.validate(errors), - LookupStore::Sharded(inner) => inner.validate(errors), LookupStore::Redis(inner) => inner.validate(errors), LookupStore::RedisCluster(inner) => inner.validate(errors), LookupStore::RedisSentinel(inner) => inner.validate(errors), } } + } impl Default for LookupStore { @@ -24192,10 +30801,6 @@ impl Pickle for LookupStore { 2u16.pickle(out); inner.pickle(out); } - LookupStore::Sharded(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } LookupStore::Redis(inner) => { 4u16.pickle(out); inner.pickle(out); @@ -24216,7 +30821,6 @@ impl Pickle for LookupStore { 0 => Pickle::unpickle(stream).map(LookupStore::PostgreSql), 1 => Pickle::unpickle(stream).map(LookupStore::MySql), 2 => Pickle::unpickle(stream).map(LookupStore::Sqlite), - 3 => Pickle::unpickle(stream).map(LookupStore::Sharded), 4 => Pickle::unpickle(stream).map(LookupStore::Redis), 5 => Pickle::unpickle(stream).map(LookupStore::RedisCluster), 6 => Pickle::unpickle(stream).map(LookupStore::RedisSentinel), @@ -24227,51 +30831,48 @@ impl Pickle for LookupStore { impl IntoValue for LookupStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { LookupStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } LookupStore::MySql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); obj } LookupStore::Sqlite(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Sqlite".into())); obj } - LookupStore::Sharded(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Sharded".into())); - obj - } LookupStore::Redis(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); obj } LookupStore::RedisCluster(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisCluster".into())); obj } LookupStore::RedisSentinel(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("RedisSentinel".into())); @@ -24292,7 +30893,6 @@ impl RegistryJsonPatch for LookupStore { LookupStoreType::PostgreSql => *self = LookupStore::PostgreSql(Default::default()), LookupStoreType::MySql => *self = LookupStore::MySql(Default::default()), LookupStoreType::Sqlite => *self = LookupStore::Sqlite(Default::default()), - LookupStoreType::Sharded => *self = LookupStore::Sharded(Default::default()), LookupStoreType::Redis => *self = LookupStore::Redis(Default::default()), LookupStoreType::RedisCluster => { *self = LookupStore::RedisCluster(Default::default()) @@ -24306,7 +30906,6 @@ impl RegistryJsonPatch for LookupStore { LookupStore::PostgreSql(inner) => inner.patch(pointer, value), LookupStore::MySql(inner) => inner.patch(pointer, value), LookupStore::Sqlite(inner) => inner.patch(pointer, value), - LookupStore::Sharded(inner) => inner.patch(pointer, value), LookupStore::Redis(inner) => inner.patch(pointer, value), LookupStore::RedisCluster(inner) => inner.patch(pointer, value), LookupStore::RedisSentinel(inner) => inner.patch(pointer, value), @@ -24320,7 +30919,6 @@ impl LookupStore { LookupStore::PostgreSql(_) => LookupStoreType::PostgreSql, LookupStore::MySql(_) => LookupStoreType::MySql, LookupStore::Sqlite(_) => LookupStoreType::Sqlite, - LookupStore::Sharded(_) => LookupStoreType::Sharded, LookupStore::Redis(_) => LookupStoreType::Redis, LookupStore::RedisCluster(_) => LookupStoreType::RedisCluster, LookupStore::RedisSentinel(_) => LookupStoreType::RedisSentinel, @@ -24349,15 +30947,39 @@ impl MailExchanger { impl Pickle for MailExchanger { fn pickle(&self, out: &mut Vec) { - self.hostname.pickle(out); + let mut bitmap = 0u64; + if self.hostname.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); + if let Some(value) = &self.hostname { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.hostname = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + stream.close_block(block); + Some(Self { + hostname: field0, + priority: field1, + }) } } @@ -24372,9 +30994,13 @@ impl Default for MailExchanger { impl IntoValue for MailExchanger { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Hostname, self.hostname.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Hostname, self.hostname.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -24423,11 +31049,6 @@ impl ObjectImpl for MailingList { for value in value.values() { value.validate(errors); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.recipients; for value in value.iter() { if value.is_empty() { @@ -24438,7 +31059,11 @@ impl ObjectImpl for MailingList { } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.unique_global_composite(Property::Email, &self.name, &self.domain_id); + i.unique_global_composite( + Property::Email, + &self.name, + IndexValue::U32(self.domain_id.document_id()), + ); i.text(Property::Text, &self.name); i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); if let Some(value) = &self.description { @@ -24447,10 +31072,6 @@ impl ObjectImpl for MailingList { for item in self.aliases.values() { item.index(i); } - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } for value in self.recipients.iter() { i.text(Property::Text, value); } @@ -24459,23 +31080,63 @@ impl ObjectImpl for MailingList { impl Pickle for MailingList { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 6, 2, bitmap); self.name.pickle(out); self.domain_id.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.aliases.pickle(out); - self.member_tenant_id.pickle(out); self.recipients.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.aliases = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.recipients = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: List = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Map = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + domain_id: field1, + description: field2, + aliases: field3, + recipients: field5, + }) } } @@ -24486,7 +31147,6 @@ impl Default for MailingList { domain_id: Default::default(), description: Default::default(), aliases: Default::default(), - member_tenant_id: Default::default(), recipients: Default::default(), } } @@ -24494,13 +31154,19 @@ impl Default for MailingList { impl IntoValue for MailingList { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Aliases, self.aliases.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Recipients, self.recipients.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Aliases, self.aliases.into_value_with(opts)); + map.insert_unchecked(Property::Recipients, self.recipients.into_value_with(opts)); JmapValue::Object(map) } } @@ -24520,9 +31186,6 @@ impl RegistryJsonPropertyPatch for MailingList { Some(Property::EmailAddress) => pointer.assert_server_set(), Some(Property::Description) => self.description.patch(pointer, value), Some(Property::Aliases) => self.aliases.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Recipients) => self .recipients .patch(pointer.with_validators(&[StringValidator::Email]), value), @@ -24535,152 +31198,6 @@ impl RegistryJsonPropertyPatch for MailingList { } } -impl ObjectImpl for MaskedEmail { - const FLAGS: u64 = OBJ_FILTER_ACCOUNT; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::MaskedEmail; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.email; - if value.is_empty() { - errors.push(ValidationError::required(Property::Email)); - } - if let Some(value) = &self.description { - if value.is_empty() { - errors.push(ValidationError::required(Property::Description)); - } - } - if let Some(value) = &self.for_domain { - if value.is_empty() { - errors.push(ValidationError::required(Property::ForDomain)); - } - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - if let Some(value) = &self.created_by { - if value.is_empty() { - errors.push(ValidationError::required(Property::CreatedBy)); - } - } - if let Some(value) = &self.expires_at { - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ExpiresAt, value)); - } - } - if let Some(value) = &self.url { - if value.is_empty() { - errors.push(ValidationError::required(Property::Url)); - } - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - i.search(Property::AccountId, &self.account_id); - } -} - -impl Pickle for MaskedEmail { - fn pickle(&self, out: &mut Vec) { - self.enabled.pickle(out); - self.account_id.pickle(out); - self.email.pickle(out); - self.description.pickle(out); - self.for_domain.pickle(out); - self.created_at.pickle(out); - self.created_by.pickle(out); - self.expires_at.pickle(out); - self.url.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enabled = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.email = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.for_domain = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.created_by = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for MaskedEmail { - fn default() -> Self { - Self { - enabled: true, - account_id: Default::default(), - email: Default::default(), - description: Default::default(), - for_domain: Default::default(), - created_at: Default::default(), - created_by: Default::default(), - expires_at: Default::default(), - url: Default::default(), - } - } -} - -impl IntoValue for MaskedEmail { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Enabled, self.enabled.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::Email, self.email.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::ForDomain, self.for_domain.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::CreatedBy, self.created_by.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for MaskedEmail { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Enabled) => self.enabled.patch(pointer, value), - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), - Some(Property::Email) => pointer.assert_server_set(), - Some(Property::Description) => self.description.patch(pointer, value), - Some(Property::ForDomain) => self.for_domain.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::CreatedBy) => self.created_by.patch(pointer, value), - Some(Property::ExpiresAt) => self.expires_at.patch(pointer.assert_read_only()?, value), - Some(Property::Url) => self.url.patch(pointer, value), - Some(property @ Property::EmailPrefix) => { - Ok(MaybeUnpatched::Unpatched { property, value }) - } - Some(property @ Property::EmailDomain) => { - Ok(MaybeUnpatched::Unpatched { property, value }) - } - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl MeilisearchStore { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -24688,12 +31205,12 @@ impl MeilisearchStore { if value.is_empty() { errors.push(ValidationError::required(Property::Url)); } - let value = &self.max_retries; + let value = &self.max_polls; if *value > 1024 { - errors.push(ValidationError::max_value(Property::MaxRetries, 1024)); + errors.push(ValidationError::max_value(Property::MaxPolls, 1024)); } if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxRetries, 1)); + errors.push(ValidationError::min_value(Property::MaxPolls, 1)); } let value = &self.http_auth; value.validate(errors); @@ -24705,31 +31222,73 @@ impl MeilisearchStore { } errors.len() == neb } + } impl Pickle for MeilisearchStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.fail_on_timeout { + bitmap |= 1 << 0; + } + if self.allow_invalid_certs { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.url.pickle(out); self.poll_interval.pickle(out); - self.max_retries.pickle(out); - self.fail_on_timeout.pickle(out); + self.max_polls.pickle(out); self.timeout.pickle(out); - self.allow_invalid_certs.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.url = Pickle::unpickle(stream)?; - this.poll_interval = Pickle::unpickle(stream)?; - this.max_retries = Pickle::unpickle(stream)?; - this.fail_on_timeout = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(500) + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 120u64 + }; + let field3: bool = if block.has(3) { block.flag(0) } else { true }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + let field6: HttpAuth = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + poll_interval: field1, + max_polls: field2, + fail_on_timeout: field3, + timeout: field4, + allow_invalid_certs: field5, + http_auth: field6, + http_headers: field7, + }) } } @@ -24738,7 +31297,7 @@ impl Default for MeilisearchStore { Self { url: Default::default(), poll_interval: Duration::from_millis(500), - max_retries: 120u64, + max_polls: 120u64, fail_on_timeout: true, timeout: Duration::from_millis(30000), allow_invalid_certs: false, @@ -24750,18 +31309,31 @@ impl Default for MeilisearchStore { impl IntoValue for MeilisearchStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::PollInterval, self.poll_interval.into_value()); - map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value()); - map.insert_unchecked(Property::FailOnTimeout, self.fail_on_timeout.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked( + Property::PollInterval, + self.poll_interval.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxPolls, self.max_polls.into_value_with(opts)); + map.insert_unchecked( + Property::FailOnTimeout, + self.fail_on_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), ); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); JmapValue::Object(map) } } @@ -24777,7 +31349,7 @@ impl RegistryJsonPropertyPatch for MeilisearchStore { .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::PollInterval) => self.poll_interval.patch(pointer, value), - Some(Property::MaxRetries) => self.max_retries.patch(pointer, value), + Some(Property::MaxPolls) => self.max_polls.patch(pointer, value), Some(Property::FailOnTimeout) => self.fail_on_timeout.patch(pointer, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), @@ -24823,17 +31395,38 @@ impl ObjectImpl for MemoryLookupKey { impl Pickle for MemoryLookupKey { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.is_glob_pattern { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 3, 1, bitmap); self.namespace.pickle(out); self.key.pickle(out); - self.is_glob_pattern.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.namespace = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.is_glob_pattern = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(0) } else { false }; + stream.close_block(block); + Some(Self { + namespace: field0, + key: field1, + is_glob_pattern: field2, + }) } } @@ -24849,10 +31442,17 @@ impl Default for MemoryLookupKey { impl IntoValue for MemoryLookupKey { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Namespace, self.namespace.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::IsGlobPattern, self.is_glob_pattern.into_value()); + map.insert_unchecked(Property::Namespace, self.namespace.into_value_with(opts)); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked( + Property::IsGlobPattern, + self.is_glob_pattern.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -24909,19 +31509,45 @@ impl ObjectImpl for MemoryLookupKeyValue { impl Pickle for MemoryLookupKeyValue { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.is_glob_pattern { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.namespace.pickle(out); self.key.pickle(out); self.value.pickle(out); - self.is_glob_pattern.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.namespace = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.value = Pickle::unpickle(stream)?; - this.is_glob_pattern = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + stream.close_block(block); + Some(Self { + namespace: field0, + key: field1, + value: field2, + is_glob_pattern: field3, + }) } } @@ -24938,11 +31564,18 @@ impl Default for MemoryLookupKeyValue { impl IntoValue for MemoryLookupKeyValue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Namespace, self.namespace.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Value, self.value.into_value()); - map.insert_unchecked(Property::IsGlobPattern, self.is_glob_pattern.into_value()); + map.insert_unchecked(Property::Namespace, self.namespace.into_value_with(opts)); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked(Property::Value, self.value.into_value_with(opts)); + map.insert_unchecked( + Property::IsGlobPattern, + self.is_glob_pattern.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -24967,229 +31600,271 @@ impl RegistryJsonPropertyPatch for MemoryLookupKeyValue { } } -impl ObjectImpl for Metric { - const FLAGS: u64 = 0; +impl ObjectImpl for Metadata { + const FLAGS: u64 = OBJ_SINGLETON; const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::Metric; + const OBJECT: ObjectType = ObjectType::Metadata; fn validate(&self, errors: &mut Vec) -> bool { - match self { - Metric::Counter(inner) => inner.validate(errors), - Metric::Gauge(inner) => inner.validate(errors), - Metric::Histogram(inner) => inner.validate(errors), + let neb = errors.len(); + if let Some(value) = &self.max_depth { + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxDepth, 1)); + } } + let value = &self.max_entry_size; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxEntrySize, 1024)); + } + let value = &self.max_size; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxSize, 1024)); + } + let value = &self.max_private_size; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxPrivateSize, 1024)); + } + let value = &self.max_entries; + if *value < 10 { + errors.push(ValidationError::min_value(Property::MaxEntries, 10)); + } + let value = &self.max_query_scan; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxQueryScan, 1)); + } + if let Some(value) = &self.imap_server_comment { + if value.is_empty() { + errors.push(ValidationError::required(Property::ImapServerComment)); + } + } + if let Some(value) = &self.imap_server_admin { + if value.is_empty() { + errors.push(ValidationError::required(Property::ImapServerAdmin)); + } + } + errors.len() == neb } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Default for Metric { - fn default() -> Self { - Metric::Counter(Default::default()) - } -} - -impl Pickle for Metric { +impl Pickle for Metadata { fn pickle(&self, out: &mut Vec) { - match self { - Metric::Counter(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - Metric::Gauge(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - Metric::Histogram(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } + let mut bitmap = 0u64; + if self.vendor_namespaces { + bitmap |= 1 << 0; } + if self.private_metadata { + bitmap |= 1 << 1; + } + if self.max_depth.is_some() { + bitmap |= 1 << 2; + } + if self.imap_server_comment.is_some() { + bitmap |= 1 << 3; + } + if self.imap_server_admin.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 11, 5, bitmap); + self.data_types.pickle(out); + if let Some(value) = &self.max_depth { + value.pickle(out); + } + self.max_entry_size.pickle(out); + self.max_size.pickle(out); + self.max_private_size.pickle(out); + self.max_entries.pickle(out); + self.max_query_scan.pickle(out); + if let Some(value) = &self.imap_server_comment { + value.pickle(out); + } + if let Some(value) = &self.imap_server_admin { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(Metric::Counter), - 1 => Pickle::unpickle(stream).map(Metric::Gauge), - 2 => Pickle::unpickle(stream).map(Metric::Histogram), - _ => None, + if stream.version() > Self::VERSION { + return None; } - } -} - -impl IntoValue for Metric { - fn into_value(self) -> JmapValue<'static> { - match self { - Metric::Counter(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Counter".into())); - obj + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + MetadataDataType::Email, + MetadataDataType::Mailbox, + MetadataDataType::SieveScript, + MetadataDataType::Calendar, + MetadataDataType::CalendarEvent, + MetadataDataType::AddressBook, + MetadataDataType::ContactCard, + MetadataDataType::FileNode, + ]) + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: bool = if block.has(2) { block.flag(1) } else { true }; + let field3: Option = if block.has(3) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None } - Metric::Gauge(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Gauge".into())); - obj + } else { + Some(8u64) + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 8192u64 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 65536u64 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 16384u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 128u64 + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 50000u64 + }; + let field9: Option = if block.has(9) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None } - Metric::Histogram(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Histogram".into())); - obj + } else { + Default::default() + }; + let field10: Option = if block.has(10) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None } - } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + data_types: field0, + vendor_namespaces: field1, + private_metadata: field2, + max_depth: field3, + max_entry_size: field4, + max_size: field5, + max_private_size: field6, + max_entries: field7, + max_query_scan: field8, + imap_server_comment: field9, + imap_server_admin: field10, + }) } } -impl RegistryJsonPatch for Metric { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - MetricType::Counter => *self = Metric::Counter(Default::default()), - MetricType::Gauge => *self = Metric::Gauge(Default::default()), - MetricType::Histogram => *self = Metric::Histogram(Default::default()), - } - } - match self { - Metric::Counter(inner) => inner.patch(pointer, value), - Metric::Gauge(inner) => inner.patch(pointer, value), - Metric::Histogram(inner) => inner.patch(pointer, value), - } - } -} - -impl Metric { - pub fn object_type(&self) -> MetricType { - match self { - Metric::Counter(_) => MetricType::Counter, - Metric::Gauge(_) => MetricType::Gauge, - Metric::Histogram(_) => MetricType::Histogram, - } - } -} - -impl MetricCount { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for MetricCount { - fn pickle(&self, out: &mut Vec) { - self.count.pickle(out); - self.metric.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.count = Pickle::unpickle(stream)?; - this.metric = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for MetricCount { +impl Default for Metadata { fn default() -> Self { Self { - count: 0u64, - metric: Default::default(), + data_types: Map::new(vec![ + MetadataDataType::Email, + MetadataDataType::Mailbox, + MetadataDataType::SieveScript, + MetadataDataType::Calendar, + MetadataDataType::CalendarEvent, + MetadataDataType::AddressBook, + MetadataDataType::ContactCard, + MetadataDataType::FileNode, + ]), + vendor_namespaces: true, + private_metadata: true, + max_depth: Some(8u64), + max_entry_size: 8192u64, + max_size: 65536u64, + max_private_size: 16384u64, + max_entries: 128u64, + max_query_scan: 50000u64, + imap_server_comment: Default::default(), + imap_server_admin: Default::default(), } } } -impl IntoValue for MetricCount { +impl IntoValue for Metadata { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Count, self.count.into_value()); - map.insert_unchecked(Property::Metric, self.metric.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(13); + map.insert_unchecked(Property::DataTypes, self.data_types.into_value_with(opts)); + map.insert_unchecked( + Property::VendorNamespaces, + self.vendor_namespaces.into_value_with(opts), + ); + map.insert_unchecked( + Property::PrivateMetadata, + self.private_metadata.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxDepth, self.max_depth.into_value_with(opts)); + map.insert_unchecked( + Property::MaxEntrySize, + self.max_entry_size.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxSize, self.max_size.into_value_with(opts)); + map.insert_unchecked( + Property::MaxPrivateSize, + self.max_private_size.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxEntries, self.max_entries.into_value_with(opts)); + map.insert_unchecked( + Property::MaxQueryScan, + self.max_query_scan.into_value_with(opts), + ); + map.insert_unchecked( + Property::ImapServerComment, + self.imap_server_comment.into_value_with(opts), + ); + map.insert_unchecked( + Property::ImapServerAdmin, + self.imap_server_admin.into_value_with(opts), + ); JmapValue::Object(map) } } -impl RegistryJsonPropertyPatch for MetricCount { +impl RegistryJsonPropertyPatch for Metadata { fn patch_property<'x>( &mut self, mut pointer: JsonPointerPatch<'_>, value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Count) => self.count.patch(pointer, value), - Some(Property::Metric) => self.metric.patch(pointer, value), - Some(property @ Property::Timestamp) => { - Ok(MaybeUnpatched::Unpatched { property, value }) - } - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl MetricSum { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for MetricSum { - fn pickle(&self, out: &mut Vec) { - self.count.pickle(out); - self.sum.pickle(out); - self.metric.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.count = Pickle::unpickle(stream)?; - this.sum = Pickle::unpickle(stream)?; - this.metric = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for MetricSum { - fn default() -> Self { - Self { - count: 0u64, - sum: 0u64, - metric: Default::default(), - } - } -} - -impl IntoValue for MetricSum { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Count, self.count.into_value()); - map.insert_unchecked(Property::Sum, self.sum.into_value()); - map.insert_unchecked(Property::Metric, self.metric.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for MetricSum { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Count) => self.count.patch(pointer, value), - Some(Property::Sum) => self.sum.patch(pointer, value), - Some(Property::Metric) => self.metric.patch(pointer, value), - Some(property @ Property::Timestamp) => { - Ok(MaybeUnpatched::Unpatched { property, value }) - } + Some(Property::DataTypes) => self.data_types.patch(pointer, value), + Some(Property::VendorNamespaces) => self.vendor_namespaces.patch(pointer, value), + Some(Property::PrivateMetadata) => self.private_metadata.patch(pointer, value), + Some(Property::MaxDepth) => self.max_depth.patch(pointer, value), + Some(Property::MaxEntrySize) => self.max_entry_size.patch(pointer, value), + Some(Property::MaxSize) => self.max_size.patch(pointer, value), + Some(Property::MaxPrivateSize) => self.max_private_size.patch(pointer, value), + Some(Property::MaxEntries) => self.max_entries.patch(pointer, value), + Some(Property::MaxQueryScan) => self.max_query_scan.patch(pointer, value), + Some(Property::ImapServerComment) => self.imap_server_comment.patch(pointer, value), + Some(Property::ImapServerAdmin) => self + .imap_server_admin + .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -25214,23 +31889,51 @@ impl ObjectImpl for Metrics { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Pickle for Metrics { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.open_telemetry.pickle(out); self.prometheus.pickle(out); self.metrics.pickle(out); self.metrics_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.open_telemetry = Pickle::unpickle(stream)?; - this.prometheus = Pickle::unpickle(stream)?; - this.metrics = Pickle::unpickle(stream)?; - this.metrics_policy = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: MetricsOtel = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: MetricsPrometheus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: EventPolicy = if block.has(3) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + open_telemetry: field0, + prometheus: field1, + metrics: field2, + metrics_policy: field3, + }) } } @@ -25247,11 +31950,21 @@ impl Default for Metrics { impl IntoValue for Metrics { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::OpenTelemetry, self.open_telemetry.into_value()); - map.insert_unchecked(Property::Prometheus, self.prometheus.into_value()); - map.insert_unchecked(Property::Metrics, self.metrics.into_value()); - map.insert_unchecked(Property::MetricsPolicy, self.metrics_policy.into_value()); + map.insert_unchecked( + Property::OpenTelemetry, + self.open_telemetry.into_value_with(opts), + ); + map.insert_unchecked(Property::Prometheus, self.prometheus.into_value_with(opts)); + map.insert_unchecked(Property::Metrics, self.metrics.into_value_with(opts)); + map.insert_unchecked( + Property::MetricsPolicy, + self.metrics_policy.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -25284,6 +31997,7 @@ impl MetricsOtel { MetricsOtel::Grpc(inner) => inner.validate(errors), } } + } impl Default for MetricsOtel { @@ -25321,6 +32035,10 @@ impl Pickle for MetricsOtel { impl IntoValue for MetricsOtel { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { MetricsOtel::Disabled => { let mut obj = jmap_tools::Map::new(); @@ -25328,14 +32046,14 @@ impl IntoValue for MetricsOtel { JmapValue::Object(obj) } MetricsOtel::Http(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Http".into())); obj } MetricsOtel::Grpc(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Grpc".into())); @@ -25379,9 +32097,9 @@ impl MetricsOtel { impl MetricsOtelGrpc { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - if let Some(value) = &self.endpoint { + if let Some(value) = &self.url { if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } } errors.len() == neb @@ -25390,24 +32108,53 @@ impl MetricsOtelGrpc { impl Pickle for MetricsOtelGrpc { fn pickle(&self, out: &mut Vec) { - self.endpoint.pickle(out); + let mut bitmap = 0u64; + if self.url.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 3, 1, bitmap); + if let Some(value) = &self.url { + value.pickle(out); + } self.interval.pickle(out); self.timeout.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.endpoint = Pickle::unpickle(stream)?; - this.interval = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + stream.close_block(block); + Some(Self { + url: field0, + interval: field1, + timeout: field2, + }) } } impl Default for MetricsOtelGrpc { fn default() -> Self { Self { - endpoint: Default::default(), + url: Default::default(), interval: Duration::from_millis(60000), timeout: Duration::from_millis(10000), } @@ -25416,10 +32163,14 @@ impl Default for MetricsOtelGrpc { impl IntoValue for MetricsOtelGrpc { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); - map.insert_unchecked(Property::Interval, self.interval.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Interval, self.interval.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); JmapValue::Object(map) } } @@ -25431,8 +32182,8 @@ impl RegistryJsonPropertyPatch for MetricsOtelGrpc { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Interval) => self.interval.patch(pointer, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), @@ -25448,9 +32199,9 @@ impl RegistryJsonPropertyPatch for MetricsOtelGrpc { impl MetricsOtelHttp { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.endpoint; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } let value = &self.http_auth; value.validate(errors); @@ -25462,34 +32213,71 @@ impl MetricsOtelHttp { } errors.len() == neb } + } impl Pickle for MetricsOtelHttp { fn pickle(&self, out: &mut Vec) { - self.endpoint.pickle(out); + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 6, 1, bitmap); + self.url.pickle(out); self.interval.pickle(out); self.timeout.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.endpoint = Pickle::unpickle(stream)?; - this.interval = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + let field4: HttpAuth = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: VecMap = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + interval: field1, + timeout: field2, + allow_invalid_certs: field3, + http_auth: field4, + http_headers: field5, + }) } } impl Default for MetricsOtelHttp { fn default() -> Self { Self { - endpoint: Default::default(), + url: Default::default(), interval: Duration::from_millis(60000), timeout: Duration::from_millis(10000), + allow_invalid_certs: false, http_auth: Default::default(), http_headers: Default::default(), } @@ -25498,12 +32286,23 @@ impl Default for MetricsOtelHttp { impl IntoValue for MetricsOtelHttp { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); - map.insert_unchecked(Property::Interval, self.interval.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(8); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Interval, self.interval.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -25515,11 +32314,12 @@ impl RegistryJsonPropertyPatch for MetricsOtelHttp { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Interval) => self.interval.patch(pointer, value), Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), Some(Property::HttpHeaders) => self .http_headers @@ -25540,6 +32340,7 @@ impl MetricsPrometheus { MetricsPrometheus::Enabled(inner) => inner.validate(errors), } } + } impl Default for MetricsPrometheus { @@ -25572,6 +32373,10 @@ impl Pickle for MetricsPrometheus { impl IntoValue for MetricsPrometheus { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { MetricsPrometheus::Disabled => { let mut obj = jmap_tools::Map::new(); @@ -25579,7 +32384,7 @@ impl IntoValue for MetricsPrometheus { JmapValue::Object(obj) } MetricsPrometheus::Enabled(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Enabled".into())); @@ -25631,19 +32436,44 @@ impl MetricsPrometheusProperties { } errors.len() == neb } + } impl Pickle for MetricsPrometheusProperties { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.auth_username.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); self.auth_secret.pickle(out); - self.auth_username.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_secret = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: SecretKeyOptional = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + auth_secret: field0, + auth_username: field1, + }) } } @@ -25658,9 +32488,16 @@ impl Default for MetricsPrometheusProperties { impl IntoValue for MetricsPrometheusProperties { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -25685,145 +32522,6 @@ impl RegistryJsonPropertyPatch for MetricsPrometheusProperties { } } -impl ObjectImpl for MetricsStore { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::MetricsStore; - - fn validate(&self, errors: &mut Vec) -> bool { - match self { - MetricsStore::Disabled => true, - MetricsStore::Default => true, - MetricsStore::FoundationDb(inner) => inner.validate(errors), - MetricsStore::PostgreSql(inner) => inner.validate(errors), - MetricsStore::MySql(inner) => inner.validate(errors), - } - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Default for MetricsStore { - fn default() -> Self { - MetricsStore::Disabled - } -} - -impl Pickle for MetricsStore { - fn pickle(&self, out: &mut Vec) { - match self { - MetricsStore::Disabled => { - 0u16.pickle(out); - } - MetricsStore::Default => { - 1u16.pickle(out); - } - MetricsStore::FoundationDb(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - MetricsStore::PostgreSql(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - MetricsStore::MySql(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(MetricsStore::Disabled), - 1 => Some(MetricsStore::Default), - 2 => Pickle::unpickle(stream).map(MetricsStore::FoundationDb), - 3 => Pickle::unpickle(stream).map(MetricsStore::PostgreSql), - 4 => Pickle::unpickle(stream).map(MetricsStore::MySql), - _ => None, - } - } -} - -impl IntoValue for MetricsStore { - fn into_value(self) -> JmapValue<'static> { - match self { - MetricsStore::Disabled => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Disabled".into())); - JmapValue::Object(obj) - } - MetricsStore::Default => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); - JmapValue::Object(obj) - } - MetricsStore::FoundationDb(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); - obj - } - MetricsStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); - obj - } - MetricsStore::MySql(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for MetricsStore { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - MetricsStoreType::Disabled => *self = MetricsStore::Disabled, - MetricsStoreType::Default => *self = MetricsStore::Default, - MetricsStoreType::FoundationDb => { - *self = MetricsStore::FoundationDb(Default::default()) - } - MetricsStoreType::PostgreSql => { - *self = MetricsStore::PostgreSql(Default::default()) - } - MetricsStoreType::MySql => *self = MetricsStore::MySql(Default::default()), - } - } - match self { - MetricsStore::Disabled => pointer.assert_eof(), - MetricsStore::Default => pointer.assert_eof(), - MetricsStore::FoundationDb(inner) => inner.patch(pointer, value), - MetricsStore::PostgreSql(inner) => inner.patch(pointer, value), - MetricsStore::MySql(inner) => inner.patch(pointer, value), - } - } -} - -impl MetricsStore { - pub fn object_type(&self) -> MetricsStoreType { - match self { - MetricsStore::Disabled => MetricsStoreType::Disabled, - MetricsStore::Default => MetricsStoreType::Default, - MetricsStore::FoundationDb(_) => MetricsStoreType::FoundationDb, - MetricsStore::PostgreSql(_) => MetricsStoreType::PostgreSql, - MetricsStore::MySql(_) => MetricsStoreType::MySql, - } - } -} - impl MtaConnectionIpHost { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -25832,9 +32530,9 @@ impl MtaConnectionIpHost { errors.push(ValidationError::required(Property::EhloHostname)); } } - let value = &self.source_ip; + let value = &self.local_ip; if !value.is_valid() { - errors.push(ValidationError::invalid(Property::SourceIp, value)); + errors.push(ValidationError::invalid(Property::LocalIp, value)); } errors.len() == neb } @@ -25842,15 +32540,39 @@ impl MtaConnectionIpHost { impl Pickle for MtaConnectionIpHost { fn pickle(&self, out: &mut Vec) { - self.ehlo_hostname.pickle(out); - self.source_ip.pickle(out); + let mut bitmap = 0u64; + if self.ehlo_hostname.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); + if let Some(value) = &self.ehlo_hostname { + value.pickle(out); + } + self.local_ip.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.ehlo_hostname = Pickle::unpickle(stream)?; - this.source_ip = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: IpAddr = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + ehlo_hostname: field0, + local_ip: field1, + }) } } @@ -25858,16 +32580,23 @@ impl Default for MtaConnectionIpHost { fn default() -> Self { Self { ehlo_hostname: Default::default(), - source_ip: Default::default(), + local_ip: Default::default(), } } } impl IntoValue for MtaConnectionIpHost { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::EhloHostname, self.ehlo_hostname.into_value()); - map.insert_unchecked(Property::SourceIp, self.source_ip.into_value()); + map.insert_unchecked( + Property::EhloHostname, + self.ehlo_hostname.into_value_with(opts), + ); + map.insert_unchecked(Property::LocalIp, self.local_ip.into_value_with(opts)); JmapValue::Object(map) } } @@ -25882,7 +32611,7 @@ impl RegistryJsonPropertyPatch for MtaConnectionIpHost { Some(Property::EhloHostname) => self .ehlo_hostname .patch(pointer.with_validators(&[StringValidator::Hostname]), value), - Some(Property::SourceIp) => self.source_ip.patch(pointer, value), + Some(Property::LocalIp) => self.local_ip.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -25913,7 +32642,7 @@ impl ObjectImpl for MtaConnectionStrategy { errors.push(ValidationError::required(Property::EhloHostname)); } } - let value = &self.source_ips; + let value = &self.local_ips; for value in value.values() { value.validate(errors); } @@ -25927,31 +32656,107 @@ impl ObjectImpl for MtaConnectionStrategy { impl Pickle for MtaConnectionStrategy { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.ehlo_hostname.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 10, 2, bitmap); self.name.pickle(out); - self.description.pickle(out); - self.ehlo_hostname.pickle(out); - self.source_ips.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + if let Some(value) = &self.ehlo_hostname { + value.pickle(out); + } + self.local_ips.pickle(out); self.connect_timeout.pickle(out); self.data_timeout.pickle(out); self.ehlo_timeout.pickle(out); self.greeting_timeout.pickle(out); self.mail_from_timeout.pickle(out); self.rcpt_to_timeout.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.ehlo_hostname = Pickle::unpickle(stream)?; - this.source_ips = Pickle::unpickle(stream)?; - this.connect_timeout = Pickle::unpickle(stream)?; - this.data_timeout = Pickle::unpickle(stream)?; - this.ehlo_timeout = Pickle::unpickle(stream)?; - this.greeting_timeout = Pickle::unpickle(stream)?; - this.mail_from_timeout = Pickle::unpickle(stream)?; - this.rcpt_to_timeout = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: List = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + ehlo_hostname: field2, + local_ips: field3, + connect_timeout: field4, + data_timeout: field5, + ehlo_timeout: field6, + greeting_timeout: field7, + mail_from_timeout: field8, + rcpt_to_timeout: field9, + }) } } @@ -25961,7 +32766,7 @@ impl Default for MtaConnectionStrategy { name: Default::default(), description: Default::default(), ehlo_hostname: Default::default(), - source_ips: Default::default(), + local_ips: Default::default(), connect_timeout: Duration::from_millis(300000), data_timeout: Duration::from_millis(600000), ehlo_timeout: Duration::from_millis(300000), @@ -25974,23 +32779,45 @@ impl Default for MtaConnectionStrategy { impl IntoValue for MtaConnectionStrategy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::EhloHostname, self.ehlo_hostname.into_value()); - map.insert_unchecked(Property::SourceIps, self.source_ips.into_value()); - map.insert_unchecked(Property::ConnectTimeout, self.connect_timeout.into_value()); - map.insert_unchecked(Property::DataTimeout, self.data_timeout.into_value()); - map.insert_unchecked(Property::EhloTimeout, self.ehlo_timeout.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::EhloHostname, + self.ehlo_hostname.into_value_with(opts), + ); + map.insert_unchecked(Property::LocalIps, self.local_ips.into_value_with(opts)); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::DataTimeout, + self.data_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::EhloTimeout, + self.ehlo_timeout.into_value_with(opts), + ); map.insert_unchecked( Property::GreetingTimeout, - self.greeting_timeout.into_value(), + self.greeting_timeout.into_value_with(opts), ); map.insert_unchecked( Property::MailFromTimeout, - self.mail_from_timeout.into_value(), + self.mail_from_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::RcptToTimeout, + self.rcpt_to_timeout.into_value_with(opts), ); - map.insert_unchecked(Property::RcptToTimeout, self.rcpt_to_timeout.into_value()); JmapValue::Object(map) } } @@ -26007,7 +32834,7 @@ impl RegistryJsonPropertyPatch for MtaConnectionStrategy { Some(Property::EhloHostname) => self .ehlo_hostname .patch(pointer.with_validators(&[StringValidator::Hostname]), value), - Some(Property::SourceIps) => self.source_ips.patch(pointer, value), + Some(Property::LocalIps) => self.local_ips.patch(pointer, value), Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), Some(Property::DataTimeout) => self.data_timeout.patch(pointer, value), Some(Property::EhloTimeout) => self.ehlo_timeout.patch(pointer, value), @@ -26063,16 +32890,20 @@ impl Pickle for MtaDeliveryExpiration { impl IntoValue for MtaDeliveryExpiration { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { MtaDeliveryExpiration::Ttl(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ttl".into())); obj } MtaDeliveryExpiration::Attempts(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Attempts".into())); @@ -26127,13 +32958,22 @@ impl MtaDeliveryExpirationAttempts { impl Pickle for MtaDeliveryExpirationAttempts { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.max_attempts.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_attempts = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + stream.close_block(block); + Some(Self { + max_attempts: field0, + }) } } @@ -26145,8 +32985,15 @@ impl Default for MtaDeliveryExpirationAttempts { impl IntoValue for MtaDeliveryExpirationAttempts { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::MaxAttempts, self.max_attempts.into_value()); + map.insert_unchecked( + Property::MaxAttempts, + self.max_attempts.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -26176,13 +33023,20 @@ impl MtaDeliveryExpirationTtl { impl Pickle for MtaDeliveryExpirationTtl { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.expire.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.expire = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(259200000) + }; + stream.close_block(block); + Some(Self { expire: field0 }) } } @@ -26196,8 +33050,12 @@ impl Default for MtaDeliveryExpirationTtl { impl IntoValue for MtaDeliveryExpirationTtl { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Expire, self.expire.into_value()); + map.insert_unchecked(Property::Expire, self.expire.into_value_with(opts)); JmapValue::Object(map) } } @@ -26256,23 +33114,70 @@ impl ObjectImpl for MtaDeliverySchedule { impl Pickle for MtaDeliverySchedule { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 6, 1, bitmap); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.expiry.pickle(out); self.notify.pickle(out); self.queue_id.pickle(out); self.retry.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.expiry = Pickle::unpickle(stream)?; - this.notify = Pickle::unpickle(stream)?; - this.queue_id = Pickle::unpickle(stream)?; - this.retry = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: MtaDeliveryExpiration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: MtaDeliveryScheduleIntervalsOrDefault = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Id = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: MtaDeliveryScheduleIntervalsOrDefault = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + expiry: field2, + notify: field3, + queue_id: field4, + retry: field5, + }) } } @@ -26291,13 +33196,20 @@ impl Default for MtaDeliverySchedule { impl IntoValue for MtaDeliverySchedule { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Expiry, self.expiry.into_value()); - map.insert_unchecked(Property::Notify, self.notify.into_value()); - map.insert_unchecked(Property::QueueId, self.queue_id.into_value()); - map.insert_unchecked(Property::Retry, self.retry.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Expiry, self.expiry.into_value_with(opts)); + map.insert_unchecked(Property::Notify, self.notify.into_value_with(opts)); + map.insert_unchecked(Property::QueueId, self.queue_id.into_value_with(opts)); + map.insert_unchecked(Property::Retry, self.retry.into_value_with(opts)); JmapValue::Object(map) } } @@ -26332,13 +33244,20 @@ impl MtaDeliveryScheduleInterval { impl Pickle for MtaDeliveryScheduleInterval { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.duration.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.duration = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + stream.close_block(block); + Some(Self { duration: field0 }) } } @@ -26352,8 +33271,12 @@ impl Default for MtaDeliveryScheduleInterval { impl IntoValue for MtaDeliveryScheduleInterval { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Duration, self.duration.into_value()); + map.insert_unchecked(Property::Duration, self.duration.into_value_with(opts)); JmapValue::Object(map) } } @@ -26391,13 +33314,20 @@ impl MtaDeliveryScheduleIntervals { impl Pickle for MtaDeliveryScheduleIntervals { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.intervals.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.intervals = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: List = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { intervals: field0 }) } } @@ -26411,8 +33341,12 @@ impl Default for MtaDeliveryScheduleIntervals { impl IntoValue for MtaDeliveryScheduleIntervals { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Intervals, self.intervals.into_value()); + map.insert_unchecked(Property::Intervals, self.intervals.into_value_with(opts)); JmapValue::Object(map) } } @@ -26473,6 +33407,10 @@ impl Pickle for MtaDeliveryScheduleIntervalsOrDefault { impl IntoValue for MtaDeliveryScheduleIntervalsOrDefault { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { MtaDeliveryScheduleIntervalsOrDefault::Default => { let mut obj = jmap_tools::Map::new(); @@ -26480,7 +33418,7 @@ impl IntoValue for MtaDeliveryScheduleIntervalsOrDefault { JmapValue::Object(obj) } MtaDeliveryScheduleIntervalsOrDefault::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); @@ -26726,6 +33664,7 @@ impl MtaExtensions { impl Pickle for MtaExtensions { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 10, 0, 0); self.chunking.pickle(out); self.deliver_by.pickle(out); self.dsn.pickle(out); @@ -26736,21 +33675,125 @@ impl Pickle for MtaExtensions { self.pipelining.pickle(out); self.require_tls.pickle(out); self.vrfy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.chunking = Pickle::unpickle(stream)?; - this.deliver_by = Pickle::unpickle(stream)?; - this.dsn = Pickle::unpickle(stream)?; - this.expn = Pickle::unpickle(stream)?; - this.future_release = Pickle::unpickle(stream)?; - this.mt_priority = Pickle::unpickle(stream)?; - this.no_soliciting = Pickle::unpickle(stream)?; - this.pipelining = Pickle::unpickle(stream)?; - this.require_tls = Pickle::unpickle(stream)?; - this.vrfy = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "15d".to_string(), + }]), + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "true".to_string(), + }]), + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "true".to_string(), + }]), + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "7d".to_string(), + }]), + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "mixer".to_string(), + }]), + } + }; + let field6: Expression = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "''".to_string(), + ..Default::default() + } + }; + let field7: Expression = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field8: Expression = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field9: Expression = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "!is_empty(authenticated_as)".to_string(), + then: "true".to_string(), + }]), + } + }; + stream.close_block(block); + Some(Self { + chunking: field0, + deliver_by: field1, + dsn: field2, + expn: field3, + future_release: field4, + mt_priority: field5, + no_soliciting: field6, + pipelining: field7, + require_tls: field8, + vrfy: field9, + }) } } @@ -26821,17 +33864,27 @@ impl Default for MtaExtensions { impl IntoValue for MtaExtensions { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Chunking, self.chunking.into_value()); - map.insert_unchecked(Property::DeliverBy, self.deliver_by.into_value()); - map.insert_unchecked(Property::Dsn, self.dsn.into_value()); - map.insert_unchecked(Property::Expn, self.expn.into_value()); - map.insert_unchecked(Property::FutureRelease, self.future_release.into_value()); - map.insert_unchecked(Property::MtPriority, self.mt_priority.into_value()); - map.insert_unchecked(Property::NoSoliciting, self.no_soliciting.into_value()); - map.insert_unchecked(Property::Pipelining, self.pipelining.into_value()); - map.insert_unchecked(Property::RequireTls, self.require_tls.into_value()); - map.insert_unchecked(Property::Vrfy, self.vrfy.into_value()); + map.insert_unchecked(Property::Chunking, self.chunking.into_value_with(opts)); + map.insert_unchecked(Property::DeliverBy, self.deliver_by.into_value_with(opts)); + map.insert_unchecked(Property::Dsn, self.dsn.into_value_with(opts)); + map.insert_unchecked(Property::Expn, self.expn.into_value_with(opts)); + map.insert_unchecked( + Property::FutureRelease, + self.future_release.into_value_with(opts), + ); + map.insert_unchecked(Property::MtPriority, self.mt_priority.into_value_with(opts)); + map.insert_unchecked( + Property::NoSoliciting, + self.no_soliciting.into_value_with(opts), + ); + map.insert_unchecked(Property::Pipelining, self.pipelining.into_value_with(opts)); + map.insert_unchecked(Property::RequireTls, self.require_tls.into_value_with(opts)); + map.insert_unchecked(Property::Vrfy, self.vrfy.into_value_with(opts)); JmapValue::Object(map) } } @@ -26887,6 +33940,7 @@ impl ObjectImpl for MtaHook { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl MtaHook { @@ -26910,36 +33964,87 @@ impl MtaHook { impl Pickle for MtaHook { fn pickle(&self, out: &mut Vec) { - self.allow_invalid_certs.pickle(out); + let mut bitmap = 0u64; + if self.temp_fail_on_error { + bitmap |= 1 << 0; + } + if self.allow_invalid_certs { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 9, 2, bitmap); self.enable.pickle(out); self.max_response_size.pickle(out); - self.temp_fail_on_error.pickle(out); self.stages.pickle(out); - self.timeout.pickle(out); self.url.pickle(out); + self.timeout.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.max_response_size = Pickle::unpickle(stream)?; - this.temp_fail_on_error = Pickle::unpickle(stream)?; - this.stages = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 52428800u64 + }; + let field2: bool = if block.has(2) { block.flag(0) } else { true }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![MtaStage::Data]) + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field6: bool = if block.has(6) { block.flag(1) } else { false }; + let field7: HttpAuth = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: VecMap = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + max_response_size: field1, + temp_fail_on_error: field2, + stages: field3, + url: field4, + timeout: field5, + allow_invalid_certs: field6, + http_auth: field7, + http_headers: field8, + }) } } impl Default for MtaHook { fn default() -> Self { Self { - allow_invalid_certs: false, enable: Expression { else_: "true".to_string(), ..Default::default() @@ -26947,8 +34052,9 @@ impl Default for MtaHook { max_response_size: 52428800u64, temp_fail_on_error: true, stages: Map::new(vec![MtaStage::Data]), - timeout: Duration::from_millis(30000), url: Default::default(), + timeout: Duration::from_millis(30000), + allow_invalid_certs: false, http_auth: Default::default(), http_headers: Default::default(), } @@ -26957,25 +34063,32 @@ impl Default for MtaHook { impl IntoValue for MtaHook { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked( - Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), - ); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); map.insert_unchecked( Property::MaxResponseSize, - self.max_response_size.into_value(), + self.max_response_size.into_value_with(opts), ); map.insert_unchecked( Property::TempFailOnError, - self.temp_fail_on_error.into_value(), + self.temp_fail_on_error.into_value_with(opts), + ); + map.insert_unchecked(Property::Stages, self.stages.into_value_with(opts)); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), ); - map.insert_unchecked(Property::Stages, self.stages.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); JmapValue::Object(map) } } @@ -26987,15 +34100,15 @@ impl RegistryJsonPropertyPatch for MtaHook { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::Enable) => self.enable.patch(pointer, value), Some(Property::MaxResponseSize) => self.max_response_size.patch(pointer, value), Some(Property::TempFailOnError) => self.temp_fail_on_error.patch(pointer, value), Some(Property::Stages) => self.stages.patch(pointer, value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::Url) => self .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), Some(Property::HttpHeaders) => self .http_headers @@ -27018,7 +34131,7 @@ impl ObjectImpl for MtaInboundSession { let neb = errors.len(); let value = &self.max_duration; value.validate(errors); - let value = &self.timeout; + let value = &self.idle_timeout; value.validate(errors); let value = &self.transfer_limit; value.validate(errors); @@ -27042,14 +34155,14 @@ impl MtaInboundSession { } } - pub fn ctx_timeout(&self) -> ExpressionContext<'_> { + pub fn ctx_idle_timeout(&self) -> ExpressionContext<'_> { ExpressionContext { - expr: &self.timeout, + expr: &self.idle_timeout, default: Some(Expression { else_: "5m".to_string(), ..Default::default() }), - property: Property::Timeout, + property: Property::IdleTimeout, allowed_variables: MTA_CONNECTION_VARIABLE, allowed_constants: &[], } @@ -27071,7 +34184,7 @@ impl MtaInboundSession { pub fn expression_ctxs(&self) -> Vec> { vec![ self.ctx_max_duration(), - self.ctx_timeout(), + self.ctx_idle_timeout(), self.ctx_transfer_limit(), ] } @@ -27079,17 +34192,48 @@ impl MtaInboundSession { impl Pickle for MtaInboundSession { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.max_duration.pickle(out); - self.timeout.pickle(out); + self.idle_timeout.pickle(out); self.transfer_limit.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_duration = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.transfer_limit = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "10m".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5m".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "262144000".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + max_duration: field0, + idle_timeout: field1, + transfer_limit: field2, + }) } } @@ -27100,7 +34244,7 @@ impl Default for MtaInboundSession { else_: "10m".to_string(), ..Default::default() }, - timeout: Expression { + idle_timeout: Expression { else_: "5m".to_string(), ..Default::default() }, @@ -27114,10 +34258,23 @@ impl Default for MtaInboundSession { impl IntoValue for MtaInboundSession { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::MaxDuration, self.max_duration.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::TransferLimit, self.transfer_limit.into_value()); + map.insert_unchecked( + Property::MaxDuration, + self.max_duration.into_value_with(opts), + ); + map.insert_unchecked( + Property::IdleTimeout, + self.idle_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::TransferLimit, + self.transfer_limit.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -27130,7 +34287,7 @@ impl RegistryJsonPropertyPatch for MtaInboundSession { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::MaxDuration) => self.max_duration.patch(pointer, value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::IdleTimeout) => self.idle_timeout.patch(pointer, value), Some(Property::TransferLimit) => self.transfer_limit.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -27183,21 +34340,55 @@ impl MtaInboundThrottle { impl Pickle for MtaInboundThrottle { fn pickle(&self, out: &mut Vec) { - self.enable.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 5, 1, bitmap); self.description.pickle(out); self.key.pickle(out); self.match_.pickle(out); self.rate.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.match_ = Pickle::unpickle(stream)?; - this.rate = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field4: Rate = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + description: field1, + key: field2, + match_: field3, + rate: field4, + }) } } @@ -27218,12 +34409,19 @@ impl Default for MtaInboundThrottle { impl IntoValue for MtaInboundThrottle { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Match, self.match_.into_value()); - map.insert_unchecked(Property::Rate, self.rate.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked(Property::Match, self.match_.into_value_with(opts)); + map.insert_unchecked(Property::Rate, self.rate.into_value_with(opts)); JmapValue::Object(map) } } @@ -27236,9 +34434,7 @@ impl RegistryJsonPropertyPatch for MtaInboundThrottle { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::Description) => { - self.description.patch(pointer.assert_read_only()?, value) - } + Some(Property::Description) => self.description.patch(pointer, value), Some(Property::Key) => self.key.patch(pointer, value), Some(Property::Match) => self.match_.patch(pointer, value), Some(Property::Rate) => self.rate.patch(pointer, value), @@ -27260,9 +34456,9 @@ impl ObjectImpl for MtaMilter { let neb = errors.len(); let value = &self.enable; value.validate(errors); - let value = &self.hostname; + let value = &self.host; if value.is_empty() { - errors.push(ValidationError::required(Property::Hostname)); + errors.push(ValidationError::required(Property::Host)); } let value = &self.port; if *value > 65535 { @@ -27302,39 +34498,132 @@ impl MtaMilter { impl Pickle for MtaMilter { fn pickle(&self, out: &mut Vec) { - self.allow_invalid_certs.pickle(out); + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.temp_fail_on_error { + bitmap |= 1 << 1; + } + if self.use_tls { + bitmap |= 1 << 2; + } + if self.flags_action.is_some() { + bitmap |= 1 << 3; + } + if self.flags_protocol.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 14, 5, bitmap); self.enable.pickle(out); - self.hostname.pickle(out); + self.host.pickle(out); self.max_response_size.pickle(out); - self.temp_fail_on_error.pickle(out); self.protocol_version.pickle(out); self.port.pickle(out); self.stages.pickle(out); - self.timeout_command.pickle(out); - self.timeout_connect.pickle(out); - self.timeout_data.pickle(out); - self.use_tls.pickle(out); - self.flags_action.pickle(out); - self.flags_protocol.pickle(out); + self.command_timeout.pickle(out); + self.connect_timeout.pickle(out); + self.data_timeout.pickle(out); + if let Some(value) = &self.flags_action { + value.pickle(out); + } + if let Some(value) = &self.flags_protocol { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.hostname = Pickle::unpickle(stream)?; - this.max_response_size = Pickle::unpickle(stream)?; - this.temp_fail_on_error = Pickle::unpickle(stream)?; - this.protocol_version = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.stages = Pickle::unpickle(stream)?; - this.timeout_command = Pickle::unpickle(stream)?; - this.timeout_connect = Pickle::unpickle(stream)?; - this.timeout_data = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.flags_action = Pickle::unpickle(stream)?; - this.flags_protocol = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { false }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 52428800u64 + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + let field5: MilterVersion = if block.has(5) { + Pickle::unpickle(stream)? + } else { + MilterVersion::V6 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 11332u64 + }; + let field7: Map = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![MtaStage::Data]) + }; + let field8: Duration = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field9: Duration = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field10: Duration = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field11: bool = if block.has(11) { block.flag(2) } else { false }; + let field12: Option = if block.has(12) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field13: Option = if block.has(13) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + allow_invalid_certs: field0, + enable: field1, + host: field2, + max_response_size: field3, + temp_fail_on_error: field4, + protocol_version: field5, + port: field6, + stages: field7, + command_timeout: field8, + connect_timeout: field9, + data_timeout: field10, + use_tls: field11, + flags_action: field12, + flags_protocol: field13, + }) } } @@ -27346,15 +34635,15 @@ impl Default for MtaMilter { else_: "true".to_string(), ..Default::default() }, - hostname: Default::default(), + host: Default::default(), max_response_size: 52428800u64, temp_fail_on_error: true, protocol_version: MilterVersion::V6, port: 11332u64, stages: Map::new(vec![MtaStage::Data]), - timeout_command: Duration::from_millis(30000), - timeout_connect: Duration::from_millis(30000), - timeout_data: Duration::from_millis(60000), + command_timeout: Duration::from_millis(30000), + connect_timeout: Duration::from_millis(30000), + data_timeout: Duration::from_millis(60000), use_tls: false, flags_action: Default::default(), flags_protocol: Default::default(), @@ -27364,33 +34653,52 @@ impl Default for MtaMilter { impl IntoValue for MtaMilter { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(16); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Hostname, self.hostname.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); map.insert_unchecked( Property::MaxResponseSize, - self.max_response_size.into_value(), + self.max_response_size.into_value_with(opts), ); map.insert_unchecked( Property::TempFailOnError, - self.temp_fail_on_error.into_value(), + self.temp_fail_on_error.into_value_with(opts), ); map.insert_unchecked( Property::ProtocolVersion, - self.protocol_version.into_value(), + self.protocol_version.into_value_with(opts), + ); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Stages, self.stages.into_value_with(opts)); + map.insert_unchecked( + Property::CommandTimeout, + self.command_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::DataTimeout, + self.data_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); + map.insert_unchecked( + Property::FlagsAction, + self.flags_action.into_value_with(opts), + ); + map.insert_unchecked( + Property::FlagsProtocol, + self.flags_protocol.into_value_with(opts), ); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Stages, self.stages.into_value()); - map.insert_unchecked(Property::TimeoutCommand, self.timeout_command.into_value()); - map.insert_unchecked(Property::TimeoutConnect, self.timeout_connect.into_value()); - map.insert_unchecked(Property::TimeoutData, self.timeout_data.into_value()); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); - map.insert_unchecked(Property::FlagsAction, self.flags_action.into_value()); - map.insert_unchecked(Property::FlagsProtocol, self.flags_protocol.into_value()); JmapValue::Object(map) } } @@ -27404,17 +34712,17 @@ impl RegistryJsonPropertyPatch for MtaMilter { match pointer.next_property() { Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::Hostname) => self - .hostname + Some(Property::Host) => self + .host .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::MaxResponseSize) => self.max_response_size.patch(pointer, value), Some(Property::TempFailOnError) => self.temp_fail_on_error.patch(pointer, value), Some(Property::ProtocolVersion) => self.protocol_version.patch(pointer, value), Some(Property::Port) => self.port.patch(pointer, value), Some(Property::Stages) => self.stages.patch(pointer, value), - Some(Property::TimeoutCommand) => self.timeout_command.patch(pointer, value), - Some(Property::TimeoutConnect) => self.timeout_connect.patch(pointer, value), - Some(Property::TimeoutData) => self.timeout_data.patch(pointer, value), + Some(Property::CommandTimeout) => self.command_timeout.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), + Some(Property::DataTimeout) => self.data_timeout.patch(pointer, value), Some(Property::UseTls) => self.use_tls.patch(pointer, value), Some(Property::FlagsAction) => self.flags_action.patch(pointer, value), Some(Property::FlagsProtocol) => self.flags_protocol.patch(pointer, value), @@ -27532,19 +34840,77 @@ impl MtaOutboundStrategy { impl Pickle for MtaOutboundStrategy { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.connection.pickle(out); self.route.pickle(out); self.schedule.pickle(out); self.tls.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.connection = Pickle::unpickle(stream)?; - this.route = Pickle::unpickle(stream)?; - this.schedule = Pickle::unpickle(stream)?; - this.tls = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'default'".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'mx'".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "is_local_domain(rcpt_domain)".to_string(), + then: "'local'".to_string(), + }]), + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'remote'".to_string(), + match_: List::from_iter([ + ExpressionMatch { + if_: "is_local_domain(rcpt_domain)".to_string(), + then: "'local'".to_string(), + }, + ExpressionMatch { + if_: "source == 'dsn'".to_string(), + then: "'dsn'".to_string(), + }, + ExpressionMatch { + if_: "source == 'report'".to_string(), + then: "'report'".to_string(), + }, + ]), + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'default'".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "retry_num > 0 && last_error == 'tls'".to_string(), + then: "'invalid-tls'".to_string(), + }]), + } + }; + stream.close_block(block); + Some(Self { + connection: field0, + route: field1, + schedule: field2, + tls: field3, + }) } } @@ -27592,11 +34958,15 @@ impl Default for MtaOutboundStrategy { impl IntoValue for MtaOutboundStrategy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Connection, self.connection.into_value()); - map.insert_unchecked(Property::Route, self.route.into_value()); - map.insert_unchecked(Property::Schedule, self.schedule.into_value()); - map.insert_unchecked(Property::Tls, self.tls.into_value()); + map.insert_unchecked(Property::Connection, self.connection.into_value_with(opts)); + map.insert_unchecked(Property::Route, self.route.into_value_with(opts)); + map.insert_unchecked(Property::Schedule, self.schedule.into_value_with(opts)); + map.insert_unchecked(Property::Tls, self.tls.into_value_with(opts)); JmapValue::Object(map) } } @@ -27663,21 +35033,55 @@ impl MtaOutboundThrottle { impl Pickle for MtaOutboundThrottle { fn pickle(&self, out: &mut Vec) { - self.enable.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 5, 1, bitmap); self.description.pickle(out); self.key.pickle(out); self.match_.pickle(out); self.rate.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.match_ = Pickle::unpickle(stream)?; - this.rate = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field4: Rate = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + description: field1, + key: field2, + match_: field3, + rate: field4, + }) } } @@ -27698,12 +35102,19 @@ impl Default for MtaOutboundThrottle { impl IntoValue for MtaOutboundThrottle { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Match, self.match_.into_value()); - map.insert_unchecked(Property::Rate, self.rate.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked(Property::Match, self.match_.into_value_with(opts)); + map.insert_unchecked(Property::Rate, self.rate.into_value_with(opts)); JmapValue::Object(map) } } @@ -27774,23 +35185,86 @@ impl MtaQueueQuota { impl Pickle for MtaQueueQuota { fn pickle(&self, out: &mut Vec) { - self.enable.pickle(out); - self.description.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.description.is_some() { + bitmap |= 1 << 1; + } + if self.messages.is_some() { + bitmap |= 1 << 2; + } + if self.size.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 6, 4, bitmap); + if let Some(value) = &self.description { + value.pickle(out); + } self.key.pickle(out); self.match_.pickle(out); - self.messages.pickle(out); - self.size.pickle(out); + if let Some(value) = &self.messages { + value.pickle(out); + } + if let Some(value) = &self.size { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.match_ = Pickle::unpickle(stream)?; - this.messages = Pickle::unpickle(stream)?; - this.size = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enable: field0, + description: field1, + key: field2, + match_: field3, + messages: field4, + size: field5, + }) } } @@ -27809,13 +35283,20 @@ impl Default for MtaQueueQuota { impl IntoValue for MtaQueueQuota { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Match, self.match_.into_value()); - map.insert_unchecked(Property::Messages, self.messages.into_value()); - map.insert_unchecked(Property::Size, self.size.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked(Property::Match, self.match_.into_value_with(opts)); + map.insert_unchecked(Property::Messages, self.messages.into_value_with(opts)); + map.insert_unchecked(Property::Size, self.size.into_value_with(opts)); JmapValue::Object(map) } } @@ -27828,9 +35309,7 @@ impl RegistryJsonPropertyPatch for MtaQueueQuota { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::Description) => { - self.description.patch(pointer.assert_read_only()?, value) - } + Some(Property::Description) => self.description.patch(pointer, value), Some(Property::Key) => self.key.patch(pointer, value), Some(Property::Match) => self.match_.patch(pointer, value), Some(Property::Messages) => self.messages.patch(pointer, value), @@ -27897,6 +35376,9 @@ impl Pickle for MtaRoute { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(MtaRoute::Mx), 1 => Pickle::unpickle(stream).map(MtaRoute::Relay), @@ -27908,23 +35390,27 @@ impl Pickle for MtaRoute { impl IntoValue for MtaRoute { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { MtaRoute::Mx(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Mx".into())); obj } MtaRoute::Relay(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Relay".into())); obj } MtaRoute::Local(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Local".into())); @@ -27987,15 +35473,39 @@ impl MtaRouteCommon { impl Pickle for MtaRouteCommon { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + }) } } @@ -28010,9 +35520,16 @@ impl Default for MtaRouteCommon { impl IntoValue for MtaRouteCommon { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -28065,21 +35582,60 @@ impl MtaRouteMx { impl Pickle for MtaRouteMx { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 5, 1, bitmap); self.ip_lookup_strategy.pickle(out); self.max_multihomed.pickle(out); self.max_mx_hosts.pickle(out); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.ip_lookup_strategy = Pickle::unpickle(stream)?; - this.max_multihomed = Pickle::unpickle(stream)?; - this.max_mx_hosts = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: MtaIpStrategy = if block.has(0) { + Pickle::unpickle(stream)? + } else { + MtaIpStrategy::V4ThenV6 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + ip_lookup_strategy: field0, + max_multihomed: field1, + max_mx_hosts: field2, + name: field3, + description: field4, + }) } } @@ -28097,15 +35653,28 @@ impl Default for MtaRouteMx { impl IntoValue for MtaRouteMx { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); map.insert_unchecked( Property::IpLookupStrategy, - self.ip_lookup_strategy.into_value(), + self.ip_lookup_strategy.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMultihomed, + self.max_multihomed.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMxHosts, + self.max_mx_hosts.into_value_with(opts), + ); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), ); - map.insert_unchecked(Property::MaxMultihomed, self.max_multihomed.into_value()); - map.insert_unchecked(Property::MaxMxHosts, self.max_mx_hosts.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); JmapValue::Object(map) } } @@ -28134,9 +35703,9 @@ impl RegistryJsonPropertyPatch for MtaRouteMx { impl MtaRouteRelay { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.address; + let value = &self.host; if value.is_empty() { - errors.push(ValidationError::required(Property::Address)); + errors.push(ValidationError::required(Property::Host)); } let value = &self.auth_secret; value.validate(errors); @@ -28171,36 +35740,100 @@ impl MtaRouteRelay { impl Pickle for MtaRouteRelay { fn pickle(&self, out: &mut Vec) { - self.address.pickle(out); + let mut bitmap = 0u64; + if self.auth_username.is_some() { + bitmap |= 1 << 0; + } + if self.allow_invalid_certs { + bitmap |= 1 << 1; + } + if self.implicit_tls { + bitmap |= 1 << 2; + } + if self.description.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 9, 4, bitmap); + self.host.pickle(out); self.auth_secret.pickle(out); - self.auth_username.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.port.pickle(out); self.protocol.pickle(out); - self.allow_invalid_certs.pickle(out); - self.implicit_tls.pickle(out); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.address = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.protocol = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.implicit_tls = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SecretKeyOptional = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 25u64 + }; + let field4: MtaProtocol = if block.has(4) { + Pickle::unpickle(stream)? + } else { + MtaProtocol::Smtp + }; + let field5: bool = if block.has(5) { block.flag(1) } else { false }; + let field6: bool = if block.has(6) { block.flag(2) } else { false }; + let field7: String = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Option = if block.has(8) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + host: field0, + auth_secret: field1, + auth_username: field2, + port: field3, + protocol: field4, + allow_invalid_certs: field5, + implicit_tls: field6, + name: field7, + description: field8, + }) } } impl Default for MtaRouteRelay { fn default() -> Self { Self { - address: Default::default(), + host: Default::default(), auth_secret: Default::default(), auth_username: Default::default(), port: 25u64, @@ -28215,19 +35848,32 @@ impl Default for MtaRouteRelay { impl IntoValue for MtaRouteRelay { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Address, self.address.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Protocol, self.protocol.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Protocol, self.protocol.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked( + Property::ImplicitTls, + self.implicit_tls.into_value_with(opts), + ); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), ); - map.insert_unchecked(Property::ImplicitTls, self.implicit_tls.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); JmapValue::Object(map) } } @@ -28239,8 +35885,8 @@ impl RegistryJsonPropertyPatch for MtaRouteRelay { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Address) => self - .address + Some(Property::Host) => self + .host .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::AuthSecret) => self.auth_secret.patch(pointer, value), Some(Property::AuthUsername) => self.auth_username.patch(pointer, value), @@ -28314,16 +35960,10 @@ impl MtaStageAuth { expr: &self.sasl_mechanisms, default: Some(Expression { else_: "false".to_string(), - match_: List::from_iter([ - ExpressionMatch { - if_: "local_port != 25 && is_tls".to_string(), - then: "[plain, login, oauthbearer, xoauth2]".to_string(), - }, - ExpressionMatch { - if_: "local_port != 25".to_string(), - then: "[oauthbearer, xoauth2]".to_string(), - }, - ]), + match_: List::from_iter([ExpressionMatch { + if_: "local_port != 25 && is_tls".to_string(), + then: "[plain, login, oauthbearer, xoauth2]".to_string(), + }]), }), property: Property::SaslMechanisms, allowed_variables: MTA_EHLO_VARIABLE, @@ -28370,21 +36010,71 @@ impl MtaStageAuth { impl Pickle for MtaStageAuth { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.max_failures.pickle(out); self.wait_on_fail.pickle(out); self.sasl_mechanisms.pickle(out); self.must_match_sender.pickle(out); self.require.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_failures = Pickle::unpickle(stream)?; - this.wait_on_fail = Pickle::unpickle(stream)?; - this.sasl_mechanisms = Pickle::unpickle(stream)?; - this.must_match_sender = Pickle::unpickle(stream)?; - this.require = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "3".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5s".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port != 25 && is_tls".to_string(), + then: "[plain, login, oauthbearer, xoauth2]".to_string(), + }]), + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "local_port != 25".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + max_failures: field0, + wait_on_fail: field1, + sasl_mechanisms: field2, + must_match_sender: field3, + require: field4, + }) } } @@ -28401,16 +36091,10 @@ impl Default for MtaStageAuth { }, sasl_mechanisms: Expression { else_: "false".to_string(), - match_: List::from_iter([ - ExpressionMatch { - if_: "local_port != 25 && is_tls".to_string(), - then: "[plain, login, oauthbearer, xoauth2]".to_string(), - }, - ExpressionMatch { - if_: "local_port != 25".to_string(), - then: "[oauthbearer, xoauth2]".to_string(), - }, - ]), + match_: List::from_iter([ExpressionMatch { + if_: "local_port != 25 && is_tls".to_string(), + then: "[plain, login, oauthbearer, xoauth2]".to_string(), + }]), }, must_match_sender: Expression { else_: "true".to_string(), @@ -28426,15 +36110,28 @@ impl Default for MtaStageAuth { impl IntoValue for MtaStageAuth { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::MaxFailures, self.max_failures.into_value()); - map.insert_unchecked(Property::WaitOnFail, self.wait_on_fail.into_value()); - map.insert_unchecked(Property::SaslMechanisms, self.sasl_mechanisms.into_value()); + map.insert_unchecked( + Property::MaxFailures, + self.max_failures.into_value_with(opts), + ); + map.insert_unchecked( + Property::WaitOnFail, + self.wait_on_fail.into_value_with(opts), + ); + map.insert_unchecked( + Property::SaslMechanisms, + self.sasl_mechanisms.into_value_with(opts), + ); map.insert_unchecked( Property::MustMatchSender, - self.must_match_sender.into_value(), + self.must_match_sender.into_value_with(opts), ); - map.insert_unchecked(Property::Require, self.require.into_value()); + map.insert_unchecked(Property::Require, self.require.into_value_with(opts)); JmapValue::Object(map) } } @@ -28530,17 +36227,48 @@ impl MtaStageConnect { impl Pickle for MtaStageConnect { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.smtp_greeting.pickle(out); self.hostname.pickle(out); self.script.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.smtp_greeting = Pickle::unpickle(stream)?; - this.hostname = Pickle::unpickle(stream)?; - this.script = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('hostname') + ' Stalwart ESMTP at your service'".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('hostname')".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + smtp_greeting: field0, + hostname: field1, + script: field2, + }) } } @@ -28565,10 +36293,17 @@ impl Default for MtaStageConnect { impl IntoValue for MtaStageConnect { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::SmtpGreeting, self.smtp_greeting.into_value()); - map.insert_unchecked(Property::Hostname, self.hostname.into_value()); - map.insert_unchecked(Property::Script, self.script.into_value()); + map.insert_unchecked( + Property::SmtpGreeting, + self.smtp_greeting.into_value_with(opts), + ); + map.insert_unchecked(Property::Hostname, self.hostname.into_value_with(opts)); + map.insert_unchecked(Property::Script, self.script.into_value_with(opts)); JmapValue::Object(map) } } @@ -28621,6 +36356,8 @@ impl ObjectImpl for MtaStageData { value.validate(errors); let value = &self.enable_spam_filter; value.validate(errors); + let value = &self.dkim_sign_domain; + value.validate(errors); errors.len() == neb } @@ -28789,6 +36526,23 @@ impl MtaStageData { } } + pub fn ctx_dkim_sign_domain(&self) -> ExpressionContext<'_> { + ExpressionContext { + expr: &self.dkim_sign_domain, + default: Some(Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "is_local_domain(sender_domain) && !is_empty(authenticated_as)" + .to_string(), + then: "sender_domain".to_string(), + }]), + }), + property: Property::DkimSignDomain, + allowed_variables: MTA_RCPT_TO_VARIABLE, + allowed_constants: &[], + } + } + pub fn expression_ctxs(&self) -> Vec> { vec![ self.ctx_add_auth_results_header(), @@ -28802,15 +36556,20 @@ impl MtaStageData { self.ctx_max_message_size(), self.ctx_script(), self.ctx_enable_spam_filter(), + self.ctx_dkim_sign_domain(), ] } } impl Pickle for MtaStageData { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.add_delivered_to_header { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 13, 1, bitmap); self.add_auth_results_header.pickle(out); self.add_date_header.pickle(out); - self.add_delivered_to_header.pickle(out); self.add_message_id_header.pickle(out); self.add_received_header.pickle(out); self.add_received_spf_header.pickle(out); @@ -28820,23 +36579,150 @@ impl Pickle for MtaStageData { self.max_message_size.pickle(out); self.script.pickle(out); self.enable_spam_filter.pickle(out); + self.dkim_sign_domain.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.add_auth_results_header = Pickle::unpickle(stream)?; - this.add_date_header = Pickle::unpickle(stream)?; - this.add_delivered_to_header = Pickle::unpickle(stream)?; - this.add_message_id_header = Pickle::unpickle(stream)?; - this.add_received_header = Pickle::unpickle(stream)?; - this.add_received_spf_header = Pickle::unpickle(stream)?; - this.add_return_path_header = Pickle::unpickle(stream)?; - this.max_messages = Pickle::unpickle(stream)?; - this.max_received_headers = Pickle::unpickle(stream)?; - this.max_message_size = Pickle::unpickle(stream)?; - this.script = Pickle::unpickle(stream)?; - this.enable_spam_filter = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field2: bool = if block.has(2) { block.flag(0) } else { true }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field6: Expression = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field7: Expression = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "10".to_string(), + ..Default::default() + } + }; + let field8: Expression = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "50".to_string(), + ..Default::default() + } + }; + let field9: Expression = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "104857600".to_string(), + ..Default::default() + } + }; + let field10: Expression = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + let field11: Expression = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "is_empty(authenticated_as)".to_string(), + ..Default::default() + } + }; + let field12: Expression = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "is_local_domain(sender_domain) && !is_empty(authenticated_as)" + .to_string(), + then: "sender_domain".to_string(), + }]), + } + }; + stream.close_block(block); + Some(Self { + add_auth_results_header: field0, + add_date_header: field1, + add_delivered_to_header: field2, + add_message_id_header: field3, + add_received_header: field4, + add_received_spf_header: field5, + add_return_path_header: field6, + max_messages: field7, + max_received_headers: field8, + max_message_size: field9, + script: field10, + enable_spam_filter: field11, + dkim_sign_domain: field12, + }) } } @@ -28906,48 +36792,73 @@ impl Default for MtaStageData { else_: "is_empty(authenticated_as)".to_string(), ..Default::default() }, + dkim_sign_domain: Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "is_local_domain(sender_domain) && !is_empty(authenticated_as)" + .to_string(), + then: "sender_domain".to_string(), + }]), + }, } } } impl IntoValue for MtaStageData { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(14); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(15); map.insert_unchecked( Property::AddAuthResultsHeader, - self.add_auth_results_header.into_value(), + self.add_auth_results_header.into_value_with(opts), + ); + map.insert_unchecked( + Property::AddDateHeader, + self.add_date_header.into_value_with(opts), ); - map.insert_unchecked(Property::AddDateHeader, self.add_date_header.into_value()); map.insert_unchecked( Property::AddDeliveredToHeader, - self.add_delivered_to_header.into_value(), + self.add_delivered_to_header.into_value_with(opts), ); map.insert_unchecked( Property::AddMessageIdHeader, - self.add_message_id_header.into_value(), + self.add_message_id_header.into_value_with(opts), ); map.insert_unchecked( Property::AddReceivedHeader, - self.add_received_header.into_value(), + self.add_received_header.into_value_with(opts), ); map.insert_unchecked( Property::AddReceivedSpfHeader, - self.add_received_spf_header.into_value(), + self.add_received_spf_header.into_value_with(opts), ); map.insert_unchecked( Property::AddReturnPathHeader, - self.add_return_path_header.into_value(), + self.add_return_path_header.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMessages, + self.max_messages.into_value_with(opts), ); - map.insert_unchecked(Property::MaxMessages, self.max_messages.into_value()); map.insert_unchecked( Property::MaxReceivedHeaders, - self.max_received_headers.into_value(), + self.max_received_headers.into_value_with(opts), ); - map.insert_unchecked(Property::MaxMessageSize, self.max_message_size.into_value()); - map.insert_unchecked(Property::Script, self.script.into_value()); + map.insert_unchecked( + Property::MaxMessageSize, + self.max_message_size.into_value_with(opts), + ); + map.insert_unchecked(Property::Script, self.script.into_value_with(opts)); map.insert_unchecked( Property::EnableSpamFilter, - self.enable_spam_filter.into_value(), + self.enable_spam_filter.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), ); JmapValue::Object(map) } @@ -28980,6 +36891,7 @@ impl RegistryJsonPropertyPatch for MtaStageData { Some(Property::MaxMessageSize) => self.max_message_size.patch(pointer, value), Some(Property::Script) => self.script.patch(pointer, value), Some(Property::EnableSpamFilter) => self.enable_spam_filter.patch(pointer, value), + Some(Property::DkimSignDomain) => self.dkim_sign_domain.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -29062,17 +36974,51 @@ impl MtaStageEhlo { impl Pickle for MtaStageEhlo { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.reject_non_fqdn.pickle(out); self.require.pickle(out); self.script.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.reject_non_fqdn = Pickle::unpickle(stream)?; - this.require = Pickle::unpickle(stream)?; - this.script = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "true".to_string(), + }]), + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "true".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + reject_non_fqdn: field0, + require: field1, + script: field2, + }) } } @@ -29100,10 +37046,17 @@ impl Default for MtaStageEhlo { impl IntoValue for MtaStageEhlo { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::RejectNonFqdn, self.reject_non_fqdn.into_value()); - map.insert_unchecked(Property::Require, self.require.into_value()); - map.insert_unchecked(Property::Script, self.script.into_value()); + map.insert_unchecked( + Property::RejectNonFqdn, + self.reject_non_fqdn.into_value_with(opts), + ); + map.insert_unchecked(Property::Require, self.require.into_value_with(opts)); + map.insert_unchecked(Property::Script, self.script.into_value_with(opts)); JmapValue::Object(map) } } @@ -29198,17 +37151,49 @@ impl MtaStageMail { impl Pickle for MtaStageMail { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.is_sender_allowed.pickle(out); self.rewrite.pickle(out); self.script.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.is_sender_allowed = Pickle::unpickle(stream)?; - this.rewrite = Pickle::unpickle(stream)?; - this.script = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "!is_empty(authenticated_as) || !key_exists('spam-block', sender_domain)" + .to_string(), + match_: List::from_iter([]), + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + is_sender_allowed: field0, + rewrite: field1, + script: field2, + }) } } @@ -29234,13 +37219,17 @@ impl Default for MtaStageMail { impl IntoValue for MtaStageMail { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); map.insert_unchecked( Property::IsSenderAllowed, - self.is_sender_allowed.into_value(), + self.is_sender_allowed.into_value_with(opts), ); - map.insert_unchecked(Property::Rewrite, self.rewrite.into_value()); - map.insert_unchecked(Property::Script, self.script.into_value()); + map.insert_unchecked(Property::Rewrite, self.rewrite.into_value_with(opts)); + map.insert_unchecked(Property::Script, self.script.into_value_with(opts)); JmapValue::Object(map) } } @@ -29382,23 +37371,78 @@ impl MtaStageRcpt { impl Pickle for MtaStageRcpt { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 6, 0, 0); self.max_failures.pickle(out); self.wait_on_fail.pickle(out); self.max_recipients.pickle(out); self.allow_relaying.pickle(out); self.rewrite.pickle(out); self.script.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_failures = Pickle::unpickle(stream)?; - this.wait_on_fail = Pickle::unpickle(stream)?; - this.max_recipients = Pickle::unpickle(stream)?; - this.allow_relaying = Pickle::unpickle(stream)?; - this.rewrite = Pickle::unpickle(stream)?; - this.script = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5s".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "100".to_string(), + ..Default::default() + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "!is_empty(authenticated_as)".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + max_failures: field0, + wait_on_fail: field1, + max_recipients: field2, + allow_relaying: field3, + rewrite: field4, + script: field5, + }) } } @@ -29435,13 +37479,29 @@ impl Default for MtaStageRcpt { impl IntoValue for MtaStageRcpt { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::MaxFailures, self.max_failures.into_value()); - map.insert_unchecked(Property::WaitOnFail, self.wait_on_fail.into_value()); - map.insert_unchecked(Property::MaxRecipients, self.max_recipients.into_value()); - map.insert_unchecked(Property::AllowRelaying, self.allow_relaying.into_value()); - map.insert_unchecked(Property::Rewrite, self.rewrite.into_value()); - map.insert_unchecked(Property::Script, self.script.into_value()); + map.insert_unchecked( + Property::MaxFailures, + self.max_failures.into_value_with(opts), + ); + map.insert_unchecked( + Property::WaitOnFail, + self.wait_on_fail.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRecipients, + self.max_recipients.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowRelaying, + self.allow_relaying.into_value_with(opts), + ); + map.insert_unchecked(Property::Rewrite, self.rewrite.into_value_with(opts)); + map.insert_unchecked(Property::Script, self.script.into_value_with(opts)); JmapValue::Object(map) } } @@ -29489,17 +37549,39 @@ impl ObjectImpl for MtaSts { impl Pickle for MtaSts { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.max_age.pickle(out); self.mode.pickle(out); self.mx_hosts.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_age = Pickle::unpickle(stream)?; - this.mode = Pickle::unpickle(stream)?; - this.mx_hosts = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(604800000) + }; + let field1: PolicyEnforcement = if block.has(1) { + Pickle::unpickle(stream)? + } else { + PolicyEnforcement::Testing + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + max_age: field0, + mode: field1, + mx_hosts: field2, + }) } } @@ -29515,10 +37597,14 @@ impl Default for MtaSts { impl IntoValue for MtaSts { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::MaxAge, self.max_age.into_value()); - map.insert_unchecked(Property::Mode, self.mode.into_value()); - map.insert_unchecked(Property::MxHosts, self.mx_hosts.into_value()); + map.insert_unchecked(Property::MaxAge, self.max_age.into_value_with(opts)); + map.insert_unchecked(Property::Mode, self.mode.into_value_with(opts)); + map.insert_unchecked(Property::MxHosts, self.mx_hosts.into_value_with(opts)); JmapValue::Object(map) } } @@ -29570,27 +37656,82 @@ impl ObjectImpl for MtaTlsStrategy { impl Pickle for MtaTlsStrategy { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.description.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 8, 2, bitmap); self.name.pickle(out); - self.allow_invalid_certs.pickle(out); self.dane.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.mta_sts.pickle(out); self.start_tls.pickle(out); self.mta_sts_timeout.pickle(out); self.tls_timeout.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.dane = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.mta_sts = Pickle::unpickle(stream)?; - this.start_tls = Pickle::unpickle(stream)?; - this.mta_sts_timeout = Pickle::unpickle(stream)?; - this.tls_timeout = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: MtaRequiredOrOptional = if block.has(2) { + Pickle::unpickle(stream)? + } else { + MtaRequiredOrOptional::Optional + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: MtaRequiredOrOptional = if block.has(4) { + Pickle::unpickle(stream)? + } else { + MtaRequiredOrOptional::Optional + }; + let field5: MtaRequiredOrOptional = if block.has(5) { + Pickle::unpickle(stream)? + } else { + MtaRequiredOrOptional::Optional + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(180000) + }; + stream.close_block(block); + Some(Self { + name: field0, + allow_invalid_certs: field1, + dane: field2, + description: field3, + mta_sts: field4, + start_tls: field5, + mta_sts_timeout: field6, + tls_timeout: field7, + }) } } @@ -29611,18 +37752,28 @@ impl Default for MtaTlsStrategy { impl IntoValue for MtaTlsStrategy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Name, self.name.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); - map.insert_unchecked(Property::Dane, self.dane.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MtaSts, self.mta_sts.into_value()); - map.insert_unchecked(Property::StartTls, self.start_tls.into_value()); - map.insert_unchecked(Property::MtaStsTimeout, self.mta_sts_timeout.into_value()); - map.insert_unchecked(Property::TlsTimeout, self.tls_timeout.into_value()); + map.insert_unchecked(Property::Dane, self.dane.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::MtaSts, self.mta_sts.into_value_with(opts)); + map.insert_unchecked(Property::StartTls, self.start_tls.into_value_with(opts)); + map.insert_unchecked( + Property::MtaStsTimeout, + self.mta_sts_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::TlsTimeout, self.tls_timeout.into_value_with(opts)); JmapValue::Object(map) } } @@ -29684,17 +37835,49 @@ impl ObjectImpl for MtaVirtualQueue { impl Pickle for MtaVirtualQueue { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 3, 1, bitmap); self.name.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.threads_per_node.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.threads_per_node = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 25u64 + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + threads_per_node: field2, + }) } } @@ -29710,10 +37893,20 @@ impl Default for MtaVirtualQueue { impl IntoValue for MtaVirtualQueue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::ThreadsPerNode, self.threads_per_node.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::ThreadsPerNode, + self.threads_per_node.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -29742,106 +37935,6 @@ impl RegistryJsonPropertyPatch for MtaVirtualQueue { } } -impl MySqlSettings { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.host; - if value.is_empty() { - errors.push(ValidationError::required(Property::Host)); - } - let value = &self.port; - if *value > 65535 { - errors.push(ValidationError::max_value(Property::Port, 65535)); - } - if *value < 1 { - errors.push(ValidationError::min_value(Property::Port, 1)); - } - let value = &self.database; - if value.is_empty() { - errors.push(ValidationError::required(Property::Database)); - } - if let Some(value) = &self.auth_username { - if value.is_empty() { - errors.push(ValidationError::required(Property::AuthUsername)); - } - } - let value = &self.auth_secret; - value.validate(errors); - errors.len() == neb - } -} - -impl Pickle for MySqlSettings { - fn pickle(&self, out: &mut Vec) { - self.host.pickle(out); - self.port.pickle(out); - self.database.pickle(out); - self.auth_username.pickle(out); - self.auth_secret.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.database = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for MySqlSettings { - fn default() -> Self { - Self { - host: Default::default(), - port: 3306u64, - database: "inbuxa".to_string(), - auth_username: Some("inbuxa".to_string()), - auth_secret: Default::default(), - } - } -} - -impl IntoValue for MySqlSettings { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Database, self.database.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for MySqlSettings { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Host) => self - .host - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Port) => self.port.patch(pointer, value), - Some(Property::Database) => self - .database - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthUsername) => self - .auth_username - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthSecret) => self.auth_secret.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl MySqlStore { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -29878,10 +37971,6 @@ impl MySqlStore { errors.push(ValidationError::min_value(Property::PoolMinConnections, 1)); } } - let value = &self.read_replicas; - for value in value.values() { - value.validate(errors); - } let value = &self.host; if value.is_empty() { errors.push(ValidationError::required(Property::Host)); @@ -29906,52 +37995,161 @@ impl MySqlStore { value.validate(errors); errors.len() == neb } + } impl Pickle for MySqlStore { fn pickle(&self, out: &mut Vec) { - self.timeout.pickle(out); - self.use_tls.pickle(out); - self.allow_invalid_certs.pickle(out); - self.max_allowed_packet.pickle(out); - self.pool_max_connections.pickle(out); - self.pool_min_connections.pickle(out); - self.read_replicas.pickle(out); + let mut bitmap = 0u64; + if self.idle_timeout.is_some() { + bitmap |= 1 << 0; + } + if self.use_tls { + bitmap |= 1 << 1; + } + if self.allow_invalid_certs { + bitmap |= 1 << 2; + } + if self.max_allowed_packet.is_some() { + bitmap |= 1 << 3; + } + if self.pool_max_connections.is_some() { + bitmap |= 1 << 4; + } + if self.pool_min_connections.is_some() { + bitmap |= 1 << 5; + } + if self.auth_username.is_some() { + bitmap |= 1 << 6; + } + let block = crate::pickle::begin_block(out, 12, 7, bitmap); + if let Some(value) = &self.idle_timeout { + value.pickle(out); + } + if let Some(value) = &self.max_allowed_packet { + value.pickle(out); + } + if let Some(value) = &self.pool_max_connections { + value.pickle(out); + } + if let Some(value) = &self.pool_min_connections { + value.pickle(out); + } + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } self.host.pickle(out); self.port.pickle(out); self.database.pickle(out); - self.auth_username.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.auth_secret.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.timeout = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.max_allowed_packet = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_min_connections = Pickle::unpickle(stream)?; - this.read_replicas = Pickle::unpickle(stream)?; - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.database = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(15000)) + }; + let field1: bool = if block.has(1) { block.flag(1) } else { false }; + let field2: bool = if block.has(2) { block.flag(2) } else { false }; + let field3: Option = if block.has(3) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(5u64) + }; + #[cfg(not(feature = "enterprise"))] + if block.has(6) { + let block = stream.open_block()?; + stream.close_block(block); + } + let field7: String = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 3306u64 + }; + let field9: String = if block.has(9) { + Pickle::unpickle(stream)? + } else { + "inbuxa".to_string() + }; + let field10: Option = if block.has(10) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("inbuxa".to_string()) + }; + let field11: SecretKeyOptional = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + idle_timeout: field0, + use_tls: field1, + allow_invalid_certs: field2, + max_allowed_packet: field3, + pool_max_connections: field4, + pool_min_connections: field5, + host: field7, + port: field8, + database: field9, + auth_username: field10, + auth_secret: field11, + }) } } impl Default for MySqlStore { fn default() -> Self { Self { - timeout: Some(Duration::from_millis(15000)), + idle_timeout: Some(Duration::from_millis(15000)), use_tls: false, allow_invalid_certs: false, max_allowed_packet: Default::default(), - pool_max_connections: Some(10u64), + pool_max_connections: Default::default(), pool_min_connections: Some(5u64), - read_replicas: Default::default(), host: Default::default(), port: 3306u64, database: "inbuxa".to_string(), @@ -29963,31 +38161,40 @@ impl Default for MySqlStore { impl IntoValue for MySqlStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); + map.insert_unchecked( + Property::IdleTimeout, + self.idle_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); map.insert_unchecked( Property::MaxAllowedPacket, - self.max_allowed_packet.into_value(), + self.max_allowed_packet.into_value_with(opts), ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolMinConnections, - self.pool_min_connections.into_value(), + self.pool_min_connections.into_value_with(opts), ); - map.insert_unchecked(Property::ReadReplicas, self.read_replicas.into_value()); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Database, self.database.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Database, self.database.into_value_with(opts)); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); JmapValue::Object(map) } } @@ -29999,13 +38206,12 @@ impl RegistryJsonPropertyPatch for MySqlStore { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::IdleTimeout) => self.idle_timeout.patch(pointer, value), Some(Property::UseTls) => self.use_tls.patch(pointer, value), Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::MaxAllowedPacket) => self.max_allowed_packet.patch(pointer, value), Some(Property::PoolMaxConnections) => self.pool_max_connections.patch(pointer, value), Some(Property::PoolMinConnections) => self.pool_min_connections.patch(pointer, value), - Some(Property::ReadReplicas) => self.read_replicas.patch(pointer, value), Some(Property::Host) => self .host .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -30029,27 +38235,27 @@ impl RegistryJsonPropertyPatch for MySqlStore { impl NatsCoordinator { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.addresses; + let value = &self.hosts; for value in value.iter() { if value.is_empty() { - errors.push(ValidationError::required(Property::Addresses)); + errors.push(ValidationError::required(Property::Hosts)); } } if value.len() < 1 { - errors.push(ValidationError::min_items(Property::Addresses, 1)); + errors.push(ValidationError::min_items(Property::Hosts, 1)); } - let value = &self.capacity_client; + let value = &self.client_capacity; if *value < 1 { - errors.push(ValidationError::min_value(Property::CapacityClient, 1)); + errors.push(ValidationError::min_value(Property::ClientCapacity, 1)); } - let value = &self.capacity_read_buffer; + let value = &self.read_buffer_capacity; if *value < 1 { - errors.push(ValidationError::min_value(Property::CapacityReadBuffer, 1)); + errors.push(ValidationError::min_value(Property::ReadBufferCapacity, 1)); } - let value = &self.capacity_subscription; + let value = &self.subscription_capacity; if *value < 1 { errors.push(ValidationError::min_value( - Property::CapacitySubscription, + Property::SubscriptionCapacity, 1, )); } @@ -30064,57 +38270,148 @@ impl NatsCoordinator { value.validate(errors); errors.len() == neb } + } impl Pickle for NatsCoordinator { fn pickle(&self, out: &mut Vec) { - self.addresses.pickle(out); - self.max_reconnects.pickle(out); - self.timeout_connection.pickle(out); - self.timeout_request.pickle(out); + let mut bitmap = 0u64; + if self.max_reconnects.is_some() { + bitmap |= 1 << 0; + } + if self.no_echo { + bitmap |= 1 << 1; + } + if self.use_tls { + bitmap |= 1 << 2; + } + if self.allow_invalid_certs { + bitmap |= 1 << 3; + } + if self.auth_username.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 14, 5, bitmap); + self.hosts.pickle(out); + if let Some(value) = &self.max_reconnects { + value.pickle(out); + } + self.connect_timeout.pickle(out); + self.request_timeout.pickle(out); self.ping_interval.pickle(out); - self.capacity_client.pickle(out); - self.capacity_read_buffer.pickle(out); - self.capacity_subscription.pickle(out); - self.no_echo.pickle(out); - self.use_tls.pickle(out); + self.client_capacity.pickle(out); + self.read_buffer_capacity.pickle(out); + self.subscription_capacity.pickle(out); self.auth_secret.pickle(out); - self.auth_username.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.credentials.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.addresses = Pickle::unpickle(stream)?; - this.max_reconnects = Pickle::unpickle(stream)?; - this.timeout_connection = Pickle::unpickle(stream)?; - this.timeout_request = Pickle::unpickle(stream)?; - this.ping_interval = Pickle::unpickle(stream)?; - this.capacity_client = Pickle::unpickle(stream)?; - this.capacity_read_buffer = Pickle::unpickle(stream)?; - this.capacity_subscription = Pickle::unpickle(stream)?; - this.no_echo = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.credentials = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["127.0.0.1:4222".to_string()]) + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 2048u64 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 65535u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 65536u64 + }; + let field8: bool = if block.has(8) { block.flag(1) } else { true }; + let field9: bool = if block.has(9) { block.flag(2) } else { false }; + let field10: bool = if block.has(10) { block.flag(3) } else { false }; + let field11: SecretKeyOptional = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: Option = if block.has(12) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("inbuxa".to_string()) + }; + let field13: SecretTextOptional = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + hosts: field0, + max_reconnects: field1, + connect_timeout: field2, + request_timeout: field3, + ping_interval: field4, + client_capacity: field5, + read_buffer_capacity: field6, + subscription_capacity: field7, + no_echo: field8, + use_tls: field9, + allow_invalid_certs: field10, + auth_secret: field11, + auth_username: field12, + credentials: field13, + }) } } impl Default for NatsCoordinator { fn default() -> Self { Self { - addresses: Map::new(vec!["127.0.0.1:4444".to_string()]), + hosts: Map::new(vec!["127.0.0.1:4222".to_string()]), max_reconnects: Default::default(), - timeout_connection: Duration::from_millis(5000), - timeout_request: Duration::from_millis(10000), + connect_timeout: Duration::from_millis(5000), + request_timeout: Duration::from_millis(10000), ping_interval: Duration::from_millis(60000), - capacity_client: 2048u64, - capacity_read_buffer: 65535u64, - capacity_subscription: 65536u64, + client_capacity: 2048u64, + read_buffer_capacity: 65535u64, + subscription_capacity: 65536u64, no_echo: true, use_tls: false, + allow_invalid_certs: false, auth_secret: Default::default(), auth_username: Some("inbuxa".to_string()), credentials: Default::default(), @@ -30124,29 +38421,55 @@ impl Default for NatsCoordinator { impl IntoValue for NatsCoordinator { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::Addresses, self.addresses.into_value()); - map.insert_unchecked(Property::MaxReconnects, self.max_reconnects.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); + map.insert_unchecked(Property::Hosts, self.hosts.into_value_with(opts)); map.insert_unchecked( - Property::TimeoutConnection, - self.timeout_connection.into_value(), - ); - map.insert_unchecked(Property::TimeoutRequest, self.timeout_request.into_value()); - map.insert_unchecked(Property::PingInterval, self.ping_interval.into_value()); - map.insert_unchecked(Property::CapacityClient, self.capacity_client.into_value()); - map.insert_unchecked( - Property::CapacityReadBuffer, - self.capacity_read_buffer.into_value(), + Property::MaxReconnects, + self.max_reconnects.into_value_with(opts), ); map.insert_unchecked( - Property::CapacitySubscription, - self.capacity_subscription.into_value(), + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::RequestTimeout, + self.request_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::PingInterval, + self.ping_interval.into_value_with(opts), + ); + map.insert_unchecked( + Property::ClientCapacity, + self.client_capacity.into_value_with(opts), + ); + map.insert_unchecked( + Property::ReadBufferCapacity, + self.read_buffer_capacity.into_value_with(opts), + ); + map.insert_unchecked( + Property::SubscriptionCapacity, + self.subscription_capacity.into_value_with(opts), + ); + map.insert_unchecked(Property::NoEcho, self.no_echo.into_value_with(opts)); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked( + Property::Credentials, + self.credentials.into_value_with(opts), ); - map.insert_unchecked(Property::NoEcho, self.no_echo.into_value()); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::Credentials, self.credentials.into_value()); JmapValue::Object(map) } } @@ -30158,18 +38481,19 @@ impl RegistryJsonPropertyPatch for NatsCoordinator { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Addresses) => self.addresses.patch(pointer, value), + Some(Property::Hosts) => self.hosts.patch(pointer, value), Some(Property::MaxReconnects) => self.max_reconnects.patch(pointer, value), - Some(Property::TimeoutConnection) => self.timeout_connection.patch(pointer, value), - Some(Property::TimeoutRequest) => self.timeout_request.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), + Some(Property::RequestTimeout) => self.request_timeout.patch(pointer, value), Some(Property::PingInterval) => self.ping_interval.patch(pointer, value), - Some(Property::CapacityClient) => self.capacity_client.patch(pointer, value), - Some(Property::CapacityReadBuffer) => self.capacity_read_buffer.patch(pointer, value), - Some(Property::CapacitySubscription) => { - self.capacity_subscription.patch(pointer, value) + Some(Property::ClientCapacity) => self.client_capacity.patch(pointer, value), + Some(Property::ReadBufferCapacity) => self.read_buffer_capacity.patch(pointer, value), + Some(Property::SubscriptionCapacity) => { + self.subscription_capacity.patch(pointer, value) } Some(Property::NoEcho) => self.no_echo.patch(pointer, value), Some(Property::UseTls) => self.use_tls.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::AuthSecret) => self.auth_secret.patch(pointer, value), Some(Property::AuthUsername) => self .auth_username @@ -30259,49 +38583,203 @@ impl ObjectImpl for NetworkListener { impl Pickle for NetworkListener { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.socket_backlog.is_some() { + bitmap |= 1 << 0; + } + if self.socket_no_delay { + bitmap |= 1 << 1; + } + if self.socket_receive_buffer_size.is_some() { + bitmap |= 1 << 2; + } + if self.socket_reuse_address { + bitmap |= 1 << 3; + } + if self.socket_reuse_port { + bitmap |= 1 << 4; + } + if self.socket_send_buffer_size.is_some() { + bitmap |= 1 << 5; + } + if self.socket_tos_v4.is_some() { + bitmap |= 1 << 6; + } + if self.socket_ttl.is_some() { + bitmap |= 1 << 7; + } + if self.use_tls { + bitmap |= 1 << 8; + } + if self.tls_ignore_client_order { + bitmap |= 1 << 9; + } + if self.implicit_tls { + bitmap |= 1 << 10; + } + if self.tls_timeout.is_some() { + bitmap |= 1 << 11; + } + if self.max_connections.is_some() { + bitmap |= 1 << 12; + } + let block = crate::pickle::begin_block(out, 19, 13, bitmap); self.name.pickle(out); self.bind.pickle(out); self.protocol.pickle(out); self.override_proxy_trusted_networks.pickle(out); - self.socket_backlog.pickle(out); - self.socket_no_delay.pickle(out); - self.socket_receive_buffer_size.pickle(out); - self.socket_reuse_address.pickle(out); - self.socket_reuse_port.pickle(out); - self.socket_send_buffer_size.pickle(out); - self.socket_tos_v4.pickle(out); - self.socket_ttl.pickle(out); - self.use_tls.pickle(out); + if let Some(value) = &self.socket_backlog { + value.pickle(out); + } + if let Some(value) = &self.socket_receive_buffer_size { + value.pickle(out); + } + if let Some(value) = &self.socket_send_buffer_size { + value.pickle(out); + } + if let Some(value) = &self.socket_tos_v4 { + value.pickle(out); + } + if let Some(value) = &self.socket_ttl { + value.pickle(out); + } self.tls_disable_cipher_suites.pickle(out); self.tls_disable_protocols.pickle(out); - self.tls_ignore_client_order.pickle(out); - self.tls_implicit.pickle(out); - self.tls_timeout.pickle(out); - self.max_connections.pickle(out); + if let Some(value) = &self.tls_timeout { + value.pickle(out); + } + if let Some(value) = &self.max_connections { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.bind = Pickle::unpickle(stream)?; - this.protocol = Pickle::unpickle(stream)?; - this.override_proxy_trusted_networks = Pickle::unpickle(stream)?; - this.socket_backlog = Pickle::unpickle(stream)?; - this.socket_no_delay = Pickle::unpickle(stream)?; - this.socket_receive_buffer_size = Pickle::unpickle(stream)?; - this.socket_reuse_address = Pickle::unpickle(stream)?; - this.socket_reuse_port = Pickle::unpickle(stream)?; - this.socket_send_buffer_size = Pickle::unpickle(stream)?; - this.socket_tos_v4 = Pickle::unpickle(stream)?; - this.socket_ttl = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.tls_disable_cipher_suites = Pickle::unpickle(stream)?; - this.tls_disable_protocols = Pickle::unpickle(stream)?; - this.tls_ignore_client_order = Pickle::unpickle(stream)?; - this.tls_implicit = Pickle::unpickle(stream)?; - this.tls_timeout = Pickle::unpickle(stream)?; - this.max_connections = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: NetworkListenerProtocol = if block.has(2) { + Pickle::unpickle(stream)? + } else { + NetworkListenerProtocol::Smtp + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(1024u64) + }; + let field5: bool = if block.has(5) { block.flag(1) } else { true }; + let field6: Option = if block.has(6) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: bool = if block.has(7) { block.flag(3) } else { true }; + let field8: bool = if block.has(8) { block.flag(4) } else { true }; + let field9: Option = if block.has(9) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field10: Option = if block.has(10) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field11: Option = if block.has(11) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field12: bool = if block.has(12) { block.flag(8) } else { true }; + let field13: Map = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field14: Map = if block.has(14) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field15: bool = if block.has(15) { block.flag(9) } else { true }; + let field16: bool = if block.has(16) { block.flag(10) } else { false }; + let field17: Option = if block.has(17) { + if block.flag(11) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(60000)) + }; + let field18: Option = if block.has(18) { + if block.flag(12) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + bind: field1, + protocol: field2, + override_proxy_trusted_networks: field3, + socket_backlog: field4, + socket_no_delay: field5, + socket_receive_buffer_size: field6, + socket_reuse_address: field7, + socket_reuse_port: field8, + socket_send_buffer_size: field9, + socket_tos_v4: field10, + socket_ttl: field11, + use_tls: field12, + tls_disable_cipher_suites: field13, + tls_disable_protocols: field14, + tls_ignore_client_order: field15, + implicit_tls: field16, + tls_timeout: field17, + max_connections: field18, + }) } } @@ -30324,59 +38802,78 @@ impl Default for NetworkListener { tls_disable_cipher_suites: Default::default(), tls_disable_protocols: Default::default(), tls_ignore_client_order: true, - tls_implicit: false, + implicit_tls: false, tls_timeout: Some(Duration::from_millis(60000)), - max_connections: Some(8192u64), + max_connections: Default::default(), } } } impl IntoValue for NetworkListener { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(21); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Bind, self.bind.into_value()); - map.insert_unchecked(Property::Protocol, self.protocol.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::Bind, self.bind.into_value_with(opts)); + map.insert_unchecked(Property::Protocol, self.protocol.into_value_with(opts)); map.insert_unchecked( Property::OverrideProxyTrustedNetworks, - self.override_proxy_trusted_networks.into_value(), + self.override_proxy_trusted_networks.into_value_with(opts), + ); + map.insert_unchecked( + Property::SocketBacklog, + self.socket_backlog.into_value_with(opts), + ); + map.insert_unchecked( + Property::SocketNoDelay, + self.socket_no_delay.into_value_with(opts), ); - map.insert_unchecked(Property::SocketBacklog, self.socket_backlog.into_value()); - map.insert_unchecked(Property::SocketNoDelay, self.socket_no_delay.into_value()); map.insert_unchecked( Property::SocketReceiveBufferSize, - self.socket_receive_buffer_size.into_value(), + self.socket_receive_buffer_size.into_value_with(opts), ); map.insert_unchecked( Property::SocketReuseAddress, - self.socket_reuse_address.into_value(), + self.socket_reuse_address.into_value_with(opts), ); map.insert_unchecked( Property::SocketReusePort, - self.socket_reuse_port.into_value(), + self.socket_reuse_port.into_value_with(opts), ); map.insert_unchecked( Property::SocketSendBufferSize, - self.socket_send_buffer_size.into_value(), + self.socket_send_buffer_size.into_value_with(opts), ); - map.insert_unchecked(Property::SocketTosV4, self.socket_tos_v4.into_value()); - map.insert_unchecked(Property::SocketTtl, self.socket_ttl.into_value()); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); + map.insert_unchecked( + Property::SocketTosV4, + self.socket_tos_v4.into_value_with(opts), + ); + map.insert_unchecked(Property::SocketTtl, self.socket_ttl.into_value_with(opts)); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); map.insert_unchecked( Property::TlsDisableCipherSuites, - self.tls_disable_cipher_suites.into_value(), + self.tls_disable_cipher_suites.into_value_with(opts), ); map.insert_unchecked( Property::TlsDisableProtocols, - self.tls_disable_protocols.into_value(), + self.tls_disable_protocols.into_value_with(opts), ); map.insert_unchecked( Property::TlsIgnoreClientOrder, - self.tls_ignore_client_order.into_value(), + self.tls_ignore_client_order.into_value_with(opts), + ); + map.insert_unchecked( + Property::ImplicitTls, + self.implicit_tls.into_value_with(opts), + ); + map.insert_unchecked(Property::TlsTimeout, self.tls_timeout.into_value_with(opts)); + map.insert_unchecked( + Property::MaxConnections, + self.max_connections.into_value_with(opts), ); - map.insert_unchecked(Property::TlsImplicit, self.tls_implicit.into_value()); - map.insert_unchecked(Property::TlsTimeout, self.tls_timeout.into_value()); - map.insert_unchecked(Property::MaxConnections, self.max_connections.into_value()); JmapValue::Object(map) } } @@ -30416,7 +38913,7 @@ impl RegistryJsonPropertyPatch for NetworkListener { Some(Property::TlsIgnoreClientOrder) => { self.tls_ignore_client_order.patch(pointer, value) } - Some(Property::TlsImplicit) => self.tls_implicit.patch(pointer, value), + Some(Property::ImplicitTls) => self.implicit_tls.patch(pointer, value), Some(Property::TlsTimeout) => self.tls_timeout.patch(pointer, value), Some(Property::MaxConnections) => self.max_connections.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -30464,17 +38961,18 @@ impl ObjectImpl for OAuthClient { errors.push(ValidationError::invalid(Property::ExpiresAt, value)); } } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.redirect_uris; for value in value.iter() { if value.is_empty() { errors.push(ValidationError::required(Property::RedirectUris)); } } + let value = &self.post_logout_redirect_uris; + for value in value.iter() { + if value.is_empty() { + errors.push(ValidationError::required(Property::PostLogoutRedirectUris)); + } + } if let Some(value) = &self.logo { if value.is_empty() { errors.push(ValidationError::required(Property::Logo)); @@ -30491,38 +38989,145 @@ impl ObjectImpl for OAuthClient { for value in self.contacts.iter() { i.text(Property::Text, value); } - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for OAuthClient { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.secret.is_some() { + bitmap |= 1 << 1; + } + if self.expires_at.is_some() { + bitmap |= 1 << 2; + } + if self.logo.is_some() { + bitmap |= 1 << 4; + } + if self.first_party { + bitmap |= 1 << 5; + } + let block = crate::pickle::begin_block(out, 13, 6, bitmap); self.client_id.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.contacts.pickle(out); - self.secret.pickle(out); + if let Some(value) = &self.secret { + value.pickle(out); + } self.created_at.pickle(out); - self.expires_at.pickle(out); - self.member_tenant_id.pickle(out); + if let Some(value) = &self.expires_at { + value.pickle(out); + } self.redirect_uris.pickle(out); - self.logo.pickle(out); + self.post_logout_redirect_uris.pickle(out); + if let Some(value) = &self.logo { + value.pickle(out); + } + self.scopes.pickle(out); + self.grant_types.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.client_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.contacts = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.redirect_uris = Pickle::unpickle(stream)?; - this.logo = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Map = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: Map = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Option = if block.has(9) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field10: bool = if block.has(10) { block.flag(5) } else { false }; + let field11: Map = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: Map = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + OAuthGrantType::AuthorizationCode, + OAuthGrantType::RefreshToken, + ]) + }; + stream.close_block(block); + Some(Self { + client_id: field0, + description: field1, + contacts: field2, + secret: field3, + created_at: field4, + expires_at: field5, + redirect_uris: field7, + post_logout_redirect_uris: field8, + logo: field9, + first_party: field10, + scopes: field11, + grant_types: field12, + }) } } @@ -30535,27 +39140,56 @@ impl Default for OAuthClient { secret: Default::default(), created_at: Default::default(), expires_at: Default::default(), - member_tenant_id: Default::default(), redirect_uris: Default::default(), + post_logout_redirect_uris: Default::default(), logo: Default::default(), + first_party: false, + scopes: Default::default(), + grant_types: Map::new(vec![ + OAuthGrantType::AuthorizationCode, + OAuthGrantType::RefreshToken, + ]), } } } impl IntoValue for OAuthClient { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::ClientId, self.client_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Contacts, self.contacts.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(15); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Contacts, self.contacts.into_value_with(opts)); if self.secret.is_some() { - map.insert_unchecked(Property::Secret, secret_value(self.secret)); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::RedirectUris, self.redirect_uris.into_value()); - map.insert_unchecked(Property::Logo, self.logo.into_value()); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::RedirectUris, + self.redirect_uris.into_value_with(opts), + ); + map.insert_unchecked( + Property::PostLogoutRedirectUris, + self.post_logout_redirect_uris.into_value_with(opts), + ); + map.insert_unchecked(Property::Logo, self.logo.into_value_with(opts)); + map.insert_unchecked(Property::FirstParty, self.first_party.into_value_with(opts)); + map.insert_unchecked(Property::Scopes, self.scopes.into_value_with(opts)); + map.insert_unchecked(Property::GrantTypes, self.grant_types.into_value_with(opts)); JmapValue::Object(map) } } @@ -30577,11 +39211,229 @@ impl RegistryJsonPropertyPatch for OAuthClient { Some(Property::Secret) => self.secret.patch(pointer, value), Some(Property::CreatedAt) => pointer.assert_server_set(), Some(Property::ExpiresAt) => self.expires_at.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::RedirectUris) => self.redirect_uris.patch(pointer, value), + Some(Property::PostLogoutRedirectUris) => { + self.post_logout_redirect_uris.patch(pointer, value) + } Some(Property::Logo) => self.logo.patch(pointer, value), + Some(Property::FirstParty) => self.first_party.patch(pointer, value), + Some(Property::Scopes) => self.scopes.patch(pointer, value), + Some(Property::GrantTypes) => self.grant_types.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl ObjectImpl for OAuthGrant { + const FLAGS: u64 = OBJ_FILTER_ACCOUNT | OBJ_SEQ_ID | OBJ_NO_CREATE | OBJ_NO_UPDATE; + const VERSION: u8 = 0; + const OBJECT: ObjectType = ObjectType::OAuthGrant; + + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.account_id; + if !value.is_valid() { + errors.push(ValidationError::required(Property::AccountId)); + } + let value = &self.client_id; + if value.is_empty() { + errors.push(ValidationError::required(Property::ClientId)); + } + if let Some(value) = &self.client_name { + if value.is_empty() { + errors.push(ValidationError::required(Property::ClientName)); + } + } + if let Some(value) = &self.remote_ip { + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::RemoteIp, value)); + } + } + let value = &self.created_at; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::CreatedAt, value)); + } + let value = &self.last_used_at; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::LastUsedAt, value)); + } + errors.len() == neb + } + + fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { + i.foreign_key(ObjectType::Account, self.account_id.into(), None); + i.search(Property::AccountId, &self.account_id); + } +} + +impl Pickle for OAuthGrant { + fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.client_name.is_some() { + bitmap |= 1 << 0; + } + if self.remote_ip.is_some() { + bitmap |= 1 << 1; + } + if self.pending { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 10, 3, bitmap); + self.account_id.pickle(out); + self.client_id.pickle(out); + if let Some(value) = &self.client_name { + value.pickle(out); + } + self.client_kind.pickle(out); + self.scopes.pickle(out); + if let Some(value) = &self.remote_ip { + value.pickle(out); + } + self.created_at.pickle(out); + self.last_used_at.pickle(out); + self.generation.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: OAuthClientKind = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: UTCDateTime = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: UTCDateTime = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field9: bool = if block.has(9) { block.flag(2) } else { false }; + stream.close_block(block); + Some(Self { + account_id: field0, + client_id: field1, + client_name: field2, + client_kind: field3, + scopes: field4, + remote_ip: field5, + created_at: field6, + last_used_at: field7, + generation: field8, + pending: field9, + }) + } +} + +impl Default for OAuthGrant { + fn default() -> Self { + Self { + account_id: Default::default(), + client_id: Default::default(), + client_name: Default::default(), + client_kind: Default::default(), + scopes: Default::default(), + remote_ip: Default::default(), + created_at: Default::default(), + last_used_at: Default::default(), + generation: 0u64, + pending: false, + } + } +} + +impl IntoValue for OAuthGrant { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(12); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::ClientId, self.client_id.into_value_with(opts)); + map.insert_unchecked(Property::ClientName, self.client_name.into_value_with(opts)); + map.insert_unchecked(Property::ClientKind, self.client_kind.into_value_with(opts)); + map.insert_unchecked(Property::Scopes, self.scopes.into_value_with(opts)); + map.insert_unchecked(Property::RemoteIp, self.remote_ip.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked( + Property::LastUsedAt, + self.last_used_at.into_value_with(opts), + ); + map.insert_unchecked(Property::Generation, self.generation.into_value_with(opts)); + map.insert_unchecked(Property::Pending, self.pending.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for OAuthGrant { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::AccountId) => self + .account_id + .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), + Some(Property::ClientId) => pointer.assert_server_set(), + Some(Property::ClientName) => pointer.assert_server_set(), + Some(Property::ClientKind) => pointer.assert_server_set(), + Some(Property::Scopes) => pointer.assert_server_set(), + Some(Property::RemoteIp) => pointer.assert_server_set(), + Some(Property::CreatedAt) => pointer.assert_server_set(), + Some(Property::LastUsedAt) => pointer.assert_server_set(), + Some(Property::Generation) => pointer.assert_server_set(), + Some(Property::Pending) => pointer.assert_server_set(), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -30602,17 +39454,6 @@ impl OidcDirectory { if value.is_empty() { errors.push(ValidationError::required(Property::IssuerUrl)); } - if let Some(value) = &self.require_audience { - if value.is_empty() { - errors.push(ValidationError::required(Property::RequireAudience)); - } - } - let value = &self.require_scopes; - for value in value.iter() { - if value.is_empty() { - errors.push(ValidationError::required(Property::RequireScopes)); - } - } let value = &self.claim_username; if value.is_empty() { errors.push(ValidationError::required(Property::ClaimUsername)); @@ -30632,47 +39473,179 @@ impl OidcDirectory { errors.push(ValidationError::required(Property::ClaimGroups)); } } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); + let value = &self.accepted_audiences; + for value in value.iter() { + if value.is_empty() { + errors.push(ValidationError::required(Property::AcceptedAudiences)); } } + if let Some(value) = &self.introspection_client_id { + if value.is_empty() { + errors.push(ValidationError::required(Property::IntrospectionClientId)); + } + } + let value = &self.introspection_client_secret; + value.validate(errors); errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for OidcDirectory { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.username_domain.is_some() { + bitmap |= 1 << 1; + } + if self.claim_name.is_some() { + bitmap |= 1 << 2; + } + if self.claim_groups.is_some() { + bitmap |= 1 << 3; + } + if self.introspection_client_id.is_some() { + bitmap |= 1 << 5; + } + if self.skip_audience_validation { + bitmap |= 1 << 6; + } + if self.allow_id_token_claims { + bitmap |= 1 << 7; + } + if self.lenient_issuer_match { + bitmap |= 1 << 8; + } + let block = crate::pickle::begin_block(out, 17, 9, bitmap); self.description.pickle(out); self.issuer_url.pickle(out); - self.require_audience.pickle(out); - self.require_scopes.pickle(out); self.claim_username.pickle(out); - self.username_domain.pickle(out); - self.claim_name.pickle(out); - self.claim_groups.pickle(out); - self.member_tenant_id.pickle(out); + if let Some(value) = &self.username_domain { + value.pickle(out); + } + if let Some(value) = &self.claim_name { + value.pickle(out); + } + if let Some(value) = &self.claim_groups { + value.pickle(out); + } + self.accepted_audiences.pickle(out); + self.opaque_token_validation.pickle(out); + if let Some(value) = &self.introspection_client_id { + value.pickle(out); + } + self.introspection_client_secret.pickle(out); + self.introspection_auth_method.pickle(out); + self.unscoped_tokens.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.issuer_url = Pickle::unpickle(stream)?; - this.require_audience = Pickle::unpickle(stream)?; - this.require_scopes = Pickle::unpickle(stream)?; - this.claim_username = Pickle::unpickle(stream)?; - this.username_domain = Pickle::unpickle(stream)?; - this.claim_name = Pickle::unpickle(stream)?; - this.claim_groups = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(0) } else { false }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + "preferred_username".to_string() + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: Map = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: OidcOpaqueTokenValidation = if block.has(9) { + Pickle::unpickle(stream)? + } else { + OidcOpaqueTokenValidation::None + }; + let field10: Option = if block.has(10) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field11: SecretKeyOptional = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: OidcIntrospectionAuth = if block.has(12) { + Pickle::unpickle(stream)? + } else { + OidcIntrospectionAuth::ClientSecretBasic + }; + let field13: OidcUnscopedTokenPolicy = if block.has(13) { + Pickle::unpickle(stream)? + } else { + OidcUnscopedTokenPolicy::Reject + }; + let field14: bool = if block.has(14) { block.flag(6) } else { false }; + let field15: bool = if block.has(15) { block.flag(7) } else { false }; + let field16: bool = if block.has(16) { block.flag(8) } else { false }; + stream.close_block(block); + Some(Self { + description: field0, + issuer_url: field1, + allow_invalid_certs: field2, + claim_username: field3, + username_domain: field4, + claim_name: field5, + claim_groups: field6, + accepted_audiences: field8, + opaque_token_validation: field9, + introspection_client_id: field10, + introspection_client_secret: field11, + introspection_auth_method: field12, + unscoped_tokens: field13, + skip_audience_validation: field14, + allow_id_token_claims: field15, + lenient_issuer_match: field16, + }) } } @@ -30681,32 +39654,89 @@ impl Default for OidcDirectory { Self { description: Default::default(), issuer_url: Default::default(), - require_audience: Default::default(), - require_scopes: Map::new(vec!["openid".to_string(), "email".to_string()]), + allow_invalid_certs: false, claim_username: "preferred_username".to_string(), username_domain: Default::default(), - claim_name: Some("name".to_string()), + claim_name: Default::default(), claim_groups: Default::default(), - member_tenant_id: Default::default(), + accepted_audiences: Default::default(), + opaque_token_validation: OidcOpaqueTokenValidation::None, + introspection_client_id: Default::default(), + introspection_client_secret: Default::default(), + introspection_auth_method: OidcIntrospectionAuth::ClientSecretBasic, + unscoped_tokens: OidcUnscopedTokenPolicy::Reject, + skip_audience_validation: false, + allow_id_token_claims: false, + lenient_issuer_match: false, } } } impl IntoValue for OidcDirectory { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::IssuerUrl, self.issuer_url.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(19); map.insert_unchecked( - Property::RequireAudience, - self.require_audience.into_value(), + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::IssuerUrl, self.issuer_url.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked( + Property::ClaimUsername, + self.claim_username.into_value_with(opts), + ); + map.insert_unchecked( + Property::UsernameDomain, + self.username_domain.into_value_with(opts), + ); + map.insert_unchecked(Property::ClaimName, self.claim_name.into_value_with(opts)); + map.insert_unchecked( + Property::ClaimGroups, + self.claim_groups.into_value_with(opts), + ); + map.insert_unchecked( + Property::AcceptedAudiences, + self.accepted_audiences.into_value_with(opts), + ); + map.insert_unchecked( + Property::OpaqueTokenValidation, + self.opaque_token_validation.into_value_with(opts), + ); + map.insert_unchecked( + Property::IntrospectionClientId, + self.introspection_client_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::IntrospectionClientSecret, + self.introspection_client_secret.into_value_with(opts), + ); + map.insert_unchecked( + Property::IntrospectionAuthMethod, + self.introspection_auth_method.into_value_with(opts), + ); + map.insert_unchecked( + Property::UnscopedTokens, + self.unscoped_tokens.into_value_with(opts), + ); + map.insert_unchecked( + Property::SkipAudienceValidation, + self.skip_audience_validation.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowIdTokenClaims, + self.allow_id_token_claims.into_value_with(opts), + ); + map.insert_unchecked( + Property::LenientIssuerMatch, + self.lenient_issuer_match.into_value_with(opts), ); - map.insert_unchecked(Property::RequireScopes, self.require_scopes.into_value()); - map.insert_unchecked(Property::ClaimUsername, self.claim_username.into_value()); - map.insert_unchecked(Property::UsernameDomain, self.username_domain.into_value()); - map.insert_unchecked(Property::ClaimName, self.claim_name.into_value()); - map.insert_unchecked(Property::ClaimGroups, self.claim_groups.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); JmapValue::Object(map) } } @@ -30724,12 +39754,7 @@ impl RegistryJsonPropertyPatch for OidcDirectory { Some(Property::IssuerUrl) => self .issuer_url .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::RequireAudience) => self - .require_audience - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::RequireScopes) => self - .require_scopes - .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::ClaimUsername) => self .claim_username .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -30740,9 +39765,27 @@ impl RegistryJsonPropertyPatch for OidcDirectory { Some(Property::ClaimGroups) => self .claim_groups .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), + Some(Property::AcceptedAudiences) => self + .accepted_audiences + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::OpaqueTokenValidation) => { + self.opaque_token_validation.patch(pointer, value) + } + Some(Property::IntrospectionClientId) => self + .introspection_client_id + .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::IntrospectionClientSecret) => { + self.introspection_client_secret.patch(pointer, value) + } + Some(Property::IntrospectionAuthMethod) => { + self.introspection_auth_method.patch(pointer, value) + } + Some(Property::UnscopedTokens) => self.unscoped_tokens.patch(pointer, value), + Some(Property::SkipAudienceValidation) => { + self.skip_audience_validation.patch(pointer, value) + } + Some(Property::AllowIdTokenClaims) => self.allow_id_token_claims.patch(pointer, value), + Some(Property::LenientIssuerMatch) => self.lenient_issuer_match.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -30759,14 +39802,34 @@ impl ObjectImpl for OidcProvider { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.auth_code_max_attempts; - if *value < 1 { - errors.push(ValidationError::min_value(Property::AuthCodeMaxAttempts, 1)); + let value = &self.auth_code_expiry; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::AuthCodeExpiry, value)); } - if *value > 1000 { - errors.push(ValidationError::max_value( - Property::AuthCodeMaxAttempts, - 1000, + if *value < Duration::from_millis(600000) { + errors.push(ValidationError::min_value(Property::AuthCodeExpiry, 600000)); + } + let value = &self.refresh_token_expiry; + if !value.is_valid() { + errors.push(ValidationError::invalid( + Property::RefreshTokenExpiry, + value, + )); + } + if *value < Duration::from_millis(86400000) { + errors.push(ValidationError::min_value( + Property::RefreshTokenExpiry, + 86400000, + )); + } + let value = &self.access_token_expiry; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::AccessTokenExpiry, value)); + } + if *value < Duration::from_millis(600000) { + errors.push(ValidationError::min_value( + Property::AccessTokenExpiry, + 600000, )); } let value = &self.encryption_key; @@ -30777,97 +39840,148 @@ impl ObjectImpl for OidcProvider { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Pickle for OidcProvider { fn pickle(&self, out: &mut Vec) { - self.auth_code_max_attempts.pickle(out); - self.anonymous_client_registration.pickle(out); - self.require_client_registration.pickle(out); + let block = crate::pickle::begin_block(out, 9, 0, 0); self.auth_code_expiry.pickle(out); self.refresh_token_expiry.pickle(out); - self.refresh_token_renewal.pickle(out); self.access_token_expiry.pickle(out); self.user_code_expiry.pickle(out); self.id_token_expiry.pickle(out); self.encryption_key.pickle(out); self.signature_algorithm.pickle(out); self.signature_key.pickle(out); + self.client_metadata_documents.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.auth_code_max_attempts = Pickle::unpickle(stream)?; - this.anonymous_client_registration = Pickle::unpickle(stream)?; - this.require_client_registration = Pickle::unpickle(stream)?; - this.auth_code_expiry = Pickle::unpickle(stream)?; - this.refresh_token_expiry = Pickle::unpickle(stream)?; - this.refresh_token_renewal = Pickle::unpickle(stream)?; - this.access_token_expiry = Pickle::unpickle(stream)?; - this.user_code_expiry = Pickle::unpickle(stream)?; - this.id_token_expiry = Pickle::unpickle(stream)?; - this.encryption_key = Pickle::unpickle(stream)?; - this.signature_algorithm = Pickle::unpickle(stream)?; - this.signature_key = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(2592000000) + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1800000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(900000) + }; + let field5: SecretKey = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: JwtSignatureAlgorithm = if block.has(6) { + Pickle::unpickle(stream)? + } else { + JwtSignatureAlgorithm::Hs256 + }; + let field7: SecretText = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: OAuthClientMetadataPolicy = if block.has(8) { + Pickle::unpickle(stream)? + } else { + OAuthClientMetadataPolicy::Disabled + }; + stream.close_block(block); + Some(Self { + auth_code_expiry: field0, + refresh_token_expiry: field1, + access_token_expiry: field2, + user_code_expiry: field3, + id_token_expiry: field4, + encryption_key: field5, + signature_algorithm: field6, + signature_key: field7, + client_metadata_documents: field8, + }) } } impl Default for OidcProvider { fn default() -> Self { Self { - auth_code_max_attempts: 3u64, - anonymous_client_registration: true, - require_client_registration: false, auth_code_expiry: Duration::from_millis(600000), refresh_token_expiry: Duration::from_millis(2592000000), - refresh_token_renewal: Duration::from_millis(345600000), access_token_expiry: Duration::from_millis(3600000), user_code_expiry: Duration::from_millis(1800000), id_token_expiry: Duration::from_millis(900000), encryption_key: Default::default(), signature_algorithm: JwtSignatureAlgorithm::Hs256, signature_key: Default::default(), + client_metadata_documents: OAuthClientMetadataPolicy::Disabled, } } } impl IntoValue for OidcProvider { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(14); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(11); map.insert_unchecked( - Property::AuthCodeMaxAttempts, - self.auth_code_max_attempts.into_value(), + Property::AuthCodeExpiry, + self.auth_code_expiry.into_value_with(opts), ); - map.insert_unchecked( - Property::AnonymousClientRegistration, - self.anonymous_client_registration.into_value(), - ); - map.insert_unchecked( - Property::RequireClientRegistration, - self.require_client_registration.into_value(), - ); - map.insert_unchecked(Property::AuthCodeExpiry, self.auth_code_expiry.into_value()); map.insert_unchecked( Property::RefreshTokenExpiry, - self.refresh_token_expiry.into_value(), - ); - map.insert_unchecked( - Property::RefreshTokenRenewal, - self.refresh_token_renewal.into_value(), + self.refresh_token_expiry.into_value_with(opts), ); map.insert_unchecked( Property::AccessTokenExpiry, - self.access_token_expiry.into_value(), + self.access_token_expiry.into_value_with(opts), + ); + map.insert_unchecked( + Property::UserCodeExpiry, + self.user_code_expiry.into_value_with(opts), + ); + map.insert_unchecked( + Property::IdTokenExpiry, + self.id_token_expiry.into_value_with(opts), + ); + map.insert_unchecked( + Property::EncryptionKey, + self.encryption_key.into_value_with(opts), ); - map.insert_unchecked(Property::UserCodeExpiry, self.user_code_expiry.into_value()); - map.insert_unchecked(Property::IdTokenExpiry, self.id_token_expiry.into_value()); - map.insert_unchecked(Property::EncryptionKey, self.encryption_key.into_value()); map.insert_unchecked( Property::SignatureAlgorithm, - self.signature_algorithm.into_value(), + self.signature_algorithm.into_value_with(opts), + ); + map.insert_unchecked( + Property::SignatureKey, + self.signature_key.into_value_with(opts), + ); + map.insert_unchecked( + Property::ClientMetadataDocuments, + self.client_metadata_documents.into_value_with(opts), ); - map.insert_unchecked(Property::SignatureKey, self.signature_key.into_value()); JmapValue::Object(map) } } @@ -30879,24 +39993,17 @@ impl RegistryJsonPropertyPatch for OidcProvider { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AuthCodeMaxAttempts) => { - self.auth_code_max_attempts.patch(pointer, value) - } - Some(Property::AnonymousClientRegistration) => { - self.anonymous_client_registration.patch(pointer, value) - } - Some(Property::RequireClientRegistration) => { - self.require_client_registration.patch(pointer, value) - } Some(Property::AuthCodeExpiry) => self.auth_code_expiry.patch(pointer, value), Some(Property::RefreshTokenExpiry) => self.refresh_token_expiry.patch(pointer, value), - Some(Property::RefreshTokenRenewal) => self.refresh_token_renewal.patch(pointer, value), Some(Property::AccessTokenExpiry) => self.access_token_expiry.patch(pointer, value), Some(Property::UserCodeExpiry) => self.user_code_expiry.patch(pointer, value), Some(Property::IdTokenExpiry) => self.id_token_expiry.patch(pointer, value), Some(Property::EncryptionKey) => self.encryption_key.patch(pointer, value), Some(Property::SignatureAlgorithm) => self.signature_algorithm.patch(pointer, value), Some(Property::SignatureKey) => self.signature_key.patch(pointer, value), + Some(Property::ClientMetadataDocuments) => { + self.client_metadata_documents.patch(pointer, value) + } Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -30923,20 +40030,6 @@ impl OtpAuth { } } -impl Pickle for OtpAuth { - fn pickle(&self, out: &mut Vec) { - self.otp_code.pickle(out); - self.otp_url.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.otp_code = Pickle::unpickle(stream)?; - this.otp_url = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for OtpAuth { fn default() -> Self { Self { @@ -30948,12 +40041,30 @@ impl Default for OtpAuth { impl IntoValue for OtpAuth { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); if self.otp_code.is_some() { - map.insert_unchecked(Property::OtpCode, secret_value(self.otp_code)); + map.insert_unchecked( + Property::OtpCode, + if opts.include_secrets { + self.otp_code.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } if self.otp_url.is_some() { - map.insert_unchecked(Property::OtpUrl, secret_value(self.otp_url)); + map.insert_unchecked( + Property::OtpUrl, + if opts.include_secrets { + self.otp_url.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } JmapValue::Object(map) } @@ -30988,9 +40099,9 @@ impl PasswordCredential { if value.is_empty() { errors.push(ValidationError::required(Property::Secret)); } - if let Some(value) = &self.otp_auth { + if let Some(value) = &self.otp_url { if value.is_empty() { - errors.push(ValidationError::required(Property::OtpAuth)); + errors.push(ValidationError::required(Property::OtpUrl)); } } if let Some(value) = &self.expires_at { @@ -31010,21 +40121,69 @@ impl PasswordCredential { impl Pickle for PasswordCredential { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.otp_url.is_some() { + bitmap |= 1 << 0; + } + if self.expires_at.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.credential_id.pickle(out); self.secret.pickle(out); - self.otp_auth.pickle(out); - self.expires_at.pickle(out); + if let Some(value) = &self.otp_url { + value.pickle(out); + } + if let Some(value) = &self.expires_at { + value.pickle(out); + } self.allowed_ips.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.credential_id = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.otp_auth = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.allowed_ips = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + credential_id: field0, + secret: field1, + otp_url: field2, + expires_at: field3, + allowed_ips: field4, + }) } } @@ -31033,7 +40192,7 @@ impl Default for PasswordCredential { Self { credential_id: Default::default(), secret: Default::default(), - otp_auth: Default::default(), + otp_url: Default::default(), expires_at: Default::default(), allowed_ips: Default::default(), } @@ -31042,14 +40201,35 @@ impl Default for PasswordCredential { impl IntoValue for PasswordCredential { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::CredentialId, self.credential_id.into_value()); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); - if self.otp_auth.is_some() { - map.insert_unchecked(Property::OtpAuth, secret_value(self.otp_auth)); + map.insert_unchecked( + Property::CredentialId, + self.credential_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); + if self.otp_url.is_some() { + map.insert_unchecked( + Property::OtpUrl, + if opts.include_secrets { + self.otp_url.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); } - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value()); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value_with(opts)); JmapValue::Object(map) } } @@ -31063,7 +40243,7 @@ impl RegistryJsonPropertyPatch for PasswordCredential { match pointer.next_property() { Some(Property::CredentialId) => pointer.assert_server_set(), Some(Property::Secret) => self.secret.patch(pointer, value), - Some(Property::OtpAuth) => self.otp_auth.patch(pointer, value), + Some(Property::OtpUrl) => self.otp_url.patch(pointer, value), Some(Property::ExpiresAt) => self.expires_at.patch(pointer, value), Some(Property::AllowedIps) => self.allowed_ips.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -31120,6 +40300,10 @@ impl Pickle for Permissions { impl IntoValue for Permissions { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Permissions::Inherit => { let mut obj = jmap_tools::Map::new(); @@ -31127,14 +40311,14 @@ impl IntoValue for Permissions { JmapValue::Object(obj) } Permissions::Merge(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Merge".into())); obj } Permissions::Replace(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Replace".into())); @@ -31183,15 +40367,29 @@ impl PermissionsList { impl Pickle for PermissionsList { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.enabled_permissions.pickle(out); self.disabled_permissions.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enabled_permissions = Pickle::unpickle(stream)?; - this.disabled_permissions = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + enabled_permissions: field0, + disabled_permissions: field1, + }) } } @@ -31206,14 +40404,18 @@ impl Default for PermissionsList { impl IntoValue for PermissionsList { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); map.insert_unchecked( Property::EnabledPermissions, - self.enabled_permissions.into_value(), + self.enabled_permissions.into_value_with(opts), ); map.insert_unchecked( Property::DisabledPermissions, - self.disabled_permissions.into_value(), + self.disabled_permissions.into_value_with(opts), ); JmapValue::Object(map) } @@ -31237,118 +40439,6 @@ impl RegistryJsonPropertyPatch for PermissionsList { } } -impl PostgreSqlSettings { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.host; - if value.is_empty() { - errors.push(ValidationError::required(Property::Host)); - } - let value = &self.port; - if *value > 65535 { - errors.push(ValidationError::max_value(Property::Port, 65535)); - } - if *value < 1 { - errors.push(ValidationError::min_value(Property::Port, 1)); - } - let value = &self.database; - if value.is_empty() { - errors.push(ValidationError::required(Property::Database)); - } - if let Some(value) = &self.auth_username { - if value.is_empty() { - errors.push(ValidationError::required(Property::AuthUsername)); - } - } - let value = &self.auth_secret; - value.validate(errors); - if let Some(value) = &self.options { - if value.is_empty() { - errors.push(ValidationError::required(Property::Options)); - } - } - errors.len() == neb - } -} - -impl Pickle for PostgreSqlSettings { - fn pickle(&self, out: &mut Vec) { - self.host.pickle(out); - self.port.pickle(out); - self.database.pickle(out); - self.auth_username.pickle(out); - self.auth_secret.pickle(out); - self.options.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.database = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.options = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for PostgreSqlSettings { - fn default() -> Self { - Self { - host: Default::default(), - port: 5432u64, - database: "inbuxa".to_string(), - auth_username: Some("inbuxa".to_string()), - auth_secret: Default::default(), - options: Default::default(), - } - } -} - -impl IntoValue for PostgreSqlSettings { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Database, self.database.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::Options, self.options.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for PostgreSqlSettings { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Host) => self - .host - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Port) => self.port.patch(pointer, value), - Some(Property::Database) => self - .database - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthUsername) => self - .auth_username - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AuthSecret) => self.auth_secret.patch(pointer, value), - Some(Property::Options) => self - .options - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl PostgreSqlStore { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -31363,10 +40453,6 @@ impl PostgreSqlStore { errors.push(ValidationError::min_value(Property::PoolMaxConnections, 1)); } } - let value = &self.read_replicas; - for value in value.values() { - value.validate(errors); - } let value = &self.host; if value.is_empty() { errors.push(ValidationError::required(Property::Host)); @@ -31396,51 +40482,151 @@ impl PostgreSqlStore { } errors.len() == neb } + } impl Pickle for PostgreSqlStore { fn pickle(&self, out: &mut Vec) { - self.timeout.pickle(out); - self.use_tls.pickle(out); - self.allow_invalid_certs.pickle(out); - self.pool_max_connections.pickle(out); + let mut bitmap = 0u64; + if self.connect_timeout.is_some() { + bitmap |= 1 << 0; + } + if self.use_tls { + bitmap |= 1 << 1; + } + if self.allow_invalid_certs { + bitmap |= 1 << 2; + } + if self.pool_max_connections.is_some() { + bitmap |= 1 << 3; + } + if self.auth_username.is_some() { + bitmap |= 1 << 4; + } + if self.options.is_some() { + bitmap |= 1 << 5; + } + let block = crate::pickle::begin_block(out, 12, 6, bitmap); + if let Some(value) = &self.connect_timeout { + value.pickle(out); + } + if let Some(value) = &self.pool_max_connections { + value.pickle(out); + } self.pool_recycling_method.pickle(out); - self.read_replicas.pickle(out); + #[cfg(not(feature = "enterprise"))] + { + let block = crate::pickle::begin_block(out, 0, 0, 0); + crate::pickle::end_block(out, block); + } self.host.pickle(out); self.port.pickle(out); self.database.pickle(out); - self.auth_username.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.auth_secret.pickle(out); - self.options.pickle(out); + if let Some(value) = &self.options { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.timeout = Pickle::unpickle(stream)?; - this.use_tls = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_recycling_method = Pickle::unpickle(stream)?; - this.read_replicas = Pickle::unpickle(stream)?; - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.database = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.options = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(15000)) + }; + let field1: bool = if block.has(1) { block.flag(1) } else { false }; + let field2: bool = if block.has(2) { block.flag(2) } else { false }; + let field3: Option = if block.has(3) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: PostgreSqlRecyclingMethod = if block.has(4) { + Pickle::unpickle(stream)? + } else { + PostgreSqlRecyclingMethod::Fast + }; + #[cfg(not(feature = "enterprise"))] + if block.has(5) { + let block = stream.open_block()?; + stream.close_block(block); + } + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 5432u64 + }; + let field8: String = if block.has(8) { + Pickle::unpickle(stream)? + } else { + "inbuxa".to_string() + }; + let field9: Option = if block.has(9) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("inbuxa".to_string()) + }; + let field10: SecretKeyOptional = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: Option = if block.has(11) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + connect_timeout: field0, + use_tls: field1, + allow_invalid_certs: field2, + pool_max_connections: field3, + pool_recycling_method: field4, + host: field6, + port: field7, + database: field8, + auth_username: field9, + auth_secret: field10, + options: field11, + }) } } impl Default for PostgreSqlStore { fn default() -> Self { Self { - timeout: Some(Duration::from_millis(15000)), + connect_timeout: Some(Duration::from_millis(15000)), use_tls: false, allow_invalid_certs: false, - pool_max_connections: Some(10u64), + pool_max_connections: Default::default(), pool_recycling_method: PostgreSqlRecyclingMethod::Fast, - read_replicas: Default::default(), host: Default::default(), port: 5432u64, database: "inbuxa".to_string(), @@ -31453,28 +40639,37 @@ impl Default for PostgreSqlStore { impl IntoValue for PostgreSqlStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::UseTls, self.use_tls.into_value()); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked(Property::UseTls, self.use_tls.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolRecyclingMethod, - self.pool_recycling_method.into_value(), + self.pool_recycling_method.into_value_with(opts), ); - map.insert_unchecked(Property::ReadReplicas, self.read_replicas.into_value()); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Database, self.database.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::Options, self.options.into_value()); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Database, self.database.into_value_with(opts)); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); + map.insert_unchecked(Property::Options, self.options.into_value_with(opts)); JmapValue::Object(map) } } @@ -31486,12 +40681,11 @@ impl RegistryJsonPropertyPatch for PostgreSqlStore { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), Some(Property::UseTls) => self.use_tls.patch(pointer, value), Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::PoolMaxConnections) => self.pool_max_connections.patch(pointer, value), Some(Property::PoolRecyclingMethod) => self.pool_recycling_method.patch(pointer, value), - Some(Property::ReadReplicas) => self.read_replicas.patch(pointer, value), Some(Property::Host) => self .host .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -31560,23 +40754,70 @@ impl ObjectImpl for PublicKey { impl Pickle for PublicKey { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.expires_at.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 6, 1, bitmap); self.account_id.pickle(out); self.key.pickle(out); self.description.pickle(out); self.created_at.pickle(out); - self.expires_at.pickle(out); + if let Some(value) = &self.expires_at { + value.pickle(out); + } self.email_addresses.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.account_id = Pickle::unpickle(stream)?; - this.key = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.email_addresses = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Map = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + account_id: field0, + key: field1, + description: field2, + created_at: field3, + expires_at: field4, + email_addresses: field5, + }) } } @@ -31595,13 +40836,23 @@ impl Default for PublicKey { impl IntoValue for PublicKey { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::EmailAddresses, self.email_addresses.into_value()); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::Key, self.key.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::EmailAddresses, + self.email_addresses.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -31643,6 +40894,7 @@ impl PublicStringOptional { PublicStringOptional::File(inner) => inner.validate(errors), } } + } impl Default for PublicStringOptional { @@ -31685,6 +40937,10 @@ impl Pickle for PublicStringOptional { impl IntoValue for PublicStringOptional { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { PublicStringOptional::None => { let mut obj = jmap_tools::Map::new(); @@ -31692,21 +40948,21 @@ impl IntoValue for PublicStringOptional { JmapValue::Object(obj) } PublicStringOptional::Value(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Value".into())); obj } PublicStringOptional::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } PublicStringOptional::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -31771,13 +41027,20 @@ impl PublicStringValue { impl Pickle for PublicStringValue { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.value.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { value: field0 }) } } @@ -31791,8 +41054,12 @@ impl Default for PublicStringValue { impl IntoValue for PublicStringValue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); + map.insert_unchecked(Property::Value, self.value.into_value_with(opts)); JmapValue::Object(map) } } @@ -31824,6 +41091,7 @@ impl PublicText { PublicText::File(inner) => inner.validate(errors), } } + } impl Default for PublicText { @@ -31862,23 +41130,27 @@ impl Pickle for PublicText { impl IntoValue for PublicText { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { PublicText::Text(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Text".into())); obj } PublicText::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } PublicText::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -31934,13 +41206,20 @@ impl PublicTextValue { impl Pickle for PublicTextValue { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.value.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { value: field0 }) } } @@ -31954,8 +41233,12 @@ impl Default for PublicTextValue { impl IntoValue for PublicTextValue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); + map.insert_unchecked(Property::Value, self.value.into_value_with(opts)); JmapValue::Object(map) } } @@ -31992,41 +41275,22 @@ impl Default for QueueExpiry { } } -impl Pickle for QueueExpiry { - fn pickle(&self, out: &mut Vec) { - match self { - QueueExpiry::Ttl(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - QueueExpiry::Attempts(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(QueueExpiry::Ttl), - 1 => Pickle::unpickle(stream).map(QueueExpiry::Attempts), - _ => None, - } - } -} - impl IntoValue for QueueExpiry { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { QueueExpiry::Ttl(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ttl".into())); obj } QueueExpiry::Attempts(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Attempts".into())); @@ -32070,18 +41334,6 @@ impl QueueExpiryAttempts { } } -impl Pickle for QueueExpiryAttempts { - fn pickle(&self, out: &mut Vec) { - self.expires_attempts.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.expires_attempts = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for QueueExpiryAttempts { fn default() -> Self { Self { @@ -32092,10 +41344,14 @@ impl Default for QueueExpiryAttempts { impl IntoValue for QueueExpiryAttempts { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); map.insert_unchecked( Property::ExpiresAttempts, - self.expires_attempts.into_value(), + self.expires_attempts.into_value_with(opts), ); JmapValue::Object(map) } @@ -32129,18 +41385,6 @@ impl QueueExpiryTtl { } } -impl Pickle for QueueExpiryTtl { - fn pickle(&self, out: &mut Vec) { - self.expires_at.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.expires_at = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for QueueExpiryTtl { fn default() -> Self { Self { @@ -32151,8 +41395,12 @@ impl Default for QueueExpiryTtl { impl IntoValue for QueueExpiryTtl { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -32175,7 +41423,7 @@ impl RegistryJsonPropertyPatch for QueueExpiryTtl { } impl ObjectImpl for QueuedMessage { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NOT_STORED | OBJ_NO_CREATE; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::QueuedMessage; @@ -32236,40 +41484,6 @@ impl ObjectImpl for QueuedMessage { fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} } -impl Pickle for QueuedMessage { - fn pickle(&self, out: &mut Vec) { - self.created_at.pickle(out); - self.next_retry.pickle(out); - self.next_notify.pickle(out); - self.blob_id.pickle(out); - self.return_path.pickle(out); - self.recipients.pickle(out); - self.received_from_ip.pickle(out); - self.received_via_port.pickle(out); - self.flags.pickle(out); - self.env_id.pickle(out); - self.priority.pickle(out); - self.size.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.created_at = Pickle::unpickle(stream)?; - this.next_retry = Pickle::unpickle(stream)?; - this.next_notify = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - this.return_path = Pickle::unpickle(stream)?; - this.recipients = Pickle::unpickle(stream)?; - this.received_from_ip = Pickle::unpickle(stream)?; - this.received_via_port = Pickle::unpickle(stream)?; - this.flags = Pickle::unpickle(stream)?; - this.env_id = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - this.size = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for QueuedMessage { fn default() -> Self { Self { @@ -32291,22 +41505,29 @@ impl Default for QueuedMessage { impl IntoValue for QueuedMessage { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::NextRetry, self.next_retry.into_value()); - map.insert_unchecked(Property::NextNotify, self.next_notify.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - map.insert_unchecked(Property::ReturnPath, self.return_path.into_value()); - map.insert_unchecked(Property::Recipients, self.recipients.into_value()); - map.insert_unchecked(Property::ReceivedFromIp, self.received_from_ip.into_value()); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::NextRetry, self.next_retry.into_value_with(opts)); + map.insert_unchecked(Property::NextNotify, self.next_notify.into_value_with(opts)); + map.insert_unchecked(Property::BlobId, self.blob_id.into_value_with(opts)); + map.insert_unchecked(Property::ReturnPath, self.return_path.into_value_with(opts)); + map.insert_unchecked(Property::Recipients, self.recipients.into_value_with(opts)); + map.insert_unchecked( + Property::ReceivedFromIp, + self.received_from_ip.into_value_with(opts), + ); map.insert_unchecked( Property::ReceivedViaPort, - self.received_via_port.into_value(), + self.received_via_port.into_value_with(opts), ); - map.insert_unchecked(Property::Flags, self.flags.into_value()); - map.insert_unchecked(Property::EnvId, self.env_id.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); - map.insert_unchecked(Property::Size, self.size.into_value()); + map.insert_unchecked(Property::Flags, self.flags.into_value_with(opts)); + map.insert_unchecked(Property::EnvId, self.env_id.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); + map.insert_unchecked(Property::Size, self.size.into_value_with(opts)); JmapValue::Object(map) } } @@ -32370,34 +41591,6 @@ impl QueuedRecipient { } } -impl Pickle for QueuedRecipient { - fn pickle(&self, out: &mut Vec) { - self.retry_count.pickle(out); - self.retry_due.pickle(out); - self.notify_count.pickle(out); - self.notify_due.pickle(out); - self.expires.pickle(out); - self.queue_name.pickle(out); - self.status.pickle(out); - self.flags.pickle(out); - self.orcpt.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.retry_count = Pickle::unpickle(stream)?; - this.retry_due = Pickle::unpickle(stream)?; - this.notify_count = Pickle::unpickle(stream)?; - this.notify_due = Pickle::unpickle(stream)?; - this.expires = Pickle::unpickle(stream)?; - this.queue_name = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - this.flags = Pickle::unpickle(stream)?; - this.orcpt = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for QueuedRecipient { fn default() -> Self { Self { @@ -32416,16 +41609,23 @@ impl Default for QueuedRecipient { impl IntoValue for QueuedRecipient { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::RetryCount, self.retry_count.into_value()); - map.insert_unchecked(Property::RetryDue, self.retry_due.into_value()); - map.insert_unchecked(Property::NotifyCount, self.notify_count.into_value()); - map.insert_unchecked(Property::NotifyDue, self.notify_due.into_value()); - map.insert_unchecked(Property::Expires, self.expires.into_value()); - map.insert_unchecked(Property::QueueName, self.queue_name.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); - map.insert_unchecked(Property::Flags, self.flags.into_value()); - map.insert_unchecked(Property::Orcpt, self.orcpt.into_value()); + map.insert_unchecked(Property::RetryCount, self.retry_count.into_value_with(opts)); + map.insert_unchecked(Property::RetryDue, self.retry_due.into_value_with(opts)); + map.insert_unchecked( + Property::NotifyCount, + self.notify_count.into_value_with(opts), + ); + map.insert_unchecked(Property::NotifyDue, self.notify_due.into_value_with(opts)); + map.insert_unchecked(Property::Expires, self.expires.into_value_with(opts)); + map.insert_unchecked(Property::QueueName, self.queue_name.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); + map.insert_unchecked(Property::Flags, self.flags.into_value_with(opts)); + map.insert_unchecked(Property::Orcpt, self.orcpt.into_value_with(opts)); JmapValue::Object(map) } } @@ -32478,15 +41678,29 @@ impl Rate { impl Pickle for Rate { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.count.pickle(out); self.period.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.count = Pickle::unpickle(stream)?; - this.period = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(0) + }; + stream.close_block(block); + Some(Self { + count: field0, + period: field1, + }) } } @@ -32501,9 +41715,13 @@ impl Default for Rate { impl IntoValue for Rate { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Count, self.count.into_value()); - map.insert_unchecked(Property::Period, self.period.into_value()); + map.insert_unchecked(Property::Count, self.count.into_value_with(opts)); + map.insert_unchecked(Property::Period, self.period.into_value_with(opts)); JmapValue::Object(map) } } @@ -32543,40 +41761,12 @@ impl Default for RecipientStatus { } } -impl Pickle for RecipientStatus { - fn pickle(&self, out: &mut Vec) { - match self { - RecipientStatus::Scheduled => { - 0u16.pickle(out); - } - RecipientStatus::Completed(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - RecipientStatus::TemporaryFailure(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - RecipientStatus::PermanentFailure(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(RecipientStatus::Scheduled), - 1 => Pickle::unpickle(stream).map(RecipientStatus::Completed), - 2 => Pickle::unpickle(stream).map(RecipientStatus::TemporaryFailure), - 3 => Pickle::unpickle(stream).map(RecipientStatus::PermanentFailure), - _ => None, - } - } -} - impl IntoValue for RecipientStatus { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { RecipientStatus::Scheduled => { let mut obj = jmap_tools::Map::new(); @@ -32584,21 +41774,21 @@ impl IntoValue for RecipientStatus { JmapValue::Object(obj) } RecipientStatus::Completed(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Completed".into())); obj } RecipientStatus::TemporaryFailure(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TemporaryFailure".into())); obj } RecipientStatus::PermanentFailure(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PermanentFailure".into())); @@ -32706,41 +41896,178 @@ impl RedisClusterStore { } errors.len() == neb } + } impl Pickle for RedisClusterStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.auth_username.is_some() { + bitmap |= 1 << 1; + } + if self.max_retry_wait.is_some() { + bitmap |= 1 << 2; + } + if self.min_retry_wait.is_some() { + bitmap |= 1 << 3; + } + if self.max_retries.is_some() { + bitmap |= 1 << 4; + } + if self.read_from_replicas { + bitmap |= 1 << 5; + } + if self.pool_timeout_create.is_some() { + bitmap |= 1 << 6; + } + if self.pool_timeout_wait.is_some() { + bitmap |= 1 << 7; + } + if self.pool_timeout_recycle.is_some() { + bitmap |= 1 << 8; + } + let block = crate::pickle::begin_block(out, 14, 9, bitmap); self.urls.pickle(out); - self.timeout.pickle(out); - self.auth_username.pickle(out); + self.connect_timeout.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.auth_secret.pickle(out); - self.max_retry_wait.pickle(out); - self.min_retry_wait.pickle(out); - self.max_retries.pickle(out); - self.read_from_replicas.pickle(out); + if let Some(value) = &self.max_retry_wait { + value.pickle(out); + } + if let Some(value) = &self.min_retry_wait { + value.pickle(out); + } + if let Some(value) = &self.max_retries { + value.pickle(out); + } self.protocol_version.pickle(out); self.pool_max_connections.pickle(out); - self.pool_timeout_create.pickle(out); - self.pool_timeout_wait.pickle(out); - self.pool_timeout_recycle.pickle(out); + if let Some(value) = &self.pool_timeout_create { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_wait { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_recycle { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.urls = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.max_retry_wait = Pickle::unpickle(stream)?; - this.min_retry_wait = Pickle::unpickle(stream)?; - this.max_retries = Pickle::unpickle(stream)?; - this.read_from_replicas = Pickle::unpickle(stream)?; - this.protocol_version = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_timeout_create = Pickle::unpickle(stream)?; - this.pool_timeout_wait = Pickle::unpickle(stream)?; - this.pool_timeout_recycle = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["redis://127.0.0.1".to_string()]) + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field2: bool = if block.has(2) { block.flag(0) } else { false }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: SecretKeyOptional = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: bool = if block.has(8) { block.flag(5) } else { true }; + let field9: RedisProtocol = if block.has(9) { + Pickle::unpickle(stream)? + } else { + RedisProtocol::Resp2 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field11: Option = if block.has(11) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field12: Option = if block.has(12) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field13: Option = if block.has(13) { + if block.flag(8) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + stream.close_block(block); + Some(Self { + urls: field0, + connect_timeout: field1, + allow_invalid_certs: field2, + auth_username: field3, + auth_secret: field4, + max_retry_wait: field5, + min_retry_wait: field6, + max_retries: field7, + read_from_replicas: field8, + protocol_version: field9, + pool_max_connections: field10, + pool_timeout_create: field11, + pool_timeout_wait: field12, + pool_timeout_recycle: field13, + }) } } @@ -32748,8 +42075,9 @@ impl Default for RedisClusterStore { fn default() -> Self { Self { urls: Map::new(vec!["redis://127.0.0.1".to_string()]), - timeout: Duration::from_millis(10000), - auth_username: Some("inbuxa".to_string()), + connect_timeout: Duration::from_millis(10000), + allow_invalid_certs: false, + auth_username: Default::default(), auth_secret: Default::default(), max_retry_wait: Default::default(), min_retry_wait: Default::default(), @@ -32766,37 +42094,57 @@ impl Default for RedisClusterStore { impl IntoValue for RedisClusterStore { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(15); - map.insert_unchecked(Property::Urls, self.urls.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); - map.insert_unchecked(Property::MaxRetryWait, self.max_retry_wait.into_value()); - map.insert_unchecked(Property::MinRetryWait, self.min_retry_wait.into_value()); - map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); + map.insert_unchecked(Property::Urls, self.urls.into_value_with(opts)); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); + map.insert_unchecked( + Property::MaxRetryWait, + self.max_retry_wait.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinRetryWait, + self.min_retry_wait.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value_with(opts)); map.insert_unchecked( Property::ReadFromReplicas, - self.read_from_replicas.into_value(), + self.read_from_replicas.into_value_with(opts), ); map.insert_unchecked( Property::ProtocolVersion, - self.protocol_version.into_value(), + self.protocol_version.into_value_with(opts), ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutCreate, - self.pool_timeout_create.into_value(), + self.pool_timeout_create.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutWait, - self.pool_timeout_wait.into_value(), + self.pool_timeout_wait.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutRecycle, - self.pool_timeout_recycle.into_value(), + self.pool_timeout_recycle.into_value_with(opts), ); JmapValue::Object(map) } @@ -32812,7 +42160,8 @@ impl RegistryJsonPropertyPatch for RedisClusterStore { Some(Property::Urls) => self .urls .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::AuthUsername) => self .auth_username .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -32874,39 +42223,155 @@ impl RedisSentinelStore { } errors.len() == neb } + } impl Pickle for RedisSentinelStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.auth_username.is_some() { + bitmap |= 1 << 1; + } + if self.sentinel_username.is_some() { + bitmap |= 1 << 2; + } + if self.pool_timeout_create.is_some() { + bitmap |= 1 << 3; + } + if self.pool_timeout_wait.is_some() { + bitmap |= 1 << 4; + } + if self.pool_timeout_recycle.is_some() { + bitmap |= 1 << 5; + } + let block = crate::pickle::begin_block(out, 13, 6, bitmap); self.urls.pickle(out); self.service_name.pickle(out); - self.timeout.pickle(out); - self.auth_username.pickle(out); + self.connect_timeout.pickle(out); + if let Some(value) = &self.auth_username { + value.pickle(out); + } self.auth_secret.pickle(out); - self.sentinel_username.pickle(out); + if let Some(value) = &self.sentinel_username { + value.pickle(out); + } self.sentinel_secret.pickle(out); self.protocol_version.pickle(out); self.pool_max_connections.pickle(out); - self.pool_timeout_create.pickle(out); - self.pool_timeout_wait.pickle(out); - self.pool_timeout_recycle.pickle(out); + if let Some(value) = &self.pool_timeout_create { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_wait { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_recycle { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.urls = Pickle::unpickle(stream)?; - this.service_name = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.auth_username = Pickle::unpickle(stream)?; - this.auth_secret = Pickle::unpickle(stream)?; - this.sentinel_username = Pickle::unpickle(stream)?; - this.sentinel_secret = Pickle::unpickle(stream)?; - this.protocol_version = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_timeout_create = Pickle::unpickle(stream)?; - this.pool_timeout_wait = Pickle::unpickle(stream)?; - this.pool_timeout_recycle = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["redis://127.0.0.1:26379".to_string()]) + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + "mymaster".to_string() + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: SecretKeyOptional = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Option = if block.has(6) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: SecretKeyOptional = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: RedisProtocol = if block.has(8) { + Pickle::unpickle(stream)? + } else { + RedisProtocol::Resp2 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field10: Option = if block.has(10) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field11: Option = if block.has(11) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field12: Option = if block.has(12) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + stream.close_block(block); + Some(Self { + urls: field0, + service_name: field1, + connect_timeout: field2, + allow_invalid_certs: field3, + auth_username: field4, + auth_secret: field5, + sentinel_username: field6, + sentinel_secret: field7, + protocol_version: field8, + pool_max_connections: field9, + pool_timeout_create: field10, + pool_timeout_wait: field11, + pool_timeout_recycle: field12, + }) } } @@ -32915,8 +42380,9 @@ impl Default for RedisSentinelStore { Self { urls: Map::new(vec!["redis://127.0.0.1:26379".to_string()]), service_name: "mymaster".to_string(), - timeout: Duration::from_millis(10000), - auth_username: Some("inbuxa".to_string()), + connect_timeout: Duration::from_millis(10000), + allow_invalid_certs: false, + auth_username: Default::default(), auth_secret: Default::default(), sentinel_username: Default::default(), sentinel_secret: Default::default(), @@ -32931,36 +42397,56 @@ impl Default for RedisSentinelStore { impl IntoValue for RedisSentinelStore { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Urls, self.urls.into_value()); - map.insert_unchecked(Property::ServiceName, self.service_name.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::AuthUsername, self.auth_username.into_value()); - map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(15); + map.insert_unchecked(Property::Urls, self.urls.into_value_with(opts)); + map.insert_unchecked( + Property::ServiceName, + self.service_name.into_value_with(opts), + ); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked( + Property::AuthUsername, + self.auth_username.into_value_with(opts), + ); + map.insert_unchecked(Property::AuthSecret, self.auth_secret.into_value_with(opts)); map.insert_unchecked( Property::SentinelUsername, - self.sentinel_username.into_value(), + self.sentinel_username.into_value_with(opts), + ); + map.insert_unchecked( + Property::SentinelSecret, + self.sentinel_secret.into_value_with(opts), ); - map.insert_unchecked(Property::SentinelSecret, self.sentinel_secret.into_value()); map.insert_unchecked( Property::ProtocolVersion, - self.protocol_version.into_value(), + self.protocol_version.into_value_with(opts), ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutCreate, - self.pool_timeout_create.into_value(), + self.pool_timeout_create.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutWait, - self.pool_timeout_wait.into_value(), + self.pool_timeout_wait.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutRecycle, - self.pool_timeout_recycle.into_value(), + self.pool_timeout_recycle.into_value_with(opts), ); JmapValue::Object(map) } @@ -32979,7 +42465,8 @@ impl RegistryJsonPropertyPatch for RedisSentinelStore { Some(Property::ServiceName) => self .service_name .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::AuthUsername) => self .auth_username .patch(pointer.with_validators(&[StringValidator::Trim]), value), @@ -33025,23 +42512,90 @@ impl RedisStore { impl Pickle for RedisStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.pool_timeout_create.is_some() { + bitmap |= 1 << 1; + } + if self.pool_timeout_wait.is_some() { + bitmap |= 1 << 2; + } + if self.pool_timeout_recycle.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 7, 4, bitmap); self.url.pickle(out); - self.timeout.pickle(out); + self.connect_timeout.pickle(out); self.pool_max_connections.pickle(out); - self.pool_timeout_create.pickle(out); - self.pool_timeout_wait.pickle(out); - self.pool_timeout_recycle.pickle(out); + if let Some(value) = &self.pool_timeout_create { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_wait { + value.pickle(out); + } + if let Some(value) = &self.pool_timeout_recycle { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.url = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - this.pool_timeout_create = Pickle::unpickle(stream)?; - this.pool_timeout_wait = Pickle::unpickle(stream)?; - this.pool_timeout_recycle = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + "redis://127.0.0.1".to_string() + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field2: bool = if block.has(2) { block.flag(0) } else { false }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field5: Option = if block.has(5) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + let field6: Option = if block.has(6) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(30000)) + }; + stream.close_block(block); + Some(Self { + url: field0, + connect_timeout: field1, + allow_invalid_certs: field2, + pool_max_connections: field3, + pool_timeout_create: field4, + pool_timeout_wait: field5, + pool_timeout_recycle: field6, + }) } } @@ -33049,7 +42603,8 @@ impl Default for RedisStore { fn default() -> Self { Self { url: "redis://127.0.0.1".to_string(), - timeout: Duration::from_millis(10000), + connect_timeout: Duration::from_millis(10000), + allow_invalid_certs: false, pool_max_connections: 10u64, pool_timeout_create: Some(Duration::from_millis(30000)), pool_timeout_wait: Some(Duration::from_millis(30000)), @@ -33060,24 +42615,35 @@ impl Default for RedisStore { impl IntoValue for RedisStore { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(9); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked( + Property::ConnectTimeout, + self.connect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutCreate, - self.pool_timeout_create.into_value(), + self.pool_timeout_create.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutWait, - self.pool_timeout_wait.into_value(), + self.pool_timeout_wait.into_value_with(opts), ); map.insert_unchecked( Property::PoolTimeoutRecycle, - self.pool_timeout_recycle.into_value(), + self.pool_timeout_recycle.into_value_with(opts), ); JmapValue::Object(map) } @@ -33093,7 +42659,8 @@ impl RegistryJsonPropertyPatch for RedisStore { Some(Property::Url) => self .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::ConnectTimeout) => self.connect_timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::PoolMaxConnections) => self.pool_max_connections.patch(pointer, value), Some(Property::PoolTimeoutCreate) => self.pool_timeout_create.patch(pointer, value), Some(Property::PoolTimeoutWait) => self.pool_timeout_wait.patch(pointer, value), @@ -33107,9 +42674,336 @@ impl RegistryJsonPropertyPatch for RedisStore { } } +impl RedisSwap { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.store; + value.validate(errors); + let value = &self.chunk_size; + if *value < 65536 { + errors.push(ValidationError::min_value(Property::ChunkSize, 65536)); + } + let value = &self.flush_changes; + if *value < 1 { + errors.push(ValidationError::min_value(Property::FlushChanges, 1)); + } + let value = &self.max_account_size; + if *value < 2048 { + errors.push(ValidationError::min_value(Property::MaxAccountSize, 2048)); + } + errors.len() == neb + } + +} + +impl Pickle for RedisSwap { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 7, 0, 0); + self.store.pickle(out); + self.chunk_size.pickle(out); + self.retention.pickle(out); + self.flush_changes.pickle(out); + self.flush_interval.pickle(out); + self.max_account_size.pickle(out); + self.min_account_size.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: RedisSwapStore = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(604800000) + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 2000u64 + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(600000) + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 1073741824 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 1048576 + }; + stream.close_block(block); + Some(Self { + store: field0, + chunk_size: field1, + retention: field2, + flush_changes: field3, + flush_interval: field4, + max_account_size: field5, + min_account_size: field6, + }) + } +} + +impl Default for RedisSwap { + fn default() -> Self { + Self { + store: Default::default(), + chunk_size: 1048576, + retention: Duration::from_millis(604800000), + flush_changes: 2000u64, + flush_interval: Duration::from_millis(600000), + max_account_size: 1073741824, + min_account_size: 1048576, + } + } +} + +impl IntoValue for RedisSwap { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(9); + map.insert_unchecked(Property::Store, self.store.into_value_with(opts)); + map.insert_unchecked(Property::ChunkSize, self.chunk_size.into_value_with(opts)); + map.insert_unchecked(Property::Retention, self.retention.into_value_with(opts)); + map.insert_unchecked( + Property::FlushChanges, + self.flush_changes.into_value_with(opts), + ); + map.insert_unchecked( + Property::FlushInterval, + self.flush_interval.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAccountSize, + self.max_account_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinAccountSize, + self.min_account_size.into_value_with(opts), + ); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for RedisSwap { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Store) => self.store.patch(pointer, value), + Some(Property::ChunkSize) => self.chunk_size.patch(pointer, value), + Some(Property::Retention) => self.retention.patch(pointer, value), + Some(Property::FlushChanges) => self.flush_changes.patch(pointer, value), + Some(Property::FlushInterval) => self.flush_interval.patch(pointer, value), + Some(Property::MaxAccountSize) => self.max_account_size.patch(pointer, value), + Some(Property::MinAccountSize) => self.min_account_size.patch(pointer, value), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + +impl RedisSwapStore { + fn validate(&self, errors: &mut Vec) -> bool { + match self { + RedisSwapStore::Default => true, + RedisSwapStore::Redis(inner) => inner.validate(errors), + RedisSwapStore::RedisCluster(inner) => inner.validate(errors), + RedisSwapStore::RedisSentinel(inner) => inner.validate(errors), + } + } + +} + +impl Default for RedisSwapStore { + fn default() -> Self { + RedisSwapStore::Default + } +} + +impl Pickle for RedisSwapStore { + fn pickle(&self, out: &mut Vec) { + match self { + RedisSwapStore::Default => { + 0u16.pickle(out); + } + RedisSwapStore::Redis(inner) => { + 1u16.pickle(out); + inner.pickle(out); + } + RedisSwapStore::RedisCluster(inner) => { + 2u16.pickle(out); + inner.pickle(out); + } + RedisSwapStore::RedisSentinel(inner) => { + 3u16.pickle(out); + inner.pickle(out); + } + } + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + match u16::unpickle(stream)? { + 0 => Some(RedisSwapStore::Default), + 1 => Pickle::unpickle(stream).map(RedisSwapStore::Redis), + 2 => Pickle::unpickle(stream).map(RedisSwapStore::RedisCluster), + 3 => Pickle::unpickle(stream).map(RedisSwapStore::RedisSentinel), + _ => None, + } + } +} + +impl IntoValue for RedisSwapStore { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + match self { + RedisSwapStore::Default => { + let mut obj = jmap_tools::Map::new(); + obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); + JmapValue::Object(obj) + } + RedisSwapStore::Redis(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("Redis".into())); + obj + } + RedisSwapStore::RedisCluster(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("RedisCluster".into())); + obj + } + RedisSwapStore::RedisSentinel(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("RedisSentinel".into())); + obj + } + } + } +} + +impl RegistryJsonPatch for RedisSwapStore { + fn patch<'x>( + &mut self, + pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + if !pointer.has_next() { + match object_type(&pointer, &value)? { + RedisSwapStoreType::Default => *self = RedisSwapStore::Default, + RedisSwapStoreType::Redis => *self = RedisSwapStore::Redis(Default::default()), + RedisSwapStoreType::RedisCluster => { + *self = RedisSwapStore::RedisCluster(Default::default()) + } + RedisSwapStoreType::RedisSentinel => { + *self = RedisSwapStore::RedisSentinel(Default::default()) + } + } + } + match self { + RedisSwapStore::Default => pointer.assert_eof(), + RedisSwapStore::Redis(inner) => inner.patch(pointer, value), + RedisSwapStore::RedisCluster(inner) => inner.patch(pointer, value), + RedisSwapStore::RedisSentinel(inner) => inner.patch(pointer, value), + } + } +} + +impl RedisSwapStore { + pub fn object_type(&self) -> RedisSwapStoreType { + match self { + RedisSwapStore::Default => RedisSwapStoreType::Default, + RedisSwapStore::Redis(_) => RedisSwapStoreType::Redis, + RedisSwapStore::RedisCluster(_) => RedisSwapStoreType::RedisCluster, + RedisSwapStore::RedisSentinel(_) => RedisSwapStoreType::RedisSentinel, + } + } +} + +impl ReloadSettingsResult { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.warnings; + for value in value.iter() { + if value.is_empty() { + errors.push(ValidationError::required(Property::Warnings)); + } + } + errors.len() == neb + } +} + +impl Default for ReloadSettingsResult { + fn default() -> Self { + Self { + warnings: Default::default(), + } + } +} + +impl IntoValue for ReloadSettingsResult { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(3); + map.insert_unchecked(Property::Warnings, self.warnings.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for ReloadSettingsResult { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::Warnings) => pointer.assert_server_set(), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + impl ObjectImpl for ReportSettings { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::ReportSettings; fn validate(&self, errors: &mut Vec) -> bool { @@ -33127,10 +43021,10 @@ impl ObjectImpl for ReportSettings { } let value = &self.outbound_report_submitter; value.validate(errors); - let value = &self.inbound_report_max_size; - if *value < (1024) { + let value = &self.max_inbound_report_size; + if *value < 1024 { errors.push(ValidationError::min_value( - Property::InboundReportMaxSize, + Property::MaxInboundReportSize, 1024, )); } @@ -33161,23 +43055,64 @@ impl ReportSettings { impl Pickle for ReportSettings { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.inbound_report_forwarding { + bitmap |= 1 << 0; + } + if self.outbound_report_domain.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.inbound_report_addresses.pickle(out); - self.inbound_report_forwarding.pickle(out); - self.outbound_report_domain.pickle(out); + if let Some(value) = &self.outbound_report_domain { + value.pickle(out); + } self.outbound_report_submitter.pickle(out); - self.inbound_report_max_size.pickle(out); + self.max_inbound_report_size.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.inbound_report_addresses = Pickle::unpickle(stream)?; - this.inbound_report_forwarding = Pickle::unpickle(stream)?; - this.outbound_report_domain = Pickle::unpickle(stream)?; - this.outbound_report_submitter = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.inbound_report_max_size = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["postmaster@*".to_string()]) + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('hostname')".to_string(), + ..Default::default() + } + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 26214400u64 + }; + stream.close_block(block); + Some(Self { + inbound_report_addresses: field0, + inbound_report_forwarding: field1, + outbound_report_domain: field2, + outbound_report_submitter: field3, + max_inbound_report_size: field4, + }) } } @@ -33191,33 +43126,37 @@ impl Default for ReportSettings { else_: "system('hostname')".to_string(), ..Default::default() }, - inbound_report_max_size: 26214400i64, + max_inbound_report_size: 26214400u64, } } } impl IntoValue for ReportSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); map.insert_unchecked( Property::InboundReportAddresses, - self.inbound_report_addresses.into_value(), + self.inbound_report_addresses.into_value_with(opts), ); map.insert_unchecked( Property::InboundReportForwarding, - self.inbound_report_forwarding.into_value(), + self.inbound_report_forwarding.into_value_with(opts), ); map.insert_unchecked( Property::OutboundReportDomain, - self.outbound_report_domain.into_value(), + self.outbound_report_domain.into_value_with(opts), ); map.insert_unchecked( Property::OutboundReportSubmitter, - self.outbound_report_submitter.into_value(), + self.outbound_report_submitter.into_value_with(opts), ); map.insert_unchecked( - Property::InboundReportMaxSize, - self.inbound_report_max_size.into_value(), + Property::MaxInboundReportSize, + self.max_inbound_report_size.into_value_with(opts), ); JmapValue::Object(map) } @@ -33242,8 +43181,8 @@ impl RegistryJsonPropertyPatch for ReportSettings { Some(Property::OutboundReportSubmitter) => { self.outbound_report_submitter.patch(pointer, value) } - Some(Property::InboundReportMaxSize) => { - self.inbound_report_max_size.patch(pointer, value) + Some(Property::MaxInboundReportSize) => { + self.max_inbound_report_size.patch(pointer, value) } Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -33283,59 +43222,45 @@ impl RocksDbStore { errors.push(ValidationError::min_value(Property::PoolWorkers, 1)); } } - let value = &self.cache_size; - if *value > 17179869184 { - errors.push(ValidationError::max_value(Property::CacheSize, 17179869184)); - } - if *value < 8388608 { - errors.push(ValidationError::min_value(Property::CacheSize, 8388608)); + if let Some(value) = &self.cache_size { + if *value > 17179869184 { + errors.push(ValidationError::max_value(Property::CacheSize, 17179869184)); + } + if *value < 8388608 { + errors.push(ValidationError::min_value(Property::CacheSize, 8388608)); + } } errors.len() == neb } } -impl Pickle for RocksDbStore { - fn pickle(&self, out: &mut Vec) { - self.path.pickle(out); - self.blob_size.pickle(out); - self.buffer_size.pickle(out); - self.pool_workers.pickle(out); - self.cache_size.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.path = Pickle::unpickle(stream)?; - this.blob_size = Pickle::unpickle(stream)?; - this.buffer_size = Pickle::unpickle(stream)?; - this.pool_workers = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.cache_size = Pickle::unpickle(stream)?; - } - Some(this) - } -} - impl Default for RocksDbStore { fn default() -> Self { Self { path: Default::default(), - blob_size: 16834u64, + blob_size: 4096u64, buffer_size: 134217728u64, pool_workers: Default::default(), - cache_size: 134217728u64, + cache_size: Default::default(), } } } impl IntoValue for RocksDbStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Path, self.path.into_value()); - map.insert_unchecked(Property::BlobSize, self.blob_size.into_value()); - map.insert_unchecked(Property::BufferSize, self.buffer_size.into_value()); - map.insert_unchecked(Property::PoolWorkers, self.pool_workers.into_value()); - map.insert_unchecked(Property::CacheSize, self.cache_size.into_value()); + map.insert_unchecked(Property::Path, self.path.into_value_with(opts)); + map.insert_unchecked(Property::BlobSize, self.blob_size.into_value_with(opts)); + map.insert_unchecked(Property::BufferSize, self.buffer_size.into_value_with(opts)); + map.insert_unchecked( + Property::PoolWorkers, + self.pool_workers.into_value_with(opts), + ); + map.insert_unchecked(Property::CacheSize, self.cache_size.into_value_with(opts)); JmapValue::Object(map) } } @@ -33374,11 +43299,6 @@ impl ObjectImpl for Role { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.role_ids; for value in value.iter() { if !value.is_valid() { @@ -33390,10 +43310,6 @@ impl ObjectImpl for Role { fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { i.text(Property::Description, &self.description); - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } for id in self.role_ids.iter() { i.foreign_key(ObjectType::Role, Some(*id), None); } @@ -33402,21 +43318,48 @@ impl ObjectImpl for Role { impl Pickle for Role { fn pickle(&self, out: &mut Vec) { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut bitmap = 0u64; + let block = crate::pickle::begin_block(out, 5, 1, bitmap); self.description.pickle(out); - self.member_tenant_id.pickle(out); self.role_ids.pickle(out); self.enabled_permissions.pickle(out); self.disabled_permissions.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.role_ids = Pickle::unpickle(stream)?; - this.enabled_permissions = Pickle::unpickle(stream)?; - this.disabled_permissions = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + description: field0, + role_ids: field2, + enabled_permissions: field3, + disabled_permissions: field4, + }) } } @@ -33424,7 +43367,6 @@ impl Default for Role { fn default() -> Self { Self { description: Default::default(), - member_tenant_id: Default::default(), role_ids: Default::default(), enabled_permissions: Default::default(), disabled_permissions: Default::default(), @@ -33434,17 +43376,23 @@ impl Default for Role { impl IntoValue for Role { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::RoleIds, self.role_ids.into_value()); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::RoleIds, self.role_ids.into_value_with(opts)); map.insert_unchecked( Property::EnabledPermissions, - self.enabled_permissions.into_value(), + self.enabled_permissions.into_value_with(opts), ); map.insert_unchecked( Property::DisabledPermissions, - self.disabled_permissions.into_value(), + self.disabled_permissions.into_value_with(opts), ); JmapValue::Object(map) } @@ -33458,9 +43406,6 @@ impl RegistryJsonPropertyPatch for Role { ) -> PatchResult<'x> { match pointer.next_property() { Some(Property::Description) => self.description.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::RoleIds) => self.role_ids.patch(pointer, value), Some(Property::EnabledPermissions) => self.enabled_permissions.patch(pointer, value), Some(Property::DisabledPermissions) => self.disabled_permissions.patch(pointer, value), @@ -33521,6 +43466,10 @@ impl Pickle for Roles { impl IntoValue for Roles { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Roles::Default => { let mut obj = jmap_tools::Map::new(); @@ -33528,7 +43477,7 @@ impl IntoValue for Roles { JmapValue::Object(obj) } Roles::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); @@ -33575,12 +43524,10 @@ impl S3Store { if value.is_empty() { errors.push(ValidationError::required(Property::Bucket)); } - let value = &self.access_key; + let value = &self.access_key_id; value.validate(errors); let value = &self.secret_key; value.validate(errors); - let value = &self.security_token; - value.validate(errors); let value = &self.session_token; value.validate(errors); if let Some(value) = &self.profile { @@ -33592,9 +43539,6 @@ impl S3Store { if *value > 10 { errors.push(ValidationError::max_value(Property::MaxRetries, 10)); } - if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxRetries, 1)); - } if let Some(value) = &self.key_prefix { if value.is_empty() { errors.push(ValidationError::required(Property::KeyPrefix)); @@ -33602,41 +43546,112 @@ impl S3Store { } errors.len() == neb } + } impl Pickle for S3Store { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.profile.is_some() { + bitmap |= 1 << 0; + } + if self.key_prefix.is_some() { + bitmap |= 1 << 1; + } + if self.allow_invalid_certs { + bitmap |= 1 << 2; + } + if self.verify_after_write { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 11, 4, bitmap); self.region.pickle(out); self.bucket.pickle(out); - self.access_key.pickle(out); + self.access_key_id.pickle(out); self.secret_key.pickle(out); - self.security_token.pickle(out); self.session_token.pickle(out); - self.profile.pickle(out); + if let Some(value) = &self.profile { + value.pickle(out); + } self.timeout.pickle(out); self.max_retries.pickle(out); - self.key_prefix.pickle(out); - self.allow_invalid_certs.pickle(out); - self.verify_after_write.pickle(out); + if let Some(value) = &self.key_prefix { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.region = Pickle::unpickle(stream)?; - this.bucket = Pickle::unpickle(stream)?; - this.access_key = Pickle::unpickle(stream)?; - this.secret_key = Pickle::unpickle(stream)?; - this.security_token = Pickle::unpickle(stream)?; - this.session_token = Pickle::unpickle(stream)?; - this.profile = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.max_retries = Pickle::unpickle(stream)?; - this.key_prefix = Pickle::unpickle(stream)?; - this.allow_invalid_certs = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.verify_after_write = Pickle::unpickle(stream)?; - } - Some(this) + let block = stream.open_block()?; + let field0: S3StoreRegion = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: PublicStringOptional = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: SecretKeyOptional = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: SecretKeyOptional = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Option = if block.has(5) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Duration = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field8: Option = if block.has(8) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field9: bool = if block.has(9) { block.flag(2) } else { false }; + let field10: bool = if block.has(10) { block.flag(3) } else { true }; + stream.close_block(block); + Some(Self { + region: field0, + bucket: field1, + access_key_id: field2, + secret_key: field3, + session_token: field4, + profile: field5, + timeout: field6, + max_retries: field7, + key_prefix: field8, + allow_invalid_certs: field9, + verify_after_write: field10, + }) } } @@ -33645,9 +43660,8 @@ impl Default for S3Store { Self { region: Default::default(), bucket: Default::default(), - access_key: Default::default(), + access_key_id: Default::default(), secret_key: Default::default(), - security_token: Default::default(), session_token: Default::default(), profile: Default::default(), timeout: Duration::from_millis(30000), @@ -33661,24 +43675,33 @@ impl Default for S3Store { impl IntoValue for S3Store { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Region, self.region.into_value()); - map.insert_unchecked(Property::Bucket, self.bucket.into_value()); - map.insert_unchecked(Property::AccessKey, self.access_key.into_value()); - map.insert_unchecked(Property::SecretKey, self.secret_key.into_value()); - map.insert_unchecked(Property::SecurityToken, self.security_token.into_value()); - map.insert_unchecked(Property::SessionToken, self.session_token.into_value()); - map.insert_unchecked(Property::Profile, self.profile.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value()); - map.insert_unchecked(Property::KeyPrefix, self.key_prefix.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(13); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); + map.insert_unchecked(Property::Bucket, self.bucket.into_value_with(opts)); + map.insert_unchecked( + Property::AccessKeyId, + self.access_key_id.into_value_with(opts), + ); + map.insert_unchecked(Property::SecretKey, self.secret_key.into_value_with(opts)); + map.insert_unchecked( + Property::SessionToken, + self.session_token.into_value_with(opts), + ); + map.insert_unchecked(Property::Profile, self.profile.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::MaxRetries, self.max_retries.into_value_with(opts)); + map.insert_unchecked(Property::KeyPrefix, self.key_prefix.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), ); map.insert_unchecked( Property::VerifyAfterWrite, - self.verify_after_write.into_value(), + self.verify_after_write.into_value_with(opts), ); JmapValue::Object(map) } @@ -33695,9 +43718,8 @@ impl RegistryJsonPropertyPatch for S3Store { Some(Property::Bucket) => self .bucket .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::AccessKey) => self.access_key.patch(pointer, value), + Some(Property::AccessKeyId) => self.access_key_id.patch(pointer, value), Some(Property::SecretKey) => self.secret_key.patch(pointer, value), - Some(Property::SecurityToken) => self.security_token.patch(pointer, value), Some(Property::SessionToken) => self.session_token.patch(pointer, value), Some(Property::Profile) => self .profile @@ -33721,13 +43743,13 @@ impl RegistryJsonPropertyPatch for S3Store { impl S3StoreCustomRegion { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.custom_endpoint; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::CustomEndpoint)); + errors.push(ValidationError::required(Property::Url)); } - let value = &self.custom_region; + let value = &self.region; if value.is_empty() { - errors.push(ValidationError::required(Property::CustomRegion)); + errors.push(ValidationError::required(Property::Region)); } errors.len() == neb } @@ -33735,32 +43757,50 @@ impl S3StoreCustomRegion { impl Pickle for S3StoreCustomRegion { fn pickle(&self, out: &mut Vec) { - self.custom_endpoint.pickle(out); - self.custom_region.pickle(out); + let block = crate::pickle::begin_block(out, 2, 0, 0); + self.url.pickle(out); + self.region.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.custom_endpoint = Pickle::unpickle(stream)?; - this.custom_region = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + url: field0, + region: field1, + }) } } impl Default for S3StoreCustomRegion { fn default() -> Self { Self { - custom_endpoint: Default::default(), - custom_region: Default::default(), + url: Default::default(), + region: Default::default(), } } } impl IntoValue for S3StoreCustomRegion { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::CustomEndpoint, self.custom_endpoint.into_value()); - map.insert_unchecked(Property::CustomRegion, self.custom_region.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Region, self.region.into_value_with(opts)); JmapValue::Object(map) } } @@ -33772,11 +43812,11 @@ impl RegistryJsonPropertyPatch for S3StoreCustomRegion { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::CustomEndpoint) => self - .custom_endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::CustomRegion) => self - .custom_region + Some(Property::Region) => self + .region .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -34031,6 +44071,10 @@ impl Pickle for S3StoreRegion { impl IntoValue for S3StoreRegion { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { S3StoreRegion::UsEast1 => { let mut obj = jmap_tools::Map::new(); @@ -34243,7 +44287,7 @@ impl IntoValue for S3StoreRegion { JmapValue::Object(obj) } S3StoreRegion::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); @@ -34415,6 +44459,57 @@ impl ObjectImpl for Search { if *value < 1 { errors.push(ValidationError::min_value(Property::IndexBatchSize, 1)); } + let value = &self.max_concurrent_index_tasks; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxConcurrentIndexTasks, + 1, + )); + } + let value = &self.max_extract_document_size; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxExtractDocumentSize, + 1, + )); + } + let value = &self.max_extract_text_size; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxExtractTextSize, 1)); + } + let value = &self.max_extract_decompressed_size; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxExtractDecompressedSize, + 1, + )); + } + let value = &self.max_extract_part_size; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxExtractPartSize, 1)); + } + let value = &self.max_extract_parts; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxExtractParts, 1)); + } + let value = &self.max_extract_archive_entries; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxExtractArchiveEntries, + 1, + )); + } + let value = &self.max_extract_pdf_objects; + if *value < 1 { + errors.push(ValidationError::min_value( + Property::MaxExtractPdfObjects, + 1, + )); + } + let value = &self.max_extract_rtf_depth; + if *value < 1 { + errors.push(ValidationError::min_value(Property::MaxExtractRtfDepth, 1)); + } errors.len() == neb } @@ -34423,33 +44518,169 @@ impl ObjectImpl for Search { impl Pickle for Search { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.index_calendar { + bitmap |= 1 << 0; + } + if self.index_contacts { + bitmap |= 1 << 1; + } + if self.index_email { + bitmap |= 1 << 2; + } + if self.index_files { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 20, 5, bitmap); self.index_batch_size.pickle(out); self.default_language.pickle(out); self.supported_languages.pickle(out); - self.index_calendar.pickle(out); self.index_calendar_fields.pickle(out); - self.index_contacts.pickle(out); self.index_contact_fields.pickle(out); - self.index_email.pickle(out); self.index_email_fields.pickle(out); - self.index_telemetry.pickle(out); - self.index_tracing_fields.pickle(out); + self.max_concurrent_index_tasks.pickle(out); + self.max_extract_document_size.pickle(out); + self.max_extract_text_size.pickle(out); + self.max_extract_decompressed_size.pickle(out); + self.max_extract_part_size.pickle(out); + self.max_extract_parts.pickle(out); + self.max_extract_archive_entries.pickle(out); + self.max_extract_pdf_objects.pickle(out); + self.max_extract_rtf_depth.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.index_batch_size = Pickle::unpickle(stream)?; - this.default_language = Pickle::unpickle(stream)?; - this.supported_languages = Pickle::unpickle(stream)?; - this.index_calendar = Pickle::unpickle(stream)?; - this.index_calendar_fields = Pickle::unpickle(stream)?; - this.index_contacts = Pickle::unpickle(stream)?; - this.index_contact_fields = Pickle::unpickle(stream)?; - this.index_email = Pickle::unpickle(stream)?; - this.index_email_fields = Pickle::unpickle(stream)?; - this.index_telemetry = Pickle::unpickle(stream)?; - this.index_tracing_fields = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field1: Locale = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Locale::EnUS + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![Locale::EnUS]) + }; + let field3: bool = if block.has(3) { block.flag(0) } else { true }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + SearchCalendarField::Title, + SearchCalendarField::Description, + SearchCalendarField::Location, + SearchCalendarField::Owner, + SearchCalendarField::Attendee, + ]) + }; + let field5: bool = if block.has(5) { block.flag(1) } else { true }; + let field6: Map = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + SearchContactField::Member, + SearchContactField::Kind, + SearchContactField::Name, + SearchContactField::Nickname, + SearchContactField::Organization, + SearchContactField::Email, + SearchContactField::Phone, + SearchContactField::OnlineService, + SearchContactField::Address, + SearchContactField::Note, + ]) + }; + let field7: bool = if block.has(7) { block.flag(2) } else { true }; + let field8: Map = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + SearchEmailField::From, + SearchEmailField::To, + SearchEmailField::Cc, + SearchEmailField::Bcc, + SearchEmailField::Subject, + SearchEmailField::Body, + SearchEmailField::Attachment, + ]) + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + let field11: bool = if block.has(11) { block.flag(4) } else { true }; + let field12: u64 = if block.has(12) { + Pickle::unpickle(stream)? + } else { + 67108864u64 + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 4194304u64 + }; + let field14: u64 = if block.has(14) { + Pickle::unpickle(stream)? + } else { + 268435456u64 + }; + let field15: u64 = if block.has(15) { + Pickle::unpickle(stream)? + } else { + 67108864u64 + }; + let field16: u64 = if block.has(16) { + Pickle::unpickle(stream)? + } else { + 10000u64 + }; + let field17: u64 = if block.has(17) { + Pickle::unpickle(stream)? + } else { + 10000u64 + }; + let field18: u64 = if block.has(18) { + Pickle::unpickle(stream)? + } else { + 1048576u64 + }; + let field19: u64 = if block.has(19) { + Pickle::unpickle(stream)? + } else { + 256u64 + }; + stream.close_block(block); + Some(Self { + index_batch_size: field0, + default_language: field1, + supported_languages: field2, + index_calendar: field3, + index_calendar_fields: field4, + index_contacts: field5, + index_contact_fields: field6, + index_email: field7, + index_email_fields: field8, + max_concurrent_index_tasks: field10, + index_files: field11, + max_extract_document_size: field12, + max_extract_text_size: field13, + max_extract_decompressed_size: field14, + max_extract_part_size: field15, + max_extract_parts: field16, + max_extract_archive_entries: field17, + max_extract_pdf_objects: field18, + max_extract_rtf_depth: field19, + }) } } @@ -34466,8 +44697,6 @@ impl Default for Search { SearchCalendarField::Location, SearchCalendarField::Owner, SearchCalendarField::Attendee, - SearchCalendarField::Start, - SearchCalendarField::Uid, ]), index_contacts: true, index_contact_fields: Map::new(vec![ @@ -34481,7 +44710,6 @@ impl Default for Search { SearchContactField::OnlineService, SearchContactField::Address, SearchContactField::Note, - SearchContactField::Uid, ]), index_email: true, index_email_fields: Map::new(vec![ @@ -34492,52 +44720,97 @@ impl Default for Search { SearchEmailField::Subject, SearchEmailField::Body, SearchEmailField::Attachment, - SearchEmailField::ReceivedAt, - SearchEmailField::SentAt, - SearchEmailField::Size, - SearchEmailField::HasAttachment, - ]), - index_telemetry: true, - index_tracing_fields: Map::new(vec![ - SearchTracingField::EventType, - SearchTracingField::QueueId, - SearchTracingField::Keywords, ]), + max_concurrent_index_tasks: 5u64, + index_files: true, + max_extract_document_size: 67108864u64, + max_extract_text_size: 4194304u64, + max_extract_decompressed_size: 268435456u64, + max_extract_part_size: 67108864u64, + max_extract_parts: 10000u64, + max_extract_archive_entries: 10000u64, + max_extract_pdf_objects: 1048576u64, + max_extract_rtf_depth: 256u64, } } } impl IntoValue for Search { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::IndexBatchSize, self.index_batch_size.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(22); + map.insert_unchecked( + Property::IndexBatchSize, + self.index_batch_size.into_value_with(opts), + ); map.insert_unchecked( Property::DefaultLanguage, - self.default_language.into_value(), + self.default_language.into_value_with(opts), ); map.insert_unchecked( Property::SupportedLanguages, - self.supported_languages.into_value(), + self.supported_languages.into_value_with(opts), + ); + map.insert_unchecked( + Property::IndexCalendar, + self.index_calendar.into_value_with(opts), ); - map.insert_unchecked(Property::IndexCalendar, self.index_calendar.into_value()); map.insert_unchecked( Property::IndexCalendarFields, - self.index_calendar_fields.into_value(), + self.index_calendar_fields.into_value_with(opts), + ); + map.insert_unchecked( + Property::IndexContacts, + self.index_contacts.into_value_with(opts), ); - map.insert_unchecked(Property::IndexContacts, self.index_contacts.into_value()); map.insert_unchecked( Property::IndexContactFields, - self.index_contact_fields.into_value(), + self.index_contact_fields.into_value_with(opts), ); - map.insert_unchecked(Property::IndexEmail, self.index_email.into_value()); + map.insert_unchecked(Property::IndexEmail, self.index_email.into_value_with(opts)); map.insert_unchecked( Property::IndexEmailFields, - self.index_email_fields.into_value(), + self.index_email_fields.into_value_with(opts), ); - map.insert_unchecked(Property::IndexTelemetry, self.index_telemetry.into_value()); map.insert_unchecked( - Property::IndexTracingFields, - self.index_tracing_fields.into_value(), + Property::MaxConcurrentIndexTasks, + self.max_concurrent_index_tasks.into_value_with(opts), + ); + map.insert_unchecked(Property::IndexFiles, self.index_files.into_value_with(opts)); + map.insert_unchecked( + Property::MaxExtractDocumentSize, + self.max_extract_document_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractTextSize, + self.max_extract_text_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractDecompressedSize, + self.max_extract_decompressed_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractPartSize, + self.max_extract_part_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractParts, + self.max_extract_parts.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractArchiveEntries, + self.max_extract_archive_entries.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractPdfObjects, + self.max_extract_pdf_objects.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxExtractRtfDepth, + self.max_extract_rtf_depth.into_value_with(opts), ); JmapValue::Object(map) } @@ -34559,8 +44832,26 @@ impl RegistryJsonPropertyPatch for Search { Some(Property::IndexContactFields) => self.index_contact_fields.patch(pointer, value), Some(Property::IndexEmail) => self.index_email.patch(pointer, value), Some(Property::IndexEmailFields) => self.index_email_fields.patch(pointer, value), - Some(Property::IndexTelemetry) => self.index_telemetry.patch(pointer, value), - Some(Property::IndexTracingFields) => self.index_tracing_fields.patch(pointer, value), + Some(Property::MaxConcurrentIndexTasks) => { + self.max_concurrent_index_tasks.patch(pointer, value) + } + Some(Property::IndexFiles) => self.index_files.patch(pointer, value), + Some(Property::MaxExtractDocumentSize) => { + self.max_extract_document_size.patch(pointer, value) + } + Some(Property::MaxExtractTextSize) => self.max_extract_text_size.patch(pointer, value), + Some(Property::MaxExtractDecompressedSize) => { + self.max_extract_decompressed_size.patch(pointer, value) + } + Some(Property::MaxExtractPartSize) => self.max_extract_part_size.patch(pointer, value), + Some(Property::MaxExtractParts) => self.max_extract_parts.patch(pointer, value), + Some(Property::MaxExtractArchiveEntries) => { + self.max_extract_archive_entries.patch(pointer, value) + } + Some(Property::MaxExtractPdfObjects) => { + self.max_extract_pdf_objects.patch(pointer, value) + } + Some(Property::MaxExtractRtfDepth) => self.max_extract_rtf_depth.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -34587,6 +44878,7 @@ impl ObjectImpl for SearchStore { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for SearchStore { @@ -34625,6 +44917,9 @@ impl Pickle for SearchStore { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Some(SearchStore::Default), 1 => Pickle::unpickle(stream).map(SearchStore::ElasticSearch), @@ -34639,6 +44934,10 @@ impl Pickle for SearchStore { impl IntoValue for SearchStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SearchStore::Default => { let mut obj = jmap_tools::Map::new(); @@ -34646,35 +44945,35 @@ impl IntoValue for SearchStore { JmapValue::Object(obj) } SearchStore::ElasticSearch(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ElasticSearch".into())); obj } SearchStore::Meilisearch(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Meilisearch".into())); obj } SearchStore::FoundationDb(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); obj } SearchStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } SearchStore::MySql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); @@ -34768,25 +45067,74 @@ impl SecondaryCredential { impl Pickle for SecondaryCredential { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.expires_at.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.credential_id.pickle(out); self.description.pickle(out); self.secret.pickle(out); self.created_at.pickle(out); - self.expires_at.pickle(out); + if let Some(value) = &self.expires_at { + value.pickle(out); + } self.permissions.pickle(out); self.allowed_ips.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.credential_id = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.secret = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.allowed_ips = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: CredentialPermissions = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: Map = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + credential_id: field0, + description: field1, + secret: field2, + created_at: field3, + expires_at: field4, + permissions: field5, + allowed_ips: field6, + }) } } @@ -34806,14 +45154,34 @@ impl Default for SecondaryCredential { impl IntoValue for SecondaryCredential { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::CredentialId, self.credential_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value()); + map.insert_unchecked( + Property::CredentialId, + self.credential_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); + map.insert_unchecked(Property::AllowedIps, self.allowed_ips.into_value_with(opts)); JmapValue::Object(map) } } @@ -34849,6 +45217,7 @@ impl SecretKey { SecretKey::File(inner) => inner.validate(errors), } } + } impl Default for SecretKey { @@ -34887,23 +45256,27 @@ impl Pickle for SecretKey { impl IntoValue for SecretKey { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SecretKey::Value(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Value".into())); obj } SecretKey::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } SecretKey::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -34959,13 +45332,22 @@ impl SecretKeyEnvironmentVariable { impl Pickle for SecretKeyEnvironmentVariable { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.variable_name.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.variable_name = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + variable_name: field0, + }) } } @@ -34979,8 +45361,15 @@ impl Default for SecretKeyEnvironmentVariable { impl IntoValue for SecretKeyEnvironmentVariable { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::VariableName, self.variable_name.into_value()); + map.insert_unchecked( + Property::VariableName, + self.variable_name.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -35015,13 +45404,20 @@ impl SecretKeyFile { impl Pickle for SecretKeyFile { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.file_path.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.file_path = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { file_path: field0 }) } } @@ -35035,8 +45431,12 @@ impl Default for SecretKeyFile { impl IntoValue for SecretKeyFile { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::FilePath, self.file_path.into_value()); + map.insert_unchecked(Property::FilePath, self.file_path.into_value_with(opts)); JmapValue::Object(map) } } @@ -35067,6 +45467,7 @@ impl SecretKeyOptional { SecretKeyOptional::File(inner) => inner.validate(errors), } } + } impl Default for SecretKeyOptional { @@ -35109,6 +45510,10 @@ impl Pickle for SecretKeyOptional { impl IntoValue for SecretKeyOptional { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SecretKeyOptional::None => { let mut obj = jmap_tools::Map::new(); @@ -35116,21 +45521,21 @@ impl IntoValue for SecretKeyOptional { JmapValue::Object(obj) } SecretKeyOptional::Value(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Value".into())); obj } SecretKeyOptional::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } SecretKeyOptional::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -35191,13 +45596,20 @@ impl SecretKeyValue { impl Pickle for SecretKeyValue { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.secret.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.secret = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { secret: field0 }) } } @@ -35211,8 +45623,19 @@ impl Default for SecretKeyValue { impl IntoValue for SecretKeyValue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); JmapValue::Object(map) } } @@ -35242,6 +45665,7 @@ impl SecretText { SecretText::File(inner) => inner.validate(errors), } } + } impl Default for SecretText { @@ -35280,23 +45704,27 @@ impl Pickle for SecretText { impl IntoValue for SecretText { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SecretText::Text(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Text".into())); obj } SecretText::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } SecretText::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -35348,6 +45776,7 @@ impl SecretTextOptional { SecretTextOptional::File(inner) => inner.validate(errors), } } + } impl Default for SecretTextOptional { @@ -35390,6 +45819,10 @@ impl Pickle for SecretTextOptional { impl IntoValue for SecretTextOptional { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SecretTextOptional::None => { let mut obj = jmap_tools::Map::new(); @@ -35397,21 +45830,21 @@ impl IntoValue for SecretTextOptional { JmapValue::Object(obj) } SecretTextOptional::Text(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Text".into())); obj } SecretTextOptional::EnvironmentVariable(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("EnvironmentVariable".into())); obj } SecretTextOptional::File(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("File".into())); @@ -35476,13 +45909,20 @@ impl SecretTextValue { impl Pickle for SecretTextValue { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.secret.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.secret = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { secret: field0 }) } } @@ -35496,8 +45936,19 @@ impl Default for SecretTextValue { impl IntoValue for SecretTextValue { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Secret, secret_value(self.secret)); + map.insert_unchecked( + Property::Secret, + if opts.include_secrets { + self.secret.into_value_with(opts) + } else { + JmapValue::Str(MASKED_PASSWORD.into()) + }, + ); JmapValue::Object(map) } } @@ -35552,29 +46003,179 @@ impl ObjectImpl for Security { impl Pickle for Security { fn pickle(&self, out: &mut Vec) { - self.abuse_ban_rate.pickle(out); - self.abuse_ban_period.pickle(out); - self.auth_ban_rate.pickle(out); - self.auth_ban_period.pickle(out); - self.loiter_ban_rate.pickle(out); - self.loiter_ban_period.pickle(out); + let mut bitmap = 0u64; + if self.abuse_ban_rate.is_some() { + bitmap |= 1 << 0; + } + if self.abuse_ban_period.is_some() { + bitmap |= 1 << 1; + } + if self.auth_ban_rate.is_some() { + bitmap |= 1 << 2; + } + if self.auth_ban_period.is_some() { + bitmap |= 1 << 3; + } + if self.loiter_ban_rate.is_some() { + bitmap |= 1 << 4; + } + if self.loiter_ban_period.is_some() { + bitmap |= 1 << 5; + } + if self.scan_ban_rate.is_some() { + bitmap |= 1 << 6; + } + if self.scan_ban_period.is_some() { + bitmap |= 1 << 7; + } + let block = crate::pickle::begin_block(out, 9, 8, bitmap); + if let Some(value) = &self.abuse_ban_rate { + value.pickle(out); + } + if let Some(value) = &self.abuse_ban_period { + value.pickle(out); + } + if let Some(value) = &self.auth_ban_rate { + value.pickle(out); + } + if let Some(value) = &self.auth_ban_period { + value.pickle(out); + } + if let Some(value) = &self.loiter_ban_rate { + value.pickle(out); + } + if let Some(value) = &self.loiter_ban_period { + value.pickle(out); + } self.scan_ban_paths.pickle(out); - self.scan_ban_rate.pickle(out); - self.scan_ban_period.pickle(out); + if let Some(value) = &self.scan_ban_rate { + value.pickle(out); + } + if let Some(value) = &self.scan_ban_period { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.abuse_ban_rate = Pickle::unpickle(stream)?; - this.abuse_ban_period = Pickle::unpickle(stream)?; - this.auth_ban_rate = Pickle::unpickle(stream)?; - this.auth_ban_period = Pickle::unpickle(stream)?; - this.loiter_ban_rate = Pickle::unpickle(stream)?; - this.loiter_ban_period = Pickle::unpickle(stream)?; - this.scan_ban_paths = Pickle::unpickle(stream)?; - this.scan_ban_rate = Pickle::unpickle(stream)?; - this.scan_ban_period = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 35u64, + period: Duration::from_millis(86400000), + }) + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 100u64, + period: Duration::from_millis(86400000), + }) + }; + let field3: Option = if block.has(3) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 150u64, + period: Duration::from_millis(86400000), + }) + }; + let field5: Option = if block.has(5) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field6: Map = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + "*.php*".to_string(), + "*.cgi*".to_string(), + "*.asp*".to_string(), + "*/wp-*".to_string(), + "*/php*".to_string(), + "*/cgi-bin*".to_string(), + "*xmlrpc*".to_string(), + "*../*".to_string(), + "*/..*".to_string(), + "*joomla*".to_string(), + "*wordpress*".to_string(), + "*drupal*".to_string(), + ]) + }; + let field7: Option = if block.has(7) { + if block.flag(6) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Rate { + count: 30u64, + period: Duration::from_millis(86400000), + }) + }; + let field8: Option = if block.has(8) { + if block.flag(7) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + abuse_ban_rate: field0, + abuse_ban_period: field1, + auth_ban_rate: field2, + auth_ban_period: field3, + loiter_ban_rate: field4, + loiter_ban_period: field5, + scan_ban_paths: field6, + scan_ban_rate: field7, + scan_ban_period: field8, + }) } } @@ -35621,19 +46222,47 @@ impl Default for Security { impl IntoValue for Security { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AbuseBanRate, self.abuse_ban_rate.into_value()); - map.insert_unchecked(Property::AbuseBanPeriod, self.abuse_ban_period.into_value()); - map.insert_unchecked(Property::AuthBanRate, self.auth_ban_rate.into_value()); - map.insert_unchecked(Property::AuthBanPeriod, self.auth_ban_period.into_value()); - map.insert_unchecked(Property::LoiterBanRate, self.loiter_ban_rate.into_value()); + map.insert_unchecked( + Property::AbuseBanRate, + self.abuse_ban_rate.into_value_with(opts), + ); + map.insert_unchecked( + Property::AbuseBanPeriod, + self.abuse_ban_period.into_value_with(opts), + ); + map.insert_unchecked( + Property::AuthBanRate, + self.auth_ban_rate.into_value_with(opts), + ); + map.insert_unchecked( + Property::AuthBanPeriod, + self.auth_ban_period.into_value_with(opts), + ); + map.insert_unchecked( + Property::LoiterBanRate, + self.loiter_ban_rate.into_value_with(opts), + ); map.insert_unchecked( Property::LoiterBanPeriod, - self.loiter_ban_period.into_value(), + self.loiter_ban_period.into_value_with(opts), + ); + map.insert_unchecked( + Property::ScanBanPaths, + self.scan_ban_paths.into_value_with(opts), + ); + map.insert_unchecked( + Property::ScanBanRate, + self.scan_ban_rate.into_value_with(opts), + ); + map.insert_unchecked( + Property::ScanBanPeriod, + self.scan_ban_period.into_value_with(opts), ); - map.insert_unchecked(Property::ScanBanPaths, self.scan_ban_paths.into_value()); - map.insert_unchecked(Property::ScanBanRate, self.scan_ban_rate.into_value()); - map.insert_unchecked(Property::ScanBanPeriod, self.scan_ban_period.into_value()); JmapValue::Object(map) } } @@ -35672,8 +46301,6 @@ impl ObjectImpl for SenderAuth { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.dkim_sign_domain; - value.validate(errors); let value = &self.dkim_verify; value.validate(errors); let value = &self.spf_ehlo_verify; @@ -35693,23 +46320,6 @@ impl ObjectImpl for SenderAuth { } impl SenderAuth { - pub fn ctx_dkim_sign_domain(&self) -> ExpressionContext<'_> { - ExpressionContext { - expr: &self.dkim_sign_domain, - default: Some(Expression { - else_: "false".to_string(), - match_: List::from_iter([ExpressionMatch { - if_: "is_local_domain(sender_domain) && !is_empty(authenticated_as)" - .to_string(), - then: "sender_domain".to_string(), - }]), - }), - property: Property::DkimSignDomain, - allowed_variables: MTA_RCPT_TO_VARIABLE, - allowed_constants: &[], - } - } - pub fn ctx_dkim_verify(&self) -> ExpressionContext<'_> { ExpressionContext { expr: &self.dkim_verify, @@ -35802,7 +46412,6 @@ impl SenderAuth { pub fn expression_ctxs(&self) -> Vec> { vec![ - self.ctx_dkim_sign_domain(), self.ctx_dkim_verify(), self.ctx_spf_ehlo_verify(), self.ctx_spf_from_verify(), @@ -35815,41 +46424,102 @@ impl SenderAuth { impl Pickle for SenderAuth { fn pickle(&self, out: &mut Vec) { - self.dkim_sign_domain.pickle(out); - self.dkim_strict.pickle(out); + let mut bitmap = 0u64; + if self.dkim_strict { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.dkim_verify.pickle(out); self.spf_ehlo_verify.pickle(out); self.spf_from_verify.pickle(out); self.arc_verify.pickle(out); self.dmarc_verify.pickle(out); self.reverse_ip_verify.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.dkim_sign_domain = Pickle::unpickle(stream)?; - this.dkim_strict = Pickle::unpickle(stream)?; - this.dkim_verify = Pickle::unpickle(stream)?; - this.spf_ehlo_verify = Pickle::unpickle(stream)?; - this.spf_from_verify = Pickle::unpickle(stream)?; - this.arc_verify = Pickle::unpickle(stream)?; - this.dmarc_verify = Pickle::unpickle(stream)?; - this.reverse_ip_verify = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "relaxed".to_string(), + match_: List::from_iter([]), + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "disable".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "relaxed".to_string(), + }]), + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "disable".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "relaxed".to_string(), + }]), + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "disable".to_string(), + match_: List::from_iter([]), + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "disable".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "relaxed".to_string(), + }]), + } + }; + let field6: Expression = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "disable".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "local_port == 25".to_string(), + then: "relaxed".to_string(), + }]), + } + }; + stream.close_block(block); + Some(Self { + dkim_strict: field0, + dkim_verify: field1, + spf_ehlo_verify: field2, + spf_from_verify: field3, + arc_verify: field4, + dmarc_verify: field5, + reverse_ip_verify: field6, + }) } } impl Default for SenderAuth { fn default() -> Self { Self { - dkim_sign_domain: Expression { - else_: "false".to_string(), - match_: List::from_iter([ExpressionMatch { - if_: "is_local_domain(sender_domain) && !is_empty(authenticated_as)" - .to_string(), - then: "sender_domain".to_string(), - }]), - }, dkim_strict: true, dkim_verify: Expression { else_: "relaxed".to_string(), @@ -35893,17 +46563,29 @@ impl Default for SenderAuth { impl IntoValue for SenderAuth { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); - map.insert_unchecked(Property::DkimStrict, self.dkim_strict.into_value()); - map.insert_unchecked(Property::DkimVerify, self.dkim_verify.into_value()); - map.insert_unchecked(Property::SpfEhloVerify, self.spf_ehlo_verify.into_value()); - map.insert_unchecked(Property::SpfFromVerify, self.spf_from_verify.into_value()); - map.insert_unchecked(Property::ArcVerify, self.arc_verify.into_value()); - map.insert_unchecked(Property::DmarcVerify, self.dmarc_verify.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(9); + map.insert_unchecked(Property::DkimStrict, self.dkim_strict.into_value_with(opts)); + map.insert_unchecked(Property::DkimVerify, self.dkim_verify.into_value_with(opts)); + map.insert_unchecked( + Property::SpfEhloVerify, + self.spf_ehlo_verify.into_value_with(opts), + ); + map.insert_unchecked( + Property::SpfFromVerify, + self.spf_from_verify.into_value_with(opts), + ); + map.insert_unchecked(Property::ArcVerify, self.arc_verify.into_value_with(opts)); + map.insert_unchecked( + Property::DmarcVerify, + self.dmarc_verify.into_value_with(opts), + ); map.insert_unchecked( Property::ReverseIpVerify, - self.reverse_ip_verify.into_value(), + self.reverse_ip_verify.into_value_with(opts), ); JmapValue::Object(map) } @@ -35916,7 +46598,6 @@ impl RegistryJsonPropertyPatch for SenderAuth { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::DkimSignDomain) => self.dkim_sign_domain.patch(pointer, value), Some(Property::DkimStrict) => self.dkim_strict.patch(pointer, value), Some(Property::DkimVerify) => self.dkim_verify.patch(pointer, value), Some(Property::SpfEhloVerify) => self.spf_ehlo_verify.patch(pointer, value), @@ -35963,24 +46644,6 @@ impl ServerResponse { } } -impl Pickle for ServerResponse { - fn pickle(&self, out: &mut Vec) { - self.response_hostname.pickle(out); - self.response_code.pickle(out); - self.response_enhanced.pickle(out); - self.response_message.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.response_hostname = Pickle::unpickle(stream)?; - this.response_code = Pickle::unpickle(stream)?; - this.response_enhanced = Pickle::unpickle(stream)?; - this.response_message = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for ServerResponse { fn default() -> Self { Self { @@ -35994,19 +46657,26 @@ impl Default for ServerResponse { impl IntoValue for ServerResponse { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); map.insert_unchecked( Property::ResponseHostname, - self.response_hostname.into_value(), + self.response_hostname.into_value_with(opts), + ); + map.insert_unchecked( + Property::ResponseCode, + self.response_code.into_value_with(opts), ); - map.insert_unchecked(Property::ResponseCode, self.response_code.into_value()); map.insert_unchecked( Property::ResponseEnhanced, - self.response_enhanced.into_value(), + self.response_enhanced.into_value_with(opts), ); map.insert_unchecked( Property::ResponseMessage, - self.response_message.into_value(), + self.response_message.into_value_with(opts), ); JmapValue::Object(map) } @@ -36046,15 +46716,37 @@ impl Service { impl Pickle for Service { fn pickle(&self, out: &mut Vec) { - self.hostname.pickle(out); - self.cleartext.pickle(out); + let mut bitmap = 0u64; + if self.hostname.is_some() { + bitmap |= 1 << 0; + } + if self.cleartext { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 2, 2, bitmap); + if let Some(value) = &self.hostname { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.hostname = Pickle::unpickle(stream)?; - this.cleartext = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(1) } else { false }; + stream.close_block(block); + Some(Self { + hostname: field0, + cleartext: field1, + }) } } @@ -36069,9 +46761,13 @@ impl Default for Service { impl IntoValue for Service { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Hostname, self.hostname.into_value()); - map.insert_unchecked(Property::Cleartext, self.cleartext.into_value()); + map.insert_unchecked(Property::Hostname, self.hostname.into_value_with(opts)); + map.insert_unchecked(Property::Cleartext, self.cleartext.into_value_with(opts)); JmapValue::Object(map) } } @@ -36096,124 +46792,6 @@ impl RegistryJsonPropertyPatch for Service { } } -impl ShardedBlobStore { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.stores; - for value in value.values() { - value.validate(errors); - } - if value.len() < 2 { - errors.push(ValidationError::min_items(Property::Stores, 2)); - } - errors.len() == neb - } -} - -impl Pickle for ShardedBlobStore { - fn pickle(&self, out: &mut Vec) { - self.stores.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.stores = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ShardedBlobStore { - fn default() -> Self { - Self { - stores: Default::default(), - } - } -} - -impl IntoValue for ShardedBlobStore { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Stores, self.stores.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ShardedBlobStore { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Stores) => self.stores.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ShardedInMemoryStore { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.stores; - for value in value.values() { - value.validate(errors); - } - if value.len() < 2 { - errors.push(ValidationError::min_items(Property::Stores, 2)); - } - errors.len() == neb - } -} - -impl Pickle for ShardedInMemoryStore { - fn pickle(&self, out: &mut Vec) { - self.stores.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.stores = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for ShardedInMemoryStore { - fn default() -> Self { - Self { - stores: Default::default(), - } - } -} - -impl IntoValue for ShardedInMemoryStore { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Stores, self.stores.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for ShardedInMemoryStore { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Stores) => self.stores.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for Sharing { const FLAGS: u64 = OBJ_SINGLETON; const VERSION: u8 = 0; @@ -36233,15 +46811,31 @@ impl ObjectImpl for Sharing { impl Pickle for Sharing { fn pickle(&self, out: &mut Vec) { - self.allow_directory_queries.pickle(out); + let mut bitmap = 0u64; + if self.allow_directory_queries { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 2, 1, bitmap); self.max_shares.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_directory_queries = Pickle::unpickle(stream)?; - this.max_shares = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { false }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + stream.close_block(block); + Some(Self { + allow_directory_queries: field0, + max_shares: field1, + }) } } @@ -36256,12 +46850,16 @@ impl Default for Sharing { impl IntoValue for Sharing { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); map.insert_unchecked( Property::AllowDirectoryQueries, - self.allow_directory_queries.into_value(), + self.allow_directory_queries.into_value_with(opts), ); - map.insert_unchecked(Property::MaxShares, self.max_shares.into_value()); + map.insert_unchecked(Property::MaxShares, self.max_shares.into_value_with(opts)); JmapValue::Object(map) } } @@ -36392,11 +46990,19 @@ impl SieveSystemInterpreter { impl Pickle for SieveSystemInterpreter { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.message_id_hostname.is_some() { + bitmap |= 1 << 0; + } + if self.no_capability_check { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 12, 2, bitmap); self.default_from_address.pickle(out); self.default_from_name.pickle(out); - self.message_id_hostname.pickle(out); - self.duplicate_expiry.pickle(out); - self.no_capability_check.pickle(out); + if let Some(value) = &self.message_id_hostname { + value.pickle(out); + } self.default_return_path.pickle(out); self.dkim_sign_domain.pickle(out); self.max_cpu_cycles.pickle(out); @@ -36405,24 +47011,98 @@ impl Pickle for SieveSystemInterpreter { self.max_received_headers.pickle(out); self.max_redirects.pickle(out); self.max_var_size.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.default_from_address = Pickle::unpickle(stream)?; - this.default_from_name = Pickle::unpickle(stream)?; - this.message_id_hostname = Pickle::unpickle(stream)?; - this.duplicate_expiry = Pickle::unpickle(stream)?; - this.no_capability_check = Pickle::unpickle(stream)?; - this.default_return_path = Pickle::unpickle(stream)?; - this.dkim_sign_domain = Pickle::unpickle(stream)?; - this.max_cpu_cycles = Pickle::unpickle(stream)?; - this.max_nested_includes = Pickle::unpickle(stream)?; - this.max_out_messages = Pickle::unpickle(stream)?; - this.max_received_headers = Pickle::unpickle(stream)?; - this.max_redirects = Pickle::unpickle(stream)?; - this.max_var_size = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'MAILER-DAEMON@' + system('domain')".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Automated Message'".to_string(), + ..Default::default() + } + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('domain')".to_string(), + ..Default::default() + } + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 1048576u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field11: u64 = if block.has(11) { + Pickle::unpickle(stream)? + } else { + 52428800u64 + }; + stream.close_block(block); + Some(Self { + default_from_address: field0, + default_from_name: field1, + message_id_hostname: field2, + no_capability_check: field3, + default_return_path: field4, + dkim_sign_domain: field5, + max_cpu_cycles: field6, + max_nested_includes: field7, + max_out_messages: field8, + max_received_headers: field9, + max_redirects: field10, + max_var_size: field11, + }) } } @@ -36438,7 +47118,6 @@ impl Default for SieveSystemInterpreter { ..Default::default() }, message_id_hostname: Default::default(), - duplicate_expiry: Duration::from_millis(604800000), no_capability_check: true, default_return_path: Default::default(), dkim_sign_domain: Expression { @@ -36457,44 +47136,59 @@ impl Default for SieveSystemInterpreter { impl IntoValue for SieveSystemInterpreter { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(15); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(14); map.insert_unchecked( Property::DefaultFromAddress, - self.default_from_address.into_value(), + self.default_from_address.into_value_with(opts), ); map.insert_unchecked( Property::DefaultFromName, - self.default_from_name.into_value(), + self.default_from_name.into_value_with(opts), ); map.insert_unchecked( Property::MessageIdHostname, - self.message_id_hostname.into_value(), - ); - map.insert_unchecked( - Property::DuplicateExpiry, - self.duplicate_expiry.into_value(), + self.message_id_hostname.into_value_with(opts), ); map.insert_unchecked( Property::NoCapabilityCheck, - self.no_capability_check.into_value(), + self.no_capability_check.into_value_with(opts), ); map.insert_unchecked( Property::DefaultReturnPath, - self.default_return_path.into_value(), + self.default_return_path.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxCpuCycles, + self.max_cpu_cycles.into_value_with(opts), ); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); - map.insert_unchecked(Property::MaxCpuCycles, self.max_cpu_cycles.into_value()); map.insert_unchecked( Property::MaxNestedIncludes, - self.max_nested_includes.into_value(), + self.max_nested_includes.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxOutMessages, + self.max_out_messages.into_value_with(opts), ); - map.insert_unchecked(Property::MaxOutMessages, self.max_out_messages.into_value()); map.insert_unchecked( Property::MaxReceivedHeaders, - self.max_received_headers.into_value(), + self.max_received_headers.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRedirects, + self.max_redirects.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxVarSize, + self.max_var_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxRedirects, self.max_redirects.into_value()); - map.insert_unchecked(Property::MaxVarSize, self.max_var_size.into_value()); JmapValue::Object(map) } } @@ -36511,7 +47205,6 @@ impl RegistryJsonPropertyPatch for SieveSystemInterpreter { Some(Property::MessageIdHostname) => self .message_id_hostname .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::DuplicateExpiry) => self.duplicate_expiry.patch(pointer, value), Some(Property::NoCapabilityCheck) => self.no_capability_check.patch(pointer, value), Some(Property::DefaultReturnPath) => self.default_return_path.patch(pointer, value), Some(Property::DkimSignDomain) => self.dkim_sign_domain.patch(pointer, value), @@ -36560,19 +47253,54 @@ impl ObjectImpl for SieveSystemScript { impl Pickle for SieveSystemScript { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.is_active { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.name.pickle(out); - self.description.pickle(out); - self.is_active.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.contents.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.is_active = Pickle::unpickle(stream)?; - this.contents = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + is_active: field2, + contents: field3, + }) } } @@ -36589,11 +47317,18 @@ impl Default for SieveSystemScript { impl IntoValue for SieveSystemScript { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::IsActive, self.is_active.into_value()); - map.insert_unchecked(Property::Contents, self.contents.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::IsActive, self.is_active.into_value_with(opts)); + map.insert_unchecked(Property::Contents, self.contents.into_value_with(opts)); JmapValue::Object(map) } } @@ -36626,7 +47361,7 @@ impl RegistryJsonPropertyPatch for SieveSystemScript { impl ObjectImpl for SieveUserInterpreter { const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 1; + const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::SieveUserInterpreter; fn validate(&self, errors: &mut Vec) -> bool { @@ -36748,6 +47483,11 @@ impl SieveUserInterpreter { impl Pickle for SieveUserInterpreter { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.max_scripts.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 26, 1, bitmap); self.default_expiry_duplicate.pickle(out); self.default_expiry_vacation.pickle(out); self.disable_capabilities.pickle(out); @@ -36772,41 +47512,192 @@ impl Pickle for SieveUserInterpreter { self.max_string_length.pickle(out); self.max_var_name_length.pickle(out); self.max_var_size.pickle(out); - self.max_scripts.pickle(out); + if let Some(value) = &self.max_scripts { + value.pickle(out); + } self.dkim_sign_domain.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.default_expiry_duplicate = Pickle::unpickle(stream)?; - this.default_expiry_vacation = Pickle::unpickle(stream)?; - this.disable_capabilities = Pickle::unpickle(stream)?; - this.allowed_notify_uris = Pickle::unpickle(stream)?; - this.protected_headers = Pickle::unpickle(stream)?; - this.default_subject = Pickle::unpickle(stream)?; - this.default_subject_prefix = Pickle::unpickle(stream)?; - this.max_cpu_cycles = Pickle::unpickle(stream)?; - this.max_header_size = Pickle::unpickle(stream)?; - this.max_includes = Pickle::unpickle(stream)?; - this.max_local_vars = Pickle::unpickle(stream)?; - this.max_match_vars = Pickle::unpickle(stream)?; - this.max_script_name_length = Pickle::unpickle(stream)?; - this.max_nested_blocks = Pickle::unpickle(stream)?; - this.max_nested_for_every = Pickle::unpickle(stream)?; - this.max_nested_includes = Pickle::unpickle(stream)?; - this.max_nested_tests = Pickle::unpickle(stream)?; - this.max_out_messages = Pickle::unpickle(stream)?; - this.max_received_headers = Pickle::unpickle(stream)?; - this.max_redirects = Pickle::unpickle(stream)?; - this.max_script_size = Pickle::unpickle(stream)?; - this.max_string_length = Pickle::unpickle(stream)?; - this.max_var_name_length = Pickle::unpickle(stream)?; - this.max_var_size = Pickle::unpickle(stream)?; - this.max_scripts = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.dkim_sign_domain = Pickle::unpickle(stream)?; + if stream.version() > Self::VERSION { + return None; } - Some(this) + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(604800000) + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(2592000000) + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Map::new(vec!["mailto".to_string()]) + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Map::new(vec![ + "Original-Subject".to_string(), + "Original-From".to_string(), + "Received".to_string(), + "Auto-Submitted".to_string(), + ]) + }; + let field5: String = if block.has(5) { + Pickle::unpickle(stream)? + } else { + "Automated reply".to_string() + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + "Auto: ".to_string() + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 5000u64 + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 1024u64 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 128u64 + }; + let field11: u64 = if block.has(11) { + Pickle::unpickle(stream)? + } else { + 30u64 + }; + let field12: u64 = if block.has(12) { + Pickle::unpickle(stream)? + } else { + 512u64 + }; + let field13: u64 = if block.has(13) { + Pickle::unpickle(stream)? + } else { + 15u64 + }; + let field14: u64 = if block.has(14) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field15: u64 = if block.has(15) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field16: u64 = if block.has(16) { + Pickle::unpickle(stream)? + } else { + 15u64 + }; + let field17: u64 = if block.has(17) { + Pickle::unpickle(stream)? + } else { + 3u64 + }; + let field18: u64 = if block.has(18) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + let field19: u64 = if block.has(19) { + Pickle::unpickle(stream)? + } else { + 1u64 + }; + let field20: u64 = if block.has(20) { + Pickle::unpickle(stream)? + } else { + 102400 + }; + let field21: u64 = if block.has(21) { + Pickle::unpickle(stream)? + } else { + 4096u64 + }; + let field22: u64 = if block.has(22) { + Pickle::unpickle(stream)? + } else { + 32u64 + }; + let field23: u64 = if block.has(23) { + Pickle::unpickle(stream)? + } else { + 4096u64 + }; + let field24: Option = if block.has(24) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(100u64) + }; + let field25: Expression = if block.has(25) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "is_local_domain(sender_domain)".to_string(), + then: "sender_domain".to_string(), + }]), + } + }; + stream.close_block(block); + Some(Self { + default_expiry_duplicate: field0, + default_expiry_vacation: field1, + disable_capabilities: field2, + allowed_notify_uris: field3, + protected_headers: field4, + default_subject: field5, + default_subject_prefix: field6, + max_cpu_cycles: field7, + max_header_size: field8, + max_includes: field9, + max_local_vars: field10, + max_match_vars: field11, + max_script_name_length: field12, + max_nested_blocks: field13, + max_nested_for_every: field14, + max_nested_includes: field15, + max_nested_tests: field16, + max_out_messages: field17, + max_received_headers: field18, + max_redirects: field19, + max_script_size: field20, + max_string_length: field21, + max_var_name_length: field22, + max_var_size: field23, + max_scripts: field24, + dkim_sign_domain: field25, + }) } } @@ -36856,72 +47747,112 @@ impl Default for SieveUserInterpreter { impl IntoValue for SieveUserInterpreter { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(28); map.insert_unchecked( Property::DefaultExpiryDuplicate, - self.default_expiry_duplicate.into_value(), + self.default_expiry_duplicate.into_value_with(opts), ); map.insert_unchecked( Property::DefaultExpiryVacation, - self.default_expiry_vacation.into_value(), + self.default_expiry_vacation.into_value_with(opts), ); map.insert_unchecked( Property::DisableCapabilities, - self.disable_capabilities.into_value(), + self.disable_capabilities.into_value_with(opts), ); map.insert_unchecked( Property::AllowedNotifyUris, - self.allowed_notify_uris.into_value(), + self.allowed_notify_uris.into_value_with(opts), ); map.insert_unchecked( Property::ProtectedHeaders, - self.protected_headers.into_value(), + self.protected_headers.into_value_with(opts), + ); + map.insert_unchecked( + Property::DefaultSubject, + self.default_subject.into_value_with(opts), ); - map.insert_unchecked(Property::DefaultSubject, self.default_subject.into_value()); map.insert_unchecked( Property::DefaultSubjectPrefix, - self.default_subject_prefix.into_value(), + self.default_subject_prefix.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxCpuCycles, + self.max_cpu_cycles.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxHeaderSize, + self.max_header_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxIncludes, + self.max_includes.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxLocalVars, + self.max_local_vars.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMatchVars, + self.max_match_vars.into_value_with(opts), ); - map.insert_unchecked(Property::MaxCpuCycles, self.max_cpu_cycles.into_value()); - map.insert_unchecked(Property::MaxHeaderSize, self.max_header_size.into_value()); - map.insert_unchecked(Property::MaxIncludes, self.max_includes.into_value()); - map.insert_unchecked(Property::MaxLocalVars, self.max_local_vars.into_value()); - map.insert_unchecked(Property::MaxMatchVars, self.max_match_vars.into_value()); map.insert_unchecked( Property::MaxScriptNameLength, - self.max_script_name_length.into_value(), + self.max_script_name_length.into_value_with(opts), ); map.insert_unchecked( Property::MaxNestedBlocks, - self.max_nested_blocks.into_value(), + self.max_nested_blocks.into_value_with(opts), ); map.insert_unchecked( Property::MaxNestedForEvery, - self.max_nested_for_every.into_value(), + self.max_nested_for_every.into_value_with(opts), ); map.insert_unchecked( Property::MaxNestedIncludes, - self.max_nested_includes.into_value(), + self.max_nested_includes.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxNestedTests, + self.max_nested_tests.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxOutMessages, + self.max_out_messages.into_value_with(opts), ); - map.insert_unchecked(Property::MaxNestedTests, self.max_nested_tests.into_value()); - map.insert_unchecked(Property::MaxOutMessages, self.max_out_messages.into_value()); map.insert_unchecked( Property::MaxReceivedHeaders, - self.max_received_headers.into_value(), + self.max_received_headers.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRedirects, + self.max_redirects.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxScriptSize, + self.max_script_size.into_value_with(opts), ); - map.insert_unchecked(Property::MaxRedirects, self.max_redirects.into_value()); - map.insert_unchecked(Property::MaxScriptSize, self.max_script_size.into_value()); map.insert_unchecked( Property::MaxStringLength, - self.max_string_length.into_value(), + self.max_string_length.into_value_with(opts), ); map.insert_unchecked( Property::MaxVarNameLength, - self.max_var_name_length.into_value(), + self.max_var_name_length.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxVarSize, + self.max_var_size.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxScripts, self.max_scripts.into_value_with(opts)); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), ); - map.insert_unchecked(Property::MaxVarSize, self.max_var_size.into_value()); - map.insert_unchecked(Property::MaxScripts, self.max_scripts.into_value()); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); JmapValue::Object(map) } } @@ -37012,19 +47943,54 @@ impl ObjectImpl for SieveUserScript { impl Pickle for SieveUserScript { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.is_active { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.name.pickle(out); - self.description.pickle(out); - self.is_active.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.contents.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.is_active = Pickle::unpickle(stream)?; - this.contents = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + description: field1, + is_active: field2, + contents: field3, + }) } } @@ -37041,11 +48007,18 @@ impl Default for SieveUserScript { impl IntoValue for SieveUserScript { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::IsActive, self.is_active.into_value()); - map.insert_unchecked(Property::Contents, self.contents.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::IsActive, self.is_active.into_value_with(opts)); + map.insert_unchecked(Property::Contents, self.contents.into_value_with(opts)); JmapValue::Object(map) } } @@ -37124,31 +48097,92 @@ impl ObjectImpl for SpamClassifier { impl Pickle for SpamClassifier { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.learn_ham_from_card { + bitmap |= 1 << 0; + } + if self.learn_spam_from_traps { + bitmap |= 1 << 1; + } + if self.train_interval.is_some() { + bitmap |= 1 << 2; + } + if self.learn_ham_from_reply { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 10, 4, bitmap); self.model.pickle(out); - self.learn_ham_from_card.pickle(out); self.learn_spam_from_rbl_hits.pickle(out); - self.learn_spam_from_traps.pickle(out); self.hold_samples_for.pickle(out); self.min_ham_samples.pickle(out); self.min_spam_samples.pickle(out); self.reservoir_capacity.pickle(out); - self.train_frequency.pickle(out); - self.learn_ham_from_reply.pickle(out); + if let Some(value) = &self.train_interval { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.model = Pickle::unpickle(stream)?; - this.learn_ham_from_card = Pickle::unpickle(stream)?; - this.learn_spam_from_rbl_hits = Pickle::unpickle(stream)?; - this.learn_spam_from_traps = Pickle::unpickle(stream)?; - this.hold_samples_for = Pickle::unpickle(stream)?; - this.min_ham_samples = Pickle::unpickle(stream)?; - this.min_spam_samples = Pickle::unpickle(stream)?; - this.reservoir_capacity = Pickle::unpickle(stream)?; - this.train_frequency = Pickle::unpickle(stream)?; - this.learn_ham_from_reply = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: SpamClassifierModel = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 2u64 + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(15552000000) + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 100u64 + }; + let field7: u64 = if block.has(7) { + Pickle::unpickle(stream)? + } else { + 1024u64 + }; + let field8: Option = if block.has(8) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some(Duration::from_millis(43200000)) + }; + let field9: bool = if block.has(9) { block.flag(3) } else { true }; + stream.close_block(block); + Some(Self { + model: field0, + learn_ham_from_card: field1, + learn_spam_from_rbl_hits: field2, + learn_spam_from_traps: field3, + hold_samples_for: field4, + min_ham_samples: field5, + min_spam_samples: field6, + reservoir_capacity: field7, + train_interval: field8, + learn_ham_from_reply: field9, + }) } } @@ -37163,7 +48197,7 @@ impl Default for SpamClassifier { min_ham_samples: 100u64, min_spam_samples: 100u64, reservoir_capacity: 1024u64, - train_frequency: Some(Duration::from_millis(43200000)), + train_interval: Some(Duration::from_millis(43200000)), learn_ham_from_reply: true, } } @@ -37171,31 +48205,47 @@ impl Default for SpamClassifier { impl IntoValue for SpamClassifier { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Model, self.model.into_value()); + map.insert_unchecked(Property::Model, self.model.into_value_with(opts)); map.insert_unchecked( Property::LearnHamFromCard, - self.learn_ham_from_card.into_value(), + self.learn_ham_from_card.into_value_with(opts), ); map.insert_unchecked( Property::LearnSpamFromRblHits, - self.learn_spam_from_rbl_hits.into_value(), + self.learn_spam_from_rbl_hits.into_value_with(opts), ); map.insert_unchecked( Property::LearnSpamFromTraps, - self.learn_spam_from_traps.into_value(), + self.learn_spam_from_traps.into_value_with(opts), + ); + map.insert_unchecked( + Property::HoldSamplesFor, + self.hold_samples_for.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinHamSamples, + self.min_ham_samples.into_value_with(opts), + ); + map.insert_unchecked( + Property::MinSpamSamples, + self.min_spam_samples.into_value_with(opts), ); - map.insert_unchecked(Property::HoldSamplesFor, self.hold_samples_for.into_value()); - map.insert_unchecked(Property::MinHamSamples, self.min_ham_samples.into_value()); - map.insert_unchecked(Property::MinSpamSamples, self.min_spam_samples.into_value()); map.insert_unchecked( Property::ReservoirCapacity, - self.reservoir_capacity.into_value(), + self.reservoir_capacity.into_value_with(opts), + ); + map.insert_unchecked( + Property::TrainInterval, + self.train_interval.into_value_with(opts), ); - map.insert_unchecked(Property::TrainFrequency, self.train_frequency.into_value()); map.insert_unchecked( Property::LearnHamFromReply, - self.learn_ham_from_reply.into_value(), + self.learn_ham_from_reply.into_value_with(opts), ); JmapValue::Object(map) } @@ -37218,7 +48268,7 @@ impl RegistryJsonPropertyPatch for SpamClassifier { Some(Property::MinHamSamples) => self.min_ham_samples.patch(pointer, value), Some(Property::MinSpamSamples) => self.min_spam_samples.patch(pointer, value), Some(Property::ReservoirCapacity) => self.reservoir_capacity.patch(pointer, value), - Some(Property::TrainFrequency) => self.train_frequency.patch(pointer, value), + Some(Property::TrainInterval) => self.train_interval.patch(pointer, value), Some(Property::LearnHamFromReply) => self.learn_ham_from_reply.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -37242,19 +48292,46 @@ impl SpamClassifierFtrlCcfh { impl Pickle for SpamClassifierFtrlCcfh { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.feature_l2_normalize { + bitmap |= 1 << 0; + } + if self.feature_log_scale { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 4, 2, bitmap); self.indicator_parameters.pickle(out); self.parameters.pickle(out); - self.feature_l2_normalize.pickle(out); - self.feature_log_scale.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.indicator_parameters = Pickle::unpickle(stream)?; - this.parameters = Pickle::unpickle(stream)?; - this.feature_l2_normalize = Pickle::unpickle(stream)?; - this.feature_log_scale = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: FtrlParameters = if block.has(0) { + Pickle::unpickle(stream)? + } else { + FtrlParameters { + num_features: ModelSize::V18, + ..Default::default() + } + }; + let field1: FtrlParameters = if block.has(1) { + Pickle::unpickle(stream)? + } else { + FtrlParameters { + num_features: ModelSize::V20, + ..Default::default() + } + }; + let field2: bool = if block.has(2) { block.flag(0) } else { true }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + indicator_parameters: field0, + parameters: field1, + feature_l2_normalize: field2, + feature_log_scale: field3, + }) } } @@ -37277,19 +48354,23 @@ impl Default for SpamClassifierFtrlCcfh { impl IntoValue for SpamClassifierFtrlCcfh { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); map.insert_unchecked( Property::IndicatorParameters, - self.indicator_parameters.into_value(), + self.indicator_parameters.into_value_with(opts), ); - map.insert_unchecked(Property::Parameters, self.parameters.into_value()); + map.insert_unchecked(Property::Parameters, self.parameters.into_value_with(opts)); map.insert_unchecked( Property::FeatureL2Normalize, - self.feature_l2_normalize.into_value(), + self.feature_l2_normalize.into_value_with(opts), ); map.insert_unchecked( Property::FeatureLogScale, - self.feature_log_scale.into_value(), + self.feature_log_scale.into_value_with(opts), ); JmapValue::Object(map) } @@ -37326,17 +48407,36 @@ impl SpamClassifierFtrlFh { impl Pickle for SpamClassifierFtrlFh { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.feature_l2_normalize { + bitmap |= 1 << 0; + } + if self.feature_log_scale { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 3, 2, bitmap); self.parameters.pickle(out); - self.feature_l2_normalize.pickle(out); - self.feature_log_scale.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.parameters = Pickle::unpickle(stream)?; - this.feature_l2_normalize = Pickle::unpickle(stream)?; - this.feature_log_scale = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: FtrlParameters = if block.has(0) { + Pickle::unpickle(stream)? + } else { + FtrlParameters { + num_features: ModelSize::V20, + ..Default::default() + } + }; + let field1: bool = if block.has(1) { block.flag(0) } else { true }; + let field2: bool = if block.has(2) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + parameters: field0, + feature_l2_normalize: field1, + feature_log_scale: field2, + }) } } @@ -37355,15 +48455,19 @@ impl Default for SpamClassifierFtrlFh { impl IntoValue for SpamClassifierFtrlFh { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Parameters, self.parameters.into_value()); + map.insert_unchecked(Property::Parameters, self.parameters.into_value_with(opts)); map.insert_unchecked( Property::FeatureL2Normalize, - self.feature_l2_normalize.into_value(), + self.feature_l2_normalize.into_value_with(opts), ); map.insert_unchecked( Property::FeatureLogScale, - self.feature_log_scale.into_value(), + self.feature_log_scale.into_value_with(opts), ); JmapValue::Object(map) } @@ -37433,16 +48537,20 @@ impl Pickle for SpamClassifierModel { impl IntoValue for SpamClassifierModel { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SpamClassifierModel::FtrlFh(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FtrlFh".into())); obj } SpamClassifierModel::FtrlCcfh(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FtrlCcfh".into())); @@ -37512,14 +48620,14 @@ impl SpamClassify { errors.push(ValidationError::required(Property::AuthenticatedAs)); } } - let value = &self.env_from; + let value = &self.mail_from; if value.is_empty() { - errors.push(ValidationError::required(Property::EnvFrom)); + errors.push(ValidationError::required(Property::MailFrom)); } - let value = &self.env_rcpt_to; + let value = &self.rcpt_to; for value in value.iter() { if value.is_empty() { - errors.push(ValidationError::required(Property::EnvRcptTo)); + errors.push(ValidationError::required(Property::RcptTo)); } } let value = &self.tags; @@ -37530,38 +48638,6 @@ impl SpamClassify { } } -impl Pickle for SpamClassify { - fn pickle(&self, out: &mut Vec) { - self.message.pickle(out); - self.remote_ip.pickle(out); - self.ehlo_domain.pickle(out); - self.authenticated_as.pickle(out); - self.is_tls.pickle(out); - self.env_from.pickle(out); - self.env_from_parameters.pickle(out); - self.env_rcpt_to.pickle(out); - self.score.pickle(out); - self.tags.pickle(out); - self.result.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.message = Pickle::unpickle(stream)?; - this.remote_ip = Pickle::unpickle(stream)?; - this.ehlo_domain = Pickle::unpickle(stream)?; - this.authenticated_as = Pickle::unpickle(stream)?; - this.is_tls = Pickle::unpickle(stream)?; - this.env_from = Pickle::unpickle(stream)?; - this.env_from_parameters = Pickle::unpickle(stream)?; - this.env_rcpt_to = Pickle::unpickle(stream)?; - this.score = Pickle::unpickle(stream)?; - this.tags = Pickle::unpickle(stream)?; - this.result = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for SpamClassify { fn default() -> Self { Self { @@ -37570,9 +48646,9 @@ impl Default for SpamClassify { ehlo_domain: Default::default(), authenticated_as: Default::default(), is_tls: true, - env_from: Default::default(), - env_from_parameters: Default::default(), - env_rcpt_to: Default::default(), + mail_from: Default::default(), + mail_from_parameters: Default::default(), + rcpt_to: Default::default(), score: Float::new(0.0f64), tags: Default::default(), result: Default::default(), @@ -37582,24 +48658,28 @@ impl Default for SpamClassify { impl IntoValue for SpamClassify { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::Message, self.message.into_value()); - map.insert_unchecked(Property::RemoteIp, self.remote_ip.into_value()); - map.insert_unchecked(Property::EhloDomain, self.ehlo_domain.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); + map.insert_unchecked(Property::Message, self.message.into_value_with(opts)); + map.insert_unchecked(Property::RemoteIp, self.remote_ip.into_value_with(opts)); + map.insert_unchecked(Property::EhloDomain, self.ehlo_domain.into_value_with(opts)); map.insert_unchecked( Property::AuthenticatedAs, - self.authenticated_as.into_value(), + self.authenticated_as.into_value_with(opts), ); - map.insert_unchecked(Property::IsTls, self.is_tls.into_value()); - map.insert_unchecked(Property::EnvFrom, self.env_from.into_value()); + map.insert_unchecked(Property::IsTls, self.is_tls.into_value_with(opts)); + map.insert_unchecked(Property::MailFrom, self.mail_from.into_value_with(opts)); map.insert_unchecked( - Property::EnvFromParameters, - self.env_from_parameters.into_value(), + Property::MailFromParameters, + self.mail_from_parameters.into_value_with(opts), ); - map.insert_unchecked(Property::EnvRcptTo, self.env_rcpt_to.into_value()); - map.insert_unchecked(Property::Score, self.score.into_value()); - map.insert_unchecked(Property::Tags, self.tags.into_value()); - map.insert_unchecked(Property::Result, self.result.into_value()); + map.insert_unchecked(Property::RcptTo, self.rcpt_to.into_value_with(opts)); + map.insert_unchecked(Property::Score, self.score.into_value_with(opts)); + map.insert_unchecked(Property::Tags, self.tags.into_value_with(opts)); + map.insert_unchecked(Property::Result, self.result.into_value_with(opts)); JmapValue::Object(map) } } @@ -37616,12 +48696,12 @@ impl RegistryJsonPropertyPatch for SpamClassify { Some(Property::EhloDomain) => self.ehlo_domain.patch(pointer, value), Some(Property::AuthenticatedAs) => self.authenticated_as.patch(pointer, value), Some(Property::IsTls) => self.is_tls.patch(pointer, value), - Some(Property::EnvFrom) => self - .env_from + Some(Property::MailFrom) => self + .mail_from .patch(pointer.with_validators(&[StringValidator::Email]), value), - Some(Property::EnvFromParameters) => self.env_from_parameters.patch(pointer, value), - Some(Property::EnvRcptTo) => self - .env_rcpt_to + Some(Property::MailFromParameters) => self.mail_from_parameters.patch(pointer, value), + Some(Property::RcptTo) => self + .rcpt_to .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::Score) => pointer.assert_server_set(), Some(Property::Tags) => pointer.assert_server_set(), @@ -37641,20 +48721,6 @@ impl SpamClassifyTag { } } -impl Pickle for SpamClassifyTag { - fn pickle(&self, out: &mut Vec) { - self.score.pickle(out); - self.disposition.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.score = Pickle::unpickle(stream)?; - this.disposition = Pickle::unpickle(stream)?; - Some(this) - } -} - impl Default for SpamClassifyTag { fn default() -> Self { Self { @@ -37666,9 +48732,16 @@ impl Default for SpamClassifyTag { impl IntoValue for SpamClassifyTag { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Score, self.score.into_value()); - map.insert_unchecked(Property::Disposition, self.disposition.into_value()); + map.insert_unchecked(Property::Score, self.score.into_value_with(opts)); + map.insert_unchecked( + Property::Disposition, + self.disposition.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -37776,6 +48849,9 @@ impl Pickle for SpamDnsblServer { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(SpamDnsblServer::Any), 1 => Pickle::unpickle(stream).map(SpamDnsblServer::Url), @@ -37791,51 +48867,55 @@ impl Pickle for SpamDnsblServer { impl IntoValue for SpamDnsblServer { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SpamDnsblServer::Any(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Any".into())); obj } SpamDnsblServer::Url(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Url".into())); obj } SpamDnsblServer::Domain(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Domain".into())); obj } SpamDnsblServer::Email(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Email".into())); obj } SpamDnsblServer::Ip(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ip".into())); obj } SpamDnsblServer::Header(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Header".into())); obj } SpamDnsblServer::Body(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Body".into())); @@ -37952,21 +49032,58 @@ impl SpamDnsblServerAny { impl Pickle for SpamDnsblServerAny { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -37984,12 +49101,19 @@ impl Default for SpamDnsblServerAny { impl IntoValue for SpamDnsblServerAny { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38072,21 +49196,58 @@ impl SpamDnsblServerBody { impl Pickle for SpamDnsblServerBody { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38104,12 +49265,19 @@ impl Default for SpamDnsblServerBody { impl IntoValue for SpamDnsblServerBody { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38192,21 +49360,58 @@ impl SpamDnsblServerDomain { impl Pickle for SpamDnsblServerDomain { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38224,12 +49429,19 @@ impl Default for SpamDnsblServerDomain { impl IntoValue for SpamDnsblServerDomain { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38312,21 +49524,58 @@ impl SpamDnsblServerEmail { impl Pickle for SpamDnsblServerEmail { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38344,12 +49593,19 @@ impl Default for SpamDnsblServerEmail { impl IntoValue for SpamDnsblServerEmail { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38432,21 +49688,58 @@ impl SpamDnsblServerHeader { impl Pickle for SpamDnsblServerHeader { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38464,12 +49757,19 @@ impl Default for SpamDnsblServerHeader { impl IntoValue for SpamDnsblServerHeader { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38552,21 +49852,58 @@ impl SpamDnsblServerIp { impl Pickle for SpamDnsblServerIp { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38584,12 +49921,19 @@ impl Default for SpamDnsblServerIp { impl IntoValue for SpamDnsblServerIp { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38672,21 +50016,58 @@ impl SpamDnsblServerUrl { impl Pickle for SpamDnsblServerUrl { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.tag.pickle(out); self.zone.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.zone = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: bool = if block.has(4) { block.flag(1) } else { true }; + stream.close_block(block); + Some(Self { + tag: field0, + zone: field1, + name: field2, + description: field3, + enable: field4, + }) } } @@ -38704,12 +50085,19 @@ impl Default for SpamDnsblServerUrl { impl IntoValue for SpamDnsblServerUrl { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Zone, self.zone.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Zone, self.zone.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); JmapValue::Object(map) } } @@ -38771,19 +50159,46 @@ impl ObjectImpl for SpamDnsblSettings { impl Pickle for SpamDnsblSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.domain_limit.pickle(out); self.email_limit.pickle(out); self.ip_limit.pickle(out); self.url_limit.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.domain_limit = Pickle::unpickle(stream)?; - this.email_limit = Pickle::unpickle(stream)?; - this.ip_limit = Pickle::unpickle(stream)?; - this.url_limit = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 50u64 + }; + stream.close_block(block); + Some(Self { + domain_limit: field0, + email_limit: field1, + ip_limit: field2, + url_limit: field3, + }) } } @@ -38800,11 +50215,18 @@ impl Default for SpamDnsblSettings { impl IntoValue for SpamDnsblSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::DomainLimit, self.domain_limit.into_value()); - map.insert_unchecked(Property::EmailLimit, self.email_limit.into_value()); - map.insert_unchecked(Property::IpLimit, self.ip_limit.into_value()); - map.insert_unchecked(Property::UrlLimit, self.url_limit.into_value()); + map.insert_unchecked( + Property::DomainLimit, + self.domain_limit.into_value_with(opts), + ); + map.insert_unchecked(Property::EmailLimit, self.email_limit.into_value_with(opts)); + map.insert_unchecked(Property::IpLimit, self.ip_limit.into_value_with(opts)); + map.insert_unchecked(Property::UrlLimit, self.url_limit.into_value_with(opts)); JmapValue::Object(map) } } @@ -38856,21 +50278,48 @@ impl ObjectImpl for SpamFileExtension { impl Pickle for SpamFileExtension { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.is_archive { + bitmap |= 1 << 0; + } + if self.is_bad { + bitmap |= 1 << 1; + } + if self.is_nz { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 5, 3, bitmap); self.extension.pickle(out); - self.is_archive.pickle(out); - self.is_bad.pickle(out); - self.is_nz.pickle(out); self.content_types.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.extension = Pickle::unpickle(stream)?; - this.is_archive = Pickle::unpickle(stream)?; - this.is_bad = Pickle::unpickle(stream)?; - this.is_nz = Pickle::unpickle(stream)?; - this.content_types = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + let field3: bool = if block.has(3) { block.flag(2) } else { false }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + extension: field0, + is_archive: field1, + is_bad: field2, + is_nz: field3, + content_types: field4, + }) } } @@ -38888,12 +50337,19 @@ impl Default for SpamFileExtension { impl IntoValue for SpamFileExtension { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Extension, self.extension.into_value()); - map.insert_unchecked(Property::IsArchive, self.is_archive.into_value()); - map.insert_unchecked(Property::IsBad, self.is_bad.into_value()); - map.insert_unchecked(Property::IsNz, self.is_nz.into_value()); - map.insert_unchecked(Property::ContentTypes, self.content_types.into_value()); + map.insert_unchecked(Property::Extension, self.extension.into_value_with(opts)); + map.insert_unchecked(Property::IsArchive, self.is_archive.into_value_with(opts)); + map.insert_unchecked(Property::IsBad, self.is_bad.into_value_with(opts)); + map.insert_unchecked(Property::IsNz, self.is_nz.into_value_with(opts)); + map.insert_unchecked( + Property::ContentTypes, + self.content_types.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -38926,257 +50382,6 @@ impl RegistryJsonPropertyPatch for SpamFileExtension { } } -impl ObjectImpl for SpamLlm { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::SpamLlm; - - fn validate(&self, errors: &mut Vec) -> bool { - match self { - SpamLlm::Disable => true, - SpamLlm::Enable(inner) => inner.validate(errors), - } - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - match self { - SpamLlm::Disable => {} - SpamLlm::Enable(object) => { - object.index(i); - } - } - } -} - -impl Default for SpamLlm { - fn default() -> Self { - SpamLlm::Disable - } -} - -impl Pickle for SpamLlm { - fn pickle(&self, out: &mut Vec) { - match self { - SpamLlm::Disable => { - 0u16.pickle(out); - } - SpamLlm::Enable(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(SpamLlm::Disable), - 1 => Pickle::unpickle(stream).map(SpamLlm::Enable), - _ => None, - } - } -} - -impl IntoValue for SpamLlm { - fn into_value(self) -> JmapValue<'static> { - match self { - SpamLlm::Disable => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Disable".into())); - JmapValue::Object(obj) - } - SpamLlm::Enable(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Enable".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for SpamLlm { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - SpamLlmType::Disable => *self = SpamLlm::Disable, - SpamLlmType::Enable => *self = SpamLlm::Enable(Default::default()), - } - } - match self { - SpamLlm::Disable => pointer.assert_eof(), - SpamLlm::Enable(inner) => inner.patch(pointer, value), - } - } -} - -impl SpamLlm { - pub fn object_type(&self) -> SpamLlmType { - match self { - SpamLlm::Disable => SpamLlmType::Disable, - SpamLlm::Enable(_) => SpamLlmType::Enable, - } - } -} - -impl SpamLlmProperties { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.categories; - for value in value.iter() { - if value.is_empty() { - errors.push(ValidationError::required(Property::Categories)); - } - } - if value.len() < 2 { - errors.push(ValidationError::min_items(Property::Categories, 2)); - } - let value = &self.confidence; - for value in value.iter() { - if value.is_empty() { - errors.push(ValidationError::required(Property::Confidence)); - } - } - let value = &self.model_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::ModelId)); - } - let value = &self.prompt; - if value.is_empty() { - errors.push(ValidationError::required(Property::Prompt)); - } - let value = &self.separator; - if value.is_empty() { - errors.push(ValidationError::required(Property::Separator)); - } - let value = &self.temperature; - if *value > Float::new(1.0) { - errors.push(ValidationError::max_value(Property::Temperature, 1)); - } - if *value < Float::new(0.0) { - errors.push(ValidationError::min_value(Property::Temperature, 0)); - } - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::AiModel, self.model_id.into(), None); - } -} - -impl Pickle for SpamLlmProperties { - fn pickle(&self, out: &mut Vec) { - self.categories.pickle(out); - self.confidence.pickle(out); - self.response_pos_category.pickle(out); - self.response_pos_confidence.pickle(out); - self.response_pos_explanation.pickle(out); - self.model_id.pickle(out); - self.prompt.pickle(out); - self.separator.pickle(out); - self.temperature.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.categories = Pickle::unpickle(stream)?; - this.confidence = Pickle::unpickle(stream)?; - this.response_pos_category = Pickle::unpickle(stream)?; - this.response_pos_confidence = Pickle::unpickle(stream)?; - this.response_pos_explanation = Pickle::unpickle(stream)?; - this.model_id = Pickle::unpickle(stream)?; - this.prompt = Pickle::unpickle(stream)?; - this.separator = Pickle::unpickle(stream)?; - this.temperature = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for SpamLlmProperties { - fn default() -> Self { - Self { - categories: Map::new(vec![ - "Unsolicited".to_string(), - "Commercial".to_string(), - "Harmful".to_string(), - "Legitimate".to_string(), - ]), - confidence: Map::new(vec![ - "High".to_string(), - "Medium".to_string(), - "Low".to_string(), - ]), - response_pos_category: 0u64, - response_pos_confidence: Some(1u64), - response_pos_explanation: Some(2u64), - model_id: Default::default(), - prompt: Default::default(), - separator: ",".to_string(), - temperature: Float::new(0.5f64), - } - } -} - -impl IntoValue for SpamLlmProperties { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::Categories, self.categories.into_value()); - map.insert_unchecked(Property::Confidence, self.confidence.into_value()); - map.insert_unchecked( - Property::ResponsePosCategory, - self.response_pos_category.into_value(), - ); - map.insert_unchecked( - Property::ResponsePosConfidence, - self.response_pos_confidence.into_value(), - ); - map.insert_unchecked( - Property::ResponsePosExplanation, - self.response_pos_explanation.into_value(), - ); - map.insert_unchecked(Property::ModelId, self.model_id.into_value()); - map.insert_unchecked(Property::Prompt, self.prompt.into_value()); - map.insert_unchecked(Property::Separator, self.separator.into_value()); - map.insert_unchecked(Property::Temperature, self.temperature.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for SpamLlmProperties { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Categories) => self.categories.patch(pointer, value), - Some(Property::Confidence) => self.confidence.patch(pointer, value), - Some(Property::ResponsePosCategory) => self.response_pos_category.patch(pointer, value), - Some(Property::ResponsePosConfidence) => { - self.response_pos_confidence.patch(pointer, value) - } - Some(Property::ResponsePosExplanation) => { - self.response_pos_explanation.patch(pointer, value) - } - Some(Property::ModelId) => self.model_id.patch(pointer, value), - Some(Property::Prompt) => self.prompt.patch(pointer, value), - Some(Property::Separator) => self - .separator - .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Temperature) => self.temperature.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for SpamPyzor { const FLAGS: u64 = OBJ_SINGLETON; const VERSION: u8 = 0; @@ -39224,25 +50429,66 @@ impl ObjectImpl for SpamPyzor { impl Pickle for SpamPyzor { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.block_count.pickle(out); - self.enable.pickle(out); self.host.pickle(out); self.port.pickle(out); self.ratio.pickle(out); self.timeout.pickle(out); self.allow_count.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.block_count = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.host = Pickle::unpickle(stream)?; - this.port = Pickle::unpickle(stream)?; - this.ratio = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.allow_count = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + "public.pyzor.org".to_string() + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 24441u64 + }; + let field4: Float = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Float::new(0.2f64) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field6: u64 = if block.has(6) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + stream.close_block(block); + Some(Self { + block_count: field0, + enable: field1, + host: field2, + port: field3, + ratio: field4, + timeout: field5, + allow_count: field6, + }) } } @@ -39250,7 +50496,7 @@ impl Default for SpamPyzor { fn default() -> Self { Self { block_count: 5u64, - enable: true, + enable: false, host: "public.pyzor.org".to_string(), port: 24441u64, ratio: Float::new(0.2f64), @@ -39262,14 +50508,18 @@ impl Default for SpamPyzor { impl IntoValue for SpamPyzor { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::BlockCount, self.block_count.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Host, self.host.into_value()); - map.insert_unchecked(Property::Port, self.port.into_value()); - map.insert_unchecked(Property::Ratio, self.ratio.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::AllowCount, self.allow_count.into_value()); + map.insert_unchecked(Property::BlockCount, self.block_count.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Host, self.host.into_value_with(opts)); + map.insert_unchecked(Property::Port, self.port.into_value_with(opts)); + map.insert_unchecked(Property::Ratio, self.ratio.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::AllowCount, self.allow_count.into_value_with(opts)); JmapValue::Object(map) } } @@ -39384,6 +50634,9 @@ impl Pickle for SpamRule { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(SpamRule::Any), 1 => Pickle::unpickle(stream).map(SpamRule::Url), @@ -39399,51 +50652,55 @@ impl Pickle for SpamRule { impl IntoValue for SpamRule { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SpamRule::Any(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Any".into())); obj } SpamRule::Url(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Url".into())); obj } SpamRule::Domain(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Domain".into())); obj } SpamRule::Email(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Email".into())); obj } SpamRule::Ip(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Ip".into())); obj } SpamRule::Header(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Header".into())); obj } SpamRule::Body(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Body".into())); @@ -39555,21 +50812,58 @@ impl SpamRuleAny { impl Pickle for SpamRuleAny { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -39587,12 +50881,19 @@ impl Default for SpamRuleAny { impl IntoValue for SpamRuleAny { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -39670,21 +50971,58 @@ impl SpamRuleBody { impl Pickle for SpamRuleBody { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -39702,12 +51040,19 @@ impl Default for SpamRuleBody { impl IntoValue for SpamRuleBody { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -39785,21 +51130,58 @@ impl SpamRuleDomain { impl Pickle for SpamRuleDomain { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -39817,12 +51199,19 @@ impl Default for SpamRuleDomain { impl IntoValue for SpamRuleDomain { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -39900,21 +51289,58 @@ impl SpamRuleEmail { impl Pickle for SpamRuleEmail { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -39932,12 +51358,19 @@ impl Default for SpamRuleEmail { impl IntoValue for SpamRuleEmail { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -40015,21 +51448,58 @@ impl SpamRuleHeader { impl Pickle for SpamRuleHeader { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -40047,12 +51517,19 @@ impl Default for SpamRuleHeader { impl IntoValue for SpamRuleHeader { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -40130,21 +51607,58 @@ impl SpamRuleIp { impl Pickle for SpamRuleIp { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -40162,12 +51676,19 @@ impl Default for SpamRuleIp { impl IntoValue for SpamRuleIp { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -40245,21 +51766,58 @@ impl SpamRuleUrl { impl Pickle for SpamRuleUrl { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.condition.pickle(out); self.name.pickle(out); - self.description.pickle(out); - self.enable.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.priority.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.condition = Pickle::unpickle(stream)?; - this.name = Pickle::unpickle(stream)?; - this.description = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.priority = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(1) } else { true }; + let field4: i64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 500i64 + }; + stream.close_block(block); + Some(Self { + condition: field0, + name: field1, + description: field2, + enable: field3, + priority: field4, + }) } } @@ -40277,12 +51835,19 @@ impl Default for SpamRuleUrl { impl IntoValue for SpamRuleUrl { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Condition, self.condition.into_value()); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Priority, self.priority.into_value()); + map.insert_unchecked(Property::Condition, self.condition.into_value_with(opts)); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Priority, self.priority.into_value_with(opts)); JmapValue::Object(map) } } @@ -40346,6 +51911,40 @@ impl ObjectImpl for SpamSettings { errors.push(ValidationError::required(Property::SpamFilterRulesUrl)); } } + let value = &self.max_body_text; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxBodyText, 1024)); + } + if *value > 16777216 { + errors.push(ValidationError::max_value(Property::MaxBodyText, 16777216)); + } + let value = &self.max_attachment_text; + if *value < 1024 { + errors.push(ValidationError::min_value( + Property::MaxAttachmentText, + 1024, + )); + } + if *value > 16777216 { + errors.push(ValidationError::max_value( + Property::MaxAttachmentText, + 16777216, + )); + } + let value = &self.max_message_text; + if *value < 1024 { + errors.push(ValidationError::min_value(Property::MaxMessageText, 1024)); + } + if *value > 67108864 { + errors.push(ValidationError::max_value( + Property::MaxMessageText, + 67108864, + )); + } + let value = &self.max_redirect_urls; + if *value > 1000 { + errors.push(ValidationError::max_value(Property::MaxRedirectUrls, 1000)); + } errors.len() == neb } @@ -40354,27 +51953,129 @@ impl ObjectImpl for SpamSettings { impl Pickle for SpamSettings { fn pickle(&self, out: &mut Vec) { - self.trust_contacts.pickle(out); - self.enable.pickle(out); - self.greylist_for.pickle(out); + let mut bitmap = 0u64; + if self.trust_contacts { + bitmap |= 1 << 0; + } + if self.enable { + bitmap |= 1 << 1; + } + if self.greylist_for.is_some() { + bitmap |= 1 << 2; + } + if self.trust_replies { + bitmap |= 1 << 3; + } + if self.spam_filter_rules_url.is_some() { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 14, 5, bitmap); + if let Some(value) = &self.greylist_for { + value.pickle(out); + } self.score_discard.pickle(out); self.score_reject.pickle(out); self.score_spam.pickle(out); - self.trust_replies.pickle(out); - self.spam_filter_rules_url.pickle(out); + if let Some(value) = &self.spam_filter_rules_url { + value.pickle(out); + } + self.max_body_text.pickle(out); + self.max_attachment_text.pickle(out); + self.max_message_text.pickle(out); + self.max_redirect_urls.pickle(out); + self.redirect_timeout.pickle(out); + self.redirect_cache_ttl.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.trust_contacts = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.greylist_for = Pickle::unpickle(stream)?; - this.score_discard = Pickle::unpickle(stream)?; - this.score_reject = Pickle::unpickle(stream)?; - this.score_spam = Pickle::unpickle(stream)?; - this.trust_replies = Pickle::unpickle(stream)?; - this.spam_filter_rules_url = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: bool = if block.has(1) { block.flag(1) } else { true }; + let field2: Option = if block.has(2) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Float = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Float::new(0.0f64) + }; + let field4: Float = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Float::new(0.0f64) + }; + let field5: Float = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Float::new(5.0f64) + }; + let field6: bool = if block.has(6) { block.flag(3) } else { true }; + let field7: Option = if block.has(7) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz".to_string()) + }; + let field8: u64 = if block.has(8) { + Pickle::unpickle(stream)? + } else { + 262144u64 + }; + let field9: u64 = if block.has(9) { + Pickle::unpickle(stream)? + } else { + 65536u64 + }; + let field10: u64 = if block.has(10) { + Pickle::unpickle(stream)? + } else { + 1048576u64 + }; + let field11: u64 = if block.has(11) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field12: Duration = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(5000) + }; + let field13: Duration = if block.has(13) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(86400000) + }; + stream.close_block(block); + Some(Self { + trust_contacts: field0, + enable: field1, + greylist_for: field2, + score_discard: field3, + score_reject: field4, + score_spam: field5, + trust_replies: field6, + spam_filter_rules_url: field7, + max_body_text: field8, + max_attachment_text: field9, + max_message_text: field10, + max_redirect_urls: field11, + redirect_timeout: field12, + redirect_cache_ttl: field13, + }) } } @@ -40389,23 +52090,72 @@ impl Default for SpamSettings { score_spam: Float::new(5.0f64), trust_replies: true, spam_filter_rules_url: None, + max_body_text: 262144u64, + max_attachment_text: 65536u64, + max_message_text: 1048576u64, + max_redirect_urls: 10u64, + redirect_timeout: Duration::from_millis(5000), + redirect_cache_ttl: Duration::from_millis(86400000), } } } impl IntoValue for SpamSettings { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::TrustContacts, self.trust_contacts.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::GreylistFor, self.greylist_for.into_value()); - map.insert_unchecked(Property::ScoreDiscard, self.score_discard.into_value()); - map.insert_unchecked(Property::ScoreReject, self.score_reject.into_value()); - map.insert_unchecked(Property::ScoreSpam, self.score_spam.into_value()); - map.insert_unchecked(Property::TrustReplies, self.trust_replies.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(16); + map.insert_unchecked( + Property::TrustContacts, + self.trust_contacts.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked( + Property::GreylistFor, + self.greylist_for.into_value_with(opts), + ); + map.insert_unchecked( + Property::ScoreDiscard, + self.score_discard.into_value_with(opts), + ); + map.insert_unchecked( + Property::ScoreReject, + self.score_reject.into_value_with(opts), + ); + map.insert_unchecked(Property::ScoreSpam, self.score_spam.into_value_with(opts)); + map.insert_unchecked( + Property::TrustReplies, + self.trust_replies.into_value_with(opts), + ); map.insert_unchecked( Property::SpamFilterRulesUrl, - self.spam_filter_rules_url.into_value(), + self.spam_filter_rules_url.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxBodyText, + self.max_body_text.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxAttachmentText, + self.max_attachment_text.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxMessageText, + self.max_message_text.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRedirectUrls, + self.max_redirect_urls.into_value_with(opts), + ); + map.insert_unchecked( + Property::RedirectTimeout, + self.redirect_timeout.into_value_with(opts), + ); + map.insert_unchecked( + Property::RedirectCacheTtl, + self.redirect_cache_ttl.into_value_with(opts), ); JmapValue::Object(map) } @@ -40428,6 +52178,12 @@ impl RegistryJsonPropertyPatch for SpamSettings { Some(Property::SpamFilterRulesUrl) => self .spam_filter_rules_url .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::MaxBodyText) => self.max_body_text.patch(pointer, value), + Some(Property::MaxAttachmentText) => self.max_attachment_text.patch(pointer, value), + Some(Property::MaxMessageText) => self.max_message_text.patch(pointer, value), + Some(Property::MaxRedirectUrls) => self.max_redirect_urls.patch(pointer, value), + Some(Property::RedirectTimeout) => self.redirect_timeout.patch(pointer, value), + Some(Property::RedirectCacheTtl) => self.redirect_cache_ttl.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -40490,6 +52246,9 @@ impl Pickle for SpamTag { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(SpamTag::Score), 1 => Pickle::unpickle(stream).map(SpamTag::Discard), @@ -40501,23 +52260,27 @@ impl Pickle for SpamTag { impl IntoValue for SpamTag { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SpamTag::Score(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Score".into())); obj } SpamTag::Discard(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Discard".into())); obj } SpamTag::Reject(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Reject".into())); @@ -40575,13 +52338,20 @@ impl SpamTagAction { impl Pickle for SpamTagAction { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.tag.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { tag: field0 }) } } @@ -40595,8 +52365,12 @@ impl Default for SpamTagAction { impl IntoValue for SpamTagAction { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Tag, self.tag.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); JmapValue::Object(map) } } @@ -40646,15 +52420,29 @@ impl SpamTagScore { impl Pickle for SpamTagScore { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.tag.pickle(out); self.score.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tag = Pickle::unpickle(stream)?; - this.score = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Float = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Float::new(0.0f64) + }; + stream.close_block(block); + Some(Self { + tag: field0, + score: field1, + }) } } @@ -40669,9 +52457,13 @@ impl Default for SpamTagScore { impl IntoValue for SpamTagScore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Tag, self.tag.into_value()); - map.insert_unchecked(Property::Score, self.score.into_value()); + map.insert_unchecked(Property::Tag, self.tag.into_value_with(opts)); + map.insert_unchecked(Property::Score, self.score.into_value_with(opts)); JmapValue::Object(map) } } @@ -40699,7 +52491,7 @@ impl RegistryJsonPropertyPatch for SpamTagScore { } impl ObjectImpl for SpamTrainingSample { - const FLAGS: u64 = OBJ_FILTER_ACCOUNT; + const FLAGS: u64 = OBJ_FILTER_ACCOUNT | OBJ_NO_UPDATE; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::SpamTrainingSample; @@ -40739,25 +52531,73 @@ impl ObjectImpl for SpamTrainingSample { impl Pickle for SpamTrainingSample { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.is_spam { + bitmap |= 1 << 0; + } + if self.account_id.is_some() { + bitmap |= 1 << 1; + } + if self.delete_after_use { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 7, 3, bitmap); self.from.pickle(out); self.subject.pickle(out); self.blob_id.pickle(out); - self.is_spam.pickle(out); - self.account_id.pickle(out); + if let Some(value) = &self.account_id { + value.pickle(out); + } self.expires_at.pickle(out); - self.delete_after_use.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.blob_id = Pickle::unpickle(stream)?; - this.is_spam = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.delete_after_use = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: BlobId = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: bool = if block.has(3) { block.flag(0) } else { false }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: bool = if block.has(6) { block.flag(2) } else { false }; + stream.close_block(block); + Some(Self { + from: field0, + subject: field1, + blob_id: field2, + is_spam: field3, + account_id: field4, + expires_at: field5, + delete_after_use: field6, + }) } } @@ -40777,14 +52617,21 @@ impl Default for SpamTrainingSample { impl IntoValue for SpamTrainingSample { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - map.insert_unchecked(Property::IsSpam, self.is_spam.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::DeleteAfterUse, self.delete_after_use.into_value()); + map.insert_unchecked(Property::From, self.from.into_value_with(opts)); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); + map.insert_unchecked(Property::BlobId, self.blob_id.into_value_with(opts)); + map.insert_unchecked(Property::IsSpam, self.is_spam.into_value_with(opts)); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); + map.insert_unchecked( + Property::DeleteAfterUse, + self.delete_after_use.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -40844,7 +52691,7 @@ impl SpfReportSettings { ExpressionContext { expr: &self.from_address, default: Some(Expression { - else_: "'noreply-spf@' + system('domain')".to_string(), + else_: "'noreply-spf@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::FromAddress, @@ -40883,7 +52730,7 @@ impl SpfReportSettings { ExpressionContext { expr: &self.dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::DkimSignDomain, @@ -40918,21 +52765,68 @@ impl SpfReportSettings { impl Pickle for SpfReportSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.from_address.pickle(out); self.from_name.pickle(out); self.send_frequency.pickle(out); self.dkim_sign_domain.pickle(out); self.subject.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from_address = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.send_frequency = Pickle::unpickle(stream)?; - this.dkim_sign_domain = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'noreply-spf@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Report Subsystem'".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "[1, 1d]".to_string(), + ..Default::default() + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'SPF Authentication Failure Report'".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + from_address: field0, + from_name: field1, + send_frequency: field2, + dkim_sign_domain: field3, + subject: field4, + }) } } @@ -40940,7 +52834,7 @@ impl Default for SpfReportSettings { fn default() -> Self { Self { from_address: Expression { - else_: "'noreply-spf@' + system('domain')".to_string(), + else_: "'noreply-spf@' + system('report_domain')".to_string(), ..Default::default() }, from_name: Expression { @@ -40952,7 +52846,7 @@ impl Default for SpfReportSettings { ..Default::default() }, dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, subject: Expression { @@ -40965,12 +52859,25 @@ impl Default for SpfReportSettings { impl IntoValue for SpfReportSettings { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::FromAddress, self.from_address.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); - map.insert_unchecked(Property::SendFrequency, self.send_frequency.into_value()); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); + map.insert_unchecked( + Property::FromAddress, + self.from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FromName, self.from_name.into_value_with(opts)); + map.insert_unchecked( + Property::SendFrequency, + self.send_frequency.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), + ); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); JmapValue::Object(map) } } @@ -41005,6 +52912,7 @@ impl SqlAuthStore { SqlAuthStore::Sqlite(inner) => inner.validate(errors), } } + } impl Default for SqlAuthStore { @@ -41047,6 +52955,10 @@ impl Pickle for SqlAuthStore { impl IntoValue for SqlAuthStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SqlAuthStore::Default => { let mut obj = jmap_tools::Map::new(); @@ -41054,21 +52966,21 @@ impl IntoValue for SqlAuthStore { JmapValue::Object(obj) } SqlAuthStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); obj } SqlAuthStore::MySql(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); obj } SqlAuthStore::Sqlite(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Sqlite".into())); @@ -41159,51 +53071,134 @@ impl SqlDirectory { errors.push(ValidationError::required(Property::QueryEmailAliases)); } } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } } } impl Pickle for SqlDirectory { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.column_class.is_some() { + bitmap |= 1 << 0; + } + if self.column_description.is_some() { + bitmap |= 1 << 1; + } + if self.query_member_of.is_some() { + bitmap |= 1 << 2; + } + if self.query_email_aliases.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 11, 5, bitmap); self.description.pickle(out); self.store.pickle(out); self.column_email.pickle(out); self.column_secret.pickle(out); - self.column_class.pickle(out); - self.column_description.pickle(out); + if let Some(value) = &self.column_class { + value.pickle(out); + } + if let Some(value) = &self.column_description { + value.pickle(out); + } self.query_login.pickle(out); self.query_recipient.pickle(out); - self.query_member_of.pickle(out); - self.query_email_aliases.pickle(out); - self.member_tenant_id.pickle(out); + if let Some(value) = &self.query_member_of { + value.pickle(out); + } + if let Some(value) = &self.query_email_aliases { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.description = Pickle::unpickle(stream)?; - this.store = Pickle::unpickle(stream)?; - this.column_email = Pickle::unpickle(stream)?; - this.column_secret = Pickle::unpickle(stream)?; - this.column_class = Pickle::unpickle(stream)?; - this.column_description = Pickle::unpickle(stream)?; - this.query_login = Pickle::unpickle(stream)?; - this.query_recipient = Pickle::unpickle(stream)?; - this.query_member_of = Pickle::unpickle(stream)?; - this.query_email_aliases = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: SqlAuthStore = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + "name".to_string() + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + "secret".to_string() + }; + let field4: Option = if block.has(4) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("type".to_string()) + }; + let field5: Option = if block.has(5) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("description".to_string()) + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + "SELECT name, secret, description, type FROM accounts WHERE name = $1 AND active = true" + .to_string() + }; + let field7: String = if block.has(7) { + Pickle::unpickle(stream)? + } else { + "SELECT name, secret, description, type FROM accounts WHERE name = $1 AND active = true" + .to_string() + }; + let field8: Option = if block.has(8) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("SELECT member_of FROM group_members WHERE name = $1".to_string()) + }; + let field9: Option = if block.has(9) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Some("SELECT address FROM emails WHERE name = $1".to_string()) + }; + stream.close_block(block); + Some(Self { + description: field0, + store: field1, + column_email: field2, + column_secret: field3, + column_class: field4, + column_description: field5, + query_login: field6, + query_recipient: field7, + query_member_of: field8, + query_email_aliases: field9, + }) } } @@ -41216,35 +53211,55 @@ impl Default for SqlDirectory { column_secret: "secret".to_string(), column_class: Some("type".to_string()), column_description: Some("description".to_string()), - query_login: "SELECT name, secret, description, type FROM accounts WHERE name = $1".to_string(), + query_login: "SELECT name, secret, description, type FROM accounts WHERE name = $1 AND active = true".to_string(), query_recipient: "SELECT name, secret, description, type FROM accounts WHERE name = $1 AND active = true".to_string(), query_member_of: Some("SELECT member_of FROM group_members WHERE name = $1".to_string()), query_email_aliases: Some("SELECT address FROM emails WHERE name = $1".to_string()), - member_tenant_id: Default::default(), } } } impl IntoValue for SqlDirectory { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(13); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Store, self.store.into_value()); - map.insert_unchecked(Property::ColumnEmail, self.column_email.into_value()); - map.insert_unchecked(Property::ColumnSecret, self.column_secret.into_value()); - map.insert_unchecked(Property::ColumnClass, self.column_class.into_value()); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Store, self.store.into_value_with(opts)); + map.insert_unchecked( + Property::ColumnEmail, + self.column_email.into_value_with(opts), + ); + map.insert_unchecked( + Property::ColumnSecret, + self.column_secret.into_value_with(opts), + ); + map.insert_unchecked( + Property::ColumnClass, + self.column_class.into_value_with(opts), + ); map.insert_unchecked( Property::ColumnDescription, - self.column_description.into_value(), + self.column_description.into_value_with(opts), + ); + map.insert_unchecked(Property::QueryLogin, self.query_login.into_value_with(opts)); + map.insert_unchecked( + Property::QueryRecipient, + self.query_recipient.into_value_with(opts), + ); + map.insert_unchecked( + Property::QueryMemberOf, + self.query_member_of.into_value_with(opts), ); - map.insert_unchecked(Property::QueryLogin, self.query_login.into_value()); - map.insert_unchecked(Property::QueryRecipient, self.query_recipient.into_value()); - map.insert_unchecked(Property::QueryMemberOf, self.query_member_of.into_value()); map.insert_unchecked( Property::QueryEmailAliases, - self.query_email_aliases.into_value(), + self.query_email_aliases.into_value_with(opts), ); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); JmapValue::Object(map) } } @@ -41284,9 +53299,6 @@ impl RegistryJsonPropertyPatch for SqlDirectory { Some(Property::QueryEmailAliases) => self .query_email_aliases .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -41311,15 +53323,16 @@ impl SqliteStore { errors.push(ValidationError::min_value(Property::PoolWorkers, 1)); } } - let value = &self.pool_max_connections; - if *value > 8192 { - errors.push(ValidationError::max_value( - Property::PoolMaxConnections, - 8192, - )); - } - if *value < 1 { - errors.push(ValidationError::min_value(Property::PoolMaxConnections, 1)); + if let Some(value) = &self.pool_max_connections { + if *value > 8192 { + errors.push(ValidationError::max_value( + Property::PoolMaxConnections, + 8192, + )); + } + if *value < 1 { + errors.push(ValidationError::min_value(Property::PoolMaxConnections, 1)); + } } errors.len() == neb } @@ -41327,17 +53340,55 @@ impl SqliteStore { impl Pickle for SqliteStore { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.pool_workers.is_some() { + bitmap |= 1 << 0; + } + if self.pool_max_connections.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 3, 2, bitmap); self.path.pickle(out); - self.pool_workers.pickle(out); - self.pool_max_connections.pickle(out); + if let Some(value) = &self.pool_workers { + value.pickle(out); + } + if let Some(value) = &self.pool_max_connections { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.path = Pickle::unpickle(stream)?; - this.pool_workers = Pickle::unpickle(stream)?; - this.pool_max_connections = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + path: field0, + pool_workers: field1, + pool_max_connections: field2, + }) } } @@ -41346,19 +53397,26 @@ impl Default for SqliteStore { Self { path: Default::default(), pool_workers: Default::default(), - pool_max_connections: 10u64, + pool_max_connections: Default::default(), } } } impl IntoValue for SqliteStore { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Path, self.path.into_value()); - map.insert_unchecked(Property::PoolWorkers, self.pool_workers.into_value()); + map.insert_unchecked(Property::Path, self.path.into_value_with(opts)); + map.insert_unchecked( + Property::PoolWorkers, + self.pool_workers.into_value_with(opts), + ); map.insert_unchecked( Property::PoolMaxConnections, - self.pool_max_connections.into_value(), + self.pool_max_connections.into_value_with(opts), ); JmapValue::Object(map) } @@ -41408,15 +53466,32 @@ impl ObjectImpl for StoreLookup { impl Pickle for StoreLookup { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.namespace.pickle(out); self.store.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.namespace = Pickle::unpickle(stream)?; - this.store = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: LookupStore = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + namespace: field0, + store: field1, + }) } } @@ -41431,9 +53506,13 @@ impl Default for StoreLookup { impl IntoValue for StoreLookup { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Namespace, self.namespace.into_value()); - map.insert_unchecked(Property::Store, self.store.into_value()); + map.insert_unchecked(Property::Namespace, self.namespace.into_value_with(opts)); + map.insert_unchecked(Property::Store, self.store.into_value_with(opts)); JmapValue::Object(map) } } @@ -41505,6 +53584,10 @@ impl Pickle for SubAddressing { impl IntoValue for SubAddressing { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { SubAddressing::Enabled => { let mut obj = jmap_tools::Map::new(); @@ -41512,7 +53595,7 @@ impl IntoValue for SubAddressing { JmapValue::Object(obj) } SubAddressing::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); @@ -41592,13 +53675,22 @@ impl SubAddressingCustom { impl Pickle for SubAddressingCustom { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.custom_rule.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.custom_rule = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + custom_rule: field0, + }) } } @@ -41612,8 +53704,12 @@ impl Default for SubAddressingCustom { impl IntoValue for SubAddressingCustom { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::CustomRule, self.custom_rule.into_value()); + map.insert_unchecked(Property::CustomRule, self.custom_rule.into_value_with(opts)); JmapValue::Object(map) } } @@ -41655,15 +53751,18 @@ impl ObjectImpl for SystemSettings { errors.push(ValidationError::required(Property::DefaultCertificateId)); } } - if let Some(value) = &self.thread_pool_size { - if *value < 1 { - errors.push(ValidationError::min_value(Property::ThreadPoolSize, 1)); - } - } let value = &self.max_connections; if *value < 1 { errors.push(ValidationError::min_value(Property::MaxConnections, 1)); } + if let Some(value) = &self.max_buffered_memory { + if *value < 16777216 { + errors.push(ValidationError::min_value( + Property::MaxBufferedMemory, + 16777216, + )); + } + } let value = &self.proxy_trusted_networks; for value in value.iter() { if !value.is_valid() { @@ -41698,29 +53797,165 @@ impl ObjectImpl for SystemSettings { impl Pickle for SystemSettings { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.default_certificate_id.is_some() { + bitmap |= 1 << 0; + } + if self.max_buffered_memory.is_some() { + bitmap |= 1 << 1; + } + if self.include_secrets { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 10, 3, bitmap); self.default_hostname.pickle(out); self.default_domain_id.pickle(out); - self.default_certificate_id.pickle(out); - self.thread_pool_size.pickle(out); + if let Some(value) = &self.default_certificate_id { + value.pickle(out); + } self.max_connections.pickle(out); + if let Some(value) = &self.max_buffered_memory { + value.pickle(out); + } self.proxy_trusted_networks.pickle(out); self.mail_exchangers.pickle(out); self.services.pickle(out); self.provider_info.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.default_hostname = Pickle::unpickle(stream)?; - this.default_domain_id = Pickle::unpickle(stream)?; - this.default_certificate_id = Pickle::unpickle(stream)?; - this.thread_pool_size = Pickle::unpickle(stream)?; - this.max_connections = Pickle::unpickle(stream)?; - this.proxy_trusted_networks = Pickle::unpickle(stream)?; - this.mail_exchangers = Pickle::unpickle(stream)?; - this.services = Pickle::unpickle(stream)?; - this.provider_info = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 8192u64 + }; + let field4: Option = if block.has(4) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: Map = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: List = if block.has(6) { + Pickle::unpickle(stream)? + } else { + List::from_iter([MailExchanger { + priority: 10u64, + ..Default::default() + }]) + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + VecMap::from_iter([ + ( + ServiceProtocol::Caldav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Carddav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Imap, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Jmap, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Managesieve, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Pop3, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Smtp, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Webdav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ]) + }; + let field8: VecMap = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: bool = if block.has(9) { block.flag(2) } else { false }; + stream.close_block(block); + Some(Self { + default_hostname: field0, + default_domain_id: field1, + default_certificate_id: field2, + max_connections: field3, + max_buffered_memory: field4, + proxy_trusted_networks: field5, + mail_exchangers: field6, + services: field7, + provider_info: field8, + include_secrets: field9, + }) } } @@ -41730,8 +53965,8 @@ impl Default for SystemSettings { default_hostname: Default::default(), default_domain_id: Default::default(), default_certificate_id: Default::default(), - thread_pool_size: Default::default(), max_connections: 8192u64, + max_buffered_memory: Default::default(), proxy_trusted_networks: Default::default(), mail_exchangers: List::from_iter([MailExchanger { priority: 10u64, @@ -41796,34 +54031,55 @@ impl Default for SystemSettings { ), ]), provider_info: Default::default(), + include_secrets: false, } } } impl IntoValue for SystemSettings { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(12); map.insert_unchecked( Property::DefaultHostname, - self.default_hostname.into_value(), + self.default_hostname.into_value_with(opts), ); map.insert_unchecked( Property::DefaultDomainId, - self.default_domain_id.into_value(), + self.default_domain_id.into_value_with(opts), ); map.insert_unchecked( Property::DefaultCertificateId, - self.default_certificate_id.into_value(), + self.default_certificate_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxConnections, + self.max_connections.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxBufferedMemory, + self.max_buffered_memory.into_value_with(opts), ); - map.insert_unchecked(Property::ThreadPoolSize, self.thread_pool_size.into_value()); - map.insert_unchecked(Property::MaxConnections, self.max_connections.into_value()); map.insert_unchecked( Property::ProxyTrustedNetworks, - self.proxy_trusted_networks.into_value(), + self.proxy_trusted_networks.into_value_with(opts), + ); + map.insert_unchecked( + Property::MailExchangers, + self.mail_exchangers.into_value_with(opts), + ); + map.insert_unchecked(Property::Services, self.services.into_value_with(opts)); + map.insert_unchecked( + Property::ProviderInfo, + self.provider_info.into_value_with(opts), + ); + map.insert_unchecked( + Property::IncludeSecrets, + self.include_secrets.into_value_with(opts), ); - map.insert_unchecked(Property::MailExchangers, self.mail_exchangers.into_value()); - map.insert_unchecked(Property::Services, self.services.into_value()); - map.insert_unchecked(Property::ProviderInfo, self.provider_info.into_value()); JmapValue::Object(map) } } @@ -41842,14 +54098,15 @@ impl RegistryJsonPropertyPatch for SystemSettings { Some(Property::DefaultCertificateId) => { self.default_certificate_id.patch(pointer, value) } - Some(Property::ThreadPoolSize) => self.thread_pool_size.patch(pointer, value), Some(Property::MaxConnections) => self.max_connections.patch(pointer, value), + Some(Property::MaxBufferedMemory) => self.max_buffered_memory.patch(pointer, value), Some(Property::ProxyTrustedNetworks) => self .proxy_trusted_networks .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::MailExchangers) => self.mail_exchangers.patch(pointer, value), Some(Property::Services) => self.services.patch(pointer, value), Some(Property::ProviderInfo) => self.provider_info.patch(pointer, value), + Some(Property::IncludeSecrets) => self.include_secrets.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -41866,38 +54123,27 @@ impl ObjectImpl for Task { fn validate(&self, errors: &mut Vec) -> bool { match self { - Task::IndexDocument(inner) => inner.validate(errors), - Task::UnindexDocument(inner) => inner.validate(errors), - Task::IndexTrace(inner) => inner.validate(errors), Task::CalendarAlarmEmail(inner) => inner.validate(errors), Task::CalendarAlarmNotification(inner) => inner.validate(errors), Task::CalendarItipMessage(inner) => inner.validate(errors), Task::MergeThreads(inner) => inner.validate(errors), Task::DmarcReport(inner) => inner.validate(errors), Task::TlsReport(inner) => inner.validate(errors), - Task::RestoreArchivedItem(inner) => inner.validate(errors), Task::DestroyAccount(inner) => inner.validate(errors), Task::AccountMaintenance(inner) => inner.validate(errors), - Task::TenantMaintenance(inner) => inner.validate(errors), Task::StoreMaintenance(inner) => inner.validate(errors), Task::SpamFilterMaintenance(inner) => inner.validate(errors), Task::AcmeRenewal(inner) => inner.validate(errors), Task::DkimManagement(inner) => inner.validate(errors), Task::DnsManagement(inner) => inner.validate(errors), + Task::ShareRevokeCleanup(inner) => inner.validate(errors), + Task::DmarcReportWindow(inner) => inner.validate(errors), + Task::TlsReportWindow(inner) => inner.validate(errors), } } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { match self { - Task::IndexDocument(object) => { - object.index(i); - } - Task::UnindexDocument(object) => { - object.index(i); - } - Task::IndexTrace(object) => { - object.index(i); - } Task::CalendarAlarmEmail(object) => { object.index(i); } @@ -41916,18 +54162,12 @@ impl ObjectImpl for Task { Task::TlsReport(object) => { object.index(i); } - Task::RestoreArchivedItem(object) => { - object.index(i); - } Task::DestroyAccount(object) => { object.index(i); } Task::AccountMaintenance(object) => { object.index(i); } - Task::TenantMaintenance(object) => { - object.index(i); - } Task::StoreMaintenance(_) => {} Task::SpamFilterMaintenance(_) => {} Task::AcmeRenewal(object) => { @@ -41939,88 +54179,85 @@ impl ObjectImpl for Task { Task::DnsManagement(object) => { object.index(i); } + Task::ShareRevokeCleanup(object) => { + object.index(i); + } + Task::DmarcReportWindow(_) => {} + Task::TlsReportWindow(_) => {} } } } impl Default for Task { fn default() -> Self { - Task::IndexDocument(Default::default()) + Task::CalendarAlarmEmail(Default::default()) } } impl Pickle for Task { fn pickle(&self, out: &mut Vec) { match self { - Task::IndexDocument(inner) => { + Task::CalendarAlarmEmail(inner) => { 0u16.pickle(out); inner.pickle(out); } - Task::UnindexDocument(inner) => { + Task::CalendarAlarmNotification(inner) => { 1u16.pickle(out); inner.pickle(out); } - Task::IndexTrace(inner) => { + Task::CalendarItipMessage(inner) => { 2u16.pickle(out); inner.pickle(out); } - Task::CalendarAlarmEmail(inner) => { + Task::MergeThreads(inner) => { 3u16.pickle(out); inner.pickle(out); } - Task::CalendarAlarmNotification(inner) => { + Task::DmarcReport(inner) => { 4u16.pickle(out); inner.pickle(out); } - Task::CalendarItipMessage(inner) => { + Task::TlsReport(inner) => { 5u16.pickle(out); inner.pickle(out); } - Task::MergeThreads(inner) => { - 6u16.pickle(out); - inner.pickle(out); - } - Task::DmarcReport(inner) => { + Task::DestroyAccount(inner) => { 7u16.pickle(out); inner.pickle(out); } - Task::TlsReport(inner) => { + Task::AccountMaintenance(inner) => { 8u16.pickle(out); inner.pickle(out); } - Task::RestoreArchivedItem(inner) => { - 9u16.pickle(out); - inner.pickle(out); - } - Task::DestroyAccount(inner) => { + Task::StoreMaintenance(inner) => { 10u16.pickle(out); inner.pickle(out); } - Task::AccountMaintenance(inner) => { + Task::SpamFilterMaintenance(inner) => { 11u16.pickle(out); inner.pickle(out); } - Task::TenantMaintenance(inner) => { + Task::AcmeRenewal(inner) => { 12u16.pickle(out); inner.pickle(out); } - Task::StoreMaintenance(inner) => { + Task::DkimManagement(inner) => { 13u16.pickle(out); inner.pickle(out); } - Task::SpamFilterMaintenance(inner) => { + Task::DnsManagement(inner) => { 14u16.pickle(out); inner.pickle(out); } - Task::AcmeRenewal(inner) => { + Task::ShareRevokeCleanup(inner) => { 15u16.pickle(out); inner.pickle(out); } - Task::DkimManagement(inner) => { + Task::DmarcReportWindow(inner) => { 16u16.pickle(out); inner.pickle(out); } - Task::DnsManagement(inner) => { + Task::TlsReportWindow(inner) => { 17u16.pickle(out); inner.pickle(out); } @@ -42028,25 +54265,26 @@ impl Pickle for Task { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(Task::IndexDocument), - 1 => Pickle::unpickle(stream).map(Task::UnindexDocument), - 2 => Pickle::unpickle(stream).map(Task::IndexTrace), - 3 => Pickle::unpickle(stream).map(Task::CalendarAlarmEmail), - 4 => Pickle::unpickle(stream).map(Task::CalendarAlarmNotification), - 5 => Pickle::unpickle(stream).map(Task::CalendarItipMessage), - 6 => Pickle::unpickle(stream).map(Task::MergeThreads), - 7 => Pickle::unpickle(stream).map(Task::DmarcReport), - 8 => Pickle::unpickle(stream).map(Task::TlsReport), - 9 => Pickle::unpickle(stream).map(Task::RestoreArchivedItem), - 10 => Pickle::unpickle(stream).map(Task::DestroyAccount), - 11 => Pickle::unpickle(stream).map(Task::AccountMaintenance), - 12 => Pickle::unpickle(stream).map(Task::TenantMaintenance), - 13 => Pickle::unpickle(stream).map(Task::StoreMaintenance), - 14 => Pickle::unpickle(stream).map(Task::SpamFilterMaintenance), - 15 => Pickle::unpickle(stream).map(Task::AcmeRenewal), - 16 => Pickle::unpickle(stream).map(Task::DkimManagement), - 17 => Pickle::unpickle(stream).map(Task::DnsManagement), + 0 => Pickle::unpickle(stream).map(Task::CalendarAlarmEmail), + 1 => Pickle::unpickle(stream).map(Task::CalendarAlarmNotification), + 2 => Pickle::unpickle(stream).map(Task::CalendarItipMessage), + 3 => Pickle::unpickle(stream).map(Task::MergeThreads), + 4 => Pickle::unpickle(stream).map(Task::DmarcReport), + 5 => Pickle::unpickle(stream).map(Task::TlsReport), + 7 => Pickle::unpickle(stream).map(Task::DestroyAccount), + 8 => Pickle::unpickle(stream).map(Task::AccountMaintenance), + 10 => Pickle::unpickle(stream).map(Task::StoreMaintenance), + 11 => Pickle::unpickle(stream).map(Task::SpamFilterMaintenance), + 12 => Pickle::unpickle(stream).map(Task::AcmeRenewal), + 13 => Pickle::unpickle(stream).map(Task::DkimManagement), + 14 => Pickle::unpickle(stream).map(Task::DnsManagement), + 15 => Pickle::unpickle(stream).map(Task::ShareRevokeCleanup), + 16 => Pickle::unpickle(stream).map(Task::DmarcReportWindow), + 17 => Pickle::unpickle(stream).map(Task::TlsReportWindow), _ => None, } } @@ -42054,37 +54292,20 @@ impl Pickle for Task { impl IntoValue for Task { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { - Task::IndexDocument(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("IndexDocument".into())); - obj - } - Task::UnindexDocument(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("UnindexDocument".into())); - obj - } - Task::IndexTrace(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("IndexTrace".into())); - obj - } Task::CalendarAlarmEmail(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("CalendarAlarmEmail".into())); obj } Task::CalendarAlarmNotification(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut().unwrap().insert_unchecked( Property::Type, JmapValue::Str("CalendarAlarmNotification".into()), @@ -42092,70 +54313,56 @@ impl IntoValue for Task { obj } Task::CalendarItipMessage(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("CalendarItipMessage".into())); obj } Task::MergeThreads(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("MergeThreads".into())); obj } Task::DmarcReport(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DmarcReport".into())); obj } Task::TlsReport(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("TlsReport".into())); obj } - Task::RestoreArchivedItem(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("RestoreArchivedItem".into())); - obj - } Task::DestroyAccount(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DestroyAccount".into())); obj } Task::AccountMaintenance(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("AccountMaintenance".into())); obj } - Task::TenantMaintenance(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("TenantMaintenance".into())); - obj - } Task::StoreMaintenance(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("StoreMaintenance".into())); obj } Task::SpamFilterMaintenance(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut().unwrap().insert_unchecked( Property::Type, JmapValue::Str("SpamFilterMaintenance".into()), @@ -42163,26 +54370,47 @@ impl IntoValue for Task { obj } Task::AcmeRenewal(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("AcmeRenewal".into())); obj } Task::DkimManagement(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DkimManagement".into())); obj } Task::DnsManagement(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("DnsManagement".into())); obj } + Task::ShareRevokeCleanup(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("ShareRevokeCleanup".into())); + obj + } + Task::DmarcReportWindow(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("DmarcReportWindow".into())); + obj + } + Task::TlsReportWindow(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("TlsReportWindow".into())); + obj + } } } } @@ -42195,9 +54423,6 @@ impl RegistryJsonPatch for Task { ) -> PatchResult<'x> { if !pointer.has_next() { match object_type(&pointer, &value)? { - TaskType::IndexDocument => *self = Task::IndexDocument(Default::default()), - TaskType::UnindexDocument => *self = Task::UnindexDocument(Default::default()), - TaskType::IndexTrace => *self = Task::IndexTrace(Default::default()), TaskType::CalendarAlarmEmail => { *self = Task::CalendarAlarmEmail(Default::default()) } @@ -42210,14 +54435,10 @@ impl RegistryJsonPatch for Task { TaskType::MergeThreads => *self = Task::MergeThreads(Default::default()), TaskType::DmarcReport => *self = Task::DmarcReport(Default::default()), TaskType::TlsReport => *self = Task::TlsReport(Default::default()), - TaskType::RestoreArchivedItem => { - *self = Task::RestoreArchivedItem(Default::default()) - } TaskType::DestroyAccount => *self = Task::DestroyAccount(Default::default()), TaskType::AccountMaintenance => { *self = Task::AccountMaintenance(Default::default()) } - TaskType::TenantMaintenance => *self = Task::TenantMaintenance(Default::default()), TaskType::StoreMaintenance => *self = Task::StoreMaintenance(Default::default()), TaskType::SpamFilterMaintenance => { *self = Task::SpamFilterMaintenance(Default::default()) @@ -42225,27 +54446,30 @@ impl RegistryJsonPatch for Task { TaskType::AcmeRenewal => *self = Task::AcmeRenewal(Default::default()), TaskType::DkimManagement => *self = Task::DkimManagement(Default::default()), TaskType::DnsManagement => *self = Task::DnsManagement(Default::default()), + TaskType::ShareRevokeCleanup => { + *self = Task::ShareRevokeCleanup(Default::default()) + } + TaskType::DmarcReportWindow => *self = Task::DmarcReportWindow(Default::default()), + TaskType::TlsReportWindow => *self = Task::TlsReportWindow(Default::default()), } } match self { - Task::IndexDocument(inner) => inner.patch(pointer, value), - Task::UnindexDocument(inner) => inner.patch(pointer, value), - Task::IndexTrace(inner) => inner.patch(pointer, value), Task::CalendarAlarmEmail(inner) => inner.patch(pointer, value), Task::CalendarAlarmNotification(inner) => inner.patch(pointer, value), Task::CalendarItipMessage(inner) => inner.patch(pointer, value), Task::MergeThreads(inner) => inner.patch(pointer, value), Task::DmarcReport(inner) => inner.patch(pointer, value), Task::TlsReport(inner) => inner.patch(pointer, value), - Task::RestoreArchivedItem(inner) => inner.patch(pointer, value), Task::DestroyAccount(inner) => inner.patch(pointer, value), Task::AccountMaintenance(inner) => inner.patch(pointer, value), - Task::TenantMaintenance(inner) => inner.patch(pointer, value), Task::StoreMaintenance(inner) => inner.patch(pointer, value), Task::SpamFilterMaintenance(inner) => inner.patch(pointer, value), Task::AcmeRenewal(inner) => inner.patch(pointer, value), Task::DkimManagement(inner) => inner.patch(pointer, value), Task::DnsManagement(inner) => inner.patch(pointer, value), + Task::ShareRevokeCleanup(inner) => inner.patch(pointer, value), + Task::DmarcReportWindow(inner) => inner.patch(pointer, value), + Task::TlsReportWindow(inner) => inner.patch(pointer, value), } } } @@ -42253,24 +54477,22 @@ impl RegistryJsonPatch for Task { impl Task { pub fn object_type(&self) -> TaskType { match self { - Task::IndexDocument(_) => TaskType::IndexDocument, - Task::UnindexDocument(_) => TaskType::UnindexDocument, - Task::IndexTrace(_) => TaskType::IndexTrace, Task::CalendarAlarmEmail(_) => TaskType::CalendarAlarmEmail, Task::CalendarAlarmNotification(_) => TaskType::CalendarAlarmNotification, Task::CalendarItipMessage(_) => TaskType::CalendarItipMessage, Task::MergeThreads(_) => TaskType::MergeThreads, Task::DmarcReport(_) => TaskType::DmarcReport, Task::TlsReport(_) => TaskType::TlsReport, - Task::RestoreArchivedItem(_) => TaskType::RestoreArchivedItem, Task::DestroyAccount(_) => TaskType::DestroyAccount, Task::AccountMaintenance(_) => TaskType::AccountMaintenance, - Task::TenantMaintenance(_) => TaskType::TenantMaintenance, Task::StoreMaintenance(_) => TaskType::StoreMaintenance, Task::SpamFilterMaintenance(_) => TaskType::SpamFilterMaintenance, Task::AcmeRenewal(_) => TaskType::AcmeRenewal, Task::DkimManagement(_) => TaskType::DkimManagement, Task::DnsManagement(_) => TaskType::DnsManagement, + Task::ShareRevokeCleanup(_) => TaskType::ShareRevokeCleanup, + Task::DmarcReportWindow(_) => TaskType::DmarcReportWindow, + Task::TlsReportWindow(_) => TaskType::TlsReportWindow, } } } @@ -42294,17 +54516,36 @@ impl TaskAccountMaintenance { impl Pickle for TaskAccountMaintenance { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 3, 0, 0); self.account_id.pickle(out); self.maintenance_type.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.account_id = Pickle::unpickle(stream)?; - this.maintenance_type = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskAccountMaintenanceType = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: TaskStatus = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + account_id: field0, + maintenance_type: field1, + status: field2, + }) } } @@ -42320,13 +54561,17 @@ impl Default for TaskAccountMaintenance { impl IntoValue for TaskAccountMaintenance { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); map.insert_unchecked( Property::MaintenanceType, - self.maintenance_type.into_value(), + self.maintenance_type.into_value_with(opts), ); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42366,6 +54611,11 @@ impl TaskCalendarAlarmEmail { if !value.is_valid() { errors.push(ValidationError::invalid(Property::EventEnd, value)); } + if let Some(value) = &self.target_account_id { + if !value.is_valid() { + errors.push(ValidationError::required(Property::TargetAccountId)); + } + } let value = &self.account_id; if !value.is_valid() { errors.push(ValidationError::required(Property::AccountId)); @@ -42380,35 +54630,118 @@ impl TaskCalendarAlarmEmail { } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { + i.foreign_key(ObjectType::Account, self.target_account_id, None); i.foreign_key(ObjectType::Account, self.account_id.into(), None); } } impl Pickle for TaskCalendarAlarmEmail { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.target_account_id.is_some() { + bitmap |= 1 << 0; + } + if self.recurrence_id.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 11, 2, bitmap); self.alarm_id.pickle(out); self.event_id.pickle(out); self.event_start.pickle(out); self.event_end.pickle(out); self.event_start_tz.pickle(out); self.event_end_tz.pickle(out); + if let Some(value) = &self.target_account_id { + value.pickle(out); + } + if let Some(value) = &self.recurrence_id { + value.pickle(out); + } self.account_id.pickle(out); self.document_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.alarm_id = Pickle::unpickle(stream)?; - this.event_id = Pickle::unpickle(stream)?; - this.event_start = Pickle::unpickle(stream)?; - this.event_end = Pickle::unpickle(stream)?; - this.event_start_tz = Pickle::unpickle(stream)?; - this.event_end_tz = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.document_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: UTCDateTime = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field6: Option = if block.has(6) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field8: Id = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: Id = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field10: TaskStatus = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + alarm_id: field0, + event_id: field1, + event_start: field2, + event_end: field3, + event_start_tz: field4, + event_end_tz: field5, + target_account_id: field6, + recurrence_id: field7, + account_id: field8, + document_id: field9, + status: field10, + }) } } @@ -42421,6 +54754,8 @@ impl Default for TaskCalendarAlarmEmail { event_end: Default::default(), event_start_tz: 0u64, event_end_tz: 0u64, + target_account_id: Default::default(), + recurrence_id: Default::default(), account_id: Default::default(), document_id: Default::default(), status: Default::default(), @@ -42430,16 +54765,34 @@ impl Default for TaskCalendarAlarmEmail { impl IntoValue for TaskCalendarAlarmEmail { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(11); - map.insert_unchecked(Property::AlarmId, self.alarm_id.into_value()); - map.insert_unchecked(Property::EventId, self.event_id.into_value()); - map.insert_unchecked(Property::EventStart, self.event_start.into_value()); - map.insert_unchecked(Property::EventEnd, self.event_end.into_value()); - map.insert_unchecked(Property::EventStartTz, self.event_start_tz.into_value()); - map.insert_unchecked(Property::EventEndTz, self.event_end_tz.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::DocumentId, self.document_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(13); + map.insert_unchecked(Property::AlarmId, self.alarm_id.into_value_with(opts)); + map.insert_unchecked(Property::EventId, self.event_id.into_value_with(opts)); + map.insert_unchecked(Property::EventStart, self.event_start.into_value_with(opts)); + map.insert_unchecked(Property::EventEnd, self.event_end.into_value_with(opts)); + map.insert_unchecked( + Property::EventStartTz, + self.event_start_tz.into_value_with(opts), + ); + map.insert_unchecked( + Property::EventEndTz, + self.event_end_tz.into_value_with(opts), + ); + map.insert_unchecked( + Property::TargetAccountId, + self.target_account_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::RecurrenceId, + self.recurrence_id.into_value_with(opts), + ); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::DocumentId, self.document_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42457,6 +54810,8 @@ impl RegistryJsonPropertyPatch for TaskCalendarAlarmEmail { Some(Property::EventEnd) => pointer.assert_server_set(), Some(Property::EventStartTz) => pointer.assert_server_set(), Some(Property::EventEndTz) => pointer.assert_server_set(), + Some(Property::TargetAccountId) => pointer.assert_server_set(), + Some(Property::RecurrenceId) => pointer.assert_server_set(), Some(Property::AccountId) => self .account_id .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), @@ -42477,6 +54832,11 @@ impl RegistryJsonPropertyPatch for TaskCalendarAlarmEmail { impl TaskCalendarAlarmNotification { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); + if let Some(value) = &self.target_account_id { + if !value.is_valid() { + errors.push(ValidationError::required(Property::TargetAccountId)); + } + } let value = &self.account_id; if !value.is_valid() { errors.push(ValidationError::required(Property::AccountId)); @@ -42491,29 +54851,90 @@ impl TaskCalendarAlarmNotification { } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { + i.foreign_key(ObjectType::Account, self.target_account_id, None); i.foreign_key(ObjectType::Account, self.account_id.into(), None); } } impl Pickle for TaskCalendarAlarmNotification { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.recurrence_id.is_some() { + bitmap |= 1 << 0; + } + if self.target_account_id.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 7, 2, bitmap); self.alarm_id.pickle(out); self.event_id.pickle(out); - self.recurrence_id.pickle(out); + if let Some(value) = &self.recurrence_id { + value.pickle(out); + } + if let Some(value) = &self.target_account_id { + value.pickle(out); + } self.account_id.pickle(out); self.document_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.alarm_id = Pickle::unpickle(stream)?; - this.event_id = Pickle::unpickle(stream)?; - this.recurrence_id = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.document_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: u64 = if block.has(0) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field2: Option = if block.has(2) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Id = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: Id = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: TaskStatus = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + alarm_id: field0, + event_id: field1, + recurrence_id: field2, + target_account_id: field3, + account_id: field4, + document_id: field5, + status: field6, + }) } } @@ -42523,6 +54944,7 @@ impl Default for TaskCalendarAlarmNotification { alarm_id: 0u64, event_id: 0u64, recurrence_id: Default::default(), + target_account_id: Default::default(), account_id: Default::default(), document_id: Default::default(), status: Default::default(), @@ -42532,13 +54954,24 @@ impl Default for TaskCalendarAlarmNotification { impl IntoValue for TaskCalendarAlarmNotification { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::AlarmId, self.alarm_id.into_value()); - map.insert_unchecked(Property::EventId, self.event_id.into_value()); - map.insert_unchecked(Property::RecurrenceId, self.recurrence_id.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::DocumentId, self.document_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(9); + map.insert_unchecked(Property::AlarmId, self.alarm_id.into_value_with(opts)); + map.insert_unchecked(Property::EventId, self.event_id.into_value_with(opts)); + map.insert_unchecked( + Property::RecurrenceId, + self.recurrence_id.into_value_with(opts), + ); + map.insert_unchecked( + Property::TargetAccountId, + self.target_account_id.into_value_with(opts), + ); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::DocumentId, self.document_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42553,6 +54986,7 @@ impl RegistryJsonPropertyPatch for TaskCalendarAlarmNotification { Some(Property::AlarmId) => pointer.assert_server_set(), Some(Property::EventId) => pointer.assert_server_set(), Some(Property::RecurrenceId) => pointer.assert_server_set(), + Some(Property::TargetAccountId) => pointer.assert_server_set(), Some(Property::AccountId) => self .account_id .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), @@ -42597,21 +55031,49 @@ impl TaskCalendarItipContents { impl Pickle for TaskCalendarItipContents { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.is_from_organizer { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 5, 1, bitmap); self.from.pickle(out); self.to.pickle(out); - self.is_from_organizer.pickle(out); self.i_calendar_data.pickle(out); self.summary.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.from = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - this.is_from_organizer = Pickle::unpickle(stream)?; - this.i_calendar_data = Pickle::unpickle(stream)?; - this.summary = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: bool = if block.has(2) { block.flag(0) } else { false }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + from: field0, + to: field1, + is_from_organizer: field2, + i_calendar_data: field3, + summary: field4, + }) } } @@ -42629,15 +55091,22 @@ impl Default for TaskCalendarItipContents { impl IntoValue for TaskCalendarItipContents { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); + map.insert_unchecked(Property::From, self.from.into_value_with(opts)); + map.insert_unchecked(Property::To, self.to.into_value_with(opts)); map.insert_unchecked( Property::IsFromOrganizer, - self.is_from_organizer.into_value(), + self.is_from_organizer.into_value_with(opts), ); - map.insert_unchecked(Property::ICalendarData, self.i_calendar_data.into_value()); - map.insert_unchecked(Property::Summary, self.summary.into_value()); + map.insert_unchecked( + Property::ICalendarData, + self.i_calendar_data.into_value_with(opts), + ); + map.insert_unchecked(Property::Summary, self.summary.into_value_with(opts)); JmapValue::Object(map) } } @@ -42690,19 +55159,43 @@ impl TaskCalendarItipMessage { impl Pickle for TaskCalendarItipMessage { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.messages.pickle(out); self.account_id.pickle(out); self.document_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.messages = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.document_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: List = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Id = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: TaskStatus = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + messages: field0, + account_id: field1, + document_id: field2, + status: field3, + }) } } @@ -42719,11 +55212,15 @@ impl Default for TaskCalendarItipMessage { impl IntoValue for TaskCalendarItipMessage { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Messages, self.messages.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::DocumentId, self.document_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::Messages, self.messages.into_value_with(opts)); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::DocumentId, self.document_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42768,6 +55265,12 @@ impl TaskDestroyAccount { if !value.is_valid() { errors.push(ValidationError::required(Property::AccountDomainId)); } + let value = &self.member_group_ids; + for value in value.iter() { + if !value.is_valid() { + errors.push(ValidationError::required(Property::MemberGroupIds)); + } + } let value = &self.status; value.validate(errors); errors.len() == neb @@ -42776,26 +55279,65 @@ impl TaskDestroyAccount { fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { i.foreign_key(ObjectType::Account, self.account_id.into(), None); i.foreign_key(ObjectType::Domain, self.account_domain_id.into(), None); + for id in self.member_group_ids.iter() { + i.foreign_key(ObjectType::Account, Some(*id), None); + } } } impl Pickle for TaskDestroyAccount { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 6, 0, 0); self.account_id.pickle(out); self.account_name.pickle(out); self.account_domain_id.pickle(out); self.account_type.pickle(out); + self.member_group_ids.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.account_id = Pickle::unpickle(stream)?; - this.account_name = Pickle::unpickle(stream)?; - this.account_domain_id = Pickle::unpickle(stream)?; - this.account_type = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Id = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: AccountType = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: TaskStatus = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + account_id: field0, + account_name: field1, + account_domain_id: field2, + account_type: field3, + member_group_ids: field4, + status: field5, + }) } } @@ -42806,6 +55348,7 @@ impl Default for TaskDestroyAccount { account_name: Default::default(), account_domain_id: Default::default(), account_type: Default::default(), + member_group_ids: Default::default(), status: Default::default(), } } @@ -42813,15 +55356,29 @@ impl Default for TaskDestroyAccount { impl IntoValue for TaskDestroyAccount { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::AccountName, self.account_name.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(8); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked( + Property::AccountName, + self.account_name.into_value_with(opts), + ); map.insert_unchecked( Property::AccountDomainId, - self.account_domain_id.into_value(), + self.account_domain_id.into_value_with(opts), ); - map.insert_unchecked(Property::AccountType, self.account_type.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked( + Property::AccountType, + self.account_type.into_value_with(opts), + ); + map.insert_unchecked( + Property::MemberGroupIds, + self.member_group_ids.into_value_with(opts), + ); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42837,6 +55394,7 @@ impl RegistryJsonPropertyPatch for TaskDestroyAccount { Some(Property::AccountName) => self.account_name.patch(pointer, value), Some(Property::AccountDomainId) => self.account_domain_id.patch(pointer, value), Some(Property::AccountType) => pointer.assert_server_set(), + Some(Property::MemberGroupIds) => pointer.assert_server_set(), Some(Property::Status) => self.status.patch(pointer, value), Some(Property::Due) => pointer.assert_server_set(), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -42867,15 +55425,29 @@ impl TaskDmarcReport { impl Pickle for TaskDmarcReport { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.report_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.report_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskStatus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + report_id: field0, + status: field1, + }) } } @@ -42890,9 +55462,13 @@ impl Default for TaskDmarcReport { impl IntoValue for TaskDmarcReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::ReportId, self.report_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::ReportId, self.report_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -42935,19 +55511,42 @@ impl TaskDnsManagement { impl Pickle for TaskDnsManagement { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.on_success_renew_certificate { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.update_records.pickle(out); - self.on_success_renew_certificate.pickle(out); self.domain_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.update_records = Pickle::unpickle(stream)?; - this.on_success_renew_certificate = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: Id = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: TaskStatus = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + update_records: field0, + on_success_renew_certificate: field1, + domain_id: field2, + status: field3, + }) } } @@ -42964,14 +55563,21 @@ impl Default for TaskDnsManagement { impl IntoValue for TaskDnsManagement { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::UpdateRecords, self.update_records.into_value()); + map.insert_unchecked( + Property::UpdateRecords, + self.update_records.into_value_with(opts), + ); map.insert_unchecked( Property::OnSuccessRenewCertificate, - self.on_success_renew_certificate.into_value(), + self.on_success_renew_certificate.into_value_with(opts), ); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -43018,15 +55624,29 @@ impl TaskDomainManagement { impl Pickle for TaskDomainManagement { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.domain_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.domain_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskStatus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + domain_id: field0, + status: field1, + }) } } @@ -43041,9 +55661,13 @@ impl Default for TaskDomainManagement { impl IntoValue for TaskDomainManagement { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -43067,162 +55691,6 @@ impl RegistryJsonPropertyPatch for TaskDomainManagement { } } -impl TaskIndexDocument { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); - } - let value = &self.document_id; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::DocumentId, value)); - } - let value = &self.status; - value.validate(errors); - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - } -} - -impl Pickle for TaskIndexDocument { - fn pickle(&self, out: &mut Vec) { - self.document_type.pickle(out); - self.account_id.pickle(out); - self.document_id.pickle(out); - self.status.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.document_type = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.document_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TaskIndexDocument { - fn default() -> Self { - Self { - document_type: Default::default(), - account_id: Default::default(), - document_id: Default::default(), - status: Default::default(), - } - } -} - -impl IntoValue for TaskIndexDocument { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::DocumentType, self.document_type.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::DocumentId, self.document_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TaskIndexDocument { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::DocumentType) => { - self.document_type.patch(pointer.assert_read_only()?, value) - } - Some(Property::AccountId) => self - .account_id - .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), - Some(Property::DocumentId) => { - self.document_id.patch(pointer.assert_read_only()?, value) - } - Some(Property::Status) => self.status.patch(pointer, value), - Some(Property::Due) => pointer.assert_server_set(), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TaskIndexTrace { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.trace_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::TraceId)); - } - let value = &self.status; - value.validate(errors); - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Trace, self.trace_id.into(), None); - } -} - -impl Pickle for TaskIndexTrace { - fn pickle(&self, out: &mut Vec) { - self.trace_id.pickle(out); - self.status.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.trace_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TaskIndexTrace { - fn default() -> Self { - Self { - trace_id: Default::default(), - status: Default::default(), - } - } -} - -impl IntoValue for TaskIndexTrace { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::TraceId, self.trace_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TaskIndexTrace { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::TraceId) => self.trace_id.patch(pointer.assert_read_only()?, value), - Some(Property::Status) => self.status.patch(pointer, value), - Some(Property::Due) => pointer.assert_server_set(), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for TaskManager { const FLAGS: u64 = OBJ_SINGLETON; const VERSION: u8 = 0; @@ -43230,12 +55698,12 @@ impl ObjectImpl for TaskManager { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.max_attempts; - if *value < 1 { - errors.push(ValidationError::min_value(Property::MaxAttempts, 1)); - } let value = &self.strategy; value.validate(errors); + let value = &self.alert_after; + if *value < 1 { + errors.push(ValidationError::min_value(Property::AlertAfter, 1)); + } errors.len() == neb } @@ -43244,36 +55712,53 @@ impl ObjectImpl for TaskManager { impl Pickle for TaskManager { fn pickle(&self, out: &mut Vec) { - self.max_attempts.pickle(out); + let block = crate::pickle::begin_block(out, 2, 0, 0); self.strategy.pickle(out); - self.total_deadline.pickle(out); + self.alert_after.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.max_attempts = Pickle::unpickle(stream)?; - this.strategy = Pickle::unpickle(stream)?; - this.total_deadline = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: TaskRetryStrategy = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: u64 = if block.has(1) { + Pickle::unpickle(stream)? + } else { + 5u64 + }; + stream.close_block(block); + Some(Self { + strategy: field0, + alert_after: field1, + }) } } impl Default for TaskManager { fn default() -> Self { Self { - max_attempts: 3u64, strategy: Default::default(), - total_deadline: Duration::from_millis(21600000), + alert_after: 5u64, } } } impl IntoValue for TaskManager { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::MaxAttempts, self.max_attempts.into_value()); - map.insert_unchecked(Property::Strategy, self.strategy.into_value()); - map.insert_unchecked(Property::TotalDeadline, self.total_deadline.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(4); + map.insert_unchecked(Property::Strategy, self.strategy.into_value_with(opts)); + map.insert_unchecked(Property::AlertAfter, self.alert_after.into_value_with(opts)); JmapValue::Object(map) } } @@ -43285,9 +55770,8 @@ impl RegistryJsonPropertyPatch for TaskManager { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::MaxAttempts) => self.max_attempts.patch(pointer, value), Some(Property::Strategy) => self.strategy.patch(pointer, value), - Some(Property::TotalDeadline) => self.total_deadline.patch(pointer, value), + Some(Property::AlertAfter) => self.alert_after.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -43326,19 +55810,43 @@ impl TaskMergeThreads { impl Pickle for TaskMergeThreads { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.account_id.pickle(out); self.thread_name.pickle(out); self.message_ids.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.account_id = Pickle::unpickle(stream)?; - this.thread_name = Pickle::unpickle(stream)?; - this.message_ids = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: TaskStatus = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + account_id: field0, + thread_name: field1, + message_ids: field2, + status: field3, + }) } } @@ -43355,11 +55863,15 @@ impl Default for TaskMergeThreads { impl IntoValue for TaskMergeThreads { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::ThreadName, self.thread_name.into_value()); - map.insert_unchecked(Property::MessageIds, self.message_ids.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::ThreadName, self.thread_name.into_value_with(opts)); + map.insert_unchecked(Property::MessageIds, self.message_ids.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -43385,98 +55897,77 @@ impl RegistryJsonPropertyPatch for TaskMergeThreads { } } -impl TaskRestoreArchivedItem { +impl TaskReportWindow { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.blob_id; - if value.is_empty() { - errors.push(ValidationError::required(Property::BlobId)); - } - let value = &self.created_at; + let value = &self.window_end; if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - let value = &self.archived_until; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::ArchivedUntil, value)); - } - let value = &self.account_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::AccountId)); + errors.push(ValidationError::invalid(Property::WindowEnd, value)); } let value = &self.status; value.validate(errors); errors.len() == neb } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Account, self.account_id.into(), None); - } } -impl Pickle for TaskRestoreArchivedItem { +impl Pickle for TaskReportWindow { fn pickle(&self, out: &mut Vec) { - self.blob_id.pickle(out); - self.archived_item_type.pickle(out); - self.created_at.pickle(out); - self.archived_until.pickle(out); - self.account_id.pickle(out); + let block = crate::pickle::begin_block(out, 2, 0, 0); + self.window_end.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.blob_id = Pickle::unpickle(stream)?; - this.archived_item_type = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.archived_until = Pickle::unpickle(stream)?; - this.account_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: UTCDateTime = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskStatus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + window_end: field0, + status: field1, + }) } } -impl Default for TaskRestoreArchivedItem { +impl Default for TaskReportWindow { fn default() -> Self { Self { - blob_id: Default::default(), - archived_item_type: Default::default(), - created_at: Default::default(), - archived_until: Default::default(), - account_id: Default::default(), + window_end: Default::default(), status: Default::default(), } } } -impl IntoValue for TaskRestoreArchivedItem { +impl IntoValue for TaskReportWindow { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::BlobId, self.blob_id.into_value()); - map.insert_unchecked( - Property::ArchivedItemType, - self.archived_item_type.into_value(), - ); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::ArchivedUntil, self.archived_until.into_value()); - map.insert_unchecked(Property::AccountId, self.account_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(4); + map.insert_unchecked(Property::WindowEnd, self.window_end.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } -impl RegistryJsonPropertyPatch for TaskRestoreArchivedItem { +impl RegistryJsonPropertyPatch for TaskReportWindow { fn patch_property<'x>( &mut self, mut pointer: JsonPointerPatch<'_>, value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::BlobId) => pointer.assert_server_set(), - Some(Property::ArchivedItemType) => pointer.assert_server_set(), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::ArchivedUntil) => pointer.assert_server_set(), - Some(Property::AccountId) => pointer.assert_server_set(), + Some(Property::WindowEnd) => pointer.assert_server_set(), Some(Property::Status) => self.status.patch(pointer, value), Some(Property::Due) => pointer.assert_server_set(), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -43528,16 +56019,20 @@ impl Pickle for TaskRetryStrategy { impl IntoValue for TaskRetryStrategy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { TaskRetryStrategy::ExponentialBackoff(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("ExponentialBackoff".into())); obj } TaskRetryStrategy::FixedDelay(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("FixedDelay".into())); @@ -43592,19 +56087,42 @@ impl TaskRetryStrategyBackoff { impl Pickle for TaskRetryStrategyBackoff { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.jitter { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 4, 1, bitmap); self.factor.pickle(out); self.initial_delay.pickle(out); self.max_delay.pickle(out); - self.jitter.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.factor = Pickle::unpickle(stream)?; - this.initial_delay = Pickle::unpickle(stream)?; - this.max_delay = Pickle::unpickle(stream)?; - this.jitter = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Float = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Float::new(2.0f64) + }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(60000) + }; + let field2: Duration = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(21600000) + }; + let field3: bool = if block.has(3) { block.flag(0) } else { true }; + stream.close_block(block); + Some(Self { + factor: field0, + initial_delay: field1, + max_delay: field2, + jitter: field3, + }) } } @@ -43613,7 +56131,7 @@ impl Default for TaskRetryStrategyBackoff { Self { factor: Float::new(2.0f64), initial_delay: Duration::from_millis(60000), - max_delay: Duration::from_millis(1800000), + max_delay: Duration::from_millis(21600000), jitter: true, } } @@ -43621,11 +56139,18 @@ impl Default for TaskRetryStrategyBackoff { impl IntoValue for TaskRetryStrategyBackoff { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::Factor, self.factor.into_value()); - map.insert_unchecked(Property::InitialDelay, self.initial_delay.into_value()); - map.insert_unchecked(Property::MaxDelay, self.max_delay.into_value()); - map.insert_unchecked(Property::Jitter, self.jitter.into_value()); + map.insert_unchecked(Property::Factor, self.factor.into_value_with(opts)); + map.insert_unchecked( + Property::InitialDelay, + self.initial_delay.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxDelay, self.max_delay.into_value_with(opts)); + map.insert_unchecked(Property::Jitter, self.jitter.into_value_with(opts)); JmapValue::Object(map) } } @@ -43658,13 +56183,20 @@ impl TaskRetryStrategyFixed { impl Pickle for TaskRetryStrategyFixed { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.delay.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.delay = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Duration = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + stream.close_block(block); + Some(Self { delay: field0 }) } } @@ -43678,8 +56210,12 @@ impl Default for TaskRetryStrategyFixed { impl IntoValue for TaskRetryStrategyFixed { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Delay, self.delay.into_value()); + map.insert_unchecked(Property::Delay, self.delay.into_value_with(opts)); JmapValue::Object(map) } } @@ -43701,6 +56237,120 @@ impl RegistryJsonPropertyPatch for TaskRetryStrategyFixed { } } +impl TaskShareRevokeCleanup { + fn validate(&self, errors: &mut Vec) -> bool { + let neb = errors.len(); + let value = &self.account_id; + if !value.is_valid() { + errors.push(ValidationError::required(Property::AccountId)); + } + let value = &self.document_id; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::DocumentId, value)); + } + let value = &self.status; + value.validate(errors); + errors.len() == neb + } + + fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { + i.foreign_key(ObjectType::Account, self.account_id.into(), None); + } +} + +impl Pickle for TaskShareRevokeCleanup { + fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); + self.object_type.pickle(out); + self.account_id.pickle(out); + self.document_id.pickle(out); + self.status.pickle(out); + crate::pickle::end_block(out, block); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + let block = stream.open_block()?; + let field0: SharedObjectType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Id = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: TaskStatus = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + object_type: field0, + account_id: field1, + document_id: field2, + status: field3, + }) + } +} + +impl Default for TaskShareRevokeCleanup { + fn default() -> Self { + Self { + object_type: Default::default(), + account_id: Default::default(), + document_id: Default::default(), + status: Default::default(), + } + } +} + +impl IntoValue for TaskShareRevokeCleanup { + fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(6); + map.insert_unchecked(Property::ObjectType, self.object_type.into_value_with(opts)); + map.insert_unchecked(Property::AccountId, self.account_id.into_value_with(opts)); + map.insert_unchecked(Property::DocumentId, self.document_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); + JmapValue::Object(map) + } +} + +impl RegistryJsonPropertyPatch for TaskShareRevokeCleanup { + fn patch_property<'x>( + &mut self, + mut pointer: JsonPointerPatch<'_>, + value: JmapValue<'x>, + ) -> PatchResult<'x> { + match pointer.next_property() { + Some(Property::ObjectType) => pointer.assert_server_set(), + Some(Property::AccountId) => self + .account_id + .patch(pointer.assert_read_only()?.assert_can_set_account()?, value), + Some(Property::DocumentId) => { + self.document_id.patch(pointer.assert_read_only()?, value) + } + Some(Property::Status) => self.status.patch(pointer, value), + Some(Property::Due) => pointer.assert_server_set(), + Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { + property: Property::Type, + value, + }), + _ => Err(PatchError::new(pointer, "Invalid property")), + } + } +} + impl TaskSpamFilterMaintenance { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -43712,15 +56362,29 @@ impl TaskSpamFilterMaintenance { impl Pickle for TaskSpamFilterMaintenance { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.maintenance_type.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.maintenance_type = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: TaskSpamFilterMaintenanceType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskStatus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + maintenance_type: field0, + status: field1, + }) } } @@ -43735,12 +56399,16 @@ impl Default for TaskSpamFilterMaintenance { impl IntoValue for TaskSpamFilterMaintenance { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); map.insert_unchecked( Property::MaintenanceType, - self.maintenance_type.into_value(), + self.maintenance_type.into_value_with(opts), ); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -43772,6 +56440,7 @@ impl TaskStatus { TaskStatus::Pending(inner) => inner.validate(errors), TaskStatus::Retry(inner) => inner.validate(errors), TaskStatus::Failed(inner) => inner.validate(errors), + TaskStatus::Stalled(inner) => inner.validate(errors), } } } @@ -43797,6 +56466,10 @@ impl Pickle for TaskStatus { 2u16.pickle(out); inner.pickle(out); } + TaskStatus::Stalled(inner) => { + 3u16.pickle(out); + inner.pickle(out); + } } } @@ -43805,6 +56478,7 @@ impl Pickle for TaskStatus { 0 => Pickle::unpickle(stream).map(TaskStatus::Pending), 1 => Pickle::unpickle(stream).map(TaskStatus::Retry), 2 => Pickle::unpickle(stream).map(TaskStatus::Failed), + 3 => Pickle::unpickle(stream).map(TaskStatus::Stalled), _ => None, } } @@ -43812,28 +56486,39 @@ impl Pickle for TaskStatus { impl IntoValue for TaskStatus { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { TaskStatus::Pending(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Pending".into())); obj } TaskStatus::Retry(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Retry".into())); obj } TaskStatus::Failed(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Failed".into())); obj } + TaskStatus::Stalled(obj) => { + let mut obj = obj.into_value_with(opts); + obj.as_object_mut() + .unwrap() + .insert_unchecked(Property::Type, JmapValue::Str("Stalled".into())); + obj + } } } } @@ -43849,12 +56534,14 @@ impl RegistryJsonPatch for TaskStatus { TaskStatusType::Pending => *self = TaskStatus::Pending(Default::default()), TaskStatusType::Retry => *self = TaskStatus::Retry(Default::default()), TaskStatusType::Failed => *self = TaskStatus::Failed(Default::default()), + TaskStatusType::Stalled => *self = TaskStatus::Stalled(Default::default()), } } match self { TaskStatus::Pending(inner) => inner.patch(pointer, value), TaskStatus::Retry(inner) => inner.patch(pointer, value), TaskStatus::Failed(inner) => inner.patch(pointer, value), + TaskStatus::Stalled(inner) => inner.patch(pointer, value), } } } @@ -43865,6 +56552,7 @@ impl TaskStatus { TaskStatus::Pending(_) => TaskStatusType::Pending, TaskStatus::Retry(_) => TaskStatusType::Retry, TaskStatus::Failed(_) => TaskStatusType::Failed, + TaskStatus::Stalled(_) => TaskStatusType::Stalled, } } } @@ -43890,19 +56578,43 @@ impl TaskStatusFailed { impl Pickle for TaskStatusFailed { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 4, 0, 0); self.created_at.pickle(out); self.failed_at.pickle(out); self.failed_attempt_number.pickle(out); self.failure_reason.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.created_at = Pickle::unpickle(stream)?; - this.failed_at = Pickle::unpickle(stream)?; - this.failed_attempt_number = Pickle::unpickle(stream)?; - this.failure_reason = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: UTCDateTime = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: UTCDateTime = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field3: String = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + created_at: field0, + failed_at: field1, + failed_attempt_number: field2, + failure_reason: field3, + }) } } @@ -43919,14 +56631,21 @@ impl Default for TaskStatusFailed { impl IntoValue for TaskStatusFailed { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::FailedAt, self.failed_at.into_value()); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::FailedAt, self.failed_at.into_value_with(opts)); map.insert_unchecked( Property::FailedAttemptNumber, - self.failed_attempt_number.into_value(), + self.failed_attempt_number.into_value_with(opts), + ); + map.insert_unchecked( + Property::FailureReason, + self.failure_reason.into_value_with(opts), ); - map.insert_unchecked(Property::FailureReason, self.failure_reason.into_value()); JmapValue::Object(map) } } @@ -43968,15 +56687,29 @@ impl TaskStatusPending { impl Pickle for TaskStatusPending { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.created_at.pickle(out); self.due.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.created_at = Pickle::unpickle(stream)?; - this.due = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: UTCDateTime = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: UTCDateTime = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + created_at: field0, + due: field1, + }) } } @@ -43991,9 +56724,13 @@ impl Default for TaskStatusPending { impl IntoValue for TaskStatusPending { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::Due, self.due.into_value()); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::Due, self.due.into_value_with(opts)); JmapValue::Object(map) } } @@ -44027,6 +56764,10 @@ impl TaskStatusRetry { if !value.is_valid() { errors.push(ValidationError::invalid(Property::Due, value)); } + let value = &self.first_failed_at; + if !value.is_valid() { + errors.push(ValidationError::invalid(Property::FirstFailedAt, value)); + } let value = &self.failure_reason; if value.is_empty() { errors.push(ValidationError::required(Property::FailureReason)); @@ -44037,19 +56778,50 @@ impl TaskStatusRetry { impl Pickle for TaskStatusRetry { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 5, 0, 0); self.created_at.pickle(out); self.due.pickle(out); self.attempt_number.pickle(out); + self.first_failed_at.pickle(out); self.failure_reason.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.created_at = Pickle::unpickle(stream)?; - this.due = Pickle::unpickle(stream)?; - this.attempt_number = Pickle::unpickle(stream)?; - this.failure_reason = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: UTCDateTime = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: UTCDateTime = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 1u64 + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + created_at: field0, + due: field1, + attempt_number: field2, + first_failed_at: field3, + failure_reason: field4, + }) } } @@ -44059,6 +56831,7 @@ impl Default for TaskStatusRetry { created_at: Default::default(), due: Default::default(), attempt_number: 1u64, + first_failed_at: Default::default(), failure_reason: Default::default(), } } @@ -44066,11 +56839,25 @@ impl Default for TaskStatusRetry { impl IntoValue for TaskStatusRetry { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(6); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::Due, self.due.into_value()); - map.insert_unchecked(Property::AttemptNumber, self.attempt_number.into_value()); - map.insert_unchecked(Property::FailureReason, self.failure_reason.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(7); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::Due, self.due.into_value_with(opts)); + map.insert_unchecked( + Property::AttemptNumber, + self.attempt_number.into_value_with(opts), + ); + map.insert_unchecked( + Property::FirstFailedAt, + self.first_failed_at.into_value_with(opts), + ); + map.insert_unchecked( + Property::FailureReason, + self.failure_reason.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -44085,6 +56872,7 @@ impl RegistryJsonPropertyPatch for TaskStatusRetry { Some(Property::CreatedAt) => pointer.assert_server_set(), Some(Property::Due) => self.due.patch(pointer, value), Some(Property::AttemptNumber) => self.attempt_number.patch(pointer, value), + Some(Property::FirstFailedAt) => pointer.assert_server_set(), Some(Property::FailureReason) => self.failure_reason.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -44106,17 +56894,46 @@ impl TaskStoreMaintenance { impl Pickle for TaskStoreMaintenance { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.shard_index.is_some() { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 3, 1, bitmap); self.maintenance_type.pickle(out); - self.shard_index.pickle(out); + if let Some(value) = &self.shard_index { + value.pickle(out); + } self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.maintenance_type = Pickle::unpickle(stream)?; - this.shard_index = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: TaskStoreMaintenanceType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: TaskStatus = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + maintenance_type: field0, + shard_index: field1, + status: field2, + }) } } @@ -44132,13 +56949,17 @@ impl Default for TaskStoreMaintenance { impl IntoValue for TaskStoreMaintenance { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(5); map.insert_unchecked( Property::MaintenanceType, - self.maintenance_type.into_value(), + self.maintenance_type.into_value_with(opts), ); - map.insert_unchecked(Property::ShardIndex, self.shard_index.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::ShardIndex, self.shard_index.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -44165,84 +56986,6 @@ impl RegistryJsonPropertyPatch for TaskStoreMaintenance { } } -impl TaskTenantMaintenance { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.tenant_id; - if !value.is_valid() { - errors.push(ValidationError::required(Property::TenantId)); - } - let value = &self.status; - value.validate(errors); - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.tenant_id.into(), None); - } -} - -impl Pickle for TaskTenantMaintenance { - fn pickle(&self, out: &mut Vec) { - self.tenant_id.pickle(out); - self.maintenance_type.pickle(out); - self.status.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.tenant_id = Pickle::unpickle(stream)?; - this.maintenance_type = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TaskTenantMaintenance { - fn default() -> Self { - Self { - tenant_id: Default::default(), - maintenance_type: Default::default(), - status: Default::default(), - } - } -} - -impl IntoValue for TaskTenantMaintenance { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::TenantId, self.tenant_id.into_value()); - map.insert_unchecked( - Property::MaintenanceType, - self.maintenance_type.into_value(), - ); - map.insert_unchecked(Property::Status, self.status.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TaskTenantMaintenance { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::TenantId) => self.tenant_id.patch(pointer.assert_read_only()?, value), - Some(Property::MaintenanceType) => self - .maintenance_type - .patch(pointer.assert_read_only()?, value), - Some(Property::Status) => self.status.patch(pointer, value), - Some(Property::Due) => pointer.assert_server_set(), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl TaskTlsReport { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -44262,15 +57005,29 @@ impl TaskTlsReport { impl Pickle for TaskTlsReport { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 2, 0, 0); self.report_id.pickle(out); self.status.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.report_id = Pickle::unpickle(stream)?; - this.status = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Id = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: TaskStatus = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + report_id: field0, + status: field1, + }) } } @@ -44285,9 +57042,13 @@ impl Default for TaskTlsReport { impl IntoValue for TaskTlsReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::ReportId, self.report_id.into_value()); - map.insert_unchecked(Property::Status, self.status.into_value()); + map.insert_unchecked(Property::ReportId, self.report_id.into_value_with(opts)); + map.insert_unchecked(Property::Status, self.status.into_value_with(opts)); JmapValue::Object(map) } } @@ -44311,112 +57072,8 @@ impl RegistryJsonPropertyPatch for TaskTlsReport { } } -impl ObjectImpl for Tenant { - const FLAGS: u64 = OBJ_SEQ_ID; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::Tenant; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.name; - if value.is_empty() { - errors.push(ValidationError::required(Property::Name)); - } - let value = &self.created_at; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::CreatedAt, value)); - } - if let Some(value) = &self.logo { - if value.is_empty() { - errors.push(ValidationError::required(Property::Logo)); - } - } - let value = &self.roles; - value.validate(errors); - let value = &self.permissions; - value.validate(errors); - errors.len() == neb - } - - fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.text(Property::Text, &self.name); - self.roles.index(i); - } -} - -impl Pickle for Tenant { - fn pickle(&self, out: &mut Vec) { - self.name.pickle(out); - self.created_at.pickle(out); - self.logo.pickle(out); - self.roles.pickle(out); - self.permissions.pickle(out); - self.quotas.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.logo = Pickle::unpickle(stream)?; - this.roles = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.quotas = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for Tenant { - fn default() -> Self { - Self { - name: Default::default(), - created_at: Default::default(), - logo: Default::default(), - roles: Default::default(), - permissions: Default::default(), - quotas: Default::default(), - } - } -} - -impl IntoValue for Tenant { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(8); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::Logo, self.logo.into_value()); - map.insert_unchecked(Property::Roles, self.roles.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::Quotas, self.quotas.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for Tenant { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Name) => self.name.patch(pointer, value), - Some(Property::CreatedAt) => pointer.assert_server_set(), - Some(Property::Logo) => self.logo.patch(pointer, value), - Some(Property::Roles) => self.roles.patch(pointer, value), - Some(Property::Permissions) => self.permissions.patch(pointer, value), - Some(Property::Quotas) => self.quotas.patch(pointer, value), - Some(Property::UsedDiskQuota) => pointer.assert_server_set(), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - impl ObjectImpl for TlsExternalReport { - const FLAGS: u64 = OBJ_FILTER_TENANT; + const FLAGS: u64 = OBJ_FILTER_TENANT | OBJ_NO_CREATE | OBJ_NO_UPDATE; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::TlsExternalReport; @@ -44446,40 +57103,72 @@ impl ObjectImpl for TlsExternalReport { if !value.is_valid() { errors.push(ValidationError::invalid(Property::ExpiresAt, value)); } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } errors.len() == neb } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); } } impl Pickle for TlsExternalReport { fn pickle(&self, out: &mut Vec) { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] + let mut bitmap = 0u64; + let block = crate::pickle::begin_block(out, 7, 1, bitmap); self.report.pickle(out); self.from.pickle(out); self.subject.pickle(out); self.to.pickle(out); self.received_at.pickle(out); self.expires_at.pickle(out); - self.member_tenant_id.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.report = Pickle::unpickle(stream)?; - this.from = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - this.to = Pickle::unpickle(stream)?; - this.received_at = Pickle::unpickle(stream)?; - this.expires_at = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: TlsReport = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + report: field0, + from: field1, + subject: field2, + to: field3, + received_at: field4, + expires_at: field5, + }) } } @@ -44492,21 +57181,23 @@ impl Default for TlsExternalReport { to: Default::default(), received_at: Default::default(), expires_at: Default::default(), - member_tenant_id: Default::default(), } } } impl IntoValue for TlsExternalReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::From, self.from.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); - map.insert_unchecked(Property::To, self.to.into_value()); - map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value()); - map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::From, self.from.into_value_with(opts)); + map.insert_unchecked(Property::Subject, self.subject.into_value_with(opts)); + map.insert_unchecked(Property::To, self.to.into_value_with(opts)); + map.insert_unchecked(Property::ReceivedAt, self.received_at.into_value_with(opts)); + map.insert_unchecked(Property::ExpiresAt, self.expires_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -44528,9 +57219,6 @@ impl RegistryJsonPropertyPatch for TlsExternalReport { .patch(pointer.with_validators(&[StringValidator::Email]), value), Some(Property::ReceivedAt) => self.received_at.patch(pointer, value), Some(Property::ExpiresAt) => self.expires_at.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -44579,27 +57267,126 @@ impl TlsFailureDetails { impl Pickle for TlsFailureDetails { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.sending_mta_ip.is_some() { + bitmap |= 1 << 0; + } + if self.receiving_mx_hostname.is_some() { + bitmap |= 1 << 1; + } + if self.receiving_mx_helo.is_some() { + bitmap |= 1 << 2; + } + if self.receiving_ip.is_some() { + bitmap |= 1 << 3; + } + if self.additional_information.is_some() { + bitmap |= 1 << 4; + } + if self.failure_reason_code.is_some() { + bitmap |= 1 << 5; + } + let block = crate::pickle::begin_block(out, 8, 6, bitmap); self.result_type.pickle(out); - self.sending_mta_ip.pickle(out); - self.receiving_mx_hostname.pickle(out); - self.receiving_mx_helo.pickle(out); - self.receiving_ip.pickle(out); + if let Some(value) = &self.sending_mta_ip { + value.pickle(out); + } + if let Some(value) = &self.receiving_mx_hostname { + value.pickle(out); + } + if let Some(value) = &self.receiving_mx_helo { + value.pickle(out); + } + if let Some(value) = &self.receiving_ip { + value.pickle(out); + } self.failed_session_count.pickle(out); - self.additional_information.pickle(out); - self.failure_reason_code.pickle(out); + if let Some(value) = &self.additional_information { + value.pickle(out); + } + if let Some(value) = &self.failure_reason_code { + value.pickle(out); + } + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.result_type = Pickle::unpickle(stream)?; - this.sending_mta_ip = Pickle::unpickle(stream)?; - this.receiving_mx_hostname = Pickle::unpickle(stream)?; - this.receiving_mx_helo = Pickle::unpickle(stream)?; - this.receiving_ip = Pickle::unpickle(stream)?; - this.failed_session_count = Pickle::unpickle(stream)?; - this.additional_information = Pickle::unpickle(stream)?; - this.failure_reason_code = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: TlsResultType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: Option = if block.has(2) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field3: Option = if block.has(3) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field4: Option = if block.has(4) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field6: Option = if block.has(6) { + if block.flag(4) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field7: Option = if block.has(7) { + if block.flag(5) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + result_type: field0, + sending_mta_ip: field1, + receiving_mx_hostname: field2, + receiving_mx_helo: field3, + receiving_ip: field4, + failed_session_count: field5, + additional_information: field6, + failure_reason_code: field7, + }) } } @@ -44620,29 +57407,39 @@ impl Default for TlsFailureDetails { impl IntoValue for TlsFailureDetails { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::ResultType, self.result_type.into_value()); - map.insert_unchecked(Property::SendingMtaIp, self.sending_mta_ip.into_value()); + map.insert_unchecked(Property::ResultType, self.result_type.into_value_with(opts)); + map.insert_unchecked( + Property::SendingMtaIp, + self.sending_mta_ip.into_value_with(opts), + ); map.insert_unchecked( Property::ReceivingMxHostname, - self.receiving_mx_hostname.into_value(), + self.receiving_mx_hostname.into_value_with(opts), ); map.insert_unchecked( Property::ReceivingMxHelo, - self.receiving_mx_helo.into_value(), + self.receiving_mx_helo.into_value_with(opts), + ); + map.insert_unchecked( + Property::ReceivingIp, + self.receiving_ip.into_value_with(opts), ); - map.insert_unchecked(Property::ReceivingIp, self.receiving_ip.into_value()); map.insert_unchecked( Property::FailedSessionCount, - self.failed_session_count.into_value(), + self.failed_session_count.into_value_with(opts), ); map.insert_unchecked( Property::AdditionalInformation, - self.additional_information.into_value(), + self.additional_information.into_value_with(opts), ); map.insert_unchecked( Property::FailureReasonCode, - self.failure_reason_code.into_value(), + self.failure_reason_code.into_value_with(opts), ); JmapValue::Object(map) } @@ -44675,7 +57472,7 @@ impl RegistryJsonPropertyPatch for TlsFailureDetails { } impl ObjectImpl for TlsInternalReport { - const FLAGS: u64 = 0; + const FLAGS: u64 = OBJ_NO_CREATE; const VERSION: u8 = 0; const OBJECT: ObjectType = ObjectType::TlsInternalReport; @@ -44715,6 +57512,7 @@ impl ObjectImpl for TlsInternalReport { impl Pickle for TlsInternalReport { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 7, 0, 0); self.policy_identifiers.pickle(out); self.mail_rua.pickle(out); self.http_rua.pickle(out); @@ -44722,18 +57520,59 @@ impl Pickle for TlsInternalReport { self.domain.pickle(out); self.created_at.pickle(out); self.deliver_at.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.policy_identifiers = Pickle::unpickle(stream)?; - this.mail_rua = Pickle::unpickle(stream)?; - this.http_rua = Pickle::unpickle(stream)?; - this.report = Pickle::unpickle(stream)?; - this.domain = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.deliver_at = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Map = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: Map = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: TlsReport = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: String = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: UTCDateTime = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: UTCDateTime = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + policy_identifiers: field0, + mail_rua: field1, + http_rua: field2, + report: field3, + domain: field4, + created_at: field5, + deliver_at: field6, + }) } } @@ -44753,17 +57592,21 @@ impl Default for TlsInternalReport { impl IntoValue for TlsInternalReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); map.insert_unchecked( Property::PolicyIdentifiers, - self.policy_identifiers.into_value(), + self.policy_identifiers.into_value_with(opts), ); - map.insert_unchecked(Property::MailRua, self.mail_rua.into_value()); - map.insert_unchecked(Property::HttpRua, self.http_rua.into_value()); - map.insert_unchecked(Property::Report, self.report.into_value()); - map.insert_unchecked(Property::Domain, self.domain.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::DeliverAt, self.deliver_at.into_value()); + map.insert_unchecked(Property::MailRua, self.mail_rua.into_value_with(opts)); + map.insert_unchecked(Property::HttpRua, self.http_rua.into_value_with(opts)); + map.insert_unchecked(Property::Report, self.report.into_value_with(opts)); + map.insert_unchecked(Property::Domain, self.domain.into_value_with(opts)); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked(Property::DeliverAt, self.deliver_at.into_value_with(opts)); JmapValue::Object(map) } } @@ -44830,23 +57673,76 @@ impl TlsReport { impl Pickle for TlsReport { fn pickle(&self, out: &mut Vec) { - self.organization_name.pickle(out); - self.contact_info.pickle(out); + let mut bitmap = 0u64; + if self.organization_name.is_some() { + bitmap |= 1 << 0; + } + if self.contact_info.is_some() { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 6, 2, bitmap); + if let Some(value) = &self.organization_name { + value.pickle(out); + } + if let Some(value) = &self.contact_info { + value.pickle(out); + } self.report_id.pickle(out); self.date_range_start.pickle(out); self.date_range_end.pickle(out); self.policies.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.organization_name = Pickle::unpickle(stream)?; - this.contact_info = Pickle::unpickle(stream)?; - this.report_id = Pickle::unpickle(stream)?; - this.date_range_start = Pickle::unpickle(stream)?; - this.date_range_end = Pickle::unpickle(stream)?; - this.policies = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: Option = if block.has(1) { + if block.flag(1) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: UTCDateTime = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field5: List = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + organization_name: field0, + contact_info: field1, + report_id: field2, + date_range_start: field3, + date_range_end: field4, + policies: field5, + }) } } @@ -44865,16 +57761,29 @@ impl Default for TlsReport { impl IntoValue for TlsReport { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(8); map.insert_unchecked( Property::OrganizationName, - self.organization_name.into_value(), + self.organization_name.into_value_with(opts), ); - map.insert_unchecked(Property::ContactInfo, self.contact_info.into_value()); - map.insert_unchecked(Property::ReportId, self.report_id.into_value()); - map.insert_unchecked(Property::DateRangeStart, self.date_range_start.into_value()); - map.insert_unchecked(Property::DateRangeEnd, self.date_range_end.into_value()); - map.insert_unchecked(Property::Policies, self.policies.into_value()); + map.insert_unchecked( + Property::ContactInfo, + self.contact_info.into_value_with(opts), + ); + map.insert_unchecked(Property::ReportId, self.report_id.into_value_with(opts)); + map.insert_unchecked( + Property::DateRangeStart, + self.date_range_start.into_value_with(opts), + ); + map.insert_unchecked( + Property::DateRangeEnd, + self.date_range_end.into_value_with(opts), + ); + map.insert_unchecked(Property::Policies, self.policies.into_value_with(opts)); JmapValue::Object(map) } } @@ -44930,6 +57839,7 @@ impl TlsReportPolicy { impl Pickle for TlsReportPolicy { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 7, 0, 0); self.policy_type.pickle(out); self.policy_strings.pickle(out); self.policy_domain.pickle(out); @@ -44937,18 +57847,56 @@ impl Pickle for TlsReportPolicy { self.total_successful_sessions.pickle(out); self.total_failed_sessions.pickle(out); self.failure_details.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.policy_type = Pickle::unpickle(stream)?; - this.policy_strings = Pickle::unpickle(stream)?; - this.policy_domain = Pickle::unpickle(stream)?; - this.mx_hosts = Pickle::unpickle(stream)?; - this.total_successful_sessions = Pickle::unpickle(stream)?; - this.total_failed_sessions = Pickle::unpickle(stream)?; - this.failure_details = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: TlsPolicyType = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Map = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: String = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 0u64 + }; + let field6: List = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + policy_type: field0, + policy_strings: field1, + policy_domain: field2, + mx_hosts: field3, + total_successful_sessions: field4, + total_failed_sessions: field5, + failure_details: field6, + }) } } @@ -44968,20 +57916,33 @@ impl Default for TlsReportPolicy { impl IntoValue for TlsReportPolicy { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(9); - map.insert_unchecked(Property::PolicyType, self.policy_type.into_value()); - map.insert_unchecked(Property::PolicyStrings, self.policy_strings.into_value()); - map.insert_unchecked(Property::PolicyDomain, self.policy_domain.into_value()); - map.insert_unchecked(Property::MxHosts, self.mx_hosts.into_value()); + map.insert_unchecked(Property::PolicyType, self.policy_type.into_value_with(opts)); + map.insert_unchecked( + Property::PolicyStrings, + self.policy_strings.into_value_with(opts), + ); + map.insert_unchecked( + Property::PolicyDomain, + self.policy_domain.into_value_with(opts), + ); + map.insert_unchecked(Property::MxHosts, self.mx_hosts.into_value_with(opts)); map.insert_unchecked( Property::TotalSuccessfulSessions, - self.total_successful_sessions.into_value(), + self.total_successful_sessions.into_value_with(opts), ); map.insert_unchecked( Property::TotalFailedSessions, - self.total_failed_sessions.into_value(), + self.total_failed_sessions.into_value_with(opts), + ); + map.insert_unchecked( + Property::FailureDetails, + self.failure_details.into_value_with(opts), ); - map.insert_unchecked(Property::FailureDetails, self.failure_details.into_value()); JmapValue::Object(map) } } @@ -45034,8 +57995,6 @@ impl ObjectImpl for TlsReportSettings { value.validate(errors); let value = &self.dkim_sign_domain; value.validate(errors); - let value = &self.subject; - value.validate(errors); errors.len() == neb } @@ -45051,7 +58010,7 @@ impl TlsReportSettings { ..Default::default() }), property: Property::ContactInfo, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } } @@ -45060,11 +58019,11 @@ impl TlsReportSettings { ExpressionContext { expr: &self.from_address, default: Some(Expression { - else_: "'noreply-tls@' + system('domain')".to_string(), + else_: "'noreply-tls@' + system('report_domain')".to_string(), ..Default::default() }), property: Property::FromAddress, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } } @@ -45077,7 +58036,7 @@ impl TlsReportSettings { ..Default::default() }), property: Property::FromName, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } } @@ -45090,7 +58049,7 @@ impl TlsReportSettings { ..Default::default() }), property: Property::MaxReportSize, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } } @@ -45099,11 +58058,11 @@ impl TlsReportSettings { ExpressionContext { expr: &self.org_name, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::OrgName, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_RCPT_DOMAIN_VARIABLE, allowed_constants: &[], } } @@ -45125,24 +58084,11 @@ impl TlsReportSettings { ExpressionContext { expr: &self.dkim_sign_domain, default: Some(Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }), property: Property::DkimSignDomain, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, - allowed_constants: &[], - } - } - - pub fn ctx_subject(&self) -> ExpressionContext<'_> { - ExpressionContext { - expr: &self.subject, - default: Some(Expression { - else_: "'TLS Aggregate Report'".to_string(), - ..Default::default() - }), - property: Property::Subject, - allowed_variables: MTA_QUEUE_HOST_VARIABLE, + allowed_variables: MTA_QUEUE_SENDER_VARIABLE, allowed_constants: &[], } } @@ -45156,13 +58102,13 @@ impl TlsReportSettings { self.ctx_org_name(), self.ctx_send_frequency(), self.ctx_dkim_sign_domain(), - self.ctx_subject(), ] } } impl Pickle for TlsReportSettings { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 7, 0, 0); self.contact_info.pickle(out); self.from_address.pickle(out); self.from_name.pickle(out); @@ -45170,20 +58116,80 @@ impl Pickle for TlsReportSettings { self.org_name.pickle(out); self.send_frequency.pickle(out); self.dkim_sign_domain.pickle(out); - self.subject.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.contact_info = Pickle::unpickle(stream)?; - this.from_address = Pickle::unpickle(stream)?; - this.from_name = Pickle::unpickle(stream)?; - this.max_report_size = Pickle::unpickle(stream)?; - this.org_name = Pickle::unpickle(stream)?; - this.send_frequency = Pickle::unpickle(stream)?; - this.dkim_sign_domain = Pickle::unpickle(stream)?; - this.subject = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: Expression = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "false".to_string(), + ..Default::default() + } + }; + let field1: Expression = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'noreply-tls@' + system('report_domain')".to_string(), + ..Default::default() + } + }; + let field2: Expression = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "'Report Subsystem'".to_string(), + ..Default::default() + } + }; + let field3: Expression = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "5242880".to_string(), + ..Default::default() + } + }; + let field4: Expression = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + let field5: Expression = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "daily".to_string(), + ..Default::default() + } + }; + let field6: Expression = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Expression { + else_: "system('report_domain')".to_string(), + ..Default::default() + } + }; + stream.close_block(block); + Some(Self { + contact_info: field0, + from_address: field1, + from_name: field2, + max_report_size: field3, + org_name: field4, + send_frequency: field5, + dkim_sign_domain: field6, + }) } } @@ -45195,7 +58201,7 @@ impl Default for TlsReportSettings { ..Default::default() }, from_address: Expression { - else_: "'noreply-tls@' + system('domain')".to_string(), + else_: "'noreply-tls@' + system('report_domain')".to_string(), ..Default::default() }, from_name: Expression { @@ -45207,7 +58213,7 @@ impl Default for TlsReportSettings { ..Default::default() }, org_name: Expression { - else_: "system('domain')".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, send_frequency: Expression { @@ -45215,11 +58221,7 @@ impl Default for TlsReportSettings { ..Default::default() }, dkim_sign_domain: Expression { - else_: "system('domain')".to_string(), - ..Default::default() - }, - subject: Expression { - else_: "'TLS Aggregate Report'".to_string(), + else_: "system('report_domain')".to_string(), ..Default::default() }, } @@ -45228,15 +58230,33 @@ impl Default for TlsReportSettings { impl IntoValue for TlsReportSettings { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::ContactInfo, self.contact_info.into_value()); - map.insert_unchecked(Property::FromAddress, self.from_address.into_value()); - map.insert_unchecked(Property::FromName, self.from_name.into_value()); - map.insert_unchecked(Property::MaxReportSize, self.max_report_size.into_value()); - map.insert_unchecked(Property::OrgName, self.org_name.into_value()); - map.insert_unchecked(Property::SendFrequency, self.send_frequency.into_value()); - map.insert_unchecked(Property::DkimSignDomain, self.dkim_sign_domain.into_value()); - map.insert_unchecked(Property::Subject, self.subject.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(9); + map.insert_unchecked( + Property::ContactInfo, + self.contact_info.into_value_with(opts), + ); + map.insert_unchecked( + Property::FromAddress, + self.from_address.into_value_with(opts), + ); + map.insert_unchecked(Property::FromName, self.from_name.into_value_with(opts)); + map.insert_unchecked( + Property::MaxReportSize, + self.max_report_size.into_value_with(opts), + ); + map.insert_unchecked(Property::OrgName, self.org_name.into_value_with(opts)); + map.insert_unchecked( + Property::SendFrequency, + self.send_frequency.into_value_with(opts), + ); + map.insert_unchecked( + Property::DkimSignDomain, + self.dkim_sign_domain.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -45255,962 +58275,6 @@ impl RegistryJsonPropertyPatch for TlsReportSettings { Some(Property::OrgName) => self.org_name.patch(pointer, value), Some(Property::SendFrequency) => self.send_frequency.patch(pointer, value), Some(Property::DkimSignDomain) => self.dkim_sign_domain.patch(pointer, value), - Some(Property::Subject) => self.subject.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl ObjectImpl for Trace { - const FLAGS: u64 = 0; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::Trace; - - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.events; - for value in value.values() { - value.validate(errors); - } - errors.len() == neb - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Pickle for Trace { - fn pickle(&self, out: &mut Vec) { - self.events.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.events = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for Trace { - fn default() -> Self { - Self { - events: Default::default(), - } - } -} - -impl IntoValue for Trace { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Events, self.events.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for Trace { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Events) => self.events.patch(pointer, value), - Some(Property::Timestamp) => pointer.assert_server_set(), - Some(Property::From) => pointer.assert_server_set(), - Some(Property::To) => pointer.assert_server_set(), - Some(Property::Size) => pointer.assert_server_set(), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceEvent { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.timestamp; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::Timestamp, value)); - } - let value = &self.key_values; - for value in value.values() { - value.validate(errors); - } - errors.len() == neb - } -} - -impl Pickle for TraceEvent { - fn pickle(&self, out: &mut Vec) { - self.event.pickle(out); - self.timestamp.pickle(out); - self.key_values.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.event = Pickle::unpickle(stream)?; - this.timestamp = Pickle::unpickle(stream)?; - this.key_values = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceEvent { - fn default() -> Self { - Self { - event: Default::default(), - timestamp: Default::default(), - key_values: Default::default(), - } - } -} - -impl IntoValue for TraceEvent { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(5); - map.insert_unchecked(Property::Event, self.event.into_value()); - map.insert_unchecked(Property::Timestamp, self.timestamp.into_value()); - map.insert_unchecked(Property::KeyValues, self.key_values.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceEvent { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Event) => self.event.patch(pointer, value), - Some(Property::Timestamp) => self.timestamp.patch(pointer, value), - Some(Property::KeyValues) => self.key_values.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceKeyValue { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - value.validate(errors); - errors.len() == neb - } -} - -impl Pickle for TraceKeyValue { - fn pickle(&self, out: &mut Vec) { - self.key.pickle(out); - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.key = Pickle::unpickle(stream)?; - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceKeyValue { - fn default() -> Self { - Self { - key: Default::default(), - value: Default::default(), - } - } -} - -impl IntoValue for TraceKeyValue { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Key, self.key.into_value()); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceKeyValue { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Key) => self.key.patch(pointer, value), - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValue { - fn validate(&self, errors: &mut Vec) -> bool { - match self { - TraceValue::String(inner) => inner.validate(errors), - TraceValue::UnsignedInt(inner) => inner.validate(errors), - TraceValue::Integer(inner) => inner.validate(errors), - TraceValue::Boolean(inner) => inner.validate(errors), - TraceValue::Float(inner) => inner.validate(errors), - TraceValue::UTCDateTime(inner) => inner.validate(errors), - TraceValue::Duration(inner) => inner.validate(errors), - TraceValue::IpAddr(inner) => inner.validate(errors), - TraceValue::List(inner) => inner.validate(errors), - TraceValue::Event(inner) => inner.validate(errors), - TraceValue::Null => true, - } - } -} - -impl Default for TraceValue { - fn default() -> Self { - TraceValue::String(Default::default()) - } -} - -impl Pickle for TraceValue { - fn pickle(&self, out: &mut Vec) { - match self { - TraceValue::String(inner) => { - 0u16.pickle(out); - inner.pickle(out); - } - TraceValue::UnsignedInt(inner) => { - 1u16.pickle(out); - inner.pickle(out); - } - TraceValue::Integer(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - TraceValue::Boolean(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - TraceValue::Float(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - TraceValue::UTCDateTime(inner) => { - 5u16.pickle(out); - inner.pickle(out); - } - TraceValue::Duration(inner) => { - 6u16.pickle(out); - inner.pickle(out); - } - TraceValue::IpAddr(inner) => { - 7u16.pickle(out); - inner.pickle(out); - } - TraceValue::List(inner) => { - 8u16.pickle(out); - inner.pickle(out); - } - TraceValue::Event(inner) => { - 9u16.pickle(out); - inner.pickle(out); - } - TraceValue::Null => { - 10u16.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Pickle::unpickle(stream).map(TraceValue::String), - 1 => Pickle::unpickle(stream).map(TraceValue::UnsignedInt), - 2 => Pickle::unpickle(stream).map(TraceValue::Integer), - 3 => Pickle::unpickle(stream).map(TraceValue::Boolean), - 4 => Pickle::unpickle(stream).map(TraceValue::Float), - 5 => Pickle::unpickle(stream).map(TraceValue::UTCDateTime), - 6 => Pickle::unpickle(stream).map(TraceValue::Duration), - 7 => Pickle::unpickle(stream).map(TraceValue::IpAddr), - 8 => Pickle::unpickle(stream).map(TraceValue::List), - 9 => Pickle::unpickle(stream).map(TraceValue::Event), - 10 => Some(TraceValue::Null), - _ => None, - } - } -} - -impl IntoValue for TraceValue { - fn into_value(self) -> JmapValue<'static> { - match self { - TraceValue::String(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("String".into())); - obj - } - TraceValue::UnsignedInt(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("UnsignedInt".into())); - obj - } - TraceValue::Integer(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Integer".into())); - obj - } - TraceValue::Boolean(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Boolean".into())); - obj - } - TraceValue::Float(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Float".into())); - obj - } - TraceValue::UTCDateTime(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("UTCDateTime".into())); - obj - } - TraceValue::Duration(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Duration".into())); - obj - } - TraceValue::IpAddr(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("IpAddr".into())); - obj - } - TraceValue::List(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("List".into())); - obj - } - TraceValue::Event(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("Event".into())); - obj - } - TraceValue::Null => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Null".into())); - JmapValue::Object(obj) - } - } - } -} - -impl RegistryJsonPatch for TraceValue { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - TraceValueType::String => *self = TraceValue::String(Default::default()), - TraceValueType::UnsignedInt => *self = TraceValue::UnsignedInt(Default::default()), - TraceValueType::Integer => *self = TraceValue::Integer(Default::default()), - TraceValueType::Boolean => *self = TraceValue::Boolean(Default::default()), - TraceValueType::Float => *self = TraceValue::Float(Default::default()), - TraceValueType::UTCDateTime => *self = TraceValue::UTCDateTime(Default::default()), - TraceValueType::Duration => *self = TraceValue::Duration(Default::default()), - TraceValueType::IpAddr => *self = TraceValue::IpAddr(Default::default()), - TraceValueType::List => *self = TraceValue::List(Default::default()), - TraceValueType::Event => *self = TraceValue::Event(Default::default()), - TraceValueType::Null => *self = TraceValue::Null, - } - } - match self { - TraceValue::String(inner) => inner.patch(pointer, value), - TraceValue::UnsignedInt(inner) => inner.patch(pointer, value), - TraceValue::Integer(inner) => inner.patch(pointer, value), - TraceValue::Boolean(inner) => inner.patch(pointer, value), - TraceValue::Float(inner) => inner.patch(pointer, value), - TraceValue::UTCDateTime(inner) => inner.patch(pointer, value), - TraceValue::Duration(inner) => inner.patch(pointer, value), - TraceValue::IpAddr(inner) => inner.patch(pointer, value), - TraceValue::List(inner) => inner.patch(pointer, value), - TraceValue::Event(inner) => inner.patch(pointer, value), - TraceValue::Null => pointer.assert_eof(), - } - } -} - -impl TraceValue { - pub fn object_type(&self) -> TraceValueType { - match self { - TraceValue::String(_) => TraceValueType::String, - TraceValue::UnsignedInt(_) => TraceValueType::UnsignedInt, - TraceValue::Integer(_) => TraceValueType::Integer, - TraceValue::Boolean(_) => TraceValueType::Boolean, - TraceValue::Float(_) => TraceValueType::Float, - TraceValue::UTCDateTime(_) => TraceValueType::UTCDateTime, - TraceValue::Duration(_) => TraceValueType::Duration, - TraceValue::IpAddr(_) => TraceValueType::IpAddr, - TraceValue::List(_) => TraceValueType::List, - TraceValue::Event(_) => TraceValueType::Event, - TraceValue::Null => TraceValueType::Null, - } - } -} - -impl TraceValueBoolean { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for TraceValueBoolean { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueBoolean { - fn default() -> Self { - Self { value: false } - } -} - -impl IntoValue for TraceValueBoolean { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueBoolean { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueDuration { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for TraceValueDuration { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueDuration { - fn default() -> Self { - Self { value: 0u64 } - } -} - -impl IntoValue for TraceValueDuration { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueDuration { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueEvent { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - for value in value.values() { - value.validate(errors); - } - errors.len() == neb - } -} - -impl Pickle for TraceValueEvent { - fn pickle(&self, out: &mut Vec) { - self.event.pickle(out); - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.event = Pickle::unpickle(stream)?; - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueEvent { - fn default() -> Self { - Self { - event: Default::default(), - value: Default::default(), - } - } -} - -impl IntoValue for TraceValueEvent { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(4); - map.insert_unchecked(Property::Event, self.event.into_value()); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueEvent { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Event) => self.event.patch(pointer, value), - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueFloat { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for TraceValueFloat { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueFloat { - fn default() -> Self { - Self { - value: Float::new(0.0f64), - } - } -} - -impl IntoValue for TraceValueFloat { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueFloat { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueInteger { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for TraceValueInteger { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueInteger { - fn default() -> Self { - Self { value: 0i64 } - } -} - -impl IntoValue for TraceValueInteger { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueInteger { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueIpAddr { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::Value, value)); - } - errors.len() == neb - } -} - -impl Pickle for TraceValueIpAddr { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueIpAddr { - fn default() -> Self { - Self { - value: Default::default(), - } - } -} - -impl IntoValue for TraceValueIpAddr { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueIpAddr { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueList { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - for value in value.values() { - value.validate(errors); - } - errors.len() == neb - } -} - -impl Pickle for TraceValueList { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueList { - fn default() -> Self { - Self { - value: Default::default(), - } - } -} - -impl IntoValue for TraceValueList { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueList { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueString { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - if value.is_empty() { - errors.push(ValidationError::required(Property::Value)); - } - errors.len() == neb - } -} - -impl Pickle for TraceValueString { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueString { - fn default() -> Self { - Self { - value: Default::default(), - } - } -} - -impl IntoValue for TraceValueString { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueString { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueUTCDateTime { - fn validate(&self, errors: &mut Vec) -> bool { - let neb = errors.len(); - let value = &self.value; - if !value.is_valid() { - errors.push(ValidationError::invalid(Property::Value, value)); - } - errors.len() == neb - } -} - -impl Pickle for TraceValueUTCDateTime { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueUTCDateTime { - fn default() -> Self { - Self { - value: Default::default(), - } - } -} - -impl IntoValue for TraceValueUTCDateTime { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueUTCDateTime { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), - Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { - property: Property::Type, - value, - }), - _ => Err(PatchError::new(pointer, "Invalid property")), - } - } -} - -impl TraceValueUnsignedInt { - fn validate(&self, _: &mut Vec) -> bool { - true - } -} - -impl Pickle for TraceValueUnsignedInt { - fn pickle(&self, out: &mut Vec) { - self.value.pickle(out); - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.value = Pickle::unpickle(stream)?; - Some(this) - } -} - -impl Default for TraceValueUnsignedInt { - fn default() -> Self { - Self { value: 0u64 } - } -} - -impl IntoValue for TraceValueUnsignedInt { - fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Value, self.value.into_value()); - JmapValue::Object(map) - } -} - -impl RegistryJsonPropertyPatch for TraceValueUnsignedInt { - fn patch_property<'x>( - &mut self, - mut pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - match pointer.next_property() { - Some(Property::Value) => self.value.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, @@ -46236,6 +58300,7 @@ impl ObjectImpl for Tracer { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Default for Tracer { @@ -46271,6 +58336,9 @@ impl Pickle for Tracer { } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + if stream.version() > Self::VERSION { + return None; + } match u16::unpickle(stream)? { 0 => Pickle::unpickle(stream).map(Tracer::Log), 1 => Pickle::unpickle(stream).map(Tracer::Stdout), @@ -46284,37 +58352,41 @@ impl Pickle for Tracer { impl IntoValue for Tracer { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { Tracer::Log(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Log".into())); obj } Tracer::Stdout(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Stdout".into())); obj } Tracer::Journal(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Journal".into())); obj } Tracer::OtelHttp(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("OtelHttp".into())); obj } Tracer::OtelGrpc(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("OtelGrpc".into())); @@ -46369,21 +58441,47 @@ impl TracerCommon { impl Pickle for TracerCommon { fn pickle(&self, out: &mut Vec) { - self.enable.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.lossy { + bitmap |= 1 << 1; + } + let block = crate::pickle::begin_block(out, 5, 2, bitmap); self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: TracingLevel = if block.has(1) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + let field3: Map = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: EventPolicy = if block.has(4) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + enable: field0, + level: field1, + lossy: field2, + events: field3, + events_policy: field4, + }) } } @@ -46401,12 +58499,19 @@ impl Default for TracerCommon { impl IntoValue for TracerCommon { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(7); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -46449,31 +58554,78 @@ impl TracerLog { impl Pickle for TracerLog { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.ansi { + bitmap |= 1 << 0; + } + if self.multiline { + bitmap |= 1 << 1; + } + if self.enable { + bitmap |= 1 << 2; + } + if self.lossy { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 10, 4, bitmap); self.path.pickle(out); self.prefix.pickle(out); self.rotate.pickle(out); - self.ansi.pickle(out); - self.multiline.pickle(out); - self.enable.pickle(out); self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.path = Pickle::unpickle(stream)?; - this.prefix = Pickle::unpickle(stream)?; - this.rotate = Pickle::unpickle(stream)?; - this.ansi = Pickle::unpickle(stream)?; - this.multiline = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: String = if block.has(1) { + Pickle::unpickle(stream)? + } else { + "stalwart.log".to_string() + }; + let field2: LogRotateFrequency = if block.has(2) { + Pickle::unpickle(stream)? + } else { + LogRotateFrequency::Daily + }; + let field3: bool = if block.has(3) { block.flag(0) } else { true }; + let field4: bool = if block.has(4) { block.flag(1) } else { false }; + let field5: bool = if block.has(5) { block.flag(2) } else { true }; + let field6: TracingLevel = if block.has(6) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field7: bool = if block.has(7) { block.flag(3) } else { false }; + let field8: Map = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: EventPolicy = if block.has(9) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + path: field0, + prefix: field1, + rotate: field2, + ansi: field3, + multiline: field4, + enable: field5, + level: field6, + lossy: field7, + events: field8, + events_policy: field9, + }) } } @@ -46481,7 +58633,7 @@ impl Default for TracerLog { fn default() -> Self { Self { path: Default::default(), - prefix: "inbuxa".to_string(), + prefix: "stalwart.log".to_string(), rotate: LogRotateFrequency::Daily, ansi: true, multiline: false, @@ -46496,17 +58648,24 @@ impl Default for TracerLog { impl IntoValue for TracerLog { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(12); - map.insert_unchecked(Property::Path, self.path.into_value()); - map.insert_unchecked(Property::Prefix, self.prefix.into_value()); - map.insert_unchecked(Property::Rotate, self.rotate.into_value()); - map.insert_unchecked(Property::Ansi, self.ansi.into_value()); - map.insert_unchecked(Property::Multiline, self.multiline.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); + map.insert_unchecked(Property::Path, self.path.into_value_with(opts)); + map.insert_unchecked(Property::Prefix, self.prefix.into_value_with(opts)); + map.insert_unchecked(Property::Rotate, self.rotate.into_value_with(opts)); + map.insert_unchecked(Property::Ansi, self.ansi.into_value_with(opts)); + map.insert_unchecked(Property::Multiline, self.multiline.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -46544,9 +58703,9 @@ impl RegistryJsonPropertyPatch for TracerLog { impl TracerOtelGrpc { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - if let Some(value) = &self.endpoint { + if let Some(value) = &self.url { if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } } let value = &self.http_auth; @@ -46559,46 +58718,113 @@ impl TracerOtelGrpc { } errors.len() == neb } + } impl Pickle for TracerOtelGrpc { fn pickle(&self, out: &mut Vec) { - self.endpoint.pickle(out); - self.enable_log_exporter.pickle(out); - self.enable_span_exporter.pickle(out); + let mut bitmap = 0u64; + if self.url.is_some() { + bitmap |= 1 << 0; + } + if self.enable_log_exporter { + bitmap |= 1 << 1; + } + if self.enable_span_exporter { + bitmap |= 1 << 2; + } + if self.enable { + bitmap |= 1 << 3; + } + if self.lossy { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 12, 5, bitmap); + if let Some(value) = &self.url { + value.pickle(out); + } self.throttle.pickle(out); self.timeout.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); - self.enable.pickle(out); self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.endpoint = Pickle::unpickle(stream)?; - this.enable_log_exporter = Pickle::unpickle(stream)?; - this.enable_span_exporter = Pickle::unpickle(stream)?; - this.throttle = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: Option = if block.has(0) { + if block.flag(0) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(1) } else { true }; + let field2: bool = if block.has(2) { block.flag(2) } else { true }; + let field3: Duration = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field5: HttpAuth = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: VecMap = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: bool = if block.has(7) { block.flag(3) } else { true }; + let field8: TracingLevel = if block.has(8) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field9: bool = if block.has(9) { block.flag(4) } else { false }; + let field10: Map = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: EventPolicy = if block.has(11) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + url: field0, + enable_log_exporter: field1, + enable_span_exporter: field2, + throttle: field3, + timeout: field4, + http_auth: field5, + http_headers: field6, + enable: field7, + level: field8, + lossy: field9, + events: field10, + events_policy: field11, + }) } } impl Default for TracerOtelGrpc { fn default() -> Self { Self { - endpoint: Default::default(), + url: Default::default(), enable_log_exporter: true, enable_span_exporter: true, throttle: Duration::from_millis(1000), @@ -46616,25 +58842,35 @@ impl Default for TracerOtelGrpc { impl IntoValue for TracerOtelGrpc { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); map.insert_unchecked( Property::EnableLogExporter, - self.enable_log_exporter.into_value(), + self.enable_log_exporter.into_value_with(opts), ); map.insert_unchecked( Property::EnableSpanExporter, - self.enable_span_exporter.into_value(), + self.enable_span_exporter.into_value_with(opts), + ); + map.insert_unchecked(Property::Throttle, self.throttle.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), ); - map.insert_unchecked(Property::Throttle, self.throttle.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); JmapValue::Object(map) } } @@ -46646,8 +58882,8 @@ impl RegistryJsonPropertyPatch for TracerOtelGrpc { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), Some(Property::EnableLogExporter) => self.enable_log_exporter.patch(pointer, value), Some(Property::EnableSpanExporter) => self.enable_span_exporter.patch(pointer, value), @@ -46674,9 +58910,9 @@ impl RegistryJsonPropertyPatch for TracerOtelGrpc { impl TracerOtelHttp { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); - let value = &self.endpoint; + let value = &self.url; if value.is_empty() { - errors.push(ValidationError::required(Property::Endpoint)); + errors.push(ValidationError::required(Property::Url)); } let value = &self.http_auth; value.validate(errors); @@ -46688,46 +58924,110 @@ impl TracerOtelHttp { } errors.len() == neb } + } impl Pickle for TracerOtelHttp { fn pickle(&self, out: &mut Vec) { - self.endpoint.pickle(out); - self.enable_log_exporter.pickle(out); - self.enable_span_exporter.pickle(out); + let mut bitmap = 0u64; + if self.allow_invalid_certs { + bitmap |= 1 << 0; + } + if self.enable_log_exporter { + bitmap |= 1 << 1; + } + if self.enable_span_exporter { + bitmap |= 1 << 2; + } + if self.enable { + bitmap |= 1 << 3; + } + if self.lossy { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 13, 5, bitmap); + self.url.pickle(out); self.throttle.pickle(out); self.timeout.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); - self.enable.pickle(out); self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.endpoint = Pickle::unpickle(stream)?; - this.enable_log_exporter = Pickle::unpickle(stream)?; - this.enable_span_exporter = Pickle::unpickle(stream)?; - this.throttle = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: bool = if block.has(1) { block.flag(0) } else { false }; + let field2: bool = if block.has(2) { block.flag(1) } else { true }; + let field3: bool = if block.has(3) { block.flag(2) } else { true }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(10000) + }; + let field6: HttpAuth = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: VecMap = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: bool = if block.has(8) { block.flag(3) } else { true }; + let field9: TracingLevel = if block.has(9) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field10: bool = if block.has(10) { block.flag(4) } else { false }; + let field11: Map = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: EventPolicy = if block.has(12) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + url: field0, + allow_invalid_certs: field1, + enable_log_exporter: field2, + enable_span_exporter: field3, + throttle: field4, + timeout: field5, + http_auth: field6, + http_headers: field7, + enable: field8, + level: field9, + lossy: field10, + events: field11, + events_policy: field12, + }) } } impl Default for TracerOtelHttp { fn default() -> Self { Self { - endpoint: Default::default(), + url: Default::default(), + allow_invalid_certs: false, enable_log_exporter: true, enable_span_exporter: true, throttle: Duration::from_millis(1000), @@ -46745,25 +59045,39 @@ impl Default for TracerOtelHttp { impl IntoValue for TracerOtelHttp { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(14); - map.insert_unchecked(Property::Endpoint, self.endpoint.into_value()); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(15); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked( + Property::AllowInvalidCerts, + self.allow_invalid_certs.into_value_with(opts), + ); map.insert_unchecked( Property::EnableLogExporter, - self.enable_log_exporter.into_value(), + self.enable_log_exporter.into_value_with(opts), ); map.insert_unchecked( Property::EnableSpanExporter, - self.enable_span_exporter.into_value(), + self.enable_span_exporter.into_value_with(opts), + ); + map.insert_unchecked(Property::Throttle, self.throttle.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), ); - map.insert_unchecked(Property::Throttle, self.throttle.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); JmapValue::Object(map) } } @@ -46775,9 +59089,10 @@ impl RegistryJsonPropertyPatch for TracerOtelHttp { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::Endpoint) => self - .endpoint + Some(Property::Url) => self + .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::EnableLogExporter) => self.enable_log_exporter.patch(pointer, value), Some(Property::EnableSpanExporter) => self.enable_span_exporter.patch(pointer, value), Some(Property::Throttle) => self.throttle.patch(pointer, value), @@ -46808,27 +59123,62 @@ impl TracerStdout { impl Pickle for TracerStdout { fn pickle(&self, out: &mut Vec) { - self.buffered.pickle(out); - self.ansi.pickle(out); - self.multiline.pickle(out); - self.enable.pickle(out); + let mut bitmap = 0u64; + if self.buffered { + bitmap |= 1 << 0; + } + if self.ansi { + bitmap |= 1 << 1; + } + if self.multiline { + bitmap |= 1 << 2; + } + if self.enable { + bitmap |= 1 << 3; + } + if self.lossy { + bitmap |= 1 << 4; + } + let block = crate::pickle::begin_block(out, 8, 5, bitmap); self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.buffered = Pickle::unpickle(stream)?; - this.ansi = Pickle::unpickle(stream)?; - this.multiline = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: bool = if block.has(1) { block.flag(1) } else { false }; + let field2: bool = if block.has(2) { block.flag(2) } else { false }; + let field3: bool = if block.has(3) { block.flag(3) } else { true }; + let field4: TracingLevel = if block.has(4) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field5: bool = if block.has(5) { block.flag(4) } else { false }; + let field6: Map = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: EventPolicy = if block.has(7) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + buffered: field0, + ansi: field1, + multiline: field2, + enable: field3, + level: field4, + lossy: field5, + events: field6, + events_policy: field7, + }) } } @@ -46849,15 +59199,22 @@ impl Default for TracerStdout { impl IntoValue for TracerStdout { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(10); - map.insert_unchecked(Property::Buffered, self.buffered.into_value()); - map.insert_unchecked(Property::Ansi, self.ansi.into_value()); - map.insert_unchecked(Property::Multiline, self.multiline.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); + map.insert_unchecked(Property::Buffered, self.buffered.into_value_with(opts)); + map.insert_unchecked(Property::Ansi, self.ansi.into_value_with(opts)); + map.insert_unchecked(Property::Multiline, self.multiline.into_value_with(opts)); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), + ); JmapValue::Object(map) } } @@ -46886,145 +59243,6 @@ impl RegistryJsonPropertyPatch for TracerStdout { } } -impl ObjectImpl for TracingStore { - const FLAGS: u64 = OBJ_SINGLETON; - const VERSION: u8 = 0; - const OBJECT: ObjectType = ObjectType::TracingStore; - - fn validate(&self, errors: &mut Vec) -> bool { - match self { - TracingStore::Disabled => true, - TracingStore::Default => true, - TracingStore::FoundationDb(inner) => inner.validate(errors), - TracingStore::PostgreSql(inner) => inner.validate(errors), - TracingStore::MySql(inner) => inner.validate(errors), - } - } - - fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} -} - -impl Default for TracingStore { - fn default() -> Self { - TracingStore::Disabled - } -} - -impl Pickle for TracingStore { - fn pickle(&self, out: &mut Vec) { - match self { - TracingStore::Disabled => { - 0u16.pickle(out); - } - TracingStore::Default => { - 1u16.pickle(out); - } - TracingStore::FoundationDb(inner) => { - 2u16.pickle(out); - inner.pickle(out); - } - TracingStore::PostgreSql(inner) => { - 3u16.pickle(out); - inner.pickle(out); - } - TracingStore::MySql(inner) => { - 4u16.pickle(out); - inner.pickle(out); - } - } - } - - fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - match u16::unpickle(stream)? { - 0 => Some(TracingStore::Disabled), - 1 => Some(TracingStore::Default), - 2 => Pickle::unpickle(stream).map(TracingStore::FoundationDb), - 3 => Pickle::unpickle(stream).map(TracingStore::PostgreSql), - 4 => Pickle::unpickle(stream).map(TracingStore::MySql), - _ => None, - } - } -} - -impl IntoValue for TracingStore { - fn into_value(self) -> JmapValue<'static> { - match self { - TracingStore::Disabled => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Disabled".into())); - JmapValue::Object(obj) - } - TracingStore::Default => { - let mut obj = jmap_tools::Map::new(); - obj.insert_unchecked(Property::Type, JmapValue::Str("Default".into())); - JmapValue::Object(obj) - } - TracingStore::FoundationDb(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("FoundationDb".into())); - obj - } - TracingStore::PostgreSql(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("PostgreSql".into())); - obj - } - TracingStore::MySql(obj) => { - let mut obj = obj.into_value(); - obj.as_object_mut() - .unwrap() - .insert_unchecked(Property::Type, JmapValue::Str("MySql".into())); - obj - } - } - } -} - -impl RegistryJsonPatch for TracingStore { - fn patch<'x>( - &mut self, - pointer: JsonPointerPatch<'_>, - value: JmapValue<'x>, - ) -> PatchResult<'x> { - if !pointer.has_next() { - match object_type(&pointer, &value)? { - TracingStoreType::Disabled => *self = TracingStore::Disabled, - TracingStoreType::Default => *self = TracingStore::Default, - TracingStoreType::FoundationDb => { - *self = TracingStore::FoundationDb(Default::default()) - } - TracingStoreType::PostgreSql => { - *self = TracingStore::PostgreSql(Default::default()) - } - TracingStoreType::MySql => *self = TracingStore::MySql(Default::default()), - } - } - match self { - TracingStore::Disabled => pointer.assert_eof(), - TracingStore::Default => pointer.assert_eof(), - TracingStore::FoundationDb(inner) => inner.patch(pointer, value), - TracingStore::PostgreSql(inner) => inner.patch(pointer, value), - TracingStore::MySql(inner) => inner.patch(pointer, value), - } - } -} - -impl TracingStore { - pub fn object_type(&self) -> TracingStoreType { - match self { - TracingStore::Disabled => TracingStoreType::Disabled, - TracingStore::Default => TracingStoreType::Default, - TracingStore::FoundationDb(_) => TracingStoreType::FoundationDb, - TracingStore::PostgreSql(_) => TracingStoreType::PostgreSql, - TracingStore::MySql(_) => TracingStoreType::MySql, - } - } -} - impl UserAccount { fn validate(&self, errors: &mut Vec) -> bool { let neb = errors.len(); @@ -47050,11 +59268,6 @@ impl UserAccount { errors.push(ValidationError::required(Property::MemberGroupIds)); } } - if let Some(value) = &self.member_tenant_id { - if !value.is_valid() { - errors.push(ValidationError::required(Property::MemberTenantId)); - } - } let value = &self.roles; value.validate(errors); let value = &self.permissions; @@ -47063,11 +59276,6 @@ impl UserAccount { for value in value.values() { value.validate(errors); } - if let Some(value) = &self.external_id { - if value.is_empty() { - errors.push(ValidationError::required(Property::ExternalId)); - } - } if let Some(value) = &self.description { if value.is_empty() { errors.push(ValidationError::required(Property::Description)); @@ -47079,7 +59287,11 @@ impl UserAccount { } fn index<'x>(&'x self, i: &mut IndexBuilder<'x>) { - i.unique_global_composite(Property::Email, &self.name, &self.domain_id); + i.unique_global_composite( + Property::Email, + &self.name, + IndexValue::U32(self.domain_id.document_id()), + ); i.text(Property::Text, &self.name); i.search(Property::Name, &self.name); i.foreign_key(ObjectType::Domain, self.domain_id.into(), None); @@ -47094,17 +59306,10 @@ impl UserAccount { for value in self.member_group_ids.iter() { i.search(Property::MemberGroupIds, value); } - i.foreign_key(ObjectType::Tenant, self.member_tenant_id, None); - if let Some(value) = &self.member_tenant_id { - i.search(Property::MemberTenantId, value); - } self.roles.index(i); for item in self.aliases.values() { item.index(i); } - if let Some(value) = &self.external_id { - i.search(Property::ExternalId, value); - } if let Some(value) = &self.description { i.text(Property::Text, value); } @@ -47114,43 +59319,125 @@ impl UserAccount { impl Pickle for UserAccount { fn pickle(&self, out: &mut Vec) { + let mut bitmap = 0u64; + if self.description.is_some() { + bitmap |= 1 << 2; + } + if self.time_zone.is_some() { + bitmap |= 1 << 3; + } + let block = crate::pickle::begin_block(out, 15, 4, bitmap); self.name.pickle(out); self.domain_id.pickle(out); self.credentials.pickle(out); self.created_at.pickle(out); self.member_group_ids.pickle(out); - self.member_tenant_id.pickle(out); self.roles.pickle(out); self.permissions.pickle(out); self.quotas.pickle(out); self.aliases.pickle(out); - self.external_id.pickle(out); - self.description.pickle(out); + if let Some(value) = &self.description { + value.pickle(out); + } self.locale.pickle(out); - self.time_zone.pickle(out); + if let Some(value) = &self.time_zone { + value.pickle(out); + } self.encryption_at_rest.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.name = Pickle::unpickle(stream)?; - this.domain_id = Pickle::unpickle(stream)?; - this.credentials = Pickle::unpickle(stream)?; - this.created_at = Pickle::unpickle(stream)?; - this.member_group_ids = Pickle::unpickle(stream)?; - this.member_tenant_id = Pickle::unpickle(stream)?; - this.roles = Pickle::unpickle(stream)?; - this.permissions = Pickle::unpickle(stream)?; - this.quotas = Pickle::unpickle(stream)?; - this.aliases = Pickle::unpickle(stream)?; - if stream.version() >= 1 { - this.external_id = Pickle::unpickle(stream)?; - } - this.description = Pickle::unpickle(stream)?; - this.locale = Pickle::unpickle(stream)?; - this.time_zone = Pickle::unpickle(stream)?; - this.encryption_at_rest = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field1: Id = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field2: List = if block.has(2) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field3: UTCDateTime = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Map = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field6: UserRoles = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Permissions = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field8: VecMap = if block.has(8) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field9: List = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: Option = if block.has(11) { + if block.flag(2) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field12: Locale = if block.has(12) { + Pickle::unpickle(stream)? + } else { + Locale::EnUS + }; + let field13: Option = if block.has(13) { + if block.flag(3) { + Some(Pickle::unpickle(stream)?) + } else { + None + } + } else { + Default::default() + }; + let field14: EncryptionAtRest = if block.has(14) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { + name: field0, + domain_id: field1, + credentials: field2, + created_at: field3, + member_group_ids: field4, + roles: field6, + permissions: field7, + quotas: field8, + aliases: field9, + description: field11, + locale: field12, + time_zone: field13, + encryption_at_rest: field14, + }) } } @@ -47162,12 +59449,10 @@ impl Default for UserAccount { credentials: Default::default(), created_at: Default::default(), member_group_ids: Default::default(), - member_tenant_id: Default::default(), roles: Default::default(), permissions: Default::default(), quotas: Default::default(), aliases: Default::default(), - external_id: Default::default(), description: Default::default(), locale: Locale::EnUS, time_zone: Default::default(), @@ -47178,24 +59463,38 @@ impl Default for UserAccount { impl IntoValue for UserAccount { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(17); - map.insert_unchecked(Property::Name, self.name.into_value()); - map.insert_unchecked(Property::DomainId, self.domain_id.into_value()); - map.insert_unchecked(Property::Credentials, self.credentials.into_value()); - map.insert_unchecked(Property::CreatedAt, self.created_at.into_value()); - map.insert_unchecked(Property::MemberGroupIds, self.member_group_ids.into_value()); - map.insert_unchecked(Property::MemberTenantId, self.member_tenant_id.into_value()); - map.insert_unchecked(Property::Roles, self.roles.into_value()); - map.insert_unchecked(Property::Permissions, self.permissions.into_value()); - map.insert_unchecked(Property::Quotas, self.quotas.into_value()); - map.insert_unchecked(Property::Aliases, self.aliases.into_value()); - map.insert_unchecked(Property::ExternalId, self.external_id.into_value()); - map.insert_unchecked(Property::Description, self.description.into_value()); - map.insert_unchecked(Property::Locale, self.locale.into_value()); - map.insert_unchecked(Property::TimeZone, self.time_zone.into_value()); + map.insert_unchecked(Property::Name, self.name.into_value_with(opts)); + map.insert_unchecked(Property::DomainId, self.domain_id.into_value_with(opts)); + map.insert_unchecked( + Property::Credentials, + self.credentials.into_value_with(opts), + ); + map.insert_unchecked(Property::CreatedAt, self.created_at.into_value_with(opts)); + map.insert_unchecked( + Property::MemberGroupIds, + self.member_group_ids.into_value_with(opts), + ); + map.insert_unchecked(Property::Roles, self.roles.into_value_with(opts)); + map.insert_unchecked( + Property::Permissions, + self.permissions.into_value_with(opts), + ); + map.insert_unchecked(Property::Quotas, self.quotas.into_value_with(opts)); + map.insert_unchecked(Property::Aliases, self.aliases.into_value_with(opts)); + map.insert_unchecked( + Property::Description, + self.description.into_value_with(opts), + ); + map.insert_unchecked(Property::Locale, self.locale.into_value_with(opts)); + map.insert_unchecked(Property::TimeZone, self.time_zone.into_value_with(opts)); map.insert_unchecked( Property::EncryptionAtRest, - self.encryption_at_rest.into_value(), + self.encryption_at_rest.into_value_with(opts), ); JmapValue::Object(map) } @@ -47217,15 +59516,11 @@ impl RegistryJsonPropertyPatch for UserAccount { Some(Property::Credentials) => self.credentials.patch(pointer, value), Some(Property::CreatedAt) => pointer.assert_server_set(), Some(Property::MemberGroupIds) => self.member_group_ids.patch(pointer, value), - Some(Property::MemberTenantId) => self - .member_tenant_id - .patch(pointer.assert_can_set_tenant()?, value), Some(Property::Roles) => self.roles.patch(pointer, value), Some(Property::Permissions) => self.permissions.patch(pointer, value), Some(Property::Quotas) => self.quotas.patch(pointer, value), Some(Property::UsedDiskQuota) => pointer.assert_server_set(), Some(Property::Aliases) => self.aliases.patch(pointer, value), - Some(Property::ExternalId) => self.external_id.patch(pointer, value), Some(Property::Description) => self.description.patch(pointer, value), Some(Property::Locale) => self.locale.patch(pointer, value), Some(Property::TimeZone) => self.time_zone.patch(pointer, value), @@ -47293,6 +59588,10 @@ impl Pickle for UserRoles { impl IntoValue for UserRoles { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { match self { UserRoles::User => { let mut obj = jmap_tools::Map::new(); @@ -47305,7 +59604,7 @@ impl IntoValue for UserRoles { JmapValue::Object(obj) } UserRoles::Custom(obj) => { - let mut obj = obj.into_value(); + let mut obj = obj.into_value_with(opts); obj.as_object_mut() .unwrap() .insert_unchecked(Property::Type, JmapValue::Str("Custom".into())); @@ -47360,25 +59659,59 @@ impl ObjectImpl for WebDav { impl Pickle for WebDav { fn pickle(&self, out: &mut Vec) { - self.enable_assisted_discovery.pickle(out); + let mut bitmap = 0u64; + if self.enable_assisted_discovery { + bitmap |= 1 << 0; + } + let block = crate::pickle::begin_block(out, 6, 1, bitmap); self.max_lock_timeout.pickle(out); self.max_locks.pickle(out); - self.dead_property_max_size.pickle(out); - self.live_property_max_size.pickle(out); - self.request_max_size.pickle(out); + self.max_live_property_size.pickle(out); + self.max_request_size.pickle(out); self.max_results.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.enable_assisted_discovery = Pickle::unpickle(stream)?; - this.max_lock_timeout = Pickle::unpickle(stream)?; - this.max_locks = Pickle::unpickle(stream)?; - this.dead_property_max_size = Pickle::unpickle(stream)?; - this.live_property_max_size = Pickle::unpickle(stream)?; - this.request_max_size = Pickle::unpickle(stream)?; - this.max_results = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: Duration = if block.has(1) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(3600000) + }; + let field2: u64 = if block.has(2) { + Pickle::unpickle(stream)? + } else { + 10u64 + }; + let field3: u64 = if block.has(3) { + Pickle::unpickle(stream)? + } else { + 250u64 + }; + let field4: u64 = if block.has(4) { + Pickle::unpickle(stream)? + } else { + 26214400 + }; + let field5: u64 = if block.has(5) { + Pickle::unpickle(stream)? + } else { + 2000u64 + }; + stream.close_block(block); + Some(Self { + enable_assisted_discovery: field0, + max_lock_timeout: field1, + max_locks: field2, + max_live_property_size: field3, + max_request_size: field4, + max_results: field5, + }) } } @@ -47388,9 +59721,8 @@ impl Default for WebDav { enable_assisted_discovery: true, max_lock_timeout: Duration::from_millis(3600000), max_locks: 10u64, - dead_property_max_size: Some(1024u64), - live_property_max_size: 250u64, - request_max_size: 26214400, + max_live_property_size: 250u64, + max_request_size: 26214400, max_results: 2000u64, } } @@ -47398,23 +59730,29 @@ impl Default for WebDav { impl IntoValue for WebDav { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(9); + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { + let mut map = jmap_tools::Map::with_capacity(8); map.insert_unchecked( Property::EnableAssistedDiscovery, - self.enable_assisted_discovery.into_value(), - ); - map.insert_unchecked(Property::MaxLockTimeout, self.max_lock_timeout.into_value()); - map.insert_unchecked(Property::MaxLocks, self.max_locks.into_value()); - map.insert_unchecked( - Property::DeadPropertyMaxSize, - self.dead_property_max_size.into_value(), + self.enable_assisted_discovery.into_value_with(opts), ); map.insert_unchecked( - Property::LivePropertyMaxSize, - self.live_property_max_size.into_value(), + Property::MaxLockTimeout, + self.max_lock_timeout.into_value_with(opts), ); - map.insert_unchecked(Property::RequestMaxSize, self.request_max_size.into_value()); - map.insert_unchecked(Property::MaxResults, self.max_results.into_value()); + map.insert_unchecked(Property::MaxLocks, self.max_locks.into_value_with(opts)); + map.insert_unchecked( + Property::MaxLivePropertySize, + self.max_live_property_size.into_value_with(opts), + ); + map.insert_unchecked( + Property::MaxRequestSize, + self.max_request_size.into_value_with(opts), + ); + map.insert_unchecked(Property::MaxResults, self.max_results.into_value_with(opts)); JmapValue::Object(map) } } @@ -47431,13 +59769,10 @@ impl RegistryJsonPropertyPatch for WebDav { } Some(Property::MaxLockTimeout) => self.max_lock_timeout.patch(pointer, value), Some(Property::MaxLocks) => self.max_locks.patch(pointer, value), - Some(Property::DeadPropertyMaxSize) => { - self.dead_property_max_size.patch(pointer, value) + Some(Property::MaxLivePropertySize) => { + self.max_live_property_size.patch(pointer, value) } - Some(Property::LivePropertyMaxSize) => { - self.live_property_max_size.patch(pointer, value) - } - Some(Property::RequestMaxSize) => self.request_max_size.patch(pointer, value), + Some(Property::MaxRequestSize) => self.max_request_size.patch(pointer, value), Some(Property::MaxResults) => self.max_results.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, @@ -47473,58 +59808,126 @@ impl ObjectImpl for WebHook { } fn index<'x>(&'x self, _: &mut IndexBuilder<'x>) {} + } impl Pickle for WebHook { fn pickle(&self, out: &mut Vec) { - self.allow_invalid_certs.pickle(out); + let mut bitmap = 0u64; + if self.enable { + bitmap |= 1 << 0; + } + if self.lossy { + bitmap |= 1 << 1; + } + if self.allow_invalid_certs { + bitmap |= 1 << 2; + } + let block = crate::pickle::begin_block(out, 13, 3, bitmap); + self.level.pickle(out); self.signature_key.pickle(out); self.throttle.pickle(out); - self.timeout.pickle(out); self.discard_after.pickle(out); self.url.pickle(out); + self.timeout.pickle(out); self.http_auth.pickle(out); self.http_headers.pickle(out); - self.enable.pickle(out); - self.level.pickle(out); - self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.allow_invalid_certs = Pickle::unpickle(stream)?; - this.signature_key = Pickle::unpickle(stream)?; - this.throttle = Pickle::unpickle(stream)?; - this.timeout = Pickle::unpickle(stream)?; - this.discard_after = Pickle::unpickle(stream)?; - this.url = Pickle::unpickle(stream)?; - this.http_auth = Pickle::unpickle(stream)?; - this.http_headers = Pickle::unpickle(stream)?; - this.enable = Pickle::unpickle(stream)?; - this.level = Pickle::unpickle(stream)?; - this.lossy = Pickle::unpickle(stream)?; - this.events = Pickle::unpickle(stream)?; - this.events_policy = Pickle::unpickle(stream)?; - Some(this) + if stream.version() > Self::VERSION { + return None; + } + let block = stream.open_block()?; + let field0: bool = if block.has(0) { block.flag(0) } else { true }; + let field1: TracingLevel = if block.has(1) { + Pickle::unpickle(stream)? + } else { + TracingLevel::Info + }; + let field2: bool = if block.has(2) { block.flag(1) } else { false }; + let field3: SecretKeyOptional = if block.has(3) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field4: Duration = if block.has(4) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(1000) + }; + let field5: Duration = if block.has(5) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(300000) + }; + let field6: String = if block.has(6) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field7: Duration = if block.has(7) { + Pickle::unpickle(stream)? + } else { + Duration::from_millis(30000) + }; + let field8: bool = if block.has(8) { block.flag(2) } else { false }; + let field9: HttpAuth = if block.has(9) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field10: VecMap = if block.has(10) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field11: Map = if block.has(11) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + let field12: EventPolicy = if block.has(12) { + Pickle::unpickle(stream)? + } else { + EventPolicy::Exclude + }; + stream.close_block(block); + Some(Self { + enable: field0, + level: field1, + lossy: field2, + signature_key: field3, + throttle: field4, + discard_after: field5, + url: field6, + timeout: field7, + allow_invalid_certs: field8, + http_auth: field9, + http_headers: field10, + events: field11, + events_policy: field12, + }) } } impl Default for WebHook { fn default() -> Self { Self { - allow_invalid_certs: false, - signature_key: Default::default(), - throttle: Duration::from_millis(1000), - timeout: Duration::from_millis(30000), - discard_after: Duration::from_millis(300000), - url: Default::default(), - http_auth: Default::default(), - http_headers: Default::default(), enable: true, level: TracingLevel::Info, lossy: false, + signature_key: Default::default(), + throttle: Duration::from_millis(1000), + discard_after: Duration::from_millis(300000), + url: Default::default(), + timeout: Duration::from_millis(30000), + allow_invalid_certs: false, + http_auth: Default::default(), + http_headers: Default::default(), events: Default::default(), events_policy: EventPolicy::Exclude, } @@ -47533,23 +59936,39 @@ impl Default for WebHook { impl IntoValue for WebHook { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(15); + map.insert_unchecked(Property::Enable, self.enable.into_value_with(opts)); + map.insert_unchecked(Property::Level, self.level.into_value_with(opts)); + map.insert_unchecked(Property::Lossy, self.lossy.into_value_with(opts)); + map.insert_unchecked( + Property::SignatureKey, + self.signature_key.into_value_with(opts), + ); + map.insert_unchecked(Property::Throttle, self.throttle.into_value_with(opts)); + map.insert_unchecked( + Property::DiscardAfter, + self.discard_after.into_value_with(opts), + ); + map.insert_unchecked(Property::Url, self.url.into_value_with(opts)); + map.insert_unchecked(Property::Timeout, self.timeout.into_value_with(opts)); map.insert_unchecked( Property::AllowInvalidCerts, - self.allow_invalid_certs.into_value(), + self.allow_invalid_certs.into_value_with(opts), + ); + map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value_with(opts)); + map.insert_unchecked( + Property::HttpHeaders, + self.http_headers.into_value_with(opts), + ); + map.insert_unchecked(Property::Events, self.events.into_value_with(opts)); + map.insert_unchecked( + Property::EventsPolicy, + self.events_policy.into_value_with(opts), ); - map.insert_unchecked(Property::SignatureKey, self.signature_key.into_value()); - map.insert_unchecked(Property::Throttle, self.throttle.into_value()); - map.insert_unchecked(Property::Timeout, self.timeout.into_value()); - map.insert_unchecked(Property::DiscardAfter, self.discard_after.into_value()); - map.insert_unchecked(Property::Url, self.url.into_value()); - map.insert_unchecked(Property::HttpAuth, self.http_auth.into_value()); - map.insert_unchecked(Property::HttpHeaders, self.http_headers.into_value()); - map.insert_unchecked(Property::Enable, self.enable.into_value()); - map.insert_unchecked(Property::Level, self.level.into_value()); - map.insert_unchecked(Property::Lossy, self.lossy.into_value()); - map.insert_unchecked(Property::Events, self.events.into_value()); - map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); JmapValue::Object(map) } } @@ -47561,21 +59980,21 @@ impl RegistryJsonPropertyPatch for WebHook { value: JmapValue<'x>, ) -> PatchResult<'x> { match pointer.next_property() { - Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), + Some(Property::Enable) => self.enable.patch(pointer, value), + Some(Property::Level) => self.level.patch(pointer, value), + Some(Property::Lossy) => self.lossy.patch(pointer, value), Some(Property::SignatureKey) => self.signature_key.patch(pointer, value), Some(Property::Throttle) => self.throttle.patch(pointer, value), - Some(Property::Timeout) => self.timeout.patch(pointer, value), Some(Property::DiscardAfter) => self.discard_after.patch(pointer, value), Some(Property::Url) => self .url .patch(pointer.with_validators(&[StringValidator::Trim]), value), + Some(Property::Timeout) => self.timeout.patch(pointer, value), + Some(Property::AllowInvalidCerts) => self.allow_invalid_certs.patch(pointer, value), Some(Property::HttpAuth) => self.http_auth.patch(pointer, value), Some(Property::HttpHeaders) => self .http_headers .patch(pointer.with_validators(&[StringValidator::Trim]), value), - Some(Property::Enable) => self.enable.patch(pointer, value), - Some(Property::Level) => self.level.patch(pointer, value), - Some(Property::Lossy) => self.lossy.patch(pointer, value), Some(Property::Events) => self.events.patch(pointer, value), Some(Property::EventsPolicy) => self.events_policy.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { @@ -47600,13 +60019,20 @@ impl ZenohCoordinator { impl Pickle for ZenohCoordinator { fn pickle(&self, out: &mut Vec) { + let block = crate::pickle::begin_block(out, 1, 0, 0); self.config.pickle(out); + crate::pickle::end_block(out, block); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { - let mut this = Self::default(); - this.config = Pickle::unpickle(stream)?; - Some(this) + let block = stream.open_block()?; + let field0: String = if block.has(0) { + Pickle::unpickle(stream)? + } else { + Default::default() + }; + stream.close_block(block); + Some(Self { config: field0 }) } } @@ -47620,8 +60046,12 @@ impl Default for ZenohCoordinator { impl IntoValue for ZenohCoordinator { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(3); - map.insert_unchecked(Property::Config, self.config.into_value()); + map.insert_unchecked(Property::Config, self.config.into_value_with(opts)); JmapValue::Object(map) } } diff --git a/crates/registry/src/types/datetime.rs b/crates/registry/src/types/datetime.rs index 5df91134..b1045b41 100644 --- a/crates/registry/src/types/datetime.rs +++ b/crates/registry/src/types/datetime.rs @@ -242,13 +242,12 @@ impl<'de> serde::Deserialize<'de> for UTCDateTime { impl Pickle for UTCDateTime { fn pickle(&self, out: &mut Vec) { - out.extend_from_slice(&self.0.to_be_bytes()); + out.extend_from_slice(&self.0.to_le_bytes()); } fn unpickle(data: &mut PickledStream<'_>) -> Option { - let mut arr = [0u8; 8]; - arr.copy_from_slice(data.read_bytes(8)?); - Some(UTCDateTime(i64::from_be_bytes(arr))) + data.read_array::<8>() + .map(|bytes| UTCDateTime(i64::from_le_bytes(bytes))) } } @@ -290,33 +289,3 @@ impl IntoValue for UTCDateTime { } } -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use crate::types::datetime::UTCDateTime; - - #[test] - fn parse_jmap_date() { - for (input, _) in [ - ("1997-11-21T09:55:06-06:00", "1997-11-21T09:55:06-06:00"), - ("1997-11-21T09:55:06+00:00", "1997-11-21T09:55:06Z"), - ("2021-01-01T09:55:06+02:00", "2021-01-01T09:55:06+02:00"), - ("2004-06-28T23:43:45.000Z", "2004-06-28T23:43:45Z"), - ("1997-11-21T09:55:06.123+00:00", "1997-11-21T09:55:06Z"), - ( - "2021-01-01T09:55:06.4567+02:00", - "2021-01-01T09:55:06+02:00", - ), - ] { - let date = UTCDateTime::from_str(input).unwrap(); - //assert_eq!(date.to_string(), expected_result); - - let timestamp = date.timestamp(); - assert_eq!( - UTCDateTime::from_timestamp(timestamp).timestamp(), - timestamp - ); - } - } -} diff --git a/crates/registry/src/types/error.rs b/crates/registry/src/types/error.rs index 483426e5..ac53e0c3 100644 --- a/crates/registry/src/types/error.rs +++ b/crates/registry/src/types/error.rs @@ -9,6 +9,7 @@ use crate::{ schema::prelude::Property, types::{EnumImpl, id::ObjectId}, }; +use compact_str::ToCompactString; use std::{borrow::Cow, fmt::Display}; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -111,9 +112,9 @@ impl Warning { pub fn log(&self) { trc::event!( Registry(trc::RegistryEvent::BuildWarning), - Source = self.object_id.object().as_str(), + ObjectType = self.object_id.object().as_str(), Id = self.object_id.id().id(), - Key = self.property.map(|key| key.as_str()), + Property = self.property.map(|key| key.as_str()), Reason = self.message.clone(), ); } @@ -125,18 +126,18 @@ impl Error { Error::Validation { object_id, errors } => { trc::event!( Registry(trc::RegistryEvent::ValidationError), - Source = object_id.object().as_str(), + ObjectType = object_id.object().as_str(), Id = object_id.id().id(), Reason = errors .iter() - .map(|err| trc::Value::from(err.to_string())) + .map(|err| trc::Value::from(err.to_compact_string())) .collect::>(), ); } Error::Build { object_id, message } => { trc::event!( Registry(trc::RegistryEvent::BuildError), - Source = object_id.object().as_str(), + ObjectType = object_id.object().as_str(), Id = object_id.id().id(), Reason = message.clone(), ); @@ -144,7 +145,7 @@ impl Error { Error::Internal { object_id, error } => { trc::event!( Registry(trc::RegistryEvent::ReadError), - Source = object_id.as_ref().map(|id| id.object().as_str()), + ObjectType = object_id.as_ref().map(|id| id.object().as_str()), Id = object_id.as_ref().map(|id| id.id().id()), CausedBy = error.clone(), ); @@ -152,7 +153,7 @@ impl Error { Error::NotFound { object_id } => { trc::event!( Registry(trc::RegistryEvent::BuildError), - Source = object_id.object().as_str(), + ObjectType = object_id.object().as_str(), Id = object_id.id().id(), Reason = "Object not found", ); diff --git a/crates/registry/src/types/float.rs b/crates/registry/src/types/float.rs index efa4cdf3..6d8520fd 100644 --- a/crates/registry/src/types/float.rs +++ b/crates/registry/src/types/float.rs @@ -105,11 +105,12 @@ impl From for Float { impl Pickle for Float { fn pickle(&self, out: &mut Vec) { - self.0.to_bits().pickle(out); + out.extend_from_slice(&self.0.to_le_bytes()); } fn unpickle(data: &mut PickledStream<'_>) -> Option { - u64::unpickle(data).map(|bits| Float(f64::from_bits(bits))) + data.read_array::<8>() + .map(|bytes| Float(f64::from_le_bytes(bytes))) } } diff --git a/crates/registry/src/types/id.rs b/crates/registry/src/types/id.rs index 57057cbc..2f23ac1e 100644 --- a/crates/registry/src/types/id.rs +++ b/crates/registry/src/types/id.rs @@ -85,12 +85,8 @@ impl Pickle for BlobId { } fn unpickle(stream: &mut PickledStream<'_>) -> Option { - stream.read_bytes(BLOB_HASH_LEN).map(|bytes| { - BlobId::new( - BlobHash::try_from_hash_slice(bytes).unwrap(), - BlobClass::default(), - ) - }) + let hash = BlobHash::try_from_hash_slice(stream.read_bytes(BLOB_HASH_LEN)?).ok()?; + Some(BlobId::new(hash, BlobClass::default())) } } diff --git a/crates/registry/src/types/index.rs b/crates/registry/src/types/index.rs index a179232a..b9fb279d 100644 --- a/crates/registry/src/types/index.rs +++ b/crates/registry/src/types/index.rs @@ -42,6 +42,7 @@ pub enum IndexValue<'x> { Bytes(Vec), U64(u64), I64(i64), + U32(u32), U16(u16), None, } @@ -161,6 +162,28 @@ impl<'x> IndexBuilder<'x> { } } +impl IndexKey<'_> { + pub fn text_with_nul(&self) -> Option { + match self { + IndexKey::Unique { + property, + value_1, + value_2, + .. + } => (value_1.has_nul() || value_2.has_nul()).then_some(*property), + IndexKey::Search { property, value } => value.has_nul().then_some(*property), + IndexKey::ForeignKey { .. } => None, + } + } +} + +impl IndexValue<'_> { + #[inline] + pub fn has_nul(&self) -> bool { + matches!(self, IndexValue::Text(text) if text.as_bytes().contains(&0)) + } +} + impl IndexSchema { pub const fn new(prop: Property, typ: IndexSchemaType, value: IndexSchemaValueType) -> Self { Self { prop, typ, value } diff --git a/crates/registry/src/types/ipaddr.rs b/crates/registry/src/types/ipaddr.rs index efbf1cfc..343de5f4 100644 --- a/crates/registry/src/types/ipaddr.rs +++ b/crates/registry/src/types/ipaddr.rs @@ -92,18 +92,13 @@ impl Pickle for std::net::IpAddr { } fn unpickle(data: &mut PickledStream<'_>) -> Option { - let kind = data.read()?; - match kind { - 4 => { - let mut arr = [0u8; 4]; - arr.copy_from_slice(data.read_bytes(4)?); - Some(std::net::IpAddr::V4(Ipv4Addr::from(arr))) - } - 6 => { - let mut arr = [0u8; 16]; - arr.copy_from_slice(data.read_bytes(16)?); - Some(std::net::IpAddr::V6(std::net::Ipv6Addr::from(arr))) - } + match data.read()? { + 4 => data + .read_array::<4>() + .map(|octets| std::net::IpAddr::V4(Ipv4Addr::from(octets))), + 6 => data + .read_array::<16>() + .map(|octets| std::net::IpAddr::V6(std::net::Ipv6Addr::from(octets))), _ => None, } } diff --git a/crates/registry/src/types/ipmask.rs b/crates/registry/src/types/ipmask.rs index 68ff6627..97e45d37 100644 --- a/crates/registry/src/types/ipmask.rs +++ b/crates/registry/src/types/ipmask.rs @@ -241,26 +241,14 @@ impl Pickle for IpAddrOrMask { fn unpickle(data: &mut PickledStream<'_>) -> Option { match data.read()? { - 4 => { - let mut addr_arr = [0u8; 4]; - addr_arr.copy_from_slice(data.read_bytes(4)?); - let mut mask_arr = [0u8; 4]; - mask_arr.copy_from_slice(data.read_bytes(4)?); - Some(IpAddrOrMask::V4 { - addr: Ipv4Addr::from(addr_arr), - mask: u32::from_be_bytes(mask_arr), - }) - } - 6 => { - let mut addr_arr = [0u8; 16]; - addr_arr.copy_from_slice(data.read_bytes(16)?); - let mut mask_arr = [0u8; 16]; - mask_arr.copy_from_slice(data.read_bytes(16)?); - Some(IpAddrOrMask::V6 { - addr: Ipv6Addr::from(addr_arr), - mask: u128::from_be_bytes(mask_arr), - }) - } + 4 => Some(IpAddrOrMask::V4 { + addr: Ipv4Addr::from(data.read_array::<4>()?), + mask: u32::from_be_bytes(data.read_array::<4>()?), + }), + 6 => Some(IpAddrOrMask::V6 { + addr: Ipv6Addr::from(data.read_array::<16>()?), + mask: u128::from_be_bytes(data.read_array::<16>()?), + }), _ => None, } } @@ -298,29 +286,3 @@ impl IntoValue for IpAddrOrMask { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ipaddrmask() { - for (mask, ip) in [ - ("10.0.0.0/8", "10.30.20.11"), - ("10.0.0.0/8", "10.0.13.73"), - ("192.168.1.1", "192.168.1.1"), - ] { - let mask = IpAddrOrMask::from_str(mask).unwrap(); - let ip = ip.parse::().unwrap(); - assert!(mask.matches(&ip)); - } - - for (mask, ip) in [ - ("10.0.0.0/8", "11.30.20.11"), - ("192.168.1.1", "193.168.1.1"), - ] { - let mask = IpAddrOrMask::from_str(mask).unwrap(); - let ip = ip.parse::().unwrap(); - assert!(!mask.matches(&ip)); - } - } -} diff --git a/crates/registry/src/types/list.rs b/crates/registry/src/types/list.rs index ff7f289d..9336a393 100644 --- a/crates/registry/src/types/list.rs +++ b/crates/registry/src/types/list.rs @@ -7,6 +7,7 @@ use crate::{ jmap::{ IntoValue, JmapValue, JsonPointerPatch, MaybeUnpatched, PatchResult, RegistryJsonPatch, + ValueOptions, }, pickle::{Pickle, PickledStream}, types::error::PatchError, @@ -72,15 +73,15 @@ where T: Pickle, { fn pickle(&self, out: &mut Vec) { - (self.0.len() as u32).pickle(out); + (self.0.len() as u64).pickle(out); for item in self.0.values() { item.pickle(out); } } fn unpickle(stream: &mut PickledStream<'_>) -> Option { - let len = u32::unpickle(stream)? as usize; - let mut vec = Self::with_capacity(len); + let len = usize::try_from(u64::unpickle(stream)?).ok()?; + let mut vec = Self::with_capacity(len.min(stream.remaining_len())); for _ in 0..len { vec.push_unchecked(T::unpickle(stream)?); } @@ -149,9 +150,13 @@ impl RegistryJsonPatch for List { impl IntoValue for List { fn into_value(self) -> JmapValue<'static> { + self.into_value_with(ValueOptions::default()) + } + + fn into_value_with(self, opts: ValueOptions) -> JmapValue<'static> { let mut map = jmap_tools::Map::with_capacity(self.0.len()); for (idx, v) in self.0 { - map.insert_unchecked(Key::Owned(idx.to_string()), v.into_value()); + map.insert_unchecked(Key::Owned(idx.to_string()), v.into_value_with(opts)); } JmapValue::Object(map) diff --git a/crates/registry/src/types/map.rs b/crates/registry/src/types/map.rs index af953c15..fc8f0d7b 100644 --- a/crates/registry/src/types/map.rs +++ b/crates/registry/src/types/map.rs @@ -101,15 +101,15 @@ where T: Pickle + MapItem, { fn pickle(&self, out: &mut Vec) { - (self.0.len() as u32).pickle(out); + (self.0.len() as u64).pickle(out); for item in &self.0 { item.pickle(out); } } fn unpickle(stream: &mut PickledStream<'_>) -> Option { - let len = u32::unpickle(stream)? as usize; - let mut vec = Vec::with_capacity(len); + let len = usize::try_from(u64::unpickle(stream)?).ok()?; + let mut vec = Vec::with_capacity(len.min(stream.remaining_len())); for _ in 0..len { vec.push(T::unpickle(stream)?); } @@ -418,27 +418,3 @@ impl IntoIterator for Map { } } -#[cfg(test)] -mod tests { - use super::*; - use crate::{jmap::JsonPointerPatch, schema::prelude::Property}; - use jmap_tools::JsonPointer; - - fn patch_member(ptr_str: &str) -> Map { - let mut map = Map::::default(); - let ptr = JsonPointer::::parse(ptr_str); - let pointer = JsonPointerPatch::new(&ptr); - map.patch(pointer, Value::Bool(true)).expect("patch failed"); - map - } - - #[test] - fn patch_map_id_digit_keys() { - for id in [0u64, 28, 29, 70, 861, 954, 957, 30554] { - let id = Id::new(id); - let key = id.as_string(); - let map = patch_member(&key); - assert_eq!(map.0, vec![id], "id {} via key '{key}'", id.id()); - } - } -} diff --git a/crates/registry/src/types/mod.rs b/crates/registry/src/types/mod.rs index ddd8528b..a8ca7ad0 100644 --- a/crates/registry/src/types/mod.rs +++ b/crates/registry/src/types/mod.rs @@ -34,16 +34,18 @@ pub trait EnumImpl: Sized + Debug + PartialEq + Eq { fn to_id(&self) -> u16; } -pub trait ObjectImpl: - Pickle + Serialize + DeserializeOwned + Default + Clone + Send + Sync -{ +pub trait ObjectImpl: Serialize + DeserializeOwned + Default + Clone + Send + Sync { const FLAGS: u64; const OBJECT: ObjectType; const VERSION: u8; fn validate(&self, errors: &mut Vec) -> bool; + fn validate_inline_secrets(&self, _errors: &mut Vec) {} fn index<'x>(&'x self, builder: &mut IndexBuilder<'x>); - fn to_pickled_vec(&self) -> Vec { + fn to_pickled_vec(&self) -> Vec + where + Self: Pickle, + { let mut out = Vec::with_capacity(256); out.push(Self::VERSION); self.pickle(&mut out); diff --git a/crates/registry/src/utils/archived_item.rs b/crates/registry/src/utils/archived_item.rs index 597ff86c..c8be0514 100644 --- a/crates/registry/src/utils/archived_item.rs +++ b/crates/registry/src/utils/archived_item.rs @@ -38,6 +38,16 @@ impl ArchivedItem { } } + pub fn size(&self) -> Option { + match self { + ArchivedItem::Email(i) => Some(i.size), + ArchivedItem::FileNode(_) + | ArchivedItem::CalendarEvent(_) + | ArchivedItem::ContactCard(_) + | ArchivedItem::SieveScript(_) => None, + } + } + pub fn created_at(&self) -> UTCDateTime { match self { ArchivedItem::Email(i) => i.received_at, diff --git a/crates/registry/src/utils/dkim.rs b/crates/registry/src/utils/dkim.rs index d02704a9..78de617e 100644 --- a/crates/registry/src/utils/dkim.rs +++ b/crates/registry/src/utils/dkim.rs @@ -5,18 +5,28 @@ */ use crate::schema::{ - enums::{DkimRotationStage, DkimSignatureType}, + enums::{DkimRotationStage, DkimSignatureType, DnsRecordType}, prelude::{DkimSignature, UTCDateTime}, + structs::DnsManagement, }; use types::id::Id; +impl DnsManagement { + pub fn publishes(&self, record: DnsRecordType) -> bool { + matches!(self, DnsManagement::Automatic(props) if props.publish_records.contains(&record)) + } +} + impl DkimSignature { pub fn rotation_due(&self) -> Option { let (stage, next_transition) = match self { DkimSignature::Dkim1Ed25519Sha256(sign) => (sign.stage, sign.next_transition_at), DkimSignature::Dkim1RsaSha256(sign) => (sign.stage, sign.next_transition_at), DkimSignature::Dkim2Ed25519Sha256(sign) => (sign.stage, sign.next_transition_at), - DkimSignature::Dkim2RsaSha256(sign) => (sign.stage, sign.next_transition_at), + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => { + (sign.stage, sign.next_transition_at) + } }; next_transition.and_then(|next_transition| { if next_transition <= UTCDateTime::now() { @@ -32,7 +42,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => sign.next_transition_at, DkimSignature::Dkim1RsaSha256(sign) => sign.next_transition_at, DkimSignature::Dkim2Ed25519Sha256(sign) => sign.next_transition_at, - DkimSignature::Dkim2RsaSha256(sign) => sign.next_transition_at, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => sign.next_transition_at, } } @@ -45,7 +56,10 @@ impl DkimSignature { DkimSignature::Dkim2Ed25519Sha256(sign) => { sign.next_transition_at = Some(next_transition) } - DkimSignature::Dkim2RsaSha256(sign) => sign.next_transition_at = Some(next_transition), + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => { + sign.next_transition_at = Some(next_transition) + } } } @@ -54,7 +68,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage, DkimSignature::Dkim1RsaSha256(sign) => sign.stage, DkimSignature::Dkim2Ed25519Sha256(sign) => sign.stage, - DkimSignature::Dkim2RsaSha256(sign) => sign.stage, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => sign.stage, } } @@ -63,7 +78,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage = stage, DkimSignature::Dkim1RsaSha256(sign) => sign.stage = stage, DkimSignature::Dkim2Ed25519Sha256(sign) => sign.stage = stage, - DkimSignature::Dkim2RsaSha256(sign) => sign.stage = stage, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => sign.stage = stage, } } @@ -72,7 +88,10 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage == DkimRotationStage::Active, DkimSignature::Dkim1RsaSha256(sign) => sign.stage == DkimRotationStage::Active, DkimSignature::Dkim2Ed25519Sha256(sign) => sign.stage == DkimRotationStage::Active, - DkimSignature::Dkim2RsaSha256(sign) => sign.stage == DkimRotationStage::Active, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => { + sign.stage == DkimRotationStage::Active + } } } @@ -85,7 +104,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => &sign.selector, DkimSignature::Dkim1RsaSha256(sign) => &sign.selector, DkimSignature::Dkim2Ed25519Sha256(sign) => &sign.selector, - DkimSignature::Dkim2RsaSha256(sign) => &sign.selector, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => &sign.selector, } } @@ -94,7 +114,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(sign) => sign.domain_id, DkimSignature::Dkim1RsaSha256(sign) => sign.domain_id, DkimSignature::Dkim2Ed25519Sha256(sign) => sign.domain_id, - DkimSignature::Dkim2RsaSha256(sign) => sign.domain_id, + DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => sign.domain_id, } } } @@ -104,17 +125,27 @@ impl DkimSignatureType { match self { Self::Dkim1Ed25519Sha256 | Self::Dkim2Ed25519Sha256 => "ed25519", Self::Dkim1RsaSha256 | Self::Dkim2RsaSha256 => "rsa", + Self::Dkim2MlDsa44Ed25519Sha512 => "mldsa44-ed25519", } } pub const fn hash(self) -> &'static str { - "sha256" + match self { + Self::Dkim2MlDsa44Ed25519Sha512 => "sha512", + Self::Dkim1Ed25519Sha256 + | Self::Dkim1RsaSha256 + | Self::Dkim2Ed25519Sha256 + | Self::Dkim2RsaSha256 => "sha256", + } } pub const fn version(self) -> &'static str { match self { Self::Dkim1Ed25519Sha256 | Self::Dkim1RsaSha256 => "1", - Self::Dkim2Ed25519Sha256 | Self::Dkim2RsaSha256 => "2", + Self::Dkim2Ed25519Sha256 | Self::Dkim2RsaSha256 | Self::Dkim2MlDsa44Ed25519Sha512 => { + "2" + } } } } + diff --git a/crates/registry/src/utils/mod.rs b/crates/registry/src/utils/mod.rs index ee626644..aeb4d172 100644 --- a/crates/registry/src/utils/mod.rs +++ b/crates/registry/src/utils/mod.rs @@ -8,7 +8,6 @@ use crate::schema::prelude::{DkimSignature, Roles, SecretText}; use types::id::Id; pub mod account; -pub mod archived_item; pub mod cron; pub mod dkim; pub mod http; @@ -32,7 +31,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(signature) => &signature.private_key, DkimSignature::Dkim1RsaSha256(signature) => &signature.private_key, DkimSignature::Dkim2Ed25519Sha256(signature) => &signature.private_key, - DkimSignature::Dkim2RsaSha256(signature) => &signature.private_key, + DkimSignature::Dkim2RsaSha256(signature) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(signature) => &signature.private_key, } } @@ -41,7 +41,8 @@ impl DkimSignature { DkimSignature::Dkim1Ed25519Sha256(signature) => &mut signature.private_key, DkimSignature::Dkim1RsaSha256(signature) => &mut signature.private_key, DkimSignature::Dkim2Ed25519Sha256(signature) => &mut signature.private_key, - DkimSignature::Dkim2RsaSha256(signature) => &mut signature.private_key, + DkimSignature::Dkim2RsaSha256(signature) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(signature) => &mut signature.private_key, } } } diff --git a/crates/registry/src/utils/report.rs b/crates/registry/src/utils/report.rs index 6dca37f6..040c0499 100644 --- a/crates/registry/src/utils/report.rs +++ b/crates/registry/src/utils/report.rs @@ -14,48 +14,49 @@ use crate::{ }; use mail_auth::{ IprevOutput, IprevResult, SpfOutput, - report::{tlsrpt::*, *}, + dmarc::{Alignment, Policy}, }; +use smtp_report::{arf::*, dmarc::*, tlsrpt::*}; use std::borrow::Cow; -impl From for Alignment { +impl From for Option { fn from(value: enums::DmarcAlignment) -> Self { match value { - enums::DmarcAlignment::Relaxed => Alignment::Relaxed, - enums::DmarcAlignment::Strict => Alignment::Strict, - enums::DmarcAlignment::Unspecified => Alignment::Unspecified, + enums::DmarcAlignment::Relaxed => Some(Alignment::Relaxed), + enums::DmarcAlignment::Strict => Some(Alignment::Strict), + enums::DmarcAlignment::Unspecified => None, } } } -impl From for enums::DmarcAlignment { - fn from(value: Alignment) -> Self { +impl From> for enums::DmarcAlignment { + fn from(value: Option) -> Self { match value { - Alignment::Relaxed => enums::DmarcAlignment::Relaxed, - Alignment::Strict => enums::DmarcAlignment::Strict, - Alignment::Unspecified => enums::DmarcAlignment::Unspecified, + Some(Alignment::Relaxed) => enums::DmarcAlignment::Relaxed, + Some(Alignment::Strict) => enums::DmarcAlignment::Strict, + None => enums::DmarcAlignment::Unspecified, } } } -impl From for Disposition { +impl From for Policy { fn from(value: enums::DmarcDisposition) -> Self { match value { - enums::DmarcDisposition::None => Disposition::None, - enums::DmarcDisposition::Quarantine => Disposition::Quarantine, - enums::DmarcDisposition::Reject => Disposition::Reject, - enums::DmarcDisposition::Unspecified => Disposition::Unspecified, + enums::DmarcDisposition::None => Policy::None, + enums::DmarcDisposition::Quarantine => Policy::Quarantine, + enums::DmarcDisposition::Reject => Policy::Reject, + enums::DmarcDisposition::Unspecified => Policy::Unspecified, } } } -impl From for enums::DmarcDisposition { - fn from(value: Disposition) -> Self { +impl From for enums::DmarcDisposition { + fn from(value: Policy) -> Self { match value { - Disposition::None => enums::DmarcDisposition::None, - Disposition::Quarantine => enums::DmarcDisposition::Quarantine, - Disposition::Reject => enums::DmarcDisposition::Reject, - Disposition::Unspecified => enums::DmarcDisposition::Unspecified, + Policy::None => enums::DmarcDisposition::None, + Policy::Quarantine => enums::DmarcDisposition::Quarantine, + Policy::Reject => enums::DmarcDisposition::Reject, + Policy::Unspecified => enums::DmarcDisposition::Unspecified, } } } @@ -80,46 +81,46 @@ impl From for enums::DmarcDiscovery { } } -impl From for ActionDisposition { +impl From for Disposition { fn from(value: enums::DmarcActionDisposition) -> Self { match value { - enums::DmarcActionDisposition::None => ActionDisposition::None, - enums::DmarcActionDisposition::Pass => ActionDisposition::Pass, - enums::DmarcActionDisposition::Quarantine => ActionDisposition::Quarantine, - enums::DmarcActionDisposition::Reject => ActionDisposition::Reject, - enums::DmarcActionDisposition::Unspecified => ActionDisposition::Unspecified, + enums::DmarcActionDisposition::None => Disposition::None, + enums::DmarcActionDisposition::Pass => Disposition::Pass, + enums::DmarcActionDisposition::Quarantine => Disposition::Quarantine, + enums::DmarcActionDisposition::Reject => Disposition::Reject, + enums::DmarcActionDisposition::Unspecified => Disposition::Unspecified, } } } -impl From for enums::DmarcActionDisposition { - fn from(value: ActionDisposition) -> Self { +impl From for enums::DmarcActionDisposition { + fn from(value: Disposition) -> Self { match value { - ActionDisposition::None => enums::DmarcActionDisposition::None, - ActionDisposition::Pass => enums::DmarcActionDisposition::Pass, - ActionDisposition::Quarantine => enums::DmarcActionDisposition::Quarantine, - ActionDisposition::Reject => enums::DmarcActionDisposition::Reject, - ActionDisposition::Unspecified => enums::DmarcActionDisposition::Unspecified, + Disposition::None => enums::DmarcActionDisposition::None, + Disposition::Pass => enums::DmarcActionDisposition::Pass, + Disposition::Quarantine => enums::DmarcActionDisposition::Quarantine, + Disposition::Reject => enums::DmarcActionDisposition::Reject, + Disposition::Unspecified => enums::DmarcActionDisposition::Unspecified, } } } -impl From for DmarcResult { +impl From for DmarcStatus { fn from(value: enums::DmarcResult) -> Self { match value { - enums::DmarcResult::Pass => DmarcResult::Pass, - enums::DmarcResult::Fail => DmarcResult::Fail, - enums::DmarcResult::Unspecified => DmarcResult::Unspecified, + enums::DmarcResult::Pass => DmarcStatus::Pass, + enums::DmarcResult::Fail => DmarcStatus::Fail, + enums::DmarcResult::Unspecified => DmarcStatus::Unspecified, } } } -impl From for enums::DmarcResult { - fn from(value: DmarcResult) -> Self { +impl From for enums::DmarcResult { + fn from(value: DmarcStatus) -> Self { match value { - DmarcResult::Pass => enums::DmarcResult::Pass, - DmarcResult::Fail => enums::DmarcResult::Fail, - DmarcResult::Unspecified => enums::DmarcResult::Unspecified, + DmarcStatus::Pass => enums::DmarcResult::Pass, + DmarcStatus::Fail => enums::DmarcResult::Fail, + DmarcStatus::Unspecified => enums::DmarcResult::Unspecified, } } } @@ -152,78 +153,78 @@ impl From for enums::DmarcPolicyOverride { } } -impl From for DkimResult { +impl From for DkimStatus { fn from(value: enums::DkimAuthResult) -> Self { match value { - enums::DkimAuthResult::None => DkimResult::None, - enums::DkimAuthResult::Pass => DkimResult::Pass, - enums::DkimAuthResult::Fail => DkimResult::Fail, - enums::DkimAuthResult::Policy => DkimResult::Policy, - enums::DkimAuthResult::Neutral => DkimResult::Neutral, - enums::DkimAuthResult::TempError => DkimResult::TempError, - enums::DkimAuthResult::PermError => DkimResult::PermError, + enums::DkimAuthResult::None => DkimStatus::None, + enums::DkimAuthResult::Pass => DkimStatus::Pass, + enums::DkimAuthResult::Fail => DkimStatus::Fail, + enums::DkimAuthResult::Policy => DkimStatus::Policy, + enums::DkimAuthResult::Neutral => DkimStatus::Neutral, + enums::DkimAuthResult::TempError => DkimStatus::TempError, + enums::DkimAuthResult::PermError => DkimStatus::PermError, } } } -impl From for enums::DkimAuthResult { - fn from(value: DkimResult) -> Self { +impl From for enums::DkimAuthResult { + fn from(value: DkimStatus) -> Self { match value { - DkimResult::None => enums::DkimAuthResult::None, - DkimResult::Pass => enums::DkimAuthResult::Pass, - DkimResult::Fail => enums::DkimAuthResult::Fail, - DkimResult::Policy => enums::DkimAuthResult::Policy, - DkimResult::Neutral => enums::DkimAuthResult::Neutral, - DkimResult::TempError => enums::DkimAuthResult::TempError, - DkimResult::PermError => enums::DkimAuthResult::PermError, + DkimStatus::None => enums::DkimAuthResult::None, + DkimStatus::Pass => enums::DkimAuthResult::Pass, + DkimStatus::Fail => enums::DkimAuthResult::Fail, + DkimStatus::Policy => enums::DkimAuthResult::Policy, + DkimStatus::Neutral => enums::DkimAuthResult::Neutral, + DkimStatus::TempError => enums::DkimAuthResult::TempError, + DkimStatus::PermError => enums::DkimAuthResult::PermError, } } } -impl From for SpfResult { +impl From for SpfStatus { fn from(value: enums::SpfAuthResult) -> Self { match value { - enums::SpfAuthResult::None => SpfResult::None, - enums::SpfAuthResult::Neutral => SpfResult::Neutral, - enums::SpfAuthResult::Pass => SpfResult::Pass, - enums::SpfAuthResult::Fail => SpfResult::Fail, - enums::SpfAuthResult::SoftFail => SpfResult::SoftFail, - enums::SpfAuthResult::TempError => SpfResult::TempError, - enums::SpfAuthResult::PermError => SpfResult::PermError, + enums::SpfAuthResult::None => SpfStatus::None, + enums::SpfAuthResult::Neutral => SpfStatus::Neutral, + enums::SpfAuthResult::Pass => SpfStatus::Pass, + enums::SpfAuthResult::Fail => SpfStatus::Fail, + enums::SpfAuthResult::SoftFail => SpfStatus::SoftFail, + enums::SpfAuthResult::TempError => SpfStatus::TempError, + enums::SpfAuthResult::PermError => SpfStatus::PermError, } } } -impl From for enums::SpfAuthResult { - fn from(value: SpfResult) -> Self { +impl From for enums::SpfAuthResult { + fn from(value: SpfStatus) -> Self { match value { - SpfResult::None => enums::SpfAuthResult::None, - SpfResult::Neutral => enums::SpfAuthResult::Neutral, - SpfResult::Pass => enums::SpfAuthResult::Pass, - SpfResult::Fail => enums::SpfAuthResult::Fail, - SpfResult::SoftFail => enums::SpfAuthResult::SoftFail, - SpfResult::TempError => enums::SpfAuthResult::TempError, - SpfResult::PermError => enums::SpfAuthResult::PermError, + SpfStatus::None => enums::SpfAuthResult::None, + SpfStatus::Neutral => enums::SpfAuthResult::Neutral, + SpfStatus::Pass => enums::SpfAuthResult::Pass, + SpfStatus::Fail => enums::SpfAuthResult::Fail, + SpfStatus::SoftFail => enums::SpfAuthResult::SoftFail, + SpfStatus::TempError => enums::SpfAuthResult::TempError, + SpfStatus::PermError => enums::SpfAuthResult::PermError, } } } -impl From for SPFDomainScope { +impl From for SpfScope { fn from(value: enums::SpfDomainScope) -> Self { match value { - enums::SpfDomainScope::Helo => SPFDomainScope::Helo, - enums::SpfDomainScope::MailFrom => SPFDomainScope::MailFrom, - enums::SpfDomainScope::Unspecified => SPFDomainScope::Unspecified, + enums::SpfDomainScope::Helo => SpfScope::Helo, + enums::SpfDomainScope::MailFrom => SpfScope::MailFrom, + enums::SpfDomainScope::Unspecified => SpfScope::Unspecified, } } } -impl From for enums::SpfDomainScope { - fn from(value: SPFDomainScope) -> Self { +impl From for enums::SpfDomainScope { + fn from(value: SpfScope) -> Self { match value { - SPFDomainScope::Helo => enums::SpfDomainScope::Helo, - SPFDomainScope::MailFrom => enums::SpfDomainScope::MailFrom, - SPFDomainScope::Unspecified => enums::SpfDomainScope::Unspecified, + SpfScope::Helo => enums::SpfDomainScope::Helo, + SpfScope::MailFrom => enums::SpfDomainScope::MailFrom, + SpfScope::Unspecified => enums::SpfDomainScope::Unspecified, } } } @@ -231,7 +232,7 @@ impl From for enums::SpfDomainScope { impl From for PolicyOverrideReason { fn from(value: structs::DmarcPolicyOverrideReason) -> Self { PolicyOverrideReason { - type_: value.override_type.into(), + kind: value.override_type.into(), comment: value.comment, } } @@ -240,15 +241,15 @@ impl From for PolicyOverrideReason { impl From for structs::DmarcPolicyOverrideReason { fn from(value: PolicyOverrideReason) -> Self { structs::DmarcPolicyOverrideReason { - override_type: value.type_.into(), + override_type: value.kind.into(), comment: value.comment, } } } -impl From for DKIMAuthResult { +impl From for DkimAuthResult { fn from(value: structs::DmarcDkimResult) -> Self { - DKIMAuthResult { + DkimAuthResult { domain: value.domain, selector: value.selector, result: value.result.into(), @@ -257,8 +258,8 @@ impl From for DKIMAuthResult { } } -impl From for structs::DmarcDkimResult { - fn from(value: DKIMAuthResult) -> Self { +impl From for structs::DmarcDkimResult { + fn from(value: DkimAuthResult) -> Self { structs::DmarcDkimResult { domain: value.domain, selector: value.selector, @@ -268,9 +269,9 @@ impl From for structs::DmarcDkimResult { } } -impl From for SPFAuthResult { +impl From for SpfAuthResult { fn from(value: structs::DmarcSpfResult) -> Self { - SPFAuthResult { + SpfAuthResult { domain: value.domain, scope: value.scope.into(), result: value.result.into(), @@ -279,8 +280,8 @@ impl From for SPFAuthResult { } } -impl From for structs::DmarcSpfResult { - fn from(value: SPFAuthResult) -> Self { +impl From for structs::DmarcSpfResult { + fn from(value: SpfAuthResult) -> Self { structs::DmarcSpfResult { domain: value.domain, scope: value.scope.into(), @@ -325,12 +326,12 @@ impl From for Record { .collect(), }, }, - identifiers: Identifier { + identifiers: Identifiers { envelope_to: value.envelope_to, envelope_from: value.envelope_from, header_from: value.header_from, }, - auth_results: AuthResult { + auth_results: AuthResults { dkim: value.dkim_results.into_iter().map(Into::into).collect(), spf: value.spf_results.into_iter().map(Into::into).collect(), }, @@ -365,20 +366,20 @@ impl From for structs::DmarcReportRecord { } } -impl From for Report { +impl From for AggregateReport { fn from(value: structs::DmarcReport) -> Self { - Report { - version: value.version.into_inner() as f32, + AggregateReport { + version: (value.version.into_inner() == 1.0).then_some(ReportVersion::V1), report_metadata: ReportMetadata { org_name: value.org_name, email: value.email, extra_contact_info: value.extra_contact_info, report_id: value.report_id, - date_range: mail_auth::report::DateRange { - begin: value.date_range_begin.timestamp() as u64, + date_range: smtp_report::dmarc::DateRange { + begin: value.date_range_start.timestamp() as u64, end: value.date_range_end.timestamp() as u64, }, - error: value.errors.into_inner(), + errors: value.errors.into_inner(), generator: value.generator, }, policy_published: PolicyPublished { @@ -395,24 +396,24 @@ impl From for Report { value.policy_failure_reporting_options.as_slice(), ), }, - record: value.records.into_iter().map(Into::into).collect(), + records: value.records.into_iter().map(Into::into).collect(), extensions: value.extensions.into_iter().map(Into::into).collect(), } } } -impl From for structs::DmarcReport { - fn from(value: Report) -> Self { +impl From for structs::DmarcReport { + fn from(value: AggregateReport) -> Self { structs::DmarcReport { - version: (value.version as f64).into(), - date_range_begin: UTCDateTime::from_timestamp( + version: value.version.map_or(0.0, |_| 1.0).into(), + date_range_start: UTCDateTime::from_timestamp( value.report_metadata.date_range.begin as i64, ), date_range_end: UTCDateTime::from_timestamp( value.report_metadata.date_range.end as i64, ), email: value.report_metadata.email, - errors: value.report_metadata.error.into(), + errors: value.report_metadata.errors.into(), extensions: List::from_iter(value.extensions.into_iter().map(Into::into)), extra_contact_info: value.report_metadata.extra_contact_info, generator: value.report_metadata.generator, @@ -432,8 +433,8 @@ impl From for structs::DmarcReport { policy_version: value .policy_published .version_published - .map(|v| v.to_string()), - records: List::from_iter(value.record.into_iter().map(Into::into)), + .map(|v| v.as_str().to_string()), + records: List::from_iter(value.records.into_iter().map(Into::into)), report_id: value.report_metadata.report_id, } } @@ -543,9 +544,9 @@ impl From for enums::ArfIdentityAlignment { } } -impl From for Feedback<'static> { +impl From for FeedbackReport<'static> { fn from(value: structs::ArfFeedbackReport) -> Self { - Feedback { + FeedbackReport { feedback_type: value.feedback_type.into(), arrival_date: value.arrival_date.map(|d| d.timestamp()), authentication_results: value @@ -558,13 +559,13 @@ impl From for Feedback<'static> { original_envelope_id: value.original_envelope_id.map(Cow::Owned), original_mail_from: value.original_mail_from.map(Cow::Owned), original_rcpt_to: value.original_rcpt_to.map(Cow::Owned), - reported_domain: value + reported_domains: value .reported_domains .into_inner() .into_iter() .map(Cow::Owned) .collect(), - reported_uri: value + reported_uris: value .reported_uris .into_inner() .into_iter() @@ -574,7 +575,10 @@ impl From for Feedback<'static> { source_ip: value.source_ip.map(|ip| ip.into_inner()), user_agent: value.user_agent.map(Cow::Owned), version: value.version as u32, - source_port: value.source_port.unwrap_or(0) as u32, + source_port: value + .source_port + .and_then(|port| u16::try_from(port).ok()) + .unwrap_or(0), auth_failure: value.auth_failure.into(), delivery_result: value.delivery_result.into(), dkim_adsp_dns: value.dkim_adsp_dns.map(Cow::Owned), @@ -592,8 +596,8 @@ impl From for Feedback<'static> { } } -impl From> for structs::ArfFeedbackReport { - fn from(value: Feedback<'_>) -> Self { +impl From> for structs::ArfFeedbackReport { + fn from(value: FeedbackReport<'_>) -> Self { let port = value.source_port; structs::ArfFeedbackReport { arrival_date: value.arrival_date.map(UTCDateTime::from_timestamp), @@ -621,24 +625,20 @@ impl From> for structs::ArfFeedbackReport { original_mail_from: value.original_mail_from.map(|s| s.into_owned()), original_rcpt_to: value.original_rcpt_to.map(|s| s.into_owned()), reported_domains: value - .reported_domain + .reported_domains .into_iter() .map(|s| s.into_owned()) .collect::>() .into(), reported_uris: value - .reported_uri + .reported_uris .into_iter() .map(|s| s.into_owned()) .collect::>() .into(), reporting_mta: value.reporting_mta.map(|s| s.into_owned()), source_ip: value.source_ip.map(IpAddr), - source_port: if port == 0 || port > 65535 { - None - } else { - Some(port as u64) - }, + source_port: (port != 0).then_some(port as u64), spf_dns: value.spf_dns.map(|s| s.into_owned()), user_agent: value.user_agent.map(|s| s.into_owned()), version: value.version as u64, @@ -668,40 +668,40 @@ impl From for enums::TlsPolicyType { } } -impl From for ResultType { +impl From for FailureType { fn from(value: enums::TlsResultType) -> Self { match value { - enums::TlsResultType::StartTlsNotSupported => ResultType::StartTlsNotSupported, - enums::TlsResultType::CertificateHostMismatch => ResultType::CertificateHostMismatch, - enums::TlsResultType::CertificateExpired => ResultType::CertificateExpired, - enums::TlsResultType::CertificateNotTrusted => ResultType::CertificateNotTrusted, - enums::TlsResultType::ValidationFailure => ResultType::ValidationFailure, - enums::TlsResultType::TlsaInvalid => ResultType::TlsaInvalid, - enums::TlsResultType::DnssecInvalid => ResultType::DnssecInvalid, - enums::TlsResultType::DaneRequired => ResultType::DaneRequired, - enums::TlsResultType::StsPolicyFetchError => ResultType::StsPolicyFetchError, - enums::TlsResultType::StsPolicyInvalid => ResultType::StsPolicyInvalid, - enums::TlsResultType::StsWebpkiInvalid => ResultType::StsWebpkiInvalid, - enums::TlsResultType::Other => ResultType::Other, + enums::TlsResultType::StartTlsNotSupported => FailureType::StartTlsNotSupported, + enums::TlsResultType::CertificateHostMismatch => FailureType::CertificateHostMismatch, + enums::TlsResultType::CertificateExpired => FailureType::CertificateExpired, + enums::TlsResultType::CertificateNotTrusted => FailureType::CertificateNotTrusted, + enums::TlsResultType::ValidationFailure => FailureType::ValidationFailure, + enums::TlsResultType::TlsaInvalid => FailureType::TlsaInvalid, + enums::TlsResultType::DnssecInvalid => FailureType::DnssecInvalid, + enums::TlsResultType::DaneRequired => FailureType::DaneRequired, + enums::TlsResultType::StsPolicyFetchError => FailureType::StsPolicyFetchError, + enums::TlsResultType::StsPolicyInvalid => FailureType::StsPolicyInvalid, + enums::TlsResultType::StsWebpkiInvalid => FailureType::StsWebpkiInvalid, + enums::TlsResultType::Other => FailureType::Other, } } } -impl From for enums::TlsResultType { - fn from(value: ResultType) -> Self { +impl From for enums::TlsResultType { + fn from(value: FailureType) -> Self { match value { - ResultType::StartTlsNotSupported => enums::TlsResultType::StartTlsNotSupported, - ResultType::CertificateHostMismatch => enums::TlsResultType::CertificateHostMismatch, - ResultType::CertificateExpired => enums::TlsResultType::CertificateExpired, - ResultType::CertificateNotTrusted => enums::TlsResultType::CertificateNotTrusted, - ResultType::ValidationFailure => enums::TlsResultType::ValidationFailure, - ResultType::TlsaInvalid => enums::TlsResultType::TlsaInvalid, - ResultType::DnssecInvalid => enums::TlsResultType::DnssecInvalid, - ResultType::DaneRequired => enums::TlsResultType::DaneRequired, - ResultType::StsPolicyFetchError => enums::TlsResultType::StsPolicyFetchError, - ResultType::StsPolicyInvalid => enums::TlsResultType::StsPolicyInvalid, - ResultType::StsWebpkiInvalid => enums::TlsResultType::StsWebpkiInvalid, - ResultType::Other => enums::TlsResultType::Other, + FailureType::StartTlsNotSupported => enums::TlsResultType::StartTlsNotSupported, + FailureType::CertificateHostMismatch => enums::TlsResultType::CertificateHostMismatch, + FailureType::CertificateExpired => enums::TlsResultType::CertificateExpired, + FailureType::CertificateNotTrusted => enums::TlsResultType::CertificateNotTrusted, + FailureType::ValidationFailure => enums::TlsResultType::ValidationFailure, + FailureType::TlsaInvalid => enums::TlsResultType::TlsaInvalid, + FailureType::DnssecInvalid => enums::TlsResultType::DnssecInvalid, + FailureType::DaneRequired => enums::TlsResultType::DaneRequired, + FailureType::StsPolicyFetchError => enums::TlsResultType::StsPolicyFetchError, + FailureType::StsPolicyInvalid => enums::TlsResultType::StsPolicyInvalid, + FailureType::StsWebpkiInvalid => enums::TlsResultType::StsWebpkiInvalid, + FailureType::Other => enums::TlsResultType::Other, } } } @@ -736,9 +736,9 @@ impl From for structs::TlsFailureDetails { } } -impl From for Policy { +impl From for PolicyResult { fn from(value: structs::TlsReportPolicy) -> Self { - Policy { + PolicyResult { policy: PolicyDetails { policy_type: value.policy_type.into(), policy_string: value.policy_strings.into_inner(), @@ -746,23 +746,23 @@ impl From for Policy { mx_host: value.mx_hosts.into_inner(), }, summary: Summary { - total_success: value.total_successful_sessions as u32, - total_failure: value.total_failed_sessions as u32, + successful_sessions: value.total_successful_sessions as u32, + failed_sessions: value.total_failed_sessions as u32, }, failure_details: value.failure_details.into_iter().map(Into::into).collect(), } } } -impl From for structs::TlsReportPolicy { - fn from(value: Policy) -> Self { +impl From for structs::TlsReportPolicy { + fn from(value: PolicyResult) -> Self { structs::TlsReportPolicy { policy_type: value.policy.policy_type.into(), policy_strings: value.policy.policy_string.into(), policy_domain: value.policy.policy_domain, mx_hosts: value.policy.mx_host.into(), - total_successful_sessions: value.summary.total_success as u64, - total_failed_sessions: value.summary.total_failure as u64, + total_successful_sessions: value.summary.successful_sessions as u64, + total_failed_sessions: value.summary.failed_sessions as u64, failure_details: List::from_iter(value.failure_details.into_iter().map(Into::into)), } } @@ -772,7 +772,7 @@ impl From for TlsReport { fn from(value: structs::TlsReport) -> Self { TlsReport { organization_name: value.organization_name, - date_range: mail_auth::report::tlsrpt::DateRange::from_timestamps( + date_range: smtp_report::tlsrpt::DateRange::from_timestamps( value.date_range_start.timestamp(), value.date_range_end.timestamp(), ), @@ -973,14 +973,9 @@ impl From<&mail_auth::DmarcResult> for structs::DmarcTroubleshootAuthResult { } } -impl From<&mail_auth::dmarc::Policy> for enums::DmarcDisposition { - fn from(value: &mail_auth::dmarc::Policy) -> Self { - match value { - mail_auth::dmarc::Policy::None => enums::DmarcDisposition::None, - mail_auth::dmarc::Policy::Quarantine => enums::DmarcDisposition::Quarantine, - mail_auth::dmarc::Policy::Reject => enums::DmarcDisposition::Reject, - mail_auth::dmarc::Policy::Unspecified => enums::DmarcDisposition::Unspecified, - } +impl From<&Policy> for enums::DmarcDisposition { + fn from(value: &Policy) -> Self { + (*value).into() } } diff --git a/crates/registry/src/utils/secret.rs b/crates/registry/src/utils/secret.rs index 69563054..d6105f01 100644 --- a/crates/registry/src/utils/secret.rs +++ b/crates/registry/src/utils/secret.rs @@ -150,3 +150,4 @@ impl SecretKeyEnvironmentVariable { } } } + diff --git a/crates/registry/src/utils/task.rs b/crates/registry/src/utils/task.rs index 97cbd7ca..a7e907fc 100644 --- a/crates/registry/src/utils/task.rs +++ b/crates/registry/src/utils/task.rs @@ -4,24 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::schema::{ - enums::Permission, - prelude::{Action, Task, TaskStatus, TaskStatusPending, UTCDateTime}, +use crate::{ + schema::{ + enums::{ + IndexType, Permission, SharedObjectType, TaskSpamFilterMaintenanceType, + TaskStoreMaintenanceType, + }, + prelude::{Action, Task, TaskStatus, TaskStatusPending, UTCDateTime}, + }, + types::{EnumImpl, index::IndexValue}, }; +use types::{collection::Collection, id::Id}; impl Task { pub fn set_status(&mut self, status: TaskStatus) { match self { - Task::IndexDocument(task) => task.status = status, - Task::UnindexDocument(task) => task.status = status, - Task::IndexTrace(task) => task.status = status, Task::CalendarAlarmEmail(task) => task.status = status, Task::CalendarAlarmNotification(task) => task.status = status, Task::CalendarItipMessage(task) => task.status = status, Task::MergeThreads(task) => task.status = status, Task::DmarcReport(task) => task.status = status, Task::TlsReport(task) => task.status = status, - Task::RestoreArchivedItem(task) => task.status = status, + Task::DmarcReportWindow(task) | Task::TlsReportWindow(task) => task.status = status, Task::DestroyAccount(task) => task.status = status, Task::AccountMaintenance(task) => task.status = status, Task::StoreMaintenance(task) => task.status = status, @@ -29,22 +33,19 @@ impl Task { Task::AcmeRenewal(task) => task.status = status, Task::DkimManagement(task) => task.status = status, Task::DnsManagement(task) => task.status = status, - Task::TenantMaintenance(task) => task.status = status, + Task::ShareRevokeCleanup(task) => task.status = status, } } pub fn status(&self) -> &TaskStatus { match self { - Task::IndexDocument(task) => &task.status, - Task::UnindexDocument(task) => &task.status, - Task::IndexTrace(task) => &task.status, Task::CalendarAlarmEmail(task) => &task.status, Task::CalendarAlarmNotification(task) => &task.status, Task::CalendarItipMessage(task) => &task.status, Task::MergeThreads(task) => &task.status, Task::DmarcReport(task) => &task.status, Task::TlsReport(task) => &task.status, - Task::RestoreArchivedItem(task) => &task.status, + Task::DmarcReportWindow(task) | Task::TlsReportWindow(task) => &task.status, Task::DestroyAccount(task) => &task.status, Task::AccountMaintenance(task) => &task.status, Task::StoreMaintenance(task) => &task.status, @@ -52,14 +53,14 @@ impl Task { Task::AcmeRenewal(task) => &task.status, Task::DkimManagement(task) => &task.status, Task::DnsManagement(task) => &task.status, - Task::TenantMaintenance(task) => &task.status, + Task::ShareRevokeCleanup(task) => &task.status, } } pub fn attempt_number(&self) -> u64 { match self.status() { TaskStatus::Pending(_) => 0, - TaskStatus::Retry(status) => status.attempt_number, + TaskStatus::Retry(status) | TaskStatus::Stalled(status) => status.attempt_number, TaskStatus::Failed(status) => status.failed_attempt_number, } } @@ -67,23 +68,23 @@ impl Task { pub fn due_timestamp(&self) -> u64 { match self.status() { TaskStatus::Pending(status) => status.due.timestamp() as u64, - TaskStatus::Retry(status) => status.due.timestamp() as u64, + TaskStatus::Retry(status) | TaskStatus::Stalled(status) => { + status.due.timestamp() as u64 + } TaskStatus::Failed(_) => u64::MAX, } } pub fn permission(&self) -> Permission { match self { - Task::IndexDocument(_) => Permission::TaskIndexDocument, - Task::UnindexDocument(_) => Permission::TaskUnindexDocument, - Task::IndexTrace(_) => Permission::TaskIndexTrace, Task::CalendarAlarmEmail(_) => Permission::TaskCalendarAlarmEmail, Task::CalendarAlarmNotification(_) => Permission::TaskCalendarAlarmNotification, Task::CalendarItipMessage(_) => Permission::TaskCalendarItipMessage, Task::MergeThreads(_) => Permission::TaskMergeThreads, Task::DmarcReport(_) => Permission::TaskDmarcReport, Task::TlsReport(_) => Permission::TaskTlsReport, - Task::RestoreArchivedItem(_) => Permission::TaskRestoreArchivedItem, + Task::DmarcReportWindow(_) => Permission::TaskDmarcReportWindow, + Task::TlsReportWindow(_) => Permission::TaskTlsReportWindow, Task::DestroyAccount(_) => Permission::TaskDestroyAccount, Task::AccountMaintenance(_) => Permission::TaskAccountMaintenance, Task::StoreMaintenance(_) => Permission::TaskStoreMaintenance, @@ -91,7 +92,62 @@ impl Task { Task::AcmeRenewal(_) => Permission::TaskAcmeRenewal, Task::DkimManagement(_) => Permission::TaskDkimManagement, Task::DnsManagement(_) => Permission::TaskDnsManagement, - Task::TenantMaintenance(_) => Permission::TaskTenantMaintenance, + Task::ShareRevokeCleanup(_) => Permission::TaskShareRevokeCleanup, + } + } + + pub fn singleton_key(&self) -> Option<(u16, u64)> { + match self { + Task::AcmeRenewal(task) | Task::DkimManagement(task) => Some((0, task.domain_id.id())), + Task::DmarcReportWindow(task) | Task::TlsReportWindow(task) => { + Some((0, task.window_end.timestamp().cast_unsigned())) + } + Task::AccountMaintenance(task) => { + Some((task.maintenance_type.to_id(), task.account_id.id())) + } + Task::StoreMaintenance(task) => match (task.maintenance_type, task.shard_index) { + (TaskStoreMaintenanceType::PurgeBlob, Some(shard_index)) => { + Some((task.maintenance_type.to_id(), shard_index)) + } + ( + TaskStoreMaintenanceType::PurgeAccounts + | TaskStoreMaintenanceType::PurgeData + | TaskStoreMaintenanceType::PurgeBlob + | TaskStoreMaintenanceType::ReindexAccounts + | TaskStoreMaintenanceType::ResetUserQuotas, + _, + ) => Some((task.maintenance_type.to_id(), u64::MAX)), + _ => None, + }, + Task::SpamFilterMaintenance(task) => match task.maintenance_type { + TaskSpamFilterMaintenanceType::Train + | TaskSpamFilterMaintenanceType::Retrain + | TaskSpamFilterMaintenanceType::UpdateRules => { + Some((task.maintenance_type.to_id(), u64::MAX)) + } + _ => None, + }, + _ => None, + } + } + + pub fn set_document_id(&mut self, document_id: Id) { + match self { + Task::CalendarAlarmEmail(task) => task.document_id = document_id, + Task::CalendarAlarmNotification(task) => task.document_id = document_id, + Task::CalendarItipMessage(task) => task.document_id = document_id, + _ => unreachable!("task type does not carry a document id"), + } + } + + pub fn size_hint(&self) -> usize { + match self { + Task::CalendarItipMessage(task) => task + .messages + .iter() + .map(|message| message.i_calendar_data.len() + message.summary.len()) + .sum(), + _ => 64, } } } @@ -99,7 +155,7 @@ impl Task { impl Action { pub fn permission(&self) -> Permission { match self { - Action::ReloadSettings => Permission::ActionReloadSettings, + Action::ReloadSettings(_) => Permission::ActionReloadSettings, Action::ReloadTlsCertificates => Permission::ActionReloadTlsCertificates, Action::ReloadLookupStores => Permission::ActionReloadLookupStores, Action::ReloadBlockedIps => Permission::ActionReloadBlockedIps, @@ -115,6 +171,14 @@ impl Action { } impl TaskStatus { + pub fn created_at(&self) -> UTCDateTime { + match self { + TaskStatus::Pending(status) => status.created_at, + TaskStatus::Retry(status) | TaskStatus::Stalled(status) => status.created_at, + TaskStatus::Failed(status) => status.created_at, + } + } + pub fn now() -> Self { let now = UTCDateTime::now(); TaskStatus::Pending(TaskStatusPending { @@ -130,3 +194,31 @@ impl TaskStatus { }) } } + +impl SharedObjectType { + pub fn from_collection(collection: Collection) -> Option { + match collection { + Collection::Mailbox => Some(SharedObjectType::Mailbox), + Collection::Calendar => Some(SharedObjectType::Calendar), + Collection::AddressBook => Some(SharedObjectType::AddressBook), + Collection::FileNode => Some(SharedObjectType::FileNode), + _ => None, + } + } + + pub fn collection(self) -> Collection { + match self { + SharedObjectType::Mailbox => Collection::Mailbox, + SharedObjectType::Calendar => Collection::Calendar, + SharedObjectType::AddressBook => Collection::AddressBook, + SharedObjectType::FileNode => Collection::FileNode, + } + } +} + +impl From<&IndexType> for IndexValue<'_> { + fn from(value: &IndexType) -> Self { + IndexValue::U16(value.to_id()) + } +} + diff --git a/crates/s3/Cargo.toml b/crates/s3/Cargo.toml new file mode 100644 index 00000000..802f78cb --- /dev/null +++ b/crates/s3/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "s3" +version = "1.0.0" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[dependencies] +reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "stream"] } +bytes = "1.12" +aws-lc-rs = "1" +encodify = "1.0" +md5 = "0.8" +quick-xml = "0.42" +crc-fast = { version = "1.10", default-features = false, features = ["std"] } +tokio = { version = "1.54", features = ["time"] } +rand = "0.10.3" +hashify = "0.3" +memchr = "2.8" + +[lints] +workspace = true + +[dev-dependencies] +jiff = "0.2.38" diff --git a/crates/s3/src/client.rs b/crates/s3/src/client.rs new file mode 100644 index 00000000..da10a129 --- /dev/null +++ b/crates/s3/src/client.rs @@ -0,0 +1,465 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Credentials, DeleteBatch, Endpoint, Error, Result, RetryPolicy, Scheme, UrlStyle, + delete::{DeleteFailure, DeleteResponse}, + endpoint::{Location, RequestUrl}, + error::read_capped, + retry::Attempts, + signer::{ + AmzDate, CanonicalRequest, EMPTY_PAYLOAD_SHA256, Sha256Hex, Signer, UNSIGNED_PAYLOAD, + }, +}; +use bytes::Bytes; +use encodify::base64::STANDARD; +use reqwest::{ + Client, Method, Response, StatusCode, + header::{AUTHORIZATION, CONTENT_LENGTH, HeaderName, RANGE}, +}; +use std::{ + fmt, + ops::Range, + sync::atomic::{AtomicBool, Ordering}, +}; + +const CONTENT_MD5_NAME: &str = "content-md5"; +const HOST_NAME: &str = "host"; +const X_AMZ_CHECKSUM_CRC32C_NAME: &str = "x-amz-checksum-crc32c"; +const X_AMZ_CONTENT_SHA256_NAME: &str = "x-amz-content-sha256"; +const X_AMZ_DATE_NAME: &str = "x-amz-date"; +const X_AMZ_SECURITY_TOKEN_NAME: &str = "x-amz-security-token"; + +const CONTENT_MD5: HeaderName = HeaderName::from_static(CONTENT_MD5_NAME); +const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static(X_AMZ_CHECKSUM_CRC32C_NAME); +const X_AMZ_CONTENT_SHA256: HeaderName = HeaderName::from_static(X_AMZ_CONTENT_SHA256_NAME); +const X_AMZ_DATE: HeaderName = HeaderName::from_static(X_AMZ_DATE_NAME); +const X_AMZ_SECURITY_TOKEN: HeaderName = HeaderName::from_static(X_AMZ_SECURITY_TOKEN_NAME); + +const MAX_DELETE_RESPONSE: usize = 1024 * 1024; +const MAX_SIGNED_HEADERS: usize = 6; + +pub struct BucketConfig<'x> { + pub endpoint: Endpoint, + pub bucket: &'x str, + pub region: &'x str, + pub style: UrlStyle, + pub credentials: Credentials, + pub retry: RetryPolicy, +} + +pub struct Bucket { + http: Client, + location: Location, + signer: Signer, + session_token: Option>, + scheme: Scheme, + retry: RetryPolicy, + checksum: AtomicBool, + multi_delete: AtomicBool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum WriteCheck { + None, + Head, +} + +struct Request<'x> { + method: Method, + url: &'x RequestUrl, + query: &'x str, + payload_hash: &'x str, + body: Option<&'x Bytes>, + content_md5: Option<&'x str>, + checksum: Option<&'x str>, + range: Option<&'x str>, +} + +impl Bucket { + pub fn new(http: Client, config: BucketConfig<'_>) -> Result { + let location = config.endpoint.locate(config.bucket, config.style)?; + Ok(Bucket { + http, + location, + signer: Signer::new(&config.credentials, config.region), + session_token: config.credentials.session_token().map(Into::into), + scheme: config.endpoint.scheme(), + retry: config.retry, + checksum: AtomicBool::new(true), + multi_delete: AtomicBool::new(true), + }) + } + + pub fn sends_checksum(&self) -> bool { + self.scheme == Scheme::Https && self.checksum.load(Ordering::Relaxed) + } + + pub fn supports_multi_delete(&self) -> bool { + self.multi_delete.load(Ordering::Relaxed) + } + + pub async fn put_object(&self, key: &str, body: Bytes) -> Result<()> { + self.put(key, body, WriteCheck::None).await + } + + pub async fn put_object_verified(&self, key: &str, body: Bytes) -> Result<()> { + self.put(key, body, WriteCheck::Head).await + } + + async fn put(&self, key: &str, body: Bytes, check: WriteCheck) -> Result<()> { + let url = self.location.object_url(key)?; + let payload_hash = match self.scheme { + Scheme::Https => None, + Scheme::Http => Some(Sha256Hex::of(&body)), + }; + let payload_hash = payload_hash + .as_ref() + .map_or(UNSIGNED_PAYLOAD, Sha256Hex::as_str); + let (mut checksum, mut content_md5) = match self.scheme { + Scheme::Http => (None, None), + Scheme::Https if self.checksum.load(Ordering::Relaxed) => { + (Some(crc32c_base64(&body)), None) + } + Scheme::Https => (None, Some(md5_base64(&body))), + }; + let mut attempts = Attempts::new(self.retry); + + loop { + let response = self + .execute( + &Request { + method: Method::PUT, + url: &url, + query: "", + payload_hash, + body: Some(&body), + content_md5: content_md5.as_deref(), + checksum: checksum.as_deref(), + range: None, + }, + &mut attempts, + ) + .await?; + if !response.status().is_success() { + let error = Error::from_response(response).await; + if checksum.is_some() && error.rejects_checksum() { + self.checksum.store(false, Ordering::Relaxed); + checksum = None; + content_md5 = Some(md5_base64(&body)); + continue; + } + return Err(error); + } + if check == WriteCheck::None || self.head(&url, &mut attempts).await?.is_some() { + return Ok(()); + } + if !attempts.can_retry() { + return Err(Error::NotVisible); + } + attempts.backoff().await; + } + } + + pub async fn get_object(&self, key: &str, range: Option>) -> Result> { + let range_header = match &range { + Some(range) if range.start >= range.end => { + return Ok(self.head_object(key).await?.map(|_| Bytes::new())); + } + Some(range) if range.end == u64::MAX => Some(format!("bytes={}-", range.start)), + Some(range) => Some(format!("bytes={}-{}", range.start, range.end - 1)), + None => None, + }; + let url = self.location.object_url(key)?; + let request = Request { + method: Method::GET, + url: &url, + query: "", + payload_hash: EMPTY_PAYLOAD_SHA256, + body: None, + content_md5: None, + checksum: None, + range: range_header.as_deref(), + }; + + let mut attempts = Attempts::new(self.retry); + loop { + let response = self.execute(&request, &mut attempts).await?; + let status = response.status(); + match status { + StatusCode::OK | StatusCode::PARTIAL_CONTENT => match response.bytes().await { + Ok(body) => { + return Ok(Some(match &range { + Some(range) if status == StatusCode::OK => slice_range(body, range), + _ => body, + })); + } + Err(err) if attempts.take_transport_retry(&err) => attempts.backoff().await, + Err(err) => return Err(Error::Transport(err)), + }, + StatusCode::NOT_FOUND => return Ok(None), + StatusCode::RANGE_NOT_SATISFIABLE => return Ok(Some(Bytes::new())), + _ => return Err(Error::from_response(response).await), + } + } + } + + pub async fn head_object(&self, key: &str) -> Result> { + let url = self.location.object_url(key)?; + self.head(&url, &mut Attempts::new(self.retry)).await + } + + async fn head(&self, url: &RequestUrl, attempts: &mut Attempts) -> Result> { + let response = self + .execute( + &Request { + method: Method::HEAD, + url, + query: "", + payload_hash: EMPTY_PAYLOAD_SHA256, + body: None, + content_md5: None, + checksum: None, + range: None, + }, + attempts, + ) + .await?; + match response.status() { + status if status.is_success() => Ok(Some( + response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse().ok()) + .unwrap_or_default(), + )), + StatusCode::NOT_FOUND => Ok(None), + _ => Err(Error::from_response(response).await), + } + } + + pub async fn delete_object(&self, key: &str) -> Result { + let url = self.location.object_url(key)?; + let response = self + .execute( + &Request { + method: Method::DELETE, + url: &url, + query: "", + payload_hash: EMPTY_PAYLOAD_SHA256, + body: None, + content_md5: None, + checksum: None, + range: None, + }, + &mut Attempts::new(self.retry), + ) + .await?; + match response.status() { + status if status.is_success() => Ok(true), + StatusCode::NOT_FOUND => Ok(false), + _ => Err(Error::from_response(response).await), + } + } + + pub async fn delete_objects(&self, batch: DeleteBatch) -> Result<()> { + if batch.is_empty() { + return Ok(()); + } + let url = self.location.bucket_url("delete"); + let mut body = batch.into_body(); + let mut attempts = Attempts::new(self.retry); + + loop { + let content_md5 = md5_base64(&body); + let payload_hash = Sha256Hex::of(&body); + let response = self + .execute( + &Request { + method: Method::POST, + url: &url, + query: "delete=", + payload_hash: payload_hash.as_str(), + body: Some(&body), + content_md5: Some(&content_md5), + checksum: None, + range: None, + }, + &mut attempts, + ) + .await?; + let status = response.status(); + if !status.is_success() { + let error = Error::from_response(response).await; + if error.is_not_implemented() { + self.multi_delete.store(false, Ordering::Relaxed); + } + return Err(error); + } + + let content_length = response.content_length(); + let response = read_capped(response, MAX_DELETE_RESPONSE + 1).await?; + if response.len() > MAX_DELETE_RESPONSE { + return Err(Error::InvalidResponse( + "DeleteObjects response exceeds 1 MiB", + )); + } + match DeleteResponse::parse(&response, content_length) { + DeleteResponse::Deleted => return Ok(()), + DeleteResponse::Malformed => { + return Err(Error::InvalidResponse( + "DeleteObjects response is not a DeleteResult", + )); + } + DeleteResponse::Rejected { code, message } => { + let not_implemented = code.as_deref() == Some("NotImplemented"); + let retryable = code.as_deref().is_some_and(RetryPolicy::is_retryable_code); + if not_implemented { + self.multi_delete.store(false, Ordering::Relaxed); + } + if !retryable || !attempts.can_retry() { + return Err(Error::Status { + status: if not_implemented { + StatusCode::NOT_IMPLEMENTED.as_u16() + } else { + status.as_u16() + }, + code: code.map(String::into_boxed_str), + message: message.map(String::into_boxed_str), + }); + } + } + DeleteResponse::Failed(failures) => { + body = retry_batch(failures, &attempts)?; + } + } + + attempts.backoff().await; + } + } + + async fn execute(&self, request: &Request<'_>, attempts: &mut Attempts) -> Result { + loop { + match self.send(request).await { + Ok(response) if attempts.retries_status(response.status()) => {} + Ok(response) => return Ok(response), + Err(err) if request.is_read() && attempts.take_transport_retry(&err) => {} + Err(err) => return Err(Error::Transport(err)), + } + attempts.backoff().await; + } + } + + async fn send(&self, request: &Request<'_>) -> reqwest::Result { + let date = AmzDate::now(); + let entries = [ + request.content_md5.map(|value| (CONTENT_MD5_NAME, value)), + Some((HOST_NAME, self.location.host())), + request + .checksum + .map(|value| (X_AMZ_CHECKSUM_CRC32C_NAME, value)), + Some((X_AMZ_CONTENT_SHA256_NAME, request.payload_hash)), + Some((X_AMZ_DATE_NAME, date.as_str())), + self.session_token + .as_deref() + .map(|value| (X_AMZ_SECURITY_TOKEN_NAME, value)), + ]; + let mut headers = [("", ""); MAX_SIGNED_HEADERS]; + let mut len = 0; + for (slot, entry) in headers.iter_mut().zip(entries.into_iter().flatten()) { + *slot = entry; + len += 1; + } + let authorization = self.signer.authorization( + &CanonicalRequest { + method: request.method.as_str(), + uri: request.url.canonical_uri(), + query: request.query, + headers: headers.get(..len).unwrap_or_default(), + payload_hash: request.payload_hash, + }, + &date, + ); + + let mut builder = self + .http + .request(request.method.clone(), request.url.as_str()) + .header(AUTHORIZATION, authorization) + .header(X_AMZ_DATE, date.as_str()) + .header(X_AMZ_CONTENT_SHA256, request.payload_hash); + if let Some(content_md5) = request.content_md5 { + builder = builder.header(CONTENT_MD5, content_md5); + } + if let Some(checksum) = request.checksum { + builder = builder.header(X_AMZ_CHECKSUM_CRC32C, checksum); + } + if let Some(token) = &self.session_token { + builder = builder.header(X_AMZ_SECURITY_TOKEN, token.as_ref()); + } + if let Some(range) = request.range { + builder = builder.header(RANGE, range); + } + if let Some(body) = request.body { + builder = builder + .header(CONTENT_LENGTH, body.len()) + .body(body.clone()); + } + builder.send().await + } +} + +impl Request<'_> { + fn is_read(&self) -> bool { + self.method == Method::GET || self.method == Method::HEAD + } +} + +impl fmt::Debug for Bucket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Bucket") + .field("location", &self.location) + .field("signer", &self.signer) + .field("scheme", &self.scheme) + .field("retry", &self.retry) + .field("checksum", &self.checksum) + .field("multi_delete", &self.multi_delete) + .finish_non_exhaustive() + } +} + +fn retry_batch(failures: Vec, attempts: &Attempts) -> Result { + let can_retry = attempts.can_retry(); + let key_len = failures.first().map_or(0, |failure| failure.key.len()); + let mut batch = DeleteBatch::with_capacity(failures.len(), key_len); + for failure in failures { + if !can_retry || !RetryPolicy::is_retryable_code(&failure.code) { + return Err(Error::DeleteFailed { + key: failure.key.into_boxed_str(), + code: failure.code.into_boxed_str(), + message: failure.message.map(String::into_boxed_str), + }); + } + batch.push(&failure.key)?; + } + Ok(batch.into_body()) +} + +fn slice_range(body: Bytes, range: &Range) -> Bytes { + let len = body.len() as u64; + let start = range.start.min(len); + let end = range.end.min(len); + body.slice(start as usize..end as usize) +} + +fn md5_base64(data: &[u8]) -> String { + STANDARD.encode(md5::compute(data).0) +} + +pub fn crc32c_base64(data: &[u8]) -> String { + STANDARD.encode(crc_fast::crc32_iscsi(data).to_be_bytes()) +} + diff --git a/crates/s3/src/credentials.rs b/crates/s3/src/credentials.rs new file mode 100644 index 00000000..e9716d67 --- /dev/null +++ b/crates/s3/src/credentials.rs @@ -0,0 +1,175 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Error, Result}; +use std::{env, path::PathBuf}; + +const DEFAULT_PROFILE: &str = "default"; + +#[derive(Clone, PartialEq, Eq)] +pub struct Credentials { + access_key: Box, + secret_key: Box, + session_token: Option>, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct ProfileEntry { + pub access_key: Option, + pub secret_key: Option, + pub session_token: Option, +} + +#[derive(Clone, Copy)] +enum ProfileField { + AccessKey, + SecretKey, + SessionToken, +} + +impl Credentials { + pub fn new(access_key: &str, secret_key: &str, session_token: Option<&str>) -> Self { + Credentials { + access_key: access_key.into(), + secret_key: secret_key.into(), + session_token: session_token.map(Into::into), + } + } + + pub fn resolve( + access_key: Option<&str>, + secret_key: Option<&str>, + session_token: Option<&str>, + profile: Option<&str>, + ) -> Result { + if let Some(access_key) = access_key { + let secret_key = secret_key.ok_or_else(|| { + Error::Credentials("an access key was given without a secret key".into()) + })?; + return Ok(Credentials::new(access_key, secret_key, session_token)); + } + if let Some(credentials) = Self::from_env() { + return Ok(credentials); + } + Self::from_profile(profile) + } + + fn from_env() -> Option { + let access_key = env::var("AWS_ACCESS_KEY_ID").ok()?; + let secret_key = env::var("AWS_SECRET_ACCESS_KEY").ok()?; + let session_token = env::var("AWS_SESSION_TOKEN") + .or_else(|_| env::var("AWS_SECURITY_TOKEN")) + .ok(); + Some(Credentials::new( + &access_key, + &secret_key, + session_token.as_deref(), + )) + } + + fn from_profile(profile: Option<&str>) -> Result { + let path = env::var_os("AWS_SHARED_CREDENTIALS_FILE") + .map(PathBuf::from) + .or_else(|| { + env::var_os("HOME").map(|home| PathBuf::from(home).join(".aws").join("credentials")) + }) + .ok_or_else(|| Error::Credentials("no credentials configured".into()))?; + let contents = std::fs::read_to_string(&path).map_err(|err| { + Error::Credentials( + format!( + "no credentials configured and {} is not readable: {err}", + path.display() + ) + .into(), + ) + })?; + let section = profile.unwrap_or(DEFAULT_PROFILE); + let entry = ProfileEntry::parse(&contents, section).ok_or_else(|| { + Error::Credentials( + format!("profile {section:?} not found in {}", path.display()).into(), + ) + })?; + match (entry.access_key, entry.secret_key) { + (Some(access_key), Some(secret_key)) => Ok(Credentials::new( + &access_key, + &secret_key, + entry.session_token.as_deref(), + )), + _ => Err(Error::Credentials( + format!( + "profile {section:?} in {} lacks aws_access_key_id or aws_secret_access_key", + path.display() + ) + .into(), + )), + } + } + + pub fn access_key(&self) -> &str { + &self.access_key + } + + pub fn secret_key(&self) -> &str { + &self.secret_key + } + + pub fn session_token(&self) -> Option<&str> { + self.session_token.as_deref() + } +} + +impl ProfileEntry { + pub(crate) fn parse(contents: &str, section: &str) -> Option { + let mut entry = None; + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + if let Some(name) = line + .strip_prefix('[') + .and_then(|line| line.strip_suffix(']')) + { + if entry.is_some() { + break; + } + if name.trim() == section { + entry = Some(ProfileEntry::default()); + } + continue; + } + let (Some(entry), Some((name, value))) = (entry.as_mut(), line.split_once('=')) else { + continue; + }; + let field = hashify::map!(name.trim().as_bytes(), ProfileField, + "aws_access_key_id" => ProfileField::AccessKey, + "aws_secret_access_key" => ProfileField::SecretKey, + "aws_session_token" => ProfileField::SessionToken, + "aws_security_token" => ProfileField::SessionToken, + ) + .copied(); + let value = Some(value.trim().to_string()); + match field { + Some(ProfileField::AccessKey) => entry.access_key = value, + Some(ProfileField::SecretKey) => entry.secret_key = value, + Some(ProfileField::SessionToken) if entry.session_token.is_none() => { + entry.session_token = value; + } + Some(ProfileField::SessionToken) | None => {} + } + } + entry + } +} + +impl std::fmt::Debug for Credentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credentials") + .field("access_key", &self.access_key) + .finish_non_exhaustive() + } +} + diff --git a/crates/s3/src/delete.rs b/crates/s3/src/delete.rs new file mode 100644 index 00000000..f6ff2c39 --- /dev/null +++ b/crates/s3/src/delete.rs @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Error, Result, + error::{ErrorBody, ErrorField, push_entity}, +}; +use bytes::Bytes; +use quick_xml::{Reader, events::Event}; +use std::fmt::Write; + +pub const MAX_DELETE_KEYS: usize = 1000; + +const BODY_START: &str = ""; +const BODY_END: &str = "true"; +const OBJECT_START: &str = ""; +const OBJECT_END: &str = ""; +const NO_SUCH_KEY: &str = "NoSuchKey"; + +#[derive(Debug, Clone)] +pub struct DeleteBatch { + body: String, + len: usize, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct DeleteFailure { + pub key: String, + pub code: String, + pub message: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum DeleteResponse { + Deleted, + Failed(Vec), + Rejected { + code: Option, + message: Option, + }, + Malformed, +} + +impl DeleteBatch { + pub fn with_capacity(keys: usize, key_len: usize) -> Self { + let keys = keys.min(MAX_DELETE_KEYS); + DeleteBatch { + body: String::with_capacity( + BODY_START.len() + + keys * (OBJECT_START.len() + key_len + OBJECT_END.len()) + + BODY_END.len(), + ), + len: 0, + } + } + + pub fn push(&mut self, key: &str) -> Result<()> { + if key.is_empty() || self.len == MAX_DELETE_KEYS { + return Err(Error::InvalidKey(key.into())); + } + if self.body.is_empty() { + self.body.push_str(BODY_START); + } + self.body.push_str(OBJECT_START); + push_escaped(&mut self.body, key); + self.body.push_str(OBJECT_END); + self.len += 1; + Ok(()) + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn is_full(&self) -> bool { + self.len == MAX_DELETE_KEYS + } + + pub(crate) fn into_body(mut self) -> Bytes { + if self.body.is_empty() { + self.body.push_str(BODY_START); + } + self.body.push_str(BODY_END); + Bytes::from(self.body) + } +} + +fn push_escaped(out: &mut String, key: &str) { + for ch in key.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + ch if ch.is_ascii_control() => { + let _ = write!(out, "&#{};", u32::from(ch)); + } + ch => out.push(ch), + } + } +} + +impl DeleteResponse { + pub(crate) fn parse(body: &[u8], content_length: Option) -> Self { + if body.is_empty() && content_length == Some(0) { + return DeleteResponse::Deleted; + } + let mut reader = Reader::from_reader(body); + loop { + match reader.read_event() { + Ok(Event::Start(tag)) => { + return match tag.local_name().as_ref() { + "DeleteResult" => Self::parse_result(&mut reader), + "Error" => { + let error = ErrorBody::parse(body); + DeleteResponse::Rejected { + code: error.code, + message: error.message, + } + } + _ => DeleteResponse::Malformed, + }; + } + Ok(Event::Empty(tag)) => { + return if tag.local_name().as_ref() == "DeleteResult" { + DeleteResponse::Deleted + } else { + DeleteResponse::Malformed + }; + } + Ok(Event::Eof | Event::End(_)) | Err(_) => return DeleteResponse::Malformed, + Ok(_) => {} + } + } + } + + fn parse_result(reader: &mut Reader<&[u8]>) -> Self { + let mut failures = Vec::new(); + let mut current: Option = None; + let mut field = ErrorField::Other; + let mut text = String::new(); + let mut depth = 0u32; + + loop { + match reader.read_event() { + Ok(Event::Start(tag)) => { + depth += 1; + if current.is_some() { + field = ErrorField::parse(tag.local_name().as_ref()); + text.clear(); + } else if depth == 1 && tag.local_name().as_ref() == "Error" { + current = Some(DeleteFailure::default()); + } + } + Ok(Event::Text(chunk)) if field != ErrorField::Other => { + text.push_str(&chunk.xml10_content()); + } + Ok(Event::GeneralRef(entity)) if field != ErrorField::Other => { + push_entity(&entity, &mut text); + } + Ok(Event::End(_)) => { + if depth == 0 { + break; + } + depth -= 1; + if let Some(failure) = current.as_mut() { + if depth == 0 { + if let Some(failure) = current.take() + && failure.code != NO_SUCH_KEY + { + failures.push(failure); + } + } else { + let value = std::mem::take(&mut text); + match field { + ErrorField::Key => failure.key = value, + ErrorField::Code => failure.code = value, + ErrorField::Message => failure.message = Some(value), + ErrorField::Other => {} + } + field = ErrorField::Other; + } + } + } + Ok(Event::Eof) | Err(_) => return DeleteResponse::Malformed, + Ok(_) => {} + } + } + + if failures.is_empty() { + DeleteResponse::Deleted + } else { + DeleteResponse::Failed(failures) + } + } +} + diff --git a/crates/s3/src/endpoint.rs b/crates/s3/src/endpoint.rs new file mode 100644 index 00000000..c589487a --- /dev/null +++ b/crates/s3/src/endpoint.rs @@ -0,0 +1,257 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Scheme { + Http, + Https, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UrlStyle { + Path, + VirtualHost, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Endpoint { + scheme: Scheme, + authority: Box, + base_path: Box, +} + +#[derive(Debug)] +pub(crate) struct Location { + url_prefix: Box, + path_offset: usize, + host: Box, +} + +impl Scheme { + fn parse(scheme: &str) -> Result { + hashify::map_ignore_case!(scheme.as_bytes(), Scheme, + "http" => Scheme::Http, + "https" => Scheme::Https, + ) + .copied() + .ok_or_else(|| Error::InvalidConfig(format!("unsupported URL scheme {scheme:?}").into())) + } + + pub fn as_str(&self) -> &'static str { + match self { + Scheme::Http => "http", + Scheme::Https => "https", + } + } + + fn default_port(&self) -> &'static str { + match self { + Scheme::Http => "80", + Scheme::Https => "443", + } + } +} + +impl Endpoint { + pub fn parse(url: &str) -> Result { + let url = url.trim(); + let (scheme, rest) = match url.split_once("://") { + Some((scheme, rest)) => (Scheme::parse(scheme)?, rest), + None => (Scheme::Https, url), + }; + let (authority, path) = rest.split_once('/').unwrap_or((rest, "")); + if authority.is_empty() + || !authority.bytes().all(|ch| { + ch.is_ascii_alphanumeric() || matches!(ch, b'.' | b'-' | b'_' | b':' | b'[' | b']') + }) + { + return Err(Error::InvalidConfig( + format!("invalid S3 endpoint host {authority:?}").into(), + )); + } + let mut authority = authority.to_ascii_lowercase(); + if let Some((host, port)) = authority.rsplit_once(':') + && !host.ends_with(':') + && !port.ends_with(']') + && port == scheme.default_port() + { + authority.truncate(host.len()); + } + + let mut base_path = String::with_capacity(path.len() + 1); + for segment in path.split('/').filter(|segment| !segment.is_empty()) { + base_path.push('/'); + push_segment(&mut base_path, segment)?; + } + + Ok(Endpoint { + scheme, + authority: authority.into_boxed_str(), + base_path: base_path.into_boxed_str(), + }) + } + + pub fn scheme(&self) -> Scheme { + self.scheme + } + + pub fn authority(&self) -> &str { + &self.authority + } + + pub(crate) fn locate(&self, bucket: &str, style: UrlStyle) -> Result { + if bucket.is_empty() || bucket.contains('/') { + return Err(Error::InvalidConfig( + format!("invalid bucket name {bucket:?}").into(), + )); + } + let scheme = self.scheme.as_str(); + match style { + UrlStyle::Path => { + let mut url = String::with_capacity( + scheme.len() + + 3 + + self.authority.len() + + self.base_path.len() + + bucket.len() + + 1, + ); + url.push_str(scheme); + url.push_str("://"); + url.push_str(&self.authority); + let path_offset = url.len(); + url.push_str(&self.base_path); + url.push('/'); + push_segment(&mut url, bucket)?; + Ok(Location { + url_prefix: url.into_boxed_str(), + path_offset, + host: self.authority.clone(), + }) + } + UrlStyle::VirtualHost => { + if !bucket.bytes().all(|ch| { + ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, b'-' | b'.') + }) { + return Err(Error::InvalidConfig( + format!("bucket {bucket:?} is not a valid host name").into(), + )); + } + let mut host = String::with_capacity(bucket.len() + 1 + self.authority.len()); + host.push_str(bucket); + host.push('.'); + host.push_str(&self.authority); + let mut url = + String::with_capacity(scheme.len() + 3 + host.len() + self.base_path.len()); + url.push_str(scheme); + url.push_str("://"); + url.push_str(&host); + let path_offset = url.len(); + url.push_str(&self.base_path); + Ok(Location { + url_prefix: url.into_boxed_str(), + path_offset, + host: host.into_boxed_str(), + }) + } + } + } +} + +impl Location { + pub(crate) fn host(&self) -> &str { + &self.host + } + + pub(crate) fn object_url(&self, key: &str) -> Result { + let mut url = String::with_capacity(self.url_prefix.len() + 1 + key.len() + key.len() / 2); + url.push_str(&self.url_prefix); + url.push('/'); + push_key(&mut url, key)?; + Ok(RequestUrl { + url, + path_offset: self.path_offset, + query_offset: None, + }) + } + + pub(crate) fn bucket_url(&self, query: &str) -> RequestUrl { + let mut url = String::with_capacity(self.url_prefix.len() + 2 + query.len()); + url.push_str(&self.url_prefix); + url.push('/'); + let query_offset = url.len(); + url.push('?'); + url.push_str(query); + RequestUrl { + url, + path_offset: self.path_offset, + query_offset: Some(query_offset), + } + } +} + +#[derive(Debug)] +pub(crate) struct RequestUrl { + url: String, + path_offset: usize, + query_offset: Option, +} + +impl RequestUrl { + pub(crate) fn as_str(&self) -> &str { + &self.url + } + + pub(crate) fn canonical_uri(&self) -> &str { + self.url + .get(self.path_offset..self.query_offset.unwrap_or(self.url.len())) + .unwrap_or("/") + } +} + +const fn is_unreserved(ch: u8) -> bool { + ch.is_ascii_alphanumeric() || matches!(ch, b'-' | b'_' | b'.' | b'~') +} + +const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF"; + +fn push_encoded(out: &mut String, segment: &str) { + for &ch in segment.as_bytes() { + if is_unreserved(ch) { + out.push(char::from(ch)); + } else { + out.push('%'); + out.push(char::from(HEX_UPPER[usize::from(ch >> 4)])); + out.push(char::from(HEX_UPPER[usize::from(ch & 0x0f)])); + } + } +} + +fn push_segment(out: &mut String, segment: &str) -> Result<()> { + if matches!(segment, "." | "..") { + return Err(Error::InvalidKey(segment.into())); + } + push_encoded(out, segment); + Ok(()) +} + +pub(crate) fn push_key(out: &mut String, key: &str) -> Result<()> { + if key.is_empty() { + return Err(Error::InvalidKey(key.into())); + } + let mut segments = key.split('/'); + if let Some(first) = segments.next() { + push_segment(out, first).map_err(|_| Error::InvalidKey(key.into()))?; + } + for segment in segments { + out.push('/'); + push_segment(out, segment).map_err(|_| Error::InvalidKey(key.into()))?; + } + Ok(()) +} + diff --git a/crates/s3/src/error.rs b/crates/s3/src/error.rs new file mode 100644 index 00000000..ad730c98 --- /dev/null +++ b/crates/s3/src/error.rs @@ -0,0 +1,267 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use quick_xml::{ + Reader, + escape::resolve_predefined_entity, + events::{BytesRef, Event}, +}; +use reqwest::{Response, StatusCode}; +use std::fmt::{self, Display}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum Error { + Transport(reqwest::Error), + Status { + status: u16, + code: Option>, + message: Option>, + }, + DeleteFailed { + key: Box, + code: Box, + message: Option>, + }, + NotVisible, + InvalidResponse(&'static str), + InvalidKey(Box), + InvalidConfig(Box), + Credentials(Box), +} + +const MAX_ERROR_BODY: usize = 64 * 1024; +const CHECKSUM_HEADER_PREFIX: &str = "x-amz-checksum-"; + +impl Error { + #[cold] + pub(crate) async fn from_response(response: Response) -> Self { + let status = response.status().as_u16(); + let body = read_capped(response, MAX_ERROR_BODY) + .await + .unwrap_or_default(); + let (code, message) = ErrorBody::parse(&body).into_parts(); + Error::Status { + status, + code, + message, + } + } + + pub fn status(&self) -> Option { + match self { + Error::Status { status, .. } => Some(*status), + Error::NotVisible => Some(StatusCode::NOT_FOUND.as_u16()), + _ => None, + } + } + + pub fn code(&self) -> Option<&str> { + match self { + Error::Status { code, .. } => code.as_deref(), + Error::DeleteFailed { code, .. } => Some(code), + _ => None, + } + } + + pub fn is_not_implemented(&self) -> bool { + matches!(self, Error::Status { status, .. } if *status == StatusCode::NOT_IMPLEMENTED.as_u16()) + } + + #[cold] + pub(crate) fn rejects_checksum(&self) -> bool { + let Error::Status { + status, + code, + message, + } = self + else { + return false; + }; + if *status == StatusCode::NOT_IMPLEMENTED.as_u16() { + return true; + } + if *status != StatusCode::BAD_REQUEST.as_u16() { + return false; + } + let Some(message) = message.as_deref().map(str::to_ascii_lowercase) else { + return false; + }; + match code.as_deref().map(str::as_bytes) { + Some(code) + if hashify::set!( + code, + "BadDigest", + "InvalidDigest", + "XAmzContentChecksumMismatch", + "XAmzContentSHA256Mismatch" + ) => + { + false + } + Some(code) + if hashify::set!( + code, + "InvalidArgument", + "InvalidRequest", + "NotImplemented", + "UnsupportedHeader" + ) => + { + message.contains("checksum") + } + _ => message.contains(CHECKSUM_HEADER_PREFIX), + } + } +} + +pub(crate) async fn read_capped( + mut response: Response, + max_len: usize, +) -> std::result::Result, reqwest::Error> { + let mut body = Vec::with_capacity(response.content_length().map_or(0, |len| { + usize::try_from(len).unwrap_or(max_len).min(max_len) + })); + while let Some(chunk) = response.chunk().await? { + let room = max_len - body.len(); + if chunk.len() >= room { + body.extend_from_slice(chunk.get(..room).unwrap_or_default()); + break; + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +#[derive(Debug, Default)] +pub(crate) struct ErrorBody { + pub code: Option, + pub message: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ErrorField { + Code, + Message, + Key, + Other, +} + +impl ErrorField { + pub(crate) fn parse(name: &str) -> Self { + hashify::map!(name.as_bytes(), ErrorField, + "Code" => ErrorField::Code, + "Message" => ErrorField::Message, + "Key" => ErrorField::Key, + ) + .copied() + .unwrap_or(ErrorField::Other) + } +} + +impl ErrorBody { + pub(crate) fn parse(body: &[u8]) -> Self { + let mut reader = Reader::from_reader(body); + let mut error = ErrorBody::default(); + let mut field = ErrorField::Other; + let mut text = String::new(); + loop { + match reader.read_event() { + Ok(Event::Start(tag)) => { + field = ErrorField::parse(tag.local_name().as_ref()); + text.clear(); + } + Ok(Event::Text(chunk)) if field != ErrorField::Other => { + text.push_str(&chunk.xml10_content()); + } + Ok(Event::GeneralRef(entity)) if field != ErrorField::Other => { + push_entity(&entity, &mut text); + } + Ok(Event::End(_)) => { + match field { + ErrorField::Code => error.code = Some(std::mem::take(&mut text)), + ErrorField::Message => error.message = Some(std::mem::take(&mut text)), + ErrorField::Key | ErrorField::Other => {} + } + field = ErrorField::Other; + } + Ok(Event::Eof) | Err(_) => break, + Ok(_) => {} + } + } + error + } + + fn into_parts(self) -> (Option>, Option>) { + ( + self.code.map(String::into_boxed_str), + self.message.map(String::into_boxed_str), + ) + } +} + +pub(crate) fn push_entity(entity: &BytesRef<'_>, out: &mut String) { + if let Ok(Some(ch)) = entity.resolve_char_ref() { + out.push(ch); + } else if let Some(text) = resolve_predefined_entity(entity) { + out.push_str(text); + } +} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Transport(err) => { + write!(f, "S3 request failed: {err}")?; + let mut source = std::error::Error::source(err); + while let Some(err) = source { + write!(f, ": {err}")?; + source = err.source(); + } + Ok(()) + } + Error::Status { + status, + code, + message, + } => { + write!(f, "S3 returned status {status}")?; + if let Some(code) = code { + write!(f, " {code}")?; + } + if let Some(message) = message { + write!(f, ": {message}")?; + } + Ok(()) + } + Error::DeleteFailed { key, code, message } => { + write!(f, "S3 failed to delete {key:?}: {code}")?; + if let Some(message) = message { + write!(f, ": {message}")?; + } + Ok(()) + } + Error::NotVisible => f.write_str(concat!( + "PUT acknowledged with 2xx but object not visible to read path; ", + "backend may be silently losing writes" + )), + Error::InvalidResponse(reason) => write!(f, "Invalid S3 response: {reason}"), + Error::InvalidKey(key) => write!(f, "Invalid S3 object key {key:?}"), + Error::InvalidConfig(reason) => write!(f, "Invalid S3 configuration: {reason}"), + Error::Credentials(reason) => write!(f, "S3 credentials: {reason}"), + } + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: reqwest::Error) -> Self { + Error::Transport(err) + } +} + diff --git a/crates/s3/src/lib.rs b/crates/s3/src/lib.rs new file mode 100644 index 00000000..b9cbec25 --- /dev/null +++ b/crates/s3/src/lib.rs @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod client; +pub mod credentials; +pub mod delete; +pub mod endpoint; +pub mod error; +pub mod retry; +pub mod signer; + +pub use client::{Bucket, BucketConfig}; +pub use credentials::Credentials; +pub use delete::DeleteBatch; +pub use endpoint::{Endpoint, Scheme, UrlStyle}; +pub use error::{Error, Result}; +pub use retry::RetryPolicy; diff --git a/crates/s3/src/retry.rs b/crates/s3/src/retry.rs new file mode 100644 index 00000000..8273893a --- /dev/null +++ b/crates/s3/src/retry.rs @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use reqwest::StatusCode; +use std::time::Duration; + +const BASE_DELAY_MS: u64 = 1_000; +const MAX_DELAY_MS: u64 = 20_000; +const MAX_SHIFT: u32 = 5; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPolicy { + max_retries: u32, +} + +impl RetryPolicy { + pub fn new(max_retries: u32) -> Self { + RetryPolicy { max_retries } + } + + pub fn max_retries(&self) -> u32 { + self.max_retries + } + + pub fn can_retry(&self, attempt: u32) -> bool { + attempt < self.max_retries + } + + pub fn delay(&self, attempt: u32) -> Duration { + let cap = (BASE_DELAY_MS << attempt.min(MAX_SHIFT)).min(MAX_DELAY_MS); + Duration::from_millis(rand::random_range(0..=cap)) + } + + pub fn is_retryable_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) + } + + pub fn is_retryable_error(err: &reqwest::Error) -> bool { + !err.is_builder() && !err.is_redirect() + } + + pub fn is_retryable_code(code: &str) -> bool { + hashify::set!( + code.as_bytes(), + "InternalError", + "ServiceUnavailable", + "SlowDown", + "RequestTimeout" + ) + } +} + +pub(crate) struct Attempts { + policy: RetryPolicy, + retries: u32, + transport_retried: bool, +} + +impl Attempts { + pub(crate) fn new(policy: RetryPolicy) -> Self { + Attempts { + policy, + retries: 0, + transport_retried: false, + } + } + + pub(crate) fn can_retry(&self) -> bool { + self.policy.can_retry(self.retries) + } + + pub(crate) fn retries_status(&self, status: StatusCode) -> bool { + RetryPolicy::is_retryable_status(status) && self.can_retry() + } + + pub(crate) fn take_transport_retry(&mut self, err: &reqwest::Error) -> bool { + if self.transport_retried || !RetryPolicy::is_retryable_error(err) || !self.can_retry() { + return false; + } + self.transport_retried = true; + true + } + + pub(crate) async fn backoff(&mut self) { + tokio::time::sleep(self.policy.delay(self.retries)).await; + self.retries += 1; + } +} diff --git a/crates/s3/src/signer.rs b/crates/s3/src/signer.rs new file mode 100644 index 00000000..1580b576 --- /dev/null +++ b/crates/s3/src/signer.rs @@ -0,0 +1,258 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Credentials; +use aws_lc_rs::{ + digest::{SHA256, digest}, + hmac::{HMAC_SHA256, Key, sign}, +}; +use std::{ + fmt, + time::{SystemTime, UNIX_EPOCH}, +}; + +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const EMPTY_PAYLOAD_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +const ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const SERVICE_SCOPE: &str = "/s3/aws4_request"; +const HEX_LOWER: &[u8; 16] = b"0123456789abcdef"; + +pub struct Signer { + access_key: Box, + secret_key: Box<[u8]>, + region: Box, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AmzDate([u8; 16]); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Sha256Hex([u8; 64]); + +pub struct CanonicalRequest<'x> { + pub method: &'x str, + pub uri: &'x str, + pub query: &'x str, + pub headers: &'x [(&'x str, &'x str)], + pub payload_hash: &'x str, +} + +impl Signer { + pub fn new(credentials: &Credentials, region: &str) -> Self { + let secret = credentials.secret_key(); + let mut secret_key = Vec::with_capacity(4 + secret.len()); + secret_key.extend_from_slice(b"AWS4"); + secret_key.extend_from_slice(secret.as_bytes()); + Signer { + access_key: credentials.access_key().into(), + secret_key: secret_key.into_boxed_slice(), + region: region.into(), + } + } + + pub fn authorization(&self, request: &CanonicalRequest<'_>, date: &AmzDate) -> String { + let canonical_hash = Sha256Hex::of(request.build().as_bytes()); + + let mut string_to_sign = String::with_capacity( + ALGORITHM.len() + 18 + 9 + self.region.len() + SERVICE_SCOPE.len() + 1 + 64, + ); + string_to_sign.push_str(ALGORITHM); + string_to_sign.push('\n'); + string_to_sign.push_str(date.as_str()); + string_to_sign.push('\n'); + self.push_scope(&mut string_to_sign, date); + string_to_sign.push('\n'); + string_to_sign.push_str(canonical_hash.as_str()); + + let signing_key = self.signing_key(date); + let signature = Sha256Hex::from_bytes( + sign(&signing_key, string_to_sign.as_bytes()) + .as_ref() + .first_chunk::<32>() + .copied() + .unwrap_or_default(), + ); + + let signed_headers_len = request + .headers + .iter() + .map(|(name, _)| name.len() + 1) + .sum::(); + let mut authorization = String::with_capacity( + ALGORITHM.len() + + 12 + + self.access_key.len() + + 10 + + self.region.len() + + SERVICE_SCOPE.len() + + 15 + + signed_headers_len + + 11 + + 64, + ); + authorization.push_str(ALGORITHM); + authorization.push_str(" Credential="); + authorization.push_str(&self.access_key); + authorization.push('/'); + self.push_scope(&mut authorization, date); + authorization.push_str(",SignedHeaders="); + request.push_signed_headers(&mut authorization); + authorization.push_str(",Signature="); + authorization.push_str(signature.as_str()); + authorization + } + + fn push_scope(&self, out: &mut String, date: &AmzDate) { + out.push_str(date.day()); + out.push('/'); + out.push_str(&self.region); + out.push_str(SERVICE_SCOPE); + } + + fn signing_key(&self, date: &AmzDate) -> Key { + let step = |key: &[u8], data: &[u8]| sign(&Key::new(HMAC_SHA256, key), data); + let date_key = step(&self.secret_key, date.day().as_bytes()); + let region_key = step(date_key.as_ref(), self.region.as_bytes()); + let service_key = step(region_key.as_ref(), b"s3"); + let signing_key = step(service_key.as_ref(), b"aws4_request"); + Key::new(HMAC_SHA256, signing_key.as_ref()) + } +} + +impl fmt::Debug for Signer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Signer") + .field("access_key", &self.access_key) + .field("region", &self.region) + .finish_non_exhaustive() + } +} + +impl CanonicalRequest<'_> { + pub fn build(&self) -> String { + let headers_len = self + .headers + .iter() + .map(|(name, value)| (name.len() + value.len()) * 2 + 3) + .sum::(); + let mut out = String::with_capacity( + self.method.len() + + self.uri.len() + + self.query.len() + + headers_len + + self.payload_hash.len() + + 5, + ); + out.push_str(self.method); + out.push('\n'); + out.push_str(self.uri); + out.push('\n'); + out.push_str(self.query); + out.push('\n'); + for (name, value) in self.headers { + out.push_str(name); + out.push(':'); + out.push_str(value.trim()); + out.push('\n'); + } + out.push('\n'); + self.push_signed_headers(&mut out); + out.push('\n'); + out.push_str(self.payload_hash); + out + } + + fn push_signed_headers(&self, out: &mut String) { + let mut headers = self.headers.iter(); + if let Some((name, _)) = headers.next() { + out.push_str(name); + } + for (name, _) in headers { + out.push(';'); + out.push_str(name); + } + } +} + +impl AmzDate { + pub fn now() -> Self { + Self::from_unix( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()), + ) + } + + pub fn from_unix(timestamp: u64) -> Self { + let days = timestamp / 86_400; + let seconds = timestamp % 86_400; + let (year, month, day) = civil_from_days(days); + let mut out = *b"00000000T000000Z"; + write_digits(&mut out[0..4], year); + write_digits(&mut out[4..6], month); + write_digits(&mut out[6..8], day); + write_digits(&mut out[9..11], seconds / 3600); + write_digits(&mut out[11..13], (seconds / 60) % 60); + write_digits(&mut out[13..15], seconds % 60); + AmzDate(out) + } + + pub fn as_str(&self) -> &str { + std::str::from_utf8(&self.0).unwrap_or_default() + } + + pub fn day(&self) -> &str { + self.as_str().get(..8).unwrap_or_default() + } +} + +fn civil_from_days(days: u64) -> (u64, u64, u64) { + let z = days + 719_468; + let era = z / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + u64::from(month <= 2); + (year, month, day) +} + +fn write_digits(out: &mut [u8], mut value: u64) { + for digit in out.iter_mut().rev() { + *digit = b'0' + (value % 10) as u8; + value /= 10; + } +} + +impl Sha256Hex { + pub fn of(data: &[u8]) -> Self { + Self::from_bytes( + digest(&SHA256, data) + .as_ref() + .first_chunk::<32>() + .copied() + .unwrap_or_default(), + ) + } + + fn from_bytes(bytes: [u8; 32]) -> Self { + let mut out = [0u8; 64]; + for ([high, low], byte) in out.as_chunks_mut::<2>().0.iter_mut().zip(bytes) { + *high = HEX_LOWER[usize::from(byte >> 4)]; + *low = HEX_LOWER[usize::from(byte & 0x0f)]; + } + Sha256Hex(out) + } + + pub fn as_str(&self) -> &str { + std::str::from_utf8(&self.0).unwrap_or_default() + } +} + diff --git a/crates/scim-proto/Cargo.toml b/crates/scim-proto/Cargo.toml index cbe450d7..87e227f3 100644 --- a/crates/scim-proto/Cargo.toml +++ b/crates/scim-proto/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "scim-proto" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] serde = { version = "1.0.229", features = ["derive"] } -serde_json = "1.0.151" -hashify = "0.2.9" +serde_json = "1.0.154" +hashify = "0.3" [lints] workspace = true diff --git a/crates/scim/Cargo.toml b/crates/scim/Cargo.toml index e9b6fa4d..2553fd8a 100644 --- a/crates/scim/Cargo.toml +++ b/crates/scim/Cargo.toml @@ -1,12 +1,13 @@ [package] name = "scim" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] scim-proto = { path = "../scim-proto" } common = { path = "../common" } jmap = { path = "../jmap" } +services = { path = "../services" } store = { path = "../store" } registry = { path = "../registry" } directory = { path = "../directory" } @@ -15,18 +16,18 @@ jmap_proto = { path = "../jmap-proto" } types = { path = "../types" } utils = { path = "../utils" } trc = { path = "../trc" } -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } +compact_str = "0.10.0" +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -xxhash-rust = { version = "0.8.18", features = ["xxh3"] } -icu_locale = "2.3.1" +xxhash-rust = { version = "0.8.19", features = ["xxh3"] } +icu_locale = "2.4.0" [dev-dependencies] [features] -test_mode = [] dev_mode = [] -enterprise = ["common/enterprise", "jmap/enterprise", "store/enterprise", "directory/enterprise", "trc/enterprise"] +enterprise = ["common/enterprise", "jmap/enterprise", "services/enterprise", "store/enterprise", "directory/enterprise", "trc/enterprise", "registry/enterprise", "http_proto/enterprise", "jmap_proto/enterprise", "utils/enterprise"] [lints] workspace = true diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index f24e36e0..3caff13a 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -1,43 +1,45 @@ [package] name = "services" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] store = { path = "../store" } common = { path = "../common" } +coordinator = { path = "../coordinator" } utils = { path = "../utils" } trc = { path = "../trc" } email = { path = "../email" } smtp = { path = "../smtp" } groupware = { path = "../groupware" } +nlp = { path = "../nlp" } +text-extract = { path = "../text-extract" } spam-filter = { path = "../spam-filter" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } -jmap-tools = { version = "0.1" } +jmap-tools = { path = "../jmap-tools" } registry = { path = "../registry" } -smtp-proto = { version = "0.2", features = ["rkyv", "serde"] } -tokio = { version = "1.53", features = ["rt"] } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } -mail-builder = { version = "1.0" } +smtp-proto = { path = "../smtp-proto", features = ["rkyv", "serde"] } +tokio = { version = "1.54", features = ["rt"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } aho-corasick = { version = "1.1" } -calcard = { version = "0.3", features = ["rkyv"] } +calcard = { path = "../calcard", features = ["rkyv"] } serde_json = "1.0" memory-stats = "1.2.0" aes-gcm = "0.11.1" -p256 = { version = "0.13", features = ["ecdh"] } +p256 = { version = "0.14", features = ["ecdh"] } hkdf = "0.13" sha2 = "0.11" reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"]} -base64 = "0.23" dns-update = { version = "0.6" } psl = "2" +compact_str = "0.10.0" [dev-dependencies] [features] -test_mode = [] -enterprise = [] +enterprise = ["store/enterprise", "common/enterprise", "coordinator/enterprise", "utils/enterprise", "trc/enterprise", "email/enterprise", "smtp/enterprise", "groupware/enterprise", "spam-filter/enterprise", "jmap_proto/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/services/src/broadcast/backlog.rs b/crates/services/src/broadcast/backlog.rs new file mode 100644 index 00000000..500b795b --- /dev/null +++ b/crates/services/src/broadcast/backlog.rs @@ -0,0 +1,447 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BroadcastBatch, MAX_BATCH_SIZE}; +use common::ipc::{ + BroadcastEvent, CacheInvalidation, PushNotification, QueueNames, RegistryChange, + ViewerStateChange, +}; +use registry::schema::prelude::ObjectType; +use std::{collections::VecDeque, mem::size_of}; +use store::{ + ahash::{AHashMap, AHashSet}, + write::TaskTypes, +}; +use types::type_state::StateChange; +use utils::map::bitmap::Bitmap; + +pub(crate) const MAX_PENDING_EVENTS: usize = 65_536; +pub(crate) const MAX_PENDING_BYTES: usize = 8 * 1024 * 1024; +const LOW_WATER_NUMERATOR: usize = 7; +const LOW_WATER_DENOMINATOR: usize = 8; +const INVALIDATE_ALL_THRESHOLD: usize = 1024; +const MAX_INVALIDATIONS_PER_EVENT: usize = 1024; +const MAX_CHUNK_BYTES: usize = 256 * 1024; + +const EVENT_COST: usize = size_of::(); +const INVALIDATION_COST: usize = size_of::() + size_of::(); +const STATE_KEY_COST: usize = size_of::<(StateKey, Vec)>() + size_of::(); +const TYPE_STATE_COST: usize = size_of::(); +const PUSH_UPDATE_COST: usize = size_of::() + size_of::(); + +#[derive(Debug, Default)] +pub(crate) struct Backlog { + ready: VecDeque, + ready_bytes: usize, + registry_changes: Vec, + queue_status: Option, + push_updates: AHashSet, + invalidate_negative: bool, + events: VecDeque, + events_bytes: usize, + state_changes: AHashMap>, + type_states: usize, + invalidations: AHashSet, + invalidate_all: bool, + refresh_queues: QueueNames, + refresh_tasks: TaskTypes, + refresh_index: bool, + in_flight: usize, + in_flight_bytes: usize, + degraded: bool, + dropped: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum StateKey { + Account(u32), + Viewer { viewer_id: u32, account_id: u32 }, +} + +#[derive(Debug, Clone, Copy)] +struct TypeState { + types: u64, + change_id: u64, +} + +impl Backlog { + pub fn push(&mut self, event: BroadcastEvent) { + match event { + BroadcastEvent::PushNotification(PushNotification::StateChange(change)) => { + self.push_state( + StateKey::Account(change.account_id), + *change.types, + change.change_id, + ); + } + BroadcastEvent::PushNotification(PushNotification::ViewerStateChange(change)) => { + self.push_state( + StateKey::Viewer { + viewer_id: change.viewer_id, + account_id: change.change.account_id, + }, + *change.change.types, + change.change.change_id, + ); + } + BroadcastEvent::PushNotification(PushNotification::EmailPush(push)) + if self.degraded => + { + let change = push.to_state_change(); + self.push_state( + StateKey::Account(change.account_id), + *change.types, + change.change_id, + ); + } + BroadcastEvent::CacheInvalidate(items) => { + for item in items { + self.push_invalidation(item); + } + } + BroadcastEvent::CacheInvalidateAll => self.invalidate_all(), + BroadcastEvent::CacheInvalidateNegative => { + self.invalidate_negative |= !self.invalidate_all; + } + BroadcastEvent::RegistryChange(change) => self.push_registry_change(change), + BroadcastEvent::MtaQueueStatus { is_running } => self.queue_status = Some(is_running), + BroadcastEvent::PushServerUpdate(account_id) => { + self.push_updates.insert(account_id); + } + BroadcastEvent::QueueRefresh(queues) => self.refresh_queues.merge(&queues), + BroadcastEvent::TaskQueueRefresh(task_types) => self.refresh_tasks.union(task_types), + BroadcastEvent::IndexQueueRefresh => self.refresh_index = true, + event => { + self.events_bytes += event_cost(&event); + self.events.push_back(event); + } + } + self.enforce_limits(); + } + + pub fn fill(&mut self, batch: &mut BroadcastBatch>) -> bool { + if !batch.messages.is_empty() { + return true; + } + if self.ready.is_empty() { + self.snapshot(); + } + let mut chunk_bytes = 0; + while batch.messages.len() < MAX_BATCH_SIZE { + let Some(event) = self.ready.front() else { + break; + }; + let cost = event_cost(event); + if !batch.messages.is_empty() && chunk_bytes + cost > MAX_CHUNK_BYTES { + break; + } + chunk_bytes += cost; + self.ready_bytes -= cost; + batch.messages.extend(self.ready.pop_front()); + } + self.in_flight = batch.messages.len(); + self.in_flight_bytes = chunk_bytes; + !batch.messages.is_empty() + } + + pub fn release(&mut self, batch: &mut BroadcastBatch>) { + batch.clear(); + self.in_flight = 0; + self.in_flight_bytes = 0; + } + + pub fn take_dropped(&mut self) -> u64 { + std::mem::take(&mut self.dropped) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn len(&self) -> usize { + self.in_flight + + self.ready.len() + + self.registry_changes.len() + + usize::from(self.queue_status.is_some()) + + self.push_updates.len() + + usize::from(self.invalidate_negative) + + self.events.len() + + self.type_states + + self.invalidations.len() + + usize::from(self.invalidate_all) + + usize::from(!self.refresh_queues.is_empty()) + + usize::from(!self.refresh_tasks.is_empty()) + + usize::from(self.refresh_index) + } + + pub fn bytes(&self) -> usize { + self.in_flight_bytes + + self.ready_bytes + + self.registry_changes.len() * EVENT_COST + + self.push_updates.len() * PUSH_UPDATE_COST + + self.events_bytes + + self.state_changes.len() * STATE_KEY_COST + + self.type_states * TYPE_STATE_COST + + self.invalidations.len() * INVALIDATION_COST + } + + fn push_state(&mut self, key: StateKey, types: u64, change_id: u64) { + if types == 0 { + return; + } + let states = self.state_changes.entry(key).or_default(); + let before = states.len(); + merge_state(states, types, change_id); + self.type_states = self.type_states + states.len() - before; + } + + fn push_invalidation(&mut self, item: CacheInvalidation) { + if self.invalidate_all && item.is_covered_by_invalidate_all() { + return; + } + self.invalidations.insert(item); + if !self.invalidate_all && self.invalidations.len() > INVALIDATE_ALL_THRESHOLD { + self.invalidate_all(); + } + } + + fn invalidate_all(&mut self) { + self.invalidate_all = true; + self.invalidate_negative = false; + self.invalidations + .retain(|item| !item.is_covered_by_invalidate_all()); + } + + fn push_registry_change(&mut self, change: RegistryChange) { + let object = registry_object(&change); + match self + .registry_changes + .iter_mut() + .find(|pending| registry_object(pending) == object) + { + Some(pending) if *pending != change => *pending = RegistryChange::Reload(object), + Some(_) => {} + None => self.registry_changes.push(change), + } + } + + fn snapshot(&mut self) { + if std::mem::take(&mut self.invalidate_all) { + self.push_ready(BroadcastEvent::CacheInvalidateAll); + } + while !self.invalidations.is_empty() { + let items = if self.invalidations.len() <= MAX_INVALIDATIONS_PER_EVENT { + self.invalidations.drain().collect::>() + } else { + let items = self + .invalidations + .iter() + .take(MAX_INVALIDATIONS_PER_EVENT) + .copied() + .collect::>(); + for item in &items { + self.invalidations.remove(item); + } + items + }; + self.push_ready(BroadcastEvent::CacheInvalidate(items)); + } + if std::mem::take(&mut self.invalidate_negative) { + self.push_ready(BroadcastEvent::CacheInvalidateNegative); + } + for change in std::mem::take(&mut self.registry_changes) { + self.push_ready(BroadcastEvent::RegistryChange(change)); + } + if let Some(is_running) = self.queue_status.take() { + self.push_ready(BroadcastEvent::MtaQueueStatus { is_running }); + } + for account_id in std::mem::take(&mut self.push_updates) { + self.push_ready(BroadcastEvent::PushServerUpdate(account_id)); + } + self.ready_bytes += std::mem::take(&mut self.events_bytes); + self.ready.append(&mut self.events); + self.type_states = 0; + for (key, states) in self.state_changes.drain() { + for state in states { + let change = StateChange { + account_id: key.account_id(), + change_id: state.change_id, + types: Bitmap::from(state.types), + }; + let notification = match key { + StateKey::Account(_) => PushNotification::StateChange(change), + StateKey::Viewer { viewer_id, .. } => { + PushNotification::ViewerStateChange(ViewerStateChange { viewer_id, change }) + } + }; + self.ready_bytes += EVENT_COST; + self.ready + .push_back(BroadcastEvent::PushNotification(notification)); + } + } + if !self.refresh_queues.is_empty() { + let queues = std::mem::take(&mut self.refresh_queues); + self.push_ready(BroadcastEvent::QueueRefresh(queues)); + } + if !self.refresh_tasks.is_empty() { + let task_types = std::mem::take(&mut self.refresh_tasks); + self.push_ready(BroadcastEvent::TaskQueueRefresh(task_types)); + } + if std::mem::take(&mut self.refresh_index) { + self.push_ready(BroadcastEvent::IndexQueueRefresh); + } + self.enforce_limits(); + } + + fn push_ready(&mut self, event: BroadcastEvent) { + self.ready_bytes += event_cost(&event); + self.ready.push_back(event); + } + + fn enforce_limits(&mut self) { + if self.len() > MAX_PENDING_EVENTS || self.bytes() > MAX_PENDING_BYTES { + self.relieve(); + } + } + + #[cold] + #[inline(never)] + fn relieve(&mut self) { + if !self.invalidate_all && !self.invalidations.is_empty() { + self.invalidate_all(); + } + if !self.degraded { + self.degrade_email_pushes(); + } + if self.len() <= MAX_PENDING_EVENTS && self.bytes() <= MAX_PENDING_BYTES { + return; + } + + let low_len = MAX_PENDING_EVENTS / LOW_WATER_DENOMINATOR * LOW_WATER_NUMERATOR; + let low_bytes = MAX_PENDING_BYTES / LOW_WATER_DENOMINATOR * LOW_WATER_NUMERATOR; + let mut dropped = 0u64; + let is_over = |backlog: &Self| backlog.len() > low_len || backlog.bytes() > low_bytes; + + if is_over(self) { + let ready_bytes = &mut self.ready_bytes; + self.ready.retain(|event| { + if matches!(event, BroadcastEvent::PushNotification(_)) { + *ready_bytes -= event_cost(event); + dropped += 1; + false + } else { + true + } + }); + } + if is_over(self) { + let mut excess_len = self.len().saturating_sub(low_len); + let mut excess_bytes = self.bytes().saturating_sub(low_bytes); + let mut removed_states = 0; + self.state_changes.retain(|_, states| { + if excess_len == 0 && excess_bytes == 0 { + return true; + } + excess_len = excess_len.saturating_sub(states.len()); + excess_bytes = + excess_bytes.saturating_sub(STATE_KEY_COST + states.len() * TYPE_STATE_COST); + removed_states += states.len(); + false + }); + self.type_states -= removed_states; + dropped += removed_states as u64; + } + while is_over(self) { + let Some(event) = self.events.pop_front() else { + break; + }; + self.events_bytes -= event_cost(&event); + dropped += 1; + } + if is_over(self) { + dropped += self.invalidations.len() as u64; + self.invalidations.clear(); + } + if dropped > 0 { + self.dropped += dropped; + self.invalidate_all(); + } + } + + fn degrade_email_pushes(&mut self) { + self.degraded = true; + let mut pushes = Vec::new(); + for (events, bytes) in [ + (&mut self.ready, &mut self.ready_bytes), + (&mut self.events, &mut self.events_bytes), + ] { + events.retain(|event| match event { + BroadcastEvent::PushNotification(PushNotification::EmailPush(push)) => { + pushes.push(push.to_state_change()); + *bytes -= EVENT_COST; + false + } + _ => true, + }); + } + for change in pushes { + self.push_state( + StateKey::Account(change.account_id), + *change.types, + change.change_id, + ); + } + } +} + +impl StateKey { + fn account_id(&self) -> u32 { + match self { + StateKey::Account(account_id) | StateKey::Viewer { account_id, .. } => *account_id, + } + } +} + +fn merge_state(states: &mut Vec, mut types: u64, change_id: u64) { + for state in states.iter_mut() { + let overlap = state.types & types; + if overlap != 0 { + if state.change_id >= change_id { + types &= !overlap; + } else { + state.types &= !overlap; + } + } + } + states.retain(|state| state.types != 0); + if types != 0 { + match states.iter_mut().find(|state| state.change_id == change_id) { + Some(state) => state.types |= types, + None => states.push(TypeState { types, change_id }), + } + } +} + +fn registry_object(change: &RegistryChange) -> ObjectType { + match change { + RegistryChange::Insert(id) | RegistryChange::Delete(id) => id.object(), + RegistryChange::Reload(object) => *object, + } +} + +fn event_cost(event: &BroadcastEvent) -> usize { + EVENT_COST + + match event { + BroadcastEvent::PushNotification(PushNotification::CalendarAlert(alert)) => { + alert.uid.capacity() + alert.alert_id.capacity() + } + BroadcastEvent::CacheInvalidate(items) => { + items.capacity() * size_of::() + } + _ => 0, + } +} + diff --git a/crates/services/src/broadcast/mod.rs b/crates/services/src/broadcast/mod.rs index 108b4130..e0df6538 100644 --- a/crates/services/src/broadcast/mod.rs +++ b/crates/services/src/broadcast/mod.rs @@ -4,20 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::ipc::{ - BroadcastEvent, CacheInvalidation, CalendarAlert, EmailPush, PushNotification, RegistryChange, +use common::{ + config::smtp::queue::QueueName, + ipc::{ + BroadcastEvent, CacheInvalidation, CalendarAlert, EmailPush, PushNotification, QueueNames, + RegistryChange, ViewerStateChange, + }, }; use registry::{ schema::prelude::ObjectType, types::{EnumImpl, id::ObjectId}, }; -use std::{borrow::Borrow, io::Write}; +use std::{collections::VecDeque, time::Duration}; +use store::write::TaskTypes; +use tokio::sync::mpsc; use types::{id::Id, type_state::StateChange}; use utils::{ - codec::leb128::{Leb128Iterator, Leb128Writer}, + codec::leb128::{Leb128Reader, Leb128Writer}, map::bitmap::Bitmap, }; +pub mod backlog; pub mod publisher; pub mod subscriber; @@ -26,8 +33,13 @@ pub(crate) struct BroadcastBatch { messages: T, } -const MAX_BATCH_SIZE: usize = 100; +pub(crate) const MAX_BATCH_SIZE: usize = 100; pub(crate) const BROADCAST_TOPIC: &str = "stwt.agora"; +pub(crate) const RETRY_INITIAL: Duration = Duration::from_millis(100); +pub(crate) const RETRY_MAX: Duration = Duration::from_secs(5); + +const STATE_CHANGE: u8 = 0; +const VIEWER_STATE_CHANGE: u8 = 15; impl BroadcastBatch> { pub fn init() -> Self { @@ -36,91 +48,161 @@ impl BroadcastBatch> { } } - pub fn insert(&mut self, message: BroadcastEvent) -> bool { - self.messages.push(message); - self.messages.len() < MAX_BATCH_SIZE + pub async fn recv(&mut self, rx: &mut mpsc::Receiver) -> bool { + if rx.recv_many(&mut self.messages, MAX_BATCH_SIZE).await == 0 { + return false; + } + self.dedup_refreshes(); + true } - pub fn serialize(&self, node_id: u16) -> Vec { - let mut serialized = - Vec::with_capacity((self.messages.len() * 10) + std::mem::size_of::()); - let _ = serialized.write_leb128(node_id); - for message in &self.messages { + pub fn take_from(&mut self, queue: &mut VecDeque) { + let count = queue.len().min(MAX_BATCH_SIZE); + self.messages.extend(queue.drain(..count)); + self.dedup_refreshes(); + } + + fn dedup_refreshes(&mut self) { + let mut seen_refreshes = 0u8; + let mut task_types = TaskTypes::default(); + let mut queues = QueueNames::default(); + self.messages.retain(|message| { + let refresh = match message { + BroadcastEvent::TaskQueueRefresh(types) => { + task_types.union(*types); + REFRESH_TASK + } + BroadcastEvent::QueueRefresh(names) => { + queues.merge(names); + REFRESH_QUEUE + } + BroadcastEvent::IndexQueueRefresh => REFRESH_INDEX, + _ => return true, + }; + let is_first = seen_refreshes & refresh == 0; + seen_refreshes |= refresh; + is_first + }); + + if seen_refreshes & (REFRESH_TASK | REFRESH_QUEUE) != 0 { + for message in &mut self.messages { + match message { + BroadcastEvent::TaskQueueRefresh(types) => *types = task_types, + BroadcastEvent::QueueRefresh(names) => *names = std::mem::take(&mut queues), + _ => {} + } + } + } + } + + pub fn clear(&mut self) { + self.messages.clear(); + } +} + +impl> BroadcastBatch { + pub fn serialize(&self, node_id: u16, read_version: Option, serialized: &mut Vec) { + let messages = self.messages.as_ref(); + serialized.clear(); + serialized.reserve( + (messages.len() * 10) + std::mem::size_of::() + std::mem::size_of::(), + ); + serialized.push_leb128(node_id); + serialized.push_leb128(read_version.unwrap_or_default()); + for message in messages { match message { BroadcastEvent::PushNotification(notification) => match notification { PushNotification::StateChange(state_change) => { - serialized.push(0u8); - let _ = serialized.write_leb128(state_change.change_id); - let _ = serialized.write_leb128(*state_change.types.as_ref()); - let _ = serialized.write_leb128(state_change.account_id); + serialized.push(STATE_CHANGE); + serialized.push_leb128(state_change.change_id); + serialized.push_leb128(*state_change.types.as_ref()); + serialized.push_leb128(state_change.account_id); + } + PushNotification::ViewerStateChange(viewer_change) => { + serialized.push(VIEWER_STATE_CHANGE); + serialized.push_leb128(viewer_change.change.change_id); + serialized.push_leb128(*viewer_change.change.types.as_ref()); + serialized.push_leb128(viewer_change.change.account_id); + serialized.push_leb128(viewer_change.viewer_id); } PushNotification::CalendarAlert(calendar_alert) => { serialized.push(1u8); - let _ = serialized.write_leb128(calendar_alert.account_id); - let _ = serialized.write_leb128(calendar_alert.event_id); - let _ = serialized - .write_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64); - let _ = serialized.write_leb128(calendar_alert.uid.len()); - let _ = serialized.write(calendar_alert.uid.as_bytes()); - let _ = serialized.write_leb128(calendar_alert.alert_id.len()); - let _ = serialized.write(calendar_alert.alert_id.as_bytes()); + serialized.push_leb128(calendar_alert.account_id); + serialized.push_leb128(calendar_alert.event_account_id); + serialized.push_leb128(calendar_alert.event_id); + serialized + .push_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64); + serialized.push_leb128(calendar_alert.uid.len()); + serialized.extend_from_slice(calendar_alert.uid.as_bytes()); + serialized.push_leb128(calendar_alert.alert_id.len()); + serialized.extend_from_slice(calendar_alert.alert_id.as_bytes()); } PushNotification::EmailPush(email_push) => { serialized.push(2u8); - let _ = serialized.write_leb128(email_push.account_id); - let _ = serialized.write_leb128(email_push.email_id); - let _ = serialized.write_leb128(email_push.change_id); + serialized.push_leb128(email_push.account_id); + serialized.push_leb128(email_push.email_id); + serialized.push_leb128(email_push.change_id); } }, BroadcastEvent::PushServerUpdate(account_id) => { serialized.push(3u8); - let _ = serialized.write_leb128(*account_id); + serialized.push_leb128(*account_id); } BroadcastEvent::RegistryChange(items) => match items { RegistryChange::Insert(id) => { serialized.push(4u8); - let _ = serialized.write_leb128(id.object().to_id()); - let _ = serialized.write_leb128(id.id().id()); + serialized.push_leb128(id.object().to_id()); + serialized.push_leb128(id.id().id()); } RegistryChange::Delete(id) => { serialized.push(5u8); - let _ = serialized.write_leb128(id.object().to_id()); - let _ = serialized.write_leb128(id.id().id()); + serialized.push_leb128(id.object().to_id()); + serialized.push_leb128(id.id().id()); } RegistryChange::Reload(object) => { serialized.push(6u8); - let _ = serialized.write_leb128(object.to_id()); + serialized.push_leb128(object.to_id()); } }, BroadcastEvent::CacheInvalidate(items) => { serialized.push(7u8); - let _ = serialized.write_leb128(items.len()); + serialized.push_leb128(items.len()); for item in items { let (marker, id) = match item { CacheInvalidation::AccessToken(id) => (0u8, *id), - CacheInvalidation::DavResources(id) => (1u8, *id), + CacheInvalidation::GroupwareResources(id) => (1u8, *id), CacheInvalidation::Domain(id) => (2u8, *id), CacheInvalidation::Account(id) => (3u8, *id), CacheInvalidation::DkimSignature(id) => (4u8, *id), - CacheInvalidation::Tenant(id) => (5u8, *id), CacheInvalidation::Role(id) => (6u8, *id), CacheInvalidation::List(id) => (7u8, *id), CacheInvalidation::DomainLogo(id) => (8u8, *id), - CacheInvalidation::TenantLogo(id) => (9u8, *id), CacheInvalidation::EmailNegative { domain_id, local_part_hash, } => { serialized.push(10u8); - let _ = serialized.write_leb128(*domain_id); - let _ = serialized.write_leb128(*local_part_hash); + serialized.push_leb128(*domain_id); + serialized.push_leb128(*local_part_hash); continue; } CacheInvalidation::DomainNegative => (11u8, 0), + CacheInvalidation::MessageCache(id) => (12u8, *id), + CacheInvalidation::PrivateMetadata(id) => (13u8, *id), + CacheInvalidation::OAuthGrant { + account_id, + grant_id, + } => { + serialized.push(14u8); + serialized.push_leb128(*account_id); + serialized.push_leb128(*grant_id); + continue; + } + CacheInvalidation::ActiveSieveScript(id) => (15u8, *id), }; serialized.push(marker); - let _ = serialized.write_leb128(id); + serialized.push_leb128(id); } } BroadcastEvent::CacheInvalidateAll => { @@ -136,58 +218,74 @@ impl BroadcastBatch> { serialized.push(11u8); } } - BroadcastEvent::QueueRefresh => { + BroadcastEvent::QueueRefresh(queues) => { serialized.push(12u8); + serialized.push_leb128(queues.len()); + for queue in queues.iter() { + serialized.extend_from_slice(queue.as_slice()); + } + } + BroadcastEvent::TaskQueueRefresh(task_types) => { + serialized.push(13u8); + serialized.push_leb128(task_types.bits()); + } + BroadcastEvent::IndexQueueRefresh => { + serialized.push(14u8); } } } - serialized - } - - pub fn clear(&mut self) { - self.messages.clear(); } } -impl BroadcastBatch -where - T: Iterator + Leb128Iterator, - I: Borrow, -{ +const REFRESH_TASK: u8 = 1; +const REFRESH_INDEX: u8 = 2; +const REFRESH_QUEUE: u8 = 4; + +impl BroadcastBatch<&[u8]> { pub fn node_id(&mut self) -> Option { - self.messages.next_leb128::() + self.messages.read_leb128::() + } + + pub fn read_version(&mut self) -> Option> { + self.messages + .read_leb128::() + .map(|version| Some(version).filter(|version| *version > 0)) } pub fn next_event(&mut self) -> Result, ()> { - if let Some(id) = self.messages.next() { - match id.borrow() { - 0 => Ok(Some(BroadcastEvent::PushNotification( - PushNotification::StateChange(StateChange { - change_id: self.messages.next_leb128().ok_or(())?, - types: Bitmap::from(self.messages.next_leb128::().ok_or(())?), - account_id: self.messages.next_leb128().ok_or(())?, - }), - ))), + if let Some(id) = self.next_byte() { + match id { + tag @ (STATE_CHANGE | VIEWER_STATE_CHANGE) => { + let change_id = self.messages.read_leb128().ok_or(())?; + let types = Bitmap::from(self.messages.read_leb128::().ok_or(())?); + let account_id = self.messages.read_leb128().ok_or(())?; + let change = StateChange { + account_id, + change_id, + types, + }; + let notification = if tag == VIEWER_STATE_CHANGE { + PushNotification::ViewerStateChange(ViewerStateChange { + viewer_id: self.messages.read_leb128().ok_or(())?, + change, + }) + } else { + PushNotification::StateChange(change) + }; + Ok(Some(BroadcastEvent::PushNotification(notification))) + } 1 => { - let account_id = self.messages.next_leb128().ok_or(())?; - let event_id = self.messages.next_leb128().ok_or(())?; - let recurrence_id = self.messages.next_leb128::().ok_or(())? as i64; - let uid_len = self.messages.next_leb128::().ok_or(())?; - let mut uid_bytes = vec![0u8; uid_len]; - for byte in uid_bytes.iter_mut() { - *byte = self.messages.next().ok_or(())?.borrow().to_owned(); - } - let uid = String::from_utf8(uid_bytes).map_err(|_| ())?; - let alert_id_len = self.messages.next_leb128::().ok_or(())?; - let mut alert_id_bytes = vec![0u8; alert_id_len]; - for byte in alert_id_bytes.iter_mut() { - *byte = self.messages.next().ok_or(())?.borrow().to_owned(); - } - let alert_id = String::from_utf8(alert_id_bytes).map_err(|_| ())?; + let account_id = self.messages.read_leb128().ok_or(())?; + let event_account_id = self.messages.read_leb128().ok_or(())?; + let event_id = self.messages.read_leb128().ok_or(())?; + let recurrence_id = self.messages.read_leb128::().ok_or(())? as i64; + let uid = self.next_string()?; + let alert_id = self.next_string()?; Ok(Some(BroadcastEvent::PushNotification( PushNotification::CalendarAlert(CalendarAlert { account_id, + event_account_id, event_id, recurrence_id: if recurrence_id == 0 { None @@ -201,18 +299,18 @@ where } 2 => Ok(Some(BroadcastEvent::PushNotification( PushNotification::EmailPush(EmailPush { - account_id: self.messages.next_leb128().ok_or(())?, - email_id: self.messages.next_leb128().ok_or(())?, - change_id: self.messages.next_leb128().ok_or(())?, + account_id: self.messages.read_leb128().ok_or(())?, + email_id: self.messages.read_leb128().ok_or(())?, + change_id: self.messages.read_leb128().ok_or(())?, }), ))), 3 => { - let account_id = self.messages.next_leb128().ok_or(())?; + let account_id = self.messages.read_leb128().ok_or(())?; Ok(Some(BroadcastEvent::PushServerUpdate(account_id))) } 4 => { - let object_id = self.messages.next_leb128().ok_or(())?; - let id = self.messages.next_leb128::().ok_or(())?; + let object_id = self.messages.read_leb128().ok_or(())?; + let id = self.messages.read_leb128::().ok_or(())?; Ok(Some(BroadcastEvent::RegistryChange( RegistryChange::Insert(ObjectId::new( ObjectType::from_id(object_id).ok_or(())?, @@ -221,8 +319,8 @@ where ))) } 5 => { - let object_id = self.messages.next_leb128().ok_or(())?; - let id = self.messages.next_leb128::().ok_or(())?; + let object_id = self.messages.read_leb128().ok_or(())?; + let id = self.messages.read_leb128::().ok_or(())?; Ok(Some(BroadcastEvent::RegistryChange( RegistryChange::Delete(ObjectId::new( ObjectType::from_id(object_id).ok_or(())?, @@ -231,37 +329,42 @@ where ))) } 6 => { - let object_id = self.messages.next_leb128().ok_or(())?; + let object_id = self.messages.read_leb128().ok_or(())?; Ok(Some(BroadcastEvent::RegistryChange( RegistryChange::Reload(ObjectType::from_id(object_id).ok_or(())?), ))) } 7 => { - let count = self.messages.next_leb128::().ok_or(())?; - let mut items = Vec::with_capacity(count); + let count = self.messages.read_leb128::().ok_or(())?; + let mut items = Vec::with_capacity(count.min(self.messages.len())); for _ in 0..count { - let marker = self.messages.next().ok_or(())?.borrow().to_owned(); - let id = self.messages.next_leb128::().ok_or(())?; + let marker = self.next_byte().ok_or(())?; + let id = self.messages.read_leb128::().ok_or(())?; items.push(match marker { 0 => CacheInvalidation::AccessToken(id), - 1 => CacheInvalidation::DavResources(id), + 1 => CacheInvalidation::GroupwareResources(id), 2 => CacheInvalidation::Domain(id), 3 => CacheInvalidation::Account(id), 4 => CacheInvalidation::DkimSignature(id), - 5 => CacheInvalidation::Tenant(id), 6 => CacheInvalidation::Role(id), 7 => CacheInvalidation::List(id), 8 => CacheInvalidation::DomainLogo(id), - 9 => CacheInvalidation::TenantLogo(id), 10 => { let local_part_hash = - self.messages.next_leb128::().ok_or(())?; + self.messages.read_leb128::().ok_or(())?; CacheInvalidation::EmailNegative { domain_id: id, local_part_hash, } } 11 => CacheInvalidation::DomainNegative, + 12 => CacheInvalidation::MessageCache(id), + 13 => CacheInvalidation::PrivateMetadata(id), + 14 => CacheInvalidation::OAuthGrant { + account_id: id, + grant_id: self.messages.read_leb128::().ok_or(())?, + }, + 15 => CacheInvalidation::ActiveSieveScript(id), _ => return Err(()), }); } @@ -271,13 +374,42 @@ where 9 => Ok(Some(BroadcastEvent::CacheInvalidateNegative)), 10 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: true })), 11 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: false })), - 12 => Ok(Some(BroadcastEvent::QueueRefresh)), + 12 => { + let count = self.messages.read_leb128::().ok_or(())?; + if count > self.messages.len() / size_of::() { + return Err(()); + } + let mut queues = QueueNames::default(); + for _ in 0..count { + let (name, rest) = self.messages.split_first_chunk::<8>().ok_or(())?; + self.messages = rest; + queues.insert(QueueName::from(*name)); + } + Ok(Some(BroadcastEvent::QueueRefresh(queues))) + } + 13 => Ok(Some(BroadcastEvent::TaskQueueRefresh( + TaskTypes::from_bits(self.messages.read_leb128::().ok_or(())?), + ))), + 14 => Ok(Some(BroadcastEvent::IndexQueueRefresh)), _ => Err(()), } } else { Ok(None) } } + + fn next_byte(&mut self) -> Option { + let (&byte, rest) = self.messages.split_first()?; + self.messages = rest; + Some(byte) + } + + fn next_string(&mut self) -> Result { + let len = self.messages.read_leb128::().ok_or(())?; + let (bytes, rest) = self.messages.split_at_checked(len).ok_or(())?; + self.messages = rest; + String::from_utf8(bytes.to_vec()).map_err(|_| ()) + } } impl BroadcastBatch { @@ -285,3 +417,4 @@ impl BroadcastBatch { Self { messages } } } + diff --git a/crates/services/src/broadcast/publisher.rs b/crates/services/src/broadcast/publisher.rs index cf81f287..3411bf74 100644 --- a/crates/services/src/broadcast/publisher.rs +++ b/crates/services/src/broadcast/publisher.rs @@ -4,52 +4,211 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - -use common::{Inner, ipc::BroadcastEvent}; +use super::{ + BROADCAST_TOPIC, BroadcastBatch, MAX_BATCH_SIZE, RETRY_INITIAL, RETRY_MAX, backlog::Backlog, +}; +use common::{IPC_CHANNEL_BUFFER, Inner, ipc::BroadcastEvent}; +use coordinator::Publisher; +use std::{collections::VecDeque, future::Future, sync::Arc}; use tokio::sync::mpsc; use trc::ClusterEvent; -use super::{BROADCAST_TOPIC, BroadcastBatch}; +const MAX_PAYLOAD_BYTES: usize = 512 * 1024; + +struct BatchPublisher { + inner: Arc, + publisher: Publisher, + payload: Vec, + node_id: u16, +} + +struct Intake<'x> { + event_rx: &'x mut mpsc::Receiver, + received: Vec, + queued: VecDeque, + backlog: Backlog, + is_open: bool, +} pub fn spawn_broadcast_publisher(inner: Arc, mut event_rx: mpsc::Receiver) { - let (coordinator, this_node_id) = { - let _core = inner.shared_core.load(); - let coordinator = inner.shared_core.load().storage.coordinator.clone(); - if coordinator.is_none() { + let (publisher, node_id) = { + let core = inner.shared_core.load(); + let Some(publisher) = core.storage.coordinator.publisher(BROADCAST_TOPIC) else { return; - } - (coordinator, _core.network.node_id as u16) + }; + (publisher, core.network.node_id as u16) }; tokio::spawn(async move { let mut batch = BroadcastBatch::init(); + let mut publisher = BatchPublisher { + inner, + publisher, + payload: Vec::new(), + node_id, + }; + let mut intake = Intake::new(&mut event_rx); + let mut backoff = RETRY_INITIAL; trc::event!(Cluster(ClusterEvent::PublisherStart)); - while let Some(event) = event_rx.recv().await { - batch.insert(event); - - while let Ok(event) = event_rx.try_recv() { - if !batch.insert(event) { - break; - } - } - - match coordinator - .publish(BROADCAST_TOPIC, batch.serialize(this_node_id)) - .await - { - Ok(_) => { - batch.clear(); + while intake.next_batch(&mut batch).await { + match intake.absorb(publisher.publish(&mut batch)).await { + Ok(()) => { + intake.backlog.release(&mut batch); + backoff = RETRY_INITIAL; } Err(err) => { - batch.clear(); trc::event!(Cluster(ClusterEvent::PublisherError), CausedBy = err); + intake.absorb(tokio::time::sleep(backoff)).await; + backoff = (backoff * 2).min(RETRY_MAX); } } + intake.report_dropped(); } trc::event!(Cluster(ClusterEvent::PublisherStop)); }); } + +impl BatchPublisher { + async fn publish( + &mut self, + batch: &mut BroadcastBatch>, + ) -> trc::Result<()> { + let read_version = self + .inner + .shared_core + .load() + .storage + .data + .read_snapshot_version(); + batch.serialize(self.node_id, read_version, &mut self.payload); + if self.payload.len() <= MAX_PAYLOAD_BYTES { + self.publisher.publish(&mut self.payload).await + } else { + self.publish_split(&mut batch.messages, read_version).await + } + } + + #[cold] + async fn publish_split( + &mut self, + events: &mut Vec, + read_version: Option, + ) -> trc::Result<()> { + while let Some(count) = next_chunk( + events, + self.node_id, + read_version, + &mut self.payload, + MAX_PAYLOAD_BYTES, + ) { + self.publisher.publish(&mut self.payload).await?; + events.drain(..count); + } + Ok(()) + } +} + +fn next_chunk( + events: &mut Vec, + node_id: u16, + read_version: Option, + payload: &mut Vec, + max_bytes: usize, +) -> Option { + while !events.is_empty() { + let mut count = events.len(); + loop { + BroadcastBatch { + messages: &events[..count], + } + .serialize(node_id, read_version, payload); + if payload.len() <= max_bytes || count == 1 { + break; + } + count /= 2; + } + if payload.len() <= max_bytes { + return Some(count); + } + match events.remove(0) { + BroadcastEvent::CacheInvalidate(mut items) if items.len() > 1 => { + let tail = items.split_off(items.len() / 2); + events.insert(0, BroadcastEvent::CacheInvalidate(tail)); + events.insert(0, BroadcastEvent::CacheInvalidate(items)); + } + _ => { + trc::event!( + Cluster(ClusterEvent::PublisherError), + Details = "Dropped a broadcast event larger than the maximum payload", + Size = payload.len(), + ); + } + } + } + None +} + +impl<'x> Intake<'x> { + fn new(event_rx: &'x mut mpsc::Receiver) -> Self { + Intake { + event_rx, + received: Vec::with_capacity(MAX_BATCH_SIZE), + queued: VecDeque::new(), + backlog: Backlog::default(), + is_open: true, + } + } + + async fn next_batch(&mut self, batch: &mut BroadcastBatch>) -> bool { + if !batch.messages.is_empty() { + return true; + } + if !self.queued.is_empty() { + batch.take_from(&mut self.queued); + return true; + } + if self.backlog.fill(batch) || (self.is_open && batch.recv(self.event_rx).await) { + return true; + } + self.is_open = false; + false + } + + async fn absorb(&mut self, until: impl Future) -> T { + tokio::pin!(until); + loop { + tokio::select! { + biased; + result = &mut until => return result, + count = self.event_rx.recv_many(&mut self.received, MAX_BATCH_SIZE), if self.is_open => { + if count == 0 { + self.is_open = false; + } + for event in self.received.drain(..) { + if self.queued.len() < IPC_CHANNEL_BUFFER && self.backlog.is_empty() { + self.queued.push_back(event); + } else { + self.backlog.push(event); + } + } + } + } + } + } + + fn report_dropped(&mut self) { + let dropped = self.backlog.take_dropped(); + if dropped > 0 { + trc::event!( + Cluster(ClusterEvent::PublisherError), + Details = "Dropped the oldest pending broadcast events", + Total = dropped, + Size = self.backlog.bytes(), + ); + } + } +} + diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index fbcd4bc1..2452335b 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -4,32 +4,61 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch}; +use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch, RETRY_INITIAL, RETRY_MAX}; use common::{ - BuildServer, Inner, - ipc::{BroadcastEvent, PushEvent, PushNotification, QueueEvent, RegistryChange}, + BuildServer, IPC_CHANNEL_BUFFER, Inner, + ipc::{BroadcastEvent, PushEvent, PushNotification, QueueEvent, QueueRefresh, RegistryChange}, }; +use compact_str::format_compact; use registry::types::EnumImpl; -use std::{sync::Arc, time::Duration}; -use tokio::sync::watch; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use store::{ + ahash::AHashSet, + rand::{self, RngExt}, + write::TaskTypes, +}; +use tokio::sync::{mpsc, watch}; use trc::{ClusterEvent, ServerEvent}; +const STABLE_SUBSCRIPTION: Duration = Duration::from_secs(1); + +struct BroadcastReceiver { + inner: Arc, + node_id: u16, + registry_tx: mpsc::Sender, +} + pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Receiver) { - let this_node_id = { - let _core = inner.shared_core.load(); - if _core.storage.coordinator.is_none() || _core.storage.registry.is_recovery_mode() { + let node_id = { + let core = inner.shared_core.load(); + if core.storage.coordinator.is_none() || core.storage.registry.is_recovery_mode() { return; } - _core.network.node_id as u16 + core.network.node_id as u16 }; tokio::spawn(async move { + let receiver = BroadcastReceiver { + registry_tx: spawn_registry_worker(inner.clone()), + inner, + node_id, + }; let mut retry_count = 0; + let mut is_resubscribe = false; trc::event!(Cluster(ClusterEvent::SubscriberStart)); 'subscribe: loop { - let coordinator = inner.shared_core.load().storage.coordinator.clone(); + let coordinator = receiver + .inner + .shared_core + .load() + .storage + .coordinator + .clone(); if coordinator.is_none() { trc::event!( Cluster(ClusterEvent::SubscriberError), @@ -38,10 +67,35 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec break; } - let mut stream = match coordinator.subscribe(BROADCAST_TOPIC).await { - Ok(stream) => { - retry_count = 0; - stream + match coordinator.subscribe(BROADCAST_TOPIC).await { + Ok(mut subscription) => { + if std::mem::replace(&mut is_resubscribe, true) { + receiver.catch_up().await; + } + let subscribed_at = Instant::now(); + let mut has_received = false; + let mut stream = subscription.stream(); + loop { + tokio::select! { + message = stream.next() => match message { + Some(message) => { + has_received = true; + receiver.receive(message.payload()).await; + } + None => { + trc::event!(Cluster(ClusterEvent::SubscriberDisconnected)); + break; + } + }, + _ = shutdown_rx.changed() => { + break 'subscribe; + } + } + } + if has_received || subscribed_at.elapsed() >= STABLE_SUBSCRIPTION { + retry_count = 0; + continue; + } } Err(err) => { trc::event!( @@ -49,165 +103,12 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec CausedBy = err, Details = "Failed to subscribe to channel" ); - - match tokio::time::timeout( - Duration::from_secs(1 << retry_count.max(6)), - shutdown_rx.changed(), - ) - .await - { - Ok(_) => { - break; - } - Err(_) => { - retry_count += 1; - continue; - } - } } - }; + } - loop { - tokio::select! { - message = stream.next() => { - match message { - Some(message) => { - let mut batch = BroadcastBatch::new(message.payload().iter()); - let node_id = match batch.node_id() { - Some(node_id) => { - if node_id != this_node_id { - node_id - } else { - trc::event!( - Cluster(ClusterEvent::MessageSkipped), - Details = message.payload() - ); - continue; - } - } - None => { - trc::event!( - Cluster(ClusterEvent::MessageInvalid), - Details = message.payload() - ); - continue; - } - }; - - inner - .shared_core - .load() - .storage - .data - .invalidate_read_snapshot(); - - loop { - match batch.next_event() { - Ok(Some(event)) => { - trc::event!( - Cluster(ClusterEvent::MessageReceived), - From = node_id, - To = this_node_id, - Details = log_event(&event), - ); - match event { - BroadcastEvent::PushNotification(notification) => { - if inner - .ipc - .push_tx - .send(PushEvent::Publish { - notification, - broadcast: false, - }) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending push notification.", - CausedBy = trc::location!() - ); - } - } - BroadcastEvent::PushServerUpdate(account_id) => { - if inner - .ipc - .push_tx - .send(PushEvent::PushServerUpdate { account_id, broadcast: false }) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending reload request.", - CausedBy = trc::location!() - ); - } - } - BroadcastEvent::CacheInvalidate(changes) => { - inner.build_server().invalidate_local_caches(&changes).await; - - } - BroadcastEvent::CacheInvalidateAll => { - inner.build_server().invalidate_all_local_caches(); - } - BroadcastEvent::CacheInvalidateNegative => { - inner.build_server().invalidate_all_local_negative_caches(); - } - BroadcastEvent::MtaQueueStatus { is_running } => { - let _ = inner - .ipc - .queue_tx - .send(QueueEvent::Paused(!is_running)) - .await; - } - BroadcastEvent::QueueRefresh => { - if inner.shared_core.load().network.roles.outbound_mta { - let _ = inner - .ipc - .queue_tx - .send(QueueEvent::Refresh) - .await; - } - } - BroadcastEvent::RegistryChange(change) => { - match Box::pin(inner.build_server().reload_registry(change)).await { - Ok(result) => { - result.log(); - } - Err(err) => { - trc::error!( - err.details("Failed to reload settings") - .caused_by(trc::location!()) - ); - } - } - } - } - } - Ok(None) => break, - Err(_) => { - trc::event!( - Cluster(ClusterEvent::MessageInvalid), - Details = message.payload() - ); - break; - } - } - } - } - None => { - trc::event!( - Cluster(ClusterEvent::SubscriberDisconnected), - ); - break; - } - } - }, - _ = shutdown_rx.changed() => { - break 'subscribe; - } - }; + match tokio::time::timeout(retry_delay(retry_count), shutdown_rx.changed()).await { + Ok(_) => break, + Err(_) => retry_count += 1, } } @@ -215,6 +116,171 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec }); } +fn retry_delay(retry_count: u32) -> Duration { + (RETRY_INITIAL * 4 * (1 << retry_count.min(6))) + .min(RETRY_MAX) + .mul_f64(rand::rng().random_range(0.75..=1.0)) +} + +impl BroadcastReceiver { + async fn catch_up(&self) { + let inner = &self.inner; + inner.build_server().invalidate_all_local_caches(); + inner.ipc.task_tx.notify(TaskTypes::ALL); + inner.ipc.index_tx.notify_one(); + if inner.shared_core.load().network.roles.outbound_mta { + let _ = inner + .ipc + .queue_tx + .send(QueueEvent::Refresh(QueueRefresh::All)) + .await; + } + } + + async fn receive(&self, payload: &[u8]) { + let mut batch = BroadcastBatch::new(payload); + let node_id = match (batch.node_id(), batch.read_version()) { + (Some(node_id), Some(read_version)) if node_id != self.node_id => { + let core = self.inner.shared_core.load(); + match read_version { + Some(version) => core.storage.data.advance_read_snapshot(version), + None => core.storage.data.invalidate_read_snapshot(), + } + node_id + } + (Some(_), Some(_)) => { + trc::event!(Cluster(ClusterEvent::MessageSkipped), Contents = payload); + return; + } + _ => { + trc::event!(Cluster(ClusterEvent::MessageInvalid), Contents = payload); + return; + } + }; + + loop { + match batch.next_event() { + Ok(Some(event)) => { + trc::event!( + Cluster(ClusterEvent::MessageReceived), + RemoteNodeId = node_id, + NodeId = self.node_id, + Contents = log_event(&event), + ); + self.apply(event).await; + tokio::task::consume_budget().await; + } + Ok(None) => break, + Err(_) => { + trc::event!(Cluster(ClusterEvent::MessageInvalid), Contents = payload); + break; + } + } + } + } + + async fn apply(&self, event: BroadcastEvent) { + let inner = &self.inner; + match event { + BroadcastEvent::PushNotification(notification) => { + inner.mark_caches_stale_for(¬ification); + inner.publish_push_notification(notification).await; + } + BroadcastEvent::PushServerUpdate(account_id) => { + if inner + .ipc + .push_tx + .send(PushEvent::PushServerUpdate { + account_id, + broadcast: false, + }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending reload request.", + CausedBy = trc::location!() + ); + } + } + BroadcastEvent::CacheInvalidate(changes) => { + inner.build_server().invalidate_local_caches(&changes); + } + BroadcastEvent::CacheInvalidateAll => { + inner.build_server().invalidate_all_local_caches(); + } + BroadcastEvent::CacheInvalidateNegative => { + inner.build_server().invalidate_all_local_negative_caches(); + } + BroadcastEvent::MtaQueueStatus { is_running } => { + let _ = inner + .ipc + .queue_tx + .send(QueueEvent::Paused(!is_running)) + .await; + } + BroadcastEvent::QueueRefresh(queues) => { + if inner.shared_core.load().network.roles.outbound_mta { + let _ = inner + .ipc + .queue_tx + .send(QueueEvent::Refresh(QueueRefresh::Queues(queues))) + .await; + } + } + BroadcastEvent::TaskQueueRefresh(task_types) => { + inner.ipc.task_tx.notify(task_types); + } + BroadcastEvent::IndexQueueRefresh => { + inner.ipc.index_tx.notify_one(); + } + BroadcastEvent::RegistryChange(change) => { + if self.registry_tx.send(change).await.is_err() { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending registry change.", + CausedBy = trc::location!() + ); + } + } + } + } +} + +fn spawn_registry_worker(inner: Arc) -> mpsc::Sender { + let (registry_tx, mut registry_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); + + tokio::spawn(async move { + let mut changes = Vec::new(); + let mut applied = AHashSet::new(); + while registry_rx + .recv_many(&mut changes, IPC_CHANNEL_BUFFER) + .await + > 0 + { + for change in changes.drain(..) { + if applied.insert(change) { + match inner.build_server().reload_registry(change).await { + Ok(result) => { + result.log(); + } + Err(err) => { + trc::error!( + err.details("Failed to reload settings") + .caused_by(trc::location!()) + ); + } + } + } + } + applied.clear(); + } + }); + + registry_tx +} + fn log_event(event: &BroadcastEvent) -> trc::Value { match event { BroadcastEvent::PushNotification(notification) => match notification { @@ -224,6 +290,13 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { state_change.change_id.into(), (*state_change.types.as_ref()).into(), ]), + PushNotification::ViewerStateChange(viewer_change) => trc::Value::Array(vec![ + "ViewerStateChange".into(), + viewer_change.change.account_id.into(), + viewer_change.viewer_id.into(), + viewer_change.change.change_id.into(), + (*viewer_change.change.types.as_ref()).into(), + ]), PushNotification::CalendarAlert(calendar_alert) => trc::Value::Array(vec![ "CalendarAlert".into(), calendar_alert.account_id.into(), @@ -261,7 +334,7 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { let mut array = Vec::with_capacity(items.len() + 1); array.push("CacheInvalidation".into()); for item in items { - array.push(format!("{:?}", item).into()); + array.push(format_compact!("{:?}", item).into()); } trc::Value::Array(array) } @@ -274,6 +347,8 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { "MtaQueuePaused".into() } } - BroadcastEvent::QueueRefresh => "QueueRefresh".into(), + BroadcastEvent::QueueRefresh(_) => "QueueRefresh".into(), + BroadcastEvent::TaskQueueRefresh(_) => "TaskQueueRefresh".into(), + BroadcastEvent::IndexQueueRefresh => "IndexQueueRefresh".into(), } } diff --git a/crates/services/src/index_queue/document.rs b/crates/services/src/index_queue/document.rs new file mode 100644 index 00000000..3f69b286 --- /dev/null +++ b/crates/services/src/index_queue/document.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Core, Server}; +use email::message::metadata::MetadataRow; +use groupware::{calendar::CalendarEventContent, contact::ContactCardContent}; +use store::{ + ValueKey, + search::IndexDocument, + write::{Archive, ArchiveBytes, SearchIndex}, +}; +use trc::AddContext; +use types::{ + collection::Collection, + field::{CalendarEventField, ContactField}, +}; + +pub(crate) fn is_email_indexed(server: &Server) -> bool { + server + .core + .email + .index_fields + .contains_key(&SearchIndex::Email) +} + +pub(crate) async fn build_email_document( + server: &Server, + account_id: u32, + document_id: u32, + row: MetadataRow, +) -> trc::Result { + let metadata = row.unarchive().caused_by(trc::location!())?; + let raw_message = server + .blob_store() + .get_blob(metadata.blob_hash().as_slice(), 0..usize::MAX) + .await + .caused_by(trc::location!())? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .details("Blob not found") + })?; + + if !metadata.has_extractable_attachments() { + return email_document(&server.core, &row, account_id, document_id, &raw_message); + } + + let core = server.core.clone(); + tokio::task::spawn_blocking(move || { + email_document(&core, &row, account_id, document_id, &raw_message) + }) + .await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .details("Email indexing task failed") + .caused_by(trc::location!()) + })? +} + +fn email_document( + core: &Core, + row: &MetadataRow, + account_id: u32, + document_id: u32, + raw_message: &[u8], +) -> trc::Result { + let index_fields = core + .email + .index_fields + .get(&SearchIndex::Email) + .ok_or_else(|| trc::StoreEvent::UnexpectedError.into_err())?; + let headers = row.raw_headers().caused_by(trc::location!())?; + Ok(row.unarchive().caused_by(trc::location!())?.index_document( + account_id, + document_id, + &headers, + raw_message, + index_fields, + core.email.default_language, + &core.email.extract_limits, + )) +} + +pub(crate) async fn build_calendar_document( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result> { + let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Calendar) else { + return Ok(None); + }; + + match server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await? + { + Some(metadata_) => Ok(Some( + metadata_ + .unarchive::() + .caused_by(trc::location!())? + .index_document( + account_id, + document_id, + index_fields, + server.core.email.default_language, + ), + )), + None => Ok(None), + } +} + +pub(crate) async fn build_contact_document( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result> { + let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Contacts) else { + return Ok(None); + }; + + match server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::ContactCard, + document_id, + ContactField::Content, + )) + .await? + { + Some(metadata_) => Ok(Some( + metadata_ + .unarchive::() + .caused_by(trc::location!())? + .index_document( + account_id, + document_id, + index_fields, + server.core.email.default_language, + ), + )), + None => Ok(None), + } +} + + +#[cfg(not(feature = "enterprise"))] +pub(crate) async fn build_tracing_span_document( + _: &Server, + _: u64, +) -> trc::Result> { + Ok(None) +} diff --git a/crates/services/src/index_queue/file.rs b/crates/services/src/index_queue/file.rs new file mode 100644 index 00000000..bad16c82 --- /dev/null +++ b/crates/services/src/index_queue/file.rs @@ -0,0 +1,212 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Server, config::mailstore::email::ExtractLimits}; +use email::message::index::attachment::document_limits; +use groupware::file::{FileNode, content::FileContentKind}; +use mail_parser::decoders::html::html_to_text; +use nlp::language::{ + Language, + detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, +}; +use std::{ + borrow::Cow, + panic::{AssertUnwindSafe, catch_unwind}, +}; +use store::{ + ValueKey, + search::{FileSearchField, IndexDocument, SearchField}, + write::{Archive, ArchiveBytes, SearchIndex}, +}; +use trc::AddContext; +use types::collection::Collection; + +const HTML_INPUT_FACTOR: usize = 4; + +pub(crate) async fn build_file_document( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result> { + let empty_document = || { + IndexDocument::new(SearchIndex::File) + .with_account_id(account_id) + .with_document_id(document_id) + }; + let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::File) else { + return Ok(Some(empty_document())); + }; + let Some(archive) = server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + let node = archive + .unarchive::() + .caused_by(trc::location!())?; + let Some(file) = node.file() else { + return Ok(None); + }; + + let limits = server.core.email.extract_limits; + let kind = + FileContentKind::detect(node.name.as_str(), file.media_type.as_deref()).filter(|_| { + (index_fields.is_empty() + || index_fields.contains(&SearchField::File(FileSearchField::Content))) + && file.size.to_native() as usize <= limits.max_document_size + }); + let Some(kind) = kind else { + return Ok(Some(empty_document())); + }; + let range = match kind { + FileContentKind::Document => 0..usize::MAX, + FileContentKind::PlainText => 0..limits.max_text_size, + FileContentKind::Html => 0..limits.max_text_size.saturating_mul(HTML_INPUT_FACTOR), + }; + let Some(contents) = server + .blob_store() + .get_blob(file.blob_hash.0.as_slice(), range) + .await + .caused_by(trc::location!())? + else { + return Ok(Some(empty_document())); + }; + + let request = FileIndexRequest { + account_id, + document_id, + file_name: node.name.to_string(), + media_type: file.media_type.as_deref().map(str::to_string), + kind, + contents, + limits, + default_language: server.core.email.default_language, + }; + if kind == FileContentKind::PlainText { + return Ok(Some(request.build())); + } + let document = tokio::task::spawn_blocking(move || request.build()) + .await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .details("File indexing task failed") + .caused_by(trc::location!()) + })?; + + Ok(Some(document)) +} + +struct FileIndexRequest { + account_id: u32, + document_id: u32, + file_name: String, + media_type: Option, + kind: FileContentKind, + contents: Vec, + limits: ExtractLimits, + default_language: Language, +} + +impl FileIndexRequest { + fn build(self) -> IndexDocument { + let mut document = IndexDocument::new(SearchIndex::File) + .with_account_id(self.account_id) + .with_document_id(self.document_id); + if let Some(text) = self.text() { + let mut detector = LanguageDetector::new(); + detector.detect(&text, MIN_LANGUAGE_SCORE); + document.index_text(FileSearchField::Content, &text, Language::Unknown); + document.set_unknown_language( + detector + .most_frequent_language() + .unwrap_or(self.default_language), + ); + } + document + } + + fn text(&self) -> Option> { + let max_text_size = self.limits.max_text_size; + let text = match self.kind { + FileContentKind::Document => self + .extract_document() + .map(Cow::Owned) + .or_else(|| self.is_rtf().then(|| self.plain_text()))?, + FileContentKind::Html => Cow::Owned(html_to_text(&lossy_prefix( + &self.contents, + max_text_size.saturating_mul(HTML_INPUT_FACTOR), + ))), + FileContentKind::PlainText => self.plain_text(), + }; + let text = match text { + Cow::Borrowed(text) => { + Cow::Borrowed(text.get(..text.floor_char_boundary(max_text_size))?) + } + Cow::Owned(mut text) => { + text.truncate(text.floor_char_boundary(max_text_size)); + Cow::Owned(text) + } + }; + (!text.trim().is_empty()).then_some(text) + } + + fn hints(&self) -> text_extract::Hints<'_> { + let hints = text_extract::Hints::new().with_file_name(&self.file_name); + match self.media_type.as_deref() { + Some(media_type) => hints.with_media_type(media_type), + None => hints, + } + } + + fn is_rtf(&self) -> bool { + self.hints().format() == Some(text_extract::Format::Rtf) + } + + fn extract_document(&self) -> Option { + let mut extractor = text_extract::Extractor::new(document_limits(&self.limits)); + let mut text = String::new(); + match catch_unwind(AssertUnwindSafe(|| { + extractor.extract(&self.contents, self.hints(), &mut text) + })) { + Ok(Ok(_)) if !text.trim().is_empty() => Some(text), + Ok(_) => None, + Err(_) => { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "File text extraction panicked", + ); + None + } + } + } + + fn plain_text(&self) -> Cow<'_, str> { + lossy_prefix(&self.contents, self.limits.max_text_size) + } +} + +fn lossy_prefix(contents: &[u8], max_len: usize) -> Cow<'_, str> { + let bytes = contents + .get(..contents.len().min(max_len)) + .unwrap_or_default(); + match std::str::from_utf8(bytes) { + Ok(text) => Cow::Borrowed(text), + Err(err) if err.error_len().is_none() => bytes + .get(..err.valid_up_to()) + .and_then(|valid| std::str::from_utf8(valid).ok()) + .map_or(Cow::Borrowed(""), Cow::Borrowed), + Err(_) => String::from_utf8_lossy(bytes), + } +} + diff --git a/crates/services/src/index_queue/index.rs b/crates/services/src/index_queue/index.rs new file mode 100644 index 00000000..eb6a405e --- /dev/null +++ b/crates/services/src/index_queue/index.rs @@ -0,0 +1,871 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::index_queue::document::{ + build_calendar_document, build_contact_document, build_email_document, + build_tracing_span_document, is_email_indexed, +}; +use crate::index_queue::file::build_file_document; +use crate::index_queue::{ + BULK_LINGER, BulkRequest, Deletion, ITEM_OVERSCAN, LOCK_EXPIRY, LOCK_MARGIN, + MAX_DELETE_CLAUSES, MAX_FLUSH_TEXT_BYTES, Partition, PartitionFailure, PartitionOutcome, + PendingItem, QueuedItem, Sink, +}; +use common::{BuildServer, Inner, Server, storage::batch::MAX_BATCH_BYTES}; +use email::message::{messagedata::EmailMessageData, metadata::MetadataRow, thread::ThreadSubject}; +use std::borrow::Cow; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; +use store::ahash::AHashMap; +use store::{ + Deserialize, IterateParams, U32_LEN, ValueKey, + search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, + write::{ + BatchBuilder, QueueDocumentId, SearchIndex, SearchIndexClass, ValueClass, + key::DeserializeBigEndian, + }, +}; +use tokio::sync::{mpsc, oneshot}; +use trc::{AddContext, SearchIndexEvent}; +use types::collection::Collection; + +#[derive(Clone, Copy)] +enum Consumed { + All, + Through { id_prefix: u32, id_suffix: u32 }, +} + +#[derive(Default)] +struct FlushBudget(usize); + +pub(crate) trait SearchIndexTask: Sync + Send { + fn process_partition( + &self, + partition: Partition, + sink: &Sink, + batch_size: usize, + is_clustered: bool, + ) -> impl Future + Send; +} + +impl SearchIndexTask for Server { + async fn process_partition( + &self, + partition: Partition, + sink: &Sink, + batch_size: usize, + is_clustered: bool, + ) -> PartitionOutcome { + // Tthe distributed lock is only needed when other nodes may be indexing as well + if is_clustered + && !self + .try_lock_search_index(partition.index, partition.partition, LOCK_EXPIRY) + .await + { + trc::event!( + SearchIndex(SearchIndexEvent::PartitionLocked), + Index = partition.index.name(), + Partition = partition.partition, + ); + + return PartitionOutcome::Locked; + } + + let mut has_cleared_index = false; + let mut outcome = drain_partition( + self, + partition, + sink, + batch_size, + is_clustered, + &mut has_cleared_index, + ) + .await; + + // Restore the queue index + if has_cleared_index + && !matches!(outcome, PartitionOutcome::Completed) + && let Err(err) = set_queue_index(self, partition).await + { + outcome = PartitionOutcome::Failed(err); + } + + if is_clustered { + self.unlock_search_index(partition.index, partition.partition) + .await; + } + + outcome + } +} + +async fn drain_partition( + server: &Server, + partition: Partition, + sink: &Sink, + batch_size: usize, + is_clustered: bool, + has_cleared_index: &mut bool, +) -> PartitionOutcome { + let mut lease = Instant::now() + Duration::from_secs(LOCK_EXPIRY); + let mut from_document = None; + + loop { + if is_clustered { + let now = Instant::now(); + + if lease.saturating_duration_since(now) <= LOCK_MARGIN { + if !server + .renew_search_index_lock(partition.index, partition.partition, LOCK_EXPIRY) + .await + { + return PartitionOutcome::Incomplete; + } + + lease = now + Duration::from_secs(LOCK_EXPIRY); + } + } + + let (items, is_exhausted) = match read_queue( + server, + partition, + batch_size, + sink.counts_deletions(), + from_document, + ) + .await + { + Ok(result) => result, + Err(err) => { + return PartitionOutcome::Failed(err); + } + }; + + if items.is_empty() { + if from_document.is_some() { + // Documents may have been queued below the cursor + from_document = None; + continue; + } else if *has_cleared_index { + return PartitionOutcome::Completed; + } else if let Err(err) = clear_queue_index(server, partition).await { + return PartitionOutcome::Failed(err); + } + + *has_cleared_index = true; + continue; + } + + if *has_cleared_index && !is_exhausted { + if let Err(err) = set_queue_index(server, partition).await { + return PartitionOutcome::Failed(err); + } + + *has_cleared_index = false; + } + + let consumed = match process_items(server, partition, &items, sink, is_exhausted).await { + Ok(consumed) => consumed, + Err(err) => return PartitionOutcome::Failed(err), + }; + let is_drained = is_exhausted && consumed == items.len(); + + from_document = if is_drained { + None + } else { + consumed + .checked_sub(1) + .and_then(|last| items.get(last)) + .map(|item| (item.id_prefix, item.id_suffix)) + }; + + *has_cleared_index |= is_drained; + } +} + +async fn process_items( + server: &Server, + partition: Partition, + items: &[QueuedItem], + sink: &Sink, + clear_index: bool, +) -> Result { + let started = Instant::now(); + let mut documents = Vec::new(); + let mut deletions = Vec::new(); + let mut batch = BatchBuilder::new(); + let mut budget = FlushBudget::default(); + + let consumed = if matches!(partition.index, SearchIndex::Email) { + process_email_items( + server, + partition, + items, + &mut documents, + &mut deletions, + &mut batch, + &mut budget, + ) + .await? + } else { + let mut consumed = Consumed::All; + for item in pending_items(items) { + if item.insert { + let document = match partition.index { + SearchIndex::Calendar => { + build_calendar_document(server, item.id_prefix, item.id_suffix).await + } + SearchIndex::Contacts => { + build_contact_document(server, item.id_prefix, item.id_suffix).await + } + SearchIndex::Tracing => { + build_tracing_span_document(server, item.document_id()).await + } + SearchIndex::File => { + build_file_document(server, item.id_prefix, item.id_suffix).await + } + SearchIndex::Email | SearchIndex::InMemory => Ok(None), + }; + + match document { + Ok(Some(document)) if !document.is_empty() => { + let fits = budget.admit(&document); + documents.push(document); + if !fits { + consumed = Consumed::Through { + id_prefix: item.id_prefix, + id_suffix: item.id_suffix, + }; + break; + } + } + Ok(_) => { + nothing_to_index(partition, item.id_prefix, item.id_suffix); + } + Err(err) => { + return Err(build_failure( + partition, + err.account_id(item.id_prefix).document_id(item.id_suffix), + )); + } + } + } else if !matches!( + partition.index, + SearchIndex::Tracing | SearchIndex::InMemory + ) { + batch.commit_point(); + deletions.push(Deletion { + index: partition.index, + account_id: item.id_prefix, + document_id: item.id_suffix, + }); + } + } + consumed + }; + let consumed = consumed.items(items); + let (processed, remaining) = items.split_at_checked(consumed).unwrap_or((items, &[])); + + // Update the search index + let total = documents.len(); + sink.flush(server, partition, documents, deletions).await?; + if total > 0 { + trc::event!( + SearchIndex(SearchIndexEvent::BatchIndexed), + Index = partition.index.name(), + Partition = partition.partition, + Total = total, + Elapsed = started.elapsed(), + ); + } + + // Delete the processed items together with any metadata scheduled for deletion + for item in processed { + batch.clear(partition.queue_class(item)).commit_point(); + } + + if clear_index && remaining.is_empty() { + batch.clear(partition.index_class()); + } + + if let Err(err) = server.store().write_batch(&mut batch).await { + let reason = err.to_string(); + trc::error!( + err.details("Failed to delete processed search index queue items") + .caused_by(trc::location!()) + ); + + return Err(reason.into()); + } + + Ok(consumed) +} + +async fn process_email_items( + server: &Server, + partition: Partition, + items: &[QueuedItem], + documents: &mut Vec, + deletions: &mut Vec, + batch: &mut BatchBuilder, + budget: &mut FlushBudget, +) -> Result { + let account_id = partition.partition; + let is_indexed = is_email_indexed(server); + let pending = pending_items(items).collect::>(); + let mut remaining = pending.as_slice(); + + while !remaining.is_empty() { + let (rows, settled) = read_email_metadata(server, account_id, remaining, is_indexed) + .await + .map_err(|err| { + let reason = err.to_string(); + trc::error!( + err.account_id(account_id) + .ctx(trc::Key::Index, partition.index.name()) + .details("Failed to read email metadata for indexing") + .caused_by(trc::location!()) + ); + PartitionFailure::from(reason) + })?; + let (round, rest) = remaining + .split_at_checked(settled) + .unwrap_or((remaining, &[])); + remaining = rest; + + let mut rows = rows.into_iter().peekable(); + for item in round { + let row = rows + .next_if(|(document_id, _)| *document_id == item.id_suffix) + .map(|(_, row)| row); + + if item.insert { + let Some(row) = row else { + nothing_to_index(partition, account_id, item.id_suffix); + continue; + }; + let document = build_email_document(server, account_id, item.id_suffix, row) + .await + .map_err(|err| { + build_failure( + partition, + err.account_id(account_id).document_id(item.id_suffix), + ) + })?; + if document.is_empty() { + nothing_to_index(partition, account_id, item.id_suffix); + } else { + let fits = budget.admit(&document); + documents.push(document); + if !fits { + return Ok(Consumed::Through { + id_prefix: account_id, + id_suffix: item.id_suffix, + }); + } + } + } else { + if let Err(err) = + delete_email_metadata(server, batch, account_id, item.id_suffix, row.as_ref()) + { + let reason = err.to_string(); + trc::error!( + err.account_id(account_id) + .document_id(item.id_suffix) + .details("Failed to delete email metadata from index") + .caused_by(trc::location!()) + ); + + return Err(reason.into()); + } + + batch.commit_point(); + deletions.push(Deletion { + index: partition.index, + account_id, + document_id: item.id_suffix, + }); + } + } + } + + Ok(Consumed::All) +} + +impl Consumed { + fn items(self, items: &[QueuedItem]) -> usize { + match self { + Consumed::All => items.len(), + Consumed::Through { + id_prefix, + id_suffix, + } => items + .partition_point(|item| (item.id_prefix, item.id_suffix) <= (id_prefix, id_suffix)), + } + } +} + +impl FlushBudget { + fn admit(&mut self, document: &IndexDocument) -> bool { + self.0 = self.0.saturating_add(document.text_len()); + self.0 < MAX_FLUSH_TEXT_BYTES + } +} + +async fn read_email_metadata( + server: &Server, + account_id: u32, + items: &[PendingItem], + is_indexed: bool, +) -> trc::Result<(Vec<(u32, MetadataRow)>, usize)> { + let document_ids = items + .iter() + .filter(|item| is_indexed || !item.insert) + .map(|item| item.id_suffix) + .collect::>(); + if document_ids.is_empty() { + return Ok((Vec::new(), items.len())); + } + + let mut rows = Vec::with_capacity(document_ids.len()); + let mut bytes = 0usize; + server + .message_metadata(account_id, &document_ids, |document_id, value| { + let row = + MetadataRow::deserialize(value).map_err(|err| err.document_id(document_id))?; + bytes = bytes.saturating_add(row.decoded_len()); + rows.push((document_id, row)); + Ok(bytes < MAX_BATCH_BYTES) + }) + .await?; + + let settled = match rows.last() { + Some((last, _)) if bytes >= MAX_BATCH_BYTES => { + items.partition_point(|item| item.id_suffix <= *last) + } + _ => items.len(), + }; + + Ok((rows, settled)) +} + +fn nothing_to_index(partition: Partition, account_id: u32, document_id: u32) { + trc::event!( + SearchIndex(SearchIndexEvent::DocumentIgnored), + Index = partition.index.name(), + AccountId = account_id, + DocumentId = document_id, + Reason = "Nothing to index", + ); +} + +#[cold] +fn build_failure(partition: Partition, err: trc::Error) -> PartitionFailure { + let reason = err.to_string(); + trc::error!( + err.ctx(trc::Key::Index, partition.index.name()) + .details("Failed to build document for indexing") + .caused_by(trc::location!()) + ); + reason.into() +} + +impl Sink { + fn counts_deletions(&self) -> bool { + matches!(self, Sink::Internal) + } + + async fn flush( + &self, + server: &Server, + partition: Partition, + documents: Vec, + deletions: Vec, + ) -> Result<(), PartitionFailure> { + if documents.is_empty() && deletions.is_empty() { + return Ok(()); + } + + match self { + Sink::Internal => flush_partition(server, partition, documents, deletions).await, + Sink::Bulk(tx) => { + let (ack, ack_rx) = oneshot::channel(); + + match tx + .send(BulkRequest { + documents, + deletions, + ack, + }) + .await + { + Ok(_) => ack_rx.await.unwrap_or_else(|_| { + Err("Bulk indexer is no longer available".to_string().into()) + }), + Err(_) => Err("Bulk indexer is no longer available".to_string().into()), + } + } + } + } +} + +pub(crate) fn spawn_bulk_indexer( + inner: Arc, + batch_size: usize, + active: Arc, +) -> mpsc::Sender { + let (tx, mut rx) = mpsc::channel::(1); + + tokio::spawn(async move { + while let Some(request) = rx.recv().await { + let mut num_documents = request.documents.len(); + let mut pending = vec![request]; + + while num_documents < batch_size + && pending.len() < std::cmp::max(active.load(Ordering::Relaxed), 1) + { + match tokio::time::timeout(BULK_LINGER, rx.recv()).await { + Ok(Some(request)) => { + num_documents += request.documents.len(); + pending.push(request); + } + Ok(None) | Err(_) => break, + } + } + + let mut documents = Vec::with_capacity(num_documents); + let mut deletions = Vec::new(); + for request in pending.iter_mut() { + documents.append(&mut request.documents); + deletions.append(&mut request.deletions); + } + + let result = flush_documents(&inner.build_server(), documents, deletions).await; + for request in pending { + let _ = request.ack.send(result.clone()); + } + } + }); + + tx +} + +async fn flush_partition( + server: &Server, + partition: Partition, + documents: Vec, + deletions: Vec, +) -> Result<(), PartitionFailure> { + let Some(store) = server.search_store().internal_fts() else { + return flush_documents(server, documents, deletions).await; + }; + + // The internal store indexes and deletes a single partition at a time + let result = if matches!(partition.index, SearchIndex::Tracing) { + store.index_trace(documents).await + } else { + store + .index_account_documents( + partition.partition, + partition.index, + documents, + deletions + .into_iter() + .map(|deletion| deletion.document_id) + .collect(), + ) + .await + }; + + result.map_err(|err| { + let reason = err.to_string(); + let failure = PartitionFailure::deferred(reason, &err); + trc::error!( + err.details("Failed to update the search index") + .ctx(trc::Key::Index, partition.index.name()) + .ctx(trc::Key::Partition, partition.partition) + .caused_by(trc::location!()) + ); + failure + }) +} + +async fn flush_documents( + server: &Server, + documents: Vec, + deletions: Vec, +) -> Result<(), PartitionFailure> { + if !documents.is_empty() + && let Err(err) = server.search_store().index(documents).await + { + let failure = PartitionFailure::deferred(err.to_string(), &err); + trc::error!( + err.details("Failed to index documents") + .caused_by(trc::location!()) + ); + + return Err(failure); + } + + if !deletions.is_empty() { + let mut indexes: AHashMap>> = AHashMap::new(); + for deletion in deletions { + indexes + .entry(deletion.index) + .or_default() + .entry(deletion.account_id) + .or_default() + .push(deletion.document_id); + } + + for (index, accounts) in indexes { + // Native stores limit the number of clauses a delete query may contain + for chunk in delete_chunks(accounts) { + let multi_account = chunk.len() > 1; + let mut query = SearchQuery::new(index); + + if multi_account { + query.add_filter(SearchFilter::Or); + } + + for (account_id, document_ids) in chunk { + let multi_document = document_ids.len() > 1; + query + .add_filter(SearchFilter::And) + .add_filter(SearchFilter::integer_eq(SearchField::AccountId, account_id)); + + if multi_document { + query.add_filter(SearchFilter::Or); + } + + for document_id in document_ids { + query.add_filter(SearchFilter::integer_eq( + SearchField::DocumentId, + document_id, + )); + } + + if multi_document { + query.add_filter(SearchFilter::End); + } + query.add_filter(SearchFilter::End); + } + + if multi_account { + query.add_filter(SearchFilter::End); + } + + if let Err(err) = server.search_store().unindex(query).await { + let failure = PartitionFailure::deferred(err.to_string(), &err); + trc::error!( + err.details("Failed to delete documents from index") + .ctx(trc::Key::Index, index.name()) + .caused_by(trc::location!()) + ); + + return Err(failure); + } + } + } + } + + Ok(()) +} + +async fn read_queue( + server: &Server, + partition: Partition, + batch_size: usize, + count_deletions: bool, + from_document: Option<(u32, u32)>, +) -> Result<(Vec, bool), PartitionFailure> { + let (mut from_class, to_class) = + SearchIndexClass::queue_range(partition.index, partition.partition); + + // Resume after the last document of the previous batch + if let Some((id_prefix, id_suffix)) = from_document + && let SearchIndexClass::Queue { + id_prefix: from_prefix, + id_suffix: from_suffix, + created_at, + .. + } = &mut from_class + { + *from_prefix = id_prefix; + *from_suffix = QueueDocumentId::Assigned(id_suffix); + *created_at = u64::MAX; + } + + let max_items = batch_size * ITEM_OVERSCAN; + let mut items: Vec = Vec::new(); + let mut documents = 0; + let mut is_insert = false; + let mut is_exhausted = true; + + server + .store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::SearchIndex(from_class)), + ValueKey::from(ValueClass::SearchIndex(to_class)), + ) + .ascending() + .expect_rows(max_items), + |key, value| { + let item = QueuedItem { + id_prefix: key.deserialize_be_u32(1)?, + id_suffix: key.deserialize_be_u32(1 + U32_LEN)?, + created_at: key.deserialize_be_u64(1 + (U32_LEN * 2))?, + insert: value.first().is_some_and(|action| *action != 0), + }; + + match items.last() { + Some(last) if last.is_same_document(&item) => { + // A deletion turns the document into a deletion for good + if !item.insert && is_insert { + is_insert = false; + if !count_deletions { + documents -= 1; + } + } + } + _ => { + if documents >= batch_size || items.len() >= max_items { + is_exhausted = false; + return Ok(false); + } + + is_insert = item.insert; + if count_deletions || is_insert { + documents += 1; + } + } + } + + items.push(item); + + Ok(true) + }, + ) + .await + .map_err(|err| { + let reason = err.to_string(); + trc::error!( + err.details("Failed to iterate over the search index queue") + .ctx(trc::Key::Index, partition.index.name()) + .ctx(trc::Key::Partition, partition.partition) + .caused_by(trc::location!()) + ); + PartitionFailure::from(reason) + })?; + + Ok((items, is_exhausted)) +} + +fn delete_chunks(accounts: AHashMap>) -> Vec)>> { + let mut chunks = Vec::new(); + let mut chunk = Vec::new(); + let mut chunk_len = 0; + + for (account_id, document_ids) in accounts { + for document_ids in document_ids.chunks(MAX_DELETE_CLAUSES) { + if chunk_len + document_ids.len() > MAX_DELETE_CLAUSES && !chunk.is_empty() { + chunks.push(std::mem::take(&mut chunk)); + chunk_len = 0; + } + + chunk_len += document_ids.len(); + chunk.push((account_id, document_ids.to_vec())); + } + } + + if !chunk.is_empty() { + chunks.push(chunk); + } + + chunks +} + +fn pending_items(items: &[QueuedItem]) -> impl Iterator { + items + .chunk_by(|prev, next| prev.is_same_document(next)) + .map(|documents| PendingItem { + id_prefix: documents[0].id_prefix, + id_suffix: documents[0].id_suffix, + insert: documents.iter().all(|item| item.insert), + }) +} + +async fn clear_queue_index(server: &Server, partition: Partition) -> Result<(), PartitionFailure> { + let mut batch = BatchBuilder::new(); + batch.clear(partition.index_class()); + write_queue_index(server, batch).await +} + +async fn set_queue_index(server: &Server, partition: Partition) -> Result<(), PartitionFailure> { + let mut batch = BatchBuilder::new(); + batch.set(partition.index_class(), vec![]); + write_queue_index(server, batch).await +} + +async fn write_queue_index( + server: &Server, + mut batch: BatchBuilder, +) -> Result<(), PartitionFailure> { + server + .store() + .write_batch(&mut batch) + .await + .map(|_| ()) + .map_err(|err| { + let reason = err.to_string(); + trc::error!( + err.details("Failed to update the search index queue") + .caused_by(trc::location!()) + ); + reason.into() + }) +} + +fn delete_email_metadata( + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] server: &Server, + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + row: Option<&MetadataRow>, +) -> trc::Result<()> { + match row { + Some(row) => { + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(document_id); + let metadata = row.unarchive().caused_by(trc::location!())?; + let headers; + let thread_subject = match metadata.thread_subject() { + ThreadSubject::Resolved(name) => Cow::Borrowed(name), + ThreadSubject::InHeaders => { + headers = row.raw_headers().caused_by(trc::location!())?; + metadata.thread_subject_in(&headers) + } + }; + metadata.unindex(batch, &thread_subject); + + } + None => { + trc::event!( + SearchIndex(SearchIndexEvent::MetadataNotFound), + Details = "E-mail metadata not found", + AccountId = account_id, + DocumentId = document_id, + ); + } + } + + Ok(()) +} + diff --git a/crates/services/src/index_queue/manager.rs b/crates/services/src/index_queue/manager.rs new file mode 100644 index 00000000..17227329 --- /dev/null +++ b/crates/services/src/index_queue/manager.rs @@ -0,0 +1,386 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::index_queue::index::{SearchIndexTask, spawn_bulk_indexer}; +use crate::index_queue::{ + CANDIDATE_OVERSCAN, FULL_SCAN_INTERVAL, IndexQueueState, MIN_SCAN_INTERVAL, Partition, + PartitionOutcome, PartitionResult, Sink, +}; +use crate::index_queue::{LOCKED_BACKOFF, MAX_RETRY_DELAY, MIN_RETRY_DELAY}; +use common::storage::search::SearchIndexStatus; +use common::{BuildServer, Inner, Server}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; +use store::{ + IterateParams, U32_LEN, U64_LEN, + write::{BatchBuilder, SearchIndex, SearchIndexClass, key::DeserializeBigEndian, now}, +}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; +use trc::SearchIndexEvent; + +pub fn spawn_index_queue_manager(inner: Arc) { + let (concurrency, batch_size, is_bulk, is_clustered) = { + let server = inner.build_server(); + + if !server.core.network.roles.search_indexing { + return; + } + + ( + server.core.email.index_concurrency, + server.core.email.index_batch_size, + server.search_store().internal_fts().is_none(), + server.core.storage.coordinator.is_enabled(), + ) + }; + + trc::event!(SearchIndex(SearchIndexEvent::ManagerStarted)); + + let (done_tx, mut done_rx) = mpsc::unbounded_channel::(); + let semaphore = Arc::new(Semaphore::new(concurrency)); + let active = Arc::new(AtomicUsize::new(0)); + let sink = Arc::new(if is_bulk { + Sink::Bulk(spawn_bulk_indexer( + inner.clone(), + batch_size, + active.clone(), + )) + } else { + Sink::Internal + }); + + tokio::spawn(async move { + let mut state = IndexQueueState::default(); + let rx = inner.ipc.index_tx.clone(); + + loop { + let server = inner.build_server(); + + // Update the status of the partitions processed since the last scan + let mut results = Vec::new(); + while let Ok(result) = done_rx.try_recv() { + state.claimed.remove(&result.partition); + results.push(result); + } + if !results.is_empty() { + update_partition_status(&server, &mut state, results).await; + } + + let sleep_for = server + .scan_index_queue( + &mut state, + &inner, + &semaphore, + &active, + &sink, + &done_tx, + batch_size, + concurrency, + is_clustered, + ) + .await; + let scanned_at = Instant::now(); + + // Wait for a signal or sleep until the next retry is due + if tokio::time::timeout(sleep_for, rx.notified()).await.is_ok() + && let Some(wait_for) = MIN_SCAN_INTERVAL.checked_sub(scanned_at.elapsed()) + { + // Coalesce bursts of notifications into a single scan + tokio::time::sleep(wait_for).await; + } + } + }); +} + +pub(crate) trait IndexQueueManager: Sync + Send { + #[allow(clippy::too_many_arguments)] + fn scan_index_queue( + &self, + state: &mut IndexQueueState, + inner: &Arc, + semaphore: &Arc, + active: &Arc, + sink: &Arc, + done_tx: &mpsc::UnboundedSender, + batch_size: usize, + concurrency: usize, + is_clustered: bool, + ) -> impl Future + Send; +} + +impl IndexQueueManager for Server { + async fn scan_index_queue( + &self, + state: &mut IndexQueueState, + inner: &Arc, + semaphore: &Arc, + active: &Arc, + sink: &Arc, + done_tx: &mpsc::UnboundedSender, + batch_size: usize, + concurrency: usize, + is_clustered: bool, + ) -> Duration { + let now_timestamp = now(); + let now_instant = Instant::now(); + let budget = concurrency * CANDIDATE_OVERSCAN; + + // Drop the partitions that were deferred because another node held their lock + state + .deferred + .retain(|_, deferred_until| *deferred_until > now_instant); + let from_partition = state.cursor.take().unwrap_or(Partition { + index: SearchIndex::Email, + partition: 0, + }); + + let (from_key, to_key) = + SearchIndexClass::control_range((from_partition.index, from_partition.partition), None); + + // Each partition is stored as a queue index key optionally followed by a status key + let mut candidates = Vec::with_capacity(budget); + let mut pending: Option = None; + let mut next_retry = u64::MAX; + let mut next_deferred: Option = None; + let mut budget_exhausted = false; + + let result = self + .store() + .iterate( + IterateParams::new(from_key, to_key) + .ascending() + .expect_rows(2 * budget + 2), + |key, value| { + if key.len() != U32_LEN + 3 { + return Ok(true); + } + let Some(index) = SearchIndex::try_from_u8(key[1]) else { + return Ok(true); + }; + let partition = Partition { + index, + partition: key.deserialize_be_u32(2)?, + }; + + if key[U32_LEN + 2] == SearchIndexClass::CONTROL_INDEX { + if let Some(pending) = pending.take() { + state.failed.remove(&pending); + candidates.push(pending); + } + + if candidates.len() >= budget { + budget_exhausted = true; + state.cursor = Some(partition); + return Ok(false); + } + + pending = Some(partition); + } else { + let status_retry = value.deserialize_be_u64(0)?; + state + .failed + .insert(partition, value.deserialize_be_u32(U64_LEN)?); + let has_queued_documents = pending == Some(partition); + pending = None; + + if status_retry > now_timestamp { + next_retry = std::cmp::min(next_retry, status_retry); + } else if has_queued_documents { + candidates.push(partition); + } else if candidates.len() < budget { + // A status without a queue index is stale, process the partition to clear it + candidates.push(partition); + } else { + budget_exhausted = true; + state.cursor = Some(partition); + return Ok(false); + } + } + + Ok(true) + }, + ) + .await; + + if let Err(err) = result { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to iterate over the search index queue.") + ); + + state.cursor = None; + let backoff = MIN_SCAN_INTERVAL + .saturating_mul(1u32 << std::cmp::min(state.scan_failures, 12)) + .min(FULL_SCAN_INTERVAL); + state.scan_failures += 1; + return backoff; + } + state.scan_failures = 0; + + if let Some(pending) = pending { + state.failed.remove(&pending); + candidates.push(pending); + } + + if !candidates.is_empty() { + trc::event!( + SearchIndex(SearchIndexEvent::PartitionsAcquired), + Total = candidates.len(), + Used = state.claimed.len(), + ); + } + + // Dispatch partitions, waiting for a free indexing slot when necessary + for partition in candidates { + if let Some(deferred_until) = state.deferred.get(&partition) { + next_deferred = + Some(next_deferred.map_or(*deferred_until, |next| next.min(*deferred_until))); + continue; + } + + if !state.claimed.insert(partition) { + continue; + } + + let Ok(permit) = semaphore.clone().acquire_owned().await else { + state.claimed.remove(&partition); + break; + }; + + let inner = inner.clone(); + let sink = sink.clone(); + let active = active.clone(); + let done_tx = done_tx.clone(); + + active.fetch_add(1, Ordering::Relaxed); + tokio::spawn(async move { + let server = inner.build_server(); + + // The slot is released by the guard + let mut slot = PartitionSlot { + partition, + outcome: PartitionOutcome::Incomplete, + active, + done_tx, + permit: Some(permit), + }; + + slot.outcome = server + .process_partition(partition, sink.as_ref(), batch_size, is_clustered) + .await; + drop(slot); + + server.notify_index_queue(); + }); + } + + // Sleep until the next partition is due, unless there is work left over + let mut sleep_for = FULL_SCAN_INTERVAL; + if budget_exhausted { + sleep_for = MIN_SCAN_INTERVAL; + } + if next_retry != u64::MAX { + sleep_for = sleep_for.min(Duration::from_secs(next_retry.saturating_sub(now()))); + } + if let Some(next_deferred) = next_deferred { + sleep_for = sleep_for.min(next_deferred.saturating_duration_since(Instant::now())); + } + + sleep_for.max(MIN_SCAN_INTERVAL) + } +} + +struct PartitionSlot { + partition: Partition, + outcome: PartitionOutcome, + active: Arc, + done_tx: mpsc::UnboundedSender, + permit: Option, +} + +impl Drop for PartitionSlot { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::Relaxed); + self.permit.take(); + let _ = self.done_tx.send(PartitionResult { + partition: self.partition, + outcome: std::mem::replace(&mut self.outcome, PartitionOutcome::Incomplete), + }); + } +} + +fn retry_delay(attempts: u32) -> u64 { + MIN_RETRY_DELAY + .saturating_mul(1 << std::cmp::min(attempts, 8)) + .min(MAX_RETRY_DELAY) +} + +async fn update_partition_status( + server: &Server, + state: &mut IndexQueueState, + results: Vec, +) { + let mut batch = BatchBuilder::new(); + + for result in results { + let partition = result.partition; + + match result.outcome { + PartitionOutcome::Completed => { + if state.failed.remove(&partition).is_some() { + batch.clear(partition.status_class()).commit_point(); + } + } + PartitionOutcome::Failed(failure) => { + let attempts = state.failed.get(&partition).copied().unwrap_or_default(); + let (reason, deferred_until) = failure.into_parts(); + let next_retry = deferred_until + .filter(|retry_at| *retry_at > now()) + .unwrap_or_else(|| now() + retry_delay(attempts)); + + trc::event!( + SearchIndex(SearchIndexEvent::PartitionRetry), + Index = partition.index.name(), + Partition = partition.partition, + Reason = reason.clone(), + NextRetry = trc::Value::Timestamp(next_retry), + ); + + state.failed.insert(partition, attempts + 1); + batch + .set( + partition.status_class(), + SearchIndexStatus { + next_retry, + attempts: attempts + 1, + reason, + } + .serialize(), + ) + .commit_point(); + } + PartitionOutcome::Locked => { + // The partition is being processed by another node + state + .deferred + .insert(partition, Instant::now() + LOCKED_BACKOFF); + } + PartitionOutcome::Incomplete => (), + } + } + + if batch.is_empty() { + return; + } + + if let Err(err) = server.store().write_batch(&mut batch).await { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to update the search index queue status.") + ); + } +} diff --git a/crates/services/src/index_queue/mod.rs b/crates/services/src/index_queue/mod.rs new file mode 100644 index 00000000..358ddaea --- /dev/null +++ b/crates/services/src/index_queue/mod.rs @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::{Duration, Instant}; +use store::{ + ahash::{AHashMap, AHashSet}, + search::IndexDocument, + write::{QueueDocumentId, SearchIndex, SearchIndexClass, ValueClass}, +}; +use tokio::sync::{mpsc, oneshot}; + +pub mod document; +pub mod file; +pub mod index; +pub mod manager; +pub mod reindex; + +const LOCK_EXPIRY: u64 = 10 * 60; +const LOCK_MARGIN: Duration = Duration::from_secs(60); +const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(5 * 60); +const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100); +const BULK_LINGER: Duration = Duration::from_millis(50); +const CANDIDATE_OVERSCAN: usize = 4; +const ITEM_OVERSCAN: usize = 4; +const MAX_DELETE_CLAUSES: usize = 256; +const MAX_FLUSH_TEXT_BYTES: usize = 128 * 1024 * 1024; +const REINDEX_QUEUE_RATE: u64 = 10_000; +const MIN_RETRY_DELAY: u64 = 30; +const MAX_RETRY_DELAY: u64 = 30 * 60; +const LOCKED_BACKOFF: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct Partition { + index: SearchIndex, + partition: u32, +} + +#[derive(Debug)] +pub(crate) struct QueuedItem { + id_prefix: u32, + id_suffix: u32, + created_at: u64, + insert: bool, +} + +#[derive(Debug)] +pub(crate) struct PendingItem { + id_prefix: u32, + id_suffix: u32, + insert: bool, +} + +#[derive(Debug)] +pub(crate) struct Deletion { + index: SearchIndex, + account_id: u32, + document_id: u32, +} + +#[derive(Debug)] +pub(crate) enum PartitionOutcome { + Completed, + Locked, + Incomplete, + Failed(PartitionFailure), +} + +#[derive(Debug, Clone)] +pub(crate) struct PartitionFailure { + reason: String, + retry_at: Option, +} + +impl PartitionFailure { + fn into_parts(self) -> (String, Option) { + (self.reason, self.retry_at) + } + + fn deferred(reason: String, err: &trc::Error) -> Self { + PartitionFailure { + reason, + retry_at: err + .value(trc::Key::NextRetry) + .and_then(|value| value.to_uint()), + } + } +} + +impl From for PartitionFailure { + fn from(reason: String) -> Self { + PartitionFailure { + reason, + retry_at: None, + } + } +} + +#[derive(Debug)] +pub(crate) struct PartitionResult { + partition: Partition, + outcome: PartitionOutcome, +} + +#[derive(Debug, Default)] +pub(crate) struct IndexQueueState { + claimed: AHashSet, + failed: AHashMap, + deferred: AHashMap, + cursor: Option, + scan_failures: u32, +} + +pub(crate) enum Sink { + Internal, + Bulk(mpsc::Sender), +} + +pub(crate) struct BulkRequest { + documents: Vec, + deletions: Vec, + ack: oneshot::Sender>, +} + +impl Partition { + fn queue_class(&self, item: &QueuedItem) -> ValueClass { + ValueClass::SearchIndex(SearchIndexClass::Queue { + index: self.index, + id_prefix: item.id_prefix, + id_suffix: QueueDocumentId::Assigned(item.id_suffix), + created_at: item.created_at, + }) + } + + fn index_class(&self) -> ValueClass { + ValueClass::SearchIndex(SearchIndexClass::QueueIndex { + index: self.index, + partition: self.partition, + }) + } + + fn status_class(&self) -> ValueClass { + ValueClass::SearchIndex(SearchIndexClass::QueueStatus { + index: self.index, + partition: self.partition, + }) + } +} + +impl QueuedItem { + fn is_same_document(&self, other: &QueuedItem) -> bool { + self.id_prefix == other.id_prefix && self.id_suffix == other.id_suffix + } +} + +impl PendingItem { + fn document_id(&self) -> u64 { + ((self.id_prefix as u64) << 32) | self.id_suffix as u64 + } +} diff --git a/crates/services/src/index_queue/reindex.rs b/crates/services/src/index_queue/reindex.rs new file mode 100644 index 00000000..146bf97a --- /dev/null +++ b/crates/services/src/index_queue/reindex.rs @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::index_queue::REINDEX_QUEUE_RATE; +use common::{Server, storage::dav::FILE_KIND_FILE}; +use email::cache::MessageCacheFetch; +use groupware::cache::GroupwareCache; +use store::{ + IterateParams, U32_LEN, ValueKey, + write::{ + BatchBuilder, CommitPriority, QueueDocumentId, SearchIndex, SearchIndexClass, ValueClass, + key::DeserializeBigEndian, + }, +}; +use std::{mem, time::Duration}; +use tokio::time::{Instant, sleep_until}; +use trc::AddContext; +use types::collection::SyncCollection; + +const ENTRY_INTERVAL: Duration = Duration::from_nanos(1_000_000_000 / REINDEX_QUEUE_RATE); + +struct PacedQueue { + batch: BatchBuilder, + entries: u32, + next: Instant, +} + + +pub(crate) async fn reindex_account(server: &Server, account_id: u32) -> trc::Result<()> { + for index in [ + SearchIndex::Email, + SearchIndex::Calendar, + SearchIndex::Contacts, + SearchIndex::File, + ] { + clear_queued_updates(server, index, account_id).await?; + } + + let calendars = server + .fetch_groupware_resources(account_id, account_id, SyncCollection::Calendar) + .await + .caused_by(trc::location!())?; + let contacts = server + .fetch_groupware_resources(account_id, account_id, SyncCollection::AddressBook) + .await + .caused_by(trc::location!())?; + let files = if server + .core + .email + .index_fields + .contains_key(&SearchIndex::File) + { + Some( + server + .fetch_groupware_resources(account_id, account_id, SyncCollection::FileNode) + .await + .caused_by(trc::location!())?, + ) + } else { + None + }; + + let mut queue = PacedQueue::new(); + + for document_id in server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .emails + .iter() + .map(|v| v.document_id()) + { + queue + .batch + .queue_document_index(SearchIndex::Email, account_id, document_id); + queue.queued(server).await?; + } + + for (document_type, cache) in [ + (SearchIndex::Calendar, &calendars), + (SearchIndex::Contacts, &contacts), + ] { + for document_id in cache.document_ids(false) { + queue + .batch + .queue_document_index(document_type, account_id, document_id); + queue.queued(server).await?; + } + } + + if let Some(cache) = &files { + for document_id in cache + .resources + .iter() + .filter(|resource| resource.file_kind() == Some(FILE_KIND_FILE)) + .map(|resource| resource.document_id()) + { + queue + .batch + .queue_document_index(SearchIndex::File, account_id, document_id); + queue.queued(server).await?; + } + } + + queue.commit(server).await +} + +async fn clear_queued_updates( + server: &Server, + index: SearchIndex, + partition: u32, +) -> trc::Result<()> { + let (from_class, to_class) = SearchIndexClass::queue_range(index, partition); + let mut batch = BatchBuilder::with_priority(CommitPriority::Background); + batch.clear(SearchIndexClass::QueueStatus { index, partition }); + + server + .store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::SearchIndex(from_class)), + ValueKey::from(ValueClass::SearchIndex(to_class)), + ) + .ascending(), + |key, value| { + if value.first().is_some_and(|action| *action != 0) { + batch + .clear(SearchIndexClass::Queue { + index, + id_prefix: key.deserialize_be_u32(1)?, + id_suffix: QueueDocumentId::Assigned( + key.deserialize_be_u32(1 + U32_LEN)?, + ), + created_at: key.deserialize_be_u64(1 + (U32_LEN * 2))?, + }) + .bulk_commit_point(); + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + server + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + + Ok(()) +} + + +impl PacedQueue { + fn new() -> Self { + PacedQueue { + batch: BatchBuilder::with_priority(CommitPriority::Background), + entries: 0, + next: Instant::now(), + } + } + + async fn queued(&mut self, server: &Server) -> trc::Result<()> { + self.entries += 1; + if self.batch.is_large_batch() { + self.commit(server).await + } else { + Ok(()) + } + } + + async fn commit(&mut self, server: &Server) -> trc::Result<()> { + if self.batch.is_empty() { + return Ok(()); + } + let batch = mem::replace( + &mut self.batch, + BatchBuilder::with_priority(CommitPriority::Background), + ); + server.commit_batch(batch).await?; + if let Some(deadline) = self.schedule(Instant::now()) { + sleep_until(deadline).await; + } + Ok(()) + } + + fn schedule(&mut self, now: Instant) -> Option { + self.next += ENTRY_INTERVAL * mem::take(&mut self.entries); + if self.next > now { + Some(self.next) + } else { + self.next = now; + None + } + } +} + diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index b249b46b..254fb0f1 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -6,6 +6,8 @@ #![warn(clippy::large_futures)] +use crate::index_queue::manager::spawn_index_queue_manager; +use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_scheduler}; use broadcast::publisher::spawn_broadcast_publisher; use common::{ BuildServer, Inner, @@ -14,9 +16,8 @@ use common::{ use state_manager::manager::spawn_push_router; use std::sync::Arc; -use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_scheduler}; - pub mod broadcast; +pub mod index_queue; pub mod state_manager; pub mod task_manager; @@ -59,6 +60,9 @@ impl SpawnServices for IpcReceivers { // Spawn task manager spawn_task_manager(inner.clone()); + // Spawn search index queue manager + spawn_index_queue_manager(inner.clone()); + // Spawn task scheduler spawn_task_scheduler(inner); } diff --git a/crates/services/src/state_manager/ece.rs b/crates/services/src/state_manager/ece.rs index dc334b90..979ccc15 100644 --- a/crates/services/src/state_manager/ece.rs +++ b/crates/services/src/state_manager/ece.rs @@ -8,7 +8,7 @@ use hkdf::Hkdf; use p256::{ PublicKey, ecdh::EphemeralSecret, - elliptic_curve::{rand_core::OsRng, sec1::ToEncodedPoint}, + elliptic_curve::{Generate, sec1::ToSec1Point}, }; use sha2::Sha256; use store::rand::RngExt; @@ -59,9 +59,9 @@ pub fn ece_encrypt( mut data: &[u8], ) -> Result, String> { let salt = store::rand::rng().random::<[u8; 16]>(); - let server_secret = EphemeralSecret::random(&mut OsRng); + let server_secret = EphemeralSecret::try_generate().map_err(|e| e.to_string())?; let server_public_key = server_secret.public_key(); - let server_public_key_bytes = server_public_key.to_encoded_point(false); + let server_public_key_bytes = server_public_key.to_sec1_point(false); let client_public_key = PublicKey::from_sec1_bytes(p256dh).map_err(|e| e.to_string())?; let shared_secret = server_secret.diffie_hellman(&client_public_key); @@ -210,42 +210,3 @@ pub fn generate_iv(nonce: &[u8], counter: usize) -> [u8; ECE_NONCE_LENGTH] { iv } -#[cfg(test)] -mod tests { - use super::*; - - fn encrypt_len(plaintext_len: usize) -> usize { - let secret = EphemeralSecret::random(&mut OsRng); - let p256dh = secret.public_key().to_encoded_point(false); - - ece_encrypt(p256dh.as_bytes(), &[0u8; 16], &vec![b'a'; plaintext_len]) - .expect("encryption failed") - .len() - } - - #[test] - fn max_plaintext_stays_within_a_single_record() { - let header = ECE_AES128GCM_HEADER_LENGTH + ECE_WEBPUSH_PUBLIC_KEY_LENGTH; - let limit = WEBPUSH_MAX_BODY_SIZE; - - let padded = (ECE_WEBPUSH_MAX_PLAINTEXT_SIZE + ECE_AES128GCM_PAD_SIZE) - .next_multiple_of(ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE); - - let len = encrypt_len(ECE_WEBPUSH_MAX_PLAINTEXT_SIZE); - assert!( - len <= limit, - "{len} exceeds the {limit} octet payload limit" - ); - assert_eq!( - len, - header + padded + ECE_TAG_LENGTH, - "expected exactly one authentication tag, i.e. a single record" - ); - - let over = encrypt_len(ECE_WEBPUSH_MAX_PLAINTEXT_SIZE + 1); - assert!( - over > limit, - "{ECE_WEBPUSH_MAX_PLAINTEXT_SIZE} is not the largest single-record plaintext" - ); - } -} diff --git a/crates/services/src/state_manager/email_push.rs b/crates/services/src/state_manager/email_push.rs index 79304002..0304c3e4 100644 --- a/crates/services/src/state_manager/email_push.rs +++ b/crates/services/src/state_manager/email_push.rs @@ -9,12 +9,11 @@ use common::{MessageStoreCache, Server}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::{ - body::ToBodyPart, - headers::{HeaderToValue, IntoForm}, + jmap::{BodyValueOptions, EmailNeeds, EmailRender}, + messagedata::MessageData, metadata::{ - ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, - ArchivedMetadataPartType, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, - MessageMetadata, MetadataHeaderName, + AddressHeader, ArchivedMessageMetadata, HeaderList, MetadataRow, MetadataStructure, + Occurrence, PartSource, RawMessage, }, }, push::EmailPush, @@ -22,27 +21,23 @@ use email::{ use jmap_proto::{ method::query::Filter, object::{ - email::{EmailFilter, EmailProperty, EmailValue, HeaderForm}, + email::{EmailFilter, EmailProperty, EmailValue}, push_subscription::EmailPushProperty, }, types::date::UTCDate, }; use jmap_tools::{Map, Property, Value}; -use mail_parser::{HeaderName, HeaderValue}; -use std::iter::Peekable; -use store::{ - ValueKey, - write::{AlignedBytes, Archive}, -}; +use mail_parser::HeaderForm; +use std::{borrow::Cow, iter::Peekable}; +use store::ValueKey; use trc::AddContext; use types::{ blob::{BlobClass, BlobId}, - blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id, + keyword::HASATTACHMENT, }; -use utils::chained_bytes::ChainedBytes; const BODY_PROPERTIES: &[EmailProperty] = &[ EmailProperty::PartId, @@ -67,7 +62,7 @@ pub async fn build_email_push_object( let properties = &config.properties; let Some(data) = server .store() - .get_value::>(ValueKey::archive( + .get_value::(ValueKey::archive( account_id, Collection::Email, document_id, @@ -77,67 +72,92 @@ pub async fn build_email_push_object( else { return Ok(None); }; - let data = data - .deserialize::() - .caused_by(trc::location!())?; - let Some(metadata_archive) = server - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await - .caused_by(trc::location!())? - else { - return Ok(None); - }; - let metadata = metadata_archive - .unarchive::() - .caused_by(trc::location!())?; - let (Some(contents), Some(root_part)) = ( - metadata.contents.first(), - metadata.contents.first().and_then(|c| c.parts.first()), - ) else { - return Ok(None); - }; - - let blob_hash = BlobHash::from(&metadata.blob_hash); - let needs_body = properties.iter().any(|property| { - matches!( - property, - EmailPushProperty::TextBody - | EmailPushProperty::HtmlBody - | EmailPushProperty::Attachments - | EmailPushProperty::BodyStructure - ) - }) || config.filter.iter().any(|filter| { + let keys = properties + .iter() + .map(EmailProperty::from) + .collect::>(); + let options = BodyValueOptions::default(); + let needs = EmailNeeds::new(&keys, BODY_PROPERTIES, &options); + let needs_headers = needs.headers + || config + .filter + .iter() + .any(|filter| matches!(filter, Filter::Property(EmailFilter::Header(_)))); + let filter_blob = config.filter.iter().any(|filter| { matches!( filter, Filter::Property(EmailFilter::Body(_) | EmailFilter::Text(_)) ) }); - let raw_body; - let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); - if needs_body { - let Some(blob) = server - .blob_store() - .get_blob(blob_hash.as_slice(), 0..usize::MAX) + let key = ValueKey::immutable( + account_id, + Collection::Email, + document_id, + EmailField::Metadata, + ); + let row; + let structure; + let raw_headers; + let (metadata, headers) = if needs_headers { + let Some(value) = server + .store() + .get_value::(key) .await .caused_by(trc::location!())? else { return Ok(None); }; - raw_body = blob; - raw_message.append( - raw_body - .get(metadata.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - } + row = value; + raw_headers = row.raw_headers().caused_by(trc::location!())?; + ( + row.unarchive().caused_by(trc::location!())?, + Some(raw_headers.as_ref()), + ) + } else { + let Some(value) = server + .store() + .get_value::(key) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + structure = value; + (structure.unarchive().caused_by(trc::location!())?, None) + }; + + let blob = if filter_blob || needs.blob { + let Some(blob) = server + .blob_store() + .get_blob(metadata.blob_hash().as_slice(), 0..usize::MAX) + .await + .caused_by(trc::location!())? + else { + return Ok(None); + }; + Some(blob) + } else { + None + }; + + let blob_id = BlobId::new( + metadata.blob_hash(), + BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id, + }, + ); + let render = EmailRender::new( + metadata, + headers, + blob.as_deref(), + &blob_id, + BODY_PROPERTIES, + &options, + ); if !config.filter.is_empty() { let cache = if config.filter.iter().any(|filter| { @@ -160,46 +180,33 @@ pub async fn build_email_push_object( None }; - if !eval_filter_node( - &mut config.filter.iter().peekable(), - &FilterContext { - data: &data, - document_id, - cache: cache.as_deref(), - metadata, - contents, - root_part, - raw_message: &raw_message, - }, - ) - .unwrap_or(true) + let context = FilterContext { + data: &data, + document_id, + cache: cache.as_deref(), + meta: metadata, + headers, + root_headers: render.root_headers(), + raw: blob + .as_deref() + .map(|blob| metadata.raw_message(headers, blob)), + }; + if !context + .eval_node(&mut config.filter.iter().peekable()) + .unwrap_or(true) { return Ok(None); } } - let blob_id = BlobId { - hash: blob_hash, - class: BlobClass::Linked { - account_id, - collection: Collection::Email.into(), - document_id, - }, - section: None, - }; - let blob_body_offset = - metadata.blob_body_offset.to_native() as isize - root_part.offset_body.to_native() as isize; - let id = Id::from_parts(data.thread_id, document_id); let mut email = Map::with_capacity(properties.len()); let mut used = 0; - for property in properties { - let key = EmailProperty::from(property); + for (property, key) in properties.iter().zip(keys) { let value: Value<'static, EmailProperty, EmailValue> = match property { EmailPushProperty::Id => id.into(), EmailPushProperty::ThreadId => Id::from(id.prefix_id()).into(), - EmailPushProperty::BlobId => blob_id.clone().into(), EmailPushProperty::MailboxIds => { let mut mailbox_ids = Map::with_capacity(data.mailboxes.len()); for mailbox in data.mailboxes.iter() { @@ -212,94 +219,22 @@ pub async fn build_email_push_object( } EmailPushProperty::Keywords => { let mut keywords = Map::with_capacity(2); - for keyword in data.keywords.iter() { - keywords.insert_unchecked(EmailProperty::Keyword(keyword.clone()), true); + for keyword in data.keywords() { + keywords.insert_unchecked(EmailProperty::Keyword(keyword), true); } Value::Object(keywords) } EmailPushProperty::Size => data.size.into(), - EmailPushProperty::ReceivedAt => EmailValue::Date(UTCDate::from_timestamp( - (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, - )) - .into(), - EmailPushProperty::Preview => { - if metadata.preview.is_empty() { - continue; - } - metadata.preview.to_string().into() + EmailPushProperty::ReceivedAt => { + EmailValue::Date(UTCDate::from_timestamp(data.received_at as i64)).into() } EmailPushProperty::HasAttachment => { - ((metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0).into() + ((data.keywords & (1 << HASATTACHMENT)) != 0).into() } - EmailPushProperty::Subject - | EmailPushProperty::SentAt - | EmailPushProperty::MessageId - | EmailPushProperty::InReplyTo - | EmailPushProperty::References - | EmailPushProperty::Sender - | EmailPushProperty::From - | EmailPushProperty::To - | EmailPushProperty::Cc - | EmailPushProperty::Bcc - | EmailPushProperty::ReplyTo => { - let (header_name, form) = match property { - EmailPushProperty::Subject => (MetadataHeaderName::Subject, HeaderForm::Text), - EmailPushProperty::SentAt => (MetadataHeaderName::Date, HeaderForm::Date), - EmailPushProperty::MessageId => { - (MetadataHeaderName::MessageId, HeaderForm::MessageIds) - } - EmailPushProperty::InReplyTo => { - (MetadataHeaderName::InReplyTo, HeaderForm::MessageIds) - } - EmailPushProperty::References => { - (MetadataHeaderName::References, HeaderForm::MessageIds) - } - EmailPushProperty::Sender => { - (MetadataHeaderName::Sender, HeaderForm::Addresses) - } - EmailPushProperty::From => (MetadataHeaderName::From, HeaderForm::Addresses), - EmailPushProperty::To => (MetadataHeaderName::To, HeaderForm::Addresses), - EmailPushProperty::Cc => (MetadataHeaderName::Cc, HeaderForm::Addresses), - EmailPushProperty::Bcc => (MetadataHeaderName::Bcc, HeaderForm::Addresses), - EmailPushProperty::ReplyTo => { - (MetadataHeaderName::ReplyTo, HeaderForm::Addresses) - } - _ => unreachable!(), - }; - root_part - .header_value(&header_name) - .map(|value| HeaderValue::from(value).into_form(&form)) - .unwrap_or_default() - } - EmailPushProperty::Header(_) => root_part.header_to_value(&key, &raw_message), - EmailPushProperty::Headers => root_part.headers_to_value(&raw_message), - EmailPushProperty::TextBody - | EmailPushProperty::HtmlBody - | EmailPushProperty::Attachments => { - let parts = match property { - EmailPushProperty::TextBody => &contents.text_body, - EmailPushProperty::HtmlBody => &contents.html_body, - EmailPushProperty::Attachments => &contents.attachments, - _ => unreachable!(), - }; - parts - .iter() - .map(|part_id| { - contents.to_body_part( - u16::from(part_id) as u32, - BODY_PROPERTIES, - &raw_message, - &blob_id, - blob_body_offset, - ) - }) - .collect::>() - .into() - } - EmailPushProperty::BodyStructure => { - contents.to_body_part(0, BODY_PROPERTIES, &raw_message, &blob_id, blob_body_offset) - } - EmailPushProperty::BodyValues => Value::Object(Map::with_capacity(0)), + _ => match render.value(&key) { + Some(value) => value.into_owned(), + None => continue, + }, }; let entry_size = key.to_cow().len() + estimate_value_size(&value) + 4; @@ -338,199 +273,188 @@ struct FilterContext<'a> { data: &'a MessageData, document_id: u32, cache: Option<&'a MessageStoreCache>, - metadata: &'a ArchivedMessageMetadata, - contents: &'a ArchivedMessageMetadataContents, - root_part: &'a ArchivedMessageMetadataPart, - raw_message: &'a ChainedBytes<'a>, + meta: &'a ArchivedMessageMetadata, + headers: Option<&'a [u8]>, + root_headers: HeaderList<'a>, + raw: Option>, } -fn eval_filter_node<'a, I>(tokens: &mut Peekable, context: &FilterContext) -> Option -where - I: Iterator>, -{ - match tokens.next()? { - operator @ (Filter::And | Filter::Or | Filter::Not) => { - let mut all = true; - let mut any = false; - while let Some(token) = tokens.peek() { - if matches!(token, Filter::Close) { - tokens.next(); - break; - } - if let Some(result) = eval_filter_node(tokens, context) { - all &= result; - any |= result; +impl FilterContext<'_> { + fn eval_node<'f, I>(&self, tokens: &mut Peekable) -> Option + where + I: Iterator>, + { + match tokens.next()? { + operator @ (Filter::And | Filter::Or | Filter::Not) => { + let mut all = true; + let mut any = false; + while let Some(token) = tokens.peek() { + if matches!(token, Filter::Close) { + tokens.next(); + break; + } + if let Some(result) = self.eval_node(tokens) { + all &= result; + any |= result; + } } + Some(match operator { + Filter::And => all, + Filter::Or => any, + _ => !any, + }) } - Some(match operator { - Filter::And => all, - Filter::Or => any, - Filter::Not => !any, - _ => unreachable!(), - }) + Filter::Property(condition) => Some(self.eval_condition(condition)), + Filter::Close => None, } - Filter::Property(condition) => Some(eval_filter_condition(condition, context)), - Filter::Close => None, } -} -fn eval_filter_condition(condition: &EmailFilter, context: &FilterContext) -> bool { - match condition { - EmailFilter::InMailbox(id) => { - let mailbox_id = id.document_id(); - context + fn eval_condition(&self, condition: &EmailFilter) -> bool { + match condition { + EmailFilter::InMailbox(id) => { + let mailbox_id = id.document_id(); + self.data + .mailboxes + .iter() + .any(|mailbox| mailbox.mailbox_id == mailbox_id) + } + EmailFilter::InMailboxOtherThan(ids) => self .data .mailboxes .iter() - .any(|mailbox| mailbox.mailbox_id == mailbox_id) - } - EmailFilter::InMailboxOtherThan(ids) => context - .data - .mailboxes - .iter() - .any(|mailbox| ids.iter().all(|id| id.document_id() != mailbox.mailbox_id)), - EmailFilter::Before(date) => received_at(context) < date.timestamp(), - EmailFilter::After(date) => received_at(context) >= date.timestamp(), - EmailFilter::MinSize(size) => context.data.size >= *size, - EmailFilter::MaxSize(size) => context.data.size < *size, - EmailFilter::HasKeyword(keyword) => { - context.data.keywords.iter().any(|value| value == keyword) - } - EmailFilter::NotKeyword(keyword) => { - !context.data.keywords.iter().any(|value| value == keyword) - } - EmailFilter::AllInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| { - cache - .in_thread(context.data.thread_id) - .all(|message| cache.has_keyword(message, keyword)) - }), - EmailFilter::SomeInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| { - cache - .in_thread(context.data.thread_id) - .any(|message| cache.has_keyword(message, keyword)) - }), - EmailFilter::NoneInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| { - !cache - .in_thread(context.data.thread_id) - .any(|message| cache.has_keyword(message, keyword)) - }), - EmailFilter::HasAttachment(value) => { - ((context.metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0) == *value - } - EmailFilter::From(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::From, - &HeaderForm::Addresses, - &matcher, - ) - }), - EmailFilter::To(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::To, - &HeaderForm::Addresses, - &matcher, - ) - }), - EmailFilter::Cc(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::Cc, - &HeaderForm::Addresses, - &matcher, - ) - }), - EmailFilter::Bcc(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::Bcc, - &HeaderForm::Addresses, - &matcher, - ) - }), - EmailFilter::Subject(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::Subject, - &HeaderForm::Text, - &matcher, - ) - }), - EmailFilter::Body(text) => { - ascii_matcher(text).is_some_and(|matcher| body_matches(context, &matcher)) - } - EmailFilter::Text(text) => ascii_matcher(text).is_some_and(|matcher| { - header_matches( - context, - &MetadataHeaderName::From, - &HeaderForm::Addresses, - &matcher, - ) || header_matches( - context, - &MetadataHeaderName::To, - &HeaderForm::Addresses, - &matcher, - ) || header_matches( - context, - &MetadataHeaderName::Cc, - &HeaderForm::Addresses, - &matcher, - ) || header_matches( - context, - &MetadataHeaderName::Bcc, - &HeaderForm::Addresses, - &matcher, - ) || header_matches( - context, - &MetadataHeaderName::Subject, - &HeaderForm::Text, - &matcher, - ) || body_matches(context, &matcher) - }), - EmailFilter::Header(parts) => { - let Some(name) = parts.first() else { - return false; - }; - let header_name = MetadataHeaderName::from( - HeaderName::parse(name.as_str()) - .unwrap_or_else(|| HeaderName::Other(name.as_str().into())), - ); - match (context.root_part.header_value(&header_name), parts.get(1)) { - (Some(value), Some(expected)) => ascii_matcher(expected).is_some_and(|matcher| { - value_contains( - &HeaderValue::from(value).into_form(&HeaderForm::Raw), - &matcher, - ) - }), - (Some(_), None) => true, - (None, _) => false, + .any(|mailbox| ids.iter().all(|id| id.document_id() != mailbox.mailbox_id)), + EmailFilter::Before(date) => self.received_at() < date.timestamp(), + EmailFilter::After(date) => self.received_at() >= date.timestamp(), + EmailFilter::MinSize(size) => self.data.size >= *size, + EmailFilter::MaxSize(size) => self.data.size < *size, + EmailFilter::HasKeyword(keyword) => self.data.has_keyword(keyword), + EmailFilter::NotKeyword(keyword) => !self.data.has_keyword(keyword), + EmailFilter::AllInThreadHaveKeyword(keyword) => self.cache.is_some_and(|cache| { + cache + .in_thread(self.data.thread_id) + .all(|message| cache.has_keyword(message, keyword)) + }), + EmailFilter::SomeInThreadHaveKeyword(keyword) => self.cache.is_some_and(|cache| { + cache + .in_thread(self.data.thread_id) + .any(|message| cache.has_keyword(message, keyword)) + }), + EmailFilter::NoneInThreadHaveKeyword(keyword) => self.cache.is_some_and(|cache| { + !cache + .in_thread(self.data.thread_id) + .any(|message| cache.has_keyword(message, keyword)) + }), + EmailFilter::HasAttachment(value) => { + ((self.data.keywords & (1 << HASATTACHMENT)) != 0) == *value } + EmailFilter::From(text) => ascii_matcher(text) + .is_some_and(|matcher| self.address_matches(AddressHeader::From, &matcher)), + EmailFilter::To(text) => ascii_matcher(text) + .is_some_and(|matcher| self.address_matches(AddressHeader::To, &matcher)), + EmailFilter::Cc(text) => ascii_matcher(text) + .is_some_and(|matcher| self.address_matches(AddressHeader::Cc, &matcher)), + EmailFilter::Bcc(text) => ascii_matcher(text) + .is_some_and(|matcher| self.address_matches(AddressHeader::Bcc, &matcher)), + EmailFilter::Subject(text) => { + ascii_matcher(text).is_some_and(|matcher| self.subject_matches(&matcher)) + } + EmailFilter::Body(text) => { + ascii_matcher(text).is_some_and(|matcher| self.body_matches(&matcher)) + } + EmailFilter::Text(text) => ascii_matcher(text).is_some_and(|matcher| { + [ + AddressHeader::From, + AddressHeader::To, + AddressHeader::Cc, + AddressHeader::Bcc, + ] + .into_iter() + .any(|header| self.address_matches(header, &matcher)) + || self.subject_matches(&matcher) + || self.body_matches(&matcher) + }), + EmailFilter::Header(parts) => match parts.first() { + Some(name) => self.header_matches(name, parts.get(1).map(String::as_str)), + None => false, + }, + EmailFilter::SentBefore(date) => self + .sent_at() + .is_some_and(|sent_at| sent_at < date.timestamp()), + EmailFilter::SentAfter(date) => self + .sent_at() + .is_some_and(|sent_at| sent_at >= date.timestamp()), + EmailFilter::InThread(id) => self.data.thread_id == id.document_id(), + EmailFilter::Id(ids) => ids.iter().any(|id| id.document_id() == self.document_id), + EmailFilter::_T(_) => false, } - EmailFilter::SentBefore(date) => { - sent_at(context).is_some_and(|sent_at| sent_at < date.timestamp()) - } - EmailFilter::SentAfter(date) => { - sent_at(context).is_some_and(|sent_at| sent_at >= date.timestamp()) - } - EmailFilter::InThread(id) => context.data.thread_id == id.document_id(), - EmailFilter::Id(ids) => ids.iter().any(|id| id.document_id() == context.document_id), - EmailFilter::_T(_) => false, } -} -fn received_at(context: &FilterContext) -> i64 { - (context.metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64 -} + fn received_at(&self) -> i64 { + self.data.received_at as i64 + } -fn sent_at(context: &FilterContext) -> Option { - match context - .root_part - .header_value(&MetadataHeaderName::Date) - .map(HeaderValue::from) - { - Some(HeaderValue::DateTime(datetime)) => Some(datetime.to_timestamp()), - _ => None, + fn sent_at(&self) -> Option { + self.meta + .root() + .envelope() + .datetime() + .map(|date| date.to_timestamp()) + } + + fn address_matches(&self, header: AddressHeader, matcher: &AhoCorasick) -> bool { + self.meta + .root() + .envelope() + .addresses(header, Occurrence::Last) + .mailboxes() + .any(|mailbox| { + mailbox + .name + .into_iter() + .chain(mailbox.address) + .any(|text| matcher.is_match(text)) + }) + } + + fn subject_matches(&self, matcher: &AhoCorasick) -> bool { + self.meta + .root() + .envelope() + .subject() + .is_some_and(|subject| matcher.is_match(subject)) + } + + fn header_matches(&self, name: &str, expected: Option<&str>) -> bool { + let Some(headers) = self.headers else { + return false; + }; + let Some(header) = self.root_headers.last_named(name, headers) else { + return false; + }; + let Some(expected) = expected else { + return true; + }; + ascii_matcher(expected).is_some_and(|matcher| { + header.raw_value(headers).is_some_and(|raw| { + HeaderForm::Text + .parse(raw) + .value() + .as_text() + .is_some_and(|text| matcher.is_match(text)) + }) + }) + } + + fn body_matches(&self, matcher: &AhoCorasick) -> bool { + let Some(raw) = self.raw else { + return false; + }; + let source = PartSource::Raw(raw); + let root = self.meta.root(); + root.text_body().chain(root.html_body()).any(|part| { + part.text(&source) + .is_some_and(|text| matcher.is_match(Cow::as_ref(&text.text))) + }) } } @@ -541,44 +465,3 @@ fn ascii_matcher(needle: &str) -> Option { .ok() } -fn header_matches( - context: &FilterContext, - name: &MetadataHeaderName, - form: &HeaderForm, - matcher: &AhoCorasick, -) -> bool { - context - .root_part - .header_value(name) - .is_some_and(|value| value_contains(&HeaderValue::from(value).into_form(form), matcher)) -} - -fn value_contains(value: &Value<'_, EmailProperty, EmailValue>, matcher: &AhoCorasick) -> bool { - match value { - Value::Str(text) => matcher.is_match(text.as_ref()), - Value::Array(values) => values.iter().any(|value| value_contains(value, matcher)), - Value::Object(map) => map.iter().any(|(_, value)| value_contains(value, matcher)), - _ => false, - } -} - -fn body_matches(context: &FilterContext, matcher: &AhoCorasick) -> bool { - context - .contents - .text_body - .iter() - .chain(context.contents.html_body.iter()) - .any(|part_id| { - context - .contents - .parts - .as_ref() - .get(u16::from(part_id) as usize) - .is_some_and(|part| { - matches!( - part.body, - ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html - ) && matcher.is_match(part.decode_contents(context.raw_message).as_str()) - }) - }) -} diff --git a/crates/services/src/state_manager/http.rs b/crates/services/src/state_manager/http.rs index 39987ba1..bb9f5242 100644 --- a/crates/services/src/state_manager/http.rs +++ b/crates/services/src/state_manager/http.rs @@ -12,6 +12,7 @@ use super::{ use crate::state_manager::{PushBatch, PushRegistration}; use calcard::jscalendar::JSCalendarDateTime; use common::{Server, ipc::PushNotification, network::webpush::Vapid}; +use compact_str::{CompactString, ToCompactString}; use email::push::{PushSubscription, Urgency}; use jmap_proto::{ object::email::{EmailProperty, EmailValue}, @@ -20,17 +21,18 @@ use jmap_proto::{ }; use jmap_tools::Value; use reqwest::{ - Client, Url, + Method, header::{AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE}, - redirect::Policy, }; -use std::net::IpAddr; use std::time::{Duration, Instant}; use store::write::now; use tokio::sync::mpsc; use trc::PushSubscriptionEvent; use types::id::Id; -use utils::map::vec_map::VecMap; +use utils::{ + map::vec_map::VecMap, + outbound::{OutboundClient, OutboundPolicy, Redirects}, +}; const MAX_ERROR_RESPONSE_LEN: usize = 1024; const MAX_REDIRECTS: usize = 4; @@ -48,7 +50,7 @@ impl PushRegistration { pub fn send( &mut self, id: Id, - push_client: &Client, + push_client: &OutboundClient, push_tx: mpsc::Sender, push_timeout: Duration, server: Server, @@ -70,10 +72,10 @@ impl PushRegistration { for notification in &batch.notifications { match notification { - PushNotification::StateChange(_) => {} + PushNotification::StateChange(_) | PushNotification::ViewerStateChange(_) => {} PushNotification::CalendarAlert(calendar_alert) => { let payload = PushObject::CalendarAlert { - account_id: calendar_alert.account_id.into(), + account_id: calendar_alert.event_account_id.into(), calendar_event_id: calendar_alert.event_id.into(), uid: calendar_alert.uid.clone(), recurrence_id: calendar_alert.recurrence_id.map(|timestamp| { @@ -204,7 +206,8 @@ impl PushRegistration { .notifications .into_iter() .filter(|notification| match notification { - PushNotification::StateChange(_) => false, + PushNotification::StateChange(_) + | PushNotification::ViewerStateChange(_) => false, PushNotification::EmailPush(email_push) => { failed_email_pushes.contains(&email_push.account_id) } @@ -222,7 +225,7 @@ impl PushRegistration { } async fn post_object( - push_client: &Client, + push_client: &OutboundClient, subscription: &PushSubscription, object: &PushObject, push_timeout: Duration, @@ -253,35 +256,37 @@ async fn post_object( } } -pub(crate) fn build_push_client() -> Client { - utils::http::http_client_builder(cfg!(feature = "test_mode")) - .redirect(Policy::custom(|attempt| match attempt.previous().last() { - Some(previous) if is_same_organization(previous, attempt.url()) => { - if attempt.previous().len() > MAX_REDIRECTS { - attempt.error("Too many redirects.") - } else { - attempt.follow() - } - } - _ => attempt.stop(), - })) - .build() - .unwrap_or_default() +pub(crate) fn build_push_client() -> OutboundClient { + OutboundClient::new( + utils::http::http_client_builder(cfg!(stalwart_tests)), + OutboundPolicy::PublicOnly, + Redirects::SameOrganization(MAX_REDIRECTS), + ) } pub(crate) async fn http_request( - push_client: &Client, + push_client: &OutboundClient, details: &PushSubscription, mut body: Vec, push_timeout: Duration, vapid: Option<&Vapid>, urgency: Urgency, ) -> bool { - let mut client = push_client - .post(details.url.as_str()) - .timeout(push_timeout) - .header("TTL", "86400") - .header("Urgency", urgency.as_str()); + let mut client = match push_client.request(Method::POST, &details.url) { + Ok(client) => client, + Err(err) => { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Push subscription URL rejected", + Url = CompactString::from(&details.url), + Reason = err.as_str() + ); + return true; + } + } + .timeout(push_timeout) + .header("TTL", "86400") + .header("Urgency", urgency.as_str()); if let Some(authorization) = vapid.and_then(|vapid| vapid.authorization(&details.url, now())) { client = client.header(AUTHORIZATION, authorization); @@ -301,7 +306,7 @@ pub(crate) async fn http_request( trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to encrypt push subscription", - Url = details.url.to_string(), + Url = CompactString::from(&details.url), Reason = err ); return true; @@ -321,7 +326,7 @@ pub(crate) async fn http_request( if status.is_success() { trc::event!( PushSubscription(PushSubscriptionEvent::Success), - Url = details.url.to_string() + Url = CompactString::from(&details.url) ); true @@ -332,7 +337,7 @@ pub(crate) async fn http_request( trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", - Url = details.url.to_string(), + Url = CompactString::from(&details.url), Code = status.as_u16(), Reason = reason, ); @@ -344,98 +349,11 @@ pub(crate) async fn http_request( trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", - Url = details.url.to_string(), - Reason = err.to_string() + Url = CompactString::from(&details.url), + Reason = err.to_compact_string() ); false } } } - -fn is_same_organization(previous: &Url, next: &Url) -> bool { - if previous.scheme() == next.scheme() - && let (Some(previous_host), Some(next_host)) = (previous.host_str(), next.host_str()) - { - if is_ip_literal(previous_host) || is_ip_literal(next_host) { - previous_host == next_host - } else { - match (psl::domain_str(previous_host), psl::domain_str(next_host)) { - (Some(previous_domain), Some(next_domain)) => previous_domain == next_domain, - _ => previous_host == next_host, - } - } - } else { - false - } -} - -fn is_ip_literal(host: &str) -> bool { - host.strip_prefix('[') - .and_then(|host| host.strip_suffix(']')) - .unwrap_or(host) - .parse::() - .is_ok() -} - -#[cfg(test)] -mod tests { - use super::is_same_organization; - use reqwest::Url; - - #[test] - fn same_organization_redirects() { - for (previous, next, expected) in [ - ( - "https://push.example.org/a", - "https://push.example.org/b", - true, - ), - ( - "https://push.example.org/a", - "https://push2.example.org/b", - true, - ), - ("https://push.example.org/a", "https://example.org/b", true), - ( - "https://push.example.org/a", - "https://push.example.org:8443/b", - true, - ), - ( - "https://push.example.org/a", - "https://push.evil.org/b", - false, - ), - ( - "https://push.example.org/a", - "http://push.example.org/b", - false, - ), - ( - "https://push.example.co.uk/a", - "https://evil.co.uk/b", - false, - ), - ("https://1.2.3.4/a", "https://1.2.3.4/b", true), - ("https://1.2.3.4/a", "https://5.6.7.8/b", false), - ("https://1.2.3.4/a", "https://5.6.3.4/b", false), - ("https://1.2.3.4/a", "https://127.0.0.1/b", false), - ("https://[2606:4700::1111]/a", "https://[::1]/b", false), - ( - "https://[2606:4700::1111]/a", - "https://[2606:4700::1111]/b", - true, - ), - ] { - let previous = Url::parse(previous).unwrap(); - let next = Url::parse(next).unwrap(); - - assert_eq!( - is_same_organization(&previous, &next), - expected, - "{previous} -> {next}" - ); - } - } -} diff --git a/crates/services/src/state_manager/manager.rs b/crates/services/src/state_manager/manager.rs index 13314e28..cd1e6635 100644 --- a/crates/services/src/state_manager/manager.rs +++ b/crates/services/src/state_manager/manager.rs @@ -4,36 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Event, PURGE_EVERY, SEND_TIMEOUT, push::spawn_push_manager}; -use crate::state_manager::IpcSubscriber; +use super::{Event, push::spawn_push_manager}; use common::{ Inner, ipc::{BroadcastEvent, PushEvent}, }; -use std::{sync::Arc, time::Instant}; -use store::ahash::AHashMap; -use tokio::sync::mpsc::{self, error::TrySendError}; +use std::sync::Arc; +use tokio::sync::mpsc; use trc::ServerEvent; -#[derive(Default)] -struct Subscriber { - ipc: Vec, - is_push: bool, -} - -#[allow(clippy::unwrap_or_default)] pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver) { let push_tx = spawn_push_manager(inner.clone()); tokio::spawn(async move { - let mut subscribers: AHashMap = AHashMap::default(); - let mut last_purge = Instant::now(); - while let Some(event) = change_rx.recv().await { - let mut purge_needed = last_purge.elapsed() >= PURGE_EVERY; - match event { PushEvent::Stop => { + inner.ipc.push_subscribers.close_all(); if push_tx.send(Event::Reset).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), @@ -44,109 +31,27 @@ pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver { - for account_id in account_ids { - subscribers - .entry(account_id) - .or_default() - .ipc - .push(IpcSubscriber { - types, - tx: tx.clone(), - }); - } - } - PushEvent::PushServerRegister { activate, expired } => { - for account_id in activate { - subscribers.entry(account_id).or_default().is_push = true; - } - - for account_id in expired { - let mut remove_account = false; - if let Some(subscriber_list) = subscribers.get_mut(&account_id) { - subscriber_list.is_push = false; - remove_account = subscriber_list.ipc.is_empty(); - } - if remove_account { - subscribers.remove(&account_id); - } - } + inner + .ipc + .push_subscribers + .register_push(&activate, &expired); } - PushEvent::Publish { - notification, - broadcast, - } => { - // Publish event to cluster - if broadcast - && let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref() - && broadcast_tx - .send(BroadcastEvent::PushNotification(notification.clone())) - .await - .is_err() - { + PushEvent::Publish { notification } => { + if push_tx.send(Event::Push { notification }).await.is_err() { trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending broadcast event.", + Server(ServerEvent::ThreadError), + Details = "Error sending push updates.", CausedBy = trc::location!() ); } - - let account_id = notification.account_id(); - if let Some(subscribers) = subscribers.get(&account_id) { - for subscriber in &subscribers.ipc { - if let Some(notification) = notification.filter_types(&subscriber.types) - { - match subscriber.tx.try_send(notification) { - Ok(()) => {} - Err(TrySendError::Full(notification)) => { - let subscriber_tx = subscriber.tx.clone(); - - tokio::spawn(async move { - // Timeout after 500ms in case there is a blocked client - if subscriber_tx - .send_timeout(notification, SEND_TIMEOUT) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = - "Error sending state change to subscriber.", - CausedBy = trc::location!() - ); - } - }); - } - Err(TrySendError::Closed(_)) => { - purge_needed = true; - } - } - } - } - - if subscribers.is_push - && push_tx.send(Event::Push { notification }).await.is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending push updates.", - CausedBy = trc::location!() - ); - } - } } PushEvent::PushServerUpdate { account_id, broadcast, } => { - // Publish event to cluster if broadcast && let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref() && broadcast_tx @@ -161,7 +66,6 @@ pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver, mut change_rx: mpsc::Receiver, - tx: mpsc::Sender, -} +use utils::map::vec_map::VecMap; #[derive(Debug)] pub struct PushRegistration { @@ -55,16 +42,14 @@ pub enum Event { Reset, } -impl IpcSubscriber { - fn is_valid(&self) -> bool { - !self.tx.is_closed() - } -} - impl PushBatch { pub fn push(&mut self, notification: PushNotification) { match notification { - PushNotification::StateChange(state_change) => { + PushNotification::StateChange(state_change) + | PushNotification::ViewerStateChange(ViewerStateChange { + change: state_change, + .. + }) => { if !state_change.types.is_empty() { let states = self .state_changes @@ -113,77 +98,3 @@ fn merge_state(states: &mut VecMap, data_type: DataType, change } } -#[cfg(test)] -mod tests { - use super::PushBatch; - use common::ipc::{EmailPush, PushNotification}; - use jmap_proto::types::state::State; - use types::{ - id::Id, - type_state::{DataType, StateChange}, - }; - use utils::map::bitmap::Bitmap; - - fn state_change(change_id: u64, types: [DataType; N]) -> PushNotification { - PushNotification::StateChange(StateChange { - account_id: 1, - change_id, - types: Bitmap::from_iter(types), - }) - } - - fn email_push(email_id: u32) -> PushNotification { - PushNotification::EmailPush(EmailPush { - account_id: 1, - email_id, - change_id: email_id.into(), - }) - } - - #[test] - fn batch_keeps_newest_state_per_type() { - let mut batch = PushBatch::default(); - batch.push(state_change(5, [DataType::Email])); - batch.push(state_change(7, [DataType::Email, DataType::Mailbox])); - batch.push(state_change(6, [DataType::Mailbox])); - - let mut failed = PushBatch::default(); - failed.push(state_change( - 4, - [DataType::Email, DataType::Mailbox, DataType::Thread], - )); - batch.merge_failed(failed); - - let states = batch.state_changes.get(&Id::from(1u32)).unwrap(); - assert_eq!(states.len(), 3); - assert_eq!(states.get(&DataType::Email), Some(&State::Exact(7))); - assert_eq!(states.get(&DataType::Mailbox), Some(&State::Exact(7))); - assert_eq!(states.get(&DataType::Thread), Some(&State::Exact(4))); - - assert!(!batch.is_empty()); - batch.clear(); - assert!(batch.is_empty()); - } - - #[test] - fn failed_notifications_are_retried_first() { - let mut batch = PushBatch::default(); - batch.push(email_push(3)); - - let mut failed = PushBatch::default(); - failed.push(email_push(1)); - failed.push(email_push(2)); - batch.merge_failed(failed); - - let email_ids = batch - .notifications - .iter() - .filter_map(|notification| match notification { - PushNotification::EmailPush(email_push) => Some(email_push.email_id), - _ => None, - }) - .collect::>(); - assert_eq!(email_ids, [1, 2, 3]); - assert!(batch.state_changes.is_empty()); - } -} diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index ecc585ab..c512ea72 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -15,7 +15,6 @@ use common::{ ipc::{PushEvent, PushNotification}, }; use email::push::{PushSubscription, PushSubscriptions, Urgency}; -use reqwest::Client; use std::{ collections::hash_map::Entry, sync::Arc, @@ -24,15 +23,31 @@ use std::{ use store::{ ValueKey, ahash::{AHashMap, AHashSet}, - write::{AlignedBytes, Archive, now}, + write::{Archive, ArchiveBytes, now}, }; use tokio::sync::mpsc; use trc::{AddContext, PushSubscriptionEvent, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; +use utils::outbound::OutboundClient; pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let (push_tx_, mut push_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); + let (register_tx, mut register_rx) = mpsc::unbounded_channel::(); + let router_tx = inner.ipc.push_tx.clone(); + + tokio::spawn(async move { + while let Some(event) = register_rx.recv().await { + if router_tx.send(event).await.is_err() { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending push server registration.", + CausedBy = trc::location!() + ); + break; + } + } + }); tokio::spawn(async move { let mut push_servers: AHashMap = AHashMap::default(); @@ -110,16 +125,11 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { // Subscribe to push events if !account_push_ids.is_empty() - && server - .inner - .ipc - .push_tx - .clone() + && register_tx .send(PushEvent::PushServerRegister { activate: account_push_ids.keys().copied().collect(), expired: vec![], }) - .await .is_err() { trc::event!( @@ -244,7 +254,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { if let Some(subscription) = newest_unverified { let current_time = Instant::now(); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if subscription.url.contains("skip_checks") { last_verify.remove(&account_id); } @@ -335,16 +345,11 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } } if (!active_account_ids.is_empty() || !inactive_account_ids.is_empty()) - && server - .inner - .ipc - .push_tx - .clone() + && register_tx .send(PushEvent::PushServerRegister { activate: active_account_ids, expired: inactive_account_ids, }) - .await .is_err() { trc::event!( @@ -355,7 +360,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } } Event::Push { notification } => { - let account_id = notification.account_id(); + let account_id = notification.recipient(); if let Some(ids) = account_push_ids.get_mut(&account_id) { let current_time = now(); let mut remove_ids = Vec::new(); @@ -415,16 +420,11 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } if ids.is_empty() { account_push_ids.remove(&account_id); - if server - .inner - .ipc - .push_tx - .clone() + if register_tx .send(PushEvent::PushServerRegister { activate: vec![], expired: vec![account_id], }) - .await .is_err() { trc::event!( @@ -528,7 +528,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } fn send_verification( - push_client: &Client, + push_client: &OutboundClient, subscription: Arc, server: &Server, push_timeout: Duration, @@ -571,7 +571,7 @@ async fn load_push_subscriptions( if let Some(push_subscriptions) = server .store() - .get_value::>(ValueKey::property( + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, diff --git a/crates/services/src/task_manager/acme.rs b/crates/services/src/task_manager/acme.rs deleted file mode 100644 index 662fc4c9..00000000 --- a/crates/services/src/task_manager/acme.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::task_manager::{TaskFailureType, TaskResult}; -use common::{Server, network::acme::AcmeError}; -use registry::schema::structs::TaskDomainManagement; -use std::time::Duration; -use store::write::now; - -pub(crate) trait AcmeTask: Sync + Send { - fn acme_management( - &self, - task: &TaskDomainManagement, - ) -> impl Future + Send; -} - -impl AcmeTask for Server { - async fn acme_management(&self, task: &TaskDomainManagement) -> TaskResult { - match acme_management(self, task).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to run ACME task") - ); - result - } - } - } -} - -#[cfg(not(feature = "test_mode"))] -const MAX_RETRIES: u32 = 3; - -#[cfg(feature = "test_mode")] -const MAX_RETRIES: u32 = 5; - -#[allow(unused_variables)] -async fn acme_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { - let mut last_temporary_error = Ok(TaskResult::temporary("")); - for retry in 0..MAX_RETRIES { - last_temporary_error = match Box::pin(server.acme_renew(task.domain_id)).await { - Ok(tasks) => return Ok(TaskResult::Success(tasks)), - Err(err) => { - if !matches!( - err, - AcmeError::NotDue(_) - | AcmeError::Internal(_) - | AcmeError::AuthInvalid(_) - | AcmeError::OrderInvalid(_) - | AcmeError::AuthTimeout { .. } - | AcmeError::Backoff { .. } - ) { - trc::event!( - Acme(trc::AcmeEvent::Error), - Id = task.domain_id.to_string(), - Total = retry as u64, - Reason = err.to_string(), - ); - } - - match err { - AcmeError::Crypto(_) - | AcmeError::Invalid(_) - | AcmeError::NotDue(_) - | AcmeError::ChallengeNotSupported { .. } => { - return Ok(TaskResult::permanent(err.to_string())); - } - AcmeError::OrderInvalid(_) | AcmeError::Json(_) | AcmeError::Registry(_) => { - return Ok(TaskResult::perpetual(err.to_string())); - } - AcmeError::Http(_) - | AcmeError::HttpStatus(_) - | AcmeError::Dns(_) - | AcmeError::AuthInvalid(_) => Ok(TaskResult::temporary(err.to_string())), - AcmeError::OrderTimeout { max_retries } - | AcmeError::AuthTimeout { max_retries } => Ok(TaskResult::Failure { - typ: TaskFailureType::Temporary, - message: err.to_string(), - max_attempts: (max_retries as u64).into(), - }), - AcmeError::Backoff { max_retries, wait } => { - return if let Some(wait) = wait { - Ok(TaskResult::Failure { - typ: TaskFailureType::Retry(now() + wait.as_secs()), - message: err.to_string(), - max_attempts: (max_retries as u64).into(), - }) - } else { - Ok(TaskResult::Failure { - typ: TaskFailureType::Temporary, - message: err.to_string(), - max_attempts: (max_retries as u64).into(), - }) - }; - } - AcmeError::Internal(error) => return Err(error), - } - } - }; - - #[cfg(not(feature = "test_mode"))] - tokio::time::sleep(Duration::from_secs(1 << (retry + 5))).await; - - #[cfg(feature = "test_mode")] - tokio::time::sleep(Duration::from_secs(1)).await; - } - last_temporary_error -} diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs deleted file mode 100644 index bb7fab9c..00000000 --- a/crates/services/src/task_manager/alarm.rs +++ /dev/null @@ -1,669 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use calcard::icalendar::{ - ArchivedICalendarParameterName, ArchivedICalendarProperty, ICalendarProperty, -}; -use common::{ - DEFAULT_LOGO_BASE64, Server, - auth::{AccountInfo, BuildAccessToken}, - config::groupware::CalendarTemplateVariable, - ipc::{CalendarAlert, PushNotification}, - network::{ServerInstance, stream::NullIo}, -}; -use groupware::{ - calendar::{ArchivedCalendarEvent, CalendarEvent}, - scheduling::{ - ItipTime, ItipValue, - format::{DateStyle, TextFormatter, hyperlink}, - }, - strip_mailto_scheme, -}; -use mail_builder::{ - MessageBuilder, - headers::{HeaderType, content_type::ContentType}, - mime::{BodyPart, MimePart}, -}; -use mail_parser::decoders::html::html_to_text; -use registry::{ - schema::{ - enums::Permission, - structs::{TaskCalendarAlarmEmail, TaskCalendarAlarmNotification}, - }, - types::EnumImpl, -}; -use smtp::core::{Session, SessionData}; -use smtp_proto::{MailFrom, RcptTo}; -use std::{sync::Arc, time::Duration}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive, now}, -}; -use trc::{AddContext, TaskManagerEvent}; -use types::collection::Collection; -use utils::{sanitize_email, template::Variables}; - -use crate::task_manager::TaskResult; - -pub(crate) trait SendAlarmTask: Sync + Send { - fn send_display_alarm( - &self, - task: &TaskCalendarAlarmNotification, - ) -> impl Future + Send; - - fn send_email_alarm( - &self, - task: &TaskCalendarAlarmEmail, - server_instance: Arc, - ) -> impl Future + Send; -} - -impl SendAlarmTask for Server { - async fn send_display_alarm(&self, task: &TaskCalendarAlarmNotification) -> TaskResult { - match send_display_alarm(self, task).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.account_id(task.account_id.document_id()) - .document_id(task.document_id.document_id()) - .caused_by(trc::location!()) - .details("Failed to process e-mail alarm") - ); - result - } - } - } - - async fn send_email_alarm( - &self, - task: &TaskCalendarAlarmEmail, - server_instance: Arc, - ) -> TaskResult { - match send_email_alarm(self, task, server_instance).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.account_id(task.account_id.document_id()) - .document_id(task.document_id.document_id()) - .caused_by(trc::location!()) - .details("Failed to process e-mail alarm") - ); - result - } - } - } -} - -async fn send_email_alarm( - server: &Server, - task: &TaskCalendarAlarmEmail, - server_instance: Arc, -) -> trc::Result { - // Obtain access token - let account_id = task.account_id.document_id(); - let document_id = task.document_id.document_id(); - let access_token = server - .access_token(account_id) - .await - .caused_by(trc::location!())? - .build(); - - if !access_token.has_permission(Permission::CalendarAlarmsSend) { - trc::event!( - Calendar(trc::CalendarEvent::AlarmSkipped), - Reason = "Account does not have permission to send calendar alarms", - AccountId = account_id, - DocumentId = document_id, - ); - return Ok(TaskResult::Success(vec![])); - } - let account_info = server - .account_info(account_id) - .await - .caused_by(trc::location!())?; - - if account_info.name().is_empty() { - trc::event!( - Calendar(trc::CalendarEvent::AlarmFailed), - Reason = "Account does not have any email addresses", - AccountId = account_id, - DocumentId = document_id, - ); - return Ok(TaskResult::Success(vec![])); - } - - // Fetch event - let Some(event_) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEvent, - document_id, - )) - .await - .caused_by(trc::location!())? - else { - trc::event!( - TaskManager(TaskManagerEvent::MetadataNotFound), - Details = "Calendar Event metadata not found", - AccountId = account_id, - DocumentId = document_id, - ); - - return Ok(TaskResult::Success(vec![])); - }; - - // Unarchive event - let event = event_ - .unarchive::() - .caused_by(trc::location!())?; - - // Build message body - let account_main_email = account_info.name(); - let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); - let logo_cid = format!("logo.{}@{account_main_domain}", now()); - let Some(tpl) = build_template(server, &account_info, task, event, &logo_cid).await? else { - return Ok(TaskResult::Success(vec![])); - }; - let txt_body = html_to_text(&tpl.body); - - // Obtain logo image - let logo = match server.logo_resource(account_main_domain).await { - Ok(logo) => logo, - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to fetch logo image") - ); - None - } - }; - let logo = if let Some(logo) = &logo { - MimePart::new( - ContentType::new(logo.content_type.as_ref()), - BodyPart::Binary(logo.contents.as_slice().into()), - ) - } else { - MimePart::new( - ContentType::new("image/png"), - BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), - ) - .transfer_encoding("base64") - } - .inline() - .cid(&logo_cid); - - // Build message - let mail_from = if let Some(from_email) = &server.core.groupware.alarms_from_email { - from_email.to_string() - } else { - format!("calendar-notification@{account_main_domain}") - }; - let message = MessageBuilder::new() - .from(( - server.core.groupware.alarms_from_name.as_str(), - mail_from.as_str(), - )) - .header("To", HeaderType::Text(tpl.to.as_str().into())) - .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) - .header("Reply-To", HeaderType::Text(account_main_email.into())) - .message_id(server.core.network.message_id()) - .subject(tpl.subject) - .body(MimePart::new( - ContentType::new("multipart/related"), - BodyPart::Multipart(vec![ - MimePart::new( - ContentType::new("multipart/alternative"), - BodyPart::Multipart(vec![ - MimePart::new( - ContentType::new("text/plain"), - BodyPart::Text(txt_body.into()), - ), - MimePart::new( - ContentType::new("text/html"), - BodyPart::Text(tpl.body.into()), - ), - ]), - ), - logo, - ]), - )) - .write_to_vec() - .unwrap_or_default(); - - // Send message - let server_ = server.clone(); - let mail_from = account_main_email.to_string(); - let to = tpl.to; - let result = tokio::spawn(async move { - let mut session = Session::::local( - server_, - server_instance, - SessionData::local(account_info, None, vec![], vec![], 0), - ); - - // MAIL FROM - let _ = session - .handle_mail_from(MailFrom { - address: mail_from.into(), - ..Default::default() - }) - .await; - if let Some(error) = session.has_failed() { - return Err(format!("Server rejected MAIL-FROM: {}", error.trim())); - } - - // RCPT TO - session.params.rcpt_errors_wait = Duration::from_secs(0); - let _ = session - .handle_rcpt_to(RcptTo { - address: to.into(), - ..Default::default() - }) - .await; - if let Some(error) = session.has_failed() { - return Err(format!("Server rejected RCPT-TO: {}", error.trim())); - } - - // DATA - session.data.message = message; - let response = session.queue_message().await; - if let smtp::core::State::Accepted(queue_id) = session.state { - Ok(queue_id) - } else { - Err(format!( - "Server rejected DATA: {}", - std::str::from_utf8(&response).unwrap().trim() - )) - } - }) - .await; - - match result { - Ok(Ok(queue_id)) => { - trc::event!( - Calendar(trc::CalendarEvent::AlarmSent), - AccountId = account_id, - DocumentId = document_id, - QueueId = queue_id, - ); - } - Ok(Err(err)) => { - trc::event!( - Calendar(trc::CalendarEvent::AlarmFailed), - AccountId = account_id, - DocumentId = document_id, - Reason = err, - ); - } - Err(_) => { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Join Error", - AccountId = account_id, - DocumentId = document_id, - CausedBy = trc::location!(), - ); - return Ok(TaskResult::temporary("Thread join error")); - } - } - - build_next_alarm(server, account_id, document_id, event) -} - -async fn send_display_alarm( - server: &Server, - task: &TaskCalendarAlarmNotification, -) -> trc::Result { - // Fetch event - let account_id = task.account_id.document_id(); - let document_id = task.document_id.document_id(); - let Some(event_) = server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEvent, - document_id, - )) - .await - .caused_by(trc::location!())? - else { - trc::event!( - TaskManager(TaskManagerEvent::MetadataNotFound), - Details = "Calendar Event metadata not found", - AccountId = account_id, - DocumentId = document_id, - ); - - return Ok(TaskResult::Success(vec![])); - }; - - // Unarchive event - let event = event_ - .unarchive::() - .caused_by(trc::location!())?; - - let recurrence_id = task.recurrence_id; - - let ical = &event.data.event; - server - .broadcast_push_notification(PushNotification::CalendarAlert(CalendarAlert { - account_id, - event_id: document_id, - recurrence_id, - uid: ical.uids().next().unwrap_or_default().to_string(), - alert_id: ical - .components - .get(task.alarm_id as usize) - .and_then(|c| c.property(&ICalendarProperty::Jsid)) - .and_then(|v| v.values.first()) - .and_then(|v| v.as_text()) - .map(|v| v.to_string()) - .unwrap_or_else(|| { - format!( - "k{}", - ical.components - .get(task.event_id as usize) - .and_then(|c| c - .component_ids - .iter() - .position(|id| id.to_native() == task.alarm_id as u32)) - .unwrap_or_default() - + 1 - ) - }), - })) - .await; - - build_next_alarm(server, account_id, document_id, event) -} - -fn build_next_alarm( - server: &Server, - account_id: u32, - document_id: u32, - event: &ArchivedCalendarEvent, -) -> trc::Result { - // Find next alarm time and write to task queue - let now = now() as i64; - if let Some(next_alarm) = - event - .data - .next_alarm(now, Default::default()) - .and_then(|next_alarm| { - // Verify minimum interval - let max_next_alarm = now + server.core.groupware.alarms_minimum_interval; - if next_alarm.alarm_time < max_next_alarm { - trc::event!( - Calendar(trc::CalendarEvent::AlarmSkipped), - Reason = "Next alarm skipped due to minimum interval", - Details = next_alarm.alarm_time - now, - AccountId = account_id, - DocumentId = document_id, - ); - event.data.next_alarm(max_next_alarm, Default::default()) - } else { - Some(next_alarm) - } - }) - { - Ok(TaskResult::Update( - next_alarm.build_write_ops(account_id, document_id), - )) - } else { - Ok(TaskResult::Success(vec![])) - } -} - -struct Details { - to: String, - subject: String, - body: String, -} - -async fn build_template( - server: &Server, - account_info: &AccountInfo, - alarm: &TaskCalendarAlarmEmail, - event: &ArchivedCalendarEvent, - logo_cid: &str, -) -> trc::Result> { - let account_id = alarm.account_id.document_id(); - let document_id = alarm.document_id.document_id(); - let (Some(event_component), Some(alarm_component)) = ( - event.data.event.components.get(alarm.event_id as usize), - event.data.event.components.get(alarm.alarm_id as usize), - ) else { - trc::event!( - TaskManager(TaskManagerEvent::MetadataNotFound), - Details = "Calendar Alarm component not found", - AccountId = account_id, - DocumentId = document_id, - ); - return Ok(None); - }; - - // Build webcal URI - let webcal_uri = match event.webcal_uri(server, account_info).await { - Ok(uri) => uri, - Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to generate webcal URI") - ); - String::from("#") - } - }; - - // Obtain alarm details - let mut summary = None; - let mut description = None; - let mut rcpt_to = None; - let mut location = None; - let mut conference = None; - let mut organizer = None; - let mut guests = vec![]; - - for entry in alarm_component.entries.iter() { - match &entry.name { - ArchivedICalendarProperty::Summary => { - summary = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Description => { - description = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Attendee => { - rcpt_to = entry - .values - .first() - .and_then(|v| v.as_text()) - .map(strip_mailto_scheme) - .and_then(sanitize_email); - } - _ => {} - } - } - - for entry in event_component.entries.iter() { - match &entry.name { - ArchivedICalendarProperty::Summary if summary.is_none() => { - summary = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Description if description.is_none() => { - description = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Location => { - location = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Conference if conference.is_none() => { - conference = entry.values.first().and_then(|v| v.as_text()); - } - ArchivedICalendarProperty::Organizer | ArchivedICalendarProperty::Attendee => { - let email = entry - .values - .first() - .and_then(|v| v.as_text()) - .map(strip_mailto_scheme); - let name = entry.params.iter().find_map(|param| { - if let ArchivedICalendarParameterName::Cn = param.name { - param.value.as_text() - } else { - None - } - }); - - if email.is_some() || name.is_some() { - if matches!(entry.name, ArchivedICalendarProperty::Organizer) { - organizer = Some((email, name)); - } else { - guests.push((email, name)); - } - } - } - _ => {} - } - } - - // Validate recipient - let rcpt_to = if let Some(rcpt_to) = rcpt_to { - if server.core.groupware.alarms_allow_external_recipients - || account_info.addresses().contains(&rcpt_to) - { - rcpt_to - } else { - trc::event!( - Calendar(trc::CalendarEvent::AlarmRecipientOverride), - Reason = "External recipient not allowed for calendar alarms", - Details = rcpt_to, - AccountId = account_id, - DocumentId = document_id, - ); - - account_info.name().to_string() - } - } else { - account_info.name().to_string() - }; - - - #[cfg(not(feature = "enterprise"))] - let template = &server.core.groupware.alarms_template; - let formatter = TextFormatter::new(account_info.locale().as_str())?; - let locale = formatter.locale; - - let start = formatter.field_to_string( - &ItipValue::Time(ItipTime { - start: alarm.event_start.timestamp(), - tz_id: alarm.event_start_tz as u16, - }), - DateStyle::Short, - ); - let end = formatter.field_to_string( - &ItipValue::Time(ItipTime { - start: alarm.event_end.timestamp(), - tz_id: alarm.event_end_tz as u16, - }), - DateStyle::Short, - ); - let subject = format!( - "{}: {} @ {}", - locale.calendar_alarm_subject_prefix, - summary.or(description).unwrap_or("No Subject"), - start - ); - let organizer = organizer - .map(|(email, name)| match (email, name) { - (Some(email), Some(name)) => format!("{} <{}>", name, email), - (Some(email), None) => email.to_string(), - (None, Some(name)) => name.to_string(), - _ => unreachable!(), - }) - .unwrap_or_else(|| account_info.name().to_string()); - let logo_cid = format!("cid:{logo_cid}"); - let mut variables = Variables::new(); - variables.insert_single(CalendarTemplateVariable::PageTitle, subject.as_str()); - variables.insert_single(CalendarTemplateVariable::Lang, locale.name); - variables.insert_single(CalendarTemplateVariable::Dir, locale.direction); - variables.insert_single( - CalendarTemplateVariable::Header, - locale.calendar_alarm_header, - ); - variables.insert_single( - CalendarTemplateVariable::Footer, - locale.calendar_alarm_footer, - ); - variables.insert_single( - CalendarTemplateVariable::ActionName, - locale.calendar_alarm_open, - ); - variables.insert_single(CalendarTemplateVariable::ActionUrl, webcal_uri.as_str()); - variables.insert_single( - CalendarTemplateVariable::AttendeesTitle, - locale.calendar_attendees, - ); - if let Some(summary) = summary.filter(|summary| !summary.is_empty()) { - variables.insert_single(CalendarTemplateVariable::EventTitle, summary); - } - variables.insert_single(CalendarTemplateVariable::LogoCid, logo_cid.as_str()); - if let Some(description) = description { - variables.insert_single(CalendarTemplateVariable::EventDescription, description); - } - variables.insert_block( - CalendarTemplateVariable::EventDetails, - [ - Some(vec![ - (CalendarTemplateVariable::Key, locale.calendar_start), - (CalendarTemplateVariable::Value, start.as_str()), - ]), - Some(vec![ - (CalendarTemplateVariable::Key, locale.calendar_end), - (CalendarTemplateVariable::Value, end.as_str()), - ]), - location.map(|location| { - vec![ - (CalendarTemplateVariable::Key, locale.calendar_location), - (CalendarTemplateVariable::Value, location), - ] - }), - conference.map(|conference| { - let mut detail = vec![ - (CalendarTemplateVariable::Key, locale.calendar_conference), - (CalendarTemplateVariable::Value, conference), - ]; - if let Some(link) = hyperlink(conference) { - detail.push((CalendarTemplateVariable::Link, link)); - } - detail - }), - Some(vec![ - (CalendarTemplateVariable::Key, locale.calendar_organizer), - (CalendarTemplateVariable::Value, organizer.as_str()), - ]), - ] - .into_iter() - .flatten(), - ); - if !guests.is_empty() { - variables.insert_block( - CalendarTemplateVariable::Attendees, - guests.into_iter().map(|(email, name)| { - [ - (CalendarTemplateVariable::Key, name.unwrap_or_default()), - (CalendarTemplateVariable::Value, email.unwrap_or_default()), - ] - }), - ); - } - Ok(Some(Details { - to: rcpt_to, - body: template.eval(&variables), - subject, - })) -} diff --git a/crates/services/src/task_manager/destroy_account.rs b/crates/services/src/task_manager/destroy_account.rs deleted file mode 100644 index 565ea224..00000000 --- a/crates/services/src/task_manager/destroy_account.rs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::task_manager::{TaskResult, deferred_retry_time}; -use common::Server; -use email::{message::metadata::MessageMetadata, sieve::SieveScript}; -use groupware::file::FileNode; -use registry::{ - schema::{ - prelude::{ObjectType, Property}, - structs::{ArchivedItem, TaskDestroyAccount}, - }, - types::EnumImpl, -}; -use store::{ - SerializeInfallible, ValueKey, - registry::RegistryQuery, - search::SearchQuery, - write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, SearchIndex, ValueClass}, -}; -use trc::AddContext; -use types::{ - blob_hash::BlobHash, - collection::Collection, - field::{EmailField, Field}, - id::Id, -}; - -pub(crate) trait DestroyAccountTask: Sync + Send { - fn destroy_account(&self, task: &TaskDestroyAccount) - -> impl Future + Send; -} - -impl DestroyAccountTask for Server { - async fn destroy_account(&self, task: &TaskDestroyAccount) -> TaskResult { - match destroy_account(self, task).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::deferred(deferred_retry_time(&err), err.to_string()); - trc::error!( - err.account_id(task.account_id.document_id()) - .details("Failed to destroy account") - ); - result - } - } - } -} - -async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Result { - let account_id = task.account_id.document_id(); - - // Destroy public keys and masked emails - for object in [ObjectType::PublicKey, ObjectType::MaskedEmail] { - let mut batch = BatchBuilder::new(); - let ids = server - .registry() - .query::>(RegistryQuery::new(object).with_account(account_id)) - .await?; - let object_id = object.to_id(); - - for id in ids { - batch - .clear(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id: id.id(), - })) - .clear(ValueClass::Registry(RegistryClass::IndexId { - object_id, - item_id: id.id(), - })) - .clear(ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId as u16, - object_id, - item_id: id.id(), - key: (account_id as u64).serialize(), - })) - .clear(ValueClass::Registry(RegistryClass::Reference { - to_object_id: ObjectType::Account as u16, - to_item_id: account_id as u64, - from_object_id: object_id, - from_item_id: id.id(), - })); - } - - if !batch.is_empty() { - server.store().write(batch.build_all()).await?; - } - } - - // Remove archived items - let mut batch = BatchBuilder::new(); - let ids = server - .registry() - .query::>(RegistryQuery::new(ObjectType::ArchivedItem).with_account(account_id)) - .await?; - for id in ids { - let object_id = ObjectType::ArchivedItem.to_id(); - let item_id = id.id(); - - if let Some(item) = server - .store() - .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id, - }))) - .await? - { - let until = item.archived_until().timestamp() as u64; - let blob_hash = item.into_blob_id().hash; - - batch - .with_account_id(account_id) - .clear(BlobOp::Link { - hash: blob_hash, - to: BlobLink::Temporary { until }, - }) - .clear(ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId.to_id(), - object_id, - item_id, - key: (account_id as u64).serialize(), - })) - .clear(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id, - })); - } - } - if !batch.is_empty() { - server.store().write(batch.build_all()).await?; - } - - // Remove search index - for index in [ - SearchIndex::Email, - SearchIndex::Contacts, - SearchIndex::Calendar, - ] { - server - .search_store() - .unindex(SearchQuery::new(index).with_account_id(account_id)) - .await?; - } - - // Unlink all accounts's blobs - destroy_account_blobs(server, account_id).await?; - - // Destroy account data - server - .store() - .danger_destroy_account(account_id) - .await - .caused_by(trc::location!())?; - - Ok(TaskResult::Success(vec![])) -} - -pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { - let mut delete_keys = Vec::new(); - for (collection, field) in [ - (Collection::Email, u8::from(EmailField::Metadata)), - (Collection::FileNode, u8::from(Field::ARCHIVE)), - (Collection::SieveScript, u8::from(Field::ARCHIVE)), - ] { - server - .all_archives(account_id, collection, field, |document_id, archive| { - match collection { - Collection::Email => { - let message = archive.unarchive::()?; - delete_keys.push(( - collection, - document_id, - BlobHash::from(&message.blob_hash), - )); - } - Collection::FileNode => { - if let Some(file) = archive.unarchive::()?.file.as_ref() { - delete_keys.push(( - collection, - document_id, - BlobHash::from(&file.blob_hash), - )); - } - } - Collection::SieveScript => { - let sieve = archive.unarchive::()?; - delete_keys.push(( - collection, - document_id, - BlobHash::from(&sieve.blob_hash), - )); - } - _ => {} - } - Ok(()) - }) - .await - .caused_by(trc::location!())?; - } - - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - - for (collection, document_id, hash) in delete_keys { - if batch.is_large_batch() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - batch.with_account_id(account_id); - } - batch - .with_collection(collection) - .with_document(document_id) - .clear(ValueClass::Blob(BlobOp::Link { - hash, - to: BlobLink::Document, - })); - } - - if !batch.is_empty() { - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - - Ok(()) -} diff --git a/crates/services/src/task_manager/imip.rs b/crates/services/src/task_manager/imip.rs deleted file mode 100644 index fdd82fb8..00000000 --- a/crates/services/src/task_manager/imip.rs +++ /dev/null @@ -1,612 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::task_manager::TaskResult; -use calcard::icalendar::{ICalendarParticipationStatus, ICalendarProperty}; -use common::{ - DEFAULT_LOGO_BASE64, Server, - auth::AccountInfo, - config::groupware::CalendarTemplateVariable, - network::{ServerInstance, stream::NullIo}, -}; -use groupware::{ - calendar::itip::ItipIngest, - scheduling::{ - ItipSummary, ItipValue, - format::{DateStyle, TextFormatter, hyperlink}, - }, -}; -use mail_builder::{ - MessageBuilder, - headers::{HeaderType, content_type::ContentType}, - mime::{BodyPart, MimePart}, -}; -use mail_parser::decoders::html::html_to_text; -use registry::{schema::structs::TaskCalendarItipMessage, types::EnumImpl}; -use smtp::core::{Session, SessionData}; -use smtp_proto::{MailFrom, RcptTo}; -use std::{sync::Arc, time::Duration}; -use store::{ahash::AHashMap, write::now}; -use trc::AddContext; -use utils::template::{Variable, Variables}; - -pub(crate) trait SendImipTask: Sync + Send { - fn send_imip( - &self, - task: &TaskCalendarItipMessage, - server_instance: Arc, - ) -> impl Future + Send; -} - -impl SendImipTask for Server { - async fn send_imip( - &self, - task: &TaskCalendarItipMessage, - server_instance: Arc, - ) -> TaskResult { - match send_imip(self, task, server_instance).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.account_id(task.account_id.document_id()) - .document_id(task.document_id.document_id()) - .caused_by(trc::location!()) - .details("Failed to send iMIP message") - ); - result - } - } - } -} - -async fn send_imip( - server: &Server, - imip: &TaskCalendarItipMessage, - server_instance: Arc, -) -> trc::Result { - // Obtain iMIP payload - let account_id = imip.account_id.document_id(); - let document_id = imip.document_id.document_id(); - - let sender_domain = imip - .messages - .iter() - .next() - .and_then(|msg| msg.from.rsplit('@').next()) - .unwrap_or("localhost"); - - // Obtain logo image - let logo = match server.logo_resource(sender_domain).await { - Ok(logo) => logo, - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to fetch logo image") - ); - None - } - }; - let logo_cid = format!("logo.{}@{sender_domain}", now()); - let logo = if let Some(logo) = &logo { - MimePart::new( - ContentType::new(logo.content_type.as_ref()), - BodyPart::Binary(logo.contents.as_slice().into()), - ) - } else { - MimePart::new( - ContentType::new("image/png"), - BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), - ) - .transfer_encoding("base64") - } - .inline() - .cid(&logo_cid); - - let account_info = server - .account_info(account_id) - .await - .caused_by(trc::location!())?; - - for itip_message in imip.messages.iter() { - let Ok(summary) = serde_json::from_str::(&itip_message.summary) else { - return Ok(TaskResult::permanent( - "Failed to parse iMIP message summary.", - )); - }; - - let organizer_info = match server - .account_id_from_email(itip_message.from.as_str(), true) - .await - { - Ok(Some(sender_id)) if sender_id != account_id => { - match server.account_info(sender_id).await { - Ok(info) => Some(info), - Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to load organizer account for iMIP sender") - ); - None - } - } - } - Ok(_) => None, - Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to resolve organizer account for iMIP sender") - ); - None - } - }; - let sender_info = organizer_info.as_ref().unwrap_or(&account_info); - - for recipient in itip_message.to.iter() { - // Build template - let tpl = build_itip_template( - server, - &account_info, - account_id, - document_id, - itip_message.from.as_str(), - recipient.as_str(), - &summary, - &logo_cid, - ) - .await?; - let txt_body = html_to_text(&tpl.body); - - // Build message - let message = MessageBuilder::new() - .from(( - sender_info.description().unwrap_or(sender_info.name()), - itip_message.from.as_str(), - )) - .to(recipient.as_str()) - .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) - .header( - "Reply-To", - HeaderType::Text(itip_message.from.as_str().into()), - ) - .message_id(server.core.network.message_id()) - .subject(&tpl.subject) - .body(MimePart::new( - ContentType::new("multipart/mixed"), - BodyPart::Multipart(vec![ - MimePart::new( - ContentType::new("multipart/related"), - BodyPart::Multipart(vec![ - MimePart::new( - ContentType::new("multipart/alternative"), - BodyPart::Multipart(vec![ - MimePart::new( - ContentType::new("text/plain"), - BodyPart::Text(txt_body.into()), - ), - MimePart::new( - ContentType::new("text/html"), - BodyPart::Text(tpl.body.as_str().into()), - ), - MimePart::new( - ContentType::new("text/calendar") - .attribute("method", summary.method()) - .attribute("charset", "utf-8"), - BodyPart::Text( - itip_message.i_calendar_data.as_str().into(), - ), - ), - ]), - ), - logo.clone(), - ]), - ), - MimePart::new( - ContentType::new("application/ics").attribute("name", "event.ics"), - BodyPart::Text(itip_message.i_calendar_data.as_str().into()), - ) - .attachment("event.ics"), - ]), - )) - .write_to_vec() - .unwrap_or_default(); - - // Send message - let server_ = server.clone(); - let server_instance = server_instance.clone(); - let sender_info = sender_info.clone(); - let from = itip_message.from.to_string(); - let to = recipient.to_string(); - tokio::spawn(async move { - let mut session = Session::::local( - server_, - server_instance, - SessionData::local(sender_info, None, vec![], vec![], 0), - ); - - // MAIL FROM - let _ = session - .handle_mail_from(MailFrom { - address: from.as_str().into(), - ..Default::default() - }) - .await; - if let Some(error) = session.has_failed() { - trc::event!( - Calendar(trc::CalendarEvent::ItipMessageError), - AccountId = account_id, - DocumentId = document_id, - From = from, - To = to, - Reason = format!("Server rejected MAIL-FROM: {}", error.trim()), - ); - return; - } - - // RCPT TO - session.params.rcpt_errors_wait = Duration::from_secs(0); - let _ = session - .handle_rcpt_to(RcptTo { - address: to.as_str().into(), - ..Default::default() - }) - .await; - if let Some(error) = session.has_failed() { - trc::event!( - Calendar(trc::CalendarEvent::ItipMessageError), - AccountId = account_id, - DocumentId = document_id, - From = from, - To = to, - Reason = format!("Server rejected RCPT-TO: {}", error.trim()), - ); - return; - } - - // DATA - session.data.message = message; - let response = session.queue_message().await; - if let smtp::core::State::Accepted(queue_id) = session.state { - trc::event!( - Calendar(trc::CalendarEvent::ItipMessageSent), - From = from, - To = to, - AccountId = account_id, - DocumentId = document_id, - QueueId = queue_id, - ); - } else { - trc::event!( - Calendar(trc::CalendarEvent::ItipMessageError), - From = from, - To = to, - AccountId = account_id, - DocumentId = document_id, - Reason = format!( - "Server rejected DATA: {}", - std::str::from_utf8(&response).unwrap().trim() - ), - ); - } - }) - .await - .map_err(|_| { - trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError)) - .caused_by(trc::location!()) - })?; - } - } - - Ok(TaskResult::Success(vec![])) -} - -pub struct Details { - pub subject: String, - pub body: String, -} - -#[allow(clippy::too_many_arguments)] -pub async fn build_itip_template( - server: &Server, - account_info: &AccountInfo, - account_id: u32, - document_id: u32, - from: &str, - to: &str, - summary: &ItipSummary, - logo_cid: &str, -) -> trc::Result
{ - #[cfg(not(feature = "enterprise"))] - let template = &server.core.groupware.itip_template; - let formatter = TextFormatter::new(account_info.locale().as_str())?; - let locale = formatter.locale; - - let mut variables = Variables::new(); - let mut subject; - let (fields, old_fields) = match summary { - ItipSummary::Invite(fields) => { - subject = format!("{}: ", locale.calendar_invitation); - - (fields, None) - } - ItipSummary::Update { - current, previous, .. - } => { - subject = format!("{}: ", locale.calendar_updated_invitation); - variables.insert_single( - CalendarTemplateVariable::Header, - locale.calendar_event_updated.to_string(), - ); - variables.insert_single(CalendarTemplateVariable::Color, "info".to_string()); - (current, Some(previous)) - } - ItipSummary::Cancel(fields) => { - subject = format!("{}: ", locale.calendar_cancelled); - variables.insert_single( - CalendarTemplateVariable::Header, - locale.calendar_event_cancelled.to_string(), - ); - variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); - (fields, None) - } - ItipSummary::Rsvp { part_stat, current } => { - let (color, value) = match part_stat { - ICalendarParticipationStatus::Accepted => { - subject = format!("{}: ", locale.calendar_accepted); - - ( - "info", - locale.calendar_participant_accepted.replace("$name", from), - ) - } - ICalendarParticipationStatus::Declined => { - subject = format!("{}: ", locale.calendar_declined); - ( - "danger", - locale.calendar_participant_declined.replace("$name", from), - ) - } - ICalendarParticipationStatus::Tentative => { - subject = format!("{}: ", locale.calendar_tentative); - ( - "warning", - locale.calendar_participant_tentative.replace("$name", from), - ) - } - ICalendarParticipationStatus::Delegated => { - subject = format!("{}: ", locale.calendar_delegated); - ( - "warning", - locale.calendar_participant_delegated.replace("$name", from), - ) - } - _ => { - subject = format!("{}: ", locale.calendar_reply); - ( - "info", - locale.calendar_participant_reply.replace("$name", from), - ) - } - }; - - variables.insert_single(CalendarTemplateVariable::Header, value); - variables.insert_single(CalendarTemplateVariable::Color, color.to_string()); - - (current, None) - } - }; - - let mut when_detail: Option<(usize, &ItipValue)> = None; - let mut details: Vec> = Vec::with_capacity(4); - for field in [ - ICalendarProperty::Summary, - ICalendarProperty::Description, - ICalendarProperty::Dtstart, - ICalendarProperty::Rrule, - ICalendarProperty::Location, - ICalendarProperty::Conference, - ] { - let mut old_entries = old_fields.into_iter().flatten().filter(|e| e.name == field); - - for entry in fields.iter().filter(|e| e.name == field) { - let field_name = match &field { - ICalendarProperty::Summary => locale.calendar_summary, - ICalendarProperty::Description => locale.calendar_description, - ICalendarProperty::Dtstart | ICalendarProperty::Rrule => locale.calendar_when, - ICalendarProperty::Location => locale.calendar_location, - ICalendarProperty::Conference => locale.calendar_conference, - _ => continue, - }; - let value = formatter.field_to_string(&entry.value, DateStyle::Long); - - let old_entry = old_entries.next(); - - match &field { - ICalendarProperty::Summary => { - subject.push_str(&value); - } - ICalendarProperty::Dtstart => { - subject.push_str(" @ "); - subject.push_str(&value); - } - ICalendarProperty::Rrule if when_detail.is_none() => { - subject.push_str(" @ "); - subject.push_str(&value); - } - _ => (), - } - - if let ICalendarProperty::Summary | ICalendarProperty::Description = &field { - let variable = if matches!(field, ICalendarProperty::Summary) { - CalendarTemplateVariable::EventTitle - } else { - CalendarTemplateVariable::EventDescription - }; - - if old_entry.is_none() { - variables.insert_single(variable, value); - continue; - } - variables.insert_single(variable, value.clone()); - } - - if matches!(field, ICalendarProperty::Rrule) - && let Some((index, start_value)) = when_detail - && let Some(detail) = details.get_mut(index) - { - if let Some(when_value) = detail.get_mut(&CalendarTemplateVariable::Value) { - when_value.push_str(", "); - when_value.push_str(&value); - } - - if let Some(old_entry) = old_entry { - detail.insert( - CalendarTemplateVariable::Changed, - locale.calendar_changed.to_string(), - ); - let old_value = detail - .entry(CalendarTemplateVariable::OldValue) - .or_insert_with(|| { - formatter.field_to_string(start_value, DateStyle::Short) - }); - old_value.push_str(", "); - old_value - .push_str(&formatter.field_to_string(&old_entry.value, DateStyle::Short)); - } - - continue; - } - - let mut detail = AHashMap::with_capacity(4); - detail.insert(CalendarTemplateVariable::Key, field_name.to_string()); - if matches!(field, ICalendarProperty::Conference) - && let Some(link) = hyperlink(&value) - { - detail.insert(CalendarTemplateVariable::Link, link.to_string()); - } - detail.insert(CalendarTemplateVariable::Value, value); - if let Some(old_entry) = old_entry { - detail.insert( - CalendarTemplateVariable::Changed, - locale.calendar_changed.to_string(), - ); - detail.insert( - CalendarTemplateVariable::OldValue, - formatter.field_to_string(&old_entry.value, DateStyle::Short), - ); - } - if matches!(field, ICalendarProperty::Dtstart) && when_detail.is_none() { - when_detail = Some((details.len(), &entry.value)); - } - details.push(detail); - } - } - if !details.is_empty() { - variables.items.insert( - CalendarTemplateVariable::EventDetails, - Variable::Block(details), - ); - } - variables.insert_single(CalendarTemplateVariable::PageTitle, subject.clone()); - variables.insert_single(CalendarTemplateVariable::Lang, locale.name.to_string()); - variables.insert_single(CalendarTemplateVariable::Dir, locale.direction.to_string()); - variables.insert_single(CalendarTemplateVariable::LogoCid, format!("cid:{logo_cid}")); - - if let Some(guests) = fields - .iter() - .find(|e| e.name == ICalendarProperty::Attendee) - && let ItipValue::Participants(guests) = &guests.value - { - variables.insert_single( - CalendarTemplateVariable::AttendeesTitle, - locale.calendar_attendees.to_string(), - ); - variables.insert_block( - CalendarTemplateVariable::Attendees, - guests.iter().map(|guest| { - [ - ( - CalendarTemplateVariable::Key, - if guest.is_organizer { - if let Some(name) = guest.name.as_ref() { - format!("{name} - {}", locale.calendar_organizer) - } else { - locale.calendar_organizer.to_string() - } - } else { - guest.name.as_deref().unwrap_or_default().to_string() - }, - ), - (CalendarTemplateVariable::Value, guest.email.to_string()), - ] - }), - ); - } - - // Add RSVP buttons - if matches!(summary, ItipSummary::Invite(_) | ItipSummary::Update { .. }) - && let Some(rsvp_url) = server - .http_rsvp_url(account_id, account_info.name(), document_id, to) - .await - { - variables.insert_single( - CalendarTemplateVariable::Rsvp, - locale.calendar_reply_as.replace("$name", to), - ); - variables.insert_block( - CalendarTemplateVariable::Actions, - [ - ( - ICalendarParticipationStatus::Accepted, - locale.calendar_yes.to_string(), - "info", - ), - ( - ICalendarParticipationStatus::Declined, - locale.calendar_no.to_string(), - "danger", - ), - ( - ICalendarParticipationStatus::Tentative, - locale.calendar_maybe.to_string(), - "warning", - ), - ] - .into_iter() - .map(|(status, title, color)| { - [ - (CalendarTemplateVariable::ActionName, title.to_string()), - (CalendarTemplateVariable::ActionUrl, rsvp_url.url(&status)), - (CalendarTemplateVariable::Color, color.to_string()), - ] - }), - ); - } - - // Add footer - variables.insert_block( - CalendarTemplateVariable::Footer, - [ - [( - CalendarTemplateVariable::Key, - locale.calendar_imip_footer_1.to_string(), - )], - [( - CalendarTemplateVariable::Key, - locale.calendar_imip_footer_2.to_string(), - )], - ], - ); - - Ok(Details { - subject, - body: template.eval(&variables), - }) -} diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs deleted file mode 100644 index 49cdb4dc..00000000 --- a/crates/services/src/task_manager/index.rs +++ /dev/null @@ -1,594 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::task_manager::{Task, TaskDetails, TaskFailureType, TaskResult, deferred_retry_time}; -use common::Server; -use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata}; -use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; -use registry::{ - schema::{ - enums::IndexDocumentType, - prelude::{ObjectType, Property}, - structs::{TaskIndexDocument, TaskIndexTrace, TaskStatus}, - }, - types::EnumImpl, -}; -use std::cmp::Ordering; -use store::{ - IterateParams, ValueKey, - ahash::AHashMap, - rand::{self, RngExt}, - search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, - write::{ - AlignedBytes, Archive, BatchBuilder, SearchIndex, TelemetryClass, ValueClass, - key::DeserializeBigEndian, now, - }, -}; -use trc::{AddContext, TaskManagerEvent}; -use types::{ - blob_hash::BlobHash, - collection::{Collection, SyncCollection}, - field::EmailField, -}; - -pub(crate) trait SearchIndexTask: Sync + Send { - fn index(&self, tasks: &[TaskDetails]) -> impl Future> + Send; -} - -const NUM_INDEXES: usize = 5; -const MISSING_DOCUMENT_MAX_ATTEMPTS: u64 = 3; -const MISSING_DOCUMENT_RETRY_DELAY: u64 = 5; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TaskType { - Insert, - Delete, -} - -enum BuildResult { - Document(IndexDocument), - NotIndexed, - NotFound, -} - -#[derive(Debug)] -pub(crate) struct IndexTaskResult { - index: IndexDocumentType, - task_type: TaskType, - pub result: TaskResult, -} - -impl SearchIndexTask for Server { - async fn index(&self, tasks: &[TaskDetails]) -> Vec { - let mut results: Vec = Vec::with_capacity(tasks.len()); - let mut batch = BatchBuilder::new(); - let mut document_insertions = Vec::new(); - let mut document_deletions: [AHashMap>; NUM_INDEXES] = - std::array::from_fn(|_| AHashMap::new()); - - for task in tasks { - match &task.task { - Task::IndexDocument(task) => { - let account_id = task.account_id.document_id(); - let document_id = task.document_id.document_id(); - - let document = match task.document_type { - IndexDocumentType::Email => { - build_email_document(self, account_id, document_id).await - } - IndexDocumentType::Calendar => { - build_calendar_document(self, account_id, document_id).await - } - IndexDocumentType::Contacts => { - build_contact_document(self, account_id, document_id).await - } - IndexDocumentType::File => { - // File indexing not implemented yet - continue; - } - }; - - // Retry non found errors in case they are due to SQL read replication lag - let result = match document { - Ok(BuildResult::Document(doc)) if !doc.is_empty() => { - document_insertions.push(doc); - TaskResult::Success(vec![]) - } - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .ctx(trc::Key::Collection, task.document_type.as_str()) - .details("Failed to build document for indexing") - ); - result - } - Ok(BuildResult::NotFound) - if attempt_number(&task.status) < MISSING_DOCUMENT_MAX_ATTEMPTS => - { - TaskResult::Failure { - typ: TaskFailureType::Retry( - now().saturating_add(MISSING_DOCUMENT_RETRY_DELAY), - ), - message: "Document not found in data store".into(), - max_attempts: Some(MISSING_DOCUMENT_MAX_ATTEMPTS), - } - } - Ok(BuildResult::NotFound) => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Collection = task.document_type.as_str(), - Reason = "Document no longer exists", - AccountId = account_id, - DocumentId = document_id, - ); - TaskResult::Ignored - } - _ => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Collection = task.document_type.as_str(), - Reason = "Nothing to index", - AccountId = account_id, - DocumentId = document_id, - ); - TaskResult::Ignored - } - }; - - results.push(IndexTaskResult { - task_type: TaskType::Insert, - index: task.document_type, - result, - }); - } - Task::IndexTrace(task) => { - let result = match build_tracing_span_document(self, task.trace_id.id()).await { - Ok(Some(doc)) if !doc.is_empty() => { - document_insertions.push(doc); - TaskResult::Success(vec![]) - } - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!( - err.id(task.trace_id.id()) - .caused_by(trc::location!()) - .details("Failed to build document for indexing") - ); - result - } - _ => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Reason = "Nothing to index", - Id = task.trace_id.id(), - ); - TaskResult::Ignored - } - }; - - results.push(IndexTaskResult { - task_type: TaskType::Insert, - index: IndexDocumentType::File, // use File index for tracing spans to avoid creating a new index type - result, - }); - } - Task::UnindexDocument(task) => { - let account_id = task.account_id.document_id(); - let document_id = task.document_id.document_id(); - let idx = match task.document_type { - IndexDocumentType::Email => { - if let Err(err) = - delete_email_metadata(self, &mut batch, account_id, document_id) - .await - { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to delete email metadata from index") - ); - results.push(IndexTaskResult { - task_type: TaskType::Delete, - index: task.document_type, - result: TaskResult::temporary( - "Failed to delete email metadata from index", - ), - }); - continue; - } - 0 - } - IndexDocumentType::Calendar => 1, - IndexDocumentType::Contacts => 2, - IndexDocumentType::File => 3, - }; - - document_deletions[idx] - .entry(account_id) - .or_default() - .push(document_id); - - results.push(IndexTaskResult { - task_type: TaskType::Delete, - index: task.document_type, - result: TaskResult::Success(vec![]), - }); - } - _ => unreachable!(), - } - } - - // Commit deletion batch to data store - if !batch.is_empty() - && let Err(err) = self.store().write(batch.build_all()).await - { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to commit index deletions to data store") - ); - for r in results.iter_mut() { - if r.task_type == TaskType::Delete - && r.result.is_success() - && r.index == IndexDocumentType::Email - { - r.result = - TaskResult::temporary("Failed to commit index deletions to data store"); - } - } - return results; - } - - // Index documents - if !document_insertions.is_empty() - && let Err(err) = self.search_store().index(document_insertions).await - { - let retry_at = deferred_retry_time(&err); - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to index documents") - ); - for r in results.iter_mut() { - if r.task_type == TaskType::Insert && r.result.is_success() { - r.result = TaskResult::deferred(retry_at, "Failed to index documents"); - } - } - return results; - } - - // Delete documents - for (accounts, index) in document_deletions.into_iter().zip([ - SearchIndex::Email, - SearchIndex::Calendar, - SearchIndex::Contacts, - ]) { - let multi_account = match accounts.len().cmp(&1) { - Ordering::Greater => true, - Ordering::Equal => false, - Ordering::Less => continue, - }; - - let mut query = SearchQuery::new(index); - if multi_account { - query.add_filter(SearchFilter::Or); - } - - for (account_id, document_ids) in accounts { - let multi_document = document_ids.len() > 1; - query - .add_filter(SearchFilter::And) - .add_filter(SearchFilter::eq(SearchField::AccountId, account_id)); - - if multi_document { - query.add_filter(SearchFilter::Or); - } - - for document_id in document_ids { - query.add_filter(SearchFilter::eq(SearchField::DocumentId, document_id)); - } - - if multi_document { - query.add_filter(SearchFilter::End); - } - query.add_filter(SearchFilter::End); - } - - if multi_account { - query.add_filter(SearchFilter::End); - } - - if let Err(err) = self.search_store().unindex(query).await { - let retry_at = deferred_retry_time(&err); - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to delete documents from index") - .ctx(trc::Key::Collection, index.name()) - ); - for r in results.iter_mut() { - if r.task_type == TaskType::Delete && r.result.is_success() { - r.result = - TaskResult::deferred(retry_at, "Failed to delete documents from index"); - } - } - return results; - } - } - - results - } -} - -pub(crate) async fn reindex_telemetry(server: &Server) -> trc::Result<()> { - let mut spans = Vec::new(); - server - .tracing_store() - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(0))), - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(u64::MAX))), - ) - .no_values(), - |key, _| { - spans.push(key.deserialize_be_u64(0)?); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - let mut batch = BatchBuilder::new(); - let now = now() as i64; - for span_id in spans { - batch.schedule_task(Task::IndexTrace(TaskIndexTrace { - trace_id: span_id.into(), - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - if batch.is_large_batch() { - server.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - - if !batch.is_empty() { - server.core.storage.data.write(batch.build_all()).await?; - } - Ok(()) -} - -pub(crate) async fn reindex_account(server: &Server, account_id: u32) -> trc::Result<()> { - let now = now() as i64; - - let mut batch = BatchBuilder::new(); - - for document_id in server - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .emails - .items - .iter() - .map(|v| v.document_id) - { - batch.schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - - if batch.is_large_batch() { - server.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - - for document_type in [IndexDocumentType::Calendar, IndexDocumentType::Contacts] { - let cache = server - .fetch_dav_resources( - account_id, - account_id, - if document_type == IndexDocumentType::Calendar { - SyncCollection::Calendar - } else { - SyncCollection::AddressBook - }, - ) - .await - .caused_by(trc::location!())?; - - for document_id in cache.document_ids(false) { - batch.schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type, - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - - if batch.is_large_batch() { - server.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - } - - if !batch.is_empty() { - server.core.storage.data.write(batch.build_all()).await?; - } - - // Request indexing - server.notify_task_queue(); - - Ok(()) -} - -fn attempt_number(status: &TaskStatus) -> u64 { - match status { - TaskStatus::Pending(_) => 0, - TaskStatus::Retry(status) => status.attempt_number, - TaskStatus::Failed(status) => status.failed_attempt_number, - } -} - -async fn build_email_document( - server: &Server, - account_id: u32, - document_id: u32, -) -> trc::Result { - let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Email) else { - return Ok(BuildResult::NotIndexed); - }; - - match server - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await? - { - Some(metadata_) => { - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; - - let raw_message = server - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) - .await - .caused_by(trc::location!())? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .details("Blob not found") - })?; - - Ok(BuildResult::Document(metadata.index_document( - account_id, - document_id, - &raw_message, - index_fields, - server.core.email.default_language, - ))) - } - None => Ok(BuildResult::NotFound), - } -} - -async fn build_calendar_document( - server: &Server, - account_id: u32, - document_id: u32, -) -> trc::Result { - let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Calendar) else { - return Ok(BuildResult::NotIndexed); - }; - - match server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::CalendarEvent, - document_id, - )) - .await? - { - Some(metadata_) => Ok(BuildResult::Document( - metadata_ - .unarchive::() - .caused_by(trc::location!())? - .index_document( - account_id, - document_id, - index_fields, - server.core.email.default_language, - ), - )), - None => Ok(BuildResult::NotFound), - } -} - -async fn build_contact_document( - server: &Server, - account_id: u32, - document_id: u32, -) -> trc::Result { - let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Contacts) else { - return Ok(BuildResult::NotIndexed); - }; - - match server - .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::ContactCard, - document_id, - )) - .await? - { - Some(metadata_) => Ok(BuildResult::Document( - metadata_ - .unarchive::() - .caused_by(trc::location!())? - .index_document( - account_id, - document_id, - index_fields, - server.core.email.default_language, - ), - )), - None => Ok(BuildResult::NotFound), - } -} - - -#[cfg(not(feature = "enterprise"))] -async fn build_tracing_span_document(_: &Server, _: u64) -> trc::Result> { - Ok(None) -} - -async fn delete_email_metadata( - server: &Server, - batch: &mut BatchBuilder, - account_id: u32, - document_id: u32, -) -> trc::Result<()> { - match server - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await? - { - Some(metadata_) => { - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(document_id); - let metadata = metadata_ - .unarchive::() - .caused_by(trc::location!())?; - metadata.unindex(batch); - - } - None => { - trc::event!( - TaskManager(TaskManagerEvent::MetadataNotFound), - Details = "E-mail metadata not found", - AccountId = account_id, - DocumentId = document_id, - ); - } - } - - Ok(()) -} diff --git a/crates/services/src/task_manager/lock.rs b/crates/services/src/task_manager/lock.rs index cb11a2e6..41365274 100644 --- a/crates/services/src/task_manager/lock.rs +++ b/crates/services/src/task_manager/lock.rs @@ -4,49 +4,114 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::*; +use crate::task_manager::LEASE_DURATION; +use common::{KV_LOCK_TASK, Server}; +use std::future::Future; +use store::{InMemoryStore, rand}; +use trc::TaskManagerEvent; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TaskLock { + id: u64, + lease: u64, +} pub trait TaskLockManager: Sync + Send { - fn try_lock_task(&self, task: u64) -> impl Future + Send; - fn remove_index_lock(&self, id: u64) -> impl Future + Send; + fn lock_task(&self, id: u64) -> impl Future> + Send; + + fn renew_task_lock(&self, lock: TaskLock) -> impl Future + Send; + + fn unlock_task(&self, lock: TaskLock) -> impl Future + Send; +} + +impl TaskLock { + pub fn new(id: u64) -> Self { + TaskLock { + id, + lease: rand::random(), + } + } + + pub fn id(&self) -> u64 { + self.id + } + + pub async fn extend(self, store: &InMemoryStore) -> trc::Result { + let key = self.id.to_be_bytes(); + let renewed = store + .renew_lease(KV_LOCK_TASK, &key, self.lease, LEASE_DURATION) + .await; + if matches!(renewed, Ok(true)) { + return Ok(true); + } + + match store + .try_lease(KV_LOCK_TASK, &key, self.lease, LEASE_DURATION) + .await + { + Ok(acquired) => Ok(acquired), + Err(err) => renewed.and(Err(err)), + } + } } impl TaskLockManager for Server { - async fn try_lock_task(&self, id: u64) -> bool { + async fn lock_task(&self, id: u64) -> Option { + let lock = TaskLock::new(id); match self .in_memory_store() - .try_lock(KV_LOCK_TASK, &id.to_be_bytes(), DEFAULT_LOCK_EXPIRY) + .try_lease(KV_LOCK_TASK, &id.to_be_bytes(), lock.lease, LEASE_DURATION) .await { - Ok(result) => { - if !result { - trc::event!( - TaskManager(TaskManagerEvent::TaskLocked), - Id = id, - Details = "Task details not available", - ); - } - result + Ok(true) => Some(lock), + Ok(false) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskLocked), + TaskId = id, + Details = "Task is locked", + ); + None } Err(err) => { - trc::error!(err.id(id).details("Failed to lock task")); + trc::error!(err.ctx(trc::Key::TaskId, id).details("Failed to lock task")); + None + } + } + } + async fn renew_task_lock(&self, lock: TaskLock) -> bool { + match lock.extend(self.in_memory_store()).await { + Ok(true) => true, + Ok(false) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskLocked), + TaskId = lock.id, + Details = "Task lease was lost while the task was running", + ); + false + } + Err(err) => { + trc::error!( + err.ctx(trc::Key::TaskId, lock.id) + .details("Failed to renew task lease") + ); false } } } - async fn remove_index_lock(&self, id: u64) { + async fn unlock_task(&self, lock: TaskLock) { if let Err(err) = self .in_memory_store() - .remove_lock(KV_LOCK_TASK, &id.to_be_bytes()) + .release_lease(KV_LOCK_TASK, &lock.id.to_be_bytes(), lock.lease) .await { trc::error!( err.details("Failed to unlock task") - .ctx(trc::Key::Id, id) + .ctx(trc::Key::TaskId, lock.id) .caused_by(trc::location!()) ); } } } + diff --git a/crates/services/src/task_manager/manager.rs b/crates/services/src/task_manager/manager.rs index 180aeac5..c8323025 100644 --- a/crates/services/src/task_manager/manager.rs +++ b/crates/services/src/task_manager/manager.rs @@ -4,52 +4,40 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::acme::AcmeTask; -use crate::task_manager::alarm::SendAlarmTask; -use crate::task_manager::destroy_account::DestroyAccountTask; -use crate::task_manager::dkim::DkimManagementTask; -use crate::task_manager::dns::DnsManagementTask; -use crate::task_manager::imip::SendImipTask; -use crate::task_manager::index::SearchIndexTask; use crate::task_manager::lock::TaskLockManager; -use crate::task_manager::maintenance::MaintenanceTask; -use crate::task_manager::merge_threads::MergeThreadsTask; -use crate::task_manager::report::{self, SubmitReportTask}; -use crate::task_manager::restore_item::RestoreItemTask; -use crate::task_manager::spam_classifier::SpamFilterMaintenanceTask; +use crate::task_manager::worker::spawn_worker; use crate::task_manager::{ - DEFAULT_LOCK_EXPIRY, Locked, QUEUE_REFRESH_INTERVAL, TaskDetails, TaskFailureType, TaskInfo, - TaskJob, TaskManagerIpc, TaskResult, + DEFAULT_LOCK_EXPIRY, Dispatch, LockState, Locked, QUEUE_REFRESH_INTERVAL, TaskDone, TaskJob, + TaskManagerIpc, TaskTypeLimits, TaskTypeQueue, }; use common::BuildServer; +use common::config::network::ClusterRoles; use common::config::server::{DEFAULT_TLS_TIMEOUT, ServerProtocol}; use common::network::limiter::ConcurrencyLimiter; use common::network::{ServerInstance, TcpAcceptor}; use common::{Inner, Server}; use registry::schema::enums::TaskType; -use registry::schema::structs::{ - Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry, -}; -use registry::types::datetime::UTCDateTime; -use registry::types::{EnumImpl, ObjectImpl}; use std::collections::hash_map::Entry; use std::future::Future; use std::time::Duration; use std::{sync::Arc, time::Instant}; +use store::ahash::AHashMap; +use store::rand::RngExt; use store::rand::seq::SliceRandom; -use store::write::key::DeserializeBigEndian; use store::{ - IterateParams, ValueKey, - write::{BatchBuilder, TaskQueueClass, ValueClass, assert::AssertValue, now}, + IterateParams, ValueKey, rand, + write::{TaskDue, TaskId, TaskQueueClass, TaskTypes, ValueClass, now}, }; -use store::{SerializeInfallible, U64_LEN, rand}; use tokio::sync::{mpsc, watch}; use trc::TaskManagerEvent; use utils::snowflake::SnowflakeIdGenerator; -const TASK_QUEUE_BUFFER: usize = 10; -const PERPETUAL_RETRY_MIN_DELAY: u64 = 3600; -const PERPETUAL_RETRY_MAX_DELAY: u64 = 21600; +pub(crate) const TASK_QUEUE_BUFFER: usize = 10; +const CANDIDATE_OVERSCAN: usize = 4; +const REMOTE_LOCK_EXPIRY: u64 = QUEUE_REFRESH_INTERVAL; +const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100); +const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(QUEUE_REFRESH_INTERVAL / 2); +const SCAN_RETRY_INTERVAL: Duration = Duration::from_secs(1); pub fn spawn_task_manager(inner: Arc) { let is_clustered = { @@ -58,7 +46,6 @@ pub fn spawn_task_manager(inner: Arc) { if !roles.account_maintenance && !roles.store_maintenance - && !roles.search_indexing && !roles.spam_training && !roles.task_manager { @@ -83,606 +70,450 @@ pub fn spawn_task_manager(inner: Arc) { }); // Spawn workers for each task type - let mut txs = Vec::with_capacity(TaskType::COUNT); - for idx in 0..TaskType::COUNT { - let task_type = TaskType::from_id(idx as u16).unwrap(); - let channel_capacity = match task_type { - TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => { - std::cmp::max( - inner.build_server().core.email.index_batch_size, - TASK_QUEUE_BUFFER, - ) - } - TaskType::DestroyAccount - | TaskType::AccountMaintenance - | TaskType::TenantMaintenance - | TaskType::StoreMaintenance => 1, - TaskType::SpamFilterMaintenance => 2, - TaskType::CalendarAlarmEmail - | TaskType::CalendarAlarmNotification - | TaskType::CalendarItipMessage - | TaskType::MergeThreads - | TaskType::DmarcReport - | TaskType::TlsReport - | TaskType::RestoreArchivedItem - | TaskType::AcmeRenewal - | TaskType::DkimManagement - | TaskType::DnsManagement => TASK_QUEUE_BUFFER, - }; - - let (tx, mut rx) = mpsc::channel::(channel_capacity); - txs.push(tx); - let inner = inner.clone(); - let server_instance = server_instance.clone(); - - if matches!( - task_type, - TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace, - ) { - tokio::spawn(async move { - while let Some(job) = rx.recv().await { - let server = inner.build_server(); - let batch_size = server.core.email.index_batch_size; - let mut batch = Vec::with_capacity(batch_size); - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - batch.push(TaskDetails { task, info: job }); - } - Ok(None) => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - - while batch.len() < batch_size { - match rx.try_recv() { - Ok(job) => { - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - batch.push(TaskDetails { task, info: job }); - } - Ok(None) => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - } - Err(_) => break, - } - } - - // Dispatch - let mut refresh_queue = false; - let results = server.index(&batch).await.into_iter().map(|r| { - refresh_queue |= r.result.is_retry(); - r.result - }); - update_tasks(&server, &mut batch, results).await; - - if refresh_queue || rx.is_empty() { - server.notify_task_queue(); - } - } - }); - } else { - let server_instance = server_instance.clone(); - tokio::spawn(async move { - while let Some(job) = rx.recv().await { - let server = inner.build_server(); - let mut refresh_queue = false; - - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - let result = match &task { - Task::CalendarAlarmEmail(task) => { - server.send_email_alarm(task, server_instance.clone()).await - } - Task::CalendarAlarmNotification(task) => { - server.send_display_alarm(task).await - } - Task::CalendarItipMessage(task) => { - server.send_imip(task, server_instance.clone()).await - } - Task::MergeThreads(task) => server.merge_threads(task).await, - Task::DmarcReport(task) => { - server - .submit_report(report::ReportId::Dmarc(task.report_id.id())) - .await - } - Task::TlsReport(task) => { - server - .submit_report(report::ReportId::Tls(task.report_id.id())) - .await - } - Task::RestoreArchivedItem(task) => server.restore_item(task).await, - Task::DestroyAccount(task) => server.destroy_account(task).await, - Task::AccountMaintenance(task) => { - server.account_maintenance(task).await - } - Task::TenantMaintenance(task) => { - server.tenant_maintenance(task).await - } - Task::StoreMaintenance(task) => { - server.store_maintenance(task).await - } - Task::SpamFilterMaintenance(task) => { - Box::pin(server.spam_filter_maintenance(task)).await - } - Task::AcmeRenewal(task) => server.acme_management(task).await, - Task::DkimManagement(task_dkim_rotation) => { - server.dkim_management(task_dkim_rotation).await - } - Task::DnsManagement(task_dns_management) => { - server.dns_management(task_dns_management).await - } - Task::IndexDocument(_) - | Task::UnindexDocument(_) - | Task::IndexTrace(_) => unreachable!(), - }; - - refresh_queue = result.is_retry(); - - update_tasks( - &server, - &mut [TaskDetails { task, info: job }], - vec![result], - ) - .await; - } - Ok(None) => { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - - if refresh_queue || rx.is_empty() { - server.notify_task_queue(); - } - } - }); - } - } + let (done_tx, mut done_rx) = mpsc::unbounded_channel::(); + let started = Instant::now(); + let queues = TaskType::ALL + .iter() + .map(|&task_type| { + let (tx, rx) = mpsc::channel::(task_type.channel_capacity()); + spawn_worker( + inner.clone(), + task_type, + rx, + done_tx.clone(), + server_instance.clone(), + ); + TaskTypeQueue::new(task_type, tx, started) + }) + .collect::>(); const REFRESH_INTERVAL: Duration = Duration::from_secs(60); tokio::spawn(async move { let mut ipc = TaskManagerIpc { - txs: txs.try_into().expect("Incorrect number of task channels"), - locked: Default::default(), + queues, + candidates: Vec::with_capacity(TASK_QUEUE_BUFFER * CANDIDATE_OVERSCAN), revision: 0, }; - let rx = inner.ipc.task_tx.clone(); loop { - // Index any queued tasks - let mut sleep_for = inner.build_server().process_tasks(&mut ipc).await; + // Release the locks held by tasks that have completed, unless the event + // was already dispatched again + while let Ok(done) = done_rx.try_recv() { + ipc.task_done(done); + } + + let flagged = inner.ipc.task_tx.take(); + let mut sleep_for = inner.build_server().process_tasks(&mut ipc, flagged).await; if is_clustered && sleep_for > REFRESH_INTERVAL { sleep_for = REFRESH_INTERVAL; } - // Wait for a signal or sleep until the next task is due - let _ = tokio::time::timeout(sleep_for, rx.notified()).await; + let _ = tokio::time::timeout(sleep_for, inner.ipc.task_tx.notified()).await; } }); } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ScanEnd { + Future(u64), + Exhausted, + Truncated(u64), + Failed, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Dispatched { + sent: usize, + released: bool, +} + +impl TaskManagerIpc { + fn task_done(&mut self, done: TaskDone) { + if let Some(queue) = self.queues.iter_mut().find(|queue| queue.typ == done.typ) + && let Entry::Occupied(entry) = queue.locked.entry(done.id) + && let LockState::InFlight { + dispatch_revision, + has_pending_due, + } = entry.get().state + && dispatch_revision == done.dispatch_revision + { + entry.remove(); + + // The queue entry could not be updated, make sure it is visited again + if !done.is_committed || has_pending_due { + queue.is_stale = true; + } + } + } +} + +impl TaskTypeQueue { + fn new(typ: TaskType, tx: mpsc::Sender, now: Instant) -> Self { + TaskTypeQueue { + typ, + tx, + locked: AHashMap::new(), + next_due: u64::MAX, + has_more: false, + is_stale: true, + has_sent: false, + scanned_at: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now), + wake_at: now, + } + } + + fn has_due_work(&self, now_timestamp: u64) -> bool { + self.has_more || self.next_due <= now_timestamp + } + + fn should_scan(&self, now: Instant, now_timestamp: u64) -> bool { + let free = self.tx.capacity(); + free > 0 + && !self.tx.is_closed() + && (self.wake_at <= now + || ((self.is_stale || self.has_more) + && (self.scanned_at + MIN_SCAN_INTERVAL <= now + || (self.has_sent + && free == self.tx.max_capacity() + && self.has_due_work(now_timestamp))))) + } + + fn next_wake(&self, now: Instant, now_timestamp: u64) -> Instant { + if self.tx.capacity() == 0 || self.tx.is_closed() { + if self.wake_at > now { + self.wake_at + } else { + now + Duration::from_secs(QUEUE_REFRESH_INTERVAL) + } + } else if self.should_scan(now, now_timestamp) { + now + } else if self.is_stale || self.has_more { + self.scanned_at + MIN_SCAN_INTERVAL + } else { + self.wake_at + } + } + + fn finish_scan( + &mut self, + scan: ScanEnd, + dispatched: Dispatched, + revision: u64, + now: Instant, + now_timestamp: u64, + ) { + let released = dispatched.released; + self.scanned_at = now; + self.is_stale = false; + self.has_sent = dispatched.sent > 0; + let mut wake_at = now + FULL_SCAN_INTERVAL; + + let evict_below = match scan { + ScanEnd::Future(due) => { + self.next_due = due; + self.has_more = released; + wake_at = wake_at.min(now + Duration::from_secs(due.saturating_sub(now_timestamp))); + u64::MAX + } + ScanEnd::Exhausted => { + self.next_due = u64::MAX; + self.has_more = released; + u64::MAX + } + ScanEnd::Truncated(last_due) => { + self.next_due = u64::MAX; + self.has_more = true; + last_due + } + ScanEnd::Failed => { + self.next_due = u64::MAX; + self.has_more = false; + wake_at = now + SCAN_RETRY_INTERVAL; + 0 + } + }; + + self.locked.retain(|_, locked| { + let keep = + locked.expires > now && (locked.revision == revision || locked.due >= evict_below); + if keep && locked.expires < wake_at { + wake_at = locked.expires; + } + keep + }); + self.wake_at = wake_at; + } +} + pub(crate) trait TaskQueueManager: Sync + Send { - fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> impl Future + Send; + fn process_tasks( + &self, + ipc: &mut TaskManagerIpc, + flagged: TaskTypes, + ) -> impl Future + Send; + + fn scan_tasks( + &self, + queue: &mut TaskTypeQueue, + candidates: &mut Vec, + revision: u64, + now: Instant, + now_timestamp: u64, + ) -> impl Future + Send; + + fn dispatch_tasks( + &self, + queue: &mut TaskTypeQueue, + candidates: &mut Vec, + revision: u64, + now: Instant, + ) -> impl Future + Send; } impl TaskQueueManager for Server { - async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration { - let now_timestamp = now(); - let from_key = ValueKey:: { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 1 }), - }; - let to_key = ValueKey:: { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::TaskQueue(TaskQueueClass::Due { - id: u64::MAX, - due: now_timestamp + QUEUE_REFRESH_INTERVAL, - }), - }; - - // Retrieve tasks pending to be processed - let mut tasks = Vec::new(); - let now = Instant::now(); - let mut next_event = None; + async fn process_tasks(&self, ipc: &mut TaskManagerIpc, flagged: TaskTypes) -> Duration { let roles = &self.core.network.roles; - ipc.revision += 1; - let _ = self + let now_timestamp = now(); + let now_instant = Instant::now(); + let mut wake_at = now_instant + Duration::from_secs(QUEUE_REFRESH_INTERVAL); + let TaskManagerIpc { + queues, + candidates, + revision, + } = ipc; + + for queue in queues + .iter_mut() + .filter(|queue| is_enabled(queue.typ, roles)) + { + if flagged.contains(queue.typ) { + queue.is_stale = true; + } + + if queue.should_scan(now_instant, now_timestamp) { + *revision += 1; + let scan = self + .scan_tasks(queue, candidates, *revision, now_instant, now_timestamp) + .await; + let dispatched = self + .dispatch_tasks(queue, candidates, *revision, now_instant) + .await; + queue.finish_scan(scan, dispatched, *revision, now_instant, now_timestamp); + } + + wake_at = wake_at.min(queue.next_wake(now_instant, now_timestamp)); + } + + wake_at.saturating_duration_since(Instant::now()) + } + + async fn scan_tasks( + &self, + queue: &mut TaskTypeQueue, + candidates: &mut Vec, + revision: u64, + now: Instant, + now_timestamp: u64, + ) -> ScanEnd { + let want = queue.tx.capacity() * CANDIDATE_OVERSCAN; + let limit = queue.locked.len() + want + 1; + let typ = queue.typ; + let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(0), + typ, + due: 0, + })); + let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(u64::MAX), + typ, + due: now_timestamp + QUEUE_REFRESH_INTERVAL, + })); + + let locked = &mut queue.locked; + let mut rows = 0; + let mut last_due = 0; + let mut end = ScanEnd::Exhausted; + candidates.clear(); + let result = self .store() .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - if key.len() == U64_LEN * 2 { - let task_due = key.deserialize_be_u64(0)?; - let task_id = key.deserialize_be_u64(U64_LEN)?; + IterateParams::new(from_key, to_key) + .ascending() + .no_values() + .limit(limit), + |key, _| { + let TaskDue { due, id, .. } = TaskDue::deserialize(key)?; + rows += 1; - if task_due <= now_timestamp { - let task_type_idx = value.deserialize_be_u16(0)?; - let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, value) - })?; - let enabled = match task_type { - TaskType::IndexDocument - | TaskType::UnindexDocument - | TaskType::IndexTrace => roles.search_indexing, - TaskType::AccountMaintenance - | TaskType::TenantMaintenance - | TaskType::DestroyAccount => roles.account_maintenance, - TaskType::StoreMaintenance => roles.store_maintenance, - TaskType::SpamFilterMaintenance => roles.spam_training, - TaskType::CalendarAlarmEmail - | TaskType::CalendarAlarmNotification - | TaskType::CalendarItipMessage - | TaskType::MergeThreads - | TaskType::DmarcReport - | TaskType::TlsReport - | TaskType::RestoreArchivedItem - | TaskType::AcmeRenewal - | TaskType::DkimManagement - | TaskType::DnsManagement => true, - }; + if due > now_timestamp { + end = ScanEnd::Future(due); + return Ok(false); + } + last_due = due; - if !enabled { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Id = task_id, - Details = task_type.as_str(), - Reason = "Task type is disabled by cluster roles.", - ); - return Ok(true); - } - - match ipc.locked.entry(task_id) { - Entry::Occupied(mut entry) => { - let locked = entry.get_mut(); - if locked.expires <= now || locked.due < task_due { - locked.expires = Instant::now() - + std::time::Duration::from_secs( - DEFAULT_LOCK_EXPIRY + 1, - ); - locked.due = task_due; - tasks.push(( - TaskJob { - id: task_id, - due: task_due, - typ: task_type, - }, - task_type_idx, - )); - } - locked.revision = ipc.revision; + // Refreshed even when not dispatched, or a stale revision would evict it + if let Some(entry) = locked.get_mut(&id) { + entry.revision = revision; + if entry.expires > now { + match &mut entry.state { + LockState::InFlight { + has_pending_due, .. + } => { + *has_pending_due |= entry.due != due; + return Ok(true); } - Entry::Vacant(entry) => { - entry.insert(Locked { - expires: Instant::now() - + std::time::Duration::from_secs( - DEFAULT_LOCK_EXPIRY + 1, - ), - due: task_due, - revision: ipc.revision, - }); - tasks.push(( - TaskJob { - id: task_id, - due: task_due, - typ: task_type, - }, - task_type_idx, - )); + LockState::Parked if entry.due >= due => { + entry.due = due; + return Ok(true); + } + LockState::Parked => { + entry.due = due; } } - - Ok(true) - } else { - next_event = Some(task_due); - Ok(false) } - } else { + } + + candidates.push(TaskJob { + id, + due, + typ, + dispatch_revision: revision, + }); + if candidates.len() < want { Ok(true) + } else { + end = ScanEnd::Truncated(due); + Ok(false) } }, ) - .await - .map_err(|err| { + .await; + + match result { + Ok(()) if end == ScanEnd::Exhausted && rows >= limit => ScanEnd::Truncated(last_due), + Ok(()) => end, + Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to iterate over task queue.") ); - }); + ScanEnd::Failed + } + } + } - if !tasks.is_empty() { - trc::event!( - TaskManager(TaskManagerEvent::TaskAcquired), - Total = tasks.len(), - Details = ipc.locked.len(), - ); + async fn dispatch_tasks( + &self, + queue: &mut TaskTypeQueue, + candidates: &mut Vec, + revision: u64, + now: Instant, + ) -> Dispatched { + let mut dispatched = Dispatched::default(); + if candidates.is_empty() { + return dispatched; } - // Shuffle tasks - if tasks.len() > 1 { - tasks.shuffle(&mut rand::rng()); + trc::event!( + TaskManager(TaskManagerEvent::TaskAcquired), + Total = candidates.len(), + Used = queue.locked.len(), + ); + + if candidates.len() > 1 { + candidates.shuffle(&mut rand::rng()); } - // Dispatch tasks - for (task_job, task_type_idx) in tasks { - let tx = &ipc.txs[task_type_idx as usize]; + let lock_expires = now + Duration::from_secs(DEFAULT_LOCK_EXPIRY + 1); + for job in candidates.drain(..) { + if queue.tx.capacity() == 0 { + dispatched.released = true; + break; + } - if tx.capacity() > 0 { - if self.try_lock_task(task_job.id).await && tx.send(task_job).await.is_err() { + let (id, due) = (job.id, job.due); + if let Some(Locked { + expires, + state: + LockState::InFlight { + has_pending_due, .. + }, + .. + }) = queue.locked.get_mut(&id) + && *expires > now + { + *has_pending_due = true; + } else if let Some(lock) = self.lock_task(id).await { + if queue + .tx + .send(Dispatch { + job, + lock, + dispatched_at: Instant::now(), + }) + .await + .is_ok() + { + dispatched.sent += 1; + queue.locked.insert( + id, + Locked { + expires: lock_expires, + due, + revision, + state: LockState::InFlight { + dispatch_revision: revision, + has_pending_due: false, + }, + }, + ); + } else { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending task.", CausedBy = trc::location!() ); + self.unlock_task(lock).await; } } else { - // If the channel is full, release the lock so it can be picked up in the next iteration - ipc.locked.remove(&task_job.id); + // Another node owns the task, retry shortly after its lock expires + queue.locked.insert( + id, + Locked { + expires: now + + Duration::from_secs( + REMOTE_LOCK_EXPIRY + rand::rng().random_range(0..=60), + ), + due, + revision, + state: LockState::Parked, + }, + ); } } - // Delete expired locks - let now = Instant::now(); - ipc.locked - .retain(|_, locked| locked.expires > now && locked.revision == ipc.revision); - Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| { - timestamp.saturating_sub(store::write::now()) - })) + dispatched } } -async fn update_tasks( - server: &Server, - tasks: &mut [TaskDetails], - results: impl IntoIterator, -) { - let mut batch = BatchBuilder::new(); - - for (task, result) in tasks.iter_mut().zip(results) { - let id = task.info.id; - batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id, - due: task.info.due, - })); - match result { - TaskResult::Success(tasks) => { - for task in tasks { - batch.schedule_task(task); - } - batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); - } - TaskResult::Ignored => { - batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); - } - TaskResult::Update(ops) => { - for op in ops { - batch.any_op(op); - } - } - TaskResult::Failure { - typ, - message, - max_attempts, - } => { - let (attempt_number, retry_since) = match task.task.status() { - TaskStatus::Pending(_) => (0, UTCDateTime::now()), - TaskStatus::Retry(status) => (status.attempt_number, status.created_at), - TaskStatus::Failed(status) => (status.failed_attempt_number, status.failed_at), - }; - let retry_at = match typ { - TaskFailureType::Retry(retry_at) => (attempt_number - < max_attempts.unwrap_or(server.core.network.task_manager.max_attempts) - && retry_at - <= (retry_since.timestamp() as u64).saturating_add( - server.core.network.task_manager.total_deadline.as_secs(), - )) - .then_some(retry_at) - .or_else(|| perpetual_retry_time(task.info.typ, attempt_number)), - TaskFailureType::Temporary => next_retry_time( - &server.core.network.task_manager, - max_attempts, - retry_since.timestamp() as u64, - attempt_number, - now(), - ) - .or_else(|| perpetual_retry_time(task.info.typ, attempt_number)), - TaskFailureType::Perpetual => { - perpetual_retry_time(task.info.typ, attempt_number) - } - TaskFailureType::Permanent => None, - }; - - let due = if let Some(retry_at) = retry_at { - trc::event!( - TaskManager(TaskManagerEvent::TaskRetry), - Id = id, - Details = task.task.name(), - Reason = message.to_string(), - NextRetry = trc::Value::Timestamp(retry_at), - ); - - task.task.set_status(TaskStatus::Retry(TaskStatusRetry { - due: UTCDateTime::from_timestamp(retry_at as i64), - attempt_number: attempt_number + 1, - failure_reason: message, - created_at: retry_since, - })); - - retry_at - } else { - trc::event!( - TaskManager(TaskManagerEvent::TaskFailed), - Id = id, - Details = task.task.name(), - Reason = message.to_string(), - ); - - task.task.set_status(TaskStatus::Failed(TaskStatusFailed { - failed_at: UTCDateTime::now(), - failed_attempt_number: attempt_number, - failure_reason: message, - created_at: retry_since, - })); - u64::MAX - }; - batch - .assert_value( - ValueClass::TaskQueue(TaskQueueClass::Task { id }), - AssertValue::Some, - ) - .set( - ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), - task.info.typ.to_id().serialize(), - ) - .set( - ValueClass::TaskQueue(TaskQueueClass::Task { id }), - task.task.to_pickled_vec(), - ); - } +#[inline(always)] +fn is_enabled(task_type: TaskType, roles: &ClusterRoles) -> bool { + match task_type { + TaskType::AccountMaintenance | TaskType::DestroyAccount | TaskType::ShareRevokeCleanup => { + roles.account_maintenance } - } - - if let Err(err) = server.store().write(batch.build_all()).await { - if err.matches(trc::EventType::Store(trc::StoreEvent::AssertValueFailed)) { - trc::event!( - TaskManager(TaskManagerEvent::TaskIgnored), - Reason = "Task was deleted while being processed; skipping update.", - ); - } else { - trc::error!(err.details("Failed to remove task(s) from queue.")); - } - } - - for task in tasks { - server.remove_index_lock(task.info.id).await; + TaskType::StoreMaintenance => roles.store_maintenance, + TaskType::SpamFilterMaintenance => roles.spam_training, + TaskType::CalendarAlarmEmail + | TaskType::CalendarAlarmNotification + | TaskType::CalendarItipMessage + | TaskType::MergeThreads + | TaskType::DmarcReport + | TaskType::TlsReport + | TaskType::DmarcReportWindow + | TaskType::TlsReportWindow + | TaskType::AcmeRenewal + | TaskType::DkimManagement + | TaskType::DnsManagement => true, } } -pub fn perpetual_retry_time(typ: TaskType, attempt: u64) -> Option { - matches!( - typ, - TaskType::AcmeRenewal - | TaskType::DkimManagement - | TaskType::IndexDocument - | TaskType::UnindexDocument - | TaskType::DestroyAccount - ) - .then(|| { - now().saturating_add( - PERPETUAL_RETRY_MIN_DELAY - .saturating_mul(1u64 << attempt.min(4)) - .min(PERPETUAL_RETRY_MAX_DELAY), - ) - }) -} - -pub fn next_retry_time( - manager: &TaskManager, - max_attempts_override: Option, - retry_since: u64, - attempt: u64, - now: u64, -) -> Option { - if attempt >= max_attempts_override.unwrap_or(manager.max_attempts) { - return None; - } - - let delay_secs: u64 = match &manager.strategy { - TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(), - TaskRetryStrategy::ExponentialBackoff(backoff) => { - let delay = (backoff.initial_delay.as_secs() as f64 - * backoff.factor.into_inner().powi(attempt as i32)) - .min(backoff.max_delay.as_secs() as f64) as u64; - - if backoff.jitter { - let jitter_factor = rand::random::() + 0.5; - ((delay as f64 * jitter_factor) as u64).min(backoff.max_delay.as_secs()) - } else { - delay - } - } - }; - - let next_time = now.saturating_add(delay_secs); - let deadline = retry_since.saturating_add(manager.total_deadline.as_secs()); - if next_time > deadline { - return None; - } - - Some(next_time) -} - -impl TaskResult { - pub fn is_success(&self) -> bool { - matches!(self, TaskResult::Success(_)) - } - - pub fn is_retry(&self) -> bool { - matches!( - self, - TaskResult::Update(_) - | TaskResult::Failure { - typ: TaskFailureType::Temporary - | TaskFailureType::Retry(_) - | TaskFailureType::Perpetual, - .. - } - ) - } -} diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index c598ada3..72f8db57 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -4,81 +4,101 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{KV_LOCK_TASK, Server}; use registry::schema::enums::TaskType; use registry::schema::structs::Task; use registry::types::EnumImpl; -use std::future::Future; -use std::time::Instant; +use std::time::{Duration, Instant}; +use store::Deserialize; use store::ahash::AHashMap; -use store::write::Operation; +use store::xxhash_rust::xxh3::xxh3_64; use tokio::sync::mpsc; -use trc::TaskManagerEvent; -pub mod acme; -pub mod alarm; -pub mod destroy_account; -pub mod dkim; -pub mod dns; -pub mod imip; -pub mod index; +use lock::TaskLock; + pub mod lock; -pub mod maintenance; pub mod manager; -pub mod merge_threads; -pub mod report; -pub mod restore_item; +pub mod outcome; +pub mod reconcile; pub mod scheduler; -pub mod spam_classifier; +pub mod tasks; +mod worker; -const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes -const DEFAULT_LOCK_EXPIRY: u64 = 60 * 60; // 1 hour +const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; +const DEFAULT_LOCK_EXPIRY: u64 = 60 * 60; +const LEASE_DURATION: u64 = 60 * 5; +const LEASE_RENEW_INTERVAL: Duration = Duration::from_secs(LEASE_DURATION / 3); pub(crate) struct TaskManagerIpc { - txs: [mpsc::Sender; TaskType::COUNT], - locked: AHashMap, + queues: Box<[TaskTypeQueue]>, + candidates: Vec, revision: u64, } +#[derive(Debug)] +pub(crate) struct TaskTypeQueue { + typ: TaskType, + tx: mpsc::Sender, + locked: AHashMap, + next_due: u64, + has_more: bool, + is_stale: bool, + has_sent: bool, + scanned_at: Instant, + wake_at: Instant, +} + #[derive(Debug)] pub(crate) struct Locked { expires: Instant, due: u64, revision: u64, + state: LockState, } -#[derive(Debug)] -pub(crate) struct TaskDetails { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LockState { + InFlight { + dispatch_revision: u64, + has_pending_due: bool, + }, + Parked, +} + +pub(crate) struct StoredTask { task: Task, - info: TaskJob, + hash: u64, } -#[derive(Debug)] +impl Deserialize for StoredTask { + fn deserialize(bytes: &[u8]) -> trc::Result { + Task::deserialize(bytes).map(|task| StoredTask { + task, + hash: xxh3_64(bytes), + }) + } +} + +#[derive(Debug, Clone, Copy)] pub(crate) struct TaskJob { id: u64, due: u64, typ: TaskType, + dispatch_revision: u64, } -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum TaskResult { - Success(Vec), - Update([Operation; 2]), - Failure { - typ: TaskFailureType, - message: String, - max_attempts: Option, - }, - Ignored, +#[derive(Debug, Clone, Copy)] +pub(crate) struct Dispatch { + job: TaskJob, + lock: TaskLock, + dispatched_at: Instant, } -#[derive(Debug, Clone, PartialEq, Eq)] -#[allow(dead_code)] -pub(crate) enum TaskFailureType { - Retry(u64), - Temporary, - Perpetual, - Permanent, +#[derive(Debug)] +pub(crate) struct TaskDone { + id: u64, + typ: TaskType, + dispatch_revision: u64, + is_committed: bool, } pub(crate) trait TaskInfo { @@ -87,67 +107,67 @@ pub(crate) trait TaskInfo { impl TaskInfo for Task { fn name(&self) -> &'static str { + self.object_type().as_str() + } +} + +impl TaskJob { + fn done(&self, is_committed: bool) -> TaskDone { + TaskDone { + id: self.id, + typ: self.typ, + dispatch_revision: self.dispatch_revision, + is_committed, + } + } +} + +pub(crate) trait TaskTypeLimits { + fn channel_capacity(self) -> usize; + + fn concurrency(self) -> usize; +} + +impl TaskTypeLimits for TaskType { + fn channel_capacity(self) -> usize { match self { - Task::IndexDocument(_) => "IndexDocument", - Task::UnindexDocument(_) => "UnindexDocument", - Task::IndexTrace(_) => "IndexTrace", - Task::CalendarAlarmEmail(_) => "CalendarAlarmEmail", - Task::CalendarAlarmNotification(_) => "CalendarAlarmNotification", - Task::CalendarItipMessage(_) => "CalendarItipMessage", - Task::MergeThreads(_) => "MergeThreads", - Task::DmarcReport(_) => "DmarcReport", - Task::TlsReport(_) => "TlsReport", - Task::RestoreArchivedItem(_) => "RestoreArchivedItem", - Task::DestroyAccount(_) => "DestroyAccount", - Task::AccountMaintenance(_) => "AccountMaintenance", - Task::StoreMaintenance(_) => "StoreMaintenance", - Task::SpamFilterMaintenance(_) => "SpamFilterMaintenance", - Task::AcmeRenewal(_) => "AcmeRenewal", - Task::DkimManagement(_) => "DkimManagement", - Task::DnsManagement(_) => "DnsManagement", - Task::TenantMaintenance(_) => "TenantMaintenance", + TaskType::DestroyAccount + | TaskType::AccountMaintenance + | TaskType::StoreMaintenance => 1, + TaskType::SpamFilterMaintenance => 2, + TaskType::CalendarAlarmEmail + | TaskType::CalendarAlarmNotification + | TaskType::CalendarItipMessage + | TaskType::MergeThreads + | TaskType::DmarcReport + | TaskType::TlsReport + | TaskType::DmarcReportWindow + | TaskType::TlsReportWindow + | TaskType::AcmeRenewal + | TaskType::DkimManagement + | TaskType::DnsManagement + | TaskType::ShareRevokeCleanup => manager::TASK_QUEUE_BUFFER, + } + } + + fn concurrency(self) -> usize { + match self { + TaskType::DestroyAccount + | TaskType::AccountMaintenance + | TaskType::StoreMaintenance + | TaskType::SpamFilterMaintenance => 1, + TaskType::MergeThreads + | TaskType::DmarcReport + | TaskType::TlsReport + | TaskType::DmarcReportWindow + | TaskType::TlsReportWindow + | TaskType::AcmeRenewal + | TaskType::DkimManagement + | TaskType::DnsManagement + | TaskType::ShareRevokeCleanup => 2, + TaskType::CalendarAlarmEmail + | TaskType::CalendarAlarmNotification + | TaskType::CalendarItipMessage => 4, } } } - -impl TaskResult { - pub fn permanent(message: impl Into) -> Self { - TaskResult::Failure { - typ: TaskFailureType::Permanent, - message: message.into(), - max_attempts: None, - } - } - - pub fn temporary(message: impl Into) -> Self { - TaskResult::Failure { - typ: TaskFailureType::Temporary, - message: message.into(), - max_attempts: None, - } - } - - pub fn perpetual(message: impl Into) -> Self { - TaskResult::Failure { - typ: TaskFailureType::Perpetual, - message: message.into(), - max_attempts: None, - } - } - - pub fn deferred(retry_at: Option, message: impl Into) -> Self { - match retry_at { - Some(retry_at) => TaskResult::Failure { - typ: TaskFailureType::Retry(retry_at), - message: message.into(), - max_attempts: None, - }, - None => TaskResult::temporary(message), - } - } -} - -pub(crate) fn deferred_retry_time(err: &trc::Error) -> Option { - err.value(trc::Key::NextRetry) - .and_then(|value| value.to_uint()) -} diff --git a/crates/services/src/task_manager/outcome.rs b/crates/services/src/task_manager/outcome.rs new file mode 100644 index 00000000..1306c0a9 --- /dev/null +++ b/crates/services/src/task_manager/outcome.rs @@ -0,0 +1,382 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::{TaskInfo, TaskJob}; +use common::Server; +use compact_str::CompactString; +use registry::{ + schema::structs::{ + Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry, + }, + types::{ObjectImpl, datetime::UTCDateTime}, +}; +use std::ops::RangeInclusive; +use store::{ + rand::{self, RngExt}, + write::{ + BatchBuilder, Operation, TaskId, TaskQueueClass, TaskTypes, ValueClass, + assert::AssertValue, now, + }, +}; +use trc::{EventType, Key, SecurityEvent, StoreEvent, TaskManagerEvent}; + +const MAX_REASON_LEN: usize = 1024; +const CONTENTION_DELAY: RangeInclusive = 1..=10; +const CONTENTION_FAST_RETRIES: u64 = 3; +const ITIP_EXPIRY: u64 = 3 * 86400; +const FDB_TRANSACTION_TOO_OLD: u64 = 1007; +const FDB_NOT_COMMITTED: u64 = 1020; +const FDB_PAYLOAD_TOO_LARGE: RangeInclusive = 2101..=2103; + +#[derive(Debug)] +pub(crate) enum Outcome { + Done(Vec), + Replace([Operation; 2]), + Partial { + remaining: Box, + failure: Failure, + }, + Retry(Failure), + Fatal(String), + Obsolete(&'static str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RetryClass { + Transient, + Throttled { retry_at: u64 }, + Contention, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ErrorClass { + Retry(RetryClass), + Fatal, +} + +#[derive(Debug)] +pub(crate) struct Failure { + class: RetryClass, + reason: String, +} + +#[derive(Debug)] +pub(crate) enum Commit { + Committed(TaskTypes), + Conflict, + Failed, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum NextAttempt { + Retry { + retry: TaskStatusRetry, + is_stalled: bool, + }, + Expired(String), +} + +impl Outcome { + pub(crate) fn done() -> Self { + Outcome::Done(Vec::new()) + } + + pub(crate) fn transient(reason: impl Into) -> Self { + Outcome::Retry(Failure::new(RetryClass::Transient, reason)) + } + + pub(crate) fn fatal(reason: impl Into) -> Self { + Outcome::Fatal(reason.into()) + } + + pub(crate) fn error(err: &trc::Error) -> Self { + match ErrorClass::of(err) { + ErrorClass::Retry(class) => Outcome::Retry(Failure::new(class, err.to_string())), + ErrorClass::Fatal => Outcome::Fatal(err.to_string()), + } + } +} + +impl Failure { + pub(crate) fn new(class: RetryClass, reason: impl Into) -> Self { + Failure { + class, + reason: reason.into(), + } + } +} + +impl ErrorClass { + pub(crate) fn of(err: &trc::Error) -> Self { + if let Some(retry_at) = err.value(Key::NextRetry).and_then(|value| match value { + trc::Value::Timestamp(retry_at) => Some(*retry_at), + value => value.to_uint(), + }) { + return ErrorClass::Retry(RetryClass::Throttled { retry_at }); + } + + match err.event_type() { + EventType::Store(StoreEvent::AssertValueFailed) => { + ErrorClass::Retry(RetryClass::Contention) + } + EventType::Store(StoreEvent::FoundationdbError) => { + match err.value(Key::Code).and_then(|value| value.to_uint()) { + Some(FDB_TRANSACTION_TOO_OLD | FDB_NOT_COMMITTED) => { + ErrorClass::Retry(RetryClass::Contention) + } + Some(code) if FDB_PAYLOAD_TOO_LARGE.contains(&code) => ErrorClass::Fatal, + _ => ErrorClass::Retry(RetryClass::Transient), + } + } + EventType::Store( + StoreEvent::DataCorruption + | StoreEvent::DecompressError + | StoreEvent::DeserializeError + | StoreEvent::NotSupported + | StoreEvent::NotConfigured + | StoreEvent::CryptoError, + ) + | EventType::Security(SecurityEvent::Unauthorized) => ErrorClass::Fatal, + _ => ErrorClass::Retry(RetryClass::Transient), + } + } +} + +impl NextAttempt { + pub(crate) fn plan( + manager: &TaskManager, + task: &Task, + class: RetryClass, + reason: String, + now: u64, + ) -> Self { + let (created_at, first_failed_at, failures) = match task.status() { + TaskStatus::Pending(status) => (status.created_at, None, 0), + TaskStatus::Retry(status) | TaskStatus::Stalled(status) => ( + status.created_at, + Some(status.first_failed_at), + status.attempt_number, + ), + TaskStatus::Failed(status) => (status.created_at, None, 0), + }; + let attempt = failures.saturating_add(1); + let delay = match class { + RetryClass::Transient => retry_delay(manager, attempt), + RetryClass::Throttled { retry_at } => retry_at + .saturating_sub(now) + .max(retry_delay(manager, attempt)), + RetryClass::Contention if attempt <= CONTENTION_FAST_RETRIES => { + rand::rng().random_range(CONTENTION_DELAY) + } + RetryClass::Contention => { + retry_delay(manager, attempt.saturating_sub(CONTENTION_FAST_RETRIES)) + } + }; + let due = now.saturating_add(delay); + + if expires_at(task).is_some_and(|expires_at| due > expires_at) { + return NextAttempt::Expired(reason); + } + + let retry = TaskStatusRetry { + created_at, + due: UTCDateTime::from_timestamp(due as i64), + attempt_number: attempt, + first_failed_at: first_failed_at + .unwrap_or_else(|| UTCDateTime::from_timestamp(now as i64)), + failure_reason: truncate_reason(reason), + }; + + NextAttempt::Retry { + retry, + is_stalled: attempt >= manager.alert_after, + } + } +} + +pub(crate) fn retry_delay(manager: &TaskManager, attempt: u64) -> u64 { + match &manager.strategy { + TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(), + TaskRetryStrategy::ExponentialBackoff(backoff) => { + let max_delay = backoff.max_delay.as_secs(); + let exponent = i32::try_from(attempt.saturating_sub(1)).unwrap_or(i32::MAX); + let delay = (backoff.initial_delay.as_secs() as f64 + * backoff.factor.into_inner().powi(exponent)) + .min(max_delay as f64) as u64; + + if backoff.jitter && delay > 1 { + rand::rng().random_range(delay / 2..=delay) + } else { + delay + } + } + } + .max(1) +} + +fn expires_at(task: &Task) -> Option { + match task { + Task::CalendarItipMessage(_) => { + Some((task.status().created_at().timestamp() as u64).saturating_add(ITIP_EXPIRY)) + } + _ => None, + } +} + +fn truncate_reason(mut reason: String) -> String { + if reason.len() > MAX_REASON_LEN { + reason.truncate(reason.floor_char_boundary(MAX_REASON_LEN)); + } + reason +} + +pub(crate) async fn commit( + server: &Server, + job: &TaskJob, + task: Task, + hash: u64, + outcome: Outcome, +) -> Commit { + let id = TaskId::Assigned(job.id); + let task_key = ValueClass::TaskQueue(TaskQueueClass::Task { id }); + let mut batch = BatchBuilder::new(); + batch + .assert_value(task_key.clone(), AssertValue::Hash(hash)) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + typ: job.typ, + due: job.due, + })); + + match outcome { + Outcome::Done(tasks) => { + batch.clear(task_key); + for task in tasks { + batch.schedule_task(task); + } + } + Outcome::Replace(ops) => { + for op in ops { + batch.any_op(op); + } + } + Outcome::Partial { remaining, failure } => { + schedule_retry(server, &mut batch, job, *remaining, failure); + } + Outcome::Retry(failure) => { + schedule_retry(server, &mut batch, job, task, failure); + } + Outcome::Fatal(reason) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskFailed), + TaskId = job.id, + Name = task.name(), + Reason = CompactString::from(reason.as_str()), + ); + + let mut task = task; + let status = TaskStatus::Failed(TaskStatusFailed { + created_at: task.status().created_at(), + failed_at: UTCDateTime::now(), + failed_attempt_number: task.attempt_number().saturating_add(1), + failure_reason: truncate_reason(reason), + }); + task.set_status(status); + batch + .set_task_due(id, job.typ, u64::MAX) + .set(task_key, task.to_pickled_vec()); + } + Outcome::Obsolete(reason) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + TaskId = job.id, + Name = task.name(), + Reason = reason, + ); + + batch.clear(task_key); + } + } + + let task_types = batch.queue_notify().task_types; + match server.store().write_batch(&mut batch).await { + Ok(_) => Commit::Committed(task_types), + Err(err) if err.is_assertion_failure() => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + TaskId = job.id, + Reason = "Task was changed or deleted while being processed; skipping update.", + ); + Commit::Conflict + } + Err(err) => { + trc::error!( + err.ctx(trc::Key::TaskId, job.id) + .details("Failed to update task after processing.") + ); + Commit::Failed + } + } +} + +fn schedule_retry( + server: &Server, + batch: &mut BatchBuilder, + job: &TaskJob, + mut task: Task, + failure: Failure, +) { + let id = TaskId::Assigned(job.id); + let task_key = ValueClass::TaskQueue(TaskQueueClass::Task { id }); + let manager = &server.core.network.task_manager; + + match NextAttempt::plan(manager, &task, failure.class, failure.reason, now()) { + NextAttempt::Retry { retry, is_stalled } => { + let due = retry.due.timestamp() as u64; + let attempt = retry.attempt_number; + if attempt == manager.alert_after { + trc::event!( + TaskManager(TaskManagerEvent::TaskStalled), + TaskId = job.id, + Name = task.name(), + Reason = CompactString::from(retry.failure_reason.as_str()), + Attempt = attempt, + NextRetry = trc::Value::Timestamp(due), + ); + } else { + trc::event!( + TaskManager(TaskManagerEvent::TaskRetry), + TaskId = job.id, + Name = task.name(), + Reason = CompactString::from(retry.failure_reason.as_str()), + Attempt = attempt, + NextRetry = trc::Value::Timestamp(due), + ); + } + + let (status, marker) = if is_stalled { + (TaskStatus::Stalled(retry), TaskQueueClass::DUE_STALLED) + } else { + (TaskStatus::Retry(retry), TaskQueueClass::DUE_RETRY) + }; + task.set_status(status); + batch + .set_task_due_marked(id, job.typ, due, Some(marker)) + .set(task_key, task.to_pickled_vec()); + } + NextAttempt::Expired(reason) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskExpired), + TaskId = job.id, + Name = task.name(), + Reason = CompactString::from(reason), + ); + + batch.clear(task_key); + } + } +} + diff --git a/crates/services/src/task_manager/reconcile.rs b/crates/services/src/task_manager/reconcile.rs new file mode 100644 index 00000000..793ad193 --- /dev/null +++ b/crates/services/src/task_manager/reconcile.rs @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::StoredTask; +use common::Server; +use registry::schema::structs::{Task, TaskStatus}; +use store::{ + ValueKey, + write::{BatchBuilder, TaskId, TaskQueueClass, ValueClass, assert::AssertValue}, +}; +use trc::AddContext; + +pub(crate) async fn insert_task_if_absent(server: &Server, task: Task) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + if let Some(id) = TaskId::singleton_of(&task) { + let key = ValueClass::TaskQueue(TaskQueueClass::Task { + id: TaskId::Assigned(id), + }); + let current = match server + .store() + .get_value::(ValueKey::from(key.clone())) + .await + .caused_by(trc::location!())? + { + Some(stored) if matches!(stored.task.status(), TaskStatus::Failed(_)) => { + AssertValue::Hash(stored.hash) + } + Some(_) => return Ok(()), + None => AssertValue::None, + }; + batch.assert_value(key, current); + } + + batch.schedule_task(task); + let queues = batch.queue_notify(); + match server.store().write_batch(&mut batch).await { + Ok(_) => { + server.notify_queues(queues).await; + Ok(()) + } + Err(err) if err.is_assertion_failure() => Ok(()), + Err(err) => Err(err.caused_by(trc::location!())), + } +} + +pub async fn schedule_replacing( + server: &Server, + batch: &mut BatchBuilder, + task: Task, +) -> trc::Result<()> { + if let Some(id) = TaskId::singleton_of(&task) + && let Some(current) = server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { + id: TaskId::Assigned(id), + }, + ))) + .await + .caused_by(trc::location!())? + { + let due = current.due_timestamp(); + if due != task.due_timestamp() { + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(id), + typ: current.object_type(), + due, + })); + } + } + + batch.schedule_task(task); + Ok(()) +} diff --git a/crates/services/src/task_manager/report.rs b/crates/services/src/task_manager/report.rs deleted file mode 100644 index 44bcaf60..00000000 --- a/crates/services/src/task_manager/report.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::task_manager::TaskResult; -use common::Server; -use smtp::reporting::{dmarc::DmarcReporting, tls::TlsReporting}; - -pub enum ReportId { - Dmarc(u64), - Tls(u64), -} - -pub(crate) trait SubmitReportTask: Sync + Send { - fn submit_report(&self, report_id: ReportId) -> impl Future + Send; -} - -impl SubmitReportTask for Server { - async fn submit_report(&self, report_id: ReportId) -> TaskResult { - match submit_report(self, report_id).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!(err.details("Failed to submit report")); - result - } - } - } -} - -async fn submit_report(server: &Server, report_id: ReportId) -> trc::Result { - match report_id { - ReportId::Dmarc(item_id) => server - .send_dmarc_aggregate_report(item_id) - .await - .map(|_| TaskResult::Success(vec![])), - ReportId::Tls(item_id) => server - .send_tls_aggregate_report(item_id) - .await - .map(|_| TaskResult::Success(vec![])), - } -} diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index 2f4495c9..743966d4 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -11,6 +11,7 @@ use std::{ time::{Instant, SystemTime}, }; +use crate::task_manager::reconcile::insert_task_if_absent; use common::{ BuildServer, Inner, LONG_1D_SLUMBER, config::{mailstore::spamfilter, telemetry::OtelMetrics}, @@ -22,7 +23,7 @@ use registry::{ }, types::EnumImpl, }; -use store::write::{BatchBuilder, now}; +use store::write::now; use trc::{ClusterEvent, Collector, MetricType, TaskManagerEvent, TelemetryEvent}; #[derive(PartialEq, Eq)] @@ -81,12 +82,12 @@ pub fn spawn_task_scheduler(inner: Arc) { } // Spam classifier training - if let Some(train_frequency) = server + if let Some(train_interval) = server .core .spam .classifier .as_ref() - .and_then(|c| c.train_frequency) + .and_then(|c| c.train_interval) { let next_train = match server.inner.data.spam_classifier.load().as_ref() { spamfilter::SpamClassifier::FhClassifier { @@ -94,8 +95,8 @@ pub fn spawn_task_scheduler(inner: Arc) { } | spamfilter::SpamClassifier::CcfhClassifier { last_trained_at, .. - } => now().saturating_sub(*last_trained_at).min(train_frequency), - spamfilter::SpamClassifier::Disabled => train_frequency, + } => now().saturating_sub(*last_trained_at).min(train_interval), + spamfilter::SpamClassifier::Disabled => train_interval, }; queue.schedule( @@ -117,13 +118,15 @@ pub fn spawn_task_scheduler(inner: Arc) { let mut next_metric_update = Instant::now(); + #[cfg(feature = "enterprise")] + let mut license_retry = LICENSE_RETRY_MIN; loop { tokio::time::sleep(queue.wake_up_time()).await; let server = inner.build_server(); let roles = &server.core.network.roles; - let mut batch = (roles.task_scheduler).then(BatchBuilder::new); + let mut tasks = (roles.task_scheduler).then(Vec::new); while let Some(event) = queue.pop() { match event.event { @@ -134,13 +137,13 @@ pub fn spawn_task_scheduler(inner: Arc) { Event::PurgeAccount, ); - if let Some(batch) = batch.as_mut() { + if let Some(tasks) = tasks.as_mut() { trc::event!( TaskManager(TaskManagerEvent::TaskQueued), Type = TaskStoreMaintenanceType::PurgeAccounts.as_str() ); - batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + tasks.push(Task::StoreMaintenance(TaskStoreMaintenance { maintenance_type: TaskStoreMaintenanceType::PurgeAccounts, status: TaskStatus::now(), shard_index: None, @@ -153,13 +156,13 @@ pub fn spawn_task_scheduler(inner: Arc) { Event::PurgeDataStore, ); - if let Some(batch) = batch.as_mut() { + if let Some(tasks) = tasks.as_mut() { trc::event!( TaskManager(TaskManagerEvent::TaskQueued), Type = TaskStoreMaintenanceType::PurgeData.as_str() ); - batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + tasks.push(Task::StoreMaintenance(TaskStoreMaintenance { maintenance_type: TaskStoreMaintenanceType::PurgeData, status: TaskStatus::now(), shard_index: None, @@ -172,13 +175,13 @@ pub fn spawn_task_scheduler(inner: Arc) { Event::PurgeBlobStore, ); - if let Some(batch) = batch.as_mut() { + if let Some(tasks) = tasks.as_mut() { trc::event!( TaskManager(TaskManagerEvent::TaskQueued), Type = TaskStoreMaintenanceType::PurgeBlob.as_str() ); - batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + tasks.push(Task::StoreMaintenance(TaskStoreMaintenance { maintenance_type: TaskStoreMaintenanceType::PurgeBlob, status: TaskStatus::now(), shard_index: None, @@ -193,7 +196,7 @@ pub fn spawn_task_scheduler(inner: Arc) { trc::event!( Cluster(ClusterEvent::NodeIdRenewed), - Id = server.registry().node_id() + NodeId = server.registry().node_id() ); let server = server.clone(); @@ -214,6 +217,10 @@ pub fn spawn_task_scheduler(inner: Arc) { #[cfg(not(feature = "enterprise"))] let is_enterprise = false; + if is_enterprise { + server.inner.refresh_live_gauges(); + } + tokio::spawn(async move { let elapsed = Instant::now(); otel.push_metrics(is_enterprise, start_time).await; @@ -276,6 +283,7 @@ pub fn spawn_task_scheduler(inner: Arc) { } } + match tokio::task::spawn_blocking(memory_stats::memory_stats).await { Ok(Some(stats)) => { Collector::update_gauge( @@ -301,26 +309,26 @@ pub fn spawn_task_scheduler(inner: Arc) { }); } Event::TrainSpamClassifier => { - if let Some(train_frequency) = server + if let Some(train_interval) = server .core .spam .classifier .as_ref() - .and_then(|c| c.train_frequency) + .and_then(|c| c.train_interval) { // Schedule next training queue.schedule( - Instant::now() + Duration::from_secs(train_frequency), + Instant::now() + Duration::from_secs(train_interval), Event::TrainSpamClassifier, ); - if let Some(batch) = batch.as_mut() { + if let Some(tasks) = tasks.as_mut() { trc::event!( TaskManager(TaskManagerEvent::TaskQueued), Type = TaskType::SpamFilterMaintenance.as_str() ); - batch.schedule_task(Task::SpamFilterMaintenance( + tasks.push(Task::SpamFilterMaintenance( TaskSpamFilterMaintenance { maintenance_type: TaskSpamFilterMaintenanceType::Train, status: TaskStatus::now(), @@ -333,16 +341,16 @@ pub fn spawn_task_scheduler(inner: Arc) { } } - if let Some(mut batch) = batch - && !batch.is_empty() - && let Err(err) = server.store().write(batch.build_all()).await - { - trc::error!(err.details("Failed to write scheduled tasks")); + for task in tasks.into_iter().flatten() { + if let Err(err) = insert_task_if_absent(&server, task).await { + trc::error!(err.details("Failed to write scheduled task")); + } } } }); } + impl Queue { pub fn schedule(&mut self, due: Instant, event: Event) { trc::event!( @@ -350,7 +358,7 @@ impl Queue { Due = trc::Value::Timestamp( now() + due.saturating_duration_since(Instant::now()).as_secs() ), - Id = event.name() + Name = event.name() ); self.heap.push(Action { due, event }); diff --git a/crates/services/src/task_manager/tasks/acme.rs b/crates/services/src/task_manager/tasks/acme.rs new file mode 100644 index 00000000..9c1f67e8 --- /dev/null +++ b/crates/services/src/task_manager/tasks/acme.rs @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::outcome::{Failure, Outcome, RetryClass}; +use common::{Server, network::acme::AcmeError}; +use compact_str::ToCompactString; +use registry::schema::structs::{ + CertificateManagement, Domain, Task, TaskDomainManagement, TaskStatus, +}; +use store::write::now; + +pub(crate) trait AcmeTask: Sync + Send { + fn acme_management(&self, task: &TaskDomainManagement) -> impl Future + Send; +} + +impl AcmeTask for Server { + async fn acme_management(&self, task: &TaskDomainManagement) -> Outcome { + match acme_management(self, task).await { + Ok(outcome) => outcome, + Err(err) => { + let outcome = Outcome::error(&err); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to run ACME task") + ); + outcome + } + } + } +} + +async fn acme_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { + match server.registry().object::(task.domain_id).await? { + Some(domain) => { + if matches!(domain.certificate_management, CertificateManagement::Manual) { + return Ok(Outcome::Obsolete( + "Domain no longer uses automatic certificate management", + )); + } + } + None => return Ok(Outcome::Obsolete("Domain no longer exists")), + } + + let err = match server.acme_renew(task.domain_id).await { + Ok(tasks) => return Ok(Outcome::Done(tasks)), + Err(err) => err, + }; + + if !matches!( + err, + AcmeError::NotDue { .. } + | AcmeError::Internal(_) + | AcmeError::AuthInvalid(_) + | AcmeError::OrderInvalid(_) + | AcmeError::AuthTimeout + | AcmeError::Backoff { .. } + ) { + trc::event!( + Acme(trc::AcmeEvent::Error), + Id = task.domain_id.to_compact_string(), + Reason = err.to_compact_string(), + ); + } + + Ok(match err { + AcmeError::NotDue { renew_at, .. } => { + Outcome::Done(vec![Task::AcmeRenewal(TaskDomainManagement { + domain_id: task.domain_id, + status: TaskStatus::at(renew_at as i64), + })]) + } + AcmeError::Crypto(_) | AcmeError::Invalid(_) | AcmeError::ChallengeNotSupported { .. } => { + Outcome::fatal(err.to_string()) + } + AcmeError::Backoff { wait: Some(wait) } => Outcome::Retry(Failure::new( + RetryClass::Throttled { + retry_at: now().saturating_add(wait.as_secs()), + }, + err.to_string(), + )), + AcmeError::Http(_) + | AcmeError::HttpStatus(_) + | AcmeError::Dns(_) + | AcmeError::Json(_) + | AcmeError::AuthInvalid(_) + | AcmeError::OrderInvalid(_) + | AcmeError::Registry(_) + | AcmeError::OrderTimeout + | AcmeError::AuthTimeout + | AcmeError::Backoff { wait: None } => Outcome::transient(err.to_string()), + AcmeError::Internal(error) => return Err(error), + }) +} diff --git a/crates/services/src/task_manager/tasks/alarm.rs b/crates/services/src/task_manager/tasks/alarm.rs new file mode 100644 index 00000000..ac3604c6 --- /dev/null +++ b/crates/services/src/task_manager/tasks/alarm.rs @@ -0,0 +1,1020 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use calcard::icalendar::{ + ArchivedICalendarComponent, ArchivedICalendarEntry, ArchivedICalendarParameterName, + ArchivedICalendarProperty, ICalendar, ICalendarComponentType, +}; +use common::{ + ArchivedDavName, DEFAULT_LOGO_BASE64, GroupwareResources, Server, + auth::{AccessToken, AccountInfo, BuildAccessToken}, + config::groupware::CalendarTemplateVariable, + ipc::{CalendarAlert, PushNotification}, + network::{ServerInstance, stream::NullIo}, +}; +use groupware::{ + cache::GroupwareCache, + calendar::{ + ArchivedCalendarEvent, ArchivedCalendarEventContent, CalendarEvent, CalendarEventContent, + EVENT_DRAFT, EVENT_HIDE_ATTENDEES, + alarm::{AlarmId, AlarmTarget, TriggeredAlarm}, + alerts::{DefaultAlerts, DefaultAlertsResolver}, + expand::RecurrenceKey, + identity::{CalendarAddresses, EventOwnership, ParticipantIdentityAddresses}, + index::ICalendarObjectUid, + participants::ParticipantVisibility, + privacy::EventPrivacy, + rights::EventAcl, + schedule::{EventAlarmScheduler, EventAlarmUsers}, + }, + scheduling::{ + ItipTime, ItipValue, + format::{DateStyle, TextFormatter, hyperlink}, + }, + strip_mailto_scheme, +}; +use mail_builder::{ + MessageBuilder, + headers::{HeaderType, content_type::ContentType}, + mime::{BodyPart, MimePart}, +}; +use mail_parser::decoders::html::html_to_text; +use registry::{ + schema::{ + enums::Permission, + structs::{TaskCalendarAlarmEmail, TaskCalendarAlarmNotification, TaskStatus}, + }, + types::EnumImpl, +}; +use smtp::{ + core::{Session, SessionData}, + inbound::data::MessageOrigin, +}; +use smtp_proto::{MailFrom, RcptTo}; +use std::{sync::Arc, time::Duration}; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes, now, serialize::rkyv_deserialize}, +}; +use trc::{AddContext, TaskManagerEvent}; +use types::{ + acl::Acl, + collection::{Collection, SyncCollection}, + field::CalendarEventField, +}; +use utils::{sanitize_email, template::Variables}; + +use crate::task_manager::{outcome::Outcome, tasks::SmtpRejection}; + +const ALARM_GRACE: i64 = 86400; +const ALARM_EXPIRED: &str = "Alarm expired before it could be delivered"; + +pub(crate) trait SendAlarmTask: Sync + Send { + fn send_display_alarm( + &self, + task: &TaskCalendarAlarmNotification, + ) -> impl Future + Send; + + fn send_email_alarm( + &self, + task: &TaskCalendarAlarmEmail, + server_instance: Arc, + ) -> impl Future + Send; +} + +impl SendAlarmTask for Server { + async fn send_display_alarm(&self, task: &TaskCalendarAlarmNotification) -> Outcome { + match send_display_alarm(self, task).await { + Ok(outcome) => outcome, + Err(err) => { + let result = Outcome::error(&err); + trc::error!( + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + result + } + } + } + + async fn send_email_alarm( + &self, + task: &TaskCalendarAlarmEmail, + server_instance: Arc, + ) -> Outcome { + match send_email_alarm(self, task, server_instance).await { + Ok(outcome) => outcome, + Err(err) => { + let result = Outcome::error(&err); + trc::error!( + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + result + } + } + } +} + +async fn send_email_alarm( + server: &Server, + task: &TaskCalendarAlarmEmail, + server_instance: Arc, +) -> trc::Result { + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); + let target_id = task + .target_account_id + .map_or(account_id, |id| id.document_id()); + let fired = FiredAlarm::new(&task.status); + let Some((meta_, event_)) = fetch_alarm_event(server, account_id, document_id).await? else { + return Ok(Outcome::Obsolete("Calendar event no longer exists")); + }; + let meta = meta_ + .unarchive::() + .caused_by(trc::location!())?; + let event = event_ + .unarchive::() + .caused_by(trc::location!())?; + let mut alarm = AlarmContext::new(server, account_id, document_id, target_id, meta).await?; + if let Some(reason) = alarm + .skip_reason(server, Some(Permission::CalendarAlarmsSend)) + .await? + { + return alarm.skip(server, event, fired, reason).await; + } + if fired.is_expired() { + return alarm.skip(server, event, fired, ALARM_EXPIRED).await; + } + let account_info = server + .account_info(target_id) + .await + .caused_by(trc::location!())?; + if account_info.name().is_empty() { + trc::event!( + Calendar(trc::CalendarEvent::AlarmFailed), + Reason = "Account does not have any email addresses", + AccountId = account_id, + DocumentId = document_id, + ); + return alarm.build_next(server, event, fired).await; + } + let recurrence_key = task + .recurrence_id + .and_then(RecurrenceKey::from_recurrence_id) + .map(RecurrenceKey::prefix); + let alarm_id = AlarmId::from_task_id(task.alarm_id); + let defaults = alarm.defaults(server, event, alarm_id).await?; + let Some(triggered) = + alarm.triggered(event, &defaults, alarm_id, task.event_id, recurrence_key) + else { + return alarm.build_next(server, event, fired).await; + }; + if triggered + .acknowledged() + .is_some_and(|acknowledged| acknowledged >= fired.alarm_time) + { + return alarm + .skip(server, event, fired, "Alarm was acknowledged") + .await; + } + let event_account_info = if target_id == account_id { + None + } else { + Some( + server + .account_info(account_id) + .await + .caused_by(trc::location!())?, + ) + }; + let attendees = alarm.attendee_list(server, &account_info, event).await?; + + // Build message body + let account_main_email = account_info.name(); + let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); + let logo_cid = format!("logo.{}@{account_main_domain}", now()); + let Some(tpl) = build_template( + server, + &account_info, + event_account_info.as_ref().unwrap_or(&account_info), + task, + meta, + event, + triggered.component(), + &attendees, + &logo_cid, + ) + .await? + else { + return alarm.build_next(server, event, fired).await; + }; + let txt_body = html_to_text(&tpl.body); + + // Obtain logo image + let logo = match server.logo_resource(account_main_domain).await { + Ok(logo) => logo, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to fetch logo image") + ); + None + } + }; + let logo = if let Some(logo) = &logo { + MimePart::new( + ContentType::new(logo.content_type.as_ref()), + BodyPart::Binary(logo.contents[..].into()), + ) + } else { + MimePart::new( + ContentType::new("image/png"), + BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), + ) + .transfer_encoding("base64") + } + .inline() + .cid(&logo_cid); + + // Build message + let mail_from = if let Some(from_email) = &server.core.groupware.alarms_from_email { + from_email.to_string() + } else { + format!("calendar-notification@{account_main_domain}") + }; + let message = MessageBuilder::new() + .from(( + server.core.groupware.alarms_from_name.as_str(), + mail_from.as_str(), + )) + .header("To", HeaderType::Text(tpl.to.as_str().into())) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .header("Reply-To", HeaderType::Text(account_main_email.into())) + .message_id(server.core.network.message_id()) + .subject(tpl.subject) + .body(MimePart::new( + ContentType::new("multipart/related"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("multipart/alternative"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("text/plain"), + BodyPart::Text(txt_body.into()), + ), + MimePart::new( + ContentType::new("text/html"), + BodyPart::Text(tpl.body.into()), + ), + ]), + ), + logo, + ]), + )) + .write_to_vec() + .unwrap_or_default(); + + // Send message + let server_ = server.clone(); + let mail_from = account_main_email.to_string(); + let to = tpl.to; + let result = tokio::spawn(async move { + let mut session = Session::::local( + server_, + server_instance, + SessionData::local(account_info, None, vec![], vec![], 0), + ); + + // MAIL FROM + let _ = session + .handle_mail_from(MailFrom { + address: mail_from.into(), + ..Default::default() + }) + .await; + if let Some(error) = session.has_failed() { + return Err(SmtpRejection::new("MAIL-FROM", &error)); + } + + // RCPT TO + session.params.rcpt_errors_wait = Duration::from_secs(0); + let _ = session + .handle_rcpt_to(RcptTo { + address: to.into(), + ..Default::default() + }) + .await; + if let Some(error) = session.has_failed() { + return Err(SmtpRejection::new("RCPT-TO", &error)); + } + + // DATA + session.data.message = message; + let response = session.queue_message(MessageOrigin::Internal).await; + if let smtp::core::State::Accepted(queue_id) = session.state { + Ok(queue_id) + } else { + Err(SmtpRejection::new( + "DATA", + std::str::from_utf8(&response).unwrap_or_default(), + )) + } + }) + .await; + + match result { + Ok(Ok(queue_id)) => { + trc::event!( + Calendar(trc::CalendarEvent::AlarmSent), + AccountId = account_id, + DocumentId = document_id, + QueueId = queue_id, + ); + } + Ok(Err(rejection)) => { + trc::event!( + Calendar(trc::CalendarEvent::AlarmFailed), + AccountId = account_id, + DocumentId = document_id, + Reason = rejection.reason.clone(), + ); + if rejection.is_temporary { + return Ok(Outcome::transient(rejection.reason)); + } + } + Err(_) => { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Join Error", + AccountId = account_id, + DocumentId = document_id, + CausedBy = trc::location!(), + ); + return Ok(Outcome::transient("Thread join error")); + } + } + + alarm.build_next(server, event, fired).await +} + +async fn send_display_alarm( + server: &Server, + task: &TaskCalendarAlarmNotification, +) -> trc::Result { + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); + let target_id = task + .target_account_id + .map_or(account_id, |id| id.document_id()); + let fired = FiredAlarm::new(&task.status); + let Some((meta_, event_)) = fetch_alarm_event(server, account_id, document_id).await? else { + return Ok(Outcome::Obsolete("Calendar event no longer exists")); + }; + let meta = meta_ + .unarchive::() + .caused_by(trc::location!())?; + let event = event_ + .unarchive::() + .caused_by(trc::location!())?; + let mut alarm = AlarmContext::new(server, account_id, document_id, target_id, meta).await?; + if let Some(reason) = alarm.skip_reason(server, None).await? { + return alarm.skip(server, event, fired, reason).await; + } + if fired.is_expired() { + return alarm.skip(server, event, fired, ALARM_EXPIRED).await; + } + let recurrence_key = task + .recurrence_id + .and_then(RecurrenceKey::from_recurrence_id) + .map(RecurrenceKey::prefix); + let alarm_id = AlarmId::from_task_id(task.alarm_id); + let defaults = alarm.defaults(server, event, alarm_id).await?; + let Some(triggered) = + alarm.triggered(event, &defaults, alarm_id, task.event_id, recurrence_key) + else { + return alarm.build_next(server, event, fired).await; + }; + if triggered + .acknowledged() + .is_some_and(|acknowledged| acknowledged >= fired.alarm_time) + { + return alarm + .skip(server, event, fired, "Alarm was acknowledged") + .await; + } + + let alert_id = match triggered.alert_id() { + Some(alert_id) => alert_id.to_string(), + None => format!("k{}", alarm.alarm_position(event, task) + 1), + }; + server + .broadcast_push_notification(PushNotification::CalendarAlert(CalendarAlert { + account_id: target_id, + event_account_id: account_id, + event_id: document_id, + recurrence_id: task.recurrence_id, + uid: event + .data + .event + .object_uid() + .unwrap_or_default() + .to_string(), + alert_id, + })) + .await; + + alarm.build_next(server, event, fired).await +} + +struct AlarmContext<'x> { + account_id: u32, + document_id: u32, + target_id: u32, + target: AlarmTarget, + meta: &'x ArchivedCalendarEvent, + calendar_ids: Vec, + access_token: AccessToken, + shared_resources: Option>, + resolver: DefaultAlertsResolver, +} + +impl<'x> AlarmContext<'x> { + async fn new( + server: &Server, + account_id: u32, + document_id: u32, + target_id: u32, + meta: &'x ArchivedCalendarEvent, + ) -> trc::Result { + let access_token = server + .access_token(target_id) + .await + .caused_by(trc::location!())? + .build(); + let shared_resources = if access_token.is_member(account_id) { + None + } else { + Some( + server + .fetch_groupware_resources(target_id, account_id, SyncCollection::Calendar) + .await + .caused_by(trc::location!())?, + ) + }; + let resolver = shared_resources + .as_ref() + .map_or_else(DefaultAlertsResolver::default, |resources| { + DefaultAlertsResolver::with_resources(account_id, resources.clone()) + }); + Ok(AlarmContext { + account_id, + document_id, + target_id, + target: AlarmTarget::for_account(account_id, target_id), + meta, + calendar_ids: meta.names.iter().map(ArchivedDavName::parent_id).collect(), + access_token, + shared_resources, + resolver, + }) + } + + async fn skip_reason( + &mut self, + server: &Server, + permission: Option, + ) -> trc::Result> { + if !server.core.groupware.alarms_enabled { + return Ok(Some("Calendar alarms are disabled")); + } + if self.meta.flags & EVENT_DRAFT != 0 { + return Ok(Some("Calendar event is a draft")); + } + if !self + .resolver + .is_subscribed( + server, + self.account_id, + self.target_id, + self.calendar_ids.iter().copied(), + ) + .await + .caused_by(trc::location!())? + { + return Ok(Some("User is not subscribed to the calendar")); + } + if permission.is_some_and(|permission| !self.access_token.has_permission(permission)) { + return Ok(Some( + "Account does not have permission to send calendar alarms", + )); + } + let Some(resources) = &self.shared_resources else { + return Ok(None); + }; + if EventPrivacy::from_flags(self.meta.flags.to_native()) != EventPrivacy::Public { + return Ok(Some("Calendar event is not public")); + } + if self.calendar_ids.iter().any(|calendar_id| { + resources.has_access_to_container(&self.access_token, *calendar_id, Acl::ReadItems) + }) { + Ok(None) + } else { + Ok(Some("User no longer has access to the calendar")) + } + } + + async fn skip( + &mut self, + server: &Server, + event: &ArchivedCalendarEventContent, + fired: FiredAlarm, + reason: &'static str, + ) -> trc::Result { + trc::event!( + Calendar(trc::CalendarEvent::AlarmSkipped), + Reason = reason, + AccountId = self.account_id, + DocumentId = self.document_id, + ); + self.build_next(server, event, fired).await + } + + async fn defaults( + &mut self, + server: &Server, + event: &ArchivedCalendarEventContent, + alarm_id: Option, + ) -> trc::Result { + if alarm_id.is_some_and(AlarmId::is_default) { + self.resolver + .resolve_for_content( + server, + self.account_id, + self.target_id, + event, + self.calendar_ids.iter().copied(), + ) + .await + .caused_by(trc::location!()) + } else { + Ok(DefaultAlerts::disabled()) + } + } + + fn triggered<'y>( + &self, + event: &'y ArchivedCalendarEventContent, + defaults: &'y DefaultAlerts, + alarm_id: Option, + event_id: u64, + recurrence_key: Option, + ) -> Option> { + let triggered = alarm_id.and_then(|alarm_id| { + event.triggered_alarm( + self.target, + alarm_id, + event_id as u16, + recurrence_key, + defaults, + ) + }); + if triggered.is_none() { + trc::event!( + TaskManager(TaskManagerEvent::MetadataNotFound), + Details = "Calendar Alarm component not found", + AccountId = self.account_id, + DocumentId = self.document_id, + ); + } + triggered + } + + fn alarm_position( + &self, + event: &ArchivedCalendarEventContent, + task: &TaskCalendarAlarmNotification, + ) -> usize { + match AlarmId::from_task_id(task.alarm_id) { + Some(AlarmId::Stored(alarm_id)) => { + let components = &event.data.event.components; + components + .get(task.event_id as usize) + .into_iter() + .flat_map(|component| component.component_ids.iter()) + .map(|id| id.to_native()) + .filter(|id| { + components + .get(*id as usize) + .is_some_and(|c| c.component_type == ICalendarComponentType::VAlarm) + }) + .position(|id| id == alarm_id as u32) + .unwrap_or_default() + } + Some(AlarmId::Personal(index)) => index as usize, + Some(AlarmId::Default(_)) | None => 0, + } + } + + async fn attendee_list( + &self, + server: &Server, + account_info: &AccountInfo, + event: &ArchivedCalendarEventContent, + ) -> trc::Result { + let Some(resources) = self + .shared_resources + .as_ref() + .filter(|_| self.meta.flags.to_native() & EVENT_HIDE_ATTENDEES != 0) + else { + return Ok(AttendeeList::All); + }; + let acl = EventAcl::for_calendars( + resources, + &self.access_token, + self.calendar_ids.iter().copied(), + ); + if acl.may_write(EventOwnership::NotOwner) { + return Ok(AttendeeList::All); + } + let identities = server + .identity_addresses(self.target_id, account_info) + .await + .caused_by(trc::location!())?; + let ical = + rkyv_deserialize::<_, ICalendar>(&event.data.event).caused_by(trc::location!())?; + Ok(if acl.may_write(identities.event_ownership(&ical)) { + AttendeeList::All + } else { + AttendeeList::OwnersAndSelf(identities) + }) + } + + async fn build_next( + &mut self, + server: &Server, + event: &ArchivedCalendarEventContent, + fired: FiredAlarm, + ) -> trc::Result { + let users = EventAlarmUsers::for_target(self.account_id, event, self.target)? + .with_event_flags(self.meta.flags.to_native()); + let now = now() as i64; + let start_time = fired + .alarm_time + .min(now) + .max(now + server.core.groupware.alarms_minimum_interval); + let next_alarm = server + .next_event_alarms( + self.account_id, + &users, + &event.data, + &self.calendar_ids, + start_time, + &mut self.resolver, + ) + .await? + .into_alarm(); + + Ok(match next_alarm { + Some(next_alarm) => { + Outcome::Replace(next_alarm.build_write_ops(self.account_id, self.document_id)) + } + None => Outcome::done(), + }) + } +} + +enum AttendeeList { + All, + OwnersAndSelf(CalendarAddresses), +} + +impl AttendeeList { + fn includes(&self, entry: &ArchivedICalendarEntry) -> bool { + match self { + AttendeeList::All => true, + AttendeeList::OwnersAndSelf(identities) => entry.is_visible_participant(identities), + } + } +} + +async fn fetch_alarm_event( + server: &Server, + account_id: u32, + document_id: u32, +) -> trc::Result, Archive)>> { + let (Some(meta), Some(event)) = ( + server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) + .await + .caused_by(trc::location!())?, + server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await + .caused_by(trc::location!())?, + ) else { + trc::event!( + TaskManager(TaskManagerEvent::MetadataNotFound), + Details = "Calendar Event metadata not found", + AccountId = account_id, + DocumentId = document_id, + ); + return Ok(None); + }; + Ok(Some((meta, event))) +} + +#[derive(Debug, Clone, Copy)] +struct FiredAlarm { + alarm_time: i64, +} + +impl FiredAlarm { + fn new(status: &TaskStatus) -> Self { + FiredAlarm { + alarm_time: match status { + TaskStatus::Pending(pending) => pending.due.timestamp(), + TaskStatus::Retry(retry) | TaskStatus::Stalled(retry) => { + retry.first_failed_at.timestamp() + } + TaskStatus::Failed(_) => now() as i64, + }, + } + } + + fn is_expired(&self) -> bool { + now() as i64 > self.alarm_time.saturating_add(ALARM_GRACE) + } +} + +struct Details { + to: String, + subject: String, + body: String, +} + +#[allow(clippy::too_many_arguments)] +async fn build_template( + server: &Server, + account_info: &AccountInfo, + event_account_info: &AccountInfo, + alarm: &TaskCalendarAlarmEmail, + meta: &ArchivedCalendarEvent, + event: &ArchivedCalendarEventContent, + alarm_component: Option<&ArchivedICalendarComponent>, + attendees: &AttendeeList, + logo_cid: &str, +) -> trc::Result> { + let account_id = alarm.account_id.document_id(); + let document_id = alarm.document_id.document_id(); + let Some(event_component) = event.data.event.components.get(alarm.event_id as usize) else { + trc::event!( + TaskManager(TaskManagerEvent::MetadataNotFound), + Details = "Calendar Alarm component not found", + AccountId = account_id, + DocumentId = document_id, + ); + return Ok(None); + }; + + // Build webcal URI + let webcal_uri = match meta.webcal_uri(server, event_account_info).await { + Ok(uri) => uri, + Err(err) => { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to generate webcal URI") + ); + String::from("#") + } + }; + + // Obtain alarm details + let mut summary = None; + let mut description = None; + let mut rcpt_to = None; + let mut location = None; + let mut conference = None; + let mut organizer = None; + let mut guests = vec![]; + + for entry in alarm_component + .into_iter() + .flat_map(|alarm_component| alarm_component.entries.iter()) + { + match &entry.name { + ArchivedICalendarProperty::Summary => { + summary = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Description => { + description = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Attendee => { + rcpt_to = entry + .values + .first() + .and_then(|v| v.as_text()) + .map(strip_mailto_scheme) + .and_then(sanitize_email); + } + _ => {} + } + } + + for entry in event_component.entries.iter() { + match &entry.name { + ArchivedICalendarProperty::Summary if summary.is_none() => { + summary = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Description if description.is_none() => { + description = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Location => { + location = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Conference if conference.is_none() => { + conference = entry.values.first().and_then(|v| v.as_text()); + } + ArchivedICalendarProperty::Attendee if !attendees.includes(entry) => {} + ArchivedICalendarProperty::Organizer | ArchivedICalendarProperty::Attendee => { + let email = entry + .values + .first() + .and_then(|v| v.as_text()) + .map(strip_mailto_scheme); + let name = entry.params.iter().find_map(|param| { + if let ArchivedICalendarParameterName::Cn = param.name { + param.value.as_text() + } else { + None + } + }); + + if email.is_some() || name.is_some() { + if matches!(entry.name, ArchivedICalendarProperty::Organizer) { + organizer = Some((email, name)); + } else { + guests.push((email, name)); + } + } + } + _ => {} + } + } + + // Validate recipient + let rcpt_to = if let Some(rcpt_to) = rcpt_to { + if server.core.groupware.alarms_allow_external_recipients + || account_info.addresses().contains(&rcpt_to) + { + rcpt_to + } else { + trc::event!( + Calendar(trc::CalendarEvent::AlarmRecipientOverride), + Reason = "External recipient not allowed for calendar alarms", + Address = rcpt_to, + AccountId = account_id, + DocumentId = document_id, + ); + + account_info.name().to_string() + } + } else { + account_info.name().to_string() + }; + + + #[cfg(not(feature = "enterprise"))] + let template = &server.core.groupware.alarms_template; + let formatter = TextFormatter::new(account_info.locale().as_str())?; + let locale = formatter.locale; + + let start = formatter.field_to_string( + &ItipValue::Time(ItipTime { + start: alarm.event_start.timestamp(), + tz_id: alarm.event_start_tz as u16, + }), + DateStyle::Short, + ); + let end = formatter.field_to_string( + &ItipValue::Time(ItipTime { + start: alarm.event_end.timestamp(), + tz_id: alarm.event_end_tz as u16, + }), + DateStyle::Short, + ); + let subject = format!( + "{}: {} @ {}", + locale.calendar_alarm_subject_prefix, + summary.or(description).unwrap_or("No Subject"), + start + ); + let organizer = organizer + .map(|(email, name)| match (email, name) { + (Some(email), Some(name)) => format!("{} <{}>", name, email), + (Some(email), None) => email.to_string(), + (None, Some(name)) => name.to_string(), + _ => unreachable!(), + }) + .unwrap_or_else(|| account_info.name().to_string()); + let logo_cid = format!("cid:{logo_cid}"); + let mut variables = Variables::new(); + variables.insert_single(CalendarTemplateVariable::PageTitle, subject.as_str()); + variables.insert_single(CalendarTemplateVariable::Lang, locale.name); + variables.insert_single(CalendarTemplateVariable::Dir, locale.direction); + variables.insert_single( + CalendarTemplateVariable::Header, + locale.calendar_alarm_header, + ); + variables.insert_single( + CalendarTemplateVariable::Footer, + locale.calendar_alarm_footer, + ); + variables.insert_single( + CalendarTemplateVariable::ActionName, + locale.calendar_alarm_open, + ); + variables.insert_single(CalendarTemplateVariable::ActionUrl, webcal_uri.as_str()); + variables.insert_single( + CalendarTemplateVariable::AttendeesTitle, + locale.calendar_attendees, + ); + if let Some(summary) = summary.filter(|summary| !summary.is_empty()) { + variables.insert_single(CalendarTemplateVariable::EventTitle, summary); + } + variables.insert_single(CalendarTemplateVariable::LogoCid, logo_cid.as_str()); + if let Some(description) = description { + variables.insert_single(CalendarTemplateVariable::EventDescription, description); + } + variables.insert_block( + CalendarTemplateVariable::EventDetails, + [ + Some(vec![ + (CalendarTemplateVariable::Key, locale.calendar_start), + (CalendarTemplateVariable::Value, start.as_str()), + ]), + Some(vec![ + (CalendarTemplateVariable::Key, locale.calendar_end), + (CalendarTemplateVariable::Value, end.as_str()), + ]), + location.map(|location| { + vec![ + (CalendarTemplateVariable::Key, locale.calendar_location), + (CalendarTemplateVariable::Value, location), + ] + }), + conference.map(|conference| { + let mut detail = vec![ + (CalendarTemplateVariable::Key, locale.calendar_conference), + (CalendarTemplateVariable::Value, conference), + ]; + if let Some(link) = hyperlink(conference) { + detail.push((CalendarTemplateVariable::Link, link)); + } + detail + }), + Some(vec![ + (CalendarTemplateVariable::Key, locale.calendar_organizer), + (CalendarTemplateVariable::Value, organizer.as_str()), + ]), + ] + .into_iter() + .flatten(), + ); + if !guests.is_empty() { + variables.insert_block( + CalendarTemplateVariable::Attendees, + guests.into_iter().map(|(email, name)| { + [ + (CalendarTemplateVariable::Key, name.unwrap_or_default()), + (CalendarTemplateVariable::Value, email.unwrap_or_default()), + ] + }), + ); + } + Ok(Some(Details { + to: rcpt_to, + body: template.eval(&variables), + subject, + })) +} diff --git a/crates/services/src/task_manager/tasks/destroy_account.rs b/crates/services/src/task_manager/tasks/destroy_account.rs new file mode 100644 index 00000000..a9abd371 --- /dev/null +++ b/crates/services/src/task_manager/tasks/destroy_account.rs @@ -0,0 +1,287 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::{outcome::Outcome, tasks::grantee_cleanup::GranteeCleanupTask}; +use common::{Server, cache::invalidate::CacheInvalidationBuilder, ipc::CacheInvalidation}; +use email::{message::metadata::MetadataStructure, sieve::SieveScript}; +use groupware::file::FileNode; +use registry::{ + schema::{ + enums::TaskAccountMaintenanceType, + prelude::{ObjectType, Property}, + structs::{Task, TaskAccountMaintenance, TaskDestroyAccount, TaskStatus}, + }, + types::EnumImpl, +}; +use store::{ + SerializeInfallible, ValueKey, + registry::RegistryQuery, + search::SearchQuery, + write::{ + Archive, ArchiveBytes, BatchBuilder, BlobLink, BlobOp, RegistryClass, SearchIndex, + ValueClass, + }, +}; +use trc::AddContext; +use types::{ + blob_hash::BlobHash, + collection::Collection, + field::{EmailField, Field}, + id::Id, +}; + +const DESTROY_WINDOW: usize = 5_000; + +pub(crate) trait DestroyAccountTask: Sync + Send { + fn destroy_account(&self, task: &TaskDestroyAccount) -> impl Future + Send; +} + +impl DestroyAccountTask for Server { + async fn destroy_account(&self, task: &TaskDestroyAccount) -> Outcome { + match destroy_account(self, task).await { + Ok(outcome) => outcome, + Err(err) => { + let result = Outcome::error(&err); + trc::error!( + err.account_id(task.account_id.document_id()) + .details("Failed to destroy account") + ); + result + } + } + } +} + +async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Result { + let account_id = task.account_id.document_id(); + + if server + .store() + .key_exists(ValueKey::from(ValueClass::Registry(RegistryClass::Item { + object_id: ObjectType::Account.to_id(), + item_id: account_id as u64, + }))) + .await + .caused_by(trc::location!())? + { + return Ok(Outcome::Done(vec![Task::AccountMaintenance( + TaskAccountMaintenance { + account_id: task.account_id, + maintenance_type: TaskAccountMaintenanceType::RebuildShares, + status: TaskStatus::now(), + }, + )])); + } + + server + .remove_grantee_references(account_id, task.member_group_ids.as_slice()) + .await + .caused_by(trc::location!())?; + + // Destroy public keys, masked emails and OAuth grants + #[cfg(not(feature = "enterprise"))] + let objects = [ObjectType::PublicKey, ObjectType::OAuthGrant]; + for object in objects { + let mut batch = BatchBuilder::new(); + let ids = server + .registry() + .query::>(RegistryQuery::new(object).with_account(account_id)) + .await?; + let object_id = object.to_id(); + + for id in ids { + batch + .clear(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: id.id(), + })) + .clear(ValueClass::Registry(RegistryClass::IndexId { + object_id, + item_id: id.id(), + })) + .clear(ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId as u16, + object_id, + item_id: id.id(), + key: (account_id as u64).serialize(), + })) + .clear(ValueClass::Registry(RegistryClass::Reference { + to_object_id: ObjectType::Account as u16, + to_item_id: account_id as u64, + from_object_id: object_id, + from_item_id: id.id(), + })) + .bulk_commit_point(); + } + + if !batch.is_empty() { + server.store().write_batch(&mut batch).await?; + } + } + + // Remove archived items + + // Remove search index + for index in [ + SearchIndex::Email, + SearchIndex::Contacts, + SearchIndex::Calendar, + SearchIndex::File, + ] { + if let Some(store) = server.search_store().internal_fts() { + store.unindex_account(index, account_id).await?; + } else { + server + .search_store() + .unindex(SearchQuery::new(index).with_account_id(account_id)) + .await?; + } + + server + .store() + .clear_search_index_queue(index, account_id) + .await?; + } + + // Unlink all accounts's blobs + destroy_account_blobs(server, account_id).await?; + + server + .destroy_viewer_metadata(account_id) + .await + .caused_by(trc::location!())?; + server + .destroy_owner_metadata(account_id) + .await + .caused_by(trc::location!())?; + + // Drop the resident caches and the writer's pending state on every node + server + .invalidate_caches( + CacheInvalidationBuilder::from(CacheInvalidation::MessageCache(account_id)) + .with_invalidation(CacheInvalidation::GroupwareResources(account_id)) + .with_invalidation(CacheInvalidation::PrivateMetadata(account_id)), + ) + .await?; + server.inner.cache.swap.forget(account_id); + + // Remove persisted cache snapshots + server.inner.cache.swap.remove_account(account_id).await; + + // Destroy account data + server + .store() + .danger_destroy_account(account_id) + .await + .caused_by(trc::location!())?; + + Ok(Outcome::done()) +} + +pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { + let window = server.store().is_postgresql().then_some(DESTROY_WINDOW); + let mut links = Vec::new(); + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + + let class = ValueClass::Immutable(EmailField::Metadata.into()); + let mut next = Some(0); + while let Some(from_document_id) = next { + next = server + .values_window( + account_id, + Collection::Email, + &class, + from_document_id, + window, + |document_id, structure: MetadataStructure| { + links.push((document_id, structure.unarchive()?.blob_hash())); + Ok(()) + }, + ) + .await + .caused_by(trc::location!())?; + clear_links( + server, + account_id, + Collection::Email, + &mut links, + &mut batch, + ) + .await?; + } + + let class = ValueClass::Property(Field::ARCHIVE.into()); + for collection in [Collection::FileNode, Collection::SieveScript] { + let mut next = Some(0); + while let Some(from_document_id) = next { + next = server + .values_window( + account_id, + collection, + &class, + from_document_id, + window, + |document_id, archive: Archive| { + let blob_hash = match collection { + Collection::FileNode => archive + .unarchive::()? + .file() + .map(|file| BlobHash::from(&file.blob_hash)), + _ => Some(BlobHash::from( + &archive.unarchive::()?.blob_hash, + )), + }; + if let Some(blob_hash) = blob_hash { + links.push((document_id, blob_hash)); + } + Ok(()) + }, + ) + .await + .caused_by(trc::location!())?; + clear_links(server, account_id, collection, &mut links, &mut batch).await?; + } + } + + if !batch.is_empty() { + server + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + } + + Ok(()) +} + +async fn clear_links( + server: &Server, + account_id: u32, + collection: Collection, + links: &mut Vec<(u32, BlobHash)>, + batch: &mut BatchBuilder, +) -> trc::Result<()> { + for (document_id, hash) in links.drain(..) { + if batch.is_large_bulk_batch() { + server + .store() + .write_batch(batch) + .await + .caused_by(trc::location!())?; + *batch = BatchBuilder::new(); + batch.with_account_id(account_id); + } + batch + .with_collection(collection) + .with_document(document_id) + .clear(ValueClass::Blob(BlobOp::Link { + hash, + to: BlobLink::Document, + })); + } + Ok(()) +} diff --git a/crates/services/src/task_manager/dkim.rs b/crates/services/src/task_manager/tasks/dkim.rs similarity index 66% rename from crates/services/src/task_manager/dkim.rs rename to crates/services/src/task_manager/tasks/dkim.rs index 341831d8..f0b4a1f1 100644 --- a/crates/services/src/task_manager/dkim.rs +++ b/crates/services/src/task_manager/tasks/dkim.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::TaskResult; +use crate::task_manager::{outcome::Outcome, tasks::append_error}; use common::{ Server, cache::invalidate::CacheInvalidationBuilder, @@ -14,6 +14,7 @@ use common::{ generate_dkim_selector, }, }; +use compact_str::{CompactString, ToCompactString}; use registry::{ schema::{ enums::{DkimRotationStage, DkimSignatureType, DnsRecordType}, @@ -25,7 +26,6 @@ use registry::{ }, types::{datetime::UTCDateTime, id::ObjectId}, }; -use std::fmt::Write; use store::{ registry::{ RegistryObject, RegistryQuery, @@ -37,59 +37,74 @@ use trc::{DkimEvent, DnsEvent}; use types::id::Id; pub(crate) trait DkimManagementTask: Sync + Send { - fn dkim_management( - &self, - task: &TaskDomainManagement, - ) -> impl Future + Send; + fn dkim_management(&self, task: &TaskDomainManagement) -> impl Future + Send; } impl DkimManagementTask for Server { - async fn dkim_management(&self, task: &TaskDomainManagement) -> TaskResult { - match dkim_management(self, task).await { - Ok(result) => result, + async fn dkim_management(&self, task: &TaskDomainManagement) -> Outcome { + let mut do_refresh = false; + let result = dkim_management(self, task, &mut do_refresh).await; + + if do_refresh + && let Err(err) = self + .invalidate_caches(CacheInvalidationBuilder::default().with_invalidation( + CacheInvalidation::DkimSignature(task.domain_id.document_id()), + )) + .await + { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to invalidate caches after DKIM management task") + ); + } + + match result { + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let outcome = Outcome::error(&err); trc::error!( err.caused_by(trc::location!()) .details("Failed to run DKIM management task") ); - result + outcome } } } } -async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { +async fn dkim_management( + server: &Server, + task: &TaskDomainManagement, + do_refresh: &mut bool, +) -> trc::Result { let Some(domain) = server.registry().object::(task.domain_id).await? else { - return Ok(TaskResult::permanent("Domain not found".to_string())); + return Ok(Outcome::Obsolete("Domain no longer exists")); }; let DkimManagement::Automatic(dkim) = domain.dkim_management else { - return Ok(TaskResult::permanent( - "Domain is not set to automatic DKIM management".to_string(), + return Ok(Outcome::Obsolete( + "Domain no longer uses automatic DKIM management", )); }; let configured = dkim.algorithms.into_inner(); if configured.is_empty() { - return Ok(TaskResult::permanent( - "No DKIM algorithms configured for domain".to_string(), - )); + return Ok(Outcome::fatal("No DKIM algorithms configured for domain")); } let mut create_signatures = configured.clone(); let mut active_types: Vec = Vec::with_capacity(configured.len()); - let dns_updater = match domain.dns_management { - DnsManagement::Automatic(props) if props.publish_records.contains(&DnsRecordType::Dkim) => { - match server.build_dns_updater(props.dns_server_id).await? { - Ok(updater) => Some((updater, props.origin.unwrap_or_else(|| domain.name.clone()))), - Err(err) => { - return Ok(TaskResult::permanent(format!( - "Failed to build DNS updater: {}", - err - ))); - } + let dns_updater = if domain.dns_management.publishes(DnsRecordType::Dkim) + && let DnsManagement::Automatic(props) = domain.dns_management + { + match server.build_dns_updater(props.dns_server_id).await? { + Ok(updater) => Some((updater, props.origin.unwrap_or_else(|| domain.name.clone()))), + Err(err) => { + return Ok(Outcome::fatal(format!( + "Failed to build DNS updater: {err}" + ))); } } - _ => None, + } else { + None }; // Fetch existing DKIM keys @@ -163,18 +178,22 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } let now = now(); - let mut do_refresh = false; let mut temporary_errors = String::new(); for algorithm in create_signatures { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] let secret = { if dkim.selector_template.contains("dummy") { match algorithm { DkimSignatureType::Dkim1Ed25519Sha256 - | DkimSignatureType::Dkim2Ed25519Sha256 => TEST_ED25519_KEY.to_string(), + | DkimSignatureType::Dkim2Ed25519Sha256 => { + test_support::TEST_ED25519_KEY.to_string() + } DkimSignatureType::Dkim1RsaSha256 | DkimSignatureType::Dkim2RsaSha256 => { - TEST_RSA_KEY.to_string() + test_support::TEST_RSA_KEY.to_string() + } + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => { + generate_dkim_private_key(algorithm).await.unwrap().unwrap() } } } else { @@ -183,19 +202,18 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R }; // Generate new key and selector - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] let secret = match generate_dkim_private_key(algorithm).await? { Ok(secret) => secret, Err(err) => { - return Ok(TaskResult::permanent(err.to_string())); + return Ok(Outcome::fatal(err)); } }; let selector = match generate_dkim_selector(&dkim.selector_template, algorithm) { Ok(selector) => selector, Err(err) => { - return Ok(TaskResult::permanent(format!( - "Failed to generate DKIM selector: {}", - err + return Ok(Outcome::fatal(format!( + "Failed to generate DKIM selector: {err}" ))); } }; @@ -207,7 +225,6 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R let signature = Dkim1Signature { stage: DkimRotationStage::Active, domain_id: task.domain_id, - member_tenant_id: domain.member_tenant_id, selector: selector.clone(), private_key, ..Default::default() @@ -218,19 +235,24 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R DkimSignature::Dkim1RsaSha256(signature) } } - DkimSignatureType::Dkim2Ed25519Sha256 | DkimSignatureType::Dkim2RsaSha256 => { + DkimSignatureType::Dkim2Ed25519Sha256 + | DkimSignatureType::Dkim2RsaSha256 + | DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => { let signature = Dkim2Signature { stage: DkimRotationStage::Active, domain_id: task.domain_id, - member_tenant_id: domain.member_tenant_id, selector: selector.clone(), private_key, ..Default::default() }; - if algorithm == DkimSignatureType::Dkim2Ed25519Sha256 { - DkimSignature::Dkim2Ed25519Sha256(signature) - } else { - DkimSignature::Dkim2RsaSha256(signature) + match algorithm { + DkimSignatureType::Dkim2Ed25519Sha256 => { + DkimSignature::Dkim2Ed25519Sha256(signature) + } + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => { + DkimSignature::Dkim2MlDsa44Ed25519Sha512(signature) + } + _ => DkimSignature::Dkim2RsaSha256(signature), } } }; @@ -239,9 +261,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R if let Some((updater, origin)) = &dns_updater { let record = generate_dkim_dns_record(&signature, &domain.name).await?; let dns_update::DnsRecord::TXT(txt_value) = &record.record else { - return Ok(TaskResult::permanent( - "DKIM record must be a TXT record".to_string(), - )); + return Ok(Outcome::fatal("DKIM record must be a TXT record")); }; let propagation_target = txt_value.clone(); let published = match updater @@ -255,35 +275,23 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R { Ok(_) => { let propagated = updater - .wait_for_txt_propagation(&record.name, origin, &propagation_target) + .check_txt_propagation(&record.name, origin, &propagation_target) .await; if !propagated { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( &mut temporary_errors, - "DKIM record {} did not propagate, will retry.", - record.name + format_args!( + "DKIM record {} did not propagate yet, will retry.", + record.name + ), ); } propagated } Err(err) => { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( &mut temporary_errors, - "Failed to publish DKIM record {}: {err}.", - record.name - ); - trc::event!( - Dns(DnsEvent::RecordCreationFailed), - Hostname = record.name.clone(), - Details = origin.clone(), - Type = "TXT", - Reason = err, + format_args!("Failed to publish DKIM record {}: {err}.", record.name), ); false } @@ -291,11 +299,11 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R let signature_transition = if published { trc::event!( Dkim(DkimEvent::SignaturePublished), - Id = selector.clone(), - Details = domain.name.clone() + Name = selector.clone(), + Domain = domain.name.clone() ); - do_refresh = true; + *do_refresh = true; UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64) } else { // Something went wrong, reschedule. @@ -318,17 +326,18 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R .await? { RegistryWriteResult::Success(_) => { + *do_refresh |= is_active; if is_active { active_types.push(algorithm); } trc::event!( Dkim(DkimEvent::SignatureCreated), - Id = selector, - Details = domain.name.clone() + Name = selector, + Domain = domain.name.clone() ); } err => { - return Ok(TaskResult::permanent(format!( + return Ok(Outcome::fatal(format!( "Failed to write DKIM signature: {err}" ))); } @@ -347,7 +356,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R let mut new_signature = signature.object.clone(); new_signature.set_next_transition(signature_transition); - if let SignatureUpdate::Failed(task_result) = update_signature( + if let SignatureUpdate::Failed(reason) = update_signature( server, signature, new_signature, @@ -356,7 +365,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ) .await? { - return Ok(task_result); + return Ok(Outcome::Fatal(reason)); } } @@ -365,9 +374,7 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R for signature in publish_signatures { let record = generate_dkim_dns_record(&signature.object, &domain.name).await?; let dns_update::DnsRecord::TXT(txt_value) = &record.record else { - return Ok(TaskResult::permanent( - "DKIM record must be a TXT record".to_string(), - )); + return Ok(Outcome::fatal("DKIM record must be a TXT record")); }; let propagation_target = txt_value.clone(); let publish_result = updater @@ -378,11 +385,11 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R vec![record.record.clone()], ) .await; - let propagation_result = match &publish_result { + let propagation_result = match publish_result { Ok(_) => Ok(updater - .wait_for_txt_propagation(&record.name, origin, &propagation_target) + .check_txt_propagation(&record.name, origin, &propagation_target) .await), - Err(err) => Err(err.clone()), + Err(err) => Err(err), }; match propagation_result { Ok(true) => { @@ -401,8 +408,8 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R trc::event!( Dkim(DkimEvent::SignaturePublished), - Id = new_signature.selector().to_string(), - Details = domain.name.clone() + Name = CompactString::from(new_signature.selector()), + Domain = domain.name.clone() ); // Write key @@ -417,28 +424,23 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R { SignatureUpdate::Written => active_types.push(signature_type), SignatureUpdate::Conflict => {} - SignatureUpdate::Failed(task_result) => return Ok(task_result), + SignatureUpdate::Failed(reason) => return Ok(Outcome::Fatal(reason)), } - do_refresh = true; + *do_refresh = true; } Ok(false) => { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( &mut temporary_errors, - "DKIM record {} did not propagate, will retry.", - record.name + format_args!( + "DKIM record {} did not propagate yet, will retry.", + record.name + ), ); } Err(err) => { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( &mut temporary_errors, - "Failed to publish DKIM record {}: {err}.", - record.name + format_args!("Failed to publish DKIM record {}: {err}.", record.name), ); } } @@ -457,9 +459,9 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R DkimSignature::Dkim1Ed25519Sha256(sign) | DkimSignature::Dkim1RsaSha256(sign) => { sign.next_transition_at = None } - DkimSignature::Dkim2Ed25519Sha256(sign) | DkimSignature::Dkim2RsaSha256(sign) => { - sign.next_transition_at = None - } + DkimSignature::Dkim2Ed25519Sha256(sign) + | DkimSignature::Dkim2RsaSha256(sign) + | DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => sign.next_transition_at = None, } match update_signature( @@ -473,10 +475,10 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R { SignatureUpdate::Written => { active_types.push(signature_type); - do_refresh = true; + *do_refresh = true; } SignatureUpdate::Conflict => active_types.push(signature_type), - SignatureUpdate::Failed(task_result) => return Ok(task_result), + SignatureUpdate::Failed(reason) => return Ok(Outcome::Fatal(reason)), } } } @@ -503,12 +505,12 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R trc::event!( Dkim(DkimEvent::SignatureRetiring), - Id = new_signature.selector().to_string(), - Details = domain.name.clone() + Name = CompactString::from(new_signature.selector()), + Domain = domain.name.clone() ); // Write key - if let SignatureUpdate::Failed(task_result) = update_signature( + if let SignatureUpdate::Failed(reason) = update_signature( server, signature, new_signature, @@ -517,9 +519,9 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ) .await? { - return Ok(task_result); + return Ok(Outcome::Fatal(reason)); } - do_refresh = true; + *do_refresh = true; } // Retire signatures @@ -545,12 +547,12 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R trc::event!( Dkim(DkimEvent::SignatureRetired), - Id = new_signature.selector().to_string(), - Details = domain.name.clone() + Name = CompactString::from(new_signature.selector()), + Domain = domain.name.clone() ); // Write key - if let SignatureUpdate::Failed(task_result) = update_signature( + if let SignatureUpdate::Failed(reason) = update_signature( server, signature, new_signature, @@ -559,19 +561,15 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R ) .await? { - return Ok(task_result); + return Ok(Outcome::Fatal(reason)); } - do_refresh = true; + *do_refresh = true; } Err(err) => { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( &mut temporary_errors, - "Failed to remove DKIM record {}: {err}.", - record + format_args!("Failed to remove DKIM record {record}: {err}."), ); } } @@ -587,19 +585,24 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R .set_rrset(origin, &record, dns_update::DnsRecordType::TXT, Vec::new()) .await { + append_error( + &mut temporary_errors, + format_args!("Failed to remove DKIM record {record}: {err}."), + ); trc::event!( Dns(DnsEvent::RecordDeletionFailed), - Hostname = record.clone(), - Details = origin.clone(), + Hostname = record, + Domain = origin.clone(), Type = "TXT", - Reason = err.to_string(), + Reason = err.to_compact_string(), ); + continue; } trc::event!( Dkim(DkimEvent::SignatureDeleted), - Id = signature.object.selector().to_string(), - Details = domain.name.clone() + Name = CompactString::from(signature.object.selector()), + Domain = domain.name.clone() ); match server @@ -615,18 +618,17 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R { Ok(RegistryWriteResult::Success(_)) => {} Ok(err) => { - return Ok(TaskResult::permanent(format!( + return Ok(Outcome::fatal(format!( "Failed to delete DKIM signature for record {record}: {err}" ))); } Err(err) => { if err.is_assertion_failure() { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( - temporary_errors, - "Failed to delete DKIM signature for record {record} due to concurrent modification, will retry.", + append_error( + &mut temporary_errors, + format_args!( + "Failed to delete DKIM signature for record {record} due to concurrent modification, will retry." + ), ); } else { return Err(err); @@ -635,39 +637,24 @@ async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::R } } - if do_refresh - && let Err(err) = server - .invalidate_caches(CacheInvalidationBuilder::default().with_invalidation( - CacheInvalidation::DkimSignature(task.domain_id.document_id()), - )) - .await - { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to invalidate caches after DKIM management task") - ); - } - if !temporary_errors.is_empty() { - Ok(TaskResult::temporary(temporary_errors)) - } else { - let tasks = if let Some(next_transition) = next_transition { - vec![Task::DkimManagement(TaskDomainManagement { + Ok(Outcome::transient(temporary_errors)) + } else if let Some(next_transition) = next_transition { + Ok(Outcome::Done(vec![Task::DkimManagement( + TaskDomainManagement { domain_id: task.domain_id, status: TaskStatus::at(next_transition.timestamp()), - })] - } else { - vec![] - }; - - Ok(TaskResult::Success(tasks)) + }, + )])) + } else { + Ok(Outcome::done()) } } enum SignatureUpdate { Written, Conflict, - Failed(TaskResult), + Failed(String), } async fn update_signature( @@ -690,17 +677,16 @@ async fn update_signature( .await { Ok(RegistryWriteResult::Success(_)) => Ok(SignatureUpdate::Written), - Ok(err) => Ok(SignatureUpdate::Failed(TaskResult::permanent(format!( + Ok(err) => Ok(SignatureUpdate::Failed(format!( "Failed to write DKIM signature for record {name}: {err}" - )))), + ))), Err(err) => { if err.is_assertion_failure() { - if !temporary_errors.is_empty() { - temporary_errors.push_str("; "); - } - let _ = write!( + append_error( temporary_errors, - "Failed to write DKIM signature for record {name} due to concurrent modification, will retry.", + format_args!( + "Failed to write DKIM signature for record {name} due to concurrent modification, will retry." + ), ); Ok(SignatureUpdate::Conflict) } else { @@ -710,38 +696,3 @@ async fn update_signature( } } -#[cfg(feature = "test_mode")] -const TEST_RSA_KEY: &str = r#"-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAv9XYXG3uK95115mB4nJ37nGeNe2CrARm1agrbcnSk5oIaEfM -ZLUR/X8gPzoiNHZcfMZEVR6bAytxUhc5EvZIZrjSuEEeny+fFd/cTvcm3cOUUbIa -UmSACj0dL2/KwW0LyUaza9z9zor7I5XdIl1M53qVd5GI62XBB76FH+Q0bWPZNkT4 -NclzTLspD/MTpNCCPhySM4Kdg5CuDczTH4aNzyS0TqgXdtw6A4Sdsp97VXT9fkPW -9rso3lrkpsl/9EQ1mR/DWK6PBmRfIuSFuqnLKY6v/z2hXHxF7IoojfZLa2kZr9Ae -d4l9WheQOTA19k5r2BmlRw/W9CrgCBo0Sdj+KQIDAQABAoIBAFPChEi/OvnulReB -ECQWhOUYuNKlFKQU++2YEvZJ4+bMn5UgnE7wfJ1pj2Pr9xlfALz+OMHNrjMxGbaV -KzdrT2uCkYcf78XjnhuH9gKIiXDUv4L4N+P3u6w8yOx4bFgOS9IjS53yDOPM7SC5 -g6dIg5aigHaHlffqIuFFv4yQMI/+Ai+zBKxS7wRhxK/7nnAuo28fe5MEdp57ho9/ -AGlDNsdg9zCgjwhokwFE3+AaD+bkUFm4gQ1XjkUFrlmnQn8vDQ0i9toEWhCj+UPY -iOKL63MJnr90MXTXWLHoFj99wBp//mYygbF9Lj8fa28/oa8LWp3Jhb7QeMgH46iv -3aLHbTECgYEA5M2dAw+nyMw9vYlkMejhwObKYP8Mr/6zcGMLCalYvRJM5iUAM0JI -H6sM6pV9/nv167cbKocj3xYPdtE7FPOn4132MLM8Ne1f8nPE64Qrcbj5WBXvLnU8 -hpWbwe2Z8h7UUMKx6q4F1/TXYkc3ScxYwfjM4mP/pLsAOgVzRSEEgrUCgYEA1qNQ -xaQHNWZ1O8WuTnqWd5JSsic6iURAmUcLeFDZY2PWhVoaQ8L/xMQhDYs1FIbLWArW -4Qq3Ibu8AbSejAKuaJz7Uf26PX+PYVUwAOO0qamCJ8d/qd6So7qWMDyAY2yXI39Y -1nMqRjr7bkEsggAZao7BKqA7ZtmogjOusBT38iUCgYEA06agJ8TDoKvOMRZ26PRU -YO0dKLzGL8eclcoI29cbj0rud7aiiMg3j5PbTuUat95TjsjDCIQaWrM9etvxm2AJ -Xfn9Uu96MyhyKQWOk46f4YMKpMElkARDCPw8KRhx39dE77AqhLyWCz8iPndCXbH6 -KPTOEl4OjYOuof2Is9nnIkECgYBh948RdsnXhNlzm8nwhiGRmBbou+EK8D0v+O5y -Tyy6IcKzgSnFzgZh8EdJ4EUtBk1f9SqY8wQdgIvSl3daXorusuA/TzkngsaV3YUY -ktZOLlF7CKLrjOyPkMWmZKcROmpNyH1q/IvKHHfQnizLdXIkYd4nL5WNX0F7lE1i -j1+QhQKBgB2lviBK7rJFwlFYdQUP1NAN2dKxMZk8uJS8JglHrM0+8nRI83HbTdEQ -vB0ManEKBkbS4T5n+gRtdEqKSDmWDTXDlrBfcdCHNQLwYtBpOotCqQn/AmfjcPBl -byAbwh4+HiZ5JISoRZpiZqy67aJNVoXmdtb/E9mi7ozzytpxMNql ------END RSA PRIVATE KEY----- -"#; - -#[cfg(feature = "test_mode")] -const TEST_ED25519_KEY: &str = r#"-----BEGIN PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIAO3hAf144lTAVjTkht3ZwBTK0CMCCd1bI0alggneN3B ------END PRIVATE KEY----- -"#; diff --git a/crates/services/src/task_manager/dns.rs b/crates/services/src/task_manager/tasks/dns.rs similarity index 74% rename from crates/services/src/task_manager/dns.rs rename to crates/services/src/task_manager/tasks/dns.rs index ab650477..3f61e6a4 100644 --- a/crates/services/src/task_manager/dns.rs +++ b/crates/services/src/task_manager/tasks/dns.rs @@ -4,56 +4,53 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::TaskResult; +use crate::task_manager::{outcome::Outcome, tasks::append_error}; use common::Server; +use compact_str::{CompactString, format_compact}; use dns_update::{CAARecord, DnsRecord, DnsRecordType, Error as DnsUpdateError, KeyValue}; use registry::schema::structs::{ DnsManagement, Domain, Task, TaskDnsManagement, TaskDomainManagement, TaskStatus, }; -use std::fmt::Write; use store::ahash::AHashMap; use trc::DnsEvent; pub(crate) trait DnsManagementTask: Sync + Send { - fn dns_management(&self, task: &TaskDnsManagement) -> impl Future + Send; + fn dns_management(&self, task: &TaskDnsManagement) -> impl Future + Send; } impl DnsManagementTask for Server { - async fn dns_management(&self, task: &TaskDnsManagement) -> TaskResult { + async fn dns_management(&self, task: &TaskDnsManagement) -> Outcome { match dns_management(self, task).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let outcome = Outcome::error(&err); trc::error!( err.caused_by(trc::location!()) .details("Failed to run DNS management task") ); - result + outcome } } } } -async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Result { +async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Result { if task.update_records.is_empty() { - return Ok(TaskResult::permanent( - "No DNS records to update".to_string(), - )); + return Ok(Outcome::Obsolete("No DNS records to update")); } let Some(domain) = server.registry().object::(task.domain_id).await? else { - return Ok(TaskResult::permanent("Domain not found".to_string())); + return Ok(Outcome::Obsolete("Domain no longer exists")); }; let DnsManagement::Automatic(props) = &domain.dns_management else { - return Ok(TaskResult::permanent( - "Domain is not set to automatic DNS management".to_string(), + return Ok(Outcome::Obsolete( + "Domain no longer uses automatic DNS management", )); }; let dns_updater = match server.build_dns_updater(props.dns_server_id).await? { Ok(updater) => updater, Err(err) => { - return Ok(TaskResult::permanent(format!( - "Failed to build DNS updater: {}", - err + return Ok(Outcome::fatal(format!( + "Failed to build DNS updater: {err}" ))); } }; @@ -72,6 +69,7 @@ async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Resul } let mut errors = String::new(); + let mut is_fatal = false; for ((name, record_type), mut recs) in by_owner { let preserve_unrelated = match record_type { DnsRecordType::TXT => !is_owned_txt_name(&name), @@ -91,9 +89,9 @@ async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Resul trc::event!( Dns(DnsEvent::RecordLookupFailed), Hostname = name.clone(), - Details = origin.to_string(), + Domain = CompactString::from(origin), Type = record_type.as_str(), - Reason = format!( + Reason = format_compact!( "DNS provider cannot list RRSet, unrelated records at this name may be overwritten: {reason}" ), ); @@ -102,10 +100,18 @@ async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Resul trc::event!( Dns(DnsEvent::RecordLookupFailed), Hostname = name.clone(), - Details = origin.to_string(), + Domain = CompactString::from(origin), Type = record_type.as_str(), - Reason = format!("DNS provider failed to list RRSet: {err}"), + Reason = format_compact!("DNS provider failed to list RRSet: {err}"), ); + append_error( + &mut errors, + format_args!( + "Failed to list DNS RRSet for {name}/{}: {err}", + record_type.as_str() + ), + ); + continue; } } } @@ -114,35 +120,47 @@ async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Resul .set_rrset(origin, &name, record_type, recs) .await { - if !errors.is_empty() { - errors.push_str("; "); - } - let _ = write!( + is_fatal |= is_fatal_dns_error(&err); + append_error( &mut errors, - "Failed to set DNS RRSet for {}/{}: {}", - name, - record_type.as_str(), - err + format_args!( + "Failed to set DNS RRSet for {name}/{}: {err}", + record_type.as_str() + ), ); } } if errors.is_empty() { if task.on_success_renew_certificate { - Ok(TaskResult::Success(vec![Task::AcmeRenewal( + Ok(Outcome::Done(vec![Task::AcmeRenewal( TaskDomainManagement { domain_id: task.domain_id, status: TaskStatus::now(), }, )])) } else { - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } + } else if is_fatal { + Ok(Outcome::fatal(errors)) } else { - Ok(TaskResult::permanent(errors)) + Ok(Outcome::transient(errors)) } } +fn is_fatal_dns_error(err: &DnsUpdateError) -> bool { + matches!( + err, + DnsUpdateError::Parse(_) + | DnsUpdateError::Serialize(_) + | DnsUpdateError::Unsupported(_) + | DnsUpdateError::InvalidRegion + | DnsUpdateError::BadRequest + | DnsUpdateError::NotFound + ) +} + fn same_rrset_family(a: &DnsRecord, b: &DnsRecord) -> bool { match (a, b) { (DnsRecord::TXT(_), DnsRecord::TXT(_)) => same_txt_family(a, b), diff --git a/crates/services/src/task_manager/tasks/grantee_cleanup.rs b/crates/services/src/task_manager/tasks/grantee_cleanup.rs new file mode 100644 index 00000000..6fbcb484 --- /dev/null +++ b/crates/services/src/task_manager/tasks/grantee_cleanup.rs @@ -0,0 +1,208 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod cleanup; +mod stage; + +use crate::task_manager::outcome::Outcome; +use cleanup::{GranteeCleanup, Purge}; +use common::Server; +use email::mailbox::Mailbox; +use groupware::{calendar::Calendar, contact::AddressBook, file::FileNode}; +use registry::schema::structs::TaskShareRevokeCleanup; +use std::iter; +use store::{ + ValueKey, + query::acl::AclQuery, + rkyv::vec::ArchivedVec, + write::{Archive, ArchiveBytes, assert::AssertValue}, +}; +use trc::{AddContext, EventType, StoreEvent}; +use types::{acl::ArchivedAclGrant, collection::Collection, id::Id}; + +pub(crate) trait GranteeCleanupTask: Sync + Send { + fn remove_grantee_references( + &self, + grantee_id: u32, + member_group_ids: &[Id], + ) -> impl Future> + Send; + + fn share_revoke_cleanup( + &self, + task: &TaskShareRevokeCleanup, + ) -> impl Future + Send; + + fn revoke_cleanup( + &self, + task: &TaskShareRevokeCleanup, + ) -> impl Future> + Send; +} + +impl GranteeCleanupTask for Server { + async fn remove_grantee_references( + &self, + grantee_id: u32, + member_group_ids: &[Id], + ) -> trc::Result<()> { + let mut items = Vec::new(); + for account_id in iter::once(grantee_id).chain( + member_group_ids + .iter() + .map(|id| id.document_id()) + .filter(|group_id| *group_id != grantee_id), + ) { + items.extend( + self.store() + .acl_query(AclQuery::HasAccess { + grant_account_id: account_id, + }) + .await + .caused_by(trc::location!())? + .into_iter() + .filter(|item| item.to_account_id != grantee_id) + .map(|item| (item.to_account_id, item.to_collection, item.to_document_id)), + ); + } + if !member_group_ids.is_empty() { + items.sort_unstable_by_key(|(owner_id, collection, document_id)| { + (*owner_id, *collection as u8, *document_id) + }); + items.dedup(); + } + + for owner_items in items.chunk_by(|a, b| a.0 == b.0) { + let Some(owner_id) = owner_items.first().map(|item| item.0) else { + continue; + }; + + let result = match self.try_account(owner_id).await { + Ok(Some(owner)) => { + GranteeCleanup::new( + self, + owner_id, + owner.changed_by(), + Purge::Grantee(grantee_id), + ) + .run( + owner_items + .iter() + .map(|(_, collection, document_id)| (*collection, *document_id)), + ) + .await + } + Ok(None) => Ok(()), + Err(err) => Err(err), + }; + + if let Err(err) = result { + skip_unless_transient(err.account_id(owner_id).caused_by(trc::location!()))?; + } + } + + Ok(()) + } + + async fn share_revoke_cleanup(&self, task: &TaskShareRevokeCleanup) -> Outcome { + match self.revoke_cleanup(task).await { + Ok(()) => Outcome::done(), + Err(err) => { + let outcome = if is_transient(&err) { + Outcome::error(&err) + } else { + Outcome::fatal(err.to_string()) + }; + trc::error!( + err.account_id(task.account_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to clean up revoked share") + ); + outcome + } + } + } + + async fn revoke_cleanup(&self, task: &TaskShareRevokeCleanup) -> trc::Result<()> { + let owner_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); + let collection = task.object_type.collection(); + let Some(owner) = self + .try_account(owner_id) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + let Some(archive) = self + .store() + .get_value::>(ValueKey::archive( + owner_id, + collection, + document_id, + )) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + GranteeCleanup::new(self, owner_id, owner.changed_by(), Purge::LostAccess) + .with_guard( + collection, + document_id, + AssertValue::Archive(archive.version), + ) + .run(iter::once((collection, document_id))) + .await + .add_context(|err| err.account_id(owner_id).document_id(document_id)) + } +} + +pub(crate) fn shared_acls( + collection: Collection, + archive: &Archive, +) -> trc::Result>> { + Ok(match collection { + Collection::Mailbox => Some(&archive.unarchive::()?.acls), + Collection::Calendar => Some(&archive.unarchive::()?.acls), + Collection::AddressBook => Some(&archive.unarchive::()?.acls), + Collection::FileNode => Some(&archive.unarchive::()?.acls), + _ => None, + }) +} + +fn skip_unless_transient(err: trc::Error) -> trc::Result<()> { + if is_transient(&err) { + Err(err) + } else { + trc::error!(err.ctx(trc::Key::Reason, "Skipped during grantee reference cleanup")); + Ok(()) + } +} + +fn is_transient(err: &trc::Error) -> bool { + matches!( + err.event_type(), + EventType::Store( + StoreEvent::AssertValueFailed + | StoreEvent::FoundationdbError + | StoreEvent::MysqlError + | StoreEvent::PostgresqlError + | StoreEvent::RocksdbError + | StoreEvent::SqliteError + | StoreEvent::RedisError + | StoreEvent::S3Error + | StoreEvent::AzureError + | StoreEvent::FilesystemError + | StoreEvent::PoolError + | StoreEvent::DataWrite + | StoreEvent::DataIterate + | StoreEvent::BlobRead + | StoreEvent::BlobWrite + | StoreEvent::BlobDelete + | StoreEvent::SqlQuery + | StoreEvent::NotConfigured + ) + ) +} diff --git a/crates/services/src/task_manager/tasks/grantee_cleanup/cleanup.rs b/crates/services/src/task_manager/tasks/grantee_cleanup/cleanup.rs new file mode 100644 index 00000000..d6551fdf --- /dev/null +++ b/crates/services/src/task_manager/tasks/grantee_cleanup/cleanup.rs @@ -0,0 +1,398 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{is_transient, skip_unless_transient}; +use common::{ + GroupwareResources, Server, + auth::{AccessToken, ChangedBy}, + storage::dav::resource::NotificationRef, +}; +use groupware::cache::GroupwareCache; +use groupware::{calendar::CalendarEventContent, file::FileNode}; +use std::{mem, sync::Arc}; +use store::{ + ahash::AHashMap, + parking_lot::Mutex, + roaring::RoaringBitmap, + write::{BatchBuilder, assert::AssertValue}, +}; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + field::{CalendarEventField, Field}, +}; + +const MAX_WRITE_ATTEMPTS: usize = 3; + +pub(super) enum Purge { + Grantee(u32), + LostAccess, +} + +pub(super) struct GranteeCleanup<'x> { + pub(super) server: &'x Server, + pub(super) owner_id: u32, + pub(super) changed_by: ChangedBy, + pub(super) purge: Purge, + pub(super) calendars: Option>, + pub(super) files: Option>, + pub(super) guard: Option<(Collection, u32, AssertValue)>, + pub(super) viewers: Mutex>>, +} + +#[derive(Default)] +struct PendingWrites { + batch: BatchBuilder, + staged: Vec<(Collection, u32)>, +} + +impl<'x> GranteeCleanup<'x> { + pub(super) fn new( + server: &'x Server, + owner_id: u32, + changed_by: ChangedBy, + purge: Purge, + ) -> Self { + GranteeCleanup { + server, + owner_id, + changed_by, + purge, + calendars: None, + files: None, + guard: None, + viewers: Mutex::new(AHashMap::new()), + } + } + + pub(super) fn is_candidate(&self, account_id: u32) -> bool { + match self.purge { + Purge::Grantee(grantee_id) => grantee_id == account_id, + Purge::LostAccess => account_id != self.owner_id, + } + } + + pub(super) fn purges_grant(&self, account_id: u32) -> bool { + matches!(self.purge, Purge::Grantee(grantee_id) if grantee_id == account_id) + } + + pub(super) fn candidates(&self, account_ids: impl Iterator) -> Vec { + let mut candidates = account_ids + .filter(|account_id| self.is_candidate(*account_id)) + .collect::>(); + candidates.sort_unstable(); + candidates.dedup(); + candidates + } + + pub(super) async fn load_viewers(&self, account_ids: &[u32]) -> trc::Result<()> { + if matches!(self.purge, Purge::Grantee(_)) { + return Ok(()); + } + for &account_id in account_ids { + let is_loaded = self.viewers.lock().contains_key(&account_id); + if is_loaded { + continue; + } + let viewer = match self.server.try_account(account_id).await { + Ok(Some(account)) => Some(AccessToken::from_account_maybe_invalid(&account)), + Ok(None) => None, + Err(err) if is_transient(&err) => return Err(err), + Err(err) => { + trc::error!( + err.account_id(account_id) + .ctx(trc::Key::Reason, "Skipped during revoked share cleanup") + ); + None + } + }; + self.viewers.lock().insert(account_id, viewer); + } + Ok(()) + } + + pub(super) fn with_guard( + mut self, + collection: Collection, + document_id: u32, + archive: AssertValue, + ) -> Self { + self.guard = Some((collection, document_id, archive)); + self + } + + pub(super) async fn run( + mut self, + objects: impl Iterator + Clone, + ) -> trc::Result<()> { + let mut calendar_ids = RoaringBitmap::new(); + let mut file_ids = RoaringBitmap::new(); + for (collection, document_id) in objects.clone() { + match collection { + Collection::Calendar => { + calendar_ids.insert(document_id); + } + Collection::FileNode => { + file_ids.insert(document_id); + } + _ => {} + } + } + if !calendar_ids.is_empty() { + self.calendars = Some(self.resources(SyncCollection::Calendar).await?); + } + if !file_ids.is_empty() { + self.files = Some(self.resources(SyncCollection::FileNode).await?); + } + let mut pending = PendingWrites::default(); + if !calendar_ids.is_empty() { + self.clean_calendar_items(&mut pending, &calendar_ids) + .await?; + } + if !file_ids.is_empty() { + self.clean_file_nodes(&mut pending, &file_ids).await?; + } + for (collection, document_id) in + objects.filter(|(collection, _)| *collection != Collection::FileNode) + { + self.clean(&mut pending, collection, document_id).await?; + } + + self.flush(&mut pending).await + } + + async fn resources(&self, collection: SyncCollection) -> trc::Result> { + self.server + .fetch_groupware_resources_at(self.owner_id, self.owner_id, collection, u64::MAX) + .await + .caused_by(trc::location!()) + } + + async fn clean_calendar_items( + &self, + pending: &mut PendingWrites, + calendar_ids: &RoaringBitmap, + ) -> trc::Result<()> { + let event_ids = self + .calendars + .as_deref() + .map(|calendars| { + calendar_ids + .iter() + .flat_map(|calendar_id| calendars.children_ids(calendar_id)) + .collect::() + }) + .unwrap_or_default(); + let mut staged_ids = RoaringBitmap::new(); + if !event_ids.is_empty() { + self.server + .archives( + self.owner_id, + Collection::CalendarEvent, + CalendarEventField::Content.field(), + &event_ids, + |document_id, archive| { + if archive + .unarchive::() + .map_or(true, |content| { + content + .preferences + .iter() + .any(|pref| self.is_candidate(pref.account_id.to_native())) + }) + { + staged_ids.insert(document_id); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + for event_id in staged_ids { + self.clean(pending, Collection::CalendarEvent, event_id) + .await?; + } + + let notifications = self + .resources(SyncCollection::CalendarEventNotification) + .await?; + let notification_ids = notifications + .resources + .iter() + .filter(|resource| { + !resource.is_container() + && resource.notification().is_some_and(|notification| { + self.selects_notification(¬ification, calendar_ids) + }) + }) + .map(|resource| resource.document_id()) + .collect::(); + drop(notifications); + for notification_id in notification_ids { + self.clean( + pending, + Collection::CalendarEventNotification, + notification_id, + ) + .await?; + } + + Ok(()) + } + + fn selects_notification( + &self, + notification: &NotificationRef<'_>, + calendar_ids: &RoaringBitmap, + ) -> bool { + notification + .dismissed_by + .iter() + .any(|account_id| self.is_candidate(*account_id)) + && match self.purge { + Purge::Grantee(_) => true, + Purge::LostAccess => notification + .calendar_ids + .iter() + .any(|calendar_id| calendar_ids.contains(*calendar_id)), + } + } + + async fn clean_file_nodes( + &self, + pending: &mut PendingWrites, + root_ids: &RoaringBitmap, + ) -> trc::Result<()> { + let mut descendant_ids = RoaringBitmap::new(); + if let Some(files) = self.files.as_deref() { + for document_id in root_ids { + if !descendant_ids.contains(document_id) + && let Some(root) = files.any_resource_path_by_id(document_id) + { + descendant_ids.extend( + files + .subtree(root.path()) + .map(|resource| resource.document_id()), + ); + } + } + } + descendant_ids -= root_ids; + let mut staged_ids = RoaringBitmap::new(); + self.server + .archives( + self.owner_id, + Collection::FileNode, + Field::ARCHIVE, + &(&descendant_ids | root_ids), + |document_id, archive| { + if archive.unarchive::().map_or(true, |node| { + node.unsubscribed + .iter() + .any(|id| self.is_candidate(id.to_native())) + || node + .acls + .iter() + .any(|grant| self.purges_grant(u32::from(grant.account_id))) + }) { + staged_ids.insert(document_id); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + descendant_ids &= &staged_ids; + for document_id in descendant_ids.iter().chain(root_ids & &staged_ids) { + self.clean(pending, Collection::FileNode, document_id) + .await?; + } + + Ok(()) + } + + async fn clean( + &self, + pending: &mut PendingWrites, + collection: Collection, + document_id: u32, + ) -> trc::Result<()> { + match self + .stage(collection, document_id, &mut pending.batch) + .await + { + Ok(true) => { + pending.staged.push((collection, document_id)); + if pending.batch.is_large_batch() { + self.flush(pending).await?; + } + Ok(()) + } + Ok(false) => Ok(()), + Err(err) if is_transient(&err) => Err(err), + Err(err) => { + self.replay(pending).await?; + skip_unless_transient( + err.account_id(self.owner_id) + .document_id(document_id) + .caused_by(trc::location!()), + ) + } + } + } + + async fn flush(&self, pending: &mut PendingWrites) -> trc::Result<()> { + if pending.staged.is_empty() { + return Ok(()); + } + match self + .server + .commit_batch(mem::take(&mut pending.batch)) + .await + { + Ok(_) => { + pending.staged.clear(); + Ok(()) + } + Err(err) if err.is_assertion_failure() => self.replay(pending).await, + Err(err) => Err(err.caused_by(trc::location!())), + } + } + + async fn replay(&self, pending: &mut PendingWrites) -> trc::Result<()> { + pending.batch = BatchBuilder::new(); + for (collection, document_id) in mem::take(&mut pending.staged) { + self.clean_one(collection, document_id).await?; + } + Ok(()) + } + + async fn clean_one(&self, collection: Collection, document_id: u32) -> trc::Result<()> { + let mut attempt = 1; + loop { + let mut batch = BatchBuilder::new(); + let result = match self.stage(collection, document_id, &mut batch).await { + Ok(true) => self.server.commit_batch(batch).await.map(|_| ()), + Ok(false) => Ok(()), + Err(err) => Err(err), + }; + match result { + Ok(()) => return Ok(()), + Err(err) if err.is_assertion_failure() && attempt < MAX_WRITE_ATTEMPTS => { + attempt += 1; + } + Err(err) => { + return skip_unless_transient( + err.account_id(self.owner_id) + .document_id(document_id) + .caused_by(trc::location!()), + ); + } + } + } + } +} diff --git a/crates/services/src/task_manager/tasks/grantee_cleanup/stage.rs b/crates/services/src/task_manager/tasks/grantee_cleanup/stage.rs new file mode 100644 index 00000000..1ae998e7 --- /dev/null +++ b/crates/services/src/task_manager/tasks/grantee_cleanup/stage.rs @@ -0,0 +1,404 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::cleanup::{GranteeCleanup, Purge}; +use common::{ + ArchivedDavName, + auth::AccessToken, + sharing::EffectiveAcl, + storage::index::{CurrentObject, ObjectIndexBuilder, RevokeCleanup, SerializableObject}, +}; +use email::mailbox::Mailbox; +use groupware::{ + calendar::{ + Calendar, CalendarEvent, CalendarEventContent, CalendarEventNotification, EVENT_HAS_ALARMS, + alarm::AlarmTarget, + schedule::{AlarmTasks, EventAlarmScheduler}, + }, + contact::AddressBook, + file::FileNode, +}; +use store::{ + ValueKey, + rkyv::vec::ArchivedVec, + write::{Archive, ArchiveBytes, BatchBuilder, now}, +}; +use types::{ + acl::ArchivedAclGrant, + collection::Collection, + field::{CalendarEventField, Field}, +}; + +impl GranteeCleanup<'_> { + pub(super) async fn stage( + &self, + collection: Collection, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result { + match collection { + Collection::Mailbox => self.stage_mailbox(document_id, batch).await, + Collection::Calendar => self.stage_calendar(document_id, batch).await, + Collection::CalendarEvent => self.stage_event(document_id, batch).await, + Collection::CalendarEventNotification => { + self.stage_notification(document_id, batch).await + } + Collection::AddressBook => self.stage_address_book(document_id, batch).await, + Collection::FileNode => self.stage_file_node(document_id, batch).await, + _ => Ok(false), + } + } + + async fn stage_mailbox(&self, document_id: u32, batch: &mut BatchBuilder) -> trc::Result { + let Some(archive) = self.archive(Collection::Mailbox, document_id).await? else { + return Ok(false); + }; + let current = archive.to_unarchived::()?; + let candidates = self.candidates(current.inner.subscribers.iter().map(|id| id.to_native())); + let has_grants = self.has_purged_grants(¤t.inner.acls); + if candidates.is_empty() && !has_grants { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + !current.inner.acls.effective_acl(viewer).is_empty() + }) + }; + if !has_grants && !candidates.iter().any(|account_id| lost(*account_id)) { + return Ok(false); + } + + let mut mailbox = current.deserialize::()?; + mailbox + .acls + .retain(|grant| !self.purges_grant(grant.account_id)); + mailbox.subscribers.retain(|id| !lost(*id)); + + self.write(Collection::Mailbox, document_id, current, mailbox, batch) + } + + async fn stage_calendar( + &self, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result { + let Some(archive) = self.archive(Collection::Calendar, document_id).await? else { + return Ok(false); + }; + let current = archive.to_unarchived::()?; + let candidates = self.candidates( + current + .inner + .preferences + .iter() + .map(|pref| pref.account_id.to_native()), + ); + let has_grants = self.has_purged_grants(¤t.inner.acls); + if candidates.is_empty() && !has_grants { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + !current.inner.acls.effective_acl(viewer).is_empty() + }) + }; + if !has_grants && !candidates.iter().any(|account_id| lost(*account_id)) { + return Ok(false); + } + + let mut calendar = current.deserialize::()?; + calendar + .acls + .retain(|grant| !self.purges_grant(grant.account_id)); + calendar.preferences.retain(|pref| !lost(pref.account_id)); + calendar.modified = now() as i64; + + self.write(Collection::Calendar, document_id, current, calendar, batch) + } + + async fn stage_event(&self, document_id: u32, batch: &mut BatchBuilder) -> trc::Result { + let Some(content) = self + .server + .store() + .get_value::>(ValueKey::property( + self.owner_id, + Collection::CalendarEvent, + document_id, + CalendarEventField::Content, + )) + .await? + else { + return Ok(false); + }; + let content = content.to_unarchived::()?; + let candidates = self.candidates( + content + .inner + .preferences + .iter() + .map(|pref| pref.account_id.to_native()), + ); + if candidates.is_empty() { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let Some(meta) = self.archive(Collection::CalendarEvent, document_id).await? else { + return Ok(false); + }; + let event = meta.to_unarchived::()?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + self.has_calendar_access( + viewer, + event.inner.names.iter().map(ArchivedDavName::parent_id), + ) + }) + }; + let lost_ids = candidates + .into_iter() + .filter(|account_id| lost(*account_id)) + .collect::>(); + if lost_ids.is_empty() { + return Ok(false); + } + + self.guard(batch); + batch + .with_account_id(self.owner_id) + .with_collection(Collection::CalendarEvent) + .with_document(document_id) + .assert_value(CalendarEventField::Content.field(), &content); + if event.inner.flags.to_native() & EVENT_HAS_ALARMS != 0 { + for &account_id in &lost_ids { + self.server + .clear_event_alarms( + self.owner_id, + document_id, + [AlarmTarget::Sharee(account_id)], + &mut AlarmTasks::None, + batch, + ) + .await?; + } + } + + let mut new_content = content.deserialize::()?; + new_content + .preferences + .retain(|pref| !lost_ids.contains(&pref.account_id)); + event.deserialize::()?.update_full( + new_content, + self.changed_by, + event, + content.inner, + self.owner_id, + document_id, + None, + batch, + )?; + Ok(true) + } + + async fn stage_notification( + &self, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result { + let Some(archive) = self + .archive(Collection::CalendarEventNotification, document_id) + .await? + else { + return Ok(false); + }; + let current = archive.to_unarchived::()?; + let candidates = self.candidates(current.inner.dismissed_ids()); + if candidates.is_empty() { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + self.has_calendar_access( + viewer, + current.inner.calendar_ids.iter().map(|id| id.to_native()), + ) + }) + }; + if !candidates.iter().any(|account_id| lost(*account_id)) { + return Ok(false); + } + + let mut notification = current.deserialize::()?; + notification.dismissed_by.retain(|id| !lost(*id)); + self.guard(batch); + + notification.update_meta(self.changed_by, current, self.owner_id, document_id, batch)?; + Ok(true) + } + + async fn stage_address_book( + &self, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result { + let Some(archive) = self.archive(Collection::AddressBook, document_id).await? else { + return Ok(false); + }; + let current = archive.to_unarchived::()?; + let candidates = self.candidates( + current + .inner + .subscribers + .iter() + .map(|id| id.to_native()) + .chain( + current + .inner + .preferences + .iter() + .map(|pref| pref.account_id.to_native()), + ), + ); + let has_grants = self.has_purged_grants(¤t.inner.acls); + if candidates.is_empty() && !has_grants { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + !current.inner.acls.effective_acl(viewer).is_empty() + }) + }; + if !has_grants && !candidates.iter().any(|account_id| lost(*account_id)) { + return Ok(false); + } + + let mut book = current.deserialize::()?; + book.acls + .retain(|grant| !self.purges_grant(grant.account_id)); + book.subscribers.retain(|id| !lost(*id)); + book.preferences.retain(|pref| !lost(pref.account_id)); + book.modified = now() as i64; + + self.write(Collection::AddressBook, document_id, current, book, batch) + } + + async fn stage_file_node( + &self, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result { + let Some(archive) = self.archive(Collection::FileNode, document_id).await? else { + return Ok(false); + }; + let current = archive.to_unarchived::()?; + let candidates = + self.candidates(current.inner.unsubscribed.iter().map(|id| id.to_native())); + let has_grants = self.has_purged_grants(¤t.inner.acls); + if candidates.is_empty() && !has_grants { + return Ok(false); + } + self.load_viewers(&candidates).await?; + let lost = |account_id| { + self.lost_access(account_id, |viewer| { + self.files + .as_deref() + .is_none_or(|files| !files.file_acl(viewer, document_id).is_empty()) + }) + }; + if !has_grants && !candidates.iter().any(|account_id| lost(*account_id)) { + return Ok(false); + } + + let mut node = current.deserialize::()?; + node.acls + .retain(|grant| !self.purges_grant(grant.account_id)); + node.unsubscribed.retain(|id| !lost(*id)); + node.stamp_update(false); + + self.write(Collection::FileNode, document_id, current, node, batch) + } + + fn has_purged_grants(&self, acls: &ArchivedVec) -> bool { + acls.iter() + .any(|grant| self.purges_grant(u32::from(grant.account_id))) + } + + fn lost_access(&self, account_id: u32, has_access: impl Fn(&AccessToken) -> bool) -> bool { + match self.purge { + Purge::Grantee(grantee_id) => account_id == grantee_id, + Purge::LostAccess => { + account_id != self.owner_id + && self.viewers.lock().get(&account_id).is_some_and(|viewer| { + viewer.as_ref().is_some_and(|viewer| { + !viewer.is_member(self.owner_id) && !has_access(viewer) + }) + }) + } + } + } + + fn has_calendar_access( + &self, + viewer: &AccessToken, + mut calendar_ids: impl Iterator, + ) -> bool { + self.calendars.as_deref().is_none_or(|calendars| { + calendar_ids.any(|calendar_id| !calendars.container_acl(viewer, calendar_id).is_empty()) + }) + } + + fn write( + &self, + collection: Collection, + document_id: u32, + current: C, + changes: N, + batch: &mut BatchBuilder, + ) -> trc::Result { + self.guard(batch); + batch + .with_account_id(self.owner_id) + .with_collection(collection) + .with_document(document_id) + .custom( + ObjectIndexBuilder::new() + .with_current(current) + .with_changes(changes) + .with_changed_by(self.changed_by) + .with_revoke_cleanup(RevokeCleanup::Inline), + )? + .commit_point(); + Ok(true) + } + + fn guard(&self, batch: &mut BatchBuilder) { + if let Some((collection, document_id, archive)) = self.guard { + batch + .with_account_id(self.owner_id) + .with_collection(collection) + .with_document(document_id) + .assert_value(Field::ARCHIVE, archive); + } + } + + async fn archive( + &self, + collection: Collection, + document_id: u32, + ) -> trc::Result>> { + self.server + .store() + .get_value::>(ValueKey::archive( + self.owner_id, + collection, + document_id, + )) + .await + } +} diff --git a/crates/services/src/task_manager/tasks/imip.rs b/crates/services/src/task_manager/tasks/imip.rs new file mode 100644 index 00000000..ccfb9d61 --- /dev/null +++ b/crates/services/src/task_manager/tasks/imip.rs @@ -0,0 +1,734 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::{ + outcome::{Failure, Outcome, RetryClass}, + tasks::{SmtpRejection, append_error}, +}; +use calcard::icalendar::{ICalendarParticipationStatus, ICalendarProperty}; +use common::{ + DEFAULT_LOGO_BASE64, Server, + auth::AccountInfo, + config::groupware::CalendarTemplateVariable, + i18n::Locale, + network::{ServerInstance, stream::NullIo}, +}; +use groupware::{ + calendar::itip::ItipIngest, + scheduling::{ + ItipSummary, ItipValue, + format::{DateStyle, TextFormatter, hyperlink}, + }, +}; +use mail_builder::{ + MessageBuilder, + headers::{HeaderType, content_type::ContentType}, + mime::{BodyPart, MimePart}, +}; +use mail_parser::decoders::html::html_to_text; +use registry::{ + schema::structs::{Task, TaskCalendarItipContents, TaskCalendarItipMessage}, + types::{EnumImpl, list::List, map::Map}, +}; +use smtp::{ + core::{Session, SessionData}, + inbound::data::MessageOrigin, +}; +use smtp_proto::{MailFrom, RcptTo}; +use std::{sync::Arc, time::Duration}; +use store::{ahash::AHashMap, write::now}; +use trc::AddContext; +use utils::template::{Template, Variable, Variables}; + +pub(crate) trait SendImipTask: Sync + Send { + fn send_imip( + &self, + task: &TaskCalendarItipMessage, + server_instance: Arc, + ) -> impl Future + Send; +} + +impl SendImipTask for Server { + async fn send_imip( + &self, + task: &TaskCalendarItipMessage, + server_instance: Arc, + ) -> Outcome { + match send_imip(self, task, server_instance).await { + Ok(outcome) => outcome, + Err(err) => { + let result = Outcome::error(&err); + trc::error!( + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to send iMIP message") + ); + result + } + } + } +} + +async fn send_imip( + server: &Server, + imip: &TaskCalendarItipMessage, + server_instance: Arc, +) -> trc::Result { + // Obtain iMIP payload + let account_id = imip.account_id.document_id(); + let document_id = imip.document_id.document_id(); + + let sender_domain = imip + .messages + .iter() + .next() + .and_then(|msg| msg.from.rsplit('@').next()) + .unwrap_or("localhost"); + + // Obtain logo image + let logo = match server.logo_resource(sender_domain).await { + Ok(logo) => logo, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to fetch logo image") + ); + None + } + }; + let logo_cid = format!("logo.{}@{sender_domain}", now()); + let logo = if let Some(logo) = &logo { + MimePart::new( + ContentType::new(logo.content_type.as_ref()), + BodyPart::Binary(logo.contents[..].into()), + ) + } else { + MimePart::new( + ContentType::new("image/png"), + BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), + ) + .transfer_encoding("base64") + } + .inline() + .cid(&logo_cid); + + let account_info = server + .account_info(account_id) + .await + .caused_by(trc::location!())?; + let formatter = TextFormatter::new(account_info.locale().as_str())?; + let mut pending = List::default(); + let mut errors = String::new(); + + for itip_message in imip.messages.iter() { + let Ok(summary) = serde_json::from_str::(&itip_message.summary) else { + return Ok(Outcome::fatal("Failed to parse iMIP message summary.")); + }; + let mut retry_to = Vec::new(); + + let organizer_info = match server + .account_id_from_email(itip_message.from.as_str(), true) + .await + { + Ok(Some(sender_id)) if sender_id != account_id => { + match server.account_info(sender_id).await { + Ok(info) => Some(info), + Err(err) => { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to load organizer account for iMIP sender") + ); + None + } + } + } + Ok(_) => None, + Err(err) => { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to resolve organizer account for iMIP sender") + ); + None + } + }; + let sender_info = organizer_info.as_ref().unwrap_or(&account_info); + let mut template = ItipTemplate::new( + server, + &formatter, + itip_message.from.as_str(), + &summary, + &logo_cid, + ); + let mut shared_body = None; + + for recipient in itip_message.to.iter() { + // Build template + let personal_body; + let body = if template + .set_recipient( + server, + &account_info, + account_id, + document_id, + recipient.as_str(), + ) + .await + { + personal_body = template.render(); + &personal_body + } else { + &*shared_body.get_or_insert_with(|| template.render()) + }; + + // Build message + let message = MessageBuilder::new() + .from(( + sender_info.description().unwrap_or(sender_info.name()), + itip_message.from.as_str(), + )) + .to(recipient.as_str()) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .header( + "Reply-To", + HeaderType::Text(itip_message.from.as_str().into()), + ) + .message_id(server.core.network.message_id()) + .subject(template.subject.as_str()) + .body(MimePart::new( + ContentType::new("multipart/mixed"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("multipart/related"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("multipart/alternative"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("text/plain"), + BodyPart::Text(body.text.as_str().into()), + ), + MimePart::new( + ContentType::new("text/html"), + BodyPart::Text(body.html.as_str().into()), + ), + MimePart::new( + ContentType::new("text/calendar") + .attribute("method", summary.method()) + .attribute("charset", "utf-8"), + BodyPart::Text( + itip_message.i_calendar_data.as_str().into(), + ), + ), + ]), + ), + logo.clone(), + ]), + ), + MimePart::new( + ContentType::new("application/ics").attribute("name", "event.ics"), + BodyPart::Text(itip_message.i_calendar_data.as_str().into()), + ) + .attachment("event.ics"), + ]), + )) + .write_to_vec() + .unwrap_or_default(); + + // Send message + let server_ = server.clone(); + let server_instance = server_instance.clone(); + let sender_info = sender_info.clone(); + let from = itip_message.from.to_string(); + let to = recipient.to_string(); + let result = tokio::spawn(async move { + let mut session = Session::::local( + server_, + server_instance, + SessionData::local(sender_info, None, vec![], vec![], 0), + ); + + // MAIL FROM + let _ = session + .handle_mail_from(MailFrom { + address: from.as_str().into(), + ..Default::default() + }) + .await; + if let Some(error) = session.has_failed() { + let rejection = SmtpRejection::new("MAIL-FROM", &error); + trc::event!( + Calendar(trc::CalendarEvent::ItipMessageError), + AccountId = account_id, + DocumentId = document_id, + From = from, + To = to, + Reason = rejection.reason.clone(), + ); + return Err(rejection); + } + + // RCPT TO + session.params.rcpt_errors_wait = Duration::from_secs(0); + let _ = session + .handle_rcpt_to(RcptTo { + address: to.as_str().into(), + ..Default::default() + }) + .await; + if let Some(error) = session.has_failed() { + let rejection = SmtpRejection::new("RCPT-TO", &error); + trc::event!( + Calendar(trc::CalendarEvent::ItipMessageError), + AccountId = account_id, + DocumentId = document_id, + From = from, + To = to, + Reason = rejection.reason.clone(), + ); + return Err(rejection); + } + + // DATA + session.data.message = message; + let response = session.queue_message(MessageOrigin::Internal).await; + if let smtp::core::State::Accepted(queue_id) = session.state { + trc::event!( + Calendar(trc::CalendarEvent::ItipMessageSent), + From = from, + To = to, + AccountId = account_id, + DocumentId = document_id, + QueueId = queue_id, + ); + Ok(()) + } else { + let rejection = SmtpRejection::new( + "DATA", + std::str::from_utf8(&response).unwrap_or_default(), + ); + trc::event!( + Calendar(trc::CalendarEvent::ItipMessageError), + From = from, + To = to, + AccountId = account_id, + DocumentId = document_id, + Reason = rejection.reason.clone(), + ); + Err(rejection) + } + }) + .await + .map_err(|_| { + trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError)) + .caused_by(trc::location!()) + })?; + + if let Err(rejection) = result + && rejection.is_temporary + { + append_error( + &mut errors, + format_args!("{recipient}: {}", rejection.reason), + ); + retry_to.push(recipient.clone()); + } + } + + if !retry_to.is_empty() { + pending.push(TaskCalendarItipContents { + from: itip_message.from.clone(), + to: Map::new(retry_to), + is_from_organizer: itip_message.is_from_organizer, + i_calendar_data: itip_message.i_calendar_data.clone(), + summary: itip_message.summary.clone(), + }); + } + } + + if pending.is_empty() { + Ok(Outcome::done()) + } else { + Ok(Outcome::Partial { + remaining: Box::new(Task::CalendarItipMessage(TaskCalendarItipMessage { + messages: pending, + account_id: imip.account_id, + document_id: imip.document_id, + status: imip.status.clone(), + })), + failure: Failure::new(RetryClass::Transient, errors), + }) + } +} + +pub struct Details { + pub subject: String, + pub body: String, +} + +pub struct RenderedBody { + pub html: String, + pub text: String, +} + +pub struct ItipTemplate<'x> { + template: &'x Template, + locale: &'static Locale, + variables: Variables, + has_rsvp: bool, + pub subject: String, +} + +#[allow(clippy::too_many_arguments)] +pub async fn build_itip_template( + server: &Server, + account_info: &AccountInfo, + account_id: u32, + document_id: u32, + from: &str, + to: &str, + summary: &ItipSummary, + logo_cid: &str, +) -> trc::Result
{ + let formatter = TextFormatter::new(account_info.locale().as_str())?; + let mut template = ItipTemplate::new(server, &formatter, from, summary, logo_cid); + template + .set_recipient(server, account_info, account_id, document_id, to) + .await; + Ok(Details { + body: template.template.eval(&template.variables), + subject: template.subject, + }) +} + +impl<'x> ItipTemplate<'x> { + pub fn new( + server: &'x Server, + formatter: &TextFormatter, + from: &str, + summary: &ItipSummary, + logo_cid: &str, + ) -> Self { + #[cfg(not(feature = "enterprise"))] + let template = &server.core.groupware.itip_template; + let locale = formatter.locale; + + let mut variables = Variables::new(); + let mut subject; + let (fields, old_fields) = match summary { + ItipSummary::Invite(fields) => { + subject = format!("{}: ", locale.calendar_invitation); + + (fields, None) + } + ItipSummary::Update { + current, previous, .. + } => { + subject = format!("{}: ", locale.calendar_updated_invitation); + variables.insert_single( + CalendarTemplateVariable::Header, + locale.calendar_event_updated.to_string(), + ); + variables.insert_single(CalendarTemplateVariable::Color, "info".to_string()); + (current, Some(previous)) + } + ItipSummary::Cancel(fields) => { + subject = format!("{}: ", locale.calendar_cancelled); + variables.insert_single( + CalendarTemplateVariable::Header, + locale.calendar_event_cancelled.to_string(), + ); + variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); + (fields, None) + } + ItipSummary::Rsvp { part_stat, current } => { + let (color, value) = match part_stat { + ICalendarParticipationStatus::Accepted => { + subject = format!("{}: ", locale.calendar_accepted); + + ( + "info", + locale.calendar_participant_accepted.replace("$name", from), + ) + } + ICalendarParticipationStatus::Declined => { + subject = format!("{}: ", locale.calendar_declined); + ( + "danger", + locale.calendar_participant_declined.replace("$name", from), + ) + } + ICalendarParticipationStatus::Tentative => { + subject = format!("{}: ", locale.calendar_tentative); + ( + "warning", + locale.calendar_participant_tentative.replace("$name", from), + ) + } + ICalendarParticipationStatus::Delegated => { + subject = format!("{}: ", locale.calendar_delegated); + ( + "warning", + locale.calendar_participant_delegated.replace("$name", from), + ) + } + _ => { + subject = format!("{}: ", locale.calendar_reply); + ( + "info", + locale.calendar_participant_reply.replace("$name", from), + ) + } + }; + + variables.insert_single(CalendarTemplateVariable::Header, value); + variables.insert_single(CalendarTemplateVariable::Color, color.to_string()); + + (current, None) + } + }; + + let mut when_detail: Option<(usize, &ItipValue)> = None; + let mut details: Vec> = Vec::with_capacity(4); + for field in [ + ICalendarProperty::Summary, + ICalendarProperty::Description, + ICalendarProperty::Dtstart, + ICalendarProperty::Rrule, + ICalendarProperty::Location, + ICalendarProperty::Conference, + ] { + let mut old_entries = old_fields.into_iter().flatten().filter(|e| e.name == field); + + for entry in fields.iter().filter(|e| e.name == field) { + let field_name = match &field { + ICalendarProperty::Summary => locale.calendar_summary, + ICalendarProperty::Description => locale.calendar_description, + ICalendarProperty::Dtstart | ICalendarProperty::Rrule => locale.calendar_when, + ICalendarProperty::Location => locale.calendar_location, + ICalendarProperty::Conference => locale.calendar_conference, + _ => continue, + }; + let value = formatter.field_to_string(&entry.value, DateStyle::Long); + + let old_entry = old_entries.next(); + + match &field { + ICalendarProperty::Summary => { + subject.push_str(&value); + } + ICalendarProperty::Dtstart if when_detail.is_none() => { + subject.push_str(" @ "); + subject.push_str(&value); + } + ICalendarProperty::Rrule if when_detail.is_none() => { + subject.push_str(" @ "); + subject.push_str(&value); + } + _ => (), + } + + if let ICalendarProperty::Summary | ICalendarProperty::Description = &field { + let variable = if matches!(field, ICalendarProperty::Summary) { + CalendarTemplateVariable::EventTitle + } else { + CalendarTemplateVariable::EventDescription + }; + + if old_entry.is_none() { + variables.insert_single(variable, value); + continue; + } + variables.insert_single(variable, value.clone()); + } + + if matches!(field, ICalendarProperty::Rrule) + && let Some((index, start_value)) = when_detail + && let Some(detail) = details.get_mut(index) + { + if let Some(when_value) = detail.get_mut(&CalendarTemplateVariable::Value) { + when_value.push_str(", "); + when_value.push_str(&value); + } + + if let Some(old_entry) = old_entry { + detail.insert( + CalendarTemplateVariable::Changed, + locale.calendar_changed.to_string(), + ); + let old_value = detail + .entry(CalendarTemplateVariable::OldValue) + .or_insert_with(|| { + formatter.field_to_string(start_value, DateStyle::Short) + }); + old_value.push_str(", "); + old_value.push_str( + &formatter.field_to_string(&old_entry.value, DateStyle::Short), + ); + } + + continue; + } + + let mut detail = AHashMap::with_capacity(4); + detail.insert(CalendarTemplateVariable::Key, field_name.to_string()); + if matches!(field, ICalendarProperty::Conference) + && let Some(link) = hyperlink(&value) + { + detail.insert(CalendarTemplateVariable::Link, link.to_string()); + } + detail.insert(CalendarTemplateVariable::Value, value); + if let Some(old_entry) = old_entry { + detail.insert( + CalendarTemplateVariable::Changed, + locale.calendar_changed.to_string(), + ); + detail.insert( + CalendarTemplateVariable::OldValue, + formatter.field_to_string(&old_entry.value, DateStyle::Short), + ); + } + if matches!(field, ICalendarProperty::Dtstart) && when_detail.is_none() { + when_detail = Some((details.len(), &entry.value)); + } + details.push(detail); + } + } + if !details.is_empty() { + variables.items.insert( + CalendarTemplateVariable::EventDetails, + Variable::Block(details), + ); + } + variables.insert_single(CalendarTemplateVariable::PageTitle, subject.clone()); + variables.insert_single(CalendarTemplateVariable::Lang, locale.name.to_string()); + variables.insert_single(CalendarTemplateVariable::Dir, locale.direction.to_string()); + variables.insert_single(CalendarTemplateVariable::LogoCid, format!("cid:{logo_cid}")); + + if let Some(guests) = fields + .iter() + .find(|e| e.name == ICalendarProperty::Attendee) + && let ItipValue::Participants(guests) = &guests.value + { + variables.insert_single( + CalendarTemplateVariable::AttendeesTitle, + locale.calendar_attendees.to_string(), + ); + variables.insert_block( + CalendarTemplateVariable::Attendees, + guests.iter().map(|guest| { + [ + ( + CalendarTemplateVariable::Key, + if guest.is_organizer { + if let Some(name) = guest.name.as_ref() { + format!("{name} - {}", locale.calendar_organizer) + } else { + locale.calendar_organizer.to_string() + } + } else { + guest.name.as_deref().unwrap_or_default().to_string() + }, + ), + (CalendarTemplateVariable::Value, guest.email.to_string()), + ] + }), + ); + } + + // Add footer + variables.insert_block( + CalendarTemplateVariable::Footer, + [ + [( + CalendarTemplateVariable::Key, + locale.calendar_imip_footer_1.to_string(), + )], + [( + CalendarTemplateVariable::Key, + locale.calendar_imip_footer_2.to_string(), + )], + ], + ); + + ItipTemplate { + template, + locale, + variables, + has_rsvp: matches!(summary, ItipSummary::Invite(_) | ItipSummary::Update { .. }), + subject, + } + } + + pub async fn set_recipient( + &mut self, + server: &Server, + account_info: &AccountInfo, + account_id: u32, + document_id: u32, + to: &str, + ) -> bool { + let locale = self.locale; + let variables = &mut self.variables; + variables.items.remove(&CalendarTemplateVariable::Rsvp); + variables.items.remove(&CalendarTemplateVariable::Actions); + + // Add RSVP buttons + if self.has_rsvp + && let Some(rsvp_url) = server + .http_rsvp_url(account_id, account_info.name(), document_id, to) + .await + { + variables.insert_single( + CalendarTemplateVariable::Rsvp, + locale.calendar_reply_as.replace("$name", to), + ); + variables.insert_block( + CalendarTemplateVariable::Actions, + [ + ( + ICalendarParticipationStatus::Accepted, + locale.calendar_yes.to_string(), + "info", + ), + ( + ICalendarParticipationStatus::Declined, + locale.calendar_no.to_string(), + "danger", + ), + ( + ICalendarParticipationStatus::Tentative, + locale.calendar_maybe.to_string(), + "warning", + ), + ] + .into_iter() + .map(|(status, title, color)| { + [ + (CalendarTemplateVariable::ActionName, title.to_string()), + (CalendarTemplateVariable::ActionUrl, rsvp_url.url(&status)), + (CalendarTemplateVariable::Color, color.to_string()), + ] + }), + ); + true + } else { + false + } + } + + pub fn render(&self) -> RenderedBody { + let html = self.template.eval(&self.variables); + RenderedBody { + text: html_to_text(&html), + html, + } + } +} diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/tasks/maintenance.rs similarity index 66% rename from crates/services/src/task_manager/maintenance.rs rename to crates/services/src/task_manager/tasks/maintenance.rs index 3c5ec410..79466914 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/tasks/maintenance.rs @@ -4,22 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - -use crate::task_manager::{ - TaskResult, - index::{reindex_account, reindex_telemetry}, +use crate::{ + index_queue::reindex::reindex_account, + task_manager::{outcome::Outcome, tasks::rebuild_shares::RebuildSharesTask}, }; use common::{ KV_ACME, KV_GREYLIST, KV_LOCK_DAV, KV_LOCK_QUEUE_MESSAGE, KV_LOCK_TASK, KV_OAUTH, - KV_QUOTA_BLOB, KV_RATE_LIMIT_AUTH, KV_RATE_LIMIT_CONTACT, KV_RATE_LIMIT_HTTP_ANONYMOUS, - KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_IMAP, KV_RATE_LIMIT_LOITER, KV_RATE_LIMIT_RCPT, - KV_RATE_LIMIT_SCAN, KV_RATE_LIMIT_SMTP, KV_SIEVE_ID, Server, + KV_OAUTH_CIMD, KV_QUOTA_BLOB, KV_RATE_LIMIT_AUTH, KV_RATE_LIMIT_CIMD, KV_RATE_LIMIT_CONTACT, + KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_IMAP, + KV_RATE_LIMIT_LOITER, KV_RATE_LIMIT_RCPT, KV_RATE_LIMIT_SCAN, KV_RATE_LIMIT_SMTP, Server, storage::index::ObjectIndexBuilder, }; use email::{ cache::MessageCacheFetch, - message::{delete::EmailDeletion, ingest::EmailIngest, metadata::MessageData}, + message::{ + delete::EmailDeletion, + messagedata::{EmailMessageData, MessageData, PendingMessageData}, + }, sieve::SieveScript, }; use groupware::{ @@ -29,26 +30,25 @@ use groupware::{ }; use registry::{ schema::{ - enums::{TaskAccountMaintenanceType, TaskStoreMaintenanceType, TaskTenantMaintenanceType}, + enums::{StorageQuota, TaskAccountMaintenanceType, TaskStoreMaintenanceType}, prelude::{Object, ObjectInner, ObjectType, Property}, - structs::{ - Task, TaskAccountMaintenance, TaskStatus, TaskStoreMaintenance, TaskTenantMaintenance, - }, + structs::{Task, TaskAccountMaintenance, TaskStatus, TaskStoreMaintenance}, }, types::EnumImpl, }; use smtp::reporting::index::ExternalReportIndex; +use std::time::Instant; use store::{ - Serialize, ValueKey, + ValueKey, rand::{self}, registry::{RegistryFilter, RegistryQuery}, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, RegistryClass, ValueClass, now}, + write::{Archive, ArchiveBytes, BatchBuilder, RegistryClass, ValueClass, now}, }; use trc::{AddContext, StoreEvent}; use types::{ collection::Collection, - field::{EmailField, MailboxField}, + field::{EmailField, Field, MailboxField}, id::Id, }; @@ -56,34 +56,30 @@ pub(crate) trait MaintenanceTask: Sync + Send { fn store_maintenance( &self, task: &TaskStoreMaintenance, - ) -> impl Future + Send; + ) -> impl Future + Send; fn account_maintenance( &self, task: &TaskAccountMaintenance, - ) -> impl Future + Send; - fn tenant_maintenance( - &self, - task: &TaskTenantMaintenance, - ) -> impl Future + Send; + ) -> impl Future + Send; } impl MaintenanceTask for Server { - async fn store_maintenance(&self, task: &TaskStoreMaintenance) -> TaskResult { + async fn store_maintenance(&self, task: &TaskStoreMaintenance) -> Outcome { match store_maintenance(self, task).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let result = Outcome::error(&err); trc::error!(err.details("Failed to perform store maintenance task")); result } } } - async fn account_maintenance(&self, task: &TaskAccountMaintenance) -> TaskResult { + async fn account_maintenance(&self, task: &TaskAccountMaintenance) -> Outcome { match account_maintenance(self, task).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let result = Outcome::error(&err); trc::error!( err.account_id(task.account_id.document_id()) .details("Failed to perform account maintenance task") @@ -93,22 +89,9 @@ impl MaintenanceTask for Server { } } - async fn tenant_maintenance(&self, task: &TaskTenantMaintenance) -> TaskResult { - match tenant_maintenance(self, task).await { - Ok(result) => result, - Err(err) => { - let result = TaskResult::temporary(err.to_string()); - trc::error!(err.details("Failed to perform tenant maintenance task")); - result - } - } - } } -async fn store_maintenance( - server: &Server, - task: &TaskStoreMaintenance, -) -> trc::Result { +async fn store_maintenance(server: &Server, task: &TaskStoreMaintenance) -> trc::Result { match task.maintenance_type { TaskStoreMaintenanceType::ReindexAccounts | TaskStoreMaintenanceType::PurgeAccounts @@ -128,10 +111,10 @@ async fn store_maintenance( .query::(RegistryQuery::new(ObjectType::Account)) .await? { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] let status = TaskStatus::at(now); - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] let status = TaskStatus::at(now + rand::RngExt::random_range(&mut rand::rng(), 0..=300)); @@ -142,20 +125,15 @@ async fn store_maintenance( })); if batch.is_large_batch() { - server.core.storage.data.write(batch.build_all()).await?; - server.notify_task_queue(); + server.commit_batch(batch).await?; batch = BatchBuilder::new(); } } if !batch.is_empty() { - server.core.storage.data.write(batch.build_all()).await?; - server.notify_task_queue(); + server.commit_batch(batch).await?; } } - TaskStoreMaintenanceType::ReindexTelemetry => { - reindex_telemetry(server).await?; - } TaskStoreMaintenanceType::PurgeData => { // Delete expired external reports let now = now(); @@ -197,14 +175,14 @@ async fn store_maintenance( } if batch.is_large_batch() { - server.store().write(batch.build_all()).await?; + server.store().write_batch(&mut batch).await?; batch = BatchBuilder::new(); } } } } if !batch.is_empty() { - server.store().write(batch.build_all()).await?; + server.store().write_batch(&mut batch).await?; } let started = Instant::now(); @@ -215,12 +193,6 @@ async fn store_maintenance( .await .caused_by(trc::location!())?; - server - .in_memory_store() - .purge_in_memory_store() - .await - .caused_by(trc::location!())?; - server .registry() .purge_dead_nodes() @@ -251,15 +223,13 @@ async fn store_maintenance( })); if batch.is_large_batch() { - server.core.storage.data.write(batch.build_all()).await?; - server.notify_task_queue(); + server.commit_batch(batch).await?; batch = BatchBuilder::new(); } } if !batch.is_empty() { - server.core.storage.data.write(batch.build_all()).await?; - server.notify_task_queue(); + server.commit_batch(batch).await?; } } } @@ -267,36 +237,27 @@ async fn store_maintenance( | TaskStoreMaintenanceType::RemoveLockQueueMessage | TaskStoreMaintenanceType::RemoveLockTask | TaskStoreMaintenanceType::RemoveLockDav - | TaskStoreMaintenanceType::RemoveSieveId | TaskStoreMaintenanceType::ResetRateLimiters | TaskStoreMaintenanceType::ResetBlobQuotas | TaskStoreMaintenanceType::RemoveAuthTokens => { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if let Some(test_var) = task.shard_index { - use crate::task_manager::TaskFailureType; + use crate::task_manager::outcome::{Failure, RetryClass}; - // Simulate success for testing purposes - match test_var { - 0 => { - return Ok(TaskResult::Success(vec![])); + return Ok(match test_var { + 0 => Outcome::done(), + 1 => Outcome::transient("Simulated temporary failure"), + 2 => Outcome::fatal("Simulated permanent failure"), + 3 => panic!("Simulated task panic"), + 4 => Outcome::Obsolete("Simulated obsolete task"), + 5 => { + Outcome::Retry(Failure::new(RetryClass::Contention, "Simulated contention")) } - 1 => { - return Ok(TaskResult::temporary( - "Simulated temporary failure".to_string(), - )); - } - 2 => { - return Ok(TaskResult::permanent("Simulated permanent failure")); - } - - retry => { - return Ok(TaskResult::Failure { - typ: TaskFailureType::Retry(retry), - message: "Simulated retry failure".to_string(), - max_attempts: None, - }); - } - } + retry_at => Outcome::Retry(Failure::new( + RetryClass::Throttled { retry_at }, + "Simulated throttled failure", + )), + }); } let prefixes = match task.maintenance_type { @@ -304,7 +265,6 @@ async fn store_maintenance( TaskStoreMaintenanceType::RemoveLockQueueMessage => &[KV_LOCK_QUEUE_MESSAGE][..], TaskStoreMaintenanceType::RemoveLockTask => &[KV_LOCK_TASK][..], TaskStoreMaintenanceType::RemoveLockDav => &[KV_LOCK_DAV][..], - TaskStoreMaintenanceType::RemoveSieveId => &[KV_SIEVE_ID][..], TaskStoreMaintenanceType::ResetRateLimiters => &[ KV_RATE_LIMIT_RCPT, KV_RATE_LIMIT_SCAN, @@ -315,9 +275,12 @@ async fn store_maintenance( KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_IMAP, + KV_RATE_LIMIT_CIMD, ][..], TaskStoreMaintenanceType::ResetBlobQuotas => &[KV_QUOTA_BLOB][..], - TaskStoreMaintenanceType::RemoveAuthTokens => &[KV_ACME, KV_OAUTH][..], + TaskStoreMaintenanceType::RemoveAuthTokens => { + &[KV_ACME, KV_OAUTH, KV_OAUTH_CIMD][..] + } _ => unreachable!(), }; @@ -328,17 +291,15 @@ async fn store_maintenance( .await?; } } - TaskStoreMaintenanceType::ResetTenantQuotas => { - } } - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } async fn account_maintenance( server: &Server, task: &TaskAccountMaintenance, -) -> trc::Result { +) -> trc::Result { match task.maintenance_type { TaskAccountMaintenanceType::Purge => { server.purge_account(task.account_id.document_id()).await?; @@ -352,29 +313,32 @@ async fn account_maintenance( TaskAccountMaintenanceType::RecalculateQuota => { recalculate_quota(server, task.account_id.document_id()).await?; } - } - - Ok(TaskResult::Success(vec![])) -} - -async fn tenant_maintenance( - server: &Server, - task: &TaskTenantMaintenance, -) -> trc::Result { - match task.maintenance_type { - TaskTenantMaintenanceType::RecalculateQuota => { - recalculate_tenant_quota(server, task.tenant_id.document_id()).await?; + TaskAccountMaintenanceType::RebuildShares => { + server.rebuild_shares(task.account_id.document_id()).await?; } } - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } + async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> { let mut quota = 0; + let mut emails = 0; + let mut events = 0; + let mut notifications = 0; + let mut submissions = 0; + + server + .message_datas(account_id, &(), |_, data| { + quota += data.size as i64; + emails += 1; + Ok(true) + }) + .await + .caused_by(trc::location!())?; for collection in [ - Collection::Email, Collection::Calendar, Collection::CalendarEvent, Collection::CalendarEventNotification, @@ -382,27 +346,33 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> Collection::ContactCard, Collection::FileNode, Collection::SieveScript, + Collection::EmailSubmission, ] { server - .archives(account_id, collection, &(), |_, archive| { + .archives(account_id, collection, Field::ARCHIVE, &(), |_, archive| { match collection { - Collection::Email => { - quota += archive.unarchive::()?.size.to_native() as i64; - } Collection::Calendar => { quota += archive.unarchive::()?.size() as i64; } Collection::CalendarEvent => { - quota += archive.unarchive::()?.size() as i64; + quota += archive.unarchive::()?.size.to_native() as i64; + events += 1; } Collection::CalendarEventNotification => { - quota += archive.unarchive::()?.size() as i64; + quota += archive + .unarchive::()? + .size + .to_native() as i64; + notifications += 1; + } + Collection::EmailSubmission => { + submissions += 1; } Collection::AddressBook => { quota += archive.unarchive::()?.size() as i64; } Collection::ContactCard => { - quota += archive.unarchive::()?.size() as i64; + quota += archive.unarchive::()?.size.to_native() as i64; } Collection::FileNode => { quota += archive.unarchive::()?.size() as i64; @@ -418,25 +388,33 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> .caused_by(trc::location!())?; } + quota += server + .metadata_used_quota(account_id) + .await + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .clear(ValueClass::Quota) - .add(ValueClass::Quota, quota); + batch.with_account_id(account_id); + for (object, count) in [ + (StorageQuota::MaxEmails, emails), + (StorageQuota::MaxEmailSubmissions, submissions), + (StorageQuota::MaxCalendarEvents, events), + (StorageQuota::MaxCalendarEventNotifications, notifications), + ] { + batch + .clear(ValueClass::ObjectCount(object)) + .add(ValueClass::ObjectCount(object), count); + } + batch.clear(ValueClass::Quota).add(ValueClass::Quota, quota); server .store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!()) .map(|_| ()) } -#[cfg(not(feature = "enterprise"))] -async fn recalculate_tenant_quota(_server: &Server, _tenant_id: u32) -> trc::Result<()> { - Ok(()) -} - async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> { let mut mailbox_count = 0; let mut email_count = 0; @@ -449,7 +427,7 @@ async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, for &mailbox_id in cache.mailboxes.index.keys() { let mailbox = server .store() - .get_value::>(ValueKey::archive( + .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, mailbox_id, @@ -474,65 +452,51 @@ async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, .caused_by(trc::location!())? .clear(MailboxField::UidCounter); server - .store() - .write(batch.build_all()) + .commit_batch(batch) .await .caused_by(trc::location!())?; mailbox_count += 1; } // Reset all UIDs - for message_id in cache.emails.items.iter().map(|i| i.document_id) { - let data = server + for message_id in cache.emails.iter().map(|i| i.document_id()) { + let Some(data) = server .store() - .get_value::>(ValueKey::archive( - account_id, - Collection::Email, - message_id, - )) + .get_value::(ValueKey::archive(account_id, Collection::Email, message_id)) .await - .caused_by(trc::location!())?; - let data_ = if let Some(data) = data { - data - } else { + .caused_by(trc::location!())? + else { continue; }; - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - let mut new_data = data - .deserialize::() - .caused_by(trc::location!())?; - - let ids = server - .assign_email_ids( - account_id, - new_data.mailboxes.iter().map(|m| m.mailbox_id), - false, - ) - .await - .caused_by(trc::location!())?; - - for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) { - uid_mailbox.uid = uid; + let mut new_data = data.clone(); + for uid_mailbox in new_data.mailboxes.iter_mut() { + uid_mailbox.uid = 0; } // Prepare write batch let mut batch = BatchBuilder::new(); + let uid_slots = + batch.reserve_uids(account_id, new_data.mailboxes.iter().map(|m| m.mailbox_id)); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(message_id) - .assert_value(ValueClass::Property(EmailField::Archive.into()), &data) + .assert_value( + ValueClass::Property(EmailField::Archive.into()), + data.change_id, + ) .set( EmailField::Archive, - Archiver::new(new_data) - .serialize() - .caused_by(trc::location!())?, + PendingMessageData { + data: new_data, + uid_slots, + thread_slot: None, + change_id: Some(data.change_id), + }, ); server .store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; email_count += 1; diff --git a/crates/services/src/task_manager/merge_threads.rs b/crates/services/src/task_manager/tasks/merge_threads.rs similarity index 74% rename from crates/services/src/task_manager/merge_threads.rs rename to crates/services/src/task_manager/tasks/merge_threads.rs index 2a8a4c03..92d59100 100644 --- a/crates/services/src/task_manager/merge_threads.rs +++ b/crates/services/src/task_manager/tasks/merge_threads.rs @@ -4,42 +4,41 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::TaskResult; +use crate::task_manager::outcome::{Failure, Outcome, RetryClass}; use common::{Server, storage::index::ObjectIndexBuilder}; use email::message::{ ingest::{ThreadMerge, has_message_id}, - metadata::MessageData, + messagedata::MessageData, }; use registry::schema::structs::TaskMergeThreads; -use std::{str::FromStr, time::Duration}; +use std::{ops::RangeInclusive, str::FromStr, time::Duration}; use store::{ IterateParams, Key, U32_LEN, ValueKey, ahash::AHashMap, rand::RngExt, - write::{ - AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, MergeResult, Params, ValueClass, - key::DeserializeBigEndian, - }, + write::{BatchBuilder, IndexPropertyClass, MergeResult, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::EmailField, }; -use utils::cheeky_hash::CheekyHash; +use utils::hash128::Hash128; const MAX_RETRIES: usize = 5; +const MAX_PASSES: usize = 5; +const RETRY_BACKOFF_MS: RangeInclusive = 50..=300; pub(crate) trait MergeThreadsTask: Sync + Send { - fn merge_threads(&self, threads: &TaskMergeThreads) -> impl Future + Send; + fn merge_threads(&self, threads: &TaskMergeThreads) -> impl Future + Send; } impl MergeThreadsTask for Server { - async fn merge_threads(&self, threads: &TaskMergeThreads) -> TaskResult { + async fn merge_threads(&self, threads: &TaskMergeThreads) -> Outcome { match merge_threads(self, threads).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let result = Outcome::error(&err); trc::error!( err.account_id(threads.account_id.document_id()) .details("Failed to merge threads") @@ -53,17 +52,22 @@ impl MergeThreadsTask for Server { async fn merge_threads( server: &Server, task_merge_threads: &TaskMergeThreads, -) -> trc::Result { - let Ok(thread_hash) = CheekyHash::from_str(&task_merge_threads.thread_name) else { - return Ok(TaskResult::permanent("Invalid thread hash")); +) -> trc::Result { + #[cfg(stalwart_tests)] + if task_merge_threads.thread_name == "bench-transient" { + return Ok(Outcome::transient("Simulated transient failure")); + } + + let Ok(thread_hash) = Hash128::from_str(&task_merge_threads.thread_name) else { + return Ok(Outcome::fatal("Invalid thread hash")); }; let Ok(mut message_ids) = task_merge_threads .message_ids .iter() - .map(|id| CheekyHash::from_str(id)) + .map(|id| u128::from_str(id)) .collect::, _>>() else { - return Ok(TaskResult::permanent("Invalid message ids")); + return Ok(Outcome::fatal("Invalid message ids")); }; message_ids.sort_unstable(); @@ -73,7 +77,7 @@ async fn merge_threads( let from_key = ValueKey { account_id, collection: Collection::Email.into(), - document_id: 0, + document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_hash, @@ -82,7 +86,7 @@ async fn merge_threads( let to_key = ValueKey { account_id, collection: Collection::Email.into(), - document_id: u32::MAX, + document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_hash, @@ -92,6 +96,8 @@ async fn merge_threads( let key_len = prefix.len(); let document_id_pos = key_len - U32_LEN; prefix.truncate(document_id_pos); + let mut merged_away: Vec = Vec::new(); + let mut passes = 0; 'retry: loop { // Merge threads @@ -108,9 +114,11 @@ async fn merge_threads( // Find matching references let references = value.get(U32_LEN..).unwrap_or_default(); let thread_id = value.deserialize_be_u32(0)?; - let document_id = key.deserialize_be_u32(document_id_pos)?; + let document_id = !key.deserialize_be_u32(document_id_pos)?; - if has_message_id(&message_ids, references) { + if has_message_id(&message_ids, references) + || merged_away.contains(&thread_id) + { thread_merge.add(thread_id, document_id); } else { // Keep track of messages with the same subject for potential future merges @@ -129,7 +137,7 @@ async fn merge_threads( if thread_merge.num_thread_ids() < 2 { // Another process merged the threads already? - return Ok(TaskResult::Success(vec![])); + return Ok(Outcome::done()); } // Add other messages with the same subject to the merge if they share a @@ -164,9 +172,9 @@ async fn merge_threads( for (&group_thread_id, document_ids) in thread_merge.thread_groups() { if thread_id != group_thread_id { for &document_id in document_ids { - if let Some(data_) = server + if let Some(data) = server .store() - .get_value::>(ValueKey::archive( + .get_value::(ValueKey::archive( account_id, Collection::Email, document_id, @@ -174,18 +182,20 @@ async fn merge_threads( .await .caused_by(trc::location!())? { - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - if data.inner.thread_id != group_thread_id { + if data.thread_id != group_thread_id { try_count += 1; + if try_count > MAX_RETRIES { + return Ok(Outcome::Retry(Failure::new( + RetryClass::Contention, + "Thread ids keep changing while merging", + ))); + } + retry_backoff().await; continue 'retry; } // Update thread id - let mut new_data = data - .deserialize::() - .caused_by(trc::location!())?; + let mut new_data = data.clone(); new_data.thread_id = thread_id; batch .with_document(document_id) @@ -202,13 +212,7 @@ async fn merge_threads( property: EmailField::Threading.into(), hash: thread_hash, }), - Params::with_capacity(3) - .with_u64(thread_id as u64) - .with_u64(group_thread_id as u64), - |params, _, bytes| { - let new_thread_id = params.u64(0) as u32; - let old_thread_id = params.u64(1) as u32; - + move |_, bytes| { let mut thread_index = bytes .filter(|v| v.len() > U32_LEN) .ok_or_else(|| { @@ -219,7 +223,7 @@ async fn merge_threads( })? .to_vec(); - if thread_index.as_slice().deserialize_be_u32(0)? != old_thread_id { + if thread_index.as_slice().deserialize_be_u32(0)? != group_thread_id { return Err( trc::StoreEvent::AssertValueFailed .into_err() @@ -228,7 +232,7 @@ async fn merge_threads( ); } - thread_index[0..U32_LEN].copy_from_slice(&new_thread_id.to_be_bytes()); + thread_index[0..U32_LEN].copy_from_slice(&thread_id.to_be_bytes()); Ok(MergeResult::Update(thread_index)) }, @@ -239,10 +243,19 @@ async fn merge_threads( } match server.commit_batch(batch).await { - Ok(_) => return Ok(TaskResult::Success(vec![])), + Ok(_) => { + passes += 1; + if passes == MAX_PASSES { + return Ok(Outcome::done()); + } + for &merged in thread_merge.thread_ids() { + if merged != thread_id && !merged_away.contains(&merged) { + merged_away.push(merged); + } + } + } Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { - let backoff = store::rand::rng().random_range(50..=300); - tokio::time::sleep(Duration::from_millis(backoff)).await; + retry_backoff().await; try_count += 1; } Err(err) => { @@ -251,3 +264,8 @@ async fn merge_threads( } } } + +async fn retry_backoff() { + let backoff = store::rand::rng().random_range(RETRY_BACKOFF_MS); + Box::pin(tokio::time::sleep(Duration::from_millis(backoff))).await; +} diff --git a/crates/services/src/task_manager/tasks/mod.rs b/crates/services/src/task_manager/tasks/mod.rs new file mode 100644 index 00000000..3ea70b0e --- /dev/null +++ b/crates/services/src/task_manager/tasks/mod.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod acme; +pub mod alarm; +pub mod destroy_account; +pub mod dkim; +pub mod dns; +pub mod grantee_cleanup; +pub mod imip; +pub mod maintenance; +pub mod merge_threads; +pub mod rebuild_shares; +pub mod report; +pub mod spam_classifier; + +use std::fmt::Write; + +pub(crate) struct SmtpRejection { + reason: String, + is_temporary: bool, +} + +impl SmtpRejection { + pub(crate) fn new(stage: &str, response: &str) -> Self { + let response = response.trim(); + SmtpRejection { + reason: format!("Server rejected {stage}: {response}"), + is_temporary: response.starts_with('4'), + } + } +} + +pub(crate) fn append_error(errors: &mut String, error: std::fmt::Arguments<'_>) { + if !errors.is_empty() { + errors.push_str("; "); + } + let _ = errors.write_fmt(error); +} diff --git a/crates/services/src/task_manager/tasks/rebuild_shares.rs b/crates/services/src/task_manager/tasks/rebuild_shares.rs new file mode 100644 index 00000000..e2e2076d --- /dev/null +++ b/crates/services/src/task_manager/tasks/rebuild_shares.rs @@ -0,0 +1,236 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::grantee_cleanup::shared_acls; +use common::{Server, cache::invalidate::CacheInvalidationBuilder}; +use registry::{schema::prelude::ObjectType, types::EnumImpl}; +use std::mem; +use store::{ + SerializeInfallible, ValueKey, + ahash::{AHashMap, AHashSet}, + roaring::RoaringBitmap, + write::{BatchBuilder, RegistryClass, ValueClass, assert::AssertValue}, +}; +use trc::AddContext; +use types::{acl::Acl, collection::Collection, field::Field}; +use utils::map::bitmap::Bitmap; + +const SHARED_COLLECTIONS: [Collection; 4] = [ + Collection::Mailbox, + Collection::Calendar, + Collection::AddressBook, + Collection::FileNode, +]; + +struct ShareFix { + document_id: u32, + archive: AssertValue, + grants: Vec<(u32, u64)>, + revokes: Vec, +} + +pub trait RevokeAccountGrants: Sync + Send { + fn revoke_account_grants( + &self, + account_id: u32, + ) -> impl Future>> + Send; +} + +impl RevokeAccountGrants for Server { + async fn revoke_account_grants(&self, account_id: u32) -> trc::Result> { + let grantee_ids = match owned_grantees(self, account_id).await { + Ok(grantee_ids) => grantee_ids, + Err(err) => { + trc::error!( + err.account_id(account_id) + .details("Failed to read shared containers, scanning every grant") + .caused_by(trc::location!()) + ); + self.store() + .acl_owned(account_id) + .await + .caused_by(trc::location!())? + .into_iter() + .map(|(grantee_id, _)| grantee_id) + .collect() + } + }; + self.store() + .acl_revoke_owner(account_id, &grantee_ids) + .await + .caused_by(trc::location!())?; + Ok(grantee_ids) + } +} + +async fn owned_grantees(server: &Server, account_id: u32) -> trc::Result> { + let mut grantee_ids = AHashSet::new(); + for collection in SHARED_COLLECTIONS { + server + .archives(account_id, collection, Field::ARCHIVE, &(), |_, archive| { + if let Some(acls) = shared_acls(collection, &archive)? { + grantee_ids.extend(acls.iter().map(|grant| u32::from(grant.account_id))); + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + } + Ok(grantee_ids) +} + +pub(crate) trait RebuildSharesTask: Sync + Send { + fn rebuild_shares(&self, account_id: u32) -> impl Future> + Send; + + fn invalidate_grantees( + &self, + grantee_ids: &mut RoaringBitmap, + ) -> impl Future> + Send; +} + +impl RebuildSharesTask for Server { + async fn rebuild_shares(&self, account_id: u32) -> trc::Result<()> { + let account = ValueClass::Registry(RegistryClass::Item { + object_id: ObjectType::Account.to_id(), + item_id: u64::from(account_id), + }); + if !self + .store() + .key_exists(ValueKey::from(account.clone())) + .await + .caused_by(trc::location!())? + { + return Ok(()); + } + + let mut indexed: AHashMap<(Collection, u32), Vec<(u32, u64)>> = AHashMap::new(); + for (grantee_id, item) in self + .store() + .acl_owned(account_id) + .await + .caused_by(trc::location!())? + { + indexed + .entry((item.to_collection, item.to_document_id)) + .or_default() + .push((grantee_id, item.permissions)); + } + + let mut fixes = Vec::new(); + let mut grantee_ids = RoaringBitmap::new(); + let new_batch = || { + let mut batch = BatchBuilder::new(); + batch.assert_value(account.clone(), AssertValue::Some); + batch + }; + let mut batch = new_batch(); + let mut has_changes = false; + for collection in SHARED_COLLECTIONS { + self.archives( + account_id, + collection, + Field::ARCHIVE, + &(), + |document_id, archive| { + let Some(acls) = shared_acls(collection, &archive)? else { + return Ok(true); + }; + let mut stale = indexed + .remove(&(collection, document_id)) + .unwrap_or_default(); + let mut grants = Vec::new(); + for grant in acls.iter() { + let grantee_id = u32::from(grant.account_id); + let rights = Bitmap::::from(&grant.grants).bitmap; + let indexed_rights = stale + .iter() + .position(|(id, _)| *id == grantee_id) + .map(|idx| stale.swap_remove(idx).1); + if indexed_rights != Some(rights) { + grants.push((grantee_id, rights)); + } + } + if !grants.is_empty() || !stale.is_empty() { + fixes.push(ShareFix { + document_id, + archive: AssertValue::Archive(archive.version), + grants, + revokes: stale + .into_iter() + .map(|(grantee_id, _)| grantee_id) + .collect(), + }); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + fixes.extend( + indexed + .extract_if(|(indexed_collection, _), _| *indexed_collection == collection) + .map(|((_, document_id), stale)| ShareFix { + document_id, + archive: AssertValue::None, + grants: Vec::new(), + revokes: stale + .into_iter() + .map(|(grantee_id, _)| grantee_id) + .collect(), + }), + ); + + for fix in fixes.drain(..) { + batch + .with_account_id(account_id) + .with_collection(collection) + .with_document(fix.document_id) + .assert_value(Field::ARCHIVE, fix.archive); + for (grantee_id, rights) in fix.grants { + batch.acl_grant(grantee_id, rights.serialize()); + grantee_ids.insert(grantee_id); + } + for grantee_id in fix.revokes { + batch.acl_revoke(grantee_id); + grantee_ids.insert(grantee_id); + } + has_changes = true; + if batch.is_large_batch() { + self.store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + self.invalidate_grantees(&mut grantee_ids).await?; + batch = new_batch(); + has_changes = false; + } + } + } + + if has_changes { + self.store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + self.invalidate_grantees(&mut grantee_ids).await?; + } + + Ok(()) + } + + async fn invalidate_grantees(&self, grantee_ids: &mut RoaringBitmap) -> trc::Result<()> { + if grantee_ids.is_empty() { + return Ok(()); + } + let mut changes = CacheInvalidationBuilder::default(); + for grantee_id in mem::take(grantee_ids) { + self.invalidate_grantee(&mut changes, grantee_id).await; + } + self.invalidate_caches(changes) + .await + .caused_by(trc::location!()) + } +} diff --git a/crates/services/src/task_manager/tasks/report.rs b/crates/services/src/task_manager/tasks/report.rs new file mode 100644 index 00000000..a69db1cc --- /dev/null +++ b/crates/services/src/task_manager/tasks/report.rs @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::outcome::Outcome; +use common::Server; +use registry::{schema::prelude::ObjectType, types::datetime::UTCDateTime}; +use smtp::reporting::{ + aggregate::window::create_window_reports, dmarc::DmarcReporting, tls::TlsReporting, +}; + +pub enum ReportId { + Dmarc(u64), + Tls(u64), + DmarcWindow(UTCDateTime), + TlsWindow(UTCDateTime), +} + +pub(crate) trait SubmitReportTask: Sync + Send { + fn submit_report(&self, report_id: ReportId) -> impl Future + Send; +} + +impl SubmitReportTask for Server { + async fn submit_report(&self, report_id: ReportId) -> Outcome { + match submit_report(self, report_id).await { + Ok(()) => Outcome::done(), + Err(err) => { + let outcome = Outcome::error(&err); + trc::error!(err.details("Failed to submit report")); + outcome + } + } + } +} + +async fn submit_report(server: &Server, report_id: ReportId) -> trc::Result<()> { + match report_id { + ReportId::Dmarc(item_id) => server.send_dmarc_aggregate_report(item_id).await, + ReportId::Tls(item_id) => server.send_tls_aggregate_report(item_id).await, + ReportId::DmarcWindow(end) => { + create_window_reports( + server, + ObjectType::DmarcInternalReport, + end.timestamp().cast_unsigned(), + ) + .await + } + ReportId::TlsWindow(end) => { + create_window_reports( + server, + ObjectType::TlsInternalReport, + end.timestamp().cast_unsigned(), + ) + .await + } + } +} diff --git a/crates/services/src/task_manager/restore_item.rs b/crates/services/src/task_manager/tasks/restore_item.rs similarity index 80% rename from crates/services/src/task_manager/restore_item.rs rename to crates/services/src/task_manager/tasks/restore_item.rs index 82dada2f..ae0c6c15 100644 --- a/crates/services/src/task_manager/restore_item.rs +++ b/crates/services/src/task_manager/tasks/restore_item.rs @@ -14,21 +14,18 @@ use registry::schema::{enums::ArchivedItemType, structs::TaskRestoreArchivedItem use store::write::{BatchBuilder, BlobLink, BlobOp}; use trc::AddContext; -use crate::task_manager::TaskResult; +use crate::task_manager::outcome::Outcome; pub(crate) trait RestoreItemTask: Sync + Send { - fn restore_item( - &self, - task: &TaskRestoreArchivedItem, - ) -> impl Future + Send; + fn restore_item(&self, task: &TaskRestoreArchivedItem) -> impl Future + Send; } impl RestoreItemTask for Server { - async fn restore_item(&self, task: &TaskRestoreArchivedItem) -> TaskResult { + async fn restore_item(&self, task: &TaskRestoreArchivedItem) -> Outcome { match restore_item(self, task).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let result = Outcome::error(&err); trc::error!( err.account_id(task.account_id.document_id()) .details("Failed to restore item") @@ -39,7 +36,7 @@ impl RestoreItemTask for Server { } } -async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::Result { +async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::Result { match task.archived_item_type { ArchivedItemType::Email => { let account_id = task.account_id.document_id(); @@ -53,7 +50,7 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R .get_blob(task.blob_id.hash.as_slice(), 0..usize::MAX) .await? else { - return Ok(TaskResult::permanent("Blob not found")); + return Ok(Outcome::Obsolete("Archived blob no longer exists")); }; match server @@ -66,6 +63,7 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R keywords: vec![], received_at: (task.created_at.timestamp() as u64).into(), source: IngestSource::Restore, + metadata: None, session_id: 0, }) .await @@ -78,20 +76,19 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R until: task.archived_until.timestamp() as u64, }, }); - server.store().write(batch.build_all()).await?; + server.store().write_batch(&mut batch).await?; - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } Err(mut err) if err.matches(trc::EventType::MessageIngest( trc::MessageIngestEvent::Error, )) => { - Ok(TaskResult::permanent( + Ok(Outcome::fatal( err.take_value(trc::Key::Reason) .and_then(|v| v.into_string()) - .unwrap() - .to_string(), + .map_or_else(|| "Message ingestion failed".to_string(), Into::into), )) } Err(err) => Err(err.caused_by(trc::location!())), @@ -100,6 +97,6 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R ArchivedItemType::FileNode | ArchivedItemType::CalendarEvent | ArchivedItemType::ContactCard - | ArchivedItemType::SieveScript => Ok(TaskResult::permanent("Not implemented")), + | ArchivedItemType::SieveScript => Ok(Outcome::fatal("Not implemented")), } } diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/tasks/spam_classifier.rs similarity index 78% rename from crates/services/src/task_manager/spam_classifier.rs rename to crates/services/src/task_manager/tasks/spam_classifier.rs index 390957e4..78cfd7da 100644 --- a/crates/services/src/task_manager/spam_classifier.rs +++ b/crates/services/src/task_manager/tasks/spam_classifier.rs @@ -4,13 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::{TaskFailureType, TaskResult}; +use crate::task_manager::outcome::{ErrorClass, Failure, Outcome, RetryClass}; use common::{ Server, ipc::{BroadcastEvent, RegistryChange}, manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource}, }; use registry::{ + pickle::Pickle, schema::{ enums::TaskSpamFilterMaintenanceType, prelude::{Object, ObjectInner, ObjectType}, @@ -19,7 +20,10 @@ use registry::{ TaskSpamFilterMaintenance, }, }, - types::{EnumImpl, ObjectImpl}, + types::{ + EnumImpl, ObjectImpl, + index::{IndexBuilder, IndexKey}, + }, }; use spam_filter::modules::classifier::SpamClassifier; use std::time::{Duration, Instant}; @@ -34,15 +38,15 @@ pub(crate) trait SpamFilterMaintenanceTask: Sync + Send { fn spam_filter_maintenance( &self, task: &TaskSpamFilterMaintenance, - ) -> impl Future + Send; + ) -> impl Future + Send; } impl SpamFilterMaintenanceTask for Server { - async fn spam_filter_maintenance(&self, task: &TaskSpamFilterMaintenance) -> TaskResult { + async fn spam_filter_maintenance(&self, task: &TaskSpamFilterMaintenance) -> Outcome { match spam_filter_maintenance(self, task).await { - Ok(result) => result, + Ok(outcome) => outcome, Err(err) => { - let result = TaskResult::temporary(err.to_string()); + let result = Outcome::error(&err); trc::error!(err.details("Failed to perform spam filter maintenance task")); result } @@ -53,16 +57,16 @@ impl SpamFilterMaintenanceTask for Server { async fn spam_filter_maintenance( server: &Server, task: &TaskSpamFilterMaintenance, -) -> trc::Result { +) -> trc::Result { match task.maintenance_type { TaskSpamFilterMaintenanceType::Train => { if !server.inner.ipc.train_task_controller.is_running() { - Box::pin(server.spam_train(false)).await?; + server.spam_train(false).await?; } } TaskSpamFilterMaintenanceType::Retrain => { if !server.inner.ipc.train_task_controller.is_running() { - Box::pin(server.spam_train(true)).await?; + server.spam_train(true).await?; } } TaskSpamFilterMaintenanceType::Reset => { @@ -80,11 +84,11 @@ async fn spam_filter_maintenance( } } - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } struct RuleUpdateError { - typ: TaskFailureType, + typ: ErrorClass, reason: String, } @@ -98,7 +102,9 @@ struct Rules { file_exts: Vec, } -trait UpstreamObject: ObjectImpl + PartialEq + From + Into { +trait UpstreamObject: + ObjectImpl + Pickle + Clone + PartialEq + From + Into +{ fn replacement_for(self, _local: &Self) -> Option { Some(self) } @@ -159,15 +165,14 @@ impl RuleUpdateResult { } } -async fn update_spam_rules(server: &Server) -> trc::Result { +async fn update_spam_rules(server: &Server) -> trc::Result { let started = Instant::now(); let rules = match fetch_spam_rules(server).await { Ok(rules) => rules, Err(err) => { - return Ok(TaskResult::Failure { - typ: err.typ, - message: err.reason, - max_attempts: None, + return Ok(match err.typ { + ErrorClass::Retry(class) => Outcome::Retry(Failure::new(class, err.reason)), + ErrorClass::Fatal => Outcome::Fatal(err.reason), }); } }; @@ -227,21 +232,21 @@ async fn update_spam_rules(server: &Server) -> trc::Result { ); if !reload_errors.is_empty() { - Ok(TaskResult::permanent(format!( + Ok(Outcome::fatal(format!( "Spam rules were stored but not activated ({}); fix the logged errors and run Reload settings", reload_errors.join("; ") ))) } else if failed > 0 { - Ok(TaskResult::permanent(format!( + Ok(Outcome::fatal(format!( "{failed} spam filter objects failed to import or update" ))) } else { - Ok(TaskResult::Success(vec![])) + Ok(Outcome::done()) } } async fn reload_and_broadcast(server: &Server, object: ObjectType) -> Result<(), String> { - match Box::pin(server.reload_registry(RegistryChange::Reload(object))).await { + match server.reload_registry(RegistryChange::Reload(object)).await { Ok(result) => { result.log(); if result.has_errors() { @@ -267,51 +272,52 @@ async fn apply_upstream( objects: Vec, ) -> trc::Result { let mut result = RuleUpdateResult::new(T::OBJECT); + let locals = registry.list::().await?; + let mut local_keys = IndexBuilder::default(); + let mut local_by_key = AHashMap::with_capacity(locals.len()); + for (index, local) in locals.iter().enumerate() { + local.object.index(&mut local_keys); + for key in local_keys.keys.drain().filter(is_unique_key) { + local_by_key.insert(key, index); + } + } for upstream in objects { - let upstream = Object::from(upstream); - let existing_id = match registry.write(RegistryWrite::insert(&upstream)).await? { - RegistryWriteResult::Success(_) => { - result.added += 1; - continue; - } - RegistryWriteResult::PrimaryKeyConflict { existing_id, .. } - if existing_id.object() == T::OBJECT => - { - existing_id - } - RegistryWriteResult::PrimaryKeyConflict { .. } => { - result.unchanged += 1; - continue; - } - _ => { - result.failed += 1; - continue; - } + let existing = { + let mut upstream_keys = IndexBuilder::default(); + upstream.index(&mut upstream_keys); + upstream_keys + .keys + .iter() + .filter(|key| is_unique_key(key)) + .find_map(|key| local_by_key.get(key)) + .and_then(|&index| locals.get(index)) }; - let Some(local) = registry.get(existing_id).await? else { - result.failed += 1; + let Some(local) = existing else { + match registry + .write(RegistryWrite::insert(&Object::from(upstream))) + .await? + { + RegistryWriteResult::Success(_) => result.added += 1, + RegistryWriteResult::PrimaryKeyConflict { .. } => result.unchanged += 1, + _ => result.failed += 1, + } continue; }; - let revision = local.revision; - let local = T::from(local); - let Some(replacement) = T::from(upstream) - .replacement_for(&local) - .filter(|replacement| replacement != &local) + + let Some(replacement) = upstream + .replacement_for(&local.object) + .filter(|replacement| replacement != &local.object) else { result.unchanged += 1; continue; }; let replacement = Object::from(replacement); - let local = Object::with_revision(local.into(), revision); + let current = Object::with_revision(local.object.clone().into(), local.revision); match registry - .write(RegistryWrite::update( - existing_id.id(), - &replacement, - &local, - )) + .write(RegistryWrite::update(local.id.id(), &replacement, ¤t)) .await? { RegistryWriteResult::Success(_) => result.updated += 1, @@ -322,23 +328,27 @@ async fn apply_upstream( Ok(result) } +fn is_unique_key(key: &IndexKey<'_>) -> bool { + matches!(key, IndexKey::Unique { .. }) +} + async fn fetch_spam_rules(server: &Server) -> Result { let Some(rules_url) = server.core.spam.spam_rules_url.as_ref() else { return Err(RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: "Spam rules resource URL not configured".to_string(), }); }; let rules_json: AHashMap> = - fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500) + fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500, false) .await .map_err(|reason| RuleUpdateError { - typ: TaskFailureType::Temporary, + typ: ErrorClass::Retry(RetryClass::Transient), reason, }) .and_then(|bytes| { serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse spam rules JSON: {err}"), }) })?; @@ -347,7 +357,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { for (object_type, values) in rules_json { let Some(object_type) = ObjectType::parse(&object_type) else { return Err(RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Invalid object type in spam rules JSON: {object_type}"), }); }; @@ -358,7 +368,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse spam rule: {err}"), }) }) @@ -369,7 +379,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse DNSBL server: {err}"), }) }) @@ -380,7 +390,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse spam tag: {err}"), }) }) @@ -391,7 +401,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse HTTP lookup: {err}"), }) }) @@ -402,7 +412,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse memory lookup key: {err}"), }) }) @@ -413,7 +423,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { .into_iter() .map(|value| { serde_json::from_value(value).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Failed to parse spam file extension: {err}"), }) }) @@ -421,7 +431,7 @@ async fn fetch_spam_rules(server: &Server) -> Result { } _ => { return Err(RuleUpdateError { - typ: TaskFailureType::Permanent, + typ: ErrorClass::Fatal, reason: format!("Unsupported object type in spam rules: {object_type:?}"), }); } diff --git a/crates/services/src/task_manager/worker.rs b/crates/services/src/task_manager/worker.rs new file mode 100644 index 00000000..241b4505 --- /dev/null +++ b/crates/services/src/task_manager/worker.rs @@ -0,0 +1,353 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::{ + Dispatch, LEASE_RENEW_INTERVAL, StoredTask, TaskDone, TaskInfo, TaskJob, TaskTypeLimits, + lock::{TaskLock, TaskLockManager}, + outcome::{Commit, Outcome, commit}, + tasks::{ + acme::AcmeTask, + alarm::SendAlarmTask, + destroy_account::DestroyAccountTask, + dkim::DkimManagementTask, + dns::DnsManagementTask, + grantee_cleanup::GranteeCleanupTask, + imip::SendImipTask, + maintenance::MaintenanceTask, + merge_threads::MergeThreadsTask, + report::{ReportId, SubmitReportTask}, + spam_classifier::SpamFilterMaintenanceTask, + }, +}; +use common::{BuildServer, Inner, Server, network::ServerInstance}; +use compact_str::CompactString; +use registry::schema::{enums::TaskType, structs::Task}; +use std::{any::Any, future::Future, pin::pin, sync::Arc}; +use store::{ + ValueKey, + write::{BatchBuilder, TaskId, TaskQueueClass, TaskTypes, ValueClass, assert::AssertValue}, +}; +use tokio::{ + sync::mpsc, + task::{self, JoinError, JoinSet}, + time::{Instant, sleep}, +}; +use trc::TaskManagerEvent; + +struct JobResult { + done: TaskDone, + written: TaskTypes, +} + +pub(crate) fn spawn_worker( + inner: Arc, + typ: TaskType, + mut rx: mpsc::Receiver, + done_tx: mpsc::UnboundedSender, + server_instance: Arc, +) { + tokio::spawn(async move { + let concurrency = typ.concurrency(); + let mut running = JoinSet::new(); + let mut in_flight: Vec<(task::Id, Dispatch)> = Vec::with_capacity(concurrency); + + loop { + let result = tokio::select! { + dispatch = rx.recv(), if running.len() < concurrency => { + let Some(dispatch) = dispatch else { + break; + }; + let handle = running.spawn(run_job( + inner.clone(), + dispatch, + server_instance.clone(), + )); + in_flight.push((handle.id(), dispatch)); + continue; + } + Some(joined) = running.join_next_with_id() => joined, + else => break, + }; + + let result = match result { + Ok((id, result)) => { + take_in_flight(&mut in_flight, id); + Some(result) + } + Err(err) => match take_in_flight(&mut in_flight, err.id()) { + Some(dispatch) => Some(recover(&inner, dispatch, err).await), + None => None, + }, + }; + + if let Some(result) = result { + let _ = done_tx.send(result.done); + if !result.written.is_empty() || rx.is_empty() { + inner.build_server().notify_task_types(result.written); + } + } + } + }); +} + +fn take_in_flight(in_flight: &mut Vec<(task::Id, Dispatch)>, id: task::Id) -> Option { + in_flight + .iter() + .position(|(running_id, _)| *running_id == id) + .map(|idx| in_flight.swap_remove(idx).1) +} + +async fn run_job( + inner: Arc, + dispatch: Dispatch, + server_instance: Arc, +) -> JobResult { + let server = inner.build_server(); + let Dispatch { + job, + lock, + dispatched_at, + } = dispatch; + + if dispatched_at.elapsed() >= LEASE_RENEW_INTERVAL && !server.renew_task_lock(lock).await { + return JobResult { + done: job.done(false), + written: TaskTypes::default(), + }; + } + + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { + id: TaskId::Assigned(job.id), + }, + ))) + .await + { + Ok(Some(StoredTask { task, hash })) if task.due_timestamp() != job.due => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + TaskId = job.id, + Name = task.name(), + Reason = "Task was rescheduled, removing its previous due date.", + ); + + clear_stale_due(&server, &job, lock, AssertValue::Hash(hash)).await + } + Ok(Some(StoredTask { task, hash })) => { + let outcome = with_lease(&server, lock, execute(&server, &task, server_instance)).await; + finish( + &server, + &job, + lock, + commit(&server, &job, task, hash, outcome).await, + ) + .await + } + Ok(None) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + TaskId = job.id, + Reason = "Task not found in store, likely already processed.", + ); + + clear_stale_due(&server, &job, lock, AssertValue::None).await + } + Err(err) => { + trc::error!( + err.ctx(trc::Key::TaskId, job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + + finish(&server, &job, lock, Commit::Failed).await + } + } +} + +async fn recover(inner: &Arc, dispatch: Dispatch, err: JoinError) -> JobResult { + let server = inner.build_server(); + let Dispatch { job, lock, .. } = dispatch; + let reason = if err.is_panic() { + format!( + "Task handler panicked: {}", + panic_message(err.into_panic().as_ref()) + ) + } else { + "Task handler was cancelled".to_string() + }; + + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Task handler terminated abnormally.", + TaskId = job.id, + Reason = CompactString::from(reason.as_str()), + CausedBy = trc::location!() + ); + + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { + id: TaskId::Assigned(job.id), + }, + ))) + .await + { + Ok(Some(StoredTask { task, hash })) if task.due_timestamp() == job.due => { + let result = commit(&server, &job, task, hash, Outcome::transient(reason)).await; + finish(&server, &job, lock, result).await + } + Ok(_) => finish(&server, &job, lock, Commit::Conflict).await, + Err(err) => { + trc::error!( + err.ctx(trc::Key::TaskId, job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + + finish(&server, &job, lock, Commit::Failed).await + } + } +} + +fn panic_message(payload: &(dyn Any + Send)) -> &str { + payload + .downcast_ref::<&str>() + .copied() + .or_else(|| payload.downcast_ref::().map(String::as_str)) + .unwrap_or("unknown panic") +} + +async fn finish(server: &Server, job: &TaskJob, lock: TaskLock, result: Commit) -> JobResult { + match result { + Commit::Committed(written) => { + server.unlock_task(lock).await; + JobResult { + done: job.done(true), + written, + } + } + Commit::Conflict => { + server.unlock_task(lock).await; + JobResult { + done: job.done(false), + written: TaskTypes::default(), + } + } + Commit::Failed => JobResult { + done: job.done(false), + written: TaskTypes::default(), + }, + } +} + +async fn clear_stale_due( + server: &Server, + job: &TaskJob, + lock: TaskLock, + task: AssertValue, +) -> JobResult { + let id = TaskId::Assigned(job.id); + let mut batch = BatchBuilder::new(); + batch + .assert_value(ValueClass::TaskQueue(TaskQueueClass::Task { id }), task) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + typ: job.typ, + due: job.due, + })); + + let result = match server.store().write_batch(&mut batch).await { + Ok(_) => Commit::Committed(TaskTypes::default()), + Err(err) if err.is_assertion_failure() => Commit::Conflict, + Err(err) => { + trc::error!( + err.ctx(trc::Key::TaskId, job.id) + .details("Failed to remove stale task due date from queue.") + ); + Commit::Failed + } + }; + + finish(server, job, lock, result).await +} + +async fn with_lease( + server: &Server, + lock: TaskLock, + task: impl Future, +) -> Outcome { + let mut task = pin!(task); + let mut renew = pin!(sleep(LEASE_RENEW_INTERVAL)); + let mut lease_lost = false; + + loop { + tokio::select! { + outcome = &mut task => return outcome, + _ = &mut renew => { + match lock.extend(server.in_memory_store()).await { + Ok(true) => lease_lost = false, + Ok(false) | Err(_) if lease_lost => {} + Ok(false) => { + lease_lost = true; + trc::event!( + TaskManager(TaskManagerEvent::TaskLocked), + TaskId = lock.id(), + Details = "Task lease was lost while the task was running", + ); + } + Err(err) => { + lease_lost = true; + trc::error!( + err.ctx(trc::Key::TaskId, lock.id()) + .details("Failed to renew task lease") + ); + } + } + renew.as_mut().reset(Instant::now() + LEASE_RENEW_INTERVAL); + } + } + } +} + +async fn execute(server: &Server, task: &Task, server_instance: Arc) -> Outcome { + match task { + Task::CalendarAlarmEmail(task) => server.send_email_alarm(task, server_instance).await, + Task::CalendarAlarmNotification(task) => server.send_display_alarm(task).await, + Task::CalendarItipMessage(task) => server.send_imip(task, server_instance).await, + Task::MergeThreads(task) => server.merge_threads(task).await, + Task::DmarcReport(task) => { + server + .submit_report(ReportId::Dmarc(task.report_id.id())) + .await + } + Task::TlsReport(task) => { + server + .submit_report(ReportId::Tls(task.report_id.id())) + .await + } + Task::DmarcReportWindow(task) => { + server + .submit_report(ReportId::DmarcWindow(task.window_end)) + .await + } + Task::TlsReportWindow(task) => { + server + .submit_report(ReportId::TlsWindow(task.window_end)) + .await + } + Task::DestroyAccount(task) => server.destroy_account(task).await, + Task::AccountMaintenance(task) => server.account_maintenance(task).await, + Task::StoreMaintenance(task) => server.store_maintenance(task).await, + Task::SpamFilterMaintenance(task) => server.spam_filter_maintenance(task).await, + Task::AcmeRenewal(task) => server.acme_management(task).await, + Task::DkimManagement(task) => server.dkim_management(task).await, + Task::DnsManagement(task) => server.dns_management(task).await, + Task::ShareRevokeCleanup(task) => server.share_revoke_cleanup(task).await, + } +} diff --git a/crates/sieve/Cargo.toml b/crates/sieve/Cargo.toml new file mode 100644 index 00000000..15e37515 --- /dev/null +++ b/crates/sieve/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sieve-rs" +description = "Sieve filter interpreter" +version = "2.0.0" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[lib] +name = "sieve" +doctest = false + +[dependencies] +mail-parser = { version = "1.0", features = ["full_encoding"] } +fancy-regex = "0.20.0" +bumpalo = { version = "3.20", features = ["collections"] } +memchr = "2.8" +encodify = "1.0" +smallvec = "1.16" +hashbrown = "0.17.1" +hashify = "0.3" +serde = { version = "1.0", features = ["derive", "rc"], optional = true } +mail-builder = { version = "2.0" } +ahash = { version = "0.8.12" } + +[dev-dependencies] +mail-parser = { version = "1.0", features = ["full_encoding", "serde"] } +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = "1.0" +evalexpr = "13.1.0" + +[features] +default = [] +serde = ["dep:serde", "mail-parser/serde"] + +[lints] +workspace = true diff --git a/crates/sieve/src/bytecode/cursor.rs b/crates/sieve/src/bytecode/cursor.rs new file mode 100644 index 00000000..4738a857 --- /dev/null +++ b/crates/sieve/src/bytecode/cursor.rs @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Corrupt, Decoded, REC_LEN, + rec::{Range, Rec, Str}, +}; + +pub(crate) struct Cursor<'a> { + code: &'a [u8], + pub pos: usize, + trace: Option, +} + +#[derive(Default)] +pub(crate) struct Trace { + pub ranges: Vec, + pub strs: Vec, + pub recs: Vec, + pub jumps: Vec, + pub masks: Vec, +} + +impl<'a> Cursor<'a> { + pub(crate) fn new(code: &'a [u8], pos: usize) -> Self { + Cursor { + code, + pos, + trace: None, + } + } + + pub(crate) fn tracing(code: &'a [u8]) -> Self { + Cursor { + code, + pos: 0, + trace: Some(Trace::default()), + } + } + + pub(crate) fn trace_mut(&mut self) -> &mut Trace { + self.trace.get_or_insert_with(Trace::default) + } + + #[inline(always)] + pub(crate) fn at_end(&self) -> bool { + self.pos >= self.code.len() + } + + #[inline(always)] + fn take(&mut self) -> Decoded<&'a [u8; N]> { + let bytes = self + .code + .get(self.pos..) + .and_then(|s| s.first_chunk::()) + .ok_or(Corrupt)?; + self.pos += N; + Ok(bytes) + } + + #[inline(always)] + pub(crate) fn u8(&mut self) -> Decoded { + self.take::<1>().map(|b| b[0]) + } + + #[inline(always)] + pub(crate) fn bool(&mut self) -> Decoded { + self.u8().map(|b| b != 0) + } + + #[inline(always)] + pub(crate) fn u16(&mut self) -> Decoded { + self.take::<2>().map(|b| u16::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn u32(&mut self) -> Decoded { + self.take::<4>().map(|b| u32::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn i32(&mut self) -> Decoded { + self.take::<4>().map(|b| i32::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn u64(&mut self) -> Decoded { + self.take::<8>().map(|b| u64::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn i64(&mut self) -> Decoded { + self.take::<8>().map(|b| i64::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn rec(&mut self) -> Decoded { + let rec = Rec::decode(self.take::()?); + if let Some(trace) = &mut self.trace { + trace.recs.push(rec); + } + Ok(rec) + } + + #[inline(always)] + pub(crate) fn range(&mut self) -> Decoded { + let range = Range { + start: self.u32()?, + len: self.u32()?, + }; + if let Some(trace) = &mut self.trace { + trace.ranges.push(range); + } + Ok(range) + } + + #[inline(always)] + pub(crate) fn str(&mut self) -> Decoded { + let s = Str { + off: self.u32()?, + len: self.u32()?, + }; + if let Some(trace) = &mut self.trace { + trace.strs.push(s); + } + Ok(s) + } + + #[inline(always)] + pub(crate) fn mask(&mut self) -> Decoded { + let mask = self.u64()?; + if let Some(trace) = &mut self.trace { + trace.masks.push(mask); + } + Ok(mask) + } + + #[inline(always)] + pub(crate) fn jump(&mut self) -> Decoded { + let target = self.u32()?; + if let Some(trace) = &mut self.trace { + trace.jumps.push(target); + } + Ok(target) + } + + #[inline(always)] + pub(crate) fn opt_i32(&mut self) -> Decoded> { + let present = self.bool()?; + let value = self.i32()?; + Ok(present.then_some(value)) + } + + #[inline(always)] + pub(crate) fn opt_u32(&mut self) -> Decoded> { + let present = self.bool()?; + let value = self.u32()?; + Ok(present.then_some(value)) + } + + #[inline(always)] + pub(crate) fn opt_u64(&mut self) -> Decoded> { + let present = self.bool()?; + let value = self.u64()?; + Ok(present.then_some(value)) + } + + #[inline(always)] + pub(crate) fn opt_i64(&mut self) -> Decoded> { + let present = self.bool()?; + let value = self.i64()?; + Ok(present.then_some(value)) + } +} + +#[derive(Default, Debug)] +pub(crate) struct Code { + pub bytes: Vec, +} + +impl Code { + #[inline(always)] + pub(crate) fn pos(&self) -> u32 { + self.bytes.len() as u32 + } + + pub(crate) fn u8(&mut self, v: u8) { + self.bytes.push(v); + } + + pub(crate) fn bool(&mut self, v: bool) { + self.bytes.push(u8::from(v)); + } + + pub(crate) fn u16(&mut self, v: u16) { + self.bytes.extend_from_slice(&v.to_le_bytes()); + } + + pub(crate) fn u32(&mut self, v: u32) { + self.bytes.extend_from_slice(&v.to_le_bytes()); + } + + pub(crate) fn i32(&mut self, v: i32) { + self.bytes.extend_from_slice(&v.to_le_bytes()); + } + + pub(crate) fn u64(&mut self, v: u64) { + self.bytes.extend_from_slice(&v.to_le_bytes()); + } + + pub(crate) fn i64(&mut self, v: i64) { + self.bytes.extend_from_slice(&v.to_le_bytes()); + } + + pub(crate) fn rec(&mut self, v: Rec) { + self.bytes.extend_from_slice(&v.encode()); + } + + pub(crate) fn range(&mut self, v: Range) { + self.u32(v.start); + self.u32(v.len); + } + + pub(crate) fn str(&mut self, v: Str) { + self.u32(v.off); + self.u32(v.len); + } + + pub(crate) fn opt_i32(&mut self, v: Option) { + self.bool(v.is_some()); + self.i32(v.unwrap_or_default()); + } + + pub(crate) fn opt_u32(&mut self, v: Option) { + self.bool(v.is_some()); + self.u32(v.unwrap_or_default()); + } + + pub(crate) fn opt_u64(&mut self, v: Option) { + self.bool(v.is_some()); + self.u64(v.unwrap_or_default()); + } + + pub(crate) fn opt_i64(&mut self, v: Option) { + self.bool(v.is_some()); + self.i64(v.unwrap_or_default()); + } + + pub(crate) fn patch_u32(&mut self, at: u32, v: u32) { + let at = at as usize; + if let Some(slot) = self.bytes.get_mut(at..at + 4) { + slot.copy_from_slice(&v.to_le_bytes()); + } else { + debug_assert!(false, "patch outside of code section"); + } + } +} diff --git a/crates/sieve/src/bytecode/emit.rs b/crates/sieve/src/bytecode/emit.rs new file mode 100644 index 00000000..1139e408 --- /dev/null +++ b/crates/sieve/src/bytecode/emit.rs @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::header_id::{HEADER_OTHER, header_id}; +use super::{ + cursor::Code, + rec::{Range, Rec, Str}, +}; +use crate::{Sieve, runtime::tests::glob::CompiledGlob, sieve::LoadError}; +use hashbrown::{HashTable, hash_table::Entry}; +use mail_parser::HeaderName; + +#[derive(Default)] +pub(crate) struct Emitter { + pub code: Code, + records: Vec, + blob: String, + strings: HashTable, + header_names: Vec, + header_map: HashTable, + hasher: ahash::RandomState, + glob_offsets: Vec, + globs: Vec, + num_regexes: u32, +} + +impl Emitter { + pub(crate) fn new() -> Self { + Emitter { + code: Code { + bytes: Vec::with_capacity(512), + }, + records: Vec::with_capacity(512), + blob: String::with_capacity(512), + strings: HashTable::with_capacity(32), + header_map: HashTable::with_capacity(8), + header_names: Vec::with_capacity(8), + ..Default::default() + } + } + + pub(crate) fn str(&mut self, text: &str) -> Str { + let hash = self.hasher.hash_one(text); + let blob = &self.blob; + let hasher = &self.hasher; + match self.strings.entry( + hash, + |s| blob_str(blob, *s) == text, + |s| hasher.hash_one(blob_str(blob, *s)), + ) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let s = Str { + off: self.blob.len() as u32, + len: text.len() as u32, + }; + entry.insert(s); + self.blob.push_str(text); + s + } + } + } + + pub(crate) fn num_records(&self) -> u32 { + (self.records.len() / super::REC_LEN) as u32 + } + + pub(crate) fn push_recs(&mut self, recs: &[Rec]) -> Range { + let start = self.num_records(); + self.records.reserve(recs.len() * super::REC_LEN); + for rec in recs { + self.records.extend_from_slice(&rec.encode()); + } + Range { + start, + len: recs.len() as u32, + } + } + + pub(crate) fn header_name(&mut self, name: &HeaderName<'_>) -> u16 { + let entry = match header_id(name) { + Some(id) => HeaderEntry::Known(id), + None => HeaderEntry::Other(self.str(name.as_str())), + }; + let hash = self.hasher.hash_one(name.as_str()); + let blob = &self.blob; + let hasher = &self.hasher; + let header_names = &self.header_names; + match self.header_map.entry( + hash, + |index| header_names.get(*index as usize) == Some(&entry), + |index| { + hasher.hash_one( + header_names + .get(*index as usize) + .map_or("", |entry| entry.as_str(blob)), + ) + }, + ) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(slot) => { + let index = self.header_names.len() as u16; + slot.insert(index); + self.header_names.push(entry); + index + } + } + } + + pub(crate) fn glob(&mut self, glob: &CompiledGlob) -> u16 { + let index = self.glob_offsets.len() as u16; + let shape = glob + .shape() + .map(|(kind, literal)| (kind, self.str(literal))); + let mut out = std::mem::take(&mut self.globs); + self.glob_offsets.push(out.len() as u32); + out.push(u8::from(glob.to_lower())); + out.push(u8::from(glob.is_ascii())); + let (kind, literal) = shape.unwrap_or((0, Str::default())); + out.push(kind); + out.extend_from_slice(&literal.off.to_le_bytes()); + out.extend_from_slice(&literal.len.to_le_bytes()); + let count_at = out.len(); + out.extend_from_slice(&0u32.to_le_bytes()); + let mut count = 0u32; + for ch in glob.encoded_chars() { + out.extend_from_slice(&ch.to_le_bytes()); + count += 1; + } + out[count_at..count_at + 4].copy_from_slice(&count.to_le_bytes()); + self.globs = out; + index + } + + pub(crate) fn regex_slot(&mut self) -> u16 { + let slot = self.num_regexes as u16; + self.num_regexes += 1; + slot + } + + pub(crate) fn finish( + self, + num_vars: u16, + num_match_vars: u16, + ) -> Result, LoadError> { + let mut header_names = Vec::with_capacity(4 + self.header_names.len() * 9); + if !self.header_names.is_empty() { + header_names.extend_from_slice(&(self.header_names.len() as u32).to_le_bytes()); + for entry in &self.header_names { + match entry { + HeaderEntry::Known(id) => header_names.push(*id), + HeaderEntry::Other(s) => { + header_names.push(HEADER_OTHER); + header_names.extend_from_slice(&s.off.to_le_bytes()); + header_names.extend_from_slice(&s.len.to_le_bytes()); + } + } + } + } + let mut globs = Vec::with_capacity(4 + self.glob_offsets.len() * 4 + self.globs.len()); + if !self.glob_offsets.is_empty() { + let table_len = 4 + self.glob_offsets.len() as u32 * 4; + globs.extend_from_slice(&(self.glob_offsets.len() as u32).to_le_bytes()); + for offset in &self.glob_offsets { + globs.extend_from_slice(&(offset + table_len).to_le_bytes()); + } + globs.extend_from_slice(&self.globs); + } + Sieve::from_parts( + self.code.bytes, + self.records, + self.blob, + header_names, + globs, + self.num_regexes, + num_vars, + num_match_vars, + ) + } +} + +fn blob_str(blob: &str, s: Str) -> &str { + blob.get(s.off as usize..(s.off + s.len) as usize) + .unwrap_or("") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HeaderEntry { + Known(u8), + Other(Str), +} + +impl HeaderEntry { + fn as_str<'a>(&self, blob: &'a str) -> &'a str { + match self { + HeaderEntry::Known(id) => super::header_id::header_from_id(*id) + .map(|name| name.as_static_str()) + .unwrap_or(""), + HeaderEntry::Other(s) => blob_str(blob, *s), + } + } +} diff --git a/crates/sieve/src/bytecode/header_id.rs b/crates/sieve/src/bytecode/header_id.rs new file mode 100644 index 00000000..9253ad5d --- /dev/null +++ b/crates/sieve/src/bytecode/header_id.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use mail_parser::HeaderName; + +pub(crate) const HEADER_OTHER: u8 = 0xff; + +macro_rules! header_ids { + ($($id:literal => $variant:ident),* $(,)?) => { + pub(crate) fn header_id(name: &HeaderName<'_>) -> Option { + match name { + $(HeaderName::$variant => Some($id),)* + _ => None, + } + } + + pub(crate) fn header_from_id(id: u8) -> Option> { + match id { + $($id => Some(HeaderName::$variant),)* + _ => None, + } + } + }; +} + +header_ids! { + 0 => Subject, + 1 => From, + 2 => To, + 3 => Cc, + 4 => Date, + 5 => Bcc, + 6 => ReplyTo, + 7 => Sender, + 8 => Comments, + 9 => InReplyTo, + 10 => Keywords, + 11 => Received, + 12 => MessageId, + 13 => References, + 14 => ReturnPath, + 15 => MimeVersion, + 16 => ContentDescription, + 17 => ContentId, + 18 => ContentLanguage, + 19 => ContentLocation, + 20 => ContentTransferEncoding, + 21 => ContentType, + 22 => ContentDisposition, + 23 => ResentTo, + 24 => ResentFrom, + 25 => ResentBcc, + 26 => ResentCc, + 27 => ResentSender, + 28 => ResentDate, + 29 => ResentMessageId, + 30 => ListArchive, + 31 => ListHelp, + 32 => ListId, + 33 => ListOwner, + 34 => ListPost, + 35 => ListSubscribe, + 36 => ListUnsubscribe, + 37 => DkimSignature, + 38 => ArcAuthenticationResults, + 39 => ArcMessageSignature, + 40 => ArcSeal, + 41 => Dkim2Signature, + 42 => MessageInstance, + 43 => AcceptLanguage, + 44 => AlternateRecipient, + 45 => ArchivedAt, + 46 => AuthenticationResults, + 47 => AutoSubmitted, + 48 => Autoforwarded, + 49 => Autosubmitted, + 50 => ContentAlternative, + 51 => ContentDuration, + 52 => ContentFeatures, + 53 => ContentMd5, + 54 => ContentTranslationType, + 55 => Conversion, + 56 => ConversionWithLoss, + 57 => DlExpansionHistory, + 58 => DeferredDelivery, + 59 => DeliveryDate, + 60 => DiscardedX400IpmsExtensions, + 61 => DiscardedX400MtsExtensions, + 62 => DiscloseRecipients, + 63 => DispositionNotificationOptions, + 64 => DispositionNotificationTo, + 65 => DowngradedFinalRecipient, + 66 => DowngradedInReplyTo, + 67 => DowngradedMessageId, + 68 => DowngradedOriginalRecipient, + 69 => DowngradedReferences, + 70 => Encoding, + 71 => Expires, + 72 => GenerateDeliveryReport, + 73 => HpOuter, + 74 => Importance, + 75 => IncompleteCopy, + 76 => Language, + 77 => LatestDeliveryTime, + 78 => ListUnsubscribePost, + 79 => MessageContext, + 80 => MessageType, + 81 => MmhsExemptedAddress, + 82 => MmhsExtendedAuthorisationInfo, + 83 => MmhsSubjectIndicatorCodes, + 84 => MmhsHandlingInstructions, + 85 => MmhsMessageInstructions, + 86 => MmhsCodressMessageIndicator, + 87 => MmhsOriginatorReference, + 88 => MmhsPrimaryPrecedence, + 89 => MmhsCopyPrecedence, + 90 => MmhsMessageType, + 91 => MmhsOtherRecipientsIndicatorTo, + 92 => MmhsOtherRecipientsIndicatorCc, + 93 => MmhsAcp127MessageIdentifier, + 94 => MmhsOriginatorPlad, + 95 => MtPriority, + 96 => Organization, + 97 => OriginalEncodedInformationTypes, + 98 => OriginalFrom, + 99 => OriginalMessageId, + 100 => OriginalRecipient, + 101 => OriginatorReturnAddress, + 102 => OriginalSubject, + 103 => PicsLabel, + 104 => PreventNonDeliveryReport, + 105 => Priority, + 106 => ReceivedSpf, + 107 => ReplyBy, + 108 => RequireRecipientValidSince, + 109 => Sensitivity, + 110 => Solicitation, + 111 => Supersedes, + 112 => TlsReportDomain, + 113 => TlsReportSubmitter, + 114 => TlsRequired, + 115 => VbrInfo, + 116 => X400ContentIdentifier, + 117 => X400ContentReturn, + 118 => X400ContentType, + 119 => X400MtsIdentifier, + 120 => X400Originator, + 121 => X400Received, + 122 => X400Recipients, + 123 => X400Trace, + 124 => ApparentlyTo, + 125 => Author, + 126 => CfblAddress, + 127 => CfblFeedbackId, + 128 => DeliveredTo, + 129 => EdiintFeatures, + 130 => EesstVersion, + 131 => ErrorsTo, + 132 => Face, + 133 => FormSub, + 134 => JabberId, + 135 => MmhsAuthorizingUsers, + 136 => Privicon, + 137 => SioLabel, + 138 => SioLabelHistory, + 139 => WrongRecipient, +} + diff --git a/crates/sieve/src/bytecode/mod.rs b/crates/sieve/src/bytecode/mod.rs new file mode 100644 index 00000000..2995b9df --- /dev/null +++ b/crates/sieve/src/bytecode/mod.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) mod cursor; +pub(crate) mod emit; +pub(crate) mod header_id; +pub(crate) mod ops; +pub(crate) mod rec; +pub(crate) mod verify; + +use crate::LoadError; + +pub(crate) const MAGIC: u32 = 0x5645_4953; +pub(crate) const FORMAT_VERSION: u16 = 1; +pub(crate) const HEADER_LEN: usize = 40; +pub(crate) const REC_LEN: usize = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Corrupt; + +pub(crate) type Decoded = Result; + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct Sections { + pub num_vars: u16, + pub num_match_vars: u16, + pub code: (usize, usize), + pub records: (usize, usize), + pub blob: (usize, usize), + pub header_names: (usize, usize), + pub globs: (usize, usize), + pub num_regexes: u32, +} + +impl Sections { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let head = bytes + .first_chunk::() + .ok_or(LoadError::Truncated)?; + let u16_at = |at: usize| u16::from_le_bytes([head[at], head[at + 1]]); + let u32_at = + |at: usize| u32::from_le_bytes([head[at], head[at + 1], head[at + 2], head[at + 3]]); + if u32_at(0) != MAGIC { + return Err(LoadError::Corrupted); + } + let version = u16_at(4); + if version != FORMAT_VERSION { + return Err(LoadError::UnsupportedVersion(version)); + } + let mut start = HEADER_LEN; + let mut section = |len: u32| -> Result<(usize, usize), LoadError> { + let len = len as usize; + let end = start.checked_add(len).ok_or(LoadError::Corrupted)?; + if end > bytes.len() { + return Err(LoadError::Truncated); + } + let range = (start, end); + start = end; + Ok(range) + }; + let code = section(u32_at(12))?; + let records = section(u32_at(16))?; + let blob = section(u32_at(20))?; + let header_names = section(u32_at(24))?; + let globs = section(u32_at(28))?; + let num_records = records.1 - records.0; + let num_regexes = u32_at(32); + if num_records % REC_LEN != 0 || num_regexes as usize > num_records / REC_LEN { + return Err(LoadError::Corrupted); + } + Ok(Sections { + num_vars: u16_at(8), + num_match_vars: u16_at(10), + code, + records, + blob, + header_names, + globs, + num_regexes, + }) + } + + pub(crate) fn write_header(&self, out: &mut Vec) { + out.extend_from_slice(&MAGIC.to_le_bytes()); + out.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&self.num_vars.to_le_bytes()); + out.extend_from_slice(&self.num_match_vars.to_le_bytes()); + for (start, end) in [ + self.code, + self.records, + self.blob, + self.header_names, + self.globs, + ] { + out.extend_from_slice(&((end - start) as u32).to_le_bytes()); + } + out.extend_from_slice(&self.num_regexes.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); + } +} diff --git a/crates/sieve/src/bytecode/ops.rs b/crates/sieve/src/bytecode/ops.rs new file mode 100644 index 00000000..e107f040 --- /dev/null +++ b/crates/sieve/src/bytecode/ops.rs @@ -0,0 +1,244 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Corrupt, Decoded, + cursor::{Code, Cursor}, + rec::{Range, Rec, Str}, +}; + +pub(crate) trait Field: Sized { + fn read(c: &mut Cursor<'_>) -> Decoded; + fn write(&self, w: &mut Code); +} + +macro_rules! primitive_field { + ($($ty:ty => $read:ident),* $(,)?) => { + $( + impl Field for $ty { + #[inline(always)] + fn read(c: &mut Cursor<'_>) -> Decoded { + c.$read() + } + + fn write(&self, w: &mut Code) { + w.$read(*self); + } + } + )* + }; +} + +primitive_field!( + u8 => u8, + bool => bool, + u16 => u16, + u32 => u32, + i32 => i32, + u64 => u64, + i64 => i64, + Rec => rec, + Range => range, + Str => str, + Option => opt_i32, + Option => opt_u32, + Option => opt_u64, + Option => opt_i64, +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Jump(pub u32); + +impl Field for Jump { + #[inline(always)] + fn read(c: &mut Cursor<'_>) -> Decoded { + c.jump().map(Jump) + } + + fn write(&self, w: &mut Code) { + w.u32(self.0); + } +} + +macro_rules! composite_field { + ($( $name:ident { $( $field:ident : $ty:ty ),* $(,)? } )*) => { + $( + #[derive(Debug, Clone, Copy, PartialEq, Default)] + pub(crate) struct $name { + $( pub $field: $ty, )* + } + + impl Field for $name { + #[inline(always)] + fn read(c: &mut Cursor<'_>) -> Decoded { + Ok(Self { + $( $field: <$ty as Field>::read(c)?, )* + }) + } + + fn write(&self, w: &mut Code) { + $( <$ty as Field>::write(&self.$field, w); )* + } + } + )* + }; +} + +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub(crate) struct Match { + pub kind: u8, + pub arg: u64, +} + +impl Field for Match { + #[inline(always)] + fn read(c: &mut Cursor<'_>) -> Decoded { + let kind = c.u8()?; + let arg = if matches!(kind, match_kind::MATCHES | match_kind::REGEX) { + c.mask()? + } else { + c.u64()? + }; + Ok(Match { kind, arg }) + } + + fn write(&self, w: &mut Code) { + w.u8(self.kind); + w.u64(self.arg); + } +} + +composite_field! { + MimeOpts { kind: u8, params: Range } + Fcc { present: bool, mailbox: Rec, mailbox_id: Rec, create: bool, flags: Range, special_use: Rec } + ByTime { kind: u8, mode: u8, trace: bool, rlimit: u64, alimit: Rec } + Zone { kind: u8, time: i64 } + Period { kind: u8, value: u64 } + DupMatch { kind: u8, value: Rec } + MetadataRef { kind: u8, name: Rec, annotation: Rec } + NotifySpec { kind: u8, items: Range } + BodyTransform { kind: u8, content_types: Range } +} + +pub(crate) mod match_kind { + pub const IS: u8 = 0; + pub const CONTAINS: u8 = 1; + pub const MATCHES: u8 = 2; + pub const REGEX: u8 = 3; + pub const VALUE: u8 = 4; + pub const COUNT: u8 = 5; + pub const LIST: u8 = 6; +} + +macro_rules! ops { + ($( $name:ident = $code:expr => { $( $field:ident : $ty:ty ),* $(,)? } )*) => { + $( + #[derive(Debug, Clone, Copy, PartialEq, Default)] + pub(crate) struct $name { + $( pub $field: $ty, )* + } + + impl $name { + pub(crate) const OP: u8 = $code; + + #[inline(always)] + pub(crate) fn decode(c: &mut Cursor<'_>) -> Decoded { + let _ = &c; + Ok(Self { + $( $field: <$ty as Field>::read(c)?, )* + }) + } + + pub(crate) fn emit(&self, w: &mut Code) { + w.u8(Self::OP); + $( <$ty as Field>::write(&self.$field, w); )* + } + } + )* + + pub(crate) fn skip_op(op: u8, c: &mut Cursor<'_>) -> Decoded<()> { + match op { + $( $code => $name::decode(c).map(|_| ()), )* + _ => Err(Corrupt), + } + } + }; +} + +ops! { + Require = 0 => { capabilities: Range } + Keep = 1 => { flags: Range } + FileInto = 2 => { copy: bool, create: bool, folder: Rec, flags: Range, mailbox_id: Rec, special_use: Rec } + Redirect = 3 => { copy: bool, list: bool, address: Rec, notify: NotifySpec, ret: u8, by_time: ByTime } + Discard = 4 => {} + Stop = 5 => {} + Invalid = 6 => { name: Str, line_num: u32, line_pos: u32 } + TestTrue = 7 => {} + TestFalse = 8 => {} + TestAddress = 9 => { header_list: Range, key_list: Range, address_part: u8, match_type: Match, comparator: u8, index: Option, mime_anychild: bool, is_not: bool } + TestEnvelope = 10 => { envelope_list: Range, key_list: Range, address_part: u8, match_type: Match, comparator: u8, zone: Option, is_not: bool } + TestExists = 11 => { header_names: Range, mime_anychild: bool, is_not: bool } + TestHeader = 12 => { header_list: Range, key_list: Range, match_type: Match, comparator: u8, index: Option, mime_opts: MimeOpts, mime_anychild: bool, is_not: bool } + TestSize = 13 => { over: bool, limit: u64, is_not: bool } + TestInvalid = 14 => { name: Str, line_num: u32, line_pos: u32 } + TestBody = 15 => { key_list: Range, body_transform: BodyTransform, match_type: Match, comparator: u8, include_subject: bool, is_not: bool } + TestConvert = 16 => { from_media_type: Rec, to_media_type: Rec, transcoding_params: Range, is_not: bool } + Convert = 17 => { from_media_type: Rec, to_media_type: Rec, transcoding_params: Range, is_not: bool } + TestDate = 18 => { header_name: Rec, key_list: Range, match_type: Match, comparator: u8, index: Option, zone: Zone, date_part: u8, mime_anychild: bool, is_not: bool } + TestCurrentDate = 19 => { zone: Option, match_type: Match, comparator: u8, date_part: u8, key_list: Range, is_not: bool } + TestDuplicate = 20 => { handle: Rec, dup_match: DupMatch, seconds: Option, last: bool, is_not: bool } + TestString = 21 => { match_type: Match, comparator: u8, source: Range, key_list: Range, is_not: bool } + TestEnvironment = 22 => { match_type: Match, comparator: u8, source: Range, key_list: Range, is_not: bool } + TestNotifyMethodCapability = 23 => { comparator: u8, match_type: Match, notification_uri: Rec, notification_capability: Rec, key_list: Range, is_not: bool } + TestValidNotifyMethod = 24 => { notification_uris: Range, is_not: bool } + TestValidExtList = 25 => { list_names: Range, is_not: bool } + TestIhave = 26 => { capabilities: Range, is_not: bool } + TestHasFlag = 27 => { comparator: u8, match_type: Match, variable_list: Range, flags: Range, is_not: bool } + TestMailboxExists = 28 => { mailbox_names: Range, is_not: bool } + TestMetadata = 29 => { match_type: Match, comparator: u8, metadata: MetadataRef, key_list: Range, is_not: bool } + TestMetadataExists = 30 => { mailbox: Rec, annotation_names: Range, is_not: bool } + TestMailboxIdExists = 31 => { mailbox_ids: Range, is_not: bool } + TestSpamTest = 32 => { value: Rec, match_type: Match, comparator: u8, percent: bool, is_not: bool } + TestVirusTest = 33 => { value: Rec, match_type: Match, comparator: u8, is_not: bool } + TestSpecialUseExists = 34 => { mailbox: Rec, attributes: Range, is_not: bool } + TestVacation = 35 => { addresses: Range, period: Period, handle: Rec, reason: Rec } + Jmp = 36 => { target: Jump } + Jz = 37 => { target: Jump } + Jnz = 38 => { target: Jump } + ForEveryPartPush = 39 => {} + ForEveryPart = 40 => { jz_pos: Jump } + ForEveryPartPop = 41 => { num_pops: u32 } + Replace = 42 => { subject: Rec, from: Rec, replacement: Rec, mime: bool } + Enclose = 43 => { subject: Rec, headers: Range, value: Rec } + ExtractText = 44 => { modifiers: Range, first: Option, name: Rec } + AddHeader = 45 => { last: bool, field_name: Rec, value: Rec } + DeleteHeader = 46 => { index: Option, comparator: u8, match_type: Match, field_name: Rec, value_patterns: Range, mime_anychild: bool } + Set = 47 => { modifiers: Range, name: Rec, value: Rec } + Clear = 48 => { local_vars_idx: u32, local_vars_num: u32, match_vars: u64 } + Notify = 49 => { from: Rec, importance: Rec, options: Range, message: Rec, fcc: Fcc, method: Rec } + Reject = 50 => { ereject: bool, reason: Rec } + Vacation = 51 => { subject: Rec, from: Rec, mime: bool, fcc: Fcc, reason: Rec } + Error = 52 => { message: Rec } + EditFlags = 53 => { action: u8, name: Rec, flags: Range } + Include = 54 => { global: bool, once: bool, optional: bool, value: Rec } + Return = 55 => {} + While = 56 => { expr: Range, jz_pos: Jump } + Eval = 57 => { expr: Range } + Let = 58 => { name: Rec, expr: Range } + TestCmd = 59 => { arguments: Range } + TestCmdTest = 60 => { arguments: Range, is_not: bool } +} + +impl From for Convert { + fn from(test: TestConvert) -> Self { + Convert { + from_media_type: test.from_media_type, + to_media_type: test.to_media_type, + transcoding_params: test.transcoding_params, + is_not: test.is_not, + } + } +} diff --git a/crates/sieve/src/bytecode/rec.rs b/crates/sieve/src/bytecode/rec.rs new file mode 100644 index 00000000..a9745bb1 --- /dev/null +++ b/crates/sieve/src/bytecode/rec.rs @@ -0,0 +1,128 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::REC_LEN; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Rec { + pub tag: u8, + pub b: u8, + pub c: u16, + pub d: u32, + pub e: u64, +} + +impl Rec { + pub(crate) const fn tagged(tag: u8) -> Self { + Rec { + tag, + b: 0, + c: 0, + d: 0, + e: 0, + } + } + + pub(crate) fn encode(self) -> [u8; REC_LEN] { + let mut out = [0u8; REC_LEN]; + out[0] = self.tag; + out[1] = self.b; + out[2..4].copy_from_slice(&self.c.to_le_bytes()); + out[4..8].copy_from_slice(&self.d.to_le_bytes()); + out[8..16].copy_from_slice(&self.e.to_le_bytes()); + out + } + + pub(crate) fn decode(bytes: &[u8; REC_LEN]) -> Self { + Rec { + tag: bytes[0], + b: bytes[1], + c: u16::from_le_bytes([bytes[2], bytes[3]]), + d: u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + e: u64::from_le_bytes([ + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], + bytes[15], + ]), + } + } + + pub(crate) fn str(self) -> Str { + Str { + off: self.d, + len: self.e as u32, + } + } + + pub(crate) fn with_str(mut self, s: Str) -> Self { + self.d = s.off; + self.e = s.len as u64; + self + } + + pub(crate) fn range(self) -> Range { + Range { + start: self.d, + len: self.e as u32, + } + } + + pub(crate) fn with_range(mut self, r: Range) -> Self { + self.d = r.start; + self.e = r.len as u64; + self + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Str { + pub off: u32, + pub len: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Range { + pub start: u32, + pub len: u32, +} + +impl Range { + pub(crate) const EMPTY: Range = Range { start: 0, len: 0 }; + + pub(crate) fn is_empty(self) -> bool { + self.len == 0 + } +} + +pub(crate) mod tag { + pub const NONE: u8 = 0; + pub const TEXT: u8 = 1; + pub const INT: u8 = 2; + pub const FLOAT: u8 = 3; + pub const VAR_LOCAL: u8 = 4; + pub const VAR_MATCH: u8 = 5; + pub const VAR_GLOBAL: u8 = 6; + pub const VAR_ENV: u8 = 7; + pub const VAR_ENVELOPE: u8 = 8; + pub const VAR_PART: u8 = 9; + pub const VAR_HEADER: u8 = 10; + pub const CONT: u8 = 11; + pub const REGEX: u8 = 12; + pub const GLOB: u8 = 13; + pub const HEADER: u8 = 14; + pub const LIST: u8 = 15; + pub const REF: u8 = 16; + pub const BIN_OP: u8 = 20; + pub const UN_OP: u8 = 21; + pub const JMP_IF: u8 = 22; + pub const CALL: u8 = 23; + pub const ARRAY_ACCESS: u8 = 24; + pub const ARRAY_BUILD: u8 = 25; + pub const MODIFIER: u8 = 30; + pub const CAPABILITY: u8 = 31; + pub const ENVELOPE: u8 = 32; + pub const NOTIFY_ITEM: u8 = 33; + pub const VARIABLE_NONE: u8 = 34; +} diff --git a/crates/sieve/src/bytecode/verify.rs b/crates/sieve/src/bytecode/verify.rs new file mode 100644 index 00000000..b1593da0 --- /dev/null +++ b/crates/sieve/src/bytecode/verify.rs @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Corrupt, Decoded, + cursor::Cursor, + ops::{self, skip_op}, + rec::{Range, Rec, tag}, +}; +use crate::{ + Sieve, + compiler::grammar::{actions::action_set::MODIFIER_REPLACE, expr::parser::ID_EXTERNAL}, +}; + +const MAX_NESTING: u8 = 8; + +pub(crate) fn verify(sieve: &Sieve<'_>) -> Decoded<()> { + let code = sieve.code(); + let mut cursor = Cursor::tracing(code); + let mut boundaries = Boundaries::new(code.len() + 1); + let mut jumps = Vec::new(); + + while !cursor.at_end() { + boundaries.set(cursor.pos); + let op = cursor.u8()?; + if op == ops::Clear::OP { + verify_clear(sieve, &mut Cursor::new(code, cursor.pos))?; + } + skip_op(op, &mut cursor)?; + let trace = cursor.trace_mut(); + for mask in trace.masks.drain(..) { + verify_mask(sieve, mask)?; + } + for range in trace.ranges.drain(..) { + verify_range(sieve, range, 0)?; + } + for s in trace.strs.drain(..) { + sieve.str(s)?; + } + for rec in trace.recs.drain(..) { + let mut iter = std::iter::empty(); + verify_rec(sieve, rec, &mut iter, 0)?; + } + jumps.append(&mut trace.jumps); + } + boundaries.set(code.len()); + + if jumps.iter().all(|target| boundaries.get(*target as usize)) { + Ok(()) + } else { + Err(Corrupt) + } +} + +fn verify_clear(sieve: &Sieve<'_>, cursor: &mut Cursor<'_>) -> Decoded<()> { + let clear = ops::Clear::decode(cursor)?; + let end = clear + .local_vars_idx + .checked_add(clear.local_vars_num) + .ok_or(Corrupt)?; + if end as usize > sieve.num_vars() { + return Err(Corrupt); + } + verify_mask(sieve, clear.match_vars) +} + +fn verify_mask(sieve: &Sieve<'_>, mask: u64) -> Decoded<()> { + let highest = (u64::BITS - mask.leading_zeros()) as usize; + if highest <= sieve.num_match_vars() { + Ok(()) + } else { + Err(Corrupt) + } +} + +struct Boundaries { + bits: Vec, +} + +impl Boundaries { + fn new(len: usize) -> Self { + Boundaries { + bits: vec![0; len.div_ceil(64)], + } + } + + #[inline(always)] + fn set(&mut self, at: usize) { + if let Some(word) = self.bits.get_mut(at / 64) { + *word |= 1 << (at % 64); + } + } + + #[inline(always)] + fn get(&self, at: usize) -> bool { + self.bits + .get(at / 64) + .is_some_and(|word| word & (1 << (at % 64)) != 0) + } +} + +fn verify_range(sieve: &Sieve<'_>, range: Range, depth: u8) -> Decoded<()> { + if depth > MAX_NESTING { + return Err(Corrupt); + } + let mut iter = sieve.recs(range)?; + while let Some(rec) = iter.next() { + if rec.tag == tag::JMP_IF { + let remaining = range.start + range.len - iter.index; + if rec.d > remaining { + return Err(Corrupt); + } + } + verify_rec(sieve, rec, &mut iter, depth)?; + } + Ok(()) +} + +fn verify_rec( + sieve: &Sieve<'_>, + rec: Rec, + following: &mut impl Iterator, + depth: u8, +) -> Decoded<()> { + match rec.tag { + tag::CALL => { + if rec.d <= ID_EXTERNAL { + Ok(()) + } else { + Err(Corrupt) + } + } + tag::VAR_LOCAL => { + if (rec.c as usize) < sieve.num_vars() { + Ok(()) + } else { + Err(Corrupt) + } + } + tag::VAR_MATCH => { + if (rec.b as usize) < sieve.num_match_vars() { + Ok(()) + } else { + Err(Corrupt) + } + } + tag::NONE + | tag::INT + | tag::FLOAT + | tag::VAR_ENVELOPE + | tag::VAR_PART + | tag::BIN_OP + | tag::UN_OP + | tag::JMP_IF + | tag::ARRAY_ACCESS + | tag::ARRAY_BUILD + | tag::ENVELOPE + | tag::NOTIFY_ITEM + | tag::VARIABLE_NONE => Ok(()), + tag::TEXT | tag::VAR_GLOBAL | tag::VAR_ENV => sieve.str(rec.str()).map(|_| ()), + tag::REGEX => { + sieve.str(rec.str())?; + if (rec.c as u32) < sieve.num_regexes() { + Ok(()) + } else { + Err(Corrupt) + } + } + tag::GLOB => { + sieve.str(rec.str())?; + sieve.glob(rec.c).map(|_| ()) + } + tag::HEADER => sieve.header_name(rec.c).map(|_| ()), + tag::LIST => verify_range(sieve, rec.range(), depth + 1), + tag::REF => verify_range( + sieve, + Range { + start: rec.d, + len: (rec.e as u32).saturating_add(1), + }, + depth + 1, + ), + tag::VAR_HEADER => { + let cont = following.next().ok_or(Corrupt)?; + if cont.tag != tag::CONT { + return Err(Corrupt); + } + let names = Range { + start: rec.d, + len: cont.c as u32, + }; + for name in sieve.recs(names)? { + if name.tag != tag::HEADER { + return Err(Corrupt); + } + sieve.header_name(name.c)?; + } + if cont.e != 0 { + sieve.str(cont.str())?; + } + Ok(()) + } + tag::CONT => Err(Corrupt), + tag::MODIFIER => { + if rec.b == MODIFIER_REPLACE { + for _ in 0..2 { + let value = following.next().ok_or(Corrupt)?; + verify_rec(sieve, value, &mut std::iter::empty(), depth + 1)?; + } + } + Ok(()) + } + tag::CAPABILITY => { + if rec.e != 0 { + sieve.str(rec.str())?; + } + Ok(()) + } + _ => Err(Corrupt), + } +} diff --git a/crates/sieve/src/compiler/emit.rs b/crates/sieve/src/compiler/emit.rs new file mode 100644 index 00000000..36ad86d2 --- /dev/null +++ b/crates/sieve/src/compiler/emit.rs @@ -0,0 +1,1132 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ContentTypePart, HeaderPart, Number, Value, VariableType, + grammar::{ + Capability, Comparator, + actions::{ + action_mime::MimeOpts, + action_redirect::{ByTime, Notify}, + action_set::Modifier, + action_vacation::Period, + }, + expr::Expression, + instruction::Instruction, + test::Test, + tests::{test_body::BodyTransform, test_date::Zone, test_duplicate::DupMatch}, + }, +}; +use crate::{ + FileCarbonCopy, Metadata, Sieve, + bytecode::{ + emit::Emitter, + ops::{self, Field, Match}, + rec::{Range, Rec, Str, tag}, + }, + sieve::LoadError, +}; + +struct Codegen<'c> { + emitter: Emitter, + constants: &'c [String], + constant_strs: Vec>, + patches: Vec<(u32, u32)>, + scratch: Vec, +} + +pub(crate) fn emit( + instructions: &[Instruction], + constants: &[String], + num_vars: u32, + num_match_vars: u32, +) -> Result, LoadError> { + let mut codegen = Codegen { + emitter: Emitter::new(), + constants, + constant_strs: vec![None; constants.len()], + patches: Vec::with_capacity(16), + scratch: Vec::with_capacity(64), + }; + let mut offsets = Vec::with_capacity(instructions.len() + 1); + for instruction in instructions { + offsets.push(codegen.emitter.code.pos()); + codegen.instruction(instruction); + } + offsets.push(codegen.emitter.code.pos()); + for (at, target) in std::mem::take(&mut codegen.patches) { + let target = offsets + .get(target as usize) + .copied() + .unwrap_or(codegen.emitter.code.pos()); + codegen.emitter.code.patch_u32(at, target); + } + codegen + .emitter + .finish(num_vars as u16, num_match_vars as u16) +} + +impl Codegen<'_> { + fn str(&mut self, text: &str) -> Str { + self.emitter.str(text) + } + + fn constant(&mut self, id: super::ConstantId) -> Str { + if let Some(Some(s)) = self.constant_strs.get(id.index()) { + return *s; + } + let text = self + .constants + .get(id.index()) + .map(|c| c.as_str()) + .unwrap_or(""); + let s = self.emitter.str(text); + if let Some(slot) = self.constant_strs.get_mut(id.index()) { + *slot = Some(s); + } + s + } + + fn flush_scratch(&mut self, start: usize) -> Range { + let range = self.emitter.push_recs(&self.scratch[start..]); + self.scratch.truncate(start); + range + } + + fn jump(&mut self, target: u32) -> ops::Jump { + self.patches.push((self.emitter.code.pos() + 1, target)); + ops::Jump(u32::MAX) + } + + fn jump_at(&mut self, offset_in_body: u32, target: u32) -> ops::Jump { + self.patches + .push((self.emitter.code.pos() + 1 + offset_in_body, target)); + ops::Jump(u32::MAX) + } + + fn value_recs(&mut self, value: &Value) { + match value { + Value::Text(id) => { + let s = self.constant(*id); + self.scratch.push(Rec::tagged(tag::TEXT).with_str(s)); + } + Value::Number(Number::Integer(n)) => self.scratch.push(Rec { + e: *n as u64, + ..Rec::tagged(tag::INT) + }), + Value::Number(Number::Float(n)) => self.scratch.push(Rec { + e: n.to_bits(), + ..Rec::tagged(tag::FLOAT) + }), + Value::Variable(var) => self.variable_recs(var), + Value::Regex(regex) => { + let s = self.str(®ex.expr); + let slot = self.emitter.regex_slot(); + self.scratch.push(Rec { + c: slot, + ..Rec::tagged(tag::REGEX).with_str(s) + }); + } + Value::Glob(glob) => { + let s = self.str(&glob.expr); + let index = self.emitter.glob(&glob.glob); + self.scratch.push(Rec { + c: index, + ..Rec::tagged(tag::GLOB).with_str(s) + }); + } + Value::Header(name) => { + let index = self.emitter.header_name(name); + self.scratch.push(Rec { + c: index, + ..Rec::tagged(tag::HEADER) + }); + } + Value::List(items) => { + let range = self.list(items); + self.scratch.push(Rec::tagged(tag::LIST).with_range(range)); + } + } + } + + fn variable_recs(&mut self, var: &VariableType) { + match var { + VariableType::Local(id) => self.scratch.push(Rec { + c: *id, + ..Rec::tagged(tag::VAR_LOCAL) + }), + VariableType::Match(id) => self.scratch.push(Rec { + b: *id, + ..Rec::tagged(tag::VAR_MATCH) + }), + VariableType::Global(name) => { + let s = self.str(name); + self.scratch.push(Rec::tagged(tag::VAR_GLOBAL).with_str(s)); + } + VariableType::Environment(name) => { + let s = self.str(name); + self.scratch.push(Rec::tagged(tag::VAR_ENV).with_str(s)); + } + VariableType::Envelope(envelope) => self.scratch.push(Rec { + b: *envelope as u8, + ..Rec::tagged(tag::VAR_ENVELOPE) + }), + VariableType::Part(part) => { + let (kind, convert) = match part { + super::MessagePart::TextBody(convert) => (0, *convert), + super::MessagePart::HtmlBody(convert) => (1, *convert), + super::MessagePart::Contents => (2, false), + super::MessagePart::Raw => (3, false), + }; + self.scratch.push(Rec { + b: kind, + c: u16::from(convert), + ..Rec::tagged(tag::VAR_PART) + }); + } + VariableType::Header(header) => { + let names: Vec = header + .name + .iter() + .map(|name| Rec { + c: self.emitter.header_name(name), + ..Rec::tagged(tag::HEADER) + }) + .collect(); + let names = self.emitter.push_recs(&names); + let (part, sub, attr) = match &header.part { + HeaderPart::Text => (0, 0, None), + HeaderPart::Date => (1, 0, None), + HeaderPart::Id => (2, 0, None), + HeaderPart::Address(part) => (3, *part as u16, None), + HeaderPart::ContentType(ct) => match ct { + ContentTypePart::Type => (4, 0, None), + ContentTypePart::Subtype => (4, 1, None), + ContentTypePart::Attribute(attr) => (4, 2, Some(attr.as_str())), + }, + HeaderPart::Received(part) => (5, part.code(), None), + HeaderPart::Raw => (6, 0, None), + HeaderPart::RawName => (7, 0, None), + HeaderPart::Exists => (8, 0, None), + }; + let attr = attr.map(|attr| self.str(attr)).unwrap_or_default(); + self.scratch.push(Rec { + b: part, + c: sub, + d: names.start, + e: ((header.index_hdr as u32 as u64) << 32) | (header.index_part as u32 as u64), + ..Rec::tagged(tag::VAR_HEADER) + }); + self.scratch.push(Rec { + c: names.len as u16, + ..Rec::tagged(tag::CONT).with_str(attr) + }); + } + } + } + + fn list(&mut self, items: &[Value]) -> Range { + let start = self.scratch.len(); + for item in items { + self.value_recs(item); + } + self.flush_scratch(start) + } + + fn inline(&mut self, value: &Value) -> Rec { + let start = self.scratch.len(); + self.value_recs(value); + self.inline_recs(start) + } + + fn inline_recs(&mut self, start: usize) -> Rec { + match self.scratch.len() - start { + 1 => { + let rec = self.scratch[start]; + self.scratch.truncate(start); + rec + } + 0 => Rec::tagged(tag::NONE), + _ => { + let range = self.flush_scratch(start); + Rec { + d: range.start, + e: (range.len - 1) as u64, + ..Rec::tagged(tag::REF) + } + } + } + } + + fn inline_opt(&mut self, value: &Option) -> Rec { + match value { + Some(value) => self.inline(value), + None => Rec::tagged(tag::NONE), + } + } + + fn variable(&mut self, var: &VariableType) -> Rec { + let start = self.scratch.len(); + self.variable_recs(var); + self.inline_recs(start) + } + + fn variable_opt(&mut self, var: &Option) -> Rec { + match var { + Some(var) => self.variable(var), + None => Rec::tagged(tag::VARIABLE_NONE), + } + } + + fn variable_list(&mut self, vars: &[VariableType]) -> Range { + let start = self.scratch.len(); + for var in vars { + self.variable_recs(var); + } + self.flush_scratch(start) + } + + fn expressions(&mut self, exprs: &[Expression]) -> Range { + let sizes: Vec = exprs + .iter() + .map(|expr| match expr { + Expression::VariableOther(var) + if matches!(var.as_ref(), VariableType::Header(_)) => + { + 2 + } + _ => 1, + }) + .collect(); + let start = self.scratch.len(); + for (index, expr) in exprs.iter().enumerate() { + match expr { + Expression::VariableLocal(id) => self.scratch.push(Rec { + c: *id, + ..Rec::tagged(tag::VAR_LOCAL) + }), + Expression::VariableMatch(id) => self.scratch.push(Rec { + b: *id, + ..Rec::tagged(tag::VAR_MATCH) + }), + Expression::VariableOther(var) => self.variable_recs(var), + Expression::ConstantInteger(n) => self.scratch.push(Rec { + e: *n as u64, + ..Rec::tagged(tag::INT) + }), + Expression::ConstantFloat(n) => self.scratch.push(Rec { + e: n.to_bits(), + ..Rec::tagged(tag::FLOAT) + }), + Expression::ConstantString(id) => { + let s = self.constant(*id); + self.scratch.push(Rec::tagged(tag::TEXT).with_str(s)); + } + Expression::BinaryOperator(op) => self.scratch.push(Rec { + b: *op as u8, + ..Rec::tagged(tag::BIN_OP) + }), + Expression::UnaryOperator(op) => self.scratch.push(Rec { + b: *op as u8, + ..Rec::tagged(tag::UN_OP) + }), + Expression::JmpIf { val, pos } => { + let skip: u32 = sizes.iter().skip(index + 1).take(*pos as usize).sum(); + self.scratch.push(Rec { + b: u8::from(*val), + d: skip, + ..Rec::tagged(tag::JMP_IF) + }); + } + Expression::Function { id, num_args } => self.scratch.push(Rec { + c: *num_args as u16, + d: *id, + ..Rec::tagged(tag::CALL) + }), + Expression::ArrayAccess => self.scratch.push(Rec::tagged(tag::ARRAY_ACCESS)), + Expression::ArrayBuild(n) => self.scratch.push(Rec { + d: *n, + ..Rec::tagged(tag::ARRAY_BUILD) + }), + } + } + self.flush_scratch(start) + } + + fn modifiers(&mut self, modifiers: &[Modifier]) -> Range { + let start = self.scratch.len(); + for modifier in modifiers { + self.scratch.push(Rec { + b: modifier.code(), + ..Rec::tagged(tag::MODIFIER) + }); + if let Modifier::Replace(replacement) = modifier { + let find = self.inline(&replacement.find); + let replace = self.inline(&replacement.replace); + self.scratch.push(find); + self.scratch.push(replace); + } + } + self.flush_scratch(start) + } + + fn capabilities(&mut self, capabilities: &[Capability]) -> Range { + let mut recs = Vec::with_capacity(capabilities.len()); + for capability in capabilities { + recs.push(match capability { + Capability::Comparator(Comparator::Other(name)) => { + let s = self.str(name); + Rec { + b: 5, + c: Comparator::Octet.code() as u16, + ..Rec::tagged(tag::CAPABILITY).with_str(s) + } + } + Capability::Comparator(comparator) => Rec { + b: 5, + c: comparator.code() as u16, + ..Rec::tagged(tag::CAPABILITY) + }, + Capability::Other(name) => { + let s = self.str(name); + Rec { + b: 6, + ..Rec::tagged(tag::CAPABILITY).with_str(s) + } + } + other => Rec { + b: other.id(), + ..Rec::tagged(tag::CAPABILITY) + }, + }); + } + self.emitter.push_recs(&recs) + } + + fn mime_opts(&mut self, opts: &MimeOpts) -> ops::MimeOpts { + match opts { + MimeOpts::Type => ops::MimeOpts { + kind: 0, + params: Range::EMPTY, + }, + MimeOpts::Subtype => ops::MimeOpts { + kind: 1, + params: Range::EMPTY, + }, + MimeOpts::ContentType => ops::MimeOpts { + kind: 2, + params: Range::EMPTY, + }, + MimeOpts::Param(params) => ops::MimeOpts { + kind: 3, + params: self.list(params), + }, + MimeOpts::None => ops::MimeOpts { + kind: 4, + params: Range::EMPTY, + }, + } + } + + fn fcc(&mut self, fcc: &Option>) -> ops::Fcc { + match fcc { + Some(fcc) => ops::Fcc { + present: true, + mailbox: self.inline(&fcc.mailbox), + mailbox_id: self.inline_opt(&fcc.mailbox_id), + create: fcc.create, + flags: self.list(&fcc.flags), + special_use: self.inline_opt(&fcc.special_use), + }, + None => ops::Fcc::default(), + } + } + + fn by_time(&mut self, by_time: &ByTime) -> ops::ByTime { + match by_time { + ByTime::Relative { + rlimit, + mode, + trace, + } => ops::ByTime { + kind: 0, + mode: *mode as u8, + trace: *trace, + rlimit: *rlimit, + alimit: Rec::tagged(tag::NONE), + }, + ByTime::Absolute { + alimit, + mode, + trace, + } => ops::ByTime { + kind: 1, + mode: *mode as u8, + trace: *trace, + rlimit: 0, + alimit: self.inline(alimit), + }, + ByTime::None => ops::ByTime { + kind: 2, + mode: 2, + trace: false, + rlimit: 0, + alimit: Rec::tagged(tag::NONE), + }, + } + } + + fn notify(&mut self, notify: &Notify) -> ops::NotifySpec { + match notify { + Notify::Never => ops::NotifySpec { + kind: 0, + items: Range::EMPTY, + }, + Notify::Items(items) => { + let recs: Vec = items + .iter() + .map(|item| Rec { + b: *item as u8, + ..Rec::tagged(tag::NOTIFY_ITEM) + }) + .collect(); + ops::NotifySpec { + kind: 1, + items: self.emitter.push_recs(&recs), + } + } + Notify::Default => ops::NotifySpec { + kind: 2, + items: Range::EMPTY, + }, + } + } + + fn emit_op(&mut self, op: &T) { + op.emit_into(&mut self.emitter.code); + } + + fn instruction(&mut self, instruction: &Instruction) { + match instruction { + Instruction::Require(capabilities) => { + let capabilities = self.capabilities(capabilities); + self.emit_op(&ops::Require { capabilities }); + } + Instruction::Keep(keep) => { + let flags = self.list(&keep.flags); + self.emit_op(&ops::Keep { flags }); + } + Instruction::FileInto(fi) => { + let op = ops::FileInto { + copy: fi.copy, + create: fi.create, + folder: self.inline(&fi.folder), + flags: self.list(&fi.flags), + mailbox_id: self.inline_opt(&fi.mailbox_id), + special_use: self.inline_opt(&fi.special_use), + }; + self.emit_op(&op); + } + Instruction::Redirect(redirect) => { + let op = ops::Redirect { + copy: redirect.copy, + list: redirect.list, + address: self.inline(&redirect.address), + notify: self.notify(&redirect.notify), + ret: redirect.return_of_content as u8, + by_time: self.by_time(&redirect.by_time), + }; + self.emit_op(&op); + } + Instruction::Discard => self.emit_op(&ops::Discard {}), + Instruction::Stop => self.emit_op(&ops::Stop {}), + Instruction::Invalid(invalid) => { + let op = ops::Invalid { + name: self.str(&invalid.name), + line_num: invalid.line_num, + line_pos: invalid.line_pos, + }; + self.emit_op(&op); + } + Instruction::Test(test) => self.test(test), + Instruction::Jmp(target) => { + let target = self.jump(*target); + self.emit_op(&ops::Jmp { target }); + } + Instruction::Jz(target) => { + let target = self.jump(*target); + self.emit_op(&ops::Jz { target }); + } + Instruction::Jnz(target) => { + let target = self.jump(*target); + self.emit_op(&ops::Jnz { target }); + } + Instruction::ForEveryPartPush => self.emit_op(&ops::ForEveryPartPush {}), + Instruction::ForEveryPart(fep) => { + let jz_pos = self.jump(fep.jz_pos); + self.emit_op(&ops::ForEveryPart { jz_pos }); + } + Instruction::ForEveryPartPop(num_pops) => { + self.emit_op(&ops::ForEveryPartPop { + num_pops: *num_pops, + }); + } + Instruction::Replace(replace) => { + let op = ops::Replace { + subject: self.inline_opt(&replace.subject), + from: self.inline_opt(&replace.from), + replacement: self.inline(&replace.replacement), + mime: replace.mime, + }; + self.emit_op(&op); + } + Instruction::Enclose(enclose) => { + let op = ops::Enclose { + subject: self.inline_opt(&enclose.subject), + headers: self.list(&enclose.headers), + value: self.inline(&enclose.value), + }; + self.emit_op(&op); + } + Instruction::ExtractText(extract) => { + let op = ops::ExtractText { + modifiers: self.modifiers(&extract.modifiers), + first: extract.first, + name: self.variable(&extract.name), + }; + self.emit_op(&op); + } + Instruction::Convert(convert) => { + let op = ops::Convert { + from_media_type: self.inline(&convert.from_media_type), + to_media_type: self.inline(&convert.to_media_type), + transcoding_params: self.list(&convert.transcoding_params), + is_not: convert.is_not, + }; + self.emit_op(&op); + } + Instruction::AddHeader(add) => { + let op = ops::AddHeader { + last: add.last, + field_name: self.inline(&add.field_name), + value: self.inline(&add.value), + }; + self.emit_op(&op); + } + Instruction::DeleteHeader(delete) => { + let op = ops::DeleteHeader { + index: delete.index, + comparator: delete.comparator.code(), + match_type: Match::from_match_type(&delete.match_type), + field_name: self.inline(&delete.field_name), + value_patterns: self.list(&delete.value_patterns), + mime_anychild: delete.mime_anychild, + }; + self.emit_op(&op); + } + Instruction::Set(set) => { + let op = ops::Set { + modifiers: self.modifiers(&set.modifiers), + name: self.variable(&set.name), + value: self.inline(&set.value), + }; + self.emit_op(&op); + } + Instruction::Clear(clear) => self.emit_op(&ops::Clear { + local_vars_idx: clear.local_vars_idx, + local_vars_num: clear.local_vars_num, + match_vars: clear.match_vars, + }), + Instruction::Notify(notify) => { + let op = ops::Notify { + from: self.inline_opt(¬ify.from), + importance: self.inline_opt(¬ify.importance), + options: self.list(¬ify.options), + message: self.inline_opt(¬ify.message), + fcc: self.fcc(¬ify.fcc), + method: self.inline(¬ify.method), + }; + self.emit_op(&op); + } + Instruction::Reject(reject) => { + let op = ops::Reject { + ereject: reject.ereject, + reason: self.inline(&reject.reason), + }; + self.emit_op(&op); + } + Instruction::Vacation(vacation) => { + let op = ops::Vacation { + subject: self.inline_opt(&vacation.subject), + from: self.inline_opt(&vacation.from), + mime: vacation.mime, + fcc: self.fcc(&vacation.fcc), + reason: self.inline(&vacation.reason), + }; + self.emit_op(&op); + } + Instruction::Error(error) => { + let op = ops::Error { + message: self.inline(&error.message), + }; + self.emit_op(&op); + } + Instruction::EditFlags(flags) => { + let op = ops::EditFlags { + action: flags.action as u8, + name: self.variable_opt(&flags.name), + flags: self.list(&flags.flags), + }; + self.emit_op(&op); + } + Instruction::Include(include) => { + let op = ops::Include { + global: include.location as u8 == 1, + once: include.once, + optional: include.optional, + value: self.inline(&include.value), + }; + self.emit_op(&op); + } + Instruction::Return => self.emit_op(&ops::Return {}), + Instruction::While(while_) => { + let expr = self.expressions(&while_.expr); + let jz_pos = self.jump_at(8, while_.jz_pos); + self.emit_op(&ops::While { expr, jz_pos }); + } + Instruction::Eval(expr) => { + let expr = self.expressions(expr); + self.emit_op(&ops::Eval { expr }); + } + Instruction::Let(let_) => { + let op = ops::Let { + name: self.variable(&let_.name), + expr: self.expressions(&let_.expr), + }; + self.emit_op(&op); + } + #[cfg(all(test, stalwart_tests))] + Instruction::TestCmd(arguments) => { + let arguments = self.list(arguments); + self.emit_op(&ops::TestCmd { arguments }); + } + } + } + + fn test(&mut self, test: &Test) { + match test { + Test::True => self.emit_op(&ops::TestTrue {}), + Test::False => self.emit_op(&ops::TestFalse {}), + Test::Address(t) => { + let op = ops::TestAddress { + header_list: self.list(&t.header_list), + key_list: self.list(&t.key_list), + address_part: t.address_part as u8, + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + index: t.index, + mime_anychild: t.mime_anychild, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Envelope(t) => { + let recs: Vec = t + .envelope_list + .iter() + .map(|envelope| Rec { + b: *envelope as u8, + ..Rec::tagged(tag::ENVELOPE) + }) + .collect(); + let op = ops::TestEnvelope { + envelope_list: self.emitter.push_recs(&recs), + key_list: self.list(&t.key_list), + address_part: t.address_part as u8, + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + zone: t.zone, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Exists(t) => { + let op = ops::TestExists { + header_names: self.list(&t.header_names), + mime_anychild: t.mime_anychild, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Header(t) => { + let op = ops::TestHeader { + header_list: self.list(&t.header_list), + key_list: self.list(&t.key_list), + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + index: t.index, + mime_opts: self.mime_opts(&t.mime_opts), + mime_anychild: t.mime_anychild, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Size(t) => self.emit_op(&ops::TestSize { + over: t.over, + limit: t.limit, + is_not: t.is_not, + }), + Test::Invalid(invalid) => { + let op = ops::TestInvalid { + name: self.str(&invalid.name), + line_num: invalid.line_num, + line_pos: invalid.line_pos, + }; + self.emit_op(&op); + } + Test::Body(t) => { + let body_transform = match &t.body_transform { + BodyTransform::Raw => ops::BodyTransform { + kind: 0, + content_types: Range::EMPTY, + }, + BodyTransform::Content(types) => ops::BodyTransform { + kind: 1, + content_types: self.list(types), + }, + BodyTransform::Text => ops::BodyTransform { + kind: 2, + content_types: Range::EMPTY, + }, + }; + let op = ops::TestBody { + key_list: self.list(&t.key_list), + body_transform, + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + include_subject: t.include_subject, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Convert(t) => { + let op = ops::TestConvert { + from_media_type: self.inline(&t.from_media_type), + to_media_type: self.inline(&t.to_media_type), + transcoding_params: self.list(&t.transcoding_params), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Date(t) => { + let zone = match &t.zone { + Zone::Time(time) => ops::Zone { + kind: 0, + time: *time, + }, + Zone::Original => ops::Zone { kind: 1, time: 0 }, + Zone::Local => ops::Zone { kind: 2, time: 0 }, + }; + let op = ops::TestDate { + header_name: self.inline(&t.header_name), + key_list: self.list(&t.key_list), + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + index: t.index, + zone, + date_part: t.date_part as u8, + mime_anychild: t.mime_anychild, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::CurrentDate(t) => { + let op = ops::TestCurrentDate { + zone: t.zone, + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + date_part: t.date_part as u8, + key_list: self.list(&t.key_list), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Duplicate(t) => { + let dup_match = match &t.dup_match { + DupMatch::Header(value) => ops::DupMatch { + kind: 0, + value: self.inline(value), + }, + DupMatch::UniqueId(value) => ops::DupMatch { + kind: 1, + value: self.inline(value), + }, + DupMatch::Default => ops::DupMatch { + kind: 2, + value: Rec::tagged(tag::NONE), + }, + }; + let op = ops::TestDuplicate { + handle: self.inline_opt(&t.handle), + dup_match, + seconds: t.seconds, + last: t.last, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::String(t) => { + let op = ops::TestString { + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + source: self.list(&t.source), + key_list: self.list(&t.key_list), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Environment(t) => { + let op = ops::TestEnvironment { + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + source: self.list(&t.source), + key_list: self.list(&t.key_list), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::NotifyMethodCapability(t) => { + let op = ops::TestNotifyMethodCapability { + comparator: t.comparator.code(), + match_type: Match::from_match_type(&t.match_type), + notification_uri: self.inline(&t.notification_uri), + notification_capability: self.inline(&t.notification_capability), + key_list: self.list(&t.key_list), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::ValidNotifyMethod(t) => { + let op = ops::TestValidNotifyMethod { + notification_uris: self.list(&t.notification_uris), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::ValidExtList(t) => { + let op = ops::TestValidExtList { + list_names: self.list(&t.list_names), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Ihave(t) => { + let op = ops::TestIhave { + capabilities: self.capabilities(&t.capabilities), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::HasFlag(t) => { + let op = ops::TestHasFlag { + comparator: t.comparator.code(), + match_type: Match::from_match_type(&t.match_type), + variable_list: self.variable_list(&t.variable_list), + flags: self.list(&t.flags), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::MailboxExists(t) => { + let op = ops::TestMailboxExists { + mailbox_names: self.list(&t.mailbox_names), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Metadata(t) => { + let metadata = match &t.medatata { + Metadata::Server { annotation } => ops::MetadataRef { + kind: 0, + name: Rec::tagged(tag::NONE), + annotation: self.inline(annotation), + }, + Metadata::Mailbox { name, annotation } => ops::MetadataRef { + kind: 1, + name: self.inline(name), + annotation: self.inline(annotation), + }, + }; + let op = ops::TestMetadata { + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + metadata, + key_list: self.list(&t.key_list), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::MetadataExists(t) => { + let op = ops::TestMetadataExists { + mailbox: self.inline_opt(&t.mailbox), + annotation_names: self.list(&t.annotation_names), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::MailboxIdExists(t) => { + let op = ops::TestMailboxIdExists { + mailbox_ids: self.list(&t.mailbox_ids), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::SpamTest(t) => { + let op = ops::TestSpamTest { + value: self.inline(&t.value), + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + percent: t.percent, + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::VirusTest(t) => { + let op = ops::TestVirusTest { + value: self.inline(&t.value), + match_type: Match::from_match_type(&t.match_type), + comparator: t.comparator.code(), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::SpecialUseExists(t) => { + let op = ops::TestSpecialUseExists { + mailbox: self.inline_opt(&t.mailbox), + attributes: self.list(&t.attributes), + is_not: t.is_not, + }; + self.emit_op(&op); + } + Test::Vacation(t) => { + let period = match &t.period { + Period::Days(days) => ops::Period { + kind: 0, + value: *days, + }, + Period::Seconds(seconds) => ops::Period { + kind: 1, + value: *seconds, + }, + Period::Default => ops::Period { kind: 2, value: 0 }, + }; + let op = ops::TestVacation { + addresses: self.list(&t.addresses), + period, + handle: self.inline_opt(&t.handle), + reason: self.inline(&t.reason), + }; + self.emit_op(&op); + } + #[cfg(all(test, stalwart_tests))] + Test::TestCmd(cmd) => { + let op = ops::TestCmdTest { + arguments: self.list(&cmd.arguments), + is_not: cmd.is_not, + }; + self.emit_op(&op); + } + } + } +} + +pub(crate) trait OpEmit { + fn emit_into(&self, code: &mut crate::bytecode::cursor::Code); +} + +macro_rules! op_emit { + ($($name:ident),* $(,)?) => { + $( + impl OpEmit for ops::$name { + fn emit_into(&self, code: &mut crate::bytecode::cursor::Code) { + self.emit(code); + } + } + )* + }; +} + +op_emit!( + Require, + Keep, + FileInto, + Redirect, + Discard, + Stop, + Invalid, + TestTrue, + TestFalse, + TestAddress, + TestEnvelope, + TestExists, + TestHeader, + TestSize, + TestInvalid, + TestBody, + TestConvert, + Convert, + TestDate, + TestCurrentDate, + TestDuplicate, + TestString, + TestEnvironment, + TestNotifyMethodCapability, + TestValidNotifyMethod, + TestValidExtList, + TestIhave, + TestHasFlag, + TestMailboxExists, + TestMetadata, + TestMetadataExists, + TestMailboxIdExists, + TestSpamTest, + TestVirusTest, + TestSpecialUseExists, + TestVacation, + Jmp, + Jz, + Jnz, + ForEveryPartPush, + ForEveryPart, + ForEveryPartPop, + Replace, + Enclose, + ExtractText, + AddHeader, + DeleteHeader, + Set, + Clear, + Notify, + Reject, + Vacation, + Error, + EditFlags, + Include, + Return, + While, + Eval, + Let, + TestCmd, + TestCmdTest, +); + +impl Field for () { + fn read(_: &mut crate::bytecode::cursor::Cursor<'_>) -> crate::bytecode::Decoded { + Ok(()) + } + + fn write(&self, _: &mut crate::bytecode::cursor::Code) {} +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_convert.rs b/crates/sieve/src/compiler/grammar/actions/action_convert.rs new file mode 100644 index 00000000..81b16fd1 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_convert.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{ + instruction::{CompilerState, Instruction}, + test::Test, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Convert { + pub from_media_type: Value, + pub to_media_type: Value, + pub transcoding_params: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_convert(&mut self) -> Result { + Ok(Test::Convert(Box::new(Convert { + from_media_type: self.parse_string()?, + to_media_type: self.parse_string()?, + transcoding_params: self.parse_strings(false)?.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_convert(&mut self) -> Result<(), CompileError> { + let cmd = Instruction::Convert(Box::new(Convert { + from_media_type: self.parse_string()?, + to_media_type: self.parse_string()?, + transcoding_params: self.parse_strings(false)?.into(), + is_not: false, + })); + self.instructions.push(cmd); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_editheader.rs b/crates/sieve/src/compiler/grammar/actions/action_editheader.rs new file mode 100644 index 00000000..41de1067 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_editheader.rs @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, RawValue, Value, + grammar::{ + Capability, Comparator, + instruction::{CompilerState, Instruction}, + }, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::MatchType; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct AddHeader { + pub last: bool, + pub field_name: Value, + pub value: Value, +} + +/* + Usage: "deleteheader" [":index" [":last"]] + [COMPARATOR] [MATCH-TYPE] + + [] + +*/ +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct DeleteHeader { + pub index: Option, + pub comparator: Comparator, + pub match_type: MatchType, + pub field_name: Value, + pub value_patterns: Box<[Value]>, + pub mime_anychild: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_addheader(&mut self) -> Result<(), CompileError> { + let mut field_name = None; + let value; + let mut last = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Last) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + last = true; + } + _ => { + let string = self.parse_raw_string_token(token_info)?; + if field_name.is_none() { + field_name = match self.resolve_header_name(string) { + Some(header_name) => header_name.into(), + None => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + }; + } else { + let header_size = match &string { + RawValue::Text(value) => value.len(), + RawValue::Value(Value::Text(id)) => self.constant(*id).len(), + _ => 0, + }; + if header_size > self.compiler.max_header_size { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::HeaderTooLong)); + } + value = self.intern_raw(string); + break; + } + } + } + } + self.instructions + .push(Instruction::AddHeader(Box::new(AddHeader { + last, + field_name: field_name.unwrap(), + value, + }))); + Ok(()) + } + + pub(crate) fn parse_deleteheader(&mut self) -> Result<(), CompileError> { + let field_name: Value; + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut index = None; + let mut index_last = false; + let mut mime = false; + let mut mime_anychild = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Index) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + index = (self.tokens.expect_number(u16::MAX as usize)? as i32).into(); + } + Token::Tag(Word::Last) => { + self.validate_argument(4, None, token_info.line_num, token_info.line_pos)?; + index_last = true; + } + Token::Tag(Word::Mime) => { + self.validate_argument( + 5, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime = true; + } + Token::Tag(Word::AnyChild) => { + self.validate_argument( + 6, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_anychild = true; + } + _ => { + let header_name = self.parse_raw_string_token(token_info)?; + field_name = match self.resolve_header_name(header_name) { + Some(field_name) => field_name, + None => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + }; + break; + } + } + } + + if !mime && mime_anychild { + return Err(self.tokens.unwrap_next()?.missing_tag(":mime")); + } + + let value_patterns = if let Some(Ok( + Token::StringConstant(_) | Token::StringVariable(_) | Token::BracketOpen, + )) = self.tokens.peek().map(|r| r.map(|t| &t.token)) + { + let key_list = self.parse_raw_strings(false)?; + self.validate_match(&match_type, &comparator, key_list)? + } else { + Vec::new() + }; + + let cmd = Instruction::DeleteHeader(Box::new(DeleteHeader { + index: if index_last { index.map(|i| -i) } else { index }, + comparator, + match_type, + field_name, + value_patterns: value_patterns.into(), + mime_anychild, + })); + self.instructions.push(cmd); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_fileinto.rs b/crates/sieve/src/compiler/grammar/actions/action_fileinto.rs new file mode 100644 index 00000000..d28abb59 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_fileinto.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{ + Capability, + instruction::{CompilerState, Instruction}, + }, + lexer::{Token, word::Word}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct FileInto { + pub copy: bool, + pub create: bool, + pub folder: Value, + pub flags: Box<[Value]>, + pub mailbox_id: Option, + pub special_use: Option, +} + +impl CompilerState<'_> { + pub(crate) fn parse_fileinto(&mut self) -> Result<(), CompileError> { + let folder; + let mut copy = false; + let mut create = false; + let mut flags = Vec::new(); + let mut mailbox_id = None; + let mut special_use = None; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Copy) => { + self.validate_argument( + 1, + Capability::Copy.into(), + token_info.line_num, + token_info.line_pos, + )?; + copy = true; + } + Token::Tag(Word::Create) => { + self.validate_argument( + 2, + Capability::Mailbox.into(), + token_info.line_num, + token_info.line_pos, + )?; + create = true; + } + Token::Tag(Word::Flags) => { + self.validate_argument( + 3, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + flags = self.parse_strings(false)?; + } + Token::Tag(Word::MailboxId) => { + self.validate_argument( + 4, + Capability::MailboxId.into(), + token_info.line_num, + token_info.line_pos, + )?; + mailbox_id = self.parse_string()?.into(); + } + Token::Tag(Word::SpecialUse) => { + self.validate_argument( + 5, + Capability::SpecialUse.into(), + token_info.line_num, + token_info.line_pos, + )?; + special_use = self.parse_string()?.into(); + } + _ => { + folder = self.parse_string_token(token_info)?; + break; + } + } + } + + self.instructions + .push(Instruction::FileInto(Box::new(FileInto { + folder, + copy, + create, + flags: flags.into(), + mailbox_id, + special_use, + }))); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_flags.rs b/crates/sieve/src/compiler/grammar/actions/action_flags.rs new file mode 100644 index 00000000..99faafb2 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_flags.rs @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, Value, VariableType, + grammar::instruction::{CompilerState, Instruction}, + lexer::{Token, word::Word}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct EditFlags { + pub action: Action, + pub name: Option, + pub flags: Box<[Value]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Action { + Set = 0, + Add = 1, + Remove = 2, +} + +impl CompilerState<'_> { + pub(crate) fn parse_flag_action(&mut self, word: Word) -> Result<(), CompileError> { + let token_info = self.tokens.unwrap_next()?; + let action = match word { + Word::SetFlag => Action::Set, + Word::AddFlag => Action::Add, + Word::RemoveFlag => Action::Remove, + _ => unreachable!(), + }; + + let instruction = Instruction::EditFlags(Box::new( + match ( + &token_info.token, + self.tokens.peek().map(|r| r.map(|t| &t.token)), + ) { + ( + Token::StringConstant(_), + Some(Ok( + Token::StringConstant(_) | Token::StringVariable(_) | Token::BracketOpen, + )), + ) => EditFlags { + name: self.parse_variable_name(token_info, false)?.into(), + flags: self.parse_strings(false)?.into(), + action, + }, + (Token::BracketOpen, _) + | ( + Token::StringConstant(_) | Token::StringVariable(_), + Some(Ok(Token::Semicolon)), + ) => EditFlags { + name: None, + flags: self.parse_strings_token(token_info)?.into(), + action, + }, + _ => { + return Err(token_info.custom(ErrorType::InvalidArguments)); + } + }, + )); + + self.instructions.push(instruction); + + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_include.rs b/crates/sieve/src/compiler/grammar/actions/action_include.rs new file mode 100644 index 00000000..4781dcb2 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_include.rs @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::instruction::{CompilerState, Instruction}, + lexer::{Token, word::Word}, +}; + +/* + +include [LOCATION] [":once"] [":optional"] + LOCATION = ":personal" / ":global" + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Include { + pub location: Location, + pub once: bool, + pub optional: bool, + pub value: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Location { + Personal = 0, + Global = 1, +} + +impl CompilerState<'_> { + pub(crate) fn parse_include(&mut self) -> Result<(), CompileError> { + let value; + let mut once = false; + let mut optional = false; + let mut location = Location::Personal; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Once) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + once = true; + } + Token::Tag(Word::Optional) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + optional = true; + } + Token::Tag(Word::Personal) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + location = Location::Personal; + } + Token::Tag(Word::Global) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + location = Location::Global; + } + _ => { + value = self.parse_string_token(token_info)?; + break; + } + } + } + + self.instructions + .push(Instruction::Include(Box::new(Include { + location, + once, + optional, + value, + }))); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_keep.rs b/crates/sieve/src/compiler/grammar/actions/action_keep.rs new file mode 100644 index 00000000..038073cc --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_keep.rs @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{ + Capability, + instruction::{CompilerState, Instruction}, + }, + lexer::{Token, word::Word}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Keep { + pub flags: Box<[Value]>, +} + +impl CompilerState<'_> { + pub(crate) fn parse_keep(&mut self) -> Result<(), CompileError> { + let cmd = Instruction::Keep(Box::new(Keep { + flags: match self.tokens.peek().map(|r| r.map(|t| &t.token)) { + Some(Ok(Token::Tag(Word::Flags))) => { + let token_info = self.tokens.next().unwrap().unwrap(); + self.validate_argument( + 0, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_strings(false)?.into() + } + _ => Vec::new().into(), + }, + })); + self.instructions.push(cmd); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_mime.rs b/crates/sieve/src/compiler/grammar/actions/action_mime.rs new file mode 100644 index 00000000..0716876e --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_mime.rs @@ -0,0 +1,209 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, VariableType, + grammar::instruction::{CompilerState, Instruction}, + lexer::{Token, word::Word}, +}; + +use super::action_set::{Modifier, Replacement}; + +#[derive(Debug, Eq, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(transparent)] +pub(crate) struct ForEveryPart { + pub jz_pos: u32, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Replace { + pub subject: Option, + pub from: Option, + pub replacement: Value, + pub mime: bool, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Enclose { + pub subject: Option, + pub headers: Box<[Value]>, + pub value: Value, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct ExtractText { + pub modifiers: Box<[Modifier]>, + pub first: Option, + pub name: VariableType, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum MimeOpts { + Type = 0, + Subtype = 1, + ContentType = 2, + Param(Box<[T]>) = 3, + None = 4, +} + +impl CompilerState<'_> { + pub(crate) fn parse_replace(&mut self) -> Result<(), CompileError> { + let mut subject = None; + let mut from = None; + let replacement; + let mut mime = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Mime) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + mime = true; + } + Token::Tag(Word::Subject) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + subject = self.parse_string()?.into(); + } + Token::Tag(Word::From) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + from = self.parse_string()?.into(); + } + _ => { + replacement = self.parse_string_token(token_info)?; + break; + } + } + } + + self.instructions + .push(Instruction::Replace(Box::new(Replace { + subject, + from, + replacement, + mime, + }))); + Ok(()) + } + + pub(crate) fn parse_enclose(&mut self) -> Result<(), CompileError> { + let mut subject = None; + let mut headers = Vec::new(); + let value; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Subject) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + subject = self.parse_string()?.into(); + } + Token::Tag(Word::Headers) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + headers = self.parse_strings(false)?; + } + _ => { + value = self.parse_string_token(token_info)?; + break; + } + } + } + + self.instructions + .push(Instruction::Enclose(Box::new(Enclose { + subject, + headers: headers.into(), + value, + }))); + Ok(()) + } + + pub(crate) fn parse_extracttext(&mut self) -> Result<(), CompileError> { + let mut modifiers = Vec::new(); + let mut first = None; + let name; + let mut is_local = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::First) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + first = (self.tokens.expect_number(u32::MAX as usize)? as u32).into(); + } + Token::Tag( + word @ (Word::Lower + | Word::Upper + | Word::LowerFirst + | Word::UpperFirst + | Word::QuoteWildcard + | Word::QuoteRegex + | Word::Length), + ) => { + let modifier = word.into(); + if !modifiers.contains(&modifier) { + modifiers.push(modifier); + } + } + Token::Tag(Word::Replace) => { + let find = self.tokens.unwrap_next()?; + let replace = self.tokens.unwrap_next()?; + modifiers.push(Modifier::Replace(Box::new(Replacement { + find: self.parse_string_token(find)?, + replace: self.parse_string_token(replace)?, + }))); + } + Token::Tag(Word::Local) => { + is_local = true; + } + _ => { + name = self.parse_variable_name(token_info, is_local)?; + break; + } + } + } + + modifiers.sort_unstable_by_key(|m| std::cmp::Reverse(m.order())); + + self.instructions + .push(Instruction::ExtractText(Box::new(ExtractText { + modifiers: modifiers.into(), + first, + name, + }))); + Ok(()) + } + + pub(crate) fn parse_mimeopts(&mut self, opts: Word) -> Result, CompileError> { + Ok(match opts { + Word::Type => MimeOpts::Type, + Word::Subtype => MimeOpts::Subtype, + Word::ContentType => MimeOpts::ContentType, + Word::Param => MimeOpts::Param(self.parse_strings(false)?.into()), + _ => MimeOpts::None, + }) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_notify.rs b/crates/sieve/src/compiler/grammar/actions/action_notify.rs new file mode 100644 index 00000000..e075162b --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_notify.rs @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + FileCarbonCopy, + compiler::{ + CompileError, ErrorType, Value, + grammar::{ + Capability, + instruction::{CompilerState, Instruction, MapLocalVars}, + }, + lexer::{Token, word::Word}, + }, + runtime::actions::action_notify::{validate_from, validate_uri}, +}; + +/* +notify [":from" string] + [":importance" <"1" / "2" / "3">] + [":options" string-list] + [":message" string] + + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Notify { + pub from: Option, + pub importance: Option, + pub options: Box<[Value]>, + pub message: Option, + pub fcc: Option>, + pub method: Value, +} + +impl CompilerState<'_> { + pub(crate) fn parse_notify(&mut self) -> Result<(), CompileError> { + let method; + let mut from = None; + let mut importance = None; + let mut message = None; + let mut options = Vec::new(); + + let mut fcc = None; + let mut create = false; + let mut flags = Vec::new(); + let mut special_use = None; + let mut mailbox_id = None; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::From) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + let address = self.parse_string()?; + if let Value::Text(address) = &address + && { + let address = self.constant(*address); + address.is_empty() || !validate_from(address) + } + { + return Err(token_info.custom(ErrorType::InvalidAddress)); + } + from = address.into(); + } + Token::Tag(Word::Message) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + message = self.parse_string()?.into(); + } + Token::Tag(Word::Importance) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + importance = self.parse_string()?.into(); + } + Token::Tag(Word::Options) => { + self.validate_argument(4, None, token_info.line_num, token_info.line_pos)?; + options = self.parse_strings(false)?; + } + Token::Tag(Word::Create) => { + self.validate_argument( + 5, + Capability::Mailbox.into(), + token_info.line_num, + token_info.line_pos, + )?; + create = true; + } + Token::Tag(Word::SpecialUse) => { + self.validate_argument( + 6, + Capability::SpecialUse.into(), + token_info.line_num, + token_info.line_pos, + )?; + special_use = self.parse_string()?.into(); + } + Token::Tag(Word::MailboxId) => { + self.validate_argument( + 7, + Capability::MailboxId.into(), + token_info.line_num, + token_info.line_pos, + )?; + mailbox_id = self.parse_string()?.into(); + } + Token::Tag(Word::Fcc) => { + self.validate_argument( + 8, + Capability::Fcc.into(), + token_info.line_num, + token_info.line_pos, + )?; + fcc = self.parse_string()?.into(); + } + Token::Tag(Word::Flags) => { + self.validate_argument( + 9, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + flags = self.parse_strings(false)?; + } + _ => { + if let Token::StringConstant(uri) = &token_info.token + && validate_uri(uri.to_string().as_ref()).is_none() + { + return Err(token_info.custom(ErrorType::InvalidURI)); + } + + method = self.parse_string_token(token_info)?; + break; + } + } + } + + if fcc.is_none() + && (create || !flags.is_empty() || special_use.is_some() || mailbox_id.is_some()) + { + return Err(self.tokens.unwrap_next()?.missing_tag(":fcc")); + } + + self.instructions.push(Instruction::Notify(Box::new(Notify { + method, + from, + importance, + options: options.into(), + message, + fcc: if let Some(fcc) = fcc { + FileCarbonCopy { + mailbox: fcc, + create, + flags: flags.into(), + special_use, + mailbox_id, + } + .into() + } else { + None + }, + }))); + Ok(()) + } +} + +impl MapLocalVars for FileCarbonCopy { + fn map_local_vars(&mut self, last_id: u16) { + self.mailbox.map_local_vars(last_id); + self.mailbox_id.map_local_vars(last_id); + self.flags.map_local_vars(last_id); + self.special_use.map_local_vars(last_id); + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_redirect.rs b/crates/sieve/src/compiler/grammar/actions/action_redirect.rs new file mode 100644 index 00000000..112a9a54 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_redirect.rs @@ -0,0 +1,289 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{ + Capability, + instruction::{CompilerState, Instruction, MapLocalVars}, + }, + lexer::{Token, word::Word}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Redirect { + pub copy: bool, + pub address: Value, + pub notify: Notify, + pub return_of_content: Ret, + pub by_time: ByTime, + pub list: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum NotifyItem { + Success = 0, + Failure = 1, + Delay = 2, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Notify { + Never = 0, + Items(Box<[NotifyItem]>) = 1, + Default = 2, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Ret { + Full = 0, + Hdrs = 1, + Default = 2, +} + +/* + + Usage: redirect [:bytimerelative / + :bytimeabsolute + [:bymode "notify"|"return"] [:bytrace]] + + +*/ + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum ByTime { + Relative { + rlimit: u64, + mode: ByMode, + trace: bool, + } = 0, + Absolute { + alimit: T, + mode: ByMode, + trace: bool, + } = 1, + None = 2, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum ByMode { + Notify = 0, + Return = 1, + Default = 2, +} + +impl CompilerState<'_> { + pub(crate) fn parse_redirect(&mut self) -> Result<(), CompileError> { + let address; + let mut copy = false; + let mut ret = Ret::Default; + let mut notify = Notify::Default; + let mut list = false; + let mut by_mode = ByMode::Default; + let mut by_trace = false; + let mut by_rlimit = None; + let mut by_alimit = None; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Copy) => { + self.validate_argument( + 1, + Capability::Copy.into(), + token_info.line_num, + token_info.line_pos, + )?; + copy = true; + } + Token::Tag(Word::List) => { + self.validate_argument( + 2, + Capability::ExtLists.into(), + token_info.line_num, + token_info.line_pos, + )?; + list = true; + } + Token::Tag(Word::ByTrace) => { + self.validate_argument( + 3, + Capability::RedirectDeliverBy.into(), + token_info.line_num, + token_info.line_pos, + )?; + by_trace = true; + } + Token::Tag(Word::ByMode) => { + self.validate_argument( + 4, + Capability::RedirectDeliverBy.into(), + token_info.line_num, + token_info.line_pos, + )?; + let by_mode_ = self.tokens.expect_static_string()?; + if let Some(mode) = lookup_by_mode(&by_mode_) { + by_mode = mode; + } else { + return Err(token_info.expected("\"notify\" or \"return\"")); + } + } + Token::Tag(Word::ByTimeRelative) => { + self.validate_argument( + 5, + Capability::RedirectDeliverBy.into(), + token_info.line_num, + token_info.line_pos, + )?; + by_rlimit = (self.tokens.expect_number(u64::MAX as usize)? as u64).into(); + } + Token::Tag(Word::ByTimeAbsolute) => { + self.validate_argument( + 5, + Capability::RedirectDeliverBy.into(), + token_info.line_num, + token_info.line_pos, + )?; + by_alimit = self.parse_string()?.into(); + } + Token::Tag(Word::Ret) => { + self.validate_argument( + 6, + Capability::RedirectDsn.into(), + token_info.line_num, + token_info.line_pos, + )?; + let ret_ = self.tokens.expect_static_string()?; + if let Some(ret_of_content) = lookup_ret(&ret_) { + ret = ret_of_content; + } else { + return Err(token_info.expected("\"FULL\" or \"HDRS\"")); + } + } + Token::Tag(Word::Notify) => { + self.validate_argument( + 7, + Capability::RedirectDsn.into(), + token_info.line_num, + token_info.line_pos, + )?; + let notify_ = self.tokens.expect_static_string()?; + if notify_.eq_ignore_ascii_case("never") { + notify = Notify::Never; + } else { + let mut items = Vec::new(); + for item in notify_.split(',') { + if let Some(item) = lookup_notify_item(item.trim()) { + items.push(item); + } + } + if !items.is_empty() { + notify = Notify::Items(items.into()); + } else { + return Err( + token_info.expected("\"NEVER\" or \"SUCCESS, FAILURE, DELAY, ..\"") + ); + } + } + } + _ => { + address = self.parse_string_token(token_info)?; + break; + } + } + } + + self.instructions + .push(Instruction::Redirect(Box::new(Redirect { + address, + copy, + notify, + return_of_content: ret, + by_time: if let Some(alimit) = by_alimit { + ByTime::Absolute { + alimit, + mode: by_mode, + trace: by_trace, + } + } else if let Some(rlimit) = by_rlimit { + ByTime::Relative { + rlimit, + mode: by_mode, + trace: by_trace, + } + } else { + ByTime::None + }, + list, + }))); + Ok(()) + } +} + +impl MapLocalVars for ByTime { + fn map_local_vars(&mut self, last_id: u16) { + if let ByTime::Absolute { alimit, .. } = self { + alimit.map_local_vars(last_id) + } + } +} + +fn lookup_by_mode(input: &str) -> Option { + hashify::map_ignore_case!( + input.as_bytes(), ByMode, + "notify" => ByMode::Notify, + "return" => ByMode::Return, + ) + .copied() +} + +fn lookup_ret(input: &str) -> Option { + hashify::map_ignore_case!( + input.as_bytes(), Ret, + "full" => Ret::Full, + "hdrs" => Ret::Hdrs, + ) + .copied() +} + +fn lookup_notify_item(input: &str) -> Option { + hashify::map_ignore_case!( + input.as_bytes(), NotifyItem, + "success" => NotifyItem::Success, + "failure" => NotifyItem::Failure, + "delay" => NotifyItem::Delay, + ) + .copied() +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_reject.rs b/crates/sieve/src/compiler/grammar/actions/action_reject.rs new file mode 100644 index 00000000..c6d6cba2 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_reject.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::instruction::{CompilerState, Instruction}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Reject { + pub ereject: bool, + pub reason: Value, +} + +impl CompilerState<'_> { + pub(crate) fn parse_reject(&mut self, ereject: bool) -> Result<(), CompileError> { + let cmd = Instruction::Reject(Box::new(Reject { + ereject, + reason: self.parse_string()?, + })); + self.instructions.push(cmd); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_require.rs b/crates/sieve/src/compiler/grammar/actions/action_require.rs new file mode 100644 index 00000000..4177b9a0 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_require.rs @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, + grammar::{ + Capability, + instruction::{CompilerState, Instruction}, + }, + lexer::{Token, tokenizer::TokenInfo}, +}; + +impl CompilerState<'_> { + fn add_capability( + &mut self, + capabilities: &mut Vec, + capability: Capability, + token_info: &TokenInfo, + ) -> Result<(), CompileError> { + if self.compiler.disabled_capabilities.contains(&capability) { + return Err(CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type: ErrorType::UnsupportedCapability(capability), + }); + } + if !self.has_capability(&capability) { + let parent_capability = if matches!(&capability, Capability::SpamTestPlus) { + Some(Capability::SpamTest) + } else { + None + }; + capabilities.push(capability.clone()); + self.block.capabilities.insert(capability); + + if let Some(capability) = parent_capability + && !self.has_capability(&capability) + { + capabilities.push(capability.clone()); + self.block.capabilities.insert(capability); + } + } + Ok(()) + } + + pub(crate) fn parse_require(&mut self) -> Result<(), CompileError> { + let mut capabilities = Vec::new(); + + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::BracketOpen => loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::StringConstant(ref value) => { + self.add_capability( + &mut capabilities, + Capability::parse(value.to_string().as_ref()), + &token_info, + )?; + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Comma => (), + Token::BracketClose => break, + _ => { + return Err(token_info.expected("']' or ','")); + } + } + } + _ => { + return Err(token_info.expected("string")); + } + } + }, + Token::StringConstant(ref value) => { + self.add_capability( + &mut capabilities, + Capability::parse(value.to_string().as_ref()), + &token_info, + )?; + } + _ => { + return Err(token_info.expected("'[' or string")); + } + } + + if !capabilities.is_empty() { + if self.block.require_pos == usize::MAX { + self.block.require_pos = self.instructions.len(); + self.instructions + .push(Instruction::Require(capabilities.into_boxed_slice())); + } else if let Some(Instruction::Require(capabilties)) = + self.instructions.get_mut(self.block.require_pos) + { + let mut merged = std::mem::take(capabilties).into_vec(); + merged.extend(capabilities); + *capabilties = merged.into_boxed_slice(); + } else { + #[cfg(all(test, stalwart_tests))] + panic!( + "Invalid require instruction position {}.", + self.block.require_pos + ) + } + } + + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_set.rs b/crates/sieve/src/compiler/grammar/actions/action_set.rs new file mode 100644 index 00000000..12578553 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_set.rs @@ -0,0 +1,242 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Envelope, + compiler::{ + CompileError, ErrorType, Value, VariableType, + grammar::{ + expr::Expression, + instruction::{CompilerState, Instruction}, + }, + lexer::{Token, tokenizer::TokenInfo, word::Word}, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Modifier { + Lower = 0, + Upper = 1, + LowerFirst = 2, + UpperFirst = 3, + QuoteWildcard = 4, + QuoteRegex = 5, + EncodeUrl = 6, + Length = 7, + Replace(Box) = 8, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Replacement { + pub find: Value, + pub replace: Value, +} + +impl Modifier { + pub fn order(&self) -> usize { + match self { + Modifier::Lower => 41, + Modifier::Upper => 40, + Modifier::LowerFirst => 31, + Modifier::UpperFirst => 30, + Modifier::QuoteWildcard => 20, + Modifier::QuoteRegex => 21, + Modifier::EncodeUrl => 15, + Modifier::Length => 10, + Modifier::Replace(_) => 40, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Set { + pub modifiers: Box<[Modifier]>, + pub name: VariableType, + pub value: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Let { + pub name: VariableType, + pub expr: Box<[Expression]>, +} + +impl CompilerState<'_> { + pub(crate) fn parse_set(&mut self) -> Result<(), CompileError> { + let mut modifiers = Vec::new(); + let mut name = None; + let mut is_local = false; + let value; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Lower + | Word::Upper + | Word::LowerFirst + | Word::UpperFirst + | Word::QuoteWildcard + | Word::QuoteRegex + | Word::Length + | Word::EncodeUrl), + ) => { + let modifier = word.into(); + if !modifiers.contains(&modifier) { + modifiers.push(modifier); + } + } + Token::Tag(Word::Replace) => { + let find = self.tokens.unwrap_next()?; + let replace = self.tokens.unwrap_next()?; + modifiers.push(Modifier::Replace(Box::new(Replacement { + find: self.parse_string_token(find)?, + replace: self.parse_string_token(replace)?, + }))); + } + Token::Tag(Word::Local) => { + is_local = true; + } + _ => { + if name.is_none() { + name = self.parse_variable_name(token_info, is_local)?.into(); + } else { + value = self.parse_string_token(token_info)?; + break; + } + } + } + } + + modifiers.sort_unstable_by_key(|m| std::cmp::Reverse(m.order())); + + self.instructions.push(Instruction::Set(Box::new(Set { + modifiers: modifiers.into(), + name: name.unwrap(), + value, + }))); + Ok(()) + } + + pub(crate) fn parse_let(&mut self) -> Result<(), CompileError> { + let name = self.tokens.unwrap_next()?; + let name = self.parse_variable_name(name, false)?; + let expr = self.parse_expr()?; + + self.instructions.push(Instruction::Let(Box::new(Let { + name, + expr: expr.into(), + }))); + Ok(()) + } + + pub(crate) fn parse_variable_name( + &mut self, + token_info: TokenInfo, + register_as_local: bool, + ) -> Result { + match token_info.token { + Token::StringConstant(value) => self + .register_variable(value.into_string(), register_as_local) + .map_err(|error_type| CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type, + }), + _ => Err(token_info.custom(ErrorType::ExpectedConstantString)), + } + } + + pub(crate) fn register_variable( + &mut self, + name: String, + register_as_local: bool, + ) -> Result { + let name = name.to_lowercase(); + if let Some((namespace, part)) = name.split_once('.') { + let mut variable = None; + hashify::fnc_map!(namespace.as_bytes(), + "global" => { + variable = VariableType::Global(part.to_string()).into(); + }, + "t" => { + variable = VariableType::Global(part.to_string()).into(); + }, + "envelope" => { + variable = Envelope::try_from(part).ok().map(VariableType::Envelope); + }, + _ => {} + ); + + variable.ok_or_else(|| ErrorType::InvalidNamespace(namespace.to_string())) + } else { + Ok(if !self.is_var_global(&name) { + VariableType::Local(self.register_local_var(name, register_as_local)) + } else { + VariableType::Global(name) + }) + } + } +} + +impl From for Modifier { + fn from(word: Word) -> Self { + match word { + Word::Lower => Modifier::Lower, + Word::Upper => Modifier::Upper, + Word::LowerFirst => Modifier::LowerFirst, + Word::UpperFirst => Modifier::UpperFirst, + Word::QuoteWildcard => Modifier::QuoteWildcard, + Word::QuoteRegex => Modifier::QuoteRegex, + Word::Length => Modifier::Length, + Word::EncodeUrl => Modifier::EncodeUrl, + _ => unreachable!(), + } + } +} + +pub(crate) const MODIFIER_LOWER: u8 = 0; +pub(crate) const MODIFIER_UPPER: u8 = 1; +pub(crate) const MODIFIER_LOWER_FIRST: u8 = 2; +pub(crate) const MODIFIER_UPPER_FIRST: u8 = 3; +pub(crate) const MODIFIER_QUOTE_WILDCARD: u8 = 4; +pub(crate) const MODIFIER_QUOTE_REGEX: u8 = 5; +pub(crate) const MODIFIER_ENCODE_URL: u8 = 6; +pub(crate) const MODIFIER_LENGTH: u8 = 7; +pub(crate) const MODIFIER_REPLACE: u8 = 8; + +impl Modifier { + pub(crate) fn code(&self) -> u8 { + match self { + Modifier::Lower => MODIFIER_LOWER, + Modifier::Upper => MODIFIER_UPPER, + Modifier::LowerFirst => MODIFIER_LOWER_FIRST, + Modifier::UpperFirst => MODIFIER_UPPER_FIRST, + Modifier::QuoteWildcard => MODIFIER_QUOTE_WILDCARD, + Modifier::QuoteRegex => MODIFIER_QUOTE_REGEX, + Modifier::EncodeUrl => MODIFIER_ENCODE_URL, + Modifier::Length => MODIFIER_LENGTH, + Modifier::Replace(_) => MODIFIER_REPLACE, + } + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/action_vacation.rs b/crates/sieve/src/compiler/grammar/actions/action_vacation.rs new file mode 100644 index 00000000..cf340c35 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/action_vacation.rs @@ -0,0 +1,232 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + FileCarbonCopy, + compiler::{ + CompileError, Value, + grammar::{ + Capability, + instruction::{CompilerState, Instruction}, + test::Test, + }, + lexer::{Token, word::Word}, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Vacation { + pub subject: Option, + pub from: Option, + pub mime: bool, + pub fcc: Option>, + pub reason: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestVacation { + pub addresses: Box<[Value]>, + pub period: Period, + pub handle: Option, + pub reason: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Period { + Days(u64) = 0, + Seconds(u64) = 1, + Default = 2, +} + +/* + +vacation [":days" number] [":subject" string] + [":from" string] [":addresses" string-list] + [":mime"] [":handle" string] + +vacation [FCC] + [":days" number | ":seconds" number] + [":subject" string] + [":from" string] + [":addresses" string-list] + [":mime"] + [":handle" string] + + +":flags" + + + FCC = ":fcc" string *FCC-OPTS + ; per Section 2.6.2 of RFC 5228, + ; the tagged arguments in FCC may appear in any order + + FCC-OPTS = CREATE / IMAP-FLAGS / SPECIAL-USE + ; each option MUST NOT appear more than once + + CREATE = ":create" + IMAP-FLAGS = ":flags" string-list + SPECIAL-USE = ":specialuse" string +*/ + +impl CompilerState<'_> { + pub(crate) fn parse_vacation(&mut self) -> Result<(), CompileError> { + let mut period = Period::Default; + let mut subject = None; + let mut from = None; + let mut handle = None; + let mut addresses = Vec::new(); + let mut mime = false; + let reason; + + let mut fcc = None; + let mut create = false; + let mut flags = Vec::new(); + let mut special_use = None; + let mut mailbox_id = None; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Mime) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + mime = true; + } + Token::Tag(Word::Create) => { + self.validate_argument( + 2, + Capability::Mailbox.into(), + token_info.line_num, + token_info.line_pos, + )?; + create = true; + } + Token::Tag(Word::Days) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + period = Period::Days(self.tokens.expect_number(u64::MAX as usize)? as u64); + } + Token::Tag(Word::Seconds) => { + self.validate_argument( + 3, + Capability::VacationSeconds.into(), + token_info.line_num, + token_info.line_pos, + )?; + period = Period::Seconds(self.tokens.expect_number(u64::MAX as usize)? as u64); + } + Token::Tag(Word::Subject) => { + self.validate_argument(4, None, token_info.line_num, token_info.line_pos)?; + subject = self.parse_string()?.into(); + } + Token::Tag(Word::From) => { + self.validate_argument(5, None, token_info.line_num, token_info.line_pos)?; + from = self.parse_string()?.into(); + } + Token::Tag(Word::Handle) => { + self.validate_argument(6, None, token_info.line_num, token_info.line_pos)?; + handle = self.parse_string()?.into(); + } + Token::Tag(Word::SpecialUse) => { + self.validate_argument( + 7, + Capability::SpecialUse.into(), + token_info.line_num, + token_info.line_pos, + )?; + special_use = self.parse_string()?.into(); + } + Token::Tag(Word::MailboxId) => { + self.validate_argument( + 8, + Capability::MailboxId.into(), + token_info.line_num, + token_info.line_pos, + )?; + mailbox_id = self.parse_string()?.into(); + } + Token::Tag(Word::Fcc) => { + self.validate_argument( + 9, + Capability::Fcc.into(), + token_info.line_num, + token_info.line_pos, + )?; + fcc = self.parse_string()?.into(); + } + Token::Tag(Word::Flags) => { + self.validate_argument( + 10, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + flags = self.parse_strings(false)?; + } + Token::Tag(Word::Addresses) => { + self.validate_argument(11, None, token_info.line_num, token_info.line_pos)?; + addresses = self.parse_strings(false)?; + } + _ => { + reason = self.parse_string_token(token_info)?; + break; + } + } + } + + if fcc.is_none() + && (create || !flags.is_empty() || special_use.is_some() || mailbox_id.is_some()) + { + return Err(self.tokens.unwrap_next()?.missing_tag(":fcc")); + } + + self.instructions + .push(Instruction::Test(Box::new(Test::Vacation(Box::new( + TestVacation { + period, + handle, + reason: reason.clone(), + addresses: addresses.into(), + }, + ))))); + + self.instructions + .push(Instruction::Jz((self.instructions.len() + 2) as u32)); + + self.instructions + .push(Instruction::Vacation(Box::new(Vacation { + reason, + subject, + from, + mime, + fcc: if let Some(fcc) = fcc { + FileCarbonCopy { + mailbox: fcc, + create, + flags: flags.into(), + special_use, + mailbox_id, + } + .into() + } else { + None + }, + }))); + + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/actions/mod.rs b/crates/sieve/src/compiler/grammar/actions/mod.rs new file mode 100644 index 00000000..c38b867f --- /dev/null +++ b/crates/sieve/src/compiler/grammar/actions/mod.rs @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod action_convert; +pub mod action_editheader; +pub mod action_fileinto; +pub mod action_flags; +pub mod action_include; +pub mod action_keep; +pub mod action_mime; +pub mod action_notify; +pub mod action_redirect; +pub mod action_reject; +pub mod action_require; +pub mod action_set; +pub mod action_vacation; diff --git a/crates/sieve/src/compiler/grammar/expr/mod.rs b/crates/sieve/src/compiler/grammar/expr/mod.rs new file mode 100644 index 00000000..6e7936e2 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/expr/mod.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ConstantId, Number, VariableType}; + +pub mod parser; +pub mod tokenizer; + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Expression { + VariableLocal(u16) = 0, + VariableMatch(u8) = 1, + VariableOther(Box) = 2, + ConstantInteger(i64) = 3, + ConstantFloat(f64) = 4, + ConstantString(ConstantId) = 5, + BinaryOperator(BinaryOperator) = 6, + UnaryOperator(UnaryOperator) = 7, + JmpIf { val: bool, pos: u32 } = 8, + Function { id: u32, num_args: u32 } = 9, + ArrayAccess = 10, + ArrayBuild(u32) = 11, +} + +impl Eq for Expression {} + +impl Expression { + pub(crate) fn variable(variable: VariableType) -> Self { + match variable { + VariableType::Local(id) => Expression::VariableLocal(id), + VariableType::Match(id) => Expression::VariableMatch(id), + other => Expression::VariableOther(Box::new(other)), + } + } + + pub(crate) fn number(number: Number) -> Self { + match number { + Number::Integer(i) => Expression::ConstantInteger(i), + Number::Float(f) => Expression::ConstantFloat(f), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum BinaryOperator { + Add = 0, + Subtract = 1, + Multiply = 2, + Divide = 3, + + And = 4, + Or = 5, + Xor = 6, + + Eq = 7, + Ne = 8, + Lt = 9, + Le = 10, + Gt = 11, + Ge = 12, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum UnaryOperator { + Not = 0, + Minus = 1, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub(crate) enum Token { + Variable(VariableType), + Function { + name: String, + id: u32, + num_args: u32, + }, + Number(Number), + String(String), + BinaryOperator(BinaryOperator), + UnaryOperator(UnaryOperator), + OpenParen, + CloseParen, + OpenBracket, + CloseBracket, + Comma, +} diff --git a/crates/sieve/src/compiler/grammar/expr/parser.rs b/crates/sieve/src/compiler/grammar/expr/parser.rs new file mode 100644 index 00000000..734a4d20 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/expr/parser.rs @@ -0,0 +1,262 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BinaryOperator, Expression, Token, tokenizer::Tokenizer}; +use crate::compiler::ConstantId; + +pub(crate) struct ExpressionParser<'x, F> +where + F: Fn(&str, bool) -> Result, +{ + pub(crate) tokenizer: Tokenizer<'x, F>, + pub(crate) output: Vec, + pub(crate) string_constants: Vec<(usize, String)>, + operator_stack: Vec<(Token, Option)>, + arg_count: Vec, +} + +pub(crate) const ID_ARRAY_ACCESS: u32 = u32::MAX; +pub(crate) const ID_ARRAY_BUILD: u32 = u32::MAX - 1; +pub(crate) const ID_EXTERNAL: u32 = u32::MAX - 2; + +impl<'x, F> ExpressionParser<'x, F> +where + F: Fn(&str, bool) -> Result, +{ + pub fn from_tokenizer(tokenizer: Tokenizer<'x, F>) -> Self { + Self { + tokenizer, + output: Vec::new(), + string_constants: Vec::new(), + operator_stack: Vec::new(), + arg_count: Vec::new(), + } + } + + pub fn parse(mut self) -> Result { + let mut last_is_var_or_fnc = false; + + while let Some(token) = self.tokenizer.next()? { + let mut is_var_or_fnc = false; + match token { + Token::Variable(v) => { + self.inc_arg_count(); + is_var_or_fnc = true; + self.output.push(Expression::variable(v)) + } + Token::Number(n) => { + self.inc_arg_count(); + self.output.push(Expression::number(n)) + } + Token::String(s) => { + self.inc_arg_count(); + self.string_constants.push((self.output.len(), s)); + self.output + .push(Expression::ConstantString(ConstantId::UNRESOLVED)) + } + Token::UnaryOperator(uop) => { + self.operator_stack.push((Token::UnaryOperator(uop), None)) + } + Token::OpenParen => self.operator_stack.push((token, None)), + Token::CloseParen | Token::CloseBracket => { + let expect_token = if matches!(token, Token::CloseParen) { + Token::OpenParen + } else { + Token::OpenBracket + }; + loop { + match self.operator_stack.pop() { + Some((t, _)) if t == expect_token => { + break; + } + Some((Token::BinaryOperator(bop), jmp_pos)) => { + self.update_jmp_pos(jmp_pos); + self.output.push(Expression::BinaryOperator(bop)) + } + Some((Token::UnaryOperator(uop), _)) => { + self.output.push(Expression::UnaryOperator(uop)) + } + _ => return Err("Mismatched parentheses".to_string()), + } + } + + if let Some((Token::Function { id, num_args, name }, _)) = + self.operator_stack.last() + { + let got_args = self.arg_count.pop().unwrap(); + if got_args != *num_args as i32 { + return Err(if *id != u32::MAX { + format!( + "Expression function {:?} expected {} arguments, got {}", + name, num_args, got_args + ) + } else { + "Missing array index".to_string() + }); + } + + let expr = match *id { + ID_ARRAY_ACCESS => Expression::ArrayAccess, + ID_ARRAY_BUILD => Expression::ArrayBuild(*num_args), + id => Expression::Function { + id, + num_args: *num_args, + }, + }; + + self.operator_stack.pop(); + self.output.push(expr); + } + + is_var_or_fnc = true; + } + Token::BinaryOperator(bop) => { + self.dec_arg_count(); + while let Some((top_token, prev_jmp_pos)) = self.operator_stack.last() { + match top_token { + Token::BinaryOperator(top_bop) => { + if bop.precedence() <= top_bop.precedence() { + let top_bop = *top_bop; + let jmp_pos = *prev_jmp_pos; + self.update_jmp_pos(jmp_pos); + self.operator_stack.pop(); + self.output.push(Expression::BinaryOperator(top_bop)); + } else { + break; + } + } + Token::UnaryOperator(top_uop) => { + let top_uop = *top_uop; + self.operator_stack.pop(); + self.output.push(Expression::UnaryOperator(top_uop)); + } + _ => break, + } + } + + // Add jump instruction for short-circuiting + let jmp_pos = match bop { + BinaryOperator::And => { + self.output.push(Expression::JmpIf { val: false, pos: 0 }); + Some(self.output.len() - 1) + } + BinaryOperator::Or => { + self.output.push(Expression::JmpIf { val: true, pos: 0 }); + Some(self.output.len() - 1) + } + _ => None, + }; + + self.operator_stack + .push((Token::BinaryOperator(bop), jmp_pos)); + } + Token::Function { id, name, num_args } => { + self.inc_arg_count(); + self.arg_count.push(0); + self.operator_stack + .push((Token::Function { id, name, num_args }, None)) + } + Token::OpenBracket => { + // Array functions + let (id, num_args, arg_count) = if last_is_var_or_fnc { + (ID_ARRAY_ACCESS, 2, 1) + } else { + self.inc_arg_count(); + (ID_ARRAY_BUILD, 0, 0) + }; + self.arg_count.push(arg_count); + self.operator_stack.push(( + Token::Function { + id, + name: String::from("array"), + num_args, + }, + None, + )); + self.operator_stack.push((token, None)); + } + Token::Comma => { + while let Some((token, jmp_pos)) = self.operator_stack.last() { + match token { + Token::OpenParen => break, + Token::BinaryOperator(bop) => { + let bop = *bop; + let jmp_pos = *jmp_pos; + self.update_jmp_pos(jmp_pos); + self.output.push(Expression::BinaryOperator(bop)); + self.operator_stack.pop(); + } + Token::UnaryOperator(uop) => { + self.output.push(Expression::UnaryOperator(*uop)); + self.operator_stack.pop(); + } + _ => break, + } + } + } + } + last_is_var_or_fnc = is_var_or_fnc; + } + + while let Some((token, jmp_pos)) = self.operator_stack.pop() { + match token { + Token::BinaryOperator(bop) => { + self.update_jmp_pos(jmp_pos); + self.output.push(Expression::BinaryOperator(bop)) + } + Token::UnaryOperator(uop) => self.output.push(Expression::UnaryOperator(uop)), + _ => return Err("Invalid token on the operator stack".to_string()), + } + } + + Ok(self) + } + + fn inc_arg_count(&mut self) { + if let Some(x) = self.arg_count.last_mut() { + *x = x.saturating_add(1); + let op_pos = self.operator_stack.len().saturating_sub(2); + match self.operator_stack.get_mut(op_pos) { + Some((Token::Function { num_args, id, .. }, _)) if *id == ID_ARRAY_BUILD => { + *num_args += 1; + } + _ => {} + } + } + } + + fn dec_arg_count(&mut self) { + if let Some(x) = self.arg_count.last_mut() { + *x = x.saturating_sub(1); + } + } + + fn update_jmp_pos(&mut self, jmp_pos: Option) { + if let Some(jmp_pos) = jmp_pos { + let cur_pos = self.output.len(); + if let Expression::JmpIf { pos, .. } = &mut self.output[jmp_pos] { + *pos = (cur_pos - jmp_pos) as u32; + } else { + #[cfg(all(test, stalwart_tests))] + panic!("Invalid jump position"); + } + } + } +} + +impl BinaryOperator { + fn precedence(&self) -> i32 { + match self { + BinaryOperator::Multiply | BinaryOperator::Divide => 7, + BinaryOperator::Add | BinaryOperator::Subtract => 6, + BinaryOperator::Gt | BinaryOperator::Ge | BinaryOperator::Lt | BinaryOperator::Le => 5, + BinaryOperator::Eq | BinaryOperator::Ne => 4, + BinaryOperator::Xor => 3, + BinaryOperator::And => 2, + BinaryOperator::Or => 1, + } + } +} diff --git a/crates/sieve/src/compiler/grammar/expr/tokenizer.rs b/crates/sieve/src/compiler/grammar/expr/tokenizer.rs new file mode 100644 index 00000000..387d2ae1 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/expr/tokenizer.rs @@ -0,0 +1,300 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{ + iter::{Enumerate, Peekable}, + slice::Iter, +}; + +use crate::compiler::Number; + +use super::{BinaryOperator, Token, UnaryOperator}; + +pub(crate) struct Tokenizer<'x, F> +where + F: Fn(&str, bool) -> Result, +{ + pub(crate) iter: Peekable>>, + token_map: F, + buf: Vec, + depth: u32, + next_token: Vec, + has_number: bool, + has_dot: bool, + has_alpha: bool, + is_start: bool, + is_eof: bool, +} + +impl<'x, F> Tokenizer<'x, F> +where + F: Fn(&str, bool) -> Result, +{ + #[cfg(all(test, stalwart_tests))] + pub fn new(expr: &'x str, token_map: F) -> Self { + Self::from_iter(expr.as_bytes().iter().enumerate().peekable(), token_map) + } + + #[allow(clippy::should_implement_trait)] + pub(crate) fn from_iter(iter: Peekable>>, token_map: F) -> Self { + Self { + iter, + buf: Vec::new(), + depth: 0, + next_token: Vec::with_capacity(2), + has_number: false, + has_dot: false, + has_alpha: false, + is_start: true, + is_eof: false, + token_map, + } + } + + #[allow(clippy::should_implement_trait)] + pub(crate) fn next(&mut self) -> Result, String> { + if let Some(token) = self.next_token.pop() { + return Ok(Some(token)); + } else if self.is_eof { + return Ok(None); + } + + while let Some((_, &ch)) = self.iter.next() { + match ch { + b'A'..=b'Z' | b'a'..=b'z' | b'_' => { + self.buf.push(ch); + self.has_alpha = true; + } + b'0'..=b'9' => { + self.buf.push(ch); + self.has_number = true; + } + b'.' => { + self.buf.push(ch); + self.has_dot = true; + } + b'}' => { + self.is_eof = true; + break; + } + b'[' if matches!(self.buf.get(0..7), Some(b"header.")) => { + self.buf.push(ch); + } + b'-' if self.buf.last().is_some_and(|c| *c == b'[') + || matches!(self.buf.get(0..7), Some(b"header.")) => + { + self.buf.push(ch); + } + b':' if self.buf.contains(&b'.') => { + self.buf.push(ch); + } + b']' if self.buf.contains(&b'[') => { + self.buf.push(b']'); + } + b'*' if self.buf.last().is_some_and(|&c| c == b'[' || c == b'.') => { + self.buf.push(ch); + } + _ => { + let prev_token = if !self.buf.is_empty() { + self.is_start = false; + self.parse_buf()?.into() + } else { + None + }; + let token = match ch { + b'&' => { + if matches!(self.iter.peek(), Some((_, b'&'))) { + self.iter.next(); + } + Token::BinaryOperator(BinaryOperator::And) + } + b'|' => { + if matches!(self.iter.peek(), Some((_, b'|'))) { + self.iter.next(); + } + Token::BinaryOperator(BinaryOperator::Or) + } + b'!' => { + if matches!(self.iter.peek(), Some((_, b'='))) { + self.iter.next(); + Token::BinaryOperator(BinaryOperator::Ne) + } else { + Token::UnaryOperator(UnaryOperator::Not) + } + } + b'^' => Token::BinaryOperator(BinaryOperator::Xor), + b'(' => { + self.depth += 1; + Token::OpenParen + } + b')' => { + if self.depth == 0 { + return Err("Unmatched close parenthesis".to_string()); + } + self.depth -= 1; + Token::CloseParen + } + b'+' => Token::BinaryOperator(BinaryOperator::Add), + b'*' => Token::BinaryOperator(BinaryOperator::Multiply), + b'/' => Token::BinaryOperator(BinaryOperator::Divide), + b'-' => { + if self.is_start { + Token::UnaryOperator(UnaryOperator::Minus) + } else { + Token::BinaryOperator(BinaryOperator::Subtract) + } + } + b'=' => match self.iter.next() { + Some((_, b'=')) => Token::BinaryOperator(BinaryOperator::Eq), + Some((_, b'>')) => Token::BinaryOperator(BinaryOperator::Ge), + Some((_, b'<')) => Token::BinaryOperator(BinaryOperator::Le), + _ => Token::BinaryOperator(BinaryOperator::Eq), + }, + b'>' => match self.iter.peek() { + Some((_, b'=')) => { + self.iter.next(); + Token::BinaryOperator(BinaryOperator::Ge) + } + _ => Token::BinaryOperator(BinaryOperator::Gt), + }, + b'<' => match self.iter.peek() { + Some((_, b'=')) => { + self.iter.next(); + Token::BinaryOperator(BinaryOperator::Le) + } + _ => Token::BinaryOperator(BinaryOperator::Lt), + }, + b',' => Token::Comma, + b'[' => Token::OpenBracket, + b']' => Token::CloseBracket, + b' ' | b'\r' | b'\n' => { + if prev_token.is_some() { + return Ok(prev_token); + } else { + continue; + } + } + b'\"' | b'\'' => { + let mut buf = Vec::with_capacity(16); + let stop_ch = ch; + let mut last_ch = 0; + let mut found_end = false; + + for (_, &ch) in self.iter.by_ref() { + if last_ch != b'\\' { + if ch != stop_ch { + buf.push(ch); + } else { + found_end = true; + break; + } + } else { + match ch { + b'n' => { + buf.push(b'\n'); + } + b'r' => { + buf.push(b'\r'); + } + b't' => { + buf.push(b'\t'); + } + _ => { + buf.push(ch); + } + } + } + + last_ch = ch; + } + + if found_end { + Token::String( + String::from_utf8(buf) + .map_err(|_| "Invalid UTF-8".to_string())?, + ) + } else { + return Err("Unterminated string".to_string()); + } + } + _ => { + return Err(format!("Invalid character {:?}", char::from(ch),)); + } + }; + self.is_start = matches!( + token, + Token::OpenParen | Token::Comma | Token::BinaryOperator(_) + ); + + return if prev_token.is_some() { + self.next_token.push(token); + Ok(prev_token) + } else { + Ok(Some(token)) + }; + } + } + } + + if self.depth > 0 { + Err("Unmatched open parenthesis".to_string()) + } else if !self.buf.is_empty() { + self.parse_buf().map(Some) + } else { + Ok(None) + } + } + + fn parse_buf(&mut self) -> Result { + let bytes = std::mem::take(&mut self.buf); + let result = self.parse_word(&String::from_utf8_lossy(&bytes)); + self.buf = bytes; + self.buf.clear(); + result + } + + fn parse_word(&mut self, buf: &str) -> Result { + if self.has_number && !self.has_alpha { + self.has_number = false; + if self.has_dot { + self.has_dot = false; + + buf.parse::() + .map(|f| Token::Number(Number::Float(f))) + .map_err(|_| format!("Invalid float value {}", buf,)) + } else { + buf.parse::() + .map(|i| Token::Number(Number::Integer(i))) + .map_err(|_| format!("Invalid integer value {}", buf,)) + } + } else { + let has_dot = self.has_dot; + let has_number = self.has_number; + + self.has_alpha = false; + self.has_number = false; + self.has_dot = false; + + if !has_number + && !has_dot + && let Some(value) = lookup_boolean(buf) + { + return Ok(Token::Number(Number::Integer(value))); + } + + (self.token_map)(buf, has_dot) + } + } +} + +fn lookup_boolean(input: &str) -> Option { + hashify::map!( + input.as_bytes(), i64, + "true" => 1, + "false" => 0, + ) + .copied() +} diff --git a/crates/sieve/src/compiler/grammar/instruction.rs b/crates/sieve/src/compiler/grammar/instruction.rs new file mode 100644 index 00000000..54079380 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/instruction.rs @@ -0,0 +1,1344 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::tests::test_ihave::Error; +use super::{ + Capability, Clear, Invalid, While, + actions::{ + action_convert::Convert, + action_editheader::{AddHeader, DeleteHeader}, + action_fileinto::FileInto, + action_flags::EditFlags, + action_include::Include, + action_keep::Keep, + action_mime::{Enclose, ExtractText, ForEveryPart, Replace}, + action_notify::Notify, + action_redirect::Redirect, + action_reject::Reject, + action_set::{Let, Set}, + action_vacation::Vacation, + }, + expr::Expression, +}; +use crate::{ + Compiler, Sieve, + compiler::{ + CompileError, ConstantId, ErrorType, RawValue, Value, VariableType, + grammar::{MatchType, test::Test}, + lexer::{Token, tokenizer::Tokenizer, word::Word}, + }, +}; +use ahash::{AHashMap, AHashSet}; +use hashbrown::HashTable; + +#[derive(Debug, Clone, Eq, PartialEq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Instruction { + Require(Box<[Capability]>) = 0, + Keep(Box) = 1, + FileInto(Box) = 2, + Redirect(Box) = 3, + Discard = 4, + Stop = 5, + Invalid(Box) = 6, + Test(Box) = 7, + Jmp(u32) = 8, + Jz(u32) = 9, + Jnz(u32) = 10, + + // RFC 5703 + ForEveryPartPush = 11, + ForEveryPart(ForEveryPart) = 12, + ForEveryPartPop(u32) = 13, + Replace(Box) = 14, + Enclose(Box) = 15, + ExtractText(Box) = 16, + + // RFC 6558 + Convert(Box) = 17, + + // RFC 5293 + AddHeader(Box) = 18, + DeleteHeader(Box) = 19, + + // RFC 5229 + Set(Box) = 20, + Clear(Box) = 21, + + // RFC 5435 + Notify(Box) = 22, + + // RFC 5429 + Reject(Box) = 23, + + // RFC 5230 + Vacation(Box) = 24, + + // RFC 5463 + Error(Box) = 25, + + // RFC 5232 + EditFlags(Box) = 26, + + // RFC 6609 + Include(Box) = 27, + Return = 28, + + // For every line extension + While(Box) = 29, + + // Expression extension + Eval(Box<[Expression]>) = 30, + Let(Box) = 31, + + // Test only + #[cfg(all(test, stalwart_tests))] + TestCmd(Box<[Value]>) = 0x7f, +} + +pub(crate) const MAX_PARAMS: usize = 11; + +#[derive(Debug)] +pub(crate) struct Block { + pub(crate) btype: Word, + pub(crate) label: Option, + pub(crate) line_num: usize, + pub(crate) line_pos: usize, + pub(crate) last_block_start: usize, + pub(crate) if_jmps: Vec, + pub(crate) break_jmps: Vec, + pub(crate) match_test_pos: Vec, + pub(crate) match_test_vars: u64, + pub(crate) vars_local: AHashMap, + pub(crate) capabilities: AHashSet, + pub(crate) require_pos: usize, +} + +pub(crate) struct CompilerState<'x> { + pub(crate) compiler: &'x Compiler, + pub(crate) tokens: Tokenizer<'x>, + pub(crate) instructions: Vec, + pub(crate) block_stack: Vec, + pub(crate) block: Block, + pub(crate) last_block_type: Word, + pub(crate) vars_global: AHashSet, + pub(crate) vars_num: usize, + pub(crate) vars_num_max: usize, + pub(crate) vars_match_max: usize, + pub(crate) vars_local: usize, + pub(crate) param_check: [bool; MAX_PARAMS], + pub(crate) includes_num: usize, + pub(crate) constants: Vec, + pub(crate) constants_map: HashTable, + pub(crate) hasher: ahash::RandomState, +} + +impl CompilerState<'_> { + pub(crate) fn intern(&mut self, text: impl AsRef) -> ConstantId { + let text = text.as_ref(); + match self.lookup_constant(text) { + Ok(id) => id, + Err(id) => { + self.constants.push(text.to_string()); + id + } + } + } + + pub(crate) fn intern_string(&mut self, text: String) -> ConstantId { + match self.lookup_constant(&text) { + Ok(id) => id, + Err(id) => { + self.constants.push(text); + id + } + } + } + + fn lookup_constant(&mut self, text: &str) -> Result { + let hash = self.hasher.hash_one(text); + let constants = &self.constants; + match self.constants_map.entry( + hash, + |id| constants.get(id.index()).is_some_and(|c| c == text), + |id| { + self.hasher + .hash_one(constants.get(id.index()).map_or("", |c| c.as_str())) + }, + ) { + hashbrown::hash_table::Entry::Occupied(entry) => Ok(*entry.get()), + hashbrown::hash_table::Entry::Vacant(entry) => { + let id = ConstantId::new(constants.len()); + entry.insert(id); + Err(id) + } + } + } + + pub(crate) fn constant(&self, id: ConstantId) -> &str { + self.constants.get(id.index()).map_or("", |v| v.as_ref()) + } + + pub(crate) fn text(&mut self, text: impl AsRef) -> Value { + Value::Text(self.intern(text)) + } + + pub(crate) fn intern_raw(&mut self, value: RawValue) -> Value { + match value { + RawValue::Text(text) => Value::Text(self.intern_string(text)), + RawValue::Number(number) => Value::Number(number), + RawValue::Value(value) => value, + } + } + + pub(crate) fn intern_raw_list(&mut self, values: Vec) -> Vec { + let mut result = Vec::with_capacity(values.len()); + for value in values { + let value = self.intern_raw(value); + result.push(value); + } + result + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Program { + pub(crate) instructions: Vec, + pub(crate) constants: Vec, + pub(crate) num_vars: u32, + pub(crate) num_match_vars: u32, +} + +impl Program { + pub(crate) fn emit(&self) -> Result, crate::sieve::LoadError> { + crate::compiler::emit::emit( + &self.instructions, + &self.constants, + self.num_vars, + self.num_match_vars, + ) + } +} + +impl Compiler { + pub fn compile(&self, script: &[u8]) -> Result, CompileError> { + let program = self.compile_ast(script)?; + program.emit().map_err(|_| CompileError { + line_num: 0, + line_pos: 0, + error_type: ErrorType::ScriptTooLong, + }) + } + + pub(crate) fn compile_ast(&self, script: &[u8]) -> Result { + if script.len() > self.max_script_size { + return Err(CompileError { + line_num: 0, + line_pos: 0, + error_type: ErrorType::ScriptTooLong, + }); + } + + let mut state = CompilerState { + compiler: self, + tokens: Tokenizer::new(self, script), + instructions: Vec::new(), + block_stack: Vec::new(), + block: Block::new(Word::Not), + last_block_type: Word::Not, + vars_global: AHashSet::new(), + vars_num: 0, + vars_num_max: 0, + vars_match_max: 0, + vars_local: 0, + param_check: [false; MAX_PARAMS], + includes_num: 0, + constants: Vec::new(), + constants_map: HashTable::new(), + hasher: ahash::RandomState::new(), + }; + + while let Some(token_info) = state.tokens.next() { + let token_info = token_info?; + state.reset_param_check(); + + match token_info.token { + Token::Identifier(instruction) => { + let mut is_new_block = None; + + match instruction { + Word::Require => { + state.parse_require()?; + } + Word::If => { + state.parse_test()?; + state.block.if_jmps.clear(); + is_new_block = Block::new(Word::If).into(); + } + Word::ElsIf => { + if let Word::If | Word::ElsIf = &state.last_block_type { + state.parse_test()?; + is_new_block = Block::new(Word::ElsIf).into(); + } else { + return Err(token_info.expected("'if' before 'elsif'")); + } + } + Word::Else => { + if let Word::If | Word::ElsIf = &state.last_block_type { + is_new_block = Block::new(Word::Else).into(); + } else { + return Err(token_info.expected("'if' or 'elsif' before 'else'")); + } + } + Word::Keep => { + state.parse_keep()?; + } + Word::FileInto => { + state.validate_argument( + 0, + Capability::FileInto.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_fileinto()?; + } + Word::Redirect => { + state.parse_redirect()?; + } + Word::Discard => { + state.instructions.push(Instruction::Discard); + } + Word::Stop => { + state.instructions.push(Instruction::Stop); + } + + // RFC 5703 + Word::ForEveryPart => { + state.validate_argument( + 0, + Capability::ForEveryPart.into(), + token_info.line_num, + token_info.line_pos, + )?; + + if state + .block_stack + .iter() + .filter(|b| matches!(&b.btype, Word::ForEveryPart)) + .count() + == self.max_nested_foreverypart + { + return Err( + token_info.custom(ErrorType::TooManyNestedForEveryParts) + ); + } + + is_new_block = if let Some(Ok(Token::Tag(Word::Name))) = + state.tokens.peek().map(|r| r.map(|t| &t.token)) + { + let tag = state.tokens.next().unwrap().unwrap(); + let label = state.tokens.expect_static_string()?; + for block in &state.block_stack { + if block.label.as_ref().is_some_and(|n| n.eq(&label)) { + return Err( + tag.custom(ErrorType::LabelAlreadyDefined(label)) + ); + } + } + Block::new(Word::ForEveryPart).with_label(label) + } else { + Block::new(Word::ForEveryPart) + } + .into(); + + state.instructions.push(Instruction::ForEveryPartPush); + state + .instructions + .push(Instruction::ForEveryPart(ForEveryPart { jz_pos: u32::MAX })); + } + Word::Break => { + if let Some(Ok(Token::Tag(Word::Name))) = + state.tokens.peek().map(|r| r.map(|t| &t.token)) + { + state.validate_argument( + 0, + Capability::ForEveryPart.into(), + token_info.line_num, + token_info.line_pos, + )?; + + let tag = state.tokens.next().unwrap().unwrap(); + let label = state.tokens.expect_static_string()?; + let mut label_found = false; + let mut num_pops = 0; + + for block in [&mut state.block] + .into_iter() + .chain(state.block_stack.iter_mut().rev()) + { + if let Word::ForEveryPart = &block.btype { + num_pops += 1; + if block.label.as_ref().is_some_and(|n| n.eq(&label)) { + state.instructions.push(Instruction::ForEveryPartPop( + num_pops as u32, + )); + block.break_jmps.push(state.instructions.len()); + label_found = true; + break; + } + } + } + + if !label_found { + return Err(tag.custom(ErrorType::LabelUndefined(label))); + } + } else { + let mut block_found = None; + if matches!(&state.block.btype, Word::ForEveryPart | Word::While) { + block_found = Some(&mut state.block); + } else { + for block in state.block_stack.iter_mut().rev() { + if matches!(&block.btype, Word::ForEveryPart | Word::While) + { + block_found = Some(block); + break; + } + } + } + + let block = block_found.ok_or_else(|| { + token_info.custom(ErrorType::BreakOutsideLoop) + })?; + if matches!(block.btype, Word::ForEveryPart) { + state.instructions.push(Instruction::ForEveryPartPop(1)); + } + + block.break_jmps.push(state.instructions.len()); + } + + state.instructions.push(Instruction::Jmp(u32::MAX)); + } + Word::Replace => { + state.validate_argument( + 0, + Capability::Replace.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_replace()?; + } + Word::Enclose => { + state.validate_argument( + 0, + Capability::Enclose.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_enclose()?; + } + Word::ExtractText => { + state.validate_argument( + 0, + Capability::ExtractText.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_extracttext()?; + } + + // RFC 6558 + Word::Convert => { + state.validate_argument( + 0, + Capability::Convert.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_convert()?; + } + + // RFC 5293 + Word::AddHeader => { + state.validate_argument( + 0, + Capability::EditHeader.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_addheader()?; + } + Word::DeleteHeader => { + state.validate_argument( + 0, + Capability::EditHeader.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_deleteheader()?; + } + + // RFC 5229 + Word::Set => { + state.validate_argument( + 0, + Capability::Variables.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_set()?; + } + + // RFC 5435 + Word::Notify => { + state.validate_argument( + 0, + Capability::Enotify.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_notify()?; + } + + // RFC 5429 + Word::Reject => { + state.validate_argument( + 0, + Capability::Reject.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_reject(false)?; + } + Word::Ereject => { + state.validate_argument( + 0, + Capability::Ereject.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_reject(true)?; + } + + // RFC 5230 + Word::Vacation => { + state.validate_argument( + 0, + Capability::Vacation.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_vacation()?; + } + + // RFC 5463 + Word::Error => { + state.validate_argument( + 0, + Capability::Ihave.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_error()?; + } + + // RFC 5232 + Word::SetFlag | Word::AddFlag | Word::RemoveFlag => { + state.validate_argument( + 0, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_flag_action(instruction)?; + } + + // RFC 6609 + Word::Include => { + if state.includes_num < self.max_includes { + state.validate_argument( + 0, + Capability::Include.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_include()?; + state.includes_num += 1; + } else { + return Err(token_info.custom(ErrorType::TooManyIncludes)); + } + } + Word::Return => { + state.validate_argument( + 0, + Capability::Include.into(), + token_info.line_num, + token_info.line_pos, + )?; + let mut num_pops = 0; + + for block in [&state.block] + .into_iter() + .chain(state.block_stack.iter().rev()) + { + if let Word::ForEveryPart = &block.btype { + num_pops += 1; + } + } + + if num_pops > 0 { + state + .instructions + .push(Instruction::ForEveryPartPop(num_pops as u32)); + } + + state.instructions.push(Instruction::Return); + } + Word::Global => { + state.validate_argument( + 0, + Capability::Include.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.validate_argument( + 0, + Capability::Variables.into(), + token_info.line_num, + token_info.line_pos, + )?; + for global in state.parse_static_strings()? { + if !state.is_var_local(&global) { + if global.len() < self.max_variable_name_size { + state.register_global_var(&global); + } else { + return Err(state + .tokens + .unwrap_next()? + .custom(ErrorType::VariableTooLong)); + } + } else { + return Err(state + .tokens + .unwrap_next()? + .custom(ErrorType::VariableIsLocal(global))); + } + } + } + + // Expressions extension + Word::Let => { + state.validate_argument( + 0, + Capability::Expressions.into(), + token_info.line_num, + token_info.line_pos, + )?; + state.parse_let()?; + } + Word::Eval => { + state.validate_argument( + 0, + Capability::Expressions.into(), + token_info.line_num, + token_info.line_pos, + )?; + let expr = state.parse_expr()?; + state + .instructions + .push(Instruction::Eval(expr.into_boxed_slice())); + } + + // While extension + Word::While => { + state.validate_argument( + 0, + Capability::While.into(), + token_info.line_num, + token_info.line_pos, + )?; + + is_new_block = Block::new(Word::While).into(); + + let expr = state.parse_expr()?; + state.instructions.push(Instruction::While(Box::new(While { + expr: expr.into(), + jz_pos: u32::MAX, + }))); + } + Word::Continue => { + state.validate_argument( + 0, + Capability::While.into(), + token_info.line_num, + token_info.line_pos, + )?; + let mut found_while = 0; + for block in [&state.block] + .into_iter() + .chain(state.block_stack.iter().rev()) + { + if let Word::While = &block.btype { + found_while += 1; + } else if found_while == 1 { + state + .instructions + .push(Instruction::Jmp(block.last_block_start as u32)); + found_while += 1; + break; + } + } + if found_while != 2 { + return Err(token_info.custom(ErrorType::ContinueOutsideLoop)); + } + } + + _ => { + if state.has_capability(&Capability::Ihave) { + state.ignore_instruction()?; + state + .instructions + .push(Instruction::Invalid(Box::new(Invalid { + name: instruction.to_string(), + line_num: token_info.line_num as u32, + line_pos: (token_info.line_pos) as u32, + }))); + continue; + } else { + return Err(CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type: ErrorType::UnexpectedToken { + expected: "command".into(), + found: instruction.to_string(), + }, + }); + } + } + } + + if let Some(mut new_block) = is_new_block { + new_block.line_num = state.tokens.line_num; + new_block.line_pos = state.tokens.pos.wrapping_sub(state.tokens.line_start); + + state.tokens.expect_token(Token::CurlyOpen)?; + if state.block_stack.len() < self.max_nested_blocks { + state.block.last_block_start = state.instructions.len() - 1; + state.block_stack.push(state.block); + state.block = new_block; + } else { + return Err(CompileError { + line_num: state.block.line_num, + line_pos: state.block.line_pos, + error_type: ErrorType::TooManyNestedBlocks, + }); + } + } else { + state.expect_instruction_end()?; + } + } + Token::CurlyClose if !state.block_stack.is_empty() => { + state.block_end(); + let mut prev_block = state.block_stack.pop().unwrap(); + match &state.block.btype { + Word::ForEveryPart => { + state + .instructions + .push(Instruction::Jmp(prev_block.last_block_start as u32)); + let cur_pos = state.instructions.len(); + if let Instruction::ForEveryPart(fep) = + &mut state.instructions[prev_block.last_block_start] + { + fep.jz_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + for pos in state.block.break_jmps { + if let Instruction::Jmp(jmp_pos) = &mut state.instructions[pos] { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + } + state.last_block_type = Word::Not; + } + Word::If | Word::ElsIf => { + let next_is_block = matches!( + state.tokens.peek().map(|r| r.map(|t| &t.token)), + Some(Ok(Token::Identifier(Word::ElsIf | Word::Else))) + ); + if next_is_block { + prev_block.if_jmps.push(state.instructions.len()); + state.instructions.push(Instruction::Jmp(u32::MAX)); + } + let cur_pos = state.instructions.len(); + if let Instruction::Jz(jmp_pos) = + &mut state.instructions[prev_block.last_block_start] + { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + if !next_is_block { + for pos in prev_block.if_jmps.drain(..) { + if let Instruction::Jmp(jmp_pos) = &mut state.instructions[pos] + { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + } + state.last_block_type = Word::Not; + } else { + state.last_block_type = state.block.btype; + } + } + Word::Else => { + let cur_pos = state.instructions.len(); + for pos in prev_block.if_jmps.drain(..) { + if let Instruction::Jmp(jmp_pos) = &mut state.instructions[pos] { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + } + state.last_block_type = Word::Else; + } + Word::While => { + state + .instructions + .push(Instruction::Jmp(prev_block.last_block_start as u32)); + let cur_pos = state.instructions.len(); + if let Instruction::While(fep) = + &mut state.instructions[prev_block.last_block_start] + { + fep.jz_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + for pos in state.block.break_jmps { + if let Instruction::Jmp(jmp_pos) = &mut state.instructions[pos] { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened."); + } + } + state.last_block_type = Word::Not; + } + _ => { + debug_assert!(false, "This should not have happened."); + } + } + + state.block = prev_block; + } + + #[cfg(all(test, stalwart_tests))] + Token::Unknown(instruction) if instruction.contains("test") => { + let has_arguments = instruction != "test"; + let mut arguments = vec![state.text(&instruction)]; + + if !has_arguments { + arguments.push(state.parse_string()?); + state + .instructions + .push(Instruction::TestCmd(arguments.into_boxed_slice())); + let mut new_block = Block::new(Word::Else); + new_block.line_num = state.tokens.line_num; + new_block.line_pos = state.tokens.pos.wrapping_sub(state.tokens.line_start); + state.tokens.expect_token(Token::CurlyOpen)?; + state.block.last_block_start = state.instructions.len() - 1; + state.block_stack.push(state.block); + state.block = new_block; + } else { + loop { + arguments.push(match state.tokens.unwrap_next()?.token { + Token::StringConstant(s) => state.intern_raw(s.into()), + Token::StringVariable(s) => state + .tokenize_string(&s, true) + .map_err(|error_type| CompileError { + line_num: 0, + line_pos: 0, + error_type, + })?, + Token::Number(n) => { + Value::Number(crate::compiler::Number::Integer(n as i64)) + } + Token::Identifier(s) => state.text(s.to_string()), + Token::Tag(s) => state.text(format!(":{s}")), + Token::Unknown(s) => state.text(s), + Token::Semicolon => break, + other => panic!("Invalid test param {other:?}"), + }); + } + state + .instructions + .push(Instruction::TestCmd(arguments.into_boxed_slice())); + } + } + + Token::Unknown(instruction) => { + if state.has_capability(&Capability::Ihave) { + state.ignore_instruction()?; + state + .instructions + .push(Instruction::Invalid(Box::new(Invalid { + name: instruction, + line_num: token_info.line_num as u32, + line_pos: (token_info.line_pos) as u32, + }))); + } else { + return Err(CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type: ErrorType::UnexpectedToken { + expected: "command".into(), + found: instruction, + }, + }); + } + } + _ => { + return Err(token_info.expected("instruction")); + } + } + } + + if !state.block_stack.is_empty() { + return Err(CompileError { + line_num: state.block.line_num, + line_pos: state.block.line_pos, + error_type: ErrorType::UnterminatedBlock, + }); + } + + // Map local variables + let mut num_vars = std::cmp::max(state.vars_num_max, state.vars_num); + if state.vars_local > 0 { + state.map_local_vars(num_vars as u16); + num_vars += state.vars_local; + } + + Ok(Program { + instructions: state.instructions, + constants: state.constants, + num_vars: num_vars as u32, + num_match_vars: state.vars_match_max as u32, + }) + } +} + +impl CompilerState<'_> { + pub(crate) fn is_var_local(&self, name: &str) -> bool { + let name = lowercase(name); + let name = name.as_ref(); + if self.block.vars_local.contains_key(name) { + true + } else { + for block in self.block_stack.iter().rev() { + if block.vars_local.contains_key(name) { + return true; + } + } + false + } + } + + pub(crate) fn is_var_global(&self, name: &str) -> bool { + self.vars_global.contains(lowercase(name).as_ref()) + } + + pub(crate) fn register_local_var(&mut self, name: String, register_as_local: bool) -> u16 { + if let Some(var_id) = self.get_local_var(&name) { + var_id + } else if !register_as_local || self.block_stack.is_empty() { + let var_id = self.vars_num as u16; + self.block_stack + .first_mut() + .unwrap_or(&mut self.block) + .vars_local + .insert(name, var_id); + self.vars_num += 1; + var_id + } else { + let var_id = u16::MAX - self.vars_local as u16; + self.block_stack + .first_mut() + .unwrap() + .vars_local + .insert(name, var_id); + self.vars_local += 1; + var_id + } + } + + pub(crate) fn register_global_var(&mut self, name: &str) { + self.vars_global.insert(name.to_ascii_lowercase()); + } + + pub(crate) fn get_local_var(&self, name: &str) -> Option { + let name = lowercase(name); + let name = name.as_ref(); + if let Some(var_id) = self.block.vars_local.get(name) { + Some(*var_id) + } else { + for block in self.block_stack.iter().rev() { + if let Some(var_id) = block.vars_local.get(name) { + return Some(*var_id); + } + } + None + } + } + + pub(crate) fn register_match_var(&mut self, num: usize) -> bool { + let mut block = &mut self.block; + + if block.match_test_pos.is_empty() { + for block_ in self.block_stack.iter_mut().rev() { + if !block_.match_test_pos.is_empty() { + block = block_; + break; + } + } + } + + if !block.match_test_pos.is_empty() { + debug_assert!(num < 63); + + for pos in &block.match_test_pos { + if let Some(Instruction::Test(test)) = self.instructions.get_mut(*pos) { + let match_type = match &mut **test { + Test::Address(t) => &mut t.match_type, + Test::Body(t) => &mut t.match_type, + Test::Date(t) => &mut t.match_type, + Test::CurrentDate(t) => &mut t.match_type, + Test::Envelope(t) => &mut t.match_type, + Test::HasFlag(t) => &mut t.match_type, + Test::Header(t) => &mut t.match_type, + Test::Metadata(t) => &mut t.match_type, + Test::NotifyMethodCapability(t) => &mut t.match_type, + Test::SpamTest(t) => &mut t.match_type, + Test::String(t) | Test::Environment(t) => &mut t.match_type, + Test::VirusTest(t) => &mut t.match_type, + _ => { + debug_assert!(false, "This should not have happened: {test:?}"); + return false; + } + }; + if let MatchType::Matches(positions) | MatchType::Regex(positions) = match_type + { + *positions |= 1 << num; + block.match_test_vars = *positions; + } else { + debug_assert!(false, "This should not have happened"); + return false; + } + } else { + debug_assert!(false, "This should not have happened"); + return false; + } + } + true + } else { + false + } + } + + pub(crate) fn block_end(&mut self) { + let vars_num_block = self.block.vars_local.len(); + if vars_num_block > 0 { + if self.vars_num > self.vars_num_max { + self.vars_num_max = self.vars_num; + } + self.vars_num -= vars_num_block; + self.instructions.push(Instruction::Clear(Box::new(Clear { + match_vars: self.block.match_test_vars, + local_vars_idx: self.vars_num as u32, + local_vars_num: vars_num_block as u32, + }))); + } else if self.block.match_test_vars != 0 { + self.instructions.push(Instruction::Clear(Box::new(Clear { + match_vars: self.block.match_test_vars, + local_vars_idx: 0, + local_vars_num: 0, + }))); + } + } + + fn map_local_vars(&mut self, last_id: u16) { + for instruction in &mut self.instructions { + match instruction { + Instruction::Test(v) => v.map_local_vars(last_id), + Instruction::Keep(k) => k.flags.map_local_vars(last_id), + Instruction::FileInto(v) => { + v.folder.map_local_vars(last_id); + v.flags.map_local_vars(last_id); + v.mailbox_id.map_local_vars(last_id); + v.special_use.map_local_vars(last_id); + } + Instruction::Redirect(v) => { + v.address.map_local_vars(last_id); + v.by_time.map_local_vars(last_id); + } + Instruction::Replace(v) => { + v.subject.map_local_vars(last_id); + v.from.map_local_vars(last_id); + v.replacement.map_local_vars(last_id); + } + Instruction::Enclose(v) => { + v.subject.map_local_vars(last_id); + v.headers.map_local_vars(last_id); + v.value.map_local_vars(last_id); + } + Instruction::ExtractText(v) => { + v.name.map_local_vars(last_id); + } + Instruction::Convert(v) => { + v.from_media_type.map_local_vars(last_id); + v.to_media_type.map_local_vars(last_id); + v.transcoding_params.map_local_vars(last_id); + } + Instruction::AddHeader(v) => { + v.field_name.map_local_vars(last_id); + v.value.map_local_vars(last_id); + } + Instruction::DeleteHeader(v) => { + v.field_name.map_local_vars(last_id); + v.value_patterns.map_local_vars(last_id); + } + Instruction::Set(v) => { + v.name.map_local_vars(last_id); + v.value.map_local_vars(last_id); + } + Instruction::Let(v) => { + v.name.map_local_vars(last_id); + v.expr.map_local_vars(last_id); + } + Instruction::While(v) => { + v.expr.map_local_vars(last_id); + } + Instruction::Eval(v) => { + v.map_local_vars(last_id); + } + Instruction::Notify(v) => { + v.from.map_local_vars(last_id); + v.importance.map_local_vars(last_id); + v.options.map_local_vars(last_id); + v.message.map_local_vars(last_id); + v.fcc.map_local_vars(last_id); + v.method.map_local_vars(last_id); + } + Instruction::Reject(v) => { + v.reason.map_local_vars(last_id); + } + Instruction::Vacation(v) => { + v.subject.map_local_vars(last_id); + v.from.map_local_vars(last_id); + v.fcc.map_local_vars(last_id); + v.reason.map_local_vars(last_id); + } + Instruction::Error(v) => { + v.message.map_local_vars(last_id); + } + Instruction::EditFlags(v) => { + v.name.map_local_vars(last_id); + v.flags.map_local_vars(last_id); + } + Instruction::Include(v) => { + v.value.map_local_vars(last_id); + } + _ => {} + } + } + } +} + +pub(crate) fn lowercase(name: &str) -> std::borrow::Cow<'_, str> { + if name.bytes().any(|b| b.is_ascii_uppercase()) { + std::borrow::Cow::Owned(name.to_ascii_lowercase()) + } else { + std::borrow::Cow::Borrowed(name) + } +} + +pub trait MapLocalVars { + fn map_local_vars(&mut self, last_id: u16); +} + +impl MapLocalVars for Test { + fn map_local_vars(&mut self, last_id: u16) { + match self { + Test::Address(v) => { + v.header_list.map_local_vars(last_id); + v.key_list.map_local_vars(last_id); + } + Test::Envelope(v) => { + v.key_list.map_local_vars(last_id); + } + Test::Exists(v) => { + v.header_names.map_local_vars(last_id); + } + Test::Header(v) => { + v.key_list.map_local_vars(last_id); + v.header_list.map_local_vars(last_id); + v.mime_opts.map_local_vars(last_id); + } + Test::Body(v) => { + v.key_list.map_local_vars(last_id); + } + Test::Convert(v) => { + v.from_media_type.map_local_vars(last_id); + v.to_media_type.map_local_vars(last_id); + v.transcoding_params.map_local_vars(last_id); + } + Test::Date(v) => { + v.key_list.map_local_vars(last_id); + v.header_name.map_local_vars(last_id); + } + Test::CurrentDate(v) => { + v.key_list.map_local_vars(last_id); + } + Test::Duplicate(v) => { + v.handle.map_local_vars(last_id); + v.dup_match.map_local_vars(last_id); + } + Test::String(v) => { + v.source.map_local_vars(last_id); + v.key_list.map_local_vars(last_id); + } + Test::Environment(v) => { + v.source.map_local_vars(last_id); + v.key_list.map_local_vars(last_id); + } + Test::NotifyMethodCapability(v) => { + v.key_list.map_local_vars(last_id); + v.notification_capability.map_local_vars(last_id); + v.notification_uri.map_local_vars(last_id); + } + Test::ValidNotifyMethod(v) => { + v.notification_uris.map_local_vars(last_id); + } + Test::ValidExtList(v) => { + v.list_names.map_local_vars(last_id); + } + Test::HasFlag(v) => { + v.variable_list.map_local_vars(last_id); + v.flags.map_local_vars(last_id); + } + Test::MailboxExists(v) => { + v.mailbox_names.map_local_vars(last_id); + } + Test::Metadata(v) => { + v.key_list.map_local_vars(last_id); + v.medatata.map_local_vars(last_id); + } + Test::MetadataExists(v) => { + v.annotation_names.map_local_vars(last_id); + v.mailbox.map_local_vars(last_id); + } + Test::MailboxIdExists(v) => { + v.mailbox_ids.map_local_vars(last_id); + } + Test::SpamTest(v) => { + v.value.map_local_vars(last_id); + } + Test::VirusTest(v) => { + v.value.map_local_vars(last_id); + } + Test::SpecialUseExists(v) => { + v.mailbox.map_local_vars(last_id); + v.attributes.map_local_vars(last_id); + } + Test::Vacation(v) => { + v.addresses.map_local_vars(last_id); + v.handle.map_local_vars(last_id); + v.reason.map_local_vars(last_id); + } + #[cfg(all(test, stalwart_tests))] + Test::TestCmd(cmd) => { + cmd.arguments.map_local_vars(last_id); + } + _ => (), + } + } +} + +impl MapLocalVars for VariableType { + fn map_local_vars(&mut self, last_id: u16) { + match self { + VariableType::Local(id) if *id > last_id => { + *id = (u16::MAX - *id) + last_id; + } + _ => (), + } + } +} + +impl MapLocalVars for Value { + fn map_local_vars(&mut self, last_id: u16) { + match self { + Value::Variable(var) => var.map_local_vars(last_id), + Value::List(items) => items.map_local_vars(last_id), + _ => (), + } + } +} + +impl MapLocalVars for Option { + fn map_local_vars(&mut self, last_id: u16) { + if let Some(value) = self { + value.map_local_vars(last_id); + } + } +} + +impl MapLocalVars for Expression { + fn map_local_vars(&mut self, last_id: u16) { + match self { + Expression::VariableLocal(id) if *id > last_id => { + *id = (u16::MAX - *id) + last_id; + } + Expression::VariableOther(var) => var.map_local_vars(last_id), + _ => (), + } + } +} + +impl MapLocalVars for [T] { + fn map_local_vars(&mut self, last_id: u16) { + for item in self { + item.map_local_vars(last_id); + } + } +} + +impl Block { + pub fn new(btype: Word) -> Self { + Block { + btype, + label: None, + line_num: 0, + line_pos: 0, + last_block_start: 0, + match_test_pos: vec![], + match_test_vars: 0, + if_jmps: vec![], + break_jmps: vec![], + vars_local: AHashMap::new(), + capabilities: AHashSet::new(), + require_pos: usize::MAX, + } + } + + pub fn with_label(mut self, label: String) -> Self { + self.label = label.into(); + self + } +} diff --git a/crates/sieve/src/compiler/grammar/mod.rs b/crates/sieve/src/compiler/grammar/mod.rs new file mode 100644 index 00000000..e6e644e4 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/mod.rs @@ -0,0 +1,921 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use mail_parser::HeaderName; +use std::fmt::Display; + +use self::{expr::Expression, instruction::CompilerState}; + +use super::{ + CompileError, ErrorType, Glob, RawValue, Regex, Value, + lexer::{Token, tokenizer::TokenInfo, word::Word}, +}; + +pub mod actions; +pub mod expr; +pub mod instruction; +pub mod test; +pub mod tests; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Capability { + Envelope = 0, + EnvelopeDsn = 1, + EnvelopeDeliverBy = 2, + FileInto = 3, + EncodedCharacter = 4, + Comparator(Comparator) = 5, + Other(String) = 6, + Body = 7, + Convert = 8, + Copy = 9, + Relational = 10, + Date = 11, + Index = 12, + Duplicate = 13, + Variables = 14, + EditHeader = 15, + ForEveryPart = 16, + Mime = 17, + Replace = 18, + Enclose = 19, + ExtractText = 20, + Enotify = 21, + RedirectDsn = 22, + RedirectDeliverBy = 23, + Environment = 24, + Reject = 25, + Ereject = 26, + ExtLists = 27, + SubAddress = 28, + Vacation = 29, + VacationSeconds = 30, + Fcc = 31, + Mailbox = 32, + MailboxId = 33, + MboxMetadata = 34, + ServerMetadata = 35, + SpecialUse = 36, + Imap4Flags = 37, + Ihave = 38, + ImapSieve = 39, + Include = 40, + Regex = 41, + SpamTest = 42, + SpamTestPlus = 43, + VirusTest = 44, + + // Extensions + Expressions = 45, + While = 46, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum AddressPart { + LocalPart = 0, + Domain = 1, + All = 2, + User = 3, + Detail = 4, + Name = 5, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum MatchType { + Is = 0, + Contains = 1, + Matches(u64) = 2, + Regex(u64) = 3, + Value(RelationalMatch) = 4, + Count(RelationalMatch) = 5, + List = 6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum RelationalMatch { + Gt = 0, + Ge = 1, + Lt = 2, + Le = 3, + Eq = 4, + Ne = 5, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Comparator { + Elbonia = 0, + Octet = 1, + AsciiCaseMap = 2, + AsciiNumeric = 3, + Other(String) = 4, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Clear { + pub(crate) local_vars_idx: u32, + pub(crate) local_vars_num: u32, + pub(crate) match_vars: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Invalid { + pub(crate) name: String, + pub(crate) line_num: u32, + pub(crate) line_pos: u32, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct While { + pub expr: Box<[Expression]>, + pub jz_pos: u32, +} + +impl CompilerState<'_> { + #[inline(always)] + pub fn expect_instruction_end(&mut self) -> Result<(), CompileError> { + self.tokens.expect_token(Token::Semicolon) + } + + pub fn ignore_instruction(&mut self) -> Result<(), CompileError> { + // Skip entire instruction + let mut curly_count = 0; + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Semicolon if curly_count == 0 => { + break; + } + Token::CurlyOpen => { + curly_count += 1; + } + Token::CurlyClose => match curly_count { + 0 => { + return Err(token_info.expected("instruction")); + } + 1 => { + break; + } + _ => curly_count -= 1, + }, + _ => (), + } + } + + Ok(()) + } + + pub fn ignore_test(&mut self) -> Result<(), CompileError> { + let mut d_count = 0; + while let Some(token_info) = self.tokens.peek() { + match token_info?.token { + Token::ParenthesisOpen => { + d_count += 1; + } + Token::ParenthesisClose => { + if d_count == 0 { + break; + } else { + d_count -= 1; + } + } + Token::Comma if d_count == 0 => { + break; + } + Token::CurlyOpen => { + break; + } + _ => (), + } + self.tokens.next(); + } + + Ok(()) + } + + pub fn parse_match_type(&mut self, word: Word) -> Result { + match word { + Word::Is => Ok(MatchType::Is), + Word::Contains => Ok(MatchType::Contains), + Word::Matches => { + self.block.match_test_pos.push(self.instructions.len()); + Ok(MatchType::Matches(0)) + } + Word::Regex => { + self.block.match_test_pos.push(self.instructions.len()); + Ok(MatchType::Regex(0)) + } + Word::List => Ok(MatchType::List), + _ => { + let token_info = self.tokens.unwrap_next()?; + if let Token::StringConstant(text) = &token_info.token + && let Some(relational) = lookup_relational(text.to_string().as_ref()) + { + return Ok(if word == Word::Value { + MatchType::Value(relational) + } else { + MatchType::Count(relational) + }); + } + Err(token_info.expected("relational match")) + } + } + } + + pub(crate) fn parse_comparator(&mut self) -> Result { + let comparator = self.tokens.expect_static_string()?; + Ok(if let Some(comparator) = lookup_comparator(&comparator) { + comparator + } else { + Comparator::Other(comparator) + }) + } + + pub(crate) fn parse_static_strings(&mut self) -> Result, CompileError> { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::BracketOpen => { + let mut strings = Vec::new(); + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::StringConstant(string) => { + strings.push(string.into_string()); + } + Token::Comma => (), + Token::BracketClose if !strings.is_empty() => break, + _ => return Err(token_info.expected("constant string")), + } + } + Ok(strings) + } + Token::StringConstant(string) => Ok(vec![string.into_string()]), + _ => Err(token_info.expected("'[' or constant string")), + } + } + + pub fn parse_string(&mut self) -> Result { + let value = self.parse_raw_string()?; + Ok(self.intern_raw(value)) + } + + pub(crate) fn parse_strings(&mut self, allow_empty: bool) -> Result, CompileError> { + let values = self.parse_raw_strings(allow_empty)?; + Ok(self.intern_raw_list(values)) + } + + pub(crate) fn parse_string_token( + &mut self, + token_info: TokenInfo, + ) -> Result { + let value = self.parse_raw_string_token(token_info)?; + Ok(self.intern_raw(value)) + } + + pub(crate) fn parse_strings_token( + &mut self, + token_info: TokenInfo, + ) -> Result, CompileError> { + let values = self.parse_raw_strings_token(token_info)?; + Ok(self.intern_raw_list(values)) + } + + pub(crate) fn parse_raw_string(&mut self) -> Result { + let next_token = self.tokens.unwrap_next()?; + match next_token.token { + Token::StringConstant(s) => Ok(s.into()), + Token::StringVariable(s) => self + .tokenize_string(&s, true) + .map(RawValue::Value) + .map_err(|error_type| CompileError { + line_num: next_token.line_num, + line_pos: next_token.line_pos, + error_type, + }), + Token::BracketOpen => { + let mut items = self.parse_raw_string_list(false)?; + match items.pop() { + Some(s) if items.is_empty() => Ok(s), + _ => Err(next_token.expected("string")), + } + } + _ => Err(next_token.expected("string")), + } + } + + pub(crate) fn parse_raw_strings( + &mut self, + allow_empty: bool, + ) -> Result, CompileError> { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::BracketOpen => self.parse_raw_string_list(allow_empty), + Token::StringConstant(s) => Ok(vec![s.into()]), + Token::StringVariable(s) => self + .tokenize_string(&s, true) + .map(|s| vec![RawValue::Value(s)]) + .map_err(|error_type| CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type, + }), + _ => Err(token_info.expected("'[' or string")), + } + } + + pub(crate) fn parse_raw_string_token( + &mut self, + token_info: TokenInfo, + ) -> Result { + match token_info.token { + Token::StringConstant(s) => Ok(s.into()), + Token::StringVariable(s) => self + .tokenize_string(&s, true) + .map(RawValue::Value) + .map_err(|error_type| CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type, + }), + _ => Err(token_info.expected("string")), + } + } + + pub(crate) fn parse_raw_strings_token( + &mut self, + token_info: TokenInfo, + ) -> Result, CompileError> { + match token_info.token { + Token::StringConstant(s) => Ok(vec![s.into()]), + Token::StringVariable(s) => self + .tokenize_string(&s, true) + .map(|s| vec![RawValue::Value(s)]) + .map_err(|error_type| CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type, + }), + Token::BracketOpen => self.parse_raw_string_list(false), + _ => Err(token_info.expected("string")), + } + } + + pub(crate) fn parse_raw_string_list( + &mut self, + allow_empty: bool, + ) -> Result, CompileError> { + let mut strings = Vec::new(); + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::StringConstant(s) => { + strings.push(s.into()); + } + Token::StringVariable(s) => { + strings.push(RawValue::Value(self.tokenize_string(&s, true).map_err( + |error_type| CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type, + }, + )?)); + } + Token::Comma => (), + Token::BracketClose if !strings.is_empty() || allow_empty => break, + _ => return Err(token_info.expected("string or string list")), + } + } + Ok(strings) + } + + #[inline(always)] + pub(crate) fn has_capability(&self, capability: &Capability) -> bool { + [&self.block] + .into_iter() + .chain(self.block_stack.iter()) + .any(|b| b.capabilities.contains(capability)) + || (capability != &Capability::Ihave && self.compiler.no_capability_check) + } + + #[inline(always)] + pub(crate) fn reset_param_check(&mut self) { + self.param_check.fill(false); + } + + #[inline(always)] + pub(crate) fn validate_argument( + &mut self, + arg_num: usize, + capability: Option, + line_num: usize, + line_pos: usize, + ) -> Result<(), CompileError> { + if arg_num > 0 { + if let Some(param) = self.param_check.get_mut(arg_num - 1) { + if !*param { + *param = true; + } else { + return Err(CompileError { + line_num, + line_pos, + error_type: ErrorType::DuplicatedParameter, + }); + } + } else { + #[cfg(all(test, stalwart_tests))] + panic!("Argument out of range {arg_num}"); + } + } + if let Some(capability) = capability + && !self.has_capability(&capability) + { + return Err(CompileError { + line_num, + line_pos, + error_type: ErrorType::UndeclaredCapability(capability), + }); + } + + Ok(()) + } + + fn header_name(&mut self, header: RawValue) -> Result { + match header { + RawValue::Text(text) => { + match HeaderName::parse(text.as_str()).map(HeaderName::into_owned) { + Some(header_name) => Ok(Value::Header(header_name)), + None => Err(self.text(text)), + } + } + RawValue::Value(Value::Text(id)) => { + match HeaderName::parse(self.constant(id)).map(HeaderName::into_owned) { + Some(header_name) => Ok(Value::Header(header_name)), + None => Err(Value::Text(id)), + } + } + other => Ok(self.intern_raw(other)), + } + } + + pub(crate) fn resolve_header_name(&mut self, header: RawValue) -> Option { + self.header_name(header).ok() + } + + pub(crate) fn resolve_header_names(&mut self, headers: Vec) -> (Vec, bool) { + let mut all_valid = true; + let mut values = Vec::with_capacity(headers.len()); + + for header in headers { + match self.header_name(header) { + Ok(value) => values.push(value), + Err(value) => { + all_valid = false; + values.push(value); + } + } + } + + (values, all_valid) + } + + pub(crate) fn validate_match( + &mut self, + match_type: &MatchType, + comparator: &Comparator, + key_list: Vec, + ) -> Result, CompileError> { + match match_type { + MatchType::Regex(_) => { + let mut keys = Vec::with_capacity(key_list.len()); + for key in key_list { + let expr = match self.match_expression(key) { + Ok(expr) => expr, + Err(key) => { + keys.push(key); + continue; + } + }; + match crate::regex::build(&expr) { + Ok(_) => keys.push(Value::Regex(Regex::new(expr))), + Err(err) => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidRegex(format!("{expr}: {err}")))); + } + } + } + Ok(keys) + } + MatchType::Matches(_) => { + let to_lower = matches!(comparator, Comparator::AsciiCaseMap); + let mut keys = Vec::with_capacity(key_list.len()); + for key in key_list { + match self.match_expression(key) { + Ok(expr) => keys.push(Value::Glob(Glob::new(expr, to_lower))), + Err(key) => keys.push(key), + } + } + Ok(keys) + } + _ => Ok(self.intern_raw_list(key_list)), + } + } + + fn match_expression(&mut self, key: RawValue) -> Result { + match key { + RawValue::Text(text) => Ok(text), + RawValue::Value(Value::Text(id)) => Ok(self.constant(id).to_string()), + other => Err(self.intern_raw(other)), + } + } +} + +impl AddressPart { + #[inline(always)] + pub(crate) fn from_code(code: u8) -> AddressPart { + match code { + 0 => AddressPart::LocalPart, + 1 => AddressPart::Domain, + 2 => AddressPart::All, + 3 => AddressPart::User, + 4 => AddressPart::Detail, + _ => AddressPart::Name, + } + } +} + +impl Capability { + pub(crate) fn id(&self) -> u8 { + match self { + Capability::Envelope => 0, + Capability::EnvelopeDsn => 1, + Capability::EnvelopeDeliverBy => 2, + Capability::FileInto => 3, + Capability::EncodedCharacter => 4, + Capability::Body => 7, + Capability::Convert => 8, + Capability::Copy => 9, + Capability::Relational => 10, + Capability::Date => 11, + Capability::Index => 12, + Capability::Duplicate => 13, + Capability::Variables => 14, + Capability::EditHeader => 15, + Capability::ForEveryPart => 16, + Capability::Mime => 17, + Capability::Replace => 18, + Capability::Enclose => 19, + Capability::ExtractText => 20, + Capability::Enotify => 21, + Capability::RedirectDsn => 22, + Capability::RedirectDeliverBy => 23, + Capability::Environment => 24, + Capability::Reject => 25, + Capability::Ereject => 26, + Capability::ExtLists => 27, + Capability::SubAddress => 28, + Capability::Vacation => 29, + Capability::VacationSeconds => 30, + Capability::Fcc => 31, + Capability::Mailbox => 32, + Capability::MailboxId => 33, + Capability::MboxMetadata => 34, + Capability::ServerMetadata => 35, + Capability::SpecialUse => 36, + Capability::Imap4Flags => 37, + Capability::Ihave => 38, + Capability::ImapSieve => 39, + Capability::Include => 40, + Capability::Regex => 41, + Capability::SpamTest => 42, + Capability::SpamTestPlus => 43, + Capability::VirusTest => 44, + Capability::Expressions => 45, + Capability::While => 46, + Capability::Comparator(_) => 5, + Capability::Other(_) => 6, + } + } + + pub(crate) fn from_id(id: u8) -> Capability { + match id { + 0 => Capability::Envelope, + 1 => Capability::EnvelopeDsn, + 2 => Capability::EnvelopeDeliverBy, + 3 => Capability::FileInto, + 4 => Capability::EncodedCharacter, + 7 => Capability::Body, + 8 => Capability::Convert, + 9 => Capability::Copy, + 10 => Capability::Relational, + 11 => Capability::Date, + 12 => Capability::Index, + 13 => Capability::Duplicate, + 14 => Capability::Variables, + 15 => Capability::EditHeader, + 16 => Capability::ForEveryPart, + 17 => Capability::Mime, + 18 => Capability::Replace, + 19 => Capability::Enclose, + 20 => Capability::ExtractText, + 21 => Capability::Enotify, + 22 => Capability::RedirectDsn, + 23 => Capability::RedirectDeliverBy, + 24 => Capability::Environment, + 25 => Capability::Reject, + 26 => Capability::Ereject, + 27 => Capability::ExtLists, + 28 => Capability::SubAddress, + 29 => Capability::Vacation, + 30 => Capability::VacationSeconds, + 31 => Capability::Fcc, + 32 => Capability::Mailbox, + 33 => Capability::MailboxId, + 34 => Capability::MboxMetadata, + 35 => Capability::ServerMetadata, + 36 => Capability::SpecialUse, + 37 => Capability::Imap4Flags, + 38 => Capability::Ihave, + 39 => Capability::ImapSieve, + 40 => Capability::Include, + 41 => Capability::Regex, + 42 => Capability::SpamTest, + 43 => Capability::SpamTestPlus, + 44 => Capability::VirusTest, + 45 => Capability::Expressions, + 46 => Capability::While, + _ => Capability::Other(String::new()), + } + } + + pub fn parse(capability: &str) -> Capability { + if let Some(capability) = lookup_capabilities(capability) { + capability + } else if let Some(comparator) = capability.strip_prefix("comparator-") { + Capability::Comparator(Comparator::Other(comparator.to_string())) + } else { + Capability::Other(capability.to_string()) + } + } + + pub fn all() -> &'static [Capability] { + &[ + Capability::Envelope, + Capability::EnvelopeDsn, + Capability::EnvelopeDeliverBy, + Capability::FileInto, + Capability::EncodedCharacter, + Capability::Comparator(Comparator::Elbonia), + Capability::Comparator(Comparator::AsciiCaseMap), + Capability::Comparator(Comparator::AsciiNumeric), + Capability::Comparator(Comparator::Octet), + Capability::Body, + Capability::Convert, + Capability::Copy, + Capability::Relational, + Capability::Date, + Capability::Index, + Capability::Duplicate, + Capability::Variables, + Capability::EditHeader, + Capability::ForEveryPart, + Capability::Mime, + Capability::Replace, + Capability::Enclose, + Capability::ExtractText, + Capability::Enotify, + Capability::RedirectDsn, + Capability::RedirectDeliverBy, + Capability::Environment, + Capability::Reject, + Capability::Ereject, + Capability::ExtLists, + Capability::SubAddress, + Capability::Vacation, + Capability::VacationSeconds, + Capability::Fcc, + Capability::Mailbox, + Capability::MailboxId, + Capability::MboxMetadata, + Capability::ServerMetadata, + Capability::SpecialUse, + Capability::Imap4Flags, + Capability::Ihave, + Capability::ImapSieve, + Capability::Include, + Capability::Regex, + Capability::SpamTest, + Capability::SpamTestPlus, + Capability::VirusTest, + ] + } +} + +fn lookup_relational(input: &str) -> Option { + hashify::map!( + input.as_bytes(), RelationalMatch, + "gt" => RelationalMatch::Gt, + "ge" => RelationalMatch::Ge, + "lt" => RelationalMatch::Lt, + "le" => RelationalMatch::Le, + "eq" => RelationalMatch::Eq, + "ne" => RelationalMatch::Ne, + ) + .copied() +} + +fn lookup_comparator(input: &str) -> Option { + hashify::fnc_map!( + input.as_bytes(), + "i;octet" => Some(Comparator::Octet), + "i;ascii-casemap" => Some(Comparator::AsciiCaseMap), + "i;ascii-numeric" => Some(Comparator::AsciiNumeric), + _ => None, + ) +} + +impl Invalid { + pub fn name(&self) -> &str { + &self.name + } + + pub fn line_num(&self) -> usize { + self.line_num as usize + } + + pub fn line_pos(&self) -> usize { + self.line_pos as usize + } +} + +impl From<&str> for Capability { + fn from(value: &str) -> Self { + Capability::parse(value) + } +} + +impl From for Capability { + fn from(value: String) -> Self { + Capability::parse(&value) + } +} + +impl Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Capability::Envelope => f.write_str("envelope"), + Capability::EnvelopeDsn => f.write_str("envelope-dsn"), + Capability::EnvelopeDeliverBy => f.write_str("envelope-deliverby"), + Capability::FileInto => f.write_str("fileinto"), + Capability::EncodedCharacter => f.write_str("encoded-character"), + Capability::Comparator(Comparator::Elbonia) => f.write_str("comparator-elbonia"), + Capability::Comparator(Comparator::Octet) => f.write_str("comparator-i;octet"), + Capability::Comparator(Comparator::AsciiCaseMap) => { + f.write_str("comparator-i;ascii-casemap") + } + Capability::Comparator(Comparator::AsciiNumeric) => { + f.write_str("comparator-i;ascii-numeric") + } + Capability::Comparator(Comparator::Other(comparator)) => f.write_str(comparator), + Capability::Body => f.write_str("body"), + Capability::Convert => f.write_str("convert"), + Capability::Copy => f.write_str("copy"), + Capability::Relational => f.write_str("relational"), + Capability::Date => f.write_str("date"), + Capability::Index => f.write_str("index"), + Capability::Duplicate => f.write_str("duplicate"), + Capability::Variables => f.write_str("variables"), + Capability::EditHeader => f.write_str("editheader"), + Capability::ForEveryPart => f.write_str("foreverypart"), + Capability::Mime => f.write_str("mime"), + Capability::Replace => f.write_str("replace"), + Capability::Enclose => f.write_str("enclose"), + Capability::ExtractText => f.write_str("extracttext"), + Capability::Enotify => f.write_str("enotify"), + Capability::RedirectDsn => f.write_str("redirect-dsn"), + Capability::RedirectDeliverBy => f.write_str("redirect-deliverby"), + Capability::Environment => f.write_str("environment"), + Capability::Reject => f.write_str("reject"), + Capability::Ereject => f.write_str("ereject"), + Capability::ExtLists => f.write_str("extlists"), + Capability::SubAddress => f.write_str("subaddress"), + Capability::Vacation => f.write_str("vacation"), + Capability::VacationSeconds => f.write_str("vacation-seconds"), + Capability::Fcc => f.write_str("fcc"), + Capability::Mailbox => f.write_str("mailbox"), + Capability::MailboxId => f.write_str("mailboxid"), + Capability::MboxMetadata => f.write_str("mboxmetadata"), + Capability::ServerMetadata => f.write_str("servermetadata"), + Capability::SpecialUse => f.write_str("special-use"), + Capability::Imap4Flags => f.write_str("imap4flags"), + Capability::Ihave => f.write_str("ihave"), + Capability::ImapSieve => f.write_str("imapsieve"), + Capability::Include => f.write_str("include"), + Capability::Regex => f.write_str("regex"), + Capability::SpamTest => f.write_str("spamtest"), + Capability::SpamTestPlus => f.write_str("spamtestplus"), + Capability::VirusTest => f.write_str("virustest"), + Capability::While => f.write_str("vnd.inbuxa.while"), + Capability::Expressions => f.write_str("vnd.inbuxa.expressions"), + Capability::Other(capability) => f.write_str(capability), + } + } +} + +fn lookup_capabilities(input: &str) -> Option { + hashify::fnc_map!( + input.as_bytes(), + "envelope" => Some(Capability::Envelope), + "envelope-dsn" => Some(Capability::EnvelopeDsn), + "envelope-deliverby" => Some(Capability::EnvelopeDeliverBy), + "fileinto" => Some(Capability::FileInto), + "encoded-character" => Some(Capability::EncodedCharacter), + "comparator-elbonia" => Some(Capability::Comparator(Comparator::Elbonia)), + "comparator-i;octet" => Some(Capability::Comparator(Comparator::Octet)), + "comparator-i;ascii-casemap" => Some(Capability::Comparator(Comparator::AsciiCaseMap)), + "comparator-i;ascii-numeric" => Some(Capability::Comparator(Comparator::AsciiNumeric)), + "body" => Some(Capability::Body), + "convert" => Some(Capability::Convert), + "copy" => Some(Capability::Copy), + "relational" => Some(Capability::Relational), + "date" => Some(Capability::Date), + "index" => Some(Capability::Index), + "duplicate" => Some(Capability::Duplicate), + "variables" => Some(Capability::Variables), + "editheader" => Some(Capability::EditHeader), + "foreverypart" => Some(Capability::ForEveryPart), + "mime" => Some(Capability::Mime), + "replace" => Some(Capability::Replace), + "enclose" => Some(Capability::Enclose), + "extracttext" => Some(Capability::ExtractText), + "enotify" => Some(Capability::Enotify), + "redirect-dsn" => Some(Capability::RedirectDsn), + "redirect-deliverby" => Some(Capability::RedirectDeliverBy), + "environment" => Some(Capability::Environment), + "reject" => Some(Capability::Reject), + "ereject" => Some(Capability::Ereject), + "extlists" => Some(Capability::ExtLists), + "subaddress" => Some(Capability::SubAddress), + "vacation" => Some(Capability::Vacation), + "vacation-seconds" => Some(Capability::VacationSeconds), + "fcc" => Some(Capability::Fcc), + "mailbox" => Some(Capability::Mailbox), + "mailboxid" => Some(Capability::MailboxId), + "mboxmetadata" => Some(Capability::MboxMetadata), + "servermetadata" => Some(Capability::ServerMetadata), + "special-use" => Some(Capability::SpecialUse), + "imap4flags" => Some(Capability::Imap4Flags), + "ihave" => Some(Capability::Ihave), + "imapsieve" => Some(Capability::ImapSieve), + "include" => Some(Capability::Include), + "regex" => Some(Capability::Regex), + "spamtest" => Some(Capability::SpamTest), + "spamtestplus" => Some(Capability::SpamTestPlus), + "virustest" => Some(Capability::VirusTest), + + // Extensions + "vnd.inbuxa.while" => Some(Capability::While), + "vnd.inbuxa.expressions" => Some(Capability::Expressions), + _ => None, + ) +} diff --git a/crates/sieve/src/compiler/grammar/test.rs b/crates/sieve/src/compiler/grammar/test.rs new file mode 100644 index 00000000..822b2f6b --- /dev/null +++ b/crates/sieve/src/compiler/grammar/test.rs @@ -0,0 +1,706 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Capability, Invalid, + actions::{action_convert::Convert, action_vacation::TestVacation}, + expr::{Expression, UnaryOperator, parser::ExpressionParser, tokenizer::Tokenizer}, + instruction::{CompilerState, Instruction}, + tests::{ + test_address::TestAddress, + test_body::TestBody, + test_date::{TestCurrentDate, TestDate}, + test_duplicate::TestDuplicate, + test_envelope::TestEnvelope, + test_exists::TestExists, + test_extlists::TestValidExtList, + test_hasflag::TestHasFlag, + test_header::TestHeader, + test_ihave::TestIhave, + test_mailbox::{TestMailboxExists, TestMetadata, TestMetadataExists}, + test_mailboxid::TestMailboxIdExists, + test_notify::{TestNotifyMethodCapability, TestValidNotifyMethod}, + test_size::TestSize, + test_spamtest::{TestSpamTest, TestVirusTest}, + test_specialuse::TestSpecialUseExists, + test_string::TestString, + }, +}; +use crate::compiler::{ + CompileError, ErrorType, + lexer::{Token, tokenizer::TokenInfo, word::Word}, +}; + +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Test { + True = 0, + False = 1, + Address(Box) = 2, + Envelope(Box) = 3, + Exists(Box) = 4, + Header(Box) = 5, + Size(Box) = 6, + Invalid(Box) = 7, + + // RFC 5173 + Body(Box) = 8, + + // RFC 6558 + Convert(Box) = 9, + + // RFC 5260 + Date(Box) = 10, + CurrentDate(Box) = 11, + + // RFC 7352 + Duplicate(Box) = 12, + + // RFC 5229 & RFC 5183 + String(Box) = 13, + Environment(Box) = 14, + + // RFC 5435 + NotifyMethodCapability(Box) = 15, + ValidNotifyMethod(Box) = 16, + + // RFC 6134 + ValidExtList(Box) = 17, + + // RFC 5463 + Ihave(Box) = 18, + + // RFC 5232 + HasFlag(Box) = 19, + + // RFC 5490 + MailboxExists(Box) = 20, + Metadata(Box) = 21, + MetadataExists(Box) = 22, + + // RFC 9042 + MailboxIdExists(Box) = 23, + + // RFC 5235 + SpamTest(Box) = 24, + VirusTest(Box) = 25, + + // RFC 8579 + SpecialUseExists(Box) = 26, + + // RFC 5230 + Vacation(Box) = 27, + + // Only test + #[cfg(all(test, stalwart_tests))] + TestCmd(Box) = 0x7f, +} + +#[cfg(all(test, stalwart_tests))] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestCommand { + pub(crate) arguments: Box<[crate::compiler::Value]>, + pub(crate) is_not: bool, +} + +#[derive(Debug)] +struct Block { + is_all: bool, + is_not: bool, + p_count: u32, + jmps: Vec, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test(&mut self) -> Result<(), CompileError> { + let mut block_stack: Vec = Vec::new(); + let mut block = Block { + is_all: false, + is_not: false, + p_count: 0, + jmps: Vec::new(), + }; + let mut is_not = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + self.reset_param_check(); + let test: Instruction = + match token_info.token { + Token::Comma + if !block_stack.is_empty() + && matches!( + self.instructions.last(), + Some(Instruction::Test(_) | Instruction::Eval(_)) + ) + && matches!( + self.tokens.peek(), + Some(Ok(TokenInfo { + token: Token::Identifier(_) | Token::Unknown(_), + .. + })) + ) => + { + is_not = block.is_not; + block.jmps.push(self.instructions.len()); + self.instructions.push(if block.is_all { + Instruction::Jz(u32::MAX) + } else { + Instruction::Jnz(u32::MAX) + }); + continue; + } + Token::ParenthesisOpen => { + block.p_count += 1; + continue; + } + Token::ParenthesisClose => { + if block.p_count > 0 { + block.p_count -= 1; + continue; + } else if let Some(prev_block) = block_stack.pop() { + let cur_pos = self.instructions.len(); + for jmp_pos in block.jmps { + if let Instruction::Jnz(jmp_pos) | Instruction::Jz(jmp_pos) = + &mut self.instructions[jmp_pos] + { + *jmp_pos = cur_pos as u32; + } else { + debug_assert!(false, "This should not have happened") + } + } + + block = prev_block; + is_not = block.is_not; + if block_stack.is_empty() { + break; + } else { + continue; + } + } else { + return Err(token_info.expected("test name")); + } + } + Token::Identifier(Word::Not) => { + if !matches!( + self.tokens.peek(), + Some(Ok(TokenInfo { + token: Token::Identifier(_) | Token::Unknown(_), + .. + })) + ) { + return Err(token_info.expected("test name")); + } + is_not = !is_not; + continue; + } + Token::Identifier(word @ (Word::AnyOf | Word::AllOf)) => { + if block_stack.len() < self.tokens.compiler.max_nested_tests { + self.tokens.expect_token(Token::ParenthesisOpen)?; + block_stack.push(block); + let (is_all, block_is_not) = if word == Word::AllOf { + if !is_not { + (true, false) + } else { + (false, true) + } + } else if !is_not { + (false, false) + } else { + (true, true) + }; + block = Block { + is_all, + is_not: block_is_not, + p_count: 0, + jmps: Vec::new(), + }; + is_not = block_is_not; + continue; + } else { + return Err(CompileError { + line_num: token_info.line_num, + line_pos: token_info.line_pos, + error_type: ErrorType::TooManyNestedTests, + }); + } + } + Token::Identifier(Word::True) => if !is_not { + Test::True + } else { + is_not = false; + Test::False + } + .into(), + Token::Identifier(Word::False) => if !is_not { + Test::False + } else { + is_not = false; + Test::True + } + .into(), + Token::Identifier(Word::Address) => self.parse_test_address()?.into(), + Token::Identifier(Word::Envelope) => { + self.validate_argument( + 0, + Capability::Envelope.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_envelope()?.into() + } + Token::Identifier(Word::Header) => self.parse_test_header()?.into(), + Token::Identifier(Word::Size) => self.parse_test_size()?.into(), + Token::Identifier(Word::Exists) => self.parse_test_exists()?.into(), + + // RFC 5173 + Token::Identifier(Word::Body) => { + self.validate_argument( + 0, + Capability::Body.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_body()?.into() + } + + // RFC 6558 + Token::Identifier(Word::Convert) => { + self.validate_argument( + 0, + Capability::Convert.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_convert()?.into() + } + + // RFC 5260 + Token::Identifier(Word::Date) => { + self.validate_argument( + 0, + Capability::Date.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_date()?.into() + } + Token::Identifier(Word::CurrentDate) => { + self.validate_argument( + 0, + Capability::Date.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_currentdate()?.into() + } + + // RFC 7352 + Token::Identifier(Word::Duplicate) => { + self.validate_argument( + 0, + Capability::Duplicate.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_duplicate()?.into() + } + + // RFC 5229 + Token::Identifier(Word::String) => { + self.validate_argument( + 0, + Capability::Variables.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_string()?.into() + } + + // RFC 5435 + Token::Identifier(Word::NotifyMethodCapability) => { + self.validate_argument( + 0, + Capability::Enotify.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_notify_method_capability()?.into() + } + Token::Identifier(Word::ValidNotifyMethod) => { + self.validate_argument( + 0, + Capability::Enotify.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_valid_notify_method()?.into() + } + + // RFC 5183 + Token::Identifier(Word::Environment) => { + self.validate_argument( + 0, + Capability::Environment.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_environment()?.into() + } + + // RFC 6134 + Token::Identifier(Word::ValidExtList) => { + self.validate_argument( + 0, + Capability::ExtLists.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_valid_ext_list()?.into() + } + + // RFC 5463 + Token::Identifier(Word::Ihave) => { + self.validate_argument( + 0, + Capability::Ihave.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_ihave()?.into() + } + + // RFC 5232 + Token::Identifier(Word::HasFlag) => { + self.validate_argument( + 0, + Capability::Imap4Flags.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_hasflag()?.into() + } + + // RFC 5490 + Token::Identifier(Word::MailboxExists) => { + self.validate_argument( + 0, + Capability::Mailbox.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_mailboxexists()?.into() + } + Token::Identifier(Word::Metadata) => { + self.validate_argument( + 0, + Capability::MboxMetadata.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_metadata()?.into() + } + Token::Identifier(Word::MetadataExists) => { + self.validate_argument( + 0, + Capability::MboxMetadata.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_metadataexists()?.into() + } + Token::Identifier(Word::ServerMetadata) => { + self.validate_argument( + 0, + Capability::ServerMetadata.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_servermetadata()?.into() + } + Token::Identifier(Word::ServerMetadataExists) => { + self.validate_argument( + 0, + Capability::ServerMetadata.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_servermetadataexists()?.into() + } + + // RFC 9042 + Token::Identifier(Word::MailboxIdExists) => { + self.validate_argument( + 0, + Capability::MailboxId.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_mailboxidexists()?.into() + } + + // RFC 5235 + Token::Identifier(Word::SpamTest) => { + self.validate_argument( + 0, + Capability::SpamTest.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_spamtest()?.into() + } + Token::Identifier(Word::VirusTest) => { + self.validate_argument( + 0, + Capability::VirusTest.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_virustest()?.into() + } + + // RFC 8579 + Token::Identifier(Word::SpecialUseExists) => { + self.validate_argument( + 0, + Capability::SpecialUse.into(), + token_info.line_num, + token_info.line_pos, + )?; + self.parse_test_specialuseexists()?.into() + } + + // Expressions extension + Token::Identifier(Word::Eval) => { + self.validate_argument( + 0, + Capability::Expressions.into(), + token_info.line_num, + token_info.line_pos, + )?; + + Instruction::Eval(self.parse_expr()?.into_boxed_slice()) + } + Token::Identifier(word) => { + self.ignore_test()?; + Test::Invalid(Box::new(Invalid { + name: word.to_string(), + line_num: token_info.line_num as u32, + line_pos: (token_info.line_pos) as u32, + })) + .into() + } + #[cfg(all(test, stalwart_tests))] + Token::Unknown(name) if name.contains("test") => { + use crate::compiler::Value; + + let mut arguments = Vec::new(); + arguments.push(self.text(&name)); + while !matches!( + self.tokens.peek().map(|r| r.map(|t| &t.token)), + Some(Ok(Token::Comma + | Token::ParenthesisClose + | Token::CurlyOpen)) + ) { + arguments.push(match self.tokens.unwrap_next()?.token { + Token::StringConstant(s) => self.intern_raw(s.into()), + Token::StringVariable(s) => self + .tokenize_string(&s, true) + .map_err(|error_type| CompileError { + line_num: 0, + line_pos: 0, + error_type, + })?, + Token::Number(n) => { + Value::Number(crate::compiler::Number::Integer(n as i64)) + } + Token::Identifier(s) => self.text(s.to_string()), + Token::Tag(s) => self.text(format!(":{s}")), + Token::Unknown(s) => self.text(s), + other => panic!("Invalid test param {other:?}"), + }); + } + Test::TestCmd(Box::new(TestCommand { + arguments: arguments.into(), + is_not: false, + })) + .into() + } + Token::Unknown(name) => { + self.ignore_test()?; + Test::Invalid(Box::new(Invalid { + name, + line_num: token_info.line_num as u32, + line_pos: (token_info.line_pos) as u32, + })) + .into() + } + _ => return Err(token_info.expected("test name")), + }; + + while block.p_count > 0 { + self.tokens.expect_token(Token::ParenthesisClose)?; + block.p_count -= 1; + } + + self.instructions + .push(if !is_not { test } else { test.set_not() }); + + if block_stack.is_empty() { + break; + } + } + + self.instructions.push(Instruction::Jz(u32::MAX)); + Ok(()) + } + + pub(crate) fn parse_expr(&mut self) -> Result, CompileError> { + let mut next_token = self.tokens.unwrap_next()?; + let expr = match next_token.token { + Token::StringConstant(s) => s.into_string().into_bytes(), + Token::StringVariable(s) => s, + _ => return Err(next_token.expected("string")), + }; + + match ExpressionParser::from_tokenizer(Tokenizer::from_iter( + expr.iter().enumerate().peekable(), + |var_name, maybe_namespace| self.parse_expr_fnc_or_var(var_name, maybe_namespace), + )) + .parse() + { + Ok(parser) => { + let mut output = parser.output; + let string_constants = parser.string_constants; + for (pos, text) in string_constants { + let id = self.intern(text); + if let Some(Expression::ConstantString(slot)) = output.get_mut(pos) { + *slot = id; + } else { + debug_assert!(false, "Expression constant slot {pos} not found."); + } + } + Ok(output) + } + Err(err) => { + let err = ErrorType::InvalidExpression(format!( + "{}: {}", + std::str::from_utf8(&expr).unwrap_or_default(), + err + )); + next_token.token = Token::StringVariable(expr); + Err(next_token.custom(err)) + } + } + } +} + +impl From for Instruction { + fn from(test: Test) -> Self { + Instruction::Test(Box::new(test)) + } +} + +impl Instruction { + pub fn set_not(mut self) -> Self { + match &mut self { + Instruction::Test(test) => match &mut **test { + Test::True => return Instruction::Test(Box::new(Test::False)), + Test::False => return Instruction::Test(Box::new(Test::True)), + Test::Address(op) => { + op.is_not = true; + } + Test::Envelope(op) => { + op.is_not = true; + } + Test::Exists(op) => { + op.is_not = true; + } + Test::Header(op) => { + op.is_not = true; + } + Test::Size(op) => { + op.is_not = true; + } + Test::Body(op) => { + op.is_not = true; + } + Test::Convert(op) => { + op.is_not = true; + } + Test::Date(op) => { + op.is_not = true; + } + Test::CurrentDate(op) => { + op.is_not = true; + } + Test::Duplicate(op) => { + op.is_not = true; + } + Test::String(op) | Test::Environment(op) => { + op.is_not = true; + } + Test::NotifyMethodCapability(op) => { + op.is_not = true; + } + Test::ValidNotifyMethod(op) => { + op.is_not = true; + } + Test::ValidExtList(op) => { + op.is_not = true; + } + Test::Ihave(op) => { + op.is_not = true; + } + Test::HasFlag(op) => { + op.is_not = true; + } + Test::MailboxExists(op) => { + op.is_not = true; + } + Test::Metadata(op) => { + op.is_not = true; + } + Test::MetadataExists(op) => { + op.is_not = true; + } + Test::MailboxIdExists(op) => { + op.is_not = true; + } + Test::SpamTest(op) => { + op.is_not = true; + } + Test::VirusTest(op) => { + op.is_not = true; + } + Test::SpecialUseExists(op) => { + op.is_not = true; + } + #[cfg(all(test, stalwart_tests))] + Test::TestCmd(op) => { + op.is_not = true; + } + Test::Vacation(_) | Test::Invalid(_) => {} + }, + Instruction::Eval(expr) => { + let mut items = std::mem::take(expr).into_vec(); + items.push(Expression::UnaryOperator(UnaryOperator::Not)); + *expr = items.into_boxed_slice(); + } + _ => (), + } + self + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/mod.rs b/crates/sieve/src/compiler/grammar/tests/mod.rs new file mode 100644 index 00000000..733bb4c3 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/mod.rs @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod test_address; +pub mod test_body; +pub mod test_date; +pub mod test_duplicate; +pub mod test_envelope; +pub mod test_environment; +pub mod test_exists; +pub mod test_extlists; +pub mod test_hasflag; +pub mod test_header; +pub mod test_ihave; +pub mod test_mailbox; +pub mod test_mailboxid; +pub mod test_notify; +pub mod test_size; +pub mod test_spamtest; +pub mod test_specialuse; +pub mod test_string; diff --git a/crates/sieve/src/compiler/grammar/tests/test_address.rs b/crates/sieve/src/compiler/grammar/tests/test_address.rs new file mode 100644 index 00000000..be07a3ad --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_address.rs @@ -0,0 +1,172 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{Capability, Comparator, instruction::CompilerState, test::Test}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{AddressPart, MatchType}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestAddress { + pub header_list: Box<[Value]>, + pub key_list: Box<[Value]>, + pub address_part: AddressPart, + pub match_type: MatchType, + pub comparator: Comparator, + pub index: Option, + + pub mime_anychild: bool, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_address(&mut self) -> Result { + let mut address_part = AddressPart::All; + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut header_list = None; + let key_list; + let mut index = None; + let mut index_last = false; + + let mut mime = false; + let mut mime_anychild = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::LocalPart + | Word::Domain + | Word::All + | Word::User + | Word::Detail + | Word::Name), + ) => { + self.validate_argument( + 1, + if matches!(word, Word::User | Word::Detail) { + Capability::SubAddress.into() + } else { + None + }, + token_info.line_num, + token_info.line_pos, + )?; + address_part = word.into(); + } + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 2, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Index) => { + self.validate_argument( + 4, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + index = (self.tokens.expect_number(u16::MAX as usize)? as i32).into(); + } + Token::Tag(Word::Last) => { + self.validate_argument( + 5, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + index_last = true; + } + Token::Tag(Word::Mime) => { + self.validate_argument( + 6, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime = true; + } + Token::Tag(Word::AnyChild) => { + self.validate_argument( + 7, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_anychild = true; + } + _ => { + if header_list.is_none() { + let headers = self.parse_raw_strings_token(token_info)?; + header_list = self.resolve_header_names(headers).0.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + + if !mime && mime_anychild { + return Err(self.tokens.unwrap_next()?.missing_tag(":mime")); + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Address(Box::new(TestAddress { + header_list: header_list.unwrap().into(), + key_list: key_list.into(), + address_part, + match_type, + comparator, + index: if index_last { index.map(|i| -i) } else { index }, + mime_anychild, + is_not: false, + }))) + } +} + +impl From for AddressPart { + fn from(word: Word) -> Self { + match word { + Word::LocalPart => AddressPart::LocalPart, + Word::Domain => AddressPart::Domain, + Word::All => AddressPart::All, + Word::User => AddressPart::User, + Word::Detail => AddressPart::Detail, + Word::Name => AddressPart::Name, + _ => unreachable!(), + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_body.rs b/crates/sieve/src/compiler/grammar/tests/test_body.rs new file mode 100644 index 00000000..407b2982 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_body.rs @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestBody { + pub key_list: Box<[Value]>, + pub body_transform: BodyTransform, + pub match_type: MatchType, + pub comparator: Comparator, + pub include_subject: bool, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum BodyTransform { + Raw = 0, + Content(Box<[Value]>) = 1, + Text = 2, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_body(&mut self) -> Result { + let mut body_transform = BodyTransform::Text; + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let key_list; + let mut include_subject = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Raw) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + body_transform = BodyTransform::Raw; + } + Token::Tag(Word::Text) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + body_transform = BodyTransform::Text; + } + Token::Tag(Word::Content) => { + self.validate_argument(1, None, token_info.line_num, token_info.line_pos)?; + body_transform = BodyTransform::Content(self.parse_strings(false)?.into()); + } + Token::Tag(Word::Subject) => { + self.validate_argument(4, None, token_info.line_num, token_info.line_pos)?; + include_subject = true; + } + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 2, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Body(Box::new(TestBody { + key_list: key_list.into(), + body_transform, + match_type, + comparator, + include_subject, + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_date.rs b/crates/sieve/src/compiler/grammar/tests/test_date.rs new file mode 100644 index 00000000..3c174471 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_date.rs @@ -0,0 +1,373 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, Number, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{StringConstant, Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestDate { + pub header_name: Value, + pub key_list: Box<[Value]>, + pub match_type: MatchType, + pub comparator: Comparator, + pub index: Option, + pub zone: Zone, + pub date_part: DatePart, + pub mime_anychild: bool, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestCurrentDate { + pub zone: Option, + pub match_type: MatchType, + pub comparator: Comparator, + pub date_part: DatePart, + pub key_list: Box<[Value]>, + pub is_not: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Zone { + Time(i64) = 0, + Original = 1, + Local = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum DatePart { + Year = 0, + Month = 1, + Day = 2, + Date = 3, + Julian = 4, + Hour = 5, + Minute = 6, + Second = 7, + Time = 8, + Iso8601 = 9, + Std11 = 10, + Zone = 11, + Weekday = 12, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_date(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut header_name = None; + let key_list; + let mut index = None; + let mut index_last = false; + let mut zone = Zone::Local; + let mut date_part = None; + + let mut mime = false; + let mut mime_anychild = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Index) => { + self.validate_argument( + 3, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + index = (self.tokens.expect_number(u16::MAX as usize)? as i32).into(); + } + Token::Tag(Word::Last) => { + self.validate_argument( + 4, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + index_last = true; + } + Token::Tag(Word::Mime) => { + self.validate_argument( + 5, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime = true; + } + Token::Tag(Word::AnyChild) => { + self.validate_argument( + 6, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_anychild = true; + } + Token::Tag(Word::OriginalZone) => { + self.validate_argument(7, None, token_info.line_num, token_info.line_pos)?; + zone = Zone::Original; + } + Token::Tag(Word::Zone) => { + self.validate_argument(7, None, token_info.line_num, token_info.line_pos)?; + zone = Zone::Time(self.parse_timezone()?); + } + _ => { + if header_name.is_none() { + let header = self.parse_raw_string_token(token_info)?; + header_name = match self.resolve_header_name(header) { + Some(header) => header.into(), + None => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + }; + } else if date_part.is_none() { + if let Token::StringConstant(string) = &token_info.token + && let Some(date_part_) = + lookup_date_part(&string.to_string().to_ascii_lowercase()) + { + date_part = date_part_.into(); + continue; + } + return Err(token_info.expected("valid date part")); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + + if !mime && mime_anychild { + return Err(self.tokens.unwrap_next()?.missing_tag(":mime")); + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Date(Box::new(TestDate { + header_name: header_name.unwrap(), + key_list: key_list.into(), + date_part: date_part.unwrap(), + match_type, + comparator, + index: if index_last { index.map(|i| -i) } else { index }, + zone, + mime_anychild, + is_not: false, + }))) + } + + pub(crate) fn parse_test_currentdate(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let key_list; + let mut zone = None; + let mut date_part = None; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Zone) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + zone = self.parse_timezone()?.into(); + } + _ => { + if date_part.is_none() { + if let Token::StringConstant(string) = &token_info.token + && let Some(date_part_) = + lookup_date_part(&string.to_string().to_ascii_lowercase()) + { + date_part = date_part_.into(); + continue; + } + return Err(token_info.expected("valid date part")); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::CurrentDate(Box::new(TestCurrentDate { + key_list: key_list.into(), + date_part: date_part.unwrap(), + match_type, + comparator, + zone, + is_not: false, + }))) + } + + pub(crate) fn parse_timezone(&mut self) -> Result { + let token_info = self.tokens.unwrap_next()?; + if let Token::StringConstant(value) = &token_info.token { + let timezone = match value { + StringConstant::String(value) => value.parse::().unwrap_or(i64::MAX), + StringConstant::Number(Number::Integer(n)) => *n, + StringConstant::Number(Number::Float(n)) => *n as i64, + }; + + return match timezone { + 0..=1400 => Ok((timezone / 100 * 3600) + (timezone % 100 * 60)), + -1200..=-1 => Ok((timezone / 100 * 3600) - (-timezone % 100 * 60)), + _ => Err(token_info.expected("invalid timezone")), + }; + } + Err(token_info.expected("string containing time zone")) + } +} + +/* + "year" => the year, "0000" .. "9999". + "month" => the month, "01" .. "12". + "day" => the day, "01" .. "31". + "date" => the date in "yyyy-mm-dd" format. + "julian" => the Modified Julian Day, that is, the date + expressed as an integer number of days since + 00:00 UTC on November 17, 1858 (using the Gregorian + calendar). This corresponds to the regular + Julian Day minus 2400000.5. Sample routines to + convert to and from modified Julian dates are + given in Appendix A. + "hour" => the hour, "00" .. "23". + "minute" => the minute, "00" .. "59". + "second" => the second, "00" .. "60". + "time" => the time in "hh:mm:ss" format. + "iso8601" => the date and time in restricted ISO 8601 format. + "std11" => the date and time in a format appropriate + for use in a Date: header field [RFC2822]. + "zone" => the time zone in use. If the user specified a + time zone with ":zone", "zone" will + contain that value. If :originalzone is specified + this value will be the original zone specified + in the date-time value. If neither argument is + specified the value will be the server's default + time zone in offset format "+hhmm" or "-hhmm". An + offset of 0 (Zulu) always has a positive sign. + "weekday" => the day of the week expressed as an integer between + "0" and "6". "0" is Sunday, "1" is Monday, etc. +*/ + +fn lookup_date_part(input: &str) -> Option { + hashify::map!( + input.as_bytes(), DatePart, + "year" => DatePart::Year, + "month" => DatePart::Month, + "day" => DatePart::Day, + "date" => DatePart::Date, + "julian" => DatePart::Julian, + "hour" => DatePart::Hour, + "minute" => DatePart::Minute, + "second" => DatePart::Second, + "time" => DatePart::Time, + "iso8601" => DatePart::Iso8601, + "std11" => DatePart::Std11, + "zone" => DatePart::Zone, + "weekday" => DatePart::Weekday, + ) + .copied() +} + +impl DatePart { + pub(crate) fn from_code(code: u8) -> DatePart { + match code { + 0 => DatePart::Year, + 1 => DatePart::Month, + 2 => DatePart::Day, + 3 => DatePart::Date, + 4 => DatePart::Julian, + 5 => DatePart::Hour, + 6 => DatePart::Minute, + 7 => DatePart::Second, + 8 => DatePart::Time, + 9 => DatePart::Iso8601, + 10 => DatePart::Std11, + 11 => DatePart::Zone, + _ => DatePart::Weekday, + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_duplicate.rs b/crates/sieve/src/compiler/grammar/tests/test_duplicate.rs new file mode 100644 index 00000000..68635ecd --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_duplicate.rs @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, Value, + grammar::instruction::{CompilerState, MapLocalVars}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestDuplicate { + pub handle: Option, + pub dup_match: DupMatch, + pub seconds: Option, + pub last: bool, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum DupMatch { + Header(Value) = 0, + UniqueId(Value) = 1, + Default = 2, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_duplicate(&mut self) -> Result { + let mut handle = None; + let mut dup_match = DupMatch::Default; + let mut seconds = None; + let mut last = false; + + while let Some(token_info) = self.tokens.peek() { + let token_info = token_info?; + let line_num = token_info.line_num; + let line_pos = token_info.line_pos; + + match token_info.token { + Token::Tag(Word::Handle) => { + self.validate_argument(1, None, line_num, line_pos)?; + self.tokens.next(); + handle = self.parse_string()?.into(); + } + Token::Tag(Word::Header) => { + self.validate_argument(2, None, line_num, line_pos)?; + self.tokens.next(); + let header = self.parse_raw_string()?; + dup_match = match self.resolve_header_name(header) { + Some(header) => DupMatch::Header(header), + None => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + }; + } + Token::Tag(Word::UniqueId) => { + self.validate_argument(2, None, line_num, line_pos)?; + self.tokens.next(); + dup_match = DupMatch::UniqueId(self.parse_string()?); + } + Token::Tag(Word::Seconds) => { + self.validate_argument(3, None, line_num, line_pos)?; + self.tokens.next(); + seconds = (self.tokens.expect_number(u64::MAX as usize)? as u64).into(); + } + Token::Tag(Word::Last) => { + self.validate_argument(4, None, line_num, line_pos)?; + self.tokens.next(); + last = true; + } + _ => break, + } + } + + Ok(Test::Duplicate(Box::new(TestDuplicate { + handle, + dup_match, + seconds, + last, + is_not: false, + }))) + } +} + +impl MapLocalVars for DupMatch { + fn map_local_vars(&mut self, last_id: u16) { + match self { + DupMatch::Header(header) => header.map_local_vars(last_id), + DupMatch::UniqueId(unique_id) => unique_id.map_local_vars(last_id), + DupMatch::Default => {} + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_envelope.rs b/crates/sieve/src/compiler/grammar/tests/test_envelope.rs new file mode 100644 index 00000000..ba3878a0 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_envelope.rs @@ -0,0 +1,233 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Envelope, + compiler::{ + CompileError, ErrorType, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, + }, +}; + +use crate::compiler::grammar::{AddressPart, MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestEnvelope { + pub envelope_list: Box<[Envelope]>, + pub key_list: Box<[Value]>, + pub address_part: AddressPart, + pub match_type: MatchType, + pub comparator: Comparator, + pub zone: Option, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_envelope(&mut self) -> Result { + let mut address_part = AddressPart::All; + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut envelope_list = None; + let key_list; + let mut zone = None; + + loop { + let mut token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::LocalPart | Word::Domain | Word::All | Word::User | Word::Detail), + ) => { + self.validate_argument( + 1, + if matches!(word, Word::User | Word::Detail) { + Capability::SubAddress.into() + } else { + None + }, + token_info.line_num, + token_info.line_pos, + )?; + address_part = word.into(); + } + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 2, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(3, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Zone) => { + self.validate_argument( + 4, + Capability::EnvelopeDeliverBy.into(), + token_info.line_num, + token_info.line_pos, + )?; + zone = self.parse_timezone()?.into(); + } + _ => { + if envelope_list.is_none() { + let mut envelopes = Vec::new(); + let line_num = token_info.line_num; + let line_pos = token_info.line_pos; + + match token_info.token { + Token::StringConstant(s) => { + match Envelope::try_from(s.into_string().to_ascii_lowercase()) { + Ok(envelope) => { + envelopes.push(envelope); + } + Err(invalid) => { + token_info.token = Token::Comma; + return Err( + token_info.custom(ErrorType::InvalidEnvelope(invalid)) + ); + } + } + } + Token::BracketOpen => loop { + let mut token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::StringConstant(s) => { + match Envelope::try_from( + s.into_string().to_ascii_lowercase(), + ) { + Ok(envelope) => { + if !envelopes.contains(&envelope) { + envelopes.push(envelope); + } + } + Err(invalid) => { + token_info.token = Token::Comma; + return Err(token_info + .custom(ErrorType::InvalidEnvelope(invalid))); + } + } + } + Token::Comma => (), + Token::BracketClose if !envelopes.is_empty() => break, + _ => return Err(token_info.expected("constant string")), + } + }, + _ => return Err(token_info.expected("constant string")), + } + + for envelope in &envelopes { + match envelope { + Envelope::ByTimeAbsolute + | Envelope::ByTimeRelative + | Envelope::ByMode + | Envelope::ByTrace => { + self.validate_argument( + 0, + Capability::EnvelopeDeliverBy.into(), + line_num, + line_pos, + )?; + } + + Envelope::Notify + | Envelope::Orcpt + | Envelope::Ret + | Envelope::Envid => { + self.validate_argument( + 0, + Capability::EnvelopeDsn.into(), + line_num, + line_pos, + )?; + } + _ => (), + } + } + + envelope_list = envelopes.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Envelope(Box::new(TestEnvelope { + envelope_list: envelope_list.unwrap().into(), + key_list: key_list.into(), + address_part, + match_type, + comparator, + zone, + is_not: false, + }))) + } +} + +impl TryFrom for Envelope { + type Error = String; + + fn try_from(value: String) -> Result { + if let Some(envelope) = lookup_envelope(&value) { + Ok(envelope) + } else { + Err(value) + } + } +} + +impl<'x> TryFrom<&'x str> for Envelope { + type Error = &'x str; + + fn try_from(value: &'x str) -> Result { + if let Some(envelope) = lookup_envelope(value) { + Ok(envelope) + } else { + Err(value) + } + } +} + +fn lookup_envelope(input: &str) -> Option { + hashify::map!( + input.as_bytes(), Envelope, + "from" => Envelope::From, + "to" => Envelope::To, + "bytimeabsolute" => Envelope::ByTimeAbsolute, + "bytimerelative" => Envelope::ByTimeRelative, + "bymode" => Envelope::ByMode, + "bytrace" => Envelope::ByTrace, + "notify" => Envelope::Notify, + "orcpt" => Envelope::Orcpt, + "ret" => Envelope::Ret, + "envid" => Envelope::Envid, + ) + .copied() +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_environment.rs b/crates/sieve/src/compiler/grammar/tests/test_environment.rs new file mode 100644 index 00000000..a44ee8ec --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_environment.rs @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, VariableType, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +use super::test_string::TestString; + +impl CompilerState<'_> { + pub(crate) fn parse_test_environment(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut name = None; + let key_list; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + if name.is_none() { + if let Token::StringConstant(s) = token_info.token { + name = Value::Variable(VariableType::Environment( + s.into_string().to_lowercase(), + )) + .into(); + } else { + return Err(token_info.expected("environment variable")); + } + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Environment(Box::new(TestString { + source: vec![name.unwrap()].into(), + key_list: key_list.into(), + match_type, + comparator, + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_exists.rs b/crates/sieve/src/compiler/grammar/tests/test_exists.rs new file mode 100644 index 00000000..0b4c101f --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_exists.rs @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, Value, + grammar::{Capability, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestExists { + pub header_names: Box<[Value]>, + pub mime_anychild: bool, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_exists(&mut self) -> Result { + let mut header_names = None; + + let mut mime = false; + let mut mime_anychild = false; + + while header_names.is_none() { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag(Word::Mime) => { + self.validate_argument( + 1, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime = true; + } + Token::Tag(Word::AnyChild) => { + self.validate_argument( + 2, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_anychild = true; + } + _ => { + let headers = self.parse_raw_strings_token(token_info)?; + let (headers, all_valid) = self.resolve_header_names(headers); + if !all_valid { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + header_names = headers.into(); + } + } + } + + if !mime && mime_anychild { + return Err(self.tokens.unwrap_next()?.missing_tag(":mime")); + } + + Ok(Test::Exists(Box::new(TestExists { + header_names: header_names.unwrap().into(), + mime_anychild, + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_extlists.rs b/crates/sieve/src/compiler/grammar/tests/test_extlists.rs new file mode 100644 index 00000000..66a71948 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_extlists.rs @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::CompileError; +use crate::compiler::Value; +use crate::compiler::grammar::instruction::CompilerState; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestValidExtList { + pub list_names: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_valid_ext_list(&mut self) -> Result { + Ok(Test::ValidExtList(Box::new(TestValidExtList { + list_names: self.parse_strings(false)?.into(), + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_hasflag.rs b/crates/sieve/src/compiler/grammar/tests/test_hasflag.rs new file mode 100644 index 00000000..c0523466 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_hasflag.rs @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, RawValue, Value, VariableType, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, tokenizer::TokenInfo, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +/* + Usage: hasflag [MATCH-TYPE] [COMPARATOR] + [] + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestHasFlag { + pub comparator: Comparator, + pub match_type: MatchType, + pub variable_list: Box<[VariableType]>, + pub flags: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_hasflag(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut is_local = false; + + let maybe_variables; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Local) => { + is_local = true; + } + _ => { + maybe_variables = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + + match self.tokens.peek() { + Some(Ok(TokenInfo { + token: Token::StringConstant(_) | Token::StringVariable(_) | Token::BracketOpen, + line_num, + line_pos, + })) => { + if !maybe_variables.is_empty() { + let line_num = *line_num; + let line_pos = *line_pos; + + let mut variable_list = Vec::with_capacity(maybe_variables.len()); + for variable in maybe_variables { + let var_name = match variable { + RawValue::Text(text) => text, + RawValue::Value(Value::Text(id)) => self.constant(id).to_string(), + _ => { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::ExpectedConstantString)); + } + }; + variable_list.push(self.register_variable(var_name, is_local).map_err( + |error_type| CompileError { + line_num, + line_pos, + error_type, + }, + )?); + } + let flags = self.parse_raw_strings(false)?; + let flags = self.validate_match(&match_type, &comparator, flags)?; + + Ok(Test::HasFlag(Box::new(TestHasFlag { + comparator, + match_type, + variable_list: variable_list.into(), + flags: flags.into(), + is_not: false, + }))) + } else { + Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::ExpectedConstantString)) + } + } + _ => { + let flags = self.validate_match(&match_type, &comparator, maybe_variables)?; + + Ok(Test::HasFlag(Box::new(TestHasFlag { + comparator, + match_type, + variable_list: Vec::new().into(), + flags: flags.into(), + is_not: false, + }))) + } + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_header.rs b/crates/sieve/src/compiler/grammar/tests/test_header.rs new file mode 100644 index 00000000..28fb270d --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_header.rs @@ -0,0 +1,172 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, ErrorType, Value, + grammar::{ + Capability, Comparator, + actions::action_mime::MimeOpts, + instruction::{CompilerState, MapLocalVars}, + }, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestHeader { + pub header_list: Box<[Value]>, + pub key_list: Box<[Value]>, + pub match_type: MatchType, + pub comparator: Comparator, + pub index: Option, + + pub mime_opts: MimeOpts, + pub mime_anychild: bool, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_header(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut header_list = None; + let key_list; + let mut index = None; + let mut index_last = false; + + let mut mime = false; + let mut mime_opts = MimeOpts::None; + let mut mime_anychild = false; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Index) => { + self.validate_argument( + 3, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + + index = (self.tokens.expect_number(u16::MAX as usize)? as i32).into(); + } + Token::Tag(Word::Last) => { + self.validate_argument( + 4, + Capability::Index.into(), + token_info.line_num, + token_info.line_pos, + )?; + + index_last = true; + } + Token::Tag(Word::Mime) => { + self.validate_argument( + 5, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime = true; + } + Token::Tag(Word::AnyChild) => { + self.validate_argument( + 6, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_anychild = true; + } + Token::Tag( + word @ (Word::Type | Word::Subtype | Word::ContentType | Word::Param), + ) => { + self.validate_argument( + 7, + Capability::Mime.into(), + token_info.line_num, + token_info.line_pos, + )?; + mime_opts = self.parse_mimeopts(word)?; + } + _ => { + if header_list.is_none() { + let headers = self.parse_raw_strings_token(token_info)?; + let (headers, all_valid) = self.resolve_header_names(headers); + if !all_valid { + return Err(self + .tokens + .unwrap_next()? + .custom(ErrorType::InvalidHeaderName)); + } + header_list = headers.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + + if !mime && (mime_anychild || mime_opts != MimeOpts::None) { + return Err(self.tokens.unwrap_next()?.missing_tag(":mime")); + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Header(Box::new(TestHeader { + header_list: header_list.unwrap().into(), + key_list: key_list.into(), + match_type, + comparator, + index: if index_last { index.map(|i| -i) } else { index }, + mime_opts, + mime_anychild, + is_not: false, + }))) + } +} + +impl MapLocalVars for MimeOpts { + fn map_local_vars(&mut self, last_id: u16) { + if let MimeOpts::Param(value) = self { + value.map_local_vars(last_id) + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_ihave.rs b/crates/sieve/src/compiler/grammar/tests/test_ihave.rs new file mode 100644 index 00000000..9b233d8e --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_ihave.rs @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::CompileError; +use crate::compiler::Value; +use crate::compiler::grammar::Capability; +use crate::compiler::grammar::instruction::{CompilerState, Instruction}; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestIhave { + pub capabilities: Box<[Capability]>, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct Error { + pub message: Value, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_ihave(&mut self) -> Result { + Ok(Test::Ihave(Box::new(TestIhave { + capabilities: self + .parse_static_strings()? + .into_iter() + .map(|n| n.into()) + .collect(), + is_not: false, + }))) + } + + pub(crate) fn parse_error(&mut self) -> Result<(), CompileError> { + let cmd = Instruction::Error(Box::new(Error { + message: self.parse_string()?, + })); + self.instructions.push(cmd); + Ok(()) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_mailbox.rs b/crates/sieve/src/compiler/grammar/tests/test_mailbox.rs new file mode 100644 index 00000000..1ecedb28 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_mailbox.rs @@ -0,0 +1,227 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Metadata, + compiler::{ + CompileError, RawValue, Value, + grammar::{ + Capability, Comparator, + instruction::{CompilerState, MapLocalVars}, + }, + lexer::{Token, word::Word}, + }, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestMailboxExists { + pub mailbox_names: Box<[Value]>, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestMetadataExists { + pub mailbox: Option, + pub annotation_names: Box<[Value]>, + pub is_not: bool, +} + +/* + +metadata [MATCH-TYPE] [COMPARATOR] + + + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestMetadata { + pub match_type: MatchType, + pub comparator: Comparator, + pub medatata: Metadata, + pub key_list: Box<[Value]>, + pub is_not: bool, +} + +/* + +servermetadata [MATCH-TYPE] [COMPARATOR] + + +*/ + +impl CompilerState<'_> { + pub(crate) fn parse_test_mailboxexists(&mut self) -> Result { + Ok(Test::MailboxExists(Box::new(TestMailboxExists { + mailbox_names: self.parse_strings(false)?.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_test_metadataexists(&mut self) -> Result { + Ok(Test::MetadataExists(Box::new(TestMetadataExists { + mailbox: self.parse_string()?.into(), + annotation_names: self.parse_strings(false)?.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_test_servermetadataexists(&mut self) -> Result { + Ok(Test::MetadataExists(Box::new(TestMetadataExists { + mailbox: None, + annotation_names: self.parse_strings(false)?.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_test_metadata(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut mailbox = None; + let mut annotation_name = None; + let key_list: Vec; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + if mailbox.is_none() { + mailbox = self.parse_string_token(token_info)?.into(); + } else if annotation_name.is_none() { + annotation_name = self.parse_string_token(token_info)?.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Metadata(Box::new(TestMetadata { + match_type, + comparator, + medatata: Metadata::Mailbox { + name: mailbox.unwrap(), + annotation: annotation_name.unwrap(), + }, + key_list: key_list.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_test_servermetadata(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut annotation_name = None; + let key_list: Vec; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + if annotation_name.is_none() { + annotation_name = self.parse_string_token(token_info)?.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::Metadata(Box::new(TestMetadata { + match_type, + comparator, + medatata: Metadata::Server { + annotation: annotation_name.unwrap(), + }, + key_list: key_list.into(), + is_not: false, + }))) + } +} + +impl MapLocalVars for Metadata { + fn map_local_vars(&mut self, last_id: u16) { + match self { + Metadata::Mailbox { name, annotation } => { + name.map_local_vars(last_id); + annotation.map_local_vars(last_id); + } + Metadata::Server { annotation } => { + annotation.map_local_vars(last_id); + } + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_mailboxid.rs b/crates/sieve/src/compiler/grammar/tests/test_mailboxid.rs new file mode 100644 index 00000000..16ec3bba --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_mailboxid.rs @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::CompileError; +use crate::compiler::Value; +use crate::compiler::grammar::instruction::CompilerState; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestMailboxIdExists { + pub mailbox_ids: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_mailboxidexists(&mut self) -> Result { + Ok(Test::MailboxIdExists(Box::new(TestMailboxIdExists { + mailbox_ids: self.parse_strings(false)?.into(), + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_notify.rs b/crates/sieve/src/compiler/grammar/tests/test_notify.rs new file mode 100644 index 00000000..597ccdf6 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_notify.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestNotifyMethodCapability { + pub comparator: Comparator, + pub match_type: MatchType, + pub notification_uri: Value, + pub notification_capability: Value, + pub key_list: Box<[Value]>, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestValidNotifyMethod { + pub notification_uris: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_valid_notify_method(&mut self) -> Result { + Ok(Test::ValidNotifyMethod(Box::new(TestValidNotifyMethod { + notification_uris: self.parse_strings(false)?.into(), + is_not: false, + }))) + } + + pub(crate) fn parse_test_notify_method_capability(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut notification_uri = None; + let mut notification_capability = None; + let key_list; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + if notification_uri.is_none() { + notification_uri = self.parse_string_token(token_info)?.into(); + } else if notification_capability.is_none() { + notification_capability = self.parse_string_token(token_info)?.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::NotifyMethodCapability(Box::new( + TestNotifyMethodCapability { + key_list: key_list.into(), + match_type, + comparator, + notification_uri: notification_uri.unwrap(), + notification_capability: notification_capability.unwrap(), + is_not: false, + }, + ))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_size.rs b/crates/sieve/src/compiler/grammar/tests/test_size.rs new file mode 100644 index 00000000..b3f18c68 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_size.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, + grammar::instruction::CompilerState, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::test::Test; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestSize { + pub over: bool, + pub limit: u64, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_size(&mut self) -> Result { + let token_info = self.tokens.unwrap_next()?; + let over = match token_info.token { + Token::Tag(Word::Over) => true, + Token::Tag(Word::Under) => false, + _ => { + return Err(token_info.expected("':over' or ':under'")); + } + }; + let token_info = self.tokens.unwrap_next()?; + if let Token::Number(limit) = token_info.token { + Ok(Test::Size(Box::new(TestSize { + over, + limit: limit as u64, + is_not: false, + }))) + } else { + Err(token_info.expected("number")) + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_spamtest.rs b/crates/sieve/src/compiler/grammar/tests/test_spamtest.rs new file mode 100644 index 00000000..8b900900 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_spamtest.rs @@ -0,0 +1,154 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +/* + Usage: spamtest [":percent"] [COMPARATOR] [MATCH-TYPE] + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestSpamTest { + pub value: Value, + pub match_type: MatchType, + pub comparator: Comparator, + pub percent: bool, + pub is_not: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestVirusTest { + pub value: Value, + pub match_type: MatchType, + pub comparator: Comparator, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_spamtest(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut percent = false; + let value; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + Token::Tag(Word::Percent) => { + self.validate_argument( + 3, + Capability::SpamTestPlus.into(), + token_info.line_num, + token_info.line_pos, + )?; + percent = true; + } + _ => { + value = self.parse_string_token(token_info)?; + break; + } + } + } + + Ok(Test::SpamTest(Box::new(TestSpamTest { + value, + percent, + match_type, + comparator, + is_not: false, + }))) + } + + pub(crate) fn parse_test_virustest(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let value; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + value = self.parse_string_token(token_info)?; + break; + } + } + } + + Ok(Test::VirusTest(Box::new(TestVirusTest { + value, + match_type, + comparator, + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_specialuse.rs b/crates/sieve/src/compiler/grammar/tests/test_specialuse.rs new file mode 100644 index 00000000..6cfb5160 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_specialuse.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{CompileError, Value, grammar::instruction::CompilerState, lexer::Token}; + +use crate::compiler::grammar::test::Test; + +/* + Usage: specialuse_exists [] + +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestSpecialUseExists { + pub mailbox: Option, + pub attributes: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_specialuseexists(&mut self) -> Result { + let mut maybe_attributes = self.parse_strings(false)?; + + match self.tokens.peek().map(|r| r.map(|t| &t.token)) { + Some(Ok(Token::StringConstant(_) | Token::StringVariable(_) | Token::BracketOpen)) => { + if maybe_attributes.len() == 1 { + Ok(Test::SpecialUseExists(Box::new(TestSpecialUseExists { + mailbox: maybe_attributes.pop(), + attributes: self.parse_strings(false)?.into(), + is_not: false, + }))) + } else { + Err(self.tokens.unwrap_next()?.expected("string")) + } + } + _ => Ok(Test::SpecialUseExists(Box::new(TestSpecialUseExists { + mailbox: None, + attributes: maybe_attributes.into(), + is_not: false, + }))), + } + } +} diff --git a/crates/sieve/src/compiler/grammar/tests/test_string.rs b/crates/sieve/src/compiler/grammar/tests/test_string.rs new file mode 100644 index 00000000..1dd2d1f8 --- /dev/null +++ b/crates/sieve/src/compiler/grammar/tests/test_string.rs @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::{ + CompileError, RawValue, Value, + grammar::{Capability, Comparator, instruction::CompilerState}, + lexer::{Token, word::Word}, +}; + +use crate::compiler::grammar::{MatchType, test::Test}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct TestString { + pub match_type: MatchType, + pub comparator: Comparator, + pub source: Box<[Value]>, + pub key_list: Box<[Value]>, + pub is_not: bool, +} + +impl CompilerState<'_> { + pub(crate) fn parse_test_string(&mut self) -> Result { + let mut match_type = MatchType::Is; + let mut comparator = Comparator::AsciiCaseMap; + let mut source = None; + let key_list: Vec; + + loop { + let token_info = self.tokens.unwrap_next()?; + match token_info.token { + Token::Tag( + word @ (Word::Is + | Word::Contains + | Word::Matches + | Word::Value + | Word::Count + | Word::Regex + | Word::List), + ) => { + self.validate_argument( + 1, + match word { + Word::Value | Word::Count => Capability::Relational.into(), + Word::Regex => Capability::Regex.into(), + Word::List => Capability::ExtLists.into(), + _ => None, + }, + token_info.line_num, + token_info.line_pos, + )?; + + match_type = self.parse_match_type(word)?; + } + Token::Tag(Word::Comparator) => { + self.validate_argument(2, None, token_info.line_num, token_info.line_pos)?; + comparator = self.parse_comparator()?; + } + _ => { + if source.is_none() { + source = self.parse_strings_token(token_info)?.into(); + } else { + key_list = self.parse_raw_strings_token(token_info)?; + break; + } + } + } + } + let key_list = self.validate_match(&match_type, &comparator, key_list)?; + + Ok(Test::String(Box::new(TestString { + source: source.unwrap().into(), + key_list: key_list.into(), + match_type, + comparator, + is_not: false, + }))) + } +} diff --git a/crates/sieve/src/compiler/lexer/mod.rs b/crates/sieve/src/compiler/lexer/mod.rs new file mode 100644 index 00000000..5bd7b208 --- /dev/null +++ b/crates/sieve/src/compiler/lexer/mod.rs @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod string; +pub mod tokenizer; +pub mod word; + +use self::word::Word; +use super::Number; +use std::{borrow::Cow, fmt::Display}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Token { + CurlyOpen, + CurlyClose, + BracketOpen, + BracketClose, + ParenthesisOpen, + ParenthesisClose, + Comma, + Semicolon, + StringConstant(StringConstant), + StringVariable(Vec), + Number(usize), + Identifier(Word), + Tag(Word), + Unknown(String), + Colon, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum StringConstant { + String(String), + Number(Number), +} + +impl StringConstant { + pub fn to_string(&'_ self) -> Cow<'_, str> { + match self { + StringConstant::String(s) => s.as_str().into(), + StringConstant::Number(n) => n.to_string().into(), + } + } + + pub fn into_string(self) -> String { + match self { + StringConstant::String(s) => s, + StringConstant::Number(n) => n.to_string(), + } + } +} + +impl Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Token::CurlyOpen => f.write_str("{"), + Token::CurlyClose => f.write_str("}"), + Token::BracketOpen => f.write_str("["), + Token::BracketClose => f.write_str("]"), + Token::ParenthesisOpen => f.write_str("("), + Token::ParenthesisClose => f.write_str(")"), + Token::Comma => f.write_str(","), + Token::Semicolon => f.write_str(";"), + Token::Colon => f.write_str(":"), + Token::Number(n) => write!(f, "{n}"), + Token::Identifier(w) => w.fmt(f), + Token::Tag(t) => write!(f, ":{t}"), + Token::Unknown(s) => f.write_str(s), + Token::StringVariable(s) => f.write_str(&String::from_utf8_lossy(s)), + Token::StringConstant(c) => match c { + StringConstant::String(s) => f.write_str(s), + StringConstant::Number(n) => write!(f, "{n}"), + }, + } + } +} diff --git a/crates/sieve/src/compiler/lexer/string.rs b/crates/sieve/src/compiler/lexer/string.rs new file mode 100644 index 00000000..959722d2 --- /dev/null +++ b/crates/sieve/src/compiler/lexer/string.rs @@ -0,0 +1,761 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Envelope, MAX_MATCH_VARIABLES, + compiler::{ + ContentTypePart, ErrorType, HeaderPart, HeaderVariable, MessagePart, Number, + ReceivedHostname, ReceivedPart, Value, VariableType, + grammar::{ + AddressPart, + expr::{self}, + instruction::CompilerState, + }, + }, +}; +use mail_parser::HeaderName; +use std::fmt::Display; + +enum State { + None, + Variable, + Encoded { + is_unicode: bool, + initial_buf_size: usize, + }, +} + +impl CompilerState<'_> { + pub(crate) fn tokenize_string( + &mut self, + bytes: &[u8], + parse_decoded: bool, + ) -> Result { + let mut state = State::None; + let mut items = Vec::with_capacity(3); + let mut last_ch = 0; + + let mut var_start_pos = usize::MAX; + let mut var_is_number = true; + let mut var_has_namespace = false; + + let mut text_has_digits = true; + let mut text_has_dots = false; + + let mut hex_start = usize::MAX; + let mut decode_buf = Vec::with_capacity(bytes.len()); + + for (pos, &ch) in bytes.iter().enumerate() { + let mut is_var_error = false; + + match state { + State::None => match ch { + b'{' if last_ch == b'$' => { + decode_buf.pop(); + var_start_pos = pos + 1; + var_is_number = true; + var_has_namespace = false; + state = State::Variable; + } + b'.' => { + if text_has_dots { + text_has_digits = false; + } else { + text_has_dots = true; + } + decode_buf.push(ch); + } + b'0'..=b'9' => { + decode_buf.push(ch); + } + _ => { + text_has_digits = false; + decode_buf.push(ch); + } + }, + State::Variable => match ch { + b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'[' | b']' | b'*' | b'-' => { + var_is_number = false; + } + b'.' => { + var_is_number = false; + var_has_namespace = true; + } + b'0'..=b'9' => {} + b'}' if pos > var_start_pos => { + // Add any text before the variable + if !decode_buf.is_empty() { + self.add_value( + &mut items, + &decode_buf, + parse_decoded, + text_has_digits, + text_has_dots, + )?; + decode_buf.clear(); + text_has_digits = true; + text_has_dots = false; + } + + // Parse variable type + let var_name = std::str::from_utf8(&bytes[var_start_pos..pos]).unwrap(); + let var_type = if !var_is_number { + self.parse_variable(var_name, var_has_namespace) + } else { + self.parse_match_variable(var_name) + }; + + match var_type { + Ok(Some(var)) => items.push(Value::Variable(var)), + Ok(None) => {} + Err(ErrorType::InvalidNamespace(_) | ErrorType::InvalidEnvelope(_)) => { + is_var_error = true; + } + Err(e) => return Err(e), + } + + state = State::None; + } + b':' => { + if parse_decoded && !var_has_namespace { + match bytes.get(var_start_pos..pos) { + Some(enc) if enc.eq_ignore_ascii_case(b"hex") => { + state = State::Encoded { + is_unicode: false, + initial_buf_size: decode_buf.len(), + }; + } + Some(enc) if enc.eq_ignore_ascii_case(b"unicode") => { + state = State::Encoded { + is_unicode: true, + initial_buf_size: decode_buf.len(), + }; + } + _ => { + is_var_error = true; + } + } + } else if var_has_namespace { + var_is_number = false; + } else { + is_var_error = true; + } + } + _ => { + is_var_error = true; + } + }, + + State::Encoded { + is_unicode, + initial_buf_size, + } => match ch { + b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => { + if hex_start == usize::MAX { + hex_start = pos; + } + } + b' ' | b'\t' | b'\r' | b'\n' | b'}' => { + if hex_start != usize::MAX { + let code = std::str::from_utf8(&bytes[hex_start..pos]).unwrap(); + hex_start = usize::MAX; + + if !is_unicode { + if let Ok(ch) = u8::from_str_radix(code, 16) { + decode_buf.push(ch); + } else { + is_var_error = true; + } + } else if let Ok(ch) = u32::from_str_radix(code, 16) { + let mut buf = [0; 4]; + decode_buf.extend_from_slice( + char::from_u32(ch) + .ok_or(ErrorType::InvalidUnicodeSequence(ch))? + .encode_utf8(&mut buf) + .as_bytes(), + ); + } else { + is_var_error = true; + } + } + if ch == b'}' { + if decode_buf.len() != initial_buf_size { + state = State::None; + } else { + is_var_error = true; + } + } + } + _ => { + is_var_error = true; + } + }, + } + + if is_var_error { + if let State::Encoded { + initial_buf_size, .. + } = state + && initial_buf_size != decode_buf.len() + { + decode_buf.truncate(initial_buf_size); + } + decode_buf.extend_from_slice(&bytes[var_start_pos - 2..pos + 1]); + hex_start = usize::MAX; + state = State::None; + } + + last_ch = ch; + } + + match state { + State::Variable => { + decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]); + } + State::Encoded { + initial_buf_size, .. + } => { + if initial_buf_size != decode_buf.len() { + decode_buf.truncate(initial_buf_size); + } + decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]); + } + State::None => (), + } + + if !decode_buf.is_empty() { + self.add_value( + &mut items, + &decode_buf, + parse_decoded, + text_has_digits, + text_has_dots, + )?; + } + + Ok(match items.len() { + 1 => items.pop().unwrap(), + 0 => self.text(""), + _ => Value::List(items.into()), + }) + } + + fn parse_match_variable(&mut self, var_name: &str) -> Result, ErrorType> { + let num = var_name + .parse() + .map_err(|_| ErrorType::InvalidNumber(var_name.to_string()))?; + if num < MAX_MATCH_VARIABLES as usize { + if self.register_match_var(num) { + let total_vars = num + 1; + if total_vars > self.vars_match_max { + self.vars_match_max = total_vars; + } + Ok(Some(VariableType::Match(num as u8))) + } else { + Ok(None) + } + } else { + Err(ErrorType::InvalidMatchVariable(num)) + } + } + + pub fn parse_variable( + &self, + var_name: &str, + maybe_namespace: bool, + ) -> Result, ErrorType> { + if !maybe_namespace { + if self.is_var_global(var_name) { + Ok(Some(VariableType::Global(var_name.to_string()))) + } else if let Some(var_id) = self.get_local_var(var_name) { + Ok(Some(VariableType::Local(var_id))) + } else { + Ok(None) + } + } else { + let lowercase_name = if var_name.is_ascii() { + super::super::grammar::instruction::lowercase(var_name) + } else { + std::borrow::Cow::Owned(var_name.to_lowercase()) + }; + let var = if let Some((namespace, name)) = lowercase_name.split_once('.') { + if name.is_empty() { + return Err(ErrorType::InvalidNamespace(var_name.to_string())); + } + + let mut var = None; + hashify::fnc_map!(namespace.as_bytes(), + "global" => { + var = VariableType::Global(name.to_string()).into(); + }, + "t" => { + var = VariableType::Global(name.to_string()).into(); + }, + "env" => { + var = VariableType::Environment(name.to_string()).into(); + }, + "envelope" => { + var = VariableType::Envelope( + lookup_envelope(name) + .ok_or_else(|| ErrorType::InvalidEnvelope(name.to_string()))?, + ) + .into(); + }, + "header" => { + var = self.parse_header_variable(name)?.into(); + }, + "body" => { + var = VariableType::Part( + lookup_body_part(name) + .ok_or_else(|| ErrorType::InvalidNamespace(name.to_string()))?, + ) + .into(); + }, + "part" => { + var = VariableType::Part( + lookup_message_part(name) + .ok_or_else(|| ErrorType::InvalidNamespace(name.to_string()))?, + ) + .into(); + }, + _ => {} + ); + + var.ok_or_else(|| ErrorType::InvalidNamespace(var_name.to_string()))? + } else if self.is_var_global(var_name) { + VariableType::Global(var_name.to_string()) + } else if let Some(var_id) = self.get_local_var(var_name) { + VariableType::Local(var_id) + } else { + return Ok(None); + }; + + Ok(Some(var)) + } + } + + fn parse_header_variable(&self, var_name: &str) -> Result { + #[derive(Debug)] + enum State { + Name, + Index, + Part, + PartIndex, + } + let mut name = vec![]; + let mut has_name = false; + let mut has_wildcard = false; + let mut hdr_name = String::new(); + let mut hdr_index = String::new(); + let mut part = String::new(); + let mut part_index = String::new(); + let mut state = State::Name; + + for ch in var_name.chars() { + match state { + State::Name => match ch { + '[' => { + state = if hdr_index.is_empty() { + State::Index + } else if part.is_empty() { + State::PartIndex + } else { + return Err(ErrorType::InvalidExpression(var_name.to_string())); + }; + has_name = true; + } + '.' => { + state = State::Part; + has_name = true; + } + ' ' | '\t' | '\r' | '\n' => {} + '*' if !has_wildcard && hdr_name.is_empty() && name.is_empty() => { + has_wildcard = true; + } + ':' if !hdr_name.is_empty() && !has_wildcard => { + name.push( + HeaderName::parse(std::mem::take(&mut hdr_name)).ok_or_else(|| { + ErrorType::InvalidExpression(var_name.to_string()) + })?, + ); + } + _ if !has_name && !has_wildcard => { + hdr_name.push(ch); + } + _ => { + return Err(ErrorType::InvalidExpression(var_name.to_string())); + } + }, + State::Index => match ch { + ']' => { + state = State::Name; + } + ' ' | '\t' | '\r' | '\n' => {} + _ => { + hdr_index.push(ch); + } + }, + State::Part => match ch { + '[' => { + state = State::PartIndex; + } + ' ' | '\t' | '\r' | '\n' => {} + _ => { + part.push(ch); + } + }, + State::PartIndex => match ch { + ']' => { + state = State::Name; + } + ' ' | '\t' | '\r' | '\n' => {} + _ => { + part_index.push(ch); + } + }, + } + } + + if !hdr_name.is_empty() { + name.push( + HeaderName::parse(hdr_name) + .ok_or_else(|| ErrorType::InvalidExpression(var_name.to_string()))?, + ); + } + + if !name.is_empty() || has_wildcard { + Ok(VariableType::Header(HeaderVariable { + name: name.into(), + part: HeaderPart::try_from(part.as_str()) + .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?, + index_hdr: match hdr_index.as_str() { + "" => { + if !has_wildcard { + -1 + } else { + 0 + } + } + "*" => 0, + _ => hdr_index + .parse() + .map(|v| if v == 0 { 1 } else { v }) + .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?, + }, + index_part: match part_index.as_str() { + "" => { + if !has_wildcard { + -1 + } else { + 0 + } + } + "*" => 0, + _ => part_index + .parse() + .map(|v| if v == 0 { 1 } else { v }) + .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?, + }, + })) + } else { + Err(ErrorType::InvalidExpression(var_name.to_string())) + } + } + + pub fn parse_expr_fnc_or_var( + &self, + var_name: &str, + maybe_namespace: bool, + ) -> Result { + match self.parse_variable(var_name, maybe_namespace) { + Ok(Some(var)) => Ok(expr::Token::Variable(var)), + _ => { + if let Some((id, num_args)) = self.compiler.functions.get(var_name) { + Ok(expr::Token::Function { + name: var_name.to_string(), + id: *id, + num_args: *num_args, + }) + } else { + Err(format!("Invalid variable or function name {var_name:?}")) + } + } + } + } + + #[inline(always)] + fn add_value( + &mut self, + items: &mut Vec, + buf: &[u8], + parse_decoded: bool, + has_digits: bool, + has_dots: bool, + ) -> Result<(), ErrorType> { + if !parse_decoded { + let value = if has_digits { + if has_dots { + match std::str::from_utf8(buf) + .ok() + .and_then(|v| (v, v.parse::().ok()?).into()) + { + Some((v, n)) if n.to_string() == v => Value::Number(Number::Float(n)), + _ => self.text(String::from_utf8_lossy(buf)), + } + } else { + match std::str::from_utf8(buf) + .ok() + .and_then(|v| (v, v.parse::().ok()?).into()) + { + Some((v, n)) if n.to_string() == v => Value::Number(Number::Integer(n)), + _ => self.text(String::from_utf8_lossy(buf)), + } + } + } else { + self.text(String::from_utf8_lossy(buf)) + }; + items.push(value); + } else { + match self.tokenize_string(buf, false)? { + Value::List(new_items) => items.extend(new_items), + item => items.push(item), + } + } + + Ok(()) + } +} + +impl TryFrom<&str> for HeaderPart { + type Error = (); + + fn try_from(value: &str) -> Result { + let (value, subvalue) = value.split_once('.').unwrap_or((value, "")); + if value.is_empty() { + return Ok(HeaderPart::Text); + } + + let mut part = None; + hashify::fnc_map!(value.as_bytes(), + "text" => { + part = HeaderPart::Text.into(); + }, + "name" => { + part = HeaderPart::Address(AddressPart::Name).into(); + }, + "addr" => { + part = if !subvalue.is_empty() { + HeaderPart::Address(AddressPart::try_from(subvalue)?) + } else { + HeaderPart::Address(AddressPart::All) + } + .into(); + }, + "type" => { + part = HeaderPart::ContentType(ContentTypePart::Type).into(); + }, + "subtype" => { + part = HeaderPart::ContentType(ContentTypePart::Subtype).into(); + }, + "attr" => { + if !subvalue.is_empty() { + part = HeaderPart::ContentType(ContentTypePart::Attribute( + subvalue.to_string(), + )) + .into(); + } + }, + "rcvd" => { + part = if !subvalue.is_empty() { + HeaderPart::Received(ReceivedPart::try_from(subvalue)?) + } else { + HeaderPart::Text + } + .into(); + }, + "id" => { + part = HeaderPart::Id.into(); + }, + "raw" => { + part = HeaderPart::Raw.into(); + }, + "raw_name" => { + part = HeaderPart::RawName.into(); + }, + "date" => { + part = HeaderPart::Date.into(); + }, + "exists" => { + part = HeaderPart::Exists.into(); + }, + _ => {} + ); + + part.ok_or(()) + } +} + +impl TryFrom<&str> for ReceivedPart { + type Error = (); + + fn try_from(value: &str) -> Result { + lookup_received_part(value).ok_or(()) + } +} + +impl TryFrom<&str> for AddressPart { + type Error = (); + + fn try_from(value: &str) -> Result { + lookup_address_part(value).ok_or(()) + } +} + +fn lookup_envelope(input: &str) -> Option { + hashify::map!( + input.as_bytes(), Envelope, + "from" => Envelope::From, + "to" => Envelope::To, + "by_time_absolute" => Envelope::ByTimeAbsolute, + "by_time_relative" => Envelope::ByTimeRelative, + "by_mode" => Envelope::ByMode, + "by_trace" => Envelope::ByTrace, + "notify" => Envelope::Notify, + "orcpt" => Envelope::Orcpt, + "ret" => Envelope::Ret, + "envid" => Envelope::Envid, + ) + .copied() +} + +fn lookup_body_part(input: &str) -> Option { + hashify::map!( + input.as_bytes(), MessagePart, + "text" => MessagePart::TextBody(false), + "html" => MessagePart::HtmlBody(false), + "to_text" => MessagePart::TextBody(true), + "to_html" => MessagePart::HtmlBody(true), + ) + .copied() +} + +fn lookup_message_part(input: &str) -> Option { + hashify::map!( + input.as_bytes(), MessagePart, + "text" => MessagePart::Contents, + "raw" => MessagePart::Raw, + ) + .copied() +} + +fn lookup_received_part(input: &str) -> Option { + hashify::map!( + input.as_bytes(), ReceivedPart, + "from" => ReceivedPart::From(ReceivedHostname::Any), + "from.name" => ReceivedPart::From(ReceivedHostname::Name), + "from.ip" => ReceivedPart::From(ReceivedHostname::Ip), + "ip" => ReceivedPart::FromIp, + "iprev" => ReceivedPart::FromIpRev, + "by" => ReceivedPart::By(ReceivedHostname::Any), + "by.name" => ReceivedPart::By(ReceivedHostname::Name), + "by.ip" => ReceivedPart::By(ReceivedHostname::Ip), + "for" => ReceivedPart::For, + "with" => ReceivedPart::With, + "tls" => ReceivedPart::TlsVersion, + "cipher" => ReceivedPart::TlsCipher, + "id" => ReceivedPart::Id, + "ident" => ReceivedPart::Ident, + "date" => ReceivedPart::Date, + "date.raw" => ReceivedPart::DateRaw, + ) + .copied() +} + +fn lookup_address_part(input: &str) -> Option { + hashify::map!( + input.as_bytes(), AddressPart, + "name" => AddressPart::Name, + "addr" => AddressPart::All, + "all" => AddressPart::All, + "addr.domain" => AddressPart::Domain, + "addr.local" => AddressPart::LocalPart, + "addr.user" => AddressPart::User, + "addr.detail" => AddressPart::Detail, + ) + .copied() +} + +impl Display for VariableType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VariableType::Local(v) => write!(f, "${{{v}}}"), + VariableType::Match(v) => write!(f, "${{{v}}}"), + VariableType::Global(v) => write!(f, "${{global.{v}}}"), + VariableType::Environment(v) => write!(f, "${{env.{v}}}"), + + VariableType::Envelope(env) => f.write_str(match env { + Envelope::From => "${{envelope.from}}", + Envelope::To => "${{envelope.to}}", + Envelope::ByTimeAbsolute => "${{envelope.by_time_absolute}}", + Envelope::ByTimeRelative => "${{envelope.by_time_relative}}", + Envelope::ByMode => "${{envelope.by_mode}}", + Envelope::ByTrace => "${{envelope.by_trace}}", + Envelope::Notify => "${{envelope.notify}}", + Envelope::Orcpt => "${{envelope.orcpt}}", + Envelope::Ret => "${{envelope.ret}}", + Envelope::Envid => "${{envelope.envit}}", + }), + + VariableType::Header(hdr) => { + write!( + f, + "${{header.{}", + hdr.name.first().map(|h| h.as_str()).unwrap_or_default() + )?; + if hdr.index_hdr != 0 { + write!(f, "[{}]", hdr.index_hdr)?; + } else { + f.write_str("[*]")?; + } + /*if hdr.part != HeaderPart::Text { + f.write_str(".")?; + f.write_str(match &hdr.part { + HeaderPart::Name => "name", + HeaderPart::Address => "address", + HeaderPart::Type => "type", + HeaderPart::Subtype => "subtype", + HeaderPart::Raw => "raw", + HeaderPart::Date => "date", + HeaderPart::Attribute(attr) => attr.as_str(), + HeaderPart::Text => unreachable!(), + })?; + }*/ + if hdr.index_part != 0 { + write!(f, "[{}]", hdr.index_part)?; + } else { + f.write_str("[*]")?; + } + f.write_str("}") + } + VariableType::Part(part) => { + write!( + f, + "${{{}", + match part { + MessagePart::TextBody(true) => "body.to_text", + MessagePart::TextBody(false) => "body.text", + MessagePart::HtmlBody(true) => "body.to_html", + MessagePart::HtmlBody(false) => "body.html", + MessagePart::Contents => "part.text", + MessagePart::Raw => "part.raw", + } + )?; + f.write_str("}") + } + } + } +} + diff --git a/crates/sieve/src/compiler/lexer/tokenizer.rs b/crates/sieve/src/compiler/lexer/tokenizer.rs new file mode 100644 index 00000000..2445e620 --- /dev/null +++ b/crates/sieve/src/compiler/lexer/tokenizer.rs @@ -0,0 +1,646 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{StringConstant, Token, word::lookup_words}; +use crate::{ + Compiler, + compiler::{CompileError, ErrorType, Number}, + runtime::eval::IntoString, +}; + +pub(crate) struct Tokenizer<'x> { + pub compiler: &'x Compiler, + bytes: &'x [u8], + cursor: usize, + pub buf: Vec, + pub next_token: Vec, + + pub pos: usize, + pub line_num: usize, + pub line_start: usize, + + pub text_line_num: usize, + pub text_line_pos: usize, + + pub token_line_num: usize, + pub token_line_pos: usize, + + pub token_is_tag: bool, + + pub last_ch: u8, + pub state: State, +} + +#[derive(Debug)] +pub(crate) struct TokenInfo { + pub(crate) token: Token, + pub(crate) line_num: usize, + pub(crate) line_pos: usize, +} + +pub(crate) enum State { + None, + BracketComment, + HashComment, + QuotedString(StringType), + MultiLine(StringType), +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct StringType { + maybe_variable: bool, + has_other: bool, + has_digits: bool, + has_dots: bool, +} + +impl StringType { + fn classify(self, buf: &[u8]) -> StringType { + let mut result = self; + let mut dots = 0; + let mut last = 0; + for &ch in buf { + match ch { + b'0'..=b'9' => result.has_digits = true, + b'.' => dots += 1, + b'-' => (), + b'{' if last == b'$' || last == b'%' => { + result.maybe_variable = true; + result.has_other = true; + } + _ => result.has_other = true, + } + last = ch; + } + if dots > 1 { + result.has_other = true; + } + result.has_dots = dots > 0; + result + } +} + +impl<'x> Tokenizer<'x> { + pub fn new(compiler: &'x Compiler, bytes: &'x [u8]) -> Self { + Tokenizer { + compiler, + bytes, + cursor: 0, + buf: Vec::with_capacity(256), + pos: usize::MAX, + line_num: 1, + line_start: usize::MAX, + text_line_num: 0, + text_line_pos: 0, + token_line_num: 0, + token_line_pos: 0, + token_is_tag: false, + next_token: Vec::with_capacity(2), + last_ch: 0, + state: State::None, + } + } + + pub fn get_current_token(&mut self) -> Option { + if !self.buf.is_empty() { + let word = std::str::from_utf8(&self.buf).unwrap(); + let token = if let Some(word) = lookup_words(word) { + if self.token_is_tag { + Token::Tag(word) + } else { + Token::Identifier(word) + } + } else if self.buf.first().unwrap().is_ascii_digit() { + let multiplier = match self.buf.last().unwrap() { + b'k' => 1024, + b'm' => 1048576, + b'g' => 1073741824, + _ => 1, + }; + + if let Ok(number) = (if multiplier > 1 && self.buf.len() > 1 { + std::str::from_utf8(&self.buf[..self.buf.len() - 1]).unwrap() + } else { + word + }) + .parse::() + { + Token::Number(number.saturating_mul(multiplier)) + } else if self.token_is_tag { + Token::Unknown(format!(":{word}")) + } else { + Token::Unknown(word.to_string()) + } + } else if self.token_is_tag { + Token::Unknown(format!(":{word}")) + } else { + Token::Unknown(word.to_string()) + }; + + let line_pos = if self.token_is_tag { + self.token_line_pos.saturating_sub(1) + } else { + self.token_line_pos + }; + self.reset_current_token(); + + Some(TokenInfo { + token, + line_num: self.token_line_num, + line_pos, + }) + } else { + None + } + } + + #[inline(always)] + pub fn reset_current_token(&mut self) { + self.buf.clear(); + self.token_is_tag = false; + } + + #[inline(always)] + pub fn token_is_tag(&mut self) { + self.token_is_tag = true; + } + + pub fn get_token(&mut self, token: Token) -> TokenInfo { + let next_token = TokenInfo { + token, + line_num: self.line_num, + line_pos: self.pos.wrapping_sub(self.line_start), + }; + if let Some(token) = self.get_current_token() { + self.next_token.push(next_token); + token + } else { + next_token + } + } + + pub fn get_string(&mut self, str_type: StringType) -> Result { + if self.buf.len() < self.compiler.max_string_size { + let str_type = str_type.classify(&self.buf); + let token = if str_type.maybe_variable { + Token::StringVariable(self.buf.to_vec()) + } else { + let constant = self.buf.to_vec().into_string(); + if !str_type.has_other && str_type.has_digits { + if !str_type.has_dots { + if let Some(number) = constant + .parse::() + .ok() + .filter(|&n| n.to_string() == constant) + { + Token::StringConstant(StringConstant::Number(Number::Integer(number))) + } else { + Token::StringConstant(StringConstant::String(constant)) + } + } else if let Some(number) = constant + .parse::() + .ok() + .filter(|&n| n.to_string() == constant) + { + Token::StringConstant(StringConstant::Number(Number::Float(number))) + } else { + Token::StringConstant(StringConstant::String(constant)) + } + } else { + Token::StringConstant(StringConstant::String(constant)) + } + }; + + self.buf.clear(); + + Ok(TokenInfo { + token, + line_num: self.text_line_num, + line_pos: self.text_line_pos, + }) + } else { + Err(CompileError { + line_num: self.text_line_num, + line_pos: self.text_line_pos, + error_type: ErrorType::StringTooLong, + }) + } + } + + #[inline(always)] + pub fn push_byte(&mut self, ch: u8) { + if self.buf.is_empty() { + self.token_line_num = self.line_num; + self.token_line_pos = self.pos.wrapping_sub(self.line_start); + } + self.buf.push(ch); + } + + #[inline(always)] + pub fn new_line(&mut self) { + self.line_num += 1; + self.line_start = self.pos; + } + + #[inline(always)] + pub fn text_start(&mut self) { + self.text_line_num = self.line_num; + self.text_line_pos = self.pos.wrapping_sub(self.line_start); + } + + #[inline(always)] + pub fn is_token_start(&self) -> bool { + self.buf.is_empty() + } + + #[inline(always)] + pub fn token_bytes(&self) -> &[u8] { + &self.buf + } + + #[inline(always)] + pub fn next_byte(&mut self) -> Option<(u8, u8)> { + self.bytes.get(self.cursor).map(|&ch| { + self.cursor += 1; + let last_ch = self.last_ch; + self.pos = self.pos.wrapping_add(1); + self.last_ch = ch; + (ch, last_ch) + }) + } + + #[inline(always)] + pub fn peek_byte(&mut self) -> Option { + self.bytes.get(self.cursor).copied() + } + + #[inline(always)] + fn skip_word_chunk(&mut self) { + let rest = &self.bytes[self.cursor..]; + let len = rest + .iter() + .position(|b| !matches!(b, b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'$')) + .unwrap_or(rest.len()); + if len > 0 { + self.buf.extend_from_slice(&rest[..len]); + self.cursor += len; + self.pos = self.pos.wrapping_add(len); + self.last_ch = rest[len - 1]; + } + } + + #[inline(always)] + fn skip_blank_chunk(&mut self) { + let rest = &self.bytes[self.cursor..]; + let len = rest + .iter() + .position(|b| !matches!(b, b' ' | b'\t' | b'\r')) + .unwrap_or(rest.len()); + if len > 0 { + self.cursor += len; + self.pos = self.pos.wrapping_add(len); + self.last_ch = rest[len - 1]; + } + } + + #[inline(always)] + fn skip_comment_chunk(&mut self) { + let rest = &self.bytes[self.cursor..]; + let len = memchr::memchr(b'\n', rest).unwrap_or(rest.len()); + if len > 0 { + self.cursor += len; + self.pos = self.pos.wrapping_add(len); + self.last_ch = rest[len - 1]; + } + } + + #[inline(always)] + fn skip_string_chunk(&mut self) { + let rest = &self.bytes[self.cursor..]; + let len = memchr::memchr3(b'"', b'\\', b'\n', rest).unwrap_or(rest.len()); + if len > 0 { + if self.buf.is_empty() { + self.token_line_num = self.line_num; + self.token_line_pos = self.pos.wrapping_sub(self.line_start); + } + self.buf.extend_from_slice(&rest[..len]); + self.cursor += len; + self.pos = self.pos.wrapping_add(len); + self.last_ch = rest[len - 1]; + } + } + + pub fn unwrap_next(&mut self) -> Result { + if let Some(token) = self.next() { + token + } else { + Err(CompileError { + line_num: self.line_num, + line_pos: self.pos.wrapping_sub(self.line_start), + error_type: ErrorType::UnexpectedEOF, + }) + } + } + + pub fn expect_token(&mut self, token: Token) -> Result<(), CompileError> { + let next_token = self.unwrap_next()?; + if next_token.token == token { + Ok(()) + } else { + Err(next_token.expected(format!("'{token}'"))) + } + } + + pub fn expect_static_string(&mut self) -> Result { + let next_token = self.unwrap_next()?; + match next_token.token { + Token::StringConstant(s) => Ok(s.into_string()), + Token::BracketOpen => { + let mut string = None; + loop { + let token_info = self.unwrap_next()?; + match token_info.token { + Token::StringConstant(string_) => { + string = string_.into(); + } + Token::BracketClose if string.is_some() => break, + _ => return Err(token_info.expected("constant string")), + } + } + Ok(string.unwrap().into_string()) + } + _ => Err(next_token.expected("constant string")), + } + } + + pub fn expect_number(&mut self, max_value: usize) -> Result { + let next_token = self.unwrap_next()?; + if let Token::Number(n) = next_token.token { + if n < max_value { + Ok(n) + } else { + Err(next_token.expected(format!("number lower than {max_value}"))) + } + } else { + Err(next_token.expected("number")) + } + } + + pub fn invalid_character(&self) -> CompileError { + CompileError { + line_num: self.line_num, + line_pos: self.pos.wrapping_sub(self.line_start), + error_type: ErrorType::InvalidCharacter(self.last_ch), + } + } + + pub fn peek(&mut self) -> Option> { + if self.next_token.is_empty() { + match self.next()? { + Ok(next_token) => self.next_token.push(next_token), + Err(err) => return Some(Err(err)), + } + } + self.next_token.last().map(Ok) + } +} + +impl Iterator for Tokenizer<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + if let Some(prev_token) = self.next_token.pop() { + return Some(Ok(prev_token)); + } + + 'outer: while let Some((ch, last_ch)) = self.next_byte() { + match self.state { + State::None => match ch { + b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'$' => { + self.push_byte(ch); + self.skip_word_chunk(); + } + b'A'..=b'Z' => { + self.push_byte(ch.to_ascii_lowercase()); + } + b':' => { + if self.is_token_start() + && matches!(self.peek_byte(), Some(b) if b.is_ascii_alphabetic()) + { + self.token_is_tag(); + } else if self.token_bytes().eq_ignore_ascii_case(b"text") { + self.state = State::MultiLine(StringType::default()); + self.text_start(); + while let Some((ch, _)) = self.next_byte() { + if ch == b'\n' { + self.new_line(); + self.reset_current_token(); + continue 'outer; + } + } + } else { + return Some(Ok(self.get_token(Token::Colon))); + //return Some(Err(self.invalid_character())); + } + } + b'"' => { + self.state = State::QuotedString(StringType::default()); + self.text_start(); + if let Some(token) = self.get_current_token() { + return Some(Ok(token)); + } + } + b'{' => { + return Some(Ok(self.get_token(Token::CurlyOpen))); + } + b'}' => { + return Some(Ok(self.get_token(Token::CurlyClose))); + } + b';' => { + return Some(Ok(self.get_token(Token::Semicolon))); + } + b',' => { + return Some(Ok(self.get_token(Token::Comma))); + } + b'[' => { + return Some(Ok(self.get_token(Token::BracketOpen))); + } + b']' => { + return Some(Ok(self.get_token(Token::BracketClose))); + } + b'(' => { + return Some(Ok(self.get_token(Token::ParenthesisOpen))); + } + b')' => { + return Some(Ok(self.get_token(Token::ParenthesisClose))); + } + b'/' => { + if let Some((b'*', _)) = self.next_byte() { + self.last_ch = 0; + self.state = State::BracketComment; + self.text_start(); + if let Some(token) = self.get_current_token() { + return Some(Ok(token)); + } + } else { + return Some(Err(self.invalid_character())); + } + } + b'#' => { + self.state = State::HashComment; + if let Some(token) = self.get_current_token() { + return Some(Ok(token)); + } + } + b'\n' => { + self.new_line(); + if let Some(token) = self.get_current_token() { + return Some(Ok(token)); + } + } + b' ' | b'\t' | b'\r' => { + self.skip_blank_chunk(); + if let Some(token) = self.get_current_token() { + return Some(Ok(token)); + } + } + _ => { + return Some(Err(self.invalid_character())); + } + }, + State::BracketComment => match ch { + b'/' if last_ch == b'*' => { + self.state = State::None; + } + b'\n' => { + self.new_line(); + } + _ => (), + }, + State::HashComment => { + if ch == b'\n' { + self.state = State::None; + self.new_line(); + } else { + self.skip_comment_chunk(); + } + } + State::QuotedString(str_type) => match ch { + b'"' if last_ch != b'\\' => { + self.state = State::None; + return Some(self.get_string(str_type)); + } + b'\n' => { + self.new_line(); + self.push_byte(b'\n'); + self.skip_string_chunk(); + } + b'\\' => { + if last_ch == b'\\' { + self.push_byte(ch); + self.skip_string_chunk(); + } + } + _ => { + let ch = if last_ch == b'\\' { + match ch { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + _ => ch, + } + } else { + ch + }; + self.push_byte(ch); + self.skip_string_chunk(); + } + }, + State::MultiLine(mut str_type) => match ch { + b'.' if last_ch == b'\n' => { + let is_eof = match (self.next_byte(), self.peek_byte()) { + (Some((b'\r', _)), Some(b'\n')) => { + self.next_byte(); + true + } + (Some((b'\n', _)), _) => true, + (Some((b'.', _)), _) => { + self.push_byte(b'.'); + false + } + (Some((ch, _)), _) => { + self.push_byte(b'.'); + self.push_byte(ch); + false + } + _ => false, + }; + + if is_eof { + self.new_line(); + self.state = State::None; + return Some(self.get_string(str_type)); + } + } + b'\n' => { + self.new_line(); + self.push_byte(b'\n'); + } + b'{' if (last_ch == b'$' || last_ch == b'%') => { + str_type.maybe_variable = true; + self.state = State::MultiLine(str_type); + self.push_byte(ch); + } + b'0'..=b'9' => { + if !str_type.has_digits { + str_type.has_digits = true; + self.state = State::MultiLine(str_type); + } + self.push_byte(ch); + } + b'.' => { + if !str_type.has_dots { + str_type.has_dots = true; + } else { + str_type.has_other = true; + } + self.state = State::MultiLine(str_type); + self.push_byte(ch); + } + _ => { + if !str_type.has_other && ch != b'-' { + str_type.has_other = true; + self.state = State::MultiLine(str_type); + } + self.push_byte(ch); + } + }, + } + } + + match self.state { + State::BracketComment | State::QuotedString(_) | State::MultiLine(_) => { + Some(Err(CompileError { + line_num: self.text_line_num, + line_pos: self.text_line_pos, + error_type: (&self.state).into(), + })) + } + _ => None, + } + } +} + +impl From<&State> for ErrorType { + fn from(state: &State) -> Self { + match state { + State::BracketComment => ErrorType::UnterminatedComment, + State::QuotedString(_) => ErrorType::UnterminatedString, + State::MultiLine(_) => ErrorType::UnterminatedMultiline, + _ => unreachable!(), + } + } +} diff --git a/crates/sieve/src/compiler/lexer/word.rs b/crates/sieve/src/compiler/lexer/word.rs new file mode 100644 index 00000000..7aa2f2e0 --- /dev/null +++ b/crates/sieve/src/compiler/lexer/word.rs @@ -0,0 +1,411 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Display; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub(crate) enum Word { + AddFlag, + AddHeader, + Address, + Addresses, + All, + AllOf, + AnyChild, + AnyOf, + Body, + Break, + ByMode, + ByTimeAbsolute, + ByTimeRelative, + ByTrace, + Comparator, + Contains, + Content, + ContentType, + Convert, + Copy, + Count, + Create, + CurrentDate, + Date, + Days, + DeleteHeader, + Detail, + Discard, + Domain, + Duplicate, + Else, + ElsIf, + Enclose, + EncodeUrl, + Envelope, + Environment, + Ereject, + Error, + Exists, + ExtractText, + False, + Fcc, + FileInto, + First, + Flags, + ForEveryPart, + From, + Global, + Handle, + HasFlag, + Header, + Headers, + If, + Ihave, + Importance, + Include, + Index, + Is, + Keep, + Last, + Length, + List, + LocalPart, + Lower, + LowerFirst, + MailboxExists, + MailboxId, + MailboxIdExists, + Matches, + Message, + Metadata, + MetadataExists, + Mime, + Name, + Not, + Notify, + NotifyMethodCapability, + Once, + Optional, + Options, + OriginalZone, + Over, + Param, + Percent, + Personal, + QuoteRegex, + QuoteWildcard, + Raw, + Redirect, + Regex, + Reject, + RemoveFlag, + Replace, + Require, + Ret, + Return, + Seconds, + ServerMetadata, + ServerMetadataExists, + Set, + SetFlag, + Size, + SpamTest, + SpecialUse, + SpecialUseExists, + Stop, + String, + Subject, + Subtype, + Text, + True, + Type, + Under, + UniqueId, + Upper, + UpperFirst, + User, + Vacation, + ValidExtList, + ValidNotifyMethod, + Value, + VirusTest, + Zone, + + // Extensions + Eval, + Local, + While, + Let, + Continue, +} + +pub(crate) fn lookup_words(input: &str) -> Option { + hashify::map!( + input.as_bytes(), Word, + "addflag" => Word::AddFlag, + "addheader" => Word::AddHeader, + "address" => Word::Address, + "addresses" => Word::Addresses, + "all" => Word::All, + "allof" => Word::AllOf, + "anychild" => Word::AnyChild, + "anyof" => Word::AnyOf, + "body" => Word::Body, + "break" => Word::Break, + "bymode" => Word::ByMode, + "bytimeabsolute" => Word::ByTimeAbsolute, + "bytimerelative" => Word::ByTimeRelative, + "bytrace" => Word::ByTrace, + "comparator" => Word::Comparator, + "contains" => Word::Contains, + "content" => Word::Content, + "contenttype" => Word::ContentType, + "convert" => Word::Convert, + "copy" => Word::Copy, + "count" => Word::Count, + "create" => Word::Create, + "currentdate" => Word::CurrentDate, + "date" => Word::Date, + "days" => Word::Days, + "deleteheader" => Word::DeleteHeader, + "detail" => Word::Detail, + "discard" => Word::Discard, + "domain" => Word::Domain, + "duplicate" => Word::Duplicate, + "else" => Word::Else, + "elsif" => Word::ElsIf, + "enclose" => Word::Enclose, + "encodeurl" => Word::EncodeUrl, + "envelope" => Word::Envelope, + "environment" => Word::Environment, + "ereject" => Word::Ereject, + "error" => Word::Error, + "exists" => Word::Exists, + "extracttext" => Word::ExtractText, + "false" => Word::False, + "fcc" => Word::Fcc, + "fileinto" => Word::FileInto, + "first" => Word::First, + "flags" => Word::Flags, + "foreverypart" => Word::ForEveryPart, + "from" => Word::From, + "global" => Word::Global, + "handle" => Word::Handle, + "hasflag" => Word::HasFlag, + "header" => Word::Header, + "headers" => Word::Headers, + "if" => Word::If, + "ihave" => Word::Ihave, + "importance" => Word::Importance, + "include" => Word::Include, + "index" => Word::Index, + "is" => Word::Is, + "keep" => Word::Keep, + "last" => Word::Last, + "length" => Word::Length, + "list" => Word::List, + "localpart" => Word::LocalPart, + "lower" => Word::Lower, + "lowerfirst" => Word::LowerFirst, + "mailboxexists" => Word::MailboxExists, + "mailboxid" => Word::MailboxId, + "mailboxidexists" => Word::MailboxIdExists, + "matches" => Word::Matches, + "message" => Word::Message, + "metadata" => Word::Metadata, + "metadataexists" => Word::MetadataExists, + "mime" => Word::Mime, + "name" => Word::Name, + "not" => Word::Not, + "notify" => Word::Notify, + "notify_method_capability" => Word::NotifyMethodCapability, + "once" => Word::Once, + "optional" => Word::Optional, + "options" => Word::Options, + "originalzone" => Word::OriginalZone, + "over" => Word::Over, + "param" => Word::Param, + "percent" => Word::Percent, + "personal" => Word::Personal, + "quoteregex" => Word::QuoteRegex, + "quotewildcard" => Word::QuoteWildcard, + "raw" => Word::Raw, + "redirect" => Word::Redirect, + "regex" => Word::Regex, + "reject" => Word::Reject, + "removeflag" => Word::RemoveFlag, + "replace" => Word::Replace, + "require" => Word::Require, + "ret" => Word::Ret, + "return" => Word::Return, + "seconds" => Word::Seconds, + "servermetadata" => Word::ServerMetadata, + "servermetadataexists" => Word::ServerMetadataExists, + "set" => Word::Set, + "setflag" => Word::SetFlag, + "size" => Word::Size, + "spamtest" => Word::SpamTest, + "specialuse" => Word::SpecialUse, + "specialuse_exists" => Word::SpecialUseExists, + "stop" => Word::Stop, + "string" => Word::String, + "subject" => Word::Subject, + "subtype" => Word::Subtype, + "text" => Word::Text, + "true" => Word::True, + "type" => Word::Type, + "under" => Word::Under, + "uniqueid" => Word::UniqueId, + "upper" => Word::Upper, + "upperfirst" => Word::UpperFirst, + "user" => Word::User, + "vacation" => Word::Vacation, + "valid_ext_list" => Word::ValidExtList, + "valid_notify_method" => Word::ValidNotifyMethod, + "value" => Word::Value, + "virustest" => Word::VirusTest, + "zone" => Word::Zone, + "eval" => Word::Eval, + "local" => Word::Local, + "while" => Word::While, + "let" => Word::Let, + "continue" => Word::Continue, + ) + .copied() +} + +impl Display for Word { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Word::AddFlag => f.write_str("addflag"), + Word::AddHeader => f.write_str("addheader"), + Word::Address => f.write_str("address"), + Word::Addresses => f.write_str("addresses"), + Word::All => f.write_str("all"), + Word::AllOf => f.write_str("allof"), + Word::AnyChild => f.write_str("anychild"), + Word::AnyOf => f.write_str("anyof"), + Word::Body => f.write_str("body"), + Word::Break => f.write_str("break"), + Word::ByMode => f.write_str("bymode"), + Word::ByTimeAbsolute => f.write_str("bytimeabsolute"), + Word::ByTimeRelative => f.write_str("bytimerelative"), + Word::ByTrace => f.write_str("bytrace"), + Word::Comparator => f.write_str("comparator"), + Word::Contains => f.write_str("contains"), + Word::Content => f.write_str("content"), + Word::ContentType => f.write_str("contenttype"), + Word::Convert => f.write_str("convert"), + Word::Copy => f.write_str("copy"), + Word::Count => f.write_str("count"), + Word::Create => f.write_str("create"), + Word::CurrentDate => f.write_str("currentdate"), + Word::Date => f.write_str("date"), + Word::Days => f.write_str("days"), + Word::DeleteHeader => f.write_str("deleteheader"), + Word::Detail => f.write_str("detail"), + Word::Discard => f.write_str("discard"), + Word::Domain => f.write_str("domain"), + Word::Duplicate => f.write_str("duplicate"), + Word::Else => f.write_str("else"), + Word::ElsIf => f.write_str("elsif"), + Word::Enclose => f.write_str("enclose"), + Word::EncodeUrl => f.write_str("encodeurl"), + Word::Envelope => f.write_str("envelope"), + Word::Environment => f.write_str("environment"), + Word::Ereject => f.write_str("ereject"), + Word::Error => f.write_str("error"), + Word::Exists => f.write_str("exists"), + Word::ExtractText => f.write_str("extracttext"), + Word::False => f.write_str("false"), + Word::Fcc => f.write_str("fcc"), + Word::FileInto => f.write_str("fileinto"), + Word::First => f.write_str("first"), + Word::Flags => f.write_str("flags"), + Word::ForEveryPart => f.write_str("foreverypart"), + Word::From => f.write_str("from"), + Word::Global => f.write_str("global"), + Word::Handle => f.write_str("handle"), + Word::HasFlag => f.write_str("hasflag"), + Word::Header => f.write_str("header"), + Word::Headers => f.write_str("headers"), + Word::If => f.write_str("if"), + Word::Ihave => f.write_str("ihave"), + Word::Importance => f.write_str("importance"), + Word::Include => f.write_str("include"), + Word::Index => f.write_str("index"), + Word::Is => f.write_str("is"), + Word::Keep => f.write_str("keep"), + Word::Last => f.write_str("last"), + Word::Length => f.write_str("length"), + Word::List => f.write_str("list"), + Word::LocalPart => f.write_str("localpart"), + Word::Lower => f.write_str("lower"), + Word::LowerFirst => f.write_str("lowerfirst"), + Word::MailboxExists => f.write_str("mailboxexists"), + Word::MailboxId => f.write_str("mailboxid"), + Word::MailboxIdExists => f.write_str("mailboxidexists"), + Word::Matches => f.write_str("matches"), + Word::Message => f.write_str("message"), + Word::Metadata => f.write_str("metadata"), + Word::MetadataExists => f.write_str("metadataexists"), + Word::Mime => f.write_str("mime"), + Word::Name => f.write_str("name"), + Word::Not => f.write_str("not"), + Word::Notify => f.write_str("notify"), + Word::NotifyMethodCapability => f.write_str("notify_method_capability"), + Word::Once => f.write_str("once"), + Word::Optional => f.write_str("optional"), + Word::Options => f.write_str("options"), + Word::OriginalZone => f.write_str("originalzone"), + Word::Over => f.write_str("over"), + Word::Param => f.write_str("param"), + Word::Percent => f.write_str("percent"), + Word::Personal => f.write_str("personal"), + Word::QuoteRegex => f.write_str("quoteregex"), + Word::QuoteWildcard => f.write_str("quotewildcard"), + Word::Raw => f.write_str("raw"), + Word::Redirect => f.write_str("redirect"), + Word::Regex => f.write_str("regex"), + Word::Reject => f.write_str("reject"), + Word::RemoveFlag => f.write_str("removeflag"), + Word::Replace => f.write_str("replace"), + Word::Require => f.write_str("require"), + Word::Ret => f.write_str("ret"), + Word::Return => f.write_str("return"), + Word::Seconds => f.write_str("seconds"), + Word::ServerMetadata => f.write_str("servermetadata"), + Word::ServerMetadataExists => f.write_str("servermetadataexists"), + Word::Set => f.write_str("set"), + Word::SetFlag => f.write_str("setflag"), + Word::Size => f.write_str("size"), + Word::SpamTest => f.write_str("spamtest"), + Word::SpecialUse => f.write_str("specialuse"), + Word::SpecialUseExists => f.write_str("specialuse_exists"), + Word::Stop => f.write_str("stop"), + Word::String => f.write_str("string"), + Word::Subject => f.write_str("subject"), + Word::Subtype => f.write_str("subtype"), + Word::Text => f.write_str("text"), + Word::True => f.write_str("true"), + Word::Type => f.write_str("type"), + Word::Under => f.write_str("under"), + Word::UniqueId => f.write_str("uniqueid"), + Word::Upper => f.write_str("upper"), + Word::UpperFirst => f.write_str("upperfirst"), + Word::User => f.write_str("user"), + Word::Vacation => f.write_str("vacation"), + Word::ValidExtList => f.write_str("valid_ext_list"), + Word::ValidNotifyMethod => f.write_str("valid_notify_method"), + Word::Value => f.write_str("value"), + Word::VirusTest => f.write_str("virustest"), + Word::Zone => f.write_str("zone"), + Word::Eval => f.write_str("eval"), + Word::Local => f.write_str("local"), + Word::While => f.write_str("while"), + Word::Let => f.write_str("let"), + Word::Continue => f.write_str("continue"), + } + } +} diff --git a/crates/sieve/src/compiler/mod.rs b/crates/sieve/src/compiler/mod.rs new file mode 100644 index 00000000..47c65d4a --- /dev/null +++ b/crates/sieve/src/compiler/mod.rs @@ -0,0 +1,689 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use self::{ + grammar::{AddressPart, Capability}, + lexer::{StringConstant, tokenizer::TokenInfo}, +}; +use crate::{ + Compiler, Envelope, FunctionMap, + runtime::{RuntimeError, tests::glob::CompiledGlob}, +}; +use ahash::{AHashMap, AHashSet}; +use mail_parser::HeaderName; +use std::{borrow::Cow, fmt::Display}; + +pub(crate) mod emit; +pub mod grammar; +pub mod lexer; + +#[derive(Debug)] +pub struct CompileError { + line_num: usize, + line_pos: usize, + error_type: ErrorType, +} + +#[derive(Debug)] +pub enum ErrorType { + InvalidCharacter(u8), + InvalidNumber(String), + InvalidMatchVariable(usize), + InvalidUnicodeSequence(u32), + InvalidNamespace(String), + InvalidRegex(String), + InvalidExpression(String), + InvalidUtf8String, + InvalidHeaderName, + InvalidArguments, + InvalidAddress, + InvalidURI, + InvalidEnvelope(String), + UnterminatedString, + UnterminatedComment, + UnterminatedMultiline, + UnterminatedBlock, + ScriptTooLong, + StringTooLong, + VariableTooLong, + VariableIsLocal(String), + HeaderTooLong, + ExpectedConstantString, + UnexpectedToken { + expected: Cow<'static, str>, + found: String, + }, + UnexpectedEOF, + TooManyNestedBlocks, + TooManyNestedTests, + TooManyNestedForEveryParts, + TooManyIncludes, + LabelAlreadyDefined(String), + LabelUndefined(String), + BreakOutsideLoop, + ContinueOutsideLoop, + UnsupportedComparator(String), + DuplicatedParameter, + UndeclaredCapability(Capability), + UnsupportedCapability(Capability), + MissingTag(Cow<'static, str>), +} + +impl Default for Compiler { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub(crate) enum Value { + Text(ConstantId) = 0, + Number(Number) = 1, + Variable(VariableType) = 2, + Regex(Regex) = 3, + Glob(Glob) = 4, + Header(HeaderName<'static>) = 5, + List(Box<[Value]>) = 6, +} + +#[derive(Debug)] +pub(crate) enum RawValue { + Text(String), + Number(Number), + Value(Value), +} + +impl From for RawValue { + fn from(value: StringConstant) -> Self { + match value { + StringConstant::String(text) => RawValue::Text(text), + StringConstant::Number(number) => RawValue::Number(number), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(transparent)] +pub struct ConstantId(u32); + +impl ConstantId { + pub(crate) const UNRESOLVED: ConstantId = ConstantId(u32::MAX); + + #[inline(always)] + pub(crate) fn new(index: usize) -> Self { + ConstantId(index as u32) + } + + #[inline(always)] + pub(crate) fn index(&self) -> usize { + self.0 as usize + } +} + +#[derive(Debug, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Regex { + pub expr: String, +} + +#[derive(Debug, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Glob { + #[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(skip, default) + )] + pub(crate) glob: CompiledGlob, + pub expr: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum VariableType { + Local(u16) = 0, + Match(u8) = 1, + Global(String) = 2, + Environment(String) = 3, + Envelope(Envelope) = 4, + Header(HeaderVariable<'static>) = 5, + Part(MessagePart) = 6, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Transform { + pub variable: Box, + pub functions: Box<[u32]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct HeaderVariable<'x> { + #[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(deserialize_with = "deserialize_header_names") + )] + pub name: Box<[HeaderName<'x>]>, + pub part: HeaderPart, + pub index_hdr: i32, + pub index_part: i32, +} + +#[cfg(any(all(test, stalwart_tests), feature = "serde"))] +fn deserialize_header_names<'de, 'x, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result]>, D::Error> { + > as serde::Deserialize>::deserialize(deserializer) + .map(Vec::into_boxed_slice) +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum MessagePart { + TextBody(bool) = 0, + HtmlBody(bool) = 1, + Contents = 2, + Raw = 3, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum HeaderPart { + Text = 0, + Date = 1, + Id = 2, + Address(AddressPart) = 3, + ContentType(ContentTypePart) = 4, + Received(ReceivedPart) = 5, + Raw = 6, + RawName = 7, + Exists = 8, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum ContentTypePart { + Type = 0, + Subtype = 1, + Attribute(String) = 2, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum ReceivedPart { + From(ReceivedHostname) = 0, + FromIp = 1, + FromIpRev = 2, + By(ReceivedHostname) = 3, + For = 4, + With = 5, + TlsVersion = 6, + TlsCipher = 7, + Id = 8, + Ident = 9, + Via = 10, + Date = 11, + DateRaw = 12, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum ReceivedHostname { + Name = 0, + Ip = 1, + Any = 2, +} + +#[derive(Debug, Clone, Copy)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Number { + Integer(i64) = 0, + Float(f64) = 1, +} + +impl ReceivedPart { + pub(crate) fn code(&self) -> u16 { + let (part, hostname): (u16, u16) = match self { + ReceivedPart::From(host) => (0, *host as u16), + ReceivedPart::FromIp => (1, 0), + ReceivedPart::FromIpRev => (2, 0), + ReceivedPart::By(host) => (3, *host as u16), + ReceivedPart::For => (4, 0), + ReceivedPart::With => (5, 0), + ReceivedPart::TlsVersion => (6, 0), + ReceivedPart::TlsCipher => (7, 0), + ReceivedPart::Id => (8, 0), + ReceivedPart::Ident => (9, 0), + ReceivedPart::Via => (10, 0), + ReceivedPart::Date => (11, 0), + ReceivedPart::DateRaw => (12, 0), + }; + part | (hostname << 8) + } + + pub(crate) fn from_code(part: u8, hostname: u8) -> ReceivedPart { + let hostname = match hostname { + 0 => ReceivedHostname::Name, + 1 => ReceivedHostname::Ip, + _ => ReceivedHostname::Any, + }; + match part { + 0 => ReceivedPart::From(hostname), + 1 => ReceivedPart::FromIp, + 2 => ReceivedPart::FromIpRev, + 3 => ReceivedPart::By(hostname), + 4 => ReceivedPart::For, + 5 => ReceivedPart::With, + 6 => ReceivedPart::TlsVersion, + 7 => ReceivedPart::TlsCipher, + 8 => ReceivedPart::Id, + 9 => ReceivedPart::Ident, + 10 => ReceivedPart::Via, + 11 => ReceivedPart::Date, + _ => ReceivedPart::DateRaw, + } + } +} + +impl Number { + #[cfg(all(test, stalwart_tests))] + pub fn to_float(&self) -> f64 { + match self { + Number::Integer(i) => *i as f64, + Number::Float(fl) => *fl, + } + } +} + +impl From for usize { + fn from(value: Number) -> Self { + match value { + Number::Integer(i) => i as usize, + Number::Float(fl) => fl as usize, + } + } +} + +impl Display for Number { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Number::Integer(i) => i.fmt(f), + Number::Float(fl) => fl.fmt(f), + } + } +} + +impl Compiler { + pub const VERSION: u32 = crate::bytecode::FORMAT_VERSION as u32; + + pub fn new() -> Self { + Compiler { + max_script_size: 1024 * 1024, + max_string_size: 4096, + max_variable_name_size: 32, + max_nested_blocks: 15, + max_nested_tests: 15, + max_nested_foreverypart: 3, + max_match_variables: 30, + max_local_variables: 128, + max_header_size: 1024, + max_includes: 6, + functions: AHashMap::new(), + no_capability_check: false, + disabled_capabilities: AHashSet::new(), + } + } + + pub fn set_max_header_size(&mut self, size: usize) { + self.max_header_size = size; + } + + pub fn with_max_header_size(mut self, size: usize) -> Self { + self.max_header_size = size; + self + } + + pub fn set_max_includes(&mut self, size: usize) { + self.max_includes = size; + } + + pub fn with_max_includes(mut self, size: usize) -> Self { + self.max_includes = size; + self + } + + pub fn set_max_nested_blocks(&mut self, size: usize) { + self.max_nested_blocks = size; + } + + pub fn with_max_nested_blocks(mut self, size: usize) -> Self { + self.max_nested_blocks = size; + self + } + + pub fn set_max_nested_tests(&mut self, size: usize) { + self.max_nested_tests = size; + } + + pub fn with_max_nested_tests(mut self, size: usize) -> Self { + self.max_nested_tests = size; + self + } + + pub fn set_max_nested_foreverypart(&mut self, size: usize) { + self.max_nested_foreverypart = size; + } + + pub fn with_max_nested_foreverypart(mut self, size: usize) -> Self { + self.max_nested_foreverypart = size; + self + } + + pub fn set_max_script_size(&mut self, size: usize) { + self.max_script_size = size; + } + + pub fn with_max_script_size(mut self, size: usize) -> Self { + self.max_script_size = size; + self + } + + pub fn set_max_string_size(&mut self, size: usize) { + self.max_string_size = size; + } + + pub fn with_max_string_size(mut self, size: usize) -> Self { + self.max_string_size = size; + self + } + + pub fn set_max_variable_name_size(&mut self, size: usize) { + self.max_variable_name_size = size; + } + + pub fn with_max_variable_name_size(mut self, size: usize) -> Self { + self.max_variable_name_size = size; + self + } + + pub fn set_max_match_variables(&mut self, size: usize) { + self.max_match_variables = size; + } + + pub fn with_max_match_variables(mut self, size: usize) -> Self { + self.max_match_variables = size; + self + } + + pub fn set_max_local_variables(&mut self, size: usize) { + self.max_local_variables = size; + } + + pub fn with_max_local_variables(mut self, size: usize) -> Self { + self.max_local_variables = size; + self + } + + pub fn register_functions(mut self, fnc_map: &mut FunctionMap) -> Self { + self.functions = std::mem::take(&mut fnc_map.map); + self + } + + pub fn with_no_capability_check(mut self, value: bool) -> Self { + self.no_capability_check = value; + self + } + + pub fn set_no_capability_check(&mut self, value: bool) { + self.no_capability_check = value; + } + + pub fn unset_capability(&mut self, capability: impl Into) { + self.disabled_capabilities.insert(capability.into()); + } + + pub fn without_capability(mut self, capability: impl Into) -> Self { + self.unset_capability(capability); + self + } + + pub fn without_capabilities( + mut self, + capabilities: impl IntoIterator>, + ) -> Self { + for capability in capabilities { + self.unset_capability(capability); + } + self + } +} + +impl CompileError { + pub fn line_num(&self) -> usize { + self.line_num + } + + pub fn line_pos(&self) -> usize { + self.line_pos + } + + pub fn error_type(&self) -> &ErrorType { + &self.error_type + } +} + +impl PartialEq for Regex { + fn eq(&self, other: &Self) -> bool { + self.expr == other.expr + } +} + +impl Eq for Regex {} + +impl TokenInfo { + pub fn expected(self, expected: impl Into>) -> CompileError { + CompileError { + line_num: self.line_num, + line_pos: self.line_pos, + error_type: ErrorType::UnexpectedToken { + expected: expected.into(), + found: self.token.to_string(), + }, + } + } + + pub fn missing_tag(self, tag: impl Into>) -> CompileError { + CompileError { + line_num: self.line_num, + line_pos: self.line_pos, + error_type: ErrorType::MissingTag(tag.into()), + } + } + + pub fn custom(self, error_type: ErrorType) -> CompileError { + CompileError { + line_num: self.line_num, + line_pos: self.line_pos, + error_type, + } + } +} + +impl Glob { + pub fn new(expr: String, to_lower: bool) -> Self { + let glob = CompiledGlob::compile(&expr, to_lower); + Self { expr, glob } + } +} + +impl PartialEq for Glob { + fn eq(&self, other: &Self) -> bool { + self.expr == other.expr + } +} + +impl Eq for Glob {} + +impl Regex { + pub fn new(expr: String) -> Self { + Self { expr } + } +} + +impl Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.error_type() { + ErrorType::InvalidCharacter(value) => { + write!(f, "Invalid character {:?}", char::from(*value)) + } + ErrorType::InvalidNumber(value) => write!(f, "Invalid number {value:?}"), + ErrorType::InvalidMatchVariable(value) => { + write!(f, "Match variable {value} out of range") + } + ErrorType::InvalidUnicodeSequence(value) => { + write!(f, "Invalid Unicode sequence {value:04x}") + } + ErrorType::InvalidNamespace(value) => write!(f, "Invalid namespace {value:?}"), + ErrorType::InvalidRegex(value) => write!(f, "Invalid regular expression {value:?}"), + ErrorType::InvalidExpression(value) => write!(f, "Invalid expression {value}"), + ErrorType::InvalidUtf8String => write!(f, "Invalid UTF-8 string"), + ErrorType::InvalidHeaderName => write!(f, "Invalid header name"), + ErrorType::InvalidArguments => write!(f, "Invalid Arguments"), + ErrorType::InvalidAddress => write!(f, "Invalid Address"), + ErrorType::InvalidURI => write!(f, "Invalid URI"), + ErrorType::InvalidEnvelope(value) => write!(f, "Invalid envelope {value:?}"), + ErrorType::UnterminatedString => write!(f, "Unterminated string"), + ErrorType::UnterminatedComment => write!(f, "Unterminated comment"), + ErrorType::UnterminatedMultiline => write!(f, "Unterminated multi-line string"), + ErrorType::UnterminatedBlock => write!(f, "Unterminated block"), + ErrorType::ScriptTooLong => write!(f, "Sieve script is too large"), + ErrorType::StringTooLong => write!(f, "String is too long"), + ErrorType::VariableTooLong => write!(f, "Variable name is too long"), + ErrorType::VariableIsLocal(value) => { + write!(f, "Variable {value:?} was already defined as local") + } + ErrorType::HeaderTooLong => write!(f, "Header value is too long"), + ErrorType::ExpectedConstantString => write!(f, "Expected a constant string"), + ErrorType::UnexpectedToken { expected, found } => { + write!(f, "Expected token {expected:?} but found {found:?}") + } + ErrorType::UnexpectedEOF => write!(f, "Unexpected end of file"), + ErrorType::TooManyNestedBlocks => write!(f, "Too many nested blocks"), + ErrorType::TooManyNestedTests => write!(f, "Too many nested tests"), + ErrorType::TooManyNestedForEveryParts => { + write!(f, "Too many nested foreverypart blocks") + } + ErrorType::TooManyIncludes => write!(f, "Too many includes"), + ErrorType::LabelAlreadyDefined(value) => write!(f, "Label {value:?} already defined"), + ErrorType::LabelUndefined(value) => write!(f, "Label {value:?} does not exist"), + ErrorType::BreakOutsideLoop => write!(f, "Break used outside of foreverypart loop"), + ErrorType::ContinueOutsideLoop => write!(f, "Continue used outside of while loop"), + ErrorType::UnsupportedComparator(value) => { + write!(f, "Comparator {value:?} is not supported") + } + ErrorType::DuplicatedParameter => write!(f, "Duplicated argument"), + ErrorType::UndeclaredCapability(value) => { + write!(f, "Undeclared capability '{value}'") + } + ErrorType::UnsupportedCapability(value) => { + write!(f, "Capability '{value}' is not supported") + } + ErrorType::MissingTag(value) => write!(f, "Missing tag {value:?}"), + }?; + + write!( + f, + " at line {}, column {}.", + self.line_num(), + self.line_pos() + ) + } +} + +impl Display for RuntimeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RuntimeError::TooManyIncludes => f.write_str("Too many nested includes"), + RuntimeError::ScriptNotFound(name) => write!(f, "Included script {name:?} not found."), + RuntimeError::InvalidInstruction { + name, + line_num, + line_pos, + } => write!( + f, + "Script executed invalid instruction {name:?} at line {line_num}, column {line_pos}." + ), + RuntimeError::ScriptErrorMessage(value) => { + write!(f, "Script reported error {value:?}.") + } + RuntimeError::CapabilityNotAllowed(value) => { + write!(f, "Capability '{value}' has been disabled.") + } + RuntimeError::CapabilityNotSupported(value) => { + write!(f, "Capability '{value}' not supported.") + } + RuntimeError::CPULimitReached => write!( + f, + "Script exceeded the maximum number of instructions allowed to execute." + ), + RuntimeError::MemoryLimitReached => { + write!(f, "Script exceeded the maximum amount of memory allowed.") + } + RuntimeError::InvalidBytecode => write!(f, "Compiled script is corrupted."), + RuntimeError::AwaitingInput => f.write_str("Script is waiting for a pending result"), + } + } +} + diff --git a/crates/sieve/src/lib.rs b/crates/sieve/src/lib.rs new file mode 100644 index 00000000..8a388e22 --- /dev/null +++ b/crates/sieve/src/lib.rs @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::{AHashMap, AHashSet}; +use compiler::grammar::Capability; +use mail_parser::{HeaderName, Message, PartId}; +use runtime::{ + RuntimeError, Variable, + context::{Frame, Pending}, + handler::Action, + message::{body::BodyCache, edits::Edits, parts::PartCursor}, +}; +use std::{ + borrow::Cow, + cell::{Cell, RefCell}, +}; + +pub mod bytecode; +pub mod compiler; +mod regex; +pub mod runtime; +pub mod sieve; + +pub use runtime::{ + Action as SieveAction, Arena, Handler, Input, Mailbox, MessageSource, Recipient, Reply, Script, + Status, +}; +pub use sieve::{LoadError, ScriptArena, Sieve}; + +pub(crate) const MAX_MATCH_VARIABLES: u32 = 63; + +#[derive(Clone)] +pub struct Compiler { + pub(crate) max_script_size: usize, + pub(crate) max_string_size: usize, + pub(crate) max_variable_name_size: usize, + pub(crate) max_nested_blocks: usize, + pub(crate) max_nested_tests: usize, + pub(crate) max_nested_foreverypart: usize, + pub(crate) max_match_variables: usize, + pub(crate) max_local_variables: usize, + pub(crate) max_header_size: usize, + pub(crate) max_includes: usize, + pub(crate) no_capability_check: bool, + pub(crate) disabled_capabilities: AHashSet, + pub(crate) functions: AHashMap, +} + +pub type Function = for<'x> fn(&Context<'x>, &[Variable<'x>]) -> Variable<'x>; + +#[derive(Default, Clone)] +pub struct FunctionMap { + pub(crate) map: AHashMap, + pub(crate) functions: Vec, +} + +#[derive(Debug, Clone)] +pub struct Runtime { + pub(crate) allowed_capabilities: AHashSet, + pub(crate) valid_notification_uris: AHashSet>, + pub(crate) valid_ext_lists: AHashSet>, + pub(crate) protected_headers: Vec>, + pub(crate) environment: AHashMap, Variable<'static>>, + pub(crate) metadata: Vec<(Metadata, Cow<'static, str>)>, + pub(crate) include_scripts: AHashMap>, + pub(crate) local_hostname: Cow<'static, str>, + pub(crate) functions: Vec, + + pub(crate) max_nested_includes: usize, + pub(crate) cpu_limit: usize, + pub(crate) memory_limit: usize, + pub(crate) max_variable_size: usize, + pub(crate) max_redirects: usize, + pub(crate) max_received_headers: usize, + pub(crate) max_header_size: usize, + pub(crate) max_header_count: usize, + pub(crate) max_header_block_size: usize, + pub(crate) max_out_messages: usize, + + pub(crate) default_vacation_expiry: u64, + pub(crate) default_duplicate_expiry: u64, + + pub(crate) vacation_use_orig_rcpt: bool, + pub(crate) vacation_default_subject: Cow<'static, str>, + pub(crate) vacation_subject_prefix: Cow<'static, str>, +} + +pub struct Context<'x> { + pub(crate) runtime: &'x Runtime, + pub(crate) user_address: Cow<'x, str>, + pub(crate) user_full_name: Cow<'x, str>, + pub(crate) current_time: i64, + + pub(crate) message: &'x Message<'x>, + pub(crate) message_size: usize, + pub(crate) edits: Edits<'x>, + pub(crate) body_cache: RefCell>, + pub(crate) envelope: Vec<(Envelope, Variable<'x>)>, + pub(crate) metadata: Vec<(Metadata, Cow<'x, str>)>, + + pub(crate) part: PartId, + pub(crate) part_iter: PartCursor, + pub(crate) part_iter_stack: Vec<(PartId, PartCursor)>, + + pub(crate) spam_status: SpamStatus, + pub(crate) virus_status: VirusStatus, + + pub(crate) script: &'x Sieve<'x>, + pub(crate) frames: Vec>, + pub(crate) local_base: usize, + pub(crate) match_base: usize, + pub(crate) pos: usize, + pub(crate) test_result: bool, + pub(crate) pending: Pending<'x>, + pub(crate) error: Option, + pub(crate) rejected: bool, + pub(crate) included: Vec>, + pub(crate) vars_global: AHashMap, Variable<'x>>, + pub(crate) vars_env: AHashMap, Variable<'x>>, + pub(crate) vars_local: Vec>, + pub(crate) vars_match: Vec>, + pub(crate) expr_stack: Vec>, + pub(crate) expr_pos: usize, + + pub(crate) flags: Vec<&'x str>, + pub(crate) actions: Vec>, + pub(crate) final_action: Option>, + pub(crate) last_message_id: usize, + pub(crate) main_message_id: usize, + + pub(crate) has_changes: bool, + pub(crate) oom: Cell, + pub(crate) dynamic_regexes: RefCell>>, + pub(crate) num_redirects: usize, + pub(crate) num_instructions: usize, + pub(crate) num_scanned_bytes: usize, + pub(crate) num_out_messages: usize, + + pub(crate) arena: &'x mut Arena, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Envelope { + From = 0, + To = 1, + ByTimeAbsolute = 2, + ByTimeRelative = 3, + ByMode = 4, + ByTrace = 5, + Notify = 6, + Orcpt = 7, + Ret = 8, + Envid = 9, +} + +impl Envelope { + #[inline(always)] + pub(crate) fn from_code(code: u8) -> Envelope { + match code { + 0 => Envelope::From, + 1 => Envelope::To, + 2 => Envelope::ByTimeAbsolute, + 3 => Envelope::ByTimeRelative, + 4 => Envelope::ByMode, + 5 => Envelope::ByTrace, + 6 => Envelope::Notify, + 7 => Envelope::Orcpt, + 8 => Envelope::Ret, + _ => Envelope::Envid, + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[repr(u8)] +pub enum Metadata { + Server { annotation: T } = 0, + Mailbox { name: T, annotation: T } = 1, +} + +pub type ExternalId = u32; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub(crate) struct FileCarbonCopy { + pub mailbox: T, + pub mailbox_id: Option, + pub create: bool, + pub flags: Box<[T]>, + pub special_use: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum Importance { + High, + Normal, + Low, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum MatchAs { + Octet, + Lowercase, + Number, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpamStatus { + Unknown, + Ham, + MaybeSpam(f64), + Spam, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum VirusStatus { + Unknown, + Clean, + Replaced, + Cured, + MaybeVirus, + Virus, +} + diff --git a/crates/sieve/src/regex.rs b/crates/sieve/src/regex.rs new file mode 100644 index 00000000..b3a538a2 --- /dev/null +++ b/crates/sieve/src/regex.rs @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use fancy_regex::{Error, Regex, RegexBuilder}; + +const SIZE_LIMIT: usize = 1 << 20; +const BACKTRACK_LIMIT: usize = 1_000_000; + +pub(crate) fn build(pattern: &str) -> Result { + RegexBuilder::new(pattern) + .backtrack_limit(BACKTRACK_LIMIT) + .delegate_size_limit(SIZE_LIMIT) + .delegate_dfa_size_limit(SIZE_LIMIT) + .build() +} + +pub(crate) fn compile(pattern: &str) -> Option { + build(pattern).ok() +} diff --git a/crates/sieve/src/runtime/actions/action_convert.rs b/crates/sieve/src/runtime/actions/action_convert.rs new file mode 100644 index 00000000..2f92fec3 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_convert.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::super::tests::TestResult; +use crate::{ + Context, Sieve, + bytecode::ops, + runtime::{ + RuntimeError, + message::{ + body::TextForm, + edits::{BodyEdit, BodyFormat, BodyKind, TransferEncoding}, + headers::HeaderRef, + parts::{PartCursor, ROOT_PART, Scope}, + }, + }, +}; +use mail_parser::{HeaderName, PartKind}; + +#[derive(Clone, Copy)] +enum Conversion { + TextToHtml, + TextPlainToHtml, + HtmlToText, +} + +impl<'x> Context<'x> { + pub(crate) fn exec_convert( + &mut self, + script: &'x Sieve<'x>, + convert: &ops::Convert, + ) -> Result { + let from_media_type = self.eval_value(script, convert.from_media_type)?; + let to_media_type = self.eval_value(script, convert.to_media_type)?; + + let from_media_type = from_media_type.to_string(); + let to_media_type = to_media_type.to_string(); + + if from_media_type.eq_ignore_ascii_case(to_media_type.as_ref()) { + return Ok(TestResult::Bool(false ^ convert.is_not)); + } + + let conversion = if (from_media_type.eq_ignore_ascii_case("text") + || from_media_type.starts_with("text/")) + && to_media_type.eq_ignore_ascii_case("text/html") + { + if from_media_type.eq_ignore_ascii_case("text") { + Conversion::TextPlainToHtml + } else { + Conversion::TextToHtml + } + } else if from_media_type.eq_ignore_ascii_case("text/html") + && to_media_type.eq_ignore_ascii_case("text/plain") + { + Conversion::HtmlToText + } else { + return Ok(TestResult::Bool(false ^ convert.is_not)); + }; + let mut did_convert = false; + let mut parts = match self.root_part() { + Some(root) => PartCursor::subtree(root, true, Scope::Message), + None => PartCursor::default(), + }; + while let Some(part) = self.advance(&mut parts) { + let (text, kind, content_type) = match (self.part_kind(part), conversion) { + (PartKind::Html, Conversion::HtmlToText) => ( + self.part_text(part, TextForm::Plain), + BodyKind::Text, + "text/plain; charset=utf-8", + ), + (PartKind::Text, Conversion::TextToHtml) => ( + self.part_text(part, TextForm::Html), + BodyKind::Html, + "text/html; charset=utf-8", + ), + (PartKind::Text, Conversion::TextPlainToHtml) + if self + .part_content_type(part) + .is_some_and(|(_, subtype)| subtype.eq_ignore_ascii_case("plain")) => + { + ( + self.part_text(part, TextForm::Html), + BodyKind::Html, + "text/html; charset=utf-8", + ) + } + _ => continue, + }; + let Some(text) = text else { + continue; + }; + let part_id = part.id(); + let (encoding, encoded_len) = TransferEncoding::select(text); + let previous_size = match self.edits.body(part_id) { + Some(edited) => edited.encoded_len, + None => part.raw().len(), + }; + + let mut headers = Vec::new(); + if part_id == ROOT_PART { + let mut add_mime_version = true; + let previous = self.part_headers(part); + headers.reserve(previous.size_hint().0 + 3); + headers.extend(previous.filter(|header| { + let name = header.name(); + add_mime_version &= name != HeaderName::MimeVersion; + !name.is_content_header() + })); + if add_mime_version { + headers.push(self.add_header(HeaderName::MimeVersion, "1.0")); + } + } + headers.push(self.add_header(HeaderName::ContentType, content_type)); + if let Some(cte) = encoding.header_value() { + headers.push(self.add_header(HeaderName::ContentTransferEncoding, cte)); + } + + let headers_size = headers.iter().map(HeaderRef::len).sum::(); + self.message_size = + (self.message_size + headers_size + encoded_len + 2).saturating_sub(previous_size); + self.edits.set_headers(part_id, headers); + self.set_part_body( + part_id, + BodyEdit { + text, + kind, + format: BodyFormat::Content, + encoding, + encoded_len, + }, + ); + did_convert = true; + } + + if did_convert { + self.has_changes = true; + } + + Ok(TestResult::Bool(did_convert ^ convert.is_not)) + } +} diff --git a/crates/sieve/src/runtime/actions/action_editheader.rs b/crates/sieve/src/runtime/actions/action_editheader.rs new file mode 100644 index 00000000..02cbeb5f --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_editheader.rs @@ -0,0 +1,172 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::grammar::Comparator, + runtime::{RuntimeError, tests::test_header::MimeOptsRef}, +}; +use mail_parser::{HeaderName, PartId}; + +impl<'x> Context<'x> { + pub(crate) fn exec_addheader( + &mut self, + script: &'x Sieve<'x>, + add: &ops::AddHeader, + ) -> Result<(), RuntimeError> { + let header_name_ = self.eval_value(script, add.field_name)?; + let header_name_ = header_name_.to_string(); + let mut header_name = String::with_capacity(header_name_.len()); + + for ch in header_name_.chars() { + if ch.is_alphanumeric() || ch == '-' { + header_name.push(ch); + } + } + + if !header_name.is_empty() + && let Some(header_name) = HeaderName::parse(self.alloc_string(header_name)) + && !self.runtime.protected_headers.contains(&header_name) + { + let header_value = self.alloc_string( + self.eval_value(script, add.value)? + .to_string() + .as_ref() + .remove_crlf(self.runtime.max_header_size), + ); + self.has_changes = true; + self.insert_header(self.part, header_name, header_value, add.last); + } + + Ok(()) + } + + pub(crate) fn exec_deleteheader( + &mut self, + script: &'x Sieve<'x>, + delete: &ops::DeleteHeader, + ) -> Result<(), RuntimeError> { + let header_name_ = self.eval_value(script, delete.field_name)?; + let header_name_ = header_name_.to_string(); + let Some(header_name) = HeaderName::parse(header_name_.as_ref()) else { + return Ok(()); + }; + let value_patterns = self.eval_keys(script, delete.value_patterns)?; + let comparator = Comparator::from_code(delete.comparator); + let match_type = delete.match_type.match_type(); + let mut deleted_headers = Vec::new(); + let mut deleted_bytes = 0; + let mut error = None; + + if self.runtime.protected_headers.contains(&header_name) { + return Ok(()); + } + + let mut meter = self.scan_meter(value_patterns.len()); + self.find_headers( + &[header_name], + delete.index, + delete.mime_anychild, + &mut meter, + |header, part_id| { + if !value_patterns.is_empty() { + let did_match = self.find_header_values(header, &MimeOptsRef::None, |value| { + for key in &value_patterns { + match self.key_matches( + script, + &comparator, + &match_type, + key, + value, + &mut Vec::new(), + ) { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + false + }); + + if error.is_some() { + return true; + } + if !did_match { + return false; + } + } + + deleted_bytes += header.len(); + deleted_headers.push((part_id, header)); + + false + }, + ); + self.settle_scan(meter)?; + + if let Some(err) = error { + return Err(err); + } + + if !deleted_headers.is_empty() { + self.has_changes = true; + for (part_id, deleted) in deleted_headers { + if let Some(part) = self.message.part(part_id) { + self.edits + .headers_mut(part) + .retain(|header| !header.same(&deleted)); + } + } + } + + self.message_size = self.message_size.saturating_sub(deleted_bytes); + Ok(()) + } + + pub(crate) fn insert_header( + &mut self, + part_id: PartId, + header_name: HeaderName<'x>, + header_value: &'x str, + last: bool, + ) { + let Some(part) = self.message.part(part_id) else { + return; + }; + let header = self.add_header(header_name, header_value); + self.message_size += header.len(); + let headers = self.edits.headers_mut(part); + if last { + headers.push(header); + } else { + headers.insert(0, header); + } + } +} + +pub(crate) trait RemoveCrLf { + fn remove_crlf(&self, max_len: usize) -> String; +} + +impl RemoveCrLf for &str { + fn remove_crlf(&self, max_len: usize) -> String { + let mut header_value = String::with_capacity(self.len()); + for ch in self.chars() { + if !['\n', '\r'].contains(&ch) { + if header_value.len() + ch.len_utf8() <= max_len { + header_value.push(ch); + } else { + return header_value; + } + } + } + header_value + } +} diff --git a/crates/sieve/src/runtime/actions/action_fileinto.rs b/crates/sieve/src/runtime/actions/action_fileinto.rs new file mode 100644 index 00000000..b133c0c8 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_fileinto.rs @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::ops, + runtime::{RuntimeError, handler::Action}, +}; + +impl<'x> Context<'x> { + pub(crate) fn exec_fileinto( + &mut self, + script: &'x Sieve<'x>, + fileinto: &ops::FileInto, + ) -> Result<(), RuntimeError> { + let folder = self.eval_str(script, fileinto.folder)?; + if let Some(created) = self.build_message_id() { + self.actions.push(created); + } + + if !fileinto.copy + && !matches!(&self.final_action, Some(Action::Keep { flags, .. }) if !flags.is_empty()) + { + self.final_action = None; + } + + let action = Action::FileInto { + folder, + flags: self.get_local_or_global_flags(script, fileinto.flags)?, + mailbox_id: self.eval_opt_str(script, fileinto.mailbox_id)?, + special_use: self.eval_opt_str(script, fileinto.special_use)?, + create: fileinto.create, + message_id: self.main_message_id, + }; + self.actions.push(action); + Ok(()) + } +} diff --git a/crates/sieve/src/runtime/actions/action_flags.rs b/crates/sieve/src/runtime/actions/action_flags.rs new file mode 100644 index 00000000..c3033521 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_flags.rs @@ -0,0 +1,191 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::{ + ops, + rec::{Range, tag}, + }, + runtime::{RuntimeError, eval::ValueRef}, +}; + +const ACTION_SET: u8 = 0; +const ACTION_ADD: u8 = 1; + +impl<'x> Context<'x> { + pub(crate) fn exec_editflags( + &mut self, + script: &'x Sieve<'x>, + edit: &ops::EditFlags, + ) -> Result<(), RuntimeError> { + if edit.name.tag == tag::NONE || edit.name.tag == tag::VARIABLE_NONE { + self.exec_editflags_implicit(script, edit) + } else { + self.exec_editflags_variable(script, edit) + } + } + + fn exec_editflags_implicit( + &mut self, + script: &'x Sieve<'x>, + edit: &ops::EditFlags, + ) -> Result<(), RuntimeError> { + let mut flags = std::mem::take(&mut self.flags); + + match edit.action { + ACTION_SET | ACTION_ADD => { + if edit.action == ACTION_SET { + flags.clear(); + } + self.tokenize_flags(script, edit.flags, |flag| { + for flag in flag.split_ascii_whitespace() { + if !flags.iter().any(|f| f.eq_ignore_ascii_case(flag)) { + flags.push(flag); + } + } + false + })?; + } + _ => { + self.tokenize_flags(script, edit.flags, |flag| { + for flag in flag.split_ascii_whitespace() { + if let Some(pos) = flags.iter().position(|f| f.eq_ignore_ascii_case(flag)) { + flags.swap_remove(pos); + } + } + false + })?; + } + } + + self.flags = flags; + Ok(()) + } + + fn exec_editflags_variable( + &mut self, + script: &'x Sieve<'x>, + edit: &ops::EditFlags, + ) -> Result<(), RuntimeError> { + let value = match edit.action { + ACTION_SET => { + let mut flags = String::new(); + self.tokenize_flags(script, edit.flags, |flag| { + if !contains_flag(&flags, flag) { + if !flags.is_empty() { + flags.push(' '); + } + flags.push_str(flag); + } + false + })?; + flags + } + ACTION_ADD => { + let mut new_flags = self + .get_variable(script, edit.name)? + .map(|v| v.to_string().into_owned()) + .unwrap_or_default(); + + self.tokenize_flags(script, edit.flags, |flag| { + if !contains_flag(&new_flags, flag) { + if !new_flags.is_empty() { + new_flags.push(' '); + } + new_flags.push_str(flag); + } + false + })?; + new_flags + } + _ => { + let flags = self + .get_variable(script, edit.name)? + .map(|v| v.to_string().into_owned()) + .unwrap_or_default(); + let mut current_flags: Vec<&str> = flags.split(' ').collect(); + self.tokenize_flags(script, edit.flags, |flag| { + if let Some(pos) = current_flags + .iter() + .position(|lflag| lflag.eq_ignore_ascii_case(flag)) + { + current_flags.swap_remove(pos); + } + false + })?; + current_flags.join(" ") + } + }; + self.set_variable(script, edit.name, value.into()) + } + + pub(crate) fn tokenize_flags( + &self, + script: &'x Sieve<'x>, + strings: Range, + mut cb: impl FnMut(&'x str) -> bool, + ) -> Result { + let mut iter = script.recs(strings)?; + let mut pos = 0; + while let Some(rec) = iter.next() { + let value = ValueRef::decode(script, rec, &mut iter)?; + let is_single = pos == 0 && iter.len() == 0; + let flag = self.eval_value_ref(script, value)?; + let flag = self.intern_cow(flag.into_string()); + if !flag.is_empty() { + if is_single { + for flag in flag.split_ascii_whitespace() { + if !flag.is_empty() && cb(flag) { + return Ok(true); + } + } + } else if cb(flag.trim()) { + return Ok(true); + } + } + pos += 1; + } + Ok(false) + } + + pub(crate) fn get_local_flags( + &self, + script: &'x Sieve<'x>, + strings: Range, + ) -> Result<&'x [&'x str], RuntimeError> { + let mut flags: smallvec::SmallVec<[&'x str; 8]> = smallvec::SmallVec::new(); + self.tokenize_flags(script, strings, |flag| { + flags.push(flag); + false + })?; + Ok(self.alloc_strs(&flags)) + } + + pub(crate) fn get_global_flags(&self) -> &'x [&'x str] { + self.alloc_strs(&self.flags) + } + + pub(crate) fn global_flags(&self) -> &[&'x str] { + &self.flags + } + + pub(crate) fn get_local_or_global_flags( + &self, + script: &'x Sieve<'x>, + strings: Range, + ) -> Result<&'x [&'x str], RuntimeError> { + if strings.is_empty() { + Ok(self.get_global_flags()) + } else { + self.get_local_flags(script, strings) + } + } +} + +fn contains_flag(flags: &str, flag: &str) -> bool { + flags.split(' ').any(|kept| kept.eq_ignore_ascii_case(flag)) +} diff --git a/crates/sieve/src/runtime/actions/action_keep.rs b/crates/sieve/src/runtime/actions/action_keep.rs new file mode 100644 index 00000000..6e7fe7e9 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_keep.rs @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::ops, + runtime::{RuntimeError, handler::Action}, +}; + +impl<'x> Context<'x> { + pub(crate) fn exec_keep( + &mut self, + script: &'x Sieve<'x>, + keep: &ops::Keep, + ) -> Result<(), RuntimeError> { + let created = self.build_message_id(); + let flags = self.get_local_or_global_flags(script, keep.flags)?; + self.final_action = Some(Action::Keep { + flags, + message_id: self.main_message_id, + }); + if let Some(created) = created { + self.actions.push(created); + } + Ok(()) + } +} diff --git a/crates/sieve/src/runtime/actions/action_mime.rs b/crates/sieve/src/runtime/actions/action_mime.rs new file mode 100644 index 00000000..e548509e --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_mime.rs @@ -0,0 +1,355 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::action_editheader::RemoveCrLf; +use crate::{ + Context, Sieve, + bytecode::{ops, rec::tag}, + runtime::{ + RuntimeError, Variable, + eval::ValueRef, + handler::Action, + message::{ + body::TextForm, + edits::{BodyEdit, BodyFormat, BodyKind, Edits, TransferEncoding}, + headers::{HeaderRef, ORIGINAL_FROM, ORIGINAL_SUBJECT}, + parts::{PartCursor, ROOT_PART, subtree_end}, + }, + }, +}; +use mail_parser::{HeaderName, MessageParser, PartKind}; +use std::fmt::Write; + +#[cfg(not(all(test, stalwart_tests)))] +use crate::runtime::platform; +#[cfg(not(all(test, stalwart_tests)))] +use mail_builder::headers::date::Date; + +impl<'x> Context<'x> { + pub(crate) fn exec_replace( + &mut self, + script: &'x Sieve<'x>, + replace: &ops::Replace, + ) -> Result<(), RuntimeError> { + let Some(part) = self.current_part() else { + return Ok(()); + }; + let part_id = part.id(); + self.edits.hide(part_id + 1..subtree_end(part)); + self.has_changes = true; + + let body = self.eval_value(script, replace.replacement)?.into_string(); + let body = self.intern_cow(body); + let has_subject = replace.subject.tag != tag::NONE; + let has_from = replace.from.tag != tag::NONE; + + let (format, encoding, encoded_len) = if replace.mime { + TransferEncoding::select_entity(body) + } else { + let (encoding, encoded_len) = TransferEncoding::select(body); + (BodyFormat::Content, encoding, encoded_len) + }; + let previous_size = match self.edits.body(part_id) { + Some(edited) => edited.encoded_len, + None => part.raw().len(), + }; + self.message_size = (self.message_size + encoded_len).saturating_sub(previous_size); + + let mut headers = Vec::new(); + if part_id == ROOT_PART { + let mut add_date = true; + let mut add_message_id = true; + let mut add_mime_version = true; + let mut has_original_from = false; + let mut kept_size = 0; + let previous = self.part_headers(part); + headers.reserve(previous.size_hint().0 + 7); + + for header in previous { + let header = match header.name() { + HeaderName::Subject if has_subject => { + self.renamed_header(header, &ORIGINAL_SUBJECT) + } + HeaderName::From if has_from => self.renamed_header(header, &ORIGINAL_FROM), + HeaderName::From => { + has_original_from = true; + header + } + HeaderName::Date => { + add_date = false; + header + } + HeaderName::MessageId => { + add_message_id = false; + header + } + HeaderName::MimeVersion => { + add_mime_version = false; + header + } + name if name.is_content_header() => continue, + _ => header, + }; + kept_size += header.len(); + headers.push(header); + } + self.message_size += kept_size; + + let mut add_from = true; + if let Some(from) = self.eval_opt(script, replace.from)? + && !from.is_empty() + { + let from = self.header_value(from.to_string().as_ref()); + headers.push(self.new_header(HeaderName::From, from)); + add_from = false; + } + if add_from && !has_original_from { + let from = self.alloc_string(self.user_from_field()); + headers.push(self.new_header(HeaderName::From, from)); + } + + if let Some(subject) = self.eval_opt(script, replace.subject)? + && !subject.is_empty() + { + let subject = self.header_value(subject.to_string().as_ref()); + headers.push(self.new_header(HeaderName::Subject, subject)); + } + + if add_date { + let date = self.alloc_string(self.current_date()); + headers.push(self.new_header(HeaderName::Date, date)); + } + + if add_message_id { + let message_id = self.alloc_string(self.generate_message_id()); + headers.push(self.new_header(HeaderName::MessageId, message_id)); + } + + if add_mime_version { + headers.push(self.new_header(HeaderName::MimeVersion, "1.0")); + } + } + + if !replace.mime { + headers.push(self.new_header(HeaderName::ContentType, "text/plain; charset=utf-8")); + } + if let Some(cte) = encoding.header_value() { + headers.push(self.new_header(HeaderName::ContentTransferEncoding, cte)); + } + + self.edits.set_headers(part_id, headers); + self.set_part_body( + part_id, + BodyEdit { + text: body, + kind: BodyKind::Text, + format, + encoding, + encoded_len, + }, + ); + + Ok(()) + } + + pub(crate) fn exec_enclose( + &mut self, + script: &'x Sieve<'x>, + enclose: &ops::Enclose, + ) -> Result<(), RuntimeError> { + let body = self.eval_value(script, enclose.value)?.into_string(); + let subject = match self.eval_opt(script, enclose.subject)? { + Some(subject) => subject + .to_string() + .as_ref() + .remove_crlf(self.runtime.max_header_size), + None => self + .root_text(&HeaderName::Subject) + .unwrap_or_default() + .to_string(), + }; + + #[cfg(all(test, stalwart_tests))] + let boundary = make_test_boundary(); + #[cfg(not(all(test, stalwart_tests)))] + let boundary = platform::make_boundary(); + + let enclosed = self.build_message(); + let mut message = String::with_capacity(512); + let _ = write!( + message, + "Content-Type: multipart/mixed; boundary=\"{boundary}\"\r\nSubject: {subject}\r\n" + ); + + let mut add_date = true; + let mut add_message_id = true; + let mut add_from = true; + + let mut iter = script.recs(enclose.headers)?; + while let Some(rec) = iter.next() { + let header = ValueRef::decode(script, rec, &mut iter)?; + let header = self.eval_value_ref(script, header)?; + if let Some((mut header_name, mut header_value)) = + header.to_string().as_ref().split_once(':') + { + header_name = header_name.trim(); + header_value = header_value.trim(); + if !header_value.is_empty() + && let Some(name) = HeaderName::parse(header_name) + && !self.runtime.protected_headers.contains(&name) + { + match &name { + HeaderName::Date => add_date = false, + HeaderName::From => add_from = false, + HeaderName::MessageId => add_message_id = false, + _ => (), + } + let header_value = header_value.remove_crlf(self.runtime.max_header_size); + let _ = write!(message, "{header_name}: {header_value}\r\n"); + } + } + } + + if add_from { + let _ = write!(message, "From: {}\r\n", self.user_from_field()); + } + if add_date { + let _ = write!(message, "Date: {}\r\n", self.current_date()); + } + if add_message_id { + let _ = write!(message, "Message-ID: {}\r\n", self.generate_message_id()); + } + let _ = write!( + message, + "\r\n--{boundary}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n{body}\r\n--{boundary}\r\nContent-Type: message/rfc822\r\n\r\n" + ); + + let mut message = message.into_bytes(); + message.reserve(enclosed.len() + boundary.len() + 8); + message.extend_from_slice(&enclosed); + message.extend_from_slice(b"\r\n--"); + message.extend_from_slice(boundary.as_bytes()); + message.extend_from_slice(b"--\r\n"); + + if !self.arena.charge(message.len()) { + return Err(RuntimeError::MemoryLimitReached); + } + let message = MessageParser::new() + .parse_owned(message) + .unwrap_or_default(); + self.message_size = message.raw().len(); + self.message = self.arena.keep_message(message); + self.edits = Edits::default(); + self.reset_body_cache(); + self.part = ROOT_PART; + self.part_iter = PartCursor::default(); + for (part, parts) in &mut self.part_iter_stack { + *part = ROOT_PART; + *parts = PartCursor::default(); + } + self.has_changes = true; + + Ok(()) + } + + pub(crate) fn exec_extracttext( + &mut self, + script: &'x Sieve<'x>, + extract: &ops::ExtractText, + ) -> Result<(), RuntimeError> { + let mut value = ""; + + if !self.part_iter_stack.is_empty() + && let Some(part) = self.current_part() + { + let text = match self.part_kind(part) { + PartKind::Text => self.part_text(part, TextForm::Source), + PartKind::Html => self.part_text(part, TextForm::Plain), + _ => None, + }; + if let Some(text) = text { + value = match extract.first { + Some(first) => text + .char_indices() + .nth(first as usize) + .and_then(|(end, _)| text.get(..end)) + .unwrap_or(text), + None => text, + }; + } + + if !extract.modifiers.is_empty() && !value.is_empty() { + let modified = + self.apply_modifiers(script, extract.modifiers, Variable::borrowed(value))?; + return self.assign_variable(script, extract.name, modified); + } + } + + self.assign_variable(script, extract.name, Variable::borrowed(value)) + } + + fn header_value(&self, value: &str) -> &'x str { + self.alloc_string(value.remove_crlf(self.runtime.max_header_size)) + } + + fn new_header(&mut self, name: HeaderName<'x>, value: &'x str) -> HeaderRef<'x> { + let header = self.add_header(name, value); + self.message_size += header.len(); + header + } + + fn current_date(&self) -> String { + #[cfg(not(all(test, stalwart_tests)))] + { + Date::new(self.current_time).to_rfc822() + } + #[cfg(all(test, stalwart_tests))] + { + "Tue, 20 Nov 2022 05:14:20 -0300".to_string() + } + } + + fn generate_message_id(&self) -> String { + #[cfg(not(all(test, stalwart_tests)))] + { + let mut header_value = Vec::with_capacity(self.runtime.local_hostname.len() + 64); + platform::write_message_id(&mut header_value, &self.runtime.local_hostname); + String::from_utf8(header_value).unwrap_or_default() + } + #[cfg(all(test, stalwart_tests))] + { + "".to_string() + } + } + + pub(crate) fn build_message_id(&mut self) -> Option> { + if self.has_changes { + self.last_message_id += 1; + self.main_message_id = self.last_message_id; + self.has_changes = false; + let message = self.build_message(); + Some(Action::CreatedMessage { + message_id: self.main_message_id, + message, + }) + } else { + None + } + } +} + +#[cfg(all(test, stalwart_tests))] +thread_local!(static COUNTER: std::cell::Cell = 0.into()); + +#[cfg(all(test, stalwart_tests))] +pub(crate) fn make_test_boundary() -> String { + format!("boundary_{}", COUNTER.with(|c| { c.replace(c.get() + 1) })) +} + +#[cfg(all(test, stalwart_tests))] +pub(crate) fn reset_test_boundary() { + COUNTER.with(|c| c.replace(0)); +} diff --git a/crates/sieve/src/runtime/actions/action_notify.rs b/crates/sieve/src/runtime/actions/action_notify.rs new file mode 100644 index 00000000..61301dec --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_notify.rs @@ -0,0 +1,600 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::action_vacation::MAX_SUBJECT_LEN; +use crate::runtime::platform; +use crate::{ + Context, Importance, Sieve, + bytecode::ops, + compiler::grammar::actions::action_redirect::{ByTime, Notify, Ret}, + runtime::{ + RuntimeError, Variable, + handler::{Action, MessageSource, Recipient}, + }, +}; +use mail_builder::headers::date::Date; +use mail_parser::HeaderName; +use std::borrow::Cow; + +const DEFAULT_IMPORTANCE_HEADERS: (&str, &str) = ("Normal", "3 (Normal)"); + +impl<'x> Context<'x> { + pub(crate) fn exec_notify( + &mut self, + script: &'x Sieve<'x>, + notify: &ops::Notify, + ) -> Result<(), RuntimeError> { + for header in self.root_headers() { + if header.is_named(&HeaderName::AutoSubmitted) + && header + .value() + .as_text() + .is_none_or(|v| !v.eq_ignore_ascii_case("no")) + { + return Ok(()); + } + } + + let uri = self.eval_str(script, notify.method)?; + let Some((scheme, params)) = parse_uri(uri) else { + return Ok(()); + }; + + let has_fcc = notify.fcc.present; + let is_mailto = scheme.eq_ignore_ascii_case("mailto") + && self.num_out_messages < self.runtime.max_out_messages; + let from = self.eval_opt(script, notify.from)?; + let importance = self.eval_opt(script, notify.importance)?; + let notify_message = self + .eval_opt(script, notify.message)? + .map(|m| m.into_string()); + + if is_mailto || has_fcc { + let params = if is_mailto { + let Some(params) = parse_mailto(params) else { + return Ok(()); + }; + params + } else { + MailtoMessage::default() + }; + let from = self.notify_from(from.as_ref()); + let importance_headers = importance.as_ref().map_or(DEFAULT_IMPORTANCE_HEADERS, |i| { + lookup_importance_headers(i.to_string().as_ref()) + .unwrap_or(DEFAULT_IMPORTANCE_HEADERS) + }); + let message = self.build_notify_message( + ¶ms, + &from, + notify_message.as_deref(), + importance_headers, + ); + + self.last_message_id += 1; + self.actions.push(Action::CreatedMessage { + message_id: self.last_message_id, + message, + }); + + if is_mailto { + let recipients = params + .to + .iter() + .chain(params.cc.iter()) + .chain(params.bcc.iter()) + .map(|addr| { + self.alloc_str( + addr.rsplit_once('<') + .and_then(|(_, addr)| addr.rsplit_once('>')) + .map_or(addr.as_str(), |(addr, _)| addr), + ) + }) + .collect(); + self.actions.push(Action::SendMessage { + source: MessageSource::Notification, + recipient: Recipient::Group(recipients), + notify: Notify::Never, + return_of_content: Ret::Default, + by_time: ByTime::None, + message_id: self.last_message_id, + }); + } + } + + if !is_mailto { + let options = self.eval_strings(script, notify.options)?; + let action = Action::Notify { + from: from.map(|f| self.intern_cow(f.into_string())), + importance: importance.map_or(Importance::Normal, |i| { + lookup_importance(i.to_string().as_ref()).unwrap_or(Importance::Normal) + }), + options: self.alloc_strs(&options), + message: match notify_message { + Some(message) => self.intern_cow(message), + None => self.subject_str().unwrap_or_default(), + }, + method: uri, + }; + self.actions.push(action); + self.num_out_messages += 1; + } + + if has_fcc { + let fcc = ¬ify.fcc; + let action = Action::FileInto { + folder: self.eval_str(script, fcc.mailbox)?, + flags: self.get_local_flags(script, fcc.flags)?, + mailbox_id: self.eval_opt_str(script, fcc.mailbox_id)?, + special_use: self.eval_opt_str(script, fcc.special_use)?, + create: fcc.create, + message_id: self.last_message_id, + }; + self.actions.push(action); + } + Ok(()) + } + + fn subject_str(&self) -> Option<&'x str> { + self.root_text(&HeaderName::Subject) + } + + fn notify_from<'a>(&self, from: Option<&'a Variable<'x>>) -> Cow<'a, str> { + if let Some(from) = from { + let from = from.to_string(); + if from + .to_ascii_lowercase() + .contains(&self.user_address.to_ascii_lowercase()) + { + return from; + } + } + self.user_from_field().into() + } + + fn build_notify_message( + &self, + params: &MailtoMessage, + from: &str, + notify_message: Option<&str>, + (importance, priority): (&str, &str), + ) -> Vec { + let message_len = params + .to + .iter() + .chain(params.cc.iter()) + .map(|a| a.len() + 4) + .sum::() + + params + .headers + .iter() + .map(|(h, v)| h.as_str().len() + v.len() + 4) + .sum::() + + params.body.as_ref().map_or(0, |b| b.len()) + + notify_message.map_or(0, |b| b.len()) + + from.len() + + 200; + + let mut message = Vec::with_capacity(message_len); + message.extend_from_slice(b"From: "); + message.extend_from_slice(from.as_bytes()); + message.extend_from_slice(b"\r\n"); + + for (header, addresses) in [("To: ", ¶ms.to), ("Cc: ", ¶ms.cc)] { + if !addresses.is_empty() { + message.extend_from_slice(header.as_bytes()); + for (pos, address) in addresses.iter().enumerate() { + if pos > 0 { + message.extend_from_slice(b", "); + } + if !address.contains('<') { + message.push(b'<'); + } + message.extend_from_slice(address.as_bytes()); + if !address.contains('<') { + message.push(b'>'); + } + } + message.extend_from_slice(b"\r\n"); + } + } + + let mut has_subject = None; + let mut has_date = false; + let mut has_message_id = false; + for (header, value) in ¶ms.headers { + match header { + HeaderName::Subject => { + has_subject = value.into(); + continue; + } + HeaderName::Date => { + has_date = true; + } + HeaderName::MessageId => { + has_message_id = true; + } + HeaderName::From => { + continue; + } + _ => (), + } + message.extend_from_slice(header.as_str().as_bytes()); + message.extend_from_slice(b": "); + message.extend_from_slice(value.as_bytes()); + message.extend_from_slice(b"\r\n"); + } + + if !has_date { + message.extend_from_slice(b"Date: "); + message.extend_from_slice(Date::new(self.current_time).to_rfc822().as_bytes()); + message.extend_from_slice(b"\r\n"); + } + + if !has_message_id { + message.extend_from_slice(b"Message-ID: "); + platform::write_message_id(&mut message, &self.runtime.local_hostname); + message.extend_from_slice(b"\r\n"); + } + + message.extend_from_slice(b"Importance: "); + message.extend_from_slice(importance.as_bytes()); + message.extend_from_slice(b"\r\n"); + + message.extend_from_slice(b"X-Priority: "); + message.extend_from_slice(priority.as_bytes()); + message.extend_from_slice(b"\r\n"); + + message.extend_from_slice(b"Subject: "); + let subject = if let Some(subject) = has_subject { + subject.as_str() + } else if let Some(subject) = notify_message { + subject + } else { + self.message.subject().unwrap_or_default() + }; + let mut iter = subject.chars().enumerate(); + let mut buf = [0; 4]; + #[allow(clippy::while_let_on_iterator)] + while let Some((pos, char)) = iter.next() { + if pos < MAX_SUBJECT_LEN { + message.extend_from_slice(char.encode_utf8(&mut buf).as_bytes()); + } else { + break; + } + } + if iter.next().is_some() { + message.extend_from_slice('…'.encode_utf8(&mut buf).as_bytes()); + } + message.extend_from_slice(b"\r\n"); + + message.extend_from_slice(b"Auto-Submitted: auto-notified\r\n"); + message.extend_from_slice(b"X-Sieve: yes\r\n"); + message.extend_from_slice(b"Content-type: text/plain; charset=utf-8\r\n\r\n"); + if let Some(body) = ¶ms.body { + message.extend_from_slice(body.as_bytes()); + } else if let Some(subject) = notify_message { + message.extend_from_slice(subject.as_bytes()); + } else if let Some(subject) = self.message.subject() { + message.extend_from_slice(subject.as_bytes()); + } + message + } +} + +pub fn validate_from(addr: &str) -> bool { + let mut has_at = false; + let mut has_dot = false; + let mut in_quote = false; + let mut in_angle = false; + let mut last_ch = 0; + + for &ch in addr.as_bytes().iter() { + match ch { + b'\"' if last_ch != b'\\' => { + in_quote = !in_quote; + } + b'<' if !in_quote => { + if !in_angle { + in_angle = true; + has_at = false; + has_dot = false; + } else { + return false; + } + } + b'>' if !in_quote => { + if in_angle { + in_angle = false; + } else { + return false; + } + } + b'@' if !in_quote => { + if !has_at && last_ch.is_ascii_alphanumeric() { + has_at = true; + } else { + return false; + } + } + b'.' if !in_quote && has_at => { + has_dot = true; + } + _ => (), + } + last_ch = ch; + } + + has_dot && has_at && !in_angle +} + +pub fn validate_uri(uri: &str) -> Option<&str> { + let (scheme, uri) = parse_uri(uri)?; + if scheme.eq_ignore_ascii_case("mailto") { + parse_mailto(uri)?; + scheme.into() + } else if ["xmpp", "tel", "http", "https"].contains(&scheme) { + scheme.into() + } else { + None + } +} + +pub(crate) fn parse_uri(uri: &str) -> Option<(&str, &str)> { + let (scheme, uri) = uri.split_once(':')?; + + if !uri.is_empty() { + Some((scheme, uri)) + } else { + None + } +} + +pub enum Mailto { + Header(HeaderName<'static>), + Body, + Other(String), +} + +enum State { + Address((HeaderName<'static>, bool)), + ParamName, + ParamValue(Mailto), +} + +#[derive(Default)] +struct MailtoMessage { + to: Vec, + cc: Vec, + bcc: Vec, + body: Option, + headers: Vec<(HeaderName<'static>, String)>, +} + +fn parse_mailto(uri: &str) -> Option { + let mut params = MailtoMessage::default(); + + let mut state = State::Address((HeaderName::To, false)); + let mut buf = Vec::new(); + let uri_ = uri.as_bytes(); + let mut iter = uri_.iter(); + let mut has_addresses = false; + + while let Some(&ch) = iter.next() { + match ch { + b'%' => { + let hex1 = hex_value(*iter.next()?); + let hex2 = hex_value(*iter.next()?); + if let (Some(hex1), Some(hex2)) = (hex1, hex2) { + let ch = (hex1 << 4) | hex2; + + match &state { + State::Address((header, has_at)) => match ch { + b',' => { + if *has_at { + insert_address( + &mut params, + header.clone(), + String::from_utf8(std::mem::take(&mut buf)).ok()?, + ); + has_addresses = true; + state = State::Address((header.clone(), false)); + } else { + return None; + } + } + b'@' => { + if !*has_at { + state = State::Address((header.clone(), true)); + buf.push(ch); + } else { + return None; + } + } + _ => { + buf.push(ch); + } + }, + _ => buf.push(ch), + } + } else { + return None; + } + } + b',' => match &state { + State::Address((header, true)) => { + insert_address( + &mut params, + header.clone(), + String::from_utf8(std::mem::take(&mut buf)).ok()?, + ); + state = State::Address((header.clone(), false)); + has_addresses = true; + } + State::ParamValue(_) => buf.push(ch), + _ => return None, + }, + b'?' => match &state { + State::Address((header, has_at)) if *has_at || buf.is_empty() => { + if !buf.is_empty() { + insert_address( + &mut params, + header.clone(), + String::from_utf8(std::mem::take(&mut buf)).ok()?, + ); + has_addresses = true; + } + state = State::ParamName; + } + State::ParamValue(_) => buf.push(ch), + _ => return None, + }, + b'@' => match &state { + State::Address((header, false)) if !buf.is_empty() => { + buf.push(ch); + state = State::Address((header.clone(), true)); + } + State::ParamName | State::ParamValue(_) => buf.push(ch), + _ => return None, + }, + b'=' => match &state { + State::ParamName if !buf.is_empty() => { + let param = String::from_utf8(std::mem::take(&mut buf)).ok()?; + state = HeaderName::parse(param) + .map(|hdr| match hdr { + HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { + State::Address((hdr, false)) + } + HeaderName::Other(param) => { + if param.eq_ignore_ascii_case("body") { + State::ParamValue(Mailto::Body) + } else { + State::ParamValue(Mailto::Other(param.into_owned())) + } + } + _ => State::ParamValue(Mailto::Header(hdr)), + }) + .unwrap_or_else(|| State::ParamValue(Mailto::Other(String::new()))); + } + State::ParamValue(_) => buf.push(ch), + _ => return None, + }, + b'&' => match state { + State::Address((header, true)) => { + if !buf.is_empty() { + insert_address( + &mut params, + header, + String::from_utf8(std::mem::take(&mut buf)).ok()?, + ); + } + state = State::ParamName; + } + State::ParamValue(param) => { + if !buf.is_empty() { + let value = String::from_utf8(std::mem::take(&mut buf)).ok()?; + match param { + Mailto::Header(header) => params.headers.push((header, value)), + Mailto::Body => params.body = value.into(), + Mailto::Other(header) => params.headers.push((header.into(), value)), + } + } + state = State::ParamName; + } + _ => return None, + }, + _ => match &state { + State::ParamName => { + if ch.is_ascii_alphanumeric() || b"-_".contains(&ch) { + buf.push(ch); + } else { + return None; + } + } + _ => { + if !ch.is_ascii_whitespace() { + buf.push(ch); + } + } + }, + } + } + + if !buf.is_empty() { + let value = String::from_utf8(std::mem::take(&mut buf)).ok()?; + match state { + State::Address((header, true)) => { + insert_address(&mut params, header, value); + has_addresses = true; + } + State::ParamName => { + params + .headers + .push((HeaderName::Other(value.into()), String::new())); + } + State::ParamValue(param) => match param { + Mailto::Header(header) => params.headers.push((header, value)), + Mailto::Body => params.body = value.into(), + Mailto::Other(header) => params + .headers + .push((HeaderName::Other(header.into()), value)), + }, + _ => return None, + } + } + + if has_addresses { Some(params) } else { None } +} + +#[inline(always)] +fn insert_address(params: &mut MailtoMessage, name: HeaderName, value: String) { + if !params + .to + .iter() + .chain(params.cc.iter()) + .chain(params.bcc.iter()) + .any(|v| v.eq_ignore_ascii_case(&value)) + { + match name { + HeaderName::To => { + params.to.push(value); + } + HeaderName::Cc => { + params.cc.push(value); + } + HeaderName::Bcc => { + params.bcc.push(value); + } + _ => (), + } + } +} + +fn lookup_importance(input: &str) -> Option { + hashify::map!( + input.as_bytes(), Importance, + "1" => Importance::High, + "3" => Importance::Low, + ) + .copied() +} + +fn lookup_importance_headers(input: &str) -> Option<(&'static str, &'static str)> { + hashify::map!( + input.as_bytes(), (&'static str, &'static str), + "1" => ("High", "1 (High)"), + "3" => ("Low", "5 (Low)"), + ) + .copied() +} + +fn hex_value(ch: u8) -> Option { + match ch { + b'0'..=b'9' => Some(ch - b'0'), + b'a'..=b'f' => Some(ch - b'a' + 10), + b'A'..=b'F' => Some(ch - b'A' + 10), + _ => None, + } +} diff --git a/crates/sieve/src/runtime/actions/action_redirect.rs b/crates/sieve/src/runtime/actions/action_redirect.rs new file mode 100644 index 00000000..58c847ef --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_redirect.rs @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::{ + ops, + rec::{Range, tag}, + }, + compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, + runtime::{ + RuntimeError, + handler::{Action, MessageSource, Recipient}, + }, +}; +use mail_parser::{DateTime, HeaderName}; + +const NOTIFY_NEVER: u8 = 0; +const NOTIFY_ITEMS: u8 = 1; +const BY_TIME_RELATIVE: u8 = 0; +const BY_TIME_ABSOLUTE: u8 = 1; + +impl<'x> Context<'x> { + pub(crate) fn exec_redirect( + &mut self, + script: &'x Sieve<'x>, + redirect: &ops::Redirect, + ) -> Result<(), RuntimeError> { + let address = self.eval_value(script, redirect.address)?; + let Some(address) = sanitize_address(address.to_string().as_ref()) else { + return Ok(()); + }; + if self.num_redirects >= self.runtime.max_redirects + || self.num_out_messages >= self.runtime.max_out_messages + || self + .root_headers() + .filter(|h| h.is_named(&HeaderName::Received)) + .count() + >= self.runtime.max_received_headers + { + return Ok(()); + } + + if !redirect.list && address.eq_ignore_ascii_case(self.user_address.as_ref()) { + return Ok(()); + } + + if !redirect.copy && matches!(&self.final_action, Some(Action::Keep { .. })) { + self.final_action = None; + } + + let notify = self.notify_spec(script, &redirect.notify)?; + let by_time = self.by_time(script, &redirect.by_time)?; + if let Some(created) = self.build_message_id() { + self.actions.push(created); + } + self.num_redirects += 1; + self.num_out_messages += 1; + let address = self.alloc_string(address); + self.actions.push(Action::SendMessage { + source: MessageSource::Redirect, + recipient: if !redirect.list { + Recipient::Address(address) + } else { + Recipient::List(address) + }, + notify, + return_of_content: ret_from_code(redirect.ret), + by_time, + message_id: self.main_message_id, + }); + Ok(()) + } + + fn notify_spec( + &self, + script: &'x Sieve<'x>, + spec: &ops::NotifySpec, + ) -> Result { + Ok(match spec.kind { + NOTIFY_NEVER => Notify::Never, + NOTIFY_ITEMS => Notify::Items(notify_items(script, spec.items)?), + _ => Notify::Default, + }) + } + + fn by_time( + &self, + script: &'x Sieve<'x>, + by_time: &ops::ByTime, + ) -> Result, RuntimeError> { + Ok(match by_time.kind { + BY_TIME_RELATIVE => ByTime::Relative { + rlimit: by_time.rlimit, + mode: by_mode_from_code(by_time.mode), + trace: by_time.trace, + }, + BY_TIME_ABSOLUTE => ByTime::Absolute { + alimit: DateTime::parse_rfc3339( + self.eval_value(script, by_time.alimit)? + .to_string() + .as_ref(), + ) + .and_then(|d| { + if d.is_valid() { + d.to_timestamp().into() + } else { + None + } + }) + .unwrap_or(0), + mode: by_mode_from_code(by_time.mode), + trace: by_time.trace, + }, + _ => ByTime::None, + }) + } +} + +fn notify_items(script: &Sieve<'_>, items: Range) -> Result, RuntimeError> { + script + .recs(items)? + .map(|rec| { + if rec.tag != tag::NOTIFY_ITEM { + return Err(RuntimeError::InvalidBytecode); + } + Ok(match rec.b { + 0 => NotifyItem::Success, + 1 => NotifyItem::Failure, + _ => NotifyItem::Delay, + }) + }) + .collect() +} + +fn ret_from_code(code: u8) -> Ret { + match code { + 0 => Ret::Full, + 1 => Ret::Hdrs, + _ => Ret::Default, + } +} + +fn by_mode_from_code(code: u8) -> ByMode { + match code { + 0 => ByMode::Notify, + 1 => ByMode::Return, + _ => ByMode::Default, + } +} + +pub(crate) fn sanitize_address(addr: &str) -> Option { + let mut result = String::with_capacity(addr.len()); + let mut in_quote = false; + let mut last_ch = '\n'; + let mut has_at = false; + let mut has_dot = false; + + for ch in addr.chars() { + match ch { + '\"' => { + if !in_quote { + in_quote = true; + } else if last_ch != '\\' { + in_quote = false; + } + } + '@' if !in_quote => { + if !has_at && !result.is_empty() { + has_at = true; + result.push(ch); + } else { + return None; + } + } + '.' if !in_quote && has_at && !has_dot => { + has_dot = true; + result.push(ch); + } + '<' => { + result.clear(); + has_at = false; + has_dot = false; + } + '>' => (), + _ => { + if !ch.is_ascii_whitespace() || in_quote { + result.push(ch); + } + } + } + last_ch = ch; + } + + if !result.is_empty() && has_at && has_dot { + Some(result) + } else { + None + } +} diff --git a/crates/sieve/src/runtime/actions/action_set.rs b/crates/sieve/src/runtime/actions/action_set.rs new file mode 100644 index 00000000..0781dc82 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_set.rs @@ -0,0 +1,168 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Sieve, + bytecode::{ + ops, + rec::{Range, tag}, + }, + compiler::grammar::actions::action_set::{ + MODIFIER_ENCODE_URL, MODIFIER_LENGTH, MODIFIER_LOWER, MODIFIER_LOWER_FIRST, + MODIFIER_QUOTE_REGEX, MODIFIER_QUOTE_WILDCARD, MODIFIER_REPLACE, MODIFIER_UPPER, + MODIFIER_UPPER_FIRST, + }, + runtime::{RuntimeError, Variable}, +}; +use std::{borrow::Cow, fmt::Write}; + +impl<'x> Context<'x> { + pub(crate) fn exec_set( + &mut self, + script: &'x Sieve<'x>, + set: &ops::Set, + ) -> Result<(), RuntimeError> { + let mut value = self.eval_value(script, set.value)?; + if !set.modifiers.is_empty() { + value = self.apply_modifiers(script, set.modifiers, value)?; + } + self.set_variable(script, set.name, value) + } + + pub(crate) fn apply_modifiers( + &self, + script: &'x Sieve<'x>, + modifiers: Range, + mut value: Variable<'x>, + ) -> Result, RuntimeError> { + let mut iter = script.recs(modifiers)?; + while let Some(modifier) = iter.next() { + if modifier.tag != tag::MODIFIER { + return Err(RuntimeError::InvalidBytecode); + } + let input = value.to_string(); + let output = match modifier.b { + MODIFIER_REPLACE => { + let find = iter.next().ok_or(RuntimeError::InvalidBytecode)?; + let replace = iter.next().ok_or(RuntimeError::InvalidBytecode)?; + input.replace( + self.eval_value(script, find)?.to_string().as_ref(), + self.eval_value(script, replace)?.to_string().as_ref(), + ) + } + kind => self.apply_modifier(kind, input.as_ref()), + }; + value = Variable::String(Cow::Owned(output)); + } + Ok(value) + } + + pub(crate) fn apply_modifier(&self, kind: u8, input: &str) -> String { + let max_len = self.runtime.max_variable_size; + match kind { + MODIFIER_LOWER => input.to_lowercase(), + MODIFIER_UPPER => input.to_uppercase(), + MODIFIER_LOWER_FIRST => { + let mut result = String::with_capacity(input.len()); + for (pos, char) in input.chars().enumerate() { + if result.len() + char.len_utf8() <= max_len { + if pos != 0 { + result.push(char); + } else { + for char in char.to_lowercase() { + result.push(char); + } + } + } else { + return result; + } + } + result + } + MODIFIER_UPPER_FIRST => { + let mut result = String::with_capacity(input.len()); + for (pos, char) in input.chars().enumerate() { + if result.len() + char.len_utf8() <= max_len { + if pos != 0 { + result.push(char); + } else { + for char in char.to_uppercase() { + result.push(char); + } + } + } else { + return result; + } + } + result + } + MODIFIER_QUOTE_WILDCARD => { + let mut result = String::with_capacity(input.len()); + for char in input.chars() { + if ['*', '\\', '?'].contains(&char) { + if result.len() + char.len_utf8() < max_len { + result.push('\\'); + result.push(char); + } else { + return result; + } + } else if result.len() + char.len_utf8() <= max_len { + result.push(char); + } else { + return result; + } + } + result + } + MODIFIER_QUOTE_REGEX => { + let mut result = String::with_capacity(input.len()); + for char in input.chars() { + if [ + '*', '\\', '?', '.', '[', ']', '(', ')', '+', '{', '}', '|', '^', '=', ':', + '$', + ] + .contains(&char) + { + if result.len() + char.len_utf8() < max_len { + result.push('\\'); + result.push(char); + } else { + return result; + } + } else if result.len() + char.len_utf8() <= max_len { + result.push(char); + } else { + return result; + } + } + result + } + MODIFIER_LENGTH => input.chars().count().to_string(), + MODIFIER_ENCODE_URL => { + let mut buf = [0; 4]; + let mut result = String::with_capacity(input.len()); + + for char in input.chars() { + if char.is_ascii_alphanumeric() || ['-', '.', '_', '~'].contains(&char) { + if result.len() < max_len { + result.push(char); + } else { + return result; + } + } else if result.len() + (char.len_utf8() * 3) <= max_len { + for byte in char.encode_utf8(&mut buf).as_bytes().iter() { + write!(result, "%{byte:02x}").ok(); + } + } else { + return result; + } + } + result + } + _ => input.to_string(), + } + } +} diff --git a/crates/sieve/src/runtime/actions/action_vacation.rs b/crates/sieve/src/runtime/actions/action_vacation.rs new file mode 100644 index 00000000..af9c6941 --- /dev/null +++ b/crates/sieve/src/runtime/actions/action_vacation.rs @@ -0,0 +1,323 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::super::tests::TestResult; +use crate::runtime::platform; +use crate::{ + Context, Envelope, Sieve, + bytecode::ops, + compiler::grammar::{ + AddressPart, + actions::action_redirect::{ByTime, Notify, Ret}, + }, + runtime::{ + RuntimeError, + handler::{Action, Handler, MessageSource, Recipient}, + }, +}; +use mail_builder::headers::date::Date; +use mail_parser::HeaderName; +use std::borrow::Cow; + +pub(crate) const MAX_SUBJECT_LEN: usize = 256; + +const PERIOD_DAYS: u8 = 0; +const PERIOD_SECONDS: u8 = 1; + +impl<'x> Context<'x> { + pub(crate) fn test_vacation>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestVacation, + handler: &mut H, + ) -> Result { + let mut from = String::new(); + let mut user_addresses: Vec> = Vec::new(); + + if self.num_out_messages >= self.runtime.max_out_messages { + return Ok(TestResult::Bool(false)); + } + + for (name, value) in &self.envelope { + if !value.is_empty() { + match name { + Envelope::From => { + from = value.to_string().to_ascii_lowercase(); + } + Envelope::To if !self.runtime.vacation_use_orig_rcpt => { + user_addresses.push(value.to_string()); + } + Envelope::Orcpt if self.runtime.vacation_use_orig_rcpt => { + user_addresses.push(value.to_string()); + } + _ => (), + } + } + } + + for address in self.eval_values(script, test.addresses)? { + let address = address.into_string(); + if !address.is_empty() { + user_addresses.push(address); + } + } + if !self.user_address.is_empty() { + user_addresses.push(Cow::Borrowed(self.user_address.as_ref())); + } + + if from.is_empty() + || user_addresses.is_empty() + || from.starts_with("mailer-daemon") + || from.starts_with("owner-") + || from.contains("-request@") + || user_addresses.iter().any(|a| a.eq_ignore_ascii_case(&from)) + { + return Ok(TestResult::Bool(false)); + } + + let mut found_rcpt = false; + let mut received_count = 0; + for header in self.root_headers() { + match header.name() { + HeaderName::To + | HeaderName::Cc + | HeaderName::Bcc + | HeaderName::ResentTo + | HeaderName::ResentBcc + | HeaderName::ResentCc + if !found_rcpt => + { + found_rcpt = self.find_addresses(header, &AddressPart::All, |addr| { + user_addresses.iter().any(|a| a.eq_ignore_ascii_case(addr)) + }); + } + HeaderName::ListArchive + | HeaderName::ListHelp + | HeaderName::ListId + | HeaderName::ListOwner + | HeaderName::ListPost + | HeaderName::ListSubscribe + | HeaderName::ListUnsubscribe => { + return Ok(TestResult::Bool(false)); + } + HeaderName::Received => { + received_count += 1; + } + HeaderName::AutoSubmitted => { + if header + .value() + .as_text() + .is_none_or(|v| !v.eq_ignore_ascii_case("no")) + { + return Ok(TestResult::Bool(false)); + } + } + name if name + .as_str() + .eq_ignore_ascii_case("X-Auto-Response-Suppress") => + { + if header.value().as_text().is_some_and(|v| { + v.to_ascii_lowercase() + .split(',') + .any(|v| ["all", "oof"].contains(&v.trim())) + }) { + return Ok(TestResult::Bool(false)); + } + } + name if name.as_str().eq_ignore_ascii_case("Precedence") + && header + .value() + .as_text() + .is_some_and(|v| v.eq_ignore_ascii_case("bulk")) => + { + return Ok(TestResult::Bool(false)); + } + _ => (), + } + } + + if found_rcpt && received_count <= self.runtime.max_received_headers { + let suffix = match self.eval_opt(script, test.handle)? { + Some(handle) => handle, + None => self.eval_value(script, test.reason)?, + }; + let id = format!("_v{}{}", from, suffix.to_string()); + let expiry = match test.period.kind { + PERIOD_DAYS => test.period.value.saturating_mul(86400), + PERIOD_SECONDS => test.period.value, + _ => self.runtime.default_vacation_expiry, + }; + TestResult::from_reply(handler.duplicate_id(self, &id, expiry, false), true) + } else { + Ok(TestResult::Bool(false)) + } + } + + pub(crate) fn exec_vacation( + &mut self, + script: &'x Sieve<'x>, + vacation: &ops::Vacation, + ) -> Result<(), RuntimeError> { + let vacation_to = self + .envelope + .iter() + .find(|(name, value)| !value.is_empty() && name == &Envelope::From) + .map_or(Cow::Borrowed(""), |(_, value)| value.clone().into_string()); + + let mut vacation_subject = self + .eval_opt(script, vacation.subject)? + .map_or(Cow::Borrowed(""), |subject| subject.into_string()); + + let mut message_id = None; + let mut vacation_to_full = None; + let mut references = None; + for header in self.root_headers() { + match header.name() { + HeaderName::Subject if vacation_subject.is_empty() => { + if let Some(subject) = header.value().as_text() { + let mut vacation_subject_ = String::with_capacity(MAX_SUBJECT_LEN); + let mut iter = self + .runtime + .vacation_subject_prefix + .chars() + .chain(subject.chars()) + .enumerate(); + + #[allow(clippy::while_let_on_iterator)] + while let Some((pos, char)) = iter.next() { + if pos < MAX_SUBJECT_LEN { + vacation_subject_.push(char); + } else { + break; + } + } + if iter.next().is_some() { + vacation_subject_.push('…'); + } + vacation_subject = vacation_subject_.into(); + } + } + HeaderName::MessageId => { + message_id = header.value().as_text(); + } + HeaderName::References => { + if let Some(raw) = header.raw_value() { + references = Some(raw); + } + } + HeaderName::From | HeaderName::Sender + if header + .value() + .as_address() + .is_some_and(|address| address.contains(vacation_to.as_ref())) => + { + if let Some(raw) = header.raw_value() { + vacation_to_full = Some(raw); + } + } + _ => (), + } + } + + let vacation_from = if let Some(from) = self.eval_opt(script, vacation.from)? { + from.into_string() + } else if !self.user_address.is_empty() { + self.user_from_field().into() + } else if let Some(addr) = self + .envelope + .iter() + .find_map(|(n, v)| if n == &Envelope::To { Some(v) } else { None }) + { + addr.to_string() + } else { + "".into() + }; + if vacation_subject.is_empty() { + vacation_subject = self.runtime.vacation_default_subject.as_ref().into(); + } + let vacation_body = self.eval_value(script, vacation.reason)?.into_string(); + let message_len = vacation_body.len() + + vacation_from.len() + + vacation_to_full.map_or(vacation_to.len(), |t| t.len()) + + vacation_subject.len() + + message_id.map_or(0, |m| m.len() * 2) + + references.map_or(0, |m| m.len()) + + 160; + + let mut message = Vec::with_capacity(message_len); + write_header(&mut message, "From: ", vacation_from.as_ref()); + if let Some(vacation_to_full) = vacation_to_full { + message.extend_from_slice(b"To:"); + message.extend_from_slice(vacation_to_full); + } else { + write_header(&mut message, "To: ", vacation_to.as_ref()); + } + write_header(&mut message, "Subject: ", vacation_subject.as_ref()); + if let Some(message_id) = message_id { + message.extend_from_slice(b"In-Reply-To: <"); + message.extend_from_slice(message_id.as_bytes()); + message.extend_from_slice(b">\r\n"); + + message.extend_from_slice(b"References: <"); + message.extend_from_slice(message_id.as_bytes()); + if let Some(references) = references { + message.extend_from_slice(b"> "); + message.extend_from_slice(references); + } else { + message.extend_from_slice(b">\r\n"); + } + } + message.extend_from_slice(b"Date: "); + message.extend_from_slice(Date::new(self.current_time).to_rfc822().as_bytes()); + message.extend_from_slice(b"\r\n"); + + message.extend_from_slice(b"Message-ID: "); + platform::write_message_id(&mut message, &self.runtime.local_hostname); + message.extend_from_slice(b"\r\n"); + + write_header(&mut message, "Auto-Submitted: ", "auto-replied"); + if !vacation.mime { + message.extend_from_slice(b"Content-type: text/plain; charset=utf-8\r\n\r\n"); + } + message.extend_from_slice(vacation_body.as_bytes()); + + let recipient = self.intern_cow(vacation_to); + self.last_message_id += 1; + self.num_out_messages += 1; + self.actions.push(Action::CreatedMessage { + message_id: self.last_message_id, + message, + }); + self.actions.push(Action::SendMessage { + source: MessageSource::Vacation, + recipient: Recipient::Address(recipient), + notify: Notify::Never, + return_of_content: Ret::Default, + by_time: ByTime::None, + message_id: self.last_message_id, + }); + + if vacation.fcc.present { + let fcc = &vacation.fcc; + let action = Action::FileInto { + folder: self.eval_str(script, fcc.mailbox)?, + flags: self.get_local_flags(script, fcc.flags)?, + mailbox_id: self.eval_opt_str(script, fcc.mailbox_id)?, + special_use: self.eval_opt_str(script, fcc.special_use)?, + create: fcc.create, + message_id: self.last_message_id, + }; + self.actions.push(action); + } + Ok(()) + } +} + +fn write_header(buf: &mut Vec, name: &str, value: &str) { + buf.extend_from_slice(name.as_bytes()); + buf.extend_from_slice(value.as_bytes()); + buf.extend_from_slice(b"\r\n"); +} diff --git a/crates/sieve/src/runtime/actions/mod.rs b/crates/sieve/src/runtime/actions/mod.rs new file mode 100644 index 00000000..f97849fb --- /dev/null +++ b/crates/sieve/src/runtime/actions/mod.rs @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod action_convert; +pub mod action_editheader; +pub mod action_fileinto; +pub mod action_flags; +pub mod action_keep; +pub mod action_mime; +pub mod action_notify; +pub mod action_redirect; +pub mod action_set; +pub mod action_vacation; diff --git a/crates/sieve/src/runtime/arena.rs b/crates/sieve/src/runtime/arena.rs new file mode 100644 index 00000000..aaf5f909 --- /dev/null +++ b/crates/sieve/src/runtime/arena.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use bumpalo::Bump; +use mail_parser::Message; +use std::{ + cell::{Cell, RefCell}, + ptr::NonNull, +}; + +#[derive(Default)] +pub struct Arena { + pub(crate) bump: Bump, + texts: RefCell>, + text_bytes: Cell, + messages: Vec>>, + message_bytes: usize, + charged: Cell, + limit: Option, +} + +unsafe impl Send for Arena {} + +impl Arena { + pub fn new() -> Self { + Self::default() + } + + pub fn with_capacity(bytes: usize) -> Self { + Arena { + bump: Bump::with_capacity(bytes), + texts: RefCell::default(), + text_bytes: Cell::new(0), + messages: Vec::new(), + message_bytes: 0, + charged: Cell::new(0), + limit: None, + } + } + + pub fn reset(&mut self) { + self.bump.reset(); + self.release(); + } + + pub(crate) fn prepare(&mut self, limit: usize) { + self.reset(); + if self.bump.allocated_bytes() > limit { + self.bump = Bump::new(); + } + self.bump.set_allocation_limit(Some(limit)); + self.limit = Some(limit); + } + + pub fn allocated_bytes(&self) -> usize { + self.bump.allocated_bytes() + self.message_bytes + self.text_bytes.get() + } + + pub(crate) fn charge(&self, bytes: usize) -> bool { + let charged = self.charged.get().saturating_add(bytes); + if let Some(limit) = self.limit { + if self.bump.allocated_bytes().saturating_add(charged) > limit { + return false; + } + self.bump.set_allocation_limit(Some(limit - charged)); + } + self.charged.set(charged); + true + } + + pub(crate) fn keep_text<'x>(&self, text: String) -> &'x str { + self.text_bytes.set(self.text_bytes.get() + text.len()); + let mut texts = self.texts.borrow_mut(); + texts.push(text); + let text = texts.last().map_or("", String::as_str); + unsafe { &*(text as *const str) } + } + + pub(crate) fn keep_charged_text<'x>(&self, text: String) -> Option<&'x str> { + self.charge(text.len()).then(|| self.keep_text(text)) + } + + pub(crate) fn keep_message<'x>(&mut self, message: Message<'static>) -> &'x Message<'x> { + self.message_bytes += message.raw().len(); + let message = NonNull::from(Box::leak(Box::new(message))); + self.messages.push(message); + unsafe { message.as_ref() } + } + + fn release(&mut self) { + self.texts.get_mut().clear(); + self.text_bytes.set(0); + self.charged.set(0); + self.message_bytes = 0; + for message in self.messages.drain(..) { + drop(unsafe { Box::from_raw(message.as_ptr()) }); + } + } +} + +impl Drop for Arena { + fn drop(&mut self) { + self.release(); + } +} diff --git a/crates/sieve/src/runtime/context.rs b/crates/sieve/src/runtime/context.rs new file mode 100644 index 00000000..832afa98 --- /dev/null +++ b/crates/sieve/src/runtime/context.rs @@ -0,0 +1,1285 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Arena; +use super::message::{ + edits::Edits, + parts::{PartCursor, ROOT_PART, Scope, empty_message}, +}; +use super::{ + RuntimeError, Variable, + handler::{Action, Handler, Input, Reply, Script, Status}, + tests::{TestResult, test_envelope::parse_envelope_address}, + variable::Array, +}; +use crate::{ + Context, Envelope, Metadata, Runtime, Sieve, SpamStatus, VirusStatus, + bytecode::{ + cursor::Cursor, + ops, + rec::{Rec, tag}, + }, + compiler::grammar::Capability, +}; +use ahash::AHashMap; +use mail_parser::{Message, PartId}; +use std::{ + borrow::Cow, + cell::{Cell, RefCell}, +}; + +#[derive(Clone, Copy)] +pub(crate) struct Frame<'x> { + pub(crate) script: &'x Sieve<'x>, + pub(crate) prev_pos: usize, + pub(crate) local_base: usize, + pub(crate) match_base: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Pending<'x> { + Idle, + Test { is_not: bool }, + Function, + Include { name: Script<'x>, optional: bool }, + Action, + Finished, + Aborted, +} + +impl<'x> Context<'x> { + pub fn new( + runtime: &'x Runtime, + message: &'x Message<'x>, + script: &'x Sieve<'x>, + arena: &'x mut Arena, + ) -> Self { + arena.prepare(runtime.memory_limit); + Self::with_message(runtime, message, script, arena) + } + + pub(crate) fn with_message( + runtime: &'x Runtime, + message: &'x Message<'x>, + script: &'x Sieve<'x>, + arena: &'x mut Arena, + ) -> Self { + let message_size = message.raw().len(); + let message = if message.part(ROOT_PART).is_some() { + message + } else { + arena.keep_message(empty_message()) + }; + Context { + runtime, + message, + script, + frames: Vec::with_capacity(2), + local_base: 0, + match_base: 0, + part: ROOT_PART, + part_iter: PartCursor::default(), + part_iter_stack: Vec::new(), + pos: 0, + test_result: false, + pending: Pending::Idle, + error: None, + rejected: false, + included: Vec::new(), + vars_global: AHashMap::new(), + vars_env: AHashMap::new(), + vars_local: Vec::with_capacity(script.num_vars()), + vars_match: Vec::with_capacity(script.num_match_vars()), + expr_stack: Vec::with_capacity(16), + expr_pos: 0, + envelope: Vec::new(), + metadata: Vec::new(), + message_size, + edits: Edits::default(), + body_cache: RefCell::new(Vec::new()), + final_action: Some(Action::Keep { + flags: &[], + message_id: 0, + }), + flags: Vec::new(), + actions: Vec::new(), + has_changes: false, + oom: Cell::new(false), + dynamic_regexes: RefCell::new(AHashMap::new()), + user_address: "".into(), + user_full_name: "".into(), + current_time: super::platform::unix_time(), + num_redirects: 0, + num_instructions: 0, + num_scanned_bytes: 0, + num_out_messages: 0, + last_message_id: 0, + main_message_id: 0, + virus_status: VirusStatus::Unknown, + spam_status: SpamStatus::Unknown, + arena, + } + } + + #[inline(always)] + pub(crate) fn alloc_str(&self, s: &str) -> &'x str { + if s.is_empty() { + return ""; + } + match self.arena.bump.try_alloc_slice_copy(s.as_bytes()) { + Ok(bytes) => unsafe { std::str::from_utf8_unchecked(extend(bytes)) }, + Err(_) => { + self.note_oom(); + "" + } + } + } + + #[inline(always)] + pub(crate) fn alloc_string(&self, s: String) -> &'x str { + self.alloc_str(&s) + } + + pub(crate) fn alloc_strs(&self, items: &[&'x str]) -> &'x [&'x str] { + if items.is_empty() { + return &[]; + } + match self.arena.bump.try_alloc_slice_copy(items) { + Ok(slice) => unsafe { extend(slice) }, + Err(_) => { + self.note_oom(); + &[] + } + } + } + + pub(crate) fn alloc_variables(&self, items: Vec>) -> &'x [Variable<'x>] { + match self.try_alloc_variables(items) { + Ok(slice) => slice, + Err(_) => { + self.note_oom(); + &[] + } + } + } + + pub(crate) fn try_alloc_variables( + &self, + items: I, + ) -> Result<&'x [Variable<'x>], RuntimeError> + where + I: IntoIterator>, + I::IntoIter: ExactSizeIterator, + { + let items = items.into_iter(); + if items.len() == 0 { + return Ok(&[]); + } + self.arena + .bump + .try_alloc_slice_fill_iter(items.map(|v| self.intern(v))) + .map(|slice| unsafe { extend(slice) }) + .map_err(|_| RuntimeError::MemoryLimitReached) + } + + #[inline(always)] + pub(crate) fn intern(&self, value: Variable<'x>) -> Variable<'x> { + match value { + Variable::String(Cow::Owned(s)) => Variable::String(Cow::Borrowed(self.alloc_str(&s))), + Variable::Array(Array::Owned(items)) => { + Variable::Array(Array::Borrowed(self.alloc_variables(items))) + } + value => value, + } + } + + #[inline(always)] + pub(crate) fn intern_cow(&self, value: Cow<'x, str>) -> &'x str { + match value { + Cow::Borrowed(s) => s, + Cow::Owned(s) => self.alloc_str(&s), + } + } + + #[cold] + pub(crate) fn note_oom(&self) { + self.oom.set(true); + } + + pub fn run>(&mut self, handler: &mut H) -> Result { + match self.pending { + Pending::Idle => { + self.pending = Pending::Action; + self.push_frame(self.script); + } + Pending::Finished => return Ok(Status::Finished), + Pending::Aborted => { + return match self.error.take() { + Some(err) => Err(err), + None => self.finish(handler), + }; + } + Pending::Action => (), + _ => return Err(RuntimeError::AwaitingInput), + } + + match self.flush_actions(handler) { + Ok(true) => (), + Ok(false) => return Ok(Status::Pending), + Err(err) => return Err(self.abort(err)), + } + + if self.frames.is_empty() { + return self.finish(handler); + } + + let cpu_limit = self.runtime.cpu_limit; + + 'outer: loop { + if self.frames.is_empty() { + return self.finish(handler); + } + let script = self.script; + let mut cur = Cursor::new(script.code(), self.pos); + + while !cur.at_end() { + self.num_instructions += 1; + if self.num_instructions > cpu_limit { + return Err(self.abort(RuntimeError::CPULimitReached)); + } + if self.oom.get() { + return Err(self.abort(RuntimeError::MemoryLimitReached)); + } + let start = cur.pos; + let op = cur.u8()?; + + match op { + ops::Jz::OP => { + let jump = ops::Jz::decode(&mut cur)?; + if !self.test_result { + cur.pos = jump.target.0 as usize; + } + continue; + } + ops::Jnz::OP => { + let jump = ops::Jnz::decode(&mut cur)?; + if self.test_result { + cur.pos = jump.target.0 as usize; + } + continue; + } + ops::Jmp::OP => { + let jump = ops::Jmp::decode(&mut cur)?; + cur.pos = jump.target.0 as usize; + continue; + } + ops::Clear::OP => { + let clear = ops::Clear::decode(&mut cur)?; + self.exec_clear(&clear); + continue; + } + ops::Discard::OP => { + self.final_action = Some(Action::Discard); + continue; + } + ops::Stop::OP => { + self.pos = cur.pos; + self.frames.clear(); + return self.finish(handler); + } + ops::Return::OP => { + self.pos = cur.pos; + self.pop_frame(); + continue 'outer; + } + _ => (), + } + + match self.exec_op(op, start, &mut cur, handler) { + Ok(Flow::Continue) => (), + Ok(Flow::Jump(target)) => { + cur.pos = target; + } + Ok(Flow::Switch) => { + continue 'outer; + } + Ok(Flow::Pending) => { + return Ok(Status::Pending); + } + Err(err) => { + return Err(self.abort(err)); + } + } + } + + self.pos = cur.pos; + self.pop_frame(); + } + } + + #[inline(always)] + fn exec_op>( + &mut self, + op: u8, + start: usize, + cur: &mut Cursor<'x>, + handler: &mut H, + ) -> Result { + let script = self.script; + let flow = match op { + ops::TestHeader::OP => { + let test = ops::TestHeader::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_header(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestString::OP => { + let test = ops::TestString::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_string(script, &test, false, handler)?; + self.test_outcome(result) + } + } + ops::TestEnvironment::OP => { + let test = ops::TestEnvironment::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_environment(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestAddress::OP => { + let test = ops::TestAddress::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_address(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestEnvelope::OP => { + let test = ops::TestEnvelope::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_envelope(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestExists::OP => { + let test = ops::TestExists::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_exists(script, &test)?; + self.test_outcome(result) + } + } + ops::TestSize::OP => { + let test = ops::TestSize::decode(cur)?; + self.pos = cur.pos; + self.test_outcome(self.test_size(&test)) + } + ops::TestBody::OP => { + let test = ops::TestBody::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_body(script, &test)?; + self.test_outcome(result) + } + } + ops::TestDate::OP => { + let test = ops::TestDate::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_date(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestCurrentDate::OP => { + let test = ops::TestCurrentDate::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_current_date(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestDuplicate::OP => { + let test = ops::TestDuplicate::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_duplicate(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestHasFlag::OP => { + let test = ops::TestHasFlag::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_hasflag(script, &test)?; + self.test_outcome(result) + } + } + ops::TestNotifyMethodCapability::OP => { + let test = ops::TestNotifyMethodCapability::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_notify_method_capability(script, &test)?; + self.test_outcome(result) + } + } + ops::TestValidNotifyMethod::OP => { + let test = ops::TestValidNotifyMethod::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_valid_notify_method(script, &test)?; + self.test_outcome(result) + } + } + ops::TestValidExtList::OP => { + let test = ops::TestValidExtList::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_valid_ext_list(script, &test)?; + self.test_outcome(result) + } + } + ops::TestIhave::OP => { + let test = ops::TestIhave::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_ihave(script, &test)?; + self.test_outcome(result) + } + } + ops::TestMailboxExists::OP => { + let test = ops::TestMailboxExists::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_mailbox_exists(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestMailboxIdExists::OP => { + let test = ops::TestMailboxIdExists::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_mailbox_id_exists(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestSpecialUseExists::OP => { + let test = ops::TestSpecialUseExists::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_special_use_exists(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestMetadata::OP => { + let test = ops::TestMetadata::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_metadata(script, &test)?; + self.test_outcome(result) + } + } + ops::TestMetadataExists::OP => { + let test = ops::TestMetadataExists::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_metadata_exists(script, &test)?; + self.test_outcome(result) + } + } + ops::TestSpamTest::OP => { + let test = ops::TestSpamTest::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_spamtest(script, &test)?; + self.test_outcome(result) + } + } + ops::TestVirusTest::OP => { + let test = ops::TestVirusTest::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_virustest(script, &test)?; + self.test_outcome(result) + } + } + ops::TestVacation::OP => { + let test = ops::TestVacation::decode(cur)?; + self.pos = cur.pos; + { + let result = self.test_vacation(script, &test, handler)?; + self.test_outcome(result) + } + } + ops::TestConvert::OP => { + let test = ops::TestConvert::decode(cur)?; + self.pos = cur.pos; + { + let result = self.exec_convert(script, &test.into())?; + self.test_outcome(result) + } + } + ops::TestTrue::OP => { + self.pos = cur.pos; + self.test_result = true; + Flow::Continue + } + ops::TestFalse::OP => { + self.pos = cur.pos; + self.test_result = false; + Flow::Continue + } + ops::TestInvalid::OP => { + let test = ops::TestInvalid::decode(cur)?; + return Err(RuntimeError::InvalidInstruction { + name: script.str(test.name)?.to_string(), + line_num: test.line_num, + line_pos: test.line_pos, + }); + } + ops::Eval::OP => { + let eval = ops::Eval::decode(cur)?; + self.pos = cur.pos; + match self.eval_expression(script, eval.expr, handler)? { + Some(result) => { + self.test_result = result.to_bool(); + Flow::Continue + } + None => { + self.pos = start; + Flow::Pending + } + } + } + ops::Let::OP => { + let let_ = ops::Let::decode(cur)?; + self.pos = cur.pos; + match self.eval_expression(script, let_.expr, handler)? { + Some(result) => { + self.set_variable(script, let_.name, result)?; + Flow::Continue + } + None => { + self.pos = start; + Flow::Pending + } + } + } + ops::While::OP => { + let while_ = ops::While::decode(cur)?; + self.pos = cur.pos; + match self.eval_expression(script, while_.expr, handler)? { + Some(result) => { + if result.to_bool() { + Flow::Continue + } else { + Flow::Jump(while_.jz_pos.0 as usize) + } + } + None => { + self.pos = start; + Flow::Pending + } + } + } + ops::Set::OP => { + let set = ops::Set::decode(cur)?; + self.pos = cur.pos; + self.exec_set(script, &set)?; + self.flush(handler)? + } + ops::Keep::OP => { + let keep = ops::Keep::decode(cur)?; + self.pos = cur.pos; + self.exec_keep(script, &keep)?; + self.flush(handler)? + } + ops::FileInto::OP => { + let fileinto = ops::FileInto::decode(cur)?; + self.pos = cur.pos; + self.exec_fileinto(script, &fileinto)?; + self.flush(handler)? + } + ops::Redirect::OP => { + let redirect = ops::Redirect::decode(cur)?; + self.pos = cur.pos; + self.exec_redirect(script, &redirect)?; + self.flush(handler)? + } + ops::Reject::OP => { + let reject = ops::Reject::decode(cur)?; + self.pos = cur.pos; + let reason = self.eval_value(script, reject.reason)?.into_string(); + self.final_action = None; + self.rejected = true; + let reason = self.intern_cow(reason); + self.actions.push(Action::Reject { + extended: reject.ereject, + reason, + }); + self.flush(handler)? + } + ops::ForEveryPart::OP => { + let fep = ops::ForEveryPart::decode(cur)?; + self.pos = cur.pos; + if let Some(next_part) = self.next_part() { + self.part = next_part; + Flow::Continue + } else if let Some((prev_part, prev_iter)) = self.part_iter_stack.pop() { + self.part_iter = prev_iter; + self.part = prev_part; + Flow::Jump(fep.jz_pos.0 as usize) + } else { + self.part = ROOT_PART; + Flow::Continue + } + } + ops::ForEveryPartPush::OP => { + self.pos = cur.pos; + let part_iter = self + .current_part() + .map(|part| { + PartCursor::subtree(part, self.part_iter_stack.is_empty(), Scope::Message) + }) + .unwrap_or_default(); + let prev_iter = std::mem::replace(&mut self.part_iter, part_iter); + self.part_iter_stack.push((self.part, prev_iter)); + Flow::Continue + } + ops::ForEveryPartPop::OP => { + let pop = ops::ForEveryPartPop::decode(cur)?; + self.pos = cur.pos; + debug_assert!( + pop.num_pops > 0 && pop.num_pops as usize <= self.part_iter_stack.len(), + "Pop out of range: {} with {} items.", + pop.num_pops, + self.part_iter_stack.len() + ); + for _ in 0..pop.num_pops { + if let Some((prev_part, prev_iter)) = self.part_iter_stack.pop() { + self.part_iter = prev_iter; + self.part = prev_part; + } else { + break; + } + } + Flow::Continue + } + ops::Replace::OP => { + let replace = ops::Replace::decode(cur)?; + self.pos = cur.pos; + self.exec_replace(script, &replace)?; + Flow::Continue + } + ops::Enclose::OP => { + let enclose = ops::Enclose::decode(cur)?; + self.pos = cur.pos; + self.exec_enclose(script, &enclose)?; + Flow::Continue + } + ops::ExtractText::OP => { + let extract = ops::ExtractText::decode(cur)?; + self.pos = cur.pos; + self.exec_extracttext(script, &extract)?; + self.flush(handler)? + } + ops::AddHeader::OP => { + let add = ops::AddHeader::decode(cur)?; + self.pos = cur.pos; + self.exec_addheader(script, &add)?; + Flow::Continue + } + ops::DeleteHeader::OP => { + let delete = ops::DeleteHeader::decode(cur)?; + self.pos = cur.pos; + self.exec_deleteheader(script, &delete)?; + Flow::Continue + } + ops::Notify::OP => { + let notify = ops::Notify::decode(cur)?; + self.pos = cur.pos; + self.exec_notify(script, ¬ify)?; + self.flush(handler)? + } + ops::Vacation::OP => { + let vacation = ops::Vacation::decode(cur)?; + self.pos = cur.pos; + self.exec_vacation(script, &vacation)?; + self.flush(handler)? + } + ops::EditFlags::OP => { + let flags = ops::EditFlags::decode(cur)?; + self.pos = cur.pos; + self.exec_editflags(script, &flags)?; + Flow::Continue + } + ops::Convert::OP => { + let convert = ops::Convert::decode(cur)?; + self.pos = cur.pos; + self.exec_convert(script, &convert)?; + Flow::Continue + } + ops::Include::OP => { + let include = ops::Include::decode(cur)?; + self.pos = cur.pos; + self.exec_include(script, &include, handler)? + } + ops::Require::OP => { + let require = ops::Require::decode(cur)?; + self.pos = cur.pos; + for rec in script.recs(require.capabilities)? { + let capability = Capability::from_rec(script, rec)?; + if !self.runtime.allowed_capabilities.contains(&capability) { + return Err(if let Capability::Other(not_supported) = capability { + RuntimeError::CapabilityNotSupported(not_supported) + } else { + RuntimeError::CapabilityNotAllowed(capability) + }); + } + } + Flow::Continue + } + ops::Error::OP => { + let error = ops::Error::decode(cur)?; + self.pos = cur.pos; + let message = self.eval_value(script, error.message)?; + return Err(RuntimeError::ScriptErrorMessage( + message.to_string().into_owned(), + )); + } + ops::Invalid::OP => { + let invalid = ops::Invalid::decode(cur)?; + return Err(RuntimeError::InvalidInstruction { + name: script.str(invalid.name)?.to_string(), + line_num: invalid.line_num, + line_pos: invalid.line_pos, + }); + } + #[cfg(all(test, stalwart_tests))] + ops::TestCmd::OP => { + let cmd = ops::TestCmd::decode(cur)?; + self.pos = cur.pos; + { + let result = self.exec_test_cmd(script, cmd.arguments, false, handler)?; + self.test_outcome(result) + } + } + #[cfg(all(test, stalwart_tests))] + ops::TestCmdTest::OP => { + let cmd = ops::TestCmdTest::decode(cur)?; + self.pos = cur.pos; + { + let result = self.exec_test_cmd(script, cmd.arguments, cmd.is_not, handler)?; + self.test_outcome(result) + } + } + _ => return Err(RuntimeError::InvalidBytecode), + }; + Ok(flow) + } + + #[inline(always)] + fn test_outcome(&mut self, result: TestResult) -> Flow { + match result { + TestResult::Bool(value) => { + self.test_result = value; + Flow::Continue + } + TestResult::Pending { is_not } => { + self.pending = Pending::Test { is_not }; + Flow::Pending + } + } + } + + #[inline(always)] + fn flush>(&mut self, handler: &mut H) -> Result { + if self.oom.get() { + return Err(RuntimeError::MemoryLimitReached); + } + if self.actions.is_empty() || self.flush_actions(handler)? { + Ok(Flow::Continue) + } else { + Ok(Flow::Pending) + } + } + + fn flush_actions>(&mut self, handler: &mut H) -> Result { + let mut actions = std::mem::take(&mut self.actions).into_iter(); + while let Some(action) = actions.next() { + match handler.action(self, action) { + Reply::Ready(()) => (), + Reply::Pending => { + let remaining: Vec> = actions.collect(); + self.actions = remaining; + self.pending = Pending::Action; + return Ok(false); + } + Reply::Error(err) => return Err(err), + } + } + Ok(true) + } + + fn finish>(&mut self, handler: &mut H) -> Result { + self.frames.clear(); + if let Some(action) = self.final_action.take() { + match action { + Action::Keep { flags, message_id } => { + let create_message = if self.has_changes { + self.build_message_id() + } else { + None + }; + let flags = if flags.is_empty() && !self.flags.is_empty() { + let flags = std::mem::take(&mut self.flags); + let flags = self.alloc_strs(&flags); + if self.oom.get() { + return Err(self.abort(RuntimeError::MemoryLimitReached)); + } + flags + } else { + flags + }; + if let Some(create_message) = create_message { + self.actions.push(create_message); + self.actions.push(Action::Keep { + flags, + message_id: self.main_message_id, + }); + } else { + self.actions.push(Action::Keep { flags, message_id }); + } + } + action => self.actions.push(action), + } + } + match self.flush_actions(handler) { + Ok(true) => { + self.pending = Pending::Finished; + Ok(Status::Finished) + } + Ok(false) => Ok(Status::Pending), + Err(err) => Err(self.abort(err)), + } + } + + fn abort(&mut self, err: RuntimeError) -> RuntimeError { + self.frames.clear(); + self.actions.clear(); + if self.pending != Pending::Aborted && !self.rejected { + self.final_action = Some(Action::Keep { + flags: &[], + message_id: 0, + }); + } + self.pending = Pending::Aborted; + err + } + + pub fn resume(&mut self, input: impl Into>) { + let input = input.into(); + match std::mem::replace(&mut self.pending, Pending::Action) { + Pending::Test { is_not } => { + self.test_result = matches!(input, Input::Bool(true)) ^ is_not; + } + Pending::Function => { + let value = match input { + Input::Value(value) => self.intern(value), + Input::Bool(value) => Variable::from(value), + _ => Variable::default(), + }; + self.expr_stack.push(value); + } + Pending::Include { name, optional } => match input { + Input::Script(Some(script)) => self.push_included(name, script), + _ if optional => (), + _ => { + let err = RuntimeError::ScriptNotFound(name.name().to_string()); + self.error = Some(self.abort(err)); + } + }, + Pending::Action => (), + Pending::Idle => { + self.pending = Pending::Idle; + } + Pending::Finished => { + self.pending = Pending::Finished; + } + Pending::Aborted => { + self.pending = Pending::Aborted; + } + } + } + + fn push_included(&mut self, name: Script<'x>, script: &'x Sieve<'x>) { + if !self.included.contains(&name) { + self.included.push(name); + } + self.push_frame(script); + } + + pub(crate) fn push_frame(&mut self, script: &'x Sieve<'x>) { + self.frames.push(Frame { + script: self.script, + prev_pos: self.pos, + local_base: self.vars_local.len(), + match_base: self.vars_match.len(), + }); + self.local_base = self.vars_local.len(); + self.match_base = self.vars_match.len(); + self.vars_local + .resize_with(self.vars_local.len() + script.num_vars(), Variable::default); + self.vars_match.resize_with( + self.vars_match.len() + script.num_match_vars(), + Variable::default, + ); + self.script = script; + self.pos = 0; + self.test_result = false; + } + + pub(crate) fn pop_frame(&mut self) { + if let Some(frame) = self.frames.pop() { + self.vars_local.truncate(frame.local_base); + self.vars_match.truncate(frame.match_base); + self.script = frame.script; + self.pos = frame.prev_pos; + self.local_base = self.frames.last().map_or(0, |f| f.local_base); + self.match_base = self.frames.last().map_or(0, |f| f.match_base); + } + } + + #[inline(always)] + pub(crate) fn local_base(&self) -> usize { + self.local_base + } + + #[inline(always)] + pub(crate) fn match_base(&self) -> usize { + self.match_base + } + + fn exec_clear(&mut self, clear: &ops::Clear) { + if clear.local_vars_num > 0 { + let base = self.local_base() + clear.local_vars_idx as usize; + if let Some(local_vars) = self + .vars_local + .get_mut(base..base + clear.local_vars_num as usize) + { + for local_var in local_vars.iter_mut() { + if !local_var.is_empty() { + *local_var = Variable::default(); + } + } + } else { + debug_assert!(false, "Failed to clear local variables: {clear:?}"); + } + } + if clear.match_vars != 0 { + self.clear_match_variables(clear.match_vars); + } + } + + fn exec_include>( + &mut self, + script: &'x Sieve<'x>, + include: &ops::Include, + handler: &mut H, + ) -> Result { + let name = self.eval_value(script, include.value)?.into_string(); + if name.is_empty() { + return Ok(Flow::Continue); + } + let name = self.intern_cow(name); + let script_name = if include.global { + Script::Global(name) + } else { + Script::Personal(name) + }; + if include.once && self.included.contains(&script_name) { + return Ok(Flow::Continue); + } + if self.frames.len() >= self.runtime.max_nested_includes { + return Err(RuntimeError::TooManyIncludes); + } + if let Some(cached) = self.runtime.include_scripts.get(name) { + self.push_included(script_name, cached); + return Ok(Flow::Switch); + } + match handler.include_script(self, script_name, include.optional) { + Reply::Ready(Some(included)) => { + self.push_included(script_name, included); + Ok(Flow::Switch) + } + Reply::Ready(None) if include.optional => Ok(Flow::Continue), + Reply::Ready(None) => Err(RuntimeError::ScriptNotFound(name.to_string())), + Reply::Error(err) => Err(err), + Reply::Pending => { + self.pending = Pending::Include { + name: script_name, + optional: include.optional, + }; + Ok(Flow::Pending) + } + } + } + + #[inline(always)] + fn next_part(&mut self) -> Option { + let mut cursor = self.part_iter; + let part = self.advance(&mut cursor); + self.part_iter = cursor; + part.map(|part| part.id()) + } + + pub fn set_envelope( + &mut self, + envelope: impl TryInto, + value: impl Into>, + ) { + if let Ok(envelope) = envelope.try_into() { + if matches!(&envelope, Envelope::From | Envelope::To) { + let value: Cow = value.into(); + if let Some(value) = parse_envelope_address(value.as_ref()) { + let value = self.alloc_str(value); + self.envelope.push((envelope, Variable::borrowed(value))); + } + } else { + self.envelope.push((envelope, Variable::from(value.into()))); + } + } + } + + pub fn with_vars_env(mut self, vars_env: AHashMap, Variable<'x>>) -> Self { + self.vars_env = vars_env; + self + } + + pub fn with_envelope_list(mut self, envelope: Vec<(Envelope, Variable<'x>)>) -> Self { + self.envelope = envelope; + self + } + + pub fn with_envelope( + mut self, + envelope: impl TryInto, + value: impl Into>, + ) -> Self { + self.set_envelope(envelope, value); + self + } + + pub fn clear_envelope(&mut self) { + self.envelope.clear() + } + + pub fn set_user_address(&mut self, from: impl Into>) { + self.user_address = from.into(); + } + + pub fn with_user_address(mut self, from: impl Into>) -> Self { + self.set_user_address(from); + self + } + + pub fn set_user_full_name(&mut self, name: &str) { + let mut name_ = String::with_capacity(name.len()); + for ch in name.chars() { + if ['\"', '\\'].contains(&ch) { + name_.push('\\'); + } + name_.push(ch); + } + self.user_full_name = name_.into(); + } + + pub fn with_user_full_name(mut self, name: &str) -> Self { + self.set_user_full_name(name); + self + } + + pub fn set_env_variable( + &mut self, + name: impl Into>, + value: impl Into>, + ) { + self.vars_env.insert(name.into(), value.into()); + } + + pub fn with_env_variable( + mut self, + name: impl Into>, + value: impl Into>, + ) -> Self { + self.set_env_variable(name, value); + self + } + + pub fn set_global_variable( + &mut self, + name: impl Into>, + value: impl Into>, + ) { + self.vars_global.insert(name.into(), value.into()); + } + + pub fn with_global_variable( + mut self, + name: impl Into>, + value: impl Into>, + ) -> Self { + self.set_global_variable(name, value); + self + } + + pub fn set_medatata( + &mut self, + name: impl Into>, + value: impl Into>, + ) { + self.metadata.push((name.into(), value.into())); + } + + pub fn with_metadata( + mut self, + name: impl Into>, + value: impl Into>, + ) -> Self { + self.set_medatata(name, value); + self + } + + pub fn set_spam_status(&mut self, status: impl Into) { + self.spam_status = status.into(); + } + + pub fn with_spam_status(mut self, status: impl Into) -> Self { + self.set_spam_status(status); + self + } + + pub fn set_virus_status(&mut self, status: impl Into) { + self.virus_status = status.into(); + } + + pub fn with_virus_status(mut self, status: impl Into) -> Self { + self.set_virus_status(status); + self + } + + pub fn set_current_time(&mut self, time: i64) { + self.current_time = time; + } + + pub fn with_current_time(mut self, time: i64) -> Self { + self.current_time = time; + self + } + + pub fn has_message_changed(&self) -> bool { + self.main_message_id > 0 + } + + pub(crate) fn user_from_field(&self) -> String { + if !self.user_full_name.is_empty() { + format!("\"{}\" <{}>", self.user_full_name, self.user_address) + } else { + self.user_address.to_string() + } + } + + pub fn global_variable_names(&self) -> impl Iterator { + self.vars_global.keys().map(|k| k.as_ref()) + } + + pub fn global_variable(&self, name: &str) -> Option<&Variable<'_>> { + self.vars_global.get(name) + } + + pub fn message(&self) -> &'x Message<'x> { + self.message + } + + pub fn part(&self) -> PartId { + self.part + } + + pub fn runtime(&self) -> &'x Runtime { + self.runtime + } + + pub fn instructions_executed(&self) -> usize { + self.num_instructions + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Flow { + Continue, + Jump(usize), + Switch, + Pending, +} + +#[inline(always)] +pub(crate) unsafe fn extend<'x, T: ?Sized>(value: &T) -> &'x T { + unsafe { &*(value as *const T) } +} + +impl Capability { + pub(crate) fn from_rec(script: &Sieve<'_>, rec: Rec) -> Result { + if rec.tag != tag::CAPABILITY { + return Err(RuntimeError::InvalidBytecode); + } + Ok(match rec.b { + 5 => { + Capability::Comparator(crate::compiler::grammar::Comparator::from_code(rec.c as u8)) + } + 6 => Capability::Other(script.str(rec.str())?.to_string()), + id => Capability::from_id(id), + }) + } +} + +#[cfg(all(test, stalwart_tests))] +impl<'x> Context<'x> { + pub(crate) fn with_runtime(self, runtime: &'x Runtime) -> Context<'x> { + Context { runtime, ..self } + } + + pub(crate) fn set_message(&mut self, message: &'x Message<'x>) { + self.message_size = message.raw().len(); + self.message = if message.part(ROOT_PART).is_some() { + message + } else { + self.arena.keep_message(empty_message()) + }; + self.edits = Edits::default(); + self.reset_body_cache(); + self.part = ROOT_PART; + } + + pub(crate) fn exec_test_cmd>( + &mut self, + script: &'x Sieve<'x>, + arguments: crate::bytecode::rec::Range, + is_not: bool, + handler: &mut H, + ) -> Result { + let arguments = self.eval_values(script, arguments)?; + match handler.function(self, u32::MAX, &arguments) { + Reply::Ready(value) => Ok(super::tests::TestResult::Bool(value.to_bool() ^ is_not)), + Reply::Pending => Ok(super::tests::TestResult::Pending { is_not }), + Reply::Error(err) => Err(err), + } + } +} diff --git a/crates/sieve/src/runtime/eval.rs b/crates/sieve/src/runtime/eval.rs new file mode 100644 index 00000000..9a3139c2 --- /dev/null +++ b/crates/sieve/src/runtime/eval.rs @@ -0,0 +1,763 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::message::{body::TextForm, headers::HeaderRef}; +use super::{RuntimeError, Variable}; +use crate::{ + Context, Envelope, Sieve, + bytecode::{ + Corrupt, Decoded, + rec::{Range, Rec, tag}, + }, + compiler::{ReceivedHostname, ReceivedPart, grammar::AddressPart}, +}; +use bumpalo::Bump; +use mail_parser::{ + ContentType, HeaderForm, HeaderName, HeaderValue, Host, Mailbox, PartKind, Received, +}; +use smallvec::SmallVec; +use std::cmp::Ordering; + +pub(crate) type Values<'x> = SmallVec<[Variable<'x>; 4]>; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum ValueRef<'x> { + Text(&'x str), + Int(i64), + Float(f64), + Local(u16), + Match(u8), + Global(&'x str), + Env(&'x str), + Envelope(Envelope), + Part { kind: u8, convert: bool }, + Header(HeaderVar<'x>), + Regex { pattern: &'x str }, + Glob { pattern: &'x str }, + HeaderName(&'x HeaderName<'static>), + List(Range), + None, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct HeaderVar<'x> { + pub names: Range, + pub part: HeaderPartRef<'x>, + pub index_hdr: i32, + pub index_part: i32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HeaderPartRef<'x> { + Text, + Date, + Id, + Address(AddressPart), + ContentType(ContentTypeRef<'x>), + Received(ReceivedPart), + Raw, + RawName, + Exists, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ContentTypeRef<'x> { + Type, + Subtype, + Attribute(&'x str), +} + +impl<'x> ValueRef<'x> { + pub(crate) fn decode( + script: &'x Sieve<'x>, + rec: Rec, + following: &mut impl Iterator, + ) -> Decoded> { + Ok(match rec.tag { + tag::TEXT => ValueRef::Text(script.str(rec.str())?), + tag::INT => ValueRef::Int(rec.e as i64), + tag::FLOAT => ValueRef::Float(f64::from_bits(rec.e)), + tag::VAR_LOCAL => ValueRef::Local(rec.c), + tag::VAR_MATCH => ValueRef::Match(rec.b), + tag::VAR_GLOBAL => ValueRef::Global(script.str(rec.str())?), + tag::VAR_ENV => ValueRef::Env(script.str(rec.str())?), + tag::VAR_ENVELOPE | tag::ENVELOPE => ValueRef::Envelope(Envelope::from_code(rec.b)), + tag::VAR_PART => ValueRef::Part { + kind: rec.b, + convert: rec.c != 0, + }, + tag::VAR_HEADER => { + let cont = following.next().ok_or(Corrupt)?; + if cont.tag != tag::CONT { + return Err(Corrupt); + } + let attr = if cont.e != 0 { + script.str(cont.str())? + } else { + "" + }; + ValueRef::Header(HeaderVar { + names: Range { + start: rec.d, + len: cont.c as u32, + }, + part: HeaderPartRef::decode(rec.b, rec.c, attr), + index_hdr: (rec.e >> 32) as u32 as i32, + index_part: rec.e as u32 as i32, + }) + } + tag::REF => { + let target = script.rec(rec.d)?; + let mut iter = script.recs(Range { + start: rec.d + 1, + len: rec.e as u32, + })?; + return ValueRef::decode(script, target, &mut iter); + } + tag::REGEX => ValueRef::Regex { + pattern: script.str(rec.str())?, + }, + tag::GLOB => ValueRef::Glob { + pattern: script.str(rec.str())?, + }, + tag::HEADER => ValueRef::HeaderName(script.header_name(rec.c)?), + tag::LIST => ValueRef::List(rec.range()), + tag::NONE | tag::VARIABLE_NONE => ValueRef::None, + _ => return Err(Corrupt), + }) + } +} + +impl ContentTypeRef<'_> { + fn select<'y>(&self, ct: ContentType<'y>) -> Option<&'y str> { + match self { + ContentTypeRef::Type => Some(ct.ctype()), + ContentTypeRef::Subtype => ct.subtype(), + ContentTypeRef::Attribute(attr) => ct + .attributes() + .find(|(name, _)| name.eq_ignore_ascii_case(attr)) + .map(|(_, value)| value), + } + } +} + +impl<'x> HeaderPartRef<'x> { + fn decode(part: u8, sub: u16, attr: &'x str) -> Self { + match part { + 0 => HeaderPartRef::Text, + 1 => HeaderPartRef::Date, + 2 => HeaderPartRef::Id, + 3 => HeaderPartRef::Address(AddressPart::from_code(sub as u8)), + 4 => HeaderPartRef::ContentType(match sub as u8 { + 0 => ContentTypeRef::Type, + 1 => ContentTypeRef::Subtype, + _ => ContentTypeRef::Attribute(attr), + }), + 5 => HeaderPartRef::Received(ReceivedPart::from_code( + (sub & 0xff) as u8, + (sub >> 8) as u8, + )), + 6 => HeaderPartRef::Raw, + 7 => HeaderPartRef::RawName, + _ => HeaderPartRef::Exists, + } + } +} + +impl<'x> Context<'x> { + #[inline(always)] + pub(crate) fn eval_value( + &self, + script: &'x Sieve<'x>, + rec: Rec, + ) -> Result, RuntimeError> { + match rec.tag { + tag::TEXT => Ok(Variable::borrowed(script.str(rec.str())?)), + tag::VAR_LOCAL => Ok(self.local_variable(rec.c)), + _ => { + let value = ValueRef::decode(script, rec, &mut std::iter::empty())?; + self.eval_value_ref(script, value) + } + } + } + + pub(crate) fn eval_value_ref( + &self, + script: &'x Sieve<'x>, + value: ValueRef<'x>, + ) -> Result, RuntimeError> { + Ok(match value { + ValueRef::Text(text) => Variable::borrowed(text), + ValueRef::Int(n) => Variable::Integer(n), + ValueRef::Float(n) => Variable::Float(n), + ValueRef::List(range) => { + let mut data = ArenaString::new_in(&self.arena.bump); + let mut iter = script.recs(range)?; + while let Some(item) = iter.next() { + match item.tag { + tag::TEXT => data.push_str(script.str(item.str())?)?, + tag::VAR_LOCAL => { + if let Some(value) = + self.vars_local.get(self.local_base() + item.c as usize) + { + data.push_variable(value)?; + } + } + tag::INT => data.push_str(&(item.e as i64).to_string())?, + tag::FLOAT => data.push_str(&f64::from_bits(item.e).to_string())?, + tag::REGEX | tag::GLOB => (), + tag::HEADER => data.push_str(script.header_name(item.c)?.as_str())?, + _ => { + let item = ValueRef::decode(script, item, &mut iter)?; + if let Some(value) = self.variable_ref(script, item)? { + data.push_variable(&value)?; + } + } + } + } + Variable::borrowed(unsafe { super::context::extend(data.into_str()) }) + } + ValueRef::Regex { pattern, .. } | ValueRef::Glob { pattern, .. } => { + Variable::borrowed(pattern) + } + ValueRef::HeaderName(name) => Variable::borrowed(name.as_str()), + ValueRef::None => Variable::default(), + other => self.variable_ref(script, other)?.unwrap_or_default(), + }) + } + + #[inline(always)] + pub(crate) fn local_variable(&self, id: u16) -> Variable<'x> { + self.vars_local + .get(self.local_base() + id as usize) + .cloned() + .unwrap_or_default() + } + + #[inline(always)] + pub(crate) fn match_variable(&self, id: u8) -> Variable<'x> { + self.vars_match + .get(self.match_base() + id as usize) + .cloned() + .unwrap_or_default() + } + + pub(crate) fn variable_ref( + &self, + script: &'x Sieve<'x>, + value: ValueRef<'x>, + ) -> Result>, RuntimeError> { + Ok(match value { + ValueRef::Local(id) => self + .vars_local + .get(self.local_base() + id as usize) + .cloned(), + ValueRef::Match(id) => self + .vars_match + .get(self.match_base() + id as usize) + .cloned(), + ValueRef::Global(name) => self.vars_global.get(name).cloned(), + ValueRef::Env(name) => self + .vars_env + .get(name) + .or_else(|| self.runtime.environment.get(name)) + .cloned(), + ValueRef::Envelope(envelope) => self.envelope.iter().find_map(|(e, v)| { + if *e == envelope { + Some(v.clone()) + } else { + None + } + }), + ValueRef::Header(header) => self.eval_header(script, &header)?, + ValueRef::Part { kind, convert } => self.eval_part(kind, convert), + ValueRef::Text(text) => Some(Variable::borrowed(text)), + ValueRef::Int(n) => Some(Variable::Integer(n)), + ValueRef::Float(n) => Some(Variable::Float(n)), + other => Some(self.eval_value_ref(script, other)?), + }) + } + + fn eval_part(&self, kind: u8, convert: bool) -> Option> { + let text = match kind { + 0 => { + let part = self.message.root().text_body().next()?; + match self.part_kind(part) { + PartKind::Text => self.part_text(part, TextForm::Source), + PartKind::Html if convert => self.part_text(part, TextForm::Plain), + _ => None, + } + } + 1 => { + let part = self.message.root().html_body().next()?; + match self.part_kind(part) { + PartKind::Html => self.part_text(part, TextForm::Source), + PartKind::Text if convert => self.part_text(part, TextForm::Html), + _ => None, + } + } + 2 => { + let part = self.current_part()?; + match self.part_kind(part) { + PartKind::Text | PartKind::Html => self.part_text(part, TextForm::Source), + PartKind::Binary | PartKind::InlineBinary => { + self.part_text(part, TextForm::Decoded) + } + _ => None, + } + } + _ => self + .current_part() + .and_then(|part| self.part_text(part, TextForm::Raw)), + }; + text.map(Variable::borrowed) + } + + pub(crate) fn eval_values( + &self, + script: &'x Sieve<'x>, + range: Range, + ) -> Result, RuntimeError> { + let mut result = Values::with_capacity(range.len as usize); + let mut iter = script.recs(range)?; + while let Some(rec) = iter.next() { + let value = ValueRef::decode(script, rec, &mut iter)?; + result.push(self.eval_value_ref(script, value)?); + } + Ok(result) + } + + pub(crate) fn eval_strings( + &self, + script: &'x Sieve<'x>, + range: Range, + ) -> Result, RuntimeError> { + let mut result = SmallVec::with_capacity(range.len as usize); + let mut iter = script.recs(range)?; + while let Some(rec) = iter.next() { + let value = ValueRef::decode(script, rec, &mut iter)?; + let value = self.eval_value_ref(script, value)?; + result.push(self.intern_cow(value.into_string())); + } + Ok(result) + } + + pub(crate) fn eval_opt( + &self, + script: &'x Sieve<'x>, + rec: Rec, + ) -> Result>, RuntimeError> { + if rec.tag == tag::NONE { + Ok(None) + } else { + self.eval_value(script, rec).map(Some) + } + } + + pub(crate) fn eval_str( + &self, + script: &'x Sieve<'x>, + rec: Rec, + ) -> Result<&'x str, RuntimeError> { + Ok(self.intern_cow(self.eval_value(script, rec)?.into_string())) + } + + pub(crate) fn eval_opt_str( + &self, + script: &'x Sieve<'x>, + rec: Rec, + ) -> Result, RuntimeError> { + if rec.tag == tag::NONE { + Ok(None) + } else { + self.eval_str(script, rec).map(Some) + } + } + + fn eval_header( + &self, + script: &'x Sieve<'x>, + header: &HeaderVar<'x>, + ) -> Result>, RuntimeError> { + let mut result: Values<'x> = SmallVec::new(); + let Some(part) = self.current_part() else { + return Ok(None); + }; + if !header.names.is_empty() { + let mut names: SmallVec<[&HeaderName<'static>; 2]> = SmallVec::new(); + for rec in script.recs(header.names)? { + names.push(script.header_name(rec.c)?); + } + match (header.index_hdr.cmp(&0), names.as_slice()) { + (Ordering::Less, [name]) if header.index_hdr == -1 => { + if let Some(h) = self.last_header(part, name) { + self.eval_header_part(header, h, &mut result); + } + } + (Ordering::Less, [name]) => { + if let Some(h) = self + .part_headers(part) + .rev() + .filter(|h| h.is_named(name)) + .nth((header.index_hdr.unsigned_abs() - 1) as usize) + { + self.eval_header_part(header, h, &mut result); + } + } + (Ordering::Greater, [name]) => { + if let Some(h) = self + .named_headers(part, name) + .nth((header.index_hdr - 1) as usize) + { + self.eval_header_part(header, h, &mut result); + } + } + (Ordering::Equal, [name]) => { + for h in self.named_headers(part, name) { + self.eval_header_part(header, h, &mut result); + } + } + (index, names) => { + let matching = self.matching_headers(part, names); + let selected = match index { + Ordering::Greater => matching.get((header.index_hdr - 1) as usize), + Ordering::Less => matching + .iter() + .rev() + .nth((header.index_hdr.unsigned_abs() - 1) as usize), + Ordering::Equal => { + for h in &matching { + self.eval_header_part(header, *h, &mut result); + } + None + } + }; + if let Some(h) = selected { + self.eval_header_part(header, *h, &mut result); + } + } + } + } else { + let source = part.message().source_bytes(); + for h in self.part_headers(part) { + match &header.part { + HeaderPartRef::Raw => { + let field = match h { + HeaderRef::Added(_) => &[][..], + h => raw_field(source, h), + }; + result.push(Variable::borrowed( + self.alloc_string(sanitize_raw_header(field)), + )); + } + HeaderPartRef::Text => { + let parsed; + let text = match h.value() { + HeaderValue::Text(text) => Some(text), + _ => { + parsed = h.parse_as(HeaderForm::Text); + parsed.value().as_text() + } + }; + if let Some(text) = text { + result.push(Variable::borrowed(self.alloc_string(format!( + "{}: {}", + h.name().as_str(), + text + )))); + } + } + _ => { + self.eval_header_part(header, h, &mut result); + } + } + } + } + + match result.len() { + 1 if header.index_hdr != 0 && header.index_part != 0 => Ok(result.pop()), + 0 => Ok(None), + _ => self + .try_alloc_variables(result) + .map(|items| Some(Variable::Array(super::variable::Array::Borrowed(items)))), + } + } + + fn eval_header_part(&self, header: &HeaderVar<'x>, h: HeaderRef<'x>, result: &mut Values<'x>) { + let index_part = header.index_part; + let var = match &header.part { + HeaderPartRef::Text => match h.value() { + HeaderValue::Text(v) if [-1, 0, 1].contains(&index_part) => { + Some(Variable::borrowed(v)) + } + HeaderValue::TextList(list) => { + match pick(list.iter(), index_part, result, |item| { + Some(Variable::borrowed(item)) + }) { + Some(var) => var, + None => return, + } + } + HeaderValue::ContentType(ct) => Some(Variable::borrowed(match ct.subtype() { + Some(subtype) => self.alloc_string(format!("{}/{}", ct.ctype(), subtype)), + None => ct.ctype(), + })), + HeaderValue::Address(list) => { + match pick(list.mailboxes(), index_part, result, |mailbox| { + Some(self.addr_to_text(mailbox)) + }) { + Some(var) => var, + None => return, + } + } + HeaderValue::DateTime(_) => h + .raw_value() + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + .map(|s| Variable::borrowed(s.trim())), + _ => None, + }, + HeaderPartRef::Address(part) => match h.value() { + HeaderValue::Address(list) => { + match pick(list.mailboxes(), index_part, result, |mailbox| { + self.addr_part(part, mailbox) + }) { + Some(var) => var, + None => return, + } + } + HeaderValue::Text(_) => { + let parsed = h.parse_as(HeaderForm::Addresses); + let Some(list) = parsed.value().as_address() else { + result.push(Variable::default()); + return; + }; + match pick(list.mailboxes(), index_part, result, |mailbox| { + part.eval_strict(mailbox) + .map(|s| Variable::borrowed(self.alloc_str(s))) + }) { + Some(var) => var, + None => return, + } + } + _ => None, + }, + HeaderPartRef::Date => match h.value() { + HeaderValue::DateTime(dt) => Some(Variable::from(dt.to_timestamp())), + _ => h + .parse_as(HeaderForm::Date) + .value() + .as_datetime() + .map(|dt| Variable::from(dt.to_timestamp())), + }, + HeaderPartRef::Id => { + let name = h.name(); + if matches!(name, HeaderName::MessageId | HeaderName::ResentMessageId) { + match h.value() { + HeaderValue::Text(id) => Some(Variable::borrowed(id)), + HeaderValue::TextList(ids) => { + result.extend(ids.iter().map(Variable::borrowed)); + return; + } + _ => None, + } + } else if name.is_structured() { + None + } else { + let parsed = h.parse_as(HeaderForm::MessageIds); + match parsed.value() { + HeaderValue::Text(id) => Some(Variable::borrowed(self.alloc_str(id))), + HeaderValue::TextList(ids) if !ids.is_empty() => { + result.extend( + ids.iter().map(|id| Variable::borrowed(self.alloc_str(id))), + ); + return; + } + _ => None, + } + } + } + HeaderPartRef::Raw => Some(Variable::borrowed( + self.alloc_string(sanitize_raw_header(h.raw_value().unwrap_or_default())), + )), + HeaderPartRef::RawName => match h { + HeaderRef::Added(_) => None, + h => Some(Variable::borrowed(h.raw_name())), + }, + HeaderPartRef::Exists => Some(Variable::from(true)), + HeaderPartRef::ContentType(part) => match h { + HeaderRef::Added(_) => h + .parse_as(HeaderForm::ContentType) + .value() + .as_content_type() + .and_then(|ct| part.select(ct)) + .map(|value| Variable::borrowed(self.alloc_str(value))), + h => h + .value() + .as_content_type() + .and_then(|ct| part.select(ct)) + .map(Variable::borrowed), + }, + HeaderPartRef::Received(part) => match h.value() { + HeaderValue::Received(rcvd) => self.received_part(part, rcvd), + _ => None, + }, + }; + + result.push(var.unwrap_or_default()); + } + + fn addr_to_text(&self, mailbox: Mailbox<'x>) -> Variable<'x> { + match (mailbox.name(), mailbox.address()) { + (Some(name), Some(address)) => { + Variable::borrowed(self.alloc_string(format!("{name} <{address}>"))) + } + (Some(name), None) => Variable::borrowed(name), + (None, Some(address)) => Variable::borrowed(self.alloc_string(format!("<{address}>"))), + (None, None) => Variable::default(), + } + } + + fn addr_part(&self, part: &AddressPart, mailbox: Mailbox<'x>) -> Option> { + match part { + AddressPart::Name => mailbox.name().map(Variable::borrowed), + AddressPart::All => mailbox.address().map(Variable::borrowed), + _ => part.eval_strict(mailbox).map(Variable::borrowed), + } + } + + pub fn received_part(&self, part: &ReceivedPart, rcvd: Received<'x>) -> Option> { + match part { + ReceivedPart::From(from) => rcvd + .from() + .or_else(|| rcvd.helo()) + .and_then(|host| self.host_variable(from, host)), + ReceivedPart::FromIp => rcvd + .from_ip() + .map(|ip| Variable::borrowed(self.alloc_string(ip.to_string()))), + ReceivedPart::FromIpRev => rcvd.from_iprev().map(Variable::borrowed), + ReceivedPart::By(by) => rcvd.by().and_then(|host| self.host_variable(by, host)), + ReceivedPart::For => rcvd.for_().map(Variable::borrowed), + ReceivedPart::With => rcvd.with().map(|v| Variable::borrowed(v.as_str())), + ReceivedPart::TlsVersion => rcvd.tls_version().map(|v| Variable::borrowed(v.as_str())), + ReceivedPart::TlsCipher => rcvd.tls_cipher().map(Variable::borrowed), + ReceivedPart::Id => rcvd.id().map(Variable::borrowed), + ReceivedPart::Ident => rcvd.ident().map(Variable::borrowed), + ReceivedPart::Via => rcvd.via().map(Variable::borrowed), + ReceivedPart::Date => rcvd.date().map(|d| Variable::from(d.to_timestamp())), + ReceivedPart::DateRaw => rcvd + .date() + .map(|d| Variable::borrowed(self.alloc_string(d.to_rfc822()))), + } + } + + fn host_variable(&self, hostname: &ReceivedHostname, host: Host<'x>) -> Option> { + match (hostname, host) { + (ReceivedHostname::Name | ReceivedHostname::Any, Host::Name(name)) => { + Some(Variable::borrowed(name)) + } + (ReceivedHostname::Ip | ReceivedHostname::Any, Host::IpAddr(ip)) => { + Some(Variable::borrowed(self.alloc_string(ip.to_string()))) + } + _ => None, + } + } +} + +fn raw_field<'x>(source: &'x [u8], header: HeaderRef<'x>) -> &'x [u8] { + match header { + HeaderRef::Parsed(header) | HeaderRef::Renamed(header, _) => source + .get(header.offset_field() as usize..header.offset_end() as usize) + .unwrap_or_default(), + HeaderRef::Added(_) => &[], + } +} + +fn pick<'x, T>( + mut items: impl DoubleEndedIterator, + index: i32, + result: &mut Values<'x>, + mut map: impl FnMut(T) -> Option>, +) -> Option>> { + match index.cmp(&0) { + Ordering::Greater => Some(items.nth((index - 1) as usize).and_then(map)), + Ordering::Less => Some( + items + .rev() + .nth((index.unsigned_abs() - 1) as usize) + .and_then(map), + ), + Ordering::Equal => { + result.extend(items.map(|item| map(item).unwrap_or_default())); + None + } + } +} + +struct ArenaString<'a> { + bytes: bumpalo::collections::Vec<'a, u8>, +} + +impl<'a> ArenaString<'a> { + #[inline(always)] + fn new_in(arena: &'a Bump) -> Self { + ArenaString { + bytes: bumpalo::collections::Vec::new_in(arena), + } + } + + #[inline(always)] + fn push_str(&mut self, s: &str) -> Result<(), RuntimeError> { + self.bytes + .try_reserve(s.len()) + .map_err(|_| RuntimeError::MemoryLimitReached)?; + self.bytes.extend_from_slice(s.as_bytes()); + Ok(()) + } + + fn push_variable(&mut self, value: &Variable<'_>) -> Result<(), RuntimeError> { + match value { + Variable::String(s) => self.push_str(s), + Variable::Integer(n) => self.push_str(&n.to_string()), + Variable::Float(n) => self.push_str(&n.to_string()), + Variable::Array(items) => self.push_str(&super::variable::array_to_string(items)), + } + } + + #[inline(always)] + fn into_str(self) -> &'a str { + unsafe { std::str::from_utf8_unchecked(self.bytes.into_bump_slice()) } + } +} + +pub(crate) trait IntoString: Sized { + fn into_string(self) -> String; +} + +impl IntoString for Vec { + fn into_string(self) -> String { + String::from_utf8(self) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) + } +} + +pub(crate) fn sanitize_raw_header(bytes: &[u8]) -> String { + let mut result = Vec::with_capacity(bytes.len()); + let mut last_is_space = false; + + for &ch in bytes { + if ch.is_ascii_whitespace() { + last_is_space = true; + } else { + if last_is_space { + result.push(b' '); + last_is_space = false; + } + result.push(ch); + } + } + + result.into_string() +} diff --git a/crates/sieve/src/runtime/expression.rs b/crates/sieve/src/runtime/expression.rs new file mode 100644 index 00000000..274e1c27 --- /dev/null +++ b/crates/sieve/src/runtime/expression.rs @@ -0,0 +1,335 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::variable::Array; +use super::{ + RuntimeError, Variable, + context::Pending, + eval::ValueRef, + handler::{Handler, Reply}, +}; +use crate::{ + Context, Sieve, + bytecode::rec::{Range, tag}, + compiler::grammar::expr::{BinaryOperator, UnaryOperator, parser::ID_EXTERNAL}, +}; +use std::borrow::Cow; + +impl<'x> Context<'x> { + pub(crate) fn eval_expression>( + &mut self, + script: &'x Sieve<'x>, + range: Range, + handler: &mut H, + ) -> Result>, RuntimeError> { + let mut pos = self.expr_pos; + let mut iter = script.recs(Range { + start: range.start + pos as u32, + len: range.len.saturating_sub(pos as u32), + })?; + + while let Some(rec) = iter.next() { + pos += 1; + match rec.tag { + tag::VAR_LOCAL => { + let value = self.local_variable(rec.c); + self.expr_stack.push(value); + } + tag::VAR_MATCH => { + let value = self.match_variable(rec.b); + self.expr_stack.push(value); + } + tag::INT => { + self.expr_stack.push(Variable::Integer(rec.e as i64)); + } + tag::FLOAT => { + self.expr_stack.push(Variable::Float(f64::from_bits(rec.e))); + } + tag::TEXT => { + self.expr_stack + .push(Variable::borrowed(script.str(rec.str())?)); + } + tag::UN_OP => { + let value = self.expr_stack.pop().unwrap_or_default(); + self.expr_stack.push(match UnaryOperator::from_code(rec.b) { + UnaryOperator::Not => value.op_not(), + UnaryOperator::Minus => value.op_minus(), + }); + } + tag::BIN_OP => { + let right = self.expr_stack.pop().unwrap_or_default(); + let left = self.expr_stack.pop().unwrap_or_default(); + self.expr_stack + .push(match BinaryOperator::from_code(rec.b) { + BinaryOperator::Add => left.op_add(right), + BinaryOperator::Subtract => left.op_subtract(right), + BinaryOperator::Multiply => left.op_multiply(right), + BinaryOperator::Divide => left.op_divide(right), + BinaryOperator::And => left.op_and(right), + BinaryOperator::Or => left.op_or(right), + BinaryOperator::Xor => left.op_xor(right), + BinaryOperator::Eq => left.op_eq(right), + BinaryOperator::Ne => left.op_ne(right), + BinaryOperator::Lt => left.op_lt(right), + BinaryOperator::Le => left.op_le(right), + BinaryOperator::Gt => left.op_gt(right), + BinaryOperator::Ge => left.op_ge(right), + }); + } + tag::CALL => { + let num_args = rec.c as usize; + let start = self.expr_stack.len().saturating_sub(num_args); + if let Some(fnc) = self.runtime.functions.get(rec.d as usize) { + let result = (fnc)(self, &self.expr_stack[start..]); + self.expr_stack.truncate(start); + let result = self.intern(result); + self.expr_stack.push(result); + } else { + let id = ID_EXTERNAL + .checked_sub(rec.d) + .ok_or(RuntimeError::InvalidBytecode)?; + let reply = handler.function(self, id, &self.expr_stack[start..]); + self.expr_stack.truncate(start); + match reply { + Reply::Ready(result) => { + let result = self.intern(result); + self.expr_stack.push(result); + } + Reply::Pending => { + self.expr_pos = pos; + self.pending = Pending::Function; + return Ok(None); + } + Reply::Error(err) => return Err(err), + } + } + } + tag::JMP_IF => { + if self.expr_stack.last().is_some_and(|v| v.to_bool()) == (rec.b != 0) { + pos += rec.d as usize; + if pos > range.len as usize { + return Err(RuntimeError::InvalidBytecode); + } + iter = script.recs(Range { + start: range.start + pos as u32, + len: range.len.saturating_sub(pos as u32), + })?; + } + } + tag::ARRAY_ACCESS => { + let index = self.expr_stack.pop().unwrap_or_default().to_usize(); + let value = match self.expr_stack.pop() { + Some(Variable::Array(array)) => { + array.get(index).cloned().unwrap_or_default() + } + Some(value) if index == 0 && !value.is_empty() => value, + _ => Variable::default(), + }; + self.expr_stack.push(value); + } + tag::ARRAY_BUILD => { + let start = self.expr_stack.len().saturating_sub(rec.d as usize); + let items = self.expr_stack.split_off(start); + self.expr_stack.push(Variable::Array(Array::Owned(items))); + } + _ => { + let value = ValueRef::decode(script, rec, &mut iter)?; + if matches!(value, ValueRef::Header(_)) { + pos += 1; + } + let value = self.variable_ref(script, value)?.unwrap_or_default(); + self.expr_stack.push(value); + } + } + } + + let result = self.expr_stack.pop().unwrap_or_default(); + self.expr_stack.clear(); + self.expr_pos = 0; + Ok(Some(result)) + } +} + +impl<'x> Variable<'x> { + pub fn op_add(self, other: Variable<'x>) -> Variable<'x> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_add(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a + b), + (Variable::Integer(i), Variable::Float(f)) + | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 + f), + (Variable::Array(a), Variable::Array(b)) => { + Variable::Array(Array::Owned(a.iter().chain(b.iter()).cloned().collect())) + } + (Variable::Array(a), b) => { + Variable::Array(Array::Owned(a.iter().cloned().chain([b]).collect())) + } + (a, Variable::Array(b)) => Variable::Array(Array::Owned( + [a].into_iter().chain(b.iter().cloned()).collect(), + )), + (Variable::String(a), b) => { + if a.is_empty() { + b + } else if b.is_empty() { + Variable::String(a) + } else { + let b = b.to_string(); + let mut result = a.into_owned(); + result.push_str(&b); + Variable::String(Cow::Owned(result)) + } + } + (a, Variable::String(b)) => { + if b.is_empty() { + a + } else { + let a = a.to_string(); + let mut result = String::with_capacity(a.len() + b.len()); + result.push_str(&a); + result.push_str(&b); + Variable::String(Cow::Owned(result)) + } + } + } + } + + pub fn op_subtract(self, other: Variable<'x>) -> Variable<'x> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_sub(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a - b), + (Variable::Integer(a), Variable::Float(b)) => Variable::Float(a as f64 - b), + (Variable::Float(a), Variable::Integer(b)) => Variable::Float(a - b as f64), + (Variable::Array(a), b) | (b, Variable::Array(a)) => Variable::Array(Array::Owned( + a.iter().filter(|v| *v != &b).cloned().collect(), + )), + (a, b) => a.parse_number().op_subtract(b.parse_number()), + } + } + + pub fn op_multiply(self, other: Variable<'x>) -> Variable<'x> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_mul(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a * b), + (Variable::Integer(i), Variable::Float(f)) + | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 * f), + (a, b) => a.parse_number().op_multiply(b.parse_number()), + } + } + + pub fn op_divide(self, other: Variable<'x>) -> Variable<'x> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => { + Variable::Float(if b != 0 { a as f64 / b as f64 } else { 0.0 }) + } + (Variable::Float(a), Variable::Float(b)) => { + Variable::Float(if b != 0.0 { a / b } else { 0.0 }) + } + (Variable::Integer(a), Variable::Float(b)) => { + Variable::Float(if b != 0.0 { a as f64 / b } else { 0.0 }) + } + (Variable::Float(a), Variable::Integer(b)) => { + Variable::Float(if b != 0 { a / b as f64 } else { 0.0 }) + } + (a, b) => a.parse_number().op_divide(b.parse_number()), + } + } + + pub fn op_and(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self.to_bool() & other.to_bool())) + } + + pub fn op_or(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self.to_bool() | other.to_bool())) + } + + pub fn op_xor(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self.to_bool() ^ other.to_bool())) + } + + pub fn op_eq(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self == other)) + } + + pub fn op_ne(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self != other)) + } + + pub fn op_lt(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self < other)) + } + + pub fn op_le(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self <= other)) + } + + pub fn op_gt(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self > other)) + } + + pub fn op_ge(self, other: Variable<'x>) -> Variable<'x> { + Variable::Integer(i64::from(self >= other)) + } + + pub fn op_not(self) -> Variable<'x> { + Variable::Integer(i64::from(!self.to_bool())) + } + + pub fn op_minus(self) -> Variable<'x> { + match self { + Variable::Integer(n) => Variable::Integer(n.saturating_neg()), + Variable::Float(n) => Variable::Float(-n), + _ => self.parse_number().op_minus(), + } + } + + pub fn parse_number(&self) -> Variable<'x> { + match self { + Variable::String(s) if !s.is_empty() => { + if let Ok(n) = s.parse::() { + Variable::Integer(n) + } else if let Ok(n) = s.parse::() { + Variable::Float(n) + } else { + Variable::Integer(0) + } + } + Variable::Integer(n) => Variable::Integer(*n), + Variable::Float(n) => Variable::Float(*n), + Variable::Array(l) => Variable::Integer(l.is_empty() as i64), + _ => Variable::Integer(0), + } + } +} + +impl BinaryOperator { + #[inline(always)] + pub(crate) fn from_code(code: u8) -> BinaryOperator { + match code { + 0 => BinaryOperator::Add, + 1 => BinaryOperator::Subtract, + 2 => BinaryOperator::Multiply, + 3 => BinaryOperator::Divide, + 4 => BinaryOperator::And, + 5 => BinaryOperator::Or, + 6 => BinaryOperator::Xor, + 7 => BinaryOperator::Eq, + 8 => BinaryOperator::Ne, + 9 => BinaryOperator::Lt, + 10 => BinaryOperator::Le, + 11 => BinaryOperator::Gt, + _ => BinaryOperator::Ge, + } + } +} + +impl UnaryOperator { + #[inline(always)] + pub(crate) fn from_code(code: u8) -> UnaryOperator { + match code { + 0 => UnaryOperator::Not, + _ => UnaryOperator::Minus, + } + } +} diff --git a/crates/sieve/src/runtime/handler.rs b/crates/sieve/src/runtime/handler.rs new file mode 100644 index 00000000..79fc4265 --- /dev/null +++ b/crates/sieve/src/runtime/handler.rs @@ -0,0 +1,204 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Context, Envelope, ExternalId, Importance, MatchAs, Sieve, + compiler::grammar::actions::action_redirect::{ByTime, Notify, Ret}, + runtime::{RuntimeError, Variable}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Reply { + Ready(T), + Pending, + Error(RuntimeError), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Finished, + Pending, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Script<'a> { + Personal(&'a str), + Global(&'a str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Mailbox<'a> { + Name(&'a str), + Id(&'a str), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Recipient<'a> { + Address(&'a str), + List(&'a str), + Group(Vec<&'a str>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MessageSource { + Redirect, + Vacation, + Notification, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Action<'a> { + Keep { + flags: &'a [&'a str], + message_id: usize, + }, + Discard, + Reject { + extended: bool, + reason: &'a str, + }, + FileInto { + folder: &'a str, + flags: &'a [&'a str], + mailbox_id: Option<&'a str>, + special_use: Option<&'a str>, + create: bool, + message_id: usize, + }, + SendMessage { + source: MessageSource, + recipient: Recipient<'a>, + notify: Notify, + return_of_content: Ret, + by_time: ByTime, + message_id: usize, + }, + Notify { + from: Option<&'a str>, + importance: Importance, + options: &'a [&'a str], + message: &'a str, + method: &'a str, + }, + SetEnvelope { + envelope: Envelope, + value: &'a str, + }, + CreatedMessage { + message_id: usize, + message: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum Input<'x> { + Bool(bool), + Value(Variable<'x>), + Script(Option<&'x Sieve<'x>>), + Continue, +} + +impl From for Input<'_> { + fn from(value: bool) -> Self { + Input::Bool(value) + } +} + +impl<'x> From> for Input<'x> { + fn from(value: Variable<'x>) -> Self { + Input::Value(value) + } +} + +impl<'x> From>> for Input<'x> { + fn from(value: Option<&'x Sieve<'x>>) -> Self { + Input::Script(value) + } +} + +impl<'x> From<&'x Sieve<'x>> for Input<'x> { + fn from(value: &'x Sieve<'x>) -> Self { + Input::Script(Some(value)) + } +} + +impl From for Reply { + fn from(value: T) -> Self { + Reply::Ready(value) + } +} + +impl<'a> Script<'a> { + pub fn name(&self) -> &'a str { + match self { + Script::Personal(name) | Script::Global(name) => name, + } + } + + pub fn is_global(&self) -> bool { + matches!(self, Script::Global(_)) + } +} + +impl std::fmt::Display for Script<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.name()) + } +} + +#[allow(unused_variables)] +pub trait Handler<'x> { + fn include_script( + &mut self, + ctx: &Context<'x>, + name: Script<'_>, + optional: bool, + ) -> Reply>> { + Reply::Ready(None) + } + + fn mailbox_exists( + &mut self, + ctx: &Context<'x>, + mailboxes: &[Mailbox<'_>], + special_use: &[&str], + ) -> Reply { + Reply::Ready(false) + } + + fn list_contains( + &mut self, + ctx: &Context<'x>, + lists: &[&str], + values: &[&str], + match_as: MatchAs, + ) -> Reply { + Reply::Ready(false) + } + + fn duplicate_id( + &mut self, + ctx: &Context<'x>, + id: &str, + expiry: u64, + last: bool, + ) -> Reply { + Reply::Ready(false) + } + + fn function( + &mut self, + ctx: &Context<'x>, + id: ExternalId, + arguments: &[Variable<'x>], + ) -> Reply> { + Reply::Ready(Variable::default()) + } + + fn action(&mut self, ctx: &Context<'x>, action: Action<'x>) -> Reply<()> { + Reply::Ready(()) + } +} diff --git a/crates/sieve/src/runtime/message/body.rs b/crates/sieve/src/runtime/message/body.rs new file mode 100644 index 00000000..c0a195e2 --- /dev/null +++ b/crates/sieve/src/runtime/message/body.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::edits::{BodyEdit, BodyKind}; +use crate::Context; +use mail_parser::{MessagePart, PartId, PartKind, html_to_text, text_to_html}; +use std::borrow::Cow; + +pub(crate) const TEXT_FORMS: usize = 6; + +pub(crate) type BodyCache<'x> = Vec<[Option<&'x str>; TEXT_FORMS]>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum TextForm { + Source = 0, + Plain = 1, + Html = 2, + Decoded = 3, + Raw = 4, + Markup = 5, +} + +impl<'x> Context<'x> { + #[inline] + pub(crate) fn part_kind(&self, part: MessagePart<'x>) -> PartKind<'x> { + match self.edits.body(part.id()) { + Some(body) => body.kind.into(), + None => part.kind(), + } + } + + pub(crate) fn part_text(&self, part: MessagePart<'x>, form: TextForm) -> Option<&'x str> { + let index = part.id() as usize; + if let Some(text) = self + .body_cache + .borrow() + .get(index) + .and_then(|forms| forms.get(form as usize).copied().flatten()) + { + return Some(text); + } + let text = match self.edits.body(part.id()) { + Some(body) => self.edited_text(*body, form)?, + None => self.decode_text(part, form)?, + }; + let mut cache = self.body_cache.borrow_mut(); + if cache.len() <= index { + cache.resize( + self.message.parts().len().max(index + 1), + [None; TEXT_FORMS], + ); + } + if let Some(slot) = cache + .get_mut(index) + .and_then(|forms| forms.get_mut(form as usize)) + { + *slot = Some(text); + } + Some(text) + } + + fn decode_text(&self, part: MessagePart<'x>, form: TextForm) -> Option<&'x str> { + match form { + TextForm::Source => part.text().map(|text| self.keep_text(text)), + TextForm::Plain => match part.kind() { + PartKind::Text => self.part_text(part, TextForm::Source), + PartKind::Html => self + .part_text(part, TextForm::Source) + .map(|html| self.keep_text(html_to_text(html).into())), + _ => None, + }, + TextForm::Html => match part.kind() { + PartKind::Html => self.part_text(part, TextForm::Source), + PartKind::Text => self + .part_text(part, TextForm::Source) + .map(|text| self.keep_text(text_to_html(text).into())), + _ => None, + }, + TextForm::Decoded => Some(self.keep_bytes(part.decoded())), + TextForm::Raw => Some(self.keep_bytes(Cow::Borrowed(part.raw_body()))), + TextForm::Markup => self + .part_text(part, TextForm::Decoded) + .map(|markup| self.keep_text(html_to_text(markup).into())), + } + } + + fn edited_text(&self, body: BodyEdit<'x>, form: TextForm) -> Option<&'x str> { + match (form, body.kind) { + (TextForm::Plain, BodyKind::Html) => Some(self.keep_charged(html_to_text(body.text))), + (TextForm::Html, BodyKind::Text) => Some(self.keep_charged(text_to_html(body.text))), + (TextForm::Markup, _) => None, + _ => Some(body.text), + } + } + + #[inline] + pub(crate) fn keep_text(&self, text: Cow<'x, str>) -> &'x str { + match text { + Cow::Borrowed(text) => text, + Cow::Owned(text) => self.arena.keep_text(text), + } + } + + fn keep_charged(&self, text: String) -> &'x str { + self.arena.keep_charged_text(text).unwrap_or_else(|| { + self.note_oom(); + "" + }) + } + + fn keep_bytes(&self, bytes: Cow<'x, [u8]>) -> &'x str { + match bytes { + Cow::Borrowed(bytes) => self.keep_text(String::from_utf8_lossy(bytes)), + Cow::Owned(bytes) => self.arena.keep_text( + String::from_utf8(bytes) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()), + ), + } + } + + pub(crate) fn set_part_body(&mut self, part: PartId, body: BodyEdit<'x>) { + self.edits.set_body(part, body); + if let Some(forms) = self.body_cache.get_mut().get_mut(part as usize) { + *forms = [None; TEXT_FORMS]; + } + } + + pub(crate) fn reset_body_cache(&mut self) { + self.body_cache.get_mut().clear(); + } +} + +impl From for PartKind<'_> { + fn from(kind: BodyKind) -> Self { + match kind { + BodyKind::Text => PartKind::Text, + BodyKind::Html => PartKind::Html, + } + } +} diff --git a/crates/sieve/src/runtime/message/edits.rs b/crates/sieve/src/runtime/message/edits.rs new file mode 100644 index 00000000..e8c6281b --- /dev/null +++ b/crates/sieve/src/runtime/message/edits.rs @@ -0,0 +1,207 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::headers::HeaderRef; +use encodify::{base64, qp}; +use mail_parser::{HeaderName, MessageParser, MessagePart, PartId}; +use std::ops::Range; + +const MAX_LINE_LEN: usize = 998; +const QP_SLACK_DIVISOR: usize = 4; + +#[derive(Default)] +pub(crate) struct Edits<'x> { + headers: Vec<(PartId, Vec>)>, + bodies: Vec<(PartId, BodyEdit<'x>)>, + hidden: Vec>, +} + +#[derive(Clone, Copy)] +pub(crate) struct BodyEdit<'x> { + pub(crate) text: &'x str, + pub(crate) kind: BodyKind, + pub(crate) format: BodyFormat, + pub(crate) encoding: TransferEncoding, + pub(crate) encoded_len: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BodyKind { + Text, + Html, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BodyFormat { + Content, + Entity { body_start: usize }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TransferEncoding { + Identity, + QuotedPrintable, + Base64, +} + +impl TransferEncoding { + pub(crate) fn select(text: &str) -> (Self, usize) { + let bytes = text.as_bytes(); + if bytes.is_ascii() && !has_long_line(bytes) { + return (TransferEncoding::Identity, bytes.len()); + } + let base64_len = base64::MIME.encoded_len(bytes.len()); + let qp_limit = base64_len.saturating_add(base64_len / QP_SLACK_DIVISOR); + match qp::BODY.encoded_len_within(bytes, qp_limit) { + Some(qp_len) => (TransferEncoding::QuotedPrintable, qp_len), + None => (TransferEncoding::Base64, base64_len), + } + } + + pub(crate) fn select_entity(entity: &str) -> (BodyFormat, Self, usize) { + let verbatim = ( + BodyFormat::Entity { body_start: 0 }, + TransferEncoding::Identity, + entity.len(), + ); + let Some(message) = MessageParser::new().parse_headers(entity) else { + return verbatim; + }; + let part = message.root_part(); + let is_composite = part.content_type().is_some_and(|content_type| { + hashify::set_ignore_case!(content_type.ctype().as_bytes(), "multipart", "message") + }); + if is_composite + || part + .headers() + .value(HeaderName::ContentTransferEncoding) + .is_some() + { + return verbatim; + } + let body_start = part.offset_body() as usize; + let Some(body) = entity.get(body_start..) else { + return verbatim; + }; + match TransferEncoding::select(body) { + (TransferEncoding::Identity, _) => verbatim, + (encoding, encoded_len) => ( + BodyFormat::Entity { body_start }, + encoding, + body_start + encoded_len, + ), + } + } + + pub(crate) fn header_value(self) -> Option<&'static str> { + match self { + TransferEncoding::Identity => None, + TransferEncoding::QuotedPrintable => Some("quoted-printable"), + TransferEncoding::Base64 => Some("base64"), + } + } + + pub(crate) fn write(self, text: &str, out: &mut Vec) { + match self { + TransferEncoding::Identity => out.extend_from_slice(text.as_bytes()), + TransferEncoding::QuotedPrintable => { + qp::BODY.encode_append(text, out); + } + TransferEncoding::Base64 => { + base64::MIME.encode_append(text, out); + } + } + } +} + +fn has_long_line(bytes: &[u8]) -> bool { + if bytes.len() <= MAX_LINE_LEN { + return false; + } + let mut line_start = 0; + memchr::memchr_iter(b'\n', bytes).any(|line_end| { + let is_long = line_end - line_start > MAX_LINE_LEN; + line_start = line_end + 1; + is_long + }) || bytes.len() - line_start > MAX_LINE_LEN +} + +impl<'x> Edits<'x> { + #[inline(always)] + pub(crate) fn headers(&self, part: PartId) -> Option<&[HeaderRef<'x>]> { + if self.headers.is_empty() { + return None; + } + self.headers + .iter() + .find(|(id, _)| *id == part) + .map(|(_, headers)| headers.as_slice()) + } + + pub(crate) fn headers_mut(&mut self, part: MessagePart<'x>) -> &mut Vec> { + let id = part.id(); + let index = match self.headers.iter().position(|(edited, _)| *edited == id) { + Some(index) => index, + None => { + self.headers + .push((id, part.headers().iter().map(HeaderRef::Parsed).collect())); + self.headers.len() - 1 + } + }; + &mut self.headers[index].1 + } + + pub(crate) fn set_headers(&mut self, part: PartId, headers: Vec>) { + match self.headers.iter_mut().find(|(id, _)| *id == part) { + Some((_, edited)) => *edited = headers, + None => self.headers.push((part, headers)), + } + } + + #[inline(always)] + pub(crate) fn body(&self, part: PartId) -> Option<&BodyEdit<'x>> { + if self.bodies.is_empty() { + return None; + } + self.bodies + .iter() + .find(|(id, _)| *id == part) + .map(|(_, body)| body) + } + + pub(crate) fn set_body(&mut self, part: PartId, body: BodyEdit<'x>) { + match self.bodies.iter_mut().find(|(id, _)| *id == part) { + Some((_, edited)) => *edited = body, + None => self.bodies.push((part, body)), + } + } + + pub(crate) fn hide(&mut self, range: Range) { + if range.is_empty() { + return; + } + self.headers.retain(|(id, _)| !range.contains(id)); + self.bodies.retain(|(id, _)| !range.contains(id)); + self.hidden.retain(|hidden| !range.contains(&hidden.start)); + self.hidden.push(range); + } + + #[inline(always)] + pub(crate) fn hidden_end(&self, part: PartId) -> Option { + if self.hidden.is_empty() { + return None; + } + self.hidden + .iter() + .find(|hidden| hidden.contains(&part)) + .map(|hidden| hidden.end) + } + + pub(crate) fn touches(&self, range: &Range) -> bool { + self.headers.iter().any(|(id, _)| range.contains(id)) + || self.bodies.iter().any(|(id, _)| range.contains(id)) + } +} diff --git a/crates/sieve/src/runtime/message/headers.rs b/crates/sieve/src/runtime/message/headers.rs new file mode 100644 index 00000000..cd5ed2c2 --- /dev/null +++ b/crates/sieve/src/runtime/message/headers.rs @@ -0,0 +1,315 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Context; +use mail_parser::{ + Header, HeaderForm, HeaderIter, HeaderKey, HeaderName, HeaderValue, MessagePart, NamedHeaders, + ParsedValue, +}; +use smallvec::SmallVec; +use std::slice; + +pub(crate) static ORIGINAL_SUBJECT: HeaderName<'static> = HeaderName::OriginalSubject; +pub(crate) static ORIGINAL_FROM: HeaderName<'static> = HeaderName::OriginalFrom; + +pub(crate) struct AddedHeader<'x> { + pub(crate) name: HeaderName<'x>, + pub(crate) value: &'x str, + content_type: Option<(&'x str, &'x str)>, +} + +#[derive(Clone, Copy)] +pub(crate) enum HeaderRef<'x> { + Parsed(Header<'x>), + Renamed(Header<'x>, &'x HeaderName<'x>), + Added(&'x AddedHeader<'x>), +} + +impl<'x> HeaderRef<'x> { + pub(crate) fn name(&self) -> HeaderName<'x> { + match self { + HeaderRef::Parsed(header) => header.name(), + HeaderRef::Renamed(_, name) => (*name).clone(), + HeaderRef::Added(added) => added.name.clone(), + } + } + + pub(crate) fn is_named(&self, name: &HeaderName<'_>) -> bool { + match self { + HeaderRef::Parsed(header) => header.name() == *name, + HeaderRef::Renamed(_, renamed) => *renamed == name, + HeaderRef::Added(added) => added.name == *name, + } + } + + pub(crate) fn value(&self) -> HeaderValue<'x> { + match self { + HeaderRef::Parsed(header) | HeaderRef::Renamed(header, _) => header.value(), + HeaderRef::Added(added) => HeaderValue::Text(added.value), + } + } + + pub(crate) fn parse_as(&self, form: HeaderForm) -> ParsedValue<'x> { + match self { + HeaderRef::Parsed(header) | HeaderRef::Renamed(header, _) => header.parse_as(form), + HeaderRef::Added(added) => form.parse(added.value.as_bytes()), + } + } + + fn position(&self) -> u32 { + match self { + HeaderRef::Parsed(header) | HeaderRef::Renamed(header, _) => header.offset_field(), + HeaderRef::Added(_) => u32::MAX, + } + } + + pub(crate) fn raw_value(&self) -> Option<&'x [u8]> { + match self { + HeaderRef::Parsed(header) | HeaderRef::Renamed(header, _) => Some(header.raw_value()), + HeaderRef::Added(_) => None, + } + } + + pub(crate) fn raw_name(&self) -> &'x str { + match self { + HeaderRef::Parsed(header) => header.raw_name(), + HeaderRef::Renamed(_, name) => name.as_str(), + HeaderRef::Added(added) => added.name.as_str(), + } + } + + pub(crate) fn len(&self) -> usize { + match self { + HeaderRef::Parsed(header) => { + header.offset_end().saturating_sub(header.offset_field()) as usize + } + HeaderRef::Renamed(header, name) => name.as_str().len() + 1 + header.raw_value().len(), + HeaderRef::Added(added) => added.name.as_str().len() + added.value.len() + 4, + } + } + + pub(crate) fn same(&self, other: &HeaderRef<'_>) -> bool { + match (self, other) { + ( + HeaderRef::Parsed(a) | HeaderRef::Renamed(a, _), + HeaderRef::Parsed(b) | HeaderRef::Renamed(b, _), + ) => a.offset_field() == b.offset_field() && a.offset_end() == b.offset_end(), + (HeaderRef::Added(a), HeaderRef::Added(b)) => std::ptr::eq(*a, *b), + _ => false, + } + } + + pub(crate) fn write(&self, source: &[u8], out: &mut Vec) { + match self { + HeaderRef::Parsed(header) => out.extend_from_slice( + source + .get(header.offset_field() as usize..header.offset_end() as usize) + .unwrap_or_default(), + ), + HeaderRef::Renamed(header, name) => { + out.extend_from_slice(name.as_str().as_bytes()); + out.push(b':'); + out.extend_from_slice(header.raw_value()); + } + HeaderRef::Added(added) => { + out.extend_from_slice(added.name.as_str().as_bytes()); + out.extend_from_slice(b": "); + out.extend_from_slice(added.value.as_bytes()); + out.extend_from_slice(b"\r\n"); + } + } + } +} + +pub(crate) enum PartHeaders<'y, 'x> { + Parsed(HeaderIter<'x>), + Edited(slice::Iter<'y, HeaderRef<'x>>), +} + +impl<'x> Iterator for PartHeaders<'_, 'x> { + type Item = HeaderRef<'x>; + + #[inline] + fn next(&mut self) -> Option> { + match self { + PartHeaders::Parsed(iter) => iter.next().map(HeaderRef::Parsed), + PartHeaders::Edited(iter) => iter.next().copied(), + } + } +} + +impl<'x> DoubleEndedIterator for PartHeaders<'_, 'x> { + #[inline] + fn next_back(&mut self) -> Option> { + match self { + PartHeaders::Parsed(iter) => iter.next_back().map(HeaderRef::Parsed), + PartHeaders::Edited(iter) => iter.next_back().copied(), + } + } +} + +pub(crate) enum NamedPartHeaders<'y, 'x> { + Parsed(NamedHeaders<'x, 'y>), + Edited(slice::Iter<'y, HeaderRef<'x>>, &'y HeaderName<'y>), +} + +impl<'x> Iterator for NamedPartHeaders<'_, 'x> { + type Item = HeaderRef<'x>; + + #[inline] + fn next(&mut self) -> Option> { + match self { + NamedPartHeaders::Parsed(iter) => iter.next().map(HeaderRef::Parsed), + NamedPartHeaders::Edited(iter, name) => { + iter.find(|header| header.is_named(name)).copied() + } + } + } +} + +impl<'x> Context<'x> { + #[inline] + pub(crate) fn part_headers(&self, part: MessagePart<'x>) -> PartHeaders<'_, 'x> { + match self.edits.headers(part.id()) { + Some(headers) => PartHeaders::Edited(headers.iter()), + None => PartHeaders::Parsed(part.headers().iter()), + } + } + + #[inline] + pub(crate) fn named_headers<'y>( + &'y self, + part: MessagePart<'x>, + name: &'y HeaderName<'y>, + ) -> NamedPartHeaders<'y, 'x> { + self.keyed_headers(part, name, name.key()) + } + + #[inline] + pub(crate) fn keyed_headers<'y>( + &'y self, + part: MessagePart<'x>, + name: &'y HeaderName<'y>, + key: HeaderKey<'y>, + ) -> NamedPartHeaders<'y, 'x> { + match self.edits.headers(part.id()) { + Some(headers) => NamedPartHeaders::Edited(headers.iter(), name), + None => NamedPartHeaders::Parsed(part.headers().all_key(key)), + } + } + + pub(crate) fn matching_headers( + &self, + part: MessagePart<'x>, + names: &[&HeaderName<'_>], + ) -> SmallVec<[HeaderRef<'x>; 8]> { + let mut matching: SmallVec<[HeaderRef<'x>; 8]> = SmallVec::new(); + if self.edits.headers(part.id()).is_some() { + matching.extend(self.part_headers(part).filter(|header| { + let name = header.name(); + names.iter().any(|wanted| **wanted == name) + })); + } else { + let headers = part.headers(); + for name in names { + matching.extend(headers.all_key(name.key()).map(HeaderRef::Parsed)); + } + matching.sort_unstable_by_key(HeaderRef::position); + matching.dedup_by_key(|header| header.position()); + } + matching + } + + pub(crate) fn root_headers(&self) -> PartHeaders<'_, 'x> { + match self.root_part() { + Some(part) => self.part_headers(part), + None => PartHeaders::Edited([].iter()), + } + } + + pub(crate) fn last_header( + &self, + part: MessagePart<'x>, + name: &HeaderName<'_>, + ) -> Option> { + match self.edits.headers(part.id()) { + Some(headers) => headers + .iter() + .rev() + .find(|header| header.is_named(name)) + .copied(), + None => part.headers().get_key(name.key()).map(HeaderRef::Parsed), + } + } + + pub(crate) fn root_text(&self, name: &HeaderName<'_>) -> Option<&'x str> { + self.root_part() + .and_then(|part| self.last_header(part, name)) + .and_then(|header| header.value().as_text()) + } + + pub(crate) fn part_content_type(&self, part: MessagePart<'x>) -> Option<(&'x str, &'x str)> { + if self.edits.headers(part.id()).is_none() { + return part + .content_type() + .map(|ct| (ct.ctype(), ct.subtype().unwrap_or_default())); + } + match self.last_header(part, &HeaderName::ContentType)? { + HeaderRef::Added(added) => added.content_type, + header => header + .value() + .as_content_type() + .map(|ct| (ct.ctype(), ct.subtype().unwrap_or_default())), + } + } + + pub(crate) fn renamed_header( + &self, + header: HeaderRef<'x>, + name: &'x HeaderName<'x>, + ) -> HeaderRef<'x> { + match header { + HeaderRef::Parsed(parsed) | HeaderRef::Renamed(parsed, _) => { + HeaderRef::Renamed(parsed, name) + } + HeaderRef::Added(added) => self.add_header(name.clone(), added.value), + } + } + + pub(crate) fn add_header(&self, name: HeaderName<'x>, value: &'x str) -> HeaderRef<'x> { + let content_type = if name == HeaderName::ContentType { + HeaderForm::ContentType + .parse(value.as_bytes()) + .value() + .as_content_type() + .map(|ct| { + ( + self.alloc_str(ct.ctype()), + self.alloc_str(ct.subtype().unwrap_or_default()), + ) + }) + } else { + None + }; + match self.arena.bump.try_alloc(AddedHeader { + name, + value, + content_type, + }) { + Ok(added) => HeaderRef::Added(unsafe { super::super::context::extend(added) }), + Err(_) => { + self.note_oom(); + HeaderRef::Added(&EMPTY_HEADER) + } + } + } +} + +static EMPTY_HEADER: AddedHeader<'static> = AddedHeader { + name: HeaderName::Other(std::borrow::Cow::Borrowed("")), + value: "", + content_type: None, +}; diff --git a/tests/src/cluster/mod.rs b/crates/sieve/src/runtime/message/mod.rs similarity index 54% rename from tests/src/cluster/mod.rs rename to crates/sieve/src/runtime/message/mod.rs index 95c16049..34c07855 100644 --- a/tests/src/cluster/mod.rs +++ b/crates/sieve/src/runtime/message/mod.rs @@ -4,5 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod broadcast; -pub mod stress; +pub(crate) mod body; +pub(crate) mod edits; +pub(crate) mod headers; +pub(crate) mod parts; +pub(crate) mod serialize; diff --git a/crates/sieve/src/runtime/message/parts.rs b/crates/sieve/src/runtime/message/parts.rs new file mode 100644 index 00000000..ce0be99f --- /dev/null +++ b/crates/sieve/src/runtime/message/parts.rs @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Context; +use mail_parser::{Message, MessageParser, MessagePart, PartId}; + +pub(crate) const ROOT_PART: PartId = 0; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct PartCursor { + next: PartId, + end: PartId, + nested: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Scope { + Message, + Nested, +} + +impl PartCursor { + pub(crate) fn subtree(part: MessagePart<'_>, include_self: bool, scope: Scope) -> Self { + let nested = scope == Scope::Nested; + PartCursor { + next: part.id() + u32::from(!include_self), + end: if nested || !part.is_message() { + subtree_end(part) + } else { + part.id() + 1 + }, + nested, + } + } +} + +pub(crate) fn empty_message() -> Message<'static> { + MessageParser::new() + .parse_owned(b"\r\n".to_vec()) + .unwrap_or_default() +} + +pub(crate) fn subtree_end(part: MessagePart<'_>) -> PartId { + let first = part.id(); + let mut last = part; + loop { + if let Some(child) = last.children().next_back() { + if child.id() <= last.id() { + break; + } + last = child; + } else if let Some(nested) = last.nested() { + let root = nested.root_part(); + if root.id() <= last.id() || root.id() == PartId::MAX { + break; + } + last = root; + } else { + break; + } + } + last.id().max(first).saturating_add(1) +} + +impl<'x> Context<'x> { + #[inline(always)] + pub(crate) fn root_part(&self) -> Option> { + self.message.part(ROOT_PART) + } + + #[inline(always)] + pub(crate) fn current_part(&self) -> Option> { + self.message.part(self.part) + } + + #[inline] + pub(crate) fn advance(&self, cursor: &mut PartCursor) -> Option> { + while cursor.next < cursor.end { + let id = cursor.next; + if let Some(end) = self.edits.hidden_end(id) { + cursor.next = end; + continue; + } + let Some(part) = self.message.part(id) else { + cursor.next = cursor.end; + return None; + }; + cursor.next = if !cursor.nested && part.is_message() { + subtree_end(part) + } else { + id + 1 + }; + return Some(part); + } + None + } +} diff --git a/crates/sieve/src/runtime/message/serialize.rs b/crates/sieve/src/runtime/message/serialize.rs new file mode 100644 index 00000000..ef8339d4 --- /dev/null +++ b/crates/sieve/src/runtime/message/serialize.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + edits::BodyFormat, + headers::HeaderRef, + parts::{ROOT_PART, subtree_end}, +}; +use crate::Context; +use mail_parser::MessagePart; + +impl<'x> Context<'x> { + pub(crate) fn build_message(&self) -> Vec { + let raw = self.message.raw(); + let mut message = Vec::with_capacity(self.message_size); + match self.root_part() { + Some(root) => { + message.extend_from_slice(slice(raw, 0, root.offset_header())); + self.write_part(&mut message, root); + message + .extend_from_slice(raw.get(root.offset_end() as usize..).unwrap_or_default()); + } + None => message.extend_from_slice(raw), + } + message + } + + fn write_part(&self, out: &mut Vec, part: MessagePart<'x>) { + let id = part.id(); + if !self.edits.touches(&(id..subtree_end(part))) { + out.extend_from_slice(part.raw()); + return; + } + + let source = part.message().source_bytes(); + let body = self.edits.body(id); + match self.edits.headers(id) { + Some(headers) => { + let (mut count_left, mut size_left) = + if id == ROOT_PART && self.runtime.has_header_block_limits() { + headers + .iter() + .filter(|header| !matches!(header, HeaderRef::Added(_))) + .fold( + ( + self.runtime.max_header_count, + self.runtime.max_header_block_size, + ), + |(count, size), header| { + (count.saturating_sub(1), size.saturating_sub(header.len())) + }, + ) + } else { + (usize::MAX, usize::MAX) + }; + for header in headers { + if matches!(header, HeaderRef::Added(_)) { + let len = header.len(); + if count_left == 0 || size_left < len { + count_left = 0; + continue; + } + count_left -= 1; + size_left -= len; + } + header.write(source, out); + } + if !body.is_some_and(|body| matches!(body.format, BodyFormat::Entity { .. })) { + out.extend_from_slice(b"\r\n"); + } + } + None => out.extend_from_slice(part.raw_headers()), + } + + if let Some(body) = body { + let (verbatim, content) = match body.format { + BodyFormat::Content => ("", body.text), + BodyFormat::Entity { body_start } => body + .text + .split_at_checked(body_start) + .unwrap_or((body.text, "")), + }; + out.extend_from_slice(verbatim.as_bytes()); + body.encoding.write(content, out); + } else if part.is_multipart() { + let mut offset = part.offset_body(); + for child in part.children() { + out.extend_from_slice(slice(source, offset, child.offset_header())); + self.write_part(out, child); + offset = child.offset_end(); + } + out.extend_from_slice(slice(source, offset, part.offset_end())); + } else { + out.extend_from_slice(part.raw_body()); + } + } +} + +#[inline(always)] +fn slice(bytes: &[u8], start: u32, end: u32) -> &[u8] { + bytes.get(start as usize..end as usize).unwrap_or_default() +} diff --git a/crates/sieve/src/runtime/mod.rs b/crates/sieve/src/runtime/mod.rs new file mode 100644 index 00000000..1f57462a --- /dev/null +++ b/crates/sieve/src/runtime/mod.rs @@ -0,0 +1,583 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod actions; +mod arena; +pub mod context; +pub mod eval; +pub mod expression; +pub mod handler; +pub(crate) mod message; +pub(crate) mod platform; +pub mod tests; +pub mod variable; +pub mod variables; + +pub use arena::Arena; +pub use handler::{ + Action, Handler, Input, Mailbox, MessageSource, Recipient, Reply, Script, Status, +}; +pub use variable::Variable; + +use crate::{ + ExternalId, Function, FunctionMap, Metadata, Runtime, Sieve, + bytecode::Corrupt, + compiler::{ + Number, + grammar::{Capability, expr::parser::ID_EXTERNAL}, + }, +}; +use ahash::{AHashMap, AHashSet}; +use mail_parser::{HeaderName, Message, MessageParser}; +use std::borrow::Cow; + +use crate::Context; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuntimeError { + TooManyIncludes, + ScriptNotFound(String), + InvalidInstruction { + name: String, + line_num: u32, + line_pos: u32, + }, + ScriptErrorMessage(String), + CapabilityNotAllowed(Capability), + CapabilityNotSupported(String), + CPULimitReached, + MemoryLimitReached, + InvalidBytecode, + AwaitingInput, +} + +impl From for RuntimeError { + fn from(_: Corrupt) -> Self { + RuntimeError::InvalidBytecode + } +} + +impl Number { + pub fn is_non_zero(&self) -> bool { + match self { + Number::Integer(n) => *n != 0, + Number::Float(n) => *n != 0.0, + } + } +} + +impl Default for Number { + fn default() -> Self { + Number::Integer(0) + } +} + +impl From for Number { + #[inline(always)] + fn from(b: bool) -> Self { + Number::Integer(i64::from(b)) + } +} + +impl From for Number { + #[inline(always)] + fn from(n: i64) -> Self { + Number::Integer(n) + } +} + +impl From for Number { + #[inline(always)] + fn from(n: f64) -> Self { + Number::Float(n) + } +} + +impl From for Number { + #[inline(always)] + fn from(n: i32) -> Self { + Number::Integer(n as i64) + } +} + +impl PartialEq for Number { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Integer(a), Self::Integer(b)) => a == b, + (Self::Float(a), Self::Float(b)) => a == b, + (Self::Integer(a), Self::Float(b)) => (*a as f64) == *b, + (Self::Float(a), Self::Integer(b)) => *a == (*b as f64), + } + } +} + +impl Eq for Number {} + +impl PartialOrd for Number { + fn partial_cmp(&self, other: &Self) -> Option { + let (a, b) = match (self, other) { + (Number::Integer(a), Number::Integer(b)) => return a.partial_cmp(b), + (Number::Float(a), Number::Float(b)) => (*a, *b), + (Number::Integer(a), Number::Float(b)) => (*a as f64, *b), + (Number::Float(a), Number::Integer(b)) => (*a, *b as f64), + }; + a.partial_cmp(&b) + } +} + +impl Runtime { + pub fn filter<'z: 'x, 'x>( + &'z self, + raw_message: &[u8], + script: &'x Sieve<'x>, + arena: &'x mut Arena, + ) -> Context<'x> { + arena.prepare(self.memory_limit); + let message = match MessageParser::new().parse_owned(raw_message.to_vec()) { + Some(message) => arena.keep_message(message), + None => arena.keep_message(Message::default()), + }; + let mut context = Context::with_message(self, message, script, arena); + context.message_size = raw_message.len(); + context + } + + pub fn filter_parsed<'z: 'x, 'x>( + &'z self, + message: &'x Message<'x>, + script: &'x Sieve<'x>, + arena: &'x mut Arena, + ) -> Context<'x> { + Context::new(self, message, script, arena) + } +} + +impl Default for Runtime { + fn default() -> Self { + Self::new() + } +} + +impl Runtime { + pub fn new() -> Self { + #[allow(unused_mut)] + let mut allowed_capabilities = AHashSet::from_iter(Capability::all().iter().cloned()); + + #[cfg(all(test, stalwart_tests))] + allowed_capabilities.insert(Capability::Other("vnd.inbuxa.testsuite".to_string())); + + Runtime { + allowed_capabilities, + environment: AHashMap::from_iter([ + ("name".into(), "Stalwart Sieve".into()), + ("version".into(), env!("CARGO_PKG_VERSION").into()), + ]), + metadata: Vec::new(), + include_scripts: AHashMap::new(), + max_nested_includes: 3, + cpu_limit: 5000, + memory_limit: 32 * 1024 * 1024, + max_variable_size: 4096, + max_redirects: 1, + max_received_headers: 10, + protected_headers: vec![HeaderName::OriginalSubject, HeaderName::OriginalFrom], + valid_notification_uris: AHashSet::new(), + valid_ext_lists: AHashSet::new(), + vacation_use_orig_rcpt: false, + vacation_default_subject: "Automated reply".into(), + vacation_subject_prefix: "Auto: ".into(), + max_header_size: 1024, + max_header_count: usize::MAX, + max_header_block_size: usize::MAX, + max_out_messages: 3, + default_vacation_expiry: 30 * 86400, + default_duplicate_expiry: 7 * 86400, + local_hostname: "localhost".into(), + functions: Vec::new(), + } + } + + pub fn set_cpu_limit(&mut self, size: usize) { + self.cpu_limit = size; + } + + pub fn with_cpu_limit(mut self, size: usize) -> Self { + self.cpu_limit = size; + self + } + + pub fn set_memory_limit(&mut self, size: usize) { + self.memory_limit = size; + } + + pub fn with_memory_limit(mut self, size: usize) -> Self { + self.memory_limit = size; + self + } + + pub fn set_max_nested_includes(&mut self, size: usize) { + self.max_nested_includes = size; + } + + pub fn with_max_nested_includes(mut self, size: usize) -> Self { + self.max_nested_includes = size; + self + } + + pub fn set_max_redirects(&mut self, size: usize) { + self.max_redirects = size; + } + + pub fn with_max_redirects(mut self, size: usize) -> Self { + self.max_redirects = size; + self + } + + pub fn set_max_out_messages(&mut self, size: usize) { + self.max_out_messages = size; + } + + pub fn with_max_out_messages(mut self, size: usize) -> Self { + self.max_out_messages = size; + self + } + + pub fn set_max_received_headers(&mut self, size: usize) { + self.max_received_headers = size; + } + + pub fn with_max_received_headers(mut self, size: usize) -> Self { + self.max_received_headers = size; + self + } + + pub fn set_max_variable_size(&mut self, size: usize) { + self.max_variable_size = size; + } + + pub fn with_max_variable_size(mut self, size: usize) -> Self { + self.max_variable_size = size; + self + } + + pub fn set_max_header_size(&mut self, size: usize) { + self.max_header_size = size; + } + + pub fn with_max_header_size(mut self, size: usize) -> Self { + self.max_header_size = size; + self + } + + pub fn set_max_header_count(&mut self, count: usize) { + self.max_header_count = count; + } + + pub fn with_max_header_count(mut self, count: usize) -> Self { + self.max_header_count = count; + self + } + + pub fn set_max_header_block_size(&mut self, size: usize) { + self.max_header_block_size = size; + } + + pub fn with_max_header_block_size(mut self, size: usize) -> Self { + self.max_header_block_size = size; + self + } + + #[inline(always)] + pub(crate) fn has_header_block_limits(&self) -> bool { + self.max_header_count != usize::MAX || self.max_header_block_size != usize::MAX + } + + pub fn set_default_vacation_expiry(&mut self, expiry: u64) { + self.default_vacation_expiry = expiry; + } + + pub fn with_default_vacation_expiry(mut self, expiry: u64) -> Self { + self.default_vacation_expiry = expiry; + self + } + + pub fn set_default_duplicate_expiry(&mut self, expiry: u64) { + self.default_duplicate_expiry = expiry; + } + + pub fn with_default_duplicate_expiry(mut self, expiry: u64) -> Self { + self.default_duplicate_expiry = expiry; + self + } + + pub fn set_capability(&mut self, capability: impl Into) { + self.allowed_capabilities.insert(capability.into()); + } + + pub fn with_capability(mut self, capability: impl Into) -> Self { + self.set_capability(capability); + self + } + + pub fn unset_capability(&mut self, capability: impl Into) { + self.allowed_capabilities.remove(&capability.into()); + } + + pub fn without_capability(mut self, capability: impl Into) -> Self { + self.unset_capability(capability); + self + } + + pub fn without_capabilities( + mut self, + capabilities: impl IntoIterator>, + ) -> Self { + for capability in capabilities { + self.allowed_capabilities.remove(&capability.into()); + } + self + } + + pub fn set_protected_header(&mut self, header_name: impl Into>) { + if let Some(header_name) = HeaderName::parse(header_name) { + self.protected_headers.push(header_name); + } + } + + pub fn with_protected_header(mut self, header_name: impl Into>) -> Self { + self.set_protected_header(header_name); + self + } + + pub fn with_protected_headers( + mut self, + header_names: impl IntoIterator>>, + ) -> Self { + self.protected_headers = header_names + .into_iter() + .filter_map(HeaderName::parse) + .collect(); + self + } + + pub fn set_env_variable( + &mut self, + name: impl Into>, + value: impl Into>, + ) { + self.environment.insert(name.into(), value.into()); + } + + pub fn with_env_variable( + mut self, + name: impl Into>, + value: impl Into>, + ) -> Self { + self.set_env_variable(name, value); + self + } + + pub fn set_medatata( + &mut self, + name: impl Into>, + value: impl Into>, + ) { + self.metadata.push((name.into(), value.into())); + } + + pub fn with_metadata( + mut self, + name: impl Into>, + value: impl Into>, + ) -> Self { + self.set_medatata(name, value); + self + } + + pub fn set_valid_notification_uri(&mut self, uri: impl Into>) { + self.valid_notification_uris.insert(uri.into()); + } + + pub fn with_valid_notification_uri(mut self, uri: impl Into>) -> Self { + self.valid_notification_uris.insert(uri.into()); + self + } + + pub fn with_valid_notification_uris( + mut self, + uris: impl IntoIterator>>, + ) -> Self { + self.valid_notification_uris = uris.into_iter().map(Into::into).collect(); + self + } + + pub fn set_valid_ext_list(&mut self, name: impl Into>) { + self.valid_ext_lists.insert(name.into()); + } + + pub fn with_valid_ext_list(mut self, name: impl Into>) -> Self { + self.set_valid_ext_list(name); + self + } + + pub fn set_vacation_use_orig_rcpt(&mut self, value: bool) { + self.vacation_use_orig_rcpt = value; + } + + pub fn with_valid_ext_lists( + mut self, + lists: impl IntoIterator>>, + ) -> Self { + self.valid_ext_lists = lists.into_iter().map(Into::into).collect(); + self + } + + pub fn with_vacation_use_orig_rcpt(mut self, value: bool) -> Self { + self.set_vacation_use_orig_rcpt(value); + self + } + + pub fn set_vacation_default_subject(&mut self, value: impl Into>) { + self.vacation_default_subject = value.into(); + } + + pub fn with_vacation_default_subject(mut self, value: impl Into>) -> Self { + self.set_vacation_default_subject(value); + self + } + + pub fn set_vacation_subject_prefix(&mut self, value: impl Into>) { + self.vacation_subject_prefix = value.into(); + } + + pub fn with_vacation_subject_prefix(mut self, value: impl Into>) -> Self { + self.set_vacation_subject_prefix(value); + self + } + + pub fn set_local_hostname(&mut self, value: impl Into>) { + self.local_hostname = value.into(); + } + + pub fn with_local_hostname(mut self, value: impl Into>) -> Self { + self.set_local_hostname(value); + self + } + + pub fn with_functions(mut self, fnc_map: &mut FunctionMap) -> Self { + self.functions = std::mem::take(&mut fnc_map.functions); + self + } + + pub fn set_functions(&mut self, fnc_map: &mut FunctionMap) { + self.functions = std::mem::take(&mut fnc_map.functions); + } + + pub fn set_include_script(&mut self, name: impl Into, script: Sieve<'static>) { + self.include_scripts.insert(name.into(), script); + } + + pub fn with_include_script(mut self, name: impl Into, script: Sieve<'static>) -> Self { + self.set_include_script(name, script); + self + } + + pub fn include_script(&self, name: &str) -> Option<&Sieve<'static>> { + self.include_scripts.get(name) + } +} + +impl FunctionMap { + pub fn new() -> Self { + FunctionMap { + map: Default::default(), + functions: Default::default(), + } + } + + pub fn with_function(self, name: impl Into, fnc: Function) -> Self { + self.with_function_args(name, fnc, 1) + } + + pub fn with_function_no_args(self, name: impl Into, fnc: Function) -> Self { + self.with_function_args(name, fnc, 0) + } + + pub fn with_function_args( + mut self, + name: impl Into, + fnc: Function, + num_args: u32, + ) -> Self { + self.map + .insert(name.into(), (self.functions.len() as u32, num_args)); + self.functions.push(fnc); + self + } + + pub fn with_external_function( + mut self, + name: impl Into, + id: ExternalId, + num_args: u32, + ) -> Self { + self.set_external_function(name, id, num_args); + self + } + + pub fn set_external_function( + &mut self, + name: impl Into, + id: ExternalId, + num_args: u32, + ) { + self.map.insert(name.into(), (ID_EXTERNAL - id, num_args)); + } +} + +impl Metadata { + pub fn server(annotation: impl Into) -> Self { + Metadata::Server { + annotation: annotation.into(), + } + } + + pub fn mailbox(name: impl Into, annotation: impl Into) -> Self { + Metadata::Mailbox { + name: name.into(), + annotation: annotation.into(), + } + } +} + +impl From for Metadata { + fn from(annotation: String) -> Self { + Metadata::Server { annotation } + } +} + +impl From<&'_ str> for Metadata { + fn from(annotation: &'_ str) -> Self { + Metadata::Server { + annotation: annotation.to_string(), + } + } +} + +impl From<(String, String)> for Metadata { + fn from((name, annotation): (String, String)) -> Self { + Metadata::Mailbox { name, annotation } + } +} + +impl From<(&'_ str, &'_ str)> for Metadata { + fn from((name, annotation): (&'_ str, &'_ str)) -> Self { + Metadata::Mailbox { + name: name.to_string(), + annotation: annotation.to_string(), + } + } +} diff --git a/crates/sieve/src/runtime/platform.rs b/crates/sieve/src/runtime/platform.rs new file mode 100644 index 00000000..2208c6de --- /dev/null +++ b/crates/sieve/src/runtime/platform.rs @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use mail_builder::headers::message_id::generate_message_id_header; +#[cfg(not(all(test, stalwart_tests)))] +use mail_builder::mime; +use std::time::SystemTime; + +#[inline(always)] +pub(crate) fn unix_time() -> i64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) as i64 +} + +#[inline(always)] +#[cfg(not(all(test, stalwart_tests)))] +pub(crate) fn make_boundary() -> String { + mime::make_boundary(".") +} + +#[inline(always)] +pub(crate) fn write_message_id(output: &mut Vec, hostname: &str) { + generate_message_id_header(output, hostname); +} diff --git a/crates/sieve/src/runtime/tests/comparator.rs b/crates/sieve/src/runtime/tests/comparator.rs new file mode 100644 index 00000000..4d6726d2 --- /dev/null +++ b/crates/sieve/src/runtime/tests/comparator.rs @@ -0,0 +1,243 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + MatchAs, + compiler::{ + Number, + grammar::{Comparator, RelationalMatch}, + }, + runtime::Variable, +}; +use std::{borrow::Cow, cmp::Ordering}; + +pub(crate) trait Comparable { + fn to_str(&'_ self) -> Cow<'_, str>; + fn to_number(&self) -> Number; +} + +impl Comparator { + #[inline(always)] + pub(crate) fn from_code(code: u8) -> Comparator { + match code { + 0 => Comparator::Elbonia, + 2 => Comparator::AsciiCaseMap, + 3 => Comparator::AsciiNumeric, + _ => Comparator::Octet, + } + } + + #[inline(always)] + pub(crate) fn code(&self) -> u8 { + match self { + Comparator::Elbonia | Comparator::Other(_) => 0, + Comparator::Octet => 1, + Comparator::AsciiCaseMap => 2, + Comparator::AsciiNumeric => 3, + } + } + + #[inline(always)] + pub(crate) fn is_casemap(&self) -> bool { + matches!(self, Comparator::AsciiCaseMap) + } + + pub(crate) fn is(&self, a: &impl Comparable, b: &impl Comparable) -> bool { + match self { + Comparator::Octet => a.to_str() == b.to_str(), + Comparator::AsciiNumeric => RelationalMatch::Eq.cmp(&a.to_number(), &b.to_number()), + _ => casemap_eq(a.to_str().as_ref(), b.to_str().as_ref()), + } + } + + pub(crate) fn contains(&self, haystack: &str, needle: &str) -> bool { + needle.is_empty() + || match self { + Comparator::Octet => haystack.contains(needle), + _ => contains_ignore_ascii_case(haystack, needle), + } + } + + pub(crate) fn relational( + &self, + relation: &RelationalMatch, + a: &impl Comparable, + b: &impl Comparable, + ) -> bool { + match self { + Comparator::Octet => relation.cmp(a.to_str().as_ref(), b.to_str().as_ref()), + Comparator::AsciiNumeric => relation.cmp(&a.to_number(), &b.to_number()), + _ => relation.matches(casemap_cmp(a.to_str().as_ref(), b.to_str().as_ref())), + } + } + + pub(crate) fn as_match(&self) -> MatchAs { + match self { + Comparator::AsciiCaseMap => MatchAs::Lowercase, + Comparator::AsciiNumeric => MatchAs::Number, + _ => MatchAs::Octet, + } + } +} + +pub(crate) fn casemap_eq(a: &str, b: &str) -> bool { + if a.is_ascii() && b.is_ascii() { + a.eq_ignore_ascii_case(b) + } else { + a.to_lowercase() == b.to_lowercase() + } +} + +pub(crate) fn casemap_cmp(a: &str, b: &str) -> Ordering { + if a.is_ascii() && b.is_ascii() { + a.bytes() + .map(|c| c.to_ascii_lowercase()) + .cmp(b.bytes().map(|c| c.to_ascii_lowercase())) + } else { + a.to_lowercase().cmp(&b.to_lowercase()) + } +} + +pub(crate) fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool { + if needle.is_empty() { + return true; + } + if !needle.is_ascii() || !haystack.is_ascii() { + return haystack.to_lowercase().contains(&needle.to_lowercase()); + } + let haystack = haystack.as_bytes(); + let needle = needle.as_bytes(); + if needle.len() > haystack.len() { + return false; + } + let first = needle[0]; + let lower = first.to_ascii_lowercase(); + let upper = first.to_ascii_uppercase(); + let last_start = haystack.len() - needle.len(); + let mut offset = 0; + while offset <= last_start { + let window = &haystack[offset..=last_start]; + let found = if lower != upper { + memchr::memchr2(lower, upper, window) + } else { + memchr::memchr(first, window) + }; + let Some(at) = found else { + return false; + }; + let start = offset + at; + if haystack[start..start + needle.len()].eq_ignore_ascii_case(needle) { + return true; + } + offset = start + 1; + } + false +} + +pub(crate) fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool { + if value.is_ascii() && prefix.is_ascii() { + value.len() >= prefix.len() + && value.as_bytes()[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes()) + } else { + value.to_lowercase().starts_with(&prefix.to_lowercase()) + } +} + +pub(crate) fn ends_with_ignore_ascii_case(value: &str, suffix: &str) -> bool { + if value.is_ascii() && suffix.is_ascii() { + value.len() >= suffix.len() + && value.as_bytes()[value.len() - suffix.len()..] + .eq_ignore_ascii_case(suffix.as_bytes()) + } else { + value.to_lowercase().ends_with(&suffix.to_lowercase()) + } +} + +impl Comparable for Variable<'_> { + fn to_str(&'_ self) -> Cow<'_, str> { + self.to_string() + } + + fn to_number(&self) -> Number { + self.to_number() + } +} + +impl Comparable for &str { + fn to_str(&'_ self) -> Cow<'_, str> { + (*self).into() + } + + fn to_number(&self) -> Number { + self.parse::() + .map(Number::Float) + .unwrap_or(Number::Float(0.0)) + } +} + +impl Comparable for str { + fn to_str(&'_ self) -> Cow<'_, str> { + self.into() + } + + fn to_number(&self) -> Number { + self.parse::() + .map(Number::Float) + .unwrap_or(Number::Float(0.0)) + } +} + +impl RelationalMatch { + #[inline(always)] + pub(crate) fn from_code(code: u64) -> RelationalMatch { + match code { + 0 => RelationalMatch::Gt, + 1 => RelationalMatch::Ge, + 2 => RelationalMatch::Lt, + 3 => RelationalMatch::Le, + 4 => RelationalMatch::Eq, + _ => RelationalMatch::Ne, + } + } + + #[inline(always)] + pub(crate) fn code(&self) -> u64 { + match self { + RelationalMatch::Gt => 0, + RelationalMatch::Ge => 1, + RelationalMatch::Lt => 2, + RelationalMatch::Le => 3, + RelationalMatch::Eq => 4, + RelationalMatch::Ne => 5, + } + } + + pub fn matches(&self, ordering: Ordering) -> bool { + match self { + RelationalMatch::Gt => ordering == Ordering::Greater, + RelationalMatch::Ge => ordering != Ordering::Less, + RelationalMatch::Lt => ordering == Ordering::Less, + RelationalMatch::Le => ordering != Ordering::Greater, + RelationalMatch::Eq => ordering == Ordering::Equal, + RelationalMatch::Ne => ordering != Ordering::Equal, + } + } + + pub fn cmp(&self, a: &T, b: &T) -> bool + where + T: PartialOrd + ?Sized, + { + match self { + RelationalMatch::Gt => a.gt(b), + RelationalMatch::Ge => a.ge(b), + RelationalMatch::Lt => a.lt(b), + RelationalMatch::Le => a.le(b), + RelationalMatch::Eq => a.eq(b), + RelationalMatch::Ne => a.ne(b), + } + } +} + diff --git a/crates/sieve/src/runtime/tests/glob.rs b/crates/sieve/src/runtime/tests/glob.rs new file mode 100644 index 00000000..f50198b3 --- /dev/null +++ b/crates/sieve/src/runtime/tests/glob.rs @@ -0,0 +1,610 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::comparator::{ + casemap_eq, contains_ignore_ascii_case, ends_with_ignore_ascii_case, + starts_with_ignore_ascii_case, +}; +use crate::{ + MAX_MATCH_VARIABLES, Sieve, + bytecode::{Corrupt, Decoded, rec::Str}, +}; +use std::char::REPLACEMENT_CHARACTER; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CompiledGlob { + shape: Option, + pattern: GlobPattern, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GlobShape { + Literal(String), + Prefix(String), + Suffix(String), + Contains(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct GlobPattern { + pattern: Vec, + to_lower: bool, + is_ascii: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PatternChar { + WildcardMany(u32), + WildcardSingle, + Char(char), +} + +const SHAPE_NONE: u8 = 0; +const SHAPE_LITERAL: u8 = 1; +const SHAPE_PREFIX: u8 = 2; +const SHAPE_SUFFIX: u8 = 3; +const SHAPE_CONTAINS: u8 = 4; + +const ENC_MANY: u32 = 1 << 31; +const ENC_SINGLE: u32 = 1 << 30; + +impl PatternChar { + fn encode(self) -> u32 { + match self { + PatternChar::WildcardMany(n) => ENC_MANY | (n & (ENC_SINGLE - 1)), + PatternChar::WildcardSingle => ENC_SINGLE, + PatternChar::Char(c) => c as u32, + } + } + + #[inline(always)] + fn decode(v: u32) -> PatternChar { + if v & ENC_MANY != 0 { + PatternChar::WildcardMany(v & (ENC_SINGLE - 1)) + } else if v & ENC_SINGLE != 0 { + PatternChar::WildcardSingle + } else { + PatternChar::Char(char::from_u32(v).unwrap_or(REPLACEMENT_CHARACTER)) + } + } +} + +pub(crate) trait Pattern { + fn len(&self) -> usize; + fn at(&self, index: usize) -> Option; + fn to_lower(&self) -> bool; + fn is_ascii(&self) -> bool; +} + +impl Pattern for GlobPattern { + #[inline(always)] + fn len(&self) -> usize { + self.pattern.len() + } + + #[inline(always)] + fn at(&self, index: usize) -> Option { + self.pattern.get(index).copied() + } + + #[inline(always)] + fn to_lower(&self) -> bool { + self.to_lower + } + + #[inline(always)] + fn is_ascii(&self) -> bool { + self.is_ascii + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct GlobView<'a> { + to_lower: bool, + is_ascii: bool, + shape: u8, + literal: &'a str, + chars: &'a [u8], +} + +impl<'a> GlobView<'a> { + pub(crate) fn parse(bytes: &'a [u8], sieve: &'a Sieve<'a>) -> Decoded { + let (head, rest) = bytes.split_first_chunk::<15>().ok_or(Corrupt)?; + let literal = sieve.str(Str { + off: u32::from_le_bytes([head[3], head[4], head[5], head[6]]), + len: u32::from_le_bytes([head[7], head[8], head[9], head[10]]), + })?; + let nchars = u32::from_le_bytes([head[11], head[12], head[13], head[14]]) as usize; + let chars = rest + .get(..nchars.checked_mul(4).ok_or(Corrupt)?) + .ok_or(Corrupt)?; + Ok(GlobView { + to_lower: head[0] != 0, + is_ascii: head[1] != 0, + shape: head[2], + literal, + chars, + }) + } + + pub(crate) fn matches(&self, value: &str) -> bool { + match self.shape { + SHAPE_NONE => matches_value(self, value), + shape => shape_matches(shape, self.literal, value, self.to_lower), + } + } + + pub(crate) fn capture( + &self, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> bool { + capture_value(self, value, capture_positions, captured_values) + } +} + +impl Pattern for GlobView<'_> { + #[inline(always)] + fn len(&self) -> usize { + self.chars.len() / 4 + } + + #[inline(always)] + fn at(&self, index: usize) -> Option { + self.chars + .get(index * 4..index * 4 + 4) + .and_then(|b| b.try_into().ok()) + .map(|b| PatternChar::decode(u32::from_le_bytes(b))) + } + + #[inline(always)] + fn to_lower(&self) -> bool { + self.to_lower + } + + #[inline(always)] + fn is_ascii(&self) -> bool { + self.is_ascii + } +} + +impl Default for CompiledGlob { + fn default() -> Self { + CompiledGlob::compile("", false) + } +} + +impl CompiledGlob { + pub fn compile(pattern: &str, to_lower: bool) -> Self { + let pattern = GlobPattern::compile(pattern, to_lower); + + CompiledGlob { + shape: pattern.classify(), + pattern, + } + } + + pub fn matches(&self, value: &str) -> bool { + match &self.shape { + Some(shape) => shape.matches(value, self.pattern.to_lower), + None => self.pattern.matches(value), + } + } + + pub fn capture( + &self, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> bool { + self.pattern + .capture(value, capture_positions, captured_values) + } + + pub(crate) fn shape(&self) -> Option<(u8, &str)> { + self.shape.as_ref().map(|shape| match shape { + GlobShape::Literal(s) => (SHAPE_LITERAL, s.as_str()), + GlobShape::Prefix(s) => (SHAPE_PREFIX, s.as_str()), + GlobShape::Suffix(s) => (SHAPE_SUFFIX, s.as_str()), + GlobShape::Contains(s) => (SHAPE_CONTAINS, s.as_str()), + }) + } + + pub(crate) fn to_lower(&self) -> bool { + self.pattern.to_lower + } + + pub(crate) fn is_ascii(&self) -> bool { + self.pattern.is_ascii + } + + pub(crate) fn encoded_chars(&self) -> impl Iterator + '_ { + self.pattern.pattern.iter().map(|c| c.encode()) + } +} + +impl GlobShape { + fn matches(&self, value: &str, to_lower: bool) -> bool { + let (shape, literal) = match self { + GlobShape::Literal(literal) => (SHAPE_LITERAL, literal), + GlobShape::Prefix(literal) => (SHAPE_PREFIX, literal), + GlobShape::Suffix(literal) => (SHAPE_SUFFIX, literal), + GlobShape::Contains(literal) => (SHAPE_CONTAINS, literal), + }; + shape_matches(shape, literal, value, to_lower) + } +} + +fn shape_matches(shape: u8, literal: &str, value: &str, to_lower: bool) -> bool { + match shape { + SHAPE_LITERAL => { + if to_lower { + casemap_eq(value, literal) + } else { + value == literal + } + } + SHAPE_PREFIX => { + if !to_lower { + value.starts_with(literal) + } else { + starts_with_ignore_ascii_case(value, literal) + } + } + SHAPE_SUFFIX => { + if !to_lower { + value.ends_with(literal) + } else { + ends_with_ignore_ascii_case(value, literal) + } + } + _ => { + if !to_lower { + value.contains(literal) + } else { + contains_ignore_ascii_case(value, literal) + } + } + } +} + +impl GlobPattern { + pub fn compile(pattern: &str, to_lower: bool) -> Self { + let mut chars = Vec::with_capacity(pattern.len()); + let mut is_escaped = false; + let mut str = pattern.chars().peekable(); + + while let Some(char) = str.next() { + match char { + '*' if !is_escaped => { + let mut num = 1; + while let Some('*') = str.peek() { + num += 1; + str.next(); + } + chars.push(PatternChar::WildcardMany(num)); + } + '?' if !is_escaped => { + chars.push(PatternChar::WildcardSingle); + } + '\\' if !is_escaped => { + is_escaped = true; + continue; + } + _ => { + if is_escaped { + is_escaped = false; + } + if to_lower && char.is_uppercase() { + for char in char.to_lowercase() { + chars.push(PatternChar::Char(char)); + } + } else { + chars.push(PatternChar::Char(char)); + } + } + } + } + + let is_ascii = chars.iter().all(|c| match c { + PatternChar::Char(char) => char.is_ascii(), + _ => true, + }); + + GlobPattern { + pattern: chars, + to_lower, + is_ascii, + } + } + + fn classify(&self) -> Option { + let mut body = self.pattern.as_slice(); + let leading = matches!(body.first(), Some(PatternChar::WildcardMany(_))); + if leading { + body = &body[1..]; + } + let trailing = matches!(body.last(), Some(PatternChar::WildcardMany(_))); + if trailing { + body = &body[..body.len() - 1]; + } + + let mut literal = String::with_capacity(body.len()); + for item in body { + match item { + PatternChar::Char(char) => literal.push(*char), + _ => return None, + } + } + + Some(match (leading, trailing) { + (false, false) => GlobShape::Literal(literal), + (false, true) => GlobShape::Prefix(literal), + (true, false) => GlobShape::Suffix(literal), + (true, true) => GlobShape::Contains(literal), + }) + } + + pub fn matches(&self, value: &str) -> bool { + matches_value(self, value) + } + + pub fn capture( + &self, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> bool { + capture_value(self, value, capture_positions, captured_values) + } +} + +fn matches_value(pattern: &impl Pattern, value: &str) -> bool { + if pattern.is_ascii() && value.is_ascii() { + matches_ascii(pattern, value.as_bytes()) + } else if pattern.to_lower() { + matches_chars(pattern, &value.to_lowercase().chars().collect::>()) + } else { + matches_chars(pattern, &value.chars().collect::>()) + } +} + +fn matches_ascii(pattern: &impl Pattern, value: &[u8]) -> bool { + let mut px = 0; + let mut nx = 0; + let mut next_px = 0; + let mut next_nx = 0; + let len = pattern.len(); + let to_lower = pattern.to_lower(); + + while px < len || nx < value.len() { + match pattern.at(px) { + Some(PatternChar::Char(char)) => { + let char = char as u8; + let matched = if to_lower { + matches!(value.get(nx), Some(nc) if nc.to_ascii_lowercase() == char) + } else { + matches!(value.get(nx), Some(nc) if *nc == char) + }; + if matched { + px += 1; + nx += 1; + continue; + } + } + Some(PatternChar::WildcardSingle) if nx < value.len() => { + px += 1; + nx += 1; + continue; + } + Some(PatternChar::WildcardMany(_)) => { + next_px = px; + next_nx = nx + 1; + px += 1; + continue; + } + _ => (), + } + if 0 < next_nx && next_nx <= value.len() { + px = next_px; + nx = next_nx; + continue; + } + return false; + } + true +} + +fn matches_chars(pattern: &impl Pattern, value: &[char]) -> bool { + let mut px = 0; + let mut nx = 0; + let mut next_px = 0; + let mut next_nx = 0; + let len = pattern.len(); + + while px < len || nx < value.len() { + match pattern.at(px) { + Some(PatternChar::Char(char)) => { + if matches!(value.get(nx), Some(nc) if *nc == char) { + px += 1; + nx += 1; + continue; + } + } + Some(PatternChar::WildcardSingle) if nx < value.len() => { + px += 1; + nx += 1; + continue; + } + Some(PatternChar::WildcardMany(_)) => { + next_px = px; + next_nx = nx + 1; + px += 1; + continue; + } + _ => (), + } + if 0 < next_nx && next_nx <= value.len() { + px = next_px; + nx = next_nx; + continue; + } + return false; + } + true +} + +fn capture_value( + pattern: &impl Pattern, + value_: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, +) -> bool { + let value = if pattern.to_lower() { + let mut value = Vec::with_capacity(value_.len()); + for char in value_.chars() { + if char.is_uppercase() { + for (pos, lowerchar) in char.to_lowercase().enumerate() { + value.push(( + lowerchar, + if pos == 0 { + char + } else { + REPLACEMENT_CHARACTER + }, + )); + } + } else { + value.push((char, char)); + } + } + value + } else { + value_.chars().map(|char| (char, char)).collect::>() + }; + + let len = pattern.len(); + let mut match_pos = vec![0usize; len]; + + let mut px = 0; + let mut nx = 0; + let mut next_px = 0; + let mut next_nx = 0; + + while px < len || nx < value.len() { + match pattern.at(px) { + Some(PatternChar::Char(char)) => { + if matches!(value.get(nx), Some(nc) if nc.0 == char) { + match_pos[px] = nx; + px += 1; + nx += 1; + continue; + } + } + Some(PatternChar::WildcardSingle) if nx < value.len() => { + match_pos[px] = nx; + px += 1; + nx += 1; + continue; + } + Some(PatternChar::WildcardMany(_)) => { + match_pos[px] = nx; + next_px = px; + next_nx = nx + 1; + px += 1; + continue; + } + _ => (), + } + if 0 < next_nx && next_nx <= value.len() { + px = next_px; + nx = next_nx; + continue; + } + return false; + } + + let mut last_pos = 0; + + captured_values.clear(); + if capture_positions & 1 != 0 { + captured_values.push((0usize, value_.to_string())); + } + + let mut wildcard_pos: usize = 1; + for px in 0..len { + if wildcard_pos > MAX_MATCH_VARIABLES as usize { + break; + } + let Some(item) = pattern.at(px) else { + break; + }; + let match_pos = match_pos[px]; + last_pos = match item { + PatternChar::WildcardMany(num) => { + let mut num = num; + while num > 1 { + if is_captured(capture_positions, wildcard_pos) { + captured_values.push((wildcard_pos, String::with_capacity(0))); + } + wildcard_pos += 1; + num -= 1; + } + + if is_captured(capture_positions, wildcard_pos) { + if let Some(range) = value.get(last_pos..match_pos) { + captured_values.push(( + wildcard_pos, + range + .iter() + .filter_map(|(_, char)| { + if char != &REPLACEMENT_CHARACTER { + Some(char) + } else { + None + } + }) + .collect::(), + )); + } else { + debug_assert!(false, "Glob pattern failure."); + return false; + } + } + wildcard_pos += 1; + match_pos + } + PatternChar::WildcardSingle => { + if is_captured(capture_positions, wildcard_pos) { + if let Some((char, orig_char)) = value.get(match_pos) { + captured_values.push(( + wildcard_pos, + (if orig_char != &REPLACEMENT_CHARACTER { + orig_char + } else { + char + }) + .to_string(), + )); + } else { + debug_assert!(false, "Glob pattern failure."); + return false; + } + } + wildcard_pos += 1; + match_pos + } + PatternChar::Char(_) => match_pos, + } + 1; + } + true +} +#[inline(always)] +fn is_captured(capture_positions: u64, wildcard_pos: usize) -> bool { + wildcard_pos <= MAX_MATCH_VARIABLES as usize && capture_positions & (1 << wildcard_pos) != 0 +} + diff --git a/crates/sieve/src/runtime/tests/matching.rs b/crates/sieve/src/runtime/tests/matching.rs new file mode 100644 index 00000000..2ad9221d --- /dev/null +++ b/crates/sieve/src/runtime/tests/matching.rs @@ -0,0 +1,232 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::glob::CompiledGlob; +use crate::{ + Context, Sieve, + bytecode::{ + ops::{Match, match_kind}, + rec::{Range, tag}, + }, + compiler::grammar::{Comparator, MatchType, RelationalMatch}, + runtime::{RuntimeError, Variable, eval::ValueRef}, +}; +use smallvec::SmallVec; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Pattern { + Dynamic, + Glob(u16), + Regex(u16), +} + +#[derive(Debug, Clone)] +pub(crate) struct Key<'x> { + pub value: Variable<'x>, + pub pattern: Pattern, +} + +pub(crate) type Keys<'x> = SmallVec<[Key<'x>; 4]>; + +impl Match { + #[inline(always)] + pub(crate) fn match_type(&self) -> MatchType { + match self.kind { + match_kind::IS => MatchType::Is, + match_kind::CONTAINS => MatchType::Contains, + match_kind::MATCHES => MatchType::Matches(self.arg), + match_kind::REGEX => MatchType::Regex(self.arg), + match_kind::VALUE => MatchType::Value(RelationalMatch::from_code(self.arg)), + match_kind::COUNT => MatchType::Count(RelationalMatch::from_code(self.arg)), + _ => MatchType::List, + } + } + + pub(crate) fn from_match_type(match_type: &MatchType) -> Match { + match match_type { + MatchType::Is => Match { + kind: match_kind::IS, + arg: 0, + }, + MatchType::Contains => Match { + kind: match_kind::CONTAINS, + arg: 0, + }, + MatchType::Matches(positions) => Match { + kind: match_kind::MATCHES, + arg: *positions, + }, + MatchType::Regex(positions) => Match { + kind: match_kind::REGEX, + arg: *positions, + }, + MatchType::Value(rel) => Match { + kind: match_kind::VALUE, + arg: rel.code(), + }, + MatchType::Count(rel) => Match { + kind: match_kind::COUNT, + arg: rel.code(), + }, + MatchType::List => Match { + kind: match_kind::LIST, + arg: 0, + }, + } + } +} + +const DYNAMIC_REGEX_CACHE: usize = 32; + +use std::borrow::Cow; + +impl<'x> Context<'x> { + pub(crate) fn eval_keys( + &self, + script: &'x Sieve<'x>, + range: Range, + ) -> Result, RuntimeError> { + let mut result = Keys::with_capacity(range.len as usize); + let mut iter = script.recs(range)?; + while let Some(rec) = iter.next() { + let pattern = match rec.tag { + tag::GLOB => Pattern::Glob(rec.c), + tag::REGEX => Pattern::Regex(rec.c), + _ => Pattern::Dynamic, + }; + let value = ValueRef::decode(script, rec, &mut iter)?; + result.push(Key { + value: self.eval_value_ref(script, value)?, + pattern, + }); + } + Ok(result) + } + + pub(crate) fn key_matches( + &self, + script: &'x Sieve<'x>, + comparator: &Comparator, + match_type: &MatchType, + key: &Key<'x>, + value: &str, + captured_values: &mut Vec<(usize, String)>, + ) -> Result { + Ok(match match_type { + MatchType::Is => comparator.is(&value, &key.value), + MatchType::Contains => comparator.contains(value, key.value.to_string().as_ref()), + MatchType::Value(relation) => comparator.relational(relation, &value, &key.value), + MatchType::Matches(capture_positions) => self.glob_matches( + script, + comparator.is_casemap(), + key, + value, + *capture_positions, + captured_values, + )?, + MatchType::Regex(capture_positions) => { + self.regex_matches(script, key, value, *capture_positions, captured_values)? + } + MatchType::Count(_) | MatchType::List => false, + }) + } + + pub(crate) fn glob_matches( + &self, + script: &'x Sieve<'x>, + to_lower: bool, + key: &Key<'x>, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> Result { + if let Pattern::Glob(index) = key.pattern { + let glob = script.glob(index)?; + Ok(if capture_positions == 0 { + glob.matches(value) + } else { + glob.capture(value, capture_positions, captured_values) + }) + } else { + let compiled = CompiledGlob::compile(key.value.to_string().as_ref(), to_lower); + Ok(if capture_positions == 0 { + compiled.matches(value) + } else { + compiled.capture(value, capture_positions, captured_values) + }) + } + } + + pub(crate) fn regex_matches( + &self, + script: &'x Sieve<'x>, + key: &Key<'x>, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> Result { + if let Pattern::Regex(slot) = key.pattern { + let pattern = key.value.to_string(); + let regex = script.regex(slot, pattern.as_ref()); + debug_assert!(regex.is_some(), "Failed to compile regex: {pattern:?}"); + Ok(regex + .is_some_and(|regex| eval_regex(regex, value, capture_positions, captured_values))) + } else { + Ok(self.dynamic_regex_matches( + key.value.clone().into_string(), + value, + capture_positions, + captured_values, + )) + } + } + + fn dynamic_regex_matches( + &self, + pattern: Cow<'x, str>, + value: &str, + capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, + ) -> bool { + let mut cache = self.dynamic_regexes.borrow_mut(); + if let Some(regex) = cache.get(pattern.as_ref()) { + return regex + .as_ref() + .is_some_and(|regex| eval_regex(regex, value, capture_positions, captured_values)); + } + let regex = crate::regex::compile(&pattern); + let matched = regex + .as_ref() + .is_some_and(|regex| eval_regex(regex, value, capture_positions, captured_values)); + if cache.len() < DYNAMIC_REGEX_CACHE { + cache.insert(self.intern_cow(pattern), regex); + } + matched + } +} + +fn eval_regex( + regex: &fancy_regex::Regex, + value: &str, + mut capture_positions: u64, + captured_values: &mut Vec<(usize, String)>, +) -> bool { + if capture_positions == 0 { + regex.is_match(value).unwrap_or_default() + } else if let Ok(Some(captures)) = regex.captures(value) { + captured_values.clear(); + while capture_positions != 0 { + let index = 63 - capture_positions.leading_zeros(); + capture_positions ^= 1 << index; + if let Some(match_var) = captures.get(index as usize) { + captured_values.push((index as usize, match_var.as_str().to_string())); + } + } + true + } else { + false + } +} diff --git a/crates/sieve/src/runtime/tests/mime.rs b/crates/sieve/src/runtime/tests/mime.rs new file mode 100644 index 00000000..90407f78 --- /dev/null +++ b/crates/sieve/src/runtime/tests/mime.rs @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::ScanMeter; +use crate::{ + Context, + runtime::message::parts::{PartCursor, Scope}, +}; +use mail_parser::{MessagePart, PartKind}; + +#[derive(Debug)] +pub(crate) enum ContentTypeFilter<'x> { + Type(&'x str), + TypeSubtype((&'x str, &'x str)), +} + +impl<'x> Context<'x> { + pub(crate) fn find_nested_parts( + &self, + ct_filter: &[ContentTypeFilter<'_>], + meter: &mut ScanMeter, + visitor_fnc: &mut impl FnMut(MessagePart<'x>, &mut ScanMeter) -> bool, + ) -> bool { + let Some(part) = self.current_part() else { + return false; + }; + let mut parts = PartCursor::subtree(part, true, Scope::Nested); + while let Some(part) = self.advance(&mut parts) { + if !meter.charge_visit() { + return true; + } + if (ct_filter.is_empty() || self.matches_content_type(part, ct_filter)) + && visitor_fnc(part, meter) + { + return true; + } + } + false + } + + fn matches_content_type( + &self, + part: MessagePart<'x>, + ct_filter: &[ContentTypeFilter<'_>], + ) -> bool { + let (ct, cst) = + self.part_content_type(part) + .unwrap_or_else(|| match self.part_kind(part) { + PartKind::Text => ("text", "plain"), + PartKind::Html => ("text", "html"), + PartKind::Message(_) => ("message", "rfc822"), + PartKind::Multipart => ("multipart", "mixed"), + _ => ("application", "octet-stream"), + }); + + ct_filter.iter().any(|ctf| match ctf { + ContentTypeFilter::Type(name) => name.eq_ignore_ascii_case(ct), + ContentTypeFilter::TypeSubtype((name, subname)) => { + name.eq_ignore_ascii_case(ct) && subname.eq_ignore_ascii_case(cst) + } + }) + } +} + +impl<'x> ContentTypeFilter<'x> { + pub(crate) fn parse(ct: &'x str) -> Option> { + let mut iter = ct.split('/'); + let name = iter.next()?; + if let Some(sub_name) = iter.next() { + if !name.is_empty() && !sub_name.is_empty() && iter.next().is_none() { + Some(ContentTypeFilter::TypeSubtype((name, sub_name))) + } else { + None + } + } else if !name.is_empty() { + Some(ContentTypeFilter::Type(name)) + } else { + None + } + } +} diff --git a/crates/sieve/src/runtime/tests/mod.rs b/crates/sieve/src/runtime/tests/mod.rs new file mode 100644 index 00000000..777b5ad6 --- /dev/null +++ b/crates/sieve/src/runtime/tests/mod.rs @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + RuntimeError, + handler::{Handler, Mailbox, Reply}, +}; +use crate::{Context, Sieve, bytecode::ops, compiler::grammar::Capability}; +use mail_parser::MessagePart; +use smallvec::SmallVec; + +pub mod comparator; +pub mod glob; +pub mod matching; +pub mod mime; +pub mod test_address; +pub mod test_body; +pub mod test_date; +pub mod test_duplicate; +pub mod test_envelope; +pub mod test_exists; +pub mod test_extlists; +pub mod test_hasflag; +pub mod test_header; +pub mod test_metadata; +pub mod test_notify; +pub mod test_size; +pub mod test_spamtest; +pub mod test_string; + +const MAX_SCAN_BYTES: usize = 192 << 20; +const PART_VISIT_BYTES: usize = 512; + +pub(crate) struct ScanMeter { + budget: usize, + spent: usize, + weight: usize, +} + +impl ScanMeter { + pub(crate) fn charge(&mut self, bytes: usize) -> bool { + self.spent = self.weight.saturating_mul(bytes).saturating_add(self.spent); + self.spent <= self.budget + } + + pub(crate) fn charge_visit(&mut self) -> bool { + self.spent = self.spent.saturating_add(PART_VISIT_BYTES); + self.spent <= self.budget + } + + pub(crate) fn charge_headers(&mut self, part: MessagePart<'_>) -> bool { + let headers = part.offset_body().saturating_sub(part.offset_header()) as usize; + self.charge_visit() && self.charge(headers) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TestResult { + Bool(bool), + Pending { is_not: bool }, +} + +impl TestResult { + #[inline(always)] + pub(crate) fn from_reply(reply: Reply, is_not: bool) -> Result { + match reply { + Reply::Ready(result) => Ok(TestResult::Bool(result ^ is_not)), + Reply::Pending => Ok(TestResult::Pending { is_not }), + Reply::Error(err) => Err(err), + } + } +} + +impl<'x> Context<'x> { + pub(crate) fn scan_meter(&self, keys: usize) -> ScanMeter { + ScanMeter { + budget: MAX_SCAN_BYTES.saturating_sub(self.num_scanned_bytes), + spent: 0, + weight: keys.max(1), + } + } + + pub(crate) fn settle_scan(&mut self, meter: ScanMeter) -> Result<(), RuntimeError> { + self.num_scanned_bytes = self.num_scanned_bytes.saturating_add(meter.spent); + if meter.spent <= meter.budget { + Ok(()) + } else { + Err(RuntimeError::CPULimitReached) + } + } + + pub(crate) fn charge_scan(&mut self, keys: usize, bytes: usize) -> Result<(), RuntimeError> { + let mut meter = self.scan_meter(keys); + meter.charge(bytes); + self.settle_scan(meter) + } + + pub(crate) fn test_ihave( + &self, + script: &'x Sieve<'x>, + test: &ops::TestIhave, + ) -> Result { + let mut result = true; + for rec in script.recs(test.capabilities)? { + let capability = Capability::from_rec(script, rec)?; + if [Capability::Variables, Capability::EncodedCharacter].contains(&capability) + || !self.runtime.allowed_capabilities.contains(&capability) + { + result = false; + break; + } + } + Ok(TestResult::Bool(result ^ test.is_not)) + } + + pub(crate) fn test_mailbox_exists>( + &self, + script: &'x Sieve<'x>, + test: &ops::TestMailboxExists, + handler: &mut H, + ) -> Result { + let names = self.eval_strings(script, test.mailbox_names)?; + let mailboxes: SmallVec<[Mailbox<'_>; 4]> = + names.iter().map(|name| Mailbox::Name(name)).collect(); + TestResult::from_reply(handler.mailbox_exists(self, &mailboxes, &[]), test.is_not) + } + + pub(crate) fn test_mailbox_id_exists>( + &self, + script: &'x Sieve<'x>, + test: &ops::TestMailboxIdExists, + handler: &mut H, + ) -> Result { + let ids = self.eval_strings(script, test.mailbox_ids)?; + let mailboxes: SmallVec<[Mailbox<'_>; 4]> = ids.iter().map(|id| Mailbox::Id(id)).collect(); + TestResult::from_reply(handler.mailbox_exists(self, &mailboxes, &[]), test.is_not) + } + + pub(crate) fn test_special_use_exists>( + &self, + script: &'x Sieve<'x>, + test: &ops::TestSpecialUseExists, + handler: &mut H, + ) -> Result { + let mailbox = self.eval_opt_str(script, test.mailbox)?; + let attributes = self.eval_strings(script, test.attributes)?; + let mailboxes: SmallVec<[Mailbox<'_>; 1]> = + mailbox.map(Mailbox::Name).into_iter().collect(); + TestResult::from_reply( + handler.mailbox_exists(self, &mailboxes, &attributes), + test.is_not, + ) + } +} diff --git a/crates/sieve/src/runtime/tests/test_address.rs b/crates/sieve/src/runtime/tests/test_address.rs new file mode 100644 index 00000000..f04de5ff --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_address.rs @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::runtime::message::headers::HeaderRef; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{AddressPart, Comparator, MatchType}, + }, + runtime::{RuntimeError, handler::Handler}, +}; +use mail_parser::{ + AddressList, HeaderForm, HeaderValue, Mailbox, parse_address_detail_part, parse_address_domain, + parse_address_local_part, parse_address_user_part, +}; +use smallvec::SmallVec; + +impl<'x> Context<'x> { + pub(crate) fn test_address>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestAddress, + handler: &mut H, + ) -> Result { + let key_list = self.eval_keys(script, test.key_list)?; + let header_list = self.parse_header_names(script, test.header_list)?; + let address_part = AddressPart::from_code(test.address_part); + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + + let result = match &match_type { + MatchType::Is | MatchType::Contains | MatchType::Value(_) => { + let mut meter = self.scan_meter(key_list.len()); + let result = self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_addresses(header, &address_part, |value| { + key_list.iter().any(|key| match &match_type { + MatchType::Is => comparator.is(&value, &key.value), + MatchType::Contains => { + comparator.contains(value, key.value.to_string().as_ref()) + } + MatchType::Value(rel_match) => { + comparator.relational(rel_match, &value, &key.value) + } + _ => false, + }) + }) + }, + ); + self.settle_scan(meter)?; + result + } + MatchType::Matches(capture_positions) | MatchType::Regex(capture_positions) => { + let mut meter = self.scan_meter(key_list.len()); + let mut captured_values = Vec::new(); + let is_matches = matches!(&match_type, MatchType::Matches(_)); + let to_lower = comparator.is_casemap(); + let mut error = None; + let result = self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_addresses(header, &address_part, |value| { + for key in &key_list { + let matched = if is_matches { + self.glob_matches( + script, + to_lower, + key, + value, + *capture_positions, + &mut captured_values, + ) + } else { + self.regex_matches( + script, + key, + value, + *capture_positions, + &mut captured_values, + ) + }; + match matched { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + false + }) + }, + ); + self.settle_scan(meter)?; + if let Some(err) = error { + return Err(err); + } + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + result + } + MatchType::Count(rel_match) => { + let mut count: i64 = 0; + let mut meter = self.scan_meter(1); + self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_addresses(header, &address_part, |value| { + if !value.is_empty() { + count += 1; + } + false + }) + }, + ); + self.settle_scan(meter)?; + + key_list + .iter() + .any(|key| rel_match.cmp(&Number::from(count), &key.value.to_number())) + } + MatchType::List => { + let mut values: Vec<&str> = Vec::new(); + let mut meter = self.scan_meter(1); + self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_addresses(header, &address_part, |value| { + if !value.is_empty() && !values.contains(&value) { + values.push(self.alloc_str(value)); + } + false + }) + }, + ); + self.settle_scan(meter)?; + + if !values.is_empty() { + let lists: SmallVec<[&str; 4]> = key_list + .iter() + .map(|key| self.intern_cow(key.value.clone().into_string())) + .collect(); + return TestResult::from_reply( + handler.list_contains(self, &lists, &values, comparator.as_match()), + test.is_not, + ); + } + + false + } + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + pub(crate) fn find_addresses( + &self, + header: HeaderRef<'x>, + part: &AddressPart, + mut visitor_fnc: impl FnMut(&str) -> bool, + ) -> bool { + if let HeaderValue::Address(list) = header.value() { + return visit_addresses(list, part, &mut visitor_fnc); + } + match header.parse_as(HeaderForm::Addresses).value().as_address() { + Some(list) => visit_addresses(list, part, &mut visitor_fnc), + None => visitor_fnc(""), + } + } +} + +fn visit_addresses( + list: AddressList<'_>, + part: &AddressPart, + visitor_fnc: &mut impl FnMut(&str) -> bool, +) -> bool { + list.mailboxes() + .any(|mailbox| part.eval(mailbox).is_some_and(&mut *visitor_fnc)) +} + +impl AddressPart { + pub(crate) fn eval<'x>(&self, mailbox: Mailbox<'x>) -> Option<&'x str> { + let email = mailbox.address().or(mailbox.name()); + match (self, email) { + (AddressPart::All, _) => email, + (AddressPart::LocalPart, Some(email)) if !email.is_empty() => { + parse_address_local_part(email) + } + (AddressPart::Domain, Some(email)) if !email.is_empty() => parse_address_domain(email), + (AddressPart::User, Some(email)) if !email.is_empty() => parse_address_user_part(email), + (AddressPart::Detail, Some(email)) if !email.is_empty() => { + parse_address_detail_part(email) + } + (AddressPart::Name, _) => mailbox.name(), + _ => email, + } + } + + pub(crate) fn eval_strict<'x>(&self, mailbox: Mailbox<'x>) -> Option<&'x str> { + match (self, mailbox.address()) { + (AddressPart::All, Some(email)) => Some(email), + (AddressPart::LocalPart, Some(email)) if !email.is_empty() => { + parse_address_local_part(email) + } + (AddressPart::Domain, Some(email)) if !email.is_empty() => parse_address_domain(email), + (AddressPart::User, Some(email)) if !email.is_empty() => parse_address_user_part(email), + (AddressPart::Detail, Some(email)) if !email.is_empty() => { + parse_address_detail_part(email) + } + (AddressPart::Name, _) => mailbox.name(), + (_, email) => email, + } + } + + pub(crate) fn eval_string<'x>(&self, addr: &'x str) -> Option<&'x str> { + if !addr.is_empty() { + match self { + AddressPart::All => addr.into(), + AddressPart::LocalPart => parse_address_local_part(addr), + AddressPart::Domain => parse_address_domain(addr), + AddressPart::User => parse_address_user_part(addr), + AddressPart::Detail => parse_address_detail_part(addr), + _ => addr.into(), + } + } else { + addr.into() + } + } +} diff --git a/crates/sieve/src/runtime/tests/test_body.rs b/crates/sieve/src/runtime/tests/test_body.rs new file mode 100644 index 00000000..4fa1f923 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_body.rs @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{TestResult, matching::Key, mime::ContentTypeFilter}; +use crate::runtime::message::{body::TextForm, headers::HeaderRef}; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::RuntimeError, +}; +use mail_parser::MessagePart; +use mail_parser::{HeaderName, PartKind}; +use smallvec::SmallVec; +use std::borrow::Cow; + +const TRANSFORM_RAW: u8 = 0; +const TRANSFORM_CONTENT: u8 = 1; +const TRANSFORM_TEXT: u8 = 2; + +impl<'x> Context<'x> { + pub(crate) fn test_body( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestBody, + ) -> Result { + let key_list = self.eval_keys(script, test.key_list)?; + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let transform = test.body_transform.kind; + + if test.include_subject && !matches!(&match_type, MatchType::Count(_) | MatchType::List) { + let subject = if transform != TRANSFORM_RAW { + self.root_text(&HeaderName::Subject).unwrap_or_default() + } else { + self.root_part() + .and_then(|part| self.last_header(part, &HeaderName::Subject)) + .map(|header| match header { + HeaderRef::Added(added) => added.value, + header => header + .raw_value() + .and_then(|raw| std::str::from_utf8(raw).ok()) + .unwrap_or_default(), + }) + .unwrap_or_default() + }; + + self.charge_scan(key_list.len(), subject.len())?; + for key in &key_list { + if self.body_key_matches(script, &comparator, &match_type, key, subject)? { + return Ok(TestResult::Bool(!test.is_not)); + } + } + } + + let mut ct_filter: SmallVec<[ContentTypeFilter<'x>; 4]> = SmallVec::new(); + if transform == TRANSFORM_CONTENT { + for ct in self.eval_strings(script, test.body_transform.content_types)? { + if ct.is_empty() { + break; + } else if let Some(ctf) = ContentTypeFilter::parse(ct) { + ct_filter.push(ctf); + } else { + return Ok(TestResult::Bool(test.is_not)); + } + } + } + + let result = if let MatchType::Count(rel_match) = &match_type { + let mut count: i64 = 0; + let mut meter = self.scan_meter(1); + self.find_nested_parts(&ct_filter, &mut meter, &mut |_, _| { + count += 1; + false + }); + self.settle_scan(meter)?; + + key_list + .iter() + .any(|key| rel_match.cmp(&Number::from(count), &key.value.to_number())) + } else { + let mut error = None; + let mut meter = self.scan_meter(key_list.len()); + let result = self.find_nested_parts(&ct_filter, &mut meter, &mut |part, meter| { + let Some(text) = self.body_text(part, transform) else { + return false; + }; + + if !meter.charge(text.len()) { + return true; + } + + for key in &key_list { + match self.body_key_matches(script, &comparator, &match_type, key, &text) { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + + false + }); + self.settle_scan(meter)?; + if let Some(err) = error { + return Err(err); + } + result + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + fn body_text(&self, part: MessagePart<'x>, transform: u8) -> Option> { + let text = match (transform, self.part_kind(part)) { + (TRANSFORM_CONTENT, PartKind::Message(nested)) => { + return Some(String::from_utf8_lossy(nested.root_part().raw_headers())); + } + (TRANSFORM_CONTENT, PartKind::Multipart) => { + return Some(match part.boundary() { + Some(boundary) => multipart_text(part.raw_body(), boundary).into(), + None => String::from_utf8_lossy(part.raw_body()), + }); + } + (TRANSFORM_RAW, _) if self.edits.body(part.id()).is_some() => { + self.part_text(part, TextForm::Raw) + } + (TRANSFORM_RAW, _) => self + .part_text(part, TextForm::Raw) + .filter(|raw| !raw.is_empty()), + (_, PartKind::Text) | (TRANSFORM_CONTENT, PartKind::Html) => { + self.part_text(part, TextForm::Source) + } + (_, PartKind::Html) => self.part_text(part, TextForm::Plain), + (TRANSFORM_TEXT, PartKind::Binary | PartKind::InlineBinary) + if self.part_content_type(part).is_some_and(|(ct, st)| { + ct.eq_ignore_ascii_case("application") && st.contains("xml") + }) => + { + self.part_text(part, TextForm::Markup) + } + (TRANSFORM_CONTENT, PartKind::Binary | PartKind::InlineBinary) => { + self.part_text(part, TextForm::Decoded) + } + _ => None, + }; + text.map(Cow::Borrowed) + } + + fn body_key_matches( + &self, + script: &'x Sieve<'x>, + comparator: &Comparator, + match_type: &MatchType, + key: &Key<'x>, + text: &str, + ) -> Result { + Ok(match match_type { + MatchType::Is => comparator.is(&text, &key.value), + MatchType::Contains => comparator.contains(text, key.value.to_string().as_ref()), + MatchType::Value(rel_match) => comparator.relational(rel_match, &text, &key.value), + MatchType::Matches(_) => self.glob_matches( + script, + comparator.is_casemap(), + key, + text, + 0, + &mut Vec::new(), + )?, + MatchType::Regex(_) => self.regex_matches(script, key, text, 0, &mut Vec::new())?, + MatchType::Count(_) | MatchType::List => false, + }) + } +} + +fn multipart_text(raw_body: &[u8], boundary: &str) -> String { + let mime_body = std::str::from_utf8(raw_body).unwrap_or_default(); + let mut mime_part = String::with_capacity(64); + if let Some((prologue, epilogue)) = mime_body.split_once(&format!("\n--{boundary}")) { + mime_part.push_str(prologue); + if let Some((_, epilogue)) = epilogue.rsplit_once(&format!("\n--{boundary}--")) { + mime_part.push_str(epilogue); + } + } + mime_part +} diff --git a/crates/sieve/src/runtime/tests/test_date.rs b/crates/sieve/src/runtime/tests/test_date.rs new file mode 100644 index 00000000..a733ef01 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_date.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::runtime::message::headers::HeaderRef; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType, tests::test_date::DatePart}, + }, + runtime::{RuntimeError, handler::Handler}, +}; +use mail_parser::{DateTime, HeaderForm, HeaderValue}; +use smallvec::SmallVec; + +impl<'x> Context<'x> { + pub(crate) fn test_date>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestDate, + handler: &mut H, + ) -> Result { + let Some(header_name) = self.parse_header_name(script, test.header_name)? else { + return Ok(TestResult::Bool(false ^ test.is_not)); + }; + let header_names = [header_name]; + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let date_part = DatePart::from_code(test.date_part); + let zone = test.zone; + + let result = match &match_type { + MatchType::Count(rel_match) => { + let mut date_count = 0; + let mut meter = self.scan_meter(1); + self.find_headers( + &header_names, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + if self.find_dates(header).is_some() { + date_count += 1; + } + false + }, + ); + self.settle_scan(meter)?; + + self.eval_values(script, test.key_list)? + .iter() + .any(|key| rel_match.cmp(&Number::from(date_count), &key.to_number())) + } + MatchType::List => { + let mut values: SmallVec<[&'x str; 4]> = SmallVec::new(); + let mut meter = self.scan_meter(1); + self.find_headers( + &header_names, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + if let Some(dt) = self.find_dates(header) { + let value = date_part.eval(&zone.eval(dt)); + if !value.is_empty() && !values.iter().any(|v| *v == value) { + values.push(self.alloc_string(value)); + } + } + false + }, + ); + self.settle_scan(meter)?; + if !values.is_empty() { + let lists = self.eval_strings(script, test.key_list)?; + return TestResult::from_reply( + handler.list_contains(self, &lists, &values, comparator.as_match()), + test.is_not, + ); + } + false + } + _ => { + let key_list = self.eval_keys(script, test.key_list)?; + let mut captured_values = Vec::new(); + let mut error = None; + let mut meter = self.scan_meter(key_list.len()); + + let result = self.find_headers( + &header_names, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + let Some(dt) = self.find_dates(header) else { + return false; + }; + let value = date_part.eval(&zone.eval(dt)); + for key in &key_list { + match self.key_matches( + script, + &comparator, + &match_type, + key, + &value, + &mut captured_values, + ) { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + false + }, + ); + self.settle_scan(meter)?; + if let Some(err) = error { + return Err(err); + } + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + result + } + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + pub(crate) fn test_current_date>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestCurrentDate, + handler: &mut H, + ) -> Result { + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let date_part = DatePart::from_code(test.date_part); + let mut result = false; + + match &match_type { + MatchType::Count(rel_match) => { + result = self + .eval_values(script, test.key_list)? + .iter() + .any(|key| rel_match.cmp(&Number::from(1.0), &key.to_number())); + } + MatchType::List => { + let value = date_part.eval(&self.current_date_time(test.zone)); + if !value.is_empty() { + let lists = self.eval_strings(script, test.key_list)?; + return TestResult::from_reply( + handler.list_contains( + self, + &lists, + &[value.as_str()], + comparator.as_match(), + ), + test.is_not, + ); + } + } + _ => { + let value = date_part.eval(&self.current_date_time(test.zone)); + let keys = self.eval_keys(script, test.key_list)?; + let mut captured_values = Vec::new(); + + for key in &keys { + if self.key_matches( + script, + &comparator, + &match_type, + key, + &value, + &mut captured_values, + )? { + result = true; + break; + } + } + + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + } + } + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + fn current_date_time(&self, zone: Option) -> DateTime { + let dt = DateTime::from_timestamp(self.current_time); + match zone { + Some(zone) => dt.to_timezone(zone), + None => dt, + } + } + + pub(crate) fn find_dates(&self, header: HeaderRef<'x>) -> Option { + match header.value() { + HeaderValue::DateTime(dt) => dt.is_valid().then_some(dt), + _ => header + .parse_as(HeaderForm::Date) + .value() + .as_datetime() + .filter(DateTime::is_valid), + } + } +} + +impl DatePart { + fn eval(&self, dt: &DateTime) -> String { + match self { + DatePart::Year => format!("{:04}", dt.year), + DatePart::Month => format!("{:02}", dt.month), + DatePart::Day => format!("{:02}", dt.day), + DatePart::Date => format!("{:04}-{:02}-{:02}", dt.year, dt.month, dt.day,), + DatePart::Julian => ((dt.julian_day() as f64 - 2400000.5) as i64).to_string(), + DatePart::Hour => format!("{:02}", dt.hour), + DatePart::Minute => format!("{:02}", dt.minute), + DatePart::Second => format!("{:02}", dt.second), + DatePart::Time => format!("{:02}:{:02}:{:02}", dt.hour, dt.minute, dt.second,), + DatePart::Iso8601 => dt.to_rfc3339(), + DatePart::Std11 => dt.to_rfc822(), + DatePart::Zone => format!( + "{}{:02}{:02}", + if dt.tz_before_gmt && (dt.tz_hour > 0 || dt.tz_minute > 0) { + "-" + } else { + "+" + }, + dt.tz_hour, + dt.tz_minute + ), + DatePart::Weekday => dt.day_of_week().to_string(), + } + } +} + +impl ops::Zone { + pub(crate) fn eval(&self, dt: DateTime) -> DateTime { + match self.kind { + 0 => dt.to_timezone(self.time), + 1 => dt, + _ => DateTime::from_timestamp(dt.to_timestamp()), + } + } +} diff --git a/crates/sieve/src/runtime/tests/test_duplicate.rs b/crates/sieve/src/runtime/tests/test_duplicate.rs new file mode 100644 index 00000000..51444efc --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_duplicate.rs @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Sieve, + bytecode::{ops, rec::Rec}, + runtime::{RuntimeError, handler::Handler}, +}; +use mail_parser::{HeaderForm, HeaderName}; +use std::borrow::Cow; + +impl<'x> Context<'x> { + pub(crate) fn test_duplicate>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestDuplicate, + handler: &mut H, + ) -> Result { + let id = match test.dup_match.kind { + 0 => self.duplicate_header_id(script, test.dup_match.value)?, + 1 => self.eval_value(script, test.dup_match.value)?.into_string(), + _ => self + .root_text(&HeaderName::MessageId) + .unwrap_or_default() + .into(), + }; + + if id.is_empty() { + return Ok(TestResult::Bool(false ^ test.is_not)); + } + + let id = match self.eval_opt(script, test.handle)? { + Some(handle) => { + let mut prefixed = handle.to_string().into_owned(); + prefixed.push_str(&id); + Cow::Owned(prefixed) + } + None => id, + }; + let expiry = test + .seconds + .unwrap_or(self.runtime.default_duplicate_expiry); + + TestResult::from_reply( + handler.duplicate_id(self, &id, expiry, test.last), + test.is_not, + ) + } + + fn duplicate_header_id( + &mut self, + script: &'x Sieve<'x>, + header_name: Rec, + ) -> Result, RuntimeError> { + let mut value = Cow::Borrowed(""); + if let Some(header_name) = self.parse_header_name(script, header_name)? { + let mut meter = self.scan_meter(1); + self.find_headers(&[header_name], None, true, &mut meter, |header, _| { + let parsed = header.parse_as(HeaderForm::MessageIds); + if let Some(ids) = parsed.value().as_text_list() + && ids.len() == 1 + && let Some(id) = ids.first().filter(|id| !id.is_empty()) + { + value = Cow::Owned(id.to_string()); + return true; + } + false + }); + self.settle_scan(meter)?; + } + Ok(value) + } +} diff --git a/crates/sieve/src/runtime/tests/test_envelope.rs b/crates/sieve/src/runtime/tests/test_envelope.rs new file mode 100644 index 00000000..61b92655 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_envelope.rs @@ -0,0 +1,256 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Envelope, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{AddressPart, Comparator, MatchType}, + }, + runtime::{RuntimeError, handler::Handler}, +}; +use mail_parser::DateTime; +use smallvec::SmallVec; + +type EnvelopeList = SmallVec<[Envelope; 4]>; + +impl<'x> Context<'x> { + pub(crate) fn test_envelope>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestEnvelope, + handler: &mut H, + ) -> Result { + let key_list = self.eval_keys(script, test.key_list)?; + let envelope_list: EnvelopeList = script + .recs(test.envelope_list)? + .map(|rec| Envelope::from_code(rec.b)) + .collect(); + let address_part = AddressPart::from_code(test.address_part); + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + + let result = match &match_type { + MatchType::Is | MatchType::Contains | MatchType::Value(_) => { + self.find_envelopes(&envelope_list, address_part, test.zone, |value| { + key_list.iter().any(|key| match &match_type { + MatchType::Is => comparator.is(&value, &key.value), + MatchType::Contains => { + comparator.contains(value, key.value.to_string().as_ref()) + } + MatchType::Value(rel_match) => { + comparator.relational(rel_match, &value, &key.value) + } + _ => false, + }) + }) + } + MatchType::Matches(capture_positions) | MatchType::Regex(capture_positions) => { + let mut captured_values = Vec::new(); + let is_matches = matches!(&match_type, MatchType::Matches(_)); + let to_lower = comparator.is_casemap(); + let mut error = None; + let result = + self.find_envelopes(&envelope_list, address_part, test.zone, |value| { + for key in &key_list { + let matched = if is_matches { + self.glob_matches( + script, + to_lower, + key, + value, + *capture_positions, + &mut captured_values, + ) + } else { + self.regex_matches( + script, + key, + value, + *capture_positions, + &mut captured_values, + ) + }; + match matched { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + false + }); + if let Some(err) = error { + return Err(err); + } + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + result + } + MatchType::Count(rel_match) => { + let mut count: i64 = 0; + self.find_envelopes(&envelope_list, address_part, test.zone, |value| { + if !value.is_empty() { + count += 1; + } + false + }); + + key_list + .iter() + .any(|key| rel_match.cmp(&Number::from(count), &key.value.to_number())) + } + MatchType::List => { + let mut values: Vec<&str> = Vec::new(); + self.find_envelopes(&envelope_list, address_part, test.zone, |value| { + if !value.is_empty() && !values.contains(&value) { + values.push(self.alloc_str(value)); + } + false + }); + + if !values.is_empty() { + let lists: SmallVec<[&str; 4]> = key_list + .iter() + .map(|key| self.intern_cow(key.value.clone().into_string())) + .collect(); + return TestResult::from_reply( + handler.list_contains(self, &lists, &values, comparator.as_match()), + test.is_not, + ); + } + + false + } + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + fn find_envelopes( + &self, + envelope_list: &[Envelope], + address_part: AddressPart, + zone: Option, + mut cb: impl FnMut(&str) -> bool, + ) -> bool { + for (name, value) in &self.envelope { + if envelope_list.contains(name) + && match (name, zone) { + (Envelope::From | Envelope::To | Envelope::Orcpt, _) => { + if let Some(value) = address_part.eval_string(value.to_string().as_ref()) { + cb(value) + } else { + false + } + } + (Envelope::ByTimeAbsolute, Some(zone)) => { + if let Some(dt) = DateTime::parse_rfc3339(value.to_string().as_ref()) { + cb(&dt.to_timezone(zone).to_rfc3339()) + } else { + cb("") + } + } + _ => cb(value.to_string().as_ref()), + } + { + return true; + } + } + false + } +} + +pub fn parse_envelope_address(addr: &str) -> Option<&str> { + let addr = addr.as_bytes(); + let mut addr_start_pos = 0; + let mut addr_end_pos = addr.len(); + let mut last_ch = 0; + let mut at_pos = 0; + let mut has_bracket = false; + let mut in_path = false; + + if addr.is_empty() { + return "".into(); + } + + for (pos, &ch) in addr.iter().enumerate() { + match ch { + b'<' => { + if pos == 0 { + addr_start_pos = pos + 1; + has_bracket = true; + } else { + return None; + } + } + b'>' => { + if has_bracket && pos == addr.len() - 1 { + if addr.len() > 2 { + has_bracket = false; + addr_end_pos = pos; + } else { + return "".into(); + } + } else { + return None; + } + } + b':' => { + if at_pos != 0 { + at_pos = 0; + addr_start_pos = pos + 1; + in_path = false; + } else { + return None; + } + } + b',' => { + if at_pos != 0 { + at_pos = 0; + in_path = true; + } else { + return None; + } + } + b'@' => { + if at_pos == 0 && pos != addr.len() - 1 { + at_pos = pos; + } else { + return None; + } + } + b'.' => { + if (at_pos != 0 && last_ch == b'.') || last_ch == b'@' { + return None; + } + } + _ => { + if ch.is_ascii_whitespace() || !ch.is_ascii() { + return None; + } + } + } + + last_ch = ch; + } + + if !has_bracket && !in_path && at_pos > addr_start_pos && addr_end_pos - 1 > at_pos { + std::str::from_utf8(&addr[addr_start_pos..addr_end_pos]).ok() + } else { + match addr.get(addr_start_pos..addr_end_pos) { + Some(addr) if at_pos == 0 && addr.eq_ignore_ascii_case(b"mailer-daemon") => { + std::str::from_utf8(addr).ok() + } + _ => None, + } + } +} diff --git a/crates/sieve/src/runtime/tests/test_exists.rs b/crates/sieve/src/runtime/tests/test_exists.rs new file mode 100644 index 00000000..86b63a67 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_exists.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Sieve, + bytecode::ops, + runtime::{ + RuntimeError, + message::parts::{PartCursor, Scope}, + tests::test_header::HeaderKeys, + }, +}; +use mail_parser::HeaderName; +use smallvec::{SmallVec, smallvec}; + +impl<'x> Context<'x> { + pub(crate) fn test_exists( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestExists, + ) -> Result { + let header_names = self.parse_header_names(script, test.header_names)?; + let Some(part) = self.current_part() else { + return Ok(TestResult::Bool(test.is_not)); + }; + if !test.mime_anychild { + let result = self.edits.hidden_end(part.id()).is_none() + && header_names + .iter() + .all(|name| self.named_headers(part, name).next().is_some()); + return Ok(TestResult::Bool(result ^ test.is_not)); + } + let keys: HeaderKeys<'_> = header_names.iter().map(HeaderName::key).collect(); + let mut header_exists: SmallVec<[bool; 8]> = smallvec![false; header_names.len()]; + let mut result = false; + let mut parts = PartCursor::subtree(part, true, Scope::Message); + let mut meter = self.scan_meter(header_names.len()); + + while let Some(part) = self.advance(&mut parts) { + if !meter.charge_headers(part) { + break; + } + for ((exists, header_name), key) in header_exists + .iter_mut() + .zip(header_names.iter()) + .zip(keys.iter()) + { + if !*exists && self.keyed_headers(part, header_name, *key).next().is_some() { + *exists = true; + } + } + + if header_exists.iter().all(|v| *v) { + result = true; + break; + } + } + self.settle_scan(meter)?; + + Ok(TestResult::Bool(result ^ test.is_not)) + } +} diff --git a/crates/sieve/src/runtime/tests/test_extlists.rs b/crates/sieve/src/runtime/tests/test_extlists.rs new file mode 100644 index 00000000..bbbda375 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_extlists.rs @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{Context, Sieve, bytecode::ops, runtime::RuntimeError}; + +impl<'x> Context<'x> { + pub(crate) fn test_valid_ext_list( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestValidExtList, + ) -> Result { + let all_valid = self + .eval_values(script, test.list_names)? + .iter() + .all(|list| { + self.runtime + .valid_ext_lists + .contains(list.to_string().as_ref()) + }); + + Ok(TestResult::Bool(all_valid ^ test.is_not)) + } +} diff --git a/crates/sieve/src/runtime/tests/test_hasflag.rs b/crates/sieve/src/runtime/tests/test_hasflag.rs new file mode 100644 index 00000000..9bccc57c --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_hasflag.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{TestResult, glob::CompiledGlob}; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::{RuntimeError, Variable, eval::ValueRef}, +}; +use smallvec::SmallVec; + +impl<'x> Context<'x> { + pub(crate) fn test_hasflag( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestHasFlag, + ) -> Result { + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let use_global_flags = test.variable_list.is_empty(); + let mut variables: SmallVec<[Variable<'x>; 4]> = SmallVec::new(); + let mut iter = script.recs(test.variable_list)?; + while let Some(rec) = iter.next() { + let value = ValueRef::decode(script, rec, &mut iter)?; + if let Some(flags) = self.variable_ref(script, value)? + && !flags.is_empty() + { + variables.push(flags); + } + } + + let result = if let MatchType::Count(rel_match) = &match_type { + let flag_count = if use_global_flags { + self.global_flags().len() + } else { + variables + .iter() + .map(|flags| flags.to_string().split(' ').count()) + .sum() + }; + + self.eval_values(script, test.flags)? + .iter() + .any(|key| rel_match.cmp(&Number::from(flag_count as i64), &key.to_number())) + } else { + let mut captured_values = Vec::new(); + let result = self.tokenize_flags(script, test.flags, |check_flag| { + if use_global_flags { + self.global_flags().iter().any(|flag| { + check_flag_matches( + &comparator, + &match_type, + flag, + check_flag, + &mut captured_values, + ) + }) + } else { + variables.iter().any(|flags| { + flags.to_string().split(' ').any(|flag| { + check_flag_matches( + &comparator, + &match_type, + flag, + check_flag, + &mut captured_values, + ) + }) + }) + } + })?; + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + result + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } +} + +fn check_flag_matches( + comparator: &Comparator, + match_type: &MatchType, + flag: &str, + check_flag: &str, + captured_values: &mut Vec<(usize, String)>, +) -> bool { + match match_type { + MatchType::Is => comparator.is(&flag, &check_flag), + MatchType::Contains => comparator.contains(flag, check_flag), + MatchType::Value(rel_match) => comparator.relational(rel_match, &flag, &check_flag), + MatchType::Matches(capture_positions) | MatchType::Regex(capture_positions) => { + let glob = CompiledGlob::compile(check_flag, comparator.is_casemap()); + if *capture_positions == 0 { + glob.matches(flag) + } else { + glob.capture(flag, *capture_positions, captured_values) + } + } + MatchType::Count(_) | MatchType::List => false, + } +} diff --git a/crates/sieve/src/runtime/tests/test_header.rs b/crates/sieve/src/runtime/tests/test_header.rs new file mode 100644 index 00000000..0c31202c --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_header.rs @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ScanMeter, TestResult}; +use crate::{ + Context, Sieve, + bytecode::{ + ops::{self, MimeOpts}, + rec::{Range, Rec, tag}, + }, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::{ + RuntimeError, + eval::ValueRef, + handler::Handler, + message::{ + headers::HeaderRef, + parts::{PartCursor, Scope}, + }, + }, +}; +use mail_parser::{HeaderForm, HeaderKey, HeaderName, HeaderValue, MessagePart, PartId}; +use smallvec::SmallVec; + +pub(crate) type HeaderNames<'x> = SmallVec<[HeaderName<'x>; 4]>; +pub(crate) type HeaderKeys<'x> = SmallVec<[HeaderKey<'x>; 4]>; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum MimeOptsRef<'x> { + None, + Type, + Subtype, + ContentType, + Param(SmallVec<[&'x str; 2]>), +} + +impl<'x> Context<'x> { + pub(crate) fn test_header>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestHeader, + handler: &mut H, + ) -> Result { + let key_list = self.eval_keys(script, test.key_list)?; + let header_list = self.parse_header_names(script, test.header_list)?; + let mime_opts = self.mime_opts(script, &test.mime_opts)?; + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + + let result = match &match_type { + MatchType::Is | MatchType::Contains | MatchType::Value(_) => { + let mut meter = self.scan_meter(key_list.len()); + let result = self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_header_values(header, &mime_opts, |value| { + key_list.iter().any(|key| match &match_type { + MatchType::Is => comparator.is(&value, &key.value), + MatchType::Contains => { + comparator.contains(value, key.value.to_string().as_ref()) + } + MatchType::Value(rel_match) => { + comparator.relational(rel_match, &value, &key.value) + } + _ => false, + }) + }) + }, + ); + self.settle_scan(meter)?; + result + } + MatchType::Matches(capture_positions) | MatchType::Regex(capture_positions) => { + let mut meter = self.scan_meter(key_list.len()); + let mut captured_values = Vec::new(); + let is_matches = matches!(&match_type, MatchType::Matches(_)); + let to_lower = comparator.is_casemap(); + let mut error = None; + let result = self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_header_values(header, &mime_opts, |value| { + for key in &key_list { + let matched = if is_matches { + self.glob_matches( + script, + to_lower, + key, + value, + *capture_positions, + &mut captured_values, + ) + } else { + self.regex_matches( + script, + key, + value, + *capture_positions, + &mut captured_values, + ) + }; + match matched { + Ok(true) => return true, + Ok(false) => (), + Err(err) => { + error = Some(err); + return true; + } + } + } + false + }) + }, + ); + self.settle_scan(meter)?; + if let Some(err) = error { + return Err(err); + } + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + result + } + MatchType::Count(rel_match) => { + let mut count: i64 = 0; + let mut meter = self.scan_meter(1); + self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + match &mime_opts { + MimeOptsRef::None => { + count += 1; + } + MimeOptsRef::Type | MimeOptsRef::Subtype | MimeOptsRef::ContentType => { + if let HeaderValue::ContentType(_) = header.value() { + count += 1; + } + } + MimeOptsRef::Param(params) => { + if let HeaderValue::ContentType(ct) = header.value() { + count += + ct.attributes() + .filter(|(name, _)| { + params.iter().any(|p| p.eq_ignore_ascii_case(name)) + }) + .count() as i64; + } + } + } + + false + }, + ); + self.settle_scan(meter)?; + + key_list + .iter() + .any(|key| rel_match.cmp(&Number::from(count), &key.value.to_number())) + } + MatchType::List => { + let mut values: Vec<&str> = Vec::new(); + let mut meter = self.scan_meter(1); + self.find_headers( + &header_list, + test.index, + test.mime_anychild, + &mut meter, + |header, _| { + self.find_header_values(header, &mime_opts, |value| { + if !value.is_empty() && !values.contains(&value) { + values.push(self.alloc_str(value)); + } + false + }) + }, + ); + self.settle_scan(meter)?; + + if !values.is_empty() { + let lists: SmallVec<[&str; 4]> = key_list + .iter() + .map(|key| self.intern_cow(key.value.clone().into_string())) + .collect(); + return TestResult::from_reply( + handler.list_contains(self, &lists, &values, comparator.as_match()), + test.is_not, + ); + } + + false + } + }; + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + pub(crate) fn mime_opts( + &self, + script: &'x Sieve<'x>, + opts: &MimeOpts, + ) -> Result, RuntimeError> { + Ok(match opts.kind { + 0 => MimeOptsRef::Type, + 1 => MimeOptsRef::Subtype, + 2 => MimeOptsRef::ContentType, + 3 => MimeOptsRef::Param( + self.eval_strings(script, opts.params)? + .into_iter() + .collect(), + ), + _ => MimeOptsRef::None, + }) + } + + pub(crate) fn parse_header_names( + &self, + script: &'x Sieve<'x>, + header_names: Range, + ) -> Result, RuntimeError> { + let mut result = HeaderNames::with_capacity(header_names.len as usize); + let mut iter = script.recs(header_names)?; + while let Some(rec) = iter.next() { + if rec.tag == tag::HEADER { + result.push(borrow_header_name(script.header_name(rec.c)?)); + } else { + let value = ValueRef::decode(script, rec, &mut iter)?; + let value = self.eval_value_ref(script, value)?; + if let Some(header_name) = + self.parse_header_name_str(self.intern_cow(value.into_string())) + { + result.push(header_name); + } + } + } + Ok(result) + } + + #[inline(always)] + pub(crate) fn parse_header_name( + &self, + script: &'x Sieve<'x>, + header_name: Rec, + ) -> Result>, RuntimeError> { + if header_name.tag == tag::HEADER { + return Ok(Some(borrow_header_name(script.header_name(header_name.c)?))); + } + let name = self.eval_str(script, header_name)?; + Ok(self.parse_header_name_str(name)) + } + + #[inline(always)] + pub(crate) fn parse_header_name_str(&self, name: &'x str) -> Option> { + HeaderName::parse(name) + } + + pub(crate) fn find_headers( + &self, + header_names: &[HeaderName<'_>], + index: Option, + any_child: bool, + meter: &mut ScanMeter, + mut visitor_fnc: impl FnMut(HeaderRef<'x>, PartId) -> bool, + ) -> bool { + let Some(part) = self.current_part() else { + return false; + }; + if !any_child { + return self.edits.hidden_end(part.id()).is_none() + && (!meter.charge_headers(part) + || self.find_part_headers( + part, + header_names.iter().map(|name| (name, name.key())), + index, + &mut visitor_fnc, + )); + } + let keys: HeaderKeys<'_> = header_names.iter().map(HeaderName::key).collect(); + let mut parts = PartCursor::subtree(part, true, Scope::Message); + while let Some(part) = self.advance(&mut parts) { + if !meter.charge_headers(part) { + return true; + } + if self.find_part_headers( + part, + header_names.iter().zip(keys.iter().copied()), + index, + &mut visitor_fnc, + ) { + return true; + } + } + false + } + + #[inline(always)] + fn find_part_headers<'s, 'n: 's>( + &self, + part: MessagePart<'x>, + mut names: impl Iterator, HeaderKey<'s>)>, + index: Option, + visitor_fnc: &mut impl FnMut(HeaderRef<'x>, PartId) -> bool, + ) -> bool { + let id = part.id(); + match self.edits.headers(id) { + None => { + let headers = part.headers(); + names.any(|(_, key)| { + visit_indexed( + || headers.all_key(key).map(HeaderRef::Parsed), + index, + |header| visitor_fnc(header, id), + ) + }) + } + Some(edited) => names.any(|(name, _)| { + visit_indexed( + || { + edited + .iter() + .filter(|header| header.is_named(name)) + .copied() + }, + index, + |header| visitor_fnc(header, id), + ) + }), + } + } + + pub(crate) fn find_header_values( + &self, + header: HeaderRef<'x>, + mime_opts: &MimeOptsRef<'_>, + mut visitor_fnc: impl FnMut(&str) -> bool, + ) -> bool { + if let MimeOptsRef::None = mime_opts { + return match (header, header.value()) { + (HeaderRef::Added(added), _) => visitor_fnc(added.value), + (header, HeaderValue::Text(text)) + if matches!( + header.name(), + HeaderName::Subject + | HeaderName::Comments + | HeaderName::ContentDescription + | HeaderName::ContentLocation + | HeaderName::ContentTransferEncoding, + ) => + { + visitor_fnc(text) + } + (header, _) => visitor_fnc( + header + .parse_as(HeaderForm::Text) + .value() + .as_text() + .unwrap_or_default(), + ), + }; + } + + let parsed; + let content_type = match header { + HeaderRef::Added(_) => { + parsed = header.parse_as(HeaderForm::ContentType); + parsed.value().as_content_type() + } + header => header.value().as_content_type(), + }; + let Some(content_type) = content_type else { + return visitor_fnc(""); + }; + + match mime_opts { + MimeOptsRef::Type => visitor_fnc(content_type.ctype()), + MimeOptsRef::Subtype => visitor_fnc(content_type.subtype().unwrap_or_default()), + MimeOptsRef::ContentType => match content_type.subtype() { + Some(subtype) => visitor_fnc(&format!("{}/{}", content_type.ctype(), subtype)), + None => visitor_fnc(content_type.ctype()), + }, + MimeOptsRef::Param(params) => { + for param in params { + for (name, value) in content_type.attributes() { + if param.eq_ignore_ascii_case(name) && visitor_fnc(value) { + return true; + } + } + } + visitor_fnc("") + } + MimeOptsRef::None => visitor_fnc(""), + } + } +} + +#[inline(always)] +fn visit_indexed<'x, I: Iterator>>( + named: impl Fn() -> I, + index: Option, + mut visit: impl FnMut(HeaderRef<'x>) -> bool, +) -> bool { + match index { + None => named().any(visit), + Some(0) => false, + Some(index) if index > 0 => named().nth(index as usize - 1).is_some_and(visit), + Some(index) => { + let position = index.unsigned_abs() as usize; + let count = named().count(); + count >= position && named().nth(count - position).is_some_and(&mut visit) + } + } +} + +pub(crate) fn borrow_header_name<'y>(name: &'y HeaderName<'static>) -> HeaderName<'y> { + match name { + HeaderName::Other(name) => HeaderName::Other(std::borrow::Cow::Borrowed(name.as_ref())), + other => other.clone(), + } +} diff --git a/crates/sieve/src/runtime/tests/test_metadata.rs b/crates/sieve/src/runtime/tests/test_metadata.rs new file mode 100644 index 00000000..9b821edc --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_metadata.rs @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Metadata, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::RuntimeError, +}; + +impl<'x> Context<'x> { + pub(crate) fn test_metadata( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestMetadata, + ) -> Result { + let mailbox = if test.metadata.kind == 1 { + Some(self.eval_value(script, test.metadata.name)?.into_string()) + } else { + None + }; + let annotation = self + .eval_value(script, test.metadata.annotation)? + .into_string(); + + let Some((_, value)) = [&self.metadata, &self.runtime.metadata] + .into_iter() + .flatten() + .find(|(m, _)| match (m, &mailbox) { + (Metadata::Server { annotation: a }, None) => a.eq_ignore_ascii_case(&annotation), + ( + Metadata::Mailbox { + name: a, + annotation: c, + }, + Some(b), + ) => a.eq(b) && c.eq_ignore_ascii_case(&annotation), + _ => false, + }) + else { + return Ok(TestResult::Bool(false ^ test.is_not)); + }; + let value = value.as_ref(); + + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let mut result = false; + + if let MatchType::Count(rel_match) = &match_type { + result = self + .eval_values(script, test.key_list)? + .iter() + .any(|key| rel_match.cmp(&Number::Float(1.0), &key.to_number())); + } else { + let keys = self.eval_keys(script, test.key_list)?; + let mut captured_values = Vec::new(); + + for key in &keys { + if self.key_matches( + script, + &comparator, + &match_type, + key, + value, + &mut captured_values, + )? { + result = true; + break; + } + } + + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + } + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + pub(crate) fn test_metadata_exists( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestMetadataExists, + ) -> Result { + let mailbox = self + .eval_opt(script, test.mailbox)? + .map(|mailbox| mailbox.into_string()); + let mut annotations = self.eval_values(script, test.annotation_names)?; + + for (metadata, _) in [&self.metadata, &self.runtime.metadata] + .into_iter() + .flatten() + { + match (metadata, mailbox.as_deref()) { + (Metadata::Server { annotation }, None) => { + annotations.retain(|a| !a.to_string().eq_ignore_ascii_case(annotation)) + } + (Metadata::Mailbox { name, annotation }, Some(mailbox)) if name.eq(mailbox) => { + annotations.retain(|a| !a.to_string().eq_ignore_ascii_case(annotation)); + } + _ => (), + } + if annotations.is_empty() { + return Ok(TestResult::Bool(true ^ test.is_not)); + } + } + + Ok(TestResult::Bool(false ^ test.is_not)) + } +} diff --git a/crates/sieve/src/runtime/tests/test_notify.rs b/crates/sieve/src/runtime/tests/test_notify.rs new file mode 100644 index 00000000..42be6d20 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_notify.rs @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::{RuntimeError, actions::action_notify::validate_uri}, +}; + +impl<'x> Context<'x> { + pub(crate) fn test_notify_method_capability( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestNotifyMethodCapability, + ) -> Result { + let uri = self.eval_value(script, test.notification_uri)?; + let uri = uri.to_string(); + let capability = self.eval_value(script, test.notification_capability)?; + if !capability.to_string().eq_ignore_ascii_case("online") + || !self.is_valid_notification_uri(uri.as_ref()) + { + return Ok(TestResult::Bool(false ^ test.is_not)); + } + + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + + if let MatchType::Count(rel_match) = &match_type { + let matched = self + .eval_values(script, test.key_list)? + .iter() + .any(|key| rel_match.cmp(&Number::from(1.0), &key.to_number())); + return Ok(TestResult::Bool(matched ^ test.is_not)); + } + + let mut captured_values = Vec::new(); + for key in &self.eval_keys(script, test.key_list)? { + let matched = match &match_type { + MatchType::Is => comparator.is(&"maybe", &key.value), + MatchType::Contains => comparator.contains("maybe", key.value.to_string().as_ref()), + MatchType::Value(relation) => comparator.relational(relation, &"maybe", &key.value), + MatchType::Matches(_) => self.glob_matches( + script, + comparator.is_casemap(), + key, + "maybe", + 0, + &mut captured_values, + )?, + MatchType::Regex(_) => { + self.regex_matches(script, key, "maybe", 0, &mut captured_values)? + } + _ => false, + }; + if matched { + return Ok(TestResult::Bool(true ^ test.is_not)); + } + } + + Ok(TestResult::Bool(false ^ test.is_not)) + } + + pub(crate) fn test_valid_notify_method( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestValidNotifyMethod, + ) -> Result { + let all_valid = self + .eval_values(script, test.notification_uris)? + .iter() + .all(|uri| self.is_valid_notification_uri(uri.to_string().as_ref())); + + Ok(TestResult::Bool(all_valid ^ test.is_not)) + } + + fn is_valid_notification_uri(&self, uri: &str) -> bool { + validate_uri(uri).is_some_and(|scheme| { + self.runtime.valid_notification_uris.contains(scheme) + || self.runtime.valid_notification_uris.contains(uri) + }) + } +} diff --git a/crates/sieve/src/runtime/tests/test_size.rs b/crates/sieve/src/runtime/tests/test_size.rs new file mode 100644 index 00000000..87ca1547 --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_size.rs @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{Context, bytecode::ops}; + +impl Context<'_> { + pub(crate) fn test_size(&self, test: &ops::TestSize) -> TestResult { + let size = self.message_size as u64; + let result = if test.over { + size > test.limit + } else { + size < test.limit + }; + TestResult::Bool(result ^ test.is_not) + } +} diff --git a/crates/sieve/src/runtime/tests/test_spamtest.rs b/crates/sieve/src/runtime/tests/test_spamtest.rs new file mode 100644 index 00000000..ca6039bc --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_spamtest.rs @@ -0,0 +1,197 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + TestResult, + matching::{Key, Pattern}, +}; +use crate::{ + Context, Sieve, SpamStatus, VirusStatus, + bytecode::{ + ops, + rec::{Rec, tag}, + }, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::{RuntimeError, Variable}, +}; + +impl<'x> Context<'x> { + pub(crate) fn test_spamtest( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestSpamTest, + ) -> Result { + let status = if test.percent { + self.spam_status.as_percentage() + } else { + self.spam_status.as_number() + }; + let count = if matches!(self.spam_status, SpamStatus::Unknown) { + 0.0 + } else { + 1.1 + }; + self.test_status(script, test, status, count) + } + + pub(crate) fn test_virustest( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestVirusTest, + ) -> Result { + let status = self.virus_status.as_number(); + let count = if matches!(self.virus_status, VirusStatus::Unknown) { + 0.0 + } else { + 1.1 + }; + let test = ops::TestSpamTest { + value: test.value, + match_type: test.match_type, + comparator: test.comparator, + percent: false, + is_not: test.is_not, + }; + self.test_status(script, &test, status, count) + } + + fn test_status( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestSpamTest, + status: Variable<'static>, + count: f64, + ) -> Result { + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + let key = self.eval_key(script, test.value)?; + let mut captured_values = Vec::new(); + + let result = match &match_type { + MatchType::Count(rel_match) => { + rel_match.cmp(&Number::from(count), &key.value.to_number()) + } + MatchType::List => false, + _ => self.key_matches( + script, + &comparator, + &match_type, + &key, + status.to_string().as_ref(), + &mut captured_values, + )?, + }; + + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + + Ok(TestResult::Bool(result ^ test.is_not)) + } + + fn eval_key(&self, script: &'x Sieve<'x>, rec: Rec) -> Result, RuntimeError> { + let pattern = match rec.tag { + tag::GLOB => Pattern::Glob(rec.c), + tag::REGEX => Pattern::Regex(rec.c), + _ => Pattern::Dynamic, + }; + Ok(Key { + value: self.eval_value(script, rec)?, + pattern, + }) + } +} + +impl SpamStatus { + pub fn from_number(number: u32) -> Self { + match number { + 1 => SpamStatus::Ham, + 2..=9 => SpamStatus::MaybeSpam(number as f64 / 10.0), + 10 => SpamStatus::Spam, + _ => SpamStatus::Unknown, + } + } + + pub(crate) fn as_number(&self) -> Variable<'static> { + Variable::Integer(match self { + SpamStatus::Unknown => 0, + SpamStatus::Ham => 1, + SpamStatus::MaybeSpam(pct) => ((pct * 10.0) as i64).clamp(2, 9), + SpamStatus::Spam => 10, + }) + } + + pub(crate) fn as_percentage(&self) -> Variable<'static> { + Variable::Integer(match self { + SpamStatus::Unknown | SpamStatus::Ham => 0, + SpamStatus::MaybeSpam(pct) => ((pct * 100.0).ceil() as i64).clamp(1, 100), + SpamStatus::Spam => 100, + }) + } +} + +impl VirusStatus { + pub fn from_number(number: u32) -> Self { + match number { + 1 => VirusStatus::Clean, + 2 => VirusStatus::Replaced, + 3 => VirusStatus::Cured, + 4 => VirusStatus::MaybeVirus, + 5 => VirusStatus::Virus, + _ => VirusStatus::Unknown, + } + } + + pub(crate) fn as_number(&self) -> Variable<'static> { + Variable::Integer(match self { + VirusStatus::Unknown => 0, + VirusStatus::Clean => 1, + VirusStatus::Replaced => 2, + VirusStatus::Cured => 3, + VirusStatus::MaybeVirus => 4, + VirusStatus::Virus => 5, + }) + } +} + +impl From for SpamStatus { + fn from(number: u32) -> Self { + SpamStatus::from_number(number) + } +} + +impl From for SpamStatus { + fn from(number: i32) -> Self { + SpamStatus::from_number(number as u32) + } +} + +impl From for SpamStatus { + fn from(number: usize) -> Self { + SpamStatus::from_number(number as u32) + } +} + +impl From for VirusStatus { + fn from(number: u32) -> Self { + VirusStatus::from_number(number) + } +} + +impl From for VirusStatus { + fn from(number: i32) -> Self { + VirusStatus::from_number(number as u32) + } +} + +impl From for VirusStatus { + fn from(number: usize) -> Self { + VirusStatus::from_number(number as u32) + } +} diff --git a/crates/sieve/src/runtime/tests/test_string.rs b/crates/sieve/src/runtime/tests/test_string.rs new file mode 100644 index 00000000..f9e98e8e --- /dev/null +++ b/crates/sieve/src/runtime/tests/test_string.rs @@ -0,0 +1,128 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::TestResult; +use crate::{ + Context, Sieve, + bytecode::ops, + compiler::{ + Number, + grammar::{Comparator, MatchType}, + }, + runtime::{RuntimeError, handler::Handler}, +}; +use smallvec::SmallVec; + +impl<'x> Context<'x> { + pub(crate) fn test_environment>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestEnvironment, + handler: &mut H, + ) -> Result { + let test = ops::TestString { + match_type: test.match_type, + comparator: test.comparator, + source: test.source, + key_list: test.key_list, + is_not: test.is_not, + }; + self.test_string(script, &test, true, handler) + } + + pub(crate) fn test_string>( + &mut self, + script: &'x Sieve<'x>, + test: &ops::TestString, + empty_is_null: bool, + handler: &mut H, + ) -> Result { + let mut result = false; + let comparator = Comparator::from_code(test.comparator); + let match_type = test.match_type.match_type(); + + match &match_type { + MatchType::Count(rel) => { + let sources = self.eval_values(script, test.source)?; + let num_items = sources.iter().filter(|x| !x.is_empty()).count() as i64; + if !empty_is_null || num_items > 0 { + for key in self.eval_values(script, test.key_list)? { + if rel.cmp(&Number::from(num_items), &key.to_number()) { + result = true; + break; + } + } + } + } + MatchType::List => { + let sources = self.eval_values(script, test.source)?; + let mut values: SmallVec<[&str; 4]> = SmallVec::with_capacity(sources.len()); + for source in &sources { + let value = self.intern_cow(source.clone().into_string()); + if !value.is_empty() && !values.contains(&value) { + values.push(value); + } + } + if !values.is_empty() { + let lists = self.eval_strings(script, test.key_list)?; + return TestResult::from_reply( + handler.list_contains(self, &lists, &values, comparator.as_match()), + test.is_not, + ); + } + } + _ => { + let mut captured_values = Vec::new(); + let sources = self.eval_values(script, test.source)?; + let keys = self.eval_keys(script, test.key_list)?; + self.charge_scan(keys.len(), sources.iter().map(|source| source.len()).sum())?; + + 'outer: for key in &keys { + for source in &sources { + if !empty_is_null || !source.is_empty() { + result = match &match_type { + MatchType::Is => comparator.is(source, &key.value), + MatchType::Contains => comparator.contains( + source.to_string().as_ref(), + key.value.to_string().as_ref(), + ), + MatchType::Value(relation) => { + comparator.relational(relation, source, &key.value) + } + MatchType::Matches(capture_positions) => self.glob_matches( + script, + comparator.is_casemap(), + key, + source.to_string().as_ref(), + *capture_positions, + &mut captured_values, + )?, + MatchType::Regex(capture_positions) => self.regex_matches( + script, + key, + source.to_string().as_ref(), + *capture_positions, + &mut captured_values, + )?, + _ => false, + }; + + if result { + break 'outer; + } + } + } + } + + if !captured_values.is_empty() { + self.set_match_variables(captured_values); + } + } + } + + Ok(TestResult::Bool(result ^ test.is_not)) + } +} diff --git a/crates/sieve/src/runtime/variable.rs b/crates/sieve/src/runtime/variable.rs new file mode 100644 index 00000000..a2591617 --- /dev/null +++ b/crates/sieve/src/runtime/variable.rs @@ -0,0 +1,435 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::compiler::Number; +use std::{borrow::Cow, cmp::Ordering, fmt::Display, hash::Hash}; + +#[derive(Debug, Clone)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +pub enum Variable<'x> { + String(Cow<'x, str>), + Integer(i64), + Float(f64), + Array(Array<'x>), +} + +#[derive(Debug, Clone)] +pub enum Array<'x> { + Borrowed(&'x [Variable<'x>]), + Owned(Vec>), +} + +impl<'x> Array<'x> { + #[inline(always)] + pub fn as_slice(&self) -> &[Variable<'x>] { + match self { + Array::Borrowed(items) => items, + Array::Owned(items) => items, + } + } + + pub fn into_vec(self) -> Vec> { + match self { + Array::Borrowed(items) => items.to_vec(), + Array::Owned(items) => items, + } + } + + #[inline(always)] + pub fn is_borrowed(&self) -> bool { + matches!(self, Array::Borrowed(_)) + } +} + +impl<'x> std::ops::Deref for Array<'x> { + type Target = [Variable<'x>]; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl<'x> From>> for Array<'x> { + fn from(items: Vec>) -> Self { + Array::Owned(items) + } +} + +impl<'x> From<&'x [Variable<'x>]> for Array<'x> { + fn from(items: &'x [Variable<'x>]) -> Self { + Array::Borrowed(items) + } +} + +impl PartialEq for Array<'_> { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl Eq for Array<'_> {} + +impl PartialOrd for Array<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + self.as_slice().partial_cmp(other.as_slice()) + } +} + +impl Hash for Array<'_> { + fn hash(&self, state: &mut H) { + self.as_slice().hash(state) + } +} + +impl Default for Variable<'_> { + #[inline(always)] + fn default() -> Self { + Variable::String(Cow::Borrowed("")) + } +} + +impl<'x> Variable<'x> { + #[inline(always)] + pub const fn empty() -> Self { + Variable::String(Cow::Borrowed("")) + } + + #[inline(always)] + pub const fn borrowed(s: &'x str) -> Self { + Variable::String(Cow::Borrowed(s)) + } + + pub fn to_string(&self) -> Cow<'_, str> { + match self { + Variable::String(s) => Cow::Borrowed(s.as_ref()), + Variable::Integer(n) => Cow::Owned(n.to_string()), + Variable::Float(n) => Cow::Owned(n.to_string()), + Variable::Array(l) => Cow::Owned(array_to_string(l)), + } + } + + pub fn into_string(self) -> Cow<'x, str> { + match self { + Variable::String(s) => s, + Variable::Integer(n) => Cow::Owned(n.to_string()), + Variable::Float(n) => Cow::Owned(n.to_string()), + Variable::Array(l) => Cow::Owned(array_to_string(&l)), + } + } + + #[inline(always)] + pub fn as_str(&self) -> Option<&str> { + match self { + Variable::String(s) => Some(s.as_ref()), + _ => None, + } + } + + pub fn to_number(&self) -> Number { + self.to_number_checked() + .unwrap_or(Number::Float(f64::INFINITY)) + } + + pub fn to_number_checked(&self) -> Option { + let s = match self { + Variable::Integer(n) => return Number::Integer(*n).into(), + Variable::Float(n) => return Number::Float(*n).into(), + Variable::String(s) if !s.is_empty() => s.as_ref(), + _ => return None, + }; + + if !s.contains('.') { + s.parse::().map(Number::Integer).ok() + } else { + s.parse::().map(Number::Float).ok() + } + } + + pub fn to_integer(&self) -> i64 { + match self { + Variable::Integer(n) => *n, + Variable::Float(n) => *n as i64, + Variable::String(s) if !s.is_empty() => s.parse::().unwrap_or(0), + _ => 0, + } + } + + pub fn to_usize(&self) -> usize { + match self { + Variable::Integer(n) => *n as usize, + Variable::Float(n) => *n as usize, + Variable::String(s) if !s.is_empty() => s.parse::().unwrap_or(0), + _ => 0, + } + } + + pub fn len(&self) -> usize { + match self { + Variable::String(s) => s.len(), + Variable::Integer(_) | Variable::Float(_) => 2, + Variable::Array(l) => l.iter().map(|v| v.len() + 2).sum(), + } + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + match self { + Variable::String(s) => s.is_empty(), + _ => false, + } + } + + #[inline(always)] + pub fn as_array(&self) -> Option<&[Variable<'x>]> { + match self { + Variable::Array(l) => Some(l), + _ => None, + } + } + + pub fn into_array(self) -> Array<'x> { + match self { + Variable::Array(l) => l, + v if !v.is_empty() => Array::Owned(vec![v]), + _ => Array::Borrowed(&[]), + } + } + + pub fn to_array(&self) -> Cow<'_, [Variable<'x>]> { + match self { + Variable::Array(l) => Cow::Borrowed(l.as_slice()), + v if !v.is_empty() => Cow::Owned(vec![v.clone()]), + _ => Cow::Borrowed(&[]), + } + } + + pub fn into_string_array(self) -> Vec { + match self { + Variable::Array(l) => l.iter().map(|i| i.to_string().into_owned()).collect(), + v if !v.is_empty() => vec![v.to_string().into_owned()], + _ => vec![], + } + } + + pub fn to_string_array(&self) -> Vec> { + match self { + Variable::Array(l) => l.iter().map(|i| i.to_string()).collect(), + v if !v.is_empty() => vec![v.to_string()], + _ => vec![], + } + } + + pub fn into_owned(self) -> Variable<'static> { + match self { + Variable::String(s) => Variable::String(Cow::Owned(s.into_owned())), + Variable::Integer(n) => Variable::Integer(n), + Variable::Float(n) => Variable::Float(n), + Variable::Array(l) => Variable::Array(Array::Owned( + l.iter().cloned().map(Variable::into_owned).collect(), + )), + } + } + + #[inline(always)] + pub fn is_borrowed(&self) -> bool { + match self { + Variable::String(s) => matches!(s, Cow::Borrowed(_)), + Variable::Array(l) => l.is_borrowed(), + _ => true, + } + } + + pub fn to_bool(&self) -> bool { + match self { + Variable::Float(f) => *f != 0.0, + Variable::Integer(n) => *n != 0, + Variable::String(s) => !s.is_empty(), + Variable::Array(a) => !a.is_empty(), + } + } +} + +pub(crate) fn array_to_string(items: &[Variable<'_>]) -> String { + let mut result = String::with_capacity(items.len() * 10); + for item in items { + if !result.is_empty() { + result.push_str("\r\n"); + } + match item { + Variable::String(v) => result.push_str(v), + Variable::Integer(v) => result.push_str(&v.to_string()), + Variable::Float(v) => result.push_str(&v.to_string()), + Variable::Array(_) => {} + } + } + result +} + +impl From for Variable<'_> { + fn from(s: String) -> Self { + Variable::String(Cow::Owned(s)) + } +} + +impl<'x> From<&'x String> for Variable<'x> { + fn from(s: &'x String) -> Self { + Variable::String(Cow::Borrowed(s.as_str())) + } +} + +impl<'x> From<&'x str> for Variable<'x> { + fn from(s: &'x str) -> Self { + Variable::String(Cow::Borrowed(s)) + } +} + +impl<'x> From> for Variable<'x> { + fn from(s: Cow<'x, str>) -> Self { + Variable::String(s) + } +} + +impl<'x> From>> for Variable<'x> { + fn from(l: Vec>) -> Self { + Variable::Array(Array::Owned(l)) + } +} + +impl<'x> From<&'x [Variable<'x>]> for Variable<'x> { + fn from(l: &'x [Variable<'x>]) -> Self { + Variable::Array(Array::Borrowed(l)) + } +} + +impl From for Variable<'_> { + fn from(n: Number) -> Self { + match n { + Number::Integer(n) => Variable::Integer(n), + Number::Float(n) => Variable::Float(n), + } + } +} + +macro_rules! integer_from { + ($($ty:ty),*) => { + $( + impl From<$ty> for Variable<'_> { + fn from(n: $ty) -> Self { + Variable::Integer(n as i64) + } + } + )* + }; +} + +integer_from!(usize, i64, u64, i32, u32); + +impl From for Variable<'_> { + fn from(n: f64) -> Self { + Variable::Float(n) + } +} + +impl From for Variable<'_> { + fn from(b: bool) -> Self { + Variable::Integer(i64::from(b)) + } +} + +impl Hash for Variable<'_> { + fn hash(&self, state: &mut H) { + match self { + Variable::String(s) => s.hash(state), + Variable::Integer(n) => n.hash(state), + Variable::Float(n) => n.to_bits().hash(state), + Variable::Array(l) => l.hash(state), + } + } +} + +impl PartialEq for Variable<'_> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Integer(a), Self::Integer(b)) => a == b, + (Self::Float(a), Self::Float(b)) => a == b, + (Self::Integer(a), Self::Float(b)) | (Self::Float(b), Self::Integer(a)) => { + *a as f64 == *b + } + (Self::String(a), Self::String(b)) => a == b, + (Self::String(_), Self::Integer(_) | Self::Float(_)) => &self.parse_number() == other, + (Self::Integer(_) | Self::Float(_), Self::String(_)) => self == &other.parse_number(), + (Self::Array(a), Self::Array(b)) => a == b, + _ => false, + } + } +} + +impl Eq for Variable<'_> {} + +#[allow(clippy::non_canonical_partial_ord_impl)] +impl PartialOrd for Variable<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Integer(a), Self::Integer(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), + (Self::Integer(a), Self::Float(b)) => (*a as f64).partial_cmp(b), + (Self::Float(a), Self::Integer(b)) => a.partial_cmp(&(*b as f64)), + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::String(_), Self::Integer(_) | Self::Float(_)) => { + self.parse_number().partial_cmp(other) + } + (Self::Integer(_) | Self::Float(_), Self::String(_)) => { + self.partial_cmp(&other.parse_number()) + } + (Self::Array(a), Self::Array(b)) => a.partial_cmp(b), + (Self::Array(_) | Self::String(_), _) => Ordering::Greater.into(), + (_, Self::Array(_)) => Ordering::Less.into(), + } + } +} + +impl Ord for Variable<'_> { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap_or(Ordering::Greater) + } +} + +impl Display for Variable<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Variable::String(v) => v.fmt(f), + Variable::Integer(v) => v.fmt(f), + Variable::Float(v) => v.fmt(f), + Variable::Array(v) => { + for (i, v) in v.iter().enumerate() { + if i > 0 { + f.write_str("\n")?; + } + v.fmt(f)?; + } + Ok(()) + } + } + } +} + +#[cfg(any(all(test, stalwart_tests), feature = "serde"))] +impl serde::Serialize for Array<'_> { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.as_slice()) + } +} + +#[cfg(any(all(test, stalwart_tests), feature = "serde"))] +impl<'de, 'x> serde::Deserialize<'de> for Array<'x> { + fn deserialize>(deserializer: D) -> Result { + Vec::>::deserialize(deserializer).map(Array::Owned) + } +} diff --git a/crates/sieve/src/runtime/variables.rs b/crates/sieve/src/runtime/variables.rs new file mode 100644 index 00000000..088e74f8 --- /dev/null +++ b/crates/sieve/src/runtime/variables.rs @@ -0,0 +1,130 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{RuntimeError, Variable, handler::Action}; +use crate::{ + Context, Envelope, Sieve, + bytecode::rec::{Rec, tag}, +}; +use std::borrow::Cow; + +impl<'x> Context<'x> { + pub(crate) fn set_variable( + &mut self, + script: &'x Sieve<'x>, + name: Rec, + mut variable: Variable<'x>, + ) -> Result<(), RuntimeError> { + if variable.len() > self.runtime.max_variable_size { + let mut new_variable = String::with_capacity(self.runtime.max_variable_size); + for ch in variable.to_string().chars() { + if ch.len_utf8() + new_variable.len() <= self.runtime.max_variable_size { + new_variable.push(ch); + } else { + break; + } + } + variable = new_variable.into(); + } + self.assign_variable(script, name, variable) + } + + pub(crate) fn assign_variable( + &mut self, + script: &'x Sieve<'x>, + name: Rec, + variable: Variable<'x>, + ) -> Result<(), RuntimeError> { + let variable = self.intern(variable); + + match name.tag { + tag::VAR_LOCAL => { + let index = self.local_base() + name.c as usize; + if let Some(var) = self.vars_local.get_mut(index) { + *var = variable; + } else { + debug_assert!(false, "Non-existent local variable {}", name.c); + } + } + tag::VAR_GLOBAL => { + let var_name = script.str(name.str())?; + if let Some(value) = self.vars_global.get_mut(var_name) { + *value = variable; + } else { + self.vars_global.insert(Cow::Borrowed(var_name), variable); + } + } + tag::VAR_ENVELOPE => { + let value = self.intern_cow(variable.into_string()); + self.set_envelope_variable(Envelope::from_code(name.b), value); + } + _ => (), + } + Ok(()) + } + + pub(crate) fn set_envelope_variable(&mut self, envelope: Envelope, value: &'x str) { + let mut did_find = false; + for (name, val) in self.envelope.iter_mut() { + if *name == envelope { + *val = Variable::borrowed(value); + did_find = true; + break; + } + } + if !did_find { + self.envelope.push((envelope, Variable::borrowed(value))); + } + self.actions.push(Action::SetEnvelope { envelope, value }); + } + + pub(crate) fn get_variable( + &self, + script: &'x Sieve<'x>, + name: Rec, + ) -> Result>, RuntimeError> { + Ok(match name.tag { + tag::VAR_LOCAL => self + .vars_local + .get(self.local_base() + name.c as usize) + .cloned(), + tag::VAR_MATCH => self + .vars_match + .get(self.match_base() + name.b as usize) + .cloned(), + tag::VAR_GLOBAL => self.vars_global.get(script.str(name.str())?).cloned(), + tag::NONE | tag::VARIABLE_NONE => None, + _ => Some(self.eval_value(script, name)?), + }) + } + + pub(crate) fn set_match_variables(&mut self, set_vars: Vec<(usize, String)>) { + let base = self.match_base(); + for (var_num, value) in set_vars { + let value = self.alloc_str(&value); + if let Some(var) = self.vars_match.get_mut(base + var_num) { + *var = Variable::borrowed(value); + } else { + debug_assert!(false, "Invalid match variable {var_num}"); + } + } + } + + pub(crate) fn clear_match_variables(&mut self, mut positions: u64) { + let base = self.match_base(); + while positions != 0 { + let index = 63 - positions.leading_zeros(); + positions ^= 1 << index; + if let Some(match_var) = self.vars_match.get_mut(base + index as usize) { + if !match_var.is_empty() { + *match_var = Variable::default(); + } + } else { + debug_assert!(false, "Failed to clear match variable at index {index}."); + } + } + } +} diff --git a/crates/sieve/src/sieve.rs b/crates/sieve/src/sieve.rs new file mode 100644 index 00000000..0d825636 --- /dev/null +++ b/crates/sieve/src/sieve.rs @@ -0,0 +1,428 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + bytecode::{ + Corrupt, Decoded, FORMAT_VERSION, HEADER_LEN, REC_LEN, Sections, + header_id::{HEADER_OTHER, header_from_id}, + rec::{Range, Rec, Str}, + verify::verify, + }, + runtime::tests::glob::GlobView, +}; +use mail_parser::HeaderName; +use std::{ + borrow::Cow, + cell::UnsafeCell, + fmt::{Debug, Display, Formatter}, + sync::OnceLock, +}; + +pub struct Sieve<'a> { + code: Cow<'a, [u8]>, + records: Cow<'a, [u8]>, + blob: Cow<'a, str>, + header_names_raw: Cow<'a, [u8]>, + globs: Cow<'a, [u8]>, + header_names: Box<[HeaderName<'static>]>, + regexes: Box<[OnceLock>]>, + num_vars: u16, + num_match_vars: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LoadError { + Truncated, + UnsupportedVersion(u16), + Corrupted, +} + +impl From for LoadError { + fn from(_: Corrupt) -> Self { + LoadError::Corrupted + } +} + +impl Display for LoadError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + LoadError::Truncated => f.write_str("Truncated Sieve script"), + LoadError::UnsupportedVersion(version) => write!( + f, + "Sieve script was compiled with format version {version}, expected {FORMAT_VERSION}" + ), + LoadError::Corrupted => f.write_str("Corrupted Sieve script"), + } + } +} + +impl std::error::Error for LoadError {} + +pub(crate) struct RecIter<'s> { + bytes: &'s [u8], + pub(crate) index: u32, +} + +impl Iterator for RecIter<'_> { + type Item = Rec; + + #[inline(always)] + fn next(&mut self) -> Option { + let (chunk, rest) = self.bytes.split_first_chunk::()?; + self.bytes = rest; + self.index += 1; + Some(Rec::decode(chunk)) + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.bytes.len() / REC_LEN; + (len, Some(len)) + } +} + +impl ExactSizeIterator for RecIter<'_> {} + +impl<'a> Sieve<'a> { + pub fn from_bytes(bytes: &'a [u8]) -> Result, LoadError> { + let sections = Self::sections(bytes)?; + let blob = std::str::from_utf8(&bytes[sections.blob.0..sections.blob.1]) + .map_err(|_| LoadError::Corrupted)?; + let sieve = Self::build(bytes, sections, Cow::Borrowed(blob))?; + verify(&sieve)?; + Ok(sieve) + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_bytes_unchecked(bytes: &'a [u8]) -> Result, LoadError> { + let sections = Self::sections(bytes)?; + let blob = + unsafe { std::str::from_utf8_unchecked(&bytes[sections.blob.0..sections.blob.1]) }; + Self::build(bytes, sections, Cow::Borrowed(blob)) + } + + fn sections(bytes: &[u8]) -> Result { + Sections::parse(bytes) + } + + fn build( + bytes: &'a [u8], + sections: Sections, + blob: Cow<'a, str>, + ) -> Result, LoadError> { + let header_names_raw = &bytes[sections.header_names.0..sections.header_names.1]; + let header_names = parse_header_names(header_names_raw, &blob)?; + Ok(Sieve { + code: Cow::Borrowed(&bytes[sections.code.0..sections.code.1]), + records: Cow::Borrowed(&bytes[sections.records.0..sections.records.1]), + blob, + header_names_raw: Cow::Borrowed(header_names_raw), + globs: Cow::Borrowed(&bytes[sections.globs.0..sections.globs.1]), + header_names, + regexes: new_regex_cache(sections.num_regexes), + num_vars: sections.num_vars, + num_match_vars: sections.num_match_vars, + }) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn from_parts( + code: Vec, + records: Vec, + blob: String, + header_names_raw: Vec, + globs: Vec, + num_regexes: u32, + num_vars: u16, + num_match_vars: u16, + ) -> Result, LoadError> { + let header_names = parse_header_names(&header_names_raw, &blob)?; + Ok(Sieve { + code: Cow::Owned(code), + records: Cow::Owned(records), + blob: Cow::Owned(blob), + header_names_raw: Cow::Owned(header_names_raw), + globs: Cow::Owned(globs), + header_names, + regexes: new_regex_cache(num_regexes), + num_vars, + num_match_vars, + }) + } + + pub fn into_owned(self) -> Sieve<'static> { + Sieve { + code: Cow::Owned(self.code.into_owned()), + records: Cow::Owned(self.records.into_owned()), + blob: Cow::Owned(self.blob.into_owned()), + header_names_raw: Cow::Owned(self.header_names_raw.into_owned()), + globs: Cow::Owned(self.globs.into_owned()), + header_names: self.header_names, + regexes: self.regexes, + num_vars: self.num_vars, + num_match_vars: self.num_match_vars, + } + } + + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(self.serialized_len()); + let mut start = HEADER_LEN; + let mut section = |len: usize| { + let range = (start, start + len); + start += len; + range + }; + Sections { + num_vars: self.num_vars, + num_match_vars: self.num_match_vars, + code: section(self.code.len()), + records: section(self.records.len()), + blob: section(self.blob.len()), + header_names: section(self.header_names_raw.len()), + globs: section(self.globs.len()), + num_regexes: self.regexes.len() as u32, + } + .write_header(&mut out); + out.extend_from_slice(&self.code); + out.extend_from_slice(&self.records); + out.extend_from_slice(self.blob.as_bytes()); + out.extend_from_slice(&self.header_names_raw); + out.extend_from_slice(&self.globs); + out + } + + pub fn serialized_len(&self) -> usize { + HEADER_LEN + + self.code.len() + + self.records.len() + + self.blob.len() + + self.header_names_raw.len() + + self.globs.len() + } + + pub fn code_len(&self) -> usize { + self.code.len() + } + + pub fn constant_count(&self) -> usize { + self.blob.len() + } + + #[inline(always)] + pub(crate) fn code(&self) -> &[u8] { + &self.code + } + + #[inline(always)] + pub(crate) fn num_vars(&self) -> usize { + self.num_vars as usize + } + + #[inline(always)] + pub(crate) fn num_match_vars(&self) -> usize { + self.num_match_vars as usize + } + + #[inline(always)] + pub(crate) fn num_records(&self) -> u32 { + (self.records.len() / REC_LEN) as u32 + } + + #[inline(always)] + pub(crate) fn num_globs(&self) -> u32 { + self.globs + .first_chunk::<4>() + .map_or(0, |b| u32::from_le_bytes(*b)) + } + + #[inline(always)] + pub(crate) fn num_regexes(&self) -> u32 { + self.regexes.len() as u32 + } + + #[inline(always)] + pub(crate) fn rec(&self, index: u32) -> Decoded { + let start = index as usize * REC_LEN; + self.records + .get(start..) + .and_then(|s| s.first_chunk::()) + .map(Rec::decode) + .ok_or(Corrupt) + } + + #[inline(always)] + pub(crate) fn recs(&self, range: Range) -> Decoded> { + let start = range.start as usize * REC_LEN; + let len = range.len as usize * REC_LEN; + self.records + .get(start..start.checked_add(len).ok_or(Corrupt)?) + .map(|bytes| RecIter { + bytes, + index: range.start, + }) + .ok_or(Corrupt) + } + + #[inline(always)] + pub(crate) fn str(&self, s: Str) -> Decoded<&str> { + let start = s.off as usize; + self.blob + .get(start..start.checked_add(s.len as usize).ok_or(Corrupt)?) + .ok_or(Corrupt) + } + + #[inline(always)] + pub(crate) fn header_name(&self, index: u16) -> Decoded<&HeaderName<'static>> { + self.header_names.get(index as usize).ok_or(Corrupt) + } + + pub(crate) fn glob(&self, index: u16) -> Decoded> { + let count = self.num_globs(); + if index as u32 >= count { + return Err(Corrupt); + } + let at = 4 + index as usize * 4; + let offset = self + .globs + .get(at..at + 4) + .and_then(|b| b.try_into().ok()) + .map(u32::from_le_bytes) + .ok_or(Corrupt)? as usize; + GlobView::parse(self.globs.get(offset..).ok_or(Corrupt)?, self) + } + + pub(crate) fn regex(&self, slot: u16, pattern: &str) -> Option<&fancy_regex::Regex> { + self.regexes + .get(slot as usize)? + .get_or_init(|| crate::regex::compile(pattern)) + .as_ref() + } +} + +fn new_regex_cache(count: u32) -> Box<[OnceLock>]> { + (0..count).map(|_| OnceLock::new()).collect() +} + +fn parse_header_names(raw: &[u8], blob: &str) -> Result]>, LoadError> { + let Some((count, mut rest)) = raw.split_first_chunk::<4>() else { + return if raw.is_empty() { + Ok(Box::default()) + } else { + Err(LoadError::Corrupted) + }; + }; + let count = u32::from_le_bytes(*count) as usize; + if count > rest.len() { + return Err(LoadError::Corrupted); + } + let mut names = Vec::with_capacity(count); + for _ in 0..count { + let (&id, tail) = rest.split_first().ok_or(LoadError::Corrupted)?; + rest = tail; + if id != HEADER_OTHER { + names.push(header_from_id(id).ok_or(LoadError::Corrupted)?); + continue; + } + let (entry, tail) = rest.split_first_chunk::<8>().ok_or(LoadError::Corrupted)?; + rest = tail; + let off = u32::from_le_bytes([entry[0], entry[1], entry[2], entry[3]]) as usize; + let len = u32::from_le_bytes([entry[4], entry[5], entry[6], entry[7]]) as usize; + let name = blob + .get(off..off.checked_add(len).ok_or(LoadError::Corrupted)?) + .ok_or(LoadError::Corrupted)?; + let name = HeaderName::parse(name) + .map(HeaderName::into_owned) + .unwrap_or_else(|| HeaderName::Other(Cow::Owned(name.to_string()))); + names.push(name); + } + if rest.is_empty() { + Ok(names.into_boxed_slice()) + } else { + Err(LoadError::Corrupted) + } +} + +#[derive(Default)] +pub struct ScriptArena { + #[allow(clippy::vec_box)] + scripts: UnsafeCell>>>, +} + +impl ScriptArena { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&self, script: Sieve<'static>) -> &Sieve<'static> { + let boxed = Box::new(script); + let stable: *const Sieve<'static> = &*boxed; + unsafe { (*self.scripts.get()).push(boxed) }; + unsafe { &*stable } + } + + pub fn len(&self) -> usize { + unsafe { (*self.scripts.get()).len() } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Clone for Sieve<'_> { + fn clone(&self) -> Self { + Sieve { + code: self.code.clone(), + records: self.records.clone(), + blob: self.blob.clone(), + header_names_raw: self.header_names_raw.clone(), + globs: self.globs.clone(), + header_names: self.header_names.clone(), + regexes: self + .regexes + .iter() + .map(|slot| { + let cell = OnceLock::new(); + if let Some(value) = slot.get() { + let _ = cell.set(value.clone()); + } + cell + }) + .collect(), + num_vars: self.num_vars, + num_match_vars: self.num_match_vars, + } + } +} + +impl PartialEq for Sieve<'_> { + fn eq(&self, other: &Self) -> bool { + self.code == other.code + && self.records == other.records + && self.blob == other.blob + && self.header_names_raw == other.header_names_raw + && self.globs == other.globs + && self.regexes.len() == other.regexes.len() + && self.num_vars == other.num_vars + && self.num_match_vars == other.num_match_vars + } +} + +impl Eq for Sieve<'_> {} + +impl Debug for Sieve<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Sieve") + .field("code_len", &self.code.len()) + .field("records", &self.num_records()) + .field("blob_len", &self.blob.len()) + .field("header_names", &self.header_names) + .field("globs", &self.num_globs()) + .field("regexes", &self.regexes.len()) + .field("num_vars", &self.num_vars) + .field("num_match_vars", &self.num_match_vars) + .finish() + } +} diff --git a/crates/smtp-proto/Cargo.toml b/crates/smtp-proto/Cargo.toml new file mode 100644 index 00000000..94edfc1c --- /dev/null +++ b/crates/smtp-proto/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "smtp-proto" +description = "SMTP protocol parser" +version = "0.2.5" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[dependencies] +memchr = "2" +rkyv = { version = "0.8", optional = true } +serde = { version = "1.0", features = ["derive"], optional = true } + +[features] +default = [] +rkyv = ["dep:rkyv"] +serde = ["dep:serde"] + +[lints] +workspace = true diff --git a/crates/smtp-proto/src/lib.rs b/crates/smtp-proto/src/lib.rs new file mode 100644 index 00000000..a8e5b925 --- /dev/null +++ b/crates/smtp-proto/src/lib.rs @@ -0,0 +1,464 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#![deny(rust_2018_idioms)] + +use std::borrow::Cow; +use std::fmt::Display; + +pub mod request; +pub mod response; +mod tokens; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub enum Request { + Ehlo { host: T }, + Lhlo { host: T }, + Helo { host: T }, + Mail { from: MailFrom }, + Rcpt { to: RcptTo }, + Bdat { chunk_size: usize, is_last: bool }, + Auth { mechanism: u64, initial_response: T }, + Noop { value: T }, + Vrfy { value: T }, + Expn { value: T }, + Help { value: T }, + Etrn { name: T }, + Atrn { domains: Vec }, + Burl { uri: T, is_last: bool }, + StartTls, + Data, + Rset, + Quit, +} + +impl Request { + /// Returns the SMTP command name for this request variant. + pub fn name(&self) -> &'static str { + match self { + Request::Ehlo { .. } => "EHLO", + Request::Helo { .. } => "HELO", + Request::Lhlo { .. } => "LHLO", + Request::Mail { .. } => "MAIL", + Request::Rcpt { .. } => "RCPT", + Request::Data => "DATA", + Request::Bdat { .. } => "BDAT", + Request::Auth { .. } => "AUTH", + Request::Noop { .. } => "NOOP", + Request::Vrfy { .. } => "VRFY", + Request::Expn { .. } => "EXPN", + Request::Help { .. } => "HELP", + Request::Etrn { .. } => "ETRN", + Request::Atrn { .. } => "ATRN", + Request::Burl { .. } => "BURL", + Request::StartTls => "STARTTLS", + Request::Rset => "RSET", + Request::Quit => "QUIT", + } + } +} + +impl Request> { + pub fn into_owned(self) -> Request { + match self { + Request::Ehlo { host } => Request::Ehlo { + host: host.into_owned(), + }, + Request::Lhlo { host } => Request::Lhlo { + host: host.into_owned(), + }, + Request::Helo { host } => Request::Helo { + host: host.into_owned(), + }, + Request::Mail { from } => Request::Mail { + from: from.into_owned(), + }, + Request::Rcpt { to } => Request::Rcpt { + to: to.into_owned(), + }, + Request::Bdat { + chunk_size, + is_last, + } => Request::Bdat { + chunk_size, + is_last, + }, + Request::Auth { + mechanism, + initial_response, + } => Request::Auth { + mechanism, + initial_response: initial_response.into_owned(), + }, + Request::Noop { value } => Request::Noop { + value: value.into_owned(), + }, + Request::Vrfy { value } => Request::Vrfy { + value: value.into_owned(), + }, + Request::Expn { value } => Request::Expn { + value: value.into_owned(), + }, + Request::Help { value } => Request::Help { + value: value.into_owned(), + }, + Request::Etrn { name } => Request::Etrn { + name: name.into_owned(), + }, + Request::Atrn { domains } => Request::Atrn { + domains: domains.into_iter().map(Cow::into_owned).collect(), + }, + Request::Burl { uri, is_last } => Request::Burl { + uri: uri.into_owned(), + is_last, + }, + Request::StartTls => Request::StartTls, + Request::Data => Request::Data, + Request::Rset => Request::Rset, + Request::Quit => Request::Quit, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub struct MailFrom { + pub address: T, + pub flags: u64, + pub size: usize, + pub trans_id: Option, + pub by: i64, + pub env_id: Option, + pub solicit: Option, + pub mtrk: Option>, + pub auth: Option, + pub hold_for: u64, + pub hold_until: u64, + pub mt_priority: i64, +} + +impl MailFrom> { + pub fn into_owned(self) -> MailFrom { + MailFrom { + address: self.address.into_owned(), + flags: self.flags, + size: self.size, + trans_id: self.trans_id.map(Cow::into_owned), + by: self.by, + env_id: self.env_id.map(Cow::into_owned), + solicit: self.solicit.map(Cow::into_owned), + mtrk: self.mtrk.map(Mtrk::into_owned), + auth: self.auth.map(Cow::into_owned), + hold_for: self.hold_for, + hold_until: self.hold_until, + mt_priority: self.mt_priority, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub struct RcptTo { + pub address: T, + pub orcpt: Option, + pub rrvs: i64, + pub flags: u64, +} + +impl RcptTo> { + pub fn into_owned(self) -> RcptTo { + RcptTo { + address: self.address.into_owned(), + orcpt: self.orcpt.map(Cow::into_owned), + rrvs: self.rrvs, + flags: self.flags, + } + } +} + +pub const MAIL_BODY_7BIT: u64 = 1 << 0; +pub const MAIL_BODY_8BITMIME: u64 = 1 << 1; +pub const MAIL_BODY_BINARYMIME: u64 = 1 << 2; +pub const MAIL_RET_FULL: u64 = 1 << 3; +pub const MAIL_RET_HDRS: u64 = 1 << 4; +pub const MAIL_SMTPUTF8: u64 = 1 << 5; +pub const MAIL_REQUIRETLS: u64 = 1 << 6; +pub const MAIL_CONPERM: u64 = 1 << 7; +pub const MAIL_BY_NOTIFY: u64 = 1 << 8; +pub const MAIL_BY_RETURN: u64 = 1 << 9; +pub const MAIL_BY_TRACE: u64 = 1 << 10; + +pub const RCPT_NOTIFY_SUCCESS: u64 = 1 << 0; +pub const RCPT_NOTIFY_FAILURE: u64 = 1 << 1; +pub const RCPT_NOTIFY_DELAY: u64 = 1 << 2; +pub const RCPT_NOTIFY_NEVER: u64 = 1 << 3; +pub const RCPT_CONNEG: u64 = 1 << 4; +pub const RCPT_RRVS_REJECT: u64 = 1 << 5; +pub const RCPT_RRVS_CONTINUE: u64 = 1 << 6; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub struct Mtrk { + pub certifier: T, + pub timeout: u64, +} + +impl Mtrk> { + pub fn into_owned(self) -> Mtrk { + Mtrk { + certifier: self.certifier.into_owned(), + timeout: self.timeout, + } + } +} + +pub const AUTH_SCRAM_SHA_256_PLUS: u64 = 1u64 << 0; +pub const AUTH_SCRAM_SHA_256: u64 = 1u64 << 1; +pub const AUTH_SCRAM_SHA_1_PLUS: u64 = 1u64 << 2; +pub const AUTH_SCRAM_SHA_1: u64 = 1u64 << 3; +pub const AUTH_OAUTHBEARER: u64 = 1u64 << 4; +pub const AUTH_XOAUTH: u64 = 1u64 << 5; +pub const AUTH_XOAUTH2: u64 = 1u64 << 6; +pub const AUTH_9798_M_DSA_SHA1: u64 = 1u64 << 7; +pub const AUTH_9798_M_ECDSA_SHA1: u64 = 1u64 << 8; +pub const AUTH_9798_M_RSA_SHA1_ENC: u64 = 1u64 << 9; +pub const AUTH_9798_U_DSA_SHA1: u64 = 1u64 << 10; +pub const AUTH_9798_U_ECDSA_SHA1: u64 = 1u64 << 11; +pub const AUTH_9798_U_RSA_SHA1_ENC: u64 = 1u64 << 12; +pub const AUTH_EAP_AES128: u64 = 1u64 << 13; +pub const AUTH_EAP_AES128_PLUS: u64 = 1u64 << 14; +pub const AUTH_ECDH_X25519_CHALLENGE: u64 = 1u64 << 15; +pub const AUTH_ECDSA_NIST256P_CHALLENGE: u64 = 1u64 << 16; +pub const AUTH_EXTERNAL: u64 = 1u64 << 17; +pub const AUTH_GS2_KRB5: u64 = 1u64 << 18; +pub const AUTH_GS2_KRB5_PLUS: u64 = 1u64 << 19; +pub const AUTH_GSS_SPNEGO: u64 = 1u64 << 20; +pub const AUTH_GSSAPI: u64 = 1u64 << 21; +pub const AUTH_KERBEROS_V4: u64 = 1u64 << 22; +pub const AUTH_KERBEROS_V5: u64 = 1u64 << 23; +pub const AUTH_NMAS_SAMBA_AUTH: u64 = 1u64 << 24; +pub const AUTH_NMAS_AUTHEN: u64 = 1u64 << 25; +pub const AUTH_NMAS_LOGIN: u64 = 1u64 << 26; +pub const AUTH_NTLM: u64 = 1u64 << 27; +pub const AUTH_OAUTH10A: u64 = 1u64 << 28; +pub const AUTH_OPENID20: u64 = 1u64 << 29; +pub const AUTH_OTP: u64 = 1u64 << 30; +pub const AUTH_SAML20: u64 = 1u64 << 31; +pub const AUTH_SECURID: u64 = 1u64 << 32; +pub const AUTH_SKEY: u64 = 1u64 << 33; +pub const AUTH_SPNEGO: u64 = 1u64 << 34; +pub const AUTH_SPNEGO_PLUS: u64 = 1u64 << 35; +pub const AUTH_SXOVER_PLUS: u64 = 1u64 << 36; +pub const AUTH_CRAM_MD5: u64 = 1u64 << 37; +pub const AUTH_DIGEST_MD5: u64 = 1u64 << 38; +pub const AUTH_LOGIN: u64 = 1u64 << 39; +pub const AUTH_PLAIN: u64 = 1u64 << 40; +pub const AUTH_ANONYMOUS: u64 = 1u64 << 41; + +pub const EXT_8BIT_MIME: u32 = 1 << 0; +pub const EXT_ATRN: u32 = 1 << 1; +pub const EXT_AUTH: u32 = 1 << 2; +pub const EXT_BINARY_MIME: u32 = 1 << 3; +pub const EXT_BURL: u32 = 1 << 4; +pub const EXT_CHECKPOINT: u32 = 1 << 5; +pub const EXT_CHUNKING: u32 = 1 << 6; +pub const EXT_CONNEG: u32 = 1 << 7; +pub const EXT_CONPERM: u32 = 1 << 8; +pub const EXT_DELIVER_BY: u32 = 1 << 9; +pub const EXT_DSN: u32 = 1 << 10; +pub const EXT_ENHANCED_STATUS_CODES: u32 = 1 << 11; +pub const EXT_ETRN: u32 = 1 << 12; +pub const EXT_FUTURE_RELEASE: u32 = 1 << 13; +pub const EXT_HELP: u32 = 1 << 14; +pub const EXT_MT_PRIORITY: u32 = 1 << 15; +pub const EXT_MTRK: u32 = 1 << 16; +pub const EXT_NO_SOLICITING: u32 = 1 << 17; +pub const EXT_ONEX: u32 = 1 << 18; +pub const EXT_PIPELINING: u32 = 1 << 19; +pub const EXT_REQUIRE_TLS: u32 = 1 << 20; +pub const EXT_RRVS: u32 = 1 << 21; +pub const EXT_SIZE: u32 = 1 << 22; +pub const EXT_SMTP_UTF8: u32 = 1 << 23; +pub const EXT_START_TLS: u32 = 1 << 24; +pub const EXT_VERB: u32 = 1 << 25; +pub const EXT_EXPN: u32 = 1 << 26; +pub const EXT_VRFY: u32 = 1 << 27; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub enum MtPriority { + #[default] + Mixer, + Stanag4406, + Nsep, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub struct EhloResponse { + pub hostname: T, + pub capabilities: u32, + + pub auth_mechanisms: u64, + pub deliver_by: u64, + pub future_release_interval: u64, + pub future_release_datetime: u64, + pub mt_priority: MtPriority, + pub no_soliciting: Option, + pub size: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub struct Response { + pub code: u16, + pub esc: [u8; 3], + pub message: T, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub enum Severity { + PositiveCompletion = 2, + PositiveIntermediate = 3, + TransientNegativeCompletion = 4, + PermanentNegativeCompletion = 5, + Invalid = 0, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +pub enum Category { + Syntax = 0, + Information = 1, + Connections = 2, + Unspecified3 = 3, + Unspecified4 = 4, + MailSystem = 5, + Invalid = 6, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + NeedsMoreData { bytes_left: usize }, + UnknownCommand, + InvalidSenderAddress, + InvalidRecipientAddress, + SyntaxError { syntax: &'static str }, + InvalidParameter { param: &'static str }, + UnsupportedParameter { param: String }, + ResponseTooLong, + InvalidResponse { code: u16 }, +} + +pub(crate) const LF: u8 = b'\n'; +pub(crate) const SP: u8 = b' '; + +pub trait IntoString: Sized { + fn into_string(self) -> String; +} + +impl IntoString for Vec { + fn into_string(self) -> String { + String::from_utf8(self) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) + } +} + +impl Default for MailFrom { + fn default() -> Self { + Self { + address: Default::default(), + flags: Default::default(), + size: Default::default(), + trans_id: Default::default(), + by: Default::default(), + env_id: Default::default(), + solicit: Default::default(), + mtrk: Default::default(), + auth: Default::default(), + hold_for: Default::default(), + hold_until: Default::default(), + mt_priority: Default::default(), + } + } +} + +impl Default for RcptTo { + fn default() -> Self { + Self { + address: Default::default(), + orcpt: Default::default(), + rrvs: Default::default(), + flags: Default::default(), + } + } +} + +impl AsRef> for EhloResponse { + fn as_ref(&self) -> &EhloResponse { + self + } +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NeedsMoreData { bytes_left } => { + write!(f, "Needs more data: {} bytes left", bytes_left) + } + Error::UnknownCommand => write!(f, "Unknown command"), + Error::InvalidSenderAddress => write!(f, "Invalid sender address"), + Error::InvalidRecipientAddress => write!(f, "Invalid recipient address"), + Error::SyntaxError { syntax } => write!(f, "Syntax error: {}", syntax), + Error::InvalidParameter { param } => write!(f, "Invalid parameter: {}", param), + Error::UnsupportedParameter { param } => { + write!(f, "Unsupported parameter: {}", param) + } + Error::ResponseTooLong => write!(f, "Response too long"), + Error::InvalidResponse { code } => write!(f, "Invalid response: {}", code), + } + } +} + +impl std::error::Error for Error {} diff --git a/crates/smtp-proto/src/request/mod.rs b/crates/smtp-proto/src/request/mod.rs new file mode 100644 index 00000000..93f5e48e --- /dev/null +++ b/crates/smtp-proto/src/request/mod.rs @@ -0,0 +1,121 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::tokens::{define_tokens_64, define_tokens_128}; + +pub mod parser; +pub mod receiver; +mod scan; + +pub const AUTH: u64 = crate::tokens::token64("AUTH"); // special, because it is `pub` instead of `pub(crate)` + +define_tokens_64! { + // SMTP commands + EHLO, + HELO, + LHLO, + MAIL, + RCPT, + DATA, + BDAT, + RSET, + VRFY, + EXPN, + HELP, + NOOP, + QUIT, + ETRN, + ATRN, + BURL, + STARTTLS, + + // Arguments + FROM, + TO, + LAST, + + // Parameters + N, + NT, + C, + R, + RT, + FULL, + HDRS, + RFC822, +} + +define_tokens_128! { + // Parameters + BODY, + SEVENBIT = "7BIT", + EIGHBITMIME = "8BITMIME", + BINARYMIME, + SIZE, + TRANSID, + BY, + NOTIFY, + ORCPT, + RET, + ENVID, + NEVER, + SUCCESS, + FAILURE, + DELAY, + SOLICIT, + MTRK, + AUTH_ = "AUTH", + HOLDFOR, + HOLDUNTIL, + SMTPUTF8, + CONPERM, + CONNEG, + MT_PRIORITY = "MT-PRIORITY", + RRVS, + REQUIRETLS, + _9798_M_DSA_SHA1 = "9798-M-DSA-SHA1", + _9798_M_ECDSA_SHA = "9798-M-ECDSA-SHA", + _9798_M_RSA_SHA1_ = "9798-M-RSA-SHA1-", + _9798_U_DSA_SHA1 = "9798-U-DSA-SHA1", + _9798_U_ECDSA_SHA = "9798-U-ECDSA-SHA", + _9798_U_RSA_SHA1_ = "9798-U-RSA-SHA1-", + ANONYMOUS, + CRAM_MD5 = "CRAM-MD5", + DIGEST_MD5 = "DIGEST-MD5", + EAP_AES128 = "EAP-AES128", + EAP_AES128_PLUS = "EAP-AES128-PLUS", + ECDH_X25519_CHAL = "ECDH-X25519-CHAL", + ECDSA_NIST256P_C = "ECDSA-NIST256P-C", + EXTERNAL, + GS2_KRB5 = "GS2-KRB5", + GS2_KRB5_PLUS = "GS2-KRB5-PLUS", + GSS_SPNEGO = "GSS-SPNEGO", + GSSAPI, + KERBEROS_V4 = "KERBEROS-V4", + KERBEROS_V5 = "KERBEROS-V5", + LOGIN, + NMAS_SAMBA_AUTH = "NMAS-SAMBA-AUTH", + NMAS_AUTHEN = "NMAS-AUTHEN", + NMAS_LOGIN = "NMAS-LOGIN", + NTLM, + OAUTH10A, + OAUTHBEARER, + OPENID20, + OTP, + PLAIN, + SAML20, + SCRAM_SHA_1 = "SCRAM-SHA-1", + SCRAM_SHA_1_PLUS = "SCRAM-SHA-1-PLUS", + SCRAM_SHA_256 = "SCRAM-SHA-256", + SCRAM_SHA_256_PL = "SCRAM-SHA-256-PL", + SECURID, + SKEY, + SPNEGO, + SPNEGO_PLUS = "SPNEGO-PLUS", + SXOVER_PLUS = "SXOVER-PLUS", + XOAUTH, + XOAUTH2, +} diff --git a/crates/smtp-proto/src/request/parser.rs b/crates/smtp-proto/src/request/parser.rs new file mode 100644 index 00000000..cdd40f1c --- /dev/null +++ b/crates/smtp-proto/src/request/parser.rs @@ -0,0 +1,1435 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; +use std::char; +use std::slice::Iter; +use std::str::from_utf8; + +use crate::*; + +use super::scan::{ + MEMCHR_MIN_LEN, TOKEN_UPPER, find_any3, find_any4, find_lf, find_non_atext, preceding_byte, + split_at_index, +}; +use super::*; + +const MAX_ADDRESS_LEN: usize = 256; +const MAX_DOMAIN_LEN: usize = 255; +const MAX_FUTURE_RELEASE_INTERVAL: usize = 999_999_999; +const MAX_EXACT_SIZE_DIGITS: u32 = usize::MAX.ilog10(); +const MAX_EXACT_INTEGER_DIGITS: u32 = i64::MAX.ilog10(); + +impl<'a> Request> { + pub fn parse(bytes: &mut Iter<'a, u8>) -> Result>, Error> { + let mut parser = Rfc5321Parser::new(bytes); + let command = parser.hashed_value()?; + if !parser.stop_char.is_ascii_whitespace() { + parser.seek_lf()?; + return Err(Error::UnknownCommand); + } + + match command { + RCPT => { + if !(parser.stop_char == LF + || parser.hashed_value()? != TO + || parser.stop_char != b':' && parser.next_char()? != b':') + && parser.next_char()? == b'<' + { + if let Some(to) = parser.address()? { + if parser.stop_char == b'>' { + return Ok(Request::Rcpt { + to: parser.rcpt_to_parameters(to)?, + }); + } + } else { + parser.seek_lf()?; + return Err(Error::InvalidRecipientAddress); + } + } + parser.seek_lf()?; + Err(Error::SyntaxError { + syntax: "RCPT TO: [parameters]", + }) + } + MAIL => { + if !(parser.stop_char == LF + || parser.hashed_value()? != FROM + || parser.stop_char != b':' && parser.next_char()? != b':') + && parser.next_char()? == b'<' + { + if let Some(from) = parser.address()? { + if parser.stop_char == b'>' { + return Ok(Request::Mail { + from: parser.mail_from_parameters(from)?, + }); + } + } else { + parser.seek_lf()?; + return Err(Error::InvalidSenderAddress); + } + } + + parser.seek_lf()?; + Err(Error::SyntaxError { + syntax: "MAIL FROM: [parameters]", + }) + } + DATA => { + parser.seek_lf()?; + Ok(Request::Data) + } + EHLO => { + if parser.stop_char != LF { + let host = parser.text()?; + parser.seek_lf()?; + if (1..=MAX_DOMAIN_LEN).contains(&host.len()) { + return Ok(Request::Ehlo { host }); + } + } + Err(Error::SyntaxError { + syntax: "EHLO domain", + }) + } + BDAT => { + if parser.stop_char != LF { + let chunk_size = parser.size()?; + if chunk_size != usize::MAX && parser.stop_char.is_ascii_whitespace() { + if parser.stop_char != LF { + match parser.hashed_value()? { + LAST => { + parser.seek_lf()?; + return Ok(Request::Bdat { + chunk_size, + is_last: true, + }); + } + 0 => { + parser.seek_lf()?; + return Ok(Request::Bdat { + chunk_size, + is_last: false, + }); + } + _ => (), + } + } else { + return Ok(Request::Bdat { + chunk_size, + is_last: false, + }); + } + } + } + parser.seek_lf()?; + Err(Error::SyntaxError { + syntax: "BDAT chunk-size [LAST]", + }) + } + AUTH => { + if parser.stop_char != LF + && let Some(mechanism) = parser.mechanism()? + { + let initial_response = if parser.stop_char != LF { + while let Some(rest) = parser.bytes.as_slice().strip_prefix(b" ") { + *parser.bytes = rest.iter(); + } + parser.text()? + } else { + Cow::Borrowed("") + }; + parser.seek_lf()?; + return Ok(Request::Auth { + mechanism, + initial_response, + }); + } + Err(Error::SyntaxError { + syntax: "AUTH mechanism [initial-response]", + }) + } + EXPN => { + if parser.stop_char != LF { + let value = parser.string()?; + parser.seek_lf()?; + if (1..=MAX_ADDRESS_LEN).contains(&value.len()) { + return Ok(Request::Expn { value }); + } + } + Err(Error::SyntaxError { + syntax: "EXPN string", + }) + } + NOOP => { + if parser.stop_char != LF { + let value = parser.string()?; + parser.seek_lf()?; + Ok(Request::Noop { value }) + } else { + Ok(Request::Noop { + value: Cow::Borrowed(""), + }) + } + } + QUIT => { + parser.seek_lf()?; + Ok(Request::Quit) + } + LHLO => { + if parser.stop_char != LF { + let host = parser.text()?; + parser.seek_lf()?; + if (1..=MAX_DOMAIN_LEN).contains(&host.len()) { + return Ok(Request::Lhlo { host }); + } + } + Err(Error::SyntaxError { + syntax: "LHLO domain", + }) + } + RSET => { + parser.seek_lf()?; + Ok(Request::Rset) + } + VRFY => { + if parser.stop_char != LF { + let value = parser.string()?; + parser.seek_lf()?; + if (1..=MAX_ADDRESS_LEN).contains(&value.len()) { + return Ok(Request::Vrfy { value }); + } + } + Err(Error::SyntaxError { + syntax: "VRFY string", + }) + } + HELP => { + if parser.stop_char != LF { + let value = parser.string()?; + parser.seek_lf()?; + Ok(Request::Help { value }) + } else { + Ok(Request::Help { + value: Cow::Borrowed(""), + }) + } + } + STARTTLS => { + parser.seek_lf()?; + Ok(Request::StartTls) + } + ETRN => { + if parser.stop_char != LF { + let name = parser.string()?; + parser.seek_lf()?; + if !name.is_empty() { + return Ok(Request::Etrn { name }); + } + } + Err(Error::SyntaxError { + syntax: "ETRN name", + }) + } + ATRN => { + if parser.stop_char != LF { + let mut domains = Vec::new(); + loop { + let domain = parser.seek_char(b',')?; + if !domain.is_empty() { + domains.push(domain); + } + if parser.stop_char != b',' { + parser.seek_lf()?; + if !domains.is_empty() { + return Ok(Request::Atrn { domains }); + } else { + break; + } + } + } + } + Err(Error::SyntaxError { + syntax: "ATRN domain[,domain]", + }) + } + BURL => { + if parser.stop_char != LF { + let uri = parser.text()?; + if !uri.is_empty() && parser.stop_char.is_ascii_whitespace() { + if parser.stop_char != LF { + match parser.hashed_value()? { + LAST => { + parser.seek_lf()?; + return Ok(Request::Burl { uri, is_last: true }); + } + 0 => { + parser.seek_lf()?; + return Ok(Request::Burl { + uri, + is_last: false, + }); + } + _ => (), + } + } else { + return Ok(Request::Burl { + uri, + is_last: false, + }); + } + } + } + parser.seek_lf()?; + Err(Error::SyntaxError { + syntax: "BURL absolute-uri [LAST]", + }) + } + HELO => { + if parser.stop_char != LF { + let host = parser.text()?; + parser.seek_lf()?; + if (1..=MAX_DOMAIN_LEN).contains(&host.len()) { + return Ok(Request::Helo { host }); + } + } + Err(Error::SyntaxError { + syntax: "HELO domain", + }) + } + _ => { + parser.seek_lf()?; + Err(Error::UnknownCommand) + } + } + } +} + +pub struct Rfc5321Parser<'x, 'y> { + bytes: &'x mut Iter<'y, u8>, + pub stop_char: u8, + pub bytes_left: usize, +} + +impl<'x, 'y> Rfc5321Parser<'x, 'y> { + pub fn new(bytes: &'x mut Iter<'y, u8>) -> Self { + let (bytes_left, _) = bytes.size_hint(); + Rfc5321Parser { + bytes, + bytes_left, + stop_char: 0, + } + } + + #[allow(clippy::while_let_on_iterator)] + pub fn hashed_value(&mut self) -> Result { + let mut value: u64 = 0; + let mut shift = 0; + + while let Some(&ch) = self.bytes.next() { + let token = TOKEN_UPPER[ch as usize]; + if token != 0 && shift < 64 { + value |= (token as u64) << shift; + shift += 8; + } else if ch != b'\r' && (ch != b' ' || value != 0) { + self.stop_char = ch; + return Ok(value); + } + } + + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[allow(clippy::while_let_on_iterator)] + pub fn hashed_value_long(&mut self) -> Result { + let mut low: u64 = 0; + let mut high: u64 = 0; + let mut shift = 0; + + while let Some(&ch) = self.bytes.next() { + let token = TOKEN_UPPER[ch as usize]; + if token != 0 && shift < 128 { + let bits = (token as u64) << (shift & 63); + if shift < 64 { + low |= bits; + } else { + high |= bits; + } + shift += 8; + } else if ch != b'\r' && (ch != b' ' || (low | high) != 0) { + self.stop_char = ch; + return Ok(low as u128 | (high as u128) << 64); + } + } + + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + pub fn address(&mut self) -> Result>, Error> { + let mut value = self.start_zero_copy(); + let mut last_ch = 0; + let mut in_quote = false; + let mut at_count = 0; + let mut lp_len = 0; + + loop { + let slice = self.bytes.as_slice(); + let pos = if in_quote { + find_any4(slice, LF, b'"', b'\\', b'\r') + } else { + find_non_atext(slice) + }; + let (found, rest) = split_at_index(slice, pos); + if let Some(prev) = preceding_byte(slice, rest) { + last_ch = prev; + } + *self.bytes = rest.iter(); + + let Some(ch) = found else { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + }; + + match ch { + b'0'..=b'9' + | b'a'..=b'z' + | b'A'..=b'Z' + | b'!' + | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'/' + | b'=' + | b'?' + | b'^' + | b'_' + | b'`' + | b'{' + | b'|' + | b'}' + | b'~' + | 0x7f..=u8::MAX => {} + b'.' if !in_quote => { + if last_ch == b'.' || last_ch == b'@' || self.len_excluding_current(&value) == 0 + { + self.stop_char = ch; + return Ok(None); + } + } + b'@' if !in_quote => { + at_count += 1; + lp_len = self.len_excluding_current(&value); + } + b'>' if !in_quote => { + self.stop_char = ch; + self.flush_excluding_current(&mut value); + let value = value.data; + + return Ok(match value.len() { + 1..=MAX_ADDRESS_LEN if at_count == 1 && lp_len > 0 => Some(value), + 10 if at_count == 0 && value.eq_ignore_ascii_case("postmaster") => { + Some("".into()) + } + 0 => Some("".into()), + _ => None, + }); + } + b'\r' => self.flush_excluding_current(&mut value), + b':' if !in_quote && self.first_excluding_current(&value) == Some(b'@') => { + // Remove source route + value = self.start_zero_copy(); + at_count = 0; + lp_len = 0; + } + // Note that if there is an @ at the start, we require `:` a to be in the string + // later, since otherwise `at_count == 1 && lp_len > 0` cannot be satisfied. So it + // doesn't matter what we do here (other than not error). + b',' if !in_quote && self.first_excluding_current(&value) == Some(b'@') => (), + b' ' if !in_quote => { + self.flush_excluding_current(&mut value); + if !value.data.is_empty() { + self.stop_char = b' '; + let value = value.data; + + let is_valid = value.is_empty() + || value.len() <= MAX_ADDRESS_LEN && at_count == 1 && lp_len > 0; + + return Ok(is_valid.then_some(value)); + } + } + b'\n' => { + self.stop_char = b'\n'; + self.flush_excluding_current(&mut value); + let value = value.data; + + let is_valid = value.is_empty() + || value.len() <= MAX_ADDRESS_LEN && at_count == 1 && lp_len > 0; + + return Ok(is_valid.then_some(value)); + } + b'\"' if !in_quote || last_ch != b'\\' => { + in_quote = !in_quote; + self.flush_excluding_current(&mut value); + } + b'\\' if in_quote => { + if last_ch != b'\\' { + self.flush_excluding_current(&mut value); + } else { + // Escaped backslash + last_ch = 0; + continue; + } + } + _ if in_quote => {} + _ => { + self.stop_char = ch; + return Ok(None); + } + } + + last_ch = ch; + } + } + + pub fn string(&mut self) -> Result, Error> { + let mut in_quote = false; + let mut value = self.start_zero_copy(); + let mut last_ch = 0; + + loop { + let slice = self.bytes.as_slice(); + let pos = if in_quote { + find_any4(slice, LF, b'"', b'\\', b'\r') + } else { + find_any4(slice, LF, b' ', b'"', b'\r') + }; + let (found, rest) = split_at_index(slice, pos); + if let Some(prev) = preceding_byte(slice, rest) { + last_ch = prev; + } + *self.bytes = rest.iter(); + + let Some(ch) = found else { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + }; + + match ch { + b' ' if !in_quote => { + self.flush_excluding_current(&mut value); + if !value.data.is_empty() { + self.stop_char = b' '; + return Ok(value.data); + } + } + b'\n' => { + self.flush_excluding_current(&mut value); + self.stop_char = b'\n'; + return Ok(value.data); + } + b'\"' if !in_quote || last_ch != b'\\' => { + in_quote = !in_quote; + self.flush_excluding_current(&mut value); + } + b'\\' if in_quote => { + if last_ch != b'\\' { + self.flush_excluding_current(&mut value); + } else { + last_ch = 0; + continue; + } + } + b'\r' => self.flush_excluding_current(&mut value), + _ => {} + } + + last_ch = ch; + } + } + + pub fn text(&mut self) -> Result, Error> { + let mut value = self.start_zero_copy(); + loop { + let slice = self.bytes.as_slice(); + let (found, rest) = split_at_index(slice, find_any3(slice, LF, b' ', b'\r')); + *self.bytes = rest.iter(); + + let Some(ch) = found else { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + }; + + self.flush_excluding_current(&mut value); + if ch != b'\r' { + self.stop_char = ch; + return Ok(value.data); + } + } + } + + #[allow(clippy::while_let_on_iterator)] + pub fn xtext(&mut self) -> Result, Error> { + let mut value = self.start_zero_copy(); + loop { + let slice = self.bytes.as_slice(); + let (found, rest) = split_at_index(slice, find_any4(slice, LF, b' ', b'+', b'\r')); + *self.bytes = rest.iter(); + + let Some(ch) = found else { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + }; + + match ch { + b'\n' | b' ' => { + self.flush_excluding_current(&mut value); + self.stop_char = ch; + return Ok(value.data); + } + b'+' => { + self.flush_excluding_current(&mut value); + + let mut hex1 = None; + + while let Some(&ch) = self.bytes.next() { + if let Some(digit) = char::from(ch).to_digit(16) { + if let Some(hex1) = hex1 { + let data = value.data.to_mut(); + data.push(char::from(((hex1 as u8) << 4) | digit as u8)); + break; + } else { + hex1 = Some(digit); + } + } else if ch == LF { + self.stop_char = b'\n'; + return Ok(value.data); + } else { + break; + } + } + + self.drop_extra(&mut value); + } + b'\r' => self.flush_excluding_current(&mut value), + _ => {} + } + } + } + + pub fn seek_char(&mut self, stop_char: u8) -> Result, Error> { + let mut value = self.start_zero_copy(); + loop { + let slice = self.bytes.as_slice(); + let (found, rest) = split_at_index(slice, find_any4(slice, LF, b' ', b'\r', stop_char)); + *self.bytes = rest.iter(); + + let Some(ch) = found else { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + }; + + match ch { + b'\n' => { + self.flush_excluding_current(&mut value); + self.stop_char = b'\n'; + return Ok(value.data); + } + b' ' => { + self.flush_excluding_current(&mut value); + if !value.data.is_empty() { + self.stop_char = b' '; + return Ok(value.data); + } + } + b'\r' => self.flush_excluding_current(&mut value), + _ if ch == stop_char => { + self.flush_excluding_current(&mut value); + self.stop_char = ch; + return Ok(value.data); + } + _ => {} + } + } + } + + #[inline(always)] + pub fn seek_lf(&mut self) -> Result<(), Error> { + if self.stop_char != LF { + if self.bytes.len() >= MEMCHR_MIN_LEN { + return self.seek_lf_bulk(); + } + + for &ch in &mut self.bytes { + if ch == LF { + return Ok(()); + } + } + + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } else { + Ok(()) + } + } + + #[inline(never)] + fn seek_lf_bulk(&mut self) -> Result<(), Error> { + let slice = self.bytes.as_slice(); + let (found, rest) = split_at_index(slice, find_lf(slice)); + *self.bytes = rest.iter(); + + if found.is_some() { + Ok(()) + } else { + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + } + + #[inline(always)] + pub fn next_char(&mut self) -> Result { + for &ch in &mut self.bytes { + match ch { + b' ' | b'\r' => (), + _ => { + self.stop_char = ch; + return Ok(ch); + } + } + } + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[inline(always)] + pub fn read_char(&mut self) -> Result { + self.bytes.next().copied().ok_or(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[allow(clippy::while_let_on_iterator)] + pub fn size(&mut self) -> Result { + let mut value: usize = 0; + let mut digits = 0; + + while let Some(&ch) = self.bytes.next() { + let digit = ch.wrapping_sub(b'0'); + if digit < 10 { + if digits == MAX_EXACT_SIZE_DIGITS { + return self.size_saturating(value, digit); + } + value = value * 10 + digit as usize; + digits += 1; + } else if ch != b'\r' && (ch != b' ' || digits != 0) { + self.stop_char = ch; + return Ok(if digits != 0 { value } else { usize::MAX }); + } + } + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[inline(never)] + fn size_saturating(&mut self, value: usize, digit: u8) -> Result { + let mut value = value.saturating_mul(10).saturating_add(digit as usize); + for &ch in &mut self.bytes { + match ch { + b'0'..=b'9' => { + value = if value != usize::MAX { + value + .saturating_mul(10) + .saturating_add((ch - b'0') as usize) + } else { + (ch - b'0') as usize + }; + } + b'\r' => (), + b' ' => { + if value != usize::MAX { + self.stop_char = b' '; + return Ok(value); + } + } + _ => { + self.stop_char = ch; + return Ok(value); + } + } + } + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[allow(clippy::while_let_on_iterator)] + pub fn integer(&mut self) -> Result { + let mut value: i64 = 0; + let mut digits = 0; + let mut multiplier = 1; + + while let Some(&ch) = self.bytes.next() { + let digit = ch.wrapping_sub(b'0'); + if digit < 10 { + if digits == MAX_EXACT_INTEGER_DIGITS { + return self.integer_saturating(value, digit, multiplier); + } + value = value * 10 + digit as i64; + digits += 1; + } else if digits == 0 && (ch == b'-' || ch == b'+') { + if ch == b'-' { + multiplier = -1; + } + } else if ch != b'\r' && (ch != b' ' || digits != 0) { + self.stop_char = ch; + return Ok(if digits != 0 { + value * multiplier + } else { + i64::MAX + }); + } + } + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + #[inline(never)] + fn integer_saturating(&mut self, value: i64, digit: u8, multiplier: i64) -> Result { + let mut value = value.saturating_mul(10).saturating_add(digit as i64); + let mut multiplier = multiplier; + + for &ch in &mut self.bytes { + match ch { + b'0'..=b'9' => { + value = if value != i64::MAX { + value.saturating_mul(10).saturating_add((ch - b'0') as i64) + } else { + (ch - b'0') as i64 + }; + } + b' ' => { + if value != i64::MAX { + self.stop_char = b' '; + return Ok(value * multiplier); + } + } + b'-' if value == i64::MAX => { + multiplier = -1; + } + b'+' if value == i64::MAX => (), + b'\r' => (), + _ => { + self.stop_char = ch; + return Ok(if value != i64::MAX { + value * multiplier + } else { + i64::MAX + }); + } + } + } + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + pub fn timestamp(&mut self) -> Result { + let mut dt = [0u32; 8]; + let mut zone_multiplier = 1; + let mut pos = 0; + let mut in_secfrac = false; + + for &ch in &mut self.bytes { + match ch { + b'0'..=b'9' if in_secfrac => (), + b'0'..=b'9' if pos < 8 => { + dt[pos] = dt[pos] + .saturating_mul(10) + .saturating_add((ch - b'0') as u32); + } + b'-' if pos <= 1 || pos == 5 => { + pos += 1; + in_secfrac = false; + } + b'+' if pos == 5 => { + zone_multiplier = -1; + pos += 1; + in_secfrac = false; + } + b'T' | b't' if pos == 2 => { + pos += 1; + } + b':' if pos == 3 || pos == 4 || pos == 6 => { + pos += 1; + } + b'.' if pos == 5 && !in_secfrac => { + in_secfrac = true; + } + b'Z' | b'z' if pos == 5 => { + pos = 8; + } + _ => { + self.stop_char = ch; + return Ok( + if pos >= 7 + && (1..=12).contains(&dt[1]) + && (1..=31).contains(&dt[2]) + && dt[3] <= 23 + && dt[4] <= 59 + && dt[5] <= 60 + && dt[6] <= 23 + && dt[7] <= 59 + { + // Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992 + let month = dt[1]; + let year_base = 4800; /* Before min year, multiple of 400. */ + let m_adj = month.wrapping_sub(3); /* March-based month. */ + let carry = i64::from(m_adj > month); + let adjust = if carry > 0 { 12 } else { 0 }; + let y_adj = dt[0] as i64 + year_base - carry; + let month_days = + ((m_adj.wrapping_add(adjust)).wrapping_mul(62719) + 769) / 2048; + let leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400; + (y_adj * 365 + leap_days + month_days as i64 + (dt[2] as i64 - 1) + - 2472632) + * 86400 + + dt[3] as i64 * 3600 + + dt[4] as i64 * 60 + + dt[5] as i64 + + ((dt[6] as i64 * 3600 + dt[7] as i64 * 60) * zone_multiplier) + } else { + i64::MAX + }, + ); + } + } + } + + Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }) + } + + pub fn mail_from_parameters( + &mut self, + address: Cow<'y, str>, + ) -> Result>, Error> { + let mut params = MailFrom { + address, + flags: 0, + size: 0, + trans_id: None, + by: 0, + env_id: None, + solicit: None, + mtrk: None, + auth: None, + hold_for: 0, + hold_until: 0, + mt_priority: 0, + }; + while self.stop_char != LF { + let key = self.hashed_value_long()?; + match key { + SMTPUTF8 if self.stop_char.is_ascii_whitespace() => { + params.flags |= MAIL_SMTPUTF8; + } + BODY if self.stop_char == b'=' => { + params.flags |= match self.hashed_value_long()? { + EIGHBITMIME if self.stop_char.is_ascii_whitespace() => MAIL_BODY_8BITMIME, + BINARYMIME if self.stop_char.is_ascii_whitespace() => MAIL_BODY_BINARYMIME, + SEVENBIT if self.stop_char.is_ascii_whitespace() => MAIL_BODY_7BIT, + _ => { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "BODY" }); + } + } + } + SIZE if self.stop_char == b'=' => { + let size = self.size()?; + if size != usize::MAX && self.stop_char.is_ascii_whitespace() { + params.size = size; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "SIZE" }); + } + } + BY if self.stop_char == b'=' => { + let time = self.integer()?; + if time != i64::MAX && self.stop_char == b';' { + params.flags |= match self.hashed_value()? { + N if self.stop_char.is_ascii_whitespace() => MAIL_BY_NOTIFY, + NT if self.stop_char.is_ascii_whitespace() => { + MAIL_BY_NOTIFY | MAIL_BY_TRACE + } + R if self.stop_char.is_ascii_whitespace() => MAIL_BY_RETURN, + RT if self.stop_char.is_ascii_whitespace() => { + MAIL_BY_RETURN | MAIL_BY_TRACE + } + _ => { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "BY" }); + } + }; + params.by = time; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "BY" }); + } + } + HOLDUNTIL if self.stop_char == b'=' => { + let hold = self.timestamp()?; + if hold != i64::MAX && hold >= 0 && self.stop_char.is_ascii_whitespace() { + params.hold_until = hold as u64; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "HOLDUNTIL" }); + } + } + HOLDFOR if self.stop_char == b'=' => { + let hold = self.size()?; + if (1..=MAX_FUTURE_RELEASE_INTERVAL).contains(&hold) + && self.stop_char.is_ascii_whitespace() + { + params.hold_for = hold as u64; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "HOLDFOR" }); + } + } + RET if self.stop_char == b'=' => { + params.flags |= match self.hashed_value()? { + FULL if self.stop_char.is_ascii_whitespace() => MAIL_RET_FULL, + HDRS if self.stop_char.is_ascii_whitespace() => MAIL_RET_HDRS, + _ => { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "RET" }); + } + }; + } + ENVID if self.stop_char == b'=' => { + let env_id = self.xtext()?; + if self.stop_char.is_ascii_whitespace() && (1..=100).contains(&env_id.len()) { + params.env_id = env_id.into(); + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "ENVID" }); + } + } + REQUIRETLS if self.stop_char.is_ascii_whitespace() => { + params.flags |= MAIL_REQUIRETLS; + } + SOLICIT if self.stop_char == b'=' => { + let solicit = self.text()?; + if !solicit.is_empty() && self.stop_char.is_ascii_whitespace() { + params.solicit = solicit.into(); + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "SOLICIT" }); + } + } + TRANSID if self.stop_char == b'=' => { + if self.next_char()? == b'<' { + let transid = self.seek_char(b'>')?; + if self.stop_char == b'>' && !transid.is_empty() { + params.trans_id = transid.into(); + self.stop_char = SP; + continue; + } + } + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "TRANSID" }); + } + MTRK if self.stop_char == b'=' => { + let certifier = self.seek_char(b':')?; + let timeout = if self.stop_char == b':' { + self.size()? + } else { + 0 + }; + + if !certifier.is_empty() + && self.stop_char.is_ascii_whitespace() + && timeout != usize::MAX + { + params.mtrk = Mtrk { + certifier, + timeout: timeout as u64, + } + .into(); + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "MTRK" }); + } + } + AUTH_ if self.stop_char == b'=' => { + let mailbox = self.xtext()?; + if (1..=MAX_ADDRESS_LEN).contains(&mailbox.len()) + && self.stop_char.is_ascii_whitespace() + { + params.auth = mailbox.into(); + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "AUTH" }); + } + } + MT_PRIORITY if self.stop_char == b'=' => { + let priority = self.integer()?; + if priority != i64::MAX && self.stop_char.is_ascii_whitespace() { + params.mt_priority = priority; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { + param: "MT-PRIORITY", + }); + } + } + CONPERM if self.stop_char.is_ascii_whitespace() => { + params.flags |= MAIL_CONPERM; + } + 0 => { + // hashed_value_long() returned 0, meaning no alphanumeric + // characters were found before hitting a non-keyword char. + // If stop_char is whitespace/LF this is just extra spacing + // between parameters (harmless). Otherwise it's garbage + // like ">", "!", etc. that should be rejected. + if !self.stop_char.is_ascii_whitespace() { + self.seek_lf()?; + return Err(Error::SyntaxError { + syntax: "MAIL FROM: [parameters]", + }); + } + } + unknown => { + let mut param = Vec::with_capacity(16); + for ch in unknown.to_le_bytes() { + if ch != 0 { + param.push(ch.to_ascii_uppercase()); + } + } + if !self.stop_char.is_ascii_whitespace() { + param.push(self.stop_char.to_ascii_uppercase()); + for &ch in &mut self.bytes { + if !ch.is_ascii_whitespace() { + param.push(ch.to_ascii_uppercase()); + } else { + self.stop_char = ch; + break; + } + } + } + + self.seek_lf()?; + return Err(Error::UnsupportedParameter { + param: param.into_string(), + }); + } + } + } + + Ok(params) + } + + pub fn rcpt_to_parameters( + &mut self, + address: Cow<'y, str>, + ) -> Result>, Error> { + let mut params = RcptTo { + address, + orcpt: None, + rrvs: 0, + flags: 0, + }; + while self.stop_char != LF { + let key = self.hashed_value_long()?; + match key { + NOTIFY if self.stop_char == b'=' => loop { + match self.hashed_value_long()? { + NEVER + if (params.flags + & (RCPT_NOTIFY_NEVER + | RCPT_NOTIFY_SUCCESS + | RCPT_NOTIFY_FAILURE + | RCPT_NOTIFY_DELAY)) + == 0 => + { + params.flags |= RCPT_NOTIFY_NEVER; + } + SUCCESS => { + params.flags |= RCPT_NOTIFY_SUCCESS; + } + FAILURE => { + params.flags |= RCPT_NOTIFY_FAILURE; + } + DELAY => { + params.flags |= RCPT_NOTIFY_DELAY; + } + _ => { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "NOTIFY" }); + } + } + if self.stop_char.is_ascii_whitespace() { + break; + } else if self.stop_char != b',' { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "NOTIFY" }); + } + }, + ORCPT if self.stop_char == b'=' => { + let v = self.hashed_value()?; + if v != RFC822 || self.stop_char != b';' { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "ORCPT" }); + } + let addr = self.xtext()?; + if self.stop_char.is_ascii_whitespace() + && (1..=MAX_ADDRESS_LEN).contains(&addr.len()) + { + params.orcpt = addr.into(); + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "ORCPT" }); + } + } + RRVS if self.stop_char == b'=' => { + let time = self.timestamp()?; + if time != i64::MAX && self.stop_char.is_ascii_whitespace() + || self.stop_char == b';' + { + let is_reject = self.stop_char != b';' + || match self.hashed_value()? { + C if self.stop_char.is_ascii_whitespace() => false, + R if self.stop_char.is_ascii_whitespace() => true, + _ => { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "RRVS" }); + } + }; + params.rrvs = time; + params.flags |= if is_reject { + RCPT_RRVS_REJECT + } else { + RCPT_RRVS_CONTINUE + }; + } else { + self.seek_lf()?; + return Err(Error::InvalidParameter { param: "RRVS" }); + } + } + CONNEG if self.stop_char.is_ascii_whitespace() => { + params.flags |= RCPT_CONNEG; + } + 0 => { + if !self.stop_char.is_ascii_whitespace() { + self.seek_lf()?; + return Err(Error::SyntaxError { + syntax: "RCPT TO: [parameters]", + }); + } + } + unknown => { + let mut param = Vec::with_capacity(16); + for ch in unknown.to_le_bytes() { + if ch != 0 { + param.push(ch.to_ascii_uppercase()); + } + } + if !self.stop_char.is_ascii_whitespace() { + param.push(self.stop_char.to_ascii_uppercase()); + for &ch in &mut self.bytes { + if !ch.is_ascii_whitespace() { + param.push(ch.to_ascii_uppercase()); + } else { + self.stop_char = ch; + break; + } + } + } + + self.seek_lf()?; + return Err(Error::UnsupportedParameter { + param: param.into_string(), + }); + } + } + } + + Ok(params) + } + + pub fn mechanism(&mut self) -> Result, Error> { + let mut trailing_chars = [0u8; 8]; + let mut pos = 0; + let mechanism = self.hashed_value_long()?; + if !self.stop_char.is_ascii_whitespace() { + trailing_chars[0] = self.stop_char; + pos += 1; + for &ch in &mut self.bytes { + if !ch.is_ascii_whitespace() { + if let Some(tch) = trailing_chars.get_mut(pos) { + *tch = ch.to_ascii_uppercase(); + } + pos += 1; + } else { + self.stop_char = ch; + break; + } + } + if !self.stop_char.is_ascii_whitespace() { + return Err(Error::NeedsMoreData { + bytes_left: self.bytes_left, + }); + } else if pos > 8 { + return Ok(0.into()); + } + } + Ok(match (mechanism, &trailing_chars[..pos]) { + (_9798_M_DSA_SHA1, b"") => AUTH_9798_M_DSA_SHA1.into(), + (_9798_M_ECDSA_SHA, b"1") => AUTH_9798_M_ECDSA_SHA1.into(), + (_9798_M_RSA_SHA1_, b"ENC") => AUTH_9798_M_RSA_SHA1_ENC.into(), + (_9798_U_DSA_SHA1, b"") => AUTH_9798_U_DSA_SHA1.into(), + (_9798_U_ECDSA_SHA, b"1") => AUTH_9798_U_ECDSA_SHA1.into(), + (_9798_U_RSA_SHA1_, b"ENC") => AUTH_9798_U_RSA_SHA1_ENC.into(), + (ANONYMOUS, b"") => AUTH_ANONYMOUS.into(), + (CRAM_MD5, b"") => AUTH_CRAM_MD5.into(), + (DIGEST_MD5, b"") => AUTH_DIGEST_MD5.into(), + (EAP_AES128, b"") => AUTH_EAP_AES128.into(), + (EAP_AES128_PLUS, b"") => AUTH_EAP_AES128_PLUS.into(), + (ECDH_X25519_CHAL, b"LENGE") => AUTH_ECDH_X25519_CHALLENGE.into(), + (ECDSA_NIST256P_C, b"HALLENGE") => AUTH_ECDSA_NIST256P_CHALLENGE.into(), + (EXTERNAL, b"") => AUTH_EXTERNAL.into(), + (GS2_KRB5, b"") => AUTH_GS2_KRB5.into(), + (GS2_KRB5_PLUS, b"") => AUTH_GS2_KRB5_PLUS.into(), + (GSS_SPNEGO, b"") => AUTH_GSS_SPNEGO.into(), + (GSSAPI, b"") => AUTH_GSSAPI.into(), + (KERBEROS_V4, b"") => AUTH_KERBEROS_V4.into(), + (KERBEROS_V5, b"") => AUTH_KERBEROS_V5.into(), + (LOGIN, b"") => AUTH_LOGIN.into(), + (NMAS_SAMBA_AUTH, b"") => AUTH_NMAS_SAMBA_AUTH.into(), + (NMAS_AUTHEN, b"") => AUTH_NMAS_AUTHEN.into(), + (NMAS_LOGIN, b"") => AUTH_NMAS_LOGIN.into(), + (NTLM, b"") => AUTH_NTLM.into(), + (OAUTH10A, b"") => AUTH_OAUTH10A.into(), + (OAUTHBEARER, b"") => AUTH_OAUTHBEARER.into(), + (OPENID20, b"") => AUTH_OPENID20.into(), + (OTP, b"") => AUTH_OTP.into(), + (PLAIN, b"") => AUTH_PLAIN.into(), + (SAML20, b"") => AUTH_SAML20.into(), + (SCRAM_SHA_1, b"") => AUTH_SCRAM_SHA_1.into(), + (SCRAM_SHA_1_PLUS, b"") => AUTH_SCRAM_SHA_1_PLUS.into(), + (SCRAM_SHA_256, b"") => AUTH_SCRAM_SHA_256.into(), + (SCRAM_SHA_256_PL, b"US") => AUTH_SCRAM_SHA_256_PLUS.into(), + (SECURID, b"") => AUTH_SECURID.into(), + (SKEY, b"") => AUTH_SKEY.into(), + (SPNEGO, b"") => AUTH_SPNEGO.into(), + (SPNEGO_PLUS, b"") => AUTH_SPNEGO_PLUS.into(), + (SXOVER_PLUS, b"") => AUTH_SXOVER_PLUS.into(), + (XOAUTH, b"") => AUTH_XOAUTH.into(), + (XOAUTH2, b"") => AUTH_XOAUTH2.into(), + (0, b"") => None, + _ => 0.into(), + }) + } + + /// Construct a new `MaybeZeroCopy`, beginning a potentially zero-copy read from the input. + #[inline(always)] + fn start_zero_copy(&self) -> MaybeZeroCopy<'y> { + MaybeZeroCopy { + remaining: self.bytes.as_slice(), + data: Cow::Borrowed(""), + } + } + + /// Get the slice starting at the last time `MaybeZeroCopy` was flushed and ending one byte + /// before where `Self` currently is. This is what will be pushed to the `MaybeZeroCopy` on + /// flush. + fn extra_before_current(&self, out: &MaybeZeroCopy<'y>) -> &'y [u8] { + // Calculate the number of bytes we have advanced our iterator from the start of the + // `MaybeZeroCopy`. Subtract one to remove the current byte. + let new_len = self.bytes.as_slice().as_ptr().addr() - out.remaining.as_ptr().addr() - 1; + &out.remaining[..new_len] + } + + /// Get the length of a `MaybeZeroCopy`. + fn len_excluding_current(&self, out: &MaybeZeroCopy<'y>) -> usize { + out.data.len() + self.extra_before_current(out).len() + } + + /// Get the first byte of a `MaybeZeroCopy`. + fn first_excluding_current(&self, out: &MaybeZeroCopy<'y>) -> Option { + out.data + .bytes() + .next() + .or_else(|| self.extra_before_current(out).first().copied()) + } + + /// Drop data up to the current cursor from the `MaybeZeroCopy`. + #[inline(always)] + fn drop_extra(&self, out: &mut MaybeZeroCopy<'y>) { + out.remaining = self.bytes.as_slice(); + } + + /// Extend a `MaybeZeroCopy` with new bytes. + fn flush_excluding_current(&self, out: &mut MaybeZeroCopy<'y>) { + let extra = self.extra_before_current(out); + self.drop_extra(out); + + if extra.is_empty() { + return; + } + + if out.data.is_empty() { + out.data = match from_utf8(extra) { + Ok(valid) => Cow::Borrowed(valid), + Err(_) => String::from_utf8_lossy(extra), + }; + } else { + if let Cow::Borrowed(s) = out.data { + let mut buf = String::with_capacity(32); + buf.push_str(s); + out.data = Cow::Owned(buf); + } + + // The same algorithm as `String::from_utf8_lossy`, but appending to an existing + // buffer. + let out = out.data.to_mut(); + for chunk in extra.utf8_chunks() { + out.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + } + } + } + } +} + +/// A state machine enabling parsing to be potentially zero-copy. +struct MaybeZeroCopy<'y> { + /// Slice from where we start parsing to the end of the input. + remaining: &'y [u8], + /// The data itself. + data: Cow<'y, str>, +} + diff --git a/crates/smtp-proto/src/request/receiver.rs b/crates/smtp-proto/src/request/receiver.rs new file mode 100644 index 00000000..706a4273 --- /dev/null +++ b/crates/smtp-proto/src/request/receiver.rs @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Error, Request}; +use memchr::{memchr, memchr2, memmem}; +use std::{borrow::Cow, slice::Iter}; + +pub const MAX_LINE_LENGTH: usize = 4096; + +#[derive(Default)] +pub struct RequestReceiver { + buf: Vec, + buf_used: bool, +} + +pub struct DataReceiver { + state: DataState, +} + +pub struct BdatReceiver { + pub is_last: bool, + bytes_left: usize, +} + +pub struct DummyDataReceiver { + is_bdat: bool, + bdat_bytes_left: usize, + state: DataState, +} + +#[derive(Clone, Copy)] +struct DataState { + crlf_dot: bool, + last_ch: u8, + prev_last_ch: u8, +} + +struct Discard; + +trait DataSink { + fn push_byte(&mut self, ch: u8); + + fn push_slice(&mut self, bytes: &[u8]); + + fn pop_byte(&mut self); + + fn take_chunk(&mut self, bytes: &mut Iter<'_, u8>, bytes_left: &mut usize) -> bool { + let rest = bytes.as_slice(); + let (chunk, tail) = rest.split_at(rest.len().min(*bytes_left)); + self.push_slice(chunk); + *bytes_left -= chunk.len(); + *bytes = tail.iter(); + *bytes_left == 0 + } +} + +impl DataSink for Vec { + fn push_byte(&mut self, ch: u8) { + self.push(ch); + } + + fn push_slice(&mut self, bytes: &[u8]) { + self.extend_from_slice(bytes); + } + + fn pop_byte(&mut self) { + self.pop(); + } +} + +impl DataSink for Discard { + fn push_byte(&mut self, _: u8) {} + + fn push_slice(&mut self, _: &[u8]) {} + + fn pop_byte(&mut self) {} +} + +impl DataState { + fn new() -> Self { + Self { + crlf_dot: false, + last_ch: 0, + prev_last_ch: 0, + } + } + + fn advance(&mut self, chunk: &[u8]) { + if let Some((&last, head)) = chunk.split_last() { + self.prev_last_ch = head.last().copied().unwrap_or(self.last_ch); + self.last_ch = last; + } + } + + fn ingest(&mut self, bytes: &mut Iter<'_, u8>, sink: &mut S) -> bool { + let mut rest = bytes.as_slice(); + + loop { + if self.last_ch == b'\n' + && self.prev_last_ch == b'\r' + && let [b'.', tail @ ..] = rest + { + self.crlf_dot = true; + self.prev_last_ch = b'\n'; + self.last_ch = b'.'; + rest = tail; + } + + while self.crlf_dot { + let Some((&ch, tail)) = rest.split_first() else { + *bytes = rest.iter(); + return false; + }; + rest = tail; + + if ch == b'\n' && self.last_ch == b'\r' { + sink.pop_byte(); + *bytes = rest.iter(); + return true; + } + + sink.push_byte(ch); + self.crlf_dot = ch == b'\r'; + self.prev_last_ch = self.last_ch; + self.last_ch = ch; + } + + let Some(pos) = find_stuffed_newline(rest) else { + sink.push_slice(rest); + self.advance(rest); + *bytes = [].iter(); + return false; + }; + + let (line, tail) = rest.split_at(pos + 1); + sink.push_slice(line); + self.advance(line); + rest = tail; + } + } +} + +#[inline(always)] +fn find_stuffed_newline(haystack: &[u8]) -> Option { + memmem::find(haystack, b"\n.") +} + +#[derive(Default)] +pub struct DummyLineReceiver {} + +#[derive(Default)] +pub struct LineReceiver { + pub buf: Vec, + pub state: T, +} + +impl RequestReceiver { + pub fn buf(&mut self) -> &mut Vec { + if self.buf_used { + self.buf.clear(); + self.buf_used = false; + } + + &mut self.buf + } + + pub fn ingest<'this, 'bytes, 'out>( + &'this mut self, + bytes: &mut Iter<'bytes, u8>, + ) -> Result>, Error> + where + 'this: 'out, + 'bytes: 'out, + { + self.buf(); + + if self.buf.is_empty() { + let buf = bytes.as_slice(); + match Request::parse(bytes) { + Err(Error::NeedsMoreData { bytes_left }) => { + if bytes_left > 0 { + if bytes_left < MAX_LINE_LENGTH { + self.buf = buf[buf.len().saturating_sub(bytes_left)..].to_vec(); + } else { + return Err(Error::ResponseTooLong); + } + } + } + result => return result, + } + } else { + let rest = bytes.as_slice(); + let free = MAX_LINE_LENGTH.saturating_sub(self.buf.len()); + let newline = memchr(b'\n', rest); + + if let Some(pos) = newline.filter(|&pos| pos < free || free == 0) { + let (line, tail) = rest.split_at(pos + 1); + self.buf.extend_from_slice(line); + *bytes = tail.iter(); + self.buf_used = true; + return Request::parse(&mut self.buf.iter()); + } else if free > 0 && rest.len() >= free { + *bytes = rest.split_at(free).1.iter(); + self.buf.clear(); + return Err(Error::ResponseTooLong); + } + + self.buf.extend_from_slice(rest); + *bytes = [].iter(); + } + + Err(Error::NeedsMoreData { bytes_left: 0 }) + } +} + +impl DataReceiver { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + state: DataState::new(), + } + } + + pub fn ingest(&mut self, bytes: &mut Iter<'_, u8>, buf: &mut Vec) -> bool { + self.state.ingest(bytes, buf) + } +} + +impl BdatReceiver { + pub fn new(chunk_size: usize, is_last: bool) -> Self { + Self { + bytes_left: chunk_size, + is_last, + } + } + + pub fn ingest(&mut self, bytes: &mut Iter<'_, u8>, buf: &mut Vec) -> bool { + buf.take_chunk(bytes, &mut self.bytes_left) + } +} + +impl DummyDataReceiver { + pub fn new_bdat(chunk_size: usize) -> Self { + Self { + bdat_bytes_left: chunk_size, + is_bdat: true, + state: DataState::new(), + } + } + + pub fn new_data(data: &DataReceiver) -> Self { + Self { + is_bdat: false, + bdat_bytes_left: 0, + state: data.state, + } + } + + pub fn ingest(&mut self, bytes: &mut Iter<'_, u8>) -> bool { + if !self.is_bdat { + self.state.ingest(bytes, &mut Discard) + } else { + Discard.take_chunk(bytes, &mut self.bdat_bytes_left) + } + } +} + +impl LineReceiver { + pub fn new(state: T) -> Self { + Self { + buf: Vec::with_capacity(32), + state, + } + } + + pub fn ingest(&mut self, bytes: &mut Iter<'_, u8>) -> bool { + let mut rest = bytes.as_slice(); + + loop { + let Some(pos) = memchr2(b'\n', b'\r', rest) else { + self.push_capped(rest); + *bytes = [].iter(); + return false; + }; + + let (head, tail) = rest.split_at(pos); + self.push_capped(head); + + let Some((&ch, tail)) = tail.split_first() else { + *bytes = [].iter(); + return false; + }; + rest = tail; + + if ch == b'\n' { + *bytes = rest.iter(); + return true; + } + } + } + + fn push_capped(&mut self, chunk: &[u8]) { + let free = MAX_LINE_LENGTH.saturating_sub(self.buf.len()); + self.buf.extend_from_slice(&chunk[..chunk.len().min(free)]); + } +} + +impl DummyLineReceiver { + pub fn ingest(&mut self, bytes: &mut Iter<'_, u8>) -> bool { + let rest = bytes.as_slice(); + + match memchr(b'\n', rest) { + Some(pos) => { + *bytes = rest.split_at(pos + 1).1.iter(); + true + } + None => { + *bytes = [].iter(); + false + } + } + } +} + diff --git a/crates/smtp-proto/src/request/scan.rs b/crates/smtp-proto/src/request/scan.rs new file mode 100644 index 00000000..4b5e4af2 --- /dev/null +++ b/crates/smtp-proto/src/request/scan.rs @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::LF; +pub(super) const MEMCHR_MIN_LEN: usize = 16; + +const SWAR_LOW: u64 = 0x0101_0101_0101_0101; +const SWAR_HIGH: u64 = 0x8080_8080_8080_8080; +const ATEXT: [bool; 256] = build_atext(); + +pub(super) const TOKEN_UPPER: [u8; 256] = { + let mut table = [0u8; 256]; + let mut index = 0; + while index < table.len() { + let ch = index as u8; + table[index] = match ch { + b'A'..=b'Z' | b'0'..=b'9' | b'-' => ch, + b'a'..=b'z' => ch - b'a' + b'A', + _ => 0, + }; + index += 1; + } + table +}; + +const fn build_atext() -> [bool; 256] { + let mut table = [false; 256]; + let mut index = 0; + while index < 256 { + table[index] = is_atext(index as u8); + index += 1; + } + table +} + +#[inline(always)] +const fn is_atext(ch: u8) -> bool { + matches!( + ch, + b'0'..=b'9' + | b'a'..=b'z' + | b'A'..=b'Z' + | b'!' + | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'/' + | b'=' + | b'?' + | b'^' + | b'_' + | b'`' + | b'{' + | b'|' + | b'}' + | b'~' + | 0x7f..=u8::MAX + ) +} + +#[inline(always)] +const fn swar_splat(byte: u8) -> u64 { + SWAR_LOW.wrapping_mul(byte as u64) +} + +#[inline(always)] +const fn swar_zero_byte(word: u64) -> u64 { + word.wrapping_sub(SWAR_LOW) & !word & SWAR_HIGH +} + +#[inline(always)] +pub(super) fn find_lf(slice: &[u8]) -> Option { + memchr::memchr(LF, slice) +} + +#[inline(always)] +pub(super) fn find_any3(slice: &[u8], a: u8, b: u8, c: u8) -> Option { + if slice.len() < MEMCHR_MIN_LEN { + slice.iter().position(|&ch| ch == a || ch == b || ch == c) + } else { + memchr::memchr3(a, b, c, slice) + } +} + +#[inline(always)] +pub(super) fn find_any4(slice: &[u8], a: u8, b: u8, c: u8, d: u8) -> Option { + let (chunks, rest) = slice.as_chunks::<8>(); + for (index, chunk) in chunks.iter().enumerate() { + let word = u64::from_le_bytes(*chunk); + let hits = swar_zero_byte(word ^ swar_splat(a)) + | swar_zero_byte(word ^ swar_splat(b)) + | swar_zero_byte(word ^ swar_splat(c)) + | swar_zero_byte(word ^ swar_splat(d)); + if hits != 0 { + return Some(index * 8 + (hits.trailing_zeros() / 8) as usize); + } + } + + rest.iter() + .position(|&ch| ch == a || ch == b || ch == c || ch == d) + .map(|pos| chunks.len() * 8 + pos) +} + +#[inline(always)] +pub(super) fn find_non_atext(slice: &[u8]) -> Option { + slice.iter().position(|&ch| !ATEXT[ch as usize]) +} + +#[inline(always)] +pub(super) fn split_at_index(slice: &[u8], pos: Option) -> (Option, &[u8]) { + match pos.and_then(|pos| slice.split_at_checked(pos + 1)) { + Some((head, tail)) => (head.last().copied(), tail), + None => (None, &slice[slice.len()..]), + } +} + +#[inline(always)] +pub(super) fn preceding_byte(slice: &[u8], rest: &[u8]) -> Option { + slice + .len() + .checked_sub(rest.len() + 2) + .and_then(|pos| slice.get(pos).copied()) +} diff --git a/crates/smtp-proto/src/response/generate.rs b/crates/smtp-proto/src/response/generate.rs new file mode 100644 index 00000000..b17a0531 --- /dev/null +++ b/crates/smtp-proto/src/response/generate.rs @@ -0,0 +1,1290 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::*; +use std::{ + fmt::{self, Display}, + io::{self, Write}, +}; + +impl EhloResponse { + pub fn new(hostname: T) -> Self { + Self { + hostname, + capabilities: 0, + auth_mechanisms: 0, + deliver_by: 0, + future_release_interval: 0, + future_release_datetime: 0, + mt_priority: MtPriority::Mixer, + no_soliciting: None, + size: 0, + } + } + + pub fn write(&self, mut writer: impl Write) -> io::Result<()> { + writer.write_all(b"250-")?; + write!(writer, "{}", self.hostname)?; + writer.write_all(b" you had me at EHLO\r\n")?; + let mut capabilities = self.capabilities; + + while capabilities != 0 { + let capability = 1 << (31 - capabilities.leading_zeros()); + capabilities ^= capability; + + writer.write_all(if capabilities != 0 { b"250-" } else { b"250 " })?; + match capability { + EXT_8BIT_MIME => writer.write_all(b"8BITMIME\r\n"), + EXT_ATRN => writer.write_all(b"ATRN\r\n"), + EXT_AUTH => { + writer.write_all(b"AUTH")?; + let mut mechanisms = self.auth_mechanisms; + while mechanisms != 0 { + let item = 1 << (63 - mechanisms.leading_zeros()); + mechanisms ^= item; + writer.write_all(b" ")?; + writer.write_all(item.to_mechanism().as_bytes())?; + } + writer.write_all(b"\r\n") + } + EXT_BINARY_MIME => writer.write_all(b"BINARYMIME\r\n"), + EXT_BURL => writer.write_all(b"BURL\r\n"), + EXT_CHECKPOINT => writer.write_all(b"CHECKPOINT\r\n"), + EXT_CHUNKING => writer.write_all(b"CHUNKING\r\n"), + EXT_CONNEG => writer.write_all(b"CONNEG\r\n"), + EXT_CONPERM => writer.write_all(b"CONPERM\r\n"), + EXT_DELIVER_BY => { + if self.deliver_by > 0 { + writer.write_all(b"DELIVERBY ")?; + write_decimal(&mut writer, self.deliver_by)?; + writer.write_all(b"\r\n") + } else { + writer.write_all(b"DELIVERBY\r\n") + } + } + EXT_DSN => writer.write_all(b"DSN\r\n"), + EXT_ENHANCED_STATUS_CODES => writer.write_all(b"ENHANCEDSTATUSCODES\r\n"), + EXT_ETRN => writer.write_all(b"ETRN\r\n"), + EXT_EXPN => writer.write_all(b"EXPN\r\n"), + EXT_VRFY => writer.write_all(b"VRFY\r\n"), + EXT_FUTURE_RELEASE => { + writer.write_all(b"FUTURERELEASE ")?; + write_decimal(&mut writer, self.future_release_interval)?; + writer.write_all(b" ")?; + write!(writer, "{}\r\n", Rfc3339(self.future_release_datetime)) + } + EXT_HELP => writer.write_all(b"HELP\r\n"), + EXT_MT_PRIORITY => { + writer.write_all(b"MT-PRIORITY ")?; + writer.write_all(match self.mt_priority { + MtPriority::Mixer => b"MIXER".as_slice(), + MtPriority::Stanag4406 => b"STANAG4406".as_slice(), + MtPriority::Nsep => b"NSEP".as_slice(), + })?; + writer.write_all(b"\r\n") + } + EXT_MTRK => writer.write_all(b"MTRK\r\n"), + EXT_NO_SOLICITING => { + if let Some(keywords) = &self.no_soliciting { + writer.write_all(b"NO-SOLICITING ")?; + writer.write_all(keywords.as_bytes())?; + writer.write_all(b"\r\n") + } else { + writer.write_all(b"NO-SOLICITING\r\n") + } + } + EXT_ONEX => writer.write_all(b"ONEX\r\n"), + EXT_PIPELINING => writer.write_all(b"PIPELINING\r\n"), + EXT_REQUIRE_TLS => writer.write_all(b"REQUIRETLS\r\n"), + EXT_RRVS => writer.write_all(b"RRVS\r\n"), + EXT_SIZE => { + if self.size > 0 { + writer.write_all(b"SIZE ")?; + write_decimal(&mut writer, self.size as u64)?; + writer.write_all(b"\r\n") + } else { + writer.write_all(b"SIZE\r\n") + } + } + EXT_SMTP_UTF8 => writer.write_all(b"SMTPUTF8\r\n"), + EXT_START_TLS => writer.write_all(b"STARTTLS\r\n"), + EXT_VERB => writer.write_all(b"VERB\r\n"), + _ => Ok(()), + }?; + } + + Ok(()) + } +} + +impl Response { + pub fn write(&self, mut writer: impl Write) -> io::Result<()> { + let [class, subject, detail] = self.esc; + if (100..1000).contains(&self.code) && class < 10 && subject < 10 && detail < 10 { + writer.write_all(&[ + b'0' + (self.code / 100) as u8, + b'0' + (self.code / 10 % 10) as u8, + b'0' + (self.code % 10) as u8, + b' ', + b'0' + class, + b'.', + b'0' + subject, + b'.', + b'0' + detail, + b' ', + ])?; + } else { + write!(writer, "{} {class}.{subject}.{detail} ", self.code)?; + } + write!(writer, "{}", self.message)?; + writer.write_all(b"\r\n") + } +} + +pub trait BitToString { + fn to_mechanism(&self) -> &'static str; +} + +impl BitToString for u64 { + fn to_mechanism(&self) -> &'static str { + match *self { + AUTH_SCRAM_SHA_256_PLUS => "SCRAM-SHA-256-PLUS", + AUTH_SCRAM_SHA_256 => "SCRAM-SHA-256", + AUTH_SCRAM_SHA_1_PLUS => "SCRAM-SHA-1-PLUS", + AUTH_SCRAM_SHA_1 => "SCRAM-SHA-1", + AUTH_OAUTHBEARER => "OAUTHBEARER", + AUTH_XOAUTH => "XOAUTH", + AUTH_XOAUTH2 => "XOAUTH2", + AUTH_9798_M_DSA_SHA1 => "9798-M-DSA-SHA1", + AUTH_9798_M_ECDSA_SHA1 => "9798-M-ECDSA-SHA1", + AUTH_9798_M_RSA_SHA1_ENC => "9798-M-RSA-SHA1-ENC", + AUTH_9798_U_DSA_SHA1 => "9798-U-DSA-SHA1", + AUTH_9798_U_ECDSA_SHA1 => "9798-U-ECDSA-SHA1", + AUTH_9798_U_RSA_SHA1_ENC => "9798-U-RSA-SHA1-ENC", + AUTH_EAP_AES128 => "EAP-AES128", + AUTH_EAP_AES128_PLUS => "EAP-AES128-PLUS", + AUTH_ECDH_X25519_CHALLENGE => "ECDH-X25519-CHALLENGE", + AUTH_ECDSA_NIST256P_CHALLENGE => "ECDSA-NIST256P-CHALLENGE", + AUTH_EXTERNAL => "EXTERNAL", + AUTH_GS2_KRB5 => "GS2-KRB5", + AUTH_GS2_KRB5_PLUS => "GS2-KRB5-PLUS", + AUTH_GSS_SPNEGO => "GSS-SPNEGO", + AUTH_GSSAPI => "GSSAPI", + AUTH_KERBEROS_V4 => "KERBEROS_V4", + AUTH_KERBEROS_V5 => "KERBEROS_V5", + AUTH_NMAS_SAMBA_AUTH => "NMAS-SAMBA-AUTH", + AUTH_NMAS_AUTHEN => "NMAS_AUTHEN", + AUTH_NMAS_LOGIN => "NMAS_LOGIN", + AUTH_NTLM => "NTLM", + AUTH_OAUTH10A => "OAUTH10A", + AUTH_OPENID20 => "OPENID20", + AUTH_OTP => "OTP", + AUTH_SAML20 => "SAML20", + AUTH_SECURID => "SECURID", + AUTH_SKEY => "SKEY", + AUTH_SPNEGO => "SPNEGO", + AUTH_SPNEGO_PLUS => "SPNEGO-PLUS", + AUTH_SXOVER_PLUS => "SXOVER-PLUS", + AUTH_CRAM_MD5 => "CRAM-MD5", + AUTH_DIGEST_MD5 => "DIGEST-MD5", + AUTH_LOGIN => "LOGIN", + AUTH_PLAIN => "PLAIN", + AUTH_ANONYMOUS => "ANONYMOUS", + _ => "", + } + } +} + +struct Rfc3339(u64); + +impl Display for Rfc3339 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let days = (self.0 / 86400) as i64; + let seconds = self.0 % 86400; + + // Ported from https://howardhinnant.github.io/date_algorithms.html#civil_from_days + let z = days + 719468; + let era = if z >= 0 { z } else { z - 146096 } / 146097; + let doe = (z - era * 146097) as u64; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe as i64 + era * 400 + i64::from(month <= 2); + + let hour = seconds / 3600; + let minute = (seconds / 60) % 60; + let second = seconds % 60; + + if (0..10000).contains(&year) { + let buffer = [ + b'0' + (year / 1000 % 10) as u8, + b'0' + (year / 100 % 10) as u8, + b'0' + (year / 10 % 10) as u8, + b'0' + (year % 10) as u8, + b'-', + b'0' + (month / 10) as u8, + b'0' + (month % 10) as u8, + b'-', + b'0' + (day / 10) as u8, + b'0' + (day % 10) as u8, + b'T', + b'0' + (hour / 10) as u8, + b'0' + (hour % 10) as u8, + b':', + b'0' + (minute / 10) as u8, + b'0' + (minute % 10) as u8, + b':', + b'0' + (second / 10) as u8, + b'0' + (second % 10) as u8, + b'Z', + ]; + f.write_str(str::from_utf8(&buffer).map_err(|_| fmt::Error)?) + } else { + write!( + f, + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z" + ) + } + } +} + +impl Response { + pub fn new(code: u16, e0: u8, e1: u8, e2: u8, message: T) -> Self { + Self { + code, + esc: [e0, e1, e2], + message, + } + } + + /// Returns the reply's numeric status. + pub fn code(&self) -> u16 { + self.code + } + + /// Returns the message included in the reply. + pub fn message(&self) -> &T { + &self.message + } + + /// Returns the status severity (first digit of the status code). + pub fn severity(&self) -> Severity { + match self.code { + 200..=299 => Severity::PositiveCompletion, + 300..=399 => Severity::PositiveIntermediate, + 400..=499 => Severity::TransientNegativeCompletion, + 500..=599 => Severity::PermanentNegativeCompletion, + _ => Severity::Invalid, + } + } + + /// Returns the status category (second digit of the status code). + pub fn category(&self) -> Category { + match (self.code / 10) % 10 { + 0 => Category::Syntax, + 1 => Category::Information, + 2 => Category::Connections, + 3 => Category::Unspecified3, + 4 => Category::Unspecified4, + 5 => Category::MailSystem, + _ => Category::Invalid, + } + } + + /// Returns the status details (third digit of the status code). + pub fn details(&self) -> u16 { + self.code % 10 + } + + /// Returns `true` if the reply is a positive completion. + pub fn is_positive_completion(&self) -> bool { + self.severity() == Severity::PositiveCompletion + } + + pub fn explain_class_code(&self) -> Option<(&'static str, &'static str)> { + match self.esc[0] { + 2 => ( + "Success", + concat!( + "Success specifies that the DSN is reporting a posi", + "tive delivery action. Detail sub-codes may provide", + " notification of transformations required for deli", + "very." + ), + ) + .into(), + 4 => ( + "Persistent Transient Failure", + concat!( + "A persistent transient failure is one in which the", + " message as sent is valid, but persistence of some", + " temporary condition has caused abandonment or del", + "ay of attempts to send the message. If this code a", + "ccompanies a delivery failure report, sending in t", + "he future may be successful." + ), + ) + .into(), + 5 => ( + "Permanent Failure", + concat!( + "A permanent failure is one which is not likely to ", + "be resolved by resending the message in the curren", + "t form. Some change to the message or the destinat", + "ion must be made for successful delivery." + ), + ) + .into(), + _ => None, + } + } + + pub fn explain_subject_code(&self) -> Option<(&'static str, &'static str)> { + match self.esc[1] { + 0 => ( + "Other or Undefined Status", + concat!("There is no additional subject information availab", "le."), + ) + .into(), + 1 => ( + "Addressing Status", + concat!( + "The address status reports on the originator or de", + "stination address. It may include address syntax o", + "r validity. These errors can generally be correcte", + "d by the sender and retried." + ), + ) + .into(), + 2 => ( + "Mailbox Status", + concat!( + "Mailbox status indicates that something having to ", + "do with the mailbox has caused this DSN. Mailbox i", + "ssues are assumed to be under the general control ", + "of the recipient." + ), + ) + .into(), + 3 => ( + "Mail System Status", + concat!( + "Mail system status indicates that something having", + " to do with the destination system has caused this", + " DSN. System issues are assumed to be under the ge", + "neral control of the destination system administra", + "tor." + ), + ) + .into(), + 4 => ( + "Network and Routing Status", + concat!( + "The networking or routing codes report status abou", + "t the delivery system itself. These system compone", + "nts include any necessary infrastructure such as d", + "irectory and routing services. Network issues are ", + "assumed to be under the control of the destination", + " or intermediate system administrator." + ), + ) + .into(), + 5 => ( + "Mail Delivery Protocol Status", + concat!( + "The mail delivery protocol status codes report fai", + "lures involving the message delivery protocol. The", + "se failures include the full range of problems res", + "ulting from implementation errors or an unreliable", + " connection." + ), + ) + .into(), + 6 => ( + "Message Content or Media Status", + concat!( + "The message content or media status codes report f", + "ailures involving the content of the message. Thes", + "e codes report failures due to translation, transc", + "oding, or otherwise unsupported message media. Mes", + "sage content or media issues are under the control", + " of both the sender and the receiver, both of whic", + "h must support a common set of supported content-t", + "ypes." + ), + ) + .into(), + 7 => ( + "Security or Policy Status", + concat!( + "The security or policy status codes report failure", + "s involving policies such as per-recipient or per-", + "host filtering and cryptographic operations. Secur", + "ity and policy status issues are assumed to be und", + "er the control of either or both the sender and re", + "cipient. Both the sender and recipient must permit", + " the exchange of messages and arrange the exchange", + " of necessary keys and certificates for cryptograp", + "hic operations." + ), + ) + .into(), + + _ => None, + } + } + + pub fn explain_status_code(&self) -> Option<(&'static str, &'static str)> { + match (self.esc[1], self.esc[2]) { + (0, 0) => ( + "Other undefined Status", + concat!( + "Other undefined status is the only undefined error", + " code." + ), + ) + .into(), + (1, 0) => ( + "Other address status", + concat!( + "Something about the address specified in the messa", + "ge caused this DSN." + ), + ) + .into(), + (1, 1) => ( + "Bad destination mailbox address", + concat!( + "The mailbox specified in the address does not exis", + "t. For Internet mail names, this means the address", + " portion to the left of the \"@\" sign is invalid.", + " This code is only useful for permanent failures." + ), + ) + .into(), + (1, 2) => ( + "Bad destination system address", + concat!( + "The destination system specified in the address do", + "es not exist or is incapable of accepting mail. Fo", + "r Internet mail names, this means the address port", + "ion to the right of the \"@\" is invalid for mail.", + " This code is only useful for permanent failures." + ), + ) + .into(), + (1, 3) => ( + "Bad destination mailbox address syntax", + concat!( + "The destination address was syntactically invalid.", + " This can apply to any field in the address. This ", + "code is only useful for permanent failures." + ), + ) + .into(), + (1, 4) => ( + "Destination mailbox address ambiguous", + concat!( + "The mailbox address as specified matches one or mo", + "re recipients on the destination system. This may ", + "result if a heuristic address mapping algorithm is", + " used to map the specified address to a local mail", + "box name." + ), + ) + .into(), + (1, 5) => ( + "Destination address valid", + concat!( + "This mailbox address as specified was valid. This ", + "status code should be used for positive delivery r", + "eports." + ), + ) + .into(), + (1, 6) => ( + "Destination mailbox has moved, No forwarding address", + concat!( + "The mailbox address provided was at one time valid", + ", but mail is no longer being accepted for that ad", + "dress. This code is only useful for permanent fail", + "ures." + ), + ) + .into(), + (1, 7) => ( + "Bad sender's mailbox address syntax", + concat!( + "The sender's address was syntactically invalid. Th", + "is can apply to any field in the address." + ), + ) + .into(), + (1, 8) => ( + "Bad sender's system address", + concat!( + "The sender's system specified in the address does ", + "not exist or is incapable of accepting return mail", + ". For domain names, this means the address portion", + " to the right of the \"@\" is invalid for mail." + ), + ) + .into(), + (1, 9) => ( + "Message relayed to non-compliant mailer", + concat!( + "The mailbox address specified was valid, but the m", + "essage has been relayed to a system that does not ", + "speak this protocol; no further information can be", + " provided." + ), + ) + .into(), + (1, 10) => ( + "Recipient address has null MX", + concat!( + "This status code is returned when the associated a", + "ddress is marked as invalid using a null MX." + ), + ) + .into(), + (2, 0) => ( + "Other or undefined mailbox status", + concat!( + "The mailbox exists, but something about the destin", + "ation mailbox has caused the sending of this DSN." + ), + ) + .into(), + (2, 1) => ( + "Mailbox disabled, not accepting messages", + concat!( + "The mailbox exists, but is not accepting messages.", + " This may be a permanent error if the mailbox will", + " never be re-enabled or a transient error if the m", + "ailbox is only temporarily disabled." + ), + ) + .into(), + (2, 2) => ( + "Mailbox full", + concat!( + "The mailbox is full because the user has exceeded ", + "a per-mailbox administrative quota or physical cap", + "acity. The general semantics implies that the reci", + "pient can delete messages to make more space avail", + "able. This code should be used as a persistent tra", + "nsient failure." + ), + ) + .into(), + (2, 3) => ( + "Message length exceeds administrative limit", + concat!( + "A per-mailbox administrative message length limit ", + "has been exceeded. This status code should be used", + " when the per-mailbox message length limit is less", + " than the general system limit. This code should b", + "e used as a permanent failure." + ), + ) + .into(), + (2, 4) => ( + "Mailing list expansion problem", + concat!( + "The mailbox is a mailing list address and the mail", + "ing list was unable to be expanded. This code may ", + "represent a permanent failure or a persistent tran", + "sient failure." + ), + ) + .into(), + (3, 0) => ( + "Other or undefined mail system status", + concat!( + "The destination system exists and normally accepts", + " mail, but something about the system has caused t", + "he generation of this DSN." + ), + ) + .into(), + (3, 1) => ( + "Mail system full", + concat!( + "Mail system storage has been exceeded. The general", + " semantics imply that the individual recipient may", + " not be able to delete material to make room for a", + "dditional messages." + ), + ) + .into(), + (3, 2) => ( + "System not accepting network messages", + concat!( + "The host on which the mailbox is resident is not a", + "ccepting messages. Examples of such conditions inc", + "lude an imminent shutdown, excessive load, or syst", + "em maintenance." + ), + ) + .into(), + (3, 3) => ( + "System not capable of selected features", + concat!( + "Selected features specified for the message are no", + "t supported by the destination system. This can oc", + "cur in gateways when features from one domain cann", + "ot be mapped onto the supported feature in another", + "." + ), + ) + .into(), + (3, 4) => ( + "Message too big for system", + concat!( + "The message is larger than per-message size limit.", + " This limit may either be for physical or administ", + "rative reasons." + ), + ) + .into(), + (3, 5) => ( + "System incorrectly configured", + concat!( + "The system is not configured in a manner that will", + " permit it to accept this message." + ), + ) + .into(), + (3, 6) => ( + "Requested priority was changed", + concat!( + "The message was accepted for relay/delivery, but t", + "he requested priority (possibly the implied defaul", + "t) was not honoured. The human readable text after", + " the status code contains the new priority, follow", + "ed by SP (space) and explanatory human readable te", + "xt." + ), + ) + .into(), + (4, 0) => ( + "Other or undefined network or routing status", + concat!( + "Something went wrong with the networking, but it i", + "s not clear what the problem is, or the problem ca", + "nnot be well expressed with any of the other provi", + "ded detail codes." + ), + ) + .into(), + (4, 1) => ( + "No answer from host", + concat!( + "The outbound connection attempt was not answered, ", + "because either the remote system was busy, or was ", + "unable to take a call." + ), + ) + .into(), + (4, 2) => ( + "Bad connection", + concat!( + "The outbound connection was established, but was u", + "nable to complete the message transaction, either ", + "because of time-out, or inadequate connection qual", + "ity." + ), + ) + .into(), + (4, 3) => ( + "Directory server failure", + concat!( + "The network system was unable to forward the messa", + "ge, because a directory server was unavailable. Th", + "e inability to connect to an Internet DNS server i", + "s one example of the directory server failure erro", + "r." + ), + ) + .into(), + (4, 4) => ( + "Unable to route", + concat!( + "The mail system was unable to determine the next h", + "op for the message because the necessary routing i", + "nformation was unavailable from the directory serv", + "er. A DNS lookup returning only an SOA (Start of A", + "dministration) record for a domain name is one exa", + "mple of the unable to route error." + ), + ) + .into(), + (4, 5) => ( + "Mail system congestion", + concat!( + "The mail system was unable to deliver the message ", + "because the mail system was congested." + ), + ) + .into(), + (4, 6) => ( + "Routing loop detected", + concat!( + "A routing loop caused the message to be forwarded ", + "too many times, either because of incorrect routin", + "g tables or a user- forwarding loop." + ), + ) + .into(), + (4, 7) => ( + "Delivery time expired", + concat!( + "The message was considered too old by the rejectin", + "g system, either because it remained on that host ", + "too long or because the time-to-live value specifi", + "ed by the sender of the message was exceeded. If p", + "ossible, the code for the actual problem found whe", + "n delivery was attempted should be returned rather", + " than this code." + ), + ) + .into(), + (5, 0) => ( + "Other or undefined protocol status", + concat!( + "Something was wrong with the protocol necessary to", + " deliver the message to the next hop and the probl", + "em cannot be well expressed with any of the other ", + "provided detail codes." + ), + ) + .into(), + (5, 1) => ( + "Invalid command", + concat!( + "A mail transaction protocol command was issued whi", + "ch was either out of sequence or unsupported." + ), + ) + .into(), + (5, 2) => ( + "Syntax error", + concat!( + "A mail transaction protocol command was issued whi", + "ch could not be interpreted, either because the sy", + "ntax was wrong or the command is unrecognized." + ), + ) + .into(), + (5, 3) => ( + "Too many recipients", + concat!( + "More recipients were specified for the message tha", + "n could have been delivered by the protocol. This ", + "error should normally result in the segmentation o", + "f the message into two, the remainder of the recip", + "ients to be delivered on a subsequent delivery att", + "empt. It is included in this list in the event tha", + "t such segmentation is not possible." + ), + ) + .into(), + (5, 4) => ( + "Invalid command arguments", + concat!( + "A valid mail transaction protocol command was issu", + "ed with invalid arguments, either because the argu", + "ments were out of range or represented unrecognize", + "d features." + ), + ) + .into(), + (5, 5) => ( + "Wrong protocol version", + concat!( + "A protocol version mis-match existed which could n", + "ot be automatically resolved by the communicating ", + "parties." + ), + ) + .into(), + (5, 6) => ( + "Authentication Exchange line is too long", + concat!( + "This enhanced status code SHOULD be returned when ", + "the server fails the AUTH command due to the clien", + "t sending a [BASE64] response which is longer than", + " the maximum buffer size available for the current", + "ly selected SASL mechanism." + ), + ) + .into(), + (6, 0) => ( + "Other or undefined media error", + concat!( + "Something about the content of a message caused it", + " to be considered undeliverable and the problem ca", + "nnot be well expressed with any of the other provi", + "ded detail codes." + ), + ) + .into(), + (6, 1) => ( + "Media not supported", + concat!( + "The media of the message is not supported by eithe", + "r the delivery protocol or the next system in the ", + "forwarding path." + ), + ) + .into(), + (6, 2) => ( + "Conversion required and prohibited", + concat!( + "The content of the message must be converted befor", + "e it can be delivered and such conversion is not p", + "ermitted. Such prohibitions may be the expression ", + "of the sender in the message itself or the policy ", + "of the sending host." + ), + ) + .into(), + (6, 3) => ( + "Conversion required but not supported", + concat!( + "The message content must be converted in order to ", + "be forwarded but such conversion is not possible o", + "r is not practical by a host in the forwarding pat", + "h. This condition may result when an ESMTP gateway", + " supports 8bit transport but is not able to downgr", + "ade the message to 7 bit as required for the next ", + "hop." + ), + ) + .into(), + (6, 4) => ( + "Conversion with loss performed", + concat!( + "This is a warning sent to the sender when message ", + "delivery was successfully but when the delivery re", + "quired a conversion in which some data was lost. T", + "his may also be a permanent error if the sender ha", + "s indicated that conversion with loss is prohibite", + "d for the message." + ), + ) + .into(), + (6, 5) => ( + "Conversion Failed", + concat!( + "A conversion was required but was unsuccessful. Th", + "is may be useful as a permanent or persistent temp", + "orary notification." + ), + ) + .into(), + (6, 6) => ( + "Message content not available", + concat!( + "The message content could not be fetched from a re", + "mote system. This may be useful as a permanent or ", + "persistent temporary notification." + ), + ) + .into(), + (6, 7) => ( + "Non-ASCII addresses not permitted for that sender/recipient", + concat!( + "This indicates the reception of a MAIL or RCPT com", + "mand that non-ASCII addresses are not permitted" + ), + ) + .into(), + (6, 8 | 10) => ( + "UTF-8 string reply is required, but not permitted by the SMTP client", + concat!( + "This indicates that a reply containing a UTF-8 str", + "ing is required to show the mailbox name, but that", + " form of response is not permitted by the SMTP cli", + "ent." + ), + ) + .into(), + (6, 9) => ( + concat!( + "UTF-8 header message cannot be transferred to ", + "one or more recipients, so the message must be rejected" + ), + concat!( + "This indicates that transaction failed after the f", + "inal \".\" of the DATA command." + ), + ) + .into(), + (7, 0) => ( + "Other or undefined security status", + concat!( + "Something related to security caused the message t", + "o be returned, and the problem cannot be well expr", + "essed with any of the other provided detail codes.", + " This status code may also be used when the condit", + "ion cannot be further described because of securit", + "y policies in force." + ), + ) + .into(), + (7, 1) => ( + "Delivery not authorized, message refused", + concat!( + "The sender is not authorized to send to the destin", + "ation. This can be the result of per-host or per-r", + "ecipient filtering. This memo does not discuss the", + " merits of any such filtering, but provides a mech", + "anism to report such." + ), + ) + .into(), + (7, 2) => ( + "Mailing list expansion prohibited", + concat!( + "The sender is not authorized to send a message to ", + "the intended mailing list." + ), + ) + .into(), + (7, 3) => ( + "Security conversion required but not possible", + concat!( + "A conversion from one secure messaging protocol to", + " another was required for delivery and such conver", + "sion was not possible." + ), + ) + .into(), + (7, 4) => ( + "Security features not supported", + concat!( + "A message contained security features such as secu", + "re authentication that could not be supported on t", + "he delivery protocol." + ), + ) + .into(), + (7, 5) => ( + "Cryptographic failure", + concat!( + "A transport system otherwise authorized to validat", + "e or decrypt a message in transport was unable to ", + "do so because necessary information such as key wa", + "s not available or such information was invalid." + ), + ) + .into(), + (7, 6) => ( + "Cryptographic algorithm not supported", + concat!( + "A transport system otherwise authorized to validat", + "e or decrypt a message was unable to do so because", + " the necessary algorithm was not supported." + ), + ) + .into(), + (7, 7) => ( + "Message integrity failure", + concat!( + "A transport system otherwise authorized to validat", + "e a message was unable to do so because the messag", + "e was corrupted or altered. This may be useful as ", + "a permanent, transient persistent, or successful d", + "elivery code." + ), + ) + .into(), + (7, 8) => ( + "Authentication credentials invalid", + concat!( + "This response to the AUTH command indicates that t", + "he authentication failed due to invalid or insuffi", + "cient authentication credentials. In this case, th", + "e client SHOULD ask the user to supply new credent", + "ials (such as by presenting a password dialog box)", + "." + ), + ) + .into(), + (7, 9) => ( + "Authentication mechanism is too weak", + concat!( + "This response to the AUTH command indicates that t", + "he selected authentication mechanism is weaker tha", + "n server policy permits for that user. The client ", + "SHOULD retry with a new authentication mechanism." + ), + ) + .into(), + (7, 10) => ( + "Encryption Needed", + concat!( + "This indicates that external strong privacy layer ", + "is needed in order to use the requested authentica", + "tion mechanism. This is primarily intended for use", + " with clear text authentication mechanisms. A clie", + "nt which receives this may activate a security lay", + "er such as TLS prior to authenticating, or attempt", + " to use a stronger mechanism." + ), + ) + .into(), + (7, 11) => ( + "Encryption required for requested authentication mechanism", + concat!( + "This response to the AUTH command indicates that t", + "he selected authentication mechanism may only be u", + "sed when the underlying SMTP connection is encrypt", + "ed. Note that this response code is documented her", + "e for historical purposes only. Modern implementat", + "ions SHOULD NOT advertise mechanisms that are not ", + "permitted due to lack of encryption, unless an enc", + "ryption layer of sufficient strength is currently ", + "being employed." + ), + ) + .into(), + (7, 12) => ( + "A password transition is needed", + concat!( + "This response to the AUTH command indicates that t", + "he user needs to transition to the selected authen", + "tication mechanism. This is typically done by auth", + "enticating once using the [PLAIN] authentication m", + "echanism. The selected mechanism SHOULD then work ", + "for authentications in subsequent sessions." + ), + ) + .into(), + (7, 13) => ( + "User Account Disabled", + concat!( + "Sometimes a system administrator will have to disa", + "ble a user's account (e.g., due to lack of payment", + ", abuse, evidence of a break-in attempt, etc). Thi", + "s error code occurs after a successful authenticat", + "ion to a disabled account. This informs the client", + " that the failure is permanent until the user cont", + "acts their system administrator to get the account", + " re-enabled. It differs from a generic authenticat", + "ion failure where the client's best option is to p", + "resent the passphrase entry dialog in case the use", + "r simply mistyped their passphrase." + ), + ) + .into(), + (7, 14) => ( + "Trust relationship required", + concat!( + "The submission server requires a configured trust ", + "relationship with a third-party server in order to", + " access the message content. This value replaces t", + "he prior use of X.7.8 for this error condition. th", + "ereby updating [RFC4468]." + ), + ) + .into(), + (7, 15) => ( + "Priority Level is too low", + concat!( + "The specified priority level is below the lowest p", + "riority acceptable for the receiving SMTP server. ", + "This condition might be temporary, for example the", + " server is operating in a mode where only higher p", + "riority messages are accepted for transfer and del", + "ivery, while lower priority messages are rejected." + ), + ) + .into(), + (7, 16) => ( + "Message is too big for the specified priority", + concat!( + "The message is too big for the specified priority.", + " This condition might be temporary, for example th", + "e server is operating in a mode where only higher ", + "priority messages below certain size are accepted ", + "for transfer and delivery." + ), + ) + .into(), + (7, 17) => ( + "Mailbox owner has changed", + concat!( + "This status code is returned when a message is rec", + "eived with a Require-Recipient-Valid-Since field o", + "r RRVS extension and the receiving system is able ", + "to determine that the intended recipient mailbox h", + "as not been under continuous ownership since the s", + "pecified date-time." + ), + ) + .into(), + (7, 18) => ( + "Domain owner has changed", + concat!( + "This status code is returned when a message is rec", + "eived with a Require-Recipient-Valid-Since field o", + "r RRVS extension and the receiving system wishes t", + "o disclose that the owner of the domain name of th", + "e recipient has changed since the specified date-t", + "ime." + ), + ) + .into(), + (7, 19) => ( + "RRVS test cannot be completed", + concat!( + "This status code is returned when a message is rec", + "eived with a Require-Recipient-Valid-Since field o", + "r RRVS extension and the receiving system cannot c", + "omplete the requested evaluation because the requi", + "red timestamp was not recorded. The message origin", + "ator needs to decide whether to reissue the messag", + "e without RRVS protection." + ), + ) + .into(), + (7, 20) => ( + "No passing DKIM signature found", + concat!( + "This status code is returned when a message did no", + "t contain any passing DKIM signatures. (This viola", + "tes the advice of Section 6.1 of [RFC6376].)" + ), + ) + .into(), + (7, 21) => ( + "No acceptable DKIM signature found", + concat!( + "This status code is returned when a message contai", + "ns one or more passing DKIM signatures, but none a", + "re acceptable. (This violates the advice of Sectio", + "n 6.1 of [RFC6376].)" + ), + ) + .into(), + (7, 22) => ( + "No valid author-matched DKIM signature found", + concat!( + "This status code is returned when a message contai", + "ns one or more passing DKIM signatures, but none a", + "re acceptable because none have an identifier(s) t", + "hat matches the author address(es) found in the Fr", + "om header field. This is a special case of X.7.21.", + " (This violates the advice of Section 6.1 of [RFC6", + "376].)" + ), + ) + .into(), + (7, 23) => ( + "SPF validation failed", + concat!( + "This status code is returned when a message comple", + "ted an SPF check that produced a \"fail\" result, ", + "contrary to local policy requirements. Used in pla", + "ce of 5.7.1 as described in Section 8.4 of [RFC720", + "8]." + ), + ) + .into(), + (7, 24) => ( + "SPF validation error", + concat!( + "This status code is returned when evaluation of SP", + "F relative to an arriving message resulted in an e", + "rror. Used in place of 4.4.3 or 5.5.2 as described", + " in Sections 8.6 and 8.7 of [RFC7208]." + ), + ) + .into(), + (7, 25) => ( + "Reverse DNS validation failed", + concat!( + "This status code is returned when an SMTP client's", + " IP address failed a reverse DNS validation check,", + " contrary to local policy requirements." + ), + ) + .into(), + (7, 26) => ( + "Multiple authentication checks failed", + concat!( + "This status code is returned when a message failed", + " more than one message authentication check, contr", + "ary to local policy requirements. The particular m", + "echanisms that failed are not specified." + ), + ) + .into(), + (7, 27) => ( + "Sender address has null MX", + concat!( + "This status code is returned when the associated s", + "ender address has a null MX, and the SMTP receiver", + " is configured to reject mail from such sender (e.", + "g., because it could not return a DSN)." + ), + ) + .into(), + (7, 28) => ( + "Mail flood detected", + concat!( + "The message appears to be part of a mail flood of ", + "similar abusive messages." + ), + ) + .into(), + (7, 29) => ( + "ARC validation failure", + concat!( + "This status code may be returned when a message fa", + "ils ARC validation." + ), + ) + .into(), + (7, 30) => ( + "REQUIRETLS support required", + concat!( + "This indicates that the message was not able to be", + " forwarded because it was received with a REQUIRET", + "LS requirement and none of the SMTP servers to whi", + "ch the message should be forwarded provide this su", + "pport." + ), + ) + .into(), + _ => None, + } + } +} + +fn write_decimal(mut writer: impl Write, value: u64) -> io::Result<()> { + let mut buffer = [0u8; 20]; + let mut remaining = value; + let mut len = 0; + + for slot in buffer.iter_mut().rev() { + *slot = b'0' + (remaining % 10) as u8; + remaining /= 10; + len += 1; + if remaining == 0 { + break; + } + } + + let (_, digits) = buffer.split_at(buffer.len() - len); + writer.write_all(digits) +} + diff --git a/crates/smtp-proto/src/response/mod.rs b/crates/smtp-proto/src/response/mod.rs new file mode 100644 index 00000000..8fad014d --- /dev/null +++ b/crates/smtp-proto/src/response/mod.rs @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Display; + +use crate::tokens::define_tokens_128; +use crate::{EhloResponse, Response}; + +pub mod generate; +pub mod parser; + +define_tokens_128! { + _8BITMIME = "8BITMIME", + ATRN, + AUTH, + BINARYMIME, + BURL, + CHECKPOINT, + CHUNKING, + CONNEG, + CONPERM, + DELIVERBY, + DSN, + ENHANCEDSTATUSCO, + ETRN, + EXPN, + VRFY, + FUTURERELEASE, + HELP, + MT_PRIORITY = "MT-PRIORITY", + MTRK, + NO_SOLICITING = "NO-SOLICITING", + ONEX, + PIPELINING, + REQUIRETLS, + RRVS, + SIZE, + SMTPUTF8, + STARTTLS, + VERB, + + // Priorities + MIXER, + STANAG4406, + NSEP, +} + +impl EhloResponse { + /// Returns the hostname of the SMTP server. + pub fn hostname(&self) -> &T { + &self.hostname + } + + /// Returns the capabilities of the SMTP server. + pub fn capabilities(&self) -> u32 { + self.capabilities + } + + /// Returns `true` if the SMTP server supports a given extension. + pub fn has_capability(&self, capability: u32) -> bool { + (self.capabilities & capability) != 0 + } + + /// Returns all supported authentication mechanisms. + pub fn auth(&self) -> u64 { + self.auth_mechanisms + } +} + +impl Display for Response { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Code: {}, Enhanced code: {}.{}.{}, Message: {}", + self.code, self.esc[0], self.esc[1], self.esc[2], self.message, + ) + } +} + +#[cfg(feature = "rkyv")] +impl Display for crate::ArchivedResponse +where + ::Archived: std::fmt::Display, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Code: {}, Enhanced code: {}.{}.{}, Message: {}", + self.code, self.esc[0], self.esc[1], self.esc[2], self.message, + ) + } +} diff --git a/crates/smtp-proto/src/response/parser.rs b/crates/smtp-proto/src/response/parser.rs new file mode 100644 index 00000000..20dd38b1 --- /dev/null +++ b/crates/smtp-proto/src/response/parser.rs @@ -0,0 +1,468 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::{request::parser::Rfc5321Parser, *}; +use memchr::memchr2; +use std::slice::Iter; + +pub const MAX_RESPONSE_LENGTH: usize = 4096; +const MULTILINE_RESERVE: usize = 128; +const CONTINUATION_RESERVE: usize = 64; + +#[derive(Default)] +pub struct ResponseReceiver { + buf: Vec, + code: u16, + esc: [u8; 3], + esc_cur: [u8; 3], + esc_restore: [u8; 10], + esc_pos: usize, + is_last: bool, + pos: usize, +} + +impl ResponseReceiver { + pub fn from_code(code: u16) -> Self { + Self { + code, + esc: [0u8; 3], + esc_cur: [0u8; 3], + esc_restore: [0u8; 10], + esc_pos: 0, + pos: 3, + is_last: false, + buf: Vec::new(), + } + } + + pub fn parse(&mut self, bytes: &mut Iter<'_, u8>) -> Result, Error> { + let mut rest = bytes.as_slice(); + let result = loop { + if self.pos > 6 { + let end = line_end(rest); + if end != 0 { + let available = MAX_RESPONSE_LENGTH.saturating_sub(self.buf.len()); + if end <= available { + let (run, tail) = rest.split_at(end); + self.buf.extend_from_slice(run); + rest = tail; + } else { + let (run, tail) = rest.split_at(available); + self.buf.extend_from_slice(run); + rest = tail.get(1..).unwrap_or_default(); + break Err(Error::ResponseTooLong); + } + } + } else if self.pos == 0 + && let [d0, d1, d2, sep, tail @ ..] = rest + && d0.is_ascii_digit() + && d1.is_ascii_digit() + && d2.is_ascii_digit() + && matches!(*sep, b' ' | b'-') + { + if self.buf.is_empty() { + self.code = self + .code + .saturating_mul(10) + .saturating_add((d0 - b'0') as u16) + .saturating_mul(10) + .saturating_add((d1 - b'0') as u16) + .saturating_mul(10) + .saturating_add((d2 - b'0') as u16); + if *sep == b'-' { + self.buf.reserve(MULTILINE_RESERVE); + } + } + self.is_last |= *sep == b' '; + self.pos = 4; + rest = tail; + continue; + } else if self.pos == 4 && self.esc_pos == 0 { + if let Some(ch) = rest.first() + && !matches!(ch, b'0'..=b'9' | b'.' | b'\r' | b'\n') + && self.buf.len() < MAX_RESPONSE_LENGTH + { + self.pos = 7; + continue; + } + if let [class, b'.', subject, b'.', detail, separator, tail @ ..] = rest + && class.is_ascii_digit() + && subject.is_ascii_digit() + && detail.is_ascii_digit() + && matches!(*separator, b' ' | b'\r') + { + self.esc = [class - b'0', subject - b'0', detail - b'0']; + self.pos = 7; + rest = tail; + continue; + } + } + + let [ch, tail @ ..] = rest else { + break Err(Error::NeedsMoreData { bytes_left: 0 }); + }; + let ch = *ch; + rest = tail; + + match self.pos { + 0..=2 => { + if ch.is_ascii_digit() { + if self.buf.is_empty() { + self.code = self + .code + .saturating_mul(10) + .saturating_add((ch - b'0') as u16); + } + self.pos += 1; + } else { + break Err(Error::SyntaxError { + syntax: "Invalid response code", + }); + } + } + 3 => match ch { + b' ' => { + self.is_last = true; + self.pos += 1; + } + b'-' => { + if self.buf.is_empty() { + self.buf.reserve(MULTILINE_RESERVE); + } + self.pos += 1; + } + b'\r' => (), + b'\n' => { + self.is_last = true; + } + _ => { + break Err(Error::SyntaxError { + syntax: "Invalid response separator", + }); + } + }, + 4..=6 => { + let mut do_restore = false; + + match ch { + b'0'..=b'9' => { + let code = &mut self.esc_cur[self.pos - 4]; + if *code < 100 { + if let Some(restore) = self.esc_restore.get_mut(self.esc_pos) { + *restore = ch; + *code = code.saturating_mul(10).saturating_add(ch - b'0'); + self.esc_pos += 1; + } else { + do_restore = true; + } + } else { + do_restore = true; + } + } + b'.' if self.pos < 6 => { + if let Some(restore) = self.esc_restore.get_mut(self.esc_pos) { + *restore = ch; + self.pos += 1; + self.esc_pos += 1; + } else { + do_restore = true; + } + } + b' ' | b'\r' | b'\n' if self.pos == 6 => { + self.esc = self.esc_cur; + self.esc_cur.fill(0); + self.pos = 7; + self.esc_pos = 0; + } + _ => { + do_restore = true; + } + } + if do_restore { + if self.esc_pos > 0 { + self.buf + .extend_from_slice(&self.esc_restore[..self.esc_pos]); + } + self.buf.push(ch); + self.pos = 7; + self.esc_pos = 0; + self.esc_cur.fill(0); + } + } + _ => (), + } + + if ch == b'\n' { + if self.is_last { + break Ok(Response { + code: self.code, + esc: self.esc, + message: std::mem::take(&mut self.buf).into_string(), + }); + } else { + self.buf.reserve(CONTINUATION_RESERVE); + self.buf.push(b'\n'); + self.pos = 0; + } + } + }; + *bytes = rest.iter(); + result + } + + pub fn reset(&mut self) { + self.is_last = false; + self.code = 0; + self.esc.fill(0); + self.esc_cur.fill(0); + self.pos = 0; + self.esc_pos = 0; + self.buf.clear(); + } +} + +impl EhloResponse { + fn parse_first_line( + rest: &mut &[u8], + bytes_left: usize, + hostname: &mut String, + ) -> Result { + let mut code: u16 = 0; + for _ in 0..3 { + let [ch, tail @ ..] = *rest else { + return Err(Error::NeedsMoreData { bytes_left }); + }; + *rest = tail; + if !ch.is_ascii_digit() { + return Err(Error::SyntaxError { + syntax: "unexpected token", + }); + } + code = code.saturating_mul(10).saturating_add((ch - b'0') as u16); + } + + if code != 250 { + return Err(Error::InvalidResponse { code }); + } + + let [separator, tail @ ..] = *rest else { + return Err(Error::NeedsMoreData { bytes_left }); + }; + *rest = tail; + let eol = match *separator { + b' ' => true, + b'-' => false, + b'\n' => return Ok(true), + _ => { + return Err(Error::SyntaxError { + syntax: "unexpected token", + }); + } + }; + + let mut buf = Vec::new(); + loop { + let end = rest + .iter() + .position(|&ch| matches!(ch, b' ' | b'\r' | b'\n')) + .unwrap_or(rest.len()); + if end > MAX_RESPONSE_LENGTH.saturating_sub(buf.len()) { + *rest = rest + .get(MAX_RESPONSE_LENGTH.saturating_sub(buf.len()) + 1..) + .unwrap_or_default(); + return Err(Error::ResponseTooLong); + } + let (run, tail) = rest.split_at(end); + buf.extend_from_slice(run); + let [ch, tail @ ..] = tail else { + *rest = tail; + return Err(Error::NeedsMoreData { bytes_left }); + }; + *rest = tail; + match *ch { + b'\r' => (), + b'\n' => break, + _ => { + let Some(lf) = rest.iter().position(|&ch| ch == b'\n') else { + *rest = &[]; + return Err(Error::NeedsMoreData { bytes_left }); + }; + *rest = rest.get(lf + 1..).unwrap_or_default(); + break; + } + } + } + + *hostname = buf.into_string(); + Ok(eol) + } + + pub fn parse(bytes: &mut Iter<'_, u8>) -> Result { + let bytes_left = bytes.len(); + let mut response = EhloResponse::default(); + let mut rest = bytes.as_slice(); + let first_line = Self::parse_first_line(&mut rest, bytes_left, &mut response.hostname); + *bytes = rest.iter(); + let mut eol = first_line?; + + let mut parser = Rfc5321Parser::new(bytes); + parser.bytes_left = bytes_left; + + while !eol { + let mut code: u16 = 0; + for _ in 0..3 { + match parser.read_char()? { + ch @ b'0'..=b'9' => { + code = code.saturating_mul(10).saturating_add((ch - b'0') as u16); + } + _ => { + return Err(Error::SyntaxError { + syntax: "unexpected token", + }); + } + } + } + + if code != 250 { + return Err(Error::InvalidResponse { code }); + } + + match parser.read_char()? { + b' ' => { + eol = true; + } + b'-' => (), + b'\n' if code < 600 => { + break; + } + _ => { + return Err(Error::SyntaxError { + syntax: "unexpected token", + }); + } + } + + response.capabilities |= match parser.hashed_value_long()? { + _8BITMIME => EXT_8BIT_MIME, + ATRN => EXT_ATRN, + AUTH => { + while parser.stop_char != LF { + if let Some(mechanism) = parser.mechanism()? { + response.auth_mechanisms |= mechanism; + } + } + + EXT_AUTH + } + BINARYMIME => EXT_BINARY_MIME, + BURL => EXT_BURL, + CHECKPOINT => EXT_CHECKPOINT, + CHUNKING => EXT_CHUNKING, + CONNEG => EXT_CONNEG, + CONPERM => EXT_CONPERM, + DELIVERBY => { + response.deliver_by = if parser.stop_char != LF { + let db = parser.size()?; + if db != usize::MAX { db as u64 } else { 0 } + } else { + 0 + }; + EXT_DELIVER_BY + } + DSN => EXT_DSN, + ENHANCEDSTATUSCO + if parser.stop_char.eq_ignore_ascii_case(&b'D') + && parser.read_char()?.eq_ignore_ascii_case(&b'E') + && parser.read_char()?.eq_ignore_ascii_case(&b'S') => + { + EXT_ENHANCED_STATUS_CODES + } + ETRN => EXT_ETRN, + EXPN => EXT_EXPN, + VRFY => EXT_VRFY, + FUTURERELEASE => { + let max_interval = if parser.stop_char != LF { + parser.size()? + } else { + 0 + }; + let max_datetime = if parser.stop_char != LF { + parser.timestamp()? + } else { + 0 + }; + + response.future_release_interval = if max_interval != usize::MAX { + max_interval as u64 + } else { + 0 + }; + response.future_release_datetime = if max_datetime != i64::MAX { + max_datetime.max(0) as u64 + } else { + 0 + }; + EXT_FUTURE_RELEASE + } + HELP => EXT_HELP, + MT_PRIORITY => { + response.mt_priority = if parser.stop_char != LF { + match parser.hashed_value_long()? { + MIXER => MtPriority::Mixer, + STANAG4406 => MtPriority::Stanag4406, + NSEP => MtPriority::Nsep, + _ => MtPriority::Mixer, + } + } else { + MtPriority::Mixer + }; + EXT_MT_PRIORITY + } + MTRK => EXT_MTRK, + NO_SOLICITING => { + response.no_soliciting = if parser.stop_char != LF { + let text = parser.text()?; + if !text.is_empty() { + text.into_owned().into() + } else { + None + } + } else { + None + }; + EXT_NO_SOLICITING + } + ONEX => EXT_ONEX, + PIPELINING => EXT_PIPELINING, + REQUIRETLS => EXT_REQUIRE_TLS, + RRVS => EXT_RRVS, + SIZE => { + response.size = if parser.stop_char != LF { + let size = parser.size()?; + if size != usize::MAX { size } else { 0 } + } else { + 0 + }; + EXT_SIZE + } + SMTPUTF8 => EXT_SMTP_UTF8, + STARTTLS => EXT_START_TLS, + VERB => EXT_VERB, + _ => 0, + }; + parser.seek_lf()?; + } + + Ok(response) + } +} + +#[inline(always)] +fn line_end(bytes: &[u8]) -> usize { + memchr2(b'\r', b'\n', bytes).unwrap_or(bytes.len()) +} + diff --git a/crates/smtp-proto/src/tokens.rs b/crates/smtp-proto/src/tokens.rs new file mode 100644 index 00000000..2359a84e --- /dev/null +++ b/crates/smtp-proto/src/tokens.rs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const fn str_to_array(s: &str) -> [u8; N] { + let s = s.as_bytes(); + let mut arr = [0; N]; + let mut i = 0; + while i < s.len() { + arr[i] = s[i]; + i += 1; + } + arr +} + +pub(crate) const fn token64(s: &str) -> u64 { + u64::from_le_bytes(str_to_array(s)) +} + +pub(crate) const fn token128(s: &str) -> u128 { + u128::from_le_bytes(str_to_array(s)) +} + +macro_rules! first { + ($head:expr$(, $tail:expr)*) => { + $head + }; +} + +macro_rules! define_tokens_64 { + ($($a:ident $(= $b:expr)?,)*) => { + $(pub(crate) const $a: u64 = $crate::tokens::token64($crate::tokens::first!($($b,)* stringify!($a)));)* + }; +} + +macro_rules! define_tokens_128 { + ($($a:ident $(= $b:expr)?,)*) => { + $(pub(crate) const $a: u128 = $crate::tokens::token128($crate::tokens::first!($($b,)* stringify!($a)));)* + }; +} + +pub(crate) use define_tokens_64; +pub(crate) use define_tokens_128; +pub(crate) use first; diff --git a/crates/smtp-report/Cargo.toml b/crates/smtp-report/Cargo.toml new file mode 100644 index 00000000..2a01223c --- /dev/null +++ b/crates/smtp-report/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "smtp-report" +description = "DMARC aggregate, ARF and TLS-RPT report generation and parsing" +version = "1.0.0" +edition = "2024" +license = "AGPL-3.0-only OR LicenseRef-SEL" +publish = false + +[lib] +name = "smtp_report" +doctest = false + +[dependencies] +mail-auth = { version = "0.14", features = ["arc"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +flate2 = "1.1" +quick-xml = "0.42" +zip = { version = "9.1", default-features = false, features = ["deflate-flate2"] } +hashify = "0.3" +memchr = "2.8" +simdutf8 = "0.1.5" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[lints] +workspace = true diff --git a/crates/smtp-report/src/arf/builder.rs b/crates/smtp-report/src/arf/builder.rs new file mode 100644 index 00000000..cfefd206 --- /dev/null +++ b/crates/smtp-report/src/arf/builder.rs @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Construction and ownership helpers for [`FeedbackReport`]. + +use super::{FeedbackReport, FeedbackType}; + +impl<'x> FeedbackReport<'x> { + /// Returns an empty report of the given type with `version` and + /// `incidents` set to 1. + pub fn new(feedback_type: FeedbackType) -> Self { + FeedbackReport { + feedback_type, + version: 1, + incidents: 1, + ..Default::default() + } + } + + /// Converts every borrowed string into an owned one, detaching the report + /// from the buffer it was parsed from. + pub fn into_owned<'y>(self) -> FeedbackReport<'y> { + FeedbackReport { + feedback_type: self.feedback_type, + arrival_date: self.arrival_date, + authentication_results: self + .authentication_results + .into_iter() + .map(|ar| ar.into_owned().into()) + .collect(), + incidents: self.incidents, + original_envelope_id: self.original_envelope_id.map(|v| v.into_owned().into()), + original_mail_from: self.original_mail_from.map(|v| v.into_owned().into()), + original_rcpt_to: self.original_rcpt_to.map(|v| v.into_owned().into()), + reported_domains: self + .reported_domains + .into_iter() + .map(|ar| ar.into_owned().into()) + .collect(), + reported_uris: self + .reported_uris + .into_iter() + .map(|ar| ar.into_owned().into()) + .collect(), + reporting_mta: self.reporting_mta.map(|v| v.into_owned().into()), + source_ip: self.source_ip, + user_agent: self.user_agent.map(|v| v.into_owned().into()), + version: self.version, + source_port: self.source_port, + auth_failure: self.auth_failure, + delivery_result: self.delivery_result, + dkim_adsp_dns: self.dkim_adsp_dns.map(|v| v.into_owned().into()), + dkim_canonicalized_body: self.dkim_canonicalized_body.map(|v| v.into_owned().into()), + dkim_canonicalized_header: self + .dkim_canonicalized_header + .map(|v| v.into_owned().into()), + dkim_domain: self.dkim_domain.map(|v| v.into_owned().into()), + dkim_identity: self.dkim_identity.map(|v| v.into_owned().into()), + dkim_selector: self.dkim_selector.map(|v| v.into_owned().into()), + dkim_selector_dns: self.dkim_selector_dns.map(|v| v.into_owned().into()), + spf_dns: self.spf_dns.map(|v| v.into_owned().into()), + identity_alignment: self.identity_alignment, + message: self.message.map(|v| v.into_owned().into()), + headers: self.headers.map(|v| v.into_owned().into()), + } + } +} diff --git a/crates/smtp-report/src/arf/generate.rs b/crates/smtp-report/src/arf/generate.rs new file mode 100644 index 00000000..fb79cedd --- /dev/null +++ b/crates/smtp-report/src/arf/generate.rs @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Serialization of [`FeedbackReport`] to the ARF field format and to a +//! complete feedback report email message. + +use crate::{ + ReportEnvelope, + arf::{AuthFailureType, DeliveryResult, FeedbackReport, FeedbackType, IdentityAlignment}, + utf8::into_string, +}; +use mail_builder::{ + MessageBuilder, + headers::{HeaderType, content_type::ContentType}, + mime::{BodyPart, MimePart, make_boundary}, +}; +use mail_parser::DateTime; +use std::{fmt::Write, io, time::SystemTime}; + +impl<'x> FeedbackReport<'x> { + /// Writes the report as an RFC 5322 `multipart/report` message with + /// `report-type=feedback-report` to `writer`. + /// + /// The parts are a `text/plain` summary, the output of + /// [`to_arf`](Self::to_arf) as `message/feedback-report`, and the original + /// message as `message/rfc822` or, when `message` is `None`, its headers + /// as `text/rfc822-headers`. The summary uses `arrival_date`, or the + /// current time when it is unset. The message carries an + /// `Auto-Submitted: auto-generated` header field. + /// + /// Fields read from `envelope`: + /// + /// - `from`: the `From` header field. + /// - `to`: the `To` header field. + /// - `subject`: the subject; when `None`, defaults to + /// `Authentication Failure Report` for [`FeedbackType::AuthFailure`] + /// and `Abuse Report` otherwise. + /// - `submitter`: the `Message-ID` host, only when `reporting_mta` is + /// `None`. When both are empty, `localhost` is used. + /// + /// `envelope.report_domain` is ignored. + /// + /// # Errors + /// + /// Returns any I/O error raised by `writer`. + pub fn write_rfc5322( + &self, + envelope: &ReportEnvelope<'_>, + writer: impl io::Write, + ) -> io::Result<()> { + let arf = self.to_arf(); + + let mut text_body = String::with_capacity(128); + if self.feedback_type == FeedbackType::AuthFailure { + write!( + &mut text_body, + "This is an authentication failure report for an email message received\r\n" + ) + } else { + write!( + &mut text_body, + "This is an email abuse report for an email message received\r\n" + ) + } + .ok(); + if let Some(ip) = &self.source_ip { + write!(&mut text_body, "from IP address {ip} ").ok(); + } + let dt = DateTime::from_timestamp(if let Some(ad) = &self.arrival_date { + *ad + } else { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) as i64 + }); + write!(&mut text_body, "on {}.\r\n", dt.to_rfc822()).ok(); + + let mut parts = vec![ + MimePart::new( + ContentType::new("text/plain"), + BodyPart::Text(text_body.into()), + ), + MimePart::new( + ContentType::new("message/feedback-report"), + BodyPart::Text(arf.into()), + ), + ]; + if let Some(message) = self.message.as_deref() { + parts.push(MimePart::new( + ContentType::new("message/rfc822"), + BodyPart::Text(message.into()), + )); + } else if let Some(headers) = self.headers.as_deref() { + parts.push(MimePart::new( + ContentType::new("text/rfc822-headers"), + BodyPart::Text(headers.into()), + )); + } + + let host = self + .reporting_mta + .as_deref() + .filter(|host| !host.is_empty()) + .or((!envelope.submitter.is_empty()).then_some(envelope.submitter)) + .unwrap_or("localhost"); + let subject = envelope.subject.unwrap_or(match self.feedback_type { + FeedbackType::AuthFailure => "Authentication Failure Report", + _ => "Abuse Report", + }); + + MessageBuilder::new() + .from(envelope.from.clone()) + .header("To", envelope.to_header()) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .message_id(format!("{}@{}", make_boundary("."), host)) + .subject(subject) + .body(MimePart::new( + ContentType::new("multipart/report").attribute("report-type", "feedback-report"), + BodyPart::Multipart(parts), + )) + .write_to(writer) + } + + /// Returns the report as an RFC 5322 message. + /// + /// Same as [`write_rfc5322`](Self::write_rfc5322), collected into a + /// `String`. + /// + /// # Errors + /// + /// Returns an error if the message is not valid UTF-8. + pub fn to_rfc5322(&self, envelope: &ReportEnvelope<'_>) -> io::Result { + let mut buf = Vec::new(); + self.write_rfc5322(envelope, &mut buf)?; + into_string(buf).map_err(io::Error::other) + } + + /// Serializes the machine-readable part of the report as ARF fields, + /// each terminated by CRLF. + /// + /// Unset optional fields are omitted. The RFC 6591 fields (`Auth-Failure` + /// through `Identity-Alignment`) are written only for + /// [`FeedbackType::AuthFailure`] reports. + pub fn to_arf(&self) -> String { + let mut arf = String::with_capacity(128); + + write!(&mut arf, "Version: {}\r\n", self.version).ok(); + write!( + &mut arf, + "Feedback-Type: {}\r\n", + match self.feedback_type { + FeedbackType::Abuse => "abuse", + FeedbackType::AuthFailure => "auth-failure", + FeedbackType::Fraud => "fraud", + FeedbackType::NotSpam => "not-spam", + FeedbackType::Other => "other", + FeedbackType::Virus => "virus", + } + ) + .ok(); + if let Some(ad) = &self.arrival_date { + let ad = DateTime::from_timestamp(*ad); + write!(&mut arf, "Arrival-Date: {}\r\n", ad.to_rfc822()).ok(); + } + + if self.feedback_type == FeedbackType::AuthFailure { + if self.auth_failure != AuthFailureType::Unspecified { + write!( + &mut arf, + "Auth-Failure: {}\r\n", + match self.auth_failure { + AuthFailureType::Adsp => "adsp", + AuthFailureType::BodyHash => "bodyhash", + AuthFailureType::Revoked => "revoked", + AuthFailureType::Signature => "signature", + AuthFailureType::Spf => "spf", + AuthFailureType::Dmarc => "dmarc", + AuthFailureType::Unspecified => unreachable!(), + } + ) + .ok(); + } + + if self.delivery_result != DeliveryResult::Unspecified { + write!( + &mut arf, + "Delivery-Result: {}\r\n", + match self.delivery_result { + DeliveryResult::Delivered => "delivered", + DeliveryResult::Spam => "spam", + DeliveryResult::Policy => "policy", + DeliveryResult::Reject => "reject", + DeliveryResult::Other => "other", + DeliveryResult::Unspecified => unreachable!(), + } + ) + .ok(); + } + if let Some(value) = &self.dkim_adsp_dns { + write!(&mut arf, "DKIM-ADSP-DNS: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_canonicalized_body { + write!(&mut arf, "DKIM-Canonicalized-Body: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_canonicalized_header { + write!(&mut arf, "DKIM-Canonicalized-Header: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_domain { + write!(&mut arf, "DKIM-Domain: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_identity { + write!(&mut arf, "DKIM-Identity: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_selector { + write!(&mut arf, "DKIM-Selector: {value}\r\n").ok(); + } + if let Some(value) = &self.dkim_selector_dns { + write!(&mut arf, "DKIM-Selector-DNS: {value}\r\n").ok(); + } + if let Some(value) = &self.spf_dns { + write!(&mut arf, "SPF-DNS: {value}\r\n").ok(); + } + if self.identity_alignment != IdentityAlignment::Unspecified { + write!( + &mut arf, + "Identity-Alignment: {}\r\n", + match self.identity_alignment { + IdentityAlignment::None => "none", + IdentityAlignment::Spf => "spf", + IdentityAlignment::Dkim => "dkim", + IdentityAlignment::DkimSpf => "dkim, spf", + IdentityAlignment::Unspecified => unreachable!(), + } + ) + .ok(); + } + } + + for value in &self.authentication_results { + write!(&mut arf, "Authentication-Results: {value}\r\n").ok(); + } + if self.incidents > 1 { + write!(&mut arf, "Incidents: {}\r\n", self.incidents).ok(); + } + if let Some(value) = &self.original_envelope_id { + write!(&mut arf, "Original-Envelope-Id: {value}\r\n").ok(); + } + if let Some(value) = &self.original_mail_from { + write!(&mut arf, "Original-Mail-From: {value}\r\n").ok(); + } + if let Some(value) = &self.original_rcpt_to { + write!(&mut arf, "Original-Rcpt-To: {value}\r\n").ok(); + } + for value in &self.reported_domains { + write!(&mut arf, "Reported-Domain: {value}\r\n").ok(); + } + for value in &self.reported_uris { + write!(&mut arf, "Reported-URI: {value}\r\n").ok(); + } + if let Some(value) = &self.reporting_mta { + write!(&mut arf, "Reporting-MTA: dns;{value}\r\n").ok(); + } + if let Some(value) = &self.source_ip { + write!(&mut arf, "Source-IP: {value}\r\n").ok(); + } + if self.source_port != 0 { + write!(&mut arf, "Source-Port: {}\r\n", self.source_port).ok(); + } + if let Some(value) = &self.user_agent { + write!(&mut arf, "User-Agent: {value}\r\n").ok(); + } + + arf + } +} + diff --git a/crates/smtp-report/src/arf/mod.rs b/crates/smtp-report/src/arf/mod.rs new file mode 100644 index 00000000..b22df9c0 --- /dev/null +++ b/crates/smtp-report/src/arf/mod.rs @@ -0,0 +1,207 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Abuse Reporting Format feedback reports. +//! +//! [`FeedbackReport`] models the machine-readable `message/feedback-report` +//! part defined by RFC 5965, extended with the authentication failure fields +//! of RFC 6591 and the `not-spam` feedback type of RFC 6430. RFC 6650 covers +//! when to send these reports. Redacting the original message as described +//! in RFC 6590 is up to the caller. +//! Reports are parsed with [`FeedbackReport::parse_rfc5322`] or +//! [`FeedbackReport::parse_arf`] and written with +//! [`FeedbackReport::write_rfc5322`] or [`FeedbackReport::to_arf`]. +//! +//! Field documentation names the ARF field each field maps to. + +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, net::IpAddr}; + +mod builder; +mod generate; +mod parse; + +/// ARF feedback report (RFC 5965), including the authentication failure +/// fields of RFC 6591. +/// +/// String fields borrow from the parsed message; call +/// [`into_owned`](Self::into_owned) to detach them. Build a new report with +/// [`FeedbackReport::new`] and struct update syntax. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct FeedbackReport<'x> { + /// Kind of feedback (`Feedback-Type`). + pub feedback_type: FeedbackType, + /// When the original message was received, in seconds since the Unix + /// epoch (`Arrival-Date`, or the legacy `Received-Date`). + pub arrival_date: Option, + /// Authentication results for the original message, one per + /// `Authentication-Results` field. + pub authentication_results: Vec>, + /// Number of incidents the report represents (`Incidents`). Parsing + /// defaults to 1; the field is written only when greater than 1. + pub incidents: u32, + /// Envelope ID of the original message (`Original-Envelope-Id`). + pub original_envelope_id: Option>, + /// Envelope sender of the original message (`Original-Mail-From`). + pub original_mail_from: Option>, + /// Envelope recipient of the original message (`Original-Rcpt-To`). + /// When the field repeats, the last value is kept. + pub original_rcpt_to: Option>, + /// Domains the report concerns, one per `Reported-Domain` field. + pub reported_domains: Vec>, + /// URIs the report concerns, one per `Reported-URI` field. + pub reported_uris: Vec>, + /// Host name of the MTA generating the report (`Reporting-MTA`), without + /// the `dns;` type prefix, which is added when writing. + pub reporting_mta: Option>, + /// IP address the original message came from (`Source-IP`). + pub source_ip: Option, + /// Software that generated the report (`User-Agent`). + pub user_agent: Option>, + /// ARF version (`Version`), 1 for RFC 5965. Parsing yields 0 when the + /// field is absent or not a number. + pub version: u32, + /// Source TCP port of the original message (`Source-Port`, RFC 6692). + /// 0 when unknown; the field is written only when nonzero. + pub source_port: u16, + /// Kind of authentication failure (`Auth-Failure`, RFC 6591). + /// Auth-failure reports only. + pub auth_failure: AuthFailureType, + /// What the receiver did with the original message (`Delivery-Result`, + /// RFC 6591). Auth-failure reports only. + pub delivery_result: DeliveryResult, + /// Retrieved ADSP record (`DKIM-ADSP-DNS`, RFC 6591). Auth-failure + /// reports only. + pub dkim_adsp_dns: Option>, + /// Base64 of the canonicalized body the verifier hashed + /// (`DKIM-Canonicalized-Body`, RFC 6591). Auth-failure reports only. + pub dkim_canonicalized_body: Option>, + /// Base64 of the canonicalized header the verifier hashed + /// (`DKIM-Canonicalized-Header`, RFC 6591). Auth-failure reports only. + pub dkim_canonicalized_header: Option>, + /// Signing domain of the failed signature (`DKIM-Domain`, RFC 6591). + /// Auth-failure reports only. + pub dkim_domain: Option>, + /// Agent or user identifier of the failed signature (`DKIM-Identity`, RFC + /// 6591). Auth-failure reports only. + pub dkim_identity: Option>, + /// Selector of the failed signature (`DKIM-Selector`, RFC 6591). + /// Auth-failure reports only. + pub dkim_selector: Option>, + /// Retrieved DKIM key record (`DKIM-Selector-DNS`, RFC 6591). + /// Auth-failure reports only. + pub dkim_selector_dns: Option>, + /// Retrieved SPF record (`SPF-DNS`, RFC 6591). Auth-failure reports only. + pub spf_dns: Option>, + /// Which mechanisms produced an aligned identifier + /// (`Identity-Alignment`, DMARC failure reports). Auth-failure reports + /// only. + pub identity_alignment: IdentityAlignment, + + /// Full original message, from the `message/rfc822` part. + pub message: Option>, + /// Header section of the original message, from the + /// `text/rfc822-headers` part. Written only when `message` is `None`. + pub headers: Option>, +} + +/// Kind of authentication failure (`Auth-Failure` field, RFC 6591). +#[derive(Debug, Clone, PartialEq, Eq, Copy, Serialize, Deserialize, Default)] +pub enum AuthFailureType { + /// The message failed an ADSP check (`adsp`). + Adsp, + /// The DKIM body hash did not match (`bodyhash`). + BodyHash, + /// The DKIM key has been revoked (`revoked`). + Revoked, + /// The DKIM signature did not verify (`signature`). + Signature, + /// The message failed an SPF check (`spf`). + Spf, + /// The message failed DMARC (`dmarc`). + Dmarc, + /// The field was absent or had an unrecognized value. Not written. + #[default] + Unspecified, +} + +/// Mechanisms that produced a DMARC-aligned identifier +/// (`Identity-Alignment` field). +#[derive(Debug, Clone, PartialEq, Eq, Copy, Serialize, Deserialize, Default)] +pub enum IdentityAlignment { + /// Neither mechanism produced an aligned identifier (`none`). + None, + /// Only SPF produced an aligned identifier (`spf`). + Spf, + /// Only DKIM produced an aligned identifier (`dkim`). + Dkim, + /// Both mechanisms produced an aligned identifier (`dkim, spf`). + DkimSpf, + /// The field was absent or had no recognized value. Not written. + #[default] + Unspecified, +} + +/// What the receiver did with the original message (`Delivery-Result` +/// field, RFC 6591). +#[derive(Debug, Clone, PartialEq, Eq, Copy, Serialize, Deserialize, Default)] +pub enum DeliveryResult { + /// Delivered to the recipient (`delivered`). + Delivered, + /// Delivered to a spam folder (`spam`). + Spam, + /// Not delivered because of local policy (`policy`). + Policy, + /// Rejected (`reject`). + Reject, + /// Any other outcome (`other`). + Other, + /// The field was absent or had an unrecognized value. Not written. + #[default] + Unspecified, +} + +/// Kind of feedback (`Feedback-Type` field, RFC 5965). +#[derive(Debug, Clone, PartialEq, Eq, Copy, Serialize, Deserialize, Default)] +pub enum FeedbackType { + /// Unsolicited or abusive email (`abuse`). + Abuse, + /// Authentication failure report, RFC 6591 (`auth-failure`). + AuthFailure, + /// Fraudulent email such as phishing (`fraud`). + Fraud, + /// The message was wrongly classified as spam, RFC 6430 (`not-spam`). + NotSpam, + /// Any other feedback (`other`). This is the `Default` value; unknown + /// `Feedback-Type` values are rejected rather than mapped here. + #[default] + Other, + /// The message contained a virus (`virus`). + Virus, +} + +impl From<&mail_auth::DkimResult> for AuthFailureType { + fn from(value: &mail_auth::DkimResult) -> Self { + match value { + mail_auth::DkimResult::Neutral(err) + | mail_auth::DkimResult::Fail(err) + | mail_auth::DkimResult::PermError(err) + | mail_auth::DkimResult::TempError(err) => match err { + mail_auth::Error::Dkim(mail_auth::dkim::DkimError::BodyHashMismatch) => { + AuthFailureType::BodyHash + } + mail_auth::Error::Arc(mail_auth::arc::ArcError::BodyHashMismatch) => { + AuthFailureType::BodyHash + } + mail_auth::Error::Dkim(mail_auth::dkim::DkimError::PublicKeyRevoked) => { + AuthFailureType::Revoked + } + _ => AuthFailureType::Signature, + }, + mail_auth::DkimResult::Pass | mail_auth::DkimResult::None => AuthFailureType::Signature, + } + } +} diff --git a/crates/smtp-report/src/arf/parse.rs b/crates/smtp-report/src/arf/parse.rs new file mode 100644 index 00000000..2d3bf250 --- /dev/null +++ b/crates/smtp-report/src/arf/parse.rs @@ -0,0 +1,288 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Parsing of ARF feedback reports from their field format and from feedback +//! report email messages. + +use crate::{ + ReportError, + arf::{AuthFailureType, DeliveryResult, FeedbackReport, FeedbackType, IdentityAlignment}, + headers::HeaderIterator, + utf8::{into_string_lossy, to_str_lossy}, +}; +use mail_parser::{HeaderForm, MessageParser, MessagePart}; +use std::borrow::Cow; + +trait PartBody { + fn body_in<'x>(&self, source: &'x [u8]) -> Cow<'x, [u8]>; +} + +impl PartBody for MessagePart<'_> { + fn body_in<'x>(&self, source: &'x [u8]) -> Cow<'x, [u8]> { + match self.text() { + Some(Cow::Borrowed(text)) => reborrow(source, text.as_bytes()), + Some(Cow::Owned(text)) => Cow::Owned(text.into_bytes()), + None => match self.decoded() { + Cow::Borrowed(body) => reborrow(source, body), + Cow::Owned(body) => Cow::Owned(body), + }, + } + } +} + +fn reborrow<'x>(source: &'x [u8], bytes: &[u8]) -> Cow<'x, [u8]> { + bytes + .as_ptr() + .addr() + .checked_sub(source.as_ptr().addr()) + .and_then(|offset| source.get(offset..offset + bytes.len())) + .map_or_else(|| Cow::Owned(bytes.to_vec()), Cow::Borrowed) +} + +impl<'x> FeedbackReport<'x> { + /// Extracts and parses the feedback report in an RFC 5322 message. + /// + /// Reads the `message/feedback-report` part with + /// [`parse_arf`](Self::parse_arf); when there are several, the last one + /// wins. A `message/rfc822` part is stored in `message` and a + /// `text/rfc822-headers` part in `headers`. Strings borrow from `message` + /// where possible. + /// + /// `max_size` caps the length of the raw message, since ARF reports embed + /// the original message. + /// + /// # Errors + /// + /// - [`ReportError::TooLarge`] if `message` is longer than `max_size`. + /// - [`ReportError::MailParse`] if `message` is not a parseable message. + /// - [`ReportError::NotFound`] if there is no `message/feedback-report` + /// part. + /// - [`ReportError::Parse`] if the feedback report part is invalid (see + /// [`parse_arf`](Self::parse_arf)). + pub fn parse_rfc5322(message: &'x [u8], max_size: usize) -> Result { + if message.len() > max_size { + return Err(ReportError::TooLarge); + } + let parsed = MessageParser::new() + .parse(message) + .ok_or(ReportError::MailParse)?; + let mut feedback = Err(ReportError::NotFound); + let mut included_message = None; + let mut included_headers = None; + + for part in parsed.root().parts() { + if let Some(nested) = part.nested() { + included_message = Some(reborrow(message, nested.raw())); + } else if part.is_content_type("message", "feedback-report") { + feedback = match part.body_in(message) { + Cow::Borrowed(arf) => FeedbackReport::parse_arf(arf), + Cow::Owned(arf) => FeedbackReport::parse_arf(&arf).map(|f| f.into_owned()), + }; + } else if part.is_content_type("text", "rfc822-headers") && part.is_text() { + included_headers = Some(part.body_in(message)); + } + } + + feedback.map(|mut feedback| { + for (feedback, included) in [ + (&mut feedback.message, included_message), + (&mut feedback.headers, included_headers), + ] { + if let Some(included) = included { + *feedback = match included { + Cow::Borrowed(bytes) => Some(to_str_lossy(bytes)), + Cow::Owned(bytes) => Some(into_string_lossy(bytes).into()), + }; + } + } + + feedback + }) + } + + /// Parses the fields of a `message/feedback-report` part. + /// + /// Field names are case-insensitive and unknown fields are ignored. + /// Fields with unrecognized values keep their default. `incidents` + /// defaults to 1. + /// + /// # Errors + /// + /// Returns [`ReportError::Parse`] if there is no `Feedback-Type` field + /// with a recognized value. + pub fn parse_arf(arf: &'x [u8]) -> Result { + let mut f = FeedbackReport { + incidents: 1, + ..Default::default() + }; + let mut has_ft = false; + + let mut fields = HeaderIterator::new(arf); + fields.seek_start(); + + for (key, value) in fields { + let txt_value = std::str::from_utf8(value).unwrap_or_default().trim(); + + hashify::fnc_map_ignore_case!(key, + b"Arrival-Date" => { + if let Some(dt) = HeaderForm::Date.parse(value).value().as_datetime() { + f.arrival_date = dt.to_timestamp().into(); + } + }, + b"Auth-Failure" => { + f.auth_failure = match hashify::map_ignore_case!(txt_value.as_bytes(), AuthFailureType, + b"adsp" => AuthFailureType::Adsp, + b"bodyhash" => AuthFailureType::BodyHash, + b"revoked" => AuthFailureType::Revoked, + b"signature" => AuthFailureType::Signature, + b"spf" => AuthFailureType::Spf, + b"dmarc" => AuthFailureType::Dmarc, + ).copied() { + Some(auth_failure) => auth_failure, + None => continue, + }; + }, + b"Authentication-Results" => { + f.authentication_results.push(txt_value.into()); + }, + b"DKIM-ADSP-DNS" => { + f.dkim_adsp_dns = Some(txt_value.into()); + }, + b"DKIM-Canonicalized-Body" => { + f.dkim_canonicalized_body = Some(txt_value.into()); + }, + b"DKIM-Canonicalized-Header" => { + f.dkim_canonicalized_header = Some(txt_value.into()); + }, + b"DKIM-Domain" => { + f.dkim_domain = Some(txt_value.into()); + }, + b"DKIM-Identity" => { + f.dkim_identity = Some(txt_value.into()); + }, + b"DKIM-Selector" => { + f.dkim_selector = Some(txt_value.into()); + }, + b"DKIM-Selector-DNS" => { + f.dkim_selector_dns = Some(txt_value.into()); + }, + b"Delivery-Result" => { + f.delivery_result = match hashify::map_ignore_case!(txt_value.as_bytes(), DeliveryResult, + b"delivered" => DeliveryResult::Delivered, + b"spam" => DeliveryResult::Spam, + b"policy" => DeliveryResult::Policy, + b"reject" => DeliveryResult::Reject, + b"other" => DeliveryResult::Other, + ).copied() { + Some(delivery_result) => delivery_result, + None => continue, + }; + }, + b"Feedback-Type" => { + f.feedback_type = match hashify::map_ignore_case!(txt_value.as_bytes(), FeedbackType, + b"abuse" => FeedbackType::Abuse, + b"auth-failure" => FeedbackType::AuthFailure, + b"fraud" => FeedbackType::Fraud, + b"not-spam" => FeedbackType::NotSpam, + b"other" => FeedbackType::Other, + b"virus" => FeedbackType::Virus, + ).copied() { + Some(feedback_type) => feedback_type, + None => continue, + }; + has_ft = true; + }, + b"Identity-Alignment" => { + for id in txt_value.split(',') { + let id = id.trim(); + hashify::fnc_map_ignore_case!(id.as_bytes(), + b"dkim" => { + f.identity_alignment = + if f.identity_alignment == IdentityAlignment::Spf { + IdentityAlignment::DkimSpf + } else { + IdentityAlignment::Dkim + }; + }, + b"spf" => { + f.identity_alignment = + if f.identity_alignment == IdentityAlignment::Dkim { + IdentityAlignment::DkimSpf + } else { + IdentityAlignment::Spf + }; + }, + b"none" => { + f.identity_alignment = IdentityAlignment::None; + break; + }, + _ => () + ); + } + }, + b"Incidents" => { + f.incidents = txt_value.parse().unwrap_or(1); + }, + b"Original-Envelope-Id" => { + f.original_envelope_id = Some(txt_value.into()); + }, + b"Original-Mail-From" => { + f.original_mail_from = Some(txt_value.into()); + }, + b"Original-Rcpt-To" => { + f.original_rcpt_to = Some(txt_value.into()); + }, + b"Reported-Domain" => { + f.reported_domains.push(txt_value.into()); + }, + b"Reported-URI" => { + f.reported_uris.push(txt_value.into()); + }, + b"Reporting-MTA" => { + f.reporting_mta = Some(if let Some(mta) = txt_value.strip_prefix("dns;") { + mta.trim().into() + } else { + txt_value.into() + }); + }, + b"Received-Date" => { + if let Some(dt) = HeaderForm::Date.parse(value).value().as_datetime() { + f.arrival_date = dt.to_timestamp().into(); + } + }, + b"SPF-DNS" => { + f.spf_dns = Some(txt_value.into()); + }, + b"Source-IP" => { + f.source_ip = if let Some((ip, _)) = txt_value.split_once(' ') { + ip.parse().ok() + } else { + txt_value.parse().ok() + }; + }, + b"Source-Port" => { + f.source_port = txt_value.parse().unwrap_or(0); + }, + b"User-Agent" => { + f.user_agent = Some(txt_value.into()); + }, + b"Version" => { + f.version = txt_value.parse().unwrap_or(0); + }, + _ => () + ); + } + + if has_ft { + Ok(f) + } else { + Err(ReportError::Parse( + "Missing Feedback-Type field".to_string(), + )) + } + } +} + diff --git a/crates/smtp-report/src/dmarc/builder.rs b/crates/smtp-report/src/dmarc/builder.rs new file mode 100644 index 00000000..f701f7f7 --- /dev/null +++ b/crates/smtp-report/src/dmarc/builder.rs @@ -0,0 +1,211 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Helpers that fill a [`Record`] and a [`PolicyPublished`] from the outputs +//! of the DKIM, DKIM2, SPF, DMARC and ARC verifiers. + +use super::PolicyPublished; +use crate::dmarc::{ + Discovery, Disposition, DkimAuthResult, DkimStatus, DmarcStatus, Record, SpfAuthResult, + SpfScope, SpfStatus, +}; +use crate::dmarc::{PolicyOverride, PolicyOverrideReason}; +use mail_auth::ArcOutput; +use mail_auth::{ + Dkim2Result, DkimOutput, DmarcOutput, SpfOutput, + dkim2::Dkim2Output, + dmarc::{DmarcRecord, Policy}, +}; +use std::fmt::Write; + +impl Record { + /// Appends one [`DkimAuthResult`] to `auth_results.dkim` for each DKIM + /// output that carries a signature. + /// + /// The domain and selector come from the signature's `d=` and `s=` tags. + /// For results other than `pass` and `none`, `human_result` holds the + /// error text. Outputs without a signature are skipped. + pub fn with_dkim_output(mut self, dkim_output: &[DkimOutput]) -> Self { + for dkim in dkim_output { + if let Some(signature) = dkim.signature() { + let (result, human_result) = match dkim.result() { + mail_auth::DkimResult::Pass => (DkimStatus::Pass, None), + mail_auth::DkimResult::Neutral(err) => { + (DkimStatus::Neutral, err.to_string().into()) + } + mail_auth::DkimResult::Fail(err) => (DkimStatus::Fail, err.to_string().into()), + mail_auth::DkimResult::PermError(err) => { + (DkimStatus::PermError, err.to_string().into()) + } + mail_auth::DkimResult::TempError(err) => { + (DkimStatus::TempError, err.to_string().into()) + } + mail_auth::DkimResult::None => (DkimStatus::None, None), + }; + + self.auth_results.dkim.push(DkimAuthResult { + domain: signature.d.to_string(), + selector: signature.s.to_string(), + result, + human_result, + }); + } + } + self + } + + /// Appends one [`DkimAuthResult`] to `auth_results.dkim` for each link of + /// the DKIM2 signature chain. + /// + /// The domain comes from the signature's `d=` tag and the selector from its + /// first `s=` entry (empty when there is none). For results other than + /// `pass` and `none`, `human_result` holds the error text. + pub fn with_dkim2_output(mut self, dkim2_output: &Dkim2Output) -> Self { + for link in dkim2_output.chain() { + let (result, human_result) = match &link.result { + Dkim2Result::Pass => (DkimStatus::Pass, None), + Dkim2Result::Fail(err) => (DkimStatus::Fail, err.to_string().into()), + Dkim2Result::PermError(err) => (DkimStatus::PermError, err.to_string().into()), + Dkim2Result::TempError(err) => (DkimStatus::TempError, err.to_string().into()), + Dkim2Result::None => (DkimStatus::None, None), + }; + + self.auth_results.dkim.push(DkimAuthResult { + domain: link.signature.d.to_string(), + selector: link + .signature + .s + .first() + .map(|value| value.selector.clone()) + .unwrap_or_default(), + result, + human_result, + }); + } + self + } + + /// Appends one [`SpfAuthResult`] for `spf_output` to `auth_results.spf`. + /// + /// `scope` records which identity was checked (`HELO` or `MAIL FROM`) and + /// is stored as given. `human_result` is left empty. + pub fn with_spf_output(mut self, spf_output: &SpfOutput, scope: SpfScope) -> Self { + self.auth_results.spf.push(SpfAuthResult { + domain: spf_output.domain().to_string(), + scope, + result: match spf_output.result() { + mail_auth::SpfResult::Pass => SpfStatus::Pass, + mail_auth::SpfResult::Fail => SpfStatus::Fail, + mail_auth::SpfResult::SoftFail => SpfStatus::SoftFail, + mail_auth::SpfResult::Neutral => SpfStatus::Neutral, + mail_auth::SpfResult::TempError => SpfStatus::TempError, + mail_auth::SpfResult::PermError => SpfStatus::PermError, + mail_auth::SpfResult::None => SpfStatus::None, + }, + human_result: None, + }); + self + } + + /// Sets `row.policy_evaluated` from a DMARC verification result. + /// + /// The `dkim` and `spf` statuses are [`DmarcStatus::Pass`] when the aligned + /// result passed and [`DmarcStatus::Fail`] otherwise. The disposition is + /// [`Disposition::Pass`] when either passed; otherwise it follows the + /// policy in `dmarc_output` (`none` when the policy is unspecified). + /// Override reasons are left untouched. + pub fn with_dmarc_output(mut self, dmarc_output: &DmarcOutput) -> Self { + self.row.policy_evaluated.disposition = if dmarc_output.dkim_result() + == &mail_auth::DmarcResult::Pass + || dmarc_output.spf_result() == &mail_auth::DmarcResult::Pass + { + Disposition::Pass + } else { + match dmarc_output.policy() { + Policy::None | Policy::Unspecified => Disposition::None, + Policy::Quarantine => Disposition::Quarantine, + Policy::Reject => Disposition::Reject, + } + }; + self.row.policy_evaluated.dkim = dmarc_output.dkim_result().into(); + self.row.policy_evaluated.spf = dmarc_output.spf_result().into(); + self + } + + /// Records a passing ARC chain as a policy override reason. + /// + /// When the ARC result is `pass`, appends a [`PolicyOverride::LocalPolicy`] + /// reason whose comment reads `arc=pass` followed by the `d=` and `s=` + /// tags of every ARC-Seal, in reverse chain order, as `as[i].d=... as[i].s=...`. + /// Any other ARC result leaves the record unchanged. + pub fn with_arc_output(mut self, arc_output: &ArcOutput) -> Self { + if arc_output.result() == &mail_auth::DkimResult::Pass { + let mut comment = "arc=pass".to_string(); + for set in arc_output.chain().iter().rev() { + let seal = &set.seal.header; + write!( + &mut comment, + " as[{}].d={} as[{}].s={}", + seal.i, seal.d, seal.i, seal.s + ) + .ok(); + } + self.row.policy_evaluated.reason.push(PolicyOverrideReason { + kind: PolicyOverride::LocalPolicy, + comment: Some(comment), + }); + } + self + } +} + +impl PolicyPublished { + /// Builds the `policy_published` element for `domain` from its parsed + /// DMARC record. + /// + /// Copies `adkim`, `aspf`, `p`, `sp`, `np` (an unspecified policy becomes + /// `none`), `testing` from the `t` tag and `fo` in its tag syntax (`0`, + /// `1`, `d`, `s` or `d:s`). The discovery method is set to + /// [`Discovery::Treewalk`] and `version_published` to `None`. + pub fn from_record(domain: impl Into, dmarc: &DmarcRecord) -> Self { + PolicyPublished { + domain: domain.into(), + adkim: Some(dmarc.adkim), + aspf: Some(dmarc.aspf), + p: published_policy(dmarc.p), + sp: published_policy(dmarc.sp), + np: published_policy(dmarc.np), + testing: dmarc.t, + discovery_method: Discovery::Treewalk, + fo: match &dmarc.fo { + mail_auth::dmarc::FailureOptions::All => "0", + mail_auth::dmarc::FailureOptions::Any => "1", + mail_auth::dmarc::FailureOptions::Dkim => "d", + mail_auth::dmarc::FailureOptions::Spf => "s", + mail_auth::dmarc::FailureOptions::DkimSpf => "d:s", + } + .to_string() + .into(), + version_published: None, + } + } +} + +impl From<&mail_auth::DmarcResult> for DmarcStatus { + fn from(result: &mail_auth::DmarcResult) -> Self { + match result { + mail_auth::DmarcResult::Pass => DmarcStatus::Pass, + _ => DmarcStatus::Fail, + } + } +} + +fn published_policy(policy: Policy) -> Policy { + match policy { + Policy::Unspecified => Policy::None, + policy => policy, + } +} diff --git a/crates/smtp-report/src/dmarc/generate.rs b/crates/smtp-report/src/dmarc/generate.rs new file mode 100644 index 00000000..770f403b --- /dev/null +++ b/crates/smtp-report/src/dmarc/generate.rs @@ -0,0 +1,543 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Serialization of [`AggregateReport`] to RFC 9990 XML and to a complete +//! report email message. + +use crate::{ + ReportEnvelope, + dmarc::{ + AggregateReport, AuthResults, DateRange, Discovery, Disposition, DkimAuthResult, + DkimStatus, DmarcStatus, Identifiers, PolicyEvaluated, PolicyOverride, + PolicyOverrideReason, PolicyPublished, Record, ReportMetadata, Row, SpfAuthResult, + SpfScope, SpfStatus, + }, +}; +use flate2::{Compression, write::GzEncoder}; +use mail_auth::dmarc::Policy; +use mail_builder::{MessageBuilder, headers::HeaderType, mime::make_boundary}; +use std::{ + borrow::Cow, + fmt::{Display, Formatter, Write}, + io, +}; + +impl AggregateReport { + /// Writes the report as an RFC 5322 message to `writer`. + /// + /// The message has a `text/plain` summary and the output of + /// [`to_xml`](Self::to_xml), gzip-compressed, as an `application/gzip` + /// attachment named `submitter!domain!begin!end.xml.gz`. It carries an + /// `Auto-Submitted: auto-generated` header field. + /// + /// Fields read from `envelope`: + /// + /// - `from`: the `From` header field. + /// - `to`: the `To` header field. + /// - `submitter`: the `Message-ID` host, the subject, the text body and + /// the file name. + /// - `subject`: the subject; when `None`, defaults to + /// `Report Domain: Submitter: Report-ID: <>`. + /// + /// The Report Domain is always `policy_published.domain`; + /// `envelope.report_domain` is ignored. + /// + /// # Errors + /// + /// Returns any I/O error raised by `writer` or by the gzip encoder. + /// + /// # Example + /// + /// ```rust,no_run + /// use mail_auth::dmarc::Policy; + /// use smtp_report::{ + /// ReportEnvelope, + /// dmarc::{ + /// AggregateReport, DateRange, Disposition, DmarcStatus, Identifiers, + /// PolicyEvaluated, PolicyPublished, Record, ReportMetadata, ReportVersion, Row, + /// }, + /// }; + /// + /// # fn main() -> std::io::Result<()> { + /// let report = AggregateReport { + /// version: Some(ReportVersion::V1), + /// report_metadata: ReportMetadata { + /// org_name: "Example Inc.".into(), + /// email: "dmarc@example.net".into(), + /// report_id: "abc-123".into(), + /// date_range: DateRange { + /// begin: 1_700_000_000, + /// end: 1_700_086_400, + /// }, + /// ..Default::default() + /// }, + /// policy_published: PolicyPublished { + /// domain: "example.org".into(), + /// p: Policy::Reject, + /// ..Default::default() + /// }, + /// records: vec![Record { + /// row: Row { + /// source_ip: Some("192.0.2.1".parse().expect("valid IP address")), + /// count: 3, + /// policy_evaluated: PolicyEvaluated { + /// disposition: Disposition::Pass, + /// dkim: DmarcStatus::Pass, + /// spf: DmarcStatus::Fail, + /// reason: vec![], + /// }, + /// }, + /// identifiers: Identifiers { + /// envelope_from: "example.org".into(), + /// header_from: "example.org".into(), + /// ..Default::default() + /// }, + /// ..Default::default() + /// }], + /// ..Default::default() + /// }; + /// + /// let envelope = ReportEnvelope { + /// from: ("Example DMARC Reporter", "noreply-dmarc@example.net").into(), + /// to: vec!["dmarc-reports@example.org"], + /// submitter: "example.net", + /// report_domain: "", + /// subject: None, + /// }; + /// + /// report.write_rfc5322(&envelope, std::io::stdout())?; + /// # Ok(()) + /// # } + /// ``` + pub fn write_rfc5322( + &self, + envelope: &ReportEnvelope<'_>, + writer: impl io::Write, + ) -> io::Result<()> { + let xml = self.to_xml(); + let mut e = GzEncoder::new(Vec::with_capacity(xml.len()), Compression::default()); + io::Write::write_all(&mut e, xml.as_bytes())?; + let compressed_bytes = e.finish()?; + + let submitter = envelope.submitter; + let domain = self.policy_published.domain.as_str(); + let report_id = self.report_metadata.report_id.as_str(); + let subject = envelope.subject.map_or_else( + || { + Cow::Owned(format!( + "Report Domain: {domain} Submitter: {submitter} Report-ID: <{report_id}>" + )) + }, + Cow::Borrowed, + ); + + MessageBuilder::new() + .from(envelope.from.clone()) + .header("To", envelope.to_header()) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .message_id(format!("{}@{}", make_boundary("."), submitter)) + .subject(subject) + .text_body(format!( + concat!( + "DMARC aggregate report from {}\r\n\r\n", + "Report Domain: {}\r\n", + "Submitter: {}\r\n", + "Report-ID: {}\r\n", + ), + submitter, domain, submitter, report_id + )) + .attachment( + "application/gzip", + format!( + "{}!{}!{}!{}.xml.gz", + submitter, + domain, + self.report_metadata.date_range.begin, + self.report_metadata.date_range.end + ), + compressed_bytes, + ) + .write_to(writer) + } + + /// Returns the report as an RFC 5322 message. + /// + /// Same as [`write_rfc5322`](Self::write_rfc5322), collected into a + /// `String`. + /// + /// # Errors + /// + /// Returns an error if the gzip encoder fails or the message is not valid + /// UTF-8. + pub fn to_rfc5322(&self, envelope: &ReportEnvelope<'_>) -> io::Result { + let mut buf = Vec::new(); + self.write_rfc5322(envelope, &mut buf)?; + String::from_utf8(buf).map_err(io::Error::other) + } + + /// Serializes the report as an RFC 9990 XML document in the + /// `urn:ietf:params:xml:ns:dmarc-2.0` namespace. + /// + /// Optional elements are omitted when unset. The `version_published` + /// element and all extensions are not written. Multiple `errors` are + /// joined with `; ` into a single `error` element, and each record gets + /// at most one `spf` result (see [`AuthResults::spf`]). + pub fn to_xml(&self) -> String { + let mut xml = String::with_capacity(128); + writeln!(&mut xml, "").ok(); + writeln!( + &mut xml, + "" + ) + .ok(); + if let Some(version) = self.version { + writeln!(&mut xml, "\t{}", version.as_str()).ok(); + } + self.report_metadata.to_xml(&mut xml); + self.policy_published.to_xml(&mut xml); + for record in &self.records { + record.to_xml(&mut xml); + } + writeln!(&mut xml, "").ok(); + xml + } +} + +impl ReportMetadata { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t").ok(); + writeln!( + xml, + "\t\t{}", + escape_xml(&self.org_name) + ) + .ok(); + writeln!(xml, "\t\t{}", escape_xml(&self.email)).ok(); + if let Some(eci) = &self.extra_contact_info { + writeln!( + xml, + "\t\t{}", + escape_xml(eci) + ) + .ok(); + } + writeln!( + xml, + "\t\t{}", + escape_xml(&self.report_id) + ) + .ok(); + self.date_range.to_xml(xml); + match self.errors.as_slice() { + [] => {} + [error] => { + writeln!(xml, "\t\t{}", escape_xml(error)).ok(); + } + errors => { + writeln!(xml, "\t\t{}", escape_xml(&errors.join("; "))).ok(); + } + } + if let Some(generator) = &self.generator { + writeln!(xml, "\t\t{}", escape_xml(generator)).ok(); + } + writeln!(xml, "\t").ok(); + } +} + +impl PolicyPublished { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t").ok(); + writeln!(xml, "\t\t{}", escape_xml(&self.domain)).ok(); + writeln!(xml, "\t\t

{}

", self.p).ok(); + if self.sp != Policy::Unspecified { + writeln!(xml, "\t\t{}", self.sp).ok(); + } + if self.np != Policy::Unspecified { + writeln!(xml, "\t\t{}", self.np).ok(); + } + if let Some(adkim) = self.adkim { + writeln!(xml, "\t\t{adkim}").ok(); + } + if let Some(aspf) = self.aspf { + writeln!(xml, "\t\t{aspf}").ok(); + } + if self.discovery_method != Discovery::Unspecified { + writeln!( + xml, + "\t\t{}", + self.discovery_method + ) + .ok(); + } + if let Some(fo) = &self.fo { + writeln!(xml, "\t\t{}", escape_xml(fo)).ok(); + } + writeln!( + xml, + "\t\t{}", + if self.testing { "y" } else { "n" } + ) + .ok(); + writeln!(xml, "\t
").ok(); + } +} + +impl DateRange { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t").ok(); + writeln!(xml, "\t\t\t{}", self.begin).ok(); + writeln!(xml, "\t\t\t{}", self.end).ok(); + writeln!(xml, "\t\t").ok(); + } +} + +impl Record { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t").ok(); + self.row.to_xml(xml); + self.identifiers.to_xml(xml); + self.auth_results.to_xml(xml); + writeln!(xml, "\t").ok(); + } +} + +impl Row { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t").ok(); + if let Some(source_ip) = &self.source_ip { + writeln!(xml, "\t\t\t{source_ip}").ok(); + } + writeln!(xml, "\t\t\t{}", self.count).ok(); + self.policy_evaluated.to_xml(xml); + writeln!(xml, "\t\t").ok(); + } +} + +impl PolicyEvaluated { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t\t").ok(); + writeln!( + xml, + "\t\t\t\t{}", + self.disposition + ) + .ok(); + writeln!(xml, "\t\t\t\t{}", self.dkim).ok(); + writeln!(xml, "\t\t\t\t{}", self.spf).ok(); + for reason in &self.reason { + reason.to_xml(xml); + } + writeln!(xml, "\t\t\t").ok(); + } +} + +impl PolicyOverrideReason { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t\t\t").ok(); + writeln!(xml, "\t\t\t\t\t{}", self.kind).ok(); + if let Some(comment) = &self.comment { + writeln!(xml, "\t\t\t\t\t{}", escape_xml(comment)).ok(); + } + writeln!(xml, "\t\t\t\t").ok(); + } +} + +impl Identifiers { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t").ok(); + if let Some(envelope_to) = &self.envelope_to { + writeln!( + xml, + "\t\t\t{}", + escape_xml(envelope_to) + ) + .ok(); + } + writeln!( + xml, + "\t\t\t{}", + escape_xml(&self.envelope_from) + ) + .ok(); + writeln!( + xml, + "\t\t\t{}", + escape_xml(&self.header_from) + ) + .ok(); + writeln!(xml, "\t\t").ok(); + } +} + +impl AuthResults { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t").ok(); + for dkim in &self.dkim { + dkim.to_xml(xml); + } + if let Some(spf) = self.spf.iter().find(|spf| spf.scope != SpfScope::Helo) { + spf.to_xml(xml); + } + writeln!(xml, "\t\t").ok(); + } +} + +impl DkimAuthResult { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t\t").ok(); + writeln!(xml, "\t\t\t\t{}", escape_xml(&self.domain)).ok(); + writeln!( + xml, + "\t\t\t\t{}", + escape_xml(&self.selector) + ) + .ok(); + writeln!(xml, "\t\t\t\t{}", self.result).ok(); + if let Some(result) = &self.human_result { + writeln!( + xml, + "\t\t\t\t{}", + escape_xml(result) + ) + .ok(); + } + writeln!(xml, "\t\t\t").ok(); + } +} + +impl SpfAuthResult { + pub(crate) fn to_xml(&self, xml: &mut String) { + writeln!(xml, "\t\t\t").ok(); + writeln!(xml, "\t\t\t\t{}", escape_xml(&self.domain)).ok(); + if self.scope == SpfScope::MailFrom { + writeln!(xml, "\t\t\t\t{}", self.scope).ok(); + } + writeln!(xml, "\t\t\t\t{}", self.result).ok(); + if let Some(result) = &self.human_result { + writeln!( + xml, + "\t\t\t\t{}", + escape_xml(result) + ) + .ok(); + } + writeln!(xml, "\t\t\t").ok(); + } +} + +impl Display for Disposition { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Disposition::None | Disposition::Unspecified => "none", + Disposition::Pass => "pass", + Disposition::Quarantine => "quarantine", + Disposition::Reject => "reject", + }) + } +} + +impl Display for DmarcStatus { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + DmarcStatus::Pass => "pass", + DmarcStatus::Fail => "fail", + DmarcStatus::Unspecified => "", + }) + } +} + +impl Display for PolicyOverride { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + PolicyOverride::TrustedForwarder => "trusted_forwarder", + PolicyOverride::MailingList => "mailing_list", + PolicyOverride::LocalPolicy => "local_policy", + PolicyOverride::PolicyTestMode => "policy_test_mode", + PolicyOverride::Other => "other", + }) + } +} + +impl Display for Discovery { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Discovery::Psl => "psl", + Discovery::Treewalk => "treewalk", + Discovery::Unspecified => "", + }) + } +} + +impl Display for DkimStatus { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + DkimStatus::None => "none", + DkimStatus::Pass => "pass", + DkimStatus::Fail => "fail", + DkimStatus::Policy => "policy", + DkimStatus::Neutral => "neutral", + DkimStatus::TempError => "temperror", + DkimStatus::PermError => "permerror", + }) + } +} + +impl Display for SpfScope { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SpfScope::Helo => "helo", + SpfScope::MailFrom | SpfScope::Unspecified => "mfrom", + }) + } +} + +impl Display for SpfStatus { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SpfStatus::None => "none", + SpfStatus::Neutral => "neutral", + SpfStatus::Pass => "pass", + SpfStatus::Fail => "fail", + SpfStatus::SoftFail => "softfail", + SpfStatus::TempError => "temperror", + SpfStatus::PermError => "permerror", + }) + } +} + +fn escape_xml(text: &str) -> Cow<'_, str> { + for ch in text.as_bytes() { + if b"\"'<>&".contains(ch) { + let mut escaped = String::with_capacity(text.len()); + for ch in text.chars() { + match ch { + '"' => { + escaped.push_str("""); + } + '\'' => { + escaped.push_str("'"); + } + '<' => { + escaped.push_str("<"); + } + '>' => { + escaped.push_str(">"); + } + '&' => { + escaped.push_str("&"); + } + _ => { + escaped.push(ch); + } + } + } + + return escaped.into(); + } + } + text.into() +} + diff --git a/crates/smtp-report/src/dmarc/mod.rs b/crates/smtp-report/src/dmarc/mod.rs new file mode 100644 index 00000000..2ee12fb4 --- /dev/null +++ b/crates/smtp-report/src/dmarc/mod.rs @@ -0,0 +1,473 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! DMARC aggregate reports (RFC 9990). +//! +//! [`AggregateReport`] models the XML `feedback` document. Reports are parsed +//! with [`AggregateReport::parse_rfc5322`] or [`AggregateReport::parse_xml`]; +//! legacy reports in the RFC 7489 Appendix C format still parse. Reports are +//! built as struct literals, with [`Record`] and [`PolicyPublished`] filled +//! from verification results by [`Record::with_dkim_output`] and the other +//! `with_*_output` helpers and [`PolicyPublished::from_record`], and written with +//! [`AggregateReport::write_rfc5322`] or [`AggregateReport::to_xml`]. +//! +//! Field documentation names the XML element each field maps to. + +use mail_auth::dmarc::{Alignment, Policy}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{self, Visitor}, +}; +use std::{ + fmt::{Display, Formatter}, + net::IpAddr, + str::FromStr, +}; + +mod builder; +mod generate; +mod parse; + +/// Reporting period covered by an aggregate report (`date_range` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct DateRange { + /// Start of the period in seconds since the Unix epoch (`begin`). + pub begin: u64, + /// End of the period in seconds since the Unix epoch (`end`). + pub end: u64, +} + +/// Information about the reporting organization (`report_metadata` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ReportMetadata { + /// Name of the reporting organization (`org_name`). + pub org_name: String, + /// Contact email address of the reporting organization (`email`). + pub email: String, + /// Additional contact details (`extra_contact_info`). + pub extra_contact_info: Option, + /// Unique identifier of the report within the reporting organization + /// (`report_id`). + pub report_id: String, + /// Reporting period (`date_range`). + pub date_range: DateRange, + /// Errors encountered while generating the report, one per `error` + /// element. + pub errors: Vec, + /// Name and version of the software that generated the report + /// (`generator`, RFC 9990 only). + #[serde(default)] + pub generator: Option, +} + +/// Policy applied to the messages of a row (`disposition` element of +/// `policy_evaluated`). +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum Disposition { + /// No action was taken (`none`). + None, + /// The messages passed DMARC (`pass`). + Pass, + /// The messages were quarantined (`quarantine`). + Quarantine, + /// The messages were rejected (`reject`). + Reject, + /// The element was absent or had an unrecognized value. Written as `none`. + #[default] + Unspecified, +} + +/// DMARC policy published by the domain owner (`policy_published` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct PolicyPublished { + /// Domain whose DMARC record was applied (`domain`). This is the Report + /// Domain of the generated report message. + pub domain: String, + /// Version of the published policy (`version_published`). Not written by + /// [`AggregateReport::to_xml`]. + #[serde(default, deserialize_with = "deserialize_version")] + pub version_published: Option, + /// DKIM identifier alignment mode (`adkim`). `None` when the element is + /// absent or unrecognized. + pub adkim: Option, + /// SPF identifier alignment mode (`aspf`). `None` when the element is + /// absent or unrecognized. + pub aspf: Option, + /// Policy for the domain (`p`). [`Policy::Unspecified`] when the element + /// is absent or unrecognized. + pub p: Policy, + /// Policy for subdomains (`sp`). [`Policy::Unspecified`] when the element + /// is absent or unrecognized; not written in that case. + pub sp: Policy, + /// Policy for non-existent subdomains (`np`, RFC 9990 only). + /// [`Policy::Unspecified`] when the element is absent or unrecognized; + /// not written in that case. + #[serde(default)] + pub np: Policy, + /// Whether the policy is in test mode (`testing`, `y` or `n`), from the + /// DMARC `t` tag. + pub testing: bool, + /// Method used to find the DMARC record (`discovery_method`, RFC 9990 + /// only). + #[serde(default)] + pub discovery_method: Discovery, + /// Failure reporting options, verbatim (`fo`). + pub fo: Option, +} + +/// Method used to discover the DMARC policy record (`discovery_method` +/// element). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum Discovery { + /// Public Suffix List lookup, as in RFC 7489 (`psl`). + Psl, + /// DNS tree walk, as in RFC 9989 (`treewalk`). + Treewalk, + /// The element was absent or had an unrecognized value. Not written. + #[default] + Unspecified, +} + +/// DMARC-aligned result of one mechanism (`dkim` and `spf` elements of +/// `policy_evaluated`). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum DmarcStatus { + /// The mechanism passed and was aligned (`pass`). + Pass, + /// The mechanism failed or was not aligned (`fail`). + Fail, + /// The element was absent or had an unrecognized value. Written as an + /// empty element. + #[default] + Unspecified, +} + +/// Reason the applied disposition differs from the published policy (`type` +/// element of `reason`). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum PolicyOverride { + /// A local policy overrode the published one (`local_policy`). + LocalPolicy, + /// The message came from a mailing list (`mailing_list`). + MailingList, + /// The policy was in test mode (`policy_test_mode`). + PolicyTestMode, + /// The message came from a trusted forwarder (`trusted_forwarder`). + TrustedForwarder, + /// `other`, or a value that is absent or unrecognized, such as the RFC + /// 7489 `forwarded` and `sampled_out` types. + #[default] + Other, +} + +/// One policy override reason (`reason` element of `policy_evaluated`). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct PolicyOverrideReason { + /// Kind of override (`type`). + pub kind: PolicyOverride, + /// Free-form explanation (`comment`). + pub comment: Option, +} + +/// Result of applying the DMARC policy to a row (`policy_evaluated` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct PolicyEvaluated { + /// Action taken on the messages (`disposition`). + pub disposition: Disposition, + /// DMARC-aligned DKIM result (`dkim`). + pub dkim: DmarcStatus, + /// DMARC-aligned SPF result (`spf`). + pub spf: DmarcStatus, + /// Reasons the disposition differs from the published policy, one per + /// `reason` element. + pub reason: Vec, +} + +/// Source and count of the messages of a record (`row` element). +#[derive(Debug, Clone, Hash, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Row { + /// IP address the messages came from (`source_ip`). `None` when the + /// element is absent or not a valid address. + pub source_ip: Option, + /// Number of messages this record covers (`count`). + pub count: u32, + /// Policy evaluation result (`policy_evaluated`). + pub policy_evaluated: PolicyEvaluated, +} + +/// Report extension declared with an `extension` element inside an +/// `extensions` element. +/// +/// Only the `name` and `definition` attributes are kept; the extension content +/// is skipped. Extensions are not written by [`AggregateReport::to_xml`]. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Extension { + /// Extension name (`name` attribute). + pub name: String, + /// URI of the extension definition (`definition` attribute). + pub definition: String, +} + +/// Identifiers of the messages of a record (`identifiers` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Identifiers { + /// Domain of the envelope recipient (`envelope_to`). + pub envelope_to: Option, + /// Domain of the `MAIL FROM` envelope sender (`envelope_from`). + pub envelope_from: String, + /// Domain of the `From` header field (`header_from`). + pub header_from: String, +} + +/// Result of verifying one DKIM signature (`result` element of an +/// `auth_results/dkim` element). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum DkimStatus { + /// The message was not signed (`none`). Also used when the element is + /// absent or has an unrecognized value. + #[default] + None, + /// The signature verified (`pass`). + Pass, + /// The signature did not verify (`fail`). + Fail, + /// The signature verified but was not accepted by local policy (`policy`). + Policy, + /// The signature could not be processed (`neutral`). + Neutral, + /// A transient error, such as a DNS timeout, prevented verification + /// (`temperror`). + TempError, + /// A permanent error, such as a malformed key record, prevented + /// verification (`permerror`). + PermError, +} + +/// Raw result of one DKIM signature (`dkim` element of `auth_results`). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct DkimAuthResult { + /// Signing domain, from the signature's `d=` tag (`domain`). + pub domain: String, + /// Selector, from the signature's `s=` tag (`selector`). + pub selector: String, + /// Verification result (`result`). + pub result: DkimStatus, + /// Human-readable detail about the result (`human_result`). + pub human_result: Option, +} + +/// Identity checked by SPF (`scope` element of an `auth_results/spf` +/// element). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum SpfScope { + /// The `HELO` identity (`helo`, RFC 7489 only). + Helo, + /// The `MAIL FROM` identity (`mfrom`). + MailFrom, + /// The element was absent or had an unrecognized value. Not written by + /// [`AggregateReport::to_xml`]. + #[default] + Unspecified, +} + +/// Result of an SPF check (`result` element of an `auth_results/spf` +/// element). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum SpfStatus { + /// No SPF record was found (`none`). Also used when the element is absent + /// or has an unrecognized value. + #[default] + None, + /// The record makes no assertion about the sender (`neutral`). + Neutral, + /// The sender is authorized (`pass`). + Pass, + /// The sender is not authorized (`fail`). + Fail, + /// The sender is probably not authorized (`softfail`). + SoftFail, + /// A transient error, such as a DNS timeout, prevented the check + /// (`temperror`). + TempError, + /// A permanent error, such as a malformed record, prevented the check + /// (`permerror`). + PermError, +} + +/// Raw result of one SPF check (`spf` element of `auth_results`). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct SpfAuthResult { + /// Domain that was checked (`domain`). + pub domain: String, + /// Identity that was checked (`scope`). + pub scope: SpfScope, + /// Check result (`result`). + pub result: SpfStatus, + /// Human-readable detail about the result (`human_result`). + pub human_result: Option, +} + +/// Raw authentication results of a record, before DMARC alignment +/// (`auth_results` element). +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct AuthResults { + /// DKIM results, one per `dkim` element. + pub dkim: Vec, + /// SPF results, one per `spf` element. + /// + /// Parsing keeps every element. [`AggregateReport::to_xml`] writes at most + /// one: the first result whose scope is not [`SpfScope::Helo`]. + pub spf: Vec, +} + +/// Results for one group of messages sharing a source and identifiers +/// (`record` element). +/// +/// Build it from verification outputs with [`Record::with_dkim_output`], +/// [`Record::with_spf_output`], [`Record::with_dmarc_output`], +/// [`Record::with_dkim2_output`] and `with_arc_output`. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Record { + /// Message source, count and policy evaluation (`row`). + pub row: Row, + /// Message identifiers (`identifiers`). + pub identifiers: Identifiers, + /// Raw DKIM and SPF results (`auth_results`). + pub auth_results: AuthResults, + /// Record-level extensions (`extensions`). Not written by + /// [`AggregateReport::to_xml`]. + pub extensions: Vec, +} + +/// DMARC aggregate report (`feedback` element), as defined in RFC 9990. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct AggregateReport { + /// Report format version (`version`). `None` when the element is absent + /// or holds a version other than 1.0; not written in that case. + #[serde(default, deserialize_with = "deserialize_version")] + pub version: Option, + /// Reporting organization and period (`report_metadata`). + pub report_metadata: ReportMetadata, + /// Policy published by the domain owner (`policy_published`). + pub policy_published: PolicyPublished, + /// Per-source results, one per `record` element. + pub records: Vec, + /// Report-level extensions (`extensions`). Not written by + /// [`AggregateReport::to_xml`]. + pub extensions: Vec, +} + +/// Aggregate report format version (`version` and `version_published` +/// elements). +/// +/// Only version 1.0 is known. Parsing any other version, from XML or with +/// serde, yields `None` in the enclosing `Option`. Serde serializes +/// [`ReportVersion::V1`] as the number `1.0` and accepts `1`, `1.0` or the +/// string `"1.0"`. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub enum ReportVersion { + /// Version 1.0. + V1, +} + +impl ReportVersion { + /// Returns the version as written in XML, such as `"1.0"`. + pub fn as_str(&self) -> &'static str { + match self { + ReportVersion::V1 => "1.0", + } + } + + fn from_number(value: f64) -> Option { + (value == 1.0).then_some(ReportVersion::V1) + } +} + +impl Display for ReportVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ReportVersion { + type Err = (); + + fn from_str(s: &str) -> Result { + s.trim() + .parse::() + .ok() + .and_then(ReportVersion::from_number) + .ok_or(()) + } +} + +impl Serialize for ReportVersion { + fn serialize(&self, serializer: S) -> Result { + match self { + ReportVersion::V1 => serializer.serialize_f32(1.0), + } + } +} + +struct VersionVisitor; + +impl<'de> Visitor<'de> for VersionVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("a report version number") + } + + fn visit_f64(self, value: f64) -> Result { + Ok(ReportVersion::from_number(value)) + } + + fn visit_u64(self, value: u64) -> Result { + Ok((value == 1).then_some(ReportVersion::V1)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok((value == 1).then_some(ReportVersion::V1)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(value.parse().ok()) + } + + fn visit_none(self) -> Result { + Ok(None) + } + + fn visit_unit(self) -> Result { + Ok(None) + } + + fn visit_some>(self, deserializer: D) -> Result { + deserializer.deserialize_any(self) + } +} + +fn deserialize_version<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + if deserializer.is_human_readable() { + deserializer.deserialize_any(VersionVisitor) + } else { + Ok(Option::::deserialize(deserializer)? + .and_then(|value| ReportVersion::from_number(value.into()))) + } +} + +impl<'de> Deserialize<'de> for ReportVersion { + fn deserialize>(deserializer: D) -> Result { + let version = if deserializer.is_human_readable() { + deserializer.deserialize_any(VersionVisitor)? + } else { + ReportVersion::from_number(f32::deserialize(deserializer)?.into()) + }; + version.ok_or_else(|| de::Error::custom("unsupported report version")) + } +} diff --git a/crates/smtp-report/src/dmarc/parse.rs b/crates/smtp-report/src/dmarc/parse.rs new file mode 100644 index 00000000..1e11530d --- /dev/null +++ b/crates/smtp-report/src/dmarc/parse.rs @@ -0,0 +1,866 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Parsing of DMARC aggregate reports from XML documents and from report +//! email messages. + +use crate::{ + ReportError, + dmarc::{ + AggregateReport, AuthResults, DateRange, Discovery, Disposition, DkimAuthResult, + DkimStatus, DmarcStatus, Extension, Identifiers, PolicyEvaluated, PolicyOverride, + PolicyOverrideReason, PolicyPublished, Record, ReportMetadata, ReportVersion, Row, + SpfAuthResult, SpfScope, SpfStatus, + }, + read_capped, +}; +use flate2::read::GzDecoder; +use mail_auth::dmarc::{Alignment, Policy}; +use mail_parser::{MessageParser, PartKind}; +use quick_xml::XmlVersion; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::reader::Reader; +use std::borrow::Cow; +use std::io::{BufRead, Cursor}; +use std::net::IpAddr; +use std::str::FromStr; + +impl AggregateReport { + /// Extracts and parses the aggregate report attached to an RFC 5322 + /// message. + /// + /// Parts are tried in order: text parts with an `xml` subtype or `.xml` + /// file name, and binary parts whose subtype (`gzip`, `zip`, `xml`) or, + /// failing that, file extension (`.gz`, `.zip`, `.xml`) identifies the + /// report. The first part that parses is returned. `max_size` caps the + /// decompressed size of each gzip stream or zip member. + /// + /// # Errors + /// + /// - [`ReportError::MailParse`] if `report` is not a parseable message. + /// - [`ReportError::TooLarge`] if a decompressed report exceeds `max_size`. + /// - [`ReportError::Decompress`] if a gzip or zip attachment is corrupt. + /// - [`ReportError::Parse`] if every candidate part failed to parse; the + /// error of the last one is returned. + /// - [`ReportError::NotFound`] if the message has no candidate part. + pub fn parse_rfc5322(report: &[u8], max_size: usize) -> Result { + let message = MessageParser::new() + .parse(report) + .ok_or(ReportError::MailParse)?; + let mut error = ReportError::NotFound; + + for part in message.root().parts() { + match part.kind() { + PartKind::Text + if part + .content_type() + .and_then(|ct| ct.subtype()) + .is_some_and(|t| t.eq_ignore_ascii_case("xml")) + || part + .attachment_name() + .and_then(|n| n.rsplit_once('.')) + .is_some_and(|(_, e)| e.eq_ignore_ascii_case("xml")) => + { + let report = part.text().unwrap_or_default(); + match AggregateReport::parse_xml(report.as_bytes()) { + Ok(feedback) => return Ok(feedback), + Err(err) => { + error = err; + } + } + } + PartKind::Binary | PartKind::InlineBinary => { + enum ReportType { + Xml, + Gzip, + Zip, + } + + let (_, ext) = part + .attachment_name() + .unwrap_or("file.none") + .rsplit_once('.') + .unwrap_or(("file", "none")); + let subtype = part + .content_type() + .and_then(|ct| ct.subtype()) + .unwrap_or("none"); + let rt = if subtype.eq_ignore_ascii_case("gzip") { + ReportType::Gzip + } else if subtype.eq_ignore_ascii_case("zip") { + ReportType::Zip + } else if subtype.eq_ignore_ascii_case("xml") { + ReportType::Xml + } else if ext.eq_ignore_ascii_case("gz") { + ReportType::Gzip + } else if ext.eq_ignore_ascii_case("zip") { + ReportType::Zip + } else if ext.eq_ignore_ascii_case("xml") { + ReportType::Xml + } else { + continue; + }; + let report = part.decoded(); + + match rt { + ReportType::Gzip => { + let report: &[u8] = report.as_ref(); + let buf = read_capped(GzDecoder::new(report), 0, max_size)?; + + match AggregateReport::parse_xml(&buf) { + Ok(feedback) => return Ok(feedback), + Err(err) => { + error = err; + } + } + } + ReportType::Zip => { + let mut archive = zip::ZipArchive::new(Cursor::new(report)) + .map_err(|err| ReportError::Decompress(err.to_string()))?; + for i in 0..archive.len() { + match archive.by_index(i) { + Ok(mut file) => { + let size_hint = file.size(); + let buf = read_capped(&mut file, size_hint, max_size)?; + match AggregateReport::parse_xml(&buf) { + Ok(feedback) => return Ok(feedback), + Err(err) => { + error = err; + } + } + } + Err(err) => { + error = ReportError::Decompress(err.to_string()); + } + } + } + } + ReportType::Xml => match AggregateReport::parse_xml(&report) { + Ok(feedback) => return Ok(feedback), + Err(err) => { + error = err; + } + }, + } + } + _ => (), + } + } + + Err(error) + } + + /// Parses an aggregate report XML document. + /// + /// Accepts RFC 9990 reports and legacy RFC 7489 Appendix C reports, with + /// or without a default namespace. Unknown elements are skipped. + /// Unrecognized values map to the fallback documented on each field or + /// type, such as an `Unspecified` or `Other` variant or `None`. + /// + /// # Errors + /// + /// Returns [`ReportError::Parse`] if the XML is malformed, the root + /// element is not `feedback`, or `report_metadata` or `policy_published` + /// is missing. + pub fn parse_xml(report: &[u8]) -> Result { + Self::parse_xml_document(report).map_err(ReportError::Parse) + } + + fn parse_xml_document(report: &[u8]) -> Result { + let mut version = None; + let mut report_metadata = None; + let mut policy_published = None; + let mut records = Vec::new(); + let mut extensions = Vec::new(); + + let mut reader = Reader::from_reader(report); + reader.config_mut().trim_text(true); + + let mut buf = Vec::with_capacity(128); + let mut found_feedback = false; + + while let Some(tag) = reader.next_tag(&mut buf)? { + let name = tag.name(); + if found_feedback { + hashify::fnc_map!(name.as_ref().as_bytes(), + b"version" => { + version = reader.next_value::(&mut buf)?; + }, + b"report_metadata" => { + report_metadata = ReportMetadata::parse(&mut reader, &mut buf)?.into(); + }, + b"policy_published" => { + policy_published = PolicyPublished::parse(&mut reader, &mut buf)?.into(); + }, + b"record" => { + records.push(Record::parse(&mut reader, &mut buf)?); + }, + b"extensions" => { + Extension::parse(&mut reader, &mut buf, &mut extensions)?; + }, + b"" => (), + _ => { + reader.skip_tag(&mut buf)?; + } + ); + } else if name.as_ref() == "feedback" { + found_feedback = true; + } else if !name.as_ref().is_empty() { + return Err(format!( + "Unexpected tag {} at position {}.", + name.as_ref(), + reader.buffer_position() + )); + } + } + + Ok(AggregateReport { + version, + report_metadata: report_metadata.ok_or("Missing feedback/report_metadata tag.")?, + policy_published: policy_published.ok_or("Missing feedback/policy_published tag.")?, + records, + extensions, + }) + } +} + +impl ReportMetadata { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut rm = ReportMetadata::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"org_name" => { + rm.org_name = reader.next_value::(buf)?.unwrap_or_default(); + }, + b"email" => { + rm.email = reader.next_value::(buf)?.unwrap_or_default(); + }, + b"extra_contact_info" => { + rm.extra_contact_info = reader.next_value::(buf)?; + }, + b"report_id" => { + rm.report_id = reader.next_value::(buf)?.unwrap_or_default(); + }, + b"date_range" => { + rm.date_range = DateRange::parse(reader, buf)?; + }, + b"error" => { + if let Some(err) = reader.next_value::(buf)? { + rm.errors.push(err); + } + }, + b"generator" => { + rm.generator = reader.next_value::(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(rm) + } +} + +impl DateRange { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut dr = DateRange::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"begin" => { + dr.begin = reader.next_value(buf)?.unwrap_or_default(); + }, + b"end" => { + dr.end = reader.next_value(buf)?.unwrap_or_default(); + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(dr) + } +} + +impl PolicyPublished { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut p = PolicyPublished::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"domain" => { + p.domain = reader.next_value::(buf)?.unwrap_or_default(); + }, + b"version_published" => { + p.version_published = reader.next_value(buf)?; + }, + b"adkim" => { + p.adkim = reader.next_value::(buf)?.and_then(|v| parse_alignment(&v)); + }, + b"aspf" => { + p.aspf = reader.next_value::(buf)?.and_then(|v| parse_alignment(&v)); + }, + b"p" => { + p.p = reader.next_value::(buf)?.map_or(Policy::Unspecified, |v| parse_policy(&v)); + }, + b"sp" => { + p.sp = reader.next_value::(buf)?.map_or(Policy::Unspecified, |v| parse_policy(&v)); + }, + b"np" => { + p.np = reader.next_value::(buf)?.map_or(Policy::Unspecified, |v| parse_policy(&v)); + }, + b"discovery_method" => { + p.discovery_method = reader.next_value(buf)?.unwrap_or_default(); + }, + b"testing" => { + p.testing = reader + .next_value::(buf)? + .is_some_and(|s| s.eq_ignore_ascii_case("y")); + }, + b"fo" => { + p.fo = reader.next_value::(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(p) + } +} + +impl Extension { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + extensions: &mut Vec, + ) -> Result<(), String> { + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"extension" => { + let mut e = Extension::default(); + if let Ok(Some(attr)) = tag.try_get_attribute("name") + && let Ok(attr) = attr.normalized_value(XmlVersion::Implicit1_0) + { + e.name = attr.to_string(); + } + if let Ok(Some(attr)) = tag.try_get_attribute("definition") + && let Ok(attr) = attr.normalized_value(XmlVersion::Implicit1_0) + { + e.definition = attr.to_string(); + } + extensions.push(e); + reader.skip_tag(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(()) + } +} + +impl Record { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut r = Record::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"row" => { + r.row = Row::parse(reader, buf)?; + }, + b"identifiers" => { + r.identifiers = Identifiers::parse(reader, buf)?; + }, + b"auth_results" => { + r.auth_results = AuthResults::parse(reader, buf)?; + }, + b"extensions" => { + Extension::parse(reader, buf, &mut r.extensions)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(r) + } +} + +impl Row { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut r = Row::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"source_ip" => { + if let Some(ip) = reader.next_value::(buf)? { + r.source_ip = ip.into(); + } + }, + b"count" => { + r.count = reader.next_value(buf)?.unwrap_or_default(); + }, + b"policy_evaluated" => { + r.policy_evaluated = PolicyEvaluated::parse(reader, buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(r) + } +} + +impl PolicyEvaluated { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut pe = PolicyEvaluated::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"disposition" => { + pe.disposition = reader.next_value(buf)?.unwrap_or_default(); + }, + b"dkim" => { + pe.dkim = reader.next_value(buf)?.unwrap_or_default(); + }, + b"spf" => { + pe.spf = reader.next_value(buf)?.unwrap_or_default(); + }, + b"reason" => { + pe.reason.push(PolicyOverrideReason::parse(reader, buf)?); + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(pe) + } +} + +impl PolicyOverrideReason { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut por = PolicyOverrideReason::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"type" => { + por.kind = reader.next_value(buf)?.unwrap_or_default(); + }, + b"comment" => { + por.comment = reader.next_value(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(por) + } +} + +impl Identifiers { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut i = Identifiers::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"envelope_to" => { + i.envelope_to = reader.next_value(buf)?; + }, + b"envelope_from" => { + i.envelope_from = reader.next_value(buf)?.unwrap_or_default(); + }, + b"header_from" => { + i.header_from = reader.next_value(buf)?.unwrap_or_default(); + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(i) + } +} + +impl AuthResults { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut ar = AuthResults::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"dkim" => { + ar.dkim.push(DkimAuthResult::parse(reader, buf)?); + }, + b"spf" => { + ar.spf.push(SpfAuthResult::parse(reader, buf)?); + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(ar) + } +} + +impl DkimAuthResult { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut dar = DkimAuthResult::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"domain" => { + dar.domain = reader.next_value(buf)?.unwrap_or_default(); + }, + b"selector" => { + dar.selector = reader.next_value(buf)?.unwrap_or_default(); + }, + b"result" => { + dar.result = reader.next_value(buf)?.unwrap_or_default(); + }, + b"human_result" => { + dar.human_result = reader.next_value(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(dar) + } +} + +impl SpfAuthResult { + pub(crate) fn parse( + reader: &mut Reader, + buf: &mut Vec, + ) -> Result { + let mut sar = SpfAuthResult::default(); + + while let Some(tag) = reader.next_tag(buf)? { + let name = tag.name(); + hashify::fnc_map!(name.as_ref().as_bytes(), + b"domain" => { + sar.domain = reader.next_value(buf)?.unwrap_or_default(); + }, + b"scope" => { + sar.scope = reader.next_value(buf)?.unwrap_or_default(); + }, + b"result" => { + sar.result = reader.next_value(buf)?.unwrap_or_default(); + }, + b"human_result" => { + sar.human_result = reader.next_value(buf)?; + }, + b"" => (), + _ => { + reader.skip_tag(buf)?; + } + ); + } + + Ok(sar) + } +} + +impl FromStr for PolicyOverride { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), PolicyOverride, + b"trusted_forwarder" => PolicyOverride::TrustedForwarder, + b"mailing_list" => PolicyOverride::MailingList, + b"local_policy" => PolicyOverride::LocalPolicy, + b"policy_test_mode" => PolicyOverride::PolicyTestMode, + ) + .copied() + .unwrap_or(PolicyOverride::Other)) + } +} + +impl FromStr for Discovery { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), Discovery, + b"psl" => Discovery::Psl, + b"treewalk" => Discovery::Treewalk, + ) + .copied() + .unwrap_or(Discovery::Unspecified)) + } +} + +impl FromStr for DmarcStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), DmarcStatus, + b"pass" => DmarcStatus::Pass, + b"fail" => DmarcStatus::Fail, + ) + .copied() + .unwrap_or(DmarcStatus::Unspecified)) + } +} + +impl FromStr for DkimStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), DkimStatus, + b"none" => DkimStatus::None, + b"pass" => DkimStatus::Pass, + b"fail" => DkimStatus::Fail, + b"policy" => DkimStatus::Policy, + b"neutral" => DkimStatus::Neutral, + b"temperror" => DkimStatus::TempError, + b"permerror" => DkimStatus::PermError, + ) + .copied() + .unwrap_or(DkimStatus::None)) + } +} + +impl FromStr for SpfStatus { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), SpfStatus, + b"none" => SpfStatus::None, + b"pass" => SpfStatus::Pass, + b"fail" => SpfStatus::Fail, + b"softfail" => SpfStatus::SoftFail, + b"neutral" => SpfStatus::Neutral, + b"temperror" => SpfStatus::TempError, + b"permerror" => SpfStatus::PermError, + ) + .copied() + .unwrap_or(SpfStatus::None)) + } +} + +impl FromStr for SpfScope { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), SpfScope, + b"helo" => SpfScope::Helo, + b"mfrom" => SpfScope::MailFrom, + ) + .copied() + .unwrap_or(SpfScope::Unspecified)) + } +} + +impl FromStr for Disposition { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(hashify::map!(s.as_bytes(), Disposition, + b"none" => Disposition::None, + b"pass" => Disposition::Pass, + b"quarantine" => Disposition::Quarantine, + b"reject" => Disposition::Reject, + ) + .copied() + .unwrap_or(Disposition::Unspecified)) + } +} + +fn parse_policy(value: &str) -> Policy { + hashify::map!(value.as_bytes(), Policy, + b"none" => Policy::None, + b"quarantine" => Policy::Quarantine, + b"reject" => Policy::Reject, + ) + .copied() + .unwrap_or(Policy::Unspecified) +} + +fn parse_alignment(value: &str) -> Option { + match value.as_bytes().first() { + Some(b'r') => Some(Alignment::Relaxed), + Some(b's') => Some(Alignment::Strict), + _ => None, + } +} + +trait ReaderHelper { + fn next_tag<'x>(&mut self, buf: &'x mut Vec) -> Result>, String>; + fn next_value(&mut self, buf: &mut Vec) -> Result, String>; + fn skip_tag(&mut self, buf: &mut Vec) -> Result<(), String>; +} + +impl ReaderHelper for Reader { + fn next_tag<'x>(&mut self, buf: &'x mut Vec) -> Result>, String> { + match self.read_event_into(buf) { + Ok(Event::Start(e)) => Ok(Some(e)), + Ok(Event::End(_)) | Ok(Event::Eof) => Ok(None), + Err(e) => Err(format!( + "Error at position {}: {:?}", + self.buffer_position(), + e + )), + _ => Ok(Some(BytesStart::new(""))), + } + } + + fn next_value(&mut self, buf: &mut Vec) -> Result, String> { + let mut value: Option = None; + + loop { + match self.read_event_into(buf) { + Ok(Event::Text(e)) => { + let v = e.xml_content(XmlVersion::Implicit1_0); + if let Some(value) = &mut value { + value.push_str(&v); + } else { + value = Some(v.into_owned()); + } + } + Ok(Event::GeneralRef(e)) => { + let v = hashify::map!(e.as_bytes(), &'static str, + b"lt" => "<", + b"gt" => ">", + b"amp" => "&", + b"apos" => "'", + b"quot" => "\"", + ) + .copied() + .map(Cow::Borrowed) + .or_else(|| { + e.resolve_char_ref() + .ok() + .flatten() + .map(|v| Cow::Owned(v.to_string())) + }) + .unwrap_or_else(|| e.xml_content(XmlVersion::Implicit1_0)); + + if let Some(value) = &mut value { + value.push_str(&v); + } else { + value = Some(v.into_owned()); + } + } + Ok(Event::End(_)) => { + break; + } + Ok(Event::Start(e)) => { + return Err(format!( + "Expected value, found unexpected tag {} at position {}.", + e.name().as_ref(), + self.buffer_position() + )); + } + Ok(Event::Eof) => { + return Err(format!( + "Expected value, found unexpected EOF at position {}.", + self.buffer_position() + )); + } + _ => (), + } + } + + Ok(value.and_then(|v| T::from_str(&v).ok())) + } + + fn skip_tag(&mut self, buf: &mut Vec) -> Result<(), String> { + let mut tag_count = 0; + loop { + match self.read_event_into(buf) { + Ok(Event::End(_)) => { + if tag_count == 0 { + break; + } else { + tag_count -= 1; + } + } + Ok(Event::Start(_)) => { + tag_count += 1; + } + Ok(Event::Eof) => { + return Err(format!( + "Expected value, found unexpected EOF at position {}.", + self.buffer_position() + )); + } + _ => (), + } + } + Ok(()) + } +} + diff --git a/crates/smtp-report/src/headers.rs b/crates/smtp-report/src/headers.rs new file mode 100644 index 00000000..07119c6c --- /dev/null +++ b/crates/smtp-report/src/headers.rs @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::{memchr, memchr2}; + +pub(crate) struct HeaderIterator<'x> { + message: &'x [u8], + pos: usize, + start_pos: usize, +} + +enum FieldScan { + Named { colon: usize, end: usize }, + Unnamed { end: usize }, + End { pos: usize }, +} + +#[inline(always)] +fn scan_field(message: &[u8], start_pos: usize, from: usize) -> FieldScan { + let mut cur = from; + + while let Some(rest) = message.get(cur..) { + let Some(offset) = memchr2(b':', b'\n', rest) else { + break; + }; + let Some((head, tail)) = rest.split_at_checked(offset) else { + break; + }; + let pos = cur + offset; + + if tail.first() == Some(&b':') { + return scan_value(message, pos); + } else if head.last() == Some(&b'\r') || pos == start_pos { + return FieldScan::End { pos: pos + 1 }; + } + + match message.get(pos + 1) { + Some(b' ' | b'\t') => cur = pos + 1, + _ => return FieldScan::Unnamed { end: pos + 1 }, + } + } + + FieldScan::End { pos: message.len() } +} + +#[inline(always)] +fn scan_value(message: &[u8], colon: usize) -> FieldScan { + let mut cur = colon + 1; + + while let Some(rest) = message.get(cur..) { + let Some(offset) = memchr(b'\n', rest) else { + break; + }; + let pos = cur + offset; + + match message.get(pos + 1) { + Some(b' ' | b'\t') => cur = pos + 1, + _ => { + return FieldScan::Named { + colon, + end: pos + 1, + }; + } + } + } + + FieldScan::End { pos: message.len() } +} + +impl<'x> HeaderIterator<'x> { + pub fn new(message: &'x [u8]) -> Self { + HeaderIterator { + message, + pos: 0, + start_pos: 0, + } + } + + pub fn seek_start(&mut self) { + let rest = self.message.get(self.pos..).unwrap_or_default(); + self.pos += rest + .iter() + .position(|ch| !ch.is_ascii_whitespace()) + .unwrap_or(rest.len()); + } +} + +impl<'x> Iterator for HeaderIterator<'x> { + type Item = (&'x [u8], &'x [u8]); + + fn next(&mut self) -> Option { + match scan_field(self.message, self.start_pos, self.pos) { + FieldScan::Named { colon, end } => { + let header_name = self.message.get(self.start_pos..colon).unwrap_or_default(); + let header_value = self.message.get(colon + 1..end).unwrap_or_default(); + + self.start_pos = end; + self.pos = end; + + Some((header_name, header_value)) + } + FieldScan::Unnamed { end } => { + let header_name = self.message.get(self.start_pos..end).unwrap_or_default(); + + self.start_pos = end; + self.pos = end; + + Some((header_name, b"")) + } + FieldScan::End { pos } => { + self.pos = pos; + + None + } + } + } +} diff --git a/crates/smtp-report/src/lib.rs b/crates/smtp-report/src/lib.rs new file mode 100644 index 00000000..66346a40 --- /dev/null +++ b/crates/smtp-report/src/lib.rs @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Parsing and generation of email authentication reports. +//! +//! Three report formats are supported, one per submodule: +//! +//! - [`dmarc`]: DMARC aggregate reports ([`dmarc::AggregateReport`], RFC 9990). +//! - [`arf`]: Abuse Reporting Format feedback reports, including +//! authentication failure reports ([`arf::FeedbackReport`], RFC 5965 and RFC 6591). +//! - [`tlsrpt`]: SMTP TLS reports ([`tlsrpt::TlsReport`], RFC 8460). +//! +//! Each report type has a `parse_rfc5322` function that extracts the report +//! from a complete email message, a function that parses the bare report +//! document (`parse_xml`, `parse_arf` or `parse_json`), and `write_rfc5322` and +//! `to_rfc5322` methods that wrap the report in a new email message. The +//! addressing of that message is taken from a [`ReportEnvelope`]. Parse +//! failures are reported as [`ReportError`]. + +pub mod arf; +pub mod dmarc; +mod headers; +pub mod tlsrpt; +mod utf8; + +pub use flate2; +pub use zip; + +use mail_builder::headers::{HeaderType, address::Address}; +use std::{fmt::Display, io::Read}; + +/// Addressing and identification of a generated report message. +/// +/// Passed to the `write_rfc5322` and `to_rfc5322` methods of every report +/// type. Not every report type reads every field; see the documentation of +/// each `write_rfc5322` method for the fields it uses. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReportEnvelope<'x> { + /// Address written to the `From` header field. + pub from: Address<'x>, + /// Recipient addresses written to the `To` header field. + pub to: Vec<&'x str>, + /// Domain of the organization submitting the report. + /// + /// Used as the `Message-ID` host, in the default subject, the text body + /// and the attachment file name. + pub submitter: &'x str, + /// Domain the report is about (TLS-RPT only). + /// + /// Written to the `TLS-Report-Domain` header field, the default subject + /// and the attachment file name. DMARC and ARF reports ignore it. + pub report_domain: &'x str, + /// Subject of the message. When `None`, each report type writes its own + /// default subject. + pub subject: Option<&'x str>, +} + +impl<'x> ReportEnvelope<'x> { + pub(crate) fn to_header(&self) -> HeaderType<'x> { + HeaderType::Address(Address::List( + self.to.iter().map(|to| (*to).into()).collect(), + )) + } +} + +/// Error returned when a report cannot be parsed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReportError { + /// The input is not a parseable RFC 5322 message. + MailParse, + /// The report document is malformed or lacks a required element. Holds + /// the parser's error message. + Parse(String), + /// A gzip or zip attachment could not be decompressed. Holds the + /// decompressor's error message. + Decompress(String), + /// The report exceeds the `max_size` passed to the parse function. + TooLarge, + /// The message contains no part that looks like a report. + NotFound, +} + +impl Display for ReportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ReportError::MailParse => f.write_str("Failed to parse the report message"), + ReportError::Parse(err) => write!(f, "Failed to parse the report: {err}"), + ReportError::Decompress(err) => write!(f, "Failed to decompress the report: {err}"), + ReportError::TooLarge => f.write_str("Report exceeds the maximum allowed size"), + ReportError::NotFound => f.write_str("No report found in the message"), + } + } +} + +impl std::error::Error for ReportError {} + +const MAX_SIZE_RESERVATION: u64 = 64 * 1024; + +pub(crate) fn read_capped( + reader: impl Read, + size_hint: u64, + max_size: usize, +) -> Result, ReportError> { + let max_size = max_size as u64; + if size_hint > max_size { + return Err(ReportError::TooLarge); + } + + let mut buf = Vec::with_capacity(size_hint.min(MAX_SIZE_RESERVATION) as usize); + reader + .take(max_size.saturating_add(1)) + .read_to_end(&mut buf) + .map_err(|err| ReportError::Decompress(err.to_string()))?; + + if buf.len() as u64 > max_size { + return Err(ReportError::TooLarge); + } + + Ok(buf) +} + +impl From for ReportError { + fn from(err: String) -> Self { + ReportError::Parse(err) + } +} + + diff --git a/crates/smtp-report/src/tlsrpt/generate.rs b/crates/smtp-report/src/tlsrpt/generate.rs new file mode 100644 index 00000000..16a5e508 --- /dev/null +++ b/crates/smtp-report/src/tlsrpt/generate.rs @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Serialization of [`TlsReport`] to JSON and to a complete TLS report email +//! message. + +use super::{DateRange, TlsReport}; +use crate::ReportEnvelope; +use flate2::{Compression, write::GzEncoder}; +use mail_builder::{ + MessageBuilder, + headers::{HeaderType, content_type::ContentType}, + mime::{BodyPart, MimePart, make_boundary}, +}; +use std::{borrow::Cow, io}; + +#[derive(serde::Deserialize)] +struct ReportHeader { + #[serde(rename = "report-id")] + #[serde(default)] + report_id: String, + #[serde(rename = "date-range")] + date_range: DateRange, +} + +impl TlsReport { + /// Writes the report as an RFC 5322 `multipart/report` message with + /// `report-type=tlsrpt` to `writer`, as described in RFC 8460 section 5.3. + /// + /// The parts are a `text/plain` summary and the output of + /// [`to_json`](Self::to_json), gzip-compressed, as an + /// `application/tlsrpt+gzip` attachment named + /// `submitter!report_domain!start!end.json.gz`, where `start` and `end` + /// are Unix timestamps. The message carries `TLS-Report-Domain`, + /// `TLS-Report-Submitter` and `Auto-Submitted: auto-generated` header + /// fields. + /// + /// Fields read from `envelope`: + /// + /// - `from`: the `From` header field. + /// - `to`: the `To` header field. + /// - `submitter`: the `TLS-Report-Submitter` header field, the + /// `Message-ID` host, the subject, the text body and the file name. + /// - `report_domain`: the `TLS-Report-Domain` header field, the subject, + /// the text body and the file name. + /// - `subject`: the subject; when `None`, defaults to + /// `Report Domain: Submitter: Report-ID: <>`. + /// + /// # Errors + /// + /// Returns any I/O error raised by `writer` or by the gzip encoder. + pub fn write_rfc5322( + &self, + envelope: &ReportEnvelope<'_>, + writer: impl io::Write, + ) -> io::Result<()> { + let json = self.to_json(); + let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); + io::Write::write_all(&mut e, json.as_bytes())?; + let bytes = e.finish()?; + Self::write_message( + envelope, + &self.report_id, + self.date_range.start_datetime.to_timestamp(), + self.date_range.end_datetime.to_timestamp(), + &bytes, + writer, + ) + } + + /// Writes a report that is already serialized as uncompressed JSON as an + /// RFC 5322 message to `writer`. + /// + /// Produces the same message as [`write_rfc5322`](Self::write_rfc5322) + /// and reads the same `envelope` fields. Only the `report-id` and + /// `date-range` members are read from `json`; the document is gzipped and + /// attached as is. + /// + /// # Errors + /// + /// Returns an error of kind [`io::ErrorKind::InvalidData`] if `json` is + /// not a JSON object with a valid `date-range` member, and any I/O error + /// raised by `writer` or by the gzip encoder. + pub fn write_rfc5322_json( + json: &[u8], + envelope: &ReportEnvelope<'_>, + writer: impl io::Write, + ) -> io::Result<()> { + let report = serde_json::from_slice::(json) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); + io::Write::write_all(&mut e, json)?; + let bytes = e.finish()?; + Self::write_message( + envelope, + &report.report_id, + report.date_range.start_datetime.to_timestamp(), + report.date_range.end_datetime.to_timestamp(), + &bytes, + writer, + ) + } + + fn write_message( + envelope: &ReportEnvelope<'_>, + report_id: &str, + start: i64, + end: i64, + bytes: &[u8], + writer: impl io::Write, + ) -> io::Result<()> { + let report_domain = envelope.report_domain; + let submitter = envelope.submitter; + let subject = envelope.subject.map_or_else( + || { + Cow::Owned(format!( + "Report Domain: {report_domain} Submitter: {submitter} Report-ID: <{report_id}>" + )) + }, + Cow::Borrowed, + ); + + MessageBuilder::new() + .from(envelope.from.clone()) + .header("To", envelope.to_header()) + .message_id(format!("{}@{}", make_boundary("."), submitter)) + .header("TLS-Report-Domain", HeaderType::Text(report_domain.into())) + .header("TLS-Report-Submitter", HeaderType::Text(submitter.into())) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .subject(subject) + .body(MimePart::new( + ContentType::new("multipart/report").attribute("report-type", "tlsrpt"), + BodyPart::Multipart(vec![ + MimePart::new( + ContentType::new("text/plain"), + BodyPart::Text( + format!( + concat!( + "TLS report from {}\r\n\r\n", + "Report Domain: {}\r\n", + "Submitter: {}\r\n", + "Report-ID: {}\r\n", + ), + submitter, report_domain, submitter, report_id + ) + .into(), + ), + ), + MimePart::new( + ContentType::new("application/tlsrpt+gzip"), + BodyPart::Binary(bytes.into()), + ) + .attachment(format!("{submitter}!{report_domain}!{start}!{end}.json.gz")), + ]), + )) + .write_to(writer) + } + + /// Returns the report as an RFC 5322 message. + /// + /// Same as [`write_rfc5322`](Self::write_rfc5322), collected into a + /// `String`. + /// + /// # Errors + /// + /// Returns an error if the gzip encoder fails or the message is not valid + /// UTF-8. + pub fn to_rfc5322(&self, envelope: &ReportEnvelope<'_>) -> io::Result { + let mut buf = Vec::new(); + self.write_rfc5322(envelope, &mut buf)?; + String::from_utf8(buf).map_err(io::Error::other) + } + + /// Serializes the report as an RFC 8460 JSON document. + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } +} + diff --git a/crates/smtp-report/src/tlsrpt/mod.rs b/crates/smtp-report/src/tlsrpt/mod.rs new file mode 100644 index 00000000..2d203f24 --- /dev/null +++ b/crates/smtp-report/src/tlsrpt/mod.rs @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! SMTP TLS reports (RFC 8460). +//! +//! [`TlsReport`] models the JSON report document of RFC 8460 section 4. +//! Reports are parsed with [`TlsReport::parse_rfc5322`] or +//! [`TlsReport::parse_json`] and written with [`TlsReport::write_rfc5322`], +//! [`TlsReport::write_rfc5322_json`] or [`TlsReport::to_json`]. Serde +//! renames every field to its JSON member name, which the field +//! documentation also gives. + +use mail_parser::DateTime; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{self, Visitor}, +}; +use std::{fmt, net::IpAddr}; + +mod generate; +mod parse; + +/// SMTP TLS report (RFC 8460 section 4). +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct TlsReport { + /// Name of the reporting organization (`organization-name`). + #[serde(rename = "organization-name")] + #[serde(default)] + pub organization_name: Option, + + /// Reporting period (`date-range`). + #[serde(rename = "date-range")] + pub date_range: DateRange, + + /// Contact address of the reporting organization (`contact-info`). + #[serde(rename = "contact-info")] + #[serde(default)] + pub contact_info: Option, + + /// Unique identifier of the report (`report-id`). + #[serde(rename = "report-id")] + #[serde(default)] + pub report_id: String, + + /// Results per applied policy (`policies`). + #[serde(rename = "policies")] + #[serde(default)] + pub policies: Vec, +} + +/// Entry of the `policies` array: a policy with its session counts and +/// failure details. +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct PolicyResult { + /// The policy that was applied (`policy`). + #[serde(rename = "policy")] + pub policy: PolicyDetails, + + /// Session counts (`summary`). + #[serde(rename = "summary")] + pub summary: Summary, + + /// Failed sessions grouped by failure (`failure-details`). + #[serde(rename = "failure-details")] + #[serde(default)] + pub failure_details: Vec, +} + +/// Description of an applied policy (`policy` object). +#[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct PolicyDetails { + /// Kind of policy (`policy-type`). + #[serde(rename = "policy-type")] + pub policy_type: PolicyType, + + /// Policy text, one entry per line or DANE TLSA record + /// (`policy-string`). + #[serde(rename = "policy-string")] + #[serde(default)] + pub policy_string: Vec, + + /// Domain the policy applies to (`policy-domain`). + #[serde(rename = "policy-domain")] + #[serde(default)] + pub policy_domain: String, + + /// MX host patterns listed in an MTA-STS policy (`mx-host`). + #[serde(rename = "mx-host")] + #[serde(default)] + pub mx_host: Vec, +} + +/// Session counts for one policy (`summary` object). +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct Summary { + /// Number of sessions that established TLS successfully + /// (`total-successful-session-count`). + #[serde(rename = "total-successful-session-count")] + #[serde(default)] + pub successful_sessions: u32, + + /// Number of sessions that failed (`total-failure-session-count`). + #[serde(rename = "total-failure-session-count")] + #[serde(default)] + pub failed_sessions: u32, +} + +/// Group of failed sessions sharing a failure type and endpoints (entry of +/// the `failure-details` array). +#[derive(Debug, Default, Hash, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct FailureDetails { + /// Kind of failure (`result-type`). + #[serde(rename = "result-type")] + pub result_type: FailureType, + + /// IP address of the sending MTA (`sending-mta-ip`). An empty or + /// unparseable address becomes `None`. + #[serde(rename = "sending-mta-ip")] + #[serde(default)] + #[serde(deserialize_with = "deserialize_ip")] + pub sending_mta_ip: Option, + + /// Host name of the receiving MX (`receiving-mx-hostname`). + #[serde(rename = "receiving-mx-hostname")] + pub receiving_mx_hostname: Option, + + /// Greeting name announced by the receiving MX (`receiving-mx-helo`). + #[serde(rename = "receiving-mx-helo")] + pub receiving_mx_helo: Option, + + /// IP address of the receiving MX (`receiving-ip`). An empty or + /// unparseable address becomes `None`. + #[serde(rename = "receiving-ip")] + #[serde(default)] + #[serde(deserialize_with = "deserialize_ip")] + pub receiving_ip: Option, + + /// Number of sessions that failed this way (`failed-session-count`). + #[serde(rename = "failed-session-count")] + #[serde(default)] + pub failed_session_count: u32, + + /// URI pointing to more information about the failure + /// (`additional-information`). + #[serde(rename = "additional-information")] + pub additional_information: Option, + + /// Free-form failure reason, such as a TLS alert (`failure-reason-code`). + #[serde(rename = "failure-reason-code")] + pub failure_reason_code: Option, +} + +/// Reporting period of a TLS report (`date-range` object). +/// +/// Both ends are RFC 3339 timestamps in JSON. A timestamp that does not parse +/// becomes the Unix epoch. +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct DateRange { + /// Start of the period (`start-datetime`). + #[serde(rename = "start-datetime")] + #[serde(serialize_with = "serialize_datetime")] + #[serde(deserialize_with = "deserialize_datetime")] + pub start_datetime: DateTime, + /// End of the period (`end-datetime`). + #[serde(rename = "end-datetime")] + #[serde(serialize_with = "serialize_datetime")] + #[serde(deserialize_with = "deserialize_datetime")] + pub end_datetime: DateTime, +} + +/// Kind of policy (`policy-type` member). +#[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize, Clone, Copy)] +pub enum PolicyType { + /// DANE TLSA policy (`tlsa`). + #[serde(rename = "tlsa")] + Tlsa, + /// MTA-STS policy (`sts`). + #[serde(rename = "sts")] + Sts, + /// No policy was found (`no-policy-found`). + #[serde(rename = "no-policy-found")] + NoPolicyFound, + /// An unrecognized policy type. + #[serde(other)] + #[default] + Other, +} + +/// Kind of TLS negotiation failure (`result-type` member, RFC 8460 section +/// 4.3). +#[derive(Debug, Default, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum FailureType { + /// The receiving MX does not support STARTTLS (`starttls-not-supported`). + #[serde(rename = "starttls-not-supported")] + StartTlsNotSupported, + /// The certificate does not match the host name + /// (`certificate-host-mismatch`). + #[serde(rename = "certificate-host-mismatch")] + CertificateHostMismatch, + /// The certificate has expired (`certificate-expired`). + #[serde(rename = "certificate-expired")] + CertificateExpired, + /// The certificate chain is not trusted (`certificate-not-trusted`). + #[serde(rename = "certificate-not-trusted")] + CertificateNotTrusted, + /// Any other validation failure (`validation-failure`). + #[serde(rename = "validation-failure")] + ValidationFailure, + /// The TLSA record is invalid (`tlsa-invalid`). + #[serde(rename = "tlsa-invalid")] + TlsaInvalid, + /// DNSSEC validation failed (`dnssec-invalid`). + #[serde(rename = "dnssec-invalid")] + DnssecInvalid, + /// DANE is required but no DNSSEC-validated TLSA records were found + /// (`dane-required`). + #[serde(rename = "dane-required")] + DaneRequired, + /// The MTA-STS policy could not be fetched (`sts-policy-fetch-error`). + #[serde(rename = "sts-policy-fetch-error")] + StsPolicyFetchError, + /// The MTA-STS policy is invalid (`sts-policy-invalid`). + #[serde(rename = "sts-policy-invalid")] + StsPolicyInvalid, + /// The MTA-STS policy host failed PKIX validation + /// (`sts-webpki-invalid`). + #[serde(rename = "sts-webpki-invalid")] + StsWebpkiInvalid, + /// An unrecognized result type. + #[serde(other)] + #[default] + Other, +} + +fn deserialize_datetime<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Ok( + DateTime::parse_rfc3339(Deserialize::deserialize(deserializer)?) + .unwrap_or_else(|| DateTime::from_timestamp(0)), + ) +} + +fn deserialize_ip<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_option(IpVisitor) +} + +struct IpVisitor; + +impl<'de> Visitor<'de> for IpVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an IP address string or null") + } + + fn visit_str(self, value: &str) -> Result { + Ok(value.parse().ok()) + } + + fn visit_none(self) -> Result { + Ok(None) + } + + fn visit_unit(self) -> Result { + Ok(None) + } + + fn visit_some>(self, deserializer: D) -> Result { + deserializer.deserialize_str(self) + } +} + +fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&datetime.to_rfc3339()) +} + +impl PolicyDetails { + /// Returns a policy of the given type for `policy_domain`, with no policy + /// strings and no MX hosts. + pub fn new(policy_type: PolicyType, policy_domain: impl Into) -> Self { + Self { + policy_type, + policy_string: vec![], + policy_domain: policy_domain.into(), + mx_host: vec![], + } + } +} + +impl FailureDetails { + /// Returns failure details of the given type with every other field + /// unset and a zero session count. + pub fn new(result_type: impl Into) -> Self { + FailureDetails { + result_type: result_type.into(), + ..Default::default() + } + } +} + +impl DateRange { + /// Builds a date range from two timestamps in seconds since the Unix + /// epoch. + pub fn from_timestamps(start_datetime: i64, end_datetime: i64) -> Self { + Self { + start_datetime: DateTime::from_timestamp(start_datetime), + end_datetime: DateTime::from_timestamp(end_datetime), + } + } +} diff --git a/crates/smtp-report/src/tlsrpt/parse.rs b/crates/smtp-report/src/tlsrpt/parse.rs new file mode 100644 index 00000000..a3c3a7a9 --- /dev/null +++ b/crates/smtp-report/src/tlsrpt/parse.rs @@ -0,0 +1,137 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! Parsing of TLS reports from JSON documents and from report email +//! messages. + +use super::TlsReport; +use crate::{ReportError, read_capped}; +use flate2::read::GzDecoder; +use mail_parser::{MessageParser, PartKind}; +use std::io::Cursor; +use zip::ZipArchive; + +impl TlsReport { + /// Parses an RFC 8460 JSON report document. + /// + /// Unknown members are ignored. Unrecognized `policy-type` and + /// `result-type` values map to the `Other` variant. + /// + /// # Errors + /// + /// Returns [`ReportError::Parse`] if the JSON is malformed or a required + /// member is missing: `date-range`, or `policy`, `summary`, `policy-type` + /// or `result-type` inside a `policies` entry. + pub fn parse_json(report: &[u8]) -> Result { + serde_json::from_slice(report).map_err(|err| ReportError::Parse(err.to_string())) + } + + /// Extracts and parses the TLS report attached to an RFC 5322 message. + /// + /// Binary parts are tried in order when their subtype (`tlsrpt+gzip`, + /// `tlsrpt+zip`, `tlsrpt+json`) or, failing that, file extension (`.gz`, + /// `.zip`, `.json`) identifies the report. The first part that parses is + /// returned. `max_size` caps the decompressed size of each gzip stream or + /// zip member. + /// + /// # Errors + /// + /// - [`ReportError::MailParse`] if `report` is not a parseable message. + /// - [`ReportError::TooLarge`] if a decompressed report exceeds `max_size`. + /// - [`ReportError::Decompress`] if a gzip or zip attachment is corrupt. + /// - [`ReportError::Parse`] if every candidate part failed to parse; the + /// error of the last one is returned. + /// - [`ReportError::NotFound`] if the message has no candidate part. + pub fn parse_rfc5322(report: &[u8], max_size: usize) -> Result { + let message = MessageParser::new() + .parse(report) + .ok_or(ReportError::MailParse)?; + let mut error = ReportError::NotFound; + + for part in message.root().parts() { + match part.kind() { + PartKind::Binary | PartKind::InlineBinary => { + enum ReportType { + Json, + Gzip, + Zip, + } + + let (_, ext) = part + .attachment_name() + .unwrap_or("file.none") + .rsplit_once('.') + .unwrap_or(("file", "none")); + let subtype = part + .content_type() + .and_then(|ct| ct.subtype()) + .unwrap_or("none"); + let rt = if subtype.eq_ignore_ascii_case("tlsrpt+gzip") { + ReportType::Gzip + } else if subtype.eq_ignore_ascii_case("tlsrpt+zip") { + ReportType::Zip + } else if subtype.eq_ignore_ascii_case("tlsrpt+json") { + ReportType::Json + } else if ext.eq_ignore_ascii_case("gz") { + ReportType::Gzip + } else if ext.eq_ignore_ascii_case("zip") { + ReportType::Zip + } else if ext.eq_ignore_ascii_case("json") { + ReportType::Json + } else { + continue; + }; + let report = part.decoded(); + + match rt { + ReportType::Gzip => { + let report: &[u8] = report.as_ref(); + let buf = read_capped(GzDecoder::new(report), 0, max_size)?; + + match Self::parse_json(&buf) { + Ok(report) => return Ok(report), + Err(err) => { + error = err; + } + } + } + ReportType::Zip => { + let mut archive = ZipArchive::new(Cursor::new(report)) + .map_err(|err| ReportError::Decompress(err.to_string()))?; + for i in 0..archive.len() { + match archive.by_index(i) { + Ok(mut file) => { + let size_hint = file.size(); + let buf = read_capped(&mut file, size_hint, max_size)?; + match Self::parse_json(&buf) { + Ok(report) => return Ok(report), + Err(err) => { + error = err; + } + } + } + Err(err) => { + error = ReportError::Decompress(err.to_string()); + } + } + } + } + ReportType::Json => match Self::parse_json(&report) { + Ok(report) => return Ok(report), + Err(err) => { + error = err; + } + }, + } + } + _ => (), + } + } + + Err(error) + } +} + diff --git a/crates/smtp-report/src/utf8.rs b/crates/smtp-report/src/utf8.rs new file mode 100644 index 00000000..6f3a6b30 --- /dev/null +++ b/crates/smtp-report/src/utf8.rs @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{borrow::Cow, string::FromUtf8Error}; + +#[inline] +pub(crate) fn into_string(bytes: Vec) -> Result { + if simdutf8::basic::from_utf8(&bytes).is_ok() { + // SAFETY: simdutf8 has just validated the whole buffer as UTF-8 and the bytes are moved unchanged. + Ok(unsafe { String::from_utf8_unchecked(bytes) }) + } else { + String::from_utf8(bytes) + } +} + +#[inline] +pub(crate) fn into_string_lossy(bytes: Vec) -> String { + into_string(bytes).unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) +} + +#[inline] +pub(crate) fn to_str_lossy(bytes: &[u8]) -> Cow<'_, str> { + match simdutf8::basic::from_utf8(bytes) { + Ok(text) => Cow::Borrowed(text), + Err(_) => String::from_utf8_lossy(bytes), + } +} diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 17e6436b..a113ba51 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -20,33 +20,40 @@ email = { path = "../email" } registry = { path = "../registry" } spam-filter = { path = "../spam-filter" } trc = { path = "../trc" } -mail-auth = { version = "0.13", features = ["rkyv"] } -mail-parser = { version = "0.11", features = ["full_encoding"] } -mail-builder = { version = "1.0" } -smtp-proto = { version = "0.2.3", features = ["rkyv", "serde"] } -sieve-rs = { version = "0.7", features = ["rkyv"] } +mail-auth = { version = "0.14" } +smtp-report = { path = "../smtp-report" } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +smtp-proto = { path = "../smtp-proto", features = ["rkyv", "serde"] } +sieve-rs = { path = "../sieve" } ahash = { version = "0.8" } -rustls = { version = "0.23.43", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +rustls = { version = "0.23.46", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } rustls-pki-types = { version = "1", features = ["std"] } rustls-webpki = { version = "0.103", default-features = false, features = ["std", "aws-lc-rs"] } -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } sha2 = "0.11" -parking_lot = "0.12" +parking_lot = "0.13" blake3 = "1.8" -rand = "0.10.2" +rand = "0.10.3" x509-parser = { version = "0.18", features = ["verify-aws"] } reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"] } serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = "0.10.0" -hashify = { version = "0.2" } -base64 = "0.23" +hashify = { version = "0.3" } +encodify = "1.0" +memchr = "2.8" + +[target.'cfg(stalwart_tests)'.dependencies] +mail-auth = { version = "0.14", features = ["test"] } + +[dev-dependencies] +mail-auth = { version = "0.14", features = ["test"] } [features] -test_mode = ["mail-auth/test"] -enterprise = [] +enterprise = ["store/enterprise", "utils/enterprise", "directory/enterprise", "common/enterprise", "email/enterprise", "registry/enterprise", "spam-filter/enterprise", "trc/enterprise"] #[[bench]] #name = "hash" diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 485b3c3e..fdc60b0a 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -9,7 +9,8 @@ use common::{ Inner, Server, auth::AccountInfo, config::smtp::auth::VerifyStrategy, - network::{ServerInstance, asn::AsnGeoLookupResult}, + expr::{Bump, Variable, if_block::IfBlock}, + network::{ServerInstance, SessionStream, asn::AsnGeoLookupResult, memory::MemoryReservation}, }; use email::message::delivery::ORCPT_ADDR_TYPE; use mail_auth::{IprevOutput, SpfOutput}; @@ -41,15 +42,24 @@ impl SmtpSessionManager { pub enum State { Request(RequestReceiver), - Bdat(BdatReceiver), + Bdat { + receiver: BdatReceiver, + bytes_left: usize, + }, Data(DataReceiver), Sasl(LineReceiver), - SkipData(DummyDataReceiver, &'static [u8]), + SkipData(DummyDataReceiver, &'static [u8], DataFailure), RequestTooLarge(DummyLineReceiver), Accepted(QueueId), None, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataFailure { + Chunk, + Transaction, +} + pub struct Session { pub hostname: String, pub state: State, @@ -76,6 +86,8 @@ pub struct SessionData { pub rcpt_errors: usize, pub rcpt_oks: usize, pub message: Vec, + pub message_size: usize, + pub message_memory: Option, pub authenticated_as: Option, pub auth_errors: usize, @@ -159,6 +171,8 @@ impl SessionData { rcpt_errors: 0, rcpt_oks: 0, message: Vec::with_capacity(0), + message_size: 0, + message_memory: None, auth_errors: 0, messages_sent: 0, bytes_left: 0, @@ -207,6 +221,18 @@ impl PartialOrd for SessionAddress { } } +impl Session { + pub async fn eval_if(&self, if_block: &IfBlock) -> Option + where + R: for<'b> TryFrom>, + { + let mut arena = Bump::new(); + self.server + .eval_if(if_block, self, &mut arena, self.data.session_id) + .await + } +} + impl Session { pub fn local( server: Server, @@ -279,6 +305,8 @@ impl SessionData { rcpt_errors: 0, rcpt_oks: 0, message, + message_size: 0, + message_memory: None, authenticated_as: Some(authenticated_as), auth_errors: 0, priority: 0, @@ -312,8 +340,14 @@ impl SessionAddress { } pub fn orcpt_parameter(&self) -> Option { - self.dsn_info - .as_deref() - .map(|orcpt| format!("{ORCPT_ADDR_TYPE}{}", orcpt.to_lowercase())) + self.dsn_info.as_deref().map(|orcpt| { + if orcpt.is_ascii() { + let mut parameter = format!("{ORCPT_ADDR_TYPE}{orcpt}"); + parameter.make_ascii_lowercase(); + parameter + } else { + format!("{ORCPT_ADDR_TYPE}{}", orcpt.to_lowercase()) + } + }) } } diff --git a/crates/smtp/src/core/params.rs b/crates/smtp/src/core/params.rs index 5e15f3f4..ef71134b 100644 --- a/crates/smtp/src/core/params.rs +++ b/crates/smtp/src/core/params.rs @@ -12,143 +12,73 @@ impl Session { pub async fn eval_session_params(&mut self) { let c = &self.server.core.smtp.session; self.data.bytes_left = self - .server - .eval_if(&c.transfer_limit, self, self.data.session_id) + .eval_if(&c.transfer_limit) .await .unwrap_or(250 * 1024 * 1024); self.data.valid_until += self - .server - .eval_if(&c.duration, self, self.data.session_id) + .eval_if(&c.duration) .await .unwrap_or_else(|| Duration::from_secs(15 * 60)); self.params.timeout = self - .server - .eval_if(&c.timeout, self, self.data.session_id) + .eval_if(&c.timeout) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); self.params.spf_ehlo = self - .server - .eval_if( - &self.server.core.smtp.mail_auth.spf.verify_ehlo, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.mail_auth.spf.verify_ehlo) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.spf_mail_from = self - .server - .eval_if( - &self.server.core.smtp.mail_auth.spf.verify_mail_from, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.mail_auth.spf.verify_mail_from) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.iprev = self - .server - .eval_if( - &self.server.core.smtp.mail_auth.iprev.verify, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.mail_auth.iprev.verify) .await .unwrap_or(VerifyStrategy::Relaxed); // Ehlo parameters let ec = &self.server.core.smtp.session.ehlo; - self.params.ehlo_require = self - .server - .eval_if(&ec.require, self, self.data.session_id) - .await - .unwrap_or(true); - self.params.ehlo_reject_non_fqdn = self - .server - .eval_if(&ec.reject_non_fqdn, self, self.data.session_id) - .await - .unwrap_or(true); + self.params.ehlo_require = self.eval_if(&ec.require).await.unwrap_or(true); + self.params.ehlo_reject_non_fqdn = self.eval_if(&ec.reject_non_fqdn).await.unwrap_or(true); // Auth parameters let ac = &self.server.core.smtp.session.auth; - self.params.auth_require = self - .server - .eval_if(&ac.require, self, self.data.session_id) - .await - .unwrap_or(false); - self.params.auth_errors_max = self - .server - .eval_if(&ac.errors_max, self, self.data.session_id) - .await - .unwrap_or(3); + self.params.auth_require = self.eval_if(&ac.require).await.unwrap_or(false); + self.params.auth_errors_max = self.eval_if(&ac.errors_max).await.unwrap_or(3); self.params.auth_errors_wait = self - .server - .eval_if(&ac.errors_wait, self, self.data.session_id) + .eval_if(&ac.errors_wait) .await .unwrap_or_else(|| Duration::from_secs(30)); // VRFY/EXPN parameters let ec = &self.server.core.smtp.session.extensions; - self.params.can_expn = self - .server - .eval_if(&ec.expn, self, self.data.session_id) - .await - .unwrap_or(false); - self.params.can_vrfy = self - .server - .eval_if(&ec.vrfy, self, self.data.session_id) - .await - .unwrap_or(false); + self.params.can_expn = self.eval_if(&ec.expn).await.unwrap_or(false); + self.params.can_vrfy = self.eval_if(&ec.vrfy).await.unwrap_or(false); } pub async fn eval_post_auth_params(&mut self) { // Refresh VRFY/EXPN parameters let ec = &self.server.core.smtp.session.extensions; - self.params.can_expn = self - .server - .eval_if(&ec.expn, self, self.data.session_id) - .await - .unwrap_or(false); - self.params.can_vrfy = self - .server - .eval_if(&ec.vrfy, self, self.data.session_id) - .await - .unwrap_or(false); + self.params.can_expn = self.eval_if(&ec.expn).await.unwrap_or(false); + self.params.can_vrfy = self.eval_if(&ec.vrfy).await.unwrap_or(false); } pub async fn eval_rcpt_params(&mut self) { let rc = &self.server.core.smtp.session.rcpt; - self.params.rcpt_errors_max = self - .server - .eval_if(&rc.errors_max, self, self.data.session_id) - .await - .unwrap_or(10); + self.params.rcpt_errors_max = self.eval_if(&rc.errors_max).await.unwrap_or(10); self.params.rcpt_errors_wait = self - .server - .eval_if(&rc.errors_wait, self, self.data.session_id) + .eval_if(&rc.errors_wait) .await .unwrap_or_else(|| Duration::from_secs(30)); - self.params.rcpt_max = self - .server - .eval_if(&rc.max_recipients, self, self.data.session_id) - .await - .unwrap_or(100); + self.params.rcpt_max = self.eval_if(&rc.max_recipients).await.unwrap_or(100); self.params.rcpt_dsn = self - .server - .eval_if( - &self.server.core.smtp.session.extensions.dsn, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.session.extensions.dsn) .await .unwrap_or(true); self.params.max_message_size = match self - .server - .eval_if::( - &self.server.core.smtp.session.data.max_message_size, - self, - self.data.session_id, - ) + .eval_if::(&self.server.core.smtp.session.data.max_message_size) .await { Some(0) => usize::MAX, diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs index 549eccd9..86d6de8e 100644 --- a/crates/smtp/src/core/throttle.rs +++ b/crates/smtp/src/core/throttle.rs @@ -6,9 +6,12 @@ use super::Session; use common::{ - KV_RATE_LIMIT_SMTP, ThrottleKey, config::smtp::*, expr::functions::ResolveVariable, + KV_RATE_LIMIT_SMTP, ThrottleKey, + config::smtp::*, + expr::{Bump, functions::ResolveVariable}, network::SessionStream, }; +use compact_str::ToCompactString; use queue::QueueQuota; use registry::schema::{enums::ExpressionVariable, structs::Rate}; use trc::SmtpEvent; @@ -19,40 +22,36 @@ pub trait NewKey: Sized { impl NewKey for QueueQuota { fn new_key(&self, e: &impl ResolveVariable, _: &str) -> ThrottleKey { + let arena = Bump::new(); let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::Rcpt) - .to_string() + e.resolve_variable(ExpressionVariable::Rcpt, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::RcptDomain) - .to_string() + e.resolve_variable(ExpressionVariable::RcptDomain, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_SENDER) != 0 { - let sender = e.resolve_variable(ExpressionVariable::Sender).into_string(); - hasher.update( - if !sender.is_empty() { - sender.as_ref() - } else { - "<>" - } - .as_bytes(), - ); + let sender = e + .resolve_variable(ExpressionVariable::Sender, &arena) + .to_str(&arena); + hasher.update(if !sender.is_empty() { sender } else { "<>" }.as_bytes()); } if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { let sender_domain = e - .resolve_variable(ExpressionVariable::SenderDomain) - .into_string(); + .resolve_variable(ExpressionVariable::SenderDomain, &arena) + .to_str(&arena); hasher.update( if !sender_domain.is_empty() { - sender_domain.as_ref() + sender_domain } else { "<>" } @@ -76,40 +75,36 @@ impl NewKey for QueueQuota { impl NewKey for QueueRateLimiter { fn new_key(&self, e: &impl ResolveVariable, context: &str) -> ThrottleKey { + let arena = Bump::new(); let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::Rcpt) - .to_string() + e.resolve_variable(ExpressionVariable::Rcpt, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::RcptDomain) - .to_string() + e.resolve_variable(ExpressionVariable::RcptDomain, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_SENDER) != 0 { - let sender = e.resolve_variable(ExpressionVariable::Sender).into_string(); - hasher.update( - if !sender.is_empty() { - sender.as_ref() - } else { - "<>" - } - .as_bytes(), - ); + let sender = e + .resolve_variable(ExpressionVariable::Sender, &arena) + .to_str(&arena); + hasher.update(if !sender.is_empty() { sender } else { "<>" }.as_bytes()); } if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { let sender_domain = e - .resolve_variable(ExpressionVariable::SenderDomain) - .into_string(); + .resolve_variable(ExpressionVariable::SenderDomain, &arena) + .to_str(&arena); hasher.update( if !sender_domain.is_empty() { - sender_domain.as_ref() + sender_domain } else { "<>" } @@ -118,43 +113,43 @@ impl NewKey for QueueRateLimiter { } if (self.keys & THROTTLE_HELO_DOMAIN) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::HeloDomain) - .to_string() + e.resolve_variable(ExpressionVariable::HeloDomain, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_AUTH_AS) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::AuthenticatedAs) - .to_string() + e.resolve_variable(ExpressionVariable::AuthenticatedAs, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_LISTENER) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::Listener) - .to_string() + e.resolve_variable(ExpressionVariable::Listener, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_MX) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::Mx) - .to_string() + e.resolve_variable(ExpressionVariable::Mx, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_REMOTE_IP) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::RemoteIp) - .to_string() + e.resolve_variable(ExpressionVariable::RemoteIp, &arena) + .to_str(&arena) .as_bytes(), ); } if (self.keys & THROTTLE_LOCAL_IP) != 0 { hasher.update( - e.resolve_variable(ExpressionVariable::LocalIp) - .to_string() + e.resolve_variable(ExpressionVariable::LocalIp, &arena) + .to_str(&arena) .as_bytes(), ); } @@ -179,24 +174,12 @@ impl Session { }; for t in throttles { - if t.expr.is_empty() - || self - .server - .eval_if(&t.expr, self, self.data.session_id) - .await - .unwrap_or(false) - { - if (t.keys & THROTTLE_RCPT_DOMAIN) != 0 { - let d = self - .data - .rcpt_to - .last() - .map(|r| r.domain.as_str()) - .unwrap_or_default(); - - if self.data.rcpt_to.iter().filter(|p| p.domain == d).count() > 1 { - continue; - } + if t.expr.is_empty() || self.eval_if(&t.expr).await.unwrap_or(false) { + if (t.keys & THROTTLE_RCPT_DOMAIN) != 0 + && let Some((last, previous)) = self.data.rcpt_to.split_last() + && previous.iter().any(|p| p.domain == last.domain) + { + continue; } // Build throttle key @@ -206,18 +189,16 @@ impl Session { match self .server .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.hash.as_slice(), &t.rate, false) + .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.hash.as_slice(), &t.rate) .await { Ok(Some(_)) => { trc::event!( Smtp(SmtpEvent::RateLimitExceeded), SpanId = self.data.session_id, - Id = t.id.to_string(), - Limit = vec![ - trc::Value::from(t.rate.count), - trc::Value::from(t.rate.period.into_inner()) - ], + Name = t.id.to_compact_string(), + Limit = t.rate.count, + Period = t.rate.period.into_inner(), ); return false; @@ -246,12 +227,7 @@ impl Session { match self .server .in_memory_store() - .is_rate_allowed( - KV_RATE_LIMIT_SMTP, - hasher.finalize().as_bytes(), - rate, - false, - ) + .is_rate_allowed(KV_RATE_LIMIT_SMTP, hasher.finalize().as_bytes(), rate) .await { Ok(None) => true, diff --git a/crates/smtp/src/inbound/auth.rs b/crates/smtp/src/inbound/auth.rs index 044cf3fc..20cb5fc8 100644 --- a/crates/smtp/src/inbound/auth.rs +++ b/crates/smtp/src/inbound/auth.rs @@ -5,16 +5,38 @@ */ use crate::core::Session; -use common::{auth::AuthRequest, network::SessionStream}; +use common::{ + auth::{ + AuthRequest, + oauth::scope::OAuthScopes, + sasl::{BearerFailure, BearerMechanism, SASL_CANCEL, SASL_EMPTY}, + }, + config::smtp::session::Mechanism, + network::SessionStream, +}; use directory::Credentials; -use mail_parser::decoders::base64::base64_decode; +use encodify::base64::LENIENT; use registry::schema::enums::Permission; use smtp_proto::{AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2, IntoString}; use trc::AuthEvent; +const AUTH_BEARER: u64 = AUTH_OAUTHBEARER | AUTH_XOAUTH2; +const CREDENTIALS_INVALID: &[u8] = b"535 5.7.8 Authentication credentials invalid.\r\n"; +const TOKEN_EXPIRED: &[u8] = b"535 5.7.8 OAuth token expired.\r\n"; +const INVALID_CHALLENGE: &[u8] = b"500 5.5.6 Invalid challenge.\r\n"; +const NOT_AUTHORIZED: &[u8] = b"550 5.7.1 Your account is not authorized to use this service.\r\n"; +const CANCELLED: &[u8] = b"501 5.7.0 Authentication cancelled.\r\n"; + pub struct SaslToken { mechanism: u64, credentials: Credentials, + pending_failure: Option, +} + +#[derive(Clone, Copy)] +enum PendingFailure { + Counted(&'static [u8]), + Uncounted(&'static [u8]), } impl SaslToken { @@ -27,6 +49,7 @@ impl SaslToken { secret: String::new(), mfa_token: None, }, + pending_failure: None, } .into(), AUTH_OAUTHBEARER | AUTH_XOAUTH2 => SaslToken { @@ -35,20 +58,56 @@ impl SaslToken { username: None, token: String::new(), }, + pending_failure: None, } .into(), _ => None, } } + + fn bearer_mechanism(&self) -> Option { + match self.mechanism { + AUTH_OAUTHBEARER => Some(BearerMechanism::OAuthBearer), + AUTH_XOAUTH2 => Some(BearerMechanism::XOAuth2), + _ => None, + } + } } impl Session { + pub async fn sasl_mechanisms(&self) -> u64 { + let mechanisms: u64 = self + .eval_if::(&self.server.core.smtp.session.auth.mechanisms) + .await + .unwrap_or_default() + .into(); + if self.server.accepts_bearer_tokens(self.stream.is_tls()) { + mechanisms + } else { + mechanisms & !AUTH_BEARER + } + } + pub async fn handle_sasl_response( &mut self, token: &mut SaslToken, response: &[u8], ) -> Result { - if response.is_empty() { + match token.pending_failure.take() { + Some(PendingFailure::Counted(failure)) => return self.auth_error(failure).await, + Some(PendingFailure::Uncounted(failure)) => { + self.write(failure).await?; + return Ok(false); + } + None => {} + } + + if response == SASL_CANCEL.as_bytes() { + self.write(CANCELLED).await?; + return Ok(false); + } + + if response.is_empty() || response == SASL_EMPTY.as_bytes() { match (token.mechanism, &token.credentials) { (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER, _) => { self.write(b"334 \r\n").await?; @@ -65,11 +124,11 @@ impl Session { } _ => (), } - } else if let Some(response) = base64_decode(response) { + } else if let Ok(response) = LENIENT.decode(response) { match (token.mechanism, &mut token.credentials) { (AUTH_PLAIN, _) => { if let Some(credentials) = Credentials::decode_sasl_challenge_plain(&response) { - return self.authenticate(credentials).await; + return self.authenticate(credentials, token).await; } } ( @@ -84,40 +143,111 @@ impl Session { Ok(true) } else { *secret = response.into_string(); - self.authenticate(std::mem::replace( + let credentials = std::mem::replace( &mut token.credentials, Credentials::Basic { username: String::new(), secret: String::new(), mfa_token: None, }, - )) - .await + ); + self.authenticate(credentials, token).await }; } (AUTH_OAUTHBEARER | AUTH_XOAUTH2, _) => { if let Some(credentials) = Credentials::decode_sasl_challenge_oauth(&response) { - return self.authenticate(credentials).await; + return self.authenticate(credentials, token).await; + } else if let Some(mechanism) = token.bearer_mechanism() { + self.malformed_credentials().await?; + return self + .bearer_challenge( + token, + mechanism, + BearerFailure::InvalidToken, + None, + PendingFailure::Counted(INVALID_CHALLENGE), + ) + .await; } } _ => (), } } - self.auth_error(b"500 5.5.6 Invalid challenge.\r\n").await + self.malformed_credentials().await?; + self.auth_error(INVALID_CHALLENGE).await } - pub async fn authenticate(&mut self, credentials: Credentials) -> Result { - // Authenticate - let result = self + async fn malformed_credentials(&mut self) -> Result<(), ()> { + let err = self .server - .authenticate(&AuthRequest::from_credentials( - credentials, - self.data.session_id, + .auth_failure( + AuthEvent::Failed + .into_err() + .details("Invalid SASL response."), self.data.remote_ip, - )) - .await - .and_then(|access_token| access_token.assert_has_permission(Permission::EmailSend)); + None, + ) + .await; + if matches!(err.as_ref(), trc::EventType::Security(_)) { + trc::error!(err.span_id(self.data.session_id)); + Err(()) + } else { + Ok(()) + } + } + + async fn bearer_challenge( + &mut self, + token: &mut SaslToken, + mechanism: BearerMechanism, + failure: BearerFailure, + credentials: Option<&Credentials>, + pending: PendingFailure, + ) -> Result { + let challenge = self + .server + .bearer_failure_challenge(mechanism, failure, credentials) + .await; + token.pending_failure = Some(pending); + self.write(format!("334 {challenge}\r\n").as_bytes()) + .await?; + Ok(true) + } + + pub async fn authenticate( + &mut self, + credentials: Credentials, + token: &mut SaslToken, + ) -> Result { + // Authenticate + let request = + AuthRequest::from_credentials(credentials, self.data.session_id, self.data.remote_ip); + let result = match self.server.authenticate(&request).await { + Ok(authenticated) => match token.bearer_mechanism() { + Some(mechanism) => match authenticated + .access_token + .assert_allows_oauth_scopes(OAuthScopes::MAIL) + { + Ok(access_token) => Ok(access_token), + Err(err) => { + trc::error!(err.span_id(self.data.session_id)); + return self + .bearer_challenge( + token, + mechanism, + BearerFailure::InsufficientScope, + Some(&request.credentials), + PendingFailure::Uncounted(NOT_AUTHORIZED), + ) + .await; + } + }, + None => Ok(authenticated.access_token), + } + .and_then(|access_token| access_token.assert_has_permission(Permission::EmailSend)), + Err(err) => Err(err), + }; let result = match result { Ok(access_token) => self.server.account_info(access_token.account_id()).await, @@ -135,16 +265,32 @@ impl Session { Err(err) => { let reason = *err.as_ref(); + if let Some(mechanism) = token.bearer_mechanism() + && let Some(failure) = BearerFailure::from_error(&err) + { + trc::error!(err.span_id(self.data.session_id)); + return self + .bearer_challenge( + token, + mechanism, + failure, + Some(&request.credentials), + PendingFailure::Counted(match reason { + trc::EventType::Auth(trc::AuthEvent::TokenExpired) => TOKEN_EXPIRED, + _ => CREDENTIALS_INVALID, + }), + ) + .await; + } + trc::error!(err.span_id(self.data.session_id)); match reason { trc::EventType::Auth(trc::AuthEvent::Failed) => { - return self - .auth_error(b"535 5.7.8 Authentication credentials invalid.\r\n") - .await; + return self.auth_error(CREDENTIALS_INVALID).await; } trc::EventType::Auth(trc::AuthEvent::TokenExpired) => { - return self.auth_error(b"535 5.7.8 OAuth token expired.\r\n").await; + return self.auth_error(TOKEN_EXPIRED).await; } trc::EventType::Auth(trc::AuthEvent::MfaRequired) => { return self @@ -158,14 +304,7 @@ impl Session { .await; } trc::EventType::Security(trc::SecurityEvent::Unauthorized) => { - self.write( - concat!( - "550 5.7.1 Your account is not authorized ", - "to use this service.\r\n" - ) - .as_bytes(), - ) - .await?; + self.write(NOT_AUTHORIZED).await?; return Ok(false); } trc::EventType::Security(_) => { diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 94b2c1e6..59bbaefe 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -6,11 +6,13 @@ use super::AuthResult; use crate::{ - core::{Session, SessionAddress, State}, - inbound::{dkim::DkimSign, milter::Modification}, + core::{Session, SessionAddress, SessionData, State}, + inbound::{dkim::DkimSign, milter::Modification, session::INSUFFICIENT_STORAGE}, queue::{ - self, Message, MessageSource, MessageWrapper, QueueEnvelope, RCPT_SPAM_MASK, - quota::HasQueueQuota, rcpt_spam_flag, spool::QueueParams, + self, Message, MessageSource, MessageWrapper, Metadata, QueueEnvelope, RCPT_SPAM_MASK, + quota::HasQueueQuota, + rcpt_spam_flag, + spool::{QueueOutcome, QueueParams}, }, reporting::analysis::{AnalyzeReport, ReportData}, scripts::ScriptResult, @@ -24,42 +26,78 @@ use common::{ session::Stage, }, }, + expr::{Bump, Variable as ExprVariable}, network::SessionStream, scripts::ScriptModification, }; +use compact_str::{CompactString, ToCompactString}; use mail_auth::{ AuthenticatedMessage, AuthenticationResults, Dkim2Result, DkimResult, DmarcResult, ReceivedSpf, SpfOutput, SpfResult, - common::{ - crypto::Algorithm, - headers::{Header, HeaderWriter}, - verify::VerifySignature, - }, + crypto::Algorithm, dkim::DkimError, dkim2::{Dkim2Dsn, Dkim2DsnFailure, Envelope as Dkim2Envelope}, dmarc::{self, verify::DmarcParameters}, + headers::HeaderWriter, + spf::verify::SpfParameters, }; -use mail_builder::headers::{date::Date, message_id::generate_message_id_header}; -use mail_parser::{MessageParser, MimeHeaders, parsers::fields::thread::thread_name}; +use mail_builder::headers::{Header, date::Date, message_id::generate_message_id_header}; +use mail_parser::{AddressList, MessageParser, PartKind, thread_name}; use registry::schema::structs::Rate; -use sieve::runtime::Variable; +use sieve::{Envelope, runtime::Variable}; use smtp_proto::{ MAIL_BY_RETURN, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use std::{ borrow::Cow, + io::Write, time::{Instant, SystemTime}, }; -use trc::{SmtpEvent, SpamEvent}; +use trc::{LimitEvent, SmtpEvent, SpamEvent}; use utils::DomainPart; +const PROCESSING_MEMORY_FACTOR: u64 = 3; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MessageOrigin { + Client, + Internal, + Submission { account_id: u32 }, +} + +impl MessageOrigin { + fn metadata(self) -> Option { + match self { + MessageOrigin::Submission { account_id } => Some(Metadata::Submission { account_id }), + MessageOrigin::Client | MessageOrigin::Internal => None, + } + } +} + impl Session { - pub async fn queue_message(&mut self) -> Cow<'static, [u8]> { + pub async fn queue_message(&mut self, origin: MessageOrigin) -> Cow<'static, [u8]> { // Parse message let raw_message = std::mem::take(&mut self.data.message); + let _message_memory = match self.data.message_memory.take() { + Some(mut memory) => { + let required = (raw_message.len() as u64).saturating_mul(PROCESSING_MEMORY_FACTOR); + if !memory.reserve_to(required) { + trc::event!( + Limit(LimitEvent::MemoryBudget), + SpanId = self.data.session_id, + Size = required, + Limit = memory.limit(), + ); + + return INSUFFICIENT_STORAGE.into(); + } + Some(memory) + } + None => None, + }; let parsed_message = match MessageParser::new() .parse(&raw_message) - .filter(|p| p.headers().iter().any(|h| !h.name.is_other())) + .filter(|p| p.headers().has_known()) { Some(parsed_message) => parsed_message, None => { @@ -72,6 +110,23 @@ impl Session { } }; + if origin == MessageOrigin::Client + && let Err(err) = self + .server + .core + .email + .limits + .validate_header_section(&parsed_message) + { + trc::event!( + Smtp(SmtpEvent::MessageTooLarge), + SpanId = self.data.session_id, + Details = err.to_string(), + ); + + return (&b"552 5.3.4 Message header too large.\r\n"[..]).into(); + } + // Authenticate message let mut auth_message = AuthenticatedMessage::from_parsed( &parsed_message, @@ -86,11 +141,7 @@ impl Session { let ac = &self.server.core.smtp.mail_auth; let rc = &self.server.core.smtp.report; if auth_message.received_headers_count() - > self - .server - .eval_if(&dc.max_received_headers, self, self.data.session_id) - .await - .unwrap_or(50) + > self.eval_if(&dc.max_received_headers).await.unwrap_or(50) { trc::event!( Smtp(SmtpEvent::LoopDetected), @@ -104,31 +155,22 @@ impl Session { // Verify DKIM let dkim = self - .server - .eval_if(&ac.dkim.verify, self, self.data.session_id) + .eval_if(&ac.dkim.verify) .await .unwrap_or(VerifyStrategy::Relaxed); let dmarc = self - .server - .eval_if(&ac.dmarc.verify, self, self.data.session_id) + .eval_if(&ac.dmarc.verify) .await .unwrap_or(VerifyStrategy::Relaxed); let dkim_output = if dkim.verify() || dmarc.verify() { // Remove insecure DKIM signatures before verification - auth_message.dkim_headers.retain(|header| { - let signature = &header.header; - if signature.algorithm() == Algorithm::RsaSha1 - || (signature.algorithm() == Algorithm::RsaSha256 && signature.b.len() < 128) + auth_message.retain_dkim_signatures(|signature| { + if signature.a == Algorithm::RsaSha1 + || (signature.a == Algorithm::RsaSha256 && signature.b.len() < 128) { - auth_message.errors.push(Header { - name: header.name, - value: header.value, - header: mail_auth::Error::Dkim(DkimError::UnsupportedAlgorithm), - }); - auth_message.has_dkim_errors = true; - false + Err(DkimError::UnsupportedAlgorithm) } else { - true + Ok(()) } }); @@ -148,13 +190,9 @@ impl Session { let rejected = strict && !pass; // Send reports for failed signatures - if let Some(rate) = self - .server - .eval_if::(&rc.dkim.send, self, self.data.session_id) - .await - { + if let Some(rate) = self.eval_if::(&rc.dkim.send).await { for output in &dkim_output { - if let Some(rcpt) = output.failure_report_addr() { + if let Some(rcpt) = output.report_address() { self.send_dkim_report(rcpt, &auth_message, &rate, rejected, output) .await; } @@ -192,8 +230,7 @@ impl Session { // Verify ARC let arc = self - .server - .eval_if(&ac.arc.verify, self, self.data.session_id) + .eval_if(&ac.arc.verify) .await .unwrap_or(VerifyStrategy::Relaxed); let arc_output = if arc.verify() { @@ -237,12 +274,13 @@ impl Session { // Verify DKIM2 let mail_from = self.data.mail_from.as_ref().unwrap(); - let dkim2_output = if (dkim.verify() || dmarc.verify()) - && (!auth_message.dkim2_signatures.is_empty() || auth_message.has_dkim2_errors) - { + let has_dkim2_fields = !auth_message.dkim2_signatures().is_empty() + || !auth_message.dkim2_instances().is_empty() + || auth_message.has_dkim2_errors(); + let dkim2_output = if (dkim.verify() || dmarc.verify()) && has_dkim2_fields { // Discard forged DKIM2-signed delivery status notifications if dkim.verify() - && !auth_message.dkim2_signatures.is_empty() + && !auth_message.dkim2_signatures().is_empty() && let Some(dsn) = parse_dkim2_dsn(&parsed_message, &auth_message, raw_message.as_slice()) && let Err(failure) = self @@ -259,29 +297,32 @@ impl Session { }, ) .await - && matches!( + && (matches!( failure, Dkim2DsnFailure::DsnChainFailed | Dkim2DsnFailure::ReturnedChainFailed | Dkim2DsnFailure::NotAligned - ) + ) || (failure == Dkim2DsnFailure::TempError && dkim.is_strict())) { trc::event!( - Smtp(SmtpEvent::Dkim2DsnDiscarded), + Smtp(SmtpEvent::Dkim2DsnRejected), SpanId = self.data.session_id, - From = mail_from.address.to_string(), + From = CompactString::from(&mail_from.address), To = self .data .rcpt_to .iter() - .map(|rcpt| trc::Value::from(rcpt.address.to_string())) + .map(|rcpt| trc::Value::from(CompactString::from(&rcpt.address))) .collect::>(), - Reason = failure.to_string(), + Details = failure.to_compact_string(), ); self.data.messages_sent += 1; - return (b"550 5.7.1 DSN rejected due to DKIM2 verification failure.\r\n"[..]) - .into(); + return if failure == Dkim2DsnFailure::TempError { + (b"451 4.7.5 Unable to verify DKIM2 DSN, try again later.\r\n"[..]).into() + } else { + (b"550 5.7.1 DSN rejected due to DKIM2 verification failure.\r\n"[..]).into() + }; } let time = Instant::now(); @@ -313,6 +354,22 @@ impl Session { ); } + if dkim.is_strict() + && matches!(output.result(), Dkim2Result::TempError(_)) + && self + .server + .eval_signers(&ac.dkim.sign, self, self.data.session_id) + .await + .is_some_and(|signers| signers.dkim2.is_some()) + && self + .server + .has_remote_domain(self.data.rcpt_to.iter().map(|rcpt| rcpt.domain.as_str())) + .await + { + return (b"451 4.7.5 Unable to verify DKIM2 signatures, try again later.\r\n"[..]) + .into(); + } + Some(output) } else { None @@ -321,7 +378,8 @@ impl Session { // Build authentication results header let mut auth_results = AuthenticationResults::new(&self.hostname); if !dkim_output.is_empty() { - auth_results = auth_results.with_dkim_results(&dkim_output, auth_message.from()) + auth_results = + auth_results.with_dkim_results(&dkim_output, auth_message.first_from_address()) } if let Some(dkim2_output) = &dkim2_output && (!matches!(dkim2_output.result(), Dkim2Result::None) @@ -330,18 +388,20 @@ impl Session { auth_results = auth_results.with_dkim2_result(dkim2_output); } if let Some(spf_ehlo) = &self.data.spf_ehlo { - auth_results = auth_results.with_spf_ehlo_result( + auth_results = auth_results.with_spf_result( spf_ehlo, - self.data.remote_ip, - &self.data.helo_domain, + &SpfParameters::helo(self.data.remote_ip, &self.data.helo_domain, &self.hostname), ); } if let Some(spf_mail_from) = &self.data.spf_mail_from { - auth_results = auth_results.with_spf_mailfrom_result( + auth_results = auth_results.with_spf_result( spf_mail_from, - self.data.remote_ip, - &mail_from.address, - &self.data.helo_domain, + &SpfParameters::mail_from( + self.data.remote_ip, + &self.data.helo_domain, + &self.hostname, + &mail_from.address, + ), ); } if let Some(iprev) = &self.data.iprev { @@ -372,7 +432,7 @@ impl Session { message: &auth_message, dkim_output: &dkim_output, dkim2_output: dkim2_output.as_ref(), - rfc5321_mail_from_domain: if !mail_from.domain.is_empty() { + mail_from_domain: if !mail_from.domain.is_empty() { &mail_from.domain } else { &self.data.helo_domain @@ -382,6 +442,25 @@ impl Session { )) .await; + if origin == MessageOrigin::Client + && !self.is_authenticated() + && dmarc_output.domain().is_empty() + && let Some((author_domain, other_domain)) = + conflicting_author_domains(parsed_message.all_from()) + { + trc::event!( + Smtp(SmtpEvent::MultipleFromDomains), + SpanId = self.data.session_id, + Domain = vec![ + trc::Value::String(author_domain.as_ref().into()), + trc::Value::String(other_domain.as_ref().into()), + ], + ); + + return (&b"550 5.7.1 Multiple author domains in From header are not allowed.\r\n"[..]) + .into(); + } + let dmarc_result = dmarc_output.result(); let pass = dmarc_result == DmarcResult::Pass; let strict = dmarc.is_strict(); @@ -402,14 +481,14 @@ impl Session { }), SpanId = self.data.session_id, Strict = strict, - Domain = dmarc_output.domain().to_string(), - Policy = dmarc_policy.to_string(), + Domain = CompactString::from(dmarc_output.domain()), + Policy = dmarc_policy.to_compact_string(), Result = trc::Error::from(&dmarc_result), Elapsed = time.elapsed(), ); // Send DMARC report - if dmarc_output.requested_reports() && !is_report && !(rejected && is_temp_fail) { + if dmarc_output.requests_reports() && !is_report && !(rejected && is_temp_fail) { self.send_dmarc_report( &auth_message, &auth_results, @@ -445,76 +524,40 @@ impl Session { } if self.data.rcpt_to.is_empty() { - self.server.analyze_report( - mail_parser::Message { - html_body: parsed_message.html_body, - text_body: parsed_message.text_body, - attachments: parsed_message.attachments, - parts: parsed_message - .parts - .into_iter() - .map(|p| p.into_owned()) - .collect(), - raw_message: b"".into(), - }, - self.data.session_id, - ); + self.server + .analyze_report(parsed_message.into_owned(), self.data.session_id); self.data.messages_sent += 1; return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); } else { - self.server.analyze_report( - mail_parser::Message { - html_body: parsed_message.html_body.clone(), - text_body: parsed_message.text_body.clone(), - attachments: parsed_message.attachments.clone(), - parts: parsed_message - .parts - .iter() - .map(|p| p.clone().into_owned()) - .collect(), - raw_message: b"".into(), - }, - self.data.session_id, - ); + self.server + .analyze_report(parsed_message.clone().into_owned(), self.data.session_id); } } // Add Received header let message_id = self.server.inner.data.queue_id_gen.generate(); let mut headers = Vec::with_capacity(64); - if self - .server - .eval_if(&dc.add_received, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&dc.add_received).await.unwrap_or(true) { self.write_received(&mut headers, message_id) } // Add authentication results header - if self - .server - .eval_if(&dc.add_auth_results, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&dc.add_auth_results).await.unwrap_or(true) { auth_results.write_header(&mut headers); } // Add Received-SPF header if let Some(spf_output) = &self.data.spf_mail_from - && self - .server - .eval_if(&dc.add_received_spf, self, self.data.session_id) - .await - .unwrap_or(true) + && self.eval_if(&dc.add_received_spf).await.unwrap_or(true) { ReceivedSpf::new( spf_output, - self.data.remote_ip, - &self.data.helo_domain, - &mail_from.address_lcase, - &self.hostname, + &SpfParameters::mail_from( + self.data.remote_ip, + &self.data.helo_domain, + &self.hostname, + &mail_from.address_lcase, + ), ) .write_header(&mut headers); } @@ -522,13 +565,7 @@ impl Session { // Run SPAM filter let mut train_spam = None; let mut spam_result = None; - if self.server.core.spam.enabled - && self - .server - .eval_if(&dc.spam_filter, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.server.core.spam.enabled && self.eval_if(&dc.spam_filter).await.unwrap_or(true) { match self .spam_classify( &parsed_message, @@ -622,6 +659,20 @@ impl Session { }; // Apply modifications + let needs_arrival_envelope = has_dkim2_fields && dkim2_output.is_none(); + let mut arrival_envelope = None; + if needs_arrival_envelope + && modifications.iter().any(|modification| { + matches!( + modification, + Modification::ChangeFrom { .. } + | Modification::AddRcpt { .. } + | Modification::DeleteRcpt { .. } + ) + }) + { + arrival_envelope = Some(ArrivalEnvelope::capture(&self.data)); + } let mut edited_message = if !modifications.is_empty() { self.data .apply_milter_modifications(modifications, &auth_message) @@ -630,19 +681,21 @@ impl Session { }; // Sieve filtering - if let Some((script, script_id)) = self - .server - .eval_if::(&dc.script, self, self.data.session_id) - .await - .and_then(|name| { + let mut sieve_redirected = false; + if let Some((script, script_id)) = + self.eval_if::(&dc.script).await.and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) { + let edited_parsed_message = edited_message + .as_deref() + .and_then(|message| MessageParser::new().parse(message)); let mut params = self .build_script_parameters("data") - .with_auth_headers(&headers); + .with_auth_headers(&headers) + .with_dkim2_verified(dkim2_output.as_ref(), &auth_message); if let Some((score, percentage)) = spam_result { params = params .with_spam_status(spam_status(Some(percentage))) @@ -672,8 +725,7 @@ impl Session { .iter() .filter_map(|r| { if matches!(r.result(), DkimResult::Pass) { - r.signature() - .map(|s| Variable::from(s.domain().to_lowercase())) + r.signature().map(|s| Variable::from(s.d.to_lowercase())) } else { None } @@ -694,20 +746,23 @@ impl Session { .map(|a| a.as_str()) .unwrap_or_default(), ) - .with_message( - edited_message - .as_deref() - .and_then(|message| MessageParser::new().parse(message)) - .unwrap_or(parsed_message), - ); + .with_message(edited_parsed_message.as_ref().unwrap_or(&parsed_message)); - let modifications = match self.run_script(script_id, script.clone(), params).await { - ScriptResult::Accept { modifications } => modifications, + let modifications = match self.run_script(script_id, script, params).await { + ScriptResult::Accept { + modifications, + redirected, + } => { + sieve_redirected = redirected; + modifications + } ScriptResult::Replace { message, modifications, + redirected, } => { edited_message = message.into(); + sieve_redirected = redirected; modifications } ScriptResult::Reject(message) => { @@ -730,6 +785,12 @@ impl Session { } } ScriptModification::SetEnvelope { name, value } => { + if needs_arrival_envelope + && arrival_envelope.is_none() + && matches!(name, Envelope::From | Envelope::To) + { + arrival_envelope = Some(ArrivalEnvelope::capture(&self.data)); + } self.data.apply_envelope_modification(name, value); } } @@ -737,19 +798,20 @@ impl Session { } // Build message - let mail_from = self.data.mail_from.clone().unwrap(); + let mail_from = self.data.mail_from.as_ref().unwrap(); + let arrival_envelope = arrival_envelope.filter(|arrival| !arrival.is_current(&self.data)); let rcpt_to = std::mem::take(&mut self.data.rcpt_to); let source = if !self.is_authenticated() { let dmarc_pass = dmarc_result.is_some_and(|result| result == DmarcResult::Pass); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { MessageSource::Unauthenticated { dmarc_pass: dmarc_pass || mail_from.address.starts_with("dmarc-"), } } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { MessageSource::Unauthenticated { dmarc_pass } } @@ -761,36 +823,18 @@ impl Session { .await; // Add Return-Path - if self - .server - .eval_if(&dc.add_return_path, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&dc.add_return_path).await.unwrap_or(true) { headers.extend_from_slice(b"Return-Path: <"); headers.extend_from_slice(message.message.return_path.as_bytes()); headers.extend_from_slice(b">\r\n"); } // Add any missing headers - if !has_date_header - && self - .server - .eval_if(&dc.add_date, self, self.data.session_id) - .await - .unwrap_or(true) - { + if !has_date_header && self.eval_if(&dc.add_date).await.unwrap_or(true) { headers.extend_from_slice(b"Date: "); - headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); - headers.extend_from_slice(b"\r\n"); + Date::now().write_header(&mut headers, 0); } - if !has_message_id_header - && self - .server - .eval_if(&dc.add_message_id, self, self.data.session_id) - .await - .unwrap_or(true) - { + if !has_message_id_header && self.eval_if(&dc.add_message_id).await.unwrap_or(true) { headers.extend_from_slice(b"Message-ID: "); generate_message_id_header(&mut headers, &self.hostname); headers.extend_from_slice(b"\r\n"); @@ -801,42 +845,74 @@ impl Session { let raw_message = edited_message.as_deref().unwrap_or(raw_message.as_slice()); message.message.size = (raw_message.len() + headers.len()) as u64; - // Verify queue quota - if let Some(metadata) = self.server.has_quota(&mut message).await { - // Queue message - let queue_id = message.queue_id; - let dkim_signers = self - .server - .eval_signers(&ac.dkim.sign, self, self.data.session_id) - .await; - if message - .queue( - QueueParams::new(raw_message, self.data.session_id, &self.server) - .with_train_spam(train_spam) - .with_raw_headers(&headers) - .with_dkim_signers(dkim_signers) - .with_original_raw_message(original_message) - .with_original_authenticated_message(auth_message) - .with_metadata(metadata), - ) - .await - { + // Queue message + let dkim_signers = self + .server + .eval_signers(&ac.dkim.sign, self, self.data.session_id) + .await; + let dkim2_proof = if dkim2_output.is_none() { + match &arrival_envelope { + Some(arrival) => { + self.server + .dkim2_signing_proof( + dkim_signers.as_deref(), + &auth_message, + &arrival.mail_from, + &arrival.rcpt_to, + ) + .await + } + None => { + self.server + .dkim2_signing_proof( + dkim_signers.as_deref(), + &auth_message, + &mail_from.address, + message.message.recipients.iter().map(|rcpt| rcpt.address()), + ) + .await + } + } + } else { + None + }; + let dkim2_verified = dkim2_output.as_ref().or(dkim2_proof.as_ref()); + let quotas = self.server.queue_quotas(&message).await; + let queue_id = message.queue_id; + match message + .queue( + QueueParams::new(raw_message, self.data.session_id, &self.server) + .with_train_spam(train_spam) + .with_raw_headers(&headers) + .with_dkim_signers(dkim_signers) + .with_original_raw_message(original_message) + .with_original_authenticated_message(&auth_message) + .with_dkim2_verified(dkim2_verified) + .with_exploded(sieve_redirected) + .with_quotas(quotas) + .with_metadata(origin.metadata()), + ) + .await + { + QueueOutcome::Queued => { self.state = State::Accepted(queue_id); self.data.messages_sent += 1; format!("250 2.0.0 Message queued with id {queue_id:x}.\r\n") .into_bytes() .into() - } else { + } + QueueOutcome::QuotaExceeded => { + (b"452 4.3.1 Mail system full, try again later.\r\n"[..]).into() + } + QueueOutcome::Failed => { (b"451 4.3.5 Unable to accept message at this time.\r\n"[..]).into() } - } else { - (b"452 4.3.1 Mail system full, try again later.\r\n"[..]).into() } } pub async fn build_message( &self, - mail_from: SessionAddress, + mail_from: &SessionAddress, mut rcpt_to: Vec, source: MessageSource, queue_id: u64, @@ -856,7 +932,7 @@ impl Session { flags: mail_from.flags | source.flags(), priority: self.data.priority, size: 0, - env_id: mail_from.dsn_info.map(|i| i.into_boxed_str()), + env_id: mail_from.dsn_info.as_deref().map(Box::from), blob_hash: Default::default(), metadata: Default::default(), received_from_ip: self.data.remote_ip, @@ -865,6 +941,7 @@ impl Session { // Add recipients let future_release = self.data.future_release; + let mut arena = Bump::new(); rcpt_to.sort_unstable(); for rcpt in rcpt_to { message.recipients.push( @@ -895,18 +972,24 @@ impl Session { }; // Resolve queue - let queue = self.server.get_queue_or_default( - &self - .server - .eval_if::( - &self.server.core.smtp.queue.queue, - &envelope, - self.data.session_id, - ) - .await - .unwrap_or_else(|| "default".to_string()), - self.data.session_id, - ); + let session_id = self.data.session_id; + let queue = self + .server + .eval_if_with( + &self.server.core.smtp.queue.queue, + &envelope, + &mut arena, + session_id, + |name| match name { + ExprVariable::String(name) => { + Some(self.server.get_queue_or_default(name, session_id)) + } + _ => None, + }, + ) + .await + .flatten() + .unwrap_or_else(|| self.server.get_queue_or_default("default", session_id)); // Set expiration and notification times let num_intervals = std::cmp::max(queue.notify.len(), 1); @@ -990,12 +1073,7 @@ impl Session { Some(b"503 5.5.1 RCPT is required first.\r\n") } else if self.data.messages_sent < self - .server - .eval_if( - &self.server.core.smtp.session.data.max_messages, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.session.data.max_messages) .await .unwrap_or(10) { @@ -1025,13 +1103,12 @@ impl Session { .as_bytes(), ); headers.extend_from_slice(b" ["); - headers.extend_from_slice(self.data.remote_ip.to_string().as_bytes()); + headers.extend_from_slice(self.data.remote_ip_str.as_bytes()); headers.extend_from_slice(b"]"); if self.data.asn_geo_data.asn.is_some() || self.data.asn_geo_data.country.is_some() { headers.extend_from_slice(b" ("); if let Some(asn) = &self.data.asn_geo_data.asn { - headers.extend_from_slice(b"AS"); - headers.extend_from_slice(asn.id.to_string().as_bytes()); + let _ = write!(headers, "AS{}", asn.id); if let Some(name) = &asn.name { headers.extend_from_slice(b" "); headers.extend_from_slice(name.as_bytes()); @@ -1063,11 +1140,72 @@ impl Session { (false, true) => b"ESMTP", (false, false) => b"ESMTPA", }); - headers.extend_from_slice(b" id "); - headers.extend_from_slice(format!("{id:X}").as_bytes()); - headers.extend_from_slice(b";\r\n\t"); - headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); - headers.extend_from_slice(b"\r\n"); + let _ = write!(headers, " id {id:X};\r\n\t"); + Date::now().write_header(headers, 0); + } +} + +fn conflicting_author_domains<'m>( + from: impl Iterator>, +) -> Option<(Cow<'m, str>, Cow<'m, str>)> { + let mut author_domain: Option> = None; + for domain in from + .flat_map(|list| list.mailboxes()) + .filter_map(|mailbox| mailbox.address()?.rsplit_once('@')) + .map(|(_, domain)| domain) + .filter(|domain| !domain.is_empty()) + { + let domain = if domain.is_ascii() { + Cow::Borrowed(domain) + } else { + domain + .to_ascii_domain() + .map_or(Cow::Borrowed(domain), |domain| { + Cow::Owned(domain.into_owned()) + }) + }; + match author_domain { + Some(author_domain) if !author_domain.eq_ignore_ascii_case(&domain) => { + return Some((author_domain, domain)); + } + Some(_) => {} + None => author_domain = Some(domain), + } + } + None +} + +struct ArrivalEnvelope { + mail_from: String, + rcpt_to: Vec, +} + +impl ArrivalEnvelope { + fn capture(data: &SessionData) -> Self { + ArrivalEnvelope { + mail_from: data + .mail_from + .as_ref() + .map(|mail_from| mail_from.address.clone()) + .unwrap_or_default(), + rcpt_to: data + .rcpt_to + .iter() + .map(|rcpt| rcpt.address.clone()) + .collect(), + } + } + + fn is_current(&self, data: &SessionData) -> bool { + data.mail_from + .as_ref() + .map_or("", |mail_from| mail_from.address.as_str()) + == self.mail_from + && self + .rcpt_to + .iter() + .map(String::as_str) + .eq(data.rcpt_to.iter().map(|rcpt| rcpt.address.as_str())) } } @@ -1087,19 +1225,19 @@ fn parse_dkim2_dsn<'x, 'r>( }) { return None; } - let mail_parser::PartType::Multipart(children) = &parsed_message.root_part().body else { + let root_part = parsed_message.root_part(); + if !matches!(root_part.kind(), PartKind::Multipart) { return None; - }; + } let mut returned = &b""[..]; let mut returned_full = false; - for child in children { - let part = parsed_message.parts.get(*child as usize)?; + for part in root_part.children() { if part.is_content_type("message", "rfc822") { - returned = raw_message.get(part.offset_body as usize..part.offset_end as usize)?; + returned = raw_message.get(part.offset_body() as usize..part.offset_end() as usize)?; returned_full = true; } else if part.is_content_type("text", "rfc822-headers") { - returned = raw_message.get(part.offset_body as usize..part.offset_end as usize)?; + returned = raw_message.get(part.offset_body() as usize..part.offset_end() as usize)?; returned_full = false; } } diff --git a/crates/smtp/src/inbound/dkim.rs b/crates/smtp/src/inbound/dkim.rs index a13afcd4..d8eca64e 100644 --- a/crates/smtp/src/inbound/dkim.rs +++ b/crates/smtp/src/inbound/dkim.rs @@ -5,18 +5,24 @@ */ use crate::queue::{MessageWrapper, Metadata, spool::QueueParams}; +use ahash::AHashSet; use common::{ Server, - config::smtp::auth::{Dkim1Signer, DkimSigners}, - expr::{functions::ResolveVariable, if_block::IfBlock}, + config::smtp::auth::{Dkim1Signer, Dkim2Signers, DkimSigners}, + expr::{Bump, functions::ResolveVariable, if_block::IfBlock}, }; use mail_auth::{ - AuthenticatedMessage, - common::headers::HeaderWriter, - dkim2::{Hop, MessageInstance}, + AuthenticatedMessage, Dkim2Result, + dkim2::{ + ChainBinding, Dkim2Error, Dkim2Output, Envelope, Hop, MessageInstance, Signature, + Verification, + }, + headers::HeaderWriter, }; -use mail_parser::{Address, parsers::MessageStream}; -use std::{collections::HashSet, sync::Arc}; +use mail_parser::HeaderForm; +use memchr::memchr_iter; +use std::{iter::once, sync::Arc}; +use trc::{Dkim2Event, EventType}; use utils::sanitize_email; pub(crate) trait DkimSign: Sync + Send { @@ -24,7 +30,7 @@ pub(crate) trait DkimSign: Sync + Send { &self, message: &mut MessageWrapper, params: &mut QueueParams<'_, '_>, - ) -> impl Future>> + Send; + ) -> impl Future + Send; fn eval_signers( &self, @@ -32,6 +38,37 @@ pub(crate) trait DkimSign: Sync + Send { resolver: &impl ResolveVariable, session_id: u64, ) -> impl Future>> + Send; + + fn dkim2_signing_proof<'x, R>( + &'x self, + signers: Option<&DkimSigners>, + message: &'x AuthenticatedMessage<'x>, + mail_from: &str, + rcpt_to: R, + ) -> impl Future>> + Send + where + R: IntoIterator + Send, IntoIter: Send> + Send; + + fn dkim2_sign_or_restart( + &self, + message: &MessageWrapper, + params: &mut QueueParams<'_, '_>, + dkim2: &Dkim2Signers, + headers: &mut Vec, + ) -> impl Future>> + Send; + + fn restart_dkim2_chain( + &self, + message: &MessageWrapper, + params: &mut QueueParams<'_, '_>, + dkim2: &Dkim2Signers, + headers: &mut Vec, + ) -> impl Future>> + Send; + + fn has_remote_domain<'x>( + &self, + domains: impl Iterator + Send, + ) -> impl Future + Send; } impl DkimSign for Server { @@ -39,120 +76,191 @@ impl DkimSign for Server { &self, message: &mut MessageWrapper, params: &mut QueueParams<'_, '_>, - ) -> Option> { - let signers = params.dkim_signers.as_ref().unwrap(); - let raw_message = params.raw_message; + ) -> SignedMessage { + let signers = params.dkim_signers.take(); + let dkim2 = signers.as_ref().and_then(|signers| signers.dkim2.as_ref()); + let chain_broken = params.dkim2_verified.is_some_and(|output| { + matches!( + output.result(), + Dkim2Result::Fail(_) | Dkim2Result::PermError(_) | Dkim2Result::TempError(_) + ) + }); - // DKIM1 signing - let mut headers = Vec::with_capacity(64); - for signer in &signers.dkim1 { - let result = match (signer, params.raw_headers) { - (Dkim1Signer::RsaSha256(signer), None) => signer.sign(raw_message), - (Dkim1Signer::Ed25519Sha256(signer), None) => signer.sign(raw_message), - (Dkim1Signer::RsaSha256(signer), Some(headers)) => { - signer.sign_chained([headers, raw_message].iter().copied()) - } - (Dkim1Signer::Ed25519Sha256(signer), Some(headers)) => { - signer.sign_chained([headers, raw_message].iter().copied()) - } - }; - - match result { - Ok(signature) => { - signature.write_header(&mut headers); - } - Err(err) => { - trc::error!( - trc::Error::from(err) - .span_id(params.session_id) - .details("Failed to sign message") - .caused_by(trc::location!()) - ); - } + let mut dkim2_headers = Vec::new(); + let stripped_message = match dkim2 { + Some(dkim2) if chain_broken => { + self.restart_dkim2_chain(message, params, dkim2, &mut dkim2_headers) + .await } - } + Some(dkim2) => { + self.dkim2_sign_or_restart(message, params, dkim2, &mut dkim2_headers) + .await + } + None => None, + }; - // DKIM2 signing - if let Some(signer) = &signers.dkim2 - && let Some(modified) = - AuthenticatedMessage::parse_with_opts(raw_message, params.raw_headers, true) + let mut headers = Vec::new(); + if let Some(signers) = &signers + && !signers.dkim1.is_empty() { - // Generate message instance - let original = params.original_authenticated_message.take().or_else(|| { - params - .original_raw_message - .and_then(AuthenticatedMessage::parse) - }); - let instance = MessageInstance::from_message(&modified, original.as_ref()); - if let Some(instance) = &instance { - instance.write_header(&mut headers); - } + let raw_message = stripped_message.as_deref().unwrap_or(params.raw_message); + headers.reserve(64 + dkim2_headers.len()); + for signer in &signers.dkim1 { + let result = match (signer, params.raw_headers) { + (Dkim1Signer::RsaSha256(signer), None) => signer.sign(raw_message), + (Dkim1Signer::Ed25519Sha256(signer), None) => signer.sign(raw_message), + (Dkim1Signer::RsaSha256(signer), Some(headers)) => { + signer.sign_chained([headers, raw_message].iter().copied()) + } + (Dkim1Signer::Ed25519Sha256(signer), Some(headers)) => { + signer.sign_chained([headers, raw_message].iter().copied()) + } + }; - // Obtain disclosed and undisclosed recipients - let envelopes = message.undisclosed_recipients(&modified); - - // Generate DKIM2 signature for disclosed recipients - if !envelopes.disclosed_recipients.is_empty() { - match signer.sign_with_message_instance( - &modified, - instance.as_ref(), - Hop::real( - message.message.return_path.as_ref(), - envelopes.disclosed_recipients, - ), - ) { + match result { Ok(signature) => { - if envelopes.undisclosed_recipients.is_empty() { - // Happy path: no undisclosed recipients, serialize signature straight to blob - signature.write_header(&mut headers); - } else { - // Undisclosed recipients present, serialize signature to metadata - let mut header = Vec::with_capacity(64); - signature.write_header(&mut header); - params.metadata.push(Metadata::Headers { - value: header.into_boxed_slice(), - id: u64::MAX, - }); - } + signature.write_header(&mut headers); } Err(err) => { trc::error!( trc::Error::from(err) .span_id(params.session_id) - .details("Failed to DKIM2 sign message") - ); - } - } - } - - // Generate DKIM2 signature for undisclosed recipients - for (pos, rcpt) in envelopes.undisclosed_recipients { - match signer.sign_with_message_instance( - &modified, - instance.as_ref(), - Hop::real(message.message.return_path.as_ref(), [rcpt]), - ) { - Ok(signature) => { - // Serialize signature to metadata - let mut header = Vec::with_capacity(64); - signature.write_header(&mut header); - params.metadata.push(Metadata::Headers { - value: header.into_boxed_slice(), - id: pos as u64, - }); - } - Err(err) => { - trc::error!( - trc::Error::from(err) - .span_id(params.session_id) - .details("Failed to DKIM2 sign message") + .details("Failed to sign message") + .caused_by(trc::location!()) ); } } } } + if headers.is_empty() { + headers = dkim2_headers; + } else { + headers.extend_from_slice(&dkim2_headers); + } - (!headers.is_empty()).then_some(headers) + SignedMessage { + headers, + stripped_message, + } + } + + async fn dkim2_signing_proof<'x, R>( + &'x self, + signers: Option<&DkimSigners>, + message: &'x AuthenticatedMessage<'x>, + mail_from: &str, + rcpt_to: R, + ) -> Option> + where + R: IntoIterator + Send, IntoIter: Send> + Send, + { + if signers.is_some_and(|signers| signers.dkim2.is_some()) + && (!message.dkim2_signatures().is_empty() + || !message.dkim2_instances().is_empty() + || message.has_dkim2_errors()) + { + Some( + self.core + .smtp + .resolvers + .dns + .verify_dkim2( + self.inner.cache.build_auth_parameters(message), + Envelope { mail_from, rcpt_to }, + ) + .await, + ) + } else { + None + } + } + + async fn dkim2_sign_or_restart( + &self, + message: &MessageWrapper, + params: &mut QueueParams<'_, '_>, + dkim2: &Dkim2Signers, + headers: &mut Vec, + ) -> Option> { + let modified = + AuthenticatedMessage::parse_with_opts(params.raw_message, params.raw_headers, true)?; + let verified = params.dkim2_verified.filter(|output| { + matches!(output.result(), Dkim2Result::Pass) + && output.verification() == Verification::Full + }); + let has_chain = !modified.dkim2_signatures().is_empty() + || !modified.dkim2_instances().is_empty() + || modified.has_dkim2_errors(); + + if !has_chain || verified.is_some() { + let parsed_original; + let original = match params.original_authenticated_message { + Some(original) => Some(original), + None if has_chain => { + parsed_original = params + .original_raw_message + .and_then(AuthenticatedMessage::parse); + parsed_original.as_ref() + } + None => None, + }; + match dkim2_sign( + message, params, dkim2, &modified, original, verified, headers, + ) { + Ok(()) => return None, + Err(mail_auth::Error::Dkim2(Dkim2Error::NotValidlySigned)) => {} + Err(err) => { + log_dkim2_error(err, params.session_id); + if !has_chain { + return None; + } + } + } + } + + self.restart_dkim2_chain(message, params, dkim2, headers) + .await + } + + async fn restart_dkim2_chain( + &self, + message: &MessageWrapper, + params: &mut QueueParams<'_, '_>, + dkim2: &Dkim2Signers, + headers: &mut Vec, + ) -> Option> { + if !self + .has_remote_domain( + message + .message + .recipients + .iter() + .map(|rcpt| rcpt.domain_part()), + ) + .await + { + return None; + } + + let stripped = strip_dkim2_headers(params.raw_message); + if let Some(fresh) = AuthenticatedMessage::parse_with_opts( + stripped.as_deref().unwrap_or(params.raw_message), + params.raw_headers, + true, + ) && let Err(err) = dkim2_sign(message, params, dkim2, &fresh, None, None, headers) + { + log_dkim2_error(err, params.session_id); + } + stripped + } + + async fn has_remote_domain<'x>(&self, domains: impl Iterator + Send) -> bool { + for domain in domains { + if !matches!(self.domain(domain).await, Ok(Some(_))) { + return true; + } + } + false } async fn eval_signers( @@ -161,8 +269,9 @@ impl DkimSign for Server { resolver: &impl ResolveVariable, session_id: u64, ) -> Option> { + let mut arena = Bump::new(); let sign_with_domain = self - .eval_if::(if_block, resolver, session_id) + .eval_if::(if_block, resolver, &mut arena, session_id) .await?; match self.dkim_signers(&sign_with_domain).await { Ok(signers) => signers, @@ -177,6 +286,162 @@ impl DkimSign for Server { } } +pub(crate) struct SignedMessage { + pub headers: Vec, + pub stripped_message: Option>, +} + +fn log_dkim2_error(err: mail_auth::Error, session_id: u64) { + let err = trc::Error::from(err); + let err = if matches!( + err.as_ref(), + EventType::Dkim2(Dkim2Event::CustodyBroken | Dkim2Event::ChainInvalid) + ) { + err + } else { + EventType::Dkim2(Dkim2Event::SigningFailed).caused_by(err) + }; + trc::error!(err.span_id(session_id).caused_by(trc::location!())); +} + +fn dkim2_sign( + message: &MessageWrapper, + params: &mut QueueParams<'_, '_>, + dkim2: &Dkim2Signers, + modified: &AuthenticatedMessage<'_>, + original: Option<&AuthenticatedMessage<'_>>, + verified: Option<&Dkim2Output<'_>>, + headers: &mut Vec, +) -> mail_auth::Result<()> { + let instance = MessageInstance::from_message(modified, original, &dkim2.hashes)?; + let signer = &dkim2.signer; + let exploded = params.exploded || message.message.recipients.len() > 1; + let return_path = message.message.return_path.as_ref(); + let envelopes = message.undisclosed_recipients(modified); + + let disclosed = if !envelopes.disclosed_recipients.is_empty() { + Some(signer.sign_with_message_instance( + modified, + instance.as_ref(), + dkim2_hop(return_path, envelopes.disclosed_recipients, exploded), + verified, + )?) + } else { + None + }; + let mut undisclosed = Vec::with_capacity(envelopes.undisclosed_recipients.len() + 1); + for (pos, rcpt) in envelopes.undisclosed_recipients { + undisclosed.push(( + pos as u64, + signer.sign_with_message_instance( + modified, + instance.as_ref(), + dkim2_hop(return_path, [rcpt], exploded), + verified, + )?, + )); + } + + if let Some(instance) = &instance { + instance.write_header(headers); + } + if let Some(signed) = disclosed { + if undisclosed.is_empty() { + headers.reserve(signature_header_capacity(&signed.signature)); + signed.signature.write_header(headers); + } else { + undisclosed.push((u64::MAX, signed)); + } + } + for (id, signed) in undisclosed { + let mut header = Vec::with_capacity(signature_header_capacity(&signed.signature)); + signed.signature.write_header(&mut header); + params.metadata.push(Metadata::Headers { + value: header.into_boxed_slice(), + id, + }); + } + + Ok(()) +} + +fn dkim2_hop<'x, R>(return_path: &'x str, rcpt_to: R, exploded: bool) -> Hop<&'x str, R, String> +where + R: IntoIterator, +{ + let hop = Hop::real(return_path, rcpt_to); + if exploded { hop.exploded() } else { hop } +} + +fn signature_header_capacity(signature: &Signature) -> usize { + let tags = match &signature.chain { + ChainBinding::Envelope { mail_from, rcpt_to } => { + mail_from.len() + rcpt_to.iter().map(|rcpt| rcpt.len() + 1).sum::() + } + ChainBinding::NextDomain(domain) => domain.len(), + }; + let len = HEADER_BASE_CAPACITY + + tags.div_ceil(3) * 4 + + signature + .s + .iter() + .map(|value| value.selector.len() + value.b.len().div_ceil(3) * 4 + 32) + .sum::(); + len + len / 32 +} + +const HEADER_BASE_CAPACITY: usize = 256; + +fn strip_dkim2_headers(message: &[u8]) -> Option> { + let mut stripped: Option> = None; + let mut offset = 0; + let mut skipping = false; + for line in header_lines(message) { + if !is_continuation(line) { + skipping = is_dkim2_field(line); + } + match &mut stripped { + Some(stripped) if !skipping => stripped.extend_from_slice(line), + None if skipping => { + let mut buf = Vec::with_capacity(message.len()); + buf.extend_from_slice(message.get(..offset).unwrap_or_default()); + stripped = Some(buf); + } + _ => {} + } + offset += line.len(); + } + let mut stripped = stripped?; + stripped.extend_from_slice(message.get(offset..).unwrap_or_default()); + Some(stripped) +} + +pub(crate) fn has_dkim2_headers(message: &[u8]) -> bool { + header_lines(message).any(|line| !is_continuation(line) && is_dkim2_field(line)) +} + +fn header_lines(message: &[u8]) -> impl Iterator { + let mut start = 0; + memchr_iter(b'\n', message) + .map(|pos| pos + 1) + .chain(once(message.len())) + .map_while(move |end| { + let line = message.get(start..end)?; + start = end; + (!matches!(line, b"" | b"\r\n" | b"\n")).then_some(line) + }) +} + +fn is_continuation(line: &[u8]) -> bool { + matches!(line.first(), Some(b' ' | b'\t')) +} + +fn is_dkim2_field(line: &[u8]) -> bool { + line.split(|&ch| ch == b':').next().is_some_and(|name| { + hashify::set_ignore_case!(name.trim_ascii(), "dkim2-signature", "message-instance") + }) +} + struct Dkim2Envelopes<'x> { undisclosed_recipients: Vec<(usize, &'x str)>, disclosed_recipients: Vec<&'x str>, @@ -194,38 +459,22 @@ impl MessageWrapper { }; } - let mut recipients = HashSet::with_capacity(self.message.recipients.len()); + let mut recipients = AHashSet::with_capacity(self.message.recipients.len()); - for addr in message.headers.iter().filter_map(|(name, value)| { + for (name, value) in message.headers() { let name = name.trim_ascii(); if name.len() == 2 && (name.eq_ignore_ascii_case(b"to") || name.eq_ignore_ascii_case(b"cc")) { - MessageStream::new(value).parse_address().into_address() - } else { - None - } - }) { - match addr { - Address::List(addrs) => { + let parsed = HeaderForm::Addresses.parse(value); + if let Some(addresses) = parsed.value().as_address() { recipients.extend( - addrs - .iter() + addresses + .mailboxes() .filter_map(|a| a.address()) .map(sanitize_or_lower), ); } - Address::Group(groups) => { - for group in groups { - recipients.extend( - group - .addresses - .iter() - .filter_map(|a| a.address()) - .map(sanitize_or_lower), - ); - } - } } } @@ -252,167 +501,3 @@ fn sanitize_or_lower(rcpt: &str) -> String { sanitize_email(rcpt).unwrap_or_else(|| rcpt.to_lowercase()) } -#[cfg(test)] -mod tests { - use super::*; - use crate::queue::{Message, QueueId, Recipient}; - use common::config::smtp::queue::QueueName; - use std::net::{IpAddr, Ipv4Addr}; - - fn wrapper(recipients: &[&str]) -> MessageWrapper { - MessageWrapper { - queue_id: 0 as QueueId, - queue_name: QueueName::default(), - is_multi_queue: false, - span_id: 0, - message: Message { - created: 0, - blob_hash: Default::default(), - return_path: "sender@example.com".into(), - recipients: recipients.iter().map(Recipient::new).collect(), - received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), - received_via_port: 0, - flags: 0, - env_id: None, - priority: 0, - size: 0, - metadata: Default::default(), - }, - } - } - - fn split(headers: &str, recipients: &[&str]) -> (Vec, Vec) { - let raw = format!("{headers}\r\nSubject: test\r\n\r\nbody\r\n"); - let auth = AuthenticatedMessage::parse(raw.as_bytes()).expect("parse message"); - let message = wrapper(recipients); - let envelopes = message.undisclosed_recipients(&auth); - let mut disclosed = envelopes - .disclosed_recipients - .iter() - .map(|s| s.to_string()) - .collect::>(); - let mut undisclosed = envelopes - .undisclosed_recipients - .iter() - .map(|(_, s)| s.to_string()) - .collect::>(); - disclosed.sort(); - undisclosed.sort(); - (disclosed, undisclosed) - } - - #[test] - fn single_recipient_is_always_disclosed() { - let (disclosed, undisclosed) = split("To: someone-else@example.com", &["bcc@example.org"]); - assert_eq!(disclosed, vec!["bcc@example.org".to_string()]); - assert!(undisclosed.is_empty()); - } - - #[test] - fn all_recipients_disclosed() { - let (disclosed, undisclosed) = split( - "To: alice@example.com, bob@example.com\r\nCc: carol@example.com", - &["alice@example.com", "bob@example.com", "carol@example.com"], - ); - assert_eq!( - disclosed, - vec![ - "alice@example.com".to_string(), - "bob@example.com".to_string(), - "carol@example.com".to_string(), - ] - ); - assert!(undisclosed.is_empty()); - } - - #[test] - fn mixed_disclosed_and_undisclosed() { - let (disclosed, undisclosed) = split( - "To: alice@example.com\r\nCc: bob@example.com", - &[ - "alice@example.com", - "bob@example.com", - "eve@secret.example.org", - "mallory@secret.example.org", - ], - ); - assert_eq!( - disclosed, - vec![ - "alice@example.com".to_string(), - "bob@example.com".to_string() - ] - ); - assert_eq!( - undisclosed, - vec![ - "eve@secret.example.org".to_string(), - "mallory@secret.example.org".to_string(), - ] - ); - } - - #[test] - fn no_to_or_cc_header_all_undisclosed() { - let (disclosed, undisclosed) = split( - "From: sender@example.com", - &["eve@example.org", "mallory@example.org"], - ); - assert!(disclosed.is_empty()); - assert_eq!( - undisclosed, - vec![ - "eve@example.org".to_string(), - "mallory@example.org".to_string() - ] - ); - } - - #[test] - fn group_addresses_are_disclosed() { - let (disclosed, undisclosed) = split( - "To: Team:alice@example.com,bob@example.com;", - &["alice@example.com", "bob@example.com", "eve@example.org"], - ); - assert_eq!( - disclosed, - vec![ - "alice@example.com".to_string(), - "bob@example.com".to_string() - ] - ); - assert_eq!(undisclosed, vec!["eve@example.org".to_string()]); - } - - #[test] - fn header_address_casing_is_ignored() { - let (disclosed, undisclosed) = split( - "To: Alice@Example.COM, BOB@EXAMPLE.com", - &["alice@example.com", "bob@example.com", "eve@example.org"], - ); - assert_eq!( - disclosed, - vec![ - "alice@example.com".to_string(), - "bob@example.com".to_string() - ] - ); - assert_eq!(undisclosed, vec!["eve@example.org".to_string()]); - } - - #[test] - fn display_names_and_brackets_are_ignored() { - let (disclosed, undisclosed) = split( - "To: \"Alice Doe\" \r\nCc: Bob ", - &["alice@example.com", "bob@example.com", "eve@example.org"], - ); - assert_eq!( - disclosed, - vec![ - "alice@example.com".to_string(), - "bob@example.com".to_string() - ] - ); - assert_eq!(undisclosed, vec!["eve@example.org".to_string()]); - } -} diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs index d82d46b8..17cf13e9 100644 --- a/crates/smtp/src/inbound/ehlo.rs +++ b/crates/smtp/src/inbound/ehlo.rs @@ -5,14 +5,9 @@ */ use crate::{core::Session, scripts::ScriptResult}; -use common::{ - config::smtp::session::{Mechanism, Stage}, - network::SessionStream, -}; -use mail_auth::{ - SpfResult, - spf::verify::{HasValidLabels, SpfParameters}, -}; +use common::{config::smtp::session::Stage, network::SessionStream}; +use compact_str::CompactString; +use mail_auth::{SpfResult, dns::has_valid_labels, spf::verify::SpfParameters}; use smtp_proto::*; use std::{ borrow::Cow, @@ -26,11 +21,11 @@ impl Session { if domain != self.data.helo_domain { // Reject non-FQDN EHLO domains - simply checks that the hostname has at least one dot - if self.params.ehlo_reject_non_fqdn && !domain.as_ref().has_valid_labels() { + if self.params.ehlo_reject_non_fqdn && !has_valid_labels(domain.as_ref()) { trc::event!( Smtp(SmtpEvent::InvalidEhlo), SpanId = self.data.session_id, - Domain = domain.as_ref().to_string(), + Domain = CompactString::from(domain.as_ref()), ); return self.write(b"550 5.5.0 Invalid EHLO domain.\r\n").await; @@ -39,7 +34,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::Ehlo), SpanId = self.data.session_id, - Domain = domain.as_ref().to_string(), + Domain = CompactString::from(domain.as_ref()), ); // SPF check @@ -47,20 +42,20 @@ impl Session { std::mem::replace(&mut self.data.helo_domain, domain.into_owned()); if self.params.spf_ehlo.verify() { let time = Instant::now(); - let spf_output = self - .server - .core - .smtp - .resolvers - .dns - .verify_spf(self.server.inner.cache.build_auth_parameters( - SpfParameters::verify_ehlo( - self.data.remote_ip, - &self.data.helo_domain, - &self.hostname, - ), - )) - .await; + let spf_output = + self.server + .core + .smtp + .resolvers + .dns + .verify_spf(self.server.inner.cache.build_auth_parameters( + SpfParameters::helo( + self.data.remote_ip, + &self.data.helo_domain, + &self.hostname, + ), + )) + .await; trc::event!( Smtp(if matches!(spf_output.result(), SpfResult::Pass) { @@ -88,12 +83,7 @@ impl Session { // Sieve filtering if let Some((script, script_id)) = self - .server - .eval_if::( - &self.server.core.smtp.session.ehlo.script, - self, - self.data.session_id, - ) + .eval_if::(&self.server.core.smtp.session.ehlo.script) .await .and_then(|name| { self.server @@ -101,11 +91,7 @@ impl Session { .map(|s| (s, name)) }) && let ScriptResult::Reject(message) = self - .run_script( - script_id, - script.clone(), - self.build_script_parameters("ehlo"), - ) + .run_script(script_id, script, self.build_script_parameters("ehlo")) .await { self.data.mail_from = None; @@ -149,88 +135,48 @@ impl Session { response.capabilities |= EXT_START_TLS; } let ec = &self.server.core.smtp.session.extensions; - let ac = &self.server.core.smtp.session.auth; let dc = &self.server.core.smtp.session.data; // Pipelining - if self - .server - .eval_if(&ec.pipelining, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&ec.pipelining).await.unwrap_or(true) { response.capabilities |= EXT_PIPELINING; } // Chunking - if self - .server - .eval_if(&ec.chunking, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&ec.chunking).await.unwrap_or(true) { response.capabilities |= EXT_CHUNKING; } // Address Expansion - if self - .server - .eval_if(&ec.expn, self, self.data.session_id) - .await - .unwrap_or(false) - { + if self.eval_if(&ec.expn).await.unwrap_or(false) { response.capabilities |= EXT_EXPN; } // Recipient Verification - if self - .server - .eval_if(&ec.vrfy, self, self.data.session_id) - .await - .unwrap_or(false) - { + if self.eval_if(&ec.vrfy).await.unwrap_or(false) { response.capabilities |= EXT_VRFY; } // Require TLS - if self - .server - .eval_if(&ec.requiretls, self, self.data.session_id) - .await - .unwrap_or(true) - { + if self.eval_if(&ec.requiretls).await.unwrap_or(true) { response.capabilities |= EXT_REQUIRE_TLS; } // DSN - if self - .server - .eval_if(&ec.dsn, self, self.data.session_id) - .await - .unwrap_or(false) - { + if self.eval_if(&ec.dsn).await.unwrap_or(false) { response.capabilities |= EXT_DSN; } // Authentication if !self.is_authenticated() { - response.auth_mechanisms = self - .server - .eval_if::(&ac.mechanisms, self, self.data.session_id) - .await - .unwrap_or_default() - .into(); + response.auth_mechanisms = self.sasl_mechanisms().await; if response.auth_mechanisms != 0 { response.capabilities |= EXT_AUTH; } } // Future release - if let Some(value) = self - .server - .eval_if::(&ec.future_release, self, self.data.session_id) - .await - { + if let Some(value) = self.eval_if::(&ec.future_release).await { response.capabilities |= EXT_FUTURE_RELEASE; response.future_release_interval = value.as_secs(); response.future_release_datetime = SystemTime::now() @@ -241,29 +187,20 @@ impl Session { } // Deliver By - if let Some(value) = self - .server - .eval_if::(&ec.deliver_by, self, self.data.session_id) - .await - { + if let Some(value) = self.eval_if::(&ec.deliver_by).await { response.capabilities |= EXT_DELIVER_BY; response.deliver_by = value.as_secs(); } // Priority - if let Some(value) = self - .server - .eval_if::(&ec.mt_priority, self, self.data.session_id) - .await - { + if let Some(value) = self.eval_if::(&ec.mt_priority).await { response.capabilities |= EXT_MT_PRIORITY; response.mt_priority = value; } // Size response.size = self - .server - .eval_if(&dc.max_message_size, self, self.data.session_id) + .eval_if(&dc.max_message_size) .await .unwrap_or(25 * 1024 * 1024); if response.size > 0 { @@ -271,11 +208,7 @@ impl Session { } // No soliciting - if let Some(value) = self - .server - .eval_if::(&ec.no_soliciting, self, self.data.session_id) - .await - { + if let Some(value) = self.eval_if::(&ec.no_soliciting).await { response.capabilities |= EXT_NO_SOLICITING; response.no_soliciting = if !value.is_empty() { value.to_string().into() diff --git a/crates/smtp/src/inbound/hooks/client.rs b/crates/smtp/src/inbound/hooks/client.rs index 252f1cfb..0f228bff 100644 --- a/crates/smtp/src/inbound/hooks/client.rs +++ b/crates/smtp/src/inbound/hooks/client.rs @@ -7,21 +7,18 @@ use common::config::smtp::session::MTAHook; use utils::HttpLimitResponse; -use super::{Request, Response}; +use super::Response; pub(super) async fn send_mta_hook_request( mta_hook: &MTAHook, - request: Request, + payload: Vec, ) -> Result { let response = mta_hook .client .post(&mta_hook.url) .timeout(mta_hook.timeout) .headers(mta_hook.headers.clone()) - .body( - serde_json::to_string(&request) - .map_err(|err| format!("Failed to serialize Hook request: {}", err))?, - ) + .body(payload) .send() .await .map_err(|err| format!("Hook request failed: {err}"))?; diff --git a/crates/smtp/src/inbound/hooks/message.rs b/crates/smtp/src/inbound/hooks/message.rs index eebb4702..d4d3178b 100644 --- a/crates/smtp/src/inbound/hooks/message.rs +++ b/crates/smtp/src/inbound/hooks/message.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Action, Queue, Response, client::send_mta_hook_request}; +use super::{Action, Queue, client::send_mta_hook_request}; use crate::{ core::Session, inbound::{ @@ -17,11 +17,8 @@ use crate::{ queue::QueueId, }; use ahash::AHashMap; -use common::{ - DAEMON_NAME, - config::smtp::session::{MTAHook, Stage}, - network::SessionStream, -}; +use common::{DAEMON_NAME, config::smtp::session::Stage, expr::Bump, network::SessionStream}; +use compact_str::ToCompactString; use mail_auth::AuthenticatedMessage; use std::time::Instant; use trc::MtaHookEvent; @@ -39,11 +36,13 @@ impl Session { } let mut modifications = Vec::new(); - for mta_hook in mta_hooks { + let mut arena = Bump::new(); + let mut payload = None; + for (idx, mta_hook) in mta_hooks.iter().enumerate() { if !mta_hook.run_on_stage.contains(&stage) || !self .server - .eval_if(&mta_hook.enable, self, self.data.session_id) + .eval_if(&mta_hook.enable, self, &mut arena, self.data.session_id) .await .unwrap_or(false) { @@ -51,7 +50,19 @@ impl Session { } let time = Instant::now(); - match self.run_mta_hook(stage, mta_hook, message, queue_id).await { + let has_next = mta_hooks + .get(idx + 1..) + .unwrap_or_default() + .iter() + .any(|next| next.run_on_stage.contains(&stage)); + let result = match payload + .get_or_insert_with(|| self.mta_hook_payload(stage, message, queue_id)) + { + Ok(body) if has_next => send_mta_hook_request(mta_hook, body.clone()).await, + Ok(body) => send_mta_hook_request(mta_hook, std::mem::take(body)).await, + Err(err) => Err(err.clone()), + }; + match result { Ok(response) => { trc::event!( MtaHook(match response.action { @@ -62,7 +73,7 @@ impl Session { }), SpanId = self.data.session_id, QueueId = queue_id, - Id = mta_hook.id.to_string(), + Name = mta_hook.id.to_compact_string(), Elapsed = time.elapsed(), ); @@ -86,7 +97,7 @@ impl Session { } super::Modification::ReplaceContents { value } => { Modification::ReplaceBody { - value: value.as_bytes().to_vec(), + value: value.into_bytes(), } } super::Modification::AddHeader { name, value } => { @@ -153,7 +164,7 @@ impl Session { trc::event!( MtaHook(MtaHookEvent::Error), SpanId = self.data.session_id, - Id = mta_hook.id.to_string(), + Name = mta_hook.id.to_compact_string(), Reason = err, Elapsed = time.elapsed(), ); @@ -168,20 +179,18 @@ impl Session { Ok(modifications) } - pub async fn run_mta_hook( + pub fn mta_hook_payload( &self, stage: Stage, - mta_hook: &MTAHook, message: Option<&AuthenticatedMessage<'_>>, queue_id: Option, - ) -> Result { - // Build request + ) -> Result, String> { let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); let request = Request { context: Context { stage: stage.into(), client: Client { - ip: self.data.remote_ip.to_string(), + ip: self.data.remote_ip_str.clone(), port: self.data.remote_port, ptr: self .data @@ -208,7 +217,7 @@ impl Session { server: Server { name: Some(DAEMON_NAME.into()), port: self.data.local_port, - ip: self.data.local_ip.to_string().into(), + ip: self.data.local_ip_str.clone().into(), }, queue: queue_id.map(|id| Queue { id: format!("{:x}", id), @@ -232,22 +241,21 @@ impl Session { }), message: message.map(|message| Message { headers: message - .raw_parsed_headers() + .headers() .iter() - .map(|(k, v)| { - ( - String::from_utf8_lossy(k).into_owned(), - String::from_utf8_lossy(v).into_owned(), - ) - }) + .map(|(k, v)| (String::from_utf8_lossy(k), String::from_utf8_lossy(v))) .collect(), server_headers: vec![], - contents: String::from_utf8_lossy(message.raw_body()).into_owned(), + contents: String::from_utf8_lossy(message.raw_body()), size: message.raw_message().len(), }), }; - send_mta_hook_request(mta_hook, request).await + let size = message.map_or(0, |message| message.raw_message().len()); + let mut payload = Vec::with_capacity(size + (size / 8) + 1024); + serde_json::to_writer(&mut payload, &request) + .map_err(|err| format!("Failed to serialize Hook request: {}", err))?; + Ok(payload) } } diff --git a/crates/smtp/src/inbound/hooks/mod.rs b/crates/smtp/src/inbound/hooks/mod.rs index bc78a312..8e415bec 100644 --- a/crates/smtp/src/inbound/hooks/mod.rs +++ b/crates/smtp/src/inbound/hooks/mod.rs @@ -8,16 +8,16 @@ pub mod client; pub mod message; use ahash::AHashMap; - use serde::{Deserialize, Serialize}; +use std::borrow::Cow; #[derive(Serialize, Deserialize)] -pub struct Request { +pub struct Request<'x> { pub context: Context, #[serde(skip_serializing_if = "Option::is_none")] pub envelope: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub message: Option, + pub message: Option>, } #[derive(Serialize, Deserialize)] @@ -113,13 +113,13 @@ pub struct Envelope { } #[derive(Serialize, Deserialize)] -pub struct Message { - pub headers: Vec<(String, String)>, +pub struct Message<'x> { + pub headers: Vec<(Cow<'x, str>, Cow<'x, str>)>, #[serde(skip_serializing_if = "Vec::is_empty")] #[serde(rename = "serverHeaders")] #[serde(default)] pub server_headers: Vec<(String, String)>, - pub contents: String, + pub contents: Cow<'x, str>, pub size: usize, } diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs index 482f1915..ce452185 100644 --- a/crates/smtp/src/inbound/mail.rs +++ b/crates/smtp/src/inbound/mail.rs @@ -9,6 +9,7 @@ use crate::{ scripts::ScriptResult, }; use common::{config::smtp::session::Stage, network::SessionStream, scripts::ScriptModification}; +use compact_str::CompactString; use mail_auth::{IprevOutput, IprevResult, SpfOutput, SpfResult, spf::verify::SpfParameters}; use mail_parser::DateTime; use registry::schema::structs::Rate; @@ -118,6 +119,7 @@ impl Session { }; let has_dsn = from.env_id.is_some(); + self.data.message_size = from.size; self.data.mail_from = SessionAddress { address, address_lcase, @@ -129,12 +131,7 @@ impl Session { // Check whether the address is allowed if !self - .server - .eval_if::( - &self.server.core.smtp.session.mail.is_allowed, - self, - self.data.session_id, - ) + .eval_if::(&self.server.core.smtp.session.mail.is_allowed) .await .unwrap_or(true) { @@ -151,12 +148,7 @@ impl Session { // Sieve filtering if let Some((script, script_id)) = self - .server - .eval_if::( - &self.server.core.smtp.session.mail.script, - self, - self.data.session_id, - ) + .eval_if::(&self.server.core.smtp.session.mail.script) .await .and_then(|name| { self.server @@ -165,14 +157,10 @@ impl Session { }) { match self - .run_script( - script_id, - script.clone(), - self.build_script_parameters("mail"), - ) + .run_script(script_id, script, self.build_script_parameters("mail")) .await { - ScriptResult::Accept { modifications } if !modifications.is_empty() => { + ScriptResult::Accept { modifications, .. } if !modifications.is_empty() => { for modification in modifications { if let ScriptModification::SetEnvelope { name, value } = modification { self.data.apply_envelope_modification(name, value); @@ -201,12 +189,7 @@ impl Session { // Address rewriting if let Some(new_address) = self - .server - .eval_if::( - &self.server.core.smtp.session.mail.rewrite, - self, - self.data.session_id, - ) + .eval_if::(&self.server.core.smtp.session.mail.rewrite) .await { let mail_from = self.data.mail_from.as_mut().unwrap(); @@ -214,7 +197,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::MailFromRewritten), SpanId = self.data.session_id, - Details = mail_from.address_lcase.clone(), + Address = mail_from.address_lcase.clone(), From = new_address.clone(), ); @@ -233,12 +216,7 @@ impl Session { match self.authenticated_as() { Some(authenticated_as) if self - .server - .eval_if( - &self.server.core.smtp.session.auth.must_match_sender, - self, - self.data.session_id, - ) + .eval_if(&self.server.core.smtp.session.auth.must_match_sender) .await .unwrap_or(true) => { @@ -252,8 +230,8 @@ impl Session { trc::event!( Smtp(SmtpEvent::MailFromUnauthorized), SpanId = self.data.session_id, - From = address_lcase.to_string(), - Details = [trc::Value::String(authenticated_as.into())] + From = CompactString::from(address_lcase), + Address = [trc::Value::String(authenticated_as.into())] .into_iter() .chain( self.authenticated_emails() @@ -275,11 +253,7 @@ impl Session { let config = &self.server.core.smtp.session.extensions; let config_data = &self.server.core.smtp.session.data; if (from.flags & MAIL_REQUIRETLS) != 0 - && !self - .server - .eval_if(&config.requiretls, self, self.data.session_id) - .await - .unwrap_or(false) + && !self.eval_if(&config.requiretls).await.unwrap_or(false) { trc::event!( Smtp(SmtpEvent::RequireTlsDisabled), @@ -291,11 +265,7 @@ impl Session { .await; } if (from.flags & (MAIL_BY_NOTIFY | MAIL_BY_RETURN)) != 0 { - if let Some(duration) = self - .server - .eval_if::(&config.deliver_by, self, self.data.session_id) - .await - { + if let Some(duration) = self.eval_if::(&config.deliver_by).await { if from.by.checked_abs().unwrap_or(0) as u64 <= duration.as_secs() && (from.by.is_positive() || (from.flags & MAIL_BY_NOTIFY) != 0) { @@ -306,7 +276,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::DeliverByInvalid), SpanId = self.data.session_id, - Details = from.by, + Value = from.by, ); return self @@ -332,18 +302,17 @@ impl Session { } if from.mt_priority != 0 { if self - .server - .eval_if::(&config.mt_priority, self, self.data.session_id) + .eval_if::(&config.mt_priority) .await .is_some() { - if (-6..6).contains(&from.mt_priority) { + if (-9..=9).contains(&from.mt_priority) { self.data.priority = from.mt_priority as i16; } else { trc::event!( Smtp(SmtpEvent::MtPriorityInvalid), SpanId = self.data.session_id, - Details = from.mt_priority, + Value = from.mt_priority, ); self.data.mail_from = None; return self.write(b"501 5.5.4 Invalid priority value.\r\n").await; @@ -361,8 +330,7 @@ impl Session { } if from.size > 0 { let max_message_size = self - .server - .eval_if::(&config_data.max_message_size, self, self.data.session_id) + .eval_if::(&config_data.max_message_size) .await .unwrap_or(25 * 1024 * 1024); @@ -393,11 +361,7 @@ impl Session { .write(b"501 5.5.4 Only one of HOLDFOR or HOLDUNTIL may be specified.\r\n") .await; } - if let Some(max_hold) = self - .server - .eval_if::(&config.future_release, self, self.data.session_id) - .await - { + if let Some(max_hold) = self.eval_if::(&config.future_release).await { let max_hold = max_hold.as_secs(); let now = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -410,7 +374,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::FutureReleaseInvalid), SpanId = self.data.session_id, - Details = from.hold_until, + Value = from.hold_until, ); self.data.mail_from = None; return self @@ -423,7 +387,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::FutureReleaseInvalid), SpanId = self.data.session_id, - Details = hold_for, + Value = hold_for, ); self.data.mail_from = None; let response = if from.hold_for != 0 { @@ -449,13 +413,7 @@ impl Session { .await; } } - if has_dsn - && !self - .server - .eval_if(&config.dsn, self, self.data.session_id) - .await - .unwrap_or(false) - { + if has_dsn && !self.eval_if(&config.dsn).await.unwrap_or(false) { trc::event!(Smtp(SmtpEvent::DsnDisabled), SpanId = self.data.session_id,); self.data.mail_from = None; return self @@ -510,12 +468,11 @@ impl Session { }), SpanId = self.data.session_id, Domain = self.data.helo_domain.clone(), - From = if !mail_from.address.is_empty() { + From = CompactString::from(if !mail_from.address.is_empty() { mail_from.address.as_str() } else { "<>" - } - .to_string(), + }), Result = trc::Error::from(&spf_output), Elapsed = time.elapsed(), ); @@ -577,12 +534,7 @@ impl Session { // Send report if let (Some(recipient), Some(rate)) = ( spf_output.report_address(), - self.server - .eval_if::( - &self.server.core.smtp.report.spf.send, - self, - self.data.session_id, - ) + self.eval_if::(&self.server.core.smtp.report.spf.send) .await, ) { // Do not send SPF auth failures to local domains, as they are likely relay attempts (which are blocked later on) diff --git a/crates/smtp/src/inbound/milter/client.rs b/crates/smtp/src/inbound/milter/client.rs index c7133b90..bb725130 100644 --- a/crates/smtp/src/inbound/milter/client.rs +++ b/crates/smtp/src/inbound/milter/client.rs @@ -5,6 +5,7 @@ */ use common::config::smtp::session::Milter; +use compact_str::ToCompactString; use rustls_pki_types::ServerName; use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, @@ -28,6 +29,7 @@ impl MilterClient { for addr in &config.addrs { match TcpStream::connect(addr).await { Ok(stream) => { + let _ = stream.set_nodelay(true); return Ok(MilterClient { stream, timeout_cmd: config.timeout_command, @@ -313,8 +315,8 @@ impl MilterClient { trc::event!( Milter(MilterEvent::Write), SpanId = self.session_id, - Id = self.id.to_string(), - Contents = action.to_string(), + Name = self.id.to_compact_string(), + Contents = action.to_compact_string(), ); tokio::time::timeout(self.timeout_cmd, async { @@ -333,8 +335,8 @@ impl MilterClient { trc::event!( Milter(MilterEvent::Read), SpanId = self.session_id, - Id = self.id.to_string(), - Contents = response.to_string(), + Name = self.id.to_compact_string(), + Contents = response.to_compact_string(), ); return Ok(response); diff --git a/crates/smtp/src/inbound/milter/message.rs b/crates/smtp/src/inbound/milter/message.rs index e910c2ba..10676bb8 100644 --- a/crates/smtp/src/inbound/milter/message.rs +++ b/crates/smtp/src/inbound/milter/message.rs @@ -13,8 +13,10 @@ use crate::{ use common::{ DAEMON_NAME, config::smtp::session::{Milter, Stage}, + expr::Bump, network::SessionStream, }; +use compact_str::ToCompactString; use mail_auth::AuthenticatedMessage; use smtp_proto::{IntoString, request::parser::Rfc5321Parser}; use std::{borrow::Cow, time::Instant}; @@ -40,11 +42,12 @@ impl Session { } let mut modifications = Vec::new(); + let mut arena = Bump::new(); for milter in milters { if !milter.run_on_stage.contains(&stage) || !self .server - .eval_if(&milter.enable, self, self.data.session_id) + .eval_if(&milter.enable, self, &mut arena, self.data.session_id) .await .unwrap_or(false) { @@ -57,7 +60,7 @@ impl Session { trc::event!( Milter(MilterEvent::ActionAccept), SpanId = self.data.session_id, - Id = milter.id.to_string(), + Name = milter.id.to_compact_string(), Elapsed = time.elapsed(), ); @@ -91,7 +94,7 @@ impl Session { }), SpanId = self.data.session_id, QueueId = queue_id, - Id = milter.id.to_string(), + Name = milter.id.to_compact_string(), Elapsed = time.elapsed(), ); @@ -119,9 +122,10 @@ impl Session { } Err(Rejection::Error(err)) => { let (code, details) = match err { - Error::Io(details) => { - (MilterEvent::IoError, trc::Value::from(details.to_string())) - } + Error::Io(details) => ( + MilterEvent::IoError, + trc::Value::from(details.to_compact_string()), + ), Error::FrameTooLarge(size) => { (MilterEvent::FrameTooLarge, trc::Value::from(size)) } @@ -130,7 +134,7 @@ impl Session { } Error::Unexpected(response) => ( MilterEvent::UnexpectedResponse, - trc::Value::from(response.to_string()), + trc::Value::from(response.to_compact_string()), ), Error::Timeout => (MilterEvent::Timeout, trc::Value::None), Error::TLSInvalidName => (MilterEvent::TlsInvalidName, trc::Value::None), @@ -140,7 +144,7 @@ impl Session { trc::event!( Milter(code), SpanId = self.data.session_id, - Id = milter.id.to_string(), + Name = milter.id.to_compact_string(), Details = details, Elapsed = time.elapsed(), ); @@ -214,16 +218,18 @@ impl Session { .assert_continue()?; // EHLO/HELO - let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); - client - .helo( - &self.data.helo_domain, - Macros::new() - .with_cipher(tls_cipher.as_ref()) - .with_tls_version(tls_version.as_ref()), - ) - .await? - .assert_continue()?; + if !self.data.helo_domain.is_empty() { + let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); + client + .helo( + &self.data.helo_domain, + Macros::new() + .with_cipher(tls_cipher.as_ref()) + .with_tls_version(tls_version.as_ref()), + ) + .await? + .assert_continue()?; + } // Mail from if let Some(mail_from) = &self.data.mail_from { @@ -262,7 +268,7 @@ impl Session { // Headers client - .headers(message.raw_parsed_headers().iter().map(|(k, v)| { + .headers(message.headers().iter().map(|(k, v)| { ( std::str::from_utf8(k).unwrap_or_default(), std::str::from_utf8(v).unwrap_or_default(), @@ -326,7 +332,7 @@ impl SessionData { Milter(MilterEvent::ParseError), SpanId = self.session_id, Details = "Failed to parse milter mailFrom parameters", - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -362,7 +368,7 @@ impl SessionData { Milter(MilterEvent::ParseError), SpanId = self.session_id, Details = "Failed to parse milter rcptTo parameters", - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -378,7 +384,11 @@ impl SessionData { self.rcpt_to.retain(|r| r.address_lcase != recipient); } Modification::ReplaceBody { value } => { - body.extend(value); + if body.is_empty() { + body = value; + } else { + body.extend(value); + } } Modification::AddHeader { name, value } => { header_changes.push((0, name, value, false)); @@ -390,7 +400,7 @@ impl SessionData { Modification::ChangeHeader { index, name, value } => { if value.is_empty() || message - .raw_parsed_headers() + .headers() .iter() .any(|(n, _)| n.eq_ignore_ascii_case(name.as_bytes())) { @@ -426,7 +436,7 @@ impl SessionData { if needs_rewrite { let mut headers = message - .raw_parsed_headers() + .headers() .iter() .map(|(h, v)| (Cow::from(*h), Cow::from(*v))) .collect::>(); diff --git a/crates/smtp/src/inbound/milter/protocol.rs b/crates/smtp/src/inbound/milter/protocol.rs index 556fc6b0..78412eea 100644 --- a/crates/smtp/src/inbound/milter/protocol.rs +++ b/crates/smtp/src/inbound/milter/protocol.rs @@ -180,86 +180,6 @@ impl Command<'_> { Command::QuitNewConnection => Command::build(SMFIC_QUIT_NC, 0), } } - - #[cfg(feature = "test_mode")] - pub fn deserialize(bytes: &[u8]) -> Command<'_> { - let mut reader = PacketReader::new(bytes); - match reader.byte() { - SMFIC_ABORT => Command::Abort, - SMFIC_BODY => Command::Body { value: &bytes[1..] }, - SMFIC_BODYEOB => Command::EndOfBody, - SMFIC_CONNECT => { - let hostname = reader.read_nul_terminated().unwrap(); - let family = reader.byte(); - let port = reader.read_u16(); - let address = std::str::from_utf8(reader.read_nul_terminated().unwrap()).unwrap(); - Command::Connect { - hostname, - port, - address: match family { - b'4' => IpAddr::V4(address.parse().unwrap()), - b'6' => IpAddr::V6(address.parse().unwrap()), - _ => unreachable!(), - }, - } - } - SMFIC_MACRO => { - let cmdcode = reader.byte(); - let mut macros = Vec::new(); - while let Some(name) = reader.read_nul_terminated() { - let value = reader.read_nul_terminated().unwrap(); - macros.push(super::Macro { - name, - value: value.into(), - }); - } - Command::Macro { - macros: super::Macros { cmdcode, macros }, - } - } - SMFIC_HEADER => { - let name = reader.read_nul_terminated().unwrap(); - let value = reader.read_nul_terminated().unwrap(); - Command::Header { name, value } - } - SMFIC_EOH => Command::EndOfHeader, - SMFIC_HELO => { - let hostname = reader.read_nul_terminated().unwrap(); - Command::Helo { hostname } - } - SMFIC_MAIL => { - let sender = reader.read_nul_terminated().unwrap(); - let mut args = Vec::new(); - while let Some(arg) = reader.read_nul_terminated() { - args.push(arg); - } - Command::MailFrom { - sender, - args: Some(args), - } - } - SMFIC_RCPT => { - let recipient = reader.read_nul_terminated().unwrap(); - let mut args = Vec::new(); - while let Some(arg) = reader.read_nul_terminated() { - args.push(arg); - } - Command::Rcpt { - recipient, - args: Some(args), - } - } - SMFIC_OPTNEG => Command::OptionNegotiation(super::Options { - version: reader.read_u32(), - actions: reader.read_u32(), - protocol: reader.read_u32(), - }), - SMFIC_QUIT => Command::Quit, - SMFIC_DATA => Command::Data, - SMFIC_QUIT_NC => Command::QuitNewConnection, - c => panic!("Unknown command: {}", char::from(c)), - } - } } impl Response { @@ -347,117 +267,6 @@ impl Response { response => Err(Error::Unexpected(response)), } } - - #[cfg(feature = "test_mode")] - pub fn serialize(&self) -> Vec { - match self { - Response::Action(action) => match action { - Action::Accept => Command::build(SMFIR_ACCEPT, 0), - Action::Continue => Command::build(SMFIR_CONTINUE, 0), - Action::Discard => Command::build(SMFIR_DISCARD, 0), - Action::Reject => Command::build(SMFIR_REJECT, 0), - Action::TempFail => Command::build(SMFIR_TEMPFAIL, 0), - Action::ReplyCode { code, text } => { - let mut buf = Command::build(SMFIR_REPLYCODE, text.len() as u32 + 4 + 1); - buf.extend(code); - buf.push(b' '); - buf.extend(text.as_bytes()); - buf.push(0x00); - buf - } - Action::Shutdown => Command::build(SMFIR_SHUTDOWN, 0), - Action::ConnectionFailure => Command::build(SMFIR_CONN_FAIL, 0), - }, - Response::Modification(modif) => match modif { - Modification::ChangeFrom { sender, args } => { - let mut buf = - Command::build(SMFIR_CHGFROM, sender.len() as u32 + args.len() as u32 + 2); - buf.extend(sender.as_bytes()); - buf.push(0x00); - buf.extend(args.as_bytes()); - buf.push(0x00); - buf - } - Modification::AddRcpt { recipient, args } => { - let mut buf = Command::build( - SMFIR_ADDRCPT_PAR, - recipient.len() as u32 + args.len() as u32 + 2, - ); - buf.extend(recipient.as_bytes()); - buf.push(0x00); - buf.extend(args.as_bytes()); - buf.push(0x00); - buf - } - Modification::DeleteRcpt { recipient } => { - let mut buf = Command::build(SMFIR_DELRCPT, recipient.len() as u32 + 1); - buf.extend(recipient.as_bytes()); - buf.push(0x00); - buf - } - Modification::ReplaceBody { value } => { - let mut buf = Command::build(SMFIR_REPLBODY, value.len() as u32); - buf.extend(value); - buf - } - Modification::AddHeader { name, value } => { - let mut buf = - Command::build(SMFIR_ADDHEADER, name.len() as u32 + value.len() as u32 + 2); - buf.extend(name.as_bytes()); - buf.push(0x00); - buf.extend(value.as_bytes()); - buf.push(0x00); - buf - } - Modification::InsertHeader { index, name, value } => { - let mut buf = Command::build( - SMFIR_INSHEADER, - name.len() as u32 - + value.len() as u32 - + std::mem::size_of::() as u32 - + 2, - ); - buf.extend(index.to_be_bytes().as_slice()); - buf.extend(name.as_bytes()); - buf.push(0x00); - buf.extend(value.as_bytes()); - buf.push(0x00); - buf - } - Modification::ChangeHeader { index, name, value } => { - let mut buf = Command::build( - SMFIR_CHGHEADER, - name.len() as u32 - + value.len() as u32 - + std::mem::size_of::() as u32 - + 2, - ); - buf.extend(index.to_be_bytes().as_slice()); - buf.extend(name.as_bytes()); - buf.push(0x00); - buf.extend(value.as_bytes()); - buf.push(0x00); - buf - } - Modification::Quarantine { reason } => { - let mut buf = Command::build(SMFIR_QUARANTINE, reason.len() as u32 + 1); - buf.extend(reason.as_bytes()); - buf.push(0x00); - buf - } - }, - Response::Progress => Command::build(SMFIR_PROGRESS, 0), - Response::Skip => Command::build(SMFIR_SKIP, 0), - Response::SetSymbols => Command::build(SMFIR_SETSYMLIST, 0), - Response::OptionNegotiation(opt) => { - let mut buf = Command::build(SMFIC_OPTNEG, 3 * std::mem::size_of::() as u32); - buf.extend(opt.version.to_be_bytes().as_slice()); - buf.extend(opt.actions.to_be_bytes().as_slice()); - buf.extend(opt.protocol.to_be_bytes().as_slice()); - buf - } - } - } } fn read_nul_terminated(bytes: &mut std::slice::Iter, expected_len: usize) -> Option { @@ -479,54 +288,3 @@ fn read_u32(bytes: &mut std::slice::Iter) -> Option { Some(u32::from_be_bytes(buf)) } -#[cfg(feature = "test_mode")] -pub struct PacketReader<'x> { - bytes: &'x [u8], - iter: std::iter::Enumerate>, -} - -#[cfg(feature = "test_mode")] -impl<'x> PacketReader<'x> { - pub fn new(bytes: &'x [u8]) -> PacketReader<'x> { - Self { - bytes, - iter: bytes.iter().enumerate(), - } - } - - pub fn byte(&mut self) -> u8 { - *self.iter.next().unwrap().1 - } - - pub fn read_nul_terminated(&mut self) -> Option<&'x [u8]> { - let (start_pos, ch) = self.iter.next()?; - let mut end_pos = start_pos; - - if *ch != 0x00 { - loop { - match self.iter.next().unwrap().1 { - 0x00 => break, - _ => end_pos += 1, - } - } - } - - Some(&self.bytes[start_pos..end_pos + 1]) - } - - pub fn read_u32(&mut self) -> u32 { - let mut buf = [0u8; 4]; - for byte in buf.iter_mut() { - *byte = self.byte(); - } - u32::from_be_bytes(buf) - } - - pub fn read_u16(&mut self) -> u16 { - let mut buf = [0u8; 2]; - for byte in buf.iter_mut() { - *byte = self.byte(); - } - u16::from_be_bytes(buf) - } -} diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index 1eb25e09..b12b1b9a 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -14,6 +14,7 @@ use common::{ network::{RcptResolution, SessionStream}, scripts::ScriptModification, }; +use compact_str::CompactString; use smtp_proto::{ RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, RcptTo, }; @@ -24,7 +25,7 @@ use utils::DomainPart; impl Session { pub async fn handle_rcpt_to(&mut self, to: RcptTo>) -> Result<(), ()> { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if self.instance.id.ends_with("-debug") { if to.address.contains("fail@") { return self.write(b"503 5.5.1 Invalid recipient.\r\n").await; @@ -93,13 +94,12 @@ impl Session { // Address rewriting and Sieve filtering let rcpt_config = &self.server.core.smtp.session.rcpt; let rcpt_script = self - .server - .eval_if::(&rcpt_config.script, self, self.data.session_id) + .eval_if::(&rcpt_config.script) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) - .map(|s| (s.clone(), name)) + .map(|s| (s, name)) }); let session_config = &self.server.core.smtp.session; @@ -117,14 +117,10 @@ impl Session { // Sieve filtering if let Some((script, script_id)) = rcpt_script { match self - .run_script( - script_id, - script.clone(), - self.build_script_parameters("rcpt"), - ) + .run_script(script_id, script, self.build_script_parameters("rcpt")) .await { - ScriptResult::Accept { modifications } if !modifications.is_empty() => { + ScriptResult::Accept { modifications, .. } if !modifications.is_empty() => { for modification in modifications { if let ScriptModification::SetEnvelope { name, value } = modification { self.data.apply_envelope_modification(name, value); @@ -152,17 +148,13 @@ impl Session { } // Address rewriting - if let Some(new_address) = self - .server - .eval_if::(&rcpt_config.rewrite, self, self.data.session_id) - .await - { + if let Some(new_address) = self.eval_if::(&rcpt_config.rewrite).await { let rcpt = self.data.rcpt_to.last_mut().unwrap(); trc::event!( Smtp(SmtpEvent::RcptToRewritten), SpanId = self.data.session_id, - Details = rcpt.address_lcase.clone(), + Address = rcpt.address_lcase.clone(), To = new_address.clone(), ); @@ -174,8 +166,8 @@ impl Session { } // Check for duplicates - let rcpt = self.data.rcpt_to.last().unwrap(); - if self.data.rcpt_to.iter().filter(|r| r == &rcpt).count() > 1 { + let (rcpt, previous) = self.data.rcpt_to.split_last().unwrap(); + if previous.contains(rcpt) { trc::event!( Smtp(SmtpEvent::RcptToDuplicate), SpanId = self.data.session_id, @@ -227,16 +219,11 @@ impl Session { let rcpt_to = self.data.rcpt_to.pop().unwrap().address_lcase; return self - .rcpt_error(b"550 5.1.2 Mailbox does not exist.\r\n", rcpt_to) + .rcpt_error(b"550 5.1.1 Mailbox does not exist.\r\n", rcpt_to) .await; } Ok(RcptResolution::UnknownDomain) => { - if !self - .server - .eval_if(&rcpt_config.relay, self, self.data.session_id) - .await - .unwrap_or(false) - { + if !self.eval_if(&rcpt_config.relay).await.unwrap_or(false) { trc::event!( Smtp(SmtpEvent::RelayNotAllowed), SpanId = self.data.session_id, @@ -245,7 +232,7 @@ impl Session { let rcpt_to = self.data.rcpt_to.pop().unwrap().address_lcase; return self - .rcpt_error(b"550 5.1.2 Relay not allowed.\r\n", rcpt_to) + .rcpt_error(b"550 5.7.1 Relay not allowed.\r\n", rcpt_to) .await; } } @@ -280,12 +267,20 @@ impl Session { .address_lcase .as_bytes(); let to_addr = self.data.rcpt_to.last().unwrap().address_lcase.as_bytes(); - let mut key = Vec::with_capacity(from_addr.len() + to_addr.len() + 1); + let mut hasher = blake3::Hasher::new(); + hasher.update(&(from_addr.len() as u64).to_be_bytes()); + hasher.update(from_addr); + hasher.update(to_addr); + let mut key = Vec::with_capacity(blake3::OUT_LEN + 1); key.push(KV_GREYLIST); - key.extend_from_slice(from_addr); - key.extend_from_slice(to_addr); + key.extend_from_slice(hasher.finalize().as_bytes()); - match self.server.in_memory_store().key_exists(key.clone()).await { + match self + .server + .in_memory_store() + .key_exists(key.as_slice()) + .await + { Ok(true) => (), Ok(false) => { match self @@ -366,7 +361,7 @@ impl Session { err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to look up mailing list member.") - .ctx(trc::Key::To, member.to_string()) + .ctx(trc::Key::To, CompactString::from(member.as_ref())) ); false } @@ -387,7 +382,7 @@ impl Session { err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to resolve mailing list member.") - .ctx(trc::Key::To, member.to_string()) + .ctx(trc::Key::To, CompactString::from(member.as_ref())) ); member.to_string() } diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs index 73b77378..6f4fb990 100644 --- a/crates/smtp/src/inbound/session.rs +++ b/crates/smtp/src/inbound/session.rs @@ -4,13 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{auth::SaslToken, data::MessageOrigin}; +use crate::core::{DataFailure, Session, State}; use common::{ - config::{server::ServerProtocol, smtp::session::Mechanism}, - expr::{self, functions::ResolveVariable, *}, - network::SessionStream, + config::server::ServerProtocol, + expr::{Bump, Variable, functions::ResolveVariable}, + network::{ + SessionStream, + memory::{MEMORY_STEP, MemoryClass}, + }, }; - -use compact_str::ToCompactString; +use compact_str::{CompactString, ToCompactString, format_compact}; use registry::schema::enums::ExpressionVariable; use smtp_proto::{ request::receiver::{ @@ -20,11 +24,12 @@ use smtp_proto::{ *, }; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use trc::{NetworkEvent, SecurityEvent, SmtpEvent}; +use trc::{LimitEvent, NetworkEvent, SecurityEvent, SmtpEvent}; -use crate::core::{Session, State}; - -use super::auth::SaslToken; +const MAX_BATCHED_RESPONSE: usize = 4096; +const MIN_DATA_BUFFER: usize = 1024; +pub(crate) const INSUFFICIENT_STORAGE: &[u8] = + b"452 4.3.1 Insufficient system storage, try again later.\r\n"; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> Result { @@ -40,7 +45,17 @@ impl Session { self.handle_rcpt_to(to).await?; } Request::Mail { from } => { - self.handle_mail_from(from).await?; + if self.server.is_memory_available(MemoryClass::SmtpInbound) { + self.handle_mail_from(from).await?; + } else { + trc::event!( + Limit(LimitEvent::MemoryBudget), + SpanId = self.data.session_id, + Size = from.size, + Limit = self.server.core.network.max_buffered_memory, + ); + self.write(INSUFFICIENT_STORAGE).await?; + } } Request::Ehlo { host } => { if self.instance.protocol == ServerProtocol::Smtp { @@ -60,7 +75,10 @@ impl Session { } else { self.write(b"354 Start mail input; end with .\r\n") .await?; - self.data.message = Vec::with_capacity(1024); + self.data.message = Vec::new(); + self.data.message_memory = Some( + self.server.memory_reservation(MemoryClass::SmtpInbound), + ); state = State::Data(DataReceiver::new()); continue 'outer; } @@ -73,16 +91,21 @@ impl Session { State::SkipData( DummyDataReceiver::new_bdat(chunk_size), response, + DataFailure::Chunk, ) } else if chunk_size.saturating_add(self.data.message.len()) < self.params.max_message_size { - if self.data.message.is_empty() { - self.data.message = Vec::with_capacity(chunk_size); - } else { - self.data.message.reserve(chunk_size); + if self.data.message_memory.is_none() { + self.data.message_memory = Some( + self.server + .memory_reservation(MemoryClass::SmtpInbound), + ); + } + State::Bdat { + receiver: BdatReceiver::new(chunk_size, is_last), + bytes_left: chunk_size, } - State::Bdat(BdatReceiver::new(chunk_size, is_last)) } else { trc::event!( Smtp(SmtpEvent::MessageTooLarge), @@ -90,10 +113,12 @@ impl Session { Size = chunk_size.saturating_add(self.data.message.len()), Limit = self.params.max_message_size, ); + self.reset(); State::SkipData( DummyDataReceiver::new_bdat(chunk_size), b"552 5.3.4 Message too big for system.\r\n", + DataFailure::Chunk, ) }; continue 'outer; @@ -102,16 +127,7 @@ impl Session { mechanism, initial_response, } => { - let auth: u64 = self - .server - .eval_if::( - &self.server.core.smtp.session.auth.mechanisms, - self, - self.data.session_id, - ) - .await - .unwrap_or_default() - .into(); + let auth = self.sasl_mechanisms().await; if auth == 0 { trc::event!( Smtp(SmtpEvent::AuthNotAllowed), @@ -123,7 +139,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::AlreadyAuthenticated), SpanId = self.data.session_id, - AccountName = authenticated_as.to_string(), + AccountName = CompactString::from(authenticated_as), ); self.write(b"503 5.5.1 Already authenticated.\r\n").await?; @@ -172,7 +188,7 @@ impl Session { ); self.write(b"220 2.0.0 Ready to start TLS.\r\n").await?; - #[cfg(any(test, feature = "test_mode"))] + #[cfg(any(test, stalwart_tests))] if self.data.helo_domain.contains("badtls") { return Err(()); } @@ -246,7 +262,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::CommandNotImplemented), SpanId = self.data.session_id, - Details = format!("{cmd:?}"), + Details = format_compact!("{cmd:?}"), ); self.write(b"502 5.5.1 Command not implemented.\r\n") @@ -332,7 +348,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::InvalidParameter), SpanId = self.data.session_id, - Details = param + Name = param ); self.write( @@ -345,7 +361,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::UnsupportedParameter), SpanId = self.data.session_id, - Details = param.clone() + Value = param.clone() ); self.write( @@ -363,17 +379,16 @@ impl Session { }, State::Data(receiver) => { if self.data.message.len() + bytes.len() < self.params.max_message_size { - if receiver.ingest(&mut iter, &mut self.data.message) { - let message = self.queue_message().await; - let num_responses = if self.instance.protocol == ServerProtocol::Smtp { - 1 - } else { - self.data.rcpt_oks - }; + if !self.grow_message_buffer(iter.len()) { + state = State::SkipData( + DummyDataReceiver::new_data(receiver), + INSUFFICIENT_STORAGE, + DataFailure::Transaction, + ); + } else if receiver.ingest(&mut iter, &mut self.data.message) { + let message = self.queue_message(MessageOrigin::Client).await; if !message.is_empty() { - for _ in 0..num_responses { - self.write(message.as_ref()).await?; - } + self.write_data_responses(message.as_ref()).await?; self.reset(); state = State::default(); } else { @@ -394,23 +409,32 @@ impl Session { state = State::SkipData( DummyDataReceiver::new_data(receiver), b"552 5.3.4 Message too big for system.\r\n", + DataFailure::Transaction, ); } } - State::Bdat(receiver) => { - if receiver.ingest(&mut iter, &mut self.data.message) { + State::Bdat { + receiver, + bytes_left, + } => { + if !self.grow_message_buffer(iter.len().min(*bytes_left)) { + let bytes_left = *bytes_left; + self.reset(); + state = State::SkipData( + DummyDataReceiver::new_bdat(bytes_left), + INSUFFICIENT_STORAGE, + DataFailure::Chunk, + ); + continue 'outer; + } + let available = iter.len(); + let completed = receiver.ingest(&mut iter, &mut self.data.message); + *bytes_left = bytes_left.saturating_sub(available - iter.len()); + if completed { if receiver.is_last { - let message = self.queue_message().await; + let message = self.queue_message(MessageOrigin::Client).await; if !message.is_empty() { - let num_responses = - if self.instance.protocol == ServerProtocol::Smtp { - 1 - } else { - self.data.rcpt_oks - }; - for _ in 0..num_responses { - self.write(message.as_ref()).await?; - } + self.write_data_responses(message.as_ref()).await?; self.reset(); } else { // Disconnect requested @@ -451,10 +475,19 @@ impl Session { break 'outer; } } - State::SkipData(receiver, response) => { + State::SkipData(receiver, response, failure) => { if receiver.ingest(&mut iter) { - self.data.message = Vec::with_capacity(0); - self.write(response).await?; + match failure { + DataFailure::Chunk => { + self.data.message = Vec::with_capacity(0); + self.data.message_memory = None; + self.write(response).await?; + } + DataFailure::Transaction => { + self.write_data_responses(response).await?; + self.reset(); + } + } state = State::default(); } else { break 'outer; @@ -480,6 +513,66 @@ impl Session { Ok(true) } + + fn grow_message_buffer(&mut self, incoming: usize) -> bool { + let required = self.data.message.len().saturating_add(incoming); + let capacity = self.data.message.capacity(); + if required <= capacity { + return true; + } + + let target = if capacity == 0 { + self.data + .message_size + .clamp(MIN_DATA_BUFFER, MEMORY_STEP as usize) + } else { + capacity.saturating_mul(2) + } + .min(self.params.max_message_size) + .max(required); + if let Some(memory) = &mut self.data.message_memory + && !memory.reserve_to(target as u64) + { + trc::event!( + Limit(LimitEvent::MemoryBudget), + SpanId = self.data.session_id, + Size = target, + Limit = memory.limit(), + ); + self.data.message = Vec::new(); + self.data.message_memory = None; + return false; + } + + self.data + .message + .reserve_exact(target - self.data.message.len()); + true + } + + async fn write_data_responses(&mut self, response: &[u8]) -> Result<(), ()> { + let count = if self.instance.protocol == ServerProtocol::Smtp { + 1 + } else { + self.data.rcpt_oks + }; + match count { + 0 => return Ok(()), + 1 => return self.write(response).await, + _ => {} + } + + let per_write = (MAX_BATCHED_RESPONSE / response.len().max(1)).clamp(1, count); + let batch = response.repeat(per_write); + let mut remaining = count; + while remaining > 0 { + let chunk = remaining.min(per_write); + self.write(batch.get(..chunk * response.len()).unwrap_or_default()) + .await?; + remaining -= chunk; + } + Ok(()) + } } impl Session { @@ -488,6 +581,8 @@ impl Session { self.data.spf_mail_from = None; self.data.rcpt_to.clear(); self.data.message = Vec::with_capacity(0); + self.data.message_memory = None; + self.data.message_size = 0; self.data.priority = 0; self.data.delivery_by = 0; self.data.future_release = 0; @@ -519,7 +614,7 @@ impl Session { trc::event!( Network(NetworkEvent::FlushError), SpanId = self.data.session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); Err(()) } @@ -528,7 +623,7 @@ impl Session { trc::event!( Network(NetworkEvent::WriteError), SpanId = self.data.session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); Err(()) @@ -537,15 +632,17 @@ impl Session { } #[inline(always)] - pub async fn read(&mut self, bytes: &mut [u8]) -> Result { - match self.stream.read(bytes).await { + pub async fn read(&mut self, bytes: &mut Vec) -> Result { + bytes.clear(); + match self.stream.read_buf(bytes).await { Ok(len) => { trc::event!( Smtp(SmtpEvent::RawInput), SpanId = self.data.session_id, Size = len, - Contents = - String::from_utf8_lossy(bytes.get(0..len).unwrap_or_default()).into_owned(), + Contents = CompactString::from(String::from_utf8_lossy( + bytes.get(0..len).unwrap_or_default() + )), ); Ok(len) @@ -554,7 +651,7 @@ impl Session { trc::event!( Network(NetworkEvent::ReadError), SpanId = self.data.session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); Err(()) @@ -564,7 +661,11 @@ impl Session { } impl ResolveVariable for Session { - fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { ExpressionVariable::Rcpt => self .data @@ -580,13 +681,14 @@ impl ResolveVariable for Session { .map(|r| r.domain.as_str()) .unwrap_or_default() .into(), - ExpressionVariable::Recipients => self - .data - .rcpt_to - .iter() - .map(|r| Variable::from(r.address_lcase.as_str())) - .collect::>() - .into(), + ExpressionVariable::Recipients => Variable::Array( + arena.alloc_slice_fill_iter( + self.data + .rcpt_to + .iter() + .map(|r| Variable::String(r.address_lcase.as_str())), + ), + ), ExpressionVariable::Sender => self .data .mail_from @@ -611,7 +713,7 @@ impl ResolveVariable for Session { ExpressionVariable::LocalIp => self.data.local_ip_str.as_str().into(), ExpressionVariable::LocalPort => self.data.local_port.into(), ExpressionVariable::IsTls => self.stream.is_tls().into(), - ExpressionVariable::Priority => self.data.priority.to_compact_string().into(), + ExpressionVariable::Priority => self.data.priority.into(), ExpressionVariable::Protocol => self.instance.protocol.as_str().into(), ExpressionVariable::Asn => self .data @@ -625,15 +727,10 @@ impl ResolveVariable for Session { .data .asn_geo_data .country - .as_ref() - .map(|c| c.as_str()) + .as_deref() .unwrap_or_default() .into(), - _ => expr::Variable::default(), + _ => Variable::default(), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } diff --git a/crates/smtp/src/inbound/spam.rs b/crates/smtp/src/inbound/spam.rs index 94e85e5e..72d472a6 100644 --- a/crates/smtp/src/inbound/spam.rs +++ b/crates/smtp/src/inbound/spam.rs @@ -9,7 +9,7 @@ use common::{config::mailstore::spamfilter::SpamFilterAction, network::SessionSt use mail_auth::{ArcOutput, DkimOutput, DmarcResult, dkim2::Dkim2Output, dmarc::Policy}; use mail_parser::Message; use spam_filter::{ - SpamFilterInput, + MessageTexts, SpamFilterInput, analysis::{ init::SpamFilterInit, score::{SpamFilterAnalyzeScore, SpamFilterScore}, @@ -27,8 +27,10 @@ impl Session { dmarc_policy: Option<&'x Policy>, ) -> SpamFilterAction { let server = &self.server; + let texts = MessageTexts::new(message, server.core.spam.text_limits); let mut ctx = server.spam_filter_init(self.build_spam_input( message, + &texts, dkim_result, dkim2_result, arc_result, @@ -45,9 +47,11 @@ impl Session { } } + #[allow(clippy::too_many_arguments)] pub fn build_spam_input<'x>( &'x self, message: &'x Message<'x>, + texts: &'x MessageTexts<'x>, dkim_result: &'x [DkimOutput<'x>], dkim2_result: Option<&'x Dkim2Output<'x>>, arc_result: Option<&'x ArcOutput>, @@ -56,6 +60,7 @@ impl Session { ) -> SpamFilterInput<'x> { SpamFilterInput { message, + texts, span_id: self.data.session_id, arc_result, spf_ehlo_result: self.data.spf_ehlo.as_ref(), @@ -69,7 +74,7 @@ impl Session { ehlo_domain: self.data.helo_domain.as_str().into(), authenticated_as: self.data.authenticated_as.as_ref().map(|a| a.name()), asn: self.data.asn_geo_data.asn.as_ref().map(|a| a.id), - country: self.data.asn_geo_data.country.as_ref().map(|c| c.as_str()), + country: self.data.asn_geo_data.country.as_deref(), is_tls: self.stream.is_tls(), env_from: self .data diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index 78492244..d68e9239 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -17,6 +17,9 @@ use std::time::Instant; use tokio_rustls::server::TlsStream; use trc::{SecurityEvent, SmtpEvent}; +const READ_BUFFER: usize = 8192; +const DATA_READ_BUFFER: usize = 65536; + impl SessionManager for SmtpSessionManager { async fn handle(self, session: network::SessionData) { // Build server and create session @@ -76,21 +79,16 @@ impl Session { let config = &self.server.core.smtp.session.connect; // Sieve filtering - if let Some((script, script_id)) = self - .server - .eval_if::(&config.script, self, self.data.session_id) - .await - .and_then(|name| { - self.server - .get_trusted_sieve_script(&name, self.data.session_id) - .map(|s| (s, name)) - }) + if let Some((script, script_id)) = + self.eval_if::(&config.script) + .await + .and_then(|name| { + self.server + .get_trusted_sieve_script(&name, self.data.session_id) + .map(|s| (s, name)) + }) && let ScriptResult::Reject(message) = self - .run_script( - script_id, - script.clone(), - self.build_script_parameters("connect"), - ) + .run_script(script_id, script, self.build_script_parameters("connect")) .await { let _ = self.write(message.as_bytes()).await; @@ -111,8 +109,7 @@ impl Session { // Obtain hostname self.hostname = self - .server - .eval_if::(&config.hostname, self, self.data.session_id) + .eval_if::(&config.hostname) .await .unwrap_or_default(); if self.hostname.is_empty() { @@ -125,8 +122,7 @@ impl Session { // Obtain greeting let greeting = self - .server - .eval_if::(&config.greeting, self, self.data.session_id) + .eval_if::(&config.greeting) .await .filter(|g| !g.is_empty()) .map(|g| format!("220 {}\r\n", g)) @@ -140,7 +136,7 @@ impl Session { } pub async fn handle_conn(&mut self) -> bool { - let mut buf = vec![0; 8192]; + let mut buf = Vec::with_capacity(READ_BUFFER); let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { @@ -153,8 +149,21 @@ impl Session { if bytes_read > 0 { if Instant::now() < self.data.valid_until && bytes_read <= self.data.bytes_left { self.data.bytes_left -= bytes_read; - match Box::pin(self.ingest(&buf[..bytes_read])).await { - Ok(true) => (), + match self.ingest(&buf).await { + Ok(true) => match &self.state { + State::Data(_) | State::Bdat { .. } + if buf.capacity() < DATA_READ_BUFFER => + { + buf = Vec::with_capacity(DATA_READ_BUFFER); + } + State::Request(_) + if buf.capacity() > READ_BUFFER + && self.data.message.is_empty() => + { + buf = Vec::with_capacity(READ_BUFFER); + } + _ => {} + }, Ok(false) => { return true; } diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs index 0a4e3a36..5eb9e6b9 100644 --- a/crates/smtp/src/inbound/vrfy.rs +++ b/crates/smtp/src/inbound/vrfy.rs @@ -6,6 +6,7 @@ use crate::core::Session; use common::network::{RcptResolution, SessionStream}; +use compact_str::CompactString; use std::{borrow::Cow, fmt::Write}; use trc::SmtpEvent; use utils::DomainPart; @@ -26,7 +27,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::Vrfy), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); self.write(format!("250 {}\r\n", address.as_ref()).as_bytes()) @@ -40,10 +41,10 @@ impl Session { trc::event!( Smtp(SmtpEvent::VrfyNotFound), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); - self.write(b"550 5.1.2 Address not found.\r\n").await + self.write(b"550 5.1.1 Address not found.\r\n").await } Err(err) => { trc::error!( @@ -60,7 +61,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::VrfyDisabled), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); self.write(b"252 2.5.1 VRFY is disabled.\r\n").await @@ -92,7 +93,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::Expn), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); self.write(result.as_bytes()).await @@ -101,10 +102,10 @@ impl Session { trc::event!( Smtp(SmtpEvent::ExpnNotFound), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); - self.write(b"550 5.1.2 Mailing list not found.\r\n").await + self.write(b"550 5.1.1 Mailing list not found.\r\n").await } Err(err) => { trc::error!( @@ -121,7 +122,7 @@ impl Session { trc::event!( Smtp(SmtpEvent::ExpnDisabled), SpanId = self.data.session_id, - To = address.as_ref().to_string(), + To = CompactString::from(address.as_ref()), ); self.write(b"252 2.5.1 EXPN is disabled.\r\n").await diff --git a/crates/smtp/src/outbound/client.rs b/crates/smtp/src/outbound/client.rs index eb0c61e4..61d01143 100644 --- a/crates/smtp/src/outbound/client.rs +++ b/crates/smtp/src/outbound/client.rs @@ -9,8 +9,9 @@ use crate::{ outbound::error::{AssertReply, ClientError, ClientResult}, queue::{Error, ErrorDetails, HostResponse, MessageWrapper, Status}, }; -use base64::{Engine, engine::general_purpose}; use directory::Credentials; +use encodify::base64::STANDARD; +use memchr::memchr_iter; use rustls::ClientConnection; use rustls_pki_types::ServerName; use smtp_proto::{ @@ -21,6 +22,7 @@ use smtp_proto::{ }, }; use std::{ + borrow::Cow, net::{IpAddr, SocketAddr}, time::Duration, }; @@ -31,10 +33,18 @@ use tokio::{ use tokio_rustls::{TlsConnector, client::TlsStream}; use trc::DeliveryEvent; +const READ_BUFFER_SIZE: usize = 8192; +const WRITE_BUFFER_SIZE: usize = 65536; +const WRITE_PREFIX_SIZE: usize = 16384; + pub struct SmtpClient { pub stream: T, pub timeout: Duration, pub session_id: u64, + buf: Box<[u8]>, + buf_pos: usize, + buf_len: usize, + out_buf: Vec, } impl SmtpClient { @@ -84,15 +94,10 @@ impl SmtpClient { credentials: &Credentials, ) -> ClientResult<()> { let mut reply = if (mechanism & (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER)) != 0 { - self.cmd( - format!( - "AUTH {} {}\r\n", - mechanism.to_mechanism(), - encode_credentials(credentials, mechanism, "")?, - ) - .as_bytes(), - ) - .await? + let mut command = format!("AUTH {} ", mechanism.to_mechanism()); + encode_credentials(credentials, mechanism, "", &mut command)?; + command.push_str("\r\n"); + self.cmd(command.as_bytes()).await? } else { self.cmd(format!("AUTH {}\r\n", mechanism.to_mechanism()).as_bytes()) .await? @@ -101,15 +106,10 @@ impl SmtpClient { for _ in 0..3 { match reply.code() { 334 => { - reply = self - .cmd( - format!( - "{}\r\n", - encode_credentials(credentials, mechanism, reply.message())? - ) - .as_bytes(), - ) - .await?; + let mut command = String::new(); + encode_credentials(credentials, mechanism, reply.message(), &mut command)?; + command.push_str("\r\n"); + reply = self.cmd(command.as_bytes()).await?; } 235 => { return Ok(()); @@ -160,12 +160,43 @@ impl SmtpClient { pub async fn write_chunks(&mut self, chunks: &[&[u8]]) -> Result<(), ClientError> { for chunk in chunks { - self.stream - .write_all(chunk) + self.write_buffered(chunk) .await .map_err(ClientError::from)?; } - self.stream.flush().await.map_err(ClientError::from) + self.flush_buffered().await.map_err(ClientError::from) + } + + async fn write_buffered(&mut self, bytes: &[u8]) -> tokio::io::Result<()> { + if bytes.len() >= WRITE_BUFFER_SIZE { + if !self.out_buf.is_empty() { + let prefix = WRITE_PREFIX_SIZE.saturating_sub(self.out_buf.len()); + let (head, rest) = bytes.split_at_checked(prefix).unwrap_or((bytes, &[])); + self.out_buf.extend_from_slice(head); + self.stream.write_all(&self.out_buf).await?; + self.out_buf.clear(); + return self.stream.write_all(rest).await; + } + return self.stream.write_all(bytes).await; + } + let room = WRITE_BUFFER_SIZE.saturating_sub(self.out_buf.len()); + let Some((head, rest)) = bytes.split_at_checked(room) else { + self.out_buf.extend_from_slice(bytes); + return Ok(()); + }; + self.out_buf.extend_from_slice(head); + self.stream.write_all(&self.out_buf).await?; + self.out_buf.clear(); + self.out_buf.extend_from_slice(rest); + Ok(()) + } + + async fn flush_buffered(&mut self) -> tokio::io::Result<()> { + if !self.out_buf.is_empty() { + self.stream.write_all(&self.out_buf).await?; + self.out_buf.clear(); + } + self.stream.flush().await } pub async fn send_message( @@ -175,90 +206,90 @@ impl SmtpClient { bdat_cmd: &mut Option, params: &SessionParams<'_>, ) -> Result<(), Status>, ErrorDetails>> { - match params - .server - .blob_store() - .get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX) - .await - { - Ok(Some(raw_message)) => { - tokio::time::timeout(params.conn_strategy.timeout_data, async { - if let Some(bdat_cmd) = bdat_cmd { - *bdat_cmd = format!( - "BDAT {} LAST\r\n", - raw_message.len() + rcpt_headers.map(|h| h.len()).unwrap_or(0) - ); - + let raw_message = params + .message_blob + .get_or_try_init(|| async { + match params + .server + .blob_store() + .get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX) + .await + { + Ok(Some(raw_message)) => Ok(raw_message), + Ok(None) => { trc::event!( - Delivery(DeliveryEvent::RawOutput), - SpanId = self.session_id, - Contents = bdat_cmd.clone(), - Size = bdat_cmd.len() + Queue(trc::QueueEvent::BlobNotFound), + SpanId = message.span_id, + BlobId = message.message.blob_hash.to_hex(), + CausedBy = trc::location!() ); - - let chunks = if let Some(rcpt_headers) = rcpt_headers { - &[bdat_cmd.as_bytes(), rcpt_headers, &raw_message][..] - } else { - &[bdat_cmd.as_bytes(), &raw_message][..] - }; - - self.write_chunks(chunks).await - } else { - trc::event!( - Delivery(DeliveryEvent::RawOutput), - SpanId = self.session_id, - Contents = "DATA\r\n", - Size = 6 - ); - - self.write_chunks(&[b"DATA\r\n"]).await?; - self.read().await?.assert_code(354)?; - if let Some(rcpt_headers) = rcpt_headers - && let Err(err) = self.write_chunks(&[rcpt_headers]).await - { - Err(err) - } else { - self.write_message(&raw_message) - .await - .map_err(ClientError::from) - } + Err(Status::TemporaryFailure(Box::new(ErrorDetails { + entity: "localhost".into(), + details: Error::Io("Queue system error.".into()), + }))) } - }) - .await - .map_err(|_| Status::timeout(params.hostname, "sending message"))? - .map_err(|err| { - Status::from_smtp_error( - params.hostname, - bdat_cmd.as_deref().unwrap_or("DATA"), - err, - ) - }) - } - Ok(None) => { - trc::event!( - Queue(trc::QueueEvent::BlobNotFound), - SpanId = message.span_id, - BlobId = message.message.blob_hash.to_hex(), - CausedBy = trc::location!() - ); - Err(Status::TemporaryFailure(ErrorDetails { - entity: "localhost".into(), - details: Error::Io("Queue system error.".into()), - })) - } - Err(err) => { - trc::error!( - err.span_id(message.span_id) - .details("Failed to fetch blobId") - .caused_by(trc::location!()) + Err(err) => { + trc::error!( + err.span_id(message.span_id) + .details("Failed to fetch blobId") + .caused_by(trc::location!()) + ); + + Err(Status::TemporaryFailure(Box::new(ErrorDetails { + entity: "localhost".into(), + details: Error::Io("Queue system error.".into()), + }))) + } + } + }) + .await?; + + tokio::time::timeout(params.conn_strategy.timeout_data, async { + if let Some(bdat_cmd) = bdat_cmd { + *bdat_cmd = format!( + "BDAT {} LAST\r\n", + raw_message.len() + rcpt_headers.map(|h| h.len()).unwrap_or(0) ); - Err(Status::TemporaryFailure(ErrorDetails { - entity: "localhost".into(), - details: Error::Io("Queue system error.".into()), - })) + trc::event!( + Delivery(DeliveryEvent::RawOutput), + SpanId = self.session_id, + Contents = bdat_cmd.clone(), + Size = bdat_cmd.len() + ); + + let chunks = if let Some(rcpt_headers) = rcpt_headers { + &[bdat_cmd.as_bytes(), rcpt_headers, raw_message][..] + } else { + &[bdat_cmd.as_bytes(), raw_message][..] + }; + + self.write_chunks(chunks).await + } else { + trc::event!( + Delivery(DeliveryEvent::RawOutput), + SpanId = self.session_id, + Contents = "DATA\r\n", + Size = 6 + ); + + self.write_chunks(&[b"DATA\r\n"]).await?; + self.read().await?.assert_code(354)?; + if let Some(rcpt_headers) = rcpt_headers { + self.write_buffered(rcpt_headers) + .await + .map_err(ClientError::from)?; + } + self.write_message(raw_message) + .await + .map_err(ClientError::from) } - } + }) + .await + .map_err(|_| Status::timeout(params.hostname, "sending message"))? + .map_err(|err| { + Status::from_smtp_error(params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) + }) } pub async fn say_helo( @@ -307,11 +338,12 @@ impl SmtpClient { } pub async fn read_ehlo(&mut self) -> ClientResult> { - let mut buf = vec![0u8; 8192]; + self.discard_buffered(); + let buf = &mut self.buf; let mut buf_concat = Vec::with_capacity(0); loop { - let br = self.stream.read(&mut buf).await?; + let br = self.stream.read(buf).await?; if br == 0 { return Err(ClientError::UnparseableReply); @@ -363,81 +395,62 @@ impl SmtpClient { } pub async fn read(&mut self) -> ClientResult> { - let mut buf = vec![0u8; 8192]; + self.discard_buffered(); + self.read_buffered().await + } + + pub async fn read_many(&mut self, num: usize) -> ClientResult>>> { + self.discard_buffered(); + let mut response = Vec::with_capacity(num); + while response.len() < num { + response.push(self.read_buffered().await?.into_box()); + } + Ok(response) + } + + async fn read_buffered(&mut self) -> ClientResult> { let mut parser = ResponseReceiver::default(); loop { - let br = self.stream.read(&mut buf).await?; + if self.buf_pos >= self.buf_len { + let br = self.stream.read(&mut self.buf).await?; + if br == 0 { + return Err(ClientError::UnparseableReply); + } - if br > 0 { trc::event!( Delivery(DeliveryEvent::RawInput), SpanId = self.session_id, - Contents = trc::Value::from_maybe_string(&buf[..br]), + Contents = + trc::Value::from_maybe_string(self.buf.get(..br).unwrap_or_default()), Size = br ); - match parser.parse(&mut buf[..br].iter()) { - Ok(reply) => return Ok(reply), - Err(err) => match err { - smtp_proto::Error::NeedsMoreData { .. } => (), - _ => { - return Err(ClientError::UnparseableReply); - } - }, - } - } else { - return Err(ClientError::UnparseableReply); + self.buf_pos = 0; + self.buf_len = br; + } + + let pending = self.buf.get(self.buf_pos..self.buf_len).unwrap_or_default(); + let mut iter = pending.iter(); + let result = parser.parse(&mut iter); + self.buf_pos = self.buf_len - iter.as_slice().len(); + + match result { + Ok(reply) => return Ok(reply), + Err(smtp_proto::Error::NeedsMoreData { .. }) => (), + Err(_) => return Err(ClientError::UnparseableReply), } } } - pub async fn read_many(&mut self, num: usize) -> ClientResult>>> { - let mut buf = vec![0u8; 1024]; - let mut response = Vec::with_capacity(num); - let mut parser = ResponseReceiver::default(); - - 'outer: loop { - let br = self.stream.read(&mut buf).await?; - - if br > 0 { - let mut iter = buf[..br].iter(); - - trc::event!( - Delivery(DeliveryEvent::RawInput), - SpanId = self.session_id, - Contents = trc::Value::from_maybe_string(&buf[..br]), - Size = br - ); - - loop { - match parser.parse(&mut iter) { - Ok(reply) => { - response.push(reply.into_box()); - if response.len() != num { - parser.reset(); - } else { - break 'outer; - } - } - Err(err) => match err { - smtp_proto::Error::NeedsMoreData { .. } => break, - _ => { - return Err(ClientError::UnparseableReply); - } - }, - } - } - } else { - return Err(ClientError::UnparseableReply); - } - } - - Ok(response) + fn discard_buffered(&mut self) { + self.buf_pos = 0; + self.buf_len = 0; } /// Sends a command to the SMTP server and waits for a reply. pub async fn cmd(&mut self, cmd: impl AsRef<[u8]>) -> ClientResult> { + self.discard_buffered(); tokio::time::timeout(self.timeout, async { let cmd = cmd.as_ref(); @@ -450,16 +463,20 @@ impl SmtpClient { self.stream.write_all(cmd).await?; self.stream.flush().await?; - self.read().await + self.read_buffered().await }) .await .map_err(|_| ClientError::Timeout)? } + pub async fn read_pipelined(&mut self) -> ClientResult> { + tokio::time::timeout(self.timeout, self.read_buffered()) + .await + .map_err(|_| ClientError::Timeout)? + } + pub async fn write_message(&mut self, message: &[u8]) -> tokio::io::Result<()> { // Transparency procedure - let mut is_cr_or_lf = false; - // As per RFC 5322bis, section 2.3: // CR and LF MUST only occur together as CRLF; they MUST NOT appear // independently in the body. @@ -474,23 +491,22 @@ impl SmtpClient { ); let mut last_pos = 0; - for (pos, byte) in message.iter().enumerate() { - if *byte == b'.' && is_cr_or_lf { - if let Some(bytes) = message.get(last_pos..pos) { - self.stream.write_all(bytes).await?; - self.stream.write_all(b".").await?; - last_pos = pos; - } - is_cr_or_lf = false; - } else { - is_cr_or_lf = *byte == b'\n' || *byte == b'\r'; + for pos in memchr_iter(b'.', message) { + if matches!( + message.get(..pos).and_then(<[u8]>::last), + Some(b'\n' | b'\r') + ) && let Some(bytes) = message.get(last_pos..pos) + { + self.write_buffered(bytes).await?; + self.write_buffered(b".").await?; + last_pos = pos; } } if let Some(bytes) = message.get(last_pos..) { - self.stream.write_all(bytes).await?; + self.write_buffered(bytes).await?; } - self.stream.write_all("\r\n.\r\n".as_bytes()).await?; - self.stream.flush().await + self.write_buffered(b"\r\n.\r\n").await?; + self.flush_buffered().await } } @@ -537,6 +553,10 @@ impl SmtpClient { })?, timeout: self.timeout, session_id: self.session_id, + buf: self.buf, + buf_pos: 0, + buf_len: 0, + out_buf: Vec::new(), }) }) .await @@ -552,10 +572,16 @@ impl SmtpClient { session_id: u64, ) -> ClientResult { tokio::time::timeout(timeout, async { + let stream = TcpStream::connect(remote_addr).await?; + let _ = stream.set_nodelay(true); Ok(SmtpClient { - stream: TcpStream::connect(remote_addr).await?, + stream, timeout, session_id, + buf: vec![0u8; READ_BUFFER_SIZE].into_boxed_slice(), + buf_pos: 0, + buf_len: 0, + out_buf: Vec::new(), }) }) .await @@ -576,11 +602,17 @@ impl SmtpClient { TcpSocket::new_v6()? }; socket.bind(SocketAddr::new(local_ip, 0))?; + let stream = socket.connect(remote_addr).await?; + let _ = stream.set_nodelay(true); Ok(SmtpClient { - stream: socket.connect(remote_addr).await?, + stream, timeout, session_id, + buf: vec![0u8; READ_BUFFER_SIZE].into_boxed_slice(), + buf_pos: 0, + buf_len: 0, + out_buf: Vec::new(), }) }) .await @@ -623,53 +655,51 @@ fn encode_credentials( credentials: &Credentials, mechanism: u64, challenge: &str, -) -> ClientResult { - Ok(general_purpose::STANDARD.encode( - match (mechanism, credentials) { - ( - AUTH_PLAIN, - Credentials::Basic { - username, secret, .. - }, - ) => { - format!("\u{0}{}\u{0}{}", username, secret) - } - ( - AUTH_LOGIN, - Credentials::Basic { - username, secret, .. - }, - ) => { - let challenge = general_purpose::STANDARD.decode(challenge)?; + out: &mut String, +) -> ClientResult<()> { + let response: Cow<'_, str> = match (mechanism, credentials) { + ( + AUTH_PLAIN, + Credentials::Basic { + username, secret, .. + }, + ) => format!("\u{0}{}\u{0}{}", username, secret).into(), + ( + AUTH_LOGIN, + Credentials::Basic { + username, secret, .. + }, + ) => { + let challenge = STANDARD.decode(challenge)?; - if b"user name" - .eq_ignore_ascii_case(challenge.get(0..9).ok_or(ClientError::InvalidChallenge)?) - || b"username".eq_ignore_ascii_case( - // Because Google makes its own standards - challenge.get(0..8).ok_or(ClientError::InvalidChallenge)?, - ) - { - &username - } else if b"password" - .eq_ignore_ascii_case(challenge.get(0..8).ok_or(ClientError::InvalidChallenge)?) - { - &secret - } else { - return Err(ClientError::InvalidChallenge); - } - .to_string() + if b"user name" + .eq_ignore_ascii_case(challenge.get(0..9).ok_or(ClientError::InvalidChallenge)?) + || b"username".eq_ignore_ascii_case( + // Because Google makes its own standards + challenge.get(0..8).ok_or(ClientError::InvalidChallenge)?, + ) + { + username.as_str().into() + } else if b"password" + .eq_ignore_ascii_case(challenge.get(0..8).ok_or(ClientError::InvalidChallenge)?) + { + secret.as_str().into() + } else { + return Err(ClientError::InvalidChallenge); } - - (AUTH_XOAUTH2, Credentials::Bearer { token, username }) => format!( - "user={}\x01auth=Bearer {}\x01\x01", - username.as_deref().unwrap_or_default(), - token - ), - (AUTH_OAUTHBEARER, Credentials::Bearer { token, .. }) => token.to_string(), - _ => return Err(ClientError::UnsupportedAuthMechanism), } - .as_bytes(), - )) + + (AUTH_XOAUTH2, Credentials::Bearer { token, username }) => format!( + "user={}\x01auth=Bearer {}\x01\x01", + username.as_deref().unwrap_or_default(), + token + ) + .into(), + (AUTH_OAUTHBEARER, Credentials::Bearer { token, .. }) => token.as_str().into(), + _ => return Err(ClientError::UnsupportedAuthMechanism), + }; + STANDARD.encode_append(response.as_bytes(), out); + Ok(()) } impl SmtpClient> { @@ -754,7 +784,7 @@ pub(crate) fn from_error_details(err: &Error) -> trc::Error { Error::UnexpectedResponse(reply) => event .details("Unexpected SMTP Response") .ctx(trc::Key::Code, reply.response.code) - .ctx(trc::Key::Details, reply.command.clone()) + .ctx(trc::Key::Command, reply.command.clone()) .ctx(trc::Key::Reason, reply.response.message.clone()), Error::ConnectionError(err) => event .details("Connection Error") diff --git a/crates/smtp/src/outbound/dane/dnssec.rs b/crates/smtp/src/outbound/dane/dnssec.rs index d2010d8f..ae9d3076 100644 --- a/crates/smtp/src/outbound/dane/dnssec.rs +++ b/crates/smtp/src/outbound/dane/dnssec.rs @@ -10,8 +10,8 @@ use common::{ }; pub use mail_auth::DnssecStatus; use mail_auth::{ - MX, RecordSet, - common::resolver::ToFqdn, + Mx, Negative, RecordSet, + dns::ToFqdn, hickory_resolver::{ net::{DnsError, NetError}, proto::{ @@ -35,7 +35,7 @@ pub trait TlsaLookup: Sync + Send { fn mx_lookup( &self, key: impl ToFqdn + Sync + Send, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; fn tlsa_lookup( &self, @@ -60,7 +60,7 @@ pub enum TlsaResult { } impl TlsaLookup for Server { - async fn mx_lookup(&self, key: impl ToFqdn + Sync + Send) -> mail_auth::Result> { + async fn mx_lookup(&self, key: impl ToFqdn + Sync + Send) -> mail_auth::Result> { if !self.core.smtp.resolvers.dnssec_available { return self .core @@ -71,16 +71,14 @@ impl TlsaLookup for Server { .await; } - let key = key.to_fqdn().into_owned().into_boxed_str(); - if let Some(value) = self.inner.cache.dns_mx.get::(key.as_ref()) - && value.dnssec_status != DnssecStatus::Indeterminate - { + let key = key.to_fqdn(); + if let Some(value) = validated_entry(self.inner.cache.dns_mx.get::(key.as_ref())) { return Ok(value); } - #[cfg(any(test, feature = "test_mode"))] + #[cfg(any(test, stalwart_tests))] if true { - return mail_auth::common::resolver::mock_resolve(key.as_ref()); + return mail_auth::dns::mock_resolve(key.as_ref()); } let mx_lookup = match self @@ -98,13 +96,13 @@ impl TlsaLookup for Server { && denial.response_code == ResponseCode::NoError { let records = RecordSet { - rrset: Arc::new([]), + records: Arc::new([]), dnssec_status: denial.dnssec_status, }; if let Some(valid_until) = denial.valid_until { self.inner.cache.dns_mx.insert_with_expiry( - key, - records.clone(), + key.into_owned().into_boxed_str(), + Ok(records.clone()), valid_until, ); } @@ -135,38 +133,42 @@ impl TlsaLookup for Server { } records.sort_unstable_by_key(|a| a.0); - let rrset: Arc<[MX]> = records + let rrset: Arc<[Mx]> = records .into_iter() - .map(|(preference, exchanges)| MX { + .map(|(preference, exchanges)| Mx { preference, exchanges: exchanges.into_boxed_slice(), }) - .collect::>(); + .collect::>(); let records = RecordSet { - rrset, + records: rrset, dnssec_status: dnssec_status.unwrap_or(DnssecStatus::Indeterminate), }; - self.inner - .cache - .dns_mx - .insert_with_expiry(key, records.clone(), mx_lookup.valid_until()); + self.inner.cache.dns_mx.insert_with_expiry( + key.into_owned().into_boxed_str(), + Ok(records.clone()), + mx_lookup.valid_until(), + ); Ok(records) } async fn tlsa_lookup(&self, key: impl ToFqdn + Sync + Send) -> mail_auth::Result { - let key = key.to_fqdn().into_owned().into_boxed_str(); + let key = key.to_fqdn(); if let Some(value) = self.inner.cache.dns_tlsa.get(key.as_ref()) { - return Ok(TlsaResult::Secure(value)); + return Ok(match value { + Some(tlsa) => TlsaResult::Secure(tlsa), + None => TlsaResult::Missing, + }); } - #[cfg(any(test, feature = "test_mode"))] + #[cfg(any(test, stalwart_tests))] if true { if key.as_ref().contains("_dnssec_bogus.") { return Ok(TlsaResult::Bogus); } - return mail_auth::common::resolver::mock_resolve(key.as_ref()); + return mail_auth::dns::mock_resolve(key.as_ref()); } let tlsa_lookup = match self @@ -181,11 +183,11 @@ impl TlsaLookup for Server { Ok(tlsa_lookup) => tlsa_lookup, Err(err) => { if let Some(denial) = NegativeAnswer::from_error(&err) { - return Ok(if denial.dnssec_status == DnssecStatus::Bogus { - TlsaResult::Bogus - } else { - TlsaResult::Missing - }); + if denial.dnssec_status == DnssecStatus::Bogus { + return Ok(TlsaResult::Bogus); + } + self.cache_missing_tlsa(key.into_owned().into_boxed_str(), denial.valid_until); + return Ok(TlsaResult::Missing); } return Err(err.into()); } @@ -247,14 +249,20 @@ impl TlsaLookup for Server { }); self.inner.cache.dns_tlsa.insert_with_expiry( - key, - tlsa.clone(), + key.into_owned().into_boxed_str(), + Some(tlsa.clone()), tlsa_lookup.valid_until(), ); Ok(TlsaResult::Secure(tlsa)) } - _ => Ok(TlsaResult::Missing), + _ => { + self.cache_missing_tlsa( + key.into_owned().into_boxed_str(), + Some(tlsa_lookup.valid_until()), + ); + Ok(TlsaResult::Missing) + } } } @@ -272,16 +280,14 @@ impl TlsaLookup for Server { .await; } - let key = key.to_fqdn().into_owned().into_boxed_str(); - if let Some(value) = self.inner.cache.dns_ipv4.get::(key.as_ref()) - && value.dnssec_status != DnssecStatus::Indeterminate - { + let key = key.to_fqdn(); + if let Some(value) = validated_entry(self.inner.cache.dns_ipv4.get::(key.as_ref())) { return Ok(value); } - #[cfg(any(test, feature = "test_mode"))] + #[cfg(any(test, stalwart_tests))] if true { - return mail_auth::common::resolver::mock_resolve(key.as_ref()); + return mail_auth::dns::mock_resolve(key.as_ref()); } let name = Name::from_str_relaxed::<&str>(key.as_ref())?; @@ -300,13 +306,13 @@ impl TlsaLookup for Server { && denial.response_code == ResponseCode::NoError { let records = RecordSet { - rrset: Arc::new([]), + records: Arc::new([]), dnssec_status: denial.dnssec_status, }; if let Some(valid_until) = denial.valid_until { self.inner.cache.dns_ipv4.insert_with_expiry( - key, - records.clone(), + key.into_owned().into_boxed_str(), + Ok(records.clone()), valid_until, ); } @@ -318,7 +324,7 @@ impl TlsaLookup for Server { let answers = lookup.answers(); let records = RecordSet { - rrset: answers + records: answers .iter() .filter_map(|record| match &record.data { RData::A(addr) => Some(addr.0), @@ -328,10 +334,11 @@ impl TlsaLookup for Server { dnssec_status: tlsa_base_status(&name, answers, RecordType::A), }; - self.inner - .cache - .dns_ipv4 - .insert_with_expiry(key, records.clone(), lookup.valid_until()); + self.inner.cache.dns_ipv4.insert_with_expiry( + key.into_owned().into_boxed_str(), + Ok(records.clone()), + lookup.valid_until(), + ); Ok(records) } @@ -350,16 +357,14 @@ impl TlsaLookup for Server { .await; } - let key = key.to_fqdn().into_owned().into_boxed_str(); - if let Some(value) = self.inner.cache.dns_ipv6.get::(key.as_ref()) - && value.dnssec_status != DnssecStatus::Indeterminate - { + let key = key.to_fqdn(); + if let Some(value) = validated_entry(self.inner.cache.dns_ipv6.get::(key.as_ref())) { return Ok(value); } - #[cfg(any(test, feature = "test_mode"))] + #[cfg(any(test, stalwart_tests))] if true { - return mail_auth::common::resolver::mock_resolve(key.as_ref()); + return mail_auth::dns::mock_resolve(key.as_ref()); } let name = Name::from_str_relaxed::<&str>(key.as_ref())?; @@ -378,13 +383,13 @@ impl TlsaLookup for Server { && denial.response_code == ResponseCode::NoError { let records = RecordSet { - rrset: Arc::new([]), + records: Arc::new([]), dnssec_status: denial.dnssec_status, }; if let Some(valid_until) = denial.valid_until { self.inner.cache.dns_ipv6.insert_with_expiry( - key, - records.clone(), + key.into_owned().into_boxed_str(), + Ok(records.clone()), valid_until, ); } @@ -396,7 +401,7 @@ impl TlsaLookup for Server { let answers = lookup.answers(); let records = RecordSet { - rrset: answers + records: answers .iter() .filter_map(|record| match &record.data { RData::AAAA(addr) => Some(addr.0), @@ -406,15 +411,30 @@ impl TlsaLookup for Server { dnssec_status: tlsa_base_status(&name, answers, RecordType::AAAA), }; - self.inner - .cache - .dns_ipv6 - .insert_with_expiry(key, records.clone(), lookup.valid_until()); + self.inner.cache.dns_ipv6.insert_with_expiry( + key.into_owned().into_boxed_str(), + Ok(records.clone()), + lookup.valid_until(), + ); Ok(records) } } +trait TlsaNegativeCache { + fn cache_missing_tlsa(&self, key: Box, valid_until: Option); +} + +impl TlsaNegativeCache for Server { + fn cache_missing_tlsa(&self, key: Box, valid_until: Option) { + self.inner.cache.dns_tlsa.insert_with_expiry( + key, + None, + valid_until.unwrap_or_else(|| Instant::now() + self.inner.cache.negative_cache_ttl), + ); + } +} + struct NegativeAnswer { response_code: ResponseCode, dnssec_status: DnssecStatus, @@ -451,6 +471,13 @@ impl NegativeAnswer { } } +fn validated_entry(entry: Option, Negative>>) -> Option> { + match entry { + Some(Ok(records)) if records.dnssec_status != DnssecStatus::Indeterminate => Some(records), + _ => None, + } +} + fn denial_dnssec_status(authorities: &[Record]) -> DnssecStatus { authorities .iter() @@ -508,120 +535,3 @@ pub(crate) fn least_secure(a: DnssecStatus, b: DnssecStatus) -> DnssecStatus { if rank(a) <= rank(b) { a } else { b } } -#[cfg(test)] -mod tests { - use super::*; - use mail_auth::hickory_resolver::proto::rr::rdata::{A, CNAME}; - use std::net::Ipv4Addr; - - fn name(value: &str) -> Name { - Name::from_ascii(value).unwrap() - } - - fn address(owner: &str, proof: Proof) -> Record { - let mut record = - Record::from_rdata(name(owner), 3600, RData::A(A(Ipv4Addr::new(192, 0, 2, 1)))); - record.proof = proof; - record - } - - fn alias(owner: &str, target: &str, proof: Proof) -> Record { - let mut record = Record::from_rdata(name(owner), 3600, RData::CNAME(CNAME(name(target)))); - record.proof = proof; - record - } - - #[test] - fn tlsa_base_status_follows_address_records() { - let query = name("mx.example.org."); - - for (proof, expected) in [ - (Proof::Secure, DnssecStatus::Secure), - (Proof::Insecure, DnssecStatus::Insecure), - (Proof::Bogus, DnssecStatus::Bogus), - (Proof::Indeterminate, DnssecStatus::Indeterminate), - ] { - assert_eq!( - tlsa_base_status(&query, &[address("mx.example.org.", proof)], RecordType::A), - expected, - "proof {proof}" - ); - } - } - - #[test] - fn tlsa_base_status_is_indeterminate_without_addresses() { - assert_eq!( - tlsa_base_status(&name("mx.example.org."), &[], RecordType::A), - DnssecStatus::Indeterminate - ); - } - - #[test] - fn tlsa_base_status_takes_least_secure_address() { - let query = name("mx.example.org."); - - assert_eq!( - tlsa_base_status( - &query, - &[ - address("mx.example.org.", Proof::Secure), - address("mx.example.org.", Proof::Insecure), - ], - RecordType::A - ), - DnssecStatus::Insecure - ); - } - - #[test] - fn tlsa_base_status_keeps_secure_alias_to_insecure_zone() { - let query = name("mx.example.org."); - - assert_eq!( - tlsa_base_status( - &query, - &[ - alias("mx.example.org.", "mx.provider.net.", Proof::Secure), - address("mx.provider.net.", Proof::Insecure), - ], - RecordType::A - ), - DnssecStatus::Secure - ); - } - - #[test] - fn tlsa_base_status_skips_insecure_alias() { - let query = name("mx.example.org."); - - assert_eq!( - tlsa_base_status( - &query, - &[ - alias("mx.example.org.", "mx.provider.net.", Proof::Insecure), - address("mx.provider.net.", Proof::Insecure), - ], - RecordType::A - ), - DnssecStatus::Insecure - ); - } - - #[test] - fn tlsa_base_status_ignores_alias_below_query_name() { - let query = name("mx.example.org."); - - assert_eq!( - tlsa_base_status( - &query, - &[ - alias("mx.provider.net.", "mx.other.net.", Proof::Secure), - address("mx.other.net.", Proof::Insecure), - ], - RecordType::A - ), - DnssecStatus::Insecure - ); - } -} diff --git a/crates/smtp/src/outbound/dane/verify.rs b/crates/smtp/src/outbound/dane/verify.rs index b464874f..fa875b8b 100644 --- a/crates/smtp/src/outbound/dane/verify.rs +++ b/crates/smtp/src/outbound/dane/verify.rs @@ -6,6 +6,7 @@ use crate::queue::{Error, ErrorDetails, HostResponse, Status}; use common::config::smtp::resolver::{Tlsa, TlsaEntry, TlsaMatching}; +use compact_str::{CompactString, ToCompactString}; use rustls_pki_types::{CertificateDer, Der, ServerName, TrustAnchor, UnixTime}; use sha2::{Digest, Sha256, Sha512}; use trc::DaneEvent; @@ -37,13 +38,13 @@ impl TlsaVerify for Tlsa { trc::event!( Dane(DaneEvent::NoCertificatesFound), SpanId = session_id, - Hostname = hostname.to_string(), + Hostname = CompactString::from(hostname), ); - return Err(Status::TemporaryFailure(ErrorDetails { + return Err(Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::DaneError("No certificates were provided by host".into()), - })); + }))); } }; @@ -55,14 +56,14 @@ impl TlsaVerify for Tlsa { trc::event!( Dane(DaneEvent::CertificateParseError), SpanId = session_id, - Hostname = hostname.to_string(), - Reason = err.to_string(), + Hostname = CompactString::from(hostname), + Reason = err.to_compact_string(), ); - return Err(Status::TemporaryFailure(ErrorDetails { + return Err(Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::DaneError("Failed to parse X.509 certificate".into()), - })); + }))); } } } @@ -80,7 +81,7 @@ impl TlsaVerify for Tlsa { trc::event!( Dane(DaneEvent::AuthenticationSuccess), SpanId = session_id, - Hostname = hostname.to_string(), + Hostname = CompactString::from(hostname), ); Ok(()) @@ -88,13 +89,13 @@ impl TlsaVerify for Tlsa { trc::event!( Dane(DaneEvent::AuthenticationFailure), SpanId = session_id, - Hostname = hostname.to_string(), + Hostname = CompactString::from(hostname), ); - Err(Status::TemporaryFailure(ErrorDetails { + Err(Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::DaneError("No matching certificates found in TLSA records".into()), - })) + }))) } } } @@ -112,7 +113,7 @@ fn verify_end_entity( trc::event!( Dane(DaneEvent::TlsaRecordMatch), SpanId = session_id, - Hostname = hostname.to_string(), + Hostname = CompactString::from(hostname), Type = "end-entity", ); return true; @@ -206,7 +207,7 @@ fn verify_trust_anchor( trc::event!( Dane(DaneEvent::TlsaRecordMatch), SpanId = session_id, - Hostname = hostname.to_string(), + Hostname = CompactString::from(hostname), Type = "trust-anchor", ); diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index d2b6b7ea..5722edfd 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -20,112 +20,132 @@ use crate::queue::spool::{DSN_RETRY, SmtpSpool}; use crate::queue::throttle::IsAllowed; use crate::queue::{ Error, FROM_REPORT, HostResponse, MessageWrapper, Metadata, QueueEnvelope, QueuedMessage, - Status, + Status, eval_strategy, }; use crate::reporting::send::MtaReportSend; use crate::{queue::ErrorDetails, reporting::tls::TlsRptOptions}; use ahash::AHashMap; -use common::Server; use common::config::smtp::queue::RoutingStrategy; +use common::config::smtp::resolver::MtaStsFailureKind; use common::config::{server::ServerProtocol, smtp::report::AggregateFrequency}; use common::ipc::{PolicyType, QueueEvent, QueueEventStatus, TlsEvent}; -use compact_str::ToCompactString; +use common::{Server, expr::Bump}; +use compact_str::{CompactString, ToCompactString, format_compact}; use mail_auth::RecordSet; -use mail_auth::{ - mta_sts::TlsRpt, - report::tlsrpt::{FailureDetails, ResultType}, -}; +use mail_auth::mta_sts::TlsRptRecord; use smtp_proto::MAIL_REQUIRETLS; +use smtp_report::tlsrpt::{FailureDetails, FailureType}; use std::sync::Arc; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, time::Instant, }; use store::write::{BatchBuilder, QueueClass, ValueClass, now}; +use tokio::sync::OnceCell; use trc::{DaneEvent, DeliveryEvent, MtaStsEvent, ServerEvent, TlsRptEvent}; impl QueuedMessage { pub fn try_deliver(self, server: Server) { + #[cfg(stalwart_tests)] + let probe_started = crate::queue::probe::DispatchProbe::dispatched(); + tokio::spawn(async move { // Lock queue event let queue_id = self.queue_id; let status = if server.try_lock_event(queue_id, self.queue_name).await { - if let Some(mut message) = server.read_message(queue_id, self.queue_name).await { - // Generate span id - message.span_id = server.inner.data.span_id_gen.generate(); - let span_id = message.span_id; + match server.read_message(queue_id, self.queue_name).await { + Ok(Some(mut message)) => { + // Generate span id + message.span_id = server.inner.data.span_id_gen.generate(); + let span_id = message.span_id; - trc::event!( - Delivery(DeliveryEvent::AttemptStart), - SpanId = message.span_id, - QueueId = message.queue_id, - QueueName = message.queue_name.to_string(), - From = if !message.message.return_path.is_empty() { - trc::Value::String(message.message.return_path.as_ref().into()) - } else { - trc::Value::String("<>".into()) - }, - To = message - .message - .recipients - .iter() - .filter_map(|r| { - if matches!( - r.status, - Status::Scheduled | Status::TemporaryFailure(_) - ) && r.queue == message.queue_name - { - Some(trc::Value::String(r.address().into())) - } else { - None - } - }) - .collect::>(), - Size = message.message.size, - Total = message.message.recipients.len(), - ); + trc::event!( + Delivery(DeliveryEvent::AttemptStart), + SpanId = message.span_id, + QueueId = message.queue_id, + QueueName = message.queue_name.to_compact_string(), + From = if !message.message.return_path.is_empty() { + trc::Value::String(message.message.return_path.as_ref().into()) + } else { + trc::Value::String("<>".into()) + }, + To = message + .message + .recipients + .iter() + .filter_map(|r| { + if matches!( + r.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) && r.queue == message.queue_name + { + Some(trc::Value::String(r.address().into())) + } else { + None + } + }) + .collect::>(), + Size = message.message.size, + Total = message.message.recipients.len(), + ); - // Attempt delivery - let start_time = Instant::now(); - let queue_event = Box::pin(self.deliver_task(server.clone(), message)).await; + // Attempt delivery + let start_time = Instant::now(); + let queue_event = self.deliver_task(server.clone(), message).await; - trc::event!( - Delivery(DeliveryEvent::AttemptEnd), - SpanId = span_id, - Elapsed = start_time.elapsed(), - ); + trc::event!( + Delivery(DeliveryEvent::AttemptEnd), + SpanId = span_id, + Elapsed = start_time.elapsed(), + ); - // Unlock event - server.unlock_event(queue_id, self.queue_name).await; + // Unlock event + server.unlock_event(queue_id, self.queue_name).await; - queue_event - } else { - // Message no longer exists, delete queue event. - let mut batch = BatchBuilder::new(); - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: self.due, - queue_id: self.queue_id, - queue_name: self.queue_name.into_inner(), - }, - ))); + queue_event + } + Ok(None) => { + // Message no longer exists, delete queue event. + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: self.due, + queue_id: self.queue_id, + queue_name: self.queue_name.into_inner(), + }, + ))); - if let Err(err) = server.store().write(batch.build_all()).await { + if let Err(err) = server.store().write_batch(&mut batch).await { + trc::error!( + err.details("Failed to delete queue event.") + .caused_by(trc::location!()) + ); + } + + // Unlock event + server.unlock_event(queue_id, self.queue_name).await; + + QueueEventStatus::Completed + } + Err(err) => { trc::error!( - err.details("Failed to delete queue event.") + err.details("Failed to read message.") + .ctx(trc::Key::QueueId, queue_id) .caused_by(trc::location!()) ); + + server.unlock_event(queue_id, self.queue_name).await; + + QueueEventStatus::Locked } - - // Unlock event - server.unlock_event(queue_id, self.queue_name).await; - - QueueEventStatus::Completed } } else { QueueEventStatus::Locked }; + #[cfg(stalwart_tests)] + crate::queue::probe::DispatchProbe::completed(probe_started); + // Notify queue manager if server .inner @@ -193,7 +213,7 @@ impl QueuedMessage { if let Err(retry_at) = server.is_allowed(throttle, &message, message.span_id).await { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), - Id = throttle.id.to_string(), + Name = throttle.id.to_compact_string(), SpanId = span_id, NextRetry = trc::Value::Timestamp(retry_at) ); @@ -207,10 +227,10 @@ impl QueuedMessage { && rcpt.queue == message.queue_name { rcpt.retry.due = retry_at; - rcpt.status = Status::TemporaryFailure(ErrorDetails { + rcpt.status = Status::TemporaryFailure(Box::new(ErrorDetails { entity: "localhost".into(), details: Error::RateLimited, - }); + })); } } @@ -223,16 +243,22 @@ impl QueuedMessage { // Group recipients by route let queue_config = &server.core.smtp.queue; let now_ = now(); + let mut arena = Bump::new(); let mut routes: AHashMap<(&str, &RoutingStrategy, Option<&[u8]>), Vec> = AHashMap::new(); - let mut has_rcpt_headers = false; let mut default_rcpt_header = None; + let mut rcpt_header_by_idx: Vec> = Vec::new(); for metadata in message.message.metadata.iter() { if let Metadata::Headers { value, id } = metadata { - has_rcpt_headers = true; if *id == u64::MAX { - default_rcpt_header = Some(value.as_ref()); - break; + default_rcpt_header.get_or_insert(value.as_ref()); + } else if let Ok(idx) = usize::try_from(*id) { + if rcpt_header_by_idx.is_empty() { + rcpt_header_by_idx.resize(message.message.recipients.len(), None); + } + if let Some(slot) = rcpt_header_by_idx.get_mut(idx) { + slot.get_or_insert(value.as_ref()); + } } } } @@ -244,26 +270,21 @@ impl QueuedMessage { && rcpt.queue == message.queue_name { let envelope = QueueEnvelope::new(&message.message, rcpt); - let route = server.get_route_or_default( - &server - .eval_if::(&queue_config.route, &envelope, message.span_id) - .await - .unwrap_or_else(|| "default".to_string()), + let route = eval_strategy( + &server, + &queue_config.route, + &envelope, + &mut arena, message.span_id, - ); + Server::get_route_or_default, + ) + .await; - // Map RCPT headers - let mut rcpt_headers = default_rcpt_header; - if has_rcpt_headers { - for metadata in message.message.metadata.iter() { - if let Metadata::Headers { value, id } = metadata - && *id == rcpt_idx as u64 - { - rcpt_headers = Some(value.as_ref()); - break; - } - } - } + let rcpt_headers = rcpt_header_by_idx + .get(rcpt_idx) + .copied() + .flatten() + .or(default_rcpt_header); routes .entry((rcpt.domain_part(), route, rcpt_headers)) @@ -274,11 +295,12 @@ impl QueuedMessage { let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); let mut delivery_results: Vec = Vec::new(); + let message_blob = OnceCell::new(); 'next_route: for ((domain, route, rcpt_headers), rcpt_idxs) in routes { trc::event!( Delivery(DeliveryEvent::DomainDeliveryStart), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), ); // Build envelope @@ -293,9 +315,9 @@ impl QueuedMessage { { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), - Id = throttle.id.to_string(), + Name = throttle.id.to_compact_string(), SpanId = span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), ); delivery_results.push(DeliveryResult::rate_limited(rcpt_idxs, retry_at)); @@ -321,13 +343,15 @@ impl QueuedMessage { }; // Prepare TLS strategy - let mut tls_strategy = server.get_tls_or_default( - &server - .eval_if::(&queue_config.tls, &envelope, message.span_id) - .await - .unwrap_or_else(|| "default".to_string()), + let mut tls_strategy = eval_strategy( + &server, + &queue_config.tls, + &envelope, + &mut arena, message.span_id, - ); + Server::get_tls_or_default, + ) + .await; // Obtain TLS reporting let tls_report = if is_smtp @@ -338,6 +362,7 @@ impl QueuedMessage { .eval_if( &server.core.smtp.report.tls.send, &envelope, + &mut arena, message.span_id, ) .await @@ -352,7 +377,7 @@ impl QueuedMessage { .smtp .resolvers .dns - .txt_lookup::( + .txt_lookup::( format!("_smtp._tls.{domain}."), Some(&server.inner.cache.dns_txt), ) @@ -362,14 +387,14 @@ impl QueuedMessage { trc::event!( TlsRpt(TlsRptEvent::RecordFetch), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Details = record .rua .iter() .map(|uri| trc::Value::from(match uri { mail_auth::mta_sts::ReportUri::Mail(uri) | mail_auth::mta_sts::ReportUri::Http(uri) => - uri.to_string(), + CompactString::from(uri), })) .collect::>(), Elapsed = time.elapsed(), @@ -381,7 +406,7 @@ impl QueuedMessage { trc::event!( TlsRpt(TlsRptEvent::RecordNotFound), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Elapsed = time.elapsed(), ); None @@ -390,7 +415,7 @@ impl QueuedMessage { trc::event!( TlsRpt(TlsRptEvent::RecordFetchError), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), CausedBy = trc::Error::from(err), Elapsed = time.elapsed(), ); @@ -415,9 +440,9 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::PolicyFetch), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Strict = mta_sts_policy.enforce(), - Details = mta_sts_policy + Hostname = mta_sts_policy .mx .iter() .map(|mx| trc::Value::String(mx.to_compact_string())) @@ -439,11 +464,11 @@ impl QueuedMessage { server.schedule_report(TlsEvent { policy: PolicyType::Sts(None), domain: domain.to_string(), - failure: FailureDetails::new(ResultType::Other) - .with_failure_reason_code( - "MTA-STS is required and no policy was found.", - ) - .into(), + failure: FailureDetails { + failure_reason_code: Some("MTA-STS is required and no policy was found.".into()), + ..FailureDetails::new(FailureType::Other) + } + .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, span_id: message.span_id, @@ -459,9 +484,11 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Sts(None), domain: domain.to_string(), - failure: FailureDetails::new(&err) - .with_failure_reason_code(err.to_string()) - .into(), + failure: FailureDetails { + failure_reason_code: Some(err.to_string()), + ..FailureDetails::new(&err) + } + .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, span_id: message.span_id, @@ -478,7 +505,7 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::PolicyNotFound), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Strict = strict, Elapsed = time.elapsed(), ); @@ -487,7 +514,7 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::PolicyFetchError), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), CausedBy = trc::Error::from(err.clone()), Strict = strict, Elapsed = time.elapsed(), @@ -497,8 +524,18 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::PolicyFetchError), SpanId = message.span_id, - Domain = domain.to_string(), - Reason = err.to_string(), + Domain = CompactString::from(domain), + Reason = err.to_compact_string(), + Strict = strict, + Elapsed = time.elapsed(), + ); + } + mta_sts::Error::Destination(err) => { + trc::event!( + MtaSts(MtaStsEvent::PolicyFetchError), + SpanId = message.span_id, + Domain = CompactString::from(domain), + Reason = err.as_str(), Strict = strict, Elapsed = time.elapsed(), ); @@ -507,12 +544,27 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::InvalidPolicy), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Reason = reason.clone(), Strict = strict, Elapsed = time.elapsed(), ); } + mta_sts::Error::Backoff(failure) => { + trc::event!( + MtaSts(if failure.kind == MtaStsFailureKind::InvalidPolicy { + MtaStsEvent::InvalidPolicy + } else { + MtaStsEvent::PolicyFetchError + }), + SpanId = message.span_id, + Domain = CompactString::from(domain), + Reason = failure.reason.to_string(), + Details = "Policy fetch skipped during back-off", + Strict = strict, + Elapsed = time.elapsed(), + ); + } } if strict { @@ -553,13 +605,13 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::MxLookupFailed), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Details = "No MX records were found, attempting implicit MX.", Elapsed = time.elapsed(), ); RecordSet { - rrset: Arc::new([]), + records: Arc::new([]), dnssec_status: DnssecStatus::Indeterminate, } } @@ -567,7 +619,7 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::MxLookupFailed), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), CausedBy = trc::Error::from(err.clone()), Elapsed = time.elapsed(), ); @@ -584,8 +636,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::MxLookup), SpanId = message.span_id, - Domain = domain.to_string(), - Details = remote_hosts_ + Domain = CompactString::from(domain), + Hostname = remote_hosts_ .iter() .map(|h| trc::Value::String(h.hostname().into())) .collect::>(), @@ -596,17 +648,17 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::NullMx), SpanId = message.span_id, - Domain = domain.to_string(), + Domain = CompactString::from(domain), Elapsed = time.elapsed(), ); delivery_results.push(DeliveryResult::domain( - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: domain.into(), details: Error::DnsError( "Domain does not accept messages (null MX)".into(), ), - }), + })), rcpt_idxs, )); continue 'next_route; @@ -627,10 +679,14 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: mta_sts_policy.into(), domain: domain.to_string(), - failure: FailureDetails::new(ResultType::ValidationFailure) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code("MX not authorized by policy.") - .into(), + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + failure_reason_code: Some( + "MX not authorized by policy.".into(), + ), + ..FailureDetails::new(FailureType::ValidationFailure) + } + .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, span_id: message.span_id, @@ -641,8 +697,8 @@ impl QueuedMessage { trc::event!( MtaSts(MtaStsEvent::NotAuthorized), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = mta_sts_policy .mx .iter() @@ -652,21 +708,21 @@ impl QueuedMessage { ); if strict { - last_status = Status::PermanentFailure(ErrorDetails { + last_status = Status::PermanentFailure(Box::new(ErrorDetails { entity: envelope.mx.into(), details: Error::MtaStsError( format!("MX {:?} not authorized by policy.", envelope.mx) .into_boxed_str(), ), - }); + })); continue 'next_host; } } else { trc::event!( MtaSts(MtaStsEvent::Authorized), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = mta_sts_policy .mx .iter() @@ -678,13 +734,15 @@ impl QueuedMessage { } // Update TLS strategy - tls_strategy = server.get_tls_or_default( - &server - .eval_if::(&queue_config.tls, &envelope, message.span_id) - .await - .unwrap_or_else(|| "default".to_string()), + tls_strategy = eval_strategy( + &server, + &queue_config.tls, + &envelope, + &mut arena, message.span_id, - ); + Server::get_tls_or_default, + ) + .await; let validated_host; let remote_host = if mx_unvalidated && tls_strategy.try_dane() { @@ -727,9 +785,9 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::IpLookup), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = resolved + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Ip = resolved .ips .iter() .map(|ip| trc::Value::from(*ip)) @@ -744,9 +802,9 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::IpLookupFailed), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = status.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Details = status.to_compact_string(), Elapsed = time.elapsed(), ); @@ -757,7 +815,11 @@ impl QueuedMessage { // Lookup DANE policy let mut dane_requires_encryption = false; - let dane_policy = if tls_strategy.try_dane() && is_smtp { + let dane_policy = if tls_strategy.try_dane() + && is_smtp + && (matches!(remote_host, NextHop::MX { .. }) + || tls_strategy.is_dane_required()) + { let time = Instant::now(); let strict = tls_strategy.is_dane_required(); @@ -775,9 +837,9 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::TlsaRecordFetch), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = format!("{tlsa:?}"), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Details = format_compact!("{tlsa:?}"), Strict = strict, Elapsed = time.elapsed(), ); @@ -787,9 +849,9 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::TlsaRecordInvalid), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = format!("{tlsa:?}"), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Details = format_compact!("{tlsa:?}"), Strict = strict, Elapsed = time.elapsed(), ); @@ -800,13 +862,17 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: tlsa.into(), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::TlsaInvalid, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "Invalid TLSA record.", - ) + failure: FailureDetails { + receiving_mx_hostname: Some( + envelope.mx.into(), + ), + failure_reason_code: Some( + "Invalid TLSA record.".into(), + ), + ..FailureDetails::new( + FailureType::TlsaInvalid, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -816,12 +882,13 @@ impl QueuedMessage { } if strict { - last_status = Status::TemporaryFailure(ErrorDetails { - entity: envelope.mx.into(), - details: Error::DaneError( - "No valid TLSA records were found".into(), - ), - }); + last_status = + Status::TemporaryFailure(Box::new(ErrorDetails { + entity: envelope.mx.into(), + details: Error::DaneError( + "No valid TLSA records were found".into(), + ), + })); continue 'next_host; } @@ -833,8 +900,8 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::BogusDnssecRecord), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = "TLSA", Strict = strict, Elapsed = time.elapsed(), @@ -846,13 +913,15 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::DnssecInvalid, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "Bogus TLSA records were found.", - ) + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + failure_reason_code: Some( + "Bogus TLSA records were found.".into(), + ), + ..FailureDetails::new( + FailureType::DnssecInvalid, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -861,20 +930,21 @@ impl QueuedMessage { .await; } - last_status = Status::TemporaryFailure(ErrorDetails { - entity: envelope.mx.into(), - details: Error::DaneError( - "Bogus TLSA records were found".into(), - ), - }); + last_status = + Status::TemporaryFailure(Box::new(ErrorDetails { + entity: envelope.mx.into(), + details: Error::DaneError( + "Bogus TLSA records were found".into(), + ), + })); continue 'next_host; } Ok(TlsaResult::Missing) => { trc::event!( Dane(DaneEvent::TlsaRecordNotDnssecSigned), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Strict = strict, Elapsed = time.elapsed(), ); @@ -886,13 +956,17 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::DaneRequired, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "No TLSA DNSSEC records found.", - ) + failure: FailureDetails { + receiving_mx_hostname: Some( + envelope.mx.into(), + ), + failure_reason_code: Some( + "No TLSA DNSSEC records found.".into(), + ), + ..FailureDetails::new( + FailureType::DaneRequired, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -901,12 +975,13 @@ impl QueuedMessage { .await; } - last_status = Status::TemporaryFailure(ErrorDetails { - entity: envelope.mx.into(), - details: Error::DaneError( - "No TLSA DNSSEC records found".into(), - ), - }); + last_status = + Status::TemporaryFailure(Box::new(ErrorDetails { + entity: envelope.mx.into(), + details: Error::DaneError( + "No TLSA DNSSEC records found".into(), + ), + })); continue 'next_host; } None @@ -923,8 +998,8 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::TlsaRecordNotFound), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Strict = strict, Elapsed = time.elapsed(), ); @@ -936,13 +1011,18 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::DaneRequired, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "No TLSA records found for MX.", - ) + failure: FailureDetails { + receiving_mx_hostname: Some( + envelope.mx.into(), + ), + failure_reason_code: Some( + "No TLSA records found for MX." + .into(), + ), + ..FailureDetails::new( + FailureType::DaneRequired, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -951,12 +1031,13 @@ impl QueuedMessage { .await; } - last_status = Status::TemporaryFailure(ErrorDetails { - entity: envelope.mx.into(), - details: Error::DaneError( - "No TLSA records found".into(), - ), - }); + last_status = + Status::TemporaryFailure(Box::new(ErrorDetails { + entity: envelope.mx.into(), + details: Error::DaneError( + "No TLSA records found".into(), + ), + })); continue 'next_host; } None @@ -964,8 +1045,8 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::TlsaRecordFetchError), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), CausedBy = trc::Error::from(err.clone()), Strict = strict, Elapsed = time.elapsed(), @@ -982,8 +1063,8 @@ impl QueuedMessage { trc::event!( Dane(DaneEvent::BogusDnssecRecord), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = dnssec_entity, Strict = strict, Elapsed = time.elapsed(), @@ -995,12 +1076,14 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), - failure: FailureDetails::new(ResultType::DnssecInvalid) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code(format!( + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + failure_reason_code: Some(format!( "Bogus {dnssec_entity} records were found." - )) - .into(), + )), + ..FailureDetails::new(FailureType::DnssecInvalid) + } + .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, span_id: message.span_id, @@ -1008,20 +1091,20 @@ impl QueuedMessage { .await; } - last_status = Status::TemporaryFailure(ErrorDetails { + last_status = Status::TemporaryFailure(Box::new(ErrorDetails { entity: envelope.mx.into(), details: Error::DaneError( format!("Bogus {dnssec_entity} records were found").into(), ), - }); + })); continue 'next_host; } _ => { trc::event!( Dane(DaneEvent::TlsaRecordNotDnssecSigned), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Strict = strict, Elapsed = time.elapsed(), ); @@ -1033,12 +1116,15 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), - failure: FailureDetails::new(ResultType::DaneRequired) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "MX host is not in a DNSSEC signed zone.", - ) - .into(), + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + failure_reason_code: Some( + "MX host is not in a DNSSEC signed zone." + .into(), + ), + ..FailureDetails::new(FailureType::DaneRequired) + } + .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, span_id: message.span_id, @@ -1046,12 +1132,12 @@ impl QueuedMessage { .await; } - last_status = Status::TemporaryFailure(ErrorDetails { + last_status = Status::TemporaryFailure(Box::new(ErrorDetails { entity: envelope.mx.into(), details: Error::DaneError( "No TLSA DNSSEC records found".into(), ), - }); + })); continue 'next_host; } None @@ -1063,8 +1149,24 @@ impl QueuedMessage { // Try each IP address 'next_ip: for remote_ip in remote_ips { - // Throttle remote host envelope.remote_ip = remote_ip; + + // Obtain connection parameters + let conn_strategy = eval_strategy( + &server, + &queue_config.connection, + &envelope, + &mut arena, + message.span_id, + Server::get_connection_or_default, + ) + .await; + + // Set source IP, if any + let ip_host = conn_strategy.source_ip(remote_ip.is_ipv4()); + envelope.local_ip = ip_host.map_or(no_ip, |ip_host| ip_host.ip); + + // Throttle remote host for throttle in &queue_config.outbound_limiters.remote { if let Err(retry_at) = server .is_allowed(throttle, &envelope, message.span_id) @@ -1073,7 +1175,7 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), SpanId = message.span_id, - Id = throttle.id.to_string(), + Name = throttle.id.to_compact_string(), RemoteIp = remote_ip, ); delivery_results @@ -1082,26 +1184,9 @@ impl QueuedMessage { } } - // Obtain connection parameters - let conn_strategy = server.get_connection_or_default( - &server - .eval_if::( - &queue_config.connection, - &envelope, - message.span_id, - ) - .await - .unwrap_or_else(|| "default".to_string()), - message.span_id, - ); - - // Set source IP, if any - let ip_host = conn_strategy.source_ip(remote_ip.is_ipv4()); - // Connect let time = Instant::now(); let mut smtp_client = match if let Some(ip_host) = ip_host { - envelope.local_ip = ip_host.ip; SmtpClient::connect_using( ip_host.ip, SocketAddr::new(remote_ip, remote_host.port()), @@ -1110,7 +1195,6 @@ impl QueuedMessage { ) .await } else { - envelope.local_ip = no_ip; SmtpClient::connect( SocketAddr::new(remote_ip, remote_host.port()), conn_strategy.timeout_connect, @@ -1122,8 +1206,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::Connect), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), LocalIp = envelope.local_ip, RemoteIp = remote_ip, RemotePort = remote_host.port(), @@ -1136,8 +1220,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::ConnectError), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), LocalIp = envelope.local_ip, RemoteIp = remote_ip, RemotePort = remote_host.port(), @@ -1164,6 +1248,7 @@ impl QueuedMessage { local_hostname, conn_strategy, capabilities: None, + message_blob: &message_blob, }; // Prepare TLS connector @@ -1192,9 +1277,9 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::GreetingFailed), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = status.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Details = status.to_compact_string(), ); last_status = status; @@ -1208,8 +1293,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::Ehlo), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = capabilities.capabilities(), Elapsed = time.elapsed(), ); @@ -1220,9 +1305,9 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::EhloRejected), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Details = status.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Details = status.to_compact_string(), Elapsed = time.elapsed(), ); @@ -1243,16 +1328,16 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::StartTls), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), - Version = format!( + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), + Version = format_compact!( "{:?}", smtp_client .tls_connection() .protocol_version() .unwrap() ), - Details = format!( + Cipher = format_compact!( "{:?}", smtp_client .tls_connection() @@ -1277,14 +1362,19 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: dane_policy.into(), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::ValidationFailure, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code( - "No matching certificates found.", - ) + failure: FailureDetails { + receiving_mx_hostname: Some( + envelope.mx.into(), + ), + receiving_ip: Some(remote_ip), + failure_reason_code: Some( + "No matching certificates found." + .into(), + ), + ..FailureDetails::new( + FailureType::ValidationFailure, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -1335,14 +1425,15 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::StartTlsUnavailable), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Code = response.as_ref().map(|r| r.code()), - Details = response - .as_ref() - .map(|r| r.message().as_ref()) - .unwrap_or("STARTTLS was not advertised by host") - .to_string(), + Details = CompactString::from( + response + .as_ref() + .map(|r| r.message().as_ref()) + .unwrap_or("STARTTLS was not advertised by host") + ), Elapsed = time.elapsed(), ); @@ -1351,12 +1442,14 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: (&mta_sts_policy, &dane_policy).into(), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::StartTlsNotSupported, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code(reason) + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + receiving_ip: Some(remote_ip), + failure_reason_code: Some(reason), + ..FailureDetails::new( + FailureType::StartTlsNotSupported, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -1387,8 +1480,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::StartTlsError), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Reason = from_mail_send_error(&error), Elapsed = time.elapsed(), ); @@ -1401,12 +1494,14 @@ impl QueuedMessage { .schedule_report(TlsEvent { policy: (&mta_sts_policy, &dane_policy).into(), domain: domain.to_string(), - failure: FailureDetails::new( - ResultType::CertificateNotTrusted, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code(error.to_string()) + failure: FailureDetails { + receiving_mx_hostname: Some(envelope.mx.into()), + receiving_ip: Some(remote_ip), + failure_reason_code: Some(error.to_string()), + ..FailureDetails::new( + FailureType::CertificateNotTrusted, + ) + } .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, @@ -1428,8 +1523,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::StartTlsDisabled), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), ); message @@ -1452,8 +1547,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::ImplicitTlsError), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Reason = from_mail_send_error(&error), ); @@ -1468,8 +1563,8 @@ impl QueuedMessage { trc::event!( Delivery(DeliveryEvent::GreetingFailed), SpanId = message.span_id, - Domain = domain.to_string(), - Hostname = envelope.mx.to_string(), + Domain = CompactString::from(domain), + Hostname = CompactString::from(envelope.mx), Details = from_error_status(&status), ); @@ -1497,15 +1592,19 @@ impl QueuedMessage { // Update status delivery_results.push(DeliveryResult::domain(last_status, rcpt_idxs)); } + drop(message_blob); // Apply status changes for delivery_result in delivery_results { match delivery_result { DeliveryResult::Domain { status, rcpt_idxs } => { - for rcpt_idx in rcpt_idxs { - message - .set_rcpt_status(status.clone(), rcpt_idx, &server) - .await; + if let Some((&last_idx, rcpt_idxs)) = rcpt_idxs.split_last() { + for &rcpt_idx in rcpt_idxs { + message + .set_rcpt_status(status.clone(), rcpt_idx, &server) + .await; + } + message.set_rcpt_status(status, last_idx, &server).await; } } DeliveryResult::Account { status, rcpt_idx } => { @@ -1583,8 +1682,8 @@ impl MessageWrapper { Delivery(DeliveryEvent::Failed), SpanId = self.span_id, QueueId = self.queue_id, - QueueName = self.queue_name.as_str().to_string(), - To = rcpt.address().to_string(), + QueueName = CompactString::from(self.queue_name.as_str()), + To = CompactString::from(rcpt.address()), Reason = from_error_details(&err.details), Details = trc::Value::Timestamp(now), Expires = rcpt @@ -1602,8 +1701,8 @@ impl MessageWrapper { Delivery(DeliveryEvent::Failed), SpanId = self.span_id, QueueId = self.queue_id, - QueueName = self.queue_name.as_str().to_string(), - To = rcpt.address().to_string(), + QueueName = CompactString::from(self.queue_name.as_str()), + To = CompactString::from(rcpt.address()), Reason = "Message expired without any delivery attempts made.", Details = trc::Value::Timestamp(now), Expires = rcpt @@ -1613,12 +1712,12 @@ impl MessageWrapper { NextDsn = trc::Value::Timestamp(rcpt.notify.due), ); - rcpt.status = Status::PermanentFailure(ErrorDetails { + rcpt.status = Status::PermanentFailure(Box::new(ErrorDetails { entity: rcpt.domain_part().into(), details: Error::Io( "Message expired without any delivery attempts made.".into(), ), - }); + })); } Status::Completed(_) | Status::PermanentFailure(_) => (), _ => { @@ -1645,14 +1744,17 @@ impl MessageWrapper { self.message.recipients[rcpt_idx].status = status; if needs_retry { + let mut arena = Bump::new(); let envelope = QueueEnvelope::new(&self.message, &self.message.recipients[rcpt_idx]); - let queue = server.get_queue_or_default( - &server - .eval_if::(&server.core.smtp.queue.queue, &envelope, self.span_id) - .await - .unwrap_or_else(|| "default".to_string()), + let queue = eval_strategy( + server, + &server.core.smtp.queue.queue, + &envelope, + &mut arena, self.span_id, - ); + Server::get_queue_or_default, + ) + .await; let rcpt = &mut self.message.recipients[rcpt_idx]; rcpt.retry.due = now() + queue.retry[std::cmp::min(rcpt.retry.inner as usize, queue.retry.len() - 1)]; @@ -1665,9 +1767,9 @@ impl MessageWrapper { pub fn set_rcpt_rate_limit(&mut self, rcpt_idx: usize, retry_at: u64) { let rcpt = &mut self.message.recipients[rcpt_idx]; rcpt.retry.due = retry_at; - rcpt.status = Status::TemporaryFailure(ErrorDetails { + rcpt.status = Status::TemporaryFailure(Box::new(ErrorDetails { entity: "localhost".into(), details: Error::RateLimited, - }); + })); } } diff --git a/crates/smtp/src/outbound/error.rs b/crates/smtp/src/outbound/error.rs index ef62ea3d..f3c0213b 100644 --- a/crates/smtp/src/outbound/error.rs +++ b/crates/smtp/src/outbound/error.rs @@ -6,6 +6,7 @@ use std::fmt::Display; +use encodify::Error as CodecError; use smtp_proto::{Response, Severity}; #[derive(Debug)] @@ -17,7 +18,7 @@ pub enum ClientError { Tls(Box), /// Base64 decode error - Base64(base64::DecodeError), + Base64(CodecError), // SMTP authentication error. InvalidChallenge, @@ -143,8 +144,8 @@ impl From for ClientError { } } -impl From for ClientError { - fn from(err: base64::DecodeError) -> Self { +impl From for ClientError { + fn from(err: CodecError) -> Self { ClientError::Base64(err) } } diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index bd38fc62..6a46c0aa 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -5,20 +5,28 @@ */ use crate::{ - inbound::dkim::DkimSign, + inbound::dkim::{DkimSign, has_dkim2_headers}, outbound::DeliveryResult, queue::{ - Error, ErrorDetails, FROM_AUTHENTICATED, FROM_UNAUTHENTICATED_DMARC, HostResponse, + Error, ErrorDetails, FROM_AUTHENTICATED, FROM_UNAUTHENTICATED_DMARC, HostResponse, Message, MessageSource, MessageWrapper, Status, UnexpectedResponse, quota::HasQueueQuota, rcpt_spam_percentage, - spool::{QueueParams, SmtpSpool}, + spool::{QueueOutcome, QueueParams, SmtpSpool}, }, + scripts::notify_flags, }; use common::Server; -use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery}; +use common::expr::{Bump, Variable, functions::ResolveVariable}; +use email::message::delivery::{ + AutogeneratedReturnPath, IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery, + SenderAuthentication, +}; +use mail_auth::AuthenticatedMessage; +use registry::schema::enums::ExpressionVariable; use smtp_proto::Response; use trc::SieveEvent; +use utils::DomainPart; impl MessageWrapper { pub(super) async fn deliver_local( @@ -45,9 +53,13 @@ impl MessageWrapper { let delivery_result = server .deliver_message(IngestMessage { sender_address: self.message.return_path.to_string(), - sender_authenticated: self.message.flags - & (FROM_UNAUTHENTICATED_DMARC | FROM_AUTHENTICATED) - != 0, + sender_authentication: if self.message.flags & FROM_AUTHENTICATED != 0 { + SenderAuthentication::Submission + } else if self.message.flags & FROM_UNAUTHENTICATED_DMARC != 0 { + SenderAuthentication::DmarcPass + } else { + SenderAuthentication::Unauthenticated + }, recipients, message_blob: self.message.blob_hash.clone(), message_size: self.message.size, @@ -60,39 +72,39 @@ impl MessageWrapper { pending_recipients.into_iter().zip(delivery_result.status) { let status = match result { - LocalDeliveryStatus::Success => Status::Completed(HostResponse { + LocalDeliveryStatus::Success => Status::Completed(Box::new(HostResponse { hostname: "localhost".into(), response: Response { code: 250, esc: [2, 1, 5], message: "OK".into(), }, - }), + })), LocalDeliveryStatus::TemporaryFailure { reason } => { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: "localhost".into(), - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: format!("RCPT TO:<{rcpt_addr}>").into_boxed_str(), response: Response { code: 451, esc: [4, 3, 0], message: reason.into(), }, - }), - }) + })), + })) } LocalDeliveryStatus::PermanentFailure { code, reason } => { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: "localhost".into(), - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: format!("RCPT TO:<{rcpt_addr}>").into_boxed_str(), response: Response { code: 550, esc: code, message: reason.into(), }, - }), - }) + })), + })) } }; statuses.push(DeliveryResult::account(status, rcpt_idx)); @@ -100,46 +112,103 @@ impl MessageWrapper { // Process autogenerated messages for autogenerated in delivery_result.autogenerated { - let mut message = server.new_message( - autogenerated.sender_address, - MessageSource::Autogenerated, - self.span_id, - ); + let return_path = match autogenerated.return_path { + AutogeneratedReturnPath::Account => autogenerated.account_address.as_str(), + AutogeneratedReturnPath::Null => "", + }; + let mut message = + server.new_message(return_path, MessageSource::Autogenerated, self.span_id); for rcpt in autogenerated.recipients { message.expand_and_add_recipient(rcpt, server).await; } + let flags = notify_flags(&autogenerated.notify); + if flags != 0 { + for rcpt in &mut message.message.recipients { + rcpt.flags |= flags; + } + } // Queue Message message.message.size = autogenerated.message.len() as u64; - if let Some(metadata) = server.has_quota(&mut message).await { - let dkim_signers = server - .eval_signers( - &server.core.sieve.untrusted_sign, - &message.message, - self.span_id, - ) - .await; + let quotas = server.queue_quotas(&message).await; + let dkim_signers = server + .eval_signers( + &server.core.sieve.untrusted_sign, + &AutogeneratedSigner { + message: &message.message, + account_address: &autogenerated.account_address, + }, + self.span_id, + ) + .await; - let _ = message - .queue( - QueueParams::new(&autogenerated.message, self.span_id, server) - .with_dkim_signers(dkim_signers) - .with_metadata(metadata), + let auth_message = if dkim_signers + .as_ref() + .is_some_and(|signers| signers.dkim2.is_some()) + && has_dkim2_headers(&autogenerated.message) + && server + .has_remote_domain( + message + .message + .recipients + .iter() + .map(|rcpt| rcpt.domain_part()), ) - .await; + .await + { + AuthenticatedMessage::parse(&autogenerated.message) } else { - trc::event!( - Sieve(SieveEvent::QuotaExceeded), - SpanId = self.span_id, - From = message.message.return_path, - To = message - .message - .recipients - .into_iter() - .map(|r| trc::Value::from(r.address().to_string())) - .collect::>(), - ); + None + }; + let dkim2_verified = match &auth_message { + Some(auth_message) => { + server + .dkim2_signing_proof( + dkim_signers.as_deref(), + auth_message, + &self.message.return_path, + rcpt_idxs + .iter() + .filter_map(|&rcpt_idx| self.message.recipients.get(rcpt_idx)) + .map(|rcpt| rcpt.address()), + ) + .await + } + None => None, + }; + + if message + .queue( + QueueParams::new(&autogenerated.message, self.span_id, server) + .with_dkim_signers(dkim_signers) + .with_dkim2_verified(dkim2_verified.as_ref()) + .with_exploded(autogenerated.exploded) + .with_quotas(quotas), + ) + .await + == QueueOutcome::QuotaExceeded + { + trc::event!(Sieve(SieveEvent::QuotaExceeded), SpanId = self.span_id); } } } } + +struct AutogeneratedSigner<'x> { + message: &'x Message, + account_address: &'x str, +} + +impl ResolveVariable for AutogeneratedSigner<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { + match variable { + ExpressionVariable::Sender => self.account_address.into(), + ExpressionVariable::SenderDomain => self.account_address.domain_part().into(), + _ => self.message.resolve_variable(variable, arena), + } + } +} diff --git a/crates/smtp/src/outbound/lookup.rs b/crates/smtp/src/outbound/lookup.rs index 3292e7cb..afd788c8 100644 --- a/crates/smtp/src/outbound/lookup.rs +++ b/crates/smtp/src/outbound/lookup.rs @@ -10,9 +10,9 @@ use crate::queue::{Error, ErrorDetails, HostResponse, Status}; use common::{ Server, config::smtp::queue::{ConnectionStrategy, HostOrIp, IpAndHost, MxConfig}, - expr::functions::ResolveVariable, + expr::{Bump, functions::ResolveVariable}, }; -use mail_auth::{DnssecStatus, IpLookupStrategy, MX, RecordSet}; +use mail_auth::{DnssecStatus, IpLookupStrategy, Mx, RecordSet}; use rand::{RngExt, seq::SliceRandom}; use registry::schema::enums::ExpressionVariable; use std::{future::Future, net::IpAddr, sync::Arc}; @@ -69,10 +69,10 @@ impl DnsLookup for Server { match result { Ok(addrs) => { - if !addrs.rrset.is_empty() { + if !addrs.records.is_empty() { dnssec_status = Some(addrs.dnssec_status); } - addrs.rrset + addrs.records } Err(_) if has_ipv6 => Arc::new([]), Err(err) => return Err(err), @@ -95,13 +95,13 @@ impl DnsLookup for Server { match result { Ok(addrs) => { - if !addrs.rrset.is_empty() { + if !addrs.records.is_empty() { dnssec_status = Some(match dnssec_status { Some(status) => least_secure(status, addrs.dnssec_status), None => addrs.dnssec_status, }); } - addrs.rrset + addrs.records } Err(_) if !ipv4_addrs.is_empty() => Arc::new([]), Err(err) => return Err(err), @@ -158,23 +158,23 @@ impl DnsLookup for Server { .. } ) { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: remote_host.hostname().into(), details: Error::DnsError("no MX record found.".into()), - }) + })) } else { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError("record not found for MX".into()), - }) + })) } } else { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError( format!("lookup error: {err}").into_boxed_str(), ), - }) + })) } })?, HostOrIp::Ip(ip) => (vec![ip], DnssecStatus::Indeterminate), @@ -184,10 +184,10 @@ impl DnsLookup for Server { if !remote_host.allow_loopback() && remote_ips.iter().any(|ip| ip.is_loopback()) { remote_ips.retain(|ip| !ip.is_loopback()); if remote_ips.is_empty() { - return Err(Status::PermanentFailure(ErrorDetails { + return Err(Status::PermanentFailure(Box::new(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError("host resolves loopback address".into()), - })); + }))); } } @@ -196,18 +196,19 @@ impl DnsLookup for Server { dnssec_status, }) } else { - Err(Status::TemporaryFailure(ErrorDetails { + let arena = Bump::new(); + Err(Status::TemporaryFailure(Box::new(ErrorDetails { entity: remote_host.hostname().into(), details: Error::DnsError( format!( "No IP addresses found for {:?}.", envelope - .resolve_variable(ExpressionVariable::Mx) - .to_string() + .resolve_variable(ExpressionVariable::Mx, &arena) + .to_str(&arena) ) .into_boxed_str(), ), - })) + }))) } } } @@ -239,17 +240,17 @@ pub trait ToNextHop { ) -> Option>>; } -impl ToNextHop for RecordSet { +impl ToNextHop for RecordSet { fn to_remote_hosts<'x, 'y: 'x>( &'x self, domain: &'y str, config: &'x MxConfig, ) -> Option>> { - if !self.rrset.is_empty() { + if !self.records.is_empty() { // Obtain max number of MX hosts to process let mut remote_hosts = Vec::with_capacity(config.max_mx); - 'outer: for mx in self.rrset.iter() { + 'outer: for mx in self.records.iter() { if mx.exchanges.len() > 1 { let mut slice = mx.exchanges.iter().collect::>(); slice.shuffle(&mut rand::rng()); diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index 1cff2b1e..89c0605c 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -10,7 +10,10 @@ use crate::{ }; use common::config::{ server::ServerProtocol, - smtp::queue::{HostOrIp, MxConfig, RelayConfig}, + smtp::{ + queue::{HostOrIp, MxConfig, RelayConfig}, + resolver::MtaStsFailureKind, + }, }; use directory::Credentials; use mail_auth::{DnssecStatus, IpLookupStrategy}; @@ -52,38 +55,38 @@ impl Status>, ErrorDetails> { | ClientError::MissingCredentials | ClientError::MissingMailFrom | ClientError::MissingRcptTo - | ClientError::Timeout => Status::TemporaryFailure(ErrorDetails { + | ClientError::Timeout => Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(err.to_string().into_boxed_str()), - }), + })), ClientError::UnexpectedReply(response) => { if response.severity() == Severity::PermanentNegativeCompletion { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: hostname.into(), - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: command.trim().into(), response: response.into_box(), - }), - }) + })), + })) } else { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: command.trim().into(), response: response.into_box(), - }), - }) + })), + })) } } ClientError::InvalidChallenge | ClientError::UnsupportedAuthMechanism | ClientError::InvalidTLSName - | ClientError::MissingStartTls => Status::PermanentFailure(ErrorDetails { + | ClientError::MissingStartTls => Status::PermanentFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(err.to_string().into_boxed_str()), - }), + })), } } @@ -91,83 +94,83 @@ impl Status>, ErrorDetails> { let entity = hostname.into(); if let Some(response) = response { if response.severity() == Severity::PermanentNegativeCompletion { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity, - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: "STARTTLS".into(), response, - }), - }) + })), + })) } else { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity, - details: Error::UnexpectedResponse(UnexpectedResponse { + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { command: "STARTTLS".into(), response, - }), - }) + })), + })) } } else { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity, details: Error::TlsError("STARTTLS not advertised by host.".into()), - }) + })) } } pub fn from_tls_error(hostname: &str, err: ClientError) -> Self { match err { - ClientError::InvalidTLSName => Status::PermanentFailure(ErrorDetails { + ClientError::InvalidTLSName => Status::PermanentFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::TlsError("Invalid hostname".into()), - }), - ClientError::Timeout => Status::TemporaryFailure(ErrorDetails { + })), + ClientError::Timeout => Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::TlsError("TLS handshake timed out".into()), - }), - ClientError::Tls(err) => Status::TemporaryFailure(ErrorDetails { + })), + ClientError::Tls(err) => Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::TlsError(format!("Handshake failed: {err}").into_boxed_str()), - }), - ClientError::Io(err) => Status::TemporaryFailure(ErrorDetails { + })), + ClientError::Io(err) => Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::TlsError(format!("I/O error: {err}").into_boxed_str()), - }), - _ => Status::PermanentFailure(ErrorDetails { + })), + _ => Status::PermanentFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::TlsError("Other TLS error".into()), - }), + })), } } pub fn timeout(hostname: &str, stage: &str) -> Self { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(format!("Timeout while {stage}").into_boxed_str()), - }) + })) } pub fn local_error() -> Self { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: "localhost".into(), details: Error::ConnectionError("Could not deliver message locally.".into()), - }) + })) } pub fn from_mail_auth_error(entity: &str, err: mail_auth::Error) -> Self { match &err { mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(code)) => { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::DnsError( format!("Domain not found: {code:?}").into_boxed_str(), ), - }) + })) } - _ => Status::TemporaryFailure(ErrorDetails { + _ => Status::TemporaryFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::DnsError(err.to_string().into_boxed_str()), - }), + })), } } @@ -175,59 +178,79 @@ impl Status>, ErrorDetails> { match &err { mta_sts::Error::Dns(err) => match err { mail_auth::Error::Dns(mail_auth::DnsError::RecordNotFound(code)) => { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( format!("Record not found: {code:?}").into_boxed_str(), ), - }) + })) } mail_auth::Error::Dns(mail_auth::DnsError::InvalidRecordType) => { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Failed to parse MTA-STS DNS record.".into()), - }) + })) } - _ => Status::TemporaryFailure(ErrorDetails { + _ => Status::TemporaryFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( format!("DNS lookup error: {err}").into_boxed_str(), ), - }), + })), }, mta_sts::Error::Http(err) => { if err.is_timeout() { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Timeout fetching policy.".into()), - }) + })) } else if err.is_connect() { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Could not reach policy host.".into()), - }) + })) } else if err.is_status() & err .status() .is_some_and(|s| s == reqwest::StatusCode::NOT_FOUND) { - Status::PermanentFailure(ErrorDetails { + Status::PermanentFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Policy not found.".into()), - }) + })) } else { - Status::TemporaryFailure(ErrorDetails { + Status::TemporaryFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Failed to fetch policy.".into()), - }) + })) } } - mta_sts::Error::InvalidPolicy(err) => Status::PermanentFailure(ErrorDetails { + mta_sts::Error::Destination(err) => Status::PermanentFailure(Box::new(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( - format!("Failed to parse policy: {err}").into_boxed_str(), + format!("Policy host rejected: {err}").into_boxed_str(), ), - }), + })), + mta_sts::Error::InvalidPolicy(err) => { + Status::PermanentFailure(Box::new(ErrorDetails { + entity: entity.into(), + details: Error::MtaStsError( + format!("Failed to parse policy: {err}").into_boxed_str(), + ), + })) + } + mta_sts::Error::Backoff(failure) => { + let details = Box::new(ErrorDetails { + entity: entity.into(), + details: Error::MtaStsError(failure.reason.clone()), + }); + match failure.kind { + MtaStsFailureKind::TemporaryFetchError => Status::TemporaryFailure(details), + MtaStsFailureKind::PermanentFetchError | MtaStsFailureKind::InvalidPolicy => { + Status::PermanentFailure(details) + } + } + } } } } @@ -297,9 +320,9 @@ impl NextHop<'_> { #[inline(always)] fn port(&self) -> u16 { match self { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] NextHop::MX { .. } => 9925, - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] NextHop::MX { .. } => 25, NextHop::Relay(host) => host.port, } @@ -308,7 +331,7 @@ impl NextHop<'_> { #[inline(always)] fn allow_loopback(&self) -> bool { match self { - NextHop::MX { .. } => cfg!(feature = "test_mode"), + NextHop::MX { .. } => cfg!(stalwart_tests), NextHop::Relay(_) => true, } } @@ -323,11 +346,11 @@ impl NextHop<'_> { #[inline(always)] fn allow_invalid_certs(&self) -> bool { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { true } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] match self { NextHop::MX { .. } => false, NextHop::Relay(host) => host.tls_allow_invalid_certs, diff --git a/crates/smtp/src/outbound/mta_sts/lookup.rs b/crates/smtp/src/outbound/mta_sts/lookup.rs index 73503d64..e221a383 100644 --- a/crates/smtp/src/outbound/mta_sts/lookup.rs +++ b/crates/smtp/src/outbound/mta_sts/lookup.rs @@ -4,20 +4,37 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Display, sync::Arc, time::Duration}; +use std::{ + fmt::Display, + sync::Arc, + time::{Duration, Instant}, +}; -#[cfg(feature = "test_mode")] +#[cfg(stalwart_tests)] pub static STS_TEST_POLICY: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); -use common::{Server, config::smtp::resolver::Policy}; -use mail_auth::{mta_sts::MtaSts, report::tlsrpt::ResultType}; +use common::{ + Server, + config::smtp::resolver::{CachedPolicy, MtaStsEntry, MtaStsFailure, MtaStsFailureKind, Policy}, +}; +use compact_str::CompactString; +use mail_auth::mta_sts::MtaStsRecord; +use smtp_report::tlsrpt::FailureType; +use trc::MtaStsEvent; +use utils::outbound::OutboundError; use super::{Error, parse::ParsePolicy}; -#[cfg(not(feature = "test_mode"))] +#[cfg(not(stalwart_tests))] +use reqwest::Method; +#[cfg(not(stalwart_tests))] use utils::HttpLimitResponse; -#[cfg(not(feature = "test_mode"))] +const FETCH_RETRY_INTERVAL: Duration = Duration::from_secs(300); +const FETCH_IN_PROGRESS: &str = "Policy fetch in progress."; +const MAX_REASON_LEN: usize = 256; + +#[cfg(not(stalwart_tests))] const MAX_POLICY_SIZE: usize = 1024 * 1024; pub trait MtaStsLookup: Sync + Send { @@ -35,13 +52,15 @@ impl MtaStsLookup for Server { domain: &str, timeout: Duration, ) -> Result, Error> { + let cache = &self.inner.cache.dns_mta_sts; + // Lookup MTA-STS TXT record let record = match self .core .smtp .resolvers .dns - .txt_lookup::( + .txt_lookup::( format!("_mta-sts.{domain}."), Some(&self.inner.cache.dns_txt), ) @@ -50,62 +69,169 @@ impl MtaStsLookup for Server { Ok(record) => record, Err(err) => { // Return the cached policy in case of failure - return if let Some(value) = self.inner.cache.dns_mta_sts.get(domain) { - Ok(value) - } else { - Err(err.into()) - }; + return cache + .get(domain) + .and_then(|entry| { + entry + .fallback(Instant::now()) + .map(|cached| cached.policy.clone()) + }) + .ok_or_else(|| err.into()); } }; - // Check if the policy has been cached - if let Some(value) = self.inner.cache.dns_mta_sts.get(domain) - && value.id == record.id - { - return Ok(value); - } + // Check if the policy or a failed fetch has been cached + let entry = match cache.get(domain) { + Some(entry) => Ok(entry), + None => { + cache + .get_value_or_guard_async(&Box::::from(domain)) + .await + } + }; + let (fallback, guard) = match entry { + Ok(entry) => { + let now = Instant::now(); + match entry.policy_for(&record.id, now) { + Some(Ok(policy)) => return Ok(policy), + Some(Err(failure)) => return Err(Error::Backoff(failure)), + None => { + let fallback = entry.fallback(now).cloned(); + if let Some(cached) = &fallback { + let retry_after = now + FETCH_RETRY_INTERVAL; + cache.insert_with_expiry( + domain.into(), + MtaStsEntry::Failed(Arc::new(MtaStsFailure { + id: record.id.as_str().into(), + kind: MtaStsFailureKind::TemporaryFetchError, + reason: FETCH_IN_PROGRESS.into(), + retry_after, + fallback: Some(cached.clone()), + })), + cached.expires.max(retry_after), + ); + } + (fallback, None) + } + } + } + Err(guard) => (None, Some(guard)), + }; // Fetch policy - #[cfg(not(feature = "test_mode"))] - let bytes = self - .core - .smtp - .mta_sts_client - .get(format!("https://mta-sts.{domain}/.well-known/mta-sts.txt")) - .timeout(timeout) - .send() - .await? - .bytes_with_limit(MAX_POLICY_SIZE) - .await? - .ok_or_else(|| Error::InvalidPolicy("Policy too large".to_string()))?; - #[cfg(feature = "test_mode")] - let bytes = STS_TEST_POLICY.lock().clone(); + let result: Result = async { + #[cfg(not(stalwart_tests))] + let bytes = self + .core + .smtp + .mta_sts_client + .request( + Method::GET, + &format!("https://mta-sts.{domain}/.well-known/mta-sts.txt"), + )? + .timeout(timeout) + .send() + .await? + .bytes_with_limit(MAX_POLICY_SIZE) + .await? + .ok_or_else(|| Error::InvalidPolicy("Policy too large".to_string()))?; + #[cfg(stalwart_tests)] + let bytes = STS_TEST_POLICY.lock().clone(); - // Parse policy - let policy = Arc::new(Policy::parse( - std::str::from_utf8(&bytes).map_err(|err| Error::InvalidPolicy(err.to_string()))?, - record.id.clone(), - )?); + Ok(Policy::parse( + std::str::from_utf8(&bytes).map_err(|err| Error::InvalidPolicy(err.to_string()))?, + record.id.clone(), + )?) + } + .await; - self.inner.cache.dns_mta_sts.insert( - domain.into(), - policy.clone(), - Duration::from_secs(if (3600..31557600).contains(&policy.max_age) { - policy.max_age - } else { - 86400 - }), - ); + let (entry, expires, result) = match result { + Ok(policy) => { + let policy = Arc::new(policy); + let expires = Instant::now() + + Duration::from_secs(if (3600..31557600).contains(&policy.max_age) { + policy.max_age + } else { + 86400 + }); + ( + MtaStsEntry::Policy(CachedPolicy { + policy: policy.clone(), + expires, + }), + expires, + Ok(policy), + ) + } + Err(err) => { + let retry_after = Instant::now() + FETCH_RETRY_INTERVAL; + let expires = fallback + .as_ref() + .map_or(retry_after, |cached| cached.expires.max(retry_after)); + let kind = err.failure_kind(); + let mut reason = err.to_string(); + reason.truncate(reason.floor_char_boundary(MAX_REASON_LEN)); + let cached = fallback.as_ref().map(|cached| cached.policy.clone()); + if cached.is_some() { + trc::event!( + MtaSts(if kind == MtaStsFailureKind::InvalidPolicy { + MtaStsEvent::InvalidPolicy + } else { + MtaStsEvent::PolicyFetchError + }), + Domain = CompactString::from(domain), + Reason = reason.clone(), + Details = "Applying the cached policy", + ); + } + let entry = MtaStsEntry::Failed(Arc::new(MtaStsFailure { + id: record.id.as_str().into(), + kind, + reason: reason.into_boxed_str(), + retry_after, + fallback, + })); + (entry, expires, cached.ok_or(err)) + } + }; - Ok(policy) + match guard { + Some(guard) => guard.insert_with_expiry(entry, expires), + None if matches!(entry, MtaStsEntry::Failed(_)) + && matches!( + cache.get(domain), + Some(MtaStsEntry::Policy(cached)) if cached.policy.id == record.id + ) => {} + None => cache.insert_with_expiry(domain.into(), entry, expires), + } + + result } } -impl From<&Error> for ResultType { +impl Error { + fn failure_kind(&self) -> MtaStsFailureKind { + match self { + Error::InvalidPolicy(_) => MtaStsFailureKind::InvalidPolicy, + Error::Destination(_) => MtaStsFailureKind::PermanentFetchError, + Error::Http(err) + if err.is_status() && err.status() == Some(reqwest::StatusCode::NOT_FOUND) => + { + MtaStsFailureKind::PermanentFetchError + } + Error::Backoff(failure) => failure.kind, + Error::Http(_) | Error::Dns(_) => MtaStsFailureKind::TemporaryFetchError, + } + } +} + +impl From<&Error> for FailureType { fn from(err: &Error) -> Self { - match &err { - Error::InvalidPolicy(_) => ResultType::StsPolicyInvalid, - _ => ResultType::StsPolicyFetchError, + match err.failure_kind() { + MtaStsFailureKind::InvalidPolicy => FailureType::StsPolicyInvalid, + MtaStsFailureKind::TemporaryFetchError | MtaStsFailureKind::PermanentFetchError => { + FailureType::StsPolicyFetchError + } } } } @@ -134,7 +260,9 @@ impl Display for Error { f.write_str("Failed to fetch policy.") } } + Error::Destination(err) => write!(f, "Policy host rejected: {err}"), Error::InvalidPolicy(err) => write!(f, "Failed to parse policy: {err}"), + Error::Backoff(failure) => f.write_str(&failure.reason), } } } @@ -151,6 +279,12 @@ impl From for Error { } } +impl From for Error { + fn from(value: OutboundError) -> Self { + Error::Destination(value) + } +} + impl From for Error { fn from(value: String) -> Self { Error::InvalidPolicy(value) diff --git a/crates/smtp/src/outbound/mta_sts/mod.rs b/crates/smtp/src/outbound/mta_sts/mod.rs index 68a6af63..9a199851 100644 --- a/crates/smtp/src/outbound/mta_sts/mod.rs +++ b/crates/smtp/src/outbound/mta_sts/mod.rs @@ -8,9 +8,15 @@ pub mod lookup; pub mod parse; pub mod verify; +use common::config::smtp::resolver::MtaStsFailure; +use std::sync::Arc; +use utils::outbound::OutboundError; + #[derive(Debug)] pub enum Error { Dns(mail_auth::Error), Http(reqwest::Error), + Destination(OutboundError), InvalidPolicy(String), + Backoff(Arc), } diff --git a/crates/smtp/src/outbound/mta_sts/parse.rs b/crates/smtp/src/outbound/mta_sts/parse.rs index 50306863..e1f9af2f 100644 --- a/crates/smtp/src/outbound/mta_sts/parse.rs +++ b/crates/smtp/src/outbound/mta_sts/parse.rs @@ -83,28 +83,3 @@ impl ParsePolicy for Policy { } } -#[cfg(test)] -mod test { - use super::ParsePolicy; - use crate::outbound::mta_sts::verify::VerifyPolicy; - use common::config::smtp::resolver::Policy; - - #[test] - fn mx_patterns_are_a_labels() { - let policy = Policy::parse( - concat!( - "version: STSv1\n", - "mode: enforce\n", - "mx: *.\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}\n", - "mx: MAIL.\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}\n", - "max_age: 604800\n" - ), - "test".to_string(), - ) - .unwrap(); - - assert!(policy.verify("mx.xn--eebajf.xn--9dbq2a")); - assert!(policy.verify("mail.xn--eebajf.xn--9dbq2a")); - assert!(!policy.verify("mx.example.org")); - } -} diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs index 7f0d5c76..b3b6ec60 100644 --- a/crates/smtp/src/outbound/session.rs +++ b/crates/smtp/src/outbound/session.rs @@ -12,16 +12,23 @@ use crate::queue::{Error, MessageWrapper, Recipient, Status}; use crate::queue::{ErrorDetails, HostResponse, UnexpectedResponse}; use common::Server; use common::config::smtp::queue::ConnectionStrategy; +use compact_str::CompactString; use directory::Credentials; use smtp_proto::{ - EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, EhloResponse, MAIL_REQUIRETLS, - MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, - RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Severity, + EXT_CHUNKING, EXT_DSN, EXT_PIPELINING, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, EhloResponse, + MAIL_REQUIRETLS, MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_DELAY, + RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Severity, +}; +use std::{fmt::Write, ops::Range, time::Instant}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::OnceCell, }; -use std::{fmt::Write, time::Instant}; -use tokio::io::{AsyncRead, AsyncWrite}; use trc::DeliveryEvent; +const MAX_PIPELINED_RCPTS: usize = 100; +const MAX_PIPELINED_BYTES: usize = 8192; + pub struct SessionParams<'x> { pub server: &'x Server, pub hostname: &'x str, @@ -31,6 +38,7 @@ pub struct SessionParams<'x> { pub local_hostname: &'x str, pub conn_strategy: &'x ConnectionStrategy, pub session_id: u64, + pub message_blob: &'x OnceCell>, } impl MessageWrapper { @@ -52,7 +60,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::Ehlo), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), Details = capabilities.capabilities(), Elapsed = time.elapsed(), ); @@ -63,7 +71,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::EhloRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); @@ -81,7 +89,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::AuthFailed), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); @@ -97,7 +105,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::Auth), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), Elapsed = time.elapsed(), ); @@ -122,8 +130,30 @@ impl MessageWrapper { // MAIL FROM let time = Instant::now(); smtp_client.timeout = params.conn_strategy.timeout_mail; - let cmd = self.build_mail_from(&capabilities); - match smtp_client.cmd(cmd.as_bytes()).await.and_then(|r| { + let mut commands = self.build_mail_from(&capabilities); + let mail_from_len = commands.len(); + let max_batch = if capabilities.has_capability(EXT_PIPELINING) { + MAX_PIPELINED_RCPTS + } else { + 1 + }; + let mut pending_rcpts = rcpt_idxs.iter().copied().filter(|rcpt_idx| { + !matches!( + &self.message.recipients[*rcpt_idx].status, + Status::Completed(_) | Status::PermanentFailure(_) + ) + }); + let mut batch = Vec::new(); + if max_batch > 1 { + self.write_rcpt_batch( + &mut pending_rcpts, + max_batch, + &capabilities, + &mut commands, + &mut batch, + ); + } + match smtp_client.cmd(commands.as_bytes()).await.and_then(|r| { if r.is_positive_completion() { Ok(r) } else { @@ -134,10 +164,10 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::MailFrom), SpanId = params.session_id, - Hostname = params.hostname.to_string(), - From = self.message.return_path.to_string(), + Hostname = CompactString::from(params.hostname), + From = CompactString::from(&*self.message.return_path), Code = response.code, - Details = response.message.to_string(), + Details = CompactString::from(response.message), Elapsed = time.elapsed(), ); } @@ -145,14 +175,26 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::MailFromRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); + if matches!(err, ClientError::UnexpectedReply(_)) { + smtp_client.timeout = params.conn_strategy.timeout_rcpt; + for _ in batch.drain(..) { + if smtp_client.read_pipelined().await.is_err() { + break; + } + } + } smtp_client.quit().await; statuses.push(DeliveryResult::domain( - Status::from_smtp_error(params.hostname, &cmd, err), + Status::from_smtp_error( + params.hostname, + commands.get(..mail_from_len).unwrap_or_default(), + err, + ), rcpt_idxs, )); return; @@ -161,85 +203,102 @@ impl MessageWrapper { // RCPT TO let mut accepted_rcpts = Vec::new(); + let mut needs_write = false; smtp_client.timeout = params.conn_strategy.timeout_rcpt; - for rcpt_idx in &rcpt_idxs { - let time = Instant::now(); - let rcpt = &self.message.recipients[*rcpt_idx]; - if matches!( - &rcpt.status, - Status::Completed(_) | Status::PermanentFailure(_) - ) { - continue; + loop { + if batch.is_empty() { + commands.clear(); + self.write_rcpt_batch( + &mut pending_rcpts, + max_batch, + &capabilities, + &mut commands, + &mut batch, + ); + if batch.is_empty() { + break; + } + needs_write = true; } - let cmd = self.build_rcpt_to(rcpt, &capabilities); - match smtp_client.cmd(cmd.as_bytes()).await { - Ok(response) => match response.severity() { - Severity::PositiveCompletion => { + for (rcpt_idx, rcpt, cmd_range) in batch.drain(..) { + let time = Instant::now(); + let result = if needs_write { + needs_write = false; + smtp_client.cmd(commands.as_bytes()).await + } else { + smtp_client.read_pipelined().await + }; + let cmd = commands.get(cmd_range).unwrap_or_default(); + + match result { + Ok(response) => match response.severity() { + Severity::PositiveCompletion => { + trc::event!( + Delivery(DeliveryEvent::RcptTo), + SpanId = params.session_id, + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), + Code = response.code, + Details = CompactString::from(&response.message), + Elapsed = time.elapsed(), + ); + + accepted_rcpts.push(( + rcpt, + rcpt_idx, + Status::Completed(Box::new(HostResponse { + hostname: params.hostname.into(), + response: response.into_box(), + })), + )); + } + severity => { + trc::event!( + Delivery(DeliveryEvent::RcptToRejected), + SpanId = params.session_id, + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), + Code = response.code, + Details = CompactString::from(&response.message), + Elapsed = time.elapsed(), + ); + + let response = ErrorDetails { + entity: params.hostname.into(), + details: Error::UnexpectedResponse(Box::new(UnexpectedResponse { + command: cmd.trim().into(), + response: response.into_box(), + })), + }; + statuses.push(DeliveryResult::account( + if severity == Severity::PermanentNegativeCompletion { + Status::PermanentFailure(Box::new(response)) + } else { + Status::TemporaryFailure(Box::new(response)) + }, + rcpt_idx, + )); + } + }, + Err(err) => { trc::event!( - Delivery(DeliveryEvent::RcptTo), + Delivery(DeliveryEvent::RcptToFailed), SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), - Code = response.code, - Details = response.message.to_string(), + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), + CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); - accepted_rcpts.push(( - rcpt, - rcpt_idx, - Status::Completed(HostResponse { - hostname: params.hostname.into(), - response: response.into_box(), - }), + // Something went wrong, abort. + smtp_client.quit().await; + statuses.push(DeliveryResult::domain( + Status::from_smtp_error(params.hostname, "", err), + rcpt_idxs, )); + return; } - severity => { - trc::event!( - Delivery(DeliveryEvent::RcptToRejected), - SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), - Code = response.code, - Details = response.message.to_string(), - Elapsed = time.elapsed(), - ); - - let response = ErrorDetails { - entity: params.hostname.into(), - details: Error::UnexpectedResponse(UnexpectedResponse { - command: cmd.trim().into(), - response: response.into_box(), - }), - }; - statuses.push(DeliveryResult::account( - if severity == Severity::PermanentNegativeCompletion { - Status::PermanentFailure(response) - } else { - Status::TemporaryFailure(response) - }, - *rcpt_idx, - )); - } - }, - Err(err) => { - trc::event!( - Delivery(DeliveryEvent::RcptToFailed), - SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), - CausedBy = from_mail_send_error(&err), - Elapsed = time.elapsed(), - ); - - // Something went wrong, abort. - smtp_client.quit().await; - statuses.push(DeliveryResult::domain( - Status::from_smtp_error(params.hostname, "", err), - rcpt_idxs, - )); - return; } } } @@ -256,7 +315,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); @@ -279,22 +338,22 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::Delivered), SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), Code = response.code, - Details = response.message.to_string(), + Details = CompactString::from(&response.message), Elapsed = time.elapsed(), ); - statuses.push(DeliveryResult::account(status, *rcpt_idx)); + statuses.push(DeliveryResult::account(status, rcpt_idx)); } } else { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), Code = response.code, - Details = response.message.to_string(), + Details = CompactString::from(&response.message), Elapsed = time.elapsed(), ); @@ -314,7 +373,7 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); @@ -340,32 +399,32 @@ impl MessageWrapper { trc::event!( Delivery(DeliveryEvent::Delivered), SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), Code = response.code, - Details = response.message.to_string(), + Details = CompactString::from(&*response.message), Elapsed = time.elapsed(), ); - Status::Completed(HostResponse { + Status::Completed(Box::new(HostResponse { hostname: params.hostname.into(), response, - }) + })) } severity => { trc::event!( Delivery(DeliveryEvent::RcptToRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), - To = rcpt.address().to_string(), + Hostname = CompactString::from(params.hostname), + To = CompactString::from(rcpt.address()), Code = response.code, - Details = response.message.to_string(), + Details = CompactString::from(&*response.message), Elapsed = time.elapsed(), ); let response = ErrorDetails { entity: params.hostname.into(), - details: Error::UnexpectedResponse( + details: Error::UnexpectedResponse(Box::new( UnexpectedResponse { command: bdat_cmd .as_deref() @@ -373,24 +432,24 @@ impl MessageWrapper { .into(), response, }, - ), + )), }; if severity == Severity::PermanentNegativeCompletion { - Status::PermanentFailure(response) + Status::PermanentFailure(Box::new(response)) } else { - Status::TemporaryFailure(response) + Status::TemporaryFailure(Box::new(response)) } } }; - statuses.push(DeliveryResult::account(status, *rcpt_idx)); + statuses.push(DeliveryResult::account(status, rcpt_idx)); } } Err(status) => { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, - Hostname = params.hostname.to_string(), + Hostname = CompactString::from(params.hostname), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); @@ -433,8 +492,32 @@ impl MessageWrapper { mail_from } - fn build_rcpt_to(&self, rcpt: &Recipient, capabilities: &EhloResponse) -> String { - let mut rcpt_to = String::with_capacity(rcpt.address().len() + 60); + fn write_rcpt_batch<'x>( + &'x self, + pending_rcpts: &mut impl Iterator, + max_batch: usize, + capabilities: &EhloResponse, + commands: &mut String, + batch: &mut Vec<(usize, &'x Recipient, Range)>, + ) { + while batch.len() < max_batch + && (batch.is_empty() || commands.len() < MAX_PIPELINED_BYTES) + && let Some(rcpt_idx) = pending_rcpts.next() + { + let rcpt = &self.message.recipients[rcpt_idx]; + let start = commands.len(); + self.write_rcpt_to(rcpt, capabilities, commands); + batch.push((rcpt_idx, rcpt, start..commands.len())); + } + } + + fn write_rcpt_to( + &self, + rcpt: &Recipient, + capabilities: &EhloResponse, + rcpt_to: &mut String, + ) { + rcpt_to.reserve(rcpt.address().len() + 60); let _ = write!(rcpt_to, "RCPT TO:<{}>", rcpt.address()); if capabilities.has_capability(EXT_DSN) { if rcpt.has_flag(RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY) { @@ -464,7 +547,6 @@ impl MessageWrapper { } } rcpt_to.push_str("\r\n"); - rcpt_to } #[inline(always)] diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs index c08271be..afac8b1c 100644 --- a/crates/smtp/src/queue/dsn.rs +++ b/crates/smtp/src/queue/dsn.rs @@ -10,9 +10,10 @@ use super::{ Recipient, Status, }; use crate::inbound::dkim::DkimSign; -use crate::queue::spool::{DSN_RETRY, QueueParams}; -use crate::queue::{MessageWrapper, UnexpectedResponse}; -use common::Server; +use crate::queue::spool::{DSN_RETRY, QueueOutcome, QueueParams}; +use crate::queue::{MessageWrapper, UnexpectedResponse, eval_strategy}; +use common::{Server, expr::Bump}; +use compact_str::{CompactString, ToCompactString}; use email::message::delivery::ORCPT_ADDR_TYPE; use mail_builder::MessageBuilder; use mail_builder::headers::HeaderType; @@ -66,6 +67,7 @@ impl SendDsn for Server { .with_dkim_signers(dkim_signers), ) .await + == QueueOutcome::Queued { message.mark_dsn_sent(); } else { @@ -101,7 +103,7 @@ impl SendDsn for Server { To = rcpt.address.clone(), Hostname = response.hostname.clone(), Code = response.response.code, - Details = response.response.message.to_string(), + Details = CompactString::from(&*response.response.message), ); } Status::TemporaryFailure(response) if rcpt.notify.due <= now => { @@ -110,7 +112,7 @@ impl SendDsn for Server { SpanId = message.span_id, To = rcpt.address.clone(), Hostname = response.entity.clone(), - Details = response.details.to_string(), + Details = response.details.to_compact_string(), NextRetry = trc::Value::Timestamp(rcpt.retry.due), Expires = rcpt .expiration_time(message.message.created) @@ -124,7 +126,7 @@ impl SendDsn for Server { SpanId = message.span_id, To = rcpt.address.clone(), Hostname = response.entity.clone(), - Details = response.details.to_string(), + Details = response.details.to_compact_string(), Total = rcpt.retry.inner, ); } @@ -269,18 +271,20 @@ impl MessageWrapper { } // Obtain hostname and sender addresses + let mut arena = Bump::new(); let from_name = server - .eval_if(&config.dsn.name, &self.message, self.span_id) + .eval_if(&config.dsn.name, &self.message, &mut arena, self.span_id) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); let from_addr = server - .eval_if(&config.dsn.address, &self.message, self.span_id) + .eval_if(&config.dsn.address, &self.message, &mut arena, self.span_id) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); let reporting_mta = server .eval_if( &server.core.smtp.report.submitter, &self.message, + &mut arena, self.span_id, ) .await @@ -376,6 +380,7 @@ impl MessageWrapper { pub async fn update_next_dsn(&mut self, server: &Server, status: DsnStatus) { let now = now(); let mut notify_changes = Vec::new(); + let mut arena = Bump::new(); for (rcpt_idx, rcpt) in self.message.recipients.iter().enumerate() { if matches!( &rcpt.status, @@ -389,11 +394,15 @@ impl MessageWrapper { let envelope = QueueEnvelope::new(&self.message, rcpt); - let queue_id = server - .eval_if::(&server.core.smtp.queue.queue, &envelope, self.span_id) - .await - .unwrap_or_else(|| "default".to_string()); - let queue = server.get_queue_or_default(&queue_id, self.span_id); + let queue = eval_strategy( + server, + &server.core.smtp.queue.queue, + &envelope, + &mut arena, + self.span_id, + Server::get_queue_or_default, + ) + .await; if let Some(next_notify) = queue.notify.get((rcpt.notify.inner + 1) as usize).copied() diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index a139f9a0..72a6c1de 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -4,56 +4,95 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Message, QueueId, Status, spool::SmtpSpool}; +use super::{ + Message, QueueId, QueuedMessage, Status, + walk::{QueueWalk, WalkStep, walk_queues}, +}; use crate::queue::{ Recipient, - spool::{INFINITE_LOCK, LOCK_EXPIRY, QUEUE_REFRESH}, + spool::{LOCK_EXPIRY, QUEUE_REFRESH}, }; use ahash::AHashMap; use common::{ - BuildServer, Inner, + BuildServer, Inner, Server, config::smtp::queue::{QueueExpiry, QueueName}, - ipc::{QueueEvent, QueueEventStatus}, + ipc::{QueueEvent, QueueEventStatus, QueueRefresh}, }; -use rand::{RngExt, seq::SliceRandom}; +use compact_str::ToCompactString; +use rand::RngExt; use std::{ collections::hash_map::Entry, sync::{Arc, atomic::Ordering}, time::{Duration, Instant}, }; -use store::write::now; +use store::{ + IterateParams, ValueKey, + write::{QueueClass, ValueClass, now}, +}; use tokio::sync::mpsc; pub struct Queue { - pub core: Arc, - pub locked: AHashMap<(QueueId, QueueName), LockedMessage>, - pub locked_revision: u64, - pub stats: AHashMap, - pub next_refresh: Instant, - pub rx: mpsc::Receiver, - pub is_paused: bool, - pub scan_from: u64, - pub scan_ceiling: u64, - pub has_pending_work: bool, - pub pending_refresh: bool, - pub urgent_refresh: bool, - pub last_scan: Instant, - pub last_full_scan: Instant, + core: Arc, + rx: mpsc::Receiver, + queues: Vec, + revision: u64, + is_paused: bool, + next_wake: Instant, + walk_at: Instant, + is_walk_due: bool, } -#[derive(Debug)] -pub struct QueueStats { - pub in_flight: usize, - pub max_in_flight: usize, - pub budget: usize, - pub last_warning: Instant, +struct VirtualQueueState { + name: QueueName, + max_in_flight: usize, + in_flight: usize, + tracked: AHashMap, + ready: Vec, + ready_at: Instant, + next_due: u64, + has_more: bool, + rescan: Rescan, + scanned_at: Instant, + wake_at: Instant, + last_warning: Instant, } -#[derive(Debug)] -pub struct LockedMessage { - pub expires: u64, - pub revision: u64, - pub due: u64, +struct Tracked { + due: u64, + revision: u64, + state: TrackedState, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Rescan { + None, + Coalesced, + Urgent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TrackedState { + Ready, + InFlight, + Parked { expires: u64 }, +} + +struct ScanCursor { + revision: u64, + want: usize, + limit: usize, + rows: usize, + taken: usize, + last_due: u64, + end: ScanEnd, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScanEnd { + Future(u64), + Exhausted, + Truncated(u64), + Failed, } impl SpawnQueue for mpsc::Receiver { @@ -71,6 +110,12 @@ impl SpawnQueue for mpsc::Receiver { const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60); const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100); const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(QUEUE_REFRESH / 2); +const SCAN_RETRY_INTERVAL: Duration = Duration::from_secs(1); +const CLUSTER_READY_TTL: Duration = Duration::from_secs(1); +const STARTUP_DELAY: Duration = Duration::from_secs(1); +const CANDIDATE_OVERSCAN: usize = 4; +const PAUSED_WAKE: Duration = Duration::from_secs(86400); +const WALK_UNKNOWN_QUEUE_ROWS: usize = 128; impl Queue { pub fn new(core: Arc, rx: mpsc::Receiver) -> Self { @@ -78,244 +123,81 @@ impl Queue { Queue { core, - locked: AHashMap::with_capacity(128), - locked_revision: 0, - stats: AHashMap::new(), - next_refresh: now + Duration::from_secs(1), - is_paused: false, rx, - scan_from: 0, - scan_ceiling: u64::MAX, - has_pending_work: false, - pending_refresh: false, - urgent_refresh: false, - last_scan: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now), - last_full_scan: now, + queues: Vec::with_capacity(8), + revision: 0, + is_paused: false, + next_wake: now + STARTUP_DELAY, + walk_at: now + FULL_SCAN_INTERVAL, + is_walk_due: true, } } pub async fn start(&mut self) { trc::event!(Queue(trc::QueueEvent::Started)); - loop { - let mut refresh_queue; + let server = self.core.build_server(); + self.reload_settings(&server); - match tokio::time::timeout( - self.next_refresh.duration_since(Instant::now()), + loop { + let event = tokio::time::timeout( + self.next_wake.saturating_duration_since(Instant::now()), self.rx.recv(), ) - .await - { + .await; + let server = self.core.build_server(); + match event { Ok(Some(event)) => { - refresh_queue = self.handle_event(event).await; + self.handle_event(event, &server); while let Ok(event) = self.rx.try_recv() { - refresh_queue = self.handle_event(event).await || refresh_queue; + self.handle_event(event, &server); } } - Err(_) => { - refresh_queue = true; - self.urgent_refresh = true; - } + Err(_) => {} Ok(None) => { break; } }; if self.is_paused { - self.next_refresh = Instant::now() + Duration::from_secs(86400); + self.next_wake = Instant::now() + PAUSED_WAKE; continue; } - self.pending_refresh |= refresh_queue; - if !self.pending_refresh && self.next_refresh > Instant::now() { - continue; - } - - // Coalesce bursts of worker notifications into a single scan - let scan_at = self.last_scan + MIN_SCAN_INTERVAL; - if !self.urgent_refresh && scan_at > Instant::now() { - self.next_refresh = scan_at; - continue; - } - - if self.scan_from != 0 && self.last_full_scan.elapsed() >= FULL_SCAN_INTERVAL { - self.scan_from = 0; - } - if self.scan_from == 0 { - self.last_full_scan = Instant::now(); - } - let scan_floor = self.scan_from; - self.pending_refresh = false; - self.urgent_refresh = false; - - // Process queue events - let server = self.core.build_server(); - let mut queue_events = server.next_event(self).await; - self.last_scan = Instant::now(); - - if queue_events.messages.len() > 3 { - queue_events.messages.shuffle(&mut rand::rng()); - } - - // A truncated scan left events behind - let now = now(); - self.has_pending_work = self.scan_ceiling != u64::MAX; - - for queue_event in &queue_events.messages { - // A message may hold more than one event key, dispatch it only once - if self - .locked - .get(&(queue_event.queue_id, queue_event.queue_name)) - .is_some_and(|locked| locked.expires > now) - { - continue; - } - - // Fetch queue stats - let stats = match self.stats.get_mut(&queue_event.queue_name) { - Some(stats) => stats, - None => { - let queue_config = - server.get_virtual_queue_or_default(&queue_event.queue_name); - self.stats.insert( - queue_event.queue_name, - QueueStats::new(queue_config.threads), - ); - self.stats.get_mut(&queue_event.queue_name).unwrap() - } - }; - - // Enforce concurrency limits - if stats.has_capacity() { - // Deliver message - stats.in_flight += 1; - self.locked.insert( - (queue_event.queue_id, queue_event.queue_name), - LockedMessage { - expires: now + INFINITE_LOCK, - revision: self.locked_revision, - due: queue_event.due, - }, - ); - queue_event.try_deliver(server.clone()); - } else { - if stats.last_warning.elapsed() >= BACK_PRESSURE_WARN_INTERVAL { - stats.last_warning = Instant::now(); - trc::event!( - Queue(trc::QueueEvent::BackPressure), - Reason = "Processing capacity for this queue exceeded.", - QueueName = queue_event.queue_name.to_string(), - Limit = stats.max_in_flight, - ); - } - self.has_pending_work = true; - if queue_event.due < self.scan_from { - self.scan_from = queue_event.due; - } - } - } - - // Remove expired locks, revisiting any event they were holding back - let scan_ceiling = self.scan_ceiling; - let mut dropped_due = u64::MAX; - self.locked.retain(|_, locked| { - let keep = locked.expires > now - && (locked.revision == self.locked_revision - || locked.due < scan_floor - || locked.due >= scan_ceiling); - if !keep && locked.due < dropped_due { - dropped_due = locked.due; - } - keep - }); - - // Do not wait for the next scheduled event while there is work left over - let mut next_refresh = queue_events.next_refresh.saturating_sub(now); - if self.has_pending_work { - next_refresh = std::cmp::min(next_refresh, FULL_SCAN_INTERVAL.as_secs()); - } - let mut next_refresh = Instant::now() + Duration::from_secs(next_refresh); - - // A released lock uncovered an event below the floor that no scan can see - if dropped_due < self.scan_from { - self.scan_from = dropped_due; - self.has_pending_work = true; - self.pending_refresh = true; - - let scan_at = self.last_scan + MIN_SCAN_INTERVAL; - if scan_at < next_refresh { - next_refresh = scan_at; - } - } - - self.next_refresh = next_refresh; + self.process(&server).await; } } - async fn handle_event(&mut self, event: QueueEvent) -> bool { + fn handle_event(&mut self, event: QueueEvent, server: &Server) { match event { QueueEvent::WorkerDone { queue_id, queue_name, status, } => { - let has_capacity = match self.stats.get_mut(&queue_name) { - Some(queue_stats) => { - queue_stats.in_flight = queue_stats.in_flight.saturating_sub(1); - queue_stats.has_capacity() - } - None => true, - }; - - match status { - QueueEventStatus::Completed => { - self.core.ipc.task_tx.notify_one(); - self.locked.remove(&(queue_id, queue_name)); - !self.locked.is_empty() || !has_capacity || self.has_pending_work - } - QueueEventStatus::Locked => { - let expires = LOCK_EXPIRY + rand::rng().random_range(5..10); - let due_in = Instant::now() + Duration::from_secs(expires); - if due_in < self.next_refresh { - self.next_refresh = due_in; - } - - // The event was not delivered, so it has to be visited again - // once the remote lock expires. - let expires = now() + expires; - let due = match self.locked.entry((queue_id, queue_name)) { - Entry::Occupied(mut entry) => { - let locked = entry.get_mut(); - locked.expires = expires; - locked.revision = self.locked_revision; - locked.due - } - Entry::Vacant(entry) => { - entry.insert(LockedMessage { - expires, - revision: self.locked_revision, - due: 0, - }); - 0 - } - }; - if due < self.scan_from { - self.scan_from = due; - } - self.locked.len() > 1 || !has_capacity || self.has_pending_work - } - QueueEventStatus::Deferred => { - self.locked.remove(&(queue_id, queue_name)); - self.scan_from = 0; - true - } + let is_cluster = server.core.storage.coordinator.is_enabled(); + virtual_queue(&mut self.queues, queue_name, server) + .worker_done(queue_id, status, is_cluster); + } + QueueEvent::Refresh(QueueRefresh::Queued(events)) => { + let now = now(); + for event in events { + virtual_queue(&mut self.queues, event.queue_name, server).add_event( + event.queue_id, + event.due, + self.revision, + now, + ); } } - QueueEvent::Refresh => { - self.scan_from = 0; - self.urgent_refresh = true; - true + QueueEvent::Refresh(QueueRefresh::Queues(names)) => { + for name in names.iter() { + virtual_queue(&mut self.queues, name, server).request_rescan(Rescan::Coalesced); + } + } + QueueEvent::Refresh(QueueRefresh::All) => { + self.is_walk_due = true; } QueueEvent::Paused(paused) => { self.core @@ -323,32 +205,550 @@ impl Queue { .queue_status .store(!paused, Ordering::Relaxed); self.is_paused = paused; - self.scan_from = 0; - self.urgent_refresh = !paused; - !paused + self.is_walk_due |= !paused; } QueueEvent::ReloadSettings => { - let server = self.core.build_server(); - let virtual_queues = &server.core.smtp.queue.virtual_queues; - for (name, settings) in virtual_queues { - if let Some(stats) = self.stats.get_mut(name) { - stats.max_in_flight = settings.threads; - } else { - self.stats.insert(*name, QueueStats::new(settings.threads)); - } - } - self.stats - .retain(|name, stats| stats.in_flight > 0 || virtual_queues.contains_key(name)); - self.scan_from = 0; - false + self.reload_settings(server); } QueueEvent::Stop => { self.rx.close(); self.is_paused = true; - false } } } + + fn reload_settings(&mut self, server: &Server) { + let virtual_queues = &server.core.smtp.queue.virtual_queues; + for (name, settings) in virtual_queues { + virtual_queue(&mut self.queues, *name, server).max_in_flight = settings.threads; + } + self.queues.retain(|queue| { + queue.in_flight > 0 + || !queue.tracked.is_empty() + || virtual_queues.contains_key(&queue.name) + }); + self.is_walk_due = true; + } + + async fn process(&mut self, server: &Server) { + let is_cluster = server.core.storage.coordinator.is_enabled(); + let mut now_instant = Instant::now(); + let mut now_timestamp = now(); + + if self.is_walk_due || self.walk_at <= now_instant { + for queue in &mut self.queues { + queue.expire_ready(now_instant, is_cluster); + queue.fill(server); + } + self.walk(server, now_instant, now_timestamp).await; + now_instant = Instant::now(); + now_timestamp = now(); + } + + self.queues + .sort_unstable_by_key(|queue| queue.service_order(now_timestamp)); + let Queue { + queues, revision, .. + } = self; + for queue in queues.iter_mut() { + queue + .service(server, revision, now_instant, now_timestamp, is_cluster) + .await; + } + + let now_instant = Instant::now(); + let now_timestamp = now(); + self.next_wake = self.queues.iter().fold( + self.walk_at + .min(now_instant + Duration::from_secs(QUEUE_REFRESH)), + |next_wake, queue| next_wake.min(queue.next_wake(now_instant, now_timestamp)), + ); + } + + async fn walk(&mut self, server: &Server, now: Instant, now_timestamp: u64) { + self.is_walk_due = false; + self.walk_at = now + FULL_SCAN_INTERVAL; + self.revision += 1; + let revision = self.revision; + let read_limit = self + .queues + .iter() + .map(|queue| queue.tracked.len() + queue.ready_capacity() + 1) + .sum::() + + WALK_UNKNOWN_QUEUE_ROWS; + + let queues = &mut self.queues; + let mut walked = vec![false; queues.len()]; + let mut current: Option<(usize, Option)> = None; + let result = walk_queues(server.store(), QueueWalk::ascending(read_limit), |event| { + let name = QueueName::from(event.queue_name); + if current + .as_ref() + .is_none_or(|(idx, _)| queues[*idx].name != name) + { + if let Some((idx, Some(cursor))) = current.take() { + let queue = &mut queues[idx]; + queue.end_scan(cursor, now, now_timestamp); + queue.fill(server); + } + let idx = queue_index(queues, name, server); + walked.resize(queues.len(), false); + walked[idx] = true; + current = Some((idx, queues[idx].begin_scan(revision, now))); + } + + match &mut current { + Some((idx, cursor @ Some(_))) => { + let queue = &mut queues[*idx]; + if let Some(open) = cursor + && queue.visit(open, event, now_timestamp) + { + WalkStep::Continue + } else { + if let Some(open) = cursor.take() { + queue.end_scan(open, now, now_timestamp); + queue.fill(server); + } + WalkStep::NextQueue + } + } + _ => WalkStep::NextQueue, + } + }) + .await; + + let is_failed = match result { + Ok(()) => false, + Err(err) => { + trc::error!( + err.details("Failed to read queues.") + .caused_by(trc::location!()) + ); + true + } + }; + if let Some((idx, Some(mut cursor))) = current { + if is_failed { + cursor.end = ScanEnd::Failed; + } + let queue = &mut queues[idx]; + queue.end_scan(cursor, now, now_timestamp); + queue.fill(server); + } + for (queue, _) in queues + .iter_mut() + .zip(walked.iter().copied().chain(std::iter::repeat(false))) + .filter(|(_, is_walked)| !is_walked) + { + if is_failed { + queue.request_rescan(Rescan::Coalesced); + } else if let Some(cursor) = queue.begin_scan(revision, now) { + queue.end_scan(cursor, now, now_timestamp); + } + } + if is_failed { + self.walk_at = now + SCAN_RETRY_INTERVAL; + } + } +} + +fn queue_index(queues: &mut Vec, name: QueueName, server: &Server) -> usize { + match queues.iter().position(|queue| queue.name == name) { + Some(idx) => idx, + None => { + queues.push(VirtualQueueState::new( + name, + server.get_virtual_queue_or_default(&name).threads, + )); + queues.len() - 1 + } + } +} + +fn virtual_queue<'x>( + queues: &'x mut Vec, + name: QueueName, + server: &Server, +) -> &'x mut VirtualQueueState { + let idx = queue_index(queues, name, server); + &mut queues[idx] +} + +impl VirtualQueueState { + fn new(name: QueueName, max_in_flight: usize) -> Self { + let now = Instant::now(); + + VirtualQueueState { + name, + max_in_flight, + in_flight: 0, + tracked: AHashMap::new(), + ready: Vec::new(), + ready_at: now, + next_due: u64::MAX, + has_more: false, + rescan: Rescan::Urgent, + scanned_at: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now), + wake_at: now + FULL_SCAN_INTERVAL, + last_warning: now.checked_sub(BACK_PRESSURE_WARN_INTERVAL).unwrap_or(now), + } + } + + fn request_rescan(&mut self, rescan: Rescan) { + self.rescan = self.rescan.max(rescan); + } + + fn is_urgent(&self, now: Instant) -> bool { + self.rescan == Rescan::Urgent || self.wake_at <= now + } + + fn should_rescan(&self, now: Instant) -> bool { + self.is_urgent(now) + || (self.rescan == Rescan::Coalesced && self.scanned_at + MIN_SCAN_INTERVAL <= now) + } + + fn service_order(&self, now_timestamp: u64) -> u8 { + if !self.ready.is_empty() || self.has_more || self.next_due <= now_timestamp { + 0 + } else if self.rescan == Rescan::Urgent { + 1 + } else { + 2 + } + } + + fn expire_ready(&mut self, now: Instant, is_cluster: bool) { + if is_cluster + && self.in_flight < self.max_in_flight + && !self.ready.is_empty() + && self.ready_at + CLUSTER_READY_TTL <= now + { + self.clear_ready(); + self.request_rescan(Rescan::Coalesced); + } + } + + fn ready_capacity(&self) -> usize { + self.max_in_flight.saturating_mul(CANDIDATE_OVERSCAN) + } + + fn worker_done(&mut self, queue_id: QueueId, status: QueueEventStatus, is_cluster: bool) { + self.in_flight = self.in_flight.saturating_sub(1); + + match status { + QueueEventStatus::Completed => { + self.tracked.remove(&queue_id); + } + QueueEventStatus::Deferred => { + self.tracked.remove(&queue_id); + if !self.has_more { + self.request_rescan(Rescan::Coalesced); + } + } + QueueEventStatus::Locked => { + if let Some(tracked) = self.tracked.get_mut(&queue_id) { + let park_for = + if is_cluster { LOCK_EXPIRY } else { 0 } + rand::rng().random_range(5..10); + tracked.state = TrackedState::Parked { + expires: now() + park_for, + }; + self.wake_at = self + .wake_at + .min(Instant::now() + Duration::from_secs(park_for)); + } + } + } + } + + fn add_event(&mut self, queue_id: QueueId, due: u64, revision: u64, now: u64) { + let has_room = self.ready.len() < self.ready_capacity(); + if due > now { + self.next_due = self.next_due.min(due); + } else if let Entry::Vacant(entry) = self.tracked.entry(queue_id) { + if has_room { + entry.insert(Tracked { + due, + revision, + state: TrackedState::Ready, + }); + if self.ready.is_empty() { + self.ready_at = Instant::now(); + } + self.ready.push(queue_id); + } else { + self.has_more = true; + } + } + } + + async fn service( + &mut self, + server: &Server, + revision: &mut u64, + now: Instant, + now_timestamp: u64, + is_cluster: bool, + ) { + self.expire_ready(now, is_cluster); + + self.fill(server); + + let mut is_scanned = false; + if self.in_flight < self.max_in_flight && self.should_rescan(now) { + *revision += 1; + self.scan(server, *revision, now, now_timestamp).await; + self.fill(server); + is_scanned = true; + } + + if !is_scanned + && self.ready.len() < self.max_in_flight + && (self.has_more || self.next_due <= now_timestamp) + { + *revision += 1; + self.scan(server, *revision, now, now_timestamp).await; + self.fill(server); + } + + if self.in_flight >= self.max_in_flight + && (self.has_more || !self.ready.is_empty()) + && self.last_warning + BACK_PRESSURE_WARN_INTERVAL <= now + { + self.last_warning = now; + trc::event!( + Queue(trc::QueueEvent::BackPressure), + Reason = "Processing capacity for this queue exceeded.", + QueueName = self.name.to_compact_string(), + Limit = self.max_in_flight, + ); + } + } + + fn next_wake(&self, now: Instant, now_timestamp: u64) -> Instant { + if self.in_flight >= self.max_in_flight { + return now + Duration::from_secs(QUEUE_REFRESH); + } + + if self.should_rescan(now) { + return now; + } + + let mut wake = self.wake_at; + if self.rescan == Rescan::Coalesced { + wake = wake.min(self.scanned_at + MIN_SCAN_INTERVAL); + } + if self.next_due != u64::MAX { + wake = wake.min(now + Duration::from_secs(self.next_due.saturating_sub(now_timestamp))); + } + wake.max(now) + } + + fn fill(&mut self, server: &Server) { + let mut rng = rand::rng(); + while self.in_flight < self.max_in_flight && !self.ready.is_empty() { + let queue_id = self + .ready + .swap_remove(rng.random_range(0..self.ready.len())); + if let Some(tracked) = self.tracked.get_mut(&queue_id) + && tracked.state == TrackedState::Ready + { + tracked.state = TrackedState::InFlight; + self.in_flight += 1; + QueuedMessage { + due: tracked.due, + queue_id, + queue_name: self.name, + } + .try_deliver(server.clone()); + } + } + } + + fn clear_ready(&mut self) { + for queue_id in self.ready.drain(..) { + if let Entry::Occupied(entry) = self.tracked.entry(queue_id) + && entry.get().state == TrackedState::Ready + { + entry.remove(); + } + } + } + + fn begin_scan(&mut self, revision: u64, now: Instant) -> Option { + self.scanned_at = now; + self.rescan = Rescan::None; + + let want = self.ready_capacity().saturating_sub(self.ready.len()); + if want == 0 { + self.has_more = true; + self.wake_at = self.wake_at.max(now + MIN_SCAN_INTERVAL); + return None; + } + if self.ready.is_empty() { + self.ready_at = now; + } + + Some(ScanCursor { + revision, + want, + limit: self.tracked.len() + want + 1, + rows: 0, + taken: 0, + last_due: 0, + end: ScanEnd::Exhausted, + }) + } + + fn visit( + &mut self, + cursor: &mut ScanCursor, + event: &store::write::QueueEvent, + now_timestamp: u64, + ) -> bool { + cursor.rows += 1; + if event.due > now_timestamp { + cursor.end = ScanEnd::Future(event.due); + return false; + } + cursor.last_due = event.due; + + let is_ready = match self.tracked.entry(event.queue_id) { + Entry::Occupied(mut entry) => { + let entry = entry.get_mut(); + entry.revision = cursor.revision; + match entry.state { + TrackedState::Parked { expires } if expires <= now_timestamp => { + entry.state = TrackedState::Ready; + entry.due = event.due; + true + } + _ => false, + } + } + Entry::Vacant(entry) => { + entry.insert(Tracked { + due: event.due, + revision: cursor.revision, + state: TrackedState::Ready, + }); + true + } + }; + + if is_ready { + self.ready.push(event.queue_id); + cursor.taken += 1; + if cursor.taken == cursor.want { + cursor.end = ScanEnd::Truncated(event.due); + return false; + } + } + if cursor.rows >= cursor.limit { + cursor.end = ScanEnd::Truncated(cursor.last_due); + return false; + } + + true + } + + fn end_scan(&mut self, cursor: ScanCursor, now: Instant, now_timestamp: u64) { + self.finish_scan(cursor.end, cursor.revision, now, now_timestamp); + } + + async fn scan(&mut self, server: &Server, revision: u64, now: Instant, now_timestamp: u64) { + let Some(mut cursor) = self.begin_scan(revision, now) else { + return; + }; + + let queue_name = self.name.into_inner(); + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: 0, + queue_id: 0, + queue_name, + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: now_timestamp + QUEUE_REFRESH, + queue_id: u64::MAX, + queue_name, + }, + ))); + + let result = server + .store() + .iterate( + IterateParams::new(from_key, to_key) + .ascending() + .no_values() + .limit(cursor.limit), + |key, _| { + let event = store::write::QueueEvent::deserialize(key)?; + Ok(self.visit(&mut cursor, &event, now_timestamp)) + }, + ) + .await; + + if let Err(err) = result { + trc::error!( + err.details("Failed to read queue.") + .caused_by(trc::location!()) + ); + cursor.end = ScanEnd::Failed; + } + self.end_scan(cursor, now, now_timestamp); + } + + fn finish_scan(&mut self, end: ScanEnd, revision: u64, now: Instant, now_timestamp: u64) { + let mut wake_at = now + FULL_SCAN_INTERVAL; + let evict_below = match end { + ScanEnd::Future(due) => { + self.next_due = due; + self.has_more = false; + self.ready_at = now; + u64::MAX + } + ScanEnd::Exhausted => { + self.next_due = u64::MAX; + self.has_more = false; + self.ready_at = now; + u64::MAX + } + ScanEnd::Truncated(last_due) => { + self.next_due = u64::MAX; + self.has_more = true; + last_due + } + ScanEnd::Failed => { + self.next_due = u64::MAX; + self.has_more = false; + wake_at = now + SCAN_RETRY_INTERVAL; + 0 + } + }; + + let mut has_evicted_ready = false; + self.tracked.retain(|_, tracked| match tracked.state { + TrackedState::InFlight => true, + state => { + let keep = tracked.revision == revision || tracked.due >= evict_below; + if !keep { + has_evicted_ready |= state == TrackedState::Ready; + } else if let TrackedState::Parked { expires } = state + && expires > now_timestamp + { + wake_at = wake_at.min(now + Duration::from_secs(expires - now_timestamp)); + } + keep + } + }); + if has_evicted_ready { + let tracked = &self.tracked; + self.ready.retain(|queue_id| tracked.contains_key(queue_id)); + } + self.wake_at = wake_at; + } } impl Message { @@ -487,20 +887,3 @@ pub trait SpawnQueue { fn discard(self); } -impl QueueStats { - pub(crate) fn new(max_in_flight: usize) -> Self { - QueueStats { - in_flight: 0, - max_in_flight, - budget: 0, - last_warning: Instant::now() - .checked_sub(BACK_PRESSURE_WARN_INTERVAL) - .unwrap_or_else(Instant::now), - } - } - - #[inline] - pub fn has_capacity(&self) -> bool { - self.in_flight < self.max_in_flight - } -} diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index bacc6c9a..82f47cc0 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -5,10 +5,10 @@ */ use common::{ + Server, config::smtp::queue::{QueueExpiry, QueueName}, - expr::{self, functions::ResolveVariable, *}, + expr::{Bump, Variable, bumpalo, functions::ResolveVariable, if_block::IfBlock}, }; -use compact_str::ToCompactString; use registry::schema::enums::ExpressionVariable; use smtp_proto::Response; use std::{ @@ -16,7 +16,7 @@ use std::{ net::{IpAddr, Ipv4Addr}, time::{Duration, Instant, SystemTime}, }; -use store::write::now; +use store::write::{ArchiveCompression, Compression, Dictionary, now}; use types::blob_hash::BlobHash; use utils::DomainPart; @@ -24,7 +24,9 @@ pub mod dsn; pub mod manager; pub mod quota; pub mod spool; +pub mod submission; pub mod throttle; +pub mod walk; pub type QueueId = u64; @@ -83,6 +85,13 @@ pub struct Message { } impl Message { + pub fn submission_account_id(&self) -> Option { + self.metadata.iter().find_map(|metadata| match metadata { + Metadata::Submission { account_id } => Some(*account_id), + _ => None, + }) + } + pub fn queued_event(&self) -> trc::QueueEvent { if (self.flags & FROM_AUTHENTICATED) != 0 { trc::QueueEvent::AuthenticatedMessageQueued @@ -118,9 +127,10 @@ pub struct MessageWrapper { serde::Deserialize, )] pub enum Metadata { - QueueSize { key: Box<[u8]>, id: u64 }, - QueueCount { key: Box<[u8]>, id: u64 }, + QueueSize { key: u128, id: u64 }, + QueueCount { key: u128, id: u64 }, Headers { value: Box<[u8]>, id: u64 }, + Submission { account_id: u32 }, } #[derive( @@ -184,11 +194,11 @@ pub enum Status { #[serde(rename = "scheduled")] Scheduled, #[serde(rename = "completed")] - Completed(T), + Completed(Box), #[serde(rename = "temp_fail")] - TemporaryFailure(E), + TemporaryFailure(Box), #[serde(rename = "perm_fail")] - PermanentFailure(E), + PermanentFailure(Box), } #[derive( @@ -219,7 +229,7 @@ pub struct HostResponse { )] pub enum Error { DnsError(Box), - UnexpectedResponse(UnexpectedResponse), + UnexpectedResponse(Box), ConnectionError(Box), TlsError(Box), DaneError(Box), @@ -319,20 +329,32 @@ impl<'x> QueueEnvelope<'x> { } } -impl<'x> ResolveVariable for QueueEnvelope<'x> { - fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'x> { +fn display<'a>(arena: &'a Bump, value: impl Display) -> Variable<'a> { + Variable::String(bumpalo::format!(in arena, "{}", value).into_bump_str()) +} + +fn addresses<'a>(arena: &'a Bump, recipients: &'a [Recipient]) -> Variable<'a> { + Variable::Array( + arena.alloc_slice_fill_iter( + recipients + .iter() + .map(|r| Variable::String(r.address.as_ref())), + ), + ) +} + +impl ResolveVariable for QueueEnvelope<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { ExpressionVariable::Sender => self.message.return_path.as_ref().into(), ExpressionVariable::SenderDomain => self.message.return_path.domain_part().into(), ExpressionVariable::RcptDomain => self.domain.into(), ExpressionVariable::Rcpt => self.rcpt.address.as_ref().into(), - ExpressionVariable::Recipients => self - .message - .recipients - .iter() - .map(|r| Variable::from(r.address.as_ref())) - .collect::>() - .into(), + ExpressionVariable::Recipients => addresses(arena, &self.message.recipients), ExpressionVariable::RetryNum => self.rcpt.retry.inner.into(), ExpressionVariable::NotifyNum => self.rcpt.notify.inner.into(), ExpressionVariable::ExpiresIn => match &self.rcpt.expires { @@ -342,7 +364,7 @@ impl<'x> ResolveVariable for QueueEnvelope<'x> { } } .into(), - ExpressionVariable::LastStatus => self.rcpt.status.to_compact_string().into(), + ExpressionVariable::LastStatus => display(arena, &self.rcpt.status), ExpressionVariable::LastError => match &self.rcpt.status { Status::Scheduled | Status::Completed(_) => "none", Status::TemporaryFailure(err) | Status::PermanentFailure(err) => { @@ -380,55 +402,42 @@ impl<'x> ResolveVariable for QueueEnvelope<'x> { .into(), ExpressionVariable::Mx => self.mx.into(), ExpressionVariable::Priority => self.message.priority.into(), - ExpressionVariable::RemoteIp => self.remote_ip.to_compact_string().into(), - ExpressionVariable::LocalIp => self.local_ip.to_compact_string().into(), - ExpressionVariable::ReceivedFromIp => { - self.message.received_from_ip.to_compact_string().into() - } + ExpressionVariable::RemoteIp => display(arena, self.remote_ip), + ExpressionVariable::LocalIp => display(arena, self.local_ip), + ExpressionVariable::ReceivedFromIp => display(arena, self.message.received_from_ip), ExpressionVariable::ReceivedViaPort => self.message.received_via_port.into(), ExpressionVariable::Size => self.message.size.into(), _ => "".into(), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } impl ResolveVariable for Message { - fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { ExpressionVariable::Sender => self.return_path.as_ref().into(), ExpressionVariable::SenderDomain => self.return_path.domain_part().into(), - ExpressionVariable::Recipients => self - .recipients - .iter() - .map(|r| Variable::from(r.address.as_ref())) - .collect::>() - .into(), + ExpressionVariable::Recipients => addresses(arena, &self.recipients), ExpressionVariable::Priority => self.priority.into(), _ => "".into(), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } impl ResolveVariable for MessageWrapper { - fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { ExpressionVariable::Sender => self.message.return_path.as_ref().into(), ExpressionVariable::SenderDomain => self.message.return_path.domain_part().into(), - ExpressionVariable::Recipients => self - .message - .recipients - .iter() - .map(|r| Variable::from(r.address.as_ref())) - .collect::>() - .into(), + ExpressionVariable::Recipients => addresses(arena, &self.message.recipients), ExpressionVariable::Priority => self.message.priority.into(), ExpressionVariable::QueueName => self.queue_name.as_str().into(), ExpressionVariable::QueueAge => now().saturating_sub(self.message.created).into(), @@ -448,18 +457,12 @@ impl ResolveVariable for MessageWrapper { "unknown" } .into(), - ExpressionVariable::ReceivedFromIp => { - self.message.received_from_ip.to_compact_string().into() - } + ExpressionVariable::ReceivedFromIp => display(arena, self.message.received_from_ip), ExpressionVariable::ReceivedViaPort => self.message.received_via_port.into(), ExpressionVariable::Size => self.message.size.into(), _ => "".into(), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } pub struct RecipientDomain<'x>(&'x str); @@ -470,17 +473,30 @@ impl<'x> RecipientDomain<'x> { } } -impl<'x> ResolveVariable for RecipientDomain<'x> { - fn resolve_variable(&self, variable: ExpressionVariable) -> expr::Variable<'x> { +impl ResolveVariable for RecipientDomain<'_> { + fn resolve_variable<'a>(&'a self, variable: ExpressionVariable, _: &'a Bump) -> Variable<'a> { match variable { ExpressionVariable::RcptDomain => self.0.into(), _ => "".into(), } } +} - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } +pub(crate) async fn eval_strategy<'x, T: Sync>( + server: &'x Server, + if_block: &IfBlock, + resolver: &impl ResolveVariable, + arena: &mut Bump, + session_id: u64, + lookup: fn(&'x Server, &str, u64) -> &'x T, +) -> &'x T { + server + .eval_if_with(if_block, resolver, arena, session_id, |value| match value { + Variable::String(name) => lookup(server, name, session_id), + _ => lookup(server, "default", session_id), + }) + .await + .unwrap_or_else(|| lookup(server, "default", session_id)) } #[inline(always)] @@ -649,18 +665,6 @@ impl Display for ArchivedErrorDetails { } } -/* - -pub trait DisplayArchivedResponse { - fn to_string(&self) -> String; +impl ArchiveCompression for Message { + const COMPRESSION: Compression = Compression::Zstd(Some(Dictionary::Common)); } - -impl DisplayArchivedResponse for ArchivedResponse> { - fn to_string(&self) -> String { - format!( - "Code: {}, Enhanced code: {}.{}.{}, Message: {}", - self.code, self.esc[0], self.esc[1], self.esc[2], self.message, - ) - } -} -*/ diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index 699f1033..57e7ac82 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -7,186 +7,142 @@ use super::{Metadata, QueueEnvelope, Status}; use crate::{core::throttle::NewKey, queue::MessageWrapper}; use ahash::AHashSet; -use common::{Server, config::smtp::queue::QueueQuota, expr::functions::ResolveVariable}; -use std::future::Future; -use store::{ - ValueKey, - write::{BatchBuilder, QueueClass, ValueClass}, +use common::{ + Server, + config::smtp::queue::QueueQuota, + expr::{Bump, functions::ResolveVariable}, }; -use trc::QueueEvent; +use std::future::Future; +use store::write::{BatchBuilder, QueueClass, QueueCounter, ValueClass}; use utils::DomainPart; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QuotaReservation { + Size { key: u128, id: u64, max: u64 }, + Count { key: u128, id: u64, max: u64 }, +} + pub trait HasQueueQuota: Sync + Send { - fn has_quota( + fn queue_quotas( &self, - message: &mut MessageWrapper, - ) -> impl Future>> + Send; - fn check_quota<'x>( - &'x self, - quota: &'x QueueQuota, - envelope: &impl ResolveVariable, - size: u64, - id: u64, - refs: &mut Vec, - session_id: u64, - ) -> impl Future + Send; + message: &MessageWrapper, + ) -> impl Future> + Send; } impl HasQueueQuota for Server { - async fn has_quota(&self, message: &mut MessageWrapper) -> Option> { - let mut quota_keys = Vec::new(); + async fn queue_quotas(&self, message: &MessageWrapper) -> Vec { + let quotas = &self.core.smtp.queue.quota; + let mut reservations = Vec::new(); + let mut arena = Bump::new(); - if !self.core.smtp.queue.quota.sender.is_empty() { - for quota in &self.core.smtp.queue.quota.sender { - if !self - .check_quota( - quota, - &message.message, - message.message.size, - 0, - &mut quota_keys, - message.span_id, - ) - .await - { - trc::event!( - Queue(QueueEvent::QuotaExceeded), - SpanId = message.span_id, - Id = quota.id.to_string(), - Type = "Sender" - ); - - return None; - } + for quota in "as.sender { + if matches_quota(self, quota, &message.message, &mut arena, message.span_id).await { + push_reservations(quota, &message.message, 0, &mut reservations); } } - if !self.core.smtp.queue.quota.rcpt_domain.is_empty() { + if !quotas.rcpt_domain.is_empty() { let mut seen_domains = AHashSet::new(); - for quota in &self.core.smtp.queue.quota.rcpt_domain { + for quota in "as.rcpt_domain { + seen_domains.clear(); for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() { - if seen_domains.insert(rcpt.address.domain_part()) - && !self - .check_quota( + if seen_domains.insert(rcpt.address.domain_part()) { + let envelope = QueueEnvelope::new(&message.message, rcpt); + if matches_quota(self, quota, &envelope, &mut arena, message.span_id).await + { + push_reservations( quota, - &QueueEnvelope::new(&message.message, rcpt), - message.message.size, - ((rcpt_idx + 1) << 32) as u64, - &mut quota_keys, - message.span_id, - ) - .await - { - trc::event!( - Queue(QueueEvent::QuotaExceeded), - SpanId = message.span_id, - Id = quota.id.to_string(), - Type = "Domain" - ); - - return None; + &envelope, + ((rcpt_idx + 1) as u64) << 32, + &mut reservations, + ); + } } } } } - for quota in &self.core.smtp.queue.quota.rcpt { + for quota in "as.rcpt { for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() { - if !self - .check_quota( - quota, - &QueueEnvelope::new(&message.message, rcpt), - message.message.size, - (rcpt_idx + 1) as u64, - &mut quota_keys, - message.span_id, - ) - .await - { - trc::event!( - Queue(QueueEvent::QuotaExceeded), - SpanId = message.span_id, - Id = quota.id.to_string(), - Type = "Recipient" - ); - - return None; + let envelope = QueueEnvelope::new(&message.message, rcpt); + if matches_quota(self, quota, &envelope, &mut arena, message.span_id).await { + push_reservations(quota, &envelope, (rcpt_idx + 1) as u64, &mut reservations); } } } - Some(quota_keys) + reservations + } +} + +async fn matches_quota( + server: &Server, + quota: &QueueQuota, + envelope: &impl ResolveVariable, + arena: &mut Bump, + session_id: u64, +) -> bool { + if quota.expr.is_empty() { + return true; + } + arena.reset(); + server + .eval_if("a.expr, envelope, arena, session_id) + .await + .unwrap_or(false) +} + +fn push_reservations( + quota: &QueueQuota, + envelope: &impl ResolveVariable, + id: u64, + reservations: &mut Vec, +) { + let key = quota.new_key(envelope, "").quota_key(); + if let Some(max) = quota.size { + reservations.push(QuotaReservation::Size { key, id, max }); + } + if let Some(max) = quota.messages { + reservations.push(QuotaReservation::Count { key, id, max }); + } +} + +impl QuotaReservation { + pub fn metadata(&self) -> Metadata { + match *self { + QuotaReservation::Size { key, id, .. } => Metadata::QueueSize { key, id }, + QuotaReservation::Count { key, id, .. } => Metadata::QueueCount { key, id }, + } } - async fn check_quota<'x>( - &'x self, - quota: &'x QueueQuota, - envelope: &impl ResolveVariable, - size: u64, - id: u64, - refs: &mut Vec, - session_id: u64, - ) -> bool { - if quota.expr.is_empty() - || self - .eval_if("a.expr, envelope, session_id) - .await - .unwrap_or(false) - { - let key = quota.new_key(envelope, ""); - if let Some(max_size) = quota.size { - let used_size = self - .core - .storage - .data - .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize( - key.as_ref().to_vec(), - )))) - .await - .unwrap_or(0) as u64; - if used_size + size > max_size { - return false; - } else { - refs.push(Metadata::QueueSize { - key: key.as_ref().into(), - id, - }); - } + pub fn reserve(&self, batch: &mut BatchBuilder, size: u64) { + match *self { + QuotaReservation::Size { key, max, .. } => { + let key = key.into(); + batch.limit_queue_counter(QueueCounter::Size(key), max).add( + ValueClass::Queue(QueueClass::QuotaSize(key)), + i64::try_from(size).unwrap_or(i64::MAX), + ); } - - if let Some(max_messages) = quota.messages { - let total_messages = self - .core - .storage - .data - .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount( - key.as_ref().to_vec(), - )))) - .await - .unwrap_or(0) as u64; - if total_messages + 1 > max_messages { - return false; - } else { - refs.push(Metadata::QueueCount { - key: key.as_ref().into(), - id, - }); - } + QuotaReservation::Count { key, max, .. } => { + let key = key.into(); + batch + .limit_queue_counter(QueueCounter::Count(key), max) + .add(ValueClass::Queue(QueueClass::QuotaCount(key)), 1); } } - true } } impl MessageWrapper { - pub fn release_quota(&mut self, batch: &mut BatchBuilder) { + pub fn release_quota(&mut self, batch: &mut BatchBuilder) -> Vec { if !self.message.metadata.iter().any(|metadata| { matches!( metadata, Metadata::QueueSize { .. } | Metadata::QueueCount { .. } ) }) { - return; + return Vec::new(); } let mut quota_ids = Vec::with_capacity(self.message.recipients.len()); @@ -204,18 +160,16 @@ impl MessageWrapper { } if !quota_ids.is_empty() { + quota_ids.sort_unstable(); let mut metadata = Vec::new(); for entry in std::mem::take(&mut self.message.metadata) { match entry { - Metadata::QueueCount { id, key } if quota_ids.contains(&id) => { - batch.add( - ValueClass::Queue(QueueClass::QuotaCount(key.into_vec())), - -1, - ); + Metadata::QueueCount { id, key } if quota_ids.binary_search(&id).is_ok() => { + batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.into())), -1); } - Metadata::QueueSize { id, key } if quota_ids.contains(&id) => { + Metadata::QueueSize { id, key } if quota_ids.binary_search(&id).is_ok() => { batch.add( - ValueClass::Queue(QueueClass::QuotaSize(key.into_vec())), + ValueClass::Queue(QueueClass::QuotaSize(key.into())), -(self.message.size as i64), ); } @@ -226,5 +180,7 @@ impl MessageWrapper { } self.message.metadata = metadata.into_boxed_slice(); } + + quota_ids } } diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index c3e80ce3..3e7b9fca 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -6,39 +6,36 @@ use super::{ ArchivedMessage, ArchivedStatus, Message, MessageSource, Metadata, QueueEnvelope, QueueId, - QueuedMessage, Recipient, Schedule, Status, + Recipient, Schedule, eval_strategy, }; -use crate::inbound::dkim::DkimSign; +use crate::inbound::dkim::{DkimSign, SignedMessage}; use crate::queue::MessageWrapper; -use crate::queue::manager::{Queue, QueueStats}; +use crate::queue::quota::QuotaReservation; use ahash::{AHashMap, AHashSet}; use common::config::smtp::auth::DkimSigners; use common::config::smtp::queue::{ArchivedQueueExpiry, QueueName}; -use common::ipc::{BroadcastEvent, QueueEvent}; +use common::ipc::{BroadcastEvent, QueueEvent, QueueRefresh, QueuedEvent, QueuedEvents}; use common::network::RcptResolution; -use common::{KV_LOCK_QUEUE_MESSAGE, Server}; -use mail_auth::AuthenticatedMessage; +use common::{KV_LOCK_QUEUE_MESSAGE, Server, expr::Bump}; +use compact_str::{CompactString, ToCompactString}; +use mail_auth::{AuthenticatedMessage, dkim2::Dkim2Output}; use registry::schema::prelude::{ObjectType, Property}; use registry::schema::structs::SpamTrainingSample; use registry::types::datetime::UTCDateTime; use registry::types::id::ObjectId; use registry::types::{EnumImpl, ObjectImpl}; use std::borrow::Cow; -use std::collections::hash_map::Entry; use std::future::Future; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use std::time::SystemTime; -use store::write::key::DeserializeBigEndian; -use store::write::serialize::rkyv_deserialize; use store::write::{ - AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, MergeResult, Params, - QueueClass, RegistryClass, ValueClass, now, + Archive, ArchiveBytes, ArchiveVersion, Archiver, AssignedIds, BatchBuilder, BlobLink, BlobOp, + MergeResult, QueueClass, RegistryClass, ValueClass, assert::AssertValue, blob::BlobLinkValue, + now, }; -use store::{ - Deserialize, IterateParams, Serialize, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, -}; -use trc::{AddContext, ServerEvent, SpamEvent}; +use store::{Deserialize, Serialize, SerializeInfallible, ValueKey}; +use trc::{AddContext, EventType, LimitEvent, ServerEvent, SpamEvent}; use types::blob::BlobId; use types::blob_hash::BlobHash; use utils::DomainPart; @@ -46,13 +43,27 @@ use utils::DomainPart; pub const LOCK_EXPIRY: u64 = 10 * 60; // 10 minutes pub const QUEUE_REFRESH: u64 = 5 * 60; // 5 minutes pub const DSN_RETRY: u64 = 5 * 60; // 5 minutes -pub(crate) const INFINITE_LOCK: u64 = 60 * 60 * 24 * 365; // 1 year -const CANDIDATE_OVERSCAN: usize = 4; -const MAX_PREALLOCATED_CANDIDATES: usize = 1024; -pub struct QueuedMessages { - pub messages: Vec, - pub next_refresh: u64, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum QueueOutcome { + Queued, + QuotaExceeded, + Failed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RegistryWrite { + Written, + Locked, + Changed, + Failed, +} + +struct LockedWrite { + queue_id: QueueId, + span_id: u64, + queues: Vec, + batch: BatchBuilder, } pub trait SmtpSpool: Sync + Send { @@ -63,8 +74,6 @@ pub trait SmtpSpool: Sync + Send { span_id: u64, ) -> MessageWrapper; - fn next_event(&self, queue: &mut Queue) -> impl Future + Send; - fn try_lock_event( &self, queue_id: QueueId, @@ -81,12 +90,18 @@ pub trait SmtpSpool: Sync + Send { &self, id: QueueId, queue_name: QueueName, - ) -> impl Future> + Send; + ) -> impl Future>> + Send; + + fn read_message_versioned( + &self, + id: QueueId, + queue_name: QueueName, + ) -> impl Future> + Send; fn read_message_archive( &self, id: QueueId, - ) -> impl Future>>> + Send; + ) -> impl Future>>> + Send; } impl SmtpSpool for Server { @@ -121,141 +136,6 @@ impl SmtpSpool for Server { } } - async fn next_event(&self, queue: &mut Queue) -> QueuedMessages { - let now = now(); - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: queue.scan_from, - queue_id: 0, - queue_name: [0; 8], - }, - ))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: now + QUEUE_REFRESH, - queue_id: u64::MAX, - queue_name: [u8::MAX; 8], - }, - ))); - - // Over-sample the free delivery slots so the shuffle still has a pool to pick from - let mut total_budget: usize = 0; - for stats in queue.stats.values_mut() { - stats.budget = stats - .max_in_flight - .saturating_sub(stats.in_flight) - .saturating_mul(CANDIDATE_OVERSCAN); - total_budget = total_budget.saturating_add(stats.budget); - } - - let mut events = QueuedMessages { - messages: Vec::with_capacity(std::cmp::min(total_budget, MAX_PREALLOCATED_CANDIDATES)), - next_refresh: now + QUEUE_REFRESH, - }; - let mut next_scan_from = u64::MAX; - let mut scan_ceiling = u64::MAX; - - queue.locked_revision += 1; - let result = self - .store() - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let due = key.deserialize_be_u64(0)?; - - if due > now { - if due < events.next_refresh { - events.next_refresh = due; - } - if due < next_scan_from { - next_scan_from = due; - } - return Ok(false); - } - - let queue_id = key.deserialize_be_u64(U64_LEN)?; - let queue_name = key - .get(U64_LEN + U64_LEN..) - .and_then(QueueName::from_bytes) - .ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Key, key) - })?; - - // Refreshed even when not dispatched, or a stale revision would evict it - let is_locked = match queue.locked.get_mut(&(queue_id, queue_name)) { - Some(locked) => { - locked.revision = queue.locked_revision; - locked.due = due; - if locked.expires > now { - if locked.expires < events.next_refresh { - events.next_refresh = locked.expires; - } - true - } else { - false - } - } - None => false, - }; - - if !is_locked { - let stats = match queue.stats.entry(queue_name) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let threads = - self.get_virtual_queue_or_default(&queue_name).threads; - let mut stats = QueueStats::new(threads); - stats.budget = threads.saturating_mul(CANDIDATE_OVERSCAN); - total_budget = total_budget.saturating_add(stats.budget); - entry.insert(stats) - } - }; - - if stats.budget > 0 { - stats.budget -= 1; - total_budget -= 1; - events.messages.push(QueuedMessage { - due, - queue_id, - queue_name, - }); - } else if due < next_scan_from { - next_scan_from = due; - } - - // Everything from here on is left for the next scan - if total_budget == 0 { - scan_ceiling = due; - if due < next_scan_from { - next_scan_from = due; - } - return Ok(false); - } - } - - Ok(true) - }, - ) - .await; - - if let Err(err) = result { - trc::error!( - err.details("Failed to read queue.") - .caused_by(trc::location!()) - ); - - queue.scan_from = 0; - queue.scan_ceiling = 0; - } else { - queue.scan_from = std::cmp::min(next_scan_from, now); - queue.scan_ceiling = scan_ceiling; - } - - events - } - async fn try_lock_event(&self, queue_id: QueueId, queue_name: QueueName) -> bool { match self .in_memory_store() @@ -271,7 +151,7 @@ impl SmtpSpool for Server { trc::event!( Queue(trc::QueueEvent::Locked), QueueId = queue_id, - QueueName = queue_name.to_string() + QueueName = queue_name.to_compact_string() ); } result @@ -303,15 +183,32 @@ impl SmtpSpool for Server { &self, queue_id: QueueId, queue_name: QueueName, - ) -> Option { + ) -> trc::Result> { + match self.read_message_archive(queue_id).await? { + Some(archive) => archive + .deserialize::() + .map(|message| Some(MessageWrapper::new(message, queue_id, queue_name))), + None => Ok(None), + } + } + + async fn read_message_versioned( + &self, + queue_id: QueueId, + queue_name: QueueName, + ) -> Option<(MessageWrapper, ArchiveVersion)> { match self .read_message_archive(queue_id) .await .and_then(|a| match a { - Some(a) => a.deserialize::().map(Some), + Some(a) => a + .deserialize::() + .map(|message| Some((message, a.version))), None => Ok(None), }) { - Ok(Some(message)) => Some(MessageWrapper::new(message, queue_id, queue_name)), + Ok(Some((message, version))) => { + Some((MessageWrapper::new(message, queue_id, queue_name), version)) + } Ok(None) => None, Err(err) => { trc::error!( @@ -327,9 +224,9 @@ impl SmtpSpool for Server { async fn read_message_archive( &self, id: QueueId, - ) -> trc::Result>> { + ) -> trc::Result>> { self.store() - .get_value::>(ValueKey::from(ValueClass::Queue( + .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(id), ))) .await @@ -339,16 +236,134 @@ impl SmtpSpool for Server { impl MessageWrapper { pub fn new(message: Message, queue_id: QueueId, queue_name: QueueName) -> Self { MessageWrapper { - is_multi_queue: message.recipients.iter().any(|rcpt| { - matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) - && rcpt.queue != queue_name - }), + is_multi_queue: message + .recipients + .iter() + .any(|rcpt| rcpt.queue != queue_name), queue_id, queue_name, span_id: 0, message, } } + + fn event_class(&self, queue_name: QueueName, due: u64) -> ValueClass { + ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { + due, + queue_id: self.queue_id, + queue_name: queue_name.into_inner(), + })) + } + +} + +impl LockedWrite { + fn new(queue_id: QueueId, span_id: u64, message: &Message) -> Self { + let mut queues = Vec::with_capacity(1); + for rcpt in &message.recipients { + if !queues.contains(&rcpt.queue) { + queues.push(rcpt.queue); + } + } + + LockedWrite { + queue_id, + span_id, + queues, + batch: BatchBuilder::new(), + } + } + + async fn commit(self, server: &Server) -> RegistryWrite { + let span_id = self.span_id; + tokio::spawn(self.write(server.clone())) + .await + .unwrap_or_else(|err| { + trc::event!( + Server(ServerEvent::ThreadError), + Reason = err.to_string(), + CausedBy = trc::location!(), + SpanId = span_id, + ); + RegistryWrite::Failed + }) + } + + async fn write(mut self, server: Server) -> RegistryWrite { + let mut locked = 0; + let mut status = None; + for queue_name in &self.queues { + match server + .in_memory_store() + .try_lock( + KV_LOCK_QUEUE_MESSAGE, + &lock_id(self.queue_id, *queue_name), + LOCK_EXPIRY, + ) + .await + { + Ok(true) => locked += 1, + Ok(false) => { + status = Some(RegistryWrite::Locked); + break; + } + Err(err) => { + trc::error!( + err.details("Failed to lock queued message.") + .ctx(trc::Key::QueueId, self.queue_id) + .span_id(self.span_id) + .caused_by(trc::location!()) + ); + status = Some(RegistryWrite::Failed); + break; + } + } + } + + let status = match status { + Some(status) => status, + None => RegistryWrite::from_result( + server.store().write_batch(&mut self.batch).await, + self.span_id, + ), + }; + + for queue_name in self.queues.iter().take(locked) { + server.unlock_event(self.queue_id, *queue_name).await; + } + + status + } +} + +impl RegistryWrite { + pub fn failure(self, failed: &'static str) -> Option<&'static str> { + match self { + RegistryWrite::Written => None, + RegistryWrite::Locked => { + Some("The message is being delivered or modified, retry later.") + } + RegistryWrite::Changed => { + Some("The message was modified during the update, retry later.") + } + RegistryWrite::Failed => Some(failed), + } + } + + fn from_result(result: trc::Result, span_id: u64) -> Self { + match result { + Ok(_) => RegistryWrite::Written, + Err(err) if err.is_assertion_failure() => RegistryWrite::Changed, + Err(err) => { + trc::error!( + err.details("Failed to update queued message.") + .span_id(span_id) + .caused_by(trc::location!()) + ); + RegistryWrite::Failed + } + } + } } fn lock_id(queue_id: QueueId, queue_name: QueueName) -> [u8; 16] { @@ -362,8 +377,11 @@ pub(crate) struct QueueParams<'x, 'y> { pub raw_message: &'x [u8], pub raw_headers: Option<&'x [u8]>, pub metadata: Vec, + pub quotas: Vec, pub original_raw_message: Option<&'x [u8]>, - pub original_authenticated_message: Option>, + pub original_authenticated_message: Option<&'x AuthenticatedMessage<'x>>, + pub dkim2_verified: Option<&'x Dkim2Output<'x>>, + pub exploded: bool, pub dkim_signers: Option>, pub session_id: u64, pub server: &'y Server, @@ -372,13 +390,12 @@ pub(crate) struct QueueParams<'x, 'y> { impl MessageWrapper { #[must_use] - pub(crate) async fn queue<'x, 'y>(mut self, mut params: QueueParams<'x, 'y>) -> bool { + pub(crate) async fn queue<'x, 'y>(mut self, mut params: QueueParams<'x, 'y>) -> QueueOutcome { // Add DKIM signatures - let dkim_headers = if params.dkim_signers.is_some() { - params.server.sign_message(&mut self, &mut params).await - } else { - None - }; + let SignedMessage { + headers: dkim_headers, + stripped_message, + } = params.server.sign_message(&mut self, &mut params).await; // Fetch params let QueueParams { @@ -387,30 +404,41 @@ impl MessageWrapper { session_id, server, train_spam, - metadata, + mut metadata, + quotas, .. } = params; let event = self.message.queued_event(); // Write blob + if let Some(stripped_message) = &stripped_message { + self.message.size = self + .message + .size + .saturating_sub(raw_message.len().saturating_sub(stripped_message.len()) as u64); + } + let raw_message = stripped_message.as_deref().unwrap_or(raw_message); let raw_headers = raw_headers.unwrap_or_default(); - let dkim_headers = dkim_headers.as_deref().unwrap_or_default(); let message = if !raw_headers.is_empty() || !dkim_headers.is_empty() { let mut message = Vec::with_capacity(raw_headers.len() + dkim_headers.len() + raw_message.len()); - message.extend_from_slice(dkim_headers); + message.extend_from_slice(&dkim_headers); message.extend_from_slice(raw_headers); message.extend_from_slice(raw_message); Cow::Owned(message) + } else if let Some(stripped_message) = stripped_message { + Cow::Owned(stripped_message) } else { raw_message.into() }; self.message.blob_hash = BlobHash::generate(message.as_ref()); + let blob_size = message.len(); // Update size if self.message.size == 0 { self.message.size = message.len() as u64; } + metadata.extend(quotas.iter().map(QuotaReservation::metadata)); self.message.metadata = metadata.into_boxed_slice(); // Reserve and write blob @@ -424,16 +452,16 @@ impl MessageWrapper { until: reserve_until, }, }, - vec![], + BlobLinkValue::new(blob_size), ); - if let Err(err) = server.store().write(batch.build_all()).await { + if let Err(err) = server.store().write_batch(&mut batch).await { trc::error!( err.details("Failed to write to store.") .span_id(session_id) .caused_by(trc::location!()) ); - return false; + return QueueOutcome::Failed; } if let Err(err) = server .blob_store() @@ -450,7 +478,143 @@ impl MessageWrapper { .caused_by(trc::location!()) ); - return false; + return QueueOutcome::Failed; + } + + + // Write message to queue + let mut batch = BatchBuilder::new(); + + // Reserve quotas + for quota in "as { + quota.reserve(&mut batch, self.message.size); + } + + let mut events = QueuedEvents::new(); + for (queue_name, due) in self.message.next_events() { + batch.set(self.event_class(queue_name, due), Vec::new()); + events.push(QueuedEvent { + queue_name, + due, + queue_id: self.queue_id, + }); + } + + if let Some((is_spam, subject)) = train_spam + && let Some(config) = &server.core.spam.classifier + { + let hold_period = now + config.hold_samples_for; + let sample = SpamTrainingSample { + account_id: None, + blob_id: BlobId::new(self.message.blob_hash.clone(), Default::default()), + delete_after_use: false, + expires_at: UTCDateTime::from_timestamp(hold_period as i64), + from: self.message.return_path.to_string(), + is_spam, + subject, + } + .to_pickled_vec(); + + let object_id = ObjectType::SpamTrainingSample.to_id(); + let item_id = server.inner.data.registry_id_gen.generate(); + batch + .set( + BlobOp::Link { + hash: self.message.blob_hash.clone(), + to: BlobLink::Temporary { until: hold_period }, + }, + BlobLinkValue::new(blob_size).with_object_id(ObjectId::new( + ObjectType::SpamTrainingSample, + item_id.into(), + )), + ) + .set( + ValueClass::Registry(RegistryClass::Item { object_id, item_id }), + sample, + ) + .set( + ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id, + key: (u32::MAX as u64).serialize(), + }), + vec![], + ); + + trc::event!( + Spam(SpamEvent::TrainSampleAdded), + Details = if is_spam { "spam" } else { "ham" }, + Expires = trc::Value::Timestamp(hold_period), + SpanId = self.span_id, + ); + } + + let archiver = Archiver::new(self.message); + let serialized = match archiver.serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.details("Failed to serialize message.") + .span_id(session_id) + .caused_by(trc::location!()) + ); + return QueueOutcome::Failed; + } + }; + self.message = archiver.into_inner(); + + batch + .clear(BlobOp::Link { + hash: self.message.blob_hash.clone(), + to: BlobLink::Temporary { + until: reserve_until, + }, + }) + .set( + BlobOp::Link { + hash: self.message.blob_hash.clone(), + to: BlobLink::Id { id: self.queue_id }, + }, + BlobLinkValue::new(blob_size), + ) + .set( + BlobOp::Commit { + hash: self.message.blob_hash.clone(), + }, + vec![], + ) + .set( + ValueClass::Queue(QueueClass::Message(self.queue_id)), + serialized, + ); + + if let Err(err) = server.store().write_batch(&mut batch).await { + if err.matches(EventType::Limit(LimitEvent::Quota)) { + trc::event!( + Queue(trc::QueueEvent::QuotaExceeded), + SpanId = session_id, + QueueId = self.queue_id, + From = trc::Value::String(self.message.return_path.as_ref().into()), + To = self + .message + .recipients + .iter() + .map(|r| trc::Value::String(r.address.as_ref().into())) + .collect::>(), + CausedBy = err + ); + + return QueueOutcome::QuotaExceeded; + } + + trc::error!( + err.details("Failed to write to store.") + .span_id(session_id) + .caused_by(trc::location!()) + ); + + return QueueOutcome::Failed; } trc::event!( @@ -477,134 +641,18 @@ impl MessageWrapper { Expires = self.message.expires(None).map(trc::Value::Timestamp), ); - // Write message to queue - let mut batch = BatchBuilder::new(); - - // Reserve quotas - for metadata in &self.message.metadata { - match metadata { - Metadata::QueueCount { key, .. } => { - batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.to_vec())), 1); - } - Metadata::QueueSize { key, .. } => { - batch.add( - ValueClass::Queue(QueueClass::QuotaSize(key.to_vec())), - self.message.size as i64, - ); - } - Metadata::Headers { .. } => {} - } - } - - for (queue_name, due) in self.message.next_events() { - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - })), - Vec::new(), - ); - } - - if let Some((is_spam, subject)) = train_spam - && let Some(config) = &server.core.spam.classifier - { - let hold_period = now + config.hold_samples_for; - let sample = SpamTrainingSample { - account_id: None, - blob_id: BlobId::new(self.message.blob_hash.clone(), Default::default()), - delete_after_use: false, - expires_at: UTCDateTime::from_timestamp(hold_period as i64), - from: self.message.return_path.to_string(), - is_spam, - subject, - } - .to_pickled_vec(); - - let object_id = ObjectType::SpamTrainingSample.to_id(); - let item_id = server.inner.data.registry_id_gen.generate(); - batch - .set( - BlobOp::Link { - hash: self.message.blob_hash.clone(), - to: BlobLink::Temporary { until: hold_period }, - }, - ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), - ) - .set( - ValueClass::Registry(RegistryClass::Item { object_id, item_id }), - sample, - ) - .set( - ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId.to_id(), - object_id, - item_id, - key: (u32::MAX as u64).serialize(), - }), - vec![], - ); - - trc::event!( - Spam(SpamEvent::TrainSampleAdded), - Details = if is_spam { "spam" } else { "ham" }, - Expires = trc::Value::Timestamp(hold_period), - SpanId = self.span_id, - ); - } - - batch - .clear(BlobOp::Link { - hash: self.message.blob_hash.clone(), - to: BlobLink::Temporary { - until: reserve_until, - }, - }) - .set( - BlobOp::Link { - hash: self.message.blob_hash.clone(), - to: BlobLink::Id { id: self.queue_id }, - }, - vec![], - ) - .set( - BlobOp::Commit { - hash: self.message.blob_hash.clone(), - }, - vec![], - ) - .set( - ValueClass::Queue(QueueClass::Message(self.queue_id)), - match Archiver::new(self.message).serialize() { - Ok(data) => data, - Err(err) => { - trc::error!( - err.details("Failed to serialize message.") - .span_id(session_id) - .caused_by(trc::location!()) - ); - return false; - } - }, - ); - - if let Err(err) = server.store().write(batch.build_all()).await { - trc::error!( - err.details("Failed to write to store.") - .span_id(session_id) - .caused_by(trc::location!()) - ); - - return false; - } - // Queue the message + let queues = server + .inner + .ipc + .broadcast_tx + .is_some() + .then(|| events.iter().map(|event| event.queue_name).collect()); if server .inner .ipc .queue_tx - .send(QueueEvent::Refresh) + .send(QueueEvent::Refresh(QueueRefresh::Queued(events))) .await .is_err() { @@ -616,9 +664,13 @@ impl MessageWrapper { ); } - server.cluster_broadcast(BroadcastEvent::QueueRefresh).await; + if let Some(queues) = queues { + server + .cluster_broadcast(BroadcastEvent::QueueRefresh(queues)) + .await; + } - true + QueueOutcome::Queued } pub async fn expand_and_add_recipient(&mut self, rcpt: impl AsRef, server: &Server) { @@ -643,7 +695,7 @@ impl MessageWrapper { err.span_id(self.span_id) .caused_by(trc::location!()) .details("Failed to resolve recipient.") - .ctx(trc::Key::To, rcpt.to_string()) + .ctx(trc::Key::To, CompactString::from(rcpt)) ); self.add_expanded_recipient(rcpt, server).await; } @@ -652,17 +704,16 @@ impl MessageWrapper { pub async fn add_expanded_recipient(&mut self, rcpt: impl AsRef, server: &Server) { self.message.recipients.push(Recipient::new(rcpt.as_ref())); - let queue = server.get_queue_or_default( - &server - .eval_if::( - &server.core.smtp.queue.queue, - &QueueEnvelope::new(&self.message, self.message.recipients.last().unwrap()), - self.span_id, - ) - .await - .unwrap_or_else(|| "default".to_string()), + let mut arena = Bump::new(); + let queue = eval_strategy( + server, + &server.core.smtp.queue.queue, + &QueueEnvelope::new(&self.message, self.message.recipients.last().unwrap()), + &mut arena, self.span_id, - ); + Server::get_queue_or_default, + ) + .await; // Update expiration let recipient = self.message.recipients.last_mut().unwrap(); @@ -679,17 +730,11 @@ impl MessageWrapper { ) -> bool { // Release quota for completed deliveries let mut batch = BatchBuilder::new(); - self.release_quota(&mut batch); + let released_quotas = self.release_quota(&mut batch); // Update message queue if let Some(prev_event) = prev_event { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: prev_event, - queue_id: self.queue_id, - queue_name: self.queue_name.into_inner(), - }, - ))); + batch.clear(self.event_class(self.queue_name, prev_event)); } let mut next_events = self.message.next_events(); if let Some(retry_at) = retry_at { @@ -697,66 +742,51 @@ impl MessageWrapper { *due = std::cmp::min(*due, retry_at); } for (queue_name, due) in next_events { - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - })), - Vec::new(), - ); + batch.set(self.event_class(queue_name, due), Vec::new()); } - let message_bytes = match Archiver::new(self.message).serialize() { - Ok(data) => data, - Err(err) => { - trc::error!( - err.details("Failed to serialize message.") - .span_id(self.span_id) - .caused_by(trc::location!()) - ); - return false; - } - }; if self.is_multi_queue { + let queue_id = self.queue_id; + let queue_name = self.queue_name; + let new_message = self.message; + batch.merge_fnc( - ValueClass::Queue(QueueClass::Message(self.queue_id)), - Params::with_capacity(3) - .with_u64(self.queue_id) - .with_bytes(self.queue_name.into_inner().to_vec()) - .with_bytes(message_bytes), - |params, _, bytes| { - let mut cur_message = as Deserialize>::deserialize( + ValueClass::Queue(QueueClass::Message(queue_id)), + move |_, bytes| { + let mut cur_message = as Deserialize>::deserialize( bytes.ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .details("Message no longer exists.") .caused_by(trc::location!()) - .ctx(trc::Key::QueueId, params.u64(0)) + .ctx(trc::Key::QueueId, queue_id) })?, ) .and_then(|archive| archive.deserialize::()) .caused_by(trc::location!())?; - let new_message_ = - as Deserialize>::deserialize(params.bytes(2)) - .caused_by(trc::location!())?; - let new_message = new_message_ - .unarchive::() - .caused_by(trc::location!())?; - - if cur_message.blob_hash.as_slice() == new_message.blob_hash.0.as_slice() + if cur_message.blob_hash.as_slice() == new_message.blob_hash.as_slice() && cur_message.recipients.len() == new_message.recipients.len() { - let queue_name = params.bytes(1); - for (rcpt_idx, rcpt) in new_message + for (cur_rcpt, rcpt) in cur_message .recipients - .iter() - .enumerate() - .filter(|(_, rcpt)| rcpt.queue.as_slice() == queue_name) + .iter_mut() + .zip(new_message.recipients.iter()) + .filter(|(cur_rcpt, _)| cur_rcpt.queue == queue_name) { - cur_message.recipients[rcpt_idx] = - rkyv_deserialize(rcpt).caused_by(trc::location!())?; + *cur_rcpt = rcpt.clone(); + } + + if !released_quotas.is_empty() { + let mut metadata = std::mem::take(&mut cur_message.metadata).into_vec(); + metadata.retain(|entry| match entry { + Metadata::QueueCount { id, .. } + | Metadata::QueueSize { id, .. } => { + released_quotas.binary_search(id).is_err() + } + Metadata::Headers { .. } | Metadata::Submission { .. } => true, + }); + cur_message.metadata = metadata.into_boxed_slice(); } Archiver::new(cur_message) @@ -768,18 +798,30 @@ impl MessageWrapper { .into_err() .details("Message blob hash or recipient count mismatch.") .caused_by(trc::location!()) - .ctx(trc::Key::QueueId, params.u64(0))) + .ctx(trc::Key::QueueId, queue_id)) } }, ); } else { + let message_bytes = match Archiver::new(self.message).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.details("Failed to serialize message.") + .span_id(self.span_id) + .caused_by(trc::location!()) + ); + return false; + } + }; + batch.set( ValueClass::Queue(QueueClass::Message(self.queue_id)), message_bytes, ); } - if let Err(err) = server.store().write(batch.build_all()).await { + if let Err(err) = server.store().write_batch(&mut batch).await { trc::error!( err.details("Failed to save changes.") .span_id(self.span_id) @@ -793,40 +835,27 @@ impl MessageWrapper { pub async fn remove(self, server: &Server, prev_event: Option) -> bool { let mut batch = BatchBuilder::new(); - - if let Some(prev_event) = prev_event { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due: prev_event, - queue_id: self.queue_id, - queue_name: self.queue_name.into_inner(), - }, - ))); - } else { - for (queue_name, due) in self.message.next_events() { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - }, - ))); - } + if self.is_multi_queue { + batch.assert_value( + ValueClass::Queue(QueueClass::Message(self.queue_id)), + AssertValue::Some, + ); } + self.clear_events(&mut batch, prev_event); // Release all quotas - for metadata in self.message.metadata { + for metadata in &self.message.metadata { match metadata { Metadata::QueueCount { key, .. } => { - batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.to_vec())), -1); + batch.add(ValueClass::Queue(QueueClass::QuotaCount((*key).into())), -1); } Metadata::QueueSize { key, .. } => { batch.add( - ValueClass::Queue(QueueClass::QuotaSize(key.to_vec())), + ValueClass::Queue(QueueClass::QuotaSize((*key).into())), -(self.message.size as i64), ); } - Metadata::Headers { .. } => {} + Metadata::Headers { .. } | Metadata::Submission { .. } => {} } } @@ -836,8 +865,18 @@ impl MessageWrapper { to: BlobLink::Id { id: self.queue_id }, }) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); + self.write_delivery_results(&mut batch); - if let Err(err) = server.store().write(batch.build_all()).await { + let result = match server.store().write_batch(&mut batch).await { + Err(err) if err.is_assertion_failure() => { + let mut batch = BatchBuilder::new(); + self.clear_events(&mut batch, prev_event); + server.store().write_batch(&mut batch).await + } + result => result, + }; + + if let Err(err) = result { trc::error!( err.details("Failed to write to update queue.") .span_id(self.span_id) @@ -849,92 +888,74 @@ impl MessageWrapper { } } + fn clear_events(&self, batch: &mut BatchBuilder, prev_event: Option) { + if let Some(prev_event) = prev_event { + batch.clear(self.event_class(self.queue_name, prev_event)); + } else { + for (queue_name, due) in self.message.next_events() { + batch.clear(self.event_class(queue_name, due)); + } + } + } + pub async fn save_registry_changes( mut self, server: &Server, prev_events: AHashMap, modified_rcpts: AHashSet, - ) -> bool { - let mut batch = BatchBuilder::new(); - self.release_quota(&mut batch); + ) -> RegistryWrite { + let mut write = LockedWrite::new(self.queue_id, self.span_id, &self.message); + let batch = &mut write.batch; + let released_quotas = self.release_quota(batch); for (queue_name, due) in prev_events { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - }, - ))); + batch.clear(self.event_class(queue_name, due)); } for (queue_name, due) in self.message.next_events() { - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - })), - Vec::new(), - ); + batch.set(self.event_class(queue_name, due), Vec::new()); } - let message_bytes = match Archiver::new(self.message).serialize() { - Ok(data) => data, - Err(err) => { - trc::error!( - err.details("Failed to serialize message.") - .span_id(self.span_id) - .caused_by(trc::location!()) - ); - return false; - } - }; - - let mut modified_bytes = Vec::with_capacity(modified_rcpts.len() * U32_LEN); - for idx in modified_rcpts { - modified_bytes.extend_from_slice(&(idx as u32).to_be_bytes()); - } + let queue_id = self.queue_id; + let new_message = self.message; batch.merge_fnc( - ValueClass::Queue(QueueClass::Message(self.queue_id)), - Params::with_capacity(3) - .with_u64(self.queue_id) - .with_bytes(modified_bytes) - .with_bytes(message_bytes), - |params, _, bytes| { - let mut cur_message = as Deserialize>::deserialize( + ValueClass::Queue(QueueClass::Message(queue_id)), + move |_, bytes| { + let mut cur_message = as Deserialize>::deserialize( bytes.ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .details("Message no longer exists.") .caused_by(trc::location!()) - .ctx(trc::Key::QueueId, params.u64(0)) + .ctx(trc::Key::QueueId, queue_id) })?, ) .and_then(|archive| archive.deserialize::()) .caused_by(trc::location!())?; - let new_message_ = - as Deserialize>::deserialize(params.bytes(2)) - .caused_by(trc::location!())?; - let new_message = new_message_ - .unarchive::() - .caused_by(trc::location!())?; - - if cur_message.blob_hash.as_slice() == new_message.blob_hash.0.as_slice() + if cur_message.blob_hash.as_slice() == new_message.blob_hash.as_slice() && cur_message.recipients.len() == new_message.recipients.len() { - cur_message.priority = new_message.priority.to_native(); - cur_message.env_id = new_message.env_id.as_ref().map(|v| v.as_ref().into()); + cur_message.priority = new_message.priority; + cur_message.env_id = new_message.env_id.clone(); - for idx in params.bytes(1).as_chunks::().0 { - let rcpt_idx = u32::from_be_bytes(*idx) as usize; + for &rcpt_idx in &modified_rcpts { if let Some(rcpt) = new_message.recipients.get(rcpt_idx) { - cur_message.recipients[rcpt_idx] = - rkyv_deserialize(rcpt).caused_by(trc::location!())?; + cur_message.recipients[rcpt_idx] = rcpt.clone(); } } + if !released_quotas.is_empty() { + let mut metadata = std::mem::take(&mut cur_message.metadata).into_vec(); + metadata.retain(|entry| match entry { + Metadata::QueueCount { id, .. } | Metadata::QueueSize { id, .. } => { + released_quotas.binary_search(id).is_err() + } + Metadata::Headers { .. } | Metadata::Submission { .. } => true, + }); + cur_message.metadata = metadata.into_boxed_slice(); + } + Archiver::new(cur_message) .serialize() .caused_by(trc::location!()) @@ -944,52 +965,44 @@ impl MessageWrapper { .into_err() .details("Message blob hash or recipient count mismatch.") .caused_by(trc::location!()) - .ctx(trc::Key::QueueId, params.u64(0))) + .ctx(trc::Key::QueueId, queue_id)) } }, ); - if let Err(err) = server.store().write(batch.build_all()).await { - trc::error!( - err.details("Failed to save changes.") - .span_id(self.span_id) - .caused_by(trc::location!()) - ); - false - } else { - true - } + write.commit(server).await } pub async fn remove_registry( self, server: &Server, prev_events: AHashMap, - ) -> bool { - let mut batch = BatchBuilder::new(); + version: ArchiveVersion, + ) -> RegistryWrite { + let mut write = LockedWrite::new(self.queue_id, self.span_id, &self.message); + let batch = &mut write.batch; + batch.assert_value( + ValueClass::Queue(QueueClass::Message(self.queue_id)), + AssertValue::Archive(version), + ); for (queue_name, due) in prev_events { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent( - store::write::QueueEvent { - due, - queue_id: self.queue_id, - queue_name: queue_name.into_inner(), - }, - ))); + batch.clear(self.event_class(queue_name, due)); } + self.write_delivery_results(batch); for metadata in self.message.metadata { match metadata { Metadata::QueueCount { key, .. } => { - batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.to_vec())), -1); + batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.into())), -1); } Metadata::QueueSize { key, .. } => { batch.add( - ValueClass::Queue(QueueClass::QuotaSize(key.to_vec())), + ValueClass::Queue(QueueClass::QuotaSize(key.into())), -(self.message.size as i64), ); } - Metadata::Headers { .. } => {} + Metadata::Headers { .. } | Metadata::Submission { .. } => {} } } @@ -1000,16 +1013,7 @@ impl MessageWrapper { }) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); - if let Err(err) = server.store().write(batch.build_all()).await { - trc::error!( - err.details("Failed to write to update queue.") - .span_id(self.span_id) - .caused_by(trc::location!()) - ); - false - } else { - true - } + write.commit(server).await } pub fn has_domain(&self, domains: &[String]) -> bool { @@ -1025,16 +1029,6 @@ impl MessageWrapper { } impl ArchivedMessage { - pub fn has_domain(&self, domains: &AHashSet) -> bool { - self.recipients.iter().any(|r| { - let domain = r.address.domain_part(); - domains.contains(domain) - }) || self - .return_path - .rsplit_once('@') - .is_some_and(|(_, domain)| domains.contains(domain)) - } - pub fn next_delivery_event(&self, queue: Option) -> Option { let mut next_delivery = None; @@ -1120,7 +1114,10 @@ impl<'x, 'y> QueueParams<'x, 'y> { train_spam: None, original_raw_message: None, original_authenticated_message: None, + dkim2_verified: None, + exploded: false, metadata: Vec::new(), + quotas: Vec::new(), } } @@ -1131,12 +1128,22 @@ impl<'x, 'y> QueueParams<'x, 'y> { pub fn with_original_authenticated_message( mut self, - authenticated_message: AuthenticatedMessage<'x>, + authenticated_message: &'x AuthenticatedMessage<'x>, ) -> Self { self.original_authenticated_message = Some(authenticated_message); self } + pub fn with_dkim2_verified(mut self, verified: Option<&'x Dkim2Output<'x>>) -> Self { + self.dkim2_verified = verified; + self + } + + pub fn with_exploded(mut self, exploded: bool) -> Self { + self.exploded = exploded; + self + } + pub fn with_original_raw_message(mut self, raw_message: &'x [u8]) -> Self { self.original_raw_message = Some(raw_message); self @@ -1157,8 +1164,13 @@ impl<'x, 'y> QueueParams<'x, 'y> { self } - pub fn with_metadata(mut self, metadata: Vec) -> Self { - self.metadata = metadata; + pub fn with_quotas(mut self, quotas: Vec) -> Self { + self.quotas = quotas; + self + } + + pub fn with_metadata(mut self, metadata: Option) -> Self { + self.metadata.extend(metadata); self } } diff --git a/crates/smtp/src/queue/submission.rs b/crates/smtp/src/queue/submission.rs new file mode 100644 index 00000000..bf0a697d --- /dev/null +++ b/crates/smtp/src/queue/submission.rs @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ArchivedError, ArchivedErrorDetails, ArchivedRecipient, ArchivedStatus, Error, ErrorDetails, + MessageWrapper, Recipient, Status, +}; +use email::submission::{Delivered, DeliveryResult, DeliveryResults, DeliveryStatus, QUEUED_REPLY}; +use store::{ + Serialize, + write::{Archiver, BatchBuilder}, +}; + +const REMOVED_REPLY: &str = "Removed from queue"; + +impl MessageWrapper { + pub(crate) fn write_delivery_results(&self, batch: &mut BatchBuilder) { + let Some(account_id) = self.message.submission_account_id() else { + return; + }; + let results = DeliveryResults( + self.message + .recipients + .iter() + .map(Recipient::delivery_result) + .collect(), + ); + match Archiver::new(results).serialize() { + Ok(value) => { + DeliveryResults::write(batch, account_id, self.queue_id, value); + } + Err(err) => { + trc::error!( + err.details("Failed to serialize delivery results.") + .span_id(self.span_id) + .account_id(account_id) + .caused_by(trc::location!()) + ); + } + } + } +} + +impl Recipient { + fn delivery_result(&self) -> DeliveryResult { + let (delivered, smtp_reply) = match &self.status { + Status::Scheduled => (Delivered::No, REMOVED_REPLY.into()), + Status::Completed(reply) => ( + Delivered::Yes, + format_response( + reply.response.code, + reply.response.esc, + &reply.response.message, + ), + ), + Status::TemporaryFailure(err) | Status::PermanentFailure(err) => { + (Delivered::No, format_error(err)) + } + }; + DeliveryResult { + address: self.address.clone(), + delivered, + smtp_reply: smtp_reply.into_boxed_str(), + } + } +} + +impl ArchivedRecipient { + pub fn delivery_status(&self) -> DeliveryStatus { + let (delivered, smtp_reply) = match &self.status { + ArchivedStatus::Scheduled => (Delivered::Queued, QUEUED_REPLY.into()), + ArchivedStatus::Completed(reply) => ( + Delivered::Yes, + format_response( + reply.response.code.into(), + reply.response.esc, + &reply.response.message, + ), + ), + ArchivedStatus::TemporaryFailure(err) => { + (Delivered::Queued, format_archived_error(err)) + } + ArchivedStatus::PermanentFailure(err) => (Delivered::No, format_archived_error(err)), + }; + DeliveryStatus { + smtp_reply, + delivered, + displayed: false, + } + } +} + +fn format_error(err: &ErrorDetails) -> String { + match &err.details { + Error::UnexpectedResponse(response) => format_response( + response.response.code, + response.response.esc, + &response.response.message, + ), + Error::DnsError(details) + | Error::Io(details) + | Error::ConnectionError(details) + | Error::TlsError(details) + | Error::DaneError(details) + | Error::MtaStsError(details) => details.to_string(), + Error::RateLimited => "Rate limited".into(), + Error::ConcurrencyLimited => "Concurrency limited".into(), + } +} + +fn format_archived_error(err: &ArchivedErrorDetails) -> String { + match &err.details { + ArchivedError::UnexpectedResponse(response) => format_response( + response.response.code.into(), + response.response.esc, + &response.response.message, + ), + ArchivedError::DnsError(details) + | ArchivedError::Io(details) + | ArchivedError::ConnectionError(details) + | ArchivedError::TlsError(details) + | ArchivedError::DaneError(details) + | ArchivedError::MtaStsError(details) => details.to_string(), + ArchivedError::RateLimited => "Rate limited".into(), + ArchivedError::ConcurrencyLimited => "Concurrency limited".into(), + } +} + +fn format_response(code: u16, [a, b, c]: [u8; 3], message: &str) -> String { + let mut reply = format!("Code: {code}, Enhanced code: {a}.{b}.{c}, Message: "); + reply.extend(message.chars().map(|ch| if ch == '\n' { ' ' } else { ch })); + reply +} diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs index 3e23f903..c5469241 100644 --- a/crates/smtp/src/queue/throttle.rs +++ b/crates/smtp/src/queue/throttle.rs @@ -6,8 +6,11 @@ use crate::core::throttle::NewKey; use common::{ - KV_RATE_LIMIT_SMTP, Server, config::smtp::QueueRateLimiter, expr::functions::ResolveVariable, + KV_RATE_LIMIT_SMTP, Server, + config::smtp::QueueRateLimiter, + expr::{Bump, functions::ResolveVariable}, }; +use compact_str::ToCompactString; use std::future::Future; use store::write::now; @@ -27,9 +30,10 @@ impl IsAllowed for Server { envelope: &impl ResolveVariable, session_id: u64, ) -> Result<(), u64> { + let mut arena = Bump::new(); if throttle.expr.is_empty() || self - .eval_if(&throttle.expr, envelope, session_id) + .eval_if(&throttle.expr, envelope, &mut arena, session_id) .await .unwrap_or(false) { @@ -37,18 +41,16 @@ impl IsAllowed for Server { match self .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.as_ref(), &throttle.rate, false) + .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.as_ref(), &throttle.rate) .await { Ok(Some(next_refill)) => { trc::event!( Queue(trc::QueueEvent::RateLimitExceeded), SpanId = session_id, - Id = throttle.id.to_string(), - Limit = vec![ - trc::Value::from(throttle.rate.count), - trc::Value::from(throttle.rate.period.into_inner()) - ], + Name = throttle.id.to_compact_string(), + Limit = throttle.rate.count, + Period = throttle.rate.period.into_inner(), ); return Err(now() + next_refill); diff --git a/crates/smtp/src/queue/walk.rs b/crates/smtp/src/queue/walk.rs new file mode 100644 index 00000000..e40bc7b7 --- /dev/null +++ b/crates/smtp/src/queue/walk.rs @@ -0,0 +1,183 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use store::{ + IterateParams, Store, ValueKey, + write::{QueueClass, QueueEvent, ValueClass}, +}; +use trc::AddContext; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WalkStep { + Continue, + NextQueue, + Stop, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct QueueWalk { + pub due_from: u64, + pub due_to: u64, + pub ascending: bool, + pub read_limit: usize, +} + +impl QueueWalk { + pub fn ascending(read_limit: usize) -> Self { + QueueWalk { + due_from: 0, + due_to: u64::MAX, + ascending: true, + read_limit, + } + } + + fn start(&self) -> QueueEvent { + self.queue_start(if self.ascending { + [0; 8] + } else { + QueueEvent::LAST_QUEUE + }) + } + + fn end(&self) -> QueueEvent { + if self.ascending { + event(QueueEvent::LAST_QUEUE, u64::MAX, u64::MAX) + } else { + event([0; 8], 0, 0) + } + } + + fn queue_start(&self, queue_name: [u8; 8]) -> QueueEvent { + if self.ascending { + event(queue_name, self.due_from, 0) + } else { + event(queue_name, self.due_to, u64::MAX) + } + } + + fn next_queue(&self, queue_name: [u8; 8]) -> Option { + let queue_name = u64::from_be_bytes(queue_name); + if self.ascending { + queue_name + .checked_add(1) + .filter(|&next| next <= u64::from_be_bytes(QueueEvent::LAST_QUEUE)) + } else { + queue_name.checked_sub(1) + } + .map(|queue_name| self.queue_start(queue_name.to_be_bytes())) + } + + fn outside_window(&self, current: &QueueEvent) -> Option> { + let is_before = if self.ascending { + current.due < self.due_from + } else { + current.due > self.due_to + }; + if is_before { + Some(Some(self.queue_start(current.queue_name))) + } else if (self.due_from..=self.due_to).contains(¤t.due) { + None + } else { + Some(self.next_queue(current.queue_name)) + } + } + + fn after(&self, current: &QueueEvent) -> Option { + if self.ascending { + match current.queue_id.checked_add(1) { + Some(queue_id) => Some(event(current.queue_name, current.due, queue_id)), + None => current + .due + .checked_add(1) + .map(|due| event(current.queue_name, due, 0)), + } + } else { + match current.queue_id.checked_sub(1) { + Some(queue_id) => Some(event(current.queue_name, current.due, queue_id)), + None => current + .due + .checked_sub(1) + .map(|due| event(current.queue_name, due, u64::MAX)), + } + } + .or_else(|| self.next_queue(current.queue_name)) + } +} + +pub async fn walk_queues( + store: &Store, + walk: QueueWalk, + mut visit: impl FnMut(&QueueEvent) -> WalkStep + Sync + Send, +) -> trc::Result<()> { + let mut bound = Some(walk.start()); + while let Some(position) = bound.take() { + let (from, to) = if walk.ascending { + (position, walk.end()) + } else { + (walk.end(), position) + }; + let mut rows = 0; + let mut last = None; + let mut next = None; + let mut is_stopped = false; + store + .iterate( + IterateParams::new(key(from), key(to)) + .set_ascending(walk.ascending) + .no_values() + .limit(walk.read_limit), + |key, _| { + let current = QueueEvent::deserialize(key)?; + rows += 1; + if let Some(skip) = walk.outside_window(¤t) { + next = Some(skip); + return Ok(false); + } + match visit(¤t) { + WalkStep::Continue => { + last = Some(current); + Ok(true) + } + WalkStep::NextQueue => { + next = Some(walk.next_queue(current.queue_name)); + Ok(false) + } + WalkStep::Stop => { + is_stopped = true; + Ok(false) + } + } + }, + ) + .await + .caused_by(trc::location!())?; + + if is_stopped { + break; + } + bound = match next { + Some(next) => next, + None if rows >= walk.read_limit => last.and_then(|last| walk.after(&last)), + None => None, + }; + } + + Ok(()) +} + +fn event(queue_name: [u8; 8], due: u64, queue_id: u64) -> QueueEvent { + QueueEvent { + due, + queue_id, + queue_name, + } +} + +fn key(event: QueueEvent) -> ValueKey { + ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(event))) +} + diff --git a/crates/smtp/src/reporting/aggregate/codec.rs b/crates/smtp/src/reporting/aggregate/codec.rs new file mode 100644 index 00000000..1710eebd --- /dev/null +++ b/crates/smtp/src/reporting/aggregate/codec.rs @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use registry::{ + schema::structs::{ + DmarcDkimResult, DmarcExtension, DmarcPolicyOverrideReason, DmarcReportRecord, + DmarcSpfResult, TlsFailureDetails, + }, + types::{EnumImpl, ipaddr::IpAddr, list::List}, +}; +use std::net::{Ipv4Addr, Ipv6Addr}; +use utils::codec::leb128::{Leb128Reader, Leb128Writer}; + +const IP_NONE: u8 = 0; +const IP_V4: u8 = 4; +const IP_V6: u8 = 6; +const NIBBLE: u16 = 0x0F; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Detail { + Full, + Compact, +} + +pub(crate) trait ReportRow: Sized { + fn encode_row(&self, context: &str, writer: &mut RowWriter<'_>); + fn decode_row(context: &str, reader: &mut RowReader<'_>) -> Option; +} + +pub(crate) struct RowWriter<'x> { + out: &'x mut Vec, + detail: Detail, +} + +pub(crate) struct RowReader<'x>(&'x [u8]); + +impl<'x> RowWriter<'x> { + pub(crate) fn new(out: &'x mut Vec, detail: Detail) -> Self { + RowWriter { out, detail } + } + + fn byte(&mut self, value: u8) { + self.out.push(value); + } + + fn id(&mut self, value: impl EnumImpl) { + self.out + .push(u8::try_from(value.to_id()).unwrap_or(u8::MAX)); + } + + fn len(&mut self, value: usize) { + self.out.push_leb128(value); + } + + fn str(&mut self, value: &str) { + self.len(value.len()); + self.out.extend_from_slice(value.as_bytes()); + } + + fn opt(&mut self, value: Option<&str>) { + match value { + Some(value) => { + self.len(value.len() + 1); + self.out.extend_from_slice(value.as_bytes()); + } + None => self.out.push(0), + } + } + + fn human(&mut self, value: Option<&str>) { + match self.detail { + Detail::Full => self.opt(value), + Detail::Compact => self.opt(None), + } + } + + fn same_or(&mut self, value: &str, reference: &str) { + if value == reference { + self.out.push(0); + } else { + self.opt(Some(value)); + } + } + + fn ip(&mut self, value: Option<&IpAddr>) { + match value.map(|ip| ip.0) { + Some(std::net::IpAddr::V4(ip)) => { + self.out.push(IP_V4); + self.out.extend_from_slice(&ip.octets()); + } + Some(std::net::IpAddr::V6(ip)) => { + self.out.push(IP_V6); + self.out.extend_from_slice(&ip.octets()); + } + None => self.out.push(IP_NONE), + } + } +} + +impl<'x> RowReader<'x> { + pub(crate) fn new(bytes: &'x [u8]) -> Self { + RowReader(bytes) + } + + fn byte(&mut self) -> Option { + let (byte, rest) = self.0.split_first()?; + self.0 = rest; + Some(*byte) + } + + fn id(&mut self) -> Option { + self.byte().and_then(|id| T::from_id(u16::from(id))) + } + + fn array(&mut self) -> Option<[u8; N]> { + let (array, rest) = self.0.split_first_chunk::()?; + self.0 = rest; + Some(*array) + } + + fn len(&mut self) -> Option { + self.0.read_leb128() + } + + fn count(&mut self) -> Option { + self.len().map(|count| count.min(self.0.len())) + } + + fn bytes(&mut self, len: usize) -> Option<&'x str> { + let (bytes, rest) = self.0.split_at_checked(len)?; + self.0 = rest; + std::str::from_utf8(bytes).ok() + } + + fn str(&mut self) -> Option { + let len = self.len()?; + self.bytes(len).map(str::to_string) + } + + fn opt(&mut self) -> Option> { + match self.len()? { + 0 => Some(None), + len => self.bytes(len - 1).map(|value| Some(value.to_string())), + } + } + + fn same_or(&mut self, reference: &str) -> Option { + self.opt() + .map(|value| value.unwrap_or_else(|| reference.to_string())) + } + + fn ip(&mut self) -> Option> { + match self.byte()? { + IP_V4 => Some(Some(IpAddr(Ipv4Addr::from(self.array::<4>()?).into()))), + IP_V6 => Some(Some(IpAddr(Ipv6Addr::from(self.array::<16>()?).into()))), + IP_NONE => Some(None), + _ => None, + } + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl ReportRow for DmarcReportRecord { + fn encode_row(&self, domain: &str, writer: &mut RowWriter<'_>) { + writer.id(self.evaluated_disposition); + writer.byte( + u8::try_from( + ((self.evaluated_dkim.to_id() & NIBBLE) << 4) + | (self.evaluated_spf.to_id() & NIBBLE), + ) + .unwrap_or(u8::MAX), + ); + writer.ip(self.source_ip.as_ref()); + writer.same_or(&self.header_from, domain); + writer.same_or(&self.envelope_from, &self.header_from); + writer.opt(self.envelope_to.as_deref()); + writer.len(self.policy_override_reasons.len()); + for reason in self.policy_override_reasons.iter() { + writer.id(reason.override_type); + writer.human(reason.comment.as_deref()); + } + writer.len(self.dkim_results.len()); + for dkim in self.dkim_results.iter() { + writer.same_or(&dkim.domain, &self.header_from); + writer.str(&dkim.selector); + writer.id(dkim.result); + writer.human(dkim.human_result.as_deref()); + } + writer.len(self.spf_results.len()); + for spf in self.spf_results.iter() { + writer.same_or(&spf.domain, &self.envelope_from); + writer.id(spf.scope); + writer.id(spf.result); + writer.human(spf.human_result.as_deref()); + } + writer.len(self.extensions.len()); + for extension in self.extensions.iter() { + writer.str(&extension.name); + writer.str(&extension.definition); + } + } + + fn decode_row(domain: &str, reader: &mut RowReader<'_>) -> Option { + let evaluated_disposition = reader.id()?; + let evaluations = u16::from(reader.byte()?); + let source_ip = reader.ip()?; + let header_from = reader.same_or(domain)?; + let envelope_from = reader.same_or(&header_from)?; + let envelope_to = reader.opt()?; + + let count = reader.count()?; + let mut policy_override_reasons = List::with_capacity(count); + for _ in 0..count { + policy_override_reasons.push_unchecked(DmarcPolicyOverrideReason { + override_type: reader.id()?, + comment: reader.opt()?, + }); + } + let count = reader.count()?; + let mut dkim_results = List::with_capacity(count); + for _ in 0..count { + dkim_results.push_unchecked(DmarcDkimResult { + domain: reader.same_or(&header_from)?, + selector: reader.str()?, + result: reader.id()?, + human_result: reader.opt()?, + }); + } + let count = reader.count()?; + let mut spf_results = List::with_capacity(count); + for _ in 0..count { + spf_results.push_unchecked(DmarcSpfResult { + domain: reader.same_or(&envelope_from)?, + scope: reader.id()?, + result: reader.id()?, + human_result: reader.opt()?, + }); + } + let count = reader.count()?; + let mut extensions = List::with_capacity(count); + for _ in 0..count { + extensions.push_unchecked(DmarcExtension { + name: reader.str()?, + definition: reader.str()?, + }); + } + + reader.is_empty().then_some(DmarcReportRecord { + source_ip, + count: 0, + evaluated_disposition, + evaluated_dkim: EnumImpl::from_id(evaluations >> 4)?, + evaluated_spf: EnumImpl::from_id(evaluations & NIBBLE)?, + policy_override_reasons, + envelope_to, + envelope_from, + header_from, + dkim_results, + spf_results, + extensions, + }) + } +} + +impl ReportRow for TlsFailureDetails { + fn encode_row(&self, _: &str, writer: &mut RowWriter<'_>) { + writer.id(self.result_type); + writer.ip(self.sending_mta_ip.as_ref()); + writer.ip(self.receiving_ip.as_ref()); + writer.opt(self.receiving_mx_hostname.as_deref()); + writer.opt(self.receiving_mx_helo.as_deref()); + writer.human(self.failure_reason_code.as_deref()); + writer.human(self.additional_information.as_deref()); + } + + fn decode_row(_: &str, reader: &mut RowReader<'_>) -> Option { + let failure = TlsFailureDetails { + result_type: reader.id()?, + sending_mta_ip: reader.ip()?, + receiving_ip: reader.ip()?, + receiving_mx_hostname: reader.opt()?, + receiving_mx_helo: reader.opt()?, + failure_reason_code: reader.opt()?, + additional_information: reader.opt()?, + failed_session_count: 0, + }; + reader.is_empty().then_some(failure) + } +} + diff --git a/crates/smtp/src/reporting/aggregate/mod.rs b/crates/smtp/src/reporting/aggregate/mod.rs new file mode 100644 index 00000000..e8f4a2f9 --- /dev/null +++ b/crates/smtp/src/reporting/aggregate/mod.rs @@ -0,0 +1,684 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod codec; +pub mod window; + +use super::{ + AggregateTimestamp, dmarc::dmarc_report_descriptor, tls::tls_report_descriptor, + tls::tls_report_policy, +}; +use codec::{Detail, ReportRow, RowReader, RowWriter}; +use common::{ + Server, + config::smtp::report::AggregateFrequency, + ipc::{ReportingEvent, ToHash}, +}; +use registry::{ + pickle::{Pickle, PickledStream}, + schema::{ + enums::TlsPolicyType, + prelude::ObjectType, + structs::{ + DmarcInternalReport, DmarcReportRecord, Task, TaskReportWindow, TaskStatus, + TlsFailureDetails, TlsInternalReport, TlsReportPolicy, + }, + }, + types::{EnumImpl, ObjectImpl, datetime::UTCDateTime}, +}; +use std::ops::Range; +use store::{ + IterateParams, Store, Subspace, U64_LEN, + write::{ + AnyClass, AnyKey, BatchBuilder, ValueClass, + key::{CounterKind, KeySerializer}, + now, + }, + xxhash_rust::xxh3::Xxh3, +}; +use trc::{AddContext, OutgoingReportEvent}; + +pub(crate) const REPORT_BURST: usize = 256; +const ROW_KEY_MAX_LEN: usize = 255; +const ROW_PREFIX_LEN: usize = 1 + U64_LEN; +const POLICY_KEY_LEN: usize = 1 + 2 + U64_LEN; +const POLICY_MARKER: u8 = u8::MAX; +const DESCRIPTOR_MARKER: u8 = u8::MAX - 1; +const DESCRIPTOR_PREFIX_LEN: usize = 1 + 2 + U64_LEN; +const DESCRIPTOR_KEY_LEN: usize = DESCRIPTOR_PREFIX_LEN + (U64_LEN * 2); +const ROW_PAYLOAD_MAX_LEN: usize = ROW_KEY_MAX_LEN - ROW_PREFIX_LEN; +const WINDOW_UNIT: u64 = 3600; +const ID_HASH_BITS: u32 = 41; +const ID_HASH_MASK: u64 = (1 << ID_HASH_BITS) - 1; +const ID_TLS_BIT: u64 = 1 << ID_HASH_BITS; +const ID_WINDOW_SHIFT: u32 = ID_HASH_BITS + 1; +const ID_WINDOW_BITS: u32 = 20; +const ID_WINDOW_MASK: u64 = (1 << ID_WINDOW_BITS) - 1; +const TLS_SUCCESS: u8 = 0; +const TLS_FAILURE: u8 = 1; +const SEND_DELAY: u64 = 300; +const SEND_JITTER: u64 = 600; +const DEFAULT_MAX_SIZE: usize = 5 * 1024 * 1024; + +pub(crate) fn report_size_limit(max_size: Option) -> usize { + match max_size.unwrap_or(DEFAULT_MAX_SIZE) { + 0 => usize::MAX, + max_size => max_size, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ReportWindow { + pub start: u64, + pub end: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReportItem { + pub object: ObjectType, + pub item_id: u64, +} + +#[derive(Default)] +pub(crate) struct ReportScheduler { + pending: Vec, + keys: Vec, + windows: Vec<(ObjectType, u64)>, +} + +struct PendingRow { + object: ObjectType, + slot: u64, + window: ReportWindow, + policy: u64, + event: usize, + key: Range, +} + +impl ReportWindow { + fn current(interval: &AggregateFrequency) -> Option { + match interval.as_secs() { + 0 => None, + secs => { + let start = interval.to_timestamp(); + Some(ReportWindow { + start, + end: start + secs, + }) + } + } + } + + fn slot(&self, object: ObjectType, domain: &str, policy: u64) -> u64 { + let mut hasher = Xxh3::new(); + hasher.update(&object.to_id().to_be_bytes()); + hasher.update(domain.as_bytes()); + hasher.update(&[0]); + hasher.update(&policy.to_be_bytes()); + hasher.update(&self.start.to_be_bytes()); + hasher.update(&self.end.to_be_bytes()); + hasher.digest() + } + + fn item_id(&self, object: ObjectType, slot: u64) -> u64 { + let object = match object { + ObjectType::TlsInternalReport => ID_TLS_BIT, + _ => 0, + }; + (((self.end / WINDOW_UNIT) & ID_WINDOW_MASK) << ID_WINDOW_SHIFT) + | object + | (slot & ID_HASH_MASK) + } + + fn task(&self, object: ObjectType) -> Option { + let task = TaskReportWindow { + window_end: UTCDateTime::from_timestamp(self.end.cast_signed()), + status: TaskStatus::at(self.end.saturating_add(SEND_DELAY).cast_signed()), + }; + match object { + ObjectType::DmarcInternalReport => Some(Task::DmarcReportWindow(task)), + ObjectType::TlsInternalReport => Some(Task::TlsReportWindow(task)), + _ => None, + } + } +} + +impl ReportItem { + pub fn new(object: ObjectType, item_id: u64) -> Self { + ReportItem { object, item_id } + } + + pub(crate) fn deliver_at(&self, window: ReportWindow) -> u64 { + window.end + SEND_DELAY + self.item_id % SEND_JITTER + } + + fn push_row_prefix(&self, out: &mut Vec) { + out.push(CounterKind::ReportRow.byte()); + out.extend_from_slice(&self.item_id.to_be_bytes()); + } + + fn row_bound(item_id: u64) -> AnyKey> { + AnyKey { + subspace: Subspace::Counter, + key: KeySerializer::new(ROW_PREFIX_LEN) + .write(CounterKind::ReportRow.byte()) + .write(item_id) + .finalize(), + } + } + + fn policy_bound(&self, item_id: u64) -> AnyKey> { + AnyKey { + subspace: Subspace::ReportOut, + key: KeySerializer::new(POLICY_KEY_LEN) + .write(POLICY_MARKER) + .write(self.object.to_id()) + .write(item_id) + .finalize(), + } + } + + fn descriptor_key(&self, window_end: u64, slot: u64) -> Vec { + KeySerializer::new(DESCRIPTOR_KEY_LEN) + .write(DESCRIPTOR_MARKER) + .write(self.object.to_id()) + .write(window_end) + .write(self.item_id) + .write(slot) + .finalize() + } + + fn descriptor_bound(object: ObjectType, window_end: u64) -> AnyKey> { + AnyKey { + subspace: Subspace::ReportOut, + key: KeySerializer::new(DESCRIPTOR_PREFIX_LEN) + .write(DESCRIPTOR_MARKER) + .write(object.to_id()) + .write(window_end) + .finalize(), + } + } + + fn policy_key(&self, policy: u64) -> Vec { + KeySerializer::new(POLICY_KEY_LEN + U64_LEN) + .write(POLICY_MARKER) + .write(self.object.to_id()) + .write(self.item_id) + .write(policy) + .finalize() + } + + async fn rows( + &self, + store: &Store, + mut cb: impl FnMut(&[u8], u64) + Send + Sync, + ) -> trc::Result<()> { + store + .iterate( + IterateParams::new( + Self::row_bound(self.item_id), + Self::row_bound(self.item_id.saturating_add(1)), + ), + |key, value| { + let count = value + .first_chunk::() + .map(|count| i64::from_le_bytes(*count)) + .ok_or_else(|| { + trc::Error::corrupted_key(key, value.into(), trc::location!()) + })?; + if let (Some(row), Ok(count @ 1..)) = + (key.get(ROW_PREFIX_LEN..), u64::try_from(count)) + { + cb(row, count); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!()) + } + + pub fn clear(&self, batch: &mut BatchBuilder) { + batch.clear_range( + Subspace::Counter, + Self::row_bound(self.item_id).key, + Self::row_bound(self.item_id.saturating_add(1)).key, + ); + if self.object == ObjectType::TlsInternalReport { + batch.clear_range( + Subspace::ReportOut, + self.policy_bound(self.item_id).key, + self.policy_bound(self.item_id.saturating_add(1)).key, + ); + } + } + + pub async fn clear_orphans(&self, store: &Store) -> trc::Result<()> { + let mut has_rows = false; + store + .iterate( + IterateParams::new( + Self::row_bound(self.item_id), + Self::row_bound(self.item_id.saturating_add(1)), + ) + .set_values(false), + |_, _| { + has_rows = true; + Ok(false) + }, + ) + .await + .caused_by(trc::location!())?; + if has_rows { + let mut batch = BatchBuilder::new(); + self.clear(&mut batch); + store + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + } + Ok(()) + } +} + +pub async fn read_dmarc_records( + store: &Store, + item_id: u64, + report: &mut DmarcInternalReport, + max_size: usize, +) -> trc::Result<()> { + let mut rows = TopRows::new(max_size, |record: &DmarcReportRecord| record.count); + let domain = report.domain.as_str(); + ReportItem::new(ObjectType::DmarcInternalReport, item_id) + .rows(store, |row, count| { + match DmarcReportRecord::decode_row(domain, &mut RowReader::new(row)) { + Some(mut record) => { + record.count = count; + rows.push(row.len(), record); + } + None => corrupted_row(item_id, row), + } + }) + .await?; + + let records = &mut report.report.records; + for record in rows.into_rows() { + records.push(record); + } + Ok(()) +} + +pub async fn read_tls_policies( + store: &Store, + item_id: u64, + report: &mut TlsInternalReport, + max_size: usize, +) -> trc::Result<()> { + let item = ReportItem::new(ObjectType::TlsInternalReport, item_id); + let mut policies: Vec<(u64, TlsReportPolicy)> = Vec::new(); + store + .iterate( + IterateParams::new( + item.policy_bound(item_id), + item.policy_bound(item_id.saturating_add(1)), + ), + |key, value| { + if let Some(policy_hash) = key.last_chunk::() { + match PickledStream::new(value) + .and_then(|mut stream| TlsReportPolicy::unpickle(&mut stream)) + { + Some(policy) => policies.push((u64::from_be_bytes(*policy_hash), policy)), + None => corrupted_row(item_id, key), + } + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + let mut failures = TopRows::new(max_size, |(_, failure): &(u64, TlsFailureDetails)| { + failure.failed_session_count + }); + let domain = report.domain.as_str(); + item.rows(store, |row, count| { + let Some((policy_hash, row)) = row.split_first_chunk::() else { + corrupted_row(item_id, row); + return; + }; + let policy_hash = u64::from_be_bytes(*policy_hash); + let index = match policies.binary_search_by_key(&policy_hash, |(hash, _)| *hash) { + Ok(index) => index, + Err(index) => { + policies.insert( + index, + ( + policy_hash, + TlsReportPolicy { + policy_type: TlsPolicyType::NoPolicyFound, + policy_domain: domain.to_string(), + ..Default::default() + }, + ), + ); + index + } + }; + let Some((_, policy)) = policies.get_mut(index) else { + return; + }; + match row.split_first() { + Some((&TLS_SUCCESS, [])) => { + policy.total_successful_sessions += count; + } + Some((&TLS_FAILURE, details)) => { + policy.total_failed_sessions += count; + match TlsFailureDetails::decode_row(domain, &mut RowReader::new(details)) { + Some(mut failure) => { + failure.failed_session_count = count; + failures.push(details.len(), (policy_hash, failure)); + } + None => corrupted_row(item_id, row), + } + } + _ => corrupted_row(item_id, row), + } + }) + .await?; + + for (policy_hash, failure) in failures.into_rows() { + if let Ok(index) = policies.binary_search_by_key(&policy_hash, |(hash, _)| *hash) + && let Some((_, policy)) = policies.get_mut(index) + { + policy.failure_details.push(failure); + } + } + for (policy_hash, policy) in policies { + report.policy_identifiers.push(policy_hash); + report.report.policies.push(policy); + } + Ok(()) +} + +struct TopRows { + rows: Vec<(usize, T)>, + size: usize, + max_size: usize, + count: F, +} + +impl u64> TopRows { + fn new(max_size: usize, count: F) -> Self { + TopRows { + rows: Vec::new(), + size: 0, + max_size, + count, + } + } + + fn push(&mut self, len: usize, row: T) { + self.size = self.size.saturating_add(len); + self.rows.push((len, row)); + if self.size > self.max_size.saturating_mul(2) { + self.trim(); + } + } + + fn trim(&mut self) { + if self.size > self.max_size { + let count = &self.count; + self.rows + .sort_unstable_by_key(|(_, row)| std::cmp::Reverse(count(row))); + let max_size = self.max_size; + let mut size = 0usize; + self.rows.retain(|(len, _)| { + size = size.saturating_add(*len); + size <= max_size + }); + self.size = self.rows.iter().map(|(len, _)| *len).sum(); + } + } + + fn into_rows(mut self) -> impl Iterator { + self.trim(); + self.rows.into_iter().map(|(_, row)| row) + } +} + +#[cold] +fn corrupted_row(item_id: u64, row: &[u8]) { + trc::error!( + trc::Error::corrupted_key(row, None, trc::location!()) + .id(item_id) + .details("Undecodable aggregate report row") + ); +} + +impl ReportScheduler { + pub(crate) async fn process( + &mut self, + server: &Server, + events: &mut Vec, + ) -> bool { + let mut stop = false; + self.pending.clear(); + self.keys.clear(); + for (index, event) in events.iter_mut().enumerate() { + match event { + ReportingEvent::Dmarc(event) => { + let Some(window) = ReportWindow::current(&event.interval) else { + continue; + }; + let object = ObjectType::DmarcInternalReport; + let policy = event.dmarc_record.to_hash(); + let record = DmarcReportRecord::from(std::mem::take(&mut event.report_record)); + let start = self.keys.len(); + if encode_row(&record, &event.domain, &mut self.keys, start) { + self.pending.push(PendingRow { + object, + slot: window.slot(object, &event.domain, policy), + window, + policy, + event: index, + key: start..self.keys.len(), + }); + } else { + oversized_row(&event.domain, event.span_id); + } + } + ReportingEvent::Tls(event) => { + let Some(window) = ReportWindow::current(&event.interval) else { + continue; + }; + let object = ObjectType::TlsInternalReport; + let policy = event.policy.to_hash(); + let start = self.keys.len(); + self.keys.extend_from_slice(&policy.to_be_bytes()); + let is_encoded = match event.failure.take() { + Some(failure) => { + self.keys.push(TLS_FAILURE); + encode_row( + &TlsFailureDetails::from(failure), + &event.domain, + &mut self.keys, + start, + ) + } + None => { + self.keys.push(TLS_SUCCESS); + true + } + }; + if is_encoded { + self.pending.push(PendingRow { + object, + slot: window.slot(object, &event.domain, 0), + window, + policy, + event: index, + key: start..self.keys.len(), + }); + } else { + oversized_row(&event.domain, event.span_id); + } + } + ReportingEvent::Stop => { + stop = true; + } + } + } + + if !self.pending.is_empty() { + let keys = self.keys.as_slice(); + self.pending.sort_unstable_by(|a, b| { + a.slot + .cmp(&b.slot) + .then_with(|| keys.get(a.key.clone()).cmp(&keys.get(b.key.clone()))) + .then(a.event.cmp(&b.event)) + }); + self.write_rows(server, events).await; + } + events.clear(); + + stop + } + + async fn write_rows(&mut self, server: &Server, events: &[ReportingEvent]) { + let ReportScheduler { + pending, + keys, + windows, + } = self; + let mut batch = BatchBuilder::new(); + let mut new_windows: Vec<(ObjectType, u64)> = Vec::new(); + + for rows in pending.chunk_by(|a, b| a.slot == b.slot) { + let Some(first) = rows.first() else { + continue; + }; + let item = + ReportItem::new(first.object, first.window.item_id(first.object, first.slot)); + let descriptor = match events.get(first.event) { + Some(ReportingEvent::Dmarc(event)) => { + dmarc_report_descriptor(event, first.window, item, first.policy) + .to_pickled_vec() + } + Some(ReportingEvent::Tls(event)) => { + tls_report_descriptor(event, first.window, item).to_pickled_vec() + } + Some(ReportingEvent::Stop) | None => continue, + }; + + let window = (first.object, first.window.end); + if !windows.contains(&window) + && !new_windows.contains(&window) + && let Some(task) = first.window.task(first.object) + { + batch.schedule_task(task); + new_windows.push(window); + } + + batch.set( + ValueClass::Any(AnyClass { + subspace: Subspace::ReportOut, + key: item.descriptor_key(first.window.end, first.slot), + }), + descriptor, + ); + add_rows(item, events, rows, keys, &mut batch); + batch.commit_point(); + } + + match server.store().write_batch(&mut batch).await { + Ok(_) => { + if !new_windows.is_empty() { + let now = now(); + windows.retain(|(_, end)| end.saturating_add(SEND_DELAY) > now); + windows.extend(new_windows); + } + } + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to write aggregate report rows") + ); + } + } + } +} + +fn add_rows( + item: ReportItem, + events: &[ReportingEvent], + rows: &[PendingRow], + keys: &[u8], + batch: &mut BatchBuilder, +) { + let mut last_policy = None; + for rows in rows.chunk_by(|a, b| keys.get(a.key.clone()) == keys.get(b.key.clone())) { + let Some(row) = rows.first() else { + continue; + }; + let Some(payload) = keys.get(row.key.clone()) else { + continue; + }; + + if let Some(ReportingEvent::Tls(event)) = events.get(row.event) + && last_policy != Some(row.policy) + { + last_policy = Some(row.policy); + let mut value = Vec::with_capacity(64); + value.push(TlsInternalReport::VERSION); + tls_report_policy(event).pickle(&mut value); + batch.set( + ValueClass::Any(AnyClass { + subspace: Subspace::ReportOut, + key: item.policy_key(row.policy), + }), + value, + ); + } + + let mut key = Vec::with_capacity(ROW_PREFIX_LEN + payload.len()); + item.push_row_prefix(&mut key); + key.extend_from_slice(payload); + batch.add( + ValueClass::Any(AnyClass { + subspace: Subspace::Counter, + key, + }), + i64::try_from(rows.len()).unwrap_or(i64::MAX), + ); + } +} + +fn encode_row(row: &impl ReportRow, context: &str, out: &mut Vec, start: usize) -> bool { + let payload = out.len(); + row.encode_row(context, &mut RowWriter::new(out, Detail::Full)); + if out.len() - start <= ROW_PAYLOAD_MAX_LEN { + return true; + } + out.truncate(payload); + row.encode_row(context, &mut RowWriter::new(out, Detail::Compact)); + if out.len() - start <= ROW_PAYLOAD_MAX_LEN { + true + } else { + out.truncate(start); + false + } +} + +#[cold] +fn oversized_row(domain: &str, span_id: u64) { + trc::event!( + OutgoingReport(OutgoingReportEvent::MaxSizeExceeded), + SpanId = span_id, + Domain = domain.to_string(), + Limit = ROW_KEY_MAX_LEN, + ); +} + diff --git a/crates/smtp/src/reporting/aggregate/window.rs b/crates/smtp/src/reporting/aggregate/window.rs new file mode 100644 index 00000000..141470bf --- /dev/null +++ b/crates/smtp/src/reporting/aggregate/window.rs @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{DESCRIPTOR_PREFIX_LEN, ReportItem}; +use crate::reporting::{ + dmarc::complete_dmarc_report, index::InternalReportIndex, tls::complete_tls_report, +}; +use common::{Server, expr::Bump}; +use registry::schema::{ + prelude::ObjectType, + structs::{DmarcInternalReport, TlsInternalReport}, +}; +use store::{ + Deserialize, IterateParams, Subspace, U64_LEN, + write::{AnyClass, AnyKey, BatchBuilder, ValueClass, assert::AssertValue}, +}; +use trc::{AddContext, OutgoingReportEvent}; + +const DESCRIPTOR_PAGE: usize = 512; + +struct Descriptor { + key: Vec, + value: Vec, +} + +impl Descriptor { + fn item_id(&self) -> Option { + self.key + .get(DESCRIPTOR_PREFIX_LEN..) + .and_then(|key| key.first_chunk::()) + .map(|id| u64::from_be_bytes(*id)) + } + + fn clear(&self, batch: &mut BatchBuilder) { + let class = ValueClass::Any(AnyClass { + subspace: Subspace::ReportOut, + key: self.key.clone(), + }); + batch + .assert_value(class.clone(), AssertValue::Some) + .clear(class); + } +} + +pub async fn create_window_reports( + server: &Server, + object: ObjectType, + window_end: u64, +) -> trc::Result<()> { + let end = ReportItem::descriptor_bound(object, window_end.saturating_add(1)); + let mut from = ReportItem::descriptor_bound(object, 0); + let mut page: Vec = Vec::with_capacity(DESCRIPTOR_PAGE); + let span_id = server.inner.data.span_id_gen.generate(); + let mut arena = Bump::new(); + + loop { + page.clear(); + server + .store() + .iterate( + IterateParams::new(from.clone(), end.clone()).limit(DESCRIPTOR_PAGE), + |key, value| { + page.push(Descriptor { + key: key.to_vec(), + value: value.to_vec(), + }); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + let is_last = page.len() < DESCRIPTOR_PAGE; + let mut ready = page.as_slice(); + if !is_last + && let Some(last_id) = page.last().and_then(Descriptor::item_id) + && let Some(split) = page.iter().position(|d| d.item_id() == Some(last_id)) + && split > 0 + { + ready = page.get(..split).unwrap_or_default(); + } + + let mut batch = BatchBuilder::new(); + for group in ready.chunk_by(|a, b| a.item_id() == b.item_id()) { + match group { + [descriptor] => { + if let Some(item_id) = descriptor.item_id() { + write_report( + server, object, item_id, descriptor, &mut batch, &mut arena, span_id, + ) + .await; + } + descriptor.clear(&mut batch); + } + descriptors => { + if let Some(item_id) = descriptors.first().and_then(Descriptor::item_id) { + ReportItem::new(object, item_id).clear(&mut batch); + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = span_id, + Id = item_id, + Total = descriptors.len(), + Details = "Aggregate report id collision, reports dropped", + ); + } + for descriptor in descriptors { + descriptor.clear(&mut batch); + } + } + } + batch.commit_point(); + } + + if !batch.is_empty() { + server + .store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + } + + match ready.last() { + Some(last) if !is_last => { + let mut key = Vec::with_capacity(last.key.len() + 1); + key.extend_from_slice(&last.key); + key.push(0); + from = AnyKey { + subspace: Subspace::ReportOut, + key, + }; + } + _ => return Ok(()), + } + } +} + +async fn write_report( + server: &Server, + object: ObjectType, + item_id: u64, + descriptor: &Descriptor, + batch: &mut BatchBuilder, + arena: &mut Bump, + span_id: u64, +) { + let written = match object { + ObjectType::DmarcInternalReport => { + match DmarcInternalReport::deserialize(&descriptor.value) { + Ok(mut report) => { + complete_dmarc_report(server, &mut report, arena, span_id).await; + report.write_header(batch, item_id); + true + } + Err(_) => false, + } + } + ObjectType::TlsInternalReport => match TlsInternalReport::deserialize(&descriptor.value) { + Ok(mut report) => { + complete_tls_report(server, &mut report, arena, span_id).await; + report.write_header(batch, item_id); + true + } + Err(_) => false, + }, + _ => false, + }; + if !written { + ReportItem::new(object, item_id).clear(batch); + trc::error!( + trc::Error::corrupted_key( + &descriptor.key, + descriptor.value.as_slice().into(), + trc::location!() + ) + .span_id(span_id) + .details("Undecodable aggregate report descriptor") + ); + } +} diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index db41102c..6b93a3a3 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -4,25 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashSet; -use common::{Server, psl}; -use mail_auth::{ - flate2::read::GzDecoder, - report::{Feedback, Report, tlsrpt::TlsReport}, - zip, -}; -use mail_parser::{Message, MessagePart, MimeHeaders, PartType}; +use common::Server; +use compact_str::{CompactString, ToCompactString, format_compact}; +use mail_parser::{Message, MessagePart, PartKind}; use registry::{ schema::structs::{ArfExternalReport, DmarcExternalReport, TlsExternalReport}, types::datetime::UTCDateTime, }; +use smtp_report::{ + arf::FeedbackReport, dmarc::AggregateReport, flate2::read::GzDecoder, tlsrpt::TlsReport, zip, +}; use std::{ borrow::Cow, io::{Cursor, Read}, }; use store::write::{BatchBuilder, now}; use trc::IncomingReportEvent; -use types::id::Id; use crate::reporting::{inbound::LogReport, index::ExternalReportIndex}; @@ -41,13 +38,26 @@ enum Format { pub(crate) struct ReportData<'x> { compression: Compression, format: Format<(), (), ()>, - data: &'x [u8], + data: Cow<'x, [u8]>, } impl<'x> ReportData<'x> { - fn from_part(part: &'x MessagePart<'x>) -> Option { - match &part.body { - PartType::Text(report) => { + fn from_part(part: MessagePart<'x>) -> Option { + let (compression, format) = Self::classify(&part)?; + Some(ReportData { + compression, + format, + data: match part.text() { + Some(Cow::Borrowed(text)) => Cow::Borrowed(text.as_bytes()), + Some(Cow::Owned(text)) => Cow::Owned(text.into_bytes()), + None => part.decoded(), + }, + }) + } + + fn classify(part: &MessagePart<'_>) -> Option<(Compression, Format<(), (), ()>)> { + match part.kind() { + PartKind::Text => { if part .content_type() .and_then(|ct| ct.subtype()) @@ -57,28 +67,16 @@ impl<'x> ReportData<'x> { .and_then(|n| n.rsplit_once('.')) .is_some_and(|(_, e)| e.eq_ignore_ascii_case("xml")) { - Some(ReportData { - compression: Compression::None, - format: Format::Dmarc(()), - data: report.as_bytes(), - }) + Some((Compression::None, Format::Dmarc(()))) } else if part.is_content_type("message", "feedback-report") { - Some(ReportData { - compression: Compression::None, - format: Format::Arf(()), - data: report.as_bytes(), - }) + Some((Compression::None, Format::Arf(()))) } else { None } } - PartType::Binary(report) | PartType::InlineBinary(report) => { + PartKind::Binary | PartKind::InlineBinary => { if part.is_content_type("message", "feedback-report") { - return Some(ReportData { - compression: Compression::None, - format: Format::Arf(()), - data: report.as_ref(), - }); + return Some((Compression::None, Format::Arf(()))); } let subtype = part @@ -109,25 +107,21 @@ impl<'x> ReportData<'x> { } }; - Some(ReportData { - compression, - format, - data: report.as_ref(), - }) + Some((compression, format)) } _ => None, } } fn extract(message: &'x Message<'x>) -> Vec { - message.parts.iter().filter_map(Self::from_part).collect() + message.root().parts().filter_map(Self::from_part).collect() } pub(crate) fn is_present(message: &Message<'_>) -> bool { message - .parts - .iter() - .any(|part| ReportData::from_part(part).is_some()) + .root() + .parts() + .any(|part| ReportData::classify(&part).is_some()) } } @@ -146,7 +140,7 @@ impl AnalyzeReport for Server { .unwrap_or_default() .into(); let to: Vec = message.to().map_or_else(Vec::new, |a| { - a.iter() + a.mailboxes() .filter_map(|a| a.address()) .map(|a| a.into()) .collect() @@ -157,16 +151,16 @@ impl AnalyzeReport for Server { for report in reports { let data = match report.compression { - Compression::None => Cow::Borrowed(report.data), + Compression::None => report.data, Compression::Gzip => { - match read_capped(GzDecoder::new(report.data), 0, max_size) { + match read_capped(GzDecoder::new(report.data.as_ref()), 0, max_size) { Ok(buf) => Cow::Owned(buf), Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, - From = from.to_string(), - Reason = err.to_string(), + From = CompactString::from(&from), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); @@ -196,8 +190,8 @@ impl AnalyzeReport for Server { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, - From = from.to_string(), - Reason = err.to_string(), + From = CompactString::from(&from), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); continue; @@ -206,8 +200,8 @@ impl AnalyzeReport for Server { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, - From = from.to_string(), - Reason = err.to_string(), + From = CompactString::from(&from), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); continue; @@ -217,7 +211,7 @@ impl AnalyzeReport for Server { }; let report = match report.format { - Format::Dmarc(_) => match Report::parse_xml(&data) { + Format::Dmarc(_) => match AggregateReport::parse_xml(&data) { Ok(report) => { // Log report.log(); @@ -227,8 +221,8 @@ impl AnalyzeReport for Server { trc::event!( IncomingReport(IncomingReportEvent::DmarcParseFailed), SpanId = session_id, - From = from.to_string(), - Reason = err, + From = CompactString::from(&from), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); @@ -245,25 +239,26 @@ impl AnalyzeReport for Server { trc::event!( IncomingReport(IncomingReportEvent::TlsRpcParseFailed), SpanId = session_id, - From = from.to_string(), - Reason = format!("{err:?}"), + From = CompactString::from(&from), + Reason = format_compact!("{err:?}"), CausedBy = trc::location!() ); continue; } }, - Format::Arf(_) => match Feedback::parse_arf(&data) { - Some(report) => { + Format::Arf(_) => match FeedbackReport::parse_arf(&data) { + Ok(report) => { // Log report.log(); Format::Arf(report.into_owned()) } - None => { + Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::ArfParseFailed), SpanId = session_id, - From = from.to_string(), + From = CompactString::from(&from), + Reason = err.to_compact_string(), CausedBy = trc::location!() ); @@ -280,68 +275,44 @@ impl AnalyzeReport for Server { match report { Format::Dmarc(report) => { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] let mut report = DmarcExternalReport { from, to: to.into(), subject, - member_tenant_id: None, expires_at: UTCDateTime::from_timestamp(expires as i64), received_at: UTCDateTime::now(), report: report.into(), }; - report.member_tenant_id = tenant_ids( - &core, - report - .domains() - .filter_map(psl::domain_str) - .collect::>(), - ) - .await; report.write_ops(&mut batch, item_id, true); } Format::Tls(report) => { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] let mut report = TlsExternalReport { from, to: to.into(), subject, - member_tenant_id: None, expires_at: UTCDateTime::from_timestamp(expires as i64), received_at: UTCDateTime::now(), report: report.into(), }; - report.member_tenant_id = tenant_ids( - &core, - report - .domains() - .filter_map(psl::domain_str) - .collect::>(), - ) - .await; report.write_ops(&mut batch, item_id, true); } Format::Arf(report) => { + #[cfg_attr(not(feature = "enterprise"), allow(unused_mut))] let mut report = ArfExternalReport { from, to: to.into(), subject, - member_tenant_id: None, expires_at: UTCDateTime::from_timestamp(expires as i64), received_at: UTCDateTime::now(), report: report.into(), }; - report.member_tenant_id = tenant_ids( - &core, - report - .domains() - .filter_map(psl::domain_str) - .collect::>(), - ) - .await; report.write_ops(&mut batch, item_id, true); } } - if let Err(err) = core.core.storage.data.write(batch.build_all()).await + if let Err(err) = core.core.storage.data.write_batch(&mut batch).await && !err.is_assertion_failure() { trc::error!( @@ -357,33 +328,6 @@ impl AnalyzeReport for Server { } } -async fn tenant_ids(server: &Server, domains: AHashSet<&str>) -> Option { - let mut tenant_ids = Vec::with_capacity(domains.len()); - for domain in domains { - if let Some(tenant_id) = server - .domain(domain) - .await - .map_err(|err| { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to lookup domain") - ); - }) - .unwrap_or_default() - .and_then(|domain| domain.id_tenant) - .map(Id::from) - && !tenant_ids.contains(&tenant_id) - { - tenant_ids.push(tenant_id); - } - } - - if tenant_ids.len() == 1 { - tenant_ids.into_iter().next() - } else { - None - } -} fn read_capped( reader: impl Read, diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs index c425c8c7..e7172d94 100644 --- a/crates/smtp/src/reporting/dkim.rs +++ b/crates/smtp/src/reporting/dkim.rs @@ -4,12 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{core::Session, reporting::send::MtaReportSend}; -use common::network::SessionStream; -use mail_auth::{ - AuthenticatedMessage, AuthenticationResults, DkimOutput, common::verify::VerifySignature, +use crate::{ + core::Session, + reporting::{ReportAddress, send::MtaReportSend}, }; +use common::{expr::Bump, network::SessionStream}; +use compact_str::CompactString; +use mail_auth::{AuthenticatedMessage, AuthenticationResults, DkimOutput}; use registry::schema::structs::Rate; +use smtp_report::{ReportEnvelope, arf::FeedbackReport}; use trc::OutgoingReportEvent; impl Session { @@ -21,6 +24,10 @@ impl Session { rejected: bool, output: &DkimOutput<'_>, ) { + if ReportAddress::checked(rcpt, self.data.session_id).is_none() { + return; + } + // Generate report let signature = if let Some(signature) = output.signature() { signature @@ -30,7 +37,7 @@ impl Session { if self .server - .is_local_report_domain(signature.domain(), self.data.session_id) + .is_local_report_domain(&signature.d, self.data.session_id) .await { return; @@ -41,57 +48,73 @@ impl Session { trc::event!( OutgoingReport(OutgoingReportEvent::DkimRateLimited), SpanId = self.data.session_id, - To = rcpt.to_string(), - Limit = vec![ - trc::Value::from(rate.count), - trc::Value::from(rate.period.into_inner()) - ], + To = CompactString::from(rcpt), + Limit = rate.count, + Period = rate.period.into_inner(), ); return; } let config = &self.server.core.smtp.report.dkim; + let mut arena = Bump::new(); let from_addr = self .server - .eval_if(&config.address, self, self.data.session_id) + .eval_if(&config.address, self, &mut arena, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + let from_name = self + .server + .eval_if(&config.name, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()); + let subject = self + .server + .eval_if(&config.subject, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "DKIM Report".to_string()); let mut report = Vec::with_capacity(128); - self.new_auth_failure(output.result().into(), rejected) - .with_authentication_results( + if let Err(err) = (FeedbackReport { + authentication_results: vec![ AuthenticationResults::new(&self.hostname) - .with_dkim_result(output, message.from()) - .to_string(), - ) - .with_dkim_domain(signature.domain()) - .with_dkim_selector(signature.selector()) - .with_dkim_identity(signature.identity()) - .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()) - .write_rfc5322( - ( - self.server - .eval_if(&config.name, self, self.data.session_id) - .await - .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) - .as_str(), - from_addr.as_str(), - ), - rcpt, - &self - .server - .eval_if(&config.subject, self, self.data.session_id) - .await - .unwrap_or_else(|| "DKIM Report".to_string()), - &mut report, - ) - .ok(); + .with_dkim_result(output, message.first_from_address()) + .to_string() + .into(), + ], + dkim_domain: Some(signature.d.as_str().into()), + dkim_selector: Some(signature.s.as_str().into()), + dkim_identity: Some(signature.identity().into()), + headers: Some( + std::str::from_utf8(message.raw_headers()) + .unwrap_or_default() + .into(), + ), + ..self.new_auth_failure(output.result().into(), rejected) + }) + .write_rfc5322( + &ReportEnvelope { + from: (from_name.as_str(), from_addr.as_str()).into(), + to: vec![rcpt], + submitter: &self.hostname, + report_domain: "", + subject: Some(&subject), + }, + &mut report, + ) { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = self.data.session_id, + To = CompactString::from(rcpt), + Reason = err.to_string(), + ); + return; + } trc::event!( OutgoingReport(OutgoingReportEvent::DkimReport), SpanId = self.data.session_id, - From = from_addr.to_string(), - To = rcpt.to_string(), + From = CompactString::from(&from_addr), + To = CompactString::from(rcpt), ); // Send report diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 57f1272c..0ecd319c 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -4,40 +4,49 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::AggregateTimestamp; use crate::{ core::Session, queue::RecipientDomain, - reporting::{index::InternalReportIndex, send::MtaReportSend}, + reporting::{ + ReportAddress, + aggregate::{ + ReportItem, ReportScheduler, ReportWindow, read_dmarc_records, report_size_limit, + }, + index::InternalReportIndex, + send::MtaReportSend, + }, }; use common::{ Server, config::smtp::report::AggregateFrequency, - ipc::{DmarcEvent, ToHash}, + expr::Bump, + ipc::{DmarcEvent, ReportingEvent}, network::SessionStream, }; -use compact_str::ToCompactString; +use compact_str::{CompactString, ToCompactString}; use mail_auth::{ ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, DmarcResult, SpfResult, - common::verify::VerifySignature, dkim2::Dkim2Output, - dmarc::{self}, - report::{AuthFailureType, IdentityAlignment, PolicyPublished, Record, SPFDomainScope}, + dmarc::{self, FailureOptions}, }; use registry::{ schema::{ enums::FailureReportingOption, - prelude::{ObjectType, Property}, - structs::{DmarcInternalReport, DmarcReport, DmarcReportRecord, Rate}, + prelude::ObjectType, + structs::{DmarcInternalReport, DmarcReport, Rate}, }, - types::{EnumImpl, ObjectImpl, datetime::UTCDateTime, map::Map}, + types::{EnumImpl, datetime::UTCDateTime, map::Map}, +}; +use smtp_report::{ + ReportEnvelope, + arf::{AuthFailureType, FeedbackReport, IdentityAlignment}, + dmarc::{AggregateReport, PolicyPublished, Record, SpfScope}, }; use std::{borrow::Cow, future::Future}; use store::{ - SerializeInfallible, U64_LEN, ValueKey, - registry::ObjectIdVersioned, - write::{BatchBuilder, RegistryClass, ValueClass, assert::AssertValue, key::KeySerializer}, + ValueKey, + write::{BatchBuilder, RegistryClass, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::DomainPart; @@ -54,8 +63,11 @@ impl Session { dkim2_output: Option<&Dkim2Output<'_>>, arc_output: &Option>, ) { - let dmarc_record = dmarc_output.dmarc_record_cloned().unwrap(); + let Some(dmarc_record) = dmarc_output.record().cloned() else { + return; + }; let config = &self.server.core.smtp.report.dmarc; + let mut arena = Bump::new(); if self .server @@ -70,7 +82,7 @@ impl Session { if !matches!(dmarc_record.psd, dmarc::Psd::Yes) && let (Some(failure_rate), Some(report_options)) = ( self.server - .eval_if::(&config.send, self, self.data.session_id) + .eval_if::(&config.send, self, &mut arena, self.data.session_id) .await, dmarc_output.failure_report(), ) @@ -82,19 +94,20 @@ impl Session { .smtp .resolvers .dns - .verify_dmarc_report_address( - dmarc_output.domain(), - dmarc_record.ruf(), - Some(&self.server.inner.cache.dns_txt), + .authorized_report_addresses( + self.server + .inner + .cache + .build_auth_parameters((dmarc_output.domain(), dmarc_record.ruf())), ) .await { - Some(rcpts) => { + Ok(rcpts) => { if !rcpts.is_empty() { let mut new_rcpts = Vec::with_capacity(rcpts.len()); for rcpt in rcpts { - if self.throttle_rcpt(rcpt.uri(), &failure_rate, "dmarc").await { + if ReportAddress::checked(rcpt.uri(), self.data.session_id).is_some() { new_rcpts.push(rcpt.uri()); } } @@ -115,7 +128,7 @@ impl Session { vec![] } } - None => { + Err(_) => { trc::event!( OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError), SpanId = self.data.session_id, @@ -130,28 +143,31 @@ impl Session { } }; - // Throttle recipient if !rcpts.is_empty() { - let mut report = Vec::with_capacity(128); let from_addr = self .server - .eval_if(&config.address, self, self.data.session_id) + .eval_if(&config.address, self, &mut arena, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_compact_string()); - let mut auth_failure = self - .new_auth_failure(AuthFailureType::Dmarc, rejected) - .with_authentication_results(auth_results.to_string()) - .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()); + let mut auth_failure = FeedbackReport { + authentication_results: vec![auth_results.to_string().into()], + headers: Some( + std::str::from_utf8(message.raw_headers()) + .unwrap_or_default() + .into(), + ), + ..self.new_auth_failure(AuthFailureType::Dmarc, rejected) + }; let dkim_aligned = matches!(dmarc_output.dkim_result(), DmarcResult::Pass); let spf_aligned = matches!(dmarc_output.spf_result(), DmarcResult::Pass); // Report the first failed signature if let ( - dmarc::Report::Dkim - | dmarc::Report::DkimSpf - | dmarc::Report::All - | dmarc::Report::Any, + FailureOptions::Dkim + | FailureOptions::DkimSpf + | FailureOptions::All + | FailureOptions::Any, Some(signature), ) = ( &report_options, @@ -171,18 +187,17 @@ impl Session { None }, ) { - auth_failure = auth_failure - .with_dkim_domain(signature.domain()) - .with_dkim_selector(signature.selector()) - .with_dkim_identity(signature.identity()); + auth_failure.dkim_domain = Some(signature.d.as_str().into()); + auth_failure.dkim_selector = Some(signature.s.as_str().into()); + auth_failure.dkim_identity = Some(signature.identity().into()); } // Report SPF failure if let ( - dmarc::Report::Spf - | dmarc::Report::DkimSpf - | dmarc::Report::All - | dmarc::Report::Any, + FailureOptions::Spf + | FailureOptions::DkimSpf + | FailureOptions::All + | FailureOptions::Any, Some(output), ) = ( &report_options, @@ -211,66 +226,101 @@ impl Session { None }, ) { - auth_failure = - auth_failure.with_spf_dns(format!("txt : {} : v=SPF1", output.domain())); + auth_failure.spf_dns = + Some(format!("txt : {} : v=SPF1", output.domain()).into()); // TODO use DNS record } - auth_failure - .with_identity_alignment(match (dkim_aligned, spf_aligned) { - (false, false) => IdentityAlignment::DkimSpf, - (false, true) => IdentityAlignment::Dkim, - (true, false) => IdentityAlignment::Spf, - (true, true) => IdentityAlignment::None, - }) - .write_rfc5322( - ( - self.server - .eval_if(&config.name, self, self.data.session_id) - .await - .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()) - .as_str(), - from_addr.as_str(), - ), - &rcpts.join(", "), - &self - .server - .eval_if(&config.subject, self, self.data.session_id) - .await - .unwrap_or_else(|| "DMARC Report".to_compact_string()), - &mut report, - ) - .ok(); + auth_failure.identity_alignment = match (dkim_aligned, spf_aligned) { + (false, false) => IdentityAlignment::DkimSpf, + (false, true) => IdentityAlignment::Dkim, + (true, false) => IdentityAlignment::Spf, + (true, true) => IdentityAlignment::None, + }; + let from_name = self + .server + .eval_if(&config.name, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()); + let subject = self + .server + .eval_if(&config.subject, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "DMARC Report".to_compact_string()); + let write_report = |to| { + let envelope = ReportEnvelope { + from: (from_name.as_str(), from_addr.as_str()).into(), + to, + submitter: &self.hostname, + report_domain: "", + subject: Some(&subject), + }; + let mut report = Vec::with_capacity(128); + auth_failure + .write_rfc5322(&envelope, &mut report) + .map(|_| (report, envelope.to)) + }; + let report = match write_report(rcpts) { + Ok((report, validated)) => { + let mut allowed = Vec::with_capacity(validated.len()); + for rcpt in &validated { + if self.throttle_rcpt(rcpt, &failure_rate, "dmarc").await { + allowed.push(*rcpt); + } + } + match allowed.len() { + 0 => None, + len if len == validated.len() => Some(Ok((report, allowed))), + _ => Some(write_report(allowed)), + } + } + Err(err) => Some(Err(err)), + }; + match report { + Some(Ok((report, rcpts))) => { + trc::event!( + OutgoingReport(OutgoingReportEvent::DmarcReport), + SpanId = self.data.session_id, + From = from_addr.clone(), + To = rcpts + .iter() + .map(|a| trc::Value::String(a.to_compact_string())) + .collect::>(), + ); - trc::event!( - OutgoingReport(OutgoingReportEvent::DmarcReport), - SpanId = self.data.session_id, - From = from_addr.to_string(), - To = rcpts - .iter() - .map(|a| trc::Value::String(a.to_compact_string())) - .collect::>(), - ); - - // Send report - self.server - .send_report( - &from_addr, - rcpts.into_iter(), - report, - &config.sign, - true, - self.data.session_id, - ) - .await; + self.server + .send_report( + &from_addr, + rcpts.into_iter(), + report, + &config.sign, + true, + self.data.session_id, + ) + .await; + } + Some(Err(err)) => { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = self.data.session_id, + Reason = err.to_string(), + ); + } + None => { + trc::event!( + OutgoingReport(OutgoingReportEvent::DmarcRateLimited), + SpanId = self.data.session_id, + Limit = failure_rate.count, + Period = failure_rate.period.into_inner(), + ); + } + } } else { trc::event!( OutgoingReport(OutgoingReportEvent::DmarcRateLimited), SpanId = self.data.session_id, - Limit = vec![ - trc::Value::from(failure_rate.count), - trc::Value::from(failure_rate.period.into_inner()) - ], + Limit = failure_rate.count, + Period = failure_rate.period.into_inner(), ); } } @@ -281,6 +331,7 @@ impl Session { .eval_if( &self.server.core.smtp.report.dmarc_aggregate.send, self, + &mut arena, self.data.session_id, ) .await @@ -291,7 +342,7 @@ impl Session { } // Report the same identifier forms that were used for alignment - let message_from = message.from(); + let message_from = message.first_from_address(); let header_from = message_from.domain_part(); let header_from = header_from .to_ascii_domain() @@ -307,17 +358,17 @@ impl Session { .unwrap_or(Cow::Borrowed(envelope_from)); // Create DMARC report record - let mut report_record = Record::new() + let mut report_record = Record::default() .with_dmarc_output(&dmarc_output) - .with_dkim_output(dkim_output) - .with_source_ip(self.data.remote_ip) - .with_header_from(header_from.as_ref()) - .with_envelope_from(envelope_from.as_ref()); + .with_dkim_output(dkim_output); + report_record.row.source_ip = Some(self.data.remote_ip); + report_record.identifiers.header_from = header_from.into_owned(); + report_record.identifiers.envelope_from = envelope_from.into_owned(); if let Some(dkim2_output) = dkim2_output { report_record = report_record.with_dkim2_output(dkim2_output); } if let Some(spf_mail_from) = &self.data.spf_mail_from { - report_record = report_record.with_spf_output(spf_mail_from, SPFDomainScope::MailFrom); + report_record = report_record.with_spf_output(spf_mail_from, SpfScope::MailFrom); } if let Some(arc_output) = arc_output { report_record = report_record.with_arc_output(arc_output); @@ -349,7 +400,7 @@ impl DmarcReporting for Server { let object_id = ObjectType::DmarcInternalReport.to_id(); let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id }); - let Some(report) = self + let Some(mut report) = self .store() .get_value::(ValueKey::from(key.clone())) .await @@ -357,24 +408,71 @@ impl DmarcReporting for Server { else { return Ok(()); }; + let span_id = self.inner.data.span_id_gen.generate(); + let max_size = self + .eval_if( + &self.core.smtp.report.dmarc_aggregate.max_size, + &RecipientDomain::new(report.domain.as_str()), + &mut Bump::new(), + span_id, + ) + .await; + read_dmarc_records( + self.store(), + item_id, + &mut report, + report_size_limit(max_size), + ) + .await?; + + let mut batch = BatchBuilder::new(); + batch.clear(key).clear(report.primary_key()); + let item = ReportItem::new(ObjectType::DmarcInternalReport, item_id); + item.clear(&mut batch); + let now = UTCDateTime::now(); + if now < report.report.date_range_end { + report.report.date_range_end = now; + } // Delete report - let mut batch = BatchBuilder::new(); - batch.clear(key).clear(RegistryClass::PrimaryKey { - object_id: object_id.into(), - index_id: Property::Domain.to_id(), - key: KeySerializer::new(report.domain.len() + U64_LEN) - .write(&report.domain) - .write(report.policy_identifier) - .finalize(), - }); self.store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; + if let Err(err) = item.clear_orphans(self.store()).await { + trc::error!( + err.span_id(span_id) + .details("Failed to clear orphaned report rows") + ); + } - let span_id = self.inner.data.span_id_gen.generate(); - let event_from = report.report.date_range_begin.timestamp() as u64; + if let Some(message) = DmarcMessage::build(self, report, span_id).await { + message.queue(self, span_id).await; + } + Ok(()) + } + + async fn schedule_dmarc(&self, event: Box) { + ReportScheduler::default() + .process(self, &mut vec![ReportingEvent::Dmarc(event)]) + .await; + } +} + +struct DmarcMessage { + from_addr: CompactString, + rcpts: Vec, + message: Vec, +} + +impl DmarcMessage { + async fn build(server: &Server, report: DmarcInternalReport, span_id: u64) -> Option { + if report.report.records.is_empty() { + return None; + } + let config = &server.core.smtp.report.dmarc_aggregate; + let mut arena = Bump::new(); + let event_from = report.report.date_range_start.timestamp() as u64; let event_to = report.report.date_range_end.timestamp() as u64; trc::event!( @@ -387,19 +485,25 @@ impl DmarcReporting for Server { ); // Verify external reporting addresses - let rua = match self + let rua = match server .core .smtp .resolvers .dns - .verify_dmarc_report_address( - &report.domain, - report.rua.as_slice(), - Some(&self.inner.cache.dns_txt), + .authorized_report_addresses( + server + .inner + .cache + .build_auth_parameters((report.domain.as_str(), report.rua.as_slice())), ) .await { - Some(rcpts) => { + Ok(rcpts) => { + let rcpts = rcpts + .into_iter() + .filter(|rcpt| ReportAddress::checked(rcpt.as_str(), span_id).is_some()) + .map(|rcpt| rcpt.as_str().to_string()) + .collect::>(); if !rcpts.is_empty() { rcpts } else { @@ -413,10 +517,10 @@ impl DmarcReporting for Server { .collect::>(), ); - return Ok(()); + return None; } } - None => { + Err(_) => { trc::event!( OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError), SpanId = span_id, @@ -427,266 +531,151 @@ impl DmarcReporting for Server { .collect::>(), ); - return Ok(()); + return None; } }; // Serialize report - let config = &self.core.smtp.report.dmarc_aggregate; - let from_addr = self + let from_addr = server .eval_if( &config.address, &RecipientDomain::new(report.domain.as_str()), + &mut arena, span_id, ) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_compact_string()); + let submitter = server + .eval_if( + &server.core.smtp.report.submitter, + &RecipientDomain::new(report.domain.as_str()), + &mut arena, + span_id, + ) + .await + .unwrap_or_else(|| "localhost".to_compact_string()); + let from_name = server + .eval_if( + &config.name, + &RecipientDomain::new(report.domain.as_str()), + &mut arena, + span_id, + ) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()); let mut message = Vec::with_capacity(2048); - let _ = mail_auth::report::Report::from(report.report).write_rfc5322( - &self - .eval_if( - &self.core.smtp.report.submitter, - &RecipientDomain::new(report.domain.as_str()), - span_id, - ) - .await - .unwrap_or_else(|| "localhost".to_compact_string()), - ( - self.eval_if( - &config.name, - &RecipientDomain::new(report.domain.as_str()), - span_id, - ) - .await - .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()) - .as_str(), - from_addr.as_str(), - ), - rua.iter().map(|a| a.as_str()), + if let Err(err) = AggregateReport::from(report.report).write_rfc5322( + &ReportEnvelope { + from: (from_name.as_str(), from_addr.as_str()).into(), + to: rua.iter().map(|a| a.as_str()).collect(), + submitter: &submitter, + report_domain: "", + subject: None, + }, &mut message, - ); + ) { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = span_id, + Reason = err.to_string(), + ); + return None; + } - // Send report - self.send_report( - &from_addr, - rua.iter(), + Some(DmarcMessage { + from_addr, + rcpts: rua, message, - &config.sign, - false, - span_id, - ) - .await; - - Ok(()) + }) } - async fn schedule_dmarc(&self, event: Box) { - let object_id = ObjectType::DmarcInternalReport.to_id(); - let policy_hash = event.dmarc_record.to_hash(); - let pk = ValueClass::Registry(RegistryClass::PrimaryKey { - object_id: object_id.into(), - index_id: Property::Domain.to_id(), - key: KeySerializer::new(event.domain.len() + U64_LEN) - .write(&event.domain) - .write(policy_hash) - .finalize(), - }); - let mut rety_count = 0; - - loop { - // Find the report by domain name - let mut batch = BatchBuilder::new(); - let report = match self - .store() - .get_value::(ValueKey::from(pk.clone())) - .await - { - Ok(Some(object_id_v)) => { - match self - .store() - .get_value::(ValueKey::from(ValueClass::Registry( - RegistryClass::Item { - object_id, - item_id: object_id_v.object_id.id().id(), - }, - ))) - .await - { - Ok(Some(report)) => Some((object_id_v, report)), - Ok(None) => { - trc::event!( - OutgoingReport(OutgoingReportEvent::NotFound), - Id = object_id_v.object_id.id().id(), - CausedBy = trc::location!(), - Details = "Failed to find DMARC report for domain" - ); - - return; - } - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to query registry for DMARC report") - ); - return; - } - } - } - Ok(None) => None, - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to query registry for DMARC report") - ); - return; - } - }; - - // Create report if missing - let config = &self.core.smtp.report.dmarc_aggregate; - let (item_id, mut report) = if let Some((mut object_id_v, report)) = report { - batch.assert_value(pk.clone(), AssertValue::U32(object_id_v.version)); - object_id_v.version += 1; - batch.set(pk.clone(), object_id_v.serialize()); - - (object_id_v.object_id.id().id(), report) - } else { - let item_id = self.inner.data.queue_id_gen.generate(); - let date_range_begin = UTCDateTime::now(); - let date_range_end = UTCDateTime::from_timestamp( - date_range_begin.timestamp() + event.interval.as_secs() as i64, - ); - let policy = - PolicyPublished::from_record(event.domain.clone(), &event.dmarc_record); - - let report = DmarcInternalReport { - created_at: date_range_begin, - deliver_at: date_range_end, - domain: event.domain.clone(), - report: DmarcReport { - report_id: format!("{}_{policy_hash}", date_range_begin.timestamp()), - date_range_begin, - date_range_end, - email: self - .eval_if( - &config.address, - &RecipientDomain::new(event.domain.as_str()), - event.span_id, - ) - .await - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), - extra_contact_info: self - .eval_if::( - &config.contact_info, - &RecipientDomain::new(event.domain.as_str()), - event.span_id, - ) - .await, - org_name: self - .eval_if::( - &config.org_name, - &RecipientDomain::new(event.domain.as_str()), - event.span_id, - ) - .await - .unwrap_or_default(), - policy_adkim: policy.adkim.into(), - policy_aspf: policy.aspf.into(), - policy_disposition: policy.p.into(), - policy_domain: policy.domain, - policy_failure_reporting_options: match event.dmarc_record.fo { - dmarc::Report::All => vec![FailureReportingOption::All], - dmarc::Report::Any => vec![FailureReportingOption::Any], - dmarc::Report::Dkim => vec![FailureReportingOption::DkimFailure], - dmarc::Report::Spf => vec![FailureReportingOption::SpfFailure], - dmarc::Report::DkimSpf => vec![ - FailureReportingOption::DkimFailure, - FailureReportingOption::SpfFailure, - ], - } - .into(), - policy_subdomain_disposition: policy.sp.into(), - policy_np: policy.np.into(), - policy_discovery_method: policy.discovery_method.into(), - policy_testing_mode: policy.testing, - policy_version: None, - version: 1.0.into(), - ..Default::default() - }, - policy_identifier: policy_hash, - rua: Map::new( - event - .dmarc_record - .rua() - .iter() - .map(|u| u.uri.clone()) - .collect(), - ), - }; - - report.write_ops(&mut batch, item_id, true); - - (item_id, report) - }; - - // Add record - let mut record = DmarcReportRecord::from(event.report_record.clone()); - if let Some(idx) = report - .report - .records - .0 - .inner - .iter() - .position(|d| d.value.eq_except_count(&record)) - { - report.report.records.0.inner[idx].value.count += 1; - } else { - record.count = 1; - report.report.records.push(record); - } - - // Write entry - let report_bytes = report.to_pickled_vec(); - let max_report_size = self - .eval_if( - &config.max_size, - &RecipientDomain::new(&event.domain), - event.span_id, - ) - .await - .unwrap_or(5 * 1024 * 1024); - if max_report_size != 0 && report_bytes.len() > max_report_size { - trc::event!( - OutgoingReport(OutgoingReportEvent::MaxSizeExceeded), - SpanId = event.span_id, - Domain = event.domain.clone(), - Details = report_bytes.len(), - Limit = max_report_size, - ); - return; - } - - batch.set( - ValueClass::Registry(RegistryClass::Item { object_id, item_id }), - report_bytes, - ); - - match self.core.storage.data.write(batch.build_all()).await { - Ok(_) => { - break; - } - Err(err) => { - if err.is_assertion_failure() && rety_count < 3 { - rety_count += 1; - continue; - } - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to write DMARC report") - ); - break; - } - } - } + async fn queue(self, server: &Server, span_id: u64) { + server + .send_report( + &self.from_addr, + self.rcpts.iter(), + self.message, + &server.core.smtp.report.dmarc_aggregate.sign, + false, + span_id, + ) + .await; } } + +pub(crate) fn dmarc_report_descriptor( + event: &DmarcEvent, + window: ReportWindow, + item: ReportItem, + policy_hash: u64, +) -> DmarcInternalReport { + let date_range_start = UTCDateTime::from_timestamp(window.start.cast_signed()); + let policy = PolicyPublished::from_record(event.domain.clone(), &event.dmarc_record); + + DmarcInternalReport { + created_at: date_range_start, + deliver_at: UTCDateTime::from_timestamp(item.deliver_at(window).cast_signed()), + domain: event.domain.clone(), + report: DmarcReport { + date_range_start, + date_range_end: UTCDateTime::from_timestamp(window.end.cast_signed()), + policy_adkim: policy.adkim.into(), + policy_aspf: policy.aspf.into(), + policy_disposition: policy.p.into(), + policy_domain: policy.domain, + policy_failure_reporting_options: match event.dmarc_record.fo { + FailureOptions::All => vec![FailureReportingOption::All], + FailureOptions::Any => vec![FailureReportingOption::Any], + FailureOptions::Dkim => vec![FailureReportingOption::DkimFailure], + FailureOptions::Spf => vec![FailureReportingOption::SpfFailure], + FailureOptions::DkimSpf => vec![ + FailureReportingOption::DkimFailure, + FailureReportingOption::SpfFailure, + ], + } + .into(), + policy_subdomain_disposition: policy.sp.into(), + policy_np: policy.np.into(), + policy_discovery_method: policy.discovery_method.into(), + policy_testing_mode: policy.testing, + policy_version: None, + version: 1.0.into(), + ..Default::default() + }, + policy_identifier: policy_hash, + rua: Map::new( + event + .dmarc_record + .rua() + .iter() + .map(|u| u.uri.clone()) + .collect(), + ), + } +} + +pub(crate) async fn complete_dmarc_report( + server: &Server, + report: &mut DmarcInternalReport, + arena: &mut Bump, + span_id: u64, +) { + let config = &server.core.smtp.report.dmarc_aggregate; + let domain = RecipientDomain::new(report.domain.as_str()); + report.report.report_id = format!( + "{}_{}", + report.report.date_range_start.timestamp(), + server.inner.data.queue_id_gen.generate() + ); + report.report.email = server + .eval_if(&config.address, &domain, arena, span_id) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + report.report.extra_contact_info = server + .eval_if::(&config.contact_info, &domain, arena, span_id) + .await; + report.report.org_name = server + .eval_if::(&config.org_name, &domain, arena, span_id) + .await + .unwrap_or_default(); +} diff --git a/crates/smtp/src/reporting/inbound.rs b/crates/smtp/src/reporting/inbound.rs index 82f1e5b5..5c2490ed 100644 --- a/crates/smtp/src/reporting/inbound.rs +++ b/crates/smtp/src/reporting/inbound.rs @@ -7,9 +7,11 @@ use crate::core::Session; use ahash::AHashMap; use common::USER_AGENT; -use mail_auth::report::{ - ActionDisposition, AuthFailureType, DeliveryResult, DmarcResult, Feedback, FeedbackType, - Report, tlsrpt::TlsReport, +use compact_str::{CompactString, format_compact}; +use smtp_report::{ + arf::{AuthFailureType, DeliveryResult, FeedbackReport, FeedbackType}, + dmarc::{AggregateReport, Disposition, DmarcStatus}, + tlsrpt::TlsReport, }; use std::{collections::hash_map::Entry, time::SystemTime}; use store::write::now; @@ -17,22 +19,24 @@ use tokio::io::{AsyncRead, AsyncWrite}; use trc::IncomingReportEvent; impl Session { - pub fn new_auth_failure(&self, ft: AuthFailureType, rejected: bool) -> Feedback<'_> { - Feedback::new(FeedbackType::AuthFailure) - .with_auth_failure(ft) - .with_arrival_date( + pub fn new_auth_failure(&self, ft: AuthFailureType, rejected: bool) -> FeedbackReport<'_> { + FeedbackReport { + auth_failure: ft, + arrival_date: Some( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) as i64, - ) - .with_source_ip(self.data.remote_ip) - .with_reporting_mta(&self.hostname) - .with_user_agent(USER_AGENT) - .with_delivery_result(if rejected { + ), + source_ip: Some(self.data.remote_ip), + reporting_mta: Some(self.hostname.as_str().into()), + user_agent: Some(USER_AGENT.into()), + delivery_result: if rejected { DeliveryResult::Reject } else { DeliveryResult::Unspecified - }) + }, + ..FeedbackReport::new(FeedbackType::AuthFailure) + } } pub fn is_report(&self) -> bool { @@ -49,7 +53,7 @@ pub(crate) trait LogReport { fn log(&self); } -impl LogReport for Report { +impl LogReport for AggregateReport { fn log(&self) { let mut dmarc_pass = 0; let mut dmarc_quarantine = 0; @@ -62,42 +66,43 @@ impl LogReport for Report { let mut spf_fail = 0; let mut spf_none = 0; - for record in self.records() { - let count = std::cmp::min(record.count(), 1); + for record in &self.records { + let count = std::cmp::min(record.row.count, 1); + let evaluated = &record.row.policy_evaluated; - match record.action_disposition() { - ActionDisposition::Pass => { + match evaluated.disposition { + Disposition::Pass => { dmarc_pass += count; } - ActionDisposition::Quarantine => { + Disposition::Quarantine => { dmarc_quarantine += count; } - ActionDisposition::Reject => { + Disposition::Reject => { dmarc_reject += count; } - ActionDisposition::None | ActionDisposition::Unspecified => { + Disposition::None | Disposition::Unspecified => { dmarc_none += count; } } - match record.dmarc_dkim_result() { - DmarcResult::Pass => { + match evaluated.dkim { + DmarcStatus::Pass => { dkim_pass += count; } - DmarcResult::Fail => { + DmarcStatus::Fail => { dkim_fail += count; } - DmarcResult::Unspecified => { + DmarcStatus::Unspecified => { dkim_none += count; } } - match record.dmarc_spf_result() { - DmarcResult::Pass => { + match evaluated.spf { + DmarcStatus::Pass => { spf_pass += count; } - DmarcResult::Fail => { + DmarcStatus::Fail => { spf_fail += count; } - DmarcResult::Unspecified => { + DmarcStatus::Unspecified => { spf_none += count; } } @@ -111,11 +116,11 @@ impl LogReport for Report { IncomingReportEvent::DmarcReport } ), - RangeFrom = trc::Value::Timestamp(self.date_range_begin()), - RangeTo = trc::Value::Timestamp(self.date_range_end()), - Domain = self.domain().to_string(), - From = self.email().to_string(), - Id = self.report_id().to_string(), + RangeFrom = trc::Value::Timestamp(self.report_metadata.date_range.begin), + RangeTo = trc::Value::Timestamp(self.report_metadata.date_range.end), + Domain = CompactString::from(&self.policy_published.domain), + From = CompactString::from(&self.report_metadata.email), + Id = CompactString::from(&self.report_metadata.report_id), DmarcPass = dmarc_pass, DmarcQuarantine = dmarc_quarantine, DmarcReject = dmarc_reject, @@ -147,7 +152,7 @@ impl LogReport for TlsReport { } trc::event!( - IncomingReport(if policy.summary.total_failure > 0 { + IncomingReport(if policy.summary.failed_sessions > 0 { IncomingReportEvent::TlsReportWithWarnings } else { IncomingReportEvent::TlsReport @@ -156,50 +161,52 @@ impl LogReport for TlsReport { trc::Value::Timestamp(self.date_range.start_datetime.to_timestamp() as u64), RangeTo = trc::Value::Timestamp(self.date_range.end_datetime.to_timestamp() as u64), Domain = policy.policy.policy_domain.clone(), - From = self.contact_info.as_deref().unwrap_or_default().to_string(), + From = CompactString::from(self.contact_info.as_deref().unwrap_or_default()), Id = self.report_id.clone(), - Policy = format!("{:?}", policy.policy.policy_type), - TotalSuccesses = policy.summary.total_success, - TotalFailures = policy.summary.total_failure, - Details = format!("{details:?}"), + Policy = format_compact!("{:?}", policy.policy.policy_type), + TotalSuccesses = policy.summary.successful_sessions, + TotalFailures = policy.summary.failed_sessions, + Details = format_compact!("{details:?}"), ); } } } -impl LogReport for Feedback<'_> { +impl LogReport for FeedbackReport<'_> { fn log(&self) { trc::event!( - IncomingReport(match self.feedback_type() { - mail_auth::report::FeedbackType::Abuse => IncomingReportEvent::AbuseReport, - mail_auth::report::FeedbackType::AuthFailure => - IncomingReportEvent::AuthFailureReport, - mail_auth::report::FeedbackType::Fraud => IncomingReportEvent::FraudReport, - mail_auth::report::FeedbackType::NotSpam => IncomingReportEvent::NotSpamReport, - mail_auth::report::FeedbackType::Other => IncomingReportEvent::OtherReport, - mail_auth::report::FeedbackType::Virus => IncomingReportEvent::VirusReport, + IncomingReport(match self.feedback_type { + FeedbackType::Abuse => IncomingReportEvent::AbuseReport, + FeedbackType::AuthFailure => IncomingReportEvent::AuthFailureReport, + FeedbackType::Fraud => IncomingReportEvent::FraudReport, + FeedbackType::NotSpam => IncomingReportEvent::NotSpamReport, + FeedbackType::Other => IncomingReportEvent::OtherReport, + FeedbackType::Virus => IncomingReportEvent::VirusReport, }), RangeFrom = trc::Value::Timestamp( - self.arrival_date() + self.arrival_date .map(|d| d as u64) .unwrap_or_else(|| { now() }) ), Domain = self - .reported_domain() + .reported_domains .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), - Hostname = self.reporting_mta().map(|d| trc::Value::String(d.into())), + Hostname = self + .reporting_mta + .as_deref() + .map(|d| trc::Value::String(d.into())), Url = self - .reported_uri() + .reported_uris .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), - RemoteIp = self.source_ip(), - Total = self.incidents(), - Result = format!("{:?}", self.delivery_result()), + RemoteIp = self.source_ip, + Total = self.incidents, + Result = format_compact!("{:?}", self.delivery_result), Details = self - .authentication_results() + .authentication_results .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), diff --git a/crates/smtp/src/reporting/index.rs b/crates/smtp/src/reporting/index.rs index 83cc9935..733b340b 100644 --- a/crates/smtp/src/reporting/index.rs +++ b/crates/smtp/src/reporting/index.rs @@ -5,8 +5,9 @@ */ use registry::{ + pickle::Pickle, schema::{ - enums::DmarcActionDisposition, + enums::{DmarcActionDisposition, TaskType}, prelude::{ObjectType, Property}, structs::{ ArfExternalReport, DmarcExternalReport, DmarcInternalReport, Task, TaskDmarcReport, @@ -24,14 +25,15 @@ use store::{ SerializeInfallible, U64_LEN, registry::ObjectIdVersioned, write::{ - BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, assert::AssertValue, + BatchBuilder, RegistryClass, TaskId, TaskQueueClass, ValueClass, assert::AssertValue, key::KeySerializer, }, xxhash_rust::xxh3::Xxh3, }; -use types::id::Id; -pub trait InternalReportIndex: ObjectImpl { +pub trait InternalReportIndex: ObjectImpl + Pickle { + const TASK_TYPE: TaskType; + fn deliver_at(&self) -> UTCDateTime; fn set_deliver_at(&mut self, at: UTCDateTime); @@ -59,20 +61,46 @@ pub trait InternalReportIndex: ObjectImpl { batch .assert_value(key.clone(), AssertValue::Hash(revision)) .clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id: item_id, + id: TaskId::Assigned(item_id), + typ: Self::TASK_TYPE, due: current_deliver_at.timestamp() as u64, })) - .set( - ValueClass::TaskQueue(TaskQueueClass::Due { - id: item_id, - due: at.timestamp() as u64, - }), - object_id.serialize(), - ) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(item_id), + typ: Self::TASK_TYPE, + due: u64::MAX, + })) + .schedule_task_with_id(item_id, self.task(item_id)) .set(key, self.to_pickled_vec()); } } + fn write_header(&self, batch: &mut BatchBuilder, item_id: u64) -> usize { + let object = Self::OBJECT; + let value = self.to_pickled_vec(); + let len = value.len(); + + batch + .set( + self.primary_key(), + ObjectIdVersioned { + object_id: ObjectId::new(object, item_id.into()), + version: 0, + } + .serialize(), + ) + .schedule_task_with_id(item_id, self.task(item_id)) + .set( + ValueClass::Registry(RegistryClass::Item { + object_id: object.to_id(), + item_id, + }), + value, + ); + + len + } + fn write_ops(&self, batch: &mut BatchBuilder, item_id: u64, is_set: bool) { let object = Self::OBJECT; let object_id = object.to_id(); @@ -97,19 +125,26 @@ pub trait InternalReportIndex: ObjectImpl { item_id, })) .clear(pk) - .clear(ValueClass::TaskQueue(TaskQueueClass::Task { id: item_id })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Task { + id: TaskId::Assigned(item_id), + })) .clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id: item_id, + id: TaskId::Assigned(item_id), + typ: Self::TASK_TYPE, due: self.deliver_at().timestamp() as u64, + })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(item_id), + typ: Self::TASK_TYPE, + due: u64::MAX, })); } } } -pub trait ExternalReportIndex: ObjectImpl { +pub trait ExternalReportIndex: ObjectImpl + Pickle { fn text(&self) -> impl Iterator; - fn tenant_id(&self) -> Option; fn expires_at(&self) -> u64; @@ -126,9 +161,6 @@ pub trait ExternalReportIndex: ObjectImpl { index_builder.text(Property::Text, text); } - if let Some(tenant_id) = self.tenant_id() { - index_builder.search(Property::MemberTenantId, tenant_id.id()); - } let (success_count, fail_count) = self.success_fail_count(); index_builder.search(Property::TotalSuccessfulSessions, success_count); @@ -151,7 +183,27 @@ pub trait ExternalReportIndex: ObjectImpl { } } +pub(crate) fn internal_report_key( + object: ObjectType, + domain: &str, + policy: Option, + window_end: u64, +) -> ValueClass { + let mut key = KeySerializer::new(domain.len() + 1 + (U64_LEN * 2)); + key.write(domain).write(0u8); + if let Some(policy) = policy { + key.write(policy); + } + ValueClass::Registry(RegistryClass::PrimaryKey { + object_id: object.to_id().into(), + index_id: Property::Domain.to_id(), + key: key.write(window_end).finalize(), + }) +} + impl InternalReportIndex for DmarcInternalReport { + const TASK_TYPE: TaskType = TaskType::DmarcReport; + fn deliver_at(&self) -> UTCDateTime { self.deliver_at } @@ -168,18 +220,18 @@ impl InternalReportIndex for DmarcInternalReport { } fn primary_key(&self) -> ValueClass { - ValueClass::Registry(RegistryClass::PrimaryKey { - object_id: ObjectType::DmarcInternalReport.to_id().into(), - index_id: Property::Domain.to_id(), - key: KeySerializer::new(self.domain.len() + U64_LEN) - .write(self.domain.as_str()) - .write(self.policy_identifier) - .finalize(), - }) + internal_report_key( + ObjectType::DmarcInternalReport, + &self.domain, + Some(self.policy_identifier), + self.report.date_range_end.timestamp().cast_unsigned(), + ) } } impl InternalReportIndex for TlsInternalReport { + const TASK_TYPE: TaskType = TaskType::TlsReport; + fn deliver_at(&self) -> UTCDateTime { self.deliver_at } @@ -196,11 +248,12 @@ impl InternalReportIndex for TlsInternalReport { } fn primary_key(&self) -> ValueClass { - ValueClass::Registry(RegistryClass::PrimaryKey { - object_id: ObjectType::TlsInternalReport.to_id().into(), - index_id: Property::Domain.to_id(), - key: self.domain.as_bytes().to_vec(), - }) + internal_report_key( + ObjectType::TlsInternalReport, + &self.domain, + None, + self.report.date_range_end.timestamp().cast_unsigned(), + ) } } @@ -241,9 +294,6 @@ impl ExternalReportIndex for ArfExternalReport { .chain(non_empty(&self.from)) } - fn tenant_id(&self) -> Option { - self.member_tenant_id - } fn expires_at(&self) -> u64 { self.expires_at.timestamp() as u64 @@ -287,9 +337,6 @@ impl ExternalReportIndex for DmarcExternalReport { .chain(non_empty(&self.from)) } - fn tenant_id(&self) -> Option { - self.member_tenant_id - } fn expires_at(&self) -> u64 { self.expires_at.timestamp() as u64 @@ -319,7 +366,7 @@ impl ExternalReportIndex for DmarcExternalReport { report.policy_domain.as_str(), report.report_id.as_str(), ], - report.date_range_begin, + report.date_range_start, report.date_range_end, )) } @@ -354,9 +401,6 @@ impl ExternalReportIndex for TlsExternalReport { .chain(non_empty(&self.from)) } - fn tenant_id(&self) -> Option { - self.member_tenant_id - } fn expires_at(&self) -> u64 { self.expires_at.timestamp() as u64 diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index add57eb4..e8c5e57e 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -5,9 +5,13 @@ */ use common::config::smtp::report::AggregateFrequency; +use compact_str::CompactString; use mail_parser::DateTime; use std::time::SystemTime; +use trc::OutgoingReportEvent; +use utils::sanitize_email; +pub mod aggregate; pub mod analysis; pub mod dkim; pub mod dmarc; @@ -18,6 +22,46 @@ pub mod send; pub mod spf; pub mod tls; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReportAddress<'x>(&'x str); + +impl<'x> ReportAddress<'x> { + const MAX_LOCAL_PART_LEN: usize = 64; + const MAX_DOMAIN_LEN: usize = 255; + const MAX_PATH_LEN: usize = 256; + const PATH_PUNCTUATION_LEN: usize = 2; + + pub fn parse(address: &'x str) -> Option { + let (local_part, domain) = address.rsplit_once('@')?; + (local_part.len() <= ReportAddress::MAX_LOCAL_PART_LEN + && domain.len() <= ReportAddress::MAX_DOMAIN_LEN + && address.len() + ReportAddress::PATH_PUNCTUATION_LEN <= ReportAddress::MAX_PATH_LEN + && !address + .chars() + .any(|ch| ch.is_control() || ch.is_whitespace()) + && sanitize_email(address).is_some()) + .then_some(ReportAddress(address)) + } + + pub fn checked(address: &'x str, session_id: u64) -> Option { + let parsed = ReportAddress::parse(address); + if parsed.is_none() { + trc::event!( + OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress), + SpanId = session_id, + Url = CompactString::from(address), + Details = "Invalid reporting address", + ); + } + parsed + } + + #[inline] + pub fn as_str(&self) -> &'x str { + self.0 + } +} + pub trait AggregateTimestamp { fn to_timestamp(&self) -> u64; fn to_timestamp_(&self, dt: DateTime) -> u64; @@ -71,3 +115,4 @@ impl AggregateTimestamp for AggregateFrequency { self.to_timestamp() + self.as_secs() } } + diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index b28b7c27..5b71d0ca 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{dmarc::DmarcReporting, tls::TlsReporting}; +use super::aggregate::{REPORT_BURST, ReportScheduler}; use common::{BuildServer, Inner, ipc::ReportingEvent}; use std::sync::Arc; use tokio::sync::mpsc; @@ -16,12 +16,11 @@ pub trait SpawnReport { impl SpawnReport for mpsc::Receiver { fn spawn(mut self, inner: Arc) { tokio::spawn(async move { - while let Some(event) = self.recv().await { - let server = inner.build_server(); - match event { - ReportingEvent::Dmarc(event) => server.schedule_dmarc(event).await, - ReportingEvent::Tls(event) => server.schedule_tls(event).await, - ReportingEvent::Stop => break, + let mut scheduler = ReportScheduler::default(); + let mut events = Vec::with_capacity(REPORT_BURST); + while self.recv_many(&mut events, REPORT_BURST).await > 0 { + if scheduler.process(&inner.build_server(), &mut events).await { + break; } } }); diff --git a/crates/smtp/src/reporting/send.rs b/crates/smtp/src/reporting/send.rs index 7bde376a..3da6a693 100644 --- a/crates/smtp/src/reporting/send.rs +++ b/crates/smtp/src/reporting/send.rs @@ -10,6 +10,7 @@ use crate::{ MessageSource, spool::{QueueParams, SmtpSpool}, }, + reporting::ReportAddress, }; use common::{Server, expr::if_block::IfBlock, ipc::ReportingEvent}; @@ -73,12 +74,17 @@ impl MtaReportSend for Server { // Build message let mut message = self.new_message(from_addr, MessageSource::Report, parent_session_id); for rcpt_ in rcpts { - message.add_expanded_recipient(rcpt_.as_ref(), self).await; + if let Some(rcpt) = ReportAddress::checked(rcpt_.as_ref(), parent_session_id) { + message.add_expanded_recipient(rcpt.as_str(), self).await; + } + } + if message.message.recipients.is_empty() { + return; } // Schedule delivery at a random time between now and the next 3 hours if !deliver_now { - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { use common::config::smtp::queue::QueueExpiry; use rand::RngExt; diff --git a/crates/smtp/src/reporting/spf.rs b/crates/smtp/src/reporting/spf.rs index 234a5c8f..111ec69a 100644 --- a/crates/smtp/src/reporting/spf.rs +++ b/crates/smtp/src/reporting/spf.rs @@ -4,10 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{core::Session, reporting::send::MtaReportSend}; -use common::network::SessionStream; -use mail_auth::{AuthenticationResults, SpfOutput, report::AuthFailureType}; +use crate::{ + core::Session, + reporting::{ReportAddress, send::MtaReportSend}, +}; +use common::{expr::Bump, network::SessionStream}; +use compact_str::CompactString; +use mail_auth::{AuthenticationResults, SpfOutput, spf::verify::SpfParameters}; use registry::schema::structs::Rate; +use smtp_report::{ + ReportEnvelope, + arf::{AuthFailureType, FeedbackReport}, +}; use trc::OutgoingReportEvent; impl Session { @@ -18,16 +26,18 @@ impl Session { rejected: bool, output: &SpfOutput, ) { + if ReportAddress::checked(rcpt, self.data.session_id).is_none() { + return; + } + // Throttle recipient if !self.throttle_rcpt(rcpt, rate, "spf").await { trc::event!( OutgoingReport(OutgoingReportEvent::SpfRateLimited), SpanId = self.data.session_id, - To = rcpt.to_string(), - Limit = vec![ - trc::Value::from(rate.count), - trc::Value::from(rate.period.into_inner()) - ], + To = CompactString::from(rcpt), + Limit = rate.count, + Period = rate.period.into_inner(), ); return; @@ -35,55 +45,67 @@ impl Session { // Generate report let config = &self.server.core.smtp.report.spf; + let mut arena = Bump::new(); let from_addr = self .server - .eval_if(&config.address, self, self.data.session_id) + .eval_if(&config.address, self, &mut arena, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + let from_name = self + .server + .eval_if(&config.name, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "Mailer Daemon".to_string()); + let subject = self + .server + .eval_if(&config.subject, self, &mut arena, self.data.session_id) + .await + .unwrap_or_else(|| "SPF Report".to_string()); + let spf_params = if let Some(mail_from) = &self.data.mail_from { + SpfParameters::mail_from( + self.data.remote_ip, + &self.data.helo_domain, + &self.hostname, + &mail_from.address, + ) + } else { + SpfParameters::helo(self.data.remote_ip, &self.data.helo_domain, &self.hostname) + }; let mut report = Vec::with_capacity(128); - self.new_auth_failure(AuthFailureType::Spf, rejected) - .with_authentication_results( - if let Some(mail_from) = &self.data.mail_from { - AuthenticationResults::new(&self.hostname).with_spf_mailfrom_result( - output, - self.data.remote_ip, - &mail_from.address, - &self.data.helo_domain, - ) - } else { - AuthenticationResults::new(&self.hostname).with_spf_ehlo_result( - output, - self.data.remote_ip, - &self.data.helo_domain, - ) - } - .to_string(), - ) - .with_spf_dns(format!("txt : {} : v=SPF1", output.domain())) // TODO use DNS record - .write_rfc5322( - ( - self.server - .eval_if(&config.name, self, self.data.session_id) - .await - .unwrap_or_else(|| "Mailer Daemon".to_string()) - .as_str(), - from_addr.as_str(), - ), - rcpt, - &self - .server - .eval_if(&config.subject, self, self.data.session_id) - .await - .unwrap_or_else(|| "SPF Report".to_string()), - &mut report, - ) - .ok(); + if let Err(err) = (FeedbackReport { + authentication_results: vec![ + AuthenticationResults::new(&self.hostname) + .with_spf_result(output, &spf_params) + .to_string() + .into(), + ], + spf_dns: Some(format!("txt : {} : v=SPF1", output.domain()).into()), + ..self.new_auth_failure(AuthFailureType::Spf, rejected) + }) + .write_rfc5322( + &ReportEnvelope { + from: (from_name.as_str(), from_addr.as_str()).into(), + to: vec![rcpt], + submitter: &self.hostname, + report_domain: "", + subject: Some(&subject), + }, + &mut report, + ) { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = self.data.session_id, + To = CompactString::from(rcpt), + Reason = err.to_string(), + ); + return; + } trc::event!( OutgoingReport(OutgoingReportEvent::SpfReport), SpanId = self.data.session_id, - To = rcpt.to_string(), - From = from_addr.to_string(), + To = CompactString::from(rcpt), + From = CompactString::from(&from_addr), ); // Send report diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 38424078..2caf1e76 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -4,10 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::AggregateTimestamp; use crate::{ queue::RecipientDomain, - reporting::{index::InternalReportIndex, send::MtaReportSend}, + reporting::{ + ReportAddress, + aggregate::{ + ReportItem, ReportScheduler, ReportWindow, read_tls_policies, report_size_limit, + }, + index::InternalReportIndex, + send::MtaReportSend, + }, }; use common::{ Server, USER_AGENT, @@ -15,45 +21,40 @@ use common::{ report::AggregateFrequency, resolver::{Mode, MxPattern, TlsaMatching}, }, - ipc::{TlsEvent, ToHash}, -}; -use mail_auth::{ - flate2::{Compression, write::GzEncoder}, - mta_sts::{ReportUri, TlsRpt}, - report::tlsrpt::{FailureDetails, PolicyDetails}, + expr::Bump, + ipc::{ReportingEvent, TlsEvent}, }; +use compact_str::{CompactString, ToCompactString}; +use mail_auth::mta_sts::{ReportUri, TlsRptRecord}; use registry::{ schema::{ enums::TlsPolicyType, - prelude::{ObjectType, Property}, - structs::{TlsFailureDetails, TlsInternalReport, TlsReport, TlsReportPolicy}, + prelude::ObjectType, + structs::{TlsInternalReport, TlsReport, TlsReportPolicy}, }, - types::{EnumImpl, ObjectImpl, datetime::UTCDateTime}, + types::{EnumImpl, datetime::UTCDateTime}, +}; +use reqwest::{Method, header::CONTENT_TYPE}; +use smtp_report::{ + ReportEnvelope, + flate2::{Compression, write::GzEncoder}, + tlsrpt::TlsReport as ExportedTlsReport, }; -use reqwest::header::CONTENT_TYPE; use std::fmt::Write; -use std::{future::Future, sync::Arc, time::Duration}; +use std::{future::Future, mem, sync::Arc, time::Duration}; use store::{ - SerializeInfallible, ValueKey, - registry::ObjectIdVersioned, - write::{BatchBuilder, RegistryClass, ValueClass, assert::AssertValue}, + ValueKey, + write::{BatchBuilder, RegistryClass, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; #[derive(Debug, Clone)] pub struct TlsRptOptions { - pub record: Arc, + pub record: Arc, pub interval: AggregateFrequency, } -#[derive(Debug, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize)] -pub struct TlsFormat { - pub rua: Vec, - pub policy: PolicyDetails, - pub records: Vec>, -} - -#[cfg(feature = "test_mode")] +#[cfg(stalwart_tests)] pub static TLS_HTTP_REPORT: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); pub trait TlsReporting: Sync + Send { @@ -70,7 +71,7 @@ impl TlsReporting for Server { let object_id = ObjectType::TlsInternalReport.to_id(); let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id }); - let Some(report) = self + let Some(mut report) = self .store() .get_value::(ValueKey::from(key.clone())) .await @@ -78,73 +79,147 @@ impl TlsReporting for Server { else { return Ok(()); }; + let span_id = self.inner.data.span_id_gen.generate(); + let max_size = self + .eval_if( + &self.core.smtp.report.tls.max_size, + &RecipientDomain::new(report.domain.as_str()), + &mut Bump::new(), + span_id, + ) + .await; + read_tls_policies( + self.store(), + item_id, + &mut report, + report_size_limit(max_size), + ) + .await?; + + let mut batch = BatchBuilder::new(); + batch.clear(key).clear(report.primary_key()); + let item = ReportItem::new(ObjectType::TlsInternalReport, item_id); + item.clear(&mut batch); + let now = UTCDateTime::now(); + if now < report.report.date_range_end { + report.report.date_range_end = now; + } // Delete report - let mut batch = BatchBuilder::new(); - batch.clear(key).clear(RegistryClass::PrimaryKey { - object_id: object_id.into(), - index_id: Property::Domain.to_id(), - key: report.domain.as_bytes().to_vec(), - }); - self.core - .storage - .data - .write(batch.build_all()) + self.store() + .write_batch(&mut batch) .await .caused_by(trc::location!())?; + if let Err(err) = item.clear_orphans(self.store()).await { + trc::error!( + err.span_id(span_id) + .details("Failed to clear orphaned report rows") + ); + } + let payload = TlsPayload::build(&mut report, span_id); - let domain_name = report.domain.as_str(); + if let Some(payload) = payload { + payload.deliver(self, &report, span_id).await; + } + Ok(()) + } + + async fn schedule_tls(&self, event: Box) { + ReportScheduler::default() + .process(self, &mut vec![ReportingEvent::Tls(event)]) + .await; + } +} + +struct TlsPayload { + json: String, + compressed_json: Vec, +} + +impl TlsPayload { + fn build(report: &mut TlsInternalReport, span_id: u64) -> Option { + if report.report.policies.is_empty() { + return None; + } let event_from = report.report.date_range_start.timestamp() as u64; let event_to = report.report.date_range_end.timestamp() as u64; - let span_id = self.inner.data.span_id_gen.generate(); trc::event!( OutgoingReport(OutgoingReportEvent::TlsAggregate), SpanId = span_id, ReportId = event_from, - Domain = domain_name.to_string(), + Domain = CompactString::from(report.domain.as_str()), RangeFrom = trc::Value::Timestamp(event_from), RangeTo = trc::Value::Timestamp(event_to), ); // Generate report - let exported_report = mail_auth::report::tlsrpt::TlsReport::from(report.report); - let json = exported_report.to_json(); - let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); - let json = match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) - { - Ok(report) => report, - Err(err) => { - trc::event!( - OutgoingReport(OutgoingReportEvent::SubmissionError), - SpanId = span_id, - Reason = err.to_string(), - Details = "Failed to compress report" - ); + let json = ExportedTlsReport::from(mem::take(&mut report.report)).to_json(); + let compressed_json = if !report.http_rua.is_empty() { + let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); + match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) { + Ok(report) => report, + Err(err) => { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = span_id, + Reason = err.to_compact_string(), + Details = "Failed to compress report" + ); - return Ok(()); + return None; + } } + } else { + Vec::new() }; + Some(TlsPayload { + json, + compressed_json, + }) + } + + async fn deliver(self, server: &Server, report: &TlsInternalReport, span_id: u64) { + let TlsPayload { + json, + compressed_json, + } = self; + let domain_name = report.domain.as_str(); // Try delivering report over HTTP for uri in report.http_rua.as_slice() { { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if uri == "https://127.0.0.1/tls" { - TLS_HTTP_REPORT.lock().extend_from_slice(&json); + TLS_HTTP_REPORT.lock().extend_from_slice(&compressed_json); - return Ok(()); + return; } - match self + let request = match server .core .smtp .tls_report_client - .post(uri) + .request(Method::POST, uri) + { + Ok(request) => request, + Err(err) => { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = span_id, + Url = CompactString::from(uri), + Reason = err.as_str(), + Details = "Report URI rejected" + ); + continue; + } + }; + + match request .timeout(Duration::from_secs(2 * 60)) .header(reqwest::header::USER_AGENT, USER_AGENT) .header(CONTENT_TYPE, "application/tlsrpt+gzip") - .body(json.to_vec()) + .body(compressed_json.clone()) .send() .await { @@ -153,16 +228,16 @@ impl TlsReporting for Server { trc::event!( OutgoingReport(OutgoingReportEvent::HttpSubmission), SpanId = span_id, - Url = uri.to_string(), + Url = CompactString::from(uri), Code = response.status().as_u16(), ); - return Ok(()); + return; } else { trc::event!( OutgoingReport(OutgoingReportEvent::SubmissionError), SpanId = span_id, - Url = uri.to_string(), + Url = CompactString::from(uri), Code = response.status().as_u16(), Details = "Invalid HTTP response" ); @@ -172,8 +247,8 @@ impl TlsReporting for Server { trc::event!( OutgoingReport(OutgoingReportEvent::SubmissionError), SpanId = span_id, - Url = uri.to_string(), - Reason = err.to_string(), + Url = CompactString::from(uri), + Reason = err.to_compact_string(), Details = "HTTP submission error" ); } @@ -182,312 +257,192 @@ impl TlsReporting for Server { } // Deliver report over SMTP - if !report.mail_rua.is_empty() { - let config = &self.core.smtp.report.tls; - let from_addr = self - .eval_if(&config.address, &RecipientDomain::new(domain_name), span_id) + let mail_rua = report + .mail_rua + .iter() + .filter(|rcpt| ReportAddress::checked(rcpt.as_str(), span_id).is_some()) + .map(|rcpt| rcpt.as_str()) + .collect::>(); + if !mail_rua.is_empty() { + let config = &server.core.smtp.report.tls; + let mut arena = Bump::new(); + let from_addr = server + .eval_if( + &config.address, + &RecipientDomain::new(domain_name), + &mut arena, + span_id, + ) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + let submitter = server + .eval_if( + &server.core.smtp.report.submitter, + &RecipientDomain::new(domain_name), + &mut arena, + span_id, + ) + .await + .unwrap_or_else(|| "localhost".to_string()); + let from_name = server + .eval_if( + &config.name, + &RecipientDomain::new(domain_name), + &mut arena, + span_id, + ) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()); let mut message = Vec::with_capacity(2048); - let _ = exported_report.write_rfc5322_from_bytes( - domain_name, - &self - .eval_if( - &self.core.smtp.report.submitter, - &RecipientDomain::new(domain_name), - span_id, - ) - .await - .unwrap_or_else(|| "localhost".to_string()), - ( - self.eval_if(&config.name, &RecipientDomain::new(domain_name), span_id) - .await - .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) - .as_str(), - from_addr.as_str(), - ), - report.mail_rua.iter().map(|v| v.as_str()), - &json, - &mut message, - ); - - // Send report - self.send_report( - &from_addr, - report.mail_rua.iter().map(|v| v.as_str()), - message, - &config.sign, - false, - span_id, - ) - .await; + let envelope = ReportEnvelope { + from: (from_name.as_str(), from_addr.as_str()).into(), + to: mail_rua, + submitter: &submitter, + report_domain: domain_name, + subject: None, + }; + match ExportedTlsReport::write_rfc5322_json(json.as_bytes(), &envelope, &mut message) { + Ok(()) => { + server + .send_report( + &from_addr, + envelope.to.iter(), + message, + &config.sign, + false, + span_id, + ) + .await; + } + Err(err) => { + trc::event!( + OutgoingReport(OutgoingReportEvent::SubmissionError), + SpanId = span_id, + Reason = err.to_string(), + ); + } + } } else { trc::event!( OutgoingReport(OutgoingReportEvent::NoRecipientsFound), SpanId = span_id, ); } - - Ok(()) } +} - async fn schedule_tls(&self, event: Box) { - let object_id = ObjectType::TlsInternalReport.to_id(); - let pk = ValueClass::Registry(RegistryClass::PrimaryKey { - object_id: object_id.into(), - index_id: Property::Domain.to_id(), - key: event.domain.as_bytes().to_vec(), - }); - let mut rety_count = 0; - let policy_hash = event.policy.to_hash(); +pub(crate) fn tls_report_descriptor( + event: &TlsEvent, + window: ReportWindow, + item: ReportItem, +) -> TlsInternalReport { + let date_range_start = UTCDateTime::from_timestamp(window.start.cast_signed()); + let mut report = TlsInternalReport { + created_at: date_range_start, + deliver_at: UTCDateTime::from_timestamp(item.deliver_at(window).cast_signed()), + domain: event.domain.clone(), + report: TlsReport { + date_range_end: UTCDateTime::from_timestamp(window.end.cast_signed()), + date_range_start, + ..Default::default() + }, + ..Default::default() + }; - loop { - // Find the report by domain name - let mut batch = BatchBuilder::new(); - let report = match self - .store() - .get_value::(ValueKey::from(pk.clone())) - .await - { - Ok(Some(object_id_v)) => { - match self - .store() - .get_value::(ValueKey::from(ValueClass::Registry( - RegistryClass::Item { - object_id, - item_id: object_id_v.object_id.id().id(), - }, - ))) - .await - { - Ok(Some(report)) => Some((object_id_v, report)), - Ok(None) => { - trc::event!( - OutgoingReport(OutgoingReportEvent::NotFound), - Id = object_id_v.object_id.id().id(), - CausedBy = trc::location!(), - Details = "Failed to find TLS report for domain" - ); - - return; - } - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to query registry for TLS report") - ); - return; - } - } - } - Ok(None) => None, - Err(err) => { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to query registry for TLS report") - ); - return; - } - }; - - // Create report if missing - let config = &self.core.smtp.report.tls; - let (item_id, mut report) = if let Some((mut object_id_v, report)) = report { - batch.assert_value(pk.clone(), AssertValue::U32(object_id_v.version)); - object_id_v.version += 1; - batch.set(pk.clone(), object_id_v.serialize()); - - (object_id_v.object_id.id().id(), report) - } else { - let item_id = self.inner.data.queue_id_gen.generate(); - let date_range_start = UTCDateTime::now(); - let date_range_end = UTCDateTime::from_timestamp( - date_range_start.timestamp() + event.interval.as_secs() as i64, - ); - - let report = TlsInternalReport { - created_at: date_range_start, - deliver_at: date_range_end, - domain: event.domain.clone(), - report: TlsReport { - report_id: format!("{}_{policy_hash}", date_range_start.timestamp()), - organization_name: self - .eval_if::( - &config.org_name, - &RecipientDomain::new(&event.domain), - event.span_id, - ) - .await - .clone(), - contact_info: self - .eval_if::( - &config.contact_info, - &RecipientDomain::new(&event.domain), - event.span_id, - ) - .await - .clone(), - date_range_end, - date_range_start, - policies: Default::default(), - }, - ..Default::default() - }; - - report.write_ops(&mut batch, item_id, true); - - (item_id, report) - }; - - let policy = if let Some(policy) = report - .policy_identifiers - .as_slice() - .iter() - .position(|id| *id == policy_hash) - .and_then(|idx| report.report.policies.0.inner.get_mut(idx)) - { - &mut policy.value - } else { - // Create policy - let mut policy = TlsReportPolicy { - policy_type: TlsPolicyType::NoPolicyFound, - policy_domain: report.domain.clone(), - ..Default::default() - }; - - match &event.policy { - common::ipc::PolicyType::Tlsa(tlsa) => { - policy.policy_type = TlsPolicyType::Tlsa; - if let Some(tlsa) = tlsa { - for entry in &tlsa.entries { - policy.policy_strings.push(format!( - "{} {} {} {}", - if entry.is_end_entity { 3 } else { 2 }, - i32::from(entry.is_spki), - match entry.matching { - TlsaMatching::Full => 0, - TlsaMatching::Sha256 => 1, - TlsaMatching::Sha512 => 2, - }, - entry.data.iter().fold( - String::with_capacity(64), - |mut s, b| { - write!(s, "{b:02X}").ok(); - s - } - ) - )); - } - } - } - common::ipc::PolicyType::Sts(sts) => { - policy.policy_type = TlsPolicyType::Sts; - if let Some(sts) = sts { - policy.policy_strings.push("version: STSv1".to_string()); - policy.policy_strings.push(format!( - "mode: {}", - match sts.mode { - Mode::Enforce => "enforce", - Mode::Testing => "testing", - Mode::None => "none", - } - )); - policy - .policy_strings - .push(format!("max_age: {}", sts.max_age)); - for mx in &sts.mx { - let mx = match mx { - MxPattern::Equals(mx) => mx.to_string(), - MxPattern::StartsWith(mx) => format!("*.{mx}"), - }; - policy.policy_strings.push(format!("mx: {mx}")); - policy.mx_hosts.push(mx); - } - } - } - _ => (), - } - - for rua in &event.tls_record.rua { - match rua { - ReportUri::Mail(mail) => { - report.mail_rua.push(mail.clone()); - } - ReportUri::Http(uri) => { - report.http_rua.push(uri.clone()); - } - } - } - - report.policy_identifiers.push(policy_hash); - report.report.policies.push(policy); - &mut report.report.policies.0.inner.last_mut().unwrap().value - }; - - // Add failure details - if let Some(mut failure) = event.failure.clone().map(TlsFailureDetails::from) { - if let Some(idx) = policy - .failure_details - .0 - .inner - .iter() - .position(|d| d.value.eq_except_count(&failure)) - { - policy.failure_details.0.inner[idx] - .value - .failed_session_count += 1; - } else { - failure.failed_session_count = 1; - policy.failure_details.push(failure); - } - - policy.total_failed_sessions += 1; - } else { - policy.total_successful_sessions += 1; + for rua in &event.tls_record.rua { + match rua { + ReportUri::Mail(mail) => { + report.mail_rua.push(mail.clone()); } - - // Write entry - let report_bytes = report.to_pickled_vec(); - let max_report_size = self - .eval_if( - &config.max_size, - &RecipientDomain::new(&event.domain), - event.span_id, - ) - .await - .unwrap_or(5 * 1024 * 1024); - if max_report_size != 0 && report_bytes.len() > max_report_size { - trc::event!( - OutgoingReport(OutgoingReportEvent::MaxSizeExceeded), - SpanId = event.span_id, - Domain = event.domain.clone(), - Details = report_bytes.len(), - Limit = max_report_size, - ); - return; - } - - batch.set( - ValueClass::Registry(RegistryClass::Item { object_id, item_id }), - report_bytes, - ); - - match self.core.storage.data.write(batch.build_all()).await { - Ok(_) => { - break; - } - Err(err) => { - if err.is_assertion_failure() && rety_count < 3 { - rety_count += 1; - continue; - } - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to write TLS report") - ); - break; - } + ReportUri::Http(uri) => { + report.http_rua.push(uri.clone()); } } } + + report +} + +pub(crate) async fn complete_tls_report( + server: &Server, + report: &mut TlsInternalReport, + arena: &mut Bump, + span_id: u64, +) { + let config = &server.core.smtp.report.tls; + let domain = RecipientDomain::new(report.domain.as_str()); + report.report.report_id = format!( + "{}_{}", + report.report.date_range_start.timestamp(), + server.inner.data.queue_id_gen.generate() + ); + report.report.organization_name = server + .eval_if::(&config.org_name, &domain, arena, span_id) + .await; + report.report.contact_info = server + .eval_if::(&config.contact_info, &domain, arena, span_id) + .await; +} + +pub(crate) fn tls_report_policy(event: &TlsEvent) -> TlsReportPolicy { + let mut policy = TlsReportPolicy { + policy_type: TlsPolicyType::NoPolicyFound, + policy_domain: event.domain.clone(), + ..Default::default() + }; + + match &event.policy { + common::ipc::PolicyType::Tlsa(tlsa) => { + policy.policy_type = TlsPolicyType::Tlsa; + if let Some(tlsa) = tlsa { + for entry in &tlsa.entries { + policy.policy_strings.push(format!( + "{} {} {} {}", + if entry.is_end_entity { 3 } else { 2 }, + i32::from(entry.is_spki), + match entry.matching { + TlsaMatching::Full => 0, + TlsaMatching::Sha256 => 1, + TlsaMatching::Sha512 => 2, + }, + entry + .data + .iter() + .fold(String::with_capacity(64), |mut s, b| { + write!(s, "{b:02X}").ok(); + s + }) + )); + } + } + } + common::ipc::PolicyType::Sts(sts) => { + policy.policy_type = TlsPolicyType::Sts; + if let Some(sts) = sts { + policy.policy_strings.push("version: STSv1".to_string()); + policy.policy_strings.push(format!( + "mode: {}", + match sts.mode { + Mode::Enforce => "enforce", + Mode::Testing => "testing", + Mode::None => "none", + } + )); + policy + .policy_strings + .push(format!("max_age: {}", sts.max_age)); + for mx in &sts.mx { + let mx = match mx { + MxPattern::Equals(mx) => mx.to_string(), + MxPattern::StartsWith(mx) => format!("*.{mx}"), + }; + policy.policy_strings.push(format!("mx: {mx}")); + policy.mx_hosts.push(mx); + } + } + } + _ => (), + } + + policy } diff --git a/crates/smtp/src/scripts/envelope.rs b/crates/smtp/src/scripts/envelope.rs index e8e860b8..49c39572 100644 --- a/crates/smtp/src/scripts/envelope.rs +++ b/crates/smtp/src/scripts/envelope.rs @@ -15,7 +15,7 @@ use crate::core::{SessionAddress, SessionData}; use email::message::delivery::ORCPT_ADDR_TYPE; impl SessionData { - pub fn apply_envelope_modification(&mut self, envelope: Envelope, value: String) { + pub fn apply_envelope_modification(&mut self, envelope: Envelope, mut value: String) { match envelope { Envelope::From => { let (address, address_lcase, domain) = if value.contains('@') { @@ -112,11 +112,10 @@ impl SessionData { } Envelope::Orcpt => { if let Some(rcpt_to) = self.rcpt_to.last_mut() { - rcpt_to.dsn_info = value - .strip_prefix(ORCPT_ADDR_TYPE) - .map(str::to_string) - .unwrap_or(value) - .into(); + if value.starts_with(ORCPT_ADDR_TYPE) { + value.drain(..ORCPT_ADDR_TYPE.len()); + } + rcpt_to.dsn_info = value.into(); } } Envelope::Envid => { diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 350dde28..a262c0e0 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -7,379 +7,400 @@ use crate::queue::{ MessageSource, quota::HasQueueQuota, - spool::{QueueParams, SmtpSpool}, + spool::{QueueOutcome, QueueParams, SmtpSpool}, }; -use common::{Server, config::smtp::queue::QueueExpiry, scripts::plugins::PluginContext}; -use mail_parser::{Encoding, Message, MessagePart, PartType}; +use common::{ + Server, + config::smtp::queue::QueueExpiry, + scripts::{ScriptModification, plugins::PluginContext}, +}; +use compact_str::{CompactString, ToCompactString, format_compact}; +use mail_parser::Message; use sieve::{ - Event, Input, MatchAs, Recipient, Sieve, - compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, + Arena, Context, Handler, Input, Mailbox, MatchAs, MessageSource as SieveMessageSource, + Recipient, Reply, Sieve, SieveAction, Status, + compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, Ret}, + runtime::{RuntimeError, Script, Variable}, }; -use smtp_proto::{ - MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, - RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, -}; -use std::{future::Future, sync::Arc, time::Instant}; +use smtp_proto::{MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS}; +use std::{future::Future, time::Instant}; use trc::SieveEvent; -use super::{ScriptModification, ScriptParameters, ScriptResult}; +use super::{ScriptParameters, ScriptResult, notify_flags}; + +const IMPLICIT_KEEP: usize = usize::MAX; +const DISCARD: usize = usize::MAX - 1; pub trait RunScript: Sync + Send { fn run_script( &self, script_id: String, - script: Arc, + script: &Sieve<'_>, params: ScriptParameters<'_>, ) -> impl Future + Send; } +struct SmtpHandler<'x> { + server: &'x Server, + script_id: &'x str, + session_id: u64, + modifications: Vec, + messages: Vec>, + reject_reason: Option, + keep_id: usize, + pending: Option>, +} + +enum PendingWork<'x> { + ListContains { + lists: Vec, + values: Vec, + }, + Function { + id: u32, + arguments: Vec>, + }, + SendMessage(SieveSend), +} + +struct SieveSend { + source: SieveMessageSource, + recipients: Vec, + notify: Notify, + return_of_content: Ret, + by_time: ByTime, + message_id: usize, +} + +impl<'x> Handler<'x> for SmtpHandler<'x> { + fn mailbox_exists(&mut self, _: &Context<'x>, _: &[Mailbox<'_>], _: &[&str]) -> Reply { + Reply::Error(RuntimeError::CapabilityNotSupported("mailbox".into())) + } + + fn duplicate_id(&mut self, _: &Context<'x>, _: &str, _: u64, _: bool) -> Reply { + Reply::Error(RuntimeError::CapabilityNotSupported("duplicate".into())) + } + + fn include_script( + &mut self, + _: &Context<'x>, + name: Script<'_>, + optional: bool, + ) -> Reply>> { + match self.server.core.sieve.trusted_script(name.name()) { + Some(script) => Reply::Ready(Some(script)), + None => { + if !optional { + trc::event!( + Sieve(SieveEvent::ScriptNotFound), + Script = CompactString::from(self.script_id), + SpanId = self.session_id, + Details = CompactString::from(name.name()), + ); + } + + Reply::Ready(None) + } + } + } + + fn list_contains( + &mut self, + _: &Context<'x>, + lists: &[&str], + values: &[&str], + match_as: MatchAs, + ) -> Reply { + let lowercase = matches!(match_as, MatchAs::Lowercase); + self.pending = Some(PendingWork::ListContains { + lists: lists.iter().map(|list| list.to_string()).collect(), + values: values + .iter() + .map(|value| { + if lowercase { + value.to_lowercase() + } else { + value.to_string() + } + }) + .collect(), + }); + + Reply::Pending + } + + fn function( + &mut self, + _: &Context<'x>, + id: u32, + arguments: &[Variable<'x>], + ) -> Reply> { + self.pending = Some(PendingWork::Function { + id, + arguments: arguments.to_vec(), + }); + + Reply::Pending + } + + fn action(&mut self, _: &Context<'x>, action: SieveAction<'_>) -> Reply<()> { + match action { + SieveAction::Keep { message_id, .. } => { + self.keep_id = message_id; + } + SieveAction::Discard => { + self.keep_id = DISCARD; + } + SieveAction::Reject { reason, .. } => { + self.reject_reason = reason.to_string().into(); + } + SieveAction::CreatedMessage { message, .. } => { + self.messages.push(message); + } + SieveAction::SetEnvelope { envelope, value } => { + self.modifications.push(ScriptModification::SetEnvelope { + name: envelope, + value: value.to_string(), + }); + } + SieveAction::SendMessage { + source, + recipient, + notify, + return_of_content, + by_time, + message_id, + } => { + let recipients = match recipient { + Recipient::Address(rcpt) => vec![rcpt.to_string()], + Recipient::Group(rcpts) => rcpts.iter().map(|rcpt| rcpt.to_string()).collect(), + Recipient::List(list) => { + trc::event!( + Sieve(SieveEvent::NotSupported), + Script = CompactString::from(self.script_id), + SpanId = self.session_id, + Name = CompactString::from(list), + Reason = "Sending to lists is not supported.", + ); + + Vec::new() + } + }; + + self.pending = Some(PendingWork::SendMessage(SieveSend { + source, + recipients, + notify, + return_of_content, + by_time, + message_id, + })); + + return Reply::Pending; + } + action => { + let capability = if matches!(action, SieveAction::FileInto { .. }) { + "fileinto" + } else { + "enotify" + }; + trc::event!( + Sieve(SieveEvent::NotSupported), + Script = CompactString::from(self.script_id), + SpanId = self.session_id, + Reason = "Unsupported action", + Details = format_compact!("{action:?}"), + ); + return Reply::Error(RuntimeError::CapabilityNotSupported(capability.into())); + } + } + + Reply::Ready(()) + } +} + impl RunScript for Server { async fn run_script( &self, script_id: String, - script: Arc, - params: ScriptParameters<'_>, + script: &Sieve<'_>, + mut params: ScriptParameters<'_>, ) -> ScriptResult { // Create filter instance let time = Instant::now(); - let mut instance = self + let session_id = params.session_id; + let empty; + let message = match params.message.take() { + Some(message) => message, + None => { + empty = Message::default(); + &empty + } + }; + let mut arena = Arena::new(); + let mut ctx = self .core .sieve .trusted_runtime - .filter_parsed(params.message.unwrap_or_else(|| Message { - parts: vec![MessagePart { - headers: vec![], - is_encoding_problem: false, - body: PartType::Text("".into()), - encoding: Encoding::None, - offset_header: 0, - offset_body: 0, - offset_end: 0, - }], - raw_message: b""[..].into(), - ..Default::default() - })) - .with_vars_env(params.variables) - .with_envelope_list(params.envelope) + .filter_parsed(message, script, &mut arena) + .with_vars_env(std::mem::take(&mut params.variables)) + .with_envelope_list(std::mem::take(&mut params.envelope)) .with_user_address(¶ms.from_addr) .with_user_full_name(¶ms.from_name); if let Some(spam_status) = params.spam_status { - instance.set_spam_status(spam_status); + ctx.set_spam_status(spam_status); } - let mut input = Input::script("__script", script); - let mut messages: Vec> = Vec::new(); - let session_id = params.session_id; - let mut reject_reason = None; - let mut modifications = vec![]; - let mut keep_id = usize::MAX; + let mut handler = SmtpHandler { + server: self, + script_id: script_id.as_str(), + session_id, + modifications: Vec::new(), + messages: Vec::new(), + reject_reason: None, + keep_id: IMPLICIT_KEEP, + pending: None, + }; // Start event loop - while let Some(result) = instance.run(input) { - match result { - Ok(event) => match event { - Event::IncludeScript { name, optional } => { - if let Some(script) = self.core.sieve.trusted_script(name.as_str()) { - input = Input::script(name, script.clone()); - } else if optional { - input = false.into(); - } else { - trc::event!( - Sieve(SieveEvent::ScriptNotFound), - Id = script_id.clone(), - SpanId = session_id, - Details = name.as_str().to_string(), - ); - break; - } - } - Event::ListContains { - lists, - values, - match_as, - } => { - input = false.into(); - 'outer: for list in lists { - if let Some(store) = self.get_lookup_store(&list) { + let mut first_redirect = None; + let mut more_redirects = Vec::new(); + let mut redirect_copies = 0; + loop { + match ctx.run(&mut handler) { + Ok(Status::Finished) => break, + Ok(Status::Pending) => { + let input = match handler.pending.take() { + Some(PendingWork::ListContains { lists, values }) => { + let mut contains = false; + + 'outer: for list in lists { + let Some(store) = self.get_lookup_store(&list) else { + trc::event!( + Sieve(SieveEvent::ListNotFound), + Script = script_id.clone(), + SpanId = session_id, + Name = list, + ); + + continue; + }; + for value in &values { - if let Ok(true) = store - .key_exists(if !matches!(match_as, MatchAs::Lowercase) { - value.clone() - } else { - value.to_lowercase() - }) - .await - { - input = true.into(); + if let Ok(true) = store.key_exists(value.as_str()).await { + contains = true; break 'outer; } } - } else { - trc::event!( - Sieve(SieveEvent::ListNotFound), - Id = script_id.clone(), - SpanId = session_id, - Details = list, - ); } + + Input::Bool(contains) } - } - Event::Function { id, arguments } => { - input = self - .core - .run_plugin( - id, - PluginContext { - session_id, - server: self, - message: instance.message(), - modifications: &mut modifications, - access_token: params.access_token, - arguments, - }, + Some(PendingWork::Function { id, arguments }) => { + self.core + .run_plugin( + id, + PluginContext { + session_id, + server: self, + message: ctx.message(), + modifications: &mut handler.modifications, + access_token: params.access_token, + arguments, + }, + ) + .await + } + Some(PendingWork::SendMessage(send)) + if send.source == SieveMessageSource::Redirect => + { + redirect_copies += send.recipients.len(); + if first_redirect.is_none() { + first_redirect = Some(send); + } else { + more_redirects.push(send); + } + + Input::Continue + } + Some(PendingWork::SendMessage(send)) => { + self.queue_sieve_message( + &handler.messages, + ctx.message().raw(), + ¶ms, + &script_id, + send, + false, ) .await; - } - Event::Keep { message_id, .. } => { - keep_id = message_id; - input = true.into(); - } - Event::Discard => { - keep_id = usize::MAX - 1; - input = true.into(); - } - Event::Reject { reason, .. } => { - reject_reason = reason.into(); - input = true.into(); - } - Event::SendMessage { - recipient, - notify, - return_of_content, - by_time, - message_id, - } => { - // Build message - let mut message = self.new_message( - params.return_path.as_str(), - MessageSource::Autogenerated, - session_id, - ); - match recipient { - Recipient::Address(rcpt) => { - message.expand_and_add_recipient(rcpt, self).await; - } - Recipient::Group(rcpt_list) => { - for rcpt in rcpt_list { - message.expand_and_add_recipient(rcpt, self).await; - } - } - Recipient::List(list) => { - trc::event!( - Sieve(SieveEvent::NotSupported), - Id = script_id.clone(), - SpanId = session_id, - Details = list, - Reason = "Sending to lists is not supported.", - ); - } + + Input::Continue } + None => Input::Continue, + }; - // Set notify flags - let mut flags = 0; - match notify { - Notify::Never => { - flags = RCPT_NOTIFY_NEVER; - } - Notify::Items(items) => { - for item in items { - flags |= match item { - NotifyItem::Success => RCPT_NOTIFY_SUCCESS, - NotifyItem::Failure => RCPT_NOTIFY_FAILURE, - NotifyItem::Delay => RCPT_NOTIFY_DELAY, - }; - } - } - Notify::Default => (), - } - if flags > 0 { - for rcpt in &mut message.message.recipients { - rcpt.flags |= flags; - } - } - - // Set ByTime flags - match by_time { - ByTime::Relative { - rlimit, - mode, - trace, - } => { - if trace { - message.message.flags |= MAIL_BY_TRACE; - } - match mode { - ByMode::Notify => { - for domain in &mut message.message.recipients { - domain.notify.due += rlimit; - } - } - ByMode::Return => { - for domain in &mut message.message.recipients { - domain.notify.due += rlimit; - } - } - ByMode::Default => (), - } - } - ByTime::Absolute { - alimit, - mode, - trace, - } => { - if trace { - message.message.flags |= MAIL_BY_TRACE; - } - match mode { - ByMode::Notify => { - for domain in &mut message.message.recipients { - domain.notify.due = alimit as u64; - } - } - ByMode::Return => { - let expires = - (alimit as u64).saturating_sub(message.message.created); - if expires > 0 { - for domain in &mut message.message.recipients { - domain.expires = QueueExpiry::Ttl(expires); - } - } - } - ByMode::Default => (), - } - } - ByTime::None => (), - }; - - // Set ret - match return_of_content { - Ret::Full => { - message.message.flags |= MAIL_RET_FULL; - } - Ret::Hdrs => { - message.message.flags |= MAIL_RET_HDRS; - } - Ret::Default => (), - } - - // Queue message - let is_forward = message_id == 0; - let raw_message = if !is_forward { - messages.get(message_id - 1).map(|m| m.as_slice()) - } else { - instance.message().raw_message().into() - }; - if let Some(raw_message) = raw_message.filter(|m| !m.is_empty()) { - if let Some(metadata) = self.has_quota(&mut message).await { - let dkim_signers = if let Some(sign_domain) = ¶ms.sign_domain { - match self.dkim_signers(sign_domain).await { - Ok(signers) => signers, - Err(err) => { - trc::error!( - err.details("Failed to obtain DKIM signers") - .caused_by(trc::location!()) - ); - - None - } - } - } else { - None - }; - - let _ = message - .queue( - QueueParams::new(raw_message, session_id, self) - .with_dkim_signers(dkim_signers) - .with_raw_headers_opt( - params.headers.filter(|_| is_forward), - ) - .with_original_raw_message( - instance.message().raw_message(), - ) - .with_metadata(metadata), - ) - .await; - } else { - trc::event!( - Sieve(SieveEvent::QuotaExceeded), - SpanId = session_id, - Id = script_id.clone(), - From = message.message.return_path, - To = message - .message - .recipients - .into_iter() - .map(|r| trc::Value::from(r.address().to_string())) - .collect::>(), - ); - } - } - - input = true.into(); - } - Event::CreatedMessage { message, .. } => { - messages.push(message); - input = true.into(); - } - Event::SetEnvelope { envelope, value } => { - modifications.push(ScriptModification::SetEnvelope { - name: envelope, - value, - }); - input = true.into(); - } - unsupported => { - trc::event!( - Sieve(SieveEvent::NotSupported), - Id = script_id.clone(), - SpanId = session_id, - Reason = "Unsupported event", - Details = format!("{unsupported:?}"), - ); - break; - } - }, + ctx.resume(input); + } Err(err) => { trc::event!( Sieve(SieveEvent::RuntimeError), - Id = script_id.clone(), + Script = script_id.clone(), SpanId = session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); + break; } } } + let redirected = redirect_copies > 0; + if first_redirect.is_some() { + let kept = handler.keep_id == 0 + || (handler.keep_id != DISCARD && handler.reject_reason.is_none()); + let exploded = redirect_copies + usize::from(kept) > 1; + for send in first_redirect.into_iter().chain(more_redirects) { + self.queue_sieve_message( + &handler.messages, + ctx.message().raw(), + ¶ms, + &script_id, + send, + exploded, + ) + .await; + } + } + + let SmtpHandler { + modifications, + messages, + reject_reason, + keep_id, + .. + } = handler; + // Keep id // 0 = use original message - // MAX = implicit keep - // MAX - 1 = discard message + // IMPLICIT_KEEP = implicit keep + // DISCARD = discard message if keep_id == 0 { trc::event!( Sieve(SieveEvent::ActionAccept), SpanId = session_id, - Id = script_id, + Script = script_id, Elapsed = time.elapsed(), ); - ScriptResult::Accept { modifications } + ScriptResult::Accept { + modifications, + redirected, + } } else if let Some(mut reject_reason) = reject_reason { trc::event!( Sieve(SieveEvent::ActionReject), - Id = script_id, + Script = script_id, SpanId = session_id, Details = reject_reason.clone(), Elapsed = time.elapsed(), @@ -398,34 +419,38 @@ impl RunScript for Server { } else { ScriptResult::Reject(format!("503 5.5.3 {reject_reason}")) } - } else if keep_id != usize::MAX - 1 { + } else if keep_id != DISCARD { if let Some(message) = messages.into_iter().nth(keep_id - 1) { trc::event!( Sieve(SieveEvent::ActionAccept), SpanId = session_id, - Id = script_id, + Script = script_id, Elapsed = time.elapsed(), ); ScriptResult::Replace { message, modifications, + redirected, } } else { trc::event!( Sieve(SieveEvent::ActionAcceptReplace), SpanId = session_id, - Id = script_id, + Script = script_id, Elapsed = time.elapsed(), ); - ScriptResult::Accept { modifications } + ScriptResult::Accept { + modifications, + redirected, + } } } else { trc::event!( Sieve(SieveEvent::ActionDiscard), SpanId = session_id, - Id = script_id, + Script = script_id, Elapsed = time.elapsed() ); @@ -433,3 +458,169 @@ impl RunScript for Server { } } } + +trait QueueSieveMessage { + fn queue_sieve_message( + &self, + messages: &[Vec], + original_raw_message: &[u8], + params: &ScriptParameters<'_>, + script_id: &str, + send: SieveSend, + exploded: bool, + ) -> impl Future + Send; +} + +impl QueueSieveMessage for Server { + async fn queue_sieve_message( + &self, + messages: &[Vec], + original_raw_message: &[u8], + params: &ScriptParameters<'_>, + script_id: &str, + send: SieveSend, + exploded: bool, + ) { + let SieveSend { + source, + recipients, + notify, + return_of_content, + by_time, + message_id, + } = send; + let session_id = params.session_id; + let is_redirect = source == SieveMessageSource::Redirect; + + // Build message + let return_path = if source == SieveMessageSource::Vacation { + "" + } else { + params.return_path.as_str() + }; + let mut message = self.new_message(return_path, MessageSource::Autogenerated, session_id); + for rcpt in recipients { + message.expand_and_add_recipient(rcpt, self).await; + } + + // Set notify flags + let flags = notify_flags(¬ify); + if flags > 0 { + for rcpt in &mut message.message.recipients { + rcpt.flags |= flags; + } + } + + // Set ByTime flags + match by_time { + ByTime::Relative { + rlimit, + mode, + trace, + } => { + if trace { + message.message.flags |= MAIL_BY_TRACE; + } + match mode { + ByMode::Notify | ByMode::Return => { + for domain in &mut message.message.recipients { + domain.notify.due += rlimit; + } + } + ByMode::Default => (), + } + } + ByTime::Absolute { + alimit, + mode, + trace, + } => { + if trace { + message.message.flags |= MAIL_BY_TRACE; + } + match mode { + ByMode::Notify => { + for domain in &mut message.message.recipients { + domain.notify.due = alimit as u64; + } + } + ByMode::Return => { + let expires = (alimit as u64).saturating_sub(message.message.created); + if expires > 0 { + for domain in &mut message.message.recipients { + domain.expires = QueueExpiry::Ttl(expires); + } + } + } + ByMode::Default => (), + } + } + ByTime::None => (), + }; + + // Set ret + match return_of_content { + Ret::Full => { + message.message.flags |= MAIL_RET_FULL; + } + Ret::Hdrs => { + message.message.flags |= MAIL_RET_HDRS; + } + Ret::Default => (), + } + + // Queue message + let is_forward = message_id == 0; + let raw_message = if !is_forward { + messages.get(message_id - 1).map(|m| m.as_slice()) + } else { + original_raw_message.into() + }; + let Some(raw_message) = raw_message.filter(|m| !m.is_empty()) else { + return; + }; + + let quotas = self.queue_quotas(&message).await; + + let dkim_signers = match ¶ms.sign_domain { + Some(sign_domain) => match self.dkim_signers(sign_domain).await { + Ok(signers) => signers, + Err(err) => { + trc::error!( + err.details("Failed to obtain DKIM signers") + .caused_by(trc::location!()) + ); + + None + } + }, + None => None, + }; + + let (dkim2_verified, verified_message) = + params.dkim2_verified.filter(|_| is_redirect).unzip(); + let queue_params = QueueParams::new(raw_message, session_id, self) + .with_dkim_signers(dkim_signers) + .with_raw_headers_opt(params.headers.filter(|_| is_forward)) + .with_original_raw_message(original_raw_message) + .with_dkim2_verified(dkim2_verified) + .with_exploded(exploded) + .with_quotas(quotas); + if message + .queue(match verified_message { + Some(verified_message) => { + queue_params.with_original_authenticated_message(verified_message) + } + None => queue_params, + }) + .await + == QueueOutcome::QuotaExceeded + { + trc::event!( + Sieve(SieveEvent::QuotaExceeded), + SpanId = session_id, + Script = CompactString::from(script_id), + ); + } + } +} diff --git a/crates/smtp/src/scripts/exec.rs b/crates/smtp/src/scripts/exec.rs index 2eeeaf18..7ca38bd8 100644 --- a/crates/smtp/src/scripts/exec.rs +++ b/crates/smtp/src/scripts/exec.rs @@ -4,13 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::SystemTime}; +use std::time::SystemTime; use common::network::SessionStream; -use mail_auth::common::resolver::ToReverseName; +use mail_auth::dns::ToReverseName; use sieve::{Envelope, Sieve, runtime::Variable}; use smtp_proto::*; +use utils::text::lowercase; use crate::{core::Session, inbound::AuthResult}; @@ -20,12 +21,12 @@ impl Session { pub fn build_script_parameters(&self, stage: &'static str) -> ScriptParameters<'_> { let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); let mut params = ScriptParameters::new() - .set_variable("remote_ip", self.data.remote_ip.to_string()) + .set_variable("remote_ip", self.data.remote_ip_str.as_str()) .set_variable("remote_ip.reverse", self.data.remote_ip.to_reverse_name()) - .set_variable("helo_domain", self.data.helo_domain.as_str().to_lowercase()) + .set_variable("helo_domain", lowercase(&self.data.helo_domain)) .set_variable( "authenticated_as", - self.authenticated_as().unwrap_or_default().to_string(), + self.authenticated_as().unwrap_or_default(), ) .set_variable( "now", @@ -47,8 +48,7 @@ impl Session { self.data .asn_geo_data .country - .as_ref() - .map(|r| r.as_str()) + .as_deref() .unwrap_or_default(), ) .set_variable( @@ -73,28 +73,26 @@ impl Session { if let Some(ip_rev) = &self.data.iprev { params = params.set_variable("iprev.result", ip_rev.result().as_str()); if let Some(ptr) = ip_rev.ptr.as_ref().and_then(|addrs| addrs.first()) { - params = params.set_variable( - "iprev.ptr", - ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase(), - ); + params = params + .set_variable("iprev.ptr", lowercase(ptr.strip_suffix('.').unwrap_or(ptr))); } } if let Some(mail_from) = &self.data.mail_from { params .envelope - .push((Envelope::From, mail_from.address_lcase.to_string().into())); + .push((Envelope::From, mail_from.address_lcase.as_str().into())); if let Some(env_id) = &mail_from.dsn_info { params .envelope - .push((Envelope::Envid, env_id.as_str().to_lowercase().into())); + .push((Envelope::Envid, lowercase(env_id).into())); } if stage != "data" { if let Some(rcpt) = self.data.rcpt_to.last() { params .envelope - .push((Envelope::To, rcpt.address_lcase.to_string().into())); + .push((Envelope::To, rcpt.address_lcase.as_str().into())); if let Some(orcpt) = rcpt.orcpt_parameter() { params.envelope.push((Envelope::Orcpt, orcpt.into())); } @@ -106,7 +104,7 @@ impl Session { let mut has_orcpts = false; for rcpt in &self.data.rcpt_to { - recipients.push(Variable::from(rcpt.address_lcase.to_string())); + recipients.push(Variable::from(rcpt.address_lcase.as_str())); orcpts.push(match rcpt.orcpt_parameter() { Some(orcpt) => { has_orcpts = true; @@ -155,19 +153,18 @@ impl Session { pub async fn run_script( &self, script_id: String, - script: Arc, + script: &Sieve<'_>, params: ScriptParameters<'_>, ) -> ScriptResult { - Box::pin( - self.server.run_script( + self.server + .run_script( script_id, script, params .with_session_id(self.data.session_id) .with_envelope(&self.server, self, self.data.session_id) .await, - ), - ) - .await + ) + .await } } diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index ff8d946a..bfff5e20 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -8,11 +8,20 @@ use std::borrow::Cow; use ahash::AHashMap; use common::{ - Server, auth::AccessToken, expr::functions::ResolveVariable, scripts::ScriptModification, + Server, + auth::AccessToken, + expr::{Bump, functions::ResolveVariable}, + scripts::ScriptModification, }; +use mail_auth::{AuthenticatedMessage, dkim2::Dkim2Output}; use mail_parser::Message; -use sieve::{Envelope, SpamStatus, runtime::Variable}; +use sieve::{ + Envelope, SpamStatus, + compiler::grammar::actions::action_redirect::{Notify, NotifyItem}, + runtime::Variable, +}; +use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS}; pub mod envelope; pub mod event_loop; @@ -22,20 +31,23 @@ pub mod exec; pub enum ScriptResult { Accept { modifications: Vec, + redirected: bool, }, Replace { message: Vec, modifications: Vec, + redirected: bool, }, Reject(String), Discard, } pub struct ScriptParameters<'x> { - message: Option>, + message: Option<&'x Message<'x>>, headers: Option<&'x [u8]>, - variables: AHashMap, Variable>, - envelope: Vec<(Envelope, Variable)>, + dkim2_verified: Option<(&'x Dkim2Output<'x>, &'x AuthenticatedMessage<'x>)>, + variables: AHashMap, Variable<'x>>, + envelope: Vec<(Envelope, Variable<'x>)>, from_addr: String, from_name: String, return_path: String, @@ -48,10 +60,11 @@ pub struct ScriptParameters<'x> { impl<'x> ScriptParameters<'x> { pub fn new() -> Self { ScriptParameters { - variables: AHashMap::with_capacity(10), + variables: AHashMap::with_capacity(24), envelope: Vec::with_capacity(6), message: None, headers: None, + dkim2_verified: None, from_addr: Default::default(), from_name: Default::default(), return_path: Default::default(), @@ -68,22 +81,23 @@ impl<'x> ScriptParameters<'x> { vars: &impl ResolveVariable, session_id: u64, ) -> Self { + let mut arena = Bump::new(); for (variable, expr) in [ (&mut self.from_addr, &server.core.sieve.from_addr), (&mut self.from_name, &server.core.sieve.from_name), (&mut self.return_path, &server.core.sieve.return_path), ] { - if let Some(value) = server.eval_if(expr, vars, session_id).await { + if let Some(value) = server.eval_if(expr, vars, &mut arena, session_id).await { *variable = value; } } self.sign_domain = server - .eval_if(&server.core.sieve.sign, vars, session_id) + .eval_if(&server.core.sieve.sign, vars, &mut arena, session_id) .await; self } - pub fn with_message(self, message: Message<'x>) -> Self { + pub fn with_message(self, message: &'x Message<'x>) -> Self { Self { message: message.into(), ..self @@ -97,6 +111,17 @@ impl<'x> ScriptParameters<'x> { } } + pub fn with_dkim2_verified( + self, + verified: Option<&'x Dkim2Output<'x>>, + verified_message: &'x AuthenticatedMessage<'x>, + ) -> Self { + Self { + dkim2_verified: verified.map(|verified| (verified, verified_message)), + ..self + } + } + pub fn with_spam_status(self, status: SpamStatus) -> Self { Self { spam_status: status.into(), @@ -107,13 +132,13 @@ impl<'x> ScriptParameters<'x> { pub fn set_variable( mut self, name: impl Into>, - value: impl Into, + value: impl Into>, ) -> Self { self.variables.insert(name.into(), value.into()); self } - pub fn set_envelope(mut self, envelope: Envelope, value: impl Into) -> Self { + pub fn set_envelope(mut self, envelope: Envelope, value: impl Into>) -> Self { self.envelope.push((envelope, value.into())); self } @@ -134,3 +159,18 @@ impl Default for ScriptParameters<'_> { Self::new() } } + +pub fn notify_flags(notify: &Notify) -> u64 { + match notify { + Notify::Never => RCPT_NOTIFY_NEVER, + Notify::Items(items) => items.iter().fold(0, |flags, item| { + flags + | match item { + NotifyItem::Success => RCPT_NOTIFY_SUCCESS, + NotifyItem::Failure => RCPT_NOTIFY_FAILURE, + NotifyItem::Delay => RCPT_NOTIFY_DELAY, + } + }), + Notify::Default => 0, + } +} diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml index 342867a1..e5f54b07 100644 --- a/crates/spam-filter/Cargo.toml +++ b/crates/spam-filter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spam-filter" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -11,30 +11,32 @@ store = { path = "../store" } trc = { path = "../trc" } common = { path = "../common" } registry = { path = "../registry" } -smtp-proto = { version = "0.2", features = ["rkyv"] } -mail-parser = { version = "0.11", features = ["full_encoding"] } -mail-auth = { version = "0.13" } -tokio = { version = "1.53", features = ["net", "macros"] } +smtp-proto = { path = "../smtp-proto", features = ["rkyv"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-auth = { version = "0.14" } +tokio = { version = "1.54", features = ["net", "macros"] } psl = "2" -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } idna = "1.1" -decancer = "3.3.3" +decancer = "4.0.0" unicode-security = "0.1.2" infer = "0.22" -hashify = "0.2" +hashify = "0.3" +memchr = "2.8" sha1 = "0.11" compact_str = "0.10.0" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true serde = { version = "1.0", features = ["derive"]} unicode-general-category = "1.1.0" -unicode-normalization = "0.1.25" +unicode-normalization = "0.1.26" +sync_wrapper = "1.0.2" +serde_json = { version = "1.0", features = ["raw_value"], optional = true } [features] -test_mode = [] -enterprise = [] +enterprise = ["common/enterprise", "dep:serde_json", "utils/enterprise", "store/enterprise", "trc/enterprise", "registry/enterprise"] [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } [lints] workspace = true diff --git a/crates/spam-filter/src/analysis/date.rs b/crates/spam-filter/src/analysis/date.rs index 849082e6..c8977ed6 100644 --- a/crates/spam-filter/src/analysis/date.rs +++ b/crates/spam-filter/src/analysis/date.rs @@ -4,14 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::SpamFilterContext; use common::Server; use mail_parser::HeaderName; +use std::future::Future; use store::write::now; -use crate::SpamFilterContext; - pub trait SpamFilterAnalyzeDate: Sync + Send { fn spam_filter_analyze_date( &self, @@ -24,7 +22,8 @@ impl SpamFilterAnalyzeDate for Server { match ctx .input .message - .header(HeaderName::Date) + .headers() + .value(HeaderName::Date) .map(|h| h.as_datetime()) { Some(Some(date)) => { diff --git a/crates/spam-filter/src/analysis/dmarc.rs b/crates/spam-filter/src/analysis/dmarc.rs index 9b4903a6..3c855b9c 100644 --- a/crates/spam-filter/src/analysis/dmarc.rs +++ b/crates/spam-filter/src/analysis/dmarc.rs @@ -4,12 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::SpamFilterContext; use common::Server; use mail_auth::{Dkim2Result, DkimResult, DmarcResult, SpfResult, dmarc::Policy}; - -use crate::SpamFilterContext; +use std::future::Future; pub trait SpamFilterAnalyzeDmarc: Sync + Send { fn spam_filter_analyze_dmarc( diff --git a/crates/spam-filter/src/analysis/domain.rs b/crates/spam-filter/src/analysis/domain.rs index ea3ed86b..1365c4df 100644 --- a/crates/spam-filter/src/analysis/domain.rs +++ b/crates/spam-filter/src/analysis/domain.rs @@ -18,9 +18,10 @@ use common::{ config::mailstore::spamfilter::{Element, Location}, }; use mail_auth::DkimResult; -use mail_parser::{HeaderName, HeaderValue, Host, parsers::MessageStream}; +use mail_parser::{HeaderForm, HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; -use std::{collections::HashSet, future::Future}; +use std::future::Future; +use store::ahash::AHashSet; pub trait SpamFilterAnalyzeDomain: Sync + Send { fn spam_filter_analyze_domain( @@ -31,188 +32,7 @@ pub trait SpamFilterAnalyzeDomain: Sync + Send { impl SpamFilterAnalyzeDomain for Server { async fn spam_filter_analyze_domain(&self, ctx: &mut SpamFilterContext<'_>) { - // Obtain email addresses and domains - let mut domains: HashSet> = HashSet::new(); - let mut emails: HashSet> = HashSet::new(); - - // Add DKIM domains - for dkim in ctx.input.dkim_result { - if dkim.result() == &DkimResult::Pass - && let Some(domain) = dkim.signature().map(|s| &s.d) - { - domains.insert(ElementLocation::new( - domain.to_lowercase(), - Location::HeaderDkimPass, - )); - } - } - - // Add Received headers - for header in ctx.input.message.headers() { - match (&header.name, &header.value) { - (HeaderName::Received, HeaderValue::Received(received)) => { - for host in [&received.from, &received.helo, &received.by] - .into_iter() - .flatten() - { - if let Host::Name(name) = host { - let host = Hostname::new(name.as_ref()); - - if host.sld.is_some() { - domains.insert(ElementLocation::new( - host.fqdn, - Location::HeaderReceived, - )); - } - } - } - } - (HeaderName::MessageId, value) => { - if let Some(mid_domain) = value - .as_text() - .and_then(|s| s.rsplit_once('@')) - .and_then(|(_, d)| { - let host = Hostname::new(d); - if host.sld.is_some() { Some(host) } else { None } - }) - { - domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid)); - } - } - (HeaderName::DispositionNotificationTo, _) => { - if let Some(address) = MessageStream::new( - ctx.input - .message - .raw_message - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(), - ) - .parse_address() - .as_address() - { - for addr in address.iter() { - if let Some(email) = addr.address() { - emails.insert(ElementLocation::new( - Recipient { - email: Email::new(email), - name: None, - }, - Location::HeaderDnt, - )); - } - } - } - } - _ => (), - } - } - - // Add EHLO domain - if ctx.output.ehlo_host.sld.is_some() { - domains.insert(ElementLocation::new( - ctx.output.ehlo_host.fqdn.clone(), - Location::Ehlo, - )); - } - - // Add PTR - if let Some(ptr) = &ctx.output.iprev_ptr { - domains.insert(ElementLocation::new(ptr.clone(), Location::Tcp)); - } - - // Add From, Envelope From and Reply-To - emails.insert(ElementLocation::new( - ctx.output.from.clone(), - Location::HeaderFrom, - )); - if let Some(reply_to) = &ctx.output.reply_to { - emails.insert(ElementLocation::new( - reply_to.clone(), - Location::HeaderReplyTo, - )); - } - emails.insert(ElementLocation::new( - Recipient { - email: ctx.output.env_from_addr.clone(), - name: None, - }, - Location::EnvelopeFrom, - )); - - // Add emails found in the message - for (part_id, part) in ctx.output.text_parts.iter().enumerate() { - let part_id = part_id as u32; - let is_body = ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id); - let tokens = match part { - TextPart::Plain { tokens, .. } => tokens, - TextPart::Html { - tokens, - html_tokens, - .. - } => { - emails.extend(html_tokens.iter().filter_map(|token| { - if let HtmlToken::StartTag { - name: A, - attributes, - .. - } = token - { - attributes.iter().find_map(|(attr, value)| { - if *attr == HREF { - let value = value.as_deref()?.strip_prefix("mailto:")?; - let email = - Email::new(value.split_once('?').map_or(value, |(e, _)| e)); - - if email.is_valid() { - return Some(ElementLocation::new( - Recipient { email, name: None }, - if is_body { - Location::BodyHtml - } else { - Location::Attachment - }, - )); - } - } - None - }) - } else { - None - } - })); - tokens - } - TextPart::None => continue, - }; - - for token in tokens { - if let TokenType::Email(email) = token { - if !ctx.input.is_train && is_body && !ctx.result.has_tag("RCPT_IN_BODY") { - for rcpt in ctx.output.all_recipients() { - if &rcpt.email == email { - ctx.result.add_tag("RCPT_IN_BODY"); - break; - } - } - } - - if email.is_valid() { - emails.insert(ElementLocation::new( - Recipient { - email: email.clone(), - name: None, - }, - if is_body { - Location::BodyText - } else { - Location::Attachment - }, - )); - } - } - } - } + let (mut domains, emails) = collect_domains_and_emails(ctx); if !ctx.input.is_train { // Validate email @@ -258,3 +78,188 @@ impl SpamFilterAnalyzeDomain for Server { ctx.output.domains = domains; } } + +type DomainsAndEmails = ( + AHashSet>, + AHashSet>, +); + +pub fn collect_domains_and_emails(ctx: &mut SpamFilterContext<'_>) -> DomainsAndEmails { + // Obtain email addresses and domains + let mut domains: AHashSet> = AHashSet::new(); + let mut emails: AHashSet> = AHashSet::new(); + + // Add DKIM domains + for dkim in ctx.input.dkim_result { + if dkim.result() == &DkimResult::Pass + && let Some(domain) = dkim.signature().map(|s| &s.d) + { + domains.insert(ElementLocation::new( + domain.to_lowercase(), + Location::HeaderDkimPass, + )); + } + } + + // Add Received headers + for header in ctx.input.message.headers() { + match (header.name(), header.value()) { + (HeaderName::Received, HeaderValue::Received(received)) => { + for host in [received.from(), received.helo(), received.by()] + .into_iter() + .flatten() + { + if let Host::Name(name) = host { + let host = Hostname::new(name); + + if host.sld.is_some() { + domains + .insert(ElementLocation::new(host.fqdn, Location::HeaderReceived)); + } + } + } + } + (HeaderName::MessageId, value) => { + if let Some(mid_domain) = + value + .as_text() + .and_then(|s| s.rsplit_once('@')) + .and_then(|(_, d)| { + let host = Hostname::new(d); + if host.sld.is_some() { Some(host) } else { None } + }) + { + domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid)); + } + } + (HeaderName::DispositionNotificationTo, _) => { + if let Some(address) = HeaderForm::Addresses + .parse(header.raw_value()) + .value() + .as_address() + { + for addr in address.mailboxes() { + if let Some(email) = addr.address() { + emails.insert(ElementLocation::new( + Recipient { + email: Email::new(email), + name: None, + }, + Location::HeaderDnt, + )); + } + } + } + } + _ => (), + } + } + + // Add EHLO domain + if ctx.output.ehlo_host.sld.is_some() { + domains.insert(ElementLocation::new( + ctx.output.ehlo_host.fqdn.clone(), + Location::Ehlo, + )); + } + + // Add PTR + if let Some(ptr) = &ctx.output.iprev_ptr { + domains.insert(ElementLocation::new(ptr.clone(), Location::Tcp)); + } + + // Add From, Envelope From and Reply-To + emails.insert(ElementLocation::new( + ctx.output.from.clone(), + Location::HeaderFrom, + )); + if let Some(reply_to) = &ctx.output.reply_to { + emails.insert(ElementLocation::new( + reply_to.clone(), + Location::HeaderReplyTo, + )); + } + emails.insert(ElementLocation::new( + Recipient { + email: ctx.output.env_from_addr.clone(), + name: None, + }, + Location::EnvelopeFrom, + )); + + // Add emails found in the message + for (part_id, part) in ctx.output.text_parts.iter().enumerate() { + let is_body = ctx.input.is_body(part_id as u32); + let (tokens, hidden_tokens) = match part { + TextPart::Plain { tokens, .. } => (tokens, &[][..]), + TextPart::Html { + tokens, + hidden_tokens, + html_tokens, + .. + } => { + emails.extend(html_tokens.iter().filter_map(|token| { + if let HtmlToken::StartTag { + name: A, + attributes, + .. + } = token + { + attributes.iter().find_map(|(attr, value)| { + if *attr == HREF { + let value = value.as_deref()?.strip_prefix("mailto:")?; + let email = + Email::new(value.split_once('?').map_or(value, |(e, _)| e)); + + if email.is_valid() { + return Some(ElementLocation::new( + Recipient { email, name: None }, + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + )); + } + } + None + }) + } else { + None + } + })); + (tokens, hidden_tokens.as_slice()) + } + TextPart::None => continue, + }; + + for token in tokens.iter().chain(hidden_tokens) { + if let TokenType::Email(email) = token { + if !ctx.input.is_train && is_body && !ctx.result.has_tag("RCPT_IN_BODY") { + for rcpt in ctx.output.all_recipients() { + if rcpt.email == **email { + ctx.result.add_tag("RCPT_IN_BODY"); + break; + } + } + } + + if email.is_valid() && !emails.contains(&**email) { + emails.insert(ElementLocation::new( + Recipient { + email: (**email).clone(), + name: None, + }, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + )); + } + } + } + } + + (domains, emails) +} diff --git a/crates/spam-filter/src/analysis/ehlo.rs b/crates/spam-filter/src/analysis/ehlo.rs index 2c2de143..9a0d9f1e 100644 --- a/crates/spam-filter/src/analysis/ehlo.rs +++ b/crates/spam-filter/src/analysis/ehlo.rs @@ -39,11 +39,11 @@ impl SpamFilterAnalyzeEhlo for Server { } if matches!( - ( - self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await, - self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await - ), - (Ok(false), Ok(false)) + self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await, + Ok(false) + ) && matches!( + self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await, + Ok(false) ) { // Helo no resolve to A or MX ctx.result.add_tag("HELO_NORES_A_OR_MX"); diff --git a/crates/spam-filter/src/analysis/from.rs b/crates/spam-filter/src/analysis/from.rs index 69a33892..35d0f105 100644 --- a/crates/spam-filter/src/analysis/from.rs +++ b/crates/spam-filter/src/analysis/from.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Email, SpamFilterContext}; +use crate::{ + Email, SpamFilterContext, + analysis::{ExcessEncoding, excess_encoding}, +}; use common::Server; use mail_parser::HeaderName; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; @@ -20,208 +23,203 @@ pub trait SpamFilterAnalyzeFrom: Sync + Send { impl SpamFilterAnalyzeFrom for Server { async fn spam_filter_analyze_from(&self, ctx: &mut SpamFilterContext<'_>) { - let mut from_count = 0; - let mut from_raw = b"".as_slice(); - let mut crt = None; - let mut dnt = None; + analyze_from(ctx); - for header in ctx.input.message.headers() { - match &header.name { - HeaderName::From => { - from_count += 1; - from_raw = ctx - .input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(); - } - HeaderName::DispositionNotificationTo => { - dnt = ctx - .input - .header_as_address(header) - .map(|s| Email::new(s.as_ref())); - } - HeaderName::Other(name) if name.eq_ignore_ascii_case("X-Confirm-Reading-To") => { - crt = ctx - .input - .header_as_address(header) - .map(|s| Email::new(s.as_ref())); - } - _ => {} - } - } - - match from_count { - 0 => { - ctx.result.add_tag("MISSING_FROM"); - } - 1 => {} - _ => { - ctx.result.add_tag("MULTIPLE_FROM"); - } - } - - let env_from_empty = ctx.output.env_from_addr.address.is_empty(); - let from_addr = &ctx.output.from.email; - let from_name = ctx.output.from.name.as_deref().unwrap_or_default(); - if from_count > 0 { - // Validate address - let from_addr_is_valid = from_addr.is_valid(); - if !from_addr_is_valid { - ctx.result.add_tag("FROM_INVALID"); - } - - // Validate from name - let from_name_trimmed = from_name.trim(); - if from_name_trimmed.is_empty() { - ctx.result.add_tag("FROM_NO_DN"); - } else if from_name_trimmed == from_addr.address { - ctx.result.add_tag("FROM_DN_EQ_ADDR"); - } else { - if from_addr_is_valid { - ctx.result.add_tag("FROM_HAS_DN"); - } - - if from_name_trimmed.contains('@') - && let Some(from_name_addr) = TypesTokenizer::new(from_name_trimmed) - .tokenize_numbers(false) - .tokenize_urls(false) - .tokenize_urls_without_scheme(false) - .tokenize_emails(true) - .filter_map(|t| match t.word { - TokenType::Email(email) => { - let email = Email::new(email); - email.is_valid().then_some(email) - } - _ => None, - }) - .next() - { - if (from_addr_is_valid - && from_name_addr.domain_part.sld != from_addr.domain_part.sld) - || (!env_from_empty - && ctx.output.env_from_addr.domain_part.sld - != from_name_addr.domain_part.sld) - || (env_from_empty - && ctx.output.ehlo_host.sld != from_name_addr.domain_part.sld) - { - ctx.result.add_tag("SPOOF_DISPLAY_NAME"); - } else { - ctx.result.add_tag("FROM_NEQ_DISPLAY_NAME"); - } - } - } - - // Check sender - if ctx.output.env_from_postmaster { - ctx.result.add_tag("FROM_BOUNCE"); - } - - if !env_from_empty && ctx.output.env_from_addr == *from_addr { - ctx.result.add_tag("FROM_EQ_ENV_FROM"); - } else if from_addr_is_valid { - if from_addr.domain_part.sld == ctx.output.ehlo_host.sld { - ctx.result.add_tag("FROMTLD_EQ_ENV_FROMTLD"); - } else if !ctx.output.env_from_postmaster { - ctx.result.add_tag("FORGED_SENDER"); - ctx.result.add_tag("FROM_NEQ_ENV_FROM"); - } - } - - // Validate FROM/TO relationship - if ctx.output.recipients_to.len() + ctx.output.recipients_cc.len() == 1 { - let rcpt = ctx - .output - .recipients_to - .first() - .or_else(|| ctx.output.recipients_cc.first()) - .unwrap(); - if rcpt.email == *from_addr { - ctx.result.add_tag("TO_EQ_FROM"); - } else if rcpt.email.domain_part.fqdn == from_addr.domain_part.fqdn { - ctx.result.add_tag("TO_DOM_EQ_FROM_DOM"); - } - } - - // Validate encoding - let from_raw_utf8 = std::str::from_utf8(from_raw); - if !from_raw.is_ascii() { - if (ctx.input.env_from_flags - & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME)) - == 0 - { - ctx.result.add_tag("FROM_NEEDS_ENCODING"); - } - - if from_raw_utf8.is_err() { - ctx.result.add_tag("INVALID_FROM_8BIT"); - } - } - - // Validate unnecessary encoding - let from_raw_utf8 = from_raw_utf8.unwrap_or_default(); - if from_name.is_ascii() - && from_addr.address.is_ascii() - && from_raw_utf8.contains("=?") - && from_raw_utf8.contains("?=") - { - if from_raw_utf8.contains("?q?") || from_raw_utf8.contains("?Q?") { - // From header is unnecessarily encoded in quoted-printable - ctx.result.add_tag("FROM_EXCESS_QP"); - } else if from_raw_utf8.contains("?b?") || from_raw_utf8.contains("?B?") { - // From header is unnecessarily encoded in base64 - ctx.result.add_tag("FROM_EXCESS_BASE64"); - } - } - - // Validate space in FROM - if !from_name.is_empty() - && !from_addr.address.is_empty() - && from_raw_utf8 - .as_bytes() - .iter() - .position(|&b| b == b'<') - .and_then(|v| from_raw_utf8.as_bytes().get(v - 1)) - .is_none_or(|v| !v.is_ascii_whitespace()) - { - ctx.result.add_tag("NO_SPACE_IN_FROM"); - } - - // Check whether read confirmation address is different to from address - if let Some(crt) = &crt - && crt != from_addr - { - ctx.result.add_tag("HEADER_RCONFIRM_MISMATCH"); - } - } - - if !env_from_empty { - // Validate envelope address - if ctx.output.env_from_addr.is_valid() { - // Mail from no resolve to A or MX - if matches!( - ( - self.dns_exists_ip(&ctx.output.env_from_addr.domain_part.fqdn) - .await, - self.dns_exists_mx(&ctx.output.env_from_addr.domain_part.fqdn) - .await - ), - (Ok(false), Ok(false)) - ) { - // Helo no resolve to A or MX - ctx.result.add_tag("FROMHOST_NORES_A_OR_MX"); - } - } else { - ctx.result.add_tag("ENV_FROM_INVALID"); - } - - // Check whether disposition notification address is different to return path - if let Some(dnt) = &dnt - && *dnt != ctx.output.env_from_addr - { - ctx.result.add_tag("HEADER_FORGED_MDN"); + if !ctx.output.env_from_addr.address.is_empty() && ctx.output.env_from_addr.is_valid() { + // Mail from no resolve to A or MX + if matches!( + self.dns_exists_ip(&ctx.output.env_from_addr.domain_part.fqdn) + .await, + Ok(false) + ) && matches!( + self.dns_exists_mx(&ctx.output.env_from_addr.domain_part.fqdn) + .await, + Ok(false) + ) { + // Helo no resolve to A or MX + ctx.result.add_tag("FROMHOST_NORES_A_OR_MX"); } } } } + +pub fn analyze_from(ctx: &mut SpamFilterContext<'_>) { + let mut from_count = 0; + let mut from_raw = b"".as_slice(); + let mut crt = None; + let mut dnt = None; + + for header in ctx.input.message.headers() { + match header.name() { + HeaderName::From => { + from_count += 1; + from_raw = header.raw_value(); + } + HeaderName::DispositionNotificationTo => { + dnt = ctx.input.header_as_address(&header).map(|s| Email::new(&s)); + } + HeaderName::Other(name) if name.eq_ignore_ascii_case("X-Confirm-Reading-To") => { + crt = ctx.input.header_as_address(&header).map(|s| Email::new(&s)); + } + _ => {} + } + } + + match from_count { + 0 => { + ctx.result.add_tag("MISSING_FROM"); + } + 1 => {} + _ => { + ctx.result.add_tag("MULTIPLE_FROM"); + } + } + + let env_from_empty = ctx.output.env_from_addr.address.is_empty(); + let from_addr = &ctx.output.from.email; + let from_name = ctx.output.from.name.as_deref().unwrap_or_default(); + if from_count > 0 { + // Validate address + let from_addr_is_valid = from_addr.is_valid(); + if !from_addr_is_valid { + ctx.result.add_tag("FROM_INVALID"); + } + + // Validate from name + let from_name_trimmed = from_name.trim(); + if from_name_trimmed.is_empty() { + ctx.result.add_tag("FROM_NO_DN"); + } else if from_name_trimmed == from_addr.address { + ctx.result.add_tag("FROM_DN_EQ_ADDR"); + } else { + if from_addr_is_valid { + ctx.result.add_tag("FROM_HAS_DN"); + } + + if from_name_trimmed.contains('@') + && let Some(from_name_addr) = TypesTokenizer::new(from_name_trimmed) + .tokenize_numbers(false) + .tokenize_urls(false) + .tokenize_urls_without_scheme(false) + .tokenize_emails(true) + .filter_map(|t| match t.word { + TokenType::Email(email) => { + let email = Email::new(email); + email.is_valid().then_some(email) + } + _ => None, + }) + .next() + { + if (from_addr_is_valid + && from_name_addr.domain_part.sld != from_addr.domain_part.sld) + || (!env_from_empty + && ctx.output.env_from_addr.domain_part.sld + != from_name_addr.domain_part.sld) + || (env_from_empty + && ctx.output.ehlo_host.sld != from_name_addr.domain_part.sld) + { + ctx.result.add_tag("SPOOF_DISPLAY_NAME"); + } else { + ctx.result.add_tag("FROM_NEQ_DISPLAY_NAME"); + } + } + } + + // Check sender + if ctx.output.env_from_postmaster { + ctx.result.add_tag("FROM_BOUNCE"); + } + + if !env_from_empty && ctx.output.env_from_addr == *from_addr { + ctx.result.add_tag("FROM_EQ_ENV_FROM"); + } else if from_addr_is_valid { + if from_addr.domain_part.sld == ctx.output.ehlo_host.sld { + ctx.result.add_tag("FROMTLD_EQ_ENV_FROMTLD"); + } else if !ctx.output.env_from_postmaster { + ctx.result.add_tag("FORGED_SENDER"); + ctx.result.add_tag("FROM_NEQ_ENV_FROM"); + } + } + + // Validate FROM/TO relationship + if ctx.output.recipients_to.len() + ctx.output.recipients_cc.len() == 1 { + let rcpt = ctx + .output + .recipients_to + .first() + .or_else(|| ctx.output.recipients_cc.first()) + .unwrap(); + if rcpt.email == *from_addr { + ctx.result.add_tag("TO_EQ_FROM"); + } else if rcpt.email.domain_part.fqdn == from_addr.domain_part.fqdn { + ctx.result.add_tag("TO_DOM_EQ_FROM_DOM"); + } + } + + // Validate encoding + let from_raw_utf8 = std::str::from_utf8(from_raw); + if !from_raw.is_ascii() { + if (ctx.input.env_from_flags + & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME)) + == 0 + { + ctx.result.add_tag("FROM_NEEDS_ENCODING"); + } + + if from_raw_utf8.is_err() { + ctx.result.add_tag("INVALID_FROM_8BIT"); + } + } + + // Validate unnecessary encoding + let from_raw_utf8 = from_raw_utf8.unwrap_or_default(); + if from_name.is_ascii() && from_addr.address.is_ascii() { + match excess_encoding(from_raw_utf8) { + ExcessEncoding::QuotedPrintable => { + // From header is unnecessarily encoded in quoted-printable + ctx.result.add_tag("FROM_EXCESS_QP"); + } + ExcessEncoding::Base64 => { + // From header is unnecessarily encoded in base64 + ctx.result.add_tag("FROM_EXCESS_BASE64"); + } + ExcessEncoding::None | ExcessEncoding::Other => {} + } + } + + // Validate space in FROM + if !from_name.is_empty() + && !from_addr.address.is_empty() + && from_raw_utf8 + .as_bytes() + .iter() + .position(|&b| b == b'<') + .and_then(|v| from_raw_utf8.as_bytes().get(v - 1)) + .is_none_or(|v| !v.is_ascii_whitespace()) + { + ctx.result.add_tag("NO_SPACE_IN_FROM"); + } + + // Check whether read confirmation address is different to from address + if let Some(crt) = &crt + && crt != from_addr + { + ctx.result.add_tag("HEADER_RCONFIRM_MISMATCH"); + } + } + + if !env_from_empty { + // Validate envelope address + if !ctx.output.env_from_addr.is_valid() { + ctx.result.add_tag("ENV_FROM_INVALID"); + } + + // Check whether disposition notification address is different to return path + if let Some(dnt) = &dnt + && *dnt != ctx.output.env_from_addr + { + ctx.result.add_tag("HEADER_FORGED_MDN"); + } + } +} diff --git a/crates/spam-filter/src/analysis/headers.rs b/crates/spam-filter/src/analysis/headers.rs index a0a4ef04..2151363f 100644 --- a/crates/spam-filter/src/analysis/headers.rs +++ b/crates/spam-filter/src/analysis/headers.rs @@ -4,13 +4,35 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{SpamFilterContext, analysis::eq_lowercase}; use common::Server; +use compact_str::CompactString; use mail_parser::HeaderName; +use std::future::Future; use store::ahash::AHashSet; -use crate::SpamFilterContext; +const TAG_BYTE: [u8; 256] = tag_byte_table(); + +const fn tag_byte_table() -> [u8; 256] { + let mut table = [b' '; 256]; + let mut idx = 0; + while idx < 256 { + let byte = idx as u8; + table[idx] = if byte.is_ascii_digit() || byte.is_ascii_uppercase() { + byte + } else if byte.is_ascii_lowercase() { + byte - 32 + } else if byte == b'-' { + b'_' + } else if byte & 0xc0 == 0x80 { + 0 + } else { + b' ' + }; + idx += 1; + } + table +} pub trait SpamFilterAnalyzeHeaders: Sync + Send { fn spam_filter_analyze_headers( @@ -22,26 +44,15 @@ pub trait SpamFilterAnalyzeHeaders: Sync + Send { impl SpamFilterAnalyzeHeaders for Server { async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) { let mut list_score = 0.0; - let mut unique_headers = AHashSet::new(); - let raw_message = ctx.input.message.raw_message(); + let mut unique_headers = AHashSet::with_capacity(13); + let raw_message = ctx.input.message.raw(); for header in ctx.input.message.headers() { // Add header exists tag - let hdr_name = header.name(); - let mut tag: String = String::with_capacity(hdr_name.len() + 5); - tag.push_str("X_HDR_"); - for ch in hdr_name.chars() { - if ch.is_ascii_alphanumeric() { - tag.push(ch.to_ascii_uppercase()); - } else if ch == '-' { - tag.push('_'); - } else { - tag.push(' '); - } - } - ctx.result.add_tag(tag); + let name = header.name(); + ctx.result.add_tag(header_exists_tag(name.as_str())); - match &header.name { + match name { HeaderName::ContentType | HeaderName::ContentTransferEncoding | HeaderName::Date @@ -55,12 +66,12 @@ impl SpamFilterAnalyzeHeaders for Server { | HeaderName::MessageId | HeaderName::References | HeaderName::InReplyTo => { - if !unique_headers.insert(header.name.clone()) { + if !unique_headers.insert(name) { ctx.result.add_tag("MULTIPLE_UNIQUE_HEADERS"); } let mut value = raw_message - .get(header.offset_start as usize..) + .get(header.offset_start() as usize..) .unwrap_or_default() .iter(); loop { @@ -92,44 +103,41 @@ impl SpamFilterAnalyzeHeaders for Server { list_score += 0.25; ctx.result.add_tag("HAS_LIST_UNSUB"); } - HeaderName::Other(name) => { - let value = header - .value() - .as_text() - .unwrap_or_default() - .trim() - .to_lowercase(); + HeaderName::Precedence => { + let value = header.value().as_text().unwrap_or_default().trim(); - if name.eq_ignore_ascii_case("Precedence") { - if value == "bulk" { - list_score += 0.25; - ctx.result.add_tag("PRECEDENCE_BULK"); - } else if value == "list" { - list_score += 0.25; - } - } else if name.eq_ignore_ascii_case("X-Loop") { - list_score += 0.125; - } else if name.eq_ignore_ascii_case("X-Priority") { - match value.parse::().unwrap_or(i32::MAX) { - 0 => { - ctx.result.add_tag("HAS_X_PRIO_ZERO"); - } - 1 => { - ctx.result.add_tag("HAS_X_PRIO_ONE"); - } - 2 => { - ctx.result.add_tag("HAS_X_PRIO_TWO"); - } - 3 | 4 => { - ctx.result.add_tag("HAS_X_PRIO_THREE"); - } - 4..=10000 => { - ctx.result.add_tag("HAS_X_PRIO_FIVE"); - } - _ => {} - } + if eq_lowercase(value, "bulk") { + list_score += 0.25; + ctx.result.add_tag("PRECEDENCE_BULK"); + } else if eq_lowercase(value, "list") { + list_score += 0.25; } } + HeaderName::XPriority => { + let value = header.value().as_text().unwrap_or_default().trim(); + + match value.parse::().unwrap_or(i32::MAX) { + 0 => { + ctx.result.add_tag("HAS_X_PRIO_ZERO"); + } + 1 => { + ctx.result.add_tag("HAS_X_PRIO_ONE"); + } + 2 => { + ctx.result.add_tag("HAS_X_PRIO_TWO"); + } + 3 | 4 => { + ctx.result.add_tag("HAS_X_PRIO_THREE"); + } + 4..=10000 => { + ctx.result.add_tag("HAS_X_PRIO_FIVE"); + } + _ => {} + } + } + HeaderName::Other(name) if name.eq_ignore_ascii_case("X-Loop") => { + list_score += 0.125; + } _ => {} } } @@ -143,3 +151,15 @@ impl SpamFilterAnalyzeHeaders for Server { } } } + +fn header_exists_tag(name: &str) -> CompactString { + let mut tag = CompactString::with_capacity(name.len() + 6); + tag.push_str("X_HDR_"); + for &byte in name.as_bytes() { + let mapped = TAG_BYTE[byte as usize]; + if mapped != 0 { + tag.push(mapped as char); + } + } + tag +} diff --git a/crates/spam-filter/src/analysis/html.rs b/crates/spam-filter/src/analysis/html.rs index c9678172..c0d01332 100644 --- a/crates/spam-filter/src/analysis/html.rs +++ b/crates/spam-filter/src/analysis/html.rs @@ -4,14 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{ + Hostname, SpamFilterContext, TextPart, + analysis::{contains_ignore_ascii_case, starts_with_ignore_ascii_case, to_lowercase_cow}, + modules::html::*, +}; use common::Server; use hyper::Uri; -use mail_parser::MimeHeaders; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; - -use crate::{Hostname, SpamFilterContext, TextPart, modules::html::*}; +use std::{borrow::Cow, future::Future}; pub trait SpamFilterAnalyzeHtml: Sync + Send { fn spam_filter_analyze_html( @@ -21,13 +22,19 @@ pub trait SpamFilterAnalyzeHtml: Sync + Send { } #[derive(Debug)] -struct Href { - url_parsed: Option, - host: Option, +struct Href<'x> { + url_parsed: Option>, + host: Option>, } impl SpamFilterAnalyzeHtml for Server { async fn spam_filter_analyze_html(&self, ctx: &mut SpamFilterContext<'_>) { + SpamFilterContext::analyze_html(ctx); + } +} + +impl SpamFilterContext<'_> { + pub fn analyze_html(ctx: &mut Self) { // Message only has text/html MIME parts if ctx.input.message.content_type().is_some_and(|ct| { ct.ctype().eq_ignore_ascii_case("text") @@ -40,9 +47,7 @@ impl SpamFilterAnalyzeHtml for Server { } for (part_id, part) in ctx.output.text_parts.iter().enumerate() { - let part_id = part_id as u32; - let is_body_part = ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id); + let is_body_part = ctx.input.is_body(part_id as u32); let (html_tokens, tokens) = if let TextPart::Html { html_tokens, @@ -76,13 +81,25 @@ impl SpamFilterAnalyzeHtml for Server { None } }) { - let url = attr.trim().to_lowercase(); - let url_parsed = url.parse::().ok(); - let href = Href { - host: url_parsed - .as_ref() - .and_then(|uri| uri.host().map(Hostname::new)), - url_parsed, + let url = to_lowercase_cow(attr.trim()); + let href = if let Some(parsed) = ctx + .output + .urls + .get(url.as_ref()) + .and_then(|url| url.element.url_parsed.as_ref()) + { + Href { + url_parsed: Some(Cow::Borrowed(&parsed.parts)), + host: Some(Cow::Borrowed(&parsed.host)), + } + } else { + let url_parsed = url.parse::().ok(); + Href { + host: url_parsed.as_ref().and_then(|uri| { + uri.host().map(|host| Cow::Owned(Hostname::new(host))) + }), + url_parsed: url_parsed.map(Cow::Owned), + } }; if is_body_part @@ -117,14 +134,16 @@ impl SpamFilterAnalyzeHtml for Server { WIDTH => &mut img_width, HEIGHT => &mut img_height, SRC => { - let src = value.to_ascii_lowercase(); - if src.starts_with("data:") && src.contains(";base64,") + if starts_with_ignore_ascii_case(value, "data:") + && contains_ignore_ascii_case(value, ";base64,") { // Has Data URI encoding ctx.result.add_tag("HAS_DATA_URI"); - } else if src.starts_with("https://") - || src.starts_with("http://") - { + } else if starts_with_ignore_ascii_case( + value, "https://", + ) || starts_with_ignore_ascii_case( + value, "http://", + ) { // Has external image ctx.result.add_tag("HAS_EXTERNAL_IMG"); } @@ -172,7 +191,7 @@ impl SpamFilterAnalyzeHtml for Server { has_equiv_refresh = true; } } else if *attr == CONTENT - && value.to_ascii_lowercase().contains("url=") + && contains_ignore_ascii_case(value, "url=") { has_content_url = true; } @@ -193,11 +212,11 @@ impl SpamFilterAnalyzeHtml for Server { value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty()) { if *attr == REL { - if value.to_ascii_lowercase().contains("stylesheet") { + if contains_ignore_ascii_case(value, "stylesheet") { has_rel_style = true; } } else if *attr == HREF - && value.to_ascii_lowercase().contains(".css") + && contains_ignore_ascii_case(value, ".css") { has_href_css = true; } @@ -232,7 +251,10 @@ impl SpamFilterAnalyzeHtml for Server { HtmlToken::Text { text } if in_head == 0 => { if let Some((href_url, href_host)) = last_href .as_ref() - .and_then(|href| Some((href.url_parsed.as_ref()?, href.host.as_ref()?))) + .and_then(|href| { + Some((href.url_parsed.as_deref()?, href.host.as_deref()?)) + }) + .filter(|_| can_contain_url(text)) { for token in TypesTokenizer::new(text.as_ref()) .tokenize_numbers(false) @@ -241,9 +263,13 @@ impl SpamFilterAnalyzeHtml for Server { .tokenize_emails(true) { let text_url = match token.word { - TokenType::Url(url) => url.to_lowercase(), + TokenType::Url(url) => to_lowercase_cow(url), TokenType::UrlNoScheme(url) => { - format!("http://{}", url.to_lowercase()) + let url = to_lowercase_cow(url); + let mut with_scheme = String::with_capacity(url.len() + 7); + with_scheme.push_str("http://"); + with_scheme.push_str(url.as_ref()); + Cow::Owned(with_scheme) } _ => continue, }; @@ -283,7 +309,8 @@ impl SpamFilterAnalyzeHtml for Server { } if is_body_part { - if in_head != 0 || in_body != 0 { + let is_whole = !ctx.input.texts.is_cut(part_id as u32); + if is_whole && (in_head != 0 || in_body != 0) { // HTML tags are not properly closed ctx.result.add_tag("HTML_UNBALANCED_TAG"); } @@ -310,6 +337,7 @@ impl SpamFilterAnalyzeHtml for Server { } match html_text_chars { + _ if !is_whole => (), 0..1024 => { ctx.result.add_tag("HTML_SHORT_1"); } @@ -329,7 +357,7 @@ impl SpamFilterAnalyzeHtml for Server { ctx.result.add_tag("HTML_TEXT_IMG_RATIO"); } - if html_uris > 0 && html_words == 0 { + if is_whole && html_uris > 0 && html_words == 0 { // Message only contains URIs in HTML ctx.result.add_tag("BODY_URI_ONLY"); } @@ -337,3 +365,9 @@ impl SpamFilterAnalyzeHtml for Server { } } } + +fn can_contain_url(text: &str) -> bool { + text.as_bytes() + .iter() + .any(|byte| matches!(byte, b'.' | b'[')) +} diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs index 0bdfe8dd..4f51abd3 100644 --- a/crates/spam-filter/src/analysis/init.rs +++ b/crates/spam-filter/src/analysis/init.rs @@ -4,28 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; - -use mail_auth::DmarcResult; -use mail_parser::{HeaderName, PartType, parsers::fields::thread::thread_name}; -use nlp::tokenizers::types::{TokenType, TypesTokenizer}; - -use crate::{ - Email, Hostname, IpParts, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, - SpamFilterResult, TextPart, - modules::html::{HEAD, HtmlToken, html_to_tokens}, -}; - use super::url::UrlParts; +use crate::{ + ContextToken, Email, Hostname, IpParts, Recipient, SpamFilterContext, SpamFilterInput, + SpamFilterOutput, SpamFilterResult, TextPart, modules::html::html_to_tokens, +}; +use common::{Server, expr::Bump}; +use mail_auth::DmarcResult; +use mail_parser::{HeaderName, Mailbox, PartKind, thread_name}; +use nlp::tokenizers::types::{TokenType, TypesTokenizer}; +use std::borrow::Cow; +use store::ahash::AHashSet; +use sync_wrapper::SyncWrapper; pub trait SpamFilterInit { fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>; } const POSTMASTER_ADDRESSES: [&str; 3] = ["postmaster", "mailer-daemon", "root"]; +const BYTES_PER_TOKEN: usize = 3; +const MAX_RESERVED_TOKENS: usize = 1 << 16; +const TAG_CAPACITY: usize = 64; impl SpamFilterInit for Server { - fn spam_filter_init<'x>(&self, mut input: SpamFilterInput<'x>) -> SpamFilterContext<'x> { + fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x> { + SpamFilterContext::new(input) + } +} + +impl<'x> SpamFilterContext<'x> { + pub fn new(mut input: SpamFilterInput<'x>) -> Self { let mut subject = ""; let mut from = None; let mut reply_to = None; @@ -35,29 +43,16 @@ impl SpamFilterInit for Server { let mut found_spam_status = false; for header in input.message.headers() { - match &header.name { + let name = header.name(); + match name { HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { - if let Some(addrs) = header.value().as_address() { - for addr in addrs.iter() { - let rcpt = Recipient { - email: Email::new(addr.address().unwrap_or_default()), - name: addr.name().and_then(|s| { - let s = s.trim(); - if !s.is_empty() { - Some(s.to_lowercase()) - } else { - None - } - }), - }; - if header.name == HeaderName::To { - recipients_to.push(rcpt); - } else if header.name == HeaderName::Cc { - recipients_cc.push(rcpt); - } else { - recipients_bcc.push(rcpt); - } - } + let recipients = match name { + HeaderName::To => &mut recipients_to, + HeaderName::Cc => &mut recipients_cc, + _ => &mut recipients_bcc, + }; + if let Some(addresses) = header.value().as_address() { + recipients.extend(addresses.mailboxes().map(Recipient::from)); } } HeaderName::ReplyTo => { @@ -68,14 +63,7 @@ impl SpamFilterInit for Server { .and_then(|addr| { Some(Recipient { email: Email::new(addr.address()?), - name: addr.name().and_then(|s| { - let s = s.trim(); - if !s.is_empty() { - Some(s.to_lowercase()) - } else { - None - } - }), + name: addr.name().and_then(trimmed_name), }) }); } @@ -85,9 +73,7 @@ impl SpamFilterInit for Server { HeaderName::From => { from = header.value().as_address().and_then(|addrs| addrs.first()); } - HeaderName::Other(name) - if input.is_train && !found_spam_status && name.eq("X-Spam-Result") => - { + HeaderName::XSpamResult if input.is_train && !found_spam_status => { for token in header .value() .as_text() @@ -115,153 +101,33 @@ impl SpamFilterInit for Server { } // Tokenize subject - let subject_tokens = TypesTokenizer::new(subject) - .tokenize_numbers(false) - .tokenize_urls(true) - .tokenize_urls_without_scheme(true) - .tokenize_emails(true) - .map(|t| match t.word { - TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), - TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), - TokenType::Integer(s) => TokenType::Integer(s.into()), - TokenType::Other(s) => TokenType::Other(s), - TokenType::Punctuation(s) => TokenType::Punctuation(s), - TokenType::Space => TokenType::Space, - TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), - TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), - TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme(UrlParts::no_scheme(s)), - TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), - TokenType::Email(e) => TokenType::Email(Email::new(e)), - TokenType::Float(s) => TokenType::Float(s.into()), - }) - .collect::>(); + let subject_tokens = tokenize(subject); // Tokenize and convert text parts - let mut text_parts = Vec::new(); - let mut text_parts_nested = Vec::new(); - let mut message_stack = Vec::new(); - let mut message_iter = input.message.parts.iter(); - - loop { - while let Some(part) = message_iter.next() { - let is_main_message = message_stack.is_empty(); - let text_part = match &part.body { - PartType::Text(text) => TextPart::Plain { - text_body: text.as_ref(), - tokens: TypesTokenizer::new(text.as_ref()) - .tokenize_numbers(false) - .tokenize_urls(true) - .tokenize_urls_without_scheme(true) - .tokenize_emails(true) - .map(|t| match t.word { - TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), - TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), - TokenType::Integer(s) => TokenType::Integer(s.into()), - TokenType::Other(s) => TokenType::Other(s), - TokenType::Punctuation(s) => TokenType::Punctuation(s), - TokenType::Space => TokenType::Space, - TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), - TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), - TokenType::UrlNoScheme(s) => { - TokenType::UrlNoScheme(UrlParts::no_scheme(s)) - } - TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), - TokenType::Email(e) => TokenType::Email(Email::new(e)), - TokenType::Float(s) => TokenType::Float(s.into()), - }) - .collect::>(), - }, - PartType::Html(html) => { - let html_tokens = html_to_tokens(html); - let text_body_len = html_tokens - .iter() - .filter_map(|t| match t { - HtmlToken::Text { text } => text.len().into(), - _ => None, - }) - .sum(); - let mut text_body = String::with_capacity(text_body_len); - let mut in_head = false; - for token in &html_tokens { - match token { - HtmlToken::StartTag { name: HEAD, .. } => { - in_head = true; - } - HtmlToken::EndTag { name: HEAD } => { - in_head = false; - } - HtmlToken::Text { text } if !in_head => { - if !text_body.is_empty() - && !text_body.ends_with(' ') - && !text.starts_with(' ') - { - text_body.push(' '); - } - text_body.push_str(text) - } - _ => {} - } - } - - TextPart::Html { - tokens: TypesTokenizer::new(&text_body) - .tokenize_numbers(false) - .tokenize_urls(true) - .tokenize_urls_without_scheme(true) - .tokenize_emails(true) - .map(|t| match t.word { - TokenType::Alphabetic(s) => { - TokenType::Alphabetic(s.to_string().into()) - } - TokenType::Alphanumeric(s) => { - TokenType::Alphanumeric(s.to_string().into()) - } - TokenType::Integer(s) => { - TokenType::Integer(s.to_string().into()) - } - TokenType::Other(s) => TokenType::Other(s), - TokenType::Punctuation(s) => TokenType::Punctuation(s), - TokenType::Space => TokenType::Space, - TokenType::Url(url) => { - TokenType::Url(UrlParts::new(url.to_string())) - } - TokenType::UrlNoHost(s) => { - TokenType::UrlNoHost(s.to_string().into()) - } - TokenType::UrlNoScheme(s) => { - TokenType::UrlNoScheme(UrlParts::no_scheme(s.to_string())) - } - TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), - TokenType::Email(e) => TokenType::Email(Email::new(e)), - TokenType::Float(s) => TokenType::Float(s.to_string().into()), - }) - .collect::>(), + let texts = input.texts; + let text_parts = input + .message + .parts() + .map(|part| match (part.kind(), texts.get(part.id())) { + (PartKind::Text, Some(text)) => TextPart::Plain { + text_body: text, + tokens: tokenize(text), + }, + (PartKind::Html, Some(html)) => { + let html_tokens = html_to_tokens(html); + match texts.html_text(part.id(), &html_tokens) { + Some(text) => TextPart::Html { + tokens: tokenize(&text.visible), + hidden_tokens: tokenize_addresses(&text.hidden), html_tokens, - text_body, - } + text_body: &text.visible, + }, + None => TextPart::None, } - PartType::Message(message) => { - message_stack.push(message_iter); - message_iter = message.parts.iter(); - TextPart::None - } - _ => TextPart::None, - }; - - if is_main_message { - text_parts.push(text_part); - } else if !matches!(text_part, TextPart::None) { - text_parts_nested.push(text_part); } - } - - if let Some(iter) = message_stack.pop() { - message_iter = iter; - } else { - break; - } - } - text_parts.extend(text_parts_nested); + _ => TextPart::None, + }) + .collect(); let subject_thread = thread_name(subject).to_string(); let env_from_addr = Email::new(input.env_from); @@ -282,11 +148,6 @@ impl SpamFilterInit for Server { .iter() .map(|rcpt| Email::new(rcpt)) .collect(), - env_to_rewritten_addr: input - .env_rcpt_rewritten_to - .iter() - .map(|rcpt| Email::new(rcpt)) - .collect(), from: Recipient { email: Email::new(from.and_then(|f| f.address()).unwrap_or_default()), name: from.and_then(|f| f.name()).map(|name| name.to_lowercase()), @@ -307,7 +168,79 @@ impl SpamFilterInit for Server { domains: Default::default(), }, input, - result: SpamFilterResult::default(), + result: SpamFilterResult { + tags: AHashSet::with_capacity(TAG_CAPACITY), + ..Default::default() + }, + arena: SyncWrapper::new(Bump::new()), } } } + +pub(crate) fn tokenize(text: &str) -> Vec> { + let mut tokens = Vec::with_capacity((text.len() / BYTES_PER_TOKEN).min(MAX_RESERVED_TOKENS)); + tokens.extend(types_tokenizer(text).map(|token| context_token(token.word))); + tokens +} + +fn tokenize_addresses(text: &str) -> Vec> { + if text.is_empty() { + return Vec::new(); + } + types_tokenizer(text) + .filter(|token| { + matches!( + token.word, + TokenType::Url(_) + | TokenType::UrlNoScheme(_) + | TokenType::IpAddr(_) + | TokenType::Email(_) + ) + }) + .map(|token| context_token(token.word)) + .collect() +} + +fn types_tokenizer(text: &str) -> TypesTokenizer<'_> { + TypesTokenizer::new(text) + .tokenize_numbers(false) + .tokenize_urls(true) + .tokenize_urls_without_scheme(true) + .tokenize_emails(true) +} + +fn context_token<'x>(word: TokenType<&'x str, &'x str, &'x str, &'x str>) -> ContextToken<'x> { + match word { + TokenType::Alphabetic(s) => TokenType::Alphabetic(Cow::Borrowed(s)), + TokenType::Alphanumeric(s) => TokenType::Alphanumeric(Cow::Borrowed(s)), + TokenType::Integer(s) => TokenType::Integer(Cow::Borrowed(s)), + TokenType::Other(s) => TokenType::Other(s), + TokenType::Punctuation(s) => TokenType::Punctuation(s), + TokenType::Space => TokenType::Space, + TokenType::Url(url) => TokenType::Url(Box::new(UrlParts::new(url))), + TokenType::UrlNoHost(s) => TokenType::UrlNoHost(Cow::Borrowed(s)), + TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme(Box::new(UrlParts::no_scheme(s))), + TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), + TokenType::Email(e) => TokenType::Email(Box::new(Email::new(e))), + TokenType::Float(s) => TokenType::Float(Cow::Borrowed(s)), + } +} + +impl From> for Recipient { + fn from(addr: Mailbox<'_>) -> Self { + Recipient { + email: Email::new(addr.address().unwrap_or_default()), + name: addr.name().and_then(trimmed_name), + } + } +} + +fn trimmed_name(name: &str) -> Option { + let name = name.trim(); + if !name.is_empty() { + Some(name.to_lowercase()) + } else { + None + } +} + diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index 1ae312c2..5417d577 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -5,13 +5,16 @@ */ use super::ElementLocation; -use crate::{IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl}; +use crate::{ + IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl, + modules::sanitize::can_be_ip_addr, +}; use common::{ Server, config::mailstore::spamfilter::{Element, IpResolver, Location}, }; use mail_auth::IprevResult; -use mail_parser::{HeaderName, HeaderValue, Host}; +use mail_parser::Host; use nlp::tokenizers::types::TokenType; use std::future::Future; use store::ahash::AHashSet; @@ -31,57 +34,54 @@ impl SpamFilterAnalyzeIp for Server { ips.insert(ElementLocation::new(ctx.input.remote_ip, Location::Tcp)); // Obtain IP addresses from Received headers - for header in ctx.input.message.headers() { - if let (HeaderName::Received, HeaderValue::Received(received)) = - (&header.name, &header.value) + for received in ctx.input.message.headers().all_received() { + if let Some(ip) = received.from_ip() + && !ip.is_loopback() + && !self.is_ip_allowed(ip) { - if let Some(ip) = received.from_ip() + ips.insert(ElementLocation::new(ip, Location::HeaderReceived)); + } + for host in [received.from(), received.helo(), received.by()] + .into_iter() + .flatten() + { + if let Host::IpAddr(ip) = host && !ip.is_loopback() && !self.is_ip_allowed(ip) { ips.insert(ElementLocation::new(ip, Location::HeaderReceived)); } - for host in [&received.from, &received.helo, &received.by] - .into_iter() - .flatten() - { - if let Host::IpAddr(ip) = host - && !ip.is_loopback() - && !self.is_ip_allowed(*ip) - { - ips.insert(ElementLocation::new(*ip, Location::HeaderReceived)); - } - } } } // Obtain IP addresses from the message body for (part_id, part) in ctx.output.text_parts.iter().enumerate() { - let part_id = part_id as u32; - let is_body = ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id); - match part { - TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => { - ips.extend(tokens.iter().filter_map(|t| { - if let TokenType::IpAddr(ip) = t { - ip.ip.map(|ip| { - ElementLocation::new( - ip, - if is_body { - Location::BodyText - } else { - Location::Attachment - }, - ) - }) - } else { - None - } - })) + let is_body = ctx.input.is_body(part_id as u32); + let (tokens, hidden_tokens) = match part { + TextPart::Plain { tokens, .. } => (tokens, &[][..]), + TextPart::Html { + tokens, + hidden_tokens, + .. + } => (tokens, hidden_tokens.as_slice()), + TextPart::None => continue, + }; + ips.extend(tokens.iter().chain(hidden_tokens).filter_map(|t| { + if let TokenType::IpAddr(ip) = t { + ip.ip.map(|ip| { + ElementLocation::new( + ip, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + ) + }) + } else { + None } - - TextPart::None => (), - } + })); } // Validate IP addresses @@ -127,7 +127,11 @@ impl SpamFilterAnalyzeIp for Server { impl IpParts { pub fn new(text: &str) -> IpParts { IpParts { - ip: text.parse().ok(), + ip: if can_be_ip_addr(text) { + text.parse().ok() + } else { + None + }, } } } diff --git a/crates/spam-filter/src/analysis/messageid.rs b/crates/spam-filter/src/analysis/messageid.rs index 6e7a5d9f..6ab753d0 100644 --- a/crates/spam-filter/src/analysis/messageid.rs +++ b/crates/spam-filter/src/analysis/messageid.rs @@ -4,12 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{Hostname, SpamFilterContext, analysis::to_lowercase_cow}; use common::Server; use mail_parser::HeaderName; - -use crate::{Hostname, SpamFilterContext}; +use std::future::Future; pub trait SpamFilterAnalyzeMid: Sync + Send { fn spam_filter_analyze_message_id( @@ -24,20 +22,18 @@ impl SpamFilterAnalyzeMid for Server { let mut mid_raw = ""; for header in ctx.input.message.headers() { - if let (HeaderName::MessageId, value) = (&header.name, &header.value) { - mid = value.as_text().unwrap_or_default(); - mid_raw = std::str::from_utf8( - &ctx.input.message.raw_message() - [header.offset_start as usize..header.offset_end as usize], - ) - .unwrap_or_default() - .trim(); + if header.name() == HeaderName::MessageId { + mid = header.value().as_text().unwrap_or_default(); + mid_raw = std::str::from_utf8(header.raw_value()) + .unwrap_or_default() + .trim(); break; } } if !mid.is_empty() { - let mid = mid.to_lowercase(); + let mid = to_lowercase_cow(mid); + let mid = mid.as_ref(); if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) { if mid_host.ip.is_some() { if mid_host.fqdn.starts_with('[') { @@ -60,29 +56,54 @@ impl SpamFilterAnalyzeMid for Server { } // From address present in Message-ID checks - for (part, sender) in [ - ("FROM", &ctx.output.from.email), - ("ENV_FROM", &ctx.output.env_from_addr), + for (tags, sender) in [ + ( + [ + "MID_CONTAINS_FROM", + "MID_RHS_MATCH_FROM", + "MID_RHS_MATCH_FROMTLD", + ], + &ctx.output.from.email, + ), + ( + [ + "MID_CONTAINS_ENV_FROM", + "MID_RHS_MATCH_ENV_FROM", + "MID_RHS_MATCH_ENV_FROMTLD", + ], + &ctx.output.env_from_addr, + ), ] { if !sender.address.is_empty() { if mid.contains(sender.address.as_str()) { - ctx.result.add_tag(format!("MID_CONTAINS_{part}")); + ctx.result.add_tag(tags[0]); } else if mid_host.fqdn == sender.domain_part.fqdn { - ctx.result.add_tag(format!("MID_RHS_MATCH_{part}")); + ctx.result.add_tag(tags[1]); } else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld) { - ctx.result.add_tag(format!("MID_RHS_MATCH_{part}TLD")); + ctx.result.add_tag(tags[2]); } } } // To/Cc addresses present in Message-ID checks + let mut has_contains = false; + let mut has_rhs_match = false; for rcpt in ctx.output.all_recipients() { if mid.contains(rcpt.email.address.as_str()) { - ctx.result.add_tag("MID_CONTAINS_TO"); + has_contains = true; } else if mid_host.fqdn == rcpt.email.domain_part.fqdn { - ctx.result.add_tag("MID_RHS_MATCH_TO"); + has_rhs_match = true; } + if has_contains && has_rhs_match { + break; + } + } + if has_contains { + ctx.result.add_tag("MID_CONTAINS_TO"); + } + if has_rhs_match { + ctx.result.add_tag("MID_RHS_MATCH_TO"); } } else { ctx.result.add_tag("INVALID_MSGID"); diff --git a/crates/spam-filter/src/analysis/mime.rs b/crates/spam-filter/src/analysis/mime.rs index f6dc0d6c..9f274746 100644 --- a/crates/spam-filter/src/analysis/mime.rs +++ b/crates/spam-filter/src/analysis/mime.rs @@ -4,20 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::HashSet, future::Future, vec}; - +use super::mime_types::{MimeMatch, mime_match}; +use crate::{PartText, SpamFilterContext, TextPart, analysis::to_lowercase_cow}; use common::{ Server, + config::mailstore::spamfilter::FileExtension, scripts::{ IsMixedCharset, - functions::{array::cosine_similarity, unicode::CharUtils}, + functions::{array::cosine_similarity, sniff::SniffPrefix, unicode::CharUtils}, }, }; -use mail_parser::{HeaderName, MimeHeaders, PartType}; +use mail_parser::{HeaderName, PartFlags, PartKind}; use nlp::tokenizers::types::TokenType; +use std::{future::Future, vec}; +use store::ahash::AHashSet; +use utils::glob::GlobMap; -use super::mime_types::{MimeMatch, mime_match}; -use crate::{SpamFilterContext, TextPart}; +const SHORT_PART_SIZE: usize = 64; pub trait SpamFilterAnalyzeMime: Sync + Send { fn spam_filter_analyze_mime( @@ -28,6 +31,12 @@ pub trait SpamFilterAnalyzeMime: Sync + Send { impl SpamFilterAnalyzeMime for Server { async fn spam_filter_analyze_mime(&self, ctx: &mut SpamFilterContext<'_>) { + SpamFilterContext::analyze_mime(ctx, &self.core.spam.lists.file_extensions); + } +} + +impl SpamFilterContext<'_> { + pub fn analyze_mime(ctx: &mut Self, file_extensions: &GlobMap) { let mut has_mime_version = false; let mut has_ct = false; let mut has_cte = false; @@ -35,14 +44,12 @@ impl SpamFilterAnalyzeMime for Server { let mut is_plain_text = false; for header in ctx.input.message.headers() { - match &header.name { + match header.name() { HeaderName::MimeVersion => { - if ctx - .input - .message - .raw_message() - .get(header.offset_field as usize..header.offset_start as usize - 1) - != Some(b"MIME-Version") + if ctx.input.message.raw().get( + header.offset_field() as usize + ..(header.offset_start() as usize).saturating_sub(1), + ) != Some(b"MIME-Version") { ctx.result.add_tag("MV_CASE"); } @@ -89,8 +96,6 @@ impl SpamFilterAnalyzeMime for Server { // Only Content-Type header without other MIME headers ctx.result.add_tag("MIME_HEADER_CTYPE_ONLY"); } - let raw_message = ctx.input.message.raw_message(); - let mut has_text_part = false; let mut is_encrypted = false; let mut is_encrypted_smime = false; @@ -99,49 +104,46 @@ impl SpamFilterAnalyzeMime for Server { let mut num_parts = 0; let mut num_parts_size = 0; - for (part_id, part) in ctx.input.message.parts.iter().enumerate() { - let part_id = part_id as u32; + for part in ctx.input.message.root().parts() { + let part_id = part.id(); let mut ct = None; let mut cd = None; - let mut ct_type = String::new(); - let mut ct_subtype = String::new(); - let mut cte = String::new(); - let mut is_attachment = ctx.input.message.attachments.contains(&part_id); + let mut ct_type = ""; + let mut ct_subtype = ""; + let mut cte = ""; + let mut is_attachment = part.is_attachment(); let mut has_content_id = false; for header in part.headers() { - match &header.name { + match header.name() { HeaderName::ContentType => { if let Some(ct_) = header.value().as_content_type() { - ct_type = ct_.ctype().to_ascii_lowercase(); - ct_subtype = ct_.subtype().unwrap_or_default().to_ascii_lowercase(); + ct_type = ct_.ctype(); + ct_subtype = ct_.subtype().unwrap_or_default(); ct = Some(ct_); } if ct_type.is_empty() { // Content-Type header can't be parsed ctx.result.add_tag("BROKEN_CONTENT_TYPE"); - } else if (ct_type == "message" && ct_subtype == "rfc822") - || (ct_type == "text" && ct_subtype == "rfc822-headers") + } else if (ct_type.eq_ignore_ascii_case("message") + && ct_subtype.eq_ignore_ascii_case("rfc822")) + || (ct_type.eq_ignore_ascii_case("text") + && ct_subtype.eq_ignore_ascii_case("rfc822-headers")) { // Message has parts ctx.result.add_tag("HAS_MESSAGE_PARTS"); } - if raw_message - .get(header.offset_start as usize..header.offset_end as usize) - .and_then(|s| s.trim_ascii_end().last()) - == Some(&b';') - { + if header.raw_value().trim_ascii_end().last() == Some(&b';') { // Content-Type header ends with a semi-colon ctx.result.add_tag("CT_EXTRA_SEMI"); } } HeaderName::ContentTransferEncoding => { - let cte_ = header.value().as_text().unwrap_or_default(); - cte = cte_.to_ascii_lowercase(); + cte = header.value().as_text().unwrap_or_default(); - if cte != cte_ { + if cte.as_bytes().iter().any(u8::is_ascii_uppercase) { ctx.result.add_tag("CTE_CASE"); } } @@ -155,221 +157,199 @@ impl SpamFilterAnalyzeMime for Server { } } - match ct_type.as_str() { - "multipart" => { - let part_ids = match &part.body { - PartType::Multipart(parts) => parts.as_slice(), - _ => &[], + if ct_type.eq_ignore_ascii_case("multipart") { + if ct_subtype.eq_ignore_ascii_case("alternative") { + let mut has_plain_part = false; + let mut has_html_part = false; + + let mut text_part_words = vec![]; + let mut text_part_uris = 0; + + let mut html_part_words = vec![]; + let mut html_part_uris = 0; + let mut is_complete = true; + + for child in part.children() { + let (words, uri_count) = match child.kind() { + PartKind::Text if !has_plain_part => { + has_plain_part = true; + (&mut text_part_words, &mut text_part_uris) + } + PartKind::Html if !has_html_part => { + has_html_part = true; + (&mut html_part_words, &mut html_part_uris) + } + _ => continue, + }; + is_complete &= !ctx.input.texts.is_cut(child.id()); + let tokens = match ctx.output.text_parts.get(child.id() as usize) { + Some( + TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. }, + ) => tokens, + _ => continue, + }; + + let mut uris = AHashSet::new(); + words.reserve(tokens.len()); + for token in tokens { + match token { + TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => { + words.push(v.as_ref()); + } + TokenType::Url(v) => { + if let Some(host) = v.url_parsed.as_ref().map(|uri| &uri.host) { + uris.insert(host.sld_or_default()); + } + } + _ => (), + } + } + + *uri_count = uris.len(); + } + + // Multipart message mostly text/html MIME + if has_html_part { + if !has_plain_part { + ctx.result.add_tag("MIME_MA_MISSING_TEXT"); + } + } else if has_plain_part { + ctx.result.add_tag("MIME_MA_MISSING_HTML"); + } + + // HTML and text parts are different + if has_plain_part + && has_html_part + && is_complete + && (!text_part_words.is_empty() || !html_part_words.is_empty()) + && cosine_similarity(&text_part_words, &html_part_words) < 0.95 + { + ctx.result.add_tag("PARTS_DIFFER"); + } + + // Odd URI count between parts + if is_complete && text_part_uris != html_part_uris { + ctx.result.add_tag("URI_COUNT_ODD"); + } + } else if ct_subtype.eq_ignore_ascii_case("mixed") { + let mut num_text_parts = 0; + let mut has_other_parts = false; + + for sub_part in part.children() { + let ctype = sub_part + .content_type() + .map(|ct| ct.ctype()) + .unwrap_or_default(); + + if ctype.eq_ignore_ascii_case("text") && !sub_part.is_attachment() { + num_text_parts += 1; + } else if !ctype.eq_ignore_ascii_case("multipart") { + has_other_parts = true; + } + } + + // Found multipart/mixed without non-textual part + if !has_other_parts && num_text_parts < 3 { + ctx.result.add_tag("CTYPE_MIXED_BOGUS"); + } + } else if ct_subtype.eq_ignore_ascii_case("encrypted") { + is_encrypted = true; + } + + continue; + } else if ct_type.eq_ignore_ascii_case("text") { + let mut is_7bit = false; + if cte.is_empty() || cte.eq_ignore_ascii_case("7bit") { + if !part.raw_body().is_ascii() { + // MIME text part claims to be ASCII but isn't + ctx.result.add_tag("BAD_CTE_7BIT"); + } + is_7bit = true; + } else if cte.eq_ignore_ascii_case("base64") { + let is_ascii = match ctx.input.texts.part(part_id) { + Some(PartText::Whole(text) | PartText::Prefix(text)) => text.is_ascii(), + Some(PartText::Skipped) => false, + Some(PartText::NotText) | None => part.decoded().is_ascii(), }; - - match ct_subtype.as_str() { - "alternative" => { - let mut has_plain_part = false; - let mut has_html_part = false; - - let mut text_part_words = vec![]; - let mut text_part_uris = 0; - - let mut html_part_words = vec![]; - let mut html_part_uris = 0; - - for text_part in part_ids - .iter() - .map(|id| &ctx.output.text_parts[*id as usize]) - { - let (tokens, words, uri_count) = match text_part { - TextPart::Plain { tokens, .. } if !has_plain_part => { - has_plain_part = true; - (tokens, &mut text_part_words, &mut text_part_uris) - } - TextPart::Html { tokens, .. } if !has_html_part => { - has_html_part = true; - (tokens, &mut html_part_words, &mut html_part_uris) - } - _ => continue, - }; - - let mut uris = HashSet::new(); - for token in tokens { - match token { - TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => { - words.push(v.as_ref()); - } - TokenType::Url(v) => { - if let Some(host) = - v.url_parsed.as_ref().map(|uri| &uri.host) - { - uris.insert(host.sld_or_default()); - } - } - _ => (), - } - } - - *uri_count = uris.len(); - } - - // Multipart message mostly text/html MIME - if has_html_part { - if !has_plain_part { - ctx.result.add_tag("MIME_MA_MISSING_TEXT"); - } - } else if has_plain_part { - ctx.result.add_tag("MIME_MA_MISSING_HTML"); - } - - // HTML and text parts are different - if has_plain_part - && has_html_part - && (!text_part_words.is_empty() || !html_part_words.is_empty()) - && cosine_similarity(&text_part_words, &html_part_words) < 0.95 - { - ctx.result.add_tag("PARTS_DIFFER"); - } - - // Odd URI count between parts - if text_part_uris != html_part_uris { - ctx.result.add_tag("URI_COUNT_ODD"); - } - } - "mixed" => { - let mut num_text_parts = 0; - let mut has_other_parts = false; - - for (sub_part_id, sub_part) in part_ids - .iter() - .map(|id| (*id, &ctx.input.message.parts[*id as usize])) - { - let ctype = sub_part - .content_type() - .map(|ct| ct.ctype()) - .unwrap_or_default(); - - if ctype.eq_ignore_ascii_case("text") - && !ctx.input.message.attachments.contains(&sub_part_id) - { - num_text_parts += 1; - } else if !ctype.eq_ignore_ascii_case("multipart") { - has_other_parts = true; - } - } - - // Found multipart/mixed without non-textual part - if !has_other_parts && num_text_parts < 3 { - ctx.result.add_tag("CTYPE_MIXED_BOGUS"); - } - } - "encrypted" => { - is_encrypted = true; - } - _ => (), + if is_ascii { + // Has text part encoded in base64 that does not contain any 8bit characters + ctx.result.add_tag("MIME_BASE64_TEXT_BOGUS"); + } else { + // Has text part encoded in base64 + ctx.result.add_tag("MIME_BASE64_TEXT"); } - - continue; } - "text" => { - let mut is_7bit = false; - match cte.as_str() { - "" | "7bit" => { - if raw_message - .get( - part.raw_body_offset() as usize..part.raw_end_offset() as usize, - ) - .is_some_and(|bytes| !bytes.is_ascii()) - { - // MIME text part claims to be ASCII but isn't - ctx.result.add_tag("BAD_CTE_7BIT"); - } - is_7bit = true; - } - "base64" => { - if part.contents().is_ascii() { - // Has text part encoded in base64 that does not contain any 8bit characters - ctx.result.add_tag("MIME_BASE64_TEXT_BOGUS"); - } else { - // Has text part encoded in base64 - ctx.result.add_tag("MIME_BASE64_TEXT"); - } - } - _ => (), - } - if !is_7bit - && ct_subtype == "plain" - && ct - .and_then(|ct| ct.attribute("charset")) - .is_none_or(|c| c.is_empty()) - { - // Charset header is missing - ctx.result.add_tag("MISSING_CHARSET"); - } + if !is_7bit + && ct_subtype.eq_ignore_ascii_case("plain") + && ct + .and_then(|ct| ct.attribute("charset")) + .is_none_or(|c| c.is_empty()) + { + // Charset header is missing + ctx.result.add_tag("MISSING_CHARSET"); + } - if ctx - .output - .text_parts - .get(part_id as usize) - .filter(|_| { - ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id) - }) - .is_some_and(|p| match p { - TextPart::Plain { text_body, .. } => text_body.is_mixed_charset(), - TextPart::Html { text_body, .. } => text_body.is_mixed_charset(), - TextPart::None => false, - }) - { - // Text part contains multiple scripts - ctx.result.add_tag("MIXED_CHARSET"); - } + if ctx + .output + .text_parts + .get(part_id as usize) + .filter(|_| ctx.input.is_body(part_id)) + .is_some_and(|p| match p { + TextPart::Plain { text_body, .. } => text_body.is_mixed_charset(), + TextPart::Html { text_body, .. } => text_body.is_mixed_charset(), + TextPart::None => false, + }) + { + // Text part contains multiple scripts + ctx.result.add_tag("MIXED_CHARSET"); + } - has_text_part = true; + has_text_part = true; + } else if ct_type.eq_ignore_ascii_case("application") { + if ct_subtype.eq_ignore_ascii_case("pkcs7-mime") { + ctx.result.add_tag("ENCRYPTED_SMIME"); + is_attachment = false; + is_encrypted_smime = true; + } else if ct_subtype.eq_ignore_ascii_case("pkcs7-signature") { + ctx.result.add_tag("SIGNED_SMIME"); + is_attachment = false; + } else if ct_subtype.eq_ignore_ascii_case("pgp-encrypted") { + ctx.result.add_tag("ENCRYPTED_PGP"); + is_attachment = false; + is_encrypted_pgp = true; + } else if ct_subtype.eq_ignore_ascii_case("pgp-signature") { + ctx.result.add_tag("SIGNED_PGP"); + is_attachment = false; + } else if ct_subtype.eq_ignore_ascii_case("octet-stream") + && !is_encrypted + && !has_content_id + && cd.is_none_or(|cd| { + !cd.ctype().eq_ignore_ascii_case("attachment") + && !cd.has_attribute("filename") + }) + { + ctx.result.add_tag("CTYPE_MISSING_DISPOSITION"); } - "application" => match ct_subtype.as_str() { - "pkcs7-mime" => { - ctx.result.add_tag("ENCRYPTED_SMIME"); - is_attachment = false; - is_encrypted_smime = true; - } - "pkcs7-signature" => { - ctx.result.add_tag("SIGNED_SMIME"); - is_attachment = false; - } - "pgp-encrypted" => { - ctx.result.add_tag("ENCRYPTED_PGP"); - is_attachment = false; - is_encrypted_pgp = true; - } - "pgp-signature" => { - ctx.result.add_tag("SIGNED_PGP"); - is_attachment = false; - } - "octet-stream" - if !is_encrypted - && !has_content_id - && cd.is_none_or(|cd| { - !cd.c_type.eq_ignore_ascii_case("attachment") - && !cd.has_attribute("filename") - }) => - { - ctx.result.add_tag("CTYPE_MISSING_DISPOSITION"); - } - _ => (), - }, - _ => (), } num_parts += 1; - num_parts_size += part.len(); + if num_parts_size < SHORT_PART_SIZE { + num_parts_size += match ctx.input.texts.part(part_id) { + Some(PartText::Whole(text)) => text.len(), + Some(PartText::Prefix(text)) => text.len() + 1, + Some(PartText::NotText | PartText::Skipped) | None => part.decoded_len(), + }; + } - let ct_full = format!("{ct_type}/{ct_subtype}"); + let is_octet_stream = ct_type.eq_ignore_ascii_case("application") + && ct_subtype.eq_ignore_ascii_case("octet-stream"); if is_attachment { // Has a MIME attachment ctx.result.add_tag("HAS_ATTACHMENT"); - if ct_full != "application/octet-stream" - && let Some(t) = infer::get(part.contents()) - { - match mime_match(t.mime_type(), &ct_full) { + if !is_octet_stream && let Some(t) = infer::get(&part.sniff_prefix()) { + match mime_match(t.mime_type(), &content_type_full(ct_type, ct_subtype)) { MimeMatch::Equal => { // Known content-type ctx.result.add_tag("MIME_GOOD"); @@ -389,13 +369,14 @@ impl SpamFilterAnalyzeMime for Server { // Attachment name contains zero-width space ctx.result.add_tag("MIME_BAD_UNICODE"); } - let attach_name = attach_name.trim().to_lowercase(); - if let Some((name, ext)) = attach_name.rsplit_once('.').and_then(|(name, ext)| { - Some((name, self.core.spam.lists.file_extensions.get(ext)?)) - }) { + let attach_name = to_lowercase_cow(attach_name.trim()); + if let Some((name, ext)) = attach_name + .rsplit_once('.') + .and_then(|(name, ext)| Some((name, file_extensions.get(ext)?))) + { let sub_ext = name .rsplit_once('.') - .and_then(|(_, ext)| self.core.spam.lists.file_extensions.get(ext)); + .and_then(|(_, ext)| file_extensions.get(ext)); if ext.is_bad { // Attachment has a bad extension @@ -412,8 +393,10 @@ impl SpamFilterAnalyzeMime for Server { } if !ext.known_types.is_empty() - && ct_full != "application/octet-stream" - && !ext.known_types.contains(&ct_full) + && !is_octet_stream + && !ext + .known_types + .contains(content_type_full(ct_type, ct_subtype).as_str()) { // Invalid attachment mime type ctx.result.add_tag("MIME_BAD_ATTACHMENT"); @@ -427,7 +410,7 @@ impl SpamFilterAnalyzeMime for Server { // Message contains no parts ctx.result.add_tag("COMPLETELY_EMPTY"); } - 1..64 if num_parts == 1 => { + 1..SHORT_PART_SIZE if num_parts == 1 => { // Message contains only one short part ctx.result.add_tag("SINGLE_SHORT_PART"); } @@ -438,5 +421,21 @@ impl SpamFilterAnalyzeMime for Server { // Message contains both text and encrypted parts ctx.result.add_tag("BOGUS_ENCRYPTED_AND_TEXT"); } + + if ctx.input.message.parts().any(|part| { + part.flags() + .intersects(PartFlags::LIMIT_REACHED | PartFlags::NESTING_LIMIT) + }) { + ctx.result.add_tag("MIME_LIMIT_EXCEEDED"); + } } } + +fn content_type_full(ct_type: &str, ct_subtype: &str) -> String { + let mut full = String::with_capacity(ct_type.len() + ct_subtype.len() + 1); + full.push_str(ct_type); + full.push('/'); + full.push_str(ct_subtype); + full.make_ascii_lowercase(); + full +} diff --git a/crates/spam-filter/src/analysis/mod.rs b/crates/spam-filter/src/analysis/mod.rs index a9c09933..32ecd9f3 100644 --- a/crates/spam-filter/src/analysis/mod.rs +++ b/crates/spam-filter/src/analysis/mod.rs @@ -8,7 +8,9 @@ use crate::{ Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, }; use common::{Server, config::mailstore::spamfilter::Location}; -use mail_parser::{Header, parsers::MessageStream}; +use compact_str::CompactString; +use mail_parser::{Header, HeaderForm}; +use memchr::memchr2_iter; use std::{ borrow::Cow, hash::{Hash, Hasher}, @@ -38,14 +40,14 @@ pub mod url; impl SpamFilterInput<'_> { - pub fn header_as_address(&self, header: &Header<'_>) -> Option> { - self.message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .map(|bytes| MessageStream::new(bytes).parse_address()) - .and_then(|addr| addr.into_address()) - .and_then(|addr| addr.into_list().into_iter().next()) - .and_then(|addr| addr.address) + pub fn header_as_address(&self, header: &Header<'_>) -> Option { + HeaderForm::Addresses + .parse(header.raw_value()) + .value() + .as_address()? + .first()? + .address() + .map(str::to_string) } } @@ -56,32 +58,43 @@ impl SpamFilterOutput<'_> { .chain(self.recipients_cc.iter()) .chain(self.recipients_bcc.iter()) } -} -impl SpamFilterContext<'_> { - pub fn text_body(&self) -> Option<&str> { - self.input - .message - .text_body - .first() - .or_else(|| self.input.message.html_body.first()) - .and_then(|idx| self.output.text_parts.get(*idx as usize)) + pub fn text_body(&self, input: &SpamFilterInput<'_>) -> Option<&str> { + input + .first_body_part() + .and_then(|idx| self.text_parts.get(idx as usize)) .and_then(|part| match part { TextPart::Plain { text_body, .. } => Some(*text_body), - TextPart::Html { text_body, .. } => Some(text_body.as_str()), + TextPart::Html { text_body, .. } => Some(*text_body), TextPart::None => None, }) } } +impl SpamFilterContext<'_> { + pub fn text_body(&self) -> Option<&str> { + self.output.text_body(&self.input) + } +} + impl SpamFilterResult { - pub fn add_tag(&mut self, tag: impl Into) { + pub fn add_tag(&mut self, tag: impl Into) { self.tags.insert(tag.into()); } pub fn has_tag(&self, tag: impl AsRef) -> bool { self.tags.contains(tag.as_ref()) } + + pub fn classifier_probability(&self) -> Option { + let mut total = 0.0f32; + let mut count = 0u32; + for confidence in self.classifier_confidence.iter().flatten() { + total += confidence; + count += 1; + } + (count > 0).then(|| total / count as f32) + } } #[derive(Debug)] @@ -104,6 +117,18 @@ impl PartialEq for ElementLocation { impl Eq for ElementLocation {} +impl std::borrow::Borrow for ElementLocation { + fn borrow(&self) -> &crate::Email { + &self.element.email + } +} + +impl std::borrow::Borrow for ElementLocation> { + fn borrow(&self) -> &str { + self.element.url.as_str() + } +} + impl ElementLocation { pub fn new(element: T, location: impl Into) -> Self { Self { @@ -146,3 +171,95 @@ pub(crate) async fn is_url_redirector(server: &Server, url: &str, span_id: u64) false } } + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ExcessEncoding { + None, + QuotedPrintable, + Base64, + Other, +} + +#[inline] +pub(crate) fn excess_encoding(text: &str) -> ExcessEncoding { + if !text.contains("=?") || !text.contains("?=") { + return ExcessEncoding::None; + } + + let bytes = text.as_bytes(); + let mut has_qp = false; + let mut has_base64 = false; + + for (pos, _) in text.match_indices('?') { + if bytes.get(pos + 2) == Some(&b'?') { + match bytes.get(pos + 1) { + Some(b'q' | b'Q') => has_qp = true, + Some(b'b' | b'B') => has_base64 = true, + _ => {} + } + } + } + + if has_qp { + ExcessEncoding::QuotedPrintable + } else if has_base64 { + ExcessEncoding::Base64 + } else { + ExcessEncoding::Other + } +} + +pub(crate) fn eq_lowercase_str(left: &str, right: &str) -> bool { + if left.is_ascii() && right.is_ascii() { + left.eq_ignore_ascii_case(right) + } else { + left.to_lowercase() == right.to_lowercase() + } +} + +pub(crate) fn to_lowercase_cow(text: &str) -> Cow<'_, str> { + if text.is_ascii() { + if text.as_bytes().iter().any(u8::is_ascii_uppercase) { + Cow::Owned(text.to_ascii_lowercase()) + } else { + Cow::Borrowed(text) + } + } else { + Cow::Owned(text.to_lowercase()) + } +} + +pub(crate) fn eq_lowercase(text: &str, lowercase_literal: &str) -> bool { + debug_assert!( + lowercase_literal + .bytes() + .all(|b| b.is_ascii() && !b.is_ascii_uppercase()) + ); + text.chars() + .flat_map(char::to_lowercase) + .eq(lowercase_literal.chars()) +} + +pub(crate) fn starts_with_ignore_ascii_case(text: &str, prefix: &str) -> bool { + text.as_bytes() + .get(..prefix.len()) + .is_some_and(|start| start.eq_ignore_ascii_case(prefix.as_bytes())) +} + +pub(crate) fn contains_ignore_ascii_case(text: &str, needle: &str) -> bool { + let haystack = text.as_bytes(); + let Some((&first, rest)) = needle.as_bytes().split_first() else { + return true; + }; + memchr2_iter( + first.to_ascii_lowercase(), + first.to_ascii_uppercase(), + haystack, + ) + .any(|pos| { + haystack + .get(pos + 1..pos + 1 + rest.len()) + .is_some_and(|tail| tail.eq_ignore_ascii_case(rest)) + }) +} + diff --git a/crates/spam-filter/src/analysis/pyzor.rs b/crates/spam-filter/src/analysis/pyzor.rs index 1f80fd81..0046f06f 100644 --- a/crates/spam-filter/src/analysis/pyzor.rs +++ b/crates/spam-filter/src/analysis/pyzor.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{future::Future, time::Instant}; - +use crate::SpamFilterContext; use common::Server; - -use crate::{SpamFilterContext, modules::pyzor::pyzor_check}; +use std::{future::Future, time::Instant}; pub trait SpamFilterAnalyzePyzor: Sync + Send { fn spam_filter_analyze_pyzor( @@ -21,7 +19,7 @@ impl SpamFilterAnalyzePyzor for Server { async fn spam_filter_analyze_pyzor(&self, ctx: &mut SpamFilterContext<'_>) { if let Some(config) = &self.core.spam.pyzor { let time = Instant::now(); - match pyzor_check(ctx.input.message, config).await { + match ctx.input.pyzor_check(config).await { Ok(Some(result)) => { let is_spam = result.code == 200 && result.count > config.min_count diff --git a/crates/spam-filter/src/analysis/received.rs b/crates/spam-filter/src/analysis/received.rs index 2ba4cb09..e676217f 100644 --- a/crates/spam-filter/src/analysis/received.rs +++ b/crates/spam-filter/src/analysis/received.rs @@ -4,13 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{Email, SpamFilterContext, analysis::eq_lowercase_str}; use common::Server; use mail_parser::{HeaderName, Host}; use smtp_proto::MAIL_SMTPUTF8; - -use crate::{Email, SpamFilterContext}; +use std::future::Future; pub trait SpamFilterAnalyzeReceived: Sync + Send { fn spam_filter_analyze_received( @@ -26,18 +24,11 @@ impl SpamFilterAnalyzeReceived for Server { let mut tls_count = 0; let is_smtputf8 = (ctx.input.env_from_flags & MAIL_SMTPUTF8) != 0; + let has_recipients = ctx.output.all_recipients().next().is_some(); for header in ctx.input.message.headers() { - if let HeaderName::Received = &header.name { - if !is_smtputf8 - && !ctx - .input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default() - .is_ascii() - { + if header.name() == HeaderName::Received { + if !is_smtputf8 && !header.raw_value().is_ascii() { // Received headers have non-ASCII characters ctx.result.add_tag("RCVD_ILLEGAL_CHARS"); } @@ -52,13 +43,14 @@ impl SpamFilterAnalyzeReceived for Server { ctx.result.add_tag("RCVD_HELO_USER"); } else if let (Some(Host::Name(helo_domain)), Some(ip_rev)) = (helo_domain, ip_rev) - && helo_domain.to_lowercase() != ip_rev.to_lowercase() + && !eq_lowercase_str(helo_domain, ip_rev) { // HELO domain does not match PTR record ctx.result.add_tag("FORGED_RCVD_TRAIL"); } - if let Some(delivered_for) = received.for_().map(Email::new) + if has_recipients + && let Some(delivered_for) = received.for_().map(Email::new) && ctx .output .all_recipients() @@ -68,7 +60,7 @@ impl SpamFilterAnalyzeReceived for Server { ctx.result.add_tag("PREVIOUSLY_DELIVERED"); } - if matches!(received.from, Some(Host::IpAddr(_))) { + if matches!(received.from(), Some(Host::IpAddr(_))) { // Received from an IP address rather than a FQDN rcvd_from_ip += 1; } diff --git a/crates/spam-filter/src/analysis/recipient.rs b/crates/spam-filter/src/analysis/recipient.rs index b04ec5d9..92dc5eac 100644 --- a/crates/spam-filter/src/analysis/recipient.rs +++ b/crates/spam-filter/src/analysis/recipient.rs @@ -4,15 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{ + SpamFilterContext, + analysis::{ExcessEncoding, excess_encoding}, +}; use common::{Server, scripts::functions::text::levenshtein_distance}; use mail_parser::HeaderName; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; +use std::future::Future; use store::ahash::HashSet; -use crate::SpamFilterContext; - pub trait SpamFilterAnalyzeRecipient: Sync + Send { fn spam_filter_analyze_recipient( &self, @@ -29,21 +30,10 @@ impl SpamFilterAnalyzeRecipient for Server { let mut has_list_id = false; for header in ctx.input.message.headers() { - match &header.name { - HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { - let raw = ctx - .input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(); - match header.name { - HeaderName::To => to_raw = raw, - HeaderName::Cc => cc_raw = raw, - HeaderName::Bcc => bcc_raw = raw, - _ => unreachable!(), - } - } + match header.name() { + HeaderName::To => to_raw = header.raw_value(), + HeaderName::Cc => cc_raw = header.raw_value(), + HeaderName::Bcc => bcc_raw = header.raw_value(), HeaderName::ListUnsubscribe => { has_list_unsubscribe = true; } @@ -87,15 +77,17 @@ impl SpamFilterAnalyzeRecipient for Server { if recipients.iter().all(|rcpt| { rcpt.name.as_ref().is_none_or(|name| name.is_ascii()) && rcpt.email.address.is_ascii() - }) && raw_utf8.contains("=?") - && raw_utf8.contains("?=") - { - if raw_utf8.contains("?q?") || raw_utf8.contains("?Q?") { - // To header is unnecessarily encoded in quoted-printable - ctx.result.add_tag("TO_EXCESS_QP"); - } else if raw_utf8.contains("?b?") || raw_utf8.contains("?B?") { - // To header is unnecessarily encoded in base64 - ctx.result.add_tag("TO_EXCESS_BASE64"); + }) { + match excess_encoding(raw_utf8) { + ExcessEncoding::QuotedPrintable => { + // To header is unnecessarily encoded in quoted-printable + ctx.result.add_tag("TO_EXCESS_QP"); + } + ExcessEncoding::Base64 => { + // To header is unnecessarily encoded in base64 + ctx.result.add_tag("TO_EXCESS_BASE64"); + } + ExcessEncoding::None | ExcessEncoding::Other => {} } } @@ -111,11 +103,17 @@ impl SpamFilterAnalyzeRecipient for Server { } } - let unique_recipients = ctx - .output - .all_recipients() - .filter(|rcpt| !rcpt.email.address.is_empty()) - .collect::>(); + let mut unique_recipients: HashSet<_> = HashSet::with_capacity_and_hasher( + ctx.output.recipients_to.len() + + ctx.output.recipients_cc.len() + + ctx.output.recipients_bcc.len(), + Default::default(), + ); + unique_recipients.extend( + ctx.output + .all_recipients() + .filter(|rcpt| !rcpt.email.address.is_empty()), + ); let rcpt_count = unique_recipients.len(); match unique_recipients.len() { @@ -218,51 +216,79 @@ impl SpamFilterAnalyzeRecipient for Server { ctx.result.add_tag("RCPT_BOUNCEMOREONE"); } - let rcpts = ctx - .output - .recipients_to - .iter() - .chain(ctx.output.recipients_cc.iter()) - .collect::>(); + let mut rcpts = + Vec::with_capacity(ctx.output.recipients_to.len() + ctx.output.recipients_cc.len()); + rcpts.extend( + ctx.output + .recipients_to + .iter() + .chain(ctx.output.recipients_cc.iter()), + ); let mut is_sorted = false; - if rcpts.len() >= 6 { + if rcpts.len() >= 6 + && rcpts + .windows(2) + .all(|pair| pair[0].email.address <= pair[1].email.address) + { // Check if the recipients list is sorted - let mut sorted = true; - for i in 1..rcpts.len() { - if rcpts[i - 1].email.address > rcpts[i].email.address { - sorted = false; - break; - } - } - if sorted { - ctx.result.add_tag("SORTED_RECIPS"); - is_sorted = true; - } + ctx.result.add_tag("SORTED_RECIPS"); + is_sorted = true; } - if !is_sorted && rcpt_count >= 5 { + let combinations = rcpts.len() * rcpts.len().saturating_sub(1) / 2; + if !is_sorted && rcpt_count >= 5 && combinations != 0 { // Look for similar recipients let mut hits = 0; - let mut combinations = 0; - for i in 0..rcpts.len() { + let mut remaining = combinations; + let is_suspicious = |hits: usize| hits as f64 / combinations as f64 > 0.65; + + 'pairs: for i in 0..rcpts.len() { for j in i + 1..rcpts.len() { let a = &rcpts[i].email; let b = &rcpts[j].email; - if levenshtein_distance(&a.local_part, &b.local_part) < 3 + if levenshtein_below(&a.local_part, &b.local_part, 3) || (a.domain_part.fqdn != b.domain_part.fqdn - && levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn) < 4) + && levenshtein_below(&a.domain_part.fqdn, &b.domain_part.fqdn, 4)) { hits += 1; } - combinations += 1; + remaining -= 1; + + if is_suspicious(hits) || !is_suspicious(hits + remaining) { + break 'pairs; + } } } - if hits as f64 / combinations as f64 > 0.65 { + if is_suspicious(hits) { ctx.result.add_tag("SUSPICIOUS_RECIPS"); } } } } + +#[inline(always)] +pub(crate) fn levenshtein_below(left: &str, right: &str, limit: usize) -> bool { + if left.is_ascii() && right.is_ascii() { + let length_diff = left.len().abs_diff(right.len()); + if length_diff >= limit { + return false; + } + if length_diff == 0 { + let mismatches = left + .as_bytes() + .iter() + .zip(right.as_bytes()) + .filter(|(a, b)| a != b) + .count(); + if mismatches < limit { + return true; + } + } + } + + levenshtein_distance(left, right) < limit +} + diff --git a/crates/spam-filter/src/analysis/replyto.rs b/crates/spam-filter/src/analysis/replyto.rs index 62db3aa3..b6f2b339 100644 --- a/crates/spam-filter/src/analysis/replyto.rs +++ b/crates/spam-filter/src/analysis/replyto.rs @@ -4,12 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{ + SpamFilterContext, + analysis::{ExcessEncoding, excess_encoding}, +}; use common::Server; use mail_parser::HeaderName; - -use crate::SpamFilterContext; +use std::future::Future; pub trait SpamFilterAnalyzeReplyTo: Sync + Send { fn spam_filter_analyze_reply_to( @@ -24,14 +25,9 @@ impl SpamFilterAnalyzeReplyTo for Server { let mut is_from_list = false; for header in ctx.input.message.headers() { - match &header.name { + match header.name() { HeaderName::ReplyTo => { - reply_to_raw = ctx - .input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(); + reply_to_raw = header.raw_value(); } HeaderName::ListUnsubscribe | HeaderName::ListId => { is_from_list = true; @@ -103,18 +99,17 @@ impl SpamFilterAnalyzeReplyTo for Server { } // Validate unnecessary encoding - let reply_to_raw_utf8 = std::str::from_utf8(reply_to_raw).unwrap_or_default(); - if reply_to.email.address.is_ascii() - && reply_to_name.is_ascii() - && reply_to_raw_utf8.contains("=?") - && reply_to_raw_utf8.contains("?=") - { - if reply_to_raw_utf8.contains("?q?") || reply_to_raw_utf8.contains("?Q?") { - // Reply-To header is unnecessarily encoded in quoted-printable - ctx.result.add_tag("REPLYTO_EXCESS_QP"); - } else if reply_to_raw_utf8.contains("?b?") || reply_to_raw_utf8.contains("?B?") { - // Reply-To header is unnecessarily encoded in base64 - ctx.result.add_tag("REPLYTO_EXCESS_BASE64"); + if reply_to.email.address.is_ascii() && reply_to_name.is_ascii() { + match excess_encoding(std::str::from_utf8(reply_to_raw).unwrap_or_default()) { + ExcessEncoding::QuotedPrintable => { + // Reply-To header is unnecessarily encoded in quoted-printable + ctx.result.add_tag("REPLYTO_EXCESS_QP"); + } + ExcessEncoding::Base64 => { + // Reply-To header is unnecessarily encoded in base64 + ctx.result.add_tag("REPLYTO_EXCESS_BASE64"); + } + ExcessEncoding::None | ExcessEncoding::Other => {} } } } else { diff --git a/crates/spam-filter/src/analysis/rules.rs b/crates/spam-filter/src/analysis/rules.rs index 5dc005a2..ae547647 100644 --- a/crates/spam-filter/src/analysis/rules.rs +++ b/crates/spam-filter/src/analysis/rules.rs @@ -4,77 +4,118 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{ + SpamFilterContext, SpamFilterInput, SpamFilterOutput, TextPart, + modules::expression::{ + EmailHeader, HEADER_NAME_INLINE, ScoreVariables, SpamFilterResolver, StringResolver, + lower_header_name, + }, +}; use common::{ Server, - config::mailstore::spamfilter::{IpResolver, Location}, -}; - -use crate::{ - SpamFilterContext, TextPart, - modules::expression::{EmailHeader, SpamFilterResolver, StringResolver}, + config::mailstore::spamfilter::{IpResolver, Location, SpamFilterRuleSet}, + expr::{Bump, functions::ResolveVariable, guard::Selection}, }; +use compact_str::CompactString; +use std::future::Future; +use store::ahash::AHashSet; pub trait SpamFilterAnalyzeRules: Sync + Send { fn spam_filter_analyze_rules( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; + + fn spam_filter_analyze_rule_set( + &self, + ctx: &mut SpamFilterContext<'_>, + stage: RuleStage, + added: Option<&mut Vec>, + ) -> impl Future + Send; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuleStage { + BeforeAi, + AfterAi, +} + +struct RulePass<'x> { + server: &'x Server, + input: &'x SpamFilterInput<'x>, + output: &'x SpamFilterOutput<'x>, + scores: ScoreVariables, } impl SpamFilterAnalyzeRules for Server { async fn spam_filter_analyze_rules(&self, ctx: &mut SpamFilterContext<'_>) { - if !self.core.spam.rules.url.is_empty() { + self.spam_filter_analyze_rule_set(ctx, RuleStage::BeforeAi, None) + .await; + self.spam_filter_analyze_rule_set(ctx, RuleStage::AfterAi, None) + .await; + } + + async fn spam_filter_analyze_rule_set( + &self, + ctx: &mut SpamFilterContext<'_>, + stage: RuleStage, + mut added: Option<&mut Vec>, + ) { + let rules: &SpamFilterRuleSet = match stage { + RuleStage::BeforeAi => &self.core.spam.rules.before_ai, + RuleStage::AfterAi => &self.core.spam.rules.after_ai, + }; + if rules.is_empty() { + return; + } + let mut arena = std::mem::take(ctx.arena.get_mut()); + let pass = RulePass { + server: self, + input: &ctx.input, + output: &ctx.output, + scores: ScoreVariables::for_rules(ctx.result.classifier_probability()), + }; + let tags = &mut ctx.result.tags; + + if !rules.url.is_empty() { for url in &ctx.output.urls { - for rule in &self.core.spam.rules.url { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &url.element, url.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval( + rules.url.select(url.location.as_str()), + &url.element, + url.location, + tags, + &mut added, + &mut arena, + ) + .await; } } - if !self.core.spam.rules.domain.is_empty() { + if !rules.domain.is_empty() { for domain in &ctx.output.domains { - let resolver = StringResolver(domain.element.as_str()); - - for rule in &self.core.spam.rules.domain { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &resolver, domain.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval( + rules.domain.select(domain.location.as_str()), + &StringResolver(domain.element.as_str()), + domain.location, + tags, + &mut added, + &mut arena, + ) + .await; } } - if !self.core.spam.rules.email.is_empty() { + if !rules.email.is_empty() { for email in &ctx.output.emails { - for rule in &self.core.spam.rules.email { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &email.element, email.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval( + rules.email.select(email.location.as_str()), + &email.element, + email.location, + tags, + &mut added, + &mut arena, + ) + .await; } for (rcpt, location) in [ @@ -82,116 +123,127 @@ impl SpamFilterAnalyzeRules for Server { (&ctx.output.recipients_cc, Location::HeaderCc), (&ctx.output.recipients_bcc, Location::HeaderBcc), ] { + let selection = rules.email.select(location.as_str()); for email in rcpt { - for rule in &self.core.spam.rules.email { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, email, location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval(selection, email, location, tags, &mut added, &mut arena) + .await; } } } - if !self.core.spam.rules.ip.is_empty() { + if !rules.ip.is_empty() { for ip in &ctx.output.ips { - let ip_resolver = IpResolver::new(ip.element); - - for rule in &self.core.spam.rules.ip { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &ip_resolver, ip.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval( + rules.ip.select(ip.location.as_str()), + &IpResolver::new(ip.element), + ip.location, + tags, + &mut added, + &mut arena, + ) + .await; } } - if !self.core.spam.rules.header.is_empty() { + if !rules.header.is_empty() { + let mut inline = [0u8; HEADER_NAME_INLINE]; + let mut header_arena = Bump::new(); for header in ctx.input.message.headers() { - let raw = String::from_utf8_lossy( - ctx.input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(), - ); - let header_resolver = EmailHeader { - header, - raw: raw.as_ref(), - }; - - for rule in &self.core.spam.rules.header { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &header_resolver, Location::BodyText), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } + header_arena.reset(); + let name = header.raw_name(); + let name_lower = lower_header_name(name, &mut inline, &header_arena); + let selection = rules.header.select(name_lower); + if selection.is_empty() { + continue; } + let header = EmailHeader { + header, + name, + name_lower, + }; + pass.eval( + selection, + &header, + Location::BodyText, + tags, + &mut added, + &mut arena, + ) + .await; } } - if !self.core.spam.rules.body.is_empty() { + if !rules.body.is_empty() { for (idx, part) in ctx.output.text_parts.iter().enumerate() { let text = match part { TextPart::Plain { text_body, .. } => *text_body, - TextPart::Html { text_body, .. } => text_body.as_str(), + TextPart::Html { text_body, .. } => *text_body, TextPart::None => continue, }; let idx = idx as u32; - let location = if ctx.input.message.text_body.contains(&idx) { + let location = if ctx.input.is_text_body(idx) { Location::BodyText - } else if ctx.input.message.html_body.contains(&idx) { + } else if ctx.input.is_html_body(idx) { Location::BodyHtml } else { Location::Attachment }; - let string_resolver = StringResolver(text); - - for rule in &self.core.spam.rules.body { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &string_resolver, location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } + pass.eval( + rules.body.select(location.as_str()), + &StringResolver(text), + location, + tags, + &mut added, + &mut arena, + ) + .await; } } - if !self.core.spam.rules.any.is_empty() { - let dummy_resolver = StringResolver(""); - for rule in &self.core.spam.rules.any { - if let Some(tag) = self - .eval_if::( - rule, - &SpamFilterResolver::new(ctx, &dummy_resolver, Location::BodyText), - ctx.input.span_id, - ) - .await + if !rules.any.is_empty() { + pass.eval( + rules.any.all(), + &StringResolver(""), + Location::BodyText, + tags, + &mut added, + &mut arena, + ) + .await; + } + + *ctx.arena.get_mut() = arena; + } +} + +impl RulePass<'_> { + async fn eval( + &self, + selection: Selection<'_>, + item: &T, + location: Location, + tags: &mut AHashSet, + added: &mut Option<&mut Vec>, + arena: &mut Bump, + ) { + for rule in selection.iter() { + if let Some(tag) = self + .server + .eval_if::( + rule, + &SpamFilterResolver::new(self.input, self.output, tags, item, location) + .with_scores(self.scores), + arena, + self.input.span_id, + ) + .await + { + if let Some(added) = added.as_deref_mut() + && !tags.contains(&tag) { - ctx.result.tags.insert(tag); + added.push(tag.clone()); } + tags.insert(tag); } } } diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index 62d59e90..19074ebd 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -5,22 +5,36 @@ */ use crate::{ - SpamFilterContext, + SpamFilterContext, SpamFilterResult, analysis::{ - classifier::SpamFilterAnalyzeClassify, date::SpamFilterAnalyzeDate, - dmarc::SpamFilterAnalyzeDmarc, domain::SpamFilterAnalyzeDomain, - ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom, - headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, ip::SpamFilterAnalyzeIp, - messageid::SpamFilterAnalyzeMid, mime::SpamFilterAnalyzeMime, - pyzor::SpamFilterAnalyzePyzor, received::SpamFilterAnalyzeReceived, - recipient::SpamFilterAnalyzeRecipient, replyto::SpamFilterAnalyzeReplyTo, - rules::SpamFilterAnalyzeRules, subject::SpamFilterAnalyzeSubject, + classifier::SpamFilterAnalyzeClassify, + date::SpamFilterAnalyzeDate, + dmarc::SpamFilterAnalyzeDmarc, + domain::SpamFilterAnalyzeDomain, + ehlo::SpamFilterAnalyzeEhlo, + from::SpamFilterAnalyzeFrom, + headers::SpamFilterAnalyzeHeaders, + html::SpamFilterAnalyzeHtml, + ip::SpamFilterAnalyzeIp, + messageid::SpamFilterAnalyzeMid, + mime::SpamFilterAnalyzeMime, + pyzor::SpamFilterAnalyzePyzor, + received::SpamFilterAnalyzeReceived, + recipient::SpamFilterAnalyzeRecipient, + replyto::SpamFilterAnalyzeReplyTo, + rules::{RuleStage, SpamFilterAnalyzeRules}, + subject::SpamFilterAnalyzeSubject, url::SpamFilterAnalyzeUrl, }, }; -use common::{Server, config::mailstore::spamfilter::SpamFilterAction}; +use common::{ + Server, + config::mailstore::spamfilter::{ClassifierConfig, SpamFilterAction, SpamFilterScoreConfig}, +}; +use compact_str::CompactString; use std::{fmt::Write, future::Future, vec}; +use utils::glob::GlobMap; pub trait SpamFilterAnalyzeScore: Sync + Send { fn spam_filter_finalize( @@ -43,141 +57,116 @@ pub struct SpamFilterScore { pub is_spam: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScoreDecision { + Reject, + Discard, +} + +#[derive(Debug, Clone)] +pub struct ScoreAccumulator { + entries: Vec<(CompactString, f32)>, + tag_score: f32, + header_len: usize, + is_spam_trap: bool, + rbl_count: u32, + decision: Option, + probability: Option, + probability_entry: Option<(&'static str, f32)>, +} + +const HEADER_BASE_LEN: usize = 60; +const HEADER_TAG_OVERHEAD: usize = 14; +const EXTRA_ENTRIES: usize = 8; + +impl ScoreAccumulator { + pub fn new(result: &SpamFilterResult, scores: &GlobMap>) -> Self { + let probability = result.classifier_probability(); + let probability_tag = probability.map(|probability| probability.spam_tag()); + let mut acc = ScoreAccumulator { + entries: Vec::with_capacity(result.tags.len() + EXTRA_ENTRIES), + tag_score: 0.0, + header_len: HEADER_BASE_LEN, + is_spam_trap: false, + rbl_count: 0, + decision: None, + probability, + probability_entry: probability_tag.map(|tag| (tag, score_of(scores, tag))), + }; + for tag in &result.tags { + if probability_tag.is_some_and(|probability_tag| tag == probability_tag) { + continue; + } + if !acc.add(tag, scores) { + break; + } + } + acc + } + + pub fn add(&mut self, tag: &CompactString, scores: &GlobMap>) -> bool { + if self.decision.is_some() { + return false; + } + let score = match scores.get(tag) { + Some(SpamFilterAction::Allow(score)) => *score, + Some(SpamFilterAction::Discard) => { + self.decision = Some(ScoreDecision::Discard); + return false; + } + Some(SpamFilterAction::Reject) => { + self.decision = Some(ScoreDecision::Reject); + return false; + } + None | Some(SpamFilterAction::Disabled) => 0.0, + }; + if tag == "SPAM_TRAP" { + self.is_spam_trap = true; + } else if score > 1.0 && tag.starts_with("RBL_") { + self.rbl_count += 1; + } + self.tag_score += score; + self.header_len += tag.len() + HEADER_TAG_OVERHEAD; + if score != 0.0 || !tag.starts_with("X_") { + self.entries.push((tag.clone(), score)); + } + true + } + + pub fn decision(&self) -> Option { + self.decision + } + + pub fn score(&self) -> f32 { + self.tag_score + self.probability_entry.map_or(0.0, |(_, score)| score) + } + + pub fn probability(&self) -> Option { + self.probability + } +} + +fn score_of(scores: &GlobMap>, tag: &str) -> f32 { + scores + .get(tag) + .and_then(|v| v.as_score()) + .copied() + .unwrap_or_default() +} + impl SpamFilterAnalyzeScore for Server { async fn spam_filter_finalize( &self, ctx: &mut SpamFilterContext<'_>, ) -> SpamFilterAction { - // Calculate final score - let mut results = vec![]; - let mut header_len = 60; - let mut is_spam_trap = false; - let mut rbl_count = 0; - - for tag in &ctx.result.tags { - let score = match self.core.spam.lists.scores.get(tag) { - Some(SpamFilterAction::Allow(score)) => *score, - Some(SpamFilterAction::Discard) => { - return SpamFilterAction::Discard; - } - Some(SpamFilterAction::Reject) => { - return SpamFilterAction::Reject; - } - None | Some(SpamFilterAction::Disabled) => 0.0, - }; - if tag == "SPAM_TRAP" { - is_spam_trap = true; - } else if score > 1.0 && tag.starts_with("RBL_") { - rbl_count += 1; - } - ctx.result.score += score; - header_len += tag.len() + 10; - if score != 0.0 || !tag.starts_with("X_") { - results.push((tag.as_str(), score)); - } - } - - let mut final_score = ctx.result.score; - let mut avg_confidence: f32 = 0.0; - let mut total_results = 0; - let mut user_results = vec![ctx.result.score; ctx.input.env_rcpt_rewritten_to.len()]; - if !ctx.result.classifier_confidence.is_empty() { - for (idx, &confidence) in ctx.result.classifier_confidence.iter().enumerate() { - if let Some(confidence) = confidence { - avg_confidence += confidence; - total_results += 1; - - let user_score = self - .core - .spam - .lists - .scores - .get(confidence.spam_tag()) - .and_then(|v| v.as_score()) - .copied() - .unwrap_or_default(); - - user_results[idx] = ctx.result.score + user_score; - } - } - - if total_results > 0 { - avg_confidence /= total_results as f32; - - let tag = avg_confidence.spam_tag(); - let score = self - .core - .spam - .lists - .scores - .get(tag) - .and_then(|v| v.as_score()) - .copied() - .unwrap_or_default(); - results.push((tag, score)); - final_score += score; - } - } - - if self.core.spam.scores.reject_threshold > 0.0 - && final_score >= self.core.spam.scores.reject_threshold - { - SpamFilterAction::Reject - } else if self.core.spam.scores.discard_threshold > 0.0 - && final_score >= self.core.spam.scores.discard_threshold - { - SpamFilterAction::Discard - } else { - let mut headers = String::with_capacity(header_len + 40); - results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0))); - headers.push_str("X-Spam-Result: "); - for (idx, (tag, score)) in results.into_iter().enumerate() { - if idx > 0 { - headers.push_str(",\r\n\t"); - } - let _ = write!(&mut headers, "{} ({:.2})", tag, score); - } - headers.push_str("\r\n"); - - if let Some((category, explanation)) = &ctx.result.llm_result { - let _ = write!(&mut headers, "X-Spam-LLM: {category} ({explanation})\r\n",); - } - - let is_spam = final_score >= self.core.spam.scores.spam_threshold; - let class = if is_spam { "spam" } else { "ham" }; - - if avg_confidence != 0.0 { - let _ = write!( - &mut headers, - "X-Spam-Score: {class}, score={final_score:.2}, avg_confidence={avg_confidence:.2}\r\n", - ); - } else { - let _ = write!( - &mut headers, - "X-Spam-Score: {class}, score={final_score:.2}\r\n", - ); - } - - // Autolearn SPAM - let mut train_spam = None; - if is_spam - && self.core.spam.classifier.as_ref().is_some_and(|c| { - (c.auto_learn_spam_trap && is_spam_trap) - || (c.auto_learn_spam_rbl_count > 0 - && rbl_count >= c.auto_learn_spam_rbl_count) - }) - { - train_spam = Some(true); - } - - SpamFilterAction::Allow(SpamFilterScore { - results: user_results, - headers, - train_spam, - score: final_score, - is_spam, - }) - } + let acc = ScoreAccumulator::new(&ctx.result, &self.core.spam.lists.scores); + finalize( + ctx, + acc, + &self.core.spam.lists.scores, + &self.core.spam.scores, + self.core.spam.classifier.as_ref(), + ) } async fn spam_filter_classify( @@ -229,7 +218,6 @@ impl SpamFilterAnalyzeScore for Server { // HTML content analysis self.spam_filter_analyze_html(ctx).await; - // Spam trap self.spam_filter_analyze_spam_trap(ctx).await; @@ -238,15 +226,187 @@ impl SpamFilterAnalyzeScore for Server { // Model classification self.spam_filter_analyze_classify(ctx).await; + if let Some(probability) = ctx.result.classifier_probability() { + ctx.result + .tags + .insert(CompactString::const_new(probability.spam_tag())); + } - // User-defined rules - self.spam_filter_analyze_rules(ctx).await; + // User-defined rules that do not depend on AI tags + self.spam_filter_analyze_rule_set(ctx, RuleStage::BeforeAi, None) + .await; + + // Provisional score + let scores = &self.core.spam.lists.scores; + let mut acc = ScoreAccumulator::new(&ctx.result, scores); + + + // User-defined rules that depend on AI tags + if acc.decision().is_none() && !self.core.spam.rules.after_ai.is_empty() { + let mut added = Vec::new(); + self.spam_filter_analyze_rule_set(ctx, RuleStage::AfterAi, Some(&mut added)) + .await; + for tag in &added { + if !acc.add(tag, scores) { + break; + } + } + } // Final score calculation - self.spam_filter_finalize(ctx).await + finalize( + ctx, + acc, + scores, + &self.core.spam.scores, + self.core.spam.classifier.as_ref(), + ) } } +pub fn finalize( + ctx: &mut SpamFilterContext<'_>, + acc: ScoreAccumulator, + scores: &GlobMap>, + thresholds: &SpamFilterScoreConfig, + classifier: Option<&ClassifierConfig>, +) -> SpamFilterAction { + match acc.decision { + Some(ScoreDecision::Reject) => return SpamFilterAction::Reject, + Some(ScoreDecision::Discard) => return SpamFilterAction::Discard, + None => {} + } + + let ScoreAccumulator { + mut entries, + tag_score, + header_len, + is_spam_trap, + rbl_count, + probability, + probability_entry, + .. + } = acc; + ctx.result.score = tag_score; + + let mut final_score = tag_score; + let mut user_results = vec![tag_score; ctx.input.env_rcpt_rewritten_to.len()]; + for (user_result, confidence) in user_results + .iter_mut() + .zip(ctx.result.classifier_confidence.iter()) + { + if let Some(confidence) = confidence { + *user_result = tag_score + score_of(scores, confidence.spam_tag()); + } + } + if let Some((tag, score)) = probability_entry { + entries.push((CompactString::const_new(tag), score)); + final_score += score; + } + let avg_confidence = probability.unwrap_or_default(); + + if thresholds.reject_threshold > 0.0 && final_score >= thresholds.reject_threshold { + SpamFilterAction::Reject + } else if thresholds.discard_threshold > 0.0 && final_score >= thresholds.discard_threshold { + SpamFilterAction::Discard + } else { + let ai_header = ctx.result.ai.header.as_deref(); + let mut headers = String::with_capacity(header_len + 40 + ai_header.map_or(0, str::len)); + entries.sort_unstable_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0))); + headers.push_str("X-Spam-Result: "); + for (idx, (tag, score)) in entries.into_iter().enumerate() { + if idx > 0 { + headers.push_str(",\r\n\t"); + } + headers.push_str(&tag); + headers.push_str(" ("); + push_two_decimals(&mut headers, score); + headers.push(')'); + } + headers.push_str("\r\n"); + + if let Some(ai_header) = ai_header { + headers.push_str(ai_header); + } + + let is_spam = final_score >= thresholds.spam_threshold; + let class = if is_spam { "spam" } else { "ham" }; + + headers.push_str("X-Spam-Score: "); + headers.push_str(class); + headers.push_str(", score="); + push_two_decimals(&mut headers, final_score); + if avg_confidence != 0.0 { + headers.push_str(", avg_confidence="); + push_two_decimals(&mut headers, avg_confidence); + } + headers.push_str("\r\n"); + + // Autolearn SPAM + let mut train_spam = None; + if is_spam + && classifier.is_some_and(|c| { + (c.auto_learn_spam_trap && is_spam_trap) + || (c.auto_learn_spam_rbl_count > 0 && rbl_count >= c.auto_learn_spam_rbl_count) + }) + { + train_spam = Some(true); + } else if let Some(ai_train) = ctx.result.ai.train_spam + && ai_train == is_spam + && classifier.is_some() + { + train_spam = Some(ai_train); + } + + SpamFilterAction::Allow(SpamFilterScore { + results: user_results, + headers, + train_spam, + score: final_score, + is_spam, + }) + } +} + +#[inline(always)] +pub(crate) fn push_two_decimals(out: &mut String, value: f32) { + let scaled = (value.abs() as f64) * 100.0; + if scaled < 1e15 { + if value.is_sign_negative() { + out.push('-'); + } + let scaled = scaled.round_ties_even() as u64; + let mut buf = [b'0'; 24]; + let mut pos = buf.len(); + let fraction = (scaled % 100) as usize; + pos -= 1; + buf[pos] = b'0' + (fraction % 10) as u8; + pos -= 1; + buf[pos] = b'0' + (fraction / 10) as u8; + pos -= 1; + buf[pos] = b'.'; + let mut integral = scaled / 100; + loop { + pos -= 1; + buf[pos] = b'0' + (integral % 10) as u8; + integral /= 10; + if integral == 0 { + break; + } + } + if let Ok(text) = std::str::from_utf8(&buf[pos..]) { + out.push_str(text); + } + } else { + push_two_decimals_cold(out, value); + } +} + +#[inline(never)] +fn push_two_decimals_cold(out: &mut String, value: f32) { + let _ = write!(out, "{value:.2}"); +} + pub trait ConfidenceStore { fn spam_tag(&self) -> &'static str; } @@ -270,3 +430,4 @@ impl ConfidenceStore for f32 { } } } + diff --git a/crates/spam-filter/src/analysis/subject.rs b/crates/spam-filter/src/analysis/subject.rs index afeacf09..0184c272 100644 --- a/crates/spam-filter/src/analysis/subject.rs +++ b/crates/spam-filter/src/analysis/subject.rs @@ -4,14 +4,44 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{ + SpamFilterContext, + analysis::{ExcessEncoding, excess_encoding}, +}; use common::Server; use mail_parser::HeaderName; use nlp::tokenizers::types::TokenType; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; +use std::future::Future; -use crate::SpamFilterContext; +const CLASS_OTHER: u8 = 0; +const CLASS_SPACE: u8 = 1; +const CLASS_UPPER: u8 = 2; +const CLASS_LOWER: u8 = 3; +const CLASS_CURRENCY: u8 = 4; +const ASCII_CLASS: [u8; 256] = ascii_class_table(); + +const fn ascii_class_table() -> [u8; 256] { + let mut table = [CLASS_OTHER; 256]; + table[0x09] = CLASS_SPACE; + table[0x0a] = CLASS_SPACE; + table[0x0b] = CLASS_SPACE; + table[0x0c] = CLASS_SPACE; + table[0x0d] = CLASS_SPACE; + table[0x20] = CLASS_SPACE; + table[b'$' as usize] = CLASS_CURRENCY; + let mut byte = b'A'; + while byte <= b'Z' { + table[byte as usize] = CLASS_UPPER; + byte += 1; + } + let mut byte = b'a'; + while byte <= b'z' { + table[byte as usize] = CLASS_LOWER; + byte += 1; + } + table +} pub trait SpamFilterAnalyzeSubject: Sync + Send { fn spam_filter_analyze_subject( @@ -25,13 +55,8 @@ impl SpamFilterAnalyzeSubject for Server { let mut subject_raw = b"".as_slice(); for header in ctx.input.message.headers() { - if header.name == HeaderName::Subject { - subject_raw = ctx - .input - .message - .raw_message() - .get(header.offset_start as usize..header.offset_end as usize) - .unwrap_or_default(); + if header.name() == HeaderName::Subject { + subject_raw = header.raw_value(); break; } } @@ -45,37 +70,62 @@ impl SpamFilterAnalyzeSubject for Server { let mut word_count = 0; let mut upper_count = 0; let mut lower_count = 0; + let mut has_currency = false; - let mut last_ch = ' '; - let mut is_ascii = true; + let subject_thread = ctx.output.subject_thread.as_str(); + let is_ascii = subject_thread.is_ascii(); - for ch in ctx.output.subject_thread.chars() { - if !ch.is_whitespace() { - if last_ch.is_whitespace() { - word_count += 1; - } - - match ch { - '$' | '€' | '£' | '¥' | '₹' | '₽' | '₿' => { - ctx.result.add_tag("SUBJECT_HAS_CURRENCY"); + if is_ascii { + let mut last_is_space = true; + for &byte in subject_thread.as_bytes() { + match ASCII_CLASS[byte as usize] { + CLASS_SPACE => { + last_is_space = true; } - _ => { - if ch.is_alphabetic() { - if ch.is_uppercase() { - upper_count += 1; - } else { - lower_count += 1; - } + class => { + if last_is_space { + word_count += 1; + } + last_is_space = false; + match class { + CLASS_UPPER => upper_count += 1, + CLASS_LOWER => lower_count += 1, + CLASS_CURRENCY => has_currency = true, + _ => {} } } } } + } else { + let mut last_ch = ' '; + for ch in subject_thread.chars() { + if !ch.is_whitespace() { + if last_ch.is_whitespace() { + word_count += 1; + } - if !ch.is_ascii() { - is_ascii = false; + match ch { + '$' | '€' | '£' | '¥' | '₹' | '₽' | '₿' => { + has_currency = true; + } + _ => { + if ch.is_alphabetic() { + if ch.is_uppercase() { + upper_count += 1; + } else { + lower_count += 1; + } + } + } + } + } + + last_ch = ch; } + } - last_ch = ch; + if has_currency { + ctx.result.add_tag("SUBJECT_HAS_CURRENCY"); } if ctx.output.subject_lc.is_empty() { @@ -128,14 +178,14 @@ impl SpamFilterAnalyzeSubject for Server { } TokenType::Email(email) => { // Subject contains recipient - if ctx.output.env_to_orig_addr.contains(email) - || ctx.output.all_recipients().any(|r| &r.email == email) + if ctx.output.env_to_orig_addr.contains(email.as_ref()) + || ctx.output.all_recipients().any(|r| r.email == **email) { ctx.result.add_tag("RCPT_IN_SUBJECT"); } else { let host = email.domain_part.sld_or_default(); for rcpt in ctx.output.all_recipients() { - if &rcpt.email == email { + if rcpt.email == **email { ctx.result.add_tag("RCPT_IN_SUBJECT"); break; } else if rcpt.email.domain_part.sld_or_default() == host { @@ -165,14 +215,17 @@ impl SpamFilterAnalyzeSubject for Server { } // Validate unnecessary encoding - let subject_raw_utf8 = subject_raw_utf8.unwrap_or_default(); - if is_ascii && subject_raw_utf8.contains("=?") && subject_raw_utf8.contains("?=") { - if subject_raw_utf8.contains("?q?") || subject_raw_utf8.contains("?Q?") { - // Subject header is unnecessarily encoded in quoted-printable - ctx.result.add_tag("SUBJ_EXCESS_QP"); - } else if subject_raw_utf8.contains("?b?") || subject_raw_utf8.contains("?B?") { - // Subject header is unnecessarily encoded in base64 - ctx.result.add_tag("SUBJ_EXCESS_BASE64"); + if is_ascii { + match excess_encoding(subject_raw_utf8.unwrap_or_default()) { + ExcessEncoding::QuotedPrintable => { + // Subject header is unnecessarily encoded in quoted-printable + ctx.result.add_tag("SUBJ_EXCESS_QP"); + } + ExcessEncoding::Base64 => { + // Subject header is unnecessarily encoded in base64 + ctx.result.add_tag("SUBJ_EXCESS_BASE64"); + } + ExcessEncoding::None | ExcessEncoding::Other => {} } } } diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index b55e7c60..c24f574e 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -10,25 +10,41 @@ use crate::modules::expression::StringResolver; use crate::modules::html::SRC; use crate::{ Hostname, SpamFilterContext, TextPart, + analysis::to_lowercase_cow, modules::html::{A, HREF, HtmlToken}, }; use common::Server; -use common::config::mailstore::spamfilter::{Element, IpResolver, Location}; +use common::config::mailstore::spamfilter::{Element, IpResolver, Location, SpamLookup}; use common::scripts::IsMixedCharset; use common::scripts::functions::unicode::CharUtils; -use hyper::{Uri, header::LOCATION}; +use compact_str::CompactString; +use hyper::{Method, Uri, header::LOCATION, http::uri::Scheme}; use nlp::tokenizers::types::TokenType; -use std::collections::HashSet; use std::hash::{Hash, Hasher}; -use std::{borrow::Cow, future::Future, time::Duration}; +use std::{borrow::Cow, future::Future, sync::Arc}; +use store::ahash::AHashSet; const HTTPS_SCHEME: &str = "https://"; +#[derive(Clone, Copy, PartialEq, Eq)] +enum UrlScheme { + Https, + Http, + Other, +} + pub trait SpamFilterAnalyzeUrl: Sync + Send { fn spam_filter_analyze_url( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; + + fn resolve_redirect( + &self, + url: &str, + fetches: &mut usize, + span_id: u64, + ) -> impl Future>> + Send; } #[derive(Clone, Debug)] @@ -36,6 +52,7 @@ pub struct UrlParts<'x> { pub url: String, pub url_original: Cow<'x, str>, pub url_parsed: Option, + pub final_host: Option>, pub has_scheme: bool, } @@ -48,109 +65,18 @@ pub struct UrlParsed { impl SpamFilterAnalyzeUrl for Server { async fn spam_filter_analyze_url(&self, ctx: &mut SpamFilterContext<'_>) { // Extract URLs - let mut urls: HashSet>> = HashSet::new(); - let mut inferred_urls: HashSet>> = HashSet::new(); - for token in &ctx.output.subject_tokens { - if let TokenType::Url(url) | TokenType::UrlNoScheme(url) = token { - collect_url( - &mut urls, - &mut inferred_urls, - url.to_owned(), - Location::HeaderSubject, - ); - } - } - for (part_id, part) in ctx.output.text_parts.iter().enumerate() { - let part_id = part_id as u32; - let is_body = ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id); - let text_location = if is_body { - Location::BodyText - } else { - Location::Attachment - }; - - let tokens = match part { - TextPart::Plain { tokens, .. } => tokens, - TextPart::Html { - html_tokens, - tokens, - .. - } => { - for token in html_tokens { - if let HtmlToken::StartTag { attributes, .. } = token { - for (attr, value) in attributes { - match value { - Some(value) if [HREF, SRC].contains(attr) => { - collect_url( - &mut urls, - &mut inferred_urls, - UrlParts::new(value.trim().to_string()), - if is_body { - Location::BodyHtml - } else { - Location::Attachment - }, - ); - } - _ => {} - } - } - } - } - tokens - } - TextPart::None => &[][..], - }; - - for token in tokens { - match token { - TokenType::Url(url) | TokenType::UrlNoScheme(url) => { - if !ctx.input.is_train - && is_body - && !ctx.result.has_tag("RCPT_DOMAIN_IN_BODY") - && let Some(url_parsed) = &url.url_parsed - { - let host = url_parsed.host.sld_or_default(); - for rcpt in ctx.output.all_recipients() { - if rcpt.email.domain_part.sld_or_default() == host { - ctx.result.add_tag("RCPT_DOMAIN_IN_BODY"); - break; - } - } - } - - collect_url(&mut urls, &mut inferred_urls, url.to_owned(), text_location); - } - _ => {} - } - } - - if is_body && !ctx.input.is_train { - let is_single = match part { - TextPart::Plain { tokens, .. } => is_single_url(tokens), - TextPart::Html { - html_tokens, - tokens, - .. - } => is_single_html_url(html_tokens, tokens), - TextPart::None => false, - }; - - if is_single { - ctx.result.add_tag("URL_ONLY"); - } - } - } - - urls.extend(inferred_urls); + let mut urls = collect_urls(ctx); if !ctx.input.is_train { - let mut redirected_urls = HashSet::new(); - let mut trusted_domains: HashSet = HashSet::new(); + let mut redirected_urls = AHashSet::new(); + let mut redirector_slds: AHashSet = AHashSet::new(); + let mut redirect_fetches = 0; + let mut has_zwsp = false; + let mut has_suspicious = false; + let mut trusted_domains: AHashSet = AHashSet::new(); { - let mut checked_domains: HashSet<&str> = HashSet::new(); + let mut checked_domains: AHashSet<&str> = AHashSet::new(); for url in &urls { if let Some(url_parsed) = &url.element.url_parsed { let host_sld = url_parsed.host.sld_or_default(); @@ -164,13 +90,23 @@ impl SpamFilterAnalyzeUrl for Server { } for url in &urls { - for ch in url.element.url.chars() { - if ch.is_zwsp() { - ctx.result.add_tag("ZERO_WIDTH_SPACE_URL"); - } + let mut url_has_zwsp = false; + if !url.element.url.is_ascii() { + for ch in url.element.url.chars() { + if ch.is_zwsp() { + url_has_zwsp = true; + if !has_zwsp { + has_zwsp = true; + ctx.result.add_tag("ZERO_WIDTH_SPACE_URL"); + } + } else if !has_suspicious && ch.is_obscured() { + has_suspicious = true; + ctx.result.add_tag("SUSPICIOUS_URL"); + } - if ch.is_obscured() { - ctx.result.add_tag("SUSPICIOUS_URL"); + if url_has_zwsp && has_suspicious { + break; + } } } @@ -183,8 +119,9 @@ impl SpamFilterAnalyzeUrl for Server { let url_parsed = if let Some(url_parsed) = &url.element.url_parsed { url_parsed } else { - // URL could not be parsed - ctx.result.add_tag("UNPARSABLE_URL"); + if !url_has_zwsp { + ctx.result.add_tag("UNPARSABLE_URL"); + } continue; }; let host_sld = url_parsed.host.sld_or_default(); @@ -197,61 +134,84 @@ impl SpamFilterAnalyzeUrl for Server { if let Some(ip) = url_parsed.host.ip { // Check IP DNSBL check_dnsbl(self, ctx, &IpResolver::new(ip), Element::Ip, url.location).await; - } else if is_url_redirector(self, host_sld, ctx.input.span_id).await { + } else if !redirector_slds.contains(host_sld) + && is_url_redirector(self, host_sld, ctx.input.span_id).await + { // Check for redirectors ctx.result.add_tag("REDIRECTOR_URL"); + redirector_slds.insert(host_sld.into()); + } + } + if !redirector_slds.is_empty() { + let redirectors = urls + .extract_if(|url| { + url.element.url.contains("://") + && url.element.url_parsed.as_ref().is_some_and(|url_parsed| { + url_parsed.host.ip.is_none() + && redirector_slds.contains(url_parsed.host.sld_or_default()) + }) + }) + .collect::>(); + + for mut url in redirectors { if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") { let mut redirect_count = 1; let mut url_redirect = url.element.request_url(); + let mut final_host = None; + let mut redirect_scheme = url.element.scheme(); while redirect_count <= 3 { - match http_get_header( - self, - url_redirect.as_ref(), - LOCATION, - Duration::from_secs(5), - ) - .await + match redirect_scheme { + UrlScheme::Https => {} + UrlScheme::Http => { + ctx.result.add_tag("URL_REDIRECTOR_INSECURE"); + break; + } + UrlScheme::Other => break, + } + + if let Some(location) = self + .resolve_redirect( + url_redirect.as_ref(), + &mut redirect_fetches, + ctx.input.span_id, + ) + .await { - Ok(Some(location)) => { - let location = UrlParts::new(location); - if let Some(location_parsed) = &location.url_parsed { - if is_url_redirector( + let location = UrlParts::new(location.to_string()); + if let Some(location_parsed) = &location.url_parsed { + if is_url_redirector( + self, + location_parsed.host.sld_or_default(), + ctx.input.span_id, + ) + .await + { + url_redirect = + Cow::Owned(location.request_url().into_owned()); + redirect_scheme = location.scheme(); + redirect_count += 1; + continue; + } else { + if is_trusted_domain( self, location_parsed.host.sld_or_default(), ctx.input.span_id, ) .await { - url_redirect = - Cow::Owned(location.request_url().into_owned()); - redirect_count += 1; - continue; - } else { - if is_trusted_domain( - self, - location_parsed.host.sld_or_default(), - ctx.input.span_id, - ) - .await - { - trusted_domains.insert( - location_parsed.host.sld_or_default().into(), - ); - } - - redirected_urls.insert(ElementLocation::new( - location, - url.location, - )); + trusted_domains.insert( + location_parsed.host.sld_or_default().into(), + ); } + + final_host = + Some(Box::from(location_parsed.host.fqdn.as_str())); + redirected_urls + .insert(ElementLocation::new(location, url.location)); } } - Ok(None) => {} - Err(err) => { - trc::error!(err.span_id(ctx.input.span_id)); - } } break; } @@ -259,7 +219,9 @@ impl SpamFilterAnalyzeUrl for Server { if redirect_count > 3 { ctx.result.add_tag("URL_REDIRECTOR_NESTED"); } + url.element.final_host = final_host; } + urls.insert(url); } } @@ -321,6 +283,151 @@ impl SpamFilterAnalyzeUrl for Server { // Update context ctx.output.urls = urls; } + + async fn resolve_redirect( + &self, + url: &str, + fetches: &mut usize, + span_id: u64, + ) -> Option> { + let cache = &self.inner.cache.spam_lookup; + let guard = match cache.get(url) { + Some(SpamLookup::Redirect(location)) => return Some(location), + Some(_) => return None, + None if *fetches >= self.core.spam.url_redirect.max_urls => return None, + None => match cache.get_value_or_guard_async(&Box::::from(url)).await { + Ok(SpamLookup::Redirect(location)) => return Some(location), + Ok(_) => return None, + Err(guard) => guard, + }, + }; + + *fetches += 1; + let (entry, ttl) = match http_get_header(self, url, LOCATION).await { + Ok(Some(location)) => ( + SpamLookup::Redirect(location.into()), + self.core.spam.url_redirect.cache_ttl, + ), + Ok(None) => (SpamLookup::None, self.core.spam.url_redirect.cache_ttl), + Err(err) => { + trc::error!(err.span_id(span_id)); + (SpamLookup::None, self.inner.cache.negative_cache_ttl) + } + }; + let location = match &entry { + SpamLookup::Redirect(location) => Some(location.clone()), + _ => None, + }; + guard.insert(entry, ttl); + location + } +} + +pub fn collect_urls( + ctx: &mut SpamFilterContext<'_>, +) -> AHashSet>> { + let mut urls: AHashSet>> = AHashSet::new(); + let mut inferred_urls: AHashSet>> = AHashSet::new(); + for token in &ctx.output.subject_tokens { + if let TokenType::Url(url) | TokenType::UrlNoScheme(url) = token { + collect_url( + &mut urls, + &mut inferred_urls, + url.as_ref(), + Location::HeaderSubject, + ); + } + } + for (part_id, part) in ctx.output.text_parts.iter().enumerate() { + let is_body = ctx.input.is_body(part_id as u32); + let text_location = if is_body { + Location::BodyText + } else { + Location::Attachment + }; + + let (tokens, hidden_tokens) = match part { + TextPart::Plain { tokens, .. } => (tokens.as_slice(), &[][..]), + TextPart::Html { + html_tokens, + tokens, + hidden_tokens, + .. + } => { + for token in html_tokens { + if let HtmlToken::StartTag { attributes, .. } = token { + for (attr, value) in attributes { + match value { + Some(value) if [HREF, SRC].contains(attr) => { + let value = value.trim(); + let url = to_lowercase_cow(value); + if !urls.contains(url.as_ref()) { + urls.insert(ElementLocation::new( + UrlParts::with_lowercase( + Cow::Owned(value.to_string()), + url.into_owned(), + ), + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + )); + } + } + _ => {} + } + } + } + } + (tokens.as_slice(), hidden_tokens.as_slice()) + } + TextPart::None => (&[][..], &[][..]), + }; + + for token in tokens.iter().chain(hidden_tokens) { + match token { + TokenType::Url(url) | TokenType::UrlNoScheme(url) => { + if !ctx.input.is_train + && is_body + && !ctx.result.has_tag("RCPT_DOMAIN_IN_BODY") + && let Some(url_parsed) = &url.url_parsed + { + let host = url_parsed.host.sld_or_default(); + for rcpt in ctx.output.all_recipients() { + if rcpt.email.domain_part.sld_or_default() == host { + ctx.result.add_tag("RCPT_DOMAIN_IN_BODY"); + break; + } + } + } + + collect_url(&mut urls, &mut inferred_urls, url.as_ref(), text_location); + } + _ => {} + } + } + + if is_body && !ctx.input.is_train { + let is_single = match part { + TextPart::Plain { tokens, .. } => is_single_url(tokens), + TextPart::Html { + html_tokens, + tokens, + .. + } => is_single_html_url(html_tokens, tokens), + TextPart::None => false, + }; + + if is_single { + ctx.result.add_tag("URL_ONLY"); + } + } + } + + urls.extend(inferred_urls); + + urls } #[allow(unreachable_code)] @@ -329,9 +436,8 @@ async fn http_get_header( server: &Server, url: &str, header: hyper::header::HeaderName, - timeout: Duration, ) -> trc::Result> { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { return if url.contains("redirect.") { Ok(url.split_once("/?").unwrap().1.to_string().into()) @@ -343,8 +449,13 @@ async fn http_get_header( .core .spam .url_client - .get(url) - .timeout(timeout) + .request(Method::GET, url) + .map_err(|err| { + trc::SieveEvent::RuntimeError + .into_err() + .reason(err) + .details("Outbound URL rejected") + })? .send() .await .map_err(|err| { @@ -363,15 +474,19 @@ async fn http_get_header( } fn collect_url( - urls: &mut HashSet>>, - inferred_urls: &mut HashSet>>, - url: UrlParts<'static>, + urls: &mut AHashSet>>, + inferred_urls: &mut AHashSet>>, + url: &UrlParts<'_>, location: Location, ) { - if url.is_explicit_link() { - urls.insert(ElementLocation::new(url, location)); + let urls = if url.is_explicit_link() { + urls } else { - inferred_urls.insert(ElementLocation::new(url, location)); + inferred_urls + }; + + if !urls.contains(url.url.as_str()) { + urls.insert(ElementLocation::new(url.to_owned(), location)); } } @@ -399,7 +514,7 @@ fn is_single_url(tokens: &[TokenType]) -> bool { } fn is_single_html_url( - html_tokens: &[HtmlToken], + html_tokens: &[HtmlToken<'_>], tokens: &[TokenType], ) -> bool { let mut url_count = 0; @@ -455,26 +570,36 @@ impl<'x> UrlParts<'x> { pub fn new(url: impl Into>) -> Self { let url_original = url.into(); let url = url_original.trim().to_lowercase(); + Self::with_lowercase(url_original, url) + } + fn with_lowercase(url_original: Cow<'x, str>, url: String) -> Self { Self { url_parsed: Self::parse(&url), url, url_original, + final_host: None, has_scheme: true, } } pub fn no_scheme(url: impl Into>) -> Self { let url_original = url.into(); - let host = url_original.trim().to_lowercase(); - let mut url = String::with_capacity(HTTPS_SCHEME.len() + host.len()); + let trimmed = url_original.trim(); + let mut url = String::with_capacity(HTTPS_SCHEME.len() + trimmed.len()); url.push_str(HTTPS_SCHEME); - url.push_str(&host); + if trimmed.is_ascii() { + url.push_str(trimmed); + url.make_ascii_lowercase(); + } else { + url.push_str(&trimmed.to_lowercase()); + } Self { url_parsed: Self::parse(&url), url, url_original, + final_host: None, has_scheme: false, } } @@ -488,6 +613,14 @@ impl<'x> UrlParts<'x> { .is_some_and(|url| url.host.fqdn.starts_with("www.")) } + fn scheme(&self) -> UrlScheme { + match self.url_parsed.as_ref().and_then(|url| url.parts.scheme()) { + Some(scheme) if *scheme == Scheme::HTTPS => UrlScheme::Https, + Some(scheme) if *scheme == Scheme::HTTP => UrlScheme::Http, + _ => UrlScheme::Other, + } + } + pub fn request_url(&self) -> Cow<'_, str> { let url = self.url_original.trim(); if self.has_scheme { @@ -498,6 +631,11 @@ impl<'x> UrlParts<'x> { } fn parse(url: &str) -> Option { + match url.as_bytes().first() { + None | Some(b'/') => return None, + _ => {} + } + url.parse::().ok().and_then(|parts| { parts .host() @@ -511,23 +649,9 @@ impl<'x> UrlParts<'x> { url: self.url.clone(), url_original: Cow::Owned(self.url_original.clone().into_owned()), url_parsed: self.url_parsed.clone(), + final_host: self.final_host.clone(), has_scheme: self.has_scheme, } } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn request_url_keeps_case() { - let url = UrlParts::new(" https://Bit.ly/3AbCdEf "); - assert_eq!(url.url, "https://bit.ly/3abcdef"); - assert_eq!(url.request_url(), "https://Bit.ly/3AbCdEf"); - - let url = UrlParts::no_scheme("Bit.ly/3AbCdEf"); - assert_eq!(url.url, "https://bit.ly/3abcdef"); - assert_eq!(url.request_url(), "https://Bit.ly/3AbCdEf"); - } -} diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index 1f773f39..e716aa8e 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -8,23 +8,32 @@ pub mod analysis; pub mod modules; +mod texts; use analysis::ElementLocation; use analysis::url::UrlParts; +use common::expr::Bump; +use compact_str::CompactString; use mail_auth::{ ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput, dkim2::Dkim2Output, dmarc::Policy, }; -use mail_parser::Message; +use mail_parser::{Message, PartId}; use modules::html::HtmlToken; use nlp::tokenizers::types::TokenType; use std::borrow::Cow; -use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::net::{IpAddr, Ipv4Addr}; use store::ahash::AHashSet; +use sync_wrapper::SyncWrapper; +use texts::is_body_part; + +pub use texts::{MessageTexts, PartText}; + +pub type ContextToken<'x> = TokenType, Box, Box>, IpParts>; pub struct SpamFilterInput<'x> { pub message: &'x Message<'x>, + pub texts: &'x MessageTexts<'x>, pub span_id: u64, // Sender authentication @@ -63,8 +72,7 @@ pub struct SpamFilterOutput<'x> { pub env_from_addr: Email, pub env_from_postmaster: bool, - pub env_to_orig_addr: HashSet, - pub env_to_rewritten_addr: HashSet, + pub env_to_orig_addr: AHashSet, pub from: Recipient, pub recipients_to: Vec, pub recipients_cc: Vec, @@ -75,50 +83,83 @@ pub struct SpamFilterOutput<'x> { pub subject_lc: String, pub subject_thread: String, pub subject_thread_lc: String, - pub subject_tokens: Vec, Email, UrlParts<'x>, IpParts>>, + pub subject_tokens: Vec>, pub ips: AHashSet>, - pub urls: HashSet>>, - pub emails: HashSet>, - pub domains: HashSet>, + pub urls: AHashSet>>, + pub emails: AHashSet>, + pub domains: AHashSet>, pub text_parts: Vec>, } #[derive(Debug)] pub struct IpParts { - ip: Option, + pub ip: Option, } pub enum TextPart<'x> { Plain { text_body: &'x str, - tokens: Vec, Email, UrlParts<'x>, IpParts>>, + tokens: Vec>, }, Html { - html_tokens: Vec, - text_body: String, - tokens: Vec, Email, UrlParts<'x>, IpParts>>, + html_tokens: Vec>, + text_body: &'x str, + tokens: Vec>, + hidden_tokens: Vec>, }, None, } #[derive(Debug, Default)] pub struct SpamFilterResult { - pub tags: AHashSet, + pub tags: AHashSet, pub classifier_confidence: Vec>, pub score: f32, pub rbl_ip_checks: usize, pub rbl_domain_checks: usize, pub rbl_url_checks: usize, pub rbl_email_checks: usize, - pub llm_result: Option<(String, String)>, + pub ai: SpamFilterAiResult, +} + +#[derive(Debug, Default)] +pub struct SpamFilterAiResult { + pub gate: Option, + pub facts: Option, + pub header: Option, + pub train_spam: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AiGate { + Run, + Authenticated, + Decided, + Condition, + NoContent, + Unavailable, +} + +impl AiGate { + pub fn as_str(&self) -> &'static str { + match self { + AiGate::Run => "run", + AiGate::Authenticated => "authenticated", + AiGate::Decided => "decided", + AiGate::Condition => "condition", + AiGate::NoContent => "no-content", + AiGate::Unavailable => "unavailable", + } + } } pub struct SpamFilterContext<'x> { pub input: SpamFilterInput<'x>, pub output: SpamFilterOutput<'x>, pub result: SpamFilterResult, + pub arena: SyncWrapper, } #[derive(Debug, Clone)] @@ -142,9 +183,14 @@ pub struct Recipient { } impl<'x> SpamFilterInput<'x> { - pub fn from_message(message: &'x Message<'x>, span_id: u64) -> Self { + pub fn from_message( + message: &'x Message<'x>, + texts: &'x MessageTexts<'x>, + span_id: u64, + ) -> Self { Self { message, + texts, span_id, arc_result: None, spf_ehlo_result: None, @@ -173,6 +219,30 @@ impl<'x> SpamFilterInput<'x> { self.is_train = true; self } + + pub fn is_text_body(&self, part_id: PartId) -> bool { + self.message + .part(part_id) + .is_some_and(|part| part.message().id() == 0 && part.in_text_body()) + } + + pub fn is_html_body(&self, part_id: PartId) -> bool { + self.message + .part(part_id) + .is_some_and(|part| part.message().id() == 0 && part.in_html_body()) + } + + pub fn is_body(&self, part_id: PartId) -> bool { + self.message.part(part_id).is_some_and(is_body_part) + } + + pub fn first_body_part(&self) -> Option { + self.message + .text_body() + .next() + .or_else(|| self.message.html_body().next()) + .map(|part| part.id()) + } } impl PartialEq for Hostname { diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index dd8ba5c7..e123bb2f 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -7,15 +7,16 @@ use crate::analysis::domain::SpamFilterAnalyzeDomain; use crate::analysis::init::SpamFilterInit; use crate::analysis::is_trusted_domain; -use crate::analysis::url::SpamFilterAnalyzeUrl; +use crate::analysis::url::{SpamFilterAnalyzeUrl, UrlParsed}; use crate::modules::html::{A, ALT, HREF, HtmlToken, IMG, SRC, TITLE}; use crate::{Email, SpamFilterContext, TextPart}; -use crate::{Hostname, SpamFilterInput}; +use crate::{Hostname, MessageTexts, SpamFilterInput}; use common::config::mailstore::spamfilter; use common::manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY}; use common::{Server, config::mailstore::spamfilter::Location, ipc::BroadcastEvent}; +use compact_str::CompactString; use mail_auth::DmarcResult; -use mail_parser::{MessageParser, MimeHeaders}; +use mail_parser::MessageParser; use nlp::classifier::feature::{ CcfhFeature, CcfhFeatureBuilder, FeatureBuilder, FhFeature, FhFeatureBuilder, Sample, UnprocessedFeature, @@ -32,16 +33,18 @@ use std::time::Instant; use std::{ borrow::Cow, collections::{HashMap, hash_map::Entry}, - hash::{Hash, RandomState}, + hash::Hash, sync::Arc, }; -use store::ahash::AHashMap; -use store::rand::seq::SliceRandom; -use store::write::{BlobLink, RegistryClass, now}; +use store::ahash::{AHashMap, AHashSet, RandomState}; +#[cfg(stalwart_tests)] +use store::rand::SeedableRng; +use store::rand::{rngs::StdRng, seq::SliceRandom}; +use store::write::{ArchiveCompression, BlobLink, Compression, RegistryClass, now}; use store::{ Deserialize, IterateParams, Serialize, ValueKey, write::{ - AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, ValueClass, + Archive, ArchiveBytes, Archiver, BatchBuilder, BlobOp, ValueClass, key::DeserializeBigEndian, }, }; @@ -140,7 +143,7 @@ impl SpamClassifier for Server { .get_blob(SPAM_TRAINER_KEY, 0..usize::MAX) .await .and_then(|archive| match archive { - Some(archive) => as Deserialize>::deserialize(&archive) + Some(archive) => as Deserialize>::deserialize(&archive) .and_then(|archive| archive.deserialize_untrusted::()) .map(Some), None => Ok(None), @@ -207,6 +210,10 @@ impl SpamClassifier for Server { let mut seen_samples = AHashMap::new(); let mut spam_count = 0; let mut ham_count = 0; + #[cfg(stalwart_tests)] + let mut rng = StdRng::seed_from_u64(42); + #[cfg(not(stalwart_tests))] + let mut rng: StdRng = store::rand::make_rng(); self.store() .iterate( IterateParams::new(from_key, to_key).descending(), @@ -235,6 +242,7 @@ impl SpamClassifier for Server { &sample, is_spam, config.reservoir_capacity, + &mut rng, ); } else { trainer.reservoir.update_counts(is_spam); @@ -328,7 +336,7 @@ impl SpamClassifier for Server { samples.extend( trainer .reservoir - .replay_samples((spam_count - ham_count) as usize, false) + .replay_samples((spam_count - ham_count) as usize, false, &mut rng) .map(|sample| TrainingTask { id: 0, sample: sample.clone(), @@ -342,7 +350,7 @@ impl SpamClassifier for Server { samples.extend( trainer .reservoir - .replay_samples((ham_count - spam_count) as usize, true) + .replay_samples((ham_count - spam_count) as usize, true, &mut rng) .map(|sample| TrainingTask { id: 0, sample: sample.clone(), @@ -354,7 +362,7 @@ impl SpamClassifier for Server { } let num_samples = samples.len(); - samples.shuffle(&mut store::rand::rng()); + samples.shuffle(&mut rng); // Spawn training task let epochs = match trainer @@ -425,8 +433,10 @@ impl SpamClassifier for Server { ); continue; }; - let mut ctx = - self.spam_filter_init(SpamFilterInput::from_message(&message, 0).train_mode()); + let texts = MessageTexts::new(&message, self.core.spam.text_limits); + let mut ctx = self.spam_filter_init( + SpamFilterInput::from_message(&message, &texts, 0).train_mode(), + ); self.spam_filter_analyze_domain(&mut ctx).await; self.spam_filter_analyze_url(&mut ctx).await; let mut tokens = self.spam_build_tokens(&ctx).await.0; @@ -548,12 +558,24 @@ impl SpamClassifier for Server { last_trained_at: now(), }, }); + let (trainer_bytes, classifier_bytes, classifier) = + tokio::task::spawn_blocking(move || { + let trainer_bytes = Archiver::new(trainer).serialize(); + let classifier_bytes = classifier.serialize(); + (trainer_bytes, classifier_bytes, classifier) + }) + .await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .details("Spam model serialization failed") + .caused_by(trc::location!()) + })?; + self.blob_store() .put_blob( SPAM_TRAINER_KEY, - &Archiver::new(trainer) - .serialize() - .caused_by(trc::location!())?, + &trainer_bytes.caused_by(trc::location!())?, self.core.email.compression, ) .await @@ -561,7 +583,7 @@ impl SpamClassifier for Server { self.blob_store() .put_blob( SPAM_CLASSIFIER_KEY, - &classifier.serialize().caused_by(trc::location!())?, + &classifier_bytes.caused_by(trc::location!())?, self.core.email.compression, ) .await @@ -606,6 +628,9 @@ impl SpamClassifier for Server { if config.log_scale { feature_builder.scale(&mut tokens); } + let mut base_features = Vec::new(); + let mut account_features = Vec::new(); + let mut base_score = None; for rcpt in &ctx.input.env_rcpt_rewritten_to { let prediction = if let Some(account_id) = self @@ -614,12 +639,27 @@ impl SpamClassifier for Server { .caused_by(trc::location!())? { has_prediction = true; - classifier - .predict_proba_sample(&feature_builder.build( + let score = if let Some(score) = base_score { + feature_builder.build_part( &tokens, - account_id.into(), + Some(account_id), + &mut account_features, + ); + score + } else { + feature_builder.build_base_and_account( + &tokens, + account_id, + &mut base_features, + &mut account_features, + ); + *base_score.insert(classifier.score(&base_features)) + }; + classifier + .predict_proba_score( + score.combine(classifier.score(&account_features)), config.l2_normalize, - )) + ) .into() } else { None @@ -631,11 +671,9 @@ impl SpamClassifier for Server { ctx.result.classifier_confidence = classifier_confidence; } else { // None of the recipients are local, default to global model prediction - let prediction = classifier.predict_proba_sample(&feature_builder.build( - &tokens, - None, - config.l2_normalize, - )); + feature_builder.build_part(&tokens, None, &mut base_features); + let prediction = classifier + .predict_proba_score(classifier.score(&base_features), config.l2_normalize); ctx.result.classifier_confidence = vec![prediction.into(); ctx.input.env_rcpt_rewritten_to.len()]; } @@ -649,6 +687,9 @@ impl SpamClassifier for Server { if config.log_scale { feature_builder.scale(&mut tokens); } + let mut base_features = Vec::new(); + let mut account_features = Vec::new(); + let mut base_score = None; for rcpt in &ctx.input.env_rcpt_rewritten_to { let prediction = if let Some(account_id) = self @@ -657,12 +698,27 @@ impl SpamClassifier for Server { .caused_by(trc::location!())? { has_prediction = true; - classifier - .predict_proba_sample(&feature_builder.build( + let score = if let Some(score) = base_score { + feature_builder.build_part( &tokens, - account_id.into(), + Some(account_id), + &mut account_features, + ); + score + } else { + feature_builder.build_base_and_account( + &tokens, + account_id, + &mut base_features, + &mut account_features, + ); + *base_score.insert(classifier.score(&base_features)) + }; + classifier + .predict_proba_score( + score.combine(classifier.score(&account_features)), config.l2_normalize, - )) + ) .into() } else { None @@ -674,11 +730,9 @@ impl SpamClassifier for Server { ctx.result.classifier_confidence = classifier_confidence; } else { // None of the recipients are local, default to global model prediction - let prediction = classifier.predict_proba_sample(&feature_builder.build( - &tokens, - None, - config.l2_normalize, - )); + feature_builder.build_part(&tokens, None, &mut base_features); + let prediction = classifier + .predict_proba_score(classifier.score(&base_features), config.l2_normalize); ctx.result.classifier_confidence = vec![prediction.into(); ctx.input.env_rcpt_rewritten_to.len()]; } @@ -696,7 +750,7 @@ impl SpamClassifier for Server { .iter() .zip(ctx.input.env_rcpt_rewritten_to.iter()) .map(|(v, rcpt)| trc::Value::Array(vec![ - trc::Value::from(rcpt.to_string()), + trc::Value::from(CompactString::from(*rcpt)), trc::Value::from(*v) ])) .collect::>(), @@ -708,49 +762,54 @@ impl SpamClassifier for Server { } async fn spam_build_tokens<'x>(&self, ctx: &'x SpamFilterContext<'_>) -> Tokens<'x> { - let mut tokens = Tokens::default(); - - // Add From addresses - if ctx - .input - .dmarc_result - .as_ref() - .is_some_and(|result| **result != DmarcResult::Pass) - { - tokens.insert(Token::Sender { value: "!".into() }); - } - for email in [&ctx.output.env_from_addr, &ctx.output.from.email] { - tokens.insert_email(email, true); + let builder = spam_collect_tokens(ctx); + let mut checked = AHashSet::new(); + let mut trusted: AHashSet = AHashSet::new(); + for domain in builder.trust_domains() { + if checked.insert(domain) && is_trusted_domain(self, domain, ctx.input.span_id).await { + trusted.insert(domain.to_string()); + } } + drop(checked); - // Add Email addresses - for email in &ctx.output.emails { - let is_sender = match &email.location { - Location::HeaderReplyTo | Location::HeaderDnt => true, - Location::BodyText - | Location::BodyHtml - | Location::Attachment - | Location::HeaderSubject => false, - _ => continue, - }; + builder.finish(|domain| trusted.contains(domain)) + } +} - if is_sender - || !is_trusted_domain( - self, - email.element.email.domain_part.sld_or_default(), - ctx.input.span_id, - ) - .await - { - tokens.insert_email(&email.element.email, is_sender); +pub struct TokenBuilder<'x> { + tokens: Tokens<'x>, + alt_tokens: Tokens<'x>, + emails: Vec<&'x Email>, + urls: Vec<&'x UrlParsed>, + hosts: Vec, +} + +impl<'x> TokenBuilder<'x> { + pub fn trust_domains(&self) -> impl Iterator { + self.emails + .iter() + .map(|email| email.domain_part.sld_or_default()) + .chain(self.urls.iter().map(|url| url.host.sld_or_default())) + .chain(self.hosts.iter().map(|host| host.sld_or_default())) + } + + pub fn finish(self, is_trusted: impl Fn(&str) -> bool) -> Tokens<'x> { + let TokenBuilder { + mut tokens, + alt_tokens, + emails, + urls, + hosts, + } = self; + + for email in emails { + if !is_trusted(email.domain_part.sld_or_default()) { + tokens.insert_email(email, false); } } - // Add URLs - for url in &ctx.output.urls { - if let Some(url) = &url.element.url_parsed - && !is_trusted_domain(self, url.host.sld_or_default(), ctx.input.span_id).await - { + for url in urls { + if !is_trusted(url.host.sld_or_default()) { if let Some(host) = &url.host.sld { tokens.insert(Token::Url { value: host.into() }); if host != &url.host.fqdn { @@ -770,119 +829,30 @@ impl SpamClassifier for Server { .filter(|v| v.chars().all(|ch| ch.is_alphabetic())) { if token.len() > 2 { - let token = truncate_word(token, MAX_TOKEN_LENGTH); tokens.insert(Token::Url { - value: format!("_{token}").into(), + value: concat_word("_", truncate_word(token, MAX_TOKEN_LENGTH)).into(), }); } } } } - // Add hostnames - for domain in &ctx.output.domains { - if matches!( - domain.location, - Location::HeaderReceived | Location::HeaderMid | Location::Ehlo | Location::Tcp - ) { - let host = Hostname::new(&domain.element); - let host_sld = host.sld_or_default(); - - if !is_trusted_domain(self, host_sld, ctx.input.span_id).await { - if !host_sld.is_empty() && host_sld != host.fqdn { - tokens.insert(Token::Hostname { - value: host_sld.to_string().into(), - }); - } + for host in hosts { + let host_sld = host.sld_or_default(); + if !is_trusted(host_sld) { + if !host_sld.is_empty() && host_sld != host.fqdn { tokens.insert(Token::Hostname { - value: host.fqdn.into(), + value: host_sld.to_string().into(), }); } + + tokens.insert(Token::Hostname { + value: host.fqdn.into(), + }); } } - // Add ASN - if let Some(asn) = ctx.input.asn { - tokens.insert(Token::Asn { - number: asn.to_be_bytes(), - }); - } - - // Add MIME and attachment indicators - for part in &ctx.input.message.parts { - if let Some(name) = part.attachment_name() - && let Some((name, ext)) = name.rsplit_once('.') - { - if !ext.is_empty() { - tokens.insert(Token::Attachment { - value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), - }); - } - let name = name.to_lowercase(); - let word_tokenizer = WordStemTokenizer::new(&name); - for token in TypesTokenizer::new(&name) { - if let TokenType::Alphabetic(word) = token.word { - word_tokenizer.tokenize(word, |token| { - tokens.insert(Token::Attachment { - value: format!( - "_{}", - truncate_word(token.as_ref(), MAX_TOKEN_LENGTH) - ) - .into(), - }); - }); - } - } - } - - if let Some(ct) = part.content_type() { - let mut ct_lower = String::with_capacity( - ct.c_type.len() + ct.c_subtype.as_ref().map_or(0, |s| s.len()), - ); - for ch in ct.c_type.chars() { - ct_lower.push(ch.to_ascii_lowercase()); - } - if let Some(st) = &ct.c_subtype { - ct_lower.push('/'); - for ch in st.chars() { - ct_lower.push(ch.to_ascii_lowercase()); - } - } - - tokens.insert(Token::MimeType { value: ct_lower }); - } - } - - // Tokenize the subject - for token in &ctx.output.subject_tokens { - tokens.insert_type( - &WordStemTokenizer::new(&ctx.output.subject_thread_lc), - token, - false, - ); - } - - // Tokenize the text parts - let body_idx = ctx - .input - .message - .html_body - .first() - .or_else(|| ctx.input.message.text_body.first()) - .map(|idx| *idx as usize); - let mut alt_tokens = Tokens::default(); - for (idx, part) in ctx.output.text_parts.iter().enumerate() { - let is_body = Some(idx) == body_idx; - if is_body - || (!ctx.input.message.text_body.contains(&(idx as u32)) - && !ctx.input.message.html_body.contains(&(idx as u32))) - { - tokens.insert_text_part(part, is_body); - } else { - alt_tokens.insert_text_part(part, false); - } - } if !alt_tokens.0.is_empty() { for (token, count) in alt_tokens.0.into_iter() { if let Entry::Vacant(entry) = tokens.0.entry(token) { @@ -895,6 +865,143 @@ impl SpamClassifier for Server { } } +pub fn spam_collect_tokens<'x>(ctx: &'x SpamFilterContext<'_>) -> TokenBuilder<'x> { + let mut tokens = Tokens::default(); + let mut emails = Vec::new(); + let mut urls = Vec::new(); + let mut hosts = Vec::new(); + + // Add From addresses + if ctx + .input + .dmarc_result + .as_ref() + .is_some_and(|result| **result != DmarcResult::Pass) + { + tokens.insert(Token::Sender { value: "!".into() }); + } + for email in [&ctx.output.env_from_addr, &ctx.output.from.email] { + tokens.insert_email(email, true); + } + + // Add Email addresses + for email in &ctx.output.emails { + let is_sender = match &email.location { + Location::HeaderReplyTo | Location::HeaderDnt => true, + Location::BodyText + | Location::BodyHtml + | Location::Attachment + | Location::HeaderSubject => false, + _ => continue, + }; + + if is_sender { + tokens.insert_email(&email.element.email, true); + } else { + emails.push(&email.element.email); + } + } + + // Add URLs + for url in &ctx.output.urls { + if let Some(url) = &url.element.url_parsed { + urls.push(url); + } + } + + // Add hostnames + for domain in &ctx.output.domains { + if matches!( + domain.location, + Location::HeaderReceived | Location::HeaderMid | Location::Ehlo | Location::Tcp + ) { + hosts.push(Hostname::new(&domain.element)); + } + } + + // Add ASN + if let Some(asn) = ctx.input.asn { + tokens.insert(Token::Asn { + number: asn.to_be_bytes(), + }); + } + + // Add MIME and attachment indicators + for part in ctx.input.message.root().parts() { + if let Some(name) = part.attachment_name() + && let Some((name, ext)) = name.rsplit_once('.') + { + if !ext.is_empty() { + tokens.insert(Token::Attachment { + value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), + }); + } + let name = name.to_lowercase(); + let word_tokenizer = WordStemTokenizer::new(&name); + for token in TypesTokenizer::new(&name) { + if let TokenType::Alphabetic(word) = token.word { + word_tokenizer.tokenize(word, |token| { + tokens.insert(Token::Attachment { + value: concat_word( + "_", + truncate_word(token.as_ref(), MAX_TOKEN_LENGTH), + ) + .into(), + }); + }); + } + } + } + + if let Some(ct) = part.content_type() { + let mut ct_lower = + String::with_capacity(ct.ctype().len() + ct.subtype().map_or(0, |s| s.len() + 1)); + ct_lower.push_str(ct.ctype()); + if let Some(st) = ct.subtype() { + ct_lower.push('/'); + ct_lower.push_str(st); + } + ct_lower.make_ascii_lowercase(); + + tokens.insert(Token::MimeType { value: ct_lower }); + } + } + + // Tokenize the subject + if !ctx.output.subject_tokens.is_empty() { + let subject_tokenizer = WordStemTokenizer::new(&ctx.output.subject_thread_lc); + for token in &ctx.output.subject_tokens { + tokens.insert_type(&subject_tokenizer, token, false); + } + } + + // Tokenize the text parts + let body_idx = ctx + .input + .message + .html_body() + .next() + .or_else(|| ctx.input.message.text_body().next()) + .map(|part| part.id() as usize); + let mut alt_tokens = Tokens(HashMap::with_hasher(RandomState::new())); + for (idx, part) in ctx.output.text_parts.iter().enumerate() { + let is_body = Some(idx) == body_idx; + if is_body || !ctx.input.is_body(idx as u32) { + tokens.insert_text_part(part, is_body); + } else { + alt_tokens.insert_text_part(part, false); + } + } + + TokenBuilder { + tokens, + alt_tokens, + emails, + urls, + hosts, + } +} + async fn delete_samples( server: &Server, samples: Vec, @@ -930,7 +1037,7 @@ async fn delete_samples( if batch.is_large_batch() { server .store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -940,7 +1047,7 @@ async fn delete_samples( if !batch.is_empty() { server .store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; } @@ -1033,7 +1140,103 @@ impl SpamTrainerClass { } } -const MAX_TOKEN_LENGTH: usize = 16; +pub const MAX_TOKEN_LENGTH: usize = 16; + +const ASCII_CASE_BUF: usize = 64; + +const EXACT_F32_INTEGERS: f64 = 16_777_216.0; + +struct WordCounts<'x>(AHashMap<&'x str, u32>); + +impl<'x> WordCounts<'x> { + fn with_capacity(tokens: usize) -> Self { + WordCounts(AHashMap::with_capacity(tokens.div_ceil(8))) + } + + #[inline(always)] + fn push(&mut self, word: &'x str) { + let count = self.0.entry(word).or_insert(0); + *count = count.saturating_add(1); + } +} + +const ASCII_IGNORED_CATEGORY: u128 = ascii_ignored_category_mask(); + +const fn ascii_ignored_category_mask() -> u128 { + let chars = b" !\"#%&'()*,-./:;?@[\\]_{}"; + let mut mask = 0u128; + let mut idx = 0; + while idx < chars.len() { + mask |= 1u128 << chars[idx]; + idx += 1; + } + mask +} + +#[inline(always)] +fn is_ascii_ignored_category(ch: char) -> bool { + let code = ch as u32; + code < 128 && ASCII_IGNORED_CATEGORY & (1u128 << code) != 0 +} + +#[inline(always)] +fn ascii_lowercase<'a>(word: &str, buf: &'a mut [u8; ASCII_CASE_BUF]) -> Cow<'a, str> { + if let Some(slot) = buf.get_mut(..word.len()) { + for (out, &byte) in slot.iter_mut().zip(word.as_bytes()) { + *out = byte.to_ascii_lowercase(); + } + if let Ok(text) = std::str::from_utf8(slot) { + return Cow::Borrowed(text); + } + } + Cow::Owned(word.to_ascii_lowercase()) +} + +fn lowercase(text: &str) -> Cow<'_, str> { + if text.is_ascii() { + if text.bytes().any(|byte| byte.is_ascii_uppercase()) { + Cow::Owned(text.to_ascii_lowercase()) + } else { + Cow::Borrowed(text) + } + } else { + Cow::Owned(text.to_lowercase()) + } +} + +fn concat_word(prefix: &str, word: &str) -> String { + let mut value = String::with_capacity(prefix.len() + word.len()); + value.push_str(prefix); + value.push_str(word); + value +} + +#[inline(always)] +fn stem_value(stem: Cow<'_, str>) -> Cow<'_, str> { + match stem { + Cow::Borrowed(text) => Cow::Borrowed(truncate_word(text, MAX_TOKEN_LENGTH)), + Cow::Owned(text) if text.len() <= MAX_TOKEN_LENGTH => Cow::Owned(text), + Cow::Owned(text) => Cow::Owned(truncate_word(&text, MAX_TOKEN_LENGTH).to_string()), + } +} + +#[inline(always)] +fn owned_stem_value(stem: Cow<'_, str>) -> Cow<'static, str> { + match stem { + Cow::Owned(text) if text.len() <= MAX_TOKEN_LENGTH => Cow::Owned(text), + other => Cow::Owned(truncate_word(other.as_ref(), MAX_TOKEN_LENGTH).to_string()), + } +} + +enum WordType<'x> { + Alphabetic(&'x str), + Alphanumeric(&'x str), + UrlNoHost(&'x str), + Integer(&'x str), + Float(&'x str), + Char(char), + IpAddr, +} #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, PartialOrd, Ord, @@ -1059,13 +1262,14 @@ pub enum Token<'x> { pub struct Tokens<'x>(pub HashMap, f32, RandomState>); impl<'x> Tokens<'x> { - fn insert_text_part(&mut self, part: &'x TextPart<'x>, is_body: bool) { + pub fn insert_text_part(&mut self, part: &'x TextPart<'x>, is_body: bool) { match part { TextPart::Plain { text_body, tokens } => { let word_tokenizer = WordStemTokenizer::new(text_body); + let mut words = WordCounts::with_capacity(tokens.len()); for token in tokens { - self.insert_type(&word_tokenizer, token, is_body); + self.insert_counted(&word_tokenizer, &mut words, token, is_body); } if is_body @@ -1076,16 +1280,20 @@ impl<'x> Tokens<'x> { value: "_null".into(), }); } + + self.flush_words(&word_tokenizer, words, is_body); } TextPart::Html { text_body, tokens, html_tokens, + .. } => { let word_tokenizer = WordStemTokenizer::new(text_body); + let mut words = WordCounts::with_capacity(tokens.len()); for token in tokens { - self.insert_type(&word_tokenizer, token, is_body); + self.insert_counted(&word_tokenizer, &mut words, token, is_body); } if is_body { @@ -1108,7 +1316,15 @@ impl<'x> Tokens<'x> { match (*name, value) { (ALT | TITLE, Some(value)) => { for token in TypesTokenizer::new(value) { - self.insert_type(&word_tokenizer, &token.word, is_body); + if let TokenType::Alphabetic(word) = token.word { + words.push(word); + } else { + self.insert_type_str( + &word_tokenizer, + &token.word, + is_body, + ); + } } } (SRC, Some(value)) => { @@ -1127,85 +1343,113 @@ impl<'x> Tokens<'x> { } } } + + self.flush_words(&word_tokenizer, words, is_body); } TextPart::None => (), } } - fn insert_type, E, U, I>( + fn insert_counted, E, U, I>( &mut self, word_tokenizer: &WordStemTokenizer, - token: &TokenType, + words: &mut WordCounts<'x>, + token: &'x TokenType, is_body: bool, ) { - match token { - TokenType::Alphabetic(word) => { - let word = word.as_ref(); - let mut set: Option = None; - let mut has_confusables = false; - let mut upper_count = 0; - for ch in word.chars() { - if ch.is_uppercase() { - upper_count += 1; - } + if let TokenType::Alphabetic(word) = token { + words.push(word.as_ref()); + } else { + self.insert_type(word_tokenizer, token, is_body); + } + } - has_confusables |= - !ch.is_ascii() && !std::iter::once(ch).nfc().eq(std::iter::once(ch).nfkc()); - set.get_or_insert_default().intersect_with(ch.into()); - } - let is_mixed_script = set.is_some_and(|set| set.is_empty()); + fn flush_words( + &mut self, + word_tokenizer: &WordStemTokenizer, + words: WordCounts<'x>, + is_body: bool, + ) { + self.0.reserve(words.0.len()); + for (word, count) in words.0 { + self.insert_alphabetic(word_tokenizer, word, is_body, count); + } + } - if (is_mixed_script || has_confusables) - && let Ok(cured_word) = decancer::cure(word, decancer::Options::default()) - { - if word.len() > MAX_TOKEN_LENGTH { - self.insert(Token::Word { - value: truncate_word(cured_word.as_str(), MAX_TOKEN_LENGTH) - .to_string() - .into(), - }); - } else { - self.insert(Token::Word { - value: String::from(cured_word).into(), - }); - } - } else { - let word = word.to_lowercase(); - word_tokenizer.tokenize(&word, |token| { - self.insert(Token::Word { - value: truncate_word(token.as_ref(), MAX_TOKEN_LENGTH) - .to_string() - .into(), - }); - }); - } + pub fn insert_type, E, U, I>( + &mut self, + word_tokenizer: &WordStemTokenizer, + token: &'x TokenType, + is_body: bool, + ) { + let word = match token { + TokenType::Alphabetic(word) => WordType::Alphabetic(word.as_ref()), + TokenType::Alphanumeric(word) => WordType::Alphanumeric(word.as_ref()), + TokenType::UrlNoHost(url) => WordType::UrlNoHost(url.as_ref()), + TokenType::Integer(word) => WordType::Integer(word.as_ref()), + TokenType::Float(word) => WordType::Float(word.as_ref()), + TokenType::Other(ch) | TokenType::Punctuation(ch) => WordType::Char(*ch), + TokenType::IpAddr(_) => WordType::IpAddr, + TokenType::Email(_) + | TokenType::Url(_) + | TokenType::UrlNoScheme(_) + | TokenType::Space => return, + }; + self.insert_word_type(word_tokenizer, word, is_body); + } - if is_body && word.len() == upper_count && word.len() > 3 { - self.insert(Token::Word { - value: "_allcaps".into(), - }); - } + pub fn insert_type_str( + &mut self, + word_tokenizer: &WordStemTokenizer, + token: &TokenType<&'x str, E, U, I>, + is_body: bool, + ) { + let word = match token { + TokenType::Alphabetic(word) => WordType::Alphabetic(word), + TokenType::Alphanumeric(word) => WordType::Alphanumeric(word), + TokenType::UrlNoHost(url) => WordType::UrlNoHost(url), + TokenType::Integer(word) => WordType::Integer(word), + TokenType::Float(word) => WordType::Float(word), + TokenType::Other(ch) | TokenType::Punctuation(ch) => WordType::Char(*ch), + TokenType::IpAddr(_) => WordType::IpAddr, + TokenType::Email(_) + | TokenType::Url(_) + | TokenType::UrlNoScheme(_) + | TokenType::Space => return, + }; + self.insert_word_type(word_tokenizer, word, is_body); + } + + fn insert_word_type( + &mut self, + word_tokenizer: &WordStemTokenizer, + word: WordType<'x>, + is_body: bool, + ) { + match word { + WordType::Alphabetic(word) => { + self.insert_alphabetic(word_tokenizer, word, is_body, 1); } - TokenType::Alphanumeric(word) => { - self.insert(Token::from_alphanumeric(word.as_ref())); + WordType::Alphanumeric(word) => { + self.insert(Token::from_alphanumeric(word)); } - TokenType::UrlNoHost(url) => { - for token in url - .as_ref() - .to_lowercase() + WordType::UrlNoHost(url) => { + for token in lowercase(url) .split(['/', '.', '_']) .filter(|v| v.chars().all(|ch| ch.is_alphabetic())) { if token.len() > 2 { - let token = truncate_word(token, MAX_TOKEN_LENGTH); self.insert(Token::Url { - value: format!("_{token}").into(), + value: concat_word("_", truncate_word(token, MAX_TOKEN_LENGTH)).into(), }); } } } - TokenType::Other(ch) | TokenType::Punctuation(ch) => { - let category = get_general_category(*ch); + WordType::Char(ch) => { + if is_ascii_ignored_category(ch) { + return; + } + let category = get_general_category(ch); if !matches!( category, GeneralCategory::ClosePunctuation @@ -1222,33 +1466,140 @@ impl<'x> Tokens<'x> { }); } } - TokenType::Integer(word) => { - self.insert(Token::from_number(false, word.as_ref())); + WordType::Integer(word) => { + self.insert(Token::from_number(false, word)); } - TokenType::Float(word) => { - self.insert(Token::from_number(true, word.as_ref())); + WordType::Float(word) => { + self.insert(Token::from_number(true, word)); } - TokenType::IpAddr(_) => { + WordType::IpAddr => { self.insert(Token::Url { value: "!ip".into(), }); } - TokenType::Email(_) - | TokenType::Url(_) - | TokenType::UrlNoScheme(_) - | TokenType::Space => {} } } - fn insert(&mut self, token: Token<'x>) { + fn insert_alphabetic( + &mut self, + word_tokenizer: &WordStemTokenizer, + word: &'x str, + is_body: bool, + count: u32, + ) { + let bytes = word.as_bytes(); + let mut ascii_bits = 0u8; + let mut upper_count = 0usize; + for &byte in bytes { + ascii_bits |= byte; + upper_count += usize::from(byte.wrapping_sub(b'A') < 26); + } + + if ascii_bits < 0x80 { + if upper_count == 0 { + word_tokenizer.tokenize(word, |stem| { + self.add( + Token::Word { + value: stem_value(stem), + }, + count, + ); + }); + return; + } + + let mut buf = [0u8; ASCII_CASE_BUF]; + let lower = ascii_lowercase(word, &mut buf); + word_tokenizer.tokenize(lower.as_ref(), |stem| { + self.add( + Token::Word { + value: owned_stem_value(stem), + }, + count, + ); + }); + + if is_body && bytes.len() == upper_count && bytes.len() > 3 { + self.add( + Token::Word { + value: "_allcaps".into(), + }, + count, + ); + } + return; + } + + let mut set: Option = None; + let mut needs_cure = false; + for ch in word.chars() { + if !ch.is_ascii() && !std::iter::once(ch).nfc().eq(std::iter::once(ch).nfkc()) { + needs_cure = true; + break; + } + let set = set.get_or_insert_default(); + set.intersect_with(ch.into()); + if set.is_empty() { + needs_cure = true; + break; + } + } + + if needs_cure && let Ok(cured_word) = decancer::cure(word, decancer::Options::default()) { + if word.len() > MAX_TOKEN_LENGTH { + self.add( + Token::Word { + value: truncate_word(&cured_word, MAX_TOKEN_LENGTH) + .to_string() + .into(), + }, + count, + ); + } else { + self.add( + Token::Word { + value: String::from(cured_word).into(), + }, + count, + ); + } + } else { + let lower = word.to_lowercase(); + word_tokenizer.tokenize(&lower, |stem| { + self.add( + Token::Word { + value: owned_stem_value(stem), + }, + count, + ); + }); + } + } + + fn add(&mut self, token: Token<'x>, count: u32) { + let entry = self.0.entry(token).or_insert(0.0); + if f64::from(*entry) + f64::from(count) <= EXACT_F32_INTEGERS { + *entry += count as f32; + } else { + for _ in 0..count { + let before = *entry; + *entry += 1.0; + if *entry == before { + break; + } + } + } + } + + pub fn insert(&mut self, token: Token<'x>) { *self.0.entry(token).or_insert(0.0) += 1.0; } - fn insert_if_missing(&mut self, token: Token<'x>) { + pub fn insert_if_missing(&mut self, token: Token<'x>) { self.0.entry(token).or_insert(1.0); } - fn insert_email(&mut self, email: &'x Email, is_sender: bool) { + pub fn insert_email(&mut self, email: &'x Email, is_sender: bool) { if !email.address.is_empty() { if is_sender { self.insert_if_missing(Token::Sender { @@ -1281,19 +1632,19 @@ impl<'x> Tokens<'x> { } impl Token<'static> { - fn from_alphanumeric(s: &str) -> Self { + pub fn from_alphanumeric(s: &str) -> Self { let mut is_hex = true; let mut is_ascii = true; let mut digit_count = 0; - for ch in s.chars() { - match ch { - 'a'..='f' | 'A'..='F' => {} - '0'..='9' => { + for &byte in s.as_bytes() { + match byte { + b'a'..=b'f' | b'A'..=b'F' => {} + b'0'..=b'9' => { digit_count += 1; } _ => { - is_ascii &= ch.is_ascii(); + is_ascii &= byte.is_ascii(); is_hex = false; } } @@ -1306,7 +1657,6 @@ impl Token<'static> { } else if !is_ascii { let word: String = if let Ok(cured) = decancer::cure(s, decancer::Options::default()) { cured - .as_str() .chars() .filter(|ch| ch.is_alphabetic()) .take(MAX_TOKEN_LENGTH) @@ -1321,30 +1671,37 @@ impl Token<'static> { Token::Word { value: word.into() } } else if s.len() > 3 && digit_count == 1 { - let word: String = s - .chars() - .filter(|ch| ch.is_alphabetic()) - .flat_map(|ch| ch.to_lowercase()) - .take(MAX_TOKEN_LENGTH) - .collect(); + let mut word = String::with_capacity(MAX_TOKEN_LENGTH.min(s.len())); + for &byte in s.as_bytes() { + if byte.is_ascii_alphabetic() { + word.push(byte.to_ascii_lowercase() as char); + if word.len() == MAX_TOKEN_LENGTH { + break; + } + } + } Token::Word { value: word.into() } } else { // Character class counts let mut upper = 0u32; let mut lower = 0u32; let mut digit = 0u32; - let mut len = 0; - let mut char_types = Vec::with_capacity(len); - for c in s.chars() { - let char_type = CharType::from_char(c); - char_types.push(char_type); + let mut run_count = 0u32; + let mut previous = None; + let bytes = s.as_bytes(); + let len = bytes.len(); + for &byte in bytes { + let char_type = CharType::from_byte(byte); match char_type { CharType::Upper => upper += 1, CharType::Lower => lower += 1, CharType::Digit => digit += 1, CharType::Other => (), } - len += 1; + if previous.is_some_and(|previous| previous != char_type) { + run_count += 1; + } + previous = Some(char_type); } // Determine dominant composition @@ -1375,7 +1732,7 @@ impl Token<'static> { // Ratio encoding (which class dominates) let max_count = upper.max(lower).max(digit); - let dominance = (max_count * 100) / len.min(1) as u32; + let dominance = max_count * 100; let ratio = match dominance { 0..=50 => b'B', // Balanced 51..=75 => b'P', // Partial dominance @@ -1383,17 +1740,6 @@ impl Token<'static> { _ => b'O', // One class only (100%) }; - // Run code - let mut run_count = 0; - if len > 1 { - let mut prev_type = char_types[0]; - for ¤t_type in char_types.iter().skip(1) { - if current_type != prev_type { - run_count += 1; - prev_type = current_type; - } - } - } let run_ratio = (run_count as f64) / ((len - 1) as f64); let run_code = match run_ratio { r if r <= 0.1 => b'0', // Very long runs (e.g., AAAABBBB) @@ -1409,7 +1755,7 @@ impl Token<'static> { } } - fn from_number(is_float: bool, num: &str) -> Self { + pub fn from_number(is_float: bool, num: &str) -> Self { Token::Number { code: [ if num.starts_with("-") { @@ -1429,29 +1775,40 @@ impl Token<'static> { } } -fn lower_prefix(prefix: &str, value: &str) -> String { +pub fn lower_prefix(prefix: &str, value: &str) -> String { let mut result = String::with_capacity(prefix.len() + value.len()); result.push_str(prefix); - for ch in value.chars() { - for lower_ch in ch.to_lowercase() { - result.push(lower_ch); + if value.is_ascii() { + let start = result.len(); + result.push_str(value); + result[start..].make_ascii_lowercase(); + } else { + for ch in value.chars() { + for lower_ch in ch.to_lowercase() { + result.push(lower_ch); + } } } result } -fn truncate_word(word: &str, max_len: usize) -> &str { +#[inline(always)] +pub fn truncate_word(word: &str, max_len: usize) -> &str { if word.len() <= max_len { word } else { - let mut pos = 0; - for (count, (idx, _)) in word.char_indices().enumerate() { - pos = idx; - if count == max_len { - break; - } + truncate_word_cold(word, max_len) + } +} + +#[inline(never)] +fn truncate_word_cold(word: &str, max_len: usize) -> &str { + match word.char_indices().nth(max_len) { + Some((idx, _)) => &word[..idx], + None => { + let last = word.char_indices().next_back().map_or(0, |(idx, _)| idx); + &word[..last] } - &word[..pos] } } @@ -1502,11 +1859,12 @@ enum CharType { } impl CharType { - fn from_char(c: char) -> CharType { - match c { - 'A'..='Z' => CharType::Upper, - 'a'..='z' => CharType::Lower, - '0'..='9' => CharType::Digit, + #[inline(always)] + fn from_byte(byte: u8) -> CharType { + match byte { + b'A'..=b'Z' => CharType::Upper, + b'a'..=b'z' => CharType::Lower, + b'0'..=b'9' => CharType::Digit, _ => CharType::Other, } } @@ -1514,6 +1872,11 @@ impl CharType { impl<'x> Default for Tokens<'x> { fn default() -> Self { - Tokens(HashMap::with_capacity(128)) + Tokens(HashMap::with_capacity_and_hasher(128, RandomState::new())) } } + +impl ArchiveCompression for SpamTrainer { + const COMPRESSION: Compression = Compression::Zstd(None); +} + diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index f0b83a7e..4b6b2eca 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -8,33 +8,44 @@ use super::expression::SpamFilterResolver; use crate::SpamFilterContext; use common::{ Server, - config::mailstore::spamfilter::{DnsBlServer, Element, IpResolver, Location}, - expr::functions::ResolveVariable, + config::mailstore::spamfilter::{DnsBlServer, Element, IpResolver, Location, SpamLookup}, + expr::{Bump, Variable, functions::ResolveVariable}, }; -use mail_auth::common::resolver::ToFqdn; -#[cfg(not(feature = "test_mode"))] +use compact_str::{CompactString, ToCompactString}; +use mail_auth::dns::ToFqdn; +#[cfg(not(stalwart_tests))] use mail_auth::hickory_resolver::{ net::{DnsError, NetError}, proto::rr::{Name, RData}, }; -#[cfg(feature = "test_mode")] -use mail_auth::{DnsError, Error}; use std::{ - net::Ipv4Addr, sync::Arc, time::{Duration, Instant}, }; +use store::xxhash_rust::xxh3::xxh3_64; use trc::SpamEvent; +#[cfg(stalwart_tests)] +use test_support::resolve_zone; + const MAX_NEGATIVE_TTL: u32 = 3600; +const EXPIRY_SPREAD_STEPS: u32 = 1024; +const EXPIRY_SPREAD_DIVISOR: u32 = 10; +const MIN_CACHE_LIFETIME: Duration = Duration::from_secs(1); +const FAILURE_CACHE_LIFETIME: Duration = Duration::from_secs(60); + +enum Zone { + Cached(SpamLookup), + Lookup(Box), +} enum DnsblAnswer { Listed { - ips: Vec, + codes: Arc<[IpResolver]>, expires: Instant, }, NotListed { - expires: Option, + expires: Instant, }, } @@ -65,33 +76,47 @@ pub(crate) async fn check_dnsbl( Element::Header | Element::Body | Element::Any => unreachable!(), }; + let mut arena = std::mem::take(ctx.arena.get_mut()); for dnsbl in &server.core.spam.dnsbl.servers { if dnsbl.scope == scope && checks < max_checks && let Some(codes) = dnsbl_codes( server, dnsbl, - SpamFilterResolver::new(ctx, resolver, location), + SpamFilterResolver::new( + &ctx.input, + &ctx.output, + &ctx.result.tags, + resolver, + location, + ), scope, &mut checks, + &mut arena, ) .await { for code in codes.iter() { - let tag = server - .eval_if::( - &dnsbl.tags, - &SpamFilterResolver::new(ctx, code, location), - ctx.input.span_id, - ) - .await; - - if let Some(tag) = tag { + if let Some(tag) = dnsbl_tag( + server, + dnsbl, + &SpamFilterResolver::new( + &ctx.input, + &ctx.output, + &ctx.result.tags, + code, + location, + ), + &mut arena, + ) + .await + { ctx.result.add_tag(tag); } } } } + *ctx.arena.get_mut() = arena; match scope { Element::Email => ctx.result.rbl_email_checks = checks, @@ -108,13 +133,29 @@ async fn dnsbl_codes( resolver: SpamFilterResolver<'_, impl ResolveVariable>, element: Element, checks: &mut usize, + arena: &mut Bump, ) -> Option> { let time = Instant::now(); + let span_id = resolver.input.span_id; + let spam_lookup = &server.inner.cache.spam_lookup; let zone = server - .eval_if::(&config.zone, &resolver, resolver.ctx.input.span_id) - .await?; + .eval_if_with(&config.zone, &resolver, arena, span_id, |zone| match zone { + Variable::String(zone) => Some(match spam_lookup.get(zone) { + Some(entry) => Zone::Cached(entry), + None => Zone::Lookup(zone.into()), + }), + _ => None, + }) + .await + .flatten()?; - #[cfg(feature = "test_mode")] + let zone = match zone { + Zone::Cached(SpamLookup::Dnsbl(codes)) => return Some(codes), + Zone::Cached(_) => return None, + Zone::Lookup(zone) => zone, + }; + + #[cfg(stalwart_tests)] { if zone.contains(".11.20.") { let parts = zone.split('.').collect::>(); @@ -130,54 +171,43 @@ async fn dnsbl_codes( } } - if let Some(codes) = server.inner.cache.dns_rbl.get(zone.as_str()) { - return codes; - } + let guard = match spam_lookup.get_value_or_guard_async(&zone).await { + Ok(SpamLookup::Dnsbl(codes)) => return Some(codes), + Ok(_) => return None, + Err(guard) => guard, + }; *checks += 1; match resolve_zone(server, zone.to_fqdn().as_ref()).await { - Ok(DnsblAnswer::Listed { ips, expires }) => { + Ok(DnsblAnswer::Listed { codes, expires }) => { + let expires = cache_expiry(&zone, time, expires, Instant::now()); trc::event!( Spam(SpamEvent::Dnsbl), - Hostname = zone.clone(), - Result = ips + Hostname = zone, + Result = codes .iter() - .map(|ip| trc::Value::from(ip.to_string())) + .map(|code| trc::Value::from(CompactString::from(code.text()))) .collect::>(), Details = element.as_str(), Elapsed = time.elapsed() ); - let codes: Arc<[IpResolver]> = ips - .into_iter() - .map(|ip| IpResolver::new(ip.into())) - .collect(); - - server.inner.cache.dns_rbl.insert_with_expiry( - zone.into(), - Some(codes.clone()), - expires, - ); + guard.insert_with_expiry(SpamLookup::Dnsbl(codes.clone()), expires); Some(codes) } Ok(DnsblAnswer::NotListed { expires }) => { + let expires = cache_expiry(&zone, time, expires, Instant::now()); trc::event!( Spam(SpamEvent::Dnsbl), - Hostname = zone.clone(), + Hostname = zone, Result = trc::Value::None, Details = element.as_str(), Elapsed = time.elapsed() ); - if let Some(expires) = expires { - server - .inner - .cache - .dns_rbl - .insert_with_expiry(zone.into(), None, expires); - } + guard.insert_with_expiry(SpamLookup::None, expires); None } @@ -190,55 +220,85 @@ async fn dnsbl_codes( CausedBy = err ); + guard.insert(SpamLookup::None, FAILURE_CACHE_LIFETIME); + None } } } -#[cfg(not(feature = "test_mode"))] -async fn resolve_zone(server: &Server, zone: &str) -> Result { - let name = Name::from_str_relaxed(zone).map_err(|err| err.to_string())?; +fn cache_expiry(zone: &str, start: Instant, expires: Instant, now: Instant) -> Instant { + spread_expiry(zone, start, expires).max(now + MIN_CACHE_LIFETIME) +} - match server.core.smtp.resolvers.dns.0.ipv4_lookup(name).await { +fn negative_expiry(ttl: Option) -> Instant { + Instant::now() + Duration::from_secs(ttl.unwrap_or_default().min(MAX_NEGATIVE_TTL).into()) +} + +fn spread_expiry(zone: &str, start: Instant, expires: Instant) -> Instant { + let step = (xxh3_64(zone.as_bytes()) % u64::from(EXPIRY_SPREAD_STEPS)) as u32; + expires + - expires.saturating_duration_since(start) / EXPIRY_SPREAD_DIVISOR * step + / EXPIRY_SPREAD_STEPS +} + +#[cfg(not(stalwart_tests))] +async fn resolve_zone(server: &Server, zone: &str) -> Result { + let name = Name::from_str_relaxed(zone).map_err(|err| err.to_compact_string())?; + + match server + .core + .smtp + .resolvers + .dns + .resolver() + .ipv4_lookup(name) + .await + { Ok(lookup) => { let expires = lookup.valid_until(); - let ips = lookup + let codes = lookup .answers() .iter() .filter_map(|record| match &record.data { - RData::A(a) => Some(a.0), + RData::A(a) => Some(IpResolver::new(a.0.into())), _ => None, }) - .collect::>(); + .collect::>(); - Ok(if !ips.is_empty() { - DnsblAnswer::Listed { ips, expires } + Ok(if !codes.is_empty() { + DnsblAnswer::Listed { codes, expires } } else { - DnsblAnswer::NotListed { - expires: Some(expires), - } + DnsblAnswer::NotListed { expires } }) } Err(NetError::Dns(DnsError::NoRecordsFound(no_records))) => Ok(DnsblAnswer::NotListed { - expires: no_records - .negative_ttl - .filter(|ttl| *ttl > 0) - .map(|ttl| Instant::now() + Duration::from_secs(ttl.min(MAX_NEGATIVE_TTL).into())), + expires: negative_expiry(no_records.negative_ttl), }), - Err(err) => Err(err.to_string()), + Err(err) => Err(err.to_compact_string()), } } -#[cfg(feature = "test_mode")] -async fn resolve_zone(server: &Server, zone: &str) -> Result { - match server.core.smtp.resolvers.dns.ipv4_lookup_raw(zone).await { - Ok(result) => Ok(DnsblAnswer::Listed { - ips: result.entry.to_vec(), - expires: result.expires, - }), - Err(Error::Dns(DnsError::RecordNotFound(_))) => Ok(DnsblAnswer::NotListed { - expires: Some(Instant::now() + Duration::from_secs(MAX_NEGATIVE_TTL.into())), - }), - Err(err) => Err(err.to_string()), - } +async fn dnsbl_tag( + server: &Server, + config: &DnsBlServer, + resolver: &SpamFilterResolver<'_, impl ResolveVariable>, + arena: &mut Bump, +) -> Option { + let tags = resolver.tags; + server + .eval_if_with( + &config.tags, + resolver, + arena, + resolver.input.span_id, + |tag| match tag { + Variable::String(tag) if !tags.contains(tag) => Some(CompactString::from(tag)), + _ => None, + }, + ) + .await + .flatten() } + + diff --git a/crates/spam-filter/src/modules/expression.rs b/crates/spam-filter/src/modules/expression.rs index c430a422..3bcb8388 100644 --- a/crates/spam-filter/src/modules/expression.rs +++ b/crates/spam-filter/src/modules/expression.rs @@ -4,400 +4,448 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - config::mailstore::spamfilter::*, - expr::{StringCow, Variable, functions::ResolveVariable}, +use crate::{ + ContextToken, Recipient, SpamFilterInput, SpamFilterOutput, TextPart, analysis::url::UrlParts, }; -use compact_str::{CompactString, ToCompactString, format_compact}; -use mail_parser::{Header, HeaderValue}; +use common::{ + config::mailstore::spamfilter::Location, + expr::{ + Bump, Variable, + bumpalo::collections::String as BumpString, + functions::ResolveVariable, + kernels::{to_lowercase, utf8_lossy}, + }, +}; +use compact_str::CompactString; +use mail_parser::{DateTime, Header, HeaderValue}; use nlp::tokenizers::types::TokenType; use registry::schema::enums::ExpressionVariable; +use std::fmt::Write; +use store::ahash::AHashSet; -use crate::{Recipient, SpamFilterContext, TextPart, analysis::url::UrlParts}; +pub(crate) const HEADER_NAME_INLINE: usize = 64; + +const RFC3339_LEN: usize = 25; +const IP_TEXT_MAX_LEN: usize = 39; pub(crate) struct SpamFilterResolver<'x, T: ResolveVariable> { - pub ctx: &'x SpamFilterContext<'x>, + pub input: &'x SpamFilterInput<'x>, + pub output: &'x SpamFilterOutput<'x>, + pub tags: &'x AHashSet, pub item: &'x T, pub location: Location, + pub scores: ScoreVariables, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ScoreVariables { + pub score: f32, + pub probability: f32, + pub threshold: f32, +} + +pub(crate) const NO_PROBABILITY: f32 = -1.0; + +impl Default for ScoreVariables { + fn default() -> Self { + ScoreVariables { + score: 0.0, + probability: NO_PROBABILITY, + threshold: 0.0, + } + } +} + +impl ScoreVariables { + pub fn for_rules(probability: Option) -> Self { + ScoreVariables { + probability: probability.unwrap_or(NO_PROBABILITY), + ..Default::default() + } + } +} + +pub(crate) struct EmailHeader<'h> { + pub header: Header<'h>, + pub name: &'h str, + pub name_lower: &'h str, +} + +pub struct StringResolver<'x>(pub &'x str); + +#[derive(Clone, Copy)] +enum RecipientPart { + Address, + Name, + Local, + Domain, } impl<'x, T: ResolveVariable> SpamFilterResolver<'x, T> { - pub fn new(ctx: &'x SpamFilterContext<'x>, item: &'x T, location: Location) -> Self { + pub fn new( + input: &'x SpamFilterInput<'x>, + output: &'x SpamFilterOutput<'x>, + tags: &'x AHashSet, + item: &'x T, + location: Location, + ) -> Self { Self { - ctx, + input, + output, + tags, item, location, + scores: ScoreVariables::default(), + } + } + + pub fn with_scores(mut self, scores: ScoreVariables) -> Self { + self.scores = scores; + self + } + + fn list<'a>(&'a self, variable: ExpressionVariable, arena: &'a Bump) -> &'a [Variable<'a>] { + let output = self.output; + match variable { + ExpressionVariable::To => { + recipients(arena, &output.recipients_to, RecipientPart::Address) + } + ExpressionVariable::ToName => { + recipients(arena, &output.recipients_to, RecipientPart::Name) + } + ExpressionVariable::ToLocal => { + recipients(arena, &output.recipients_to, RecipientPart::Local) + } + ExpressionVariable::ToDomain => { + recipients(arena, &output.recipients_to, RecipientPart::Domain) + } + ExpressionVariable::Cc => { + recipients(arena, &output.recipients_cc, RecipientPart::Address) + } + ExpressionVariable::CcName => { + recipients(arena, &output.recipients_cc, RecipientPart::Name) + } + ExpressionVariable::CcLocal => { + recipients(arena, &output.recipients_cc, RecipientPart::Local) + } + ExpressionVariable::CcDomain => { + recipients(arena, &output.recipients_cc, RecipientPart::Domain) + } + ExpressionVariable::Bcc => { + recipients(arena, &output.recipients_bcc, RecipientPart::Address) + } + ExpressionVariable::BccName => { + recipients(arena, &output.recipients_bcc, RecipientPart::Name) + } + ExpressionVariable::BccLocal => { + recipients(arena, &output.recipients_bcc, RecipientPart::Local) + } + ExpressionVariable::BccDomain => { + recipients(arena, &output.recipients_bcc, RecipientPart::Domain) + } + ExpressionVariable::EnvTo => strs( + arena, + output.env_to_orig_addr.iter().map(|e| e.address.as_str()), + ), + ExpressionVariable::SubjectWords => words(arena, &output.subject_tokens), + ExpressionVariable::BodyWords => self + .input + .message + .html_body() + .next() + .and_then(|part| output.text_parts.get(part.id() as usize)) + .map(|part| match part { + TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => { + words(arena, tokens) + } + TextPart::None => &[], + }) + .unwrap_or_default(), + _ => &[], } } } impl ResolveVariable for SpamFilterResolver<'_, T> { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { + let (input, output) = (self.input, self.output); match variable { - ExpressionVariable::RemoteIp => self.ctx.input.remote_ip.to_compact_string().into(), - ExpressionVariable::RemoteIpPtr => self - .ctx - .output - .iprev_ptr - .as_deref() - .unwrap_or_default() - .into(), - ExpressionVariable::HeloDomain => self.ctx.output.ehlo_host.fqdn.as_str().into(), + ExpressionVariable::RemoteIp => { + let mut ip = BumpString::with_capacity_in(IP_TEXT_MAX_LEN, arena); + let _ = write!(ip, "{}", input.remote_ip); + ip.into_bump_str().into() + } + ExpressionVariable::RemoteIpPtr => { + output.iprev_ptr.as_deref().unwrap_or_default().into() + } + ExpressionVariable::HeloDomain => output.ehlo_host.fqdn.as_str().into(), ExpressionVariable::AuthenticatedAs => { - self.ctx.input.authenticated_as.unwrap_or_default().into() + input.authenticated_as.unwrap_or_default().into() } - ExpressionVariable::Asn => self.ctx.input.asn.unwrap_or_default().into(), - ExpressionVariable::Country => self.ctx.input.country.unwrap_or_default().into(), - ExpressionVariable::IsTls => self.ctx.input.is_tls.into(), - ExpressionVariable::EnvFrom => self.ctx.output.env_from_addr.address.as_str().into(), - ExpressionVariable::EnvFromLocal => { - self.ctx.output.env_from_addr.local_part.as_str().into() + ExpressionVariable::Asn => input.asn.unwrap_or_default().into(), + ExpressionVariable::Country => input.country.unwrap_or_default().into(), + ExpressionVariable::IsTls => input.is_tls.into(), + ExpressionVariable::EnvFrom => output.env_from_addr.address.as_str().into(), + ExpressionVariable::EnvFromLocal => output.env_from_addr.local_part.as_str().into(), + ExpressionVariable::EnvFromDomain => { + output.env_from_addr.domain_part.fqdn.as_str().into() } - ExpressionVariable::EnvFromDomain => self - .ctx - .output - .env_from_addr - .domain_part - .fqdn - .as_str() - .into(), - ExpressionVariable::EnvTo => self - .ctx - .output - .env_to_orig_addr - .iter() - .map(|e| Variable::from(e.address.as_str())) - .collect::>() - .into(), - ExpressionVariable::From => self.ctx.output.from.email.address.as_str().into(), - ExpressionVariable::FromName => self - .ctx - .output - .from - .name - .as_deref() - .unwrap_or_default() - .into(), - ExpressionVariable::FromLocal => self.ctx.output.from.email.local_part.as_str().into(), - ExpressionVariable::FromDomain => { - self.ctx.output.from.email.domain_part.fqdn.as_str().into() - } - ExpressionVariable::ReplyTo => self - .ctx - .output + ExpressionVariable::From => output.from.email.address.as_str().into(), + ExpressionVariable::FromName => output.from.name.as_deref().unwrap_or_default().into(), + ExpressionVariable::FromLocal => output.from.email.local_part.as_str().into(), + ExpressionVariable::FromDomain => output.from.email.domain_part.fqdn.as_str().into(), + ExpressionVariable::ReplyTo => output .reply_to .as_ref() .map(|r| r.email.address.as_str()) .unwrap_or_default() .into(), - ExpressionVariable::ReplyToName => self - .ctx - .output + ExpressionVariable::ReplyToName => output .reply_to .as_ref() .and_then(|r| r.name.as_deref()) .unwrap_or_default() .into(), - ExpressionVariable::ReplyToLocal => self - .ctx - .output + ExpressionVariable::ReplyToLocal => output .reply_to .as_ref() .map(|r| r.email.local_part.as_str()) .unwrap_or_default() .into(), - ExpressionVariable::ReplyToDomain => self - .ctx - .output + ExpressionVariable::ReplyToDomain => output .reply_to .as_ref() .map(|r| r.email.domain_part.fqdn.as_str()) .unwrap_or_default() .into(), - ExpressionVariable::To => self - .ctx - .output - .recipients_to - .iter() - .map(|r| Variable::from(r.email.address.as_str())) - .collect::>() - .into(), - ExpressionVariable::ToName => self - .ctx - .output - .recipients_to - .iter() - .filter_map(|r| Variable::from(r.name.as_deref()?).into()) - .collect::>() - .into(), - ExpressionVariable::ToLocal => self - .ctx - .output - .recipients_to - .iter() - .map(|r| Variable::from(r.email.local_part.as_str())) - .collect::>() - .into(), - ExpressionVariable::ToDomain => self - .ctx - .output - .recipients_to - .iter() - .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) - .collect::>() - .into(), - ExpressionVariable::Cc => self - .ctx - .output - .recipients_cc - .iter() - .map(|r| Variable::from(r.email.address.as_str())) - .collect::>() - .into(), - ExpressionVariable::CcName => self - .ctx - .output - .recipients_cc - .iter() - .filter_map(|r| Variable::from(r.name.as_deref()?).into()) - .collect::>() - .into(), - ExpressionVariable::CcLocal => self - .ctx - .output - .recipients_cc - .iter() - .map(|r| Variable::from(r.email.local_part.as_str())) - .collect::>() - .into(), - ExpressionVariable::CcDomain => self - .ctx - .output - .recipients_cc - .iter() - .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) - .collect::>() - .into(), - ExpressionVariable::Bcc => self - .ctx - .output - .recipients_bcc - .iter() - .map(|r| Variable::from(r.email.address.as_str())) - .collect::>() - .into(), - ExpressionVariable::BccName => self - .ctx - .output - .recipients_bcc - .iter() - .filter_map(|r| Variable::from(r.name.as_deref()?).into()) - .collect::>() - .into(), - ExpressionVariable::BccLocal => self - .ctx - .output - .recipients_bcc - .iter() - .map(|r| Variable::from(r.email.local_part.as_str())) - .collect::>() - .into(), - ExpressionVariable::BccDomain => self - .ctx - .output - .recipients_bcc - .iter() - .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) - .collect::>() - .into(), + ExpressionVariable::EnvTo + | ExpressionVariable::To + | ExpressionVariable::ToName + | ExpressionVariable::ToLocal + | ExpressionVariable::ToDomain + | ExpressionVariable::Cc + | ExpressionVariable::CcName + | ExpressionVariable::CcLocal + | ExpressionVariable::CcDomain + | ExpressionVariable::Bcc + | ExpressionVariable::BccName + | ExpressionVariable::BccLocal + | ExpressionVariable::BccDomain + | ExpressionVariable::SubjectWords + | ExpressionVariable::BodyWords => Variable::Array(self.list(variable, arena)), ExpressionVariable::Body | ExpressionVariable::BodyText => { - self.ctx.text_body().unwrap_or_default().into() + output.text_body(input).unwrap_or_default().into() } - ExpressionVariable::BodyHtml => self - .ctx - .input + ExpressionVariable::BodyHtml => input .message - .html_body - .first() - .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize)) + .html_body() + .next() + .and_then(|part| output.text_parts.get(part.id() as usize)) .map(|part| { if let TextPart::Html { text_body, .. } = part { - text_body.as_str() + *text_body } else { "" } }) .unwrap_or_default() .into(), - ExpressionVariable::BodyRaw => Variable::from(CompactString::from_utf8_lossy( - self.ctx.input.message.raw_message(), - )), - ExpressionVariable::Subject => self.ctx.output.subject_lc.as_str().into(), - ExpressionVariable::SubjectThread => self.ctx.output.subject_thread_lc.as_str().into(), + ExpressionVariable::BodyRaw => utf8_lossy(input.message.raw(), arena).into(), + ExpressionVariable::Subject => output.subject_lc.as_str().into(), + ExpressionVariable::SubjectThread => output.subject_thread_lc.as_str().into(), ExpressionVariable::Location => self.location.as_str().into(), - ExpressionVariable::SubjectWords => self - .ctx - .output - .subject_tokens - .iter() - .filter_map(|w| match w { - TokenType::Alphabetic(w) - | TokenType::Alphanumeric(w) - | TokenType::Integer(w) - | TokenType::Float(w) => Some(Variable::from(w.as_ref())), - _ => None, - }) - .collect::>() - .into(), - ExpressionVariable::BodyWords => self - .ctx - .input - .message - .html_body - .first() - .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize)) - .map(|part| match part { - TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => tokens - .iter() - .filter_map(|w| match w { - TokenType::Alphabetic(w) - | TokenType::Alphanumeric(w) - | TokenType::Integer(w) - | TokenType::Float(w) => Some(Variable::from(w.as_ref())), - _ => None, - }) - .collect::>(), - TextPart::None => vec![], - }) - .unwrap_or_default() - .into(), - variable => self.item.resolve_variable(variable), + ExpressionVariable::SpamScore => Variable::Float(f64::from(self.scores.score)), + ExpressionVariable::SpamProbability => { + Variable::Float(f64::from(self.scores.probability)) + } + ExpressionVariable::SpamThreshold => Variable::Float(f64::from(self.scores.threshold)), + variable => self.item.resolve_variable(variable, arena), } } - fn resolve_global(&self, variable: &str) -> Variable<'_> { - Variable::Integer(self.ctx.result.tags.contains(variable).into()) + fn resolve_global<'a>(&'a self, variable: &str, _: &'a Bump) -> Variable<'a> { + Variable::Integer(self.tags.contains(variable).into()) } } -pub(crate) struct EmailHeader<'x> { - pub header: &'x Header<'x>, - pub raw: &'x str, -} - impl ResolveVariable for EmailHeader<'_> { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { - ExpressionVariable::Name => self.header.name().into(), - ExpressionVariable::NameLower => { - CompactString::from_str_to_lowercase(self.header.name()).into() - } + ExpressionVariable::Name => self.name.into(), + ExpressionVariable::NameLower => self.name_lower.into(), ExpressionVariable::Value | ExpressionVariable::ValueLower - | ExpressionVariable::Attributes => match &self.header.value { - HeaderValue::Text(text) => { - if variable == ExpressionVariable::ValueLower { - CompactString::from_str_to_lowercase(text).into() - } else { - text.as_ref().into() - } - } - HeaderValue::TextList(list) => Variable::Array( - list.iter() - .map(|text| { - Variable::String(if variable == ExpressionVariable::ValueLower { - StringCow::Owned(CompactString::from_str_to_lowercase(text)) - } else { - StringCow::Borrowed(text.as_ref()) - }) - }) - .collect(), - ), - HeaderValue::Address(address) => { - Variable::Array(if matches!(variable, ExpressionVariable::ValueLower) { - address - .iter() - .filter_map(|a| { - a.address.as_ref().map(|text| { - Variable::String( - if variable == ExpressionVariable::ValueLower { - StringCow::Owned(CompactString::from_str_to_lowercase( - text, - )) - } else { - StringCow::Borrowed(text.as_ref()) - }, - ) - }) - }) - .collect() - } else { - address - .iter() - .filter_map(|a| { - a.name.as_ref().map(|text| { - Variable::String( - if variable == ExpressionVariable::ValueLower { - StringCow::Owned(CompactString::from_str_to_lowercase( - text, - )) - } else { - StringCow::Borrowed(text.as_ref()) - }, - ) - }) - }) - .collect() - }) - } - HeaderValue::DateTime(date_time) => { - CompactString::new(date_time.to_rfc3339()).into() - } - HeaderValue::ContentType(ct) => { - if variable != ExpressionVariable::Attributes { - if let Some(st) = ct.subtype() { - format_compact!("{}/{}", ct.ctype(), st).into() - } else { - ct.ctype().into() - } - } else { - Variable::Array( - ct.attributes() - .map(|attr| { - attr.iter() - .map(|attr| { - Variable::from(format_compact!( - "{}={}", attr.name, attr.value - )) - }) - .collect::>() - }) - .unwrap_or_default(), - ) - } - } - HeaderValue::Received(_) => { - if variable == ExpressionVariable::ValueLower { - CompactString::from_str_to_lowercase(self.raw.trim()).into() - } else { - self.raw.trim().into() - } - } - HeaderValue::Empty => "".into(), - }, - ExpressionVariable::Raw => self.raw.into(), - ExpressionVariable::RawLower => CompactString::from_str_to_lowercase(self.raw).into(), + | ExpressionVariable::Attributes => header_value(self.header, variable, arena), + ExpressionVariable::Raw => utf8_lossy(self.header.raw_value(), arena).into(), + ExpressionVariable::RawLower => { + to_lowercase(utf8_lossy(self.header.raw_value(), arena), arena).into() + } _ => Variable::Integer(0), } } +} - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) +pub(crate) fn lower_header_name<'b>( + name: &'b str, + inline: &'b mut [u8; HEADER_NAME_INLINE], + arena: &'b Bump, +) -> &'b str { + if name.is_ascii() + && name.bytes().any(|b| b.is_ascii_uppercase()) + && let Some(out) = inline.get_mut(..name.len()) + { + out.copy_from_slice(name.as_bytes()); + out.make_ascii_lowercase(); + if let Ok(lower) = std::str::from_utf8(out) { + return lower; + } + } + to_lowercase(name, arena) +} + +fn rfc3339<'a>(date: &DateTime, arena: &'a Bump) -> &'a str { + let mut out = BumpString::with_capacity_in(RFC3339_LEN, arena); + let _ = write!( + out, + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + date.year, date.month, date.day, date.hour, date.minute, date.second + ); + let _ = if date.tz_hour != 0 || date.tz_minute != 0 { + let sign = if date.tz_before_gmt { '-' } else { '+' }; + write!(out, "{sign}{:02}:{:02}", date.tz_hour, date.tz_minute) + } else { + out.write_char('Z') + }; + out.into_bump_str() +} + +fn header_value<'a>( + header: Header<'a>, + variable: ExpressionVariable, + arena: &'a Bump, +) -> Variable<'a> { + let lower = variable == ExpressionVariable::ValueLower; + let text = |text: &'a str| -> &'a str { + if lower { + to_lowercase(text, arena) + } else { + text + } + }; + match header.value() { + HeaderValue::Text(value) => text(value).into(), + HeaderValue::TextList(list) if list.len() == 1 => { + text(list.first().unwrap_or_default()).into() + } + HeaderValue::TextList(list) => Variable::Array(strs(arena, list.iter().map(&text))), + HeaderValue::Address(address) => { + Variable::Array(if variable != ExpressionVariable::Attributes { + let addresses = || address.mailboxes().filter_map(|mailbox| mailbox.address()); + fill_strs(arena, addresses().count(), addresses().map(&text)) + } else { + filtered_strs(arena, || { + address.mailboxes().filter_map(|mailbox| mailbox.name()) + }) + }) + } + HeaderValue::DateTime(date_time) => Variable::String(rfc3339(&date_time, arena)), + HeaderValue::ContentType(ct) => { + if variable != ExpressionVariable::Attributes { + if let Some(st) = ct.subtype() { + let mut out = + BumpString::with_capacity_in(ct.ctype().len() + st.len() + 1, arena); + out.push_str(ct.ctype()); + out.push('/'); + out.push_str(st); + Variable::String(out.into_bump_str()) + } else { + ct.ctype().into() + } + } else { + Variable::Array(arena.alloc_slice_fill_iter(ct.attributes().map( + |(name, value)| { + let mut out = + BumpString::with_capacity_in(name.len() + value.len() + 1, arena); + out.push_str(name); + out.push('='); + out.push_str(value); + Variable::String(out.into_bump_str()) + }, + ))) + } + } + HeaderValue::Received(_) => text(utf8_lossy(header.raw_value(), arena).trim()).into(), + HeaderValue::Empty => "".into(), + } +} + +fn strs<'a, I>(arena: &'a Bump, items: I) -> &'a [Variable<'a>] +where + I: IntoIterator, + I::IntoIter: ExactSizeIterator, +{ + arena.alloc_slice_fill_iter(items.into_iter().map(Variable::String)) +} + +fn filtered_strs<'a, I>(arena: &'a Bump, items: impl Fn() -> I) -> &'a [Variable<'a>] +where + I: Iterator, +{ + fill_strs(arena, items().count(), items()) +} + +fn fill_strs<'a>( + arena: &'a Bump, + len: usize, + items: impl Iterator, +) -> &'a [Variable<'a>] { + let out = arena.alloc_slice_fill_copy(len, Variable::default()); + for (slot, item) in out.iter_mut().zip(items) { + *slot = Variable::String(item); + } + out +} + +fn words<'a>(arena: &'a Bump, tokens: &'a [ContextToken<'_>]) -> &'a [Variable<'a>] { + filtered_strs(arena, || { + tokens.iter().filter_map(|token| match token { + TokenType::Alphabetic(word) + | TokenType::Alphanumeric(word) + | TokenType::Integer(word) + | TokenType::Float(word) => Some(word.as_ref()), + _ => None, + }) + }) +} + +fn recipients<'a>( + arena: &'a Bump, + list: &'a [Recipient], + part: RecipientPart, +) -> &'a [Variable<'a>] { + match part { + RecipientPart::Address => strs(arena, list.iter().map(|r| r.email.address.as_str())), + RecipientPart::Name => { + filtered_strs(arena, || list.iter().filter_map(|r| r.name.as_deref())) + } + RecipientPart::Local => strs(arena, list.iter().map(|r| r.email.local_part.as_str())), + RecipientPart::Domain => strs( + arena, + list.iter().map(|r| r.email.domain_part.fqdn.as_str()), + ), } } impl ResolveVariable for Recipient { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>(&'a self, variable: ExpressionVariable, _: &'a Bump) -> Variable<'a> { match variable { ExpressionVariable::Email | ExpressionVariable::Value => { Variable::from(self.email.address.as_str()) @@ -409,14 +457,10 @@ impl ResolveVariable for Recipient { _ => Variable::Integer(0), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } impl ResolveVariable for UrlParts<'_> { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>(&'a self, variable: ExpressionVariable, _: &'a Bump) -> Variable<'a> { match variable { ExpressionVariable::Url | ExpressionVariable::Value => { Variable::from(self.url.as_str()) @@ -473,32 +517,11 @@ impl ResolveVariable for UrlParts<'_> { _ => Variable::Integer(0), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } -pub struct StringResolver<'x>(pub &'x str); - impl ResolveVariable for StringResolver<'_> { - fn resolve_variable(&self, _: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>(&'a self, _: ExpressionVariable, _: &'a Bump) -> Variable<'a> { Variable::from(self.0) } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } -pub struct StringListResolver<'x>(pub &'x [String]); - -impl ResolveVariable for StringListResolver<'_> { - fn resolve_variable(&self, _: ExpressionVariable) -> Variable<'_> { - Variable::Array(self.0.iter().map(|v| Variable::from(v.as_str())).collect()) - } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } -} diff --git a/crates/spam-filter/src/modules/html.rs b/crates/spam-filter/src/modules/html.rs index 3e9431c6..4d469a2e 100644 --- a/crates/spam-filter/src/modules/html.rs +++ b/crates/spam-filter/src/modules/html.rs @@ -4,24 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::scripts::functions::unicode::CharUtils; use mail_parser::decoders::html::add_html_token; +use memchr::{memchr_iter, memchr3_iter}; +use std::borrow::Cow; #[derive(Debug, Eq, PartialEq, Clone, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] -pub enum HtmlToken { +pub enum HtmlToken<'x> { StartTag { name: u64, - attributes: Vec<(u64, Option)>, + attributes: Vec<(u64, Option>)>, is_self_closing: bool, }, EndTag { name: u64, }, Comment { - text: String, + text: Cow<'x, str>, }, Text { - text: String, + text: Cow<'x, str>, }, } @@ -73,10 +76,464 @@ pub(crate) const HTTP_EQUIV: u64 = (b'h' as u64) | ((b'q' as u64) << 48) | ((b'u' as u64) << 56); -pub fn html_to_tokens(input: &str) -> Vec { - let input = input.as_bytes(); - let mut iter = input.iter().enumerate().peekable(); - let mut tags = vec![]; +const fn tag_name(name: &[u8]) -> u64 { + let mut key = 0u64; + let mut index = 0; + while index < name.len() && index < 8 { + key |= (name[index] as u64) << (8 * index); + index += 1; + } + key +} + +const STYLE: u64 = tag_name(b"style"); +const CLASS: u64 = tag_name(b"class"); +const HIDDEN: u64 = tag_name(b"hidden"); +const SCRIPT: u64 = tag_name(b"script"); +const TEMPLATE: u64 = tag_name(b"template"); +const NOSCRIPT: u64 = tag_name(b"noscript"); +const BLOCKQUOTE: u64 = tag_name(b"blockquote"); +const BR: u64 = tag_name(b"br"); +const HR: u64 = tag_name(b"hr"); +const INPUT: u64 = tag_name(b"input"); +const WBR: u64 = tag_name(b"wbr"); +const AREA: u64 = tag_name(b"area"); +const BASE: u64 = tag_name(b"base"); +const COL: u64 = tag_name(b"col"); +const EMBED: u64 = tag_name(b"embed"); +const SOURCE: u64 = tag_name(b"source"); +const TRACK: u64 = tag_name(b"track"); +const PARAM: u64 = tag_name(b"param"); + +const fn is_void_tag(name: u64) -> bool { + matches!( + name, + BR | IMG + | HR + | INPUT + | META + | LINK + | WBR + | AREA + | BASE + | COL + | EMBED + | SOURCE + | TRACK + | PARAM + ) +} + +const fn is_non_content_tag(name: u64) -> bool { + matches!(name, HEAD | STYLE | SCRIPT | TITLE | TEMPLATE | NOSCRIPT) +} + +pub(crate) const fn byte_table(bytes: &[u8]) -> [bool; 256] { + let mut table = [false; 256]; + let mut index = 0; + while index < bytes.len() { + table[bytes[index] as usize] = true; + index += 1; + } + table +} + +pub(crate) static TEXT_STOP: [bool; 256] = byte_table(b"<&; \t\r\n"); +static COMMENT_SPACE: [bool; 256] = byte_table(b" \t\r\n"); +static VALUE_STOP: [bool; 256] = byte_table(b">\" \t\r\n"); +static TAG_KEY_STOP: [bool; 256] = key_stop_table(); +static TAG_KEY_FOLD: [u8; 256] = key_fold_table(); + +const REUSE_BUFFER_LEN: usize = 1024; + +const fn key_fold_table() -> [u8; 256] { + let mut table = [0u8; 256]; + let mut index = 0; + while index < 256 { + let ch = index as u8; + table[index] = if ch.is_ascii_uppercase() { + ch - b'A' + b'a' + } else { + ch + }; + index += 1; + } + table +} + +const fn key_stop_table() -> [bool; 256] { + let mut table = [true; 256]; + let mut index = 0; + while index < 256 { + let ch = index as u8; + if ch.is_ascii_lowercase() + || ch.is_ascii_uppercase() + || ch.is_ascii_digit() + || ch == b'-' + || ch == b'_' + { + table[index] = false; + } + index += 1; + } + table +} + +#[inline(always)] +pub(crate) fn scan_until(bytes: &[u8], from: usize, stop: &[bool; 256]) -> usize { + match bytes.get(from..) { + Some(rest) => rest + .iter() + .position(|&ch| stop[ch as usize]) + .map_or(bytes.len(), |offset| from + offset), + None => bytes.len(), + } +} + +const SWAR_ONES: u64 = 0x0101_0101_0101_0101; +const SWAR_HIGH: u64 = 0x8080_8080_8080_8080; + +#[inline(always)] +fn scan_until_byte(bytes: &[u8], from: usize, needle: u8) -> usize { + let mut pos = from; + let broadcast = SWAR_ONES.wrapping_mul(needle as u64); + while let Some(chunk) = bytes.get(pos..).and_then(|rest| rest.first_chunk::<8>()) { + let word = u64::from_le_bytes(*chunk) ^ broadcast; + let found = word.wrapping_sub(SWAR_ONES) & !word & SWAR_HIGH; + if found != 0 { + return pos + (found.trailing_zeros() / 8) as usize; + } + pos += 8; + } + match bytes.get(pos..) { + Some(rest) => rest + .iter() + .position(|&ch| ch == needle) + .map_or(bytes.len(), |offset| pos + offset), + None => bytes.len(), + } +} + +#[inline(always)] +fn find_comment_end(bytes: &[u8], from: usize) -> Option { + let mut pos = from; + loop { + pos = scan_until_byte(bytes, pos, b'-'); + match bytes.get(pos..pos + 3) { + Some(b"-->") => return Some(pos + 2), + Some(_) => pos += 1, + None => return None, + } + } +} + +#[inline(always)] +pub(crate) fn push_span(text: &mut String, input: &str, start: usize, end: usize, add_space: bool) { + let span = &input.as_bytes()[start..end]; + if matches!(span, [b'&', .., b';']) { + add_html_token(text, span, add_space); + } else { + if add_space { + text.push(' '); + } + text.push_str(&input[start..end]); + } +} + +struct TextBuilder { + owned: String, + hint: usize, + start: usize, + end: usize, +} + +impl TextBuilder { + #[inline(always)] + fn new() -> Self { + Self { + owned: String::new(), + hint: 16, + start: 0, + end: 0, + } + } + + #[inline(always)] + fn is_empty(&self) -> bool { + self.end == self.start && self.owned.is_empty() + } + + #[inline(always)] + fn push(&mut self, input: &str, start: usize, end: usize, add_space: bool) { + let bytes = input.as_bytes(); + let span = &bytes[start..end]; + let is_entity = matches!(span, [b'&', .., b';']); + + if self.end > self.start { + if !is_entity { + if !add_space { + if start == self.end { + self.end = end; + return; + } + } else if start == self.end + 1 && bytes[self.end] == b' ' { + self.end = end; + return; + } + } + let pending_start = self.start; + let pending_end = self.end; + self.start = 0; + self.end = 0; + if self.owned.capacity() == 0 { + self.owned.reserve( + self.hint + .max((pending_end - pending_start) + (end - start) + 1), + ); + } + self.owned.push_str(&input[pending_start..pending_end]); + } else if self.owned.is_empty() { + if !is_entity { + if !add_space { + self.start = start; + self.end = end; + return; + } else if start > 0 && bytes[start - 1] == b' ' { + self.start = start - 1; + self.end = end; + return; + } + } + if self.owned.capacity() == 0 { + self.owned.reserve(self.hint.max((end - start) + 1)); + } + } + + if is_entity { + add_html_token(&mut self.owned, span, add_space); + } else { + if add_space { + self.owned.push(' '); + } + self.owned.push_str(&input[start..end]); + } + } + + #[inline(always)] + fn take<'x>(&mut self, input: &'x str) -> Cow<'x, str> { + if self.end > self.start { + let text = &input[self.start..self.end]; + self.start = 0; + self.end = 0; + Cow::Borrowed(text) + } else if self.owned.len() <= REUSE_BUFFER_LEN { + let text = String::from(self.owned.as_str()); + self.owned.clear(); + Cow::Owned(text) + } else { + self.hint = self.owned.len(); + Cow::Owned(std::mem::take(&mut self.owned)) + } + } +} + +fn parse_comment<'x>(input: &'x str, start: usize, tags: &mut Vec>) -> usize { + let bytes = input.as_bytes(); + let (end, is_terminated) = match find_comment_end(bytes, start + 1) { + Some(end) => (end, true), + None => (bytes.len(), false), + }; + + let mut pos = scan_until(bytes, start, &COMMENT_SPACE).min(end); + let text = if pos >= end { + Cow::Borrowed(&input[start..end]) + } else { + let mut comment = String::with_capacity(end - start); + comment.push_str(&input[start..pos]); + while pos < end { + comment.push(' '); + pos += 1; + let stop = scan_until(bytes, pos, &COMMENT_SPACE).min(end); + comment.push_str(&input[pos..stop]); + pos = stop; + } + Cow::Owned(comment) + }; + + tags.push(HtmlToken::Comment { text }); + + if is_terminated { end + 1 } else { end } +} + +fn parse_value<'x>( + input: &'x str, + mut pos: usize, + in_quote: &mut bool, + attributes: &mut Vec<(u64, Option>)>, +) -> (usize, bool) { + let bytes = input.as_bytes(); + let mut first_start = 0; + let mut first_end = 0; + let mut owned: Option = None; + let mut is_tag_end = false; + + loop { + let stop = if *in_quote { + scan_until_byte(bytes, pos, b'"') + } else { + scan_until(bytes, pos, &VALUE_STOP) + }; + if stop > pos { + if first_end == first_start { + first_start = pos; + first_end = stop; + } else { + match &mut owned { + Some(owned) => owned.push_str(&input[pos..stop]), + None => { + let mut value = + String::with_capacity((first_end - first_start) + (stop - pos)); + value.push_str(&input[first_start..first_end]); + value.push_str(&input[pos..stop]); + owned = Some(value); + } + } + } + } + pos = stop; + match bytes.get(pos) { + None => break, + Some(b'"') => { + pos += 1; + if *in_quote { + *in_quote = false; + break; + } + *in_quote = true; + } + Some(b'>') => { + pos += 1; + is_tag_end = true; + break; + } + Some(_) => { + pos += 1; + break; + } + } + } + + let value = match owned { + Some(owned) => Some(Cow::Owned(owned)), + None if first_end > first_start => Some(Cow::Borrowed(&input[first_start..first_end])), + None => None, + }; + if let Some(value) = value { + if let Some((_, slot)) = attributes.last_mut() { + *slot = Some(value); + } else { + attributes.push((0, Some(value))); + } + } + + (pos, is_tag_end) +} + +fn parse_tag<'x>(input: &'x str, mut pos: usize, tags: &mut Vec>) -> usize { + let bytes = input.as_bytes(); + let mut is_end_tag = false; + + while let Some(&ch) = bytes.get(pos) { + if ch == b'/' { + is_end_tag = true; + } else if !ch.is_ascii_whitespace() { + break; + } + pos += 1; + } + + let mut in_quote = false; + let mut is_self_closing = false; + let mut key: u64 = 0; + let mut shift = 0; + let mut tag = 0; + let mut attributes: Vec<(u64, Option>)> = Vec::new(); + + while let Some(&ch) = bytes.get(pos) { + if !TAG_KEY_STOP[ch as usize] { + if shift < 64 { + key |= (TAG_KEY_FOLD[ch as usize] as u64) << shift; + shift += 8; + } + pos += 1; + continue; + } + pos += 1; + match ch { + b'/' if !in_quote => { + is_self_closing = true; + } + b'>' if !in_quote => { + if shift != 0 { + if tag == 0 { + tag = key; + } else { + attributes.push((key, None)); + } + } + break; + } + b'"' => { + in_quote = !in_quote; + } + b'=' if !in_quote => { + while matches!(bytes.get(pos), Some(ch) if ch.is_ascii_whitespace()) { + pos += 1; + } + + if shift != 0 { + attributes.push((key, None)); + key = 0; + shift = 0; + } + + let (next, is_tag_end) = parse_value(input, pos, &mut in_quote, &mut attributes); + pos = next; + if is_tag_end { + break; + } + } + b' ' | b'\t' | b'\r' | b'\n' if shift != 0 => { + if tag == 0 { + tag = key; + } else { + attributes.push((key, None)); + } + key = 0; + shift = 0; + } + _ => {} + } + } + + if tag != 0 { + if is_end_tag { + tags.push(HtmlToken::EndTag { name: tag }); + } else { + tags.push(HtmlToken::StartTag { + name: tag, + attributes, + is_self_closing, + }); + } + } + + pos +} + +pub fn html_to_tokens(input: &str) -> Vec> { + let bytes = input.as_bytes(); + let mut tags = Vec::new(); let mut is_token_start = true; let mut is_after_space = false; @@ -85,212 +542,58 @@ pub fn html_to_tokens(input: &str) -> Vec { let mut token_start = 0; let mut token_end = 0; - let mut text = String::with_capacity(16); + let mut text = TextBuilder::new(); + let mut pos = 0; - while let Some((mut pos, &ch)) = iter.next() { + while let Some(&ch) = bytes.get(pos) { match ch { b'<' => { if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space, - ); + text.push(input, token_start, token_end + 1, is_after_space); is_after_space = false; is_token_start = true; } if !text.is_empty() { tags.push(HtmlToken::Text { - text: text.as_str().into(), + text: text.take(input), }); - text.clear(); } - while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace()) { + pos += 1; + while matches!(bytes.get(pos), Some(ch) if ch.is_ascii_whitespace()) { pos += 1; - iter.next(); } - if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { - let mut comment = Vec::new(); - let mut last_ch: u8 = 0; - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if comment.len() > 2 - && matches!(comment.last(), Some(b'-')) - && matches!(comment.get(comment.len() - 2), Some(b'-')) => - { - break; - } - b' ' | b'\t' | b'\r' | b'\n' => { - if last_ch != b' ' { - comment.push(b' '); - } else { - last_ch = b' '; - } - continue; - } - _ => { - comment.push(ch); - } - } - last_ch = ch; - } - tags.push(HtmlToken::Comment { - text: String::from_utf8(comment).unwrap_or_default(), - }); + pos = if matches!(bytes.get(pos..pos + 3), Some(b"!--")) { + parse_comment(input, pos, &mut tags) } else { - let mut is_end_tag = false; - loop { - match iter.peek() { - Some(&(_, &b'/')) => { - is_end_tag = true; - //pos += 1; - iter.next(); - } - Some((_, ch)) if ch.is_ascii_whitespace() => { - //pos += 1; - iter.next(); - } - _ => break, - } - } - - let mut in_quote = false; - let mut is_self_closing = false; - - let mut key: u64 = 0; - let mut shift = 0; - - let mut tag = 0; - let mut attributes: Vec<(u64, Option)> = vec![]; - - 'outer: while let Some((_, &ch)) = iter.next() { - match ch { - b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' if shift < 64 => { - key |= (ch as u64) << shift; - shift += 8; - } - b'A'..=b'Z' if shift < 64 => { - key |= ((ch - b'A' + b'a') as u64) << shift; - shift += 8; - } - b'/' if !in_quote => { - is_self_closing = true; - } - b'>' if !in_quote => { - if shift != 0 { - if tag == 0 { - tag = key; - } else { - attributes.push((key, None)); - } - } - break; - } - b'"' => { - in_quote = !in_quote; - } - b'=' if !in_quote => { - while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace()) - { - iter.next(); - } - - if shift != 0 { - attributes.push((key, None)); - key = 0; - shift = 0; - } - - let mut value = vec![]; - - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if !in_quote => { - if !value.is_empty() { - let value = - String::from_utf8(value).unwrap_or_default(); - if let Some((_, v)) = attributes.last_mut() { - *v = value.into(); - } else { - // Broken attribute - attributes.push((0, Some(value))); - } - } - break 'outer; - } - b'"' => { - if in_quote { - in_quote = false; - break; - } else { - in_quote = true; - } - } - b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { - break; - } - _ => { - value.push(ch); - } - } - } - - if !value.is_empty() { - let value = String::from_utf8(value).unwrap_or_default(); - if let Some((_, v)) = attributes.last_mut() { - *v = value.into(); - } else { - // Broken attribute - attributes.push((0, Some(value))); - } - } - } - b' ' | b'\t' | b'\r' | b'\n' if shift != 0 => { - if tag == 0 { - tag = key; - } else { - attributes.push((key, None)); - } - key = 0; - shift = 0; - } - _ => {} - } - } - - if tag != 0 { - if is_end_tag { - tags.push(HtmlToken::EndTag { name: tag }); - } else { - tags.push(HtmlToken::StartTag { - name: tag, - attributes, - is_self_closing, - }); - } - } - } + parse_tag(input, pos, &mut tags) + }; continue; } b' ' | b'\t' | b'\r' | b'\n' => { if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], + text.push( + input, + token_start, + token_end + 1, is_after_space && !is_new_line, ); is_new_line = false; } is_after_space = true; is_token_start = true; + pos += 1; + while matches!(bytes.get(pos), Some(b' ' | b'\t' | b'\r' | b'\n')) { + pos += 1; + } continue; } b'&' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..token_end + 1], + text.push( + input, + token_start, + token_end + 1, is_after_space && !is_new_line, ); is_new_line = false; @@ -298,14 +601,11 @@ pub fn html_to_tokens(input: &str) -> Vec { is_after_space = false; } b';' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..pos + 1], - is_after_space && !is_new_line, - ); + text.push(input, token_start, pos + 1, is_after_space && !is_new_line); is_token_start = true; is_after_space = false; is_new_line = false; + pos += 1; continue; } _ => (), @@ -315,134 +615,340 @@ pub fn html_to_tokens(input: &str) -> Vec { token_start = pos; is_token_start = false; } - token_end = pos; + let run_end = scan_until(bytes, pos + 1, &TEXT_STOP); + token_end = run_end - 1; + pos = run_end; } if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], + text.push( + input, + token_start, + token_end + 1, is_after_space && !is_new_line, ); } if !text.is_empty() { tags.push(HtmlToken::Text { - text: text.as_str().into(), + text: text.take(input), }); } tags } -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn test_html_to_tokens_text() { - let input = "Hello, world!"; - let tokens = html_to_tokens(input); - assert_eq!( - tokens, - vec![HtmlToken::Text { - text: "Hello, world!".into() - }] - ); - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QuoteMode { + Keep, + Trim, +} - #[test] - fn test_html_to_tokens_start_tag() { - let input = "
"; - let tokens = html_to_tokens(input); - assert_eq!( - tokens, - vec![HtmlToken::StartTag { - name: 7760228, - attributes: vec![], - is_self_closing: false - }] - ); - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VisibleTextOptions { + pub quotes: QuoteMode, + pub max_chars: usize, +} - #[test] - fn test_html_to_tokens_end_tag() { - let input = "
"; - let tokens = html_to_tokens(input); - assert_eq!(tokens, vec![HtmlToken::EndTag { name: 7760228 }]); - } +impl VisibleTextOptions { + pub const UNLIMITED: Self = VisibleTextOptions { + quotes: QuoteMode::Keep, + max_chars: usize::MAX, + }; +} - #[test] - fn test_html_to_tokens_comment() { - let input = ""; - let tokens = html_to_tokens(input); - assert_eq!( - tokens, - vec![HtmlToken::Comment { - text: "!-- This is a comment --".into() - }] - ); - } +#[derive(Debug, Default)] +pub struct HtmlText { + pub visible: String, + pub hidden: String, +} - #[test] - fn test_html_to_tokens_mixed() { - let input = "
Hello, " world " !
"; - let tokens = html_to_tokens(input); - assert_eq!( - tokens, - vec![ - HtmlToken::StartTag { - name: 7760228, - attributes: vec![], - is_self_closing: false - }, - HtmlToken::Text { - text: "Hello,".into() - }, - HtmlToken::StartTag { - name: 1851879539, - attributes: vec![], - is_self_closing: false - }, - HtmlToken::Text { - text: " \" world \"".into() - }, - HtmlToken::EndTag { name: 1851879539 }, - HtmlToken::Text { text: " !".into() }, - HtmlToken::EndTag { name: 7760228 } - ] - ); - } +struct SkippedElement { + name: u64, + depth: u32, + is_hidden_text: bool, +} - #[test] - fn test_html_to_tokens_with_attributes() { - let input = r#""#; - let tokens = html_to_tokens(input); - assert_eq!( - tokens, - vec![ - HtmlToken::StartTag { - name: 500186508905, - attributes: vec![ - (1701869940, Some("text".into())), - (435761734006, Some("test".into())) - ], - is_self_closing: false - }, - HtmlToken::StartTag { - name: 111516266162547, - attributes: vec![], - is_self_closing: true - }, - HtmlToken::StartTag { - name: 6647407, - attributes: vec![(1920234593, None)], - is_self_closing: true - }, - HtmlToken::StartTag { - name: 97, - attributes: vec![(98, Some("1".into())), (98, None), (99, Some("123".into()))], - is_self_closing: false - } - ] - ); +const ASCII_SCAN_CHUNK: usize = 64; +const PRINTABLE_ASCII_COUNT: u8 = 95; +const DEL: u8 = 0x7F; +const INVISIBLE_LEAD_2: u8 = 0xC2; +const INVISIBLE_LEAD_3: u8 = 0xE2; +const INVISIBLE_LEAD_BOM: u8 = 0xEF; + +impl HtmlText { + pub fn new(tokens: &[HtmlToken<'_>]) -> Self { + let mut hidden = String::new(); + let (visible, _) = + html_visible_text(tokens, VisibleTextOptions::UNLIMITED, Some(&mut hidden)); + HtmlText { visible, hidden } } } + +pub fn html_visible_text( + tokens: &[HtmlToken<'_>], + options: VisibleTextOptions, + mut hidden: Option<&mut String>, +) -> (String, bool) { + let text_len = tokens + .iter() + .map(|token| match token { + HtmlToken::Text { text } => text.len() + 1, + _ => 0, + }) + .sum::(); + let mut out = String::with_capacity(text_len.min(options.max_chars.saturating_mul(2))); + let mut remaining = options.max_chars; + let mut skipped: Option = None; + + for token in tokens { + match token { + HtmlToken::StartTag { + name, + attributes, + is_self_closing, + } => { + if let Some(element) = &mut skipped { + if *name == element.name && !*is_self_closing { + element.depth += 1; + } + } else if !*is_self_closing + && !is_void_tag(*name) + && (is_non_content_tag(*name) + || (options.quotes == QuoteMode::Trim && *name == BLOCKQUOTE) + || is_hidden(attributes, options.quotes)) + { + skipped = Some(SkippedElement { + name: *name, + depth: 1, + is_hidden_text: *name != HEAD, + }); + } + } + HtmlToken::EndTag { name } => { + if let Some(element) = &mut skipped + && *name == element.name + { + element.depth -= 1; + if element.depth == 0 { + skipped = None; + } + } + } + HtmlToken::Text { text } => match &skipped { + None => { + if needs_space(&out, text) { + if remaining == 0 { + return trimmed(out, true); + } + out.push(' '); + remaining -= 1; + } + match push_visible(&mut out, text, &mut remaining) { + Pushed::Clean => {} + Pushed::Stripped => { + if let Some(hidden) = hidden.as_mut() { + push_hidden(hidden, text); + } + } + Pushed::Truncated => return trimmed(out, true), + } + } + Some(element) if element.is_hidden_text => { + if let Some(hidden) = hidden.as_mut() { + push_hidden(hidden, text); + } + } + Some(_) => {} + }, + HtmlToken::Comment { .. } => {} + } + } + + trimmed(out, false) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Pushed { + Clean, + Stripped, + Truncated, +} + +fn needs_space(out: &str, text: &str) -> bool { + !out.is_empty() && !out.ends_with(' ') && !text.starts_with(' ') +} + +fn push_hidden(hidden: &mut String, text: &str) { + if needs_space(hidden, text) { + hidden.push(' '); + } + hidden.push_str(text); +} + +fn trimmed(mut out: String, truncated: bool) -> (String, bool) { + let len = out.trim_end().len(); + out.truncate(len); + (out, truncated) +} + +fn push_visible(out: &mut String, text: &str, remaining: &mut usize) -> Pushed { + if text.len() <= *remaining && is_visible_ascii(text) { + out.push_str(text); + *remaining -= text.len(); + Pushed::Clean + } else if has_control_bytes(text) { + push_segments(out, text.split(is_invisible), remaining) + } else { + push_segments(out, split_invisible_unicode(text), remaining) + } +} + +fn push_segments<'x>( + out: &mut String, + mut segments: impl Iterator, + remaining: &mut usize, +) -> Pushed { + let mut result = Pushed::Clean; + if let Some(first) = segments.next() { + if !push_segment(out, first, remaining) { + return Pushed::Truncated; + } + for segment in segments { + result = Pushed::Stripped; + if !push_segment(out, segment, remaining) { + return Pushed::Truncated; + } + } + } + result +} + +fn push_segment(out: &mut String, segment: &str, remaining: &mut usize) -> bool { + let chars = segment.chars().count(); + if chars > *remaining { + out.extend(segment.chars().take(*remaining)); + return false; + } + out.push_str(segment); + *remaining -= chars; + true +} + +fn split_invisible_unicode(text: &str) -> impl Iterator { + let mut leads = memchr3_iter( + INVISIBLE_LEAD_2, + INVISIBLE_LEAD_3, + INVISIBLE_LEAD_BOM, + text.as_bytes(), + ); + let mut start = Some(0); + std::iter::from_fn(move || { + let from = start?; + for pos in leads.by_ref() { + if let Some(ch) = text.get(pos..).and_then(|rest| rest.chars().next()) + && is_invisible(ch) + { + start = Some(pos + ch.len_utf8()); + return text.get(from..pos); + } + } + start = None; + text.get(from..) + }) +} + +fn is_hidden(attributes: &[(u64, Option>)], quotes: QuoteMode) -> bool { + attributes.iter().any(|(key, value)| match *key { + HIDDEN => true, + STYLE => value.as_deref().is_some_and(is_hidden_style), + CLASS if quotes == QuoteMode::Trim => value.as_deref().is_some_and(|value| { + value.split_ascii_whitespace().any(|class| { + hashify::set_ignore_case!( + class.as_bytes(), + "gmail_quote", + "gmail_signature", + "moz-signature", + "moz-cite-prefix", + "yahoo_quoted", + ) + }) + }), + _ => false, + }) +} + +fn is_hidden_style(style: &str) -> bool { + let style = style.as_bytes(); + memchr_iter(b':', style).any(|colon| { + let Some(rest) = style.get(colon + 1..) else { + return false; + }; + if !matches!( + rest.iter().find(|ch| !ch.is_ascii_whitespace()), + Some(b'n' | b'N' | b'h' | b'H' | b'a' | b'A' | b'0' | b'.' | b'+' | b'-') + ) { + return false; + } + let property = style + .get(..colon) + .and_then(|head| head.rsplit(|&ch| ch == b';').next()) + .unwrap_or_default() + .trim_ascii(); + let value = || { + let value = rest + .split(|&ch| ch == b';') + .next() + .unwrap_or_default() + .trim_ascii(); + value + .strip_suffix(b"!important") + .map_or(value, <[u8]>::trim_ascii_end) + }; + hashify::fnc_map_ignore_case!(property, + "display" => value().eq_ignore_ascii_case(b"none"), + "visibility" => value().eq_ignore_ascii_case(b"hidden"), + "mso-hide" => value().eq_ignore_ascii_case(b"all"), + "font-size" => is_zero_length(value()), + "opacity" => is_zero_length(value()), + "max-height" => is_zero_length(value()), + "max-width" => is_zero_length(value()), + "line-height" => is_zero_length(value()), + _ => false + ) + }) +} + +fn is_zero_length(value: &[u8]) -> bool { + std::str::from_utf8(value).is_ok_and(|value| { + value.trim_start_matches(['+', '-']).starts_with(['0', '.']) + && value + .trim_end_matches(|ch: char| ch.is_ascii_alphabetic() || ch == '%') + .parse::() + .is_ok_and(|value| value == 0.0) + }) +} + +pub(crate) fn is_visible_ascii(text: &str) -> bool { + text.as_bytes().chunks(ASCII_SCAN_CHUNK).all(|chunk| { + chunk.iter().fold(true, |visible, &ch| { + visible + & ((ch.wrapping_sub(b' ') < PRINTABLE_ASCII_COUNT) + | (ch == b'\t') + | (ch == b'\n') + | (ch == b'\r')) + }) + }) +} + +fn has_control_bytes(text: &str) -> bool { + !text.as_bytes().chunks(ASCII_SCAN_CHUNK).all(|chunk| { + chunk.iter().fold(true, |visible, &ch| { + visible & ((ch >= b' ') | (ch == b'\t') | (ch == b'\n') | (ch == b'\r')) & (ch != DEL) + }) + }) +} + +pub(crate) fn is_invisible(ch: char) -> bool { + ch.is_zwsp() || ch.is_obscured() || (ch.is_control() && !matches!(ch, '\n' | '\t' | '\r')) +} + diff --git a/crates/spam-filter/src/modules/pyzor.rs b/crates/spam-filter/src/modules/pyzor.rs index 27160113..c382b57b 100644 --- a/crates/spam-filter/src/modules/pyzor.rs +++ b/crates/spam-filter/src/modules/pyzor.rs @@ -4,76 +4,38 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::SpamFilterInput; use common::config::mailstore::spamfilter::PyzorConfig; -use mail_parser::{Message, PartType, decoders::html::add_html_token}; +use compact_str::ToCompactString; +use mail_parser::PartKind; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; + +use super::html::{TEXT_STOP, byte_table, push_span, scan_until}; use sha1::{Digest, Sha1}; use std::{ borrow::Cow, net::SocketAddr, + ops::Range, time::{Duration, SystemTime}, }; use tokio::net::UdpSocket; use utils::HexEncode; -const MIN_LINE_LENGTH: usize = 8; -const ATOMIC_NUM_LINES: usize = 4; -const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; +static TAG_STOP: [bool; 256] = byte_table(b"<>/! \t\r\n"); +static STYLE_STOP: [bool; 256] = byte_table(b"< \t\r\n"); +static COMMENT_MARK: [bool; 256] = byte_table(b"->"); + +pub(crate) const MIN_LINE_LENGTH: usize = 8; +pub(crate) const ATOMIC_NUM_LINES: usize = 4; +pub(crate) const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; #[derive(Default, Debug, PartialEq, Eq)] -pub(crate) struct PyzorResponse { +pub struct PyzorResponse { pub code: u32, pub count: u64, pub wl_count: u64, } -pub(crate) async fn pyzor_check( - message: &Message<'_>, - config: &PyzorConfig, -) -> trc::Result> { - let Some(request) = message.pyzor_check_message() else { - return Ok(None); - }; - - #[cfg(feature = "test_mode")] - { - if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { - return Ok(PyzorResponse { - code: 200, - count: 1000, - wl_count: 0, - } - .into()); - } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { - return Ok(PyzorResponse { - code: 200, - count: 60, - wl_count: 10, - } - .into()); - } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { - return Ok(PyzorResponse { - code: 200, - count: 50, - wl_count: 20, - } - .into()); - } - } - - // Send message to address - pyzor_send_message(config.address, config.timeout, &request) - .await - .map(Into::into) - .map_err(|err| { - trc::SpamEvent::PyzorError - .into_err() - .ctx(trc::Key::Url, config.address.to_string()) - .reason(err) - .details("Pyzor failed") - }) -} - async fn pyzor_send_message( addr: SocketAddr, timeout: Duration, @@ -130,7 +92,7 @@ async fn pyzor_send_message( } } -trait PyzorWrite { +pub trait PyzorWrite { fn write_all(&mut self, data: &[u8]); } @@ -146,94 +108,119 @@ impl PyzorWrite for Sha1 { } } -trait PyzorDigest { - fn pyzor_digest(&self, writer: W) -> Option; -} +impl SpamFilterInput<'_> { + pub(crate) async fn pyzor_check( + &self, + config: &PyzorConfig, + ) -> trc::Result> { + let Some(request) = self.pyzor_check_message() else { + return Ok(None); + }; -pub trait PyzorCheck { - fn pyzor_check_message(&self) -> Option; -} + #[cfg(stalwart_tests)] + { + if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { + return Ok(PyzorResponse { + code: 200, + count: 1000, + wl_count: 0, + } + .into()); + } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { + return Ok(PyzorResponse { + code: 200, + count: 60, + wl_count: 10, + } + .into()); + } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { + return Ok(PyzorResponse { + code: 200, + count: 50, + wl_count: 20, + } + .into()); + } + } -impl PyzorDigest for Message<'_> { - fn pyzor_digest(&self, writer: W) -> Option { + pyzor_send_message(config.address, config.timeout, &request) + .await + .map(Into::into) + .map_err(|err| { + trc::SpamEvent::PyzorError + .into_err() + .ctx(trc::Key::Url, config.address.to_compact_string()) + .reason(err) + .details("Pyzor failed") + }) + } + + pub fn pyzor_digest(&self, writer: W) -> Option { + let texts = self.texts; let parts = self - .parts - .iter() - .filter_map(|part| match &part.body { - PartType::Text(text) => Some(text.as_ref().into()), - PartType::Html(html) => Some(html_to_text(html.as_ref()).into()), + .message + .root() + .parts() + .filter_map(|part| match part.kind() { + PartKind::Text => texts.get(part.id()).map(Cow::Borrowed), + PartKind::Html => Some(Cow::Owned(html_to_text(texts.get(part.id())?))), _ => None, }) - .collect::>>(); + .collect::>(); pyzor_digest(writer, parts.iter().flat_map(|text| text.lines())) } -} -impl PyzorCheck for Message<'_> { - fn pyzor_check_message(&self) -> Option { + pub fn pyzor_check_message(&self) -> Option { let time = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); - pyzor_create_message( - self, + self.pyzor_create_message( time, (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16, ) } + + fn pyzor_create_message(&self, time: u64, thread: u16) -> Option { + let hash = self.pyzor_digest(Sha1::new())?.finalize().hex_encode(); + let mut hash_key = Sha1::new(); + hash_key.update("anonymous:".as_bytes()); + let hash_key = hash_key.finalize().hex_encode(); + + let message = format!( + "Op: check\nOp-Digest: {hash}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" + ); + let mut msg_hash = Sha1::new(); + msg_hash.update(message.as_bytes()); + let msg_hash = msg_hash.finalize(); + + let mut sig = Sha1::new(); + sig.update(msg_hash); + sig.update(format!(":{time}:{hash_key}")); + let sig = sig.finalize().hex_encode(); + + Some(format!("{message}\nSig: {sig}\n")) + } } -fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> Option { - // Hash message - let hash = message.pyzor_digest(Sha1::new())?.finalize().hex_encode(); - // Hash key - let mut hash_key = Sha1::new(); - hash_key.update("anonymous:".as_bytes()); - let hash_key = hash_key.finalize().hex_encode(); - - // Hash message - let message = format!( - "Op: check\nOp-Digest: {hash}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" - ); - let mut msg_hash = Sha1::new(); - msg_hash.update(message.as_bytes()); - let msg_hash = msg_hash.finalize(); - - // Sign - let mut sig = Sha1::new(); - sig.update(msg_hash); - sig.update(format!(":{time}:{hash_key}")); - let sig = sig.finalize().hex_encode(); - - Some(format!("{message}\nSig: {sig}\n")) -} - -fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> Option +pub fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> Option where I: Iterator, W: PyzorWrite, { - let mut result = Vec::with_capacity(16); + let mut buffer = String::new(); + let mut kept: Vec> = Vec::with_capacity(16); for line in lines { - let mut clean_line = String::with_capacity(line.len()); + if line.len() < MIN_LINE_LENGTH { + continue; + } + buffer.reserve(line.len()); + let line_start = buffer.len(); let mut token_start = usize::MAX; let mut token_end = usize::MAX; - let add_line = |line: &mut String, span: &str| { - if !span.contains(char::from(0)) { - if span.len() < 10 { - line.push_str(span); - } - } else { - let span = span.replace(char::from(0), ""); - if span.len() < 10 { - line.push_str(&span); - } - } - }; - for token in TypesTokenizer::new(line) { match token.word { TokenType::Alphabetic(_) @@ -254,7 +241,7 @@ where | TokenType::IpAddr(_) | TokenType::Email(_) => { if token_start != usize::MAX { - add_line(&mut clean_line, &line[token_start..token_end]); + add_span(&mut buffer, &line[token_start..token_end]); token_start = usize::MAX; token_end = usize::MAX; } @@ -263,38 +250,53 @@ where } if token_start != usize::MAX { - add_line(&mut clean_line, &line[token_start..token_end]); + add_span(&mut buffer, &line[token_start..token_end]); } - if clean_line.len() >= MIN_LINE_LENGTH { - result.push(clean_line); + if buffer.len() - line_start >= MIN_LINE_LENGTH { + kept.push(line_start..buffer.len()); + } else { + buffer.truncate(line_start); } } - if result.is_empty() { + if kept.is_empty() { return None; } - if result.len() > ATOMIC_NUM_LINES { + if kept.len() > ATOMIC_NUM_LINES { for (offset, length) in DIGEST_SPEC { for i in 0..*length { - if let Some(line) = result.get((*offset * result.len() / 100) + i) { - writer.write_all(line.as_bytes()); + if let Some(line) = kept.get((*offset * kept.len() / 100) + i) { + writer.write_all(buffer[line.clone()].as_bytes()); } } } } else { - for line in result { - writer.write_all(line.as_bytes()); + for line in kept { + writer.write_all(buffer[line].as_bytes()); } } Some(writer) } -fn html_to_text(input: &str) -> String { +#[inline(always)] +fn add_span(line: &mut String, span: &str) { + if !span.as_bytes().contains(&0) { + if span.len() < 10 { + line.push_str(span); + } + } else if span.len() - span.as_bytes().iter().filter(|&&ch| ch == 0).count() < 10 { + for part in span.split('\0') { + line.push_str(part); + } + } +} + +pub fn html_to_text(input: &str) -> String { let mut result = String::with_capacity(input.len()); - let input = input.as_bytes(); + let bytes = input.as_bytes(); let mut in_tag = false; let mut in_comment = false; @@ -311,112 +313,150 @@ fn html_to_text(input: &str) -> String { let mut tag_token_pos = 0; let mut comment_pos = 0; - for (pos, ch) in input.iter().enumerate() { - if !in_comment { - match ch { - b'<' => { - if !(in_tag || in_style || in_script || is_token_start) { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - is_after_space = false; - } + let mut pos = 0; - tag_token_pos = 0; - in_tag = true; - is_token_start = true; - is_tag_close = false; - continue; + while let Some(&ch) = bytes.get(pos) { + if in_comment { + let stop = scan_until(bytes, pos, &COMMENT_MARK); + if stop > pos { + comment_pos = 0; + } + pos = stop; + match bytes.get(pos) { + None => break, + Some(b'-') => { + comment_pos += 1; } - b'>' if in_tag => { - if tag_token_pos == 1 - && let Some(tag) = input.get(token_start..token_end + 1) - { - if tag.eq_ignore_ascii_case(b"style") { - in_style = !is_tag_close; - } else if tag.eq_ignore_ascii_case(b"script") { - in_script = !is_tag_close; - } + Some(_) => { + if comment_pos == 2 { + in_comment = false; + in_tag = false; + is_token_start = true; } + comment_pos = 0; + } + } + pos += 1; + continue; + } - in_tag = false; - is_token_start = true; - is_after_space = !result.is_empty(); - - continue; - } - b'/' if in_tag => { - if tag_token_pos == 0 { - is_tag_close = true; - } - continue; - } - b'!' if in_tag && tag_token_pos == 0 => { - if let Some(b"--") = input.get(pos + 1..pos + 3) { - in_comment = true; - continue; - } - } - b' ' | b'\t' | b'\r' | b'\n' => { - if !(in_tag || in_style || in_script) { - if !is_token_start { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - } - is_after_space = true; - } - - is_token_start = true; - continue; - } - b'&' if !(in_tag || is_token_start || in_style || in_script) => { - add_html_token( + match ch { + b'<' => { + if !(in_tag || in_style || in_script || is_token_start) { + push_span( &mut result, - &input[token_start..token_end + 1], + input, + token_start, + token_end + 1, is_after_space, ); - is_token_start = true; is_after_space = false; } - b';' if !(in_tag || is_token_start || in_style || in_script) => { - add_html_token(&mut result, &input[token_start..pos + 1], is_after_space); - is_token_start = true; - is_after_space = false; + + tag_token_pos = 0; + in_tag = true; + is_token_start = true; + is_tag_close = false; + pos += 1; + continue; + } + b'>' if in_tag => { + if tag_token_pos == 1 + && let Some(tag) = bytes.get(token_start..token_end + 1) + { + if tag.eq_ignore_ascii_case(b"style") { + in_style = !is_tag_close; + } else if tag.eq_ignore_ascii_case(b"script") { + in_script = !is_tag_close; + } + } + + in_tag = false; + is_token_start = true; + is_after_space = !result.is_empty(); + pos += 1; + continue; + } + b'/' if in_tag => { + if tag_token_pos == 0 { + is_tag_close = true; + } + pos += 1; + continue; + } + b'!' if in_tag && tag_token_pos == 0 => { + if let Some(b"--") = bytes.get(pos + 1..pos + 3) { + in_comment = true; + pos += 1; continue; } - _ => (), } - if is_token_start { - token_start = pos; - is_token_start = false; - if in_tag { - tag_token_pos += 1; + b' ' | b'\t' | b'\r' | b'\n' => { + if !(in_tag || in_style || in_script) { + if !is_token_start { + push_span( + &mut result, + input, + token_start, + token_end + 1, + is_after_space, + ); + } + is_after_space = true; } + + is_token_start = true; + pos += 1; + while matches!(bytes.get(pos), Some(b' ' | b'\t' | b'\r' | b'\n')) { + pos += 1; + } + continue; } - token_end = pos; - } else { - match ch { - b'-' => comment_pos += 1, - b'>' if comment_pos == 2 => { - comment_pos = 0; - in_comment = false; - in_tag = false; - is_token_start = true; - } - _ => comment_pos = 0, + b'&' if !(in_tag || is_token_start || in_style || in_script) => { + push_span( + &mut result, + input, + token_start, + token_end + 1, + is_after_space, + ); + is_token_start = true; + is_after_space = false; + } + b';' if !(in_tag || is_token_start || in_style || in_script) => { + push_span(&mut result, input, token_start, pos + 1, is_after_space); + is_token_start = true; + is_after_space = false; + pos += 1; + continue; + } + _ => (), + } + + if is_token_start { + token_start = pos; + is_token_start = false; + if in_tag { + tag_token_pos += 1; } } + let stop = if in_tag { + scan_until(bytes, pos + 1, &TAG_STOP) + } else if in_style || in_script { + scan_until(bytes, pos + 1, &STYLE_STOP) + } else { + scan_until(bytes, pos + 1, &TEXT_STOP) + }; + token_end = stop - 1; + pos = stop; } if !(in_tag || is_token_start || in_style || in_script) { - add_html_token( + push_span( &mut result, - &input[token_start..token_end + 1], + input, + token_start, + token_end + 1, is_after_space, ); } @@ -425,369 +465,3 @@ fn html_to_text(input: &str) -> String { result } -#[cfg(test)] -mod test { - use std::time::Duration; - - use mail_parser::MessageParser; - use sha1::Digest; - use sha1::Sha1; - use utils::HexEncode; - - use super::pyzor_create_message; - use super::pyzor_send_message; - use super::{PyzorDigest, html_to_text, pyzor_digest}; - - use super::PyzorResponse; - - #[ignore] - #[tokio::test] - async fn send_message() { - assert_eq!( - pyzor_send_message( - "public.pyzor.org:24441".parse().unwrap(), - Duration::from_secs(10), - concat!( - "Op: check\n", - "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", - "Thread: 49005\n", - "PV: 2.1\n", - "User: anonymous\n", - "Time: 1697468672\n", - "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" - ), - ) - .await - .unwrap(), - PyzorResponse { - code: 200, - count: 0, - wl_count: 0 - } - ); - } - - #[test] - fn message_pyzor() { - let message = pyzor_create_message( - &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), - 1697468672, - 49005, - ) - .unwrap(); - - assert_eq!( - message, - concat!( - "Op: check\n", - "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", - "Thread: 49005\n", - "PV: 2.1\n", - "User: anonymous\n", - "Time: 1697468672\n", - "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" - ) - ); - } - - #[test] - fn digest_pyzor() { - // HTML stripping - assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED); - - // Token stripping - for strip_me in [ - "t@abc.com", - "t1@abc.com", - "t+a@abc.com", - "t.a@abc.com", - "0A2D3f%a#S", - "3sddkf9jdkd9", - "@@#@@@@@@@@@", - "http://spammer.com/special-offers?buy=now", - ] { - assert_eq!( - String::from_utf8( - pyzor_digest(Vec::new(), format!("Test {strip_me} Test2").lines()).unwrap() - ) - .unwrap(), - "TestTest2" - ); - } - - // Test short lines - assert_eq!( - String::from_utf8( - pyzor_digest( - Vec::new(), - concat!("This line is included\n", "not this\n", "This also").lines(), - ) - .unwrap() - ) - .unwrap(), - "ThislineisincludedThisalso" - ); - - // Test atomic - assert_eq!( - String::from_utf8( - pyzor_digest( - Vec::new(), - "All this message\nShould be included\nIn the digest".lines(), - ) - .unwrap() - ) - .unwrap(), - "AllthismessageShouldbeincludedInthedigest" - ); - - // Test spec - let mut text = String::new(); - for i in 0..100 { - text += format!("Line{i} test test test\n").as_str(); - } - let mut expected = String::new(); - for i in [20, 21, 22, 60, 61, 62] { - expected += format!("Line{i}testtesttest").as_str(); - } - assert_eq!( - String::from_utf8(pyzor_digest(Vec::new(), text.lines()).unwrap()).unwrap(), - expected - ); - - // Test email parsing - for (input, expected) in [ - ( - HTML_TEXT, - concat!( - "Emailspam,alsoknownasjunkemailorbulkemail,isasubset", - "ofspaminvolvingnearlyidenticalmessagessenttonumerous", - "byemail.Clickingonlinksinspamemailmaysendusersto", - "byemail.Clickingonlinksinspamemailmaysendusersto", - "phishingwebsitesorsitesthatarehostingmalware.", - "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,", - "isasubsetofspaminvolvingnearlyidenticalmessage", - "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse", - "rstophishingwebsitesorsitesthatarehostingmalware." - ), - ), - (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."), - (TEXT_ATTACHMENT, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"), - ] { - assert_eq!( - String::from_utf8( - MessageParser::new() - .parse(input) - .unwrap() - .pyzor_digest(Vec::new()) - .unwrap() - ) - .unwrap(), - expected, - "failed for {input}" - ) - } - - // Test SHA hash - assert_eq!( - MessageParser::new() - .parse(HTML_TEXT_STYLE_SCRIPT) - .unwrap() - .pyzor_digest(Sha1::new()) - .unwrap() - .finalize() - .hex_encode(), - "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", - ) - } - - const HTML_TEXT: &str = r#"MIME-Version: 1.0 -Sender: chirila@gapps.spamexperts.com -Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) -Date: Thu, 16 Jan 2014 10:43:31 +0200 -Delivered-To: chirila@gapps.spamexperts.com -X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd - ---001a11c25ff293069304f0126bfd -Content-Type: text/plain; charset=ISO-8859-1 - -Email spam. - -Email spam, also known as junk email or unsolicited bulk email, is a subset -of electronic spam involving nearly identical messages sent to numerous -recipients by email. Clicking on links in spam email may send users to -phishing web sites or sites that are hosting malware. - ---001a11c25ff293069304f0126bfd -Content-Type: text/html; charset=ISO-8859-1 -Content-Transfer-Encoding: quoted-printable - -
Email spam.

Email spam, also= - known as junk email or unsolicited bulk email, is a subset of electronic s= -pam involving nearly identical messages sent to numerous recipients by emai= -l. Clicking on links in spam email may send users to phishing web sites or = -sites that are hosting malware.
-
- ---001a11c25ff293069304f0126bfd-- -"#; - - const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0 -Sender: chirila@gapps.spamexperts.com -Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) -Date: Thu, 16 Jan 2014 10:43:31 +0200 -Delivered-To: chirila@gapps.spamexperts.com -X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd - ---001a11c25ff293069304f0126bfd -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test. - ---001a11c25ff293069304f0126bfd -Content-Type: text/html; charset=ISO-8859-1 -Content-Transfer-Encoding: quoted-printable - -
- - -
This is a test.
-
- ---001a11c25ff293069304f0126bfd-- -"#; - - const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test mailing - ---f46d040a62c49bb1c404f027e8ca-- ---f46d040a62c49bb1c804f027e8cc -Content-Type: image/png; name="tar.png" -Content-Disposition: attachment; filename="tar.png" -Content-Transfer-Encoding: base64 -X-Attachment-Id: f_hqjas5ad0 - -iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD -QmCC ---f46d040a62c49bb1c804f027e8cc--"#; - - const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test ma\0iling ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test ma\0\0\0iling ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Te\0\0\0st -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test mailing ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\" - -This is a test mailing ---f46d040a62c49bb1c804f027e8cc--"; - - const HTML_RAW: &str = r#"Email spam -

Email spam, also known as junk email -or unsolicited bulk email (UBE), is a subset of -electronic spam -involving nearly identical messages sent to numerous recipients by -email. Clicking on -links in spam email may send users to phishing -web sites or sites that are hosting malware."#; - - const HTML_RAW_STRIPED: &str = concat!( - "Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE ),", - " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email", - " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ." - ); -} diff --git a/crates/spam-filter/src/modules/sanitize.rs b/crates/spam-filter/src/modules/sanitize.rs index f01f618e..1e852cac 100644 --- a/crates/spam-filter/src/modules/sanitize.rs +++ b/crates/spam-filter/src/modules/sanitize.rs @@ -4,55 +4,82 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{Email, Hostname}; use std::net::IpAddr; -use crate::{Email, Hostname}; +const MAX_IP_ADDR_LEN: usize = 45; + +#[inline] +pub(crate) fn can_be_ip_addr(text: &str) -> bool { + let bytes = text.as_bytes(); + if bytes.len() > MAX_IP_ADDR_LEN { + return false; + } + match bytes.first() { + Some(b':') => true, + Some(byte) if byte.is_ascii_digit() => true, + Some(byte) if byte.is_ascii_hexdigit() => bytes.contains(&b':'), + _ => false, + } +} + +#[inline] +fn ip_candidate(fqdn: &str) -> Option<&str> { + let text = fqdn + .strip_prefix('[') + .and_then(|inner| inner.strip_suffix(']')) + .unwrap_or(fqdn); + + can_be_ip_addr(text).then_some(text) +} + +fn decode_punycode(fqdn: &str) -> String { + let mut decoded = String::with_capacity(fqdn.len()); + + for part in fqdn.split('.') { + if !decoded.is_empty() { + decoded.push('.'); + } + + if let Some(puny) = part + .strip_prefix("xn--") + .and_then(idna::punycode::decode_to_string) + .filter(|puny| idna::domain_to_ascii(puny).is_ok_and(|reencoded| reencoded == part)) + { + decoded.push_str(&puny); + } else { + decoded.push_str(part); + } + } + + decoded +} + +fn second_level_domain(fqdn: &str) -> Option { + psl::domain(fqdn.as_bytes()).and_then(|domain| { + if domain.suffix().typ().is_some() { + std::str::from_utf8(domain.as_bytes()).ok().map(Into::into) + } else { + None + } + }) +} impl Hostname { pub fn new(host: &str) -> Self { - let mut fqdn = host.trim_end_matches('.').to_lowercase(); + Hostname::from_lowercase(host.trim_end_matches('.').to_lowercase()) + } - // Decode punycode + fn from_lowercase(mut fqdn: String) -> Self { if fqdn.contains("xn--") { - let mut decoded = String::with_capacity(fqdn.len()); - - for part in fqdn.split('.') { - if !decoded.is_empty() { - decoded.push('.'); - } - - if let Some(puny) = part - .strip_prefix("xn--") - .and_then(idna::punycode::decode_to_string) - .filter(|puny| { - idna::domain_to_ascii(puny).is_ok_and(|reencoded| reencoded == part) - }) - { - decoded.push_str(&puny); - } else { - decoded.push_str(part); - } - } - - fqdn = decoded; + fqdn = decode_punycode(&fqdn); } - let ip = fqdn - .strip_prefix('[') - .and_then(|ip| ip.strip_suffix(']')) - .unwrap_or(&fqdn) - .parse::() - .ok(); + let ip = ip_candidate(&fqdn).and_then(|text| text.parse::().ok()); Hostname { sld: if ip.is_none() { - psl::domain(fqdn.as_bytes()).and_then(|domain| { - if domain.suffix().typ().is_some() { - std::str::from_utf8(domain.as_bytes()).ok().map(Into::into) - } else { - None - } - }) + second_level_domain(&fqdn) } else { None }, @@ -60,6 +87,10 @@ impl Hostname { fqdn, } } + + pub fn sld_or_default(&self) -> &str { + self.sld.as_deref().unwrap_or(self.fqdn.as_str()) + } } impl Email { @@ -69,65 +100,9 @@ impl Email { Email { local_part: local_part.into(), - domain_part: Hostname::new(domain), + domain_part: Hostname::from_lowercase(domain.trim_end_matches('.').into()), address, } } } -impl Hostname { - pub fn sld_or_default(&self) -> &str { - self.sld.as_deref().unwrap_or(self.fqdn.as_str()) - } -} - -#[cfg(test)] -mod test { - use crate::{Email, Hostname}; - - #[test] - fn hostname_punycode_round_trip() { - for (host, fqdn, sld) in [ - ("mail.example.com", "mail.example.com", Some("example.com")), - ( - "MAIL.Example.CO.UK.", - "mail.example.co.uk", - Some("example.co.uk"), - ), - ( - "mail.xn--eebajf.xn--9dbq2a", - "mail.\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}", - Some("\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}"), - ), - ("xn--gmail-.com", "xn--gmail-.com", Some("xn--gmail-.com")), - ( - "xn--example-.org", - "xn--example-.org", - Some("xn--example-.org"), - ), - ("xn--.com", "xn--.com", Some("xn--.com")), - ("127.0.0.1", "127.0.0.1", None), - ] { - let parsed = Hostname::new(host); - assert_eq!(parsed.fqdn, fqdn, "fqdn of {host:?}"); - assert_eq!(parsed.sld.as_deref(), sld, "sld of {host:?}"); - } - } - - #[test] - fn email_a_label_and_u_label_are_equal() { - assert_eq!( - Email::new("bill@xn--eebajf.xn--9dbq2a"), - Email::new("bill@\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}") - ); - assert_ne!( - Email::new("bill@example.com"), - Email::new("bob@example.com") - ); - assert_ne!(Email::new("postmaster"), Email::new("mailer-daemon")); - assert_ne!( - Email::new("victim@xn--gmail-.com"), - Email::new("victim@gmail.com") - ); - } -} diff --git a/crates/spam-filter/src/texts.rs b/crates/spam-filter/src/texts.rs new file mode 100644 index 00000000..d4bfc0e9 --- /dev/null +++ b/crates/spam-filter/src/texts.rs @@ -0,0 +1,165 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::modules::html::{HtmlText, HtmlToken}; +use common::config::mailstore::spamfilter::SpamTextLimits; +use mail_parser::{Message, MessagePart, PartId}; +use memchr::{memrchr, memrchr3}; +use std::{borrow::Cow, sync::OnceLock}; + + +const MAX_TEXT_BYTES_PER_RAW_BYTE: usize = 3; + +pub struct MessageTexts<'x> { + texts: Vec>, + html_texts: Box<[OnceLock]>, +} + +pub enum PartText<'x> { + NotText, + Whole(Cow<'x, str>), + Prefix(Cow<'x, str>), + Skipped, +} + +impl<'x> MessageTexts<'x> { + pub fn new(message: &'x Message<'x>, limits: SpamTextLimits) -> Self { + let max_text = message + .raw() + .len() + .saturating_mul(MAX_TEXT_BYTES_PER_RAW_BYTE); + if max_text + <= limits + .max_body + .min(limits.max_attachment) + .min(limits.max_message) + { + return Self::with_texts( + message + .parts() + .map(|part| part.text().map_or(PartText::NotText, PartText::Whole)) + .collect(), + ); + } + + let mut texts: Vec<_> = message + .parts() + .map(|part| { + if part.is_text() { + PartText::Skipped + } else { + PartText::NotText + } + }) + .collect(); + + let html = message.html_body().find(MessagePart::is_text); + let text = message + .text_body() + .find(MessagePart::is_text) + .filter(|text| html.is_none_or(|html| html.id() != text.id())); + let primary = [html.map(|part| part.id()), text.map(|part| part.id())]; + let other_body = message.parts().filter(|part| { + part.is_text() && is_body_part(*part) && !primary.contains(&Some(part.id())) + }); + let rest = message + .parts() + .filter(|part| part.is_text() && !is_body_part(*part)); + + let mut budget = limits.max_message; + for part in [html, text] + .into_iter() + .flatten() + .chain(other_body) + .chain(rest) + { + let part_limit = if is_body_part(part) { + limits.max_body + } else { + limits.max_attachment + }; + let text = PartText::bounded(part, part_limit.min(budget)); + budget = budget.saturating_sub(text.text().map_or(0, str::len)); + if let Some(slot) = texts.get_mut(part.id() as usize) { + *slot = text; + } + } + + Self::with_texts(texts) + } + + fn with_texts(texts: Vec>) -> Self { + MessageTexts { + html_texts: texts.iter().map(|_| OnceLock::new()).collect(), + texts, + } + } + + pub fn get(&self, part_id: PartId) -> Option<&str> { + self.part(part_id)?.text() + } + + pub fn part(&self, part_id: PartId) -> Option<&PartText<'x>> { + self.texts.get(part_id as usize) + } + + pub fn is_cut(&self, part_id: PartId) -> bool { + self.part(part_id).is_some_and(PartText::is_cut) + } + + pub fn html_text(&self, part_id: PartId, html_tokens: &[HtmlToken<'_>]) -> Option<&HtmlText> { + self.html_texts + .get(part_id as usize) + .map(|text| text.get_or_init(|| HtmlText::new(html_tokens))) + } +} + +impl<'x> PartText<'x> { + fn bounded(part: MessagePart<'x>, limit: usize) -> Self { + if limit == 0 { + return PartText::Skipped; + } + let Some(text) = part.text_prefix_bytes(limit.saturating_add(char::MAX_LEN_UTF8)) else { + return PartText::NotText; + }; + if text.len() <= limit { + return PartText::Whole(text); + } + match cut_end(&text, limit) { + 0 => PartText::Skipped, + end => PartText::Prefix(match text { + Cow::Borrowed(text) => Cow::Borrowed(text.get(..end).unwrap_or(text)), + Cow::Owned(mut text) => { + text.truncate(end); + Cow::Owned(text) + } + }), + } + } + + pub fn text(&self) -> Option<&str> { + match self { + PartText::Whole(text) | PartText::Prefix(text) => Some(text), + PartText::NotText | PartText::Skipped => None, + } + } + + pub fn is_cut(&self) -> bool { + matches!(self, PartText::Prefix(_) | PartText::Skipped) + } +} + +fn cut_end(text: &str, limit: usize) -> usize { + let start = limit / 2; + text.as_bytes() + .get(start..limit) + .and_then(|tail| memrchr(b'\n', tail).or_else(|| memrchr3(b' ', b'\t', b'>', tail))) + .map_or_else(|| text.floor_char_boundary(limit), |pos| start + pos + 1) +} + +pub(crate) fn is_body_part(part: MessagePart<'_>) -> bool { + part.message().id() == 0 && (part.in_text_body() || part.in_html_body()) +} diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 477502ca..f896b6ca 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "store" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] @@ -11,68 +11,74 @@ trc = { path = "../trc" } registry = { path = "../registry" } rocksdb = { version = "0.24", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.11", features = ["embedded-fdb-include", "fdb-7_4"], optional = true } -rusqlite = { version = "0.40", features = ["bundled"], optional = true } -rust-s3 = { version = "0.38", default-features = false, features = ["tokio-rustls-tls"], optional = true } -reqwest_s3 = { package = "reqwest", version = "0.12", default-features = false, features = ["rustls-tls-native-roots"], optional = true } -azure_core = { version = "0.21.0", optional = true } -azure_storage = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true } -azure_storage_blobs = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true } +rusqlite = { version = "0.41", features = ["bundled"], optional = true } +s3 = { path = "../s3", optional = true } +azure_core = { version = "1.1", default-features = false, features = ["reqwest", "tokio"], optional = true } +azure_storage_blob = { version = "1.1", default-features = false, features = ["tokio"], optional = true } +azure_identity = { version = "1.0", default-features = false, features = ["tokio"], optional = true } reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "stream"]} -tokio = { version = "1.53", features = ["sync", "fs", "io-util"] } +tokio = { version = "1.54", features = ["sync", "fs", "io-util", "time", "rt", "rt-multi-thread"] } r2d2 = { version = "0.8.10", optional = true } futures = { version = "0.3", optional = true } -rand = "0.10.2" +libc = { version = "0.2.191", optional = true } +rand = "0.10.3" roaring = "0.11" rayon = { version = "1.12", optional = true } serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.12", features = ["serde"] } -xxhash-rust = { version = "0.8.18", features = ["xxh3"] } -parking_lot = "0.12" +xxhash-rust = { version = "0.8.19", features = ["xxh3"] } +parking_lot = "0.13" +papaya = "0.2.5" lru-cache = { version = "0.1.2", optional = true } num_cpus = { version = "1.17", optional = true } blake3 = "1.8" lz4_flex = { version = "0.14", features = ["alloc"], default-features = false } +zstd = { version = "0.14", default-features = false } deadpool-postgres = { version = "0.14", optional = true } +hashbrown = { version = "0.17.1", default-features = false, features = ["inline-more"], optional = true } tokio-postgres = { version = "0.7.18", features = ["with-serde_json-1"], optional = true } tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["aws_lc_rs", "tls12"] } -rustls = { version = "0.23.43", optional = true, default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +rustls = { version = "0.23.46", optional = true, default-features = false, features = ["std", "aws_lc_rs", "tls12"] } rustls-pki-types = { version = "1", optional = true } aws-lc-rs = { version = "1", optional = true } x509-parser = { version = "0.18", optional = true } bytes = { version = "1.12", optional = true } mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "minimal"], optional = true } -serde_json = { version = "1.0.151" } +serde_json = { version = "1.0.154" } flate2 = "1.1" -redis = { version = "1.6", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls", "cluster-async", "sentinel"], optional = true } +redis = { version = "1.8", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls", "cluster-async", "sentinel"], optional = true } deadpool = { version = "0.13", features = ["managed"], optional = true } -arc-swap = "1.9.2" +arc-swap = "1.9.3" bitpacking = "0.9.3" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = "0.10.0" gethostname = "1.1.0" radsort = "0.1.1" +indexmap = "2.14.2" +nohash-hasher = "0.2.0" +tinyvec = { version = "1.13.3", features = ["alloc"] } +encodify = "1.0" [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } [features] # Data Stores rocks = ["rocksdb", "rayon", "num_cpus"] sqlite = ["rusqlite", "rayon", "r2d2", "num_cpus", "lru-cache"] -postgres = ["tokio-postgres", "deadpool", "deadpool-postgres", "tokio-rustls", "rustls", "aws-lc-rs", "rustls-pki-types", "x509-parser", "futures", "bytes"] +postgres = ["tokio-postgres", "deadpool", "deadpool-postgres", "hashbrown", "tokio-rustls", "rustls", "aws-lc-rs", "rustls-pki-types", "x509-parser", "futures", "bytes"] mysql = ["mysql_async", "futures"] -foundation = ["foundationdb", "futures"] +foundation = ["foundationdb", "futures", "libc"] fdb-chunked-bm = [] # Blob stores -s3 = ["rust-s3", "dep:reqwest_s3"] -azure = ["azure_core", "azure_storage", "azure_storage_blobs", "futures"] +s3 = ["dep:s3", "bytes"] +azure = ["azure_core", "azure_storage_blob", "azure_identity", "aws-lc-rs", "futures"] # In-memory stores redis = ["dep:redis", "deadpool", "deadpool/rt_tokio_1", "futures"] -enterprise = [] -test_mode = [] +enterprise = ["utils/enterprise", "trc/enterprise", "registry/enterprise"] [lints] workspace = true diff --git a/crates/store/src/backend/azure/mod.rs b/crates/store/src/backend/azure/mod.rs index 9360d5e2..3b7ca5c6 100644 --- a/crates/store/src/backend/azure/mod.rs +++ b/crates/store/src/backend/azure/mod.rs @@ -4,166 +4,358 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use azure_core::error::ErrorKind; -use azure_core::{ExponentialRetryOptions, RetryOptions, StatusCode}; -use azure_storage::StorageCredentials; -use azure_storage_blobs::prelude::{ClientBuilder, ContainerClient}; -use futures::stream::StreamExt; -use registry::schema::structs::{self}; -use std::sync::Arc; -use std::{fmt::Display, io::Write, ops::Range}; -use utils::codec::base32_custom::Base32Writer; +mod shared_key; use crate::BlobStore; +use azure_core::{ + cloud::CloudConfiguration, + credentials::{Secret, TokenCredential}, + http::{ + ClientOptions, ExponentialRetryOptions, RequestContent, RetryOptions, StatusCode, + Transport, Url, headers::HeaderName, + }, + time::Duration as SdkDuration, +}; +use azure_identity::{ + ClientAssertionCredentialOptions, ClientSecretCredential, ClientSecretCredentialOptions, + ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, + WorkloadIdentityCredential, WorkloadIdentityCredentialOptions, +}; +use azure_storage_blob::{ + BlobClient, BlobContainerClient, BlobContainerClientOptions, + models::{BlobClientDownloadOptions, HttpRange}, +}; +use encodify::base32::STALWART; +use futures::StreamExt; +use registry::schema::{ + enums::AzureEnvironment, + structs::{self, AzureAuth}, +}; +use shared_key::SharedKeyPolicy; +use std::{fmt::Display, ops::Range, path::PathBuf, sync::Arc, time::Duration}; +use utils::http::http_client_builder; + +const CONTENT_RANGE: HeaderName = HeaderName::from_static("content-range"); +const RETRY_MAX_DELAY: Duration = Duration::from_secs(30); +const MAX_PREALLOCATION: usize = 64 * 1024 * 1024; pub struct AzureStore { - client: ContainerClient, - prefix: Option, + client: BlobContainerClient, + #[cfg(stalwart_tests)] + container: BlobContainerClient, + prefix: Option>, } impl AzureStore { pub async fn open(config: structs::AzureStore) -> Result { - let credentials = match ( - config.access_key.secret().await?.map(|v| v.into_owned()), - config.sas_token.secret().await?.map(|v| v.into_owned()), - ) { - (Some(access_key), None) => { - StorageCredentials::access_key(config.storage_account.clone(), access_key) + validate_account(&config.storage_account)?; + validate_container(&config.container)?; + let mut container_url = container_url( + config.url.as_deref(), + &config.storage_account, + &config.container, + config.environment, + )?; + let uses_token_credential = !matches!( + config.auth, + AzureAuth::SharedKey(_) | AzureAuth::SasToken(_) + ); + if uses_token_credential && container_url.scheme() != "https" { + return Err(format!( + "Azure Entra ID authentication requires an https URL, got {container_url}" + )); + } + + let http_client = http_client_builder(false) + .timeout(config.timeout.into_inner()) + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .build() + .map_err(|err| format!("Failed to build Azure HTTP client: {err}"))?; + let mut client_options = ClientOptions { + transport: Some(Transport::new(Arc::new(http_client))), + ..Default::default() + }; + let credential_options = ClientOptions { + cloud: cloud_configuration(config.environment).map(Arc::new), + ..client_options.clone() + }; + client_options.retry = retry_options(config.timeout.into_inner(), config.max_retries); + + let credential: Option> = match config.auth { + AzureAuth::SharedKey(auth) => { + let account_key = auth.account_key.secret().await?; + client_options + .per_try_policies + .push(Arc::new(SharedKeyPolicy::new( + &config.storage_account, + &account_key, + )?)); + None } - (None, Some(sas_token)) => match StorageCredentials::sas_token(sas_token) { - Ok(cred) => cred, - Err(err) => { - return Err(format!("Failed to create credentials: {err:?}")); + AzureAuth::SasToken(auth) => { + let sas_token = auth.sas_token.secret().await?; + let sas_token = sas_token.trim(); + let sas_token = sas_token.strip_prefix('?').unwrap_or(sas_token); + if sas_token.is_empty() { + return Err("Azure SAS token is empty".to_string()); } - }, - _ => { - return Err(concat!( - "Failed to create credentials: exactly one of ", - "'azure-access-key' and 'sas-token' must be specified" - ) - .to_string()); + container_url.set_query(Some(sas_token)); + None } + AzureAuth::ClientSecret(auth) => Some( + ClientSecretCredential::new( + &auth.tenant_id, + auth.client_id, + Secret::new(auth.client_secret.secret().await?.into_owned()), + Some(ClientSecretCredentialOptions { + client_options: credential_options, + }), + ) + .map_err(|err| format!("Failed to create Azure client secret credential: {err}"))?, + ), + AzureAuth::ManagedIdentity(auth) => Some( + ManagedIdentityCredential::new(Some(ManagedIdentityCredentialOptions { + user_assigned_id: auth.client_id.map(UserAssignedId::ClientId), + client_options: credential_options, + })) + .map_err(|err| { + format!("Failed to create Azure managed identity credential: {err}") + })?, + ), + AzureAuth::WorkloadIdentity(auth) => Some( + WorkloadIdentityCredential::new(Some(WorkloadIdentityCredentialOptions { + credential_options: ClientAssertionCredentialOptions { + client_options: credential_options, + }, + client_id: auth.client_id, + tenant_id: auth.tenant_id, + token_file_path: auth.token_file.map(PathBuf::from), + })) + .map_err(|err| { + format!("Failed to create Azure workload identity credential: {err}") + })?, + ), + }; + + let (directory, prefix) = split_prefix(config.key_prefix.as_deref().unwrap_or_default()); + let blobs_url = blobs_url(&container_url, directory)?; + let options = BlobContainerClientOptions { + client_options, + ..Default::default() }; Ok(BlobStore::Azure(Arc::new(AzureStore { - client: ClientBuilder::new(config.storage_account, credentials) - .retry(RetryOptions::exponential( - ExponentialRetryOptions::default().max_retries(config.max_retries as u32 * 2), - )) - .container_client(config.container), - prefix: config.key_prefix, + #[cfg(stalwart_tests)] + container: BlobContainerClient::new( + container_url, + credential.clone(), + Some(options.clone()), + ) + .map_err(|err| format!("Failed to create Azure container client: {err}"))?, + client: BlobContainerClient::new(blobs_url, credential, Some(options)) + .map_err(|err| format!("Failed to create Azure container client: {err}"))?, + prefix: (!prefix.is_empty()).then(|| prefix.into()), }))) } - pub(crate) async fn get_blob( - &self, - key: &[u8], - range: Range, - ) -> trc::Result>> { - let blob_client = self.client.blob_client(self.build_key(key)); + pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { + let blob_client = self.client.blob_client(&self.build_key(key)); - let mut stream = blob_client.get(); - let mut buf = if range.end == usize::MAX { - // Let's turn this into a proper RangeFrom. - stream = stream.range(range.start..); - // We don't know how big to expect the result to be. - Vec::new() - } else { - stream = stream.range(range.clone()); - Vec::with_capacity(range.end - range.start) + let http_range = match (range.start, range.end) { + (0, usize::MAX) => None, + (start, usize::MAX) => Some(HttpRange::from_offset(start as u64)), + (start, end) if start < end => Some(HttpRange::new(start as u64, (end - start) as u64)), + _ => return empty_range(&blob_client).await, }; - let mut stream = stream.into_stream(); - while let Some(response) = stream.next().await { - let err = match response { - Ok(chunks) => { - let mut chunks = chunks.data; - let mut err = None; - while let Some(chunk) = chunks.next().await { - match chunk { - Ok(ref data) => { - buf.extend(data); - } - Err(e) => { - err = Some(e); - break; - } - } - } - err - } - Err(e) => Some(e), - }; - - if let Some(e) = err { - return if matches!( - e.kind(), - ErrorKind::HttpResponse { - status: StatusCode::NotFound, - .. - } - ) { - Ok(None) - } else { - Err(trc::StoreEvent::AzureError.reason(e)) + let result = match blob_client + .download(Some(BlobClientDownloadOptions { + range: http_range, + ..Default::default() + })) + .await + { + Ok(result) => result, + Err(err) => { + return match err.http_status() { + Some(StatusCode::NotFound) => Ok(None), + Some(StatusCode::RequestedRangeNotSatisfiable) => Ok(Some(Vec::new())), + _ => Err(into_error(err)), }; } + }; + + let total_len = result + .headers + .get_optional_str(&CONTENT_RANGE) + .and_then(|value| value.rsplit_once('/')) + .and_then(|(_, total)| total.parse::().ok()) + .or_else(|| { + result + .properties + .content_length + .and_then(|len| usize::try_from(len).ok()) + }) + .unwrap_or_default(); + let mut buf = Vec::with_capacity( + total_len + .min(range.end) + .saturating_sub(range.start) + .min(MAX_PREALLOCATION), + ); + let mut body = result.body; + while let Some(chunk) = body.next().await { + buf.extend_from_slice(&chunk.map_err(into_error)?); } Ok(Some(buf)) } - pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { - let blob_client = self.client.blob_client(self.build_key(key)); - - // We unfortunately have to make a copy of `data`. This is because the Azure SDK wants to - // coerce the body into a value of type azure_core::Body, which doesn't have a lifetime - // parameter and so cannot hold any non-static references (directly or indirectly). - let data = data.to_vec(); - - blob_client - .put_block_blob(data) - .into_future() + pub async fn put_blob(&self, key: &[u8], data: Vec) -> trc::Result<()> { + self.client + .blob_client(&self.build_key(key)) + .upload(RequestContent::from(data), None) .await - .map_err(into_error)?; - - Ok(()) + .map(|_| ()) + .map_err(into_error) } - pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { - let blob_client = self.client.blob_client(self.build_key(key)); - - if let Err(e) = blob_client.delete().into_future().await { - if matches!( - e.kind(), - ErrorKind::HttpResponse { - status: StatusCode::NotFound, - .. - } - ) { - Ok(false) - } else { - Err(trc::StoreEvent::AzureError.reason(e)) - } - } else { - Ok(true) + pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { + match self + .client + .blob_client(&self.build_key(key)) + .delete(None) + .await + { + Ok(_) => Ok(true), + Err(err) if err.http_status() == Some(StatusCode::NotFound) => Ok(false), + Err(err) => Err(into_error(err)), } } fn build_key(&self, key: &[u8]) -> String { if let Some(prefix) = &self.prefix { - let mut writer = - Base32Writer::with_raw_capacity(prefix.len() + (key.len().div_ceil(4) * 5)); - writer.push_string(prefix); - writer.write_all(key).unwrap(); - writer.finalize() + let mut out = String::with_capacity(prefix.len() + STALWART.encoded_len(key.len())); + out.push_str(prefix); + STALWART.encode_append(key, &mut out); + out } else { - Base32Writer::from_bytes(key).finalize() + STALWART.encode(key) } } } -#[inline(always)] +async fn empty_range(blob_client: &BlobClient) -> trc::Result>> { + blob_client + .exists() + .await + .map(|exists| exists.then(Vec::new)) + .map_err(into_error) +} + +fn validate_account(account: &str) -> Result<(), String> { + if (3..=24).contains(&account.len()) + && account + .bytes() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit()) + { + Ok(()) + } else { + Err(format!( + "Invalid Azure storage account name {account:?}: expected 3 to 24 lowercase letters or digits" + )) + } +} + +fn validate_container(container: &str) -> Result<(), String> { + let bytes = container.as_bytes(); + let is_alnum = |ch: &u8| ch.is_ascii_lowercase() || ch.is_ascii_digit(); + if (3..=63).contains(&bytes.len()) + && bytes.first().is_some_and(is_alnum) + && bytes.last().is_some_and(is_alnum) + && bytes.iter().all(|ch| is_alnum(ch) || *ch == b'-') + && !bytes.windows(2).any(|pair| pair == b"--") + { + Ok(()) + } else { + Err(format!( + "Invalid Azure container name {container:?}: expected 3 to 63 lowercase letters, digits or single hyphens" + )) + } +} + +fn container_url( + url: Option<&str>, + account: &str, + container: &str, + environment: AzureEnvironment, +) -> Result { + let mut container_url = match url { + Some(url) => Url::parse(url).map_err(|err| format!("Invalid Azure URL {url:?}: {err}"))?, + None => Url::parse(&format!( + "https://{account}.{}", + endpoint_suffix(environment) + )) + .map_err(|err| format!("Invalid Azure storage account name: {err}"))?, + }; + container_url + .path_segments_mut() + .map_err(|_| "Azure URL cannot be used as a base URL".to_string())? + .pop_if_empty() + .push(container); + Ok(container_url) +} + +fn split_prefix(prefix: &str) -> (&str, &str) { + prefix.rsplit_once('/').unwrap_or(("", prefix)) +} + +fn blobs_url(container_url: &Url, directory: &str) -> Result { + let mut url = container_url.clone(); + url.path_segments_mut() + .map_err(|_| "Azure URL cannot be used as a base URL".to_string())? + .extend(directory.split('/').filter(|segment| !segment.is_empty())); + Ok(url) +} + +fn endpoint_suffix(environment: AzureEnvironment) -> &'static str { + match environment { + AzureEnvironment::Public => "blob.core.windows.net", + AzureEnvironment::China => "blob.core.chinacloudapi.cn", + AzureEnvironment::UsGovernment => "blob.core.usgovcloudapi.net", + } +} + +fn cloud_configuration(environment: AzureEnvironment) -> Option { + match environment { + AzureEnvironment::Public => None, + AzureEnvironment::China => Some(CloudConfiguration::AzureChina), + AzureEnvironment::UsGovernment => Some(CloudConfiguration::AzureGovernment), + } +} + +fn retry_options(timeout: Duration, max_retries: u64) -> RetryOptions { + let max_retries = u32::try_from(max_retries).unwrap_or(u32::MAX); + if max_retries == 0 { + return RetryOptions::none(); + } + let attempts = max_retries.saturating_add(1); + let max_total_elapsed = timeout + .saturating_add(RETRY_MAX_DELAY) + .saturating_mul(attempts); + RetryOptions::exponential(ExponentialRetryOptions { + max_retries, + max_total_elapsed: SdkDuration::try_from(max_total_elapsed).unwrap_or(SdkDuration::MAX), + max_delay: SdkDuration::try_from(RETRY_MAX_DELAY).unwrap_or(SdkDuration::MAX), + ..Default::default() + }) +} + +#[cold] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::AzureError.reason(err) } + diff --git a/crates/store/src/backend/azure/shared_key.rs b/crates/store/src/backend/azure/shared_key.rs new file mode 100644 index 00000000..874aa50c --- /dev/null +++ b/crates/store/src/backend/azure/shared_key.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use aws_lc_rs::hmac; +use azure_core::{ + error::{Error, ErrorKind}, + http::{ + Context, Request, + headers::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE, HeaderName, IF_MATCH, MS_DATE}, + policies::{Policy, PolicyResult}, + }, + time::{OffsetDateTime, to_rfc7231}, +}; +use encodify::base64::STANDARD; +use std::{borrow::Cow, fmt, fmt::Write, future::Future, pin::Pin, sync::Arc}; +use tinyvec::TinyVec; + +const CANONICAL_HEADER_PREFIX: &str = "x-ms-"; +const SIGNED_HEADERS_BEFORE_LENGTH: [HeaderName; 2] = [ + HeaderName::from_static("content-encoding"), + HeaderName::from_static("content-language"), +]; +const SIGNED_HEADERS_AFTER_LENGTH: [HeaderName; 8] = [ + HeaderName::from_static("content-md5"), + CONTENT_TYPE, + HeaderName::from_static("date"), + HeaderName::from_static("if-modified-since"), + IF_MATCH, + HeaderName::from_static("if-none-match"), + HeaderName::from_static("if-unmodified-since"), + HeaderName::from_static("range"), +]; + +pub(super) struct SharedKeyPolicy { + account: Box, + key: hmac::Key, +} + +impl SharedKeyPolicy { + pub(super) fn new(account: &str, account_key: &str) -> Result { + let key = STANDARD + .decode(account_key.trim()) + .map_err(|err| format!("Invalid Azure storage account key: {err}"))?; + Ok(Self { + account: account.into(), + key: hmac::Key::new(hmac::HMAC_SHA256, &key), + }) + } + + fn string_to_sign(&self, request: &Request) -> String { + let headers = request.headers(); + let url = request.url(); + let mut out = String::with_capacity(256 + url.path().len()); + + out.push_str(request.method().as_str()); + out.push('\n'); + for name in &SIGNED_HEADERS_BEFORE_LENGTH { + out.push_str(headers.get_optional_str(name).unwrap_or_default()); + out.push('\n'); + } + match headers.get_optional_str(&CONTENT_LENGTH) { + Some("0") | None => { + if let Some(len) = request.body().len().filter(|len| *len > 0) { + let _ = write!(out, "{len}"); + } + } + Some(len) => out.push_str(len), + } + out.push('\n'); + for name in &SIGNED_HEADERS_AFTER_LENGTH { + out.push_str(headers.get_optional_str(name).unwrap_or_default()); + out.push('\n'); + } + + let mut ms_headers: TinyVec<[(&str, &str); 16]> = headers + .iter() + .filter(|(name, _)| name.as_str().starts_with(CANONICAL_HEADER_PREFIX)) + .map(|(name, value)| (name.as_str(), value.as_str().trim())) + .collect(); + ms_headers.sort_unstable_by(|a, b| a.0.cmp(b.0)); + for (name, value) in ms_headers { + out.push_str(name); + out.push(':'); + out.push_str(value); + out.push('\n'); + } + + out.push('/'); + out.push_str(&self.account); + out.push_str(url.path()); + + if url.query().is_some_and(|query| !query.is_empty()) { + let mut params: TinyVec<[(Cow<'_, str>, Cow<'_, str>); 4]> = url + .query_pairs() + .map(|(name, value)| { + if name.bytes().any(|ch| ch.is_ascii_uppercase()) { + (Cow::Owned(name.to_ascii_lowercase()), value) + } else { + (name, value) + } + }) + .collect(); + params.sort_unstable(); + let mut last_name = None; + for (name, value) in ¶ms { + if last_name == Some(name.as_ref()) { + out.push(','); + } else { + out.push('\n'); + out.push_str(name); + out.push(':'); + last_name = Some(name.as_ref()); + } + out.push_str(value); + } + } + + out + } + + fn authorization(&self, request: &Request) -> String { + let signature = hmac::sign(&self.key, self.string_to_sign(request).as_bytes()); + let mut out = String::with_capacity(self.account.len() + 56); + out.push_str("SharedKey "); + out.push_str(&self.account); + out.push(':'); + STANDARD.encode_append(signature.as_ref(), &mut out); + out + } +} + +impl fmt::Debug for SharedKeyPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SharedKeyPolicy") + .field("account", &self.account) + .finish_non_exhaustive() + } +} + +impl Policy for SharedKeyPolicy { + fn send<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + ctx: &'life1 Context, + request: &'life2 mut Request, + next: &'life3 [Arc], + ) -> Pin + Send + 'async_trait>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + let [next_policy, rest @ ..] = next else { + return Err(Error::with_message( + ErrorKind::Other, + "Shared Key policy must not be the last policy in the pipeline", + )); + }; + request.insert_header(MS_DATE, to_rfc7231(&OffsetDateTime::now_utc())); + let authorization = self.authorization(request); + request.insert_header(AUTHORIZATION, authorization); + next_policy.send(ctx, request, rest).await + }) + } +} + diff --git a/crates/store/src/backend/elastic/main.rs b/crates/store/src/backend/elastic/main.rs index bf55282a..1e96dd43 100644 --- a/crates/store/src/backend/elastic/main.rs +++ b/crates/store/src/backend/elastic/main.rs @@ -10,8 +10,8 @@ use crate::{ SearchStore, backend::elastic::ElasticSearchStore, search::{ - CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, - TracingSearchField, + CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, + SearchableField, TracingSearchField, }, }; use registry::schema::structs; @@ -43,6 +43,7 @@ impl ElasticSearchStore { self.create_index::().await?; self.create_index::().await?; self.create_index::().await?; + self.create_index::().await?; self.create_index::().await?; Ok(()) } @@ -112,29 +113,6 @@ impl ElasticSearchStore { } } } - - #[cfg(feature = "test_mode")] - pub async fn drop_indexes(&self) -> trc::Result<()> { - use crate::write::SearchIndex; - - for index in &[ - SearchIndex::Email, - SearchIndex::Calendar, - SearchIndex::Contacts, - SearchIndex::Tracing, - ] { - assert_success( - self.client - .delete(format!("{}/{}", self.url, index.index_name())) - .send() - .await, - ) - .await - .map(|_| ())?; - } - - Ok(()) - } } pub(crate) async fn assert_success(response: Result) -> trc::Result { @@ -152,3 +130,4 @@ pub(crate) async fn assert_success(response: Result) -> trc::Re Err(err) => Err(trc::StoreEvent::ElasticsearchError.reason(err)), } } + diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index d0936f93..2d16796b 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -51,26 +51,15 @@ pub struct DeleteByQueryResponse { impl SearchField { pub fn es_schema(&self) -> Value { match self { - SearchField::AccountId - | SearchField::DocumentId - | SearchField::Email(EmailSearchField::Size) => json!({ + SearchField::AccountId | SearchField::DocumentId => json!({ "type": "integer" }), SearchField::Id - | SearchField::Email(EmailSearchField::SentAt | EmailSearchField::ReceivedAt) - | SearchField::Calendar(CalendarSearchField::Start) | SearchField::Tracing(TracingSearchField::QueueId | TracingSearchField::EventType) => { json!({ "type": "long" }) } - SearchField::Email(EmailSearchField::HasAttachment) => json!({ - "type": "boolean" - }), - SearchField::Calendar(CalendarSearchField::Uid) - | SearchField::Contact(ContactSearchField::Uid) => json!({ - "type": "keyword", - }), SearchField::Email( EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject, ) => json!({ @@ -87,7 +76,7 @@ impl SearchField { "enabled": true }) } - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] SearchField::Email(EmailSearchField::Bcc | EmailSearchField::Cc) => { json!({ "type": "text", @@ -98,7 +87,7 @@ impl SearchField { } }) } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] SearchField::Email(EmailSearchField::Bcc | EmailSearchField::Cc) => { json!({ "type": "text" @@ -124,7 +113,7 @@ impl SearchField { | ContactSearchField::Address | ContactSearchField::Note, ) - | SearchField::File(FileSearchField::Name | FileSearchField::Content) + | SearchField::File(FileSearchField::Content) | SearchField::Tracing(TracingSearchField::Keywords) => json!({ "type": "text" }), diff --git a/crates/store/src/backend/elastic/search.rs b/crates/store/src/backend/elastic/search.rs index b741ec2c..88193b5e 100644 --- a/crates/store/src/backend/elastic/search.rs +++ b/crates/store/src/backend/elastic/search.rs @@ -9,13 +9,13 @@ use crate::{ DeleteByQueryResponse, ElasticSearchStore, SearchResponse, main::assert_success, }, search::{ - IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter, - SearchOperator, SearchQuery, SearchValue, + IndexDocument, KeyValueMatch, SearchField, SearchFilter, SearchQuery, SearchResults, + SearchValue, TextMatch, }, write::SearchIndex, }; use serde_json::{Map, Value, json}; -use std::fmt::Write; +use std::{cmp::Ordering, fmt::Write}; impl ElasticSearchStore { pub async fn index(&self, documents: Vec) -> trc::Result<()> { @@ -54,14 +54,13 @@ impl ElasticSearchStore { .map(|_| ()) } - pub async fn query( + pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], - sort: &[SearchComparator], - ) -> trc::Result> { + ) -> trc::Result { let mut search_after: Option = None; - let mut results = Vec::new(); + let mut results = R::default(); let mut has_more = true; while has_more { @@ -70,10 +69,6 @@ impl ElasticSearchStore { Some(("query".to_string(), build_query(filters))), Some(("size".to_string(), Value::from(10_000))), Some(("_source".to_string(), Value::from(false))), - Some(( - "sort".to_string(), - build_sort(sort, R::field().field_name()), - )), search_after .take() .map(|sa| ("search_after".to_string(), sa)), @@ -107,7 +102,7 @@ impl ElasticSearchStore { for hit in response.hits.hits { search_after = hit.sort; - results.push(R::from_u64(hit.id)); + results.insert(hit.id); } } @@ -167,82 +162,81 @@ fn build_query(filters: &[SearchFilter]) -> Value { for filter in filters { match filter { - SearchFilter::Operator { field, op, value } => { - if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) - { - let SearchValue::Text { value, .. } = value else { - debug_assert!(false, "Invalid value type for text field"); - continue; - }; - - if op != &SearchOperator::Equal { - conditions.push(json!({ - "match": { field.field_name(): { - "query": value, - "operator": "and" - } } - })); - } else { - conditions.push(json!({ - "match_phrase": { field.field_name(): value } - })); + SearchFilter::Text { + field, + op, + value, + language: _, + } => { + if field.is_text() { + match op { + TextMatch::Exact => { + conditions.push(json!({ + "match_phrase": { field.field_name(): value } + })); + } + TextMatch::Standard => { + conditions.push(json!({ + "match": { field.field_name(): { + "query": value, + "operator": "and" + } } + })); + } + TextMatch::Prefix => { + conditions.push(json!({ + "match_phrase_prefix": { field.field_name(): value } + })); + } } } else { - let value = match value { - SearchValue::Text { value, .. } => json!(value), - SearchValue::Int(value) => json!(value), - SearchValue::Uint(value) => json!(value), - SearchValue::Boolean(value) => json!(value), - SearchValue::KeyValues(kv) => { - let (key, value) = kv.iter().next().unwrap(); - - let cond = if !value.is_empty() { - if op == &SearchOperator::Equal { - json!({ - "term": { - format!("{}.{}.keyword", field.field_name(), key): value - } - }) - } else { - json!({ - "match": { - format!("{}.{}", field.field_name(), key): value - } - }) - } - } else { - json!({ - "exists": { "field": format!("{}.{}", field.field_name(), key) } - }) - }; - - conditions.push(cond); - continue; - } - }; - - let cond = match op { - SearchOperator::Equal | SearchOperator::Contains => json!({ - "term": { field.field_name(): value } - }), - op => { - let op = match op { - SearchOperator::LowerThan => "lt", - SearchOperator::LowerEqualThan => "lte", - SearchOperator::GreaterThan => "gt", - SearchOperator::GreaterEqualThan => "gte", - _ => unreachable!(), - }; - - json!({ - "range": { field.field_name(): { op: value } } - }) - } + let cond = if op == &TextMatch::Prefix { + json!({ "prefix": { field.field_name(): value } }) + } else { + json!({ "term": { field.field_name(): value } }) }; conditions.push(cond); } } + SearchFilter::KeyValue { field, key, op } => { + let cond = match op { + KeyValueMatch::Equals(value) => json!({ + "term": { + format!("{}.{}.keyword", field.field_name(), key): value + } + }), + KeyValueMatch::Contains(value) => json!({ + "match": { + format!("{}.{}", field.field_name(), key): value + } + }), + KeyValueMatch::Exists => json!({ + "exists": { "field": format!("{}.{}", field.field_name(), key) } + }), + }; + + conditions.push(cond); + } + SearchFilter::Integer { field, op, value } => { + let clamped = (*value).min(i64::MAX as u64); + let cond = match op { + Ordering::Equal => json!({ + "term": { field.field_name(): clamped } + }), + Ordering::Less if *value > clamped => json!({ + "range": { field.field_name(): { "lte": clamped } } + }), + Ordering::Less => json!({ + "range": { field.field_name(): { "lt": clamped } } + }), + Ordering::Greater => json!({ + "range": { field.field_name(): { "gt": clamped } } + }), + }; + + conditions.push(cond); + } SearchFilter::And | SearchFilter::Or | SearchFilter::Not => { stack.push((logical_op, conditions)); @@ -291,30 +285,6 @@ fn build_query(filters: &[SearchFilter]) -> Value { } } -fn build_sort(sort: &[SearchComparator], tie_breaker: &str) -> Value { - Value::Array( - sort.iter() - .filter_map(|comp| match comp { - SearchComparator::Field { field, ascending } => { - let field = if field.is_text() { - format!("{}.keyword", field.field_name()) - } else { - field.field_name().to_string() - }; - - Some(json!({ - field: if *ascending { "asc" } else { "desc" } - })) - } - _ => None, - }) - .chain([json!({ - tie_breaker: "asc" - })]) - .collect(), - ) -} - fn json_serialize(request: &mut String, document: &IndexDocument) { request.push('{'); for (idx, (k, v)) in document.fields.iter().enumerate() { diff --git a/crates/store/src/backend/ephemeral/blob.rs b/crates/store/src/backend/ephemeral/blob.rs index 3ea62afb..35d25bfb 100644 --- a/crates/store/src/backend/ephemeral/blob.rs +++ b/crates/store/src/backend/ephemeral/blob.rs @@ -5,7 +5,7 @@ */ use super::EphemeralStore; -use crate::SUBSPACE_BLOBS; +use crate::Subspace; use std::ops::Range; impl EphemeralStore { @@ -17,7 +17,7 @@ impl EphemeralStore { let state = self.state.read(); Ok(state .subspaces - .get(&SUBSPACE_BLOBS) + .get(&Subspace::Blobs) .and_then(|m| m.get(key)) .map(|bytes| { if range.start == 0 && range.end == usize::MAX { @@ -35,7 +35,7 @@ impl EphemeralStore { let mut state = self.state.write(); state .subspaces - .entry(SUBSPACE_BLOBS) + .entry(Subspace::Blobs) .or_default() .insert(key.to_vec(), data.to_vec()); Ok(()) @@ -43,7 +43,7 @@ impl EphemeralStore { pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let mut state = self.state.write(); - if let Some(map) = state.subspaces.get_mut(&SUBSPACE_BLOBS) { + if let Some(map) = state.subspaces.get_mut(&Subspace::Blobs) { map.remove(key); } Ok(true) diff --git a/crates/store/src/backend/ephemeral/mod.rs b/crates/store/src/backend/ephemeral/mod.rs index f5945bef..1e47f2fa 100644 --- a/crates/store/src/backend/ephemeral/mod.rs +++ b/crates/store/src/backend/ephemeral/mod.rs @@ -9,6 +9,7 @@ pub mod main; pub mod read; pub mod write; +use crate::Subspace; use ahash::AHashMap; use parking_lot::RwLock; use std::collections::BTreeMap; @@ -18,5 +19,5 @@ pub struct EphemeralStore { } pub(crate) struct EphemeralState { - pub(crate) subspaces: AHashMap, Vec>>, + pub(crate) subspaces: AHashMap, Vec>>, } diff --git a/crates/store/src/backend/ephemeral/read.rs b/crates/store/src/backend/ephemeral/read.rs index 41036ea7..1b8980d1 100644 --- a/crates/store/src/backend/ephemeral/read.rs +++ b/crates/store/src/backend/ephemeral/read.rs @@ -5,7 +5,7 @@ */ use super::EphemeralStore; -use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; +use crate::{Deserialize, Fetch, IterateParams, Key, ValueKey, write::ValueClass}; impl EphemeralStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -40,23 +40,30 @@ impl EphemeralStore { params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { + let limit = match params.fetch { + Fetch::Limit(rows) => usize::try_from(rows).unwrap_or(usize::MAX), + Fetch::All | Fetch::Expect(_) => usize::MAX, + }; let subspace = params.begin.subspace(); let begin = params.begin.serialize(0); let end = params.end.serialize(0); + if begin > end { + return Ok(()); + } let state = self.state.read(); let Some(map) = state.subspaces.get(&subspace) else { return Ok(()); }; if params.ascending { - for (k, v) in map.range(begin..=end) { - if !cb(k.as_slice(), v.as_slice())? || params.first { + for (k, v) in map.range(begin..=end).take(limit) { + if !cb(k.as_slice(), v.as_slice())? { break; } } } else { - for (k, v) in map.range(begin..=end).rev() { - if !cb(k.as_slice(), v.as_slice())? || params.first { + for (k, v) in map.range(begin..=end).rev().take(limit) { + if !cb(k.as_slice(), v.as_slice())? { break; } } @@ -64,6 +71,32 @@ impl EphemeralStore { Ok(()) } + pub(crate) async fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + let subspace = ranges[0].begin.subspace(); + let state = self.state.read(); + let Some(map) = state.subspaces.get(&subspace) else { + return Ok(()); + }; + + 'outer: for params in ranges { + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); + if begin > end { + continue; + } + for (k, v) in map.range(begin..=end) { + if !cb(k.as_slice(), v.as_slice())? { + break 'outer; + } + } + } + Ok(()) + } + pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, diff --git a/crates/store/src/backend/ephemeral/write.rs b/crates/store/src/backend/ephemeral/write.rs index 23b87408..a46f309e 100644 --- a/crates/store/src/backend/ephemeral/write.rs +++ b/crates/store/src/backend/ephemeral/write.rs @@ -6,75 +6,64 @@ use super::EphemeralStore; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_INDEXES, - SUBSPACE_LOGS, SUBSPACE_QUOTA, + Key, Shape, Subspace, backend::deserialize_i64_le, - write::{AssignedIds, Batch, MergeResult, Operation, ValueClass, ValueOp}, + write::{Advance, AssignedIds, Batch, BatchCursor, LogSet, MergeResult, ValueOp}, }; +use std::collections::BTreeMap; impl EphemeralStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let mut result = AssignedIds::default(); - let has_changes = !batch.changes.is_empty(); + pub(crate) async fn write( + &self, + batch: Batch<'_>, + result: &mut AssignedIds, + ) -> trc::Result<()> { + let mut cursor = BatchCursor::new(&batch); + let mut log_buf = Vec::new(); + let mut log_scratch = Vec::new(); let mut state = self.state.write(); - if has_changes { - let map = state.subspaces.entry(SUBSPACE_COUNTER).or_default(); - for &account_id in batch.changes.keys() { - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); - let next = match map.get(&key) { - Some(bytes) => deserialize_i64_le(&key, bytes)? + 1, - None => 1, - }; - map.insert(key, next.to_le_bytes().to_vec()); - result.push_change_id(account_id, next as u64); + if batch.has_allocations() { + let map = state.subspaces.entry(Subspace::Counter).or_default(); + let mut key_buf = Vec::with_capacity(16); + let read = |map: &BTreeMap, Vec>, key: &[u8]| match map.get(key) { + Some(bytes) => deserialize_i64_le(key, bytes), + None => Ok(0), + }; + + for bound in batch.bounds() { + key_buf.clear(); + bound.serialize_key_into(&mut key_buf, 0); + bound.verify(read(map, &key_buf)?.saturating_add(bound.by))?; + } + + for allocation in batch.allocations() { + key_buf.clear(); + allocation.serialize_key_into(&mut key_buf, 0); + let last_id = read(map, &key_buf)? + allocation.increment_by(); + map.insert(key_buf.clone(), last_id.to_le_bytes().to_vec()); + result.apply(allocation, last_id)?; } } + let mut key_buf = Vec::with_capacity(64); for op in batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; - } - } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let subspace = class.subspace(collection); - let key = class.serialize(account_id, collection, document_id, 0); + match cursor.advance(op, result) { + Advance::Cursor => {} + Advance::Value { class, op } => { + let subspace = cursor.subspace(class); + let key = cursor.value_key_owned(class, 0); let map = state.subspaces.entry(subspace).or_default(); match op { ValueOp::Set(value) => { - map.insert(key, std::mem::take(value)); - } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; + let value = value.resolve(result)?.into_owned(); map.insert(key, value); } ValueOp::MergeFnc(merge_op) => { - let merge_result = (merge_op.fnc)( - &merge_op.params, - &result, - map.get(&key).map(|v| v.as_slice()), - )?; + let merge_result = + (merge_op.0)(result, map.get(&key).map(|v| v.as_slice()))?; match merge_result { MergeResult::Update(value) => { @@ -108,38 +97,46 @@ impl EphemeralStore { } } } - Operation::Index { field, key, set } => { - let index_key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: key.as_slice(), - } - .serialize(0); - let map = state.subspaces.entry(SUBSPACE_INDEXES).or_default(); - if *set { + Advance::Index { field, key, set } => { + cursor.index_key(field, key, &mut key_buf, 0); + let index_key = key_buf.clone(); + let map = state.subspaces.entry(Subspace::Indexes).or_default(); + if set { map.insert(index_key, Vec::new()); } else { map.remove(&index_key); } } - Operation::Log { collection, set } => { - let log_key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, - } - .serialize(0); - let map = state.subspaces.entry(SUBSPACE_LOGS).or_default(); - map.insert(log_key, std::mem::take(set)); + Advance::Log { collection, set } => { + cursor.log_key(collection, result, &mut key_buf, 0); + let log_key = key_buf.clone(); + let bytes = match set { + LogSet::Bytes(bytes) => std::mem::take(bytes), + LogSet::Pending(changes) => { + changes.serialize_into( + collection.is_prefixed(), + result, + &mut log_scratch, + &mut log_buf, + ); + std::mem::take(&mut log_buf) + } + }; + let map = state.subspaces.entry(Subspace::Logs).or_default(); + map.insert(log_key, bytes); } - Operation::AssertValue { + Advance::ClearRange { subspace, from, to } => { + if let Some(map) = state.subspaces.get_mut(&subspace) { + let mut cleared = map.split_off(from); + map.append(&mut cleared.split_off(to)); + } + } + Advance::Assert { class, assert_value, } => { - let subspace = class.subspace(collection); - let key = class.serialize(account_id, collection, document_id, 0); + let subspace = cursor.subspace(class); + let key = cursor.value_key_owned(class, 0); let matches = state .subspaces .get(&subspace) @@ -154,7 +151,7 @@ impl EphemeralStore { } } - Ok(result) + Ok(()) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { @@ -176,7 +173,11 @@ impl EphemeralStore { pub(crate) async fn purge_store(&self) -> trc::Result<()> { let mut state = self.state.write(); - for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { if let Some(map) = state.subspaces.get_mut(&subspace) { let keys: Vec> = map .iter() diff --git a/crates/store/src/backend/foundationdb/assembly.rs b/crates/store/src/backend/foundationdb/assembly.rs new file mode 100644 index 00000000..5d318410 --- /dev/null +++ b/crates/store/src/backend/foundationdb/assembly.rs @@ -0,0 +1,303 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::MAX_VALUE_SIZE; +use crate::{Fetch, write::key::KeySerializer}; +use foundationdb::future::{FdbKeyValue, FdbValues}; +use std::iter::once; + +pub(super) struct ChunkedValueCollector { + pub(super) key: Vec, + pub(super) bytes: Vec, +} + +pub(super) struct RowSink { + cb: F, + pub(super) left: u64, +} + +impl RowSink +where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, +{ + pub(super) fn new(cb: F, fetch: Fetch) -> Self { + RowSink { + cb, + left: match fetch { + Fetch::Limit(rows) => u64::from(rows), + Fetch::All | Fetch::Expect(_) => u64::MAX, + }, + } + } + + fn deliver(&mut self, key: &[u8], value: &[u8]) -> trc::Result { + self.left -= 1; + Ok((self.cb)(key.get(1..).unwrap_or_default(), value)? && self.left != 0) + } +} + +impl ChunkedValueCollector { + pub(super) fn tail_end(&self) -> Vec { + KeySerializer::new(self.key.len() + 1) + .write(self.key.as_slice()) + .write(u8::MAX) + .finalize() + } +} + +#[derive(Default)] +pub(super) struct ReverseRun { + held: Option, + key: Vec, + suffixes: Vec, + first: Vec, + chunks: Vec, + value: Vec, +} + +#[derive(Clone, Copy)] +enum ChunkLink { + Sibling, + Base, +} + +impl ChunkLink { + fn of(held: &[u8], row: &FdbKeyValue) -> Option { + let key = row.key(); + let (_, prefix) = held.split_last()?; + if row.value().len() < MAX_VALUE_SIZE { + None + } else if key == prefix { + Some(ChunkLink::Base) + } else if key.len() == held.len() && key.starts_with(prefix) { + Some(ChunkLink::Sibling) + } else { + None + } + } +} + +impl ReverseRun { + pub(super) fn members(&self) -> u64 { + self.suffixes.len() as u64 + u64::from(self.held.is_some()) + } + + fn hold(&mut self, row: &FdbKeyValue, values: bool) { + let key = row.key(); + self.key.clear(); + self.key.extend_from_slice(key); + self.suffixes.clear(); + self.suffixes.extend(key.last()); + self.first.clear(); + self.chunks.clear(); + if values { + self.first.extend_from_slice(row.value()); + } + } + + fn push(&mut self, row: &FdbKeyValue, values: bool) { + self.suffixes.extend(row.key().last()); + if values { + self.chunks.extend_from_slice(row.value()); + } + } + + fn complete( + &mut self, + base: &FdbKeyValue, + sink: &mut RowSink, + values: bool, + ) -> trc::Result + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + self.suffixes.clear(); + self.value.clear(); + if values { + self.value + .reserve(base.value().len() + self.chunks.len() + self.first.len()); + self.value.extend_from_slice(base.value()); + for chunk in self.chunks.as_chunks::().0.iter().rev() { + self.value.extend_from_slice(chunk); + } + self.value.extend_from_slice(&self.first); + } + sink.deliver(base.key(), &self.value) + } + + fn flush(&mut self, sink: &mut RowSink) -> trc::Result + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + let ReverseRun { + key, + suffixes, + first, + chunks, + .. + } = self; + let mut values = once(first.as_slice()).chain( + chunks + .as_chunks::() + .0 + .iter() + .map(|chunk| chunk.as_slice()), + ); + for suffix in suffixes.drain(..) { + if let Some(last) = key.last_mut() { + *last = suffix; + } + if !sink.deliver(key, values.next().unwrap_or_default())? { + return Ok(false); + } + } + Ok(true) + } + + fn batch( + &mut self, + rows: FdbValues, + sink: &mut RowSink, + values: bool, + ) -> trc::Result + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + let previous = self.held.take(); + let mut local = previous.as_ref().and_then(|batch| batch.last()); + for row in rows.iter() { + let held = match local { + Some(prev) => prev.key(), + None if !self.suffixes.is_empty() => self.key.as_slice(), + None => { + local = Some(row); + continue; + } + }; + match ChunkLink::of(held, row) { + Some(link) => { + if let Some(prev) = local.take() { + self.hold(prev, values); + } + match link { + ChunkLink::Sibling => self.push(row, values), + ChunkLink::Base => { + if !self.complete(row, sink, values)? { + return Ok(false); + } + } + } + } + None => { + let continues = match local.replace(row) { + Some(prev) => sink.deliver(prev.key(), prev.value())?, + None => self.flush(sink)?, + }; + if !continues { + return Ok(false); + } + } + } + } + if local.is_some() { + self.held = if rows.is_empty() { + previous + } else { + Some(rows) + }; + } + Ok(true) + } + + fn finish(&mut self, sink: &mut RowSink) -> trc::Result<()> + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + match self.held.take() { + Some(batch) => { + if let Some(row) = batch.last() { + sink.deliver(row.key(), row.value())?; + } + } + None => { + self.flush(sink)?; + } + } + Ok(()) + } +} + +pub(super) enum Assembly { + Forward(Option), + Reverse(ReverseRun), +} + +impl Assembly { + pub(super) fn batch( + &mut self, + rows: FdbValues, + sink: &mut RowSink, + values: bool, + ) -> trc::Result + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + match self { + Assembly::Forward(pending) => { + for row in rows.iter() { + let (key, value) = (row.key(), row.value()); + if let Some(chunk) = pending { + if key.len() == chunk.key.len() + 1 && key.starts_with(&chunk.key) { + if values { + chunk.bytes.extend_from_slice(value); + } + continue; + } + if values && !sink.deliver(&chunk.key, &chunk.bytes)? { + return Ok(false); + } + *pending = None; + } + if value.len() < MAX_VALUE_SIZE { + if !sink.deliver(key, value)? { + return Ok(false); + } + } else if values { + *pending = Some(ChunkedValueCollector { + key: key.to_vec(), + bytes: value.to_vec(), + }); + } else { + *pending = Some(ChunkedValueCollector { + key: key.to_vec(), + bytes: Vec::new(), + }); + if !sink.deliver(key, &[])? { + return Ok(false); + } + } + } + Ok(true) + } + Assembly::Reverse(run) => run.batch(rows, sink, values), + } + } + + pub(super) fn finish(&mut self, sink: &mut RowSink, values: bool) -> trc::Result<()> + where + F: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result, + { + match self { + Assembly::Forward(pending) => { + if values && let Some(chunk) = pending.take() { + sink.deliver(&chunk.key, &chunk.bytes)?; + } + } + Assembly::Reverse(run) => run.finish(sink)?, + } + Ok(()) + } +} diff --git a/crates/store/src/backend/foundationdb/blob.rs b/crates/store/src/backend/foundationdb/blob.rs index 459d24e7..b506dd6c 100644 --- a/crates/store/src/backend/foundationdb/blob.rs +++ b/crates/store/src/backend/foundationdb/blob.rs @@ -4,17 +4,33 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{FdbStore, MAX_VALUE_SIZE}; +use super::{FdbStore, MAX_VALUE_SIZE, into_error}; use crate::{ - IterateParams, SUBSPACE_BLOBS, - backend::foundationdb::into_error, - write::{AnyKey, key::KeySerializer}, + Subspace, + write::{commit_backoff, key::KeySerializer}, }; -use std::ops::Range; -use trc::AddContext; -use types::blob_hash::BLOB_HASH_LEN; +use foundationdb::{FdbError, KeySelector, RangeOption, Transaction, options}; +use futures::TryStreamExt; +use std::{ops::Range, time::Instant}; + +const BLOB_CHUNK_LEN: usize = std::mem::size_of::(); impl FdbStore { + fn blob_chunk_key(key: &[u8], chunk: u16) -> Vec { + KeySerializer::new(key.len() + 1 + BLOB_CHUNK_LEN) + .write(Subspace::Blobs.byte()) + .write(key) + .write(chunk) + .finalize() + } + + fn blob_chunk_range(key: &[u8]) -> (Vec, Vec) { + let mut end = Self::blob_chunk_key(key, u16::MAX); + end.push(0); + + (Self::blob_chunk_key(key, 0), end) + } + pub(crate) async fn get_blob( &self, key: &[u8], @@ -22,109 +38,120 @@ impl FdbStore { ) -> trc::Result>> { let block_start = range.start / MAX_VALUE_SIZE; let bytes_start = range.start % MAX_VALUE_SIZE; - let block_end = (range.end / MAX_VALUE_SIZE) + 1; + let blob_range = range.end.saturating_sub(range.start); - let begin = KeySerializer::new(key.len() + 2) - .write(key) - .write(block_start as u16) - .finalize(); - let end = KeySerializer::new(key.len() + 2) - .write(key) - .write(block_end as u16) - .finalize(); - let key_len = begin.len(); + if block_start > u16::MAX as usize { + return Ok(None); + } + let begin = Self::blob_chunk_key(key, block_start as u16); + let (_, end) = Self::blob_chunk_range(key); + let chunk_key_len = begin.len(); + let mut retry_count = 0; + let start = Instant::now(); let mut blob_data: Option> = None; - let blob_range = range.end - range.start; + let mut last_key: Vec = Vec::new(); + let mut trx = self.read_trx().await?; - self.iterate( - IterateParams::new( - AnyKey { - subspace: SUBSPACE_BLOBS, - key: begin, - }, - AnyKey { - subspace: SUBSPACE_BLOBS, - key: end, - }, - ), - |key, value| { - if key.len() == key_len { - if let Some(blob_data) = &mut blob_data { - blob_data.extend_from_slice( - value - .get( - ..std::cmp::min( - blob_range.saturating_sub(blob_data.len()), - value.len(), - ), - ) - .unwrap_or(&[]), - ); - if blob_data.len() == blob_range { - return Ok(false); - } - } else { - let blob_size = if blob_range <= (5 * (1 << 20)) { - blob_range - } else if value.len() == MAX_VALUE_SIZE { - MAX_VALUE_SIZE * 2 - } else { - value.len() - }; - let mut blob_data_ = Vec::with_capacity(blob_size); - blob_data_.extend_from_slice( - value - .get( - bytes_start - ..std::cmp::min(bytes_start + blob_range, value.len()), - ) - .unwrap_or(&[]), - ); - let is_done = blob_data_.len() == blob_range; - blob_data = blob_data_.into(); - if is_done { - return Ok(false); - } - } + loop { + match Self::read_blob( + &trx, + &begin, + &end, + &mut last_key, + &mut blob_data, + chunk_key_len, + bytes_start, + blob_range, + ) + .await + { + Ok(()) => return Ok(blob_data), + Err(err) => { + trx = self + .on_read_error(trx, err, &mut retry_count, start) + .await?; } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - Ok(blob_data) + } + } } - pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { - const N_CHUNKS: usize = (1 << 5) - 1; - let last_chunk = std::cmp::max( - (data.len() / MAX_VALUE_SIZE) - + if !data.len().is_multiple_of(MAX_VALUE_SIZE) { - 1 - } else { - 0 - }, - 1, - ) - 1; - let mut trx = self.db.create_trx().map_err(into_error)?; + #[allow(clippy::too_many_arguments)] + async fn read_blob( + trx: &Transaction, + begin: &[u8], + end: &[u8], + last_key: &mut Vec, + blob_data: &mut Option>, + chunk_key_len: usize, + bytes_start: usize, + blob_range: usize, + ) -> Result<(), FdbError> { + let begin_selector = if last_key.is_empty() { + KeySelector::first_greater_or_equal(begin) + } else { + KeySelector::first_greater_than(last_key.clone()) + }; + let mut chunks = trx.get_ranges_keyvalues( + RangeOption { + begin: begin_selector, + end: KeySelector::first_greater_or_equal(end), + mode: options::StreamingMode::WantAll, + ..Default::default() + }, + true, + ); - for (chunk_pos, chunk_bytes) in data.chunks(MAX_VALUE_SIZE).enumerate() { - trx.set( - &KeySerializer::new(key.len() + 3) - .write(SUBSPACE_BLOBS) - .write(key) - .write(chunk_pos as u16) - .finalize(), - chunk_bytes, - ); - if chunk_pos == last_chunk || (chunk_pos > 0 && chunk_pos % N_CHUNKS == 0) { - self.commit(trx, false).await?; - if chunk_pos < last_chunk { - trx = self.db.create_trx().map_err(into_error)?; - } else { - break; + while let Some(chunk) = chunks.try_next().await? { + let chunk_key = chunk.key(); + if chunk_key.len() != chunk_key_len { + continue; + } + last_key.clear(); + last_key.extend_from_slice(chunk_key); + let value = chunk.value(); + + match blob_data { + Some(blob_data) => { + blob_data.extend_from_slice( + value + .get( + ..std::cmp::min( + blob_range.saturating_sub(blob_data.len()), + value.len(), + ), + ) + .unwrap_or(&[]), + ); + if blob_data.len() == blob_range { + break; + } + } + None => { + let blob_size = if blob_range <= (5 * (1 << 20)) { + blob_range + } else if value.len() == MAX_VALUE_SIZE { + MAX_VALUE_SIZE * 2 + } else { + value.len() + }; + let mut data = Vec::with_capacity(blob_size); + data.extend_from_slice( + value + .get( + bytes_start + ..std::cmp::min( + bytes_start.saturating_add(blob_range), + value.len(), + ), + ) + .unwrap_or(&[]), + ); + let is_done = data.len() == blob_range; + *blob_data = Some(data); + if is_done { + break; + } } } } @@ -132,25 +159,80 @@ impl FdbStore { Ok(()) } + pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { + const N_CHUNKS: usize = (1 << 5) - 1; + + let chunks = data.chunks(MAX_VALUE_SIZE).collect::>(); + if chunks.len() > u16::MAX as usize + 1 { + return Err(trc::StoreEvent::FoundationdbError + .ctx(trc::Key::Reason, "Blob exceeds the maximum chunk count")); + } + + if chunks.is_empty() { + return self.delete_blob(key).await.map(|_| ()); + } + + for (pos, group) in chunks.chunks(N_CHUNKS).enumerate() { + self.put_blob_chunks(key, pos * N_CHUNKS, group).await?; + } + + Ok(()) + } + + async fn put_blob_chunks( + &self, + key: &[u8], + first_chunk: usize, + chunks: &[&[u8]], + ) -> trc::Result<()> { + let start = Instant::now(); + let mut retry_count = 0; + + loop { + let trx = self.db.create_trx().map_err(into_error)?; + if first_chunk == 0 { + let (begin, end) = Self::blob_chunk_range(key); + trx.clear_range(&begin, &end); + } + for (pos, bytes) in chunks.iter().enumerate() { + trx.set( + &Self::blob_chunk_key(key, (first_chunk + pos) as u16), + bytes, + ); + } + + if self.commit_attempt(trx, retry_count, start).await? { + return Ok(()); + } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } + } + pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { - if key.len() < BLOB_HASH_LEN { + if key.is_empty() { return Ok(false); } - let trx = self.db.create_trx().map_err(into_error)?; - trx.clear_range( - &KeySerializer::new(key.len() + 3) - .write(SUBSPACE_BLOBS) - .write(key) - .write(0u16) - .finalize(), - &KeySerializer::new(key.len() + 3) - .write(SUBSPACE_BLOBS) - .write(key) - .write(u16::MAX) - .finalize(), - ); + let (begin, end) = Self::blob_chunk_range(key); + let start = Instant::now(); + let mut retry_count = 0; - self.commit(trx, false).await + loop { + let trx = self.db.create_trx().map_err(into_error)?; + trx.clear_range(&begin, &end); + + if self.commit_attempt(trx, retry_count, start).await? { + return Ok(true); + } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } } } diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index 1a044fee..52d7e7b3 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -13,6 +13,10 @@ use std::sync::Arc; static FDB_NETWORK: Mutex> = Mutex::new(None); +extern "C" fn stop_fdb_network() { + drop(FDB_NETWORK.lock().take()); +} + impl FdbStore { pub async fn open(config: structs::FoundationDbStore) -> Result { { @@ -26,6 +30,7 @@ impl FdbStore { .map_err(|err| format!("Failed to boot FoundationDB: {err:?}"))? }; *guard = Some(network); + unsafe { libc::atexit(stop_fdb_network) }; } } @@ -60,6 +65,7 @@ impl FdbStore { Ok(Store::FoundationDb(Arc::new(Self { db, version: Default::default(), + windows: Default::default(), }))) } } diff --git a/crates/store/src/backend/foundationdb/mod.rs b/crates/store/src/backend/foundationdb/mod.rs index f189a7e0..485dd20f 100644 --- a/crates/store/src/backend/foundationdb/mod.rs +++ b/crates/store/src/backend/foundationdb/mod.rs @@ -10,12 +10,20 @@ use std::{ time::{Duration, Instant}, }; +mod assembly; pub mod blob; pub mod main; +mod preread; pub mod read; +mod window; pub mod write; -const MAX_VALUE_SIZE: usize = 100000; +pub(crate) const MAX_VALUE_SIZE: usize = 100000; +pub(crate) const FDB_VALUE_TOO_LARGE: i32 = 2103; +const FDB_BATCH_THROTTLED: i32 = 1051; +const MAX_THROTTLED_TIME: Duration = Duration::from_secs(15 * 60); + +const MAX_EXACT_MODE_ROWS: u64 = 256; const REFRESH_READ_VERSION_AFTER: Duration = Duration::from_secs(1); const MAX_READ_VERSION_AGE: Duration = Duration::from_secs(4); @@ -23,11 +31,13 @@ const MAX_READ_VERSION_AGE: Duration = Duration::from_secs(4); pub struct FdbStore { db: Database, version: ReadVersion, + windows: window::WindowLoad, } pub(crate) struct ReadVersion { base: Instant, version: AtomicI64, + committed: AtomicI64, obtained: AtomicU64, refreshing: AtomicBool, } @@ -46,7 +56,7 @@ impl ReadVersion { .saturating_sub(self.obtained.load(Ordering::Acquire)) } - fn store_max(&self, version: i64) { + fn store_max(&self, version: i64) -> bool { let mut current = self.version.load(Ordering::Relaxed); while version > current { match self.version.compare_exchange_weak( @@ -55,10 +65,11 @@ impl ReadVersion { Ordering::Release, Ordering::Relaxed, ) { - Ok(_) => break, + Ok(_) => return true, Err(actual) => current = actual, } } + false } fn refreshed(&self, version: i64) { @@ -67,9 +78,20 @@ impl ReadVersion { } fn raise_floor(&self, version: i64) { + self.committed.fetch_max(version, Ordering::AcqRel); self.store_max(version); } + fn advance(&self, version: i64) { + if self.store_max(version) { + self.obtained.store(self.now(), Ordering::Release); + } + } + + fn last_committed(&self) -> Option { + Some(self.committed.load(Ordering::Acquire)).filter(|version| *version > 0) + } + fn expire(&self) { self.version.store(0, Ordering::Release); } @@ -92,6 +114,7 @@ impl Default for ReadVersion { Self { base: Instant::now(), version: AtomicI64::new(0), + committed: AtomicI64::new(0), obtained: AtomicU64::new(0), refreshing: AtomicBool::new(false), } @@ -112,3 +135,4 @@ fn into_error(error: FdbError) -> trc::Error { .reason(error.message()) .ctx(trc::Key::Code, error.code()) } + diff --git a/crates/store/src/backend/foundationdb/preread.rs b/crates/store/src/backend/foundationdb/preread.rs new file mode 100644 index 00000000..da59c9e6 --- /dev/null +++ b/crates/store/src/backend/foundationdb/preread.rs @@ -0,0 +1,189 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + MAX_VALUE_SIZE, + window::{Isolation, WindowError, WindowLoad, WindowRead, WindowRows, read_window}, +}; +use crate::{ + WITH_SUBSPACE, + write::{Advance, AssignedIds, Batch, BatchCursor, Operation, ValueOp}, +}; +use ahash::AHashSet; +use foundationdb::{FdbError, Transaction}; +use std::ops::Range; +use xxhash_rust::xxh3::xxh3_64; + +const MIN_PREREAD_MERGES: usize = 2; + +enum Slot { + Sequential, + Missing, + Value(Range), + Head(Range), +} + +pub(super) enum Prefetched<'x> { + Value(Option<&'x [u8]>), + Head(&'x [u8]), + Sequential, +} + +#[derive(Default)] +pub(super) struct MergePreRead { + keys: Vec, + reads: Vec<(Range, usize)>, + values: Vec, + slots: Vec, + next: usize, + reads_own_writes: bool, +} + +fn window_reads<'k>( + keys: &'k [u8], + reads: &'k [(Range, usize)], +) -> impl Iterator)> + Send + use<'k> { + reads + .iter() + .enumerate() + .filter_map(move |(index, (range, _))| { + keys.get(range.clone()) + .map(|key| (index, WindowRead::Key(key))) + }) +} + +impl MergePreRead { + pub(super) fn plan(batch: &mut Batch<'_>, ids: &mut AssignedIds) -> Option { + let merges = batch + .ops + .iter() + .filter(|op| { + matches!( + op, + Operation::Value { + op: ValueOp::MergeFnc(_), + .. + } + ) + }) + .take(MIN_PREREAD_MERGES) + .count(); + if merges < MIN_PREREAD_MERGES { + return None; + } + + let mark = ids.mark(); + let mut plan = MergePreRead::default(); + let mut cursor = BatchCursor::new(batch); + let mut written = AHashSet::new(); + let mut range_cleared = false; + let mut key = Vec::with_capacity(64); + for op in batch.ops.iter_mut() { + match cursor.advance(op, ids) { + Advance::Value { class, op } => { + cursor.value_key(class, &mut key, WITH_SUBSPACE); + let first_write = written.insert(xxh3_64(&key)); + match op { + ValueOp::MergeFnc(_) if first_write && !range_cleared => { + let start = plan.keys.len(); + plan.keys.extend_from_slice(&key); + plan.reads.push((start..plan.keys.len(), plan.slots.len())); + plan.slots.push(Slot::Missing); + } + ValueOp::MergeFnc(_) => { + plan.slots.push(Slot::Sequential); + plan.reads_own_writes = true; + } + ValueOp::AddAndGet(_) => plan.reads_own_writes = true, + _ => {} + } + } + Advance::Assert { .. } => plan.reads_own_writes = true, + Advance::ClearRange { .. } => { + range_cleared = true; + plan.reads_own_writes = true; + } + _ => {} + } + } + ids.rollback(mark); + Some(plan) + } + + pub(super) async fn read( + &mut self, + trx: &Transaction, + load: &WindowLoad, + ) -> Result<(), FdbError> { + let MergePreRead { + keys, + reads, + values, + slots, + next, + .. + } = self; + *next = 0; + values.clear(); + for (_, slot) in reads.iter() { + if let Some(slot) = slots.get_mut(*slot) { + *slot = Slot::Missing; + } + } + let result = read_window( + trx, + load, + WindowRows::Raw, + Isolation::Serializable, + window_reads(keys, reads), + |index, _, value| { + if let Some(slot) = reads.get(index).and_then(|(_, slot)| slots.get_mut(*slot)) { + let start = values.len(); + values.extend_from_slice(value); + *slot = if value.len() < MAX_VALUE_SIZE { + Slot::Value(start..values.len()) + } else { + Slot::Head(start..values.len()) + }; + } + Ok(true) + }, + ) + .await; + match result { + Ok(()) => Ok(()), + Err(WindowError::Store(_)) => { + slots.fill_with(|| Slot::Sequential); + Ok(()) + } + Err(WindowError::Fdb { error, .. }) => Err(error), + } + } + + pub(super) fn reads_own_writes(&self) -> bool { + self.reads_own_writes + } + + pub(super) fn next_merge(&mut self) -> Prefetched<'_> { + let index = self.next; + self.next += 1; + match self.slots.get(index) { + Some(Slot::Value(range)) => self + .values + .get(range.clone()) + .map_or(Prefetched::Sequential, |value| { + Prefetched::Value(Some(value)) + }), + Some(Slot::Head(range)) => self + .values + .get(range.clone()) + .map_or(Prefetched::Sequential, Prefetched::Head), + Some(Slot::Missing) => Prefetched::Value(None), + Some(Slot::Sequential) | None => Prefetched::Sequential, + } + } +} + diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 7694fe76..e108166c 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -5,20 +5,27 @@ */ use super::{ - FdbStore, MAX_READ_VERSION_AGE, MAX_VALUE_SIZE, REFRESH_READ_VERSION_AFTER, into_error, + FdbStore, MAX_EXACT_MODE_ROWS, MAX_READ_VERSION_AGE, MAX_VALUE_SIZE, + REFRESH_READ_VERSION_AFTER, + assembly::{Assembly, ReverseRun, RowSink}, + into_error, + window::{Isolation, Resume, WindowError, WindowRows, read_window}, }; use crate::{ - Deserialize, IterateParams, Key, ValueKey, WITH_SUBSPACE, + Deserialize, Fetch, ITERATE_BATCH_GROWTH, IterateParams, Key, ValueKey, WITH_SUBSPACE, backend::deserialize_i64_le, write::{MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, ValueClass, key::KeySerializer}, }; use foundationdb::{ FdbError, KeySelector, RangeOption, Transaction, future::FdbSlice, - options::{self}, + options::{self, TransactionOption}, }; use futures::TryStreamExt; -use std::time::Instant; +use std::{ + mem::{swap, take}, + time::Instant, +}; #[allow(dead_code)] pub(crate) enum ChunkedValue { @@ -27,11 +34,6 @@ pub(crate) enum ChunkedValue { None, } -struct ChunkedValueCollector { - key: Vec, - bytes: Vec, -} - impl FdbStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where @@ -40,10 +42,9 @@ impl FdbStore { let key = key.serialize(WITH_SUBSPACE); let mut retry_count = 0; let start = Instant::now(); + let mut trx = self.read_trx().await?; loop { - let trx = self.read_trx().await?; - match read_chunked_value(&key, &trx, true).await { Ok(ChunkedValue::Single(bytes)) => { return U::deserialize_with_key(key.get(1..).unwrap_or_default(), &bytes) @@ -55,7 +56,8 @@ impl FdbStore { } Ok(ChunkedValue::None) => return Ok(None), Err(err) => { - self.on_read_error(trx, err, &mut retry_count, start) + trx = self + .on_read_error(trx, err, &mut retry_count, start) .await?; } } @@ -66,15 +68,14 @@ impl FdbStore { let key = key.serialize(WITH_SUBSPACE); let mut retry_count = 0; let start = Instant::now(); + let mut trx = self.read_trx().await?; loop { - let trx = self.read_trx().await?; - - match read_chunked_value(&key, &trx, true).await { - Ok(ChunkedValue::Single(_) | ChunkedValue::Chunked { .. }) => return Ok(true), - Ok(ChunkedValue::None) => return Ok(false), + match trx.get(&key, true).await { + Ok(value) => return Ok(value.is_some()), Err(err) => { - self.on_read_error(trx, err, &mut retry_count, start) + trx = self + .on_read_error(trx, err, &mut retry_count, start) .await?; } } @@ -84,152 +85,195 @@ impl FdbStore { pub(crate) async fn iterate( &self, params: IterateParams, - mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let begin = params.begin.serialize(WITH_SUBSPACE); let end = params.end.serialize(WITH_SUBSPACE); + let mut sink = RowSink::new(cb, params.fetch); + let mut assembly = if params.ascending { + Assembly::Forward(None) + } else { + Assembly::Reverse(ReverseRun::default()) + }; + let mut expect_window = match params.fetch { + Fetch::Expect(rows) => Some((u64::from(rows) + 1).max(MIN_EXPECT_WINDOW)), + Fetch::All | Fetch::Limit(_) => None, + }; let mut retry_count = 0; let start = Instant::now(); + let mut last_key = vec![]; + let mut read_key = vec![]; + let mut next_trx: Option = None; + let mut carried = false; - if !params.first { - let mut last_key = vec![]; - let mut chunked_key: Option = None; - - 'outer: loop { - let begin_selector = if last_key.is_empty() { - KeySelector::first_greater_or_equal(&begin) - } else { - KeySelector::first_greater_than(&last_key) + 'outer: loop { + let (begin_selector, end_selector) = if last_key.is_empty() { + ( + KeySelector::first_greater_or_equal(&begin), + KeySelector::first_greater_than(&end), + ) + } else if params.ascending { + ( + KeySelector::first_greater_than(&last_key), + KeySelector::first_greater_than(&end), + ) + } else { + ( + KeySelector::first_greater_or_equal(&begin), + KeySelector::first_greater_or_equal(&last_key), + ) + }; + let tail_end: Vec; + let mut range = RangeOption { + begin: begin_selector, + end: end_selector, + mode: options::StreamingMode::WantAll, + reverse: !params.ascending, + ..Default::default() + }; + let rows = match params.fetch { + Fetch::All => None, + Fetch::Expect(_) => expect_window, + Fetch::Limit(_) => match &assembly { + Assembly::Forward(Some(chunk)) if params.values && sink.left == 1 => { + tail_end = chunk.tail_end(); + if tail_end <= end { + range.end = KeySelector::first_greater_or_equal(&tail_end); + } + None + } + Assembly::Forward(pending) => { + Some(sink.left - u64::from(params.values && pending.is_some())) + } + Assembly::Reverse(run) => Some(sink.left.saturating_sub(run.members()) + 1), + }, + }; + if let Some(rows) = rows { + range.limit = Some(fetch_limit(rows)); + range.mode = match params.fetch { + Fetch::Expect(_) => options::StreamingMode::WantAll, + Fetch::All | Fetch::Limit(_) => bounded_mode(rows), }; - - let trx = self.read_trx().await?; - let mut values = trx.get_ranges( - RangeOption { - begin: begin_selector, - end: KeySelector::first_greater_than(&end), - mode: options::StreamingMode::WantAll, - reverse: !params.ascending, - ..Default::default() - }, - true, - ); - - let mut last_key_ = vec![]; - loop { - match values.try_next().await { - Ok(Some(values)) => { - let mut key = &[] as &[u8]; - for value in values.iter() { - key = value.key(); - - // Check whether we are collecting a chunked value - let cb_key = key.get(1..).unwrap_or_default(); - let cb_value = value.value(); - - if let Some(chunk) = &mut chunked_key { - if chunk.key.len() + 1 == cb_key.len() - && cb_key[..chunk.key.len()] == chunk.key[..] - { - // This is a chunk of the current value - if params.values { - chunk.bytes.extend_from_slice(cb_value); - } - continue; - } else { - // Return collected chunked value - if !cb(&chunk.key, &chunk.bytes)? { - return Ok(()); - } - - // Reset collector - chunked_key = None; - } - } - - if cb_value.len() < MAX_VALUE_SIZE { - if !cb(cb_key, cb_value)? { - return Ok(()); - } - } else { - // Start collecting chunked value - chunked_key = Some(ChunkedValueCollector { - key: cb_key.to_vec(), - bytes: if params.values { - cb_value.to_vec() - } else { - Vec::new() - }, - }); - } - } - if values.more() { - last_key_ = key.to_vec(); - } - } - Ok(None) => { - // Return any chunked value collected - if let Some(chunked_key) = chunked_key.take() { - cb(&chunked_key.key, &chunked_key.bytes)?; - } - - break 'outer; - } - Err(e) => { - drop(values); - if e.code() == 1007 && !last_key_.is_empty() { - // Transaction is too old to perform reads or be committed - last_key = last_key_; - continue 'outer; - } else if e.is_retryable() - && retry_count < MAX_COMMIT_ATTEMPTS - && start.elapsed() < MAX_COMMIT_TIME - { - // Transient error such as a cached read version ahead of lagging - // storage servers (code 1009); resume from the last key read, - // refresh the read version and back off before retrying. - if !last_key_.is_empty() { - last_key = last_key_; - } - self.version.expire(); - trx.on_error(e).await.map_err(into_error)?; - retry_count += 1; - continue 'outer; - } else { - return Err(into_error(e)); - } - } - } - } } - } else { - loop { - let trx = self.read_trx().await?; - let mut values = trx.get_ranges_keyvalues( - RangeOption { - begin: KeySelector::first_greater_or_equal(&begin), - end: KeySelector::first_greater_than(&end), - mode: options::StreamingMode::Small, - reverse: !params.ascending, - ..Default::default() - }, - true, - ); - match values.try_next().await { - Ok(Some(value)) => { - cb(value.key().get(1..).unwrap_or_default(), value.value())?; - break; + let trx = match next_trx.take() { + Some(trx) => trx, + None => self.read_trx().await?, + }; + let progressed = take(&mut carried); + let mut stream = trx.get_ranges(range, true); + read_key.clear(); + + loop { + match stream.try_next().await { + Ok(Some(values)) => { + #[cfg(stalwart_tests)] + super::probe::range_batch(&values); + read_key.clear(); + if values.more() + && let Some(last) = values.last() + { + read_key.extend_from_slice(last.key()); + } + if !assembly.batch(values, &mut sink, params.values)? { + return Ok(()); + } + } + Ok(None) => { + drop(stream); + if read_key.is_empty() { + return assembly.finish(&mut sink, params.values); + } + expect_window = expect_window + .map(|rows| rows.saturating_mul(u64::from(ITERATE_BATCH_GROWTH))); + swap(&mut last_key, &mut read_key); + next_trx = Some(trx); + carried = true; + continue 'outer; } - Ok(None) => break, Err(e) => { - drop(values); - self.on_read_error(trx, e, &mut retry_count, start).await?; + drop(stream); + if e.code() == 1007 && (progressed || !read_key.is_empty()) { + // Transaction is too old to perform reads or be committed + if !read_key.is_empty() { + swap(&mut last_key, &mut read_key); + } + self.version.expire(); + continue 'outer; + } else if e.is_retryable() + && retry_count < MAX_COMMIT_ATTEMPTS + && start.elapsed() < MAX_COMMIT_TIME + { + // Transient error such as a cached read version ahead of lagging + // storage servers (code 1009); resume from the last key read, + // refresh the read version and back off before retrying. + if !read_key.is_empty() { + swap(&mut last_key, &mut read_key); + } + self.version.expire(); + let trx = trx.on_error(e).await.map_err(into_error)?; + next_trx = Some(self.apply_read_version(trx).await?); + retry_count += 1; + continue 'outer; + } else { + return Err(into_error(e)); + } } } } } + } - Ok(()) + pub(crate) async fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + let rows = if ranges.first().is_none_or(|params| params.values) { + WindowRows::Values + } else { + WindowRows::Keys + }; + let bounds = ranges + .iter() + .map(|params| { + ( + params.begin.serialize(WITH_SUBSPACE), + params.end.serialize(WITH_SUBSPACE), + ) + }) + .collect::>(); + let mut resume = Resume::default(); + let mut retry_count = 0; + let start = Instant::now(); + let mut trx = self.read_trx().await?; + + loop { + let result = read_window( + &trx, + &self.windows, + rows, + Isolation::Snapshot, + resume.reads(&bounds), + |_, key, value| cb(key.get(1..).unwrap_or_default(), value), + ) + .await; + + match result { + Ok(()) => return Ok(()), + Err(WindowError::Store(err)) => return Err(err), + Err(WindowError::Fdb { + error, + resume: next, + progressed, + }) => { + resume = next; + trx = self + .on_window_error(trx, error, progressed, &mut retry_count, start) + .await?; + } + } + } } pub(crate) async fn get_counter( @@ -239,26 +283,26 @@ impl FdbStore { let key = key.into().serialize(WITH_SUBSPACE); let mut retry_count = 0; let start = Instant::now(); + let mut trx = self.read_trx().await?; loop { - let trx = self.read_trx().await?; match trx.get(&key, true).await { Ok(Some(bytes)) => return deserialize_i64_le(&key, &bytes), Ok(None) => return Ok(0), Err(e) => { - self.on_read_error(trx, e, &mut retry_count, start).await?; + trx = self.on_read_error(trx, e, &mut retry_count, start).await?; } } } } - async fn on_read_error( + pub(super) async fn on_read_error( &self, trx: Transaction, err: FdbError, retry_count: &mut u32, start: Instant, - ) -> trc::Result<()> { + ) -> trc::Result { if err.is_retryable() && *retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME @@ -267,16 +311,38 @@ impl FdbStore { // load (code 1009); expire it so the retry obtains a fresh read version, then let // FoundationDB back off before retrying. self.version.expire(); - trx.on_error(err).await.map_err(into_error)?; + let trx = trx.on_error(err).await.map_err(into_error)?; *retry_count += 1; - Ok(()) + self.apply_read_version(trx).await } else { Err(into_error(err)) } } + pub(super) async fn on_window_error( + &self, + trx: Transaction, + err: FdbError, + progressed: bool, + retry_count: &mut u32, + start: Instant, + ) -> trc::Result { + if err.code() == 1007 && progressed { + self.version.expire(); + self.read_trx().await + } else { + self.on_read_error(trx, err, retry_count, start).await + } + } + pub(crate) async fn read_trx(&self) -> trc::Result { let trx = self.db.create_trx().map_err(into_error)?; + self.apply_read_version(trx).await + } + + pub(super) async fn apply_read_version(&self, trx: Transaction) -> trc::Result { + trx.set_option(TransactionOption::ReadYourWritesDisable) + .map_err(into_error)?; let version = self.version.current(); let age = self.version.age(); @@ -300,6 +366,32 @@ impl FdbStore { pub(crate) fn invalidate_read_snapshot(&self) { self.version.expire(); } + + pub(crate) fn read_snapshot_version(&self) -> Option { + self.version + .last_committed() + .and_then(|version| u64::try_from(version).ok()) + } + + pub(crate) fn advance_read_snapshot(&self, version: u64) { + if let Ok(version) = i64::try_from(version) { + self.version.advance(version); + } + } +} + +const MIN_EXPECT_WINDOW: u64 = 64; + +fn bounded_mode(rows: u64) -> options::StreamingMode { + if rows <= MAX_EXACT_MODE_ROWS { + options::StreamingMode::Exact + } else { + options::StreamingMode::WantAll + } +} + +fn fetch_limit(rows: u64) -> usize { + usize::try_from(rows).unwrap_or(usize::MAX) } pub(crate) async fn read_chunked_value( @@ -309,26 +401,50 @@ pub(crate) async fn read_chunked_value( ) -> Result { if let Some(bytes) = trx.get(key, snapshot).await? { if bytes.len() < MAX_VALUE_SIZE { - Ok(ChunkedValue::Single(bytes)) - } else { - let mut value = Vec::with_capacity(bytes.len() * 2); - value.extend_from_slice(&bytes); - let mut key = KeySerializer::new(key.len() + 1) - .write(key) - .write(0u8) - .finalize(); - - while let Some(bytes) = trx.get(&key, snapshot).await? { - value.extend_from_slice(&bytes); - *key.last_mut().unwrap() += 1; - } - - Ok(ChunkedValue::Chunked { - bytes: value, - n_chunks: *key.last().unwrap(), - }) + return Ok(ChunkedValue::Single(bytes)); } + Ok(match read_chunk_tail(key, trx, snapshot, &bytes).await? { + Some(chunked) => chunked, + None => ChunkedValue::Single(bytes), + }) } else { Ok(ChunkedValue::None) } } + +pub(crate) async fn read_chunk_tail( + key: &[u8], + trx: &Transaction, + snapshot: bool, + head: &[u8], +) -> Result, FdbError> { + let mut chunk_key = KeySerializer::new(key.len() + 1) + .write(key) + .write(0u8) + .finalize(); + let Some(chunk) = trx.get(&chunk_key, snapshot).await? else { + return Ok(None); + }; + + let mut value = Vec::with_capacity(head.len() + chunk.len()); + value.extend_from_slice(head); + value.extend_from_slice(&chunk); + let mut n_chunks = 1u8; + loop { + if let Some(last) = chunk_key.last_mut() { + *last = n_chunks; + } + match trx.get(&chunk_key, snapshot).await? { + Some(chunk) => { + value.extend_from_slice(&chunk); + n_chunks += 1; + } + None => break, + } + } + + Ok(Some(ChunkedValue::Chunked { + bytes: value, + n_chunks, + })) +} diff --git a/crates/store/src/backend/foundationdb/window.rs b/crates/store/src/backend/foundationdb/window.rs new file mode 100644 index 00000000..84f4a3b3 --- /dev/null +++ b/crates/store/src/backend/foundationdb/window.rs @@ -0,0 +1,430 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{MAX_VALUE_SIZE, assembly::ChunkedValueCollector}; +use foundationdb::{ + FdbError, FdbResult, KeySelector, RangeOption, Transaction, + future::{FdbSlice, FdbValues}, + options::StreamingMode, +}; +use futures::{FutureExt, StreamExt, future::Either, stream::FuturesOrdered}; +use std::{ + cmp::Ordering, + mem, + sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}, +}; + +const WINDOW_BUDGET: usize = 64; +const MIN_WINDOW: usize = 16; +const MAX_WINDOW: usize = 64; +const BATCH_BYTES: usize = 1 << 20; + +#[derive(Default)] +pub(crate) struct WindowLoad(AtomicUsize); + +struct WindowSlot<'x>(&'x AtomicUsize); + +#[derive(Clone, Copy)] +pub(super) enum WindowRead<'k> { + Key(&'k [u8]), + Range { begin: &'k [u8], end: &'k [u8] }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum WindowRows { + Values, + Keys, + Raw, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Isolation { + Snapshot, + Serializable, +} + +pub(super) enum WindowError { + Fdb { + error: FdbError, + resume: Resume, + progressed: bool, + }, + Store(trc::Error), +} + +#[derive(Default)] +pub(super) struct Resume { + index: usize, + begin: Option>, +} + +enum Issued<'k> { + Exists(&'k [u8], FdbResult), + Value(&'k [u8], FdbResult>), + Rows(&'k [u8], &'k [u8], FdbResult), +} + +struct Head { + rows: WindowRows, + snapshot: bool, + largest: usize, + progressed: bool, + last_key: Vec, + pending: Option, +} + +pub(super) async fn read_window<'k>( + trx: &Transaction, + load: &WindowLoad, + rows: WindowRows, + isolation: Isolation, + reads: impl IntoIterator)>, + mut cb: impl FnMut(usize, &[u8], &[u8]) -> trc::Result + Send, +) -> Result<(), WindowError> { + let _slot = load.enter(); + let mut reads = reads.into_iter(); + let mut window = FuturesOrdered::new(); + let mut width = MIN_WINDOW; + let snapshot = isolation == Isolation::Snapshot; + let mut head = Head { + rows, + snapshot, + largest: 0, + progressed: false, + last_key: Vec::new(), + pending: None, + }; + + loop { + while window.len() < width + && let Some((index, read)) = reads.next() + { + window.push_back(issue(trx, rows, snapshot, index, read)); + } + let Some((index, issued)) = window.next().await else { + return Ok(()); + }; + if !head.complete(trx, index, issued, &mut cb).await? { + return Ok(()); + } + width = (BATCH_BYTES / head.largest.max(1)).clamp(1, load.width()); + } +} + +impl WindowLoad { + fn enter(&self) -> WindowSlot<'_> { + self.0.fetch_add(1, AtomicOrdering::Relaxed); + WindowSlot(&self.0) + } + + fn width(&self) -> usize { + (WINDOW_BUDGET / self.0.load(AtomicOrdering::Relaxed).max(1)).clamp(MIN_WINDOW, MAX_WINDOW) + } +} + +impl Drop for WindowSlot<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, AtomicOrdering::Relaxed); + } +} + +impl Resume { + pub(super) fn reads<'k>( + &'k self, + bounds: &'k [(Vec, Vec)], + ) -> impl Iterator)> + Send { + bounds + .iter() + .enumerate() + .skip(self.index) + .filter_map(move |(index, (begin, end))| { + let begin = match &self.begin { + Some(resumed) if index == self.index => resumed, + _ => begin, + }; + match begin.cmp(end) { + Ordering::Less => Some((index, WindowRead::Range { begin, end })), + Ordering::Equal => Some((index, WindowRead::Key(begin))), + Ordering::Greater => None, + } + }) + } +} + +fn issue<'k>( + trx: &Transaction, + rows: WindowRows, + snapshot: bool, + index: usize, + read: WindowRead<'k>, +) -> impl Future)> + Send + use<'k> { + match read { + WindowRead::Key(key) if rows == WindowRows::Keys => Either::Left(Either::Left( + trx.get_key(&KeySelector::first_greater_or_equal(key), snapshot) + .map(move |found| (index, Issued::Exists(key, found))), + )), + WindowRead::Key(key) => Either::Left(Either::Right( + trx.get(key, snapshot) + .map(move |value| (index, Issued::Value(key, value))), + )), + WindowRead::Range { begin, end } => Either::Right( + trx.get_range( + &range_option( + KeySelector::first_greater_or_equal(begin), + KeySelector::first_greater_than(end), + ), + 1, + snapshot, + ) + .map(move |values| (index, Issued::Rows(begin, end, values))), + ), + } +} + +impl Head { + async fn complete( + &mut self, + trx: &Transaction, + index: usize, + issued: Issued<'_>, + cb: &mut F, + ) -> Result + where + F: FnMut(usize, &[u8], &[u8]) -> trc::Result + Send, + { + match issued { + Issued::Exists(key, found) => { + let found = found.map_err(|error| self.fail(error, index, None))?; + if *found == *key { + self.deliver(cb, index, key, &[]) + } else { + self.progressed = true; + Ok(true) + } + } + Issued::Value(key, value) => { + match value.map_err(|error| self.fail(error, index, None))? { + Some(value) if value.len() < MAX_VALUE_SIZE || self.rows == WindowRows::Raw => { + self.deliver(cb, index, key, &value) + } + Some(value) => { + let mut chunk = ChunkedValueCollector { + key: key.to_vec(), + bytes: value.to_vec(), + }; + drop(value); + let tail = chunk.tail_end(); + if let Err(error) = read_chunks( + trx, + &chunk.key, + &chunk.key, + &tail, + &mut chunk.bytes, + self.snapshot, + ) + .await + { + return Err(self.fail(error, index, None)); + } + self.deliver(cb, index, key, &chunk.bytes) + } + None => { + self.progressed = true; + Ok(true) + } + } + } + Issued::Rows(begin, end, values) => { + self.last_key.clear(); + let values = values.map_err(|error| self.fail(error, index, Some(begin)))?; + self.range(trx, index, begin, end, values, cb).await + } + } + } + + async fn range( + &mut self, + trx: &Transaction, + index: usize, + begin: &[u8], + end: &[u8], + mut values: FdbValues, + cb: &mut F, + ) -> Result + where + F: FnMut(usize, &[u8], &[u8]) -> trc::Result + Send, + { + loop { + if !self.rows_of(index, &values, cb)? { + return Ok(false); + } + if !values.more() || values.is_empty() { + break; + } + drop(values); + let next = trx.get_range( + &range_option( + KeySelector::first_greater_than(self.last_key.as_slice()), + KeySelector::first_greater_than(end), + ), + 1, + self.snapshot, + ); + values = next + .await + .map_err(|error| self.fail(error, index, Some(begin)))?; + } + + if self.rows == WindowRows::Values + && let Some(chunk) = &mut self.pending + { + let tail = chunk.tail_end(); + if end < tail.as_slice() + && let Err(error) = + read_chunks(trx, &chunk.key, end, &tail, &mut chunk.bytes, self.snapshot).await + { + return Err(self.fail(error, index, Some(begin))); + } + } + match self.pending.take() { + Some(chunk) => self.deliver(cb, index, &chunk.key, &chunk.bytes), + None => { + self.progressed = true; + Ok(true) + } + } + } + + fn rows_of( + &mut self, + index: usize, + values: &FdbValues, + cb: &mut F, + ) -> Result + where + F: FnMut(usize, &[u8], &[u8]) -> trc::Result + Send, + { + #[cfg(stalwart_tests)] + super::probe::range_batch(values); + let mut bytes = 0; + for row in values.iter() { + let (key, value) = (row.key(), row.value()); + bytes += value.len(); + if let Some(chunk) = &mut self.pending + && is_chunk_of(key, &chunk.key) + { + if self.rows == WindowRows::Values { + chunk.bytes.extend_from_slice(value); + } + continue; + } + if let Some(chunk) = self.pending.take() + && !self.deliver(cb, index, &chunk.key, &chunk.bytes)? + { + return Ok(false); + } + let delivered = match self.rows { + WindowRows::Raw => self.deliver(cb, index, key, value)?, + _ if value.len() >= MAX_VALUE_SIZE => { + self.pending = Some(ChunkedValueCollector { + key: key.to_vec(), + bytes: if self.rows == WindowRows::Values { + value.to_vec() + } else { + Vec::new() + }, + }); + true + } + WindowRows::Values => self.deliver(cb, index, key, value)?, + WindowRows::Keys => self.deliver(cb, index, key, &[])?, + }; + if !delivered { + return Ok(false); + } + } + if let Some(last) = values.last() { + self.last_key.clear(); + self.last_key.extend_from_slice(last.key()); + } + self.largest = self.largest.max(bytes); + Ok(true) + } + + fn deliver( + &mut self, + cb: &mut F, + index: usize, + key: &[u8], + value: &[u8], + ) -> Result + where + F: FnMut(usize, &[u8], &[u8]) -> trc::Result + Send, + { + self.progressed = true; + self.largest = self.largest.max(value.len()); + cb(index, key, value).map_err(WindowError::Store) + } + + fn fail(&mut self, error: FdbError, index: usize, begin: Option<&[u8]>) -> WindowError { + let begin = begin.map(|begin| match self.pending.take() { + Some(chunk) => chunk.key, + None if self.last_key.is_empty() => begin.to_vec(), + None => { + let mut after = mem::take(&mut self.last_key); + after.push(0); + after + } + }); + WindowError::Fdb { + error, + resume: Resume { index, begin }, + progressed: self.progressed, + } + } +} + +async fn read_chunks( + trx: &Transaction, + base: &[u8], + after: &[u8], + tail: &[u8], + bytes: &mut Vec, + snapshot: bool, +) -> FdbResult<()> { + let mut chunks = range_option( + KeySelector::first_greater_than(after), + KeySelector::first_greater_or_equal(tail), + ); + loop { + let values = trx.get_range(&chunks, 1, snapshot).await?; + #[cfg(stalwart_tests)] + super::probe::range_batch(&values); + for row in values.iter() { + if is_chunk_of(row.key(), base) { + bytes.extend_from_slice(row.value()); + } + } + match chunks.next_range(&values) { + Some(next) => chunks = next, + None => return Ok(()), + } + } +} + +fn range_option<'a>(begin: KeySelector<'a>, end: KeySelector<'a>) -> RangeOption<'a> { + RangeOption { + begin, + end, + mode: StreamingMode::WantAll, + reverse: false, + ..Default::default() + } +} + +fn is_chunk_of(key: &[u8], base: &[u8]) -> bool { + key.len() == base.len() + 1 && key.starts_with(base) +} diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 06c79078..401f5c03 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -5,15 +5,17 @@ */ use super::{ - FdbStore, MAX_VALUE_SIZE, into_error, - read::{ChunkedValue, read_chunked_value}, + FDB_BATCH_THROTTLED, FDB_VALUE_TOO_LARGE, FdbStore, MAX_THROTTLED_TIME, MAX_VALUE_SIZE, + into_error, + preread::{MergePreRead, Prefetched}, + read::{ChunkedValue, read_chunk_tail, read_chunked_value}, }; use crate::{ backend::deserialize_i64_le, write::{ - AssignedIds, Batch, IndexPropertyClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, - Operation, QueueClass, RegistryClass, SearchIndexType, TaskQueueClass, TelemetryClass, - ValueClass, ValueOp, key::KeySerializer, + Advance, Allocation, AnyKey, AssignedIds, Batch, BatchCursor, CommitPriority, CounterBound, + LogSet, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MAX_CONCURRENT_ALLOCATIONS, MergeResult, + ValueClass, ValueOp, commit_backoff, exhausted, key::KeySerializer, }, *, }; @@ -22,117 +24,188 @@ use foundationdb::{ options::{self, MutationType}, }; use futures::TryStreamExt; -use rand::RngExt; -use std::{ - borrow::Cow, - cmp::Ordering, - time::{Duration, Instant}, -}; -use trc::AddContext; +use std::{borrow::Cow, cmp::Ordering, ops::Range, time::Instant}; + +const PURGE_BATCH_SIZE: usize = 1024; + +enum Previous<'x> { + Prefetched(Option<&'x [u8]>), + Read(ChunkedValue), +} + +struct Throttle { + priority: CommitPriority, + since: Option, +} + +struct CounterKeys { + buf: Vec, + ranges: Vec>, + reservation_key: Vec, + reservation_total: Vec, +} impl FdbStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { - let start = Instant::now(); + pub(crate) async fn write( + &self, + mut batch: Batch<'_>, + result: &mut AssignedIds, + ) -> trc::Result<()> { + let mut start = Instant::now(); let mut retry_count = 0; - let has_changes = !batch.changes.is_empty(); + let mut throttle = Throttle::new(batch.priority); + let mark = result.mark(); + let counter_keys = counter_keys(&batch); + let mut log_buf = Vec::new(); + let mut log_scratch = Vec::new(); - loop { - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let mut result = AssignedIds::default(); + let mut next_trx: Option = None; - let trx = self.db.create_trx().map_err(into_error)?; + 'retry: loop { + let mut cursor = BatchCursor::new(&batch); + result.rollback(mark); - if has_changes { - for &account_id in batch.changes.keys() { - debug_assert!(account_id != u32::MAX); - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, WITH_SUBSPACE); - let change_id = - if let Some(bytes) = trx.get(&key, false).await.map_err(into_error)? { - deserialize_i64_le(&key, &bytes)? + 1 - } else { - 1 - }; - trx.set(&key, &change_id.to_le_bytes()[..]); - result.push_change_id(account_id, change_id as u64); - } + let trx = match next_trx.take() { + Some(trx) => trx, + None => self.db.create_trx().map_err(into_error)?, + }; + if batch.priority == CommitPriority::Background { + trx.set_option(options::TransactionOption::PriorityBatch) + .map_err(into_error)?; } + #[cfg(stalwart_tests)] + super::probe::report_conflicts(&trx); - for op in batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; + let mut preread = if counter_keys.is_empty() { + let preread = MergePreRead::plan(&mut batch, result); + if preread + .as_ref() + .is_some_and(|preread| !preread.reads_own_writes()) + { + trx.set_option(options::TransactionOption::ReadYourWritesDisable) + .map_err(into_error)?; + } + preread + } else { + if let Err(err) = allocate_counters(&trx, &batch, &counter_keys, result).await { + match err { + ApplyError::Retryable(err) => { + next_trx = Some( + self.on_write_error( + trx, + err, + &mut retry_count, + &mut start, + &mut throttle, + ) + .await?, + ); + continue 'retry; + } + ApplyError::Fatal(err) => { + trx.cancel(); + return Err(err); } } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let mut key = - class.serialize(account_id, collection, document_id, WITH_SUBSPACE); + } + MergePreRead::plan(&mut batch, result) + }; + if let Some(preread) = &mut preread + && let Err(err) = preread.read(&trx, &self.windows).await + { + next_trx = Some( + self.on_write_error(trx, err, &mut retry_count, &mut start, &mut throttle) + .await?, + ); + continue 'retry; + } + + let mut key_buf = Vec::with_capacity(64); + for op in batch.ops.iter_mut() { + match cursor.advance(op, result) { + Advance::Cursor => {} + Advance::Value { class, op } => { + cursor.value_key(class, &mut key_buf, WITH_SUBSPACE); match op { - ValueOp::Set(value) => { - if !chunk_value(&trx, &mut key, value, class, None).await { + ValueOp::Set(set_value) => { + let value = set_value.resolve(result)?; + if !chunk_value(&trx, &mut key_buf, &value, class, None).await { trx.cancel(); return Err(trc::StoreEvent::FoundationdbError - .ctx(trc::Key::Reason, "Value is too large")); - } - } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; - if !chunk_value(&trx, &mut key, &value, class, None).await { - trx.cancel(); - return Err(trc::StoreEvent::FoundationdbError - .ctx(trc::Key::Reason, "Value is too large")); + .ctx(trc::Key::Reason, "Value is too large") + .ctx(trc::Key::Code, FDB_VALUE_TOO_LARGE)); } } ValueOp::MergeFnc(merge_op) => { - let (merge_result, prev_num_chunks) = - match read_chunked_value(&key, &trx, false) - .await - .map_err(into_error) - .caused_by(trc::location!())? - { - ChunkedValue::Single(slice) => ( - (merge_op.fnc)( - &merge_op.params, - &result, - Some(slice.as_ref()), - )?, - 1, - ), - ChunkedValue::Chunked { bytes, n_chunks } => ( - (merge_op.fnc)( - &merge_op.params, - &result, - Some(bytes.as_ref()), - )?, - n_chunks as usize + 1, - ), - ChunkedValue::None => { - ((merge_op.fnc)(&merge_op.params, &result, None)?, 0) - } - }; + let previous = match preread.as_mut().map(MergePreRead::next_merge) + { + Some(Prefetched::Value(value)) => { + Ok(Previous::Prefetched(value)) + } + Some(Prefetched::Head(head)) => { + read_chunk_tail(&key_buf, &trx, false, head).await.map( + |tail| match tail { + Some(chunked) => Previous::Read(chunked), + None => Previous::Prefetched(Some(head)), + }, + ) + } + Some(Prefetched::Sequential) | None => { + read_chunked_value(&key_buf, &trx, false) + .await + .map(Previous::Read) + } + }; + let previous = match previous { + Ok(previous) => previous, + Err(err) => { + next_trx = Some( + self.on_write_error( + trx, + err, + &mut retry_count, + &mut start, + &mut throttle, + ) + .await?, + ); + continue 'retry; + } + }; + let (current_value, merge_input, prev_num_chunks) = match &previous + { + Previous::Prefetched(Some(bytes)) => { + (Some(*bytes), Some(*bytes), 1) + } + Previous::Prefetched(None) + | Previous::Read(ChunkedValue::None) => (None, None, 0), + Previous::Read(ChunkedValue::Single(slice)) => { + (Some(slice.as_ref()), Some(slice.as_ref()), 1) + } + Previous::Read(ChunkedValue::Chunked { bytes, n_chunks }) => { + (None, Some(bytes.as_slice()), *n_chunks as usize + 1) + } + }; + let merge_result = (merge_op.0)(result, merge_input)?; match merge_result { MergeResult::Update(value) => { - if !chunk_value( + if let Some(append_bytes) = + current_value.and_then(|current| { + value + .strip_prefix(current) + .filter(|_| value.len() < MAX_VALUE_SIZE) + }) + { + trx.atomic_op( + &key_buf, + append_bytes, + MutationType::AppendIfFits, + ); + } else if !chunk_value( &trx, - &mut key, + &mut key_buf, &value, class, Some(prev_num_chunks), @@ -141,85 +214,126 @@ impl FdbStore { { trx.cancel(); return Err(trc::StoreEvent::FoundationdbError - .ctx(trc::Key::Reason, "Value is too large")); + .ctx(trc::Key::Reason, "Value is too large") + .ctx(trc::Key::Code, FDB_VALUE_TOO_LARGE)); } } MergeResult::Delete => { if prev_num_chunks > 1 { - clear_chunks(&trx, &key, None).await; + clear_chunks(&trx, &key_buf, None).await; } else { - trx.clear(&key); + trx.clear(&key_buf); } } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { - trx.atomic_op(&key, &by.to_le_bytes()[..], MutationType::Add); + trx.atomic_op(&key_buf, &by.to_le_bytes()[..], MutationType::Add); } ValueOp::AddAndGet(by) => { - let num = if let Some(bytes) = - trx.get(&key, false).await.map_err(into_error)? - { - deserialize_i64_le(&key, &bytes)? + *by + let current = match trx.get(&key_buf, false).await { + Ok(current) => current, + Err(err) => { + next_trx = Some( + self.on_write_error( + trx, + err, + &mut retry_count, + &mut start, + &mut throttle, + ) + .await?, + ); + continue 'retry; + } + }; + let num = if let Some(bytes) = current { + deserialize_i64_le(&key_buf, &bytes)? + *by } else { *by }; - trx.set(&key, &num.to_le_bytes()[..]); + trx.set(&key_buf, &num.to_le_bytes()[..]); result.push_counter_id(num); } ValueOp::Clear => { - if is_chunked_value(key[0], class) { - clear_chunks(&trx, &key, None).await; + if class.may_be_chunked() { + clear_chunks(&trx, &key_buf, None).await; } else { - trx.clear(&key); + trx.clear(&key_buf); } } } } - Operation::Index { field, key, set } => { - let key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: &*key, - } - .serialize(WITH_SUBSPACE); + Advance::Index { field, key, set } => { + cursor.index_key(field, key, &mut key_buf, WITH_SUBSPACE); - if *set { - trx.set(&key, &[]); + if set { + trx.set(&key_buf, &[]); } else { - trx.clear(&key); + trx.clear(&key_buf); } } - Operation::Log { collection, set } => { - let key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, - } - .serialize(WITH_SUBSPACE); + Advance::Log { collection, set } => { + cursor.log_key(collection, result, &mut key_buf, WITH_SUBSPACE); - trx.set(&key, set); + match set { + LogSet::Bytes(bytes) => { + trx.set(&key_buf, bytes); + } + LogSet::Pending(changes) => { + changes.serialize_into( + collection.is_prefixed(), + result, + &mut log_scratch, + &mut log_buf, + ); + trx.set(&key_buf, &log_buf); + } + } } - Operation::AssertValue { + Advance::ClearRange { subspace, from, to } => { + key_buf.clear(); + AnyKey { + subspace, + key: from, + } + .serialize_into(&mut key_buf, WITH_SUBSPACE); + let mut end = Vec::with_capacity(to.len() + 1); + AnyKey { subspace, key: to }.serialize_into(&mut end, WITH_SUBSPACE); + trx.clear_range(&key_buf, &end); + } + Advance::Assert { class, assert_value, } => { - let key = - class.serialize(account_id, collection, document_id, WITH_SUBSPACE); + cursor.value_key(class, &mut key_buf, WITH_SUBSPACE); - let matches = match read_chunked_value(&key, &trx, false).await { + let matches = match read_chunked_value(&key_buf, &trx, false).await { Ok(ChunkedValue::Single(bytes)) => assert_value.matches(bytes.as_ref()), Ok(ChunkedValue::Chunked { bytes, .. }) => { assert_value.matches(bytes.as_ref()) } Ok(ChunkedValue::None) => assert_value.is_none(), - Err(_) => false, + Err(err) => { + next_trx = Some( + self.on_write_error( + trx, + err, + &mut retry_count, + &mut start, + &mut throttle, + ) + .await?, + ); + continue 'retry; + } }; if !matches { + if let Ok(read_version) = trx.get_read_version().await { + self.version.advance(read_version); + } trx.cancel(); return Err(trc::StoreEvent::AssertValueFailed.into()); } @@ -227,130 +341,252 @@ impl FdbStore { } } - if self - .commit( - trx, - retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME, - ) - .await? - { - return Ok(result); - } else { - let backoff = rand::rng().random_range(50..=100); - tokio::time::sleep(Duration::from_millis(backoff)).await; - retry_count += 1; + let will_retry = retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME; + #[cfg(stalwart_tests)] + let commit = match super::probe::injected_throttle(batch.priority) { + Some(err) => { + trx.cancel(); + Ok(Err(err)) + } + None => self.try_commit(trx).await, + }; + #[cfg(not(stalwart_tests))] + let commit = self.try_commit(trx).await; + match commit? { + Ok(()) => return Ok(()), + Err(err) if throttle.waits(err) => { + start = Instant::now(); + tokio::time::sleep(commit_backoff(MAX_COMMIT_ATTEMPTS)).await; + } + Err(err) => { + retry_not_committed(err, will_retry, retry_count)?; + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } } } } - pub(crate) async fn commit(&self, trx: Transaction, will_retry: bool) -> trc::Result { + async fn on_write_error( + &self, + trx: Transaction, + err: FdbError, + retry_count: &mut u32, + start: &mut Instant, + throttle: &mut Throttle, + ) -> trc::Result { + if throttle.waits(err) { + trx.cancel(); + *start = Instant::now(); + tokio::time::sleep(commit_backoff(MAX_COMMIT_ATTEMPTS)).await; + self.db.create_trx().map_err(into_error) + } else if !err.is_retryable() { + Err(into_error(err)) + } else if *retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME { + #[cfg(stalwart_tests)] + crate::write::record_retry(*retry_count); + let trx = trx.on_error(err).await.map_err(into_error)?; + *retry_count += 1; + Ok(trx) + } else { + Err(exhausted(into_error(err), *retry_count)) + } + } + + pub(super) async fn commit_attempt( + &self, + trx: Transaction, + retry_count: u32, + start: Instant, + ) -> trc::Result { + let will_retry = retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME; + + match self.try_commit(trx).await? { + Ok(()) => Ok(true), + Err(err) => retry_not_committed(err, will_retry, retry_count).map(|()| false), + } + } + + async fn try_commit(&self, trx: Transaction) -> trc::Result> { match trx.commit().await { Ok(result) => { + #[cfg(stalwart_tests)] + super::probe::commit_ok(); let commit_version = result.committed_version().map_err(into_error)?; self.version.raise_floor(commit_version); - Ok(true) + Ok(Ok(())) } Err(err) => { - if will_retry { - err.on_error().await.map_err(into_error)?; - Ok(false) - } else { - Err(into_error(FdbError::from(err))) - } + #[cfg(stalwart_tests)] + super::probe::commit_failed(&err).await; + Ok(Err(FdbError::from(err))) } } } pub(crate) async fn purge_store(&self) -> trc::Result<()> { - // Obtain all zero counters - let mut delete_keys = Vec::new(); - for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { - let trx = self.db.create_trx().map_err(into_error)?; - let from_key = [subspace, 0u8]; - let to_key = [subspace, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]; - - let mut values = trx.get_ranges_keyvalues( - RangeOption { - begin: KeySelector::first_greater_or_equal(&from_key[..]), - end: KeySelector::first_greater_or_equal(&to_key[..]), - mode: options::StreamingMode::WantAll, - reverse: false, - ..Default::default() - }, - true, - ); - - while let Some(value) = values.try_next().await.map_err(into_error)? { - if value.value().iter().all(|byte| *byte == 0) { - delete_keys.push(value.key().to_vec()); - } - } - } - - if delete_keys.is_empty() { - return Ok(()); - } - - // Delete keys - let integer = 0i64.to_le_bytes(); - for chunk in delete_keys.chunks(1024) { + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { + let from_key = [subspace.byte()]; + let to_key = [subspace.byte() + 1]; + let mut delete_keys: Vec> = Vec::new(); + let mut last_key: Vec = Vec::new(); let mut retry_count = 0; - loop { - let trx = self.db.create_trx().map_err(into_error)?; - for key in chunk { - trx.atomic_op(key, &integer, MutationType::CompareAndClear); - } + let mut start = Instant::now(); + let mut next_trx: Option = None; - if self.commit(trx, retry_count < MAX_COMMIT_ATTEMPTS).await? { - break; + 'scan: loop { + let trx = match next_trx.take() { + Some(trx) => trx, + None => self.read_trx().await?, + }; + trx.set_option(options::TransactionOption::PriorityBatch) + .map_err(into_error)?; + let mut made_progress = false; + + let begin = if last_key.is_empty() { + KeySelector::first_greater_or_equal(&from_key[..]) } else { - retry_count += 1; + KeySelector::first_greater_than(last_key.clone()) + }; + let mut values = trx.get_ranges_keyvalues( + RangeOption { + begin, + end: KeySelector::first_greater_or_equal(&to_key[..]), + mode: options::StreamingMode::WantAll, + reverse: false, + ..Default::default() + }, + true, + ); + + loop { + match values.try_next().await { + Ok(Some(value)) => { + made_progress = true; + last_key.clear(); + last_key.extend_from_slice(value.key()); + + if value.value().iter().all(|byte| *byte == 0) { + delete_keys.push(value.key().to_vec()); + + if delete_keys.len() >= PURGE_BATCH_SIZE { + drop(values); + self.purge_counters(&mut delete_keys).await?; + retry_count = 0; + start = Instant::now(); + continue 'scan; + } + } + } + Ok(None) => break 'scan, + Err(err) => { + drop(values); + if made_progress { + retry_count = 0; + start = Instant::now(); + } + if err.is_retryable() + && retry_count < MAX_COMMIT_ATTEMPTS + && start.elapsed() < MAX_COMMIT_TIME + { + self.version.expire(); + let trx = trx.on_error(err).await.map_err(into_error)?; + next_trx = Some(self.apply_read_version(trx).await?); + retry_count += 1; + continue 'scan; + } + return Err(into_error(err)); + } + } } } + + self.purge_counters(&mut delete_keys).await?; } Ok(()) } + async fn purge_counters(&self, delete_keys: &mut Vec>) -> trc::Result<()> { + if delete_keys.is_empty() { + return Ok(()); + } + + let integer = 0i64.to_le_bytes(); + let start = Instant::now(); + let mut retry_count = 0; + + loop { + let trx = self.db.create_trx().map_err(into_error)?; + trx.set_option(options::TransactionOption::PriorityBatch) + .map_err(into_error)?; + for key in delete_keys.iter() { + trx.atomic_op(key, &integer, MutationType::CompareAndClear); + } + + if self.commit_attempt(trx, retry_count, start).await? { + delete_keys.clear(); + return Ok(()); + } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } + } + pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let from = from.serialize(WITH_SUBSPACE); let to = to.serialize(WITH_SUBSPACE); + let start = Instant::now(); + let mut retry_count = 0; - let trx = self.db.create_trx().map_err(into_error)?; - trx.clear_range(&from, &to); - self.commit(trx, false).await.map(|_| ()) + loop { + let trx = self.db.create_trx().map_err(into_error)?; + trx.clear_range(&from, &to); + + if self.commit_attempt(trx, retry_count, start).await? { + return Ok(()); + } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } } } -fn is_chunked_subspace(subspace: u8) -> bool { - matches!( - subspace, - crate::SUBSPACE_PROPERTY - | crate::SUBSPACE_SEARCH_INDEX - | crate::SUBSPACE_QUEUE_MESSAGE - | crate::SUBSPACE_TASK_QUEUE - | crate::SUBSPACE_DIRECTORY - | crate::SUBSPACE_REGISTRY - | crate::SUBSPACE_DELETED_ITEMS - | crate::SUBSPACE_SPAM_SAMPLES - | crate::SUBSPACE_REPORT_IN - | crate::SUBSPACE_REPORT_OUT - | crate::SUBSPACE_TELEMETRY_SPAN - ) +impl Throttle { + fn new(priority: CommitPriority) -> Self { + Throttle { + priority, + since: None, + } + } + + fn waits(&mut self, err: FdbError) -> bool { + self.priority == CommitPriority::Background + && err.code() == FDB_BATCH_THROTTLED + && self.since.get_or_insert_with(Instant::now).elapsed() < MAX_THROTTLED_TIME + } } -fn is_chunked_value(subspace: u8, class: &ValueClass) -> bool { - is_chunked_subspace(subspace) - && match class { - ValueClass::Property(_) - | ValueClass::IndexProperty(IndexPropertyClass::Hash { .. }) - | ValueClass::Registry(RegistryClass::Item { .. }) - | ValueClass::Queue(QueueClass::Message(_)) - | ValueClass::TaskQueue(TaskQueueClass::Task { .. }) - | ValueClass::Telemetry(TelemetryClass::Span(_)) => true, - ValueClass::SearchIndex(index) => matches!(index.typ, SearchIndexType::Document), - _ => false, - } +fn retry_not_committed(err: FdbError, will_retry: bool, retry_count: u32) -> trc::Result<()> { + if !err.is_retryable_not_committed() { + Err(into_error(err)) + } else if will_retry { + Ok(()) + } else { + Err(exhausted(into_error(err), retry_count)) + } } async fn clear_chunks(trx: &Transaction, key: &[u8], from_chunk: Option) { @@ -399,7 +635,7 @@ async fn chunk_value( class: &ValueClass, prev_num_chunks: Option, ) -> bool { - let num_chunks = if value.len() > MAX_VALUE_SIZE { + let num_chunks = if value.len() >= MAX_VALUE_SIZE { value.len().div_ceil(MAX_VALUE_SIZE) } else { 1 @@ -409,13 +645,19 @@ async fn chunk_value( return false; } - if is_chunked_value(key[0], class) + debug_assert!( + num_chunks == 1 || class.may_be_chunked() || matches!(class, ValueClass::Any(_)), + "{class:?} was split into {num_chunks} chunks but is not declared chunkable, \ + stale chunks will be spliced onto a later shorter value" + ); + + if class.may_be_chunked() && prev_num_chunks.is_none_or(|prev_num_chunks| prev_num_chunks > num_chunks) { clear_chunks(trx, key, Some((num_chunks - 1) as u8)).await; } - if value.len() > MAX_VALUE_SIZE { + if value.len() >= MAX_VALUE_SIZE { for (pos, chunk) in value.chunks(MAX_VALUE_SIZE).enumerate() { match pos.cmp(&1) { Ordering::Less => {} @@ -434,3 +676,139 @@ async fn chunk_value( true } + +enum ApplyError { + Retryable(FdbError), + Fatal(trc::Error), +} + +impl From for ApplyError { + fn from(err: trc::Error) -> Self { + ApplyError::Fatal(err) + } +} + +impl CounterKeys { + #[inline(always)] + fn is_empty(&self) -> bool { + self.ranges.is_empty() + } + + #[inline(always)] + fn key(&self, index: usize) -> &[u8] { + &self.buf[self.ranges[index].clone()] + } +} + +fn counter_keys(batch: &Batch<'_>) -> CounterKeys { + let num_accounts = batch.change_accounts.len(); + let total = num_accounts + batch.reservations.len() + batch.bounds.len(); + let mut keys = CounterKeys { + buf: Vec::with_capacity(total * 16), + ranges: Vec::with_capacity(total), + reservation_key: Vec::with_capacity(batch.reservations.len()), + reservation_total: Vec::with_capacity(batch.reservations.len()), + }; + let mut key = Vec::with_capacity(16); + + for counter in batch.change_accounts { + debug_assert!(counter.account_id != u32::MAX); + let start = keys.buf.len(); + Allocation::ChangeId(*counter).serialize_key_into(&mut keys.buf, WITH_SUBSPACE); + keys.ranges.push(start..keys.buf.len()); + } + + for reservation in batch.reservations { + key.clear(); + Allocation::Reserved(*reservation).serialize_key_into(&mut key, WITH_SUBSPACE); + + let key_index = match keys.ranges[num_accounts..] + .iter() + .position(|range| keys.buf[range.clone()] == key[..]) + { + Some(offset) => { + keys.reservation_total[offset] += reservation.count as i64; + num_accounts + offset + } + None => { + let start = keys.buf.len(); + keys.buf.extend_from_slice(&key); + keys.ranges.push(start..keys.buf.len()); + keys.reservation_total.push(reservation.count as i64); + keys.ranges.len() - 1 + } + }; + keys.reservation_key.push(key_index); + } + + for bound in batch.bounds { + let start = keys.buf.len(); + bound.serialize_key_into(&mut keys.buf, WITH_SUBSPACE); + keys.ranges.push(start..keys.buf.len()); + } + + keys +} + +async fn allocate_counters( + trx: &Transaction, + batch: &Batch<'_>, + keys: &CounterKeys, + result: &mut AssignedIds, +) -> Result<(), ApplyError> { + let num_accounts = batch.change_accounts.len(); + let num_reserved = num_accounts + keys.reservation_total.len(); + let mut cursors = vec![0i64; keys.reservation_total.len()]; + + for chunk_start in (0..keys.ranges.len()).step_by(MAX_CONCURRENT_ALLOCATIONS) { + let chunk_end = (chunk_start + MAX_CONCURRENT_ALLOCATIONS).min(keys.ranges.len()); + let values = futures::future::try_join_all((chunk_start..chunk_end).map(|index| { + let snapshot = index >= num_reserved + && batch + .bounds + .get(index - num_reserved) + .is_some_and(CounterBound::allows_overshoot); + trx.get(keys.key(index), snapshot) + })) + .await + .map_err(ApplyError::Retryable)?; + + for (offset, value) in values.into_iter().enumerate() { + let index = chunk_start + offset; + let key = keys.key(index); + let current = match value { + Some(bytes) => deserialize_i64_le(key, &bytes)?, + None => 0, + }; + + if index < num_accounts { + let account = &batch.change_accounts[index]; + let change_id = current + 1; + trx.set(key, &change_id.to_le_bytes()[..]); + result.push_change_id(*account, change_id as u64); + } else if index < num_reserved { + let key_offset = index - num_accounts; + let last_id = current + keys.reservation_total[key_offset]; + trx.set(key, &last_id.to_le_bytes()[..]); + cursors[key_offset] = current; + } else { + let bound = &batch.bounds[index - num_reserved]; + let value = current.saturating_add(bound.by); + bound.verify(value)?; + if bound.allows_overshoot() { + trx.atomic_op(key, &bound.by.to_le_bytes()[..], MutationType::Add); + } else { + trx.set(key, &value.to_le_bytes()[..]); + } + } + } + } + + for (reservation, key_index) in batch.reservations.iter().zip(keys.reservation_key.iter()) { + let cursor = &mut cursors[key_index - num_accounts]; + *cursor += reservation.count as i64; + result.fill_slots(reservation.first_slot, reservation.count, *cursor as u32); + } + + Ok(()) +} diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index d9a4b5d7..893e103c 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -5,13 +5,13 @@ */ use crate::BlobStore; +use encodify::base32::STALWART; use registry::schema::structs; use std::{io::SeekFrom, ops::Range, path::PathBuf, sync::Arc}; use tokio::{ fs::{self, File}, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; -use utils::codec::base32_custom::Base32Writer; pub struct FsStore { path: PathBuf, @@ -101,7 +101,7 @@ impl FsStore { for byte in key.iter().take(self.hash_levels) { path.push(format!("{:x}", byte)); } - path.push(Base32Writer::from_bytes(key).finalize()); + path.push(STALWART.encode(key)); path } } diff --git a/crates/store/src/backend/http/config.rs b/crates/store/src/backend/http/config.rs index 8279d4a0..3abab8b8 100644 --- a/crates/store/src/backend/http/config.rs +++ b/crates/store/src/backend/http/config.rs @@ -13,6 +13,7 @@ use std::{ collections::hash_map::Entry, sync::atomic::{AtomicBool, AtomicU64}, }; +use utils::http::http_client_builder; impl LookupStores { pub async fn parse_http(&mut self, bp: &mut Bootstrap) { @@ -24,10 +25,30 @@ impl LookupStores { continue; } + let (client, custom_user_agent) = match http + .http_auth + .build_headers(http.http_headers, None) + .await + .and_then(|headers| { + let custom_user_agent = headers.contains_key(reqwest::header::USER_AGENT); + http_client_builder(http.allow_invalid_certs) + .pool_max_idle_per_host(0) + .default_headers(headers) + .build() + .map(|client| (client, custom_user_agent)) + .map_err(|err| format!("Failed to build HTTP client: {err}")) + }) { + Ok(result) => result, + Err(err) => { + bp.build_error(id, err); + continue; + } + }; + let http_config = HttpStoreConfig { url: http.url, retry: http.retry.as_secs(), - refresh: http.refresh.as_secs(), + refresh_interval: http.refresh_interval.as_secs(), timeout: http.timeout.into_inner(), gzipped: http.is_gzipped, max_size: http.max_size as usize, @@ -43,6 +64,7 @@ impl LookupStores { }, }, id: http.namespace, + custom_user_agent, }; match self.stores.entry(http_config.id.as_str().into()) { @@ -52,7 +74,7 @@ impl LookupStores { expires: AtomicU64::new(0), in_flight: AtomicBool::new(false), config: http_config, - client: utils::http::unpooled_http_client(false), + client, }; entry.insert(InMemoryStore::Http(store.into())); diff --git a/crates/store/src/backend/http/lookup.rs b/crates/store/src/backend/http/lookup.rs index 76282dba..5d149fd5 100644 --- a/crates/store/src/backend/http/lookup.rs +++ b/crates/store/src/backend/http/lookup.rs @@ -11,7 +11,7 @@ use compact_str::ToCompactString; use rand::seq::IndexedRandom; use std::{ borrow::Cow, - io::{BufRead, BufReader}, + io::{self, BufRead, BufReader, Read}, sync::{Arc, atomic::Ordering}, time::Instant, }; @@ -38,7 +38,7 @@ impl HttpStoreGet for Arc { } fn contains(&self, key: &str) -> bool { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { if self.config.url.contains("phishtank.com") || self.config.url.contains("openphish.com") @@ -69,7 +69,7 @@ impl HttpStoreGet for Arc { let expires = match this.try_refresh().await { Ok(list) => { this.entries.store(list.into()); - this.config.refresh + this.config.refresh_interval } Err(err) => { trc::error!(err); @@ -88,21 +88,22 @@ impl HttpStoreGet for Arc { impl HttpStore { async fn try_refresh(&self) -> trc::Result>> { let time = Instant::now(); - let agent = BROWSER_USER_AGENTS.choose(&mut rand::rng()).unwrap(); - let response = self + let mut request = self .client .get(&self.config.url) - .timeout(self.config.timeout) - .header(reqwest::header::USER_AGENT, *agent) - .send() - .await - .map_err(|err| { - trc::StoreEvent::HttpStoreError - .into_err() - .reason(err) - .ctx(trc::Key::Url, self.config.url.to_compact_string()) - .details("Failed to build request") - })?; + .timeout(self.config.timeout); + if !self.config.custom_user_agent + && let Some(agent) = BROWSER_USER_AGENTS.choose(&mut rand::rng()) + { + request = request.header(reqwest::header::USER_AGENT, *agent); + } + let response = request.send().await.map_err(|err| { + trc::StoreEvent::HttpStoreError + .into_err() + .reason(err) + .ctx(trc::Key::Url, self.config.url.to_compact_string()) + .details("Failed to build request") + })?; if !response.status().is_success() { trc::bail!( @@ -134,7 +135,7 @@ impl HttpStore { .details("Resource is too large") })?; - let reader: Box = if self.config.gzipped { + let reader: Box = if self.config.gzipped { Box::new(flate2::read::GzDecoder::new(&bytes[..])) } else { Box::new(&bytes[..]) @@ -166,16 +167,24 @@ impl HttpStore { impl HttpStoreConfig { fn parse_entries( &self, - reader: impl BufRead, - ) -> std::io::Result>> { + mut reader: impl BufRead, + ) -> io::Result>> { let mut entries = AHashMap::new(); - for (pos, line) in reader.lines().enumerate() { - let line_ = line?; + let mut buf = Vec::new(); + let mut entry_key = String::new(); + let mut entry_value = String::new(); + + for pos in 0usize.. { + let line = match read_line(&mut reader, &mut buf, self.max_size)? { + Line::Text(line) => line, + Line::Oversize => continue, + Line::Eof => break, + }; match &self.format { HttpStoreFormat::List => { - let line = line_.trim(); - if !line.is_empty() { + let line = line.trim(); + if !line.is_empty() && line.len() <= self.max_entry_size { entries.insert(lookup_key(line).into_owned(), Value::Integer(1)); } } @@ -188,11 +197,12 @@ impl HttpStoreConfig { let mut in_quote = false; let mut col_num = 0; let mut last_ch = ' '; + let mut oversize = false; - let mut entry_key: String = String::new(); - let mut entry_value: String = String::new(); + entry_key.clear(); + entry_value.clear(); - for ch in line_.chars() { + for ch in line.chars() { match ch { '"' if last_ch != '\\' => { in_quote = !in_quote; @@ -208,11 +218,13 @@ impl HttpStoreConfig { } else if col_num == *index_key { entry_key.push(ch); if entry_key.len() > self.max_entry_size { + oversize = true; break; } } else if index_value.is_some_and(|v| col_num == v) { entry_value.push(ch); if entry_value.len() > self.max_entry_size { + oversize = true; break; } } @@ -222,17 +234,13 @@ impl HttpStoreConfig { last_ch = ch; } - if !entry_key.is_empty() { + if !oversize && !entry_key.is_empty() { let entry_value = if !entry_value.is_empty() { - Value::Text(entry_value.into()) + Value::Text(Cow::Owned(String::from(entry_value.as_str()))) } else { Value::Integer(1) }; - let entry_key = match lookup_key(&entry_key) { - Cow::Owned(key) => key, - Cow::Borrowed(_) => entry_key, - }; - entries.insert(entry_key, entry_value); + entries.insert(lookup_key(&entry_key).into_owned(), entry_value); } } _ => (), @@ -247,103 +255,46 @@ impl HttpStoreConfig { } } +enum Line<'x> { + Text(&'x str), + Oversize, + Eof, +} + +fn read_line<'x>( + reader: &mut impl BufRead, + buf: &'x mut Vec, + max_len: usize, +) -> io::Result> { + buf.clear(); + let limit = u64::try_from(max_len).unwrap_or(u64::MAX).saturating_add(1); + if reader.by_ref().take(limit).read_until(b'\n', buf)? == 0 { + return Ok(Line::Eof); + } + let buf: &'x [u8] = buf; + let line = match buf.strip_suffix(b"\n") { + Some(line) => line.strip_suffix(b"\r").unwrap_or(line), + None if buf.len() > max_len => { + reader.skip_until(b'\n')?; + return Ok(Line::Oversize); + } + None => buf, + }; + str::from_utf8(line) + .map(Line::Text) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err)) +} + fn lookup_key(key: &str) -> Cow<'_, str> { - if key.bytes().any(|b| !b.is_ascii() || b.is_ascii_uppercase()) { + let needs_lowercase = if key.is_ascii() { + key.bytes().any(|b| b.is_ascii_uppercase()) + } else { + key.chars().any(|ch| !ch.to_lowercase().eq([ch])) + }; + if needs_lowercase { Cow::Owned(key.to_lowercase()) } else { Cow::Borrowed(key) } } -#[cfg(test)] -mod tests { - use super::*; - use arc_swap::ArcSwap; - use reqwest::Client; - use std::{ - sync::atomic::{AtomicBool, AtomicU64}, - time::Duration, - }; - - fn http_store(format: HttpStoreFormat, feed: &str) -> Arc { - let config = HttpStoreConfig { - id: "test".into(), - url: "https://lists.example.org/feed".into(), - retry: 0, - refresh: 0, - timeout: Duration::from_secs(1), - gzipped: false, - max_size: 1024 * 1024, - max_entries: 100, - max_entry_size: 512, - format, - }; - let entries = config - .parse_entries(feed.as_bytes()) - .expect("feed is readable"); - - Arc::new(HttpStore { - entries: ArcSwap::from_pointee(entries), - expires: AtomicU64::new(u64::MAX), - in_flight: AtomicBool::new(false), - config, - client: Client::new(), - }) - } - - #[test] - fn list_keys_ignore_case() { - let store = http_store( - HttpStoreFormat::List, - "https://phish.example.org/Account/Verify?Token=AbC123\n\ - https://PHISH.example.net/lower\n", - ); - - assert!(store.contains("https://phish.example.org/account/verify?token=abc123")); - assert!(store.contains("https://phish.example.org/Account/Verify?Token=AbC123")); - assert!(store.contains("https://phish.example.net/lower")); - assert!(store.contains("HTTPS://PHISH.EXAMPLE.NET/LOWER")); - assert!(!store.contains("https://phish.example.org/account/verify")); - } - - #[test] - fn csv_keys_ignore_case() { - let store = http_store( - HttpStoreFormat::Csv { - index_key: 1, - index_value: None, - separator: ',', - skip_first: true, - }, - "phish_id,url,phish_detail_url\n\ - 1,\"https://phish.example.org/Login.PHP?Id=Xy\",https://phishtank.example/1\n", - ); - - assert!(store.contains("https://phish.example.org/login.php?id=xy")); - assert!(store.contains("https://phish.example.org/Login.PHP?Id=Xy")); - assert!(!store.contains("phish_id")); - assert!(!store.contains("url")); - } - - #[test] - fn csv_values_keep_case() { - let store = http_store( - HttpStoreFormat::Csv { - index_key: 0, - index_value: Some(1), - separator: ',', - skip_first: false, - }, - "Example.ORG,Some Value\n", - ); - - assert_eq!( - store.get("example.org"), - Some(Value::Text("Some Value".into())) - ); - assert_eq!( - store.get("EXAMPLE.org"), - Some(Value::Text("Some Value".into())) - ); - } -} diff --git a/crates/store/src/backend/http/mod.rs b/crates/store/src/backend/http/mod.rs index 42836ec4..10fba30d 100644 --- a/crates/store/src/backend/http/mod.rs +++ b/crates/store/src/backend/http/mod.rs @@ -22,13 +22,14 @@ pub struct HttpStoreConfig { pub id: String, pub url: String, pub retry: u64, - pub refresh: u64, + pub refresh_interval: u64, pub timeout: Duration, pub gzipped: bool, pub max_size: usize, pub max_entries: usize, pub max_entry_size: usize, pub format: HttpStoreFormat, + pub custom_user_agent: bool, } #[derive(Debug, Clone)] @@ -50,3 +51,4 @@ pub struct HttpStore { pub config: HttpStoreConfig, pub client: reqwest::Client, } + diff --git a/crates/store/src/backend/local/entry.rs b/crates/store/src/backend/local/entry.rs new file mode 100644 index 00000000..d22f4b40 --- /dev/null +++ b/crates/store/src/backend/local/entry.rs @@ -0,0 +1,266 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Deserialize; +use std::{ + borrow::Borrow, + hash::{Hash, Hasher}, + sync::atomic::{AtomicI64, AtomicU64, Ordering}, +}; + +const INLINE_KEY_LEN: usize = 46; +const EXPIRY_MASK: u64 = (1 << 62) - 1; +const COUNTER_RANK: u64 = 0; +const VALUE_RANK: u64 = 1 << 62; + +pub(super) const NEVER: u64 = u64::MAX; +const DEAD: u64 = 0; +const MIN_EXPIRY: u64 = DEAD + 1; + +pub(super) enum EntryKey { + Inline { + len: u8, + bytes: [u8; INLINE_KEY_LEN], + }, + Heap(Box<[u8]>), +} + +pub(super) struct Entry { + expires: AtomicU64, + payload: Payload, +} + +enum Payload { + Value(Box<[u8]>), + Counter(AtomicI64), + Lock, + Lease(u64), +} + +pub(super) enum Liveness { + Live, + Expired, + Dead, +} + +const _: () = assert!(size_of::() == 48); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(size_of::() == 32); + +impl EntryKey { + #[inline] + pub(super) fn as_bytes(&self) -> &[u8] { + match self { + EntryKey::Inline { len, bytes } => bytes.get(..usize::from(*len)).unwrap_or_default(), + EntryKey::Heap(bytes) => bytes, + } + } +} + +impl From<&[u8]> for EntryKey { + fn from(key: &[u8]) -> Self { + let mut bytes = [0u8; INLINE_KEY_LEN]; + match (bytes.get_mut(..key.len()), u8::try_from(key.len())) { + (Some(inline), Ok(len)) => { + inline.copy_from_slice(key); + EntryKey::Inline { len, bytes } + } + _ => EntryKey::Heap(key.into()), + } + } +} + +impl From> for EntryKey { + fn from(key: Vec) -> Self { + if key.len() <= INLINE_KEY_LEN { + EntryKey::from(key.as_slice()) + } else { + EntryKey::Heap(key.into_boxed_slice()) + } + } +} + +impl Borrow<[u8]> for EntryKey { + #[inline] + fn borrow(&self) -> &[u8] { + self.as_bytes() + } +} + +impl Hash for EntryKey { + #[inline] + fn hash(&self, state: &mut H) { + self.as_bytes().hash(state) + } +} + +impl PartialEq for EntryKey { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.as_bytes() == other.as_bytes() + } +} + +impl Eq for EntryKey {} + +impl Entry { + #[inline] + pub(super) fn expiry_at(now: u64, ttl: u64) -> u64 { + now.saturating_add(ttl).max(MIN_EXPIRY) + } + + pub(super) fn value(value: Vec, expires: u64) -> Self { + Entry { + expires: AtomicU64::new(expires), + payload: Payload::Value(value.into_boxed_slice()), + } + } + + pub(super) fn counter(value: i64, expires: u64) -> Self { + Entry { + expires: AtomicU64::new(expires), + payload: Payload::Counter(AtomicI64::new(value)), + } + } + + pub(super) fn lock(expires: u64) -> Self { + Entry { + expires: AtomicU64::new(expires), + payload: Payload::Lock, + } + } + + pub(super) fn lease(token: u64, expires: u64) -> Self { + Entry { + expires: AtomicU64::new(expires), + payload: Payload::Lease(token), + } + } + + #[inline] + pub(super) fn is_lease(&self, token: u64) -> bool { + matches!(self.payload, Payload::Lease(owner) if owner == token) + } + + pub(super) fn extend_lease(&self, token: u64, expires: u64) -> bool { + if !self.is_lease(token) { + return false; + } + let mut current = self.expires(); + loop { + if current == DEAD { + return false; + } + match self.expires.compare_exchange_weak( + current, + expires, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return true, + Err(actual) => current = actual, + } + } + } + + #[inline] + pub(super) fn expires(&self) -> u64 { + self.expires.load(Ordering::Acquire) + } + + #[inline] + pub(super) fn is_live(&self, now: u64) -> bool { + self.expires() > now + } + + #[inline] + pub(super) fn is_counter(&self) -> bool { + matches!(self.payload, Payload::Counter(_)) + } + + pub(super) fn deserialize(&self) -> trc::Result> { + match &self.payload { + Payload::Value(value) => T::deserialize(value).map(Some), + Payload::Lock => T::deserialize(&[]).map(Some), + Payload::Lease(token) => T::deserialize(&token.to_be_bytes()).map(Some), + Payload::Counter(_) => Ok(None), + } + } + + #[cfg(all(test, stalwart_tests))] + pub(super) fn counter_value(&self) -> i64 { + match &self.payload { + Payload::Counter(counter) => counter.load(Ordering::Relaxed), + Payload::Value(_) | Payload::Lock | Payload::Lease(_) => 0, + } + } + + #[inline] + pub(super) fn increment(&self, delta: i64, expires: Option, now: u64) -> Option { + let Payload::Counter(counter) = &self.payload else { + return None; + }; + let mut current = self.expires(); + loop { + if current <= now { + return None; + } + match expires { + Some(expires) if expires != current => { + match self.expires.compare_exchange_weak( + current, + expires, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + _ => break, + } + } + Some( + counter + .fetch_add(delta, Ordering::Relaxed) + .wrapping_add(delta), + ) + } + + pub(super) fn eviction_score(&self) -> Option { + let rank = match self.payload { + Payload::Counter(_) => COUNTER_RANK, + Payload::Value(_) => VALUE_RANK, + Payload::Lock | Payload::Lease(_) => return None, + }; + Some(rank | self.expires().min(EXPIRY_MASK)) + } + + #[inline] + pub(super) fn mark_dead(&self) -> bool { + self.expires.swap(DEAD, Ordering::AcqRel) != DEAD + } + + pub(super) fn mark_dead_if_expired(&self, now: u64) -> Liveness { + let mut current = self.expires(); + loop { + if current == DEAD { + return Liveness::Dead; + } else if current > now { + return Liveness::Live; + } + match self.expires.compare_exchange_weak( + current, + DEAD, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Liveness::Expired, + Err(actual) => current = actual, + } + } + } +} diff --git a/crates/store/src/backend/local/maintenance.rs b/crates/store/src/backend/local/maintenance.rs new file mode 100644 index 00000000..f93d63f0 --- /dev/null +++ b/crates/store/src/backend/local/maintenance.rs @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + LocalStore, + entry::{Entry, Liveness}, +}; +use std::sync::atomic::Ordering; + +const SWEEP_INTERVAL: u64 = 30; + +impl LocalStore { + pub(super) fn reserve(&self, now: u64) -> trc::Result<()> { + self.sweep_if_due(now); + while self + .state + .len + .try_update(Ordering::AcqRel, Ordering::Acquire, |len| { + (len < self.max_entries).then_some(len + 1) + }) + .is_err() + { + if !self.make_room(now) { + return Err(trc::StoreEvent::UnexpectedError + .into_err() + .details("In-memory store is full")); + } + } + Ok(()) + } + + #[inline] + pub(super) fn release(&self) { + self.state.len.fetch_sub(1, Ordering::AcqRel); + } + + #[inline] + pub(super) fn kill(&self, entry: &Entry) { + if entry.mark_dead() { + self.release(); + } + } + + pub(super) fn kill_expired(&self, entry: &Entry, now: u64) -> bool { + match entry.mark_dead_if_expired(now) { + Liveness::Live => false, + Liveness::Expired => { + self.release(); + true + } + Liveness::Dead => true, + } + } + + pub(super) fn sweep(&self, now: u64) { + self.entries + .pin() + .retain(|_, entry| !self.kill_expired(entry, now)); + } + + fn sweep_if_due(&self, now: u64) { + let next_sweep = self.state.next_sweep.load(Ordering::Relaxed); + if now >= next_sweep + && self + .state + .next_sweep + .compare_exchange( + next_sweep, + now.saturating_add(SWEEP_INTERVAL), + Ordering::Relaxed, + Ordering::Relaxed, + ) + .is_ok() + { + self.sweep(now); + } + } + + #[cold] + #[inline(never)] + fn make_room(&self, now: u64) -> bool { + let _eviction = self.eviction.lock(); + if self.len() < self.max_entries { + return true; + } + + let entries = self.entries.pin(); + let mut expired = false; + let mut scores = Vec::with_capacity(self.len()); + entries.retain(|_, entry| { + if self.kill_expired(entry, now) { + expired = true; + false + } else { + scores.extend(entry.eviction_score()); + true + } + }); + + let target = self.max_entries - self.eviction_batch; + let len = self.len(); + if len > target && !scores.is_empty() { + let excess = len - target; + let (threshold, ties_evicted, ties_total) = if excess < scores.len() { + let (below, threshold, above) = scores.select_nth_unstable(excess - 1); + let threshold = *threshold; + let below_ties = below.iter().filter(|score| **score == threshold).count(); + let above_ties = above.iter().filter(|score| **score == threshold).count(); + ( + threshold, + excess - (below.len() - below_ties), + below_ties + above_ties + 1, + ) + } else { + (u64::MAX, 0, 1) + }; + + let mut ties_stride = 0; + entries.retain(|_, entry| { + let evict = self.kill_expired(entry, now) + || match entry.eviction_score() { + Some(score) if score < threshold => true, + Some(score) if score == threshold => { + ties_stride += ties_evicted; + if ties_stride >= ties_total { + ties_stride -= ties_total; + true + } else { + false + } + } + _ => false, + }; + if evict { + self.kill(entry); + } + !evict + }); + } + + expired || !scores.is_empty() || self.len() < self.max_entries + } +} diff --git a/crates/store/src/backend/local/mod.rs b/crates/store/src/backend/local/mod.rs new file mode 100644 index 00000000..45620867 --- /dev/null +++ b/crates/store/src/backend/local/mod.rs @@ -0,0 +1,252 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod entry; +mod maintenance; + +use self::entry::{Entry, EntryKey, NEVER}; +use crate::Deserialize; +use ahash::RandomState; +use papaya::{Compute, HashMap, Operation, ResizeMode}; +use parking_lot::Mutex; +use std::{ + fmt, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, +}; + +pub struct LocalStore { + entries: HashMap, + state: SharedState, + max_entries: usize, + eviction_batch: usize, + eviction: Mutex<()>, +} + +#[repr(align(128))] +struct SharedState { + len: AtomicUsize, + next_sweep: AtomicU64, +} + +impl LocalStore { + pub const DEFAULT_MAX_ENTRIES: usize = 1 << 20; + const EVICTION_DIVISOR: usize = 16; + + pub fn new(max_entries: usize) -> Self { + let max_entries = max_entries.max(1); + LocalStore { + entries: HashMap::builder() + .hasher(RandomState::new()) + .resize_mode(ResizeMode::Blocking) + .build(), + state: SharedState { + len: AtomicUsize::new(0), + next_sweep: AtomicU64::new(0), + }, + max_entries, + eviction_batch: (max_entries / Self::EVICTION_DIVISOR).max(1), + eviction: Mutex::new(()), + } + } + + pub fn key_set( + &self, + key: Vec, + value: Vec, + expires: Option, + now: u64, + ) -> trc::Result<()> { + let expires = expires.map_or(NEVER, |ttl| Entry::expiry_at(now, ttl)); + self.reserve(now)?; + if let Some(previous) = self + .entries + .pin() + .insert(EntryKey::from(key), Entry::value(value, expires)) + { + self.kill(previous); + } + Ok(()) + } + + pub fn key_incr( + &self, + key: Vec, + delta: i64, + expires: Option, + now: u64, + ) -> trc::Result { + let expires = expires.map(|ttl| Entry::expiry_at(now, ttl)); + let entries = self.entries.pin(); + loop { + if let Some(value) = entries + .get(key.as_slice()) + .and_then(|entry| entry.increment(delta, expires, now)) + { + return Ok(value); + } + + self.reserve(now)?; + match entries.compute(EntryKey::from(key.as_slice()), |current| match current { + Some((_, entry)) if !self.kill_expired(entry, now) && entry.is_counter() => { + Operation::Abort(()) + } + _ => Operation::Insert(Entry::counter(delta, expires.unwrap_or(NEVER))), + }) { + Compute::Inserted(..) => return Ok(delta), + Compute::Updated { + old: (_, previous), .. + } => { + self.kill(previous); + return Ok(delta); + } + Compute::Aborted(_) | Compute::Removed(..) => self.release(), + } + } + } + + pub fn key_get(&self, key: &[u8], now: u64) -> trc::Result> { + match self.entries.pin().get(key) { + Some(entry) if entry.is_live(now) => entry.deserialize(), + _ => Ok(None), + } + } + + pub fn key_exists(&self, key: &[u8], now: u64) -> bool { + self.entries + .pin() + .get(key) + .is_some_and(|entry| entry.is_live(now)) + } + + pub fn key_delete(&self, key: &[u8], now: u64) -> bool { + match self.entries.pin().remove(key) { + Some(previous) => { + let is_live = previous.is_live(now); + self.kill(previous); + is_live + } + None => false, + } + } + + pub fn key_delete_prefix(&self, prefix: &[u8]) { + if !prefix.is_empty() { + self.entries.pin().retain(|key, entry| { + if key.as_bytes().starts_with(prefix) { + self.kill(entry); + false + } else { + true + } + }); + } + } + + pub fn try_lock(&self, key: Vec, duration: u64, now: u64) -> trc::Result { + self.try_insert(key, duration, now, Entry::lock) + } + + pub fn renew_lock(&self, key: Vec, duration: u64, now: u64) -> trc::Result<()> { + self.reserve(now)?; + if let Some(previous) = self.entries.pin().insert( + EntryKey::from(key), + Entry::lock(Entry::expiry_at(now, duration)), + ) { + self.kill(previous); + } + Ok(()) + } + + pub fn try_lease( + &self, + key: Vec, + token: u64, + duration: u64, + now: u64, + ) -> trc::Result { + self.try_insert(key, duration, now, |expires| Entry::lease(token, expires)) + } + + fn try_insert( + &self, + key: Vec, + duration: u64, + now: u64, + entry: impl Fn(u64) -> Entry, + ) -> trc::Result { + let entries = self.entries.pin(); + if entries + .get(key.as_slice()) + .is_some_and(|entry| entry.is_live(now)) + { + return Ok(false); + } + + self.reserve(now)?; + let expires = Entry::expiry_at(now, duration); + match entries.compute(EntryKey::from(key), |current| match current { + Some((_, entry)) if !self.kill_expired(entry, now) => Operation::Abort(()), + _ => Operation::Insert(entry(expires)), + }) { + Compute::Inserted(..) => Ok(true), + Compute::Updated { + old: (_, previous), .. + } => { + self.kill(previous); + Ok(true) + } + Compute::Aborted(_) | Compute::Removed(..) => { + self.release(); + Ok(false) + } + } + } + + pub fn renew_lease(&self, key: &[u8], token: u64, duration: u64, now: u64) -> bool { + self.entries.pin().get(key).is_some_and(|entry| { + entry.is_live(now) && entry.extend_lease(token, Entry::expiry_at(now, duration)) + }) + } + + pub fn release_lease(&self, key: Vec, token: u64) -> bool { + match self + .entries + .pin() + .compute(EntryKey::from(key), |current| match current { + Some((_, entry)) if entry.is_lease(token) => Operation::Remove, + _ => Operation::Abort(()), + }) { + Compute::Removed(_, previous) => { + self.kill(previous); + true + } + Compute::Inserted(..) | Compute::Updated { .. } | Compute::Aborted(_) => false, + } + } + + pub fn len(&self) -> usize { + self.state.len.load(Ordering::Relaxed) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Default for LocalStore { + fn default() -> Self { + Self::new(Self::DEFAULT_MAX_ENTRIES) + } +} + +impl fmt::Debug for LocalStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalStore") + .field("len", &self.len()) + .field("max_entries", &self.max_entries) + .finish() + } +} diff --git a/crates/store/src/backend/meili/main.rs b/crates/store/src/backend/meili/main.rs index 6db9f62e..0fea5674 100644 --- a/crates/store/src/backend/meili/main.rs +++ b/crates/store/src/backend/meili/main.rs @@ -8,8 +8,8 @@ use crate::{ SearchStore, backend::meili::{MeiliSearchStore, Task, TaskStatus, TaskUid}, search::{ - CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField, - TracingSearchField, + CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField, + SearchableField, TracingSearchField, }, }; use registry::schema::structs; @@ -49,7 +49,7 @@ impl MeiliSearchStore { client: ms.client, url: ms.url, task_poll_interval: config.poll_interval.into_inner(), - task_poll_retries: config.max_retries as usize, + task_poll_retries: config.max_polls as usize, task_fail_on_timeout: config.fail_on_timeout, }))) } @@ -58,6 +58,7 @@ impl MeiliSearchStore { self.create_index::().await?; self.create_index::().await?; self.create_index::().await?; + self.create_index::().await?; self.create_index::().await?; Ok(()) } @@ -86,6 +87,9 @@ impl MeiliSearchStore { let index_name = T::index().index_name(); if self.index_exists(index_name).await? { + if self.index_max_total_hits(index_name).await? != Some(MAX_TOTAL_HITS) { + self.update_index_pagination(index_name).await?; + } return Ok(()); } @@ -131,7 +135,7 @@ impl MeiliSearchStore { } } - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] filterable.push(Value::String("bcc".into())); if !searchable.is_empty() { @@ -162,6 +166,27 @@ impl MeiliSearchStore { Ok(()) } + async fn index_max_total_hits(&self, index_uid: &str) -> trc::Result> { + let response = assert_success( + self.client + .get(format!( + "{}/indexes/{}/settings/pagination", + self.url, index_uid + )) + .send() + .await, + ) + .await?; + let body = response + .text() + .await + .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?; + + serde_json::from_str::(&body) + .map(|value| value.get("maxTotalHits").and_then(Value::as_u64)) + .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err)) + } + async fn update_index_pagination(&self, index_uid: &str) -> trc::Result { let response = assert_success( self.client @@ -198,44 +223,6 @@ impl MeiliSearchStore { self.wait_for_task(response).await } - #[cfg(feature = "test_mode")] - pub async fn drop_indexes(&self) -> trc::Result<()> { - use crate::write::SearchIndex; - - for index in &[ - SearchIndex::Email, - SearchIndex::Calendar, - SearchIndex::Contacts, - SearchIndex::Tracing, - ] { - let response = self - .client - .delete(format!("{}/indexes/{}", self.url, index.index_name())) - .send() - .await - .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?; - - match response.status().as_u16() { - 200..=299 => { - self.wait_for_task(response).await?; - } - 400..=499 => { - // Index does not exist - return Ok(()); - } - _ => { - let status = response.status(); - let msg = response.text().await.unwrap_or_default(); - return Err(trc::StoreEvent::MeilisearchError - .reason(msg) - .ctx(trc::Key::Code, status.as_u16())); - } - } - } - - Ok(()) - } - pub(crate) async fn wait_for_task(&self, response: Response) -> trc::Result { let response_body = response.text().await.map_err(|err| { trc::StoreEvent::MeilisearchError @@ -323,3 +310,4 @@ pub(crate) async fn assert_success(response: Result) -> trc::Re Err(err) => Err(trc::StoreEvent::MeilisearchError.reason(err)), } } + diff --git a/crates/store/src/backend/meili/search.rs b/crates/store/src/backend/meili/search.rs index 4c08ebc8..fedca8eb 100644 --- a/crates/store/src/backend/meili/search.rs +++ b/crates/store/src/backend/meili/search.rs @@ -14,7 +14,7 @@ use crate::{ }; use ahash::AHashSet; use serde_json::{Map, Value, json}; -use std::fmt::{Display, Write}; +use std::{cmp::Ordering, fmt::Write}; const MAX_SEARCH_RESULTS: usize = 10_000; @@ -55,7 +55,7 @@ impl MeiliSearchStore { let response = assert_success( self.client - .put(format!( + .post(format!( "{}/indexes/{}/documents", self.url, index.index_name() @@ -71,15 +71,14 @@ impl MeiliSearchStore { Ok(()) } - pub async fn query( + pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], - sort: &[SearchComparator], - ) -> trc::Result> { + ) -> trc::Result { let filter_group = build_query(filters); - if filter_group.q.is_empty() && sort.is_empty() { + if filter_group.q.is_empty() { return self.fetch_documents(index, &filter_group.filter).await; } @@ -115,25 +114,8 @@ impl MeiliSearchStore { } } - if !sort.is_empty() { - let sort_arr: Vec = sort - .iter() - .filter_map(|comp| match comp { - SearchComparator::Field { field, ascending } => Some(Value::String(format!( - "{}:{}", - field.field_name(), - if *ascending { "asc" } else { "desc" } - ))), - _ => None, - }) - .collect(); - if !sort_arr.is_empty() { - body.insert("sort".to_string(), Value::Array(sort_arr)); - } - } - let url = format!("{}/indexes/{}/search", self.url, index.index_name()); - let mut results = Vec::new(); + let mut results = R::default(); let mut offset = 0; loop { @@ -162,7 +144,9 @@ impl MeiliSearchStore { .hits; let total = hits.len(); - results.extend(hits.into_iter().map(|hit| R::from_u64(hit.id))); + for hit in hits { + results.insert(hit.id); + } if total < MAX_SEARCH_RESULTS { break; @@ -174,7 +158,7 @@ impl MeiliSearchStore { trc::event!( Store(trc::StoreEvent::MeilisearchError), Reason = "Search results were truncated", - Collection = index.index_name(), + Index = index.index_name(), Total = offset, ); break; @@ -184,17 +168,17 @@ impl MeiliSearchStore { Ok(results) } - async fn fetch_documents( + async fn fetch_documents( &self, index: SearchIndex, filter: &str, - ) -> trc::Result> { + ) -> trc::Result { let url = format!( "{}/indexes/{}/documents/fetch", self.url, index.index_name() ); - let mut results = Vec::new(); + let mut results = R::default(); let mut offset = 0; loop { @@ -233,7 +217,9 @@ impl MeiliSearchStore { .results; let total = documents.len(); - results.extend(documents.into_iter().map(|hit| R::from_u64(hit.id))); + for document in documents { + results.insert(document.id); + } if total < MAX_SEARCH_RESULTS { break; @@ -295,23 +281,13 @@ fn build_query(filters: &[SearchFilter]) -> FilterGroup { for f in filters { match f { - SearchFilter::Operator { field, op, value } => { - if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) - { - let value = match value { - SearchValue::Text { value, .. } => value, - _ => { - debug_assert!( - false, - "Text field search with non-text value is not supported" - ); - "" - } - }; - + SearchFilter::Text { + field, op, value, .. + } => { + if field.is_text() { search_on.insert(field.field_name()); - if matches!(op, SearchOperator::Equal) { + if op == &TextMatch::Exact { queries.insert(format!("{value:?}")); } else { for token in value.split_whitespace() { @@ -327,37 +303,50 @@ fn build_query(filters: &[SearchFilter]) -> FilterGroup { } } - match value { - SearchValue::Text { value, .. } => { - filter.push_str(field.field_name()); - filter.push(' '); - op.write_meli_op(&mut filter, format!("{value:?}")); - } - SearchValue::KeyValues(kv) => { - let (key, value) = kv.iter().next().unwrap(); - filter.push_str(field.field_name()); - filter.push('.'); - filter.push_str(key); - filter.push(' '); - op.write_meli_op(&mut filter, format!("{value:?}")); - } - SearchValue::Int(v) => { - filter.push_str(field.field_name()); - filter.push(' '); - op.write_meli_op(&mut filter, v); - } - SearchValue::Uint(v) => { - filter.push_str(field.field_name()); - filter.push(' '); - op.write_meli_op(&mut filter, v); - } - SearchValue::Boolean(v) => { - filter.push_str(field.field_name()); - filter.push(' '); - op.write_meli_op(&mut filter, v); - } + filter.push_str(field.field_name()); + let _ = write!(filter, " = {value:?}"); + } + } + SearchFilter::KeyValue { field, key, op } => { + if !filter.is_empty() && !filter.ends_with('(') { + match operator { + SearchFilter::And => filter.push_str(" AND "), + SearchFilter::Or => filter.push_str(" OR "), + _ => (), } } + + filter.push_str(field.field_name()); + filter.push('.'); + filter.push_str(key); + match op { + KeyValueMatch::Equals(value) => { + let _ = write!(filter, " = {value:?}"); + } + KeyValueMatch::Contains(value) => { + let _ = write!(filter, " CONTAINS {value:?}"); + } + KeyValueMatch::Exists => { + filter.push_str(" EXISTS"); + } + } + } + SearchFilter::Integer { field, op, value } => { + if !filter.is_empty() && !filter.ends_with('(') { + match operator { + SearchFilter::And => filter.push_str(" AND "), + SearchFilter::Or => filter.push_str(" OR "), + _ => (), + } + } + + filter.push_str(field.field_name()); + let cmp = match op { + Ordering::Less => "<", + Ordering::Equal => "=", + Ordering::Greater => ">", + }; + let _ = write!(filter, " {cmp} {value}"); } SearchFilter::And | SearchFilter::Or => { if !filter.is_empty() && !filter.ends_with('(') { @@ -434,28 +423,6 @@ fn build_query(filters: &[SearchFilter]) -> FilterGroup { } } -impl SearchOperator { - fn write_meli_op(&self, query: &mut String, value: impl Display) { - match self { - SearchOperator::LowerThan => { - let _ = write!(query, "< {value}"); - } - SearchOperator::LowerEqualThan => { - let _ = write!(query, "<= {value}"); - } - SearchOperator::GreaterThan => { - let _ = write!(query, "> {value}"); - } - SearchOperator::GreaterEqualThan => { - let _ = write!(query, ">= {value}"); - } - SearchOperator::Equal | SearchOperator::Contains => { - let _ = write!(query, "= {value}"); - } - } - } -} - fn json_serialize(request: &mut String, document: &IndexDocument) { let mut id = 0u64; let mut is_first = true; diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index b3508a36..99ae94cd 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -12,12 +12,15 @@ pub mod ephemeral; pub mod foundationdb; pub mod fs; pub mod http; +pub mod local; pub mod meili; pub mod memory; #[cfg(feature = "mysql")] pub mod mysql; #[cfg(feature = "postgres")] pub mod postgres; +#[cfg(any(feature = "postgres", feature = "mysql"))] +pub(crate) mod range_batch; #[cfg(feature = "redis")] pub mod redis; #[cfg(feature = "rocks")] @@ -31,7 +34,146 @@ pub mod sqlite; pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize; pub const MAX_TOKEN_MASK: usize = MAX_TOKEN_LENGTH - 1; +#[cfg(any(feature = "postgres", feature = "mysql"))] +pub(crate) fn default_pool_max_connections() -> usize { + use std::thread::available_parallelism; + + const CONNECTIONS_PER_CORE: usize = 4; + const MIN_CONNECTIONS: usize = 16; + const MAX_CONNECTIONS: usize = 32; + + available_parallelism() + .map_or(MIN_CONNECTIONS, |cores| { + cores.get().saturating_mul(CONNECTIONS_PER_CORE) + }) + .clamp(MIN_CONNECTIONS, MAX_CONNECTIONS) +} + +#[cfg(any(feature = "rocks", feature = "sqlite"))] +pub(crate) mod worker { + use rayon::ThreadPool; + use tokio::{ + runtime::{Handle, RuntimeFlavor}, + sync::{Semaphore, oneshot}, + }; + + const MIN_POOL_WORKERS: usize = 4; + const READ_PERMITS_PER_WORKER: usize = 4; + + #[derive(Clone, Copy)] + pub(crate) enum Access { + Read, + Write, + } + + pub(crate) struct BlockingPermits { + reads: Semaphore, + writes: Semaphore, + } + + pub(crate) fn pool_workers(configured: Option) -> usize { + configured + .filter(|v| *v > 0) + .map(|v| v as usize) + .unwrap_or_else(num_cpus::get) + .max(MIN_POOL_WORKERS) + } + + impl BlockingPermits { + pub(crate) fn new(pool_workers: usize) -> Self { + Self { + reads: Semaphore::new( + pool_workers + .saturating_mul(READ_PERMITS_PER_WORKER) + .min(Semaphore::MAX_PERMITS), + ), + writes: Semaphore::new(pool_workers.min(Semaphore::MAX_PERMITS)), + } + } + + pub(crate) fn block( + &self, + access: Access, + f: U, + ) -> impl Future> + where + U: FnMut() -> trc::Result + Send, + V: Send, + { + let permits = match access { + Access::Read => &self.reads, + Access::Write => &self.writes, + }; + async move { + let _permit = match permits.try_acquire() { + Ok(permit) => permit, + Err(_) => permits.acquire().await.map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err) + })?, + }; + block(f) + } + } + } + + pub(crate) async fn spawn(pool: &ThreadPool, f: U) -> trc::Result + where + U: FnOnce() -> trc::Result + Send + 'static, + V: Send + 'static, + { + let (tx, rx) = oneshot::channel(); + + pool.spawn(move || { + tx.send(f()).ok(); + }); + + match rx.await { + Ok(result) => result, + Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err)), + } + } + + pub(crate) fn block(mut f: U) -> trc::Result + where + U: FnMut() -> trc::Result + Send, + V: Send, + { + #[cfg(stalwart_tests)] + let mut f = super::block_hook::wrap(&mut f); + match Handle::try_current().map(|handle| handle.runtime_flavor()) { + Ok(RuntimeFlavor::MultiThread) => tokio::task::block_in_place(&mut f), + _ => f(), + } + } +} + + #[allow(dead_code)] +#[cfg(any(feature = "postgres", feature = "mysql", feature = "sqlite"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ValueColumn { + Skip, + Bytes, + Counter, +} + +#[cfg(any(feature = "postgres", feature = "mysql", feature = "sqlite"))] +impl ValueColumn { + #[inline] + pub(crate) fn new(values: bool, subspace: crate::Subspace) -> Self { + match (values, subspace.shape()) { + (false, _) => ValueColumn::Skip, + (true, crate::Shape::Counter) => ValueColumn::Counter, + (true, _) => ValueColumn::Bytes, + } + } + + #[inline] + pub(crate) fn is_selected(self) -> bool { + self != ValueColumn::Skip + } +} + fn deserialize_i64_le(key: &[u8], bytes: &[u8]) -> trc::Result { Ok(i64::from_le_bytes(bytes[..].try_into().map_err(|_| { trc::Error::corrupted_key(key, bytes.into(), trc::location!()) diff --git a/crates/store/src/backend/mysql/blob.rs b/crates/store/src/backend/mysql/blob.rs index 983a358c..363055c1 100644 --- a/crates/store/src/backend/mysql/blob.rs +++ b/crates/store/src/backend/mysql/blob.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::ops::Range; - -use mysql_async::prelude::Queryable; - use super::{MysqlStore, into_error}; +use mysql_async::prelude::Queryable; +use std::ops::Range; impl MysqlStore { pub(crate) async fn get_blob( diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index d7ab0d3d..9543fd33 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -6,10 +6,10 @@ use super::{MysqlStore, into_error}; use crate::{ - backend::mysql::MysqlSearchField, + backend::mysql::{MysqlSearchField, sql::SqlStatements}, search::{ - CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, - TracingSearchField, + CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, + SearchableField, TracingSearchField, }, *, }; @@ -18,6 +18,10 @@ use mysql_async::{ Conn, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable, }; +const STMT_CACHE_SIZE: usize = 64; +const SESSION_SETUP: &str = + "SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; SET SESSION autocommit = 1"; + impl MysqlStore { pub async fn open(config: structs::MySqlStore) -> Result { let mut opts = OptsBuilder::default() @@ -26,8 +30,10 @@ impl MysqlStore { .pass(config.auth_secret.secret().await?.map(|v| v.into_owned())) .db_name(Some(config.database)) .max_allowed_packet(config.max_allowed_packet.map(|v| v as usize)) - .wait_timeout(config.timeout.map(|t| t.as_secs() as usize)) + .wait_timeout(config.idle_timeout.map(|t| t.as_secs() as usize)) + .stmt_cache_size(STMT_CACHE_SIZE) .client_found_rows(true) + .setup(vec![SESSION_SETUP]) .tcp_port(config.port as u16); if config.use_tls { @@ -38,35 +44,21 @@ impl MysqlStore { )); } - // Configure connection pool - let mut pool_min = PoolConstraints::default().min(); - let mut pool_max = PoolConstraints::default().max(); - if let Some(n_size) = config.pool_min_connections { - pool_min = n_size as usize; - } - if let Some(n_size) = config.pool_max_connections { - pool_max = n_size as usize; - } opts = opts.pool_opts( - PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()), + PoolOpts::default() + .with_constraints(pool_constraints( + config.pool_max_connections, + config.pool_min_connections, + backend::default_pool_max_connections, + )?) + .with_reset_connection(false), ); - let mut replicas = vec![]; - for replica in config.read_replicas { - replicas.push(Store::MySQL(Arc::new(MysqlStore { - conn_pool: Pool::new( - opts.clone() - .ip_or_hostname(replica.host) - .user(replica.auth_username) - .pass(replica.auth_secret.secret().await?.map(|v| v.into_owned())) - .db_name(Some(replica.database)) - .tcp_port(replica.port as u16), - ), - }))) - } + let sql = Arc::new(SqlStatements::new()); let primary = Store::MySQL(Arc::new(MysqlStore { conn_pool: Pool::new(opts), + sql, })); @@ -75,69 +67,31 @@ impl MysqlStore { pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + check_prepared_statement_budget(&mut conn).await; + + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| !subspace.is_internal_fts()) + { + let table = subspace.name(); + let columns = match subspace.shape() { + Shape::Value => { + let value_type = match subspace { + Subspace::Blobs | Subspace::Immutable => "LONGBLOB", + _ => "MEDIUMBLOB", + }; + format!("k VARBINARY(255) NOT NULL, v {value_type} NOT NULL, PRIMARY KEY (k)") + } + Shape::Presence => "k VARBINARY(767) NOT NULL, PRIMARY KEY (k)".to_string(), + Shape::Counter => { + "k VARBINARY(255) NOT NULL, v BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (k)" + .to_string() + } + }; - for table in [ - SUBSPACE_ACL, - SUBSPACE_TASK_QUEUE, - SUBSPACE_DELETED_ITEMS, - SUBSPACE_SPAM_SAMPLES, - SUBSPACE_BLOB_LINK, - SUBSPACE_IN_MEMORY_VALUE, - SUBSPACE_PROPERTY, - SUBSPACE_REGISTRY, - SUBSPACE_REGISTRY_PK, - SUBSPACE_DIRECTORY, - SUBSPACE_QUEUE_MESSAGE, - SUBSPACE_QUEUE_EVENT, - SUBSPACE_REPORT_OUT, - SUBSPACE_REPORT_IN, - SUBSPACE_LOGS, - SUBSPACE_TELEMETRY_SPAN, - SUBSPACE_TELEMETRY_METRIC, - ] { - let table = char::from(table); conn.query_drop(format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k VARBINARY(255) NOT NULL, - v MEDIUMBLOB NOT NULL, - PRIMARY KEY (k) - ) ENGINE=InnoDB" - )) - .await - .map_err(into_error)?; - } - - conn.query_drop(format!( - "CREATE TABLE IF NOT EXISTS {} ( - k VARBINARY(255) NOT NULL, - v LONGBLOB NOT NULL, - PRIMARY KEY (k) - ) ENGINE=InnoDB", - char::from(SUBSPACE_BLOBS), - )) - .await - .map_err(into_error)?; - - for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); - conn.query_drop(format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k BLOB, - PRIMARY KEY (k(400)) - ) ENGINE=InnoDB" - )) - .await - .map_err(into_error)?; - } - - for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { - conn.query_drop(format!( - "CREATE TABLE IF NOT EXISTS {} ( - k VARBINARY(255) NOT NULL, - v BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (k) - ) ENGINE=InnoDB", - char::from(table) + "CREATE TABLE IF NOT EXISTS {table} ({columns}) ENGINE=InnoDB" )) .await .map_err(into_error)?; @@ -152,13 +106,46 @@ impl MysqlStore { create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; - //create_search_tables::(&mut conn).await?; + create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; Ok(()) } } +fn pool_constraints( + max: Option, + min: Option, + default_max: impl FnOnce() -> usize, +) -> Result { + let min = min.map_or_else(|| PoolConstraints::default().min(), |min| min as usize); + let max = max.map_or_else(|| default_max().max(min), |max| max as usize); + PoolConstraints::new(min.min(max), max) + .ok_or_else(|| format!("Invalid MySQL poolMaxConnections {max}: must be at least 1")) +} + +async fn check_prepared_statement_budget(conn: &mut Conn) { + let opts = conn.opts(); + let statements = opts + .pool_opts() + .constraints() + .max() + .saturating_mul(opts.stmt_cache_size()) as u64; + if let Ok(Some(limit)) = conn + .query_first::("SELECT @@GLOBAL.max_prepared_stmt_count") + .await + && statements > limit + { + trc::event!( + Registry(trc::RegistryEvent::BuildWarning), + Property = "poolMaxConnections", + Reason = "Prepared statements cached by this node can exceed the server's max_prepared_stmt_count", + Total = statements, + Limit = limit, + ); + } +} + async fn create_search_tables( conn: &mut Conn, ) -> trc::Result<()> { @@ -210,3 +197,4 @@ async fn create_search_tables( Ok(()) } + diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs index cdbaad9e..f1aa4fbe 100644 --- a/crates/store/src/backend/mysql/mod.rs +++ b/crates/store/src/backend/mysql/mod.rs @@ -12,17 +12,22 @@ use crate::{ write::SearchIndex, }; use mysql_async::Pool; -use std::fmt::Display; +use std::{fmt::Display, sync::Arc}; + +use self::sql::SqlStatements; pub mod blob; pub mod lookup; pub mod main; +mod pooled; pub mod read; pub mod search; +pub mod sql; pub mod write; pub struct MysqlStore { pub(crate) conn_pool: Pool, + pub(crate) sql: Arc, } #[inline(always)] @@ -30,15 +35,19 @@ fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::MysqlError.reason(err) } +const ER_DUP_ENTRY: u16 = 1062; const ER_UNKNOWN_ERROR: u16 = 1105; const ER_TRANS_CACHE_FULL: u16 = 1197; const ER_LOCK_WAIT_TIMEOUT: u16 = 1205; +const ER_LOCK_DEADLOCK: u16 = 1213; const ER_LOCK_TABLE_FULL: u16 = 1206; const ER_STATEMENT_TIMEOUT: u16 = 1969; const ER_QUERY_TIMEOUT: u16 = 3024; pub(crate) const DELETE_CHUNK_SIZE: usize = 1000; pub(crate) const MIN_DELETE_CHUNK_SIZE: usize = 10; +pub(crate) const ITERATE_CHUNK_SIZE: usize = 1000; +pub(crate) const MIN_ITERATE_CHUNK_SIZE: usize = 10; #[inline(always)] pub(crate) fn is_timeout_error(err: &mysql_async::Error) -> bool { @@ -89,19 +98,12 @@ impl MysqlSearchField for EmailSearchField { EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "atta", - EmailSearchField::ReceivedAt => "rcvd", - EmailSearchField::SentAt => "sent", - EmailSearchField::Size => "size", - EmailSearchField::HasAttachment => "hatt", EmailSearchField::Headers => "hdrs", } } fn column_type(&self) -> &'static str { match self { - EmailSearchField::ReceivedAt | EmailSearchField::SentAt => "BIGINT", - EmailSearchField::Size => "INT", - EmailSearchField::HasAttachment => "BOOLEAN", EmailSearchField::Headers => "JSON", EmailSearchField::From => "TEXT", EmailSearchField::To => "TEXT", @@ -122,16 +124,11 @@ impl MysqlSearchField for CalendarSearchField { CalendarSearchField::Location => "locn", CalendarSearchField::Owner => "ownr", CalendarSearchField::Attendee => "atnd", - CalendarSearchField::Start => "strt", - CalendarSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { - match self { - CalendarSearchField::Start => "BIGINT NOT NULL", - _ => "TEXT", - } + "TEXT" } } @@ -148,13 +145,12 @@ impl MysqlSearchField for ContactSearchField { ContactSearchField::Address => "addr", ContactSearchField::Note => "note", ContactSearchField::Kind => "kind", - ContactSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { - ContactSearchField::Kind | ContactSearchField::Uid => "TEXT", + ContactSearchField::Kind => "TEXT", _ => "TEXT", } } @@ -163,14 +159,12 @@ impl MysqlSearchField for ContactSearchField { impl MysqlSearchField for FileSearchField { fn column(&self) -> &'static str { match self { - FileSearchField::Name => "name", FileSearchField::Content => "body", } } fn column_type(&self) -> &'static str { match self { - FileSearchField::Name => "TEXT", FileSearchField::Content => "MEDIUMTEXT", } } diff --git a/crates/store/src/backend/mysql/pooled.rs b/crates/store/src/backend/mysql/pooled.rs new file mode 100644 index 00000000..80466328 --- /dev/null +++ b/crates/store/src/backend/mysql/pooled.rs @@ -0,0 +1,64 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::into_error; +use futures::FutureExt; +use mysql_async::Conn; +use std::ops::{Deref, DerefMut}; + +pub(super) struct PooledConn { + conn: Option, + detach: bool, +} + +impl PooledConn { + pub(super) fn new(conn: Conn) -> Self { + PooledConn { + conn: Some(conn), + detach: false, + } + } + + pub(super) fn end_transaction( + &mut self, + result: Result<(), mysql_async::Error>, + ) -> trc::Result<()> { + result.map_err(|err| { + self.detach = true; + into_error(err) + }) + } +} + +impl Deref for PooledConn { + type Target = Conn; + + #[inline] + fn deref(&self) -> &Conn { + self.conn + .as_ref() + .expect("the connection is only taken when the guard is dropped") + } +} + +impl DerefMut for PooledConn { + #[inline] + fn deref_mut(&mut self) -> &mut Conn { + self.conn + .as_mut() + .expect("the connection is only taken when the guard is dropped") + } +} + +impl Drop for PooledConn { + fn drop(&mut self) { + if self.detach + && let Some(conn) = self.conn.take() + { + drop(conn.disconnect().now_or_never()); + } + } +} diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index 3fc387e0..21902156 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -5,9 +5,25 @@ */ use super::{MysqlStore, into_error, is_timeout_error}; -use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; +use crate::{ + Deserialize, Fetch, IterateParams, Key, Subspace, ValueKey, + backend::{ + ValueColumn, + mysql::{ITERATE_CHUNK_SIZE, MIN_ITERATE_CHUNK_SIZE, pooled::PooledConn}, + range_batch::{Bounds, RangeAttempt, RangeCursor, Timeout}, + }, + dispatch::store::point_keys, + write::{AnyKey, ChunkPlan, ValueClass, backoff}, +}; use futures::TryStreamExt; -use mysql_async::{Row, prelude::Queryable}; +use mysql_async::{Conn, IsolationLevel, Row, Transaction, TxOpts, Value, prelude::Queryable}; +use std::fmt::Write; + +pub(super) const POINT_BUCKETS: [usize; 4] = [16, 64, 256, MAX_POINT_KEYS]; +pub(super) const MAX_POINT_KEYS: usize = 768; +const DEGRADED_POINT_KEYS: usize = 256; +const MIN_POINT_KEYS: usize = 16; +const MAX_RANGES_PER_STMT: usize = 64; impl MysqlStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -16,10 +32,7 @@ impl MysqlStore { { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn - .prep(format!( - "SELECT v FROM {} WHERE k = ?", - char::from(key.subspace()) - )) + .prep(&*self.sql.get(key.subspace()).get_value) .await .map_err(into_error)?; let key = key.serialize(0); @@ -38,10 +51,7 @@ impl MysqlStore { pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn - .prep(format!( - "SELECT 1 FROM {} WHERE k = ?", - char::from(key.subspace()) - )) + .prep(&*self.sql.get(key.subspace()).key_exists) .await .map_err(into_error)?; let key = key.serialize(0); @@ -56,94 +66,135 @@ impl MysqlStore { params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { + debug_assert_ne!(params.fetch, Fetch::Limit(0)); let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; - let table = char::from(params.begin.subspace()); - let begin = params.begin.serialize(0); - let end = params.end.serialize(0); - let keys = if params.values { "k, v" } else { "k" }; - - let s = conn - .prep(&match (params.first, params.ascending) { - (true, true) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" - ) - } - (true, false) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" - ) - } - (false, true) => { - format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") - } - (false, false) => { - format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC") - } - }) - .await - .map_err(into_error)?; - let mut from = begin; - let mut to = end; - let mut resume_key = None; + let subspace = params.begin.subspace(); + let stmts = self.sql.get(subspace); + let column = ValueColumn::new(params.values, subspace); + let mut from = params.begin.serialize(0); + let mut to = params.end.serialize(0); + let mut last_key = Vec::new(); + let mut resuming = false; + let mut plan = ChunkPlan::new(params.fetch); + let mut attempt = 0; loop { - let mut last_key = None; - let mut timed_out = false; + let limit = plan.limit(resuming); + let (statement, bound) = match limit { + Some(limit) if limit > 1 => ( + stmts.iterate_limit(params.ascending, params.values), + Some(limit), + ), + limit => ( + stmts.iterate(limit.is_some(), params.ascending, params.values), + None, + ), + }; + let mut skip_resumed = resuming; + let mut progressed = false; + let mut fetched = 0u64; + let mut timed_out = None; - { - let mut rows = conn - .exec_stream::(&s, (from.clone(), to.clone())) - .await - .map_err(into_error)?; - - loop { + let result = match conn.prep(statement).await { + Ok(s) => match bound { + Some(bound) => { + conn.exec_stream::(&s, (from.clone(), to.clone(), bound)) + .await + } + None => { + conn.exec_stream::(&s, (from.clone(), to.clone())) + .await + } + }, + Err(err) => Err(err), + }; + match result { + Ok(mut rows) => loop { match rows.try_next().await { Ok(Some(mut row)) => { - let value = if params.values { - row.take_opt::, _>(1) - .unwrap_or_else(|| Ok(vec![])) - .map_err(into_error)? - } else { - vec![] - }; + let value = RowValue::take(&mut row, column)?; let key = row .take_opt::, _>(0) .unwrap_or_else(|| Ok(vec![])) .map_err(into_error)?; - if resume_key.take().is_some_and(|resumed| resumed == key) { - continue; + fetched += 1; + if skip_resumed { + skip_resumed = false; + if key == last_key { + continue; + } } - if !cb(&key, &value)? { + if !cb(&key, value.as_slice())? || !plan.consume_row() { return Ok(()); } - last_key = Some(key); + last_key = key; + progressed = true; } Ok(None) => break, - Err(err) => { - if params.first || last_key.is_none() || !is_timeout_error(&err) { - return Err(into_error(err)); - } - timed_out = true; + Err(err) if is_timeout_error(&err) => { + timed_out = Some(err); break; } + Err(err) => return Err(into_error(err)), } - } + }, + Err(err) if is_timeout_error(&err) => timed_out = Some(err), + Err(err) => return Err(into_error(err)), } - match last_key { - Some(last_key) if timed_out => { - if params.ascending { - from.clone_from(&last_key); - } else { - to.clone_from(&last_key); - } - resume_key = Some(last_key); + if progressed { + attempt = 0; + match timed_out { + None if limit.is_some_and(|limit| fetched >= u64::from(limit)) => plan.grow(), + None => return Ok(()), + Some(_) => (), } - _ => return Ok(()), + if params.ascending { + from.clone_from(&last_key); + } else { + to.clone_from(&last_key); + } + resuming = true; + } else { + match timed_out { + Some(err) + if !plan + .degrade(ITERATE_CHUNK_SIZE, MIN_ITERATE_CHUNK_SIZE, &mut attempt) + .await => + { + return Err(into_error(err)); + } + Some(_) => (), + None => return Ok(()), + } + } + } + } + + pub(crate) async fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + let Some(first) = ranges.first() else { + return Ok(()); + }; + let subspace = first.begin.subspace(); + let column = ValueColumn::new(first.values, subspace); + let mut conn = PooledConn::new(self.conn_pool.get_conn().await.map_err(into_error)?); + + match point_keys(&ranges) { + Some(keys) => iterate_points(&mut conn, subspace.name(), &keys, column, &mut cb).await, + None => { + let bounds = ranges + .iter() + .map(|params| (params.begin.serialize(0), params.end.serialize(0))) + .collect::>(); + self.iterate_ranges(conn, subspace, bounds, column, &mut cb) + .await } } } @@ -153,11 +204,11 @@ impl MysqlStore { key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); - let table = char::from(key.subspace()); + let subspace = key.subspace(); let key = key.serialize(0); let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn - .prep(format!("SELECT v FROM {table} WHERE k = ?")) + .prep(&*self.sql.get(subspace).get_value) .await .map_err(into_error)?; match conn.exec_first::(&s, (key,)).await { @@ -167,3 +218,470 @@ impl MysqlStore { } } } + +#[derive(Clone, Copy)] +pub(super) enum PointRows { + Keys, + Values, + ForUpdate, +} + +impl From for PointRows { + fn from(values: bool) -> Self { + if values { + PointRows::Values + } else { + PointRows::Keys + } + } +} + +type RangeCallback<'y> = + &'y mut (dyn for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Send + Sync); + +enum Emitted { + Done, + Stopped, + TimedOut { + error: mysql_async::Error, + delivered: bool, + }, +} + +async fn iterate_points( + conn: &mut PooledConn, + table: &str, + keys: &[Vec], + column: ValueColumn, + cb: RangeCallback<'_>, +) -> trc::Result<()> { + let mut pending = keys; + let mut last_key = Vec::new(); + let mut plan = ChunkPlan::new(Fetch::All); + let mut attempt = 0; + + loop { + let size = plan + .limit(false) + .map_or(MAX_POINT_KEYS, |size| size as usize); + let before = pending.len(); + let error = match points_attempt(conn, table, &mut pending, size, column, &mut last_key, cb) + .await? + { + None => return Ok(()), + Some(Timeout::Begin(_)) if backoff(&mut attempt).await => continue, + Some(Timeout::Begin(error)) => return Err(into_error(error)), + Some(Timeout::Statement(error)) => error, + }; + pending = keys_after(pending, &last_key); + if pending.is_empty() { + return Ok(()); + } else if pending.len() < before { + attempt = 0; + } else if !plan + .degrade( + DEGRADED_POINT_KEYS + .min(pending.len() / 2) + .max(MIN_POINT_KEYS), + MIN_POINT_KEYS, + &mut attempt, + ) + .await + { + return Err(into_error(error)); + } + } +} + +async fn points_attempt( + conn: &mut PooledConn, + table: &str, + pending: &mut &[Vec], + size: usize, + column: ValueColumn, + last_key: &mut Vec, + cb: RangeCallback<'_>, +) -> trc::Result>> { + if pending.len() <= size { + let (query, params) = + point_statement(table, PointRows::from(column.is_selected()), pending); + return match emit(&mut **conn, &query, params, column, false, last_key, cb).await? { + Emitted::Done | Emitted::Stopped => Ok(None), + Emitted::TimedOut { error, .. } => Ok(Some(Timeout::Statement(error))), + }; + } + + let error = match begin_read(conn).await? { + Ok(mut trx) => { + while !pending.is_empty() { + let (chunk, rest) = pending.split_at(pending.len().min(size)); + let (query, params) = + point_statement(table, PointRows::from(column.is_selected()), chunk); + match emit(&mut trx, &query, params, column, false, last_key, cb).await? { + Emitted::Done => *pending = rest, + Emitted::Stopped => return Ok(None), + Emitted::TimedOut { error, .. } => { + trx.rollback().await.map_err(into_error)?; + return Ok(Some(Timeout::Statement(error))); + } + } + } + trx.commit().await.map_err(into_error)?; + return Ok(None); + } + Err(error) => error, + }; + end_failed_begin(conn).await?; + Ok(Some(Timeout::Begin(error))) +} + +pub(super) fn point_statement>( + table: &str, + rows: PointRows, + keys: &[K], +) -> (String, Vec) { + let slots = POINT_BUCKETS + .into_iter() + .find(|bucket| *bucket >= keys.len()) + .unwrap_or(MAX_POINT_KEYS); + let (columns, order) = match rows { + PointRows::Keys => ("k", " ORDER BY k"), + PointRows::Values => ("k, v", " ORDER BY k"), + PointRows::ForUpdate => ("k, v", " FOR UPDATE"), + }; + let mut query = String::with_capacity(slots * 2 + table.len() + 48); + let _ = write!(query, "SELECT {columns} FROM {table} WHERE k IN (?"); + for _ in 1..slots { + query.push_str(",?"); + } + query.push(')'); + query.push_str(order); + + let mut params = Vec::with_capacity(slots); + params.extend(keys.iter().map(|key| Value::Bytes(key.as_ref().to_vec()))); + params.resize(slots, Value::NULL); + (query, params) +} + +impl MysqlStore { + async fn iterate_ranges( + &self, + mut conn: PooledConn, + subspace: Subspace, + mut bounds: Vec, + column: ValueColumn, + cb: RangeCallback<'_>, + ) -> trc::Result<()> { + let table = subspace.name(); + let mut cursor = RangeCursor::default(); + let mut per_statement = MAX_RANGES_PER_STMT; + let mut attempt = 0; + + loop { + match ranges_attempt( + &mut conn, + table, + &mut bounds, + &mut cursor, + per_statement, + column, + cb, + ) + .await? + { + RangeAttempt::Finished => return Ok(()), + RangeAttempt::BeginTimedOut(error) => { + if !backoff(&mut attempt).await { + return Err(error); + } + } + RangeAttempt::TimedOut { delivered: true } => attempt = 0, + RangeAttempt::TimedOut { delivered: false } => { + match cursor.halve(per_statement, bounds.len()) { + Some(size) => per_statement = size, + None => { + drop(conn); + return self + .iterate_each(subspace, &bounds, &cursor, column, cb) + .await; + } + } + } + } + } + } + + async fn iterate_each( + &self, + subspace: Subspace, + bounds: &[Bounds], + cursor: &RangeCursor, + column: ValueColumn, + cb: RangeCallback<'_>, + ) -> trc::Result<()> { + let mut stopped = false; + for (begin, end) in cursor.remaining(bounds) { + let params = IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { + subspace, + key: end.to_vec(), + }, + ) + .set_values(column.is_selected()); + self.iterate(params, |key, value| { + let more = cb(key, value)?; + stopped = !more; + Ok(more) + }) + .await?; + if stopped { + break; + } + } + Ok(()) + } +} + +fn range_query(table: &str, values: bool, count: usize) -> String { + debug_assert!(count > 0); + let mut query = String::with_capacity(count * 24 + 40); + query.push_str(if values { + "SELECT k, v FROM " + } else { + "SELECT k FROM " + }); + query.push_str(table); + query.push_str(" WHERE "); + for i in 0..count { + if i > 0 { + query.push_str(" OR "); + } + query.push_str("(k >= ? AND k <= ?)"); + } + query.push_str(" ORDER BY k ASC"); + query +} + +fn range_params(bounds: &[Bounds]) -> Vec { + bounds + .iter() + .flat_map(|(begin, end)| [Value::Bytes(begin.clone()), Value::Bytes(end.clone())]) + .collect() +} + +async fn ranges_attempt( + conn: &mut PooledConn, + table: &str, + bounds: &mut [Bounds], + cursor: &mut RangeCursor, + size: usize, + column: ValueColumn, + cb: RangeCallback<'_>, +) -> trc::Result { + let total = bounds.len(); + if cursor.start >= total { + return Ok(RangeAttempt::Finished); + } else if total - cursor.start <= size { + let chunk = bounds.get(cursor.start..).unwrap_or_default(); + return Ok( + match emit( + &mut **conn, + &range_query(table, column.is_selected(), chunk.len()), + range_params(chunk), + column, + cursor.resumed, + &mut cursor.last_key, + cb, + ) + .await? + { + Emitted::Done | Emitted::Stopped => RangeAttempt::Finished, + Emitted::TimedOut { delivered, .. } => { + if delivered { + cursor.advance(bounds, total); + } + RangeAttempt::TimedOut { delivered } + } + }, + ); + } + + let error = match begin_read(conn).await? { + Ok(mut trx) => { + while cursor.start < total { + let end = (cursor.start + size).min(total); + let chunk = bounds.get(cursor.start..end).unwrap_or_default(); + match emit( + &mut trx, + &range_query(table, column.is_selected(), chunk.len()), + range_params(chunk), + column, + cursor.resumed, + &mut cursor.last_key, + cb, + ) + .await? + { + Emitted::Done => { + cursor.start = end; + cursor.resumed = false; + } + Emitted::Stopped => return Ok(RangeAttempt::Finished), + Emitted::TimedOut { delivered, .. } => { + if delivered { + cursor.advance(bounds, end); + } + trx.rollback().await.map_err(into_error)?; + return Ok(RangeAttempt::TimedOut { delivered }); + } + } + } + trx.commit().await.map_err(into_error)?; + return Ok(RangeAttempt::Finished); + } + Err(error) => error, + }; + end_failed_begin(conn).await?; + Ok(RangeAttempt::BeginTimedOut(into_error(error))) +} + +async fn begin_read(conn: &mut Conn) -> trc::Result, mysql_async::Error>> { + let mut tx_opts = TxOpts::default(); + tx_opts + .with_consistent_snapshot(true) + .with_isolation_level(IsolationLevel::RepeatableRead); + #[cfg(stalwart_tests)] + if crate::backend::sql_fault::is_armed() { + conn.query_drop("START TRANSACTION WITH CONSISTENT SNAPSHOT") + .await + .map_err(into_error)?; + return Ok(Err(mysql_async::Error::Server(mysql_async::ServerError { + code: super::ER_LOCK_WAIT_TIMEOUT, + message: "Injected begin timeout".into(), + state: "HY000".into(), + }))); + } + match conn.start_transaction(tx_opts).await { + Ok(trx) => Ok(Ok(trx)), + Err(error) if is_timeout_error(&error) => Ok(Err(error)), + Err(error) => Err(into_error(error)), + } +} + +async fn end_failed_begin(conn: &mut PooledConn) -> trc::Result<()> { + #[cfg(stalwart_tests)] + let statement = if crate::backend::sql_fault::fire() { + "SELECT 1 FROM injected_missing_table" + } else { + "ROLLBACK" + }; + #[cfg(not(stalwart_tests))] + let statement = "ROLLBACK"; + let ended = conn.query_drop(statement).await; + conn.end_transaction(ended) +} + +async fn emit( + q: &mut Q, + query: &str, + params: Vec, + column: ValueColumn, + resumed: bool, + last_key: &mut Vec, + cb: RangeCallback<'_>, +) -> trc::Result { + let s = match q.prep(query).await { + Ok(s) => s, + Err(error) if is_timeout_error(&error) => { + return Ok(Emitted::TimedOut { + error, + delivered: false, + }); + } + Err(error) => return Err(into_error(error)), + }; + let mut rows = match q.exec_stream::(&s, params).await { + Ok(rows) => rows, + Err(error) if is_timeout_error(&error) => { + return Ok(Emitted::TimedOut { + error, + delivered: false, + }); + } + Err(error) => return Err(into_error(error)), + }; + let mut skip_resumed = resumed; + let mut delivered = false; + + loop { + match rows.try_next().await { + Ok(Some(mut row)) => { + let value = RowValue::take(&mut row, column)?; + let key = row + .take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![])) + .map_err(into_error)?; + + if skip_resumed { + skip_resumed = false; + if key == *last_key { + continue; + } + } + + if !cb(&key, value.as_slice())? { + return Ok(Emitted::Stopped); + } + + *last_key = key; + delivered = true; + } + Ok(None) => return Ok(Emitted::Done), + Err(error) if is_timeout_error(&error) => { + return Ok(Emitted::TimedOut { error, delivered }); + } + Err(error) => return Err(into_error(error)), + } + } +} + +enum RowValue { + Bytes(Vec), + Counter([u8; 8]), +} + +impl RowValue { + #[inline] + fn take(row: &mut Row, column: ValueColumn) -> trc::Result { + match column { + ValueColumn::Skip => Ok(RowValue::Bytes(Vec::new())), + ValueColumn::Bytes => row + .take_opt::, _>(1) + .unwrap_or_else(|| Ok(Vec::new())) + .map(RowValue::Bytes) + .map_err(into_error), + ValueColumn::Counter => row + .take_opt::(1) + .unwrap_or(Ok(0)) + .map(|value| RowValue::Counter(value.to_le_bytes())) + .map_err(into_error), + } + } + + #[inline] + fn as_slice(&self) -> &[u8] { + match self { + RowValue::Bytes(bytes) => bytes, + RowValue::Counter(bytes) => bytes, + } + } +} + +fn keys_after<'x>(keys: &'x [Vec], last_key: &[u8]) -> &'x [Vec] { + keys.split_at(keys.partition_point(|key| key.as_slice() <= last_key)) + .1 +} diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index eb05291c..9f441a56 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -5,104 +5,138 @@ */ use crate::{ + U64_LEN, backend::{ MAX_TOKEN_LENGTH, mysql::{ DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, into_error, is_chunk_too_large_error, + read::{MAX_POINT_KEYS, POINT_BUCKETS}, }, }, search::{ - IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, - SearchQuery, SearchValue, + IndexDocument, KeyValueMatch, SearchFilter, SearchQuery, SearchResults, SearchValue, + TextMatch, }, write::SearchIndex, }; -use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable}; +use mysql_async::{TxOpts, Value, prelude::Queryable}; use nlp::tokenizers::word::WordTokenizer; -use std::fmt::Write; +use roaring::RoaringBitmap; +use std::{cmp::Ordering, fmt::Write}; + +const ROW_BUCKETS: [usize; 5] = [256, 64, 16, 4, 1]; +const MAX_BATCH_ROWS: usize = ROW_BUCKETS[0]; +const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024; impl MysqlStore { - pub async fn index(&self, documents: Vec) -> trc::Result<()> { + pub async fn index(&self, mut documents: Vec) -> trc::Result<()> { + documents.sort_by_key(|document| document.index as u8); + let mut batches = row_batches(documents).into_iter(); let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; - let mut tx_opts = TxOpts::default(); - tx_opts - .with_consistent_snapshot(false) - .with_isolation_level(IsolationLevel::ReadCommitted); - let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?; - for document in documents { - let index = document.index; - let primary_keys = index.primary_keys(); - let all_fields = index.all_fields(); - let mut fields = document.fields; - let mut values = Vec::with_capacity(fields.len() + 2); - let mut query = format!("INSERT INTO {} (", index.mysql_table()); - - for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { - if i > 0 { - query.push(','); + match (batches.next(), batches.len()) { + (Some(batch), 0) => conn + .exec_drop(batch.statement(), batch.values) + .await + .map_err(into_error), + (Some(batch), _) => { + let mut trx = conn + .start_transaction(TxOpts::default()) + .await + .map_err(into_error)?; + for batch in std::iter::once(batch).chain(batches) { + trx.exec_drop(batch.statement(), batch.values) + .await + .map_err(into_error)?; } - query.push_str(field.column()); + trx.commit().await.map_err(into_error) } - - query.push_str(") VALUES ("); - - for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { - if i > 0 { - query.push(','); - } - - if let Some(value) = fields.remove(field) { - query.push('?'); - values.push(value); - } else { - query.push_str("NULL"); - } - } - - query.push_str(") ON DUPLICATE KEY UPDATE "); - for (i, field) in all_fields.iter().enumerate() { - if i > 0 { - query.push(','); - } - let column = field.column(); - let _ = write!(&mut query, "{column} = VALUES({column})"); - } - - let s = trx.prep(&query).await.map_err(into_error)?; - - trx.exec_drop(&s, values).await.map_err(into_error)?; + (None, _) => Ok(()), } - - trx.commit().await.map_err(into_error) } - pub async fn query( + pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], - sort: &[SearchComparator], - ) -> trc::Result> { + ) -> trc::Result { let mut query = format!( "SELECT {} FROM {}", R::field().column(), index.mysql_table() ); let params = build_filter(&mut query, filters); - if !sort.is_empty() { - build_sort(&mut query, sort); - } - let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn.prep(query).await.map_err(into_error)?; conn.exec::(s, params) .await - .map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect()) + .map(|r| { + let mut results = R::default(); + for id in r { + results.insert(id as u64); + } + results + }) .map_err(into_error) } + pub async fn filter_candidates( + &self, + index: SearchIndex, + filters: &[SearchFilter], + candidates: &[u32], + ) -> trc::Result { + let mut base = format!("SELECT docid FROM {}", index.mysql_table()); + let params = build_filter(&mut base, filters); + base.push_str(if filters.is_empty() { + " WHERE docid IN (?" + } else { + " AND docid IN (?" + }); + let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + let mut results = RoaringBitmap::new(); + let mut query = String::with_capacity(base.len() + 2 * MAX_POINT_KEYS); + + for chunk in candidates.chunks(MAX_POINT_KEYS) { + let Some(&last) = chunk.last() else { + continue; + }; + let slots = POINT_BUCKETS + .into_iter() + .find(|bucket| *bucket >= chunk.len()) + .unwrap_or(MAX_POINT_KEYS); + query.clear(); + query.push_str(&base); + for _ in 1..slots { + query.push_str(",?"); + } + query.push(')'); + + let mut values = Vec::with_capacity(params.len() + slots); + values.extend(params.iter().cloned()); + values.extend( + chunk + .iter() + .chain(std::iter::repeat_n(&last, slots - chunk.len())) + .map(|document_id| Value::Int(i64::from(*document_id))), + ); + let s = conn.prep(&query).await.map_err(into_error)?; + for document_id in conn + .exec::(s, values) + .await + .map_err(into_error)? + { + if let Ok(document_id) = u32::try_from(document_id) { + results.insert(document_id); + } + } + } + + Ok(results) + } + pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { let table = filter.index.mysql_table(); let mut query = format!("DELETE FROM {table} "); @@ -152,7 +186,12 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { for filter in filters { match filter { - SearchFilter::Operator { field, op, value } => { + SearchFilter::Text { + field, + op, + value, + language: _, + } => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), @@ -163,15 +202,11 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { is_first = false; } - if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) - { - let (value, mode) = match (value, op) { - (SearchValue::Text { value, .. }, SearchOperator::Equal) => { - (Value::Bytes(format!("{value:?}").into_bytes()), "BOOLEAN") - } - (SearchValue::Text { value, .. }, ..) => { + if field.is_text() { + let text_query = match op { + TextMatch::Exact => format!("{value:?}"), + TextMatch::Standard => { let mut text_query = String::with_capacity(value.len() + 1); - for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { if !text_query.is_empty() { text_query.push(' '); @@ -179,39 +214,90 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { text_query.push('+'); text_query.push_str(&item.word); } - - (Value::Bytes(text_query.into_bytes()), "BOOLEAN") + text_query } - _ => { - debug_assert!(false, "Invalid search value for text field"); - continue; + TextMatch::Prefix => { + let mut text_query = String::with_capacity(value.len() + 2); + for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { + if !text_query.is_empty() { + text_query.push(' '); + } + text_query.push('+'); + text_query.push_str(&item.word); + text_query.push('*'); + } + text_query } }; - let _ = write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column()); - values.push(value); - } else if let SearchValue::KeyValues(kv) = value { - let (key, value) = kv.iter().next().unwrap(); - - values.push(Value::Bytes(format!("$.{key:?}").into_bytes())); - - if !value.is_empty() { - if op == &SearchOperator::Equal { - let _ = write!(query, "JSON_EXTRACT({}, ?) = ?", field.column()); - values.push(Value::Bytes(value.as_bytes().to_vec())); - } else { - let _ = write!(query, "JSON_EXTRACT({}, ?) LIKE ?", field.column(),); - values.push(Value::Bytes(format!("%{value}%").into_bytes())); - } - } else { - let _ = write!(query, "JSON_CONTAINS_PATH({}, 'one', ?)", field.column(),); - } + let _ = write!( + query, + "MATCH({}) AGAINST(? IN BOOLEAN MODE)", + field.column() + ); + values.push(Value::Bytes(text_query.into_bytes())); } else { query.push_str(field.column()); - query.push(' '); - op.write_mysql(query); - values.push(to_mysql(value)); + match op { + TextMatch::Prefix => { + let _ = write!(query, " LIKE CONCAT(?, '%')"); + } + _ => { + query.push_str(" = ?"); + } + } + values.push(Value::Bytes(value.as_bytes().to_vec())); } } + SearchFilter::KeyValue { field, key, op } => { + if !is_first { + match operator { + SearchFilter::And => query.push_str(" AND "), + SearchFilter::Or => query.push_str(" OR "), + _ => (), + } + } else { + is_first = false; + } + + values.push(Value::Bytes(format!("$.{key:?}").into_bytes())); + + match op { + KeyValueMatch::Equals(value) => { + let _ = write!(query, "JSON_EXTRACT({}, ?) = ?", field.column()); + values.push(Value::Bytes(value.as_bytes().to_vec())); + } + KeyValueMatch::Contains(value) => { + let _ = write!( + query, + "LOWER(JSON_UNQUOTE(JSON_EXTRACT({}, ?))) LIKE ?", + field.column() + ); + values.push(Value::Bytes( + format!("%{}%", value.to_lowercase()).into_bytes(), + )); + } + KeyValueMatch::Exists => { + let _ = write!(query, "JSON_CONTAINS_PATH({}, 'one', ?)", field.column()); + } + } + } + SearchFilter::Integer { field, op, value } => { + if !is_first { + match operator { + SearchFilter::And => query.push_str(" AND "), + SearchFilter::Or => query.push_str(" OR "), + _ => (), + } + } else { + is_first = false; + } + let cmp = match op { + Ordering::Less => "<", + Ordering::Equal => "=", + Ordering::Greater => ">", + }; + let _ = write!(query, "{} {cmp} {value}", field.column()); + } SearchFilter::And | SearchFilter::Or => { if !is_first { match operator { @@ -240,7 +326,7 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { } operator_stack.push((operator, is_first)); - operator = &SearchFilter::And; + operator = &SearchFilter::Or; is_first = true; query.push_str("NOT ("); } @@ -253,7 +339,7 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { SearchFilter::DocumentSet(_) => { debug_assert!( false, - "DocumentSet filters are not supported in Postgres backend" + "DocumentSet filters are not supported in MySQL backend" ) } } @@ -262,53 +348,114 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { values } -fn build_sort(query: &mut String, sort: &[SearchComparator]) { - query.push_str(" ORDER BY "); - for (i, comparator) in sort.iter().enumerate() { - if i > 0 { - query.push_str(", "); +struct RowBatch { + index: SearchIndex, + rows: usize, + values: Vec, +} + +impl RowBatch { + fn statement(&self) -> String { + let index = self.index; + let columns = index.primary_keys().iter().chain(index.all_fields()); + let num_columns = columns.clone().count(); + let mut query = + String::with_capacity(64 + self.rows * (2 * num_columns + 3) + 32 * num_columns); + let _ = write!(query, "INSERT INTO {} (", index.mysql_table()); + for (pos, field) in columns.enumerate() { + if pos > 0 { + query.push(','); + } + query.push_str(field.column()); } - match comparator { - SearchComparator::Field { field, ascending } => { - query.push_str(field.column()); - if *ascending { - query.push_str(" ASC"); - } else { - query.push_str(" DESC"); + query.push_str(") VALUES "); + for row in 0..self.rows { + if row > 0 { + query.push(','); + } + query.push('('); + for column in 0..num_columns { + if column > 0 { + query.push(','); } + query.push('?'); } - SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { - debug_assert!( - false, - "DocumentSet and SortedSet comparators are not supported " - ); + query.push(')'); + } + query.push_str(" ON DUPLICATE KEY UPDATE "); + for (pos, field) in index.all_fields().iter().enumerate() { + if pos > 0 { + query.push(','); } + let column = field.column(); + let _ = write!(query, "{column} = VALUES({column})"); + } + query + } +} + +fn row_batches(documents: Vec) -> Vec { + let mut batches = Vec::new(); + let mut values = Vec::new(); + let mut current = None; + let mut rows = 0; + let mut bytes = 0; + + for mut document in documents { + let index = document.index; + let row_bytes = document.fields.values().map(value_len).sum::(); + if let Some(current) = current + && (current != index || rows == MAX_BATCH_ROWS || bytes + row_bytes > MAX_BATCH_BYTES) + { + split_rows(&mut batches, current, std::mem::take(&mut values), rows); + rows = 0; + bytes = 0; + } + + for field in index.primary_keys().iter().chain(index.all_fields()) { + values.push( + document + .fields + .remove(field) + .map_or(Value::NULL, Value::from), + ); + } + current = Some(index); + rows += 1; + bytes += row_bytes; + } + + if let Some(current) = current { + split_rows(&mut batches, current, values, rows); + } + + batches +} + +fn split_rows(batches: &mut Vec, index: SearchIndex, values: Vec, rows: usize) { + let num_columns = values.len() / rows.max(1); + let mut values = values.into_iter(); + let mut remaining = rows; + for bucket in ROW_BUCKETS { + while remaining >= bucket { + batches.push(RowBatch { + index, + rows: bucket, + values: values.by_ref().take(bucket * num_columns).collect(), + }); + remaining -= bucket; } } } -impl SearchOperator { - fn write_mysql(&self, query: &mut String) { - match self { - SearchOperator::LowerThan => { - let _ = write!(query, "< ?"); - } - SearchOperator::LowerEqualThan => { - let _ = write!(query, "<= ?"); - } - SearchOperator::GreaterThan => { - let _ = write!(query, "> ?"); - } - SearchOperator::GreaterEqualThan => { - let _ = write!(query, ">= ?"); - } - SearchOperator::Equal => { - let _ = write!(query, "= ?"); - } - SearchOperator::Contains => { - let _ = write!(query, "LIKE '%' CONCAT('%', ?, '%')"); - } - } +fn value_len(value: &SearchValue) -> usize { + match value { + SearchValue::Text { value, .. } => value.len(), + SearchValue::KeyValues(values) => values + .iter() + .map(|(key, value)| key.len() + value.len()) + .sum(), + SearchValue::Int(_) | SearchValue::Uint(_) | SearchValue::Boolean(_) => U64_LEN, } } @@ -333,15 +480,3 @@ impl From for Value { } } } - -fn to_mysql(value: &SearchValue) -> Value { - match value { - SearchValue::Text { value, .. } => Value::Bytes(value.as_bytes().to_vec()), - SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) - .map(|v| Value::Bytes(v.into_bytes())) - .unwrap_or(Value::NULL), - SearchValue::Int(i) => Value::Int(*i), - SearchValue::Uint(i) => Value::Int(*i as i64), - SearchValue::Boolean(b) => Value::Int(*b as i64), - } -} diff --git a/crates/store/src/backend/mysql/sql.rs b/crates/store/src/backend/mysql/sql.rs new file mode 100644 index 00000000..11646522 --- /dev/null +++ b/crates/store/src/backend/mysql/sql.rs @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Subspace; + +pub(crate) struct SubspaceSql { + pub(crate) get_value: Box, + pub(crate) get_value_for_update: Box, + pub(crate) key_exists: Box, + pub(crate) insert_value: Box, + pub(crate) update_value: Box, + pub(crate) upsert_value: Box, + pub(crate) insert_presence: Box, + pub(crate) delete_key: Box, + pub(crate) increment: Box, + pub(crate) decrement: Box, + pub(crate) increment_returning: Box, + pub(crate) delete_range: Box, + pub(crate) delete_range_limit: Box, + pub(crate) range_boundary: Box, + pub(crate) purge_zero: Box, + pub(crate) purge_zero_range: Box, + pub(crate) purge_zero_from: Box, + pub(crate) purge_boundary: Box, + iterate: [Box; 8], + iterate_limit: [Box; 4], +} + +pub(crate) struct SqlStatements { + statements: Box<[SubspaceSql]>, +} + +impl SubspaceSql { + fn new(subspace: Subspace) -> Self { + let table = subspace.name(); + let range = |order: &str, limit: &str, columns: &str| { + format!( + "SELECT {columns} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k {order}{limit}" + ) + .into_boxed_str() + }; + + Self { + get_value: format!("SELECT v FROM {table} WHERE k = ?").into_boxed_str(), + get_value_for_update: format!("SELECT v FROM {table} WHERE k = ? FOR UPDATE") + .into_boxed_str(), + key_exists: format!("SELECT 1 FROM {table} WHERE k = ?").into_boxed_str(), + insert_value: format!("INSERT INTO {table} (k, v) VALUES (?, ?)").into_boxed_str(), + update_value: format!("UPDATE {table} SET v = ? WHERE k = ?").into_boxed_str(), + upsert_value: format!( + "INSERT INTO {table} (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)" + ) + .into_boxed_str(), + insert_presence: format!( + "INSERT INTO {table} (k) VALUES (?) ON DUPLICATE KEY UPDATE k = k" + ) + .into_boxed_str(), + delete_key: format!("DELETE FROM {table} WHERE k = ?").into_boxed_str(), + increment: format!( + "INSERT INTO {table} (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = v + VALUES(v)" + ) + .into_boxed_str(), + decrement: format!("UPDATE {table} SET v = v + ? WHERE k = ?").into_boxed_str(), + increment_returning: format!( + "INSERT INTO {table} (k, v) VALUES (?, LAST_INSERT_ID(?)) ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + ?)" + ) + .into_boxed_str(), + delete_range: format!("DELETE FROM {table} WHERE k >= ? AND k < ?").into_boxed_str(), + delete_range_limit: format!( + "DELETE FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT ?" + ) + .into_boxed_str(), + range_boundary: format!( + "SELECT k FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT ?, 1" + ) + .into_boxed_str(), + purge_zero: format!("DELETE FROM {table} WHERE v = 0").into_boxed_str(), + purge_zero_range: format!("DELETE FROM {table} WHERE v = 0 AND k >= ? AND k < ?") + .into_boxed_str(), + purge_zero_from: format!("DELETE FROM {table} WHERE v = 0 AND k >= ?").into_boxed_str(), + purge_boundary: format!( + "SELECT k FROM {table} WHERE k >= ? ORDER BY k ASC LIMIT ?, 1" + ) + .into_boxed_str(), + iterate: [ + range("ASC", "", "k"), + range("ASC", "", "k, v"), + range("DESC", "", "k"), + range("DESC", "", "k, v"), + range("ASC", " LIMIT 1", "k"), + range("ASC", " LIMIT 1", "k, v"), + range("DESC", " LIMIT 1", "k"), + range("DESC", " LIMIT 1", "k, v"), + ], + iterate_limit: [ + range("ASC", " LIMIT ?", "k"), + range("ASC", " LIMIT ?", "k, v"), + range("DESC", " LIMIT ?", "k"), + range("DESC", " LIMIT ?", "k, v"), + ], + } + } + + #[inline(always)] + pub(crate) fn iterate(&self, first: bool, ascending: bool, values: bool) -> &str { + &self.iterate[((first as usize) << 2) | ((!ascending as usize) << 1) | (values as usize)] + } + + #[inline(always)] + pub(crate) fn iterate_limit(&self, ascending: bool, values: bool) -> &str { + &self.iterate_limit[((!ascending as usize) << 1) | (values as usize)] + } +} + +impl SqlStatements { + pub(crate) fn new() -> Self { + Self { + statements: Subspace::ALL + .iter() + .copied() + .map(SubspaceSql::new) + .collect(), + } + } + + #[inline(always)] + pub(crate) fn get(&self, subspace: Subspace) -> &SubspaceSql { + &self.statements[subspace.index()] + } +} diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 797219ef..39ec3104 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -4,57 +4,108 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +mod reads; + +use self::reads::{ReadSet, missing_read, unprefixed}; use super::{ - DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_chunk_too_large_error, + DELETE_CHUNK_SIZE, ER_DUP_ENTRY, ER_LOCK_DEADLOCK, ER_LOCK_WAIT_TIMEOUT, MIN_DELETE_CHUNK_SIZE, + MysqlStore, into_error, is_chunk_too_large_error, sql::SubspaceSql, }; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - SUBSPACE_REGISTRY_IDX, + Key, Shape, Subspace, WITH_SUBSPACE, write::{ - AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, - ValueClass, ValueOp, + Advance, AssignedIds, Batch, BatchCursor, ChunkedRetry, LogSet, MAX_COMMIT_ATTEMPTS, + MAX_COMMIT_TIME, MergeResult, NewKeyAsserts, Operation, SetValue, ValueOp, commit_backoff, + exhausted, }, }; -use ahash::AHashMap; -use mysql_async::{Conn, Error, IsolationLevel, TxOpts, params, prelude::Queryable}; -use rand::RngExt; -use std::time::{Duration, Instant}; +use mysql_async::{Conn, Error, ServerError, TxOpts, prelude::Queryable}; +use std::{borrow::Cow, time::Instant}; #[derive(Debug)] enum CommitError { Mysql(mysql_async::Error), Internal(trc::Error), - //Retry, +} + +struct AutocommitWrite<'x> { + subspace: Subspace, + key: Vec, + op: AutocommitOp<'x>, +} + +enum AutocommitOp<'x> { + Set(&'x mut SetValue), + SetPresence, + Add(i64), + AddAndGet(i64), + Clear, +} + +enum WriteShape<'x, 'y> { + Autocommit(AutocommitWrite<'x>), + Transaction(&'x mut Batch<'y>), } impl MysqlStore { - pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write( + &self, + mut batch: Batch<'_>, + assigned_ids: &mut AssignedIds, + ) -> trc::Result<()> { let start = Instant::now(); let mut retry_count = 0; + let mut new_key_asserts = NewKeyAsserts::CheckedByInsert; let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + let mark = assigned_ids.mark(); + let mut shape = match AutocommitWrite::parse(&mut batch, assigned_ids) { + Some(write) => WriteShape::Autocommit(write), + None => WriteShape::Transaction(&mut batch), + }; loop { - let err = match self.write_trx(&mut conn, &mut batch).await { - Ok(result) => { - return Ok(result); + assigned_ids.rollback(mark); + let result = match &mut shape { + WriteShape::Autocommit(write) => { + self.write_autocommit(&mut conn, write, assigned_ids).await + } + WriteShape::Transaction(batch) => { + self.write_trx(&mut conn, batch, assigned_ids, new_key_asserts) + .await + } + }; + let err = match result { + Ok(()) => { + return Ok(()); } Err(err) => err, }; - let _ = conn.query_drop("ROLLBACK;").await; - match err { CommitError::Mysql(Error::Server(err)) - if [1062, 1213].contains(&err.code) + if [ER_LOCK_DEADLOCK, ER_LOCK_WAIT_TIMEOUT, ER_DUP_ENTRY] + .contains(&err.code) && retry_count < MAX_COMMIT_ATTEMPTS - && start.elapsed() < MAX_COMMIT_TIME => {} - /*CommitError::Retry => { - if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!())); + && start.elapsed() < MAX_COMMIT_TIME => + { + if err.code == ER_DUP_ENTRY { + new_key_asserts = NewKeyAsserts::Read; } - }*/ + } + CommitError::Mysql(Error::Server(err)) if err.code == ER_DUP_ENTRY => { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .reason("Unique violation") + .caused_by(trc::location!())); + } + CommitError::Mysql( + err @ Error::Server(ServerError { + code: ER_LOCK_DEADLOCK | ER_LOCK_WAIT_TIMEOUT, + .. + }), + ) => { + return Err(exhausted(into_error(err), retry_count)); + } CommitError::Mysql(err) => { return Err(into_error(err)); } @@ -63,105 +114,134 @@ impl MysqlStore { } } - let backoff = rand::rng().random_range(50..=300); - tokio::time::sleep(Duration::from_millis(backoff)).await; + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; retry_count += 1; } } + async fn write_autocommit( + &self, + conn: &mut Conn, + write: &mut AutocommitWrite<'_>, + result: &mut AssignedIds, + ) -> Result<(), CommitError> { + let stmts = self.sql.get(write.subspace); + let key = &write.key; + + match &mut write.op { + AutocommitOp::Set(value) => { + let value = value.resolve(result)?; + let s = conn.prep(&*stmts.upsert_value).await?; + conn.exec_drop(&s, (key, &*value)).await?; + if conn.affected_rows() == 0 { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .caused_by(trc::location!()) + .into()); + } + } + AutocommitOp::SetPresence => { + let s = conn.prep(&*stmts.insert_presence).await?; + conn.exec_drop(&s, (key,)).await?; + } + AutocommitOp::Add(by) if *by >= 0 => { + let s = conn.prep(&*stmts.increment).await?; + conn.exec_drop(&s, (key, *by)).await?; + } + AutocommitOp::Add(by) => { + let s = conn.prep(&*stmts.decrement).await?; + conn.exec_drop(&s, (*by, key)).await?; + } + AutocommitOp::AddAndGet(by) => { + let s = conn.prep(&*stmts.increment_returning).await?; + conn.exec_drop(&s, (key, *by, *by)).await?; + result.push_counter_id( + conn.last_insert_id().ok_or_else(missing_last_insert_id)? as i64 + ); + } + AutocommitOp::Clear => { + let s = conn.prep(&*stmts.delete_key).await?; + conn.exec_drop(&s, (key,)).await?; + } + } + + Ok(()) + } + async fn write_trx( &self, conn: &mut Conn, batch: &mut Batch<'_>, - ) -> Result { - let has_changes = !batch.changes.is_empty(); - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let mut asserted_values = AHashMap::new(); - let mut tx_opts = TxOpts::default(); - tx_opts - .with_consistent_snapshot(false) - .with_isolation_level(IsolationLevel::ReadCommitted); - let mut trx = conn.start_transaction(tx_opts).await?; - let mut result = AssignedIds::default(); + result: &mut AssignedIds, + new_key_asserts: NewKeyAsserts, + ) -> Result<(), CommitError> { + let sql = &*self.sql; + let mut trx = conn.start_transaction(TxOpts::default()).await?; + let mut log_buf = Vec::new(); + let mut log_scratch = Vec::new(); - if has_changes { - for &account_id in batch.changes.keys() { - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); - let s = trx - .prep(concat!( - "INSERT INTO n (k, v) VALUES (:k, LAST_INSERT_ID(1)) ", - "ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + 1)" - )) + if batch.has_id_allocations() { + let incr = trx + .prep(&*sql.get(Subspace::Counter).increment_returning) + .await?; + let mut key_buf = Vec::with_capacity(16); + + for allocation in batch.id_allocations() { + key_buf.clear(); + allocation.serialize_key_into(&mut key_buf, 0); + let increment_by = allocation.increment_by(); + trx.exec_drop(&incr, (&key_buf, increment_by, increment_by)) .await?; - trx.exec_drop(&s, params! {"k" => key}).await?; - let s = trx.prep("SELECT LAST_INSERT_ID()").await?; - let change_id = trx.exec_first::(&s, ()).await?.ok_or_else(|| { - mysql_async::Error::Io(mysql_async::IoError::Io(std::io::Error::other( - "LAST_INSERT_ID() did not return a value", - ))) - })?; - result.push_change_id(account_id, change_id as u64); + let counter = trx.last_insert_id().ok_or_else(missing_last_insert_id)? as i64; + if let Err(err) = result.apply(allocation, counter) { + trx.rollback().await?; + return Err(err.into()); + } } } - for op in batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; - } - } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let key = class.serialize(account_id, collection, document_id, 0); - let subspace = class.subspace(collection); - let table = char::from(subspace); + let mut key_buf = Vec::with_capacity(64); + let mut reads = ReadSet::collect(batch, result, &mut key_buf, new_key_asserts); + if reads.needs_fetch() { + reads.fetch(&mut trx, sql).await?; + } + + let mut cursor = BatchCursor::new(batch); + let index_stmts = sql.get(Subspace::Indexes); + let log_stmts = sql.get(Subspace::Logs); + for (op_index, op) in batch.ops.iter_mut().enumerate() { + match cursor.advance(op, result) { + Advance::Cursor => {} + Advance::Value { class, op } => { + cursor.value_key(class, &mut key_buf, WITH_SUBSPACE); + let slot = reads.find(&key_buf); + let key = unprefixed(&key_buf); + let subspace = cursor.subspace(class); + let stmts = sql.get(subspace); match op { - ValueOp::Set(value) => { - if subspace != SUBSPACE_REGISTRY_IDX { - let exists = asserted_values.get(&key); - let s = if let Some(exists) = exists { - if *exists { - trx.prep(format!( - "UPDATE {} SET v = :v WHERE k = :k", - table - )) - .await? - } else { - trx.prep(format!( - "INSERT INTO {} (k, v) VALUES (:k, :v)", - table - )) - .await? + ValueOp::Set(set_value) => { + let value = set_value.resolve(result)?; + let value = &*value; + if !matches!(subspace.shape(), Shape::Presence) { + let updated = match slot.filter(|slot| reads.is_asserted(*slot)) { + Some(slot) if reads.exists(slot) => { + let s = trx.prep(&*stmts.update_value).await?; + trx.exec_drop(&s, (value, key)).await + } + Some(_) => { + let s = trx.prep(&*stmts.insert_value).await?; + trx.exec_drop(&s, (key, value)).await + } + None => { + let s = trx.prep(&*stmts.upsert_value).await?; + trx.exec_drop(&s, (key, value)).await } - } else { - trx - .prep( - format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table), - ) - .await? }; - match trx - .exec_drop(&s, params! {"k" => key, "v" => &*value}) - .await - { + match updated { Ok(_) => { if trx.affected_rows() == 0 { trx.rollback().await?; @@ -176,198 +256,116 @@ impl MysqlStore { return Err(err.into()); } } + + if let Some(slot) = slot { + reads.written(slot, op_index, Cow::Borrowed(value)); + } } else { - let s = trx.prep("INSERT IGNORE INTO b (k) VALUES (?)").await?; + let s = trx.prep(&*stmts.insert_presence).await?; trx.exec_drop(&s, (key,)).await?; } } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; - let exists = asserted_values.get(&key); - let s = if let Some(exists) = exists { - if *exists { - trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table)) - .await? - } else { - trx.prep(format!( - "INSERT INTO {} (k, v) VALUES (:k, :v)", - table - )) - .await? - } - } else { - trx - .prep( - format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table), - ) - .await? - }; - - match trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await { - Ok(_) => { - if trx.affected_rows() == 0 { - trx.rollback().await?; - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!()) - .into()); - } - } - Err(err) => { - trx.rollback().await?; - return Err(err.into()); - } - } - } ValueOp::MergeFnc(merge_op) => { - let s = trx - .prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) - .await?; - let (exists, merge_result) = trx - .exec_first::, _, _>(&s, (&key,)) - .await? - .map(|bytes| { - (merge_op.fnc)(&merge_op.params, &result, Some(bytes.as_ref())) - .map(|v| (true, v)) - .map_err(CommitError::from) - }) - .unwrap_or_else(|| { - (merge_op.fnc)(&merge_op.params, &result, None) - .map(|v| (false, v)) - .map_err(CommitError::from) - })?; + let slot = slot.ok_or_else(missing_read)?; + let exists = reads.exists(slot); - let s = if exists { - trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table)) - .await? - } else { - trx.prep(format!("INSERT INTO {} (k, v) VALUES (:k, :v)", table)) - .await? - }; - - match merge_result { + match (merge_op.0)(result, reads.value(slot)?)? { MergeResult::Update(value) => { - if let Err(err) = - trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await - { + let updated = if exists { + let s = trx.prep(&*stmts.update_value).await?; + trx.exec_drop(&s, (&value, key)).await + } else { + let s = trx.prep(&*stmts.insert_value).await?; + trx.exec_drop(&s, (key, &value)).await + }; + + if let Err(err) = updated { trx.rollback().await?; return Err(err.into()); } + + reads.written(slot, op_index, Cow::Owned(value)); } MergeResult::Delete if exists => { - // Update asserted value - if let Some(exists) = asserted_values.get_mut(&key) { - *exists = false; - } - - let s = trx - .prep(format!("DELETE FROM {} WHERE k = ?", table)) - .await?; + let s = trx.prep(&*stmts.delete_key).await?; trx.exec_drop(&s, (key,)).await?; + reads.cleared(slot); } - _ => (), + MergeResult::Delete | MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { if *by >= 0 { - let s = trx - .prep(format!( - concat!( - "INSERT INTO {} (k, v) VALUES (?, ?) ", - "ON DUPLICATE KEY UPDATE v = v + VALUES(v)" - ), - table - )) - .await?; + let s = trx.prep(&*stmts.increment).await?; trx.exec_drop(&s, (key, &*by)).await?; } else { - let s = trx - .prep(format!("UPDATE {table} SET v = v + ? WHERE k = ?")) - .await?; + let s = trx.prep(&*stmts.decrement).await?; trx.exec_drop(&s, (&*by, key)).await?; } } ValueOp::AddAndGet(by) => { - let s = trx - .prep(format!( - concat!( - "INSERT INTO {} (k, v) VALUES (:k, LAST_INSERT_ID(:v)) ", - "ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + :v)" - ), - table - )) - .await?; - trx.exec_drop(&s, params! {"k" => key, "v" => &*by}).await?; - let s = trx.prep("SELECT LAST_INSERT_ID()").await?; + let s = trx.prep(&*stmts.increment_returning).await?; + trx.exec_drop(&s, (key, *by, *by)).await?; result.push_counter_id( - trx.exec_first::(&s, ()).await?.ok_or_else(|| { - mysql_async::Error::Io(mysql_async::IoError::Io( - std::io::Error::other( - "LAST_INSERT_ID() did not return a value", - ), - )) - })?, + trx.last_insert_id().ok_or_else(missing_last_insert_id)? as i64, ); } ValueOp::Clear => { - // Update asserted value - if let Some(exists) = asserted_values.get_mut(&key) { - *exists = false; - } - - let s = trx - .prep(format!("DELETE FROM {} WHERE k = ?", table)) - .await?; + let s = trx.prep(&*stmts.delete_key).await?; trx.exec_drop(&s, (key,)).await?; + if let Some(slot) = slot { + reads.cleared(slot); + } } } } - Operation::Index { field, key, set } => { - let key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: &*key, - } - .serialize(0); + Advance::Index { field, key, set } => { + cursor.index_key(field, key, &mut key_buf, 0); + let key = &key_buf; - let s = if *set { - trx.prep("INSERT IGNORE INTO i (k) VALUES (?)").await? + let s = if set { + trx.prep(&*index_stmts.insert_presence).await? } else { - trx.prep("DELETE FROM i WHERE k = ?").await? + trx.prep(&*index_stmts.delete_key).await? }; trx.exec_drop(&s, (key,)).await?; } - Operation::Log { collection, set } => { - let key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, + Advance::Log { collection, set } => { + cursor.log_key(collection, result, &mut key_buf, WITH_SUBSPACE); + let value = match set { + LogSet::Bytes(bytes) => &*bytes, + LogSet::Pending(changes) => { + changes.serialize_into( + collection.is_prefixed(), + result, + &mut log_scratch, + &mut log_buf, + ); + &log_buf + } + }; + + let s = trx.prep(&*log_stmts.upsert_value).await?; + trx.exec_drop(&s, (unprefixed(&key_buf), value)).await?; + if let Some(slot) = reads.find(&key_buf) { + reads.written(slot, op_index, Cow::Borrowed(value)); } - .serialize(0); - - let s = trx - .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") - .await?; - - trx.exec_drop(&s, (key, &*set)).await?; } - Operation::AssertValue { + Advance::ClearRange { subspace, from, to } => { + reads.clear_range(subspace, from, to); + let s = trx.prep(&*sql.get(subspace).delete_range).await?; + trx.exec_drop(&s, (from, to)).await?; + } + Advance::Assert { class, assert_value, } => { - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); - - let s = trx - .prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) - .await?; - let (exists, matches) = trx - .exec_first::, _, _>(&s, (&key,)) - .await? - .map(|bytes| (true, assert_value.matches(&bytes))) - .unwrap_or_else(|| (false, assert_value.is_none())); + cursor.value_key(class, &mut key_buf, WITH_SUBSPACE); + let slot = reads.find(&key_buf).ok_or_else(missing_read)?; + let matches = match reads.value(slot)? { + Some(value) => assert_value.matches(value), + None => assert_value.is_none(), + }; if !matches { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed @@ -375,18 +373,38 @@ impl MysqlStore { .caused_by(trc::location!()) .into()); } - asserted_values.insert(key, exists); + reads.asserted(slot); } } } - trx.commit().await.map(|_| result).map_err(Into::into) + if !batch.bounds.is_empty() { + let incr = trx + .prep(&*sql.get(Subspace::Counter).increment_returning) + .await?; + for bound in batch.bounds { + key_buf.clear(); + bound.serialize_key_into(&mut key_buf, 0); + trx.exec_drop(&incr, (&key_buf, bound.by, bound.by)).await?; + let value = trx.last_insert_id().ok_or_else(missing_last_insert_id)? as i64; + if let Err(err) = bound.verify(value) { + trx.rollback().await?; + return Err(err.into()); + } + } + } + + trx.commit().await.map_err(Into::into) } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; - for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - purge_table(&mut conn, char::from(subspace)).await?; + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { + purge_table(&mut conn, self.sql.get(subspace)).await?; } Ok(()) @@ -394,34 +412,49 @@ impl MysqlStore { pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; - let table = char::from(from.subspace()); + let stmts = self.sql.get(from.subspace()); let mut from = from.serialize(0); let to = to.serialize(0); - let delete = conn - .prep(format!("DELETE FROM {table} WHERE k >= ? AND k < ?")) + let mut retry = ChunkedRetry::bounded(DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE); + + let delete_limit = conn + .prep(&*stmts.delete_range_limit) + .await + .map_err(into_error)?; + match conn + .exec_drop(&delete_limit, (&from, &to, DELETE_CHUNK_SIZE as u64)) + .await + { + Ok(_) if conn.affected_rows() < DELETE_CHUNK_SIZE as u64 => return Ok(()), + Ok(_) => (), + Err(err) if is_chunk_too_large_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } + } + Err(err) => return Err(into_error(err)), + } + + let delete = conn.prep(&*stmts.delete_range).await.map_err(into_error)?; + let boundary = conn + .prep(&*stmts.range_boundary) .await .map_err(into_error)?; - let mut chunk_size = DELETE_CHUNK_SIZE; loop { - let boundary = conn - .prep(format!( - "SELECT k FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}" - )) - .await - .map_err(into_error)?; + let chunk_size = retry.chunk_size().unwrap_or(DELETE_CHUNK_SIZE) as u64; loop { let next = match conn - .exec_first::, _, _>(&boundary, (&from, &to)) + .exec_first::, _, _>(&boundary, (&from, &to, chunk_size)) .await { Ok(next) => next, - Err(err) - if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => - { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Err(err) if is_chunk_too_large_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -431,11 +464,11 @@ impl MysqlStore { .exec_drop(&delete, (&from, next.as_ref().unwrap_or(&to))) .await { - Ok(_) => (), - Err(err) - if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => - { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Ok(_) => retry.progressed(), + Err(err) if is_chunk_too_large_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -450,11 +483,8 @@ impl MysqlStore { } } -async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { - let s = conn - .prep(format!("DELETE FROM {table} WHERE v = 0")) - .await - .map_err(into_error)?; +async fn purge_table(conn: &mut Conn, stmts: &SubspaceSql) -> trc::Result<()> { + let s = conn.prep(&*stmts.purge_zero).await.map_err(into_error)?; match conn.exec_drop(&s, ()).await { Ok(_) => return Ok(()), @@ -463,33 +493,33 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { } let purge = conn - .prep(format!( - "DELETE FROM {table} WHERE v = 0 AND k >= ? AND k < ?" - )) + .prep(&*stmts.purge_zero_range) .await .map_err(into_error)?; let purge_last = conn - .prep(format!("DELETE FROM {table} WHERE v = 0 AND k >= ?")) + .prep(&*stmts.purge_zero_from) .await .map_err(into_error)?; - let mut chunk_size = DELETE_CHUNK_SIZE; + let boundary = conn + .prep(&*stmts.purge_boundary) + .await + .map_err(into_error)?; + let mut retry = ChunkedRetry::bounded(DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE); let mut from = Vec::new(); loop { - let boundary = conn - .prep(format!( - "SELECT k FROM {table} WHERE k >= ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}" - )) - .await - .map_err(into_error)?; + let chunk_size = retry.chunk_size().unwrap_or(DELETE_CHUNK_SIZE) as u64; loop { - let next = match conn.exec_first::, _, _>(&boundary, (&from,)).await { + let next = match conn + .exec_first::, _, _>(&boundary, (&from, chunk_size)) + .await + { Ok(next) => next, - Err(err) - if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => - { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Err(err) if is_chunk_too_large_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -501,11 +531,11 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { }; match result { - Ok(_) => (), - Err(err) - if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => - { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Ok(_) => retry.progressed(), + Err(err) if is_chunk_too_large_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -519,6 +549,12 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { } } +fn missing_last_insert_id() -> mysql_async::Error { + mysql_async::Error::Io(mysql_async::IoError::Io(std::io::Error::other( + "LAST_INSERT_ID() did not return a value", + ))) +} + impl From for CommitError { fn from(err: trc::Error) -> Self { CommitError::Internal(err) @@ -530,3 +566,65 @@ impl From for CommitError { CommitError::Mysql(err) } } + +impl<'x> AutocommitWrite<'x> { + fn parse(batch: &'x mut Batch<'_>, ids: &mut AssignedIds) -> Option { + if batch.has_allocations() { + return None; + } + + let mut writes = batch.ops.iter().filter(|op| { + !matches!( + op, + Operation::AccountId { .. } + | Operation::Collection { .. } + | Operation::DocumentId { .. } + ) + }); + if writes.next().is_none() || writes.next().is_some() { + return None; + } + + let mut cursor = BatchCursor::new(batch); + for op in batch.ops.iter_mut() { + match cursor.advance(op, ids) { + Advance::Cursor => {} + Advance::Value { class, op } => { + let subspace = cursor.subspace(class); + let op = match op { + ValueOp::Set(_) if matches!(subspace.shape(), Shape::Presence) => { + AutocommitOp::SetPresence + } + ValueOp::Set(value) => AutocommitOp::Set(value), + ValueOp::AtomicAdd(by) => AutocommitOp::Add(*by), + ValueOp::AddAndGet(by) => AutocommitOp::AddAndGet(*by), + ValueOp::Clear => AutocommitOp::Clear, + ValueOp::MergeFnc(_) => return None, + }; + return Some(AutocommitWrite { + subspace, + key: cursor.value_key_owned(class, 0), + op, + }); + } + Advance::Index { field, key, set } => { + return Some(AutocommitWrite { + subspace: Subspace::Indexes, + key: cursor.index_key_owned(field, key, 0), + op: if set { + AutocommitOp::SetPresence + } else { + AutocommitOp::Clear + }, + }); + } + Advance::Log { .. } | Advance::Assert { .. } | Advance::ClearRange { .. } => { + return None; + } + } + } + + None + } +} + diff --git a/crates/store/src/backend/mysql/write/reads.rs b/crates/store/src/backend/mysql/write/reads.rs new file mode 100644 index 00000000..91a48893 --- /dev/null +++ b/crates/store/src/backend/mysql/write/reads.rs @@ -0,0 +1,269 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::CommitError; +use crate::{ + Shape, Subspace, WITH_SUBSPACE, + backend::mysql::{ + into_error, + read::{MAX_POINT_KEYS, PointRows, point_statement}, + sql::SqlStatements, + }, + write::{ + Advance, AssignedIds, Batch, BatchCursor, NewKeyAsserts, Operation, ValueOp, insert_follows, + }, +}; +use futures::TryStreamExt; +use mysql_async::{Row, Transaction, prelude::Queryable}; +use std::{borrow::Cow, mem, ops::Range}; + +#[derive(Default)] +pub(super) struct ReadSet { + keys: Vec, + entries: Vec, +} + +struct Entry { + key: Range, + subspace: Subspace, + fetch: bool, + asserted: bool, + last_read: usize, + current: Current, +} + +enum Current { + Absent, + Present, + Stored(Vec), +} + +impl ReadSet { + pub(super) fn collect( + batch: &mut Batch<'_>, + ids: &mut AssignedIds, + key_buf: &mut Vec, + new_key_asserts: NewKeyAsserts, + ) -> Self { + let mut reads = ReadSet::default(); + let mut cursor = BatchCursor::new(batch); + let mut pending_document = false; + let mut remaining = &mut *batch.ops; + let mut index = 0; + + while let Some((op, rest)) = mem::take(&mut remaining).split_first_mut() { + remaining = rest; + let op_index = index; + index += 1; + if let Operation::DocumentId { document_id } = op { + pending_document = document_id.is_pending(); + } + + let (class, asserts_absent) = match cursor.advance(op, ids) { + Advance::Value { + class, + op: ValueOp::MergeFnc(_), + } => (class, false), + Advance::Assert { + class, + assert_value, + } => (class, assert_value.is_none()), + _ => continue, + }; + let subspace = cursor.subspace(class); + let checked_by_insert = new_key_asserts == NewKeyAsserts::CheckedByInsert + && asserts_absent + && pending_document + && !matches!(subspace.shape(), Shape::Presence) + && insert_follows(class, remaining) + && !cursor.is_document_independent(class, key_buf); + cursor.value_key(class, key_buf, WITH_SUBSPACE); + reads.push(subspace, key_buf, !checked_by_insert, op_index); + } + + reads.seal(); + reads + } + + fn push(&mut self, subspace: Subspace, key: &[u8], fetch: bool, op_index: usize) { + let start = self.keys.len(); + self.keys.extend_from_slice(key); + self.entries.push(Entry { + key: start..self.keys.len(), + subspace, + fetch, + asserted: false, + last_read: op_index, + current: Current::Absent, + }); + } + + fn seal(&mut self) { + let ReadSet { keys, entries } = self; + entries.sort_unstable_by(|a, b| a.key(keys).cmp(b.key(keys))); + entries.dedup_by(|next, kept| { + let duplicate = next.key(keys) == kept.key(keys); + if duplicate { + kept.fetch |= next.fetch; + kept.last_read = kept.last_read.max(next.last_read); + } + duplicate + }); + } + + pub(super) async fn fetch( + &mut self, + trx: &mut Transaction<'_>, + sql: &SqlStatements, + ) -> Result<(), CommitError> { + let ReadSet { keys, entries } = self; + let mut fetch_keys: Vec<&[u8]> = Vec::new(); + let mut fetch_slots = Vec::new(); + + for run in entries.chunk_by_mut(|a, b| a.subspace == b.subspace) { + let Some(subspace) = run.first().map(|entry| entry.subspace) else { + continue; + }; + fetch_keys.clear(); + fetch_slots.clear(); + for (slot, entry) in run.iter().enumerate().filter(|(_, entry)| entry.fetch) { + fetch_keys.push(unprefixed(entry.key(keys))); + fetch_slots.push(slot); + } + + match (fetch_keys.as_slice(), fetch_slots.as_slice()) { + ([], _) => {} + ([key], [slot]) => { + let s = trx.prep(&*sql.get(subspace).get_value_for_update).await?; + let value = trx.exec_first::, _, _>(&s, (key,)).await?; + if let Some(entry) = run.get_mut(*slot) { + entry.current = value.map_or(Current::Absent, Current::Stored); + } + } + _ => { + for (chunk_keys, chunk_slots) in fetch_keys + .chunks(MAX_POINT_KEYS) + .zip(fetch_slots.chunks(MAX_POINT_KEYS)) + { + let (query, params) = + point_statement(subspace.name(), PointRows::ForUpdate, chunk_keys); + let s = trx.prep(&query).await?; + let mut rows = trx.exec_stream::(&s, params).await?; + while let Some(mut row) = rows.try_next().await? { + let value = row + .take_opt::, _>(1) + .transpose() + .map_err(into_error)?; + let key = row + .take_opt::, _>(0) + .transpose() + .map_err(into_error)?; + let entry = key + .and_then(|key| chunk_keys.binary_search(&key.as_slice()).ok()) + .and_then(|index| chunk_slots.get(index)) + .and_then(|slot| run.get_mut(*slot)) + .ok_or_else(missing_read)?; + entry.current = Current::Stored(value.unwrap_or_default()); + } + } + } + } + } + + Ok(()) + } + + #[inline] + pub(super) fn find(&self, key: &[u8]) -> Option { + self.entries + .binary_search_by(|entry| entry.key(&self.keys).cmp(key)) + .ok() + } + + #[inline] + pub(super) fn value(&self, slot: usize) -> Result, CommitError> { + match self.entries.get(slot).map(|entry| &entry.current) { + Some(Current::Stored(value)) => Ok(Some(value)), + Some(Current::Absent) => Ok(None), + Some(Current::Present) | None => Err(missing_read()), + } + } + + #[inline] + pub(super) fn exists(&self, slot: usize) -> bool { + self.entries + .get(slot) + .is_some_and(|entry| !matches!(entry.current, Current::Absent)) + } + + #[inline] + pub(super) fn is_asserted(&self, slot: usize) -> bool { + self.entries.get(slot).is_some_and(|entry| entry.asserted) + } + + #[inline] + pub(super) fn asserted(&mut self, slot: usize) { + if let Some(entry) = self.entries.get_mut(slot) { + entry.asserted = true; + } + } + + #[inline] + pub(super) fn written(&mut self, slot: usize, op_index: usize, value: Cow<'_, [u8]>) { + if let Some(entry) = self.entries.get_mut(slot) { + entry.current = if op_index < entry.last_read { + Current::Stored(value.into_owned()) + } else { + Current::Present + }; + } + } + + pub(super) fn clear_range(&mut self, subspace: Subspace, from: &[u8], to: &[u8]) { + let ReadSet { keys, entries } = self; + let range = from..to; + for entry in entries.iter_mut() { + if entry.subspace == subspace && range.contains(&unprefixed(entry.key(keys))) { + entry.current = Current::Absent; + } + } + } + + #[inline] + pub(super) fn cleared(&mut self, slot: usize) { + if let Some(entry) = self.entries.get_mut(slot) { + entry.current = Current::Absent; + } + } + + #[inline] + pub(super) fn needs_fetch(&self) -> bool { + self.entries.iter().any(|entry| entry.fetch) + } +} + +impl Entry { + #[inline] + fn key<'a>(&self, keys: &'a [u8]) -> &'a [u8] { + keys.get(self.key.clone()).unwrap_or_default() + } +} + +#[inline] +pub(super) fn unprefixed(key: &[u8]) -> &[u8] { + key.split_first().map_or(key, |(_, key)| key) +} + +#[cold] +pub(super) fn missing_read() -> CommitError { + CommitError::Internal( + trc::StoreEvent::UnexpectedError + .into_err() + .details("Merge or assert key was not read") + .caused_by(trc::location!()), + ) +} + diff --git a/crates/store/src/backend/postgres/blob.rs b/crates/store/src/backend/postgres/blob.rs index 66d437c0..ebffc513 100644 --- a/crates/store/src/backend/postgres/blob.rs +++ b/crates/store/src/backend/postgres/blob.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::ops::Range; - -use crate::backend::postgres::into_pool_error; - use super::{PostgresStore, into_error}; +use crate::backend::postgres::into_pool_error; +use std::ops::Range; impl PostgresStore { pub(crate) async fn get_blob( @@ -21,6 +19,8 @@ impl PostgresStore { .prepare_cached("SELECT v FROM t WHERE k = $1") .await .map_err(into_error)?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Read); conn.query_opt(&s, &[&key]) .await .and_then(|row| { @@ -49,6 +49,8 @@ impl PostgresStore { ) .await .map_err(into_error)?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::AutocommitWrite); conn.execute(&s, &[&key, &data]) .await .map_err(into_error) @@ -61,6 +63,8 @@ impl PostgresStore { .prepare_cached("DELETE FROM t WHERE k = $1") .await .map_err(into_error)?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::AutocommitWrite); conn.execute(&s, &[&key]) .await .map_err(into_error) diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index c33f9781..f0e47ce4 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -8,20 +8,21 @@ use super::{PostgresStore, into_error}; use crate::{ backend::postgres::{ PsqlSearchField, into_pool_error, + pool::{PgManager, PgObject, PgPool}, search::{PG_FALLBACK_LANG, PG_LANGS, PG_UNSTEMMED_LANG}, + sql::SqlStatements, tls::MakeRustlsConnect, }, search::{ - CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, - TracingSearchField, + CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, + SearchableField, TracingSearchField, }, *, }; use ::registry::schema::{enums::PostgreSqlRecyclingMethod, structs}; use ahash::AHashSet; -use deadpool_postgres::{ - Config, ManagerConfig, Object, Pool, PoolConfig, RecyclingMethod, Runtime, -}; +use compact_str::ToCompactString; +use deadpool_postgres::{Config, Manager, ManagerConfig, PoolConfig, RecyclingMethod, Runtime}; use tokio_postgres::NoTls; use utils::tls::rustls_client_config; @@ -33,7 +34,7 @@ impl PostgresStore { cfg.user = config.auth_username; cfg.password = config.auth_secret.secret().await?.map(|v| v.into_owned()); cfg.port = (config.port as u16).into(); - cfg.connect_timeout = config.timeout.map(|t| t.into_inner()); + cfg.connect_timeout = config.connect_timeout.map(|t| t.into_inner()); cfg.options = config.options; cfg.manager = Some(ManagerConfig { recycling_method: match config.pool_recycling_method { @@ -42,47 +43,22 @@ impl PostgresStore { PostgreSqlRecyclingMethod::Clean => RecyclingMethod::Clean, }, }); - if let Some(max_conn) = config.pool_max_connections { - cfg.pool = PoolConfig::new(max_conn as usize).into(); - } + cfg.pool = PoolConfig::new( + config + .pool_max_connections + .map_or_else(backend::default_pool_max_connections, |max| max as usize), + ) + .into(); - let primary_pool = if config.use_tls { - cfg.create_pool( - Some(Runtime::Tokio1), - MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)?), - ) - } else { - cfg.create_pool(Some(Runtime::Tokio1), NoTls) - } - .map_err(|e| format!("Failed to create connection pool: {e}"))?; + let primary_pool = create_pool(&cfg, config.use_tls, config.allow_invalid_certs)?; let ts_configs = discover_ts_configs(&primary_pool).await; + let sql = Arc::new(SqlStatements::new()); - let mut replicas = vec![]; - for replica in config.read_replicas { - let mut cfg = cfg.clone(); - cfg.dbname = replica.database.into(); - cfg.host = replica.host.into(); - cfg.user = replica.auth_username; - cfg.password = replica.auth_secret.secret().await?.map(|v| v.into_owned()); - cfg.port = (replica.port as u16).into(); - cfg.options = replica.options; - replicas.push(Store::PostgreSQL(Arc::new(PostgresStore { - conn_pool: if config.use_tls { - cfg.create_pool( - Some(Runtime::Tokio1), - MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)?), - ) - } else { - cfg.create_pool(Some(Runtime::Tokio1), NoTls) - } - .map_err(|e| format!("Failed to create connection pool: {e}"))?, - ts_configs: ts_configs.clone(), - }))); - } let primary = Store::PostgreSQL(Arc::new(PostgresStore { conn_pool: primary_pool, ts_configs, + sql, })); @@ -92,63 +68,26 @@ impl PostgresStore { pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - for table in [ - SUBSPACE_ACL, - SUBSPACE_TASK_QUEUE, - SUBSPACE_DELETED_ITEMS, - SUBSPACE_SPAM_SAMPLES, - SUBSPACE_BLOB_LINK, - SUBSPACE_IN_MEMORY_VALUE, - SUBSPACE_PROPERTY, - SUBSPACE_REGISTRY, - SUBSPACE_REGISTRY_PK, - SUBSPACE_QUEUE_MESSAGE, - SUBSPACE_QUEUE_EVENT, - SUBSPACE_REPORT_OUT, - SUBSPACE_REPORT_IN, - SUBSPACE_LOGS, - SUBSPACE_BLOBS, - SUBSPACE_DIRECTORY, - SUBSPACE_TELEMETRY_SPAN, - SUBSPACE_TELEMETRY_METRIC, - ] { - let table = char::from(table); - conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k BYTEA PRIMARY KEY, - v BYTEA NOT NULL - )" + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| !subspace.is_internal_fts()) + { + let table = subspace.name(); + let (columns, storage) = match subspace.shape() { + Shape::Value => ( + "k BYTEA PRIMARY KEY, v BYTEA NOT NULL", + " WITH (fillfactor = 85)", ), - &[], - ) - .await - .map_err(into_error)?; - } + Shape::Presence => ("k BYTEA PRIMARY KEY", ""), + Shape::Counter => ( + "k BYTEA PRIMARY KEY, v BIGINT NOT NULL DEFAULT 0", + " WITH (fillfactor = 60)", + ), + }; - for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k BYTEA PRIMARY KEY - )" - ), - &[], - ) - .await - .map_err(into_error)?; - } - - for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { - conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {} ( - k BYTEA PRIMARY KEY, - v BIGINT NOT NULL DEFAULT 0 - )", - char::from(table) - ), + &format!("CREATE TABLE IF NOT EXISTS {table} ({columns}){storage}"), &[], ) .await @@ -164,7 +103,7 @@ impl PostgresStore { create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; - //create_search_tables::(&conn).await?; + create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; Ok(()) @@ -172,7 +111,7 @@ impl PostgresStore { } async fn create_search_tables( - conn: &Object, + conn: &PgObject, ) -> trc::Result<()> { let table_name = T::index().psql_table(); let mut query = format!("CREATE TABLE IF NOT EXISTS {} (", table_name); @@ -185,11 +124,7 @@ async fn create_search_tables( // Add other columns for field in T::all_fields() { - query.push_str(&format!("{} {}", field.column(), field.column_type())); - if let Some(sort_type) = field.sort_column_type() { - query.push_str(&format!(", {} {}", field.sort_column().unwrap(), sort_type)); - } - query.push_str(", "); + query.push_str(&format!("{} {}, ", field.column(), field.column_type())); } // Add primary key constraint @@ -217,7 +152,7 @@ async fn create_search_tables( } if field.is_indexed() { - let column_name = field.sort_column().unwrap_or(field.column()); + let column_name = field.column(); let create_index_query = format!( "CREATE INDEX IF NOT EXISTS idx_{table_name}_{column_name} ON {table_name}({column_name})", ); @@ -230,7 +165,27 @@ async fn create_search_tables( Ok(()) } -async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> { +fn create_pool(cfg: &Config, use_tls: bool, allow_invalid_certs: bool) -> Result { + let pg_config = cfg + .get_pg_config() + .map_err(|e| format!("Failed to create connection pool: {e}"))?; + let manager = if use_tls { + Manager::from_config( + pg_config, + MakeRustlsConnect::new(rustls_client_config(allow_invalid_certs)?), + cfg.get_manager_config(), + ) + } else { + Manager::from_config(pg_config, NoTls, cfg.get_manager_config()) + }; + PgPool::builder(PgManager::new(manager)) + .config(cfg.get_pool_config()) + .runtime(Runtime::Tokio1) + .build() + .map_err(|e| format!("Failed to create connection pool: {e}")) +} + +async fn discover_ts_configs(pool: &PgPool) -> AHashSet<&'static str> { let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]); match probe_ts_configs(pool).await { @@ -245,7 +200,7 @@ async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> { trc::event!( Store(trc::StoreEvent::PostgresqlError), Details = "Failed to query pg_ts_config, assuming english only", - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -253,7 +208,7 @@ async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> { ts_configs } -async fn probe_ts_configs(pool: &Pool) -> trc::Result> { +async fn probe_ts_configs(pool: &PgPool) -> trc::Result> { let conn = pool.get().await.map_err(into_pool_error)?; conn.query("SELECT cfgname::text FROM pg_ts_config", &[]) diff --git a/crates/store/src/backend/postgres/mod.rs b/crates/store/src/backend/postgres/mod.rs index 39d50409..39aa043c 100644 --- a/crates/store/src/backend/postgres/mod.rs +++ b/crates/store/src/backend/postgres/mod.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use self::pool::PgPool; use crate::{ search::{ CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField, @@ -12,33 +13,40 @@ use crate::{ write::SearchIndex, }; use ahash::AHashSet; -use deadpool_postgres::Pool; +use compact_str::CompactString; +use std::sync::Arc; use tokio_postgres::error::SqlState; +use self::sql::SqlStatements; + pub mod blob; pub mod lookup; pub mod main; +pub mod pool; +mod pooled; pub mod read; pub mod search; +pub mod sql; pub mod tls; pub mod write; pub struct PostgresStore { - pub(crate) conn_pool: Pool, + pub(crate) conn_pool: PgPool, pub(crate) ts_configs: AHashSet<&'static str>, + pub(crate) sql: Arc, } #[inline(always)] fn into_error(err: tokio_postgres::error::Error) -> trc::Error { let mut local_err = trc::StoreEvent::PostgresqlError.reason(error_chain(&err)); if let Some(db_err) = err.as_db_error() { - local_err = local_err.code(db_err.code().code().to_string()); + local_err = local_err.code(CompactString::from(db_err.code().code())); if let Some(detail) = db_err.detail() { - local_err = local_err.details(detail.to_string()); + local_err = local_err.details(CompactString::from(detail)); } if let Some(hint) = db_err.hint() { - local_err = local_err.caused_by(hint.to_string()); + local_err = local_err.caused_by(CompactString::from(hint)); } } local_err @@ -60,6 +68,8 @@ fn error_chain(err: &(dyn std::error::Error + 'static)) -> String { pub(crate) const DELETE_CHUNK_SIZE: usize = 1000; pub(crate) const MIN_DELETE_CHUNK_SIZE: usize = 10; +pub(crate) const ITERATE_CHUNK_SIZE: usize = 1000; +pub(crate) const MIN_ITERATE_CHUNK_SIZE: usize = 10; #[inline(always)] pub(crate) fn is_timeout_error(err: &tokio_postgres::Error) -> bool { @@ -89,13 +99,25 @@ impl SearchIndex { SearchIndex::InMemory => "", } } + + fn reads_account_first(&self) -> bool { + matches!(self, SearchIndex::Calendar | SearchIndex::Contacts) + } +} + +impl SearchField { + fn holds_long_text(&self) -> bool { + matches!( + self, + SearchField::Calendar(CalendarSearchField::Description) + | SearchField::Contact(ContactSearchField::Note) + ) + } } trait PsqlSearchField { fn column(&self) -> &'static str; fn column_type(&self) -> &'static str; - fn sort_column_type(&self) -> Option<&'static str>; - fn sort_column(&self) -> Option<&'static str>; } impl PsqlSearchField for EmailSearchField { @@ -108,47 +130,16 @@ impl PsqlSearchField for EmailSearchField { EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "atta", - EmailSearchField::ReceivedAt => "rcvd", - EmailSearchField::SentAt => "sent", - EmailSearchField::Size => "size", - EmailSearchField::HasAttachment => "hatt", EmailSearchField::Headers => "hdrs", } } fn column_type(&self) -> &'static str { match self { - EmailSearchField::ReceivedAt | EmailSearchField::SentAt => "BIGINT", - EmailSearchField::Size => "INTEGER", - EmailSearchField::HasAttachment => "BOOLEAN", EmailSearchField::Headers => "JSONB", _ => "TSVECTOR", } } - - fn sort_column_type(&self) -> Option<&'static str> { - match self { - EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject => { - Some("TEXT") - } - #[cfg(feature = "test_mode")] - EmailSearchField::Cc | EmailSearchField::Bcc => Some("TEXT"), - _ => None, - } - } - - fn sort_column(&self) -> Option<&'static str> { - match self { - EmailSearchField::From => Some("s_fr"), - EmailSearchField::To => Some("s_to"), - EmailSearchField::Subject => Some("s_sj"), - #[cfg(feature = "test_mode")] - EmailSearchField::Bcc => Some("s_bc"), - #[cfg(feature = "test_mode")] - EmailSearchField::Cc => Some("s_cc"), - _ => None, - } - } } impl PsqlSearchField for CalendarSearchField { @@ -159,25 +150,11 @@ impl PsqlSearchField for CalendarSearchField { CalendarSearchField::Location => "locn", CalendarSearchField::Owner => "ownr", CalendarSearchField::Attendee => "atnd", - CalendarSearchField::Start => "strt", - CalendarSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { - match self { - CalendarSearchField::Start => "BIGINT", - CalendarSearchField::Uid => "TEXT", - _ => "TSVECTOR", - } - } - - fn sort_column_type(&self) -> Option<&'static str> { - None - } - - fn sort_column(&self) -> Option<&'static str> { - None + "TSVECTOR" } } @@ -194,30 +171,20 @@ impl PsqlSearchField for ContactSearchField { ContactSearchField::Address => "addr", ContactSearchField::Note => "note", ContactSearchField::Kind => "kind", - ContactSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { - ContactSearchField::Kind | ContactSearchField::Uid => "TEXT", + ContactSearchField::Kind => "TEXT", _ => "TSVECTOR", } } - - fn sort_column_type(&self) -> Option<&'static str> { - None - } - - fn sort_column(&self) -> Option<&'static str> { - None - } } impl PsqlSearchField for FileSearchField { fn column(&self) -> &'static str { match self { - FileSearchField::Name => "name", FileSearchField::Content => "body", } } @@ -225,14 +192,6 @@ impl PsqlSearchField for FileSearchField { fn column_type(&self) -> &'static str { "TSVECTOR" } - - fn sort_column_type(&self) -> Option<&'static str> { - None - } - - fn sort_column(&self) -> Option<&'static str> { - None - } } impl PsqlSearchField for TracingSearchField { fn column(&self) -> &'static str { @@ -250,14 +209,6 @@ impl PsqlSearchField for TracingSearchField { TracingSearchField::Keywords => "TSVECTOR", } } - - fn sort_column_type(&self) -> Option<&'static str> { - None - } - - fn sort_column(&self) -> Option<&'static str> { - None - } } impl PsqlSearchField for SearchField { @@ -286,26 +237,4 @@ impl PsqlSearchField for SearchField { SearchField::Tracing(field) => field.column_type(), } } - - fn sort_column_type(&self) -> Option<&'static str> { - match self { - SearchField::Email(field) => field.sort_column_type(), - SearchField::Calendar(field) => field.sort_column_type(), - SearchField::Contact(field) => field.sort_column_type(), - SearchField::File(field) => field.sort_column_type(), - SearchField::Tracing(field) => field.sort_column_type(), - SearchField::AccountId | SearchField::DocumentId | SearchField::Id => None, - } - } - - fn sort_column(&self) -> Option<&'static str> { - match self { - SearchField::Email(field) => field.sort_column(), - SearchField::Calendar(field) => field.sort_column(), - SearchField::Contact(field) => field.sort_column(), - SearchField::File(field) => field.sort_column(), - SearchField::Tracing(field) => field.sort_column(), - SearchField::AccountId | SearchField::DocumentId | SearchField::Id => None, - } - } } diff --git a/crates/store/src/backend/postgres/pool.rs b/crates/store/src/backend/postgres/pool.rs new file mode 100644 index 00000000..5b278378 --- /dev/null +++ b/crates/store/src/backend/postgres/pool.rs @@ -0,0 +1,128 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use deadpool::managed::{self, Metrics, RecycleError, RecycleResult}; +use deadpool_postgres::{ClientWrapper, Manager}; +use futures::{FutureExt, StreamExt}; +use std::{ + ops::{Deref, DerefMut}, + pin::Pin, +}; +use tokio_postgres::{Error, SimpleQueryStream}; + +const ROLLBACK: &str = "ROLLBACK"; + +pub type PgPool = managed::Pool; +pub type PgObject = managed::Object; + +pub struct PgManager(Manager); + +pub struct PgConn { + client: ClientWrapper, + rollback: Option>>, +} + +pub(crate) enum RollbackOutcome { + Succeeded, + Failed, + Pending, +} + +impl PgManager { + pub(crate) fn new(manager: Manager) -> Self { + PgManager(manager) + } +} + +impl managed::Manager for PgManager { + type Type = PgConn; + type Error = Error; + + async fn create(&self) -> Result { + self.0.create().await.map(|client| PgConn { + client, + rollback: None, + }) + } + + async fn recycle(&self, conn: &mut PgConn, metrics: &Metrics) -> RecycleResult { + conn.check_deferred_rollback()?; + self.0.recycle(&mut conn.client, metrics).await?; + conn.check_deferred_rollback() + } + + fn detach(&self, conn: &mut PgConn) { + self.0.detach(&mut conn.client); + } +} + +impl PgConn { + pub(crate) fn defer_rollback(&mut self) -> bool { + if !matches!(self.rollback_outcome(), RollbackOutcome::Succeeded) { + return false; + } + #[cfg(stalwart_tests)] + let statement = if crate::backend::sql_fault::fire() { + crate::backend::sql_fault::FAILING_ROLLBACK + } else { + ROLLBACK + }; + #[cfg(not(stalwart_tests))] + let statement = ROLLBACK; + match self.client.simple_query_raw(statement).now_or_never() { + Some(Ok(stream)) => { + self.rollback = Some(Box::pin(stream)); + true + } + _ => false, + } + } + + fn check_deferred_rollback(&mut self) -> RecycleResult { + if let RollbackOutcome::Failed = self.rollback_outcome() { + #[cfg(stalwart_tests)] + crate::backend::sql_fault::record_detection( + crate::backend::sql_fault::Detection::Checkout, + ); + Err(RecycleError::message("Deferred ROLLBACK failed")) + } else { + Ok(()) + } + } + + pub(crate) fn rollback_outcome(&mut self) -> RollbackOutcome { + let Some(mut stream) = self.rollback.take() else { + return RollbackOutcome::Succeeded; + }; + loop { + match stream.next().now_or_never() { + Some(Some(Ok(_))) => (), + Some(None) => return RollbackOutcome::Succeeded, + Some(Some(Err(_))) => return RollbackOutcome::Failed, + None => { + self.rollback = Some(stream); + return RollbackOutcome::Pending; + } + } + } + } +} + +impl Deref for PgConn { + type Target = ClientWrapper; + + #[inline] + fn deref(&self) -> &ClientWrapper { + &self.client + } +} + +impl DerefMut for PgConn { + #[inline] + fn deref_mut(&mut self) -> &mut ClientWrapper { + &mut self.client + } +} diff --git a/crates/store/src/backend/postgres/pooled.rs b/crates/store/src/backend/postgres/pooled.rs new file mode 100644 index 00000000..5cf2e2b7 --- /dev/null +++ b/crates/store/src/backend/postgres/pooled.rs @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{into_error, pool::PgObject}; +use deadpool::managed::Object; +use std::ops::{Deref, DerefMut}; + +pub(super) struct PooledConn { + object: Option, + on_drop: OnDrop, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum OnDrop { + Return, + RollBack, + Detach, +} + +impl PooledConn { + pub(super) fn new(object: PgObject) -> Self { + PooledConn { + object: Some(object), + on_drop: OnDrop::Return, + } + } + + #[inline] + pub(super) fn set_detach(&mut self, detach: bool) { + self.on_drop = if detach { + OnDrop::Detach + } else { + OnDrop::Return + }; + } + + #[inline] + pub(super) fn transaction_started(&mut self) { + self.on_drop = OnDrop::RollBack; + } + + pub(super) fn abandon_transaction(&mut self) { + self.on_drop = if self.defer_rollback() { + OnDrop::Return + } else { + OnDrop::Detach + }; + } + + pub(super) fn end_transaction( + &mut self, + result: Result<(), tokio_postgres::Error>, + ) -> trc::Result<()> { + match result { + Ok(()) => { + self.on_drop = OnDrop::Return; + Ok(()) + } + Err(err) => { + self.on_drop = OnDrop::Detach; + Err(into_error(err)) + } + } + } +} + +impl Deref for PooledConn { + type Target = PgObject; + + #[inline] + fn deref(&self) -> &PgObject { + self.object + .as_ref() + .expect("the connection is only taken when the guard is dropped") + } +} + +impl DerefMut for PooledConn { + #[inline] + fn deref_mut(&mut self) -> &mut PgObject { + self.object + .as_mut() + .expect("the connection is only taken when the guard is dropped") + } +} + +impl Drop for PooledConn { + fn drop(&mut self) { + if self.on_drop == OnDrop::RollBack && self.object.is_some() { + self.abandon_transaction(); + } + if self.on_drop == OnDrop::Detach + && let Some(object) = self.object.take() + { + drop(Object::take(object)); + } + } +} diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index 813d853d..52504e01 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -6,10 +6,28 @@ use super::{PostgresStore, into_error, is_timeout_error}; use crate::{ - Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error, - write::ValueClass, + Deserialize, Fetch, IterateParams, Key, Subspace, ValueKey, + backend::{ + ValueColumn, + postgres::{ + ITERATE_CHUNK_SIZE, MIN_ITERATE_CHUNK_SIZE, into_pool_error, pooled::PooledConn, + }, + range_batch::{Bounds, RangeAttempt, RangeCursor, Timeout}, + }, + dispatch::store::point_keys, + write::{AnyKey, ChunkPlan, ValueClass, backoff}, }; use futures::{TryStreamExt, pin_mut}; +use std::fmt::Write; +use tokio_postgres::{Row, RowStream, types::ToSql}; + +pub(super) const MAX_POINT_KEYS: usize = 4096; +const DEGRADED_POINT_KEYS: usize = 512; +const MIN_POINT_KEYS: usize = 16; +const MAX_RANGES_PER_STMT: usize = 64; +const BEGIN_READ: &str = "START TRANSACTION ISOLATION LEVEL REPEATABLE READ"; +const COMMIT: &str = "COMMIT"; +const ROLLBACK: &str = "ROLLBACK"; impl PostgresStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -18,13 +36,12 @@ impl PostgresStore { { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn - .prepare_cached(&format!( - "SELECT v FROM {} WHERE k = $1", - char::from(key.subspace()) - )) + .prepare_cached(&self.sql.get(key.subspace()).get_value) .await .map_err(into_error)?; let key = key.serialize(0); + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Read); conn.query_opt(&s, &[&key]) .await .map_err(into_error) @@ -40,13 +57,12 @@ impl PostgresStore { pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn - .prepare_cached(&format!( - "SELECT 1 FROM {} WHERE k = $1", - char::from(key.subspace()) - )) + .prepare_cached(&self.sql.get(key.subspace()).key_exists) .await .map_err(into_error)?; let key = key.serialize(0); + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Read); conn.query_opt(&s, &[&key]) .await .map_err(into_error) @@ -58,90 +74,155 @@ impl PostgresStore { params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { + debug_assert_ne!(params.fetch, Fetch::Limit(0)); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - let table = char::from(params.begin.subspace()); - let begin = params.begin.serialize(0); - let end = params.end.serialize(0); - let keys = if params.values { "k, v" } else { "k" }; - - let s = conn - .prepare_cached(&match (params.first, params.ascending) { - (true, true) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1" - ) - } - (true, false) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC LIMIT 1" - ) - } - (false, true) => { - format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC") - } - (false, false) => { - format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC") - } - }) - .await.map_err(into_error)?; - let mut from = begin; - let mut to = end; - let mut resume_key: Option> = None; + let subspace = params.begin.subspace(); + let stmts = self.sql.get(subspace); + let column = ValueColumn::new(params.values, subspace); + let mut from = params.begin.serialize(0); + let mut to = params.end.serialize(0); + let mut last_key = Vec::new(); + let mut resuming = false; + let mut plan = ChunkPlan::new(params.fetch); + let mut attempt = 0; loop { - let mut last_key = None; - let mut timed_out = false; + let limit = plan.limit(resuming); + let (statement, bound) = match limit { + Some(limit) if limit > 1 => ( + stmts.iterate_limit(params.ascending, params.values), + Some(i64::from(limit)), + ), + limit => ( + stmts.iterate(limit.is_some(), params.ascending, params.values), + None, + ), + }; + let mut skip_resumed = resuming; + let mut progressed = false; + let mut fetched = 0u64; + let mut timed_out = None; - { - let rows = conn - .query_raw(&s, &[&from, &to]) - .await - .map_err(into_error)?; - - pin_mut!(rows); - - loop { - match rows.try_next().await { - Ok(Some(row)) => { - let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?; - let value = if params.values { - row.try_get::<_, &[u8]>(1).map_err(into_error)? - } else { - b"".as_slice() - }; - - if resume_key.take().is_some_and(|resumed| resumed == key) { - continue; - } - - if !cb(key, value)? { - return Ok(()); - } - - last_key = Some(key.to_vec()); + let result = match conn.prepare_cached(statement).await { + Ok(s) => { + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Iterate); + match &bound { + Some(bound) => { + conn.query_raw(&s, [&from as &(dyn ToSql + Sync), &to, bound]) + .await } - Ok(None) => break, - Err(err) => { - if params.first || last_key.is_none() || !is_timeout_error(&err) { - return Err(into_error(err)); + None => conn.query_raw(&s, &[&from, &to]).await, + } + } + Err(err) => Err(err), + }; + match result { + Ok(rows) => { + pin_mut!(rows); + + loop { + match rows.try_next().await { + Ok(Some(row)) => { + let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?; + let counter; + let value = match column { + ValueColumn::Skip => b"".as_slice(), + ValueColumn::Bytes => { + row.try_get::<_, &[u8]>(1).map_err(into_error)? + } + ValueColumn::Counter => { + counter = read_counter(&row)?; + counter.as_slice() + } + }; + + fetched += 1; + #[cfg(stalwart_tests)] + super::probe::row_sent(); + if skip_resumed { + skip_resumed = false; + if key == last_key.as_slice() { + continue; + } + } + + if !cb(key, value)? || !plan.consume_row() { + #[cfg(stalwart_tests)] + super::probe::drain(rows.as_mut()).await; + return Ok(()); + } + + last_key.clear(); + last_key.extend_from_slice(key); + progressed = true; } - timed_out = true; - break; + Ok(None) => break, + Err(err) if is_timeout_error(&err) => { + timed_out = Some(err); + break; + } + Err(err) => return Err(into_error(err)), } } } + Err(err) if is_timeout_error(&err) => timed_out = Some(err), + Err(err) => return Err(into_error(err)), } - match last_key { - Some(last_key) if timed_out => { - if params.ascending { - from.clone_from(&last_key); - } else { - to.clone_from(&last_key); - } - resume_key = Some(last_key); + if progressed { + attempt = 0; + match timed_out { + None if limit.is_some_and(|limit| fetched >= u64::from(limit)) => plan.grow(), + None => return Ok(()), + Some(_) => (), } - _ => return Ok(()), + if params.ascending { + from.clone_from(&last_key); + } else { + to.clone_from(&last_key); + } + resuming = true; + } else { + match timed_out { + Some(err) + if !plan + .degrade(ITERATE_CHUNK_SIZE, MIN_ITERATE_CHUNK_SIZE, &mut attempt) + .await => + { + return Err(into_error(err)); + } + Some(_) => (), + None => return Ok(()), + } + } + } + } + + pub(crate) async fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + let Some(first) = ranges.first() else { + return Ok(()); + }; + let subspace = first.begin.subspace(); + let column = ValueColumn::new(first.values, subspace); + let mut conn = PooledConn::new(self.conn_pool.get().await.map_err(into_pool_error)?); + + match point_keys(&ranges) { + Some(keys) => { + let query = self.sql.get(subspace).iterate_points(column.is_selected()); + iterate_points(&mut conn, query, &keys, column, &mut cb).await + } + None => { + let bounds = ranges + .iter() + .map(|params| (params.begin.serialize(0), params.end.serialize(0))) + .collect::>(); + self.iterate_ranges(conn, subspace, bounds, column, &mut cb) + .await } } } @@ -151,14 +232,16 @@ impl PostgresStore { key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); - let table = char::from(key.subspace()); + let subspace = key.subspace(); let key = key.serialize(0); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn - .prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1")) + .prepare_cached(&self.sql.get(subspace).get_value) .await .map_err(into_error)?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Read); match conn.query_opt(&s, &[&key]).await { Ok(Some(row)) => row.try_get(0).map_err(into_error), Ok(None) => Ok(0), @@ -166,3 +249,401 @@ impl PostgresStore { } } } + +type RangeCallback<'y> = + &'y mut (dyn for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Send + Sync); + +enum Emitted { + Done, + Stopped, + TimedOut { + error: tokio_postgres::Error, + delivered: bool, + }, +} + +async fn iterate_points( + conn: &mut PooledConn, + query: &str, + keys: &[Vec], + column: ValueColumn, + cb: RangeCallback<'_>, +) -> trc::Result<()> { + let mut pending = keys; + let mut last_key = Vec::new(); + let mut plan = ChunkPlan::new(Fetch::All); + let mut attempt = 0; + + loop { + let size = plan + .limit(false) + .map_or(MAX_POINT_KEYS, |size| size as usize); + let before = pending.len(); + let error = match points_attempt(conn, query, &mut pending, size, column, &mut last_key, cb) + .await? + { + None => return Ok(()), + Some(Timeout::Begin(_)) if backoff(&mut attempt).await => continue, + Some(Timeout::Begin(error)) => return Err(into_error(error)), + Some(Timeout::Statement(error)) => error, + }; + pending = keys_after(pending, &last_key); + if pending.is_empty() { + return Ok(()); + } else if pending.len() < before { + attempt = 0; + } else if !plan + .degrade( + DEGRADED_POINT_KEYS + .min(pending.len() / 2) + .max(MIN_POINT_KEYS), + MIN_POINT_KEYS, + &mut attempt, + ) + .await + { + return Err(into_error(error)); + } + } +} + +async fn points_attempt( + conn: &mut PooledConn, + query: &str, + pending: &mut &[Vec], + size: usize, + column: ValueColumn, + last_key: &mut Vec, + cb: RangeCallback<'_>, +) -> trc::Result>> { + if pending.len() <= size { + let rows = match conn.prepare_cached(query).await { + Ok(statement) => { + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Iterate); + conn.query_raw(&statement, [pending as &(dyn ToSql + Sync)]) + .await + } + Err(error) => Err(error), + }; + return match emit(rows, column, false, last_key, cb).await? { + Emitted::Done | Emitted::Stopped => Ok(None), + Emitted::TimedOut { error, .. } => Ok(Some(Timeout::Statement(error))), + }; + } + + if let Err(error) = begin_read(conn).await? { + return Ok(Some(Timeout::Begin(error))); + } + let statement = match conn.prepare_cached(query).await { + Ok(statement) => statement, + Err(error) if is_timeout_error(&error) => { + let ended = conn.batch_execute(ROLLBACK).await; + conn.end_transaction(ended)?; + return Ok(Some(Timeout::Statement(error))); + } + Err(error) => return Err(into_error(error)), + }; + while !pending.is_empty() { + let (chunk, rest) = pending.split_at(pending.len().min(size)); + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Iterate); + let rows = conn + .query_raw(&statement, [&chunk as &(dyn ToSql + Sync)]) + .await; + match emit(rows, column, false, last_key, cb).await? { + Emitted::Done => *pending = rest, + Emitted::Stopped => return Ok(None), + Emitted::TimedOut { error, .. } => { + let ended = conn.batch_execute(ROLLBACK).await; + conn.end_transaction(ended)?; + return Ok(Some(Timeout::Statement(error))); + } + } + } + let ended = commit_read(conn).await; + conn.end_transaction(ended)?; + Ok(None) +} + +impl PostgresStore { + async fn iterate_ranges( + &self, + mut conn: PooledConn, + subspace: Subspace, + mut bounds: Vec, + column: ValueColumn, + cb: RangeCallback<'_>, + ) -> trc::Result<()> { + let table = subspace.name(); + let mut cursor = RangeCursor::default(); + let mut per_statement = MAX_RANGES_PER_STMT; + let mut attempt = 0; + + loop { + match ranges_attempt( + &mut conn, + table, + &mut bounds, + &mut cursor, + per_statement, + column, + cb, + ) + .await? + { + RangeAttempt::Finished => return Ok(()), + RangeAttempt::BeginTimedOut(error) => { + if !backoff(&mut attempt).await { + return Err(error); + } + } + RangeAttempt::TimedOut { delivered: true } => attempt = 0, + RangeAttempt::TimedOut { delivered: false } => { + match cursor.halve(per_statement, bounds.len()) { + Some(size) => per_statement = size, + None => { + drop(conn); + return self + .iterate_each(subspace, &bounds, &cursor, column, cb) + .await; + } + } + } + } + } + } + + async fn iterate_each( + &self, + subspace: Subspace, + bounds: &[Bounds], + cursor: &RangeCursor, + column: ValueColumn, + cb: RangeCallback<'_>, + ) -> trc::Result<()> { + let mut stopped = false; + for (begin, end) in cursor.remaining(bounds) { + let params = IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { + subspace, + key: end.to_vec(), + }, + ) + .set_values(column.is_selected()); + self.iterate(params, |key, value| { + let more = cb(key, value)?; + stopped = !more; + Ok(more) + }) + .await?; + if stopped { + break; + } + } + Ok(()) + } +} + +fn range_query(table: &str, values: bool, count: usize) -> String { + debug_assert!(count > 0); + let mut query = String::with_capacity(count * 28 + 40); + let columns = if values { "k, v" } else { "k" }; + let _ = write!(query, "SELECT {columns} FROM {table} WHERE "); + for i in 0..count { + if i > 0 { + query.push_str(" OR "); + } + let _ = write!(query, "(k >= ${} AND k <= ${})", i * 2 + 1, i * 2 + 2); + } + query.push_str(" ORDER BY k ASC"); + query +} + +async fn ranges_attempt( + conn: &mut PooledConn, + table: &str, + bounds: &mut [Bounds], + cursor: &mut RangeCursor, + size: usize, + column: ValueColumn, + cb: RangeCallback<'_>, +) -> trc::Result { + let total = bounds.len(); + if cursor.start >= total { + return Ok(RangeAttempt::Finished); + } else if total - cursor.start <= size { + let chunk = bounds.get(cursor.start..).unwrap_or_default(); + let rows = async { + let s = conn + .prepare_cached(&range_query(table, column.is_selected(), chunk.len())) + .await?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Iterate); + conn.query_raw( + &s, + chunk + .iter() + .flat_map(|(begin, end)| [begin, end]) + .collect::>(), + ) + .await + } + .await; + return Ok( + match emit(rows, column, cursor.resumed, &mut cursor.last_key, cb).await? { + Emitted::Done | Emitted::Stopped => RangeAttempt::Finished, + Emitted::TimedOut { delivered, .. } => { + if delivered { + cursor.advance(bounds, total); + } + RangeAttempt::TimedOut { delivered } + } + }, + ); + } + + if let Err(error) = begin_read(conn).await? { + return Ok(RangeAttempt::BeginTimedOut(into_error(error))); + } + while cursor.start < total { + let end = (cursor.start + size).min(total); + let chunk = bounds.get(cursor.start..end).unwrap_or_default(); + let rows = async { + let s = conn + .prepare_cached(&range_query(table, column.is_selected(), chunk.len())) + .await?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::Iterate); + conn.query_raw( + &s, + chunk + .iter() + .flat_map(|(begin, end)| [begin, end]) + .collect::>(), + ) + .await + } + .await; + match emit(rows, column, cursor.resumed, &mut cursor.last_key, cb).await? { + Emitted::Done => { + cursor.start = end; + cursor.resumed = false; + } + Emitted::Stopped => return Ok(RangeAttempt::Finished), + Emitted::TimedOut { delivered, .. } => { + if delivered { + cursor.advance(bounds, end); + } + let ended = conn.batch_execute(ROLLBACK).await; + conn.end_transaction(ended)?; + return Ok(RangeAttempt::TimedOut { delivered }); + } + } + } + let ended = commit_read(conn).await; + conn.end_transaction(ended)?; + Ok(RangeAttempt::Finished) +} + +async fn commit_read(conn: &PooledConn) -> Result<(), tokio_postgres::Error> { + #[cfg(stalwart_tests)] + if crate::backend::sql_fault::fire() { + return conn.batch_execute("SELECT 1/0").await; + } + conn.batch_execute(COMMIT).await +} + +async fn begin_read(conn: &mut PooledConn) -> trc::Result> { + conn.transaction_started(); + match conn.batch_execute(BEGIN_READ).await { + Ok(()) => Ok(Ok(())), + Err(error) => { + conn.abandon_transaction(); + if is_timeout_error(&error) { + Ok(Err(error)) + } else { + Err(into_error(error)) + } + } + } +} + +async fn emit( + rows: Result, + column: ValueColumn, + resumed: bool, + last_key: &mut Vec, + cb: RangeCallback<'_>, +) -> trc::Result { + let rows = match rows { + Ok(rows) => rows, + Err(error) if is_timeout_error(&error) => { + return Ok(Emitted::TimedOut { + error, + delivered: false, + }); + } + Err(error) => return Err(into_error(error)), + }; + pin_mut!(rows); + let mut skip_resumed = resumed; + let mut delivered = false; + + loop { + match rows.try_next().await { + Ok(Some(row)) => { + #[cfg(stalwart_tests)] + super::probe::row_sent(); + let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?; + let counter; + let value = match column { + ValueColumn::Skip => b"".as_slice(), + ValueColumn::Bytes => row.try_get::<_, &[u8]>(1).map_err(into_error)?, + ValueColumn::Counter => { + counter = read_counter(&row)?; + counter.as_slice() + } + }; + + if skip_resumed { + skip_resumed = false; + if key == last_key.as_slice() { + continue; + } + } + + if !cb(key, value)? { + #[cfg(stalwart_tests)] + super::probe::drain(rows.as_mut()).await; + return Ok(Emitted::Stopped); + } + + last_key.clear(); + last_key.extend_from_slice(key); + delivered = true; + } + Ok(None) => return Ok(Emitted::Done), + Err(error) if is_timeout_error(&error) => { + return Ok(Emitted::TimedOut { error, delivered }); + } + Err(error) => return Err(into_error(error)), + } + } +} + +#[inline] +fn read_counter(row: &Row) -> trc::Result<[u8; 8]> { + row.try_get::<_, i64>(1) + .map(i64::to_le_bytes) + .map_err(into_error) +} + +fn keys_after<'x>(keys: &'x [Vec], last_key: &[u8]) -> &'x [Vec] { + keys.split_at(keys.partition_point(|key| key.as_slice() <= last_key)) + .1 +} diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index ca083e40..95ec62ee 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -10,18 +10,60 @@ use crate::{ into_pool_error, is_timeout_error, }, search::{ - IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, - SearchQuery, SearchValue, + IndexDocument, KeyValueMatch, SearchField, SearchFilter, SearchQuery, SearchResults, + SearchValue, TextMatch, TracingSearchField, }, write::SearchIndex, }; +use futures::{TryStreamExt, pin_mut}; use nlp::language::Language; -use std::fmt::Write; +use roaring::RoaringBitmap; +use std::{cmp::Ordering, fmt::Write}; use tokio_postgres::{ - IsolationLevel, - types::{FromSql, ToSql, Type, WrongType}, + RowStream, + types::{FromSql, IsNull, ToSql, Type, WrongType}, }; +const MAX_TSVECTOR_CHARS: usize = 250_000; +const MAX_KEYWORD_CHARS: usize = 512; +pub const DENSE_ACCOUNT_DOCUMENTS: u64 = 32_768; +const ACCOUNT_FIRST_DOCUMENTS: u64 = 4_096; +const ACCOUNT_FIRST_COLUMNS_CAPACITY: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TsQueryForm { + Inline, + InitPlan, + DenseInitPlan, +} + +impl TsQueryForm { + fn of(documents: u64) -> Self { + if documents >= DENSE_ACCOUNT_DOCUMENTS { + TsQueryForm::DenseInitPlan + } else { + TsQueryForm::InitPlan + } + } + + fn open(self) -> &'static str { + match self { + TsQueryForm::Inline => " @@ ", + TsQueryForm::InitPlan | TsQueryForm::DenseInitPlan => " @@ (SELECT ", + } + } + + fn close(self) -> &'static str { + match self { + TsQueryForm::Inline => "", + TsQueryForm::InitPlan | TsQueryForm::DenseInitPlan => ")", + } + } +} + +#[cfg(stalwart_tests)] +pub static QUERY_LOG: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + impl PostgresStore { fn ts_config(&self, language: &Language) -> &'static str { pg_lang(language) @@ -29,139 +71,216 @@ impl PostgresStore { .unwrap_or(PG_UNSTEMMED_LANG) } - pub async fn index(&self, documents: Vec) -> trc::Result<()> { - let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?; - let trx = conn - .build_transaction() - .isolation_level(IsolationLevel::ReadCommitted) - .start() - .await - .map_err(into_error)?; - - for document in documents { - let index = document.index; - let primary_keys = index.primary_keys(); - let all_fields = index.all_fields(); - let fields = document.fields; - let mut values = Vec::with_capacity(fields.len() + 2); - let mut query = format!("INSERT INTO {} (", index.psql_table()); - - for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { - if i > 0 { - query.push(','); - } - query.push_str(field.column()); - - if let Some(sort_column) = field.sort_column() { - query.push(','); - query.push_str(sort_column); - } + pub async fn index(&self, mut documents: Vec) -> trc::Result<()> { + let mut buffer = String::new(); + for document in &mut documents { + document.split_identifiers(&mut buffer); + } + documents.sort_by_key(DocumentKey::of); + documents.dedup_by(|later, kept| { + let is_duplicate = DocumentKey::of(later) == DocumentKey::of(kept); + if is_duplicate { + std::mem::swap(later, kept); } + is_duplicate + }); + let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - query.push_str(") VALUES ("); - - for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { - if i > 0 { - query.push(','); - } - - if let Some(value) = fields.get(field) { - let value_ref = format!("${}", values.len() + 1); - let (text_len, language) = if let SearchValue::Text { value, language } = value - { - (value.len(), self.ts_config(language)) - } else { - (0, PG_UNSTEMMED_LANG) - }; - - if field.is_text() { - let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})"); - } else if text_len > 512 { - query.push_str("left("); - query.push_str(&value_ref); - query.push_str(",512)"); - } else { - query.push_str(&value_ref); - } - - if field.sort_column().is_some() { - if text_len > 255 { - query.push_str(",left("); - query.push_str(&value_ref); - query.push_str(",255)"); - } else { - query.push(','); - query.push_str(&value_ref); - } - } - - values.push(value as &(dyn ToSql + Sync)); - } else { - query.push_str("NULL"); - if field.sort_column().is_some() { - query.push_str(",NULL"); - } - } - } - - query.push_str(") ON CONFLICT ("); - for (i, pkey) in primary_keys.iter().enumerate() { - if i > 0 { - query.push(','); - } - query.push_str(pkey.column()); - } - query.push_str(") DO UPDATE SET "); - for (i, field) in all_fields.iter().enumerate() { - if i > 0 { - query.push(','); - } - let column = field.column(); - let _ = write!(&mut query, "{column} = EXCLUDED.{column}"); - } - - trx.execute(&query, &values).await.map_err(into_error)?; + for documents in documents.chunk_by(|a, b| a.index == b.index) { + let Some(index) = documents.first().map(|document| document.index) else { + continue; + }; + let statement = conn + .prepare_cached(&upsert_statement(index)) + .await + .map_err(into_error)?; + let params = self.upsert_params(index, documents); + conn.execute_raw(&statement, params.iter()) + .await + .map_err(into_error)?; } - trx.commit().await.map_err(into_error) + Ok(()) } - pub async fn query( + fn upsert_params<'x>( + &self, + index: SearchIndex, + documents: &'x [IndexDocument], + ) -> Vec> { + let (primary_keys, all_fields) = (index.primary_keys(), index.all_fields()); + let mut params = Vec::with_capacity(2 * (primary_keys.len() + all_fields.len())); + + for field in primary_keys.iter().chain(all_fields) { + let values = documents.iter().map(|document| document.fields.get(field)); + if field.is_text() { + let (values, configs) = values + .map(|value| { + let config = match value { + Some(SearchValue::Text { language, .. }) => self.ts_config(language), + _ => PG_UNSTEMMED_LANG, + }; + (value, config) + }) + .unzip(); + params.push(UpsertParam::Values(values)); + params.push(UpsertParam::Configs(configs)); + } else { + params.push(UpsertParam::Values(values.collect())); + } + } + + params + } + + pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], - sort: &[SearchComparator], - ) -> trc::Result> { + ) -> trc::Result { let mut query = format!("SELECT {} FROM {}", R::field().column(), index.psql_table()); - let params = self.build_filter(&mut query, filters); - if !sort.is_empty() { - build_sort(&mut query, sort); - } - let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - let s = conn.prepare_cached(&query).await.map_err(into_error)?; + let params = self.build_filter(&mut query, filters, TsQueryForm::Inline); + self.select(query, ¶ms, TsQueryForm::Inline).await + } - conn.query(&s, params.as_slice()) - .await - .and_then(|rows| { - rows.into_iter() - .map(|row| row.try_get::<_, DocId>(0).map(|v| R::from_u64(v.0))) - .collect::, _>>() - }) - .map_err(into_error) + pub async fn query_account( + &self, + index: SearchIndex, + filters: &[SearchFilter], + mask: &RoaringBitmap, + ) -> trc::Result { + if index.reads_account_first() + && let Some((first_id, last_id)) = mask.min().zip(mask.max()) + && u64::from(last_id - first_id) < ACCOUNT_FIRST_DOCUMENTS + && IntegerColumn::Int4.binds(last_id.into()) + && let Some(account_id) = account_filter(filters) + && let Some(columns) = account_first_columns(R::field(), filters) + { + return self + .query_account_first(index, filters, &columns, account_id, first_id, last_id) + .await; + } + + let mut query = format!("SELECT {} FROM {}", R::field().column(), index.psql_table()); + let form = TsQueryForm::of(mask.len()); + let params = self.build_filter(&mut query, filters, form); + self.select(query, ¶ms, form).await + } + + async fn query_account_first( + &self, + index: SearchIndex, + filters: &[SearchFilter], + columns: &[&str], + account_id: u64, + first_id: u32, + last_id: u32, + ) -> trc::Result { + let mut where_clause = String::new(); + let mut params = self.build_filter(&mut where_clause, filters, TsQueryForm::Inline); + + let mut query = String::with_capacity(where_clause.len() + 160); + let _ = write!(query, "SELECT {} FROM (SELECT ", R::field().column()); + for (pos, column) in columns.iter().enumerate() { + if pos > 0 { + query.push_str(", "); + } + query.push_str(column); + } + let document_column = SearchField::DocumentId.column(); + let _ = write!( + query, + " FROM {} WHERE {} = ${} AND {document_column} >= ${} AND {document_column} <= ${} \ + OFFSET 0) c{where_clause}", + index.psql_table(), + SearchField::AccountId.column(), + params.len() + 1, + params.len() + 2, + params.len() + 3, + ); + params.push(FilterParam::Integer(account_id, IntegerColumn::Int4)); + params.push(FilterParam::Integer(first_id.into(), IntegerColumn::Int4)); + params.push(FilterParam::Integer(last_id.into(), IntegerColumn::Int4)); + self.select(query, ¶ms, TsQueryForm::Inline).await + } + + pub async fn filter_candidates( + &self, + index: SearchIndex, + filters: &[SearchFilter], + candidates: &[u32], + ) -> trc::Result { + let mut query = format!("SELECT docid FROM {}", index.psql_table()); + let mut params = self.build_filter(&mut query, filters, TsQueryForm::Inline); + let candidates = candidates + .iter() + .map(|document_id| document_id.cast_signed()) + .collect::>(); + let _ = write!( + query, + "{} docid = ANY(${}::int4[])", + if filters.is_empty() { " WHERE" } else { " AND" }, + params.len() + 1 + ); + params.push(FilterParam::Ids(&candidates)); + self.select(query, ¶ms, TsQueryForm::Inline).await + } + + async fn select( + &self, + query: String, + params: &[FilterParam<'_>], + form: TsQueryForm, + ) -> trc::Result { + let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?; + let typed = params.iter().map(|param| (param, param.sql_type())); + let (results, num_rows) = match form { + TsQueryForm::Inline | TsQueryForm::InitPlan => { + collect_ids::( + conn.query_typed_raw(&query, typed) + .await + .map_err(into_error)?, + ) + .await? + } + TsQueryForm::DenseInitPlan => { + let trx = conn.transaction().await.map_err(into_error)?; + trx.batch_execute("SET LOCAL jit = off") + .await + .map_err(into_error)?; + let results = collect_ids::( + trx.query_typed_raw(&query, typed) + .await + .map_err(into_error)?, + ) + .await?; + trx.commit().await.map_err(into_error)?; + results + } + }; + + #[cfg(stalwart_tests)] + if let Ok(mut log) = QUERY_LOG.lock() { + log.push((query, num_rows)); + } + #[cfg(not(stalwart_tests))] + let _ = num_rows; + + Ok(results) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { debug_assert!(!filter.filters.is_empty()); let table = filter.index.psql_table(); let mut where_clause = String::new(); - let params = self.build_filter(&mut where_clause, &filter.filters); + let params = self.build_filter(&mut where_clause, &filter.filters, TsQueryForm::Inline); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached(&format!("DELETE FROM {table}{where_clause}")) .await .map_err(into_error)?; - match conn.execute(&s, params.as_slice()).await { + match conn.execute_raw(&s, params.iter()).await { Ok(deleted) => return Ok(deleted), Err(err) if is_timeout_error(&err) => (), Err(err) => return Err(into_error(err)), @@ -179,7 +298,7 @@ impl PostgresStore { .map_err(into_error)?; loop { - match conn.execute(&s, params.as_slice()).await { + match conn.execute_raw(&s, params.iter()).await { Ok(0) => return Ok(deleted), Ok(affected) => deleted += affected, Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { @@ -196,7 +315,8 @@ impl PostgresStore { &self, query: &mut String, filters: &'x [SearchFilter], - ) -> Vec<&'x (dyn ToSql + Sync)> { + form: TsQueryForm, + ) -> Vec> { if filters.is_empty() { return Vec::new(); } @@ -208,7 +328,12 @@ impl PostgresStore { for filter in filters { match filter { - SearchFilter::Operator { field, op, value } => { + SearchFilter::Text { + field, + op, + value, + language, + } => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), @@ -219,55 +344,108 @@ impl PostgresStore { is_first = false; } let value_pos = values.len() + 1; - if field.is_text() - && matches!(op, SearchOperator::Equal | SearchOperator::Contains) - { - query.push_str(field.column()); - query.push(' '); + query.push_str(field.column()); + if field.is_text() { + query.push_str(form.open()); + let config = self.ts_config(language); + match op { + TextMatch::Exact | TextMatch::Standard => { + let method = if matches!(op, TextMatch::Exact) { + "phraseto_tsquery" + } else { + "plainto_tsquery" + }; - let language = match &value { - SearchValue::Text { language, .. } => *language, - _ => Language::None, - }; - let config = self.ts_config(&language); - let method = match op { - SearchOperator::Equal => "phraseto_tsquery", - _ => "plainto_tsquery", - }; - - if matches!(language, Language::None) { - let _ = write!(query, "@@ {method}('{config}', ${value_pos})"); - } else { - let _ = write!(query, "@@ ({method}('{config}', ${value_pos})"); - for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] { - if fallback != config && self.ts_configs.contains(fallback) { - let _ = - write!(query, " || {method}('{fallback}', ${value_pos})"); + if matches!(language, Language::None) { + let _ = write!(query, "{method}('{config}', ${value_pos})"); + } else { + let _ = write!(query, "({method}('{config}', ${value_pos})"); + for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] { + if fallback != config && self.ts_configs.contains(fallback) + { + let _ = write!( + query, + " || {method}('{fallback}', ${value_pos})" + ); + } + } + query.push(')'); } } - query.push(')'); + TextMatch::Prefix => { + let _ = write!( + query, + "to_tsquery('{PG_UNSTEMMED_LANG}', coalesce(nullif(array_to_string(tsvector_to_array(to_tsvector('{PG_UNSTEMMED_LANG}', ${value_pos})), ':* & '), '') || ':*', ''))" + ); + } } - values.push(value as &(dyn ToSql + Sync)); - } else if let SearchValue::KeyValues(kv) = value { - query.push_str(field.column()); - query.push(' '); - - let (key, value) = kv.iter().next().unwrap(); - values.push(key as &(dyn ToSql + Sync)); - - if !value.is_empty() { - let _ = write!(query, "->> ${value_pos} "); - op.write_pqsql(query, values.len() + 1); - values.push(value as &(dyn ToSql + Sync)); - } else { - let _ = write!(query, " ? ${value_pos}"); + query.push_str(form.close()); + } else { + match op { + TextMatch::Prefix => { + let _ = write!(query, " LIKE ${value_pos} || '%'"); + } + _ => { + let _ = write!(query, " = ${value_pos}"); + } + } + } + values.push(FilterParam::Text(value)); + } + SearchFilter::KeyValue { field, key, op } => { + if !is_first { + match operator { + SearchFilter::And => query.push_str(" AND "), + SearchFilter::Or => query.push_str(" OR "), + _ => (), } } else { - query.push_str(field.sort_column().unwrap_or(field.column())); - query.push(' '); - - op.write_pqsql(query, value_pos); - values.push(value as &(dyn ToSql + Sync)); + is_first = false; + } + query.push_str(field.column()); + query.push(' '); + let key_pos = values.len() + 1; + values.push(FilterParam::Text(key)); + match op { + KeyValueMatch::Equals(value) => { + let value_pos = values.len() + 1; + let _ = write!(query, "->> ${key_pos} = ${value_pos}"); + values.push(FilterParam::Text(value)); + } + KeyValueMatch::Contains(value) => { + let value_pos = values.len() + 1; + let _ = + write!(query, "->> ${key_pos} ILIKE '%' || ${value_pos} || '%'"); + values.push(FilterParam::Text(value)); + } + KeyValueMatch::Exists => { + let _ = write!(query, " ? ${key_pos}"); + } + } + } + SearchFilter::Integer { field, op, value } => { + if !is_first { + match operator { + SearchFilter::And => query.push_str(" AND "), + SearchFilter::Or => query.push_str(" OR "), + _ => (), + } + } else { + is_first = false; + } + let cmp = match op { + Ordering::Less => "<", + Ordering::Equal => "=", + Ordering::Greater => ">", + }; + match IntegerColumn::of(field).filter(|column| column.binds(*value)) { + Some(column) => { + let _ = write!(query, "{} {cmp} ${}", field.column(), values.len() + 1); + values.push(FilterParam::Integer(*value, column)); + } + None => { + let _ = write!(query, "{} {cmp} {value}", field.column()); + } } } SearchFilter::And | SearchFilter::Or => { @@ -298,7 +476,7 @@ impl PostgresStore { } operator_stack.push((operator, is_first)); - operator = &SearchFilter::And; + operator = &SearchFilter::Or; is_first = true; query.push_str("NOT ("); } @@ -321,31 +499,6 @@ impl PostgresStore { } } -fn build_sort(query: &mut String, sort: &[SearchComparator]) { - query.push_str(" ORDER BY "); - for (i, comparator) in sort.iter().enumerate() { - if i > 0 { - query.push_str(", "); - } - match comparator { - SearchComparator::Field { field, ascending } => { - query.push_str(field.sort_column().unwrap_or(field.column())); - if *ascending { - query.push_str(" ASC"); - } else { - query.push_str(" DESC"); - } - } - SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { - debug_assert!( - false, - "DocumentSet and SortedSet comparators are not supported " - ); - } - } - } -} - impl ToSql for SearchValue { fn to_sql( &self, @@ -418,6 +571,275 @@ impl ToSql for SearchValue { } } +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct DocumentKey(u8, u64, u64); + +impl DocumentKey { + fn of(document: &IndexDocument) -> Self { + let mut keys = + document + .index + .primary_keys() + .iter() + .map(|field| match document.fields.get(field) { + Some(SearchValue::Uint(value)) => *value, + Some(SearchValue::Int(value)) => value.cast_unsigned(), + _ => 0, + }); + DocumentKey( + document.index as u8, + keys.next().unwrap_or_default(), + keys.next().unwrap_or_default(), + ) + } +} + +async fn collect_ids(rows: RowStream) -> trc::Result<(R, usize)> { + pin_mut!(rows); + let mut results = R::default(); + let mut num_rows = 0; + while let Some(row) = rows.try_next().await.map_err(into_error)? { + results.insert(row.try_get::<_, DocId>(0).map_err(into_error)?.0); + num_rows += 1; + } + Ok((results, num_rows)) +} + +fn upsert_statement(index: SearchIndex) -> String { + let primary_keys = index.primary_keys(); + let all_fields = index.all_fields(); + let mut columns = String::with_capacity(128); + let mut values = String::with_capacity(512); + let mut arrays = String::with_capacity(256); + let mut aliases = String::with_capacity(128); + let mut param = 0; + + for (pos, field) in primary_keys.iter().chain(all_fields).enumerate() { + if pos > 0 { + columns.push(','); + values.push(','); + arrays.push(','); + aliases.push(','); + } + param += 1; + columns.push_str(field.column()); + let _ = write!(aliases, "v{pos}"); + + if field.is_text() { + let _ = write!(arrays, "${param}::text[],${}::text[]", param + 1); + let _ = write!( + values, + "to_tsvector(c{pos}::regconfig,left(v{pos},{MAX_TSVECTOR_CHARS}))" + ); + let _ = write!(aliases, ",c{pos}"); + param += 1; + } else { + let (array_type, value) = match IntegerColumn::of(field) { + Some(column) => (column.type_name(), None), + None if field.is_json() => ("jsonb", None), + None => ("text", Some(MAX_KEYWORD_CHARS)), + }; + let _ = write!(arrays, "${param}::{array_type}[]"); + match value { + Some(max_chars) => { + let _ = write!(values, "left(v{pos},{max_chars})"); + } + None => { + let _ = write!(values, "v{pos}"); + } + } + } + } + + let mut statement = format!( + "INSERT INTO {} ({columns}) SELECT {values} FROM unnest({arrays}) AS u({aliases}) ON CONFLICT (", + index.psql_table() + ); + for (pos, field) in primary_keys.iter().enumerate() { + if pos > 0 { + statement.push(','); + } + statement.push_str(field.column()); + } + statement.push_str(") DO UPDATE SET "); + for (pos, field) in all_fields.iter().enumerate() { + if pos > 0 { + statement.push(','); + } + let column = field.column(); + let _ = write!(statement, "{column} = EXCLUDED.{column}"); + } + statement +} + +enum UpsertParam<'x> { + Values(Vec>), + Configs(Vec<&'static str>), +} + +impl std::fmt::Debug for UpsertParam<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + UpsertParam::Values(values) => write!(f, "Values({})", values.len()), + UpsertParam::Configs(configs) => write!(f, "Configs({configs:?})"), + } + } +} + +impl ToSql for UpsertParam<'_> { + fn to_sql( + &self, + ty: &Type, + out: &mut bytes::BytesMut, + ) -> Result> { + match self { + UpsertParam::Values(values) => values.to_sql(ty, out), + UpsertParam::Configs(configs) => configs.to_sql(ty, out), + } + } + + fn accepts(_: &Type) -> bool { + true + } + + fn to_sql_checked( + &self, + ty: &Type, + out: &mut bytes::BytesMut, + ) -> Result> { + match self { + UpsertParam::Values(values) => values.to_sql_checked(ty, out), + UpsertParam::Configs(configs) => configs.to_sql_checked(ty, out), + } + } +} + +fn account_first_columns( + result: SearchField, + filters: &[SearchFilter], +) -> Option> { + let mut columns = Vec::with_capacity(ACCOUNT_FIRST_COLUMNS_CAPACITY); + columns.push(result.column()); + for filter in filters { + if let SearchFilter::Text { field, .. } + | SearchFilter::KeyValue { field, .. } + | SearchFilter::Integer { field, .. } = filter + { + if field.holds_long_text() { + return None; + } + if !columns.contains(&field.column()) { + columns.push(field.column()); + } + } + } + Some(columns) +} + +fn account_filter(filters: &[SearchFilter]) -> Option { + let mut depth = 0usize; + for filter in filters { + match filter { + SearchFilter::And | SearchFilter::Or | SearchFilter::Not => depth += 1, + SearchFilter::End => depth = depth.saturating_sub(1), + SearchFilter::Integer { + field: SearchField::AccountId, + op: Ordering::Equal, + value, + } if depth == 0 && IntegerColumn::Int4.binds(*value) => return Some(*value), + _ => {} + } + } + None +} + +#[derive(Debug, Clone, Copy)] +enum IntegerColumn { + Int4, + Int8, +} + +impl IntegerColumn { + fn of(field: &SearchField) -> Option { + match field { + SearchField::AccountId | SearchField::DocumentId => Some(IntegerColumn::Int4), + SearchField::Id + | SearchField::Tracing(TracingSearchField::EventType | TracingSearchField::QueueId) => { + Some(IntegerColumn::Int8) + } + _ => None, + } + } + + fn type_name(self) -> &'static str { + match self { + IntegerColumn::Int4 => "int4", + IntegerColumn::Int8 => "int8", + } + } + + fn binds(self, value: u64) -> bool { + match self { + IntegerColumn::Int4 => i32::try_from(value).is_ok(), + IntegerColumn::Int8 => i64::try_from(value).is_ok(), + } + } +} + +#[derive(Debug)] +enum FilterParam<'x> { + Text(&'x str), + Integer(u64, IntegerColumn), + Ids(&'x [i32]), +} + +impl FilterParam<'_> { + fn sql_type(&self) -> Type { + match self { + FilterParam::Text(_) => Type::TEXT, + FilterParam::Integer(_, IntegerColumn::Int4) => Type::INT4, + FilterParam::Integer(_, IntegerColumn::Int8) => Type::INT8, + FilterParam::Ids(_) => Type::INT4_ARRAY, + } + } +} + +impl ToSql for FilterParam<'_> { + fn to_sql( + &self, + ty: &Type, + out: &mut bytes::BytesMut, + ) -> Result> { + match self { + FilterParam::Text(value) => value.to_sql(ty, out), + FilterParam::Integer(value, _) => match *ty { + Type::INT4 => i32::try_from(*value)?.to_sql(ty, out), + _ => i64::try_from(*value)?.to_sql(ty, out), + }, + FilterParam::Ids(ids) => ids.to_sql(ty, out), + } + } + + fn accepts(_: &Type) -> bool { + true + } + + fn to_sql_checked( + &self, + ty: &Type, + out: &mut bytes::BytesMut, + ) -> Result> { + match self { + FilterParam::Text(value) => value.to_sql_checked(ty, out), + FilterParam::Integer(value, _) => match *ty { + Type::INT4 => i32::try_from(*value)?.to_sql_checked(ty, out), + _ => i64::try_from(*value)?.to_sql_checked(ty, out), + }, + FilterParam::Ids(ids) => ids.to_sql_checked(ty, out), + } + } +} + struct DocId(u64); impl FromSql<'_> for DocId { @@ -437,31 +859,6 @@ impl FromSql<'_> for DocId { } } -impl SearchOperator { - fn write_pqsql(&self, query: &mut String, value_pos: usize) { - match self { - SearchOperator::LowerThan => { - let _ = write!(query, "< ${value_pos}"); - } - SearchOperator::LowerEqualThan => { - let _ = write!(query, "<= ${value_pos}"); - } - SearchOperator::GreaterThan => { - let _ = write!(query, "> ${value_pos}"); - } - SearchOperator::GreaterEqualThan => { - let _ = write!(query, ">= ${value_pos}"); - } - SearchOperator::Equal => { - let _ = write!(query, "= ${value_pos}"); - } - SearchOperator::Contains => { - let _ = write!(query, "LIKE '%' || ${value_pos} || '%'"); - } - } - } -} - pub(super) const PG_FALLBACK_LANG: &str = "english"; pub(super) const PG_UNSTEMMED_LANG: &str = "simple"; diff --git a/crates/store/src/backend/postgres/sql.rs b/crates/store/src/backend/postgres/sql.rs new file mode 100644 index 00000000..9b233fc8 --- /dev/null +++ b/crates/store/src/backend/postgres/sql.rs @@ -0,0 +1,156 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Subspace; + +pub(crate) struct SubspaceSql { + pub(crate) get_value: Box, + pub(crate) get_value_for_update: Box, + pub(crate) get_values_for_update: Box, + pub(crate) key_exists: Box, + pub(crate) insert_value: Box, + pub(crate) update_value: Box, + pub(crate) upsert_value: Box, + pub(crate) insert_presence: Box, + pub(crate) delete_key: Box, + pub(crate) delete_keys: Box, + pub(crate) increment: Box, + pub(crate) decrement: Box, + pub(crate) increment_returning: Box, + pub(crate) increment_bounded: Box, + pub(crate) delete_range: Box, + pub(crate) range_boundary: Box, + pub(crate) purge_zero: Box, + pub(crate) purge_zero_range: Box, + pub(crate) purge_zero_from: Box, + pub(crate) purge_boundary: Box, + iterate: [Box; 8], + iterate_limit: [Box; 4], + iterate_points: [Box; 2], +} + +const POINT_KEYS: &str = "((SELECT $1::bytea[]))::bytea[]"; + +pub(crate) struct SqlStatements { + statements: Box<[SubspaceSql]>, +} + +impl SubspaceSql { + fn new(subspace: Subspace) -> Self { + let table = subspace.name(); + let range = |order: &str, limit: &str, columns: &str| { + format!( + "SELECT {columns} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k {order}{limit}" + ) + .into_boxed_str() + }; + + Self { + get_value: format!("SELECT v FROM {table} WHERE k = $1").into_boxed_str(), + get_value_for_update: format!("SELECT v FROM {table} WHERE k = $1 FOR UPDATE") + .into_boxed_str(), + get_values_for_update: format!( + "SELECT k, v FROM {table} WHERE k = ANY({POINT_KEYS}) ORDER BY k FOR UPDATE" + ) + .into_boxed_str(), + key_exists: format!("SELECT 1 FROM {table} WHERE k = $1").into_boxed_str(), + insert_value: format!("INSERT INTO {table} (k, v) VALUES ($1, $2)").into_boxed_str(), + update_value: format!("UPDATE {table} SET v = $2 WHERE k = $1").into_boxed_str(), + upsert_value: format!( + "INSERT INTO {table} (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" + ) + .into_boxed_str(), + insert_presence: format!( + "INSERT INTO {table} (k) VALUES ($1) ON CONFLICT (k) DO NOTHING" + ) + .into_boxed_str(), + delete_key: format!("DELETE FROM {table} WHERE k = $1").into_boxed_str(), + delete_keys: format!("DELETE FROM {table} WHERE k = ANY({POINT_KEYS})") + .into_boxed_str(), + increment: format!( + "INSERT INTO {table} (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = {table}.v + EXCLUDED.v" + ) + .into_boxed_str(), + decrement: format!("UPDATE {table} SET v = v + $1 WHERE k = $2").into_boxed_str(), + increment_returning: format!( + "INSERT INTO {table} (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = {table}.v + EXCLUDED.v RETURNING v" + ) + .into_boxed_str(), + increment_bounded: format!( + "INSERT INTO {table} (k, v) VALUES ($1, CASE WHEN $2::BIGINT <= $3::BIGINT THEN $2::BIGINT END) ON CONFLICT (k) DO UPDATE SET v = CASE WHEN {table}.v + $2::BIGINT <= $3::BIGINT THEN {table}.v + $2::BIGINT END RETURNING v" + ) + .into_boxed_str(), + delete_range: format!("DELETE FROM {table} WHERE k >= $1 AND k < $2").into_boxed_str(), + range_boundary: format!( + "SELECT k FROM {table} WHERE k >= $1 AND k < $2 ORDER BY k ASC LIMIT 1 OFFSET $3" + ) + .into_boxed_str(), + purge_zero: format!("DELETE FROM {table} WHERE v = 0").into_boxed_str(), + purge_zero_range: format!("DELETE FROM {table} WHERE v = 0 AND k >= $1 AND k < $2") + .into_boxed_str(), + purge_zero_from: format!("DELETE FROM {table} WHERE v = 0 AND k >= $1") + .into_boxed_str(), + purge_boundary: format!( + "SELECT k FROM {table} WHERE k >= $1 ORDER BY k ASC LIMIT 1 OFFSET $2" + ) + .into_boxed_str(), + iterate: [ + range("ASC", "", "k"), + range("ASC", "", "k, v"), + range("DESC", "", "k"), + range("DESC", "", "k, v"), + range("ASC", " LIMIT 1", "k"), + range("ASC", " LIMIT 1", "k, v"), + range("DESC", " LIMIT 1", "k"), + range("DESC", " LIMIT 1", "k, v"), + ], + iterate_limit: [ + range("ASC", " LIMIT $3", "k"), + range("ASC", " LIMIT $3", "k, v"), + range("DESC", " LIMIT $3", "k"), + range("DESC", " LIMIT $3", "k, v"), + ], + iterate_points: [ + format!("SELECT k FROM {table} WHERE k = ANY({POINT_KEYS}) ORDER BY k") + .into_boxed_str(), + format!("SELECT k, v FROM {table} WHERE k = ANY({POINT_KEYS}) ORDER BY k") + .into_boxed_str(), + ], + } + } + + #[inline(always)] + pub(crate) fn iterate(&self, first: bool, ascending: bool, values: bool) -> &str { + &self.iterate[((first as usize) << 2) | ((!ascending as usize) << 1) | (values as usize)] + } + + #[inline(always)] + pub(crate) fn iterate_limit(&self, ascending: bool, values: bool) -> &str { + &self.iterate_limit[((!ascending as usize) << 1) | (values as usize)] + } + + #[inline(always)] + pub(crate) fn iterate_points(&self, values: bool) -> &str { + &self.iterate_points[values as usize] + } +} + +impl SqlStatements { + pub(crate) fn new() -> Self { + Self { + statements: Subspace::ALL + .iter() + .copied() + .map(SubspaceSql::new) + .collect(), + } + } + + #[inline(always)] + pub(crate) fn get(&self, subspace: Subspace) -> &SubspaceSql { + &self.statements[subspace.index()] + } +} diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index d91e7f6e..32040312 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -4,398 +4,122 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +mod flight; +mod reads; + +use self::{ + flight::{Param, Request, StatementId, StatementKind, Transaction, dispatch}, + reads::{Pass, ReadSet, missing_read, unprefixed}, +}; use super::{PostgresStore, into_error, is_timeout_error}; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - SUBSPACE_REGISTRY_IDX, - backend::postgres::{DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, into_pool_error}, + Key, Shape, Subspace, WITH_SUBSPACE, + backend::postgres::{ + DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, into_pool_error, pool::PgObject, sql::SubspaceSql, + }, write::{ - AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, - ValueClass, ValueOp, + Advance, AssignedIds, Batch, BatchCursor, ChunkedRetry, LogSet, MAX_COMMIT_ATTEMPTS, + MAX_COMMIT_TIME, MergeResult, NewKeyAsserts, ValueOp, commit_backoff, exhausted, }, }; -use ahash::AHashMap; -use deadpool_postgres::Object; -use rand::RngExt; -use std::time::{Duration, Instant}; -use tokio_postgres::{IsolationLevel, error::SqlState}; +use futures::TryStreamExt; +use std::{borrow::Cow, iter, mem, ops::Range, pin::pin, time::Instant}; +use tokio_postgres::error::SqlState; + +const KEY_LEN_HINT: usize = 32; +const MIN_DELETE_RUN: usize = 16; +const MAX_DELETE_KEYS: usize = 1000; #[derive(Debug)] enum CommitError { Postgres(tokio_postgres::Error), Internal(trc::Error), - //Retry, +} + +struct Write<'x> { + statement: StatementId, + target: Target, + param: Param<'x>, +} + +enum Target { + Key(Range), + Keys(Range), } impl PostgresStore { - pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { - let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?; + pub(crate) async fn write( + &self, + mut batch: Batch<'_>, + assigned_ids: &mut AssignedIds, + ) -> trc::Result<()> { let start = Instant::now(); let mut retry_count = 0; + let mut new_key_asserts = NewKeyAsserts::CheckedByInsert; + let mark = assigned_ids.mark(); loop { - match self.write_trx(&mut conn, &mut batch).await { - Ok(result) => { - return Ok(result); - } - Err(err) => { - match err { - CommitError::Postgres(err) => match err.code() { - Some( - &SqlState::T_R_SERIALIZATION_FAILURE - | &SqlState::T_R_DEADLOCK_DETECTED, - ) if retry_count < MAX_COMMIT_ATTEMPTS - && start.elapsed() < MAX_COMMIT_TIME => {} - Some(&SqlState::UNIQUE_VIOLATION) => { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .reason("Unique violation") - .caused_by(trc::location!())); - } - _ => return Err(into_error(err)), - }, - CommitError::Internal(err) => return Err(err), - /*CommitError::Retry => { - if retry_count > MAX_COMMIT_ATTEMPTS - || start.elapsed() > MAX_COMMIT_TIME - { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!())); - } - }*/ - } - - let backoff = rand::rng().random_range(50..=300); - tokio::time::sleep(Duration::from_millis(backoff)).await; - retry_count += 1; - } + assigned_ids.rollback(mark); + let mut trx = Transaction::new( + self.conn_pool.get().await.map_err(into_pool_error)?, + &self.sql, + ); + let result = write_pipelined(&mut trx, &mut batch, assigned_ids, new_key_asserts).await; + let poisoned = trx.previous_rollback_failed(); + if result.is_err() && !poisoned { + trx.rollback(); } - } - } + drop(trx); - async fn write_trx( - &self, - conn: &mut Object, - batch: &mut Batch<'_>, - ) -> Result { - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let mut asserted_values = AHashMap::new(); - let trx = conn - .build_transaction() - .isolation_level(IsolationLevel::ReadCommitted) - .start() - .await?; - let mut result = AssignedIds::default(); - let has_changes = !batch.changes.is_empty(); + let err = match result { + Ok(()) => return Ok(()), + Err(CommitError::Postgres(err)) => err, + Err(CommitError::Internal(err)) => return Err(err), + }; - if has_changes { - for &account_id in batch.changes.keys() { - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); - let s = trx - .prepare_cached(concat!( - "INSERT INTO n (k, v) VALUES ($1, 1) ", - "ON CONFLICT(k) DO UPDATE SET v = n.v + 1 RETURNING v" - )) - .await?; - let change_id = trx - .query_one(&s, &[&key]) - .await - .and_then(|row| row.try_get::<_, i64>(0))?; - result.push_change_id(account_id, change_id as u64); + let is_contention = poisoned + || match err.code() { + Some(&SqlState::UNIQUE_VIOLATION) => { + new_key_asserts = NewKeyAsserts::Read; + false + } + Some( + &SqlState::T_R_SERIALIZATION_FAILURE + | &SqlState::T_R_DEADLOCK_DETECTED + | &SqlState::LOCK_NOT_AVAILABLE + | &SqlState::QUERY_CANCELED + | &SqlState::IDLE_IN_TRANSACTION_SESSION_TIMEOUT, + ) => true, + _ => return Err(into_error(err)), + }; + + if retry_count >= MAX_COMMIT_ATTEMPTS || start.elapsed() >= MAX_COMMIT_TIME { + return Err(if is_contention { + exhausted(into_error(err), retry_count) + } else { + trc::StoreEvent::AssertValueFailed + .into_err() + .reason("Unique violation") + .caused_by(trc::location!()) + }); } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; } - - for op in batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; - } - } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let key = class.serialize(account_id, collection, document_id, 0); - let subspace = class.subspace(collection); - let table = char::from(subspace); - - match op { - ValueOp::Set(value) => { - if subspace != SUBSPACE_REGISTRY_IDX { - let s = if let Some(exists) = asserted_values.get(&key) { - if *exists { - trx.prepare_cached(&format!( - "UPDATE {} SET v = $2 WHERE k = $1", - table - )) - .await? - } else { - trx.prepare_cached(&format!( - "INSERT INTO {} (k, v) VALUES ($1, $2)", - table - )) - .await? - } - } else { - trx.prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES ($1, $2) ", - "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" - ), - table - )) - .await? - }; - - if trx.execute(&s, &[&key, &(*value)]).await? == 0 { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!()) - .into()); - } - } else { - let s = trx - .prepare_cached( - "INSERT INTO b (k) VALUES ($1) ON CONFLICT (k) DO NOTHING", - ) - .await?; - trx.execute(&s, &[&key]).await?; - } - } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; - - let s = if let Some(exists) = asserted_values.get(&key) { - if *exists { - trx.prepare_cached(&format!( - "UPDATE {} SET v = $2 WHERE k = $1", - table - )) - .await? - } else { - trx.prepare_cached(&format!( - "INSERT INTO {} (k, v) VALUES ($1, $2)", - table - )) - .await? - } - } else { - trx.prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES ($1, $2) ", - "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" - ), - table - )) - .await? - }; - - if trx.execute(&s, &[&key, &value]).await? == 0 { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!()) - .into()); - } - } - ValueOp::MergeFnc(merge_op) => { - let s = trx - .prepare_cached(&format!( - "SELECT v FROM {} WHERE k = $1 FOR UPDATE", - table - )) - .await?; - let (exists, merge_result) = trx - .query_opt(&s, &[&key]) - .await? - .map(|row| { - row.try_get::<_, &[u8]>(0) - .map_err(CommitError::from) - .and_then(|v| { - (merge_op.fnc)(&merge_op.params, &result, Some(v)) - .map(|v| (true, v)) - .map_err(CommitError::from) - }) - }) - .unwrap_or_else(|| { - (merge_op.fnc)(&merge_op.params, &result, None) - .map(|v| (false, v)) - .map_err(CommitError::from) - })?; - - match merge_result { - MergeResult::Update(value) => { - let s = if exists { - trx.prepare_cached(&format!( - "UPDATE {} SET v = $2 WHERE k = $1", - table - )) - .await? - } else { - trx.prepare_cached(&format!( - "INSERT INTO {} (k, v) VALUES ($1, $2)", - table - )) - .await? - }; - - trx.execute(&s, &[&key, &value]).await?; - } - MergeResult::Delete if exists => { - let s = trx - .prepare_cached(&format!( - "DELETE FROM {} WHERE k = $1", - table - )) - .await?; - trx.execute(&s, &[&key]).await?; - - // Update asserted value - if let Some(exists) = asserted_values.get_mut(&key) { - *exists = false; - } - } - _ => (), - } - } - ValueOp::AtomicAdd(by) => { - if *by >= 0 { - let s = trx - .prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES ($1, $2) ", - "ON CONFLICT(k) DO UPDATE SET v = {}.v + EXCLUDED.v" - ), - table, table - )) - .await?; - trx.execute(&s, &[&key, &*by]).await?; - } else { - let s = trx - .prepare_cached(&format!( - "UPDATE {table} SET v = v + $1 WHERE k = $2" - )) - .await?; - trx.execute(&s, &[&*by, &key]).await?; - } - } - ValueOp::AddAndGet(by) => { - let s = trx - .prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES ($1, $2) ", - "ON CONFLICT(k) DO UPDATE SET v = {}.v + EXCLUDED.v RETURNING v" - ), - table, table - )) - .await?; - result.push_counter_id( - trx.query_one(&s, &[&key, &*by]) - .await - .and_then(|row| row.try_get::<_, i64>(0))?, - ); - } - ValueOp::Clear => { - let s = trx - .prepare_cached(&format!("DELETE FROM {} WHERE k = $1", table)) - .await?; - trx.execute(&s, &[&key]).await?; - - // Update asserted value - if let Some(exists) = asserted_values.get_mut(&key) { - *exists = false; - } - } - } - } - Operation::Index { field, key, set } => { - let key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: &*key, - } - .serialize(0); - - let s = if *set { - trx.prepare_cached( - "INSERT INTO i (k) VALUES ($1) ON CONFLICT (k) DO NOTHING", - ) - .await? - } else { - trx.prepare_cached("DELETE FROM i WHERE k = $1").await? - }; - trx.execute(&s, &[&key]).await?; - } - Operation::Log { collection, set } => { - let key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, - } - .serialize(0); - - let s = trx - .prepare_cached(concat!( - "INSERT INTO l (k, v) VALUES ($1, $2) ", - "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" - )) - .await?; - - trx.execute(&s, &[&key, &*set]).await?; - } - Operation::AssertValue { - class, - assert_value, - } => { - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); - - let s = trx - .prepare_cached(&format!("SELECT v FROM {} WHERE k = $1 FOR UPDATE", table)) - .await?; - let (exists, matches) = trx - .query_opt(&s, &[&key]) - .await? - .map(|row| { - row.try_get::<_, &[u8]>(0) - .map_or((true, false), |v| (true, assert_value.matches(v))) - }) - .unwrap_or_else(|| (false, assert_value.is_none())); - if !matches { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .caused_by(trc::location!()) - .into()); - } - asserted_values.insert(key, exists); - } - } - } - - trx.commit().await.map(|_| result).map_err(Into::into) } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - purge_table(&conn, char::from(subspace)).await?; + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { + purge_table(&conn, self.sql.get(subspace)).await?; } Ok(()) @@ -403,39 +127,43 @@ impl PostgresStore { pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - let table = char::from(from.subspace()); + let stmts = self.sql.get(from.subspace()); let mut from = from.serialize(0); let to = to.serialize(0); let delete = conn - .prepare_cached(&format!("DELETE FROM {table} WHERE k >= $1 AND k < $2")) + .prepare_cached(&stmts.delete_range) .await .map_err(into_error)?; + #[cfg(stalwart_tests)] + super::probe::record(super::probe::Statement::AutocommitWrite); match conn.execute(&delete, &[&from, &to]).await { Ok(_) => return Ok(()), Err(err) if is_timeout_error(&err) => (), Err(err) => return Err(into_error(err)), } - let mut chunk_size = DELETE_CHUNK_SIZE; + let mut retry = ChunkedRetry::bounded(DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE); + + let boundary = conn + .prepare_cached(&stmts.range_boundary) + .await + .map_err(into_error)?; loop { - let boundary = conn - .prepare_cached(&format!( - "SELECT k FROM {table} WHERE k >= $1 AND k < $2 ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}" - )) - .await - .map_err(into_error)?; + let chunk_size = retry.chunk_size().unwrap_or(DELETE_CHUNK_SIZE) as i64; loop { - let next = match conn.query_opt(&boundary, &[&from, &to]).await { + let next = match conn.query_opt(&boundary, &[&from, &to, &chunk_size]).await { Ok(next) => match next { Some(row) => Some(row.try_get::<_, Vec>(0).map_err(into_error)?), None => None, }, - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Err(err) if is_timeout_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -445,9 +173,11 @@ impl PostgresStore { .execute(&delete, &[&from, next.as_ref().unwrap_or(&to)]) .await { - Ok(_) => (), - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Ok(_) => retry.progressed(), + Err(err) if is_timeout_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -462,9 +192,544 @@ impl PostgresStore { } } -async fn purge_table(conn: &Object, table: char) -> trc::Result<()> { +async fn write_pipelined( + trx: &mut Transaction<'_>, + batch: &mut Batch<'_>, + ids: &mut AssignedIds, + new_key_asserts: NewKeyAsserts, +) -> Result<(), CommitError> { + let mut reads = ReadSet::default(); + let mut key_buf = Vec::with_capacity(KEY_LEN_HINT); + let deferred = reads.collect( + batch, + ids, + &mut key_buf, + Pass::BeforeAllocation(new_key_asserts), + ); + + if batch.has_id_allocations() || !reads.is_empty() { + read_flight(trx, batch, ids, &mut reads, 0).await?; + + if deferred > 0 { + let from = reads.len(); + reads.collect(batch, ids, &mut key_buf, Pass::AfterAllocation); + if reads.len() > from { + read_flight(trx, batch, ids, &mut reads, from).await?; + } + } + + #[cfg(stalwart_tests)] + super::probe::between_flights().await; + } + + let mut keys = KeyBuffer { + bytes: Vec::with_capacity(batch.ops.len() * KEY_LEN_HINT), + deletes: Vec::new(), + }; + let mut writes = Vec::with_capacity(batch.ops.len()); + build_writes( + trx, + batch, + ids, + &mut reads, + &mut key_buf, + &mut keys, + &mut writes, + )?; + let deletes = keys + .deletes + .iter() + .map(|key| keys.bytes.get(key.clone()).unwrap_or_default()) + .collect::>(); + write_flight( + trx, + ids, + Keys { + keys: &keys.bytes, + deletes: &deletes, + }, + &writes, + ) + .await +} + +struct KeyBuffer { + bytes: Vec, + deletes: Vec>, +} + +#[derive(Clone, Copy)] +struct Keys<'r> { + keys: &'r [u8], + deletes: &'r [&'r [u8]], +} + +async fn read_flight( + trx: &mut Transaction<'_>, + batch: &Batch<'_>, + ids: &mut AssignedIds, + reads: &mut ReadSet, + from: usize, +) -> Result<(), CommitError> { + let begin = !trx.in_transaction(); + let mut counter_keys = Vec::new(); + let mut counters = Vec::new(); + if begin { + for allocation in batch.id_allocations() { + let start = counter_keys.len(); + allocation.serialize_key_into(&mut counter_keys, 0); + counters.push((allocation.increment_by(), start..counter_keys.len())); + } + } + let counter = (!counters.is_empty()) + .then(|| trx.statement_id(Subspace::Counter, StatementKind::IncrementReturning)); + let plan = reads.keys.plan(from, trx); + trx.prepare().await?; + + if begin { + trx.begin(); + } + let trx = &*trx; + let requests = begin + .then_some(Request::Begin) + .into_iter() + .chain(counter.into_iter().flat_map(|statement| { + let statement = trx.statement(statement); + counters.iter().map(|(by, key)| Request::Counter { + statement, + key: counter_keys.get(key.clone()).unwrap_or_default(), + by: *by, + }) + })) + .chain(plan.single.iter().map(|(_, key, statement)| Request::Read { + statement: trx.statement(*statement), + key, + })) + .chain( + plan.chunks + .iter() + .map(|(statement, range)| Request::ReadMany { + statement: trx.statement(*statement), + keys: plan.many_keys.get(range.clone()).unwrap_or_default(), + }), + ); + let mut flight = dispatch(trx.client(), requests); + flight.send().await?; + + if begin { + flight.next().await?.done()?; + if counter.is_some() { + for allocation in batch.id_allocations() { + ids.apply(allocation, flight.next().await?.counter()?)?; + } + } + } + + for (slot, ..) in &plan.single { + reads.values.read(*slot, flight.next().await?.value()?); + } + + for (_, range) in &plan.chunks { + let slots = plan.many_slots.get(range.clone()).unwrap_or_default(); + let keys = plan.many_keys.get(range.clone()).unwrap_or_default(); + for slot in slots { + reads.values.read(*slot, None); + } + + let mut rows = pin!(flight.next().await?.stream()?); + while let Some(row) = rows.try_next().await? { + let slot = { + let key: &[u8] = row.try_get(0)?; + keys.binary_search(&key) + .ok() + .and_then(|index| slots.get(index)) + .copied() + } + .ok_or_else(missing_read)?; + reads.values.read(slot, Some(row)); + } + } + + Ok(()) +} + +fn build_writes<'x>( + trx: &mut Transaction<'_>, + batch: &'x mut Batch<'_>, + ids: &mut AssignedIds, + reads: &mut ReadSet, + key_buf: &mut Vec, + keys: &mut KeyBuffer, + writes: &mut Vec>, +) -> Result<(), CommitError> { + let KeyBuffer { + bytes: keys, + deletes, + } = keys; + let mut cursor = BatchCursor::new(batch); + let mut log_scratch = Vec::new(); + let checked_by_insert = mem::take(&mut reads.checked_by_insert); + let mut checked_by_insert = checked_by_insert.iter().copied().peekable(); + + for (index, op) in batch.ops.iter_mut().enumerate() { + match cursor.advance(op, ids) { + Advance::Cursor => {} + Advance::Value { class, op } => { + cursor.value_key(class, key_buf, WITH_SUBSPACE); + let subspace = cursor.subspace(class); + let slot = reads.keys.find(subspace, key_buf); + + match op { + ValueOp::Set(value) => { + let value = value.resolve(ids)?; + if matches!(subspace.shape(), Shape::Presence) { + let statement = + trx.statement_id(subspace, StatementKind::InsertPresence); + push_write(writes, keys, statement, key_buf, Param::Key); + } else { + let kind = match slot { + Some(slot) if reads.values.is_asserted(slot) => { + if reads.values.exists(slot) { + StatementKind::Update + } else { + StatementKind::Insert + } + } + _ => StatementKind::Upsert, + }; + let statement = trx.statement_id(subspace, kind); + let write = + push_write(writes, keys, statement, key_buf, Param::Value(value)); + if let Some(slot) = slot { + reads.values.written(slot, write); + } + } + } + ValueOp::MergeFnc(merge) => { + let slot = slot.ok_or_else(missing_read)?; + let exists = reads.values.exists(slot); + let merged = (merge.0)(ids, reads.values.value(slot, writes)?)?; + + match merged { + MergeResult::Update(value) => { + let kind = if exists { + StatementKind::Update + } else { + StatementKind::Insert + }; + let statement = trx.statement_id(subspace, kind); + let write = push_write( + writes, + keys, + statement, + key_buf, + Param::Value(Cow::Owned(value)), + ); + reads.values.written(slot, write); + } + MergeResult::Delete if exists => { + push_delete(trx, writes, keys, deletes, subspace, key_buf); + reads.values.cleared(slot); + } + MergeResult::Delete | MergeResult::Skip => {} + } + } + ValueOp::AtomicAdd(by) => { + let (kind, param) = if *by >= 0 { + (StatementKind::Increment, Param::Add(*by)) + } else { + (StatementKind::Decrement, Param::Subtract(*by)) + }; + let statement = trx.statement_id(subspace, kind); + push_write(writes, keys, statement, key_buf, param); + } + ValueOp::AddAndGet(by) => { + let statement = + trx.statement_id(subspace, StatementKind::IncrementReturning); + push_write(writes, keys, statement, key_buf, Param::AddAndGet(*by)); + } + ValueOp::Clear => { + push_delete(trx, writes, keys, deletes, subspace, key_buf); + if let Some(slot) = slot { + reads.values.cleared(slot); + } + } + } + } + Advance::Index { field, key, set } => { + cursor.index_key(field, key, key_buf, WITH_SUBSPACE); + if set { + let statement = + trx.statement_id(Subspace::Indexes, StatementKind::InsertPresence); + push_write(writes, keys, statement, key_buf, Param::Key); + } else { + push_delete(trx, writes, keys, deletes, Subspace::Indexes, key_buf); + } + } + Advance::Log { collection, set } => { + cursor.log_key(collection, ids, key_buf, WITH_SUBSPACE); + let value = match set { + LogSet::Bytes(bytes) => Cow::Borrowed(bytes.as_slice()), + LogSet::Pending(changes) => { + let mut value = Vec::new(); + changes.serialize_into( + collection.is_prefixed(), + ids, + &mut log_scratch, + &mut value, + ); + Cow::Owned(value) + } + }; + let slot = reads.keys.find(Subspace::Logs, key_buf); + let statement = trx.statement_id(Subspace::Logs, StatementKind::Upsert); + let write = push_write(writes, keys, statement, key_buf, Param::Value(value)); + if let Some(slot) = slot { + reads.values.written(slot, write); + } + } + Advance::ClearRange { subspace, from, to } => { + reads.clear_range(subspace, from, to); + let statement = trx.statement_id(subspace, StatementKind::DeleteRange); + let start = keys.len(); + keys.extend_from_slice(from); + writes.push(Write { + statement, + target: Target::Key(start..keys.len()), + param: Param::Until(Cow::Borrowed(to)), + }); + } + Advance::Assert { + class, + assert_value, + } => { + cursor.value_key(class, key_buf, WITH_SUBSPACE); + let subspace = cursor.subspace(class); + let checked = checked_by_insert.next_if_eq(&index).is_some(); + let slot = match reads.keys.find(subspace, key_buf) { + Some(slot) => slot, + None if checked => reads.assume_absent(subspace, key_buf), + None => return Err(missing_read()), + }; + let matches = match reads.values.value(slot, writes)? { + Some(value) => assert_value.matches(value), + None => assert_value.is_none(), + }; + if !matches { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .caused_by(trc::location!()) + .into()); + } + reads.values.asserted(slot); + } + } + } + + if !batch.bounds.is_empty() { + let statement = trx.statement_id(Subspace::Counter, StatementKind::IncrementBounded); + for bound in batch.bounds { + let start = keys.len(); + bound.serialize_key_into(keys, 0); + writes.push(Write { + statement, + target: Target::Key(start..keys.len()), + param: Param::Bounded(*bound), + }); + } + } + + Ok(()) +} + +async fn write_flight( + trx: &mut Transaction<'_>, + ids: &mut AssignedIds, + keys: Keys<'_>, + writes: &[Write<'_>], +) -> Result<(), CommitError> { + trx.prepare().await?; + + let begin = !trx.in_transaction(); + if begin { + match writes { + [] => return Ok(()), + [write] => return write_autocommit(trx, ids, keys, write).await, + _ => trx.begin(), + } + } + + let requests = begin + .then_some(Request::Begin) + .into_iter() + .chain(writes.iter().map(|write| write.request(trx, keys))) + .chain(iter::once(Request::Commit)); + let mut flight = dispatch(trx.client(), requests); + flight.send().await?; + let result = async { + if begin { + flight.next().await?.done()?; + } + for write in writes { + let response = flight.next().await.map_err(|err| write.map_error(err))?; + if let Param::AddAndGet(_) = write.param { + ids.push_counter_id(response.counter()?); + } else if let Param::Bounded(_) = write.param { + response.counter()?; + } else { + let rows = response.rows()?; + debug_assert!( + rows > 0 || trx.statement_kind(write.statement) != StatementKind::Update, + "update of a key read for update affected no rows" + ); + } + } + flight.next().await?.done() + } + .await; + drop(flight); + trx.commit_sent(); + + result +} + +async fn write_autocommit( + trx: &Transaction<'_>, + ids: &mut AssignedIds, + keys: Keys<'_>, + write: &Write<'_>, +) -> Result<(), CommitError> { + #[cfg(stalwart_tests)] + { + super::probe::record_flight(); + super::probe::record(super::probe::Statement::AutocommitWrite); + } + + let response = write + .request(trx, keys) + .send(trx.client()) + .await + .map_err(|err| write.map_error(err.into()))?; + if let Param::AddAndGet(_) = write.param { + ids.push_counter_id(response.counter()?); + } else if let Param::Bounded(_) = write.param { + response.counter()?; + } else { + response.rows()?; + } + Ok(()) +} + +fn push_write<'x>( + writes: &mut Vec>, + keys: &mut Vec, + statement: StatementId, + key: &[u8], + param: Param<'x>, +) -> usize { + let start = keys.len(); + keys.extend_from_slice(unprefixed(key)); + writes.push(Write { + statement, + target: Target::Key(start..keys.len()), + param, + }); + writes.len() - 1 +} + +fn push_delete( + trx: &mut Transaction<'_>, + writes: &mut Vec>, + keys: &mut Vec, + deletes: &mut Vec>, + subspace: Subspace, + key: &[u8], +) { + let start = keys.len(); + keys.extend_from_slice(unprefixed(key)); + let key = start..keys.len(); + + if let Some(Write { + statement, + target: Target::Keys(run), + .. + }) = writes.last_mut() + && run.len() < MAX_DELETE_KEYS + && trx.is_statement(*statement, subspace, StatementKind::DeleteMany) + { + deletes.push(key); + run.end = deletes.len(); + return; + } + + let delete = trx.statement_id(subspace, StatementKind::Delete); + writes.push(Write { + statement: delete, + target: Target::Key(key), + param: Param::Key, + }); + + let run = writes + .iter() + .rev() + .take(MIN_DELETE_RUN) + .take_while(|write| write.statement == delete && matches!(write.target, Target::Key(_))) + .count(); + if run == MIN_DELETE_RUN { + let from = deletes.len(); + deletes.extend( + writes + .drain(writes.len() - MIN_DELETE_RUN..) + .filter_map(|write| match write.target { + Target::Key(key) => Some(key), + Target::Keys(_) => None, + }), + ); + writes.push(Write { + statement: trx.statement_id(subspace, StatementKind::DeleteMany), + target: Target::Keys(from..deletes.len()), + param: Param::Key, + }); + } +} + +impl<'x> Write<'x> { + #[inline] + fn request<'r>(&'r self, trx: &'r Transaction<'_>, keys: Keys<'r>) -> Request<'r> { + let statement = trx.statement(self.statement); + match &self.target { + Target::Key(key) => Request::Write { + statement, + key: keys.keys.get(key.clone()).unwrap_or_default(), + param: &self.param, + }, + Target::Keys(run) => Request::WriteMany { + statement, + keys: keys.deletes.get(run.clone()).unwrap_or_default(), + }, + } + } + + #[inline] + fn value(&self) -> Option<&[u8]> { + self.param.value() + } + + fn map_error(&self, err: CommitError) -> CommitError { + match (&self.param, err) { + (Param::Bounded(bound), CommitError::Postgres(err)) + if err.code() == Some(&SqlState::NOT_NULL_VIOLATION) => + { + CommitError::Internal(bound.exceeded()) + } + (_, err) => err, + } + } +} + +async fn purge_table(conn: &PgObject, stmts: &SubspaceSql) -> trc::Result<()> { let s = conn - .prepare_cached(&format!("DELETE FROM {table} WHERE v = 0")) + .prepare_cached(&stmts.purge_zero) .await .map_err(into_error)?; @@ -475,34 +740,33 @@ async fn purge_table(conn: &Object, table: char) -> trc::Result<()> { } let purge = conn - .prepare_cached(&format!( - "DELETE FROM {table} WHERE v = 0 AND k >= $1 AND k < $2" - )) + .prepare_cached(&stmts.purge_zero_range) .await .map_err(into_error)?; let purge_last = conn - .prepare_cached(&format!("DELETE FROM {table} WHERE v = 0 AND k >= $1")) + .prepare_cached(&stmts.purge_zero_from) .await .map_err(into_error)?; - let mut chunk_size = DELETE_CHUNK_SIZE; + let boundary = conn + .prepare_cached(&stmts.purge_boundary) + .await + .map_err(into_error)?; + let mut retry = ChunkedRetry::bounded(DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE); let mut from = Vec::new(); loop { - let boundary = conn - .prepare_cached(&format!( - "SELECT k FROM {table} WHERE k >= $1 ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}" - )) - .await - .map_err(into_error)?; + let chunk_size = retry.chunk_size().unwrap_or(DELETE_CHUNK_SIZE) as i64; loop { - let next = match conn.query_opt(&boundary, &[&from]).await { + let next = match conn.query_opt(&boundary, &[&from, &chunk_size]).await { Ok(next) => match next { Some(row) => Some(row.try_get::<_, Vec>(0).map_err(into_error)?), None => None, }, - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Err(err) if is_timeout_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), @@ -514,9 +778,11 @@ async fn purge_table(conn: &Object, table: char) -> trc::Result<()> { }; match result { - Ok(_) => (), - Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => { - chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE); + Ok(_) => retry.progressed(), + Err(err) if is_timeout_error(&err) => { + if !retry.degrade().await { + return Err(into_error(err)); + } break; } Err(err) => return Err(into_error(err)), diff --git a/crates/store/src/backend/postgres/write/flight.rs b/crates/store/src/backend/postgres/write/flight.rs new file mode 100644 index 00000000..229e7ef8 --- /dev/null +++ b/crates/store/src/backend/postgres/write/flight.rs @@ -0,0 +1,414 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::CommitError; +use crate::{ + Subspace, + backend::postgres::{ + pool::{PgObject, RollbackOutcome}, + pooled::PooledConn, + sql::SqlStatements, + }, + write::CounterBound, +}; +use deadpool_postgres::ClientWrapper; +use futures::future::{MaybeDone, maybe_done, poll_fn}; +use std::{borrow::Cow, future::Future, pin::Pin, task::Poll, vec::IntoIter}; +use tokio_postgres::{Client, Row, RowStream, Statement}; + +const BEGIN: &str = "BEGIN ISOLATION LEVEL READ COMMITTED"; +const COMMIT: &str = "COMMIT"; + +pub(super) struct Flight { + pending: IntoIter>>>, + #[cfg(stalwart_tests)] + probes: Vec, +} + +pub(super) struct Transaction<'s> { + conn: PooledConn, + in_transaction: bool, + sql: &'s SqlStatements, + statements: Vec<(Subspace, StatementKind)>, + prepared: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum StatementKind { + Insert, + Update, + Upsert, + InsertPresence, + Delete, + DeleteMany, + DeleteRange, + Increment, + Decrement, + IncrementReturning, + IncrementBounded, + ReadForUpdate, + ReadManyForUpdate, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct StatementId(usize); + +pub(super) enum Param<'x> { + Key, + Value(Cow<'x, [u8]>), + Add(i64), + Subtract(i64), + AddAndGet(i64), + Bounded(CounterBound), + Until(Cow<'x, [u8]>), +} + +pub(super) enum Request<'r> { + Begin, + Commit, + Counter { + statement: &'r Statement, + key: &'r [u8], + by: i64, + }, + Read { + statement: &'r Statement, + key: &'r [u8], + }, + ReadMany { + statement: &'r Statement, + keys: &'r [&'r [u8]], + }, + Write { + statement: &'r Statement, + key: &'r [u8], + param: &'r Param<'r>, + }, + WriteMany { + statement: &'r Statement, + keys: &'r [&'r [u8]], + }, +} + +pub(super) enum Response { + Done, + Rows(u64), + Counter(i64), + Value(Option), + Stream(RowStream), +} + +impl<'s> Transaction<'s> { + pub(super) fn new(object: PgObject, sql: &'s SqlStatements) -> Self { + Transaction { + conn: PooledConn::new(object), + in_transaction: false, + sql, + statements: Vec::new(), + prepared: Vec::new(), + } + } + + #[inline] + pub(super) fn client(&self) -> &Client { + &self.conn + } + + #[inline] + pub(super) fn in_transaction(&self) -> bool { + self.in_transaction + } + + pub(super) fn statement_id(&mut self, subspace: Subspace, kind: StatementKind) -> StatementId { + let entry = (subspace, kind); + StatementId( + match self.statements.iter().position(|item| *item == entry) { + Some(id) => id, + None => { + self.statements.push(entry); + self.statements.len() - 1 + } + }, + ) + } + + #[inline] + pub(super) fn statement(&self, id: StatementId) -> &Statement { + &self.prepared[id.0] + } + + #[inline] + pub(super) fn is_statement( + &self, + id: StatementId, + subspace: Subspace, + kind: StatementKind, + ) -> bool { + self.statements.get(id.0) == Some(&(subspace, kind)) + } + + #[inline] + pub(super) fn statement_kind(&self, id: StatementId) -> StatementKind { + self.statements[id.0].1 + } + + pub(super) async fn prepare(&mut self) -> Result<(), CommitError> { + if self.prepared.len() < self.statements.len() { + Box::pin(self.prepare_missing()).await?; + } + Ok(()) + } + + async fn prepare_missing(&mut self) -> Result<(), CommitError> { + let client: &ClientWrapper = &self.conn; + for &(subspace, kind) in self + .statements + .get(self.prepared.len()..) + .unwrap_or_default() + { + self.prepared + .push(client.prepare_cached(kind.text(self.sql, subspace)).await?); + } + Ok(()) + } + + #[inline] + pub(super) fn begin(&mut self) { + self.in_transaction = true; + self.conn.set_detach(true); + } + + #[inline] + pub(super) fn commit_sent(&mut self) { + self.in_transaction = false; + self.conn.set_detach(false); + } + + pub(super) fn rollback(&mut self) { + if self.in_transaction { + self.in_transaction = false; + if self.conn.defer_rollback() { + self.conn.set_detach(false); + } + } + } + + pub(super) fn previous_rollback_failed(&mut self) -> bool { + let failed = matches!(self.conn.rollback_outcome(), RollbackOutcome::Failed); + if failed { + #[cfg(stalwart_tests)] + crate::backend::sql_fault::record_detection( + crate::backend::sql_fault::Detection::Retry, + ); + self.conn.set_detach(true); + } + failed + } +} + +impl StatementKind { + fn text(self, sql: &SqlStatements, subspace: Subspace) -> &str { + let stmts = sql.get(subspace); + match self { + StatementKind::Insert => &stmts.insert_value, + StatementKind::Update => &stmts.update_value, + StatementKind::Upsert => &stmts.upsert_value, + StatementKind::InsertPresence => &stmts.insert_presence, + StatementKind::Delete => &stmts.delete_key, + StatementKind::DeleteMany => &stmts.delete_keys, + StatementKind::DeleteRange => &stmts.delete_range, + StatementKind::Increment => &stmts.increment, + StatementKind::Decrement => &stmts.decrement, + StatementKind::IncrementReturning => &stmts.increment_returning, + StatementKind::IncrementBounded => &stmts.increment_bounded, + StatementKind::ReadForUpdate => &stmts.get_value_for_update, + StatementKind::ReadManyForUpdate => &stmts.get_values_for_update, + } + } +} + +impl Param<'_> { + #[inline] + pub(super) fn value(&self) -> Option<&[u8]> { + match self { + Param::Value(value) => Some(value), + _ => None, + } + } +} + +impl<'r> Request<'r> { + pub(super) async fn send(self, client: &'r Client) -> Result { + match self { + Request::Begin => client.batch_execute(BEGIN).await.map(|_| Response::Done), + Request::Commit => client.batch_execute(COMMIT).await.map(|_| Response::Done), + Request::Counter { statement, key, by } => client + .query_one(statement, &[&key, &by]) + .await? + .try_get(0) + .map(Response::Counter), + Request::Read { statement, key } => client + .query_opt(statement, &[&key]) + .await + .map(Response::Value), + Request::ReadMany { statement, keys } => client + .query_raw(statement, [keys]) + .await + .map(Response::Stream), + Request::Write { + statement, + key, + param, + } => match param { + Param::Key => client.execute(statement, &[&key]).await.map(Response::Rows), + Param::Value(value) => client + .execute(statement, &[&key, &value.as_ref()]) + .await + .map(Response::Rows), + Param::Add(by) => client + .execute(statement, &[&key, by]) + .await + .map(Response::Rows), + Param::Subtract(by) => client + .execute(statement, &[by, &key]) + .await + .map(Response::Rows), + Param::AddAndGet(by) => client + .query_one(statement, &[&key, by]) + .await? + .try_get(0) + .map(Response::Counter), + Param::Bounded(bound) => client + .query_one(statement, &[&key, &bound.by, &bound.max]) + .await? + .try_get(0) + .map(Response::Counter), + Param::Until(to) => client + .execute(statement, &[&key, &to.as_ref()]) + .await + .map(Response::Rows), + }, + Request::WriteMany { statement, keys } => client + .execute(statement, &[&keys]) + .await + .map(Response::Rows), + } + } +} + +impl Response { + pub(super) fn done(self) -> Result<(), CommitError> { + match self { + Response::Done => Ok(()), + _ => Err(unexpected_response()), + } + } + + pub(super) fn rows(self) -> Result { + match self { + Response::Rows(rows) => Ok(rows), + _ => Err(unexpected_response()), + } + } + + pub(super) fn counter(self) -> Result { + match self { + Response::Counter(value) => Ok(value), + _ => Err(unexpected_response()), + } + } + + pub(super) fn value(self) -> Result, CommitError> { + match self { + Response::Value(row) => Ok(row), + _ => Err(unexpected_response()), + } + } + + pub(super) fn stream(self) -> Result { + match self { + Response::Stream(stream) => Ok(stream), + _ => Err(unexpected_response()), + } + } +} + +pub(super) fn dispatch<'r>( + client: &'r Client, + requests: impl Iterator>, +) -> Flight> + 'r> { + #[cfg(stalwart_tests)] + let mut probes = Vec::with_capacity(requests.size_hint().0); + let pending = requests + .map(|request| { + #[cfg(stalwart_tests)] + probes.push(request.probe()); + Box::pin(maybe_done(request.send(client))) + }) + .collect::>() + .into_iter(); + + Flight { + pending, + #[cfg(stalwart_tests)] + probes, + } +} + +impl Flight +where + F: Future>, +{ + pub(super) async fn send(&mut self) -> Result<(), CommitError> { + #[cfg(stalwart_tests)] + crate::backend::postgres::probe::record_flight(); + #[cfg(stalwart_tests)] + let mut probes = self.probes.iter(); + + let pending = self.pending.as_mut_slice(); + let failed = poll_fn(|cx| { + for future in pending.iter_mut() { + if future.as_mut().poll(cx).is_ready() + && let Some(Err(_)) = future.as_mut().output_mut() + { + return Poll::Ready(future.as_mut().take_output()); + } + #[cfg(stalwart_tests)] + if let Some(statement) = probes.next() { + crate::backend::postgres::probe::record(*statement); + } + } + Poll::Ready(None) + }) + .await; + + match failed { + Some(Err(err)) => Err(err.into()), + _ => Ok(()), + } + } + + pub(super) async fn next(&mut self) -> Result { + let mut future = self.pending.next().ok_or_else(unexpected_response)?; + future.as_mut().await; + future + .as_mut() + .take_output() + .ok_or_else(unexpected_response)? + .map_err(CommitError::from) + } +} + +#[cold] +fn unexpected_response() -> CommitError { + CommitError::Internal( + trc::StoreEvent::UnexpectedError + .into_err() + .details("Unexpected pipelined response") + .caused_by(trc::location!()), + ) +} + diff --git a/crates/store/src/backend/postgres/write/reads.rs b/crates/store/src/backend/postgres/write/reads.rs new file mode 100644 index 00000000..e1c73d74 --- /dev/null +++ b/crates/store/src/backend/postgres/write/reads.rs @@ -0,0 +1,402 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CommitError, Write, + flight::{StatementId, StatementKind, Transaction}, +}; +use crate::{ + Shape, Subspace, WITH_SUBSPACE, + write::{ + Advance, AssignedIds, Batch, BatchCursor, NewKeyAsserts, Operation, ValueOp, insert_follows, + }, +}; +use ahash::RandomState; +use hashbrown::HashTable; +use std::{mem, ops::Range}; +use tokio_postgres::Row; + +pub(super) const PG_PRE_READ_PER_KEY_MAX: usize = 8; +pub(super) const PG_READ_BATCH_KEYS: usize = 256; +const LINEAR_LOOKUP_MAX: usize = 16; +const SUBSPACES: usize = Subspace::ALL.len(); + +const _: () = assert!(SUBSPACES <= u32::BITS as usize); + +#[derive(Default)] +pub(super) struct ReadSet { + pub(super) keys: ReadKeys, + pub(super) values: ReadValues, + pub(super) checked_by_insert: Vec, +} + +#[derive(Default)] +pub(super) struct ReadKeys { + bytes: Vec, + spans: Vec, + index: Option, + subspaces: u32, +} + +#[derive(Default)] +pub(super) struct ReadValues(Vec); + +#[derive(Default)] +pub(super) struct ReadPlan<'k> { + pub(super) single: Vec<(usize, &'k [u8], StatementId)>, + pub(super) many_keys: Vec<&'k [u8]>, + pub(super) many_slots: Vec, + pub(super) chunks: Vec<(StatementId, Range)>, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Pass { + BeforeAllocation(NewKeyAsserts), + AfterAllocation, +} + +struct KeySpan { + start: usize, + end: usize, + subspace: Subspace, +} + +struct KeyIndex { + slots: HashTable, + hasher: RandomState, +} + +#[derive(Default)] +struct ReadValue { + current: Current, + asserted: bool, +} + +#[derive(Default)] +enum Current { + #[default] + Unread, + Absent, + Stored(Row), + Written(usize), +} + +impl ReadSet { + pub(super) fn collect( + &mut self, + batch: &mut Batch<'_>, + ids: &mut AssignedIds, + key_buf: &mut Vec, + pass: Pass, + ) -> usize { + let mut cursor = BatchCursor::new(batch); + let mut pending_document = false; + let mut deferred = 0; + let mut checked_by_insert = mem::take(&mut self.checked_by_insert); + let mut next_checked = 0; + let mut remaining = &mut *batch.ops; + let mut index = 0; + + while let Some((op, rest)) = mem::take(&mut remaining).split_first_mut() { + remaining = rest; + let op_index = index; + index += 1; + + if let Operation::DocumentId { document_id } = op { + pending_document = document_id.is_pending(); + if pending_document && pass != Pass::AfterAllocation { + continue; + } + } + + let (class, asserts_absent) = match cursor.advance(op, ids) { + Advance::Value { + class, + op: ValueOp::MergeFnc(_), + } => (class, false), + Advance::Assert { + class, + assert_value, + } => (class, assert_value.is_none()), + _ => continue, + }; + let subspace = cursor.subspace(class); + + match pass { + Pass::BeforeAllocation(_) if !pending_document => { + cursor.value_key(class, key_buf, WITH_SUBSPACE); + self.insert(subspace, key_buf); + } + Pass::BeforeAllocation(asserts) => { + if cursor.is_document_independent(class, key_buf) { + self.insert(subspace, key_buf); + } else if asserts == NewKeyAsserts::CheckedByInsert + && asserts_absent + && !matches!(subspace.shape(), Shape::Presence) + && insert_follows(class, remaining) + { + checked_by_insert.push(op_index); + } else { + deferred += 1; + } + } + Pass::AfterAllocation if pending_document => { + if checked_by_insert.get(next_checked) == Some(&op_index) { + next_checked += 1; + } else { + cursor.value_key(class, key_buf, WITH_SUBSPACE); + self.insert(subspace, key_buf); + } + } + Pass::AfterAllocation => {} + } + } + + self.checked_by_insert = checked_by_insert; + deferred + } + + pub(super) fn assume_absent(&mut self, subspace: Subspace, key: &[u8]) -> usize { + self.keys.push(subspace, key); + self.values.0.push(ReadValue { + current: Current::Absent, + asserted: false, + }); + self.values.0.len() - 1 + } + + pub(super) fn clear_range(&mut self, subspace: Subspace, from: &[u8], to: &[u8]) { + if self.keys.subspaces & (1 << subspace.index()) == 0 { + return; + } + let range = from..to; + for (span, value) in self.keys.spans.iter().zip(self.values.0.iter_mut()) { + if span.subspace == subspace && range.contains(&span.key(&self.keys.bytes)) { + value.current = Current::Absent; + } + } + } + + fn insert(&mut self, subspace: Subspace, key: &[u8]) { + if self.keys.find(subspace, key).is_none() { + self.keys.push(subspace, key); + self.values.0.push(ReadValue::default()); + } + } + + #[inline] + pub(super) fn len(&self) -> usize { + self.keys.spans.len() + } + + #[inline] + pub(super) fn is_empty(&self) -> bool { + self.keys.spans.is_empty() + } +} + +impl ReadKeys { + pub(super) fn find(&self, subspace: Subspace, key: &[u8]) -> Option { + if self.subspaces & (1 << subspace.index()) == 0 { + return None; + } + + match &self.index { + Some(index) => index + .slots + .find(index.hasher.hash_one(key), |slot| { + self.spans + .get(*slot) + .is_some_and(|span| span.prefixed(&self.bytes) == key) + }) + .copied(), + None => self + .spans + .iter() + .position(|span| span.prefixed(&self.bytes) == key), + } + } + + fn push(&mut self, subspace: Subspace, key: &[u8]) { + let slot = self.spans.len(); + let start = self.bytes.len(); + self.bytes.extend_from_slice(key); + self.spans.push(KeySpan { + start, + end: self.bytes.len(), + subspace, + }); + self.subspaces |= 1 << subspace.index(); + + let ReadKeys { + bytes, + spans, + index, + .. + } = self; + match index { + Some(index) => index.insert(slot, bytes, spans), + None if spans.len() > LINEAR_LOOKUP_MAX => { + let mut new_index = KeyIndex { + slots: HashTable::with_capacity(spans.len() * 2), + hasher: RandomState::new(), + }; + for slot in 0..spans.len() { + new_index.insert(slot, bytes, spans); + } + *index = Some(new_index); + } + None => {} + } + } + + pub(super) fn plan(&self, from: usize, trx: &mut Transaction<'_>) -> ReadPlan<'_> { + let spans = self.spans.get(from..).unwrap_or_default(); + let mut counts = [0usize; SUBSPACES]; + for span in spans { + counts[span.subspace.index()] += 1; + } + + let mut plan = ReadPlan::default(); + for (slot, span) in (from..).zip(spans) { + if counts[span.subspace.index()] <= PG_PRE_READ_PER_KEY_MAX { + plan.single.push(( + slot, + span.key(&self.bytes), + trx.statement_id(span.subspace, StatementKind::ReadForUpdate), + )); + } else { + plan.many_slots.push(slot); + } + } + + if !plan.many_slots.is_empty() { + plan.many_slots.sort_unstable_by(|a, b| { + self.spans[*a] + .prefixed(&self.bytes) + .cmp(self.spans[*b].prefixed(&self.bytes)) + }); + plan.many_keys = plan + .many_slots + .iter() + .map(|slot| self.spans[*slot].key(&self.bytes)) + .collect(); + + let mut start = 0; + for run in plan + .many_slots + .chunk_by(|a, b| self.spans[*a].subspace == self.spans[*b].subspace) + { + let end = start + run.len(); + if let Some(slot) = run.first() { + let statement = trx + .statement_id(self.spans[*slot].subspace, StatementKind::ReadManyForUpdate); + for chunk_start in (start..end).step_by(PG_READ_BATCH_KEYS) { + plan.chunks.push(( + statement, + chunk_start..(chunk_start + PG_READ_BATCH_KEYS).min(end), + )); + } + } + start = end; + } + } + + plan + } +} + +impl ReadValues { + #[inline] + pub(super) fn read(&mut self, slot: usize, row: Option) { + self.0[slot].current = row.map_or(Current::Absent, Current::Stored); + } + + #[inline] + pub(super) fn written(&mut self, slot: usize, write: usize) { + self.0[slot].current = Current::Written(write); + } + + #[inline] + pub(super) fn cleared(&mut self, slot: usize) { + self.0[slot].current = Current::Absent; + } + + #[inline] + pub(super) fn asserted(&mut self, slot: usize) { + self.0[slot].asserted = true; + } + + #[inline] + pub(super) fn is_asserted(&self, slot: usize) -> bool { + self.0[slot].asserted + } + + #[inline] + pub(super) fn exists(&self, slot: usize) -> bool { + matches!( + self.0[slot].current, + Current::Stored(_) | Current::Written(_) + ) + } + + pub(super) fn value<'a>( + &'a self, + slot: usize, + writes: &'a [Write<'_>], + ) -> Result, CommitError> { + match &self.0[slot].current { + Current::Absent => Ok(None), + Current::Stored(row) => row + .try_get(row.len().saturating_sub(1)) + .map(Some) + .map_err(CommitError::from), + Current::Written(write) => Ok(writes.get(*write).and_then(Write::value)), + Current::Unread => Err(missing_read()), + } + } +} + +impl KeyIndex { + fn insert(&mut self, slot: usize, bytes: &[u8], spans: &[KeySpan]) { + let hash_of = |slot: &usize| { + self.hasher.hash_one( + spans + .get(*slot) + .map_or(&[][..], |span| span.prefixed(bytes)), + ) + }; + let hash = hash_of(&slot); + self.slots.insert_unique(hash, slot, hash_of); + } +} + +impl KeySpan { + #[inline] + fn prefixed<'a>(&self, bytes: &'a [u8]) -> &'a [u8] { + bytes.get(self.start..self.end).unwrap_or_default() + } + + #[inline] + fn key<'a>(&self, bytes: &'a [u8]) -> &'a [u8] { + unprefixed(self.prefixed(bytes)) + } +} + +#[inline] +pub(super) fn unprefixed(key: &[u8]) -> &[u8] { + key.split_first().map_or(key, |(_, key)| key) +} + +#[cold] +pub(super) fn missing_read() -> CommitError { + CommitError::Internal( + trc::StoreEvent::UnexpectedError + .into_err() + .details("Merge or assert key was not read") + .caused_by(trc::location!()), + ) +} diff --git a/crates/store/src/backend/range_batch.rs b/crates/store/src/backend/range_batch.rs new file mode 100644 index 00000000..7c475d68 --- /dev/null +++ b/crates/store/src/backend/range_batch.rs @@ -0,0 +1,64 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) type Bounds = (Vec, Vec); + +pub(crate) enum RangeAttempt { + Finished, + TimedOut { delivered: bool }, + BeginTimedOut(trc::Error), +} + +pub(crate) enum Timeout { + Begin(E), + Statement(E), +} + +#[derive(Default)] +pub(crate) struct RangeCursor { + pub(crate) start: usize, + pub(crate) resumed: bool, + pub(crate) last_key: Vec, +} + +impl RangeCursor { + pub(crate) fn advance(&mut self, bounds: &mut [Bounds], end: usize) { + let last_key = self.last_key.as_slice(); + self.start = bounds.get(self.start..end).map_or(end, |chunk| { + self.start + chunk.partition_point(|(_, range_end)| range_end.as_slice() <= last_key) + }); + self.resumed = match bounds.get_mut(self.start) { + Some((begin, _)) if begin.as_slice() <= last_key => { + begin.clone_from(&self.last_key); + true + } + _ => false, + }; + } + + pub(crate) fn halve(&self, per_statement: usize, total: usize) -> Option { + let size = per_statement.min(total.saturating_sub(self.start)); + (size > 1).then_some(size / 2) + } + + pub(crate) fn remaining<'x>( + &'x self, + bounds: &'x [Bounds], + ) -> impl Iterator, &'x [u8])> + 'x { + bounds + .iter() + .enumerate() + .skip(self.start) + .map(|(position, (begin, end))| { + let mut begin = begin.clone(); + if position == self.start && self.resumed { + begin.push(0); + } + (begin, end.as_slice()) + }) + } +} + diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index 5206700b..6e7adc87 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -7,7 +7,15 @@ use super::{RedisPool, RedisStore, into_error}; use crate::{Deserialize, write::now}; use deadpool::managed::{Manager, Object, Pool}; -use redis::{AsyncCommands, RedisError, RedisResult, RetryMethod}; +use futures::future::try_join_all; +use redis::{ + AsyncCommands, Cmd, ErrorKind, RedisError, RedisResult, RetryMethod, Script, Value, + cluster_async::ClusterConnection, + cluster_routing::{MultipleNodeRoutingInfo, RoutingInfo, SingleNodeRoutingInfo}, + from_redis_value, +}; + +const SCAN_COUNT: usize = 100; impl RedisStore { pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option) -> trc::Result<()> { @@ -70,7 +78,105 @@ impl RedisStore { } } - pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> { + pub async fn try_lease(&self, key: &[u8], token: u64, expires: u64) -> trc::Result { + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| { + Self::try_lease_(conn, key, token, expires).await + }) + .await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| { + Self::try_lease_(conn, key, token, expires).await + }) + .await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| { + Self::try_lease_(conn, key, token, expires).await + }) + .await + } + } + } + + pub async fn renew_lease(&self, key: &[u8], token: u64, expires: u64) -> trc::Result { + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_renew, key, token, Some(expires)) + .await + }) + .await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_renew, key, token, Some(expires)) + .await + }) + .await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_renew, key, token, Some(expires)) + .await + }) + .await + } + } + } + + pub async fn release_lease(&self, key: &[u8], token: u64) -> trc::Result { + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_release, key, token, None) + .await + }) + .await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_release, key, token, None) + .await + }) + .await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| { + self.lease_script_(conn, &self.lease_release, key, token, None) + .await + }) + .await + } + } + } + + pub async fn renew_lock(&self, key: &[u8], expires: u64) -> trc::Result<()> { + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| { + Self::renew_lock_(conn, key, expires).await + }) + .await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| { + Self::renew_lock_(conn, key, expires).await + }) + .await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| { + Self::renew_lock_(conn, key, expires).await + }) + .await + } + } + } + + pub async fn key_delete(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { with_conn(pool, async |conn| Self::key_delete_(conn, key).await).await @@ -84,23 +190,126 @@ impl RedisStore { } } - pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { + pub async fn key_set_many( + &self, + items: &[(Vec, &[u8])], + expires: Option, + ) -> trc::Result<()> { + if items.is_empty() { + return Ok(()); + } match &self.pool { RedisPool::Single(pool) => { with_conn(pool, async |conn| { - Self::key_delete_prefix_(conn, prefix).await + Self::key_set_many_(conn, items, expires).await }) .await } RedisPool::Cluster(pool) => { with_conn(pool, async |conn| { - Self::key_delete_prefix_(conn, prefix).await + Self::key_set_many_(conn, items, expires).await }) .await } RedisPool::Sentinel(pool) => { with_conn(pool, async |conn| { - Self::key_delete_prefix_(conn, prefix).await + Self::key_set_many_(conn, items, expires).await + }) + .await + } + } + } + + pub async fn key_get_many(&self, keys: &[Vec]) -> trc::Result>>> { + if keys.is_empty() { + return Ok(Vec::new()); + } + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| Self::key_get_many_(conn, keys).await).await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| Self::key_get_many_(conn, keys).await).await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| Self::key_get_many_(conn, keys).await).await + } + } + } + + pub async fn key_unlink_many(&self, keys: &[Vec]) -> trc::Result<()> { + if keys.is_empty() { + return Ok(()); + } + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| Self::key_unlink_many_(conn, keys).await).await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| Self::key_unlink_many_(conn, keys).await).await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| Self::key_unlink_many_(conn, keys).await).await + } + } + } + + async fn key_set_many_( + conn: &mut impl AsyncCommands, + items: &[(Vec, &[u8])], + expires: Option, + ) -> RedisResult<()> { + let mut pipe = redis::Pipeline::with_capacity(items.len()); + for (key, value) in items { + pipe.cmd("SET").arg(key).arg(*value); + if let Some(expires) = expires { + pipe.arg("EX").arg(expires); + } + pipe.ignore(); + } + + pipe.query_async(conn).await + } + + async fn key_get_many_( + conn: &mut impl AsyncCommands, + keys: &[Vec], + ) -> RedisResult>>> { + let mut cmd = redis::cmd("MGET"); + for key in keys { + cmd.arg(key); + } + + cmd.query_async(conn).await + } + + async fn key_unlink_many_(conn: &mut impl AsyncCommands, keys: &[Vec]) -> RedisResult<()> { + let mut cmd = redis::cmd("UNLINK"); + for key in keys { + cmd.arg(key); + } + + cmd.query_async(conn).await + } + + pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { + let pattern = scan_pattern(prefix); + match &self.pool { + RedisPool::Single(pool) => { + with_conn(pool, async |conn| { + Self::key_delete_prefix_(conn, &pattern).await + }) + .await + } + RedisPool::Cluster(pool) => { + with_conn(pool, async |conn| { + Self::cluster_key_delete_prefix_(conn, &pattern).await + }) + .await + } + RedisPool::Sentinel(pool) => { + with_conn(pool, async |conn| { + Self::key_delete_prefix_(conn, &pattern).await }) .await } @@ -126,20 +335,6 @@ impl RedisStore { value.map(T::deserialize_owned).transpose() } - pub async fn counter_get(&self, key: &[u8]) -> trc::Result { - match &self.pool { - RedisPool::Single(pool) => { - with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await - } - RedisPool::Cluster(pool) => { - with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await - } - RedisPool::Sentinel(pool) => { - with_conn(pool, async |conn| Self::counter_get_(conn, key).await).await - } - } - } - pub async fn key_exists(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { @@ -158,14 +353,6 @@ impl RedisStore { redis::cmd("GET").arg(key).query_async(conn).await } - async fn counter_get_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult { - redis::cmd("GET") - .arg(key) - .query_async::>(conn) - .await - .map(|value| value.unwrap_or(0)) - } - async fn key_exists_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult { conn.exists(key).await } @@ -218,25 +405,65 @@ impl RedisStore { .map(|reply| reply.is_some()) } - async fn key_delete_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<()> { - conn.del(key).await + async fn try_lease_( + conn: &mut impl AsyncCommands, + key: &[u8], + token: u64, + expires: u64, + ) -> RedisResult { + redis::cmd("SET") + .arg(key) + .arg(token) + .arg("NX") + .arg("EX") + .arg(expires as i64) + .query_async::>(conn) + .await + .map(|reply| reply.is_some()) } - async fn key_delete_prefix_(conn: &mut impl AsyncCommands, prefix: &[u8]) -> RedisResult<()> { - let mut pattern = Vec::with_capacity(prefix.len() + 1); - pattern.extend_from_slice(prefix); - pattern.push(b'*'); + async fn lease_script_( + &self, + conn: &mut impl AsyncCommands, + script: &Script, + key: &[u8], + token: u64, + expires: Option, + ) -> RedisResult { + let mut invocation = script.key(key); + invocation.arg(token); + if let Some(expires) = expires { + invocation.arg(expires as i64); + } + invocation + .invoke_async::(conn) + .await + .map(|result| result > 0) + } + async fn renew_lock_( + conn: &mut impl AsyncCommands, + key: &[u8], + expires: u64, + ) -> RedisResult<()> { + redis::cmd("SET") + .arg(key) + .arg(now() + expires) + .arg("EX") + .arg(expires as i64) + .query_async::<()>(conn) + .await + } + + async fn key_delete_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult { + conn.del::<_, usize>(key).await.map(|deleted| deleted > 0) + } + + async fn key_delete_prefix_(conn: &mut impl AsyncCommands, pattern: &[u8]) -> RedisResult<()> { let mut cursor = 0; loop { - let (new_cursor, keys): (u64, Vec>) = redis::cmd("SCAN") - .cursor_arg(cursor) - .arg("MATCH") - .arg(&pattern) - .arg("COUNT") - .arg(100) - .query_async(conn) - .await?; + let (new_cursor, keys): (u64, Vec>) = + scan_cmd(cursor, pattern).query_async(conn).await?; if !keys.is_empty() { conn.del::<_, ()>(&keys).await?; @@ -249,6 +476,98 @@ impl RedisStore { } } } + + async fn cluster_key_delete_prefix_( + conn: &mut ClusterConnection, + pattern: &[u8], + ) -> RedisResult<()> { + let Value::Map(primaries) = conn + .route_command( + scan_cmd(0, pattern), + RoutingInfo::MultiNode((MultipleNodeRoutingInfo::AllMasters, None)), + ) + .await? + else { + return Err(RedisError::from(( + ErrorKind::UnexpectedReturnType, + "Unexpected reply to a SCAN sent to all cluster primaries", + ))); + }; + + try_join_all(primaries.into_iter().map(|(node, first_page)| { + Self::cluster_node_key_delete_prefix_(conn.clone(), pattern, node, first_page) + })) + .await?; + + Ok(()) + } + + async fn cluster_node_key_delete_prefix_( + mut conn: ClusterConnection, + pattern: &[u8], + node: Value, + first_page: Value, + ) -> RedisResult<()> { + let node = node_routing(node)?; + let (mut cursor, mut keys): (u64, Vec>) = + from_redis_value(first_page.extract_error()?)?; + loop { + if !keys.is_empty() { + Self::key_unlink_many_(&mut conn, &keys).await?; + } + + if cursor == 0 { + return Ok(()); + } + + (cursor, keys) = from_redis_value( + conn.route_command(scan_cmd(cursor, pattern), node.clone()) + .await? + .extract_error()?, + )?; + } + } +} + +fn scan_pattern(prefix: &[u8]) -> Vec { + let mut pattern = Vec::with_capacity(prefix.len() + 1); + for &byte in prefix { + if matches!(byte, b'*' | b'?' | b'[' | b'\\') { + pattern.push(b'\\'); + } + pattern.push(byte); + } + pattern.push(b'*'); + pattern +} + +fn scan_cmd(cursor: u64, pattern: &[u8]) -> Cmd { + let mut cmd = redis::cmd("SCAN"); + cmd.cursor_arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(SCAN_COUNT); + cmd +} + +fn node_routing(node: Value) -> RedisResult { + let node: String = from_redis_value(node)?; + let Some((host, port)) = node + .rsplit_once(':') + .and_then(|(host, port)| Some((host, port.parse().ok()?))) + else { + return Err(RedisError::from(( + ErrorKind::UnexpectedReturnType, + "Invalid cluster node address", + node, + ))); + }; + + Ok(RoutingInfo::SingleNode(SingleNodeRoutingInfo::ByAddress { + host: host.to_string(), + port, + })) } async fn with_conn( @@ -263,7 +582,7 @@ where match operation(conn.as_mut()).await { Ok(value) => Ok(value), Err(err) => { - if is_stale_connection(&err) { + if is_stale_connection(&err) || err.is_timeout() { drop(Object::take(conn)); } Err(into_error(err)) diff --git a/crates/store/src/backend/redis/mod.rs b/crates/store/src/backend/redis/mod.rs index 94835a97..2f1fc2ca 100644 --- a/crates/store/src/backend/redis/mod.rs +++ b/crates/store/src/backend/redis/mod.rs @@ -7,10 +7,10 @@ use crate::InMemoryStore; use deadpool::{ Runtime, - managed::{Manager, Pool}, + managed::{Manager, Pool, QueueMode}, }; use redis::{ - Client, ConnectionAddr, IntoConnectionInfo, ProtocolVersion, Script, TlsMode, + Client, ConnectionAddr, ConnectionInfo, IntoConnectionInfo, ProtocolVersion, Script, TlsMode, cluster::{ClusterClient, ClusterClientBuilder}, cluster_read_routing::RandomReplicaStrategy, sentinel::{SentinelClient, SentinelClientBuilder, SentinelServerType}, @@ -24,10 +24,14 @@ use std::{fmt::Display, sync::Arc}; pub mod lookup; pub mod pool; +const CLUSTER_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + #[derive(Debug)] pub struct RedisStore { pub pool: RedisPool, incr_expire: Script, + lease_renew: Script, + lease_release: Script, } pub struct RedisConnectionManager { @@ -60,6 +64,18 @@ impl RedisStore { redis.call('EXPIRE', KEYS[1], ARGV[2]) return redis.call('GET', KEYS[1])", ), + lease_renew: Script::new( + "if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], ARGV[2]) + end + return 0", + ), + lease_release: Script::new( + "if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0", + ), } } @@ -67,9 +83,9 @@ impl RedisStore { Ok(InMemoryStore::Redis(Arc::new(RedisStore::new( RedisPool::Single(build_pool( RedisConnectionManager { - client: Client::open(config.url) + client: Client::open(connection_info(config.url, config.allow_invalid_certs)?) .map_err(|err| format!("Failed to open Redis client: {err:?}"))?, - timeout: config.timeout.into_inner(), + timeout: config.connect_timeout.into_inner(), }, config.pool_max_connections, config.pool_timeout_create, @@ -80,7 +96,12 @@ impl RedisStore { } pub async fn open_cluster(config: structs::RedisClusterStore) -> Result { - let mut builder = ClusterClientBuilder::new(config.urls); + let mut nodes = Vec::with_capacity(config.urls.len()); + for url in config.urls { + nodes.push(connection_info(url, config.allow_invalid_certs)?); + } + let mut builder = + ClusterClientBuilder::new(nodes).response_timeout(CLUSTER_RESPONSE_TIMEOUT); if let Some(value) = config.auth_username { builder = builder.username(value); } @@ -111,7 +132,7 @@ impl RedisStore { RedisPool::Cluster(build_pool( RedisClusterConnectionManager { client, - timeout: config.timeout.into_inner(), + timeout: config.connect_timeout.into_inner(), }, config.pool_max_connections, config.pool_timeout_create, @@ -127,9 +148,7 @@ impl RedisStore { let mut sentinels = Vec::with_capacity(config.urls.len()); let mut tls_mode = None; for url in config.urls { - let info = url - .into_connection_info() - .map_err(|err| format!("Invalid Redis Sentinel URL: {err}"))?; + let info = connection_info(url, config.allow_invalid_certs)?; let url_tls_mode = match info.addr() { ConnectionAddr::TcpTls { insecure: true, .. } => Some(TlsMode::Insecure), ConnectionAddr::TcpTls { @@ -183,7 +202,7 @@ impl RedisStore { RedisPool::Sentinel(build_pool( RedisSentinelConnectionManager { client: tokio::sync::Mutex::new(client), - timeout: config.timeout.into_inner(), + timeout: config.connect_timeout.into_inner(), }, config.pool_max_connections, config.pool_timeout_create, @@ -194,6 +213,20 @@ impl RedisStore { } } +fn connection_info(url: String, allow_invalid_certs: bool) -> Result { + let info = url + .into_connection_info() + .map_err(|err| format!("Invalid Redis URL: {err}"))?; + let mut addr = info.addr().clone(); + match &mut addr { + ConnectionAddr::TcpTls { insecure, .. } if allow_invalid_certs && !*insecure => { + *insecure = true; + Ok(info.set_addr(addr)) + } + _ => Ok(info), + } +} + fn build_pool( manager: M, max_size: u64, @@ -203,6 +236,7 @@ fn build_pool( ) -> Result, String> { Pool::builder(manager) .runtime(Runtime::Tokio1) + .queue_mode(QueueMode::Lifo) .max_size(max_size as usize) .create_timeout(create_timeout.map(|v| v.into_inner())) .wait_timeout(wait_timeout.map(|v| v.into_inner())) diff --git a/crates/store/src/backend/redis/pool.rs b/crates/store/src/backend/redis/pool.rs index 341b630e..a5a811a3 100644 --- a/crates/store/src/backend/redis/pool.rs +++ b/crates/store/src/backend/redis/pool.rs @@ -13,14 +13,20 @@ use redis::{ aio::{ConnectionLike, MultiplexedConnection}, cluster_async::ClusterConnection, }; +use std::time::Duration; + +const PING_IF_IDLE_FOR: Duration = Duration::from_secs(5); impl managed::Manager for RedisConnectionManager { type Type = MultiplexedConnection; type Error = trc::Error; async fn create(&self) -> Result { - match tokio::time::timeout(self.timeout, self.client.get_multiplexed_async_connection()) - .await + match tokio::time::timeout( + self.timeout, + Box::pin(self.client.get_multiplexed_async_connection()), + ) + .await { Ok(conn) => conn.map_err(into_error), Err(_) => Err(trc::StoreEvent::RedisError.ctx(trc::Key::Details, "Connection Timeout")), @@ -30,12 +36,9 @@ impl managed::Manager for RedisConnectionManager { async fn recycle( &self, conn: &mut MultiplexedConnection, - _: &managed::Metrics, + metrics: &managed::Metrics, ) -> managed::RecycleResult { - conn.req_packed_command(&redis::cmd("PING")) - .await - .map(|_| ()) - .map_err(|err| managed::RecycleError::Backend(into_error(err))) + ping_if_idle(conn, metrics).await } } @@ -53,12 +56,9 @@ impl managed::Manager for RedisClusterConnectionManager { async fn recycle( &self, conn: &mut ClusterConnection, - _: &managed::Metrics, + metrics: &managed::Metrics, ) -> managed::RecycleResult { - conn.req_packed_command(&redis::cmd("PING")) - .await - .map(|_| ()) - .map_err(|err| managed::RecycleError::Backend(into_error(err))) + ping_if_idle(conn, metrics).await } } @@ -68,7 +68,7 @@ impl managed::Manager for RedisSentinelConnectionManager { async fn create(&self) -> Result { let mut client = self.client.lock().await; - match tokio::time::timeout(self.timeout, client.get_async_connection()).await { + match tokio::time::timeout(self.timeout, Box::pin(client.get_async_connection())).await { Ok(conn) => conn.map_err(into_error), Err(_) => Err(trc::StoreEvent::RedisError.ctx(trc::Key::Details, "Connection Timeout")), } @@ -77,11 +77,22 @@ impl managed::Manager for RedisSentinelConnectionManager { async fn recycle( &self, conn: &mut MultiplexedConnection, - _: &managed::Metrics, + metrics: &managed::Metrics, ) -> managed::RecycleResult { - conn.req_packed_command(&redis::cmd("PING")) - .await - .map(|_| ()) - .map_err(|err| managed::RecycleError::Backend(into_error(err))) + ping_if_idle(conn, metrics).await } } + +async fn ping_if_idle( + conn: &mut impl ConnectionLike, + metrics: &managed::Metrics, +) -> managed::RecycleResult { + if metrics.last_used() < PING_IF_IDLE_FOR { + return Ok(()); + } + conn.req_packed_command(&redis::cmd("PING")) + .await + .map(|_| ()) + .map_err(|err| managed::RecycleError::Backend(into_error(err))) +} + diff --git a/crates/store/src/backend/rocksdb/blob.rs b/crates/store/src/backend/rocksdb/blob.rs index 5a7f8722..02db6508 100644 --- a/crates/store/src/backend/rocksdb/blob.rs +++ b/crates/store/src/backend/rocksdb/blob.rs @@ -7,16 +7,19 @@ use std::ops::Range; use super::{CF_BLOBS, RocksDbStore, into_error}; +use rocksdb::WriteBatchWithTransaction; impl RocksDbStore { - pub(crate) async fn get_blob( + pub(crate) fn get_blob( &self, key: &[u8], range: Range, - ) -> trc::Result>> { + ) -> impl Future>>> { let db = self.db.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { - db.get_pinned_cf(&db.cf_handle(CF_BLOBS).unwrap(), key) + db.get_pinned_cf(&db.cf_handle(CF_BLOBS).unwrap(), &key) .map(|obj| { obj.map(|bytes| { if range.start == 0 && range.end == usize::MAX { @@ -31,25 +34,49 @@ impl RocksDbStore { }) .map_err(into_error) }) - .await } - pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { + pub(crate) fn put_blob( + &self, + key: &[u8], + data: Vec, + ) -> impl Future> { let db = self.db.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { - db.put_cf(&db.cf_handle(CF_BLOBS).unwrap(), key, data) + db.put_cf(&db.cf_handle(CF_BLOBS).unwrap(), &key, &data) .map_err(into_error) }) - .await } - pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { + pub(crate) fn delete_blob(&self, key: &[u8]) -> impl Future> { let db = self.db.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { - db.delete_cf(&db.cf_handle(CF_BLOBS).unwrap(), key) + db.delete_cf(&db.cf_handle(CF_BLOBS).unwrap(), &key) .map_err(into_error) .map(|_| true) }) - .await + } + + pub(crate) async fn delete_blobs(&self, keys: &[&[u8]]) -> trc::Result<()> { + let db = self.db.clone(); + let mut batch = WriteBatchWithTransaction::::default(); + { + let cf = db.cf_handle(CF_BLOBS).ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("Blob column family not found") + .caused_by(trc::location!()) + })?; + for key in keys { + batch.delete_cf(&cf, key); + } + } + + self.spawn_worker(move || db.write(batch).map_err(into_error)) + .await } } diff --git a/crates/store/src/backend/rocksdb/blob_gc.rs b/crates/store/src/backend/rocksdb/blob_gc.rs new file mode 100644 index 00000000..1b0677f2 --- /dev/null +++ b/crates/store/src/backend/rocksdb/blob_gc.rs @@ -0,0 +1,229 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CfHandle, RocksDbStore, into_error}; +use crate::Subspace; +use rocksdb::{ + BottommostLevelCompaction, BoundColumnFamily, CompactOptions, MultiThreaded, + OptimisticTransactionDB, +}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; +use tokio::sync::oneshot; + +const BLOB_GC_GARBAGE_RATIO: u64 = 2; +const SETTLE_BACKOFF: u64 = 2; +const MIN_BLOB_GC_BYTES: u64 = 32 * 1024 * 1024; +const BLOB_GC_SUBSPACES: [Subspace; 2] = [Subspace::Blobs, Subspace::Immutable]; +const LIVE_BLOB_BYTES: &str = "rocksdb.live-blob-file-size"; +const BLOB_GARBAGE_BYTES: &str = "rocksdb.live-blob-file-garbage-size"; +const TABLE_PROPERTIES: &str = "rocksdb.aggregated-table-properties"; +const TABLE_ENTRIES: &str = "# entries="; +const TABLE_DELETIONS: &str = "# deletions="; +const TABLE_PROPERTY_DELIMITER: &str = "; "; +const MEMTABLE_ENTRIES: [&str; 2] = [ + "rocksdb.num-entries-active-mem-table", + "rocksdb.num-entries-imm-mem-tables", +]; +const MEMTABLE_DELETIONS: [&str; 2] = [ + "rocksdb.num-deletes-active-mem-table", + "rocksdb.num-deletes-imm-mem-tables", +]; +const ENABLE_BLOB_GC: [(&str, &str); 2] = [ + ("enable_blob_garbage_collection", "true"), + ("blob_garbage_collection_age_cutoff", "1.0"), +]; +const DISABLE_BLOB_GC: [(&str, &str); 1] = [("enable_blob_garbage_collection", "false")]; +const BLOB_GC_THREAD: &str = "rocksdb-blob-gc"; + +type Db = OptimisticTransactionDB; +type Cf<'db> = Arc>; +type Backoffs = [SettleBackoff; BLOB_GC_SUBSPACES.len()]; + +#[derive(Default)] +pub(super) struct BlobGcState { + floors: [AtomicU64; BLOB_GC_SUBSPACES.len()], + seen: [AtomicU64; BLOB_GC_SUBSPACES.len()], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct SettleBackoff { + floor: u64, + seen: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BlobBytes { + total: u64, + garbage: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PendingDeletes { + entries: u64, + deletions: u64, +} + +impl RocksDbStore { + pub(crate) async fn collect_blob_garbage(&self) -> trc::Result<()> { + let db = self.db.clone(); + let backoffs = self.blob_gc.load(); + let (tx, rx) = oneshot::channel(); + std::thread::Builder::new() + .name(BLOB_GC_THREAD.into()) + .spawn(move || { + tx.send(run_pass(&db, backoffs)).ok(); + }) + .map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?; + let backoffs = rx + .await + .map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))??; + self.blob_gc.store(backoffs); + Ok(()) + } +} + +impl BlobGcState { + fn load(&self) -> Backoffs { + std::array::from_fn(|family| SettleBackoff { + floor: self.floors[family].load(Ordering::Relaxed), + seen: self.seen[family].load(Ordering::Relaxed), + }) + } + + fn store(&self, backoffs: Backoffs) { + for ((floor, seen), backoff) in self.floors.iter().zip(&self.seen).zip(backoffs) { + floor.store(backoff.floor, Ordering::Relaxed); + seen.store(backoff.seen, Ordering::Relaxed); + } + } +} + +impl SettleBackoff { + fn observe(&mut self, deletions: u64) { + if deletions <= self.seen { + self.floor /= SETTLE_BACKOFF; + } + self.seen = deletions; + } + + fn allows(self, deletions: u64) -> bool { + deletions >= self.floor + } + + fn after_settle(pending: u64, settled: u64) -> Self { + Self { + floor: pending.saturating_mul(SETTLE_BACKOFF), + seen: settled, + } + } +} + +fn run_pass(db: &Db, mut backoffs: Backoffs) -> trc::Result { + let mut opts = CompactOptions::default(); + opts.set_bottommost_level_compaction(BottommostLevelCompaction::ForceOptimized); + + for (subspace, backoff) in BLOB_GC_SUBSPACES.into_iter().zip(&mut backoffs) { + let cf = db.subspace_handle(subspace); + let mut bytes = BlobBytes::read(db, &cf)?; + if bytes.total < MIN_BLOB_GC_BYTES { + continue; + } + + if !bytes.over_threshold() { + let pending = PendingDeletes::read(db, &cf)?; + backoff.observe(pending.deletions); + if !backoff.allows(pending.deletions) || !bytes.with_pending(pending).over_threshold() { + continue; + } + db.compact_range_cf_opt(&cf, None::<&[u8]>, None::<&[u8]>, &opts); + bytes = BlobBytes::read(db, &cf)?; + if bytes.total < MIN_BLOB_GC_BYTES || !bytes.over_threshold() { + *backoff = SettleBackoff::after_settle( + pending.deletions, + PendingDeletes::read(db, &cf)?.deletions, + ); + continue; + } + } + + db.set_options_cf(&cf, &ENABLE_BLOB_GC) + .map_err(into_error)?; + db.compact_range_cf_opt(&cf, None::<&[u8]>, None::<&[u8]>, &opts); + db.set_options_cf(&cf, &DISABLE_BLOB_GC) + .map_err(into_error)?; + *backoff = SettleBackoff::default(); + } + + Ok(backoffs) +} + +impl BlobBytes { + fn read(db: &Db, cf: &Cf<'_>) -> trc::Result { + Ok(Self { + total: int_property(db, cf, LIVE_BLOB_BYTES)?, + garbage: int_property(db, cf, BLOB_GARBAGE_BYTES)?, + }) + } + + fn over_threshold(self) -> bool { + self.garbage.saturating_mul(BLOB_GC_GARBAGE_RATIO) >= self.total + } + + fn with_pending(self, pending: PendingDeletes) -> Self { + let puts = pending.entries.saturating_sub(pending.deletions); + let unaccounted = self.total.saturating_sub(self.garbage); + let pending_garbage = match puts { + 0 => unaccounted, + puts => u64::try_from( + u128::from(unaccounted) * u128::from(pending.deletions) / u128::from(puts), + ) + .unwrap_or(u64::MAX) + .min(unaccounted), + }; + Self { + total: self.total, + garbage: self.garbage + pending_garbage, + } + } +} + +impl PendingDeletes { + fn read(db: &Db, cf: &Cf<'_>) -> trc::Result { + let tables = db + .property_value_cf(cf, TABLE_PROPERTIES) + .map_err(into_error)? + .unwrap_or_default(); + let mut pending = Self::parse(&tables); + for (entries, deletions) in MEMTABLE_ENTRIES.into_iter().zip(MEMTABLE_DELETIONS) { + pending.entries += int_property(db, cf, entries)?; + pending.deletions += int_property(db, cf, deletions)?; + } + Ok(pending) + } + + fn parse(tables: &str) -> Self { + let property = |name: &str| { + tables + .split(TABLE_PROPERTY_DELIMITER) + .find_map(|property| property.strip_prefix(name)?.parse().ok()) + .unwrap_or(0) + }; + Self { + entries: property(TABLE_ENTRIES), + deletions: property(TABLE_DELETIONS), + } + } +} + +fn int_property(db: &Db, cf: &Cf<'_>, name: &str) -> trc::Result { + db.property_int_value_cf(cf, name) + .map(Option::unwrap_or_default) + .map_err(into_error) +} + diff --git a/crates/store/src/backend/rocksdb/cache.rs b/crates/store/src/backend/rocksdb/cache.rs new file mode 100644 index 00000000..3b11afeb --- /dev/null +++ b/crates/store/src/backend/rocksdb/cache.rs @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use rocksdb::{Cache, LruCacheOptions}; +use utils::memory::available_memory; + +const MIN_CACHE_SHARD: usize = 8 * 1024 * 1024; +const MAX_CACHE_SHARD_BITS: u32 = 6; +const MIN_DEFAULT_CACHE_SIZE: u64 = 256 * 1024 * 1024; +const MAX_DEFAULT_CACHE_SIZE: u64 = 8 * 1024 * 1024 * 1024; +const DEFAULT_CACHE_MEMORY_SHARE: u64 = 8; +const MAX_FLOOR_MEMORY_SHARE: u64 = 4; + +pub(super) fn block_cache(cache_size: Option) -> Cache { + let capacity = + usize::try_from(cache_size.unwrap_or_else(default_cache_size)).unwrap_or(usize::MAX); + let mut opts = LruCacheOptions::default(); + opts.set_capacity(capacity); + opts.set_num_shard_bits(shard_bits(capacity)); + Cache::new_lru_cache_opts(&opts) +} + +fn shard_bits(capacity: usize) -> i32 { + (capacity / MIN_CACHE_SHARD) + .max(1) + .ilog2() + .min(MAX_CACHE_SHARD_BITS) as i32 +} + +fn default_cache_size() -> u64 { + cache_size_for(available_memory()) +} + +fn cache_size_for(memory: Option) -> u64 { + memory.map_or(MIN_DEFAULT_CACHE_SIZE, |memory| { + (memory / DEFAULT_CACHE_MEMORY_SHARE).clamp( + MIN_DEFAULT_CACHE_SIZE.min(memory / MAX_FLOOR_MEMORY_SHARE), + MAX_DEFAULT_CACHE_SIZE, + ) + }) +} + diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 1b5d1517..54b3d42b 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -4,15 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{CF_BLOBS, RocksDbStore}; -use crate::*; +use super::{CfHandle, RocksDbStore, cache::block_cache}; +use crate::{ + backend::worker::{self, Access, BlockingPermits}, + *, +}; use ::registry::schema::structs; use rocksdb::{ - BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBCompressionType, MergeOperands, + BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBCompressionType, MultiThreaded, OptimisticTransactionDB, Options, }; use std::path::PathBuf; -use tokio::sync::oneshot; const MIN_WRITE_BUFFER_SIZE: usize = 4 * 1024 * 1024; const MAX_WRITE_BUFFER_SIZE: usize = 64 * 1024 * 1024; @@ -24,23 +26,76 @@ const CHURN_DELETION_WINDOW: usize = 4096; const CHURN_DELETION_TRIGGER: usize = 1024; const CHURN_DELETION_RATIO: f64 = 0.5; const BYTES_PER_SYNC: u64 = 1024 * 1024; +const COLD_MIN_BLOB_SIZE: u64 = 16 * 1024; +const LOGS_PERIODIC_COMPACTION: u64 = 24 * 60 * 60; const MAX_LOG_FILE_SIZE: usize = 10 * 1024 * 1024; const KEEP_LOG_FILE_NUM: usize = 5; +const SCAN_FLUSH_SUBSPACES: [Subspace; 3] = [ + Subspace::QueueEvent, + Subspace::SearchQueue, + Subspace::TaskQueue, +]; +const SCAN_FLUSH_OPTIONS: [(&str, &str); 2] = [ + ("memtable_op_scan_flush_trigger", "10000"), + ("memtable_avg_op_scan_flush_trigger", "1000"), +]; #[derive(Clone, Copy)] enum CfProfile { /// Read through `get_value` / `key_exists`, so a whole key bloom filter pays off. PointLookup, - /// Read only through `iterate`, which never consults a whole key bloom filter. + /// Read through `iterate`, which never consults a whole key bloom filter, and point + /// read only on administrative paths, so no bloom filter is built. Scan, + /// Scanned like `Scan` and also point read per message or per command (`get_value`, or + /// the GetForUpdate behind `merge_fnc`), so a whole key bloom filter is built. + ScanLookup, /// Point read and point deleted at a high rate. Churn, + HashedChurn, /// Scanned from the oldest key and point deleted once consumed, with empty values. Queue, - /// Counters updated through the merge operator. + /// Counters, read for update and rewritten by every commit that changes them. Counter, /// Blob values held in RocksDB blob files. Blob, + /// Write-once values, large ones offloaded to RocksDB blob files. + Cold, +} + +trait RocksDbProfile { + fn profile(self) -> CfProfile; +} + +impl RocksDbProfile for Subspace { + fn profile(self) -> CfProfile { + match self { + Subspace::Counter => CfProfile::Counter, + Subspace::Blobs => CfProfile::Blob, + Subspace::Immutable => CfProfile::Cold, + Subspace::Indexes + | Subspace::Acl + | Subspace::Logs + | Subspace::TelemetryMetric + | Subspace::RegistryIndex => CfProfile::Scan, + Subspace::SearchTerm | Subspace::IndexProperty => CfProfile::ScanLookup, + Subspace::BlobLink => CfProfile::HashedChurn, + Subspace::TaskQueue + | Subspace::DeletedItems + | Subspace::QueueMessage + | Subspace::ReportOut + | Subspace::ReportIn + | Subspace::SpamSamples => CfProfile::Churn, + Subspace::Property + | Subspace::Registry + | Subspace::TelemetrySpan + | Subspace::RegistryPrimaryKey + | Subspace::Directory + | Subspace::SearchDocument + | Subspace::System => CfProfile::PointLookup, + Subspace::QueueEvent | Subspace::SearchQueue => CfProfile::Queue, + } + } } impl RocksDbStore { @@ -55,56 +110,35 @@ impl RocksDbStore { ) })?; - let cache = Cache::new_lru_cache(config.cache_size as usize); + let cache = block_cache(config.cache_size); let write_buffer_size = ((config.buffer_size as usize) / 4).clamp(MIN_WRITE_BUFFER_SIZE, MAX_WRITE_BUFFER_SIZE); let mut cfs = Vec::new(); - // Counters - for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { - cfs.push(ColumnFamilyDescriptor::new( - std::str::from_utf8(&[subspace]).unwrap(), - cf_options(CfProfile::Counter, &cache, write_buffer_size), - )); - } + for subspace in Subspace::ALL.iter().copied() { + let profile = subspace.profile(); + let mut cf_opts = cf_options(profile, &cache, write_buffer_size); - // Blobs - let mut cf_opts = cf_options(CfProfile::Blob, &cache, write_buffer_size); - cf_opts.set_enable_blob_files(true); - cf_opts.set_min_blob_size(config.blob_size); - cf_opts.set_enable_blob_gc(false); - cf_opts.set_blob_gc_age_cutoff(1.0); - cfs.push(ColumnFamilyDescriptor::new(CF_BLOBS, cf_opts)); + if let Some(min_blob_size) = match profile { + CfProfile::Blob => Some(config.blob_size), + CfProfile::Cold => Some(COLD_MIN_BLOB_SIZE), + _ => None, + } { + cf_opts.set_enable_blob_files(true); + cf_opts.set_min_blob_size(min_blob_size); + cf_opts.set_blob_cache(&cache); + cf_opts.set_enable_blob_gc(false); - // Other cfs - for (subspace, profile) in [ - (SUBSPACE_INDEXES, CfProfile::Scan), - (SUBSPACE_ACL, CfProfile::Scan), - (SUBSPACE_TASK_QUEUE, CfProfile::Churn), - (SUBSPACE_DELETED_ITEMS, CfProfile::Churn), - (SUBSPACE_BLOB_LINK, CfProfile::Churn), - (SUBSPACE_IN_MEMORY_VALUE, CfProfile::Churn), - (SUBSPACE_PROPERTY, CfProfile::PointLookup), - (SUBSPACE_REGISTRY, CfProfile::PointLookup), - (SUBSPACE_QUEUE_MESSAGE, CfProfile::Churn), - (SUBSPACE_QUEUE_EVENT, CfProfile::Queue), - (SUBSPACE_REPORT_OUT, CfProfile::Churn), - (SUBSPACE_REPORT_IN, CfProfile::Churn), - (SUBSPACE_LOGS, CfProfile::Scan), - (SUBSPACE_TELEMETRY_SPAN, CfProfile::PointLookup), - (SUBSPACE_TELEMETRY_METRIC, CfProfile::Scan), - (SUBSPACE_SEARCH_INDEX, CfProfile::Scan), - (SUBSPACE_SPAM_SAMPLES, CfProfile::Churn), - (SUBSPACE_REGISTRY_IDX, CfProfile::Scan), - (SUBSPACE_REGISTRY_PK, CfProfile::PointLookup), - (SUBSPACE_DIRECTORY, CfProfile::PointLookup), - (LEGACY_SUBSPACE_BITMAP_TEXT, CfProfile::Scan), - (LEGACY_SUBSPACE_BITMAP_TAG, CfProfile::Scan), - ] { - cfs.push(ColumnFamilyDescriptor::new( - std::str::from_utf8(&[subspace]).unwrap(), - cf_options(profile, &cache, write_buffer_size), - )); + if matches!(profile, CfProfile::Cold) { + cf_opts.set_blob_compression_type(DBCompressionType::Lz4); + } + } + + if matches!(subspace, Subspace::Logs) { + cf_opts.set_periodic_compaction_seconds(LOGS_PERIODIC_COMPACTION); + } + + cfs.push(ColumnFamilyDescriptor::new(subspace.name(), cf_opts)); } let mut db_opts = Options::default(); @@ -119,62 +153,59 @@ impl RocksDbStore { db_opts.set_max_log_file_size(MAX_LOG_FILE_SIZE); db_opts.set_keep_log_file_num(KEEP_LOG_FILE_NUM); + #[cfg(stalwart_tests)] + if super::probe::COLLECT_STATISTICS.load(std::sync::atomic::Ordering::Relaxed) { + db_opts.enable_statistics(); + } + + let pool_workers = worker::pool_workers(config.pool_workers); + + let db: OptimisticTransactionDB = + OptimisticTransactionDB::open_cf_descriptors(&db_opts, idx_path, cfs) + .map_err(|err| format!("Failed to open database: {:?}", err))?; + + for subspace in SCAN_FLUSH_SUBSPACES { + db.set_options_cf(&db.subspace_handle(subspace), &SCAN_FLUSH_OPTIONS) + .map_err(|err| format!("Failed to set {} options: {:?}", subspace.name(), err))?; + } + Ok(Store::RocksDb(Arc::new(RocksDbStore { - db: OptimisticTransactionDB::open_cf_descriptors(&db_opts, idx_path, cfs) - .map_err(|err| format!("Failed to open database: {:?}", err))? - .into(), + db: Arc::new(db), worker_pool: rayon::ThreadPoolBuilder::new() - .num_threads(std::cmp::max( - config - .pool_workers - .filter(|v| *v > 0) - .map(|v| v as usize) - .unwrap_or_else(num_cpus::get), - 4, - )) + .num_threads(pool_workers) .build() .map_err(|err| format!("Failed to build worker pool: {:?}", err))?, + blocking: BlockingPermits::new(pool_workers), + blob_gc: Default::default(), + counter_slots: super::slots::CounterSlots::new(pool_workers), + #[cfg(stalwart_tests)] + probe: Default::default(), }))) } - pub async fn spawn_worker(&self, mut f: U) -> trc::Result + pub(crate) fn spawn_worker(&self, f: U) -> impl Future> + where + U: FnOnce() -> trc::Result + Send + 'static, + V: Send + 'static, + { + worker::spawn(&self.worker_pool, f) + } + + pub(crate) fn block_reader(&self, f: U) -> impl Future> where U: FnMut() -> trc::Result + Send, - V: Sync + Send + 'static, + V: Send, { - let (tx, rx) = oneshot::channel(); - - self.worker_pool.scope(|s| { - s.spawn(|_| { - tx.send(f()).ok(); - }); - }); - - match rx.await { - Ok(result) => result, - Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err)), - } - } -} - -pub fn numeric_value_merge( - _key: &[u8], - value: Option<&[u8]>, - operands: &MergeOperands, -) -> Option> { - let mut value = if let Some(value) = value { - i64::from_le_bytes(value.try_into().ok()?) - } else { - 0 - }; - - for op in operands.iter() { - value += i64::from_le_bytes(op.try_into().ok()?); + self.blocking.block(Access::Read, f) } - let mut bytes = Vec::with_capacity(std::mem::size_of::()); - bytes.extend_from_slice(&value.to_le_bytes()); - Some(bytes) + pub(crate) fn block_writer(&self, f: U) -> impl Future> + where + U: FnMut() -> trc::Result + Send, + V: Send, + { + self.blocking.block(Access::Write, f) + } } fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Options { @@ -187,17 +218,24 @@ fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Op opts.set_write_buffer_size(write_buffer_size); opts.set_max_write_buffer_number(4); + if !matches!(profile, CfProfile::Scan | CfProfile::Queue) { + block_opts.set_bloom_filter(BLOOM_BITS_PER_KEY, false); + } + match profile { CfProfile::PointLookup => { - block_opts.set_bloom_filter(BLOOM_BITS_PER_KEY, false); opts.set_compression_type(DBCompressionType::Lz4); } - CfProfile::Scan => { + CfProfile::Scan | CfProfile::ScanLookup => { block_opts.set_block_size(SCAN_BLOCK_SIZE); opts.set_compression_type(DBCompressionType::Lz4); + opts.add_compact_on_deletion_collector_factory( + CHURN_DELETION_WINDOW, + CHURN_DELETION_TRIGGER, + CHURN_DELETION_RATIO, + ); } CfProfile::Churn => { - block_opts.set_bloom_filter(BLOOM_BITS_PER_KEY, false); opts.set_compression_type(DBCompressionType::Lz4); opts.set_target_file_size_base(CHURN_TARGET_FILE_SIZE); opts.add_compact_on_deletion_collector_factory( @@ -206,6 +244,10 @@ fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Op CHURN_DELETION_RATIO, ); } + CfProfile::HashedChurn => { + opts.set_compression_type(DBCompressionType::Lz4); + opts.set_target_file_size_base(CHURN_TARGET_FILE_SIZE); + } CfProfile::Queue => { block_opts.set_block_size(SCAN_BLOCK_SIZE); opts.set_compression_type(DBCompressionType::None); @@ -217,14 +259,14 @@ fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Op ); } CfProfile::Counter => { - block_opts.set_bloom_filter(BLOOM_BITS_PER_KEY, false); opts.set_compression_type(DBCompressionType::None); - opts.set_merge_operator_associative("merge", numeric_value_merge); } CfProfile::Blob => { - block_opts.set_bloom_filter(BLOOM_BITS_PER_KEY, false); opts.set_compression_type(DBCompressionType::None); } + CfProfile::Cold => { + opts.set_compression_type(DBCompressionType::Lz4); + } } opts.set_block_based_table_factory(&block_opts); diff --git a/crates/store/src/backend/rocksdb/mod.rs b/crates/store/src/backend/rocksdb/mod.rs index ed409815..de67509c 100644 --- a/crates/store/src/backend/rocksdb/mod.rs +++ b/crates/store/src/backend/rocksdb/mod.rs @@ -4,37 +4,62 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{Subspace, backend::worker::BlockingPermits}; +use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB}; use std::sync::Arc; -use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB}; - -use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS}; - pub mod blob; +mod blob_gc; +mod cache; pub mod main; pub mod read; +mod slots; pub mod write; -static CF_LOGS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_LOGS]) }; -static CF_INDEXES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEXES]) }; -static CF_BLOBS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOBS]) }; +static CF_BLOBS: &str = Subspace::Blobs.name(); pub(crate) trait CfHandle { - fn subspace_handle(&self, subspace: u8) -> Arc>; + fn subspace_handle(&self, subspace: Subspace) -> Arc>; } impl CfHandle for OptimisticTransactionDB { #[inline(always)] - fn subspace_handle(&self, subspace: u8) -> Arc> { - let subspace = &[subspace]; - self.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace) }) - .unwrap() + fn subspace_handle(&self, subspace: Subspace) -> Arc> { + self.cf_handle(subspace.name()).unwrap() + } +} + +const CF_SLOTS: usize = u8::MAX as usize + 1; + +pub(crate) struct CfCache<'db> { + db: &'db OptimisticTransactionDB, + handles: [Option>>; CF_SLOTS], +} + +impl<'db> CfCache<'db> { + pub(crate) fn new(db: &'db OptimisticTransactionDB) -> Self { + Self { + db, + handles: [const { None }; CF_SLOTS], + } + } + + #[inline(always)] + pub(crate) fn get(&mut self, subspace: Subspace) -> &Arc> { + let db = self.db; + + self.handles[subspace.byte() as usize].get_or_insert_with(|| db.subspace_handle(subspace)) } } pub struct RocksDbStore { db: Arc>, worker_pool: rayon::ThreadPool, + blocking: BlockingPermits, + blob_gc: blob_gc::BlobGcState, + counter_slots: slots::CounterSlots, + #[cfg(stalwart_tests)] + probe: probe::RocksDbProbe, } #[inline(always)] diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index 4f1c1fe2..1980741c 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -4,62 +4,66 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{RocksDbStore, into_error}; +use super::{ + RocksDbStore, into_error, + slots::{queue_quota_read_options, sum_slots}, +}; use crate::{ - Deserialize, IterateParams, Key, ValueKey, backend::rocksdb::CfHandle, write::ValueClass, + Deserialize, Fetch, IterateParams, Key, ValueKey, backend::rocksdb::CfHandle, write::ValueClass, }; use rocksdb::ReadOptions; +use std::{cmp::Ordering, mem}; + +const MULTI_GET_CHUNK: usize = 256; impl RocksDbStore { - pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> + pub(crate) fn get_value(&self, key: impl Key) -> impl Future>> where U: Deserialize + 'static, { let db = self.db.clone(); + let subspace = key.subspace(); + let key = key.serialize(0); + self.spawn_worker(move || { - let subspace = &[key.subspace()]; - let key = key.serialize(0); - db.get_pinned_cf( - &db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) }) - .unwrap(), - &key, - ) - .map_err(into_error) - .and_then(|value| { - if let Some(value) = value { - U::deserialize_with_key(&key, &value).map(Some) - } else { - Ok(None) - } - }) + let cf = db.subspace_handle(subspace); + db.get_pinned_cf(&cf, &key) + .map_err(into_error) + .and_then(|value| { + if let Some(value) = value { + U::deserialize_with_key(&key, &value).map(Some) + } else { + Ok(None) + } + }) }) - .await } - pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result { + pub(crate) fn key_exists(&self, key: impl Key) -> impl Future> { let db = self.db.clone(); + let subspace = key.subspace(); + let key = key.serialize(0); + self.spawn_worker(move || { - let subspace = &[key.subspace()]; - let key = key.serialize(0); - db.get_pinned_cf( - &db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) }) - .unwrap(), - &key, - ) - .map_err(into_error) - .map(|value| value.is_some()) + let cf = db.subspace_handle(subspace); + db.get_pinned_cf(&cf, &key) + .map_err(into_error) + .map(|value| value.is_some()) }) - .await } - pub(crate) async fn iterate( + pub(crate) fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, - ) -> trc::Result<()> { + ) -> impl Future> { + let mut remaining = match params.fetch { + Fetch::Limit(rows) => u64::from(rows), + Fetch::All | Fetch::Expect(_) => u64::MAX, + }; let db = self.db.clone(); - self.spawn_worker(move || { + self.block_reader(move || { let cf = db.subspace_handle(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); @@ -71,6 +75,7 @@ impl RocksDbStore { let mut read_opts = ReadOptions::default(); read_opts.set_iterate_lower_bound(begin.as_slice()); read_opts.set_iterate_upper_bound(upper_bound); + read_opts.set_auto_readahead_size(true); let mut it = db.raw_iterator_cf_opt(&cf, read_opts); if params.ascending { @@ -89,7 +94,8 @@ impl RocksDbStore { &[][..] }; - if !cb(key, value)? || params.first { + remaining -= 1; + if !cb(key, value)? || remaining == 0 { return Ok(()); } @@ -102,18 +108,138 @@ impl RocksDbStore { it.status().map_err(into_error) }) - .await } - pub(crate) async fn get_counter( + pub(crate) fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> impl Future> { + let db = self.db.clone(); + + self.block_reader(move || { + let Some((first, rest)) = ranges.split_first() else { + return Ok(()); + }; + let cf = db.subspace_handle(first.begin.subspace()); + let mut point = Vec::new(); + let mut point_end = Vec::new(); + if first.values + && ranges.iter().all(|params| { + point.clear(); + params.begin.serialize_into(&mut point, 0); + point_end.clear(); + params.end.serialize_into(&mut point_end, 0); + point == point_end + }) + { + let snapshot = (ranges.len() > MULTI_GET_CHUNK).then(|| db.snapshot()); + let mut read_opts = ReadOptions::default(); + if let Some(snapshot) = &snapshot { + read_opts.set_snapshot(snapshot); + } + let mut keys: Vec> = Vec::with_capacity(ranges.len().min(MULTI_GET_CHUNK)); + for chunk in ranges.chunks(MULTI_GET_CHUNK) { + keys.resize_with(chunk.len(), Vec::new); + for (key, params) in keys.iter_mut().zip(chunk) { + key.clear(); + params.begin.serialize_into(key, 0); + } + let values = + db.batched_multi_get_cf_opt(&cf, &keys, keys.is_sorted(), &read_opts); + for (key, value) in keys.iter().zip(values) { + if let Some(value) = value.map_err(into_error)? + && !cb(key, &value)? + { + return Ok(()); + } + } + } + return Ok(()); + } + let mut lower_bound = first.begin.serialize(0); + let mut upper_bound = first.end.serialize(0); + let mut begin = Vec::with_capacity(lower_bound.capacity()); + let mut end = Vec::with_capacity(upper_bound.capacity()); + + for params in rest { + begin.clear(); + params.begin.serialize_into(&mut begin, 0); + if begin < lower_bound { + mem::swap(&mut begin, &mut lower_bound); + } + + end.clear(); + params.end.serialize_into(&mut end, 0); + if end > upper_bound { + mem::swap(&mut end, &mut upper_bound); + } + } + upper_bound.push(0u8); + + let mut read_opts = ReadOptions::default(); + read_opts.set_iterate_lower_bound(lower_bound); + read_opts.set_iterate_upper_bound(upper_bound); + read_opts.set_auto_readahead_size(true); + + let mut it = db.raw_iterator_cf_opt(&cf, read_opts); + + for params in &ranges { + begin.clear(); + params.begin.serialize_into(&mut begin, 0); + end.clear(); + params.end.serialize_into(&mut end, 0); + it.seek(&begin); + + while it.valid() { + let Some(key) = it.key() else { + break; + }; + let is_last = match key.cmp(end.as_slice()) { + Ordering::Less => false, + Ordering::Equal => true, + Ordering::Greater => break, + }; + let value = if params.values { + it.value().unwrap_or_default() + } else { + &[][..] + }; + + if !cb(key, value)? { + return Ok(()); + } + + if is_last { + break; + } + + it.next(); + } + + it.status().map_err(into_error)?; + } + + Ok(()) + }) + } + + pub(crate) fn get_counter( &self, key: impl Into> + Sync + Send, - ) -> trc::Result { + ) -> impl Future> { let key = key.into(); let db = self.db.clone(); + let subspace = key.subspace(); + let slots = key.class.is_queue_quota(); + let key = key.serialize(0); + self.spawn_worker(move || { - let cf = self.db.subspace_handle(key.subspace()); - let key = key.serialize(0); + let cf = db.subspace_handle(subspace); + if slots { + let mut iter = db.raw_iterator_cf_opt(&cf, queue_quota_read_options()); + return sum_slots(&mut iter, &key, 0).map(|sum| sum.total); + } db.get_pinned_cf(&cf, &key) .map_err(into_error) @@ -127,6 +253,6 @@ impl RocksDbStore { }) }) }) - .await } } + diff --git a/crates/store/src/backend/rocksdb/slots.rs b/crates/store/src/backend/rocksdb/slots.rs new file mode 100644 index 00000000..eea174f0 --- /dev/null +++ b/crates/store/src/backend/rocksdb/slots.rs @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::into_error; +use crate::{backend::deserialize_i64_le, write::key::CounterKind}; +use rocksdb::{DBAccess, DBRawIteratorWithThreadMode, ReadOptions}; +use std::sync::atomic::{AtomicBool, Ordering}; + +const SLOT_KEY_LEN: usize = 18; +const MAX_COUNTER_SLOTS: usize = u8::MAX as usize + 1; +const QUEUE_QUOTA_END: u8 = CounterKind::QueueQuotaSize.byte() + 1; + +pub(crate) struct CounterSlots(Box<[AtomicBool]>); + +pub(crate) struct CounterSlot<'x> { + slots: &'x CounterSlots, + index: usize, +} + +pub(crate) struct SlotSum { + pub(crate) total: i64, + pub(crate) own: i64, +} + +impl CounterSlots { + pub(crate) fn new(writers: usize) -> Self { + CounterSlots( + (0..writers.clamp(1, MAX_COUNTER_SLOTS)) + .map(|_| AtomicBool::new(false)) + .collect(), + ) + } + + pub(crate) fn acquire(&self) -> Option> { + self.0 + .iter() + .position(|slot| { + slot.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + }) + .map(|index| CounterSlot { slots: self, index }) + } +} + +impl CounterSlot<'_> { + #[inline] + pub(crate) fn byte(&self) -> u8 { + u8::try_from(self.index).unwrap_or(u8::MAX) + } +} + +impl Drop for CounterSlot<'_> { + fn drop(&mut self) { + if let Some(slot) = self.slots.0.get(self.index) { + slot.store(false, Ordering::Release); + } + } +} + +pub(crate) fn queue_quota_read_options() -> ReadOptions { + let mut options = ReadOptions::default(); + options.set_iterate_upper_bound(vec![QUEUE_QUOTA_END]); + options +} + +pub(crate) fn sum_slots( + iter: &mut DBRawIteratorWithThreadMode<'_, D>, + key: &[u8], + own_slot: u8, +) -> trc::Result { + iter.seek(key); + let mut sum = SlotSum { total: 0, own: 0 }; + while let (Some(slot_key), Some(value)) = (iter.key(), iter.value()) { + let Some(suffix) = slot_key.strip_prefix(key) else { + break; + }; + let value = deserialize_i64_le(slot_key, value)?; + sum.total = sum.total.saturating_add(value); + if suffix == [own_slot] { + sum.own = value; + } + iter.next(); + } + iter.status().map_err(into_error)?; + Ok(sum) +} + +#[derive(Default)] +pub(crate) struct SlotGroups(Vec); + +struct SlotGroup { + base: Box<[u8]>, + total: i64, + keys: Vec>, +} + +impl SlotGroups { + pub(crate) fn add(&mut self, key: Box<[u8]>, value: i64) { + let Some((_, base)) = key.split_last() else { + return; + }; + if key.len() != SLOT_KEY_LEN + || !matches!( + key.first().copied(), + Some(kind) if kind == CounterKind::QueueQuotaCount.byte() + || kind == CounterKind::QueueQuotaSize.byte() + ) + { + return; + } + match self.0.last_mut() { + Some(group) if *group.base == *base => { + group.total = group.total.saturating_add(value); + group.keys.push(key); + } + _ => self.0.push(SlotGroup { + base: base.into(), + total: value, + keys: vec![key], + }), + } + } + + pub(crate) fn zero_sum(&self) -> impl Iterator]> { + self.0 + .iter() + .filter(|group| group.total == 0) + .map(|group| group.keys.as_slice()) + } +} diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 3d8d2ac4..460f776e 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -4,103 +4,151 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{CF_INDEXES, CF_LOGS, CfHandle, RocksDbStore, into_error}; +use super::{ + CfCache, CfHandle, RocksDbStore, into_error, + slots::{CounterSlot, SlotGroups, queue_quota_read_options, sum_slots}, +}; use crate::{ - Deserialize, IndexKey, Key, LogKey, SUBSPACE_BLOBS, SUBSPACE_COUNTER, - SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, + Deserialize, Key, Shape, Subspace, backend::deserialize_i64_le, write::{ - AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, - ValueClass, ValueOp, + Advance, Allocation, AssignedIds, Batch, BatchCursor, LogSet, MAX_COMMIT_ATTEMPTS, + MAX_COMMIT_TIME, MergeResult, ValueOp, commit_backoff, exhausted, }, }; -use rand::RngExt; use rocksdb::{ - BottommostLevelCompaction, BoundColumnFamily, CompactOptions, ErrorKind, IteratorMode, - OptimisticTransactionDB, OptimisticTransactionOptions, WriteOptions, + BoundColumnFamily, ErrorKind, IteratorMode, OptimisticTransactionDB, + OptimisticTransactionOptions, ReadOptions, WriteOptions, }; -use std::{ - sync::Arc, - thread::sleep, - time::{Duration, Instant}, -}; - -const BLOB_GC_MIN_GARBAGE: u64 = 256 * 1024 * 1024; -const BLOB_GC_GARBAGE_RATIO: f64 = 0.25; +use std::{sync::Arc, time::Instant}; impl RocksDbStore { - pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write( + &self, + mut batch: Batch<'_>, + assigned_ids: &mut AssignedIds, + ) -> trc::Result<()> { let db = self.db.clone(); + let mut retry_count = 0; + let start = Instant::now(); + let mark = assigned_ids.mark(); - self.spawn_worker(move || { - let mut txn = RocksDBTransaction { - db: &db, - cf_indexes: db.cf_handle(CF_INDEXES).unwrap(), - cf_logs: db.cf_handle(CF_LOGS).unwrap(), - txn_opts: OptimisticTransactionOptions::default(), - batch: &mut batch, - }; - txn.txn_opts.set_snapshot(true); + loop { + let attempt = self + .block_writer(|| { + assigned_ids.rollback(mark); + let slot = self.counter_slots.acquire(); + let mut txn = RocksDBTransaction { + db: &db, + txn_opts: OptimisticTransactionOptions::default(), + batch: &mut batch, + result: assigned_ids, + counter_slot: slot.as_ref().map_or(0, CounterSlot::byte), + #[cfg(stalwart_tests)] + probe: &self.probe, + }; + txn.txn_opts.set_snapshot(true); - // Begin write - let mut retry_count = 0; - let start = Instant::now(); - loop { - match txn.commit() { - Ok(result) => { - return Ok(result); + match txn.commit() { + Ok(()) => Ok(Ok(())), + Err(CommitError::Internal(err)) => Err(err), + Err(CommitError::RocksDB(err)) => Ok(Err(err)), } - Err(CommitError::Internal(err)) => return Err(err), - Err(CommitError::RocksDB(err)) => match err.kind() { - ErrorKind::Busy | ErrorKind::MergeInProgress | ErrorKind::TryAgain - if retry_count < MAX_COMMIT_ATTEMPTS - && start.elapsed() < MAX_COMMIT_TIME => - { - let backoff = rand::rng().random_range(50..=300); - sleep(Duration::from_millis(backoff)); - retry_count += 1; - } - _ => return Err(into_error(err)), - }, + }) + .await?; + + match attempt { + Ok(()) => { + #[cfg(stalwart_tests)] + self.probe.record_commit(retry_count); + return Ok(()); } + Err(err) => match err.kind() { + ErrorKind::Busy | ErrorKind::MergeInProgress | ErrorKind::TryAgain => { + if retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME { + #[cfg(stalwart_tests)] + { + self.probe.record_retry(err.kind()); + crate::write::record_retry(retry_count); + } + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } else { + return Err(exhausted(into_error(err), retry_count)); + } + } + _ => return Err(into_error(err)), + }, } - }) - .await + } } - pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { + pub(crate) fn delete_range( + &self, + from: impl Key, + to: impl Key, + ) -> impl Future> { let db = self.db.clone(); + let subspace = from.subspace(); + let from = from.serialize(0); + let to = to.serialize(0); + self.spawn_worker(move || { db.delete_range_cf( - &db.cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap()) - .unwrap(), - from.serialize(0), - to.serialize(0), + &db.subspace_handle(subspace), + from.as_slice(), + to.as_slice(), ) .map_err(into_error) }) - .await } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { - for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - let cf = db - .cf_handle(std::str::from_utf8(&[subspace]).unwrap()) - .unwrap(); + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { + let cf = db.subspace_handle(subspace); let mut delete_keys = Vec::new(); + let mut slots = SlotGroups::default(); for row in db.iterator_cf(&cf, IteratorMode::Start) { let (key, value) = row.map_err(into_error)?; if i64::deserialize(&value)? == 0 { delete_keys.push(key); + } else if subspace == Subspace::Counter { + let value = deserialize_i64_le(&key, &value)?; + slots.add(key, value); } } let txn_opts = OptimisticTransactionOptions::default(); + for keys in slots.zero_sum() { + let txn = db.transaction_opt(&WriteOptions::default(), &txn_opts); + let mut total = 0i64; + for key in keys { + if let Some(value) = txn + .get_pinned_for_update_cf(&cf, key, true) + .map_err(into_error)? + { + total = total.saturating_add(deserialize_i64_le(key, &value)?); + } + } + if total == 0 { + for key in keys { + txn.delete_cf(&cf, key).map_err(into_error)?; + } + commit_unless_busy(txn)?; + } else { + txn.rollback().map_err(into_error)?; + } + } + for key in delete_keys { let txn = db.transaction_opt(&WriteOptions::default(), &txn_opts); if txn @@ -110,47 +158,36 @@ impl RocksDbStore { .unwrap_or(false) { txn.delete_cf(&cf, key).map_err(into_error)?; - txn.commit().map_err(into_error)?; + commit_unless_busy(txn)?; } else { txn.rollback().map_err(into_error)?; } } } - let cf = db.subspace_handle(SUBSPACE_BLOBS); - let total = db - .property_int_value_cf(&cf, "rocksdb.live-blob-file-size") - .map_err(into_error)? - .unwrap_or_default(); - let garbage = db - .property_int_value_cf(&cf, "rocksdb.live-blob-file-garbage-size") - .map_err(into_error)? - .unwrap_or_default(); - - if garbage >= BLOB_GC_MIN_GARBAGE - && garbage as f64 >= total as f64 * BLOB_GC_GARBAGE_RATIO - { - db.set_options_cf(&cf, &[("enable_blob_garbage_collection", "true")]) - .map_err(into_error)?; - let mut opts = CompactOptions::default(); - opts.set_bottommost_level_compaction(BottommostLevelCompaction::Force); - db.compact_range_cf_opt(&cf, None::<&[u8]>, None::<&[u8]>, &opts); - db.set_options_cf(&cf, &[("enable_blob_garbage_collection", "false")]) - .map_err(into_error)?; - } - Ok(()) }) - .await + .await?; + self.collect_blob_garbage().await + } +} + +fn commit_unless_busy(txn: rocksdb::Transaction<'_, OptimisticTransactionDB>) -> trc::Result<()> { + match txn.commit() { + Ok(()) => Ok(()), + Err(err) if matches!(err.kind(), ErrorKind::Busy | ErrorKind::TryAgain) => Ok(()), + Err(err) => Err(into_error(err)), } } struct RocksDBTransaction<'x, 'y> { db: &'x OptimisticTransactionDB, - cf_indexes: Arc>, - cf_logs: Arc>, txn_opts: OptimisticTransactionOptions, batch: &'x mut Batch<'y>, + result: &'x mut AssignedIds, + counter_slot: u8, + #[cfg(stalwart_tests)] + probe: &'x super::probe::RocksDbProbe, } enum CommitError { @@ -158,149 +195,172 @@ enum CommitError { RocksDB(rocksdb::Error), } +fn read_counter( + txn: &rocksdb::Transaction<'_, OptimisticTransactionDB>, + cf: &Arc>, + key: &[u8], +) -> Result { + match txn.get_pinned_for_update_cf(cf, key, true)? { + Some(bytes) => deserialize_i64_le(key, &bytes).map_err(CommitError::from), + None => Ok(0), + } +} + impl RocksDBTransaction<'_, '_> { - fn commit(&mut self) -> Result { - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let mut result = AssignedIds::default(); - let has_changes = !self.batch.changes.is_empty(); + fn commit(&mut self) -> Result<(), CommitError> { + let mut cursor = BatchCursor::new(self.batch); + let result = &mut *self.result; + let mut log_buf = Vec::new(); + let mut log_scratch = Vec::new(); + let mut cfs = CfCache::new(self.db); let txn = self .db .transaction_opt(&WriteOptions::default(), &self.txn_opts); - if has_changes { - let cf = self.db.cf_handle("n").unwrap(); - for &account_id in self.batch.changes.keys() { - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); - let change_id = txn - .get_pinned_for_update_cf(&cf, &key, true) - .map_err(CommitError::from) - .and_then(|bytes| { - if let Some(bytes) = bytes { - deserialize_i64_le(&key, &bytes) - .map(|v| v + 1) - .map_err(CommitError::from) - } else { - Ok(1) - } - })?; - txn.put_cf(&cf, &key, &change_id.to_le_bytes()[..])?; - result.push_change_id(account_id, change_id as u64); + if self.batch.has_allocations() { + let cf = cfs.get(Subspace::Counter).clone(); + let mut key_buf = Vec::with_capacity(16); + let mut slots = None; + + for allocation in self.batch.allocations() { + key_buf.clear(); + allocation.serialize_key_into(&mut key_buf, 0); + match allocation { + Allocation::Bounded(bound) if bound.allows_overshoot() => { + let iter = slots.get_or_insert_with(|| { + txn.raw_iterator_cf_opt(&cf, queue_quota_read_options()) + }); + let sum = sum_slots(iter, &key_buf, self.counter_slot)?; + bound.verify(sum.total.saturating_add(bound.by))?; + key_buf.push(self.counter_slot); + let value = sum.own.saturating_add(bound.by); + txn.put_cf(&cf, &key_buf, &value.to_le_bytes()[..])?; + } + _ => { + let last_id = + read_counter(&txn, &cf, &key_buf)? + allocation.increment_by(); + txn.put_cf(&cf, &key_buf, &last_id.to_le_bytes()[..])?; + result.apply(allocation, last_id)?; + } + } } } + let mut key_buf = Vec::with_capacity(64); for op in self.batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; + match cursor.advance(op, result) { + Advance::Cursor => {} + Advance::Value { class, op } => { + cursor.value_key(class, &mut key_buf, 0); + if matches!(op, ValueOp::AtomicAdd(_)) && class.is_queue_quota() { + key_buf.push(self.counter_slot); } - } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let key = class.serialize(account_id, collection, document_id, 0); - let cf = self.db.subspace_handle(class.subspace(collection)); + let key = &key_buf; + let cf = cfs.get(cursor.subspace(class)); match op { ValueOp::Set(value) => { - txn.put_cf(&cf, &key, value)?; - } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; - - txn.put_cf(&cf, &key, value)?; + let value = value.resolve(result)?; + txn.put_cf(cf, key, &*value)?; } ValueOp::MergeFnc(merge_op) => { - let merge_result = (merge_op.fnc)( - &merge_op.params, - &result, - txn.get_pinned_for_update_cf(&cf, &key, true)?.as_deref(), + let merge_result = (merge_op.0)( + result, + txn.get_pinned_for_update_cf(cf, key, true)?.as_deref(), )?; match merge_result { MergeResult::Update(value) => { - txn.put_cf(&cf, &key, value)?; + txn.put_cf(cf, key, value)?; } MergeResult::Delete => { - txn.delete_cf(&cf, &key)?; + txn.delete_cf(cf, key)?; } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { - txn.merge_cf(&cf, &key, &by.to_le_bytes()[..])?; + let value = read_counter(&txn, cf, key)?.saturating_add(*by); + txn.put_cf(cf, key, &value.to_le_bytes()[..])?; } ValueOp::AddAndGet(by) => { let num = txn - .get_pinned_for_update_cf(&cf, &key, true) + .get_pinned_for_update_cf(cf, key, true) .map_err(CommitError::from) .and_then(|bytes| { if let Some(bytes) = bytes { - deserialize_i64_le(&key, &bytes) + deserialize_i64_le(key, &bytes) .map(|v| v + *by) .map_err(CommitError::from) } else { Ok(*by) } })?; - txn.put_cf(&cf, &key, &num.to_le_bytes()[..])?; + txn.put_cf(cf, key, &num.to_le_bytes()[..])?; result.push_counter_id(num); } ValueOp::Clear => { - txn.delete_cf(&cf, &key)?; + txn.delete_cf(cf, key)?; } } } - Operation::Index { field, key, set } => { - let key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: &*key, - } - .serialize(0); + Advance::Index { field, key, set } => { + cursor.index_key(field, key, &mut key_buf, 0); + let key = &key_buf; + let cf = cfs.get(Subspace::Indexes); - if *set { - txn.put_cf(&self.cf_indexes, &key, [])?; + if set { + txn.put_cf(cf, key, [])?; } else { - txn.delete_cf(&self.cf_indexes, &key)?; + txn.delete_cf(cf, key)?; } } - Operation::Log { collection, set } => { - let key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, - } - .serialize(0); + Advance::Log { collection, set } => { + cursor.log_key(collection, result, &mut key_buf, 0); + let cf = cfs.get(Subspace::Logs); - txn.put_cf(&self.cf_logs, &key, set)?; + match set { + LogSet::Bytes(bytes) => { + txn.put_cf(cf, &key_buf, &*bytes)?; + } + LogSet::Pending(changes) => { + changes.serialize_into( + collection.is_prefixed(), + result, + &mut log_scratch, + &mut log_buf, + ); + txn.put_cf(cf, &key_buf, &log_buf)?; + } + } } - Operation::AssertValue { + Advance::ClearRange { subspace, from, to } => { + let cf = cfs.get(subspace); + let mut options = ReadOptions::default(); + options.set_iterate_upper_bound(to.to_vec()); + let mut iter = txn.raw_iterator_cf_opt(cf, options); + iter.seek(from); + let mut keys = Vec::new(); + while let Some(key) = iter.key() { + keys.push(key.to_vec()); + iter.next(); + } + iter.status()?; + drop(iter); + for key in keys { + txn.delete_cf(cf, key)?; + } + } + Advance::Assert { class, assert_value, } => { - let key = class.serialize(account_id, collection, document_id, 0); - let cf = self.db.subspace_handle(class.subspace(collection)); + cursor.value_key(class, &mut key_buf, 0); + let cf = cfs.get(cursor.subspace(class)); let matches = txn - .get_pinned_for_update_cf(&cf, &key, true)? + .get_pinned_for_update_cf(cf, &key_buf, true)? .map(|value| assert_value.matches(&value)) .unwrap_or_else(|| assert_value.is_none()); @@ -314,7 +374,10 @@ impl RocksDBTransaction<'_, '_> { } } - txn.commit().map(|_| result).map_err(Into::into) + #[cfg(stalwart_tests)] + self.probe.stall(); + + txn.commit().map_err(Into::into) } } diff --git a/crates/store/src/backend/s3/mod.rs b/crates/store/src/backend/s3/mod.rs index d46949bb..c91efe56 100644 --- a/crates/store/src/backend/s3/mod.rs +++ b/crates/store/src/backend/s3/mod.rs @@ -5,255 +5,196 @@ */ use crate::BlobStore; -use registry::schema::structs; -use s3::{Bucket, Region, creds::Credentials}; -use std::{io::Write, ops::Range, sync::Arc, time::Duration}; -use utils::codec::base32_custom::Base32Writer; +use encodify::base32::STALWART; +use registry::schema::structs::{self, S3StoreRegion}; +use s3::{ + Bucket, BucketConfig, Credentials, DeleteBatch, Endpoint, RetryPolicy, UrlStyle, + delete::MAX_DELETE_KEYS, +}; +use std::{ops::Range, sync::Arc}; +use trc::StoreEvent; +use utils::http::http1_client_builder; pub struct S3Store { - bucket: Box, - prefix: Option, - max_retries: u32, + bucket: Bucket, + prefix: Option>, verify_after_write: bool, } impl S3Store { pub async fn open(config: structs::S3Store) -> Result { - // Obtain region and endpoint from config - let region = match config.region { - structs::S3StoreRegion::UsEast1 => Region::UsEast1, - structs::S3StoreRegion::UsEast2 => Region::UsEast2, - structs::S3StoreRegion::UsWest1 => Region::UsWest1, - structs::S3StoreRegion::UsWest2 => Region::UsWest2, - structs::S3StoreRegion::CaCentral1 => Region::CaCentral1, - structs::S3StoreRegion::AfSouth1 => Region::Custom { - region: "af-south-1".into(), - endpoint: "s3.af-south-1.amazonaws.com".into(), - }, - structs::S3StoreRegion::ApEast1 => Region::ApEast1, - structs::S3StoreRegion::ApSouth1 => Region::ApSouth1, - structs::S3StoreRegion::ApNortheast1 => Region::ApNortheast1, - structs::S3StoreRegion::ApNortheast2 => Region::ApNortheast2, - structs::S3StoreRegion::ApNortheast3 => Region::ApNortheast3, - structs::S3StoreRegion::ApSoutheast1 => Region::ApSoutheast1, - structs::S3StoreRegion::ApSoutheast2 => Region::ApSoutheast2, - structs::S3StoreRegion::CnNorth1 => Region::CnNorth1, - structs::S3StoreRegion::CnNorthwest1 => Region::CnNorthwest1, - structs::S3StoreRegion::EuNorth1 => Region::EuNorth1, - structs::S3StoreRegion::EuCentral1 => Region::EuCentral1, - structs::S3StoreRegion::EuCentral2 => Region::EuCentral2, - structs::S3StoreRegion::EuWest1 => Region::EuWest1, - structs::S3StoreRegion::EuWest2 => Region::EuWest2, - structs::S3StoreRegion::EuWest3 => Region::EuWest3, - structs::S3StoreRegion::IlCentral1 => Region::IlCentral1, - structs::S3StoreRegion::MeSouth1 => Region::MeSouth1, - structs::S3StoreRegion::SaEast1 => Region::SaEast1, - structs::S3StoreRegion::DoNyc3 => Region::DoNyc3, - structs::S3StoreRegion::DoAms3 => Region::DoAms3, - structs::S3StoreRegion::DoSgp1 => Region::DoSgp1, - structs::S3StoreRegion::DoFra1 => Region::DoFra1, - structs::S3StoreRegion::Yandex => Region::Yandex, - structs::S3StoreRegion::WaUsEast1 => Region::WaUsEast1, - structs::S3StoreRegion::WaUsEast2 => Region::WaUsEast2, - structs::S3StoreRegion::WaUsCentral1 => Region::WaUsCentral1, - structs::S3StoreRegion::WaUsWest1 => Region::WaUsWest1, - structs::S3StoreRegion::WaCaCentral1 => Region::WaCaCentral1, - structs::S3StoreRegion::WaEuCentral1 => Region::WaEuCentral1, - structs::S3StoreRegion::WaEuCentral2 => Region::WaEuCentral2, - structs::S3StoreRegion::WaEuWest1 => Region::WaEuWest1, - structs::S3StoreRegion::WaEuWest2 => Region::WaEuWest2, - structs::S3StoreRegion::WaApNortheast1 => Region::WaApNortheast1, - structs::S3StoreRegion::WaApNortheast2 => Region::WaApNortheast2, - structs::S3StoreRegion::WaApSoutheast1 => Region::WaApSoutheast1, - structs::S3StoreRegion::WaApSoutheast2 => Region::WaApSoutheast2, - structs::S3StoreRegion::Custom(custom) => Region::Custom { - region: custom.custom_region, - endpoint: custom.custom_endpoint, - }, - }; - let credentials = Credentials::new( - config.access_key.value().await?.as_deref(), + let (region, endpoint) = region_endpoint(&config.region); + let endpoint = Endpoint::parse(endpoint).map_err(|err| err.to_string())?; + let credentials = Credentials::resolve( + config.access_key_id.value().await?.as_deref(), config.secret_key.secret().await?.as_deref(), - config.security_token.secret().await?.as_deref(), config.session_token.secret().await?.as_deref(), config.profile.as_deref(), ) - .map_err(|err| format!("Failed to create credentials: {err:?}"))?; + .map_err(|err| format!("Failed to create credentials: {err}"))?; + let http = http1_client_builder(config.allow_invalid_certs) + .timeout(config.timeout.into_inner()) + .build() + .map_err(|err| format!("Failed to build the S3 HTTP client: {err}"))?; + let bucket = Bucket::new( + http, + BucketConfig { + endpoint, + bucket: &config.bucket, + region, + style: UrlStyle::Path, + credentials, + retry: RetryPolicy::new(u32::try_from(config.max_retries).unwrap_or(u32::MAX)), + }, + ) + .map_err(|err| format!("Failed to create bucket: {err}"))?; Ok(BlobStore::S3(Arc::new(S3Store { - bucket: Bucket::new(&config.bucket, region, credentials) - .map_err(|err| format!("Failed to create bucket: {err:?}"))? - .with_path_style() - .set_dangerous_config(config.allow_invalid_certs, config.allow_invalid_certs) - .map_err(|err| format!("Failed to create bucket: {err:?}"))? - .with_request_timeout(config.timeout.into_inner()) - .map_err(|err| format!("Failed to create bucket: {err:?}"))?, - max_retries: config.max_retries as u32, - prefix: config.key_prefix, + bucket, + prefix: config.key_prefix.map(String::into_boxed_str), verify_after_write: config.verify_after_write, }))) } - pub(crate) async fn get_blob( - &self, - key: &[u8], - range: Range, - ) -> trc::Result>> { - let path = self.build_key(key); - let mut retries_left = self.max_retries; - - loop { - let response = if range.start != 0 || range.end != usize::MAX { - self.bucket - .get_object_range( - &path, - range.start as u64, - Some(range.end.saturating_sub(1) as u64), - ) - .await - } else { - self.bucket.get_object(&path).await - } - .map_err(into_error)?; - - match response.status_code() { - 200..=299 => return Ok(Some(response.to_vec())), - 404 => return Ok(None), - 500..=599 if retries_left > 0 => { - // wait backoff - tokio::time::sleep(Duration::from_secs( - 1 << (self.max_retries - retries_left).min(6), - )) - .await; - - retries_left -= 1; - } - code => { - return Err(trc::StoreEvent::S3Error - .reason(String::from_utf8_lossy(response.as_slice())) - .ctx(trc::Key::Code, code)); - } - } - } + pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { + let range = (range.start != 0 || range.end != usize::MAX) + .then(|| range.start as u64..u64::try_from(range.end).unwrap_or(u64::MAX)); + self.bucket + .get_object(&self.build_key(key), range) + .await + .map(|body| body.map(Vec::from)) + .map_err(into_error) } - pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { + pub async fn put_blob(&self, key: &[u8], data: Vec) -> trc::Result<()> { let path = self.build_key(key); - let mut retries_left = self.max_retries; - - loop { - let response = self - .bucket - .put_object(&path, data) - .await - .map_err(into_error)?; - - match response.status_code() { - 200..=299 => { - if !self.verify_after_write { - return Ok(()); - } - - // Some S3-compatible backends acknowledge a PUT before the - // write is durable. HEAD the object to confirm it is visible - // to the read path before reporting success. - let (_, head_status) = - self.bucket.head_object(&path).await.map_err(into_error)?; - - match head_status { - 200..=299 => return Ok(()), - 404 | 500..=599 if retries_left > 0 => { - tokio::time::sleep(Duration::from_secs( - 1 << (self.max_retries - retries_left).min(6), - )) - .await; - - retries_left -= 1; - } - 404 => { - return Err(trc::StoreEvent::S3Error - .reason(concat!( - "PUT acknowledged with 2xx but object not visible", - "to read path; backend may be silently losing writes" - )) - .ctx(trc::Key::Code, head_status)); - } - code => { - return Err(trc::StoreEvent::S3Error - .reason("HEAD verification failed after PUT") - .ctx(trc::Key::Code, code)); - } - } - } - 500..=599 if retries_left > 0 => { - // wait backoff - tokio::time::sleep(Duration::from_secs( - 1 << (self.max_retries - retries_left).min(6), - )) - .await; - - retries_left -= 1; - } - code => { - return Err(trc::StoreEvent::S3Error - .reason(String::from_utf8_lossy(response.as_slice())) - .ctx(trc::Key::Code, code)); - } - } + let data = bytes::Bytes::from(data); + let sent_checksum = self.bucket.sends_checksum(); + let result = if self.verify_after_write { + self.bucket.put_object_verified(&path, data).await + } else { + self.bucket.put_object(&path, data).await + }; + if sent_checksum && !self.bucket.sends_checksum() { + trc::event!( + Store(StoreEvent::S3Error), + Reason = "The S3 service refused the x-amz-checksum-crc32c header; uploads continue with Content-MD5", + ); } + result.map_err(into_error) } - pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { - let mut retries_left = self.max_retries; + pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { + self.bucket + .delete_object(&self.build_key(key)) + .await + .map_err(into_error) + } - loop { - let response = self - .bucket - .delete_object(self.build_key(key)) - .await - .map_err(into_error)?; - - match response.status_code() { - 200..=299 => return Ok(true), - 404 => return Ok(false), - 500..=599 if retries_left > 0 => { - // wait backoff - tokio::time::sleep(Duration::from_secs( - 1 << (self.max_retries - retries_left).min(6), - )) - .await; - - retries_left -= 1; - } - code => { - return Err(trc::StoreEvent::S3Error - .reason(String::from_utf8_lossy(response.as_slice())) - .ctx(trc::Key::Code, code)); + pub async fn delete_blobs(&self, keys: &[&[u8]]) -> trc::Result<()> { + if !self.bucket.supports_multi_delete() { + return self.delete_each(keys).await; + } + let key_len = self.key_len(keys.first().map_or(0, |key| key.len())); + let mut path = String::with_capacity(key_len); + for (index, chunk) in keys.chunks(MAX_DELETE_KEYS).enumerate() { + let mut batch = DeleteBatch::with_capacity(chunk.len(), key_len); + for key in chunk { + path.clear(); + self.build_key_into(key, &mut path); + batch.push(&path).map_err(into_error)?; + } + match self.bucket.delete_objects(batch).await { + Ok(()) => {} + Err(err) if err.is_not_implemented() => { + return self + .delete_each(keys.get(index * MAX_DELETE_KEYS..).unwrap_or_default()) + .await; } + Err(err) => return Err(into_error(err)), } } + Ok(()) + } + + async fn delete_each(&self, keys: &[&[u8]]) -> trc::Result<()> { + for key in keys { + self.delete_blob(key).await?; + } + Ok(()) + } + + fn key_len(&self, key_len: usize) -> usize { + self.prefix.as_ref().map_or(0, |prefix| prefix.len()) + STALWART.encoded_len(key_len) } fn build_key(&self, key: &[u8]) -> String { + let mut out = String::with_capacity(self.key_len(key.len())); + self.build_key_into(key, &mut out); + out + } + + fn build_key_into(&self, key: &[u8], out: &mut String) { if let Some(prefix) = &self.prefix { - let mut writer = - Base32Writer::with_raw_capacity(prefix.len() + (key.len().div_ceil(4) * 5)); - writer.push_string(prefix); - writer.write_all(key).unwrap(); - writer.finalize() - } else { - Base32Writer::from_bytes(key).finalize() + out.push_str(prefix); } + STALWART.encode_append(key, out); } } -fn into_error(err: impl std::error::Error) -> trc::Error { - let mut reason = err.to_string(); - let mut source = err.source(); - while let Some(err) = source { - reason.push_str(": "); - reason.push_str(&err.to_string()); - source = err.source(); +fn region_endpoint(region: &S3StoreRegion) -> (&str, &str) { + match region { + S3StoreRegion::UsEast1 => ("us-east-1", "s3.us-east-1.amazonaws.com"), + S3StoreRegion::UsEast2 => ("us-east-2", "s3.us-east-2.amazonaws.com"), + S3StoreRegion::UsWest1 => ("us-west-1", "s3.us-west-1.amazonaws.com"), + S3StoreRegion::UsWest2 => ("us-west-2", "s3.us-west-2.amazonaws.com"), + S3StoreRegion::CaCentral1 => ("ca-central-1", "s3.ca-central-1.amazonaws.com"), + S3StoreRegion::AfSouth1 => ("af-south-1", "s3.af-south-1.amazonaws.com"), + S3StoreRegion::ApEast1 => ("ap-east-1", "s3.ap-east-1.amazonaws.com"), + S3StoreRegion::ApSouth1 => ("ap-south-1", "s3.ap-south-1.amazonaws.com"), + S3StoreRegion::ApNortheast1 => ("ap-northeast-1", "s3.ap-northeast-1.amazonaws.com"), + S3StoreRegion::ApNortheast2 => ("ap-northeast-2", "s3.ap-northeast-2.amazonaws.com"), + S3StoreRegion::ApNortheast3 => ("ap-northeast-3", "s3.ap-northeast-3.amazonaws.com"), + S3StoreRegion::ApSoutheast1 => ("ap-southeast-1", "s3.ap-southeast-1.amazonaws.com"), + S3StoreRegion::ApSoutheast2 => ("ap-southeast-2", "s3.ap-southeast-2.amazonaws.com"), + S3StoreRegion::CnNorth1 => ("cn-north-1", "s3.cn-north-1.amazonaws.com.cn"), + S3StoreRegion::CnNorthwest1 => ("cn-northwest-1", "s3.cn-northwest-1.amazonaws.com.cn"), + S3StoreRegion::EuNorth1 => ("eu-north-1", "s3.eu-north-1.amazonaws.com"), + S3StoreRegion::EuCentral1 => ("eu-central-1", "s3.eu-central-1.amazonaws.com"), + S3StoreRegion::EuCentral2 => ("eu-central-2", "s3.eu-central-2.amazonaws.com"), + S3StoreRegion::EuWest1 => ("eu-west-1", "s3.eu-west-1.amazonaws.com"), + S3StoreRegion::EuWest2 => ("eu-west-2", "s3.eu-west-2.amazonaws.com"), + S3StoreRegion::EuWest3 => ("eu-west-3", "s3.eu-west-3.amazonaws.com"), + S3StoreRegion::IlCentral1 => ("il-central-1", "s3.il-central-1.amazonaws.com"), + S3StoreRegion::MeSouth1 => ("me-south-1", "s3.me-south-1.amazonaws.com"), + S3StoreRegion::SaEast1 => ("sa-east-1", "s3.sa-east-1.amazonaws.com"), + S3StoreRegion::DoNyc3 => ("nyc3", "nyc3.digitaloceanspaces.com"), + S3StoreRegion::DoAms3 => ("ams3", "ams3.digitaloceanspaces.com"), + S3StoreRegion::DoSgp1 => ("sgp1", "sgp1.digitaloceanspaces.com"), + S3StoreRegion::DoFra1 => ("fra1", "fra1.digitaloceanspaces.com"), + S3StoreRegion::Yandex => ("ru-central1", "storage.yandexcloud.net"), + S3StoreRegion::WaUsEast1 => ("us-east-1", "s3.us-east-1.wasabisys.com"), + S3StoreRegion::WaUsEast2 => ("us-east-2", "s3.us-east-2.wasabisys.com"), + S3StoreRegion::WaUsCentral1 => ("us-central-1", "s3.us-central-1.wasabisys.com"), + S3StoreRegion::WaUsWest1 => ("us-west-1", "s3.us-west-1.wasabisys.com"), + S3StoreRegion::WaCaCentral1 => ("ca-central-1", "s3.ca-central-1.wasabisys.com"), + S3StoreRegion::WaEuCentral1 => ("eu-central-1", "s3.eu-central-1.wasabisys.com"), + S3StoreRegion::WaEuCentral2 => ("eu-central-2", "s3.eu-central-2.wasabisys.com"), + S3StoreRegion::WaEuWest1 => ("eu-west-1", "s3.eu-west-1.wasabisys.com"), + S3StoreRegion::WaEuWest2 => ("eu-west-2", "s3.eu-west-2.wasabisys.com"), + S3StoreRegion::WaApNortheast1 => ("ap-northeast-1", "s3.ap-northeast-1.wasabisys.com"), + S3StoreRegion::WaApNortheast2 => ("ap-northeast-2", "s3.ap-northeast-2.wasabisys.com"), + S3StoreRegion::WaApSoutheast1 => ("ap-southeast-1", "s3.ap-southeast-1.wasabisys.com"), + S3StoreRegion::WaApSoutheast2 => ("ap-southeast-2", "s3.ap-southeast-2.wasabisys.com"), + S3StoreRegion::Custom(custom) => (&custom.region, &custom.url), } - trc::StoreEvent::S3Error.reason(reason) } + +fn into_error(err: s3::Error) -> trc::Error { + let error = StoreEvent::S3Error.reason(err.to_string()); + match err.status() { + Some(status) => error.ctx(trc::Key::Code, status), + None => error, + } +} + diff --git a/crates/store/src/backend/sqlite/blob.rs b/crates/store/src/backend/sqlite/blob.rs index 29f05789..83b7ab02 100644 --- a/crates/store/src/backend/sqlite/blob.rs +++ b/crates/store/src/backend/sqlite/blob.rs @@ -9,20 +9,22 @@ use std::ops::Range; use rusqlite::OptionalExtension; use super::{SqliteStore, into_error}; +use crate::Subspace; impl SqliteStore { - pub(crate) async fn get_blob( + pub(crate) fn get_blob( &self, key: &[u8], range: Range, - ) -> trc::Result>> { + ) -> impl Future>>> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - let mut result = conn - .prepare_cached("SELECT v FROM t WHERE k = ?") - .map_err(into_error)?; - result + conn.prepare_cached(&sql.get(Subspace::Blobs).get_value) + .map_err(into_error)? .query_row([&key], |row| { Ok({ let bytes = row.get_ref(0)?.as_bytes()?; @@ -39,32 +41,39 @@ impl SqliteStore { .optional() .map_err(into_error) }) - .await } - pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { + pub(crate) fn put_blob( + &self, + key: &[u8], + data: Vec, + ) -> impl Future> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - conn.prepare_cached("INSERT OR REPLACE INTO t (k, v) VALUES (?, ?)") + conn.prepare_cached(&sql.get(Subspace::Blobs).upsert_value) .map_err(into_error)? - .execute([key, data]) + .execute([key.as_slice(), data.as_slice()]) .map_err(into_error) .map(|_| ()) }) - .await } - pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { + pub(crate) fn delete_blob(&self, key: &[u8]) -> impl Future> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let key = key.to_vec(); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - conn.prepare_cached("DELETE FROM t WHERE k = ?") + conn.prepare_cached(&sql.get(Subspace::Blobs).delete_key) .map_err(into_error)? .execute([key]) .map_err(into_error) - .map(|_| true) + .map(|rows| rows > 0) }) - .await } } diff --git a/crates/store/src/backend/sqlite/lookup.rs b/crates/store/src/backend/sqlite/lookup.rs index 8dfbbd2f..890a9615 100644 --- a/crates/store/src/backend/sqlite/lookup.rs +++ b/crates/store/src/backend/sqlite/lookup.rs @@ -4,20 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{SqliteStore, into_error}; +use crate::{IntoRows, QueryResult, QueryType, Value, backend::worker::Access}; use rusqlite::{Row, Rows, ToSql, types::FromSql}; -use crate::{IntoRows, QueryResult, QueryType, Value}; - -use super::{SqliteStore, into_error}; - impl SqliteStore { - pub(crate) async fn sql_query( + pub(crate) fn sql_query( &self, query: &str, params_: &[Value<'_>], - ) -> trc::Result { + ) -> impl Future> { let manager = self.conn_pool.clone(); - self.spawn_worker(move || { + let access = match T::query_type() { + QueryType::Execute => Access::Write, + QueryType::Exists | QueryType::QueryAll | QueryType::QueryOne => Access::Read, + }; + self.block(access, move || { let conn = manager.get().map_err(into_error)?; let mut s = conn.prepare_cached(query).map_err(into_error)?; let params = params_ @@ -42,7 +44,6 @@ impl SqliteStore { )), } }) - .await } } @@ -97,10 +98,7 @@ impl IntoRows for Rows<'_> { .map(|s| { ( s.column_count(), - s.column_names() - .into_iter() - .map(String::from) - .collect::>(), + s.column_names().map(String::from).collect::>(), ) }) .unwrap_or((0, Vec::new())); diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 4631e770..459f69f7 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -4,119 +4,65 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{SqliteStore, into_error, pool::SqliteConnectionManager}; -use crate::*; +use super::{SqliteStore, into_error, pool::SqliteConnectionManager, sql::SqlStatements}; +use crate::{ + backend::worker::{self, Access, BlockingPermits}, + *, +}; use ::registry::schema::structs; use r2d2::Pool; -use tokio::sync::oneshot; +use rusqlite::Connection; + +const PAGE_SIZE: u32 = 8192; +const CACHE_SIZE_KIB: i64 = -32768; +const MMAP_SIZE: u64 = 256 * 1024 * 1024; +const WAL_AUTOCHECKPOINT: u32 = 8192; +const BUSY_TIMEOUT_MS: u32 = 30000; +const STATEMENT_CACHE_CAPACITY: usize = 256; +const MIN_POOL_CONNECTIONS: u32 = 4; +const MAX_POOL_CONNECTIONS: u32 = 16; impl SqliteStore { pub fn open(config: structs::SqliteStore) -> Result { + let pool_workers = worker::pool_workers(config.pool_workers); + let pool_max_connections = config.pool_max_connections.map_or_else( + || { + u32::try_from(pool_workers) + .unwrap_or(u32::MAX) + .clamp(MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS) + }, + |max| u32::try_from(max).unwrap_or(u32::MAX).max(1), + ); + Ok(Store::SQLite(Arc::new(SqliteStore { conn_pool: Pool::builder() - .max_size(config.pool_max_connections as u32) - .build(SqliteConnectionManager::file(&config.path).with_init(|c| { - c.execute_batch(concat!( - "PRAGMA journal_mode = WAL; ", - "PRAGMA synchronous = NORMAL; ", - "PRAGMA temp_store = memory;", - "PRAGMA busy_timeout = 30000;" - )) - })) + .max_size(pool_max_connections) + .max_lifetime(None) + .idle_timeout(None) + .build(SqliteConnectionManager::file(&config.path).with_init(init_connection)) .map_err(|err| format!("Failed to build connection pool: {err}"))?, worker_pool: rayon::ThreadPoolBuilder::new() - .num_threads(std::cmp::max( - config - .pool_workers - .filter(|v| *v > 0) - .map(|v| v as usize) - .unwrap_or_else(num_cpus::get), - 4, - )) + .num_threads(pool_workers) .build() .map_err(|err| format!("Failed to build worker pool: {err}"))?, + blocking: BlockingPermits::new(pool_workers), + sql: Arc::new(SqlStatements::new()), }))) } - #[cfg(feature = "test_mode")] - pub fn open_memory() -> trc::Result { - use super::into_error; - - let db = Self { - conn_pool: Pool::builder() - .max_size(1) - .build(SqliteConnectionManager::memory()) - .map_err(into_error)?, - worker_pool: rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .build() - .map_err(|err| { - into_error(err).ctx(trc::Key::Reason, "Failed to build worker pool") - })?, - }; - db.create_tables()?; - Ok(db) - } - pub(crate) fn create_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().map_err(into_error)?; - for table in [ - SUBSPACE_ACL, - SUBSPACE_TASK_QUEUE, - SUBSPACE_DELETED_ITEMS, - SUBSPACE_SPAM_SAMPLES, - SUBSPACE_BLOB_LINK, - SUBSPACE_IN_MEMORY_VALUE, - SUBSPACE_PROPERTY, - SUBSPACE_REGISTRY, - SUBSPACE_REGISTRY_PK, - SUBSPACE_QUEUE_MESSAGE, - SUBSPACE_QUEUE_EVENT, - SUBSPACE_REPORT_OUT, - SUBSPACE_REPORT_IN, - SUBSPACE_LOGS, - SUBSPACE_BLOBS, - SUBSPACE_TELEMETRY_SPAN, - SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_SEARCH_INDEX, - SUBSPACE_DIRECTORY, - ] { - let table = char::from(table); - conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k BLOB PRIMARY KEY, - v BLOB NOT NULL - )" - ), - [], - ) - .map_err(into_error)?; - } + for subspace in Subspace::ALL.iter().copied() { + let table = subspace.name(); + let columns = match subspace.shape() { + Shape::Value => "k BLOB PRIMARY KEY, v BLOB NOT NULL", + Shape::Presence => "k BLOB PRIMARY KEY", + Shape::Counter => "k BLOB PRIMARY KEY, v INTEGER NOT NULL DEFAULT 0", + }; - for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {table} ( - k BLOB PRIMARY KEY - )" - ), - [], - ) - .map_err(into_error)?; - } - - for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { - conn.execute( - &format!( - "CREATE TABLE IF NOT EXISTS {} ( - k BLOB PRIMARY KEY, - v INTEGER NOT NULL DEFAULT 0 - )", - char::from(table) - ), + &format!("CREATE TABLE IF NOT EXISTS {table} ({columns})"), [], ) .map_err(into_error)?; @@ -125,22 +71,60 @@ impl SqliteStore { Ok(()) } - pub async fn spawn_worker(&self, mut f: U) -> trc::Result + pub(crate) fn spawn_worker(&self, f: U) -> impl Future> + where + U: FnOnce() -> trc::Result + Send + 'static, + V: Send + 'static, + { + worker::spawn(&self.worker_pool, f) + } + + pub(crate) fn block(&self, access: Access, f: U) -> impl Future> where U: FnMut() -> trc::Result + Send, - V: Sync + Send + 'static, + V: Send, { - let (tx, rx) = oneshot::channel(); + self.blocking.block(access, f) + } - self.worker_pool.scope(|s| { - s.spawn(|_| { - tx.send(f()).ok(); - }); - }); + pub(crate) fn block_reader(&self, f: U) -> impl Future> + where + U: FnMut() -> trc::Result + Send, + V: Send, + { + self.blocking.block(Access::Read, f) + } - match rx.await { - Ok(result) => result, - Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err)), - } + pub(crate) fn block_writer(&self, f: U) -> impl Future> + where + U: FnMut() -> trc::Result + Send, + V: Send, + { + self.blocking.block(Access::Write, f) } } + +fn init_connection(conn: &mut Connection) -> Result<(), rusqlite::Error> { + conn.set_prepared_statement_cache_capacity(STATEMENT_CACHE_CAPACITY); + conn.execute_batch(&format!( + concat!( + "PRAGMA page_size = {}; ", + "PRAGMA journal_mode = WAL; ", + "PRAGMA synchronous = NORMAL; ", + "PRAGMA cache_size = {}; ", + "PRAGMA mmap_size = {}; ", + "PRAGMA temp_store = memory; ", + "PRAGMA busy_timeout = {}; ", + "PRAGMA wal_autocheckpoint = {};" + ), + PAGE_SIZE, CACHE_SIZE_KIB, MMAP_SIZE, BUSY_TIMEOUT_MS, WAL_AUTOCHECKPOINT + ))?; + + let journal_mode: String = conn.query_row("PRAGMA journal_mode", [], |row| row.get(0))?; + if !journal_mode.eq_ignore_ascii_case("wal") { + conn.execute_batch("PRAGMA synchronous = FULL;")?; + } + + Ok(()) +} + diff --git a/crates/store/src/backend/sqlite/mod.rs b/crates/store/src/backend/sqlite/mod.rs index 4346f68d..d23cba6a 100644 --- a/crates/store/src/backend/sqlite/mod.rs +++ b/crates/store/src/backend/sqlite/mod.rs @@ -4,20 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use self::pool::SqliteConnectionManager; +use self::{pool::SqliteConnectionManager, sql::SqlStatements}; +use crate::backend::worker::BlockingPermits; use r2d2::Pool; -use std::fmt::Display; +use std::{fmt::Display, sync::Arc}; pub mod blob; pub mod lookup; pub mod main; pub mod pool; pub mod read; +pub mod sql; pub mod write; pub struct SqliteStore { pub(crate) conn_pool: Pool, pub(crate) worker_pool: rayon::ThreadPool, + pub(crate) blocking: BlockingPermits, + pub(crate) sql: Arc, } #[inline(always)] diff --git a/crates/store/src/backend/sqlite/pool.rs b/crates/store/src/backend/sqlite/pool.rs index 23082f1e..3224d503 100644 --- a/crates/store/src/backend/sqlite/pool.rs +++ b/crates/store/src/backend/sqlite/pool.rs @@ -16,7 +16,6 @@ enum Source { type InitFn = dyn Fn(&mut Connection) -> Result<(), rusqlite::Error> + Send + Sync + 'static; -/// An `r2d2::ManageConnection` for `rusqlite::Connection`s. pub struct SqliteConnectionManager { source: Source, flags: OpenFlags, @@ -34,9 +33,6 @@ impl fmt::Debug for SqliteConnectionManager { } impl SqliteConnectionManager { - /// Creates a new `SqliteConnectionManager` from file. - /// - /// See `rusqlite::Connection::open` pub fn file>(path: P) -> Self { Self { source: Source::File(path.as_ref().to_path_buf()), @@ -45,7 +41,6 @@ impl SqliteConnectionManager { } } - /// Creates a new `SqliteConnectionManager` from memory. pub fn memory() -> Self { Self { source: Source::Memory, @@ -54,28 +49,10 @@ impl SqliteConnectionManager { } } - /// Converts `SqliteConnectionManager` into one that sets OpenFlags upon - /// connection creation. - /// - /// See `rustqlite::OpenFlags` for a list of available flags. pub fn with_flags(self, flags: OpenFlags) -> Self { Self { flags, ..self } } - /// Converts `SqliteConnectionManager` into one that calls an initialization - /// function upon connection creation. Could be used to set PRAGMAs, for - /// example. - /// - /// ### Example - /// - /// Make a `SqliteConnectionManager` that sets the `foreign_keys` pragma to - /// true for every connection. - /// - /// ```rust,no_run - /// # use r2d2_sqlite::{SqliteConnectionManager}; - /// let manager = SqliteConnectionManager::file("app.db") - /// .with_init(|c| c.execute_batch("PRAGMA foreign_keys=1;")); - /// ``` pub fn with_init(self, init: F) -> Self where F: Fn(&mut Connection) -> Result<(), rusqlite::Error> + Send + Sync + 'static, @@ -85,11 +62,6 @@ impl SqliteConnectionManager { } } -fn sleeper(_: i32) -> bool { - std::thread::sleep(std::time::Duration::from_millis(200)); - true -} - impl r2d2::ManageConnection for SqliteConnectionManager { type Connection = Connection; type Error = rusqlite::Error; @@ -99,12 +71,9 @@ impl r2d2::ManageConnection for SqliteConnectionManager { Source::File(ref path) => Connection::open_with_flags(path, self.flags), Source::Memory => Connection::open_in_memory_with_flags(self.flags), } - .and_then(|mut c| { - c.busy_handler(Some(sleeper))?; - match self.init { - None => Ok(c), - Some(ref init) => init(&mut c).map(|_| c), - } + .and_then(|mut c| match self.init { + None => Ok(c), + Some(ref init) => init(&mut c).map(|_| c), }) } @@ -112,7 +81,7 @@ impl r2d2::ManageConnection for SqliteConnectionManager { conn.execute_batch("") } - fn has_broken(&self, _: &mut Connection) -> bool { - false + fn has_broken(&self, conn: &mut Connection) -> bool { + !conn.is_autocommit() } } diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index c4fe10df..9e46eecf 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -5,140 +5,176 @@ */ use super::{SqliteStore, into_error}; -use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; +use crate::{ + Deserialize, Fetch, IterateParams, Key, ValueKey, backend::ValueColumn, write::ValueClass, +}; use rusqlite::OptionalExtension; impl SqliteStore { - pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> + pub(crate) fn get_value(&self, key: impl Key) -> impl Future>> where U: Deserialize + 'static, { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let subspace = key.subspace(); + let key = key.serialize(0); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - let mut result = conn - .prepare_cached(&format!( - "SELECT v FROM {} WHERE k = ?", - char::from(key.subspace()) - )) - .map_err(into_error)?; - let key = key.serialize(0); - result - .query_row([&key], |row| { - U::deserialize_with_key(&key, row.get_ref(0)?.as_bytes()?) - .map_err(|err| rusqlite::Error::ToSqlConversionFailure(err.into())) - }) + let bytes = conn + .prepare_cached(&sql.get(subspace).get_value) + .map_err(into_error)? + .query_row([&key], |row| Ok(row.get_ref(0)?.as_bytes()?.to_vec())) .optional() - .map_err(into_error) + .map_err(into_error)?; + + match bytes { + Some(bytes) => U::deserialize_with_key(&key, &bytes).map(Some), + None => Ok(None), + } }) - .await } - pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result { + pub(crate) fn key_exists(&self, key: impl Key) -> impl Future> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let subspace = key.subspace(); + let key = key.serialize(0); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - let mut result = conn - .prepare_cached(&format!( - "SELECT 1 FROM {} WHERE k = ?", - char::from(key.subspace()) - )) - .map_err(into_error)?; - let key = key.serialize(0); - result + conn.prepare_cached(&sql.get(subspace).key_exists) + .map_err(into_error)? .query_row([&key], |_| Ok(())) .optional() .map(|opt| opt.is_some()) .map_err(into_error) }) - .await } - pub(crate) async fn iterate( + pub(crate) fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, - ) -> trc::Result<()> { + ) -> impl Future> { + let mut remaining = match params.fetch { + Fetch::Limit(rows) => u64::from(rows), + Fetch::All | Fetch::Expect(_) => u64::MAX, + }; let manager = self.conn_pool.clone(); - self.spawn_worker(move || { + let subspace = params.begin.subspace(); + let column = ValueColumn::new(params.values, subspace); + let sql = self.sql.get(subspace).iterate( + params.fetch == Fetch::Limit(1), + params.ascending, + params.values, + ); + + self.block_reader(move || { let conn = manager.get().map_err(into_error)?; - let table = char::from(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); - let keys = if params.values { "k, v" } else { "k" }; - let mut query = conn - .prepare_cached(&match (params.first, params.ascending) { - (true, true) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" - ) - } - (true, false) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" - ) - } - (false, true) => { - format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") - } - (false, false) => { - format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC" - ) - } - }) - .map_err(into_error)?; + let mut query = conn.prepare_cached(sql).map_err(into_error)?; let mut rows = query.query([&begin, &end]).map_err(into_error)?; - if params.values { + while let Some(row) = rows.next().map_err(into_error)? { + let key = row + .get_ref(0) + .map_err(into_error)? + .as_bytes() + .map_err(into_error)?; + let counter; + let value = match column { + ValueColumn::Skip => b"".as_slice(), + ValueColumn::Bytes => row + .get_ref(1) + .map_err(into_error)? + .as_bytes() + .map_err(into_error)?, + ValueColumn::Counter => { + counter = read_counter(row)?; + counter.as_slice() + } + }; + + remaining -= 1; + if !cb(key, value)? || remaining == 0 { + break; + } + } + + Ok(()) + }) + } + + pub(crate) fn iterate_many( + &self, + ranges: Vec>, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> impl Future> { + let manager = self.conn_pool.clone(); + let subspace = ranges[0].begin.subspace(); + let column = ValueColumn::new(ranges[0].values, subspace); + let sql = self + .sql + .get(subspace) + .iterate(false, true, column.is_selected()); + + self.block_reader(move || { + let conn = manager.get().map_err(into_error)?; + let mut stmt = conn.prepare_cached(sql).map_err(into_error)?; + + 'outer: for params in &ranges { + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); + let mut rows = stmt.query([&begin, &end]).map_err(into_error)?; + while let Some(row) = rows.next().map_err(into_error)? { let key = row .get_ref(0) .map_err(into_error)? .as_bytes() .map_err(into_error)?; - let value = row - .get_ref(1) - .map_err(into_error)? - .as_bytes() - .map_err(into_error)?; - - if !cb(key, value)? { - break; - } - } - } else { - while let Some(row) = rows.next().map_err(into_error)? { - if !cb( - row.get_ref(0) + let counter; + let value = match column { + ValueColumn::Skip => b"".as_slice(), + ValueColumn::Bytes => row + .get_ref(1) .map_err(into_error)? .as_bytes() .map_err(into_error)?, - b"", - )? { - break; + ValueColumn::Counter => { + counter = read_counter(row)?; + counter.as_slice() + } + }; + + if !cb(key, value)? { + break 'outer; } } } Ok(()) }) - .await } - pub(crate) async fn get_counter( + pub(crate) fn get_counter( &self, key: impl Into> + Sync + Send, - ) -> trc::Result { + ) -> impl Future> { let key = key.into(); - let table = char::from(key.subspace()); - let key = key.serialize(0); let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let subspace = key.subspace(); + let key = key.serialize(0); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; match conn - .prepare_cached(&format!("SELECT v FROM {table} WHERE k = ?")) + .prepare_cached(&sql.get(subspace).get_value) .map_err(into_error)? .query_row([&key], |row| row.get::<_, i64>(0)) { @@ -147,6 +183,12 @@ impl SqliteStore { Err(e) => Err(into_error(e)), } }) - .await } } + +#[inline] +fn read_counter(row: &rusqlite::Row<'_>) -> trc::Result<[u8; 8]> { + row.get::<_, i64>(1) + .map(i64::to_le_bytes) + .map_err(into_error) +} diff --git a/crates/store/src/backend/sqlite/sql.rs b/crates/store/src/backend/sqlite/sql.rs new file mode 100644 index 00000000..6e4a77c1 --- /dev/null +++ b/crates/store/src/backend/sqlite/sql.rs @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Subspace; + +pub(crate) struct SubspaceSql { + pub(crate) get_value: Box, + pub(crate) key_exists: Box, + pub(crate) upsert_value: Box, + pub(crate) insert_presence: Box, + pub(crate) delete_key: Box, + pub(crate) increment: Box, + pub(crate) decrement: Box, + pub(crate) increment_returning: Box, + pub(crate) delete_range: Box, + pub(crate) purge_zero: Box, + iterate: [Box; 8], +} + +pub(crate) struct SqlStatements { + statements: Box<[SubspaceSql]>, +} + +impl SubspaceSql { + fn new(subspace: Subspace) -> Self { + let table = subspace.name(); + let range = |order: &str, limit: &str, columns: &str| { + format!( + "SELECT {columns} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k {order}{limit}" + ) + .into_boxed_str() + }; + + Self { + get_value: format!("SELECT v FROM {table} WHERE k = ?").into_boxed_str(), + key_exists: format!("SELECT 1 FROM {table} WHERE k = ?").into_boxed_str(), + upsert_value: format!( + "INSERT INTO {table} (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v" + ) + .into_boxed_str(), + insert_presence: format!("INSERT OR IGNORE INTO {table} (k) VALUES (?)") + .into_boxed_str(), + delete_key: format!("DELETE FROM {table} WHERE k = ?").into_boxed_str(), + increment: format!( + "INSERT INTO {table} (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = v + excluded.v" + ) + .into_boxed_str(), + decrement: format!("UPDATE {table} SET v = v + ? WHERE k = ?").into_boxed_str(), + increment_returning: format!( + "INSERT INTO {table} (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = v + excluded.v RETURNING v" + ) + .into_boxed_str(), + delete_range: format!("DELETE FROM {table} WHERE k >= ? AND k < ?").into_boxed_str(), + purge_zero: format!("DELETE FROM {table} WHERE v = 0").into_boxed_str(), + iterate: [ + range("ASC", "", "k"), + range("ASC", "", "k, v"), + range("DESC", "", "k"), + range("DESC", "", "k, v"), + range("ASC", " LIMIT 1", "k"), + range("ASC", " LIMIT 1", "k, v"), + range("DESC", " LIMIT 1", "k"), + range("DESC", " LIMIT 1", "k, v"), + ], + } + } + + #[inline(always)] + pub(crate) fn iterate(&self, first: bool, ascending: bool, values: bool) -> &str { + &self.iterate[((first as usize) << 2) | ((!ascending as usize) << 1) | (values as usize)] + } +} + +impl SqlStatements { + pub(crate) fn new() -> Self { + Self { + statements: Subspace::ALL + .iter() + .copied() + .map(SubspaceSql::new) + .collect(), + } + } + + #[inline(always)] + pub(crate) fn get(&self, subspace: Subspace) -> &SubspaceSql { + &self.statements[subspace.index()] + } +} diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index eb9860ef..4c4af4c3 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -4,89 +4,102 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{SqliteStore, into_error}; +use super::{SqliteStore, into_error, sql::SqlStatements}; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - SUBSPACE_REGISTRY_IDX, - write::{AssignedIds, Batch, MergeResult, Operation, ValueClass, ValueOp}, + Key, Shape, Subspace, + write::{ + Advance, AssignedIds, Batch, BatchCursor, LogSet, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, + MergeResult, ValueOp, commit_backoff, + }, }; -use rusqlite::{OptionalExtension, TransactionBehavior, params}; +use rusqlite::{ErrorCode, OptionalExtension, Transaction, TransactionBehavior, params}; +use std::time::Instant; use trc::AddContext; impl SqliteStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write( + &self, + mut batch: Batch<'_>, + assigned_ids: &mut AssignedIds, + ) -> trc::Result<()> { + let mut retry_count = 0; + let start = Instant::now(); + let mark = assigned_ids.mark(); + + loop { + if self.write_once(&mut batch, assigned_ids, mark).await? { + return Ok(()); + } else if retry_count >= MAX_COMMIT_ATTEMPTS || start.elapsed() >= MAX_COMMIT_TIME { + return Err(trc::StoreEvent::SqliteError + .into_err() + .details("Database is locked") + .caused_by(trc::location!())); + } + + #[cfg(stalwart_tests)] + crate::write::record_retry(retry_count); + tokio::time::sleep(commit_backoff(retry_count)).await; + retry_count += 1; + } + } + + fn write_once( + &self, + batch: &mut Batch<'_>, + assigned_ids: &mut AssignedIds, + mark: crate::write::AssignedIdsMark, + ) -> impl Future> { let manager = self.conn_pool.clone(); - self.spawn_worker(move || { + let sql = &*self.sql; + + self.block_writer(move || { let mut conn = manager.get().map_err(into_error)?; - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - let mut change_id = 0u64; - let trx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .map_err(into_error) - .caused_by(trc::location!())?; - let mut result = AssignedIds::default(); - let has_changes = !batch.changes.is_empty(); + let mut cursor = BatchCursor::new(batch); + let trx = match conn.transaction_with_behavior(TransactionBehavior::Immediate) { + Ok(trx) => trx, + Err(err) if is_busy(&err) => return Ok(false), + Err(err) => return Err(into_error(err)).caused_by(trc::location!()), + }; + let result = &mut *assigned_ids; + result.rollback(mark); + let mut log_buf = Vec::new(); + let mut log_scratch = Vec::new(); - if has_changes { - for &account_id in batch.changes.keys() { - let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); - let change_id = trx - .prepare_cached(concat!( - "INSERT INTO n (k, v) VALUES (?, ?) ", - "ON CONFLICT(k) DO UPDATE SET v = v + ", - "excluded.v RETURNING v" - )) - .map_err(into_error) - .caused_by(trc::location!())? - .query_row(params![&key, &1i64], |row| row.get::<_, i64>(0)) - .map_err(into_error) - .caused_by(trc::location!())?; - result.push_change_id(account_id, change_id as u64); + if batch.has_allocations() { + let mut key_buf = Vec::with_capacity(16); + + for allocation in batch.allocations() { + key_buf.clear(); + allocation.serialize_key_into(&mut key_buf, 0); + let last_id = incr_counter(&trx, sql, &key_buf, allocation.increment_by())?; + result.apply(allocation, last_id)?; } } + let mut key_buf = Vec::with_capacity(64); for op in batch.ops.iter_mut() { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - if has_changes { - change_id = result.set_current_change_id(account_id)?; - } - } - Operation::Collection { - collection: collection_, - } => { - collection = u8::from(*collection_); - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Value { class, op } => { - let key = class.serialize(account_id, collection, document_id, 0); - let subspace = class.subspace(collection); - let table = char::from(subspace); + match cursor.advance(op, result) { + Advance::Cursor => {} + Advance::Value { class, op } => { + cursor.value_key(class, &mut key_buf, 0); + let key = &key_buf; + let subspace = cursor.subspace(class); + let stmts = sql.get(subspace); match op { - ValueOp::Set(value) => { - if subspace != SUBSPACE_REGISTRY_IDX { - trx.prepare_cached(&format!( - "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([&key, value]) - .map_err(into_error) - .caused_by(trc::location!())?; + ValueOp::Set(set_value) => { + let value = set_value.resolve(result)?; + let value = &*value; + if !matches!(subspace.shape(), Shape::Presence) { + trx.prepare_cached(&stmts.upsert_value) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([key.as_slice(), value]) + .map_err(into_error) + .caused_by(trc::location!())?; } else { - trx.prepare_cached("INSERT OR IGNORE INTO b (k) VALUES (?)") + trx.prepare_cached(&stmts.insert_presence) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) @@ -94,107 +107,71 @@ impl SqliteStore { .caused_by(trc::location!())?; } } - ValueOp::SetFnc(set_op) => { - let value = (set_op.fnc)(&set_op.params, &result)?; - trx.prepare_cached(&format!( - "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([&key, &value]) - .map_err(into_error) - .caused_by(trc::location!())?; - } ValueOp::MergeFnc(merge_op) => { let merge_result = trx - .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table)) + .prepare_cached(&stmts.get_value) .map_err(into_error) .caused_by(trc::location!())? .query_row([&key], |row| { - Ok((merge_op.fnc)( - &merge_op.params, - &result, + Ok((merge_op.0)( + &*result, Some(row.get_ref(0)?.as_bytes()?), )) }) .optional() .map_err(into_error) .caused_by(trc::location!())? - .unwrap_or_else(|| { - (merge_op.fnc)(&merge_op.params, &result, None) - })?; + .unwrap_or_else(|| (merge_op.0)(&*result, None))?; match merge_result { MergeResult::Update(value) => { - trx.prepare_cached(&format!( - "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([&key, &value]) - .map_err(into_error) - .caused_by(trc::location!())?; + trx.prepare_cached(&stmts.upsert_value) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([key, &value]) + .map_err(into_error) + .caused_by(trc::location!())?; } MergeResult::Delete => { - trx.prepare_cached(&format!( - "DELETE FROM {} WHERE k = ?", - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([&key]) - .map_err(into_error) - .caused_by(trc::location!())?; + trx.prepare_cached(&stmts.delete_key) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([&key]) + .map_err(into_error) + .caused_by(trc::location!())?; } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { if *by >= 0 { - trx.prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES (?, ?) ", - "ON CONFLICT(k) DO UPDATE SET v = v + excluded.v" - ), - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute(params![&key, *by]) - .map_err(into_error) - .caused_by(trc::location!())?; + trx.prepare_cached(&stmts.increment) + .map_err(into_error) + .caused_by(trc::location!())? + .execute(params![&key, *by]) + .map_err(into_error) + .caused_by(trc::location!())?; } else { - trx.prepare_cached(&format!( - "UPDATE {table} SET v = v + ? WHERE k = ?" - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute(params![*by, &key]) - .map_err(into_error) - .caused_by(trc::location!())?; + trx.prepare_cached(&stmts.decrement) + .map_err(into_error) + .caused_by(trc::location!())? + .execute(params![*by, &key]) + .map_err(into_error) + .caused_by(trc::location!())?; } } ValueOp::AddAndGet(by) => { result.push_counter_id( - trx.prepare_cached(&format!( - concat!( - "INSERT INTO {} (k, v) VALUES (?, ?) ", - "ON CONFLICT(k) DO UPDATE SET v = v + ", - "excluded.v RETURNING v" - ), - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .query_row(params![&key, &*by], |row| row.get::<_, i64>(0)) - .map_err(into_error) - .caused_by(trc::location!())?, + trx.prepare_cached(&stmts.increment_returning) + .map_err(into_error) + .caused_by(trc::location!())? + .query_row(params![&key, &*by], |row| row.get::<_, i64>(0)) + .map_err(into_error) + .caused_by(trc::location!())?, ); } ValueOp::Clear => { - trx.prepare_cached(&format!("DELETE FROM {} WHERE k = ?", table)) + trx.prepare_cached(&stmts.delete_key) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) @@ -203,25 +180,21 @@ impl SqliteStore { } } } - Operation::Index { field, key, set } => { - let key = IndexKey { - account_id, - collection, - document_id, - field: *field, - key: &*key, - } - .serialize(0); + Advance::Index { field, key, set } => { + cursor.index_key(field, key, &mut key_buf, 0); + let key = &key_buf; - if *set { - trx.prepare_cached("INSERT OR IGNORE INTO i (k) VALUES (?)") + let stmts = sql.get(Subspace::Indexes); + + if set { + trx.prepare_cached(&stmts.insert_presence) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) .map_err(into_error) .caused_by(trc::location!())?; } else { - trx.prepare_cached("DELETE FROM i WHERE k = ?") + trx.prepare_cached(&stmts.delete_key) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) @@ -229,30 +202,46 @@ impl SqliteStore { .caused_by(trc::location!())?; } } - Operation::Log { collection, set } => { - let key = LogKey { - account_id, - collection: u8::from(*collection), - change_id, - } - .serialize(0); + Advance::Log { collection, set } => { + cursor.log_key(collection, result, &mut key_buf, 0); + let key = &key_buf; + let value = match set { + LogSet::Bytes(bytes) => &*bytes, + LogSet::Pending(changes) => { + changes.serialize_into( + collection.is_prefixed(), + result, + &mut log_scratch, + &mut log_buf, + ); + &log_buf + } + }; - trx.prepare_cached("INSERT OR REPLACE INTO l (k, v) VALUES (?, ?)") + trx.prepare_cached(&sql.get(Subspace::Logs).upsert_value) .map_err(into_error) .caused_by(trc::location!())? - .execute([&key, set]) + .execute([key.as_slice(), value.as_slice()]) .map_err(into_error) .caused_by(trc::location!())?; } - Operation::AssertValue { + Advance::ClearRange { subspace, from, to } => { + trx.prepare_cached(&sql.get(subspace).delete_range) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([from, to]) + .map_err(into_error) + .caused_by(trc::location!())?; + } + Advance::Assert { class, assert_value, } => { - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); + cursor.value_key(class, &mut key_buf, 0); + let key = &key_buf; let matches = trx - .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table)) + .prepare_cached(&sql.get(cursor.subspace(class)).get_value) .map_err(into_error) .caused_by(trc::location!())? .query_row([&key], |row| { @@ -274,17 +263,26 @@ impl SqliteStore { } } - trx.commit().map(|_| result).map_err(into_error) + match trx.commit() { + Ok(()) => Ok(true), + Err(err) if is_busy(&err) => Ok(false), + Err(err) => Err(into_error(err)).caused_by(trc::location!()), + } }) - .await } - pub(crate) async fn purge_store(&self) -> trc::Result<()> { + pub(crate) fn purge_store(&self) -> impl Future> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - conn.prepare_cached(&format!("DELETE FROM {} WHERE v = 0", char::from(subspace),)) + for subspace in Subspace::ALL + .iter() + .copied() + .filter(|subspace| matches!(subspace.shape(), Shape::Counter)) + { + conn.prepare_cached(&sql.get(subspace).purge_zero) .map_err(into_error) .caused_by(trc::location!())? .execute([]) @@ -292,28 +290,58 @@ impl SqliteStore { .caused_by(trc::location!())?; } + conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .map_err(into_error) + .caused_by(trc::location!())?; + Ok(()) }) - .await } - pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { + pub(crate) fn delete_range( + &self, + from: impl Key, + to: impl Key, + ) -> impl Future> { let manager = self.conn_pool.clone(); + let sql = self.sql.clone(); + let subspace = from.subspace(); + let from = from.serialize(0); + let to = to.serialize(0); + self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - conn.prepare_cached(&format!( - "DELETE FROM {} WHERE k >= ? AND k < ?", - char::from(from.subspace()), - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([from.serialize(0), to.serialize(0)]) - .map_err(into_error) - .caused_by(trc::location!())?; + conn.prepare_cached(&sql.get(subspace).delete_range) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([from, to]) + .map_err(into_error) + .caused_by(trc::location!())?; Ok(()) }) - .await } } + +fn incr_counter( + trx: &Transaction<'_>, + sql: &SqlStatements, + key: &[u8], + by: i64, +) -> trc::Result { + trx.prepare_cached(&sql.get(Subspace::Counter).increment_returning) + .map_err(into_error) + .caused_by(trc::location!())? + .query_row(params![&key, &by], |row| row.get::<_, i64>(0)) + .map_err(into_error) + .caused_by(trc::location!()) +} + +fn is_busy(err: &rusqlite::Error) -> bool { + matches!( + err, + rusqlite::Error::SqliteFailure(err, _) + if matches!(err.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) + ) +} diff --git a/crates/store/src/build/cluster.rs b/crates/store/src/build/cluster.rs index ad328a7d..0eb049a7 100644 --- a/crates/store/src/build/cluster.rs +++ b/crates/store/src/build/cluster.rs @@ -5,14 +5,15 @@ */ use crate::{ - IterateParams, RegistryStore, RegistryStoreInner, Store, U16_LEN, U32_LEN, U64_LEN, ValueKey, + IterateParams, RegistryStore, RegistryStoreInner, Store, U64_LEN, ValueKey, write::{ - BatchBuilder, ValueClass, + BatchBuilder, MergeResult, ValueClass, assert::AssertValue, - key::{DeserializeBigEndian, KeySerializer}, + key::{KeySerializer, node_id_from_key}, now, }, }; +use compact_str::CompactString; use registry::{ schema::{enums::ClusterNodeStatus, structs::ClusterNode}, types::datetime::UTCDateTime, @@ -38,6 +39,11 @@ struct NodeClaim { assert: AssertValue, } +struct NodeLease<'x> { + last_renewal: u64, + hostname: &'x str, +} + impl RegistryStoreInner { pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> { let mut retry_count = 0; @@ -53,13 +59,14 @@ impl RegistryStoreInner { .assert_value(ValueClass::NodeId(claim.node_id), claim.assert) .set( ValueClass::NodeId(claim.node_id), - KeySerializer::new(self.env_hostname.len() + U64_LEN) - .write(now) - .write(&self.env_hostname) - .finalize(), + NodeLease { + last_renewal: now, + hostname: &self.env_hostname, + } + .serialize(), ); - match self.store.write(batch.build_all()).await { + match self.store.write_batch(&mut batch).await { Ok(_) => { self.node_id = claim.node_id; break slots; @@ -107,35 +114,21 @@ impl RegistryStore { pub async fn refresh_node_id_lease(&self) -> trc::Result<()> { let node_id = self.0.node_id; - let assert = match NodeSlot::list(&self.0.store, now()) - .await - .caused_by(trc::location!())? - .into_iter() - .find(|slot| slot.node_id == node_id) - { - Some(slot) if slot.is_owned_by(&self.0.env_hostname) => AssertValue::Hash(slot.hash), - Some(slot) => { - return Err(trc::StoreEvent::AssertValueFailed - .into_err() - .details("Node id lease is held by another host") - .ctx(trc::Key::Id, node_id) - .ctx(trc::Key::Hostname, slot.hostname)); - } - None => AssertValue::None, - }; + let hostname = self.0.env_hostname.clone(); + let last_renewal = now(); let mut batch = BatchBuilder::new(); - batch.assert_value(ValueClass::NodeId(node_id), assert).set( - ValueClass::NodeId(node_id), - KeySerializer::new(self.0.env_hostname.len() + U64_LEN) - .write(now()) - .write(&self.0.env_hostname) - .finalize(), - ); + batch.merge_fnc(ValueClass::NodeId(node_id), move |_, value| { + NodeLease { + last_renewal, + hostname: &hostname, + } + .renew(node_id, value) + }); self.0 .store - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!()) .map(|_| ()) @@ -175,31 +168,23 @@ impl NodeSlot { ) .ascending(), |key, value| { - if key.len() == U16_LEN * 3 { - let node_id = key.deserialize_be_u16(U32_LEN)?; - - match ( - value.deserialize_be_u64(0), - value - .get(U64_LEN..) - .and_then(|bytes| std::str::from_utf8(bytes).ok()) - .filter(|text| !text.is_empty()), - ) { - (Ok(last_renewal), Some(hostname)) => { + if let Some(node_id) = node_id_from_key(key) { + match NodeLease::parse(value) { + Some(lease) => { slots.push(NodeSlot { node_id, - hostname: hostname.to_string(), - last_renewal, - elapsed: now.saturating_sub(last_renewal), + hostname: lease.hostname.to_string(), + last_renewal: lease.last_renewal, + elapsed: now.saturating_sub(lease.last_renewal), hash: xxhash_rust::xxh3::xxh3_64(value), }); } - _ => { + None => { trc::error!( trc::StoreEvent::DataCorruption .into_err() .details("Invalid node id lease") - .ctx(trc::Key::Id, node_id) + .ctx(trc::Key::NodeId, node_id) ); } } @@ -258,23 +243,31 @@ impl NodeSlot { store: &Store, slots: impl Iterator, ) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); for slot in slots { - let mut batch = BatchBuilder::new(); - batch - .assert_value( - ValueClass::NodeId(slot.node_id), - AssertValue::Hash(slot.hash), - ) - .clear(ValueClass::NodeId(slot.node_id)); - - if let Err(err) = store.write(batch.build_all()).await - && !err.is_assertion_failure() - { - return Err(err.caused_by(trc::location!())); - } + let hash = slot.hash; + batch.merge_fnc(ValueClass::NodeId(slot.node_id), move |_, value| { + Ok(NodeSlot::release_if_unchanged(hash, value)) + }); } - Ok(()) + if batch.is_empty() { + Ok(()) + } else { + store + .write_batch(&mut batch) + .await + .caused_by(trc::location!()) + .map(|_| ()) + } + } + + fn release_if_unchanged(hash: u64, value: Option<&[u8]>) -> MergeResult { + if value.is_some_and(|value| xxhash_rust::xxh3::xxh3_64(value) == hash) { + MergeResult::Delete + } else { + MergeResult::Skip + } } fn is_owned_by(&self, hostname: &str) -> bool { @@ -304,6 +297,50 @@ impl NodeSlot { } } +impl<'x> NodeLease<'x> { + fn parse(value: &'x [u8]) -> Option { + let (last_renewal, hostname) = value.split_first_chunk::()?; + let hostname = std::str::from_utf8(hostname) + .ok() + .filter(|hostname| !hostname.is_empty())?; + + Some(NodeLease { + last_renewal: u64::from_be_bytes(*last_renewal), + hostname, + }) + } + + fn serialize(&self) -> Vec { + KeySerializer::new(self.hostname.len() + U64_LEN) + .write(self.last_renewal) + .write(self.hostname) + .finalize() + } + + fn renew(&self, node_id: u16, current: Option<&[u8]>) -> trc::Result { + if let Some(current) = current { + match NodeLease::parse(current) { + Some(lease) if lease.hostname == self.hostname => {} + Some(lease) => { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .details("Node id lease is held by another host") + .ctx(trc::Key::NodeId, node_id) + .ctx(trc::Key::Hostname, CompactString::from(lease.hostname))); + } + None => { + return Err(trc::StoreEvent::AssertValueFailed + .into_err() + .details("Invalid node id lease") + .ctx(trc::Key::NodeId, node_id)); + } + } + } + + Ok(MergeResult::Update(self.serialize())) + } +} + impl From for ClusterNode { fn from(slot: NodeSlot) -> Self { ClusterNode { @@ -314,3 +351,4 @@ impl From for ClusterNode { } } } + diff --git a/crates/store/src/build/data.rs b/crates/store/src/build/data.rs index 591e8a13..b124f0ae 100644 --- a/crates/store/src/build/data.rs +++ b/crates/store/src/build/data.rs @@ -4,11 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Store, registry::bootstrap::Bootstrap}; -use registry::schema::{ - prelude::ObjectType, - structs::{DataStore, MetricsStore, TracingStore}, -}; +use crate::Store; +use registry::schema::structs::DataStore; #[allow(unreachable_patterns)] impl Store { @@ -33,63 +30,4 @@ impl Store { } } - pub async fn build_tracing(bp: &mut Bootstrap) -> Option { - let result = match bp.setting_infallible::().await { - TracingStore::Disabled => Ok(None), - TracingStore::Default => Ok(Some(bp.data_store.clone())), - #[cfg(feature = "foundation")] - TracingStore::FoundationDb(store) => { - crate::backend::foundationdb::FdbStore::open(store) - .await - .map(Some) - } - #[cfg(feature = "postgres")] - TracingStore::PostgreSql(store) => crate::backend::postgres::PostgresStore::open(store) - .await - .map(Some), - #[cfg(feature = "mysql")] - TracingStore::MySql(store) => crate::backend::mysql::MysqlStore::open(store) - .await - .map(Some), - _ => Err("Binary was not compiled with the selected tracing store backend".to_string()), - }; - - match result { - Ok(store) => store, - Err(err) => { - bp.build_warning(ObjectType::TracingStore.singleton(), err); - None - } - } - } - - pub async fn build_metrics(bp: &mut Bootstrap) -> Option { - let result = match bp.setting_infallible::().await { - MetricsStore::Disabled => Ok(None), - MetricsStore::Default => Ok(Some(bp.data_store.clone())), - #[cfg(feature = "foundation")] - MetricsStore::FoundationDb(store) => { - crate::backend::foundationdb::FdbStore::open(store) - .await - .map(Some) - } - #[cfg(feature = "postgres")] - MetricsStore::PostgreSql(store) => crate::backend::postgres::PostgresStore::open(store) - .await - .map(Some), - #[cfg(feature = "mysql")] - MetricsStore::MySql(store) => crate::backend::mysql::MysqlStore::open(store) - .await - .map(Some), - _ => Err("Binary was not compiled with the selected metrics store backend".to_string()), - }; - - match result { - Ok(store) => store, - Err(err) => { - bp.build_warning(ObjectType::MetricsStore.singleton(), err); - None - } - } - } } diff --git a/crates/store/src/build/lookup.rs b/crates/store/src/build/lookup.rs index 495f5d48..38b696f1 100644 --- a/crates/store/src/build/lookup.rs +++ b/crates/store/src/build/lookup.rs @@ -5,7 +5,15 @@ */ use crate::{LookupStores, registry::bootstrap::Bootstrap}; -use registry::schema::structs::{LookupStore, StoreLookup}; +#[cfg(any( + feature = "postgres", + feature = "mysql", + feature = "sqlite", + feature = "enterprise", + feature = "redis" +))] +use registry::schema::structs::LookupStore; +use registry::schema::structs::StoreLookup; use std::collections::hash_map::Entry; impl LookupStores { @@ -28,18 +36,18 @@ impl LookupStores { LookupStore::PostgreSql(postgre_sql_store) => { crate::backend::postgres::PostgresStore::open(postgre_sql_store) .await - .map(crate::InMemoryStore::Store) + .map(crate::InMemoryStore::Sql) } #[cfg(feature = "mysql")] LookupStore::MySql(my_sql_store) => { crate::backend::mysql::MysqlStore::open(my_sql_store) .await - .map(crate::InMemoryStore::Store) + .map(crate::InMemoryStore::Sql) } #[cfg(feature = "sqlite")] LookupStore::Sqlite(sqlite_store) => { crate::backend::sqlite::SqliteStore::open(sqlite_store) - .map(crate::InMemoryStore::Store) + .map(crate::InMemoryStore::Sql) } #[cfg(feature = "redis")] LookupStore::Redis(redis_store) => { @@ -49,6 +57,10 @@ impl LookupStores { LookupStore::RedisCluster(redis_cluster_store) => { crate::backend::redis::RedisStore::open_cluster(redis_cluster_store).await } + #[cfg(feature = "redis")] + LookupStore::RedisSentinel(redis_sentinel_store) => { + crate::backend::redis::RedisStore::open_sentinel(redis_sentinel_store).await + } _ => Err( "Binary was not compiled with the selected lookup store backend".to_string(), ), diff --git a/crates/store/src/build/memory.rs b/crates/store/src/build/memory.rs index 754b6351..2930ee2c 100644 --- a/crates/store/src/build/memory.rs +++ b/crates/store/src/build/memory.rs @@ -4,15 +4,26 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{InMemoryStore, registry::bootstrap::Bootstrap}; +use crate::{InMemoryStore, backend::local::LocalStore, registry::bootstrap::Bootstrap}; use registry::schema::{prelude::ObjectType, structs}; +use std::sync::Arc; #[allow(unreachable_patterns)] impl InMemoryStore { pub async fn build(bp: &mut Bootstrap) -> Option { let result = match bp.setting_infallible::().await { structs::InMemoryStore::Default => { - return Some(InMemoryStore::Store(bp.data_store.clone())); + if matches!( + bp.setting_infallible::().await, + structs::Coordinator::Disabled + ) { + return Some(InMemoryStore::Local(Arc::new(LocalStore::default()))); + } + Err(concat!( + "The local in-memory store cannot be used when a cluster coordinator ", + "is configured, use a Redis in-memory store or disable the coordinator" + ) + .to_string()) } #[cfg(feature = "redis")] structs::InMemoryStore::Redis(redis_store) => { diff --git a/crates/store/src/build/mod.rs b/crates/store/src/build/mod.rs index 0a140869..ab4f83ca 100644 --- a/crates/store/src/build/mod.rs +++ b/crates/store/src/build/mod.rs @@ -11,3 +11,4 @@ pub mod lookup; pub mod memory; pub mod registry; pub mod search; +pub mod swap; diff --git a/crates/store/src/build/registry.rs b/crates/store/src/build/registry.rs index ac622a42..d9f235f5 100644 --- a/crates/store/src/build/registry.rs +++ b/crates/store/src/build/registry.rs @@ -128,38 +128,5 @@ impl RegistryStore { inner.store = store; inner } - - #[cfg(feature = "test_mode")] - pub fn clone_with_public_url(&self, url: String) -> Self { - let mut inner = self.0.as_ref().clone(); - inner.env_public_url = Some(url); - Self(inner.into()) - } - - #[cfg(feature = "test_mode")] - pub async fn new( - path: &str, - store: Store, - hostname: String, - push_shard_id: u32, - cluster_role: Option, - ) -> Self { - Self::from_inner( - RegistryStoreInner { - local_path: PathBuf::from(path), - store, - node_id: 0, - env_recovery_mode: false, - env_recovery_admin: Some(("admin".to_string(), "popolna_zapora".to_string())), - env_cluster_role: cluster_role, - env_push_shard_id: push_shard_id, - env_hostname: hostname, - env_public_url: None, - id_generator: utils::snowflake::SnowflakeIdGenerator::new(), - }, - true, - ) - .await - .unwrap() - } } + diff --git a/crates/store/src/build/search.rs b/crates/store/src/build/search.rs index 608b958a..40bfc19b 100644 --- a/crates/store/src/build/search.rs +++ b/crates/store/src/build/search.rs @@ -16,7 +16,13 @@ impl SearchStore { pub async fn build(bp: &mut Bootstrap) -> Option { let result = match bp.setting_infallible::().await { structs::SearchStore::Default => { - return Some(SearchStore::Store(bp.data_store.clone())); + return Some(match bp.data_store.clone() { + #[cfg(feature = "postgres")] + crate::Store::PostgreSQL(store) => SearchStore::PostgreSQL(store), + #[cfg(feature = "mysql")] + crate::Store::MySQL(store) => SearchStore::MySQL(store), + store => SearchStore::Store(store), + }); } structs::SearchStore::ElasticSearch(elastic_search_store) => { ElasticSearchStore::open(elastic_search_store).await @@ -34,13 +40,19 @@ impl SearchStore { structs::SearchStore::PostgreSql(postgre_sql_store) => { crate::backend::postgres::PostgresStore::open(postgre_sql_store) .await - .map(SearchStore::Store) + .map(|store| match store { + crate::Store::PostgreSQL(store) => SearchStore::PostgreSQL(store), + _ => unreachable!(), + }) } #[cfg(feature = "mysql")] structs::SearchStore::MySql(my_sql_store) => { crate::backend::mysql::MysqlStore::open(my_sql_store) .await - .map(SearchStore::Store) + .map(|store| match store { + crate::Store::MySQL(store) => SearchStore::MySQL(store), + _ => unreachable!(), + }) } _ => Err("Binary was not compiled with the selected search store backend".to_string()), }; diff --git a/crates/store/src/build/swap.rs b/crates/store/src/build/swap.rs new file mode 100644 index 00000000..bbff42df --- /dev/null +++ b/crates/store/src/build/swap.rs @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{BlobStore, InMemoryStore, registry::bootstrap::Bootstrap}; +use registry::schema::structs; + +const NO_FILESYSTEM: &str = "The filesystem blob store cannot back the cache swap tier because \ + it skips writes whose length is unchanged, which silently discards \ + a snapshot after a flag change. Configure the local file cache swap \ + backend instead"; + +#[allow(unreachable_patterns)] +impl BlobStore { + pub async fn build_cache_swap( + _bp: &mut Bootstrap, + config: structs::BlobSwapStore, + configured: &BlobStore, + ) -> Result { + match config { + structs::BlobSwapStore::Default => { + if matches!(configured, BlobStore::Fs(_)) { + Err(NO_FILESYSTEM.to_string()) + } else { + Ok(configured.clone()) + } + } + #[cfg(feature = "foundation")] + structs::BlobSwapStore::FoundationDb(store) => { + crate::backend::foundationdb::FdbStore::open(store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "postgres")] + structs::BlobSwapStore::PostgreSql(store) => { + crate::backend::postgres::PostgresStore::open(store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "mysql")] + structs::BlobSwapStore::MySql(store) => crate::backend::mysql::MysqlStore::open(store) + .await + .map(BlobStore::Store), + #[cfg(feature = "s3")] + structs::BlobSwapStore::S3(store) => crate::backend::s3::S3Store::open(store).await, + #[cfg(feature = "azure")] + structs::BlobSwapStore::Azure(store) => { + crate::backend::azure::AzureStore::open(store).await + } + structs::BlobSwapStore::FileSystem(_) => Err(NO_FILESYSTEM.to_string()), + _ => { + Err("Binary was not compiled with the selected cache swap blob backend".to_string()) + } + } + } +} + +#[allow(unreachable_patterns)] +impl InMemoryStore { + pub async fn build_cache_swap( + _bp: &mut Bootstrap, + config: structs::RedisSwapStore, + configured: &InMemoryStore, + ) -> Result { + match config { + structs::RedisSwapStore::Default => { + if configured.supports_chunked_keys() { + Ok(configured.clone()) + } else { + Err( + "The default cache swap store requires a Redis in-memory backend, \ + either directly or as every shard of a sharded store" + .to_string(), + ) + } + } + #[cfg(feature = "redis")] + structs::RedisSwapStore::Redis(store) => { + crate::backend::redis::RedisStore::open_single(store).await + } + #[cfg(feature = "redis")] + structs::RedisSwapStore::RedisCluster(store) => { + crate::backend::redis::RedisStore::open_cluster(store).await + } + #[cfg(feature = "redis")] + structs::RedisSwapStore::RedisSentinel(store) => { + crate::backend::redis::RedisStore::open_sentinel(store).await + } + _ => Err( + "Binary was not compiled with the selected cache swap Redis backend".to_string(), + ), + } + } +} diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index 8e01ac86..331667b0 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -5,7 +5,7 @@ */ use crate::{BlobStore, CompressionAlgo, Store, U32_LEN}; -use std::{ops::Range, time::Instant}; +use std::{borrow::Cow, ops::Range, time::Instant}; use trc::{AddContext, StoreEvent}; const MAGIC_MARKER: u8 = 0xa0; @@ -13,10 +13,84 @@ const LZ4_MARKER: u8 = MAGIC_MARKER | 0x01; //const ZSTD_MARKER: u8 = MAGIC_MARKER | 0x02; const NONE_MARKER: u8 = 0x00; +fn decode<'x>(key: &[u8], data: &'x [u8]) -> trc::Result> { + match data.split_last() { + Some((&LZ4_MARKER, payload)) => lz4_flex::decompress_size_prepended(payload) + .map(Cow::Owned) + .map_err(|err| { + trc::StoreEvent::DecompressError + .reason(err) + .ctx(trc::Key::Key, key) + .ctx(trc::Key::CausedBy, trc::location!()) + }), + Some((&NONE_MARKER, payload)) => Ok(Cow::Borrowed(payload)), + Some(_) => { + trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key); + Ok(Cow::Borrowed(data)) + } + None => Ok(Cow::Borrowed(data)), + } +} + +fn store_uncompressed(data: &[u8]) -> Vec { + let mut uncompressed = Vec::with_capacity(data.len() + 1); + uncompressed.extend_from_slice(data); + uncompressed.push(NONE_MARKER); + uncompressed +} + impl BlobStore { pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { + #[cfg(stalwart_tests)] + { + crate::dispatch::StoreOps::count_blob_get(); + crate::dispatch::ops::wait(crate::dispatch::Latency::BlobGet).await; + } + let start_time = Instant::now(); - let result = match &self { + let result = self.get_raw_blob(key).await.caused_by(trc::location!())?; + + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_blob_get_bytes(&result); + + trc::event!( + Store(StoreEvent::BlobRead), + Key = key, + Elapsed = start_time.elapsed(), + Size = result.as_ref().map_or(0, |data| data.len()), + ); + + let Some(mut data) = result else { + return Ok(None); + }; + + let mut data = match decode(key, &data)? { + Cow::Owned(decompressed) => decompressed, + Cow::Borrowed(payload) => { + let len = payload.len(); + data.truncate(len); + data + } + }; + + if range.start == 0 { + if range.end > data.len() { + Ok(Some(data)) + } else { + data.truncate(range.end); + Ok(Some(data)) + } + } else { + Ok(Some( + data.get(range.start..range.end) + .unwrap_or_default() + .to_vec(), + )) + } + } + + async fn get_raw_blob(&self, key: &[u8]) -> trc::Result>> { + match &self { BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await, @@ -37,59 +111,6 @@ impl BlobStore { #[cfg(feature = "azure")] BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await, } - .caused_by(trc::location!())?; - - trc::event!( - Store(StoreEvent::BlobRead), - Key = key, - Elapsed = start_time.elapsed(), - Size = result.as_ref().map_or(0, |data| data.len()), - ); - - let Some(mut data) = result else { - return Ok(None); - }; - - let mut data = match data.last().copied() { - Some(LZ4_MARKER) => { - lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default()) - .map_err(|err| { - trc::StoreEvent::DecompressError - .reason(err) - .ctx(trc::Key::Key, key) - .ctx(trc::Key::CausedBy, trc::location!()) - })? - } - Some(NONE_MARKER) => { - if !data.is_empty() { - data.truncate(data.len() - 1); - } - data - } - Some(_) => { - trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key); - - data - } - None => { - return Ok(Some(data)); - } - }; - - if range.start == 0 { - if range.end > data.len() { - Ok(Some(data)) - } else { - data.truncate(range.end); - Ok(Some(data)) - } - } else { - Ok(Some( - data.get(range.start..range.end) - .unwrap_or_default() - .to_vec(), - )) - } } pub async fn put_blob( @@ -99,12 +120,6 @@ impl BlobStore { compression: CompressionAlgo, ) -> trc::Result<()> { let data = match compression { - CompressionAlgo::None => { - let mut uncompressed = Vec::with_capacity(data.len() + 1); - uncompressed.extend_from_slice(data); - uncompressed.push(NONE_MARKER); - uncompressed - } CompressionAlgo::Lz4 => { let mut compressed = vec![ @@ -116,20 +131,32 @@ impl BlobStore { let compressed_len = lz4_flex::compress_into(data, &mut compressed[U32_LEN..]).unwrap(); - // Prepend the length of the uncompressed data - compressed[..U32_LEN].copy_from_slice(&(data.len() as u32).to_le_bytes()); + if compressed_len + U32_LEN < data.len() { + // Prepend the length of the uncompressed data + compressed[..U32_LEN].copy_from_slice(&(data.len() as u32).to_le_bytes()); - // Truncate to the actual size - compressed.truncate(compressed_len + U32_LEN + 1); - compressed + // Truncate to the actual size + compressed.truncate(compressed_len + U32_LEN + 1); + compressed + } else { + store_uncompressed(data) + } } + CompressionAlgo::None => store_uncompressed(data), }; + #[cfg(stalwart_tests)] + { + crate::dispatch::StoreOps::count_blob_put(data.len()); + crate::dispatch::ops::wait(crate::dispatch::Latency::BlobPut).await; + } + let start_time = Instant::now(); + let size = data.len(); let result = match &self { BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] - Store::SQLite(store) => store.put_blob(key, &data).await, + Store::SQLite(store) => store.put_blob(key, data).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.put_blob(key, &data).await, #[cfg(feature = "postgres")] @@ -137,15 +164,15 @@ impl BlobStore { #[cfg(feature = "mysql")] Store::MySQL(store) => store.put_blob(key, &data).await, #[cfg(feature = "rocks")] - Store::RocksDb(store) => store.put_blob(key, &data).await, + Store::RocksDb(store) => store.put_blob(key, data).await, Store::Ephemeral(store) => store.put_blob(key, &data).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobStore::Fs(store) => store.put_blob(key, &data).await, #[cfg(feature = "s3")] - BlobStore::S3(store) => store.put_blob(key, &data).await, + BlobStore::S3(store) => store.put_blob(key, data).await, #[cfg(feature = "azure")] - BlobStore::Azure(store) => store.put_blob(key, &data).await, + BlobStore::Azure(store) => store.put_blob(key, data).await, } .caused_by(trc::location!()); @@ -153,13 +180,16 @@ impl BlobStore { Store(StoreEvent::BlobWrite), Key = key, Elapsed = start_time.elapsed(), - Size = data.len(), + Size = size, ); result } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_blob_delete(); + let start_time = Instant::now(); let result = match &self { BlobStore::Store(store) => match store { @@ -192,4 +222,42 @@ impl BlobStore { result } + + pub async fn delete_blobs(&self, keys: &[&[u8]]) -> trc::Result<()> { + match self { + #[cfg(feature = "rocks")] + BlobStore::Store(Store::RocksDb(store)) => { + delete_blobs_timed(keys, store.delete_blobs(keys)).await + } + #[cfg(feature = "s3")] + BlobStore::S3(store) => delete_blobs_timed(keys, store.delete_blobs(keys)).await, + _ => { + for key in keys { + self.delete_blob(key).await?; + } + Ok(()) + } + } + } } + +#[cfg(any(feature = "rocks", feature = "s3"))] +async fn delete_blobs_timed( + keys: &[&[u8]], + delete: impl Future>, +) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_blob_delete(); + + let start_time = Instant::now(); + let result = delete.await.caused_by(trc::location!()); + + trc::event!( + Store(StoreEvent::BlobWrite), + Total = keys.len(), + Elapsed = start_time.elapsed(), + ); + + result +} + diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 8f2f957b..9981ec2c 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -4,114 +4,89 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{ + Deserialize, InMemoryStore, Store, U64_LEN, Value, + backend::{http::lookup::HttpStoreGet, memory::StaticMemoryStore}, + write::now, +}; use registry::schema::structs::Rate; use std::borrow::Cow; use trc::AddContext; -#[allow(unused_imports)] -use crate::{ - Deserialize, InMemoryStore, IterateParams, QueryResult, Store, U64_LEN, Value, ValueKey, - write::{ - BatchBuilder, Operation, ValueClass, ValueOp, - key::{DeserializeBigEndian, KeySerializer}, - now, - }, -}; -use crate::{ - SerializeInfallible, - backend::{http::lookup::HttpStoreGet, memory::StaticMemoryStore}, - write::{InMemoryClass, assert::AssertValue}, -}; - pub struct KeyValue { pub key: Vec, pub value: T, pub expires: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RateWindow { + pub index: u64, + pub expires_in: u64, +} + +impl RateWindow { + pub fn new(rate: &Rate, now: u64) -> Self { + let period = rate.period.as_secs().max(1); + RateWindow { + index: now / period, + expires_in: period - now % period, + } + } + + pub fn bucket(&self, prefix: u8, key: &[u8]) -> Vec { + let mut bucket = Vec::with_capacity(key.len() + U64_LEN + 1); + bucket.push(prefix); + bucket.extend_from_slice(key); + bucket.extend_from_slice(&self.index.to_be_bytes()); + bucket + } +} + impl InMemoryStore { pub async fn key_set(&self, kv: KeyValue>) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - let mut batch = BatchBuilder::new(); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(kv.key)), - op: ValueOp::Set( - KeySerializer::new(kv.value.len() + U64_LEN) - .write(kv.expires.map_or(u64::MAX, |expires| now() + expires)) - .write(kv.value.as_slice()) - .finalize(), - ), - }); - store.write(batch.build_all()).await.map(|_| ()) - } + InMemoryStore::Local(store) => store.key_set(kv.key, kv.value, kv.expires, now()), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } - pub async fn counter_incr(&self, kv: KeyValue, return_value: bool) -> trc::Result { + pub async fn key_incr(&self, kv: KeyValue) -> trc::Result { + self.key_incr_at(kv, now()).await + } + + async fn key_incr_at(&self, kv: KeyValue, now: u64) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - let mut batch = BatchBuilder::new(); - - if let Some(expires) = kv.expires { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(kv.key.clone())), - op: ValueOp::Set( - KeySerializer::new(U64_LEN * 2) - .write(0u64) - .write(now() + expires) - .finalize(), - ), - }); - } - - if return_value { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), - op: ValueOp::AddAndGet(kv.value), - }); - - store - .write(batch.build_all()) - .await - .and_then(|r| r.last_counter_id()) - } else { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), - op: ValueOp::AtomicAdd(kv.value), - }); - - store.write(batch.build_all()).await.map(|_| 0) - } - } + InMemoryStore::Local(store) => store.key_incr(kv.key, kv.value, kv.expires, now), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } - pub async fn key_delete(&self, key: impl Into>) -> trc::Result<()> { + pub async fn key_delete(&self, key: impl Into>) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - let mut batch = BatchBuilder::new(); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key.into().into_bytes())), - op: ValueOp::Clear, - }); - store.write(batch.build_all()).await.map(|_| ()) - } + InMemoryStore::Local(store) => Ok(store.key_delete(key.into().as_bytes(), now())), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } @@ -119,18 +94,19 @@ impl InMemoryStore { } pub async fn counter_delete(&self, key: impl Into>) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - let mut batch = BatchBuilder::new(); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Counter(key.into().into_bytes())), - op: ValueOp::Clear, - }); - store.write(batch.build_all()).await.map(|_| ()) + InMemoryStore::Local(store) => { + store.key_delete(key.into().as_bytes(), now()); + Ok(()) } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Redis(store) => { + store.key_delete(key.into().as_bytes()).await.map(|_| ()) + } + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } @@ -138,38 +114,17 @@ impl InMemoryStore { } pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - if prefix.is_empty() { - return Ok(()); - } - - let from_range = prefix.to_vec(); - let mut to_range = Vec::with_capacity(prefix.len() + 3); - to_range.extend_from_slice(prefix); - to_range.extend_from_slice([u8::MAX, u8::MAX, u8::MAX].as_ref()); - - store - .delete_range( - ValueKey::from(ValueClass::InMemory(InMemoryClass::Counter( - from_range.clone(), - ))), - ValueKey::from(ValueClass::InMemory(InMemoryClass::Counter( - to_range.clone(), - ))), - ) - .await?; - - store - .delete_range( - ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(from_range))), - ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(to_range))), - ) - .await + InMemoryStore::Local(store) => { + store.key_delete_prefix(prefix); + Ok(()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } @@ -180,13 +135,11 @@ impl InMemoryStore { &self, key: impl Into>, ) -> trc::Result> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => store - .get_value::>(ValueKey::from(ValueClass::InMemory( - InMemoryClass::Key(key.into().into_bytes()), - ))) - .await - .map(|value| value.and_then(|v| v.into())), + InMemoryStore::Local(store) => store.key_get(key.into().as_bytes(), now()), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_get(key.into().as_bytes()).await, InMemoryStore::Static(store) => Ok(match store.as_ref() { @@ -204,36 +157,17 @@ impl InMemoryStore { InMemoryStore::Http(store) => { Ok(store.get(key.into().as_str()).map(|value| T::from(value))) } - } - .caused_by(trc::location!()) - } - - pub async fn counter_get(&self, key: impl Into>) -> trc::Result { - match self { - InMemoryStore::Store(store) => { - store - .get_counter(ValueKey::from(ValueClass::InMemory( - InMemoryClass::Counter(key.into().into_bytes()), - ))) - .await - } - #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.counter_get(key.into().as_bytes()).await, - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { - Err(trc::StoreEvent::NotSupported.into_err()) - } + InMemoryStore::Sql(_) => Err(trc::StoreEvent::NotSupported.into_err()), } .caused_by(trc::location!()) } pub async fn key_exists(&self, key: impl Into>) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => store - .get_value::>(ValueKey::from(ValueClass::InMemory( - InMemoryClass::Key(key.into().into_bytes()), - ))) - .await - .map(|value| matches!(value, Some(LookupValue::Value(Empty)))), + InMemoryStore::Local(store) => Ok(store.key_exists(key.into().as_bytes(), now())), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_exists(key.into().as_bytes()).await, InMemoryStore::Static(store) => Ok(match store.as_ref() { @@ -241,6 +175,7 @@ impl InMemoryStore { StaticMemoryStore::Set(set) => set.contains(key.into().as_str()), }), InMemoryStore::Http(store) => Ok(store.contains(key.into().as_str())), + InMemoryStore::Sql(_) => Err(trc::StoreEvent::NotSupported.into_err()), } .caused_by(trc::location!()) } @@ -250,89 +185,37 @@ impl InMemoryStore { prefix: u8, key: &[u8], rate: &Rate, - soft_check: bool, ) -> trc::Result> { let now = now(); - let period = rate.period.as_secs().max(1); - let range_start = now / period; - let range_end = (range_start * period) + period; - let expires_in = range_end - now; + let window = RateWindow::new(rate, now); - let mut bucket = Vec::with_capacity(key.len() + U64_LEN + 1); - bucket.push(prefix); - bucket.extend_from_slice(key); - bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); - - let requests = if !soft_check { - self.counter_incr(KeyValue::new(bucket, 1).expires(expires_in), true) - .await - .caused_by(trc::location!())? - } else { - self.counter_get(bucket).await.caused_by(trc::location!())? + 1 - }; + let requests = self + .key_incr_at( + KeyValue::new(window.bucket(prefix, key), 1).expires(window.expires_in), + now, + ) + .await + .caused_by(trc::location!())?; if requests <= rate.count as i64 { Ok(None) } else { - Ok(Some(expires_in)) + Ok(Some(window.expires_in)) } } + pub async fn rate_reset(&self, prefix: u8, key: &[u8], rate: &Rate) -> trc::Result<()> { + self.counter_delete(RateWindow::new(rate, now()).bucket(prefix, key)) + .await + } + pub async fn try_lock(&self, prefix: u8, key: &[u8], duration: u64) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + match self { - InMemoryStore::Store(store) => { - let key = KeyValue::<()>::build_key(prefix, key); - let lock_expiry = match store - .get_value::(ValueKey::from(ValueClass::InMemory(InMemoryClass::Key( - key.clone(), - )))) - .await - { - Ok(lock_expiry) => lock_expiry, - Err(err) - if err.matches(trc::EventType::Store(trc::StoreEvent::DataCorruption)) => - { - // TODO remove in 1.0 - let mut batch = BatchBuilder::new(); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key.clone())), - op: ValueOp::Clear, - }); - store - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - None - } - Err(err) => { - return Err(err - .details("Failed to read lock.") - .caused_by(trc::location!())); - } - }; - - let now = now(); - if lock_expiry.is_some_and(|expiry| expiry > now) { - return Ok(false); - } - - let key: ValueClass = ValueClass::InMemory(InMemoryClass::Key(key)); - let mut batch = BatchBuilder::new(); - batch.assert_value( - key.clone(), - match lock_expiry { - Some(value) => AssertValue::U64(value), - None => AssertValue::None, - }, - ); - batch.set(key.clone(), (now + duration).serialize()); - match store.write(batch.build_all()).await { - Ok(_) => Ok(true), - Err(err) if err.is_assertion_failure() => Ok(false), - Err(err) => Err(err - .details("Failed to lock event.") - .caused_by(trc::location!())), - } + InMemoryStore::Local(store) => { + store.try_lock(KeyValue::<()>::build_key(prefix, key), duration, now()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => { @@ -340,7 +223,84 @@ impl InMemoryStore { .try_lock(&KeyValue::<()>::build_key(prefix, key), duration) .await } - InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + Err(trc::StoreEvent::NotSupported.into_err()) + } + } + } + + pub async fn renew_lock(&self, prefix: u8, key: &[u8], duration: u64) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + + match self { + InMemoryStore::Local(store) => { + store.renew_lock(KeyValue::<()>::build_key(prefix, key), duration, now()) + } + #[cfg(feature = "redis")] + InMemoryStore::Redis(store) => { + store + .renew_lock(&KeyValue::<()>::build_key(prefix, key), duration) + .await + } + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + Err(trc::StoreEvent::NotSupported.into_err()) + } + } + } + + pub async fn try_lease( + &self, + prefix: u8, + key: &[u8], + token: u64, + duration: u64, + ) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + + let key = KeyValue::<()>::build_key(prefix, key); + match self { + InMemoryStore::Local(store) => store.try_lease(key, token, duration, now()), + #[cfg(feature = "redis")] + InMemoryStore::Redis(store) => store.try_lease(&key, token, duration).await, + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + Err(trc::StoreEvent::NotSupported.into_err()) + } + } + } + + pub async fn renew_lease( + &self, + prefix: u8, + key: &[u8], + token: u64, + duration: u64, + ) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + + let key = KeyValue::<()>::build_key(prefix, key); + match self { + InMemoryStore::Local(store) => Ok(store.renew_lease(&key, token, duration, now())), + #[cfg(feature = "redis")] + InMemoryStore::Redis(store) => store.renew_lease(&key, token, duration).await, + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + Err(trc::StoreEvent::NotSupported.into_err()) + } + } + } + + pub async fn release_lease(&self, prefix: u8, key: &[u8], token: u64) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_in_memory(self).await; + + let key = KeyValue::<()>::build_key(prefix, key); + match self { + InMemoryStore::Local(store) => Ok(store.release_lease(key, token)), + #[cfg(feature = "redis")] + InMemoryStore::Redis(store) => store.release_lease(&key, token).await, + InMemoryStore::Sql(_) | InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } @@ -349,99 +309,12 @@ impl InMemoryStore { pub async fn remove_lock(&self, prefix: u8, key: &[u8]) -> trc::Result<()> { self.key_delete(KeyValue::<()>::build_key(prefix, key)) .await - } - - pub async fn purge_in_memory_store(&self) -> trc::Result<()> { - match self { - InMemoryStore::Store(store) => { - // Delete expired keys and counters - let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8]))); - let to_key = - ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10]))); - - let current_time = now(); - let mut expired_keys = Vec::new(); - let mut expired_counters = Vec::new(); - store - .iterate(IterateParams::new(from_key, to_key), |key, value| { - let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?; - if expiry == 0 { - if value - .deserialize_be_u64(U64_LEN) - .caused_by(trc::location!())? - <= current_time - { - expired_counters.push(key.to_vec()); - } - } else if expiry <= current_time { - expired_keys.push(key.to_vec()); - } - Ok(true) - }) - .await - .caused_by(trc::location!())?; - - if !expired_keys.is_empty() { - let mut batch = BatchBuilder::new(); - for key in expired_keys { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key)), - op: ValueOp::Clear, - }); - if batch.is_large_batch() { - store - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - } - if !batch.is_empty() { - store - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - } - - if !expired_counters.is_empty() { - let mut batch = BatchBuilder::new(); - for key in expired_counters { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), - op: ValueOp::Clear, - }); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key)), - op: ValueOp::Clear, - }); - if batch.is_large_batch() { - store - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - } - if !batch.is_empty() { - store - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - } - } - #[cfg(feature = "redis")] - InMemoryStore::Redis(_) => {} - InMemoryStore::Static(_) | InMemoryStore::Http(_) => {} - } - - Ok(()) + .map(|_| ()) } pub fn is_sql(&self) -> bool { match self { - InMemoryStore::Store(store) => store.is_sql(), + InMemoryStore::Sql(store) => store.is_sql(), _ => false, } } @@ -450,14 +323,29 @@ impl InMemoryStore { match self { #[cfg(feature = "redis")] InMemoryStore::Redis(_) => true, - InMemoryStore::Static(_) => false, + _ => false, + } + } + + pub fn is_remote(&self) -> bool { + match self { + #[cfg(feature = "redis")] + InMemoryStore::Redis(_) => true, + _ => false, + } + } + + pub fn supports_chunked_keys(&self) -> bool { + match self { + #[cfg(feature = "redis")] + InMemoryStore::Redis(_) => true, _ => false, } } pub fn into_store(self) -> Option { match self { - InMemoryStore::Store(store) => Some(store), + InMemoryStore::Sql(store) => Some(store), _ => None, } } @@ -519,15 +407,6 @@ impl LookupKey<'_> { } } - pub fn into_bytes(self) -> Vec { - match self { - LookupKey::String(string) => string.into_bytes(), - LookupKey::StringRef(string) => string.as_bytes().to_vec(), - LookupKey::Bytes(bytes) => bytes, - LookupKey::BytesRef(bytes) => bytes.to_vec(), - } - } - pub fn as_bytes(&self) -> &[u8] { match self { LookupKey::String(string) => string.as_bytes(), @@ -574,43 +453,6 @@ impl KeyValue { } } -struct Empty; - -enum LookupValue { - Value(T), - None, -} - -impl Deserialize for LookupValue { - fn deserialize(bytes: &[u8]) -> trc::Result { - bytes.deserialize_be_u64(0).and_then(|expires| { - Ok(if expires > now() { - LookupValue::Value( - T::deserialize(bytes.get(U64_LEN..).unwrap_or_default()) - .caused_by(trc::location!())?, - ) - } else { - LookupValue::None - }) - }) - } -} - -impl Deserialize for Empty { - fn deserialize(_bytes: &[u8]) -> trc::Result { - Ok(Empty) - } -} - -impl From> for Option { - fn from(value: LookupValue) -> Self { - match value { - LookupValue::Value(value) => Some(value), - LookupValue::None => None, - } - } -} - impl From> for String { fn from(value: Value<'static>) -> Self { match value { diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index 205affdc..88033044 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -12,6 +12,37 @@ pub mod lookup; pub mod search; pub mod store; + +#[cfg(stalwart_tests)] +pub use ops::{CommitFailureGuard, Latency, LatencyGuard, OpsSnapshot, StoreOps, SubspaceOps}; + + +#[cfg(stalwart_tests)] +pub use rocksdb_skips::RocksDbSkips; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScanGap { + gap: u32, + ratio: u32, +} + +pub const POINT_SCAN_GAP: ScanGap = ScanGap::new(0, 2); +pub const MAX_SCAN_GAP: ScanGap = ScanGap::new(8, 64); +pub const ROCKSDB_SCAN_GAP: ScanGap = ScanGap::new(8, 17); +const MAX_RANGES_RESERVE: usize = 4096; + +#[derive(Debug, PartialEq, Eq)] +pub enum ScanShape { + Range(u32, u32), + Ranges(Vec<(u32, u32)>), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReadShape { + Ranges(ScanGap), + Keys, +} + impl Store { pub fn id(&self) -> &'static str { match self { @@ -29,6 +60,37 @@ impl Store { Self::None => "none", } } + + pub fn scan_gap(&self) -> ScanGap { + match self { + #[cfg(feature = "postgres")] + Self::PostgreSQL(_) => POINT_SCAN_GAP, + #[cfg(feature = "mysql")] + Self::MySQL(_) => POINT_SCAN_GAP, + #[cfg(feature = "rocks")] + Self::RocksDb(_) => ROCKSDB_SCAN_GAP, + _ => MAX_SCAN_GAP, + } + } + + pub fn read_shape(&self) -> ReadShape { + match self { + #[cfg(feature = "postgres")] + Self::PostgreSQL(_) => ReadShape::Keys, + #[cfg(feature = "mysql")] + Self::MySQL(_) => ReadShape::Keys, + #[cfg(feature = "foundation")] + Self::FoundationDb(_) => ReadShape::Ranges(POINT_SCAN_GAP), + _ => ReadShape::Ranges(self.scan_gap()), + } + } +} + +impl ScanGap { + pub const fn new(gap: u32, ratio: u32) -> Self { + assert!(ratio > gap, "the scan ratio must exceed the gap"); + ScanGap { gap, ratio } + } } #[allow(clippy::len_without_is_empty)] @@ -38,6 +100,52 @@ pub trait DocumentSet: Sync + Send { fn contains(&self, id: u32) -> bool; fn len(&self) -> usize; fn iterate(&self) -> impl Iterator; + + fn is_dense(&self, gap: ScanGap) -> bool { + self.len() as u64 * u64::from(gap.ratio) >= u64::from(self.max() - self.min()) + } + + fn scan_shape(&self, gap: ScanGap) -> ScanShape { + let min = self.min(); + let max = self.max(); + if self.len() == 0 || self.is_dense(gap) { + return ScanShape::Range(min, max); + } + + let ranges = self.scan_ranges(gap); + match ranges.as_slice() { + [] => ScanShape::Range(min, max), + &[(from, to)] => ScanShape::Range(from, to), + _ => ScanShape::Ranges(ranges), + } + } + + fn scan_ranges(&self, gap: ScanGap) -> Vec<(u32, u32)> { + if self.len() == 0 { + return Vec::new(); + } + if self.is_dense(gap) { + let min = self.min(); + return vec![(min, self.max().saturating_sub(1).max(min))]; + } + + let mut ids = self.iterate(); + let Some(first) = ids.next() else { + return Vec::new(); + }; + let mut ranges = Vec::with_capacity(self.len().min(MAX_RANGES_RESERVE)); + let mut range = (first, first); + for id in ids { + if id - range.1 <= gap.gap { + range.1 = id; + } else { + ranges.push(range); + range = (id, id); + } + } + ranges.push(range); + ranges + } } impl DocumentSet for RoaringBitmap { @@ -62,6 +170,28 @@ impl DocumentSet for RoaringBitmap { } } +impl DocumentSet for u32 { + fn min(&self) -> u32 { + *self + } + + fn max(&self) -> u32 { + self.saturating_add(1) + } + + fn contains(&self, id: u32) -> bool { + *self == id + } + + fn len(&self) -> usize { + 1 + } + + fn iterate(&self) -> impl Iterator { + std::iter::once(*self) + } +} + impl DocumentSet for Vec { fn contains(&self, id: u32) -> bool { self.binary_search(&id).is_ok() @@ -104,4 +234,9 @@ impl DocumentSet for () { fn iterate(&self) -> impl Iterator { std::iter::empty() } + + fn scan_ranges(&self, _: ScanGap) -> Vec<(u32, u32)> { + vec![(0, u32::MAX)] + } } + diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index 5e018442..5fbed3bd 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -4,23 +4,50 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +#[cfg(feature = "postgres")] +use crate::backend::postgres::search::DENSE_ACCOUNT_DOCUMENTS; use crate::{ SearchStore, Store, search::{ - IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, - SearchValue, + IndexDocument, QueryResults, SearchField, SearchFilter, SearchQuery, split::{SplitFilter, split_filters}, }, write::SearchIndex, }; +use roaring::{RoaringBitmap, RoaringTreemap}; use std::cmp::Ordering; use trc::AddContext; +pub const MAX_FILTER_CANDIDATES: usize = 16_384; +#[cfg(feature = "postgres")] +const DENSE_QUERY_FIXED_ROWS: f64 = 12_000.0; +#[cfg(feature = "postgres")] +const DENSE_QUERY_ROWS_PER_DOCUMENT: f64 = 0.011; + impl SearchStore { - pub async fn query_account(&self, query: SearchQuery) -> trc::Result> { + pub async fn query_account(&self, mut query: SearchQuery) -> trc::Result> { + let comparators = std::mem::take(&mut query.comparators); + let results = self.filter_account(query).await?; + + Ok(match results.len().cmp(&1) { + Ordering::Equal => vec![results.min().unwrap()], + Ordering::Less => vec![], + Ordering::Greater if !comparators.is_empty() => { + QueryResults::new(results, comparators).into_sorted() + } + Ordering::Greater => results.iter().collect(), + }) + } + + pub async fn filter_account(&self, query: SearchQuery) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_search_query(); + + debug_assert!(query.comparators.is_empty()); + // Pre-filter by mask if query.mask.is_empty() { - return Ok(vec![]); + return Ok(RoaringBitmap::new()); } // If the store does not support FTS, use the internal FTS store @@ -28,23 +55,25 @@ impl SearchStore { return store.query_account(query).await; } - // If all filters and comparators are external, delegate to the underlying store + // If all filters are external, delegate to the underlying store let mut account_id = u32::MAX; let mut has_local_filters = false; let mut has_external_filters = false; for filter in &query.filters { match filter { - SearchFilter::Operator { + SearchFilter::Integer { + value, field: SearchField::AccountId, - op: SearchOperator::Equal, - value: SearchValue::Uint(id), + .. } => { - account_id = *id as u32; + account_id = *value as u32; } SearchFilter::DocumentSet(_) => { has_local_filters = true; } - SearchFilter::Operator { .. } => { + SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } + | SearchFilter::Integer { .. } => { has_external_filters = true; } _ => (), @@ -57,25 +86,15 @@ impl SearchStore { .caused_by(trc::location!())); } - if !has_local_filters && !has_external_filters && query.comparators.is_empty() { - return Ok(query.mask.iter().collect()); + if !has_local_filters && !has_external_filters { + return Ok(query.mask); } - if !has_local_filters && query.comparators.iter().all(|c| c.is_external()) { + if !has_local_filters { return self - .sub_query(query.index, &query.filters, &query.comparators) + .sub_query(query.index, &query.filters, &query.mask) .await - .map(|results| { - if !results.is_empty() || has_external_filters { - results - .into_iter() - .filter(|id| query.mask.contains(*id)) - .collect() - } else { - // Database sort is broken, return masked results - query.mask.iter().collect() - } - }) + .map(|results| results & query.mask) .caused_by(trc::location!()); } @@ -93,10 +112,7 @@ impl SearchStore { SplitFilter::External(external) => { // Execute sub-query filters.push(SearchFilter::DocumentSet( - self.sub_query(query.index, &external, &[]) - .await? - .into_iter() - .collect(), + self.sub_query(query.index, &external, &query.mask).await?, )); } SplitFilter::Internal(filter) => { @@ -111,184 +127,171 @@ impl SearchStore { }; // Merge results locally - let results = SearchQuery::new(query.index) + Ok(SearchQuery::new(query.index) .with_filters(filters) .with_mask(query.mask) - .filter(); + .filter() + .into_bitmap()) + } - let total_results = results.results().len(); - match total_results.cmp(&1) { - Ordering::Equal => Ok(vec![results.results().min().unwrap()]), - Ordering::Less => Ok(vec![]), - Ordering::Greater => { - if !query.comparators.is_empty() { - let mut local = Vec::with_capacity(query.comparators.len()); - let mut external = Vec::with_capacity(query.comparators.len()); - let mut external_first = false; - for (pos, comparator) in query.comparators.into_iter().enumerate() { - if comparator.is_external() { - external.push(comparator); - if pos == 0 { - external_first = true; - } - } else { - local.push(comparator); - } - } - - if !external.is_empty() { - let mut results = results.results().clone(); - let filters = vec![ - SearchFilter::Operator { - field: SearchField::AccountId, - op: SearchOperator::Equal, - value: SearchValue::Uint(account_id as u64), - }, - SearchFilter::Operator { - field: SearchField::DocumentId, - op: SearchOperator::GreaterEqualThan, - value: SearchValue::Uint(results.min().unwrap() as u64), - }, - SearchFilter::Operator { - field: SearchField::DocumentId, - op: SearchOperator::LowerEqualThan, - value: SearchValue::Uint(results.max().unwrap() as u64), - }, - ]; - - let mut ordered_results = Vec::with_capacity(total_results as usize); - for ordered_result in - self.sub_query(query.index, &filters, &external).await? - { - if results.remove(ordered_result) { - ordered_results.push(ordered_result); - } - } - // Add any remaining results not yet in the index - ordered_results.extend(results); - - if local.is_empty() { - return Ok(ordered_results); - } - - let comparator = SearchComparator::SortedSet { - set: ordered_results - .into_iter() - .enumerate() - .map(|(pos, id)| (id, pos as u32)) - .collect(), - ascending: true, - }; - - if external_first { - local.insert(0, comparator); - } else { - local.push(comparator); - } - } - - Ok(results.with_comparators(local).into_sorted()) - } else { - Ok(results.results().iter().collect()) - } + #[cfg_attr(not(feature = "postgres"), allow(unused_variables))] + pub fn account_query_rows(&self, documents: u64) -> Option { + match self { + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(_) if documents >= DENSE_ACCOUNT_DOCUMENTS => { + Some(DENSE_QUERY_FIXED_ROWS + DENSE_QUERY_ROWS_PER_DOCUMENT * documents as f64) } + _ => None, } } + pub fn supports_windowed_walk(&self) -> bool { + match self { + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(_) => true, + _ => false, + } + } + + pub fn supports_candidate_filter(&self) -> bool { + match self { + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(_) => true, + #[cfg(feature = "mysql")] + SearchStore::MySQL(_) => true, + _ => false, + } + } + + pub async fn filter_candidates( + &self, + query: &SearchQuery, + candidates: &[u32], + ) -> trc::Result { + debug_assert!(candidates.len() <= MAX_FILTER_CANDIDATES); + + if self.supports_candidate_filter() && has_external_filters_only(&query.filters) { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_search_query(); + + self.sub_query_candidates(query.index, &query.filters, candidates) + .await + .caused_by(trc::location!()) + } else { + self.filter_account(SearchQuery { + index: query.index, + filters: query.filters.clone(), + comparators: Vec::new(), + mask: candidates.iter().copied().collect(), + }) + .await + } + } + + #[cfg_attr( + not(any(feature = "postgres", feature = "mysql")), + allow(unused_variables) + )] + async fn sub_query_candidates( + &self, + index: SearchIndex, + filters: &[SearchFilter], + candidates: &[u32], + ) -> trc::Result { + match self { + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => { + store.filter_candidates(index, filters, candidates).await + } + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.filter_candidates(index, filters, candidates).await, + _ => Err(trc::StoreEvent::NotSupported.into_err()), + } + } + + #[cfg_attr(not(feature = "postgres"), allow(unused_variables))] async fn sub_query( &self, index: SearchIndex, filters: &[SearchFilter], - sort: &[SearchComparator], - ) -> trc::Result> { + mask: &RoaringBitmap, + ) -> trc::Result { match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.query(index, filters, sort).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.query(index, filters, sort).await, - _ => unreachable!(), - }, - SearchStore::ElasticSearch(store) => store.query(index, filters, sort).await, - SearchStore::MeiliSearch(store) => store.query(index, filters, sort).await, + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => store.query_account(index, filters, mask).await, + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.query(index, filters).await, + SearchStore::ElasticSearch(store) => store.query(index, filters).await, + SearchStore::MeiliSearch(store) => store.query(index, filters).await, + SearchStore::Store(_) => unreachable!(), } } - pub async fn query_global(&self, query: SearchQuery) -> trc::Result> { + pub async fn query_global(&self, query: SearchQuery) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_search_query(); + match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => { - store - .query(query.index, &query.filters, &query.comparators) - .await - } - #[cfg(feature = "mysql")] - Store::MySQL(store) => { - store - .query(query.index, &query.filters, &query.comparators) - .await - } - store => store.query_global(query).await, - }, - SearchStore::ElasticSearch(store) => { - store - .query(query.index, &query.filters, &query.comparators) - .await - } - SearchStore::MeiliSearch(store) => { - store - .query(query.index, &query.filters, &query.comparators) - .await - } + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => store.query(query.index, &query.filters).await, + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.query(query.index, &query.filters).await, + SearchStore::Store(store) => store.query_global(query).await, + SearchStore::ElasticSearch(store) => store.query(query.index, &query.filters).await, + SearchStore::MeiliSearch(store) => store.query(query.index, &query.filters).await, } } pub async fn index(&self, documents: Vec) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_search_index(); + match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.index(documents).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.index(documents).await, - store => store.index(documents).await, - }, + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => store.index(documents).await, + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.index(documents).await, SearchStore::ElasticSearch(store) => store.index(documents).await, SearchStore::MeiliSearch(store) => store.index(documents).await, + SearchStore::Store(_) => unreachable!(), } } pub async fn unindex(&self, query: SearchQuery) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_search_unindex(); + match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.unindex(query).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.unindex(query).await, - store => store.unindex(query).await.map(|_| 0), - }, + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => store.unindex(query).await, + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.unindex(query).await, SearchStore::ElasticSearch(store) => store.unindex(query).await, SearchStore::MeiliSearch(store) => store.unindex(query).await, + SearchStore::Store(_) => unreachable!(), } } pub fn internal_fts(&self) -> Option<&Store> { match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(_) => None, - #[cfg(feature = "mysql")] - Store::MySQL(_) => None, - store => Some(store), - }, + SearchStore::Store(store) => Some(store), _ => None, } } + pub fn splits_identifiers(&self) -> bool { + match self { + SearchStore::Store(_) => true, + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(_) => true, + _ => false, + } + } + pub fn is_mysql(&self) -> bool { match self { #[cfg(feature = "mysql")] - SearchStore::Store(Store::MySQL(_)) => true, + SearchStore::MySQL(_) => true, _ => false, } } @@ -296,7 +299,7 @@ impl SearchStore { pub fn is_postgres(&self) -> bool { match self { #[cfg(feature = "postgres")] - SearchStore::Store(Store::PostgreSQL(_)) => true, + SearchStore::PostgreSQL(_) => true, _ => false, } } @@ -311,27 +314,40 @@ impl SearchStore { pub async fn create_indexes(&self) -> trc::Result<()> { match self { - SearchStore::Store(store) => match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.create_search_tables().await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.create_search_tables().await, - _ => Ok(()), - }, + #[cfg(feature = "postgres")] + SearchStore::PostgreSQL(store) => store.create_search_tables().await, + #[cfg(feature = "mysql")] + SearchStore::MySQL(store) => store.create_search_tables().await, + SearchStore::Store(_) => Ok(()), SearchStore::ElasticSearch(store) => store.create_indexes().await, SearchStore::MeiliSearch(store) => store.create_indexes().await, } } } -impl SearchFilter { - pub fn is_external(&self) -> bool { - matches!(self, SearchFilter::Operator { .. }) +fn has_external_filters_only(filters: &[SearchFilter]) -> bool { + let mut has_external_filters = false; + for filter in filters { + match filter { + SearchFilter::DocumentSet(_) => return false, + SearchFilter::Integer { + field: SearchField::AccountId, + .. + } => (), + SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } + | SearchFilter::Integer { .. } => has_external_filters = true, + SearchFilter::And | SearchFilter::Or | SearchFilter::Not | SearchFilter::End => (), + } } + has_external_filters } -impl SearchComparator { +impl SearchFilter { pub fn is_external(&self) -> bool { - matches!(self, SearchComparator::Field { .. }) + matches!( + self, + SearchFilter::Text { .. } | SearchFilter::KeyValue { .. } + ) } } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index af693bf4..6932c8c5 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -4,21 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::DocumentSet; use crate::{ - Deserialize, IterateParams, Key, QueryResult, SUBSPACE_COUNTER, SUBSPACE_INDEXES, - SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, + Deserialize, IterateParams, Key, QueryResult, Store, Subspace, U32_LEN, Value, ValueKey, write::{ - AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, Operation, ValueClass, ValueOp, - key::{DeserializeBigEndian, KeySerializer}, + AnyKey, AssignedIds, Batch, BatchBuilder, ValueClass, + key::{CounterKind, KeySerializer}, }, }; use compact_str::ToCompactString; use std::time::Instant; use trc::{AddContext, StoreEvent}; -use types::collection::Collection; impl Store { + #[cfg(not(stalwart_tests))] pub async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, @@ -40,7 +38,42 @@ impl Store { .caused_by(trc::location!()) } + #[cfg(stalwart_tests)] + pub async fn get_value(&self, key: impl Key) -> trc::Result> + where + U: Deserialize + 'static, + { + let key = crate::dispatch::StoreOps::count_get_value(key); + crate::dispatch::ops::wait(crate::dispatch::Latency::GetValue).await; + + let result = match self { + #[cfg(feature = "sqlite")] + Self::SQLite(store) => store.get_value(key).await, + #[cfg(feature = "foundation")] + Self::FoundationDb(store) => store.get_value(key).await, + #[cfg(feature = "postgres")] + Self::PostgreSQL(store) => store.get_value(key).await, + #[cfg(feature = "mysql")] + Self::MySQL(store) => store.get_value(key).await, + #[cfg(feature = "rocks")] + Self::RocksDb(store) => store.get_value(key).await, + Self::Ephemeral(store) => store.get_value(key).await, + Self::None => Err(trc::StoreEvent::NotConfigured.into()), + } + .caused_by(trc::location!()); + + crate::dispatch::StoreOps::count_get_value_hit(&result); + + result + } + pub async fn key_exists(&self, key: impl Key) -> trc::Result { + #[cfg(stalwart_tests)] + { + crate::dispatch::StoreOps::count_key_exists(&key); + crate::dispatch::ops::wait(crate::dispatch::Latency::GetValue).await; + } + match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.key_exists(key).await, @@ -63,6 +96,20 @@ impl Store { params: IterateParams, cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { + if params.fetch == crate::Fetch::Limit(0) { + return Ok(()); + } + #[cfg(stalwart_tests)] + let params = crate::dispatch::StoreOps::count_iterate(params); + #[cfg(stalwart_tests)] + let rows = crate::dispatch::ops::RowCounter::new(params.begin.subspace()); + #[cfg(stalwart_tests)] + let cb = rows.wrap(cb); + #[cfg(stalwart_tests)] + crate::dispatch::ops::wait(crate::dispatch::Latency::Iterate).await; + #[cfg(stalwart_tests)] + let skips = crate::dispatch::rocksdb_skips::SkipProbe::begin(self, params.begin.subspace()); + let start_time = Instant::now(); let result = match self { #[cfg(feature = "sqlite")] @@ -80,6 +127,13 @@ impl Store { } .caused_by(trc::location!()); + #[cfg(stalwart_tests)] + if let Some(skips) = skips { + skips.finish(); + } + #[cfg(stalwart_tests)] + rows.finish().await; + trc::event!( Store(StoreEvent::DataIterate), Elapsed = start_time.elapsed(), @@ -88,10 +142,80 @@ impl Store { result } + pub async fn iterate_many( + &self, + ranges: Vec>, + cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + debug_assert!(ranges.iter().all(|params| { + params.ascending + && params.fetch == crate::Fetch::All + && params.values == ranges[0].values + && params.begin.subspace() == ranges[0].begin.subspace() + && params.end.subspace() == ranges[0].begin.subspace() + })); + debug_assert!( + ranges.windows(2).all(|pair| match pair { + [previous, next] => previous.end.serialize(0) < next.begin.serialize(0), + _ => true, + }), + "iterate_many ranges must be ascending and disjoint" + ); + if ranges.is_empty() { + return Ok(()); + } + let start_time = Instant::now(); + let total = ranges.len(); + + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_iterate_many(&ranges); + #[cfg(stalwart_tests)] + let rows = crate::dispatch::ops::RowCounter::new(ranges[0].begin.subspace()); + #[cfg(stalwart_tests)] + let cb = rows.wrap(cb); + #[cfg(stalwart_tests)] + crate::dispatch::ops::wait(crate::dispatch::Latency::Iterate).await; + + let result = match self { + #[cfg(feature = "sqlite")] + Self::SQLite(store) => store.iterate_many(ranges, cb).await, + #[cfg(feature = "foundation")] + Self::FoundationDb(store) => store.iterate_many(ranges, cb).await, + #[cfg(feature = "postgres")] + Self::PostgreSQL(store) => store.iterate_many(ranges, cb).await, + #[cfg(feature = "mysql")] + Self::MySQL(store) => store.iterate_many(ranges, cb).await, + #[cfg(feature = "rocks")] + Self::RocksDb(store) => store.iterate_many(ranges, cb).await, + Self::Ephemeral(store) => store.iterate_many(ranges, cb).await, + Self::None => Err(trc::StoreEvent::NotConfigured.into()), + } + .caused_by(trc::location!()); + + #[cfg(stalwart_tests)] + rows.finish().await; + + trc::event!( + Store(StoreEvent::DataIterate), + Elapsed = start_time.elapsed(), + Total = total, + ); + + result + } + pub async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { + #[cfg(stalwart_tests)] + let key = { + let key = key.into(); + crate::dispatch::StoreOps::count_get_counter(&key); + crate::dispatch::ops::wait(crate::dispatch::Latency::GetValue).await; + key + }; + match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.get_counter(key).await, @@ -128,7 +252,7 @@ impl Store { trc::event!( Store(trc::StoreEvent::SqlQuery), - Details = query.to_compact_string(), + Query = query.to_compact_string(), Value = params.as_slice(), Result = &result, ); @@ -136,25 +260,56 @@ impl Store { result.caused_by(trc::location!()) } - pub async fn write(&self, batch: Batch<'_>) -> trc::Result { + pub async fn write_batch(&self, builder: &mut BatchBuilder) -> trc::Result { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_write_batch(); + + let mut assigned_ids = AssignedIds::default(); + let mut commit_points = builder.commit_points(); + + for commit_point in commit_points.iter() { + let batch = builder.build_one(commit_point); + self.write(batch, &mut assigned_ids).await?; + } + + Ok(assigned_ids) + } + + pub(crate) async fn write( + &self, + batch: Batch<'_>, + assigned_ids: &mut AssignedIds, + ) -> trc::Result<()> { + #[cfg(stalwart_tests)] + { + crate::dispatch::StoreOps::count_commit(&batch); + crate::dispatch::ops::wait(crate::dispatch::Latency::Write).await; + if let Some(err) = crate::dispatch::ops::injected_commit_failure() { + return Err(err); + } + } + let start_time = Instant::now(); let ops = batch.ops.len(); let result = match self { #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.write(batch).await, + Self::SQLite(store) => store.write(batch, assigned_ids).await, #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.write(batch).await, + Self::FoundationDb(store) => store.write(batch, assigned_ids).await, #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.write(batch).await, + Self::PostgreSQL(store) => store.write(batch, assigned_ids).await, #[cfg(feature = "mysql")] - Self::MySQL(store) => store.write(batch).await, + Self::MySQL(store) => store.write(batch, assigned_ids).await, #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.write(batch).await, - Self::Ephemeral(store) => store.write(batch).await, + Self::RocksDb(store) => store.write(batch, assigned_ids).await, + Self::Ephemeral(store) => store.write(batch, assigned_ids).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), }; + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_commit_result(&result); + trc::event!( Store(StoreEvent::DataWrite), Elapsed = start_time.elapsed(), @@ -164,26 +319,6 @@ impl Store { result } - pub async fn assign_document_ids( - &self, - account_id: u32, - collection: Collection, - num_ids: u64, - ) -> trc::Result { - // Increment UID next - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(collection) - .add_and_get(ValueClass::DocumentId, num_ids as i64); - self.write(batch.build_all()).await.and_then(|v| { - v.last_counter_id().map(|id| { - debug_assert!(id >= num_ids as i64, "{} < {}", id, num_ids); - id as u32 - }) - }) - } - pub async fn purge_store(&self) -> trc::Result<()> { match self { #[cfg(feature = "sqlite")] @@ -203,6 +338,9 @@ impl Store { } pub async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_delete_range(); + match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.delete_range(from, to).await, @@ -220,87 +358,35 @@ impl Store { .caused_by(trc::location!()) } - pub async fn delete_documents( - &self, - subspace: u8, - account_id: u32, - collection: u8, - collection_offset: Option, - document_ids: &impl DocumentSet, - ) -> trc::Result<()> { - // Serialize keys - let (from_key, to_key) = if collection_offset.is_some() { - ( - KeySerializer::new(U32_LEN + 2) - .write(account_id) - .write(collection), - KeySerializer::new(U32_LEN + 2) - .write(account_id) - .write(collection + 1), - ) - } else { - ( - KeySerializer::new(U32_LEN).write(account_id), - KeySerializer::new(U32_LEN).write(account_id + 1), - ) - }; - - // Find keys to delete - let mut delete_keys = Vec::new(); - self.iterate( - IterateParams::new( - AnyKey { - subspace, - key: from_key.finalize(), - }, - AnyKey { - subspace, - key: to_key.finalize(), - }, - ) - .no_values(), - |key, _| { - if collection_offset.is_none_or(|offset| { - key.get(key.len() - U32_LEN - offset).copied() == Some(collection) - }) { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if document_ids.contains(document_id) { - delete_keys.push(key.to_vec()); - } - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - // Remove keys - let mut batch = BatchBuilder::new(); - - for key in delete_keys { - if batch.is_large_batch() { - self.write(std::mem::take(&mut batch).build_all()) - .await - .caused_by(trc::location!())?; - } - batch.any_op(Operation::Value { - class: ValueClass::Any(AnyClass { subspace, key }), - op: ValueOp::Clear, - }); - } - - if !batch.is_empty() { - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - - Ok(()) - } - pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> { - for subspace in [SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER] { + for kind in CounterKind::ACCOUNT_KINDS { + self.delete_range( + AnyKey { + subspace: Subspace::Counter, + key: KeySerializer::new(U32_LEN + 1) + .write(kind.byte()) + .write(account_id) + .finalize(), + }, + AnyKey { + subspace: Subspace::Counter, + key: KeySerializer::new(U32_LEN + 1) + .write(kind.byte()) + .write(account_id + 1) + .finalize(), + }, + ) + .await + .caused_by(trc::location!())?; + } + + for subspace in [ + Subspace::Logs, + Subspace::Indexes, + Subspace::Property, + Subspace::Immutable, + Subspace::IndexProperty, + ] { self.delete_range( AnyKey { subspace, @@ -332,23 +418,6 @@ impl Store { .await .caused_by(trc::location!())?; - self.delete_range( - ValueKey { - account_id, - collection: 0, - document_id: 0, - class: ValueClass::Property(0), - }, - ValueKey { - account_id: account_id + 1, - collection: 0, - document_id: 0, - class: ValueClass::Property(0), - }, - ) - .await - .caused_by(trc::location!())?; - Ok(()) } @@ -370,4 +439,37 @@ impl Store { store.invalidate_read_snapshot(); } } + + pub fn read_snapshot_version(&self) -> Option { + match self { + #[cfg(feature = "foundation")] + Self::FoundationDb(store) => store.read_snapshot_version(), + _ => None, + } + } + + pub fn advance_read_snapshot(&self, version: u64) { + #[cfg(feature = "foundation")] + if let Self::FoundationDb(store) = self { + store.advance_read_snapshot(version); + } + #[cfg(not(feature = "foundation"))] + let _ = version; + } +} + +#[cfg(any(feature = "postgres", feature = "mysql"))] +pub(crate) fn point_keys(ranges: &[IterateParams]) -> Option>> { + let mut keys = Vec::with_capacity(ranges.len()); + let mut end = Vec::new(); + for params in ranges { + let begin = params.begin.serialize(0); + end.clear(); + params.end.serialize_into(&mut end, 0); + if begin != end { + return None; + } + keys.push(begin); + } + Some(keys) } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ae776ba5..9b2a35d9 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -28,6 +28,7 @@ use crate::backend::{elastic::ElasticSearchStore, meili::MeiliSearchStore}; use ahash::AHashMap; use backend::{ephemeral::EphemeralStore, fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; use std::{borrow::Cow, path::PathBuf, sync::Arc}; +use types::collection::SyncCollection; use write::ValueClass; pub trait Deserialize: Sized + Sync + Send { @@ -61,8 +62,16 @@ pub trait SerializeInfallible { pub(crate) const WITH_SUBSPACE: u32 = 1; pub trait Key: Sync + Send + Clone { - fn serialize(&self, flags: u32) -> Vec; - fn subspace(&self) -> u8; + fn serialize_into(&self, buf: &mut Vec, flags: u32); + fn key_len_hint(&self) -> usize; + fn subspace(&self) -> Subspace; + + #[inline] + fn serialize(&self, flags: u32) -> Vec { + let mut buf = Vec::with_capacity(self.key_len_hint() + 1); + self.serialize_into(&mut buf, flags); + buf + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -96,48 +105,262 @@ pub struct LogKey { pub change_id: u64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PrivateLogKey { + pub account_id: u32, + pub collection: SyncCollection, + pub viewer: u32, + pub change_id: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ChangeLogKey { + Shared(LogKey), + Private(PrivateLogKey), +} + +pub const U128_LEN: usize = std::mem::size_of::(); pub const U64_LEN: usize = std::mem::size_of::(); pub const U32_LEN: usize = std::mem::size_of::(); pub const U16_LEN: usize = std::mem::size_of::(); -pub const SUBSPACE_ACL: u8 = b'a'; -pub const SUBSPACE_TASK_QUEUE: u8 = b'f'; -pub const SUBSPACE_INDEXES: u8 = b'i'; -pub const SUBSPACE_BLOB_LINK: u8 = b'k'; -pub const SUBSPACE_BLOBS: u8 = b't'; -pub const SUBSPACE_LOGS: u8 = b'l'; -pub const SUBSPACE_COUNTER: u8 = b'n'; -pub const SUBSPACE_IN_MEMORY_VALUE: u8 = b'm'; -pub const SUBSPACE_IN_MEMORY_COUNTER: u8 = b'y'; -pub const SUBSPACE_PROPERTY: u8 = b'p'; -pub const SUBSPACE_REGISTRY: u8 = b's'; -pub const SUBSPACE_REGISTRY_IDX: u8 = b'b'; -pub const SUBSPACE_REGISTRY_PK: u8 = b'g'; -pub const SUBSPACE_DIRECTORY: u8 = b'd'; -pub const SUBSPACE_QUEUE_MESSAGE: u8 = b'e'; -pub const SUBSPACE_QUEUE_EVENT: u8 = b'q'; -pub const SUBSPACE_QUOTA: u8 = b'u'; -pub const SUBSPACE_REPORT_OUT: u8 = b'h'; -pub const SUBSPACE_REPORT_IN: u8 = b'r'; -pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o'; -pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x'; -pub const SUBSPACE_SEARCH_INDEX: u8 = b'z'; -pub const SUBSPACE_DELETED_ITEMS: u8 = b'j'; -pub const SUBSPACE_SPAM_SAMPLES: u8 = b'w'; +const _: () = { + let mut index = 0; + while index < Subspace::ALL.len() { + assert!(Subspace::ALL[index].index() == index); + index += 1; + } +}; -// TODO: Remove in v1.0 -pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v'; -pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[repr(u8)] +pub enum Subspace { + Acl = b'a', + RegistryIndex = b'b', + Immutable = b'c', + Directory = b'd', + QueueMessage = b'e', + TaskQueue = b'f', + RegistryPrimaryKey = b'g', + ReportOut = b'h', + Indexes = b'i', + DeletedItems = b'j', + BlobLink = b'k', + Logs = b'l', + Counter = b'n', + TelemetrySpan = b'o', + Property = b'p', + QueueEvent = b'q', + ReportIn = b'r', + Registry = b's', + Blobs = b't', + IndexProperty = b'v', + SpamSamples = b'w', + TelemetryMetric = b'x', + SearchTerm = b'z', + SearchDocument = b'm', + SearchQueue = b'y', + System = b'u', +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Shape { + Value, + Presence, + Counter, +} + +impl Subspace { + pub const ALL: &'static [Subspace] = &[ + Subspace::Acl, + Subspace::RegistryIndex, + Subspace::Immutable, + Subspace::Directory, + Subspace::QueueMessage, + Subspace::TaskQueue, + Subspace::RegistryPrimaryKey, + Subspace::ReportOut, + Subspace::Indexes, + Subspace::DeletedItems, + Subspace::BlobLink, + Subspace::Logs, + Subspace::Counter, + Subspace::TelemetrySpan, + Subspace::Property, + Subspace::QueueEvent, + Subspace::ReportIn, + Subspace::Registry, + Subspace::Blobs, + Subspace::IndexProperty, + Subspace::SpamSamples, + Subspace::TelemetryMetric, + Subspace::SearchTerm, + Subspace::SearchDocument, + Subspace::SearchQueue, + Subspace::System, + ]; + + #[inline(always)] + pub const fn byte(self) -> u8 { + self as u8 + } + + #[inline(always)] + pub const fn index(self) -> usize { + match self { + Subspace::Acl => 0, + Subspace::RegistryIndex => 1, + Subspace::Immutable => 2, + Subspace::Directory => 3, + Subspace::QueueMessage => 4, + Subspace::TaskQueue => 5, + Subspace::RegistryPrimaryKey => 6, + Subspace::ReportOut => 7, + Subspace::Indexes => 8, + Subspace::DeletedItems => 9, + Subspace::BlobLink => 10, + Subspace::Logs => 11, + Subspace::Counter => 12, + Subspace::TelemetrySpan => 13, + Subspace::Property => 14, + Subspace::QueueEvent => 15, + Subspace::ReportIn => 16, + Subspace::Registry => 17, + Subspace::Blobs => 18, + Subspace::IndexProperty => 19, + Subspace::SpamSamples => 20, + Subspace::TelemetryMetric => 21, + Subspace::SearchTerm => 22, + Subspace::SearchDocument => 23, + Subspace::SearchQueue => 24, + Subspace::System => 25, + } + } + + pub const fn try_from_byte(byte: u8) -> Option { + Some(match byte { + b'a' => Subspace::Acl, + b'b' => Subspace::RegistryIndex, + b'c' => Subspace::Immutable, + b'd' => Subspace::Directory, + b'e' => Subspace::QueueMessage, + b'f' => Subspace::TaskQueue, + b'g' => Subspace::RegistryPrimaryKey, + b'h' => Subspace::ReportOut, + b'i' => Subspace::Indexes, + b'j' => Subspace::DeletedItems, + b'k' => Subspace::BlobLink, + b'l' => Subspace::Logs, + b'n' => Subspace::Counter, + b'o' => Subspace::TelemetrySpan, + b'p' => Subspace::Property, + b'q' => Subspace::QueueEvent, + b'r' => Subspace::ReportIn, + b's' => Subspace::Registry, + b't' => Subspace::Blobs, + b'v' => Subspace::IndexProperty, + b'w' => Subspace::SpamSamples, + b'x' => Subspace::TelemetryMetric, + b'z' => Subspace::SearchTerm, + b'm' => Subspace::SearchDocument, + b'y' => Subspace::SearchQueue, + b'u' => Subspace::System, + _ => return None, + }) + } + + pub const fn name(self) -> &'static str { + match self { + Subspace::Acl => "a", + Subspace::RegistryIndex => "b", + Subspace::Immutable => "c", + Subspace::Directory => "d", + Subspace::QueueMessage => "e", + Subspace::TaskQueue => "f", + Subspace::RegistryPrimaryKey => "g", + Subspace::ReportOut => "h", + Subspace::Indexes => "i", + Subspace::DeletedItems => "j", + Subspace::BlobLink => "k", + Subspace::Logs => "l", + Subspace::Counter => "n", + Subspace::TelemetrySpan => "o", + Subspace::Property => "p", + Subspace::QueueEvent => "q", + Subspace::ReportIn => "r", + Subspace::Registry => "s", + Subspace::Blobs => "t", + Subspace::IndexProperty => "v", + Subspace::SpamSamples => "w", + Subspace::TelemetryMetric => "x", + Subspace::SearchTerm => "z", + Subspace::SearchDocument => "m", + Subspace::SearchQueue => "y", + Subspace::System => "u", + } + } + + pub const fn shape(self) -> Shape { + match self { + Subspace::Indexes | Subspace::RegistryIndex => Shape::Presence, + Subspace::Counter => Shape::Counter, + Subspace::Acl + | Subspace::Immutable + | Subspace::Directory + | Subspace::QueueMessage + | Subspace::TaskQueue + | Subspace::RegistryPrimaryKey + | Subspace::ReportOut + | Subspace::DeletedItems + | Subspace::BlobLink + | Subspace::Logs + | Subspace::TelemetrySpan + | Subspace::Property + | Subspace::QueueEvent + | Subspace::ReportIn + | Subspace::Registry + | Subspace::Blobs + | Subspace::IndexProperty + | Subspace::SpamSamples + | Subspace::TelemetryMetric + | Subspace::SearchTerm + | Subspace::SearchDocument + | Subspace::SearchQueue + | Subspace::System => Shape::Value, + } + } + + pub const fn is_internal_fts(self) -> bool { + matches!(self, Subspace::SearchTerm | Subspace::SearchDocument) + } +} + +impl From for u8 { + #[inline(always)] + fn from(subspace: Subspace) -> Self { + subspace as u8 + } +} + +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) enum Fetch { + #[default] + All, + Expect(u32), + Limit(u32), +} #[derive(Clone)] pub struct IterateParams { begin: T, end: T, - first: bool, ascending: bool, values: bool, + fetch: Fetch, } +pub const ITERATE_BATCH_GROWTH: u32 = 4; + #[derive(Clone, Default)] pub struct LookupStores { pub stores: AHashMap, InMemoryStore>, @@ -173,13 +396,18 @@ pub enum BlobStore { #[derive(Clone)] pub enum SearchStore { Store(Store), + #[cfg(feature = "postgres")] + PostgreSQL(Arc), + #[cfg(feature = "mysql")] + MySQL(Arc), ElasticSearch(Arc), MeiliSearch(Arc), } #[derive(Clone, Debug)] pub enum InMemoryStore { - Store(Store), + Sql(Store), + Local(Arc), #[cfg(feature = "redis")] Redis(Arc), Http(Arc), @@ -256,6 +484,12 @@ impl From for SearchStore { } } +impl From for InMemoryStore { + fn from(store: backend::local::LocalStore) -> Self { + Self::Local(Arc::new(store)) + } +} + #[cfg(feature = "redis")] impl From for InMemoryStore { fn from(store: backend::redis::RedisStore) -> Self { @@ -269,12 +503,6 @@ impl From for SearchStore { } } -impl From for InMemoryStore { - fn from(store: Store) -> Self { - Self::Store(store) - } -} - impl From for BlobStore { fn from(store: Store) -> Self { Self::Store(store) @@ -289,7 +517,7 @@ impl Default for BlobStore { impl Default for InMemoryStore { fn default() -> Self { - Self::Store(Store::None) + Self::Sql(Store::None) } } @@ -634,6 +862,14 @@ impl Store { !matches!(self, Self::None) } + pub fn chunk_size(&self) -> Option { + match self { + #[cfg(feature = "foundation")] + Store::FoundationDb(_) => Some(backend::foundationdb::MAX_VALUE_SIZE), + _ => None, + } + } + pub fn is_same(&self, other: &Store) -> bool { match (self, other) { #[cfg(feature = "sqlite")] @@ -687,6 +923,26 @@ impl Store { } } + #[inline(always)] + pub fn is_embedded(&self) -> bool { + match self { + #[cfg(feature = "rocks")] + Store::RocksDb(_) => true, + #[cfg(feature = "sqlite")] + Store::SQLite(_) => true, + _ => false, + } + } + + #[inline(always)] + pub fn is_postgresql(&self) -> bool { + match self { + #[cfg(feature = "postgres")] + Store::PostgreSQL(_) => true, + _ => false, + } + } + #[inline(always)] pub fn is_ephemeral(&self) -> bool { matches!(self, Self::Ephemeral(_)) @@ -735,3 +991,4 @@ impl From> for () { unreachable!() } } + diff --git a/crates/store/src/query/acl.rs b/crates/store/src/query/acl.rs index 9b74aa81..d22fccd5 100644 --- a/crates/store/src/query/acl.rs +++ b/crates/store/src/query/acl.rs @@ -9,10 +9,12 @@ use trc::AddContext; use types::collection::Collection; use crate::{ - Deserialize, IterateParams, Store, U32_LEN, ValueKey, + Deserialize, IterateParams, Store, Subspace, U32_LEN, ValueKey, write::{BatchBuilder, ValueClass, key::DeserializeBigEndian}, }; +const MAX_REVOKE_RANGES: usize = 1_000; + pub enum AclQuery { SharedWith { grant_account_id: u32, @@ -81,65 +83,69 @@ impl Store { .map(|_| results) } - pub async fn acl_revoke_all(&self, account_id: u32) -> trc::Result> { - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Acl(0), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Acl(u32::MAX), - }; - - let mut delete_keys = Vec::new(); - let mut revoked_accounts = AHashSet::new(); + pub async fn acl_owned(&self, account_id: u32) -> trc::Result> { + let mut results = Vec::new(); self.iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { + IterateParams::new( + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Acl(0), + }, + ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Acl(u32::MAX), + }, + ) + .ascending(), + |key, value| { if account_id == key.deserialize_be_u32(U32_LEN)? { - let owner_account_id = key.deserialize_be_u32(0)?; - revoked_accounts.insert(owner_account_id); - delete_keys.push((owner_account_id, AclItem::deserialize(key)?)); + results.push(( + key.deserialize_be_u32(0)?, + AclItem::deserialize(key)?.with_permissions(u64::deserialize(value)?), + )); } Ok(true) }, ) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!()) + .map(|_| results) + } - // Remove permissions + pub async fn acl_revoke_owner( + &self, + owner_account_id: u32, + grantee_ids: &AHashSet, + ) -> trc::Result<()> { let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - let mut last_collection = Collection::None; - for (revoke_account_id, acl_item) in delete_keys.into_iter() { - if batch.is_large_batch() { - self.write(batch.build_all()) + let mut ranges = 0; + for grantee_id in grantee_ids { + if ranges == MAX_REVOKE_RANGES || batch.is_large_batch() { + self.write_batch(&mut batch) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); - batch.with_account_id(account_id); - last_collection = Collection::None; + ranges = 0; } - if acl_item.to_collection != last_collection { - batch.with_collection(acl_item.to_collection); - last_collection = acl_item.to_collection; - } - batch - .with_document(acl_item.to_document_id) - .acl_revoke(revoke_account_id); + ranges += 1; + let from = (u64::from(*grantee_id) << 32) | u64::from(owner_account_id); + batch.clear_range( + Subspace::Acl, + from.to_be_bytes().to_vec(), + from.saturating_add(1).to_be_bytes().to_vec(), + ); } if !batch.is_empty() { - self.write(batch.build_all()) + self.write_batch(&mut batch) .await .caused_by(trc::location!())?; } - - Ok(revoked_accounts) + Ok(()) } } diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index 8c150434..1e1171ce 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -4,26 +4,66 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{ + IterateParams, LogKey, Store, U64_LEN, + write::{ + LogCollection, + key::DeserializeBigEndian, + log::{ + BASE_LISTS, CHANGE_LISTS, CONTAINER_INSERTS, CONTAINER_LISTS, CONTAINER_METADATA, + CONTAINER_PROPERTY_CHANGES, CONTAINER_UPDATES, ITEM_INSERTS, ITEM_METADATA, + ITEM_UPDATES, PRESENCE_EXTENDED, zigzag_decode, + }, + }, +}; +use ahash::AHashMap; use trc::AddContext; use types::collection::{SyncCollection, VanishedCollection}; -use utils::codec::leb128::Leb128Iterator; - -use crate::{ - IterateParams, LogKey, Store, U32_LEN, U64_LEN, - write::{LogCollection, key::DeserializeBigEndian}, -}; +use utils::codec::leb128::Leb128Reader; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Change { InsertContainer(u64), UpdateContainer(u64), - UpdateContainerProperty(u64), + UpdateContainerPartial(u64, PartialChange), DeleteContainer(u64), InsertItem(u64), UpdateItem(u64), + UpdateItemMetadata(u64), DeleteItem(u64), } +const _: () = assert!(size_of::() == 16); + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Default)] +#[repr(transparent)] +pub struct PartialChange(u8); + +impl PartialChange { + pub const PROPERTIES: PartialChange = PartialChange(1); + pub const METADATA: PartialChange = PartialChange(1 << 1); + + #[inline(always)] + pub const fn has_properties(self) -> bool { + self.0 & Self::PROPERTIES.0 != 0 + } + + #[inline(always)] + pub const fn has_metadata(self) -> bool { + self.0 & Self::METADATA.0 != 0 + } + + #[inline(always)] + pub const fn is_metadata_only(self) -> bool { + self.0 == Self::METADATA.0 + } + + #[inline(always)] + pub const fn union(self, other: PartialChange) -> Self { + PartialChange(self.0 | other.0) + } +} + #[derive(Debug)] pub struct Changes { pub changes: Vec, @@ -32,6 +72,10 @@ pub struct Changes { pub container_change_id: Option, pub item_change_id: Option, pub is_truncated: bool, + container_index: AHashMap, + item_index: AHashMap, + discarded: Option>, + live: usize, } #[derive(Debug, Clone, Copy)] @@ -40,10 +84,11 @@ pub enum Query { Since(u64), SinceInclusive(u64), RangeInclusive(u64, u64), + Range(u64, u64), } pub trait DeserializeVanished: Sized + Sync + Send { - fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option; + fn deserialize_vanished(bytes: &[u8], items: &mut Vec) -> Option<()>; } impl Default for Changes { @@ -55,10 +100,20 @@ impl Default for Changes { container_change_id: None, item_change_id: None, is_truncated: false, + container_index: AHashMap::new(), + item_index: AHashMap::new(), + discarded: None, + live: 0, } } } +impl Changes { + pub fn needs_full_rebuild(&self, since: u64) -> bool { + self.is_truncated || (since != 0 && self.to_change_id == 0) + } +} + impl Store { pub async fn changes( &self, @@ -70,7 +125,7 @@ impl Store { collection_, LogCollection::Sync(SyncCollection::ShareNotification) ); - let collection = u8::from(collection_); + let is_prefixed = collection_.is_prefixed(); let (is_inclusive, from_change_id, to_change_id) = match query { Query::All => (true, 0, u64::MAX), @@ -79,17 +134,10 @@ impl Store { Query::RangeInclusive(from_change_id, to_change_id) => { (true, from_change_id, to_change_id) } + Query::Range(from_change_id, to_change_id) => (false, from_change_id, to_change_id), }; - let from_key = LogKey { - account_id, - collection, - change_id: from_change_id, - }; - let to_key = LogKey { - account_id, - collection, - change_id: to_change_id, - }; + let from_key = collection_.log_key(account_id, from_change_id); + let to_key = collection_.log_key(account_id, to_change_id); let mut changelog = Changes::default(); @@ -102,13 +150,13 @@ impl Store { changelog.is_truncated = true; return Ok(true); } - if changelog.changes.is_empty() { + if changelog.live == 0 { changelog.from_change_id = change_id; } changelog.to_change_id = change_id; if !is_share_log { let (has_container_changes, has_item_changes) = - changelog.deserialize(value).ok_or_else(|| { + changelog.deserialize(value, is_prefixed).ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; if has_container_changes { @@ -118,7 +166,7 @@ impl Store { changelog.item_change_id = Some(change_id); } } else { - changelog.changes.push(Change::InsertItem(change_id)); + changelog.push_share_notification(change_id); } } else { changelog.from_change_id = change_id; @@ -130,6 +178,8 @@ impl Store { .await .caused_by(trc::location!())?; + changelog.finalize(); + Ok(changelog) } @@ -147,6 +197,7 @@ impl Store { Query::RangeInclusive(from_change_id, to_change_id) => { (true, from_change_id, to_change_id) } + Query::Range(from_change_id, to_change_id) => (false, from_change_id, to_change_id), }; let from_key = LogKey { account_id, @@ -165,20 +216,14 @@ impl Store { IterateParams::new(from_key, to_key).ascending(), |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; - if is_inclusive || change_id != from_change_id { - let mut iter = value.iter().peekable(); - - while iter.peek().is_some() { - if let Some(item) = T::deserialize_vanished(&mut iter) { - vanished.push(item); - } else { - return Err(trc::Error::corrupted_key( - key, - value.into(), - trc::location!(), - )); - } - } + if (is_inclusive || change_id != from_change_id) + && T::deserialize_vanished(value, &mut vanished).is_none() + { + return Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )); } Ok(true) }, @@ -189,22 +234,64 @@ impl Store { Ok(vanished) } + pub async fn vanished_uids( + &self, + account_id: u32, + mailbox_id: u32, + query: Query, + ) -> trc::Result> { + let collection = u8::from(LogCollection::Vanished(VanishedCollection::Email)); + let (is_inclusive, from_change_id, to_change_id) = match query { + Query::All => (true, 0, u64::MAX), + Query::Since(change_id) => (false, change_id, u64::MAX), + Query::SinceInclusive(change_id) => (true, change_id, u64::MAX), + Query::RangeInclusive(from_change_id, to_change_id) => { + (true, from_change_id, to_change_id) + } + Query::Range(from_change_id, to_change_id) => (false, from_change_id, to_change_id), + }; + let from_key = LogKey { + account_id, + collection, + change_id: from_change_id, + }; + let to_key = LogKey { + account_id, + collection, + change_id: to_change_id, + }; + + let mut uids = Vec::new(); + + self.iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + if (is_inclusive || change_id != from_change_id) + && decode_vanished_uids(value, mailbox_id, &mut uids).is_none() + { + return Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + Ok(uids) + } + pub async fn get_last_change_id( &self, account_id: u32, collection: LogCollection, ) -> trc::Result> { - let collection = u8::from(collection); - let from_key = LogKey { - account_id, - collection, - change_id: 0, - }; - let to_key = LogKey { - account_id, - collection, - change_id: u64::MAX, - }; + let from_key = collection.log_key(account_id, 0); + let to_key = collection.log_key(account_id, u64::MAX); let mut last_change_id = None; @@ -238,136 +325,179 @@ impl From for LogCollection { } impl Changes { - pub fn deserialize(&mut self, bytes: &[u8]) -> Option<(bool, bool)> { - let mut bytes_it = bytes.iter(); + fn push_change(&mut self, id: u64, change: Change, is_container: bool) { + let index = if is_container { + &mut self.container_index + } else { + &mut self.item_index + }; - let container_inserts: usize = bytes_it.next_leb128()?; - let container_updates: usize = bytes_it.next_leb128()?; - let container_property_changes: usize = bytes_it.next_leb128()?; - let container_deletes: usize = bytes_it.next_leb128()?; - - let item_inserts: usize = bytes_it.next_leb128()?; - let item_updates: usize = bytes_it.next_leb128()?; - let item_deletes: usize = bytes_it.next_leb128()?; - - let has_container_changes = - container_inserts + container_updates + container_property_changes + container_deletes - > 0; - let has_item_changes = item_inserts + item_updates + item_deletes > 0; - - if container_inserts > 0 { - for _ in 0..container_inserts { - self.changes - .push(Change::InsertContainer(bytes_it.next_leb128()?)); - } - } - - if container_updates > 0 || container_property_changes > 0 { - 'update_outer: for change_pos in 0..(container_updates + container_property_changes) { - let id = bytes_it.next_leb128()?; - let mut is_property_change = change_pos >= container_updates; - - for (idx, change) in self.changes.iter().enumerate() { - match change { - Change::InsertContainer(insert_id) if *insert_id == id => { - // Item updated after inserted, no need to count this change. - continue 'update_outer; - } - Change::UpdateContainer(update_id) if *update_id == id => { - // Move update to the front - is_property_change = false; - self.changes.remove(idx); - break; - } - Change::UpdateContainerProperty(update_id) if *update_id == id => { - // Move update to the front - self.changes.remove(idx); - break; - } - _ => (), - } + if let Some(pos) = index.get(&id).copied() { + let current = self.changes[pos]; + let (is_insert, is_update, is_delete) = match current { + Change::InsertContainer(_) | Change::InsertItem(_) => (true, false, false), + Change::UpdateContainer(_) | Change::UpdateItem(_) => (false, true, false), + Change::DeleteContainer(_) | Change::DeleteItem(_) => (false, false, true), + Change::UpdateContainerPartial(..) | Change::UpdateItemMetadata(_) => { + (false, false, false) } + }; - self.changes.push(if !is_property_change { - Change::UpdateContainer(id) - } else { - Change::UpdateContainerProperty(id) - }); - } - } - - if container_deletes > 0 { - 'delete_outer: for _ in 0..container_deletes { - let id = bytes_it.next_leb128()?; - - 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { - match change { - Change::InsertContainer(insert_id) if *insert_id == id => { - self.changes.remove(idx); - continue 'delete_outer; - } - Change::UpdateContainer(update_id) if *update_id == id => { - self.changes.remove(idx); - break 'delete_inner; - } - _ => (), - } + match change { + Change::UpdateContainer(_) + | Change::UpdateContainerPartial(..) + | Change::UpdateItem(_) + | Change::UpdateItemMetadata(_) + if is_insert || is_delete => + { + return; } - - self.changes.push(Change::DeleteContainer(id)); - } - } - - // Item changes - if item_inserts > 0 { - for _ in 0..item_inserts { - self.changes - .push(Change::InsertItem(bytes_it.next_leb128()?)); - } - } - - if item_updates > 0 { - 'update_outer: for _ in 0..item_updates { - let id = bytes_it.next_leb128()?; - - for (idx, change) in self.changes.iter().enumerate() { - match change { - Change::InsertItem(insert_id) if *insert_id == id => { - // Item updated after inserted, no need to count this change. - continue 'update_outer; - } - Change::UpdateItem(update_id) if *update_id == id => { - // Move update to the front - self.changes.remove(idx); - break; - } - _ => (), + Change::UpdateContainerPartial(..) | Change::UpdateItemMetadata(_) if is_update => { + return; + } + Change::UpdateContainerPartial(_, partial) => { + if let Change::UpdateContainerPartial(_, current) = current { + self.changes[pos] = + Change::UpdateContainerPartial(id, current.union(partial)); + } else { + self.changes[pos] = change; } + return; + } + Change::DeleteContainer(_) | Change::DeleteItem(_) if is_insert => { + index.remove(&id); + self.discard(pos); + self.live -= 1; + return; + } + _ => { + self.changes[pos] = change; + return; } - - self.changes.push(Change::UpdateItem(id)); } } - if item_deletes > 0 { - 'delete_outer: for _ in 0..item_deletes { - let id = bytes_it.next_leb128()?; + index.insert(id, self.changes.len()); + self.changes.push(change); + self.live += 1; + } - 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { - match change { - Change::InsertItem(insert_id) if *insert_id == id => { - self.changes.remove(idx); - continue 'delete_outer; + fn discard(&mut self, pos: usize) { + let discarded = self.discarded.get_or_insert_default(); + if discarded.len() <= pos { + discarded.resize(pos + 1, false); + } + discarded[pos] = true; + } + + pub(crate) fn push_share_notification(&mut self, change_id: u64) { + self.changes.push(Change::InsertItem(change_id)); + self.live += 1; + } + + pub(crate) fn finalize(&mut self) { + if let Some(discarded) = self.discarded.take() { + let mut pos = 0; + self.changes.retain(|_| { + let keep = !discarded.get(pos).copied().unwrap_or(false); + pos += 1; + keep + }); + } + + self.container_index = AHashMap::new(); + self.item_index = AHashMap::new(); + } + + pub fn deserialize(&mut self, bytes: &[u8], is_prefixed: bool) -> Option<(bool, bool)> { + let (&first, rest) = bytes.split_first()?; + let mut bytes = rest; + let (mut has_container_changes, mut has_item_changes) = self.deserialize_lists( + &mut bytes, + u16::from(first & !PRESENCE_EXTENDED), + is_prefixed, + )?; + + if first & PRESENCE_EXTENDED != 0 { + let (&extended, rest) = bytes.split_first()?; + bytes = rest; + if u32::from(extended) >> (CHANGE_LISTS - BASE_LISTS) != 0 { + return None; + } + let (containers, items) = + self.deserialize_lists(&mut bytes, u16::from(extended) << BASE_LISTS, is_prefixed)?; + has_container_changes |= containers; + has_item_changes |= items; + } + + Some((has_container_changes, has_item_changes)) + } + + #[inline(always)] + fn deserialize_lists( + &mut self, + bytes: &mut &[u8], + presence: u16, + is_prefixed: bool, + ) -> Option<(bool, bool)> { + let mut counts = [0usize; CHANGE_LISTS]; + let mut bits = presence; + while bits != 0 { + let slot = bits.trailing_zeros() as usize; + bits &= bits - 1; + *counts.get_mut(slot)? = bytes.read_leb128()?; + } + + let mut has_container_changes = false; + let mut has_item_changes = false; + let mut bits = presence; + while bits != 0 { + let slot = bits.trailing_zeros() as usize; + bits &= bits - 1; + let count = *counts.get(slot)?; + + if CONTAINER_LISTS & (1 << slot) != 0 { + has_container_changes |= count > 0; + let mut prev = 0u64; + for _ in 0..count { + prev += bytes.read_leb128::()?; + let change = match slot { + CONTAINER_INSERTS => Change::InsertContainer(prev), + CONTAINER_UPDATES => Change::UpdateContainer(prev), + CONTAINER_PROPERTY_CHANGES => { + Change::UpdateContainerPartial(prev, PartialChange::PROPERTIES) } - Change::UpdateItem(update_id) if *update_id == id => { - self.changes.remove(idx); - break 'delete_inner; + CONTAINER_METADATA => { + Change::UpdateContainerPartial(prev, PartialChange::METADATA) } - _ => (), - } + _ => Change::DeleteContainer(prev), + }; + self.push_change(prev, change, true); } + } else { + has_item_changes |= count > 0; + let mut prev_prefix = 0i64; + let mut prev_document_id = 0i64; + let mut prev_id = 0u64; - self.changes.push(Change::DeleteItem(id)); + for _ in 0..count { + let id = if is_prefixed { + prev_prefix += bytes.read_leb128::()? as i64; + prev_document_id += zigzag_decode(bytes.read_leb128::()?); + ((prev_prefix as u64) << 32) | (prev_document_id as u64 & u32::MAX as u64) + } else { + prev_id += bytes.read_leb128::()?; + prev_id + }; + + let change = match slot { + ITEM_INSERTS => Change::InsertItem(id), + ITEM_UPDATES => Change::UpdateItem(id), + ITEM_METADATA => Change::UpdateItemMetadata(id), + _ => Change::DeleteItem(id), + }; + self.push_change(id, change, false); + } } } @@ -394,40 +524,30 @@ impl Changes { impl Change { pub fn item_id(&self) -> Option { match self { - Change::InsertItem(id) => Some(*id), - Change::UpdateItem(id) => Some(*id), - Change::DeleteItem(id) => Some(*id), + Change::InsertItem(id) + | Change::UpdateItem(id) + | Change::UpdateItemMetadata(id) + | Change::DeleteItem(id) => Some(*id), _ => None, } } pub fn container_id(&self) -> Option { match self { - Change::InsertContainer(id) => Some(*id), - Change::UpdateContainer(id) => Some(*id), - Change::UpdateContainerProperty(id) => Some(*id), - Change::DeleteContainer(id) => Some(*id), + Change::InsertContainer(id) + | Change::UpdateContainer(id) + | Change::UpdateContainerPartial(id, _) + | Change::DeleteContainer(id) => Some(*id), _ => None, } } pub fn try_unwrap_item_id(self) -> Option { - match self { - Change::InsertItem(id) => Some(id), - Change::UpdateItem(id) => Some(id), - Change::DeleteItem(id) => Some(id), - _ => None, - } + self.item_id() } pub fn try_unwrap_container_id(self) -> Option { - match self { - Change::InsertContainer(id) => Some(id), - Change::UpdateContainer(id) => Some(id), - Change::UpdateContainerProperty(id) => Some(id), - Change::DeleteContainer(id) => Some(id), - _ => None, - } + self.container_id() } pub fn is_container_change(&self) -> bool { @@ -435,7 +555,7 @@ impl Change { self, Change::InsertContainer(_) | Change::UpdateContainer(_) - | Change::UpdateContainerProperty(_) + | Change::UpdateContainerPartial(..) | Change::DeleteContainer(_) ) } @@ -443,45 +563,91 @@ impl Change { pub fn is_item_change(&self) -> bool { matches!( self, - Change::InsertItem(_) | Change::UpdateItem(_) | Change::DeleteItem(_) + Change::InsertItem(_) + | Change::UpdateItem(_) + | Change::UpdateItemMetadata(_) + | Change::DeleteItem(_) ) } + + pub fn is_metadata_only(&self) -> bool { + match self { + Change::UpdateItemMetadata(_) => true, + Change::UpdateContainerPartial(_, partial) => partial.is_metadata_only(), + _ => false, + } + } } -impl DeserializeVanished for u64 { - fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { - let mut num = [0u8; U64_LEN]; - for i in num.iter_mut() { - *i = *bytes.next()?; +pub(crate) fn decode_vanished_uids( + mut bytes: &[u8], + mailbox_id: u32, + uids: &mut Vec, +) -> Option<()> { + let mut group = 0u64; + + while let Some(group_delta) = bytes.read_leb128::() { + group += group_delta; + let count: usize = bytes.read_leb128()?; + + if group as u32 != mailbox_id { + bytes.skip_leb128(count)?; + continue; + } + + let mut uid = 0u64; + for _ in 0..count { + uid += bytes.read_leb128::()?; + uids.push(uid as u32); } - Some(u64::from_be_bytes(num)) } + + Some(()) } impl DeserializeVanished for (u32, u32) { - fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { - let mut num1 = [0u8; U32_LEN]; - let mut num2 = [0u8; U32_LEN]; - for i in num1.iter_mut().chain(num2.iter_mut()) { - *i = *bytes.next()?; + fn deserialize_vanished(mut bytes: &[u8], items: &mut Vec) -> Option<()> { + let mut group = 0u64; + + while let Some(group_delta) = bytes.read_leb128::() { + group += group_delta; + let count: usize = bytes.read_leb128()?; + + let mut id = 0u64; + for _ in 0..count { + id += bytes.read_leb128::()?; + items.push((group as u32, id as u32)); + } } - Some((u32::from_be_bytes(num1), u32::from_be_bytes(num2))) + + Some(()) } } impl DeserializeVanished for String { - fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { - let mut name = Vec::with_capacity(16); + fn deserialize_vanished(mut bytes: &[u8], items: &mut Vec) -> Option<()> { + let first = items.len(); - loop { - let byte = bytes.next()?; - if *byte != 0 { - name.push(*byte); + while !bytes.is_empty() { + let shared = if items.len() != first { + bytes.read_leb128::()? } else { - break; + 0 + }; + let len = bytes.read_leb128::()?; + let (suffix, rest) = bytes.split_at_checked(len)?; + let suffix = std::str::from_utf8(suffix).ok()?; + + let mut name = String::with_capacity(shared + len); + if shared != 0 { + name.push_str(items.last()?.get(..shared)?); } + name.push_str(suffix); + items.push(name); + bytes = rest; } - String::from_utf8(name).ok() + Some(()) } } + diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 4e4fd4e7..a6c2114c 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -7,19 +7,31 @@ pub mod acl; pub mod log; -use crate::{IterateParams, Key}; +use crate::{Fetch, IterateParams, Key}; impl IterateParams { pub fn new(begin: T, end: T) -> Self { IterateParams { begin, end, - first: false, ascending: true, values: true, + fetch: Fetch::All, } } + pub fn expect_rows(mut self, rows: usize) -> Self { + if !matches!(self.fetch, Fetch::Limit(_)) { + self.fetch = Fetch::Expect(u32::try_from(rows).unwrap_or(u32::MAX)); + } + self + } + + pub fn limit(mut self, rows: usize) -> Self { + self.fetch = Fetch::Limit(u32::try_from(rows).unwrap_or(u32::MAX)); + self + } + pub fn set_ascending(mut self, ascending: bool) -> Self { self.ascending = ascending; self @@ -40,9 +52,8 @@ impl IterateParams { self } - pub fn only_first(mut self) -> Self { - self.first = true; - self + pub fn only_first(self) -> Self { + self.limit(1) } pub fn no_values(mut self) -> Self { diff --git a/crates/store/src/registry/bootstrap.rs b/crates/store/src/registry/bootstrap.rs index 87494bdc..6de49240 100644 --- a/crates/store/src/registry/bootstrap.rs +++ b/crates/store/src/registry/bootstrap.rs @@ -6,6 +6,7 @@ use crate::{RegistryStore, Store, registry::RegistryObject}; use registry::{ + pickle::Pickle, schema::{ prelude::{Object, ObjectType, Property}, structs::ClusterRole, @@ -68,68 +69,33 @@ impl Bootstrap { pub async fn setting>(&mut self) -> trc::Result { let object_id = T::OBJECT.singleton(); - - if let Some(setting) = self.registry.object::(object_id.id()).await? { - let mut errors = Vec::new(); - if setting.validate(&mut errors) { - return Ok(setting); - } - self.errors.push(Error::Validation { object_id, errors }); - } - - Ok(T::default()) + Ok(self + .registry + .get(object_id) + .await? + .map(T::from) + .filter(|setting| self.validate(object_id, setting)) + .unwrap_or_default()) } pub async fn setting_infallible>(&mut self) -> T { - match self.setting::().await { - Ok(setting) => setting, - Err(err) => { - if !self.has_fatal_errors { - self.errors.push(Error::Internal { - object_id: Some(T::OBJECT.singleton()), - error: err, - }); - self.has_fatal_errors = true; - } - T::default() - } - } + let object_id = T::OBJECT.singleton(); + self.fetch_object(object_id) + .await + .map(T::from) + .filter(|setting| self.validate(object_id, setting)) + .unwrap_or_default() } pub async fn get_infallible>(&mut self, id: Id) -> Option { - match self.registry.object::(id).await { - Ok(Some(setting)) => { - let mut errors = Vec::new(); - if setting.validate(&mut errors) { - Some(setting) - } else { - self.errors.push(Error::Validation { - object_id: ObjectId::new(T::OBJECT, id), - errors, - }); - None - } - } - Ok(None) => { - self.errors.push(Error::NotFound { - object_id: ObjectId::new(T::OBJECT, id), - }); - None - } - Err(err) => { - if !self.has_fatal_errors { - self.errors.push(Error::Internal { - object_id: Some(ObjectId::new(T::OBJECT, id)), - error: err, - }); - self.has_fatal_errors = true; - } - None - } - } + let object_id = ObjectId::new(T::OBJECT, id); + self.fetch_existing_object(object_id) + .await + .map(T::from) + .filter(|setting| self.validate(object_id, setting)) } - pub async fn list_infallible>( + pub async fn list_infallible>( &mut self, ) -> Vec> { match self.registry.list::().await { @@ -138,18 +104,43 @@ impl Bootstrap { .filter(|object| self.validate(object.id, &object.object)) .collect(), Err(err) => { - if !self.has_fatal_errors { - self.errors.push(Error::Internal { - object_id: None, - error: err, - }); - self.has_fatal_errors = true; - } + self.internal_error(None, err); Vec::new() } } } + async fn fetch_object(&mut self, object_id: ObjectId) -> Option { + match self.registry.get(object_id).await { + Ok(object) => object, + Err(err) => { + self.internal_error(Some(object_id), err); + None + } + } + } + + async fn fetch_existing_object(&mut self, object_id: ObjectId) -> Option { + match self.registry.get(object_id).await { + Ok(Some(object)) => Some(object), + Ok(None) => { + self.errors.push(Error::NotFound { object_id }); + None + } + Err(err) => { + self.internal_error(Some(object_id), err); + None + } + } + } + + fn internal_error(&mut self, object_id: Option, error: trc::Error) { + if !self.has_fatal_errors { + self.errors.push(Error::Internal { object_id, error }); + self.has_fatal_errors = true; + } + } + pub fn build_error(&mut self, id: ObjectId, message: impl Into) { self.errors.push(Error::Build { object_id: id, diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs index 45c2874b..d5f0a6ec 100644 --- a/crates/store/src/registry/get.rs +++ b/crates/store/src/registry/get.rs @@ -5,20 +5,18 @@ */ use crate::{ - IterateParams, RegistryStore, SUBSPACE_REGISTRY, U16_LEN, U64_LEN, ValueKey, + IterateParams, RegistryStore, U16_LEN, ValueKey, registry::{RegistryObject, local::RegistryInit}, - write::{ - AnyClass, RegistryClass, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - }, + write::{RegistryClass, ValueClass, key::DeserializeBigEndian}, }; use registry::{ - pickle::PickledStream, + pickle::{Pickle, PickledStream}, schema::prelude::{Object, ObjectType}, types::{EnumImpl, ObjectImpl, id::ObjectId}, }; use trc::AddContext; use types::id::Id; +use xxhash_rust::xxh3::xxh3_64; impl RegistryStore { pub async fn get(&self, object_id: ObjectId) -> trc::Result> { @@ -53,53 +51,56 @@ impl RegistryStore { .map(|v| v.map(T::from)) } - pub async fn list>(&self) -> trc::Result>> { - let object_type = T::OBJECT; - + pub async fn list>( + &self, + ) -> trc::Result>> { let mut results = Vec::new(); - self.0 - .store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY, - key: KeySerializer::new(U16_LEN) - .write(object_type.to_id()) - .finalize(), - })), - ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY, - key: KeySerializer::new(U16_LEN + U64_LEN) - .write(object_type.to_id()) - .write(u64::MAX) - .finalize(), - })), - ), - |key, value| { - let id = key.deserialize_be_u64(U16_LEN)?; - let object = PickledStream::new(value) - .and_then(|mut stream| T::unpickle(&mut stream)) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .id(id) - .details(object_type.as_str()) - .ctx(trc::Key::Value, value) - })?; - - results.push(RegistryObject { - id: ObjectId::new(object_type, Id::new(id)), - object, - revision: xxhash_rust::xxh3::xxh3_64(value), - }); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; + self.iterate_objects(T::OBJECT, &mut |id, revision, value| { + let Some(object) = + PickledStream::new(value).and_then(|mut stream| T::unpickle(&mut stream)) + else { + return false; + }; + results.push(RegistryObject { + id, + object, + revision, + }); + true + }) + .await?; Ok(results) } + + async fn iterate_objects( + &self, + object_type: ObjectType, + cb: &mut (dyn FnMut(ObjectId, u64, &[u8]) -> bool + Send + Sync), + ) -> trc::Result<()> { + let (begin, end) = RegistryClass::item_range(object_type.to_id()); + self.0 + .store + .iterate(IterateParams::new(begin, end), |key, value| { + let id = key.deserialize_be_u64(U16_LEN)?; + if cb( + ObjectId::new(object_type, Id::new(id)), + xxh3_64(value), + value, + ) { + Ok(true) + } else { + Err( + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .id(id) + .details(object_type.as_str()) + .ctx(trc::Key::Value, value), + ) + } + }) + .await + .caused_by(trc::location!()) + } } diff --git a/crates/store/src/registry/mod.rs b/crates/store/src/registry/mod.rs index 3550f6f5..f5305c89 100644 --- a/crates/store/src/registry/mod.rs +++ b/crates/store/src/registry/mod.rs @@ -17,11 +17,8 @@ use crate::{ use registry::{ pickle::{Pickle, PickledStream}, schema::{ - prelude::{Object, ObjectInner, ObjectType, Property}, - structs::{ - ArchivedItem, DmarcInternalReport, Metric, SpamTrainingSample, Task, TlsInternalReport, - Trace, - }, + prelude::{OBJ_NOT_STORED, Object, ObjectInner, ObjectType, Property}, + structs::{DmarcInternalReport, SpamTrainingSample, Task, TlsInternalReport}, }, types::{EnumImpl, ObjectImpl, id::ObjectId}, }; @@ -39,6 +36,7 @@ pub struct RegistryQuery { pub filters: Vec, pub(crate) start: RegistryQueryStart, pub(crate) limit: Option, + pub(crate) order: RegistryOrder, } #[derive(Debug, Clone, Copy, Default)] @@ -51,6 +49,20 @@ pub(crate) enum RegistryQueryStart { None, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum RegistryOrder { + #[default] + Any, + Ascending, + Descending, +} + +#[derive(Debug)] +pub struct RegistryPage { + pub ids: T, + pub position: Option, +} + #[derive(Debug)] pub struct RegistryFilter { pub property: Property, @@ -88,6 +100,7 @@ impl Deserialize for Object { fn deserialize_with_key(key: &[u8], bytes: &[u8]) -> trc::Result { let revision = xxhash_rust::xxh3::xxh3_64(bytes); ObjectType::from_id(key.deserialize_be_u16(0)?) + .filter(|object_type| object_type.flags() & OBJ_NOT_STORED == 0) .and_then(|object_id| ObjectInner::unpickle(object_id, &mut PickledStream::new(bytes)?)) .map(|inner| Object { revision, inner }) .ok_or_else(|| { @@ -129,18 +142,6 @@ impl Deserialize for SpamTrainingSample { } } -impl Deserialize for ArchivedItem { - fn deserialize(bytes: &[u8]) -> trc::Result { - PickledStream::new(bytes) - .and_then(|mut stream| Self::unpickle(&mut stream)) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} impl Deserialize for TlsInternalReport { fn deserialize(bytes: &[u8]) -> trc::Result { @@ -211,28 +212,3 @@ impl Deserialize for ObjectIdVersioned { } } -impl Deserialize for Trace { - fn deserialize(bytes: &[u8]) -> trc::Result { - PickledStream::new(bytes) - .and_then(|mut stream| Self::unpickle(&mut stream)) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} - -impl Deserialize for Metric { - fn deserialize(bytes: &[u8]) -> trc::Result { - PickledStream::new(bytes) - .and_then(|mut stream| Self::unpickle(&mut stream)) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} diff --git a/crates/store/src/registry/query.rs b/crates/store/src/registry/query.rs index d7898b71..82af2197 100644 --- a/crates/store/src/registry/query.rs +++ b/crates/store/src/registry/query.rs @@ -5,29 +5,33 @@ */ use crate::{ - IterateParams, RegistryStore, SUBSPACE_REGISTRY_IDX, SUBSPACE_REGISTRY_PK, Store, U16_LEN, - U64_LEN, ValueKey, + Deserialize, IterateParams, Key, RegistryStore, Store, U16_LEN, U64_LEN, ValueKey, + registry::write::TEXT_TERMINATOR, registry::{ RegistryFilter, RegistryFilterOp, RegistryFilterValue, RegistryObjectCounter, - RegistryQuery, RegistryQueryStart, - }, - write::{ - AnyClass, RegistryClass, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, + RegistryOrder, RegistryPage, RegistryQuery, RegistryQueryStart, }, + write::{RegistryClass, ValueClass, key::DeserializeBigEndian}, }; use ahash::AHashSet; use registry::{ - schema::prelude::{OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, ObjectType, Property}, + schema::prelude::{OBJ_FILTER_ACCOUNT, OBJ_SINGLETON, ObjectType, Property}, types::{EnumImpl, id::ObjectId}, }; use roaring::RoaringBitmap; -use std::{borrow::Cow, ops::BitAndAssign}; +use std::ops::{BitAndAssign, Bound, RangeBounds}; use trc::AddContext; use types::id::Id; impl RegistryStore { pub async fn query(&self, query: RegistryQuery) -> trc::Result { + self.query_page(query).await.map(|page| page.ids) + } + + pub async fn query_page( + &self, + query: RegistryQuery, + ) -> trc::Result> { if query.filters.is_empty() { return all_ids::(&self.0.store, query).await; } @@ -35,9 +39,14 @@ impl RegistryStore { let mut u64_buffer; let mut u16_buffer; let mut bool_buffer = [0u8; 1]; + let mut text_buffer = Vec::new(); let mut results = ResultsPagination::::new(&query); - for filter in &query.filters { + let mut merged = 0u64; + for (position, filter) in query.filters.iter().enumerate() { + if position < u64::BITS as usize && merged & (1 << position) != 0 { + continue; + } if filter.op == RegistryFilterOp::TextMatch { if let RegistryFilterValue::String(text) = &filter.value { let mut matches = ResultsPagination::::new(&query); @@ -46,14 +55,16 @@ impl RegistryStore { .split(|c: char| !c.is_alphanumeric()) .filter(|s| s.len() > 1) { - let word = if word + text_buffer.clear(); + if word .chars() .all(|ch| ch.is_lowercase() || !ch.is_alphabetic()) { - Cow::Borrowed(word) + text_buffer.extend_from_slice(word.as_bytes()); } else { - Cow::Owned(word.to_lowercase()) - }; + text_buffer.extend_from_slice(word.to_lowercase().as_bytes()); + } + text_buffer.push(TEXT_TERMINATOR); let mut result = ResultsPagination::::new(&query); @@ -61,8 +72,9 @@ impl RegistryStore { &self.0.store, query.object_type, filter.property.to_id(), - word.as_bytes(), + &text_buffer, RegistryFilterOp::Equal, + RangeKeys::Matching, &mut result, ) .await?; @@ -71,9 +83,9 @@ impl RegistryStore { matches = result; } else { matches.list.intersect(&result.list); - if !matches.list.has_items() { - break; - } + } + if !matches.list.has_items() { + break; } } @@ -89,6 +101,12 @@ impl RegistryStore { } } else { let value = match &filter.value { + RegistryFilterValue::String(v) if !filter.is_pk => { + text_buffer.clear(); + text_buffer.extend_from_slice(v.as_bytes()); + text_buffer.push(TEXT_TERMINATOR); + &text_buffer + } RegistryFilterValue::String(v) => v.as_bytes(), RegistryFilterValue::Bytes(v) => v.as_slice(), RegistryFilterValue::U64(v) => { @@ -105,6 +123,18 @@ impl RegistryStore { } }; + let keys = if filter.op == RegistryFilterOp::Equal + && match filter.value { + RegistryFilterValue::U64(_) + | RegistryFilterValue::U16(_) + | RegistryFilterValue::Boolean(_) => true, + RegistryFilterValue::String(_) => !filter.is_pk, + RegistryFilterValue::Bytes(_) => false, + } { + RangeKeys::Matching + } else { + RangeKeys::Filtered + }; let mut result = ResultsPagination::::new(&query); if !filter.is_pk { index_range( @@ -113,16 +143,25 @@ impl RegistryStore { filter.property.to_id(), value, filter.op, + keys, &mut result, ) .await? } else { + let mut bounds = op_bounds(filter.op, value); + if let Some((upper_position, upper)) = upper_bound_of(&query.filters, position) + && upper_position < u64::BITS as usize + && merged & (1 << upper_position) == 0 + { + bounds.1 = upper; + merged |= 1 << upper_position; + } pk_range( &self.0.store, query.object_type, filter.property.to_id(), - value, - filter.op, + bounds, + keys, &mut result, ) .await? @@ -177,6 +216,54 @@ impl RegistryStore { .await } + pub async fn first_primary_key( + &self, + object_type: Option, + lookups: &[(Property, &[u8])], + ) -> trc::Result> { + if let [(property, key)] = lookups { + return self + .primary_key(object_type, *property, key.to_vec()) + .await + .map(|found| found.map(|object_id| (0, object_id))); + } + + let object_id = object_type.map(|object| object.to_id()); + let mut points = lookups + .iter() + .enumerate() + .map(|(position, (property, key))| { + let key = ValueKey::from(ValueClass::Registry(RegistryClass::PrimaryKey { + object_id, + index_id: property.to_id(), + key: key.to_vec(), + })); + (key.serialize(0), key, position) + }) + .collect::>(); + points.sort_unstable_by(|a, b| a.0.cmp(&b.0).then(a.2.cmp(&b.2))); + points.dedup_by(|next, kept| next.0 == kept.0); + + let ranges = points + .iter() + .map(|(_, key, _)| IterateParams::new(key.clone(), key.clone())) + .collect(); + let mut found: Option<(usize, ObjectId)> = None; + self.store() + .iterate_many(ranges, |key, value| { + if let Ok(index) = points.binary_search_by(|(point, ..)| point.as_slice().cmp(key)) + && let Some((_, _, position)) = points.get(index) + && found.is_none_or(|(first, _)| *position < first) + { + found = Some((*position, ObjectId::deserialize(value)?)); + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + Ok(found) + } + pub async fn sort_by_index( &self, object: ObjectType, @@ -189,21 +276,11 @@ impl RegistryStore { let object_id = object.to_id(); let index_id = property.to_id(); - let begin = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_IDX, - key: KeySerializer::new(U16_LEN * 2) - .write(object_id) - .write(index_id) - .finalize(), - })); - let end = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_IDX, - key: KeySerializer::new((U16_LEN * 2) + U64_LEN) - .write(object_id) - .write(index_id) - .write(u64::MAX) - .finalize(), - })); + let (begin, end) = RegistryClass::index_range( + object_id, + (index_id, &[], None), + (index_id, &[], Some(u64::MAX)), + ); self.0 .store @@ -249,21 +326,11 @@ impl RegistryStore { let object_id = object.to_id(); let index_id = property.to_id(); - let begin = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_PK, - key: KeySerializer::new(U16_LEN * 2) - .write(object_id) - .write(index_id) - .finalize(), - })); - let end = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_PK, - key: KeySerializer::new((U16_LEN * 2) + U64_LEN) - .write(object_id) - .write(index_id) - .write(u64::MAX) - .finalize(), - })); + let (begin, end) = RegistryClass::pk_range( + object_id, + (index_id, &[], None), + (index_id, &[], Some(u64::MAX)), + ); self.0 .store @@ -297,43 +364,41 @@ impl RegistryStore { } } -async fn all_ids(store: &Store, query: RegistryQuery) -> trc::Result { - let mut bm = T::default(); +async fn all_ids( + store: &Store, + query: RegistryQuery, +) -> trc::Result> { let object_id = query.object_type.to_id(); - - let (item_id, mut offset) = match query.start { - RegistryQueryStart::Index(index) => (0, index), - RegistryQueryStart::Anchor(anchor) => (anchor + 1, 0), - RegistryQueryStart::None => (0, 0), - }; + let mut results = ResultsPagination::::new(&query); + let params = results.bounded( + IterateParams::new( + ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { + object_id, + item_id: 0, + })), + ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { + object_id, + item_id: u64::MAX, + })), + ) + .no_values(), + RangeKeys::Matching, + ); store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { - object_id, - item_id, - })), - ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { - object_id, - item_id: u64::MAX, - })), - ) - .no_values() - .ascending(), - |key, _| { - if offset == 0 { - bm.push(key.deserialize_be_u64(U16_LEN * 2)?); - Ok(query.limit.is_none_or(|limit| bm.count() < limit)) - } else { - offset -= 1; - Ok(true) - } - }, - ) + .iterate(params, |key, _| { + Ok(results.push(key.deserialize_be_u64(U16_LEN * 2)?)) + }) .await - .caused_by(trc::location!()) - .map(|_| bm) + .caused_by(trc::location!())?; + + Ok(results.finalize()) +} + +#[derive(Clone, Copy)] +enum RangeKeys { + Matching, + Filtered, } async fn index_range( @@ -342,6 +407,7 @@ async fn index_range( index_id: u16, match_value: &[u8], op: RegistryFilterOp, + keys: RangeKeys, results: &mut ResultsPagination, ) -> trc::Result<()> { let ((from_value, from_doc_id, from_index_id), (end_value, end_doc_id, end_index_id)) = match op @@ -365,106 +431,124 @@ async fn index_range( }; let object_id = object.to_id(); - let begin = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_IDX, - key: KeySerializer::new((U16_LEN * 2) + U64_LEN + from_value.len()) - .write(object_id) - .write(from_index_id) - .write(from_value) - .write(from_doc_id) - .finalize(), - })); - let end = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_IDX, - key: KeySerializer::new((U16_LEN * 2) + U64_LEN + end_value.len()) - .write(object_id) - .write(end_index_id) - .write(end_value) - .write(end_doc_id) - .finalize(), - })); - - let prefix = KeySerializer::new(U16_LEN * 2) - .write(object_id) - .write(index_id) - .finalize(); + let (begin, end) = RegistryClass::index_range( + object_id, + (from_index_id, from_value, Some(from_doc_id)), + (end_index_id, end_value, Some(end_doc_id)), + ); + let prefix = RegistryClass::prefix(object_id, index_id); + let params = results.bounded(IterateParams::new(begin, end).no_values(), keys); store - .iterate( - IterateParams::new(begin, end).no_values().ascending(), - |key, _| { - if !key.starts_with(&prefix) { - return Ok(false); - } + .iterate(params, |key, _| { + if !key.starts_with(&prefix) { + return Ok(false); + } - let id_pos = key.len() - U64_LEN; - let value = key - .get(U16_LEN * 2..id_pos) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; + let id_pos = key.len() - U64_LEN; + let value = key + .get(U16_LEN * 2..id_pos) + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - let matches = match op { - RegistryFilterOp::LowerThan => value < match_value, - RegistryFilterOp::LowerEqualThan => value <= match_value, - RegistryFilterOp::GreaterThan => value > match_value, - RegistryFilterOp::GreaterEqualThan => value >= match_value, - RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => value == match_value, - }; + let matches = match op { + RegistryFilterOp::LowerThan => value < match_value, + RegistryFilterOp::LowerEqualThan => value <= match_value, + RegistryFilterOp::GreaterThan => value > match_value, + RegistryFilterOp::GreaterEqualThan => value >= match_value, + RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => value == match_value, + }; - if matches { - Ok(results.push(key.deserialize_be_u64(id_pos)?)) - } else { - Ok(true) - } - }, - ) + if matches { + Ok(results.push(key.deserialize_be_u64(id_pos)?)) + } else { + Ok(true) + } + }) .await .caused_by(trc::location!()) - .inspect(|_| results.list.sort()) + .inspect(|_| results.scanned()) +} + +fn op_bounds(op: RegistryFilterOp, value: &[u8]) -> (Bound<&[u8]>, Bound<&[u8]>) { + match op { + RegistryFilterOp::LowerThan => (Bound::Unbounded, Bound::Excluded(value)), + RegistryFilterOp::LowerEqualThan => (Bound::Unbounded, Bound::Included(value)), + RegistryFilterOp::GreaterThan => (Bound::Excluded(value), Bound::Unbounded), + RegistryFilterOp::GreaterEqualThan => (Bound::Included(value), Bound::Unbounded), + RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => { + (Bound::Included(value), Bound::Included(value)) + } + } +} + +fn upper_bound_of(filters: &[RegistryFilter], lower: usize) -> Option<(usize, Bound<&[u8]>)> { + let filter = filters.get(lower)?; + if !filter.is_pk + || !matches!( + filter.op, + RegistryFilterOp::GreaterThan | RegistryFilterOp::GreaterEqualThan + ) + { + return None; + } + filters + .iter() + .enumerate() + .skip(lower + 1) + .filter(|(_, upper)| upper.is_pk && upper.property == filter.property) + .find_map(|(position, upper)| { + let value = match &upper.value { + RegistryFilterValue::String(value) => value.as_bytes(), + RegistryFilterValue::Bytes(value) => value.as_slice(), + _ => return None, + }; + match upper.op { + RegistryFilterOp::LowerThan => Some((position, Bound::Excluded(value))), + RegistryFilterOp::LowerEqualThan => Some((position, Bound::Included(value))), + _ => None, + } + }) } async fn pk_range( store: &Store, object: ObjectType, index_id: u16, - match_value: &[u8], - op: RegistryFilterOp, + bounds: (Bound<&[u8]>, Bound<&[u8]>), + keys: RangeKeys, results: &mut ResultsPagination, ) -> trc::Result<()> { - let ((from_value, from_index_id), (end_value, end_index_id)) = match op { - RegistryFilterOp::LowerThan => ((&[][..], index_id), (match_value, index_id)), - RegistryFilterOp::LowerEqualThan => ((&[][..], index_id), (match_value, index_id)), - RegistryFilterOp::GreaterThan => ((match_value, index_id), (&[][..], index_id + 1)), - RegistryFilterOp::GreaterEqualThan => ((match_value, index_id), (&[][..], index_id + 1)), - RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => { - ((match_value, index_id), (match_value, index_id)) - } + if match bounds { + (Bound::Included(from), Bound::Included(to)) => from > to, + ( + Bound::Included(from) | Bound::Excluded(from), + Bound::Included(to) | Bound::Excluded(to), + ) => from >= to, + _ => false, + } { + return Ok(()); + } + + let (from_value, from_index_id) = match bounds.0 { + Bound::Included(value) | Bound::Excluded(value) => (value, index_id), + Bound::Unbounded => (&[][..], index_id), + }; + let (end_value, end_index_id) = match bounds.1 { + Bound::Included(value) | Bound::Excluded(value) => (value, index_id), + Bound::Unbounded => (&[][..], index_id + 1), }; let object_id = object.to_id(); - let begin = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_PK, - key: KeySerializer::new((U16_LEN * 2) + from_value.len()) - .write(object_id) - .write(from_index_id) - .write(from_value) - .finalize(), - })); - let end = ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY_PK, - key: KeySerializer::new((U16_LEN * 2) + end_value.len()) - .write(object_id) - .write(end_index_id) - .write(end_value) - .finalize(), - })); - - let prefix = KeySerializer::new(U16_LEN * 2) - .write(object_id) - .write(index_id) - .finalize(); + let (begin, end) = RegistryClass::pk_range( + object_id, + (from_index_id, from_value, None), + (end_index_id, end_value, None), + ); + let prefix = RegistryClass::prefix(object_id, index_id); + let params = results.bounded(IterateParams::new(begin, end), keys); store - .iterate(IterateParams::new(begin, end).ascending(), |key, value| { + .iterate(params, |key, value| { if !key.starts_with(&prefix) { return Ok(false); } @@ -473,15 +557,7 @@ async fn pk_range( .get(U16_LEN * 2..) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - let matches = match op { - RegistryFilterOp::LowerThan => key < match_value, - RegistryFilterOp::LowerEqualThan => key <= match_value, - RegistryFilterOp::GreaterThan => key > match_value, - RegistryFilterOp::GreaterEqualThan => key >= match_value, - RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => key == match_value, - }; - - if matches { + if RangeBounds::<[u8]>::contains(&bounds, key) { Ok(results.push(value.deserialize_be_u64(U16_LEN)?)) } else { Ok(true) @@ -489,7 +565,7 @@ async fn pk_range( }) .await .caused_by(trc::location!()) - .inspect(|_| results.list.sort()) + .inspect(|_| results.scanned()) } pub trait RegistryQueryResults: Default + Sized + Sync + Send { @@ -498,7 +574,7 @@ pub trait RegistryQueryResults: Default + Sized + Sync + Send { fn intersect(&mut self, other: &Self); fn count(&self) -> usize; fn sort(&mut self); - fn into_list(self) -> impl Iterator; + fn into_list(self) -> impl DoubleEndedIterator; } impl RegistryQueryResults for Vec { @@ -548,7 +624,7 @@ impl RegistryQueryResults for Vec { } } - fn into_list(self) -> impl Iterator { + fn into_list(self) -> impl DoubleEndedIterator { self.into_iter().map(|id| id.id()) } } @@ -572,7 +648,7 @@ impl RegistryQueryResults for RoaringBitmap { fn sort(&mut self) {} - fn into_list(self) -> impl Iterator { + fn into_list(self) -> impl DoubleEndedIterator { self.into_iter().map(|id| id as u64) } } @@ -596,80 +672,134 @@ impl RegistryQueryResults for RegistryObjectCounter { fn sort(&mut self) {} - fn into_list(self) -> impl Iterator { + fn into_list(self) -> impl DoubleEndedIterator { Vec::new().into_iter() } } struct ResultsPagination { list: T, + start: RegistryQueryStart, offset: usize, anchor: Option, + skipped: usize, limit: Option, + order: RegistryOrder, deferred_pagination: bool, } impl ResultsPagination { fn new(query: &RegistryQuery) -> Self { let (anchor, offset) = match query.start { - RegistryQueryStart::Index(index) => (None, index), + RegistryQueryStart::Index(index) => { + (None, usize::try_from(index).unwrap_or(usize::MAX)) + } RegistryQueryStart::Anchor(anchor) => (Some(anchor), 0), RegistryQueryStart::None => (None, 0), }; Self { list: T::default(), - offset: offset as usize, + start: query.start, + offset, anchor, + skipped: 0, limit: query.limit, + order: query.order, deferred_pagination: query.filters.len() > 1 - || query.filters.first().is_some_and(|f| { - if let (RegistryFilterOp::TextMatch, RegistryFilterValue::String(value)) = - (&f.op, &f.value) - { - value.chars().any(|c| !c.is_alphanumeric()) && value.len() > 1 - } else { - false - } - }), + || query + .filters + .first() + .is_some_and(|f| match (&f.op, &f.value) { + (RegistryFilterOp::TextMatch, RegistryFilterValue::String(value)) => { + value.chars().any(|c| !c.is_alphanumeric()) && value.len() > 1 + } + (RegistryFilterOp::TextMatch | RegistryFilterOp::Equal, _) => false, + _ => query.order != RegistryOrder::Any, + }), + } + } + + fn bounded(&self, params: IterateParams, keys: RangeKeys) -> IterateParams { + let params = params + .set_ascending(self.deferred_pagination || self.order != RegistryOrder::Descending); + match (self.rows_needed(), keys) { + (Some(rows), RangeKeys::Matching) => params.limit(rows), + (_, RangeKeys::Matching | RangeKeys::Filtered) => params, + } + } + + fn rows_needed(&self) -> Option { + if self.deferred_pagination || self.anchor.is_some() { + None + } else { + self.limit.map(|limit| self.offset.saturating_add(limit)) } } fn push(&mut self, id: u64) -> bool { - if !self.deferred_pagination { - if self.offset > 0 { - self.offset -= 1; - true - } else if let Some(anchor) = self.anchor { - if id == anchor { - self.anchor = None; - } - true - } else { - self.list.push(id); - self.limit.is_none_or(|limit| self.list.count() < limit) - } - } else { + if self.deferred_pagination { self.list.push(id); true + } else if self.offset > 0 { + self.offset -= 1; + true + } else if self.anchor.is_some_and(|anchor| anchor != id) { + self.skipped += 1; + true + } else { + self.anchor = None; + match self.limit { + Some(limit) if self.list.count() >= limit => false, + limit => { + self.list.push(id); + limit.is_none_or(|limit| self.list.count() < limit) + } + } } } - fn finalize(mut self) -> T { - if self.deferred_pagination - && self.list.has_items() - && (self.limit.is_some() || self.anchor.is_some() || self.offset > 0) - { - let list = std::mem::take(&mut self.list); - self.deferred_pagination = false; + fn scanned(&mut self) { + if self.deferred_pagination || self.order != RegistryOrder::Descending { + self.list.sort(); + } + } - for item in list.into_list() { - if !self.push(item) { - break; - } + fn paginate(&mut self, ids: impl Iterator) { + for id in ids { + if !self.push(id) { + break; } } - self.list + } + + fn finalize(mut self) -> RegistryPage { + if self.deferred_pagination + && (self.limit.is_some() + || self.anchor.is_some() + || self.offset > 0 + || self.order == RegistryOrder::Descending) + { + self.deferred_pagination = false; + let ids = std::mem::take(&mut self.list).into_list(); + if self.order == RegistryOrder::Descending { + self.paginate(ids.rev()); + } else { + self.paginate(ids); + } + } + + RegistryPage { + position: match self.start { + RegistryQueryStart::Index(_) if self.offset > 0 => Some(0), + RegistryQueryStart::Index(index) => { + Some(usize::try_from(index).unwrap_or(usize::MAX)) + } + RegistryQueryStart::Anchor(_) => self.anchor.is_none().then_some(self.skipped), + RegistryQueryStart::None => Some(0), + }, + ids: self.list, + } } } @@ -680,9 +810,20 @@ impl RegistryQuery { filters: Vec::new(), start: RegistryQueryStart::None, limit: None, + order: RegistryOrder::Any, } } + pub fn with_ascending(mut self) -> Self { + self.order = RegistryOrder::Ascending; + self + } + + pub fn with_descending(mut self) -> Self { + self.order = RegistryOrder::Descending; + self + } + pub fn with_anchor(mut self, anchor: u64) -> Self { self.start = RegistryQueryStart::Anchor(anchor); self @@ -718,19 +859,6 @@ impl RegistryQuery { } } - pub fn with_tenant(mut self, tenant_id: Option) -> Self { - if let Some(tenant_id) = tenant_id - && self.object_type.flags() & OBJ_FILTER_TENANT != 0 - { - let filter = RegistryFilter::equal(Property::MemberTenantId, tenant_id, false); - if self.filters.is_empty() { - self.filters.push(filter); - } else { - self.filters.insert(0, filter); - } - } - self - } pub fn filter(mut self, filter: RegistryFilter) -> Self { self.filters.push(filter); @@ -941,3 +1069,4 @@ impl From for RegistryFilterValue { RegistryFilterValue::Boolean(value) } } + diff --git a/crates/store/src/registry/write.rs b/crates/store/src/registry/write.rs index 61f2c630..db437ed8 100644 --- a/crates/store/src/registry/write.rs +++ b/crates/store/src/registry/write.rs @@ -7,14 +7,14 @@ use crate::{ IterateParams, RegistryStore, SerializeInfallible, U16_LEN, U64_LEN, ValueKey, write::{ - BatchBuilder, RegistryClass, ValueClass, + BatchBuilder, MAX_ACCOUNT_ID, RegistryClass, ValueClass, assert::AssertValue, key::{DeserializeBigEndian, KeySerializer}, }, }; use registry::{ schema::prelude::{ - OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SEQ_ID, OBJ_SINGLETON, Object, ObjectInner, + OBJ_FILTER_ACCOUNT, OBJ_NOT_STORED, OBJ_SEQ_ID, OBJ_SINGLETON, Object, ObjectInner, ObjectType, Property, }, types::{ @@ -149,16 +149,26 @@ impl RegistryStore { allowed_orphan_types, } => { return if object_id.object().flags() & OBJ_SINGLETON == 0 { - self.delete(object_id, object, allowed_orphan_types).await + Box::pin(self.delete(object_id, object, allowed_orphan_types)).await } else { Ok(RegistryWriteResult::CannotDeleteSingleton) }; } } + if object_flags & OBJ_NOT_STORED != 0 && object_type != ObjectType::DataStore { + return Ok(RegistryWriteResult::NotSupported); + } + // Validate object let mut errors = Vec::new(); object.validate(&mut errors); + if let Some(property) = set_index.keys.iter().find_map(IndexKey::text_with_nul) { + errors.push(ValidationError::Invalid { + property, + value: "Indexed text cannot contain U+0000".to_string(), + }); + } if !errors.is_empty() { return Ok(RegistryWriteResult::ValidationError { errors }); } @@ -176,17 +186,16 @@ impl RegistryStore { } // Validate foreign keys - let tenant_id = object.inner.member_tenant_id().map(|id| id.id()); let account_id = object .inner .account_id() .map(|id| id.id()) .or_else(|| (object_type == ObjectType::Account).then_some(item_id)); - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] let set_keys = &set_index.keys; - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] let set_keys = set_index .keys .iter() @@ -223,26 +232,10 @@ impl RegistryStore { return Ok(RegistryWriteResult::InvalidForeignKey { object_id: *foreign_id, }); - } else if let Some(tenant_id) = tenant_id - && (object_flags & OBJ_FILTER_TENANT) != 0 - && !self - .0 - .store - .key_exists(ValueKey::from(ValueClass::Registry( - RegistryClass::Index { - index_id: Property::MemberTenantId.to_id(), - object_id, - item_id, - key: IndexValue::U64(tenant_id).serialize(), - }, - ))) - .await - .caused_by(trc::location!())? - { - return Ok(RegistryWriteResult::InvalidForeignKey { - object_id: *foreign_id, - }); - } else if (object_flags & OBJ_FILTER_ACCOUNT) != 0 + } + + + if (object_flags & OBJ_FILTER_ACCOUNT) != 0 && let Some(account_id) = account_id && !self .0 @@ -292,21 +285,6 @@ impl RegistryStore { } } - // Assign id - if generate_id { - let mut id_batch = BatchBuilder::new(); - id_batch.add_and_get( - ValueClass::Registry(RegistryClass::IdCounter { object_id }), - 1, - ); - item_id = self - .0 - .store - .write(id_batch.build_all()) - .await - .and_then(|v| v.last_counter_id())? as u64; - } - // It's pickle time! let out = object.inner.to_pickled_vec(); if out.len() > MAX_OBJECT_PAYLOAD_SIZE { @@ -322,6 +300,27 @@ impl RegistryStore { }); } + // Assign id + if generate_id { + let mut id_batch = BatchBuilder::new(); + id_batch.add_and_get( + ValueClass::Registry(RegistryClass::IdCounter { object_id }), + 1, + ); + item_id = self + .0 + .store + .write_batch(&mut id_batch) + .await? + .last_counter_id() as u64; + if object_type == ObjectType::Account && item_id > u64::from(MAX_ACCOUNT_ID) { + return Err(trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Id, item_id) + .details("Account id space exhausted")); + } + } + // Build batch if write_id { batch.set( @@ -339,7 +338,7 @@ impl RegistryStore { ); self.store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .map(|_| RegistryWriteResult::Success(Id::new(item_id))) } @@ -405,9 +404,10 @@ impl RegistryStore { })) .registry_index(object_type_id, item_id, clear_index.keys.iter(), false); + self.0 .store - .write(batch.build_all()) + .write_batch(&mut batch) .await .map(|_| RegistryWriteResult::Success(Id::from(item_id))) .caused_by(trc::location!()) @@ -435,20 +435,16 @@ impl RegistryStore { .iterate( IterateParams::new(from_key, to_key).no_values().ascending(), |key, _| { - if key.len() == (U16_LEN * 2) + (U64_LEN * 2) { - let object = - ObjectType::from_id(key.deserialize_be_u16(U64_LEN + U16_LEN)?) - .ok_or_else(|| { - trc::EventType::Registry( - trc::RegistryEvent::DeserializationError, - ) + let object = + ObjectType::from_id(key.deserialize_be_u16((U16_LEN * 2) + U64_LEN)?) + .ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) .into_err() .caused_by(trc::location!()) .ctx(trc::Key::Key, key) - })?; - let id = key.deserialize_be_u64(U64_LEN + U16_LEN + U16_LEN)?; - linked.push(ObjectId::new(object, Id::new(id))); - } + })?; + let id = key.deserialize_be_u64((U16_LEN * 3) + U64_LEN)?; + linked.push(ObjectId::new(object, Id::new(id))); Ok(true) }, @@ -479,7 +475,7 @@ impl BatchBuilder { index_id: property.to_id(), object_id, item_id, - key: value.serialize(), + key: search_key(value), }, vec![], ), @@ -525,19 +521,40 @@ impl BatchBuilder { } } +pub(crate) const TEXT_TERMINATOR: u8 = 0; + fn serialize_composite_key(value_1: &IndexValue<'_>, value_2: &IndexValue<'_>) -> Vec { let mut key = value_1.serialize(); match value_2 { - IndexValue::Text(text) => key.extend_from_slice(text.as_bytes()), - IndexValue::Bytes(bytes) => key.extend_from_slice(bytes), + IndexValue::Text(text) => { + key.push(TEXT_TERMINATOR); + key.extend_from_slice(text.as_bytes()); + } + IndexValue::Bytes(bytes) => { + key.push(TEXT_TERMINATOR); + key.extend_from_slice(bytes); + } IndexValue::U64(num) => key.extend_from_slice(&num.to_be_bytes()), IndexValue::I64(num) => key.extend_from_slice(&num.to_be_bytes()), + IndexValue::U32(num) => key.extend_from_slice(&num.to_be_bytes()), IndexValue::U16(num) => key.extend_from_slice(&num.to_be_bytes()), IndexValue::None => {} } key } +fn search_key(value: &IndexValue<'_>) -> Vec { + match value { + IndexValue::Text(text) => { + let mut key = Vec::with_capacity(text.len() + 1); + key.extend_from_slice(text.as_bytes()); + key.push(TEXT_TERMINATOR); + key + } + value => value.serialize(), + } +} + impl SerializeInfallible for IndexValue<'_> { fn serialize(&self) -> Vec { match self { @@ -545,6 +562,7 @@ impl SerializeInfallible for IndexValue<'_> { IndexValue::Bytes(bytes) => bytes.clone(), IndexValue::U64(num) => num.to_be_bytes().to_vec(), IndexValue::I64(num) => num.to_be_bytes().to_vec(), + IndexValue::U32(num) => num.to_be_bytes().to_vec(), IndexValue::U16(num) => num.to_be_bytes().to_vec(), IndexValue::None => vec![], } diff --git a/crates/store/src/search/bm_u32.rs b/crates/store/src/search/bm_u32.rs deleted file mode 100644 index 32040fe9..00000000 --- a/crates/store/src/search/bm_u32.rs +++ /dev/null @@ -1,278 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{ - IterateParams, Store, U32_LEN, ValueKey, - search::*, - write::{ - SEARCH_INDEX_MAX_FIELD_LEN, SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, - SearchIndexType, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - }, -}; -use ahash::AHashMap; -use roaring::RoaringBitmap; -use std::{ - collections::hash_map::Entry, - ops::{BitAndAssign, BitOrAssign}, -}; -use trc::AddContext; -use utils::cheeky_hash::CheekyHash; - -#[derive(Default)] -pub(super) struct BitmapCache { - cache: AHashMap<(CheekyHash, u8), Option>, -} - -impl BitmapCache { - pub async fn merge_bitmaps( - &mut self, - store: &Store, - index: SearchIndex, - account_id: u32, - hashes: impl Iterator, - field: u8, - is_union: bool, - ) -> trc::Result> { - let mut result = RoaringBitmap::new(); - for (idx, hash) in hashes.enumerate() { - match self.cache.entry((hash, field)) { - Entry::Occupied(entry) => { - if let Some(bm) = entry.get() { - if is_union { - result.bitor_assign(bm); - } else if idx == 0 { - result = bm.clone(); - } else { - result.bitand_assign(bm); - if result.is_empty() { - return Ok(None); - } - } - } else if !is_union { - return Ok(None); - } - } - Entry::Vacant(entry) => { - let from_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: 0, - }, - typ: SearchIndexType::Term { hash, field }, - })); - let to_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: u32::MAX, - }, - typ: SearchIndexType::Term { hash, field }, - })); - let key_len = (U32_LEN * 2) + hash.len() + 2; - let mut documents = RoaringBitmap::new(); - store - .iterate( - IterateParams::new(from_key, to_key).no_values().ascending(), - |key, _| { - if key.len() == key_len { - documents.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if !documents.is_empty() { - if is_union { - result.bitor_assign(&documents); - } else if idx == 0 { - result = documents.clone(); - } else { - result.bitand_assign(&documents); - if result.is_empty() { - entry.insert(Some(documents)); - return Ok(None); - } - } - entry.insert(Some(documents)); - } else if !is_union { - entry.insert(None); - return Ok(None); - } - } - } - } - - if !result.is_empty() { - Ok(Some(result)) - } else { - Ok(None) - } - } -} - -pub(crate) async fn range_to_bitmap( - store: &Store, - index: SearchIndex, - account_id: u32, - field_id: u8, - match_value: &[u8], - op: SearchOperator, -) -> trc::Result> { - let ((from_value, from_doc_id, from_field), (end_value, end_doc_id, end_field)) = match op { - SearchOperator::LowerThan => ((&[][..], 0, field_id), (match_value, 0, field_id)), - SearchOperator::LowerEqualThan => { - ((&[][..], 0, field_id), (match_value, u32::MAX, field_id)) - } - SearchOperator::GreaterThan => ( - (match_value, u32::MAX, field_id), - (&[][..], u32::MAX, field_id + 1), - ), - SearchOperator::GreaterEqualThan => ( - (match_value, 0, field_id), - (&[][..], u32::MAX, field_id + 1), - ), - SearchOperator::Equal | SearchOperator::Contains => ( - (match_value, 0, field_id), - (match_value, u32::MAX, field_id), - ), - }; - - let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: from_doc_id, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: from_field, - data: from_value.to_vec(), - }, - }, - })); - - let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: end_doc_id, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: end_field, - data: end_value.to_vec(), - }, - }, - })); - - let mut bm = RoaringBitmap::new(); - let prefix = KeySerializer::new(U32_LEN + 2) - .write(index.as_u8() | 1 << 6) - .write(account_id) - .write(field_id) - .finalize(); - let prefix_len = prefix.len(); - - store - .iterate( - IterateParams::new(begin, end).no_values().ascending(), - |key, _| { - if !key.starts_with(&prefix) { - return Ok(false); - } - - let id_pos = key.len() - U32_LEN; - let value = key - .get(prefix_len..id_pos) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - - let matches = match op { - SearchOperator::LowerThan => value < match_value, - SearchOperator::LowerEqualThan => value <= match_value, - SearchOperator::GreaterThan => value > match_value, - SearchOperator::GreaterEqualThan => value >= match_value, - SearchOperator::Equal | SearchOperator::Contains => value == match_value, - }; - - if matches { - bm.insert(key.deserialize_be_u32(id_pos)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if !bm.is_empty() { - Ok(Some(bm)) - } else { - Ok(None) - } -} - -pub(crate) async fn sort_order( - store: &Store, - index: SearchIndex, - account_id: u32, - field_id: u8, -) -> trc::Result> { - let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: 0, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id, - data: vec![0u8], - }, - }, - })); - let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: u32::MAX, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id, - data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], - }, - }, - })); - - let mut last_value = Vec::new(); - let mut results = AHashMap::new(); - let mut pos = 0; - store - .iterate( - IterateParams::new(begin, end).no_values().ascending(), - |key, _| { - let value = key - .get(U32_LEN + 2..key.len() - U32_LEN) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - if value != last_value { - pos += 1; - last_value = value.to_vec(); - } - - results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?, pos); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - Ok(results) -} diff --git a/crates/store/src/search/bm_u64.rs b/crates/store/src/search/bm_u64.rs deleted file mode 100644 index 3fda59a7..00000000 --- a/crates/store/src/search/bm_u64.rs +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{ - IterateParams, Store, U64_LEN, ValueKey, - search::*, - write::{ - SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, - ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - }, -}; -use ahash::AHashMap; -use roaring::RoaringTreemap; -use std::{ - collections::hash_map::Entry, - ops::{BitAndAssign, BitOrAssign}, -}; -use trc::AddContext; -use utils::cheeky_hash::CheekyHash; - -#[derive(Default)] -pub(super) struct TreemapCache { - cache: AHashMap<(CheekyHash, u8), Option>, -} - -impl TreemapCache { - pub async fn merge_treemaps( - &mut self, - store: &Store, - index: SearchIndex, - hashes: impl Iterator, - field: u8, - is_union: bool, - ) -> trc::Result> { - let mut result = RoaringTreemap::new(); - for (idx, hash) in hashes.enumerate() { - match self.cache.entry((hash, field)) { - Entry::Occupied(entry) => { - if let Some(bm) = entry.get() { - if is_union { - result.bitor_assign(bm); - } else if idx == 0 { - result = bm.clone(); - } else { - result.bitand_assign(bm); - if result.is_empty() { - return Ok(None); - } - } - } else if !is_union { - return Ok(None); - } - } - Entry::Vacant(entry) => { - let from_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: 0 }, - typ: SearchIndexType::Term { hash, field }, - })); - let to_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: u64::MAX }, - typ: SearchIndexType::Term { hash, field }, - })); - let key_len = U64_LEN + hash.len() + 2; - let mut documents = RoaringTreemap::new(); - store - .iterate( - IterateParams::new(from_key, to_key).no_values().ascending(), - |key, _| { - if key.len() == key_len { - documents.insert(key.deserialize_be_u64(key.len() - U64_LEN)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if !documents.is_empty() { - if is_union { - result.bitor_assign(&documents); - } else if idx == 0 { - result = documents.clone(); - } else { - result.bitand_assign(&documents); - if result.is_empty() { - entry.insert(Some(documents)); - return Ok(None); - } - } - entry.insert(Some(documents)); - } else if !is_union { - entry.insert(None); - return Ok(None); - } - } - } - } - - if !result.is_empty() { - Ok(Some(result)) - } else { - Ok(None) - } - } -} - -pub(crate) async fn range_to_treemap( - store: &Store, - index: SearchIndex, - field_id: u8, - match_value: &[u8], - op: SearchOperator, -) -> trc::Result> { - let ((from_value, from_id, from_field), (end_value, end_id, end_field)) = match op { - SearchOperator::LowerThan => ((&[][..], 0, field_id), (match_value, 0, field_id)), - SearchOperator::LowerEqualThan => { - ((&[][..], 0, field_id), (match_value, u64::MAX, field_id)) - } - SearchOperator::GreaterThan => ( - (match_value, u64::MAX, field_id), - (&[][..], u64::MAX, field_id + 1), - ), - SearchOperator::GreaterEqualThan => ( - (match_value, 0, field_id), - (&[][..], u64::MAX, field_id + 1), - ), - SearchOperator::Equal | SearchOperator::Contains => ( - (match_value, 0, field_id), - (match_value, u64::MAX, field_id), - ), - }; - - let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: from_id }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: from_field, - data: from_value.to_vec(), - }, - }, - })); - - let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: end_id }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: end_field, - data: end_value.to_vec(), - }, - }, - })); - - let mut bm = RoaringTreemap::new(); - let prefix = KeySerializer::new(U64_LEN + 2) - .write(index.as_u8() | 1 << 6) - .write(field_id) - .finalize(); - let prefix_len = prefix.len(); - - store - .iterate( - IterateParams::new(begin, end).no_values().ascending(), - |key, _| { - if !key.starts_with(&prefix) { - return Ok(false); - } - - let id_pos = key.len() - U64_LEN; - let value = key - .get(prefix_len..id_pos) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - - let matches = match op { - SearchOperator::LowerThan => value < match_value, - SearchOperator::LowerEqualThan => value <= match_value, - SearchOperator::GreaterThan => value > match_value, - SearchOperator::GreaterEqualThan => value >= match_value, - SearchOperator::Equal | SearchOperator::Contains => value == match_value, - }; - - if matches { - bm.insert(key.deserialize_be_u64(id_pos)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if !bm.is_empty() { - Ok(Some(bm)) - } else { - Ok(None) - } -} diff --git a/crates/store/src/search/codec.rs b/crates/store/src/search/codec.rs new file mode 100644 index 00000000..1629194a --- /dev/null +++ b/crates/store/src/search/codec.rs @@ -0,0 +1,350 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Subspace, U16_LEN, U32_LEN, + search::{MAX_STORED_DOCUMENT_SIZE, SearchField}, + write::{ + AnyKey, Dictionary, SearchIndex, + compress::{COMPRESS_WATERMARK_DICTIONARY, compress, decompress_into}, + }, +}; +use utils::{ + cheeky_hash::CheekyHash, + codec::leb128::{Leb128, Leb128Reader, Leb128Writer}, +}; + +const RANGE_PADDING: [u8; 22] = [u8::MAX; 22]; +const DOCUMENT_PLAIN: u8 = 0; +const DOCUMENT_ZSTD: u8 = 1; + +pub(crate) const ACCOUNT_TERM_BASE_LEN: usize = U32_LEN + 2; +pub(crate) const GLOBAL_TERM_BASE_LEN: usize = 2; + +pub(crate) struct Writer { + buf: Vec, +} + +pub(crate) struct AccountTermKey<'x> { + pub term: &'x [u8], + pub len: u8, + pub field: u8, + pub block_id: u16, +} + +pub(crate) struct PrefixKey<'x> { + pub term: &'x [u8], + pub len: u8, + pub block_id: u16, +} + +pub(crate) struct Reader<'x> { + bytes: &'x [u8], +} + +impl<'x> Reader<'x> { + pub fn new(bytes: &'x [u8]) -> Self { + Reader { bytes } + } + + #[inline(always)] + pub fn leb128(&mut self) -> Option { + self.bytes.read_leb128() + } + + #[inline(always)] + pub fn skip_leb128(&mut self, count: usize) -> Option<()> { + self.bytes.skip_leb128(count) + } + + pub fn u8(&mut self) -> Option { + let (&byte, rest) = self.bytes.split_first()?; + self.bytes = rest; + Some(byte) + } + + pub fn slice(&mut self, len: usize) -> Option<&'x [u8]> { + let (slice, rest) = self.bytes.split_at_checked(len)?; + self.bytes = rest; + Some(slice) + } + + pub fn term(&mut self) -> Option { + let len = self.u8()?; + let key = self.slice((len as usize).min(CheekyHash::HASH_SIZE))?; + CheekyHash::from_key_bytes(key, len) + } +} + +impl Writer { + pub fn with_capacity(capacity: usize) -> Self { + Writer { + buf: Vec::with_capacity(capacity + 1), + } + } + + pub fn into_document(self) -> Vec { + let mut plain = self.buf; + + if plain.len() >= COMPRESS_WATERMARK_DICTIONARY + && let Ok(mut compressed) = compress(Some(Dictionary::Term), &plain, 1) + && compressed.len() < plain.len() + { + compressed.push(DOCUMENT_ZSTD); + return compressed; + } + + plain.reserve_exact(1); + plain.push(DOCUMENT_PLAIN); + plain + } + + pub fn push_leb128(&mut self, value: T) { + self.buf.push_leb128(value); + } + + pub fn push_u8(&mut self, value: u8) { + self.buf.push(value); + } + + pub fn push_term(&mut self, term: &CheekyHash) { + self.buf.push(term.len() as u8); + self.buf.extend_from_slice(term.as_key()); + } +} + +pub(crate) fn read_document<'x>(stored: &'x [u8], scratch: &'x mut Vec) -> Option<&'x [u8]> { + let (marker, payload) = stored.split_last()?; + + match *marker { + DOCUMENT_PLAIN => Some(payload), + DOCUMENT_ZSTD => decompress_into(payload, scratch, MAX_STORED_DOCUMENT_SIZE) + .ok() + .map(|_| scratch.as_slice()), + _ => None, + } +} + +pub(crate) fn term_key(key: Vec) -> AnyKey> { + AnyKey { + subspace: Subspace::SearchTerm, + key, + } +} + +pub(crate) fn document_key(key: Vec) -> AnyKey> { + AnyKey { + subspace: Subspace::SearchDocument, + key, + } +} + +pub(crate) fn account_region_range(index: SearchIndex, account_id: u32) -> (Vec, Vec) { + let mut begin = Vec::with_capacity(1 + U32_LEN + RANGE_PADDING.len()); + begin.push(index.to_u8()); + begin.extend_from_slice(&account_id.to_be_bytes()); + let mut end = begin.clone(); + end.extend_from_slice(&RANGE_PADDING); + (begin, end) +} + +fn account_term_base(index: SearchIndex, account_id: u32, group: u8, capacity: usize) -> Vec { + let mut key = Vec::with_capacity(ACCOUNT_TERM_BASE_LEN + capacity); + key.push(index.to_u8()); + key.extend_from_slice(&account_id.to_be_bytes()); + key.push(group); + key +} + +pub(crate) fn account_term_prefix_range( + index: SearchIndex, + account_id: u32, + group: u8, + prefix: &[u8], +) -> (Vec, Vec) { + let mut begin = account_term_base(index, account_id, group, prefix.len() + RANGE_PADDING.len()); + begin.extend_from_slice(prefix); + let mut end = begin.clone(); + end.extend_from_slice(&RANGE_PADDING); + (begin, end) +} + +pub(crate) fn account_term_range( + index: SearchIndex, + account_id: u32, + field: u8, + term: &CheekyHash, +) -> (Vec, Vec) { + let mut begin = account_term_base( + index, + account_id, + index.term_group(field), + term.key_len() + U16_LEN + 3, + ); + begin.extend_from_slice(term.as_key()); + begin.push(term.len() as u8); + begin.push(field); + let mut end = begin.clone(); + end.extend_from_slice(&[u8::MAX; U16_LEN + 1]); + (begin, end) +} + +pub(crate) fn account_term_block_key( + index: SearchIndex, + account_id: u32, + field: u8, + term: &CheekyHash, + block: u16, +) -> Vec { + let mut key = account_term_base( + index, + account_id, + index.term_group(field), + term.key_len() + U16_LEN + 3, + ); + key.extend_from_slice(term.as_key()); + key.push(term.len() as u8); + key.push(field); + key.extend_from_slice(&block.to_be_bytes()); + key +} + +pub(crate) fn account_term_block_range( + index: SearchIndex, + account_id: u32, + field: u8, + term: &CheekyHash, + from_block: u16, + until_block: u16, +) -> (Vec, Vec) { + let begin = account_term_block_key(index, account_id, field, term, from_block); + let mut end = account_term_block_key(index, account_id, field, term, until_block); + end.push(u8::MAX); + (begin, end) +} + +pub(crate) fn account_terms_range( + index: SearchIndex, + account_id: u32, + group: u8, + first: &CheekyHash, + last: &CheekyHash, +) -> (Vec, Vec) { + let mut begin = account_term_base(index, account_id, group, first.key_len() + 1); + let mut end = begin.clone(); + begin.extend_from_slice(first.as_key()); + begin.push(first.len() as u8); + end.extend_from_slice(last.as_key()); + end.push(last.len() as u8); + end.extend_from_slice(&[u8::MAX; U16_LEN + 2]); + (begin, end) +} + +pub(crate) fn parse_account_term_key(key: &[u8]) -> Option> { + let (rest, block) = key + .get(ACCOUNT_TERM_BASE_LEN..)? + .split_last_chunk::()?; + let (field, rest) = rest.split_last()?; + let (len, term) = rest.split_last()?; + Some(AccountTermKey { + term, + len: *len, + field: *field, + block_id: u16::from_be_bytes(*block), + }) +} + +pub(crate) fn global_term_prefix_range( + index: SearchIndex, + field: u8, + prefix: &[u8], +) -> (Vec, Vec) { + let mut begin = Vec::with_capacity(prefix.len() + RANGE_PADDING.len() + 2); + begin.push(index.to_u8()); + begin.push(field); + begin.extend_from_slice(prefix); + let mut end = begin.clone(); + end.extend_from_slice(&RANGE_PADDING); + (begin, end) +} + +pub(crate) fn account_document_key( + index: SearchIndex, + account_id: u32, + document_id: u32, +) -> Vec { + let mut key = Vec::with_capacity(1 + U32_LEN * 2); + key.push(index.to_u8()); + key.extend_from_slice(&account_id.to_be_bytes()); + key.extend_from_slice(&document_id.to_be_bytes()); + key +} + +pub(crate) fn global_term_block_range( + index: SearchIndex, + field: u8, + term: &CheekyHash, + from_block: u16, + until_block: u16, +) -> (Vec, Vec) { + let mut begin = Vec::with_capacity(GLOBAL_TERM_BASE_LEN + term.key_len() + U16_LEN + 2); + begin.push(index.to_u8()); + begin.push(field); + begin.extend_from_slice(term.as_key()); + begin.push(term.len() as u8); + let mut end = begin.clone(); + begin.extend_from_slice(&from_block.to_be_bytes()); + end.extend_from_slice(&until_block.to_be_bytes()); + end.push(u8::MAX); + (begin, end) +} + +pub(crate) fn global_document_id_range( + index: SearchIndex, + from_block: u16, + until_block: u16, +) -> (Vec, Vec) { + let mut begin = Vec::with_capacity(U16_LEN + 3); + begin.push(index.to_u8()); + begin.push(SearchField::Id.u8_id()); + let mut end = begin.clone(); + begin.extend_from_slice(&from_block.to_be_bytes()); + end.extend_from_slice(&until_block.to_be_bytes()); + end.push(u8::MAX); + (begin, end) +} + +pub(crate) fn parse_prefix_key( + key: &[u8], + base_len: usize, + block_len: usize, +) -> Option> { + let mut suffix = key.get(base_len..)?; + let mut block_id = 0; + if block_len > 0 { + let (rest, block) = suffix.split_at_checked(suffix.len().checked_sub(block_len)?)?; + block_id = match block { + [block] => *block as u16, + [hi, lo] => u16::from_be_bytes([*hi, *lo]), + _ => return None, + }; + suffix = rest; + } + let (len, term) = suffix.split_last()?; + Some(PrefixKey { + term, + len: *len, + block_id, + }) +} + +pub(crate) fn parse_block_id(key: &[u8]) -> Option { + key.get(key.len().checked_sub(U16_LEN)?..) + .and_then(|bytes| bytes.try_into().ok()) + .map(u16::from_be_bytes) +} + + diff --git a/crates/store/src/search/document.rs b/crates/store/src/search/document.rs index 974648d2..ec9f36b3 100644 --- a/crates/store/src/search/document.rs +++ b/crates/store/src/search/document.rs @@ -4,7 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +#[cfg(feature = "postgres")] +use crate::backend::MAX_TOKEN_LENGTH; use crate::search::*; +#[cfg(feature = "postgres")] +use nlp::tokenizers::space::SpaceTokenizer; impl IndexDocument { pub fn new(index: SearchIndex) -> Self { @@ -28,6 +32,26 @@ impl IndexDocument { self } + #[cfg(feature = "postgres")] + pub(crate) fn split_identifiers(&mut self, buffer: &mut String) { + for (field, value) in self.fields.iter_mut() { + if let SearchValue::Text { value, .. } = value + && field.is_identifier() + { + buffer.clear(); + buffer.reserve(2 * value.len() + 1); + for word in SpaceTokenizer::new(value, MAX_TOKEN_LENGTH) { + buffer.push_str(&word); + buffer.push(' '); + } + if !buffer.is_empty() { + buffer.push_str(value); + std::mem::swap(value, buffer); + } + } + } + } + pub fn with_id(mut self, id: u64) -> Self { self.fields.insert(SearchField::Id, SearchValue::Uint(id)); self @@ -113,10 +137,27 @@ impl IndexDocument { } } + pub fn index(&self) -> SearchIndex { + self.index + } + pub fn is_empty(&self) -> bool { self.fields.is_empty() } + pub fn text_len(&self) -> usize { + self.fields + .values() + .map(|value| match value { + SearchValue::Text { value, .. } => value.len(), + SearchValue::KeyValues(map) => { + map.iter().map(|(key, value)| key.len() + value.len()).sum() + } + SearchValue::Int(_) | SearchValue::Uint(_) | SearchValue::Boolean(_) => 0, + }) + .sum() + } + pub fn has_field(&self, field: &SearchField) -> bool { self.fields.contains_key(field) } @@ -137,54 +178,52 @@ impl IndexDocument { } impl SearchFilter { - pub fn cond( - field: impl Into, - op: SearchOperator, - value: impl Into, - ) -> Self { - SearchFilter::Operator { + pub fn text_eq(field: impl Into, value: impl Into) -> Self { + SearchFilter::Text { field: field.into(), - op, + op: TextMatch::Exact, + value: value.into(), + language: Language::None, + } + } + + pub fn text_prefix(field: impl Into, value: impl Into) -> Self { + SearchFilter::Text { + field: field.into(), + op: TextMatch::Prefix, + value: value.into(), + language: Language::None, + } + } + + pub fn integer_lt(field: impl Into, value: impl Into) -> Self { + SearchFilter::Integer { + field: field.into(), + op: Ordering::Less, value: value.into(), } } - pub fn eq(field: impl Into, value: impl Into) -> Self { - SearchFilter::Operator { + pub fn integer_eq(field: impl Into, value: impl Into) -> Self { + SearchFilter::Integer { field: field.into(), - op: SearchOperator::Equal, + op: Ordering::Equal, value: value.into(), } } - pub fn lt(field: impl Into, value: impl Into) -> Self { - SearchFilter::Operator { + pub fn signed_eq(field: impl Into, value: impl Into) -> Self { + SearchFilter::Integer { field: field.into(), - op: SearchOperator::LowerThan, - value: value.into(), + op: Ordering::Equal, + value: crate::search::tokenize::zigzag(value.into()), } } - pub fn le(field: impl Into, value: impl Into) -> Self { - SearchFilter::Operator { + pub fn integer_gt(field: impl Into, value: impl Into) -> Self { + SearchFilter::Integer { field: field.into(), - op: SearchOperator::LowerEqualThan, - value: value.into(), - } - } - - pub fn gt(field: impl Into, value: impl Into) -> Self { - SearchFilter::Operator { - field: field.into(), - op: SearchOperator::GreaterThan, - value: value.into(), - } - } - - pub fn ge(field: impl Into, value: impl Into) -> Self { - SearchFilter::Operator { - field: field.into(), - op: SearchOperator::GreaterEqualThan, + op: Ordering::Greater, value: value.into(), } } @@ -204,34 +243,23 @@ impl SearchFilter { language: Language, ) -> Self { let text = text.into(); - let (is_exact, text) = if let Some(text) = text + let (op, text) = if let Some(text) = text .strip_prefix('"') .and_then(|t| t.strip_suffix('"')) .or_else(|| text.strip_prefix('\'').and_then(|t| t.strip_suffix('\''))) { - (true, text.to_string()) + (TextMatch::Exact, text.to_string()) + } else if let Some(text) = text.strip_suffix('*') { + (TextMatch::Prefix, text.trim().to_string()) } else { - (false, text) + (TextMatch::Standard, text) }; - if !matches!(language, Language::None) && is_exact { - SearchFilter::Operator { - field: field.into(), - op: SearchOperator::Equal, - value: SearchValue::Text { - value: text, - language, - }, - } - } else { - SearchFilter::Operator { - field: field.into(), - op: SearchOperator::Contains, - value: SearchValue::Text { - value: text, - language, - }, - } + SearchFilter::Text { + field: field.into(), + op, + value: text, + language, } } @@ -251,13 +279,6 @@ impl SearchFilter { } impl SearchComparator { - pub fn field(field: impl Into, ascending: bool) -> Self { - Self::Field { - field: field.into(), - ascending, - } - } - pub fn set(set: RoaringBitmap, ascending: bool) -> Self { Self::DocumentSet { set, ascending } } @@ -265,20 +286,6 @@ impl SearchComparator { pub fn sorted_set(set: AHashMap, ascending: bool) -> Self { Self::SortedSet { set, ascending } } - - pub fn ascending(field: impl Into) -> Self { - Self::Field { - field: field.into(), - ascending: true, - } - } - - pub fn descending(field: impl Into) -> Self { - Self::Field { - field: field.into(), - ascending: false, - } - } } #[inline(always)] @@ -313,3 +320,4 @@ fn sanitize_text(text: &str) -> String { write_sanitized(&mut out, text); out } + diff --git a/crates/store/src/search/fields.rs b/crates/store/src/search/fields.rs index 7cf7e263..d5875ac7 100644 --- a/crates/store/src/search/fields.rs +++ b/crates/store/src/search/fields.rs @@ -24,44 +24,12 @@ impl SearchableField for EmailSearchField { SearchField::Email(EmailSearchField::Subject), SearchField::Email(EmailSearchField::Body), SearchField::Email(EmailSearchField::Attachment), - SearchField::Email(EmailSearchField::ReceivedAt), - SearchField::Email(EmailSearchField::SentAt), - SearchField::Email(EmailSearchField::Size), - SearchField::Email(EmailSearchField::HasAttachment), SearchField::Email(EmailSearchField::Headers), ] } fn is_indexed(&self) -> bool { - #[cfg(not(feature = "test_mode"))] - { - matches!( - self, - EmailSearchField::From - | EmailSearchField::To - | EmailSearchField::Subject - | EmailSearchField::ReceivedAt - | EmailSearchField::SentAt - | EmailSearchField::Size - | EmailSearchField::HasAttachment, - ) - } - - #[cfg(feature = "test_mode")] - { - matches!( - self, - EmailSearchField::From - | EmailSearchField::To - | EmailSearchField::Subject - | EmailSearchField::ReceivedAt - | EmailSearchField::SentAt - | EmailSearchField::Size - | EmailSearchField::HasAttachment - | EmailSearchField::Bcc - | EmailSearchField::Cc - ) - } + false } fn is_text(&self) -> bool { @@ -94,17 +62,15 @@ impl SearchableField for CalendarSearchField { SearchField::Calendar(CalendarSearchField::Location), SearchField::Calendar(CalendarSearchField::Owner), SearchField::Calendar(CalendarSearchField::Attendee), - SearchField::Calendar(CalendarSearchField::Start), - SearchField::Calendar(CalendarSearchField::Uid), ] } fn is_indexed(&self) -> bool { - matches!(self, CalendarSearchField::Start | CalendarSearchField::Uid) + false } fn is_text(&self) -> bool { - !self.is_indexed() + true } } @@ -129,16 +95,15 @@ impl SearchableField for ContactSearchField { SearchField::Contact(ContactSearchField::OnlineService), SearchField::Contact(ContactSearchField::Address), SearchField::Contact(ContactSearchField::Note), - SearchField::Contact(ContactSearchField::Uid), ] } fn is_indexed(&self) -> bool { - matches!(self, ContactSearchField::Uid | ContactSearchField::Kind) + matches!(self, ContactSearchField::Kind) } fn is_text(&self) -> bool { - !self.is_indexed() + !matches!(self, ContactSearchField::Kind) } } @@ -152,10 +117,7 @@ impl SearchableField for FileSearchField { } fn all_fields() -> &'static [SearchField] { - &[ - SearchField::File(FileSearchField::Name), - SearchField::File(FileSearchField::Content), - ] + &[SearchField::File(FileSearchField::Content)] } fn is_indexed(&self) -> bool { @@ -219,6 +181,20 @@ impl SearchField { } } + #[cfg(feature = "postgres")] + pub(crate) fn is_identifier(&self) -> bool { + matches!( + self, + SearchField::Contact( + ContactSearchField::Email + | ContactSearchField::Phone + | ContactSearchField::OnlineService + | ContactSearchField::Member + ) | SearchField::Calendar(CalendarSearchField::Owner | CalendarSearchField::Attendee) + ) + } + + #[cfg(feature = "postgres")] pub(crate) fn is_json(&self) -> bool { matches!(self, SearchField::Email(EmailSearchField::Headers)) } diff --git a/crates/store/src/search/index.rs b/crates/store/src/search/index.rs index a45bf46c..57995bf3 100644 --- a/crates/store/src/search/index.rs +++ b/crates/store/src/search/index.rs @@ -5,332 +5,344 @@ */ use crate::{ - Deserialize, IterateParams, Store, U64_LEN, ValueKey, + IterateParams, Store, U32_LEN, ValueKey, search::{ - IndexDocument, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, - term::{TermIndex, TermIndexBuilder}, - }, - write::{ - AlignedBytes, Archive, BatchBuilder, SEARCH_INDEX_MAX_FIELD_LEN, SearchIndex, - SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, ValueClass, - key::DeserializeBigEndian, + GLOBAL_BUCKET_SHIFT, IndexDocument, MAX_DOCUMENT_SIZE, SearchField, SearchValue, codec, + term::{AccountIndexer, GlobalIndexer, deserialize_term_fields}, }, + write::{BatchBuilder, SearchIndex, SearchIndexClass, ValueClass}, }; -use ahash::AHashMap; +use ahash::AHashSet; use trc::AddContext; -use utils::cheeky_hash::CheekyHash; + +const STORED_DOCUMENT_BATCH: usize = 1024; impl Store { - pub(crate) async fn index(&self, documents: Vec) -> trc::Result<()> { - let truncate_at = if self.is_foundationdb() { 1_048_576 } else { 0 }; + pub async fn index_account_documents( + &self, + account_id: u32, + index: SearchIndex, + index_documents: Vec, + unindex_document_ids: Vec, + ) -> trc::Result<()> { + let diffs = matches!( + index, + SearchIndex::Email | SearchIndex::Calendar | SearchIndex::Contacts | SearchIndex::File + ); + let documents = numbered_documents(index_documents)?; - for document in documents { - let mut batch = BatchBuilder::new(); - let index = document.index; - let mut old_term_index = None; - - if matches!(index, SearchIndex::Calendar | SearchIndex::Contacts) { - let mut account_id = None; - let mut document_id = None; - for (field, value) in &document.fields { - if let SearchValue::Uint(id) = value { - match field { - SearchField::AccountId => { - account_id = Some(*id as u32); - } - SearchField::DocumentId => { - document_id = Some(*id as u32); - } - _ => {} - } - } - } - - if let (Some(account_id), Some(document_id)) = (account_id, document_id) - && let Some(archive) = self - .get_value::>(ValueKey::from( - ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id, - }, - typ: SearchIndexType::Document, - }), - )) - .await - .caused_by(trc::location!())? - { - old_term_index = Some(archive); - } - } - - let term_index_builder = TermIndexBuilder::build(document, truncate_at); - if let Some(old_term_index) = old_term_index { - let old_term_index = old_term_index - .unarchive::() - .caused_by(trc::location!())?; - term_index_builder - .index - .merge_index(&mut batch, index, term_index_builder.id, old_term_index) - .caused_by(trc::location!())?; - } else { - term_index_builder - .index - .write_index(&mut batch, index, term_index_builder.id) - .caused_by(trc::location!())?; - } - - let mut commit_points = batch.commit_points(); - for commit_point in commit_points.iter() { - let batch = batch.build_one(commit_point); - self.write(batch).await.caused_by(trc::location!())?; - } + let mut read_ids = unindex_document_ids.clone(); + if diffs { + read_ids.extend(documents.iter().map(|(document_id, _)| *document_id)); } + let stored = self + .stored_documents(index, account_id, read_ids) + .await + .caused_by(trc::location!())?; + + let mut batch = account_batch( + index, + account_id, + documents, + &unindex_document_ids, + &stored, + diffs, + )?; + self.write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + Ok(()) } - pub(crate) async fn unindex(&self, query: SearchQuery) -> trc::Result<()> { - let index = query.index; - let mut account_documents: AHashMap> = AHashMap::new(); - let mut ids = vec![]; - let mut to_id = None; - let mut last_account_id = None; + async fn stored_documents( + &self, + index: SearchIndex, + account_id: u32, + mut document_ids: Vec, + ) -> trc::Result)>> { + document_ids.sort_unstable(); + document_ids.dedup(); + let mut stored = Vec::with_capacity(document_ids.len()); + for chunk in document_ids.chunks(STORED_DOCUMENT_BATCH) { + let ranges = chunk + .iter() + .map(|document_id| { + let key = codec::account_document_key(index, account_id, *document_id); + IterateParams::new(codec::document_key(key.clone()), codec::document_key(key)) + }) + .collect::>(); + let mut corrupted = None; + self.iterate_many(ranges, |key, value| { + match key + .get(key.len().wrapping_sub(U32_LEN)..) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_be_bytes) + { + Some(document_id) => { + stored.push((document_id, value.to_vec())); + Ok(true) + } + None => { + corrupted = Some(trc::Error::corrupted_key( + key, + Some(value), + trc::location!(), + )); + Ok(false) + } + } + }) + .await?; + if let Some(err) = corrupted { + return Err(err); + } + } + Ok(stored) + } - for filter in query.filters { - match filter { - SearchFilter::Operator { field, op, value } => match (field, value) { - (SearchField::AccountId, SearchValue::Uint(id)) - if op == SearchOperator::Equal => - { - last_account_id = Some(id as u32); - account_documents.entry(id as u32).or_default(); - } - (SearchField::DocumentId, SearchValue::Uint(id)) - if op == SearchOperator::Equal && last_account_id.is_some() => - { - account_documents - .get_mut(&last_account_id.unwrap()) - .unwrap() - .push(id as u32); - } - (SearchField::Id, SearchValue::Uint(id)) => match op { - SearchOperator::LowerThan => { - to_id = Some(id.saturating_sub(1)); - } - SearchOperator::LowerEqualThan => { - to_id = Some(id); - } - SearchOperator::Equal => { - ids.push(id); - } - _ => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .reason("Unsupported operator for Id field")); - } - }, - filter => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details(format!("Unsupported unindex filter {filter:?}"))); - } - }, - SearchFilter::And | SearchFilter::Or | SearchFilter::End => {} - SearchFilter::Not | SearchFilter::DocumentSet(_) => { - return Err(trc::StoreEvent::UnexpectedError + pub async fn index_trace(&self, documents: Vec) -> trc::Result<()> { + let mut global_indexer = GlobalIndexer::default(); + + for document in documents.into_iter().rev() { + let document_id = document + .fields + .get(&SearchField::Id) + .and_then(|v| match v { + SearchValue::Uint(v) => Some(*v), + _ => None, + }) + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError .into_err() - .details(format!("Unsupported unindex filter {filter:?}"))); - } - } - } + .details("Missing id for document") + })?; - // Delete by account and document ids - for (account_id, document_ids) in account_documents { - if !document_ids.is_empty() { - for document_id in document_ids { - let Some(archive) = self - .get_value::>(ValueKey::from( - ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id, - }, - typ: SearchIndexType::Document, - }), - )) - .await - .caused_by(trc::location!())? - else { - continue; - }; - let term_index = archive - .unarchive::() - .caused_by(trc::location!())?; - let mut batch = BatchBuilder::new(); - term_index.delete_index( - &mut batch, - index, - SearchIndexId::Account { - account_id, - document_id, - }, - ); - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - } else { - // Delete all documents for the account - self.delete_range( - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: 0, - }, - typ: SearchIndexType::Document, - })), - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: u32::MAX, - }, - typ: SearchIndexType::Document, - })), - ) - .await - .caused_by(trc::location!())?; - - self.delete_range( - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: 0, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: 0, - data: vec![0u8], - }, - }, - })), - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: u32::MAX, - }, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: u8::MAX, - data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], - }, - }, - })), - ) - .await - .caused_by(trc::location!())?; - - self.delete_range( - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: 0, - }, - typ: SearchIndexType::Term { - hash: CheekyHash::NULL, - field: 0, - }, - })), - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Account { - account_id, - document_id: u32::MAX, - }, - typ: SearchIndexType::Term { - hash: CheekyHash::FULL, - field: u8::MAX, - }, - })), - ) - .await - .caused_by(trc::location!())?; - } - } - - // Delete by global ids - for id in ids { - let Some(archive) = self - .get_value::>(ValueKey::from(ValueClass::SearchIndex( - SearchIndexClass { - index, - id: SearchIndexId::Global { id }, - typ: SearchIndexType::Document, - }, - ))) - .await - .caused_by(trc::location!())? - else { + if global_indexer.documents.contains_key(&document_id) { continue; - }; - let term_index = archive - .unarchive::() - .caused_by(trc::location!())?; - let mut batch = BatchBuilder::new(); - term_index.delete_index(&mut batch, index, SearchIndexId::Global { id }); - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; + } + + if let Some(size) = global_indexer.insert(document, document_id) { + trc::event!( + SearchIndex(trc::SearchIndexEvent::DocumentTruncated), + DocumentId = document_id, + Size = size, + Limit = MAX_DOCUMENT_SIZE, + ); + } } - // Delete ranges - if let Some(to_id) = to_id { - let mut batches = Vec::new(); - self.iterate( - IterateParams::new( - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: 0 }, - typ: SearchIndexType::Document, - })), - ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { - index, - id: SearchIndexId::Global { id: to_id }, - typ: SearchIndexType::Document, - })), - ), - |key, value| { - let archive = as Deserialize>::deserialize(value)?; - let term_index = archive.unarchive::()?; - let mut batch = BatchBuilder::new(); - term_index.delete_index( - &mut batch, - index, - SearchIndexId::Global { - id: key.deserialize_be_u64(key.len() - U64_LEN)?, - }, - ); - batches.push(batch); + let mut batch = global_indexer.build_batch(SearchIndex::Tracing); + self.write_batch(&mut batch) + .await + .caused_by(trc::location!())?; - Ok(true) - }, + Ok(()) + } + + pub async fn unindex_account(&self, index: SearchIndex, account_id: u32) -> trc::Result<()> { + let (begin, end) = codec::account_region_range(index, account_id); + + self.delete_range(codec::term_key(begin.clone()), codec::term_key(end.clone())) + .await + .caused_by(trc::location!())?; + + self.delete_range(codec::document_key(begin), codec::document_key(end)) + .await + .caused_by(trc::location!())?; + + Ok(()) + } + + pub async fn clear_search_index_queue( + &self, + index: SearchIndex, + partition: u32, + ) -> trc::Result<()> { + let (from_class, to_class) = SearchIndexClass::queue_range(index, partition); + self.delete_range(ValueKey::from(from_class), ValueKey::from(to_class)) + .await + .caused_by(trc::location!())?; + + let mut batch = BatchBuilder::new(); + batch + .clear(SearchIndexClass::QueueIndex { index, partition }) + .clear(SearchIndexClass::QueueStatus { index, partition }); + self.write_batch(&mut batch) + .await + .caused_by(trc::location!()) + .map(|_| ()) + } + + pub async fn unindex_traces(&self, before_id: u64) -> trc::Result<()> { + let last_bucket = (before_id >> GLOBAL_BUCKET_SHIFT) + .checked_sub(1) + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("Invalid id cutoff for global unindex") + })? as u16; + let last_id = + ((last_bucket as u64) << GLOBAL_BUCKET_SHIFT) | ((1 << GLOBAL_BUCKET_SHIFT) - 1); + let mut delete_terms = AHashSet::new(); + let mut has_documents = false; + let mut scratch = Vec::new(); + + self.iterate( + IterateParams::new( + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalDocument { + index: SearchIndex::Tracing, + document_id: 0, + })), + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalDocument { + index: SearchIndex::Tracing, + document_id: last_id + 1, + })), + ), + |key, value| { + has_documents = true; + deserialize_term_fields(value, &mut scratch, |term_hash, mut field_mask| { + while field_mask != 0 { + let item = 31 - field_mask.leading_zeros(); + field_mask ^= 1 << item; + delete_terms.insert((term_hash, item as u8)); + } + }) + .ok_or_else(|| trc::Error::corrupted_key(key, value.into(), trc::location!()))?; + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if has_documents { + // Delete terms + for (term_hash, field) in delete_terms { + self.delete_range( + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalTerm { + index: SearchIndex::Tracing, + field, + term: term_hash, + block_id: 0, + })), + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalTerm { + index: SearchIndex::Tracing, + field, + term: term_hash, + block_id: last_bucket + 1, + })), + ) + .await + .caused_by(trc::location!())?; + } + + // Delete global documents + self.delete_range( + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalDocument { + index: SearchIndex::Tracing, + document_id: 0, + })), + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::GlobalDocument { + index: SearchIndex::Tracing, + document_id: last_id + 1, + })), ) .await .caused_by(trc::location!())?; - for mut batch in batches { - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } + // Delete global document ids + self.delete_range( + ValueKey::from(ValueClass::SearchIndex( + SearchIndexClass::GlobalDocumentId { + index: SearchIndex::Tracing, + block_id: 0, + }, + )), + ValueKey::from(ValueClass::SearchIndex( + SearchIndexClass::GlobalDocumentId { + index: SearchIndex::Tracing, + block_id: last_bucket + 1, + }, + )), + ) + .await + .caused_by(trc::location!())?; } Ok(()) } } + +pub(crate) fn numbered_documents( + index_documents: Vec, +) -> trc::Result> { + let mut documents = Vec::with_capacity(index_documents.len()); + for document in index_documents.into_iter().rev() { + let document_id = document + .fields + .get(&SearchField::DocumentId) + .and_then(|v| match v { + SearchValue::Uint(v) => Some(*v as u32), + _ => None, + }) + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("Missing documentId for document") + })?; + documents.push((document_id, document)); + } + Ok(documents) +} + +pub(crate) fn account_batch( + index: SearchIndex, + account_id: u32, + documents: Vec<(u32, IndexDocument)>, + unindex_document_ids: &[u32], + stored: &[(u32, Vec)], + diffs: bool, +) -> trc::Result { + let mut account_indexer = AccountIndexer::default(); + for document_id in unindex_document_ids { + if let Some(document) = stored_document(stored, *document_id) { + account_indexer + .remove(document, *document_id) + .caused_by(trc::location!())?; + } + } + + for (document_id, document) in documents { + if account_indexer.documents.contains_key(&document_id) { + continue; + } + + if let Some(size) = account_indexer.insert(document, document_id) { + trc::event!( + SearchIndex(trc::SearchIndexEvent::DocumentTruncated), + AccountId = account_id, + DocumentId = document_id, + Size = size, + Limit = MAX_DOCUMENT_SIZE, + ); + } + + if diffs && let Some(current_document) = stored_document(stored, document_id) { + account_indexer + .diff(current_document, document_id) + .caused_by(trc::location!())?; + } + } + + account_indexer.build_batch(index, account_id) +} + +pub(crate) fn stored_document(stored: &[(u32, Vec)], document_id: u32) -> Option<&[u8]> { + stored + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|position| stored.get(position)) + .map(|(_, document)| document.as_slice()) +} + diff --git a/crates/store/src/search/leaf.rs b/crates/store/src/search/leaf.rs new file mode 100644 index 00000000..f1e0dae3 --- /dev/null +++ b/crates/store/src/search/leaf.rs @@ -0,0 +1,329 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::search::{ + KeyValueMatch, MAX_PLANNED_TOKENS, MAX_QUERY_LEAVES, MAX_QUERY_PLANNED_TOKENS, SearchField, + SearchFilter, TextMatch, + tokenize::{integer_term, key_value_term, stem_term, tokenize}, +}; +use ahash::AHashSet; +use nlp::language::Language; +use roaring::RoaringBitmap; +use std::{borrow::Cow, cmp::Ordering, mem, ops::Range}; +use utils::cheeky_hash::{CheekyHash, CheekyHashSet}; + +pub(crate) type Probe = (u8, CheekyHash); + +pub(crate) enum Leaf { + Empty, + Groups { + field: u8, + groups: Vec<[Option; 4]>, + }, + Terms { + field: u8, + terms: Vec, + }, + Phrase { + field: u8, + words: Vec, + }, + Prefix { + field: u8, + prefix: Box<[u8]>, + }, + Resolved(RoaringBitmap), + UnsupportedRange, +} + +pub(crate) fn single_term_prefix(value: &str) -> Cow<'_, str> { + if value.chars().any(char::is_uppercase) { + Cow::Owned(value.to_lowercase()) + } else { + Cow::Borrowed(value) + } +} + +impl Leaf { + pub(crate) fn plan(filter: &SearchFilter) -> Self { + match filter { + SearchFilter::Text { + field, + op, + value, + language, + } => { + let field = field.u8_id(); + match op { + TextMatch::Standard => { + let stemmed = !matches!(language, Language::None | Language::Unknown); + let mut groups: Vec<[Option; 4]> = Vec::new(); + let mut distinct = DistinctTerms::default(); + let mut buf = String::new(); + tokenize(value, *language, |token| { + let word = CheekyHash::new(token.word.as_bytes()); + if !distinct.is_new(word) { + return true; + } + let mut group = [Some(word), None, None, None]; + if stemmed { + group[1] = Some(stem_term(&token.word, &mut buf)); + if let Some(stem) = &token.stem { + group[2] = Some(CheekyHash::new(stem.as_bytes())); + group[3] = Some(stem_term(stem, &mut buf)); + } + } + groups.push(group); + distinct.has_room(&mut groups, |group| group[0]) + }); + if groups.is_empty() { + Leaf::Empty + } else { + Leaf::Groups { field, groups } + } + } + TextMatch::Exact => { + let mut words = Vec::new(); + tokenize(value, *language, |token| { + words.push(CheekyHash::new(token.word.as_bytes())); + words.len() < MAX_PLANNED_TOKENS + }); + match words.len() { + 0 => Leaf::Empty, + 1 => Leaf::Terms { + field, + terms: words, + }, + _ => Leaf::Phrase { field, words }, + } + } + TextMatch::Prefix => { + let prefix = single_term_prefix(value); + let prefix = prefix.as_bytes(); + if prefix.is_empty() || prefix.len() > CheekyHash::HASH_SIZE { + Leaf::Empty + } else { + Leaf::Prefix { + field, + prefix: prefix.into(), + } + } + } + } + } + SearchFilter::Integer { field, op, value } => { + if *op == Ordering::Equal { + Leaf::Terms { + field: field.u8_id(), + terms: vec![integer_term(*value)], + } + } else { + Leaf::UnsupportedRange + } + } + SearchFilter::KeyValue { field, key, op } => { + let key = key + .chars() + .filter(|ch| !ch.is_control()) + .map(|ch| ch.to_ascii_lowercase()) + .collect::(); + let value = match op { + KeyValueMatch::Equals(value) | KeyValueMatch::Contains(value) => value.as_str(), + KeyValueMatch::Exists => "", + }; + let mut terms = Vec::new(); + let mut distinct = DistinctTerms::default(); + let mut buf = String::new(); + tokenize(value, Language::None, |token| { + let term = key_value_term(&key, &token.word, &mut buf); + if !distinct.is_new(term) { + return true; + } + terms.push(term); + distinct.has_room(&mut terms, |term| Some(*term)) + }); + if terms.is_empty() { + terms.push(CheekyHash::new(key.as_bytes())); + } + Leaf::Terms { + field: field.u8_id(), + terms, + } + } + SearchFilter::DocumentSet(_) + | SearchFilter::And + | SearchFilter::Or + | SearchFilter::Not + | SearchFilter::End => Leaf::Empty, + } + } + + fn planned_terms(&self) -> usize { + match self { + Leaf::Groups { groups, .. } => groups.len(), + Leaf::Terms { terms, .. } => terms.len(), + Leaf::Phrase { words, .. } => words.len(), + Leaf::Prefix { .. } => 1, + Leaf::Empty | Leaf::Resolved(_) | Leaf::UnsupportedRange => 0, + } + } + + pub(crate) fn add_probes(&self, probes: &mut AHashSet) { + match self { + Leaf::Groups { field, groups } => { + probes.extend( + groups + .iter() + .flatten() + .flatten() + .map(|hash| (*field, *hash)), + ); + } + Leaf::Terms { field, terms } => { + probes.extend(terms.iter().map(|hash| (*field, *hash))); + } + Leaf::Phrase { field, words } => { + probes.extend(words.iter().map(|hash| (*field, *hash))); + } + Leaf::Empty | Leaf::Prefix { .. } | Leaf::Resolved(_) | Leaf::UnsupportedRange => {} + } + } +} + +#[cold] +fn query_too_large() -> trc::Error { + trc::LimitEvent::SizeRequest + .into_err() + .details("Too many search terms in one query") +} + +#[derive(Default)] +struct DistinctTerms(CheekyHashSet); + +impl DistinctTerms { + fn is_new(&mut self, term: CheekyHash) -> bool { + self.0.is_empty() || self.0.insert(term) + } + + fn has_room( + &mut self, + planned: &mut Vec, + planned_term: impl Fn(&T) -> Option, + ) -> bool { + if planned.len() == MAX_PLANNED_TOKENS && self.0.is_empty() { + self.0.reserve(MAX_PLANNED_TOKENS); + planned.retain(|item| planned_term(item).is_some_and(|term| self.0.insert(term))); + } + planned.len() < MAX_PLANNED_TOKENS + } +} + +pub(crate) struct LeafPlan { + leaves: Vec, + units: Vec>, + leaf_units: Vec, + fetched: Vec, +} + +impl LeafPlan { + pub(crate) fn new(filters: &[SearchFilter]) -> trc::Result { + let mut planned = 0usize; + let mut leaves = Vec::new(); + let mut units: Vec> = Vec::new(); + let mut leaf_units = Vec::new(); + let mut depth = 0usize; + let mut open_unit: Option<(usize, usize)> = None; + for filter in filters { + match filter { + SearchFilter::And => depth += 1, + SearchFilter::Or | SearchFilter::Not => { + depth += 1; + if open_unit.is_none() { + units.push(leaves.len()..leaves.len()); + open_unit = Some((depth, units.len() - 1)); + } + } + SearchFilter::End => { + if open_unit.is_some_and(|(unit_depth, _)| unit_depth == depth) { + open_unit = None; + } + depth = depth.saturating_sub(1); + } + SearchFilter::DocumentSet(_) => {} + filter if !is_leaf(filter) => {} + filter => { + if leaves.len() == MAX_QUERY_LEAVES { + return Err(query_too_large()); + } + let leaf = Leaf::plan(filter); + planned += leaf.planned_terms(); + if planned > MAX_QUERY_PLANNED_TOKENS { + return Err(query_too_large()); + } + let unit = match open_unit { + Some((_, unit)) => unit, + None => { + units.push(leaves.len()..leaves.len()); + units.len() - 1 + } + }; + leaves.push(leaf); + if let Some(range) = units.get_mut(unit) { + range.end = leaves.len(); + } + leaf_units.push(unit); + } + } + } + let fetched = vec![false; units.len()]; + Ok(LeafPlan { + leaves, + units, + leaf_units, + fetched, + }) + } + + pub(crate) fn unfetched_unit(&mut self, leaf: usize) -> Option> { + let unit = *self.leaf_units.get(leaf)?; + let fetched = self.fetched.get_mut(unit)?; + if *fetched { + None + } else { + *fetched = true; + self.units.get(unit).cloned() + } + } + + pub(crate) fn leaves(&self, unit: Range) -> &[Leaf] { + self.leaves.get(unit).unwrap_or_default() + } + + pub(crate) fn leaves_mut(&mut self, unit: Range) -> &mut [Leaf] { + self.leaves.get_mut(unit).unwrap_or_default() + } + + pub(crate) fn take(&mut self, leaf: usize) -> Leaf { + self.leaves + .get_mut(leaf) + .map_or(Leaf::Empty, |leaf| mem::replace(leaf, Leaf::Empty)) + } +} + +pub(crate) fn is_leaf(filter: &SearchFilter) -> bool { + matches!( + filter, + SearchFilter::Text { .. } | SearchFilter::Integer { .. } | SearchFilter::KeyValue { .. } + ) && !matches!( + filter, + SearchFilter::Integer { + field: SearchField::AccountId, + op: Ordering::Equal, + .. + } + ) +} + diff --git a/crates/store/src/search/local.rs b/crates/store/src/search/local.rs index ab312a07..56e43c18 100644 --- a/crates/store/src/search/local.rs +++ b/crates/store/src/search/local.rs @@ -50,6 +50,14 @@ impl SearchQuery { self } + pub fn index(&self) -> SearchIndex { + self.index + } + + pub fn filters(&self) -> &[SearchFilter] { + &self.filters + } + pub fn with_comparator(mut self, comparator: SearchComparator) -> Self { self.comparators.push(comparator); self @@ -61,11 +69,8 @@ impl SearchQuery { } pub fn with_account_id(mut self, account_id: u32) -> Self { - self.filters.push(SearchFilter::cond( - SearchField::AccountId, - SearchOperator::Equal, - SearchValue::Uint(account_id as u64), - )); + self.filters + .push(SearchFilter::integer_eq(SearchField::AccountId, account_id)); self } @@ -101,7 +106,9 @@ impl SearchQuery { break; } } - SearchFilter::Operator { .. } => { + SearchFilter::Integer { .. } + | SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } => { continue; } }; @@ -142,12 +149,15 @@ impl SearchQuery { // And short-circuit if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { + let mut depth = 0u32; while let Some(filter) = filters.peek() { - if matches!(filter, SearchFilter::End) { - break; - } else { - filters.next(); + match filter { + SearchFilter::And | SearchFilter::Or | SearchFilter::Not => depth += 1, + SearchFilter::End if depth == 0 => break, + SearchFilter::End => depth -= 1, + _ => {} } + filters.next(); } } } @@ -210,7 +220,6 @@ impl QueryResults { *ascending, ) } - SearchComparator::Field { .. } => continue, }; let ordering = if is_ascending { a.cmp(&b) } else { b.cmp(&a) }; diff --git a/crates/store/src/search/mod.rs b/crates/store/src/search/mod.rs index f23fe6cc..74b3d20b 100644 --- a/crates/store/src/search/mod.rs +++ b/crates/store/src/search/mod.rs @@ -4,35 +4,35 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod bm_u32; -pub mod bm_u64; +pub(crate) mod codec; pub mod document; pub mod fields; pub mod index; +pub(crate) mod leaf; pub mod local; pub mod query; pub mod split; -pub mod term; +pub(crate) mod term; +pub mod tokenize; use crate::write::SearchIndex; use ahash::AHashMap; use nlp::language::Language; -use roaring::RoaringBitmap; +use roaring::{RoaringBitmap, RoaringTreemap}; use std::cmp::Ordering; use std::collections::hash_map::Entry; -use std::fmt::Display; use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; use utils::map::vec_map::VecMap; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SearchOperator { - LowerThan, - LowerEqualThan, - GreaterThan, - GreaterEqualThan, - Equal, - Contains, -} +pub const MAX_PLANNED_TOKENS: usize = 64; +pub const MAX_QUERY_PLANNED_TOKENS: usize = 4_096; +pub const MAX_QUERY_LEAVES: usize = 1_024; + +pub(crate) const GLOBAL_BUCKET_BITS: u32 = 16; +pub(crate) const GLOBAL_BUCKET_SHIFT: u32 = u64::BITS - GLOBAL_BUCKET_BITS; +pub(crate) const ACCOUNT_BLOCK_SHIFT: u32 = 16; +pub(crate) const MAX_DOCUMENT_SIZE: usize = 1024 * 1024; +pub(crate) const MAX_STORED_DOCUMENT_SIZE: usize = MAX_DOCUMENT_SIZE * 8; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum SearchField { @@ -55,10 +55,6 @@ pub enum EmailSearchField { Subject, Body, Attachment, - ReceivedAt, - SentAt, - Size, - HasAttachment, Headers, } @@ -69,8 +65,6 @@ pub enum CalendarSearchField { Location, Owner, Attendee, - Start, - Uid, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -85,12 +79,10 @@ pub enum ContactSearchField { OnlineService, Address, Note, - Uid, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum FileSearchField { - Name, Content, } @@ -110,8 +102,8 @@ pub enum SearchValue { Boolean(bool), } -pub trait SearchDocumentId: Sized + Copy + Display { - fn from_u64(id: u64) -> Self; +pub trait SearchResults: Sized + Default { + fn insert(&mut self, id: u64); fn field() -> SearchField; } @@ -125,10 +117,21 @@ pub struct SearchQuery { #[derive(Debug, PartialEq, Clone, Default)] pub enum SearchFilter { - Operator { + Text { field: SearchField, - op: SearchOperator, - value: SearchValue, + op: TextMatch, + value: String, + language: Language, + }, + Integer { + field: SearchField, + op: Ordering, + value: u64, + }, + KeyValue { + field: SearchField, + key: String, + op: KeyValueMatch, }, DocumentSet(RoaringBitmap), And, @@ -138,12 +141,22 @@ pub enum SearchFilter { End, } +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum TextMatch { + Exact, + Standard, + Prefix, +} + +#[derive(Debug, PartialEq, Clone)] +pub enum KeyValueMatch { + Equals(String), + Contains(String), + Exists, +} + #[derive(Debug)] pub enum SearchComparator { - Field { - field: SearchField, - ascending: bool, - }, DocumentSet { set: RoaringBitmap, ascending: bool, @@ -241,24 +254,24 @@ impl From for SearchValue { } } -impl SearchDocumentId for u32 { - fn from_u64(id: u64) -> Self { - id as u32 - } - +impl SearchResults for RoaringBitmap { fn field() -> SearchField { SearchField::DocumentId } + + fn insert(&mut self, id: u64) { + self.insert(id as u32); + } } -impl SearchDocumentId for u64 { - fn from_u64(id: u64) -> Self { - id - } - +impl SearchResults for RoaringTreemap { fn field() -> SearchField { SearchField::Id } + + fn insert(&mut self, id: u64) { + self.insert(id); + } } pub trait SearchableField: Sized { @@ -271,7 +284,32 @@ pub trait SearchableField: Sized { impl Eq for SearchFilter {} +pub(crate) const TERM_GROUP_TEXT: u8 = 0; +pub(crate) const TERM_GROUP_KEY_VALUE: u8 = 1; + impl SearchIndex { + pub(crate) const fn term_group(&self, field: u8) -> u8 { + match self { + SearchIndex::Email + if field == SearchField::Email(EmailSearchField::Headers).u8_id() => + { + TERM_GROUP_KEY_VALUE + } + _ => TERM_GROUP_TEXT, + } + } + + pub(crate) fn group_fields(&self, group: u8) -> u32 { + if matches!(self, SearchIndex::InMemory) { + return 0; + } + self.all_fields() + .iter() + .map(SearchField::u8_id) + .filter(|field| self.term_group(*field) == group) + .fold(0, |fields, field| fields | (1 << field)) + } + pub fn index_name(&self) -> &'static str { match self { SearchIndex::Email => "st_email", @@ -284,6 +322,53 @@ impl SearchIndex { } } +impl SearchField { + pub(crate) const fn u8_id(&self) -> u8 { + match self { + SearchField::AccountId => 0, + SearchField::DocumentId => 1, + SearchField::Id => 2, + SearchField::Email(field) => match field { + EmailSearchField::From => 3, + EmailSearchField::To => 4, + EmailSearchField::Cc => 5, + EmailSearchField::Bcc => 6, + EmailSearchField::Subject => 7, + EmailSearchField::Body => 8, + EmailSearchField::Attachment => 9, + EmailSearchField::Headers => 10, + }, + SearchField::Calendar(field) => match field { + CalendarSearchField::Title => 3, + CalendarSearchField::Description => 4, + CalendarSearchField::Location => 5, + CalendarSearchField::Owner => 6, + CalendarSearchField::Attendee => 7, + }, + SearchField::Contact(field) => match field { + ContactSearchField::Member => 3, + ContactSearchField::Kind => 4, + ContactSearchField::Name => 5, + ContactSearchField::Nickname => 6, + ContactSearchField::Organization => 7, + ContactSearchField::Email => 8, + ContactSearchField::Phone => 9, + ContactSearchField::OnlineService => 10, + ContactSearchField::Address => 11, + ContactSearchField::Note => 12, + }, + SearchField::File(field) => match field { + FileSearchField::Content => 4, + }, + SearchField::Tracing(field) => match field { + TracingSearchField::EventType => 3, + TracingSearchField::QueueId => 4, + TracingSearchField::Keywords => 5, + }, + } + } +} + impl SearchField { pub fn field_name(&self) -> &'static str { match self { @@ -298,10 +383,6 @@ impl SearchField { EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "attach", - EmailSearchField::ReceivedAt => "rcvd", - EmailSearchField::SentAt => "sent", - EmailSearchField::Size => "size", - EmailSearchField::HasAttachment => "has_att", EmailSearchField::Headers => "headers", }, SearchField::Calendar(field) => match field { @@ -310,8 +391,6 @@ impl SearchField { CalendarSearchField::Location => "loc", CalendarSearchField::Owner => "owner", CalendarSearchField::Attendee => "attendee", - CalendarSearchField::Start => "start", - CalendarSearchField::Uid => "uid", }, SearchField::Contact(field) => match field { ContactSearchField::Member => "member", @@ -324,10 +403,8 @@ impl SearchField { ContactSearchField::OnlineService => "online", ContactSearchField::Address => "addr", ContactSearchField::Note => "note", - ContactSearchField::Uid => "uid", }, SearchField::File(field) => match field { - FileSearchField::Name => "name", FileSearchField::Content => "content", }, SearchField::Tracing(field) => match field { diff --git a/crates/store/src/search/query.rs b/crates/store/src/search/query.rs index 171ca4a6..7ffaab06 100644 --- a/crates/store/src/search/query.rs +++ b/crates/store/src/search/query.rs @@ -5,419 +5,1105 @@ */ use crate::{ - Store, - backend::MAX_TOKEN_LENGTH, + IterateParams, Store, U16_LEN, U32_LEN, search::{ - QueryResults, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, - SearchValue, - bm_u32::{BitmapCache, range_to_bitmap, sort_order}, - bm_u64::{TreemapCache, range_to_treemap}, + ACCOUNT_BLOCK_SHIFT, GLOBAL_BUCKET_SHIFT, KeyValueMatch, SearchField, SearchFilter, + SearchQuery, TERM_GROUP_KEY_VALUE, TERM_GROUP_TEXT, TextMatch, codec, + leaf::{Leaf, LeafPlan, Probe, single_term_prefix}, + term::matches_phrase, + tokenize::{integer_term, key_value_term, tokenize}, + }, + write::{ + AnyKey, SearchIndex, + lazybitmap::{LazyBitmap, LazyTreemap}, }, - write::SEARCH_INDEX_MAX_FIELD_LEN, }; -use nlp::{language::stemmer::Stemmer, tokenizers::space::SpaceTokenizer}; +use ahash::{AHashMap, AHashSet}; +use compact_str::format_compact; +use nlp::language::Language; use roaring::{RoaringBitmap, RoaringTreemap}; -use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; +use std::{cmp::Ordering, iter, mem}; use utils::cheeky_hash::CheekyHash; -impl Store { - pub(crate) async fn query_account(&self, query: SearchQuery) -> trc::Result> { - struct State { - pub op: SearchFilter, - pub bm: Option, - } - let mut state: State = State { - op: SearchFilter::And, - bm: None, - }; - let mut stack = Vec::new(); - let mask = query.mask; - let mut bitmaps = BitmapCache::default(); - let mut account_id = u32::MAX; +const PHRASE_DOCUMENT_BATCH: usize = 1024; +const MAX_WINDOW_RUNS: usize = 8; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Group { + And, + Or, + Not, +} + +impl Store { + pub async fn query_account(&self, query: SearchQuery) -> trc::Result { + self.query_account_in(query, None).await + } + + pub async fn query_account_window( + &self, + mut query: SearchQuery, + window: RoaringBitmap, + ) -> trc::Result { + let mask_blocks = blocks(&query.mask).count(); + query.mask &= &window; + if query.mask.is_empty() { + return Ok(RoaringBitmap::new()); + } + let window = BlockWindow::new(&query.mask, mask_blocks); + self.query_account_in(query, window).await + } + + async fn query_account_in( + &self, + query: SearchQuery, + window: Option, + ) -> trc::Result { + let mut account_id = None; for filter in &query.filters { - if let SearchFilter::Operator { + if let SearchFilter::Integer { field: SearchField::AccountId, - value: SearchValue::Uint(id), - .. + op: Ordering::Equal, + value, } = filter { - account_id = *id as u32; - break; + account_id = Some(*value as u32); + } + } + let Some(account_id) = account_id else { + return Err(trc::StoreEvent::UnexpectedError + .into_err() + .details("Account ID filter is required for account queries")); + }; + + let mut filters = query.filters; + let mut plan = LeafPlan::new(&filters)?; + let mut ctx = AccountContext { + store: self, + index: query.index, + account_id, + mask: &query.mask, + window, + terms: AHashMap::new(), + prefixes: AHashMap::new(), + }; + + let mut next_leaf = 0usize; + let mut groups: Vec = Vec::new(); + let mut skip_depth: Option = None; + let mut has_filters = false; + + for filter in filters.iter_mut() { + match filter { + SearchFilter::And => groups.push(Group::And), + SearchFilter::Or => groups.push(Group::Or), + SearchFilter::Not => groups.push(Group::Not), + SearchFilter::End => { + groups.pop(); + if skip_depth.is_some_and(|depth| groups.len() < depth) { + skip_depth = None; + } + } + SearchFilter::Integer { + field: SearchField::AccountId, + op: Ordering::Equal, + .. + } => {} + SearchFilter::DocumentSet(set) => { + has_filters = true; + if skip_depth.is_none() + && set.is_empty() + && !matches!(groups.last(), Some(Group::Or | Group::Not)) + { + skip_depth = Some(groups.len()); + } + } + SearchFilter::Text { .. } + | SearchFilter::Integer { .. } + | SearchFilter::KeyValue { .. } => { + has_filters = true; + let leaf = next_leaf; + next_leaf += 1; + let result = if skip_depth.is_none() { + if let Some(unit) = plan.unfetched_unit(leaf) { + ctx.prefetch(plan.leaves(unit.clone())).await?; + ctx.verify_phrases(plan.leaves_mut(unit)).await?; + } + ctx.evaluate(plan.take(leaf))? + } else { + RoaringBitmap::new() + }; + if skip_depth.is_none() + && result.is_empty() + && !matches!(groups.last(), Some(Group::Or | Group::Not)) + { + skip_depth = Some(groups.len()); + } + *filter = SearchFilter::DocumentSet(result); + } } } - if account_id == u32::MAX { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Account ID must be specified before other filters")); + if has_filters { + Ok(SearchQuery::new(query.index) + .with_filters(filters) + .with_mask(query.mask) + .filter() + .into_bitmap()) + } else { + Ok(query.mask) } + } - let mut results; - if query.filters.len() > 1 { - let mut filters = query.filters.into_iter().peekable(); - while let Some(filter) = filters.next() { - let mut result = match filter { - SearchFilter::Operator { field, op, value } => { - if matches!(field, SearchField::AccountId) { - continue; - } + pub async fn query_global(&self, query: SearchQuery) -> trc::Result { + let mut from_id = 0u64; + let mut until_id = u64::MAX; + let mut has_filters = false; + let mut groups: Vec = Vec::new(); - if field.is_text() - && matches!(op, SearchOperator::Contains | SearchOperator::Equal) - { - let (value, language) = match value { - SearchValue::Text { value, language } => (value, language), - _ => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Expected text value for text field")); - } - }; - - if op == SearchOperator::Equal { - bitmaps - .merge_bitmaps( - self, - query.index, - account_id, - language - .tokenize_text(&value, MAX_TOKEN_LENGTH) - .map(|token| CheekyHash::new(token.word.as_bytes())), - field.u8_id(), - false, - ) - .await? - } else { - let mut result = RoaringBitmap::new(); - for token in Stemmer::new(&value, language, MAX_TOKEN_LENGTH) { - let mut tokens = Vec::with_capacity(3); - tokens.push(CheekyHash::new(token.word.as_bytes())); - tokens.push(CheekyHash::new( - format!("{}*", token.word).as_bytes(), - )); - if let Some(stemmed_word) = token.stemmed_word { - tokens.push(CheekyHash::new( - format!("{stemmed_word}*").as_bytes(), - )); - } - let union = bitmaps - .merge_bitmaps( - self, - query.index, - account_id, - tokens.into_iter(), - field.u8_id(), - true, - ) - .await?; - if let Some(union) = union { - if result.is_empty() { - result = union; - } else { - result.bitand_assign(&union); - if result.is_empty() { - break; - } - } - } else { - result.clear(); - break; - } - } - if !result.is_empty() { - Some(result) - } else { - None - } + for filter in &query.filters { + match filter { + SearchFilter::And => groups.push(Group::And), + SearchFilter::Or => groups.push(Group::Or), + SearchFilter::End => { + groups.pop(); + } + SearchFilter::Integer { + field: SearchField::Id, + op, + value, + } => { + if groups.contains(&Group::Or) { + return Err(trc::StoreEvent::NotSupported + .into_err() + .details("Id filters are only supported as conjunctions")); + } + match op { + Ordering::Greater => match value.checked_add(1) { + Some(value) => from_id = from_id.max(value), + None => { + from_id = 1; + until_id = 0; } - } else if field.is_json() { - let (key, value) = match value { - SearchValue::KeyValues(kv) => kv.into_iter().next().unwrap(), - _ => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Expected text value for text field")); - } - }; - - if !value.is_empty() { - bitmaps - .merge_bitmaps( - self, - query.index, - account_id, - SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH).map( - |value| { - CheekyHash::new(format!("{key} {value}").as_bytes()) - }, - ), - field.u8_id(), - true, - ) - .await? - } else { - bitmaps - .merge_bitmaps( - self, - query.index, - account_id, - [CheekyHash::new(key.as_bytes())].into_iter(), - field.u8_id(), - false, - ) - .await? + }, + Ordering::Less => match value.checked_sub(1) { + Some(value) => until_id = until_id.min(value), + None => { + from_id = 1; + until_id = 0; } - } else if field.is_indexed() { - let value = match value { - SearchValue::Text { value, .. } => { - let mut value = value.into_bytes(); - value.truncate(SEARCH_INDEX_MAX_FIELD_LEN); - value - } - SearchValue::Int(v) => (v as u64).to_be_bytes().to_vec(), - SearchValue::Uint(v) => v.to_be_bytes().to_vec(), - SearchValue::Boolean(v) => vec![v as u8], - SearchValue::KeyValues(_) => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Expected non key-value for non-text field")); - } - }; - - range_to_bitmap( - self, - query.index, - account_id, - field.u8_id(), - &value, - op, - ) - .await? - } else { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details(format!("Field {field:?} is not indexed"))); + }, + Ordering::Equal => { + from_id = from_id.max(*value); + until_id = until_id.min(*value); } } - SearchFilter::DocumentSet(bitmap) => Some(bitmap), - op @ (SearchFilter::And | SearchFilter::Or | SearchFilter::Not) => { + } + SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } + | SearchFilter::Integer { .. } => { + has_filters = true; + } + filter @ (SearchFilter::Not | SearchFilter::DocumentSet(_)) => { + return Err(trc::StoreEvent::NotSupported + .into_err() + .details(format_compact!( + "Unsupported global query filter {filter:?}" + ))); + } + } + } + + if from_id > until_id { + return Ok(RoaringTreemap::new()); + } + + let mut ctx = GlobalContext { + store: self, + index: query.index, + from_block: (from_id >> GLOBAL_BUCKET_SHIFT) as u16, + until_block: (until_id >> GLOBAL_BUCKET_SHIFT) as u16, + terms: AHashMap::new(), + prefixes: AHashMap::new(), + }; + + let mut results = if has_filters { + struct State { + op: Group, + acc: Option, + } + let mut state = State { + op: Group::And, + acc: None, + }; + let mut stack = Vec::new(); + let mut skip_depth: Option = None; + + for filter in query.filters { + let result = match filter { + SearchFilter::And => { stack.push(state); - state = State { op, bm: None }; + state = State { + op: Group::And, + acc: None, + }; + continue; + } + SearchFilter::Or => { + stack.push(state); + state = State { + op: Group::Or, + acc: None, + }; continue; } SearchFilter::End => { if let Some(prev_state) = stack.pop() { - let bm = state.bm; + let result = state.acc.unwrap_or_default(); state = prev_state; - bm + if skip_depth.is_some_and(|depth| stack.len() < depth) { + skip_depth = None; + } + result } else { break; } } - }; - - // Apply logical operation - if let Some(dest) = &mut state.bm { - match state.op { - SearchFilter::And => { - if let Some(result) = result { - dest.bitand_assign(result); - } else { - dest.clear(); - } - } - SearchFilter::Or => { - if let Some(result) = result { - dest.bitor_assign(result); - } - } - SearchFilter::Not => { - if let Some(mut result) = result { - result.bitxor_assign(&mask); - dest.bitand_assign(result); - } - } - _ => unreachable!(), - } - } else if let Some(result_) = &mut result { - if let SearchFilter::Not = state.op { - result_.bitxor_assign(&mask); - } - state.bm = result; - } else if let SearchFilter::Not = state.op { - state.bm = Some(mask.clone()); - } else { - state.bm = Some(RoaringBitmap::new()); - } - - // And short circuit - if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { - while let Some(filter) = filters.peek() { - if matches!(filter, SearchFilter::End) { - break; + SearchFilter::Integer { + field: SearchField::Id, + .. + } => continue, + filter @ (SearchFilter::Text { .. } + | SearchFilter::Integer { .. } + | SearchFilter::KeyValue { .. }) => { + if skip_depth.is_none() { + ctx.resolve(filter).await? } else { - filters.next(); + RoaringTreemap::new() } } - } - } - - results = state.bm.unwrap_or_default(); - results.bitand_assign(&mask); - } else { - results = mask; - } - - if results.len() > 1 && !query.comparators.is_empty() { - let mut comparators = Vec::with_capacity(query.comparators.len()); - for comparator in query.comparators { - let comparator = match comparator { - SearchComparator::Field { field, ascending } => SearchComparator::SortedSet { - set: sort_order(self, query.index, account_id, field.u8_id()).await?, - ascending, - }, - _ => comparator, + SearchFilter::Not | SearchFilter::DocumentSet(_) => unreachable!(), }; - comparators.push(comparator); - } - - Ok(QueryResults::new(results, comparators).into_sorted()) - } else { - Ok(results.into_iter().collect::>()) - } - } - - pub(crate) async fn query_global(&self, query: SearchQuery) -> trc::Result> { - struct State { - pub op: SearchFilter, - pub bm: Option, - } - let mut state: State = State { - op: SearchFilter::And, - bm: None, - }; - let mut stack = Vec::new(); - let mut filters = query.filters.into_iter().peekable(); - let mut bitmaps = TreemapCache::default(); - - while let Some(filter) = filters.next() { - let result = match filter { - SearchFilter::Operator { field, op, value } => { - if field.is_text() { - let value = match value { - SearchValue::Text { value, .. } => value, - _ => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Expected text value for text field")); - } - }; - - bitmaps - .merge_treemaps( - self, - query.index, - SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH) - .map(|word| CheekyHash::new(word.as_bytes())), - field.u8_id(), - false, - ) - .await? - } else if field.is_indexed() || matches!(field, SearchField::Id) { - let value = match value { - SearchValue::Text { value, .. } => value.into_bytes(), - SearchValue::Int(v) => (v as u64).to_be_bytes().to_vec(), - SearchValue::Uint(v) => v.to_be_bytes().to_vec(), - SearchValue::Boolean(v) => vec![v as u8], - SearchValue::KeyValues(_) => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Expected non key-value for non-text field")); - } - }; - - range_to_treemap(self, query.index, field.u8_id(), &value, op).await? - } else { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details(format!("Field {field:?} is not indexed"))); - } - } - SearchFilter::DocumentSet(_) | SearchFilter::Not => { - return Err(trc::StoreEvent::UnexpectedError - .into_err() - .details("Unsupported filter in global search")); - } - op @ (SearchFilter::And | SearchFilter::Or) => { - stack.push(state); - state = State { op, bm: None }; - continue; - } - SearchFilter::End => { - if let Some(prev_state) = stack.pop() { - let bm = state.bm; - state = prev_state; - bm - } else { - break; - } - } - }; - - // Apply logical operation - if let Some(dest) = &mut state.bm { match state.op { - SearchFilter::And => { - if let Some(result) = result { - dest.bitand_assign(result); + Group::And => { + if let Some(acc) = &mut state.acc { + *acc &= result; } else { - dest.clear(); + state.acc = Some(result); + } + if skip_depth.is_none() && state.acc.as_ref().unwrap().is_empty() { + skip_depth = Some(stack.len()); } } - SearchFilter::Or => { - if let Some(result) = result { - dest.bitor_assign(result); + Group::Or => { + if let Some(acc) = &mut state.acc { + *acc |= result; + } else { + state.acc = Some(result); } } - _ => unreachable!(), + Group::Not => unreachable!(), } - } else if result.is_some() { - state.bm = result; - } else { - state.bm = Some(RoaringTreemap::new()); } - // And short circuit - if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { - while let Some(filter) = filters.peek() { - if matches!(filter, SearchFilter::End) { - break; - } else { - filters.next(); + while let Some(mut prev_state) = stack.pop() { + let result = state.acc.unwrap_or_default(); + match prev_state.op { + Group::And => { + if let Some(acc) = &mut prev_state.acc { + *acc &= result; + } else { + prev_state.acc = Some(result); + } } + Group::Or => { + if let Some(acc) = &mut prev_state.acc { + *acc |= result; + } else { + prev_state.acc = Some(result); + } + } + Group::Not => unreachable!(), } + state = prev_state; } - } - if query.comparators.iter().all(|c| { - matches!( - c, - SearchComparator::Field { - field: SearchField::Id, - ascending: false - } - ) - }) { - Ok(state - .bm - .unwrap_or_default() - .into_iter() - .rev() - .collect::>()) + state.acc.unwrap_or_default() } else { - Ok(state.bm.unwrap_or_default().into_iter().collect::>()) + ctx.document_ids().await? + }; + + if from_id > 0 { + results.remove_range(..from_id); } + if until_id < u64::MAX { + results.remove_range(until_id + 1..); + } + + Ok(results) } } + +struct AccountContext<'x> { + store: &'x Store, + index: SearchIndex, + account_id: u32, + mask: &'x RoaringBitmap, + window: Option, + terms: AHashMap, + prefixes: AHashMap, +} + +struct BlockWindow { + runs: Vec<(u16, u16)>, +} + +impl BlockWindow { + fn new(masked: &RoaringBitmap, mask_blocks: usize) -> Option { + let mut runs: Vec<(u16, u16)> = Vec::new(); + let mut window_blocks = 0; + for block in blocks(masked) { + window_blocks += 1; + if let Some((_, last)) = runs.last_mut() + && last.checked_add(1) == Some(block) + { + *last = block; + } else if runs.len() == MAX_WINDOW_RUNS { + return None; + } else { + runs.push((block, block)); + } + } + (window_blocks < mask_blocks).then_some(BlockWindow { runs }) + } +} + +fn blocks(documents: &RoaringBitmap) -> impl Iterator + '_ { + let mut documents = documents.iter(); + let mut exhausted = false; + iter::from_fn(move || { + if exhausted { + return None; + } + let block = documents.next()? >> ACCOUNT_BLOCK_SHIFT; + match (block + 1).checked_mul(1 << ACCOUNT_BLOCK_SHIFT) { + Some(next_block) => documents.advance_to(next_block), + None => exhausted = true, + } + Some(block as u16) + }) +} + +struct PhraseCheck { + leaf: usize, + field: u8, + words: Vec, + candidates: RoaringBitmap, + matched: RoaringBitmap, +} + +fn probe_order(index: SearchIndex, probe: &Probe) -> (u8, &[u8], usize, u8) { + ( + index.term_group(probe.0), + probe.1.as_key(), + probe.1.len(), + probe.0, + ) +} + +fn is_stem_term_of(stem: &CheekyHash, word: &CheekyHash) -> bool { + stem.len() == word.len() + 1 + && stem.len() <= CheekyHash::HASH_SIZE + && stem + .as_key() + .strip_suffix(b"*") + .is_some_and(|stem| stem == word.as_key()) +} + +fn into_disjoint(ranges: &mut Vec>>>) { + ranges.sort_unstable_by(|a, b| a.begin.key.cmp(&b.begin.key)); + ranges.dedup_by(|next, kept| { + if next.begin.key <= kept.end.key { + if next.end.key > kept.end.key { + mem::swap(&mut kept.end, &mut next.end); + } + true + } else { + false + } + }); +} + +struct TermRanges { + index: SearchIndex, + account_id: u32, +} + +type TermRange = IterateParams>>; + +impl TermRanges { + fn same_term(&self, a: &Probe, b: &Probe) -> bool { + a.1 == b.1 && self.index.term_group(a.0) == self.index.term_group(b.0) + } + + fn windowed(&self, wanted: &[Probe], window: &BlockWindow, ranges: &mut Vec) { + for (field, term) in wanted { + for (from_block, until_block) in &window.runs { + let (begin, end) = if from_block == until_block { + let key = codec::account_term_block_key( + self.index, + self.account_id, + *field, + term, + *from_block, + ); + (key.clone(), key) + } else { + codec::account_term_block_range( + self.index, + self.account_id, + *field, + term, + *from_block, + *until_block, + ) + }; + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + } + } + } + + fn full(&self, leaves: &[Leaf], wanted: &[Probe], ranges: &mut Vec) { + let wide = self.wide_terms(leaves, wanted); + let mut all_fields_term: Option<(u8, CheekyHash)> = None; + for run in wanted.chunk_by(|a, b| self.same_term(a, b)) { + let Some((field, term)) = run.first().copied() else { + continue; + }; + let group = self.index.term_group(field); + if !wide.contains(&(group, term)) { + all_fields_term = None; + for (field, term) in run { + let (begin, end) = + codec::account_term_range(self.index, self.account_id, *field, term); + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + } + continue; + } + let first = match all_fields_term { + Some((previous_group, previous)) + if previous_group == group && is_stem_term_of(&term, &previous) => + { + ranges.pop(); + previous + } + _ => term, + }; + let (begin, end) = + codec::account_terms_range(self.index, self.account_id, group, &first, &term); + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + all_fields_term = Some((group, term)); + } + } + + fn wide_terms(&self, leaves: &[Leaf], wanted: &[Probe]) -> AHashSet<(u8, CheekyHash)> { + let text_fields = self.index.group_fields(TERM_GROUP_TEXT); + let key_value_fields = self.index.group_fields(TERM_GROUP_KEY_VALUE); + let mut wide = AHashSet::new(); + for run in wanted.chunk_by(|a, b| self.same_term(a, b)) { + let Some((field, term)) = run.first() else { + continue; + }; + let group = self.index.term_group(*field); + let group_fields = match group { + TERM_GROUP_TEXT => text_fields, + _ => key_value_fields, + }; + let fields = run + .iter() + .fold(0u32, |fields, (field, _)| fields | (1 << field)); + if fields == group_fields { + wide.insert((group, *term)); + } + } + if !wide.is_empty() { + for leaf in leaves { + if let Leaf::Groups { field, groups } = leaf { + let group = self.index.term_group(*field); + for [word, variants @ ..] in groups { + if word.is_some_and(|word| wide.contains(&(group, word))) { + wide.extend(variants.iter().flatten().map(|variant| (group, *variant))); + } + } + } + } + } + wide + } +} + +impl AccountContext<'_> { + async fn prefetch(&mut self, leaves: &[Leaf]) -> trc::Result<()> { + let mut probes = AHashSet::new(); + let mut prefixes = Vec::new(); + for leaf in leaves { + leaf.add_probes(&mut probes); + if let Leaf::Prefix { field, prefix } = leaf + && !prefixes.iter().any(|(other_field, other): &(u8, &[u8])| { + other_field == field && *other == &prefix[..] + }) + { + prefixes.push((*field, &prefix[..])); + } + } + self.fetch(leaves, probes, &prefixes).await + } + + fn evaluate(&self, leaf: Leaf) -> trc::Result { + match leaf { + Leaf::Empty => Ok(RoaringBitmap::new()), + Leaf::Groups { field, groups } => { + let mut result: Option = None; + for group in &groups { + let mut matches = RoaringBitmap::new(); + for hash in group.iter().flatten() { + if let Some(bitmap) = self.terms.get(&(field, *hash)) { + matches |= bitmap; + } + } + let result = match &mut result { + Some(result) => { + *result &= matches; + result + } + None => result.insert(matches), + }; + if result.is_empty() { + return Ok(RoaringBitmap::new()); + } + } + Ok(result.unwrap_or_default()) + } + Leaf::Terms { field, terms } => Ok(self.intersect(field, &terms)), + Leaf::Prefix { field, prefix } => Ok(self + .prefixes + .get(&(field, CheekyHash::new(&prefix))) + .cloned() + .unwrap_or_default()), + Leaf::Resolved(bitmap) => Ok(bitmap), + Leaf::Phrase { .. } => Err(trc::StoreEvent::UnexpectedError + .into_err() + .details("Unverified phrase")), + Leaf::UnsupportedRange => Err(trc::StoreEvent::NotSupported + .into_err() + .details("Integer range filters are not supported")), + } + } + + fn intersect(&self, field: u8, terms: &[CheekyHash]) -> RoaringBitmap { + let Some((first, rest)) = terms.split_first() else { + return RoaringBitmap::new(); + }; + let Some(first) = self + .terms + .get(&(field, *first)) + .filter(|first| !first.is_empty()) + else { + return RoaringBitmap::new(); + }; + let mut result: Option = None; + for term in rest { + let Some(matches) = self.terms.get(&(field, *term)) else { + return RoaringBitmap::new(); + }; + let result = match &mut result { + Some(result) => { + *result &= matches; + result + } + None => result.insert(first & matches), + }; + if result.is_empty() { + return RoaringBitmap::new(); + } + } + result.unwrap_or_else(|| first.clone()) + } + + async fn verify_phrases(&self, leaves: &mut [Leaf]) -> trc::Result<()> { + let mut checks = Vec::new(); + let mut documents = RoaringBitmap::new(); + for (index, leaf) in leaves.iter_mut().enumerate() { + if let Leaf::Phrase { field, words } = leaf { + let mut candidates = self.intersect(*field, words); + candidates &= self.mask; + if candidates.is_empty() { + *leaf = Leaf::Resolved(candidates); + } else { + documents |= &candidates; + checks.push(PhraseCheck { + leaf: index, + field: *field, + words: mem::take(words), + candidates, + matched: RoaringBitmap::new(), + }); + } + } + } + if checks.is_empty() { + return Ok(()); + } + + let mut ordinals = Vec::new(); + let mut failure = Vec::new(); + let mut scratch = Vec::new(); + let mut documents = documents.iter().peekable(); + while documents.peek().is_some() { + let ranges = documents + .by_ref() + .take(PHRASE_DOCUMENT_BATCH) + .map(|document_id| { + let key = codec::account_document_key(self.index, self.account_id, document_id); + IterateParams::new(codec::document_key(key.clone()), codec::document_key(key)) + }) + .collect::>(); + let mut corrupted = None; + + self.store + .iterate_many(ranges, |key, value| { + let verified = key + .get(key.len().wrapping_sub(U32_LEN)..) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_be_bytes) + .zip(codec::read_document(value, &mut scratch)) + .and_then(|(document_id, document)| { + for check in checks + .iter_mut() + .filter(|check| check.candidates.contains(document_id)) + { + if matches_phrase( + document, + check.field, + &check.words, + &mut ordinals, + &mut failure, + )? { + check.matched.insert(document_id); + } + } + Some(()) + }); + if verified.is_some() { + Ok(true) + } else { + corrupted = Some(trc::Error::corrupted_key( + key, + Some(value), + trc::location!(), + )); + Ok(false) + } + }) + .await?; + + if let Some(err) = corrupted { + return Err(err); + } + } + + for check in checks { + if let Some(leaf) = leaves.get_mut(check.leaf) { + *leaf = Leaf::Resolved(check.matched); + } + } + Ok(()) + } + + async fn fetch( + &mut self, + leaves: &[Leaf], + mut probes: AHashSet, + prefixes: &[(u8, &[u8])], + ) -> trc::Result<()> { + let index = self.index; + probes.retain(|probe| !self.terms.contains_key(probe)); + let mut wanted = Vec::with_capacity(probes.len()); + for probe in &probes { + self.terms.insert(*probe, RoaringBitmap::new()); + wanted.push(*probe); + } + wanted.sort_unstable_by(|a, b| probe_order(index, a).cmp(&probe_order(index, b))); + + let mut ranges = Vec::with_capacity(wanted.len() + prefixes.len()); + let planner = TermRanges { + index, + account_id: self.account_id, + }; + match &self.window { + Some(window) => planner.windowed(&wanted, window, &mut ranges), + None => planner.full(leaves, &wanted, &mut ranges), + } + let mut fetch_prefixes = Vec::new(); + for (field, prefix) in prefixes { + let cache_key = (*field, CheekyHash::new(prefix)); + if !self.prefixes.contains_key(&cache_key) { + let (begin, end) = codec::account_term_prefix_range( + index, + self.account_id, + index.term_group(*field), + prefix, + ); + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + fetch_prefixes.push((*field, *prefix, cache_key, RoaringBitmap::new())); + } + } + if ranges.is_empty() { + return Ok(()); + } + into_disjoint(&mut ranges); + + let terms = &mut self.terms; + let mut corrupted = None; + + self.store + .iterate_many(ranges, |key, value| { + let Some((hash, parsed)) = codec::parse_account_term_key(key).and_then(|parsed| { + CheekyHash::from_key_bytes(parsed.term, parsed.len).map(|hash| (hash, parsed)) + }) else { + corrupted = Some(trc::Error::corrupted_key( + key, + Some(value), + trc::location!(), + )); + return Ok(false); + }; + let field = parsed.field; + + let probe = (field, hash); + let is_probe = probes.contains(&probe); + let is_prefix_term = parsed.len as usize <= CheekyHash::HASH_SIZE + && parsed.term.last() != Some(&b'*'); + let mut prefix_matches = fetch_prefixes + .iter_mut() + .filter(|(prefix_field, prefix, _, _)| { + is_prefix_term && *prefix_field == field && parsed.term.starts_with(prefix) + }) + .peekable(); + + if is_probe || prefix_matches.peek().is_some() { + match LazyBitmap::deserialize_delta( + value, + (parsed.block_id as u32) << ACCOUNT_BLOCK_SHIFT, + ) { + Ok(bitmap) => { + for (_, _, _, matches) in prefix_matches { + *matches |= &bitmap.0; + } + if is_probe && let Some(term) = terms.get_mut(&probe) { + *term |= bitmap.0; + } + } + Err(err) => { + corrupted = Some(err); + return Ok(false); + } + } + } + + Ok(true) + }) + .await?; + + if let Some(err) = corrupted { + return Err(err); + } + + for (_, _, cache_key, matches) in fetch_prefixes { + self.prefixes.insert(cache_key, matches); + } + + Ok(()) + } +} + +struct GlobalContext<'x> { + store: &'x Store, + index: SearchIndex, + from_block: u16, + until_block: u16, + terms: AHashMap<(u8, CheekyHash), RoaringTreemap>, + prefixes: AHashMap<(u8, CheekyHash), RoaringTreemap>, +} + +impl GlobalContext<'_> { + async fn resolve(&mut self, filter: SearchFilter) -> trc::Result { + match filter { + SearchFilter::Text { + field, + op, + value, + language, + } => { + let field = field.u8_id(); + match op { + TextMatch::Exact | TextMatch::Standard => { + let mut probes = Vec::new(); + tokenize(&value, language, |token| { + probes.push((field, CheekyHash::new(token.word.as_bytes()))); + true + }); + if probes.is_empty() { + return Ok(RoaringTreemap::new()); + } + self.fetch(probes.iter().copied(), None).await?; + Ok(self.intersect(probes.iter().copied())) + } + TextMatch::Prefix => { + let prefix = single_term_prefix(&value); + let prefix = prefix.as_bytes(); + if prefix.is_empty() || prefix.len() > CheekyHash::HASH_SIZE { + return Ok(RoaringTreemap::new()); + } + self.fetch(std::iter::empty(), Some((field, prefix))) + .await?; + Ok(self.prefix(field, prefix).clone()) + } + } + } + SearchFilter::Integer { field, op, value } => { + if op == Ordering::Equal { + let probe = (field.u8_id(), integer_term(value)); + self.fetch(std::iter::once(probe), None).await?; + Ok(self.term(&probe).clone()) + } else { + Err(trc::StoreEvent::NotSupported + .into_err() + .details("Integer range filters are not supported")) + } + } + SearchFilter::KeyValue { field, key, op } => { + let field = field.u8_id(); + let key = key + .chars() + .filter(|ch| !ch.is_control()) + .map(|ch| ch.to_ascii_lowercase()) + .collect::(); + let value = match &op { + KeyValueMatch::Equals(value) | KeyValueMatch::Contains(value) => value.as_str(), + KeyValueMatch::Exists => "", + }; + let mut probes = Vec::new(); + let mut buf = String::new(); + tokenize(value, Language::None, |token| { + probes.push((field, key_value_term(&key, &token.word, &mut buf))); + true + }); + if probes.is_empty() { + let probe = (field, CheekyHash::new(key.as_bytes())); + self.fetch(std::iter::once(probe), None).await?; + return Ok(self.term(&probe).clone()); + } + self.fetch(probes.iter().copied(), None).await?; + Ok(self.intersect(probes.iter().copied())) + } + _ => unreachable!(), + } + } + + fn intersect(&self, mut probes: impl Iterator) -> RoaringTreemap { + let Some(first) = probes.next() else { + return RoaringTreemap::new(); + }; + let first = self.term(&first); + if first.is_empty() { + return RoaringTreemap::new(); + } + let mut result: Option = None; + for probe in probes { + let matches = self.term(&probe); + if let Some(result) = &mut result { + *result &= matches; + } else { + result = Some(first & matches); + } + if result.as_ref().unwrap().is_empty() { + return RoaringTreemap::new(); + } + } + result.unwrap_or_else(|| first.clone()) + } + + async fn fetch( + &mut self, + probes: impl Iterator, + prefix: Option<(u8, &[u8])>, + ) -> trc::Result<()> { + let mut ranges = Vec::new(); + let mut fetch_probes: AHashSet<(u8, CheekyHash)> = AHashSet::new(); + for probe in probes { + if !self.terms.contains_key(&probe) && fetch_probes.insert(probe) { + let (begin, end) = codec::global_term_block_range( + self.index, + probe.0, + &probe.1, + self.from_block, + self.until_block, + ); + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + self.terms.insert(probe, RoaringTreemap::new()); + } + } + let mut fetch_prefix = None; + if let Some((field, prefix)) = prefix { + let cache_key = (field, CheekyHash::new(prefix)); + if !self.prefixes.contains_key(&cache_key) { + let (begin, end) = codec::global_term_prefix_range(self.index, field, prefix); + ranges.push(IterateParams::new( + codec::term_key(begin), + codec::term_key(end), + )); + fetch_prefix = Some((field, prefix, cache_key)); + } + } + if ranges.is_empty() { + return Ok(()); + } + into_disjoint(&mut ranges); + + let mut prefix_matches = RoaringTreemap::new(); + let terms = &mut self.terms; + let mut corrupted = None; + + self.store + .iterate_many(ranges, |key, value| { + let Some((field, hash, parsed)) = key + .get(codec::GLOBAL_TERM_BASE_LEN - 1) + .copied() + .zip(codec::parse_prefix_key( + key, + codec::GLOBAL_TERM_BASE_LEN, + U16_LEN, + )) + .and_then(|(field, parsed)| { + CheekyHash::from_key_bytes(parsed.term, parsed.len) + .map(|hash| (field, hash, parsed)) + }) + else { + corrupted = Some(trc::Error::corrupted_key( + key, + Some(value), + trc::location!(), + )); + return Ok(false); + }; + + let probe = (field, hash); + let is_probe = fetch_probes.contains(&probe); + let has_prefix = fetch_prefix.is_some_and(|(prefix_field, prefix, _)| { + prefix_field == field + && parsed.len as usize <= CheekyHash::HASH_SIZE + && parsed.term.last() != Some(&b'*') + && parsed.term.starts_with(prefix) + }); + + if is_probe || has_prefix { + match LazyTreemap::deserialize_delta( + value, + (parsed.block_id as u64) << GLOBAL_BUCKET_SHIFT, + ) { + Ok(treemap) => { + if has_prefix { + prefix_matches |= &treemap.0; + } + if is_probe { + *terms.get_mut(&probe).unwrap() |= treemap.0; + } + } + Err(err) => { + corrupted = Some(err); + return Ok(false); + } + } + } + + Ok(true) + }) + .await?; + + if let Some(err) = corrupted { + return Err(err); + } + + if let Some((_, _, cache_key)) = fetch_prefix { + self.prefixes.insert(cache_key, prefix_matches); + } + + Ok(()) + } + + async fn document_ids(&self) -> trc::Result { + let (begin, end) = + codec::global_document_id_range(self.index, self.from_block, self.until_block); + let mut results = RoaringTreemap::new(); + let mut corrupted = None; + + self.store + .iterate( + IterateParams::new(codec::term_key(begin), codec::term_key(end)), + |key, value| { + let decoded = codec::parse_block_id(key).and_then(|block_id| { + LazyTreemap::deserialize_delta( + value, + (block_id as u64) << GLOBAL_BUCKET_SHIFT, + ) + .ok() + }); + if let Some(treemap) = decoded { + results |= treemap.0; + Ok(true) + } else { + corrupted = Some(trc::Error::corrupted_key( + key, + Some(value), + trc::location!(), + )); + Ok(false) + } + }, + ) + .await?; + + if let Some(err) = corrupted { + return Err(err); + } + + Ok(results) + } + + fn term(&self, probe: &(u8, CheekyHash)) -> &RoaringTreemap { + &self.terms[probe] + } + + fn prefix(&self, field: u8, prefix: &[u8]) -> &RoaringTreemap { + &self.prefixes[&(field, CheekyHash::new(prefix))] + } +} + + diff --git a/crates/store/src/search/split.rs b/crates/store/src/search/split.rs index 4a5a767c..da41cdfb 100644 --- a/crates/store/src/search/split.rs +++ b/crates/store/src/search/split.rs @@ -35,19 +35,21 @@ pub(crate) fn split_filters(filters_in: Vec) -> Option { - account_id = id; + account_id = value; } - SearchFilter::Operator { .. } => { + SearchFilter::Text { .. } + | SearchFilter::KeyValue { .. } + | SearchFilter::Integer { .. } => { operators.entry(op_stack.len()).or_default().push(filter); } SearchFilter::DocumentSet(docs) => match document_sets.entry(op_stack.len()) { Entry::Occupied(mut entry) => { - if matches!(op_stack.last(), Some(SearchFilter::Or)) { + if matches!(op_stack.last(), Some(SearchFilter::Or | SearchFilter::Not)) { entry.get_mut().bitor_assign(&docs); } else { entry.get_mut().bitand_assign(&docs); @@ -95,7 +97,9 @@ pub(crate) fn split_filters(filters_in: Vec) -> Option {} + SearchFilter::Text { .. } + | SearchFilter::Integer { .. } + | SearchFilter::KeyValue { .. } => {} SearchFilter::DocumentSet(_) => { if depth == 0 && j > i { break; @@ -109,13 +113,15 @@ pub(crate) fn split_filters(filters_in: Vec) -> Option i + 1; + let add_or = matches!( + split.last(), + Some(SplitFilter::Internal(SearchFilter::Or | SearchFilter::Not)) + ) && j > i + 1; if add_or { external_filters.push(SearchFilter::Or); } @@ -132,577 +138,3 @@ pub(crate) fn split_filters(filters_in: Vec) -> Option, Vec)> = vec![ - // Test 1: Operator followed by document set at depth 0 - ( - "Operator then document set at depth 0", - vec![account_id(42), other_op("test"), doc_set(&[1, 2, 3])], - vec![ - SplitFilter::External(vec![account_id(42), other_op("test")]), - SplitFilter::Internal(doc_set(&[1, 2, 3])), - ], - ), - // Test 2: Document set followed by operator at depth 0 - ( - "Document set then operator at depth 0", - vec![account_id(42), doc_set(&[1, 2, 3]), other_op("test")], - vec![ - SplitFilter::External(vec![account_id(42), other_op("test")]), - SplitFilter::Internal(doc_set(&[1, 2, 3])), - ], - ), - // Test 3: Multiple document sets with operator in between - ( - "Multiple document sets at depth 0 with operator", - vec![ - account_id(42), - doc_set(&[1, 2]), - other_op("middle"), - doc_set(&[2, 4]), - ], - vec![ - SplitFilter::External(vec![account_id(42), other_op("middle")]), - SplitFilter::Internal(doc_set(&[2])), - ], - ), - // Test 4: Document set at depth 0, then AND group - ( - "Document set then AND group", - vec![ - account_id(42), - doc_set(&[1, 2]), - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 5: AND group followed by document set at depth 0 - ( - "AND group then document set", - vec![ - account_id(42), - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - doc_set(&[1, 2]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 6: Operator at depth 0, then OR group, then document set - ( - "Operator, OR group, then document set", - vec![ - account_id(42), - other_op("pre"), - SearchFilter::Or, - other_op("a"), - other_op("b"), - SearchFilter::End, - doc_set(&[1, 2, 3]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - other_op("a"), - other_op("b"), - SearchFilter::End, - other_op("pre"), - ]), - SplitFilter::Internal(doc_set(&[1, 2, 3])), - ], - ), - // Test 7: Document set, OR group, operator - ( - "Document set, OR group, operator", - vec![ - account_id(42), - doc_set(&[1, 2]), - SearchFilter::Or, - other_op("a"), - other_op("b"), - SearchFilter::End, - other_op("post"), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - other_op("a"), - other_op("b"), - SearchFilter::End, - other_op("post"), - ]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 8: Multiple OR branches with document sets between - ( - "Multiple OR branches with document sets between", - vec![ - account_id(42), - SearchFilter::Or, - other_op("a"), - SearchFilter::End, - doc_set(&[1, 2]), - SearchFilter::Or, - other_op("b"), - SearchFilter::End, - doc_set(&[1, 2, 5, 6]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - other_op("a"), - SearchFilter::End, - SearchFilter::Or, - other_op("b"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 9: Document sets at different depths - depth 0 and inside AND - ( - "Document sets at different depths in AND", - vec![ - account_id(42), - doc_set(&[1, 2]), - SearchFilter::And, - other_op("a"), - doc_set(&[2, 3]), - SearchFilter::End, - ], - vec![ - SplitFilter::Internal(SearchFilter::And), - SplitFilter::External(vec![account_id(42), other_op("a")]), - SplitFilter::Internal(doc_set(&[2, 3])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 10: Operator, AND group with doc set inside, operator - ( - "Operator, AND(operator, doc_set), operator", - vec![ - account_id(42), - other_op("pre"), - SearchFilter::And, - other_op("a"), - doc_set(&[1, 2, 3]), - SearchFilter::End, - other_op("post"), - ], - vec![ - SplitFilter::Internal(SearchFilter::And), - SplitFilter::External(vec![account_id(42), other_op("a")]), - SplitFilter::Internal(doc_set(&[1, 2, 3])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("pre"), other_op("post")]), - ], - ), - // Test 11: Document set, nested groups, document set - ( - "Doc set, AND(OR(a,b)), doc set", - vec![ - account_id(42), - SearchFilter::Or, - doc_set(&[1, 2]), - other_op("c"), - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - doc_set(&[3, 4]), - SearchFilter::End, - ], - vec![ - SplitFilter::Internal(SearchFilter::Or), - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - SearchFilter::And, - other_op("a"), - other_op("b"), - SearchFilter::End, - other_op("c"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[1, 2, 3, 4])), - SplitFilter::Internal(SearchFilter::End), - ], - ), - // Test 12: OR with nested AND containing document sets, followed by operator - ( - "OR(AND(doc_set, doc_set), operator) followed by operator", - vec![ - account_id(42), - SearchFilter::Or, - SearchFilter::And, - doc_set(&[1, 2]), - doc_set(&[2, 3]), - SearchFilter::End, - other_op("b"), - SearchFilter::End, - other_op("post"), - ], - vec![ - SplitFilter::Internal(SearchFilter::Or), - SplitFilter::Internal(SearchFilter::And), - SplitFilter::Internal(doc_set(&[2])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("b")]), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("post")]), - ], - ), - // Test 13: Complex: doc set, AND group, doc set, OR group, doc set - ( - "Complex: doc, AND, doc, OR, doc", - vec![ - account_id(42), - doc_set(&[1, 2, 3]), - SearchFilter::And, - other_op("a"), - SearchFilter::End, - doc_set(&[1, 2, 3, 5]), - SearchFilter::Or, - other_op("b"), - SearchFilter::End, - doc_set(&[1, 2, 3, 6]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::And, - other_op("a"), - SearchFilter::End, - SearchFilter::Or, - other_op("b"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[1, 2, 3])), - ], - ), - // Test 14: Operator, NOT group, document set - ( - "Operator, NOT(operator), document set", - vec![ - account_id(42), - other_op("pre"), - SearchFilter::Not, - other_op("a"), - SearchFilter::End, - doc_set(&[1, 2]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Not, - other_op("a"), - SearchFilter::End, - other_op("pre"), - ]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 15: Document set, NOT group, operator - ( - "Document set, NOT(operator), operator", - vec![ - account_id(42), - doc_set(&[1, 2]), - SearchFilter::Not, - other_op("a"), - doc_set(&[3, 4]), - SearchFilter::End, - other_op("post"), - ], - vec![ - SplitFilter::Internal(SearchFilter::Not), - SplitFilter::External(vec![account_id(42), other_op("a")]), - SplitFilter::Internal(doc_set(&[3, 4])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("post")]), - SplitFilter::Internal(doc_set(&[1, 2])), - ], - ), - // Test 16: Alternating doc sets and operators - ( - "Alternating: doc, op, doc, op, doc", - vec![ - account_id(42), - doc_set(&[1]), - other_op("a"), - doc_set(&[1, 2]), - other_op("b"), - doc_set(&[1, 3]), - ], - vec![ - SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), - SplitFilter::Internal(doc_set(&[1])), - ], - ), - // Test 17: Multiple operators, then OR group with doc set inside, then doc set - ( - "Multiple ops, OR(op, doc_set), doc", - vec![ - account_id(42), - other_op("a"), - SearchFilter::Or, - other_op("c"), - doc_set(&[1, 2]), - SearchFilter::End, - other_op("b"), - doc_set(&[3, 4]), - ], - vec![ - SplitFilter::Internal(SearchFilter::Or), - SplitFilter::External(vec![account_id(42), other_op("c")]), - SplitFilter::Internal(doc_set(&[1, 2])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), - SplitFilter::Internal(doc_set(&[3, 4])), - ], - ), - // Test 18: Doc set before and after nested OR(AND(op)) - ( - "Doc, OR(AND(op)), doc", - vec![ - account_id(42), - doc_set(&[1]), - SearchFilter::Or, - SearchFilter::And, - other_op("a"), - other_op("c"), - SearchFilter::End, - other_op("b"), - SearchFilter::End, - doc_set(&[2]), - ], - vec![ - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - SearchFilter::And, - other_op("a"), - other_op("c"), - SearchFilter::End, - other_op("b"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[])), - ], - ), - // Test 19: AND group with doc set, operator between, OR group with doc set - ( - "AND(op, doc), op, OR(op, doc)", - vec![ - account_id(42), - SearchFilter::And, - other_op("a"), - doc_set(&[1, 2]), - SearchFilter::End, - other_op("middle"), - SearchFilter::Or, - other_op("b"), - other_op("c"), - doc_set(&[3, 4]), - SearchFilter::End, - ], - vec![ - SplitFilter::Internal(SearchFilter::And), - SplitFilter::External(vec![account_id(42), other_op("a")]), - SplitFilter::Internal(doc_set(&[1, 2])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::Internal(SearchFilter::Or), - SplitFilter::External(vec![ - account_id(42), - SearchFilter::Or, - other_op("b"), - other_op("c"), - SearchFilter::End, - ]), - SplitFilter::Internal(doc_set(&[3, 4])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::External(vec![account_id(42), other_op("middle")]), - ], - ), - // Test 20: Deep nesting with document sets at multiple levels - ( - "Deep nesting: doc, AND(doc, OR(doc, AND(op, doc)))", - vec![ - account_id(42), - doc_set(&[1]), - SearchFilter::And, - doc_set(&[2]), - SearchFilter::Or, - doc_set(&[3]), - SearchFilter::And, - other_op("a"), - doc_set(&[4]), - SearchFilter::End, - SearchFilter::End, - SearchFilter::End, - ], - vec![ - SplitFilter::Internal(SearchFilter::And), - SplitFilter::Internal(SearchFilter::Or), - SplitFilter::Internal(SearchFilter::And), - SplitFilter::External(vec![account_id(42), other_op("a")]), - SplitFilter::Internal(doc_set(&[4])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::Internal(doc_set(&[3])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::Internal(doc_set(&[2])), - SplitFilter::Internal(SearchFilter::End), - SplitFilter::Internal(doc_set(&[1])), - ], - ), - ]; - - for (description, input, expected) in test_cases { - println!("------ Running test: {} ------", description); - let result = split_filters(input.clone()); - assert!(result.is_some(), "Test '{}' returned None", description); - - let result = result.unwrap(); - if result != expected { - print_split_filter_code(&result); - } - assert_eq!(result, expected, "Test '{description}' failed",); - } - } - - fn account_id(id: u64) -> SearchFilter { - SearchFilter::Operator { - field: SearchField::AccountId, - op: SearchOperator::Equal, - value: SearchValue::Uint(id), - } - } - - fn other_op(value: &str) -> SearchFilter { - SearchFilter::Operator { - field: SearchField::DocumentId, - op: SearchOperator::Equal, - value: SearchValue::Text { - value: value.to_string(), - language: Language::None, - }, - } - } - - fn doc_set(ids: &[u32]) -> SearchFilter { - let mut bitmap = RoaringBitmap::new(); - for id in ids { - bitmap.insert(*id); - } - SearchFilter::DocumentSet(bitmap) - } - - fn print_split_filter_code(splits: &[SplitFilter]) { - println!("vec!["); - for split in splits { - match split { - SplitFilter::Internal(filter) => { - print!(" SplitFilter::Internal("); - print_search_filter_code(filter, 0); - println!("),"); - } - SplitFilter::External(filters) => { - println!(" SplitFilter::External(vec!["); - for filter in filters { - print!(" "); - print_search_filter_code(filter, 2); - println!(","); - } - println!(" ]),"); - } - } - } - println!("]"); - } - - fn print_search_filter_code(filter: &SearchFilter, indent_level: usize) { - let indent = " ".repeat(indent_level); - match filter { - SearchFilter::Operator { field, op, value } => match (field, op, value) { - (SearchField::AccountId, SearchOperator::Equal, SearchValue::Uint(id)) => { - print!("account_id({})", id); - } - ( - SearchField::DocumentId, - SearchOperator::Equal, - SearchValue::Text { value, .. }, - ) => { - print!("other_op(\"{}\")", value); - } - _ => { - println!("SearchFilter::Operator {{"); - println!("{} field: {:?},", indent, field); - println!("{} op: {:?},", indent, op); - println!("{} value: {:?},", indent, value); - print!("{}}}", indent); - } - }, - SearchFilter::DocumentSet(bitmap) => { - let ids: Vec = bitmap.iter().collect(); - if ids.is_empty() { - print!("doc_set(&[])"); - } else if ids.len() <= 5 { - print!("doc_set(&["); - for (i, id) in ids.iter().enumerate() { - if i > 0 { - print!(", "); - } - print!("{}", id); - } - print!("])"); - } else { - // For large bitmaps, create inline - println!("{{"); - println!("{} let mut bitmap = RoaringBitmap::new();", indent); - for id in ids { - println!("{} bitmap.insert({});", indent, id); - } - print!("{} doc_set_bitmap(bitmap)", indent); - println!(); - print!("{}}}", indent); - } - } - SearchFilter::And => print!("SearchFilter::And"), - SearchFilter::Or => print!("SearchFilter::Or"), - SearchFilter::Not => print!("SearchFilter::Not"), - SearchFilter::End => print!("SearchFilter::End"), - } - } -} diff --git a/crates/store/src/search/term.rs b/crates/store/src/search/term.rs index f56a8001..e2cd0f3b 100644 --- a/crates/store/src/search/term.rs +++ b/crates/store/src/search/term.rs @@ -5,447 +5,763 @@ */ use crate::{ - Serialize, - backend::MAX_TOKEN_LENGTH, - search::*, + Serialize, U32_LEN, + search::{ + ACCOUNT_BLOCK_SHIFT, EmailSearchField, GLOBAL_BUCKET_SHIFT, IndexDocument, + MAX_DOCUMENT_SIZE, SearchField, SearchValue, + codec::{self}, + tokenize::{integer_term, key_value_term, stem_term, tokenize, zigzag}, + }, write::{ - Archiver, BatchBuilder, SEARCH_INDEX_MAX_FIELD_LEN, SearchIndexClass, SearchIndexField, - SearchIndexId, SearchIndexType, ValueClass, + BatchBuilder, MergeResult, SearchIndex, SearchIndexClass, + lazybitmap::{LazyBitmap, LazyTreemap}, }, }; -use ahash::AHashSet; -use nlp::{ - language::stemmer::Stemmer, - tokenizers::{space::SpaceTokenizer, word::WordTokenizer}, -}; -use utils::{ - cheeky_hash::{CheekyBTreeMap, CheekyHash}, - map::bitmap::BitPop, -}; +use ahash::{AHashMap, AHashSet}; +use indexmap::{IndexMap, IndexSet, map::Entry}; +use nlp::language::Language; +use roaring::{RoaringBitmap, RoaringTreemap}; +use std::sync::Arc; +use utils::cheeky_hash::{CheekyHash, CheekyHashMap}; -#[derive(Debug, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -pub(crate) struct TermIndex { - terms: Vec, - fields: Vec, +const MAX_MERGES_PER_COMMIT: usize = 1000; + +#[derive(Default)] +pub(crate) struct AccountIndexer { + postings: IndexSet, + entries: Vec, + pub documents: AHashMap>>, + posting_overflow: bool, } -#[derive(Debug, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct PostingKey { + term: CheekyHash, + field: u8, + block: u16, +} + +#[derive(Clone, Copy)] +struct PostingEntry { + posting: u32, + document: u32, + op: PostingOp, +} + +#[derive(Clone, Copy)] +enum PostingOp { + Removed, + Inserted, + Stored, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TermAdmission { + Rejected, + Repeated, + FirstInField, +} + +const _: () = assert!(std::mem::size_of::() <= 12); + +#[derive(Default)] +pub(crate) struct GlobalIndexer { + pub terms: CheekyHashMap>>>, + pub documents: AHashMap, +} + +#[derive(Default)] +pub(crate) struct Document { + pub terms: IndexMap>, + pub fields: IndexMap, ahash::RandomState>, + size: usize, + truncated: bool, +} + +const TERM_OVERHEAD: usize = 4; +const POSITION_SIZE: usize = 3; +const FIELD_OVERHEAD: usize = 5; + +#[derive(Default)] pub(crate) struct Term { - hash: CheekyHash, - fields: u32, + id: u32, + field_mask: u32, } -pub(crate) struct TermIndexBuilder { - pub(crate) index: TermIndex, - pub(crate) id: SearchIndexId, -} +impl AccountIndexer { + pub fn insert(&mut self, index_document: IndexDocument, document_id: u32) -> Option { + let document = self.index_document(index_document, document_id); + let truncated = document.truncated_size(); + self.documents + .insert(document_id, Some(document.encode().into_document())); + truncated + } -impl TermIndexBuilder { - pub fn build(document: IndexDocument, truncate_at: usize) -> Self { - let mut terms: CheekyBTreeMap = CheekyBTreeMap::new(); - let mut fields: Vec = Vec::new(); - let mut account_id = None; - let mut document_id = None; - let mut id = None; + pub(crate) fn index_document( + &mut self, + index_document: IndexDocument, + document_id: u32, + ) -> Document { + let mut document = Document::default(); + let mut buf = String::new(); + let mut body = None; + let mut attach = None; - for (field, value) in document.fields { + for (field, value) in &index_document.fields { match field { - SearchField::Id => { - if let SearchValue::Uint(v) = value { - fields.push(SearchIndexField { - field_id: field.u8_id(), - data: v.to_be_bytes().to_vec(), - }); - id = Some(v); - } - - continue; + SearchField::AccountId | SearchField::DocumentId => {} + SearchField::Email(EmailSearchField::Body) => { + body = Some((field.u8_id(), value)); } - SearchField::AccountId => { - if let SearchValue::Uint(v) = value { - account_id = Some(v); - } - continue; + SearchField::Email(EmailSearchField::Attachment) => { + attach = Some((field.u8_id(), value)); } - SearchField::DocumentId => { - if let SearchValue::Uint(v) = value { - document_id = Some(v); - } - continue; + _ => { + self.insert_value(&mut document, &mut buf, document_id, field.u8_id(), value); } - _ => {} } + } - let field = match value { - SearchValue::Text { value, language } => { - if field.is_text() { - let value = if truncate_at > 0 && value.len() > truncate_at { - let pos = value.floor_char_boundary(truncate_at); - &value[..pos] - } else { - &value - }; + if let Some((field_id, value)) = body { + self.insert_value(&mut document, &mut buf, document_id, field_id, value); + } - match language { - Language::Unknown => { - for token in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { - terms - .entry(CheekyHash::new(token.word.as_bytes())) - .or_default() - .bit_push(field.u8_id()); - } - } - Language::None => { - for token in SpaceTokenizer::new(value, MAX_TOKEN_LENGTH) { - terms - .entry(CheekyHash::new(token.as_bytes())) - .or_default() - .bit_push(field.u8_id()); - } - } - _ => { - for token in Stemmer::new(value, language, MAX_TOKEN_LENGTH) { - terms - .entry(CheekyHash::new(token.word.as_bytes())) - .or_default() - .bit_push(field.u8_id()); + if let Some((field_id, value)) = attach { + self.insert_value(&mut document, &mut buf, document_id, field_id, value); + } - if let Some(stemmed_word) = token.stemmed_word { - terms - .entry(CheekyHash::new( - format!("{}*", stemmed_word).as_bytes(), - )) - .or_default() - .bit_push(field.u8_id()); - } - } + document + } + + fn insert_value( + &mut self, + document: &mut Document, + buf: &mut String, + document_id: u32, + field_id: u8, + value: &SearchValue, + ) { + match value { + SearchValue::Text { value, language } => { + let language = *language; + let add_position = language != Language::None; + tokenize(value, language, |token| { + let term = CheekyHash::new(token.word.as_bytes()); + match document.add_term(term, field_id, add_position) { + TermAdmission::Rejected => return false, + TermAdmission::Repeated => return true, + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + if let Some(stem) = token.stem { + let term = stem_term(&stem, buf); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => return false, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); } } } + true + }); + } + SearchValue::KeyValues(map) => { + for (key, value) in map { + let term = CheekyHash::new(key.as_bytes()); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => break, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + tokenize(value, Language::None, |token| { + let term = key_value_term(key, &token.word, buf); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => return false, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + true + }); + } + } + SearchValue::Int(v) => { + self.add_single_term(document, document_id, integer_term(zigzag(*v)), field_id); + } + SearchValue::Uint(v) => { + self.add_single_term(document, document_id, integer_term(*v), field_id); + } + SearchValue::Boolean(v) => { + self.add_single_term(document, document_id, CheekyHash::new([*v as u8]), field_id); + } + } + } - if field.is_indexed() { - let mut data = value.into_bytes(); - data.truncate(SEARCH_INDEX_MAX_FIELD_LEN); + fn add_single_term( + &mut self, + document: &mut Document, + document_id: u32, + term: CheekyHash, + field_id: u8, + ) { + if document.add_term(term, field_id, false) == TermAdmission::FirstInField { + self.add_term(document_id, term, field_id); + } + } - SearchIndexField { - field_id: field.u8_id(), - data, + pub fn diff(&mut self, current_document: &[u8], document_id: u32) -> trc::Result<()> { + self.add_stored(current_document, document_id, PostingOp::Stored) + } + + pub fn remove(&mut self, current_document: &[u8], document_id: u32) -> trc::Result<()> { + self.documents.insert(document_id, None); + self.add_stored(current_document, document_id, PostingOp::Removed) + } + + fn add_stored( + &mut self, + current_document: &[u8], + document_id: u32, + op: PostingOp, + ) -> trc::Result<()> { + let block = (document_id >> ACCOUNT_BLOCK_SHIFT) as u16; + let mut scratch = Vec::new(); + deserialize_term_fields(current_document, &mut scratch, |term, mut field_mask| { + while field_mask != 0 { + let field = 31 - field_mask.leading_zeros(); + field_mask ^= 1 << field; + self.push( + PostingKey { + term, + field: field as u8, + block, + }, + document_id, + op, + ); + } + }) + .ok_or_else(|| trc::Error::corrupted_key(current_document, None, trc::location!())) + } + + fn add_term(&mut self, document_id: u32, term: CheekyHash, field: u8) { + self.push( + PostingKey { + term, + field, + block: (document_id >> ACCOUNT_BLOCK_SHIFT) as u16, + }, + document_id, + PostingOp::Inserted, + ); + } + + fn push(&mut self, key: PostingKey, document: u32, op: PostingOp) { + let (posting, _) = self.postings.insert_full(key); + match u32::try_from(posting) { + Ok(posting) => self.entries.push(PostingEntry { + posting, + document, + op, + }), + Err(_) => self.posting_overflow = true, + } + } + + pub fn build_batch(self, index: SearchIndex, account_id: u32) -> trc::Result { + if self.posting_overflow { + return Err(trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Too many postings in one index batch.")); + } + let mut batch = BatchBuilder::new(); + let mut entries = self.entries; + radsort::sort_by_key(&mut entries, |entry| (entry.posting, entry.document)); + + let mut updates = Vec::with_capacity(entries.len()); + let mut postings = Vec::with_capacity(self.postings.len()); + for (key, entries) in self + .postings + .into_iter() + .zip(entries.chunk_by(|a, b| a.posting == b.posting)) + { + let start = updates.len(); + for document in entries.chunk_by(|a, b| a.document == b.document) { + let mut update = None; + for entry in document { + update = match entry.op { + PostingOp::Removed => Some(false), + PostingOp::Inserted => Some(true), + PostingOp::Stored if update.is_some() => None, + PostingOp::Stored => Some(false), + }; + } + if let (Some(insert), Some(entry)) = (update, document.first()) { + updates.push((entry.document, insert)); + } + } + if updates.len() > start { + postings.push((key, start..updates.len())); + } + } + postings.sort_unstable_by(|(a, _), (b, _)| { + account_term_key_order(index, a.field, &a.term, a.block) + .cmp(&account_term_key_order(index, b.field, &b.term, b.block)) + }); + + let updates: Arc<[(u32, bool)]> = updates.into(); + let mut num_merge_ops = 0; + for (key, range) in postings { + let block_base = (key.block as u32) << ACCOUNT_BLOCK_SHIFT; + let updates = updates.clone(); + batch.merge_fnc( + SearchIndexClass::Term { + index, + account_id, + field: key.field, + term: key.term, + block_id: key.block, + }, + move |_, bytes| { + let updates = updates.get(range.clone()).unwrap_or_default(); + if let Some(bytes) = bytes { + let mut map = LazyBitmap::deserialize_delta(bytes, block_base)?; + let mut has_changes = false; + for (document_id, do_insert) in updates { + if *do_insert { + has_changes |= map.0.insert(*document_id); + } else { + has_changes |= map.0.remove(*document_id); + } + } + if !has_changes { + Ok(MergeResult::Skip) + } else if !map.0.is_empty() { + Ok(MergeResult::Update(map.serialize_optimized(block_base))) + } else { + Ok(MergeResult::Delete) } } else { - continue; - } - } - SearchValue::KeyValues(map) => { - for (key, value) in map { - terms - .entry(CheekyHash::new(key.as_bytes())) - .or_default() - .bit_push(field.u8_id()); - for token in SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH) { - terms - .entry(CheekyHash::new(format!("{key} {token}").as_bytes())) - .or_default() - .bit_push(field.u8_id()); + let mut map = RoaringBitmap::new(); + for (document_id, _) in updates.iter().filter(|(_, do_insert)| *do_insert) { + map.insert(*document_id); + } + if !map.is_empty() { + Ok(MergeResult::Update( + LazyBitmap(map).serialize_optimized(block_base), + )) + } else { + Ok(MergeResult::Skip) } } + }, + ); + if num_merge_ops >= MAX_MERGES_PER_COMMIT { + batch.add_commit_point(); + num_merge_ops = 0; + } else { + num_merge_ops += 1; + } + } - continue; - } - SearchValue::Int(v) => SearchIndexField { - field_id: field.u8_id(), - data: (v as u64).to_be_bytes().to_vec(), - }, - SearchValue::Uint(v) => SearchIndexField { - field_id: field.u8_id(), - data: v.to_be_bytes().to_vec(), - }, - SearchValue::Boolean(v) => SearchIndexField { - field_id: field.u8_id(), - data: vec![v as u8], - }, + let mut documents = self.documents.into_iter().collect::>(); + documents.sort_unstable_by_key(|(document_id, _)| *document_id); + for (document_id, document) in documents { + let class = SearchIndexClass::Document { + index, + account_id, + document_id, + }; + match document { + Some(document) => batch.set(class, document).commit_point(), + None => batch.clear(class).commit_point(), }; - - fields.push(field); } - TermIndexBuilder { - index: TermIndex { - terms: terms - .into_iter() - .map(|(k, v)| Term { hash: k, fields: v }) - .collect(), - fields, - }, - id: match (account_id, document_id, id) { - (Some(account_id), Some(document_id), None) => SearchIndexId::Account { - account_id: account_id as u32, - document_id: document_id as u32, - }, - (None, None, Some(id)) => SearchIndexId::Global { id }, - _ => { - debug_assert!( - false, - "Invalid combination of AccountId {account_id:?}, DocumentId {document_id:?} and Id {id:?} fields" - ); - SearchIndexId::Global { id: 0 } - } - }, - } + Ok(batch) } } -impl TermIndex { - pub fn write_index( - self, - batch: &mut BatchBuilder, - index: SearchIndex, - id: SearchIndexId, - ) -> trc::Result<()> { - let archive = Archiver::new(self); - batch - .set( - ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Document, - }), - archive.serialize()?, - ) - .commit_point(); +impl GlobalIndexer { + pub fn insert(&mut self, index_document: IndexDocument, document_id: u64) -> Option { + let mut document = Document::default(); + let mut buf = String::new(); - for term in archive.inner.terms { - let mut fields = term.fields; - while let Some(field) = fields.bit_pop() { - batch - .set( - ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Term { - hash: term.hash, - field, - }, - }), - vec![], - ) - .commit_point(); + self.terms + .entry(CheekyHash::new("_id")) + .or_default() + .entry(SearchField::Id.u8_id()) + .or_default() + .entry((document_id >> GLOBAL_BUCKET_SHIFT) as u16) + .or_default() + .insert(document_id); + + for (field, value) in index_document.fields { + if field == SearchField::Id { + continue; + } + + let field_id = field.u8_id(); + + match value { + SearchValue::Text { value, language } => { + tokenize(&value, language, |token| { + let term = CheekyHash::new(token.word.as_bytes()); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => return false, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + true + }); + } + SearchValue::KeyValues(map) => { + for (key, value) in map { + let term = CheekyHash::new(key.as_bytes()); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => break, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + tokenize(&value, Language::None, |token| { + let term = key_value_term(&key, &token.word, &mut buf); + match document.add_term(term, field_id, false) { + TermAdmission::Rejected => return false, + TermAdmission::Repeated => {} + TermAdmission::FirstInField => { + self.add_term(document_id, term, field_id); + } + } + true + }); + } + } + SearchValue::Int(v) => { + let term = integer_term(zigzag(v)); + if document.add_term(term, field_id, false) == TermAdmission::FirstInField { + self.add_term(document_id, term, field_id); + } + } + SearchValue::Uint(v) => { + let term = integer_term(v); + if document.add_term(term, field_id, false) == TermAdmission::FirstInField { + self.add_term(document_id, term, field_id); + } + } + SearchValue::Boolean(v) => { + let term = CheekyHash::new([v as u8]); + if document.add_term(term, field_id, false) == TermAdmission::FirstInField { + self.add_term(document_id, term, field_id); + } + } } } - for field in archive.inner.fields { + let truncated = document.truncated_size(); + self.documents.insert(document_id, document); + truncated + } + + fn add_term(&mut self, document_id: u64, term: CheekyHash, field_id: u8) { + self.terms + .entry(term) + .or_default() + .entry(field_id) + .or_default() + .entry((document_id >> GLOBAL_BUCKET_SHIFT) as u16) + .or_default() + .insert(document_id); + } + + pub fn build_batch(self, index: SearchIndex) -> BatchBuilder { + let mut batch = BatchBuilder::new(); + + // Build terms + let mut postings = Vec::with_capacity(self.terms.len()); + for (term, fields) in self.terms { + for (field, blocks) in fields { + for (block_id, document_ids) in blocks { + postings.push((field, term, block_id, document_ids)); + } + } + } + postings.sort_unstable_by(|a, b| { + term_key_order(a.0, &a.1, a.2).cmp(&term_key_order(b.0, &b.1, b.2)) + }); + + let mut num_merge_ops = 0; + for (field, term, block_id, document_ids) in postings { + let key = if field != SearchField::Id.u8_id() { + SearchIndexClass::GlobalTerm { + index, + field, + term, + block_id, + } + } else { + SearchIndexClass::GlobalDocumentId { index, block_id } + }; + + batch.merge_fnc(key, move |_, bytes| { + if let Some(bytes) = bytes { + let mut treemap = LazyTreemap::deserialize_delta( + bytes, + (block_id as u64) << GLOBAL_BUCKET_SHIFT, + )?; + let cur_len = treemap.0.len(); + for document_id in &document_ids { + treemap.0.insert(*document_id); + } + if treemap.0.len() > cur_len { + Ok(MergeResult::Update( + treemap.serialize_delta((block_id as u64) << GLOBAL_BUCKET_SHIFT), + )) + } else { + Ok(MergeResult::Skip) + } + } else { + let mut treemap = RoaringTreemap::new(); + + for document_id in &document_ids { + treemap.insert(*document_id); + } + + Ok(MergeResult::Update( + LazyTreemap(treemap) + .serialize_delta((block_id as u64) << GLOBAL_BUCKET_SHIFT), + )) + } + }); + + if num_merge_ops >= 1000 { + batch.add_commit_point(); + num_merge_ops = 0; + } else { + num_merge_ops += 1; + } + } + + // Add documents + let mut documents = self.documents.into_iter().collect::>(); + documents.sort_unstable_by_key(|(document_id, _)| *document_id); + for (document_id, document) in documents { batch .set( - ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Index { field }, - }), - vec![], + SearchIndexClass::GlobalDocument { index, document_id }, + document.serialize().unwrap_or_default(), ) .commit_point(); } - Ok(()) + batch + } +} + +impl Document { + fn add_term(&mut self, term: CheekyHash, field_id: u8, add_position: bool) -> TermAdmission { + let mut cost = if add_position { + POSITION_SIZE + + if self.fields.contains_key(&field_id) { + 0 + } else { + FIELD_OVERHEAD + } + } else { + 0 + }; + + let field_mask = 1 << field_id; + let term_id = self.terms.len() as u32; + let (term_id, admission) = match self.terms.entry(term) { + Entry::Occupied(entry) => { + if self.size + cost > MAX_DOCUMENT_SIZE { + self.truncated = true; + return TermAdmission::Rejected; + } + let term = entry.into_mut(); + let admission = if term.field_mask & field_mask == 0 { + TermAdmission::FirstInField + } else { + TermAdmission::Repeated + }; + term.field_mask |= field_mask; + (term.id, admission) + } + Entry::Vacant(entry) => { + cost += term.key_len() + TERM_OVERHEAD; + if self.size + cost > MAX_DOCUMENT_SIZE { + self.truncated = true; + return TermAdmission::Rejected; + } + + entry.insert(Term { + id: term_id, + field_mask, + }); + (term_id, TermAdmission::FirstInField) + } + }; + + self.size += cost; + + if add_position { + self.fields.entry(field_id).or_default().push(term_id); + } + + admission } - pub fn merge_index( - self, - batch: &mut BatchBuilder, - index: SearchIndex, - id: SearchIndexId, - old_term: &ArchivedTermIndex, - ) -> trc::Result<()> { - let archive = Archiver::new(self); - batch - .set( - ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Document, - }), - archive.serialize()?, - ) - .commit_point(); + fn truncated_size(&self) -> Option { + self.truncated.then_some(self.size) + } +} - let mut old_terms = AHashSet::with_capacity(old_term.terms.len()); - let mut old_fields = AHashSet::with_capacity(old_term.fields.len()); - for term in old_term.terms.iter() { - let mut fields = term.fields.to_native(); - while let Some(field) = fields.bit_pop() { - old_terms.insert(SearchIndexType::Term { - hash: term.hash.to_native(), - field, - }); +impl Document { + pub(crate) fn encode(&self) -> codec::Writer { + let mut writer = + codec::Writer::with_capacity(self.terms.len() * (CheekyHash::HASH_SIZE + U32_LEN)); + writer.push_leb128(self.terms.len()); + for (term_hash, term) in &self.terms { + writer.push_term(term_hash); + writer.push_leb128(term.field_mask); + } + writer.push_leb128(self.fields.len()); + for (field, positions) in &self.fields { + writer.push_u8(*field); + writer.push_leb128(positions.len()); + for position in positions { + writer.push_leb128(*position); } } - for field in old_term.fields.iter() { - old_fields.insert(SearchIndexField { - field_id: field.field_id, - data: field.data.to_vec(), - }); - } - for term in archive.inner.terms { - let mut fields = term.fields; - while let Some(field) = fields.bit_pop() { - let typ = SearchIndexType::Term { - hash: term.hash, - field, - }; + writer + } +} - if !old_terms.remove(&typ) { - batch - .set( - ValueClass::SearchIndex(SearchIndexClass { index, id, typ }), - vec![], - ) - .commit_point(); +impl Serialize for Document { + fn serialize(&self) -> trc::Result> { + Ok(self.encode().into_document()) + } +} + +pub(crate) fn matches_phrase( + document: &[u8], + field: u8, + words: &[CheekyHash], + ordinals: &mut Vec, + failure: &mut Vec, +) -> Option { + let mut reader = codec::Reader::new(document); + + let items = reader.leb128::()?; + ordinals.clear(); + ordinals.resize(words.len(), u32::MAX); + let mut found_all = false; + for ordinal in 0..items { + let term_hash = reader.term()?; + reader.leb128::()?; + if !found_all { + let mut num_found = 0; + for (word, slot) in words.iter().zip(ordinals.iter_mut()) { + if *slot == u32::MAX { + if *word == term_hash { + *slot = ordinal as u32; + num_found += 1; + } + } else { + num_found += 1; } } + found_all = num_found == words.len(); } + } + if !found_all { + return Some(false); + } - for field in archive.inner.fields { - if !old_fields.remove(&field) { - batch - .set( - ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Index { field }, - }), - vec![], - ) - .commit_point(); + let num_fields = reader.leb128::()?; + for _ in 0..num_fields { + let field_id = reader.u8()?; + let num_positions = reader.leb128::()?; + if field_id == field { + failure.clear(); + failure.resize(ordinals.len(), 0); + let mut prefix_len = 0; + for i in 1..ordinals.len() { + while prefix_len > 0 && ordinals[i] != ordinals[prefix_len] { + prefix_len = failure[prefix_len - 1]; + } + if ordinals[i] == ordinals[prefix_len] { + prefix_len += 1; + } + failure[i] = prefix_len; } - } - for typ in old_terms { - batch - .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ })) - .commit_point(); - } - - for field in old_fields { - batch - .clear(ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Index { field }, - })) - .commit_point(); - } - - Ok(()) - } -} - -impl ArchivedTermIndex { - pub fn delete_index(&self, batch: &mut BatchBuilder, index: SearchIndex, id: SearchIndexId) { - batch - .clear(ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Document, - })) - .commit_point(); - - for term in self.terms.iter() { - let mut fields = term.fields.to_native(); - while let Some(field) = fields.bit_pop() { - batch - .clear(ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Term { - hash: term.hash.to_native(), - field, - }, - })) - .commit_point(); + let mut matched = 0; + for _ in 0..num_positions { + let position = reader.leb128::()?; + while matched > 0 && position != ordinals[matched] { + matched = failure[matched - 1]; + } + if position == ordinals[matched] { + matched += 1; + if matched == ordinals.len() { + return Some(true); + } + } } - } - - for field in self.fields.iter() { - batch - .clear(ValueClass::SearchIndex(SearchIndexClass { - index, - id, - typ: SearchIndexType::Index { - field: SearchIndexField { - field_id: field.field_id, - data: field.data.to_vec(), - }, - }, - })) - .commit_point(); + return Some(false); + } else { + reader.skip_leb128(num_positions)?; } } + + Some(true) } -impl SearchIndex { - pub(crate) fn as_u8(&self) -> u8 { - match self { - SearchIndex::Email => 0, - SearchIndex::Calendar => 1, - SearchIndex::Contacts => 2, - SearchIndex::File => 3, - SearchIndex::Tracing => 4, - SearchIndex::InMemory => unreachable!(), - } +pub(crate) fn deserialize_term_fields( + stored: &[u8], + scratch: &mut Vec, + mut cb: impl FnMut(CheekyHash, u32), +) -> Option<()> { + let mut reader = codec::Reader::new(codec::read_document(stored, scratch)?); + + let items = reader.leb128::()?; + + for _ in 0..items { + let term_hash = reader.term()?; + let field_mask = reader.leb128::()?; + cb(term_hash, field_mask); } + + Some(()) } -impl SearchField { - pub(crate) fn u8_id(&self) -> u8 { - match self { - SearchField::AccountId => 0, - SearchField::DocumentId => 1, - SearchField::Id => 2, - SearchField::Email(field) => match field { - EmailSearchField::From => 3, - EmailSearchField::To => 4, - EmailSearchField::Cc => 5, - EmailSearchField::Bcc => 6, - EmailSearchField::Subject => 7, - EmailSearchField::Body => 8, - EmailSearchField::Attachment => 9, - EmailSearchField::ReceivedAt => 10, - EmailSearchField::SentAt => 11, - EmailSearchField::Size => 12, - EmailSearchField::HasAttachment => 13, - EmailSearchField::Headers => 14, - }, - SearchField::Calendar(field) => match field { - CalendarSearchField::Title => 3, - CalendarSearchField::Description => 4, - CalendarSearchField::Location => 5, - CalendarSearchField::Owner => 6, - CalendarSearchField::Attendee => 7, - CalendarSearchField::Start => 8, - CalendarSearchField::Uid => 9, - }, - SearchField::Contact(field) => match field { - ContactSearchField::Member => 3, - ContactSearchField::Kind => 4, - ContactSearchField::Name => 5, - ContactSearchField::Nickname => 6, - ContactSearchField::Organization => 7, - ContactSearchField::Email => 8, - ContactSearchField::Phone => 9, - ContactSearchField::OnlineService => 10, - ContactSearchField::Address => 11, - ContactSearchField::Note => 12, - ContactSearchField::Uid => 13, - }, - SearchField::File(field) => match field { - FileSearchField::Name => 3, - FileSearchField::Content => 4, - }, - SearchField::Tracing(field) => match field { - TracingSearchField::EventType => 3, - TracingSearchField::QueueId => 4, - TracingSearchField::Keywords => 5, - }, - } - } +fn term_key_order(field: u8, term: &CheekyHash, block_id: u16) -> (u8, &[u8], usize, u16) { + (field, term.as_key(), term.len(), block_id) } + +fn account_term_key_order( + index: SearchIndex, + field: u8, + term: &CheekyHash, + block_id: u16, +) -> (u8, &[u8], usize, u8, u16) { + ( + index.term_group(field), + term.as_key(), + term.len(), + field, + block_id, + ) +} + diff --git a/crates/store/src/search/tokenize.rs b/crates/store/src/search/tokenize.rs new file mode 100644 index 00000000..3fb2df64 --- /dev/null +++ b/crates/store/src/search/tokenize.rs @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::backend::MAX_TOKEN_LENGTH; +use nlp::{ + language::{Language, stemmer::Stemmer}, + tokenizers::{space::SpaceTokenizer, word::WordTokenizer}, +}; +use std::borrow::Cow; +use utils::cheeky_hash::CheekyHash; + +pub struct QueryToken<'x> { + pub word: Cow<'x, str>, + pub stem: Option>, +} + +pub fn tokenize<'x>(text: &'x str, language: Language, mut cb: impl FnMut(QueryToken<'x>) -> bool) { + match language { + Language::None => { + for word in SpaceTokenizer::new(text, MAX_TOKEN_LENGTH) { + if !cb(QueryToken { word, stem: None }) { + return; + } + } + } + Language::Unknown => { + for token in WordTokenizer::new(text, MAX_TOKEN_LENGTH) { + if !cb(QueryToken { + word: token.word, + stem: None, + }) { + return; + } + } + } + _ => { + for token in Stemmer::new(text, language, MAX_TOKEN_LENGTH) { + if !cb(QueryToken { + word: token.word, + stem: token.stemmed_word, + }) { + return; + } + } + } + } +} + +pub(crate) fn stem_term(stem: &str, buf: &mut String) -> CheekyHash { + buf.clear(); + buf.push_str(stem); + buf.push('*'); + CheekyHash::new(buf.as_bytes()) +} + +pub(crate) fn key_value_term(key: &str, value: &str, buf: &mut String) -> CheekyHash { + buf.clear(); + buf.push_str(key); + buf.push(' '); + buf.push_str(value); + CheekyHash::new(buf.as_bytes()) +} + +pub(crate) fn integer_term(value: u64) -> CheekyHash { + CheekyHash::new(&value.to_be_bytes()[std::cmp::min(7, value.leading_zeros() as usize / 8)..]) +} + +#[inline(always)] +pub(crate) fn zigzag(value: i64) -> u64 { + (value.wrapping_shl(1) ^ (value >> 63)) as u64 +} + diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 74fa986f..b126389a 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -5,25 +5,35 @@ */ use super::{ - Batch, BatchBuilder, ChangedCollection, IntoOperations, Operation, ValueClass, ValueOp, - assert::ToAssertValue, log::VanishedItem, + AccountCounter, Batch, BatchBuilder, ChangedCollection, CommitPriority, IntoOperations, + Operation, QueueCounter, QueueNotify, ValueClass, ValueOp, assert::ToAssertValue, + counter::CounterKey, log::VanishedItem, }; use crate::{ - SerializeInfallible, U32_LEN, + Deserialize, Shape, Subspace, U32_LEN, + search::GLOBAL_BUCKET_SHIFT, write::{ - LogCollection, MergeFnc, MergeOperation, Params, SetFnc, SetOperation, TaskQueueClass, + AssignedIds, ChangeCounter, CommitPointOffsets, LogCollection, LogSet, MergeOperation, + MergeResult, PendingId, PendingTask, QueueDocumentId, Reservation, ReservationClass, + SearchIndex, SearchIndexClass, SetValue, SizedSetValue, Slot, SlotRange, TaskId, + TaskQueueClass, TaskTypes, }, }; use registry::{ - schema::structs::Task, - types::{EnumImpl, ObjectImpl}, + schema::{enums::TaskType, structs::Task}, + types::ObjectImpl, }; use types::{ - collection::{Collection, SyncCollection, VanishedCollection}, + collection::{ChangeGroup, Collection, SyncCollection, VanishedCollection}, field::FieldType, + id::Id, }; use utils::{map::vec_map::VecMap, snowflake::SnowflakeIdGenerator}; +const MAX_COMMIT_BYTES: usize = 5_000_000; +const MAX_COMMIT_OPS: usize = 1000; +const MAX_BULK_COMMIT_OPS: usize = 5000; + impl BatchBuilder { pub fn new() -> Self { Self { @@ -33,10 +43,26 @@ impl BatchBuilder { current_document_id: None, changes: Default::default(), changed_collections: Default::default(), + change_accounts: Vec::new(), + reservations: Vec::new(), + bounds: Vec::new(), + counters: Default::default(), + next_slot: 0, batch_size: 0, batch_ops: 0, - has_assertions: false, commit_points: Vec::new(), + last_archive_hash: None, + last_index_partition: None, + has_index_tasks: false, + task_types: TaskTypes::default(), + priority: CommitPriority::Default, + } + } + + pub fn with_priority(priority: CommitPriority) -> Self { + Self { + priority, + ..Self::new() } } @@ -61,23 +87,111 @@ impl BatchBuilder { } pub fn with_document(&mut self, document_id: u32) -> &mut Self { + self.with_pending_document(PendingId::Assigned(document_id)) + } + + pub fn create_document(&mut self, slot: Slot) -> &mut Self { + self.with_pending_document(PendingId::Slot(slot)) + } + + pub fn with_pending_document(&mut self, document_id: PendingId) -> &mut Self { self.ops.push(Operation::DocumentId { document_id }); self.current_document_id = Some(document_id); - self.has_assertions = false; self } + pub fn reserve_document_id(&mut self, account_id: u32, collection: Collection) -> Slot { + self.reserve(ReservationClass::DocumentId { + account_id, + collection, + }) + } + + pub fn reserve_document_ids( + &mut self, + account_id: u32, + collection: Collection, + count: u32, + ) -> SlotRange { + SlotRange::new( + self.reserve_many( + ReservationClass::DocumentId { + account_id, + collection, + }, + count, + ), + count, + ) + } + + pub fn reserve_uid(&mut self, account_id: u32, mailbox_id: u32) -> Slot { + self.reserve(ReservationClass::Uid { + account_id, + mailbox_id, + }) + } + + pub fn reserve_uids( + &mut self, + account_id: u32, + mailbox_ids: impl IntoIterator, + ) -> SlotRange { + let first_slot = Slot::new(self.next_slot); + let mut count = 0; + for mailbox_id in mailbox_ids { + self.reserve(ReservationClass::Uid { + account_id, + mailbox_id, + }); + count += 1; + } + + SlotRange::new(first_slot, count) + } + + fn reserve(&mut self, class: ReservationClass) -> Slot { + let next_slot = self.next_slot; + let reservations_start = self.reservations_start(); + if self.reservations.len() > reservations_start + && let Some(last) = self.reservations.last_mut() + && last.class == class + && last.first_slot.index() + last.count as usize == next_slot as usize + { + last.count += 1; + self.next_slot += 1; + return Slot::new(next_slot); + } + + self.reserve_many(class, 1) + } + + fn reserve_many(&mut self, class: ReservationClass, count: u32) -> Slot { + let first_slot = Slot::new(self.next_slot); + self.next_slot = self + .next_slot + .checked_add(count) + .expect("too many reserved ids"); + self.reservations.push(Reservation { + class, + first_slot, + count, + }); + first_slot + } + pub fn assert_value( &mut self, class: impl Into, value: impl ToAssertValue, ) -> &mut Self { + let class = class.into(); + self.settle_counter(&class, true); self.ops.push(Operation::AssertValue { - class: class.into(), + class, assert_value: value.to_assert_value(), }); self.batch_ops += 1; - self.has_assertions = true; self } @@ -123,7 +237,15 @@ impl BatchBuilder { pub fn add(&mut self, class: impl Into, value: i64) -> &mut Self { let class = class.into(); - self.batch_size += class.serialized_size() + std::mem::size_of::(); + if let Some(key) = CounterKey::new(&class, self.current_account_id) { + if self.counters.add(key, value) { + self.batch_size += class.key_len_hint() + std::mem::size_of::(); + self.batch_ops += 1; + } + return self; + } + + self.batch_size += class.key_len_hint() + std::mem::size_of::(); self.ops.push(Operation::Value { class, op: ValueOp::AtomicAdd(value), @@ -132,9 +254,30 @@ impl BatchBuilder { self } + pub fn limit_counter(&mut self, counter: AccountCounter, max: u64) -> &mut Self { + debug_assert!( + self.current_account_id.is_some(), + "a counter bound needs an account" + ); + self.counters.limit( + CounterKey::account(self.current_account_id, counter), + i64::try_from(max).unwrap_or(i64::MAX), + ); + self + } + + pub fn limit_queue_counter(&mut self, counter: QueueCounter, max: u64) -> &mut Self { + self.counters.limit( + CounterKey::Queue(counter), + i64::try_from(max).unwrap_or(i64::MAX), + ); + self + } + pub fn add_and_get(&mut self, class: impl Into, value: i64) -> &mut Self { let class = class.into(); - self.batch_size += class.serialized_size() + (std::mem::size_of::() * 2); + self.settle_counter(&class, true); + self.batch_size += class.key_len_hint() + (std::mem::size_of::() * 2); self.ops.push(Operation::Value { class, op: ValueOp::AddAndGet(value), @@ -143,10 +286,15 @@ impl BatchBuilder { self } - pub fn set(&mut self, class: impl Into, value: impl Into>) -> &mut Self { + pub fn set( + &mut self, + class: impl Into, + value: impl Into, + ) -> &mut Self { let class = class.into(); - let value = value.into(); - self.batch_size += class.serialized_size() + value.len(); + self.settle_counter(&class, false); + let SizedSetValue { value, size_hint } = value.into(); + self.batch_size += class.key_len_hint() + size_hint; self.ops.push(Operation::Value { class, op: ValueOp::Set(value), @@ -155,35 +303,48 @@ impl BatchBuilder { self } - pub fn set_fnc( - &mut self, - class: impl Into, - params: Params, - fnc: SetFnc, - ) -> &mut Self { - self.ops.push(Operation::Value { - class: class.into(), - op: ValueOp::SetFnc(SetOperation { fnc, params }), - }); - self - } - pub fn merge_fnc( &mut self, class: impl Into, - params: Params, - fnc: MergeFnc, + fnc: impl Fn(&AssignedIds, Option<&[u8]>) -> trc::Result + Send + Sync + 'static, ) -> &mut Self { + let class = class.into(); + self.settle_counter(&class, true); + self.batch_size += class.key_len_hint(); self.ops.push(Operation::Value { - class: class.into(), - op: ValueOp::MergeFnc(MergeOperation { fnc, params }), + class, + op: ValueOp::MergeFnc(MergeOperation(Box::new(fnc))), }); + self.batch_ops += 1; + self + } + + pub fn clear_if_equals(&mut self, class: impl Into, value: T) -> &mut Self + where + T: Deserialize + PartialEq + Send + Sync + 'static, + { + self.merge_fnc(class, move |_, bytes| match bytes { + Some(bytes) if T::deserialize(bytes)? == value => Ok(MergeResult::Delete), + _ => Ok(MergeResult::Skip), + }) + } + + pub fn set_archive_hash(&mut self, hash: Option) -> &mut Self { + self.last_archive_hash = hash; self } pub fn clear(&mut self, class: impl Into) -> &mut Self { let class = class.into(); - self.batch_size += class.serialized_size(); + if let Some(key) = CounterKey::new(&class, self.current_account_id) { + if self.counters.reset(key) { + self.batch_size += class.key_len_hint(); + self.batch_ops += 1; + } + return self; + } + self.settle_counter(&class, false); + self.batch_size += class.key_len_hint(); self.ops.push(Operation::Value { class, op: ValueOp::Clear, @@ -196,7 +357,7 @@ impl BatchBuilder { self.batch_size += (U32_LEN * 3) + op.len(); self.ops.push(Operation::Value { class: ValueClass::Acl(grant_account_id), - op: ValueOp::Set(op), + op: ValueOp::Set(SetValue::Fixed(op)), }); self.batch_ops += 1; self @@ -215,7 +376,7 @@ impl BatchBuilder { pub fn log_item_insert( &mut self, collection: SyncCollection, - prefix: Option, + prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) @@ -232,7 +393,7 @@ impl BatchBuilder { pub fn log_item_update( &mut self, collection: SyncCollection, - prefix: Option, + prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) @@ -249,7 +410,7 @@ impl BatchBuilder { pub fn log_item_delete( &mut self, collection: SyncCollection, - prefix: Option, + prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) @@ -299,7 +460,7 @@ impl BatchBuilder { pub fn log_container_property_change( &mut self, collection: SyncCollection, - document_id: u32, + document_id: PendingId, ) -> &mut Self { if let Some(account_id) = self.current_account_id { self.changes @@ -309,6 +470,63 @@ impl BatchBuilder { self } + pub fn log_container_metadata(&mut self, collection: SyncCollection) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_container_metadata(collection, document_id); + } + self + } + + pub fn log_item_metadata( + &mut self, + collection: SyncCollection, + prefix: Option, + ) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_item_metadata(collection, prefix, document_id); + } + self + } + + pub fn log_private_container_metadata( + &mut self, + collection: SyncCollection, + viewer: u32, + ) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_private_container_metadata(collection, viewer, document_id); + } + self + } + + pub fn log_private_item_metadata( + &mut self, + collection: SyncCollection, + viewer: u32, + prefix: Option, + ) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_private_item_metadata(collection, viewer, prefix, document_id); + } + self + } + pub fn log_vanished_item( &mut self, collection: VanishedCollection, @@ -328,7 +546,7 @@ impl BatchBuilder { &mut self, notification_id: u64, notify_account_id: u32, - value: impl SerializeInfallible, + value: impl Into, ) -> &mut Self { self.changed_collections .get_mut_or_insert(notify_account_id) @@ -338,16 +556,34 @@ impl BatchBuilder { notification_id, notify_account_id, }, - value.serialize(), + value, ) } + fn sort_allocations_for_lock_order(&mut self) { + let reservations_start = self.reservations_start(); + self.reservations[reservations_start..].sort_unstable_by_key(|reservation| { + (reservation.class.sort_key(), reservation.first_slot) + }); + + let change_accounts_start = self.change_accounts_start(); + self.change_accounts[change_accounts_start..] + .sort_unstable_by_key(|entry| (entry.account_id, entry.group)); + } + fn serialize_changes(&mut self) { if !self.changes.is_empty() { for (account_id, changelog) in std::mem::take(&mut self.changes) { + if changelog.changes.is_empty() + && changelog.vanished.is_empty() + && changelog.private.is_empty() + { + continue; + } self.with_account_id(account_id); // Serialize changes + let mut scratch = Vec::new(); for (collection, changes) in changelog.changes.into_iter() { let cc = self.changed_collections.get_mut_or_insert(account_id); if changes.has_container_changes() { @@ -356,50 +592,199 @@ impl BatchBuilder { if changes.has_item_changes() { cc.changed_items.insert(collection); } + self.register_change_group(account_id, collection.change_group()); + let set = match changes.serialize(collection.is_prefixed(), &mut scratch) { + Some(bytes) => LogSet::Bytes(bytes), + None => LogSet::Pending(Box::new(changes)), + }; self.ops.push(Operation::Log { collection: LogCollection::Sync(collection), - set: changes.serialize(), + set, }); } // Serialize vanished items for (collection, vanished) in changelog.vanished.into_iter() { + self.register_change_group(account_id, collection.change_group()); self.ops.push(Operation::Log { collection: LogCollection::Vanished(collection), - set: vanished.serialize(), + set: LogSet::Bytes(vanished.serialize(collection.is_named(), &mut scratch)), + }); + } + + for (log, changes) in changelog.private.into_iter() { + self.register_change_group(account_id, log.collection.change_group()); + let set = match changes.serialize(log.collection.is_prefixed(), &mut scratch) { + Some(bytes) => LogSet::Bytes(bytes), + None => LogSet::Pending(Box::new(changes)), + }; + self.ops.push(Operation::Log { + collection: LogCollection::Private { + collection: log.collection, + viewer: log.viewer, + }, + set, }); } } } } + #[inline(always)] + fn change_accounts_start(&self) -> usize { + self.commit_points + .last() + .map_or(0, |commit_point| commit_point.change_accounts) + } + + #[inline(always)] + fn reservations_start(&self) -> usize { + self.commit_points + .last() + .map_or(0, |commit_point| commit_point.reservations) + } + + fn register_change_group(&mut self, account_id: u32, group: ChangeGroup) { + let entry = ChangeCounter { account_id, group }; + if !self.change_accounts[self.change_accounts_start()..].contains(&entry) { + self.change_accounts.push(entry); + } + } + + #[inline] pub fn commit_point(&mut self) -> &mut Self { if self.is_large_batch() { - self.serialize_changes(); - self.commit_points.push(self.ops.len()); - self.batch_ops = 0; - self.batch_size = 0; - if let Some(account_id) = self.current_account_id { - self.ops.push(Operation::AccountId { account_id }); - } - if let Some(collection) = self.current_collection { - self.ops.push(Operation::Collection { collection }); - } + self.add_commit_point() + } else { + self + } + } + + #[inline] + pub fn bulk_commit_point(&mut self) -> &mut Self { + if self.is_large_bulk_batch() { + self.add_commit_point() + } else { + self + } + } + + pub fn add_commit_point(&mut self) -> &mut Self { + self.serialize_changes(); + self.flush_counters(); + self.sort_allocations_for_lock_order(); + self.commit_points.push(CommitPointOffsets { + ops: self.ops.len(), + reservations: self.reservations.len(), + change_accounts: self.change_accounts.len(), + bounds: self.bounds.len(), + }); + self.batch_ops = 0; + self.batch_size = 0; + self.last_index_partition = None; + if let Some(account_id) = self.current_account_id { + self.ops.push(Operation::AccountId { account_id }); + } + if let Some(collection) = self.current_collection { + self.ops.push(Operation::Collection { collection }); } self } + pub fn has_commit_points(&self) -> bool { + !self.commit_points.is_empty() + } + + pub fn commit_point_count(&self) -> usize { + self.commit_points.len() + } + #[inline] pub fn is_large_batch(&self) -> bool { - self.batch_size > 5_000_000 || self.batch_ops > 1000 + self.batch_size > MAX_COMMIT_BYTES || self.batch_ops > MAX_COMMIT_OPS + } + + #[inline] + pub fn is_large_bulk_batch(&self) -> bool { + self.batch_size > MAX_COMMIT_BYTES || self.batch_ops > MAX_BULK_COMMIT_OPS + } + + fn settle_counter(&mut self, class: &ValueClass, keep_pending: bool) { + if !self.counters.is_empty() + && let Some(key) = CounterKey::new(class, self.current_account_id) + { + let (reset, delta) = self.counters.settle(key); + if keep_pending && reset { + self.ops.push(Operation::Value { + class: class.clone(), + op: ValueOp::Clear, + }); + } + if keep_pending && delta != 0 { + self.ops.push(Operation::Value { + class: class.clone(), + op: ValueOp::AtomicAdd(delta), + }); + } + } + } + + fn flush_counters(&mut self) { + if !self.counters.is_empty() { + self.counters + .flush(self.current_account_id, &mut self.ops, &mut self.bounds); + } } pub fn any_op(&mut self, op: Operation) -> &mut Self { + match op { + Operation::Value { + class, + op: ValueOp::AtomicAdd(value), + } => return self.add(class, value), + Operation::Value { + class, + op: ValueOp::Clear, + } if CounterKey::new(&class, self.current_account_id).is_some() => { + return self.clear(class); + } + Operation::Value { ref class, ref op } => { + self.settle_counter(class, !matches!(op, ValueOp::Set(_) | ValueOp::Clear)); + } + Operation::AssertValue { ref class, .. } => self.settle_counter(class, true), + Operation::AccountId { account_id } => self.current_account_id = Some(account_id), + Operation::Collection { collection } => self.current_collection = Some(collection), + Operation::DocumentId { document_id } => self.current_document_id = Some(document_id), + Operation::ClearRange { + subspace, + ref from, + ref to, + } => { + if matches!(subspace.shape(), Shape::Counter) && !self.counters.is_empty() { + self.counters.clear_range(subspace, from, to); + } + self.batch_size += from.len() + to.len(); + } + Operation::Index { .. } | Operation::Log { .. } => {} + } + if let Operation::Value { class, op } = &op { - self.batch_size += class.serialized_size(); + self.batch_size += class.key_len_hint(); if let ValueOp::Set(value) = op { self.batch_size += value.len(); + + match class { + ValueClass::TaskQueue(TaskQueueClass::Due { typ, due, .. }) + if *due != u64::MAX => + { + self.task_types.insert(*typ); + } + ValueClass::SearchIndex(SearchIndexClass::Queue { .. }) => { + self.has_index_tasks = true; + } + _ => {} + } } } @@ -408,6 +793,10 @@ impl BatchBuilder { self } + pub fn clear_range(&mut self, subspace: Subspace, from: Vec, to: Vec) -> &mut Self { + self.any_op(Operation::ClearRange { subspace, from, to }) + } + pub fn custom(&mut self, value: impl IntoOperations) -> trc::Result<&mut Self> { value.build(self)?; Ok(self) @@ -421,31 +810,46 @@ impl BatchBuilder { self.current_collection } - pub fn last_document_id(&self) -> Option { + pub fn last_document_id(&self) -> Option { self.current_document_id } + pub fn last_archive_hash(&self) -> Option { + self.last_archive_hash + } + pub fn commit_points(&mut self) -> CommitPointIterator { self.serialize_changes(); + self.flush_counters(); + self.sort_allocations_for_lock_order(); CommitPointIterator { commit_points: std::mem::take(&mut self.commit_points), - commit_point_last: self.ops.len(), - offset_start: 0, + commit_point_last: CommitPointOffsets { + ops: self.ops.len(), + reservations: self.reservations.len(), + change_accounts: self.change_accounts.len(), + bounds: self.bounds.len(), + }, + offset_start: CommitPointOffsets::default(), } } pub fn build_one(&mut self, commit_point: CommitPoint) -> Batch<'_> { Batch { - changes: &self.changed_collections, + change_accounts: &self.change_accounts + [commit_point.change_account_start..commit_point.change_account_end], + reservations: &self.reservations + [commit_point.reservation_start..commit_point.reservation_end], + bounds: &self.bounds[commit_point.bound_start..commit_point.bound_end], ops: &mut self.ops[commit_point.offset_start..commit_point.offset_end], + priority: self.priority, } } - pub fn build_all(&mut self) -> Batch<'_> { - self.serialize_changes(); - Batch { - changes: &self.changed_collections, - ops: self.ops.as_mut_slice(), + pub fn queue_notify(&self) -> QueueNotify { + QueueNotify { + task_types: self.task_types, + search_index: self.has_index_tasks, } } @@ -470,44 +874,185 @@ impl BatchBuilder { } pub fn is_empty(&self) -> bool { - self.batch_ops == 0 + self.batch_ops == 0 && self.commit_points.is_empty() && self.changes.is_empty() } pub fn schedule_task(&mut self, task: Task) -> &mut Self { - let due = task.due_timestamp(); - let class = task.object_type().to_id(); - let task = task.to_pickled_vec(); - let id = SnowflakeIdGenerator::global_id().unwrap_or_default(); - - self.set(ValueClass::TaskQueue(TaskQueueClass::Task { id }), task) - .set( - ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), - class.serialize(), - ) + match task.singleton_key() { + Some((discriminant, key)) => self.schedule_singleton_task(discriminant, key, task), + None => self + .schedule_task_with_id(SnowflakeIdGenerator::global_id().unwrap_or_default(), task), + } } pub fn schedule_task_with_id(&mut self, id: u64, task: Task) -> &mut Self { - let due = task.due_timestamp(); - let class = task.object_type().to_id(); - let task = task.to_pickled_vec(); + self.push_task(TaskId::Assigned(id), task, None) + } - self.set(ValueClass::TaskQueue(TaskQueueClass::Task { id }), task) + pub fn schedule_document_task(&mut self, id: TaskId, task: Task) -> trc::Result<&mut Self> { + let document_id = self.document_id_for_task()?; + + Ok(self.push_task(id, task, Some(document_id))) + } + + pub fn schedule_task_with_document(&mut self, task: Task) -> trc::Result<&mut Self> { + let document_id = self.document_id_for_task()?; + let id = TaskId::Assigned(SnowflakeIdGenerator::global_id().unwrap_or_default()); + + Ok(self.push_task(id, task, Some(document_id))) + } + + fn document_id_for_task(&self) -> trc::Result { + self.current_document_id.ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No document is set for a document task.") + }) + } + + pub fn schedule_singleton_task( + &mut self, + discriminant: u16, + key: u64, + task: Task, + ) -> &mut Self { + let typ = task.object_type(); + let id = TaskId::Assigned(TaskId::singleton(typ, discriminant, key)); + self.clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + typ, + due: u64::MAX, + })) + .push_task(id, task, None) + } + + fn push_task( + &mut self, + id: TaskId, + mut task: Task, + document_id: Option, + ) -> &mut Self { + let due = task.due_timestamp(); + let typ = task.object_type(); + let marker = TaskQueueClass::due_marker(task.status()); + + let value = match document_id { + Some(PendingId::Slot(slot)) => SetValue::serializable(PendingTask { task, slot }), + Some(PendingId::Assigned(document_id)) => { + task.set_document_id(Id::from(document_id)); + SetValue::fixed(task.to_pickled_vec()) + } + None => SetValue::fixed(task.to_pickled_vec()), + }; + + self.set(ValueClass::TaskQueue(TaskQueueClass::Task { id }), value) + .set_task_due_marked(id, typ, due, marker) + } + + pub fn set_task_due(&mut self, id: TaskId, typ: TaskType, due: u64) -> &mut Self { + self.set_task_due_marked(id, typ, due, None) + } + + pub fn set_task_due_marked( + &mut self, + id: TaskId, + typ: TaskType, + due: u64, + marker: Option, + ) -> &mut Self { + if due != u64::MAX { + self.task_types.insert(typ); + } + self.set( + ValueClass::TaskQueue(TaskQueueClass::Due { id, typ, due }), + marker.map_or_else(Vec::new, |marker| vec![marker]), + ) + } + + pub fn clear_document_task(&mut self, id: TaskId, typ: TaskType, due: u64) -> &mut Self { + self.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { id, typ, due })) + } + + pub fn queue_document_index( + &mut self, + index: SearchIndex, + account_id: u32, + document_id: impl Into, + ) -> &mut Self { + let document_id = document_id.into(); + self.queue_index_task(index, account_id).set( + ValueClass::SearchIndex(SearchIndexClass::Queue { + index, + id_prefix: account_id, + id_suffix: document_id, + created_at: SnowflakeIdGenerator::global_id_with_sequence_id(0).unwrap_or_default(), + }), + vec![1u8], + ) + } + + pub fn queue_document_unindex( + &mut self, + index: SearchIndex, + account_id: u32, + document_id: impl Into, + ) -> &mut Self { + let document_id = document_id.into(); + self.queue_index_task(index, account_id).set( + ValueClass::SearchIndex(SearchIndexClass::Queue { + index, + id_prefix: account_id, + id_suffix: document_id, + created_at: SnowflakeIdGenerator::global_id_with_sequence_id(0).unwrap_or_default(), + }), + vec![0u8], + ) + } + + pub fn queue_trace_index(&mut self, id: u64) -> &mut Self { + self.queue_index_task(SearchIndex::Tracing, (id >> GLOBAL_BUCKET_SHIFT) as u32) .set( - ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), - class.serialize(), + ValueClass::SearchIndex(SearchIndexClass::Queue { + index: SearchIndex::Tracing, + id_prefix: (id >> 32) as u32, + id_suffix: QueueDocumentId::Assigned(id as u32), + created_at: SnowflakeIdGenerator::global_id_with_sequence_id(0) + .unwrap_or_default(), + }), + vec![1u8], ) } + + fn queue_index_task(&mut self, index: SearchIndex, partition: u32) -> &mut Self { + self.has_index_tasks = true; + if self.last_index_partition != Some((index, partition)) { + self.last_index_partition = Some((index, partition)); + self.set( + ValueClass::SearchIndex(SearchIndexClass::QueueIndex { index, partition }), + vec![], + ); + } + + self + } } pub struct CommitPointIterator { - commit_points: Vec, - commit_point_last: usize, - offset_start: usize, + commit_points: Vec, + commit_point_last: CommitPointOffsets, + offset_start: CommitPointOffsets, } pub struct CommitPoint { pub offset_start: usize, pub offset_end: usize, + pub reservation_start: usize, + pub reservation_end: usize, + pub change_account_start: usize, + pub change_account_end: usize, + pub bound_start: usize, + pub bound_end: usize, } impl CommitPointIterator { @@ -516,41 +1061,26 @@ impl CommitPointIterator { .iter() .copied() .chain([self.commit_point_last]) - .map(|offset_end| { + .map(|end| { let point = CommitPoint { - offset_start: self.offset_start, - offset_end, + offset_start: self.offset_start.ops, + offset_end: end.ops, + reservation_start: self.offset_start.reservations, + reservation_end: end.reservations, + change_account_start: self.offset_start.change_accounts, + change_account_end: end.change_accounts, + bound_start: self.offset_start.bounds, + bound_end: end.bounds, }; - self.offset_start = offset_end; + self.offset_start = end; point }) } } -impl Batch<'_> { - pub fn is_atomic(&self) -> bool { - !self.ops.iter().any(|op| { - matches!( - op, - Operation::AssertValue { .. } - | Operation::Value { - op: ValueOp::AddAndGet(_), - .. - } - ) - }) - } - - pub fn first_account_id(&self) -> Option { - self.ops.iter().find_map(|op| match op { - Operation::AccountId { account_id } => Some(*account_id), - _ => None, - }) - } -} - impl Default for BatchBuilder { fn default() -> Self { Self::new() } } + diff --git a/crates/store/src/write/bitpack.rs b/crates/store/src/write/bitpack.rs index 92695edf..5736399a 100644 --- a/crates/store/src/write/bitpack.rs +++ b/crates/store/src/write/bitpack.rs @@ -5,19 +5,34 @@ */ use bitpacking::{BitPacker, BitPacker1x, BitPacker4x, BitPacker8x}; -use utils::codec::leb128::Leb128Reader; +use utils::codec::leb128::Leb128; use super::key::KeySerializer; -#[derive(Default)] +const MAX_BLOCK_LEN: usize = BitPacker8x::BLOCK_LEN; + pub struct BitpackIterator<'x> { pub(crate) bytes: &'x [u8], pub(crate) bytes_offset: usize, - pub(crate) chunk: Vec, + pub(crate) chunk: [u32; MAX_BLOCK_LEN], + pub(crate) chunk_len: usize, pub(crate) chunk_offset: usize, pub items_left: u32, } +impl Default for BitpackIterator<'_> { + fn default() -> Self { + BitpackIterator { + bytes: &[], + bytes_offset: 0, + chunk: [0u32; MAX_BLOCK_LEN], + chunk_len: 0, + chunk_offset: 0, + items_left: 0, + } + } +} + #[derive(Clone, Copy)] pub(crate) struct BitBlockPacker { bitpacker_1: BitPacker1x, @@ -27,8 +42,7 @@ pub(crate) struct BitBlockPacker { } impl KeySerializer { - pub fn bitpack_sorted(self, items: &[u32]) -> Self { - let mut serializer = self; + pub fn bitpack_sorted(&mut self, items: &[u32]) -> &mut Self { let mut bitpacker = BitBlockPacker::new(); let mut compressed = vec![0u8; 4 * BitPacker8x::BLOCK_LEN]; @@ -36,13 +50,13 @@ impl KeySerializer { let len = items.len(); let mut initial_value = None; - serializer = serializer.write_leb128(len as u32); + self.write_leb128(len as u32); while pos < len { let block_len = match len - pos { 0..=31 => { for val in &items[pos..] { - serializer = serializer.write_leb128(*val); + self.write_leb128(*val); } break; } @@ -60,14 +74,12 @@ impl KeySerializer { &mut compressed[..], num_bits, ); - serializer = serializer - .write(num_bits) - .write(&compressed[..compressed_len]); + self.write(num_bits).write(&compressed[..compressed_len]); initial_value = chunk[chunk.len() - 1].into(); pos += block_len; } - serializer + self } } @@ -82,14 +94,12 @@ impl<'x> BitpackIterator<'x> { } pub fn new(bytes: &'x [u8]) -> Option { - bytes - .read_leb128::() - .map(|(items_left, bytes_offset)| BitpackIterator { - bytes, - bytes_offset, - items_left, - ..Default::default() - }) + u32::from_leb128(bytes).map(|(items_left, bytes_offset)| BitpackIterator { + bytes, + bytes_offset, + items_left, + ..Default::default() + }) } } @@ -97,15 +107,16 @@ impl Iterator for BitpackIterator<'_> { type Item = u32; fn next(&mut self) -> Option { - if let Some(item) = self.chunk.get(self.chunk_offset) { + if self.chunk_offset < self.chunk_len { + let item = self.chunk[self.chunk_offset]; self.chunk_offset += 1; - return Some(*item); + return Some(item); } let block_len = match self.items_left { 0 => return None, 1..=31 => { self.items_left -= 1; - let (item, bytes_read) = self.bytes.get(self.bytes_offset..)?.read_leb128()?; + let (item, bytes_read) = u32::from_leb128(self.bytes.get(self.bytes_offset..)?)?; self.bytes_offset += bytes_read; return Some(item); } @@ -117,22 +128,23 @@ impl Iterator for BitpackIterator<'_> { let bitpacker = BitBlockPacker::with_block_len(block_len); let num_bits = *self.bytes.get(self.bytes_offset)?; let bytes_read = ((num_bits as usize) * block_len / 8) + 1; - let initial_value = self.chunk.last().copied(); - - self.chunk = vec![0u32; block_len]; - self.chunk_offset = 1; + let initial_value = self.chunk_len.checked_sub(1).map(|last| self.chunk[last]); + let compressed = self + .bytes + .get(self.bytes_offset + 1..self.bytes_offset + bytes_read)?; bitpacker.decompress_strictly_sorted( initial_value, - self.bytes - .get(self.bytes_offset + 1..self.bytes_offset + bytes_read)?, - &mut self.chunk[..], + compressed, + &mut self.chunk[..block_len], num_bits, ); + self.chunk_len = block_len; + self.chunk_offset = 1; self.bytes_offset += bytes_read; self.items_left -= block_len as u32; - self.chunk.first().copied() + Some(self.chunk[0]) } } @@ -326,57 +338,3 @@ impl BitPacker for BitBlockPacker { } } -#[cfg(test)] -mod tests { - - use super::*; - - #[test] - fn bitpack_roundtrip() { - for num_positions in [ - 1, - 10, - BitPacker1x::BLOCK_LEN, - BitPacker4x::BLOCK_LEN, - BitPacker8x::BLOCK_LEN, - BitPacker8x::BLOCK_LEN + BitPacker4x::BLOCK_LEN + BitPacker1x::BLOCK_LEN, - BitPacker8x::BLOCK_LEN + BitPacker4x::BLOCK_LEN + BitPacker1x::BLOCK_LEN + 1, - (BitPacker8x::BLOCK_LEN * 3) - + (BitPacker4x::BLOCK_LEN * 3) - + (BitPacker1x::BLOCK_LEN * 3) - + 1, - (BitPacker8x::BLOCK_LEN * 32) + 1, - ] { - let serialized = KeySerializer::new(num_positions * std::mem::size_of::()) - .bitpack_sorted( - &(0..num_positions) - .map(|i| (i * i) as u32) - .collect::>(), - ) - .finalize(); - - println!( - "Testing block {num_positions} with {} size...", - serialized.len() - ); - - let mut iter = BitpackIterator::new(&serialized).unwrap(); - - assert_eq!( - iter.items_left, num_positions as u32, - "failed for num_positions: {}", - num_positions - ); - - for i in 0..num_positions { - assert_eq!( - iter.next(), - Some((i * i) as u32), - "failed for position: {}", - i - ); - } - assert_eq!(iter.next(), None, "expected end of iterator"); - } - } -} diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index fd64d190..e0b07414 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -8,7 +8,7 @@ use super::{BlobOp, Operation, ValueClass, ValueOp, key::DeserializeBigEndian, n use crate::{ BlobStore, Deserialize, IterateParams, SerializeInfallible, Store, U16_LEN, U32_LEN, U64_LEN, ValueKey, - write::{BatchBuilder, BlobLink, RegistryClass}, + write::{BatchBuilder, BlobLink, RegistryClass, SetValue, SizedSetValue}, }; use registry::{ schema::prelude::Property, @@ -20,6 +20,17 @@ use types::{ blob::BlobClass, blob_hash::{BLOB_HASH_LEN, BlobHash}, }; +use utils::codec::leb128::{Leb128, Leb128Reader, Leb128Writer}; + +const OBJECT_ID_LEN: usize = U16_LEN + U64_LEN; +const MAX_LEB128_U64_LEN: usize = 10; +const BLOB_DELETE_CHUNK: usize = 1_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobLinkValue { + pub size: u64, + pub object_id: Option, +} #[derive(Debug, PartialEq, Eq)] pub struct BlobQuota { @@ -27,6 +38,57 @@ pub struct BlobQuota { pub count: usize, } +impl BlobLinkValue { + pub fn new(size: impl TryInto) -> Self { + BlobLinkValue { + size: size.try_into().unwrap_or(u64::MAX), + object_id: None, + } + } + + pub fn with_object_id(mut self, object_id: ObjectId) -> Self { + self.object_id = Some(object_id); + self + } +} + +impl SerializeInfallible for BlobLinkValue { + fn serialize(&self) -> Vec { + let mut bytes = Vec::with_capacity( + self.size.leb128_len() + self.object_id.map_or(0, |_| OBJECT_ID_LEN), + ); + bytes.push_leb128(self.size); + if let Some(object_id) = &self.object_id { + bytes.extend_from_slice(&object_id.serialize()); + } + bytes + } +} + +impl Deserialize for BlobLinkValue { + fn deserialize(bytes: &[u8]) -> trc::Result { + let mut remaining = bytes; + let size = remaining.read_leb128::(); + match (size, remaining.len()) { + (Some(size), 0) => Ok(BlobLinkValue::new(size)), + (Some(size), OBJECT_ID_LEN) => { + Ok(BlobLinkValue::new(size).with_object_id(ObjectId::deserialize(remaining)?)) + } + _ => Err(trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid blob link value") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!())), + } + } +} + +impl From for SizedSetValue { + fn from(value: BlobLinkValue) -> Self { + SetValue::fixed(value.serialize()) + } +} + impl Store { pub async fn blob_exists(&self, hash: impl AsRef + Sync + Send) -> trc::Result { self.key_exists(ValueKey { @@ -41,11 +103,11 @@ impl Store { .caused_by(trc::location!()) } - pub async fn blob_has_access( + pub async fn blob_link_size( &self, hash: impl AsRef + Sync + Send, class: impl AsRef + Sync + Send, - ) -> trc::Result { + ) -> trc::Result> { let key = match class.as_ref() { BlobClass::Reserved { account_id, @@ -72,10 +134,12 @@ impl Store { to: BlobLink::Document, }), }, - _ => return Ok(false), + _ => return Ok(None), }; - self.key_exists(key).await + self.get_value::(key) + .await + .map(|value| value.map(|value| value.size)) } pub async fn purge_blobs_all_shards(&self, blob_store: BlobStore) -> trc::Result<()> { @@ -133,20 +197,31 @@ impl Store { state.finalize(BlobHash::default()); // Delete expired or unlinked blobs + let mut hashes = Vec::with_capacity(BLOB_DELETE_CHUNK.min(state.delete_keys.len())); for (_, op) in &state.delete_keys { if let BlobOp::Commit { hash } = op { - blob_store - .delete_blob(hash.as_ref()) - .await - .caused_by(trc::location!())?; + hashes.push(hash.as_ref()); + if hashes.len() == BLOB_DELETE_CHUNK { + blob_store + .delete_blobs(&hashes) + .await + .caused_by(trc::location!())?; + hashes.clear(); + } } } + if !hashes.is_empty() { + blob_store + .delete_blobs(&hashes) + .await + .caused_by(trc::location!())?; + } // Delete hashes let mut batch = BatchBuilder::new(); for (account_id, op) in state.delete_keys { - if batch.is_large_batch() { - self.write(batch.build_all()) + if batch.is_large_bulk_batch() { + self.write_batch(&mut batch) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -162,8 +237,8 @@ impl Store { }); } for (account_id, object_id) in state.delete_registry { - if batch.is_large_batch() { - self.write(batch.build_all()) + if batch.is_large_bulk_batch() { + self.write_batch(&mut batch) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -185,7 +260,7 @@ impl Store { })); } if !batch.is_empty() { - self.write(batch.build_all()) + self.write_batch(&mut batch) .await .caused_by(trc::location!())?; } @@ -195,9 +270,9 @@ impl Store { trc::event!( Store(StoreEvent::BlobStorePurged), - Id = shard_index as u16, - Expires = total_deleted, - Total = total_active, + Partition = shard_index as u16, + Total = total_deleted, + Used = total_active, Elapsed = started.elapsed() ); @@ -272,9 +347,11 @@ impl BlobPurgeState { to: BlobLink::Temporary { until }, }, )); - if value.len() == U16_LEN + U64_LEN { + if value.len() > MAX_LEB128_U64_LEN + && let Some((_, object_id)) = value.split_last_chunk::() + { self.delete_registry - .push((account_id, ObjectId::deserialize(value)?)); + .push((account_id, ObjectId::deserialize(object_id)?)); } } else { self.last_hash_is_linked = true; @@ -294,3 +371,4 @@ impl BlobPurgeState { } } } + diff --git a/crates/store/src/write/compress.rs b/crates/store/src/write/compress.rs new file mode 100644 index 00000000..7185b655 --- /dev/null +++ b/crates/store/src/write/compress.rs @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{cell::RefCell, sync::LazyLock}; +use zstd::{ + bulk::{Compressor, Decompressor}, + dict::{DecoderDictionary, EncoderDictionary}, + zstd_safe::{ + CParameter, compress_bound, get_dict_id_from_dict, get_dict_id_from_frame, + get_frame_content_size, + }, +}; + +pub const COMPRESSION_LEVEL: i32 = 3; +pub const TERM_COMPRESSION_LEVEL: i32 = 6; +pub const COMPRESS_WATERMARK: usize = 256; +pub const COMPRESS_WATERMARK_DICTIONARY: usize = 64; +pub const MAX_ARCHIVE_SIZE: usize = 64 * 1024 * 1024; + +const DICTIONARIES: [&[u8]; 6] = [ + include_bytes!("../../../../resources/zstd/common-v1.dict"), + include_bytes!("../../../../resources/zstd/email-v1.dict"), + include_bytes!("../../../../resources/zstd/calendar-v1.dict"), + include_bytes!("../../../../resources/zstd/contact-v1.dict"), + include_bytes!("../../../../resources/zstd/sieve-v1.dict"), + include_bytes!("../../../../resources/zstd/term-v1.dict"), +]; + +const ALL_DICTIONARIES: [Dictionary; DICTIONARIES.len()] = [ + Dictionary::Common, + Dictionary::Email, + Dictionary::Calendar, + Dictionary::Contact, + Dictionary::Sieve, + Dictionary::Term, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dictionary { + Common, + Email, + Calendar, + Contact, + Sieve, + Term, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compression { + None, + Zstd(Option), +} + +pub trait ArchiveCompression { + const COMPRESSION: Compression = Compression::Zstd(None); +} + +impl ArchiveCompression for () { + const COMPRESSION: Compression = Compression::None; +} + +impl Dictionary { + #[inline(always)] + fn index(self) -> usize { + match self { + Dictionary::Common => 0, + Dictionary::Email => 1, + Dictionary::Calendar => 2, + Dictionary::Contact => 3, + Dictionary::Sieve => 4, + Dictionary::Term => 5, + } + } + + #[inline(always)] + const fn level(self) -> i32 { + match self { + Dictionary::Term => TERM_COMPRESSION_LEVEL, + _ => COMPRESSION_LEVEL, + } + } +} + +#[inline(always)] +pub const fn compress_watermark(dictionary: Option) -> usize { + if dictionary.is_some() { + COMPRESS_WATERMARK_DICTIONARY + } else { + COMPRESS_WATERMARK + } +} + +static ENCODER_DICTIONARIES: LazyLock>> = LazyLock::new(|| { + DICTIONARIES + .iter() + .zip(ALL_DICTIONARIES) + .map(|(dict, dictionary)| EncoderDictionary::copy(dict, dictionary.level())) + .collect() +}); + +static DECODER_DICTIONARIES: LazyLock)>> = + LazyLock::new(|| { + let mut dictionaries: Vec<(u32, DecoderDictionary<'static>)> = + Vec::with_capacity(DICTIONARIES.len()); + for dict in DICTIONARIES { + let id = get_dict_id_from_dict(dict) + .expect("shipped compression dictionary carries no identifier") + .get(); + if !dictionaries.iter().any(|(known, _)| *known == id) { + dictionaries.push((id, DecoderDictionary::copy(dict))); + } + } + dictionaries + }); + +pub fn init() { + LazyLock::force(&ENCODER_DICTIONARIES); + LazyLock::force(&DECODER_DICTIONARIES); +} + +const CODEC_SLOTS: usize = DICTIONARIES.len() + 1; + +thread_local! { + static COMPRESSORS: RefCell<[Option>; CODEC_SLOTS]> = + const { RefCell::new([const { None }; CODEC_SLOTS]) }; + static DECOMPRESSORS: RefCell<[Option>; CODEC_SLOTS]> = + const { RefCell::new([const { None }; CODEC_SLOTS]) }; +} + +fn new_compressor(dictionary: Option) -> std::io::Result> { + let mut compressor = match dictionary { + Some(dictionary) => { + Compressor::with_prepared_dictionary(&ENCODER_DICTIONARIES[dictionary.index()])? + } + None => Compressor::new(COMPRESSION_LEVEL)?, + }; + compressor.set_parameter(CParameter::ChecksumFlag(true))?; + compressor.set_parameter(CParameter::ContentSizeFlag(true))?; + Ok(compressor) +} + +pub fn compress( + dictionary: Option, + input: &[u8], + trailer_len: usize, +) -> std::io::Result> { + let slot = dictionary.map_or(0, |dictionary| dictionary.index() + 1); + let mut output = Vec::with_capacity(compress_bound(input.len())); + + COMPRESSORS.with_borrow_mut(|compressors| { + let compressor = match &mut compressors[slot] { + Some(compressor) => compressor, + empty => empty.insert(new_compressor(dictionary)?), + }; + compressor.compress_to_buffer(input, &mut output) + })?; + output.reserve(trailer_len); + + Ok(output) +} + +fn new_decompressor(slot: usize) -> std::io::Result> { + match slot.checked_sub(1) { + None => Decompressor::new(), + Some(index) => match DECODER_DICTIONARIES.get(index) { + Some((_, dictionary)) => Decompressor::with_prepared_dictionary(dictionary), + None => Err(std::io::Error::other(format!( + "unknown decompression slot {slot}" + ))), + }, + } +} + +fn with_decompressor( + slot: usize, + f: impl FnOnce(&mut Decompressor<'static>) -> std::io::Result, +) -> std::io::Result { + DECOMPRESSORS.with_borrow_mut(|decompressors| { + let decompressor = match &mut decompressors[slot] { + Some(decompressor) => decompressor, + empty => empty.insert(new_decompressor(slot)?), + }; + f(decompressor) + }) +} + +fn decoder_slot(frame: &[u8]) -> std::io::Result { + match get_dict_id_from_frame(frame).map(|id| id.get()) { + Some(id) => DECODER_DICTIONARIES + .iter() + .position(|(known, _)| *known == id) + .map(|slot| slot + 1) + .ok_or_else(|| std::io::Error::other(format!("unknown compression dictionary {id}"))), + None => Ok(0), + } +} + +fn frame_capacity(frame: &[u8], limit: usize) -> std::io::Result { + match get_frame_content_size(frame) { + Ok(Some(size)) if size <= limit as u64 => Ok(size as usize), + Ok(Some(size)) => Err(std::io::Error::other(format!( + "archive of {size} bytes exceeds the {limit} byte limit" + ))), + Ok(None) => Err(std::io::Error::other( + "compression frame declares no content size", + )), + Err(err) => Err(std::io::Error::other(format!( + "malformed compression frame header: {err}" + ))), + } +} + +pub fn decompress(frame: &[u8]) -> std::io::Result> { + let slot = decoder_slot(frame)?; + let capacity = frame_capacity(frame, MAX_ARCHIVE_SIZE)?; + + with_decompressor(slot, |decompressor| { + decompressor.decompress(frame, capacity) + }) +} + +pub fn decompress_into(frame: &[u8], output: &mut Vec, limit: usize) -> std::io::Result<()> { + let slot = decoder_slot(frame)?; + let capacity = frame_capacity(frame, limit)?; + + output.clear(); + output.reserve(capacity); + with_decompressor(slot, |decompressor| { + decompressor.decompress_to_buffer(frame, output) + }) + .map(|_| ()) +} diff --git a/crates/store/src/write/counter.rs b/crates/store/src/write/counter.rs new file mode 100644 index 00000000..2fc280f7 --- /dev/null +++ b/crates/store/src/write/counter.rs @@ -0,0 +1,353 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Operation, QueueClass, ValueClass, ValueOp, key::CounterKind}; +use crate::{Subspace, U16_LEN, U32_LEN}; +use ahash::AHashMap; +use registry::{schema::enums::StorageQuota, types::EnumImpl}; +use std::mem; +use utils::hash128::Hash128; + +const LINEAR_LOOKUP_MAX: usize = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AccountCounter { + Quota, + ObjectCount(StorageQuota), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum QueueCounter { + Count(Hash128), + Size(Hash128), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CounterBound { + pub(crate) key: CounterKey, + pub(crate) by: i64, + pub(crate) max: i64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum CounterKey { + Account { + account_id: u32, + counter: AccountCounter, + }, + Queue(QueueCounter), +} + +#[derive(Debug)] +struct PendingCounter { + key: CounterKey, + delta: i64, + max: Option, + reset: bool, +} + +#[derive(Debug, Default)] +pub(super) struct PendingCounters { + counters: Vec, + index: Option>, +} + +impl CounterKey { + #[inline] + pub(super) fn new(class: &ValueClass, account_id: Option) -> Option { + let counter = match class { + ValueClass::Quota => AccountCounter::Quota, + ValueClass::ObjectCount(object) => AccountCounter::ObjectCount(*object), + ValueClass::Queue(QueueClass::QuotaCount(key)) => { + return Some(CounterKey::Queue(QueueCounter::Count(*key))); + } + ValueClass::Queue(QueueClass::QuotaSize(key)) => { + return Some(CounterKey::Queue(QueueCounter::Size(*key))); + } + _ => return None, + }; + Some(CounterKey::account(account_id, counter)) + } + + #[inline] + pub(super) fn account(account_id: Option, counter: AccountCounter) -> Self { + CounterKey::Account { + account_id: account_id.unwrap_or(u32::MAX), + counter, + } + } + + fn class_and_account(self) -> (ValueClass, u32) { + match self { + CounterKey::Account { + account_id, + counter, + } => (counter.into(), account_id), + CounterKey::Queue(counter) => (counter.into(), 0), + } + } + + fn class(self) -> ValueClass { + self.class_and_account().0 + } + + fn sort_key(self) -> (u8, u32, u8, Hash128) { + match self { + CounterKey::Account { + account_id, + counter: AccountCounter::Quota, + } => (CounterKind::Quota.byte(), account_id, 0, Hash128::default()), + CounterKey::Account { + account_id, + counter: AccountCounter::ObjectCount(object), + } => ( + CounterKind::ObjectCount.byte(), + account_id, + object as u8, + Hash128::default(), + ), + CounterKey::Queue(QueueCounter::Count(key)) => { + (CounterKind::QueueQuotaCount.byte(), 0, 0, key) + } + CounterKey::Queue(QueueCounter::Size(key)) => { + (CounterKind::QueueQuotaSize.byte(), 0, 0, key) + } + } + } +} + +impl PendingCounters { + #[inline] + pub(super) fn is_empty(&self) -> bool { + self.counters.is_empty() + } + + pub(super) fn add(&mut self, key: CounterKey, by: i64) -> bool { + let counter = self.entry(key); + let was_idle = counter.delta == 0; + counter.delta = counter.delta.saturating_add(by); + was_idle + } + + pub(super) fn limit(&mut self, key: CounterKey, max: i64) { + let counter = self.entry(key); + counter.max = Some(counter.max.map_or(max, |current| current.min(max))); + } + + pub(super) fn reset(&mut self, key: CounterKey) -> bool { + let counter = self.entry(key); + counter.delta = 0; + counter.max = None; + !mem::replace(&mut counter.reset, true) + } + + pub(super) fn settle(&mut self, key: CounterKey) -> (bool, i64) { + match self + .position(key) + .and_then(|pos| self.counters.get_mut(pos)) + { + Some(counter) => { + counter.max = None; + (mem::take(&mut counter.reset), mem::take(&mut counter.delta)) + } + None => (false, 0), + } + } + + pub(super) fn clear_range(&mut self, subspace: Subspace, from: &[u8], to: &[u8]) { + let range = from..to; + let mut key = Vec::with_capacity(U32_LEN + U16_LEN); + for counter in &mut self.counters { + let (class, account_id) = counter.key.class_and_account(); + if class.subspace(0) == subspace { + key.clear(); + class.serialize_into(&mut key, account_id, 0, 0, 0); + if range.contains(&key.as_slice()) { + counter.delta = 0; + counter.max = None; + } + } + } + } + + pub(super) fn flush( + &mut self, + mut account_id: Option, + ops: &mut Vec, + bounds: &mut Vec, + ) { + self.counters + .sort_unstable_by_key(|counter| counter.key.sort_key()); + for pending in &mut self.counters { + let delta = mem::take(&mut pending.delta); + let reset = mem::take(&mut pending.reset); + match (pending.key, pending.max) { + _ if delta == 0 && !reset => {} + (key @ (CounterKey::Account { .. } | CounterKey::Queue(_)), Some(max)) + if delta > 0 && !reset => + { + bounds.push(CounterBound { + key, + by: delta, + max, + }) + } + (key, _) => { + if let CounterKey::Account { + account_id: counter_account_id, + .. + } = key + && account_id != Some(counter_account_id) + { + ops.push(Operation::AccountId { + account_id: counter_account_id, + }); + account_id = Some(counter_account_id); + } + if reset { + ops.push(Operation::Value { + class: key.class(), + op: ValueOp::Clear, + }); + } + if delta != 0 { + ops.push(Operation::Value { + class: key.class(), + op: ValueOp::AtomicAdd(delta), + }); + } + } + } + } + self.counters.retain(|counter| counter.max.is_some()); + self.index = (self.counters.len() > LINEAR_LOOKUP_MAX).then(|| self.build_index()); + } + + fn position(&self, key: CounterKey) -> Option { + match &self.index { + Some(index) => index.get(&key).copied(), + None => self.counters.iter().rposition(|counter| counter.key == key), + } + } + + fn entry(&mut self, key: CounterKey) -> &mut PendingCounter { + if let Some(pos) = self.position(key) { + return &mut self.counters[pos]; + } + + let pos = self.counters.len(); + self.counters.push(PendingCounter { + key, + delta: 0, + max: None, + reset: false, + }); + match &mut self.index { + Some(index) => { + index.insert(key, pos); + } + None if self.counters.len() > LINEAR_LOOKUP_MAX => { + self.index = Some(self.build_index()); + } + None => {} + } + &mut self.counters[pos] + } + + fn build_index(&self) -> AHashMap { + self.counters + .iter() + .enumerate() + .map(|(pos, counter)| (counter.key, pos)) + .collect() + } +} + +impl CounterBound { + #[inline] + pub fn serialize_key_into(&self, buf: &mut Vec, flags: u32) { + let (class, account_id) = self.key.class_and_account(); + class.serialize_into(buf, account_id, 0, 0, flags) + } + + #[inline] + pub fn verify(&self, value: i64) -> trc::Result<()> { + if value <= self.max { + Ok(()) + } else { + Err(self.exceeded_at(value)) + } + } + + #[cold] + #[inline(never)] + fn exceeded_at(&self, value: i64) -> trc::Error { + self.exceeded().ctx( + trc::Key::Used, + u64::try_from(value.saturating_sub(self.by)).unwrap_or_default(), + ) + } + + #[cfg(any(feature = "foundation", feature = "rocks"))] + #[inline] + pub(crate) fn allows_overshoot(&self) -> bool { + matches!(self.key, CounterKey::Queue(_)) + } + + #[cold] + #[inline(never)] + pub(crate) fn exceeded(&self) -> trc::Error { + let err = trc::LimitEvent::Quota + .into_err() + .ctx(trc::Key::Limit, u64::try_from(self.max).unwrap_or_default()) + .ctx(trc::Key::Size, u64::try_from(self.by).unwrap_or_default()); + match self.key { + CounterKey::Account { + account_id, + counter: AccountCounter::Quota, + } => err.account_id(account_id), + CounterKey::Account { + account_id, + counter: AccountCounter::ObjectCount(object), + } => err + .account_id(account_id) + .ctx(trc::Key::ObjectType, object.as_str()), + _ => err, + } + } +} + +impl ValueClass { + #[cfg(feature = "rocks")] + #[inline] + pub(crate) fn is_queue_quota(&self) -> bool { + matches!( + self, + ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) + ) + } +} + +impl From for ValueClass { + #[inline] + fn from(counter: QueueCounter) -> Self { + ValueClass::Queue(match counter { + QueueCounter::Count(key) => QueueClass::QuotaCount(key), + QueueCounter::Size(key) => QueueClass::QuotaSize(key), + }) + } +} + +impl From for ValueClass { + #[inline] + fn from(counter: AccountCounter) -> Self { + match counter { + AccountCounter::Quota => ValueClass::Quota, + AccountCounter::ObjectCount(object) => ValueClass::ObjectCount(object), + } + } +} + diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index e6f6dddb..d877ae84 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -5,35 +5,40 @@ */ use super::{ - AnyKey, BlobOp, InMemoryClass, QueueClass, TaskQueueClass, TelemetryClass, ValueClass, + AnyKey, BlobOp, QueueClass, QueueEvent, TaskDue, TaskQueueClass, TelemetryClass, ValueClass, }; use crate::{ - IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_LINK, SUBSPACE_COUNTER, - SUBSPACE_DELETED_ITEMS, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER, - SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, - SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REGISTRY, - SUBSPACE_REGISTRY_IDX, SUBSPACE_REGISTRY_PK, SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, - SUBSPACE_SEARCH_INDEX, SUBSPACE_SPAM_SAMPLES, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, WITH_SUBSPACE, + ChangeLogKey, IndexKey, IndexKeyPrefix, Key, LogKey, PrivateLogKey, Subspace, U16_LEN, U32_LEN, + U64_LEN, ValueKey, WITH_SUBSPACE, + search::{GLOBAL_BUCKET_SHIFT, SearchField}, write::{ - BlobLink, IndexPropertyClass, RegistryClass, SearchIndex, SearchIndexId, SearchIndexType, + BlobLink, IndexPropertyClass, PRIVATE_LOG, QueueDocumentId, RegistryClass, SearchIndex, + SearchIndexClass, metadata::MetadataClass, }, }; -use registry::schema::prelude::ObjectType; -use std::convert::TryInto; +use registry::{ + schema::{enums::TaskType, prelude::ObjectType, structs::TaskStatus}, + types::EnumImpl, +}; +use std::{borrow::BorrowMut, convert::TryInto, mem}; use types::{ blob_hash::BLOB_HASH_LEN, collection::{Collection, SyncCollection}, - field::{Field, MailboxField}, + field::{ + CalendarEventField, CalendarNotificationField, ContactField, EmailField, Field, + MailboxField, + }, }; -use utils::codec::leb128::Leb128_; - -pub struct KeySerializer { - pub buf: Vec, +use utils::{ + codec::leb128::{Leb128, Leb128Writer}, + hash128::Hash128, +}; +pub struct KeySerializer> { + buf: B, } pub trait KeySerialize { - fn serialize(&self, buf: &mut Vec); + fn to_key_bytes(&self) -> impl AsRef<[u8]>; } pub trait DeserializeBigEndian { @@ -42,70 +47,91 @@ pub trait DeserializeBigEndian { fn deserialize_be_u64(&self, index: usize) -> trc::Result; } -impl KeySerializer { +impl KeySerializer> { + #[inline(always)] pub fn new(capacity: usize) -> Self { Self { buf: Vec::with_capacity(capacity), } } - pub fn write(mut self, value: T) -> Self { - value.serialize(&mut self.buf); - self + #[inline(always)] + pub fn finalize(&mut self) -> Vec { + mem::take(&mut self.buf) } +} - pub fn write_leb128(mut self, value: T) -> Self { - T::to_leb128_bytes(value, &mut self.buf); - self +impl<'x> KeySerializer<&'x mut Vec> { + #[inline(always)] + pub fn borrowed(buf: &'x mut Vec) -> Self { + Self { buf } } +} - pub fn finalize(self) -> Vec { +impl>> KeySerializer { + #[inline(always)] + pub fn write(&mut self, value: T) -> &mut Self { self.buf + .borrow_mut() + .extend_from_slice(value.to_key_bytes().as_ref()); + self + } + + #[inline(always)] + pub fn write_leb128(&mut self, value: T) -> &mut Self { + self.buf.borrow_mut().push_leb128(value); + self } } impl KeySerialize for u8 { - fn serialize(&self, buf: &mut Vec) { - buf.push(*self); - } -} - -impl KeySerialize for &str { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(self.as_bytes()); - } -} - -impl KeySerialize for &String { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(self.as_bytes()); + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + [*self] } } impl KeySerialize for &[u8] { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(self); + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + *self } } -impl KeySerialize for u32 { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(&self.to_be_bytes()); +impl KeySerialize for &str { + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + self.as_bytes() } } -impl KeySerialize for u16 { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(&self.to_be_bytes()); +impl KeySerialize for &String { + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + self.as_bytes() } } -impl KeySerialize for u64 { - fn serialize(&self, buf: &mut Vec) { - buf.extend_from_slice(&self.to_be_bytes()); +impl KeySerialize for Hash128 { + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + self.as_be_bytes() } } +macro_rules! impl_key_serialize_be { + ($($ty:ty),+) => {$( + impl KeySerialize for $ty { + #[inline(always)] + fn to_key_bytes(&self) -> impl AsRef<[u8]> { + self.to_be_bytes() + } + } + )+}; +} + +impl_key_serialize_be!(u16, u32, u64); + impl DeserializeBigEndian for &[u8] { fn deserialize_be_u16(&self, index: usize) -> trc::Result { self.get(index..index + U16_LEN) @@ -177,26 +203,41 @@ impl ValueKey { class: ValueClass::Property(Field::ARCHIVE.into()), } } + + pub fn immutable( + account_id: u32, + collection: impl Into, + document_id: u32, + field: impl Into, + ) -> ValueKey { + ValueKey { + account_id, + collection: collection.into(), + document_id, + class: ValueClass::Immutable(field.into()), + } + } } impl Key for IndexKeyPrefix { - fn serialize(&self, flags: u32) -> Vec { - { - if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(std::mem::size_of::() + 1) - .write(crate::SUBSPACE_INDEXES) - } else { - KeySerializer::new(std::mem::size_of::()) - } - } - .write(self.account_id) - .write(self.collection) - .write(self.field) - .finalize() + fn key_len_hint(&self) -> usize { + IndexKeyPrefix::len() } - fn subspace(&self) -> u8 { - SUBSPACE_INDEXES + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + let mut serializer = KeySerializer::borrowed(buf); + if (flags & WITH_SUBSPACE) != 0 { + serializer.write(Subspace::Indexes.byte()); + } + + serializer + .write(self.account_id) + .write(self.collection) + .write(self.field); + } + + fn subspace(&self) -> Subspace { + Subspace::Indexes } } @@ -207,34 +248,87 @@ impl IndexKeyPrefix { } impl Key for LogKey { - fn subspace(&self) -> u8 { - SUBSPACE_LOGS + fn subspace(&self) -> Subspace { + Subspace::Logs } - fn serialize(&self, flags: u32) -> Vec { - { - if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(std::mem::size_of::() + 1).write(crate::SUBSPACE_LOGS) - } else { - KeySerializer::new(std::mem::size_of::()) - } + fn key_len_hint(&self) -> usize { + U32_LEN + 1 + U64_LEN + } + + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + let mut serializer = KeySerializer::borrowed(buf); + if (flags & WITH_SUBSPACE) != 0 { + serializer.write(Subspace::Logs.byte()); + } + + serializer + .write(self.account_id) + .write(self.collection) + .write(self.change_id); + } +} + +impl Key for PrivateLogKey { + fn subspace(&self) -> Subspace { + Subspace::Logs + } + + fn key_len_hint(&self) -> usize { + U32_LEN + 1 + U32_LEN + U64_LEN + } + + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + let mut serializer = KeySerializer::borrowed(buf); + if (flags & WITH_SUBSPACE) != 0 { + serializer.write(Subspace::Logs.byte()); + } + + serializer + .write(self.account_id) + .write(PRIVATE_LOG | u8::from(self.collection)) + .write(self.viewer) + .write(self.change_id); + } +} + +impl Key for ChangeLogKey { + fn subspace(&self) -> Subspace { + Subspace::Logs + } + + fn key_len_hint(&self) -> usize { + match self { + ChangeLogKey::Shared(key) => key.key_len_hint(), + ChangeLogKey::Private(key) => key.key_len_hint(), + } + } + + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + match self { + ChangeLogKey::Shared(key) => key.serialize_into(buf, flags), + ChangeLogKey::Private(key) => key.serialize_into(buf, flags), } - .write(self.account_id) - .write(self.collection) - .write(self.change_id) - .finalize() } } impl + Sync + Send + Clone> Key for ValueKey { - fn subspace(&self) -> u8 { + fn subspace(&self) -> Subspace { self.class.as_ref().subspace(self.collection) } - fn serialize(&self, flags: u32) -> Vec { - self.class - .as_ref() - .serialize(self.account_id, self.collection, self.document_id, flags) + fn key_len_hint(&self) -> usize { + self.class.as_ref().key_len_hint() + } + + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + self.class.as_ref().serialize_into( + buf, + self.account_id, + self.collection, + self.document_id, + flags, + ) } } @@ -246,31 +340,56 @@ impl ValueClass { document_id: u32, flags: u32, ) -> Vec { - let serializer = if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(self.serialized_size() + 2).write(self.subspace(collection)) - } else { - KeySerializer::new(self.serialized_size() + 1) - }; + let mut buf = Vec::with_capacity(self.key_len_hint() + 1); + self.serialize_into(&mut buf, account_id, collection, document_id, flags); + buf + } - match self { + pub fn serialize_into( + &self, + buf: &mut Vec, + account_id: u32, + collection: u8, + document_id: u32, + flags: u32, + ) { + let start = buf.len(); + let with_subspace = (flags & WITH_SUBSPACE) != 0; + let mut serializer = KeySerializer::borrowed(buf); + if with_subspace { + serializer.write(self.subspace(collection).byte()); + } + + let serializer = match self { ValueClass::Property(property) => serializer .write(account_id) .write(collection) .write(*property) .write(document_id), + ValueClass::Immutable(property) => serializer + .write(account_id) + .write(collection) + .write(*property) + .write(document_id), + ValueClass::MailboxUid => serializer + .write(CounterKind::MailboxUid.byte()) + .write(account_id) + .write(document_id), ValueClass::IndexProperty(property) => match property { - IndexPropertyClass::Hash { property, hash } => serializer - .write(account_id) - .write(collection) - .write(*property) - .write(hash.as_bytes()) - .write(document_id), IndexPropertyClass::Integer { property, value } => serializer .write(account_id) .write(collection) .write(*property) + .write(IndexPropertyClass::KIND_INTEGER) .write(*value) .write(document_id), + IndexPropertyClass::Hash { property, hash } => serializer + .write(account_id) + .write(collection) + .write(*property) + .write(IndexPropertyClass::KIND_HASH) + .write(*hash) + .write(!document_id), }, ValueClass::Acl(grant_account_id) => serializer .write(*grant_account_id) @@ -278,8 +397,14 @@ impl ValueClass { .write(collection) .write(document_id), ValueClass::TaskQueue(task) => match task { - TaskQueueClass::Task { id } => serializer.write(0u64).write(*id), - TaskQueueClass::Due { id, due } => serializer.write(*due).write(*id), + TaskQueueClass::Task { id } => serializer + .write(TaskQueueClass::KIND_TASK) + .write(id.resolve(account_id, document_id)), + TaskQueueClass::Due { id, typ, due } => serializer + .write(TaskQueueClass::KIND_DUE) + .write(*typ as u16) + .write(*due) + .write(id.resolve(account_id, document_id)), }, ValueClass::Blob(op) => match op { BlobOp::Commit { hash } => serializer.write::<&[u8]>(hash.as_ref()), @@ -296,10 +421,6 @@ impl ValueClass { .write(*until), }, }, - ValueClass::InMemory(lookup) => match lookup { - InMemoryClass::Key(key) => serializer.write(key.as_slice()), - InMemoryClass::Counter(key) => serializer.write(key.as_slice()), - }, ValueClass::Registry(registry) => match registry { RegistryClass::Item { object_id, item_id } => { serializer.write(*object_id).write(*item_id) @@ -323,6 +444,7 @@ impl ValueClass { from_object_id, from_item_id, } => serializer + .write(RegistryClass::REFERENCE_MARKER) .write(*to_object_id) .write(*to_item_id) .write(*from_object_id) @@ -335,26 +457,45 @@ impl ValueClass { .write((*object_id).unwrap_or(u16::MAX)) .write(*index_id) .write(key.as_slice()), - RegistryClass::IdCounter { object_id } => serializer.write(*object_id), + RegistryClass::IdCounter { object_id } => serializer + .write(CounterKind::RegistryId.byte()) + .write(*object_id), }, ValueClass::Queue(queue) => match queue { QueueClass::Message(queue_id) => serializer.write(*queue_id), QueueClass::MessageEvent(event) => serializer + .write(event.queue_name.as_slice()) .write(event.due) - .write(event.queue_id) - .write(event.queue_name.as_slice()), - QueueClass::QuotaCount(key) => serializer.write(0u8).write(key.as_slice()), - QueueClass::QuotaSize(key) => serializer.write(1u8).write(key.as_slice()), + .write(event.queue_id), + QueueClass::QuotaCount(key) => serializer + .write(CounterKind::QueueQuotaCount.byte()) + .write(*key), + QueueClass::QuotaSize(key) => serializer + .write(CounterKind::QueueQuotaSize.byte()) + .write(*key), }, ValueClass::Telemetry(telemetry) => match telemetry { TelemetryClass::Span(span_id) => serializer.write(*span_id), TelemetryClass::Metric(metric_id) => serializer.write(*metric_id), }, - ValueClass::DocumentId => serializer.write(account_id).write(collection), - ValueClass::ChangeId => serializer.write(account_id), - ValueClass::Quota => serializer.write(account_id).write(u8::MAX), - ValueClass::TenantQuota(tenant_id) => serializer.write(*tenant_id).write(u8::MAX - 1), - ValueClass::NodeId(node_id) => serializer.write(u32::MAX).write(*node_id), + ValueClass::DocumentId => serializer + .write(CounterKind::DocumentId.byte()) + .write(account_id) + .write(collection), + ValueClass::ChangeId(group) => serializer + .write(CounterKind::ChangeId.byte()) + .write(account_id) + .write(group.to_u8()), + ValueClass::Quota => serializer + .write(CounterKind::Quota.byte()) + .write(account_id), + ValueClass::ObjectCount(object) => serializer + .write(CounterKind::ObjectCount.byte()) + .write(account_id) + .write(*object as u8), + ValueClass::NodeId(node_id) => { + serializer.write(SystemKind::NodeId as u8).write(*node_id) + } ValueClass::ShareNotification { notification_id, notify_account_id, @@ -362,122 +503,155 @@ impl ValueClass { .write(*notify_account_id) .write(u8::from(SyncCollection::ShareNotification)) .write(*notification_id), - ValueClass::SearchIndex(index) => match &index.typ { - SearchIndexType::Term { field, hash } => { - let class = index.index.as_u8(); - match &index.id { - SearchIndexId::Account { - account_id, - document_id, - } => serializer - .write(class) - .write(*account_id) - .write(hash.payload()) - .write(hash.payload_len()) - .write(*field) - .write(*document_id), - SearchIndexId::Global { id } => serializer - .write(class) - .write(hash.payload()) - .write(hash.payload_len()) - .write(*field) - .write(*id), - } - } - SearchIndexType::Index { field } => { - let class = index.index.as_u8() | 1 << 6; - match &index.id { - SearchIndexId::Account { - account_id, - document_id, - } => serializer - .write(class) - .write(*account_id) - .write(field.field_id) - .write(field.data.as_slice()) - .write(*document_id), - SearchIndexId::Global { id } => serializer - .write(class) - .write(field.field_id) - .write(field.data.as_slice()) - .write(*id), - } - } - SearchIndexType::Document => { - let class = index.index.as_u8() | 2 << 6; - match &index.id { - SearchIndexId::Account { - account_id, - document_id, - } => serializer - .write(class) - .write(*account_id) - .write(*document_id), - SearchIndexId::Global { id } => serializer.write(class).write(*id), - } + ValueClass::SearchIndex(search) => match search { + SearchIndexClass::Term { + index, + account_id, + field, + term, + block_id, + } => serializer + .write(index.to_u8()) + .write(*account_id) + .write(index.term_group(*field)) + .write(term.as_key()) + .write(term.len() as u8) + .write(*field) + .write(*block_id), + SearchIndexClass::Document { + index, + account_id, + document_id, + } => serializer + .write(index.to_u8()) + .write(*account_id) + .write(*document_id), + SearchIndexClass::GlobalTerm { + index, + field, + term, + block_id, + } => serializer + .write(index.to_u8()) + .write(*field) + .write(term.as_key()) + .write(term.len() as u8) + .write(*block_id), + SearchIndexClass::GlobalDocument { index, document_id } => { + serializer.write(index.to_u8()).write(*document_id) } + SearchIndexClass::GlobalDocumentId { index, block_id } => serializer + .write(index.to_u8()) + .write(SearchField::Id.u8_id()) + .write(*block_id), + SearchIndexClass::Queue { + index, + id_prefix, + id_suffix, + created_at, + } => serializer + .write(index.to_u8()) + .write(*id_prefix) + .write(id_suffix.resolve(document_id)) + .write(*created_at), + SearchIndexClass::QueueIndex { index, partition } => serializer + .write(SearchIndexClass::QUEUE_CONTROL) + .write(index.to_u8()) + .write(*partition) + .write(SearchIndexClass::CONTROL_INDEX), + SearchIndexClass::QueueStatus { index, partition } => serializer + .write(SearchIndexClass::QUEUE_CONTROL) + .write(index.to_u8()) + .write(*partition) + .write(SearchIndexClass::CONTROL_STATUS), }, ValueClass::Any(any) => serializer.write(any.key.as_slice()), - } - .finalize() + ValueClass::Metadata(class) => { + let serializer = serializer.write(account_id).write(METADATA_COLLECTION); + match class { + MetadataClass::Shared => serializer + .write(MetadataClass::SHARED) + .write(collection) + .write(document_id), + MetadataClass::Private { viewer } => serializer + .write(MetadataClass::PRIVATE) + .write(*viewer) + .write(collection) + .write(document_id), + MetadataClass::Viewer { viewer } => serializer + .write(MetadataClass::VIEWER) + .write(collection) + .write(*viewer), + MetadataClass::Owner { owner } => serializer + .write(MetadataClass::OWNER) + .write(*owner) + .write(collection), + } + } + }; + + debug_assert_eq!( + serializer.buf.len() - start, + self.key_len_hint() + usize::from(with_subspace), + "key length hint disagrees with the serialized key for {self:?}" + ); } } impl + Sync + Send + Clone> Key for IndexKey { - fn subspace(&self) -> u8 { - SUBSPACE_INDEXES + fn subspace(&self) -> Subspace { + Subspace::Indexes } - fn serialize(&self, flags: u32) -> Vec { - let key = self.key.as_ref(); - { - if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(std::mem::size_of::>() + key.len() + 1) - .write(crate::SUBSPACE_INDEXES) - } else { - KeySerializer::new(std::mem::size_of::>() + key.len()) - } + fn key_len_hint(&self) -> usize { + IndexKeyPrefix::len() + self.key.as_ref().len() + U32_LEN + } + + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + let mut serializer = KeySerializer::borrowed(buf); + if (flags & WITH_SUBSPACE) != 0 { + serializer.write(Subspace::Indexes.byte()); } - .write(self.account_id) - .write(self.collection) - .write(self.field) - .write(key) - .write(self.document_id) - .finalize() + + serializer + .write(self.account_id) + .write(self.collection) + .write(self.field) + .write(self.key.as_ref()) + .write(self.document_id); } } impl + Sync + Send + Clone> Key for AnyKey { - fn serialize(&self, flags: u32) -> Vec { - let key = self.key.as_ref(); - if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(key.len() + 1).write(self.subspace) - } else { - KeySerializer::new(key.len()) - } - .write(key) - .finalize() + fn key_len_hint(&self) -> usize { + self.key.as_ref().len() } - fn subspace(&self) -> u8 { + fn serialize_into(&self, buf: &mut Vec, flags: u32) { + let mut serializer = KeySerializer::borrowed(buf); + if (flags & WITH_SUBSPACE) != 0 { + serializer.write(self.subspace.byte()); + } + + serializer.write(self.key.as_ref()); + } + + fn subspace(&self) -> Subspace { self.subspace } } const MAILBOX_COLLECTION: u8 = Collection::Mailbox as u8; -const MAILBOX_COUNTER_FIELD: u8 = MailboxField::UidCounter as u8; -const REG_ARCHIVED_ITEM: u16 = ObjectType::ArchivedItem as u16; +const METADATA_COLLECTION: u8 = Collection::Metadata as u8; +const EMAIL_COLLECTION: u8 = Collection::Email as u8; const REG_SPAM_SAMPLE: u16 = ObjectType::SpamTrainingSample as u16; const REG_ACCOUNT: u16 = ObjectType::Account as u16; const REG_DOMAIN: u16 = ObjectType::Domain as u16; -const REG_TENANT: u16 = ObjectType::Tenant as u16; const REG_ROLE: u16 = ObjectType::Role as u16; const REG_OAUTH_CLIENT: u16 = ObjectType::OAuthClient as u16; +const REG_OAUTH_GRANT: u16 = ObjectType::OAuthGrant as u16; const REG_MAILING_LIST: u16 = ObjectType::MailingList as u16; -const REG_MASKED_EMAIL: u16 = ObjectType::MaskedEmail as u16; const REG_PUBLIC_KEY: u16 = ObjectType::PublicKey as u16; -const REG_TRACE: u16 = ObjectType::Trace as u16; -const REG_METRIC: u16 = ObjectType::Metric as u16; const REPORT_EXTERNAL_ARF: u16 = ObjectType::ArfExternalReport as u16; const REPORT_EXTERNAL_DMARC: u16 = ObjectType::DmarcExternalReport as u16; const REPORT_EXTERNAL_TLS: u16 = ObjectType::TlsExternalReport as u16; @@ -485,21 +659,25 @@ const REPORT_INTERNAL_DMARC: u16 = ObjectType::DmarcInternalReport as u16; const REPORT_INTERNAL_TLS: u16 = ObjectType::TlsInternalReport as u16; impl ValueClass { - pub fn serialized_size(&self) -> usize { + pub fn key_len_hint(&self) -> usize { match self { - ValueClass::Property(_) => U32_LEN * 2 + 3, - ValueClass::IndexProperty(p) => match p { - IndexPropertyClass::Hash { hash, .. } => U32_LEN * 2 + 3 + hash.len(), - IndexPropertyClass::Integer { .. } => U32_LEN * 2 + 3 + U64_LEN, - }, - ValueClass::Acl(_) => U32_LEN * 3 + 2, - ValueClass::InMemory(InMemoryClass::Counter(v) | InMemoryClass::Key(v)) => v.len(), + ValueClass::Property(_) | ValueClass::Immutable(_) => (U32_LEN * 2) + 2, + ValueClass::MailboxUid => (U32_LEN * 2) + 1, + ValueClass::IndexProperty(p) => { + (U32_LEN * 2) + + 3 + + match p { + IndexPropertyClass::Hash { .. } => Hash128::LEN, + IndexPropertyClass::Integer { .. } => U64_LEN, + } + } + ValueClass::Acl(_) => (U32_LEN * 3) + 1, ValueClass::Registry(registry) => match registry { - RegistryClass::Item { .. } => U16_LEN + U64_LEN + 1, - RegistryClass::Reference { .. } => ((U16_LEN + U64_LEN) * 2) + 1, - RegistryClass::Index { key, .. } => (U16_LEN * 2) + U64_LEN + key.len() + 1, - RegistryClass::PrimaryKey { key, .. } => (U16_LEN * 2) + key.len() + 1, - RegistryClass::IndexId { .. } => U16_LEN + U64_LEN + 1, + RegistryClass::Item { .. } => U16_LEN + U64_LEN, + RegistryClass::Reference { .. } => U16_LEN + ((U16_LEN + U64_LEN) * 2), + RegistryClass::Index { key, .. } => (U16_LEN * 2) + U64_LEN + key.len(), + RegistryClass::PrimaryKey { key, .. } => (U16_LEN * 2) + key.len(), + RegistryClass::IndexId { .. } => (U16_LEN * 2) + U64_LEN, RegistryClass::IdCounter { .. } => U16_LEN + 1, }, ValueClass::Blob(op) => match op { @@ -508,100 +686,162 @@ impl ValueClass { BLOB_HASH_LEN + match to { BlobLink::Id { .. } => U64_LEN, - BlobLink::Document => U32_LEN * 2 + 1, + BlobLink::Document => (U32_LEN * 2) + 1, BlobLink::Temporary { .. } => U32_LEN + U64_LEN, } } }, - ValueClass::TaskQueue(_) => (U64_LEN * 2) + 1, + ValueClass::TaskQueue(task) => match task { + TaskQueueClass::Task { .. } => U64_LEN + 1, + TaskQueueClass::Due { .. } => (U64_LEN * 2) + U16_LEN + 1, + }, ValueClass::Queue(q) => match q { QueueClass::Message(_) => U64_LEN, - QueueClass::MessageEvent(_) => U64_LEN * 3, - QueueClass::QuotaCount(v) | QueueClass::QuotaSize(v) => v.len(), + QueueClass::MessageEvent(_) => QueueEvent::KEY_LEN, + QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_) => Hash128::LEN + 1, }, ValueClass::Telemetry(telemetry) => match telemetry { - TelemetryClass::Span(_) | TelemetryClass::Metric(_) => U64_LEN + 1, + TelemetryClass::Span(_) | TelemetryClass::Metric(_) => U64_LEN, }, - ValueClass::DocumentId | ValueClass::Quota | ValueClass::TenantQuota(_) => U32_LEN + 1, - ValueClass::ChangeId => U32_LEN, + ValueClass::DocumentId | ValueClass::ChangeId(_) | ValueClass::ObjectCount(_) => { + U32_LEN + 2 + } + ValueClass::Quota => U32_LEN + 1, ValueClass::ShareNotification { .. } => U32_LEN + U64_LEN + 1, - ValueClass::NodeId(_) => (U16_LEN * 3) + 1, - ValueClass::SearchIndex(v) => match &v.typ { - SearchIndexType::Term { hash, .. } => U64_LEN + hash.len() + 2, - SearchIndexType::Index { field, .. } => 1 + field.data.len() + U64_LEN, - SearchIndexType::Document => match &v.id { - SearchIndexId::Account { .. } => 1 + U32_LEN * 2, - SearchIndexId::Global { .. } => 1 + U64_LEN, - }, + ValueClass::NodeId(_) => U16_LEN + 1, + ValueClass::SearchIndex(v) => match v { + SearchIndexClass::Term { term, .. } => U32_LEN + term.key_len() + U16_LEN + 4, + SearchIndexClass::Document { .. } => (U32_LEN * 2) + 1, + SearchIndexClass::GlobalTerm { term, .. } => term.key_len() + U16_LEN + 3, + SearchIndexClass::GlobalDocument { .. } => U64_LEN + 1, + SearchIndexClass::GlobalDocumentId { .. } => U16_LEN + 2, + SearchIndexClass::Queue { .. } => (U32_LEN * 2) + U64_LEN + 1, + SearchIndexClass::QueueIndex { .. } | SearchIndexClass::QueueStatus { .. } => { + U32_LEN + 3 + } }, ValueClass::Any(v) => v.key.len(), + ValueClass::Metadata(class) => match class { + MetadataClass::Private { .. } => (U32_LEN * 3) + 3, + MetadataClass::Shared + | MetadataClass::Viewer { .. } + | MetadataClass::Owner { .. } => (U32_LEN * 2) + 3, + }, } } - pub fn subspace(&self, collection: u8) -> u8 { + pub fn subspace(&self, collection: u8) -> Subspace { match self { ValueClass::Property(field) => { - if collection == MAILBOX_COLLECTION && *field == MAILBOX_COUNTER_FIELD { - SUBSPACE_COUNTER - } else { - SUBSPACE_PROPERTY - } + debug_assert!( + !(collection == MAILBOX_COLLECTION + && *field == u8::from(MailboxField::UidCounter)), + "the mailbox UID counter must be written as ValueClass::MailboxUid" + ); + debug_assert!( + !(collection == EMAIL_COLLECTION + && (*field == u8::from(EmailField::Metadata) + || *field == u8::from(EmailField::SortKeys))), + "email field {field} must be written as ValueClass::Immutable" + ); + Subspace::Property } - ValueClass::IndexProperty { .. } => SUBSPACE_PROPERTY, - ValueClass::Acl(_) => SUBSPACE_ACL, - ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, + ValueClass::Immutable(_) => Subspace::Immutable, + ValueClass::IndexProperty { .. } => Subspace::IndexProperty, + ValueClass::MailboxUid => Subspace::Counter, + ValueClass::Acl(_) => Subspace::Acl, + ValueClass::TaskQueue { .. } => Subspace::TaskQueue, ValueClass::Blob(op) => match op { - BlobOp::Commit { .. } | BlobOp::Link { .. } => SUBSPACE_BLOB_LINK, + BlobOp::Commit { .. } | BlobOp::Link { .. } => Subspace::BlobLink, }, ValueClass::Registry(registry) => match registry { RegistryClass::Item { object_id, .. } => match *object_id { - REG_ACCOUNT | REG_DOMAIN | REG_TENANT | REG_ROLE | REG_OAUTH_CLIENT - | REG_MAILING_LIST | REG_MASKED_EMAIL | REG_PUBLIC_KEY => SUBSPACE_DIRECTORY, - REG_ARCHIVED_ITEM => SUBSPACE_DELETED_ITEMS, - REG_SPAM_SAMPLE => SUBSPACE_SPAM_SAMPLES, - REG_TRACE => SUBSPACE_TELEMETRY_SPAN, - REG_METRIC => SUBSPACE_TELEMETRY_METRIC, + REG_ACCOUNT | REG_DOMAIN | REG_ROLE | REG_OAUTH_CLIENT | REG_MAILING_LIST + | REG_PUBLIC_KEY | REG_OAUTH_GRANT => Subspace::Directory, + REG_SPAM_SAMPLE => Subspace::SpamSamples, REPORT_EXTERNAL_ARF | REPORT_EXTERNAL_DMARC | REPORT_EXTERNAL_TLS => { - SUBSPACE_REPORT_IN + Subspace::ReportIn } - REPORT_INTERNAL_DMARC | REPORT_INTERNAL_TLS => SUBSPACE_REPORT_OUT, - _ => SUBSPACE_REGISTRY, + REPORT_INTERNAL_DMARC | REPORT_INTERNAL_TLS => Subspace::ReportOut, + _ => Subspace::Registry, }, RegistryClass::IndexId { .. } | RegistryClass::Index { .. } => { - SUBSPACE_REGISTRY_IDX + Subspace::RegistryIndex } RegistryClass::Reference { .. } | RegistryClass::PrimaryKey { .. } => { - SUBSPACE_REGISTRY_PK + Subspace::RegistryPrimaryKey } - RegistryClass::IdCounter { .. } => SUBSPACE_COUNTER, - }, - ValueClass::NodeId(_) => SUBSPACE_REGISTRY_PK, - ValueClass::InMemory(lookup) => match lookup { - InMemoryClass::Key(_) => SUBSPACE_IN_MEMORY_VALUE, - InMemoryClass::Counter(_) => SUBSPACE_IN_MEMORY_COUNTER, + RegistryClass::IdCounter { .. } => Subspace::Counter, }, + ValueClass::NodeId(_) => Subspace::System, ValueClass::Queue(queue) => match queue { - QueueClass::Message(_) => SUBSPACE_QUEUE_MESSAGE, - QueueClass::MessageEvent(_) => SUBSPACE_QUEUE_EVENT, - QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_) => SUBSPACE_QUOTA, + QueueClass::Message(_) => Subspace::QueueMessage, + QueueClass::MessageEvent(_) => Subspace::QueueEvent, + QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_) => Subspace::Counter, }, ValueClass::Telemetry(telemetry) => match telemetry { - TelemetryClass::Span { .. } => SUBSPACE_TELEMETRY_SPAN, - TelemetryClass::Metric { .. } => SUBSPACE_TELEMETRY_METRIC, + TelemetryClass::Span { .. } => Subspace::TelemetrySpan, + TelemetryClass::Metric { .. } => Subspace::TelemetryMetric, }, ValueClass::DocumentId - | ValueClass::ChangeId + | ValueClass::ChangeId(_) | ValueClass::Quota - | ValueClass::TenantQuota(_) => SUBSPACE_COUNTER, - ValueClass::ShareNotification { .. } => SUBSPACE_LOGS, - ValueClass::SearchIndex(_) => SUBSPACE_SEARCH_INDEX, + | ValueClass::ObjectCount(_) => Subspace::Counter, + ValueClass::ShareNotification { .. } => Subspace::Logs, + ValueClass::SearchIndex(search) => match search { + SearchIndexClass::Term { .. } + | SearchIndexClass::GlobalTerm { .. } + | SearchIndexClass::GlobalDocumentId { .. } => Subspace::SearchTerm, + SearchIndexClass::Document { .. } | SearchIndexClass::GlobalDocument { .. } => { + Subspace::SearchDocument + } + SearchIndexClass::Queue { .. } + | SearchIndexClass::QueueIndex { .. } + | SearchIndexClass::QueueStatus { .. } => Subspace::SearchQueue, + }, ValueClass::Any(any) => any.subspace, + ValueClass::Metadata(_) => Subspace::Property, + } + } + + pub fn may_be_chunked(&self) -> bool { + match self { + ValueClass::Property(field) => { + *field == u8::from(Field::ARCHIVE) + || *field == ContactField::Content.as_u8() + || *field == CalendarEventField::Content.as_u8() + || *field == CalendarNotificationField::Content.as_u8() + } + ValueClass::Immutable(field) => *field == u8::from(EmailField::Metadata), + ValueClass::IndexProperty(IndexPropertyClass::Hash { .. }) + | ValueClass::Registry(RegistryClass::Item { .. }) + | ValueClass::Queue(QueueClass::Message(_)) + | ValueClass::TaskQueue(TaskQueueClass::Task { .. }) + | ValueClass::Telemetry(TelemetryClass::Span(_)) + | ValueClass::Metadata(MetadataClass::Shared | MetadataClass::Private { .. }) => true, + ValueClass::SearchIndex(search) => matches!( + search, + SearchIndexClass::Document { .. } + | SearchIndexClass::GlobalTerm { .. } + | SearchIndexClass::GlobalDocument { .. } + | SearchIndexClass::GlobalDocumentId { .. } + ), + _ => false, } } } +pub fn node_id_from_key(key: &[u8]) -> Option { + match key { + [kind, node_id @ ..] if *kind == SystemKind::NodeId as u8 && node_id.len() == U16_LEN => { + node_id.try_into().ok().map(u16::from_be_bytes) + } + _ => None, + } +} + pub fn is_node_id_key(key: &[u8]) -> bool { - key.len() == U32_LEN + U16_LEN && key.starts_with(&u32::MAX.to_be_bytes()) + node_id_from_key(key).is_some() } impl From for ValueKey { @@ -638,6 +878,282 @@ impl From for ValueClass { } } +impl RegistryClass { + pub const REFERENCE_MARKER: u16 = u16::MAX - 1; + + pub fn prefix(object_id: u16, index_id: u16) -> Vec { + KeySerializer::new(U16_LEN * 2) + .write(object_id) + .write(index_id) + .finalize() + } + + pub fn item_range(object_id: u16) -> (AnyKey>, AnyKey>) { + let subspace = ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: 0, + }) + .subspace(0); + + ( + AnyKey { + subspace, + key: KeySerializer::new(U16_LEN).write(object_id).finalize(), + }, + AnyKey { + subspace, + key: KeySerializer::new(U16_LEN + U64_LEN) + .write(object_id) + .write(u64::MAX) + .finalize(), + }, + ) + } + + pub fn index_range( + object_id: u16, + from: (u16, &[u8], Option), + to: (u16, &[u8], Option), + ) -> (AnyKey>, AnyKey>) { + ( + AnyKey { + subspace: Subspace::RegistryIndex, + key: Self::bound(object_id, from), + }, + AnyKey { + subspace: Subspace::RegistryIndex, + key: Self::bound(object_id, to), + }, + ) + } + + pub fn pk_range( + object_id: u16, + from: (u16, &[u8], Option), + to: (u16, &[u8], Option), + ) -> (AnyKey>, AnyKey>) { + ( + AnyKey { + subspace: Subspace::RegistryPrimaryKey, + key: Self::bound(object_id, from), + }, + AnyKey { + subspace: Subspace::RegistryPrimaryKey, + key: Self::bound(object_id, to), + }, + ) + } + + fn bound(object_id: u16, (index_id, value, item_id): (u16, &[u8], Option)) -> Vec { + let mut serializer = KeySerializer::new((U16_LEN * 2) + value.len() + U64_LEN); + serializer.write(object_id).write(index_id).write(value); + if let Some(item_id) = item_id { + serializer.write(item_id); + } + serializer.finalize() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum CounterKind { + DocumentId = 0x00, + ChangeId = 0x01, + ObjectCount = 0x02, + Quota = 0x03, + MailboxUid = 0x04, + RegistryId = 0x06, + ReportRow = 0x07, + QueueQuotaCount = 0x08, + QueueQuotaSize = 0x09, +} + +impl CounterKind { + pub const ACCOUNT_KINDS: [CounterKind; 5] = [ + CounterKind::DocumentId, + CounterKind::ChangeId, + CounterKind::ObjectCount, + CounterKind::Quota, + CounterKind::MailboxUid, + ]; + + #[inline(always)] + pub const fn byte(self) -> u8 { + self as u8 + } +} + +#[repr(u8)] +pub enum SystemKind { + SchemaVersion = 0x00, + NodeId = 0x01, +} + +impl QueueEvent { + pub const KEY_LEN: usize = U64_LEN * 3; + pub const TENANT_INDEX_QUEUE: [u8; 8] = [u8::MAX; 8]; + pub const LAST_QUEUE: [u8; 8] = (u64::MAX - 1).to_be_bytes(); + + pub fn deserialize(key: &[u8]) -> trc::Result { + if let Some((queue_name, rest)) = key.split_first_chunk::() + && let Some((due, rest)) = rest.split_first_chunk::() + && let Some((queue_id, [])) = rest.split_first_chunk::() + { + Ok(QueueEvent { + due: u64::from_be_bytes(*due), + queue_id: u64::from_be_bytes(*queue_id), + queue_name: *queue_name, + }) + } else { + Err(trc::Error::corrupted_key(key, None, trc::location!())) + } + } +} + +impl TaskQueueClass { + pub const KIND_TASK: u8 = 0x00; + pub const KIND_DUE: u8 = 0x01; + pub const DUE_RETRY: u8 = 0x01; + pub const DUE_STALLED: u8 = 0x02; + + pub fn due_marker(status: &TaskStatus) -> Option { + match status { + TaskStatus::Retry(_) => Some(Self::DUE_RETRY), + TaskStatus::Stalled(_) => Some(Self::DUE_STALLED), + TaskStatus::Pending(_) | TaskStatus::Failed(_) => None, + } + } + + #[inline] + pub fn due_marker_of(value: &[u8]) -> Option { + value.first().copied() + } +} + +impl TaskDue { + pub fn deserialize(key: &[u8]) -> trc::Result { + if let Some((&TaskQueueClass::KIND_DUE, rest)) = key.split_first() + && let Some((typ, rest)) = rest.split_first_chunk::() + && let Some((due, id)) = rest.split_first_chunk::() + && let Ok(id) = <[u8; U64_LEN]>::try_from(id) + && let Some(typ) = TaskType::from_id(u16::from_be_bytes(*typ)) + { + Ok(TaskDue { + typ, + due: u64::from_be_bytes(*due), + id: u64::from_be_bytes(id), + }) + } else { + Err(trc::Error::corrupted_key(key, None, trc::location!())) + } + } +} + +impl IndexPropertyClass { + pub const KIND_INTEGER: u8 = 0x00; + pub const KIND_HASH: u8 = 0x01; + + pub fn key_len(&self) -> usize { + IndexKeyPrefix::len() + + 1 + + match self { + IndexPropertyClass::Hash { .. } => Hash128::LEN, + IndexPropertyClass::Integer { .. } => U64_LEN, + } + + U32_LEN + } +} + +impl SearchIndexClass { + pub const QUEUE_CONTROL: u8 = u8::MAX; + pub const CONTROL_INDEX: u8 = 0x00; + pub const CONTROL_STATUS: u8 = 0x01; + + pub fn control_range( + from: (SearchIndex, u32), + to: Option<(SearchIndex, u32)>, + ) -> (AnyKey>, AnyKey>) { + let begin = KeySerializer::new(U32_LEN + 3) + .write(Self::QUEUE_CONTROL) + .write(from.0.to_u8()) + .write(from.1) + .write(Self::CONTROL_INDEX) + .finalize(); + let end = match to { + Some((index, partition)) => KeySerializer::new(U32_LEN + 3) + .write(Self::QUEUE_CONTROL) + .write(index.to_u8()) + .write(partition) + .write(u8::MAX) + .finalize(), + None => KeySerializer::new(2) + .write(Self::QUEUE_CONTROL) + .write(u8::MAX) + .finalize(), + }; + + ( + AnyKey { + subspace: Subspace::SearchQueue, + key: begin, + }, + AnyKey { + subspace: Subspace::SearchQueue, + key: end, + }, + ) + } + + pub fn queue_range(index: SearchIndex, partition: u32) -> (Self, Self) { + let (from_prefix, to_prefix) = if matches!(index, SearchIndex::Tracing) { + let bucket = partition << (GLOBAL_BUCKET_SHIFT - 32); + (bucket, bucket | ((1 << (GLOBAL_BUCKET_SHIFT - 32)) - 1)) + } else { + (partition, partition) + }; + + ( + SearchIndexClass::Queue { + index, + id_prefix: from_prefix, + id_suffix: QueueDocumentId::Assigned(0), + created_at: 0, + }, + SearchIndexClass::Queue { + index, + id_prefix: to_prefix, + id_suffix: QueueDocumentId::Assigned(u32::MAX), + created_at: u64::MAX, + }, + ) + } + + pub fn queue_partition(index: SearchIndex, id_prefix: u32) -> u32 { + if matches!(index, SearchIndex::Tracing) { + id_prefix >> (GLOBAL_BUCKET_SHIFT - 32) + } else { + id_prefix + } + } +} + +impl From for ValueClass { + fn from(value: SearchIndexClass) -> Self { + ValueClass::SearchIndex(value) + } +} + +impl From for ValueKey { + fn from(value: SearchIndexClass) -> Self { + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::SearchIndex(value), + } + } +} + impl SearchIndex { pub fn to_u8(&self) -> u8 { match self { @@ -683,3 +1199,4 @@ impl SearchIndex { } } } + diff --git a/crates/store/src/write/lazybitmap.rs b/crates/store/src/write/lazybitmap.rs new file mode 100644 index 00000000..0d2edc21 --- /dev/null +++ b/crates/store/src/write/lazybitmap.rs @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use roaring::{RoaringBitmap, RoaringTreemap}; +use utils::codec::leb128::{Leb128Iterator, Leb128Writer}; + +const IS_ROARING: u8 = 1; + +pub(crate) struct LazyBitmap(pub RoaringBitmap); +pub(crate) struct LazyTreemap(pub RoaringTreemap); + +impl LazyBitmap { + pub fn serialize_optimized(&mut self, block_base: u32) -> Vec { + self.0.optimize(); + let roaring_size = self.0.serialized_size(); + + if roaring_size <= self.0.len() as usize { + return self.serialize_roaring(roaring_size); + } + + let delta = self.serialize_delta(block_base); + if roaring_size + 1 < delta.len() { + self.serialize_roaring(roaring_size) + } else { + delta + } + } + + fn serialize_roaring(&self, roaring_size: usize) -> Vec { + let mut serialized = Vec::with_capacity(roaring_size + 1); + serialized.push(IS_ROARING); + let _ = self.0.serialize_into(&mut serialized); + serialized + } + + fn serialize_delta(&self, mut delta: u32) -> Vec { + let mut serialized = Vec::with_capacity((self.0.len() as usize * 2) + 1); + serialized.push(0); + for id in self.0.iter() { + serialized.push_leb128(id - delta); + delta = id; + } + serialized + } + + pub fn deserialize_delta(bytes: &[u8], mut delta: u32) -> trc::Result { + let Some((is_roaring, bytes)) = bytes + .split_at_checked(1) + .map(|(byte, bytes)| (byte == [IS_ROARING], bytes)) + else { + return Err(trc::Error::corrupted_key(bytes, None, trc::location!())); + }; + + if is_roaring { + RoaringBitmap::deserialize_from(bytes) + .ok() + .map(Self) + .ok_or_else(|| trc::Error::corrupted_key(bytes, None, trc::location!())) + } else { + let mut bitmap = RoaringBitmap::new(); + let mut iter = bytes.iter().copied(); + + while let Some(delta_id) = iter.next_leb128::() { + delta = delta + .checked_add(delta_id) + .filter(|delta| bitmap.try_push(*delta).is_ok()) + .ok_or_else(|| trc::Error::corrupted_key(bytes, None, trc::location!()))?; + } + + if iter.next().is_none() { + Ok(Self(bitmap)) + } else { + Err(trc::Error::corrupted_key(bytes, None, trc::location!())) + } + } + } +} + +impl LazyTreemap { + pub fn serialize_delta(&self, mut delta: u64) -> Vec { + let mut serialized = Vec::with_capacity(self.0.len() as usize * 8); + for id in self.0.iter() { + serialized.push_leb128(id - delta); + delta = id; + } + serialized + } + + pub fn deserialize_delta(bytes: &[u8], mut delta: u64) -> trc::Result { + let mut treemap = RoaringTreemap::new(); + let mut iter = bytes.iter().copied(); + + while let Some(delta_id) = iter.next_leb128::() { + delta = delta + .checked_add(delta_id) + .filter(|delta| treemap.try_push(*delta).is_ok()) + .ok_or_else(|| trc::Error::corrupted_key(bytes, None, trc::location!()))?; + } + + if iter.next().is_none() { + Ok(Self(treemap)) + } else { + Err(trc::Error::corrupted_key(bytes, None, trc::location!())) + } + } +} + +impl AsRef for LazyBitmap { + fn as_ref(&self) -> &RoaringBitmap { + &self.0 + } +} + diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index d15e10a1..2e3ac046 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -4,24 +4,232 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{SerializeInfallible, U64_LEN}; -use ahash::AHashSet; +use crate::{ + U16_LEN, U64_LEN, + write::{AssignedIds, PendingId}, +}; +use ahash::{AHashSet, RandomState}; +use indexmap::IndexSet; use types::collection::{SyncCollection, VanishedCollection}; -use utils::{codec::leb128::Leb128Vec, map::vec_map::VecMap}; +use utils::{codec::leb128::Leb128Writer, map::vec_map::VecMap}; -use super::key::KeySerializer; +pub const CONTAINER_INSERTS: usize = 0; +pub const CONTAINER_UPDATES: usize = 1; +pub const CONTAINER_PROPERTY_CHANGES: usize = 2; +pub const CONTAINER_DELETES: usize = 3; +pub const ITEM_INSERTS: usize = 4; +pub const ITEM_UPDATES: usize = 5; +pub const ITEM_DELETES: usize = 6; +pub const CONTAINER_METADATA: usize = 7; +pub const ITEM_METADATA: usize = 8; +pub const CHANGE_LISTS: usize = 9; + +pub(crate) const PRESENCE_EXTENDED: u8 = 0x80; +pub(crate) const BASE_LISTS: usize = CONTAINER_METADATA; +pub(crate) const CONTAINER_LISTS: u16 = (1 << CONTAINER_INSERTS) + | (1 << CONTAINER_UPDATES) + | (1 << CONTAINER_PROPERTY_CHANGES) + | (1 << CONTAINER_DELETES) + | (1 << CONTAINER_METADATA); +pub(crate) const ITEM_LISTS: u16 = + (1 << ITEM_INSERTS) | (1 << ITEM_UPDATES) | (1 << ITEM_DELETES) | (1 << ITEM_METADATA); + +const _: () = assert!(CHANGE_LISTS == ITEM_METADATA + 1); +const _: () = assert!(BASE_LISTS == ITEM_DELETES + 1); +const _: () = assert!(BASE_LISTS < u8::BITS as usize); +const _: () = assert!(CHANGE_LISTS - BASE_LISTS < u8::BITS as usize); +const _: () = assert!(CONTAINER_LISTS & ITEM_LISTS == 0); +const _: () = assert!((CONTAINER_LISTS | ITEM_LISTS).count_ones() as usize == CHANGE_LISTS); + +const CHANGE_SET_THRESHOLD: usize = 256; +const CHANGE_SET_SCAN_LIMIT: usize = 32; + +const CONTAINER_BYTES_HINT: usize = 2; +const ITEM_BYTES_HINT: usize = 2; +const PREFIXED_ITEM_BYTES_HINT: usize = 3; +const VANISHED_ID_BYTES_HINT: usize = 2; + +#[derive(Debug)] +pub enum ChangeSet { + Small(Vec), + Large(AHashSet), +} + +pub const PENDING_ID_MARKER: u64 = (u32::MAX as u64) << 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PendingChange { + pub prefix: Option, + pub id: PendingId, +} + +#[inline(always)] +pub const fn is_pending_id(id: u64) -> bool { + id & PENDING_ID_MARKER == PENDING_ID_MARKER +} + +impl Default for ChangeSet { + fn default() -> Self { + ChangeSet::Small(Vec::new()) + } +} + +pub enum ChangeSetIter<'x, T> { + Small(std::slice::Iter<'x, T>), + Large(std::collections::hash_set::Iter<'x, T>), +} + +impl<'x, T> Iterator for ChangeSetIter<'x, T> { + type Item = &'x T; + + fn next(&mut self) -> Option { + match self { + ChangeSetIter::Small(items) => items.next(), + ChangeSetIter::Large(items) => items.next(), + } + } +} + +impl ChangeSet { + pub fn iter(&self) -> ChangeSetIter<'_, T> { + match self { + ChangeSet::Small(items) => ChangeSetIter::Small(items.iter()), + ChangeSet::Large(items) => ChangeSetIter::Large(items.iter()), + } + } +} + +impl<'x, T> IntoIterator for &'x ChangeSet { + type Item = &'x T; + type IntoIter = ChangeSetIter<'x, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl> ChangeSet { + pub fn insert(&mut self, value: T) -> bool { + match self { + ChangeSet::Small(items) => { + if items.contains(&value) { + return false; + } + items.push(value); + if items.len() > CHANGE_SET_THRESHOLD { + *self = ChangeSet::Large(items.iter().copied().collect()); + } + true + } + ChangeSet::Large(items) => items.insert(value), + } + } + + pub fn remove(&mut self, value: &T) -> bool { + if let ChangeSet::Small(items) = self + && items.len() > CHANGE_SET_SCAN_LIMIT + { + *self = ChangeSet::Large(items.iter().copied().collect()); + } + + match self { + ChangeSet::Small(items) => { + if let Some(pos) = items.iter().position(|item| item == value) { + items.swap_remove(pos); + true + } else { + false + } + } + ChangeSet::Large(items) => items.remove(value), + } + } + + pub fn contains(&self, value: &T) -> bool { + match self { + ChangeSet::Small(items) => items.contains(value), + ChangeSet::Large(items) => items.contains(value), + } + } + + pub fn len(&self) -> usize { + match self { + ChangeSet::Small(items) => items.len(), + ChangeSet::Large(items) => items.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + ChangeSet::Small(items) => items.is_empty(), + ChangeSet::Large(items) => items.is_empty(), + } + } + + fn collect_unsorted(&self, out: &mut Vec) { + out.clear(); + match self { + ChangeSet::Small(items) => out.extend(items.iter().map(|item| (*item).into())), + ChangeSet::Large(items) => out.extend(items.iter().map(|item| (*item).into())), + } + } + + fn collect_sorted(&self, out: &mut Vec) { + self.collect_unsorted(out); + out.sort_unstable(); + } +} + +#[inline(always)] +pub fn zigzag_encode(value: i64) -> u64 { + ((value << 1) ^ (value >> 63)) as u64 +} + +#[inline(always)] +pub fn zigzag_decode(value: u64) -> i64 { + ((value >> 1) as i64) ^ -((value & 1) as i64) +} + +#[inline(always)] +fn write_ids(buf: &mut Vec, ids: &[u64]) { + let mut prev = 0u64; + for id in ids { + buf.push_leb128(*id - prev); + prev = *id; + } +} + +#[inline(always)] +fn write_prefixed_ids(buf: &mut Vec, ids: &[u64]) { + let mut prev_prefix = 0i64; + let mut prev_document_id = 0i64; + for id in ids { + let prefix = (*id >> 32) as i64; + let document_id = (*id & u32::MAX as u64) as i64; + buf.push_leb128((prefix - prev_prefix) as u64); + buf.push_leb128(zigzag_encode(document_id - prev_document_id)); + prev_prefix = prefix; + prev_document_id = document_id; + } +} #[derive(Default, Debug)] pub(crate) struct ChangeLogBuilder { pub changes: VecMap, pub vanished: VecMap, + pub private: VecMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct PrivateLog { + pub collection: SyncCollection, + pub viewer: u32, } #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum VanishedItem { Name(String), Id(u64), - IdPair(u32, u32), } #[derive(Default, Debug)] @@ -29,34 +237,115 @@ pub(crate) struct VanishedItems(Vec); #[derive(Default, Debug)] pub struct Changes { - pub item_inserts: AHashSet, - pub item_updates: AHashSet, - pub item_deletes: AHashSet, + pub item_inserts: ChangeSet, + pub item_updates: ChangeSet, + pub item_deletes: ChangeSet, - pub container_inserts: AHashSet, - pub container_updates: AHashSet, - pub container_deletes: AHashSet, - pub container_property_changes: AHashSet, + pub container_inserts: ChangeSet, + pub container_updates: ChangeSet, + pub container_deletes: ChangeSet, + pub container_property_changes: ChangeSet, + + pub metadata: Option>, + + pending: IndexSet, +} + +#[derive(Default, Debug)] +pub struct MetadataChanges { + pub containers: ChangeSet, + pub items: ChangeSet, +} + +impl MetadataChanges { + #[cold] + fn bytes_hint(&self, item_bytes_hint: usize) -> usize { + 1 + CHANGE_LISTS - BASE_LISTS + + (self.containers.len() * CONTAINER_BYTES_HINT) + + (self.items.len() * item_bytes_hint) + } +} + +impl Changes { + fn build_id(&mut self, prefix: Option, document_id: PendingId) -> u64 { + match (prefix, document_id) { + (None, PendingId::Assigned(document_id)) => document_id as u64, + (Some(PendingId::Assigned(prefix)), PendingId::Assigned(document_id)) => { + debug_assert_ne!(prefix, u32::MAX, "prefix collides with the pending marker"); + ((prefix as u64) << 32) | document_id as u64 + } + _ => { + let change = PendingChange { + prefix, + id: document_id, + }; + PENDING_ID_MARKER | self.pending.insert_full(change).0 as u64 + } + } + } + + fn resolve_id(&self, id: u64, ids: &AssignedIds) -> u64 { + if !is_pending_id(id) { + return id; + } + + let Some(change) = self + .pending + .get_index((id & u32::MAX as u64) as usize) + .copied() + else { + debug_assert!(false, "pending change is out of range"); + return id; + }; + let document_id = change.id.resolve(ids) as u64; + + match change.prefix { + Some(prefix) => ((prefix.resolve(ids) as u64) << 32) | document_id, + None => document_id, + } + } + + #[inline(always)] + pub fn has_pending(&self) -> bool { + !self.pending.is_empty() + } + + #[inline(always)] + fn supersede_container_metadata(&mut self, id: u64) { + if let Some(metadata) = &mut self.metadata { + metadata.containers.remove(&id); + } + } + + #[inline(always)] + fn supersede_item_metadata(&mut self, id: u64) { + if let Some(metadata) = &mut self.metadata { + metadata.items.remove(&id); + } + } } impl ChangeLogBuilder { - pub fn log_container_insert(&mut self, collection: SyncCollection, document_id: u32) { + pub fn log_container_insert(&mut self, collection: SyncCollection, document_id: PendingId) { let changes = self.changes.get_mut_or_insert(collection); - if changes.container_deletes.remove(&document_id) { - changes.container_updates.insert(document_id); + let id = changes.build_id(None, document_id); + changes.supersede_container_metadata(id); + if changes.container_deletes.remove(&id) { + changes.container_updates.insert(id); } else { - changes.container_inserts.insert(document_id); + changes.container_inserts.insert(id); } } pub fn log_item_insert( &mut self, collection: SyncCollection, - prefix: Option, - document_id: u32, + prefix: Option, + document_id: PendingId, ) { - let id = build_id(prefix, document_id); let changes = self.changes.get_mut_or_insert(collection); + let id = changes.build_id(prefix, document_id); + changes.supersede_item_metadata(id); if changes.item_deletes.remove(&id) { changes.item_updates.insert(id); } else { @@ -64,52 +353,97 @@ impl ChangeLogBuilder { } } - pub fn log_container_update(&mut self, collection: SyncCollection, document_id: u32) { - self.changes - .get_mut_or_insert(collection) - .container_updates - .insert(document_id); + pub fn log_container_update(&mut self, collection: SyncCollection, document_id: PendingId) { + let changes = self.changes.get_mut_or_insert(collection); + let id = changes.build_id(None, document_id); + changes.supersede_container_metadata(id); + changes.container_updates.insert(id); } - pub fn log_container_property_update(&mut self, collection: SyncCollection, document_id: u32) { - self.changes - .get_mut_or_insert(collection) - .container_property_changes - .insert(document_id); + pub fn log_container_property_update( + &mut self, + collection: SyncCollection, + document_id: PendingId, + ) { + let changes = self.changes.get_mut_or_insert(collection); + let id = changes.build_id(None, document_id); + changes.container_property_changes.insert(id); } pub fn log_item_update( &mut self, collection: SyncCollection, - prefix: Option, - document_id: u32, + prefix: Option, + document_id: PendingId, ) { - self.changes - .get_mut_or_insert(collection) - .item_updates - .insert(build_id(prefix, document_id)); + let changes = self.changes.get_mut_or_insert(collection); + let id = changes.build_id(prefix, document_id); + changes.supersede_item_metadata(id); + changes.item_updates.insert(id); } - pub fn log_container_delete(&mut self, collection: SyncCollection, document_id: u32) { + pub fn log_container_delete(&mut self, collection: SyncCollection, document_id: PendingId) { let changes = self.changes.get_mut_or_insert(collection); - let id = document_id; + let id = changes.build_id(None, document_id); changes.container_updates.remove(&id); changes.container_property_changes.remove(&id); + changes.supersede_container_metadata(id); changes.container_deletes.insert(id); } pub fn log_item_delete( &mut self, collection: SyncCollection, - prefix: Option, - document_id: u32, + prefix: Option, + document_id: PendingId, ) { let changes = self.changes.get_mut_or_insert(collection); - let id = build_id(prefix, document_id); + let id = changes.build_id(prefix, document_id); changes.item_updates.remove(&id); + changes.supersede_item_metadata(id); changes.item_deletes.insert(id); } + pub fn log_container_metadata(&mut self, collection: SyncCollection, document_id: PendingId) { + self.changes + .get_mut_or_insert(collection) + .container_metadata_change(document_id); + } + + pub fn log_item_metadata( + &mut self, + collection: SyncCollection, + prefix: Option, + document_id: PendingId, + ) { + self.changes + .get_mut_or_insert(collection) + .item_metadata_change(prefix, document_id); + } + + pub fn log_private_container_metadata( + &mut self, + collection: SyncCollection, + viewer: u32, + document_id: PendingId, + ) { + self.private + .get_mut_or_insert(PrivateLog { collection, viewer }) + .container_metadata_change(document_id); + } + + pub fn log_private_item_metadata( + &mut self, + collection: SyncCollection, + viewer: u32, + prefix: Option, + document_id: PendingId, + ) { + self.private + .get_mut_or_insert(PrivateLog { collection, viewer }) + .item_metadata_change(prefix, document_id); + } + pub fn log_vanished_item( &mut self, collection: VanishedCollection, @@ -122,69 +456,209 @@ impl ChangeLogBuilder { } } -#[inline(always)] -fn build_id(prefix: Option, document_id: u32) -> u64 { - if let Some(prefix) = prefix { - ((prefix as u64) << 32) | document_id as u64 - } else { - document_id as u64 - } -} - impl Changes { + fn container_metadata_change(&mut self, document_id: PendingId) { + let id = self.build_id(None, document_id); + if !self.container_inserts.contains(&id) + && !self.container_updates.contains(&id) + && !self.container_deletes.contains(&id) + { + self.metadata.get_or_insert_default().containers.insert(id); + } + } + + fn item_metadata_change(&mut self, prefix: Option, document_id: PendingId) { + let id = self.build_id(prefix, document_id); + if !self.item_inserts.contains(&id) + && !self.item_updates.contains(&id) + && !self.item_deletes.contains(&id) + { + self.metadata.get_or_insert_default().items.insert(id); + } + } + pub fn has_container_changes(&self) -> bool { !self.container_inserts.is_empty() || !self.container_updates.is_empty() || !self.container_property_changes.is_empty() || !self.container_deletes.is_empty() + || self + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.containers.is_empty()) } pub fn has_item_changes(&self) -> bool { !self.item_inserts.is_empty() || !self.item_updates.is_empty() || !self.item_deletes.is_empty() + || self + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.items.is_empty()) } } -impl SerializeInfallible for Changes { - fn serialize(&self) -> Vec { - let mut buf = Vec::with_capacity( - 1 + (self.item_inserts.len() - + self.item_updates.len() - + self.item_deletes.len() - + self.container_inserts.len() - + self.container_updates.len() - + self.container_property_changes.len() - + self.container_deletes.len() - + 4) - * std::mem::size_of::(), - ); +impl Changes { + pub fn serialize(&self, is_prefixed: bool, scratch: &mut Vec) -> Option> { + if self.has_pending() { + return None; + } + let mut buf = Vec::new(); + self.write_into(is_prefixed, None, scratch, &mut buf); + Some(buf) + } - buf.push_leb128(self.container_inserts.len()); - buf.push_leb128(self.container_updates.len()); - buf.push_leb128(self.container_property_changes.len()); - buf.push_leb128(self.container_deletes.len()); - buf.push_leb128(self.item_inserts.len()); - buf.push_leb128(self.item_updates.len()); - buf.push_leb128(self.item_deletes.len()); + pub fn serialize_into( + &self, + is_prefixed: bool, + ids: &AssignedIds, + scratch: &mut Vec, + buf: &mut Vec, + ) { + let pending = self.has_pending().then_some(ids); + self.write_into(is_prefixed, pending, scratch, buf); + } - for list in [ - &self.container_inserts, - &self.container_updates, - &self.container_property_changes, - &self.container_deletes, - ] { - for id in list { - buf.push_leb128(*id); + fn write_into( + &self, + is_prefixed: bool, + ids: Option<&AssignedIds>, + scratch: &mut Vec, + buf: &mut Vec, + ) { + let container_lists = [ + (CONTAINER_INSERTS, &self.container_inserts), + (CONTAINER_UPDATES, &self.container_updates), + (CONTAINER_PROPERTY_CHANGES, &self.container_property_changes), + (CONTAINER_DELETES, &self.container_deletes), + ]; + let item_lists = [ + (ITEM_INSERTS, &self.item_inserts), + (ITEM_UPDATES, &self.item_updates), + (ITEM_DELETES, &self.item_deletes), + ]; + + let mut presence = 0u8; + for (slot, list) in container_lists.iter() { + if !list.is_empty() { + presence |= 1 << slot; } } - for list in [&self.item_inserts, &self.item_updates, &self.item_deletes] { - for id in list { - buf.push_leb128(*id); + for (slot, list) in item_lists.iter() { + if !list.is_empty() { + presence |= 1 << slot; } } - buf + let containers = self.container_inserts.len() + + self.container_updates.len() + + self.container_property_changes.len() + + self.container_deletes.len(); + let items = self.item_inserts.len() + self.item_updates.len() + self.item_deletes.len(); + let item_bytes_hint = if is_prefixed { + PREFIXED_ITEM_BYTES_HINT + } else { + ITEM_BYTES_HINT + }; + + let mut capacity = + 1 + BASE_LISTS + (containers * CONTAINER_BYTES_HINT) + (items * item_bytes_hint); + if let Some(metadata) = &self.metadata { + capacity += metadata.bytes_hint(item_bytes_hint); + } + + buf.clear(); + buf.reserve(capacity); + buf.push(presence); + + for (slot, list) in container_lists.iter() { + if presence & (1 << slot) != 0 { + buf.push_leb128(list.len()); + } + } + for (slot, list) in item_lists.iter() { + if presence & (1 << slot) != 0 { + buf.push_leb128(list.len()); + } + } + + for (slot, list) in container_lists.iter() { + if presence & (1 << slot) == 0 { + continue; + } + self.collect(list, scratch, ids); + write_ids(buf, scratch); + } + + for (slot, list) in item_lists.iter() { + if presence & (1 << slot) == 0 { + continue; + } + self.collect(list, scratch, ids); + if is_prefixed { + write_prefixed_ids(buf, scratch); + } else { + write_ids(buf, scratch); + } + } + + if let Some(metadata) = &self.metadata { + self.serialize_metadata(metadata, is_prefixed, ids, scratch, buf); + } + } + + #[cold] + fn serialize_metadata( + &self, + metadata: &MetadataChanges, + is_prefixed: bool, + ids: Option<&AssignedIds>, + scratch: &mut Vec, + buf: &mut Vec, + ) { + let lists = [ + (CONTAINER_METADATA, &metadata.containers), + (ITEM_METADATA, &metadata.items), + ]; + let mut extended = 0u8; + for (slot, list) in lists.iter() { + if !list.is_empty() { + extended |= 1 << (slot - BASE_LISTS); + } + } + if extended == 0 { + return; + } + if let Some(presence) = buf.first_mut() { + *presence |= PRESENCE_EXTENDED; + } + buf.push(extended); + + for (_, list) in lists.iter().filter(|(_, list)| !list.is_empty()) { + buf.push_leb128(list.len()); + } + for (slot, list) in lists.iter().filter(|(_, list)| !list.is_empty()) { + self.collect(list, scratch, ids); + if is_prefixed && *slot == ITEM_METADATA { + write_prefixed_ids(buf, scratch); + } else { + write_ids(buf, scratch); + } + } + } + + fn collect(&self, list: &ChangeSet, scratch: &mut Vec, ids: Option<&AssignedIds>) { + let Some(ids) = ids else { + list.collect_sorted(scratch); + return; + }; + + list.collect_unsorted(scratch); + for id in scratch.iter_mut() { + *id = self.resolve_id(*id, ids); + } + scratch.sort_unstable(); } } @@ -194,12 +668,6 @@ impl From for VanishedItem { } } -impl From for VanishedItem { - fn from(value: u64) -> Self { - VanishedItem::Id(value) - } -} - impl From<(u32, u32)> for VanishedItem { fn from(value: (u32, u32)) -> Self { VanishedItem::Id((value.0 as u64) << 32 | value.1 as u64) @@ -209,26 +677,106 @@ impl From<(u32, u32)> for VanishedItem { impl VanishedItem { pub fn serialized_size(&self) -> usize { match self { - VanishedItem::Name(name) => name.len() + 1, - VanishedItem::Id(_) | VanishedItem::IdPair(..) => U64_LEN, + VanishedItem::Name(name) => name.len() + U16_LEN, + VanishedItem::Id(_) => U16_LEN, } } } -impl SerializeInfallible for VanishedItems { - fn serialize(&self) -> Vec { - let mut buf = KeySerializer::new(64); +fn shared_prefix_len(prev: &str, name: &str) -> usize { + let prev = prev.as_bytes(); + let bytes = name.as_bytes(); + let max_len = prev.len().min(bytes.len()); + let mut len = 0; - for item in &self.0 { - buf = match item { - VanishedItem::Name(name) => buf.write(name.as_bytes()).write(0u8), - VanishedItem::Id(id) => buf.write(id.to_be_bytes().as_slice()), - VanishedItem::IdPair(a, b) => buf - .write(a.to_be_bytes().as_slice()) - .write(b.to_be_bytes().as_slice()), - }; + while len + U64_LEN <= max_len { + let a = u64::from_le_bytes(prev[len..len + U64_LEN].try_into().unwrap()); + let b = u64::from_le_bytes(bytes[len..len + U64_LEN].try_into().unwrap()); + if a != b { + len += (a ^ b).trailing_zeros() as usize / 8; + break; + } + len += U64_LEN; + } + while len < max_len && prev[len] == bytes[len] { + len += 1; + } + while len > 0 && !name.is_char_boundary(len) { + len -= 1; + } + + len +} + +impl VanishedItems { + pub fn serialize(&self, is_named: bool, scratch: &mut Vec) -> Vec { + let mut buf = Vec::with_capacity(if is_named { + self.0.iter().map(|item| item.serialized_size()).sum() + } else { + self.0.len() * VANISHED_ID_BYTES_HINT + U64_LEN + }); + + if is_named { + let mut prev: Option<&str> = None; + + for item in &self.0 { + match item { + VanishedItem::Name(name) => { + if let Some(prev) = prev { + let shared = shared_prefix_len(prev, name); + buf.push_leb128(shared); + buf.push_leb128(name.len() - shared); + buf.extend_from_slice(&name.as_bytes()[shared..]); + } else { + buf.push_leb128(name.len()); + buf.extend_from_slice(name.as_bytes()); + } + prev = Some(name); + } + VanishedItem::Id(_) => { + debug_assert!(false, "id logged for a name-based vanished collection"); + } + } + } + + return buf; } - buf.finalize() + scratch.clear(); + scratch.extend(self.0.iter().filter_map(|item| match item { + VanishedItem::Id(id) => Some(*id), + VanishedItem::Name(_) => { + debug_assert!(false, "name logged for an id-based vanished collection"); + None + } + })); + scratch.sort_unstable(); + scratch.dedup(); + + let mut pos = 0; + let mut prev_group = 0u64; + while pos < scratch.len() { + let group = scratch[pos] >> 32; + let mut end = pos; + while end < scratch.len() && scratch[end] >> 32 == group { + end += 1; + } + + buf.push_leb128(group - prev_group); + buf.push_leb128(end - pos); + + let mut prev_id = 0u64; + for id in &scratch[pos..end] { + let id = *id & u32::MAX as u64; + buf.push_leb128(id - prev_id); + prev_id = id; + } + + prev_group = group; + pos = end; + } + + buf } } + diff --git a/crates/store/src/write/metadata.rs b/crates/store/src/write/metadata.rs new file mode 100644 index 00000000..05c3400d --- /dev/null +++ b/crates/store/src/write/metadata.rs @@ -0,0 +1,184 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Archive, ArchiveBytes, ArchiveVersion, ValueClass}; +use crate::{Deserialize, SerializeInfallible, U32_LEN, U64_LEN}; +use types::metadata::{EncodedMetadata, MetadataKinds, MetadataView, STORAGE_TRAILER_CAPACITY}; + +pub const METADATA_COMPRESS_WATERMARK: usize = 1024; + +const TRAILER_LEN: usize = U32_LEN + 1; +const _: () = assert!(TRAILER_LEN == STORAGE_TRAILER_CAPACITY); + +#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] +pub enum MetadataClass { + Shared, + Private { viewer: u32 }, + Viewer { viewer: u32 }, + Owner { owner: u32 }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredMetadata { + bytes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataBuf { + bytes: Box<[u8]>, + stored_len: u32, + hash: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)] +pub struct ViewerState { + pub change_id: u64, + pub containers: u32, +} + +const VIEWER_STATE_LEN: usize = U64_LEN + U32_LEN; + +impl MetadataClass { + pub(crate) const SHARED: u8 = 0; + pub(crate) const PRIVATE: u8 = 1; + pub const VIEWER: u8 = 2; + pub const OWNER: u8 = 3; +} + +impl From for ValueClass { + fn from(class: MetadataClass) -> Self { + ValueClass::Metadata(class) + } +} + +impl StoredMetadata { + pub fn new(container: EncodedMetadata) -> trc::Result { + Archive::serialize_raw(container.into_bytes(), METADATA_COMPRESS_WATERMARK) + .map(|bytes| StoredMetadata { bytes }) + } + + pub fn trailer_hash(stored: &[u8]) -> trc::Result { + stored + .split_last() + .and_then(|(_, contents)| contents.last_chunk::()) + .map(|hash| u32::from_be_bytes(*hash)) + .ok_or_else(|| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Metadata container carries no trailer") + .ctx(trc::Key::Value, stored) + .caused_by(trc::location!()) + }) + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } + + pub fn view<'x>(stored: &'x [u8], scratch: &'x mut Vec) -> trc::Result> { + let contents = Archive::::deserialize_raw(stored, scratch)?; + unsafe { MetadataView::from_trusted(contents) }.ok_or_else(|| invalid_header(contents)) + } +} + +#[cold] +fn invalid_header(contents: &[u8]) -> trc::Error { + trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid metadata container header") + .ctx(trc::Key::Value, contents) + .caused_by(trc::location!()) +} + +impl MetadataBuf { + pub fn from_view(view: &MetadataView<'_>, stored_len: u32, hash: u32) -> Self { + MetadataBuf { + bytes: view.as_bytes().into(), + stored_len, + hash, + } + } + + pub fn read(stored: &[u8]) -> trc::Result { + let archive = as Deserialize>::deserialize(stored)?; + match archive.version { + ArchiveVersion::Hashed { hash } + if unsafe { MetadataView::from_trusted(&archive.inner) }.is_some() => + { + Ok(MetadataBuf { + bytes: archive.inner.into_boxed_slice(), + stored_len: u32::try_from(stored.len()).unwrap_or(u32::MAX), + hash, + }) + } + _ => Err(invalid_header(&archive.inner)), + } + } + + pub fn view(&self) -> MetadataView<'_> { + unsafe { MetadataView::from_trusted(&self.bytes) }.unwrap_or_else(MetadataView::empty) + } + + pub fn kinds(&self) -> MetadataKinds { + self.view().kinds() + } + + pub fn stored_len(&self) -> u32 { + self.stored_len + } + + pub fn hash(&self) -> u32 { + self.hash + } +} + +impl Deserialize for MetadataBuf { + fn deserialize(bytes: &[u8]) -> trc::Result { + Self::read(bytes) + } +} + +impl SerializeInfallible for ViewerState { + fn serialize(&self) -> Vec { + let mut bytes = Vec::with_capacity(VIEWER_STATE_LEN); + bytes.extend_from_slice(&self.change_id.to_be_bytes()); + bytes.extend_from_slice(&self.containers.to_be_bytes()); + bytes + } +} + +impl Deserialize for ViewerState { + fn deserialize(bytes: &[u8]) -> trc::Result { + bytes + .split_first_chunk::() + .and_then(|(change_id, containers)| { + Some(ViewerState { + change_id: u64::from_be_bytes(*change_id), + containers: u32::from_be_bytes(containers.try_into().ok()?), + }) + }) + .ok_or_else(|| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid metadata viewer state") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + }) + } +} + diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index d52fda2b..de6d64e5 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -5,33 +5,52 @@ */ use self::assert::AssertValue; -use crate::backend::MAX_TOKEN_LENGTH; +use crate::{ + ChangeLogKey, IndexKey, Key, LogKey, PrivateLogKey, Subspace, U32_LEN, U64_LEN, + backend::MAX_TOKEN_LENGTH, +}; use log::ChangeLogBuilder; +use metadata::MetadataClass; use nlp::tokenizers::word::WordTokenizer; -use rkyv::util::AlignedVec; -use std::{collections::HashSet, hash::Hash, time::SystemTime}; +use registry::{ + schema::{ + enums::{StorageQuota, TaskType}, + structs::Task, + }, + types::{EnumImpl, ObjectImpl}, +}; +use std::{borrow::Cow, collections::HashSet, hash::Hash, time::SystemTime}; +use tinyvec::TinyVec; use types::{ blob_hash::BlobHash, - collection::{Collection, SyncCollection, VanishedCollection}, + collection::{ChangeGroup, Collection, SyncCollection, VanishedCollection}, field::{ CalendarEventField, CalendarNotificationField, ContactField, EmailField, EmailSubmissionField, Field, MailboxField, PrincipalField, SieveField, }, + id::Id, }; use utils::{ cheeky_hash::CheekyHash, + hash128::Hash128, map::{bitmap::Bitmap, vec_map::VecMap}, }; +use xxhash_rust::xxh3::xxh3_64; pub mod assert; pub mod batch; pub mod bitpack; pub mod blob; +pub mod compress; +pub mod counter; pub mod key; +pub mod lazybitmap; pub mod log; +pub mod metadata; pub mod serialize; -pub(crate) const ARCHIVE_ALIGNMENT: usize = 16; +pub use compress::{ArchiveCompression, Compression, Dictionary}; +pub use counter::{AccountCounter, CounterBound, QueueCounter}; #[derive(Debug, Clone)] pub struct Archive { @@ -46,18 +65,20 @@ pub enum ArchiveVersion { Unversioned, } -#[derive(Debug, Clone)] -pub enum AlignedBytes { - Aligned(AlignedVec), - Vec(Vec), -} +pub type ArchiveBytes = Vec; + +const _: () = assert!(std::mem::size_of::() == 4); +const _: () = assert!(std::mem::align_of::() == 1); +const _: () = assert!(std::mem::align_of::() == 1); +const _: () = assert!(std::mem::size_of::() <= 48); +const _: () = assert!(std::mem::size_of::() <= 88); pub struct Archiver where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, + Vec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, @@ -65,71 +86,439 @@ where { pub inner: T, pub flags: u8, + pub compression: Compression, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +#[repr(transparent)] +pub struct Slot(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct SlotRange { + first: Slot, + count: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PendingId { + Assigned(u32), + Slot(Slot), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ChangeCounter { + pub account_id: u32, + pub group: ChangeGroup, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct ChangeId { + pub counter: ChangeCounter, + pub change_id: u64, } #[derive(Debug, Default)] pub struct AssignedIds { - pub ids: Vec, - current_change_id: Option, + slots: TinyVec<[u32; 8]>, + change_ids: TinyVec<[ChangeId; 2]>, + counter: Option, + archive_hash: Option, + current_change_id: u64, } -#[derive(Debug)] -pub enum AssignedId { - Counter(i64), - ChangeId(ChangeId), +#[derive(Debug, Clone, Copy, Default)] +pub struct CommitPointOffsets { + pub ops: usize, + pub reservations: usize, + pub change_accounts: usize, + pub bounds: usize, } #[derive(Debug, Clone, Copy)] -pub struct ChangeId { - pub account_id: u32, - pub change_id: u64, +pub struct AssignedIdsMark { + slots: usize, + change_ids: usize, + counter: Option, + archive_hash: Option, } #[cfg(any( feature = "rocks", feature = "postgres", feature = "mysql", - feature = "foundation" + feature = "foundation", + feature = "sqlite" ))] pub(crate) use commit_limits::{MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME}; +#[cfg(any( + feature = "rocks", + feature = "postgres", + feature = "mysql", + feature = "foundation", + feature = "sqlite" +))] +pub(crate) use commit_limits::commit_backoff; + #[cfg(any( feature = "rocks", feature = "postgres", feature = "mysql", feature = "foundation" ))] +pub(crate) use commit_limits::exhausted; + +#[cfg(all( + stalwart_tests, + any( + feature = "rocks", + feature = "postgres", + feature = "mysql", + feature = "foundation", + feature = "sqlite" + ) +))] +pub(crate) use commit_limits::record_retry; + +#[cfg(any(feature = "postgres", feature = "mysql"))] +pub(crate) use commit_limits::{ChunkPlan, ChunkedRetry, backoff}; + +#[cfg(any(feature = "postgres", feature = "mysql"))] +mod new_key; + +#[cfg(any(feature = "postgres", feature = "mysql"))] +pub(crate) use new_key::{NewKeyAsserts, insert_follows}; + +#[cfg(any( + feature = "rocks", + feature = "postgres", + feature = "mysql", + feature = "foundation", + feature = "sqlite" +))] +#[cfg_attr(stalwart_tests, path = ".")] mod commit_limits { + #[cfg(any(feature = "postgres", feature = "mysql"))] + use crate::{Fetch, ITERATE_BATCH_GROWTH}; + use rand::RngExt; use std::time::Duration; - #[cfg(not(feature = "test_mode"))] - pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 10; - #[cfg(not(feature = "test_mode"))] + const PRODUCTION_MAX_COMMIT_ATTEMPTS: u32 = 24; + + #[cfg(not(stalwart_tests))] + pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = PRODUCTION_MAX_COMMIT_ATTEMPTS; + #[cfg(not(stalwart_tests))] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(10); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 1000; - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); + + const MIN_COMMIT_BACKOFF_US: u64 = 250; + const MAX_COMMIT_BACKOFF_US: u64 = 1_000_000; + const MAX_COMMIT_BACKOFF_SHIFT: u32 = 12; + + pub(crate) fn commit_backoff(attempt: u32) -> Duration { + #[cfg(stalwart_tests)] + crate::dispatch::StoreOps::count_commit_retry(); + + let ceiling = (MIN_COMMIT_BACKOFF_US << attempt.min(MAX_COMMIT_BACKOFF_SHIFT)) + .min(MAX_COMMIT_BACKOFF_US); + + Duration::from_micros(rand::rng().random_range(0..=ceiling)) + } + + + #[cfg(stalwart_tests)] + pub(crate) use test_support::record_retry; + + #[cfg(any( + feature = "rocks", + feature = "postgres", + feature = "mysql", + feature = "foundation" + ))] + pub(crate) fn exhausted(err: trc::Error, retry_count: u32) -> trc::Error { + err.ctx(trc::Key::Attempt, retry_count + 1) + } + + #[cfg(any(feature = "postgres", feature = "mysql"))] + pub(crate) struct ChunkedRetry { + chunk_size: Option, + first_chunk_size: usize, + min_chunk_size: usize, + retry_count: u32, + } + + #[cfg(any(feature = "postgres", feature = "mysql"))] + impl ChunkedRetry { + pub(crate) fn bounded(first_chunk_size: usize, min_chunk_size: usize) -> Self { + Self { + chunk_size: Some(first_chunk_size), + first_chunk_size, + min_chunk_size, + retry_count: 0, + } + } + + pub(crate) fn chunk_size(&self) -> Option { + self.chunk_size + } + + pub(crate) fn progressed(&mut self) { + self.retry_count = 0; + } + + pub(crate) async fn degrade(&mut self) -> bool { + match self.chunk_size { + None => { + self.chunk_size = Some(self.first_chunk_size); + true + } + Some(chunk_size) if chunk_size > self.min_chunk_size => { + self.chunk_size = Some((chunk_size / 2).max(self.min_chunk_size)); + true + } + Some(_) => backoff(&mut self.retry_count).await, + } + } + } + + #[cfg(any(feature = "postgres", feature = "mysql"))] + pub(crate) async fn backoff(attempt: &mut u32) -> bool { + if *attempt < MAX_COMMIT_ATTEMPTS { + tokio::time::sleep(commit_backoff(*attempt)).await; + *attempt += 1; + true + } else { + false + } + } + + #[cfg(any(feature = "postgres", feature = "mysql"))] + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) enum ChunkPlan { + Unbounded, + Growing { next: u32 }, + Fixed { next: u32 }, + Capped { remaining: u32, next: u32 }, + } + + #[cfg(any(feature = "postgres", feature = "mysql"))] + impl ChunkPlan { + pub(crate) fn new(fetch: Fetch) -> Self { + match fetch { + Fetch::All => ChunkPlan::Unbounded, + Fetch::Expect(rows) => ChunkPlan::Growing { + next: rows.saturating_add(1), + }, + Fetch::Limit(rows) => ChunkPlan::Capped { + remaining: rows, + next: rows, + }, + } + } + + pub(crate) fn limit(self, resuming: bool) -> Option { + let rows = match self { + ChunkPlan::Unbounded => return None, + ChunkPlan::Growing { next } | ChunkPlan::Fixed { next } => next, + ChunkPlan::Capped { remaining, next } => remaining.min(next), + }; + Some(rows.saturating_add(u32::from(resuming))) + } + + pub(crate) fn consume_row(&mut self) -> bool { + match self { + ChunkPlan::Capped { remaining, .. } => { + *remaining = remaining.saturating_sub(1); + *remaining > 0 + } + _ => true, + } + } + + pub(crate) fn grow(&mut self) { + if let ChunkPlan::Growing { next } = self { + *next = next.saturating_mul(ITERATE_BATCH_GROWTH); + } + } + + fn shrink(&mut self, first: u32, min: u32) -> bool { + match *self { + ChunkPlan::Unbounded => { + *self = ChunkPlan::Fixed { next: first }; + true + } + ChunkPlan::Growing { next } | ChunkPlan::Fixed { next } => { + *self = ChunkPlan::Fixed { + next: Self::halve(next, first, min), + }; + next > min + } + ChunkPlan::Capped { remaining, next } => { + let current = remaining.min(next); + *self = ChunkPlan::Capped { + remaining, + next: Self::halve(current, first, min), + }; + current > min + } + } + } + + fn halve(current: u32, first: u32, min: u32) -> u32 { + (current / 2).min(first).max(min).min(current) + } + + pub(crate) async fn degrade( + &mut self, + first: usize, + min: usize, + attempt: &mut u32, + ) -> bool { + self.shrink( + u32::try_from(first).unwrap_or(u32::MAX), + u32::try_from(min).unwrap_or(u32::MAX), + ) || backoff(attempt).await + } + } } +const UNAVAILABLE_RETRY_AFTER: u64 = 1; +const EXHAUSTED_RETRY_AFTER: u64 = 5; + +pub fn unavailable_retry_after(err: &trc::Error) -> Option { + if !matches!( + err.event_type(), + trc::EventType::Store(trc::StoreEvent::FoundationdbError) + ) { + return None; + } + + match err.value_as_uint(trc::Key::Code)? { + 1007 | 1009 | 1020 | 1021 | 1037 | 1038 | 1039 | 1042 | 1051 | 1078 | 1213 | 1223 + | 1235 | 1242 => Some(if err.value(trc::Key::Attempt).is_some() { + EXHAUSTED_RETRY_AFTER + } else { + UNAVAILABLE_RETRY_AFTER + }), + _ => None, + } +} + +pub fn is_commit_contention(err: &trc::Error) -> bool { + matches!( + err.event_type(), + trc::EventType::Store( + trc::StoreEvent::FoundationdbError + | trc::StoreEvent::RocksdbError + | trc::StoreEvent::PostgresqlError + | trc::StoreEvent::MysqlError + ) + ) && err.value(trc::Key::Attempt).is_some() +} + + #[derive(Debug)] pub struct Batch<'x> { - pub(crate) changes: &'x VecMap, + pub(crate) change_accounts: &'x [ChangeCounter], + pub(crate) reservations: &'x [Reservation], + pub(crate) bounds: &'x [CounterBound], pub(crate) ops: &'x mut [Operation], + #[cfg_attr(not(feature = "foundation"), allow(dead_code))] + pub(crate) priority: CommitPriority, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommitPriority { + Default, + Background, +} + +#[derive(Debug, Clone, Copy)] +pub struct Reservation { + pub class: ReservationClass, + pub first_slot: Slot, + pub count: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReservationClass { + DocumentId { + account_id: u32, + collection: Collection, + }, + Uid { + account_id: u32, + mailbox_id: u32, + }, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct QueueNotify { + pub task_types: TaskTypes, + pub search_index: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TaskTypes(u32); + +const _: () = assert!(TaskType::COUNT <= u32::BITS as usize); + +impl TaskTypes { + pub const ALL: TaskTypes = TaskTypes(u32::MAX >> (u32::BITS as usize - TaskType::COUNT)); + + pub const fn from_bits(bits: u32) -> Self { + TaskTypes(bits & Self::ALL.0) + } + + pub const fn bits(self) -> u32 { + self.0 + } + + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + pub fn insert(&mut self, typ: TaskType) { + self.0 |= Self::bit(typ); + } + + pub fn union(&mut self, other: TaskTypes) { + self.0 |= other.0; + } + + pub fn contains(self, typ: TaskType) -> bool { + self.0 & Self::bit(typ) != 0 + } + + const fn bit(typ: TaskType) -> u32 { + 1 << typ as u32 + } } #[derive(Debug)] pub struct BatchBuilder { current_account_id: Option, current_collection: Option, - current_document_id: Option, + current_document_id: Option, changes: VecMap, changed_collections: VecMap, - has_assertions: bool, + change_accounts: Vec, + reservations: Vec, + bounds: Vec, + counters: counter::PendingCounters, + next_slot: u32, batch_size: usize, batch_ops: usize, - commit_points: Vec, + commit_points: Vec, + last_archive_hash: Option, + last_index_partition: Option<(SearchIndex, u32)>, + has_index_tasks: bool, + task_types: TaskTypes, + priority: CommitPriority, ops: Vec, } @@ -140,7 +529,7 @@ pub struct ChangedCollection { pub share_notification_id: Option, } -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug)] pub enum Operation { AccountId { account_id: u32, @@ -149,7 +538,7 @@ pub enum Operation { collection: Collection, }, DocumentId { - document_id: u32, + document_id: PendingId, }, AssertValue { class: ValueClass, @@ -166,22 +555,357 @@ pub enum Operation { }, Log { collection: LogCollection, - set: Vec, + set: LogSet, }, + ClearRange { + subspace: Subspace, + from: Vec, + to: Vec, + }, +} + +#[derive(Debug)] +pub enum LogSet { + Bytes(Vec), + Pending(Box), } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub enum LogCollection { Sync(SyncCollection), Vanished(VanishedCollection), + Private { + collection: SyncCollection, + viewer: u32, + }, +} + +pub const PRIVATE_LOG: u8 = 0x80; + +impl LogCollection { + #[inline(always)] + pub fn change_group(&self) -> ChangeGroup { + match self { + LogCollection::Sync(collection) | LogCollection::Private { collection, .. } => { + collection.change_group() + } + LogCollection::Vanished(collection) => collection.change_group(), + } + } + + #[inline(always)] + pub fn is_prefixed(&self) -> bool { + match self { + LogCollection::Sync(collection) | LogCollection::Private { collection, .. } => { + collection.is_prefixed() + } + LogCollection::Vanished(_) => false, + } + } + + #[inline(always)] + pub fn log_key(&self, account_id: u32, change_id: u64) -> ChangeLogKey { + match *self { + LogCollection::Private { collection, viewer } => ChangeLogKey::Private(PrivateLogKey { + account_id, + collection, + viewer, + change_id, + }), + collection => ChangeLogKey::Shared(LogKey { + account_id, + collection: u8::from(collection), + change_id, + }), + } + } +} + +impl Batch<'_> { + #[inline(always)] + pub fn has_allocations(&self) -> bool { + self.has_id_allocations() || !self.bounds.is_empty() + } + + #[inline(always)] + pub fn has_id_allocations(&self) -> bool { + !self.change_accounts.is_empty() || !self.reservations.is_empty() + } + + #[inline] + pub fn allocations(&self) -> impl Iterator + '_ { + self.id_allocations() + .chain(self.bounds.iter().map(|bound| Allocation::Bounded(*bound))) + } + + #[inline] + pub fn id_allocations(&self) -> impl Iterator + '_ { + self.change_accounts + .iter() + .map(|counter| Allocation::ChangeId(*counter)) + .chain( + self.reservations + .iter() + .map(|reservation| Allocation::Reserved(*reservation)), + ) + } + + #[inline(always)] + pub fn bounds(&self) -> &[CounterBound] { + self.bounds + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Allocation { + ChangeId(ChangeCounter), + Reserved(Reservation), + Bounded(CounterBound), +} + +#[derive(Debug)] +pub struct BatchCursor { + account_id: u32, + collection: u8, + change_group: ChangeGroup, + document_id: u32, + has_changes: bool, +} + +#[derive(Debug)] +pub enum Advance<'x> { + Cursor, + Value { + class: &'x ValueClass, + op: &'x mut ValueOp, + }, + Index { + field: u8, + key: &'x [u8], + set: bool, + }, + Log { + collection: LogCollection, + set: &'x mut LogSet, + }, + Assert { + class: &'x ValueClass, + assert_value: &'x AssertValue, + }, + ClearRange { + subspace: Subspace, + from: &'x [u8], + to: &'x [u8], + }, +} + +impl BatchCursor { + pub fn new(batch: &Batch<'_>) -> Self { + BatchCursor { + account_id: u32::MAX, + collection: u8::MAX, + change_group: SyncCollection::None.change_group(), + document_id: u32::MAX, + has_changes: !batch.change_accounts.is_empty(), + } + } + + #[inline(always)] + pub fn advance<'x>(&mut self, op: &'x mut Operation, ids: &mut AssignedIds) -> Advance<'x> { + match op { + Operation::AccountId { account_id } => { + self.account_id = *account_id; + if self.has_changes { + ids.set_current_change_id(self.account_id, self.change_group); + } + Advance::Cursor + } + Operation::Collection { collection } => { + self.collection = u8::from(*collection); + self.change_group = collection.change_group(); + if self.has_changes { + ids.set_current_change_id(self.account_id, self.change_group); + } + Advance::Cursor + } + Operation::DocumentId { document_id } => { + self.document_id = document_id.resolve(ids); + Advance::Cursor + } + Operation::Value { class, op } => Advance::Value { class, op }, + Operation::Index { field, key, set } => Advance::Index { + field: *field, + key: key.as_slice(), + set: *set, + }, + Operation::Log { collection, set } => Advance::Log { + collection: *collection, + set, + }, + Operation::AssertValue { + class, + assert_value, + } => Advance::Assert { + class, + assert_value, + }, + Operation::ClearRange { subspace, from, to } => Advance::ClearRange { + subspace: *subspace, + from, + to, + }, + } + } + + #[inline(always)] + pub fn account_id(&self) -> u32 { + self.account_id + } + + #[inline(always)] + pub fn collection(&self) -> u8 { + self.collection + } + + #[inline(always)] + pub fn document_id(&self) -> u32 { + self.document_id + } + + #[inline(always)] + pub fn subspace(&self, class: &ValueClass) -> Subspace { + class.subspace(self.collection) + } + + #[inline(always)] + pub fn value_key(&self, class: &ValueClass, buf: &mut Vec, flags: u32) { + buf.clear(); + class.serialize_into( + buf, + self.account_id, + self.collection, + self.document_id, + flags, + ); + } + + #[inline(always)] + pub fn value_key_owned(&self, class: &ValueClass, flags: u32) -> Vec { + class.serialize(self.account_id, self.collection, self.document_id, flags) + } + + #[inline(always)] + pub fn index_key(&self, field: u8, key: &[u8], buf: &mut Vec, flags: u32) { + buf.clear(); + IndexKey { + account_id: self.account_id, + collection: self.collection, + document_id: self.document_id, + field, + key, + } + .serialize_into(buf, flags); + } + + #[inline(always)] + pub fn index_key_owned(&self, field: u8, key: &[u8], flags: u32) -> Vec { + IndexKey { + account_id: self.account_id, + collection: self.collection, + document_id: self.document_id, + field, + key, + } + .serialize(flags) + } + + #[inline(always)] + pub fn log_key( + &self, + collection: LogCollection, + ids: &AssignedIds, + buf: &mut Vec, + flags: u32, + ) { + let change_id = ids + .change_id(self.account_id, collection.change_group()) + .unwrap_or_default(); + debug_assert!( + change_id != 0, + "no change id was allocated for this account" + ); + buf.clear(); + collection + .log_key(self.account_id, change_id) + .serialize_into(buf, flags); + } +} + +impl Allocation { + pub fn serialize_key_into(&self, buf: &mut Vec, flags: u32) { + match self { + Allocation::ChangeId(counter) => ValueClass::ChangeId(counter.group).serialize_into( + buf, + counter.account_id, + 0, + 0, + flags, + ), + Allocation::Reserved(reservation) => reservation.serialize_key_into(buf, flags), + Allocation::Bounded(bound) => bound.serialize_key_into(buf, flags), + } + } + + #[inline(always)] + pub fn increment_by(&self) -> i64 { + match self { + Allocation::ChangeId(_) => 1, + Allocation::Reserved(reservation) => reservation.count as i64, + Allocation::Bounded(bound) => bound.by, + } + } +} + +impl ReservationClass { + #[inline(always)] + pub fn sort_key(&self) -> (u8, u32, u32) { + match *self { + ReservationClass::DocumentId { + account_id, + collection, + } => (0, account_id, u8::from(collection) as u32), + ReservationClass::Uid { + account_id, + mailbox_id, + } => (1, account_id, mailbox_id), + } + } +} + +impl Reservation { + pub fn serialize_key_into(&self, buf: &mut Vec, flags: u32) { + match self.class { + ReservationClass::DocumentId { + account_id, + collection, + } => { + ValueClass::DocumentId.serialize_into(buf, account_id, collection.into(), 0, flags) + } + ReservationClass::Uid { + account_id, + mailbox_id, + } => ValueClass::MailboxUid.serialize_into(buf, account_id, 0, mailbox_id, flags), + } + } } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum ValueClass { Property(u8), + Immutable(u8), IndexProperty(IndexPropertyClass), + MailboxUid, Acl(u32), - InMemory(InMemoryClass), TaskQueue(TaskQueueClass), Blob(BlobOp), Registry(RegistryClass), @@ -194,50 +918,87 @@ pub enum ValueClass { notify_account_id: u32, }, DocumentId, - ChangeId, + ChangeId(ChangeGroup), Quota, - TenantQuota(u32), + ObjectCount(StorageQuota), NodeId(u16), + Metadata(MetadataClass), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum IndexPropertyClass { - Hash { property: u8, hash: CheekyHash }, + Hash { property: u8, hash: Hash128 }, Integer { property: u8, value: u64 }, } -#[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub struct SearchIndexClass { - pub index: SearchIndex, - pub id: SearchIndexId, - pub typ: SearchIndexType, -} - -#[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub enum SearchIndexType { - Term { field: u8, hash: CheekyHash }, - Index { field: SearchIndexField }, - Document, -} - -pub(crate) const SEARCH_INDEX_MAX_FIELD_LEN: usize = 128; - -#[derive(Debug, PartialEq, Eq, Clone, Hash, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] -pub struct SearchIndexField { - pub(crate) field_id: u8, - pub(crate) data: Vec, -} - #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] -pub enum SearchIndexId { - Account { account_id: u32, document_id: u32 }, - Global { id: u64 }, +pub enum SearchIndexClass { + Term { + index: SearchIndex, + account_id: u32, + field: u8, + term: CheekyHash, + block_id: u16, + }, + Document { + index: SearchIndex, + account_id: u32, + document_id: u32, + }, + GlobalTerm { + index: SearchIndex, + field: u8, + term: CheekyHash, + block_id: u16, + }, + GlobalDocument { + index: SearchIndex, + document_id: u64, + }, + GlobalDocumentId { + index: SearchIndex, + block_id: u16, + }, + Queue { + index: SearchIndex, + id_prefix: u32, + id_suffix: QueueDocumentId, + created_at: u64, + }, + QueueIndex { + index: SearchIndex, + partition: u32, + }, + QueueStatus { + index: SearchIndex, + partition: u32, + }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum TaskQueueClass { - Task { id: u64 }, - Due { id: u64, due: u64 }, + Task { id: TaskId }, + Due { id: TaskId, typ: TaskType, due: u64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TaskDue { + pub typ: TaskType, + pub due: u64, + pub id: u64, +} + +#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] +pub enum TaskId { + Assigned(u64), + Document, + DocumentTarget(u32), +} + +#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] +pub enum QueueDocumentId { + Assigned(u32), + Current, } #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] @@ -252,16 +1013,10 @@ pub enum SearchIndex { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct AnyClass { - pub subspace: u8, + pub subspace: Subspace, pub key: Vec, } -#[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub enum InMemoryClass { - Key(Vec), - Counter(Vec), -} - #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum RegistryClass { Item { @@ -298,8 +1053,8 @@ pub enum RegistryClass { pub enum QueueClass { Message(u64), MessageEvent(QueueEvent), - QuotaCount(Vec), - QuotaSize(Vec), + QuotaCount(Hash128), + QuotaSize(Hash128), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -315,10 +1070,9 @@ pub struct QueueEvent { pub queue_name: [u8; 8], } -#[derive(Debug, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Default)] pub enum ValueOp { - Set(Vec), - SetFnc(SetOperation), + Set(SetValue), MergeFnc(MergeOperation), AtomicAdd(i64), AddAndGet(i64), @@ -326,40 +1080,177 @@ pub enum ValueOp { Clear, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Patch { + pub offset: u32, + pub source: PatchSource, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PatchSource { + SlotBeU32(Slot), + SlotArchivedU32 { slot: Slot, base: u32 }, + ChangeIdBe, +} + +pub const MAX_CONCURRENT_ALLOCATIONS: usize = 16; + pub enum MergeResult { Update(Vec), Skip, Delete, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Param { - I64(i64), - U64(u64), - String(String), - Bytes(Vec), - Bool(bool), +pub type MergeFnc = + Box) -> trc::Result + Send + Sync>; + +pub trait SerializeWithIds: Send + Sync { + fn serialize_with_ids(&mut self, ids: &AssignedIds) -> trc::Result<(Vec, Option)>; + + fn size_hint(&self) -> usize; } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[repr(transparent)] -pub struct Params(Vec); +pub struct SerializeOperation(pub(crate) Box); -pub type SetFnc = fn(&Params, &AssignedIds) -> trc::Result>; -pub type MergeFnc = fn(&Params, &AssignedIds, Option<&[u8]>) -> trc::Result; - -#[derive(Debug, Clone)] -pub struct MergeOperation { - pub(crate) fnc: MergeFnc, - pub(crate) params: Params, +#[derive(Debug)] +pub enum SetValue { + Fixed(Vec), + Patched(Vec, Box<[Patch]>), + Serializable(SerializeOperation), } -#[derive(Debug, Clone)] -pub struct SetOperation { - pub(crate) fnc: SetFnc, - pub(crate) params: Params, +#[derive(Debug)] +pub struct SizedSetValue { + pub value: SetValue, + pub size_hint: usize, } +impl SetValue { + pub fn fixed(payload: Vec) -> SizedSetValue { + SizedSetValue { + size_hint: payload.len(), + value: SetValue::Fixed(payload), + } + } + + pub fn patched(payload: Vec, patches: Vec) -> SizedSetValue { + debug_assert!( + patches + .iter() + .all(|patch| (patch.offset as usize) < payload.len()), + "patch offset is outside the payload" + ); + + SizedSetValue { + size_hint: payload.len(), + value: SetValue::Patched(payload, patches.into_boxed_slice()), + } + } + + pub fn serializable(object: impl SerializeWithIds + 'static) -> SizedSetValue { + SizedSetValue { + size_hint: object.size_hint(), + value: SetValue::Serializable(SerializeOperation(Box::new(object))), + } + } + + pub fn resolve(&mut self, ids: &mut AssignedIds) -> trc::Result> { + match self { + SetValue::Fixed(payload) => Ok(Cow::Borrowed(payload)), + SetValue::Patched(payload, patches) => { + Patch::apply(patches, payload, ids); + Ok(Cow::Borrowed(payload)) + } + SetValue::Serializable(object) => { + let (payload, hash) = (object.0).serialize_with_ids(ids)?; + ids.set_archive_hash(hash); + Ok(Cow::Owned(payload)) + } + } + } + + pub fn len(&self) -> usize { + match self { + SetValue::Fixed(payload) | SetValue::Patched(payload, _) => payload.len(), + SetValue::Serializable(..) => 0, + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl From> for SizedSetValue { + fn from(payload: Vec) -> Self { + SetValue::fixed(payload) + } +} + +impl From<&[u8]> for SizedSetValue { + fn from(payload: &[u8]) -> Self { + SetValue::fixed(payload.to_vec()) + } +} + +impl From<[u8; N]> for SizedSetValue { + fn from(payload: [u8; N]) -> Self { + SetValue::fixed(payload.to_vec()) + } +} + +impl From<&str> for SizedSetValue { + fn from(payload: &str) -> Self { + SetValue::fixed(payload.as_bytes().to_vec()) + } +} + +impl From for SizedSetValue { + fn from(payload: String) -> Self { + SetValue::fixed(payload.into_bytes()) + } +} + +impl From> for SizedSetValue { + fn from(payload: Cow<'_, [u8]>) -> Self { + SetValue::fixed(payload.into_owned()) + } +} + +impl From<(Vec, Vec)> for SizedSetValue { + fn from((payload, patches): (Vec, Vec)) -> Self { + SetValue::patched(payload, patches) + } +} + +impl From for SizedSetValue { + fn from(document_id: PendingId) -> Self { + match document_id { + PendingId::Assigned(document_id) => SetValue::fixed(document_id.to_be_bytes().to_vec()), + PendingId::Slot(slot) => SetValue::patched( + 0u32.to_be_bytes().to_vec(), + vec![Patch { + offset: 0, + source: PatchSource::SlotBeU32(slot), + }], + ), + } + } +} + +impl From> for SizedSetValue { + fn from(object: Box) -> Self { + SizedSetValue { + size_hint: object.size_hint(), + value: SetValue::Serializable(SerializeOperation(object)), + } + } +} + +#[repr(transparent)] +pub struct MergeOperation(pub(crate) MergeFnc); + #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum BlobOp { Commit { hash: BlobHash }, @@ -375,7 +1266,7 @@ pub enum BlobLink { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct AnyKey> { - pub subspace: u8, + pub subspace: Subspace, pub key: T, } @@ -415,62 +1306,347 @@ impl AsRef for ValueClass { } } +impl Slot { + #[inline(always)] + pub fn new(index: u32) -> Self { + Slot(index) + } + + #[inline(always)] + pub fn index(self) -> usize { + self.0 as usize + } +} + +impl SlotRange { + #[inline(always)] + pub fn new(first: Slot, count: u32) -> Self { + SlotRange { first, count } + } + + #[inline(always)] + pub fn first(self) -> Slot { + self.first + } + + #[inline(always)] + pub fn len(self) -> usize { + self.count as usize + } + + #[inline(always)] + pub fn is_empty(self) -> bool { + self.count == 0 + } + + #[inline(always)] + pub fn iter(self) -> impl ExactSizeIterator { + (self.first.0..self.first.0.saturating_add(self.count)).map(Slot) + } +} + +impl PendingId { + #[inline(always)] + pub fn resolve(self, ids: &AssignedIds) -> u32 { + match self { + PendingId::Assigned(document_id) => document_id, + PendingId::Slot(slot) => ids.slot(slot), + } + } + + #[inline(always)] + pub fn slot(self) -> Option { + match self { + PendingId::Slot(slot) => Some(slot), + PendingId::Assigned(_) => None, + } + } + + #[inline(always)] + pub fn assigned(self) -> Option { + match self { + PendingId::Assigned(document_id) => Some(document_id), + PendingId::Slot(_) => None, + } + } + + #[inline(always)] + pub fn is_pending(self) -> bool { + matches!(self, PendingId::Slot(_)) + } +} + +impl From for PendingId { + #[inline(always)] + fn from(document_id: u32) -> Self { + PendingId::Assigned(document_id) + } +} + +impl TaskId { + #[inline(always)] + pub fn resolve(self, account_id: u32, document_id: u32) -> u64 { + match self { + TaskId::Assigned(id) => id, + TaskId::Document => { + debug_assert!(account_id <= MAX_ACCOUNT_ID); + Id::from_parts(account_id, document_id).id() + } + TaskId::DocumentTarget(target_id) => { + TaskId::document_target(account_id, document_id, target_id) + } + } + } + + pub fn document_target(account_id: u32, document_id: u32, target_id: u32) -> u64 { + const GOLDEN_RATIO: u64 = 0x9E37_79B9_7F4A_7C15; + const MIX_LOW: u64 = 0xBF58_476D_1CE4_E5B9; + const MIX_HIGH: u64 = 0x94D0_49BB_1331_11EB; + + let mut hash = Id::from_parts(account_id, document_id).id().rotate_left(17) + ^ (target_id as u64).wrapping_mul(GOLDEN_RATIO); + hash ^= hash >> 30; + hash = hash.wrapping_mul(MIX_LOW); + hash ^= hash >> 27; + hash = hash.wrapping_mul(MIX_HIGH); + hash ^= hash >> 31; + (hash & !DERIVED_ID_KIND) | TARGET_ID_MARKER + } + + pub fn singleton_of(task: &Task) -> Option { + task.singleton_key() + .map(|(discriminant, key)| TaskId::singleton(task.object_type(), discriminant, key)) + } + + pub fn singleton(typ: TaskType, discriminant: u16, key: u64) -> u64 { + let key = + (u128::from(typ as u16) << 80) | (u128::from(discriminant) << 64) | u128::from(key); + (xxh3_64(&key.to_be_bytes()) & !DERIVED_ID_KIND) | SINGLETON_ID_MARKER + } + + pub fn shared_object( + account_id: u32, + collection: Collection, + document_id: u32, + window: u64, + ) -> u64 { + let key = (u128::from(Id::from_parts(account_id, document_id).id()) << 64) + | (u128::from(u8::from(collection)) << 56) + | u128::from(window & SHARED_OBJECT_WINDOW_MASK); + (xxh3_64(&key.to_be_bytes()) & !DERIVED_ID_KIND) | SHARED_OBJECT_ID_MARKER + } +} + +pub const MAX_ACCOUNT_ID: u32 = (1 << 30) - 1; +const DERIVED_ID_KIND: u64 = 0b11 << 62; +const SINGLETON_ID_MARKER: u64 = 0b01 << 62; +const TARGET_ID_MARKER: u64 = 0b10 << 62; +const SHARED_OBJECT_ID_MARKER: u64 = 0b11 << 62; +const SHARED_OBJECT_WINDOW_MASK: u64 = (1 << 56) - 1; + +impl QueueDocumentId { + #[inline(always)] + pub fn resolve(self, document_id: u32) -> u32 { + match self { + QueueDocumentId::Assigned(document_id) => document_id, + QueueDocumentId::Current => document_id, + } + } +} + +impl From for QueueDocumentId { + #[inline(always)] + fn from(document_id: u32) -> Self { + QueueDocumentId::Assigned(document_id) + } +} + +impl From for TaskId { + #[inline(always)] + fn from(id: u64) -> Self { + TaskId::Assigned(id) + } +} + +impl From for PendingId { + #[inline(always)] + fn from(slot: Slot) -> Self { + PendingId::Slot(slot) + } +} + impl AssignedIds { + #[inline(always)] + pub fn mark(&self) -> AssignedIdsMark { + AssignedIdsMark { + slots: self.slots.len(), + change_ids: self.change_ids.len(), + counter: self.counter, + archive_hash: self.archive_hash, + } + } + + #[inline(always)] + pub fn rollback(&mut self, mark: AssignedIdsMark) { + self.slots.truncate(mark.slots); + self.change_ids.truncate(mark.change_ids); + self.counter = mark.counter; + self.archive_hash = mark.archive_hash; + self.current_change_id = 0; + } + + pub fn push_archive_hash(&mut self, hash: u32) { + self.archive_hash = Some(hash); + } + + pub fn set_archive_hash(&mut self, hash: Option) { + if hash.is_some() { + self.archive_hash = hash; + } + } + + #[inline(always)] + pub fn last_archive_hash(&self) -> Option { + self.archive_hash + } + pub fn push_counter_id(&mut self, id: i64) { - self.ids.push(AssignedId::Counter(id)); + self.counter = Some(id); } - pub fn push_change_id(&mut self, account_id: u32, change_id: u64) { - self.ids.push(AssignedId::ChangeId(ChangeId { + pub fn push_change_id(&mut self, counter: ChangeCounter, change_id: u64) { + self.change_ids.push(ChangeId { counter, change_id }); + } + + #[inline] + pub fn apply(&mut self, allocation: Allocation, last_id: i64) -> trc::Result<()> { + match allocation { + Allocation::ChangeId(counter) => self.push_change_id(counter, last_id as u64), + Allocation::Reserved(reservation) => { + self.fill_slots(reservation.first_slot, reservation.count, last_id as u32) + } + Allocation::Bounded(bound) => bound.verify(last_id)?, + } + Ok(()) + } + + pub fn fill_slots(&mut self, first_slot: Slot, count: u32, last_id: u32) { + debug_assert!(last_id >= count, "{last_id} < {count}"); + debug_assert_ne!(last_id, u32::MAX, "document id space is exhausted"); + + let first = first_slot.index(); + if self.slots.len() < first + count as usize { + self.slots.resize(first + count as usize, u32::MAX); + } + let base = last_id - count + 1; + for offset in 0..count { + self.slots[first + offset as usize] = base + offset; + } + } + + #[inline(always)] + pub fn slot(&self, slot: Slot) -> u32 { + let document_id = self.slots[slot.index()]; + debug_assert_ne!(document_id, u32::MAX, "slot {slot:?} was never assigned"); + document_id + } + + #[inline(always)] + pub fn slots(&self, range: SlotRange) -> impl Iterator + '_ { + let first = range.first().index(); + self.slots[first..first + range.len()] + .iter() + .inspect(|id| { + debug_assert_ne!(**id, u32::MAX, "a slot in the range was never assigned"); + }) + .copied() + } + + pub fn change_counters(&self) -> impl Iterator + '_ { + self.change_ids.iter().map(|id| id.counter) + } + + pub fn change_ids( + &self, + account_id: u32, + group: impl Into, + ) -> impl Iterator + '_ { + let counter = ChangeCounter { account_id, - change_id, - })); + group: group.into(), + }; + self.change_ids + .iter() + .filter(move |id| id.counter == counter) + .map(|id| id.change_id) } - pub fn last_change_id(&self, account_id: u32) -> trc::Result { - self.ids + pub fn change_id(&self, account_id: u32, group: impl Into) -> Option { + let counter = ChangeCounter { + account_id, + group: group.into(), + }; + self.change_ids .iter() - .filter_map(|id| match id { - AssignedId::ChangeId(change_id) if change_id.account_id == account_id => { - Some(change_id.change_id) + .filter(|id| id.counter == counter) + .map(|id| id.change_id) + .next_back() + } + + pub fn last_change_id(&self, account_id: u32, group: impl Into) -> u64 { + let group = group.into(); + let change_id = self.change_id(account_id, group); + debug_assert!( + change_id.is_some(), + "no change id was created for account {account_id} group {group:?}" + ); + change_id.unwrap_or_default() + } + + #[inline(always)] + pub fn current_change_id(&self) -> u64 { + debug_assert_ne!(self.current_change_id, 0, "no current change id is set"); + self.current_change_id + } + + #[inline(always)] + pub fn try_current_change_id(&self) -> Option { + Some(self.current_change_id).filter(|change_id| *change_id != 0) + } + + #[inline(always)] + pub(crate) fn set_current_change_id(&mut self, account_id: u32, group: ChangeGroup) -> u64 { + self.current_change_id = self.change_id(account_id, group).unwrap_or_default(); + self.current_change_id + } + + #[inline(always)] + pub fn last_counter_id(&self) -> i64 { + debug_assert!(self.counter.is_some(), "no counter id was created"); + self.counter.unwrap_or_default() + } +} + +impl Patch { + pub fn apply(patches: &[Patch], payload: &mut [u8], ids: &AssignedIds) { + for patch in patches { + let offset = patch.offset as usize; + match patch.source { + PatchSource::SlotBeU32(slot) => { + payload[offset..offset + U32_LEN] + .copy_from_slice(&ids.slot(slot).to_be_bytes()); } - _ => None, - }) - .next_back() - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No change ids were created") - }) - } - - pub fn current_change_id(&self) -> trc::Result { - self.current_change_id.ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No current change id is set") - }) - } - - pub(crate) fn set_current_change_id(&mut self, account_id: u32) -> trc::Result { - let change_id = self.last_change_id(account_id)?; - self.current_change_id = Some(change_id); - Ok(change_id) - } - - pub fn last_counter_id(&self) -> trc::Result { - self.ids - .iter() - .filter_map(|id| match id { - AssignedId::Counter(counter_id) => Some(*counter_id), - _ => None, - }) - .next_back() - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No counter ids were created") - }) + PatchSource::SlotArchivedU32 { slot, base } => { + payload[offset..offset + U32_LEN] + .copy_from_slice(&(ids.slot(slot) + base).to_le_bytes()); + Archive::::restamp_hash(payload); + } + PatchSource::ChangeIdBe => { + payload[offset..offset + U64_LEN] + .copy_from_slice(&ids.current_change_id().to_be_bytes()); + } + } + } } } @@ -502,6 +1678,7 @@ impl From for u8 { match value { LogCollection::Sync(col) => col as u8, LogCollection::Vanished(col) => col as u8, + LogCollection::Private { collection, .. } => PRIVATE_LOG | collection as u8, } } } @@ -526,13 +1703,19 @@ impl From for ValueClass { impl From for ValueClass { fn from(value: EmailField) -> Self { - ValueClass::Property(value.into()) + match value { + EmailField::Metadata | EmailField::SortKeys => ValueClass::Immutable(value.into()), + _ => ValueClass::Property(value.into()), + } } } impl From for ValueClass { fn from(value: MailboxField) -> Self { - ValueClass::Property(value.into()) + match value { + MailboxField::UidCounter => ValueClass::MailboxUid, + _ => ValueClass::Property(value.into()), + } } } @@ -560,141 +1743,32 @@ impl From for ValueClass { } } -impl PartialEq for MergeOperation { - fn eq(&self, other: &Self) -> bool { - self.params == other.params +pub struct PendingTask { + pub task: Task, + pub slot: Slot, +} + +impl SerializeWithIds for PendingTask { + fn serialize_with_ids(&mut self, ids: &AssignedIds) -> trc::Result<(Vec, Option)> { + self.task.set_document_id(Id::from(ids.slot(self.slot))); + + Ok((self.task.to_pickled_vec(), None)) + } + + fn size_hint(&self) -> usize { + self.task.size_hint() } } -impl Eq for MergeOperation {} - -impl PartialEq for SetOperation { - fn eq(&self, other: &Self) -> bool { - self.params == other.params +impl std::fmt::Debug for SerializeOperation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SerializeOperation") } } -impl Eq for SetOperation {} - -impl Hash for MergeOperation { - fn hash(&self, state: &mut H) { - self.params.hash(state); +impl std::fmt::Debug for MergeOperation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("MergeOperation") } } -impl Hash for SetOperation { - fn hash(&self, state: &mut H) { - self.params.hash(state); - } -} - -impl SetOperation { - pub fn params(&self) -> &Params { - &self.params - } -} - -impl MergeOperation { - pub fn params(&self) -> &Params { - &self.params - } -} - -impl Params { - pub fn with_capacity(capacity: usize) -> Self { - Self(Vec::with_capacity(capacity)) - } - - pub fn new() -> Self { - Self(Vec::new()) - } - - pub fn with_i64(mut self, value: i64) -> Self { - self.0.push(Param::I64(value)); - self - } - - pub fn with_u64(mut self, value: u64) -> Self { - self.0.push(Param::U64(value)); - self - } - - pub fn with_string(mut self, value: String) -> Self { - self.0.push(Param::String(value)); - self - } - - pub fn with_str(mut self, value: &str) -> Self { - self.0.push(Param::String(value.to_string())); - self - } - - pub fn with_bytes(mut self, value: Vec) -> Self { - self.0.push(Param::Bytes(value)); - self - } - - pub fn with_bool(mut self, value: bool) -> Self { - self.0.push(Param::Bool(value)); - self - } - - pub fn i64(&self, idx: usize) -> i64 { - match &self.0[idx] { - Param::I64(v) => *v, - _ => panic!("Param at index {} is not an i64", idx), - } - } - - pub fn u64(&self, idx: usize) -> u64 { - match &self.0[idx] { - Param::U64(v) => *v, - _ => panic!("Param at index {} is not a u64", idx), - } - } - - pub fn string(&self, idx: usize) -> &str { - match &self.0[idx] { - Param::String(v) => v.as_str(), - _ => panic!("Param at index {} is not a String", idx), - } - } - - pub fn bytes(&self, idx: usize) -> &[u8] { - match &self.0[idx] { - Param::Bytes(v) => v.as_slice(), - _ => panic!("Param at index {} is not Bytes", idx), - } - } - - pub fn bool(&self, idx: usize) -> bool { - match &self.0[idx] { - Param::Bool(v) => *v, - _ => panic!("Param at index {} is not a bool", idx), - } - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn as_slice(&self) -> &[Param] { - &self.0 - } -} - -impl Default for Params { - fn default() -> Self { - Self::new() - } -} - -impl AsRef<[Param]> for Params { - fn as_ref(&self) -> &[Param] { - &self.0 - } -} diff --git a/crates/store/src/write/new_key.rs b/crates/store/src/write/new_key.rs new file mode 100644 index 00000000..77f80187 --- /dev/null +++ b/crates/store/src/write/new_key.rs @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BatchCursor, Operation, ValueClass, ValueOp}; +use crate::WITH_SUBSPACE; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NewKeyAsserts { + CheckedByInsert, + Read, +} + +impl BatchCursor { + pub(crate) fn is_document_independent( + &self, + class: &ValueClass, + key_buf: &mut Vec, + ) -> bool { + key_buf.clear(); + class.serialize_into(key_buf, self.account_id, self.collection, 0, WITH_SUBSPACE); + let len = key_buf.len(); + class.serialize_into( + key_buf, + self.account_id, + self.collection, + u32::MAX, + WITH_SUBSPACE, + ); + let independent = key_buf + .split_at_checked(len) + .is_some_and(|(first, second)| first == second); + key_buf.truncate(len); + independent + } +} + +pub(crate) fn insert_follows(class: &ValueClass, rest: &[Operation]) -> bool { + rest.iter() + .find_map(|op| match op { + Operation::Value { class: next, op } if next == class => { + Some(matches!(op, ValueOp::Set(_))) + } + Operation::AssertValue { class: next, .. } if next == class => Some(false), + Operation::AccountId { .. } + | Operation::Collection { .. } + | Operation::DocumentId { .. } => Some(false), + _ => None, + }) + .unwrap_or(false) +} + diff --git a/crates/store/src/write/serialize.rs b/crates/store/src/write/serialize.rs index 0f0aea16..1f741c4f 100644 --- a/crates/store/src/write/serialize.rs +++ b/crates/store/src/write/serialize.rs @@ -4,66 +4,87 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ARCHIVE_ALIGNMENT, AlignedBytes, Archive, ArchiveVersion, Archiver}; +use super::{ + Archive, ArchiveBytes, ArchiveVersion, Archiver, + compress::{ + ArchiveCompression, Compression, MAX_ARCHIVE_SIZE, compress, compress_watermark, + decompress, decompress_into, + }, +}; use crate::{Deserialize, Serialize, SerializeInfallible, U32_LEN, U64_LEN, Value}; use compact_str::format_compact; -use rkyv::util::AlignedVec; use roaring::{RoaringBitmap, RoaringTreemap}; const MAGIC_MARKER: u8 = 1 << 7; const VERSIONED: u8 = 1 << 6; const HASHED: u8 = 1 << 5; const LZ4_COMPRESSED: u8 = 1 << 4; +const ZSTD_COMPRESSED: u8 = 1 << 3; -const COMPRESS_WATERMARK: usize = 8192; +const SERIALIZE_CAPACITY: usize = 1024; -fn validate_marker_and_contents(bytes: &[u8]) -> Option<(bool, &[u8], ArchiveVersion)> { +#[derive(Clone, Copy, PartialEq, Eq)] +enum Encoding { + Plain, + Zstd, + Lz4, +} + +fn split_hash(encoding: Encoding, contents: &[u8]) -> Option<(&[u8], u32)> { + match encoding { + Encoding::Zstd => contents + .get(contents.len().checked_sub(U32_LEN)?..)? + .try_into() + .ok() + .map(|hash| (contents, u32::from_be_bytes(hash))), + Encoding::Plain | Encoding::Lz4 => contents + .split_at_checked(contents.len().checked_sub(U32_LEN)?) + .and_then(|(contents, archive_hash)| { + let hash = xxhash_rust::xxh3::xxh3_64(contents) as u32; + (hash.to_be_bytes().as_slice() == archive_hash).then_some((contents, hash)) + }), + } +} + +fn validate_marker_and_contents(bytes: &[u8]) -> Option<(Encoding, &[u8], ArchiveVersion)> { let (marker, contents) = bytes .split_last() .filter(|(marker, _)| (**marker & MAGIC_MARKER) != 0)?; - let is_uncompressed = (marker & LZ4_COMPRESSED) == 0; + let encoding = if marker & ZSTD_COMPRESSED != 0 { + Encoding::Zstd + } else if marker & LZ4_COMPRESSED != 0 { + Encoding::Lz4 + } else { + Encoding::Plain + }; + if marker & VERSIONED != 0 { let (contents, change_id) = contents - .split_at_checked(contents.len() - U64_LEN) + .split_at_checked(contents.len().checked_sub(U64_LEN)?) .and_then(|(contents, change_id)| { change_id .try_into() .ok() .map(|change_id| (contents, u64::from_be_bytes(change_id))) })?; - contents - .split_at_checked(contents.len() - U32_LEN) - .and_then(|(contents, archive_hash)| { - let hash = xxhash_rust::xxh3::xxh3_64(contents) as u32; - if hash.to_be_bytes().as_slice() == archive_hash { - Some(( - is_uncompressed, - contents, - ArchiveVersion::Versioned { change_id, hash }, - )) - } else { - None - } - }) + split_hash(encoding, contents).map(|(contents, hash)| { + ( + encoding, + contents, + ArchiveVersion::Versioned { change_id, hash }, + ) + }) } else if marker & HASHED != 0 { - contents - .split_at_checked(contents.len() - U32_LEN) - .and_then(|(contents, archive_hash)| { - let hash = xxhash_rust::xxh3::xxh3_64(contents) as u32; - if hash.to_be_bytes().as_slice() == archive_hash { - Some((is_uncompressed, contents, ArchiveVersion::Hashed { hash })) - } else { - None - } - }) + split_hash(encoding, contents) + .map(|(contents, hash)| (encoding, contents, ArchiveVersion::Hashed { hash })) } else { - Some((is_uncompressed, contents, ArchiveVersion::Unversioned)) + Some((encoding, contents, ArchiveVersion::Unversioned)) } } -impl Deserialize for Archive { +impl Deserialize for Archive { fn deserialize(bytes: &[u8]) -> trc::Result { - let (is_uncompressed, contents, version) = + let (encoding, contents, version) = validate_marker_and_contents(bytes).ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() @@ -72,21 +93,19 @@ impl Deserialize for Archive { .caused_by(trc::location!()) })?; - if is_uncompressed { - let mut bytes = AlignedVec::with_capacity(contents.len()); - bytes.extend_from_slice(contents); - Ok(Archive { + match encoding { + Encoding::Plain => Ok(Archive { version, - inner: AlignedBytes::Aligned(bytes), - }) - } else { - aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) + inner: contents.to_vec(), + }), + Encoding::Zstd => zstd_inflate(contents).map(|inner| Archive { version, inner }), + Encoding::Lz4 => lz4_deflate(contents).map(|inner| Archive { version, inner }), } } fn deserialize_owned(mut bytes: Vec) -> trc::Result { - let (is_uncompressed, contents, version) = validate_marker_and_contents(&bytes) - .ok_or_else(|| { + let (encoding, contents, version) = + validate_marker_and_contents(&bytes).ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() .details("Archive integrity compromised") @@ -94,45 +113,118 @@ impl Deserialize for Archive { .caused_by(trc::location!()) })?; - if is_uncompressed { - bytes.truncate(contents.len()); - if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { + match encoding { + Encoding::Plain => { + let contents_len = contents.len(); + bytes.truncate(contents_len); Ok(Archive { version, - inner: AlignedBytes::Vec(bytes), - }) - } else { - let mut aligned = AlignedVec::with_capacity(bytes.len()); - aligned.extend_from_slice(&bytes); - Ok(Archive { - version, - inner: AlignedBytes::Aligned(aligned), + inner: bytes, }) } - } else { - aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) + Encoding::Zstd => zstd_inflate(contents).map(|inner| Archive { version, inner }), + Encoding::Lz4 => lz4_deflate(contents).map(|inner| Archive { version, inner }), + } + } +} + +impl Archive { + pub fn deserialize_with_prefix(prefix: &[u8], bytes: &[u8]) -> trc::Result { + let (encoding, contents, version) = + validate_marker_and_contents(bytes).ok_or_else(|| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Archive integrity compromised") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + })?; + + let inner = match encoding { + Encoding::Plain => [prefix, contents].concat(), + Encoding::Zstd => [prefix, zstd_inflate(contents)?.as_slice()].concat(), + Encoding::Lz4 => [prefix, lz4_deflate(contents)?.as_slice()].concat(), + }; + + Ok(Archive { version, inner }) + } +} + +impl Archive { + pub fn serialize_raw(mut contents: Vec, compress_watermark: usize) -> trc::Result> { + let flags = MAGIC_MARKER | HASHED; + if contents.len() >= compress_watermark { + let mut compressed = compress(None, &contents, 1).map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + })?; + if compressed.len() < contents.len() { + compressed.push(flags | ZSTD_COMPRESSED); + return Ok(compressed); + } + } + + let hash = xxhash_rust::xxh3::xxh3_64(&contents) as u32; + contents.reserve_exact(U32_LEN + 1); + contents.extend_from_slice(&hash.to_be_bytes()); + contents.push(flags); + Ok(contents) + } + + pub fn deserialize_raw<'x>(bytes: &'x [u8], scratch: &'x mut Vec) -> trc::Result<&'x [u8]> { + let (encoding, contents, version) = + validate_marker_and_contents(bytes).ok_or_else(|| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Archive integrity compromised") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + })?; + if version == ArchiveVersion::Unversioned { + return Err(trc::StoreEvent::DataCorruption + .into_err() + .details("Raw archive carries no integrity hash") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!())); + } + + match encoding { + Encoding::Plain => Ok(contents), + Encoding::Zstd => { + decompress_into(contents, scratch, MAX_ARCHIVE_SIZE).map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, contents) + .caused_by(trc::location!()) + .reason(err) + })?; + Ok(scratch.as_slice()) + } + Encoding::Lz4 => { + *scratch = lz4_deflate(contents)?; + Ok(scratch.as_slice()) + } } } } #[inline] -fn aligned_lz4_deflate(archive: &[u8]) -> trc::Result { - lz4_flex::block::uncompressed_size(archive) - .and_then(|(uncompressed_size, archive)| { - let mut bytes = AlignedVec::with_capacity(uncompressed_size); - unsafe { - // SAFETY: `new_len` is equal to `capacity` and vector is initialized by lz4_flex. - bytes.set_len(uncompressed_size); - } - lz4_flex::decompress_into(archive, &mut bytes)?; - Ok(AlignedBytes::Aligned(bytes)) - }) - .map_err(|err| { - trc::StoreEvent::DecompressError - .ctx(trc::Key::Value, archive) - .caused_by(trc::location!()) - .reason(err) - }) +fn zstd_inflate(archive: &[u8]) -> trc::Result { + decompress(archive).map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, archive) + .caused_by(trc::location!()) + .reason(err) + }) +} + +#[inline] +fn lz4_deflate(archive: &[u8]) -> trc::Result { + lz4_flex::block::decompress_size_prepended(archive).map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, archive) + .caused_by(trc::location!()) + .reason(err) + }) } impl Serialize for Archiver @@ -140,87 +232,65 @@ where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, + Vec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, { fn serialize(&self) -> trc::Result> { - rkyv::to_bytes::(&self.inner) - .map_err(|err| { - trc::StoreEvent::DeserializeError - .caused_by(trc::location!()) - .reason(err) - }) - .map(|input| { - let input = input.as_ref(); - let input_len = input.len(); - let version_offset = ((self.flags & VERSIONED != 0) as usize) * U64_LEN; - let mut bytes = if input_len > COMPRESS_WATERMARK { - let mut bytes = vec![ - self.flags | LZ4_COMPRESSED; - lz4_flex::block::get_maximum_output_size(input_len) - + (U32_LEN * 2) - + version_offset - + 1 - ]; + let version_offset = ((self.flags & VERSIONED != 0) as usize) * U64_LEN; + let trailer_len = U32_LEN + version_offset + 1; + let bytes = rkyv::api::high::to_bytes_in::<_, rkyv::rancor::Error>( + &self.inner, + Vec::with_capacity( + (std::mem::size_of::() + trailer_len).max(SERIALIZE_CAPACITY), + ), + ) + .map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + })?; - // Compress the data - let compressed_len = - lz4_flex::compress_into(input, &mut bytes[U32_LEN..]).unwrap(); + if let Compression::Zstd(dictionary) = self.compression + && bytes.len() >= compress_watermark(dictionary) + { + let mut compressed = + compress(dictionary, &bytes, version_offset + 1).map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + })?; - if compressed_len < input_len { - // Prepend the length of the uncompressed data - bytes[..U32_LEN].copy_from_slice(&(input_len as u32).to_le_bytes()); - - if self.flags & HASHED != 0 { - // Hash the compressed data including the length - let hash = - xxhash_rust::xxh3::xxh3_64(&bytes[..compressed_len + U32_LEN]) - as u32; - - // Add the hash - bytes[compressed_len + U32_LEN..compressed_len + (U32_LEN * 2)] - .copy_from_slice(&hash.to_be_bytes()); - - // Truncate to the actual size - bytes.truncate(compressed_len + (U32_LEN * 2) + version_offset + 1); - } else { - // Truncate to the actual size - bytes.truncate(compressed_len + U32_LEN + 1); - } - - return bytes; - } - bytes.clear(); - bytes - } else { - Vec::with_capacity(input_len + U32_LEN + version_offset + 1) - }; - - bytes.extend_from_slice(input); - if self.flags & HASHED != 0 { - bytes.extend_from_slice( - &(xxhash_rust::xxh3::xxh3_64(input) as u32).to_be_bytes(), - ); - } + if compressed.len() < bytes.len() { if version_offset != 0 { - bytes.extend_from_slice(0u64.to_be_bytes().as_slice()); + compressed.extend_from_slice(0u64.to_be_bytes().as_slice()); } - bytes.push(self.flags); - bytes - }) + compressed.push(self.flags | ZSTD_COMPRESSED); + return Ok(compressed); + } + } + + let mut bytes = bytes; + bytes.reserve_exact(trailer_len); + if self.flags & HASHED != 0 { + let hash = (xxhash_rust::xxh3::xxh3_64(&bytes) as u32).to_be_bytes(); + bytes.extend_from_slice(&hash); + } + if version_offset != 0 { + bytes.extend_from_slice(0u64.to_be_bytes().as_slice()); + } + bytes.push(self.flags); + + Ok(bytes) } } -impl Archive { +impl Archive { #[inline] pub fn as_bytes(&self) -> &[u8] { - match &self.inner { - AlignedBytes::Vec(bytes) => bytes.as_slice(), - AlignedBytes::Aligned(bytes) => bytes.as_slice(), - } + self.inner.as_slice() } pub fn unarchive(&self) -> trc::Result<&::Archived> @@ -230,6 +300,8 @@ impl Archive { rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { + const { assert!(std::mem::align_of::() == 1) }; + let bytes = self.as_bytes(); if self.version != ArchiveVersion::Unversioned { if bytes.len() >= std::mem::size_of::() { @@ -264,6 +336,8 @@ impl Archive { rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { + const { assert!(std::mem::align_of::() == 1) }; + let bytes = self.as_bytes(); if bytes.len() >= std::mem::size_of::() { rkyv::access::(bytes).map_err(|err| { @@ -346,28 +420,6 @@ impl Archive { }) } - pub fn into_inner(self) -> Vec { - let mut bytes = match self.inner { - AlignedBytes::Vec(bytes) => bytes, - AlignedBytes::Aligned(bytes) => bytes.to_vec(), - }; - match self.version { - ArchiveVersion::Versioned { change_id, hash } => { - bytes.extend_from_slice(&hash.to_be_bytes()); - bytes.extend_from_slice(&change_id.to_be_bytes()); - bytes.push(MAGIC_MARKER | VERSIONED | HASHED); - } - ArchiveVersion::Hashed { hash } => { - bytes.extend_from_slice(&hash.to_be_bytes()); - bytes.push(MAGIC_MARKER | HASHED); - } - ArchiveVersion::Unversioned => { - bytes.push(MAGIC_MARKER); - } - } - bytes - } - pub fn extract_hash(bytes: &[u8]) -> Option { let marker = *bytes.last()?; if marker & VERSIONED != 0 { @@ -382,6 +434,48 @@ impl Archive { None } } + + fn hash_offset(bytes: &[u8]) -> Option { + let marker = *bytes.last()?; + if marker & VERSIONED != 0 { + bytes.len().checked_sub(U32_LEN + U64_LEN + 1) + } else if marker & HASHED != 0 { + bytes.len().checked_sub(U32_LEN + 1) + } else { + None + } + } + + pub fn restamp_hash(bytes: &mut [u8]) { + debug_assert!( + bytes + .last() + .is_none_or(|marker| marker & (ZSTD_COMPRESSED | LZ4_COMPRESSED) == 0), + "a compressed archive cannot be rehashed" + ); + + if let Some(offset) = Self::hash_offset(bytes) { + let hash = (xxhash_rust::xxh3::xxh3_64(&bytes[..offset]) as u32).to_be_bytes(); + bytes[offset..offset + U32_LEN].copy_from_slice(&hash); + } + } +} + +impl Archiver +where + T: ArchiveCompression + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + Vec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >, +{ + pub fn new(inner: T) -> Self { + Self::with_compression(inner, T::COMPRESSION) + } } impl Archiver @@ -389,16 +483,17 @@ where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, + Vec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, { - pub fn new(inner: T) -> Self { + pub fn with_compression(inner: T, compression: Compression) -> Self { Self { inner, flags: MAGIC_MARKER | HASHED, + compression, } } @@ -410,6 +505,7 @@ where Self { inner: self.inner, flags: self.flags | VERSIONED, + compression: self.compression, } } @@ -417,6 +513,7 @@ where Self { inner: self.inner, flags: MAGIC_MARKER, + compression: self.compression, } } @@ -425,6 +522,25 @@ where .serialize() .map(|bytes| ((bytes.len() - U64_LEN - 1) as u64, bytes)) } + + pub fn serialize_patchable(self) -> trc::Result<(u32, Vec)> { + let flags = self.flags; + + Self { + inner: self.inner, + flags, + compression: Compression::None, + } + .serialize() + .map(|bytes| { + let payload_len = bytes.len() + - 1 + - ((flags & HASHED != 0) as usize) * U32_LEN + - ((flags & VERSIONED != 0) as usize) * U64_LEN; + + (payload_len as u32, bytes) + }) + } } impl Archive<&T> @@ -484,6 +600,8 @@ where T::Archived: for<'a> rkyv::bytecheck::CheckBytes> + rkyv::Deserialize>, { + const { assert!(std::mem::align_of::() == 1) }; + rkyv::access::(input).map_err(|err| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) @@ -492,6 +610,14 @@ where }) } +pub struct RawValue(pub Vec); + +impl Deserialize for RawValue { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(RawValue(bytes.to_vec())) + } +} + impl SerializeInfallible for u32 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() @@ -569,11 +695,11 @@ impl From> for Archive { } } -impl Default for Archive { +impl Default for Archive { fn default() -> Self { Archive { version: ArchiveVersion::Unversioned, - inner: AlignedBytes::Aligned(AlignedVec::new()), + inner: Vec::new(), } } } @@ -623,3 +749,4 @@ impl Deserialize for RoaringTreemap { }) } } + diff --git a/crates/text-extract/CREDITS.md b/crates/text-extract/CREDITS.md new file mode 100644 index 00000000..bc809db9 --- /dev/null +++ b/crates/text-extract/CREDITS.md @@ -0,0 +1,507 @@ +# Credits and third-party material + +This file records the third-party code, algorithms, data and test files used by the `text-extract` crate, and the license under which each was used. It is meant for maintainers and for license review of the project's AGPL-3.0 and commercial distributions. Anything not listed here was written for this project. + +Most of the crate's code was written from scratch. Where another project's code or data was adapted, the source file is named below. Projects under copyleft licenses (MuPDF, poppler) were consulted for ideas only; nothing was copied from them. + +## 1. Code and algorithms adapted from other projects + +Unless noted otherwise, the material below was adapted (translated to Rust, restructured), not copied verbatim. + +### pdf.js (Mozilla Foundation), Apache-2.0 + +Source: , commit d52fdf411a6e4d338180687456e0df019e28475e. Copyright Mozilla Foundation. + +Object layer (`src/pdf/`, lexer, xref, repair): + +- `src/core/parser.js` `Lexer.getNumber`: collapsing repeated leading minus signs, skipping line breaks after a sign, reading a lone sign or dot as 0, ignoring a minus sign inside a number. +- `src/core/parser.js` `Lexer.getObj`: a non-printable byte followed by a printable one is a one-byte keyword; a stray `)` is consumed so the lexer always makes progress. +- `src/core/parser.js` `Lexer.getHexString`: invalid hex characters ignored, odd digit count padded with 0. +- `src/core/xref.js` `readXRefTable`: the first-subsection fix (`1 N` whose first entry is free becomes `0 N`). +- `src/core/xref.js` `readXRef`: queue order (table, `/XRefStm`, `/Prev`) with a set of visited offsets; `/Prev N 0 R` accepted as offset N. +- `src/core/xref.js` `fetchCompressed`: looking up an object-stream member by number when the index does not match (bug 1978317). +- `src/core/xref.js` recovery: trailer candidates validated by a `/Root` that reaches a `/Pages` dictionary. +- `src/core/string_utils.js` `PDFStringTranslateTable` and `stringToPDFString`: the PDFDocEncoding table, UTF-16LE BOM support, removal of ESC-delimited language codes. +- `src/core/document.js` `checkHeader`: header search in the first 1024 bytes and header-relative offsets. + +Stream filters (`src/pdf/filter/`): + +- `src/core/lzw_stream.js` (table walked backwards to emit a code, EarlyChange width rule), `ascii_85_stream.js` (final group padding, `~` as end of data), `ascii_hex_stream.js` (non-hex bytes ignored), `run_length_stream.js`, `predictor_stream.js` (PNG row filters, Paeth tie order, TIFF predictor 2 for 1 to 16 bits per component). Semantics and structure only, no verbatim code. + +Decryption (`src/pdf/crypt/`): + +- `src/core/crypto.js`: the structure of the key derivation, lenient AES-CBC framing (IV prefix, trailing partial block dropped, PKCS#7 padding stripped only when valid), the Algorithm 2.B round loop and termination rule, the fallback when `/Length` is given in bytes, zero-padding a short version 4 file key to 16 bytes (issue 19484), forcing AES-256 for every crypt filter when V is 5 (bug 2046659). + +Text layer (`src/pdf/font/`, `content/`, `layout/`, `forms.rs`): + +- `src/core/evaluator.js` (`getTextContent`): a real space glyph only counts when the pen moved (the 0.03 em rule), zero-width glyphs never move the previous ink end, gaps measured from the ink end before character spacing, the per-document cache of form XObjects that drew no text, the ancestor set that stops form cycles, `/Font` in ExtGState dictionaries, keeping the last font when `Tf` names an unknown resource. +- `src/core/evaluator.js` (`_simpleFontToUnicode`, font loading): the list of ten simple-font names written as GBK bytes (SimSun, SimHei, SimKai, SimFang and variants, XiaoBiaoSong), reproduced in `src/pdf/font/simple.rs` (`GBK_FONT_NAMES`); `/ToUnicode /Identity-H` given as a name; empty ToUnicode treated as absent; `.notdef` Differences entries leaving the base encoding in place. +- `src/core/cmap.js` and `evaluator.js` (`readToUnicode`): odd-length and one-byte destination strings left-padded with zero (issue 18099), bfrange array destinations mapped for `min(range, array length)` codes. +- `src/core/parser.js` (`findDefaultInlineStreamEnd`, ASCIIHex and ASCII85 end finders): inline image end detection. +- `src/shared/util.js` (`normalizeUnicode`): NFKC of ligatures and Arabic presentation forms at emission. + +Tables and font programs (data copied, see also section 2): + +- `src/core/encodings.js`: StandardEncoding, WinAnsiEncoding, MacRomanEncoding, MacExpertEncoding, SymbolSetEncoding, ZapfDingbatsEncoding and ExpertEncoding tables (`src/pdf/tables/encoding_data.rs`). Modified: MacRoman code 0xF0 (`apple`) is left undefined, as in the PDF specification. +- `src/core/glyphlist.js`: 121 glyph names pdf.js adds to the Adobe Glyph List (mostly TeX math extension names). Modified: `intercal` maps to U+22BA instead of U+1D40; underscore ligature names dropped. +- `src/core/standard_fonts.js`: standard-14 font name aliases (`getStdFontMap`) and the font name normalisation rule. +- `src/core/evaluator.js` and `src/core/unicode.js`: numeric glyph name heuristics (`Gxx`, `g00xx`, `Cdd`/`cdd` with hexadecimal fallback). +- `src/core/cff_parser.js` `CFFStandardStrings`, `src/core/charsets.js` `ExpertCharset` and `ExpertSubsetCharset` (stored in `src/pdf/font/program/cff_data.rs`), `src/core/fonts_utils.js` `MacStandardGlyphOrdering` (stored in `src/pdf/font/program/post_data.rs`). +- `src/core/type1_parser.js` (`extractFontHeader`): scanning cleartext tokens for `dup / put` up to `def`. Modified: a later `/Encoding` replaces an earlier one, radix numbers accepted, string tokens skipped whole, scanning stops at `eexec`. + +### Apache PDFBox (The Apache Software Foundation), Apache-2.0 + +Source: , commit 2245d1c2bb3bbcfe56136a693c395317316ace96. + +- `pdfparser/BaseParser.java` and `COSParser.java`: after `stream`, skip spaces then exactly one EOL; `validateStreamLength` (trust `/Length` only when `endstream` follows); a missing `endstream` bounded by `endobj`. +- `pdfparser/BruteForceParser.java`: during repair, an object-stream member replaces an uncompressed definition only when the object stream appears later in the file. +- `pdfparser/PDFXrefStreamParser.java`: `/W` validation and stopping an xref stream section when its data runs out. +- `pdmodel/PDPageTree.java` (PDFBOX-5009, PDFBOX-3953): skipping page-tree nodes already visited. +- `filter/FlateFilterDecoderStream.java`: skipping the two zlib header bytes and inflating raw deflate, so bad Adler-32 checksums and trailing bytes are harmless (idea only). +- `pdmodel/encryption/StandardSecurityHandler.java`: zero-padding a short revision 4 key (PDFBOX-5955), accepting the empty owner password (ideas only). +- `text/PDFTextStripper.java`: duplicate glyph suppression with a tolerance of a third of the glyph width (fake bold, shadows), the space threshold capped at half the font's space width, ActualText honoured on any marked-content tag with duplicate suppression disabled inside it. +- `contentstream/PDFStreamEngine.java` (`showText`): the advance formula, word spacing applied only to single-byte code 32, TJ adjustments. +- `pdmodel/font/PDFont.java` (`getSpaceWidth`): space width from the code that maps to U+0020. +- `pdmodel/font/PDType0Font.java` (`toUnicode`): the CID font Unicode chain. +- `resources/glyphlist/additional.txt`: 7 TeX glyph names not in pdf.js (`bracketleftmath`, `bracketrightmath`, `epsilon1`, `equalmath`, `parenleftmath`, `parenrightmath`, `plusmath`), data copied into `src/pdf/tables/glyph_data.rs`. +- `resources/afm/`: the Adobe Core 14 AFM files, source of the standard-14 widths (see section 2). + +### pdfminer.six, MIT + +Source: . + +- `pdfminer/pdfdocument.py` `_get_objects`: sequential tokenisation of an object stream from `/First` when member offsets are wrong. +- `pdfminer/cmapdb.py` (`CMapParser`): ignoring the count before `begin*` operators and reading until the matching `end*`. +- `pdfminer/layout.py`: a char margin of two glyph widths as the point where a forward jump starts a new line. +- `name2unicode`: accepting lowercase hexadecimal in `uniXXXX` names and rejecting a ligature name with an unknown component (behaviour only). +- User-then-owner password order in decryption (idea only). + +### pypdf, BSD-3-Clause + +Source: , commit 54d3518. + +- `pypdf/_reader.py` `_get_object_from_stream` and `_sanitize_pdf15_xref_stream_index_pairs`: clamping `/N` by `decoded_len / 3` and xref-stream entry counts by the decoded length. +- `pypdf/_font.py`: a cap of 100,000 `/W` entries; `pypdf/_cmap.py`: a cap on one CMap destination string, decoding legacy CJK CMaps with a codec for the whole code. +- `pypdf/_encryption.py`: `/P` as an unsigned 32-bit value, key length from the crypt filter `/Length` in bytes, trying the owner password with the empty string (ideas only). +- Gzip-wrapped and header-less Flate fallbacks (raw deflate at offset 0, then offset 2). + +### hayro, MIT or Apache-2.0 + +Source: . + +- `hayro-syntax/src/xref.rs`, `src/data.rs`, `src/object/*`: offsets-only cross-reference storage, lazy borrowed object views, negative caching of object streams that failed to decode, and the list of known gaps (zero-width `/W`, page-tree DAG fan-out, unbounded nesting, per-stream `endstream` rescans) that shaped our defences. +- In-place PNG predictor decoding so `/Columns` never drives an allocation. +- `hayro-cmap`: the `usecmap` depth limit of 16 and accepting `bfchar` entries as CIDs in encoding CMaps. + +### lopdf, MIT + +- `src/reader.rs`: correcting a `startxref` or `/Prev` pointer that is off by a few bytes by searching a 64-byte window each side for `xref` or an object header. +- Gzip-wrapped and header-less Flate fallbacks (shared with pypdf, above). + +### pdf_oxide, MIT or Apache-2.0 + +- `src/document.rs` and `src/xref.rs`: probing header-relative and absolute offsets when the header is not at byte 0, and searching the whole file backwards for `startxref`. +- Font Unicode chain: ToUnicode entries mapping to U+FFFD, noncharacters or controls treated as misses; Identity fonts invert the embedded TrueType cmap through CIDToGIDMap before the Adobe collection tables. + +### Runtime dependencies used by the PDF code + +Linked as ordinary Cargo dependencies, listed here for completeness: flate2 (MIT or Apache-2.0) with the zlib-rs backend (Zlib), memchr (Unlicense or MIT), aws-lc-rs (Apache-2.0 or ISC) for AES and SHA-2, and md5 (Apache-2.0 or MIT). RC4 is implemented in the crate. + +### Consulted for ideas only (no code or data copied) + +- MuPDF (Artifex), AGPL-3.0: repair policy (run once, last occurrence wins, absolute offsets, object number cap of 8,388,607, `startxref 0` treated as missing, zero-width xref-stream field defaults); PNG predictor edge cases; the n-byte versus 16-byte MD5 variant of Algorithm 7 and forcing AESV3 for revisions 5 and 6; text heuristics from `pdf-op-run.c`, `stext-device.c`, `pdf-type3.c` and `pdf-cmap.c` (control-character ToUnicode targets, writing-direction line breaks, no synthetic spaces after CJK, overprint suppression, Type3 ASCII fallback, invalid-code codespace length). +- poppler, GPL-2.0-or-later (some files GPL-3.0): a sorted `endstream` index for recovering wrong `/Length` values; the 1,000,000-member object-stream limit; the frozen 4096-entry LZW table; numeric glyph name detection (`testForNumericNames`) and CIDSystemInfo ordering `UCS`; merging spacing accents into combining marks and estimating a Type3 font's em (`TextOutputDev.cc`). + +The 13 spacing-accent to combining-mark pairs in `tables::combining_accent` are Unicode facts (mostly the compatibility decompositions of the spacing accents), checked against poppler and PDFBox behaviour. + +### Specifications followed + +- ISO 32000-1:2008 and ISO 32000-2:2020 (PDF 1.7 and 2.0): syntax, filters, section 7.6 security handler algorithms, text state and glyph displacement (9.4.4), Type 0 code splitting (9.7.6.2), ToUnicode (9.10.3), marked content and ActualText (14.9.4), forms (12.7), annotations (12.5), text strings (7.9.2.2), Annex D encodings. +- Adobe Supplement to ISO 32000, BaseVersion 1.7, ExtensionLevel 3 (revision 5 encryption). +- Adobe Technical Notes #5014 and #5099 (CMaps), #5176 (CFF), #5040 (PFB), Adobe Type 1 Font Format, PostScript Language Reference 3rd edition (radix numbers). +- OpenType specification 1.9 (`cmap`, `post`, `maxp`, font collections). +- Unicode Standard Annex #9 (simplified per-line bidi reordering). + +## 2. Data + +All tables in `src/pdf/tables/` were generated from the sources below; the generator is not part of the repository. + +### Adobe Glyph List and ITC Zapf Dingbats Glyph List + +- Source: (`glyphlist.txt`, Adobe Glyph List 2.0, and `zapfdingbats.txt`, ITC Zapf Dingbats Glyph List 2.0, both dated September 20, 2002). +- Used for: glyph name to Unicode mapping (all 4281 AGL entries and all 201 Zapf Dingbats entries) in `src/pdf/tables/glyph_data.rs`. The name algorithm follows the Adobe Glyph List Specification (). +- License: BSD-3-Clause: + +``` +``` +Copyright 2002-2019 Adobe (http://www.adobe.com/). + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +Neither the name of Adobe nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +``` + +### Adobe CMap and CID resources + +- Sources: (`pdf2unicode/Adobe-Japan1-UCS2` (Supplement 7), `Adobe-GB1-UCS2`, `Adobe-CNS1-UCS2`, `Adobe-Korea1-UCS2`; commit 2dd5e53fb74a01718b9dfd448a0d1cce6fff2aa5) and (the predefined CMaps of ISO 32000 Table 118 plus the UTF-8 and UTF-32 Unicode CMaps; commit f5cf3bca7fdfeaceb77aa82847e974f2306c20b4). +- Used for: `src/pdf/tables/cid_japan1.bin`, `cid_gb1.bin`, `cid_cns1.bin`, `cid_korea1.bin` (CID to Unicode for the four Adobe character collections, loaded by `cid_data.rs`), and the predefined CMap table in `cmap_data.rs`. +- Modifications: CID tables re-encoded (delta-coded values, raw deflate); U+FFFD entries dropped; variation selectors U+FE00 to U+FE0F removed from mapped sequences. +- License: BSD-3-Clause. Both repositories carry the same text; the copyright lines are "Copyright 1990-2023 Adobe. All rights reserved." (cmap-resources) and "Copyright 1990-2019 Adobe. All rights reserved." (mapping-resources-pdf): + +``` +``` +Copyright 1990-2023 Adobe. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +Neither the name of Adobe nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +``` + +### Adobe Core 14 AFM metrics + +- Source: the Adobe Core 14 AFM files as redistributed by Apache PDFBox (`pdfbox/src/main/resources/org/apache/pdfbox/resources/afm/`, commit 2245d1c2bb3bbcfe56136a693c395317316ace96). The same metrics appear in pdf.js `src/core/metrics.js`. +- Used for: the advance widths (`WX`) of the 14 standard fonts in `src/pdf/tables/font_data.rs`. Only the widths are kept; the AFM files are not distributed. +- Modification notice: the widths were extracted from the AFM files and re-encoded as Rust tables (a shared width palette plus per-font indices); Oblique variants share the upright widths, which are identical in the AFM files. +- License notice (from `MustRead.html` accompanying the AFM files, unmodified): + +``` +``` +This file and the 14 PostScript(R) AFM files it accompanies may be used, +copied, and distributed for any purpose and without charge, with or without +modification, provided that all copyright notices are retained; that the AFM +files are not distributed without this file; that all modifications to this +file or any of the AFM files are prominently noted in the modified file(s); +and that this paragraph is not modified. Adobe Systems has no responsibility +or obligation to support the use of the AFM files. +``` +``` + +- Copyright notices of the individual AFM files: + - Courier, Courier-Oblique: Copyright (c) 1989, 1990, 1991, 1992, 1993, 1997 Adobe Systems Incorporated. All Rights Reserved. + - Courier-Bold, Courier-BoldOblique: Copyright (c) 1989, 1990, 1991, 1993, 1997 Adobe Systems Incorporated. All Rights Reserved. + - Helvetica, Helvetica-Bold, Helvetica-Oblique, Helvetica-BoldOblique: Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems Incorporated. All Rights Reserved. Helvetica is a trademark of Linotype-Hell AG and/or its subsidiaries. + - Times-Roman, Times-Bold, Times-Italic, Times-BoldItalic: Copyright (c) 1985, 1987, 1989, 1990, 1993, 1997 Adobe Systems Incorporated. All Rights Reserved. Times is a trademark of Linotype-Hell AG and/or its subsidiaries. + - Symbol: Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems Incorporated. All rights reserved. + - ZapfDingbats: Copyright (c) 1985, 1987, 1988, 1989, 1997 Adobe Systems Incorporated. All Rights Reserved. ITC Zapf Dingbats is a registered trademark of International Typeface Corporation. + +### Unicode Character Database + +- Source: Unicode 18.0.0 `UnicodeData.txt` and `DerivedNormalizationProps.txt` (). +- Used for: NFKC mappings of the presentation forms U+FB00 to U+FB4F, U+FB50 to U+FDFF and U+FE70 to U+FEFF (`src/pdf/tables/presentation_data.rs`). +- License: Unicode License v3: + +``` +``` +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. +``` +``` + +### PDF specification tables + +ISO 32000-1:2008 and ISO 32000-2:2020, Annex D (character sets and encodings) and section 9.6.2.2 (standard Type 1 fonts) were used to check the encoding tables and PDFDocEncoding (WinAnsi bullet for unused codes, 0xA0 as space, 0xAD as hyphen, MacRoman 0xDB as `currency`, the extra MacRoman space at 0xCA). The password padding constant in `src/pdf/crypt/` and the CFF standard strings (Appendix A of Technical Note #5176) and the 258 Macintosh glyph names (OpenType `post`) are specification data. + +## 3. Test fixtures + +Paths are relative to `crates/text-extract/`. Fixtures are used only by tests and are not compiled into the library. + +### Generated for this project + +These contain no third-party material beyond what the Notes column states and are covered by the project's own license. + +| Path | How it was made | +|---|---| +| `tests/fixtures/pdf/crypt/r2_rc4_40.pdf` | Generated by `.ignore/pdf/dev/crypt/vectors/generate.py` with qpdf, pikepdf and pypdf | +| `tests/fixtures/pdf/crypt/r3_empty_owner.pdf` | Same generator (pypdf, empty owner password) | +| `tests/fixtures/pdf/crypt/r3_rc4_128.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r3_user_pw.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r4_aes_128.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r4_aes_128_nometa.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r4_rc4_128.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r5_aes_256.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r6_aes_256.pdf` | Same generator | +| `tests/fixtures/pdf/crypt/r6_empty_owner.pdf` | Same generator (pypdf, empty owner password) | +| `tests/fixtures/pdf/crypt/r6_user_pw.pdf` | Same generator | +| `tests/fixtures/pdf/text/gen/acroform-filled.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/acroform-values-without-appearances.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/actualtext-spans.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/annotations-comments.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/arabic-cairo.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/broken-xref-and-trailer-missing.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/broken-xref-wrong-offsets.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/chromium-skia-two-column.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/cjk-chinese-cairo.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/cjk-cid-predefined-cmap.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/cjk-japanese-weasyprint.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/cjk-korean-cairo.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/cjk-vertical-writing.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/differences-encoding-no-tounicode.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/encrypted-aes-128.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/encrypted-rc4-40.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/filters-lzw-ascii85-hex-rle.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/form-xobject-and-inline-image.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/hebrew-cairo.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/identity-h-cairo-ligatures.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/incremental-update-prev-chain.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/invoice-weasyprint.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/libreoffice-writer-letter.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/linearized.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/rotated-pages.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/scanned-ocr-invisible-text.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/spacing-kerning-geometry.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/truetype-subset-multiscript.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/type1-winansi-std14.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/type3-matplotlib-chart.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/pdf/text/gen/xref-stream-object-streams.pdf` | Generated by `.ignore/pdf/oracle/gen_fixtures.py` | +| `tests/fixtures/real/textutil.docx` | Written by the macOS Cocoa text system (`textutil`), as its metadata shows; local generation, generator command not recorded | +| `tests/fixtures/real/textutil.odt` | Same (metadata generator `CocoaODFWriter/2685.7`) | +| `tests/fixtures/real/textutil.rtf` | Same (`\cocoartf2870` header) | +| `tests/fixtures/hostile/docx_billion_laughs.docx` | Minimal synthetic archive (entity expansion); hand-built for `tests/hostile.rs`, generator not recorded | +| `tests/fixtures/hostile/docx_deep_nesting.docx` | Minimal synthetic archive (deeply nested XML); generator not recorded | +| `tests/fixtures/hostile/docx_lzma_member.docx` | Minimal synthetic archive (LZMA-compressed member); generator not recorded | +| `tests/fixtures/hostile/ods_repeated_rows.ods` | Minimal synthetic archive (huge `number-rows-repeated`); generator not recorded | +| `tests/fixtures/hostile/xlsx_sparse_corners.xlsx` | Minimal synthetic archive (cells at opposite sheet corners); generator not recorded | +| `tests/fixtures/hostile/xlsx_sst_uniquecount.xlsx` | Minimal synthetic archive (inflated shared-string count); generator not recorded | + +Font program test data in `src/pdf/font/program/tests/data/`, generated by the font parser's `gen/gen_fixtures.py` (not in the repository) with fontTools (MIT, used only as a tool) from the fonts named below: + +| Path | Source font | Font license | +|---|---|---| +| `src/pdf/font/program/tests/data/cmr10-type1.t1` | Cleartext portion (first 2587 bytes plus 32 bytes of the encrypted part) of the CMR10 subset embedded in a TUGboat article PDF; AMS Computer Modern Type 1 fonts, Copyright (c) 1997, 2009 American Mathematical Society | SIL Open Font License 1.1 | +| `src/pdf/font/program/tests/data/cmr10-type1c.cff` | CMR10 Type1C subset embedded in the CTAN `lshort` (English) PDF; same font | SIL Open Font License 1.1 | +| `src/pdf/font/program/tests/data/dejavu-latin.ttf` | Subset of DejaVu Sans 2.37, cmap rebuilt (formats 6, 4, 12) | Bitstream Vera license, DejaVu changes public domain | +| `src/pdf/font/program/tests/data/dejavu-symbol.ttf` | Subset of DejaVu Sans 2.37, cmap rebuilt (format 0, (3,0) format 4, (3,4) format 2) | Same | +| `src/pdf/font/program/tests/data/dejavu-pair.ttc` | Collection of two DejaVu Sans 2.37 subsets | Same | +| `src/pdf/font/program/tests/data/noto-cjk-cid.otf` | Noto Sans CJK JP 2.004 (`NotoSansCJK-Regular.ttc`, font 0), subset to eight characters | SIL Open Font License 1.1 | +| `src/pdf/font/program/tests/data/noto-cjk-cid.cff` | The `CFF ` table of `noto-cjk-cid.otf` | SIL Open Font License 1.1 | +| `src/pdf/font/program/tests/data/cmr10-type1.t1.expect` | fontTools' view of the fixture (from the complete font program), written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/cmr10-type1c.cff.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/dejavu-latin.ttf.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/dejavu-symbol.ttf.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/dejavu-pair.ttc.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/noto-cjk-cid.otf.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | +| `src/pdf/font/program/tests/data/noto-cjk-cid.cff.expect` | Expectation written by `gen/oracle.py` | Project license (generated) | + +The filter test vectors in `src/pdf/filter/testdata/` come from the project-owned `scripts/gen_vectors.py` (Python standard library `zlib`, ASCII85 strings from CPython `base64.a85encode`) and were cross-checked with qpdf (through pikepdf) and pypdf. Synthetic PDFs in `tests/common/pdf.rs` are built by the tests at run time. + +### Taken from other projects + +All files below are unmodified copies. Copyright The Apache Software Foundation; license Apache-2.0 (). The NOTICE text for each project is in section 4. + +| Path | Source | License | +|---|---|---| +| `tests/fixtures/pdf/text/tika/testAnnotations.pdf` | Apache Tika, , `tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/` | Apache-2.0 | +| `tests/fixtures/pdf/text/tika/testExtraSpaces.pdf` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/pdf/text/tika/testOptionalHyphen.pdf` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/pdf/text/tika/testPDF_acroform3.pdf` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/pdf/text/pdfbox/GlyphLayoutBidi.pdf` | Apache PDFBox, ; embeds Noto Sans Arabic and DejaVu Sans subsets | Apache-2.0 (embedded fonts: OFL 1.1, Bitstream Vera license) | +| `tests/fixtures/pdf/text/pdfbox/GlyphLayoutLigaturesAndKerning_ActualText.pdf` | Apache PDFBox, same commit, `pdfbox-layout-awt/src/test/resources/pdf/`; embeds Fira Code, DejaVu Sans and Noto subsets | Apache-2.0 (embedded fonts: OFL 1.1, Bitstream Vera license) | +| `tests/fixtures/pdf/text/pdfbox/sampleForSpec.pdf` | Apache PDFBox, same commit, `pdfbox/src/test/resources/input/sampleForSpec.pdf` | Apache-2.0 | +| `tests/fixtures/real/testEXCEL_textbox.xlsx` | Apache Tika, `tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/` | Apache-2.0 | +| `tests/fixtures/real/testEXCEL_headers_footers.xlsx` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/real/testPPT_charts.pptx` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/real/testPPT_masterText.pptx` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/real/testRTFIgnoredControlWord.rtf` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/real/testRTFTIKA_1713.rtf` | Apache Tika, same directory | Apache-2.0 | +| `tests/fixtures/real/48539.xlsx` | Apache POI, , `test-data/spreadsheet/` | Apache-2.0 | +| `tests/fixtures/real/testEXCEL.xlsx` | Apache Tika test documents (parser module test resources) | Apache-2.0 | +| `tests/fixtures/real/testPPT.pptx` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testPPT_various.pptx` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testWORD.docx` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testWORD_various.docx` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testODFwithOOo3.odt` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testOpenOffice2.odt` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testRTF.rtf` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testRTFVarious.rtf` | Apache Tika test documents | Apache-2.0 | +| `tests/fixtures/real/testEPUB.epub` | Apache Tika test documents | Apache-2.0 | + +## 4. NOTICE + +The Apache-2.0 test files listed above are redistributed with the source tree. Section 4(d) of the Apache License requires the NOTICE text of each originating project to accompany them. The texts below were taken from the projects' repositories (`apache/tika` `NOTICE.txt`, `apache/pdfbox` `NOTICE.txt`, `apache/poi` `legal/NOTICE`) on 2026-09-27. + +### Apache Tika + +``` +Apache Tika +Copyright 2007-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata +This software contains code derived from UCAR/Unidata's NetCDF library. + +Tika-parsers component uses CDDL/LGPL dual-licensed dependency: jhighlight (https://github.com/codelibs/jhighlight) + +IPTC Photo Metadata descriptions Copyright 2010 International Press Telecommunications Council. + +Tika-mimetypes.xml includes mimetype definitions that were adapted from the PRONOM Technical Registry +by The National Archives (http://www.nationalarchives.gov.uk/PRONOM/Default.aspx). PRONOM is published +under the Open Government License 3.0 (http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) +``` + +### Apache PDFBox + +``` +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved +``` + +### Apache POI + +``` +Apache POI +Copyright 2003-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, (dead link), +which was acquired by Oracle Corporation in 2008. + + +Note: The ASF Secretary has on hand a Software Grant Agreement (SGA) from +BEA Systems, Inc. dated 9 Sep 2003 for XMLBeans signed by their EVP/CFO. + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project + and . +Copyright (c) 2009-2018 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT + +ExceptionUtils is derived from `scala.util.control.NonFatal` in scala-library +which was released under the Apache 2.0 license. + +Copyright (c) 2002-2023 EPFL +Copyright (c) 2011-2023 Lightbend, Inc. + +Scala includes software developed at +LAMP/EPFL (https://lamp.epfl.ch/) and +Lightbend, Inc. (https://www.lightbend.com/). +``` + +### pdf.js + +pdf.js has no NOTICE file. Its attribution, for the code and data adapted in sections 1 and 2: + +``` +pdf.js +Copyright Mozilla Foundation +Licensed under the Apache License, Version 2.0 +https://github.com/mozilla/pdf.js +``` + +The full Apache License 2.0 text is at . diff --git a/crates/text-extract/Cargo.toml b/crates/text-extract/Cargo.toml new file mode 100644 index 00000000..d2fd8a9c --- /dev/null +++ b/crates/text-extract/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "text-extract" +version = "1.0.0" +edition = "2024" + +[dependencies] +memchr = "2.8" +encoding_rs = "0.8" +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +mail-parser = { version = "1.0", default-features = false } +hashify = "0.3" +aws-lc-rs = { version = "1" } +md5 = "0.8.1" + +[dev-dependencies] +zip = { version = "9.1", default-features = false, features = ["deflate-flate2-zlib-rs"] } + +[lints] +workspace = true diff --git a/crates/text-extract/src/epub.rs b/crates/text-extract/src/epub.rs new file mode 100644 index 00000000..e46195cb --- /dev/null +++ b/crates/text-extract/src/epub.rs @@ -0,0 +1,219 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + html::HtmlText, + output::{Output, Separator}, + package::{Arena, Package, Span}, + xml::{Handler, Tag, Text, stream::Part}, +}; +use memchr::memrchr; + +pub(crate) const CONTAINER: &[u8] = b"META-INF/container.xml"; + +#[derive(Debug, Clone, Copy)] +struct Item { + id: Span, + href: Span, +} + +#[derive(Default)] +pub(crate) struct Scratch { + arena: Arena, + manifest: Vec, + spine: Vec, + chapter: HtmlText, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Element { + Rootfile, + Item, + ItemRef, +} + +impl Element { + fn parse(local: &[u8]) -> Option { + hashify::map!(local, Element, + b"rootfile" => Element::Rootfile, + b"item" => Element::Item, + b"itemref" => Element::ItemRef, + ) + .copied() + } +} + +fn is_html_media_type(media_type: &[u8]) -> bool { + hashify::set_ignore_case!( + media_type.trim_ascii(), + b"application/xhtml+xml", + b"text/html", + b"application/html", + b"text/x-oeb1-document", + ) +} + +fn has_html_extension(name: &[u8]) -> bool { + memrchr(b'.', name) + .and_then(|dot| name.get(dot + 1..)) + .is_some_and(|extension| { + hashify::set_ignore_case!(extension, b"xhtml", b"html", b"htm", b"xht") + }) +} + +struct ContainerHandler<'x> { + arena: &'x mut Arena, + rootfile: Option, +} + +impl Handler for ContainerHandler<'_> { + fn start(&mut self, tag: &Tag<'_>, _out: &mut Output<'_>) { + if self.rootfile.is_some() || Element::parse(tag.local()) != Some(Element::Rootfile) { + return; + } + let mut path = None; + let mut is_package = true; + for (name, value) in tag.attributes() { + hashify::fnc_map!(name, + b"full-path" => { path = Some(value); }, + b"media-type" => { + is_package = hashify::set_ignore_case!(value, b"application/oebps-package+xml"); + }, + _ => {} + ); + } + if is_package { + self.rootfile = path.and_then(|path| self.arena.push_decoded(path)); + } + } + + fn end(&mut self, _name: &[u8], _out: &mut Output<'_>) {} + + fn text(&mut self, _text: Text<'_>, _out: &mut Output<'_>) {} +} + +struct PackageHandler<'x> { + arena: &'x mut Arena, + manifest: &'x mut Vec, + spine: &'x mut Vec, + source: Span, + limit: usize, + capped: bool, +} + +impl Handler for PackageHandler<'_> { + fn start(&mut self, tag: &Tag<'_>, _out: &mut Output<'_>) { + match Element::parse(tag.local()) { + Some(Element::Item) if self.manifest.len() < self.limit => { + let (mut id, mut href, mut html) = (None, None, None); + for (name, value) in tag.attributes() { + hashify::fnc_map!(name, + b"id" => { id = Some(value); }, + b"href" => { href = Some(value); }, + b"media-type" => { html = Some(is_html_media_type(value)); }, + _ => {} + ); + } + let (Some(id), Some(href)) = (id, href) else { + return; + }; + if !html.unwrap_or_else(|| has_html_extension(href)) { + return; + } + let mark = self.arena.mark(); + match ( + self.arena.push_decoded(id), + self.arena.push_resolved(self.source, href), + ) { + (Some(id), Some(href)) => self.manifest.push(Item { id, href }), + _ => self.arena.rewind(mark), + } + } + Some(Element::ItemRef) if self.spine.len() >= self.limit => self.capped = true, + Some(Element::ItemRef) => { + if let Some(id) = tag + .attributes() + .find_map(|(name, value)| hashify::set!(name, b"idref").then_some(value)) + .and_then(|id| self.arena.push_decoded(id)) + { + self.spine.push(id); + } + } + _ => {} + } + } + + fn end(&mut self, _name: &[u8], _out: &mut Output<'_>) {} + + fn text(&mut self, _text: Text<'_>, _out: &mut Output<'_>) {} +} + +pub(crate) fn extract(package: &mut Package<'_, '_>, scratch: &mut Scratch, out: &mut Output<'_>) { + let Scratch { + arena, + manifest, + spine, + chapter, + } = scratch; + arena.clear(); + manifest.clear(); + spine.clear(); + let limit = package.budget.parts; + + let mut container = ContainerHandler { + arena, + rootfile: None, + }; + package.scan_plumbing(CONTAINER, &mut container, out); + if let Some(rootfile) = container.rootfile + && let Some(member) = package.archive.find(arena.get(rootfile)) + { + let mut handler = PackageHandler { + arena, + manifest, + spine, + source: rootfile, + limit, + capped: false, + }; + package.scan_member_as(&member, Part::Plumbing, &mut handler, out); + package.budget.truncated |= handler.capped; + } + + manifest.sort_unstable_by(|left, right| arena.get(left.id).cmp(arena.get(right.id))); + let mut chapters = 0usize; + for &idref in spine.iter() { + if package.stopped(out) { + return; + } + let id = arena.get(idref); + let index = manifest.partition_point(|item| arena.get(item.id) < id); + let Some(item) = manifest.get(index).filter(|item| arena.get(item.id) == id) else { + continue; + }; + chapter.reset(); + if package.scan(arena.get(item.href), chapter, out) { + chapters += 1; + out.separator(Separator::Newline); + } + } + if chapters > 0 { + return; + } + let archive = package.archive; + for member in archive + .members() + .filter(|member| member.is_readable() && has_html_extension(member.name)) + { + if package.stopped(out) { + return; + } + chapter.reset(); + if package.scan_member(&member, chapter, out) { + out.separator(Separator::Newline); + } + } +} diff --git a/crates/text-extract/src/html.rs b/crates/text-extract/src/html.rs new file mode 100644 index 00000000..ae3fdd72 --- /dev/null +++ b/crates/text-extract/src/html.rs @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + output::{Output, Separator}, + xml::{Handler, Skip, Tag, Text, local_name}, +}; +use mail_parser::decoders::html::add_html_token; + +const MAX_ENTITY_TOKEN: usize = 40; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Element { + Block, + Cell, + Head, + Script, + Style, + Template, + Ruby, +} + +impl Element { + fn parse(local: &[u8]) -> Option { + hashify::map_ignore_case!(local, Element, + b"head" => Element::Head, + b"script" => Element::Script, + b"style" => Element::Style, + b"template" => Element::Template, + b"rt" => Element::Ruby, + b"rp" => Element::Ruby, + b"p" => Element::Block, + b"div" => Element::Block, + b"br" => Element::Block, + b"li" => Element::Block, + b"ul" => Element::Block, + b"ol" => Element::Block, + b"tr" => Element::Block, + b"h1" => Element::Block, + b"h2" => Element::Block, + b"h3" => Element::Block, + b"h4" => Element::Block, + b"h5" => Element::Block, + b"h6" => Element::Block, + b"blockquote" => Element::Block, + b"pre" => Element::Block, + b"section" => Element::Block, + b"article" => Element::Block, + b"header" => Element::Block, + b"footer" => Element::Block, + b"nav" => Element::Block, + b"aside" => Element::Block, + b"table" => Element::Block, + b"dt" => Element::Block, + b"dd" => Element::Block, + b"dl" => Element::Block, + b"hr" => Element::Block, + b"figure" => Element::Block, + b"figcaption" => Element::Block, + b"details" => Element::Block, + b"summary" => Element::Block, + b"noscript" => Element::Block, + b"address" => Element::Block, + b"main" => Element::Block, + b"body" => Element::Block, + b"caption" => Element::Block, + b"title" => Element::Block, + b"td" => Element::Cell, + b"th" => Element::Cell, + ) + .copied() + } +} + +pub(crate) fn tag_separator(local: &[u8]) -> Option { + match Element::parse(local)? { + Element::Block => Some(Separator::Newline), + Element::Cell => Some(Separator::Space), + _ => None, + } +} + +#[derive(Default)] +pub(crate) struct HtmlText { + skip: Skip, + entity: String, +} + +impl HtmlText { + pub(crate) fn reset(&mut self) { + self.skip.reset(); + } +} + +impl Handler for HtmlText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let element = Element::parse(tag.local()); + if self.skip.on_start(element) { + return; + } + match element { + Some(Element::Block) => out.separator(Separator::Newline), + Some(Element::Cell) => out.separator(Separator::Space), + Some(skipped) => self.skip.begin(skipped), + None => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + let element = Element::parse(local_name(name)); + if self.skip.on_end(element) { + return; + } + match element { + Some(Element::Block) => out.separator(Separator::Newline), + Some(Element::Cell) => out.separator(Separator::Space), + _ => {} + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + if self.skip.active() { + return; + } + text.push(out); + } + + fn entity(&mut self, name: &[u8], out: &mut Output<'_>) { + if self.skip.active() { + return; + } + let mut token = [0u8; MAX_ENTITY_TOKEN]; + let Some(slot) = token.get_mut(..name.len() + 2) else { + return; + }; + let [first, middle @ .., last] = slot else { + return; + }; + *first = b'&'; + middle.copy_from_slice(name); + *last = b';'; + self.entity.clear(); + add_html_token(&mut self.entity, slot, false); + if !self.entity.starts_with('&') { + out.push_str(&self.entity); + } + } +} diff --git a/crates/text-extract/src/lib.rs b/crates/text-extract/src/lib.rs new file mode 100644 index 00000000..98783ea5 --- /dev/null +++ b/crates/text-extract/src/lib.rs @@ -0,0 +1,405 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod epub; +mod html; +mod odf; +mod ooxml; +mod output; +mod package; +mod pdf; +mod rtf; +mod utf8; +mod xml; +mod zip; + +use output::Output; +use package::Package; +use xml::stream::{Budget, Buffers}; +use zip::{Archive, Entry, MemberData, ReadRanges}; + +const MIMETYPE: &[u8] = b"mimetype"; +const MAX_MIMETYPE: usize = 128; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Format { + Docx, + Xlsx, + Pptx, + Odt, + Ods, + Odp, + Epub, + Rtf, + Pdf, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct Hints<'a> { + media_type: Option<(&'a str, &'a str)>, + file_name: Option<&'a str>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Limits { + pub max_input_bytes: usize, + pub max_output_bytes: usize, + pub max_entries: usize, + pub max_parts: usize, + pub max_part_bytes: u64, + pub max_total_bytes: u64, + pub max_rtf_depth: usize, + pub max_pdf_objects: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Extraction { + pub format: Format, + pub truncated: bool, + pub bytes_written: usize, + pub bytes_decompressed: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + Unsupported, + TooLarge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Failure { + pub error: Error, + pub bytes_decompressed: u64, +} + +#[derive(Default)] +pub struct Extractor { + limits: Limits, + entries: Vec, + claimed: ReadRanges, + buffers: Buffers, + ooxml: ooxml::Scratch, + epub: epub::Scratch, + rtf: rtf::Scratch, + pdf: pdf::Scratch, +} + +enum Container { + Ooxml, + Odf(Option), + Epub, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + max_input_bytes: 64 << 20, + max_output_bytes: 4 << 20, + max_entries: 10_000, + max_parts: 10_000, + max_part_bytes: 64 << 20, + max_total_bytes: 256 << 20, + max_rtf_depth: 256, + max_pdf_objects: 1 << 20, + } + } +} + +impl<'a> Hints<'a> { + pub fn new() -> Self { + Hints::default() + } + + pub fn with_media_type(mut self, media_type: &'a str) -> Self { + let essence = media_type + .split_once(';') + .map_or(media_type, |(essence, _)| essence) + .trim(); + self.media_type = Some(essence.split_once('/').unwrap_or((essence, ""))); + self + } + + pub fn with_media_type_parts(mut self, type_: &'a str, subtype: &'a str) -> Self { + self.media_type = Some((type_, subtype)); + self + } + + pub fn with_file_name(mut self, file_name: &'a str) -> Self { + self.file_name = Some(file_name); + self + } + + pub fn format(&self) -> Option { + self.media_type_format().or_else(|| self.extension_format()) + } + + pub fn may_be_supported(&self) -> bool { + match self.media_type { + None => self.file_name.is_none() || self.extension_format().is_some(), + Some(_) => { + self.media_type_format().is_some() + || self.extension_format().is_some() + || (self.is_generic_media_type() && !self.has_extension()) + } + } + } + + fn media_type_format(&self) -> Option { + let (type_, subtype) = self.media_type?; + if hashify::set_ignore_case!(type_.as_bytes(), b"text") { + return hashify::map_ignore_case!(subtype.as_bytes(), Format, + b"rtf" => Format::Rtf, + b"pdf" => Format::Pdf, + b"x-pdf" => Format::Pdf, + ) + .copied(); + } + if !hashify::set_ignore_case!(type_.as_bytes(), b"application") { + return None; + } + hashify::map_ignore_case!(subtype.as_bytes(), Format, + b"vnd.openxmlformats-officedocument.wordprocessingml.document" => Format::Docx, + b"vnd.openxmlformats-officedocument.wordprocessingml.template" => Format::Docx, + b"vnd.ms-word.document.macroenabled.12" => Format::Docx, + b"vnd.ms-word.template.macroenabled.12" => Format::Docx, + b"vnd.openxmlformats-officedocument.spreadsheetml.sheet" => Format::Xlsx, + b"vnd.openxmlformats-officedocument.spreadsheetml.template" => Format::Xlsx, + b"vnd.ms-excel.sheet.macroenabled.12" => Format::Xlsx, + b"vnd.ms-excel.template.macroenabled.12" => Format::Xlsx, + b"vnd.openxmlformats-officedocument.presentationml.presentation" => Format::Pptx, + b"vnd.openxmlformats-officedocument.presentationml.slideshow" => Format::Pptx, + b"vnd.openxmlformats-officedocument.presentationml.template" => Format::Pptx, + b"vnd.ms-powerpoint.presentation.macroenabled.12" => Format::Pptx, + b"vnd.ms-powerpoint.slideshow.macroenabled.12" => Format::Pptx, + b"vnd.oasis.opendocument.text" => Format::Odt, + b"vnd.oasis.opendocument.text-template" => Format::Odt, + b"vnd.oasis.opendocument.spreadsheet" => Format::Ods, + b"vnd.oasis.opendocument.spreadsheet-template" => Format::Ods, + b"vnd.oasis.opendocument.presentation" => Format::Odp, + b"vnd.oasis.opendocument.presentation-template" => Format::Odp, + b"epub+zip" => Format::Epub, + b"rtf" => Format::Rtf, + b"x-rtf" => Format::Rtf, + b"pdf" => Format::Pdf, + b"x-pdf" => Format::Pdf, + b"acrobat" => Format::Pdf, + b"vnd.pdf" => Format::Pdf, + ) + .copied() + } + + fn is_generic_media_type(&self) -> bool { + self.media_type.is_some_and(|(type_, subtype)| { + hashify::set_ignore_case!(type_.as_bytes(), b"application") + && hashify::set_ignore_case!( + subtype.as_bytes(), + b"octet-stream", + b"zip", + b"x-zip", + b"x-zip-compressed", + b"msword", + b"vnd.ms-excel", + b"vnd.ms-powerpoint", + b"vnd.ms-office", + b"force-download", + b"x-download", + b"download", + b"binary", + b"unknown", + ) + }) + } + + fn has_extension(&self) -> bool { + self.file_name.is_some_and(|name| { + name.rsplit_once('.') + .is_some_and(|(stem, _)| !stem.is_empty()) + }) + } + + fn extension_format(&self) -> Option { + let (_, extension) = self.file_name?.rsplit_once('.')?; + hashify::map_ignore_case!(extension.as_bytes().trim_ascii_end(), Format, + b"docx" => Format::Docx, + b"docm" => Format::Docx, + b"dotx" => Format::Docx, + b"dotm" => Format::Docx, + b"xlsx" => Format::Xlsx, + b"xlsm" => Format::Xlsx, + b"xltx" => Format::Xlsx, + b"xltm" => Format::Xlsx, + b"pptx" => Format::Pptx, + b"pptm" => Format::Pptx, + b"ppsx" => Format::Pptx, + b"ppsm" => Format::Pptx, + b"potx" => Format::Pptx, + b"potm" => Format::Pptx, + b"odt" => Format::Odt, + b"ott" => Format::Odt, + b"ods" => Format::Ods, + b"ots" => Format::Ods, + b"odp" => Format::Odp, + b"otp" => Format::Odp, + b"epub" => Format::Epub, + b"rtf" => Format::Rtf, + b"pdf" => Format::Pdf, + ) + .copied() + } +} + +impl Extractor { + pub fn new(limits: Limits) -> Self { + Extractor { + limits, + ..Default::default() + } + } + + pub fn limits(&self) -> &Limits { + &self.limits + } + + pub fn limits_mut(&mut self) -> &mut Limits { + &mut self.limits + } + + pub fn extract( + &mut self, + data: &[u8], + hints: Hints<'_>, + out: &mut String, + ) -> Result { + if data.len() > self.limits.max_input_bytes { + return Err(Error::TooLarge.into()); + } + let original_len = out.len(); + self.extract_into(data, hints, out) + .inspect_err(|_| out.truncate(original_len)) + } + + fn extract_into( + &mut self, + data: &[u8], + hints: Hints<'_>, + out: &mut String, + ) -> Result { + let mut output = Output::new(out, self.limits.max_output_bytes); + if rtf::is_rtf(data) { + rtf::extract(data, &mut self.rtf, self.limits.max_rtf_depth, &mut output); + return Ok(Extraction { + format: Format::Rtf, + truncated: output.is_full(), + bytes_written: output.written(), + bytes_decompressed: 0, + }); + } + if pdf::is_pdf(data) { + let summary = pdf::extract(data, &mut self.pdf, &self.limits, &mut output)?; + return Ok(Extraction { + format: Format::Pdf, + truncated: output.is_full() || summary.truncated, + bytes_written: output.written(), + bytes_decompressed: summary.used_bytes, + }); + } + + let Extractor { + limits, + entries, + claimed, + buffers, + ooxml, + epub, + .. + } = self; + let archive = Archive::open(data, entries, limits.max_entries).ok_or(Error::Unsupported)?; + claimed.clear(); + let mut package = Package { + archive: &archive, + buffers, + claimed, + budget: Budget { + part_bytes: limits.max_part_bytes, + total_bytes: limits.max_total_bytes, + parts: limits.max_parts, + used_bytes: 0, + used_parts: 0, + truncated: archive.entries_truncated, + }, + }; + let container = detect(&mut package, hints).ok_or(Error::Unsupported)?; + let format = match container { + Container::Ooxml => ooxml::extract(&mut package, ooxml, &mut output), + Container::Odf(declared) => odf::extract(&mut package, declared, &mut output), + Container::Epub => { + epub::extract(&mut package, epub, &mut output); + Some(Format::Epub) + } + } + .ok_or(Failure { + error: Error::Unsupported, + bytes_decompressed: package.budget.used_bytes, + })?; + Ok(Extraction { + format, + truncated: output.is_full() || package.budget.truncated, + bytes_written: output.written(), + bytes_decompressed: package.budget.used_bytes, + }) + } +} + +impl From for Failure { + fn from(error: Error) -> Self { + Failure { + error, + bytes_decompressed: 0, + } + } +} + +fn detect(package: &mut Package<'_, '_>, hints: Hints<'_>) -> Option { + let archive = package.archive; + let declared = + archive + .find(MIMETYPE) + .and_then(|member| match archive.read(&member, package.claimed)? { + MemberData::Stored(value) if value.len() <= MAX_MIMETYPE => Some(value), + _ => None, + }); + if let Some(declared) = declared { + if hashify::set!(declared.trim_ascii(), b"application/epub+zip") { + return Some(Container::Epub); + } + if let Some(format) = odf::format_from_mimetype(declared) { + return Some(Container::Odf(Some(format))); + } + } + let has_odf_content = archive.contains(odf::CONTENT); + let has_epub_container = archive.contains(epub::CONTAINER); + if ooxml::is_package(package) { + return Some(Container::Ooxml); + } + match (has_odf_content, has_epub_container) { + (true, true) => Some(match hints.format() { + Some(Format::Epub) => Container::Epub, + _ => Container::Odf(None), + }), + (true, false) => Some(Container::Odf(None)), + (false, true) => Some(Container::Epub), + (false, false) => None, + } +} + +pub fn extract( + data: &[u8], + hints: Hints<'_>, + limits: &Limits, + out: &mut String, +) -> Result { + Extractor::new(limits.clone()).extract(data, hints, out) +} diff --git a/crates/text-extract/src/odf.rs b/crates/text-extract/src/odf.rs new file mode 100644 index 00000000..560de69d --- /dev/null +++ b/crates/text-extract/src/odf.rs @@ -0,0 +1,277 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Format, + output::{Output, Separator}, + package::Package, + xml::{ + Handler, Skip, Tag, Text, + attr::{push_decoded, split_prefix}, + }, +}; + +pub(crate) const CONTENT: &[u8] = b"content.xml"; +const STYLES: &[u8] = b"styles.xml"; +const OBJECT_CONTENT: &[u8] = b"/content.xml"; + +pub(crate) fn format_from_mimetype(mimetype: &[u8]) -> Option { + hashify::map!(mimetype.trim_ascii(), Format, + b"application/vnd.oasis.opendocument.text" => Format::Odt, + b"application/vnd.oasis.opendocument.text-template" => Format::Odt, + b"application/vnd.oasis.opendocument.text-master" => Format::Odt, + b"application/vnd.oasis.opendocument.text-master-template" => Format::Odt, + b"application/vnd.oasis.opendocument.text-web" => Format::Odt, + b"application/vnd.oasis.opendocument.spreadsheet" => Format::Ods, + b"application/vnd.oasis.opendocument.spreadsheet-template" => Format::Ods, + b"application/vnd.oasis.opendocument.presentation" => Format::Odp, + b"application/vnd.oasis.opendocument.presentation-template" => Format::Odp, + ) + .copied() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Element { + Body, + MasterStyles, + TextBody, + SpreadsheetBody, + PresentationBody, + Ignored(Ignored), + Space, + Newline, + Block, + Frame, + Boundary, + PageField, + Cell, + Table, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ignored { + TrackedChanges, + BinaryData, + IndexSource, + Metadata, +} + +impl Element { + fn classify(name: &[u8]) -> Option { + let (_, local) = split_prefix(name); + match Element::parse(local) { + Some(Element::Ignored(Ignored::Metadata)) => { + let prefix = name.get(..name.len() - local.len()).unwrap_or_default(); + hashify::set!(prefix, b"dc:", b"meta:") + .then_some(Element::Ignored(Ignored::Metadata)) + } + element => element, + } + } + + fn parse(local: &[u8]) -> Option { + hashify::map!(local, Element, + b"body" => Element::Body, + b"master-styles" => Element::MasterStyles, + b"header" => Element::Block, + b"footer" => Element::Block, + b"header-left" => Element::Block, + b"footer-left" => Element::Block, + b"header-first" => Element::Block, + b"footer-first" => Element::Block, + b"text" => Element::TextBody, + b"spreadsheet" => Element::SpreadsheetBody, + b"presentation" => Element::PresentationBody, + b"tracked-changes" => Element::Ignored(Ignored::TrackedChanges), + b"binary-data" => Element::Ignored(Ignored::BinaryData), + b"table-of-content-source" => Element::Ignored(Ignored::IndexSource), + b"alphabetical-index-source" => Element::Ignored(Ignored::IndexSource), + b"illustration-index-source" => Element::Ignored(Ignored::IndexSource), + b"table-index-source" => Element::Ignored(Ignored::IndexSource), + b"object-index-source" => Element::Ignored(Ignored::IndexSource), + b"user-index-source" => Element::Ignored(Ignored::IndexSource), + b"bibliography-source" => Element::Ignored(Ignored::IndexSource), + b"creator" => Element::Ignored(Ignored::Metadata), + b"creator-initials" => Element::Ignored(Ignored::Metadata), + b"date" => Element::Ignored(Ignored::Metadata), + b"date-string" => Element::Ignored(Ignored::Metadata), + b"s" => Element::Space, + b"tab" => Element::Space, + b"line-break" => Element::Newline, + b"p" => Element::Newline, + b"h" => Element::Newline, + b"list-item" => Element::Block, + b"table-row" => Element::Block, + b"frame" => Element::Frame, + b"page" => Element::Block, + b"note-body" => Element::Block, + b"annotation" => Element::Block, + b"title" => Element::Block, + b"desc" => Element::Block, + b"note-citation" => Element::Space, + b"ruby-text" => Element::Boundary, + b"page-number" => Element::PageField, + b"page-count" => Element::PageField, + b"table-cell" => Element::Cell, + b"covered-table-cell" => Element::Cell, + b"table" => Element::Table, + ) + .copied() + } +} + +struct OdfText { + scope: Element, + active: u32, + skip: Skip, + body_kind: Option, + awaiting_body_kind: bool, + requires_body_kind: bool, + unsupported_body: bool, +} + +impl OdfText { + fn new(scope: Element, requires_body_kind: bool) -> Self { + OdfText { + scope, + active: 0, + skip: Skip::default(), + body_kind: None, + awaiting_body_kind: false, + requires_body_kind, + unsupported_body: false, + } + } +} + +fn table_name<'a>(tag: &Tag<'a>) -> Option<&'a [u8]> { + tag.attributes() + .find_map(|(name, value)| match split_prefix(name) { + (true, local) if hashify::set!(local, b"name") => Some(value), + _ => None, + }) +} + +fn is_prompt_frame(tag: &Tag<'_>) -> bool { + tag.attributes() + .any(|(name, value)| match split_prefix(name) { + (true, b"placeholder") => value == b"true", + (true, b"class") => hashify::set!( + value, + b"title", + b"subtitle", + b"outline", + b"notes", + b"text", + b"object", + b"chart", + b"table", + b"graphic", + b"orgchart", + b"page", + b"handout", + ), + _ => false, + }) +} + +impl Handler for OdfText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let element = Element::classify(tag.name); + if element == Some(self.scope) { + self.active = self.active.saturating_add(1); + self.awaiting_body_kind = self.scope == Element::Body; + return; + } + if self.active == 0 || self.skip.on_start(element) { + return; + } + if self.awaiting_body_kind { + self.awaiting_body_kind = false; + self.body_kind = match element { + Some(Element::TextBody) => Some(Format::Odt), + Some(Element::SpreadsheetBody) => Some(Format::Ods), + Some(Element::PresentationBody) => Some(Format::Odp), + _ => None, + }; + self.unsupported_body = self.requires_body_kind && self.body_kind.is_none(); + } + match element { + Some(ignored @ Element::Ignored(_)) => self.skip.begin(ignored), + Some(Element::PageField) if self.scope == Element::MasterStyles => { + self.skip.begin(Element::PageField) + } + Some(Element::Frame) if self.scope == Element::MasterStyles && is_prompt_frame(tag) => { + self.skip.begin(Element::Frame) + } + Some(Element::Space | Element::Boundary) => out.separator(Separator::Space), + Some(Element::Newline) => out.separator(Separator::Newline), + Some(Element::Table) if self.body_kind == Some(Format::Ods) => { + if let Some(name) = table_name(tag) { + out.separator(Separator::Newline); + push_decoded(name, out); + out.separator(Separator::Newline); + } + } + _ => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + let element = Element::classify(name); + if element == Some(self.scope) { + self.active = self.active.saturating_sub(1); + return; + } + if self.active == 0 || self.skip.on_end(element) { + return; + } + match element { + Some(Element::Newline | Element::Block | Element::Frame) => { + out.separator(Separator::Newline) + } + Some(Element::Cell | Element::Boundary) => out.separator(Separator::Space), + _ => {} + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + if self.active > 0 && !self.skip.active() { + text.push(out); + } + } + + fn aborted(&self) -> bool { + self.unsupported_body + } +} + +pub(crate) fn extract( + package: &mut Package<'_, '_>, + declared: Option, + out: &mut Output<'_>, +) -> Option { + let mut content = OdfText::new(Element::Body, declared.is_none()); + if !package.scan(CONTENT, &mut content, out) { + return None; + } + let format = declared.or(content.body_kind)?; + out.separator(Separator::Newline); + package.scan(STYLES, &mut OdfText::new(Element::MasterStyles, false), out); + out.separator(Separator::Newline); + let archive = package.archive; + for member in archive.members().filter(|member| { + member.name.len() > OBJECT_CONTENT.len() && member.name.ends_with(OBJECT_CONTENT) + }) { + if package.stopped(out) { + break; + } + if package.scan_member(&member, &mut OdfText::new(Element::Body, false), out) { + out.separator(Separator::Newline); + } + } + Some(format) +} diff --git a/crates/text-extract/src/ooxml/chart.rs b/crates/text-extract/src/ooxml/chart.rs new file mode 100644 index 00000000..27bce445 --- /dev/null +++ b/crates/text-extract/src/ooxml/chart.rs @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::number::{DateSystem, NumberFormat, ValueBuffer, push_number}; +use crate::{ + output::{Output, Separator}, + xml::{Handler, Skip, Tag, Text, local_name}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ChartValues { + Strings, + All, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ChartElement { + RichText, + Value, + Paragraph, + StringData, + NumberData, + FormatCode, + Level, + Date1904, + Category, + Skipped, +} + +impl ChartElement { + fn parse(local: &[u8]) -> Option { + hashify::map!(local, ChartElement, + b"t" => ChartElement::RichText, + b"v" => ChartElement::Value, + b"pt" => ChartElement::Value, + b"p" => ChartElement::Paragraph, + b"tx" => ChartElement::StringData, + b"txData" => ChartElement::StringData, + b"strCache" => ChartElement::StringData, + b"strLit" => ChartElement::StringData, + b"multiLvlStrCache" => ChartElement::StringData, + b"strDim" => ChartElement::StringData, + b"numCache" => ChartElement::NumberData, + b"numLit" => ChartElement::NumberData, + b"numDim" => ChartElement::NumberData, + b"formatCode" => ChartElement::FormatCode, + b"lvl" => ChartElement::Level, + b"date1904" => ChartElement::Date1904, + b"cat" => ChartElement::Category, + b"xVal" => ChartElement::Category, + b"extLst" => ChartElement::Skipped, + b"Fallback" => ChartElement::Skipped, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Collect { + Nothing, + Text, + Number, + FormatCode, +} + +pub(crate) struct ChartText { + values: ChartValues, + skip: Skip, + strings: u32, + numbers: u32, + collect: Collect, + number: ValueBuffer, + format: NumberFormat, + dates: DateSystem, + categories_done: bool, +} + +impl ChartText { + pub(crate) fn new(values: ChartValues) -> Self { + ChartText { + values, + skip: Skip::default(), + strings: 0, + numbers: 0, + collect: Collect::Nothing, + number: ValueBuffer::default(), + format: NumberFormat::General, + dates: DateSystem::Epoch1900, + categories_done: false, + } + } + + fn value_mode(&mut self) -> Collect { + if self.strings > 0 { + Collect::Text + } else if self.numbers > 0 && self.values == ChartValues::All { + self.number.clear(); + Collect::Number + } else { + Collect::Nothing + } + } +} + +impl Handler for ChartText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let element = ChartElement::parse(tag.local()); + if self.skip.on_start(element) { + return; + } + match element { + Some(ChartElement::RichText) => self.collect = Collect::Text, + Some(ChartElement::Value) => self.collect = self.value_mode(), + Some(ChartElement::StringData) => self.strings = self.strings.saturating_add(1), + Some(ChartElement::NumberData) => { + self.numbers = self.numbers.saturating_add(1); + self.format = NumberFormat::General; + } + Some(ChartElement::FormatCode) if self.numbers > 0 => { + self.number.clear(); + self.collect = Collect::FormatCode; + } + Some(ChartElement::Level) if self.numbers > 0 => { + if let Some(code) = tag + .attributes() + .find_map(|(name, value)| hashify::set!(name, b"formatCode").then_some(value)) + { + self.format = NumberFormat::classify(code); + } + } + Some(ChartElement::Date1904) => self.dates = DateSystem::from_value(tag), + Some(ChartElement::Category) + if self.values == ChartValues::Strings || self.categories_done => + { + self.skip.begin(ChartElement::Category) + } + Some(ChartElement::Skipped) => self.skip.begin(ChartElement::Skipped), + Some(ChartElement::Paragraph) => out.separator(Separator::Newline), + Some(ChartElement::FormatCode | ChartElement::Level | ChartElement::Category) + | None => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + let element = ChartElement::parse(local_name(name)); + if self.skip.on_end(element) { + return; + } + match element { + Some(ChartElement::RichText | ChartElement::Value) => { + if self.collect == Collect::Number { + push_number(self.number.value(), self.format, self.dates, out); + self.number.clear(); + } + self.collect = Collect::Nothing; + out.separator(Separator::Space); + } + Some(ChartElement::FormatCode) if self.collect == Collect::FormatCode => { + self.format = NumberFormat::classify(self.number.value()); + self.number.clear(); + self.collect = Collect::Nothing; + } + Some(ChartElement::StringData) => self.strings = self.strings.saturating_sub(1), + Some(ChartElement::NumberData) => self.numbers = self.numbers.saturating_sub(1), + Some(ChartElement::Paragraph) => out.separator(Separator::Newline), + Some(ChartElement::Category) => self.categories_done = true, + Some( + ChartElement::FormatCode + | ChartElement::Level + | ChartElement::Date1904 + | ChartElement::Skipped, + ) + | None => {} + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + match self.collect { + Collect::Nothing => {} + Collect::Text => text.push(out), + Collect::Number => { + if !self.number.push(text.as_bytes(), out) { + self.collect = Collect::Text; + } + } + Collect::FormatCode => { + if !self.number.fill(text.as_bytes()) { + self.collect = Collect::Nothing; + } + } + } + } +} diff --git a/crates/text-extract/src/ooxml/comments.rs b/crates/text-extract/src/ooxml/comments.rs new file mode 100644 index 00000000..d87bde48 --- /dev/null +++ b/crates/text-extract/src/ooxml/comments.rs @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::text::RunText; +use crate::{ + output::{Mark, Output, Separator}, + xml::{Handler, Tag, Text, local_name}, +}; + +const THREADED_MIRROR: &str = "[Threaded comment]"; + +#[derive(Default)] +pub(crate) struct CommentsText { + text: RunText, + comment: Option, + discarded: String, +} + +impl Handler for CommentsText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + if self.comment.is_none() && tag.local() == b"comment" { + self.comment = Some(out.mark()); + } + self.text.start(tag, out); + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + self.text.end(name, out); + match local_name(name) { + b"comment" => { + if let Some(mark) = self.comment.take() + && out.since(mark).trim_start().starts_with(THREADED_MIRROR) + { + out.cut(mark, &mut self.discarded, 0); + } + } + b"threadedComment" => out.separator(Separator::Newline), + _ => {} + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + self.text.text(text, out); + } +} diff --git a/crates/text-extract/src/ooxml/layout.rs b/crates/text-extract/src/ooxml/layout.rs new file mode 100644 index 00000000..be2be791 --- /dev/null +++ b/crates/text-extract/src/ooxml/layout.rs @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::text::RunText; +use crate::{ + output::Output, + xml::{Handler, Tag, Text, local_name}, +}; + +#[derive(Default)] +pub(crate) struct LayoutText { + text: RunText, + shapes: u32, + placeholder: Option, + done: bool, +} + +fn is_shape(local: &[u8]) -> bool { + hashify::set!(local, b"sp", b"graphicFrame", b"pic", b"cxnSp") +} + +impl Handler for LayoutText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let local = tag.local(); + if is_shape(local) { + self.shapes = self.shapes.saturating_add(1); + } else if self.placeholder.is_none() && self.shapes > 0 && hashify::set!(local, b"ph") { + self.placeholder = Some(self.shapes); + } + if self.placeholder.is_none() { + self.text.start(tag, out); + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + if self.placeholder.is_none() { + self.text.end(name, out); + } + let local = local_name(name); + self.done |= hashify::set!(local, b"cSld"); + if is_shape(local) { + if self.placeholder == Some(self.shapes) { + self.placeholder = None; + } + self.shapes = self.shapes.saturating_sub(1); + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + if self.placeholder.is_none() { + self.text.text(text, out); + } + } + + fn aborted(&self) -> bool { + self.done + } +} diff --git a/crates/text-extract/src/ooxml/mod.rs b/crates/text-extract/src/ooxml/mod.rs new file mode 100644 index 00000000..a27bdfbe --- /dev/null +++ b/crates/text-extract/src/ooxml/mod.rs @@ -0,0 +1,265 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod chart; +mod comments; +mod layout; +mod number; +mod pptx; +mod rels; +mod sheet; +mod styles; +mod text; +mod xlsx; + +use crate::{ + Format, + output::{Output, Separator}, + package::{Arena, Package, Span}, + xml::stream::Part, +}; +use chart::{ChartText, ChartValues}; +use comments::CommentsText; +use rels::{Rel, RelKind, RelsHandler}; +use styles::Styles; +use text::{MainPart, RunText}; + +const ROOT_RELS: &[u8] = b"_rels/.rels"; +const MAIN_PART_FALLBACKS: [&[u8]; 3] = [ + b"word/document.xml", + b"xl/workbook.xml", + b"ppt/presentation.xml", +]; + +#[derive(Default)] +pub(crate) struct Scratch { + arena: Arena, + rels: Vec, + child_rels: Vec, + nested_rels: Vec, + ids: Vec, + styles: Styles, +} + +pub(crate) struct Relations<'s> { + arena: &'s mut Arena, + child_rels: &'s mut Vec, + nested_rels: &'s mut Vec, + limit: usize, +} + +pub(crate) fn is_package(package: &Package<'_, '_>) -> bool { + package.archive.contains(ROOT_RELS) + || MAIN_PART_FALLBACKS + .iter() + .any(|name| package.archive.contains(name)) +} + +pub(crate) fn extract( + package: &mut Package<'_, '_>, + scratch: &mut Scratch, + out: &mut Output<'_>, +) -> Option { + let Scratch { + arena, + rels, + child_rels, + nested_rels, + ids, + styles, + } = scratch; + arena.clear(); + rels.clear(); + child_rels.clear(); + nested_rels.clear(); + ids.clear(); + styles.clear(); + let limit = package.budget.parts; + + let root = arena.push(b"")?; + let root_rels = arena.push(ROOT_RELS)?; + load_rels(package, arena, rels, root_rels, root, limit, out); + let main = rels + .iter() + .find(|rel| rel.kind == RelKind::OfficeDocument) + .map(|rel| rel.target) + .filter(|target| package.archive.contains(arena.get(*target))) + .or_else(|| { + let fallback = MAIN_PART_FALLBACKS + .iter() + .find(|name| package.archive.contains(name))?; + arena.push(fallback) + })?; + + let member = package.archive.find(arena.get(main))?; + let mut main_part = MainPart::new(arena, ids, limit); + package.scan_member_as(&member, Part::Content, &mut main_part, out); + let format = main_part.format()?; + let dates = main_part.dates(); + package.budget.truncated |= main_part.capped(); + out.separator(Separator::Newline); + + rels.clear(); + let main_rels = arena.push_rels_path(main)?; + load_rels(package, arena, rels, main_rels, main, limit, out); + + let mut relations = Relations { + arena, + child_rels, + nested_rels, + limit, + }; + match format { + Format::Docx => { + let Relations { + arena, nested_rels, .. + } = relations; + scan_related( + package, + arena, + rels, + nested_rels, + limit, + ChartValues::All, + out, + ); + } + Format::Xlsx => xlsx::extract(package, &mut relations, rels, ids, styles, dates, out), + _ => pptx::extract(package, &mut relations, rels, ids, out), + } + Some(format) +} + +fn load_rels( + package: &mut Package<'_, '_>, + arena: &mut Arena, + rels: &mut Vec, + path: Span, + source: Span, + limit: usize, + out: &mut Output<'_>, +) -> bool { + let Some(member) = package.archive.find(arena.get(path)) else { + return false; + }; + let mut handler = RelsHandler { + arena, + rels, + source, + limit, + }; + package.scan_member_as(&member, Part::Plumbing, &mut handler, out) +} + +impl Relations<'_> { + fn load_children( + &mut self, + package: &mut Package<'_, '_>, + part: Span, + out: &mut Output<'_>, + ) -> bool { + self.child_rels.clear(); + match self.arena.push_rels_path(part) { + Some(path) => load_rels( + package, + self.arena, + self.child_rels, + path, + part, + self.limit, + out, + ), + None => false, + } + } + + fn scan_children( + &mut self, + package: &mut Package<'_, '_>, + values: ChartValues, + out: &mut Output<'_>, + ) { + let Relations { + arena, + child_rels, + nested_rels, + limit, + } = self; + scan_related(package, arena, child_rels, nested_rels, *limit, values, out); + } +} + +fn scan_related( + package: &mut Package<'_, '_>, + arena: &mut Arena, + rels: &[Rel], + nested_rels: &mut Vec, + limit: usize, + values: ChartValues, + out: &mut Output<'_>, +) { + for rel in rels { + if package.stopped(out) { + return; + } + match rel.kind { + RelKind::Comments | RelKind::ThreadedComments => { + package.scan(arena.get(rel.target), &mut CommentsText::default(), out); + } + RelKind::Footnotes + | RelKind::Endnotes + | RelKind::DiagramData + | RelKind::Header + | RelKind::Footer + | RelKind::NotesSlide => { + package.scan(arena.get(rel.target), &mut RunText::default(), out); + } + RelKind::Chart => { + package.scan(arena.get(rel.target), &mut ChartText::new(values), out); + } + RelKind::Drawing => { + if package.scan(arena.get(rel.target), &mut RunText::default(), out) { + out.separator(Separator::Newline); + scan_drawing(package, arena, rel.target, nested_rels, limit, values, out); + } + } + _ => continue, + } + out.separator(Separator::Newline); + } +} + +fn scan_drawing( + package: &mut Package<'_, '_>, + arena: &mut Arena, + drawing: Span, + nested_rels: &mut Vec, + limit: usize, + values: ChartValues, + out: &mut Output<'_>, +) { + let mark = arena.mark(); + nested_rels.clear(); + if let Some(path) = arena.push_rels_path(drawing) { + load_rels(package, arena, nested_rels, path, drawing, limit, out); + } + for rel in nested_rels.iter() { + if package.stopped(out) { + break; + } + match rel.kind { + RelKind::Chart => { + package.scan(arena.get(rel.target), &mut ChartText::new(values), out); + } + RelKind::DiagramData => { + package.scan(arena.get(rel.target), &mut RunText::default(), out); + } + _ => continue, + } + out.separator(Separator::Newline); + } + arena.rewind(mark); +} diff --git a/crates/text-extract/src/ooxml/number/decimal.rs b/crates/text-extract/src/ooxml/number/decimal.rs new file mode 100644 index 00000000..92067201 --- /dev/null +++ b/crates/text-extract/src/ooxml/number/decimal.rs @@ -0,0 +1,285 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{NumberFormat, SIGNIFICANT_DIGITS}; +use crate::{output::Output, xml::Text}; +use memchr::memchr; + +const MAX_INTEGER_DIGITS: usize = 15; +const MAX_DIGITS: usize = 40; +const PERCENT_SHIFT: usize = 2; +const ZEROS: &str = "000000000000000000000000000000"; + +#[derive(Debug, Clone, Copy)] +enum Precision { + Shortest, + Fixed(usize), +} + +impl Precision { + fn limit(self, point: usize) -> usize { + match self { + Precision::Shortest => usize::MAX, + Precision::Fixed(decimals) => point + decimals, + } + } +} + +pub(crate) fn push_exact(text: Text<'_>, format: NumberFormat, out: &mut Output<'_>) -> bool { + let Some(decimal) = Decimal::parse(text.as_bytes()) else { + return false; + }; + match format { + NumberFormat::General | NumberFormat::Text => decimal.push(text, Precision::Shortest, out), + NumberFormat::Fixed(decimals) => { + decimal.push(text, Precision::Fixed(usize::from(decimals)), out) + } + NumberFormat::Percent(decimals) => decimal.push_percent(usize::from(decimals), out), + _ => false, + } +} + +#[inline(always)] +pub(crate) fn push_integer(text: Text<'_>, format: NumberFormat, out: &mut Output<'_>) -> bool { + let integral = matches!( + format, + NumberFormat::General | NumberFormat::Text | NumberFormat::Fixed(0) + ); + if integral && is_short_integer(text.as_bytes()) { + text.push(out); + true + } else { + false + } +} + +#[inline(always)] +fn is_short_integer(bytes: &[u8]) -> bool { + match bytes { + [b'0'..=b'9'] => true, + [b'1'..=b'9', rest @ ..] => { + rest.len() < MAX_INTEGER_DIGITS && rest.iter().all(u8::is_ascii_digit) + } + _ => false, + } +} + +struct Decimal<'a> { + negative: bool, + integer: &'a [u8], + fraction: &'a [u8], +} + +impl<'a> Decimal<'a> { + fn parse(raw: &'a [u8]) -> Option> { + let (negative, unsigned) = match raw.strip_prefix(b"-") { + Some(unsigned) => (true, unsigned), + None => (false, raw), + }; + let (integer, fraction) = match memchr(b'.', unsigned) { + Some(dot) => ( + unsigned.get(..dot)?, + unsigned + .get(dot + 1..) + .filter(|fraction| !fraction.is_empty())?, + ), + None => (unsigned, b"".as_slice()), + }; + let canonical = integer == b"0" || integer.first().is_some_and(|&digit| digit != b'0'); + (canonical + && integer.len() <= MAX_INTEGER_DIGITS + && integer.len() + fraction.len() <= MAX_DIGITS + && integer.iter().all(u8::is_ascii_digit) + && fraction.iter().all(u8::is_ascii_digit)) + .then_some(Decimal { + negative, + integer, + fraction, + }) + } + + fn first_significant(&self) -> usize { + if self.integer == b"0" { + 1 + self + .fraction + .iter() + .take_while(|&&digit| digit == b'0') + .count() + } else { + 0 + } + } + + fn is_zero(&self) -> bool { + self.integer == b"0" && self.fraction.iter().all(|&digit| digit == b'0') + } + + fn digit(&self, index: usize) -> Option { + match index.checked_sub(self.integer.len()) { + Some(offset) => self.fraction.get(offset).copied(), + None => self.integer.get(index).copied(), + } + } + + fn push(&self, text: Text<'_>, precision: Precision, out: &mut Output<'_>) -> bool { + let first = self.first_significant(); + let total = self.integer.len() + self.fraction.len(); + let exact = total.saturating_sub(first) <= SIGNIFICANT_DIGITS + && !(self.negative && self.is_zero()) + && match precision { + Precision::Shortest => self.fraction.last() != Some(&b'0'), + Precision::Fixed(decimals) => self.fraction.len() <= decimals, + }; + if !exact { + return self.push_rounded(first, precision, "", out); + } + text.push(out); + if let Precision::Fixed(decimals) = precision + && decimals > self.fraction.len() + { + if self.fraction.is_empty() { + out.push_str("."); + } + out.push_str(ZEROS.get(..decimals - self.fraction.len()).unwrap_or(ZEROS)); + } + true + } + + fn push_percent(&self, decimals: usize, out: &mut Output<'_>) -> bool { + let mut digits = [b'0'; MAX_DIGITS + PERCENT_SHIFT]; + let point = self.integer.len() + PERCENT_SHIFT; + let (moved, rest) = self + .fraction + .split_at(self.fraction.len().min(PERCENT_SHIFT)); + let copied = [ + (0, self.integer), + (self.integer.len(), moved), + (point, rest), + ] + .into_iter() + .all(|(at, bytes)| match digits.get_mut(at..at + bytes.len()) { + Some(slot) => { + slot.copy_from_slice(bytes); + true + } + None => false, + }); + if !copied { + return false; + } + let leading = digits.get(..point - 1).map_or(0, |head| { + head.iter().take_while(|&&digit| digit == b'0').count() + }); + let (Some(integer), Some(fraction)) = ( + digits.get(leading..point), + digits.get(point..point + rest.len()), + ) else { + return false; + }; + if integer.len() > MAX_INTEGER_DIGITS { + return false; + } + let shifted = Decimal { + negative: self.negative, + integer, + fraction, + }; + shifted.push_rounded( + shifted.first_significant(), + Precision::Fixed(decimals), + "%", + out, + ) + } + + fn push_rounded( + &self, + first: usize, + precision: Precision, + suffix: &str, + out: &mut Output<'_>, + ) -> bool { + let point = self.integer.len(); + let total = point + self.fraction.len(); + let keep = (first + SIGNIFICANT_DIGITS) + .min(precision.limit(point)) + .min(total) + .max(point); + let mut digits = [b'0'; MAX_DIGITS + 1]; + let (Some(kept), Some(fraction)) = + (digits.get_mut(1..=keep), self.fraction.get(..keep - point)) + else { + return false; + }; + let (integer_slot, fraction_slot) = kept.split_at_mut(point); + integer_slot.copy_from_slice(self.integer); + fraction_slot.copy_from_slice(fraction); + let mut carry = self.digit(keep).is_some_and(|digit| digit >= b'5'); + for slot in kept.iter_mut().rev() { + if !carry { + break; + } + if *slot == b'9' { + *slot = b'0'; + } else { + *slot += 1; + carry = false; + } + } + let start = if carry { + if let Some(slot) = digits.first_mut() { + *slot = b'1'; + } + 0 + } else { + 1 + }; + let (Some(integer), Some(mut fraction)) = + (digits.get(start..=point), digits.get(point + 1..=keep)) + else { + return false; + }; + if matches!(precision, Precision::Shortest) { + while let [rest @ .., b'0'] = fraction { + fraction = rest; + } + } + let padding = match precision { + Precision::Fixed(decimals) => decimals.saturating_sub(fraction.len()), + Precision::Shortest => 0, + }; + let negative = self.negative && !integer.iter().chain(fraction).all(|&digit| digit == b'0'); + let mut rendered = [0u8; MAX_DIGITS + 4]; + let mut len = 0; + let mut append = |bytes: &[u8]| { + if let Some(slot) = rendered.get_mut(len..len + bytes.len()) { + slot.copy_from_slice(bytes); + len += bytes.len(); + } + }; + if negative { + append(b"-"); + } + append(integer); + if !fraction.is_empty() || padding > 0 { + append(b"."); + append(fraction); + } + match std::str::from_utf8(rendered.get(..len).unwrap_or_default()) { + Ok(text) => { + out.push_str(text); + if padding > 0 { + out.push_str(ZEROS.get(..padding).unwrap_or(ZEROS)); + } + if !suffix.is_empty() { + out.push_str(suffix); + } + true + } + Err(_) => false, + } + } +} diff --git a/crates/text-extract/src/ooxml/number/mod.rs b/crates/text-extract/src/ooxml/number/mod.rs new file mode 100644 index 00000000..8d3586d0 --- /dev/null +++ b/crates/text-extract/src/ooxml/number/mod.rs @@ -0,0 +1,241 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod decimal; +mod render; + +use crate::{output::Output, xml::Tag}; +use memchr::memmem; + +pub(crate) use decimal::{push_exact, push_integer}; +pub(crate) use render::{push_boolean, push_number}; + +const SIGNIFICANT_DIGITS: usize = 15; +const MAX_DECIMALS: u8 = 30; +const MAX_VALUE: usize = 64; +const ENCODED_QUOTE: &[u8] = b"quot;"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DateSystem { + Epoch1900, + Epoch1904, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) enum NumberFormat { + #[default] + General, + Text, + Fixed(u8), + Percent(u8), + Date, + DateTime, + Time, + ElapsedTime, +} + +impl DateSystem { + pub(crate) fn from_properties(tag: &Tag<'_>) -> Self { + Self::from_attribute(tag, b"date1904") + } + + pub(crate) fn from_value(tag: &Tag<'_>) -> Self { + Self::from_attribute(tag, b"val") + } + + fn from_attribute(tag: &Tag<'_>, attribute: &[u8]) -> Self { + let date1904 = tag + .attributes() + .any(|(name, value)| name == attribute && hashify::set!(value, b"1", b"true", b"on")); + if date1904 { + DateSystem::Epoch1904 + } else { + DateSystem::Epoch1900 + } + } +} + +impl NumberFormat { + pub(crate) fn builtin(id: u32) -> NumberFormat { + match id { + 1 | 3 | 5 | 6 | 37 | 38 | 41 | 42 => NumberFormat::Fixed(0), + 2 | 4 | 7 | 8 | 39 | 40 | 43 | 44 => NumberFormat::Fixed(2), + 9 => NumberFormat::Percent(0), + 10 => NumberFormat::Percent(2), + 14..=17 | 27..=31 | 36 | 50..=58 => NumberFormat::Date, + 18..=21 | 32..=35 | 45 | 47 => NumberFormat::Time, + 22 => NumberFormat::DateTime, + 46 => NumberFormat::ElapsedTime, + 49 => NumberFormat::Text, + _ => NumberFormat::General, + } + } + + pub(crate) fn classify(code: &[u8]) -> NumberFormat { + let mut tokens = FormatTokens::default(); + let mut rest = code; + while let Some((&byte, tail)) = rest.split_first() { + rest = tail; + match byte { + b';' => break, + b'"' => { + rest = rest + .iter() + .position(|&byte| byte == b'"') + .and_then(|end| rest.get(end + 1..)) + .unwrap_or_default(); + } + b'&' if rest.starts_with(ENCODED_QUOTE) => { + let literal = rest.get(ENCODED_QUOTE.len()..).unwrap_or_default(); + rest = memmem::find(literal, b""") + .and_then(|end| literal.get(end + ENCODED_QUOTE.len() + 1..)) + .unwrap_or_default(); + } + b'\\' | b'_' | b'*' => rest = rest.get(1..).unwrap_or_default(), + b'[' => { + let end = rest + .iter() + .position(|&byte| byte == b']') + .unwrap_or(rest.len()); + let (bracket, tail) = rest.split_at(end); + if let Some(first) = bracket.first() + && matches!(first.to_ascii_lowercase(), b'h' | b'm' | b's') + && bracket.iter().all(|byte| byte.eq_ignore_ascii_case(first)) + { + tokens.elapsed = true; + tokens.time = true; + } + rest = tail.get(1..).unwrap_or_default(); + } + b'g' | b'G' if starts_with_ignore_case(rest, b"eneral") => { + rest = rest.get(6..).unwrap_or_default(); + } + b'a' | b'A' if starts_with_ignore_case(rest, b"m/p") => { + tokens.time = true; + rest = rest.get(3..).unwrap_or_default(); + } + b'y' | b'Y' | b'd' | b'D' => tokens.date = true, + b'h' | b'H' | b's' | b'S' => tokens.time = true, + b'm' | b'M' => tokens.month_or_minute = true, + b'%' => tokens.percent = true, + b'@' => tokens.text = true, + b'/' => tokens.fraction = true, + b'e' | b'E' if matches!(rest.first(), Some(b'+' | b'-')) => { + tokens.scientific = true + } + b'0' | b'#' | b'?' => { + tokens.digits = true; + if tokens.in_decimals && !tokens.percent { + tokens.decimals = tokens.decimals.saturating_add(1); + } + } + b'.' => tokens.in_decimals = true, + _ => {} + } + } + tokens.format() + } +} + +#[derive(Default)] +struct FormatTokens { + date: bool, + time: bool, + month_or_minute: bool, + elapsed: bool, + percent: bool, + text: bool, + fraction: bool, + scientific: bool, + digits: bool, + in_decimals: bool, + decimals: u8, +} + +impl FormatTokens { + fn format(&self) -> NumberFormat { + let date = self.date || (self.month_or_minute && !self.time); + match (date, self.time) { + (true, true) => NumberFormat::DateTime, + (true, false) => NumberFormat::Date, + (false, true) if self.elapsed => NumberFormat::ElapsedTime, + (false, true) => NumberFormat::Time, + (false, false) if self.percent => { + NumberFormat::Percent(self.decimals.min(MAX_DECIMALS)) + } + (false, false) if self.text && !self.digits => NumberFormat::Text, + (false, false) if self.digits && !self.fraction && !self.scientific => { + NumberFormat::Fixed(self.decimals.min(MAX_DECIMALS)) + } + (false, false) => NumberFormat::General, + } + } +} + +fn starts_with_ignore_case(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .get(..needle.len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(needle)) +} + +pub(crate) struct ValueBuffer { + bytes: [u8; MAX_VALUE], + len: usize, +} + +impl Default for ValueBuffer { + fn default() -> Self { + ValueBuffer { + bytes: [0; MAX_VALUE], + len: 0, + } + } +} + +impl ValueBuffer { + pub(crate) fn clear(&mut self) { + self.len = 0; + } + + pub(crate) fn is_empty(&self) -> bool { + self.len == 0 + } + + pub(crate) fn value(&self) -> &[u8] { + self.bytes.get(..self.len).unwrap_or_default() + } + + #[must_use] + pub(crate) fn fill(&mut self, text: &[u8]) -> bool { + let end = self.len + text.len(); + match self.bytes.get_mut(self.len..end) { + Some(slot) => { + slot.copy_from_slice(text); + self.len = end; + true + } + None => false, + } + } + + #[must_use] + pub(crate) fn push(&mut self, text: &[u8], out: &mut Output<'_>) -> bool { + let end = self.len + text.len(); + match self.bytes.get_mut(self.len..end) { + Some(slot) => { + slot.copy_from_slice(text); + self.len = end; + true + } + None => { + out.push_utf8(self.value()); + out.push_utf8(text); + self.len = 0; + false + } + } + } +} diff --git a/crates/text-extract/src/ooxml/number/render.rs b/crates/text-extract/src/ooxml/number/render.rs new file mode 100644 index 00000000..0dee9e2b --- /dev/null +++ b/crates/text-extract/src/ooxml/number/render.rs @@ -0,0 +1,294 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{DateSystem, NumberFormat, SIGNIFICANT_DIGITS, push_exact, push_integer}; +use crate::{output::Output, xml::Text}; +use std::fmt::{self, Write}; + +const SECONDS_PER_DAY: u32 = 86_400; +const MAX_SERIAL: f64 = 2_958_466.0; +const LEAP_DAY_SERIAL: i64 = 60; +const DAYS_1899_12_30_TO_UNIX: i64 = -25_569; +const DAYS_1899_12_31_TO_UNIX: i64 = -25_568; +const DAYS_1904_01_01_TO_UNIX: i64 = -24_107; +const MIN_PLAIN: f64 = 1e-9; +const MAX_PLAIN: f64 = 1e15; +const RENDER_CAPACITY: usize = 64; +const MAX_SERIAL_DAYS: i64 = 2_958_466; +const MAX_UNIX_DAYS: i64 = 2_932_896; +const MAX_SERIAL_DIGITS: usize = 7; + +struct Rendered { + bytes: [u8; RENDER_CAPACITY], + len: usize, +} + +impl Rendered { + fn new() -> Self { + Rendered { + bytes: [0; RENDER_CAPACITY], + len: 0, + } + } + + fn as_str(&self) -> &str { + std::str::from_utf8(self.bytes.get(..self.len).unwrap_or_default()).unwrap_or_default() + } + + fn clear(&mut self) { + self.len = 0; + } + + fn bytes(&self) -> &[u8] { + self.bytes.get(..self.len).unwrap_or_default() + } + + fn push_digits(&mut self, value: u64, width: usize) -> fmt::Result { + let mut digits = [b'0'; 20]; + let mut remaining = value; + let mut used = 0; + for slot in digits.iter_mut().rev() { + *slot = b'0' + (remaining % 10) as u8; + remaining /= 10; + used += 1; + if remaining == 0 && used >= width { + break; + } + } + let start = digits.len() - used; + let text = + std::str::from_utf8(digits.get(start..).unwrap_or_default()).map_err(|_| fmt::Error)?; + self.write_str(text) + } +} + +impl Write for Rendered { + fn write_str(&mut self, text: &str) -> fmt::Result { + let end = self.len.checked_add(text.len()).ok_or(fmt::Error)?; + self.bytes + .get_mut(self.len..end) + .ok_or(fmt::Error)? + .copy_from_slice(text.as_bytes()); + self.len = end; + Ok(()) + } +} + +pub(crate) fn push_boolean(raw: &[u8], out: &mut Output<'_>) { + match raw.trim_ascii() { + b"1" => out.push_str("TRUE"), + b"0" => out.push_str("FALSE"), + other => out.push_utf8(other), + } +} + +pub(crate) fn push_number( + raw: &[u8], + format: NumberFormat, + dates: DateSystem, + out: &mut Output<'_>, +) { + let trimmed = raw.trim_ascii(); + if push_integer(Text::raw(trimmed), format, out) || push_exact(Text::raw(trimmed), format, out) + { + return; + } + let mut rendered = Rendered::new(); + if matches!(format, NumberFormat::Date | NumberFormat::DateTime) + && let Some(moment) = whole_days(trimmed).and_then(|days| Moment::from_days(days, 0, dates)) + && render_moment(&moment, format, &mut rendered).is_ok() + { + out.push_str(rendered.as_str()); + return; + } + let Some(value) = std::str::from_utf8(trimmed) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|value| value.is_finite()) + else { + out.push_utf8(trimmed); + return; + }; + rendered.clear(); + if render(value, format, dates, &mut rendered, out).is_ok() { + out.push_str(rendered.as_str()); + } else { + out.push_utf8(trimmed); + } +} + +fn render( + value: f64, + format: NumberFormat, + dates: DateSystem, + rendered: &mut Rendered, + out: &mut Output<'_>, +) -> fmt::Result { + match format { + NumberFormat::General | NumberFormat::Text => { + if write!(rendered, "{value}").is_ok() + && push_exact(Text::raw(rendered.bytes()), format, out) + { + rendered.clear(); + return Ok(()); + } + rendered.clear(); + write_general(value, rendered) + } + NumberFormat::Fixed(_) | NumberFormat::Percent(_) => { + write!(rendered, "{value}")?; + if push_exact(Text::raw(rendered.bytes()), format, out) { + rendered.clear(); + Ok(()) + } else { + Err(fmt::Error) + } + } + NumberFormat::Date | NumberFormat::DateTime | NumberFormat::Time => { + let moment = Moment::from_serial(value, dates).ok_or(fmt::Error)?; + render_moment(&moment, format, rendered) + } + NumberFormat::ElapsedTime => { + let seconds = (value.abs() * f64::from(SECONDS_PER_DAY)).round(); + if seconds >= MAX_SERIAL * f64::from(SECONDS_PER_DAY) { + return Err(fmt::Error); + } + let seconds = seconds as u64; + if value < 0.0 && seconds > 0 { + rendered.write_char('-')?; + } + rendered.push_digits(seconds / 3600, 1)?; + rendered.write_char(':')?; + rendered.push_digits(seconds / 60 % 60, 2)?; + rendered.write_char(':')?; + rendered.push_digits(seconds % 60, 2) + } + } +} + +fn render_moment(moment: &Moment, format: NumberFormat, rendered: &mut Rendered) -> fmt::Result { + match format { + NumberFormat::Date => moment.write_date(rendered), + NumberFormat::DateTime => { + moment.write_date(rendered)?; + rendered.write_char(' ')?; + moment.write_time(rendered) + } + _ => moment.write_time(rendered), + } +} + +fn whole_days(raw: &[u8]) -> Option { + if raw.is_empty() || raw.len() > MAX_SERIAL_DIGITS { + return None; + } + raw.iter().try_fold(0i64, |days, &digit| { + Some(days * 10 + i64::from(digit.checked_sub(b'0').filter(|value| *value < 10)?)) + }) +} + +fn write_general(value: f64, rendered: &mut Rendered) -> fmt::Result { + write!(rendered, "{:.*e}", SIGNIFICANT_DIGITS - 1, value)?; + let rounded = rendered.as_str().parse::().map_err(|_| fmt::Error)?; + rendered.clear(); + if rounded == 0.0 { + rendered.write_char('0') + } else if (MIN_PLAIN..MAX_PLAIN).contains(&rounded.abs()) { + write!(rendered, "{rounded}") + } else { + write!(rendered, "{rounded:e}") + } +} + +struct Moment { + year: i64, + month: u32, + day: u32, + seconds: u32, +} + +impl Moment { + fn from_serial(serial: f64, dates: DateSystem) -> Option { + if !(0.0..MAX_SERIAL).contains(&serial) { + return None; + } + let whole = serial.floor(); + let mut days = whole as i64; + let mut seconds = ((serial - whole) * f64::from(SECONDS_PER_DAY)).round() as u32; + if seconds >= SECONDS_PER_DAY { + days += 1; + seconds -= SECONDS_PER_DAY; + } + Moment::from_days(days, seconds, dates) + } + + fn from_days(days: i64, seconds: u32, dates: DateSystem) -> Option { + if !(0..MAX_SERIAL_DAYS).contains(&days) { + return None; + } + let unix_days = match dates { + DateSystem::Epoch1904 => days + DAYS_1904_01_01_TO_UNIX, + DateSystem::Epoch1900 if days == LEAP_DAY_SERIAL => { + return Some(Moment { + year: 1900, + month: 2, + day: 29, + seconds, + }); + } + DateSystem::Epoch1900 if days < LEAP_DAY_SERIAL => days + DAYS_1899_12_31_TO_UNIX, + DateSystem::Epoch1900 => days + DAYS_1899_12_30_TO_UNIX, + }; + if unix_days > MAX_UNIX_DAYS { + return None; + } + let (year, month, day) = civil_from_days(unix_days); + Some(Moment { + year, + month, + day, + seconds, + }) + } + + fn write_date(&self, rendered: &mut Rendered) -> fmt::Result { + rendered.push_digits(u64::try_from(self.year).map_err(|_| fmt::Error)?, 4)?; + rendered.write_char('-')?; + rendered.push_digits(u64::from(self.month), 2)?; + rendered.write_char('-')?; + rendered.push_digits(u64::from(self.day), 2) + } + + fn write_time(&self, rendered: &mut Rendered) -> fmt::Result { + rendered.push_digits(u64::from(self.seconds / 3600), 2)?; + rendered.write_char(':')?; + rendered.push_digits(u64::from(self.seconds / 60 % 60), 2)?; + rendered.write_char(':')?; + rendered.push_digits(u64::from(self.seconds % 60), 2) + } +} + +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + ( + year, + u32::try_from(month).unwrap_or(1), + u32::try_from(day).unwrap_or(1), + ) +} diff --git a/crates/text-extract/src/ooxml/pptx.rs b/crates/text-extract/src/ooxml/pptx.rs new file mode 100644 index 00000000..aa26ec6e --- /dev/null +++ b/crates/text-extract/src/ooxml/pptx.rs @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Relations, + chart::ChartValues, + layout::LayoutText, + load_rels, + rels::{Rel, RelKind, find_kind, find_sorted, sort_by_id}, + text::RunText, +}; +use crate::{ + output::{Output, Separator}, + package::{Package, Span}, +}; + +pub(super) fn extract( + package: &mut Package<'_, '_>, + relations: &mut Relations<'_>, + rels: &mut Vec, + slides: &[Span], + out: &mut Output<'_>, +) { + sort_by_id(relations.arena, rels); + for &slide in slides { + if package.stopped(out) { + return; + } + let Some(rel) = find_sorted(relations.arena, rels, slide, |kind| kind == RelKind::Slide) + else { + continue; + }; + if !package.scan( + relations.arena.get(rel.target), + &mut RunText::default(), + out, + ) { + continue; + } + out.separator(Separator::Newline); + let mark = relations.arena.mark(); + if relations.load_children(package, rel.target, out) { + relations.scan_children(package, ChartValues::All, out); + scan_layout(package, relations, out); + } + relations.arena.rewind(mark); + } +} + +fn scan_layout(package: &mut Package<'_, '_>, relations: &mut Relations<'_>, out: &mut Output<'_>) { + let Some(layout) = find_kind(relations.child_rels, RelKind::SlideLayout) else { + return; + }; + if package.stopped(out) + || !package.scan(relations.arena.get(layout), &mut LayoutText::default(), out) + { + return; + } + out.separator(Separator::Newline); + relations.nested_rels.clear(); + let Some(path) = relations.arena.push_rels_path(layout) else { + return; + }; + load_rels( + package, + relations.arena, + relations.nested_rels, + path, + layout, + relations.limit, + out, + ); + if let Some(master) = find_kind(relations.nested_rels, RelKind::SlideMaster) + && !package.stopped(out) + && package.scan(relations.arena.get(master), &mut LayoutText::default(), out) + { + out.separator(Separator::Newline); + } +} diff --git a/crates/text-extract/src/ooxml/rels.rs b/crates/text-extract/src/ooxml/rels.rs new file mode 100644 index 00000000..fa52d516 --- /dev/null +++ b/crates/text-extract/src/ooxml/rels.rs @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + output::Output, + package::{Arena, Span}, + xml::{Handler, Tag, Text}, +}; +use memchr::memrchr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RelKind { + OfficeDocument, + Header, + Footer, + Footnotes, + Endnotes, + Comments, + ThreadedComments, + DiagramData, + SharedStrings, + Styles, + Worksheet, + Chartsheet, + Drawing, + Chart, + Slide, + SlideLayout, + SlideMaster, + NotesSlide, +} + +impl RelKind { + fn parse(relationship_type: &[u8]) -> Option { + let suffix = memrchr(b'/', relationship_type) + .and_then(|slash| relationship_type.get(slash + 1..)) + .unwrap_or(relationship_type); + hashify::map!(suffix, RelKind, + b"officeDocument" => RelKind::OfficeDocument, + b"header" => RelKind::Header, + b"footer" => RelKind::Footer, + b"footnotes" => RelKind::Footnotes, + b"endnotes" => RelKind::Endnotes, + b"comments" => RelKind::Comments, + b"threadedComment" => RelKind::ThreadedComments, + b"diagramData" => RelKind::DiagramData, + b"sharedStrings" => RelKind::SharedStrings, + b"styles" => RelKind::Styles, + b"worksheet" => RelKind::Worksheet, + b"chartsheet" => RelKind::Chartsheet, + b"drawing" => RelKind::Drawing, + b"chart" => RelKind::Chart, + b"chartEx" => RelKind::Chart, + b"slide" => RelKind::Slide, + b"slideLayout" => RelKind::SlideLayout, + b"slideMaster" => RelKind::SlideMaster, + b"notesSlide" => RelKind::NotesSlide, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Rel { + pub(crate) id: Span, + pub(crate) kind: RelKind, + pub(crate) target: Span, +} + +pub(crate) struct RelsHandler<'x> { + pub(crate) arena: &'x mut Arena, + pub(crate) rels: &'x mut Vec, + pub(crate) source: Span, + pub(crate) limit: usize, +} + +impl Handler for RelsHandler<'_> { + fn start(&mut self, tag: &Tag<'_>, _out: &mut Output<'_>) { + if !hashify::set!(tag.local(), b"Relationship") || self.rels.len() >= self.limit { + return; + } + let (mut id, mut kind, mut target, mut external) = (None, None, None, false); + for (name, value) in tag.attributes() { + hashify::fnc_map!(name, + b"Id" => { id = Some(value); }, + b"Type" => { kind = RelKind::parse(value); }, + b"Target" => { target = Some(value); }, + b"TargetMode" => { external = hashify::set_ignore_case!(value, b"External"); }, + _ => {} + ); + } + let (Some(id), Some(kind), Some(target), false) = (id, kind, target, external) else { + return; + }; + let mark = self.arena.mark(); + match ( + self.arena.push_decoded(id), + self.arena.push_resolved(self.source, target), + ) { + (Some(id), Some(target)) => self.rels.push(Rel { id, kind, target }), + _ => self.arena.rewind(mark), + } + } + + fn end(&mut self, _name: &[u8], _out: &mut Output<'_>) {} + + fn text(&mut self, _text: Text<'_>, _out: &mut Output<'_>) {} +} + +pub(crate) fn sort_by_id(arena: &Arena, rels: &mut Vec) { + rels.sort_unstable_by(|left, right| { + arena + .get(left.id) + .cmp(arena.get(right.id)) + .then(left.id.start().cmp(&right.id.start())) + }); + rels.dedup_by(|later, earlier| arena.get(later.id) == arena.get(earlier.id)); +} + +pub(crate) fn find_sorted( + arena: &Arena, + sorted: &[Rel], + id: Span, + accept: impl Fn(RelKind) -> bool, +) -> Option { + let id = arena.get(id); + let index = sorted + .binary_search_by(|rel| arena.get(rel.id).cmp(id)) + .ok()?; + sorted.get(index).filter(|rel| accept(rel.kind)).copied() +} + +pub(crate) fn find_kind(rels: &[Rel], kind: RelKind) -> Option { + rels.iter() + .find(|rel| rel.kind == kind) + .map(|rel| rel.target) +} diff --git a/crates/text-extract/src/ooxml/sheet.rs b/crates/text-extract/src/ooxml/sheet.rs new file mode 100644 index 00000000..3eea2720 --- /dev/null +++ b/crates/text-extract/src/ooxml/sheet.rs @@ -0,0 +1,312 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + number::{ + DateSystem, NumberFormat, ValueBuffer, push_boolean, push_exact, push_integer, push_number, + }, + styles::parse_index, +}; +use crate::{ + output::{Output, Separator}, + xml::{Handler, Skip, Tag, Text, local_name}, +}; +use memchr::memchr; + +const COLOR_CODE_LEN: u8 = 6; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SheetElement { + Cell, + Value, + InlineText, + Phonetic, + Extension, + Row, + HeaderFooter, +} + +impl SheetElement { + #[inline] + fn parse(local: &[u8]) -> Option { + hashify::map!(local, SheetElement, + b"c" => SheetElement::Cell, + b"v" => SheetElement::Value, + b"t" => SheetElement::InlineText, + b"rPh" => SheetElement::Phonetic, + b"extLst" => SheetElement::Extension, + b"row" => SheetElement::Row, + b"headerFooter" => SheetElement::HeaderFooter, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum Cell { + #[default] + None, + Raw, + Number(NumberFormat), + Boolean, + Inline, + Ignored, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum Collect { + #[default] + Nothing, + Raw, + Buffered, + HeaderFooter, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum Code { + #[default] + Text, + Ampersand, + FontName, + FontSize, + Color(u8), +} + +pub(crate) struct SheetText<'x> { + formats: &'x [NumberFormat], + dates: DateSystem, + skip: Skip, + cell: Cell, + collect: Collect, + value: ValueBuffer, + code: Code, + header_footer: bool, +} + +impl<'x> SheetText<'x> { + pub(crate) fn new(formats: &'x [NumberFormat], dates: DateSystem) -> Self { + let styled = formats + .iter() + .any(|&format| format != NumberFormat::General); + SheetText { + formats: if styled { formats } else { &[] }, + dates, + skip: Skip::default(), + cell: Cell::None, + collect: Collect::Nothing, + value: ValueBuffer::default(), + code: Code::Text, + header_footer: false, + } + } + + fn cell_kind(&self, tag: &Tag<'_>) -> Cell { + if tag.attrs.is_none() { + return Cell::Ignored; + } + let mut kind = Cell::Number(NumberFormat::General); + let mut style = None; + for (name, value) in tag.attributes() { + match name { + [b't'] => { + kind = hashify::map!(value, Cell, + b"n" => Cell::Number(NumberFormat::General), + b"str" => Cell::Raw, + b"d" => Cell::Raw, + b"b" => Cell::Boolean, + b"inlineStr" => Cell::Inline, + ) + .copied() + .unwrap_or(Cell::Ignored); + } + [b's'] => style = Some(value), + _ => {} + } + } + match (kind, style) { + (Cell::Number(_), Some(style)) if !self.formats.is_empty() => { + Cell::Number(self.format_of(style)) + } + (other, _) => other, + } + } + + fn format_of(&self, style: &[u8]) -> NumberFormat { + let index = match style { + [digit @ b'0'..=b'9'] => Some(u32::from(digit - b'0')), + _ => parse_index(style), + }; + index + .and_then(|index| self.formats.get(usize::try_from(index).ok()?)) + .copied() + .unwrap_or_default() + } + + fn flush_value(&mut self, out: &mut Output<'_>) { + let raw = self.value.value(); + match self.cell { + Cell::Number(format) => push_number(raw, format, self.dates, out), + Cell::Boolean => push_boolean(raw, out), + _ => out.push_utf8(raw), + } + self.value.clear(); + } + + fn header_footer(&mut self, mut text: &[u8], out: &mut Output<'_>) { + while !text.is_empty() { + match self.code { + Code::Text => { + let end = memchr(b'&', text).unwrap_or(text.len()); + let (plain, rest) = text.split_at(end); + out.push_utf8(plain); + if !rest.is_empty() { + self.code = Code::Ampersand; + } + text = rest.get(1..).unwrap_or_default(); + } + Code::Ampersand => { + let Some((&byte, rest)) = text.split_first() else { + return; + }; + self.code = match byte { + b'&' => { + out.push_str("&"); + Code::Text + } + b'"' => Code::FontName, + b'0'..=b'9' => Code::FontSize, + b'K' => Code::Color(COLOR_CODE_LEN), + b'L' | b'C' | b'R' | b'P' | b'N' | b'D' | b'T' | b'A' | b'F' | b'Z' + | b'G' => { + out.separator(Separator::Space); + Code::Text + } + _ => Code::Text, + }; + text = rest; + } + Code::FontName => match memchr(b'"', text) { + Some(end) => { + self.code = Code::Text; + text = text.get(end + 1..).unwrap_or_default(); + } + None => return, + }, + Code::FontSize => { + let digits = text + .iter() + .position(|byte| !byte.is_ascii_digit()) + .unwrap_or(text.len()); + if digits < text.len() { + self.code = Code::Text; + } + text = text.get(digits..).unwrap_or_default(); + } + Code::Color(remaining) => { + let skipped = usize::from(remaining).min(text.len()); + let left = remaining.saturating_sub(u8::try_from(skipped).unwrap_or(u8::MAX)); + self.code = if left == 0 { + Code::Text + } else { + Code::Color(left) + }; + text = text.get(skipped..).unwrap_or_default(); + } + } + } + } +} + +impl Handler for SheetText<'_> { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let element = SheetElement::parse(tag.local()); + if self.skip.on_start(element) { + return; + } + if self.header_footer { + out.separator(Separator::Newline); + self.code = Code::Text; + self.collect = Collect::HeaderFooter; + return; + } + match element { + Some(SheetElement::Cell) => self.cell = self.cell_kind(tag), + Some(SheetElement::Value) => { + self.collect = match self.cell { + Cell::Raw => Collect::Raw, + Cell::Number(_) | Cell::Boolean => { + self.value.clear(); + Collect::Buffered + } + _ => Collect::Nothing, + } + } + Some(SheetElement::InlineText) => { + self.collect = if self.cell == Cell::Inline { + Collect::Raw + } else { + Collect::Nothing + } + } + Some(SheetElement::HeaderFooter) => self.header_footer = true, + Some(skipped @ (SheetElement::Phonetic | SheetElement::Extension)) => { + self.skip.begin(skipped) + } + Some(SheetElement::Row) | None => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + let element = SheetElement::parse(local_name(name)); + if self.skip.on_end(element) { + return; + } + if self.header_footer { + self.header_footer = element != Some(SheetElement::HeaderFooter); + self.collect = Collect::Nothing; + out.separator(Separator::Newline); + return; + } + match element { + Some(SheetElement::Value) => { + if self.collect == Collect::Buffered { + self.flush_value(out); + } + self.collect = Collect::Nothing; + } + Some(SheetElement::InlineText) => self.collect = Collect::Nothing, + Some(SheetElement::Cell) => { + self.cell = Cell::None; + out.separator(Separator::Space); + } + Some(SheetElement::Row) => { + self.collect = Collect::Nothing; + out.separator(Separator::Newline); + } + _ => {} + } + } + + #[inline] + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + match self.collect { + Collect::Nothing => {} + Collect::Raw => text.push(out), + Collect::Buffered => { + if let Cell::Number(format) = self.cell + && text.is_closed_by_tag() + && self.value.is_empty() + && (push_integer(text, format, out) || push_exact(text, format, out)) + { + self.collect = Collect::Raw; + } else if !self.value.push(text.as_bytes(), out) { + self.collect = Collect::Raw; + } + } + Collect::HeaderFooter => self.header_footer(text.as_bytes(), out), + } + } +} diff --git a/crates/text-extract/src/ooxml/styles.rs b/crates/text-extract/src/ooxml/styles.rs new file mode 100644 index 00000000..52a98217 --- /dev/null +++ b/crates/text-extract/src/ooxml/styles.rs @@ -0,0 +1,139 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::number::NumberFormat; +use crate::{ + output::Output, + xml::{Handler, Tag, Text, attr::decode_into, local_name}, +}; + +const MAX_CUSTOM_FORMATS: usize = 4096; +const MAX_CELL_FORMATS: usize = 65_536; +const MAX_FORMAT_CODE: usize = 1024; + +#[derive(Default)] +pub(crate) struct Styles { + cell_formats: Vec, + custom: Vec<(u32, NumberFormat)>, + decoded: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Section { + Other, + NumberFormats, + CellFormats, +} + +pub(crate) struct StylesHandler<'x> { + styles: &'x mut Styles, + section: Section, + done: bool, +} + +impl Styles { + pub(crate) fn clear(&mut self) { + self.cell_formats.clear(); + self.custom.clear(); + } + + pub(crate) fn cell_formats(&self) -> &[NumberFormat] { + &self.cell_formats + } + + pub(crate) fn handler(&mut self) -> StylesHandler<'_> { + StylesHandler { + styles: self, + section: Section::Other, + done: false, + } + } + + fn define(&mut self, tag: &Tag<'_>) { + if self.custom.len() >= MAX_CUSTOM_FORMATS { + return; + } + let (mut id, mut code) = (None, None); + for (name, value) in tag.attributes() { + hashify::fnc_map!(name, + b"numFmtId" => { id = parse_index(value); }, + b"formatCode" => { code = Some(value); }, + _ => {} + ); + } + let (Some(id), Some(code)) = (id, code) else { + return; + }; + self.decoded.clear(); + decode_into( + code.get(..MAX_FORMAT_CODE).unwrap_or(code), + &mut self.decoded, + ); + if let Err(index) = self.custom.binary_search_by_key(&id, |&(custom, _)| custom) { + self.custom + .insert(index, (id, NumberFormat::classify(&self.decoded))); + } + } + + fn resolve(&mut self, tag: &Tag<'_>) { + if self.cell_formats.len() >= MAX_CELL_FORMATS { + return; + } + let id = tag + .attributes() + .find_map(|(name, value)| hashify::set!(name, b"numFmtId").then(|| parse_index(value))) + .flatten() + .unwrap_or(0); + let format = match self.custom.binary_search_by_key(&id, |&(custom, _)| custom) { + Ok(index) => self + .custom + .get(index) + .map_or(NumberFormat::General, |&(_, format)| format), + Err(_) => NumberFormat::builtin(id), + }; + self.cell_formats.push(format); + } +} + +impl Handler for StylesHandler<'_> { + fn start(&mut self, tag: &Tag<'_>, _out: &mut Output<'_>) { + let local = tag.local(); + match self.section { + Section::NumberFormats if hashify::set!(local, b"numFmt") => self.styles.define(tag), + Section::CellFormats if hashify::set!(local, b"xf") => self.styles.resolve(tag), + _ => hashify::fnc_map!(local, + b"numFmts" => { self.section = Section::NumberFormats; }, + b"cellXfs" => { self.section = Section::CellFormats; }, + _ => {} + ), + } + } + + fn end(&mut self, name: &[u8], _out: &mut Output<'_>) { + if hashify::set!(local_name(name), b"numFmts", b"cellXfs") { + self.done |= self.section == Section::CellFormats; + self.section = Section::Other; + } + } + + fn text(&mut self, _text: Text<'_>, _out: &mut Output<'_>) {} + + fn aborted(&self) -> bool { + self.done + } +} + +pub(crate) fn parse_index(value: &[u8]) -> Option { + let digits = value.trim_ascii(); + if digits.is_empty() { + return None; + } + digits.iter().try_fold(0u32, |index, &digit| { + index.checked_mul(10)?.checked_add(u32::from( + digit.checked_sub(b'0').filter(|value| *value < 10)?, + )) + }) +} diff --git a/crates/text-extract/src/ooxml/text.rs b/crates/text-extract/src/ooxml/text.rs new file mode 100644 index 00000000..30ba2998 --- /dev/null +++ b/crates/text-extract/src/ooxml/text.rs @@ -0,0 +1,252 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::number::DateSystem; +use crate::{ + Format, + output::{Output, Separator}, + package::{Arena, Span}, + xml::{ + Handler, Skip, Tag, Text, + attr::{push_decoded, split_prefix}, + local_name, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Run { + Text, + DeletedText, + FieldInstruction, + DeletedFieldInstruction, + Fallback, + Phonetic, + MoveFrom, + TabStops, + Tab, + Cell, + Break, + Paragraph, + Block, + Boundary, + NoBreakHyphen, +} + +impl Run { + fn parse(local: &[u8]) -> Option { + hashify::map!(local, Run, + b"t" => Run::Text, + b"text" => Run::Text, + b"delText" => Run::DeletedText, + b"instrText" => Run::FieldInstruction, + b"delInstrText" => Run::DeletedFieldInstruction, + b"Fallback" => Run::Fallback, + b"rPh" => Run::Phonetic, + b"moveFrom" => Run::MoveFrom, + b"tabs" => Run::TabStops, + b"tab" => Run::Tab, + b"ptab" => Run::Tab, + b"tc" => Run::Cell, + b"br" => Run::Break, + b"cr" => Run::Break, + b"p" => Run::Paragraph, + b"tr" => Run::Block, + b"si" => Run::Block, + b"txbxContent" => Run::Block, + b"comment" => Run::Block, + b"footnote" => Run::Block, + b"endnote" => Run::Block, + b"noBreakHyphen" => Run::NoBreakHyphen, + b"rt" => Run::Boundary, + b"sym" => Run::Boundary, + b"num" => Run::Boundary, + b"den" => Run::Boundary, + ) + .copied() + } +} + +#[derive(Default)] +pub(crate) struct RunText { + skip: Skip, + in_text: bool, +} + +impl Handler for RunText { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + let element = Run::parse(tag.local()); + if self.skip.on_start(element) { + return; + } + match element { + Some(Run::Text) => self.in_text = true, + Some( + skipped @ (Run::DeletedText + | Run::FieldInstruction + | Run::DeletedFieldInstruction + | Run::Fallback + | Run::Phonetic + | Run::MoveFrom + | Run::TabStops), + ) => self.skip.begin(skipped), + Some(Run::Tab | Run::Cell | Run::Boundary) => out.separator(Separator::Space), + Some(Run::Break | Run::Paragraph) => out.separator(Separator::Newline), + Some(Run::NoBreakHyphen) => out.push_str("-"), + Some(Run::Block) | None => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + let element = Run::parse(local_name(name)); + if self.skip.on_end(element) { + return; + } + match element { + Some(Run::Text) => self.in_text = false, + Some(Run::Paragraph | Run::Block) => out.separator(Separator::Newline), + Some(Run::Cell | Run::Boundary) => out.separator(Separator::Space), + _ => {} + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + if self.in_text { + text.push(out); + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RootElement { + Document, + Workbook, + Presentation, + Sheet, + SlideId, + WorkbookProperties, +} + +impl RootElement { + fn parse(local: &[u8]) -> Option { + hashify::map!(local, RootElement, + b"document" => RootElement::Document, + b"workbook" => RootElement::Workbook, + b"presentation" => RootElement::Presentation, + b"sheet" => RootElement::Sheet, + b"sldId" => RootElement::SlideId, + b"workbookPr" => RootElement::WorkbookProperties, + ) + .copied() + } +} + +enum Root { + Pending, + Document(RunText), + Workbook, + Presentation, + Unknown, +} + +pub(crate) struct MainPart<'x> { + root: Root, + arena: &'x mut Arena, + ids: &'x mut Vec, + limit: usize, + dates: DateSystem, + capped: bool, +} + +impl<'x> MainPart<'x> { + pub(crate) fn new(arena: &'x mut Arena, ids: &'x mut Vec, limit: usize) -> Self { + MainPart { + root: Root::Pending, + arena, + ids, + limit, + dates: DateSystem::Epoch1900, + capped: false, + } + } + + pub(crate) fn capped(&self) -> bool { + self.capped + } + + pub(crate) fn dates(&self) -> DateSystem { + self.dates + } + + pub(crate) fn format(&self) -> Option { + match self.root { + Root::Document(_) => Some(Format::Docx), + Root::Workbook => Some(Format::Xlsx), + Root::Presentation => Some(Format::Pptx), + Root::Pending | Root::Unknown => None, + } + } + + fn collect_reference(&mut self, tag: &Tag<'_>, out: Option<&mut Output<'_>>) { + let mut reference = None; + let mut sheet_name = None; + for (name, value) in tag.attributes() { + match split_prefix(name) { + (true, local) if hashify::set!(local, b"id") => reference = Some(value), + (false, local) if hashify::set!(local, b"name") => sheet_name = Some(value), + _ => {} + } + } + if let (Some(out), Some(sheet_name)) = (out, sheet_name) { + push_decoded(sheet_name, out); + out.separator(Separator::Newline); + } + if self.ids.len() >= self.limit { + self.capped = true; + } else if let Some(id) = reference.and_then(|id| self.arena.push_decoded(id)) { + self.ids.push(id); + } + } +} + +impl Handler for MainPart<'_> { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>) { + if let Root::Document(text) = &mut self.root { + text.start(tag, out); + return; + } + let element = RootElement::parse(tag.local()); + match (&self.root, element) { + (Root::Pending, Some(RootElement::Document)) => { + self.root = Root::Document(RunText::default()) + } + (Root::Pending, Some(RootElement::Workbook)) => self.root = Root::Workbook, + (Root::Pending, Some(RootElement::Presentation)) => self.root = Root::Presentation, + (Root::Pending, _) => self.root = Root::Unknown, + (Root::Workbook, Some(RootElement::Sheet)) => self.collect_reference(tag, Some(out)), + (Root::Presentation, Some(RootElement::SlideId)) => self.collect_reference(tag, None), + (Root::Workbook, Some(RootElement::WorkbookProperties)) => { + self.dates = DateSystem::from_properties(tag); + } + _ => {} + } + } + + fn end(&mut self, name: &[u8], out: &mut Output<'_>) { + if let Root::Document(text) = &mut self.root { + text.end(name, out); + } + } + + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>) { + if let Root::Document(handler) = &mut self.root { + handler.text(text, out); + } + } + + fn aborted(&self) -> bool { + matches!(self.root, Root::Unknown) + } +} diff --git a/crates/text-extract/src/ooxml/xlsx.rs b/crates/text-extract/src/ooxml/xlsx.rs new file mode 100644 index 00000000..29167a81 --- /dev/null +++ b/crates/text-extract/src/ooxml/xlsx.rs @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Relations, + chart::ChartValues, + number::DateSystem, + rels::{Rel, RelKind, find_kind, find_sorted, sort_by_id}, + sheet::SheetText, + styles::Styles, + text::RunText, +}; +use crate::{ + output::{Output, Separator}, + package::{Package, Span}, +}; + +pub(super) fn extract( + package: &mut Package<'_, '_>, + relations: &mut Relations<'_>, + rels: &mut Vec, + sheets: &[Span], + styles: &mut Styles, + dates: DateSystem, + out: &mut Output<'_>, +) { + if let Some(target) = find_kind(rels, RelKind::Styles) { + package.scan_plumbing(relations.arena.get(target), &mut styles.handler(), out); + } + if let Some(shared) = find_kind(rels, RelKind::SharedStrings) { + package.scan(relations.arena.get(shared), &mut RunText::default(), out); + out.separator(Separator::Newline); + } + sort_by_id(relations.arena, rels); + for &sheet in sheets { + if package.stopped(out) { + return; + } + let Some(rel) = find_sorted(relations.arena, rels, sheet, |kind| { + matches!(kind, RelKind::Worksheet | RelKind::Chartsheet) + }) else { + continue; + }; + if rel.kind == RelKind::Worksheet { + let mut handler = SheetText::new(styles.cell_formats(), dates); + if !package.scan(relations.arena.get(rel.target), &mut handler, out) { + continue; + } + out.separator(Separator::Newline); + } + let mark = relations.arena.mark(); + if relations.load_children(package, rel.target, out) { + relations.scan_children(package, ChartValues::Strings, out); + } + relations.arena.rewind(mark); + } +} diff --git a/crates/text-extract/src/output.rs b/crates/text-extract/src/output.rs new file mode 100644 index 00000000..b6d0ea0c --- /dev/null +++ b/crates/text-extract/src/output.rs @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const MIN_AMORTIZED_CAPACITY: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Separator { + None, + Space, + Newline, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Mark { + len: usize, + pending: Separator, +} + +pub(crate) struct Output<'a> { + buf: &'a mut String, + start: usize, + limit: usize, + pending: Separator, + full: bool, +} + +impl<'a> Output<'a> { + pub(crate) fn new(buf: &'a mut String, max_bytes: usize) -> Self { + let start = buf.len(); + Output { + limit: start.saturating_add(max_bytes), + buf, + start, + pending: Separator::None, + full: false, + } + } + + #[inline] + pub(crate) fn is_full(&self) -> bool { + self.full + } + + #[inline] + pub(crate) fn separator(&mut self, separator: Separator) { + self.pending = self.pending.max(separator); + } + + pub(crate) fn push_str(&mut self, text: &str) { + if self.full || text.is_empty() { + return; + } + if text.bytes().all(|byte| byte.is_ascii_whitespace()) { + self.separator(Separator::Space); + return; + } + if self.pending != Separator::None { + let separator = std::mem::replace(&mut self.pending, Separator::None); + if self.buf.len() > self.start { + if self.buf.len() >= self.limit { + self.full = true; + return; + } + self.reserve(text.len() + 1); + self.buf.push(if separator == Separator::Newline { + '\n' + } else { + ' ' + }); + } + } + let room = self.limit.saturating_sub(self.buf.len()); + if text.len() <= room { + self.reserve(text.len()); + self.buf.push_str(text); + } else { + let text = text + .get(..text.floor_char_boundary(room)) + .unwrap_or_default(); + self.reserve(text.len()); + self.buf.push_str(text); + self.full = true; + } + } + + #[inline] + fn reserve(&mut self, additional: usize) { + if self.buf.capacity() - self.buf.len() < additional { + self.grow(additional); + } + } + + #[cold] + #[inline(never)] + fn grow(&mut self, additional: usize) { + let capacity = self.buf.capacity(); + let room = self.limit.saturating_sub(self.buf.len()); + let additional = additional.min(room); + if capacity.saturating_mul(2).max(MIN_AMORTIZED_CAPACITY) <= self.limit { + self.buf.reserve(additional); + } else { + self.buf.reserve_exact(additional.max(capacity).min(room)); + } + } + + pub(crate) fn push_utf8(&mut self, bytes: &[u8]) { + if let Ok(text) = std::str::from_utf8(bytes) { + self.push_str(text); + return; + } + for chunk in bytes.utf8_chunks() { + self.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + self.push_char(char::REPLACEMENT_CHARACTER); + } + } + } + + pub(crate) fn push_char(&mut self, ch: char) { + let mut encoded = [0u8; 4]; + self.push_str(ch.encode_utf8(&mut encoded)); + } + + pub(crate) fn mark(&self) -> Mark { + Mark { + len: self.buf.len(), + pending: self.pending, + } + } + + pub(crate) fn cut(&mut self, mark: Mark, into: &mut String, max_bytes: usize) { + let start = mark.len.max(self.start); + if let Some(tail) = self.buf.get(start..).map(str::trim) + && !tail.is_empty() + { + let room = max_bytes.saturating_sub(into.len() + 1); + if !into.is_empty() && room > 0 { + into.push(' '); + } + into.push_str( + tail.get(..tail.floor_char_boundary(room)) + .unwrap_or_default(), + ); + } + self.buf.truncate(start); + self.pending = mark.pending; + self.full = self.full && self.buf.len() >= self.limit; + } + + pub(crate) fn since(&self, mark: Mark) -> &str { + self.buf.get(mark.len.max(self.start)..).unwrap_or_default() + } + + pub(crate) fn written(&self) -> usize { + self.buf.len().saturating_sub(self.start) + } +} + diff --git a/crates/text-extract/src/package.rs b/crates/text-extract/src/package.rs new file mode 100644 index 00000000..dc8d167a --- /dev/null +++ b/crates/text-extract/src/package.rs @@ -0,0 +1,266 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + output::Output, + xml::{ + Handler, + attr::decode_into, + stream::{Budget, Buffers, Part}, + }, + zip::{Archive, Member, ReadRanges}, +}; +use memchr::{memmem, memrchr}; +use std::ops::Range; + +const MAX_ARENA: usize = 1 << 20; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Span { + start: u32, + end: u32, +} + +impl Span { + pub(crate) fn start(self) -> u32 { + self.start + } + + fn range(self) -> Range { + self.start as usize..self.end as usize + } +} + +#[derive(Default)] +pub(crate) struct Arena { + bytes: Vec, + scratch: Vec, + decoded: Vec, +} + +impl Arena { + pub(crate) fn clear(&mut self) { + self.bytes.clear(); + } + + pub(crate) fn mark(&self) -> usize { + self.bytes.len() + } + + pub(crate) fn rewind(&mut self, mark: usize) { + self.bytes.truncate(mark); + } + + pub(crate) fn get(&self, span: Span) -> &[u8] { + self.bytes.get(span.range()).unwrap_or_default() + } + + fn span_from(&mut self, start: usize) -> Option { + if self.bytes.len() > MAX_ARENA { + self.bytes.truncate(start); + return None; + } + Some(Span { + start: u32::try_from(start).ok()?, + end: u32::try_from(self.bytes.len()).ok()?, + }) + } + + pub(crate) fn push(&mut self, bytes: &[u8]) -> Option { + let start = self.bytes.len(); + self.bytes + .extend_from_slice(bytes.get(..MAX_ARENA.min(bytes.len()))?); + self.span_from(start) + } + + pub(crate) fn push_decoded(&mut self, raw: &[u8]) -> Option { + let start = self.bytes.len(); + decode_into(raw.get(..MAX_ARENA.min(raw.len()))?, &mut self.bytes); + self.span_from(start) + } + + pub(crate) fn push_rels_path(&mut self, part: Span) -> Option { + let start = self.bytes.len(); + let part = part.range(); + let file_len = self + .bytes + .get(part.clone())? + .rsplit(|&byte| byte == b'/') + .next() + .map_or(0, <[u8]>::len); + let dir_end = part.end - file_len; + self.bytes.extend_from_within(part.start..dir_end); + self.bytes.extend_from_slice(b"_rels/"); + self.bytes.extend_from_within(dir_end..part.end); + self.bytes.extend_from_slice(b".rels"); + self.span_from(start) + } + + pub(crate) fn push_resolved(&mut self, source: Span, target: &[u8]) -> Option { + self.scratch.clear(); + decode_into( + target.get(..MAX_ARENA.min(target.len()))?, + &mut self.scratch, + ); + self.decoded.clear(); + percent_decode_into( + self.scratch + .split(|&byte| byte == b'#') + .next() + .unwrap_or_default(), + &mut self.decoded, + ); + if self.decoded.is_empty() || memmem::find(&self.decoded, b"://").is_some() { + return None; + } + let start = self.bytes.len(); + if self.decoded.first() != Some(&b'/') { + let source = source.range(); + if let Some(mut pieces) = self + .bytes + .get(source.clone()) + .map(|path| path.rsplitn(2, |&byte| byte == b'/')) + && let (Some(_), Some(dir)) = (pieces.next(), pieces.next()) + { + let dir_end = source.start + dir.len(); + self.bytes.extend_from_within(source.start..dir_end); + } + } + for segment in self.decoded.split(|&byte| byte == b'/' || byte == b'\\') { + hashify::fnc_map!(segment, + b"" => {}, + b"." => {}, + b".." => { + let current = self.bytes.get(start..).unwrap_or_default(); + let keep = memrchr(b'/', current).unwrap_or(0); + self.bytes.truncate(start + keep); + }, + _ => { + if self.bytes.len() > start { + self.bytes.push(b'/'); + } + self.bytes.extend_from_slice(segment); + } + ); + if self.bytes.len() > MAX_ARENA { + self.bytes.truncate(start); + return None; + } + } + self.span_from(start) + } +} + +fn percent_decode_into(source: &[u8], target: &mut Vec) { + let mut pieces = source.split(|&byte| byte == b'%'); + if let Some(head) = pieces.next() { + target.extend_from_slice(head); + } + for piece in pieces { + match split_escape(piece) { + Some((value, rest)) => { + target.push(value); + target.extend_from_slice(rest); + } + None => { + target.push(b'%'); + target.extend_from_slice(piece); + } + } + } +} + +fn split_escape(piece: &[u8]) -> Option<(u8, &[u8])> { + let [high, low, rest @ ..] = piece else { + return None; + }; + Some(((hex_digit(*high)? << 4) | hex_digit(*low)?, rest)) +} + +#[inline] +fn hex_digit(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +pub(crate) struct Package<'a, 's> { + pub(crate) archive: &'s Archive<'a>, + pub(crate) buffers: &'s mut Buffers, + pub(crate) claimed: &'s mut ReadRanges, + pub(crate) budget: Budget, +} + +impl<'a> Package<'a, '_> { + pub(crate) fn scan( + &mut self, + name: &[u8], + handler: &mut H, + out: &mut Output<'_>, + ) -> bool { + if out.is_full() { + return false; + } + match self.archive.find(name) { + Some(member) => self.scan_member(&member, handler, out), + None => false, + } + } + + pub(crate) fn scan_member( + &mut self, + member: &Member<'a>, + handler: &mut H, + out: &mut Output<'_>, + ) -> bool { + self.scan_member_as(member, Part::Content, handler, out) + } + + pub(crate) fn scan_plumbing( + &mut self, + name: &[u8], + handler: &mut H, + out: &mut Output<'_>, + ) -> bool { + match self.archive.find(name) { + Some(member) => self.scan_member_as(&member, Part::Plumbing, handler, out), + None => false, + } + } + + pub(crate) fn scan_member_as( + &mut self, + member: &Member<'a>, + part: Part, + handler: &mut H, + out: &mut Output<'_>, + ) -> bool { + if out.is_full() { + return false; + } + match self.archive.read(member, self.claimed) { + Some(data) => { + self.buffers + .scan(data, part, &mut self.budget, handler, out); + true + } + None => false, + } + } + + pub(crate) fn stopped(&mut self, out: &Output<'_>) -> bool { + if out.is_full() { + return true; + } + let exhausted = self.budget.exhausted(); + self.budget.truncated |= exhausted; + exhausted + } +} + diff --git a/crates/text-extract/src/pdf/content/form.rs b/crates/text-extract/src/pdf/content/form.rs new file mode 100644 index 00000000..eb07fb2c --- /dev/null +++ b/crates/text-extract/src/pdf/content/form.rs @@ -0,0 +1,149 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Frame, Interpreter, MAX_FORM_DEPTH, MAX_MARKED_DEPTH, MAX_TEXTLESS, ops::Operand, state::Matrix, +}; +use crate::{ + output::Output, + pdf::{ + decode::DecodeOutcome, + document::Document, + layout::GlyphCounts, + object::{Name, ObjRef, Object}, + text_string::decode_text_string, + }, +}; + +const MATRIX_LEN: usize = 6; + +impl Interpreter { + pub(super) fn form<'d>( + &mut self, + doc: &'d Document<'_>, + frame: &Frame<'d>, + name: Name<'_>, + out: &mut Output<'_>, + ) { + if frame.depth >= MAX_FORM_DEPTH || doc.exhausted() { + return; + } + let Some(id) = self.resources.xobject(doc, frame.resources, name) else { + return; + }; + let key = ( + id, + self.state.fingerprint(&self.text_matrix, &self.line_matrix), + ); + if self.textless.contains(&id) + || self.suppressed.contains(&key) + || self.ancestors.contains(&id) + { + return; + } + let Object::Stream(stream) = doc.get(id) else { + return; + }; + if !doc + .get_name(stream.dict, b"Subtype") + .is_some_and(|subtype| subtype.is(b"Form")) + { + return; + } + let matrix = doc + .get_array(stream.dict, b"Matrix") + .and_then(|array| { + let mut values = [0f64; MATRIX_LEN]; + let mut count = 0; + for (slot, value) in values.iter_mut().zip(doc.array_iter(array)) { + *slot = value.as_f64()?; + count += 1; + } + (count == MATRIX_LEN).then(|| Matrix::new(values)) + }) + .unwrap_or(Matrix::IDENTITY); + let resources = doc.get_dict(stream.dict, b"Resources").or(frame.resources); + let mut buf = self.pooled_buffer(); + let (data, outcome) = doc.stream_bytes(stream, &mut buf); + if !outcome.is_failure() { + let saved = (self.state, self.text_matrix, self.line_matrix); + let mark = self.stack.enter(); + self.state.ctm = matrix.then(&self.state.ctm); + let before = self.layout.counts(); + self.ancestors.push(id); + self.run(doc, data, resources, frame.depth + 1, out); + self.ancestors.pop(); + self.stack.leave(mark); + (self.state, self.text_matrix, self.line_matrix) = saved; + self.continued = false; + if outcome == DecodeOutcome::Complete && !out.is_full() && !doc.exhausted() { + self.remember_textless(key, before); + } + } + buf.clear(); + self.pool.push(buf); + } + + fn remember_textless(&mut self, key: (ObjRef, u64), before: GlyphCounts) { + let after = self.layout.counts(); + if after.candidates == before.candidates { + if self.textless.len() < MAX_TEXTLESS { + self.textless.insert(key.0); + } + } else if after.placed == before.placed + && !before.in_actual_text + && self.suppressed.len() < MAX_TEXTLESS + { + self.suppressed.insert(key); + } + } + + fn pooled_buffer(&mut self) -> Vec { + self.pool + .iter() + .enumerate() + .max_by_key(|(_, buffer)| buffer.capacity()) + .map(|(index, _)| index) + .map(|index| self.pool.swap_remove(index)) + .unwrap_or_default() + } + + pub(super) fn begin_marked(&mut self, frame: &mut Frame<'_>, actual: bool) { + if self.marked.len() >= MAX_MARKED_DEPTH { + frame.marked_overflow = frame.marked_overflow.saturating_add(1); + } else { + self.marked.push(actual); + } + } + + pub(super) fn begin_marked_properties<'d>( + &mut self, + doc: &'d Document<'_>, + frame: &mut Frame<'d>, + properties: Operand<'_>, + ) { + let value = match properties { + Operand::Dict(dict) => dict.get(b"ActualText").map(|value| doc.resolve(value)), + Operand::Name(name) => self.resources.actual_text(doc, frame.resources, name), + _ => None, + }; + let mut actual = false; + if let Some(Object::Str(value)) = value { + self.actual_text.clear(); + decode_text_string(&doc.string(value), &mut self.actual_text); + actual = self.layout.begin_actual_text(&self.actual_text); + } + self.begin_marked(frame, actual); + } + + pub(super) fn end_marked(&mut self, frame: &mut Frame<'_>, out: &mut Output<'_>) { + if frame.marked_overflow > 0 { + frame.marked_overflow -= 1; + } else if self.marked.len() > frame.marked_base && self.marked.pop() == Some(true) { + self.layout.end_actual_text(out); + } + } +} diff --git a/crates/text-extract/src/pdf/content/inline/markers.rs b/crates/text-extract/src/pdf/content/inline/markers.rs new file mode 100644 index 00000000..beeecacc --- /dev/null +++ b/crates/text-extract/src/pdf/content/inline/markers.rs @@ -0,0 +1,173 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::{ + content::ops::Op, + lexer::{Lexer, Token, is_whitespace}, + seek::Seek, +}; +use memchr::{memchr, memmem}; +use std::collections::VecDeque; + +pub(super) const END_MARKER: &[u8] = b"EI"; +pub(super) const DATA_MARKER: &[u8] = b"ID"; +pub(super) const ASCII85_END: &[u8] = b"~>"; +const VALIDATION_WINDOW: usize = 96; +const MAX_VALIDATED_CANDIDATES: usize = 256; +const MAX_NULS: usize = 1; + +#[derive(Debug, Clone, Copy)] +struct Candidate { + marker: usize, + plausible: bool, +} + +#[derive(Debug, Default)] +struct Candidates { + origin: usize, + frontier: usize, + queue: VecDeque, +} + +#[derive(Debug, Default)] +pub(crate) struct Markers { + hex_end: Seek, + ascii85_end: Seek, + end: Seek, + data: Seek, + candidates: Candidates, +} + +impl Markers { + pub(super) fn hex_end(&mut self, data: &[u8], pos: usize) -> Option { + self.hex_end.find(pos, |from| { + memchr(b'>', data.get(from..)?).map(|offset| from + offset) + }) + } + + pub(super) fn ascii85_end(&mut self, data: &[u8], pos: usize) -> Option { + self.ascii85_end + .find(pos, |from| find(data, from, ASCII85_END)) + } + + pub(super) fn end(&mut self, data: &[u8], pos: usize) -> Option { + self.end.find(pos, |from| find(data, from, END_MARKER)) + } + + pub(super) fn data(&mut self, data: &[u8], pos: usize) -> Option { + self.data.find(pos, |from| find(data, from, DATA_MARKER)) + } + + pub(super) fn scan(&mut self, data: &[u8], start: usize) -> Option { + let candidates = &mut self.candidates; + candidates.restart(start); + let head = data + .get(start..) + .filter(|rest| rest.starts_with(END_MARKER)) + .filter(|_| !preceded(data, start) && delimited_after(data, start)) + .map(|_| Candidate { + marker: start, + plausible: plausible_tail(data, start + END_MARKER.len()), + }); + let mut fallback = None; + let mut queued = 0usize; + for index in 0..=MAX_VALIDATED_CANDIDATES { + let candidate = match (head, index) { + (Some(head), 0) => head, + _ => { + let Some(candidate) = candidates.get(data, queued) else { + break; + }; + queued += 1; + candidate + } + }; + let end = candidate.marker + END_MARKER.len(); + if index == MAX_VALIDATED_CANDIDATES || candidate.plausible { + return Some(end); + } + fallback = fallback.or(Some(end)); + } + fallback + } +} + +impl Candidates { + fn restart(&mut self, start: usize) { + if start < self.origin || self.frontier < start { + self.queue.clear(); + self.frontier = start; + } + self.origin = start; + while self + .queue + .front() + .is_some_and(|candidate| candidate.marker < start) + { + self.queue.pop_front(); + } + } + + fn get(&mut self, data: &[u8], index: usize) -> Option { + if let Some(candidate) = self.queue.get(index) { + return Some(*candidate); + } + let from = self.frontier; + let marker = memmem::find_iter(data.get(from..)?, END_MARKER) + .map(|offset| from + offset) + .find(|&marker| preceded(data, marker) && delimited_after(data, marker)); + self.frontier = marker.map_or(data.len(), |marker| marker + 1); + let candidate = Candidate { + marker: marker?, + plausible: plausible_tail(data, marker? + END_MARKER.len()), + }; + self.queue.push_back(candidate); + Some(candidate) + } +} + +fn find(data: &[u8], from: usize, needle: &[u8]) -> Option { + memmem::find(data.get(from..)?, needle).map(|offset| from + offset) +} + +fn preceded(data: &[u8], marker: usize) -> bool { + marker + .checked_sub(1) + .and_then(|before| data.get(before)) + .is_none_or(|&byte| is_whitespace(byte)) +} + +pub(super) fn delimited_after(data: &[u8], marker: usize) -> bool { + data.get(marker + END_MARKER.len()) + .is_none_or(|&byte| is_whitespace(byte) || byte == b'<' || byte == b'/') +} + +fn plausible_tail(data: &[u8], end: usize) -> bool { + let window = data + .get(end..) + .map(|rest| rest.get(..VALIDATION_WINDOW).unwrap_or(rest)) + .unwrap_or_default(); + let nuls = window.iter().filter(|&&byte| byte == 0).count(); + if nuls > MAX_NULS + || window + .iter() + .any(|&byte| byte != 0 && !is_whitespace(byte) && !(0x20..0x7F).contains(&byte)) + { + return false; + } + let mut lexer = Lexer::new(window); + while let Some(token) = lexer.next() { + match token { + Token::Keyword(keyword) => { + return Op::is_known(keyword) || lexer.at_end(); + } + Token::Error => return false, + _ => {} + } + } + true +} + diff --git a/crates/text-extract/src/pdf/content/inline/mod.rs b/crates/text-extract/src/pdf/content/inline/mod.rs new file mode 100644 index 00000000..250a364a --- /dev/null +++ b/crates/text-extract/src/pdf/content/inline/mod.rs @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod markers; + +pub(crate) use markers::Markers; + +use crate::pdf::{ + lexer::{Lexer, Token, is_whitespace}, + object::Object, +}; +use markers::{ASCII85_END, DATA_MARKER, END_MARKER, delimited_after}; + +const MAX_DICT_ENTRIES: usize = 64; +const MAX_END_GAP: usize = 256; +const BITS_PER_BYTE: u64 = 8; + +#[derive(Debug, Default)] +struct ImageInfo { + length: Option, + filtered: bool, + hex: bool, + ascii85: bool, + width: Option, + height: Option, + bits: Option, + components: Option, + mask: bool, +} + +pub(crate) fn skip(lexer: &mut Lexer<'_>, markers: &mut Markers) { + let Some(info) = read_dict(lexer, markers) else { + return; + }; + let data = lexer.data(); + let start = data_start(data, lexer.pos()); + let end = info + .length + .and_then(|length| accept_at(data, start, length)) + .or_else(|| info.filter_end(data, start, markers)) + .or_else(|| { + info.unfiltered_size() + .and_then(|size| accept_at(data, start, size)) + }) + .or_else(|| markers.scan(data, start)) + .unwrap_or(data.len()); + lexer.set_pos(end); +} + +fn read_dict(lexer: &mut Lexer<'_>, markers: &mut Markers) -> Option { + let mut info = ImageInfo::default(); + for _ in 0..MAX_DICT_ENTRIES * 2 { + let before = lexer.pos(); + match lexer.next()? { + Token::Keyword(b"ID") => return Some(info), + Token::Keyword(b"EI") => return None, + Token::Keyword(_) => { + lexer.set_pos(before); + return None; + } + Token::Name(key) => { + let value = Object::read(lexer, None).unwrap_or_default(); + info.set(key, value); + } + _ => {} + } + } + let found = markers.data(lexer.data(), lexer.pos())?; + lexer.set_pos(found + DATA_MARKER.len()); + Some(info) +} + +impl ImageInfo { + fn set(&mut self, key: &[u8], value: Object<'_>) { + let count = || value.as_int().and_then(|value| u64::try_from(value).ok()); + hashify::fnc_map!(key, + b"L" => { self.length = count(); }, + b"Length" => { self.length = count(); }, + b"W" => { self.width = count(); }, + b"Width" => { self.width = count(); }, + b"H" => { self.height = count(); }, + b"Height" => { self.height = count(); }, + b"BPC" => { self.bits = count(); }, + b"BitsPerComponent" => { self.bits = count(); }, + b"IM" => { self.mask = value.as_bool().unwrap_or(false); }, + b"ImageMask" => { self.mask = value.as_bool().unwrap_or(false); }, + b"CS" => self.set_color_space(value), + b"ColorSpace" => self.set_color_space(value), + b"F" => self.set_filter(value), + b"Filter" => self.set_filter(value), + _ => {} + ); + } + + fn set_color_space(&mut self, value: Object<'_>) { + self.components = match value { + Object::Name(name) => hashify::map!(name.raw(), u64, + b"G" => 1, + b"DeviceGray" => 1, + b"I" => 1, + b"Indexed" => 1, + b"RGB" => 3, + b"DeviceRGB" => 3, + b"CMYK" => 4, + b"DeviceCMYK" => 4, + ) + .copied(), + Object::Array(array) => match array.get(0).and_then(|first| first.as_name()) { + Some(name) if name.is(b"I") || name.is(b"Indexed") => Some(1), + _ => None, + }, + _ => None, + } + } + + fn set_filter(&mut self, value: Object<'_>) { + let first = match value { + Object::Array(array) => array.get(0).and_then(|first| first.as_name()), + other => other.as_name(), + }; + if let Some(name) = first { + self.filtered = true; + self.hex = name.is(b"AHx") || name.is(b"ASCIIHexDecode"); + self.ascii85 = name.is(b"A85") || name.is(b"ASCII85Decode"); + } + } + + fn filter_end(&self, data: &[u8], start: usize, markers: &mut Markers) -> Option { + let after = if self.hex { + markers.hex_end(data, start)? + 1 + } else if self.ascii85 { + markers.ascii85_end(data, start)? + ASCII85_END.len() + } else { + return None; + }; + let marker = markers.end(data, after)?; + delimited_after(data, marker).then_some(marker + END_MARKER.len()) + } + + fn unfiltered_size(&self) -> Option { + if self.filtered { + return None; + } + let (components, bits) = if self.mask { + (1, 1) + } else { + (self.components?, self.bits?) + }; + let row_bits = self.width?.checked_mul(components)?.checked_mul(bits)?; + row_bits.div_ceil(BITS_PER_BYTE).checked_mul(self.height?) + } +} + +fn data_start(data: &[u8], pos: usize) -> usize { + match (data.get(pos), data.get(pos + 1)) { + (Some(b'\r'), Some(b'\n')) => pos + 2, + (Some(&byte), _) if is_whitespace(byte) => pos + 1, + _ => pos, + } + .min(data.len()) +} + +fn accept_at(data: &[u8], start: usize, length: u64) -> Option { + let end = start.checked_add(usize::try_from(length).ok()?)?; + let rest = data.get(end..)?; + let skip = rest + .iter() + .take(MAX_END_GAP + 1) + .take_while(|&&byte| is_whitespace(byte)) + .count(); + if skip > MAX_END_GAP { + return None; + } + let marker = end + skip; + (data.get(marker..)?.starts_with(END_MARKER) && delimited_after(data, marker)) + .then_some(marker + END_MARKER.len()) +} + diff --git a/crates/text-extract/src/pdf/content/mod.rs b/crates/text-extract/src/pdf/content/mod.rs new file mode 100644 index 00000000..7af25425 --- /dev/null +++ b/crates/text-extract/src/pdf/content/mod.rs @@ -0,0 +1,334 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod form; +mod inline; +mod ops; +mod resources; +mod state; +mod text; + +use self::{ + ops::{Op, Operand, Operands}, + resources::{Lookup, ResourceCache}, + state::{GraphicsState, Matrix, StateStack, finite}, +}; +use super::{ + document::Document, + font::FontCache, + layout::Layout, + lexer::{Lexer, Token}, + object::{Dict, Name, ObjRef, Object, Str}, +}; +use crate::output::Output; +use std::collections::HashSet; + +pub(crate) const MAX_FORM_DEPTH: usize = 16; +pub(crate) const MAX_CONSECUTIVE_ERRORS: usize = 1024; +pub(crate) const MAX_MARKED_DEPTH: usize = 1024; +const MAX_TEXTLESS: usize = 1 << 16; +const MAX_RETAINED: usize = 1 << 20; +const MAX_POOLED: usize = 2; + +#[derive(Default)] +pub(crate) struct Interpreter { + fonts: FontCache, + resources: ResourceCache, + textless: HashSet, + suppressed: HashSet<(ObjRef, u64)>, + ancestors: Vec, + stack: StateStack, + state: GraphicsState, + text_matrix: Matrix, + line_matrix: Matrix, + continued: bool, + marked: Vec, + pool: Vec>, + font_buf: Vec, + glyph_text: String, + string_buf: Vec, + actual_text: String, + layout: Layout, +} + +struct Frame<'d> { + resources: Option>, + depth: usize, + marked_base: usize, + marked_overflow: usize, + inline: inline::Markers, +} + +impl Default for Matrix { + fn default() -> Self { + Matrix::IDENTITY + } +} + +impl Interpreter { + pub(crate) fn begin_document(&mut self) { + self.fonts.clear(); + self.resources.clear(); + self.textless.clear(); + self.suppressed.clear(); + } + + pub(crate) fn shrink(&mut self) { + self.fonts.shrink(); + self.resources.shrink(); + self.textless = HashSet::new(); + self.suppressed = HashSet::new(); + self.stack.shrink(); + self.pool.truncate(MAX_POOLED); + for buffer in self + .pool + .iter_mut() + .chain(std::iter::once(&mut self.font_buf)) + { + buffer.clear(); + buffer.shrink_to(MAX_RETAINED); + } + self.glyph_text = String::new(); + self.string_buf = Vec::new(); + self.actual_text = String::new(); + self.layout.shrink(); + } + + pub(crate) fn layout(&mut self) -> &mut Layout { + &mut self.layout + } + + pub(crate) fn run_page<'d>( + &mut self, + doc: &'d Document<'_>, + content: &[u8], + resources: Option>, + media_box: Option<[f64; 4]>, + out: &mut Output<'_>, + ) { + self.layout.begin_page(media_box); + self.suppressed.clear(); + self.state = GraphicsState::default(); + self.stack.clear(); + self.text_matrix = Matrix::IDENTITY; + self.line_matrix = Matrix::IDENTITY; + self.continued = false; + self.marked.clear(); + self.ancestors.clear(); + self.run(doc, content, resources, 0, out); + self.fonts.end_page(); + } + + fn run<'d>( + &mut self, + doc: &'d Document<'_>, + content: &[u8], + resources: Option>, + depth: usize, + out: &mut Output<'_>, + ) { + let mut frame = Frame { + resources, + depth, + marked_base: self.marked.len(), + marked_overflow: 0, + inline: inline::Markers::default(), + }; + let mut lexer = Lexer::new(content); + let mut operands = Operands::default(); + let mut errors = 0usize; + while !out.is_full() && errors < MAX_CONSECUTIVE_ERRORS { + let before = lexer.pos(); + let Some(token) = lexer.next() else { + break; + }; + match token { + Token::Int(value) => operands.push(Operand::Number(value as f64)), + Token::Real(value) => operands.push(Operand::Number(finite(value))), + Token::Name(raw) => operands.push(Operand::Name(Name::new(raw))), + Token::Literal(raw) => operands.push(Operand::Str(Str::literal(raw, None))), + Token::Hex(raw) => operands.push(Operand::Str(Str::hex(raw, None))), + Token::ArrayOpen | Token::DictOpen => { + lexer.set_pos(before); + match Object::read(&mut lexer, None) { + Some(object) => operands.push(Operand::from_object(object)), + None => { + lexer.next(); + errors += 1; + } + } + } + Token::Keyword(b"true" | b"false" | b"null") => operands.push(Operand::Other), + Token::Keyword(b"obj" | b"endobj" | b"endstream") => break, + Token::Keyword(keyword) => { + let op = Op::parse(keyword); + if op != Op::Unknown + && self.execute(op, &operands, &mut lexer, doc, &mut frame, out) + { + errors = 0; + } else { + errors += 1; + } + operands.clear(); + } + Token::ArrayClose + | Token::DictClose + | Token::BraceOpen + | Token::BraceClose + | Token::Error => errors += 1, + } + } + while self.marked.len() > frame.marked_base { + if self.marked.pop() == Some(true) { + self.layout.end_actual_text(out); + } + } + } + + fn execute<'d>( + &mut self, + op: Op, + operands: &Operands<'_>, + lexer: &mut Lexer<'_>, + doc: &'d Document<'_>, + frame: &mut Frame<'d>, + out: &mut Output<'_>, + ) -> bool { + match op { + Op::Save => self.stack.save(&self.state), + Op::Restore => self.stack.restore(&mut self.state), + Op::Concat => { + let Some(values) = operands.numbers::<6>() else { + return false; + }; + self.state.ctm = Matrix::new(values).then(&self.state.ctm); + } + Op::SetState => { + let Some([Operand::Name(name)]) = operands.last::<1>() else { + return false; + }; + let lookup = Lookup { + doc, + fonts: &mut self.fonts, + buf: &mut self.font_buf, + }; + if let Some((font, size)) = self.resources.state_font(lookup, frame.resources, name) + { + self.state.text.font = font; + self.state.text.size = finite(size); + } + } + Op::BeginText => { + self.text_matrix = Matrix::IDENTITY; + self.line_matrix = Matrix::IDENTITY; + self.continued = false; + } + Op::EndText | Op::Ignored | Op::Unknown => {} + Op::CharSpacing | Op::WordSpacing | Op::HorizontalScale | Op::Leading | Op::Rise => { + let Some([value]) = operands.numbers::<1>() else { + return false; + }; + let value = finite(value); + let text = &mut self.state.text; + match op { + Op::CharSpacing => text.char_spacing = value, + Op::WordSpacing => text.word_spacing = value, + Op::HorizontalScale => text.set_scale(value), + Op::Leading => text.leading = value, + _ => text.rise = value, + } + } + Op::Font => { + let Some([Operand::Name(name), Operand::Number(size)]) = operands.last::<2>() + else { + return false; + }; + let lookup = Lookup { + doc, + fonts: &mut self.fonts, + buf: &mut self.font_buf, + }; + if let Some(font) = self.resources.font(lookup, frame.resources, name) { + self.state.text.font = font; + } + self.state.text.size = finite(size); + } + Op::MoveText | Op::MoveTextLeading => { + let Some([x, y]) = operands.numbers::<2>() else { + return false; + }; + if op == Op::MoveTextLeading { + self.state.text.leading = -finite(y); + } + self.line_matrix.translate(x, y); + self.text_matrix = self.line_matrix; + self.continued = false; + } + Op::TextMatrix => { + let Some(values) = operands.numbers::<6>() else { + return false; + }; + self.line_matrix = Matrix::new(values); + self.text_matrix = self.line_matrix; + self.continued = false; + } + Op::NextLine => self.next_line(), + Op::Show | Op::NextLineShow => { + let Some([Operand::Str(value)]) = operands.last::<1>() else { + return false; + }; + if op == Op::NextLineShow { + self.next_line(); + } + self.show_str(value, out); + } + Op::NextLineShowSpaced => { + let Some( + [ + Operand::Number(word), + Operand::Number(char), + Operand::Str(value), + ], + ) = operands.last::<3>() + else { + return false; + }; + self.state.text.word_spacing = finite(word); + self.state.text.char_spacing = finite(char); + self.next_line(); + self.show_str(value, out); + } + Op::ShowArray => { + let Some([Operand::Array(array)]) = operands.last::<1>() else { + return false; + }; + self.show_array(array, out); + } + Op::XObject => { + let Some([Operand::Name(name)]) = operands.last::<1>() else { + return false; + }; + self.form(doc, frame, name, out); + } + Op::BeginImage => inline::skip(lexer, &mut frame.inline), + Op::BeginMarked => self.begin_marked(frame, false), + Op::BeginMarkedProperties => { + let Some([_, properties]) = operands.last::<2>() else { + return false; + }; + self.begin_marked_properties(doc, frame, properties); + } + Op::EndMarked => self.end_marked(frame, out), + } + true + } + + fn next_line(&mut self) { + self.line_matrix.translate(0.0, -self.state.text.leading); + self.text_matrix = self.line_matrix; + self.continued = false; + } +} diff --git a/crates/text-extract/src/pdf/content/ops.rs b/crates/text-extract/src/pdf/content/ops.rs new file mode 100644 index 00000000..f3d730ed --- /dev/null +++ b/crates/text-extract/src/pdf/content/ops.rs @@ -0,0 +1,213 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::object::{Array, Dict, Name, Object, Str}; + +pub(crate) const MAX_OPERANDS: usize = 33; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Op { + Save, + Restore, + Concat, + SetState, + BeginText, + EndText, + CharSpacing, + WordSpacing, + HorizontalScale, + Leading, + Font, + Rise, + MoveText, + MoveTextLeading, + TextMatrix, + NextLine, + Show, + ShowArray, + NextLineShow, + NextLineShowSpaced, + XObject, + BeginImage, + BeginMarked, + BeginMarkedProperties, + EndMarked, + Ignored, + Unknown, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) enum Operand<'a> { + #[default] + Other, + Number(f64), + Name(Name<'a>), + Str(Str<'a>), + Array(Array<'a>), + Dict(Dict<'a>), +} + +#[derive(Debug)] +pub(crate) struct Operands<'a> { + items: [Operand<'a>; MAX_OPERANDS], + len: usize, +} + +impl Op { + pub(crate) fn parse(keyword: &[u8]) -> Op { + hashify::map!(keyword, Op, + b"q" => Op::Save, + b"Q" => Op::Restore, + b"cm" => Op::Concat, + b"gs" => Op::SetState, + b"BT" => Op::BeginText, + b"ET" => Op::EndText, + b"Tc" => Op::CharSpacing, + b"Tw" => Op::WordSpacing, + b"Tz" => Op::HorizontalScale, + b"TL" => Op::Leading, + b"Tf" => Op::Font, + b"Ts" => Op::Rise, + b"Td" => Op::MoveText, + b"TD" => Op::MoveTextLeading, + b"Tm" => Op::TextMatrix, + b"T*" => Op::NextLine, + b"Tj" => Op::Show, + b"TJ" => Op::ShowArray, + b"'" => Op::NextLineShow, + b"\"" => Op::NextLineShowSpaced, + b"Do" => Op::XObject, + b"BI" => Op::BeginImage, + b"BMC" => Op::BeginMarked, + b"BDC" => Op::BeginMarkedProperties, + b"EMC" => Op::EndMarked, + b"b" => Op::Ignored, + b"B" => Op::Ignored, + b"b*" => Op::Ignored, + b"B*" => Op::Ignored, + b"BX" => Op::Ignored, + b"c" => Op::Ignored, + b"CS" => Op::Ignored, + b"cs" => Op::Ignored, + b"d" => Op::Ignored, + b"d0" => Op::Ignored, + b"d1" => Op::Ignored, + b"DP" => Op::Ignored, + b"EX" => Op::Ignored, + b"f" => Op::Ignored, + b"F" => Op::Ignored, + b"f*" => Op::Ignored, + b"G" => Op::Ignored, + b"g" => Op::Ignored, + b"h" => Op::Ignored, + b"i" => Op::Ignored, + b"ID" => Op::Ignored, + b"EI" => Op::Ignored, + b"j" => Op::Ignored, + b"J" => Op::Ignored, + b"K" => Op::Ignored, + b"k" => Op::Ignored, + b"l" => Op::Ignored, + b"m" => Op::Ignored, + b"M" => Op::Ignored, + b"MP" => Op::Ignored, + b"n" => Op::Ignored, + b"re" => Op::Ignored, + b"RG" => Op::Ignored, + b"rg" => Op::Ignored, + b"ri" => Op::Ignored, + b"s" => Op::Ignored, + b"S" => Op::Ignored, + b"SC" => Op::Ignored, + b"sc" => Op::Ignored, + b"SCN" => Op::Ignored, + b"scn" => Op::Ignored, + b"sh" => Op::Ignored, + b"Tr" => Op::Ignored, + b"v" => Op::Ignored, + b"w" => Op::Ignored, + b"W" => Op::Ignored, + b"W*" => Op::Ignored, + b"y" => Op::Ignored, + ) + .copied() + .unwrap_or(Op::Unknown) + } + + pub(crate) fn is_known(keyword: &[u8]) -> bool { + Op::parse(keyword) != Op::Unknown + } +} + +impl<'a> Operand<'a> { + pub(crate) fn from_object(object: Object<'a>) -> Self { + match object { + Object::Int(value) => Operand::Number(value as f64), + Object::Real(value) if value.is_finite() => Operand::Number(value), + Object::Name(name) => Operand::Name(name), + Object::Str(value) => Operand::Str(value), + Object::Array(array) => Operand::Array(array), + Object::Dict(dict) => Operand::Dict(dict), + _ => Operand::Other, + } + } + + pub(crate) fn number(&self) -> Option { + match self { + Operand::Number(value) => Some(*value), + _ => None, + } + } +} + +impl Default for Operands<'_> { + fn default() -> Self { + Operands { + items: [Operand::Other; MAX_OPERANDS], + len: 0, + } + } +} + +impl<'a> Operands<'a> { + #[inline] + pub(crate) fn clear(&mut self) { + self.len = 0; + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn len(&self) -> usize { + self.len + } + + #[inline] + pub(crate) fn push(&mut self, operand: Operand<'a>) { + if self.len == MAX_OPERANDS { + self.items.copy_within(1.., 0); + self.len -= 1; + } + if let Some(slot) = self.items.get_mut(self.len) { + *slot = operand; + self.len += 1; + } + } + + pub(crate) fn last(&self) -> Option<[Operand<'a>; N]> { + let start = self.len.checked_sub(N)?; + let slice = self.items.get(start..self.len)?; + <[Operand<'a>; N]>::try_from(slice).ok() + } + + pub(crate) fn numbers(&self) -> Option<[f64; N]> { + let operands = self.last::()?; + let mut values = [0f64; N]; + for (slot, operand) in values.iter_mut().zip(operands.iter()) { + *slot = operand.number()?; + } + Some(values) + } +} + diff --git a/crates/text-extract/src/pdf/content/resources.rs b/crates/text-extract/src/pdf/content/resources.rs new file mode 100644 index 00000000..9ae78061 --- /dev/null +++ b/crates/text-extract/src/pdf/content/resources.rs @@ -0,0 +1,243 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::{ + document::Document, + font::{FontCache, FontId}, + object::{Dict, Name, ObjRef, Object}, +}; +use std::collections::{HashMap, HashSet}; + +const MAX_CACHED_NAME: usize = 32; +const MAX_ENTRIES: usize = 1 << 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum Category { + Font, + XObject, + State, + Properties, +} + +type Location = (usize, usize); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct Key { + resources: Location, + category: Category, + name: [u8; MAX_CACHED_NAME], + len: u8, +} + +#[derive(Debug, Clone, Copy)] +enum Cached { + Reference(ObjRef), + Font(Option), + XObject(Option), + State(Option<(FontId, f64)>), + Plain, +} + +pub(crate) struct Lookup<'c, 'd, 'a> { + pub(crate) doc: &'d Document<'a>, + pub(crate) fonts: &'c mut FontCache, + pub(crate) buf: &'c mut Vec, +} + +#[derive(Debug, Default)] +pub(crate) struct ResourceCache { + map: HashMap, + indexed: HashSet<(Location, Category)>, +} + +impl Category { + fn key(self) -> &'static [u8] { + match self { + Category::Font => b"Font", + Category::XObject => b"XObject", + Category::State => b"ExtGState", + Category::Properties => b"Properties", + } + } +} + +fn location(resources: Option>) -> Location { + resources.map_or((0, 0), |dict| { + (dict.body().as_ptr() as usize, dict.body().len()) + }) +} + +impl Key { + fn new(resources: Location, category: Category, name: &[u8]) -> Option { + let len = u8::try_from(name.len()) + .ok() + .filter(|&len| usize::from(len) <= MAX_CACHED_NAME)?; + let mut stored = [0u8; MAX_CACHED_NAME]; + stored.get_mut(..name.len())?.copy_from_slice(name); + Some(Key { + resources, + category, + name: stored, + len, + }) + } +} + +impl ResourceCache { + pub(crate) fn clear(&mut self) { + self.map.clear(); + self.indexed.clear(); + } + + pub(crate) fn shrink(&mut self) { + self.map = HashMap::new(); + self.indexed = HashSet::new(); + } + + fn store(&mut self, key: Option, value: Cached) { + if let Some(key) = key { + if self.map.len() >= MAX_ENTRIES { + self.clear(); + } + self.map.insert(key, value); + } + } + + fn index(&mut self, doc: &Document<'_>, resources: Option>, category: Category) { + let place = location(resources); + if !self.indexed.insert((place, category)) { + return; + } + let Some(group) = resources.and_then(|resources| doc.get_dict(resources, category.key())) + else { + return; + }; + for (name, value) in group.iter() { + if self.map.len() >= MAX_ENTRIES { + return; + } + if let Object::Ref(id) = value + && let Some(key) = Key::new(place, category, &name.decoded()) + { + self.map.insert(key, Cached::Reference(id)); + } + } + } + + fn find<'d>( + &mut self, + doc: &'d Document<'_>, + resources: Option>, + category: Category, + name: &[u8], + ) -> (Option, Option, Option>) { + let key = Key::new(location(resources), category, name); + if let Some(key) = key { + if let Some(&cached) = self.map.get(&key) { + return (Some(key), Some(cached), None); + } + self.index(doc, resources, category); + if let Some(&cached) = self.map.get(&key) { + return (Some(key), Some(cached), None); + } + } + (key, None, entry(doc, resources, category.key(), name)) + } + + pub(crate) fn font( + &mut self, + lookup: Lookup<'_, '_, '_>, + resources: Option>, + name: Name<'_>, + ) -> Option { + let decoded = name.decoded(); + let (key, cached, direct) = self.find(lookup.doc, resources, Category::Font, &decoded); + let value = match cached { + Some(Cached::Font(font)) => return font, + Some(Cached::Reference(id)) => Some(Object::Ref(id)), + _ => direct, + }; + let font = value.and_then(|value| lookup.fonts.load(lookup.doc, value, lookup.buf)); + self.store(key, Cached::Font(font)); + font + } + + pub(crate) fn xobject( + &mut self, + doc: &Document<'_>, + resources: Option>, + name: Name<'_>, + ) -> Option { + let decoded = name.decoded(); + let (key, cached, direct) = self.find(doc, resources, Category::XObject, &decoded); + let id = match cached { + Some(Cached::XObject(id)) => return id, + Some(Cached::Reference(id)) => Some(id), + _ => direct.and_then(|value| value.as_ref()), + }; + self.store(key, Cached::XObject(id)); + id + } + + pub(crate) fn state_font( + &mut self, + lookup: Lookup<'_, '_, '_>, + resources: Option>, + name: Name<'_>, + ) -> Option<(FontId, f64)> { + let decoded = name.decoded(); + let doc = lookup.doc; + let (key, cached, direct) = self.find(doc, resources, Category::State, &decoded); + let value = match cached { + Some(Cached::State(font)) => return font, + Some(Cached::Reference(id)) => Some(Object::Ref(id)), + _ => direct, + }; + let font = value + .and_then(|value| doc.resolve(value).as_dict()) + .and_then(|state| doc.get_array(state, b"Font")) + .and_then(|pair| { + let mut items = pair.iter(); + let (font, size) = (items.next()?, doc.resolve(items.next()?).as_f64()?); + Some((lookup.fonts.load(doc, font, lookup.buf)?, size)) + }); + self.store(key, Cached::State(font)); + font + } + + pub(crate) fn actual_text<'d>( + &mut self, + doc: &'d Document<'_>, + resources: Option>, + name: Name<'_>, + ) -> Option> { + let decoded = name.decoded(); + let (key, cached, direct) = self.find(doc, resources, Category::Properties, &decoded); + let value = match cached { + Some(Cached::Plain) => return None, + Some(Cached::Reference(id)) => Some(Object::Ref(id)), + _ => direct, + }; + let text = value + .and_then(|value| doc.resolve(value).as_dict()) + .map(|dict| doc.dict_get(dict, b"ActualText")) + .filter(|text| !text.is_null()); + if text.is_none() { + self.store(key, Cached::Plain); + } + text + } +} + +pub(crate) fn entry<'d>( + doc: &'d Document<'_>, + resources: Option>, + category: &[u8], + name: &[u8], +) -> Option> { + let group = doc.get_dict(resources?, category)?; + group.get(name) +} diff --git a/crates/text-extract/src/pdf/content/state.rs b/crates/text-extract/src/pdf/content/state.rs new file mode 100644 index 00000000..e980e87e --- /dev/null +++ b/crates/text-extract/src/pdf/content/state.rs @@ -0,0 +1,213 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::font::FontId; +use std::hash::{DefaultHasher, Hash, Hasher}; + +pub(crate) const MAX_SAVED_STATES: usize = 256; +const MAX_COORDINATE: f64 = 1e12; +const PERCENT: f64 = 100.0; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Matrix { + pub(crate) a: f64, + pub(crate) b: f64, + pub(crate) c: f64, + pub(crate) d: f64, + pub(crate) e: f64, + pub(crate) f: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct TextState { + pub(crate) char_spacing: f64, + pub(crate) word_spacing: f64, + pub(crate) scale: f64, + pub(crate) leading: f64, + pub(crate) rise: f64, + pub(crate) font: FontId, + pub(crate) size: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct GraphicsState { + pub(crate) ctm: Matrix, + pub(crate) text: TextState, +} + +#[derive(Debug, Default)] +pub(crate) struct StateStack { + saved: Vec, + overflow: usize, + floor: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StackMark { + saved: usize, + overflow: usize, + floor: usize, +} + +pub(crate) fn finite(value: f64) -> f64 { + if value.is_finite() { + value.clamp(-MAX_COORDINATE, MAX_COORDINATE) + } else { + 0.0 + } +} + +impl Matrix { + pub(crate) const IDENTITY: Matrix = Matrix { + a: 1.0, + b: 0.0, + c: 0.0, + d: 1.0, + e: 0.0, + f: 0.0, + }; + + pub(crate) fn new(values: [f64; 6]) -> Self { + let [a, b, c, d, e, f] = values.map(finite); + Matrix { a, b, c, d, e, f } + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn translation(x: f64, y: f64) -> Self { + Matrix { + e: finite(x), + f: finite(y), + ..Matrix::IDENTITY + } + } + + pub(crate) fn then(&self, next: &Matrix) -> Matrix { + Matrix { + a: finite(self.a * next.a + self.b * next.c), + b: finite(self.a * next.b + self.b * next.d), + c: finite(self.c * next.a + self.d * next.c), + d: finite(self.c * next.b + self.d * next.d), + e: finite(self.e * next.a + self.f * next.c + next.e), + f: finite(self.e * next.b + self.f * next.d + next.f), + } + } + + #[inline] + pub(crate) fn translate(&mut self, x: f64, y: f64) { + self.e = finite(self.e + x * self.a + y * self.c); + self.f = finite(self.f + x * self.b + y * self.d); + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn apply(&self, x: f64, y: f64) -> (f64, f64) { + ( + x * self.a + y * self.c + self.e, + x * self.b + y * self.d + self.f, + ) + } +} + +impl Default for TextState { + fn default() -> Self { + TextState { + char_spacing: 0.0, + word_spacing: 0.0, + scale: 1.0, + leading: 0.0, + rise: 0.0, + font: FontId::STANDARD, + size: 0.0, + } + } +} + +impl TextState { + pub(crate) fn set_scale(&mut self, percent: f64) { + self.scale = finite(percent) / PERCENT; + } +} + +impl GraphicsState { + pub(crate) fn fingerprint(&self, text: &Matrix, line: &Matrix) -> u64 { + let mut hasher = DefaultHasher::new(); + for matrix in [&self.ctm, text, line] { + [matrix.a, matrix.b, matrix.c, matrix.d, matrix.e, matrix.f] + .map(f64::to_bits) + .hash(&mut hasher); + } + let state = &self.text; + [ + state.char_spacing, + state.word_spacing, + state.scale, + state.leading, + state.rise, + state.size, + ] + .map(f64::to_bits) + .hash(&mut hasher); + state.font.hash(&mut hasher); + hasher.finish() + } +} + +impl Default for GraphicsState { + fn default() -> Self { + GraphicsState { + ctm: Matrix::IDENTITY, + text: TextState::default(), + } + } +} + +impl StateStack { + pub(crate) fn clear(&mut self) { + self.saved.clear(); + self.overflow = 0; + self.floor = 0; + } + + pub(crate) fn shrink(&mut self) { + self.clear(); + self.saved.shrink_to(MAX_SAVED_STATES); + } + + pub(crate) fn save(&mut self, state: &GraphicsState) { + if self.saved.len() < MAX_SAVED_STATES { + self.saved.push(*state); + } else { + self.overflow = self.overflow.saturating_add(1); + } + } + + pub(crate) fn restore(&mut self, state: &mut GraphicsState) { + if self.overflow > 0 { + self.overflow -= 1; + } else if self.saved.len() > self.floor + && let Some(saved) = self.saved.pop() + { + *state = saved; + } + } + + pub(crate) fn enter(&mut self) -> StackMark { + let mark = StackMark { + saved: self.saved.len(), + overflow: self.overflow, + floor: self.floor, + }; + self.floor = self.saved.len(); + self.overflow = 0; + mark + } + + pub(crate) fn leave(&mut self, mark: StackMark) { + self.saved.truncate(mark.saved); + self.overflow = mark.overflow; + self.floor = mark.floor; + } +} + diff --git a/crates/text-extract/src/pdf/content/text.rs b/crates/text-extract/src/pdf/content/text.rs new file mode 100644 index 00000000..ca396245 --- /dev/null +++ b/crates/text-extract/src/pdf/content/text.rs @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Interpreter, state::finite}; +use crate::{ + output::Output, + pdf::{ + layout::{Placement, space_threshold}, + object::{Array, Object, Str}, + }, +}; + +const THOUSANDTHS: f64 = 1000.0; +const MIN_LENGTH: f64 = 1e-9; +const FALLBACK_DIRECTION: (f64, f64) = (1.0, 0.0); +const DEGENERATE_SIZE: f64 = 1.0; + +fn normalize(vector: (f64, f64)) -> Option<(f64, f64)> { + let length = vector.0.hypot(vector.1); + (length > MIN_LENGTH && length.is_finite()).then(|| (vector.0 / length, vector.1 / length)) +} + +impl Interpreter { + pub(super) fn show(&mut self, bytes: &[u8], out: &mut Output<'_>) { + let Interpreter { + fonts, + state, + text_matrix, + glyph_text, + layout, + continued, + .. + } = self; + let text = state.text; + let font = fonts.font(text.font); + let metrics = font.metrics(); + let vertical = font.is_vertical(); + let size = if text.size.abs() > MIN_LENGTH { + text.size + } else { + DEGENERATE_SIZE + }; + let scale = text.scale; + let mut matrix = text_matrix.then(&state.ctm); + let x_axis = (matrix.a * size * scale, matrix.b * size * scale); + let y_axis = (matrix.c * size, matrix.d * size); + let em_scale = f64::from(metrics.em_scale); + let (dir, em) = if vertical { + ( + normalize((-y_axis.0, -y_axis.1)), + (matrix.a * size).hypot(matrix.b * size) * em_scale, + ) + } else { + (normalize(x_axis), y_axis.0.hypot(y_axis.1) * em_scale) + }; + let dir = dir.unwrap_or(FALLBACK_DIRECTION); + let em = if em.is_finite() && em > MIN_LENGTH { + em + } else { + 1.0 + }; + let threshold = space_threshold(metrics.space_width); + let spacing_em = finite(text.char_spacing * scale / size.abs()); + let mut rest = bytes; + while !rest.is_empty() { + let Some((glyph, glyph_str)) = font.glyph(rest, glyph_text) else { + break; + }; + let width = f64::from(glyph.width); + let spacing = text.char_spacing + + if glyph.space_code { + text.word_spacing + } else { + 0.0 + }; + let origin = ( + matrix.e + text.rise * matrix.c, + matrix.f + text.rise * matrix.d, + ); + let end = if vertical { + (origin.0 + width * y_axis.0, origin.1 + width * y_axis.1) + } else { + (origin.0 + width * x_axis.0, origin.1 + width * x_axis.1) + }; + layout.glyph( + out, + glyph_str, + &Placement { + origin, + end, + dir, + em, + space_threshold: threshold, + spacing: spacing_em, + continued: *continued, + vertical, + font: text.font, + }, + ); + *continued = true; + if vertical { + let advance = finite(width * size + spacing); + matrix.translate(0.0, advance); + text_matrix.translate(0.0, advance); + } else { + let advance = finite((width * size + spacing) * scale); + matrix.translate(advance, 0.0); + text_matrix.translate(advance, 0.0); + } + rest = rest.get(glyph.len.max(1)..).unwrap_or_default(); + } + } + + pub(super) fn show_str(&mut self, value: Str<'_>, out: &mut Output<'_>) { + if let Some(raw) = value.plain() { + self.show(raw, out); + return; + } + let mut buf = std::mem::take(&mut self.string_buf); + buf.clear(); + value.decode_into(&mut buf); + self.show(&buf, out); + self.string_buf = buf; + } + + pub(super) fn show_array(&mut self, array: Array<'_>, out: &mut Output<'_>) { + for item in array.iter() { + if out.is_full() { + return; + } + match item { + Object::Str(value) => self.show_str(value, out), + other => { + let Some(adjust) = other.as_f64() else { + continue; + }; + let text = self.state.text; + let shift = finite(-adjust / THOUSANDTHS * text.size); + if self.fonts.font(text.font).is_vertical() { + self.text_matrix.translate(0.0, shift); + } else { + self.text_matrix.translate(finite(shift * text.scale), 0.0); + } + } + } + } + } +} diff --git a/crates/text-extract/src/pdf/crypt/aes.rs b/crates/text-extract/src/pdf/crypt/aes.rs new file mode 100644 index 00000000..677d8537 --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/aes.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use aws_lc_rs::cipher::{ + AES_128, AES_256, Algorithm, DecryptingKey, DecryptionContext, EncryptingKey, + EncryptionContext, UnboundCipherKey, +}; +use aws_lc_rs::iv::FixedLength; + +pub(crate) const BLOCK_LEN: usize = 16; + +#[derive(Clone, Copy)] +pub(crate) enum AesKind { + Aes128, + Aes256, +} + +impl AesKind { + fn algorithm(self) -> &'static Algorithm { + match self { + AesKind::Aes128 => &AES_128, + AesKind::Aes256 => &AES_256, + } + } + + pub(crate) fn decrypting_key(self, key: &[u8]) -> Option { + UnboundCipherKey::new(self.algorithm(), key) + .and_then(DecryptingKey::cbc) + .ok() + } +} + +pub(crate) fn decrypt_with_iv(cipher: &DecryptingKey, data: &[u8], out: &mut Vec) { + let Some((iv, body)) = data.split_first_chunk::() else { + return; + }; + let whole_blocks = body.len() - body.len() % BLOCK_LEN; + let Some(body) = body.get(..whole_blocks).filter(|body| !body.is_empty()) else { + return; + }; + let start = out.len(); + out.extend_from_slice(body); + let plain_len = out + .get_mut(start..) + .and_then(|buf| { + cipher + .decrypt(buf, DecryptionContext::Iv128(FixedLength::from(iv))) + .ok() + }) + .map(|plain| plain.len() - valid_padding(plain)); + match plain_len { + Some(len) => out.truncate(start + len), + None => out.truncate(start), + } +} + +fn valid_padding(plain: &[u8]) -> usize { + let Some(&last) = plain.last() else { + return 0; + }; + let pad = usize::from(last); + if (1..=BLOCK_LEN).contains(&pad) + && plain.len() >= pad + && plain.iter().rev().take(pad).all(|byte| *byte == last) + { + pad + } else { + 0 + } +} + +pub(crate) fn decrypt_zero_iv(kind: AesKind, key: &[u8], data: &mut [u8]) -> Option<()> { + kind.decrypting_key(key)? + .decrypt( + data, + DecryptionContext::Iv128(FixedLength::from([0u8; BLOCK_LEN])), + ) + .ok() + .map(|_| ()) +} + +pub(crate) fn encrypt_aes128_cbc(key: &[u8], iv: &[u8; BLOCK_LEN], data: &mut [u8]) -> Option<()> { + UnboundCipherKey::new(&AES_128, key) + .and_then(EncryptingKey::cbc) + .and_then(|cipher| { + cipher.less_safe_encrypt(data, EncryptionContext::Iv128(FixedLength::from(iv))) + }) + .ok() + .map(|_| ()) +} diff --git a/crates/text-extract/src/pdf/crypt/derive.rs b/crates/text-extract/src/pdf/crypt/derive.rs new file mode 100644 index 00000000..0e00f9b1 --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/derive.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::key::FileKey; +use super::legacy::{Legacy, OwnerHash, PADDED_LEN}; +use super::modern::{ENTRY_LEN, Modern}; +use super::{CryptError, EncryptDict}; + +const DEFAULT_KEY_LEN: usize = 5; +const FULL_KEY_LEN: usize = 16; +const MIN_KEY_BITS: i64 = 40; +const MAX_KEY_BITS: i64 = 128; +const MIN_KEY_BYTES: i64 = 5; +const MAX_KEY_BYTES: i64 = 16; +const MIN_U_LEN_R2: usize = 32; +const MIN_U_LEN_R3: usize = 16; +const R5: i64 = 5; +const R6: i64 = 6; + +pub(crate) fn legacy_key(dict: &EncryptDict<'_>, password: &[u8]) -> Result { + let min_u_len = if dict.r == 2 { + MIN_U_LEN_R2 + } else { + MIN_U_LEN_R3 + }; + let o = dict.o.get(..PADDED_LEN).ok_or(CryptError::Malformed)?; + if dict.u.len() < min_u_len { + return Err(CryptError::Malformed); + } + let base = Legacy { + revision: dict.r, + o, + u: dict.u, + p: dict.p as u32, + id0: dict.id0, + encrypt_metadata: dict.encrypt_metadata, + }; + let metadata_flags = [ + Some(dict.encrypt_metadata), + (dict.r >= 4).then_some(!dict.encrypt_metadata), + ]; + let lengths = KeyLengths::new(dict); + let variants = || { + lengths.iter().flat_map(move |key_len| { + metadata_flags + .into_iter() + .flatten() + .map(move |encrypt_metadata| { + ( + key_len, + Legacy { + encrypt_metadata, + ..base + }, + ) + }) + }) + }; + variants() + .find_map(|(key_len, legacy)| legacy.user_key(password, key_len)) + .or_else(|| { + variants().find_map(|(key_len, legacy)| { + legacy + .owner_key(password, key_len, OwnerHash::KeyLength) + .or_else(|| { + (key_len < FULL_KEY_LEN) + .then(|| legacy.owner_key(password, key_len, OwnerHash::FullDigest)) + .flatten() + }) + }) + }) + .ok_or(if lengths.has_invalid { + CryptError::Malformed + } else { + CryptError::PasswordRequired + }) +} + +pub(crate) fn modern_key(dict: &EncryptDict<'_>, password: &[u8]) -> Result { + let o = dict + .o + .first_chunk::() + .ok_or(CryptError::Malformed)?; + let u = dict + .u + .first_chunk::() + .ok_or(CryptError::Malformed)?; + let alternate = if dict.r == R6 { R5 } else { R6 }; + let mut scratch = Vec::new(); + for revision in [dict.r, alternate] { + let modern = Modern { + revision, + o, + u, + oe: dict.oe, + ue: dict.ue, + }; + if let Some(key) = modern.user_key(password, &mut scratch)? { + return Ok(key); + } + if let Some(key) = modern.owner_key(password, &mut scratch)? { + return Ok(key); + } + } + Err(CryptError::PasswordRequired) +} + +struct KeyLengths { + candidates: [Option; 3], + has_invalid: bool, +} + +impl KeyLengths { + fn new(dict: &EncryptDict<'_>) -> Self { + if dict.v == 1 { + return KeyLengths { + candidates: [Some(DEFAULT_KEY_LEN), None, None], + has_invalid: false, + }; + } + let length = dict.length.and_then(key_len_from_entry); + let filter_length = dict.crypt_filter_length.and_then(key_len_from_entry); + let declared = if dict.v >= 4 { + filter_length.or(length) + } else { + length.or(filter_length) + }; + let has_invalid = + declared.is_none() && (dict.length.is_some() || dict.crypt_filter_length.is_some()); + KeyLengths { + candidates: [declared, Some(DEFAULT_KEY_LEN), Some(FULL_KEY_LEN)], + has_invalid, + } + } + + fn iter(&self) -> impl Iterator + '_ { + self.candidates + .iter() + .enumerate() + .filter_map(move |(pos, candidate)| { + let len = (*candidate)?; + (!self + .candidates + .iter() + .take(pos) + .any(|seen| *seen == Some(len))) + .then_some(len) + }) + } +} + +fn key_len_from_entry(value: i64) -> Option { + let bytes = if (MIN_KEY_BITS..=MAX_KEY_BITS).contains(&value) && value % 8 == 0 { + value / 8 + } else if (MIN_KEY_BYTES..=MAX_KEY_BYTES).contains(&value) { + value + } else { + return None; + }; + usize::try_from(bytes).ok() +} diff --git a/crates/text-extract/src/pdf/crypt/key.rs b/crates/text-extract/src/pdf/crypt/key.rs new file mode 100644 index 00000000..14c12ad5 --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/key.rs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) const MAX_KEY_LEN: usize = 32; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct FileKey { + bytes: [u8; MAX_KEY_LEN], + len: usize, +} + +impl FileKey { + pub(crate) fn from_slice(key: &[u8]) -> Option { + let mut bytes = [0u8; MAX_KEY_LEN]; + bytes.get_mut(..key.len())?.copy_from_slice(key); + Some(FileKey { + bytes, + len: key.len(), + }) + } + + pub(crate) fn as_slice(&self) -> &[u8] { + self.bytes.get(..self.len).unwrap_or(&self.bytes) + } + + pub(crate) fn len(&self) -> usize { + self.len + } + + pub(crate) fn zero_extend(&mut self, len: usize) { + if (self.len..=MAX_KEY_LEN).contains(&len) { + self.len = len; + } + } + + pub(crate) fn xor(&self, value: u8) -> Self { + let mut derived = *self; + for byte in derived.bytes.iter_mut().take(self.len) { + *byte ^= value; + } + derived + } +} diff --git a/crates/text-extract/src/pdf/crypt/legacy.rs b/crates/text-extract/src/pdf/crypt/legacy.rs new file mode 100644 index 00000000..55caa792 --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/legacy.rs @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::key::FileKey; +use super::rc4::Rc4; + +pub(crate) const PADDED_LEN: usize = 32; +const PASSWORD_PAD: [u8; PADDED_LEN] = [ + 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08, + 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a, +]; +const U_CHECK_LEN: usize = 16; +const MD5_ROUNDS: usize = 50; +const RC4_ROUNDS: u8 = 20; +const METADATA_MARKER: [u8; 4] = [0xff; 4]; + +#[derive(Clone, Copy)] +pub(crate) enum OwnerHash { + KeyLength, + FullDigest, +} + +#[derive(Clone, Copy)] +pub(crate) struct Legacy<'a> { + pub(crate) revision: i64, + pub(crate) o: &'a [u8], + pub(crate) u: &'a [u8], + pub(crate) p: u32, + pub(crate) id0: &'a [u8], + pub(crate) encrypt_metadata: bool, +} + +impl Legacy<'_> { + pub(crate) fn user_key(&self, password: &[u8], key_len: usize) -> Option { + let key = self.file_key(&pad_password(password), key_len)?; + self.matches_u(&key).then_some(key) + } + + pub(crate) fn owner_key( + &self, + password: &[u8], + key_len: usize, + owner_hash: OwnerHash, + ) -> Option { + let mut hash = md5::compute(pad_password(password)).0; + if self.revision >= 3 { + let rehashed_len = match owner_hash { + OwnerHash::KeyLength => key_len, + OwnerHash::FullDigest => hash.len(), + }; + for _ in 0..MD5_ROUNDS { + hash = md5::compute(hash.get(..rehashed_len)?).0; + } + } + let owner_key = FileKey::from_slice(hash.get(..key_len)?)?; + let mut user_password = pad_password(self.o); + if self.revision >= 3 { + for round in (0..RC4_ROUNDS).rev() { + Rc4::new(owner_key.xor(round).as_slice()).apply_in_place(&mut user_password); + } + } else { + Rc4::new(owner_key.as_slice()).apply_in_place(&mut user_password); + } + self.user_key(&user_password, key_len) + } + + fn file_key(&self, padded_password: &[u8; PADDED_LEN], key_len: usize) -> Option { + let mut context = md5::Context::new(); + context.consume(padded_password); + context.consume(self.o); + context.consume(self.p.to_le_bytes()); + context.consume(self.id0); + if self.revision >= 4 && !self.encrypt_metadata { + context.consume(METADATA_MARKER); + } + let mut hash = context.finalize().0; + if self.revision >= 3 { + for _ in 0..MD5_ROUNDS { + hash = md5::compute(hash.get(..key_len)?).0; + } + } + FileKey::from_slice(hash.get(..key_len)?) + } + + fn matches_u(&self, key: &FileKey) -> bool { + if self.revision >= 3 { + let mut context = md5::Context::new(); + context.consume(PASSWORD_PAD); + context.consume(self.id0); + let mut check = context.finalize().0; + Rc4::new(key.as_slice()).apply_in_place(&mut check); + for round in 1..RC4_ROUNDS { + Rc4::new(key.xor(round).as_slice()).apply_in_place(&mut check); + } + self.u.get(..U_CHECK_LEN) == Some(check.as_slice()) + } else { + let mut check = PASSWORD_PAD; + Rc4::new(key.as_slice()).apply_in_place(&mut check); + self.u.get(..PADDED_LEN) == Some(check.as_slice()) + } + } +} + +fn pad_password(password: &[u8]) -> [u8; PADDED_LEN] { + let mut padded = [0u8; PADDED_LEN]; + for (slot, byte) in padded + .iter_mut() + .zip(password.iter().chain(PASSWORD_PAD.iter())) + { + *slot = *byte; + } + padded +} diff --git a/crates/text-extract/src/pdf/crypt/mod.rs b/crates/text-extract/src/pdf/crypt/mod.rs new file mode 100644 index 00000000..a2374b8a --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/mod.rs @@ -0,0 +1,207 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod aes; +mod derive; +mod key; +mod legacy; +mod modern; +mod rc4; + + +use aws_lc_rs::cipher::DecryptingKey; + +use self::aes::AesKind; +use self::key::FileKey; +use self::rc4::Rc4; + +const STANDARD_FILTER: &[u8] = b"Standard"; +const AES_SALT: &[u8] = b"sAlT"; +const AES128_KEY_LEN: usize = 16; +const OBJECT_KEY_MAX_LEN: usize = 16; +const OBJECT_KEY_EXTRA: usize = 5; +const UNPUBLISHED_V3: i64 = 3; +const MODERN_V: i64 = 5; +const PADDED_KEY_V: i64 = 4; +const PADDED_KEY_R: i64 = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CryptMethod { + Identity, + Rc4, + AesV2, + AesV3, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Target { + String, + Stream, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CryptError { + UnsupportedHandler, + PasswordRequired, + Malformed, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct EncryptDict<'a> { + pub(crate) filter: &'a [u8], + pub(crate) v: i64, + pub(crate) r: i64, + pub(crate) length: Option, + pub(crate) o: &'a [u8], + pub(crate) u: &'a [u8], + pub(crate) oe: &'a [u8], + pub(crate) ue: &'a [u8], + pub(crate) p: i64, + pub(crate) encrypt_metadata: bool, + pub(crate) stream_method: Option, + pub(crate) string_method: Option, + pub(crate) crypt_filter_length: Option, + pub(crate) id0: &'a [u8], +} + +#[derive(Debug)] +pub(crate) struct Decryptor { + key: FileKey, + aes256: Option, + stream_method: CryptMethod, + string_method: CryptMethod, + #[cfg(all(test, stalwart_tests))] + encrypt_metadata: bool, +} + +impl Decryptor { + pub(crate) fn new(dict: &EncryptDict<'_>) -> Result { + Self::with_password(dict, &[]) + } + + pub(crate) fn with_password( + dict: &EncryptDict<'_>, + password: &[u8], + ) -> Result { + if dict.filter != STANDARD_FILTER || dict.v == UNPUBLISHED_V3 { + return Err(CryptError::UnsupportedHandler); + } + let version = match (dict.v, dict.r) { + (1 | 2 | 4, 2..=4) => dict.v, + (1 | 2 | 4 | 5, 5 | 6) => MODERN_V, + _ => return Err(CryptError::Malformed), + }; + let (string_method, stream_method) = resolve_methods(version, dict)?; + if string_method == CryptMethod::Identity && stream_method == CryptMethod::Identity { + return Ok(Decryptor { + key: FileKey::default(), + aes256: None, + stream_method, + string_method, + #[cfg(all(test, stalwart_tests))] + encrypt_metadata: dict.encrypt_metadata, + }); + } + let mut key = if version == MODERN_V { + derive::modern_key(dict, password)? + } else { + derive::legacy_key(dict, password)? + }; + if dict.v == PADDED_KEY_V || dict.r == PADDED_KEY_R { + key.zero_extend(AES128_KEY_LEN); + } + let aes256 = (string_method == CryptMethod::AesV3 || stream_method == CryptMethod::AesV3) + .then(|| AesKind::Aes256.decrypting_key(key.as_slice())) + .flatten(); + Ok(Decryptor { + key, + aes256, + stream_method, + string_method, + #[cfg(all(test, stalwart_tests))] + encrypt_metadata: dict.encrypt_metadata || version < PADDED_KEY_V, + }) + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn encrypts_metadata(&self) -> bool { + self.encrypt_metadata + } + + pub(crate) fn method(&self, target: Target) -> CryptMethod { + match target { + Target::String => self.string_method, + Target::Stream => self.stream_method, + } + } + + pub(crate) fn decrypt( + &self, + target: Target, + object: u32, + generation: u16, + data: &[u8], + out: &mut Vec, + ) { + match self.method(target) { + CryptMethod::Identity => out.extend_from_slice(data), + CryptMethod::Rc4 => { + let key = self.object_key(object, generation, false); + Rc4::new(key.as_slice()).apply_into(data, out); + } + CryptMethod::AesV2 => { + let key = self.object_key(object, generation, true); + if let Some(cipher) = AesKind::Aes128.decrypting_key(key.as_slice()) { + aes::decrypt_with_iv(&cipher, data, out); + } + } + CryptMethod::AesV3 => { + if let Some(cipher) = &self.aes256 { + aes::decrypt_with_iv(cipher, data, out); + } + } + } + } + + fn object_key(&self, object: u32, generation: u16, aes: bool) -> FileKey { + let mut context = md5::Context::new(); + context.consume(self.key.as_slice()); + context.consume(object.to_le_bytes().get(..3).unwrap_or_default()); + context.consume(generation.to_le_bytes()); + if aes { + context.consume(AES_SALT); + } + let hash = context.finalize().0; + let len = (self.key.len() + OBJECT_KEY_EXTRA).min(OBJECT_KEY_MAX_LEN); + hash.get(..len) + .and_then(FileKey::from_slice) + .unwrap_or_default() + } +} + +fn resolve_methods( + version: i64, + dict: &EncryptDict<'_>, +) -> Result<(CryptMethod, CryptMethod), CryptError> { + match version { + 1 | 2 => Ok((CryptMethod::Rc4, CryptMethod::Rc4)), + 4 => { + let resolve = |method: Option| match method { + Some(CryptMethod::AesV3) => Err(CryptError::Malformed), + method => Ok(method.unwrap_or(CryptMethod::Identity)), + }; + Ok((resolve(dict.string_method)?, resolve(dict.stream_method)?)) + } + 5 => { + let resolve = |method: Option| match method { + None | Some(CryptMethod::Identity) => CryptMethod::Identity, + Some(_) => CryptMethod::AesV3, + }; + Ok((resolve(dict.string_method), resolve(dict.stream_method))) + } + _ => Err(CryptError::Malformed), + } +} diff --git a/crates/text-extract/src/pdf/crypt/modern.rs b/crates/text-extract/src/pdf/crypt/modern.rs new file mode 100644 index 00000000..a6aac2ff --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/modern.rs @@ -0,0 +1,156 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use aws_lc_rs::digest::{self, Algorithm, Context, SHA256, SHA384, SHA512}; + +use super::CryptError; +use super::aes::{self, AesKind, BLOCK_LEN}; +use super::key::{FileKey, MAX_KEY_LEN}; + +pub(crate) const ENTRY_LEN: usize = 48; +pub(crate) const MAX_PASSWORD_LEN: usize = 127; +const HASH_LEN: usize = 32; +const SALT_LEN: usize = 8; +const MAX_DIGEST_LEN: usize = 64; +const R6_REPEAT: usize = 64; +const R6_MIN_ROUNDS: u32 = 64; +const R6_ROUND_SLACK: u32 = 32; + +pub(crate) struct Modern<'a> { + pub(crate) revision: i64, + pub(crate) o: &'a [u8; ENTRY_LEN], + pub(crate) u: &'a [u8; ENTRY_LEN], + pub(crate) oe: &'a [u8], + pub(crate) ue: &'a [u8], +} + +struct Entry<'a> { + hash: &'a [u8; HASH_LEN], + validation_salt: &'a [u8; SALT_LEN], + key_salt: &'a [u8; SALT_LEN], +} + +impl<'a> Entry<'a> { + fn split(entry: &'a [u8; ENTRY_LEN]) -> Option { + let (hash, salts) = entry.split_first_chunk::()?; + let (validation_salt, key_salt) = salts.split_first_chunk::()?; + Some(Entry { + hash, + validation_salt, + key_salt: key_salt.first_chunk::()?, + }) + } +} + +impl Modern<'_> { + pub(crate) fn user_key( + &self, + password: &[u8], + scratch: &mut Vec, + ) -> Result, CryptError> { + let password = password.get(..MAX_PASSWORD_LEN).unwrap_or(password); + let entry = Entry::split(self.u).ok_or(CryptError::Malformed)?; + if self.hash(password, entry.validation_salt, &[], scratch)? != *entry.hash { + return Ok(None); + } + let intermediate = self.hash(password, entry.key_salt, &[], scratch)?; + unwrap_file_key(&intermediate, self.ue).map(Some) + } + + pub(crate) fn owner_key( + &self, + password: &[u8], + scratch: &mut Vec, + ) -> Result, CryptError> { + let password = password.get(..MAX_PASSWORD_LEN).unwrap_or(password); + let entry = Entry::split(self.o).ok_or(CryptError::Malformed)?; + if self.hash(password, entry.validation_salt, self.u, scratch)? != *entry.hash { + return Ok(None); + } + let intermediate = self.hash(password, entry.key_salt, self.u, scratch)?; + unwrap_file_key(&intermediate, self.oe).map(Some) + } + + fn hash( + &self, + password: &[u8], + salt: &[u8; SALT_LEN], + user_entry: &[u8], + scratch: &mut Vec, + ) -> Result<[u8; HASH_LEN], CryptError> { + let initial = concat_digest(&SHA256, password, salt, user_entry); + let initial = initial + .as_ref() + .first_chunk::() + .ok_or(CryptError::Malformed)?; + if self.revision >= 6 { + hardened_hash(password, initial, user_entry, scratch).ok_or(CryptError::Malformed) + } else { + Ok(*initial) + } + } +} + +fn concat_digest(algorithm: &'static Algorithm, a: &[u8], b: &[u8], c: &[u8]) -> digest::Digest { + let mut context = Context::new(algorithm); + context.update(a); + context.update(b); + context.update(c); + context.finish() +} + +fn hardened_hash( + password: &[u8], + initial: &[u8; HASH_LEN], + user_entry: &[u8], + scratch: &mut Vec, +) -> Option<[u8; HASH_LEN]> { + let mut k = [0u8; MAX_DIGEST_LEN]; + k.get_mut(..HASH_LEN)?.copy_from_slice(initial); + let mut k_len = HASH_LEN; + scratch.clear(); + scratch.reserve(R6_REPEAT * (password.len() + MAX_DIGEST_LEN + user_entry.len())); + let mut round = 0u32; + loop { + let current = k.get(..k_len)?; + scratch.clear(); + for _ in 0..R6_REPEAT { + scratch.extend_from_slice(password); + scratch.extend_from_slice(current); + scratch.extend_from_slice(user_entry); + } + let (aes_key, rest) = current.split_first_chunk::()?; + aes::encrypt_aes128_cbc(aes_key, rest.first_chunk::()?, scratch)?; + let selector = scratch + .iter() + .take(BLOCK_LEN) + .map(|byte| u32::from(*byte)) + .sum::() + % 3; + let algorithm = match selector { + 0 => &SHA256, + 1 => &SHA384, + _ => &SHA512, + }; + let next = digest::digest(algorithm, scratch); + k_len = next.as_ref().len(); + k.get_mut(..k_len)?.copy_from_slice(next.as_ref()); + round += 1; + let last = u32::from(*scratch.last()?); + if round >= R6_MIN_ROUNDS && last + R6_ROUND_SLACK <= round { + break; + } + } + k.first_chunk::().copied() +} + +fn unwrap_file_key(intermediate: &[u8; HASH_LEN], wrapped: &[u8]) -> Result { + let mut key = *wrapped + .first_chunk::() + .ok_or(CryptError::Malformed)?; + aes::decrypt_zero_iv(AesKind::Aes256, intermediate, &mut key).ok_or(CryptError::Malformed)?; + FileKey::from_slice(&key).ok_or(CryptError::Malformed) +} diff --git a/crates/text-extract/src/pdf/crypt/rc4.rs b/crates/text-extract/src/pdf/crypt/rc4.rs new file mode 100644 index 00000000..989c06ca --- /dev/null +++ b/crates/text-extract/src/pdf/crypt/rc4.rs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) struct Rc4 { + state: [u8; 256], + i: u8, + j: u8, +} + +impl Rc4 { + pub(crate) fn new(key: &[u8]) -> Self { + let mut state = [0u8; 256]; + for (slot, value) in state.iter_mut().zip(0u8..=255) { + *slot = value; + } + let mut j = 0u8; + for (i, key_byte) in (0..256usize).zip(key.iter().cycle()) { + j = j.wrapping_add(state[i]).wrapping_add(*key_byte); + state.swap(i, usize::from(j)); + } + Rc4 { state, i: 0, j: 0 } + } + + fn next_byte(&mut self) -> u8 { + self.i = self.i.wrapping_add(1); + let si = self.state[usize::from(self.i)]; + self.j = self.j.wrapping_add(si); + self.state.swap(usize::from(self.i), usize::from(self.j)); + let sj = self.state[usize::from(self.i)]; + self.state[usize::from(si.wrapping_add(sj))] + } + + pub(crate) fn apply_in_place(&mut self, data: &mut [u8]) { + for byte in data { + *byte ^= self.next_byte(); + } + } + + pub(crate) fn apply_into(&mut self, data: &[u8], out: &mut Vec) { + out.reserve(data.len()); + out.extend(data.iter().map(|byte| byte ^ self.next_byte())); + } +} diff --git a/crates/text-extract/src/pdf/decode.rs b/crates/text-extract/src/pdf/decode.rs new file mode 100644 index 00000000..86760fe0 --- /dev/null +++ b/crates/text-extract/src/pdf/decode.rs @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + filter::{Filter, Params}, + object::{Dict, Name, Object}, +}; + +pub(crate) const MAX_STAGES: usize = 8; + +const IMAGE_FILTERS: &[&[u8]] = &[ + b"DCTDecode", + b"DCT", + b"JPXDecode", + b"JBIG2Decode", + b"CCITTFaxDecode", + b"CCF", +]; +const FULL_NAMES: &[(&[u8], Filter)] = &[ + (b"FlateDecode", Filter::Flate), + (b"LZWDecode", Filter::Lzw), + (b"ASCII85Decode", Filter::Ascii85), + (b"ASCIIHexDecode", Filter::AsciiHex), + (b"RunLengthDecode", Filter::RunLength), +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DecodeOutcome { + Complete, + Truncated, + Corrupt, + Image, + Unsupported, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Stage { + Codec(Filter, Params), + Crypt { identity: bool }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Decodable, + Image, + Unsupported, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Chain { + stages: [Option; MAX_STAGES], + len: usize, + status: Status, +} + +impl DecodeOutcome { + pub(crate) fn is_failure(self) -> bool { + matches!(self, DecodeOutcome::Corrupt | DecodeOutcome::Unsupported) + } +} + +impl Chain { + pub(crate) fn parse<'d>(dict: Dict<'d>, resolve: &impl Fn(Object<'d>) -> Object<'d>) -> Self { + let mut chain = Chain { + stages: [None; MAX_STAGES], + len: 0, + status: Status::Decodable, + }; + let (mut filter, mut params, mut abbreviated) = (None, None, None); + for (key, value) in dict.iter() { + hashify::fnc_map!(key.decoded(), + b"Filter" => filter = Some(value), + b"DecodeParms" => params = Some(value), + b"DP" => abbreviated = Some(value), + _ => {} + ); + } + let params = params.or(abbreviated).map(resolve).unwrap_or_default(); + match filter.map(resolve).unwrap_or_default() { + Object::Null => {} + Object::Name(name) => { + let params = match params { + Object::Array(array) => array.get(0).map(resolve).unwrap_or_default(), + other => other, + }; + chain.push(name, params.as_dict(), resolve); + } + Object::Array(filters) => { + let mut params = params.as_array().map(|params| params.iter()); + for filter in filters.iter() { + let entry = params + .as_mut() + .and_then(Iterator::next) + .map(resolve) + .and_then(|entry| entry.as_dict()); + let Some(name) = resolve(filter).as_name() else { + chain.status = Status::Unsupported; + break; + }; + chain.push(name, entry, resolve); + if chain.status != Status::Decodable { + break; + } + } + } + _ => chain.status = Status::Unsupported, + } + chain + } + + pub(crate) fn early_outcome(&self) -> Option { + match self.status { + Status::Decodable => None, + Status::Image => Some(DecodeOutcome::Image), + Status::Unsupported => Some(DecodeOutcome::Unsupported), + } + } + + pub(crate) fn stages(&self) -> impl Iterator + '_ { + self.stages.iter().take(self.len).flatten().copied() + } + + pub(crate) fn codecs(&self) -> impl Iterator + '_ { + self.stages().filter_map(|stage| match stage { + Stage::Codec(filter, params) => Some((filter, params)), + Stage::Crypt { .. } => None, + }) + } + + pub(crate) fn is_passthrough(&self) -> bool { + self.status == Status::Decodable && self.codecs().next().is_none() + } + + pub(crate) fn crypt_stage(&self) -> Option { + self.stages().find_map(|stage| match stage { + Stage::Crypt { identity } => Some(identity), + Stage::Codec(..) => None, + }) + } + + fn push<'d>( + &mut self, + name: Name<'d>, + params: Option>, + resolve: &impl Fn(Object<'d>) -> Object<'d>, + ) { + let decoded = name.decoded(); + let stage = if let Some(filter) = Filter::from_name(&decoded).or_else(|| { + FULL_NAMES + .iter() + .find(|(full, _)| full.eq_ignore_ascii_case(&decoded)) + .map(|&(_, filter)| filter) + }) { + Stage::Codec( + filter, + params.map_or_else(Params::default, |dict| read_params(dict, resolve)), + ) + } else if name.is(b"Crypt") { + Stage::Crypt { + identity: params + .and_then(|dict| dict.get(b"Name")) + .map(resolve) + .is_none_or(|name| name.is_null() || name.is_name(b"Identity")), + } + } else { + self.status = if IMAGE_FILTERS.iter().any(|image| name.is(image)) { + Status::Image + } else { + Status::Unsupported + }; + return; + }; + match self.stages.get_mut(self.len) { + Some(slot) => { + *slot = Some(stage); + self.len += 1; + } + None => self.status = Status::Unsupported, + } + } +} + +fn read_params<'d>(dict: Dict<'d>, resolve: &impl Fn(Object<'d>) -> Object<'d>) -> Params { + let mut params = Params::default(); + for (key, value) in dict.iter() { + let Some(value) = resolve(value).as_int() else { + continue; + }; + hashify::fnc_map!(key.raw(), + b"Predictor" => { params.predictor = value; }, + b"Colors" => { params.colors = value; }, + b"BitsPerComponent" => { params.bits_per_component = value; }, + b"Columns" => { params.columns = value; }, + b"EarlyChange" => { params.early_change = value; }, + _ => {} + ); + } + params +} + diff --git a/crates/text-extract/src/pdf/document/crypt.rs b/crates/text-extract/src/pdf/document/crypt.rs new file mode 100644 index 00000000..11ae1ba3 --- /dev/null +++ b/crates/text-extract/src/pdf/document/crypt.rs @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Document, OpenError, table::Table}; +use crate::pdf::{ + crypt::{CryptMethod, Decryptor, EncryptDict, Target}, + object::{Dict, ObjRef, Object}, +}; +use std::borrow::Cow; + +const MODERN_VERSION: i64 = 4; + +pub(crate) struct Crypt { + decryptor: Decryptor, + encrypt: Option, +} + +impl Crypt { + pub(super) fn stream_key( + &self, + id: ObjRef, + crypt_stage: Option, + ) -> Option<(&Decryptor, ObjRef)> { + let exempt = crypt_stage == Some(true) + || Some(id) == self.encrypt + || self.decryptor.method(Target::Stream) == CryptMethod::Identity; + (!exempt).then_some((&self.decryptor, id)) + } + + pub(super) fn string_key(&self, owner: Option) -> Option<(&Decryptor, ObjRef)> { + let owner = owner.filter(|&owner| Some(owner) != self.encrypt)?; + (self.decryptor.method(Target::String) != CryptMethod::Identity) + .then_some((&self.decryptor, owner)) + } +} + +impl<'a> Document<'a> { + pub(super) fn setup_crypt( + &self, + table: &Table<'_>, + trailers: &[Dict<'a>], + ) -> Result<(), OpenError> { + let Some((index, encrypt)) = trailers + .iter() + .enumerate() + .find_map(|(index, trailer)| trailer.get(b"Encrypt").map(|value| (index, value))) + else { + return Ok(()); + }; + let Object::Dict(dict) = self.resolve_in(table, encrypt) else { + return match self.resolve_in(table, encrypt) { + Object::Null => Ok(()), + _ => Err(OpenError::Encrypted), + }; + }; + let id0 = trailers + .get(index) + .and_then(|trailer| trailer.get(b"ID")) + .or_else(|| trailers.iter().find_map(|trailer| trailer.get(b"ID"))) + .map(|id| self.resolve_in(table, id)) + .and_then(|id| id.as_array()) + .and_then(|id| id.get(0)) + .map(|first| self.resolve_in(table, first)) + .and_then(|first| first.as_str()) + .map_or(Cow::Borrowed(&[][..]), |first| first.decoded()); + let value = |key: &[u8]| self.resolve_in(table, dict.get(key).unwrap_or_default()); + let bytes = |key: &[u8]| { + value(key) + .as_str() + .map_or(Cow::Borrowed(&[][..]), |value| value.decoded()) + }; + let filter = value(b"Filter") + .as_name() + .map_or(Cow::Borrowed(&[][..]), |name| name.decoded()); + let version = value(b"V").as_int().unwrap_or(0); + let revision = value(b"R").as_int().unwrap_or(match version { + ..2 => 2, + 2 | 3 => 3, + 4 => 4, + _ => 6, + }); + let (o, u, oe, ue) = (bytes(b"O"), bytes(b"U"), bytes(b"OE"), bytes(b"UE")); + let filters = value(b"CF").as_dict(); + let method = |key: &[u8]| -> Result, OpenError> { + if version < MODERN_VERSION { + return Ok(None); + } + let Some(name) = value(key).as_name() else { + return Ok(None); + }; + if name.is(b"Identity") { + return Ok(Some(CryptMethod::Identity)); + } + let Some(entry) = filters + .and_then(|filters| filters.get(name.raw())) + .map(|entry| self.resolve_in(table, entry)) + .and_then(|entry| entry.as_dict()) + else { + return Ok(Some(CryptMethod::Identity)); + }; + let cfm = self.resolve_in(table, entry.get(b"CFM").unwrap_or_default()); + match cfm.as_name() { + None => Ok(Some(CryptMethod::Identity)), + Some(cfm) if cfm.is(b"None") => Ok(Some(CryptMethod::Identity)), + Some(cfm) if cfm.is(b"V2") => Ok(Some(CryptMethod::Rc4)), + Some(cfm) if cfm.is(b"AESV2") => Ok(Some(CryptMethod::AesV2)), + Some(cfm) if cfm.is(b"AESV3") => Ok(Some(CryptMethod::AesV3)), + Some(_) => Err(OpenError::Encrypted), + } + }; + let crypt_filter_length = value(b"StmF") + .as_name() + .and_then(|name| filters?.get(name.raw())) + .map(|entry| self.resolve_in(table, entry)) + .and_then(|entry| entry.as_dict()) + .and_then(|entry| self.resolve_in(table, entry.get(b"Length")?).as_int()); + let encrypt_dict = EncryptDict { + filter: &filter, + v: version, + r: revision, + length: value(b"Length").as_int(), + o: &o, + u: &u, + oe: &oe, + ue: &ue, + p: value(b"P").as_int().unwrap_or(0), + encrypt_metadata: value(b"EncryptMetadata").as_bool().unwrap_or(true), + stream_method: method(b"StmF")?, + string_method: method(b"StrF")?, + crypt_filter_length, + id0: &id0, + }; + let decryptor = Decryptor::new(&encrypt_dict).map_err(|_| OpenError::Encrypted)?; + let _ = self.crypt.set(Crypt { + decryptor, + encrypt: encrypt.as_ref(), + }); + Ok(()) + } +} diff --git a/crates/text-extract/src/pdf/document/fetch.rs b/crates/text-extract/src/pdf/document/fetch.rs new file mode 100644 index 00000000..de87808f --- /dev/null +++ b/crates/text-extract/src/pdf/document/fetch.rs @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Document, table::Table}; +use crate::pdf::{ + object::{Dict, Indirect, ObjRef, Object, Stream}, + objstm::ObjStm, + xref::{Entry, NO_SLOT}, +}; + +const MIN_RETAINED_OBJSTM: usize = 32 << 20; +const MAX_NESTED_LOADS: u32 = 4; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Broken; + +impl<'a> Document<'a> { + pub(super) fn fetch<'t>( + &'t self, + table: &'t Table<'_>, + id: ObjRef, + ) -> Result, Broken> { + if self.source.scan_exhausted() { + return Ok(Object::Null); + } + match table.entry(id.num) { + Entry::Offset { offset, .. } => self.fetch_direct(table, offset, id.num).ok_or(Broken), + Entry::Compressed { stream, index } => { + self.fetch_compressed(table, stream, index, id.num) + } + Entry::Missing | Entry::Free { .. } => Err(Broken), + } + } + + pub(super) fn resolve_in<'t>(&'t self, table: &'t Table<'_>, object: Object<'t>) -> Object<'t> { + let mut current = object; + for _ in 0..super::MAX_REFERENCE_CHAIN { + match current { + Object::Ref(id) => current = self.fetch(table, id).unwrap_or_default(), + resolved => return resolved, + } + } + Object::Null + } + + fn parse_direct( + &self, + table: &Table<'_>, + offset: u32, + num: u32, + streams: bool, + ) -> Option> { + let data = self.source.data; + table.positions(offset).find_map(|pos| { + let object = + Indirect::parse(data, pos, streams).filter(|object| object.id.num == num)?; + self.source + .charge_scan(object.end.saturating_sub(pos)) + .then_some(object) + }) + } + + pub(super) fn fetch_direct( + &self, + table: &Table<'_>, + offset: u32, + num: u32, + ) -> Option> { + let object = self.parse_direct(table, offset, num, true)?; + Some(match (object.value, object.stream_start) { + (Object::Dict(dict), Some(start)) => { + let length = self.stream_length(table, dict, object.id); + Object::Stream(Stream { + dict, + data: self.source.stream_data(start, length), + id: object.id, + }) + } + (value, _) => value, + }) + } + + fn stream_length(&self, table: &Table<'_>, dict: Dict<'a>, owner: ObjRef) -> Option { + match dict.get(b"Length")? { + Object::Int(length) => Some(length), + Object::Ref(id) if id.num != owner.num => match table.entry(id.num) { + Entry::Offset { offset, .. } => self + .parse_direct(table, offset, id.num, false)? + .value + .as_int(), + Entry::Compressed { stream, index } if stream != owner.num => self + .fetch_compressed(table, stream, index, id.num) + .ok()? + .as_int(), + _ => None, + }, + _ => None, + } + } + + fn fetch_compressed<'t>( + &'t self, + table: &'t Table<'_>, + stream: u32, + index: u32, + num: u32, + ) -> Result, Broken> { + let Entry::Offset { offset, slot, .. } = table.entry(stream) else { + return Err(Broken); + }; + if slot == NO_SLOT || stream == num { + return Err(Broken); + } + let slot = table.slot(slot).ok_or(Broken)?; + let loaded = match slot.cell.get() { + Some(loaded) => loaded, + None => { + if slot.busy.get() || self.nesting.get() >= MAX_NESTED_LOADS { + return Ok(Object::Null); + } + slot.busy.set(true); + self.nesting.set(self.nesting.get() + 1); + let loaded = self.load_objstm(table, offset, stream); + self.nesting.set(self.nesting.get() - 1); + slot.busy.set(false); + slot.cell.get_or_init(|| loaded) + } + }; + let objstm = loaded.as_ref().ok_or(Broken)?; + let (object, consumed) = objstm.object(index, num).ok_or(Broken)?; + if !self.source.charge_scan(consumed) { + return Ok(Object::Null); + } + Ok(object) + } + + pub(super) fn load_objstm(&self, table: &Table<'_>, offset: u32, num: u32) -> Option { + let retained = self.retained.get(); + let cap = MIN_RETAINED_OBJSTM.max(self.source.data.len()); + if retained >= cap { + self.source.mark_truncated(); + return None; + } + let stream = self.fetch_direct(table, offset, num)?.as_stream()?; + let mut data = Vec::new(); + self.decode_stream(stream, &mut data); + data.shrink_to_fit(); + if retained.saturating_add(data.len()) > cap { + self.source.mark_truncated(); + return None; + } + self.retained.set(retained + data.len()); + let count = self.resolve_in(table, stream.dict.get(b"N").unwrap_or_default()); + let first = self.resolve_in(table, stream.dict.get(b"First").unwrap_or_default()); + Some(ObjStm::new(data, count.as_int(), first.as_int())) + } +} diff --git a/crates/text-extract/src/pdf/document/mod.rs b/crates/text-extract/src/pdf/document/mod.rs new file mode 100644 index 00000000..f43ded5a --- /dev/null +++ b/crates/text-extract/src/pdf/document/mod.rs @@ -0,0 +1,310 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod crypt; +mod fetch; +mod recover; +mod streams; +mod table; + +use self::{ + crypt::Crypt, + recover::Recovered, + table::{Slot, Table, assign_slots}, +}; +use super::{ + object::{Array, Dict, Name, ObjRef, Object, Stream}, + source::{Codec, Source}, + xref::{self, Builder, Entries}, +}; +use crate::xml::stream::Budget; +use std::{ + borrow::Cow, + cell::{Cell, OnceCell}, +}; + +pub(crate) const MAX_REFERENCE_CHAIN: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OpenError { + Encrypted, + Unusable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OpenFailure { + pub(crate) error: OpenError, + pub(crate) used_bytes: u64, +} + +#[derive(Default)] +pub(crate) struct DocScratch { + entries: Entries, + codec: Codec, +} + +pub(crate) struct Document<'a> { + source: Source<'a>, + primary: Table<'a>, + trailers: Vec>, + repaired: OnceCell>, + repairing: Cell, + catalog: Cell>, + page_tree: Cell>, + crypt: OnceCell, + retained: Cell, + nesting: Cell, + max_objects: usize, +} + +impl DocScratch { + pub(crate) fn shrink(&mut self, max_bytes: usize) { + self.entries.shrink(max_bytes); + self.codec.shrink(max_bytes); + } +} + +impl<'a> Document<'a> { + pub(crate) fn open( + data: &'a [u8], + scratch: &'a mut DocScratch, + budget: Budget, + max_objects: usize, + ) -> Result { + let DocScratch { entries, codec } = scratch; + let source = Source::new(data, codec, budget); + let sections = { + let mut builder = Builder::new(entries, data.len(), max_objects); + let sections = xref::read_sections(&source, &mut builder); + builder.finish(); + sections + }; + let slots = assign_slots(entries); + let entries: &'a Entries = entries; + let (trailers, bias, found) = match sections { + Some(sections) => (sections.trailers, sections.bias, true), + None => (Vec::new(), source.header, false), + }; + let header = source.header; + let document = Document { + source, + primary: Table::new(Cow::Borrowed(entries), Slot::many(slots), bias, header), + trailers, + repaired: OnceCell::new(), + repairing: Cell::new(false), + catalog: Cell::new(None), + page_tree: Cell::new(None), + crypt: OnceCell::new(), + retained: Cell::new(0), + nesting: Cell::new(0), + max_objects, + }; + if !found { + document.repair(); + } + let fail = |document: &Document<'_>, error| OpenFailure { + error, + used_bytes: document.source.used_bytes(), + }; + if document.crypt.get().is_none() + && let Err(error) = document.setup_crypt(document.table(), &document.trailers()) + { + return Err(fail(&document, error)); + } + if !document.locate_root() { + return Err(fail(&document, OpenError::Unusable)); + } + Ok(document) + } + + pub(crate) fn finish(self) -> Budget { + self.source.into_budget() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn used_bytes(&self) -> u64 { + self.source.used_bytes() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn truncated(&self) -> bool { + self.source.truncated() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn repaired(&self) -> bool { + self.repaired.get().is_some() + } + + pub(crate) fn exhausted(&self) -> bool { + self.source.budget_exhausted() || self.source.scan_exhausted() + } + + fn table(&self) -> &Table<'a> { + self.repaired + .get() + .map_or(&self.primary, |recovered| &recovered.table) + } + + fn trailers(&self) -> Vec> { + let mut trailers = self.trailers.clone(); + if let Some(recovered) = self.repaired.get() { + trailers.extend_from_slice(&recovered.trailers); + } + trailers + } + + pub(crate) fn get(&self, id: ObjRef) -> Object<'_> { + match self.fetch(self.table(), id) { + Ok(object) => object, + Err(_) if self.repair() => self.fetch(self.table(), id).unwrap_or_default(), + Err(_) => Object::Null, + } + } + + pub(crate) fn resolve<'d>(&'d self, object: Object<'d>) -> Object<'d> { + let mut current = object; + for _ in 0..MAX_REFERENCE_CHAIN { + match current { + Object::Ref(id) => current = self.get(id), + resolved => return resolved, + } + } + Object::Null + } + + pub(crate) fn dict_get<'d>(&'d self, dict: Dict<'d>, key: &[u8]) -> Object<'d> { + if !self.source.charge_scan(dict.body().len()) { + return Object::Null; + } + dict.get(key) + .map_or(Object::Null, |value| self.resolve(value)) + } + + pub(crate) fn get_int(&self, dict: Dict<'_>, key: &[u8]) -> Option { + self.dict_get(dict, key).as_int() + } + + pub(crate) fn get_f64(&self, dict: Dict<'_>, key: &[u8]) -> Option { + self.dict_get(dict, key).as_f64() + } + + pub(crate) fn get_name<'d>(&'d self, dict: Dict<'d>, key: &[u8]) -> Option> { + self.dict_get(dict, key).as_name() + } + + pub(crate) fn get_dict<'d>(&'d self, dict: Dict<'d>, key: &[u8]) -> Option> { + match self.dict_get(dict, key) { + Object::Dict(value) => Some(value), + _ => None, + } + } + + pub(crate) fn get_array<'d>(&'d self, dict: Dict<'d>, key: &[u8]) -> Option> { + self.dict_get(dict, key).as_array() + } + + pub(crate) fn get_stream<'d>(&'d self, dict: Dict<'d>, key: &[u8]) -> Option> { + self.dict_get(dict, key).as_stream() + } + + pub(crate) fn array_iter<'d>( + &'d self, + array: Array<'d>, + ) -> impl Iterator> + 'd { + let allowed = self.source.charge_scan(array.body().len()); + array + .iter() + .take_while(move |_| allowed) + .map(move |item| self.resolve(item)) + } + + pub(crate) fn catalog(&self) -> Option> { + match self.get(self.catalog.get()?) { + Object::Dict(dict) => Some(dict), + _ => None, + } + } + + pub(crate) fn page_tree_root(&self) -> Option> { + match self.catalog() { + Some(catalog) => Some(catalog.get(b"Pages").unwrap_or_default()), + None => self.page_tree.get().map(Object::Ref), + } + } + + pub(crate) fn acroform_fields(&self) -> Option> { + let form = self.get_dict(self.catalog()?, b"AcroForm")?; + self.get_array(form, b"Fields") + } + + fn locate_root(&self) -> bool { + if self.root_from_trailers(&self.trailers) { + return true; + } + self.repair(); + let Some(recovered) = self.repaired.get() else { + return false; + }; + if self.root_from_trailers(&recovered.trailers) { + return true; + } + if let Some(root) = recovered + .catalogs + .iter() + .copied() + .find(|&id| self.is_catalog(id)) + { + self.catalog.set(Some(root)); + return true; + } + if let Some(tree) = recovered + .page_trees + .iter() + .copied() + .find(|&id| matches!(self.get(id), Object::Dict(_))) + { + self.page_tree.set(Some(tree)); + return true; + } + !recovered.pages.is_empty() + } + + fn root_from_trailers(&self, trailers: &[Dict<'a>]) -> bool { + for root in trailers.iter().filter_map(|trailer| trailer.get(b"Root")) { + match root { + Object::Ref(id) if self.is_catalog(id) => { + self.catalog.set(Some(id)); + return true; + } + Object::Dict(catalog) => { + if let Some(Object::Ref(tree)) = catalog.get(b"Pages") + && matches!(self.get(tree), Object::Dict(_)) + { + self.page_tree.set(Some(tree)); + return true; + } + } + _ => {} + } + } + false + } + + fn is_catalog(&self, id: ObjRef) -> bool { + match self.get(id) { + Object::Dict(catalog) => matches!(self.dict_get(catalog, b"Pages"), Object::Dict(_)), + _ => false, + } + } + + pub(crate) fn fallback_pages(&self) -> &[ObjRef] { + self.repair(); + self.repaired + .get() + .map_or(&[], |recovered| recovered.pages.as_slice()) + } +} diff --git a/crates/text-extract/src/pdf/document/recover.rs b/crates/text-extract/src/pdf/document/recover.rs new file mode 100644 index 00000000..4fd4ce73 --- /dev/null +++ b/crates/text-extract/src/pdf/document/recover.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Document, + table::{Slot, Table}, +}; +use crate::pdf::{ + object::{Dict, ObjRef}, + repair::{Kind, Scan}, + xref::{Builder, Entries, Entry, NO_SLOT, read_stream_section}, +}; +use std::{borrow::Cow, cmp::Reverse}; + +const CANDIDATES_PER_OBJECT: usize = 2; + +pub(crate) struct Recovered<'a> { + pub(super) table: Table<'a>, + pub(super) trailers: Vec>, + pub(super) catalogs: Vec, + pub(super) page_trees: Vec, + pub(super) pages: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct Candidate { + num: u32, + position: u32, + entry: Entry, +} + +#[derive(Debug, Clone, Copy)] +struct Located { + id: ObjRef, + position: u32, + order: u32, + kind: Kind, + entry: Entry, +} + +struct Container { + num: u32, + position: u32, + slot: u32, +} + +impl<'a> Document<'a> { + pub(super) fn repair(&self) -> bool { + if self.repaired.get().is_some() || self.repairing.get() { + return false; + } + self.repairing.set(true); + let recovered = self.recover(); + self.repairing.set(false); + self.repaired.set(recovered).is_ok() + } + + fn recover(&self) -> Recovered<'a> { + let data = self.source.data; + let scan = Scan::run(&self.source); + let dense_cap = Builder::dense_cap(data.len(), self.max_objects); + let limit = dense_cap.saturating_mul(CANDIDATES_PER_OBJECT); + let mut candidates: Vec = scan + .objects + .iter() + .take(limit) + .map(|found| Candidate { + num: found.id.num, + position: found.offset, + entry: Entry::Offset { + offset: found.offset, + generation: found.id.generation, + slot: NO_SLOT, + }, + }) + .collect(); + let mut trailers: Vec<(u32, Dict<'a>)> = scan + .trailers + .iter() + .filter_map(|trailer| Some((trailer.offset, trailer.dict(data)?))) + .collect(); + let mut harvested_candidates = Vec::new(); + let mut harvested = Entries::default(); + for found in scan.objects.iter().filter(|found| found.kind == Kind::XRef) { + if self.source.scan_exhausted() { + break; + } + let mut builder = Builder::sparse(&mut harvested, limit); + let dict = read_stream_section(&self.source, found.offset as usize, &mut builder, 0); + builder.finish(); + let Some(dict) = dict else { + continue; + }; + trailers.push((found.offset, dict)); + let room = limit.saturating_sub(harvested_candidates.len()); + harvested_candidates.extend( + harvested + .iter() + .filter(|(_, entry)| matches!(entry, Entry::Compressed { .. })) + .take(room) + .map(|(num, entry)| Candidate { + num, + position: found.offset, + entry, + }), + ); + } + trailers.sort_by_key(|&(offset, _)| Reverse(offset)); + let trailers: Vec> = trailers.into_iter().map(|(_, dict)| dict).collect(); + + let mut direct = winners(&mut candidates, dense_cap); + if self.crypt.get().is_none() { + let lookup = Table::new(Cow::Borrowed(&direct), Vec::new(), 0, 0); + let _ = self.setup_crypt(&lookup, &trailers); + } + let mut located: Vec = scan + .objects + .iter() + .filter(|found| !matches!(found.kind, Kind::Plain | Kind::XRef)) + .map(|found| Located { + id: found.id, + position: found.offset, + order: 0, + kind: found.kind, + entry: Entry::Offset { + offset: found.offset, + generation: found.id.generation, + slot: NO_SLOT, + }, + }) + .filter(|found| direct.get(found.id.num) == found.entry) + .collect(); + let containers: Vec = located + .iter() + .filter(|found| found.kind == Kind::ObjStm) + .zip(0u32..) + .map(|(found, slot)| Container { + num: found.id.num, + position: found.position, + slot, + }) + .collect(); + let mut container_numbers: Vec = + containers.iter().map(|container| container.num).collect(); + container_numbers.sort_unstable(); + let room = limit.saturating_sub(candidates.len()); + candidates.extend( + harvested_candidates + .into_iter() + .filter(|candidate| match candidate.entry { + Entry::Compressed { stream, .. } => { + container_numbers.binary_search(&stream).is_ok() + } + _ => false, + }) + .take(room), + ); + candidates.sort_by_key(|candidate| (candidate.num, candidate.position)); + for container in &containers { + if let Some(Entry::Offset { slot, .. }) = direct.get_mut(container.num) { + *slot = container.slot; + } + if let Ok(index) = candidates + .binary_search_by_key(&(container.num, container.position), |candidate| { + (candidate.num, candidate.position) + }) + && let Some(Candidate { + entry: Entry::Offset { slot, .. }, + .. + }) = candidates.get_mut(index) + { + *slot = container.slot; + } + } + let table = Table::new(Cow::Owned(direct), Slot::many(containers.len()), 0, 0); + for container in &containers { + let Some(cell) = table.slot(container.slot) else { + continue; + }; + let loaded = self.load_objstm(&table, container.position, container.num); + let Some(objstm) = cell.cell.get_or_init(|| loaded) else { + continue; + }; + for (member, index) in objstm.members() { + if self.source.scan_exhausted() { + break; + } + let entry = Entry::Compressed { + stream: container.num, + index, + }; + if candidates.len() < limit { + candidates.push(Candidate { + num: member, + position: container.position, + entry, + }); + } + let kind = objstm + .object(index, member) + .filter(|&(_, consumed)| self.source.charge_scan(consumed)) + .and_then(|(object, _)| object.as_dict()) + .map_or(Kind::Plain, Kind::of); + if matches!(kind, Kind::Catalog | Kind::PageTree | Kind::Page) + && let Some(id) = ObjRef::new(i64::from(member), 0) + { + located.push(Located { + id, + position: container.position, + order: index.saturating_add(1), + kind, + entry, + }); + } + } + } + let entries = winners(&mut candidates, dense_cap); + located.retain(|found| entries.get(found.id.num) == found.entry); + located.sort_by_key(|found| (found.position, found.order)); + let by_kind = |kind: Kind| -> Vec { + located + .iter() + .filter(|found| found.kind == kind) + .map(|found| found.id) + .collect() + }; + let mut catalogs = by_kind(Kind::Catalog); + catalogs.reverse(); + let mut page_trees = by_kind(Kind::PageTree); + page_trees.reverse(); + let pages = by_kind(Kind::Page); + let Table { slots, .. } = table; + Recovered { + table: Table::new(Cow::Owned(entries), slots, 0, 0), + trailers, + catalogs, + page_trees, + pages, + } + } +} + +fn winners(candidates: &mut [Candidate], dense_cap: usize) -> Entries { + candidates.sort_by_key(|candidate| (candidate.num, candidate.position)); + let mut entries = Entries::default(); + for group in candidates.chunk_by(|left, right| left.num == right.num) { + if let Some(last) = group.last() { + entries.put_sorted(last.num, last.entry, dense_cap); + } + } + entries +} diff --git a/crates/text-extract/src/pdf/document/streams.rs b/crates/text-extract/src/pdf/document/streams.rs new file mode 100644 index 00000000..9f067f16 --- /dev/null +++ b/crates/text-extract/src/pdf/document/streams.rs @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Document; +use crate::pdf::{ + crypt::{Decryptor, Target}, + decode::{Chain, DecodeOutcome}, + object::{ObjRef, Str, Stream}, +}; +use std::borrow::Cow; + +impl<'a> Document<'a> { + pub(crate) fn decode_stream(&self, stream: Stream<'_>, out: &mut Vec) -> DecodeOutcome { + if self.source.budget_exhausted() { + self.source.mark_truncated(); + return DecodeOutcome::Truncated; + } + let chain = Chain::parse(stream.dict, &|object| self.resolve(object)); + let decrypt = self.stream_key(stream, &chain); + self.source.decode(&chain, stream.data, decrypt, out) + } + + pub(crate) fn stream_bytes<'s>( + &self, + stream: Stream<'s>, + out: &'s mut Vec, + ) -> (&'s [u8], DecodeOutcome) { + out.clear(); + if self.source.budget_exhausted() { + self.source.mark_truncated(); + return (out, DecodeOutcome::Truncated); + } + let chain = Chain::parse(stream.dict, &|object| self.resolve(object)); + let decrypt = self.stream_key(stream, &chain); + if chain.is_passthrough() && decrypt.is_none() { + return self.source.borrow(stream.data); + } + let outcome = self.source.decode(&chain, stream.data, decrypt, out); + (out, outcome) + } + + fn stream_key(&self, stream: Stream<'_>, chain: &Chain) -> Option<(&Decryptor, ObjRef)> { + self.crypt + .get() + .filter(|_| !stream.dict.is_type(b"XRef")) + .and_then(|crypt| crypt.stream_key(stream.id, chain.crypt_stage())) + } + + pub(crate) fn string<'s>(&self, value: Str<'s>) -> Cow<'s, [u8]> { + let decoded = value.decoded(); + match self + .crypt + .get() + .and_then(|crypt| crypt.string_key(value.owner())) + { + Some((decryptor, id)) => { + let mut out = Vec::with_capacity(decoded.len()); + decryptor.decrypt(Target::String, id.num, id.generation, &decoded, &mut out); + Cow::Owned(out) + } + None => decoded, + } + } +} diff --git a/crates/text-extract/src/pdf/document/table.rs b/crates/text-extract/src/pdf/document/table.rs new file mode 100644 index 00000000..fbbf04a5 --- /dev/null +++ b/crates/text-extract/src/pdf/document/table.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::{ + objstm::ObjStm, + xref::{Entries, Entry, NO_SLOT}, +}; +use std::{ + borrow::Cow, + cell::{Cell, OnceCell}, +}; + +#[derive(Default)] +pub(crate) struct Slot { + pub(super) cell: OnceCell>, + pub(super) busy: Cell, +} + +pub(crate) struct Table<'a> { + pub(super) entries: Cow<'a, Entries>, + pub(super) slots: Vec, + bases: [usize; 2], +} + +impl<'a> Table<'a> { + pub(crate) fn new( + entries: Cow<'a, Entries>, + slots: Vec, + bias: usize, + header: usize, + ) -> Self { + let other = if bias == header { 0 } else { header }; + Table { + entries, + slots, + bases: [bias, other], + } + } + + #[inline] + pub(crate) fn entry(&self, num: u32) -> Entry { + self.entries.get(num) + } + + pub(crate) fn positions(&self, offset: u32) -> impl Iterator + '_ { + let [first, second] = self.bases; + let offset = offset as usize; + std::iter::once(first) + .chain((second != first).then_some(second)) + .filter_map(move |base| base.checked_add(offset)) + } + + pub(crate) fn slot(&self, index: u32) -> Option<&Slot> { + self.slots.get(index as usize) + } +} + +impl Slot { + pub(crate) fn many(count: usize) -> Vec { + std::iter::repeat_with(Slot::default).take(count).collect() + } +} + +pub(crate) fn assign_slots(entries: &mut Entries) -> usize { + let mut streams: Vec = entries + .iter() + .filter_map(|(_, entry)| match entry { + Entry::Compressed { stream, .. } => Some(stream), + _ => None, + }) + .collect(); + streams.sort_unstable(); + streams.dedup(); + let mut next = 0u32; + for stream in streams { + if let Some(Entry::Offset { slot, .. }) = entries.get_mut(stream) + && *slot == NO_SLOT + { + *slot = next; + next += 1; + } + } + next as usize +} diff --git a/crates/text-extract/src/pdf/filter/ascii85.rs b/crates/text-extract/src/pdf/filter/ascii85.rs new file mode 100644 index 00000000..3a540f83 --- /dev/null +++ b/crates/text-extract/src/pdf/filter/ascii85.rs @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::memchr; + +use super::output::Output; + +const FIRST_DIGIT: u8 = b'!'; +const LAST_DIGIT: u8 = b'u'; +const ZERO_GROUP: u8 = b'z'; +const END: u8 = b'~'; +const GROUP: usize = 5; +const BASE: u32 = 85; + +pub(super) fn decode(input: &[u8], output: &mut Output<'_>) -> bool { + let input = input.trim_ascii_start(); + let input = input.strip_prefix(b"<~").unwrap_or(input); + let body = memchr(END, input) + .and_then(|end| input.get(..end)) + .unwrap_or(input); + output.reserve(body.len() / GROUP * 4 + 4); + let mut corrupt = false; + let mut value = 0u32; + let mut count = 0usize; + for &byte in body { + match byte { + FIRST_DIGIT..=LAST_DIGIT => { + value = value + .wrapping_mul(BASE) + .wrapping_add(u32::from(byte - FIRST_DIGIT)); + count += 1; + if count == GROUP { + if !output.push(&value.to_be_bytes()) { + return corrupt; + } + value = 0; + count = 0; + } + } + ZERO_GROUP if count == 0 => { + if !output.push(&[0; 4]) { + return corrupt; + } + } + b'\0' | b'\t' | b'\n' | b'\x0c' | b'\r' | b' ' | b'\x08' | b'\x7f' => {} + _ => corrupt = true, + } + } + match count { + 0 => {} + 1 => corrupt = true, + _ => { + let padded = (count..GROUP).fold(value, |value, _| { + value + .wrapping_mul(BASE) + .wrapping_add(u32::from(LAST_DIGIT - FIRST_DIGIT)) + }); + if let Some(bytes) = padded.to_be_bytes().get(..count - 1) { + output.push(bytes); + } + } + } + corrupt +} diff --git a/crates/text-extract/src/pdf/filter/ascii_hex.rs b/crates/text-extract/src/pdf/filter/ascii_hex.rs new file mode 100644 index 00000000..b0d32c60 --- /dev/null +++ b/crates/text-extract/src/pdf/filter/ascii_hex.rs @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::memchr; + +use super::output::Output; + +const END: u8 = b'>'; + +pub(super) fn decode(input: &[u8], output: &mut Output<'_>) -> bool { + let body = memchr(END, input) + .and_then(|end| input.get(..end)) + .unwrap_or(input); + output.reserve(body.len() / 2 + 1); + let mut corrupt = false; + let mut high: Option = None; + for &byte in body { + let nibble = match byte { + b'0'..=b'9' => byte - b'0', + b'a'..=b'f' => byte - b'a' + 10, + b'A'..=b'F' => byte - b'A' + 10, + b'\0' | b'\t' | b'\n' | b'\x0c' | b'\r' | b' ' => continue, + _ => { + corrupt = true; + continue; + } + }; + match high.take() { + Some(high) => { + if !output.push_byte(high << 4 | nibble) { + return corrupt; + } + } + None => high = Some(nibble), + } + } + if let Some(high) = high { + output.push_byte(high << 4); + } + corrupt +} diff --git a/crates/text-extract/src/pdf/filter/flate.rs b/crates/text-extract/src/pdf/filter/flate.rs new file mode 100644 index 00000000..2da5e1fb --- /dev/null +++ b/crates/text-extract/src/pdf/filter/flate.rs @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use flate2::{Decompress, FlushDecompress, Status}; +use memchr::memchr; + +use super::stage::{Source, Step}; + +const MIN_STEP: usize = 4096; +const EXPANSION_GUESS: usize = 4; +const ZLIB_HEADER: usize = 2; +const DICTIONARY_ID: usize = 4; + +pub(super) struct FlateSource<'a> { + inflate: &'a mut Decompress, + data: &'a [u8], + fallback: Option<&'a [u8]>, + hint: usize, +} + +impl<'a> FlateSource<'a> { + pub(super) fn new(inflate: &'a mut Decompress, input: &'a [u8]) -> Self { + let (data, fallback) = plan(input); + inflate.reset(false); + Self { + inflate, + data, + fallback, + hint: input.len().saturating_mul(EXPANSION_GUESS).max(MIN_STEP), + } + } + + fn retry(&mut self) -> bool { + match self.fallback.take() { + Some(data) if self.inflate.total_out() == 0 => { + self.inflate.reset(false); + self.data = data; + true + } + _ => false, + } + } +} + +impl Source for FlateSource<'_> { + fn produce(&mut self, out: &mut Vec, cap: usize) -> Step { + loop { + let len = out.len(); + if len >= cap { + return Step::More; + } + out.resize(cap, 0); + let (in_before, out_before) = (self.inflate.total_in(), self.inflate.total_out()); + let result = self.inflate.decompress( + self.data, + out.get_mut(len..).unwrap_or_default(), + FlushDecompress::None, + ); + let consumed = delta(self.inflate.total_in(), in_before); + let produced = delta(self.inflate.total_out(), out_before); + out.truncate(len + produced); + self.data = self.data.get(consumed..).unwrap_or_default(); + let stalled = match result { + Ok(Status::StreamEnd) => { + self.data = &[]; + self.fallback = None; + return Step::Done { corrupt: false }; + } + Ok(_) => consumed == 0 && produced == 0, + Err(_) => true, + }; + if stalled && !self.retry() { + self.data = &[]; + return Step::Done { corrupt: true }; + } + } + } + + fn size_hint(&self) -> usize { + self.hint + } +} + +fn delta(after: u64, before: u64) -> usize { + usize::try_from(after.saturating_sub(before)).unwrap_or(usize::MAX) +} + +fn plan(input: &[u8]) -> (&[u8], Option<&[u8]>) { + if let Some(body) = zlib_body(input).or_else(|| zlib_body(input.trim_ascii_start())) { + return (body, None); + } + if let Some(body) = gzip_body(input) { + return (body, None); + } + (input, input.get(ZLIB_HEADER..)) +} + +fn zlib_body(data: &[u8]) -> Option<&[u8]> { + const DEFLATE: u8 = 8; + const MAX_WINDOW: u8 = 7; + const PRESET_DICTIONARY: u8 = 0x20; + const CHECK_MODULUS: u16 = 31; + let [cmf, flg, body @ ..] = data else { + return None; + }; + let valid = cmf & 0x0f == DEFLATE + && cmf >> 4 <= MAX_WINDOW + && (u16::from(*cmf) << 8 | u16::from(*flg)) % CHECK_MODULUS == 0; + match (valid, flg & PRESET_DICTIONARY != 0) { + (false, _) => None, + (true, false) => Some(body), + (true, true) => body.get(DICTIONARY_ID..), + } +} + +fn gzip_body(data: &[u8]) -> Option<&[u8]> { + const HEADER_CRC: u8 = 0x02; + const EXTRA: u8 = 0x04; + const NAME: u8 = 0x08; + const COMMENT: u8 = 0x10; + let [0x1f, 0x8b, 8, flags, _, _, _, _, _, _, rest @ ..] = data else { + return None; + }; + let mut rest = rest; + if flags & EXTRA != 0 { + let [low, high, extra @ ..] = rest else { + return None; + }; + rest = extra.get(usize::from(u16::from_le_bytes([*low, *high]))..)?; + } + for flag in [NAME, COMMENT] { + if flags & flag != 0 { + rest = rest.get(memchr(0, rest)? + 1..)?; + } + } + if flags & HEADER_CRC != 0 { + rest = rest.get(2..)?; + } + Some(rest) +} diff --git a/crates/text-extract/src/pdf/filter/lzw.rs b/crates/text-extract/src/pdf/filter/lzw.rs new file mode 100644 index 00000000..767cab5c --- /dev/null +++ b/crates/text-extract/src/pdf/filter/lzw.rs @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::ops::Range; + +use super::stage::{Source, Step}; + +const TABLE_SIZE: usize = 4096; +const INDEX_MASK: usize = TABLE_SIZE - 1; +const CLEAR: u16 = 256; +const END: u16 = 257; +const FIRST_FREE: u16 = 258; +const MIN_WIDTH: u32 = 9; +const EXPANSION_GUESS: usize = 3; +const MIN_STEP: usize = 4096; + +pub(crate) struct LzwTable { + prefix: [u16; TABLE_SIZE], + suffix: [u8; TABLE_SIZE], + length: [u16; TABLE_SIZE], + sequence: [u8; TABLE_SIZE], +} + +impl LzwTable { + pub(super) fn boxed() -> Box { + let mut table = Box::new(Self { + prefix: [0; TABLE_SIZE], + suffix: [0; TABLE_SIZE], + length: [1; TABLE_SIZE], + sequence: [0; TABLE_SIZE], + }); + for (code, suffix) in (0..=u8::MAX).zip(table.suffix.iter_mut()) { + *suffix = code; + } + table + } + + fn expand(&mut self, code: u16) -> usize { + let length = usize::from(self.length[usize::from(code) & INDEX_MASK]); + let Some(slots) = self.sequence.get_mut(..length) else { + return 0; + }; + let mut code = usize::from(code); + for slot in slots.iter_mut().rev() { + *slot = self.suffix[code & INDEX_MASK]; + code = usize::from(self.prefix[code & INDEX_MASK]); + } + length + } + + fn add(&mut self, next: u16, prefix: u16, first: u8) { + let next = usize::from(next) & INDEX_MASK; + self.prefix[next] = prefix; + self.suffix[next] = first; + self.length[next] = self.length[usize::from(prefix) & INDEX_MASK].saturating_add(1); + } +} + +pub(super) struct LzwSource<'a> { + table: &'a mut LzwTable, + input: &'a [u8], + buffer: u32, + buffered: u32, + width: u32, + next: u16, + early: u16, + previous: Option, + pending: Range, + finished: Option, + hint: usize, +} + +impl<'a> LzwSource<'a> { + pub(super) fn new(table: &'a mut LzwTable, input: &'a [u8], early_change: bool) -> Self { + Self { + table, + input, + buffer: 0, + buffered: 0, + width: MIN_WIDTH, + next: FIRST_FREE, + early: u16::from(early_change), + previous: None, + pending: 0..0, + finished: None, + hint: input.len().saturating_mul(EXPANSION_GUESS).max(MIN_STEP), + } + } + + fn read(&mut self) -> Option { + while self.buffered < self.width { + let (byte, rest) = self.input.split_first()?; + self.input = rest; + self.buffer = (self.buffer << 8) | u32::from(*byte); + self.buffered += 8; + } + self.buffered -= self.width; + let code = (self.buffer >> self.buffered) & ((1 << self.width) - 1); + self.buffer &= (1 << self.buffered) - 1; + u16::try_from(code).ok() + } + + fn emit(&mut self, out: &mut Vec, cap: usize) -> bool { + let room = cap.saturating_sub(out.len()); + let take = self.pending.len().min(room); + let from = self.pending.start; + if let Some(chunk) = self.table.sequence.get(from..from + take) { + out.extend_from_slice(chunk); + } + self.pending.start += take; + self.pending.is_empty() + } + + fn step(&mut self, code: u16) -> Result<(), bool> { + match code { + CLEAR => { + self.width = MIN_WIDTH; + self.next = FIRST_FREE; + self.previous = None; + return Ok(()); + } + END => return Err(false), + _ => {} + } + let length = if code < self.next { + self.table.expand(code) + } else if let Some(previous) = self.previous.filter(|_| code == self.next) { + let length = self.table.expand(previous); + let first = self.table.sequence.first().copied().unwrap_or_default(); + match self.table.sequence.get_mut(length) { + Some(slot) => *slot = first, + None => return Err(true), + } + length + 1 + } else { + return Err(true); + }; + if let Some(previous) = self.previous + && usize::from(self.next) < TABLE_SIZE + { + let first = self.table.sequence.first().copied().unwrap_or_default(); + self.table.add(self.next, previous, first); + self.next += 1; + self.width = match self.next + self.early { + 2048.. => 12, + 1024.. => 11, + 512.. => 10, + _ => MIN_WIDTH, + }; + } + self.previous = Some(code); + self.pending = 0..length; + Ok(()) + } +} + +impl Source for LzwSource<'_> { + fn produce(&mut self, out: &mut Vec, cap: usize) -> Step { + loop { + if !self.emit(out, cap) { + return Step::More; + } + if let Some(corrupt) = self.finished { + return Step::Done { corrupt }; + } + if out.len() >= cap { + return Step::More; + } + let Some(code) = self.read() else { + self.finished = Some(false); + continue; + }; + if let Err(corrupt) = self.step(code) { + self.finished = Some(corrupt); + } + } + } + + fn size_hint(&self) -> usize { + self.hint + } +} diff --git a/crates/text-extract/src/pdf/filter/mod.rs b/crates/text-extract/src/pdf/filter/mod.rs new file mode 100644 index 00000000..fd3db0ad --- /dev/null +++ b/crates/text-extract/src/pdf/filter/mod.rs @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod ascii85; +mod ascii_hex; +mod flate; +mod lzw; +mod output; +mod predictor; +mod run_length; +mod stage; + + +use flate2::Decompress; + +use self::flate::FlateSource; +use self::lzw::{LzwSource, LzwTable}; +use self::output::Output; +use self::predictor::Layout; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Filter { + Flate, + Lzw, + Ascii85, + AsciiHex, + RunLength, +} + +impl Filter { + pub(crate) fn from_name(name: &[u8]) -> Option { + hashify::map!(name, Filter, + b"FlateDecode" => Filter::Flate, + b"Fl" => Filter::Flate, + b"LZWDecode" => Filter::Lzw, + b"LZW" => Filter::Lzw, + b"ASCII85Decode" => Filter::Ascii85, + b"A85" => Filter::Ascii85, + b"ASCIIHexDecode" => Filter::AsciiHex, + b"AHx" => Filter::AsciiHex, + b"RunLengthDecode" => Filter::RunLength, + b"RL" => Filter::RunLength, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Params { + pub(crate) predictor: i64, + pub(crate) colors: i64, + pub(crate) bits_per_component: i64, + pub(crate) columns: i64, + pub(crate) early_change: i64, +} + +impl Default for Params { + fn default() -> Self { + Self { + predictor: 1, + colors: 1, + bits_per_component: 8, + columns: 1, + early_change: 1, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[must_use] +pub(crate) struct Outcome { + pub(crate) truncated: bool, + pub(crate) corrupt: bool, +} + +pub(crate) struct Decoder { + inflate: Decompress, + lzw: Option>, + probe: Vec, +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl Decoder { + pub(crate) fn new() -> Self { + Self { + inflate: Decompress::new(false), + lzw: None, + probe: Vec::with_capacity(1), + } + } + + pub(crate) fn decode( + &mut self, + filter: Filter, + params: &Params, + input: &[u8], + out: &mut Vec, + limit: usize, + ) -> Outcome { + match filter { + Filter::Flate => { + let (layout, invalid) = Layout::from_params(params); + let mut source = FlateSource::new(&mut self.inflate, input); + stage::run(&mut source, layout, out, limit, &mut self.probe).or_corrupt(invalid) + } + Filter::Lzw => { + let (layout, invalid) = Layout::from_params(params); + let table = self.lzw.get_or_insert_with(LzwTable::boxed); + let mut source = LzwSource::new(table, input, params.early_change != 0); + stage::run(&mut source, layout, out, limit, &mut self.probe).or_corrupt(invalid) + } + Filter::Ascii85 => Output::run(out, limit, |output| ascii85::decode(input, output)), + Filter::AsciiHex => Output::run(out, limit, |output| ascii_hex::decode(input, output)), + Filter::RunLength => { + Output::run(out, limit, |output| run_length::decode(input, output)) + } + } + } +} + +impl Outcome { + fn or_corrupt(self, corrupt: bool) -> Self { + Self { + truncated: self.truncated, + corrupt: self.corrupt | corrupt, + } + } +} diff --git a/crates/text-extract/src/pdf/filter/output.rs b/crates/text-extract/src/pdf/filter/output.rs new file mode 100644 index 00000000..e0d6f67f --- /dev/null +++ b/crates/text-extract/src/pdf/filter/output.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Outcome; + +pub(super) struct Output<'a> { + out: &'a mut Vec, + start: usize, + end: usize, + truncated: bool, +} + +impl<'a> Output<'a> { + pub(super) fn run( + out: &'a mut Vec, + limit: usize, + decode: impl FnOnce(&mut Self) -> bool, + ) -> Outcome { + let start = out.len(); + let end = start.saturating_add(limit); + let mut output = Self { + out, + start, + end, + truncated: false, + }; + let corrupt = decode(&mut output); + Outcome { + truncated: output.truncated, + corrupt, + } + } + + fn room(&self) -> usize { + self.end.saturating_sub(self.out.len()) + } + + pub(super) fn reserve(&mut self, hint: usize) { + self.out.reserve_exact(hint.min(self.room())); + } + + fn grow(&mut self, additional: usize) { + if self.out.capacity() - self.out.len() < additional { + let produced = self.out.len() - self.start; + self.out + .reserve_exact(additional.max(produced).min(self.room())); + } + } + + pub(super) fn push(&mut self, data: &[u8]) -> bool { + match data.get(..self.room()) { + Some(fitting) if fitting.len() < data.len() => { + self.grow(fitting.len()); + self.out.extend_from_slice(fitting); + self.truncated = true; + false + } + _ => { + self.grow(data.len()); + self.out.extend_from_slice(data); + true + } + } + } + + pub(super) fn push_byte(&mut self, byte: u8) -> bool { + if self.room() > 0 { + self.grow(1); + self.out.push(byte); + true + } else { + self.truncated = true; + false + } + } + + pub(super) fn push_repeat(&mut self, byte: u8, count: usize) -> bool { + let room = self.room(); + if count <= room { + self.grow(count); + self.out.resize(self.out.len() + count, byte); + true + } else { + self.grow(room); + self.out.resize(self.end, byte); + self.truncated = true; + false + } + } +} diff --git a/crates/text-extract/src/pdf/filter/predictor.rs b/crates/text-extract/src/pdf/filter/predictor.rs new file mode 100644 index 00000000..910bee7a --- /dev/null +++ b/crates/text-extract/src/pdf/filter/predictor.rs @@ -0,0 +1,319 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::cell::Cell; + +use super::Params; + +const MAX_COLORS: i64 = 32; +const DEFAULT_BITS: i64 = 8; +const TIFF: i64 = 2; +const PNG_FIRST: i64 = 10; +const PNG_LAST: i64 = 15; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Layout { + None, + Tiff(Tiff), + Png(Png), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Tiff { + row_bytes: usize, + colors: usize, + bits: u32, + samples: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Png { + row_bytes: usize, + pixel_bytes: usize, +} + +struct Geometry { + row_bytes: usize, + colors: usize, + bits: u32, + samples: usize, +} + +impl Layout { + pub(super) fn from_params(params: &Params) -> (Self, bool) { + let png = match params.predictor { + ..=1 => return (Self::None, false), + TIFF => false, + PNG_FIRST..=PNG_LAST => true, + _ => return (Self::None, true), + }; + let Some(geometry) = Geometry::from_params(params) else { + return (Self::None, true); + }; + let layout = if png { + Self::Png(Png { + row_bytes: geometry.row_bytes, + pixel_bytes: (geometry.colors * geometry.bits as usize) + .div_ceil(8) + .max(1), + }) + } else { + Self::Tiff(Tiff { + row_bytes: geometry.row_bytes, + colors: geometry.colors, + bits: geometry.bits, + samples: geometry.samples, + }) + }; + (layout, false) + } +} + +impl Geometry { + fn from_params(params: &Params) -> Option { + let colors = params.colors.clamp(1, MAX_COLORS); + let bits = match params.bits_per_component { + ..=0 => DEFAULT_BITS, + bits @ (1 | 2 | 4 | 8 | 16) => bits, + _ => return None, + }; + if params.columns < 1 { + return None; + } + let samples = u64::try_from(params.columns) + .ok()? + .checked_mul(colors.unsigned_abs())?; + let row_bytes = samples.checked_mul(bits.unsigned_abs())?.div_ceil(8); + let row_bytes = usize::try_from(row_bytes).ok()?; + row_bytes.checked_add(1)?; + Some(Self { + row_bytes, + colors: usize::try_from(colors).ok()?, + bits: u32::try_from(bits).ok()?, + samples: usize::try_from(samples).ok()?, + }) + } +} + +impl Tiff { + pub(super) fn apply(&self, data: &mut [u8]) { + let rows = data.chunks_mut(self.row_bytes); + match self.bits { + 8 => rows.for_each(|row| self.row8(row)), + 16 => rows.for_each(|row| self.row16(row)), + _ => rows.for_each(|row| self.row_packed(row)), + } + } + + fn row8(&self, row: &mut [u8]) { + let cells = Cell::from_mut(row).as_slice_of_cells(); + for (current, left) in cells.iter().skip(self.colors).zip(cells) { + current.set(current.get().wrapping_add(left.get())); + } + } + + fn row16(&self, row: &mut [u8]) { + let cells = Cell::from_mut(row).as_slice_of_cells(); + let samples = cells.chunks(2); + for (current, left) in samples.clone().skip(self.colors).zip(samples) { + match (current, left) { + ([high, low], [left_high, left_low]) => { + let sum = u16::from_be_bytes([high.get(), low.get()]) + .wrapping_add(u16::from_be_bytes([left_high.get(), left_low.get()])); + let [sum_high, sum_low] = sum.to_be_bytes(); + high.set(sum_high); + low.set(sum_low); + } + ([high], [left_high, _]) => high.set(high.get().wrapping_add(left_high.get())), + _ => {} + } + } + } + + fn row_packed(&self, row: &mut [u8]) { + let bits = self.bits as usize; + let mask = (1u8 << self.bits) - 1; + let samples = self.samples.min(row.len() * 8 / bits); + let mut previous = [0u8; MAX_COLORS as usize]; + let offsets = (0..samples).map(|sample| sample * bits); + for (offset, color) in offsets.zip((0..self.colors).cycle()) { + let shift = 8 - bits - offset % 8; + if let (Some(byte), Some(previous)) = (row.get_mut(offset / 8), previous.get_mut(color)) + { + let value = (((*byte >> shift) & mask).wrapping_add(*previous)) & mask; + *previous = value; + *byte = (*byte & !(mask << shift)) | (value << shift); + } + } + } +} + +impl Png { + pub(super) fn rows(&self, out: &mut Vec, start: usize, decoded: &mut usize) -> bool { + let stride = self.row_bytes + 1; + let first = *decoded - start < self.row_bytes; + let cells = Cell::from_mut(out.as_mut_slice()).as_slice_of_cells(); + let mut sources = cells + .get(*decoded..) + .unwrap_or_default() + .chunks_exact(stride); + let mut targets = cells + .get(*decoded..) + .unwrap_or_default() + .chunks_exact(self.row_bytes); + let mut unknown = false; + let mut count = 0; + if first && let (Some(source), Some(row)) = (sources.next(), targets.next()) { + unknown |= self.decode(row, source, None); + count += 1; + } + let above = (*decoded + count * self.row_bytes).saturating_sub(self.row_bytes); + let ups = cells + .get(above..) + .unwrap_or_default() + .chunks_exact(self.row_bytes); + for ((source, row), up) in sources.zip(targets).zip(ups) { + unknown |= self.decode(row, source, Some(up)); + count += 1; + } + let raw = *decoded + count * stride; + *decoded += count * self.row_bytes; + if raw > *decoded { + let len = out.len(); + out.copy_within(raw..len, *decoded); + out.truncate(len - (raw - *decoded)); + } + unknown + } + + pub(super) fn partial( + &self, + out: &mut Vec, + start: usize, + decoded: usize, + end: usize, + ) -> bool { + let width = out + .len() + .saturating_sub(decoded + 1) + .min(end.saturating_sub(decoded)); + let cells = Cell::from_mut(out.as_mut_slice()).as_slice_of_cells(); + let source = cells.get(decoded..).and_then(|rest| rest.get(..=width)); + let row = cells.get(decoded..).and_then(|rest| rest.get(..width)); + let up = if decoded - start >= self.row_bytes { + cells.get(decoded - self.row_bytes..decoded) + } else { + None + }; + let unknown = match (source, row) { + (Some(source), Some(row)) if width > 0 => self.decode(row, source, up), + _ => false, + }; + out.truncate(decoded + width); + unknown + } + + fn decode(&self, row: Row<'_>, source: Row<'_>, up: Option>) -> bool { + match source.split_first() { + Some((kind, source)) => !decode_png_row(kind.get(), row, source, up, self.pixel_bytes), + None => false, + } + } +} + +const NONE: u8 = 0; +const SUB: u8 = 1; +const UP: u8 = 2; +const AVERAGE: u8 = 3; +const PAETH: u8 = 4; + +type Row<'a> = &'a [Cell]; + +fn decode_png_row( + kind: u8, + row: Row<'_>, + source: Row<'_>, + up: Option>, + pixel_bytes: usize, +) -> bool { + match (kind, up) { + (NONE, _) | (UP, None) => { + for (current, raw) in row.iter().zip(source) { + current.set(raw.get()); + } + } + (SUB, _) | (PAETH, None) => sub(row, source, pixel_bytes), + (UP, Some(up)) => { + for ((current, raw), above) in row.iter().zip(source).zip(up) { + current.set(raw.get().wrapping_add(above.get())); + } + } + (AVERAGE, None) => { + let pixels = row.iter().zip(source); + for (current, raw) in pixels.clone().take(pixel_bytes) { + current.set(raw.get()); + } + for ((current, raw), left) in pixels.skip(pixel_bytes).zip(row) { + current.set(raw.get().wrapping_add(left.get() >> 1)); + } + } + (AVERAGE, Some(up)) => average(row, source, up, pixel_bytes), + (PAETH, Some(up)) => paeth(row, source, up, pixel_bytes), + _ => { + for (current, raw) in row.iter().zip(source) { + current.set(raw.get()); + } + return false; + } + } + true +} + +fn sub(row: Row<'_>, source: Row<'_>, pixel_bytes: usize) { + let pixels = row.iter().zip(source); + for (current, raw) in pixels.clone().take(pixel_bytes) { + current.set(raw.get()); + } + for ((current, raw), left) in pixels.skip(pixel_bytes).zip(row) { + current.set(raw.get().wrapping_add(left.get())); + } +} + +fn average(row: Row<'_>, source: Row<'_>, up: Row<'_>, pixel_bytes: usize) { + let pixels = row.iter().zip(source).zip(up); + for ((current, raw), above) in pixels.clone().take(pixel_bytes) { + current.set(raw.get().wrapping_add(above.get() >> 1)); + } + for (((current, raw), above), left) in pixels.skip(pixel_bytes).zip(row) { + let mean = (u16::from(left.get()) + u16::from(above.get())) >> 1; + current.set(raw.get().wrapping_add(mean as u8)); + } +} + +fn paeth(row: Row<'_>, source: Row<'_>, up: Row<'_>, pixel_bytes: usize) { + let pixels = row.iter().zip(source).zip(up); + for ((current, raw), above) in pixels.clone().take(pixel_bytes) { + current.set(raw.get().wrapping_add(above.get())); + } + for ((((current, raw), above), left), above_left) in pixels.skip(pixel_bytes).zip(row).zip(up) { + let predicted = paeth_predict(left.get(), above.get(), above_left.get()); + current.set(raw.get().wrapping_add(predicted)); + } +} + +fn paeth_predict(left: u8, above: u8, above_left: u8) -> u8 { + let (a, b, c) = (i16::from(left), i16::from(above), i16::from(above_left)); + let pa = (b - c).abs(); + let pb = (a - c).abs(); + let pc = (a + b - 2 * c).abs(); + if pa <= pb && pa <= pc { + left + } else if pb <= pc { + above + } else { + above_left + } +} diff --git a/crates/text-extract/src/pdf/filter/run_length.rs b/crates/text-extract/src/pdf/filter/run_length.rs new file mode 100644 index 00000000..131ebd05 --- /dev/null +++ b/crates/text-extract/src/pdf/filter/run_length.rs @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::output::Output; + +const END: u8 = 128; +const REPEAT_BASE: usize = 257; + +pub(super) fn decode(input: &[u8], output: &mut Output<'_>) -> bool { + output.reserve(input.len()); + let mut data = input; + loop { + match data { + [] | [END, ..] => return false, + [length @ 0..END, rest @ ..] => { + let count = usize::from(*length) + 1; + let Some((literal, rest)) = rest.split_at_checked(count) else { + output.push(rest); + return true; + }; + if !output.push(literal) { + return false; + } + data = rest; + } + [length, byte, rest @ ..] => { + if !output.push_repeat(*byte, REPEAT_BASE - usize::from(*length)) { + return false; + } + data = rest; + } + [_] => return true, + } + } +} diff --git a/crates/text-extract/src/pdf/filter/stage.rs b/crates/text-extract/src/pdf/filter/stage.rs new file mode 100644 index 00000000..ac0f6f2b --- /dev/null +++ b/crates/text-extract/src/pdf/filter/stage.rs @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Outcome; +use super::predictor::{Layout, Png}; + +pub(super) enum Step { + More, + Done { corrupt: bool }, +} + +pub(super) trait Source { + fn produce(&mut self, out: &mut Vec, cap: usize) -> Step; + fn size_hint(&self) -> usize; +} + +pub(super) fn run( + source: &mut impl Source, + layout: Layout, + out: &mut Vec, + limit: usize, + probe: &mut Vec, +) -> Outcome { + let start = out.len(); + let end = start.saturating_add(limit); + match layout { + Layout::None => settle(fill(source, out, start, end), source, probe), + Layout::Tiff(tiff) => { + let step = fill(source, out, start, end.saturating_add(1)); + tiff.apply(out.get_mut(start..).unwrap_or_default()); + let truncated = out.len() > end; + out.truncate(end); + Outcome { + truncated, + corrupt: matches!(step, Step::Done { corrupt: true }), + } + } + Layout::Png(png) => run_png(source, png, out, start, end, probe), + } +} + +fn run_png( + source: &mut impl Source, + png: Png, + out: &mut Vec, + start: usize, + end: usize, + probe: &mut Vec, +) -> Outcome { + let mut decoded = start; + let mut unknown = false; + loop { + let step = fill(source, out, start, end.saturating_add(1)); + unknown |= png.rows(out, start, &mut decoded); + match step { + Step::Done { corrupt } => { + unknown |= png.partial(out, start, decoded, end); + return Outcome { + truncated: false, + corrupt: corrupt | unknown, + }; + } + Step::More if out.len() <= end => {} + Step::More => { + unknown |= png.partial(out, start, decoded, end); + return Outcome { + truncated: has_more(source, probe), + corrupt: unknown, + }; + } + } + } +} + +fn fill(source: &mut impl Source, out: &mut Vec, start: usize, cap: usize) -> Step { + let hint = source.size_hint(); + loop { + let len = out.len(); + if len >= cap { + return Step::More; + } + let step = (cap - len).min(len.saturating_sub(start).max(hint)); + out.reserve_exact(step); + if let done @ Step::Done { .. } = source.produce(out, len + step) { + return done; + } + } +} + +fn settle(step: Step, source: &mut impl Source, probe: &mut Vec) -> Outcome { + match step { + Step::Done { corrupt } => Outcome { + truncated: false, + corrupt, + }, + Step::More => Outcome { + truncated: has_more(source, probe), + corrupt: false, + }, + } +} + +fn has_more(source: &mut impl Source, probe: &mut Vec) -> bool { + probe.clear(); + let _ = source.produce(probe, 1); + let more = !probe.is_empty(); + probe.clear(); + more +} diff --git a/crates/text-extract/src/pdf/font/cache.rs b/crates/text-extract/src/pdf/font/cache.rs new file mode 100644 index 00000000..983b49cc --- /dev/null +++ b/crates/text-extract/src/pdf/font/cache.rs @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::Font; +use crate::pdf::{ + document::Document, + object::{ObjRef, Object}, +}; +use std::{ + cell::OnceCell, + collections::{HashMap, VecDeque}, +}; + +pub(crate) const MAX_FONTS: usize = 4096; +pub(crate) const MAX_FONT_HEAP: usize = 16 << 20; +const MAX_TRANSIENT: usize = 64; +const MAX_TRANSIENT_HEAP: usize = 16 << 20; +const TRANSIENT_BIT: u32 = 1 << 31; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct FontId(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum FontKey { + Ref(ObjRef), + Direct(usize, usize), +} + +struct Transient { + serial: u32, + key: FontKey, + size: usize, + font: Font, +} + +pub(crate) struct FontCache { + fonts: Vec, + keys: HashMap, + transient: VecDeque, + serial: u32, + heap: usize, + transient_heap: usize, + standard: OnceCell, +} + +impl Default for FontCache { + fn default() -> Self { + FontCache { + fonts: Vec::new(), + keys: HashMap::new(), + transient: VecDeque::new(), + serial: 0, + heap: 0, + transient_heap: 0, + standard: OnceCell::new(), + } + } +} + +impl FontId { + pub(crate) const STANDARD: FontId = FontId(u32::MAX); + + #[cfg(all(test, stalwart_tests))] + pub(crate) const FIRST: FontId = FontId(0); +} + +impl FontCache { + pub(crate) fn clear(&mut self) { + self.fonts.clear(); + self.keys.clear(); + self.end_page(); + self.serial = 0; + self.heap = 0; + } + + pub(crate) fn shrink(&mut self) { + self.clear(); + self.fonts.shrink_to(0); + self.keys.shrink_to(0); + self.transient.shrink_to(0); + } + + pub(crate) fn load( + &mut self, + doc: &Document<'_>, + value: Object<'_>, + buf: &mut Vec, + ) -> Option { + let key = match value { + Object::Ref(id) => FontKey::Ref(id), + Object::Dict(dict) => FontKey::Direct(dict.body().as_ptr() as usize, dict.body().len()), + _ => return None, + }; + if let Some(&id) = self.keys.get(&key) { + return Some(id); + } + if let Some(stored) = self.transient.iter().find(|stored| stored.key == key) { + return Some(FontId(TRANSIENT_BIT | stored.serial)); + } + let dict = doc.resolve(value).as_dict()?; + let font = Font::load(doc, dict, buf); + let size = font.heap_size(); + if self.fonts.len() < MAX_FONTS && self.heap.saturating_add(size) <= MAX_FONT_HEAP { + let id = FontId(u32::try_from(self.fonts.len()).ok()?); + self.heap += size; + self.fonts.push(font); + self.keys.insert(key, id); + return Some(id); + } + while !self.transient.is_empty() + && (self.transient.len() >= MAX_TRANSIENT + || self.transient_heap.saturating_add(size) > MAX_TRANSIENT_HEAP) + { + if let Some(evicted) = self.transient.pop_front() { + self.transient_heap -= evicted.size; + } + } + self.serial = self.serial.wrapping_add(1) & !TRANSIENT_BIT; + self.transient_heap += size; + self.transient.push_back(Transient { + serial: self.serial, + key, + size, + font, + }); + Some(FontId(TRANSIENT_BIT | self.serial)) + } + + #[inline] + pub(crate) fn font(&self, id: FontId) -> &Font { + if id.0 & TRANSIENT_BIT == 0 { + return self + .fonts + .get(id.0 as usize) + .unwrap_or_else(|| self.standard()); + } + let serial = id.0 & !TRANSIENT_BIT; + self.transient + .front() + .map(|first| serial.wrapping_sub(first.serial) & !TRANSIENT_BIT) + .and_then(|index| self.transient.get(index as usize)) + .filter(|stored| stored.serial == serial) + .map_or_else(|| self.standard(), |stored| &stored.font) + } + + fn standard(&self) -> &Font { + self.standard.get_or_init(Font::standard) + } + + pub(crate) fn end_page(&mut self) { + self.transient.clear(); + self.transient_heap = 0; + } +} diff --git a/crates/text-extract/src/pdf/font/cmap/mod.rs b/crates/text-extract/src/pdf/font/cmap/mod.rs new file mode 100644 index 00000000..a68064b1 --- /dev/null +++ b/crates/text-extract/src/pdf/font/cmap/mod.rs @@ -0,0 +1,456 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{code::code_value, ranges::RangeMap}; +use crate::pdf::{ + lexer::{Lexer, Token}, + object::{Array, Object}, + tables::{CodeRange, PredefinedCmap, glyph_unicode}, +}; + +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 17; +pub(crate) const MAX_CODESPACE_RANGES: usize = 40; +const MAX_CODE_BYTES: usize = 4; +const MAX_TARGET_UNITS: usize = 256; +const LENGTH_SHIFT: u32 = 32; +const MAX_UNIT: u32 = 0xFFFF; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CmapKind { + ToUnicode, + Encoding, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Target { + Units { offset: u32, len: u32 }, + Cid(u32), +} + +#[derive(Debug)] +pub(crate) struct Cmap { + kind: CmapKind, + entries: RangeMap, + units: Vec, + codespace: Vec, + source_lengths: u8, + vertical: bool, + base: Option, +} + +#[derive(Clone, Copy)] +enum Block { + Codespace, + BfChar, + BfRange, + CidChar, + CidRange, +} + +impl Block { + fn from_keyword(keyword: &[u8]) -> Option<(Block, &'static [u8])> { + hashify::map!(keyword, (Block, &'static [u8]), + b"begincodespacerange" => (Block::Codespace, b"endcodespacerange"), + b"beginbfchar" => (Block::BfChar, b"endbfchar"), + b"beginbfrange" => (Block::BfRange, b"endbfrange"), + b"begincidchar" => (Block::CidChar, b"endcidchar"), + b"begincidrange" => (Block::CidRange, b"endcidrange"), + ) + .copied() + } + + fn arity(self) -> usize { + match self { + Block::Codespace | Block::BfChar | Block::CidChar => 2, + Block::BfRange | Block::CidRange => 3, + } + } +} + +impl Cmap { + pub(crate) fn new(kind: CmapKind) -> Self { + Cmap { + kind, + entries: RangeMap::default(), + units: Vec::new(), + codespace: Vec::new(), + source_lengths: 0, + vertical: false, + base: None, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn is_vertical(&self) -> bool { + self.vertical + } + + pub(crate) fn set_base(&mut self, base: Option) { + self.base = base.or(self.base); + } + + pub(crate) fn set_vertical(&mut self) { + self.vertical = true; + } + + pub(crate) fn base(&self) -> Option { + self.base + } + + pub(crate) fn codespace(&self) -> &[CodeRange] { + &self.codespace + } + + pub(crate) fn heap_size(&self) -> usize { + self.entries.heap_size() + + self.units.capacity() * 2 + + self.codespace.capacity() * std::mem::size_of::() + } + + pub(crate) fn uniform_source_length(&self) -> Option { + match self.source_lengths.count_ones() { + 1 => Some(self.source_lengths.trailing_zeros() as usize), + _ => None, + } + } + + pub(crate) fn parse(&mut self, data: &[u8]) { + let mut lexer = Lexer::new(data); + let mut previous: [Option>; 2] = [None, None]; + while self.entries.len() < MAX_CMAP_ENTRIES { + if let Some(object) = Object::read(&mut lexer, None) { + let [_, last] = previous; + previous = [last, Some(object)]; + continue; + } + match lexer.next() { + Some(Token::Keyword(keyword)) => { + self.keyword(keyword, &previous, &mut lexer); + previous = [None, None]; + } + Some(_) => previous = [None, None], + None => break, + } + } + } + + fn keyword( + &mut self, + keyword: &[u8], + previous: &[Option>; 2], + lexer: &mut Lexer<'_>, + ) { + if let Some((block, end)) = Block::from_keyword(keyword) { + self.block(block, end, lexer); + return; + } + hashify::fnc_map!(keyword, + b"usecmap" => { + if let [_, Some(Object::Name(name))] = previous { + self.base = self + .base + .or_else(|| PredefinedCmap::from_name(&name.decoded())); + } + }, + b"def" => { + if let [Some(Object::Name(key)), Some(value)] = previous + && key.is(b"WMode") + { + self.vertical = value.as_int() == Some(1); + } + }, + b"endcmap" => lexer.set_pos(lexer.data().len()), + _ => {} + ); + } + + fn block(&mut self, block: Block, end: &[u8], lexer: &mut Lexer<'_>) { + let mut operands: [Object<'_>; 3] = [Object::Null; 3]; + let mut count = 0usize; + loop { + if self.entries.len() >= MAX_CMAP_ENTRIES { + return; + } + match Object::read(lexer, None) { + Some(object) => { + if let Some(slot) = operands.get_mut(count) { + *slot = object; + } + count += 1; + if count == block.arity() { + self.entry(block, &operands); + count = 0; + } + } + None => { + let before = lexer.pos(); + match lexer.next() { + Some(Token::Keyword(keyword)) if keyword == end => return, + Some(Token::Keyword(_)) => { + lexer.set_pos(before); + return; + } + Some(_) => count = 0, + None => return, + } + } + } + } + } + + fn entry(&mut self, block: Block, operands: &[Object<'_>; 3]) { + let [first, second, third] = operands; + match block { + Block::Codespace => { + if let (Some(low), Some(high)) = (source(first), source(second)) { + self.add_codespace(&low, &high); + } + } + Block::BfChar => { + if let Some(code) = source(first) { + self.add_bf(&code, &code, second); + } + } + Block::CidChar => { + if let (Some(code), Some(cid)) = (source(first), second.as_int()) { + self.add_cid(&code, &code, cid); + } + } + Block::BfRange => { + if let (Some(low), Some(high)) = (source(first), source(second)) { + match third { + Object::Array(array) => self.add_bf_array(&low, &high, *array), + target => self.add_bf(&low, &high, target), + } + } + } + Block::CidRange => { + if let (Some(low), Some(high), Some(cid)) = + (source(first), source(second), third.as_int()) + { + self.add_cid(&low, &high, cid); + } + } + } + } + + fn add_codespace(&mut self, low: &[u8], high: &[u8]) { + if low.len() != high.len() || self.codespace.len() >= MAX_CODESPACE_RANGES { + return; + } + let (mut lows, mut highs) = ([0u8; MAX_CODE_BYTES], [0u8; MAX_CODE_BYTES]); + for ((slot_low, slot_high), (&byte_low, &byte_high)) in lows + .iter_mut() + .zip(highs.iter_mut()) + .zip(low.iter().zip(high.iter())) + { + *slot_low = byte_low.min(byte_high); + *slot_high = byte_low.max(byte_high); + } + if let Ok(len) = u8::try_from(low.len()) { + self.codespace.push(CodeRange::new(len, lows, highs)); + } + } + + fn key(&self, code: &[u8]) -> Option { + let value = u64::from(code_value(code)?); + Some(match self.kind { + CmapKind::ToUnicode => value, + CmapKind::Encoding => (code.len() as u64) << LENGTH_SHIFT | value, + }) + } + + fn range(&mut self, low: &[u8], high: &[u8]) -> Option<(u64, u64)> { + if low.len() != high.len() && self.kind == CmapKind::Encoding { + return None; + } + let (lo, hi) = (self.key(low)?, self.key(high)?); + if hi < lo { + return None; + } + if let Some(bit) = u32::try_from(low.len()) + .ok() + .and_then(|len| 1u8.checked_shl(len)) + { + self.source_lengths |= bit; + } + Some((lo, hi)) + } + + fn push(&mut self, lo: u64, hi: u64, target: Target) { + self.entries.push(lo, hi, target); + } + + fn add_cid(&mut self, low: &[u8], high: &[u8], cid: i64) { + let (Some((lo, hi)), Ok(cid)) = (self.range(low, high), u32::try_from(cid)) else { + return; + }; + let target = match self.kind { + CmapKind::Encoding => Target::Cid(cid), + CmapKind::ToUnicode => match self.units_for_char(cid) { + Some(target) => target, + None => return, + }, + }; + self.push(lo, hi, target); + } + + fn add_bf(&mut self, low: &[u8], high: &[u8], target: &Object<'_>) { + let Some((lo, hi)) = self.range(low, high) else { + return; + }; + let target = match self.kind { + CmapKind::Encoding => match target { + Object::Str(value) => code_value(&value.decoded()).map(Target::Cid), + other => other + .as_int() + .and_then(|cid| u32::try_from(cid).ok()) + .map(Target::Cid), + }, + CmapKind::ToUnicode => self.text_target(target), + }; + if let Some(target) = target { + self.push(lo, hi, target); + } + } + + fn add_bf_array(&mut self, low: &[u8], high: &[u8], array: Array<'_>) { + let Some((lo, hi)) = self.range(low, high) else { + return; + }; + let span = hi - lo; + for (offset, item) in (0..=span).zip(array.iter()) { + if self.entries.len() >= MAX_CMAP_ENTRIES { + return; + } + let target = match self.kind { + CmapKind::Encoding => item + .as_str() + .and_then(|value| code_value(&value.decoded())) + .map(Target::Cid), + CmapKind::ToUnicode => self.text_target(&item), + }; + if let Some(target) = target { + self.push(lo + offset, lo + offset, target); + } + } + } + + fn text_target(&mut self, target: &Object<'_>) -> Option { + match target { + Object::Str(value) => self.units_for_bytes(&value.decoded()), + Object::Name(name) => { + let mut text = String::new(); + glyph_unicode(&name.decoded(), &mut text).then_some(())?; + self.units_for_str(&text) + } + other => u32::try_from(other.as_int()?) + .ok() + .and_then(|value| self.units_for_char(value)), + } + } + + fn units_for_bytes(&mut self, bytes: &[u8]) -> Option { + if bytes.is_empty() { + return None; + } + let offset = u32::try_from(self.units.len()).ok()?; + let padded = bytes.len() % 2 == 1; + let mut units = 0u32; + let mut pairs = bytes.as_chunks::<2>(); + if padded { + let (&first, rest) = bytes.split_first()?; + self.units.push(u16::from(first)); + units += 1; + pairs = rest.as_chunks::<2>(); + } + for pair in pairs.0.iter().take(MAX_TARGET_UNITS) { + self.units.push(u16::from_be_bytes(*pair)); + units += 1; + } + Some(Target::Units { offset, len: units }) + } + + fn units_for_str(&mut self, text: &str) -> Option { + let offset = u32::try_from(self.units.len()).ok()?; + let before = self.units.len(); + self.units + .extend(text.encode_utf16().take(MAX_TARGET_UNITS)); + let len = u32::try_from(self.units.len() - before).ok()?; + (len > 0).then_some(Target::Units { offset, len }) + } + + fn units_for_char(&mut self, value: u32) -> Option { + let ch = char::from_u32(value)?; + let mut buf = [0u16; 2]; + let offset = u32::try_from(self.units.len()).ok()?; + let encoded = ch.encode_utf16(&mut buf); + self.units.extend_from_slice(encoded); + Some(Target::Units { + offset, + len: encoded.len() as u32, + }) + } + + pub(crate) fn finish(&mut self) { + self.entries.finish(); + self.units.shrink_to_fit(); + } + + pub(crate) fn unicode(&self, code: u32, out: &mut String) -> bool { + let Some((target, delta)) = self.entries.find(u64::from(code)) else { + return false; + }; + let Target::Units { offset, len } = target else { + return false; + }; + let Some(units) = usize::try_from(offset) + .ok() + .zip(usize::try_from(len).ok()) + .and_then(|(offset, len)| self.units.get(offset..offset.checked_add(len)?)) + else { + return false; + }; + let Some((&last, head)) = units.split_last() else { + return false; + }; + let Some(last) = u64::from(last) + .checked_add(delta) + .filter(|&value| value <= u64::from(MAX_UNIT)) + .and_then(|value| u16::try_from(value).ok()) + else { + return false; + }; + let mark = out.len(); + for ch in char::decode_utf16(head.iter().copied().chain(std::iter::once(last))) { + match ch { + Ok(ch) => out.push(ch), + Err(_) => { + out.truncate(mark); + return false; + } + } + } + out.len() > mark + } + + pub(crate) fn cid(&self, code: u32, len: usize) -> Option { + let key = (len as u64) << LENGTH_SHIFT | u64::from(code); + match self.entries.find(key)? { + (Target::Cid(start), delta) => u32::try_from(u64::from(start).checked_add(delta)?).ok(), + (Target::Units { .. }, _) => None, + } + } +} + +fn source(object: &Object<'_>) -> Option> { + let bytes = object.as_str()?.decoded(); + (!bytes.is_empty() && bytes.len() <= MAX_CODE_BYTES).then(|| bytes.into_owned()) +} + diff --git a/crates/text-extract/src/pdf/font/code.rs b/crates/text-extract/src/pdf/font/code.rs new file mode 100644 index 00000000..e655d647 --- /dev/null +++ b/crates/text-extract/src/pdf/font/code.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::tables::CodeRange; + +pub(crate) const MAX_CODE_LEN: usize = 4; +const BYTE_VALUES: usize = 256; +const AMBIGUOUS: u8 = u8::MAX; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Code { + pub(crate) value: u32, + pub(crate) len: u8, + pub(crate) valid: bool, +} + +#[derive(Debug, Clone)] +pub(crate) struct CodeSplitter { + ranges: Vec, + lengths: Box<[u8; BYTE_VALUES]>, + shortest: Box<[u8; BYTE_VALUES]>, +} + +pub(crate) fn code_value(bytes: &[u8]) -> Option { + if bytes.is_empty() || bytes.len() > MAX_CODE_LEN { + return None; + } + Some( + bytes + .iter() + .fold(0u32, |value, &byte| value << 8 | u32::from(byte)), + ) +} + +impl Code { + pub(crate) fn bytes(&self) -> impl Iterator { + let len = u32::from(self.len.min(MAX_CODE_LEN as u8)); + let value = self.value; + (0..len) + .rev() + .map(move |index| (value >> (index * 8)) as u8) + } +} + +impl CodeSplitter { + pub(crate) fn fixed(len: u8) -> Self { + let len = len.clamp(1, MAX_CODE_LEN as u8); + CodeSplitter { + ranges: Vec::new(), + lengths: Box::new([len; BYTE_VALUES]), + shortest: Box::new([len; BYTE_VALUES]), + } + } + + pub(crate) fn from_ranges(ranges: &[CodeRange], fallback: u8) -> Self { + if ranges.is_empty() { + return CodeSplitter::fixed(fallback); + } + let mut lengths = Box::new([0u8; BYTE_VALUES]); + let mut shortest = Box::new([0u8; BYTE_VALUES]); + for ((byte, length), short) in (0..=u8::MAX) + .zip(lengths.iter_mut()) + .zip(shortest.iter_mut()) + { + for range in ranges.iter().filter(|range| range.accepts_prefix(&[byte])) { + let len = u8::try_from(range.len()).unwrap_or(AMBIGUOUS); + *length = match *length { + 0 => len, + current if current == len => len, + _ => AMBIGUOUS, + }; + *short = match *short { + 0 => len, + current => current.min(len), + }; + } + } + CodeSplitter { + ranges: ranges.to_vec(), + lengths, + shortest, + } + } + + pub(crate) fn split(&self, bytes: &[u8]) -> Option { + let &first = bytes.first()?; + let index = usize::from(first); + match self.lengths.get(index).copied().unwrap_or_default() { + 0 => Some(Code { + value: u32::from(first), + len: 1, + valid: false, + }), + AMBIGUOUS => Some(self.split_slow(bytes, index)), + len => Some(take(bytes, usize::from(len), true)), + } + } + + fn split_slow(&self, bytes: &[u8], index: usize) -> Code { + for len in 1..=MAX_CODE_LEN { + let Some(prefix) = bytes.get(..len) else { + break; + }; + if self.ranges.iter().any(|range| range.contains(prefix)) { + return take(bytes, len, true); + } + if !self.ranges.iter().any(|range| range.accepts_prefix(prefix)) { + break; + } + } + let len = self.shortest.get(index).copied().unwrap_or(1).max(1); + take(bytes, usize::from(len), false) + } +} + +fn take(bytes: &[u8], len: usize, valid: bool) -> Code { + let taken = bytes.get(..len).unwrap_or(bytes); + Code { + value: code_value(taken).unwrap_or_default(), + len: taken.len() as u8, + valid: valid && taken.len() == len, + } +} + diff --git a/crates/text-extract/src/pdf/font/composite.rs b/crates/text-extract/src/pdf/font/composite.rs new file mode 100644 index 00000000..c1bbdeb2 --- /dev/null +++ b/crates/text-extract/src/pdf/font/composite.rs @@ -0,0 +1,401 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Glyph, Metrics, ToUnicode, + cmap::{Cmap, CmapKind}, + code::{Code, CodeSplitter}, + embedded::{CidToGid, GidText, ProgramKind, program}, + program::{Cff, TrueType}, + unicode::{Verdict, verdict}, + widths::{CidWidths, VerticalAdvances}, +}; +use crate::pdf::{ + document::Document, + object::{Dict, ObjRef, Object, Stream}, + tables::{CidCollection, CmapDecoder, PredefinedCmap}, +}; + +pub(crate) const MAX_USECMAP_DEPTH: usize = 16; +const WIDTH_SCALE: f32 = 1000.0; +const DEFAULT_CODE_LEN: u8 = 2; +const HALF: f32 = 0.5; +const UCS_ORDERING: &[u8] = b"UCS"; + +#[derive(Debug)] +enum Encoding { + Identity { vertical: bool }, + Embedded(Box), + Predefined(PredefinedCmap), +} + +#[derive(Debug)] +pub(crate) struct CompositeFont { + splitter: CodeSplitter, + encoding: Encoding, + to_unicode: Option, + collection: Option, + ucs: bool, + gid_text: GidText, + cid_to_gid: CidToGid, + widths: CidWidths, + vertical: Option, +} + +impl Encoding { + fn load(doc: &Document<'_>, dict: Dict<'_>, buf: &mut Vec) -> Self { + match doc.dict_get(dict, b"Encoding") { + Object::Name(name) => match PredefinedCmap::from_name(&name.decoded()) { + Some(cmap) if cmap.decoder() == CmapDecoder::Identity => Encoding::Identity { + vertical: cmap.is_vertical(), + }, + Some(cmap) => Encoding::Predefined(cmap), + None => Encoding::Identity { vertical: false }, + }, + Object::Stream(stream) => Encoding::Embedded(Box::new(embedded(doc, stream, buf))), + _ => Encoding::Identity { vertical: false }, + } + } + + fn is_vertical(&self) -> bool { + match self { + Encoding::Identity { vertical } => *vertical, + Encoding::Embedded(cmap) => { + cmap.is_vertical() || cmap.base().is_some_and(|base| base.is_vertical()) + } + Encoding::Predefined(cmap) => cmap.is_vertical(), + } + } + + fn splitter(&self) -> CodeSplitter { + match self { + Encoding::Identity { .. } => CodeSplitter::fixed(DEFAULT_CODE_LEN), + Encoding::Predefined(cmap) => { + CodeSplitter::from_ranges(cmap.codespace(), DEFAULT_CODE_LEN) + } + Encoding::Embedded(cmap) => { + let ranges = match cmap.codespace() { + [] => cmap.base().map(|base| base.codespace()).unwrap_or_default(), + ranges => ranges, + }; + let fallback = cmap + .uniform_source_length() + .and_then(|len| u8::try_from(len).ok()) + .unwrap_or(DEFAULT_CODE_LEN); + CodeSplitter::from_ranges(ranges, fallback) + } + } + } + + fn cid(&self, code: Code) -> Option { + match self { + Encoding::Identity { .. } => Some(code.value), + Encoding::Embedded(cmap) => cmap.cid(code.value, usize::from(code.len)).or_else(|| { + cmap.base() + .filter(|base| base.decoder() == CmapDecoder::Identity) + .map(|_| code.value) + }), + Encoding::Predefined(_) => None, + } + } + + fn decoder(&self) -> Option { + match self { + Encoding::Identity { .. } => None, + Encoding::Embedded(cmap) => cmap.base(), + Encoding::Predefined(cmap) => Some(*cmap), + } + .filter(|cmap| cmap.is_text_encoding()) + } + + fn collection(&self) -> Option { + match self { + Encoding::Identity { .. } => None, + Encoding::Embedded(cmap) => cmap.base().and_then(|base| base.collection()), + Encoding::Predefined(cmap) => cmap.collection(), + } + } + + fn heap_size(&self) -> usize { + match self { + Encoding::Embedded(cmap) => cmap.heap_size(), + _ => 0, + } + } +} + +fn embedded(doc: &Document<'_>, stream: Stream<'_>, buf: &mut Vec) -> Cmap { + let mut chain: Vec> = vec![stream]; + let mut seen: Vec = vec![stream.id]; + let mut cmap = Cmap::new(CmapKind::Encoding); + let mut vertical = doc.get_int(stream.dict, b"WMode") == Some(1); + while chain.len() < MAX_USECMAP_DEPTH { + let Some(current) = chain.last() else { + break; + }; + match doc.dict_get(current.dict, b"UseCMap") { + Object::Stream(parent) if !seen.contains(&parent.id) => { + seen.push(parent.id); + vertical |= doc.get_int(parent.dict, b"WMode") == Some(1); + chain.push(parent); + } + Object::Name(name) => { + cmap.set_base(PredefinedCmap::from_name(&name.decoded())); + break; + } + _ => break, + } + } + for stream in chain.iter().rev() { + let (data, outcome) = doc.stream_bytes(*stream, buf); + if !outcome.is_failure() { + cmap.parse(data); + } + } + if vertical { + cmap.set_vertical(); + } + cmap.finish(); + cmap +} + +impl CompositeFont { + pub(crate) fn load( + doc: &Document<'_>, + dict: Dict<'_>, + to_unicode: Option, + buf: &mut Vec, + ) -> (CompositeFont, Metrics, bool) { + let encoding = Encoding::load(doc, dict, buf); + let descendant = doc + .get_array(dict, b"DescendantFonts") + .and_then(|fonts| doc.array_iter(fonts).next()) + .and_then(|font| font.as_dict()) + .filter(|font| { + !doc.get_name(*font, b"Subtype") + .is_some_and(|subtype| subtype.is(b"Type0")) + }); + let (widths, collection, ucs) = match descendant { + Some(descendant) => { + let (registry, ordering) = doc + .get_dict(descendant, b"CIDSystemInfo") + .map(|info| { + let text = |key: &[u8]| { + doc.dict_get(info, key) + .as_str() + .map(|value| doc.string(value).into_owned()) + .unwrap_or_default() + }; + (text(b"Registry"), text(b"Ordering")) + }) + .unwrap_or_default(); + ( + CidWidths::load(doc, descendant), + CidCollection::from_ordering(®istry, &ordering), + ordering.trim_ascii() == UCS_ORDERING, + ) + } + None => (CidWidths::empty(), None, false), + }; + let vertical = encoding.is_vertical(); + let mut font = CompositeFont { + splitter: encoding.splitter(), + collection: collection.or_else(|| encoding.collection()), + vertical: match (vertical, descendant) { + (true, Some(descendant)) => Some(VerticalAdvances::load(doc, descendant)), + (true, None) => Some(VerticalAdvances::empty()), + _ => None, + }, + encoding, + to_unicode, + ucs, + gid_text: GidText::default(), + cid_to_gid: CidToGid::Identity, + widths, + }; + if font.to_unicode.is_none() + && font.encoding.decoder().is_none() + && let Some(descendant) = descendant + { + font.load_program(doc, descendant, buf); + } + let metrics = Metrics { + em_scale: 1.0, + space_width: font.space_width(), + }; + (font, metrics, vertical) + } + + pub(crate) fn legacy(cmap: PredefinedCmap) -> (CompositeFont, Metrics) { + let encoding = Encoding::Predefined(cmap); + let font = CompositeFont { + splitter: encoding.splitter(), + collection: encoding.collection(), + vertical: None, + encoding, + to_unicode: None, + ucs: false, + gid_text: GidText::default(), + cid_to_gid: CidToGid::Identity, + widths: CidWidths::empty(), + }; + let metrics = Metrics { + em_scale: 1.0, + space_width: font.space_width(), + }; + (font, metrics) + } + + fn load_program(&mut self, doc: &Document<'_>, descendant: Dict<'_>, buf: &mut Vec) { + let Some(descriptor) = doc.get_dict(descendant, b"FontDescriptor") else { + return; + }; + let mut map = Vec::new(); + let table = CidToGid::load(doc, descendant, &mut map); + let Some((kind, data)) = program(doc, descriptor, buf) else { + return; + }; + match kind { + ProgramKind::TrueType => { + if let Some(font) = TrueType::parse(data) { + self.gid_text = GidText::from_truetype(&font, data.len()); + self.cid_to_gid = table; + } + } + ProgramKind::Cff | ProgramKind::OpenType => { + let cff = Cff::parse(data); + match &cff { + Some(cff) if cff.is_cid() => self.cid_to_gid = CidToGid::from_charset(cff), + Some(cff) => self.gid_text = GidText::from_cff(cff, data.len()), + None => self.cid_to_gid = table, + } + if kind == ProgramKind::OpenType + && let Some(font) = TrueType::parse(data) + { + let text = GidText::from_truetype(&font, data.len()); + if !text.is_empty() { + self.gid_text = text; + } + } + } + ProgramKind::Type1 => {} + } + } + + fn space_width(&self) -> Option { + let mut text = String::new(); + let code = Code { + value: u32::from(b' '), + len: DEFAULT_CODE_LEN, + valid: true, + }; + self.text(code, &mut text); + (text == " ") + .then(|| self.width(code, &text) / WIDTH_SCALE) + .filter(|width| *width > 0.0) + } + + pub(crate) fn heap_size(&self) -> usize { + self.encoding.heap_size() + + self.to_unicode.as_ref().map_or(0, ToUnicode::heap_size) + + self.gid_text.heap_size() + + self.cid_to_gid.heap_size() + + self.widths.heap_size() + + self + .vertical + .as_ref() + .map_or(0, VerticalAdvances::heap_size) + } + + pub(crate) fn glyph(&self, bytes: &[u8], out: &mut String) -> Option { + let code = self.splitter.split(bytes)?; + if code.valid { + self.text(code, out); + } + let width = match &self.vertical { + Some(advances) => match self.encoding.cid(code) { + Some(cid) => advances.advance(cid), + None => advances.default_advance(), + }, + None => self.width(code, out), + }; + Some(Glyph { + len: usize::from(code.len), + width: width / WIDTH_SCALE, + space_code: code.len == 1 && code.value == u32::from(b' '), + }) + } + + fn width(&self, code: Code, text: &str) -> f32 { + match self.encoding.cid(code) { + Some(cid) => self.widths.width(cid), + None if code.len == 1 || text.chars().all(is_narrow) && !text.is_empty() => { + self.widths.default_width() * HALF + } + None => self.widths.default_width(), + } + } + + fn text(&self, code: Code, out: &mut String) { + let mark = out.len(); + let accept = |out: &mut String| match verdict(out.get(mark..).unwrap_or_default()) { + Verdict::Accept => true, + Verdict::Space => { + out.truncate(mark); + out.push(' '); + true + } + Verdict::Reject | Verdict::PrivateUse => { + out.truncate(mark); + false + } + }; + if let Some(to_unicode) = &self.to_unicode + && to_unicode.unicode(code.value, out) + && accept(out) + { + return; + } + let cid = self.encoding.cid(code); + if cid.is_none() + && let Some(decoder) = self.encoding.decoder() + && decoder.decode(code, out) + && accept(out) + { + return; + } + let Some(cid) = cid else { + return; + }; + if !self.gid_text.is_empty() + && let Some(text) = self + .cid_to_gid + .gid(cid) + .and_then(|gid| self.gid_text.get(gid)) + { + out.push_str(text); + if accept(out) { + return; + } + } + if let Some(collection) = self.collection + && collection.unicode(cid, out) + && accept(out) + { + return; + } + if self.ucs + && let Some(ch) = char::from_u32(cid) + { + out.push(ch); + accept(out); + } + } +} + +fn is_narrow(ch: char) -> bool { + matches!(u32::from(ch), 0x20..=0x7E | 0xA0..=0x24F | 0xFF61..=0xFF9F) +} diff --git a/crates/text-extract/src/pdf/font/embedded.rs b/crates/text-extract/src/pdf/font/embedded.rs new file mode 100644 index 00000000..b83c87ae --- /dev/null +++ b/crates/text-extract/src/pdf/font/embedded.rs @@ -0,0 +1,212 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + program::{Cff, TrueType}, + unicode::NameResolver, +}; +use crate::pdf::{ + decode::DecodeOutcome, + document::Document, + object::{Dict, Object}, +}; + +const MAX_GLYPHS: usize = 1 << 16; +const MAX_GLYPH_TEXT: usize = 64; +const TEXT_PER_PROGRAM_BYTE: usize = 2; +const MIN_TEXT_CAP: usize = 64 << 10; +const MAX_TEXT_CAP: usize = 1 << 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProgramKind { + Type1, + Cff, + TrueType, + OpenType, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct GidText { + gids: Vec, + spans: Vec<(u32, u16)>, + text: String, +} + +#[derive(Debug, Clone, Default)] +pub(crate) enum CidToGid { + #[default] + Identity, + Table(Box<[u16]>), + Charset(Box<[(u16, u16)]>), +} + +pub(crate) fn has_program(descriptor: Option>) -> bool { + descriptor.is_some_and(|descriptor| { + [&b"FontFile"[..], b"FontFile2", b"FontFile3"] + .iter() + .any(|key| descriptor.contains(key)) + }) +} + +pub(crate) fn program<'b>( + doc: &'b Document<'_>, + descriptor: Dict<'b>, + buf: &'b mut Vec, +) -> Option<(ProgramKind, &'b [u8])> { + let (kind, stream) = [ + (&b"FontFile"[..], ProgramKind::Type1), + (b"FontFile2", ProgramKind::TrueType), + (b"FontFile3", ProgramKind::Cff), + ] + .into_iter() + .find_map(|(key, kind)| Some((kind, doc.get_stream(descriptor, key)?)))?; + let kind = match kind { + ProgramKind::Cff + if doc + .get_name(stream.dict, b"Subtype") + .is_some_and(|name| name.is(b"OpenType")) => + { + ProgramKind::OpenType + } + kind => kind, + }; + let (data, outcome) = doc.stream_bytes(stream, buf); + (!outcome.is_failure() && outcome != DecodeOutcome::Image && !data.is_empty()) + .then_some((kind, data)) +} + +impl GidText { + pub(crate) fn from_truetype(font: &TrueType<'_>, program_len: usize) -> Self { + let mut table = GidText::default(); + let map = font.unicode_map(); + let count = font + .num_glyphs() + .map_or(MAX_GLYPHS, usize::from) + .min(MAX_GLYPHS); + let resolver = NameResolver::default(); + let cap = text_cap(program_len); + let mut mapped = map.iter().peekable(); + for gid in (0..=u16::MAX).take(count) { + if table.text.len() >= cap { + break; + } + let mark = table.text.len(); + while mapped.next_if(|(glyph, _)| *glyph < gid).is_some() {} + if let Some((_, ch)) = mapped.next_if(|(glyph, _)| *glyph == gid) { + table.text.push(ch); + } else if let Some(name) = font.glyph_name(gid) { + resolver.resolve(name, &mut table.text); + } + table.close(gid, mark); + } + table.text.shrink_to_fit(); + table + } + + pub(crate) fn from_cff(font: &Cff<'_>, program_len: usize) -> Self { + let mut table = GidText::default(); + let resolver = NameResolver::default(); + let cap = text_cap(program_len); + for gid in 0..font.num_glyphs() { + if table.text.len() >= cap { + break; + } + let mark = table.text.len(); + if let Some(name) = font.glyph_name(gid) { + resolver.resolve(name, &mut table.text); + } + table.close(gid, mark); + } + table.text.shrink_to_fit(); + table + } + + fn close(&mut self, gid: u16, mark: usize) { + let len = self.text.len() - mark; + match (u32::try_from(mark), u16::try_from(len)) { + (Ok(start), Ok(len)) if len > 0 && usize::from(len) <= MAX_GLYPH_TEXT => { + self.gids.push(gid); + self.spans.push((start, len)); + } + _ => self.text.truncate(mark), + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.gids.is_empty() + } + + pub(crate) fn get(&self, gid: u16) -> Option<&str> { + let index = self.gids.binary_search(&gid).ok()?; + let &(start, len) = self.spans.get(index)?; + let start = start as usize; + self.text.get(start..start + usize::from(len)) + } + + pub(crate) fn heap_size(&self) -> usize { + self.gids.capacity() * 2 + self.spans.capacity() * 8 + self.text.capacity() + } +} + +fn text_cap(program_len: usize) -> usize { + program_len + .saturating_mul(TEXT_PER_PROGRAM_BYTE) + .clamp(MIN_TEXT_CAP, MAX_TEXT_CAP) +} + +impl CidToGid { + pub(crate) fn load(doc: &Document<'_>, descendant: Dict<'_>, buf: &mut Vec) -> Self { + match doc.dict_get(descendant, b"CIDToGIDMap") { + Object::Stream(stream) => { + let (data, outcome) = doc.stream_bytes(stream, buf); + if outcome.is_failure() { + return CidToGid::Identity; + } + let (pairs, _) = data.as_chunks::<2>(); + CidToGid::Table( + pairs + .iter() + .take(MAX_GLYPHS) + .map(|pair| u16::from_be_bytes(*pair)) + .collect(), + ) + } + _ => CidToGid::Identity, + } + } + + pub(crate) fn from_charset(font: &Cff<'_>) -> Self { + let mut pairs: Vec<(u16, u16)> = (0..font.num_glyphs()) + .filter_map(|gid| Some((font.glyph_cid(gid)?, gid))) + .collect(); + pairs.sort_unstable(); + pairs.dedup_by_key(|(cid, _)| *cid); + CidToGid::Charset(pairs.into_boxed_slice()) + } + + pub(crate) fn gid(&self, cid: u32) -> Option { + match self { + CidToGid::Identity => u16::try_from(cid).ok(), + CidToGid::Table(table) => usize::try_from(cid) + .ok() + .and_then(|cid| table.get(cid)) + .copied(), + CidToGid::Charset(pairs) => { + let cid = u16::try_from(cid).ok()?; + let index = pairs.binary_search_by_key(&cid, |(key, _)| *key).ok()?; + pairs.get(index).map(|(_, gid)| *gid) + } + } + } + + pub(crate) fn heap_size(&self) -> usize { + match self { + CidToGid::Identity => 0, + CidToGid::Table(table) => table.len() * 2, + CidToGid::Charset(pairs) => pairs.len() * 4, + } + } +} diff --git a/crates/text-extract/src/pdf/font/mod.rs b/crates/text-extract/src/pdf/font/mod.rs new file mode 100644 index 00000000..87461ca4 --- /dev/null +++ b/crates/text-extract/src/pdf/font/mod.rs @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod cache; +mod cmap; +mod code; +mod composite; +mod embedded; +mod predefined; +mod program; +mod ranges; +mod simple; +mod unicode; +mod widths; + + +pub(crate) use cache::{FontCache, FontId}; + +use self::{ + cmap::{Cmap, CmapKind}, + composite::CompositeFont, + simple::{SimpleFont, SimpleKind}, +}; +use super::{ + document::Document, + object::{Dict, Object}, + tables::PredefinedCmap, +}; + +const IDENTITY_PREFIX: &[u8] = b"Identity"; +const GBK_CMAP: &[u8] = b"GBK-EUC-H"; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Metrics { + pub(crate) em_scale: f32, + pub(crate) space_width: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Glyph { + pub(crate) len: usize, + pub(crate) width: f32, + pub(crate) space_code: bool, +} + +#[derive(Debug)] +pub(crate) enum ToUnicode { + Map(Cmap), + Identity, +} + +#[derive(Debug)] +enum Kind { + Simple(SimpleFont), + Composite(Box), +} + +#[derive(Debug)] +pub(crate) struct Font { + kind: Kind, + metrics: Metrics, + vertical: bool, +} + +impl ToUnicode { + fn load(doc: &Document<'_>, dict: Dict<'_>, buf: &mut Vec) -> Option { + match doc.dict_get(dict, b"ToUnicode") { + Object::Stream(stream) => { + let (data, outcome) = doc.stream_bytes(stream, buf); + if outcome.is_failure() { + return None; + } + let mut cmap = Cmap::new(CmapKind::ToUnicode); + cmap.parse(data); + cmap.finish(); + (!cmap.is_empty()).then_some(ToUnicode::Map(cmap)) + } + Object::Name(name) if name.raw().starts_with(IDENTITY_PREFIX) => { + Some(ToUnicode::Identity) + } + _ => None, + } + } + + pub(crate) fn unicode(&self, code: u32, out: &mut String) -> bool { + match self { + ToUnicode::Map(cmap) => cmap.unicode(code, out), + ToUnicode::Identity => char::from_u32(code).map(|ch| out.push(ch)).is_some(), + } + } + + fn heap_size(&self) -> usize { + match self { + ToUnicode::Map(cmap) => cmap.heap_size(), + ToUnicode::Identity => 0, + } + } +} + +impl Font { + pub(crate) fn load(doc: &Document<'_>, dict: Dict<'_>, buf: &mut Vec) -> Self { + let subtype = doc.get_name(dict, b"Subtype"); + let is = |name: &[u8]| subtype.is_some_and(|subtype| subtype.is(name)); + let to_unicode = ToUnicode::load(doc, dict, buf); + if is(b"Type0") { + let (font, metrics, vertical) = CompositeFont::load(doc, dict, to_unicode, buf); + return Font { + kind: Kind::Composite(Box::new(font)), + metrics, + vertical, + }; + } + if to_unicode.is_none() + && !is(b"Type3") + && SimpleFont::carries_gbk(doc, dict) + && let Some(cmap) = PredefinedCmap::from_name(GBK_CMAP) + { + let (font, metrics) = CompositeFont::legacy(cmap); + return Font { + kind: Kind::Composite(Box::new(font)), + metrics, + vertical: false, + }; + } + let kind = if is(b"TrueType") { + SimpleKind::TrueType + } else if is(b"Type3") { + SimpleKind::Type3 + } else { + SimpleKind::Type1 + }; + let (font, metrics) = SimpleFont::load(doc, dict, kind, to_unicode.as_ref(), buf); + Font { + kind: Kind::Simple(font), + metrics, + vertical: false, + } + } + + pub(crate) fn standard() -> Self { + let (font, metrics) = SimpleFont::standard(); + Font { + kind: Kind::Simple(font), + metrics, + vertical: false, + } + } + + #[inline] + pub(crate) fn metrics(&self) -> Metrics { + self.metrics + } + + #[inline] + pub(crate) fn is_vertical(&self) -> bool { + self.vertical + } + + pub(crate) fn heap_size(&self) -> usize { + std::mem::size_of::() + + match &self.kind { + Kind::Simple(font) => font.heap_size(), + Kind::Composite(font) => font.heap_size(), + } + } + + #[inline] + pub(crate) fn glyph<'s>( + &'s self, + bytes: &[u8], + scratch: &'s mut String, + ) -> Option<(Glyph, &'s str)> { + match &self.kind { + Kind::Simple(font) => { + let &code = bytes.first()?; + let (text, width) = font.glyph(code); + Some(( + Glyph { + len: 1, + width, + space_code: code == b' ', + }, + text, + )) + } + Kind::Composite(font) => { + scratch.clear(); + let glyph = font.glyph(bytes, scratch)?; + Some((glyph, scratch.as_str())) + } + } + } +} diff --git a/crates/text-extract/src/pdf/font/predefined.rs b/crates/text-extract/src/pdf/font/predefined.rs new file mode 100644 index 00000000..807c64a0 --- /dev/null +++ b/crates/text-extract/src/pdf/font/predefined.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::code::{Code, MAX_CODE_LEN}; +use crate::pdf::tables::{CmapDecoder, PredefinedCmap}; +use encoding_rs::{BIG5, EUC_JP, EUC_KR, Encoding, GB18030, GBK, SHIFT_JIS}; + +const JIS_HIGH_BIT: u8 = 0x80; + +impl PredefinedCmap { + pub(crate) fn is_text_encoding(self) -> bool { + !matches!(self.decoder(), CmapDecoder::Identity | CmapDecoder::EucTw) + } + + pub(crate) fn decode(self, code: Code, out: &mut String) -> bool { + let mut bytes = [0u8; MAX_CODE_LEN]; + let mut len = 0; + for (slot, byte) in bytes.iter_mut().zip(code.bytes()) { + *slot = byte; + len += 1; + } + let bytes = bytes.get(..len).unwrap_or_default(); + let mark = out.len(); + match self.decoder() { + CmapDecoder::Identity | CmapDecoder::EucTw => return false, + CmapDecoder::Utf16Be => { + let (units, _) = bytes.as_chunks::<2>(); + for ch in char::decode_utf16(units.iter().map(|pair| u16::from_be_bytes(*pair))) { + match ch { + Ok(ch) => out.push(ch), + Err(_) => { + out.truncate(mark); + return false; + } + } + } + } + CmapDecoder::Utf8 => match std::str::from_utf8(bytes) { + Ok(text) => out.push_str(text), + Err(_) => return false, + }, + CmapDecoder::Utf32Be => match char::from_u32(code.value) { + Some(ch) => out.push(ch), + None => return false, + }, + CmapDecoder::JisRowCell => { + let mut shifted = [0u8; MAX_CODE_LEN]; + for (slot, byte) in shifted.iter_mut().zip(bytes) { + *slot = byte | JIS_HIGH_BIT; + } + return legacy(EUC_JP, shifted.get(..bytes.len()).unwrap_or_default(), out); + } + decoder => { + if let [byte] = bytes + && let Some(ch) = self.single_byte(*byte) + { + out.push(ch); + return true; + } + let encoding = match decoder { + CmapDecoder::Gbk => GBK, + CmapDecoder::Gb18030 => GB18030, + CmapDecoder::Big5 => BIG5, + CmapDecoder::ShiftJis => SHIFT_JIS, + CmapDecoder::EucJp => EUC_JP, + _ => EUC_KR, + }; + return legacy(encoding, bytes, out); + } + } + out.len() > mark + } +} + +fn legacy(encoding: &'static Encoding, bytes: &[u8], out: &mut String) -> bool { + match encoding.decode_without_bom_handling_and_without_replacement(bytes) { + Some(text) if !text.is_empty() => { + out.push_str(&text); + true + } + _ => false, + } +} + diff --git a/crates/text-extract/src/pdf/font/program/cff.rs b/crates/text-extract/src/pdf/font/program/cff.rs new file mode 100644 index 00000000..452aec41 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/cff.rs @@ -0,0 +1,390 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::cff_data::{ + EXPERT_CHARSET, EXPERT_SUBSET_CHARSET, STANDARD_STRING_NAMES, STANDARD_STRING_OFFSETS, +}; +use super::read::{NameTable, ReadBytes, Reader}; +use super::{BuiltinEncoding, CodeNames, Sfnt}; + +const STANDARD_STRINGS: NameTable = NameTable { + names: STANDARD_STRING_NAMES, + offsets: &STANDARD_STRING_OFFSETS, +}; +const STANDARD_STRING_COUNT: usize = 391; +const ISO_ADOBE_LAST_SID: u16 = 228; +const CFF_MAJOR_VERSION: u8 = 1; +const HEADER_SIZE_FIELD: usize = 2; +const MAX_OFFSET_SIZE: usize = 4; +const MAX_OPERANDS: usize = 48; +const PREDEFINED_ISO_ADOBE: usize = 0; +const PREDEFINED_EXPERT: usize = 1; +const PREDEFINED_EXPERT_SUBSET: usize = 2; +const PREDEFINED_STANDARD_ENCODING: usize = 0; +const PREDEFINED_EXPERT_ENCODING: usize = 1; +const ENCODING_FORMAT_MASK: u8 = 0x7F; +const ENCODING_SUPPLEMENT_FLAG: u8 = 0x80; + +const OP_ESCAPE: u8 = 12; +const OP_CHARSET: u16 = 15; +const OP_ENCODING: u16 = 16; +const OP_CHAR_STRINGS: u16 = 17; +const OP_ROS: u16 = 0x0C00 | 30; +const OPERAND_SHORT: u8 = 28; +const OPERAND_LONG: u8 = 29; +const OPERAND_REAL: u8 = 30; +const REAL_END_NIBBLE: u8 = 0x0F; + +#[derive(Debug, Clone)] +pub(crate) struct Cff<'x> { + data: &'x [u8], + strings: Index<'x>, + num_glyphs: u16, + charset: Charset, + encoding_offset: usize, + ros: Option<[i64; 3]>, +} + +#[cfg(all(test, stalwart_tests))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Ros<'x> { + pub(crate) registry: &'x [u8], + pub(crate) ordering: &'x [u8], + pub(crate) supplement: i64, +} + +#[derive(Debug, Clone)] +enum Charset { + IsoAdobe, + Expert, + ExpertSubset, + Identity, + Custom(Box<[u16]>), +} + +#[derive(Debug, Clone, Copy, Default)] +struct Index<'x> { + count: usize, + off_size: usize, + offsets: &'x [u8], + data: &'x [u8], +} + +#[derive(Debug, Default)] +struct TopDict { + charset: i64, + encoding: i64, + char_strings: Option, + ros: Option<[i64; 3]>, +} + +impl<'x> Cff<'x> { + pub(crate) fn parse(data: &'x [u8]) -> Option { + match Sfnt::parse(data) { + Some(sfnt) => Self::parse_bare(sfnt.table(b"CFF ")?), + None => Self::parse_bare(data), + } + } + + fn parse_bare(data: &'x [u8]) -> Option { + if data.first() != Some(&CFF_MAJOR_VERSION) { + return None; + } + let header_size = usize::from(data.be_u8(HEADER_SIZE_FIELD)?); + let (_, top_dicts_at) = Index::parse(data, header_size)?; + let (top_dicts, strings_at) = Index::parse(data, top_dicts_at)?; + let (strings, _) = Index::parse(data, strings_at).unwrap_or_default(); + let top = TopDict::parse(top_dicts.get(0)?); + let char_strings = usize::try_from(top.char_strings?).ok()?; + let num_glyphs = Index::count(data, char_strings)?; + let charset = Charset::parse( + data, + usize::try_from(top.charset).ok()?, + num_glyphs, + top.ros.is_some(), + )?; + Some(Self { + data, + strings, + num_glyphs, + charset, + encoding_offset: usize::try_from(top.encoding).unwrap_or(usize::MAX), + ros: top.ros, + }) + } + + pub(crate) fn num_glyphs(&self) -> u16 { + self.num_glyphs + } + + pub(crate) fn is_cid(&self) -> bool { + self.ros.is_some() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn ros(&self) -> Option> { + let [registry, ordering, supplement] = self.ros?; + Some(Ros { + registry: self.string(u16::try_from(registry).ok()?)?, + ordering: self.string(u16::try_from(ordering).ok()?)?, + supplement, + }) + } + + fn string(&self, sid: u16) -> Option<&'x [u8]> { + let sid = usize::from(sid); + match sid.checked_sub(STANDARD_STRING_COUNT) { + None => STANDARD_STRINGS.get(sid), + Some(index) => self.strings.get(index), + } + } + + pub(crate) fn glyph_name(&self, glyph: u16) -> Option<&'x [u8]> { + if self.is_cid() { + return None; + } + self.string(self.glyph_sid(glyph)?) + } + + pub(crate) fn glyph_cid(&self, glyph: u16) -> Option { + if !self.is_cid() { + return None; + } + self.glyph_sid(glyph) + } + + pub(crate) fn builtin_encoding(&self) -> Option> { + if self.is_cid() { + return None; + } + match self.encoding_offset { + PREDEFINED_STANDARD_ENCODING => Some(BuiltinEncoding::Standard), + PREDEFINED_EXPERT_ENCODING => Some(BuiltinEncoding::Expert), + offset => { + let mut reader = Reader::at(self.data, offset)?; + let format = reader.u8()?; + let mut names = CodeNames::new(); + self.read_encoding(&mut reader, format, &mut names); + Some(BuiltinEncoding::Custom(names)) + } + } + } + + fn read_encoding(&self, reader: &mut Reader<'x>, format: u8, names: &mut CodeNames<'x>) { + let mut glyph = 1u16; + let mut assign = |code: u8| { + if let Some(name) = self.glyph_name(glyph) { + names.set(code, name); + } + glyph = glyph.saturating_add(1); + }; + let complete = match format & ENCODING_FORMAT_MASK { + 0 => read_codes(reader, &mut assign), + 1 => read_code_ranges(reader, &mut assign), + _ => None, + }; + if complete.is_some() && format & ENCODING_SUPPLEMENT_FLAG != 0 { + self.read_supplements(reader, names); + } + } + + fn read_supplements(&self, reader: &mut Reader<'x>, names: &mut CodeNames<'x>) -> Option<()> { + for _ in 0..reader.u8()? { + let code = reader.u8()?; + let sid = reader.u16()?; + if let Some(name) = self.string(sid) { + names.set(code, name); + } + } + Some(()) + } + + fn glyph_sid(&self, glyph: u16) -> Option { + if glyph >= self.num_glyphs { + return None; + } + let Some(index) = usize::from(glyph).checked_sub(1) else { + return Some(0); + }; + match &self.charset { + Charset::IsoAdobe => (glyph <= ISO_ADOBE_LAST_SID).then_some(glyph), + Charset::Expert => EXPERT_CHARSET.get(index).copied(), + Charset::ExpertSubset => EXPERT_SUBSET_CHARSET.get(index).copied(), + Charset::Identity => Some(glyph), + Charset::Custom(sids) => sids.get(index).copied(), + } + } +} + +fn read_codes(reader: &mut Reader<'_>, assign: &mut impl FnMut(u8)) -> Option<()> { + for _ in 0..reader.u8()? { + assign(reader.u8()?); + } + Some(()) +} + +fn read_code_ranges(reader: &mut Reader<'_>, assign: &mut impl FnMut(u8)) -> Option<()> { + for _ in 0..reader.u8()? { + let first = reader.u8()?; + let left = reader.u8()?; + for code in first..=first.saturating_add(left) { + assign(code); + } + } + Some(()) +} + +impl Charset { + fn parse(data: &[u8], offset: usize, num_glyphs: u16, cid: bool) -> Option { + match offset { + PREDEFINED_ISO_ADOBE | PREDEFINED_EXPERT | PREDEFINED_EXPERT_SUBSET if cid => { + Some(Charset::Identity) + } + PREDEFINED_ISO_ADOBE => Some(Charset::IsoAdobe), + PREDEFINED_EXPERT => Some(Charset::Expert), + PREDEFINED_EXPERT_SUBSET => Some(Charset::ExpertSubset), + offset => { + let mut reader = Reader::at(data, offset)?; + let format = reader.u8()?; + let wanted = usize::from(num_glyphs.saturating_sub(1)); + let mut sids = Vec::with_capacity(wanted.min(reader.remaining().len())); + match format { + 0 => { + while sids.len() < wanted { + let Some(sid) = reader.u16() else { break }; + sids.push(sid); + } + } + 1 | 2 => { + while sids.len() < wanted { + let Some(first) = reader.u16() else { break }; + let left = if format == 1 { + reader.u8().map(u16::from) + } else { + reader.u16() + }; + let Some(left) = left else { break }; + let room = wanted - sids.len(); + let run = usize::from(left).saturating_add(1).min(room); + sids.extend((first..=u16::MAX).take(run)); + } + } + _ => return None, + } + Some(Charset::Custom(sids.into_boxed_slice())) + } + } + } +} + +impl<'x> Index<'x> { + fn parse(data: &'x [u8], at: usize) -> Option<(Self, usize)> { + let count = usize::from(data.be_u16(at)?); + if count == 0 { + return Some((Self::default(), at.checked_add(2)?)); + } + let off_size = usize::from(data.be_u8(at.checked_add(2)?)?); + if !(1..=MAX_OFFSET_SIZE).contains(&off_size) { + return None; + } + let offsets_at = at.checked_add(3)?; + let offsets = data.range(offsets_at, (count + 1) * off_size)?; + let data_at = offsets_at + offsets.len(); + let mut index = Self { + count, + off_size, + offsets, + data: &[], + }; + let data_len = index.offset(count)?.checked_sub(1)?; + let payload = data.get(data_at..)?; + index.data = payload.get(..data_len).unwrap_or(payload); + Some((index, data_at.checked_add(data_len)?)) + } + + fn count(data: &'x [u8], at: usize) -> Option { + let (index, _) = Self::parse(data, at)?; + u16::try_from(index.count).ok() + } + + fn offset(&self, index: usize) -> Option { + let bytes = self + .offsets + .range(index.checked_mul(self.off_size)?, self.off_size)?; + Some( + bytes + .iter() + .fold(0usize, |value, &byte| value << 8 | usize::from(byte)), + ) + } + + fn get(&self, index: usize) -> Option<&'x [u8]> { + let start = self.offset(index)?.checked_sub(1)?; + let end = self.offset(index.checked_add(1)?)?.checked_sub(1)?; + self.data.get(start..end) + } +} + +impl TopDict { + fn parse(dict: &[u8]) -> Self { + let mut top = TopDict::default(); + let mut operands = [0i64; MAX_OPERANDS]; + let mut len = 0usize; + let mut reader = Reader::new(dict); + while let Some(byte) = reader.u8() { + let operand = match byte { + OPERAND_SHORT => reader.u16().map(|value| i64::from(value.cast_signed())), + OPERAND_LONG => reader.u32().map(|value| i64::from(value.cast_signed())), + OPERAND_REAL => skip_real(&mut reader).map(|()| 0), + 32..=246 => Some(i64::from(byte) - 139), + 247..=250 => reader + .u8() + .map(|next| (i64::from(byte) - 247) * 256 + i64::from(next) + 108), + 251..=254 => reader + .u8() + .map(|next| -(i64::from(byte) - 251) * 256 - i64::from(next) - 108), + _ => { + let operator = if byte == OP_ESCAPE { + let Some(next) = reader.u8() else { break }; + u16::from(OP_ESCAPE) << 8 | u16::from(next) + } else { + u16::from(byte) + }; + top.apply(operator, operands.get(..len).unwrap_or_default()); + len = 0; + continue; + } + }; + let Some(operand) = operand else { break }; + if let Some(slot) = operands.get_mut(len) { + *slot = operand; + len += 1; + } + } + top + } + + fn apply(&mut self, operator: u16, operands: &[i64]) { + match (operator, operands) { + (OP_CHARSET, [.., value]) => self.charset = *value, + (OP_ENCODING, [.., value]) => self.encoding = *value, + (OP_CHAR_STRINGS, [.., value]) => self.char_strings = Some(*value), + (OP_ROS, [.., registry, ordering, supplement]) => { + self.ros = Some([*registry, *ordering, *supplement]) + } + _ => {} + } + } +} + +fn skip_real(reader: &mut Reader<'_>) -> Option<()> { + loop { + let byte = reader.u8()?; + if byte >> 4 == REAL_END_NIBBLE || byte & 0x0F == REAL_END_NIBBLE { + return Some(()); + } + } +} diff --git a/crates/text-extract/src/pdf/font/program/cff_data.rs b/crates/text-extract/src/pdf/font/program/cff_data.rs new file mode 100644 index 00000000..650ebcd1 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/cff_data.rs @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) static STANDARD_STRING_NAMES: &[u8] = b".notdefspaceexclamquotedblnumbersigndollarpercentampersandquoterightparenleftparenrightasteriskpluscommahyphenperiodslashzeroonetwothreefourfivesixseveneightninecolonsemicolonlessequalgreaterquestionatABCDEFGHIJKLMNOPQRSTUVWXYZbracketleftbackslashbracketrightasciicircumunderscorequoteleftabcdefghijklmnopqrstuvwxyzbraceleftbarbracerightasciitildeexclamdowncentsterlingfractionyenflorinsectioncurrencyquotesinglequotedblleftguillemotleftguilsinglleftguilsinglrightfiflendashdaggerdaggerdblperiodcenteredparagraphbulletquotesinglbasequotedblbasequotedblrightguillemotrightellipsisperthousandquestiondowngraveacutecircumflextildemacronbrevedotaccentdieresisringcedillahungarumlautogonekcaronemdashAEordfeminineLslashOslashOEordmasculineaedotlessilslashoslashoegermandblsonesuperiorlogicalnotmutrademarkEthonehalfplusminusThornonequarterdividebrokenbardegreethornthreequarterstwosuperiorregisteredminusethmultiplythreesuperiorcopyrightAacuteAcircumflexAdieresisAgraveAringAtildeCcedillaEacuteEcircumflexEdieresisEgraveIacuteIcircumflexIdieresisIgraveNtildeOacuteOcircumflexOdieresisOgraveOtildeScaronUacuteUcircumflexUdieresisUgraveYacuteYdieresisZcaronaacuteacircumflexadieresisagravearingatildeccedillaeacuteecircumflexedieresisegraveiacuteicircumflexidieresisigraventildeoacuteocircumflexodieresisograveotildescaronuacuteucircumflexudieresisugraveyacuteydieresiszcaronexclamsmallHungarumlautsmalldollaroldstyledollarsuperiorampersandsmallAcutesmallparenleftsuperiorparenrightsuperiortwodotenleaderonedotenleaderzerooldstyleoneoldstyletwooldstylethreeoldstylefouroldstylefiveoldstylesixoldstylesevenoldstyleeightoldstylenineoldstylecommasuperiorthreequartersemdashperiodsuperiorquestionsmallasuperiorbsuperiorcentsuperiordsuperioresuperiorisuperiorlsuperiormsuperiornsuperiorosuperiorrsuperiorssuperiortsuperiorffffifflparenleftinferiorparenrightinferiorCircumflexsmallhyphensuperiorGravesmallAsmallBsmallCsmallDsmallEsmallFsmallGsmallHsmallIsmallJsmallKsmallLsmallMsmallNsmallOsmallPsmallQsmallRsmallSsmallTsmallUsmallVsmallWsmallXsmallYsmallZsmallcolonmonetaryonefittedrupiahTildesmallexclamdownsmallcentoldstyleLslashsmallScaronsmallZcaronsmallDieresissmallBrevesmallCaronsmallDotaccentsmallMacronsmallfiguredashhypheninferiorOgoneksmallRingsmallCedillasmallquestiondownsmalloneeighththreeeighthsfiveeighthsseveneighthsonethirdtwothirdszerosuperiorfoursuperiorfivesuperiorsixsuperiorsevensuperioreightsuperiorninesuperiorzeroinferioroneinferiortwoinferiorthreeinferiorfourinferiorfiveinferiorsixinferiorseveninferioreightinferiornineinferiorcentinferiordollarinferiorperiodinferiorcommainferiorAgravesmallAacutesmallAcircumflexsmallAtildesmallAdieresissmallAringsmallAEsmallCcedillasmallEgravesmallEacutesmallEcircumflexsmallEdieresissmallIgravesmallIacutesmallIcircumflexsmallIdieresissmallEthsmallNtildesmallOgravesmallOacutesmallOcircumflexsmallOtildesmallOdieresissmallOEsmallOslashsmallUgravesmallUacutesmallUcircumflexsmallUdieresissmallYacutesmallThornsmallYdieresissmall001.000001.001001.002001.003BlackBoldBookLightMediumRegularRomanSemibold"; +pub(super) static STANDARD_STRING_OFFSETS: [u16; 392] = [ + 0, 7, 12, 18, 26, 36, 42, 49, 58, 68, 77, 87, 95, 99, 104, 110, 116, 121, 125, 128, 131, 136, + 140, 144, 147, 152, 157, 161, 166, 175, 179, 184, 191, 199, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 227, 238, 247, 259, 270, 280, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 324, 327, 337, 347, 357, + 361, 369, 377, 380, 386, 393, 401, 412, 424, 437, 450, 464, 466, 468, 474, 480, 489, 503, 512, + 518, 532, 544, 557, 571, 579, 590, 602, 607, 612, 622, 627, 633, 638, 647, 655, 659, 666, 678, + 684, 689, 695, 697, 708, 714, 720, 722, 734, 736, 744, 750, 756, 758, 768, 779, 789, 791, 800, + 803, 810, 819, 824, 834, 840, 849, 855, 860, 873, 884, 894, 899, 902, 910, 923, 932, 938, 949, + 958, 964, 969, 975, 983, 989, 1000, 1009, 1015, 1021, 1032, 1041, 1047, 1053, 1059, 1070, 1079, + 1085, 1091, 1097, 1103, 1114, 1123, 1129, 1135, 1144, 1150, 1156, 1167, 1176, 1182, 1187, 1193, + 1201, 1207, 1218, 1227, 1233, 1239, 1250, 1259, 1265, 1271, 1277, 1288, 1297, 1303, 1309, 1315, + 1321, 1332, 1341, 1347, 1353, 1362, 1368, 1379, 1396, 1410, 1424, 1438, 1448, 1465, 1483, 1497, + 1511, 1523, 1534, 1545, 1558, 1570, 1582, 1593, 1606, 1619, 1631, 1644, 1663, 1677, 1690, 1699, + 1708, 1720, 1729, 1738, 1747, 1756, 1765, 1774, 1783, 1792, 1801, 1810, 1812, 1815, 1818, 1835, + 1853, 1868, 1882, 1892, 1898, 1904, 1910, 1916, 1922, 1928, 1934, 1940, 1946, 1952, 1958, 1964, + 1970, 1976, 1982, 1988, 1994, 2000, 2006, 2012, 2018, 2024, 2030, 2036, 2042, 2048, 2061, 2070, + 2076, 2086, 2101, 2113, 2124, 2135, 2146, 2159, 2169, 2179, 2193, 2204, 2214, 2228, 2239, 2248, + 2260, 2277, 2286, 2298, 2309, 2321, 2329, 2338, 2350, 2362, 2374, 2385, 2398, 2411, 2423, 2435, + 2446, 2457, 2470, 2482, 2494, 2505, 2518, 2531, 2543, 2555, 2569, 2583, 2596, 2607, 2618, 2634, + 2645, 2659, 2669, 2676, 2689, 2700, 2711, 2727, 2741, 2752, 2763, 2779, 2793, 2801, 2812, 2823, + 2834, 2850, 2861, 2875, 2882, 2893, 2904, 2915, 2931, 2945, 2956, 2966, 2980, 2987, 2994, 3001, + 3008, 3013, 3017, 3021, 3026, 3032, 3039, 3044, 3052, +]; + +pub(super) static EXPERT_CHARSET: [u16; 165] = [ + 1, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 13, 14, 15, 99, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 27, 28, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 109, 110, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 158, 155, 163, 319, 320, 321, 322, 323, 324, 325, 326, 150, 164, 169, 327, 328, + 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, +]; + +pub(super) static EXPERT_SUBSET_CHARSET: [u16; 86] = [ + 1, 231, 232, 235, 236, 237, 238, 13, 14, 15, 99, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 27, 28, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 109, 110, 267, 268, 269, 270, 272, 300, 301, 302, 305, 314, 315, 158, 155, 163, 320, 321, + 322, 323, 324, 325, 326, 150, 164, 169, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, + 338, 339, 340, 341, 342, 343, 344, 345, 346, +]; diff --git a/crates/text-extract/src/pdf/font/program/cmap.rs b/crates/text-extract/src/pdf/font/program/cmap.rs new file mode 100644 index 00000000..52324cfd --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/cmap.rs @@ -0,0 +1,395 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::read::ReadBytes; + +const RECORDS_OFFSET: usize = 4; +const RECORD_LEN: usize = 8; +const RECORD_SUBTABLE_FIELD: usize = 4; +const FORMAT0_GLYPHS: usize = 6; +const FORMAT0_LEN: usize = 256; +const FORMAT2_KEYS: usize = 6; +const FORMAT2_KEY_COUNT: usize = 256; +const FORMAT2_SUBHEADERS: usize = FORMAT2_KEYS + 2 * FORMAT2_KEY_COUNT; +const FORMAT2_SUBHEADER_LEN: usize = 8; +const FORMAT2_RANGE_FIELD: usize = 6; +const FORMAT4_SEG_COUNT_X2: usize = 6; +const FORMAT4_END_CODES: usize = 14; +const FORMAT4_RESERVED_PAD: usize = 2; +const FORMAT6_FIRST_CODE: usize = 6; +const FORMAT6_GLYPHS: usize = 10; +const FORMAT12_LENGTH: usize = 4; +const FORMAT12_GROUP_COUNT: usize = 12; +const FORMAT12_GROUPS: usize = 16; +const FORMAT12_GROUP_LEN: usize = 12; +const SHORT_LENGTH_FIELD: usize = 2; +const MAX_CODE_POINT: u32 = 0x10_FFFF; +const BYTE_CODES: u32 = 256; +const BYTE_BITS: u32 = 8; +const BYTE_MASK: u32 = 0xFF; +const FORMAT2_KEY_SCALE: u16 = 8; + +pub(crate) const PLATFORM_UNICODE: u16 = 0; +pub(crate) const PLATFORM_MAC: u16 = 1; +pub(crate) const PLATFORM_WINDOWS: u16 = 3; +pub(crate) const WINDOWS_SYMBOL: u16 = 0; +pub(crate) const WINDOWS_UNICODE_BMP: u16 = 1; +pub(crate) const WINDOWS_UNICODE_FULL: u16 = 10; +pub(crate) const MAC_ROMAN: u16 = 0; + +const UNICODE_PREFERENCE: [(u16, u16); 7] = [ + (PLATFORM_WINDOWS, WINDOWS_UNICODE_FULL), + (PLATFORM_UNICODE, 4), + (PLATFORM_WINDOWS, WINDOWS_UNICODE_BMP), + (PLATFORM_UNICODE, 3), + (PLATFORM_UNICODE, 2), + (PLATFORM_UNICODE, 1), + (PLATFORM_UNICODE, 0), +]; + +type U16Be = [u8; 2]; +type Group = [u8; FORMAT12_GROUP_LEN]; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Cmap<'x> { + data: &'x [u8], + records: &'x [[u8; RECORD_LEN]], +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct CmapSubtable<'x> { + format: Format<'x>, +} + +#[derive(Debug, Clone, Copy)] +enum Format<'x> { + ByteEncoding(&'x [u8]), + HighByte(&'x [u8]), + SegmentDelta(SegmentDelta<'x>), + Trimmed { first: u32, glyphs: &'x [U16Be] }, + SegmentedCoverage(&'x [Group]), +} + +struct Segment { + start: u16, + delta: u16, + range_at: usize, + range: usize, +} + +struct SubHeader<'x> { + data: &'x [u8], + first: u32, + count: u32, + delta: u16, + range: usize, +} + +#[derive(Debug, Clone, Copy)] +struct SegmentDelta<'x> { + ends: &'x [U16Be], + starts: &'x [U16Be], + deltas: &'x [U16Be], + ranges: &'x [u8], +} + +impl<'x> Cmap<'x> { + pub(crate) fn parse(data: &'x [u8]) -> Option { + let count = usize::from(data.be_u16(2)?); + let (records, _) = data.get(RECORDS_OFFSET..)?.as_chunks::(); + let records = records.get(..count).unwrap_or(records); + Some(Self { data, records }) + } + + pub(crate) fn subtables(&self) -> impl Iterator)> + '_ { + self.records.iter().filter_map(|record| { + let platform = record.be_u16(0)?; + let encoding = record.be_u16(2)?; + let offset = usize::try_from(record.be_u32(RECORD_SUBTABLE_FIELD)?).ok()?; + let subtable = CmapSubtable::parse(self.data.get(offset..)?)?; + Some((platform, encoding, subtable)) + }) + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn subtable(&self, platform: u16, encoding: u16) -> Option> { + self.subtables() + .find(|&(p, e, _)| p == platform && e == encoding) + .map(|(_, _, subtable)| subtable) + } + + pub(crate) fn unicode(&self) -> Option> { + let mut best: Option<(usize, CmapSubtable<'x>)> = None; + for (platform, encoding, subtable) in self.subtables() { + let Some(rank) = UNICODE_PREFERENCE + .iter() + .position(|&key| key == (platform, encoding)) + else { + continue; + }; + if best.is_none_or(|(best_rank, _)| rank < best_rank) { + best = Some((rank, subtable)); + if rank == 0 { + break; + } + } + } + best.map(|(_, subtable)| subtable) + } +} + +impl<'x> CmapSubtable<'x> { + fn parse(data: &'x [u8]) -> Option { + let format = match data.be_u16(0)? { + 0 => { + let glyphs = bounded(data, SHORT_LENGTH_FIELD).get(FORMAT0_GLYPHS..)?; + Format::ByteEncoding(glyphs.get(..FORMAT0_LEN).unwrap_or(glyphs)) + } + 2 => Format::HighByte(bounded(data, SHORT_LENGTH_FIELD)), + 4 => Format::SegmentDelta(SegmentDelta::parse(bounded(data, SHORT_LENGTH_FIELD))?), + 6 => { + let data = bounded(data, SHORT_LENGTH_FIELD); + let first = u32::from(data.be_u16(FORMAT6_FIRST_CODE)?); + let count = usize::from(data.be_u16(FORMAT6_FIRST_CODE + 2)?); + let glyphs = chunks::<2>(data, FORMAT6_GLYPHS, count)?; + Format::Trimmed { first, glyphs } + } + 12 => { + let len = usize::try_from(data.be_u32(FORMAT12_LENGTH)?).ok()?; + let data = data.get(..len).unwrap_or(data); + let count = usize::try_from(data.be_u32(FORMAT12_GROUP_COUNT)?).ok()?; + Format::SegmentedCoverage(chunks::( + data, + FORMAT12_GROUPS, + count, + )?) + } + _ => return None, + }; + Some(Self { format }) + } + + pub(crate) fn glyph(&self, code: u32) -> Option { + let glyph = match self.format { + Format::ByteEncoding(glyphs) => u16::from(*glyphs.get(usize::try_from(code).ok()?)?), + Format::HighByte(data) => high_byte_glyph(data, code)?, + Format::SegmentDelta(table) => { + let code = u16::try_from(code).ok()?; + let index = table + .ends + .partition_point(|&end| u16::from_be_bytes(end) < code); + table.segment(index)?.glyph(table.ranges, code)? + } + Format::Trimmed { first, glyphs } => { + let index = usize::try_from(code.checked_sub(first)?).ok()?; + u16::from_be_bytes(*glyphs.get(index)?) + } + Format::SegmentedCoverage(groups) => { + let index = groups.partition_point(|group| group_end(group) < code); + let group = groups.get(index)?; + let start = group.be_u32(0)?; + let offset = code.checked_sub(start)?; + u16::try_from(group.be_u32(8)?.checked_add(offset)?).ok()? + } + }; + (glyph != 0).then_some(glyph) + } + + pub(crate) fn for_each_mapping(&self, glyph_limit: u32, mut f: impl FnMut(u32, u16)) { + let mut emit = |code: u32, glyph: u16| { + if glyph != 0 && u32::from(glyph) < glyph_limit { + f(code, glyph); + } + }; + match self.format { + Format::ByteEncoding(glyphs) => { + for (code, &glyph) in (0u32..).zip(glyphs) { + emit(code, u16::from(glyph)); + } + } + Format::HighByte(data) => { + for (high, key) in (0u32..).zip(high_byte_keys(data)) { + let key = u16::from_be_bytes(*key); + let Some(subheader) = SubHeader::parse(data, key) else { + continue; + }; + if key == 0 { + if let Some(glyph) = subheader.glyph(high) { + emit(high, glyph); + } + continue; + } + let end = subheader + .first + .saturating_add(subheader.count) + .min(BYTE_CODES); + for low in subheader.first..end { + if let Some(glyph) = subheader.glyph(low) { + emit(high << BYTE_BITS | low, glyph); + } + } + } + } + Format::SegmentDelta(table) => { + let mut next_code = 0u32; + for (index, &end) in table.ends.iter().enumerate() { + let Some(segment) = table.segment(index) else { + break; + }; + let start = u32::from(segment.start).max(next_code); + let end = u32::from(u16::from_be_bytes(end)); + if start > end { + continue; + } + next_code = end + 1; + for code in (start..=end).filter_map(|code| u16::try_from(code).ok()) { + if let Some(glyph) = segment.glyph(table.ranges, code) { + emit(u32::from(code), glyph); + } + } + } + } + Format::Trimmed { first, glyphs } => { + for (code, glyph) in (first..).zip(glyphs) { + emit(code, u16::from_be_bytes(*glyph)); + } + } + Format::SegmentedCoverage(groups) => { + let mut next_code = 0u32; + for group in groups { + let (Some(group_start), Some(end), Some(first_glyph)) = + (group.be_u32(0), group.be_u32(4), group.be_u32(8)) + else { + continue; + }; + let start = group_start.max(next_code); + let end = end.min(MAX_CODE_POINT); + if start > end { + continue; + } + next_code = end + 1; + let Some(first_glyph) = first_glyph.checked_add(start - group_start) else { + continue; + }; + let Some(room) = glyph_limit.checked_sub(first_glyph) else { + continue; + }; + let last = end.min(start.saturating_add(room.saturating_sub(1))); + for (code, glyph) in (start..=last).zip(first_glyph..) { + if let Ok(glyph) = u16::try_from(glyph) { + emit(code, glyph); + } + } + } + } + } + } +} + +impl<'x> SegmentDelta<'x> { + fn parse(data: &'x [u8]) -> Option { + let seg_count_x2 = usize::from(data.be_u16(FORMAT4_SEG_COUNT_X2)?); + let seg_count = seg_count_x2 / 2; + let starts_at = FORMAT4_END_CODES + seg_count_x2 + FORMAT4_RESERVED_PAD; + let deltas_at = starts_at + seg_count_x2; + let ranges_at = deltas_at + seg_count_x2; + Some(Self { + ends: chunks::<2>(data, FORMAT4_END_CODES, seg_count)?, + starts: chunks::<2>(data, starts_at, seg_count).unwrap_or_default(), + deltas: chunks::<2>(data, deltas_at, seg_count).unwrap_or_default(), + ranges: data.get(ranges_at..).unwrap_or_default(), + }) + } + + fn segment(&self, index: usize) -> Option { + let range_at = index.checked_mul(2)?; + Some(Segment { + start: u16::from_be_bytes(*self.starts.get(index)?), + delta: u16::from_be_bytes(*self.deltas.get(index)?), + range_at, + range: usize::from(self.ranges.be_u16(range_at)?), + }) + } +} + +impl Segment { + fn glyph(&self, ranges: &[u8], code: u16) -> Option { + let offset = code.checked_sub(self.start)?; + if self.range == 0 { + return Some(code.wrapping_add(self.delta)); + } + let at = self + .range_at + .checked_add(self.range)? + .checked_add(usize::from(offset) * 2)?; + Some(apply_delta(ranges.be_u16(at)?, self.delta)) + } +} + +fn bounded(data: &[u8], length_field: usize) -> &[u8] { + data.be_u16(length_field) + .and_then(|len| data.get(..usize::from(len))) + .unwrap_or(data) +} + +fn chunks(data: &[u8], at: usize, count: usize) -> Option<&[[u8; N]]> { + let (chunks, _) = data.get(at..)?.as_chunks::(); + Some(chunks.get(..count).unwrap_or(chunks)) +} + +fn group_end(group: &Group) -> u32 { + group.be_u32(4).unwrap_or(0) +} + +fn high_byte_keys(data: &[u8]) -> &[U16Be] { + chunks::<2>(data, FORMAT2_KEYS, FORMAT2_KEY_COUNT).unwrap_or_default() +} + +impl<'x> SubHeader<'x> { + fn parse(data: &'x [u8], key: u16) -> Option { + let at = FORMAT2_SUBHEADERS + usize::from(key / FORMAT2_KEY_SCALE) * FORMAT2_SUBHEADER_LEN; + let data = data.get(at..)?; + Some(Self { + data, + first: u32::from(data.be_u16(0)?), + count: u32::from(data.be_u16(2)?), + delta: data.be_u16(4)?, + range: usize::from(data.be_u16(FORMAT2_RANGE_FIELD)?), + }) + } + + fn glyph(&self, low: u32) -> Option { + let index = low + .checked_sub(self.first) + .filter(|&index| index < self.count)?; + let at = FORMAT2_RANGE_FIELD + .checked_add(self.range)? + .checked_add(usize::try_from(index).ok()? * 2)?; + Some(apply_delta(self.data.be_u16(at)?, self.delta)) + } +} + +fn apply_delta(glyph: u16, delta: u16) -> u16 { + if glyph == 0 { + 0 + } else { + glyph.wrapping_add(delta) + } +} + +fn high_byte_glyph(data: &[u8], code: u32) -> Option { + let keys = high_byte_keys(data); + let (high, low) = if code < BYTE_CODES { + (code, code) + } else { + (code >> BYTE_BITS, code & BYTE_MASK) + }; + let key = u16::from_be_bytes(*keys.get(usize::try_from(high).ok()?)?); + if (key == 0) != (code < BYTE_CODES) { + return None; + } + SubHeader::parse(data, key)?.glyph(low) +} diff --git a/crates/text-extract/src/pdf/font/program/mod.rs b/crates/text-extract/src/pdf/font/program/mod.rs new file mode 100644 index 00000000..cb2d5e57 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/mod.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod cff; +mod cff_data; +mod cmap; +mod post; +mod post_data; +mod read; +mod sfnt; +mod truetype; +mod type1; + + +pub(crate) use self::cff::Cff; +pub(crate) use self::cmap::{Cmap, CmapSubtable}; +pub(crate) use self::post::Post; +pub(crate) use self::sfnt::Sfnt; +pub(crate) use self::truetype::TrueType; + +const CODE_COUNT: usize = 256; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum BuiltinEncoding<'x> { + Standard, + Expert, + Custom(CodeNames<'x>), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CodeNames<'x> { + names: Box<[Option<&'x [u8]>; CODE_COUNT]>, +} + +impl<'x> CodeNames<'x> { + fn new() -> Self { + Self { + names: Box::new([None; CODE_COUNT]), + } + } + + fn set(&mut self, code: u8, name: &'x [u8]) { + if let Some(slot) = self.names.get_mut(usize::from(code)) { + *slot = Some(name); + } + } + + pub(crate) fn get(&self, code: u8) -> Option<&'x [u8]> { + self.names.get(usize::from(code)).copied().flatten() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn iter(&self) -> impl Iterator + '_ { + (0..=u8::MAX) + .zip(self.names.iter()) + .filter_map(|(code, name)| name.map(|name| (code, name))) + } + + pub(crate) fn is_empty(&self) -> bool { + self.names.iter().all(Option::is_none) + } +} diff --git a/crates/text-extract/src/pdf/font/program/post.rs b/crates/text-extract/src/pdf/font/program/post.rs new file mode 100644 index 00000000..68a64239 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/post.rs @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::post_data::{MAC_GLYPH_NAMES, MAC_GLYPH_OFFSETS}; +use super::read::{NameTable, ReadBytes, Reader}; + +const VERSION_1: u32 = 0x0001_0000; +const VERSION_2: u32 = 0x0002_0000; +const VERSION_2_5: u32 = 0x0002_5000; +const HEADER_LEN: usize = 32; +const MAC_GLYPH_COUNT: usize = 258; + +const MAC_GLYPHS: NameTable = NameTable { + names: MAC_GLYPH_NAMES, + offsets: &MAC_GLYPH_OFFSETS, +}; + +#[derive(Debug, Clone)] +pub(crate) struct Post<'x> { + names: PostNames<'x>, +} + +#[derive(Debug, Clone)] +enum PostNames<'x> { + Standard, + Indexed { + indices: &'x [[u8; 2]], + strings: &'x [u8], + offsets: Box<[u32]>, + }, + Shifted(&'x [u8]), +} + +impl<'x> Post<'x> { + pub(crate) fn parse(data: &'x [u8]) -> Option { + let names = match data.be_u32(0)? { + VERSION_1 => PostNames::Standard, + VERSION_2 => { + let mut reader = Reader::at(data, HEADER_LEN)?; + let count = usize::from(reader.u16()?); + let (indices, _) = reader.remaining().as_chunks::<2>(); + let indices = indices.get(..count).unwrap_or(indices); + let strings = reader + .remaining() + .get(indices.len() * 2..) + .unwrap_or_default(); + PostNames::Indexed { + indices, + strings, + offsets: pascal_string_offsets(strings), + } + } + VERSION_2_5 => { + let mut reader = Reader::at(data, HEADER_LEN)?; + let count = usize::from(reader.u16()?); + let offsets = reader.remaining(); + PostNames::Shifted(offsets.get(..count).unwrap_or(offsets)) + } + _ => return None, + }; + Some(Self { names }) + } + + pub(crate) fn glyph_name(&self, glyph: u16) -> Option<&'x [u8]> { + let glyph = usize::from(glyph); + match &self.names { + PostNames::Standard => MAC_GLYPHS.get(glyph), + PostNames::Indexed { + indices, + strings, + offsets, + } => { + let index = usize::from(u16::from_be_bytes(*indices.get(glyph)?)); + match index.checked_sub(MAC_GLYPH_COUNT) { + None => MAC_GLYPHS.get(index), + Some(string) => { + let at = usize::try_from(*offsets.get(string)?).ok()?; + let (&len, rest) = strings.get(at..)?.split_first()?; + rest.get(..usize::from(len)) + } + } + } + PostNames::Shifted(offsets) => { + let shift = i8::from_be_bytes([*offsets.get(glyph)?]); + let index = glyph.checked_add_signed(isize::from(shift))?; + MAC_GLYPHS.get(index) + } + } + } +} + +fn pascal_string_offsets(strings: &[u8]) -> Box<[u32]> { + let mut offsets = Vec::new(); + let mut at = 0usize; + while let Some(&len) = strings.get(at) { + let Ok(offset) = u32::try_from(at) else { + break; + }; + offsets.push(offset); + at += usize::from(len) + 1; + } + offsets.into_boxed_slice() +} diff --git a/crates/text-extract/src/pdf/font/program/post_data.rs b/crates/text-extract/src/pdf/font/program/post_data.rs new file mode 100644 index 00000000..2dd07739 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/post_data.rs @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) static MAC_GLYPH_NAMES: &[u8] = b".notdef.nullnonmarkingreturnspaceexclamquotedblnumbersigndollarpercentampersandquotesingleparenleftparenrightasteriskpluscommahyphenperiodslashzeroonetwothreefourfivesixseveneightninecolonsemicolonlessequalgreaterquestionatABCDEFGHIJKLMNOPQRSTUVWXYZbracketleftbackslashbracketrightasciicircumunderscoregraveabcdefghijklmnopqrstuvwxyzbraceleftbarbracerightasciitildeAdieresisAringCcedillaEacuteNtildeOdieresisUdieresisaacuteagraveacircumflexadieresisatildearingccedillaeacuteegraveecircumflexedieresisiacuteigraveicircumflexidieresisntildeoacuteograveocircumflexodieresisotildeuacuteugraveucircumflexudieresisdaggerdegreecentsterlingsectionbulletparagraphgermandblsregisteredcopyrighttrademarkacutedieresisnotequalAEOslashinfinityplusminuslessequalgreaterequalyenmupartialdiffsummationproductpiintegralordfeminineordmasculineOmegaaeoslashquestiondownexclamdownlogicalnotradicalflorinapproxequalDeltaguillemotleftguillemotrightellipsisnonbreakingspaceAgraveAtildeOtildeOEoeendashemdashquotedblleftquotedblrightquoteleftquoterightdividelozengeydieresisYdieresisfractioncurrencyguilsinglleftguilsinglrightfifldaggerdblperiodcenteredquotesinglbasequotedblbaseperthousandAcircumflexEcircumflexAacuteEdieresisEgraveIacuteIcircumflexIdieresisIgraveOacuteOcircumflexappleOgraveUacuteUcircumflexUgravedotlessicircumflextildemacronbrevedotaccentringcedillahungarumlautogonekcaronLslashlslashScaronscaronZcaronzcaronbrokenbarEthethYacuteyacuteThornthornminusmultiplyonesuperiortwosuperiorthreesuperioronehalfonequarterthreequartersfrancGbrevegbreveIdotaccentScedillascedillaCacutecacuteCcaronccarondcroat"; +pub(super) static MAC_GLYPH_OFFSETS: [u16; 259] = [ + 0, 7, 12, 28, 33, 39, 47, 57, 63, 70, 79, 90, 99, 109, 117, 121, 126, 132, 138, 143, 147, 150, + 153, 158, 162, 166, 169, 174, 179, 183, 188, 197, 201, 206, 213, 221, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 260, 269, 281, 292, 302, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 342, 345, 355, + 365, 374, 379, 387, 393, 399, 408, 417, 423, 429, 440, 449, 455, 460, 468, 474, 480, 491, 500, + 506, 512, 523, 532, 538, 544, 550, 561, 570, 576, 582, 588, 599, 608, 614, 620, 624, 632, 639, + 645, 654, 664, 674, 683, 692, 697, 705, 713, 715, 721, 729, 738, 747, 759, 762, 764, 775, 784, + 791, 793, 801, 812, 824, 829, 831, 837, 849, 859, 869, 876, 882, 893, 898, 911, 925, 933, 949, + 955, 961, 967, 969, 971, 977, 983, 995, 1008, 1017, 1027, 1033, 1040, 1049, 1058, 1066, 1074, + 1087, 1101, 1103, 1105, 1114, 1128, 1142, 1154, 1165, 1176, 1187, 1193, 1202, 1208, 1214, 1225, + 1234, 1240, 1246, 1257, 1262, 1268, 1274, 1285, 1291, 1299, 1309, 1314, 1320, 1325, 1334, 1338, + 1345, 1357, 1363, 1368, 1374, 1380, 1386, 1392, 1398, 1404, 1413, 1416, 1419, 1425, 1431, 1436, + 1441, 1446, 1454, 1465, 1476, 1489, 1496, 1506, 1519, 1524, 1530, 1536, 1546, 1554, 1562, 1568, + 1574, 1580, 1586, 1592, +]; diff --git a/crates/text-extract/src/pdf/font/program/read.rs b/crates/text-extract/src/pdf/font/program/read.rs new file mode 100644 index 00000000..a16cf12e --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/read.rs @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) trait ReadBytes { + fn be_u8(&self, at: usize) -> Option; + fn be_u16(&self, at: usize) -> Option; + fn be_u32(&self, at: usize) -> Option; + fn range(&self, at: usize, len: usize) -> Option<&[u8]>; +} + +impl ReadBytes for [u8] { + fn be_u8(&self, at: usize) -> Option { + self.get(at).copied() + } + + fn be_u16(&self, at: usize) -> Option { + self.get(at..)? + .first_chunk::<2>() + .map(|bytes| u16::from_be_bytes(*bytes)) + } + + fn be_u32(&self, at: usize) -> Option { + self.get(at..)? + .first_chunk::<4>() + .map(|bytes| u32::from_be_bytes(*bytes)) + } + + fn range(&self, at: usize, len: usize) -> Option<&[u8]> { + self.get(at..at.checked_add(len)?) + } +} + +pub(super) struct Reader<'x> { + data: &'x [u8], +} + +impl<'x> Reader<'x> { + pub(super) fn new(data: &'x [u8]) -> Self { + Self { data } + } + + pub(super) fn at(data: &'x [u8], offset: usize) -> Option { + data.get(offset..).map(Self::new) + } + + pub(super) fn u8(&mut self) -> Option { + let (&byte, rest) = self.data.split_first()?; + self.data = rest; + Some(byte) + } + + pub(super) fn u16(&mut self) -> Option { + let (bytes, rest) = self.data.split_first_chunk::<2>()?; + self.data = rest; + Some(u16::from_be_bytes(*bytes)) + } + + pub(super) fn u32(&mut self) -> Option { + let (bytes, rest) = self.data.split_first_chunk::<4>()?; + self.data = rest; + Some(u32::from_be_bytes(*bytes)) + } + + pub(super) fn remaining(&self) -> &'x [u8] { + self.data + } +} + +pub(super) struct NameTable { + pub(super) names: &'static [u8], + pub(super) offsets: &'static [u16], +} + +impl NameTable { + pub(super) fn get(&self, index: usize) -> Option<&'static [u8]> { + let &[start, end] = self.offsets.get(index..)?.first_chunk::<2>()?; + self.names.get(usize::from(start)..usize::from(end)) + } +} diff --git a/crates/text-extract/src/pdf/font/program/sfnt.rs b/crates/text-extract/src/pdf/font/program/sfnt.rs new file mode 100644 index 00000000..25f9f52d --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/sfnt.rs @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::read::ReadBytes; + +const COLLECTION_TAG: &[u8; 4] = b"ttcf"; +const COLLECTION_FIRST_OFFSET: usize = 12; +const NUM_TABLES_OFFSET: usize = 4; +const RECORDS_OFFSET: usize = 12; +const RECORD_LEN: usize = 16; +const RECORD_OFFSET_FIELD: usize = 8; +const RECORD_LENGTH_FIELD: usize = 12; +const VERSION_TRUETYPE: u32 = 0x0001_0000; +const VERSION_OPEN_TYPE: &[u8; 4] = b"OTTO"; +const VERSION_APPLE: &[u8; 4] = b"true"; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Sfnt<'x> { + data: &'x [u8], + records: &'x [[u8; RECORD_LEN]], +} + +impl<'x> Sfnt<'x> { + pub(crate) fn parse(data: &'x [u8]) -> Option { + let offset = if data.first_chunk::<4>() == Some(COLLECTION_TAG) { + usize::try_from(data.be_u32(COLLECTION_FIRST_OFFSET)?).ok()? + } else { + 0 + }; + let header = data.get(offset..)?; + let version = header.first_chunk::<4>()?; + if u32::from_be_bytes(*version) != VERSION_TRUETYPE + && version != VERSION_OPEN_TYPE + && version != VERSION_APPLE + { + return None; + } + let num_tables = usize::from(header.be_u16(NUM_TABLES_OFFSET)?); + let (records, _) = header.get(RECORDS_OFFSET..)?.as_chunks::(); + let records = records.get(..num_tables).unwrap_or(records); + Some(Self { data, records }) + } + + pub(crate) fn table(&self, tag: &[u8; 4]) -> Option<&'x [u8]> { + let record = self + .records + .iter() + .find(|record| record.first_chunk::<4>() == Some(tag))?; + let offset = usize::try_from(record.be_u32(RECORD_OFFSET_FIELD)?).ok()?; + let len = usize::try_from(record.be_u32(RECORD_LENGTH_FIELD)?).ok()?; + let table = self.data.get(offset..)?; + Some(table.get(..len).unwrap_or(table)) + } +} diff --git a/crates/text-extract/src/pdf/font/program/truetype.rs b/crates/text-extract/src/pdf/font/program/truetype.rs new file mode 100644 index 00000000..eb842552 --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/truetype.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::cmap::{MAC_ROMAN, PLATFORM_MAC, PLATFORM_WINDOWS, WINDOWS_SYMBOL}; +use super::read::ReadBytes; +use super::{Cmap, CmapSubtable, Post, Sfnt}; + +const MAXP_NUM_GLYPHS: usize = 4; +const GLYPH_SPACE: u32 = 0x1_0000; +const SYMBOL_PAGES: [u32; 4] = [0x0000, 0xF000, 0xF100, 0xF200]; +const NO_CODE_POINT: u32 = u32::MAX; + +#[derive(Debug, Clone)] +pub(crate) struct TrueType<'x> { + num_glyphs: Option, + #[cfg(all(test, stalwart_tests))] + cmap: Option>, + unicode: Option>, + symbol: Option>, + mac_roman: Option>, + post: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct GlyphUnicodeMap { + glyphs: Box<[u16]>, + code_points: Box<[char]>, +} + +impl<'x> TrueType<'x> { + pub(crate) fn parse(data: &'x [u8]) -> Option { + let sfnt = Sfnt::parse(data)?; + let num_glyphs = sfnt + .table(b"maxp") + .and_then(|maxp| maxp.be_u16(MAXP_NUM_GLYPHS)); + let cmap = sfnt.table(b"cmap").and_then(Cmap::parse); + let mut unicode = None; + let mut symbol = None; + let mut mac_roman = None; + if let Some(cmap) = &cmap { + unicode = cmap.unicode(); + for (platform, encoding, subtable) in cmap.subtables() { + match (platform, encoding) { + (PLATFORM_WINDOWS, WINDOWS_SYMBOL) if symbol.is_none() => { + symbol = Some(subtable) + } + (PLATFORM_MAC, MAC_ROMAN) if mac_roman.is_none() => mac_roman = Some(subtable), + _ => {} + } + } + } + Some(Self { + num_glyphs, + #[cfg(all(test, stalwart_tests))] + cmap, + unicode, + symbol, + mac_roman, + post: sfnt.table(b"post").and_then(Post::parse), + }) + } + + pub(crate) fn num_glyphs(&self) -> Option { + self.num_glyphs + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn cmap(&self) -> Option<&Cmap<'x>> { + self.cmap.as_ref() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn unicode_cmap(&self) -> Option<&CmapSubtable<'x>> { + self.unicode.as_ref() + } + + pub(crate) fn glyph_name(&self, glyph: u16) -> Option<&'x [u8]> { + if !self.contains(glyph) { + return None; + } + self.post.as_ref()?.glyph_name(glyph) + } + + pub(crate) fn symbolic_glyph(&self, code: u8) -> Option { + let code = u32::from(code); + self.symbol + .as_ref() + .and_then(|symbol| { + SYMBOL_PAGES + .iter() + .find_map(|page| symbol.glyph(page | code)) + }) + .or_else(|| self.mac_roman.as_ref()?.glyph(code)) + .filter(|&glyph| self.contains(glyph)) + } + + pub(crate) fn unicode_map(&self) -> GlyphUnicodeMap { + let Some(subtable) = &self.unicode else { + return GlyphUnicodeMap::default(); + }; + let limit = self.num_glyphs.map_or(GLYPH_SPACE, u32::from); + let mut dense = vec![NO_CODE_POINT; usize::try_from(limit).unwrap_or_default()]; + subtable.for_each_mapping(limit, |code, glyph| { + if char::from_u32(code).is_none() { + return; + } + if let Some(slot) = dense.get_mut(usize::from(glyph)) + && (*slot == NO_CODE_POINT || rank(code) < rank(*slot)) + { + *slot = code; + } + }); + let mapped = dense.iter().filter(|&&code| code != NO_CODE_POINT).count(); + let mut glyphs = Vec::with_capacity(mapped); + let mut code_points = Vec::with_capacity(mapped); + for (glyph, code) in dense.into_iter().enumerate() { + if let (Ok(glyph), Some(code)) = (u16::try_from(glyph), char::from_u32(code)) { + glyphs.push(glyph); + code_points.push(code); + } + } + GlyphUnicodeMap { + glyphs: glyphs.into_boxed_slice(), + code_points: code_points.into_boxed_slice(), + } + } + + fn contains(&self, glyph: u16) -> bool { + self.num_glyphs.is_none_or(|count| glyph < count) + } +} + +fn rank(code: u32) -> (bool, u32) { + let private_use = matches!(code, 0xE000..=0xF8FF | 0xF_0000..); + (private_use, code) +} + +impl GlyphUnicodeMap { + #[cfg(all(test, stalwart_tests))] + pub(crate) fn get(&self, glyph: u16) -> Option { + let index = self.glyphs.binary_search(&glyph).ok()?; + self.code_points.get(index).copied() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn len(&self) -> usize { + self.glyphs.len() + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn is_empty(&self) -> bool { + self.glyphs.is_empty() + } + + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.glyphs + .iter() + .copied() + .zip(self.code_points.iter().copied()) + } +} diff --git a/crates/text-extract/src/pdf/font/program/type1.rs b/crates/text-extract/src/pdf/font/program/type1.rs new file mode 100644 index 00000000..c90f5c9f --- /dev/null +++ b/crates/text-extract/src/pdf/font/program/type1.rs @@ -0,0 +1,222 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BuiltinEncoding, CodeNames}; + +const PFB_MARKER: u8 = 0x80; +const PFB_ASCII: u8 = 1; +const PFB_BINARY: u8 = 2; +const PFB_HEADER_LEN: usize = 6; +const MAX_RADIX: u32 = 36; + +impl<'x> BuiltinEncoding<'x> { + pub(crate) fn from_type1(program: &'x [u8]) -> Option { + if program.first() == Some(&PFB_MARKER) { + PfbSegments { data: program }.find_map(parse_cleartext) + } else { + parse_cleartext(program) + } + } +} + +struct PfbSegments<'x> { + data: &'x [u8], +} + +impl<'x> Iterator for PfbSegments<'x> { + type Item = &'x [u8]; + + fn next(&mut self) -> Option<&'x [u8]> { + loop { + let (header, rest) = self.data.split_first_chunk::()?; + let [PFB_MARKER, kind, len @ ..] = *header else { + return None; + }; + if kind != PFB_ASCII && kind != PFB_BINARY { + return None; + } + let len = usize::try_from(u32::from_le_bytes(len)).ok()?; + let (segment, rest) = rest.split_at_checked(len).unwrap_or((rest, &[])); + self.data = rest; + if kind == PFB_ASCII { + return Some(segment); + } + } + } +} + +fn parse_cleartext(data: &[u8]) -> Option> { + let mut tokens = Tokens { data }; + let mut encoding = None; + while let Some(token) = tokens.next() { + if token == b"/Encoding" + && let Some(parsed) = parse_encoding(&mut tokens) + { + encoding = Some(parsed); + } + } + encoding +} + +fn parse_encoding<'x>(tokens: &mut Tokens<'x>) -> Option> { + let first = tokens.next()?; + if first == b"StandardEncoding" { + return Some(BuiltinEncoding::Standard); + } + parse_int(first)?; + let mut names = CodeNames::new(); + while let Some(token) = tokens.next() { + match token { + b"dup" => { + let Some(code) = tokens.next().and_then(parse_int) else { + continue; + }; + let Some(name) = tokens.next().and_then(|token| token.strip_prefix(b"/")) else { + continue; + }; + if let Ok(code) = u8::try_from(code) { + names.set(code, name); + } + } + b"def" => break, + _ => {} + } + } + Some(BuiltinEncoding::Custom(names)) +} + +fn parse_int(token: &[u8]) -> Option { + let (negative, unsigned) = match token { + [b'-', rest @ ..] => (true, rest), + [b'+', rest @ ..] => (false, rest), + _ => (false, token), + }; + let mut parts = unsigned.splitn(2, |&byte| byte == b'#'); + let head = parts.next()?; + let value = match parts.next() { + Some(digits) => { + let radix = u32::try_from(parse_digits(head, 10)?) + .ok() + .filter(|radix| (2..=MAX_RADIX).contains(radix))?; + parse_digits(digits, radix)? + } + None => parse_digits(head, 10)?, + }; + Some(if negative { -value } else { value }) +} + +fn parse_digits(digits: &[u8], radix: u32) -> Option { + if digits.is_empty() { + return None; + } + digits.iter().try_fold(0i64, |value, &byte| { + let digit = char::from(byte).to_digit(radix)?; + value + .checked_mul(i64::from(radix))? + .checked_add(i64::from(digit)) + }) +} + +struct Tokens<'x> { + data: &'x [u8], +} + +fn is_whitespace(byte: u8) -> bool { + matches!(byte, b'\0' | b'\t' | b'\n' | b'\x0c' | b'\r' | b' ') +} + +fn is_delimiter(byte: u8) -> bool { + matches!( + byte, + b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' + ) +} + +fn is_regular(byte: u8) -> bool { + !is_whitespace(byte) && !is_delimiter(byte) +} + +impl<'x> Tokens<'x> { + fn skip_space_and_comments(&mut self) { + loop { + let start = self.data.iter().position(|&byte| !is_whitespace(byte)); + self.data = self + .data + .get(start.unwrap_or(self.data.len())..) + .unwrap_or_default(); + if self.data.first() != Some(&b'%') { + return; + } + let end = self + .data + .iter() + .position(|&byte| byte == b'\n' || byte == b'\r'); + self.data = self + .data + .get(end.unwrap_or(self.data.len())..) + .unwrap_or_default(); + } + } + + fn split(&mut self, len: usize) -> &'x [u8] { + let (token, rest) = self.data.split_at_checked(len).unwrap_or((self.data, &[])); + self.data = rest; + token + } + + fn string_len(&self) -> usize { + let mut depth = 0usize; + let mut escaped = false; + for (index, &byte) in self.data.iter().enumerate() { + match byte { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return index + 1; + } + } + _ => {} + } + } + self.data.len() + } + + fn run_len(&self, skip: usize) -> usize { + let run = self.data.get(skip..).unwrap_or_default(); + skip + run + .iter() + .position(|&byte| !is_regular(byte)) + .unwrap_or(run.len()) + } +} + +impl<'x> Iterator for Tokens<'x> { + type Item = &'x [u8]; + + fn next(&mut self) -> Option<&'x [u8]> { + self.skip_space_and_comments(); + let len = match *self.data.first()? { + b'(' => self.string_len(), + b'<' => self + .data + .iter() + .position(|&byte| byte == b'>') + .map_or(self.data.len(), |end| end + 1), + b'/' => self.run_len(1), + byte if is_delimiter(byte) => 1, + _ => self.run_len(0), + }; + let token = self.split(len); + if token == b"eexec" { + self.data = &[]; + return None; + } + Some(token) + } +} diff --git a/crates/text-extract/src/pdf/font/ranges.rs b/crates/text-extract/src/pdf/font/ranges.rs new file mode 100644 index 00000000..8680d33d --- /dev/null +++ b/crates/text-extract/src/pdf/font/ranges.rs @@ -0,0 +1,130 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{cmp::Reverse, collections::BTreeMap}; + +#[derive(Debug, Clone, Copy)] +struct Entry { + lo: u64, + hi: u64, + base: u64, + value: T, +} + +#[derive(Debug, Clone)] +pub(crate) struct RangeMap { + entries: Vec>, +} + +impl Default for RangeMap { + fn default() -> Self { + RangeMap { + entries: Vec::new(), + } + } +} + +impl RangeMap { + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn heap_size(&self) -> usize { + self.entries.capacity() * std::mem::size_of::>() + } + + pub(crate) fn push(&mut self, lo: u64, hi: u64, value: T) { + if lo <= hi { + self.entries.push(Entry { + lo, + hi, + base: lo, + value, + }); + } + } + + pub(crate) fn entries(&self) -> impl Iterator + '_ { + self.entries + .iter() + .map(|entry| (entry.lo, entry.hi, entry.value)) + } + + pub(crate) fn finish(&mut self) { + if self.entries.is_empty() { + return; + } + let mut ordered: Vec<(usize, Entry)> = self.entries.drain(..).enumerate().collect(); + ordered.sort_by_key(|(sequence, entry)| (entry.lo, *sequence)); + let overlapping = ordered + .windows(2) + .any(|pair| matches!(pair, [(_, left), (_, right)] if right.lo <= left.hi)); + if !overlapping { + self.entries + .extend(ordered.into_iter().map(|(_, entry)| entry)); + self.entries.shrink_to_fit(); + return; + } + ordered.sort_by_key(|(sequence, _)| Reverse(*sequence)); + let mut covered: BTreeMap = BTreeMap::new(); + let mut overlapped: Vec<(u64, u64)> = Vec::new(); + for (_, entry) in ordered { + let start = match covered.range(..=entry.lo).next_back() { + Some((&low, &high)) if high >= entry.lo => low, + _ => entry.lo, + }; + overlapped.clear(); + overlapped.extend( + covered + .range(start..=entry.hi) + .map(|(&low, &high)| (low, high)), + ); + let mut cursor = Some(entry.lo); + let (mut merged_lo, mut merged_hi) = (entry.lo, entry.hi); + for &(low, high) in &overlapped { + if let Some(next) = cursor + && low > next + { + self.entries.push(Entry { + lo: next, + hi: low - 1, + ..entry + }); + } + cursor = cursor.and_then(|next| next.max(high).checked_add(1)); + merged_lo = merged_lo.min(low); + merged_hi = merged_hi.max(high); + covered.remove(&low); + } + if let Some(next) = cursor + && next <= entry.hi + { + self.entries.push(Entry { lo: next, ..entry }); + } + covered.insert(merged_lo, merged_hi); + } + self.entries.sort_unstable_by_key(|entry| entry.lo); + self.entries.shrink_to_fit(); + } + + pub(crate) fn find(&self, key: u64) -> Option<(T, u64)> { + let index = self.entries.partition_point(|entry| entry.lo <= key); + let entry = self.entries.get(index.checked_sub(1)?)?; + (key <= entry.hi).then(|| (entry.value, key - entry.base)) + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn is_disjoint(&self) -> bool { + self.entries + .windows(2) + .all(|pair| matches!(pair, [left, right] if left.hi < right.lo)) + } +} + diff --git a/crates/text-extract/src/pdf/font/simple/encoding.rs b/crates/text-extract/src/pdf/font/simple/encoding.rs new file mode 100644 index 00000000..c8263a9d --- /dev/null +++ b/crates/text-extract/src/pdf/font/simple/encoding.rs @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CODE_COUNT, SPACE}; +use crate::pdf::{ + document::Document, + font::{ + ToUnicode, + program::{BuiltinEncoding, CodeNames, TrueType}, + unicode::{NameResolver, Verdict, verdict}, + }, + object::{Array, Object}, + tables::BaseEncoding, +}; +use std::ops::RangeInclusive; + +const PRINTABLE_ASCII: RangeInclusive = 0x21..=0x7E; + +pub(super) struct Differences { + names: Vec, + spans: Box<[Option<(u32, u16)>; CODE_COUNT]>, +} + +pub(super) enum Builtin<'x> { + None, + Encoding(BaseEncoding), + Names(CodeNames<'x>), + TrueType(Box>), +} + +pub(super) struct Sources<'s, 'x> { + pub(super) to_unicode: Option<&'s ToUnicode>, + pub(super) differences: &'s Differences, + pub(super) base: Option, + pub(super) builtin: &'s Builtin<'x>, + pub(super) fallback: BaseEncoding, + pub(super) resolver: NameResolver, +} + +impl Differences { + pub(super) fn load(doc: &Document<'_>, array: Option>) -> Self { + let mut differences = Differences { + names: Vec::new(), + spans: Box::new([None; CODE_COUNT]), + }; + let Some(array) = array else { + return differences; + }; + let mut code: Option = None; + for item in doc.array_iter(array) { + match item { + Object::Name(name) => { + let Some(current) = code else { + continue; + }; + code = Some(current + 1); + if name.is(b".notdef") { + continue; + } + let Some(slot) = differences.spans.get_mut(current) else { + code = None; + continue; + }; + let start = differences.names.len(); + differences.names.extend(name.bytes()); + if let (Ok(start), Ok(len)) = ( + u32::try_from(start), + u16::try_from(differences.names.len() - start), + ) { + *slot = Some((start, len)); + } + } + other => { + code = other + .as_int() + .and_then(|value| usize::try_from(value).ok()) + .filter(|&value| value < CODE_COUNT); + } + } + } + differences + } + + pub(super) fn name(&self, code: u8) -> Option<&[u8]> { + let (start, len) = (*self.spans.get(usize::from(code))?)?; + let start = start as usize; + self.names.get(start..start + usize::from(len)) + } + + pub(super) fn names(&self) -> impl Iterator + Clone { + (0..=u8::MAX).filter_map(|code| self.name(code)) + } +} + +impl Builtin<'_> { + pub(super) fn name(&self, code: u8) -> Option<&[u8]> { + match self { + Builtin::Encoding(encoding) => encoding.glyph_name(code).map(str::as_bytes), + Builtin::Names(names) => names.get(code), + Builtin::TrueType(font) => font.glyph_name(font.symbolic_glyph(code)?), + Builtin::None => None, + } + } +} + +impl Sources<'_, '_> { + pub(super) fn text(&self, code: u8, out: &mut String) { + let mark = out.len(); + let mut private_use = false; + if let Some(to_unicode) = self.to_unicode + && to_unicode.unicode(u32::from(code), out) + { + match verdict(out.get(mark..).unwrap_or_default()) { + Verdict::Accept => return, + Verdict::Space => { + out.truncate(mark); + out.push(' '); + return; + } + Verdict::Reject => out.truncate(mark), + Verdict::PrivateUse => { + out.truncate(mark); + private_use = true; + } + } + } + if let Some(name) = self.differences.name(code) { + if !self.resolver.resolve(name, out) && !private_use { + last_resort(code, out); + } + return; + } + if let Some(ch) = self.base.and_then(|base| base.unicode(code)) { + out.push(ch); + return; + } + if self.builtin_text(code, out) { + return; + } + if private_use { + return; + } + match self.fallback.unicode(code) { + Some(ch) => out.push(ch), + None => last_resort(code, out), + } + } + + fn builtin_text(&self, code: u8, out: &mut String) -> bool { + match self.builtin { + Builtin::None => false, + Builtin::Encoding(encoding) => encoding.unicode(code).map(|ch| out.push(ch)).is_some(), + Builtin::Names(names) => match names.get(code) { + Some(name) => { + if !self.resolver.resolve(name, out) { + last_resort(code, out); + } + true + } + None => false, + }, + Builtin::TrueType(font) => { + let Some(glyph) = font.symbolic_glyph(code) else { + return false; + }; + if let Some(name) = font.glyph_name(glyph) + && self.resolver.resolve(name, out) + { + return true; + } + false + } + } + } + + pub(super) fn glyph_name(&self, code: u8) -> Option<&[u8]> { + self.differences + .name(code) + .or_else(|| self.base?.glyph_name(code).map(str::as_bytes)) + .or_else(|| self.builtin.name(code)) + .or_else(|| self.fallback.glyph_name(code).map(str::as_bytes)) + } +} + +pub(super) fn last_resort(code: u8, out: &mut String) { + if PRINTABLE_ASCII.contains(&code) || code == SPACE { + out.push(char::from(code)); + } +} + +impl<'x> Builtin<'x> { + pub(super) fn from_encoding(encoding: BuiltinEncoding<'x>) -> Self { + match encoding { + BuiltinEncoding::Standard => Builtin::Encoding(BaseEncoding::Standard), + BuiltinEncoding::Expert => Builtin::Encoding(BaseEncoding::Expert), + BuiltinEncoding::Custom(names) if !names.is_empty() => Builtin::Names(names), + BuiltinEncoding::Custom(_) => Builtin::None, + } + } +} diff --git a/crates/text-extract/src/pdf/font/simple/mod.rs b/crates/text-extract/src/pdf/font/simple/mod.rs new file mode 100644 index 00000000..287e604a --- /dev/null +++ b/crates/text-extract/src/pdf/font/simple/mod.rs @@ -0,0 +1,273 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod encoding; +mod widths; + +use self::{ + encoding::{Builtin, Differences, Sources, last_resort}, + widths::{Widths, metric_width, substitute}, +}; +use super::{ + Metrics, ToUnicode, + embedded::{ProgramKind, has_program, program}, + program::{BuiltinEncoding, Cff, TrueType}, + unicode::NameResolver, +}; +use crate::pdf::{ + document::Document, + object::{Dict, Object}, + tables::{BaseEncoding, StandardFont}, +}; +use std::ops::RangeInclusive; + +pub(crate) const CODE_COUNT: usize = 256; +const WIDTH_SCALE: f32 = 1000.0; +const DEFAULT_WIDTH: f32 = 500.0; +const SYMBOLIC: i64 = 1 << 2; +const NONSYMBOLIC: i64 = 1 << 5; +const SPACE: u8 = b' '; +const TYPE3_EM_RATIO: RangeInclusive = 0.2..=1.5; +const TYPE3_EM_FACTOR: f64 = 2.0; +const GBK_FONT_NAMES: &[&[u8]] = &[ + b"\xCB\xCE\xCC\xE5", + b"\xBA\xDA\xCC\xE5", + b"\xBF\xAC\xCC\xE5", + b"\xB7\xC2\xCB\xCE", + b"\xBF\xAC\xCC\xE5_GB2312", + b"\xB7\xC2\xCB\xCE_GB2312", + b"\xC1\xA5\xCA\xE9", + b"\xD0\xC2\xCB\xCE", + b"\xB7\xC2\xCB\xCE\xCC\xE5", + b"\xD0\xA1\xB1\xEA\xCB\xCE", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SimpleKind { + Type1, + TrueType, + Type3, +} + +#[derive(Debug, Clone, Copy, Default)] +struct Entry { + start: u32, + len: u16, + width: f32, +} + +#[derive(Debug, Clone)] +pub(crate) struct SimpleFont { + entries: Box<[Entry; CODE_COUNT]>, + text: String, +} + +impl SimpleFont { + pub(crate) fn carries_gbk(doc: &Document<'_>, dict: Dict<'_>) -> bool { + let win_ansi = doc + .get_name(dict, b"Encoding") + .is_some_and(|name| name.is(b"WinAnsiEncoding")); + win_ansi + && !has_program(doc.get_dict(dict, b"FontDescriptor")) + && doc.get_name(dict, b"BaseFont").is_some_and(|name| { + let name = name.decoded(); + GBK_FONT_NAMES.iter().any(|known| name.as_ref() == *known) + }) + } + + pub(crate) fn load( + doc: &Document<'_>, + dict: Dict<'_>, + kind: SimpleKind, + to_unicode: Option<&ToUnicode>, + buf: &mut Vec, + ) -> (SimpleFont, Metrics) { + let base_font = doc + .get_name(dict, b"BaseFont") + .map(|name| name.decoded().into_owned()) + .unwrap_or_default(); + let descriptor = doc.get_dict(dict, b"FontDescriptor"); + let flags = descriptor + .and_then(|descriptor| doc.get_int(descriptor, b"Flags")) + .unwrap_or(0); + let symbolic = flags & SYMBOLIC != 0 && flags & NONSYMBOLIC == 0; + let standard = match kind { + SimpleKind::Type3 => None, + _ => StandardFont::from_base_font(&base_font), + }; + let (base, differences) = match doc.dict_get(dict, b"Encoding") { + Object::Name(name) => ( + BaseEncoding::from_name(&name.decoded()), + Differences::load(doc, None), + ), + Object::Dict(encoding) => ( + doc.get_name(encoding, b"BaseEncoding") + .and_then(|name| BaseEncoding::from_name(&name.decoded())), + Differences::load(doc, doc.get_array(encoding, b"Differences")), + ), + _ => (None, Differences::load(doc, None)), + }; + let zapf = standard == Some(StandardFont::ZapfDingbats) + || base_font.windows(8).any(|window| window == b"Dingbats"); + let resolver = NameResolver::for_names(zapf, differences.names()); + let fallback = match kind { + SimpleKind::TrueType => BaseEncoding::WinAnsi, + _ => BaseEncoding::Standard, + }; + let wants_program = base.is_none() + && kind != SimpleKind::Type3 + && to_unicode.is_none() + && has_program(descriptor); + let loaded = match descriptor.filter(|_| wants_program) { + Some(descriptor) => program(doc, descriptor, buf), + None => None, + }; + let mut builtin = match loaded { + Some((ProgramKind::Type1, data)) => { + BuiltinEncoding::from_type1(data).map_or(Builtin::None, Builtin::from_encoding) + } + Some((ProgramKind::Cff, data)) => Cff::parse(data) + .and_then(|cff| cff.builtin_encoding()) + .map_or(Builtin::None, Builtin::from_encoding), + Some((ProgramKind::OpenType, data)) => { + match Cff::parse(data).and_then(|cff| cff.builtin_encoding()) { + Some(encoding) => Builtin::from_encoding(encoding), + None => TrueType::parse(data) + .map_or(Builtin::None, |font| Builtin::TrueType(Box::new(font))), + } + } + Some((ProgramKind::TrueType, data)) if symbolic => TrueType::parse(data) + .map_or(Builtin::None, |font| Builtin::TrueType(Box::new(font))), + _ => Builtin::None, + }; + if matches!(builtin, Builtin::None) + && base.is_none() + && let Some(standard) = standard.filter(|standard| standard.is_symbolic()) + { + builtin = Builtin::Encoding(standard.builtin_encoding()); + } + let sources = Sources { + to_unicode, + differences: &differences, + base, + builtin: &builtin, + fallback, + resolver, + }; + let widths = Widths::load(doc, dict, descriptor, kind); + let metrics_font = standard.unwrap_or_else(|| substitute(&base_font)); + let font = SimpleFont::build(|code, text| { + sources.text(code, text); + widths.explicit(code).unwrap_or_else(|| match kind { + SimpleKind::Type3 => widths.missing.unwrap_or(0.0) * widths.type3_scale, + _ => { + widths.missing.unwrap_or_else(|| { + sources + .glyph_name(code) + .and_then(|name| metric_width(metrics_font, name)) + .unwrap_or(DEFAULT_WIDTH) + }) / WIDTH_SCALE + } + }) + }); + let metrics = Metrics { + em_scale: match kind { + SimpleKind::Type3 => font.type3_em_scale(widths.type3_em), + _ => 1.0, + }, + space_width: font.space_width(), + }; + (font, metrics) + } + + pub(crate) fn standard() -> (SimpleFont, Metrics) { + let encoding = BaseEncoding::WinAnsi; + let font = SimpleFont::build(|code, text| { + match encoding.unicode(code) { + Some(ch) => text.push(ch), + None => last_resort(code, text), + } + encoding + .glyph_name(code) + .and_then(|name| metric_width(StandardFont::Helvetica, name.as_bytes())) + .unwrap_or(DEFAULT_WIDTH) + / WIDTH_SCALE + }); + let metrics = Metrics { + em_scale: 1.0, + space_width: font.space_width(), + }; + (font, metrics) + } + + fn build(mut fill: impl FnMut(u8, &mut String) -> f32) -> SimpleFont { + let mut font = SimpleFont { + entries: Box::new([Entry::default(); CODE_COUNT]), + text: String::new(), + }; + for (code, entry) in (0..=u8::MAX).zip(font.entries.iter_mut()) { + let mark = font.text.len(); + let width = fill(code, &mut font.text); + *entry = Entry { + start: u32::try_from(mark).unwrap_or(u32::MAX), + len: u16::try_from(font.text.len() - mark).unwrap_or_default(), + width, + }; + } + font.text.shrink_to_fit(); + font + } + + fn type3_em_scale(&self, nominal: f64) -> f32 { + let (sum, count) = self + .entries + .iter() + .filter(|entry| entry.width > 0.0) + .fold((0f64, 0u32), |(sum, count), entry| { + (sum + f64::from(entry.width), count + 1) + }); + if count == 0 || nominal <= 0.0 { + return nominal.max(f64::MIN_POSITIVE) as f32; + } + let average = sum / f64::from(count); + if TYPE3_EM_RATIO.contains(&(average / nominal)) { + nominal as f32 + } else { + (average * TYPE3_EM_FACTOR) as f32 + } + } + + fn space_width(&self) -> Option { + let is_space = |code: u8| self.glyph(code).0 == " "; + let code = if is_space(SPACE) { + Some(SPACE) + } else { + (0..=u8::MAX).find(|&code| is_space(code)) + }; + code.map(|code| self.glyph(code).1) + .filter(|width| *width > 0.0) + } + + #[inline] + pub(crate) fn glyph(&self, code: u8) -> (&str, f32) { + match self.entries.get(usize::from(code)) { + Some(entry) => { + let start = entry.start as usize; + ( + self.text + .get(start..start + usize::from(entry.len)) + .unwrap_or_default(), + entry.width, + ) + } + None => ("", 0.0), + } + } + + pub(crate) fn heap_size(&self) -> usize { + std::mem::size_of::<[Entry; CODE_COUNT]>() + self.text.capacity() + } +} diff --git a/crates/text-extract/src/pdf/font/simple/widths.rs b/crates/text-extract/src/pdf/font/simple/widths.rs new file mode 100644 index 00000000..a184f003 --- /dev/null +++ b/crates/text-extract/src/pdf/font/simple/widths.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CODE_COUNT, SimpleKind, WIDTH_SCALE}; +use crate::pdf::{document::Document, font::widths::sanitize, object::Dict, tables::StandardFont}; + +const DEFAULT_TYPE3_SCALE: f64 = 0.001; +const MATRIX_LEN: usize = 6; +const SERIF_HINTS: &[&[u8]] = &[ + b"Times", + b"Serif", + b"Roman", + b"Georgia", + b"Garamond", + b"Book", + b"Cambria", + b"Palatino", + b"Minion", +]; +const SANS_HINT: &[u8] = b"Sans"; +const BOLD_HINTS: &[&[u8]] = &[b"Bold", b"Black", b"Heavy"]; +const WIDTH_ALIASES: &[(&[u8], &[u8])] = &[ + (b"nbspace", b"space"), + (b"nonbreakingspace", b"space"), + (b"sfthyphen", b"hyphen"), + (b"softhyphen", b"hyphen"), +]; + +pub(super) struct Widths { + pub(super) explicit: Box<[Option; CODE_COUNT]>, + pub(super) missing: Option, + pub(super) type3_scale: f32, + pub(super) type3_em: f64, +} + +impl Widths { + pub(super) fn load( + doc: &Document<'_>, + dict: Dict<'_>, + descriptor: Option>, + kind: SimpleKind, + ) -> Self { + let (type3_scale, type3_em) = match kind { + SimpleKind::Type3 => type3_scales(doc, dict), + _ => (1.0 / WIDTH_SCALE, 1.0), + }; + let mut widths = Widths { + explicit: Box::new([None; CODE_COUNT]), + missing: descriptor + .and_then(|descriptor| doc.get_f64(descriptor, b"MissingWidth")) + .and_then(sanitize) + .filter(|width| *width > 0.0), + type3_scale, + type3_em, + }; + let first = doc + .get_int(dict, b"FirstChar") + .and_then(|first| usize::try_from(first).ok()) + .unwrap_or(0); + if let Some(array) = doc.get_array(dict, b"Widths") + && let Some(slots) = widths.explicit.get_mut(first..) + { + for (slot, value) in slots.iter_mut().zip(doc.array_iter(array)) { + *slot = value + .as_f64() + .and_then(sanitize) + .map(|width| width * type3_scale); + } + } + widths + } + + pub(super) fn explicit(&self, code: u8) -> Option { + self.explicit.get(usize::from(code)).copied().flatten() + } +} + +fn type3_scales(doc: &Document<'_>, dict: Dict<'_>) -> (f32, f64) { + let mut matrix = [DEFAULT_TYPE3_SCALE, 0.0, 0.0, DEFAULT_TYPE3_SCALE, 0.0, 0.0]; + if let Some(array) = doc.get_array(dict, b"FontMatrix") { + let mut values = [0f64; MATRIX_LEN]; + let mut count = 0; + for (slot, value) in values.iter_mut().zip(doc.array_iter(array)) { + if let Some(value) = value.as_f64() { + *slot = value; + count += 1; + } + } + if count == MATRIX_LEN { + matrix = values; + } + } + let [a, _, _, d, _, _] = matrix; + let horizontal = if a != 0.0 { a } else { DEFAULT_TYPE3_SCALE }; + let vertical = if d != 0.0 { d.abs() } else { horizontal.abs() }; + (horizontal as f32, vertical * f64::from(WIDTH_SCALE)) +} + +pub(super) fn metric_width(font: StandardFont, name: &[u8]) -> Option { + font.width(name) + .or_else(|| { + WIDTH_ALIASES + .iter() + .find(|(alias, _)| *alias == name) + .and_then(|(_, target)| font.width(target)) + }) + .map(f32::from) +} + +pub(super) fn substitute(base_font: &[u8]) -> StandardFont { + let contains = |needle: &[u8]| { + base_font + .windows(needle.len()) + .any(|window| window == needle) + }; + let bold = BOLD_HINTS.iter().any(|hint| contains(hint)); + let serif = !contains(SANS_HINT) && SERIF_HINTS.iter().any(|hint| contains(hint)); + match (serif, bold) { + (true, true) => StandardFont::TimesBold, + (true, false) => StandardFont::TimesRoman, + (false, true) => StandardFont::HelveticaBold, + (false, false) => StandardFont::Helvetica, + } +} diff --git a/crates/text-extract/src/pdf/font/unicode.rs b/crates/text-extract/src/pdf/font/unicode.rs new file mode 100644 index 00000000..d9776511 --- /dev/null +++ b/crates/text-extract/src/pdf/font/unicode.rs @@ -0,0 +1,148 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::pdf::tables::{ + CodeRadix, GlyphCode, glyph_name_code, glyph_unicode, zapf_dingbats_unicode, +}; + +const NOTDEF: &[u8] = b".notdef"; +const MAX_INDEX_DIGITS: usize = 3; +const INDEXED_CODES: std::ops::RangeInclusive = 0x20..=0xFF; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Verdict { + Accept, + Space, + Reject, + PrivateUse, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Numeric { + Glyph(CodeRadix), + Indexed, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct NameResolver { + pub(crate) zapf: bool, + pub(crate) numeric: Option, +} + +pub(crate) fn verdict(text: &str) -> Verdict { + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return Verdict::Reject; + }; + if chars.next().is_some() { + return if text.chars().all(|ch| ch == '\0') { + Verdict::Reject + } else { + Verdict::Accept + }; + } + match u32::from(first) { + 0x08..=0x0D => Verdict::Space, + 0x00..=0x1F | 0x7F..=0x9F | 0xFFFD..=0xFFFF | 0xFDD0..=0xFDEF => Verdict::Reject, + code if is_private_use(code) => Verdict::PrivateUse, + _ => Verdict::Accept, + } +} + +pub(crate) fn is_private_use(code: u32) -> bool { + matches!(code, 0xE000..=0xF8FF | 0xF_0000..=0x10_FFFF) +} + +impl NameResolver { + pub(crate) fn for_names<'n>(zapf: bool, names: impl Iterator + Clone) -> Self { + let numeric = numeric_radix(names.clone()) + .map(Numeric::Glyph) + .or_else(|| (!zapf && indexed(names)).then_some(Numeric::Indexed)); + NameResolver { zapf, numeric } + } + + pub(crate) fn resolve(&self, name: &[u8], out: &mut String) -> bool { + if name.is_empty() || name == NOTDEF { + return false; + } + let mark = out.len(); + let found = if self.zapf { + zapf_dingbats_unicode(name, out) + } else { + glyph_unicode(name, out) + }; + if found { + if verdict(out.get(mark..).unwrap_or_default()) == Verdict::Accept { + return true; + } + out.truncate(mark); + } + let code = match self.numeric { + Some(Numeric::Glyph(radix)) => match glyph_name_code(name, radix) { + Some(GlyphCode::Code(code)) => Some(code), + _ => None, + }, + Some(Numeric::Indexed) => index(name).filter(|code| INDEXED_CODES.contains(code)), + None => None, + }; + match code.and_then(char::from_u32) { + Some(ch) if verdict(ch.encode_utf8(&mut [0u8; 4])) == Verdict::Accept => { + out.push(ch); + true + } + _ => false, + } + } +} + +fn index(name: &[u8]) -> Option { + let digits = name.strip_prefix(b"a")?; + if digits.is_empty() || digits.len() > MAX_INDEX_DIGITS { + return None; + } + digits.iter().try_fold(0u32, |value, &byte| { + byte.is_ascii_digit() + .then(|| value * 10 + u32::from(byte - b'0')) + }) +} + +fn indexed<'n>(names: impl Iterator) -> bool { + let mut any = false; + for name in names { + if index(name).is_none() { + return false; + } + any = true; + } + any +} + +fn numeric_radix<'n>(names: impl Iterator + Clone) -> Option { + let mut any = false; + let mut radix = CodeRadix::Decimal; + for name in names.clone() { + any = true; + match glyph_name_code(name, CodeRadix::Decimal)? { + GlyphCode::Code(_) => {} + GlyphCode::NeedsHex => radix = CodeRadix::Hex, + } + } + if !any { + return None; + } + if radix == CodeRadix::Hex + && !names.into_iter().all(|name| { + matches!( + glyph_name_code(name, CodeRadix::Hex), + Some(GlyphCode::Code(_)) + ) + }) + { + return None; + } + Some(radix) +} + diff --git a/crates/text-extract/src/pdf/font/widths.rs b/crates/text-extract/src/pdf/font/widths.rs new file mode 100644 index 00000000..b59f767c --- /dev/null +++ b/crates/text-extract/src/pdf/font/widths.rs @@ -0,0 +1,247 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::ranges::RangeMap; +use crate::pdf::{ + document::Document, + object::{Array, Dict, Object}, +}; + +pub(crate) const MAX_WIDTH_ENTRIES: usize = 100_000; +pub(crate) const MAX_WIDTH: f64 = 10_000.0; +pub(crate) const DEFAULT_CID_WIDTH: f32 = 1000.0; +const DEFAULT_VERTICAL_ADVANCE: f32 = -1000.0; +const HORIZONTAL_GROUP: usize = 1; +const VERTICAL_GROUP: usize = 3; + +#[derive(Debug, Clone, Copy, PartialEq)] +struct Span { + lo: u32, + hi: u32, + start: u32, + per_code: bool, +} + +#[derive(Debug, Clone, Default)] +struct Runs { + spans: Box<[Span]>, + values: Box<[f32]>, +} + +#[derive(Debug, Clone)] +pub(crate) struct CidWidths { + runs: Runs, + default: f32, +} + +#[derive(Debug, Clone)] +pub(crate) struct VerticalAdvances { + runs: Runs, + default: f32, +} + +impl Runs { + fn read(doc: &Document<'_>, array: Option>, group: usize) -> Self { + let Some(array) = array else { + return Runs::default(); + }; + let mut map = RangeMap::default(); + read_groups(doc, array, group, &mut map); + let mut spans: Vec = Vec::new(); + let mut values: Vec = Vec::new(); + for (lo, hi, value) in map.entries() { + let (Ok(lo), Ok(hi), Ok(start)) = ( + u32::try_from(lo), + u32::try_from(hi), + u32::try_from(values.len()), + ) else { + continue; + }; + let adjacent = |span: &Span| span.hi.checked_add(1) == Some(lo); + match spans.last_mut() { + Some(last) + if !last.per_code + && adjacent(last) + && values.get(last.start as usize) == Some(&value) => + { + last.hi = hi; + } + Some(last) if last.per_code && lo == hi && adjacent(last) => { + last.hi = hi; + values.push(value); + } + _ => { + spans.push(Span { + lo, + hi, + start, + per_code: lo == hi, + }); + values.push(value); + } + } + } + Runs { + spans: spans.into_boxed_slice(), + values: values.into_boxed_slice(), + } + } + + fn find(&self, cid: u32) -> Option { + let index = self.spans.partition_point(|span| span.lo <= cid); + let span = self.spans.get(index.checked_sub(1)?)?; + if cid > span.hi { + return None; + } + let offset = if span.per_code { cid - span.lo } else { 0 }; + self.values + .get(span.start as usize + offset as usize) + .copied() + } + + fn heap_size(&self) -> usize { + self.spans.len() * std::mem::size_of::() + + self.values.len() * std::mem::size_of::() + } +} + +pub(crate) fn sanitize(value: f64) -> Option { + value + .is_finite() + .then(|| value.clamp(-MAX_WIDTH, MAX_WIDTH) as f32) +} + +impl CidWidths { + pub(crate) fn empty() -> Self { + CidWidths { + runs: Runs::default(), + default: DEFAULT_CID_WIDTH, + } + } + + pub(crate) fn load(doc: &Document<'_>, dict: Dict<'_>) -> Self { + let default = doc + .get_f64(dict, b"DW") + .and_then(sanitize) + .unwrap_or(DEFAULT_CID_WIDTH); + CidWidths { + runs: Runs::read(doc, doc.get_array(dict, b"W"), HORIZONTAL_GROUP), + default, + } + } + + pub(crate) fn width(&self, cid: u32) -> f32 { + self.runs.find(cid).unwrap_or(self.default) + } + + pub(crate) fn default_width(&self) -> f32 { + self.default + } + + pub(crate) fn heap_size(&self) -> usize { + self.runs.heap_size() + } +} + +impl VerticalAdvances { + pub(crate) fn empty() -> Self { + VerticalAdvances { + runs: Runs::default(), + default: DEFAULT_VERTICAL_ADVANCE, + } + } + + pub(crate) fn default_advance(&self) -> f32 { + self.default + } + + pub(crate) fn load(doc: &Document<'_>, dict: Dict<'_>) -> Self { + let default = doc + .get_array(dict, b"DW2") + .and_then(|array| doc.array_iter(array).nth(1)) + .and_then(|value| value.as_f64()) + .and_then(sanitize) + .unwrap_or(DEFAULT_VERTICAL_ADVANCE); + VerticalAdvances { + runs: Runs::read(doc, doc.get_array(dict, b"W2"), VERTICAL_GROUP), + default, + } + } + + pub(crate) fn advance(&self, cid: u32) -> f32 { + self.runs.find(cid).unwrap_or(self.default) + } + + pub(crate) fn heap_size(&self) -> usize { + self.runs.heap_size() + } +} + +fn read_groups(doc: &Document<'_>, array: Array<'_>, group: usize, map: &mut RangeMap) { + let mut pending = [0f64; 2 + VERTICAL_GROUP]; + let mut count = 0usize; + for item in doc.array_iter(array) { + if map.len() >= MAX_WIDTH_ENTRIES { + break; + } + match item { + Object::Array(inner) => { + if count == 1 + && let [start, ..] = pending + && let Some(first) = code(start) + { + read_list(doc, inner, first, group, map); + } + count = 0; + } + other => match other.as_f64() { + Some(value) => { + if let Some(slot) = pending.get_mut(count) { + *slot = value; + } + count += 1; + if count == 2 + group { + let [lo, hi, width, ..] = pending; + if let (Some(lo), Some(hi), Some(width)) = + (code(lo), code(hi), sanitize(width)) + { + map.push(lo, hi, width); + } + count = 0; + } + } + None => count = 0, + }, + } + } + map.finish(); +} + +fn read_list( + doc: &Document<'_>, + list: Array<'_>, + first: u64, + group: usize, + map: &mut RangeMap, +) { + let mut cid = first; + for (index, value) in doc.array_iter(list).enumerate() { + if map.len() >= MAX_WIDTH_ENTRIES { + return; + } + if index % group != 0 { + continue; + } + if let Some(width) = value.as_f64().and_then(sanitize) { + map.push(cid, cid, width); + } + cid = cid.saturating_add(1); + } +} + +fn code(value: f64) -> Option { + (value.is_finite() && (0.0..=f64::from(u32::MAX)).contains(&value)).then_some(value as u64) +} diff --git a/crates/text-extract/src/pdf/forms.rs b/crates/text-extract/src/pdf/forms.rs new file mode 100644 index 00000000..bfc2adea --- /dev/null +++ b/crates/text-extract/src/pdf/forms.rs @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + document::Document, + layout::{Layout, fingerprint}, + object::{Array, ArrayIter, Name, Object}, + pages::{Page, Visited}, + text_string::decode_text_string, +}; +use crate::output::Output; +use std::collections::HashSet; + +pub(crate) const MAX_FIELD_DEPTH: usize = 64; +pub(crate) const MAX_FIELDS: usize = 100_000; +const MAX_ANNOTATIONS: usize = 100_000; +const PASSWORD_FLAG: i64 = 1 << 13; +const SKIPPED_ANNOTATIONS: &[&[u8]] = &[b"Popup", b"Link", b"Widget"]; +const MAX_RETAINED: usize = 1 << 16; + +#[derive(Debug, Default)] +pub(crate) struct FormScratch { + seen: HashSet, + text: String, + bytes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FieldType { + Text, + Choice, + Button, + Signature, +} + +#[derive(Clone, Copy)] +struct Inherited<'d> { + kind: Option, + flags: i64, + options: Option>, +} + +impl FormScratch { + pub(crate) fn shrink(&mut self) { + self.seen = HashSet::new(); + self.text.clear(); + self.text.shrink_to(MAX_RETAINED); + self.bytes.clear(); + self.bytes.shrink_to(MAX_RETAINED); + } + + fn emit(&mut self, layout: &mut Layout, out: &mut Output<'_>) { + let text = self.text.trim(); + if !text.is_empty() && self.seen.insert(fingerprint(text)) { + layout.block(out, text); + } + self.text.clear(); + } +} + +impl FieldType { + fn from_name(name: Name<'_>) -> Option { + hashify::map!(name.raw(), FieldType, + b"Tx" => FieldType::Text, + b"Ch" => FieldType::Choice, + b"Btn" => FieldType::Button, + b"Sig" => FieldType::Signature, + ) + .copied() + } +} + +pub(crate) fn annotations( + doc: &Document<'_>, + page: &Page<'_>, + layout: &mut Layout, + out: &mut Output<'_>, + scratch: &mut FormScratch, +) { + let Some(annots) = page.annots(doc) else { + return; + }; + scratch.seen.clear(); + for annot in doc.array_iter(annots).take(MAX_ANNOTATIONS) { + if out.is_full() || doc.exhausted() { + return; + } + let Object::Dict(dict) = annot else { + continue; + }; + if doc.get_name(dict, b"Subtype").is_some_and(|subtype| { + SKIPPED_ANNOTATIONS + .iter() + .any(|skipped| subtype.is(skipped)) + }) { + continue; + } + text_value(doc, doc.dict_get(dict, b"Contents"), scratch); + if scratch.text.trim().is_empty() { + scratch.text.clear(); + text_value(doc, doc.dict_get(dict, b"RC"), scratch); + strip_markup(&mut scratch.text); + } + scratch.emit(layout, out); + } +} + +pub(crate) fn acroform( + doc: &Document<'_>, + layout: &mut Layout, + out: &mut Output<'_>, + scratch: &mut FormScratch, + visited: &mut Visited, +) { + let Some(fields) = doc.acroform_fields() else { + return; + }; + scratch.seen.clear(); + visited.clear(); + let root = Inherited { + kind: None, + flags: 0, + options: None, + }; + let mut stack: Vec<(ArrayIter<'_>, Inherited<'_>)> = vec![(fields.iter(), root)]; + let mut count = 0usize; + while let Some((kids, inherited)) = stack.last_mut() { + if out.is_full() || doc.exhausted() || count >= MAX_FIELDS { + return; + } + let inherited = *inherited; + let Some(kid) = kids.next() else { + stack.pop(); + continue; + }; + count += 1; + if let Some(id) = kid.as_ref() + && !visited.insert(id.num) + { + continue; + } + let Object::Dict(node) = doc.resolve(kid) else { + continue; + }; + let current = Inherited { + kind: doc + .get_name(node, b"FT") + .and_then(FieldType::from_name) + .or(inherited.kind), + flags: doc.get_int(node, b"Ff").unwrap_or(inherited.flags), + options: doc.get_array(node, b"Opt").or(inherited.options), + }; + if let Some(value) = node.get(b"V") { + field_value(doc, ¤t, doc.resolve(value), scratch); + scratch.emit(layout, out); + } + if let Some(children) = doc.get_array(node, b"Kids") + && stack.len() < MAX_FIELD_DEPTH + { + stack.push((children.iter(), current)); + } + } +} + +fn field_value( + doc: &Document<'_>, + field: &Inherited<'_>, + value: Object<'_>, + scratch: &mut FormScratch, +) { + match field.kind { + Some(FieldType::Button | FieldType::Signature) => {} + Some(FieldType::Text) if field.flags & PASSWORD_FLAG != 0 => {} + Some(FieldType::Choice) => match value { + Object::Array(values) => { + for item in doc.array_iter(values) { + choice(doc, field.options, item, scratch); + scratch.text.push('\n'); + } + } + other => choice(doc, field.options, other, scratch), + }, + _ => text_value(doc, value, scratch), + } +} + +fn choice( + doc: &Document<'_>, + options: Option>, + value: Object<'_>, + scratch: &mut FormScratch, +) { + let Object::Str(selected) = value else { + return; + }; + let selected = doc.string(selected); + let display = options.and_then(|options| { + doc.array_iter(options).find_map(|option| match option { + Object::Array(pair) => { + let mut items = doc.array_iter(pair); + let export = items.next()?.as_str()?; + let display = items.next()?.as_str()?; + (*doc.string(export) == *selected).then_some(display) + } + _ => None, + }) + }); + match display { + Some(display) => decode_text_string(&doc.string(display), &mut scratch.text), + None => decode_text_string(&selected, &mut scratch.text), + } +} + +fn text_value(doc: &Document<'_>, value: Object<'_>, scratch: &mut FormScratch) { + match value { + Object::Str(text) => decode_text_string(&doc.string(text), &mut scratch.text), + Object::Stream(stream) => { + scratch.bytes.clear(); + if !doc.decode_stream(stream, &mut scratch.bytes).is_failure() { + decode_text_string(&scratch.bytes, &mut scratch.text); + } + } + _ => {} + } +} + +fn strip_markup(text: &mut String) { + if !text.contains('<') && !text.contains('&') { + return; + } + let mut out = String::with_capacity(text.len()); + let mut rest = text.as_str(); + while let Some(ch) = rest.chars().next() { + match ch { + '<' => { + out.push(' '); + rest = rest.split_once('>').map_or("", |(_, after)| after); + } + '&' => { + let (entity, after) = rest + .get(1..) + .and_then(|body| body.split_once(';')) + .filter(|(entity, _)| entity.len() <= 8) + .unwrap_or(("", "")); + match decode_entity(entity) { + Some(decoded) => { + out.push(decoded); + rest = after; + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + other => { + out.push(other); + rest = rest.get(other.len_utf8()..).unwrap_or_default(); + } + } + } + *text = out; +} + +fn decode_entity(entity: &str) -> Option { + hashify::fnc_map!(entity.as_bytes(), + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + "nbsp" => Some(' '), + _ => { + let number = entity.strip_prefix('#')?; + let value = match number.strip_prefix(['x', 'X']) { + Some(hex) => u32::from_str_radix(hex, 16).ok()?, + None => number.parse().ok()?, + }; + char::from_u32(value) + } + ) +} + diff --git a/crates/text-extract/src/pdf/layout/dedupe.rs b/crates/text-extract/src/pdf/layout/dedupe.rs new file mode 100644 index 00000000..6efec871 --- /dev/null +++ b/crates/text-extract/src/pdf/layout/dedupe.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const CAPACITY: usize = 8192; +const MAX_LOAD: usize = CAPACITY * 3 / 8; +const MAX_PROBES: usize = 32; +const MIN_TOLERANCE: f64 = 0.05; +const WIDTH_TOLERANCE: f64 = 1.0 / 3.0; +const MIN_EXPONENT: i64 = -16; +const MAX_EXPONENT: i64 = 32; +const EXPONENT_BIAS: i64 = 1023; +const MANTISSA_BITS: u32 = 52; +const EXPONENT_MASK: u64 = 0x7FF; +const HALF: f64 = 0.5; +const FNV_OFFSET: u64 = 0xCBF2_9CE4_8422_2325; +const FNV_PRIME: u64 = 0x0100_0000_01B3; +const MIX: u64 = 0x9E37_79B9_7F4A_7C15; +const MIX_SHIFT: u32 = 29; +const KEY_SHIFT: u32 = 32; + +#[derive(Debug, Clone, Copy, Default)] +struct Slot { + key: u32, + generation: u32, + x: f64, + y: f64, +} + +#[derive(Debug)] +pub(crate) struct Dedupe { + slots: Vec, + generation: u32, + count: usize, +} + +impl Default for Dedupe { + fn default() -> Self { + Dedupe { + slots: Vec::new(), + generation: 1, + count: 0, + } + } +} + +fn neighbours(value: f64, cell: f64, tolerance: f64) -> (f64, Option) { + let base = (value / cell).floor(); + let offset = value - base * cell; + let neighbour = if offset < tolerance { + Some(base - 1.0) + } else if cell - offset < tolerance { + Some(base + 1.0) + } else { + None + }; + (base, neighbour) +} + +impl Dedupe { + pub(crate) fn clear(&mut self) { + self.count = 0; + self.generation = self.generation.wrapping_add(1); + if self.generation == 0 { + self.slots + .iter_mut() + .for_each(|slot| *slot = Slot::default()); + self.generation = 1; + } + } + + pub(crate) fn is_duplicate( + &mut self, + text: &str, + origin: (f64, f64), + width: f64, + em: f64, + ) -> bool { + if !(em.is_finite() && em > 0.0 && origin.0.is_finite() && origin.1.is_finite()) { + return false; + } + if self.slots.len() != CAPACITY { + self.slots = vec![Slot::default(); CAPACITY]; + } + let exponent = ((em.to_bits() >> MANTISSA_BITS & EXPONENT_MASK) as i64 - EXPONENT_BIAS) + .clamp(MIN_EXPONENT, MAX_EXPONENT); + let cell = f64::from_bits(((exponent + 1 + EXPONENT_BIAS) as u64) << MANTISSA_BITS); + let tolerance = (width.abs() * WIDTH_TOLERANCE) + .max(MIN_TOLERANCE * em) + .min(cell * HALF); + let (column, next_column) = neighbours(origin.0, cell, tolerance); + let (row, next_row) = neighbours(origin.1, cell, tolerance); + let text_hash = mix(fnv(text.as_bytes()), exponent as u64); + let home = bucket(text_hash, column, row); + let found = self.matches(home, origin, tolerance) + || next_column + .is_some_and(|next| self.matches(bucket(text_hash, next, row), origin, tolerance)) + || next_row.is_some_and(|next| { + self.matches(bucket(text_hash, column, next), origin, tolerance) + }) + || next_column + .zip(next_row) + .is_some_and(|(x, y)| self.matches(bucket(text_hash, x, y), origin, tolerance)); + if found { + return true; + } + if self.count >= MAX_LOAD { + self.clear(); + } + self.insert(home, origin); + false + } + + fn matches(&self, hash: u64, origin: (f64, f64), tolerance: f64) -> bool { + let start = hash as usize % CAPACITY; + let key = (hash >> KEY_SHIFT) as u32; + for probe in 0..MAX_PROBES { + let Some(slot) = self.slots.get((start + probe) % CAPACITY) else { + return false; + }; + if slot.generation != self.generation { + return false; + } + if slot.key == key + && (slot.x - origin.0).abs() < tolerance + && (slot.y - origin.1).abs() < tolerance + { + return true; + } + } + false + } + + fn insert(&mut self, hash: u64, origin: (f64, f64)) { + let start = hash as usize % CAPACITY; + let key = (hash >> KEY_SHIFT) as u32; + let generation = self.generation; + for probe in 0..MAX_PROBES { + if let Some(slot) = self.slots.get_mut((start + probe) % CAPACITY) + && slot.generation != generation + { + *slot = Slot { + key, + generation, + x: origin.0, + y: origin.1, + }; + self.count += 1; + return; + } + } + } +} + +pub(crate) fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(FNV_OFFSET, |hash, &byte| { + (hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME) + }) +} + +fn mix(hash: u64, value: u64) -> u64 { + let mixed = (hash ^ value).wrapping_mul(MIX); + mixed ^ (mixed >> MIX_SHIFT) +} + +fn bucket(text_hash: u64, x: f64, y: f64) -> u64 { + mix(mix(text_hash, x as i64 as u64), y as i64 as u64) +} + diff --git a/crates/text-extract/src/pdf/layout/line.rs b/crates/text-extract/src/pdf/layout/line.rs new file mode 100644 index 00000000..bedaea72 --- /dev/null +++ b/crates/text-extract/src/pdf/layout/line.rs @@ -0,0 +1,329 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::script::{is_combining, is_dropped, is_neutral, is_rtl, is_space, mirror}; +use crate::{ + output::{Output, Separator}, + pdf::tables::presentation_form, +}; + +pub(crate) const MAX_LINE_BYTES: usize = 4096; + +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub(crate) struct Extent { + pub(crate) start: f64, + pub(crate) end: f64, + pub(crate) em: f64, + pub(crate) threshold: f64, +} + +#[derive(Debug, Clone, Copy)] +struct Unit { + extent: Extent, + start: usize, + end: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SpaceMark { + at: usize, + generation: u32, +} + +#[derive(Debug, Default)] +pub(crate) struct Line { + text: String, + units: Vec, + ordered: String, + scratch: String, + provisional: Vec, + rtl: bool, + generation: u32, +} + +impl Line { + pub(crate) fn clear(&mut self) { + self.text.clear(); + self.units.clear(); + self.provisional.clear(); + self.rtl = false; + self.generation = self.generation.wrapping_add(1); + } + + pub(crate) fn shrink(&mut self) { + self.clear(); + self.text.shrink_to(MAX_LINE_BYTES); + self.units.shrink_to(MAX_LINE_BYTES); + self.provisional.shrink_to(MAX_LINE_BYTES); + self.ordered = String::new(); + self.scratch = String::new(); + } + + pub(crate) fn separate(&mut self, out: &mut Output<'_>, separator: Separator) { + match separator { + Separator::None => {} + Separator::Space => { + if self.text.is_empty() { + out.separator(Separator::Space); + } else if !self.text.ends_with(' ') { + self.text.push(' '); + } + } + Separator::Newline => { + self.flush(out); + out.separator(Separator::Newline); + } + } + } + + pub(crate) fn separate_tentatively(&mut self) -> Option { + if self.text.is_empty() || self.text.ends_with(' ') { + return None; + } + let at = self.text.len(); + self.text.push(' '); + Some(SpaceMark { + at, + generation: self.generation, + }) + } + + pub(crate) fn separate_provisionally(&mut self, out: &mut Output<'_>) { + let at = self.text.len(); + self.separate(out, Separator::Space); + if self.text.len() > at { + self.provisional.push(at); + } + } + + pub(crate) fn retract(&mut self, mark: SpaceMark) { + if mark.generation != self.generation || self.text.as_bytes().get(mark.at) != Some(&b' ') { + return; + } + self.text.remove(mark.at); + for unit in self.units.iter_mut().filter(|unit| unit.start > mark.at) { + unit.start -= 1; + unit.end -= 1; + } + self.provisional.retain(|&at| at != mark.at); + for at in self.provisional.iter_mut().filter(|at| **at > mark.at) { + *at -= 1; + } + } + + pub(crate) fn retract_provisional(&mut self) { + let text = &self.text; + self.provisional + .retain(|&at| text.as_bytes().get(at) == Some(&b' ')); + if self.provisional.is_empty() { + return; + } + self.scratch.clear(); + let mut kept_from = 0; + for &at in &self.provisional { + self.scratch + .push_str(self.text.get(kept_from..at).unwrap_or_default()); + kept_from = at + 1; + } + self.scratch + .push_str(self.text.get(kept_from..).unwrap_or_default()); + std::mem::swap(&mut self.text, &mut self.scratch); + let mut removed = self.provisional.iter().peekable(); + let mut shift = 0; + for unit in &mut self.units { + while removed.next_if(|&&at| at < unit.start).is_some() { + shift += 1; + } + unit.start -= shift; + unit.end -= shift; + } + self.provisional.clear(); + } + + pub(crate) fn push(&mut self, out: &mut Output<'_>, text: &str, extent: Extent) { + let start = self.text.len(); + if text.bytes().all(|byte| (0x21..0x7F).contains(&byte)) { + self.text.push_str(text); + } else { + for ch in text.chars() { + if is_space(ch) { + if !self.text.is_empty() && !self.text.ends_with(' ') { + self.text.push(' '); + } + } else if !is_dropped(ch) { + self.rtl |= is_rtl(ch); + match presentation_form(ch) { + Some(form) => self.text.push_str(form), + None => self.text.push(ch), + } + } + } + } + if self.text.len() > start { + self.units.push(Unit { + extent, + start, + end: self.text.len(), + }); + } + if self.text.len() >= MAX_LINE_BYTES { + self.flush(out); + } + } + + pub(crate) fn append_mark(&mut self, mark: char) { + self.text.push(mark); + if let Some(unit) = self.units.last_mut() { + unit.end = self.text.len(); + } + } + + pub(crate) fn flush(&mut self, out: &mut Output<'_>) { + if self.text.is_empty() { + return; + } + let text = if self.rtl { + self.reorder(); + self.ordered.as_str() + } else { + self.text.as_str() + }; + let trimmed = text.trim_matches(' '); + if text.starts_with(' ') { + out.separator(Separator::Space); + } + out.push_str(trimmed); + if trimmed.len() < text.len() && text.ends_with(' ') { + out.separator(Separator::Space); + } + self.clear(); + } + + fn reorder(&mut self) { + self.units + .sort_by(|left, right| left.extent.start.total_cmp(&right.extent.start)); + self.scratch.clear(); + let mut previous: Option<&Unit> = None; + for unit in &self.units { + if let Some(before) = previous { + let em = before.extent.em.max(unit.extent.em); + let gap = (unit.extent.start - before.extent.end) / em; + if gap > unit.extent.threshold + && !self.scratch.is_empty() + && !self.scratch.ends_with(' ') + { + self.scratch.push(' '); + } + } + push_visual( + self.text.get(unit.start..unit.end).unwrap_or_default(), + &mut self.scratch, + ); + previous = Some(unit); + } + let (rtl, ltr) = self + .scratch + .chars() + .fold((0usize, 0usize), |(rtl, ltr), ch| { + if is_rtl(ch) && !is_combining(ch) { + (rtl + 1, ltr) + } else if !is_neutral(ch) { + (rtl, ltr + 1) + } else { + (rtl, ltr) + } + }); + self.ordered.clear(); + if rtl >= ltr { + reverse_line(&self.scratch, &mut self.ordered); + } else { + reverse_runs(&self.scratch, &mut self.ordered); + } + } +} + +fn push_visual(logical: &str, out: &mut String) { + let mut chars = logical.chars(); + let first = chars.next(); + if chars.next().is_none() || !first.is_some_and(is_rtl) { + out.push_str(logical); + return; + } + let mut end = logical.len(); + for (index, ch) in logical.char_indices().rev() { + if !is_combining(ch) { + out.push_str(logical.get(index..end).unwrap_or_default()); + end = index; + } + } + out.push_str(logical.get(..end).unwrap_or_default()); +} + +fn is_strong_ltr(ch: char) -> bool { + (!is_neutral(ch) && !is_rtl(ch)) || ch.is_numeric() +} + +fn reverse_line(visual: &str, out: &mut String) { + let chars: Vec<(usize, char)> = visual.char_indices().collect(); + let mut end = visual.len(); + let mut cursor = chars.len(); + while let Some(index) = cursor.checked_sub(1) { + cursor = index; + let Some(&(offset, ch)) = chars.get(index) else { + break; + }; + if is_combining(ch) { + continue; + } + if is_strong_ltr(ch) { + let mut start = offset; + let mut probe = index; + while let Some(previous) = probe.checked_sub(1) { + let Some(&(at, before)) = chars.get(previous) else { + break; + }; + if is_rtl(before) && !is_combining(before) { + break; + } + probe = previous; + if is_strong_ltr(before) { + start = at; + cursor = previous; + } + } + out.push_str(visual.get(start..end).unwrap_or_default()); + end = start; + } else { + out.push(mirror(ch)); + out.push_str(visual.get(offset + ch.len_utf8()..end).unwrap_or_default()); + end = offset; + } + } +} + +fn reverse_runs(visual: &str, out: &mut String) { + let mut segment: Option<(usize, usize)> = None; + let mut copied = 0usize; + for (index, ch) in visual.char_indices() { + if is_rtl(ch) && !is_combining(ch) { + let end = index + ch.len_utf8(); + segment = Some(segment.map_or((index, end), |(start, _)| (start, end))); + } else if !is_neutral(ch) + && let Some((start, end)) = segment.take() + { + out.push_str(visual.get(copied..start).unwrap_or_default()); + reverse_line(visual.get(start..end).unwrap_or_default(), out); + copied = end; + } + } + if let Some((start, end)) = segment { + out.push_str(visual.get(copied..start).unwrap_or_default()); + reverse_line(visual.get(start..end).unwrap_or_default(), out); + copied = end; + } + out.push_str(visual.get(copied..).unwrap_or_default()); +} + diff --git a/crates/text-extract/src/pdf/layout/mod.rs b/crates/text-extract/src/pdf/layout/mod.rs new file mode 100644 index 00000000..20482b2e --- /dev/null +++ b/crates/text-extract/src/pdf/layout/mod.rs @@ -0,0 +1,539 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod dedupe; +mod line; +mod script; +mod shift; + + +use self::{ + dedupe::Dedupe, + line::{Extent, Line, SpaceMark}, + script::{is_combining, is_dropped, is_rtl, is_space, is_unspaced}, + shift::Run, +}; +use crate::{ + output::{Output, Separator}, + pdf::{font::FontId, tables::combining_accent}, +}; + +const SAME_DIRECTION: f64 = 0.99; +const NEWLINE_ACROSS: f64 = 0.7; +const BACKWARD_JUMP: f64 = -0.5; +const BACKWARD_ACROSS: f64 = 0.3; +const FORWARD_NEWLINE: f64 = 2.0; +const PEN_MOVED: f64 = 0.03; +const MIN_SPACE: f64 = 0.08; +const BASE_SPACE: f64 = 0.15; +const SPACE_WIDTH_FACTOR: f64 = 0.5; +const PAGE_MARGIN_EM: f64 = 1.0; +const MIN_EM: f64 = 1e-6; +const MIN_ADVANCE: f64 = 1e-9; +const SOFT_HYPHEN: char = '\u{AD}'; +const ACCENT_OVERLAP: f64 = 0.3; +const ACCENT_ACROSS: f64 = 1.5; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Placement { + pub(crate) origin: (f64, f64), + pub(crate) end: (f64, f64), + pub(crate) dir: (f64, f64), + pub(crate) em: f64, + pub(crate) space_threshold: f64, + pub(crate) spacing: f64, + pub(crate) continued: bool, + pub(crate) vertical: bool, + pub(crate) font: FontId, +} + +#[derive(Debug, Clone, Copy)] +struct Previous { + origin: (f64, f64), + end: (f64, f64), + dir: (f64, f64), + em: f64, + spacing: f64, + vertical: bool, + font: FontId, + last: char, + rtl: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct PendingAccent { + spacing: char, + mark: char, + placement: Placement, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Gap { + separator: Separator, + provisional: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum Class { + Empty, + Space, + SoftHyphen, + Visible { first: char, last: char, rtl: bool }, +} + +#[derive(Debug, Default)] +pub(crate) struct Layout { + previous: Option, + pending_space: bool, + spaced_line: bool, + soft_hyphen: bool, + media_box: Option<[f64; 4]>, + dedupe: Dedupe, + line: Line, + actual: Option, + actual_text: String, + accent: Option, + run: Run, + tentative: Option, + glyphs: u64, + candidates: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct GlyphCounts { + pub(crate) candidates: u64, + pub(crate) placed: u64, + pub(crate) in_actual_text: bool, +} + +pub(crate) fn fingerprint(text: &str) -> u64 { + dedupe::fnv(text.as_bytes()) +} + +pub(crate) fn space_threshold(space_width: Option) -> f64 { + space_width + .map_or(BASE_SPACE, |width| { + BASE_SPACE.min(SPACE_WIDTH_FACTOR * f64::from(width)) + }) + .max(MIN_SPACE) +} + +impl Class { + fn of(text: &str) -> Class { + if let &[byte] = text.as_bytes() + && byte.is_ascii_graphic() + { + let ch = char::from(byte); + return Class::Visible { + first: ch, + last: ch, + rtl: false, + }; + } + let mut chars = text + .chars() + .filter(|&ch| !is_dropped(ch) || ch == SOFT_HYPHEN); + let Some(first) = chars.next() else { + return Class::Empty; + }; + if text.chars().all(|ch| ch == SOFT_HYPHEN) { + return Class::SoftHyphen; + } + let mut visible = text.chars().filter(|&ch| !is_dropped(ch) && !is_space(ch)); + let Some(first_visible) = visible.next() else { + return if is_space(first) { + Class::Space + } else { + Class::Empty + }; + }; + let last = visible.next_back().unwrap_or(first_visible); + Class::Visible { + first: first_visible, + last, + rtl: is_rtl(first_visible), + } + } +} + +impl From for Gap { + fn from(separator: Separator) -> Self { + Gap { + separator, + provisional: false, + } + } +} + +impl Extent { + fn of(placement: &Placement) -> Self { + let (from, to) = ( + dot(placement.origin, placement.dir), + dot(placement.end, placement.dir), + ); + Extent { + start: from.min(to), + end: from.max(to), + em: placement.em.max(MIN_EM), + threshold: placement.space_threshold, + } + } +} + +fn dot(a: (f64, f64), b: (f64, f64)) -> f64 { + a.0 * b.0 + a.1 * b.1 +} + +fn cross(a: (f64, f64), b: (f64, f64)) -> f64 { + a.0 * b.1 - a.1 * b.0 +} + +fn sub(a: (f64, f64), b: (f64, f64)) -> (f64, f64) { + (a.0 - b.0, a.1 - b.1) +} + +fn single_accent(text: &str) -> Option<(char, char)> { + let mut chars = text.chars(); + let spacing = chars.next()?; + if chars.next().is_some() { + return None; + } + if is_combining(spacing) { + return Some((spacing, spacing)); + } + combining_accent(spacing).map(|mark| (spacing, mark)) +} + +fn overlaps( + accent: &Placement, + origin: (f64, f64), + end: (f64, f64), + dir: (f64, f64), + em: f64, +) -> bool { + let span = |from: (f64, f64), to: (f64, f64)| { + let (a, b) = (dot(from, dir), dot(to, dir)); + (a.min(b), a.max(b)) + }; + let (accent_start, accent_end) = span(accent.origin, accent.end); + let (base_start, base_end) = span(origin, end); + let width = (accent_end - accent_start).max(MIN_ADVANCE); + let shared = accent_end.min(base_end) - accent_start.max(base_start); + let across = cross(dir, sub(accent.origin, origin)).abs(); + shared / width > ACCENT_OVERLAP && across < ACCENT_ACROSS * em.max(MIN_EM) +} + +impl Layout { + pub(crate) fn begin_page(&mut self, media_box: Option<[f64; 4]>) { + self.previous = None; + self.pending_space = false; + self.spaced_line = false; + self.soft_hyphen = false; + self.accent = None; + self.media_box = media_box; + self.dedupe.clear(); + self.actual = None; + self.run = Run::default(); + self.tentative = None; + } + + pub(crate) fn shrink(&mut self) { + self.line.shrink(); + self.actual_text = String::new(); + } + + pub(crate) fn counts(&self) -> GlyphCounts { + GlyphCounts { + candidates: self.candidates, + placed: self.glyphs, + in_actual_text: self.actual.is_some(), + } + } + + pub(crate) fn end_page(&mut self, out: &mut Output<'_>) { + self.flush_accent(out); + self.end_actual_text(out); + self.line.flush(out); + out.separator(Separator::Newline); + } + + pub(crate) fn block(&mut self, out: &mut Output<'_>, text: &str) { + self.line.separate(out, Separator::Newline); + self.line.push(out, text, Extent::default()); + self.line.separate(out, Separator::Newline); + } + + pub(crate) fn begin_actual_text(&mut self, text: &str) -> bool { + if self.actual.is_some() { + return false; + } + self.actual_text.clear(); + self.actual_text.push_str(text); + self.actual = Some(false); + true + } + + pub(crate) fn end_actual_text(&mut self, out: &mut Output<'_>) { + if let Some(emitted) = self.actual.take() + && !emitted + { + let extent = self.last_extent(); + self.line.separate(out, Separator::Space); + let text = std::mem::take(&mut self.actual_text); + self.line.push(out, &text, extent); + self.actual_text = text; + self.glyphs = self.glyphs.saturating_add(1); + self.candidates = self.candidates.saturating_add(1); + } + } + + fn outside_page(&self, placement: &Placement) -> bool { + let Some([left, bottom, right, top]) = self.media_box else { + return false; + }; + let margin = placement.em * PAGE_MARGIN_EM; + let (x, y) = placement.origin; + x < left - margin || x > right + margin || y < bottom - margin || y > top + margin + } + + pub(crate) fn glyph(&mut self, out: &mut Output<'_>, text: &str, placement: &Placement) { + self.candidates = self.candidates.saturating_add(1); + if self.outside_page(placement) { + return; + } + if let Some((spacing, mark)) = single_accent(text) { + if !self.attach_accent(mark, placement) { + self.flush_accent(out); + self.accent = Some(PendingAccent { + spacing, + mark, + placement: *placement, + }); + } + return; + } + if let Some(accent) = self.accent.take() { + if matches!(Class::of(text), Class::Visible { .. }) + && overlaps( + &accent.placement, + placement.origin, + placement.end, + placement.dir, + placement.em, + ) + { + let before = self.glyphs; + self.place(out, text, placement); + if self.glyphs > before && self.actual.is_none() { + self.line.append_mark(accent.mark); + } + return; + } + self.place( + out, + accent.spacing.encode_utf8(&mut [0u8; 4]), + &accent.placement, + ); + } + self.place(out, text, placement); + } + + fn flush_accent(&mut self, out: &mut Output<'_>) { + if let Some(accent) = self.accent.take() { + self.place( + out, + accent.spacing.encode_utf8(&mut [0u8; 4]), + &accent.placement, + ); + } + } + + fn attach_accent(&mut self, mark: char, placement: &Placement) -> bool { + let Some(previous) = self.previous.filter(|previous| { + previous.last != SOFT_HYPHEN && dot(previous.dir, placement.dir) >= SAME_DIRECTION + }) else { + return false; + }; + if !overlaps( + placement, + previous.origin, + previous.end, + previous.dir, + previous.em, + ) { + return false; + } + if self.actual.is_none() { + self.line.append_mark(mark); + } + true + } + + fn place(&mut self, out: &mut Output<'_>, text: &str, placement: &Placement) { + let (first, last, rtl) = match Class::of(text) { + Class::Empty => return, + Class::Space => { + if !self.spaced_line { + self.line.retract_provisional(); + } + self.pending_space = true; + self.spaced_line = true; + return; + } + Class::SoftHyphen => { + self.soft_hyphen = true; + self.remember(placement, SOFT_HYPHEN, false); + return; + } + Class::Visible { first, last, rtl } => (first, last, rtl), + }; + let advance = dot(sub(placement.end, placement.origin), placement.dir); + if self.actual.is_none() + && self + .dedupe + .is_duplicate(text, placement.origin, advance, placement.em) + { + return; + } + self.glyphs = self.glyphs.saturating_add(1); + let gap = self.separator(placement, first, rtl); + let mut separator = gap.separator; + let shifted = self.shift(placement, separator, first, last); + if self.soft_hyphen && separator == Separator::Newline { + separator = Separator::None; + } + if separator == Separator::Newline { + self.spaced_line = false; + } + self.soft_hyphen = false; + let tentative = self.apply_shift(shifted, &mut separator); + let extent = Extent::of(placement); + match self.actual { + Some(true) => {} + Some(false) => { + self.line.separate(out, separator); + let actual = std::mem::take(&mut self.actual_text); + self.line.push(out, &actual, extent); + self.actual_text = actual; + self.actual = Some(true); + } + None => { + if gap.provisional && separator == Separator::Space { + self.line.separate_provisionally(out); + } else { + self.line.separate(out, separator); + } + if tentative { + self.tentative = self.line.separate_tentatively(); + } + self.line.push(out, text, extent); + } + } + if self.actual.is_some() + && self + .actual_text + .chars() + .all(|ch| is_space(ch) || is_dropped(ch)) + { + self.soft_hyphen = true; + } + self.remember(placement, last, rtl); + self.pending_space = false; + } + + fn last_extent(&self) -> Extent { + self.previous.map_or_else(Extent::default, |previous| { + let end = dot(previous.end, previous.dir); + Extent { + start: end, + end, + em: previous.em, + threshold: BASE_SPACE, + } + }) + } + + fn remember(&mut self, placement: &Placement, last: char, rtl: bool) { + let advance = dot(sub(placement.end, placement.origin), placement.dir).abs(); + let end = match self.previous { + Some(previous) if advance < MIN_ADVANCE => previous.end, + _ => placement.end, + }; + self.previous = Some(Previous { + origin: placement.origin, + end, + dir: placement.dir, + em: placement.em, + spacing: placement.spacing, + vertical: placement.vertical, + font: placement.font, + last, + rtl, + }); + } + + fn separator(&self, placement: &Placement, first: char, rtl: bool) -> Gap { + let Some(previous) = self.previous else { + return if self.pending_space { + Separator::Space + } else { + Separator::None + } + .into(); + }; + if previous.vertical != placement.vertical + || dot(previous.dir, placement.dir) < SAME_DIRECTION + { + return Separator::Newline.into(); + } + let em = previous.em.max(placement.em).max(MIN_EM); + let delta = sub(placement.origin, previous.end); + let mut along = dot(delta, previous.dir) / em; + let across = (cross(previous.dir, delta) / em).abs(); + if rtl && previous.rtl && dot(sub(placement.origin, previous.origin), previous.dir) < 0.0 { + along = dot(sub(previous.origin, placement.end), previous.dir) / em; + } + if across >= NEWLINE_ACROSS { + return Separator::Newline.into(); + } + if along < BACKWARD_JUMP { + return if across > BACKWARD_ACROSS { + Separator::Newline + } else { + Separator::Space + } + .into(); + } + if along > FORWARD_NEWLINE { + return Separator::Newline.into(); + } + let pending = self.pending_space && along > PEN_MOVED; + let relaxed = if placement.continued { + previous.spacing.max(0.0) + } else { + 0.0 + }; + let spaced_threshold = placement.space_threshold.max(BASE_SPACE) + relaxed; + let threshold = if self.spaced_line { + spaced_threshold + } else { + placement.space_threshold + relaxed + }; + let synthetic = along > threshold; + if pending || synthetic && !is_unspaced(previous.last) && !is_unspaced(first) { + Gap { + separator: Separator::Space, + provisional: !pending + && along <= spaced_threshold + && previous.font == placement.font + && previous.last.is_alphanumeric() + && first.is_alphanumeric(), + } + } else { + Separator::None.into() + } + } +} diff --git a/crates/text-extract/src/pdf/layout/script.rs b/crates/text-extract/src/pdf/layout/script.rs new file mode 100644 index 00000000..6330a9fe --- /dev/null +++ b/crates/text-extract/src/pdf/layout/script.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) fn is_space(ch: char) -> bool { + matches!( + ch, + '\t' | '\n' + | '\u{0B}' + | '\u{0C}' + | '\r' + | ' ' + | '\u{85}' + | '\u{A0}' + | '\u{1680}' + | '\u{2000}' + ..='\u{200B}' + | '\u{2028}' + | '\u{2029}' + | '\u{202F}' + | '\u{205F}' + | '\u{3000}' + | '\u{FEFF}' + ) +} + +pub(crate) fn is_dropped(ch: char) -> bool { + let code = u32::from(ch); + ch.is_control() + || matches!(code, 0xAD | 0xFFFD..=0xFFFF | 0xFDD0..=0xFDEF) + || matches!(code, 0xE000..=0xF8FF | 0xF_0000..=0x10_FFFF) +} + +pub(crate) fn is_unspaced(ch: char) -> bool { + matches!( + u32::from(ch), + 0x0E00..=0x0EFF + | 0x0F00..=0x0FFF + | 0x1000..=0x109F + | 0x1780..=0x17FF + | 0x3000..=0x30FF + | 0x3400..=0x4DBF + | 0x4E00..=0x9FFF + | 0xF900..=0xFAFF + | 0xFF00..=0xFFEF + | 0x2_0000..=0x3_FFFF + ) +} + +pub(crate) fn is_rtl(ch: char) -> bool { + !ch.is_numeric() + && matches!( + u32::from(ch), + 0x0590..=0x08FF | 0xFB1D..=0xFDFF | 0xFE70..=0xFEFF | 0x1_0800..=0x1_0FFF | 0x1_E800..=0x1_EFFF + ) +} + +pub(crate) fn is_combining(ch: char) -> bool { + matches!( + u32::from(ch), + 0x0300..=0x036F | 0x0591..=0x05BD | 0x05BF..=0x05C7 | 0x0610..=0x061A | 0x064B..=0x065F + | 0x0670 | 0x06D6..=0x06ED | 0x1AB0..=0x1AFF | 0x1DC0..=0x1DFF | 0x20D0..=0x20FF + | 0xFE20..=0xFE2F + ) +} + +pub(crate) fn is_neutral(ch: char) -> bool { + !ch.is_alphabetic() || is_combining(ch) +} + +pub(crate) fn mirror(ch: char) -> char { + match ch { + '(' => ')', + ')' => '(', + '[' => ']', + ']' => '[', + '{' => '}', + '}' => '{', + '<' => '>', + '>' => '<', + '\u{AB}' => '\u{BB}', + '\u{BB}' => '\u{AB}', + other => other, + } +} + diff --git a/crates/text-extract/src/pdf/layout/shift.rs b/crates/text-extract/src/pdf/layout/shift.rs new file mode 100644 index 00000000..8f68c57b --- /dev/null +++ b/crates/text-extract/src/pdf/layout/shift.rs @@ -0,0 +1,223 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Layout, MIN_EM, Placement, cross, script::is_unspaced, sub}; +use crate::output::Separator; + +const MIN_SHIFT: f64 = 0.2; +const SMALLER_FONT: f64 = 1.15; +const WORD_LETTERS: usize = 3; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum Level { + #[default] + Base, + Raised, + Lowered, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Direction { + Up, + Down, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Shifted { + Joined, + Space, + Tentative, + Retract, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct Glyph { + first: char, + last: char, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct Run { + level: Level, + recent: [Glyph; WORD_LETTERS], + count: usize, + marker: bool, + detached: bool, +} + +fn is_marker(ch: char) -> bool { + is_marker_start(ch) || matches!(ch, ',' | '-' | '\u{2013}') +} + +fn is_marker_start(ch: char) -> bool { + if ch.is_ascii() { + ch.is_ascii_digit() || matches!(ch, '*' | '#') + } else { + matches!( + ch, + '\u{A7}' | '\u{B6}' | '\u{2016}' | '\u{2020}' | '\u{2021}' | '\u{2217}' | '\u{22C6}' + ) || ch.is_numeric() + } +} + +fn is_word(ch: char) -> bool { + if ch.is_ascii() { + ch.is_ascii_alphabetic() + } else { + ch.is_alphabetic() && !is_unspaced(ch) + } +} + +impl Glyph { + fn is_marker(&self) -> bool { + is_marker(self.first) && is_marker(self.last) + } + + fn is_word(&self) -> bool { + is_word(self.first) && is_word(self.last) + } +} + +impl Run { + fn restart(&mut self, glyph: Glyph) { + self.level = Level::Base; + self.next_word(glyph); + } + + fn next_word(&mut self, glyph: Glyph) { + self.count = 0; + self.push(glyph); + self.marker = self.level != Level::Base && glyph.is_marker(); + } + + fn is_raised(&self) -> bool { + self.level == Level::Raised + } + + fn advance(&mut self, shift: f64, growth: f64, glyph: Glyph) -> Shifted { + let direction = if shift >= MIN_SHIFT { + Direction::Up + } else if shift <= -MIN_SHIFT { + Direction::Down + } else { + return self.extend(glyph); + }; + let grew = growth > SMALLER_FONT; + let (level, shifted) = match (self.level, direction) { + (Level::Raised, Direction::Up) | (Level::Lowered, Direction::Down) => { + return self.extend(glyph); + } + (Level::Lowered, Direction::Up) => (Level::Base, Shifted::Joined), + (Level::Base, Direction::Up) if grew => (Level::Base, Shifted::Joined), + (Level::Base, Direction::Up) => (Level::Raised, self.opening(glyph)), + (Level::Base, Direction::Down) if !grew => (Level::Lowered, Shifted::Joined), + (Level::Base, Direction::Down) => { + let marker = self.count <= WORD_LETTERS && self.word().iter().all(Glyph::is_marker); + (Level::Base, Self::closing(marker, glyph)) + } + (Level::Raised, Direction::Down) => ( + Level::Base, + Self::closing(self.detached && self.marker, glyph), + ), + }; + self.level = level; + self.next_word(glyph); + shifted + } + + fn opening(&mut self, glyph: Glyph) -> Shifted { + let word = self.word(); + let long_word = word.len() >= WORD_LETTERS && word.iter().all(Glyph::is_word); + self.detached = long_word || !word.last().is_some_and(Glyph::is_word); + if long_word && is_marker_start(glyph.first) { + Shifted::Tentative + } else { + Shifted::Joined + } + } + + fn closing(marker: bool, glyph: Glyph) -> Shifted { + if marker && is_word(glyph.first) { + Shifted::Space + } else { + Shifted::Joined + } + } + + fn word(&self) -> &[Glyph] { + self.recent + .get(WORD_LETTERS - self.count.min(WORD_LETTERS)..) + .unwrap_or_default() + } + + fn push(&mut self, glyph: Glyph) { + let [_, middle, last] = self.recent; + self.recent = [middle, last, glyph]; + self.count = self.count.saturating_add(1); + } + + fn extend(&mut self, glyph: Glyph) -> Shifted { + self.push(glyph); + if self.level == Level::Base || !self.marker { + return Shifted::Joined; + } + self.marker = glyph.is_marker(); + if self.level == Level::Raised && !self.marker { + Shifted::Retract + } else { + Shifted::Joined + } + } +} + +impl Layout { + pub(super) fn shift( + &mut self, + placement: &Placement, + separator: Separator, + first: char, + last: char, + ) -> Shifted { + let glyph = Glyph { first, last }; + let Some(previous) = self.previous.filter(|previous| { + separator != Separator::Newline && !previous.vertical && !placement.vertical + }) else { + self.run.restart(glyph); + return Shifted::Joined; + }; + let em = previous.em.max(placement.em).max(MIN_EM); + let shift = cross(previous.dir, sub(placement.origin, previous.end)) / em; + let growth = placement.em / previous.em.max(MIN_EM); + let shifted = self.run.advance(shift, growth, glyph); + if separator == Separator::Space { + self.run.next_word(glyph); + } + shifted + } + + pub(super) fn apply_shift(&mut self, shifted: Shifted, separator: &mut Separator) -> bool { + if *separator != Separator::None || !self.run.is_raised() { + self.tentative = None; + } + if *separator != Separator::None || self.actual.is_some() { + return false; + } + match shifted { + Shifted::Joined => false, + Shifted::Space => { + *separator = Separator::Space; + false + } + Shifted::Tentative => true, + Shifted::Retract => { + if let Some(mark) = self.tentative.take() { + self.line.retract(mark); + } + false + } + } + } +} diff --git a/crates/text-extract/src/pdf/lexer/mod.rs b/crates/text-extract/src/pdf/lexer/mod.rs new file mode 100644 index 00000000..71ac0e31 --- /dev/null +++ b/crates/text-extract/src/pdf/lexer/mod.rs @@ -0,0 +1,234 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod number; + + +use memchr::{memchr, memchr2, memchr3}; + +const WHITESPACE: u8 = 1; +const DELIMITER: u8 = 2; +static CLASSES: [u8; 256] = { + let mut table = [0u8; 256]; + table[0x00] = WHITESPACE; + table[0x09] = WHITESPACE; + table[0x0A] = WHITESPACE; + table[0x0C] = WHITESPACE; + table[0x0D] = WHITESPACE; + table[0x20] = WHITESPACE; + table[b'(' as usize] = DELIMITER; + table[b')' as usize] = DELIMITER; + table[b'<' as usize] = DELIMITER; + table[b'>' as usize] = DELIMITER; + table[b'[' as usize] = DELIMITER; + table[b']' as usize] = DELIMITER; + table[b'{' as usize] = DELIMITER; + table[b'}' as usize] = DELIMITER; + table[b'/' as usize] = DELIMITER; + table[b'%' as usize] = DELIMITER; + table +}; + +#[inline] +pub(crate) fn is_whitespace(byte: u8) -> bool { + CLASSES[byte as usize] == WHITESPACE +} + +#[inline] +pub(crate) fn is_regular(byte: u8) -> bool { + CLASSES[byte as usize] == 0 +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum Token<'a> { + Int(i64), + Real(f64), + Name(&'a [u8]), + Literal(&'a [u8]), + Hex(&'a [u8]), + ArrayOpen, + ArrayClose, + DictOpen, + DictClose, + BraceOpen, + BraceClose, + Keyword(&'a [u8]), + Error, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Lexer<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Lexer<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Lexer { data, pos: 0 } + } + + pub(crate) fn at(data: &'a [u8], pos: usize) -> Self { + Lexer { + data, + pos: pos.min(data.len()), + } + } + + #[inline] + pub(crate) fn pos(&self) -> usize { + self.pos + } + + #[inline] + pub(crate) fn data(&self) -> &'a [u8] { + self.data + } + + #[inline] + pub(crate) fn set_pos(&mut self, pos: usize) { + self.pos = pos.min(self.data.len()); + } + + #[inline] + pub(crate) fn rest(&self) -> &'a [u8] { + self.data.get(self.pos..).unwrap_or_default() + } + + #[inline] + pub(crate) fn at_end(&self) -> bool { + self.pos >= self.data.len() + } + + #[inline] + fn byte(&self, pos: usize) -> Option { + self.data.get(pos).copied() + } + + pub(crate) fn skip_whitespace(&mut self) { + while let Some(byte) = self.byte(self.pos) { + if is_whitespace(byte) { + self.pos += 1; + } else if byte == b'%' { + self.pos = match memchr2(b'\r', b'\n', self.rest()) { + Some(offset) => self.pos + offset, + None => self.data.len(), + }; + } else { + break; + } + } + } + + pub(crate) fn peek(&self) -> Option> { + let mut probe = *self; + probe.next() + } + + #[allow(clippy::should_implement_trait)] + pub(crate) fn next(&mut self) -> Option> { + self.skip_whitespace(); + let start = self.pos; + let byte = self.byte(start)?; + self.pos += 1; + Some(match byte { + b'0'..=b'9' | b'+' | b'-' | b'.' => self.number(start), + b'/' => { + let end = self.regular_run(self.pos); + let name = self.data.get(self.pos..end).unwrap_or_default(); + self.pos = end; + Token::Name(name) + } + b'(' => self.literal(), + b'<' => { + if self.byte(self.pos) == Some(b'<') { + self.pos += 1; + Token::DictOpen + } else { + self.hex() + } + } + b'>' => { + if self.byte(self.pos) == Some(b'>') { + self.pos += 1; + Token::DictClose + } else { + Token::Error + } + } + b'[' => Token::ArrayOpen, + b']' => Token::ArrayClose, + b'{' => Token::BraceOpen, + b'}' => Token::BraceClose, + b')' => Token::Error, + _ => { + if !(0x20..=0x7F).contains(&byte) + && self + .byte(self.pos) + .is_some_and(|next| (0x21..0x7F).contains(&next) && is_regular(next)) + { + return Some(Token::Keyword( + self.data.get(start..self.pos).unwrap_or_default(), + )); + } + let end = self.regular_run(self.pos); + self.pos = end; + Token::Keyword(self.data.get(start..end).unwrap_or_default()) + } + }) + } + + fn regular_run(&self, from: usize) -> usize { + self.data + .get(from..) + .unwrap_or_default() + .iter() + .position(|&byte| !is_regular(byte)) + .map_or(self.data.len(), |offset| from + offset) + } + + fn literal(&mut self) -> Token<'a> { + let start = self.pos; + let mut depth = 1u32; + let mut pos = start; + loop { + let Some(offset) = memchr3(b'(', b')', b'\\', self.data.get(pos..).unwrap_or_default()) + else { + self.pos = self.data.len(); + return Token::Literal(self.data.get(start..).unwrap_or_default()); + }; + pos += offset; + match self.byte(pos) { + Some(b'\\') => pos += 2, + Some(b'(') => { + depth = depth.saturating_add(1); + pos += 1; + } + _ => { + depth -= 1; + if depth == 0 { + self.pos = pos + 1; + return Token::Literal(self.data.get(start..pos).unwrap_or_default()); + } + pos += 1; + } + } + } + } + + fn hex(&mut self) -> Token<'a> { + let start = self.pos; + match memchr(b'>', self.rest()) { + Some(offset) => { + self.pos = start + offset + 1; + Token::Hex(self.data.get(start..start + offset).unwrap_or_default()) + } + None => { + self.pos = self.data.len(); + Token::Hex(self.data.get(start..).unwrap_or_default()) + } + } + } +} diff --git a/crates/text-extract/src/pdf/lexer/number.rs b/crates/text-extract/src/pdf/lexer/number.rs new file mode 100644 index 00000000..1e605ec3 --- /dev/null +++ b/crates/text-extract/src/pdf/lexer/number.rs @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Lexer, Token}; + +const MAX_REAL: f64 = f32::MAX as f64; +const MAX_FRACTION_DIGITS: usize = 18; +const MAX_EXACT_DIGITS: usize = 15; +const FRACTION_SCALES: [f64; MAX_FRACTION_DIGITS] = { + let mut scales = [0.0; MAX_FRACTION_DIGITS]; + let mut scale = 1.0; + let mut index = 0; + while index < MAX_FRACTION_DIGITS { + scale /= 10.0; + scales[index] = scale; + index += 1; + } + scales +}; + +impl<'a> Lexer<'a> { + pub(super) fn number(&mut self, start: usize) -> Token<'a> { + let mut pos = start; + let mut negative = false; + while let Some(sign @ (b'+' | b'-')) = self.byte(pos) { + negative |= sign == b'-'; + pos += 1; + } + if pos > start { + while let Some(b'\r' | b'\n') = self.byte(pos) { + pos += 1; + } + } + let mut integer = 0u64; + let mut integer_digits = 0usize; + let mut fraction = 0f64; + let mut fraction_digits = 0usize; + let mut dots = 0u32; + let mut digits = false; + for &byte in self.data.get(pos..).unwrap_or_default() { + let digit = byte.wrapping_sub(b'0'); + if digit < 10 { + digits = true; + if dots == 0 { + integer = integer.wrapping_mul(10).wrapping_add(u64::from(digit)); + integer_digits += 1; + } else if let Some(scale) = + FRACTION_SCALES.get(fraction_digits).filter(|_| dots == 1) + { + fraction += f64::from(digit) * scale; + fraction_digits += 1; + } + } else if byte == b'.' { + dots += 1; + } else if byte != b'+' && byte != b'-' { + break; + } + pos += 1; + } + if integer_digits > MAX_EXACT_DIGITS { + return self.wide_number(start); + } + self.pos = pos.max(start + 1); + if !digits { + return Token::Int(0); + } + let integer = integer as i64; + if dots == 0 { + return Token::Int(if negative { -integer } else { integer }); + } + let magnitude = (integer as f64 + fraction).min(MAX_REAL); + Token::Real(if negative { -magnitude } else { magnitude }) + } + + pub(super) fn wide_number(&mut self, start: usize) -> Token<'a> { + let mut pos = start; + let mut negative = false; + while let Some(sign @ (b'+' | b'-')) = self.byte(pos) { + negative |= sign == b'-'; + pos += 1; + } + if pos > start { + while let Some(b'\r' | b'\n') = self.byte(pos) { + pos += 1; + } + } + let mut integer: Option = Some(0); + let mut wide = 0f64; + let mut fraction = 0f64; + let mut scale = 1f64; + let mut fraction_digits = 0usize; + let mut dots = 0u32; + let mut digits = false; + while let Some(byte) = self.byte(pos) { + match byte { + b'0'..=b'9' => { + let digit = byte - b'0'; + digits = true; + if dots == 0 { + integer = integer + .and_then(|value| value.checked_mul(10)) + .and_then(|value| value.checked_add(i64::from(digit))); + wide = wide * 10.0 + f64::from(digit); + } else if dots == 1 && fraction_digits < MAX_FRACTION_DIGITS { + scale /= 10.0; + fraction += f64::from(digit) * scale; + fraction_digits += 1; + } + } + b'.' => dots += 1, + b'-' | b'+' => {} + _ => break, + } + pos += 1; + } + self.pos = pos.max(start + 1); + if !digits { + return Token::Int(0); + } + match integer { + Some(value) if dots == 0 => Token::Int(if negative { -value } else { value }), + _ => { + let magnitude = (wide + fraction).min(MAX_REAL); + Token::Real(if negative { -magnitude } else { magnitude }) + } + } + } +} diff --git a/crates/text-extract/src/pdf/mod.rs b/crates/text-extract/src/pdf/mod.rs new file mode 100644 index 00000000..21d8cafc --- /dev/null +++ b/crates/text-extract/src/pdf/mod.rs @@ -0,0 +1,125 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod content; +mod crypt; +mod decode; +mod document; +mod filter; +mod font; +mod forms; +mod layout; +mod lexer; +mod object; +mod objstm; +mod pages; +mod repair; +mod seek; +mod source; +mod tables; +mod text_string; +mod xref; + + +use self::{ + content::Interpreter, + document::{DocScratch, Document, OpenError}, + forms::FormScratch, + pages::{Pages, Visited}, +}; +use crate::{Error, Failure, Limits, output::Output, xml::stream::Budget}; +use memchr::memmem; + +const SIGNATURE: &[u8] = b"%PDF-"; +const SIGNATURE_WINDOW: usize = 1024; +const ZIP_LOCAL_HEADER: &[u8] = b"PK\x03\x04"; +const MAX_RETAINED_SCRATCH: usize = 1 << 20; + +#[derive(Default)] +pub(crate) struct Scratch { + document: DocScratch, + visited: Visited, + contents: Vec, + interpreter: Interpreter, + forms: FormScratch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Summary { + pub(crate) used_bytes: u64, + pub(crate) truncated: bool, +} + +pub(crate) fn is_pdf(data: &[u8]) -> bool { + !data.starts_with(ZIP_LOCAL_HEADER) + && memmem::find(data.get(..SIGNATURE_WINDOW).unwrap_or(data), SIGNATURE).is_some() +} + +impl Scratch { + fn shrink(&mut self) { + self.document.shrink(MAX_RETAINED_SCRATCH); + self.visited.shrink(MAX_RETAINED_SCRATCH); + self.contents.clear(); + self.contents.shrink_to(MAX_RETAINED_SCRATCH); + self.interpreter.shrink(); + self.forms.shrink(); + } +} + +pub(crate) fn extract( + data: &[u8], + scratch: &mut Scratch, + limits: &Limits, + out: &mut Output<'_>, +) -> Result { + let budget = Budget { + part_bytes: limits.max_part_bytes, + total_bytes: limits.max_total_bytes, + parts: usize::MAX, + used_bytes: 0, + used_parts: 0, + truncated: false, + }; + let Scratch { + document: document_scratch, + visited, + contents, + interpreter, + forms, + } = scratch; + let result = match Document::open(data, document_scratch, budget, limits.max_pdf_objects) { + Ok(document) => { + interpreter.begin_document(); + for page in Pages::new(&document, visited) { + if out.is_full() { + break; + } + let (content, _) = page.contents(&document, contents); + let media_box = page.media_box(&document); + interpreter.run_page(&document, content, page.resources, media_box, out); + forms::annotations(&document, &page, interpreter.layout(), out, forms); + interpreter.layout().end_page(out); + } + if !out.is_full() { + forms::acroform(&document, interpreter.layout(), out, forms, visited); + interpreter.layout().end_page(out); + } + let budget = document.finish(); + Ok(Summary { + used_bytes: budget.used_bytes, + truncated: budget.truncated, + }) + } + Err(failure) => Err(Failure { + error: match failure.error { + OpenError::Encrypted | OpenError::Unusable => Error::Unsupported, + }, + bytes_decompressed: failure.used_bytes, + }), + }; + scratch.shrink(); + result +} diff --git a/crates/text-extract/src/pdf/object/container.rs b/crates/text-extract/src/pdf/object/container.rs new file mode 100644 index 00000000..e52a51c6 --- /dev/null +++ b/crates/text-extract/src/pdf/object/container.rs @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Name, ObjRef, Object}; +use crate::pdf::lexer::{Lexer, Token}; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Array<'a> { + body: &'a [u8], + owner: Option, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Dict<'a> { + body: &'a [u8], + owner: Option, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ArrayIter<'a> { + lexer: Lexer<'a>, + owner: Option, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct DictIter<'a> { + lexer: Lexer<'a>, + owner: Option, +} + +pub(super) fn skip<'a>(lexer: &mut Lexer<'a>) -> &'a [u8] { + skip_container(lexer).0 +} + +pub(crate) fn skip_container<'a>(lexer: &mut Lexer<'a>) -> (&'a [u8], bool) { + let data = lexer.data(); + let start = lexer.pos(); + let mut depth = 1usize; + loop { + let before = lexer.pos(); + let Some(token) = lexer.next() else { + return (data.get(start..).unwrap_or_default(), false); + }; + match token { + Token::ArrayOpen | Token::DictOpen => depth += 1, + Token::ArrayClose | Token::DictClose => { + depth -= 1; + if depth == 0 { + return (data.get(start..before).unwrap_or_default(), true); + } + } + Token::Keyword( + b"endobj" | b"stream" | b"endstream" | b"obj" | b"trailer" | b"xref", + ) => { + lexer.set_pos(before); + return (data.get(start..before).unwrap_or_default(), false); + } + _ => {} + } + } +} + +impl<'a> Array<'a> { + pub(crate) fn new(body: &'a [u8], owner: Option) -> Self { + Array { body, owner } + } + + pub(crate) fn iter(&self) -> ArrayIter<'a> { + ArrayIter { + lexer: Lexer::new(self.body), + owner: self.owner, + } + } + + pub(crate) fn get(&self, index: usize) -> Option> { + self.iter().nth(index) + } + + pub(crate) fn body(&self) -> &'a [u8] { + self.body + } +} + +impl<'a> Dict<'a> { + pub(crate) fn new(body: &'a [u8], owner: Option) -> Self { + Dict { body, owner } + } + + pub(crate) fn iter(&self) -> DictIter<'a> { + DictIter { + lexer: Lexer::new(self.body), + owner: self.owner, + } + } + + pub(crate) fn get(&self, key: &[u8]) -> Option> { + self.iter() + .filter(|(name, _)| name.is(key)) + .last() + .map(|(_, value)| value) + } + + pub(crate) fn contains(&self, key: &[u8]) -> bool { + self.iter().any(|(name, _)| name.is(key)) + } + + pub(crate) fn is_type(&self, name: &[u8]) -> bool { + self.get(b"Type").is_some_and(|value| value.is_name(name)) + } + + pub(crate) fn body(&self) -> &'a [u8] { + self.body + } +} + +impl<'a> Iterator for ArrayIter<'a> { + type Item = Object<'a>; + + fn next(&mut self) -> Option { + loop { + if let Some(object) = Object::read(&mut self.lexer, self.owner) { + return Some(object); + } + self.lexer.next()?; + } + } +} + +impl<'a> Iterator for DictIter<'a> { + type Item = (Name<'a>, Object<'a>); + + fn next(&mut self) -> Option { + loop { + match self.lexer.next()? { + Token::Name(raw) => { + if let Some(value) = Object::read(&mut self.lexer, self.owner) { + return Some((Name::new(raw), value)); + } + } + Token::ArrayOpen | Token::DictOpen => { + skip(&mut self.lexer); + } + _ => {} + } + } + } +} + diff --git a/crates/text-extract/src/pdf/object/mod.rs b/crates/text-extract/src/pdf/object/mod.rs new file mode 100644 index 00000000..a87d3cbe --- /dev/null +++ b/crates/text-extract/src/pdf/object/mod.rs @@ -0,0 +1,318 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod container; +mod string; + +pub(crate) use container::{Array, ArrayIter, Dict, skip_container}; +pub(crate) use string::{Name, Str}; + +use super::lexer::{Lexer, Token, is_regular, is_whitespace}; + +pub(crate) const MAX_OBJECT_NUMBER: u32 = 8_388_607; +const STREAM_KEYWORD: &[u8] = b"stream"; +const OBJ_KEYWORD: &[u8] = b"obj"; +const REF_KEYWORD: &[u8] = b"R"; +const MAX_NUM_DIGITS: usize = 10; +const MAX_GEN_DIGITS: usize = 5; +const MAX_GEN_TOKEN: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct ObjRef { + pub(crate) num: u32, + pub(crate) generation: u16, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Stream<'a> { + pub(crate) dict: Dict<'a>, + pub(crate) data: &'a [u8], + pub(crate) id: ObjRef, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) enum Object<'a> { + #[default] + Null, + Bool(bool), + Int(i64), + Real(f64), + Name(Name<'a>), + Str(Str<'a>), + Array(Array<'a>), + Dict(Dict<'a>), + Ref(ObjRef), + Stream(Stream<'a>), +} + +pub(crate) struct Indirect<'a> { + pub(crate) id: ObjRef, + pub(crate) value: Object<'a>, + pub(crate) stream_start: Option, + pub(crate) end: usize, +} + +impl ObjRef { + pub(crate) fn new(num: i64, generation: i64) -> Option { + let num = u32::try_from(num) + .ok() + .filter(|num| (1..=MAX_OBJECT_NUMBER).contains(num))?; + Some(ObjRef { + num, + generation: u16::try_from(generation).ok()?, + }) + } +} + +impl<'a> Object<'a> { + #[inline] + pub(crate) fn is_null(&self) -> bool { + matches!(self, Object::Null) + } + + pub(crate) fn as_int(&self) -> Option { + match *self { + Object::Int(value) => Some(value), + Object::Real(value) if value.is_finite() => Some(value as i64), + _ => None, + } + } + + pub(crate) fn as_f64(&self) -> Option { + match *self { + Object::Int(value) => Some(value as f64), + Object::Real(value) if value.is_finite() => Some(value), + _ => None, + } + } + + pub(crate) fn as_bool(&self) -> Option { + match *self { + Object::Bool(value) => Some(value), + _ => None, + } + } + + pub(crate) fn as_name(&self) -> Option> { + match *self { + Object::Name(name) => Some(name), + _ => None, + } + } + + pub(crate) fn as_str(&self) -> Option> { + match *self { + Object::Str(value) => Some(value), + _ => None, + } + } + + pub(crate) fn as_array(&self) -> Option> { + match *self { + Object::Array(array) => Some(array), + _ => None, + } + } + + pub(crate) fn as_dict(&self) -> Option> { + match *self { + Object::Dict(dict) => Some(dict), + Object::Stream(stream) => Some(stream.dict), + _ => None, + } + } + + pub(crate) fn as_stream(&self) -> Option> { + match *self { + Object::Stream(stream) => Some(stream), + _ => None, + } + } + + pub(crate) fn as_ref(&self) -> Option { + match *self { + Object::Ref(id) => Some(id), + _ => None, + } + } + + pub(crate) fn is_name(&self, name: &[u8]) -> bool { + self.as_name().is_some_and(|value| value.is(name)) + } + + pub(crate) fn read(lexer: &mut Lexer<'a>, owner: Option) -> Option { + let saved = *lexer; + let object = match lexer.next()? { + Token::Int(value) => return Some(Object::integer_or_ref(value, lexer)), + Token::Real(value) => Object::Real(value), + Token::Name(raw) => Object::Name(Name::new(raw)), + Token::Literal(raw) => Object::Str(Str::literal(raw, owner)), + Token::Hex(raw) => Object::Str(Str::hex(raw, owner)), + Token::ArrayOpen => Object::Array(Array::new(container::skip(lexer), owner)), + Token::DictOpen => Object::Dict(Dict::new(container::skip(lexer), owner)), + Token::Keyword(b"true") => Object::Bool(true), + Token::Keyword(b"false") => Object::Bool(false), + Token::Keyword(b"null") => Object::Null, + Token::Keyword(_) + | Token::ArrayClose + | Token::DictClose + | Token::BraceOpen + | Token::BraceClose + | Token::Error => { + *lexer = saved; + return None; + } + }; + Some(object) + } + + fn integer_or_ref(value: i64, lexer: &mut Lexer<'a>) -> Self { + let mut probe = *lexer; + probe.skip_whitespace(); + let rest = probe.rest(); + if rest + .first() + .is_some_and(|&byte| byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b'.')) + && rest + .iter() + .take(MAX_GEN_TOKEN + 1) + .take_while(|&&byte| is_regular(byte)) + .count() + <= MAX_GEN_TOKEN + && let Some(Token::Int(generation)) = probe.next() + && let Some(end) = keyword_after(probe, REF_KEYWORD) + { + lexer.set_pos(end); + return ObjRef::new(value, generation).map_or(Object::Null, Object::Ref); + } + Object::Int(value) + } +} + +impl<'a> Indirect<'a> { + pub(crate) fn header(data: &'a [u8], pos: usize) -> Option<(ObjRef, Lexer<'a>)> { + let mut lexer = Lexer::at(data, pos); + lexer.skip_whitespace(); + let (num, after_num) = digits(data, lexer.pos(), MAX_NUM_DIGITS)?; + let gap = spaces(data, after_num); + if gap == 0 { + return None; + } + let (generation, after_gen) = match digits(data, after_num + gap, MAX_GEN_DIGITS) { + Some((generation, end)) => (generation, end + spaces(data, end)), + None => (0, after_num + gap), + }; + if !keyword_at(data, after_gen, OBJ_KEYWORD) { + return None; + } + let id = ObjRef::new(num, generation)?; + Some((id, Lexer::at(data, after_gen + OBJ_KEYWORD.len()))) + } + + pub(crate) fn header_before(data: &[u8], keyword: usize) -> Option { + if !keyword_at(data, keyword, OBJ_KEYWORD) { + return None; + } + let digits_end = run_start(data, keyword, is_whitespace); + let last = run_start(data, digits_end, |byte| byte.is_ascii_digit()); + if last == digits_end { + return None; + } + let gap = run_start(data, last, is_whitespace); + let first = run_start(data, gap, |byte| byte.is_ascii_digit()); + let full = (gap < last && first < gap).then_some(first); + full.into_iter().chain([last]).find(|&start| { + start + .checked_sub(1) + .and_then(|before| data.get(before)) + .is_none_or(|&before| !is_regular(before)) + && Indirect::header(data, start) + .is_some_and(|(_, lexer)| lexer.pos() == keyword + OBJ_KEYWORD.len()) + }) + } + + pub(crate) fn parse(data: &'a [u8], pos: usize, streams: bool) -> Option { + let (id, mut lexer) = Indirect::header(data, pos)?; + let value = Object::read(&mut lexer, Some(id)).unwrap_or_default(); + lexer.skip_whitespace(); + let mut end = lexer.pos(); + let stream_start = match value { + Object::Dict(_) if streams => keyword_after(lexer, STREAM_KEYWORD).map(|after| { + end = after; + stream_data_start(data, after) + }), + _ => None, + }; + Some(Indirect { + id, + value, + stream_start, + end, + }) + } +} + +pub(crate) fn keyword_after(mut lexer: Lexer<'_>, keyword: &[u8]) -> Option { + lexer.skip_whitespace(); + let pos = lexer.pos(); + keyword_at(lexer.data(), pos, keyword).then_some(pos + keyword.len()) +} + +pub(crate) fn stream_data_start(data: &[u8], after_keyword: usize) -> usize { + let mut pos = after_keyword; + while let Some(b' ' | b'\t') = data.get(pos) { + pos += 1; + } + match (data.get(pos), data.get(pos + 1)) { + (Some(b'\r'), Some(b'\n')) => pos + 2, + (Some(b'\r' | b'\n'), _) => pos + 1, + _ => pos, + } + .min(data.len()) +} + +fn digits(data: &[u8], pos: usize, max: usize) -> Option<(i64, usize)> { + let run = data + .get(pos..)? + .iter() + .take(max + 1) + .take_while(|byte| byte.is_ascii_digit()) + .count(); + if run == 0 || run > max { + return None; + } + let value = data + .get(pos..pos + run)? + .iter() + .fold(0i64, |value, &digit| value * 10 + i64::from(digit - b'0')); + Some((value, pos + run)) +} + +fn run_start(data: &[u8], end: usize, class: impl Fn(u8) -> bool) -> usize { + let run = data + .get(..end) + .unwrap_or_default() + .iter() + .rev() + .take_while(|&&byte| class(byte)) + .count(); + end - run +} + +fn spaces(data: &[u8], pos: usize) -> usize { + data.get(pos..) + .unwrap_or_default() + .iter() + .take_while(|&&byte| is_whitespace(byte)) + .count() +} + +pub(crate) fn keyword_at(data: &[u8], pos: usize, keyword: &[u8]) -> bool { + data.get(pos..) + .and_then(|rest| rest.strip_prefix(keyword)) + .is_some_and(|rest| rest.first().is_none_or(|&byte| !is_regular(byte))) +} + diff --git a/crates/text-extract/src/pdf/object/string.rs b/crates/text-extract/src/pdf/object/string.rs new file mode 100644 index 00000000..909d210d --- /dev/null +++ b/crates/text-extract/src/pdf/object/string.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::ObjRef; +use std::borrow::Cow; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Name<'a> { + raw: &'a [u8], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Encoding { + Literal, + Hex, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Str<'a> { + raw: &'a [u8], + encoding: Encoding, + owner: Option, +} + +pub(crate) struct NameBytes<'a> { + rest: &'a [u8], +} + +impl<'a> Name<'a> { + pub(crate) fn new(raw: &'a [u8]) -> Self { + Name { raw } + } + + pub(crate) fn raw(&self) -> &'a [u8] { + self.raw + } + + pub(crate) fn bytes(&self) -> NameBytes<'a> { + NameBytes { rest: self.raw } + } + + pub(crate) fn is(&self, other: &[u8]) -> bool { + if !self.raw.contains(&b'#') { + return self.raw == other; + } + self.bytes().eq(other.iter().copied()) + } + + pub(crate) fn decoded(&self) -> Cow<'a, [u8]> { + if self.raw.contains(&b'#') { + Cow::Owned(self.bytes().collect()) + } else { + Cow::Borrowed(self.raw) + } + } +} + +impl Iterator for NameBytes<'_> { + type Item = u8; + + fn next(&mut self) -> Option { + let (&first, rest) = self.rest.split_first()?; + if first == b'#' + && let [high, low, tail @ ..] = rest + && let (Some(high), Some(low)) = (hex_value(*high), hex_value(*low)) + { + self.rest = tail; + return Some(high << 4 | low); + } + self.rest = rest; + Some(first) + } +} + +impl<'a> Str<'a> { + pub(crate) fn literal(raw: &'a [u8], owner: Option) -> Self { + Str { + raw, + encoding: Encoding::Literal, + owner, + } + } + + pub(crate) fn hex(raw: &'a [u8], owner: Option) -> Self { + Str { + raw, + encoding: Encoding::Hex, + owner, + } + } + + pub(crate) fn owner(&self) -> Option { + self.owner + } + + pub(crate) fn plain(&self) -> Option<&'a [u8]> { + (self.encoding == Encoding::Literal && !self.raw.iter().any(|&b| b == b'\\' || b == b'\r')) + .then_some(self.raw) + } + + pub(crate) fn decoded(&self) -> Cow<'a, [u8]> { + if let Some(raw) = self.plain() { + return Cow::Borrowed(raw); + } + let mut out = Vec::with_capacity(self.raw.len()); + self.decode_into(&mut out); + Cow::Owned(out) + } + + pub(crate) fn decode_into(&self, out: &mut Vec) { + match self.encoding { + Encoding::Literal => decode_literal(self.raw, out), + Encoding::Hex => decode_hex(self.raw, out), + } + } +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn decode_hex(raw: &[u8], out: &mut Vec) { + out.reserve(raw.len() / 2 + 1); + let mut high = None; + for digit in raw.iter().filter_map(|&byte| hex_value(byte)) { + match high.take() { + None => high = Some(digit), + Some(value) => out.push(value << 4 | digit), + } + } + if let Some(value) = high { + out.push(value << 4); + } +} + +fn decode_literal(raw: &[u8], out: &mut Vec) { + out.reserve(raw.len()); + let mut bytes = raw.iter().copied().peekable(); + while let Some(byte) = bytes.next() { + match byte { + b'\\' => { + let Some(escaped) = bytes.next() else { + break; + }; + match escaped { + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'b' => out.push(0x08), + b'f' => out.push(0x0C), + b'0'..=b'7' => { + let mut value = u32::from(escaped - b'0'); + for _ in 0..2 { + match bytes.peek() { + Some(&digit @ b'0'..=b'7') => { + value = value * 8 + u32::from(digit - b'0'); + bytes.next(); + } + _ => break, + } + } + out.push((value & 0xFF) as u8); + } + b'\r' => { + bytes.next_if_eq(&b'\n'); + } + b'\n' => {} + other => out.push(other), + } + } + b'\r' => { + bytes.next_if_eq(&b'\n'); + out.push(b'\n'); + } + other => out.push(other), + } + } +} + diff --git a/crates/text-extract/src/pdf/objstm.rs b/crates/text-extract/src/pdf/objstm.rs new file mode 100644 index 00000000..2bd66df6 --- /dev/null +++ b/crates/text-extract/src/pdf/objstm.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + lexer::{Lexer, Token}, + object::Object, +}; +use std::cell::OnceCell; + +const MAX_MEMBERS: usize = 1_000_000; +const MIN_MEMBER_BYTES: usize = 3; + +#[derive(Debug, Clone, Copy)] +struct Member { + num: u32, + offset: u32, +} + +pub(crate) struct ObjStm { + data: Vec, + first: usize, + members: Vec, + by_number: OnceCell>, + sequential: OnceCell>, +} + +impl ObjStm { + pub(crate) fn new(data: Vec, count: Option, first: Option) -> Self { + let limit = count + .and_then(|count| usize::try_from(count).ok()) + .unwrap_or(MAX_MEMBERS) + .min(MAX_MEMBERS) + .min(data.len() / MIN_MEMBER_BYTES); + let declared_first = first + .and_then(|first| usize::try_from(first).ok()) + .filter(|&first| first <= data.len()); + let header = data + .get(..declared_first.unwrap_or(data.len())) + .unwrap_or_default(); + let mut lexer = Lexer::new(header); + let mut members = Vec::new(); + let mut end = 0; + while members.len() < limit { + let mut probe = lexer; + let (Some(Token::Int(num)), Some(Token::Int(offset))) = (probe.next(), probe.next()) + else { + break; + }; + lexer = probe; + end = lexer.pos(); + if let (Ok(num), Ok(offset)) = (u32::try_from(num), u32::try_from(offset)) { + members.push(Member { num, offset }); + } + } + ObjStm { + first: declared_first.unwrap_or(end), + data, + members, + by_number: OnceCell::new(), + sequential: OnceCell::new(), + } + } + + pub(crate) fn members(&self) -> impl Iterator + '_ { + self.members + .iter() + .zip(0u32..) + .map(|(member, index)| (member.num, index)) + } + + pub(crate) fn object(&self, index: u32, num: u32) -> Option<(Object<'_>, usize)> { + let index = match self.members.get(index as usize) { + Some(member) if member.num == num => index, + _ => self.index_of(num)?, + }; + let member = self.members.get(index as usize)?; + let start = self.first.checked_add(member.offset as usize)?; + if let Some(found) = self.read_at(start) { + return Some(found); + } + let sequential = self.sequential.get_or_init(|| self.sequential_starts()); + self.read_at(*sequential.get(index as usize)? as usize) + } + + fn read_at(&self, start: usize) -> Option<(Object<'_>, usize)> { + if start >= self.data.len() { + return None; + } + let mut lexer = Lexer::at(&self.data, start); + let object = Object::read(&mut lexer, None)?; + Some((object, lexer.pos() - start)) + } + + fn index_of(&self, num: u32) -> Option { + let sorted = self.by_number.get_or_init(|| { + let mut sorted: Vec<(u32, u32)> = self.members().collect(); + sorted.sort_unstable(); + sorted + }); + let position = sorted.partition_point(|&(member, _)| member < num); + sorted + .get(position) + .filter(|&&(member, _)| member == num) + .map(|&(_, index)| index) + } + + fn sequential_starts(&self) -> Vec { + let mut lexer = Lexer::at(&self.data, self.first); + let mut starts = Vec::with_capacity(self.members.len()); + while starts.len() < self.members.len() { + lexer.skip_whitespace(); + let Ok(start) = u32::try_from(lexer.pos()) else { + break; + }; + if Object::read(&mut lexer, None).is_some() { + starts.push(start); + } else if lexer.next().is_none() { + break; + } + } + starts + } +} + diff --git a/crates/text-extract/src/pdf/pages.rs b/crates/text-extract/src/pdf/pages.rs new file mode 100644 index 00000000..fc8addcb --- /dev/null +++ b/crates/text-extract/src/pdf/pages.rs @@ -0,0 +1,355 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + decode::DecodeOutcome, + document::Document, + object::{Array, ArrayIter, Dict, MAX_OBJECT_NUMBER, ObjRef, Object}, +}; + +pub(crate) const MAX_TREE_DEPTH: usize = 64; +const WORD_BITS: usize = u64::BITS as usize; +const CONTENT_SEPARATOR: u8 = b'\n'; + +#[derive(Default)] +pub(crate) struct Visited { + words: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Page<'d> { + pub(crate) dict: Dict<'d>, + #[cfg_attr( + not(all(test, stalwart_tests)), + expect( + dead_code, + reason = "page identity is reported by the corpus inspection test" + ) + )] + pub(crate) id: Option, + pub(crate) resources: Option>, + pub(crate) media_box: Option>, + #[cfg_attr( + not(all(test, stalwart_tests)), + expect( + dead_code, + reason = "rotation never changes content order, only the page-tree tests read it" + ) + )] + pub(crate) rotate: i64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Contents { + pub(crate) streams: usize, + pub(crate) failures: usize, + pub(crate) truncated: bool, +} + +#[derive(Clone, Copy)] +struct Inherited<'d> { + resources: Option>, + media_box: Option>, + rotate: i64, +} + +enum Kids<'d> { + Many(ArrayIter<'d>), + One(Option>), +} + +struct Frame<'d> { + kids: Kids<'d>, + inherited: Inherited<'d>, +} + +pub(crate) struct Pages<'d, 'a, 'v> { + doc: &'d Document<'a>, + visited: &'v mut Visited, + root: Option>, + stack: Vec>, + yielded: usize, + fallback: Option, +} + +impl Visited { + pub(crate) fn clear(&mut self) { + self.words.clear(); + } + + pub(crate) fn shrink(&mut self, max_bytes: usize) { + self.words.clear(); + self.words.shrink_to(max_bytes / std::mem::size_of::()); + } + + pub(crate) fn insert(&mut self, num: u32) -> bool { + if num > MAX_OBJECT_NUMBER { + return false; + } + let (word, bit) = (num as usize / WORD_BITS, num as usize % WORD_BITS); + if word >= self.words.len() { + self.words.resize(word + 1, 0); + } + let Some(slot) = self.words.get_mut(word) else { + return false; + }; + let mask = 1u64 << bit; + let fresh = *slot & mask == 0; + *slot |= mask; + fresh + } +} + +impl<'d, 'a, 'v> Pages<'d, 'a, 'v> { + pub(crate) fn new(doc: &'d Document<'a>, visited: &'v mut Visited) -> Self { + visited.clear(); + Pages { + root: doc.page_tree_root(), + doc, + visited, + stack: Vec::new(), + yielded: 0, + fallback: None, + } + } + + fn visit(&mut self, node: Object<'d>, inherited: Inherited<'d>) -> Option> { + let id = node.as_ref(); + if let Some(id) = id + && !self.visited.insert(id.num) + { + return None; + } + let Object::Dict(dict) = self.doc.resolve(node) else { + return None; + }; + let doc = self.doc; + let (mut resources, mut rotate, mut kids, mut kind) = (None, None, None, None); + let mut media_box = None; + for (key, value) in dict.iter() { + hashify::fnc_map!(key.decoded(), + b"Resources" => resources = Some(value), + b"MediaBox" => media_box = Some(value), + b"Rotate" => rotate = Some(value), + b"Kids" => kids = Some(value), + b"Type" => kind = Some(value), + _ => {} + ); + } + let inherited = Inherited { + resources: resources + .and_then(|value| doc.resolve(value).as_dict()) + .or(inherited.resources), + media_box: media_box + .and_then(|value| doc.resolve(value).as_array()) + .or(inherited.media_box), + rotate: rotate + .and_then(|value| doc.resolve(value).as_int()) + .unwrap_or(inherited.rotate), + }; + let kids = match kids.map(|value| (value, doc.resolve(value))) { + Some((_, Object::Array(array))) => Some(Kids::Many(array.iter())), + Some((Object::Ref(kid), Object::Dict(_))) => Some(Kids::One(Some(Object::Ref(kid)))), + _ => None, + }; + let kind = kind.and_then(|value| doc.resolve(value).as_name()); + let is_tree = match kind { + Some(kind) if kind.is(b"Pages") => true, + Some(kind) if kind.is(b"Page") => false, + _ => kids.is_some(), + }; + if !is_tree { + self.yielded += 1; + return Some(Page { + dict, + id, + resources: inherited.resources, + media_box: inherited.media_box, + rotate: inherited.rotate, + }); + } + if let Some(kids) = kids + && self.stack.len() < MAX_TREE_DEPTH + { + self.stack.push(Frame { kids, inherited }); + } + None + } + + fn next_fallback(&mut self) -> Option> { + let doc = self.doc; + let candidates = doc.fallback_pages(); + let mut index = self.fallback.unwrap_or(0); + while let Some(&id) = candidates.get(index) { + index += 1; + self.fallback = Some(index); + if !self.visited.insert(id.num) { + continue; + } + let Object::Dict(dict) = doc.get(id) else { + continue; + }; + let inherited = Inherited::from_ancestors(doc, dict); + return Some(Page { + dict, + id: Some(id), + resources: inherited.resources, + media_box: inherited.media_box, + rotate: inherited.rotate, + }); + } + self.fallback = Some(index); + None + } +} + +impl<'d> Inherited<'d> { + fn from_ancestors(doc: &'d Document<'_>, page: Dict<'d>) -> Self { + let mut inherited = Inherited { + resources: None, + media_box: None, + rotate: 0, + }; + let mut rotate = None; + let mut node = Some(page); + for _ in 0..MAX_TREE_DEPTH { + let Some(dict) = node else { + break; + }; + inherited.resources = inherited + .resources + .or_else(|| doc.get_dict(dict, b"Resources")); + inherited.media_box = inherited + .media_box + .or_else(|| doc.get_array(dict, b"MediaBox")); + rotate = rotate.or_else(|| doc.get_int(dict, b"Rotate")); + if inherited.resources.is_some() && inherited.media_box.is_some() && rotate.is_some() { + break; + } + node = doc.get_dict(dict, b"Parent"); + } + inherited.rotate = rotate.unwrap_or(0); + inherited + } +} + +impl<'d> Iterator for Pages<'d, '_, '_> { + type Item = Page<'d>; + + fn next(&mut self) -> Option> { + loop { + if self.doc.exhausted() { + return None; + } + if let Some(root) = self.root.take() { + let root_page = self.visit( + root, + Inherited { + resources: None, + media_box: None, + rotate: 0, + }, + ); + if root_page.is_some() { + return root_page; + } + continue; + } + if let Some(frame) = self.stack.last_mut() { + let inherited = frame.inherited; + let kid = match &mut frame.kids { + Kids::Many(kids) => kids.next(), + Kids::One(kid) => kid.take(), + }; + match kid { + Some(kid) => { + if let Some(page) = self.visit(kid, inherited) { + return Some(page); + } + } + None => { + self.stack.pop(); + } + } + continue; + } + if self.yielded > 0 { + return None; + } + return self.next_fallback(); + } + } +} + +impl<'d> Page<'d> { + pub(crate) fn media_box(&self, doc: &'d Document<'_>) -> Option<[f64; 4]> { + let mut values = [0f64; 4]; + let mut count = 0; + for (slot, value) in values.iter_mut().zip(doc.array_iter(self.media_box?)) { + *slot = value.as_f64()?; + count += 1; + } + let [x0, y0, x1, y1] = values; + (count == values.len()).then(|| [x0.min(x1), y0.min(y1), x0.max(x1), y0.max(y1)]) + } + + pub(crate) fn annots(&self, doc: &'d Document<'_>) -> Option> { + doc.get_array(self.dict, b"Annots") + } + + pub(crate) fn contents<'r>( + &self, + doc: &'d Document<'_>, + out: &'r mut Vec, + ) -> (&'r [u8], Contents) + where + 'd: 'r, + { + out.clear(); + match doc.dict_get(self.dict, b"Contents") { + Object::Stream(stream) => { + let (data, outcome) = doc.stream_bytes(stream, out); + let contents = Contents { + streams: 1, + failures: usize::from(outcome.is_failure()), + truncated: outcome == DecodeOutcome::Truncated, + }; + (data, contents) + } + Object::Array(array) => { + let contents = Page::concatenate(doc, array, out); + (out, contents) + } + _ => (out, Contents::default()), + } + } + + fn concatenate(doc: &'d Document<'_>, array: Array<'d>, out: &mut Vec) -> Contents { + let mut contents = Contents::default(); + let mut append = |object: Object<'_>, contents: &mut Contents| { + let Some(stream) = object.as_stream() else { + return; + }; + if !out.is_empty() { + out.push(CONTENT_SEPARATOR); + } + contents.streams += 1; + match doc.decode_stream(stream, out) { + DecodeOutcome::Truncated => contents.truncated = true, + outcome if outcome.is_failure() => contents.failures += 1, + _ => {} + } + }; + for item in doc.array_iter(array) { + if doc.exhausted() { + contents.truncated = true; + break; + } + append(item, &mut contents); + } + contents + } +} diff --git a/crates/text-extract/src/pdf/repair.rs b/crates/text-extract/src/pdf/repair.rs new file mode 100644 index 00000000..05cac1dc --- /dev/null +++ b/crates/text-extract/src/pdf/repair.rs @@ -0,0 +1,226 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + lexer::{Lexer, Token, is_regular, is_whitespace}, + object::{ + Dict, Indirect, ObjRef, Object, keyword_after, keyword_at, skip_container, + stream_data_start, + }, + seek::Seek, + source::Source, +}; +use memchr::{memchr2, memmem::Finder}; + +const TRAILER: &[u8] = b"trailer"; +const STREAM: &[u8] = b"stream"; +const OBJ: &[u8] = b"obj"; +const TRAILER_LEN: u32 = TRAILER.len() as u32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Plain, + XRef, + ObjStm, + Catalog, + PageTree, + Page, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Found { + pub(crate) id: ObjRef, + pub(crate) offset: u32, + pub(crate) kind: Kind, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Trailer { + pub(crate) offset: u32, + pub(crate) end: u32, +} + +#[derive(Default)] +pub(crate) struct Scan { + pub(crate) objects: Vec, + pub(crate) trailers: Vec, +} + +struct Headers<'a> { + data: &'a [u8], + finder: Finder<'static>, + seek: Seek, +} + +impl Kind { + pub(crate) fn of(dict: Dict<'_>) -> Self { + let Some(kind) = dict.get(b"Type").and_then(|kind| kind.as_name()) else { + return if dict.contains(b"Parent") && dict.contains(b"Contents") { + Kind::Page + } else { + Kind::Plain + }; + }; + hashify::fnc_map!(kind.decoded(), + b"XRef" => Kind::XRef, + b"ObjStm" => Kind::ObjStm, + b"Catalog" => Kind::Catalog, + b"Pages" => { + if dict.contains(b"Parent") { + Kind::Plain + } else { + Kind::PageTree + } + }, + b"Page" => Kind::Page, + _ => Kind::Plain + ) + } +} + +impl Scan { + pub(crate) fn run(source: &Source<'_>) -> Self { + let data = source.data; + let mut headers = Headers::new(data); + let mut scan = Scan::default(); + let mut pos = 0usize; + while let Some(&byte) = data.get(pos) { + if is_whitespace(byte) { + pos += 1; + } else if byte == b'%' { + pos = memchr2(b'\r', b'\n', data.get(pos..).unwrap_or_default()) + .map_or(data.len(), |offset| pos + offset); + } else if byte.is_ascii_digit() + && pos + .checked_sub(1) + .and_then(|before| data.get(before)) + .is_none_or(|&before| !is_regular(before)) + { + pos = match Indirect::header(data, pos) { + Some((id, lexer)) => { + let body = lexer.pos(); + let limit = headers.next(body); + let Some(end) = scan.object(source, pos, id, body, limit) else { + break; + }; + end + } + None => run_end(data, pos), + }; + } else if is_regular(byte) { + if keyword_at(data, pos, TRAILER) { + scan.trailer(pos, headers.next(pos + TRAILER.len())); + } + pos = run_end(data, pos); + } else { + pos += 1; + } + } + scan.bound_trailers(); + scan + } + + fn object( + &mut self, + source: &Source<'_>, + start: usize, + id: ObjRef, + body: usize, + limit: usize, + ) -> Option { + let bounded = source.data.get(..limit.max(body)).unwrap_or(source.data); + let (kind, end, lexed) = body_end(source, bounded, body); + if !source.charge_scan(lexed.saturating_sub(body)) { + return None; + } + if let Ok(offset) = u32::try_from(start) { + self.objects.push(Found { id, offset, kind }); + } + Some(end.max(body).max(start + 1)) + } + + fn trailer(&mut self, keyword: usize, limit: usize) { + if let (Ok(offset), Ok(end)) = + (u32::try_from(keyword + TRAILER.len()), u32::try_from(limit)) + { + self.trailers.push(Trailer { offset, end }); + } + } + + fn bound_trailers(&mut self) { + let mut next_start = u32::MAX; + for trailer in self.trailers.iter_mut().rev() { + trailer.end = trailer.end.min(next_start); + next_start = trailer.offset.saturating_sub(TRAILER_LEN); + } + } +} + +impl Trailer { + pub(crate) fn dict<'a>(&self, data: &'a [u8]) -> Option> { + let bounded = data.get(..self.end as usize).unwrap_or(data); + Object::read(&mut Lexer::at(bounded, self.offset as usize), None)?.as_dict() + } +} + +impl<'a> Headers<'a> { + fn new(data: &'a [u8]) -> Self { + Headers { + data, + finder: Finder::new(OBJ), + seek: Seek::default(), + } + } + + fn next(&mut self, from: usize) -> usize { + let Headers { data, finder, seek } = self; + seek.find(from, |from| { + finder.find_iter(data.get(from..)?).find_map(|offset| { + Indirect::header_before(data, from + offset).filter(|&start| start >= from) + }) + }) + .unwrap_or(data.len()) + } +} + +fn run_end(data: &[u8], pos: usize) -> usize { + data.get(pos..) + .unwrap_or_default() + .iter() + .position(|&byte| !is_regular(byte)) + .map_or(data.len(), |offset| pos + offset) + .max(pos + 1) +} + +fn body_end(source: &Source<'_>, bounded: &[u8], body: usize) -> (Kind, usize, usize) { + let mut lexer = Lexer::at(bounded, body); + lexer.skip_whitespace(); + let dict_start = lexer.pos(); + if lexer.next() != Some(Token::DictOpen) { + return (Kind::Plain, dict_start, lexer.pos()); + } + let (span, closed) = skip_container(&mut lexer); + if !closed { + return (Kind::Plain, dict_start + 2, lexer.pos()); + } + let dict = Dict::new(span, None); + let kind = Kind::of(dict); + let after_dict = lexer.pos(); + let Some(after_keyword) = keyword_after(lexer, STREAM) else { + return (kind, after_dict, after_dict); + }; + let start = stream_data_start(source.data, after_keyword); + let length = dict.get(b"Length").and_then(|length| match length { + Object::Int(value) => Some(value), + _ => None, + }); + ( + kind, + source.stream_end(start, length).unwrap_or(start), + after_keyword, + ) +} + diff --git a/crates/text-extract/src/pdf/seek.rs b/crates/text-extract/src/pdf/seek.rs new file mode 100644 index 00000000..dca3cdea --- /dev/null +++ b/crates/text-extract/src/pdf/seek.rs @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[derive(Debug, Clone, Copy)] +struct Searched { + from: usize, + found: Option, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct Seek { + last: Option, +} + +impl Seek { + pub(crate) fn find( + &mut self, + pos: usize, + search: impl FnOnce(usize) -> Option, + ) -> Option { + if let Some(Searched { from, found }) = self.last + && from <= pos + && found.is_none_or(|found| found >= pos) + { + return found; + } + let found = search(pos); + self.last = Some(Searched { from: pos, found }); + found + } +} + diff --git a/crates/text-extract/src/pdf/source.rs b/crates/text-extract/src/pdf/source.rs new file mode 100644 index 00000000..642b8e23 --- /dev/null +++ b/crates/text-extract/src/pdf/source.rs @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + crypt::{Decryptor, Target}, + decode::{Chain, DecodeOutcome}, + filter::Decoder, + lexer::is_whitespace, + object::{ObjRef, keyword_at}, +}; +use crate::xml::stream::Budget; +use memchr::memmem; +use std::cell::{Cell, OnceCell, RefCell}; + +const ENDSTREAM: &[u8] = b"endstream"; +const ENDOBJ: &[u8] = b"endobj"; +const MAX_END_MARKERS: usize = 4 << 20; +const MAX_ENDSTREAM_GAP: usize = 256; +const SCAN_FACTOR: u64 = 16; +const SCAN_BASE: u64 = 64 << 20; + +#[derive(Default)] +pub(crate) struct Codec { + decoder: Option, + stages: [Vec; 2], + decrypted: Vec, +} + +#[derive(Default)] +struct Ends { + endstream: Vec, + endobj: Vec, +} + +pub(crate) struct Source<'a> { + pub(crate) data: &'a [u8], + pub(crate) header: usize, + ends: OnceCell, + budget: RefCell, + scan_left: Cell, + codec: RefCell<&'a mut Codec>, +} + +impl Codec { + pub(crate) fn shrink(&mut self, max: usize) { + for buffer in self + .stages + .iter_mut() + .chain(std::iter::once(&mut self.decrypted)) + { + buffer.clear(); + buffer.shrink_to(max); + } + } +} + +impl Ends { + fn build(data: &[u8]) -> Self { + let collect = |needle: &[u8]| -> Vec { + memmem::find_iter(data, needle) + .map_while(|pos| u32::try_from(pos).ok()) + .take(MAX_END_MARKERS) + .collect() + }; + Ends { + endstream: collect(ENDSTREAM), + endobj: collect(ENDOBJ), + } + } + + fn first_after(positions: &[u32], start: usize) -> Option { + let index = positions.partition_point(|&pos| (pos as usize) < start); + positions.get(index).map(|&pos| pos as usize) + } +} + +impl<'a> Source<'a> { + pub(crate) fn new(data: &'a [u8], codec: &'a mut Codec, budget: Budget) -> Self { + let header = memmem::find(data.get(..1024).unwrap_or(data), b"%PDF-").unwrap_or(0); + let scan = SCAN_FACTOR + .saturating_mul(data.len() as u64) + .saturating_add(SCAN_BASE); + Source { + data, + header, + ends: OnceCell::new(), + budget: RefCell::new(budget), + scan_left: Cell::new(scan), + codec: RefCell::new(codec), + } + } + + pub(crate) fn charge_scan(&self, bytes: usize) -> bool { + let left = self.scan_left.get(); + match left.checked_sub(bytes as u64) { + Some(rest) => { + self.scan_left.set(rest); + true + } + None => { + self.scan_left.set(0); + self.mark_truncated(); + false + } + } + } + + pub(crate) fn scan_exhausted(&self) -> bool { + self.scan_left.get() == 0 + } + + pub(crate) fn mark_truncated(&self) { + self.budget.borrow_mut().truncated = true; + } + + pub(crate) fn budget_exhausted(&self) -> bool { + self.budget.borrow().exhausted() + } + + pub(crate) fn used_bytes(&self) -> u64 { + self.budget.borrow().used_bytes + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn truncated(&self) -> bool { + self.budget.borrow().truncated + } + + pub(crate) fn into_budget(self) -> Budget { + self.budget.into_inner() + } + + pub(crate) fn stream_data(&self, start: usize, length: Option) -> &'a [u8] { + let end = self.stream_end(start, length).unwrap_or(self.data.len()); + self.data.get(start..end).unwrap_or_default() + } + + pub(crate) fn stream_end(&self, start: usize, length: Option) -> Option { + let data = self.data; + if let Some(end) = length + .and_then(|length| usize::try_from(length).ok()) + .and_then(|length| start.checked_add(length)) + .filter(|&end| end <= data.len()) + && self.endstream_follows(end) + { + return Some(end); + } + let end = self.find_end(start)?; + let body = data.get(start..end).unwrap_or_default(); + let trimmed = body + .strip_suffix(b"\r\n") + .or_else(|| body.strip_suffix(b"\n")) + .or_else(|| body.strip_suffix(b"\r")) + .unwrap_or(body); + Some(start + trimmed.len()) + } + + fn endstream_follows(&self, end: usize) -> bool { + let skip = self + .data + .get(end..) + .unwrap_or_default() + .iter() + .take(MAX_ENDSTREAM_GAP + 1) + .take_while(|&&byte| is_whitespace(byte)) + .count(); + if skip > MAX_ENDSTREAM_GAP { + return false; + } + let pos = end + skip; + self.data + .get(pos..) + .is_some_and(|rest| rest.starts_with(ENDSTREAM)) + || keyword_at(self.data, pos, ENDOBJ) + } + + fn find_end(&self, start: usize) -> Option { + let ends = self.ends.get_or_init(|| Ends::build(self.data)); + let endstream = Ends::first_after(&ends.endstream, start); + let endobj = Ends::first_after(&ends.endobj, start); + match (endstream, endobj) { + (Some(a), Some(b)) => Some(a.min(b)), + (found, None) | (None, found) => found, + } + } + + pub(crate) fn decode( + &self, + chain: &Chain, + raw: &[u8], + decrypt: Option<(&Decryptor, ObjRef)>, + out: &mut Vec, + ) -> DecodeOutcome { + if let Some(outcome) = chain.early_outcome() { + return outcome; + } + let Ok(mut codec) = self.codec.try_borrow_mut() else { + let mut fallback = Codec::default(); + return self.run(&mut fallback, chain, raw, decrypt, out); + }; + self.run(&mut codec, chain, raw, decrypt, out) + } + + pub(crate) fn borrow<'s>(&self, data: &'s [u8]) -> (&'s [u8], DecodeOutcome) { + let taken = data.len().min(self.part_cap()); + self.charge(taken); + if taken < data.len() { + self.mark_truncated(); + return ( + data.get(..taken).unwrap_or_default(), + DecodeOutcome::Truncated, + ); + } + (data, DecodeOutcome::Complete) + } + + fn run( + &self, + codec: &mut Codec, + chain: &Chain, + raw: &[u8], + decrypt: Option<(&Decryptor, ObjRef)>, + out: &mut Vec, + ) -> DecodeOutcome { + let Codec { + decoder, + stages, + decrypted, + } = codec; + let input: &[u8] = match decrypt { + Some((decryptor, id)) => { + decrypted.clear(); + decryptor.decrypt(Target::Stream, id.num, id.generation, raw, decrypted); + decrypted + } + None => raw, + }; + let count = chain.codecs().count(); + if count == 0 { + let (taken, outcome) = self.borrow(input); + out.extend_from_slice(taken); + return outcome; + } + let decoder = decoder.get_or_insert_with(Decoder::new); + let [first, second] = stages; + let mut current: Option = None; + let mut outcome = DecodeOutcome::Complete; + for (index, (filter, params)) in chain.codecs().enumerate() { + let cap = self.part_cap(); + if cap == 0 { + self.mark_truncated(); + return DecodeOutcome::Truncated; + } + let last = index + 1 == count; + let (source, target): (&[u8], &mut Vec) = match current { + None => (input, if last { &mut *out } else { &mut *first }), + Some(true) => (first, if last { &mut *out } else { &mut *second }), + Some(false) => (second, if last { &mut *out } else { &mut *first }), + }; + let before = if last { + target.len() + } else { + target.clear(); + 0 + }; + if source.is_empty() { + break; + } + let result = decoder.decode(filter, ¶ms, source, target, cap); + self.charge(target.len().saturating_sub(before)); + if result.truncated { + self.mark_truncated(); + outcome = DecodeOutcome::Truncated; + } else if result.corrupt && outcome == DecodeOutcome::Complete { + outcome = DecodeOutcome::Corrupt; + } + current = Some(!matches!(current, Some(true))); + } + outcome + } + + fn part_cap(&self) -> usize { + usize::try_from(self.budget.borrow().part_cap()).unwrap_or(usize::MAX) + } + + fn charge(&self, bytes: usize) { + self.budget.borrow_mut().charge(bytes); + } +} diff --git a/crates/text-extract/src/pdf/tables/cid.rs b/crates/text-extract/src/pdf/tables/cid.rs new file mode 100644 index 00000000..aae22b77 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/cid.rs @@ -0,0 +1,198 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::OnceLock; + +use flate2::{Decompress, FlushDecompress, Status}; + +use super::cid_data::{ + CNS1, CNS1_RAW_LEN, EXTRA_SENTINEL, GB1, GB1_RAW_LEN, JAPAN1, JAPAN1_RAW_LEN, KOREA1, + KOREA1_RAW_LEN, +}; + +const REGISTRY: &[u8] = b"Adobe"; +const COLLECTION_COUNT: usize = 4; + +static TABLES: [OnceLock>; COLLECTION_COUNT] = + [const { OnceLock::new() }; COLLECTION_COUNT]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CidCollection { + Japan1, + Gb1, + Cns1, + Korea1, +} + +impl CidCollection { + pub(crate) fn from_ordering(registry: &[u8], ordering: &[u8]) -> Option { + if registry.trim_ascii() != REGISTRY { + return None; + } + hashify::map!(ordering.trim_ascii(), CidCollection, + b"Japan1" => CidCollection::Japan1, + b"GB1" => CidCollection::Gb1, + b"CNS1" => CidCollection::Cns1, + b"Korea1" => CidCollection::Korea1, + ) + .copied() + } + + pub(crate) fn unicode(self, cid: u32, out: &mut String) -> bool { + self.table().is_some_and(|table| table.append(cid, out)) + } + + fn table(self) -> Option<&'static CidTable> { + let (packed, raw_len) = self.source(); + TABLES + .get(self as usize)? + .get_or_init(|| CidTable::inflate(packed, raw_len)) + .as_ref() + } + + fn source(self) -> (&'static [u8], usize) { + match self { + Self::Japan1 => (JAPAN1, JAPAN1_RAW_LEN), + Self::Gb1 => (GB1, GB1_RAW_LEN), + Self::Cns1 => (CNS1, CNS1_RAW_LEN), + Self::Korea1 => (KOREA1, KOREA1_RAW_LEN), + } + } +} + +struct CidTable { + values: Box<[u16]>, + extra_cids: Box<[u16]>, + extra_offsets: Box<[u16]>, + extra_text: Box, +} + +impl CidTable { + fn inflate(packed: &[u8], raw_len: usize) -> Option { + let mut raw = Vec::with_capacity(raw_len); + let status = Decompress::new(false) + .decompress_vec(packed, &mut raw, FlushDecompress::Finish) + .ok()?; + if status != Status::StreamEnd || raw.len() != raw_len { + return None; + } + Self::parse(&raw) + } + + fn parse(raw: &[u8]) -> Option { + let mut reader = Reader { rest: raw }; + let count = usize::from(reader.u16()?); + let extra_count = usize::from(reader.u16()?); + let unit_count = usize::from(reader.u16()?); + let low = reader.take(count)?; + let high = reader.take(count)?; + let cid_deltas = reader.take(extra_count * 2)?; + let lengths = reader.take(extra_count)?; + let units = reader.take(unit_count * 2)?; + if !reader.rest.is_empty() { + return None; + } + + let mut previous = 0u16; + let values = low + .iter() + .zip(high) + .map(|(&low, &high)| { + previous = previous.wrapping_add(u16::from_le_bytes([low, high])); + previous + }) + .collect(); + let mut previous = 0u16; + let extra_cids = cid_deltas + .as_chunks::<2>() + .0 + .iter() + .map(|delta| { + previous = previous.wrapping_add(u16::from_le_bytes(*delta)); + previous + }) + .collect(); + + let mut units = units + .as_chunks::<2>() + .0 + .iter() + .map(|unit| u16::from_le_bytes(*unit)); + let mut extra_text = String::with_capacity(unit_count * 3); + let mut extra_offsets = Vec::with_capacity(extra_count + 1); + extra_offsets.push(0); + for &length in lengths { + for ch in char::decode_utf16(units.by_ref().take(usize::from(length))) { + extra_text.push(ch.ok()?); + } + extra_offsets.push(u16::try_from(extra_text.len()).ok()?); + } + + Some(Self { + values, + extra_cids, + extra_offsets: extra_offsets.into_boxed_slice(), + extra_text: extra_text.into_boxed_str(), + }) + } + + fn append(&self, cid: u32, out: &mut String) -> bool { + let Some(&value) = usize::try_from(cid) + .ok() + .and_then(|cid| self.values.get(cid)) + else { + return false; + }; + match value { + 0 => false, + EXTRA_SENTINEL => self.append_extra(cid, out).is_some(), + value => char::from_u32(u32::from(value)) + .map(|ch| out.push(ch)) + .is_some(), + } + } + + fn append_extra(&self, cid: u32, out: &mut String) -> Option<()> { + let index = self + .extra_cids + .binary_search(&u16::try_from(cid).ok()?) + .ok()?; + let &[start, end] = self.extra_offsets.get(index..index + 2)? else { + return None; + }; + out.push_str(self.extra_text.get(usize::from(start)..usize::from(end))?); + Some(()) + } +} + +struct Reader<'a> { + rest: &'a [u8], +} + +impl<'a> Reader<'a> { + fn take(&mut self, len: usize) -> Option<&'a [u8]> { + let (head, rest) = self.rest.split_at_checked(len)?; + self.rest = rest; + Some(head) + } + + fn u16(&mut self) -> Option { + let (head, rest) = self.rest.split_first_chunk::<2>()?; + self.rest = rest; + Some(u16::from_le_bytes(*head)) + } +} + +#[cfg(all(test, stalwart_tests))] +pub(super) fn table_stats(collection: CidCollection) -> Option<(usize, usize, usize)> { + collection.table().map(|table| { + ( + table.values.len(), + table.extra_cids.len(), + table.extra_text.len(), + ) + }) +} diff --git a/crates/text-extract/src/pdf/tables/cid_cns1.bin b/crates/text-extract/src/pdf/tables/cid_cns1.bin new file mode 100644 index 0000000000000000000000000000000000000000..520c27953848ebc83183f71cafbedde435343656 GIT binary patch literal 31281 zcmV(&K;ggb{B@WVN4p0MSGA4LtSh_A?y|7BySux)y99!}6C6T!F7NY z*4@Wvy6e4Xc66WTIp@6J^_@T7>pBCq-Cf;Xb=NJwTe@d*y4>a#DNuR*zxt`Vkoh9l zVtKHJu*o%bq$CU2oXdC2#?ceK9^wJu81>GDVTj*m$R&%67Nm zAG~-mIpRx(&>|wI`Z5NJ2pp!!79qF#yXsGps=ve)LQ}X=7*BbOajkcGxPI<$6ipav z4V8A$z7i3!eJ&c{vtSza(|m?u`okbeO1XZebuf$%I+xwixp&R6qmt_~{v(@2|M5)2 zvK>_EM*JW8oBdCu@@Emx>~tQnCy>m)QhtOm{{~j%CckefE=wWv5Fzsjs8%7Bxf*LLa&yb$eT%AvPHRo^RHWj!adaT5<$L&{zVo_}X3wkmbc@H- z$w1$L#?!=dpkvNvg}BLIgfCA^eDZrr7@kr_5SqpukB!TPiX0ZIj`RX5X&e}>B0_M* zj)abAP7%7&ZQp?UYkyedkN;?2!=FwXpa~%*4;a>&%Rp5Up7GjWi3Q z+fs=&uAd>K9G)pz8%X|W-x&*}U+YeSfT;#k+L2fZ5oUd0`%3GGBsQ@QDz5EaGf(8* zam|&1eQ{kJP)kOUAyTg*nSsVa3d3q?DI;wm5URYST=YZ}wurZk4<^0v9*!5dBB@bf zF`H!u`AdNe;0-LHO@%lNchksB776)?Yhs?6Rc`F~*qOq9`5nHIfa$&lKH(x^FpC4~ z>3Jr%ZQ%ZnCW^y_tlU(LqY8w`#f#Rq=vTKLCE+5wNhv%~B$0Tjn4XM0?}I$B+%fNY zCnyQ)W{{eC3ao1VMX|;;++F3#`g( z66f$CC5&>)>~c*rMc5MX$}8YoxwsmZ9IimF^C=7zG1(dpo3jZSC%oT9TlT9_J$1%5OgW{`TZpYLNlpCsFq!5gPGO`OJ2uf zJ^#(0_(}4)%$QG=1YD3FHFL6mix6ToS>(kkPRnz`m~d%a)lZC9fJu#2Qms!~j4>-; ze7-FX`dR#hmE8#qeksL5#ivz-G`UusS2@4hOG`ylL2}zXlk~EYF{pmRGOq|~pi(9n zKNG#&v}cqBXSb+D6$Q1X@OpBAtYii>B8x8P@Tx+oAk(o8f>pm{Tl~C|Lkwo4RFb1m z8h1yZ={lC?_*B}+dP{$i9H>An5@CMLDe~4zT=%`nE3!yXrk6yCaulDUG(wQZ>+_MO zMD>SOL=$3JL79YZT`EMhe&d%}(Oi(P@H3n90%<10!b$UmpjN9T{xqrfE$x8?>}<28 zQYsso%ofT+O*5L%fS>qB5|QaQu0pxK0L)z?wPTJhWnsMsVI9FsPeyE zSz^}i38rg?v7Ju0sbmOD&4|e}R-mvU?Mu6S?;=+8Ic$Z>XrzHPy3K8PcCH?O^exhd zmXu8Njp8YOlo6cU2PMX>1fFVUAl_7oAp20!$ABBd;=jj%*m#LrCahRXnZ?<|F%CgcO7iA}RuT7T2|Xz(2$ zy`clw2~$@^3~e!du19m041P@BAet6TB8GJ9&X^B+2?c_fla-iS3LxW+qX~H`rE=IH z3+EqdE)f}w-sjKw>w6X!Rm@`~rlNqnJj}-s3+@%mEERDYmC0ziS`JgvKrca&F7f%W zaa?^i_v)}hizp-XC0n^OsWFWK**{-YwS*-4kmM$x!_sm?w9!I={9DZiO1ZW`G^N#Q zo-vbc6U(I)OEvQV-^q9pm4l^(m+0QbgwS}X4vX)1)O)7|()r>N;u~%DB6=|~(HjY4 z5YYu^nBLW83-;(VB!DWmuq;(1ry^qN*S545{CGyFNL}#G_sx9xpnR4$n-XYzFOp)j<*-2zBwB?Ppc>X8vG_y%ij+CI z7h5DTnN%im$-!B5V1=1c>WC zpenU_FB69`A1xIPxlF}@0ugvoTS5j~@M$7enB@gQIjoE>{AYl`Gb;eosJ4 zC5Y)b%aJ6Rz#13|Yx`9pdJ>HXZrU!b7?6=;@G}$yP}GE?l@!*@CBpKqAK6MDl)mQd z5M_1=W2j{=Iy@&cvWR^li^`u9WtJAoaiN9&j3)#}r};|c&xO$@5#Iz?3}sS%uXHld zbklBnOf9J03=5UL-Y`{~i9UsXl^N%32+Q=<`zWdm$!c3v`5P}{z4N@afq4BgUD5pY z5#z-cT!Td1r5MT>mBko;z~zPbuQCIv@g=}$^!Z%AD3fZ%T5fv(yrr#$9x7C<+YMTpk_U0R1LH7I)Q#q@IbYHQUqKI+s}U&xlM zG5KYPB)Q6kQ*gms-;*&~NdoID((3X#OLB*wR9OLtYiN^e%>)aRKa zJ460(0ks=}*WeDz+WOUJ&{G;QNg?_(3hq7H&VQlu+MH`dO^MODAaX|IA5g~3?)8!nho z*E^$Fz{-3-?Ytj9=Y?fyB5(G25>A(BD=I#5hTN(>h+mbHiV7+6mx?OFw~`7%QN_Qx zSg((~VxP>*oEEP2GNlI_Go$iTgVK~HQai^iB`HD(g+r+Ih2BL(!fHtlUgBjoumMF> zp@cSVJ`$CIHGGLbm1}0H-+vq_a2gp|i9dBdqBM4)sD$*OLgjN`GZr@_mJ3u3A&-dj zB&!-cj$7TgFVQk@a_QQ_1+OzFj@LW2Zf7U=5Yy_;FZW0Afn zFwbnF%EyN;{Bb)ux1=`}YDo_)rWu?SA;}V_M+qY(te3t(Zdeh^Qys_xmd+=XY{z7Y ziNTbKGS}!IMZXspQ~YweUb8Blw;JTtaVig>9mVO*m0$7o8O&>Q zBXM$-YykYsFdeOp(Wd=16AQ~N)t}aa+)7K2_f;sQ)tBi+!UxtbyJnu)_jWaZb+-ug|r}v{Jp=I2%9LPrAckB#uCY_NW7csLy7on`MAMQe1YPhE?3>Km`#j? z!e#ViSXV8p#5h<}!knSXqs*eg5Gk`D3JLflCIlBZTe_|EZ@ zDt8d>7l{fC)lC|+Z{@7el<<6MVaST6oHWMoqqw{_pYqEpM}z{OJc;)M3OC>-{P&ip z!C~gq3dQ;?6`fh3r7$28FKjM;;h$e8BR@=6K3Auydw=P^BGO91>i=keQQ{M2FjM5H zM8{HQyr6uQ(?*sxwK8eFG=Uj@-|-A@V&;G*`WE_NQ(`qCz|Sm?_bZZ_^^ELt(_qmO zs!@~Xb1|k(!t#po?_1z>!AB@jH~u18QcPuYK7~bsRwSk_>SdVHUI@_gZyHp;5oqL# zD(bW7s999_909y7)?%Ylzqt;xK25>qh3F{$z*vEi@=P%$Y6lU9^g3^n4xlJQ8=a-D zDZW?|psoDC%=D!oRjDp8JETRs2TBO{cZtfPV|Y7wrBkH(%jgVzqMk@A zBJHZ+N(^;YxG7++ZLyE}?yuOuk?$fJT?^H7V!i;^6X|`$W|AwK+_(kin{3jNzKUMB zNL5fCf8i?dnwWBg$$9@&T*B!#SX%iNajH0($cx5W9Y=HDQ58q`%A=Kf3#XBo=L)X~ zW0DFtMr;ju(9}}$rRL`VU4U5=oO3J}Q&k(&te&%U!_qo?#I!{x&&dta@tpCUR`r-^ zS=@B;8YN9-OKw}PtF~$>?}vMfy|k3LgCOu@Or`<1Yxfgq^TniDaw8j&Hj&zAQHs5E z99tyYi3jkf{R+cPEuVZ1HhSIaQ?179;CyTbK>Kl;R-7A5=?y9a5fgE4fA1<(Ly?X; zWRV_v3bdHHHezs+KEKdsCw1bAjQ;PDg|g~eZ!d9?**num<7X*hRe91v|8tVL5ZEz& zdA%6IKe@I_iYf@;Gvu{}dOy&_ zLZ3ZK@s!EL(56;`h{wY9DQ4k({a?C&2u*)6A-}*s+cXbji;yDODC6mn*}D5;+s<@R$<~Py2El>)U6*Zhn=_v3q|uMUu;8n zCS)n8_mAO6IlZEOpy+$L0j7QT593%uWU4p3*uWb*F_L21Kqv4a^H#^OY2^f2h3;6om4T?d}pm96O9| zvhZC^D%`@pX#IDeVk%*3zn|#)3`Gvq9H5sT!*^a0?%Aa;n|BI9S#s7wOL5NuEf!cz zl0x|jVTpm^>jZYsF{SqWc`F&XniUBNrfJRUcC5t+AMpl4^%*p}CKW*z_VbbGuI+oZ zxu&pl%F>p6t)T&V5<-NoRPU=aw`)EvyzeEO!&0nZGF}~kjV~q%#rXm+{J55nGi+p) zy0ku3+Dh>UvV|g7`U^|W$D|J|6vdAFB~|EnmPC*-Kk-}E?81wvml+lX&cSPi)7i+d zwo}ba1;2+G-bkp@NLnRYOdeEtR=AH)HE5y708#N^j=bj?MwBOD%M#45rpdM&{o{k$ z*73#T7dHBXu3VciNheO8R&$nBrwbl75beXO@Wz?umC@ZL)Qh!A<&)E53jmk+2(OJ-S2hC-k3>i*`^)3>}C%v_@Vsxnj-Rg+|2G1Q!1wx zB)<*rRJB-kMx!uT_&U6At&dm})Q8?$xiR^13DvPx$G;;L7RzkvOO3Zb7fSt}M#QW1 z-u`KHSW!`7_~9}9t|fep=1ac_JuOmkJBvg|8^xdQdJPmXJ$jdFSUn=J) zk^?_*@~9eB#!~cVJamYv96l;4sZ#WwmXyT!6lYkXe>Doj7!gm2E;X4@EY|D&@=1^> zkJ&|yrHr&7F<2B-*DB0X0phL&K|_UMgoK$nHlHSeS~0_FI&_v$m@Ubc5}miGfYVY- z7((b?l#fMk@h?;qt?i6Q6fnUeCRs4meixz?JyR41_@;AN!vu~t8)dJ-q=i#tD9*j5 zAI<0~sfOqzG^ICUO>f;diZM!z+YKUt8=opM(4qpt-(2#c>$_TsVttU@Ak`H5exz%K zQ}cQ3o3p70u04P%F;HGbT#Aax5Q&Q`B^Q%M|1kVd)I8cOaDpwK#TijdDjSwB z6b6(0m|}6ZiP7WGABPH~TFx6x=$!cQ&rMzj7aJsWCH9-_(?{Qw5y>+TnzA~749R@R zC+)zc+<+MKbn+kt<-7US`rD<>3EX-!gIKjy1rE;x# zWN?SAyq8Z=ZWv8W`ARlbDW!74b;g?~)`&GSjTdbJQsg`~8wIgMTJ~?ERbyfbg2(Pr^ATMf=#xf}tHMzk>7Mxi_E7sv> z6OQwlay_vK5-}t*-J8AQ1965)T;k{7xG6^l_7(Q{7SNo;Sc|OSEYuHtLmjX9GCJMh zQ&2A5hqDy=7jLS5CcMMPAY7z4+TVv1=LMBiu(0n#ax3XGBC^DkWd+0v>-{sO_p#hu zGYgBon8%?x!8(eH&=qEBDIbc9$jdRyX@Q{|`*R}%#!T22{j;#o?tZl&No7=&Wz2Sw zT4HSYS)LLdu)A?|3B~M~ZlDaI-SmPyf5vZGvz*cx4H)}sd`CARWTVL>HSl8YEhjHZ zuWI2+bP}V4=dqU#`{Qa&nBLwdiG=|3TJ{ql@9@rN7%XN1kbO-?4nRPv7e$LS3OynF zutrJvVv77mTR1H6y&Gv%#Zn{1m?hsUjFg2n>?Imr3Q=$g&-Y?^iWiz%Y7=2Vtogn{ zWRaU#TmyW4;D<)R2Gm0#)0#=n% zUL)r6X`QNsIfpBYu|-*=Ec7$$#JlQyRj5qfQOzk5N>$Y>c#Fhf`X)6(NW_%@T#Kg( zc%8zqiXL9c(yLdINzVf~`uKljH6aB-7i)z`DFaS6)nHRi%lA`+n#2$J?lXSmO{eNA zdA}#-sFKGO>t^Ms(v+<1F0ToG>6acP&-0QM`ew#8%FwDxMB3{4exE{QX$Dc4eh_2e zee_@WMky*zl;K^H8=@@^% z(yS4$rpB5jIh?Lhu{0Q){k2R^R+TtkEh8=YzHW@c+$u>0sX$6nkR+k?qe^dDzl_~I z?z6(;Wl2d{w8vr+Xzt2rr9}P)x8i>ft|#H2lNRxrq1gqZMjwk5VP>s=R_J>=z$}zp z=NUX0(tF|1t{e%u->A9qL(7C}PlAX(vyvVQ!5>R%(a%Ej6edwrQk5^d0Ch{>?DFwJ zseT1vxwJ@;XAu1L@|irQ519(3qGCbsrJ6E>;#IswYBW)_f@Jv(BU2gPf?0Cc)MbKB zfQ+9=GXqS%Qazyamk603O<|*0H>4$yv4^#sjtUeh`vEY!pljdfzv0K!+Zu&)(>lf{Z+TH~W*3o4mKCZSC^Up6hzSo?Ko zu^$r`;s2hKcnhkel1kFgs^C|Zk}5q(;U^npgwg7PicG9urU;8n_~(XlUz4~{?EL({ofH2cRawZ8JDoq5n%W#|Stn5cAV1%1G@N zL-3sIUxejF+-5MpB7qZn{k|{h0%Ic5yr6FqO7%}YT_LNU+k*T=X9x1)XD^2rX+;&E zM-}*dR)u^HkmoH6Qzo+#h-3T4>q(GY52R{|s$!2c+1FUhml_`{i?V_@bmtVg;_$bW z$_spS_Y^_G8<`drMDo~=QjM-0GX2a2EZiBJK@F}?n+$Tkk~j8> zqD~j8*dlR}c$@5TZFP?af++lPq{iqI#;4?0{JYttcEb}s^+;7$C%Mr?an1CEO z+UG&>+Fk%0gB_?<}18+2ZIXT%bWWsNY^ z;L*=jsvL~1@_`RBgUQwh8 zQ!KH(vPgP@M^teo98;jmuOg;No>=cyjFno>EY`LYnLkxjn|XN=#SxJpjG)c5a8weS z5Gc>$H9mQQM2P+xBGwA|-s61;qt{EZiEqQ1Ymg<3u=iLSv+qV+M?{G-1GsEIQ@*!R zA0p&Y5yu4@cMK{5XXKmWR~Ylr_huVTp((ENRi4$SrN1RduvAPpsw{F{az*8)e|QL0 zjf`0(DUlUPq}fzp?xzkE?%>y=I3rYHDwY!S){&4x3yVZ1*0S(Bp-Uf}6jEfd1Z9=~ z68NptO^O_?W{RQofsD#4PDe)yIXEjsY6#YtzIs(mqf+DQ`Ih93j&6OClbFp!WllZS z!T34APxX?Ec_s0uB46U%hL5GOo50V1s`;T#a#me+fmd!!36)enzT15%5eR1-G79qo zKHjwWaqUD6mV1j8@*F0ANcmxcd|G*pMkMegD95Z9iw~5G zT_DB945JeASijl7`kKsI??N1!tSg@pkcK(4WseB5XF*XU*8W5+-4_cIkycWo6i88^ zkk^wke-=!NVovLqCpLPC3yuC?1q)`H5UL(--_{bCrYIJ9oogOcB>oU~tD4zD(Pw%D zmZt**b#6h2q^DuE;z=mUK0WX=ZR+r}{te&MKfh1)&8g%4)L-Dj20T40FW~D3Jj$rA z2}!gB3;5pT34wYram~(zj4jL zlj;Da;3qe(Qt(2kpC+5)m1c!JzA!N2h14&jxGwuj$r)=r2oHYKkQXE$l3u?pWZQno zFQg4rY?ePS{^DbxRUcy^`b)*$xw?R)^g<~XGlfbMtF@2|j6k|l{KNcW5ik0+DxPK% z-<@sbm4yl;RYVEV!!ruq{l z2FXHrzY>oZ2AiZ=atYymsPqzPfsd)!>)VgTeg%-^87(7_R$?LwtyC}O1nHnUg1E%1 zh>}@YBlC}jw(QQN=}D768+46GafXkfoKRd6q|g1_eNk(wqBPBiEwKIeB(`wA3fN^20_60Rb=lO1vBLx&DPFl{`}n zA-|+pCQqa8nB6Ilb;dC)&BJK12c$VihX3V}3 z5esMh`fwpd7{8MIs3fU(ju*|4YN;x|P#UGr^)eHoXm47-yA(%ZoI0^0Pv{AwDh`#HLax~X3gNtW{`58z?mgx+Q*=wIJyK6S>)tfgB?HkDkh-O>CfwFpgVgxYmz)1X?6 zsT1i*Bs4LY<(mc+2SPY3JT9UD*31g3M6S(fOvHrr=PM|9trsyHEaL3~O&Pq5MoPhx z<-M9=xt^i*F9w|5R;Xl4ct|aRPm;4*xy~#$NqbWiS3-EsrHB;$yy%;c zvDz2C-dm)nNGrWYY#}0nR8)-r>q#9D2WhOCO#9;`}mMt^@QVDz;nv%_FOyS`?c*Q;X?7$4k)H_U~ zxS*hYMiRNxCq6h`X#O(lYZWhxHrqf-WO{SxFWyCF3~9JHw`xO?utDI5iJP0~VzZc< z+eP?kuIHcv!fQ9hxaSSS%5A{u;A1U&2ima3vCpD+E=|B0Oc0 z{BzidjRG-4KGt6_(?bo4B8izuNcVa8@5@3zwpdiv$Pz~9Z8C|C%wE!J2OB`?2{CP&0Q?KA9QeAjO(qqm6|{Bl^%=%tqy%ZqG{;@?|+e>dHFiGph*tEF?A* z7eYRe8u`sytco(Io({;RgK{hii>Nph60#hFKbK8OjBMol)q;egz0+hRDxBQ^>gWub9$wadjt_2Zb3cOtA(MBNTo zB?vqp#WCqT%VfxA7$qjESWj8z5F=|@DKb-}5ee%>bHpMfb6}2y7Z@^0CX6!Z~1GCCL}*j)9a z*;g-^q^5;!#9BFTU`2gMXR%gKGjzAWnjZ~)L|7}L$N_TkwxUo(l!X$?M9ZpaIPhwG zu}W_lBecUyhhbqG8Q^20Nf8BBVwTe;K9?=m=Vv1NeHpPB0mn>!MTKI0NRCWH#g=Ha z^9&4cl43&=Z}H+;pG<&0jZcA>K^I8z;<^VWgXt+e zB{s`GaWSb>Zg@sHjkuu1n@dtu_!e2r$O~#>A-Mvb8D?ZV5c&ro*2o#?IuuVu4ZwK9 z2D9+NA~1S|5rlE0Tr~@>Aor)p62e<34IP@I)G~+l7PEJulGn-1l*~xg6Z%Gx02M?u zq=F<7j~;znPl4`QpTIP%it(DWJ)6lTyoILb4Ix-!%w@&Je!M0$Rq$TbYnVc5)cNZ$ z=JFMWLV-8aN3cMIDyCw2-!7A}sd_bVE@sh=tN0%YU!poMY$mipWRMvsO$upD*Y1E9 zjT9OT=Bf3ZC08se=KVSDn}{)LBXp7iCfp<`!9FxG>*RWwVUz(|q-G0isacb>8e*Vy zzOO4X321G(e6c7`%88Sz@im*(_Qi6@of8++Lai|?aV7l5`c$0= z&n0H8;!T3LfydpACNa-@>qKT!1nGzJE&llfX;DP{Y9@k}#MBiSY+*Ot67a@8;L?QO zw$L_4Pw`=CIwfln6_wLt&O&#m`?i=YEdha2Tx&ID`M_gvsdcXi^8_9b z5@*H~u-Sp;pdwACOz$NUTQXGz?Otkmuk28*nj$^%Myd@S1MC=sw+gZnB2J`uvmh^F zrP_V8r3*z&e1TGt-s!Q(;LU4ynPf&SXOOWPu`*9;6mvSZTy~xpE~ZFVLX$2+bygsj zM!%7fq?!qDBx{%`d3c6Nq!qWtoPaI(6HmtP?K@#OBgBxydw5KnMfYaRJ=@gB0_g{@ z6qOK(R>WuomNBRzqAe`HZ8>>SAMa%LCkk(QDoem~)>+IYekGD3h1Rd=5+QxqwNb0b z^QJvQ3L_B&K%=c7U7zML3qD*#ePZFa_t};&NG*h>Vc5Cs93VQiG=g z!vaZ-N(#dL@&#|OMAjsnC5y*QCydR{@y3sXc}CdPYW?zCI2}T0vX4JpHgWoW5SF9!Z6-=;s-KjgQ$eX1{zXCiLj!xk5?#Q ziJsM(lx8z8$c!iTa!U~_*D9rjM2Pq%{@kR!K)fnfQS2YqlWh9BX<+#a6ppdX>R%D_ zH-98gMiDA++)t$Dr{>5Mb2jJS6X$8-Z9@bH?naO`q& zsd{nhM=cQoI%CjmK9aD%@p#E-o>>vTaQ+%8Lo62Rm_=d%YsF%nAUd!twg$!{hHM-K zn%L>DH8t?4TGXZ!V4@xIc#Lafimcg-<%Il0D2n=6IZwioLli_8WP3lq7PVBtM?d_T~RoU%jRY187tu@Q4VTt(x9R^Qb6U}eo`UbKRJ*hALk{U>zn zMfGv$b?;d-5ZW<`tN%~CmP>yb@>GZjDWSf5ge4+_bgg;?h`VK zl2;tE(sO#&p4gddyQHK0Ez#nM4Kl5LcSpS1yH0zi-@EDq>1I;~otibq`{8lrDL9i5 z^`ok!cez&g+7{iZ&2&5d+rVtNkrEJ^a;N+?y=2)cl<#xCa{WHC?CTBi4TMItxH}ko zHT_{BLcjXA%LqmX^iy}t%nJ0)Te96Z3AMc6eb~V5GQZybO}dRozc2dwu?q4j;=iuE ze7Dwb0})!X24y^wD7@SIy+bEQ@aST9Ai^LviqMJ`4WFXXhta7)e>MAKLF+c5nf;LR z$77MUMWJdfccO>Q(fGt$2+htM{WNmLq5%&(B9z#CWa!6+!@s=iB)hHpFw!oQ+TNPa zH99Sx`}|898d!2=FqjsBldP7yPas+Sa^R;l1`Zx6a`en)lUglPqJGDz7wyoyL#R zc2>gdn8)_Me|T+Zoy~11ZtK>#z^uj-a&FG*7f;deS-t&7=Om+a}Y<5i&MOe!KQ(ua3W2 zJN|h$X2%Tc&VgZd^D}O>LtR_Wo3sb*orvmdJ|Xm_;-31gc8=yzo7@)uU4v(zShNu# zSlX%3=JC-8En0^LcO8}#vB+l*4DqHWO1QbWO&i_!==G&dMVGq0s2kR5ch;Z9celKp zXl4H9N4?wT!;aRgMMC1r<>=d)IqdXZhTAF2R@}?JhR};u=Mg6H(eDO+gsbFKF`Bxw z-K|mKtI*f@PSA6&6+(MS{3h6)wGCAU`cT8=i;vVqCx$hxC0Ye*Ly8iFUe-b1%G;Kk z?}N~|0|;&XeroK&*J$iP6rK!J!s=c>nsTMfU#2#wZp8d%X73*^Z8{$R8#>ZQiNG=@ zwwRzr+vZxDzi1vGZu}b+jcllxh8F%{T`zpN-vJkdkcL<`m{@wByQ#IZ(WE3`&W8X=J}{5x|OVZ*OTS&UAuf7j&3q8)YU zTPo|QyWg`-2B75mzgibj*yN*N9_MwU0GLu2ZOyO4RH^!W@ddbQ@|0Z&i~+PNzRPD| zk|up*`1MqTB9eAaMcLi|`KxW?I}e+Wq4)Pa>s7eUxU%Yp)N)7GtE<1GrhRJDsM7Mx zi~Y{ncym_Q=Sy_g#MJHyAKv%h-)wrFE`j?hS5F7;32L2PjqP!)4QC7+8t{aw=wC(yPX@wMMEUr=I7g0>$&zO-^r@G@cDTf;DdE1jTimX z4QxdnD*jrf)aW~1J2`aCn1x4cze4L4q5A(!-3jW25cQy$Kp|A)okWNJ?1j+L*5QAi z3;6-f8C-XlX3E@@OLNz;s-m1}zfJ5^sS2W`V}I;DReo*A?!KpA(|3{26=fCc?Fe1} zW3L@uLef!-qXl(r`^_Kb`69HzxNNl7HdOtA_v6Qubi$0=AF6(UlVK?^G`XxTl?(zxysU? zzJ8gvILCL}dwtg(zw@WeLEGkOcF%8Zn^q~c;`-}Mt8u-pad`BdHv7k;F^5JRLlQmu zayfY$y47y>mY~cDJ6S8_A~fLi7$ZPRb^zGo`29O~L|sKwb`5Ae3m~b@9faB^H9m?~ z|Mnieo?4~%gw%f3m)CmT_;IC|GK8*mva+$|{4S{NfykqPuxGFDLsvT^)V68Et~7sa z#NGLoj>8OV7yB#YfA4=MkgtKfW1c;$#fiHg%nKH8n(K3})4WeZ>y~Ts2q6&c-JT<7 zMb+Z-(wCmJRIRpY=C2)ZL`x-8H|GVMtD#6gS$#p$yX=<@bCIgL=+?9NuUpaUS_u7q z?iY0Mb*09YSFF3;e(LV;x9)2n`Rgxlw0$4HJJ0s8JT92pg^MYyYgZB!c zsOpE~IS(o|y*#%<^>BkKYO^HOcj(DsCu-=~guhO4-vze%(`(VXUxRL|vc8>JJ1oK+ zo-sT7c)f*vmwZV%Id?^Mbha!JNtwQ094L!qxxEx1>c z{X=HV+Ep-m=#`Z7imsa%ea~3_sAXu(wXAsL#+SVVw#R-w(fi!;t{s==%AQ4Tk25{q z^6}1^V-x1JLJ=Q7kA2m6?6`fIOQ;fhSe44fyVr5~aoh6^;iI;Oe48J4?djqdX)Bk- zb{{cypU=`G5vRm?&HG;exb4=50+WBuP3eIP-hk7n|76O{=~W^p$dJCic3Z``{#}f! z-*;rZ_VH@?)8k?5p4>R}dhP0pBNkV$#yuI2_e>em;!EGgeD<}xW9y0s92&aQH*);^ zz5`-Rf88FpEAvINT9Y0&@VdBVhGu^aL;afbqd#rS+PUg$4C*$y$)lgIf~n{GG!yTS zv?}25{l$h-YXIf=8j}#c@aGYq-esdvM*!w*pH=47=mNK7Cr8Qj)xpi@eVg@Q@%GSw zg~yu5$a_^kR2=}n^Y%(s!$AMk54|yWR8+nAzyA?!hL8iLpPhb0MjHnYZX3g_L6^V2 z?GKJYC-t3hp&H}!6B=H}pBjQLbwjA*FKsfvi-y3W&mR}NH{4bC#IWwvF|=zwRUr6A z$9K3fBkbyTJ<~(LkyiW?vFYk^O9eC2?lyY&a@h}IyMC<@huFUw0HI6$l7kPD`(4ph#r6dAzqJpiJ}`nm_GOKrXQ+K`#CW3o)~SMhfor; zE9ZTf?rAn!n^}DHa@U6YM6tg&vP1@DT7kIh<%xS|AGSK`znh*IAwb}WCYleSPd<%n z58im6deO9^x^q@<3p7}&pIC5(e|QvY1093 z&cf&&Zh)IsdL^SzmDfJu!$dn6%hp$|1A0`65z3FNzI~)bjE*us{`JN{I2kSu`L6e% zQ|EeH!>VU#LGrg7-hLXswCAB zRQGgO*}}Edy`-;?Z`+62F~_uEy>0K-%7^YLMvyItKRp1ox;Nm|#sxo>+dI`8Wu7&nZlT&uvjFaXo?bZl$^qp2#cbS zwC<5vKSqWqS3HmHbX)Y3?`n}$=|@+k>NLXU+{-=Ya-*|mEc{NaREWJQ)o$DW$UsAd zHCS{!=gq>^1x(Kz>ET$;(zeb6B-P#jeB!HNXtXpRvMf|#n5a>;ZvL}2-CPtq>SN>< z(e5M3r$)~1!m50{yxG8r$})DQA|xokY2ULBg}K?j&mY#UO$8x&wSSDFIxp?J&kfp+ z6}NZEt(97D?Bpw5tX&el)gl}pn@$}VhwN!c>1Xe&3+WfG?5SqH*J#FUWY+J#EWPBr z(|;m6Vr@g^#tElR)CkI&bKUoPpTuB!ySa&mf`)DS9X_&MQ*F$p;wJe-y0uBLxM_NC z(Rvl{J>~4+>(Yp*O|2?#svp^qDL-_^WyLt!d-$T8l?*2*pG9&iZA@_-B79(1}$e7h7+HM4RSkM$AG_A}Ys9CUuh~Wr;em_@R>FJ~wS2 z!iE-fTU%+;A?5zz2P^92yKcW_dn&p!Ri*yP*Xoi_>*tQ{D!tuRCfEEVRqgH(f+Gv7 z#gXc*Ua^+yS91&-SP30HDmSnW6G1UlUZXG?U!VM&ctMx9(KDN7^i!&T^*K=ZU4)S74>11y6=bZimbA~`tZ9UbNuLq z7kmEL`EsA>Elu~#a!CNE5_N+F`eAd3@DUnV4iJemK}P!TKOL`^CSj!-;Cz5ux%5X##v#0(WoQq5X@rz)c>cv!ISN$O#uRT<4FU~5=g?Lo- za%JeaTCc5F+?SX1x!U`WH9rWr@7~FKr47I3`OJFovCfOO4^qmF{w1;e!fjt-;wg$M z-)Zp=i!0BMdLc7LCY8CMWSJG#|gcf}Uq~yL&eKbkn~fPrKe;bf&u@AyNVf%k+uWM&FoG zY@UE*RCHJ;8nxA6AqX#J6dW{+_0lCZQ_ zK|{7~i|Dg*`5%2CZ%0p?S}p|yM%+I;d|m!j%cAD|#95QR?8~p;ZGPnL?L!9sy%YU! zf65cpt6Jz)WCB5iM3MwpsTEFf`Gm+s*g`MDNIFjMTe@rAes-=p%DJa>?nTGF{QrS}%>T>ltN%|fdP9hf zr)57D=bo*`M_IXv5KGxA0Ek5@6fD~E-wj^b!5YfdXR`It78|ch`)Q0yhevK(=+3dp z1}Tc|d_|>_qKHexPR2tAYyCsH#-N-N!!hk(qGS9ogg6E)9SiPt?39jfW(cy&*LIOj z+{pHaB93`5aY)}f7wBJU4^uh};oKL5+?M#xLAzb^o!SFqJ4YnQxol#mg4pOJ_73b^ zfo+%AN4E`H$|fL`l+Fkj(XJV-hYThR@PjHt8gdE+%da{3;as|dAJz?aV;Dg$s_pD@ z-dB1nNGa2ZOPJUpVYgc-`#6sAOYf#^u;CIVDYX*^P3{+4yJ_p-whm03 z0)!}**zt!T&O=?TQ@*TYC(EV~SKal_sTHOFF>x1|i_^$WeM>p%6oZv5rK`eDirqPg zUF>!4U_?k6uDfi)p*EPfsSa}T&cRJY?8f3mF$eX;HC`#XG;%3xS(Xx~29+LOebS(BKoHRXJ0n9O>SqGmG#)cJ`X~$+Y%l8c4M|P@HWoX zl2V;;(1?*sz;0G$U#1gmTr=AM63$@qi3G+a+O9;D`Q0fdlY}!eMOEWQY zlbqA`O9{d*=UVy~yThVOj?VwCBjg;2IMm$(Aziv)hezT9T?gZ7cRsnwd2c#W}S;{`_L`FN#VkdUG^v_NQ z?PRN6WiMjnj$&@haUqp&zPsd{qhc3axXj-n4&nxOF2CUtUKwC?Lwk4AcALXBuOkSv z0Rkyhxn{8spW}y}s_B%z!%8tio?z(ashsg-snD@gCEeiG-80)s!)_W}+TpQYhbKnv z>gO<9;&N=9mWo4Nu0JyxkUpQVUM+WU<2*$Hd06!cyud)=N*s}knKh)a-oTf z7#mz*;__iGKkINq9;`r4yeDq|oJ4vUhTUlsXZKxh2JJkF%J^VM_(sa8hLbI{ z2h#sb>}&l8k?%Rm#RKeE1yvSR9I|kz6?XVD%E=JyT$dZFV>gs_LXO>^yJ0ACB2p=~ z5NFuo9FcNomxFznIF#7McY@p|aYC7WSE}_`nIpDv#JtEYMzxVb+~Q}q*qFGfwHxCJ z@>DFR89GB=LOig(%yN)>m}F}-E`xb)*nlaUSvnNRX-|}Am5MVcEP%W2ZlJtbsUf%9)he2X@4(j>MmG>Kuh!cM*5z<`^0~k}TIhyBl#~ z2!%Xyf0x0#bKF6IRb%0ZCJv0F+*A@#4(xMj3$_Z~o$Ita%`WV>&4Hbs#Gy&9=+$ZO z*sU~yTonm>SmsDA+}7acII%lsxp^)3Y}%2Tx`QmC+;Tjpnqg;Te~GfzXBf)P0b3I3 zS~KGikR4HRsj%$j6K5U31(vupHz1|%#eR!DRdVnfBPaQ!6kD+yg>HJ_@+`J-&;{12 zigeq%w7qu3z?dvTu3Xwd7YT7!Zs8Ti9lne1Rr>Ae(XPX|4deD@svJg|&dZ>MM$1Z*u`5kLrrV3Ezdtz%Kh z!8p5^OEuj3j6+z2#5R4_K8UgkQ!LfUdlX&`3$}Qk7je?zMW4nfN6*k;7n?~*) zzci+%IJ?yC_9>z5XF&_eHS8(_c&37Gp}J=k$UTywkY^r=aGBCy*_7C=l!`rK$+Ahh z!_T{zC4hJ7NoG zSlSiaTD8WdHpsn3fn0$j{HyWC-ObtDx2tmCpvJuhjhvySJ0uX78+T@Wo?g^hBX?zx zj?pOOnVfkfcP{EZ3SEwiX;%TAxRqMniz@bH&fbatwe;aAI}qevNOftqBL%WoR@}D{XPtsLlYd*(3p@Tf4h(Wl?NPMws94$J7iE#V zwz$=nJSqTX0uyI~L_D96I7E)!ji*eN&Lj17B?6^!Ch;t!+M*cuqC%O6ANB$+@yMv1 zw|nXab`_1WNBz>ho@GN|@K_fyA)b^VH;6-?p|caI9Pow9fGTp=Sz<3g5qFIDD5w+n z#LuHFi9HHaq)c7WUPtnLPK7)QBd83PxWlYRsm#_*IUyOBeLRIcC1pQHBM;+rRTc>G zC{?+NVAw-;9(jgm>hD>Sc9V%)71fopcvRp#o-(PcrOG_?bH$j%0}z~L z8jlC&_B7Me^2?-&_6**Yxm+b4B-+C|SCPk!T#Q||;BxjD6OXXNZE5!8#jPAw_F;iN z7RSURKXX+N@xK(cJu8+NdDPhL6i^-|LC?oa9`ND{l3e9L?7jfEhqkDUn{!4J9%(c7 zEVdDQibp(REYC#WqnzUM@1D=qh?|oyQ5SO%u+hEP zM2JTyTFOO_r=n$^X}WB;=44L+ zvFFMjT1Lt^5#;$)%t^UM?p_&kkCg~2I~>8tQ~li|3}=1O)AwN4gG)ypA}z%2={QAP z56|5nNq9aUbE~8~s{?NDMmUlb;z_hyQ0i8I@_6{^mKA?|M(xZx~uGt#nIk1Gy!qwHg4H|=zHyUr(o zt`e`)uF5=cao1Ey?sqY_`n zVY@xjM)sFo{>^&=o=;-ko{o8}+4F@3&neuU3ie3fVW&uGFUmWy9lL=KA#U$KAZJ|e zs0z71Y;-dqkN2tEo<6xb0k>eJOp@dIRIW@k<$CSxUt)0Niqjp)RR-+ci@&IB5KUYK zcjOTPxW6RgP6zQI&yhrzad{W)xJS9^GIp;85obA>xIvg(K!M%RmL{%3f!kw4hqw0# zcwE&5mti`JUS(ovH*9hDww~DS`Dl>1c^A)ek30SDZ$!AK44&bVv*_v;`#N4*AfCCa zr;Bxa804Nrc)r)XpeudknYJhyYt zl|9~6@Q81%*M_9I0XQy`nb^w(WztmS@s1br&;pNFXs|nVn3TZ??2Zd=QIO+#9w`&^ zxV^&XR_Q65;u5z3J>Dbmc(2ie1edlXhhg`ZDbHKqC$?1zN}nIlPGzCpaEkVL=a6=f zQcATEJ3|G6-9iSJq}*$-l$-l@ymjHM(<0AT88LCadFc|a!yjPc83wt%FF;)HtvNa8 zEUi)G-v-H!%HpHO!~<+S9#VRi5XwjbVdD8lh5HKx=dA=#_9qH9u%!^DXy-#@hIqV_ zV10z=R37YnU668S-p<4oJGkMY$IikN?V%xdc2MrvOmG>ZU2(U^>$a}9RcsegWs(3_ z85E;3ISuj5i(Lrm>ARfIEZxr#kE*Hta}icWQKb*tDcbJn7`vvh#Phv^(yFi1<6!p^ zh{G$`pKa1^KA3Wrs4;ST0_pyImvUc_yXg_vlYY-2!YyY*o=LXj4LDM!K14jqwzj*^ zmzJy<>)Qsj^|7V>YXLR|w0$AcX0%T1#*9apM-k$B42<0$QeltRh-`X?JzkI`q|D1h z&SxfN9`Sp;^+M1&F|ig>tJIvql+`9BdE~m=VPB%;; z&lkx&A_B*Y5grfi+}>?EZzTs)RX$P3^_n2&IixdzZ7MEbCN!}p30t}_>#5g7F2PnRC?t03Eoj&SiO5gP zTFr{iWW^%tQ#6I&B3uo|8&QG(_P_dVD!!5$Q9^B)>Xr3de=kw21pR;H^UmY-%m4iO z&sT&IRBgk&4U3A0p6}5p_J267RZZ&Vqk$p1SE6zW&)jQ+Wp>CHxIE?y55gCS#6LCLuT&!QCj$NZ)vt;NAH1x#K z69lbZg7BjqCx@IwC2}+boxIy>$jKqGYV^y>w!O1p661*bX@hfe(T3@YE}cGdsVllnx?D;rcYg2XE(nc1fR6cs+qR%i=qlX~U7ZO; ztXl}1(TA>eM7IC%Z}M)0vWKD@Ut&<{cWiH>c4*a7G^%NIY0{-%LCqlELs-V;T zTW?tP^T<25wOj2%xm)0M!Hjn7$!ezwt z`w+SZV!f~hEp3aCiES}^GrDh8kmkU@2dMdj^F2|s1ziswen^_FTKW)9H2u4YZKS_Z zlSjn53&)RO159iD=zwr&e&&(K=uD@!j~hL{N;;WZ+|6Bl_-lo?pEsku6A;=z4jmm~ zUCs>zOWOCm5v7|5C-zK46ZRrB@`zPzuvu>Ve(u>mv1|I;CY$yXi2W6VLzZt>@3wX< z!7G=oL?~8$^Cp#kc;1HCo@hNVX!Id8EMz@ou-)fTNFeLce7o_4O-5+v6tobnvknEK zScKLtI@_VD_4~g+&WLSMADHlIGg_R~Z2ZzyOI9y$%G@I)Aqp?G{%VQU7_2)Z8i>-C z!Yntxjoqv6()Q-@3IK`=L4cMeK=X*BBf40>9{dAZ0J2)W) zS~kx{V-CW06Bf`Mv~lic%)GUJoqFjigtpEqJ%`Ydkq8YtZtdBc7Bb6vV(Y9?hppOk z;L(BYyPMXSj&@kNvW3QPYM^Q8CTWjuv`4o-!=e65VJm^K8g7D`lso&p6)>t>*LX}bKv9{Y9H|b`rpmkp#?vn!!Q>b1;iXc&-S88 zyV1>dM}Uhab_AMy2Q4g|evCZcHMw_&?W@kiBfjCI=vX2!Xw}jqNlTACYJ#j^p*=Aa zft~030%i5Vu&wr*OlSPaRX83 ztLS27deY2_Wb(s(N3UI#mvo}qPTGB|Ejq)>0Lt1NfW?5C-D!<%$_9%Y)M|Uu78Lta zJL^wfJ*joO%DB6e_O@8N2;FOWqy6;@pegrSPV9T7i9c#F{z2Css5#nE^Ips5vp3(r zKsx@le8qCBq_)2wa3Xdd+R(H#a%lFj8T$L&rbmDDvKdDPDDc_4AgwdF53=e$8b3fl z?T!TFo2kP(9)Wv6)+2(-lkGE8Rxigw`y0UzkAtNE7H%+-%$+0sHs2)e&>5Q@tXu|+ zwC>#a_D}t>u!7n-#d#;{t*(h4K2w@mMG{D}2-v?}iC zh@nqFhkx7ueSTmr*|+!2&RvtI#ZK8{dA%R)!Y}t7yf1hgB;Jj#bm)Xuki?^VtUvdL zwxb!2pCp-%&fFuw8x)IH_xl|XcBNH)m!gqT%SJSbdN*qoS-Qje`w~FG{Pk-V+wO>+ zHDP?sG!eT1)B;^A(rUA@Qez3!S3-BHKd4V7S4r3BcVRC184F`06Y>iMi(KndC+9tu zgmDl2*xjo*R*a~>I5aC(9gEc4C@V2DvGr@W3)ih&HueA*{>UR2yTyJ#9$k4DJ^=_C zcKo~T3sR5AcH9oUyw>sJn?)kK5+a}OpK{si~^u=p# zzOUKJ`cqenmbY%XcF`6lYYiG#b>?v2s|!NTlYn0ati1MjLE}R6v*RP5s1e}pvW73j z2gVojtLI2aRt`6cTG_HeebtV8u}z<8?4zsd#^lv zPs8-|L#N;F>E3wU+#ch)Z93C;Q}FHn4{{8u9fwkv9vQhJ)4%=wpwW;1xY)4rJpV~6 z7o|Kqm=J!85zn>)y8pO!Ya6$ERFoZYKRg{Z{1%Y}emf#DV%!1x3a9CDojMyNn8B?qAz~oHcd0-eHp{r)f%_mt?5_`U)2|{LYM@q1c7`$O^-z z&S!t`)8y2%`q!3LRGKZ;1TF6SrRDT9Cat{U)#ETy6}+E&1?3V!6P8o?MQGlbBYtTp zFarm6{o`}&t>3Z$;sd_-uR!Ur^i7bWMnBy$l+C~(WCayn;z2Zz>~TK z=Ai_3Z81@u^3_s!sPmQed8IwSoET0Vb^$p#EV2sK^48Qm*5nZhb)+ z7p^O{lGRmvfw$>;385UnbqoJMy=$k<_@UVDjVeoC)P)#hX)t)D#b?w%Pu(udO)wqa z`&Nn8EM9}q;jdP+i5NGr-ZvDKEHS4agxqtdf9ODs*~!o z@0XS0iq$)=T(*kD9gS@m9h|<@>V?;7#8T~&N#boFb+nokDHpE^%v_Ath*nGb4nsuvn>(J+llB&-Z{P6W4a0vad;D@^Heh?D(S%2B1 z16t3$Wekxkj{X&OcocZW191{avF2?kT(`lr#Of_Kd^&{IV_KNAaqh;so92wELgt{| zlVa7IQO%Q;KUR@ULR$#gH~#zCAzyb())*ITwfji;_fx#}2R}yLe=!sMI@(5NqHXcf zT=a+U(Dv_sMALsH-~CA592{E>&zMpXy*=>zi{Kf6PdnaJrKgXpIc>cZP1`j&Rz0%Z z>YpG`n2ONP{@GKlAp?Lkw;!PW*UayLR}cqJvc@MMSZjoGVC+#%?$X_l8%+YZnBLAL00z%36!NS0q>I)us(ioR6A=(1kn$UicfDEAK7jn8oynF z{@J@-L~ofr4$a)QAM|A$+K<3RZ+#7s(SD?vVRd5%#vUL@Rfr-!O5au?Nu@ z*twYm8a8I)o`ZtDmNiX#NOt82vi@f3Xsa6maplbh7e5Mi*H2PGY?M3#-22F@hMq@N z-@G7VmJA;R^8URAyZAT9dADpN? z21kFJT28$+uJCttY~w?v;m(y=Y82aP+r*&dr8o3H-S$?OE$FwvmYY5vy8J}(`M!Oe z(dgiC7y_x$u!d7@13P{>#TFFZo}%8nwKtE?)hj120lXa^PR{f{-Tzzp-v+D%rvPrD zVGKfF0=&M&S`*@O2&KPkfX45q(9=B=@68)Mso|5x)*8Z!RUc~HX^rP?oV&I2mCo0Z z2u(oyDT4pF@RRf{wRvH{%)Jx-*MI&a@qc*&tK3zHx0Pyt8T<Uhj_$cb?bT z`yZzK4QnrvAks9sqjB~p+7dE zM~#O!@-2SX8?m3Q0p>j0^=YB`-l?d90a+N79F`Q0k`NAe%<~OaoOo=~ZpZPY-PXtl zLepc`sCSQ}d*U9O>e|wW>Tg&5B{VEkRlx7F#;(UF2T|a9-Q`X{ z>RimEJrjRP__hz7s9`Ccvc5;j<3{K)YAm|j7}>(|M+l<*CVzKJnt0M$Q7V3I{qbY;DagzE z%ghkn>pNIOd`KieR<`aSbiajf9{7)YElV@iR;5jK4nbOtdR!>go*Nx*ba*NhcW^N1 zS}&vZ*G1jRAeWK#_sOg&we=?-FQV=c+FgV{*1^yPN3QT^&;3XK##cKcok@w1P6}O( z5W3YCasrTU2-x|O)k{|Y2aUF7qE=^Y`+cj4&hEVvetdrPzcxQ`w~mipg}}ku_naq! zohY+x<%oYX;H|dM>#mZ{&!#!9eFy@sc5*uh=KWaP1`urSM##RecAB^0O6Sru^ENnd zvF$Fc-OK7W{^c`|D880UiCkWm-qORrnp#sQw45(D0oge>;^?BR{T|m&_h0guG+u~( zi!7PZ89iIAkM;#|kEQnQ$z5zSrpg4;UwZ|#7z`xYR?`*AynAYMU$)bwRbRncgN#HV z#mKNJj}LX}xhFS&^YCLE=3S){weoRj-2U0s&GG9dRhd4et!6m7t4>~XGkNr5^+$|& z)az2^uW0!?gcof2{zp9hXLiE=fA+2p4(NuCVT2hfBr~wUt5EA_P&7i5sK6GlzWUYZ zbGrYWO^^*fhS)FRZ?Tb(9divk=4=`(LoBiWa5b8<={tejG?zF3xNkfhZY`|%b=mmA ztsr^*dUO@S((pb9(Acda*+UTo6F6}3g*V^!t!?w>{SGuI&vYDJ_-p;jN&zjc5z=J# z$BCmFP131;1O%8~a(Frs3&>#HGeSvt-C=9SI;K?x+fS@bZ5^t)-0W=&QwGYV!*fJA zKI-NtT8ukT^=gqUch`!j;TLbh7{`Z!rJ~1aeI|`O`O`*4&GQt;A_~(qro}W3h)**V zrno;FP>4gb0i-EgMcsm6d-kqGvrwrfg9V{CVq_F2!0Z2he5d?|g5ZA;r{W9|1* z_PJ<#!}f7pG*d8^tuF5TN88VpUhkNZaibO6`&ip5tuxTJt3~!%Ip;p-q=&K1>!8aq zlY=9rbTW3zIp8yPTLER~XX(f;>aoY3Yh-s$V<%0t+q{l@SR4N+j}@27QR=*xcAJZK za;j8Pr81T}WmvjpofM1QIO?Lx{h0Qkl5sntoTHUe>VBR3dCy*K*9gjkQ>ApXHhLWL zwMtLq9N&FB+ODIe+E2Uj%r!DoW+stShp`*oE}i3?6iVFWRhILt+c}X_S6zQ49;+7H zcx>e|XQNZ(yp#6O3Cd27JHI)n+$rbPA{&*Z8t9;mvr+8QKo@`9|2k$6*;gQjf0P@a zX%GH-a@$kB*hx9#VWrrKD=tlOAx$Y>(YCdeN=@d5Mc8fK?ld^or|i6t*>uC9 zZyfTs>%N0`GTYT6q_kTGWwU3_CVPo(9FdKmQs=5^RM)0o61YZ=LSU<~jf`!5VX((R zKZkr_Y}!e&JAGv_igKa|<)PK3yp-6ebJsJE$f_#vr!@{ zgSswz=A7t9JN3Er4lBo4=jc}L5WArw?SWb%`#2&ut*4#7igId$#Leg7GM4B-aL2tO zdrzsgv-UAqr;nwbcY12P3t8AQ_~MvXYTkj&l0x(qqo!8pJw0)dIx>Ec zdI+@F34_$V6WgmtC$v&eitDbP0Oy+|4pwi7o233BZm9ZUTxYc(99IN-QefW9gfZ$) z37yrtgu&`PFy6+v4(bsw-}r<+>YWKg)P3N(W%2FReB3hiw)j5kU*o&0Yr*mC#GYzH zd|P!uLVKXKn|dkq4~g%o-URd=j~}9D6NjjGCw5oA1e#Amdk_6>5(cPKp??>wZ9W{g zO&F;D5!QGwZnQckVYIqw!Z7uEpkY<~F!hVLVd_GlZ8SZaHknCyrM80M8C5v{Cm09wo$&RyT*aZ^n01 zk4fmHj)66N0-Bq_+;?C*4bID8>kId#!}WFJJE}K;tgpw9Q9lA6?vERzmIGa1foHX$ zFA1*60s50+n+7d6ajd#ATtnb~bwWG!61b-Z@aO}a`v%)Kr(}8rMyI3)Vjk^s6qku(%QGMX;{9@vYUhKzD}4_g8a3 zesSnrp_4r&wdVp;rHbptqV2W?t>Zy2k$dUjkdwE^_I0IsYXreW7xk7w6BTl z33}5S^t`qDIE=Le+K<4C1JI|0xvcBo4D*!--kphWr%sJ)s}6&)ZUK+Cz`3U|PkXrT z5$IM2(0voM?_j-NfHxh0S0&I!!*x5s_ErPUpMW-B(0>}PH^aGI(3Zsafc1A!)3DD# zI|BR^;M$G}9o0907vBRtr=Z`eKflA+O%sQ!`S>pCQGhK~0ZY!scUIp4U0e^co(=17 z0W@`hbD;@c!4A5rMIg)fAph>L9iPxeJteV=Its9&7wBC!oL>OzNr~^Oo(SzI=<-%* zXQ6Kc==SfRkKcf{&!E>Wp??D$Uj)5f12z=}_O%UYvVbgyT6G!DTh~wpw)NmT6|_~L zXHUVNte7z~uA|Lf-a>y8VAJ<;J=9G=_P2nRV}NM_$g>K_=L2ki04(wWIo$)?+Xw5< z27djMFiIT``*F~}9Q30Z@HGUs^AdWix4}IzpwA~jZaHw?0OO2=HFSX%2$-G(=Mq65 zqCwvE!4|XQ`@)*~s=4^d>PN7a=WyNcU>94V^??1qVOs)orNG+9fuD#>=%>yA+Uvkv zyJ75=FkT_(#0|LiF|=La2mSyatOT5F4SY!f{j%ypI_PXS7;7`==3eN#0sNW)GI|fV zqyQOdfe+`P9RM923fo7ZHy6Pt-2#2F+KUmc{}cSpXc&JO@URn{TLpe%F6dkUjM)SD z_X_-q5BMho^i>02mIe2=g*7XI&Xz!<9?qYIb_8s%64=rku%%1j7uvu$y+Jm=LVqZ% zvla0CC^R+jXdA2}3hZii!f^Fk(6=?fe=W4n&<;Q|#*bHb0{*Xo``X8i1ex?zOX51I zBSAmE0B>pnPrHH+TY%?babwj@q16LE5TK_G*iJUcI|j~WK$`%~>i_cK`~u)nK49XX zfVn@x{E0Bv2Ee6?Ad8y7hdUteP~d4r=>Hq$xsWhk{RMRM6U^5OjwiyLjd^o!8O-_ry+bYnX6L7o+jtgOnfX|0+?FL?51{ynqPOXRaS-^HaTJZ(OzXurL3v`&khAYDMKG1p+XQVa;O1fIy9jf)u;MJ}-e*|r1kkHrVV(+bE)A{^fHn`XPJsDd0+3GROY=X!yx`@nrxd_N1c-UVG*4KachLx;ls zE8+Z@xHjN_+JTOYQU?L%q=Ihr1exXm{nddME%>BzaYMka+rwDB)MJ3gS+K?w@D+c8 zpT7;;J%9&k;6pXAO$7Q@1AjjP4^4@EVU67Zn}&kV1z!Pt>?go@O@MDJAb#l! zG=GKTi@*~W_-z3hGy}ak33~Sv&|(CgZUA^O6y|>heS=|+Bp4?a@bM+k+a3CD!}SAT z9Sm%gI$-jZP}#CNh`)jqhtDe${JV9_(+wKw4S zDVUc)+?5$W0d%30`XTVE2k>_R@OmNerVH@lS8MzSMX#pgK?uk zA4hBE9iJzz^ow< zJGTW{j)a)JH^lf8!H*7ySa=ZNQg6_`iRv@Z5)y|2MzjW>)>eH5uC0+cSzQn0UJTcB zfE}-4{`w$?SeSbj@LC0Qz5#uio;V%&G8V9YAn>lO`aSTl7<@$+pl>O}Ru@1p%|AL4Ujhxu z3-oOd%>4=Y-2`M(AN1%F(76%DnE-3-33g)5RhGf{0nqxvbz^}yS~%AXu(%iK*#f|^ z7SMLUm_6XymB5oKFi)H{_Jno;^zRqoV=UbN0p!sh`1BiGcNF;75Bj~~+G}95O28fl zXsHK2_Brt4B%EspvOWp4$HLYKvTq1>dIIjpU?ZKum(2z|xCOF*2V>8I*nAc6^*-o$ z6X4x?IR6^vm<9CAfb}$h_@EcuKLzy70KR1s%y$U-D}x*^!J1Zq9c%+x_5~W>f<6wg z>OT0Lk--0h5c>~@cy0x>`!Gik&^!}(s{mROVZN=fP75420N#heSh3I_4XqWdp)%Mt z0-owY&IV{Fpq&Jn%msSVVZS!WPy^d1@%=#FUDSHu@c|ek5_I4J$p0%Gzkt{=55`Cb zULFDe^&QZZ2yOwY~0NOxk ztpFF|VLKmmZakd34?cGY&{+xi7zJz01RP~ye-6;q4*H{k2UVdlKx-3NOAWX$8RU`y z*HnOZ6=>Z8%?zy~*ho!~-C4j3D=ut?Yez!=LKt@&U;+((@({H35XXE5UUdb0Jd@BD z*3%d4c@XIP5YW>hY9;XCaAH5e$9|CCcT|@H8J34w`HVHD0ehMR^WV4DGXUr0z~^$H z%L4&N8-vbO1Q{wo?rnfxe~@)mh~*!FF13a^u0gDp2AF#XV*N(29S(C9L%fy^*Drx~ z32gR5+zcD8?t(TBbnQ2|ZUFG;9gNojd{%eBy)5uSzXBdVfOty-b36w8d=J;Qi~kO6 zyo>r-;tchCh*7?P&j|n@c?7QM1NXNDK6V4FstuU+8_=}{?CJ^FparmZH=OSVdN~2` z{UGcg0G*f%7>$7r3HYa139~@2+W|)QP(O$t1^6=%;-yaN$FPK8EHA{eJ0#3+qntd~G*Z@{3(ag)Kvj#a+}dF=(6JcIKiK+j^~ z++t|&pb4<{0vP)w$Y~h(zzy-!A-0A*67cs8Y+pe82lAOrSkp7uSAs3xhy8V+J6W)< zDG;xBO@z4I8e@aqUx2w(kmGv6wLd^i`5Rzdd!V^KU}*-#o;vWYO6ac({&*UUYmFOT zgHEM`&sqcP*T7n2AnPYE_DNWKH|U!WbbbdobQEN`3)XJ6tJZ!?qS|JHq`vp}!j({|>R z2l{Kk9KFDQJ_KFJ2YG)Fa=8uXOmI#GZ9X(J@GS@K`v_Qg8P;m0%?tV$fSeb=9G&1E z5!kVH%!P1mFBt0t%pV5l!l7xQ>EN6<#A-i-?!AC}F9R)$!6vNqT!ArHz}nQ{CvJny zeF6SA0SuTA<1PUkX%2D!9H75B*v&n_kJccY7`R^z`ZFH%_!{tC0qc;0P80yPTn5as z=4abL=1)Nmeh{nO0_?=Vi$x&AQ-B>^LAQp2{Fj4%4h1>qf{l*>xk+H#6KEK2t$hI= zngB2TfTtfIwhIQIa0udsD-a7?W8}eLGuB!{G2}M&VI8kPrW1ko2SDG?pc_}=-XmZu zpWxmPpacHkE9SwtH($>2> zCP0gT{RHq=6Tw%NKnnq1765Yj73Av;?KwhHhC^`Py77?Ob)i}&Aw3d+fkAOA?+5~8iLHC9Oy>$El^&f!I z8z3I!z-}qP`U_w$6vTyHpsj(n9-0sI-Gz1;aMJ>9FX-hA@Hr!(J%&~h@VYTH6SV4} zi}j!lg=-F4^H9*+KR^e&Lf-QNwkr~bsn;-}pdXYemm zpgn{Z3mB;cY#9QWxF4DUV)$sVpP#_a3E=uapvMOQQ~m_G`~);q10A&1C8j~!3wV1D z*4!WDQy=&~Abud^q;1vHLFVg0eup5A(}BJ&jvEO4@2B<#xv1fMHrPRD(32a0EmjP? z4?4gBe}95`G=RS~L067L4mKP7^K0Pi6|jZRfGHO6Bfr7@XF$J?0j4~FajY@%5uh;_ z_N8zx4{YWKSo;jniJf4(3jt5YLyRIoo>dF5<~PWR4#8MgK>im&XUjwEq6d1bf#3fE z_V*j;-dK?9A3*yw@blw>*_!9Wel*~b1@7U2AHmQ+A9ymvS`P+&8w|0Gl{XI{ zzU^SG`@p&CaP4DQ|0m$n0N_(?Ya9%Gr{R1Y%ySm*I}5T@fz8YZEXoJ{>i}z0CJq8W zJxDzj#-4AjiGrOi1N&|c*IWj?dI)@*XSEraqbgjt8rEm6o%R5XegSf*1!LBMHLV0W zieSza;OA5z*Bc=3j&Lp;aABFX-Ue~c72uH%*su|39SgExVO_p}+b6)*c7or&4)|lO zYb^$P6+rhKunT{%g*(8Xi6F1@ki-8CbKiwE1X}CMaL#HEZvoePgWgYrHARCw`oUVp z0`~`kN2THh4vic{+sbr)g=D^0Vvo&Jpcdz literal 0 HcmV?d00001 diff --git a/crates/text-extract/src/pdf/tables/cid_data.rs b/crates/text-extract/src/pdf/tables/cid_data.rs new file mode 100644 index 00000000..2ce81b46 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/cid_data.rs @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) const EXTRA_SENTINEL: u16 = 0xD800; + +pub(super) static JAPAN1: &[u8] = include_bytes!("cid_japan1.bin"); +pub(super) const JAPAN1_RAW_LEN: usize = 70992; +#[cfg(all(test, stalwart_tests))] +pub(super) const JAPAN1_CHECKSUM: u64 = 0xE2311E4E34FE602D; +pub(super) static GB1: &[u8] = include_bytes!("cid_gb1.bin"); +pub(super) const GB1_RAW_LEN: usize = 60616; +#[cfg(all(test, stalwart_tests))] +pub(super) const GB1_CHECKSUM: u64 = 0x5D8285994CF43CD6; +pub(super) static CNS1: &[u8] = include_bytes!("cid_cns1.bin"); +pub(super) const CNS1_RAW_LEN: usize = 50439; +#[cfg(all(test, stalwart_tests))] +pub(super) const CNS1_CHECKSUM: u64 = 0x1FCAE917FBE43122; +pub(super) static KOREA1: &[u8] = include_bytes!("cid_korea1.bin"); +pub(super) const KOREA1_RAW_LEN: usize = 40936; +#[cfg(all(test, stalwart_tests))] +pub(super) const KOREA1_CHECKSUM: u64 = 0x5077AF3F128E992C; diff --git a/crates/text-extract/src/pdf/tables/cid_gb1.bin b/crates/text-extract/src/pdf/tables/cid_gb1.bin new file mode 100644 index 0000000000000000000000000000000000000000..ae1a3894e8ed4d444f405e83cc7c12ad50918382 GIT binary patch literal 19897 zcmV(rK<>Zoya!-ZRl6>{_u6fGP0wV~LwbeK5_%U9q^mRq1(7Nuq6k<(1XNU{sR$^D zA_z(iy(B+?Qq!PHGS#Dbsz z_}?lY!tf=5vF;_nopO~YhY`3J?y{_EI3m$zw;;YWwnEFU_fw)Z!; zFUD0w7LR}7{G-TASB7tSdE~@vbBvF}7Ogn{`t0b+Pv?6y&V+k19;O_;4h>+>@_)N= z+>B^91^1!byDNhNkEO34H}m5+(?&MNes$((svgcxezg9papd9gx)o7zuOC7UQ9Cy% z*8QGV9t7X4e1Wyl!~FyAol{_N;`*zxxoa-{NRA1Z;8FZp%y73kfBZREaAACC*6^&D z%`2z=nSZp?w|3)^Q)lXlA6C5=qk5(MOl)rQ&tDjdrriAD#9QMf7YkIc=nhU_cA_xc z$O_TVn0j|+)9v1<<-M;m&gbktap^@(P3+E{*S{FEm!5xV3+aQ%QudpYtQ9kMAG$rXBXn`~o-0Fme^wuz`}48Y3;bnluZi1I=46he zUyNK@>-FjCYu_FAJ5M|0R=0V2!oIzg@m22(uGyL?KdEY75jV70T%$d4tHFEmk@wp& z!kXLg=jbkb+i_Rbq`@ozqS>o7C_*Ym{(i#_rt6WiS%O}m_~c>l(YSJE5@`F3N-S&T56j|t>6uDo%WZ~@D+B0&$PdAEA-r)pH^7^I6@X}YGW)CI6 z_pdfR*mILut>m46g1~O_XxAbC)wJ!}FRv_aesyo*j8{htgP%0z z-3@(h?bTPf!^Rcx8ee3+w|=2<`WE$4^>f+fe{7qg@Lqmx>DUP)X9q0x{OI9u*7?x3 z_J!NZL~qXIs0;qw+I})gQMdHQOFtdot-L$(LXpnfZ`ppX;+yk7Vl6~m*v?yP)~>{5 z2YNQmyQBN~uQ!pc8J`pcA8GR5@z%KYD^wHHTzOI(e`-I(v+s4*WA{2ywR2UY0z_jh3GsV-rGEDoX^-(zn2^z`}L;g zLnX5(d#3)`b}DvH>y_92 z;`jCZ5#>coP9@iz@~3Z|KYi}9>=>ATXXJ|Fx4({`F=Wp1QR>Y4GxDub+3J(~POl!5 z*IfO@8$S(wKkM~#8}|o7v$wpM`)(*LsHW-3v4ssj$=}VHz0!+4?WHw^KOcN@g!3G2 z+Kiqdt-NCkvtj84&iQSe!piZ)urZ(cMeWNx`gH$KSvgy$$(u*q{Owxh4*8EqYB#XL z%jd7$nf>_BCs*FRfc^RN>|*RI^w0U9eslK2!7}k|x8vy@%Xfd)67~sg(Kh9p{PPRH zdwbK@FKhSa4XEDS?X_=9*gMj%-zY&ddaiHkURtp4r%#4IKYB;{=0872ib}MblNaVL zS&7f>rNeh$xwLHH&l?yG1qJm7cb47q%USg1$mYGjJ=e-THAP$f!qmlSDLykl8hc~Z zh=&(X^_G2;(e!?%ApqJu@7j%b(U%VY1<$SCdLy!>`#99_+v1{4-)=J2a1-0#oj&{i z;&UGk8{2+8{x8Me-O9AvkH)6W*?lAA`irqK8MM8@hoc{+zxjdx>aELn>^Zz6<;&IE zCa=v56d!*(XqK)ea+LP~tg8FvH8U&qk0qhG=fbw8espPahwl2@8M0rl|9ZS!#6Va4 zHt%ZPhQ&3C9VPi9Lh+$nKCf%cmCDlvXUiun{%hKT7|lz=9+Agh{qs<1@(ka-58wVq zbatBW7=>7&6s70Uz4Id5q~YWD?I!CbZ7=MNRLz^8)TEsE%eU_I-6vP#L*T|w2hQ9y zYUP}DXE%Sh<%`!w^Y1CZ!?d=ZN`gOX)yY$fxbFvk)6VvL81w zxOF6A!Tsm{lI_~}a|tbYar8G?nT&xqf&;!-k&RBCrI>U6*s6CwS^Hqn2d6jjr!JST zkAEy(ORQc{$19r_Ox&N}GwFlUL(?+TXkk;s_f0yv7#S~U%MkLLT3>@!EsFkX!K{zo z9b1(?Z9UQ9do}*nrB^1OkDvF`@h@)<_AEUUps#!D{s+I|6{|v*E%6uc3hv^)R(YXu zB=-5px0WTIfrgYVF? zuU|@CvHSezw-29btDDYM2J90i&Ft87Eq7JMd%mIc>C3x_8Na^2YhKZhSKnV!x$tfE zrKw=WVrcWrKb?cVTr!=%|KleC>&{Pw5Aoicx}#(HhV6#;aBFw5=E_!Wt=j94E_1uE+%s+7$sIK*d;VJg{n;tK z>05$d|1p5qGh9}Q3<(}$B!7CXcdKADckZakE7m2iSQ>o&`8_x2%+ArqF9Np*#->l% z+y(_6e0Y88Z@PQ)x2vP)=U&-7iARGdQN(>?^UMP^+qzG@ zcj47>UXyR1955=e>h}}-$~Wrw{OmU&>ZMQBGe$?&?(`SWJ8ddnPIlh(X@kw{bJiXa6=luZk7cgqqdd-YxX64D) zptP6I{xNdPYmb_0vxdmR2Y>Lv&7~9P&#eMSxJ13AYItG9(hHg)>>rR7Kg)*->c;#& zd*lVTX!rVCTMRtk*}+`H!0CS!d3OZ-loxSYGRe&&FPIJG)ue~ziJGPuLG@H$HNP%?@SwVY5TkSNjKk`5cl)pNbjPbe){rpSsXsC=q#uF z^S358McmA}H-G8S+>1>06aVAl@M!7xIpK50MO}z$+xyC?&rYoS{mX9)uBM(k)-!r( z5O=BjbMkj~pE~pWyz`s*B{X>Sh}*YHT7%+v^^*>No;7Pk-rd0I=N_L=xPCXJzIdYO zS|z70{Ezhyl45=j_#pd4>C)T3cQ+qz`Dt+IBId?6^x!q|!#U0D9YVot_l?#Gf1T`aBpZ%_?@V*f9;5J@54qEfH zWNnq97QVG1bluiF6^nMC43y0K>xv@3v^wqW^f4y{1yg@w4Hbw--YNPa@qEy#BO|gy zf6kWNesp$7$=NR_+&PEW=C6C$Ipdea%o#i8UmLw|&f3!lw_aI36WYfvNEVM@ko)P5 z{WG5G1%8cb)*c`j6^n&2kjE2kE zXOkPAk19;NeSmq&FJ(~fxm(lS13g+F#cc_^d3Nj5y8CbUt~fvZkC%j}3nr#N$8=ls zfpkIT#@kuO+Gf@3y1y7p4I6KLs4i*jsf~%#ex~2}W%`yAE7ni_$$ih_12>olRR@p8 zp2LeJ$NxC1cn$mZgmFgbC%uw_-`>hJv@O`V_WP}Fn=^h{m3Ar~jo6+vt8L=I#ve9a zc_jT=w^{b$?P=npE78@z1nj?0b9*zsA?>?Er#3`Q{9u0?Q$)`yTRZgE!eFnj-RN)a z-@z`g&i>$yHDhL^C!e11)1`%<#_;#unEv4H`on`+6VDQ<4|09i$wQyt?fGa&j(+ff zxCPik&&}@7JrXY~pYR}W%lBJKM*TXWxuo>$`ip?K{<-vBRev1&bnMmQX*X6YUcB|H zdpPUs58X~=(HQ(y$onJobpLCgyqJFB$bd}Frn!3__>NI|Z`|-p{tMuxwDg2G2w}%p z-;LcecCH8Ov1ry`Q;U5{=17bOr!l_Uusz|E42Xk_;H!f)VF_m|8VK+pMe9OAAZm};rg@^W}^LZ7z5{`R$UF z(HS8n%JMUrmG9N=7&TTO^3DEL@R-_N-@JAvwD_;v_yD;r)qb6*O zyYhPE?Yku%i^2I8kX~}+*ph1Toa6IXof|sy(+A2mZNoml+&HwJBU|>vV&S`Uy;qcd z@l%`n$0d(^8M?if60>+mcQ0L6+4OWFydgmU%Y>^)RomrP6l+JM$2X45|5Cc{mG0&1 z7{j)B##fzexVMV;Ws;V>e6AqukbfAzy!+emhTQpA?|yh9=QB`vEknio% znV6gt8cLhIY4(w_oRAMU`?sE4kuv4NTupYvptkz2=Qe%H5FNSdx#r0NS{nvg6{uOM9w0lym0H+^t7~r<~27 zvwHc+E$Ee+4+MqBw(eTHX?JhN#X0Z1@#CsLZ@uyP`#>?TdE3BT$%NdGcDfBS-2Up5 z;>2c7(uo}G&y?Q|`>$L-l^v-p;Gewon1=)U74^-E1R77wFdV~if1Cxg1c+`{Hb!z0 zN&f3JToa~Y^rV)?joBX$>VuN#c|izJS1^Yf{nT2P#1NMs-}(Xo74CG6lH;8ae0peu zKba?%dLwfm3VLucHxdB8D87Z%(!A09CBE#H`KMyFUp&oe`?|?ZORtDcdG7MUv}FF` zTj&1DxzvNRNM;TT(cv({fB_Sxp(F~!0KfF#J#xXJ&NXDQg z446MU5BHI9YX-nX(yr$g99XzeT{VKvnvWRixDt-3AqI$=eN|)^MmG%WZ4L7)W!JC= zA%N60r|9UNnphx}3dOo8Q{UlE<2Q$|Z{C6eo=#lJKyV;n<8m+HeaXMq4_3A^yXeYL zMsaF*cUP3U$g3J3C@CF5@8rOl%87#*@Y&CPRU6*8pBJFki|R@yXR$R&B=YBU#E8IS z9u0>(8Ey2o-eEn3!|(gy3<1DKHTJ~ud-4OyCWy+E+01x_5|&lyUV9>YPe~_z++aCZ z@_-S{_5deSX0s5F5|3V`F6^$Cu3@?{0*<;mx!G6L^uVxC5}?%!BAV^fV5W(dNqqP$W|-j3Q7osD2&{$VcHqZ&WC4=4!b! zb5zl~ZnyF`aS11n#qslx8r&-6`31HXCj;CRVK~4{ab>0!O+jYn7zM)O0DoHdf_f&Z zu7sI@7d!whn*wW^D`Hh4GUR@ch&#NyX)t!%;I8xwsw0|Afl#xkIu6_#1V2JWEmwXw zddFV~OepHrryf{(V(ZvizlpyO$bzeT{Ez0hOqzD-KJYN=1FC%QBvsvSgsWtY$%K2v z<(hycDA}oH80V?01qPNsTMD>QbhTLT!wo|DMuG61wcq~17;|Udo%4QxM^|A6w$hIz zd^_}XrW@P~+}W_vOUlqJN~2@Aj`olsLBqPA>0WYV?iv!E_=rpr0)`0ID3a=W@Yq^I zc*{h%B&tFgC^STXDuFa+Km)TSO!&?%QwuRJCL;heh9jINo*a$gkRps;M2ui^ssUGM zN8d1cgJyRO8sLb>Ss7*>VU*?!pNdQyULQo7QEY|aFL(Pc{M$Nejn6nv)+PGZA zPV0j4*vqfZ(;$8dLwpanSWReKYXq1oxe04@ttm2F1au`R2 z47kNJsx=;lmk`YMz?Aw1dMGzLNyV-1#5Q{8B-UiSoK+cG78!hB85#X&${;uf?sBUS z@INwyUlCSjWRbOd%bO5U>AbU_2g7AiPkmK$_r@%JG@#yhK#RLq?Ttfe@PxD~gGUg{ zO$r87@#yZCpZk47{H;iM+P)XGul&I*U(oTOBR<5P^zL@s>l+7S1`e!c!#MpeKrM

N%V!=%bqPN$CtN(45+o)Wqc zmTH?<6ll5QvWcJgEeVFk5-7ygiBS{A+`m0Dwh_PM;KDd4Y zf_Q^5ZLF%8(pVt&1wHCf3ZWo`IqvNAS)Y(N2Fo!V5x~6!O-bjfg-ndcW{Jr+0}bqQ zFUF*oh#6BAvG|}Zq}!{w7SQM%Saak<2`qx)M!z!*L=yJyZm*|Yc zq`|mQ;?LK5yuJv)N};)w2xz@-8ZP1mtMr~6E#kwGn(`0_=3%t0ILBZRdJ>Exy0I;Q zWuVav&wEXqWpYikp-kRu#LSxs1c{J{Ei0ISiK*DMm1Re0vNi2y8Y6bRk?7N2m1Rt7uwx^d-?(Fo6R$U(3>C_yvs41V})L3}btD zx&u~3eW!;!L9L}pB72$4PRdZNvP&5?Q6GkT3ac+(&nti!9uW_omJG>NE;2N1oFR|7%s3}JK{)EwH_ zA76qEO}xq=zehapRwXM|q={7AGlaKyh7Zf28@*`lZW0FHw2W{=2aAt;AzBuIGusiS z4(2NZ8;8i`5_d#mLUzpBVW1IHQ2mTB&-^Q&N(0W=63QNs)z1E#`8_aKlN#I})uMuWkVtVt{qpgabQn*3X55b$>n z^W+jdJ%(Tywk?i!c($MZHlJq*xQv&Zs*MqLHo=7jG$7O)e**oJ3K&JsKS#D*g(l*;u<|o;iZPTusRIKdoVr&)3ZQ1gQgSe ziQzOopyTNCtufy+^qx9`AVmnm8G~3|3~XS7vk1&by8_Pz_-u+^^6-3&o7t7&qbtm} zm@>1~b!{0LI9wC4Y_ZuK8@9QxIjbF^vtZz>S95`ZB#tja0Y+j5I*e^JqJYHEgQ0-~ z$UOj~>v?h(mq1zQ5qRrF(^H6X86d?p1c?wvBZeh05QY(l@%!_x8gyJSIOYBOyJ+R1 z#R1@((_er7(ZaNEehl@X?`zf!Uvg+;Wb${ih_6L~wPEn0?7~oTRlLXL%6gH;nA^~yct=f&UukycrFbqI0pmGv728fJ{U_1WWwCBH0Kac{$A$ql3a-raLc*N;B- z=CK3FSIE7&+DW}hA?XVjt+{mUr2dTb!Ux~Pj=S%_DzyGXQTfN~eX4YS#?JBE5%p0` z+QRFLr_cN9!}mTqyZ&-wrJp`hT)%c1qkI0=bg$J*I4=)4`_ZbS+J)oJOS97xM|NhP zeykX@>*n^CnkEeIy z{L*U&#!p{X`sGWpI^mmN#3l?%Ce}2r7(Qvp*|rc?$nm+d;-g`A#vW^aK6Q+cb*%D< za_ifv>F;}W?`-+#`NHLajd7=cI4&E0@5()P#=zvJ)C1k9@$D_g4qfj^|A9qJkKlb& z_wk5jpC8(Mt+Fa1XeX`u^#g;-$C0nJe(!$pR^jP(*?|2%_r6~@q%rs1^ZOOeIakEf z-%*Lzbx1GmN&H4XGveiA8#PawYKvajF!bj$e@ybDu^tW16Qg^cgpXc1>zDJ1dtaOL zb}x&5cF3+d_ir^0yS{6gWPj+##dEGDzQ3kyFfVTRQvSSy`-h&|miztOGw&t*ypyx( ztu5N7Hs7(yQ%@b8zH-|j<||+Oj+;mhnfzPNhZnD`8}`lewA%7(;7tUHYp?&f3qQPGlz%yTNGWia?YIe4GBp5|ubb3c7kxbQ4m{71o_e{y!lH|pPid*{t%J0IxNj}Ga0HDvwP2g8`TJ+E(F zDoU@U4?4L8o2^lnok*@(8XEo-du#QOB#*DhKl*KUqxR7DPh!Bg-CHJ=-9Ecymhav7 zf4=?>=R{H0{5RT<3ieGk3|p9w94!B1(klZLuOI&QQe7N=`FW3@w)bv)x5($qZ_0%G zH+)3wdL`*l?!os?`P>oDZE3#n`R{*~%spjTw)`{3YV~p9t~sojnKScS8Z;*Y2PHL* z9yzmYXGY7~uUC&;`37@rOHkpJ{hvPefA`jB@9giq(Kh;SXz6#+pG>;b=zh+3r}t-9 zLvP)@)^LC?T!8$(sCn)6a~~xKy<|qZPg|=tZqhhyT^b;@SZi_g6eL~oFh05dAU1!cL?tS5%WdGz* z+rE1+_{Z08B;VV!Fel{8q;*Zdc(ddW=48BC`HFVi+?~4m0T+9&v#0*a_-ib0!|f@9 z9_KzO9-8{+Ta1y*xSjVOsOG;lS^diCvN5OeOZ8t1hkpJ2n(`T0If=Pj)n|7%j2RWO z?eduu^4T%oyB7=Y|2SXP=(~Mk*ogc4MqSmtF{?QHs|l}8-m}F3Uz@fhYIn@BPevYk zxbad|=pzv`c>9Mx&i!ch9xNiVGqvJ&&%Kl2@V*sYz10UkA%^6yy+G`qb?1e=8OnoR zy!?LebW#?thH#qQgNGCSc^pcd#m6r+?8T@7?@Zx2m6W#bl9BW(w*se|OrK)r+n^s6vYFu4DO>T@j?k{5tIv=bU^_ z&4!eSPBMP=%a!kGdN=*{c-GUBClPt^?)qi3F3&m#15+c<5YbE!9tIxUAn(6!6uz_V z(AT?#DQP>-Y?7j5(sA{IN5+pn5UAh!)@Z_rHY1!U?iQyFyb;e# z*YXT#DB($8c>s}UJ~)K33`Vw_8a8+eId_CjX@-Z>IwTBa;yK}Scj-D0z1#P0N^uJv z!6vZ-qI2Rt9)0rUnlGO>Atnac!k$plJU-Gl=wh{K&?nnMPE5F!He&;kbhurA5lLV4 z1WYjjGovv`YIx;5$`$)RC1qh%ffd7|^YlTjx-br$ADHV+Vs%V%qoO>1T;0IhdG9D; zNpjWHA37st+xM4#`n_&#*tKR3lhhOjddI`xiHK;$s~1D>x_P@lfIYQS>*K0<6Vlz5 zC}u1k|1^MTVPFTp)xuJK6UrD%FJE)+#XlEp4@hWbyK{XTK4tSszh$4(K>Czl=LsqY zJsB%Mz7k6;!F;$NtO@njF1kjz_p)%e?!d_k?$L2DDDq2$b@VxsUkf7JWzn-Y8K>#L zB4B|ow2AMD1xR{X1o~DmBUpDt+|1t!Eej>*h~Il};tmy7N*y?CF2#<{`s_y6b5OqrU60dT14myo?% znRY-0!&^FFwh0dfGfW-wjGG{pSJQb6UJBC=!oRs3e*(`Oh$^^9xj>k^Oa!A@$P5dw zVd-4}KnXKOC#?N~quX!}WBx*L+}x7j1ZI2JG&_d5&7hgP7ZkR%w1{Om;Ar2iEs5!U z3wA`zZvh0RJO>BBgsUxJ60r?yxzTbf3>=*lVgHz8QkeB6W}U5RX{HIF)}9Ag|5~1c zS~@KhSgs(HLt{2p$El~%6A+G|;7W>PE&Zo?2*N&hvk6Q;2|(D8zvbgrdf75Q;#fa0n;t_PK4bO>vooQvvv&m! zg_~Jxv%64y6|w0TwL6|=WYp3e+U8?tgv@L=z$}&ofMy;SLlIk}YxP0crZ5XBZXMfh zY0KM$wP!ak5ohD5RX23=Puw&ZZu$&@Sbfei3qVI~-eRAysp_Hj*9dCyI?E#Hz)E+a zm}!Eh&e;42!qM=X9|Npyzs)9gWDB5Sd?Xbu&LDs7T>YPmw9B&`fRI?OrNA#^hhxaYS*OsZ4|eRgV?wU)Mg|ATw>Dt zX~g~z>q1eRdCW5)fYSjj4>k>Cy@=SxvJGf{o=sDB9)qwxgKkllVx{X?KfZY~|>vh|NQJP7~FlJq%`3pr+Yb^kDnti#Lh2;hq z)ln=?ju1AZ;Fiyts=?5s~tO(OQ1XV)kK>^Z{K7?#VxH77VKW?yq6ps#fudEi;YBcP8toCVXe)oF5b ztN+24EeKoXBpmq+AlB%%8G&kcdwR4j0|gA*n5I;OSaPqk^5QNtLttl02i9a{FBaC! zL9^#dTa~vLH){<+s93>#3l7_(6tM=S#cn1afo&-Z*q?~NE>4MA5P=P@SVIVL;1Ltd zvX({57h;{2Lj{PlbOJ|Zv1Sz-V#Qz7n6*tDv1Lok{skqSIZ~l{uI6zurqxyK9*R=a zslBF9gV^CpD6kJ?%gcy0x}Hr000q<$fZYL{q$RNPl)wde04k^i)}c`bj#yB#rK;NN zggJ|%w)k`SH$t}+H(P?S`-3^yF~qJg8~!leMKeJLvy+N-wyuFUBp)U0OFFYY_SRK z6~*qoPAKTO#};6YjA~7Xu+{p&Trv@JWLeM_u=%-TPXzjFwfw|fI(i$wEN0g5;giFMKl;`AEJ8@P?#2F?@)Y+h_%x+Alqw$C_n zi3vj6G-g{I>KMazqdg-c)LP`IfPh+{0OC?`9RS9jvQa0f!ECkOnT${j*qzbRS((2u z`%`Q!j%Cl=`W)cI%@$B$aRy7wV}1J{E80ixkO06aLcIl8svANrxl}AH3hCP(B8`;E)WygT1(2~QET=@?frv8vDT!9 zSz{V;lq>>(K6u5Eu55-d12_~jTMnb44rQZuU}$q2nqBeMv)^QZf`%$Ets`0b4IGY9i%krmV|ERYB2MO32XAQwE*^(-;o1QXKe&HLckP#nD50X zy@suz*r^c|qZ$B3>@HwtiEP}79mZ4J7Q{x}Q0(X09wAPTLmYgFJzQ+vuC?*7eY4hk z9JNi|o=_>M(_SiVsMZuXb}|8RBz@c62n;Cs&)N{#^E~R>c-ZTQeQFN6hKev?TPIFo zpkON6qn1AeP|FCQ<`HKL;e@(~Jx8I4JyKEUkpi1rxaiiBsm-VVDD4)tB*T{C?|h0Y zUInl{;Zi?}{vC&ixHkEgC1Q4F3wG@$91lcY+i_r5lXFIn57>=sM?us=2!Jac_e_#^ z=-&iXQ5OJeV?F@la15JI*zyem`lJ_!b?iU@wg#(feno80U|0Y&a9JP<+goQ#T{TBJ z0$i;P_oeqNCpHn(39JCMT`kc1ndY?*8)79l>@BGYGXlhUw#7j-S=%lO+vebz zVf%p^b4Z%o1gnqHt>hDi*;AesOrVZHvs08n#-9d000MDLRFY{?g;Sgen25NaG+i3$_}jv9yAfDD47eXy+smbykPYR}BT z$(mZr0pjSF>`fO6V5fQQn#3#}8#V1&#{ez`z=3{jhNQc~L&RE2VF$PD1h~%12Aqi) zbs>%rJJal9CcH0WMFB64e2=<>kqZv6U@1pu>?F!ROmkHK~V z!x6nGuy+`aXmoJ}N0);-k~w1U-YpcFCASc^2(xA<%toZy_7oi{hZ~-iP8`XQHfd+BaKS8JPQo10EK4TKCx)p%EVXtnz)=pdb3yVu}9Jf$I!B!V-;bM*+97miP$>9hH>gcC2O3P=`C()Q2Ss2>9ggG!GKD{A}$<+J-jj3YD(b{fNLf1L-FG_ z6zd=-oO=LFn9_GqW`J0?Gf`^SfLehW0;oG^=4RV=;u^Q*a>TkX>SP{(0~OkL7^#)Z z(bw5~M+~-a3sN&JLdiw0d*haMq0gXQVCPUAt*;$fI}$0{w{^DPggUnst@Fffw5iS9 z2&H|cAOZj=q^%3NXrqm-|31vw)7o}}Q5WLIk&_*xTiIp`Gy69OPpt0Yh$jWJksAAffr z>L)Q5Hm?ug>fC#AW;n`jC$Q6agc&AcG{g?*tg6EiYJZ2jKr}13N`NsMZa=t#yD~$F z3(kW3w5vEw?bmH76+x}BV53!W$}TaWGPS5<$j$f5fI0gh!s=Yju+VJ zJgaN`UHgMS)Ask>M#TZ80ky{uPVE{T{*Bny<$AcuvUB6X!kM-l00>7_vcYt7_A_r5 zn@Y_?`ftF@7Ng;IGxs4L5Jyh7GMk7C zpXnr2G26K@$Du6XXi{D14I5OXW+z+VTTOyHjujvdV}oZ1DhvSIez4P3DEq81W=|kC zbO;=~Jcz58Q5qJ+nS?>#gz4HiAT}?jQgpz1^20)ZAkQ4)0Px=&Kto|lE<&C0=U@S0 zN_y=JeH>v1+sP(22I6QajI-PWQG&(5F@%BP9Ndwltlf!YqBe+xIQk_!@j!7Hn;kJB zjfEiypi+;76`&EQljt=eIg4A3i^mVkw8gL7(zz-`bJKiRBp$wctr&5|2CqBZg)dQzY*M15Tu^mLC1WzAKY3I0bM@_J@5Nynn z1#twh`Wym6sXLQ>32oH69gg%l-rD#038nc5&mKQ=9OHvw`=-3Z(H*$W^bX&Yu5cJ- zBaY!vCp4Fv)M*8?au?LnPvj9&@a};lMPI{!&+czrazbi_&wW9 z^?41P%&lv;^Gw$5%cxS$2KZS7^RXSu6)$9=hF}xLI9L7 zJDdZ2T9|H&l`z;@xhWMIxO6kN1WDO~!GN8^wVzzGgfUL3y@1j^JxlvjsCer(q}>sz zzDlM1VV13q+c=O2`E>JL;SD`pixTE@-iP*Naa&+eklMX0D~=xm%{MI zfAFlwuHEzB?C&_R4g&veU-&Nw`@cpn;NNoV|KkGwk3Fv6M*Y$scK?%N8Ts#?V?n>a z^oLWwJ^H0z`lGyG5B<__)_>ud{e5-+y8ovr9sFOhDDZFBRlmdjKa*(y?6På+D zbn^YuFaI|N3I4x#ROWspntQ(K~OFr4RMgPCIBK$ z7r2)1Rw1EW2nr-2()wllY5Hsa4q2|CWVt^CRl_6{P#b_+ZZN+~YE4%lj^b~gNdAY{ z*pA;Ml=rLt!RP*KBp&)7BKiNx`*rbTj>Sv~E=J9P0-zcr_7AP`&+TOuJqbPg$NUKr z_H8izjQv;UNcQ{}DyaP5tN(wp-WNvQWy&BjkmSf2DoqGfC{YLzRY;3h6R$$Z;?N$^ zXly|(DTBg_Lt4>DdZu5=$W}Qj<8*bT7I}7Xq(*;_Y=#u-46P=k`B@Cu)6mXkbJB*Z zjQPorllWI@qwf(U8p}I(ii}011vL{q&)kScVSS*gHxPz6Pea=py;)E*M9Lvg z{~Qez_c(%s!D9fL#YQ5w(W|rBryyQkoRmv? zwGG4Jk^%Xg(gB^JISh@cyD6zaJpgTh2HyZ8dBQyo5`N!{M=uwt*h;ZFPv}w8JiN*s z&DZNlB+{+DU__U(P2Cf(z-9av1tjV)Wv+yrs19wXO6S?c*RdN$=!)ZW0_r=k)O^1t zyn)a1D;Q9UbSE~o@gwiDMD?oh=D6lo(!;YJNA=|+GJ5?3irjm(f(k;Xj#1Ua@e=PR zOuD*u{2jmY&^DlUGq5VgT!;A5QN3^@&Y|fuwBePb5L}fWe~&~GtEBENJehAJABk5Y zO}@5VNKePL2h zHr~UDfoSRed9f9#1$86RqjUx2#9~}f>d9zli`okO>r;9dO<@J?Q4KzIN%xg0@@BE{ z*4UQ(A&;V~H~~t|JZn_&Wk)J$r;IH;%Yr@AYBU0!k*6a&IX=aP zK{X+d!V0`w;25ZpU#>-i-08)zzs|k;@uZSAVV6SVD=vw9D027A?neBJwZV0y9)UD` zx5g+v-a(g12V}E5#L9qtqrVQXgojiqA$FUPrWhb;G$;Z*8ui^Eq$;(XqtI4xc?va% zYLrLkLSeaX3P{Gr!a9orb^eeXtyhf7hs3pYfx>EIvZ}KbgSZ@&gi$Y5jjtdr1S%53 zVVyoVS?;P9BDk?Tt}?7jLhp$z<*>TRIH+8cd_R;d;mcGkRlK{R#Xl{g&@1r{k6DZQ z`&87%!6l(+6UignA+JI}CNziZNIkQ;08YrASO}3a84d7BI=4ZZm@xwFZs4l9Sz{_= zZtL0c^(6^14{j4#Ac!_X55n3=0m5NtO5>VHy%3S}nK`szy}aBQ=8+5gwMX9&i+Vh- zr6NSO(4$V@t%}c1P)L$52z*J9lTwjryvvGv;t|+D7K`a^9;hPZa!}QndwfU)Zaq8fPE-@j^{=bZkv?sZ@N;JZwDf|m0zwjs2)h^_VZ zNfV1oLQDN*ModZXNG!&c5`T6MV{nbWNjJ$T?LiB0wy#dAhgxBGdURDUu0(qx8I=q& z*R2^NLo19%5(-0MX=GYptu7v6VUJ<9l<*a1@;HVf4>v4xG$?FME%nU}(YGhn)VRk# z#ygW+>jK~&zWB-*R&L|)bPkUvq$#h*=ZnMAho@=%F(oIkqB5jc+cv$nFpTuQ!%QlW z(mi_ybmopv=l4hwJwTaQMe=hQP05)G#t7*2x?}DtCVtIAFN5JFYq&P8+-}@gG1z#FcIR_Q7hyq6jEJO zhfd|zts9gZ=%Km|4K9TO8$EkFk~2_I;e;&EIZWBC)N(7Cf=*7af`f4(9nmTyN9iAj zbk#G%%Cv!jcUXQkgG!pmU!{kYrd|_C@J=L_Q|(cu^XV|^wS!r=2vLP*Bvjf%FskVU z9g`-ugU&{_R>Rh?{Uu#SU+yE1#4Mj|y{L6aV+U~ShB#duV=se%jmAn?%OiZCI+;Q2 ziPf}3m&BuV?Jd8yK)9V_l!P{L5v*zOL-*&bh8$G20dP|5n zT}TGXb7T;PiL2d%bJ&o$M9;z`iO{W-j)K&CGnCLxzradyCdP*Jozcd!=yHUJ%+(Rx zVwf{D%eP7%SrVTq@bO7AMro_#>fP#jk?P7u#76~Viorek?l@UEra|34tRmoHQtg1O z5d}^7`0jLMbarG#V@Mab28xzpRZ}uKV!28N)p)QQS+arbN1}Rei<0CIxssAjgf?_~ z>H}dT+HU9y&KICw0DZ(TCg1XF8jRx-NdQ#hUWw5>4Zdju?_fF)(4}@y&QN=EI&>|G z3Q4<6nH2_Wh#6~X$vR1M(d&C=Z<@z9^PG^q|rgl;$Dnp z@&bL0c4SRwh_**!^kLSr+C`9|HcCxa#1`W^Z#m5`H}Yvn2WXwrRTTdyxP&$0Hs({K z^lo+=fZY>HprN;*F+{!$=wd;cVenmukQsw2lImg58}lHxgD4%5?cr7nc@dt_O-6Vr zsr9PDak@+v(-_j(%FqtHA@;6ig7V;iHhO0_U15~bz00+J?iCtH4NZAK_o^Avn;8$4 zitA<3oLe!1&O*YYD_+6tQ89Q>t8Z0?jt%V6}CM3Hh(Bl8Z) zz*+6A@HTa;kf{-onQkI!yHup>VD|953I-I6sPS)*GhiOpIG`$AndL3iDcq{thU3`- zl^GgO34_(h(7UyZ$c{j=4h?VdAc@CHNTid?j6BqzT^i9`IqHdTm(~kWl?znKcLzX~ zwD_uSt==GNWese_0RDg$T1SdaE(m2y>2kI#vANSnSl^YD&&5J61-1LNBfNI4u_vj6 zsPOWv#bLDuLg1cpy`|koW>+8=fs0IaFR0ZoCxBTv{!g)A4kimydsWA`7KH+@+Y!p{ z;AU0}(#506RHLqm{d4J}<_^-6Y;cptK7|Ozz#LXB$&M;#Dg(R8Zi!)3713+}fh7cw zG#GHTxJb&T(qO@C}R+22Q!q9fKNhIQ>apCAfU*<#4_JL&# z$Uis2ASezg){@d8Rlc$=DTEOTz4WIvwWb$}rJMIZ6aeFZMfN#obLQC`hcn_NopYQ^ z63)t%ab#s>mh7|V*`m6fy;q9JC^NDX357VTjAZ3~-+$oy%lGqn{)^{%hL2Je>Y~2e zwi#;pSvVg=k_uG>#=$c>k#ohpb@G?LmpJkF@v5U@tb#J$VgBfUO^I7eef=r+43hQs zlQ0lJ1IT*KW2lb)IziD;PljC?!-{L7#S)!lKL^2M&RiTag7EQwd7~FaJ`HYz?~Rvi zvJ?lQxu@fB!Jx;#=r2ho=+S$P_npq&F2PqEbu9_{ zDo!*6(VCjqMUMd<)Sx`t+O&!i5@#SCR0XAh@^r=)qhrdqdA^#7o^Etex>vGBbcQKj z5XP6yNI(`_v(FO#;#`J!Qt#nIZ{2c|{24ZcqU{1i-K^V9UH;>fUdX_i_Gty}%pZa2 z|1o$oPv&zb-E!L5yn6IpQ~y`N$4_ENHhDhpu2|3poCHcP^a8GGCT}6Fi_k^^25mAK zD$O=@r@Tg?w0qNP3f1-fH@H%}1I%9IxmND5KtG_3?-8wG&i;mk42}R{p*oSfjCC1M zq(Fho-LDImoxA?K4>7qet$g62tPFFxngz)f4^4^kgAkdR{gq(l=hAu$(_S?_EZ1CK zxJ^Clw8dp2NwOtK;ltncSfB-7vYSF*vfn`kVE_V^DcCCzNzX1dhCpQSe3wWz@aJL- zX>Y@>!9uLcj9wkikR$iL#aM~+ft4P-)jhxub$Y$f752-C{@Qn}wzy7GPrEiQPk7#?w+eha&hCp3C~J*AUFU~g(4~EyM~qXy zS;F5ARQcK8uKQS(Ey={3upYvqsBh9Hk0tMevMl~v!uooX z8iwN5G=i)VZvGlQ2{vmUeR)Fn%{@<}zRp9SH&!Sq)e5vmh)1rc>MT%)kk3o^sm0I3 zhoFR%iMtRr(>*5%vIb4BsjN;$`V9JL?aMXwgL}&||Ng}ST$7suT9X+3s(Z9_Nd5r( zf7_{UdZ5dIQg6rV62^ki?2YlthEyktY(#SysoA5u^Qv~pnw_Q=xSen5B~zR&CedtL zRouu*hXmxwgjq&l$_nIAnZ)rwzFWm@`~$D-PJ{Zhj4(qFLr4A4qouYJZXeAsti)_2 zCDyAg!rb3*96&TAyAG%XYzDEdd@HT5nF^e~c6q}(0@XVzr){}Y-}d8n`a{j1J%!X) zRkNWEu80jdZs-c%bMc!V6ytG>UM(~rxN}lQc3*>w5R%T?EqGVK-+1o0``V3`uq<=< z9#D(w{q{Wa2VFzR+MVc12#dM%KDz%p1=S8qU7LEQC7XGHjK{NOC>QQflX)hHg3)#i zQWYZ0KCbQz$W37d-m(;r``8+Ug7o%WB? zSx7HMNZ&m{>Phq{)71T!y!##f8KJ-acT4}0c93}j9WXmIMb&f_(s8L`BJ4xe=?{0tm#A*-kjl- zvLQ``Dn8#N{#9V|Gh#DcM4Y5`_a1<3K7(J@OwU{J1K)7( zT)2iw!4N)j9aE&$0(#@cbwnlYENoSfddKmFNhxQP8|;+HlKk-GWBD+>#BI+F@J-3- zXQNBSE71FdBgLjH$_qFhSY7Hj3FgbLQzcYpCU@NRHe3HYn2*i+a?sTe+-Wt>V8QHB z^#&}alY1=AGEx4;0NGOhdCcrmY*5;XYEd*Ucr|N{!iy{C# zO-WZHAveC3A8n>8%0wUgy?KnMdHsMtpZ6YGf8*hFg`e9EO)GT&p3Z=t_b$wuN#d{s zb#P1X+&kGl`mgPnc`^UYhVww8Vm46eR~56;K{~+0YoZ)|xbu&A&;=7cn~R?iQ;+$9 zz~SNyH&-|ZSo_GKN_BB_;{^b}LQBi5EAH2=ZwZbRcUQ%BI}NUMHumtl(d+N8TB7Z7 z-6;xT678}R>7ow?*jzaF8a3gDT9qBrTHf)t;i}T*RtY$2RKVL!r^m0;UhYY3GE}Oy z&2`1et)G+qQ3*1+(wDXh65q$>@Y_r;ab`#C%zuET!2JW@REA8QV8OM8FOk0I&QZ3F z63ta;Rk7hujE*h*P>BWZ*&3sek8SqPO@ycN4?uegLbKdt73#*q8?Ho1qxt$bS)(o~ zd`1wz%FU9@g5*$=E7hUt@mFskYI7_xSjj0m4K_(fmv`UeiD#-mR*{wJJERt1X1l#{7{Qga5ofyPNz2y>+#EW^Q1 zZOiuRkbcwbmIjCJ(_uJoDHRc#4hH7R@eDK^IQ(jHYxzLVu8yIV>&=9*8S@PZJllZ_ zA*a&l1U0v8^;RvnMFMiUAAUiwiMsuk5UhNAJCz1|V=YKRF>+xL+e zZ!w%L!+}-*0;B#K<(vkn5)#2L^VvRB5SVMSN15CW+s~gaxN|klbtL?%?dLi$Fe6tm zc>}`1k0-TFSIe?>|NQbej7t_40^2O-e%PfMpZ5JA+@`50$@5bNr*4wP$EwoZyKHQn zxd<=O>l!`SVa^=ZCWdlnA$LS3PG?{D8|y62arB`H^z`XMRBf1)+QFhb$A_0oZ{|;@)M-_gGM2Fc72=(;bCsfTJc_8#__Zm z-JDuY0-==8aYZ@AjF!#@@;>vrn|=Ey#aT`8H0sS&6OQofWqpQ4J0A8w@=XErFJ=UX z-XKS+6Q|^NU0TmL+@jsoMdXpG?9l@x#w(ODsun*@pFC-Kf8aB!&H;rF>1CB-s1?=u zG?J@~4cWf7ClYT3nvbzM0*>CuCjBk;9m9?INN-bJQTOl0N3cF~r@y3*3|UB+8y&kU zsu5GMD=7#M2?ONCJo4!jxx$U@|H>yv+@>(j;JUY&4&U}CMSJMnM#zFJ_4lzI*2*No zJO>&F+Lsw=-SxbaviZc{R7XvK857I#*#t~im&qWqXFE!%SX=yQ#qXLQSVtZO#YpPm zYz^5zeR;Kb`;ye}L|zQn9nrZ`8J~z|3A_2iDFAwF8Jg7(W)~zm&~fo_2M&rsdRWT0 zn<9xbL8#~Sy6jF#g$voow@4lJj<)9cQ*(S)UY!JGHs?xzgS7p5I7tI4Yx#R3KZg+{IpyAG;?HQt3{{8TvG?w@mmng1 z)d(PFKn`n8$u67fY@&>*Y+R}ISsvrk_m5T8cUMFo8V?4EMQ2Phg33K)W>km>jC{iXYNA=EzJ zbkU6|Qa2TQmvn#f(TuKubR@XivHafu9L4{W)kWz>M&0~y&-u>##jA&1svtE!iI%Yw zI#5L{|B8p9SiBujm3Lt#K`~6 zu~JUxe(LyKn9jkCcL&24_RM*#DCHG46){mJYl6i_?a=L#_M73)EmM^gD+z z1ubsn0eK1TE&(F%{&Vx9NWc2`eZT*AFEh8dw=z3BJ3F(tckyXU4DW`JF#5-jdK-TI zevuPz`BvZDokdc*ppheU_>Adv7quGcZ1vU0y!z=Yx*65z^465HZ<+^RkXK!pGiPM4 zqV=uo+D$Dvhhq#%U_1v13u8~0&Yr=~+@xLngty!O(0r8;&ql=A03qJC3* z&HkeORyv>dI8~6+hyLkmu`_A4epc@Th1aQ`7GNPrulGok<`)PnP<=D2L)ZJ7JOjqj zL+3)1Xq1QJeJ6ENck=2J^|WeN_3GFYZ+Iws;~_VSrqji$hkkd573G2__Prytz3WF; zM4s_WrK(Wblev!0N(|4bL4G0Yq!%-l{^f^>{vF51FBmKG3*u`dzTx;6#Rgq`;jM-v zZ&rA-o$-Kf$l_9M%hcFC;-iP3elYS-;nGVdc5Q#??a6*einf@(a_r6@XGd3WvTRAs zdBx;kV!JH7w{}+elvUM^)o(e!dGrr$E#;1yi(BuS%cdQeGNfvWhEZx@fZKGTZfR3& zJIj?1>fEY+Wzx-#t;h8)eD8*TX86~EFArV)+PmK^?y&Og@sU+m9JQ9(ln~$dtxH?_ z2-9{v2~9uSZgRn-(34X`dq%0%+p#U)+T-UZ20pb+Hj)2MVZ#k>_v`_ZoKd35|&bjqPIkc?kpVN^LC@D2luC)FFLr}mhibk zaD3SVW}o>pH-BlUK1V7%x<w(2<>dZq z9_vs2THXn}Kh4Lz@J`>>`gKjJ^Gl4~itKx@;g6g4jUL!2qQN|U#br-EK0K@0%WO~J zJVItn{^X~YO?MP6UL$k-`^x)GUv2dA-Tq6O{jjD&$S=LqzFNZjt#8z3aq-~^>uklw z)!W!_Xb)S_8^YG}!Q7y-%>pj;dUJ5+cNShVB}(N^EDKb=Sks83w}rT&D@IN1xIg9k z4<|advR@1jURCbiP+Ej3My}S9Yaz7l?3+zm+&H>wyw9$V-c zbGuipQnks9wx`09?wr4PtNN}U&do(?^h(`Td_%2tW00{$iE)?O>E54rZu*`(@6H=} zdTWop?R|DHeK_R7$eIuKkS3ljC+Chi@&2^$Mq7=27LQE3cC(7PV`7)aGe2AQre{=n z`tWWChey7=FD6(#>VM#s)2@t@VU=9-SSuXI6)Y8!xo@Bw^N&dK+h1YlnZn>|&kw_G~X0P{uy} zie={-Py1UR_UL-6>gsDZ&dxZp&sePK)HAOw|H`?cZ0%{a4sRRKU=7*!{`wh(R}B05 zaK@wyFActKztv{m+so85_0*SZZrYl5_p#-SC!uNP+Cs}}T`O3uxv9w79sVCA@Wac5 z+lF<%yKuKYvq0CAjyth~f>t*DNm|%wW7U+e?~EVoygA}X%=)i&t4hqQd-R}ZR!6^E zV!64eFQ!h~P;_mVGw+RCvi<6@J5yfi?nyXu=;STG<%?cAKjd_koussLbhq!^{dZUU zc-2prUh4FyQN^+=YK-3T{;bI>2eml$;G+^BEw|V+OHJ1g(@zYHe{EgSkPDmG!B^Y< z5;Y?=quHpPeSdEA)8chaYkp~)dbscT@OCfFI~7-WQklf_yDn7oncQdLfscIme_$JX zrc7khI#oXj`s^*4JYE_5Q4rlRN*}S+Ota8><@K3D zcwIT_b>B(*-fVo~TGh_w+Ar!p;@3h2j^J@&;a;N2YwR%~9(%lZ5+`4CE`rs$wE8odna;%yE7Q?A2XW#sI;d1wg zYHz(<`R#_^?n@^#U%NABY2npUwSmJz&5_nsJ3iHI_&TxeyKCDd)?GM$YO524Th2UU z+bj<|_;U5GQoLC5{nLk*HCs~YtYQF>|1F;NXa@4I~S`MDn% z$A#Z7+ib}9QtZUXYX`2VG^PLOZf(ikeV=c5eN4TMNA}HYHLF#{8w{n$0G|u`nEj~GhR)&aIeVxGHX2X!G}5q#y4%vTBsey zu2bKrGkL@0sGo{SJHmENZ|xhiphlNljZSXdY3Uuad9*Faa_w5I-=RgF%^x?ovwz1G zvrm)CKQDdv)xr}~x9omYv+Gg6g)5I1%{b+|f6}WPCcP@`-B9DqJ8kUqxf&~esbQ&g z_U4J1^}F>O9Cm#|_adtw+ZK#z^>LA_d+*-56Mt-du{y=uHzl&MjL>D<@(cBf-mQAH zR_TKcOFj&rT(R5=|7&UoQ~U3B?YrrFs>2UWHheod^qbC|+%*Euj%-$iZL*G=Ed0Fl z?F+;DU%gg(+{_L?)W2TOap{8sE#8VM*3b9-d2I$YIJsoe(oQ#bEUxVyyY|s|!{kO^ zxNBc6yk$U}W;G{%-zPk{(fi>Ebt{FwJF4l0E2P|x%AUHxOZqK#PdR_4{Dbg8HB+ze zST$_my-eY2C8Ld%}(#ZM%p2Ty5fV49FPK@>5c$S$IiH4PLF(WmeqDxiyZAhF#*Z{7XyyYjc*nLYN+%hkk$hl5!9YA3ibJ?~Y& z&vr^BS z*SCBh$QCYfTYJ-yHwG=O5Ocf(|1e=rZ2ZL!wvcSQ>U{ z(c+@Ld=@+R%Js{?oYd~ak)7$}eq{HZ!u8ajYSfzkRlz=)n>JKe#`rgDdBnY;bGc7; zkGnQ~Z2yQ#-JBm48eBTD^4*`8wFqok@%Wq*o7$7Tk(N2%bv-rth~M0g{>^-TTv@jGyfr_rT@*U->vG49Rj>QCXs(t!v2xvsQO$lf z_$5ABef*^d%;Fc;zG#uu5$BuT*z=&{tdHr-Lwc@d-_MxrX*0Ftwl%j5*SAh;pc_|G zJ@c76`Qpe1vg_cbiwnoT+S78Y-=I=~cShHK;%LD-)U9~&?!Y%%F8{t+opvEhuXJ7? z_|50efL6sPRe$jAg9+x1b=69v8z6Mi%x%5p^M_Nun$&jByiLuNM;kxymr*O!}AUXM5+;Cy8`=7FS#`NgGr}{ z)_r5cXII<#O&e@q(DZJb*zG$?SL)kf%BHAVNyQ}o^qR}(I*%N8snylC9bfNs({|l= zwmqZ9_+~qzHni*8#1`|Q)yhH}TF)t5uxRb25lgR>?-uz=|GjT_`?~p;3&$t4?W>j& zx~;7?VojYN&bFCd;;m*ai>4`EXTLM2%lOJZ)ArUluyAU0nl!HF074#&8e3ZNE!KTq zVzKJurcEwq?=hjz7tKGq>+|~l@$(DZl@>2*-Ra_!w$*<=*le*nyj`osbI15^UC#>+ zNf0^ChZW{7g}$BsrUguX6UX_fk64C7GT-yFHT|UsOS@CTCMoA{N#V&(%ec+@+d3{) zN%f(nO*eeg&p1yNk+~yxBOX@dH#bXvY|HR9$N|?&c%1aA*+-V-RJ}gK=P~tBRZ;TK z6jYwt(qz-@*C=met=43UnmgI&%M5Um>`Slk$8Uc+j`;aZW($Xov12I zG&bH9lW~uv2}Y>`cY~Vm$@d?`RlVeY*d{)SxGJQY45?L5``&D~oj=1l&jombjI@w& z^{#5X(prkVdan3YD?OWDxWeTUryWULfM_`&6-W+`zaDVWPrg&-eCQQ5B)KJ<(USdK zw&at&6+J;|I^PHBzKZNIr3tL!ZjocWQ!lB4pmIzUJS$u3tkn0WU=C=#`(%htXSClH zMLTEm&@%gs1x>`k~=tvroi>l?`b9w0ruS)&fP{H)v5gHi8)+r5AW$=59t&tEvzB zw5>rFSh|%LsE(I7##!!(T%=9UGw3pwks*nKm}z5Bfsu(8DhslZsYax;2R@t~_-?VJ zi|M8(ip}Dad4GP#a2>1pQ${LhdaSCY#Ei^viL+*U!qUymd^AQd@}kNM7K!sUtB-Yo z)Tm~fluSLx&KIYAwV%SnsVJ+GF)foyszf<~W%4{!-$Gp>3U*$vNIW|mAy6h8MBbog z>OD%TGri|$GVm|@s?U=fzch%byz!C$#mH-EzUfzsgk3IVymCE5@;wutTv)wT(9rf1 z=QU?KO?O;qb)w+?vlaF$wdpKTD}qNl!K0 z@}x#YW!{axBHob#xDbz8{BSz=!1_%K|4cidruV3f$&^=RiQ^O=M0tNBi||;GeMIS@ z59KPYuvSmI<`cWi^uXti=dp0Cc=dB!!fElc*zT73V9DB#?-kiw>_(ce?>bM*A}7S) zOG?y%W+6w6cFW~7>*Y2d4s|8H;<525r9vLxXAgp}l}L>^LHCw1RQ%cFk;TkF{%WWy zE0n3so@r7a>qXTe>ZoMp4F&_3&dH+Uk*MX(xlbMs|B-sGlzb>W4pt?OKH^MvK}=8g z5uzSj9(v>lA-vdWF((PkPm*P7O|?GKKT^$p?1_#>Urs!FD?$xSzYt$`N7-Eo2aBeu zf?==YaroMC1ip9?){k-SK-W9jeD~D)NJ5zttt_ z-cdufW0%c|?2h93VzuXD(EY20@9ZtVC+I|6rW$p%PDaFSX1i_)IA>3mFHGuY&Ug}9 zSQp@bL4W(n!;oct55}Abl~;7V6Sw>2?bQwjJ~E!WbFE*d{Xz(R@WvupO-+9if8Vq> z)?(FJFZl6+xBS&FI;?#u15`*lNRE?=&__CbrXoUsQyDMl1leFz<&1;%^i)#R@{f$N zpWb*m;QTdxsHar1bJqfH`5oqr8R0(Htfh-zeP~NR8{|nzYaM$yHS|nrsquorK@XSO zLXVgV+`AQY?SAPL{apuNx$5`VU%HucvCjAPPaKJQm0s#o2A= z9nuC6hC*>3kz!oou!9Ma~I6kMwCa zL%K=I6iuAo2pX3ul%t|x4ASwcsOJrW?TMB1=y;1K?1{%uzHdvDoat#9{uVlUtlcBa znWEb0df-hVTDh~mTc34D%gKS)!qTn;8m0S^K{nczNI?j`&GAg&JuqC2DwCLc->yGb z)FLaHs_nXuIUVlmJr;V?SAAj=IoT6>*K$uzud^~JQE$Rew4K*8V!b27W+iP&3#yukC zcnZ4HW=zj0c99Cs`=)@x@6_E|^~t6l=KBgwXC>05TV~H)Df2*u_*6Z2(N^wiLGFk; z>NCl3h7U`F~gzO;s8j5lNi zJb74NJkJ|bt;Nnij!G5J`oyLlDRnC(^ua@$E%nxWJ&eVYE?WxJc@$V5;w9$$mblgzf=< z**Bx)C05v!W(caX>9NDJl<)V5n#=-AWg!pE>gH0ulAxps9D8i$d0D+<6R2dT97kb7 z;yBUHjOj-I#T_yQ0j59tQ+1Jw=0Qsz1l=voGVVvH(uV^cdoC9IrP!hN>bW4*{KWb2 zTE-39*I{|cJ(%{&6YgH(R#xO{XoFDK`4ZyslIs#ICvAZa&%99w9uC}M3n_g6LA4)z zF1s&P*KM9Px3zet+=E-8sdoj--ZzpTMpB>qsp^#)kq_@Y`Jt|^NYn-8Nyv^Esqs9= zX~~v&*Ky4W7R#extMd4Mh9T&&uh}Fo?Ii0WEFwKi3qNtNYyNHDE~L|Q_cMGVAF#(H z-S;tn!&+GXn7_u~t)==OV+GDz#p`JnY4+!t=Zgj1<1FHF|Jy$5!&2O>5}vz{sdIZ} z0mhg`;BNUSVE+ib;+q+ELN2?-;z$p=u3UE-%=c{60&*Cnq&H{7d74+`alU+ z9{O432bspmeavw=GcMh}t8|*A*oTi2?AQGDoSF{m#rnVq#u{f}yw+)-Ngiz;6pT6p zlcfv+6wj5Qi8C|p4{5q=dlIQWip-=66*6THS82z0wOLby=RWRpU(ScGLC<2)o1*icrQFdCx{+ZQWXy;zzoIvUN^$LYmxocJ9b05B~waM zxQx32M%iP?%rIqU!c-3Gq%6nlm>j9kl!du{tq-V^H{7s3ei&p{`E9K!rJKt?^kbq< zlHz~1q{B3!ivAMgi?QzS$5zk- z(TgzYONO*?eJ0HPI_lAk4@ahle`7}6?kGKD`gG1Cae7fycnU^#nq=o8NuLjAjW9eC zo)`q48VsIH3pK+qvs$|@yy;W}GK=2G^c58)BMcNSrH< zU4}kgq-jbuYLuxdGJ~Z*34$3$$y7|K?vNq|Q>KSYcRn^{$Q-AFE~w5Kcr%cvgG8K? z0Xkab^)}EK6dxsl+}KouhjLO-=pB;)ibn>^{Bnf#N#jjy20ZBkzn_R4g`)bp5n>p^Mkn2F$8+*N;R_zOiMg^XkaAZftMT zXjQ*t$9$g~?Nf`rNl3&m^0um^;@RNqr&$t`iIWw?L}876kxq3^zCz_Mnt#QkESn(Iwfh24L^4Serw=60KPvs0|GMj^*&#|R&@KMy2JPYD`DY#+8ltoZl>uRlN%owO zM4Law;&XH;rIv9;mcQgbz4a2yo!41$--_h)jm!!;H3AFaa7Sv`|MG>%^e_EWZ&4Sk^3(elnrtHFV zWZ>uAXVxckLRDWVM$^;SLq1&*GaiEXcxr7`~{Tin^ijU!|vok+;oDe9y0dw%rL zSZ(MRyE0w(5*fw6NlFGDdvKeO;(>&$b!0{n5^|l8vwk~52r&>6eC?(8idJcH^WM-o zajf3?je$Iu5M1H0B`tB#Id^Em!bh5QzA!K#F}!%8NxFc*XcE;ZJhVvM^jo2=CLJ}@ z4pQsf{nY2BIx-Wh`hL=<|B`|sZ~K&wFS@*1k^0ox)Lybl%-DiLi%0d3gcf+xYt;LV zU#S@P#wy2y=p#{8@3BC&fMhe4_qq3S%Q^k3)Rm6M9>MMoE zUy1RJER?3J7`fLmeowI~78c&2K=+4juF;J@R9tO*S!`I#U^Lz?eJhSvg(H)FyY{mO zuuOf_=-va1f$wLCkV>Q*5`6p5S{?SzhV@qyMn7yDRHI99of6GnoiEzfyxS!F(yid= zYQ>%5{+ul@bm+HJxDPXfBXl(fq}T1S;qhRhKAkOuKj?YpSa@X}^`T)`zJj((i>a>I*T?6HNG6hHQH+LbVM+SPu9 z2rvu4pP#zJr?8U{@ovK=|6j=T=s(H7T{$UE`&({jfW@PS?9axHe(3(0_L+mRso!c7 zU^@uiU!jGXeMA1_5t~!vqmuK;-*pVg!$z2uI_J5Z^=tX6{f=>@A@lD{Uo3}=6FGIR zYj(c7Q09wVzU0c@)$g86ef`MBle2r>yRh?lLFwx1=Oq}pvh(ECxvhQx&@8`fRsG#8 zJ@mpMvj2s52K`R@c+lOHQ>ulnApm{B85>3zrvT2OPz8B3ENOI)h*NdM5_E#Ri}s{iExZ%JHI z^l4I%o+x&d6o-fdq!g9569PPC92lc$YcdW_-;!o*8EMYw*MCEq1;MYK_zVw=S$0-r zhgEI%(9=wMoBGf+$fZk9F8y&i;ma^F_2$=<`(_4Mozuu;m}iWxKqFmeZ+N~0{V*$2 zExIfxR2~?d=>Z2j>ztW&-mp=If>nAI>MVbJTA4V{$EhrnNssh)V`?l@+2!p#6i`}S zPu)y=`a7Q$K2a#^m#B7CY2jZ;+i#nGGk`2tVJ_?RX&3&B^s^Oq3@ZgCu7Z*&z(WYn zqYHCnr~W!T)!IbAY5u1T_}#dAt-wE9n%taaAH|4y>&)3xTfnOyocwLYCs`+yYO z2MB!LM)P{>@l~ke%YV(-3VEh$9E&deD;67+uz~4 z4SkV5^WAgq%$0M}`McxlCE03?eA&L}V#_t|iFZfNXy`h$>&k#qXPsrLknd-|`}&w= zBZo|G+Oa;FKJd-1pKtbmRHfSJPR*)J{-k=$n7g($<9-=4q1VdSTdm_W{^(@a|J{F? zWMl2$T-w}Tf5(LU+Kgx8=9eu5*zjPrr}48l(*^IjUz>7yV*RcV<{{^UpGjVBXI5ZI z>$_%eNO^S%o;TMZr1}o&&pZ*<+yAPDUJV*#xu>y!uJ~zO zQ*#SnUsrRmdqdO4SJf8xN5ypf=w;}ToCggH!U<{l%7|TL=sP|K_<%YavTg)kB;B;- ziPV`R3E5Z5c@-E!NYP{D_FKcccH3CF^-isqn=dEbKf&}l840Os6QIyZBhG$FNc8Ck z=wrSIoO18V~i`tZBXv*62la#`b^52|1%*zy>1{ip1p`35DNr`k?f|)^A z|FlV8fTR8~*5VvB>kU#NGk>^+B*q4CqCY3-xvJFyYuplz)SBQY6f0dYfR;3=zWT}~ z4CeR({(|DK=nV!@lBx#l9KN5uEshjvL0QLmCnqu*DcC6Q=OY!+o1kL9s31#FaY?3% z&S|U17m8-QXpZ#L#~V7xdXa|7&M1>fXAM&XsaTl5LG}T|NdKWMloaiarHEJ-%<*-Y zAeEKd}%!gJzQd%-mK{N@1MK(m+^_hCbAo$6=Igs-&9`fcQL!c!XOhCD_ z)usdlC-^|i1fgG96Q>VXXoOjjIK@;_Ure&aOEyz=TENCDHm-PB0AH~{<3p+VM7j6s@Z30L24i+H>8O%mnK+=WM!Y0vf&`EkK2n9vi zKxsw25HDJ-7F|SzNK0rW>$^T!&{?EXJk#kZ3*x!pLXs3*Q4kIG#1ca}zC^G^;A40` z(8L4=%r`~x5sBRP777(*8|M=dM0vi5ztw+TXMg{Gdu6V0u*s+ZQ^dG1$zDpKk&3O5 zMPk&fP$~*KCfIcry&{Pgs_@$MBM^OBi1<4iKJ$+x7_YNv9z zgZXINM&)vfP|9Yf23c`XhmBIBVygHOGaGzm(O5KYND)5+wbWrtjG=L*Wv3Y2i1Slq zYnbdSmnss@>->U*5H30-bby)GJHy)yh5c$6c_!H+MQ5bo(5e=>!WNxCd0i=qH-zg2 zJuh3kD%4(F*2!V|Z~Ab>LvQ;jetw)STJP&CiN(3rYn0&XQO=@aLZRZxdf6&9RrErP zY=@EN>&sgN+3rI{djlpY!40X=Xy%&9)ST#J;Q@8M>v&v5-2SZ1(TlB? zyS^#2Uo@na<`voR-hJ-{D~VS_OZdo^f|hxR`GEVknBIIKSSOgS-@9ioa`&j&`ei@$ zNPOXU`C>^WkB1-R*}c%fQ@6^yDynNLT)mmrlNb6O`b8teNunj=d5#9(5LeVP7#?2* z`_F2jDL!{UTSz4xW3pMtaaJaPYAYlK)uhoPOG)59KIZSZnoW74mvHatSE&9X579Hd z;=q0O`Twy#&8+-WfxEPI<{tV^uMvYTQz7qc{A0iIFYD|yQAM(C?ORFc+$%EWaSFn=ivqYb10{zvy-^T>}>U}Q* zo>ORN9Q48;8S)?f7Yz|5>@OW6r=5ECxvtR4%L>#zcwOdLp3#}a?HVCLRvBtV6bIh_UVnB4CAh1SR!Dwcn2BC4kHi5KKh3FDae^ap4 zsB;LMyZEkpoof3VB-u9V-Pe|GWqP@&X!ebhLqwlgr@cr~>exV|&1DURZ&+$v4x3}W z(wiAMr{KS_>Dsj3rDn{Ogvi@YKD^OUDd4MNRL6DMXBVZpZ-*=Kf~Cqu9c7MID;276 zMU4mIj~Q3?b6)NI*&vG?m!#JPB;F6@ueFpcdZTB|++K@(saY!VsFHCj>1i))X|P_Y z(t!?}{%$N^>v*WHkY1D(PN{fx?EtS zwlR)Zj8fyDbPg-ErW>Uj?=EY@g<=N_6hT+o-oaiJ|c_t{Bw^^CW9TE?A6WGP8 zURq3EE!OjcLQbmJQ?aPTi$yqJola*eN)=Y7c6{(6qeqxeQPbIy9e`Cb7jgg$m&vq{Wd@XAULeA?ce=0_2SGe=> z)S!zy#@jxb(qm<(rO=fb#z=A(CM`}6J$c({Rsu~v;A92Zn{CliYBf;FN9GIu+|Xom z3gr%Sz7C7d$lFO2qg8erLj?Az3 zXJTe%Fj~d!&A@gpC>A^`>ehCrVa^O*=kG2Djz{CYydYbwz31B+-$<6?ImWN)AI|BA z{&+1=fWg-EBv-^!LQourdHtZl=7HZZ>-<+gVpE{iBJH#8zNB+hJ29&tWfn!p^NNos zioT*P;Ih>0CTG@1NyFb0z*H5?ObudoN17PMi^Y~yq)e7sn5uyLaJAU+MDRaFh1W4E zNu14bTb|#{F4GIqoX?~EdxmeTYZx~t#-K`6YPzS)6+b6um-m8i-R7&&6WUZ0I93iE zO=cDZo;TP7BWJ&(vpiuz0pK<8%%ayto0fOgM;c^3_@Y!9xPdg@q!;TPH=8+aceUOq zi$N{cz%65t0bKlwH+j0=IQ7TnJ)^EO<1}fqe$Zk`5faET##=Rp2AG*$h`tbX!wLgD!(U0RV9eMwly@&_FU@=R2ID2ghw7Nf3W#P46>zviVVhxBKrhgcSiUErzu<1!_Es+-!c2u+e!IimpOqc;CzcwVaH4s`spZ!+rwEG zLN@C;6XX{xI;#=5b`reaDt0d1!tB(CnV5-Z2JT2hCV{LmlOl2^$v};|31sm(CL23n zJX!4rqc!|I5M0P1^17qtO<$4TbNa^x@l_79e&4HQ_Eh8e^<85mvsDVWuWP1b-*)A0 zg!_QEsBd*HCM=rQrORe(N=>7mUZ>DSyDI)fYU*Sg|Ni@`w6{!owa8twB7A2pd6y_@ zJ`6nklfGeGph?dPiB7R*qg6zXbm&Z9EwVKDLFo7;CxUY?T8r$d9OWdzsXS@bq} z6n}y=RjeW_-if@;@#fZga`zX9KEg;Zlu>q&zD#En=^$;XsHjBuRfwKHZp=q7U*0Ei z6-5{0Vk~A6Mw5j%&nC0MZ#0+$j#n&pMPgsph^O}=LTfUv$$FlO0`JQtmu z7OOss(IKk4u#6Ha*5;Q!Dn(kt65#Lpk$;65dT zIBImrA*nFptRff1J=U8$O?*S}?XjhLN*Fg}Y8g&3GVZGm3w!h2OyculsO4OsP{!7At_js})=0(Q0o-EI%EDI64R2y_(Un8jq}jIx!1L>pzE1}MCb6c->_ z9h^>E2J;n6FxxRR4L2~O;^U0574VONOg83>>c%I8h4}GBZPIsQydj$L&aq)iWj)hH za@GM+C6%CPp+o>aXoC)XU6nbOg!NE|ueo@a8~gJQu3dfhwDCq%lTwD@#}r z80|)0>C>@?E+R;X)K%yB1j9>uo34^SEmxMWMB{5S=-SW_nMIeBLuLe4EYq^M%nT(% zXnjsCeiR*RIQ3{)q|W@F7Dvq`b@e$=Kn>np|z7Aa645?Wp#%*lE| zFgawa0|tbnktN6`D3ZxXlyrh52xgHJ{A8+FWm(e8vR|#}hycaHe07PSteu?AX)WPj zz)u&Y(^U{n;d0et4GJ0fvO1Ax`cTPk(H@~W^g1V)no?J5X_$(Q4$CoF)TokI!C<0 zw=W-Js8rt~6|{fNk8EKw=%Yfc43wFmx9Y5wI2(*FP7-{IQW|GBhf=3G8fFYbA&cZc z*kp0amM|DEX2q(oQpP|%4~En-xnN~IuLLMe$9#llktTx_X#+FRs0-AaDF}}C3N*6; zUyO_|FFUP9@G}IM#Q6F!k~%qkg3S(Iuo-2W-VoymD{Zt}BtB5j1g^Xi(A+QF0D6XT z4MMqy7*39}2G|un4GXE@9|F@01&vzCC<)FmiDyhS`nB{EVS-FBODsS%8hz|$zfidp zZECG(=ELj;-S{HBJ;d@VtuDjZ(FfU$c0W;Pw40SOT!6pXrk8dbi&^{dx=GLlMN1i& zQxqhL=c!V_Q9^2$<2NRs1*fXg?|#Cv$fph=?H743wo9e8Q_h2q`+}^kyT={OO<$!; zC-1Y7-+F|eB_k%0kaNE!t=#iqna|{!zHuj=n}Yc=rPuj{OgnUa{XprBOQjEAys>$& zC&2tmQlX2?)1kNofWCyyOU-l3HQSKu8WZb1C8)2Dv6Sr2m3UL={22c)pN2e!~D;QVt{5B zfT{xH7nCYz6zvR5eF1E7uxNP;T);%eYhS=qc+Rd+#xWZ&`g2s|DGvh635t;yOn#O) z&ca!YLbxu{E`81w!FZ9F0(3)dGV_%LaMT!D9fn}`#dt7z>^_`eJ9rKcY|JOj;1Gn9Ma;a(W)V23sn?$OAzf6GWZ^^$$H^=RoNTdc7G; zHa=1mW#F-7<#oQ)NX=joga2=1{=CFHftPYP^iLG0Ig;ie*(S-pe6Y-TPA}<$1s#)o zcuo|;oCN~G;S*Sd&H(wD!yx-ny#c&>#SSf?B3MWvfigH*LL@8rfGRMnfnf)Bk;S8k z;M_ZOXaF-SJhq20JJumQjg;Ss# znZnCvurd5>oC&-?gA@jQ;tUe64*+ozsNHDsH;Tq0HWMf4qIeN*(82-WZ{dB6SH!nqZsawi9aWE#t5ruFf&;aV`aJ^ zSohI1=HzuUFNeiwO8_PjO!P3tr!08%K|IK>l@C&w-t23ZWrM+~1Pj_eN7-!B2kJS& z>LX~9rVZ+HBCiBfI|U^s@L>4S2n!4+PAA%C&5HYAoG%lVuRHS`r&Am-B!ekm0MhUh zGl&AGXk%Kps1xexj^btW?hec`gtfA@GolVY~#!6O2JoG{Tq=*n-je zFpxJ{_A&WrPa%X?Ryr9tiXxOEFhOT|qJ1jI00pjZs^_hs20|5^mD&`GuS|8aq8Jno z=mk3-Y(tAIQr?G!+jxVX`ZFGKaV8isoCtJ)9tNu!oEi~~YE;mm(gMu}yUieqg0C-( zx+S7r;#daQsb-*v^1^^mf{g-i0(Z&yasVgFCT37H$6f&Ow(9rQp@s;a`WhJxRHz>l z{6PD14jU(?I>5KpXHN6Fd?x|;vMQv+~V;F(zlwZNHRrqF2^JGlMk3b2ACRpeF@rDn`<HWo6Wl zr%n^Io200Olt8_MJ73Qf3LU7J!hwY_Pl1syN_I^@@Mh60O8zxM0xVWPP8;6f3RYnZ z9LNxf>9t4m@Pov>mBDaRl!h?*Fo{>pJaiqn;{r?#z(xJ3GK_+cz>B~Xouqv-3Ff0Sts&5z+DfgBZ}zhe1~xNUwAY(o1BNkQ!452OQWydL z0yq*>r!DK5Wo_cJL3Mi8{!bbm>c6_rg3pAW@fzT&8hR#_oFdVLoKZtnqQcb~Mlu=P z1*8U_fjBG?aCq`xW*AXV=oycfh`*qs6lJSPt;JH>~-EI zo{~q;b2_FYr1wg3YoqlJ+IQyFLQ3;ZKi2v=fIdDRd+#J=+MZ7RBc|;DU_?{5V904f zz^XuiYVF01HeXTggG?|@s@mn1eS%CKj9eVGR)Ku$pf?EaApo}XoQG(y7~D4!qHyOz zxOOYG(g@6}t10jim?lcB>tdM^SD1aXN^LCc=It_Tli01h5ReoHn;bR^g>aTh~Kz47)-5wiK`E9}#d#!TF=& z3eag(6+nz+a29Pa9yT}BcGs$uc(-t<&<)xSJXoC29*i$T9Dj1b#5= z=8FN?B(@gPgOmj^^C}=C(phWgv2l8%_~y`%S#X&$vUV^yApCj>>6a(JR5v1j=lL`W`Qt za>7FtN#oJ0TT{AB8B9LCeVJ+2DA9~ik%1w@*P2H7^QtDH&|DBtY#fAGst&t=NTSvb zEr1FKgpM6KuMJxQW-iZVY92d`TIiUh`0tBa-S&Ou*Mr>%%{3Cyd7x10F=xQZf{(d_ zI;oHgOj^x0OI5ZpWfb`+l+fEQj@}d`@LXW*O9oC}&xj;}kFqTTYNYq9RAE+kRs$EnND2eC3!w#JAx{8AlyN+Np!Gyw3olpT>HKWS!z|~~K+tpMPM@WH63&aW% zR2`~nF(5hMOY>~NEHHa@qDX(DRriQY^WP*><@NjS!KfE#Rx2RAq$vW>a-efz2vT7^ z;eg5hm!n&)7wLJk@yU*i38TSVm%$_AWU#;q=PvLgjPnA@KJYMoJohg38hJ^b0|^E@Ip8BYc7ba^s6~@}7?R}LNefKHF?XoRaETY-CXrEChs*>@ z)HJCVnKqDofx^x;TcdsZP-Wse&Yzws$aCrK-T;YInF4hoyJX%_b=OKroFQ<;?103veoPg)*u2^>3OQY4rO6mST^X*u^qAd3Df zkxvBuLJT~J8}H8=Z=~X(z8`~Q_5-LRk$IS;vW-f&E-%oq=fUPwckjbhFC^ zZ{4qQ(N((E7y-s*X#MXu+luTe^L++avKg6DqQ(yw9=Uy^<>Tj!EcU0w`+`fn!emx{ zy-qBAvDbog1#fw7fZuD>9k$$7JUX)xh%r9NJi)Ulw*Vb*E}HXQX98N7xyx6oiEWjb zkx0~kUrD9viYD_7dpTkjz!%kajQk=qgV18<>OvWvzDm1&KjHyNs*)rHDr;iT`vpiF zX>s!gXz+x$L~WLap+IGh6L0&b6d;wSn?c6tKIkIBtGz{{=~>M)Y`lgOY2`)7^qj=Y zOb*>WU{pl(_6U}dOdLY{nRiLVlP<%|I&ispGi^wwSb4>w-eryrI#KGghUoX3Pik{7 zeQ1D2a0919DWW!e0FaG7qA$qYUSo;tWDjTb$TmqL7w&SpbCHGCfy~InbS07y{{sR0 zDbs{kzBG9YxfHlT3T7uKl7+`){~lj}g|&^FIBEE>c*}N!tk}itEoSoeAYxQAanPirSBh{zDv0Tp8sBAVtC@P9IluS!+z``^ zeutxO`b2OWfld5Xp9hf_OYIvl^Ab4ryzt7f;$bW7M&dYVEA?aWS+&Sz--n_r;kOFAcw z`O3QA@3cNx=6U&0aI?unKTv%59onWwiD}B6h^o+i9`GBqsc`4;L*l8x%Nva8&=`37mU|)Kh!=b1W{!Z7|k;4H%rqAJQ0=e8=jJEk9P2-&xAin5fi>u8{SG+ z&kH))o%e;Bia(Z2%DGmv-EflYm8xD}PV>`p96GN{m6O5us+xNv*V)Uu%5?+*z_8O^ zE=u#Jd22*&F6}6_H@`=bO-K$o71f_|Zr0l9Jx93nI3Q}8Ks+?321Eq|s!rFP%S%-b zZFx@PN9SoEQL|Z*(*`xWWon+bc?*M3;05yL7;iXVo8KIh+ahmn56vM=Rr5K=IRTZ} zZ{+lYs*-%9dJpp*JcTE#d2T!;N_MHaS58%*?X0yY*EUUnQXfANR^t~FWNX8G&N6Syp%)5 z+JaeLalmRUw{v+{;8Z>WV9ctjHf~jtyTF#iTQz?tc~>$H=5j4RnIvyWYt&N$tE$YC zr!R8HhpOhQFXxY_dFWEfJ!76a(ADweB|Co?dh@`LQ`Ouaf=i@d|Df2Be9clx{#7V1 zE%E#FYM!Uro%I)ZLCr46hjau3VLzHzH@uN1uASad`;z-qiAS_*}c=xb`%0 zd3y+c?HA@5?Rk_Efg5=UP)VLKl21$K_KG)*3!`~Ol;oXfa%ZPJsynB#xf(Zb8trGj zm1lx|iqm;!>}=BX4|4AMUCuj>o?1Z0m_sG<%Mx*1(`aeE=4pT4Rp>lp11~fcOS1=GUR{-)G;3B+^Vk7-21Xw3n`bK7J^7owV@xGD z+_Ot;`c2-EMe>S4lNDJ-Xab%?L0&bVSLD^FM^T>nIgd)tH=19xP7A5L3rx8U%{x0k zH3MmHz2AR|qc2(o&SKB;!fIaTX1Bn0GvAc{jOxfsh5FPSkY8q1HJ{GR?Zl@E%VW6a zz3?UwpX6iJ;o?t?NA)T7lBf4_i_1G#=4nr!8A8orG|AT#d=RF37y9y>VW*<=iW(tL ztIxdsjWg&oiZDC8sRr0u>a)g`^!Qw``jifOS{Xkpeos9HLTa_o_2#!Up6v$pd5@Bw ztu3z>%;8%8zSEBKJD##D^4d1}hlTpA?0eB;>3KurM8Ri?A!LKKz=f5bzwmj?^PZdJ zeow=y=2JJ>OwAy7Y&Wf0*+Gll%H~eV>c@*64zEMVm^VN4^SM`tOut>+zU@*qHO2oI zF07Q-h^DSzXL{Mt@WIr6S2*!NKlGGWvV#b~QF+er?LC(8}O?Is&) zPaU)(Y0!r+ui9GM!UxmuMet zSbgz{#T^)Y)m7+z#Nom3_5Iwp_Ufb7f;aD-<3}vB^K4b^N}KnA<4Tqd8N%M~AJUkR z5pQWEYm1=TzRzg`$&3Lr`@eu=%bPv*F^+I-XuRa?DaMDuiq-&?2{~0--LJ+I zvWT>WySL(!v#Wo|5t5d7Vl&*!AZ`--FwX>&>pf-90#|*(G9aDQ{a@$MY{vMoA4+U?KXZ{(PS#Q)Oh2^btXk&x^MCwYe#&AxtZdY+O8%z)s4ndII6 zD3)seY2~>_luBOykUTwIbo<0xg_iVmbnZ<`g3beK`-Qg*E+^mMDa|6LE#i~x9n{}HrGW5LDP4Zw4uJn{2YJDEkT9*8zqSKH zRaNrmcn!zD+BEVv?CVg;p*$N$qW{Wm329LHDf}#07TdzebIeOZ&Z@uEI{!-{@=GC& zhk3%m>N!vDOYU%`X05Sh{(m9UVSkc;xh9P!yHDF=xp5>EqEo`W_t$B-ZfGaBS#B); zO<79~oU3u8^iEdb*kRrA{}hJ2cnFpF|4=;s6S(p1|K+jryG6r9Zmw%bZjyC%ZHwC7 zw5de%eixxk+xD{z{ff_k`&08MA)8>hZ2KYfqPmN4qUNl1ZAD z@`<*xC1=^3i%}}n=R6Nu{8tEz`nQ$*lp*`yL-u|We;{1YWJ{kVJ{la#HQ}E7aPIF| zQT$tK$T}^q^JjgxoE;Hahll*ncl$ehH}ysOY-ZZMu!FkGhWj!faOtwyZnS<^)`n*WW`SpCkr}$9(34o)P9Ir1{;nU$i*y7P)=7 z@B!ZDsIjTWA7#nA|N7nWRSkc0X_JQk9h2X-D$mBP`FsiBA1Zyvsy%!2tr6byj9p9e z#D3jUdmub*;ESnlwPe`a-cWeYVOcRy$^eZCt|4T!@Ot)Yw04XhpB;MtRY3E8nMx*( z^hSK#{Z|{*Y6&j9k00&5>QlMXXPxI0GIVklv$f~mWMv~jpLf6x_@eEctZzb3v2}T> zFlf_~R!G6G9wXi&W8c87TVCQXJG2e$xr0lqJTFhu#^kW^xz7QfJ=(vL^=V~X?A@aE zOZHO*jUO5-$+|vJVfdF?!bPtqhbMp4IwyOFkO@E*V`G3X6HZ5u${}dVU^4wRO;pu> z8mnjbCLg?l^lIy?d&-c`yUKc>5A^Z=kEt(T$|-ZnPulk}T8@ZoK}g^CyiNF~Y5Nx5 zuQ&FenU!pPofoy~`^v=o*nwy-f|SdDbJPScH;4X4eW<=bO#h_{9rrr<=Z_I@{VMs{ zBjwK=uTB3wGstf^u%CY4(Q7X0Irr&E?O8(J=L!F=!^@J63!ZlmmBfGC@h>{D@wb0v zWp&p7flM_vJw+y1Kh?J6Tvgrrdkd`i&;wc)Z}`{l0IE{|Eweg)HOaMQd`M4sBs zlDk90$o>{_Sp4@n$BVPhx!OKU@q6%QJ>sG1Xr8^kd3Wq&??VapZhbyZiafQm%UcduywAME9Gbn3I;s8QA;$X|*_7vZ z0bWdXUOTh|d1@>2|CK&%qmsKulB&I?^~ii%A^H9*W|8nbBU&{)UBcb`HD#p4WKewH zD~+59Rjp4Ea_7a1;W~21Ovr8PvlXn#o8_4%eEF+)A7!li)%#usJg3llIOypg8S)SQ z{~aO?^DT8e=l{fL#~Wh-fze#-Xja7QhL@4de7 zk;5f^+w_@pZ(pe%ALqUCbHZ~{cm9l!PM>96crAz=i=o}3{L^1v3#t-n`Spt^Xudoz zeNDf~_RY3}t>1X%)6O>Hx&vyxV4|i^hgLuDvDV7Q&*en99XRiKE$_)gFD@KPE(Va? z-63xe%jVdHz&zZ`^7JMRhrs!TBilHOEkesE$ zG)FDx$hsKwCK=N)!X4pW@Fs5E$~wl7j*-7AgW#o}CQ;=p1T+Phm>m4nMrz6F(i|kKk<`jSAlWxWG+y#FVM=I`4g^}uQMBn!g7sm^g z^d;K4@B4sWeLr^`zxsY2l3)2i!`cRI{AWq8%zA2=YyV;)H%>i}yKl}L%t_m8PEP0U zkI%Y9*G0SXhP5q7s3A%sQgfV{z1r6TNso`R@!nE<2}VIkMcn=4=LnIvfNr1U5vy*W zX#b$;=ehIpNZOu+Jqb_e-81)7Lb?E`U3w+rd^$@z zG`*GGZ0~=q+LhtfL}skdcKG10PtN}muf2iPCfe@24YjkbHmJ{?Lu(Q3)y10M;YHR-3z4i~ z4G7dAo3g^2vQ9Y`Lv^wlXQ^sh%eSJFSamD>UV|xX^>m0>-BA0}jw)o^Q_H8*zRx15 z^7r}Tr~W`H{Xh`%9eE>GYMgEKl!y6k&YW1`r!4lW6+kMzapkvrQTgppm8;E}@Ll}?5KSkE$iP%{_ptDI{w!s|G#%f**w!= zzRTZwbRjTNcg>p8yAlj!Uy^Pm`f9VS*LcWUA5DW9d|ubzUICb2pz#OH_$_G{UUjNZ z9KTr37Yw+aZu=qRVaW}4qbgh}rtB>$Tzm4+;qf7BwC{^bL^;d(nmT-Iq*cI=%v8ft za&=n1w>a|qf@-ci$+`b%Hj@9n?q6MNDf)u=gIN|j?kjnJJIVR#wwk-sq~LRm`i=|Q zJoX{?o*08Ok4_#*ODA*ZbcT!}UhBTi;yT&~kyih2B>tmEBl_(>EQ6~iM+ravUCo&^ z;x8#?!l)Nr{HtT!m_PE=;Ws>W`*%M6_-{T2PJ3QM|H20MdoS0`{y)@Ln)jUJLI27W z{OQvRcx3*+t*_fH`h9$Bv-l5G((3Eq*I8bbXg3pjKo5$qz*t?NFPArNeGroi4PeG zIDy1KehR57L&**>N_9|9W9`dmm`Wx#Vu#v9FfYR4o&fYD0xm%GSW5`hBS4LQSsya& z44}s^ng)~NP%49Z1QG-|2+juhiI9szE*q(eApz+PhM3w=TVir3&rGWNNpo~EBI=MzWlYzv6-w0&dLvpBzAw>+! zrrO6T@s5DCdq2Ub@sHt09BCGR1Y*p(=luev3Bw9dn;j#2zl@m&=RdQYtKa%#zw>S* zq}AI>{#SG44JDE{;M_Z_t@tfP;GZ|#2)Jc}Hikp%1riIq4~2d(LQDem%4WtvDvgUA zDNb|4i8-Z>HiD|7W|)S$!GsxOJ#qUtvIwNyfd1RHLEk2r5RdLEYHlIJzFxci@Z7b z-B`5{Zlsmi+k#6 zF7W@7k1vuY_P323?z!c$=a&Dhq~zx3Ucfmnr{!J`>_wc)mAd@glb`aqe!cBdzEYky z3fZ6LAbWxQ{}y?s;7JuCL%`>%K_);h8}Q^E;!DOEjB`#*}N|RVJ(g$4yZ!}83@Gn2!TYPt-|5@3qz~LTY~}WFhEPS^%8GwX5_g! zN6NFuj+iAo5?V|?NI#E$zbT7{oc39Dzy+ZMZ3W5OcNU0=Lau4|jXA3*S{@6oQZc+M zV;_hR!>|q`%D)cz<6ZC3W!X<0QsG@S&}W5N4#zc0O(>5Fl&#fi1uFe9H^ucY4r2JM z7CA6v$Qm8Ej!yIp1S$`ynm1uUi;@=jPY?%gDE!DE8YiGG< z_AGUI!i?AW%YLQcXL3B-1+zVY%9x<)mXTR^U&S!=A#=$@W|3f;?aPUJwomfh!(qTbS8&*`Qt_Nfp5^rLvyEhV`-^G+lI=hE;QmXL^iOQdw%*CB+aV|_$-S5avuBxO7b7+%fEuE>{w*D;I&9W zKMPM?{Yq^SGpn2??k~q__`zHaf8I~S1;=Z+#3vfQ-dDpv&d~6Z<{GZnO2dW^HQZ{l zhA-jWmj-G$y`zS^cGmFAFKf8n1PxD^qv51?HGF3*mW+5yCk@vgq2YnAYIyhn4a=i6 zJQixC?$%h>jwTxZwxxzo_tD~;w%5YrMr*h#mhj$om>jdq=~MI%s%K zFAaB|r(x?54e!Brz0(`pI8npDe5_#+^Df1(1!-vBQ^TW?!ur^j{+Me8=6ZzqN(&9o z#rC$s^erc8Scfz?k>?}3V_Oh6dQHQxBOW_a!^^Sl{V;t4|740r|E8TXY)vToLQth!pCPW+%3~0_G}; zC6B-wQ&5&1Sj*xbm=`(x#W3tU4DZ5{3$;bQA{cG24$uk zQrr9kl(#M#E;C5Ok^MF4s{5LTD`CnHu#JnbgcNMWGVJAe?49os&qYbD+7+b&`FR%0 zcc6?FK)f08Zfx%mq}+}8HSEncI1=AO+yZ-+L&*`|)^IR#>@{qo|3?~DP=;5i!Z*+s*xT!WLkxl)t&wbicX_0zC63aES6yBhv^kcRJK zY5itu_~gqPKGRXdpZC)6Ygpcyu^JA>GrrW#Heui-XrG%U^2@bFd|uIep)u7($4xGuKn=bjq=7~636RV;V3hMQrn zWv~q&AT{I-Z>{E}$xXeXqe+QKPf5fV5PyNS>9PK;SZXa-Kyr6%^Bm-4?`|3vyK8uS z4-J3mr2*U05Bb={btq+wE6jZW>*tZHM#K)p?;~}ihif=)B=Ud0hCdvo;hETiZxDa{ zI))L?M|=#qU3{E|ry)LqI1)KG1hI3XhR039yNGp@HM|0G@;k`UDH=XCO~cA`5J#X*de|y))vzh@-G)ihqoJLVOSL*Vyl$%+HoghWebk?|044 z%dpRxyX%PTQ3k!*1bKzBw*>p;8|)WxocuF=!i61Jpm8?C0(?Qo=}X*dKacJ|cbDaubW$R_XZ zI#>(0$NsjxrQxxt4WlsETeG$FEs^gviH@ra&={ZxKpfR^{o(wg_;^x+lp!S%mqJ_` zaROrPZX(7fVtgXRX}Bz6Ems_g#W?Mpn01cBa~$T@&Ly+XrLxYYv(5=w=Q3I6vRUWE ztg}{DcBllFS32ulChMGgCoZe*Sj>67+E44lvH*$Lnli}2c%-Hb z)>ax*l)<0YuZh^=vdH0hq_`|nR2oYui>b6pO{5;VpeU5S*g67epZ7# zt7R&KXr$dOND3kC|95)ffAaoMak_%5 z3CLPgfMx*A0a^gG1ZV}&8lVlpO8{*F+5xl&=m5|Wpc6o6fGz-C0lEQn2j~ItGC)s& zUI4w_zqLobx;8FL|CPl4h(mrSAdljZ2Z>n9|9bz=_kJw&P8>iyKuLg70HpyE0LlQA z1xN%a2T&fM0zgH8N&uAsssL04s0L6SpawuqfLZ{x0qOwM1z^cgUh@B>;J;bh{;ya5 z*DL?im;*qYy#_E4U=YAyfFS@w0fqq#2N(e`5?~a-Xn@xN#sG{3cmv=~fN=oh0VV)U z1b7Qz62RL4lif{79k4W;gRffx{oN7({Sdc|Z4;}(=drtEzl^;eyFT`-*cGvBW515Q zANy_WMu@!`n-O~kueI>f*q>rg#4d->H3;p799k+Z#R>?ogV3JXU65*L>{SR|h1lfS zLlCJ}m*lmEf#a@CEZesfDkYXd? zV?gmAp?UjrXg>H4(p>r1s;YxrHHXE4U}z;GI-EQ9q<$EOvG8fo}dY}W%^ z0Sd>ppI31`up_pG!_~?RthXUj_y||b`XJ?FkxTb*RcQ~dh6Uql-q*;10~lWdSMZME z+T^lf8VVife!}Qt|*PHzK!(VZC4DI$<>|tsio}8m@OHVhIP4!Y-KdAhvN9 zmevE;M{8lb%3+OzG0lBkV_c4Wc!an%mQx*D&0#H-`fB(+tg8@GeiPSDZOH9aowHYb znR_Po$X=vjG4dy2poT}{%ASa8&VzBaa1lz^WL!}U#PoNNliITb=H7-p?1?q*#P!$? zNRRv`mg(g`a@*VjSDi8Md|W4-gSfwUMcIpSrEwy1<9+17X~flBCU+I=kCU^rS2LOW z2c*x4d|!%dny0WA6WV64J2Q79l(tET_aHusxL79*yO1Llv4tMQy-+?5A`V6#nX%9A zA?|=Nolx4Apafh-{0jDZEaJt8YoLtI?32Bg%G~yx6; zb<0kO&mcD7diOx&Z(&^FJ%&>L9ftjJoXkRe5Bq5l_LCcN=`lFEun*26Zi~F`gm@6* z0f>hpZjL?o4&t?l_anZCYvL_&P5e0G09^k~K<)4;Vi)2Eh>M`~mq*+Oaeu_YDCGi5 zvl($CT-SaVwZH;nzY^-68d znmcnFa8wLJ{1f7uAEBl|9oh!*PQ+&s-$h&ppQju|EF6 zY{hNtueRX9YnqjJCt<(V$9sKUu9Q^N)epLPb$Uk9ODGSW@xG>QnR_$J-W#2=H7|1y z#}RQKTjY!3m8cPpVo!aAqq@;dZ;7)|LxU9J)2smlvb8lsZOtL(5ety|T9T%{Irj&% zy}2WjYM?fmigU~k@ADGW_1|JUuA-#1aYZCg!1~_9=Nm_nve%F@Rw^4a_ZZY(p*V6n z;RqRqa&T_0cdV~ZS&C09n&3PoVu?RvDP>WIMB}Ipz&fwsXg-2H;KovVy6Pi%DKDAS z1XKTjqq3o^T(Xzr(^DQ{xhhIRYnO zp1^a`6_-9~ zjH_AFj?UThJ9B&HX3z4>JsaEG!!uRU9Z2g4 zS8uco81z8XlrgSVNvBb}x5An~Lk{mn+-l*IAK-2pgWs84=41)0e604%i++6(7V?&_d~2ctgH zLL7Ag70JKk{iO2qWdT|U6au1PJCq8*s;WwfJ);n?kmT=;BiHfC-k za``Q!+8^804eK0)Scf$<#y-B{ed3NSXo&S!M%fjx=0Z4XSEB6dvCMenauIB6GPc!# zR_T{BvoUjL;y8R2XW<1{dqu3*h4r39Tp7y@?3s<3`#qHY-e}9Edo3c2xq>yeK>6K- zR_V#!*|schEEU6KO8*Z4tJeM?7u1SL04j8Hwd8 zSsvUWfE4#G>^%jkO&FAonY$(EmwMRpB&_vK*SB{#~-4mvioyT;C@DjI&k%a-s!l{#l5tx(X&wL#j?R^XjvAlB&5TxHDeP z-tWP1zXx*%p``rS*4v)GNiDE1&gbm+&~*K}q> zzkzFC$~^QXrXlX*y5zos8haD=jb=kL_gowWU!#r)M4i(WTU{Q#nIPoJ&uF7mL0h@` z2rXBA? zkgvN@%FZVeT$C6YNEK{}lIX4#RSu zpmf$lxon0z5?7;imO{zvj57|0{k#)YQ!GMxpJo z6HDlVGWHs3olm_UH_i&lbG+lolhV+&-R-rFD`Ly*x|HPlu^+`Y1ZWJ<6ref4Qh?foA?+}Sk@V1G(4*G$kSlasy}@NPp;xBl2XO>CctX|Pg1J7>VX_9pyhu7 zXq-~bC4qJ?2e1)fG+6KR0FEYCa|NeVchz)Pcbx^8m0aD`7oa;pr<58lnq0%x5?~a- zV1Nq%Cjc@565xIffbGz}zDeRsnk?6Jop85v6@r%5POj+^;ZC)bTCShmwOy@}YP*WT zxhBAHfUW=w06qbD4`4mO8Gw?&p^%i?uAu;FDRo?<+;!l(j%#^JUDy5Odaia!^;}B< zX1MFQHo$e`lzOfv0JT!;yMiJ9OE^a*)rZzJfO;Fa9s;a$uf@I94O|0L8n})E90524 zZ~@>rz$t(f=!<$u4P8wE>I3uuxB$==`so9J#N>w1s)nwz06~z}0MH#k1-P8j2x@KQ zst7PQxv^_(QezhZT~Y-g4d4mDQGhWiOfe|03sn> zdw??l$5Pt343It@&d#JZuFe2AleGD;jq5FFvk9yXmh_TqO!7-0Q*B-Dq_(bW0R2IZ z-2frUZCz~urUI-7xgM0%4$`!PdfT}^NNMMK43G-&D5bs2=x*;y1UL=hQvjZn4lWCf zfw};W#@Mtc_*o} zYZJgY2;Wcc0`2JnyzJt7#ofiV8(35ZWi?&_)wvso>GR{-7y_zqwbKnUC`1n?R_ zb$|~6_5)l1a0C31($#f4rJKtFv)^j~=>Vb0-CPL(lL4*+#KZl^DcxP4C3knNOzz>D znbgB|5PZ!$5cW^$0TTVPYiH8Su7SxtU8g~>JK+2dKwW_C09UnmfCHclN+k7yxv!V& z3Bb6dIxfHDUaqMryFrFN0vtaexP03*bY5 zF99wBB)fY-dK1n{1sZv6mp1M}=!3LEi~lC#SJ0xY>RO4jB(ZPm7DPYyynx?T2~7zblvpSyZTFGWLB&g0r#S??+dVmIQhXh*L? zPxLW{>$|EY_jQdF$({J-qSPQzl_(GUAS|HSXycI*Yg`S_}PN&fb5ktqP1w zA?OQqLA)I?LmQwns#%pZB(dZK zi35lSC<#yspfo@NKpB9t0Eqyjz-nqjT0su&Il1CQ}ff(dL400g`xe$YGjKLPfAbl}ddkk_U26+;LT!}%x#2{y4 zkT)^Nn;2|)47MufO!)zZvy5`z`O~VHv#h|VBRvAw+!YjgL%tf-ZGfC4CXC^ zdCOqlGMKjv<}HhP%VOTLn71tEEsJ@}V&1Ztw=Cu@i+Rgp-bBosh)jpmzkNvmdHayEx$Whpms4Iocw6ahgO@^Y{oc}ZdnqpGZG)0oZ!*MX z^_REDynW{FwK&Whk9p&>a(er)B;rz8Z!*Lo&%AFl#9@2mur+bmo7pv$%#sjq@0LPb z8gT;RGKk9}PDILLbLcCDc}r!EDqt~kgGW4DGss4A$@U3 zT^!ODhm^%3T~SC^6w(!ibVVUuQAk%5(iMetMIl{LNLLin4S-))#C>6OG>3U6o-{@n zP8tC;g>XGG5X!89cp2_ox)Hkwym)Pg)6cQ%0iT4}-4I#~e*N{>4Y3y??t1Jlh|%us z!ChL1AkXgDqZmF3xwYL)Ns#Vp?0raa4e&R3@9(dH#To(3v7e)rcMmP_9%xmcK`fw! zoPzeI2W`ieXg9jhw!4k^7}|2v(1QCMac^7)eFLpCg7#K>w5>XzEwvC=C+nl-^f6jT W3($r+h1NDabyJ1@3v+qN;^IkT8EtJ%%2s(!$&bFaK1 zbX27teXTkh!${xrKZ8`<9rN3629 zZS3x$BXT3HoZ;Bf0(;RVterPpcjm5dye<36a$^J|kx}k`l(tF?Hy$N!C0EpGGwqiLw@)Gv%XEmvAnQi%= zcfi*;W$5uB-aJ@%7P^Y)l~=WU6xE(GOzOH!1|rU!#lx_8jzL1YGRSX0S+B6k{EsAe zDLfMa%a8ze4Rh}kSAat^l@1#h>Jg_W6{Rl4{J>}NE7>r74qxx80Kv6Pw^7amdu-h2 zJ63VTCwoQh$z#Ve@UcfVhKsaen_V;NBLV#{(-s4lqC983sB7BQ`@mKcAFzp^cqB_i z(;iG$lq@#V(};3mRn||=jxv7*pXqRN;l1LchFpoA_V;`_!s7HlDaB7IuUp3 zL<}>56XSk^vjkHOHE|PVF3CW!rqQ}41Y5yj$>{JD9w(tF;A4O^?ds?xxe11+BA9fC zHwC;jtTVK=S)eI%8k+R%;|v}ETUywk3$g-5SV~zKRI#@O^eC%1x*;IFrL5`ia2Trs zu__6Qan*%+O0lmpw$yw+yjeorsE0u~S4_#R=0llJebs@v(%(O>(h5?(O&!eZoTOuM z&@Ba#gP<5xv2g&UA~_k9`X)iz$$#*fYlbXTkWwBfl>b1b6wJ%LsWH*;Q_N6y$CuSu zS@!RNxh)0R;E2h2qcCR$9EC*9do9bJwPeC%tMCgDF-`Jn3X*#02{D6I6gY8WI_GK&3EAX1<~5=xNfT00rrgBcx4)37J$w(&3cJ zE-XyI_(9`kdAAumd>s1;iJ`rk6W#MGEnmg{nKYPmWHH%k58dtPYz~Vawz$}2m^4&` zZiwpwAlWSh{`O1|I^kV1<-}snZ#=*PAnk%~)<*Zpgq)m!^#^UuS;@Wl1#XlXV`03i z4~HO?F`_*D7UEJ97H(VWe=va7EC} zvK>Zfx)W{5MhpZ;O(7 z=uciQe&8WbSu*6TS~-)-m8#h-i6$8(elaq8p*mb@4dn0glQV+UhMYY!GM)yF=Ao&G z1WHC8)qYc}KX1*_=I+rm+Y)tkUt&V##5t0)nP*DPNmiL}r3leD0YeuL>8AlIM?tdm zXHF$SbmV6V`D(K{f^_{DuxBB5$>B1Q4jI{f=2D&}r#$HBNG5Fzivu)UbpMQ`bB5;u zLY5%1FTj(Q)qx?g2@O0L0Cqhqy~!DdjMvd!bBmSOGEL%j1uyN^EDg&i?8T_r@|A2h zW2~Y68=cKX{7}q7u*8inGv-O2*RBu?6Va+^jp1>?(`ZyroT{2D!C6Umv35)fZcJSa zeN_lFy}r9_oNkI??`3Tx)rGn^L0Wc~N;Z-ywz&0)5c#u5d#`o^|A$y4StqkgxN$=m zLzl!_*s_hmF%F9HZxVRY0Mn-C*UC2O^iY5jo-m$O#3W6mlL|{UD)!kFDG%Lz~IV|=j;C1qCDq-C}?K^OCzpxjmVE>#P;!$EH{4EX1C2!L)(Ue zq9cc&x1v${NB5PrWob8AVS}?%8ZP~Df9rEc{YT^O{h1x$+Vzml9?|%Y|{sBm*@I6JGtXLcOQlASe4({a0883OGn59 z-}dvL7Z7!~01CDnTO+QE8NLw>FtL%@T3A`_Fkh?g!#5k&P@96fn9J2?9{H>Lwp)-4 z8*M&0<^xPenXLp^yQ{UUi-?aaGEcVtGf*>3;+|KVVa5V7MJRkKeLbLy3c0uO3O?K$9x2>WLBqn zTWaU@U1l~iSIpCd>KWaIAseF&Gu&`0NrZdeFSl_ReWR9w55H>_i^-J$ykCix#_`BxN(oI~RK=qAb-<;O(#iw2|nb+u3EcV^EU=?Jr6ni-eb{u$#Y>xJEQT*^HnQpllEP)X=)Y4ka5mn_4C!ZsJf>7HoUX%5$`xR zXWdZQ7@HLxCp#4Om&%YX< z%-Nw;Q@h0UAL8n7w$DO#%F4Y>9<{&sc9x4q-Br1A zBLby#r>$3e_3yS`&RaZg5Gr<_vG03+a-6Fc4}(FLt~uWw7t>KlV4ySaSL=BQook$f zVCdD$evfT=e{P7c%zu!+l)~;?@liS3-r_YDeeO>EO~v(^1vYu}eVH)*%{w{N=O@=+ zz9U#xJpFwHk2DMM^?+|`c(&Rq$4CC(S?Y&~`NBIdG1IaZG8<45;|NzhMlY{)UAW)7 zEr6P;AL~1YlWL!eMYqVW=7G3aTmGyXTy1I8jl9u<((pS0m&kTTPR@$8hc~E|l$_1V zv0da6b~g4F0V@t+zLp7D)ulo1(Qh@tA^-rPS z^BtChXVfrbjr@EGQwzr@CgZmd`pM^!Q}g16HozBW2bj71AupqI(Auo(o4aG{Twc0T z(S&{9)<;r_^9_h4NOymAsrA|0Po@L`5MEcRLuMm5XreQA2D7^x?!n&L2AAA0`bZQ?6G#O%=|)dCb-es53+31urE+m zDmjRn&@K`1>AaUZy_`0_d8o7!!=cyPrd8oFo@sl}3653WLD?<<r!P|FHH56eutc zqFB{sSDN`?89Ed$@Ab}Rt8wR&&knQV5Z>17F_XQ`b?}b6Zn;rgX8huW|8-%-AGt`L z{{pYD5pijO$a(hyQ33Y4#h4o?= z!*|SCgV9)=^EJ0anR*+p0B1c9d|s^)_*ND<$!@TXH}5JT{Q79w-ff|7i)ayMP;RWe zMz3Y>T!~qA&?WW5V6|`@U}kp#ZrzDhfbY~g=b_BGJj}V(WwYlWYmX2vg(6H1-T7Uz77# z?`#0)W7U9^7sTScT}vV$WVsxY<5Jc*!-Zz_dm_`yJj_>`#oD zAJ_P49dEr#vu$~`P=;q)PdjDM@EqVXi|yP)EmXan;B-59s`^;=HmA@@EuB|lcF`33 zGNXlGf+CSJCtQQ77#jnC;rlUGL z`i#ep3$m|MHau@V@~^D-SqXUupfcMD->&+eTU^xPv@0v`TE@@(yD`1g*0qQqZF}<0 zwo+RG{2jLi*05J{$QjCoz75za(1!jU9Zd=tN|b1-p#nJq8*-1H%tPZhM6QqWRkYB|R+Ux(Ru*A)8&d53*b65&VDCgBagypCt<@F=h= z^_5`!B76JSSEF_=c`&ZFg$!a(yGZ%76&~0puHoRr%`|_Muo!Kt2V?2%wQe<`y?kM;w+ug!AT=A~On2o){; z=wr}n##f=zZ0Hsc>6MM>;;U6_eSVl*NJ#JdVRrD?veB<_{gT;Lu0B=gN*@C6+n6!~ z`~#qBd9ERRxw5W%k^@pxgKXnx=)z^V@*hLjym~GXzGvWc%fxbjWxaDyf+Szq_xMnH zT$jyi+gUTbq+c$yYi}4u+D;UIDattH*(}w6MT0#*-aqQ?*Rh*yeySdI&3uHetoF|= zlk?H|Bcw7;+1y==qZ)rYv|QJFESa;fdJF=*$qyPC+txO^1)j>AEcdg{`JJ+?KdWg!KRErg!woFI7?kHaa+1dWVmk2hHl4l#3-8h*lFf&e%JVR zR4{A{u->r>z_((XG?$}4w;wL&YX+lBSZS>^Vd$%KbNGHWsKj^Y5AAAWe7;0hTnumP^B=yUmZq^x#ooN&&bI2h4{2X@g1V0R zRN^yqI;c^0CA&4s6N7E*;hmng{)$HtQ-fuLWWT3!t~lFfqb}v~>!i+|+je@~fx?JQ zT5_*%klKMo53P~QssFt3b?t%zCRL3P2YbXOI=Jwa@1=3G}^;c0KJmMLHf6b8d-Kg;`J=aNw>(^C zQq)gs8*(|;KE|tuf=9>)v!WQDjHd3hD^p^=Oj9U|ZWR2FNSjiB_W+z?l+|fVvz}FECX&EN5f8<96TutGN}FqE~)M1X}LXS=)3s97h~f zKHR+D4`A3D0N`Mjh;`25Cz{TW^sbiN`eD3E6pLMJ`T5>F`}iSOx3JQPE1_>WpKlE; zPw_jWa&2nQz^ok4hrVB3#oTR+Oml2NixuWo4M%kfuV~%&PONRP%?3loUr8lDOC&8k z9-vi$ZSz9n_j+7Qn`YoTbt3(6lE7+jy&2O^^ax2LJR$B@bD!~voMrG@YNJo9CjWyS2qd-MQ>HBR|4Ffxm8!oq_{;Oy$Nig4Wj&9>kGKa&8fDX%sspgwcwhs;B<*uPM z?ZQj!PFAcO9)7tlr|WL$VTq|k_sfRK646u3#`RunhrBI8>TT8iCor+pwk4OR#(9eW z#8pKx%ljWyQFuZn1Bs|+95XxPE+x03^Jdd- zY6$YR3)flwZUJ&@^@EUx;#C$p1La)TE6`Wz$F3TGC!X9v=*7Aw%n|;(MKgot&3Cy-AM3qC zdT`2)E>6E;D9N4_b9$M&{Bm<-7wh%1FB3bqcWBaw!w{@cE8({0|&??Ynl@7v=)CQFaqEFdP%U_xwz`L6Q4n4m1_EDqS zoGt|gEG36E9PhvL6%R`9`mOG%pQlWvoA4+&q^&X3OkMK1UN_a;i{0M!&lH-mv$&OR zuHD$BmIke0wq2i8&Z`aIrEf~Z*Xr9&x}LSrG~bfjZNuHWrxU1Fa+WMrBP~sIUkmRS z(%Cmm9LHK~>%KYmy2iG!Q&%dFEhFix3oRmq_!W8%oe8wCr|gbzYf9?vG+?-{zy-6< z>XuM;`k~FcP?i!1d-plOnPj!ZRVM|+S5ZCxrsX|DS825_0WB_fk?LBxAth8Zuj8f0 zgd%|FST#DO3v%(=m2zK5ck+@WDcVBQTqE-eLk2iLRJi!W-(Ij$RJr z4q;|>?7QGkG=C30%0Wl5tq2`b1R(acjJ~z&_6ft%)%`*=%CGJV*F_Q$Uhu5iVj zFSNSN8oV}_zr0K$)$}}mog72roFqm5vOT`D2vk4aTWGE7Tqzl>DSZ!nc+mpsBb=9- zQee3;hsc-*_j8)>2f@ywy-9cIszty5 zqpAt}b_Knr|hXa$C&T z1^b*RE#X4fHg*(=n1I%HtuQ&pmu>W&_raf7)+=e%r_GcFHa`HR*q!1n2ck*(6jjSj2tKK z_5vlUtlmA5tm{L-CSTO!b$GGk@?Fw|uWOP%8H<^r2Yh+T+n<_ygNEEeGx86)LaXN- zriBiJ+*0hp{g4Jgl77b|Q=mbDmt#(dP5PF!ecs*Kr>`m1A#5huo9mzS;{{D033kZ@ zSwLdv@e~(mdMT-y0t`aV8dw>?x1EjFGyeJr$3M$22uE z54PliC@!Dqv?0_)Z!O5g9T)h-tA-H9w+BcvZLv`cmdJYZ7Bvc)zE4dgYrR=U*_5HZ z)ZvWG-`*P<8K!f>-)w>7hX;+&H<>#3ZhVlVD^3xgH99>?aM%H+PB`fgX&m*(tqbHb zQ)2~^@~C05wwaV#B*a|2myEUk;GaVxLKLX>kJ+9N&LMqo-pt7_CifS~{A16Sk}(Fy zfud?DedDHUGXe8J1Fl9WV=24J*~!>BNd(w~O^Pay4>gLLlMsD`b4>~DgopcoW5#F6 ziR_I(=M90xJBH|QxiZXVVUqli`8ST~9{g(KGO|02dWn}9(mNN-509E29kRz>p#k<2 zBs-l|t)G9H7g+4*F%QRD`G&tzu$5!}9+E%dxjEcF!BNhXf*lg_B9QHfv-c#Hbec>b z+e44pUfsioOnGmhao&uKdoam5I-&m@j_tLUseUi>a0PsHRW{<84jgv5zlJ((`HlQ@ zx`Y~A(!1xIBRk|#`VJLwLWr!oBsf25?noR+ViE8v`PnUZ`-Xp=5Yzr9xc3cxK3WL$ z1N*+U`Qa2XY$bgvNDZp7e6QFuoXK*yXap+;0NOP_xrE0E z%Ht}8N*+;Qpn=3~9PRz^wme0CEb|xf+F(A%3$GDmmoR->D?NDe$DN zJx|3#GA@xt^9&1!x&EW%WyqjX84bS&gP3u?v~=c=PNdJ+rqE{0y2s@HF+9M^CLoZL z;_Hz4P1iqG5BAKr!vLi*aUOpp+u{_irhtRue~^9)3sFHsu?dwL!+YAP8wI>!*rv}xCNC|1_**yP|}c1Qf3h-vH+hwM*i z@Gq<-%%F*|&oMeIrmFXDZ;*hmcU0-93Qh$*ww^>Vz7Zw4ep(Uqo2inQf|Q0fc7f9x zypwPN0Tf;~-6(ERwG51M9MxYy6#N~Zj0jD<$2&xWDuPo_%Z(JG-;o$){T~E7i6ZKp z1SoUc$t3)OnMS+ukhld?OG871aF!&}%-TBH>41GIb2A4wdAc1<@ zsIf^HD)eYH6fqJuYB2>!DC4oazd1ApXQs%y@SW(ddc%B^DBMDRQLIViHcC;GB8CA& zy&>*UB#d!+B1rx0T^@gLa}F&D1Rjc;-GEjD3G=XJT{1k3#cd$B=G>8n?}Eh zg|KBN0C9pQQZ>uW#PNpNns1uZa$Tv!j(U*FM4(Vc0=*b}VoysSSeEW+b%>;Fad8iz z$Bcl;IKb{GF)}#$#tcokP9%zy5);8`!ze8;Np?6kMq;|hbX8iEGg*>KS*&Gy(f|cz zgVprQC?)bA>fHl);0pHRWQw#b@=T{0c=G{UOlt?CTn|Ur5tqq@%b0X>Sx1iAroqVO z>^8$=*{Wrb`z$X(5jaUHVHCPlK>+deNF5#;D%HK?`z{R3is*I*9t10PKMe$M0AWt_ z4ia9m(nF0xyzvwpy=9QDr2UufK!T$`KQZ=5+8fj?z2a6rZoqgWiF2-Td?U&H4~eZD z1b!la%?QhRU<;9$8XERp8KyF^H@h5lT~*eB=|^%4vw^mx5f*kW%77-+prLl??YKFF zg&d4@R1NSxRV-&%kQ+!6f&rA-_u45vR050aF7A zB`-nnbsnC-cLk(F(1ypn?hJF6yM0j`eX58_#(TR;1zDO2* z@`=z|1GjNeqEbA-E=f8Ot3wY_;$8zHq6GuC}(|kaq1Z7Qd^2>RXjx-T$LK7!mtdJg*LfEn{Ux2opj(nZ%ZX?pTS0 zQx9#mR5ina6EHNqK;!FcVr}(dg7IM!PH`KN4`*;dLzkkeaM>A7Z{tYRf6|1u86 zPuM6m;Y>6~{RfsL7L;kQa9_;HOKbgj*_4B$VXS&cGyp|sMKU5Ggs{#i^Cq0h>v@ts zVm+iPJSE}!#9y^K-?Dc^=wsErZN0w9k{HCO?$Ko?M{_97RpG6YDFkOtU}GNQzM?~( z1a%O&s-)I2T*R&2qkUPS8cFg2QEDM)7z496AsfBv&EN1w}<+sz%hI(_-;( zQvcBvm~AbJb=de|gG(jUzdXrgR$q8IbEzcTsaG71?c8!00j=9=t@uwzg) z{;EW+(ftKQ6)&wfn8om+SS=V?9Pp>gV_ot@!uC-Pk6m9a0FPg-qah#RZ7)_ELlq@+ zLto{~N(@97FEt^k$5Bt!Wwdd0&Y&)&4*=~419KevS56|a2S;jGlNrYRONW;l5AEIps3Jg(!k5P8^2yU+6fjJs0x~#i#Q9O?| zzehuSewt~2i1DBLAoG}{kaGXY+MBOa;bNDflJv3s!bGnI!&n^# zAWQR@%29w`Du<=A(6pCv!0>r`-YCSAyKghc!;w0O_{{474lvpqaAs}ZWdA8|i7jeA z`$*l$dY80`$<$%W9j2^diSSy0`mE-S=UF<3gXY{19TSJJA$>aLi!4r@>Rtu_c=n09 zot8PyzY%Y>7ivu_12ZI$t}(g=L zqvaC%BhGyYV^B_G6n*PqGQer{#>DjpoVe5w9slHI++^?@RF{rqSYC#YaZjVs8(TW` zI|4PkWC3;=qe?0qVQoVL9~HbdQIw*O7WHs;tc%vafF!$zWj?hYc^gVTBDt~}3AcK- z|8}I``;azFbjko%b{Zytjg)FlQk$UC@R)WdFid$K@76xx@Yq5f;yfAQcQ7YB*9>&G zL|G`Me2{9kUsY09+`>JrY=fbZq4+;~DK{?}FmyIr?{p=sOHjAUc{B(je)}c*z)@;IjW0BiwPIv@uVPZ1 zj9h>|I-wFU87v^hmclS-HK!*&FXm4Z26xmDj7&LlWrWgv4N%V9L zs$ykq@!qK-39Wi*l;Z;!sy#Wze5}!pP zfk+#qZX!tmMI?TI31d-QZ27^eL|QFsHu-8%oFhhjTroX3s8&)SZ-zYDB5AyFJCxN9 zmWB-o1`$p(#vpkgnURpZ4uXg}K5~O$qRx!v(eg>y7ho0yh+4U3S!z;UgWiZBvbt9C z7jMKtT&c7@cowX9K4VbjArJg?Q=VS#n)A)`@v{WLlr-82#Q5=h^GLdoOXMja)Tbu4 z4Wbb2lQKDvQ5$gAPh}Q(fMI}=XFz)WJXb(sjD{v-Cjo-&6kakYR7e69*C!&e#{HCbx_gZIJ5DCdM{t*T}RC@{OKYX^(nG*5p_6tCP#u z7EqK_>aAFsKHOEN(kCd*S&QXE(l|`>NGP$(D|HE#ERr!e>smqQ$p-iWwOcMoaVUu0 z0#{tzOu*y3H>Y?Wn@$p_q_dZ zP;>NkJBUg7M4VZ$x8^S-xsoutLh;V{@BP+Pws*a$g+HEvJAOUuW|V)<(6MRck~CAW z{m;jj{NzBdrYy8QMVntBim)Oxb9QKb5t1dT_)82J^8VZY{+9~=-^JVM_RT&swfjF8 zpdi(@&1&gv|6q=*P11xjfMp^#$QV1q zz1jMI78=<3kFDFO{XX>5e60nJ*F4!AW$&=!cpGp&2KQu5lgBHm|2DD=GbKt1)<7^K z8YBukB6j_!Oy0XP=Oi@&Oghq8cceGwN^Q=X(46*9af&kmiIFQZRykXh{4P%5I5M7yLsn(sJm!_ToHXZ2*w1T(JgWo$;@d7xMYEh6p+s?CsTJB?G8Nj9ZK`h?F9tXLdyp-Z4`SEt zz;>z3#CD;Rz;$uM!F2(ViE5Xbisn|8DZD=WTX;>IB$r{g`1Znd4k_YkVO}_GT%+`2 zj9F->hMLDTT4uf8S7r_F!8AMkP9VNT(T1_>jlEvUJX>~wU@G#jC+`c}C(m1vAj6L3 zJm;5vUvDOU5n`9gvGplTpxq=v;I(LykU6gHtZg*YsTDqpV6NcGG?uf-0sJgSpu9(4 zUS`ik-bVMrFGr?-I&egoV539-6CutCSx?REgS{+nRy4py2lBiSSB&#@DX9try_%+e z-M_l}Ab@a#<=0@WBU&2HKBd{szCHO{xA`^Vxzs=bNcY6}Zxg;9 z3Jk8DH0}nm$?ci>bUp-2@EdX~KnyawAR2^`>9jonFClM>TCf#m&>*Z{u;)N*7tDel zL~|P=XG3p1_tTP#Mtfahojv(1aO=36!u0&YTq~&tb~Mr`iq<()7}v?o+(3$Hd=hpE zf+n-`GG`th^xF)~7GU88H;ePTeiA+K+Rrgb$N;vHc${e4!gi4^=s~cK2XgvrZxd#E zNn7!KNjq=2N$&}xk + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::cid::CidCollection; +use super::cmaps::{CmapDecoder, CmapInfo, CodeRange}; + +pub(super) static CMAP_NAMES: &str = "83pv-RKSJ-H90ms-RKSJ-H90ms-RKSJ-V90msp-RKSJ-H90msp-RKSJ-V90pv-RKSJ-H90pv-RKSJ-VAdd-RKSJ-HAdd-RKSJ-VB5pc-HB5pc-VCNS-EUC-HCNS-EUC-VETen-B5-HETen-B5-VETenms-B5-HETenms-B5-VEUC-HEUC-VExt-RKSJ-HExt-RKSJ-VGB-EUC-HGB-EUC-VGBK-EUC-HGBK-EUC-VGBK2K-HGBK2K-VGBKp-EUC-HGBKp-EUC-VGBpc-EUC-HGBpc-EUC-VHHKscs-B5-HHKscs-B5-VIdentity-HIdentity-VKSC-EUC-HKSC-EUC-VKSCms-UHC-HKSCms-UHC-HW-HKSCms-UHC-HW-VKSCms-UHC-VKSCpc-EUC-HKSCpc-EUC-VUniCNS-UCS2-HUniCNS-UCS2-VUniCNS-UTF16-HUniCNS-UTF16-VUniCNS-UTF32-HUniCNS-UTF32-VUniCNS-UTF8-HUniCNS-UTF8-VUniGB-UCS2-HUniGB-UCS2-VUniGB-UTF16-HUniGB-UTF16-VUniGB-UTF32-HUniGB-UTF32-VUniGB-UTF8-HUniGB-UTF8-VUniJIS-UCS2-HUniJIS-UCS2-HW-HUniJIS-UCS2-HW-VUniJIS-UCS2-VUniJIS-UTF16-HUniJIS-UTF16-VUniJIS-UTF32-HUniJIS-UTF32-VUniJIS-UTF8-HUniJIS-UTF8-VUniKS-UCS2-HUniKS-UCS2-VUniKS-UTF16-HUniKS-UTF16-VUniKS-UTF32-HUniKS-UTF32-VUniKS-UTF8-HUniKS-UTF8-VV"; +pub(super) static CMAP_OFFSETS: [u16; 80] = [ + 0, 11, 22, 33, 45, 57, 68, 79, 89, 99, 105, 111, 120, 129, 138, 147, 158, 169, 174, 179, 189, + 199, 207, 215, 224, 233, 240, 247, 257, 267, 277, 287, 288, 298, 308, 318, 328, 337, 346, 357, + 371, 385, 396, 407, 418, 431, 444, 458, 472, 486, 500, 513, 526, 538, 550, 563, 576, 589, 602, + 614, 626, 639, 655, 671, 684, 698, 712, 726, 740, 753, 766, 778, 790, 803, 816, 829, 842, 854, + 866, 867, +]; + +pub(super) static CMAP_INFO: [CmapInfo; 79] = [ + CmapInfo { + ranges: 0, + range_count: 5, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 0, + patch_count: 5, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 0, + range_count: 5, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 0, + patch_count: 5, + }, + CmapInfo { + ranges: 0, + range_count: 5, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 0, + patch_count: 5, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 9, + range_count: 3, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 6, + patch_count: 4, + }, + CmapInfo { + ranges: 9, + range_count: 3, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 6, + patch_count: 4, + }, + CmapInfo { + ranges: 12, + range_count: 5, + decoder: CmapDecoder::EucTw, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 12, + range_count: 5, + decoder: CmapDecoder::EucTw, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 17, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 17, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 17, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 17, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 19, + range_count: 3, + decoder: CmapDecoder::EucJp, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 19, + range_count: 3, + decoder: CmapDecoder::EucJp, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 5, + range_count: 4, + decoder: CmapDecoder::ShiftJis, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 5, + patch_count: 1, + }, + CmapInfo { + ranges: 22, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 22, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 24, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 24, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 26, + range_count: 3, + decoder: CmapDecoder::Gb18030, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 26, + range_count: 3, + decoder: CmapDecoder::Gb18030, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 24, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 24, + range_count: 2, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 29, + range_count: 3, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 4, + }, + CmapInfo { + ranges: 29, + range_count: 3, + decoder: CmapDecoder::Gbk, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 4, + }, + CmapInfo { + ranges: 32, + range_count: 1, + decoder: CmapDecoder::JisRowCell, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 33, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 33, + range_count: 2, + decoder: CmapDecoder::Big5, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 35, + range_count: 1, + decoder: CmapDecoder::Identity, + collection: None, + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 35, + range_count: 1, + decoder: CmapDecoder::Identity, + collection: None, + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 22, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 22, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 36, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 36, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 36, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 36, + range_count: 2, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 38, + range_count: 3, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 14, + patch_count: 5, + }, + CmapInfo { + ranges: 38, + range_count: 3, + decoder: CmapDecoder::EucKr, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 14, + patch_count: 5, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Cns1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Cns1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Gb1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Gb1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Japan1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 41, + range_count: 2, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 43, + range_count: 3, + decoder: CmapDecoder::Utf16Be, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 46, + range_count: 1, + decoder: CmapDecoder::Utf32Be, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Korea1), + vertical: false, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 47, + range_count: 4, + decoder: CmapDecoder::Utf8, + collection: Some(CidCollection::Korea1), + vertical: true, + patches: 10, + patch_count: 0, + }, + CmapInfo { + ranges: 32, + range_count: 1, + decoder: CmapDecoder::JisRowCell, + collection: Some(CidCollection::Japan1), + vertical: true, + patches: 10, + patch_count: 0, + }, +]; + +pub(super) static CODE_RANGES: [CodeRange; 51] = [ + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xA0, 0x00, 0x00, 0x00], [0xDF, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xFD, 0x00, 0x00, 0x00], [0xFF, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x81, 0x40, 0x00, 0x00], [0x9F, 0xFC, 0x00, 0x00]), + CodeRange::new(2, [0xE0, 0x40, 0x00, 0x00], [0xFC, 0xFC, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xA0, 0x00, 0x00, 0x00], [0xDF, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x81, 0x40, 0x00, 0x00], [0x9F, 0xFC, 0x00, 0x00]), + CodeRange::new(2, [0xE0, 0x40, 0x00, 0x00], [0xFC, 0xFC, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xFD, 0x00, 0x00, 0x00], [0xFF, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0x40, 0x00, 0x00], [0xFC, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0xA1, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(4, [0x8E, 0xA1, 0xA1, 0xA1], [0x8E, 0xA1, 0xFE, 0xFE]), + CodeRange::new(4, [0x8E, 0xA2, 0xA1, 0xA1], [0x8E, 0xA2, 0xFE, 0xFE]), + CodeRange::new(4, [0x8E, 0xA3, 0xA1, 0xA1], [0x8E, 0xA3, 0xFE, 0xFE]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0x40, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x8E, 0xA0, 0x00, 0x00], [0x8E, 0xDF, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0xA1, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0xA1, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x81, 0x40, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x7F, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x81, 0x40, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(4, [0x81, 0x30, 0x81, 0x30], [0xFE, 0x39, 0xFE, 0x39]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xFD, 0x00, 0x00, 0x00], [0xFF, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0xA1, 0x00, 0x00], [0xFC, 0xFE, 0x00, 0x00]), + CodeRange::new(2, [0x21, 0x21, 0x00, 0x00], [0x7E, 0x7E, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x87, 0x40, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(2, [0x00, 0x00, 0x00, 0x00], [0xFF, 0xFF, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x80, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0x81, 0x41, 0x00, 0x00], [0xFE, 0xFE, 0x00, 0x00]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x84, 0x00, 0x00, 0x00]), + CodeRange::new(1, [0xFE, 0x00, 0x00, 0x00], [0xFF, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xA1, 0x41, 0x00, 0x00], [0xFD, 0xFE, 0x00, 0x00]), + CodeRange::new(2, [0x00, 0x00, 0x00, 0x00], [0xD7, 0xFF, 0x00, 0x00]), + CodeRange::new(2, [0xE0, 0x00, 0x00, 0x00], [0xFF, 0xFF, 0x00, 0x00]), + CodeRange::new(2, [0x00, 0x00, 0x00, 0x00], [0xD7, 0xFF, 0x00, 0x00]), + CodeRange::new(2, [0xE0, 0x00, 0x00, 0x00], [0xFF, 0xFF, 0x00, 0x00]), + CodeRange::new(4, [0xD8, 0x00, 0xDC, 0x00], [0xDB, 0xFF, 0xDF, 0xFF]), + CodeRange::new(4, [0x00, 0x00, 0x00, 0x00], [0x00, 0x10, 0xFF, 0xFF]), + CodeRange::new(1, [0x00, 0x00, 0x00, 0x00], [0x7F, 0x00, 0x00, 0x00]), + CodeRange::new(2, [0xC0, 0x80, 0x00, 0x00], [0xDF, 0xBF, 0x00, 0x00]), + CodeRange::new(3, [0xE0, 0x80, 0x80, 0x00], [0xEF, 0xBF, 0xBF, 0x00]), + CodeRange::new(4, [0xF0, 0x80, 0x80, 0x80], [0xF7, 0xBF, 0xBF, 0xBF]), +]; + +pub(super) static PATCH_BYTES: [u8; 19] = [ + 0x80, 0xA0, 0xFD, 0xFE, 0xFF, 0xA0, 0x80, 0xFD, 0xFE, 0xFF, 0x80, 0xFD, 0xFE, 0xFF, 0x81, 0x82, + 0x83, 0xFE, 0xFF, +]; +pub(super) static PATCH_CHARS: [char; 19] = [ + '\u{5c}', '\u{ff60}', '\u{a9}', '\u{2122}', '\u{2026}', '\u{ff60}', '\u{5c}', '\u{a9}', + '\u{2122}', '\u{2026}', '\u{fc}', '\u{a9}', '\u{2122}', '\u{2026}', '\u{20a9}', '\u{2010}', + '\u{a9}', '\u{2122}', '\u{22ef}', +]; diff --git a/crates/text-extract/src/pdf/tables/cmaps.rs b/crates/text-extract/src/pdf/tables/cmaps.rs new file mode 100644 index 00000000..d0c1e491 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/cmaps.rs @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::cid::CidCollection; +use super::cmap_data::{ + CMAP_INFO, CMAP_NAMES, CMAP_OFFSETS, CODE_RANGES, PATCH_BYTES, PATCH_CHARS, +}; +use super::names::NameTable; + +static CMAPS: NameTable = NameTable::new(CMAP_NAMES, &CMAP_OFFSETS); + +const MAX_CODE_LEN: usize = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CmapDecoder { + Identity, + Utf16Be, + Utf8, + Utf32Be, + Gbk, + Gb18030, + Big5, + EucTw, + ShiftJis, + EucJp, + JisRowCell, + EucKr, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct CodeRange { + len: u8, + low: [u8; MAX_CODE_LEN], + high: [u8; MAX_CODE_LEN], +} + +impl CodeRange { + pub(crate) const fn new(len: u8, low: [u8; MAX_CODE_LEN], high: [u8; MAX_CODE_LEN]) -> Self { + Self { len, low, high } + } + + pub(crate) fn len(self) -> usize { + usize::from(self.len) + } + + pub(crate) fn contains(self, code: &[u8]) -> bool { + code.len() == self.len() && self.accepts_prefix(code) + } + + pub(crate) fn accepts_prefix(self, prefix: &[u8]) -> bool { + prefix.len() <= self.len() + && prefix + .iter() + .zip(self.low.iter().zip(self.high.iter())) + .all(|(byte, (low, high))| (low..=high).contains(&byte)) + } +} + +pub(super) struct CmapInfo { + pub(super) ranges: u8, + pub(super) range_count: u8, + pub(super) decoder: CmapDecoder, + pub(super) collection: Option, + pub(super) vertical: bool, + pub(super) patches: u8, + pub(super) patch_count: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PredefinedCmap(usize); + +impl PredefinedCmap { + pub(crate) fn from_name(name: &[u8]) -> Option { + CMAPS.find(name).map(Self) + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn name(self) -> &'static str { + CMAPS.get(self.0).unwrap_or_default() + } + + pub(crate) fn codespace(self) -> &'static [CodeRange] { + self.info() + .and_then(|info| { + let start = usize::from(info.ranges); + CODE_RANGES.get(start..start + usize::from(info.range_count)) + }) + .unwrap_or_default() + } + + pub(crate) fn decoder(self) -> CmapDecoder { + self.info() + .map_or(CmapDecoder::Identity, |info| info.decoder) + } + + pub(crate) fn collection(self) -> Option { + self.info().and_then(|info| info.collection) + } + + pub(crate) fn is_vertical(self) -> bool { + self.info().is_some_and(|info| info.vertical) + } + + pub(crate) fn single_byte(self, byte: u8) -> Option { + let info = self.info()?; + let start = usize::from(info.patches); + let end = start + usize::from(info.patch_count); + let position = PATCH_BYTES.get(start..end)?.binary_search(&byte).ok()?; + PATCH_CHARS.get(start + position).copied() + } + + fn info(self) -> Option<&'static CmapInfo> { + CMAP_INFO.get(self.0) + } +} diff --git a/crates/text-extract/src/pdf/tables/encoding.rs b/crates/text-extract/src/pdf/tables/encoding.rs new file mode 100644 index 00000000..f8078efe --- /dev/null +++ b/crates/text-extract/src/pdf/tables/encoding.rs @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::encoding_data::{ENCODINGS, FIRST_CODE, NO_GLYPH, STD_NAMES, STD_OFFSETS, STD_UNICODE}; +use super::names::NameTable; + +pub(super) static STD_GLYPHS: NameTable = NameTable::new(STD_NAMES, &STD_OFFSETS); + +const PDFDOC_SPACING_FIRST: u8 = 0x18; +const PDFDOC_SPACING: [u16; 8] = [ + 0x02D8, 0x02C7, 0x02C6, 0x02D9, 0x02DD, 0x02DB, 0x02DA, 0x02DC, +]; +const PDFDOC_HIGH_FIRST: u8 = 0x80; +const PDFDOC_HIGH: [u16; 33] = [ + 0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044, 0x2039, 0x203A, 0x2212, 0x2030, + 0x201E, 0x201C, 0x201D, 0x2018, 0x2019, 0x201A, 0x2122, 0xFB01, 0xFB02, 0x0141, 0x0152, 0x0160, + 0x0178, 0x017D, 0x0131, 0x0142, 0x0153, 0x0161, 0x017E, 0x0000, 0x20AC, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BaseEncoding { + Standard, + WinAnsi, + MacRoman, + MacExpert, + Symbol, + ZapfDingbats, + Expert, +} + +impl BaseEncoding { + pub(crate) fn from_name(name: &[u8]) -> Option { + hashify::map!(name, BaseEncoding, + b"StandardEncoding" => BaseEncoding::Standard, + b"WinAnsiEncoding" => BaseEncoding::WinAnsi, + b"MacRomanEncoding" => BaseEncoding::MacRoman, + b"MacExpertEncoding" => BaseEncoding::MacExpert, + b"SymbolSetEncoding" => BaseEncoding::Symbol, + b"ZapfDingbatsEncoding" => BaseEncoding::ZapfDingbats, + b"ExpertEncoding" => BaseEncoding::Expert, + ) + .copied() + } + + pub(crate) fn glyph_name(self, code: u8) -> Option<&'static str> { + STD_GLYPHS.get(self.glyph_index(code)?) + } + + pub(crate) fn unicode(self, code: u8) -> Option { + std_char(self.glyph_index(code)?) + } + + fn glyph_index(self, code: u8) -> Option { + let index = *ENCODINGS + .get(self as usize)? + .get(usize::from(code.checked_sub(FIRST_CODE)?))?; + (index != NO_GLYPH).then_some(usize::from(index)) + } +} + +pub(super) fn std_glyph_char(name: &[u8]) -> Option { + std_char(STD_GLYPHS.find(name)?) +} + +fn std_char(index: usize) -> Option { + char::from_u32(u32::from(*STD_UNICODE.get(index)?)) +} + +pub(crate) fn pdfdoc_char(byte: u8) -> Option { + let mapped = match byte { + b'\t' | b'\n' | b'\r' | 0x20..=0x7E | 0xA1..=0xFF => return Some(char::from(byte)), + 0x18..=0x1F => PDFDOC_SPACING.get(usize::from(byte - PDFDOC_SPACING_FIRST)), + 0x80..=0xA0 => PDFDOC_HIGH.get(usize::from(byte - PDFDOC_HIGH_FIRST)), + _ => None, + }; + mapped + .filter(|value| **value != 0) + .and_then(|value| char::from_u32(u32::from(*value))) +} diff --git a/crates/text-extract/src/pdf/tables/encoding_data.rs b/crates/text-extract/src/pdf/tables/encoding_data.rs new file mode 100644 index 00000000..13787130 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/encoding_data.rs @@ -0,0 +1,242 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) const FIRST_CODE: u8 = 0x20; +pub(super) const NO_GLYPH: u16 = 0xFFFF; + +pub(super) static STD_NAMES: &str = "AAEAEsmallAacuteAacutesmallAbreveAcircumflexAcircumflexsmallAcutesmallAdieresisAdieresissmallAgraveAgravesmallAlphaAmacronAogonekAringAringsmallAsmallAtildeAtildesmallBBetaBrevesmallBsmallCCacuteCaronsmallCcaronCcedillaCcedillasmallCedillasmallChiCircumflexsmallCsmallDDcaronDcroatDeltaDieresissmallDotaccentsmallDsmallEEacuteEacutesmallEcaronEcircumflexEcircumflexsmallEdieresisEdieresissmallEdotaccentEgraveEgravesmallEmacronEogonekEpsilonEsmallEtaEthEthsmallEuroFFsmallGGammaGbreveGcommaaccentGravesmallGsmallHHsmallHungarumlautsmallIIacuteIacutesmallIcircumflexIcircumflexsmallIdieresisIdieresissmallIdotaccentIfrakturIgraveIgravesmallImacronIogonekIotaIsmallJJsmallKKappaKcommaaccentKsmallLLacuteLambdaLcaronLcommaaccentLslashLslashsmallLsmallMMacronsmallMsmallMuNNacuteNcaronNcommaaccentNsmallNtildeNtildesmallNuOOEOEsmallOacuteOacutesmallOcircumflexOcircumflexsmallOdieresisOdieresissmallOgoneksmallOgraveOgravesmallOhungarumlautOmacronOmegaOmicronOslashOslashsmallOsmallOtildeOtildesmallPPhiPiPsiPsmallQQsmallRRacuteRcaronRcommaaccentRfrakturRhoRingsmallRsmallSSacuteScaronScaronsmallScedillaScommaaccentSigmaSsmallTTauTcaronTcommaaccentThetaThornThornsmallTildesmallTsmallUUacuteUacutesmallUcircumflexUcircumflexsmallUdieresisUdieresissmallUgraveUgravesmallUhungarumlautUmacronUogonekUpsilonUpsilon1UringUsmallVVsmallWWsmallXXiXsmallYYacuteYacutesmallYdieresisYdieresissmallYsmallZZacuteZcaronZcaronsmallZdotaccentZetaZsmallaa1a10a100a101a102a103a104a105a106a107a108a109a11a110a111a112a117a118a119a12a120a121a122a123a124a125a126a127a128a129a13a130a131a132a133a134a135a136a137a138a139a14a140a141a142a143a144a145a146a147a148a149a15a150a151a152a153a154a155a156a157a158a159a16a160a161a162a163a164a165a166a167a168a169a17a170a171a172a173a174a175a176a177a178a179a18a180a181a182a183a184a185a186a187a188a189a19a190a191a192a193a194a195a196a197a198a199a2a20a200a201a202a203a204a205a206a21a22a23a24a25a26a27a28a29a3a30a31a32a33a34a35a36a37a38a39a4a40a41a42a43a44a45a46a47a48a49a5a50a51a52a53a54a55a56a57a58a59a6a60a61a62a63a64a65a66a67a68a69a7a70a71a72a73a74a75a76a77a78a79a8a81a82a83a84a85a86a87a88a89a9a90a91a92a93a94a95a96a97a98a99aacuteabreveacircumflexacuteadieresisaeagravealephalphaamacronampersandampersandsmallangleangleleftanglerightaogonekappleapproxequalaringarrowbotharrowdblbotharrowdbldownarrowdblleftarrowdblrightarrowdbluparrowdownarrowhorizexarrowleftarrowrightarrowuparrowvertexasciicircumasciitildeasteriskasteriskmathasuperioratatildebbackslashbarbetabraceexbraceleftbraceleftbtbraceleftmidbracelefttpbracerightbracerightbtbracerightmidbracerighttpbracketleftbracketleftbtbracketleftexbracketlefttpbracketrightbracketrightbtbracketrightexbracketrighttpbrevebrokenbarbsuperiorbulletccacutecaroncarriagereturnccaronccedillacedillacentcentinferiorcentoldstylecentsuperiorchicirclemultiplycirclepluscircumflexclubcoloncolonmonetarycommacommaaccentcommainferiorcommasuperiorcongruentcopyrightcopyrightsanscopyrightserifcurrencyddaggerdaggerdbldcarondcroatdegreedeltadiamonddieresisdividedollardollarinferiordollaroldstyledollarsuperiordotaccentdotlessidotmathdsuperioreeacuteecaronecircumflexedieresisedotaccentegraveeighteightinferioreightoldstyleeightsuperiorelementellipsisemacronemdashemptysetendasheogonekepsilonequalequivalenceesuperioretaethexclamexclamdownexclamdownsmallexclamsmallexistentialfffffifflfifiguredashfivefiveeighthsfiveinferiorfiveoldstylefivesuperiorflflorinfourfourinferiorfouroldstylefoursuperiorfractionggammagbrevegcommaaccentgermandblsgradientgravegreatergreaterequalguillemotleftguillemotrightguilsinglleftguilsinglrighthhearthungarumlauthyphenhypheninferiorhyphensuperioriiacuteicircumflexidieresisigraveimacroninfinityintegralintegralbtintegralexintegraltpintersectioniogonekiotaisuperiorjkkappakcommaaccentllacutelambdalcaronlcommaaccentlesslessequallogicalandlogicalnotlogicalorlozengelslashlsuperiormmacronminusminutemsuperiormumultiplynnacutencaronncommaaccentninenineinferiornineoldstyleninesuperiornotelementnotequalnotsubsetnsuperiorntildenunumbersignooacuteocircumflexodieresisoeogonekograveohungarumlautomacronomegaomega1omicrononeonedotenleaderoneeighthonefittedonehalfoneinferioroneoldstyleonequarteronesuperioronethirdordfeminineordmasculineoslashosuperiorotildepparagraphparenleftparenleftbtparenleftexparenleftinferiorparenleftsuperiorparenlefttpparenrightparenrightbtparenrightexparenrightinferiorparenrightsuperiorparenrighttppartialdiffpercentperiodperiodcenteredperiodinferiorperiodsuperiorperpendicularperthousandphiphi1piplusplusminusproductpropersubsetpropersupersetproportionalpsiqquestionquestiondownquestiondownsmallquestionsmallquotedblquotedblbasequotedblleftquotedblrightquoteleftquoterightquotesinglbasequotesinglerracuteradicalradicalexrcaronrcommaaccentreflexsubsetreflexsupersetregisteredregistersansregisterserifrhoringrsuperiorrupiahssacutescaronscedillascommaaccentsecondsectionsemicolonsevenseveneighthsseveninferiorsevenoldstylesevensuperiorsigmasigma1similarsixsixinferiorsixoldstylesixsuperiorslashspacespadessuperiorsterlingsuchthatsummationttautcarontcommaaccentthereforethetatheta1thornthreethreeeighthsthreeinferiorthreeoldstylethreequartersthreequartersemdashthreesuperiortildetrademarktrademarksanstrademarkseriftsuperiortwotwodotenleadertwoinferiortwooldstyletwosuperiortwothirdsuuacuteucircumflexudieresisugraveuhungarumlautumacronunderscoreunionuniversaluogonekupsilonuringvwweierstrassxxiyyacuteydieresisyenzzacutezcaronzdotaccentzerozeroinferiorzerooldstylezerosuperiorzeta"; +pub(super) static STD_OFFSETS: [u16; 804] = [ + 0, 1, 3, 10, 16, 27, 33, 44, 60, 70, 79, 93, 99, 110, 115, 122, 129, 134, 144, 150, 156, 167, + 168, 172, 182, 188, 189, 195, 205, 211, 219, 232, 244, 247, 262, 268, 269, 275, 281, 286, 299, + 313, 319, 320, 326, 337, 343, 354, 370, 379, 393, 403, 409, 420, 427, 434, 441, 447, 450, 453, + 461, 465, 466, 472, 473, 478, 484, 496, 506, 512, 513, 519, 536, 537, 543, 554, 565, 581, 590, + 604, 614, 622, 628, 639, 646, 653, 657, 663, 664, 670, 671, 676, 688, 694, 695, 701, 707, 713, + 725, 731, 742, 748, 749, 760, 766, 768, 769, 775, 781, 793, 799, 805, 816, 818, 819, 821, 828, + 834, 845, 856, 872, 881, 895, 906, 912, 923, 936, 943, 948, 955, 961, 972, 978, 984, 995, 996, + 999, 1001, 1004, 1010, 1011, 1017, 1018, 1024, 1030, 1042, 1050, 1053, 1062, 1068, 1069, 1075, + 1081, 1092, 1100, 1112, 1117, 1123, 1124, 1127, 1133, 1145, 1150, 1155, 1165, 1175, 1181, 1182, + 1188, 1199, 1210, 1226, 1235, 1249, 1255, 1266, 1279, 1286, 1293, 1300, 1308, 1313, 1319, 1320, + 1326, 1327, 1333, 1334, 1336, 1342, 1343, 1349, 1360, 1369, 1383, 1389, 1390, 1396, 1402, 1413, + 1423, 1427, 1433, 1434, 1436, 1439, 1443, 1447, 1451, 1455, 1459, 1463, 1467, 1471, 1475, 1479, + 1482, 1486, 1490, 1494, 1498, 1502, 1506, 1509, 1513, 1517, 1521, 1525, 1529, 1533, 1537, 1541, + 1545, 1549, 1552, 1556, 1560, 1564, 1568, 1572, 1576, 1580, 1584, 1588, 1592, 1595, 1599, 1603, + 1607, 1611, 1615, 1619, 1623, 1627, 1631, 1635, 1638, 1642, 1646, 1650, 1654, 1658, 1662, 1666, + 1670, 1674, 1678, 1681, 1685, 1689, 1693, 1697, 1701, 1705, 1709, 1713, 1717, 1721, 1724, 1728, + 1732, 1736, 1740, 1744, 1748, 1752, 1756, 1760, 1764, 1767, 1771, 1775, 1779, 1783, 1787, 1791, + 1795, 1799, 1803, 1807, 1810, 1814, 1818, 1822, 1826, 1830, 1834, 1838, 1842, 1846, 1850, 1852, + 1855, 1859, 1863, 1867, 1871, 1875, 1879, 1883, 1886, 1889, 1892, 1895, 1898, 1901, 1904, 1907, + 1910, 1912, 1915, 1918, 1921, 1924, 1927, 1930, 1933, 1936, 1939, 1942, 1944, 1947, 1950, 1953, + 1956, 1959, 1962, 1965, 1968, 1971, 1974, 1976, 1979, 1982, 1985, 1988, 1991, 1994, 1997, 2000, + 2003, 2006, 2008, 2011, 2014, 2017, 2020, 2023, 2026, 2029, 2032, 2035, 2038, 2040, 2043, 2046, + 2049, 2052, 2055, 2058, 2061, 2064, 2067, 2070, 2072, 2075, 2078, 2081, 2084, 2087, 2090, 2093, + 2096, 2099, 2101, 2104, 2107, 2110, 2113, 2116, 2119, 2122, 2125, 2128, 2131, 2137, 2143, 2154, + 2159, 2168, 2170, 2176, 2181, 2186, 2193, 2202, 2216, 2221, 2230, 2240, 2247, 2252, 2263, 2268, + 2277, 2289, 2301, 2313, 2326, 2336, 2345, 2357, 2366, 2376, 2383, 2394, 2405, 2415, 2423, 2435, + 2444, 2446, 2452, 2453, 2462, 2465, 2469, 2476, 2485, 2496, 2508, 2519, 2529, 2541, 2554, 2566, + 2577, 2590, 2603, 2616, 2628, 2642, 2656, 2670, 2675, 2684, 2693, 2699, 2700, 2706, 2711, 2725, + 2731, 2739, 2746, 2750, 2762, 2774, 2786, 2789, 2803, 2813, 2823, 2827, 2832, 2845, 2850, 2861, + 2874, 2887, 2896, 2905, 2918, 2932, 2940, 2941, 2947, 2956, 2962, 2968, 2974, 2979, 2986, 2994, + 3000, 3006, 3020, 3034, 3048, 3057, 3065, 3072, 3081, 3082, 3088, 3094, 3105, 3114, 3124, 3130, + 3135, 3148, 3161, 3174, 3181, 3189, 3196, 3202, 3210, 3216, 3223, 3230, 3235, 3246, 3255, 3258, + 3261, 3267, 3277, 3292, 3303, 3314, 3315, 3317, 3320, 3323, 3325, 3335, 3339, 3350, 3362, 3374, + 3386, 3388, 3394, 3398, 3410, 3422, 3434, 3442, 3443, 3448, 3454, 3466, 3476, 3484, 3489, 3496, + 3508, 3521, 3535, 3548, 3562, 3563, 3568, 3580, 3586, 3600, 3614, 3615, 3621, 3632, 3641, 3647, + 3654, 3662, 3670, 3680, 3690, 3700, 3712, 3719, 3723, 3732, 3733, 3734, 3739, 3751, 3752, 3758, + 3764, 3770, 3782, 3786, 3795, 3805, 3815, 3824, 3831, 3837, 3846, 3847, 3853, 3858, 3864, 3873, + 3875, 3883, 3884, 3890, 3896, 3908, 3912, 3924, 3936, 3948, 3958, 3966, 3975, 3984, 3990, 3992, + 4002, 4003, 4009, 4020, 4029, 4031, 4037, 4043, 4056, 4063, 4068, 4074, 4081, 4084, 4098, 4107, + 4116, 4123, 4134, 4145, 4155, 4166, 4174, 4185, 4197, 4203, 4212, 4218, 4219, 4228, 4237, 4248, + 4259, 4276, 4293, 4304, 4314, 4326, 4338, 4356, 4374, 4386, 4397, 4404, 4410, 4424, 4438, 4452, + 4465, 4476, 4479, 4483, 4485, 4489, 4498, 4505, 4517, 4531, 4543, 4546, 4547, 4555, 4567, 4584, + 4597, 4605, 4617, 4629, 4642, 4651, 4661, 4675, 4686, 4687, 4693, 4700, 4709, 4715, 4727, 4739, + 4753, 4763, 4775, 4788, 4791, 4795, 4804, 4810, 4811, 4817, 4823, 4831, 4843, 4849, 4856, 4865, + 4870, 4882, 4895, 4908, 4921, 4926, 4932, 4939, 4942, 4953, 4964, 4975, 4980, 4985, 4990, 4999, + 5007, 5015, 5024, 5025, 5028, 5034, 5046, 5055, 5060, 5066, 5071, 5076, 5088, 5101, 5114, 5127, + 5146, 5159, 5164, 5173, 5186, 5200, 5209, 5212, 5226, 5237, 5248, 5259, 5268, 5269, 5275, 5286, + 5295, 5301, 5314, 5321, 5331, 5336, 5345, 5352, 5359, 5364, 5365, 5366, 5377, 5378, 5380, 5381, + 5387, 5396, 5399, 5400, 5406, 5412, 5422, 5426, 5438, 5450, 5462, 5466, +]; +pub(super) static STD_UNICODE: [u16; 803] = [ + 0x0041, 0x00C6, 0xF7E6, 0x00C1, 0xF7E1, 0x0102, 0x00C2, 0xF7E2, 0xF7B4, 0x00C4, 0xF7E4, 0x00C0, + 0xF7E0, 0x0391, 0x0100, 0x0104, 0x00C5, 0xF7E5, 0xF761, 0x00C3, 0xF7E3, 0x0042, 0x0392, 0xF6F4, + 0xF762, 0x0043, 0x0106, 0xF6F5, 0x010C, 0x00C7, 0xF7E7, 0xF7B8, 0x03A7, 0xF6F6, 0xF763, 0x0044, + 0x010E, 0x0110, 0x2206, 0xF7A8, 0xF6F7, 0xF764, 0x0045, 0x00C9, 0xF7E9, 0x011A, 0x00CA, 0xF7EA, + 0x00CB, 0xF7EB, 0x0116, 0x00C8, 0xF7E8, 0x0112, 0x0118, 0x0395, 0xF765, 0x0397, 0x00D0, 0xF7F0, + 0x20AC, 0x0046, 0xF766, 0x0047, 0x0393, 0x011E, 0x0122, 0xF760, 0xF767, 0x0048, 0xF768, 0xF6F8, + 0x0049, 0x00CD, 0xF7ED, 0x00CE, 0xF7EE, 0x00CF, 0xF7EF, 0x0130, 0x2111, 0x00CC, 0xF7EC, 0x012A, + 0x012E, 0x0399, 0xF769, 0x004A, 0xF76A, 0x004B, 0x039A, 0x0136, 0xF76B, 0x004C, 0x0139, 0x039B, + 0x013D, 0x013B, 0x0141, 0xF6F9, 0xF76C, 0x004D, 0xF7AF, 0xF76D, 0x039C, 0x004E, 0x0143, 0x0147, + 0x0145, 0xF76E, 0x00D1, 0xF7F1, 0x039D, 0x004F, 0x0152, 0xF6FA, 0x00D3, 0xF7F3, 0x00D4, 0xF7F4, + 0x00D6, 0xF7F6, 0xF6FB, 0x00D2, 0xF7F2, 0x0150, 0x014C, 0x2126, 0x039F, 0x00D8, 0xF7F8, 0xF76F, + 0x00D5, 0xF7F5, 0x0050, 0x03A6, 0x03A0, 0x03A8, 0xF770, 0x0051, 0xF771, 0x0052, 0x0154, 0x0158, + 0x0156, 0x211C, 0x03A1, 0xF6FC, 0xF772, 0x0053, 0x015A, 0x0160, 0xF6FD, 0x015E, 0x0218, 0x03A3, + 0xF773, 0x0054, 0x03A4, 0x0164, 0x0162, 0x0398, 0x00DE, 0xF7FE, 0xF6FE, 0xF774, 0x0055, 0x00DA, + 0xF7FA, 0x00DB, 0xF7FB, 0x00DC, 0xF7FC, 0x00D9, 0xF7F9, 0x0170, 0x016A, 0x0172, 0x03A5, 0x03D2, + 0x016E, 0xF775, 0x0056, 0xF776, 0x0057, 0xF777, 0x0058, 0x039E, 0xF778, 0x0059, 0x00DD, 0xF7FD, + 0x0178, 0xF7FF, 0xF779, 0x005A, 0x0179, 0x017D, 0xF6FF, 0x017B, 0x0396, 0xF77A, 0x0061, 0x2701, + 0x2721, 0x275E, 0x2761, 0x2762, 0x2763, 0x2764, 0x2710, 0x2765, 0x2766, 0x2767, 0x2660, 0x261B, + 0x2665, 0x2666, 0x2663, 0x2709, 0x2708, 0x2707, 0x261E, 0x2460, 0x2461, 0x2462, 0x2463, 0x2464, + 0x2465, 0x2466, 0x2467, 0x2468, 0x2469, 0x270C, 0x2776, 0x2777, 0x2778, 0x2779, 0x277A, 0x277B, + 0x277C, 0x277D, 0x277E, 0x277F, 0x270D, 0x2780, 0x2781, 0x2782, 0x2783, 0x2784, 0x2785, 0x2786, + 0x2787, 0x2788, 0x2789, 0x270E, 0x278A, 0x278B, 0x278C, 0x278D, 0x278E, 0x278F, 0x2790, 0x2791, + 0x2792, 0x2793, 0x270F, 0x2794, 0x2192, 0x27A3, 0x2194, 0x2195, 0x2799, 0x279B, 0x279C, 0x279D, + 0x279E, 0x2711, 0x279F, 0x27A0, 0x27A1, 0x27A2, 0x27A4, 0x27A5, 0x27A6, 0x27A7, 0x27A8, 0x27A9, + 0x2712, 0x27AB, 0x27AD, 0x27AF, 0x27B2, 0x27B3, 0x27B5, 0x27B8, 0x27BA, 0x27BB, 0x27BC, 0x2713, + 0x27BD, 0x27BE, 0x279A, 0x27AA, 0x27B6, 0x27B9, 0x2798, 0x27B4, 0x27B7, 0x27AC, 0x2702, 0x2714, + 0x27AE, 0x27B1, 0x2703, 0x2750, 0x2752, 0x276E, 0x2770, 0x2715, 0x2716, 0x2717, 0x2718, 0x2719, + 0x271A, 0x271B, 0x271C, 0x2722, 0x2704, 0x2723, 0x2724, 0x2725, 0x2726, 0x2727, 0x2605, 0x2729, + 0x272A, 0x272B, 0x272C, 0x260E, 0x272D, 0x272E, 0x272F, 0x2730, 0x2731, 0x2732, 0x2733, 0x2734, + 0x2735, 0x2736, 0x2706, 0x2737, 0x2738, 0x2739, 0x273A, 0x273B, 0x273C, 0x273D, 0x273E, 0x273F, + 0x2740, 0x271D, 0x2741, 0x2742, 0x2743, 0x2744, 0x2745, 0x2746, 0x2747, 0x2748, 0x2749, 0x274A, + 0x271E, 0x274B, 0x25CF, 0x274D, 0x25A0, 0x274F, 0x2751, 0x25B2, 0x25BC, 0x25C6, 0x2756, 0x271F, + 0x25D7, 0x2758, 0x2759, 0x275A, 0x276F, 0x2771, 0x2772, 0x2773, 0x2768, 0x2720, 0x2769, 0x276C, + 0x276D, 0x276A, 0x276B, 0x2774, 0x2775, 0x275B, 0x275C, 0x275D, 0x00E1, 0x0103, 0x00E2, 0x00B4, + 0x00E4, 0x00E6, 0x00E0, 0x2135, 0x03B1, 0x0101, 0x0026, 0xF726, 0x2220, 0x2329, 0x232A, 0x0105, + 0xF8FF, 0x2248, 0x00E5, 0x2194, 0x21D4, 0x21D3, 0x21D0, 0x21D2, 0x21D1, 0x2193, 0xF8E7, 0x2190, + 0x2192, 0x2191, 0xF8E6, 0x005E, 0x007E, 0x002A, 0x2217, 0xF6E9, 0x0040, 0x00E3, 0x0062, 0x005C, + 0x007C, 0x03B2, 0xF8F4, 0x007B, 0xF8F3, 0xF8F2, 0xF8F1, 0x007D, 0xF8FE, 0xF8FD, 0xF8FC, 0x005B, + 0xF8F0, 0xF8EF, 0xF8EE, 0x005D, 0xF8FB, 0xF8FA, 0xF8F9, 0x02D8, 0x00A6, 0xF6EA, 0x2022, 0x0063, + 0x0107, 0x02C7, 0x21B5, 0x010D, 0x00E7, 0x00B8, 0x00A2, 0xF6DF, 0xF7A2, 0xF6E0, 0x03C7, 0x2297, + 0x2295, 0x02C6, 0x2663, 0x003A, 0x20A1, 0x002C, 0xF6C3, 0xF6E1, 0xF6E2, 0x2245, 0x00A9, 0xF8E9, + 0xF6D9, 0x00A4, 0x0064, 0x2020, 0x2021, 0x010F, 0x0111, 0x00B0, 0x03B4, 0x2666, 0x00A8, 0x00F7, + 0x0024, 0xF6E3, 0xF724, 0xF6E4, 0x02D9, 0x0131, 0x22C5, 0xF6EB, 0x0065, 0x00E9, 0x011B, 0x00EA, + 0x00EB, 0x0117, 0x00E8, 0x0038, 0x2088, 0xF738, 0x2078, 0x2208, 0x2026, 0x0113, 0x2014, 0x2205, + 0x2013, 0x0119, 0x03B5, 0x003D, 0x2261, 0xF6EC, 0x03B7, 0x00F0, 0x0021, 0x00A1, 0xF7A1, 0xF721, + 0x2203, 0x0066, 0xFB00, 0xFB03, 0xFB04, 0xFB01, 0x2012, 0x0035, 0x215D, 0x2085, 0xF735, 0x2075, + 0xFB02, 0x0192, 0x0034, 0x2084, 0xF734, 0x2074, 0x2044, 0x0067, 0x03B3, 0x011F, 0x0123, 0x00DF, + 0x2207, 0x0060, 0x003E, 0x2265, 0x00AB, 0x00BB, 0x2039, 0x203A, 0x0068, 0x2665, 0x02DD, 0x002D, + 0xF6E5, 0xF6E6, 0x0069, 0x00ED, 0x00EE, 0x00EF, 0x00EC, 0x012B, 0x221E, 0x222B, 0x2321, 0xF8F5, + 0x2320, 0x2229, 0x012F, 0x03B9, 0xF6ED, 0x006A, 0x006B, 0x03BA, 0x0137, 0x006C, 0x013A, 0x03BB, + 0x013E, 0x013C, 0x003C, 0x2264, 0x2227, 0x00AC, 0x2228, 0x25CA, 0x0142, 0xF6EE, 0x006D, 0x00AF, + 0x2212, 0x2032, 0xF6EF, 0x00B5, 0x00D7, 0x006E, 0x0144, 0x0148, 0x0146, 0x0039, 0x2089, 0xF739, + 0x2079, 0x2209, 0x2260, 0x2284, 0x207F, 0x00F1, 0x03BD, 0x0023, 0x006F, 0x00F3, 0x00F4, 0x00F6, + 0x0153, 0x02DB, 0x00F2, 0x0151, 0x014D, 0x03C9, 0x03D6, 0x03BF, 0x0031, 0x2024, 0x215B, 0xF6DC, + 0x00BD, 0x2081, 0xF731, 0x00BC, 0x00B9, 0x2153, 0x00AA, 0x00BA, 0x00F8, 0xF6F0, 0x00F5, 0x0070, + 0x00B6, 0x0028, 0xF8ED, 0xF8EC, 0x208D, 0x207D, 0xF8EB, 0x0029, 0xF8F8, 0xF8F7, 0x208E, 0x207E, + 0xF8F6, 0x2202, 0x0025, 0x002E, 0x00B7, 0xF6E7, 0xF6E8, 0x22A5, 0x2030, 0x03C6, 0x03D5, 0x03C0, + 0x002B, 0x00B1, 0x220F, 0x2282, 0x2283, 0x221D, 0x03C8, 0x0071, 0x003F, 0x00BF, 0xF7BF, 0xF73F, + 0x0022, 0x201E, 0x201C, 0x201D, 0x2018, 0x2019, 0x201A, 0x0027, 0x0072, 0x0155, 0x221A, 0xF8E5, + 0x0159, 0x0157, 0x2286, 0x2287, 0x00AE, 0xF8E8, 0xF6DA, 0x03C1, 0x02DA, 0xF6F1, 0xF6DD, 0x0073, + 0x015B, 0x0161, 0x015F, 0x0219, 0x2033, 0x00A7, 0x003B, 0x0037, 0x215E, 0x2087, 0xF737, 0x2077, + 0x03C3, 0x03C2, 0x223C, 0x0036, 0x2086, 0xF736, 0x2076, 0x002F, 0x0020, 0x2660, 0xF6F2, 0x00A3, + 0x220B, 0x2211, 0x0074, 0x03C4, 0x0165, 0x0163, 0x2234, 0x03B8, 0x03D1, 0x00FE, 0x0033, 0x215C, + 0x2083, 0xF733, 0x00BE, 0xF6DE, 0x00B3, 0x02DC, 0x2122, 0xF8EA, 0xF6DB, 0xF6F3, 0x0032, 0x2025, + 0x2082, 0xF732, 0x00B2, 0x2154, 0x0075, 0x00FA, 0x00FB, 0x00FC, 0x00F9, 0x0171, 0x016B, 0x005F, + 0x222A, 0x2200, 0x0173, 0x03C5, 0x016F, 0x0076, 0x0077, 0x2118, 0x0078, 0x03BE, 0x0079, 0x00FD, + 0x00FF, 0x00A5, 0x007A, 0x017A, 0x017E, 0x017C, 0x0030, 0x2080, 0xF730, 0x2070, 0x03B6, +]; + +pub(super) static ENCODINGS: [[u16; 224]; 7] = [ + [ + 740, 536, 696, 631, 504, 674, 414, 701, 661, 667, 437, 684, 485, 575, 675, 739, 798, 644, + 766, 754, 554, 547, 735, 727, 519, 621, 483, 726, 602, 531, 566, 692, 440, 0, 21, 25, 35, + 42, 61, 63, 69, 72, 87, 89, 93, 101, 105, 113, 134, 139, 141, 149, 157, 166, 182, 184, 186, + 189, 195, 455, 443, 459, 435, 779, 700, 202, 442, 467, 494, 512, 541, 559, 572, 578, 593, + 594, 597, 610, 617, 632, 659, 691, 704, 719, 746, 772, 785, 786, 788, 790, 794, 447, 444, + 451, 436, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 537, 474, 743, 558, + 793, 553, 725, 493, 703, 698, 568, 570, 571, 545, 552, 65535, 528, 495, 496, 676, 65535, + 660, 466, 702, 697, 699, 569, 524, 680, 65535, 693, 65535, 565, 407, 481, 761, 611, 463, + 508, 502, 65535, 716, 473, 65535, 574, 637, 469, 526, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 1, 65535, 654, + 65535, 65535, 65535, 65535, 98, 129, 114, 655, 65535, 65535, 65535, 65535, 65535, 409, + 65535, 65535, 65535, 509, 65535, 65535, 608, 656, 636, 563, 65535, 65535, 65535, 65535, + ], + [ + 740, 536, 696, 631, 504, 674, 414, 703, 661, 667, 437, 684, 485, 575, 675, 739, 798, 644, + 766, 754, 554, 547, 735, 727, 519, 621, 483, 726, 602, 531, 566, 692, 440, 0, 21, 25, 35, + 42, 61, 63, 69, 72, 87, 89, 93, 101, 105, 113, 134, 139, 141, 149, 157, 166, 182, 184, 186, + 189, 195, 455, 443, 459, 435, 779, 565, 202, 442, 467, 494, 512, 541, 559, 572, 578, 593, + 594, 597, 610, 617, 632, 659, 691, 704, 719, 746, 772, 785, 786, 788, 790, 794, 447, 444, + 451, 436, 466, 60, 466, 702, 553, 697, 524, 495, 496, 481, 680, 151, 570, 114, 466, 197, + 466, 466, 700, 701, 698, 699, 466, 528, 526, 761, 762, 721, 571, 636, 466, 796, 192, 740, + 537, 474, 743, 493, 793, 464, 725, 502, 490, 654, 568, 605, 575, 712, 611, 499, 685, 770, + 760, 407, 615, 660, 676, 473, 652, 655, 569, 651, 648, 758, 693, 11, 3, 6, 19, 9, 16, 1, + 29, 51, 43, 46, 48, 81, 73, 75, 77, 58, 110, 123, 116, 118, 132, 120, 616, 129, 173, 167, + 169, 171, 190, 162, 563, 410, 404, 406, 441, 408, 422, 409, 472, 518, 513, 515, 516, 582, + 579, 580, 581, 535, 629, 638, 633, 634, 658, 635, 503, 656, 776, 773, 774, 775, 791, 753, + 792, + ], + [ + 740, 536, 696, 631, 504, 674, 414, 703, 661, 667, 437, 684, 485, 575, 675, 739, 798, 644, + 766, 754, 554, 547, 735, 727, 519, 621, 483, 726, 602, 531, 566, 692, 440, 0, 21, 25, 35, + 42, 61, 63, 69, 72, 87, 89, 93, 101, 105, 113, 134, 139, 141, 149, 157, 166, 182, 184, 186, + 189, 195, 455, 443, 459, 435, 779, 565, 202, 442, 467, 494, 512, 541, 559, 572, 578, 593, + 594, 597, 610, 617, 632, 659, 691, 704, 719, 746, 772, 785, 786, 788, 790, 794, 447, 444, + 451, 436, 65535, 9, 16, 29, 43, 110, 120, 171, 404, 410, 406, 408, 441, 422, 472, 513, 518, + 515, 516, 579, 582, 580, 581, 629, 633, 638, 634, 635, 658, 773, 776, 774, 775, 495, 499, + 474, 743, 725, 466, 660, 563, 712, 490, 762, 407, 502, 626, 1, 129, 584, 685, 603, 567, + 793, 615, 673, 745, 686, 683, 585, 654, 655, 127, 409, 656, 693, 537, 605, 706, 553, 421, + 38, 568, 569, 524, 740, 11, 19, 132, 114, 636, 528, 526, 698, 699, 700, 701, 503, 607, 792, + 192, 558, 493, 570, 571, 545, 552, 496, 676, 702, 697, 680, 6, 46, 3, 48, 51, 73, 75, 77, + 81, 116, 118, 65535, 123, 167, 169, 173, 509, 481, 761, 611, 463, 508, 716, 473, 574, 637, + 469, + ], + [ + 740, 539, 71, 476, 506, 507, 415, 8, 665, 671, 767, 645, 485, 575, 675, 558, 800, 650, 769, + 757, 556, 550, 737, 730, 521, 623, 483, 726, 65535, 759, 65535, 695, 65535, 65535, 65535, + 65535, 59, 65535, 65535, 651, 648, 758, 646, 755, 548, 728, 653, 771, 65535, 65535, 65535, + 65535, 65535, 65535, 542, 545, 552, 543, 544, 664, 65535, 670, 33, 576, 67, 18, 24, 34, 41, + 56, 62, 68, 70, 86, 88, 92, 100, 103, 109, 131, 138, 140, 148, 156, 165, 181, 183, 185, + 188, 194, 201, 484, 647, 718, 164, 65535, 65535, 439, 477, 65535, 65535, 65535, 65535, 4, + 12, 7, 10, 20, 17, 30, 44, 52, 47, 49, 74, 82, 76, 78, 111, 117, 124, 119, 121, 133, 168, + 174, 170, 172, 65535, 522, 555, 756, 736, 520, 729, 152, 65535, 475, 768, 65535, 39, 65535, + 27, 657, 549, 65535, 487, 677, 191, 65535, 505, 65535, 65535, 163, 65535, 622, 799, 198, 2, + 130, 694, 649, 99, 65535, 65535, 65535, 65535, 65535, 65535, 31, 65535, 65535, 65535, + 65535, 65535, 115, 546, 577, 65535, 65535, 65535, 65535, 538, 65535, 193, 65535, 652, 770, + 760, 557, 551, 738, 731, 624, 801, 65535, 533, 717, 765, 65535, 65535, 592, 742, 511, + 65535, 65535, 65535, 65535, 65535, 609, 122, 23, 102, 465, 628, 614, 488, 678, 40, 147, + 65535, 65535, 65535, 65535, + ], + [ + 740, 536, 781, 631, 540, 674, 414, 744, 661, 667, 438, 684, 485, 612, 675, 739, 798, 644, + 766, 754, 554, 547, 735, 727, 519, 621, 483, 726, 602, 531, 566, 692, 489, 13, 22, 32, 38, + 55, 135, 64, 57, 85, 752, 90, 95, 104, 112, 128, 136, 161, 146, 155, 158, 178, 733, 127, + 187, 137, 200, 455, 750, 459, 679, 779, 707, 412, 445, 478, 500, 530, 681, 560, 534, 591, + 682, 595, 599, 615, 630, 643, 683, 751, 715, 732, 747, 783, 642, 641, 789, 690, 802, 447, + 444, 451, 734, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 60, 179, 613, 603, 558, 584, + 553, 482, 501, 573, 741, 423, 431, 433, 432, 429, 499, 685, 724, 567, 616, 689, 673, 466, + 503, 626, 532, 421, 524, 434, 430, 470, 411, 80, 145, 787, 479, 480, 527, 589, 780, 688, + 711, 627, 687, 710, 523, 625, 416, 564, 714, 492, 764, 686, 706, 510, 605, 604, 606, 424, + 426, 428, 427, 425, 607, 417, 713, 491, 763, 745, 666, 663, 662, 458, 457, 456, 450, 449, + 448, 446, 65535, 418, 585, 588, 587, 586, 672, 669, 668, 462, 461, 460, 454, 453, 452, + 65535, + ], + [ + 740, 203, 310, 314, 328, 339, 350, 221, 220, 219, 215, 222, 233, 244, 255, 266, 210, 277, + 288, 299, 311, 319, 320, 321, 322, 323, 324, 325, 326, 361, 372, 383, 393, 204, 327, 329, + 330, 331, 332, 333, 334, 335, 336, 337, 338, 340, 341, 342, 343, 344, 345, 346, 347, 348, + 349, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 373, 374, 375, 376, 377, 315, 378, 316, 379, 380, 381, 382, 384, 385, 386, + 387, 401, 402, 403, 205, 65535, 392, 394, 397, 398, 395, 396, 317, 388, 318, 389, 390, 391, + 399, 400, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 206, 207, 208, 209, 211, 212, 213, + 218, 217, 216, 214, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 234, 235, 236, 237, + 238, 239, 240, 241, 242, 243, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 267, 268, 270, 271, 306, 272, 302, 273, 274, 275, + 276, 278, 279, 280, 281, 269, 282, 283, 284, 285, 286, 287, 303, 289, 309, 290, 312, 291, + 65535, 313, 292, 293, 307, 294, 304, 308, 295, 305, 296, 297, 298, 300, 301, 65535, + ], + [ + 740, 539, 71, 65535, 506, 507, 415, 8, 665, 671, 767, 645, 485, 575, 675, 558, 800, 650, + 769, 757, 556, 550, 737, 730, 521, 623, 483, 726, 488, 759, 678, 695, 65535, 439, 465, 477, + 511, 533, 65535, 65535, 65535, 592, 65535, 65535, 609, 614, 628, 657, 65535, 65535, 717, + 742, 765, 65535, 542, 545, 552, 543, 544, 664, 65535, 670, 33, 577, 67, 18, 24, 34, 41, 56, + 62, 68, 70, 86, 88, 92, 100, 103, 109, 131, 138, 140, 148, 156, 165, 181, 183, 185, 188, + 194, 201, 484, 647, 718, 164, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, 65535, + 538, 476, 99, 65535, 65535, 152, 198, 39, 23, 27, 65535, 40, 65535, 65535, 102, 65535, + 65535, 546, 576, 65535, 65535, 122, 147, 31, 65535, 65535, 65535, 651, 648, 758, 694, 646, + 755, 548, 728, 653, 771, 65535, 65535, 801, 652, 770, 760, 557, 551, 738, 731, 522, 624, + 799, 649, 768, 756, 555, 549, 736, 729, 520, 622, 475, 505, 677, 487, 12, 4, 7, 20, 10, 17, + 2, 30, 52, 44, 47, 49, 82, 74, 76, 78, 59, 111, 124, 117, 119, 133, 121, 115, 130, 174, + 168, 170, 172, 191, 163, 193, + ], +]; diff --git a/crates/text-extract/src/pdf/tables/font_data.rs b/crates/text-extract/src/pdf/tables/font_data.rs new file mode 100644 index 00000000..a07a8043 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/font_data.rs @@ -0,0 +1,283 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::fonts::StandardFont; + +pub(super) static WIDTH_PALETTE: [u16; 207] = [ + 138, 167, 180, 191, 200, 214, 220, 222, 234, 238, 247, 250, 260, 266, 274, 275, 276, 277, 278, + 280, 299, 300, 310, 317, 326, 329, 330, 333, 334, 344, 348, 350, 355, 365, 366, 370, 382, 384, + 389, 390, 392, 394, 400, 408, 410, 411, 415, 416, 420, 422, 438, 439, 444, 453, 458, 460, 463, + 469, 471, 474, 476, 480, 493, 494, 500, 509, 520, 521, 523, 537, 540, 541, 544, 549, 552, 555, + 556, 564, 570, 571, 576, 577, 581, 584, 588, 592, 595, 600, 603, 606, 608, 611, 612, 620, 626, + 631, 643, 645, 658, 667, 668, 672, 675, 677, 682, 686, 687, 689, 690, 692, 694, 695, 696, 701, + 707, 708, 713, 719, 722, 723, 732, 737, 741, 743, 744, 747, 748, 749, 750, 753, 754, 755, 759, + 760, 761, 762, 763, 768, 771, 776, 778, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, + 795, 815, 816, 823, 826, 828, 831, 832, 833, 834, 836, 838, 841, 846, 855, 863, 865, 867, 873, + 874, 883, 888, 889, 890, 892, 894, 910, 911, 917, 918, 920, 921, 923, 924, 927, 928, 930, 931, + 933, 939, 944, 945, 946, 960, 961, 967, 970, 974, 975, 980, 987, 1000, 1015, 1016, 1042, +]; + +pub(super) static LATIN_GLYPHS: [u16; 315] = [ + 0, 1, 3, 5, 6, 9, 11, 14, 15, 16, 19, 21, 25, 26, 28, 29, 35, 36, 37, 38, 42, 43, 45, 46, 48, + 50, 51, 53, 54, 58, 60, 61, 63, 65, 66, 69, 72, 73, 75, 77, 79, 81, 83, 84, 87, 89, 91, 93, 94, + 96, 97, 98, 101, 105, 106, 107, 108, 110, 113, 114, 116, 118, 120, 123, 125, 126, 129, 132, + 134, 139, 141, 142, 143, 144, 149, 150, 151, 153, 154, 157, 159, 160, 162, 166, 167, 169, 171, + 173, 175, 176, 177, 180, 182, 184, 186, 189, 190, 192, 195, 196, 197, 199, 202, 404, 405, 406, + 407, 408, 409, 410, 413, 414, 419, 422, 435, 436, 437, 440, 441, 442, 443, 444, 447, 451, 455, + 459, 463, 464, 466, 467, 468, 469, 471, 472, 473, 474, 481, 483, 485, 486, 490, 493, 494, 495, + 496, 497, 498, 499, 502, 503, 504, 508, 509, 512, 513, 514, 515, 516, 517, 518, 519, 524, 525, + 526, 528, 529, 531, 535, 536, 537, 541, 545, 547, 552, 553, 554, 558, 559, 561, 562, 563, 565, + 566, 567, 568, 569, 570, 571, 572, 574, 575, 578, 579, 580, 581, 582, 583, 590, 593, 594, 596, + 597, 598, 600, 601, 602, 603, 605, 607, 608, 610, 611, 612, 615, 616, 617, 618, 619, 620, 621, + 626, 629, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 644, 648, 651, 652, 654, 655, 656, + 658, 659, 660, 661, 667, 673, 674, 675, 676, 680, 684, 685, 691, 692, 693, 696, 697, 698, 699, + 700, 701, 702, 703, 704, 705, 706, 708, 709, 712, 716, 719, 720, 721, 722, 723, 725, 726, 727, + 735, 739, 740, 743, 745, 746, 748, 749, 753, 754, 758, 760, 761, 762, 766, 770, 772, 773, 774, + 775, 776, 777, 778, 779, 782, 784, 785, 786, 788, 790, 791, 792, 793, 794, 795, 796, 797, 798, +]; +pub(super) static LATIN_WIDTHS: [[u8; 315]; 6] = [ + [ + 99, 203, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 118, 118, 118, 118, 118, 118, 118, 92, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 118, 76, 91, 140, 140, 140, 118, 18, 18, 18, 18, 18, 18, + 18, 18, 64, 99, 99, 76, 76, 76, 76, 76, 160, 118, 118, 118, 118, 118, 140, 203, 140, 140, + 140, 140, 140, 140, 140, 140, 99, 140, 118, 118, 118, 118, 99, 99, 99, 99, 99, 91, 91, 91, + 99, 118, 118, 118, 118, 118, 118, 118, 118, 118, 99, 192, 99, 99, 99, 99, 91, 91, 91, 91, + 76, 76, 76, 76, 27, 76, 174, 76, 76, 99, 76, 76, 57, 83, 38, 204, 76, 76, 18, 12, 28, 28, + 18, 18, 27, 12, 31, 64, 64, 27, 64, 64, 27, 76, 27, 18, 18, 11, 121, 76, 76, 76, 76, 96, + 76, 42, 27, 83, 76, 27, 18, 76, 76, 76, 76, 76, 76, 76, 76, 203, 76, 203, 76, 76, 83, 76, + 18, 27, 18, 64, 76, 64, 76, 76, 1, 76, 76, 76, 91, 27, 83, 73, 76, 76, 27, 27, 76, 27, 27, + 7, 18, 18, 18, 18, 18, 7, 7, 64, 64, 7, 7, 20, 7, 83, 73, 83, 58, 7, 160, 27, 83, 76, 83, + 76, 76, 76, 76, 76, 73, 76, 76, 76, 76, 76, 76, 192, 27, 76, 76, 76, 76, 161, 161, 27, 35, + 33, 91, 76, 76, 69, 27, 27, 60, 174, 18, 18, 203, 83, 83, 76, 76, 91, 32, 27, 27, 27, 7, 7, + 7, 3, 27, 27, 53, 27, 27, 121, 27, 64, 64, 64, 64, 64, 76, 18, 76, 76, 18, 18, 76, 87, 18, + 23, 18, 76, 76, 161, 27, 27, 203, 76, 27, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 64, 118, + 64, 64, 64, 64, 76, 64, 64, 64, 64, 76, + ], + [ + 118, 203, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, + 118, 92, 99, 99, 99, 99, 99, 99, 99, 99, 99, 118, 76, 91, 140, 140, 140, 118, 18, 18, 18, + 18, 18, 18, 18, 18, 76, 118, 118, 91, 91, 91, 91, 91, 160, 118, 118, 118, 118, 118, 140, + 203, 140, 140, 140, 140, 140, 140, 140, 140, 99, 140, 118, 118, 118, 118, 99, 99, 99, 99, + 99, 91, 91, 91, 99, 118, 118, 118, 118, 118, 118, 118, 118, 118, 99, 192, 99, 99, 99, 99, + 91, 91, 91, 91, 76, 76, 76, 76, 27, 76, 174, 76, 76, 118, 76, 76, 83, 83, 38, 200, 76, 91, + 18, 19, 38, 38, 27, 27, 27, 19, 31, 76, 76, 27, 76, 76, 27, 76, 27, 27, 18, 11, 121, 76, + 91, 76, 76, 123, 91, 42, 27, 83, 76, 27, 18, 76, 76, 76, 76, 76, 76, 76, 76, 203, 76, 203, + 76, 76, 83, 91, 27, 27, 27, 91, 76, 91, 76, 76, 1, 91, 91, 91, 91, 27, 83, 73, 76, 76, 27, + 27, 91, 27, 27, 18, 18, 18, 18, 18, 18, 18, 18, 76, 76, 18, 18, 42, 18, 83, 73, 83, 63, 18, + 174, 27, 83, 91, 83, 91, 91, 91, 91, 76, 73, 91, 76, 91, 91, 91, 91, 192, 27, 91, 91, 91, + 76, 161, 161, 27, 35, 33, 91, 91, 91, 76, 27, 27, 63, 174, 18, 18, 203, 83, 83, 91, 91, 91, + 59, 64, 64, 64, 18, 18, 18, 9, 38, 38, 73, 38, 38, 121, 27, 76, 76, 76, 76, 76, 76, 27, 76, + 76, 18, 18, 76, 87, 27, 38, 27, 91, 76, 161, 27, 27, 203, 76, 27, 91, 91, 91, 91, 91, 91, + 91, 76, 91, 91, 76, 140, 76, 76, 76, 76, 76, 64, 64, 64, 64, 76, + ], + [ + 118, 174, 118, 118, 118, 118, 118, 118, 118, 118, 118, 99, 99, 99, 99, 99, 118, 118, 118, + 92, 91, 91, 91, 91, 91, 91, 91, 91, 91, 118, 64, 76, 118, 118, 118, 118, 27, 27, 27, 27, + 27, 27, 27, 27, 38, 118, 118, 91, 91, 91, 91, 91, 174, 118, 118, 118, 118, 118, 118, 174, + 118, 118, 118, 118, 118, 118, 118, 118, 76, 118, 99, 99, 99, 99, 76, 76, 76, 76, 76, 91, + 91, 91, 76, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 192, 118, 118, 118, 118, 91, + 91, 91, 91, 52, 52, 52, 52, 27, 52, 99, 52, 52, 140, 52, 52, 57, 71, 64, 183, 52, 64, 18, + 4, 61, 61, 27, 27, 27, 4, 31, 52, 52, 27, 52, 52, 27, 64, 27, 18, 11, 11, 133, 64, 64, 64, + 64, 84, 64, 42, 27, 77, 64, 27, 18, 52, 52, 52, 52, 52, 52, 52, 64, 203, 52, 203, 64, 52, + 77, 64, 27, 27, 27, 76, 64, 76, 64, 64, 1, 64, 64, 64, 64, 27, 77, 73, 64, 64, 27, 27, 64, + 27, 27, 18, 18, 18, 18, 18, 18, 18, 18, 64, 64, 18, 18, 29, 18, 77, 73, 77, 58, 18, 140, + 27, 77, 64, 77, 64, 64, 64, 64, 64, 73, 64, 64, 64, 64, 64, 64, 118, 27, 64, 64, 64, 64, + 128, 128, 21, 16, 22, 64, 64, 64, 53, 27, 27, 60, 160, 11, 11, 203, 77, 77, 64, 52, 52, 43, + 52, 52, 52, 27, 27, 27, 2, 27, 27, 53, 27, 27, 133, 27, 38, 38, 38, 38, 38, 64, 18, 64, 64, + 18, 11, 64, 87, 18, 24, 18, 64, 64, 128, 21, 27, 201, 64, 21, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 118, 64, 64, 64, 64, 64, 52, 52, 52, 52, 64, + ], + [ + 118, 203, 118, 118, 118, 118, 118, 118, 118, 118, 118, 99, 118, 118, 118, 118, 118, 118, + 118, 92, 99, 99, 99, 99, 99, 99, 99, 99, 99, 118, 64, 91, 140, 140, 140, 140, 38, 38, 38, + 38, 38, 38, 38, 38, 64, 140, 140, 99, 99, 99, 99, 99, 192, 118, 118, 118, 118, 118, 140, + 203, 140, 140, 140, 140, 140, 140, 140, 140, 91, 140, 118, 118, 118, 118, 76, 76, 76, 76, + 76, 99, 99, 99, 91, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 203, 118, 118, 118, + 118, 99, 99, 99, 99, 64, 64, 64, 64, 27, 64, 118, 64, 64, 160, 64, 64, 82, 66, 64, 188, 64, + 76, 18, 6, 41, 41, 27, 27, 27, 6, 31, 52, 52, 27, 52, 52, 27, 64, 27, 27, 11, 11, 125, 64, + 76, 64, 64, 101, 76, 42, 27, 78, 64, 27, 18, 52, 52, 52, 52, 52, 52, 52, 64, 203, 52, 203, + 64, 52, 78, 64, 27, 27, 27, 76, 64, 76, 64, 64, 1, 64, 64, 64, 76, 27, 78, 73, 64, 64, 27, + 27, 76, 27, 27, 18, 18, 18, 18, 18, 18, 18, 27, 76, 76, 18, 18, 41, 18, 78, 73, 78, 63, 18, + 160, 27, 78, 76, 78, 76, 76, 76, 76, 64, 73, 76, 64, 64, 64, 64, 64, 118, 27, 64, 64, 64, + 64, 128, 128, 21, 21, 26, 64, 64, 76, 70, 27, 27, 63, 203, 11, 11, 203, 78, 78, 76, 64, 64, + 75, 64, 64, 64, 27, 27, 27, 18, 52, 52, 73, 52, 52, 125, 27, 38, 38, 38, 38, 38, 64, 27, + 64, 64, 18, 11, 64, 87, 27, 47, 27, 76, 64, 128, 21, 27, 203, 64, 21, 76, 76, 76, 76, 76, + 76, 76, 64, 76, 76, 64, 118, 64, 64, 64, 64, 64, 52, 52, 52, 52, 64, + ], + [ + 91, 174, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 99, 99, 99, 99, 118, 118, 118, 92, 91, 91, + 91, 91, 91, 91, 91, 91, 91, 118, 64, 91, 118, 118, 118, 118, 27, 27, 27, 27, 27, 27, 27, + 27, 52, 99, 99, 76, 76, 91, 76, 76, 160, 99, 99, 99, 99, 99, 118, 192, 118, 118, 118, 118, + 118, 118, 118, 118, 91, 118, 91, 91, 91, 91, 64, 64, 64, 64, 64, 76, 76, 76, 91, 118, 118, + 118, 118, 118, 118, 118, 118, 118, 91, 160, 91, 76, 76, 76, 76, 76, 76, 76, 64, 64, 64, 64, + 27, 64, 99, 64, 64, 140, 64, 64, 49, 71, 64, 182, 64, 64, 18, 15, 42, 42, 38, 38, 27, 15, + 31, 52, 52, 27, 52, 52, 27, 64, 27, 27, 11, 11, 133, 64, 64, 64, 64, 72, 64, 42, 27, 102, + 64, 27, 18, 52, 52, 52, 52, 52, 52, 52, 64, 174, 52, 174, 64, 52, 102, 64, 27, 38, 18, 64, + 64, 64, 64, 64, 1, 64, 64, 64, 64, 27, 102, 73, 64, 64, 27, 27, 64, 27, 27, 18, 18, 18, 18, + 18, 18, 18, 18, 52, 52, 18, 18, 21, 18, 102, 73, 102, 58, 18, 118, 27, 102, 64, 102, 64, + 64, 64, 64, 64, 73, 64, 64, 64, 64, 64, 64, 99, 27, 64, 64, 64, 64, 128, 128, 21, 16, 22, + 64, 64, 64, 68, 27, 27, 60, 160, 11, 11, 203, 102, 102, 64, 64, 64, 48, 76, 76, 76, 27, 27, + 27, 5, 38, 38, 53, 38, 38, 133, 27, 38, 38, 38, 38, 38, 64, 27, 64, 64, 18, 11, 64, 87, 18, + 21, 18, 64, 64, 128, 21, 27, 201, 64, 21, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 52, 99, + 52, 52, 52, 52, 64, 38, 38, 38, 38, 64, + ], + [ + 99, 192, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 118, 118, 118, 92, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 118, 64, 99, 118, 118, 118, 140, 38, 38, 38, 38, 38, 38, 38, + 38, 64, 99, 99, 91, 91, 91, 91, 91, 174, 118, 118, 118, 118, 118, 118, 192, 118, 118, 118, + 118, 118, 118, 118, 118, 91, 118, 99, 99, 99, 99, 76, 76, 76, 76, 76, 91, 91, 91, 91, 118, + 118, 118, 118, 118, 118, 118, 118, 118, 99, 174, 99, 91, 91, 91, 91, 91, 91, 91, 64, 64, + 64, 64, 27, 64, 118, 64, 64, 140, 64, 64, 78, 78, 64, 159, 64, 64, 18, 6, 30, 30, 27, 27, + 27, 6, 31, 52, 52, 27, 52, 52, 27, 64, 27, 27, 11, 11, 125, 64, 64, 64, 64, 90, 64, 42, 27, + 78, 64, 27, 18, 52, 52, 52, 52, 52, 52, 52, 64, 203, 52, 203, 64, 52, 78, 64, 38, 38, 27, + 76, 64, 76, 64, 64, 1, 64, 64, 64, 64, 27, 78, 73, 64, 64, 27, 27, 76, 27, 27, 18, 18, 18, + 18, 18, 18, 18, 18, 64, 64, 18, 18, 36, 18, 78, 73, 89, 63, 18, 140, 27, 89, 80, 78, 76, + 76, 76, 76, 64, 73, 76, 64, 64, 64, 64, 64, 118, 27, 64, 64, 64, 64, 128, 128, 21, 13, 21, + 64, 64, 64, 64, 27, 27, 63, 160, 11, 11, 203, 78, 78, 64, 64, 64, 75, 64, 64, 64, 27, 27, + 27, 18, 38, 38, 73, 38, 38, 125, 27, 38, 38, 38, 38, 38, 64, 27, 64, 64, 18, 11, 64, 87, + 18, 34, 18, 64, 64, 128, 21, 27, 203, 64, 21, 76, 76, 76, 76, 76, 76, 76, 64, 76, 76, 52, + 99, 64, 52, 52, 52, 64, 38, 38, 38, 38, 64, + ], +]; + +pub(super) static SYMBOL_GLYPHS: [u16; 190] = [ + 13, 22, 32, 38, 55, 57, 60, 64, 80, 85, 90, 95, 104, 112, 127, 128, 135, 136, 137, 145, 146, + 155, 158, 161, 178, 179, 187, 200, 411, 412, 414, 416, 417, 418, 420, 421, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 433, 434, 438, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 457, 458, 459, 460, 461, 462, 466, 470, 478, 479, 480, 482, 483, 485, 489, 491, + 492, 499, 500, 501, 503, 510, 519, 523, 524, 527, 530, 531, 532, 534, 536, 540, 547, 553, 554, + 558, 560, 564, 566, 567, 573, 584, 585, 586, 587, 588, 589, 591, 595, 599, 602, 603, 604, 605, + 606, 607, 612, 613, 615, 616, 621, 625, 626, 627, 630, 631, 641, 642, 643, 644, 661, 662, 663, + 666, 667, 668, 669, 672, 673, 674, 675, 679, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, + 692, 706, 707, 710, 711, 713, 714, 715, 724, 726, 727, 732, 733, 734, 735, 739, 740, 741, 744, + 745, 747, 750, 751, 752, 754, 763, 764, 766, 779, 780, 781, 783, 787, 789, 798, 802, +]; +pub(super) static SYMBOL_WIDTHS: [u8; 190] = [ + 118, 99, 118, 92, 91, 118, 128, 88, 105, 27, 118, 105, 174, 118, 137, 118, 136, 137, 152, 152, + 76, 85, 91, 122, 108, 93, 97, 91, 155, 95, 140, 137, 25, 25, 147, 73, 206, 206, 88, 202, 202, + 88, 88, 203, 202, 202, 88, 88, 64, 4, 73, 63, 61, 63, 63, 63, 61, 63, 63, 63, 27, 37, 37, 37, + 27, 37, 37, 37, 55, 98, 73, 137, 137, 129, 18, 11, 73, 147, 147, 42, 63, 129, 73, 11, 64, 116, + 203, 155, 51, 73, 73, 88, 27, 73, 64, 64, 64, 1, 45, 116, 73, 73, 129, 116, 14, 105, 105, 105, + 137, 25, 73, 73, 73, 73, 88, 116, 88, 63, 73, 10, 80, 73, 64, 116, 73, 116, 67, 64, 105, 116, + 73, 64, 27, 37, 37, 37, 27, 37, 37, 37, 63, 160, 11, 98, 67, 88, 73, 73, 73, 155, 116, 116, + 116, 105, 52, 73, 64, 116, 116, 147, 147, 73, 45, 18, 64, 88, 51, 73, 64, 18, 11, 129, 51, 116, + 51, 167, 67, 95, 64, 143, 175, 64, 64, 137, 116, 80, 202, 62, 64, 63, +]; + +pub(super) static DINGBAT_GLYPHS: [u16; 202] = [ + 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, + 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 740, +]; +pub(super) static DINGBAT_WIDTHS: [u8; 202] = [ + 199, 109, 100, 120, 72, 72, 178, 179, 99, 133, 133, 94, 195, 110, 86, 139, 108, 148, 147, 191, + 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 73, 145, 145, 145, 145, 145, 145, 145, 145, + 145, 145, 166, 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 179, 145, 145, 145, 145, 145, + 145, 145, 145, 145, 145, 190, 177, 163, 185, 205, 54, 185, 181, 186, 187, 187, 193, 161, 170, + 157, 185, 180, 188, 189, 56, 172, 162, 199, 169, 112, 171, 133, 194, 168, 197, 158, 170, 186, + 131, 198, 181, 126, 162, 138, 173, 126, 138, 173, 169, 196, 165, 112, 171, 199, 135, 132, 65, + 44, 135, 134, 79, 103, 136, 133, 132, 130, 143, 201, 145, 145, 147, 150, 151, 154, 155, 146, + 164, 155, 117, 160, 154, 158, 184, 124, 119, 127, 147, 149, 111, 146, 139, 137, 149, 132, 114, + 115, 104, 113, 156, 153, 63, 146, 146, 114, 106, 112, 107, 143, 144, 116, 148, 74, 142, 148, + 170, 134, 135, 132, 176, 176, 145, 141, 69, 50, 0, 17, 46, 65, 44, 8, 8, 39, 81, 39, 16, 16, + 23, 23, 28, 28, 40, 40, 100, 18, +]; + +pub(super) static ALIAS_NAMES: &str = "ArialArial-BlackArial-Black-BoldArial-Black-BoldItalicArial-Black-ItalicArial-BoldArial-BoldItalicArial-BoldItalicMTArial-BoldItalicMT-BoldItalicArial-BoldMTArial-BoldMT-BoldArial-ItalicArial-ItalicMTArial-ItalicMT-ItalicArialBlackArialBlack-BoldArialBlack-BoldItalicArialBlack-ItalicArialMTArialNarrowArialNarrow-BoldArialNarrow-BoldItalicArialNarrow-ItalicArialUnicodeMSArialUnicodeMS-BoldArialUnicodeMS-BoldItalicArialUnicodeMS-ItalicCourierCourier-BoldCourier-BoldItalicCourier-BoldObliqueCourier-ItalicCourier-ObliqueCourierNewCourierNew-BoldCourierNew-BoldItalicCourierNew-ItalicCourierNewPS-BoldItalicMTCourierNewPS-BoldMTCourierNewPS-ItalicMTCourierNewPSMTDingbatsHelveticaHelvetica-BoldHelvetica-BoldItalicHelvetica-BoldObliqueHelvetica-ItalicHelvetica-NarrowHelvetica-Narrow-BoldHelvetica-Narrow-BoldObliqueHelvetica-Narrow-ObliqueHelvetica-ObliqueHelveticaLTStd-BoldSymbolSymbol-BoldSymbol-BoldItalicSymbol-ItalicTimes-BoldTimes-BoldItalicTimes-ItalicTimes-RomanTimesNewRomanTimesNewRoman-BoldTimesNewRoman-BoldItalicTimesNewRoman-ItalicTimesNewRomanPSTimesNewRomanPS-BoldTimesNewRomanPS-BoldItalicTimesNewRomanPS-BoldItalicMTTimesNewRomanPS-BoldMTTimesNewRomanPS-ItalicTimesNewRomanPS-ItalicMTTimesNewRomanPSMTTimesNewRomanPSMT-BoldTimesNewRomanPSMT-BoldItalicTimesNewRomanPSMT-ItalicZapfDingbats"; +pub(super) static ALIAS_OFFSETS: [u16; 78] = [ + 0, 5, 16, 32, 54, 72, 82, 98, 116, 145, 157, 174, 186, 200, 221, 231, 246, 267, 284, 291, 302, + 318, 340, 358, 372, 391, 416, 437, 444, 456, 474, 493, 507, 522, 532, 547, 568, 585, 610, 629, + 650, 664, 672, 681, 695, 715, 736, 752, 768, 789, 817, 841, 858, 877, 883, 894, 911, 924, 934, + 950, 962, 973, 986, 1004, 1028, 1048, 1063, 1083, 1109, 1137, 1159, 1181, 1205, 1222, 1244, + 1272, 1296, 1308, +]; +pub(super) static ALIAS_FONTS: [StandardFont; 77] = [ + StandardFont::Helvetica, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaBold, + StandardFont::HelveticaBold, + StandardFont::HelveticaOblique, + StandardFont::HelveticaOblique, + StandardFont::HelveticaOblique, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::Helvetica, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::Courier, + StandardFont::CourierBold, + StandardFont::CourierBoldOblique, + StandardFont::CourierBoldOblique, + StandardFont::CourierOblique, + StandardFont::CourierOblique, + StandardFont::Courier, + StandardFont::CourierBold, + StandardFont::CourierBoldOblique, + StandardFont::CourierOblique, + StandardFont::CourierBoldOblique, + StandardFont::CourierBold, + StandardFont::CourierOblique, + StandardFont::Courier, + StandardFont::ZapfDingbats, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::Helvetica, + StandardFont::HelveticaBold, + StandardFont::HelveticaBoldOblique, + StandardFont::HelveticaOblique, + StandardFont::HelveticaOblique, + StandardFont::HelveticaBold, + StandardFont::Symbol, + StandardFont::Symbol, + StandardFont::Symbol, + StandardFont::Symbol, + StandardFont::TimesBold, + StandardFont::TimesBoldItalic, + StandardFont::TimesItalic, + StandardFont::TimesRoman, + StandardFont::TimesRoman, + StandardFont::TimesBold, + StandardFont::TimesBoldItalic, + StandardFont::TimesItalic, + StandardFont::TimesRoman, + StandardFont::TimesBold, + StandardFont::TimesBoldItalic, + StandardFont::TimesBoldItalic, + StandardFont::TimesBold, + StandardFont::TimesItalic, + StandardFont::TimesItalic, + StandardFont::TimesRoman, + StandardFont::TimesBold, + StandardFont::TimesBoldItalic, + StandardFont::TimesItalic, + StandardFont::ZapfDingbats, +]; diff --git a/crates/text-extract/src/pdf/tables/fonts.rs b/crates/text-extract/src/pdf/tables/fonts.rs new file mode 100644 index 00000000..1e317eaa --- /dev/null +++ b/crates/text-extract/src/pdf/tables/fonts.rs @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::encoding::{BaseEncoding, STD_GLYPHS}; +use super::font_data::{ + ALIAS_FONTS, ALIAS_NAMES, ALIAS_OFFSETS, DINGBAT_GLYPHS, DINGBAT_WIDTHS, LATIN_GLYPHS, + LATIN_WIDTHS, SYMBOL_GLYPHS, SYMBOL_WIDTHS, WIDTH_PALETTE, +}; +use super::names::NameTable; + +static ALIASES: NameTable = NameTable::new(ALIAS_NAMES, &ALIAS_OFFSETS); + +const COURIER_WIDTH: u16 = 600; +const SUBSET_TAG_LEN: usize = 7; +const MAX_ALIAS: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StandardFont { + Courier, + CourierBold, + CourierOblique, + CourierBoldOblique, + Helvetica, + HelveticaBold, + HelveticaOblique, + HelveticaBoldOblique, + TimesRoman, + TimesBold, + TimesItalic, + TimesBoldItalic, + Symbol, + ZapfDingbats, +} + +enum Metrics { + Monospace, + Latin(usize), + Symbol, + Dingbats, +} + +impl StandardFont { + pub(crate) fn from_base_font(name: &[u8]) -> Option { + let mut buf = [0u8; MAX_ALIAS]; + let mut len = 0; + for &byte in strip_subset_tag(name) { + let normalized = match byte { + b',' | b'_' => b'-', + byte if byte.is_ascii_whitespace() => continue, + byte => byte, + }; + *buf.get_mut(len)? = normalized; + len += 1; + } + ALIAS_FONTS.get(ALIASES.find(buf.get(..len)?)?).copied() + } + + pub(crate) fn width(self, glyph_name: &[u8]) -> Option { + let (glyphs, widths): (&[u16], &[u8]) = match self.metrics() { + Metrics::Monospace => return Some(COURIER_WIDTH), + Metrics::Latin(row) => (&LATIN_GLYPHS, LATIN_WIDTHS.get(row)?), + Metrics::Symbol => (&SYMBOL_GLYPHS, &SYMBOL_WIDTHS), + Metrics::Dingbats => (&DINGBAT_GLYPHS, &DINGBAT_WIDTHS), + }; + let index = u16::try_from(STD_GLYPHS.find(glyph_name)?).ok()?; + let position = glyphs.binary_search(&index).ok()?; + WIDTH_PALETTE + .get(usize::from(*widths.get(position)?)) + .copied() + } + + pub(crate) fn builtin_encoding(self) -> BaseEncoding { + match self { + Self::Symbol => BaseEncoding::Symbol, + Self::ZapfDingbats => BaseEncoding::ZapfDingbats, + _ => BaseEncoding::Standard, + } + } + + pub(crate) fn is_symbolic(self) -> bool { + matches!(self, Self::Symbol | Self::ZapfDingbats) + } + + fn metrics(self) -> Metrics { + match self { + Self::Courier | Self::CourierBold | Self::CourierOblique | Self::CourierBoldOblique => { + Metrics::Monospace + } + Self::Helvetica | Self::HelveticaOblique => Metrics::Latin(0), + Self::HelveticaBold | Self::HelveticaBoldOblique => Metrics::Latin(1), + Self::TimesRoman => Metrics::Latin(2), + Self::TimesBold => Metrics::Latin(3), + Self::TimesItalic => Metrics::Latin(4), + Self::TimesBoldItalic => Metrics::Latin(5), + Self::Symbol => Metrics::Symbol, + Self::ZapfDingbats => Metrics::Dingbats, + } + } +} + +fn strip_subset_tag(name: &[u8]) -> &[u8] { + match name.split_at_checked(SUBSET_TAG_LEN) { + Some((tag, rest)) + if tag.split_last().is_some_and(|(last, head)| { + *last == b'+' && head.iter().all(u8::is_ascii_uppercase) + }) => + { + rest + } + _ => name, + } +} diff --git a/crates/text-extract/src/pdf/tables/glyph_data.rs b/crates/text-extract/src/pdf/tables/glyph_data.rs new file mode 100644 index 00000000..15eee125 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/glyph_data.rs @@ -0,0 +1,391 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) const BLOCK_SIZE: usize = 16; +pub(super) const PREFIX_LIMIT: u8 = 0x30; +pub(super) const TOKEN_BASE: u8 = 0x80; +pub(super) const MAX_NAME: usize = 48; +pub(super) const MULTI_BASE: u16 = 0xD800; + +pub(super) static TOKEN_TEXT: &str = "cyrillicarabichebrewonospacekoreankatakanasquarearmeniangujaratiirclebengalihiraganahalfwidthbelowparenrmukhialthaierdevaotaccentinopomofoleftacuteanedright000umflextionaboveregravehookcmbvowelsignaresisideographicatacstrokedageshenyrillicleieudisplayhadotdesirarolateodorigsymbolshseveafiiicossmilngzhouurnblquowooktritaupoldithopguekwhihesckeammaemvasskiyeokutunotomwiocdaxtehubphnoit00umrimzaelabkngurlstiddhtfircisgnbrahyepoog"; +pub(super) static TOKEN_OFFSETS: [u16; 129] = [ + 0, 8, 14, 20, 28, 34, 42, 48, 56, 64, 69, 76, 84, 93, 98, 103, 109, 111, 115, 117, 121, 129, + 131, 138, 142, 147, 149, 151, 156, 159, 165, 167, 169, 174, 176, 181, 185, 188, 197, 199, 203, + 214, 216, 218, 224, 230, 232, 239, 241, 244, 251, 253, 256, 258, 260, 262, 264, 266, 268, 270, + 272, 274, 280, 282, 284, 286, 290, 292, 294, 296, 298, 304, 307, 309, 311, 313, 316, 319, 321, + 323, 325, 327, 329, 331, 333, 335, 338, 340, 342, 344, 346, 348, 350, 352, 354, 360, 362, 364, + 366, 368, 370, 372, 374, 376, 378, 380, 382, 384, 386, 388, 390, 392, 395, 397, 400, 402, 405, + 407, 410, 412, 414, 416, 418, 420, 422, 424, 426, 428, 430, +]; + +pub(super) static NAME_BLOCKS: &[u8] = b"\x00AE\x98\x02m\xaar\x9f\x01b\xa1\xc0\x98\x06\x80\x06\xb3\x8d\x06\xa2\x06\xa3\xa0\x06\x9el\xb4\x01c\xa6\x9f\x02\x89\x05\x9d\x98\x0b\xb3\x8d\x0b\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02u\xb9\x00A\x80\x01d\xc8\xa2\x02i\x92\xa7\x80\x09m\xaar\x9f\x02\xe1\x8d\x04m\xaar\x9f\x01\xa3\xa0\x01ie\x80\x02nv\x92t\x9ab\xa1\xc0\x01l\xe9\xa9\x9f\xc3\x01m\x83\x01r\x95g\x98\x05\x8d\x01yb\x87\x00Bc\x89\x01d\x94\x00B\xb3\x8d\x01e\x80\x02n\x87\x01\xa3\x01l\x95e\x8d\x01m\x83\x01t\xd2b\xa6\x00Ca\x87\x02r\x9f\x01c\x9a\xc5\xb8\x98\x02\x89\x05\x9d\x01\xb3\x04\xaac\xadt\x01\xb2\x87\x02e\xf1\xb2\xb5\x99\x80\x00C\xd6\x80\x03\xb4\xd7\xadd\x92\xf1\xb2\xb5\x99\x80\x0c\x80\x04i\x92\xa7\x80\x03h\x87\x03k\xb2kas\xb5\x99\x80\x03v\x92\x9ec\x90\xab\x80\x02\xcb\x01m\x83\x01o\x87\x00DZ\x02c\xa6\x9f\x01a\x87\x02fr\xc2\x99\x01c\x9a\xc5\xb8\x02\x89\x00Dci\xf8\x9d\x8d\x01d\x94\x04\x8d\x01e\x80\x02\xc2\xd2\x9ec\x02l\xcdg\xa1\xd4\x01\xa3\x01i\x92\xa7\x08Acu\xb9\x08G\xb6\xc0\x02g\xd9\xdag\xa1\xd4\x01je\x80\x01l\x95e\x8d\x01m\x83\x01s\xb8\xbe\x01t\xd2b\xa6\x00Dz\x02c\xa6\x9f\x02e\xf1\xb2\xb5\x99\x80\x03\x80\x02\xd6\x80\x00Eb\xa1\xc0\x01c\x9a\xc5\xb8b\xa1\xc0\x02h\x87\x02\x89\x05\x9d\x98\x0b\x8d\x0b\xb3\x8d\x0b\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02\xae\x00Ed\xc8\xa2\x02\xe1\x04\x8d\x01f\x80\x01h\x87\x02\xcb\xa0\x01\xbc\xf6\xb7m\x99\x02nv\x92t\x9ab\xa1\xc0\x02o\x9e\xf7\x9a\x80\x01l\x80\x02ev\xad\xb7m\x99\x01m\xaar\x9f\x98\x07\xa2\x02\x80\x02\x83\x01n\x80\x00En\xb4\xd7\xadd\x92\x80\x02g\x03\xd6\x80\x02\xa3\x80\x01\xd2\xad\x01p\xb5l\x9ft\x9f\xc3\x01r\x80\x02ev\x92s\x9a\x09\x80\x01s\x80\x02\xb4\xd7\xadd\x92\x80\x02h\x01t\x87\x03t\x9f\xc3\x02\xc5\xb4\x06\x8d\x00Ezh\x03c\xa6\x9f\x03\xa1v\x92s\x9a\x00Fc\x89\x01d\x94\x01\xe7\x87\x02\xc2\xd2\x9ec\x01\xa3\x01i\xcd\x80\x02v\x92\xe2\x99\x01m\x83\x01our\xb7m\x99\x00GB\x86\x01\x98\x02m\xdaafr\xc2\x99\x02\xf2i\xaa\xd2\x9ec\x00Gc\xa6\x9f\x02\x9a\xc5\xb8\x02\x89\x05\x9d\x01\xb3\x04\xaac\xadt\x01e\x80\x01\xb2d\x87\x02\xdb\xf5\xaf\xa3\x80\x03\xab\x80\x03\xcet\xc7\x80\x02\xcb\x01im\x87\x01je\x80\x01m\xaar\x9f\x02\x83\x00G\xb6\xc0\x01\xc4\x90l\xa3\x02t\xb7\xd8\x00H18533\x0443\x0451\x0122073\x01P\x86\x01a\xf1\xb2\xb5\x99\x80\x02\xb4\xd7\xadd\x92\x80\x02rd\xb5\xfa\x80\x01b\xa6\x02\xa1\xc0\x8d\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x00H\xd0\x92\xa7\x02\x94\x04\x8d\x01m\x83\x01o\x87\x02r\xc2\xd2\x9ec\x01\xe0g\xa6\xed\xb8\xdf\x01z\x86\x00IA\x80\x01J\x01U\x80\x01b\xa1\xc0\x01c\xa6\x9f\x02\x89\x02\xae\x01d\xc8\xa2\x00I\xd0\x92\xa7\x98\x09\x80\x02\xe1\x04\x8d\x01eb\xa1\xc0\x80\x02\x80\x01\xa3\xa0\x01i\x80\x02nv\x92t\x9ab\xa1\xc0\x02\xbe\xbbt\x80\x01m\xaar\x9f\x80\x02\x83\x01ni\x87\x01o\x80\x02\xcdafr\xc2\x99\x04\xd0\x92\xa7\x00I\xe1\xa9\x9f\xc3\x01\xab\x01\x9el\xb4\x06\x8d\x01zh\xebsa\x80\x07d\xc8\xa2\x80\x00Ja\x87\x01c\x89\x05\x9d\x01e\x80\x01\xd6h\x87\x01m\x83\x00KB\x86\x01K\x86\x01aba\xbekir\x80\x02cu\xb9\x00Ka\x80\x02\xb4\xd7\xadd\x92\x80\x02\xa3\x80\x02\xab\x80\x02v\x92\x9ec\x90\xab\x80\x01c\xa6\x9f\x02\x9a\xc5\xb8\x02\x89\x01\xb3\x8d\x01\xe7\x87\x02n\x87\x01\xb2\x80\x02e\xc2\xd2\x9ec\x02\xcb\x01je\x80\x01l\x95e\x8d\x00Km\x83\x01\xd2pa\x80\x05g\xa1\xd4\x01\xb5\x80\x00LJ\x01L\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x8d\x01\xb3\x04\xaac\xadt\x04\x8d\x09m\xaar\x9f\x01iwn\x87\x01j\x02e\x80\x00Ll\x95e\x8d\x01m\x83\x00MB\x86\x01\xaar\x9f\x03u\xb9\x01c\x89\x01d\x94\x04\x8d\x01\xad\x87\x01m\x83\x01t\xc7\x9a\x00NJ\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x8d\x01d\x94\x00N\xb3\x8d\x01\xa3\x97\x01\x95\x92\xe2\x99\x01j\x02e\x80\x01l\x95e\x8d\x01m\x83\x01\xca\x87\x00Ob\xa6r\x9a\x80\x07\xd0\x92\xa7\x80\x02\xa1\xc0\x01c\xa6\x9f\x02\xadt\x92\x9a\x9el\xb4\x02\x89\x05\x9d\x98\x0b\xb3\x8d\x00Oci\xf8\x9d\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02\xae\x01d\xc8\x98\x04\xa2\x02i\x92\xa7\x80\x02\xe1\x8d\x01h\x87\x02m\x02\xcb\xa0\x03rn\x05\x98\x05\xb3\x8d\x05\xa2\x05\xa3\xa0\x00Oh\xbbn\x9el\xb4\x01i\x02nv\x92t\x9ab\xa1\xc0\x01m\xaar\x9f\x98\x07\xa2\x02ega\x80\x05g\xa1\xd4\x05\xb7\xe0d\x80\x05\x9etlo\x80\x06\x9f\xc3\x02\xc2r\x9ft\x9f\xc3\x02\x83\x01n\x92\xe2\x99\x01\xff\x9f\xd4\x07m\xaar\x9f\x02p\xad\x00Os\xb8\xbe\x98\x02t\xb7\xd8\x98\x01t\x80\x02\xc5\xb4\x98\x06\xd0\x92\xa7\x00P\x98\x01c\x89\x01d\x94\x01e\x80\x02h\x87\x02m\xf5\xaf\xa3\x80\x01\xa3\x01iwr\x87\x01m\x83\x01\xb5\x80\x00Qc\x89\x00Qm\x83\x00Ra\x87\x01c\x9a\xc5\xb8\x02\x89\x01d\xc8\xa2\x02\x94\x04\x8d\x09m\xaar\x9f\x01\xe7\x87\x01\x95v\x92t\x9ab\xa1\xc0\x01l\x95e\x8d\x01m\x83\x01\xc4\x90l\x95v\x92t\x9a\x0es\xce\x92i\xbb\x00SF01\x9c0\x032\x9c0\x00SF03\x9c0\x034\x9c0\x035\x9c0\x036\x9c0\x037\x9c0\x038\x9c0\x039\x9c0\x021\x9c\xec\x031\x9c0\x039\x9c0\x022\x9c\xec\x031\x9c0\x032\x9c0\x033\x9c0\x034\x9c0\x035\x9c0\x00SF26\x9c0\x037\x9c0\x038\x9c0\x0236\x9c0\x037\x9c0\x038\x9c0\x039\x9c0\x024\x9c\xec\x031\x9c0\x032\x9c0\x033\x9c0\x034\x9c0\x035\x9c0\x036\x9c0\x037\x9c0\x038\x9c0\x00SF49\x9c0\x025\x9c\xec\x031\x9c0\x032\x9c0\x033\x9c0\x034\x9c0\x01\x98d\x94\x02mp\xbc\xa1\xd4\x01c\xa6\x9fd\x94\x02hwa\x05\x80\x05\xd0\x92\xa7\x80\x02\x89\x05\x9d\x01d\x94\x04\x8d\x00S\xb3\x8dd\x94\x01\xe7\x87\x02v\xad\xb7m\x99\x01\xb2\x87\x03\x80\x02c\xb2\x80\x02e\xc2\xd2\x9ec\x02\xb2\x80\x02im\xaa\xd2\x9ec\x01ix\xb7m\x99\x01m\x83\x01oft\xb5\xfa\x80\x01\x9eg\xdag\xa1\xd4\x00Tb\xa6\x01c\x9a\xc5\xb8\x02\x89\x00Tci\xf8\x9d\x8d\x01d\x94\x04\x8d\x01e\x80\x02\xb4\xd7\xadd\x92\x80\x02n\xb7m\x99\x02t\xbf\x80\x01\xa3\x02\xa1\x92\xe2\x99\x01iwn\x87\x01l\x95e\x8d\x01m\x83\x01o\x87\x02ne\xf7\xc0\x04\xb5x\x04two\x00T\xa1t\xb7f\xafx\xa3\x01\xbf\x80\x02\xd6\x80\x01w\xf0v\x92\xe2\x99\x02o\xb7m\x99\x00Ub\xa1\xc0\x01c\xa6\x9f\x02\x89\x05\x9d\x8d\x02\xae\x01d\xc8\x98\x04\xa2\x02i\x92\xa7\x98\x09\x8d\x09c\xa6\x9f\x0a\xae\x00U\xd0\x92\xa7\xa2\x09m\xaar\x9f\x02\xe1\x8d\x01\xa3\xa0\x03rn\x05\x98\x05\xb3\x8d\x05\xa2\x05\xa3\xa0\x05\x9el\xb4\x02\xe0g\xa6\xed\xb8\xdf\x80\x01\x95v\x92t\x9ab\xa1\xc0\x01k\x80\x01m\xaar\x9f\x80\x07\xd0\x92\xa7\x02\x83\x00Up\xb5l\x9f\x98\xa3\xbdg\xa1\xd4\x08fr\xc2\x99\x07\xd0\x92\xa7\x0f\xa3\xbdg\xa1\xd4\x07\xa3\xbd\x07t\x9f\xc3\x01\xbe\xbbt\x80\x02t\xb6\xbc\xf6\x80\x09\xab\x80\x01\x9el\xb4\x06\x98\x06\x8d\x00Vc\x89\x01\xb3\x8d\x01e\x80\x02w\x87\x00V\xa3\x01m\x83\x01o\x87\x01\x9el\xb4\x00W\x98\x01c\x89\x05\x9d\x01\xd0\x92\xa7\x02\x94\x04\x8d\x01\xa2\x01m\x83\x00Xc\x89\x01\xd0\x92\xa7\x02\x94\x01\xe7\x87\x00Xm\x83\x00Y\xa9\x80\x01c\x89\x05\x9d\x01d\x94\x04\x8d\x01\x92i\x80\x03u\xd0\x92\xa7\x80\x01\xa2\x01\xa3\x05\xa0\x01i\x87\x02\x80\x02wn\x87\x01m\x83\x01\x9el\xb4\x00Yusb\xbc\x80\x06io\x9e\xf7\x9a\x80\x03l\xebt\xaf\x80\x09io\x9e\xf7\x9a\x80\x00Za\x87\x01c\x89\x05\x9d\x01\xb3\x04\x8d\x01e\x80\x02\xb4\xd7\xadd\x92\x80\x03i\x92\xa7\x80\x01\xd6\x87\x03b\xa1\xc0\x80\x03\x80\x03\xb4\xd7\xadd\x92\x80\x00Zh\x9ai\x92\xa7\x80\x01l\x95e\x8d\x01m\x83\x01\xab\x00aa\x8a\x02\x93\x02\x88\x04\x8f\x02m\xa9\xb6\xd3\x8f\x02ru\x86\x02\xa5\x8a\x0b\x93\x0b\x88\x01bb\xa1via\x9e\x9fm\xa6k\x87\x0c\xb5\xfa\x93\x02\xadg\x90i\x00ab\x96\x02\xa1\xc0\x98\x06\x80\x06\xb3\x8d\x06\xa2\x06\xa3\xa0\x06\x9el\xb4\x01c\xa6\x9f\x02\x89\x05\x9d\x98\x0b\xb3\x8d\x0b\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02u\xb9\x8d\xa4\x05\xa4\x00\x98c\xe2b\x05\x93\x05l\xcam\xba\x05t\x9fe\xa4\x02\xae\x01d\xc8\xa2\x02\xe5k\xd3\x8f\x02e\xdc\x02i\x92\xa7\x80\x09m\xaar\x9f\x02\xe1\x8d\x04m\xaar\x9f\x01e\x98\x02\x84\x02m\xaar\x9f\x01\xf7i\xec208\x00\xc108941\x041\xec17\x088\x089\x0720\x081\x082\x083\x084\x085\x086\x087\x088\x089\x0730\x081\x00\xc11\xec32\x083\x084\x085\x086\x087\x088\x089\x0740\x081\x082\x083\x084\x085\x086\x087\x00\xc11\xec48\x089\x0750\x081\x082\x083\x084\x085\x086\x087\x088\x089\x0760\x081\x082\x083\x00\xc11\xec64\x085\x086\x087\x088\x089\x0770\x081\x082\x083\x084\x085\x086\x087\x088\x089\x00\xc11\xec80\x081\x082\x083\x084\x085\x086\x087\x088\x089\x0790\x081\x082\x083\x084\x085\x00\xc11\xec96\x087\x088\x089\x061\xec\x081\x082\x083\x084\x085\x086\x087\x088\x089\x0710\x0745\x00\xc110146\x087\x088\x0792\x083\x084\x085\x086\x06831\x082\x0746\x04299\x043\xec\x061\x0457381\x088\x00\xc157392\x083\x084\x085\x086\x087\x088\x089\x064\xec\x081\x083\x087\x089\x0710\x081\x082\x00\xc157413\x084\x085\x086\x087\x088\x089\x0720\x081\x082\x083\x084\x085\x086\x087\x088\x00\xc157429\x0730\x081\x082\x083\x084\x0740\x081\x082\x083\x084\x085\x086\x088\x089\x0750\x00\xc157451\x082\x083\x084\x085\x086\x087\x088\x0770\x06505\x086\x087\x088\x089\x0711\x082\x00\xc157513\x084\x089\x0734\x06636\x0745\x0758\x0764\x085\x086\x087\x088\x089\x0770\x081\x082\x00\xc157673\x084\x085\x086\x087\x088\x089\x0780\x081\x082\x083\x084\x085\x086\x087\x088\x00\xc157689\x0790\x084\x085\x067\xec\x085\x0716\x087\x088\x0723\x0793\x084\x085\x086\x087\x088\x00\xc157799\x068\xec\x081\x082\x083\x084\x086\x087\x0739\x0741\x082\x06929\x0461248\x0789\x06352\x06573\x00\xc161574\x085\x06664\x053167\x054937\x01\x88\x03\x8f\x01\x8b\x02\xcb\xa0\x01i\x8a\x03\x96\x02\x93\x02e\x80\x02\x88\x04\x8f\x02m\xa9\xb6\xd3\x8f\x00a\x95\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x03v\x92t\x9ab\xa1\xc0\x02\xa5\x8a\x0b\x93\x0b\x88\x01\x85\x09\x8c\x02o\xa1\x99\x01\xaff\x04\x81\x04\xac\x82\x04f\x95\x90\x81\x04\xb2\xef\xa0\x81\x00\x90ef\xb2\xef\xa0f\x95\x90\x81\x09\x8d\x81\x0ef\x95\x90\x81\x05eb\xa1w\x04\xb8m\x9a\x82\x04\xdad\xe5\xa0\x81\x0ef\x95\x90\x81\x06ksu\xb6\x81\x0bf\x95\x90\x81\x0b\x95i\x9e\x90\x81\x0bm\x9ai\x90\x81\x04p\xa9\xfc\x82\x04q\xd9\xa9s\x82\x02\xaf\xc9\x90\x02\xe9\xa9\x9f\xc3\x01m\x83\x00\xd9p\x92s\x99dm\x83\x02\x86\x01nb\x96\x02gb\x96\x04r\xaa\xd8t\x97\x0eB\xbc\x11g\x0eb\xbc\x11g\x0a\x9b\x0fB\xbc\x12g\x0fb\xbc\x12g\x03kh\x99khu\x91\x03\xaf\xfb\xaa\xd8t\x97\x00\x99g\xaf\xfb\xaa\xd8t\x97v\x92\x9ec\x90\x0c\x9b\x11v\x92\x9ec\x90\x03\xf4\xb7m\x02o\xb9\xafia\x02ud\xa9\xcd\x93\x03sv\xa6a\x8a\x08\x93\x08\x88\x01pa\xa9o\x86\x03\xa1n\x02\xc3t\xb7p\xd6\x87\x0am\xba\x02p\xb7\xaa\xd6s\x05xe\xc9\x90\xbbi\xdage\x06im\xa9\xf0\xfd\xc9\x90\x00\xa6aeae\x84\x05\x84\x02c\x02\xbch\xd1\x90fr\x95g\x03\xf2\x98\x05\x8d\x02\xb7w\xe5\xbed\xcan\x09\x97\x09\x9b\x09\xce\x06\xcan\x97\x09\x9b\x09\xd5\xb9\x05\xd6add\xcanm\xba\x09\x97m\xba\x09\x9bm\xba\x00\xa6\xb7w\xd6ad\xcem\xba\x06\xcb\x97\x09\x9b\x05\x97bo\xd1\x90f\x09d\xc8\x0c\xab\x09ov\x92\x9b\x09t\xd2h\x90f\x09\xd5\xb9\x05n\xbb\xd1ea\xf4\x0awe\xf4\x05\x9bbo\xd1\x90f\x0ad\xc8\xab\x0a\xd6avy\x0aov\x92\x97\x0at\xd2h\x90f\x00\xa6\xb7w\x9b\xd5\xb9\x05sou\xd1ea\xf4\x0awe\xf4\x05\xcdb\x97\x08\x9b\x05\xcedn\x09b\xbf\x08\xcanba\xbf\x07\x97\x0bofd\xcan\x07\x9b\x07\xd5\xb9\x01\xd7i\xc2i\xf8\xedm\x83\x05\x9el\xb4m\x83\x03\xeept\x07t\xc7\x9a\x00a\xc4\x90l\x8b\x06\x85\x0e\x8c\x02t\x92\xf9k\x90t\x9fe\x81\x09\xb6b\xc2\x08m\x83\x08\xc4\x90l\x07m\x02ympto\x9ec\x90l\xfd\xc9\x90\x01tm\x83\x02\xc4\x90l\x02\xc7\x9a\x01u\x8a\x03\x96\x02\x93\x02\x88\x00au\xd3\x8f\x02l\xadg\xd1m\xa6k\x8a\x02m\xa9\xb6\xd3\x8f\x02\xa5\x8a\x0b\x93\x0b\x88\x01\xdcg\xb6\xb2\x93\x01yb\x87\x02\x95\x04\x90t\x9fe\x82\x04\x82\x00ba\x8a\x02cks\xb8\xbeB\xbc\x0cg\x09b\xbc\x0cg\x00b\xaaks\xb8\xbem\x83\x02\x93\x02\x88\x04\x8f\x02\x8b\x03t\x91\x02\x85\x02rd\xc8\x03m\x83\x01b\x96\x01c\x89\x01d\x94\x04\x8d\x01e\xd9\x9a\xb5x\xb9\xad\xd1\xea\xb9s\x02cau\xbf\x03\xae\x00b\xe7\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x04\xb6g\x99a\x03m\x9ai\x90\x81\x05\xdb\x95i\x9e\x90\x81\x08s\xcf\xa9\x9a\x81\x03\xea\x9ff\x95\x90\x81\x02\x85\x02n\x87\x02t\x03a\xbdg\xa1\xd4\x03\xac\x09\x82\x03\x82\x03\xb6fe\x82\x00b\xb2\x8a\x03\x93\x03\x88\x05\x8f\x02\xcb\x01i\x8b\x02\x85\x02\xb8bi\x90cl\xc2k\x02nd\xbcu\x8f\x02ru\x86\x01l\xaakc\x89\x05\xd0\xd9\x9fd\x06\xcan\xfe\x95t\x95g\xcc\x99g\xaf\x05\x97\xfe\x95t\x95g\xfe\x95t\x92\x11\xcc\x99g\xaf\x07n\x9ecul\xa6\xfb\xaa\xd8t\x97\x00\xc8\xaakl\xad\x9ecul\xa6\xfb\xaa\xd8t\x97v\x92\x9ec\x90\x16\x9b\x1bv\x92\x9ec\x90\x06\xca\x92\x97\xcc\x99g\xaf\x0a\x9b\xcc\x99g\xaf\x05\xa1ct\x99g\xaf\x06\xbc\xf6\xfe\x95t\x95g\xfe\x95t\x92\x12\xcc\x99g\xaf\x05\xc4\x90l\x86\x07\xc5\x95gf\xaae\x06\xc9a\xa1\x06t\xa6\x05\xcep\x92\x97\xcc\x99g\xaf\x0a\x9b\xcc\x99g\xaf\x08o\x95t\x95g\xc4\x90l\xcc\x99g\xaf\x0f\xcc\x99g\xaf\x00\xc8\x99k\x02\x95e\x8d\x02\xe4k\x01m\x83\x01obai\xdai\x91\x02\x8b\x02\x85\x01\x8e\x01q\x86\x01r\xaa\xe7\x9epd\xcan\x97\x0d\x9b\x09\xce\x97\x0b\x9b\x05\x97B\xbc\x0cg\x09b\xbc\x00\xfb\xaae\x97b\xbcg\x09m\x83\x09\xc4\x90l\x09v\x92\x9ec\x90\x05\x9bB\xbc\x0dg\x0ab\xbc\x0dg\x0am\x83\x0a\xc4\x90l\x0av\x92\x9ec\x90\x04\xd8t\x97B\xbc\x0eg\x0bb\xbc\x0eg\x0bm\xa9h\x00\xfb\xaa\xd8t\x97m\x83\x07\x9bB\xbc\x0fg\x0cb\xbc\x0fg\x0cm\xa9h\x0d\x83\x02e\xc0\x8d\xa4\x05\xa4\x05\x95v\x92t\x9a\x8d\xa4\x0d\xa4\x0ddo\xe8\xaf\xa4\x02idge\x8d\xa4\x06\x95v\x92t\x9a\x8d\xa4\x01\xab\x01t\xd2b\xa6\x00bu\x8b\x02\x85\x02l\xaft\x95v\x92\xbf\x06\xd2\x92\xa9\xbb\x04\xbf\xfd\x00ca\x87\x02\x8a\x02\x93\x02\x88\x04\x8f\x02l\x86\x02nd\xb6b\x95du\x8a\x0b\xa4\x0b\x93\x0b\x88\x02psl\xe4k\x00ca\xa1of\x03\x9f\x8d\xa4\x05\xa4\x01b\x96\x01c\x9a\xc5\xb8\x98\x02\x89\x05\x9d\x02\xf3\x01\xb3\x04\xaac\xadt\x02\x86\x01\x9a\xc5\xb8\xa4\x02\xc5\x95g\x97B\xbc\x0eg\x0bb\xbc\x0eg\x00ce\xc5\x95g\x9bB\xbc\x0fg\x0cb\xbc\x0fg\x02n\x9eg\xb6\xb4\x04m\x83\x01\xb2\x87\x03\x8a\x03\x93\x03\x88\x05\x8f\x02b\x96\x02e\xf1\xb2\xb5\x99\x80\x03ckm\xa6k\x04\xae\x03\xb4\xd7\xadd\x92\xf1\xb2\xb5\x99\x80\x00ch\x9ae\xd7\xadd\x92\x80\x04i\x92\xa7\x80\x03h\x87\x03k\xb2kas\xb5\x99\x80\x03v\x92\x9ec\x90\xab\x80\x02\xb0ch\xaa\x89\x84\x08\x8e\x84\x07c\x89\x84\x07\x84\x07\x8e\x84\x02\xe4h\x99g\x91\x07\x91\x05\x95g\x91\x05oe\x91\x03ok\x01\xb0c\xaa\x89\x84\x00c\xb0ca\x8e\x84\x05c\x89\x84\x05\x84\x05\x8e\x84\x05u\x8e\x84\x02\xf8\xaf\x06c\xd2yrt\x06\xb3\xb1\x09\xb9\xe6\x06mul\x9eply\xb1\x0e\xb9\xe6\x06\xe1\x06plus\xb1\x0a\xb9\xe6\x07\xc3t\x90m\xa6k\x06\xe3\xd1\x97h\x90f\xc8\xaak\x00c\x89\xe3\xd1\x9bh\x90f\xc8\xaak\x04\x9d\x8d\xa4\x0a\xa4\x01\xaf\xa6\x02\xc2k\x90\xc0\xcf\xa6\x05d\xadt\x90\x05l\xa9\x92\x90\x05\xa1t\xb7f\xafx\x02\xe8su\xeb\xc8\xaak\x08\xd5\xb9\x01mc\xe8\x9a\x86\x02\x83\x02\x86d\x86\x01o\x87\x02l\x9fm\x83\x05\xb5\xfa\x00c\xcf\x9f\xc4\x90l\x05\xcc\x99\xd3l\xa6h\x90fm\xba\x0fm\xba\x02m\xda\xa0\xa4\x0a\x9b\xa4\x06\xb6b\xc2\x07m\xadi\x99\x05m\x83\x05\xa1v\x92s\x9a\xa0\xa4\x0dm\xba\x05\xc4\x90l\x05t\xc7\x9a\xa0\xa4\x0bm\xba\x03pa\xdd\x02nt\x95\xb9gr\x90\xb1\x0c\xb9\xe6\x00c\x9ftour\x95\xb9gr\x90\x04\xb7l\x07ACK\x07BEL\x08S\x07CAN\x08R\x07DC1\x092\x093\x094\x08EL\x08LE\x07EM\x08NQ\x08OT\x00c\x9ft\xb7lESC\x08TB\x09X\x07FF\x08S\x07GS\x07HT\x07LF\x07NAK\x07RS\x07SI\x08O\x09T\x08TX\x08UB\x08YN\x00c\x9ft\xb7lUS\x07VT\x02p\xb7duct\xb1\x09\xb9\xe6\x02rn\x92\xfb\xaa\xd8t\x97\x11\x8c\x11v\x92\x9ec\x90\x0d\x9b\x12\x8c\x12v\x92\x9ec\x90\x03\xfe\xb6\x9e\x9f\x86\x02\x86\x02v\x92kg\x86\x01\x8e\x01ruzei\xb7\x01\xf4\xa1tch\x9a\x00c\xf3y\x99d\x05\xbb\x01yrB\xa1\xc0\x03F\xafx\x03b\xa1\xc0\x03f\xafx\x00\xe5\x87\x02\x8a\x02d\x81\x03e\xdc\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02ge\xbe\x06\x82\x03uj\xa6a\x9e\x00\xe5\xd3\x8f\x02\x8b\x02\x85\x02l\x81\x03et\x05\xac\x0b\x82\x05h\xa9afp\xa9\xfc\x0f\x82\x0a\xbfg\xcf\x0f\x82\x06eb\xa1w\x06i\xeeq\x0a\x82\x06o\xb8m\x0a\x82\x00d\x90etp\xa9\xfc\x0a\x82\x05q\xd9\xa9s\x0b\x82\x06\xe8\xdfs\x0b\x82\x05\xbfg\xcf\x0a\x82\x06\xd6\xdc\x0a\x82\x05ts\x92e\x0a\x82\x03f\x95\x90\x81\x02m\xda\x81\x05l\xca\x81\x05t\x99\x90t\x9fe\x81\x00d\xd9m\xa9\x99\x81\x02n\xe5\x02rga\x82\x05\x97\x82\x02\xb5apne\xed\xa9a\x80\xa4\x01\xc8G\xb6\xc0\x03\x99g\xaf\xfb\xaa\xd8t\x97\x13v\x92\x9ec\x90\x0f\x9b\x14v\x92\x9ec\x90\x04\xf8h\x95v\x92t\x9a\x8d\xa4\x05\xb7w\x97\x08\x9b\x03d\x99\xe5\x03\xa2\x08\xa4\x00d\xc8\x95\xb9gr\x90\x03l\xcal\x95e\x0a\xa4\x03ov\x92l\x95e\xa4\x03p\xeem\xdb\xba\x03v\x92\x9ec\x90b\xa6\x0bl\x95e\xa0\xa4\x02\x96\x02\x86\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x8d\x01\xe5\x8a\x03\x93\x03\x88\x05\x8f\x00dd\x90\x81\x04f\x95\x90\x81\x02d\xb2\x93\x02\xb2\x8a\x04\x93\x04\x88\x06\x8f\x02\x94\x04\x8d\x01ecim\x90\xbfp\xa6\xa9\xbb\x81\x10p\x92\xb5\x99\x03\xae\x02hi\x82\x04\xb6g\x99a\x02\xc2\xd2\x9ec\x02\x85\x00\xb4\xaf\xb9\x97\x06\x9b\x03t\xa9\xc7\x9a\x02n\xe2\x95\xa9\xbbm\x95us\x9f\xad\xed\x92\xa9\xbb\x8a\x02zh\x01\xb2\x8a\x03\x93\x03\x88\x05\x8f\x02\xcb\x01i\x90y\x9ek\xa9\x9f\xc3\x0e\xa4\x03m\x9fdsu\xeb\xd5\xb9\x02\x92\xa7\x98\x08\x8d\xa4\x08\xa4\x00\xd0\x92\xa7\xa2\x08t\x9f\xc3\x02\x8b\x02\x85\x02tt\xe2\xa6k\x02vi\xb4s\x04\xb5\x9fs\xb8\xbe\x01je\x80\x01ks\xb2\xb4\x01l\x95e\x8d\x02\x86\x01m\xaar\x9f\x02\x83\x01n\xc8\xe4k\x01\xe4\xb2\xe5\x91\x02\xb4k\x91\x00do\x8b\x02\x85\x02ll\xa6m\x83\x06\xc4\x90l\x02\xf2\x02ru\x86\x02t\xaac\xadt\xa4\x03\x8d\xa4\x09\xe2b\x03\x85\x03\xaf\xddj\x08\xab\xa3\x03t\x9ac\x89\x02\xe8\xafy\xbap\xa9\xfc\x0e\x82\x02wnt\xaak\x8d\xa4\x00d\xcant\xaakm\xba\x01\x8e\x01\xcd\xc5\x02\xd2b\xa6\x01u\x8b\x02\x85\x01z\x02\x90t\x9fe\x02c\xa6\x9f\x03\xf3\x02e\xf1\xb2\xb5\x99\x80\x03\x80\x02\xd6\x80\x00e\xa6\xd1\x01\x8a\x02\x96\x00eb\xa1\xc0\x01c\x99d\xb6\x93\x07\x88\x07\xa5\x93\x10\x88\x02\x9a\xc5\xb8b\xa1\xc0\x02h\x87\x03yiwn\x87\x02\x89\x05\x9d\x98\x0b\x8d\x0b\xb3\x8d\x0b\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02\xae\x00\x9a\xc8\xa2\x02e\xdc\x02\xe1\x04\x8d\x01e\xd3\x8f\x02m\xa9\xb6\xd3\x8f\x01f\x80\x01\x88\x01h\x87\x02b\x96\x02i\xb6g\x99a\x02\xcb\xa0\x01ib\x96\x02g\xf6\x81\x05\x8a\x05c\x89\x00e\xbc\xf6c\x89\x95v\x92\xbfs\x99\xdd\x92if\x05\x93\x05e\xadc\x89\x08\x8e\x09\x92i\xba\x05\x88\x07\x8f\x05h\xaak\x81\x07\xc6\x06\xea\xb9be\xd9\x9a\x05\xa8\x8e\x05m\x83\x05\x8e\x06\x92i\xba\x08\xb5\x99\x05\xb7m\x99\x00e\xbc\xf6\x91\x02nv\x92t\x9ab\xa1\xc0\x02o\x9e\xf7\x9a\x80\x01\x85\x09\x8c\x02\x9fk\xa6\xd3\x8f\x03\xa1\x99\x01l\x80\x02ev\xadc\x89\x06\x8e\x07\x92i\xba\x06\xb7m\x99\x02lip\xb5sv\x92\x9ec\x90\x01m\xaar\x9f\x98\x07\xa2\x02\x80\x00\xdb\xe5\xbev\x92\x9ec\x90\x02\x83\x02p\xb2\xb5\xc4\xa6k\x87\x01nb\x96\x02\x80\x02\xe5\xbev\x92\x9ec\x90\x03e\xd7\xadd\x92\x80\x02g\x03b\x96\x03\xd6\x80\x02\xa3\x80\x02sp\xaae\x01o\x84\x02p\xad\x05cl\xc3\x9a\x05\xa1v\x92s\x9a\x00e\xd2\xad\xa1v\x92s\x9acl\xc3\x9a\x0d\xa3\x01\x8e\x02\xb5l\x9f1\x07t\x9f\xc3\x01\xc9\x90m\xa9h\x06\x83\x05\xc4\x90l\x06\xce\x92i\xbb\x01rb\x96\x02\x80\x02ev\x92s\x9a\x09\x80\x01s\x80\x02\xb4\xd7\xadd\x92\x80\x02h\x00e\xbec\xf3\x03\xbbt\x93\x06\xa5\x93\x03\xa1v\x92s\x9alo\xd2\x03s\xc9\xa9\xa1v\x92s\x9a\x02m\x90l\x8b\x06\x85\x0e\x8c\x02\x9em\xa9\x9a\x01t\x87\x03t\x9f\xc3\x02\xc5\xb4\x06\x8d\x02n\xfc\xcdfoukh\x82\x0c\x97\x82\x07\x82\x00etna\xf6\x90eft\x82\x02\xc7\x9a\x01u\x84\x02\xb7\x01\xa5\x8a\x0a\x93\x0a\x88\x01xc\xb8m\x87\x06d\xc8\x06m\x83\x01zh\x03c\xa6\x9f\x04\xf3\x03\xa1v\x92s\x9a\x03\xcd\xc5\x00fa\x93\x00fa\xd3\x8f\x02h\xa1n\xd6\xeb\x02t\xb2\x81\x05l\xca\x81\x05t\x99\x81\x01b\x96\x01c\x89\x01d\x94\x01\xe7\x81\x05m\xadi\x99\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02\xc2\xd2\x9ec\x02m\x90e\x01if\xb9\xadc\x89\x00\xf7f\xb9\xad\x8e\x08\x92i\xba\x02ll\x9abox\x06\xa1ct\x02n\x90kaf\x08\xac\x0e\x82\x08\x82\x08q\xd9\xa9s\x0e\x82\x08\xbee\xdc\x0d\x82\x05m\xdb\x08\x82\x05n\xe0\x08\x82\x00f\x95\x90pe\x07\x82\x05tsa\xd0\x0a\x82\x02r\xf4t\x9fech\x95e\xbf\x02\xbee\xfd\x02\xcd\x80\x02\xc0\x81\x04\x8a\x04c\x89\x0a\x95v\x92\xbfs\x99\xdd\x92if\x04\x93\x04\x88\x06\x8f\x04h\xaak\x81\x06\xc6\x00\xf7\xc0\xa8\x8e\x04m\x83\x04\x8e\x05\x92i\xba\x07\xb5\x99\x04\xb7m\x99\x04\x91\x01lo\xbb\x97B\xbc\x0cg\x09b\xbc\x0cg\x05\x9bB\xbc\x0dg\x0ab\xbc\x0dg\x01m\x83\x00fm\x86\x01of\x99\x91\x04\x91\x02\xf2m\x99\x91\x02r\x90l\x02ur\x81\x04\x8a\x04c\x89\x0a\x95v\x92\xbfs\x99\xdd\x92if\x04\x93\x04\x88\x06\x8f\x04h\xaak\x81\x06\xc6\x04\xa8\x8e\x04m\x83\x00fo\xc7\xed\x92\xa9\xbb\x8a\x04\x8e\x05\x92i\xba\x07\xb5\x99\x04\xb7m\x99\x04\xb9\xadc\x89\x08\x8e\x09\x92i\xba\x05\xb2i\x06t\x9fech\x95e\xbf\x01\x8e\x01r\x99c\x00ga\x8a\x02cu\xb9\x02\x93\x02f\x81\x00gaff\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02\x88\x04\x8f\x02\x8b\x02\x85\x02mm\x90\xa9\x95\xc4\x90l\x05s\xce\x92i\xbb\x02\xf2i\xaa\xd2\x9ec\x01b\x96\x01c\xa6\x9f\x02\x9a\xc5\xb8\x02\x89\x05\x9d\x01\xb3\x00gd\x94\x01e\x80\x02\x8b\x02\x85\x02\xe2etr\xc2\x90l\xfd\xc9\x90\x02\xa1\xbe\xaac\xadt\x82\x06\x82\x06muqd\xd9\x82\x03s\xb2yim\xaac\xadt\x82\x09\x82\x02\xcdm\xa6k\x01\xb2\x8a\x03d\x87\x04e\xdc\x03\x88\x05\x8f\x00g\xb2\x95\x81\x05f\x95\x90\x81\x05\x95i\x9e\x90\x81\x05m\x9ai\x90\x81\x02\xdb\xf5\xaf\xa3\x80\x03\xab\x80\x03\xcet\xc7\x80\x02\xb2\x93\x04\xd3\x8f\x02\xcb\x02z\x86\x01i\x8b\x02\x85\x02m\x87\x03\xf0\x05\xac\x00gim\xf0\xac\x82\x05\x82\x01je\x80\x01l\xe1t\x90\x95v\x92t\x9a\xab\x07\xf4\xd2\x0b\x95v\x92t\x9a\x0bm\xba\x0b\xa1v\x92s\x9a\x13m\xba\x13s\xce\x92i\xbb\x0b\xab\x11\xa1v\x92s\x9a\x01m\xaar\x9f\x02\x83\x01o\x8b\x02\x85\x00g\x8e\x03\x86\x01\xb6\xc0\x8d\xa4\x05\xa4\x06\xe2b\x05\x93\x05l\xcam\xba\x05m\x83\x05t\x9fe\xa4\x02e\xa9\x92e\xc9\x90\xbb\xaf\xdd\x07m\x83\x07o\xa1\xc9iv\x90\xadt\x09\xaf\xdd\x08v\x92e\xc9\x90\x07\xc4\x90l\x01\xd7\xeept\x00g\xab\x01u\x8b\x02\x85\x02\xb6mu\x86\x01y\x86\x00\xb2\xf1\xb2\xb5\x99\x80\x03lt\x9fe\x81\x02\x8a\x02\xb4\xd7\xadd\x92\x80\x04\xdc\x02\x88\x04\x8f\x02h\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x04\xb6g\x99a\x00\xb2hm\x9ai\x90\x81\x02\xebu\x86\x02\x85\x0a\x8c\x02l\x99t\xd3\x8f\x02\xef\x81\x05d\xd9\xda\x81\x0at\x99\x81\x05f\xa9\xb2\x81\x0at\x99\x81\x05l\xca\x81\x08kas\xb6\x81\x0dt\x99\x81\x05suk\xe0\x81\x02n\xd3lf\xc5l\x92\x02rd\xb5\xfa\x80\x00h\xa6\xfe\x9f\x97b\xa6b\xce\x07\x9bb\xa6b\xce\x02\x86\x02\xcdfp\xa9\xfc\x0a16\x0a23\x0bf\x0a\x82\x0an\xa6\xb7w\x82\x0a\xc9\xa6t\x92\x82\x0a\xe3\xb4\x82\x05q\xd9\xa9s\x0b1b\x0b28\x0b34\x0b\x82\x00h\xa9afq\xd9\xa9sn\xa6\xb7w\x82\x0b\xc9\xa6t\x92\x82\x0b\xe3\xb4\x82\x05\xbfg\xcf\x0a17\x0a24\x0a30\x0a\x82\x0an\xa6\xb7w\x82\x0a\xc9\xa6t\x92\x82\x0a\xe3\xb4\x82\x03\xe3\xb4\x07r\x07\xf4\x01b\xa6\x02\x96\x00hb\xa1\xc0\x8d\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x01\xd0\x92\xa7\x02\x94\x04\x8d\x01e\x02\xa6tsu\xeb\xc8\xaak\x09\xd5\xb9\x02\xac\x08\x82\x02h\x90t\x9fe\x81\x04\xb6b\xc2\x03eb\xa1w\x03f\x95\x90\x90t\x9fe\x81\x00\xd6hf\x95\x90\x90ttwo\x81\x09\xb6b\xc2\x03\xb2\xef\xa0f\x95\x90\x81\x0d\xf9\xcf\xa9\x9a\x81\x03\x95i\x9e\x90\x90t\x9fe\x81\x0b\xb6b\xc2\x04\xb6g\x99a\x03m\x9ai\x90\x90t\x9fe\x81\x0a\xb6b\xc2\x02i\xbfi\x92a\x86\x02\x85\x0a\x8c\x03u\xcd\xa6u\x86\x02\xf2\xa3\x02r\xdfu\x86\x02t\x00\xd6t\x82\x01\xa3\x05s\xce\x92i\xbb\x01\xb0h\xaa\x89\x84\x06\x8e\x84\x05c\x89\x84\x05\x84\x05\x8e\x84\x02\x8b\x02\x85\x0a\x8c\x02\xeeq\x0514\x0521\x06d\x05\x82\x00hi\xeeqn\xa6\xb7w\x82\x05\xc9\xa6t\x92\x82\x05\xe3\xb4\x82\x01l\x95e\x8d\x01m\x83\x01o\x87\x02hip\x91\x04\xb6g\x99a\x02\x85\x0a\x8c\x02\xb8m\x0519\x0526\x0532\x05\x82\x05n\xa6\xb7w\x82\x00ho\xb8m\xc9\xa6t\x92\x82\x05\xe3\xb4\x82\x02\xeakhuk\x91\x02ok\xa0c\xe2b\x04\xa4\x04p\x90\xa9\x90iz\x9a\x8d\xa4\x04\xa1t\xb7f\xafx\x8d\xa4\x03n\x86\x02r\xc2\xd2\x9ec\x04z\x9ft\x90b\xa6\x03n\xa4\x02tspr\x95gs\x02u\xbf\x01\x8e\x01s\xce\x92i\xbb\x01t\xc7\x9a\x00hu\x8b\x02i\xebo\x86\x02\x85\x0a\x8c\x02\xf2\xa6\xed\xb8\xdf\xa4\x01v\x01y\xe9\xadm\x83\x06\xc4\x90l\x06two\x00ia\x80\x01\x8a\x02\x96\x02\xa1\xc0\x01c\xa6\x9f\x02\x89\x02\xae\x00id\xc8\xa2\x02e\xff\xb6p\xd6\xa6\xd1c\x89\x09\xf7\xa1c\x89\x09\xc2\x90li\x99ce\x8e\x0bc\x90l\x8e\x0c\xadt\xa1c\x89\x0clo\xbf\x0c\xe2\xda\x10\x97\x0d\xf2r\xa9u\xb8\x9e\x9f\x8e\x0dr\xa1ctc\x89\x0be\xa6\xd1\x8e\x0cnt\x92p\xee\xbf\x8e\x0cxc\xf0l\xadtc\x89\x0bfes\x9ev\x90\x8e\x0c\x95\x99ci\x90c\x89\x00\xa8f\x95\x99ci\x90\x8e\x0d\xa1\x8e\x0b\xb2\xc0\x8e\x0c\xbchc\x89\x0b\xeb\x92a\x9e\x9fm\xa6k\x0b\xb8b\xbbc\x89\x10\x8e\x0ceftc\x89\x0c\xcac\x89\x0bm\x9a\xc2\x95ec\x89\x0dt\x90\x8e\x0co\x9f\x8e\x0bn\xd9e\x8e\x0bp\x92i\xba\x0cr\x95tc\x89\x0b\xa1\xaah\x8e\x00\xa8\xa1p\xa1s\xadt\x8e\x0dsou\xf8e\x8e\x0c\xbc\xf6c\x89\x0b\xbfc\xa1tc\x89\x0dlf\x8e\x0c\xe4iety\x8e\x0cp\xaae\x0deci\x90\x8e\x0ct\xe4k\x8e\x0dudy\x8e\x0c\xe0\x8e\x0dp\x92vi\xbf\x8e\x0bw\xa9\x92\x8e\x0co\xba\x8e\x0bz\x92o\x09met\x90c\x89\x00i\xb4\xff\xb6\xe9mo\x9fc\x89\x09n\xd9ec\x89\x09s\xe0c\x89\x09w\xa9\x92c\x89\x0ao\xbac\x89\x03\xdc\x02i\x92\xa7\x98\x09\x80\x02\xe1\x8d\x01eb\xa1\xc0\x80\x02\x80\x02\xe0g\xaa\x89\x84\x06\x8e\x84\x05c\x89\x84\x05\x84\x05\x8e\x84\x00i\x88\x03\x8f\x01\x8b\x02\xcb\xa0\x01i\x8a\x02\x80\x02\x93\x02\x88\x04\x8f\x02m\xa9\xb6\xd3\x8f\x02nv\x92t\x9ab\xa1\xc0\x02\xbe\xbbt\x80\x02\xa5\x8a\x0b\x93\x0b\x88\x01j\x00i\x85\x09\x8c\x02o\xa1\x99\x01l\xb4\x02uy\x82\x01m\xaar\x9f\x80\x03geo\xb6pp\xb7xim\xa9\xf0\xfd\xc9\x90\x03t\xb6\xd3\x8f\x02\x83\x01nc\xa1m\xadt\x02i\x87\x02\xb9gr\x90b\xe1t\xe2\x08\xb1\x08\xb9\xe6\x09\xd2\x04\xf8\x90\x00\x95t\x92\xbfc\x9e\x9f\xb1\x0c\xb9\xe6\x03i\x86\x02vbul\xaft\x03c\x89\x03\xc4i\xaff\xaae\x01o\x80\x02\xcd\xd0\x92\xa7\x0ct\x9f\xc3\x04l\xa9\x95\x04t\x9f\xc3\x01\x8e\x01r\xbcu\x8f\x01\xc4\x90l\x8b\x06\x85\x0e\x8c\x00\xf9\xbe\xa6\x8a\x02t\xb7\xd8\x01t\x92a\x9e\x9f\x8b\x09\x85\x02\xc5\xb4\x06\x8d\x01\xe8\x96\x02\x80\x01\xa5\x8a\x0a\x93\x0a\x88\x01zh\xebsa\x80\x07d\xc8\xa2\x80\x00ja\x87\x02\x8a\x02\x93\x00ja\x88\x04\x8f\x01b\x96\x01c\xa6\x9f\x02\x89\x05\x9d\x02\xb7\xdd\x9a\xcd\xc5\x01\xb3\xaf\xdd\xab\x01e\x80\x02\xdb\x81\x04f\x95\x90\x81\x04\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x02h\x81\x03f\x95\x90\x81\x01\xb2\x8a\x00j\xb2\x93\x03\x88\x05\x8f\x02\xe7\x87\x01\xf9\x01m\x83\x01\x8e\x01s\xce\x92i\xbb\x00kaba\xbekir\x80\x03\xadg\x90i\x02cu\xb9\x03\xae\x02\xb4\xd7\xadd\x92\x80\x04\xdc\x02f\x03\x81\x00kaf\xac\x09\x82\x03f\x95\x90\x81\x03\x82\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x03\xb6fe\x82\x02\x88\x04\x8f\x02\x8b\x03\xcb\x80\x02\x85\x0a\x8c\x02ppa\xbdg\xa1\xd4\x03\xfdo\xe0m\xb0m\x84\x08\xe9\xb0\xe9\x84\x00kap\xfdo\xe0p\xb0p\x84\x08\xdd\x99gp\xb0p\x84\x02\xb7\xeei\x86\x02\xbei\xe5a\xdfo\x81\x0b\xea\xb5\xb4be\xa6\x95g\x81\x03m\x90l\x85\x03\xc9a\xa1\x03\xb6\x81\x05t\x99\x81\x03t\xb7\xd8\x80\x02\xcdhi\xb6p\xb7l\x9fgm\xa6k\x8c\x02v\x92\x9ec\x90\xab\x80\x01b\x96\x01c\x90\x86\x03r\x9f\x02\x9a\xc5\xb8\x00kc\x89\x01\xb3\x8d\x01\xe7\x87\x03i\xb6g\x99a\x02\x85\x0a\x8c\x02n\x87\x02\xc4\x90l\x85\x01g\xa1\xadl\x99d\xc2\x01\xb2\x8a\x03\x80\x03\x93\x03\x88\x05\x8f\x03h\x81\x04f\x95\x90\x81\x00k\xb2h\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x02e\xc2\xd2\x9ec\x02\xb2\x93\x04\xd3\x8f\x02\xb0kh\xaa\x89\x84\x08\x8e\x84\x07c\x89\x84\x07\x84\x07\x8e\x84\x02ok\xb2i\x91\x05\x9f\x91\x05u\xa9\x91\x05wai\x91\x03m\xdf\x91\x03ok\x00kho\xb6kh\x99g\x91\x02z\x86\x01i\x8b\x02\x85\x0a\x8c\x02\xb7\xd3\xb6mu\x86\x04meet\xbbu\x86\x04\x86\x02\xfdok\xaa\x89\x84\x07\x8e\x84\x06c\x89\x84\x06\x84\x06\x8e\x84\x06\xb5\xc3\x84\x01je\x80\x01l\x95e\x8d\x00kl\x86\x01mc\xe8\x9a\x86\x02\x83\x02\x86d\x86\x01o\x8b\x03m\x86\x02kai\x91\x04\xcdk\x99a\x0a\x8c\x02\xd2o\x86\x02ppa\x80\x02\xa1\x99\xf4\x99d\xa6d\xbd\x03\x9f\xf9\xa4\x01\x8e\x03\x86\x01\xb5\x80\x00kt\x86\x02\xc7\x9a\x01u\x8b\x02\x85\x0a\x8c\x01v\x86\x01w\x86\x00\xb8\x8a\x02\x93\x02\x88\x04\x8f\x02kkh\x99gyao\x91\x02m\x90eff\x95\x90\x81\x07\xb2\xef\xa0f\x95\x90\x81\x11\xf9\xcf\xa9\x9a\x81\x0c\x8df\x95\x90\x81\x00\xb8m\x90ef\xb2\xef\x8d\xf9\xcf\xa9\x9a\x81\x07\xf9\xcf\xa9\x9a\x81\x07\xdad\xe5\xa0f\x95\x90\x81\x11\xf9\xcf\xa9\x9a\x81\x04\xb6b\xc2\x03b\xe5\xab\x03\x9a\x05\xac\x0b\x82\x05\x82\x06o\xb8m\x0a\xac\x10\x82\x0a\x82\x03f\x95\x90\x81\x03\xb2h\x95i\x9e\x90\x81\x00\xb8m\x95i\x9e\x90\x81\x03je\xdb\x95i\x9e\x90\x81\x03k\xb2h\x95i\x9e\x90\x81\x03\xb8m\xd6h\xf9\xcf\xa9\x9a\x81\x03m\x9ai\x90\x81\x05\xdb\xb2h\x95i\x9e\x90\x81\x07\x95i\x9e\x90\x81\x07je\xdb\x95i\x9e\x90\x81\x07k\xb2h\x95i\x9e\x90\x81\x02rgec\x89\x01b\xa6\x02\xf0t\x02\x96\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x8d\x00l\xb3\x04\xaac\xadt\x04\x8d\x09m\xaar\x9f\x01eft\x99g\xaf\xa0\xa4\x04t\xaak\x8d\xa4\x02s\xbf\xc9\x90\xbbg\xa1\xa9\x92\x04m\x83\x04o\xa1\xc9iv\x90\xadt\x06g\xa1\xa9\x92\x05v\x92e\xc9\x90\x04\xc4\x90l\x02zh\x01f\xc8\xe4k\x01\xa3\xa1t\xb7f\xafx\x01i\xb6\x00liwn\x87\x01j\x02e\x80\x01l\x02a\x93\x03\x88\x02\x95e\x8d\x02\xb8\x93\x02v\xe4\x90\xc2\x8a\x09\x93\x09\xa5\x8a\x12\x93\x01m\xf5\xaf\x9el\xb4\x02\x83\x02\x86\x01\xe4hu\xb8\x91\x00l\xff\xc2\x90\x99d\xb1\x0a\xb9\xe6\x07n\xe1\xa1v\x92s\x9a\x07\xbb\xb1\x09\xb9\xe6\x02l\x95g\x91\x02\xf2s\x02wl\x95ec\xadt\x92l\x95e\x08mb\x07\xe5\xbe\x9a\x01\x8e\x01\x86\x01ts\xb2\xb4\x01u\x91\x01v\xe4\x90\xc2\x8a\x08\x93\x00lv\xe4\x90\xc2\xa5\x8a\x11\x93\x01x\x86\x00\xda\x8a\x02cr\x9f\x8d\xa4\x06\xa4\x06l\xcam\xba\x06m\x83\x03u\xb9\x02\x93\x02\x88\x04\x8f\x02\xb2pakh\x82\x08\x97\x82\x03i\xb6g\x99a\x02\xc2h\xa9\xcdw\x90\xca\x97\x91\x00\xda\xc2h\xa9\xcdw\x90\xca\x9b\x91\x0b\x91\x0b\xcep\x92\x97\x91\x03\xd4l\xca\x97\x91\x08\x9b\x91\x05\x91\x05\xcep\x92\x97\x91\x03h\x99ak\xa9\x97\x91\x0a\x91\x03\xcdikhu\x97\x91\x09\x91\x04h\xcf\xca\x97\x91\x09\x9b\x91\x06\x91\x06\xcep\x92\x97\x91\x04r\xc5\xca\x97\x91\x00\xda\xebr\xc5\xca\x9b\x91\x06\x91\x06\xcep\x92\x97\x91\x03y\xd9ok\x91\x02\x85\x0a\x8c\x02\xaf\x02nsy\x9f\x86\x02qaf\x82\x02rs\x02s\xbb\xaa\x89\x82\x03\xc9a\xa1\x01b\x96\x02\x86\x01c\x89\x02\xe8\x9a\x86\x00md\x94\x04\x8d\x01e\xdb\x81\x04f\x95\x90\x81\x04\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x06\xdb\x95i\x9e\x90\x81\x09s\xcf\xa9\x9a\x81\x03t\xbbu\x86\x02\x8b\x02izi\x92a\x86\x02\x85\x0a\x8c\x02m\x03\xac\x09\x82\x00m\xdb\x82\x02n\x87\x02rk\xb2\x82\x06\xd8fu\xb8\x82\x0c\x97\x82\x06\x97\x82\x01\xa3\x02z\x86\x01\xf5l\x9a\xe1\x85\x8c\x04\xe1\x02e\xed\xaa\x89\x84\x06\x8e\x84\x05c\x89\x84\x05\x84\x05p\x99\xb5\xc3\x84\x07\xa1n\x84\x00m\xb0mp\xb0p\x84\x05\xb5\xc3\x84\x02\x8b\x02\x85\x0a\x8c\x02nus\x8d\xa4\x05c\x89\x05m\xba\x05plus\x02\xeeba\xa6u\x86\x04\x86\x01l\x9fg\xafgt\xc7\x9a\x02\x86\x01mc\xe8\x9a\x86\x02\x83\x02\x86d\x86\x00mo\x8b\x03m\x86\x02\x85\x0a\x8c\x02l\x86\x02\xda\x91\x02v\x92s\x86\x0cd\x86\x01\x8e\x03\x86\x01s\x86\x01t\xc7\x9a\x01u1\x02a\x86\x02chg\xa1\xa9\x92\x04\xaf\xdd\x00muf\x86\x02g\xa1\xd4\x03\x86\x02\x8b\x02\x85\x0a\x8c\x02l\x86\x02m\x86\x02n\xfc\x82\x05\x97\x82\x02\xb5c\x90\xea\xb9\x0bd\xc8\x05fl\xa9\xb5\xfa\x05\xbe\xa6p\xb5\xfa\x03\x86\x02v\x86\x00muw\x86\x01vmega\x86\x02\x86\x01wmega\x86\x02\x86\x00na\x8a\x03\xb8\x02\x93\x02\x88\x04\x8f\x02\x8b\x02\x85\x0a\x8c\x02p\xc3t\xb7p\xd6\x02\x86\x01b\x96\x00nbsp\xaae\x01c\x9a\xc5\xb8\x02\x89\x05\x9d\x8d\x01d\x94\x04\x8d\x01e\x8b\x02\x85\x0a\x8c\x02w\xbeeq\xf0\xb5\xfa\x01f\x86\x01ga\x8a\x03\x93\x03\x88\x05\x8f\x02\x9f\xd3\x91\x00n\x8b\x02\xcb\x97\x05\xa1t\xb7f\xafx\x01\xb0n\xaa\x89\x84\x06\x8e\x84\x05c\xb0c\x84\x07\xf8\xaf\x84\x05h\xb0h\x84\x05\x84\x05p\x99\xb5\xc3\x84\x07\xa1n\x84\x05\xb5\xc3\x84\x05\x9e\xd8\xdf\x84\x02\x8b\x02\x85\x0a\x8c\x00nik\xb2h\xeb\x97\x91\x08\x91\x02ne\x81\x04\x8a\x04c\x89\x0a\x95v\x92\xbfs\x99\xdd\x92if\x04\x93\x04\x88\x06\x8f\x04h\xaak\x81\x06\xc6\x04\xa8\x8e\x04m\x83\x04\x8e\x05\x92i\xba\x07\xb5\x99\x00n\x95\x92\xe2\x99\x04\xb9\xadc\x89\x08\x8e\x09\x92i\xba\x05\xb2i\x01j\x02e\x80\x01\x85\x09\x8c\x01\xafg\x9bl\x9fg\x02\x95e\x8d\x01m\x83\x02\x86\x01na\x8a\x03\x93\x03\x88\x00nna\xd3\x8f\x02na\x93\x01o\x8b\x02\x85\x0a\x8c\x02nb\xa1ak\x95gsp\xaae\x03\xad\x91\x03u\x91\x02\x9f\x81\x04f\x95\x90\x81\x04gh\xe0na\x81\x0af\x95\x90\x81\x04\xd6h\x95i\x9e\x90\x81\x04\x95i\x9e\x90\x81\x04je\xdb\x95i\x9e\x90\x81\x09s\xcf\xa9\x9a\x81\x00\xea\x9fm\x9ai\x90\x81\x06\xdb\x95i\x9e\x90\x81\x09s\xcf\xa9\x9a\x81\x04\xea\x9ff\x95\x90\x81\x02tc\x9f\xcd\x95s\x03e\xafm\xadtof\x03g\xa1\xa9\x92\x0a\xea\xa1\xc9\x90\x0d\xaf\xdd\x03id\xad\x9ec\x90\x03\xaf\xdd\x07\xea\xa1\xc9\x90\x03p\xa6\x90\xafl\x04\xa1c\x9aes\x03succe\x9as\x05p\x92\xbft\x00n\xca\x87\x01\x8e\x01s\x86\x01u\x8b\x02\x85\x0a\x8c\x03\xcd\x8a\x05\x93\x05\x88\x07\x8f\x02mb\x92\xb5\xfam\x83\x0a\xc4\x90l\x03\x92\x90\xb5g\xf2\xa1\xd4\x0bl\xca\x92g\xa1\xd4\x05o\x02n\x00n\xe0\xac\x09\x82\x03\x82\x01v\x86\x01w\x86\x01ya\x8a\x03\x93\x03\x88\x05\x8f\x00o\x99g\x91\x01b\xa6r\x9a\x07\x80\x07\xd0\x92\xa7\x80\x02\xadg\x90i\x02\x96\x02\xa1\xc0\x00\xe4\x99d\xb6\x93\x07\x88\x07\xa5\x93\x10\x88\x03r\x9f\x02\x89\x05\x9d\x98\x0b\xb3\x8d\x0b\xa2\x0b\xa3\xa0\x0b\x9el\xb4\x02\xae\x01d\xc8\x98\x04\xa2\x02e\xdc\x02i\x92\xa7\x80\x00o\xb3\x8d\x01e\x84\x01g\x9f\xd4\xa4\x02uj\xa6a\x9e\x01h\x87\x02i\xb6g\x99a\x02\xcb\xa0\x03rn\x05\x98\x05\xb3\x8d\x05\xa2\x05\xa3\xa0\x05\x9el\xb4\x01i\x02nv\x92t\x9ab\xa1\xc0\x01\x85\x00o\x85\x8c\x02o\xa1\x99\x01\xaf\x82\x01m\xaar\x9f\x98\x07\xa2\x02\x93\x02ega\x80\x05l\xa9\x95cl\xc3\x9a\x05\xb7\xe0d\x80\x05\x9etlo\x80\x06\x9f\xc3\x02\x88\x02\xc2r\x9ft\x9f\xc3\x02\x83\x01ne\x81\x03\x8a\x00\x9fec\x89\x09\x95v\x92\xbfs\x99\xdd\x92if\x03\x93\x03\x88\x05\x8f\x03h\xaak\x81\x05\xc6\x03\xa8\x8e\x03m\x83\x03n\xed\x92\xa9\xbb\x8a\x03\x8e\x04\x92i\xba\x06\xb5\x99\x03\xb7m\x99\x03\x91\x01\xff\x9f\xd4\x00o\xff\x9f\xd4m\xaar\x9f\x03u\x8f\x02m\xa9\xb6\xd3\x8f\x02p\xad\x01\x8e\x02\xadbul\xaft\x02\x9e\x9f\x01r\xd1\xff\x9f\x90\x01\xbe\xbbt\x93\x06\xa5\x93\x02\xb8\xbe\x98\x02m\x90l\x8b\x06\x85\x0e\x8c\x02t\xb7\xd8\x98\x01t\x80\x00o\x9el\xb4\x98\x06\xd0\x92\xa7\x01\xe8\x96\x01v\x92l\x95e\x08c\xadt\x92l\x95e\x09mb\x08\xe5\xbe\x9a\x09\xc8wavy\x08wavy\x04\xd7o\xa1\x02\xca\xf0\xb5\xfa\x8a\x0a\x93\x0a\x88\x00pa\xd9ps\x86\x03s\xadto\x86\x02\x8a\x00p\x98\x02\x93\x02g\x9a\xcan\x04\xce\x03uj\xa6a\x9e\x04\x8f\x02\x8b\x02iy\x99\xeai\x91\x02\x85\x02l\xa9\x90iza\x9e\x9f\x80\xa4\x03\xe4hka\x80\x02n\xb5\xc3\x84\x02r\x90\xafl\x03\xad\x97B\xbc\x0cg\x09\x90t\x9fe\x81\x00\x8e\x97b\xbc\x0cg\x09m\xa9h\x0a\x83\x09\xc4\x90l\x09v\x92\x9ec\x90\x05\x9bB\xbc\x0dg\x0a\x90t\x9fe\x81\x0ab\xbc\x0dg\x0am\xa9h\x0b\x83\x0a\xc4\x90l\x0av\x92\x9ec\x90\x02\xbfq\x82\x00pa\xbe\xcd\x82\x03\xc9a\xa1\x02\xcdh\x0511\x06d\x052a\x05\x82\x05n\xa6\xb7w\x82\x05\xc9\xa6t\x92\x82\x05\xe3\xb4\x82\x02z\x92\x82\x01b\x96\x01c\x89\x01d\x94\x01e\x02\x80\x00p\x9aage\xbe\x08\x82\x02ezi\x86\x02f\x95\x90\xac\x82\x02h\x81\x05m\xadi\x99\x03eb\xa1w\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x04\xb6g\x99a\x03m\x9ai\x90\x81\x02\x85\x02m\xf5\xaf\xa3\x80\x02\xb6fe\x82\x03c\xadt\x81\x07m\x83\x00p\x92c\xadt\xc4\x90l\x03i\xba\x87\x06\x8c\x06m\x83\x06\xc4\x90l\x04sp\xe2\xad\xbc\xa1\xd4\xa4\x02\xbf\xcd\x01f\x86\x01\xb2\x8a\x03\x93\x03\x88\x05\x8f\x02\xb0\xe9\xaa\x89\x84\x08\x8e\x84\x07c\x89\x84\x07\x84\x00\xe9\xb0\xe9\x8e\x84\x03l\xa9\x95\x03n\xd1u\x91\x03\xbdg\xa1\xd4\x02\xcb\x03\xe9\x99\x91\x05\xe0g\x91\x03s\xd9p\xb2o\x91\x01\xb0p\xaa\x89\x84\x06\x8e\x84\x05c\xb0c\x84\x07\xf8\xaf\x84\x05\xde\x84\x06o\xa1\x99\x05\x8e\x84\x05\xb5\xc3\xde\x84\x00p\xb0p\xb5\xc3\x84\x09\x9e\xd8\xdf\x84\x05\xd1\xb0\xd1\x84\x06i\xd8\xdf\x84\x02\x8b\x02\x85\x02\xbdg\xa1\xd4\x02wr\x87\x01l\x99ckov\x922pi\x0d1\x02us\x8d\xa4\x04c\x89\x04m\xa9h\x05\xba\x06n\xc3p\xaae\x04\xc4\x90l\x00plu\xdd\xce\x92i\xbb\x01m\x83\x02\x86\x01o\x8b\x02\x95t\x95g\x95\xb4xd\xcan\xd5\xb9\x0d\x97\xd5\xb9\x0d\x9b\xd5\xb9\x0d\xce\xd5\xb9\x02\x85\x02p\xb8\x91\x02\xf4\x90m\xa6k\x0af\xaae\x01\x8e\x01\xa1c\x9aes\x03\xd7\xeep\x9e\x9f\x02ime\x00p\xeem\xdb\xba\x05\xa1v\x92s\x9a\x02\xbauct\xb1\x07\xb9\xe6\x03jec\x9e\xc0\x03l\x9fg\x9ak\x99a\x03p\xf0l\xbb\x04\xbb\x9e\x9f\x01\xb5\x80\x03lipne\xed\xa9a\x80\xa4\x02\x86\x01u\x8b\x02\x85\x01v\x86\x01w\x86\x00qa\x93\x00qad\xda\x82\x02f\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02m\xa9s\x0610\x07a\x07c\x0627\x079\x0633\x06\xb4\x06\x82\x06n\xa6\xb7w\x82\x06q\xa9\x99\x82\x00q\xd9\xa9sq\xa9\x99n\xa6\xb7w\x82\x0b\xc9\xa6t\x92\x82\x0b\xe3\xb4\x82\x07u\xa6t\x92\x82\x06\xe3\xb4\x82\x02rneyp\xa6a\x82\x01b\x96\x01c\x89\x01\xa3\x01m\x83\x01of\x03\xac\x09\x82\x03h\xa9afp\xa9\xfc\x0d\x82\x08\xbfg\xcf\x00qofh\xa9af\xbfg\xcf\x82\x04eb\xa1w\x04i\xeeq\x08\x82\x04o\xb8m\x08\x82\x03p\xa9\xfc\x08\x82\x03q\xd9\xa9s\x09\x82\x04\xe8\xdfs\x09\x82\x03\xbfg\xcf\x08\x82\x04\xd6\xdc\x08\x82\x00qofts\x92e\x08\x82\x01\x8e\x01u\xa6t\x92\xea\xb9\x02b\xdfs\x0618\x0625\x0631\x06\x82\x06n\xa6\xb7w\x82\x06\xc9\xa6t\x92\x82\x06\xe3\xb4\x82\x02es\x9e\x9f\x81\x0am\xadi\x99\x08g\xa1\xd4\x08m\x83\x00\xc9\xe1\x9a\xc8m\x83\x08p\xeeme\x0d\xa1v\x92s\x9a\x05\x97\xa1v\x92s\x9a\x05\xa1v\x92s\x9a\x06\xbc\xf6n\x05s\x95g\xafm\x83\x00\xb6\x87\x02\x8a\x02\x93\x03\xc2\x90B\xbc\x0ag\x07b\xbc\x0ag\x08t\x07tp\x00\xb6d\xc2\x90v\x92\xb9x\x03ov\x92s\x86\x0ed\x86\x03\x86\x02fe\x04\x82\x02\x88\x04\x8f\x02\x8b\x02\x85\x0a\x8c\x02l\xca\x92\xd0ag\x9f\x90\x8a\x02m\xf5l\x9aiag\x9f\x90\x8a\x03\xbe\xbbn\x02\x9eo\x01b\x96\x00\xf8\x9a\xc5\xb8\x02\x89\x01d\xc8\xa2\x02\x94\x04\x8d\x09m\xaar\x9f\x01ef\x92\xadc\xdb\xa6k\x02h\x81\x05m\xadi\x99\x03f\x95\x90\x81\x03i\xb6g\x99a\x03y\xe7\x90ef\xb8m\x81\x02\x85\x0a\x8c\x02\xbe\x04\xac\x82\x00\xa1\xbeh\xa9afp\xa9\xfc\x0e\x82\x09\xbfg\xcf\x0e\x82\x05eb\xa1w\x05i\xeeq\x09\x82\x05o\xb8m\x09\x82\x04p\xa9\xfc\x09\x82\x04q\xd9\xa9s\x0a\x82\x05\xe8\xdfs\x0a\x82\x04\xbfg\xcf\x00\xa1\xbe\xbfg\xcf\x82\x05\xd6\xdc\x09\x82\x04ts\x92e\x09\x82\x02v\x92s\x9a\x9el\xb4\x03ia\x82\x05mug\xb6\xbe\x82\x03l\xff\xc2\x90n\xe1\x01\xf7\xbe\xa3\x09\xa1v\x92s\x9a\x01\xb2\x8a\x03\x93\x02\xcb\x05t\xc7\x9a\x0bs\xce\x92i\xbb\x00rho\xbdg\xa1\xd4\x03\x9ec\xa3m\xba\x01\xb0l\xaa\x89\x84\x06\x8e\x84\x05c\x89\x84\x05h\xb0h\x84\x05\xde\x84\x0b\xb5\xc3\x84\x06o\xa1\x99\x05m\xb0m\x84\x05p\x99\xb5\xc3\x84\x07\xa1n\x84\x06h\xb0\xe9\x84\x06\xb0p\x84\x0a\xb5\xc3\x84\x05\xb5\xc3\x84\x00r\xb0l\xd1\xb0\xd1\x84\x06i\xd8\xdf\x84\x05\xfd\xbb\x95h\xb0h\x84\x02g\xf6\x99g\xaf\x05t\xaak\x8d\xa4\x06\xee\x99g\xaf\x02\x8b\x02\x85\x0a\x8c\x02\xf2\x8d\xa4\x04\xa4\x04h\x90f\x97\x0c\x87\x0c\x8d\xa4\x0cc\xadt\x92\x9a\x08\x9b\x00r\x95gh\x90f\x9b\x8d\xa4\x0dc\xadt\x92\x9a\x03v\x92t\x9ab\xa1\xc0\x02tt\xbbu\x86\x01l\x95e\x8d\x02\x9fg\xafg\x08t\xc7\x9a\x01m\x83\x01o\x8b\x02\x85\x0a\x8c\x02rua\x91\x01\x8e\x01\xb6\x8a\x03\x93\x03\xd3\x8f\x00r\xa1h\x81\x04f\x95\x90\x81\x02v\xe4\x90\xc2\x8a\x09\x93\x09\x88\x09\xa5\x8a\x12\x93\x12\x88\x01t\xc8\xe4k\x02\xc7\x9a\x07s\xce\x92i\xbb\x01u\x8b\x02\x85\x0a\x8c\x02pe\xdb\xa6k\x8a\x05\xb5\xfa\x8a\x00ru\x91\x01v\xe4\x90\xc2\x8a\x08\x93\x08\x88\x08\xa5\x8a\x11\x93\x11\x88\x00sa\x8a\x02c\xdf\x9a\x94\x02d\x81\x03e\xdc\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02\x88\x04\x8f\x00sa\x8b\x02\x85\x0a\x8c\x02ll\x90\xb8hou\x90ay\xd6was\x90\xb8m\x81\x02m\xd4h\x06\xac\x0c\x82\x06\x82\x02\xb6aa\x91\x05e\x91\x05i\xdaim\x90ai\x91\x0au\x99\x91\x05m\x91\x05\x91\x04e\x91\x04ii\x97\x91\x00s\xa6aii\x91\x05\x97\x91\x05\x91\x04o\x91\x04uee\x97\x91\x07\x91\x06\x97\x91\x06\x91\x05\x91\x05u\x91\x01b\x96\x01c\xa6\x9fd\x94\x02hwa\x05\x80\x05\xd0\x92\xa7\x80\x05\xa3\x00\xd7\x89\x05\x9d\x01d\x94\x04\x8d\x09d\x94\x01ea\xd3ll\x8d\xa4\x02c\x9fdt\x9fech\x95e\xbf\x02\xad\x81\x04f\x95\x90\x81\x04\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x02g\xcf\x0513\x06f\x052c\x05\x82\x00\xbfg\xcfn\xa6\xb7w\x82\x05\xc9\xa6t\x92\x82\x05\xcd\x82\x05\xe3\xb4\x82\x02h\x87\x03i\xb6g\x99a\x02\x85\x0a\x8c\x02m\xc2\xcf\x9f\x81\x09m\x83\x09\xc4\x90l\x04vo\xc2\x9am\xa6kk\x99a\x12\x8c\x02n\x9e\x86\x04o\x86\x02v\xad\x81\x00\xbfv\xad\x8a\x05c\x89\x0b\x95v\x92\xbfs\x99\xdd\x92if\x05\x93\x05\x88\x07\x8f\x05h\xaak\x81\x07\xc6\x05\xa8\x8e\x05m\x83\x05\x8e\x06\x92i\xba\x08\xb5\x99\x05\xb7m\x99\x05\xb9\xadc\x89\x09\x8e\x00\xbfv\xad\xb9\xadp\x92i\xba\x06\xb2i\x01f\xd1y\xe9\xad\x01\xb2\x87\x03\x8a\x03\x80\x03d\xe5\x81\x06d\xd9\xda\x81\x0bt\x99\x81\x06f\xa9\xb2\x81\x0bt\x99\x81\x06kas\xb6\x81\x0bt\x99\x81\x04e\x05d\xa6k\x05l\xbc\xf6\x00s\xb2\xb4m\x9ai\xed\x05\xdc\x03\x88\x05\x8f\x03l\xbee\xaft\x82\x02b\x96\x02c\xb2\x80\x02e\xad\x81\x05f\x95\x90\x81\x05\x95i\x9e\x90\x81\x05m\x9ai\x90\x81\x03\xc2\xd2\x9ec\x03q\xf0\x06\x82\x03\xdc\x05115\x00\xbee\xdc15\x0522\x06e\x05\x82\x05n\xa6\xb7w\x82\x05\xc9\xa6t\x92\x82\x05\xe3\xb4\x82\x02\xb2\x80\x02im\xaa\xd2\x9ec\x03n\x04\xac\x0a\x82\x0a\xbe\x95\xb3\x11\x82\x0b\x95\xb3\x10\x82\x00\xbe\x95\xb3\x82\x04\x82\x04\xbe\x95\xb3\x0b\x82\x05\x95\xb3\x0a\x82\x02\xcb\x01\xbc\xdaf\x95\x90\x05l\xe0\xa9e\xbdg\xa1\xd4\x02\x8b\x02\x85\x0a\x8c\x02luq\x82\x05\x97\x82\x02n\xb3\x82\x02\xc3\xaa\x89\x84\x00\xb5\xc3a\x8e\x84\x04c\xb0c\x84\x06\xf8\xaf\x84\x04\xde\x84\x05o\xa1\x99\x04n\xb0n\x84\x04\x8e\x84\x05\xb0p\x84\x04\x9e\xd8\xdf\x84\x02x\x81\x03\x8a\x03c\x89\x09\x95v\x92\xbfs\x99\xdd\x92if\x03\x93\x03\x88\x05\x8f\x00\xb5xh\xaak\x81\x05\xc6\x03\xa8\x8e\x03m\x83\x03\x8e\x04\x92i\xba\x06\xb5\x99\x03\xb7m\x99\x03\xb9\xadc\x89\x08ur\xa1ncyd\xad\xe2\x95\xa9\xbb\x8a\x07\x8e\x08\x92i\xba\x04\xb2i\x01\xb8\xbeB\xbc\x08g\x05b\xbc\x00s\xb8\xbeb\xbcg\x05m\x83\x02\x9fg\x05d\x94\x01mi\xaff\xaae\x02\x83\x01ofpasuq\x82\x03\xd1y\xe9\xad\x04\xb5\xfa\x80\x02\x8b\x02\x85\x0a\x8c\x02lidusl\x9fgov\x92\xb8y\xa4\x07\xbe\xbbtov\x92\xb8y\xa4\x02ru\xb5\x91\x02s\x90a\x91\x00s\xc3o\x91\x03ua\x91\x01p\xaa\xe7\xaak\x81\x03\xb4su\xeb\xc8\xaak\x09\xd5\xb9\x03\xa1n\x01\xc9a\xa1\x8d\xa4\x06cc\x07m\x06\xd0ag\x9f\x90c\xb7s\xbe\xa9chf\xc5l\x06h\xbbiz\x9ft\x90f\xc5l\x06kg\x07m\x08cap\xeb\x90\x06ln\x07\xff\x00\x86mg\x07\xc5\x07m\x07\x86d\x06\xbb\xd1\xff\x9f\x90c\xb7s\xbe\xa9chf\xc5l\x06\xcep\x92\x97t\xcf\xca\x92\x9bf\xc5l\x0b\x9bt\xcf\xca\x92\x97f\xc5l\x06v\x92\x9ec\x90f\xc5l\x06\xd5\xb9\xe3\xd1\xc4\x90l\xc8\xaak\x01r\x86\x01sa\x8a\x03\x93\x03\x88\x03\xf2c\xb0c\x84\x05h\xb0h\x84\x05\xb0\xf2\x84\x00\xdd\x99g\xde\x84\x05n\xb0n\x84\x05p\xb0p\x84\x05\xb5\xc3\x84\x05\x9e\xd8\xdf\x84\x01t\x92l\x95gm\x83\x02\xb7\xd8l\x9fgov\x92\xb8y\xa4\x06\xbe\xbbtov\x92\xb8y\xa4\x01\xe8\xbft\x06\xea\xb9\xc9\x90\x06o\xa1\xc9\x90\x02cce\x9as\x02\x8b\x02\x85\x0a\x8c\x03\xe0\x81\x00s\xed\xda\x9e\x9f\xb1\x09\xb9\xe6\x02n\x02p\x92\xbft\x08\xea\xb9\xc9\x90\x08o\xa1\xc9\x90\x01v\x86\x01youwa\x92a\x86\x00\xcd\x8a\x02ckd\xcan\x04\x97\x02\x93\x02\x88\x04\x8f\x02h\x81\x03f\x95\x90\x81\x00\xcdh\x95i\x9e\x90\x81\x04\xb6g\x99a\x03m\x9ai\x90\x81\x02\xf9you\x92a\x86\x02\x85\x0a\x8c\x02twe\xf0\x81\x02v\x03\xe5ges\x08h\x09\x82\x03\x82\x01b\xa6\x02\x96\x01cc\xf3\x02\x9a\xc5\xb8\x00tc\xd6h\x81\x05f\x95\x90\x81\x05\x95i\x9e\x90\x81\x05m\x9ai\x90\x81\x07\xdb\x95i\x9e\x90\x81\x02\x89\x05\x9d\x8d\x01\xd0\x92\xa7\x02\x94\x04\x8d\x01e\x80\x02\xb4\xd7\xadd\x92\x80\x02h\x81\x03f\x95\x90\x81\x03\xb2h\x95i\x9e\x90\x81\x07s\xcf\xa9\x9a\x81\x00\xb9h\x95i\x9e\x90\x81\x04\xb6g\x99a\x03je\xdb\x95i\x9e\x90\x81\x08s\xcf\xa9\x9a\x81\x03m\xa6bu\xcd\x81\x0af\x95\x90\x81\x04\x9ai\x90\x81\x05\xdb\x95i\x9e\x90\x81\x08s\xcf\xa9\x9a\x81\x03\xea\x9ff\x95\x90\x81\x02\x85\x0a\x8c\x02\xaf\xe9\x9fe\x09\xc8\xaak\x03\xf9\xb2g\x9ao\xb8\x82\x07qet\x99a\x82\x00t\xadc\x89\x03\xa8\x8e\x03\x8e\x04\x92i\xba\x03\xb7m\x99\x02\xbe\x02t\x03\xac\x09\x82\x03\x82\x03\xbf\x80\x02vir\x82\x05\x97\x82\x01\xb2\x8a\x03\x93\x03\x88\x00t\xb2\xd3\x8f\x03l\x81\x04f\x95\x90\x81\x03nt\xb2kh\xa9l\xca\x97\x91\x0e\x9b\x91\x0b\x91\x0b\xcep\x92\x97\x91\x02\xe7\x81\x04f\x95\x90\x81\x04\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x03\xa1exi\xf4s\x03\xcd\xbdg\xa1\xd4\x02\xb0\xd1\xaa\x89\x84\x08\x8e\x84\x07c\x89\x84\x00\xd1\xb0\xd1\x84\x07\x8e\x84\x03r\xb9\xadc\x89\x08\x8e\x09\x92i\xba\x02\x9f\x99gm\x9f\xd1o\x91\x03ok\x03\xe9\xdf\xb2o\x91\x03t\xb2h\x99\x91\x06n\x91\x05\x9fg\x91\x05\xe0g\x91\x03us\x99d\x80\x08s\xbfp\xa6\xa9\xbb\x81\x12p\x92\xb5\x99\x02\xa1e\x81\x00\xd1\xa1e\x8a\x05c\x89\x0b\x95v\x92\xbfs\x99\xdd\x92if\x05\x93\x05\x88\x07\x8f\x05h\xaak\x81\x07\xc6\x05\xa8\x8e\x05m\x83\x05n\xed\x92\xa9\xbb\x8a\x05\x8e\x06\x92i\xba\x08\xb5\x99\x05\xb7m\x99\x05\x91\x00\xd1z\x86\x01i\x8b\x02\x85\x0a\x8c\x03e\xdf\xaa\x89\x84\x07\x8e\x84\x06c\x89\x84\x06\x84\x06\x8e\x84\x02l\xb4\x8d\xa4\x05\xa4\x06\xe2b\x05do\xe8\xaf\xa4\x05\xd2\x92\xa9\xbb\x06v\x92\xb8y\xa4\x05v\x92\x9ec\x90\xa4\x00\x9el\xb4\xe3\xb4\x09r\x09\xf4\x02me\xd7\x89\x02pe\xb2\x82\x06\x97\x82\x03p\xbcu\x8f\x02tlo\x80\xa4\x02wn\x87\x01l\x95e\x8d\x01m\x83\x01o\x87\x02\x8b\x02\x85\x0a\x8c\x02neba\xa1\xe6\xb6h\xbchm\xba\x00t\x9feba\xa1\xe6r\x90\xcam\xba\x07h\xbchm\xba\x07l\xcam\xba\x07midm\xba\x04\xf7\xc0\x04\xb5x\x04two\x03\xc3\x03\x86\x02p\xa9ak\x91\x02rtoi\xbf\xbe\xf0l\xfb\xaa\xd8t\x97\x18\xc4\x90l\x18v\x92\x9ec\x90\x14\x9b\x19\xc4\x90l\x19v\x92\x9ec\x90\x00to\xcdo\x91\x01p\x90\xa9\x90\xa3\x03\xa1n\x01\xa1t\xb7f\xafx\xa3\x02iagdn\x05lf\x05rt\x05\xce\x01s\x02a\xd0\x05\xac\x0b\x82\x05\x82\x02e\x80\x03\xa1\x0512\x00ts\x92e1e\x052b\x05\x82\x05n\xa6\xb7w\x82\x05\xc9\xa6t\x92\x82\x05\xe3\xb4\x82\x02\xd6\x80\x01\xcd\x8a\x03\x93\x03\x88\x05\x8f\x02\xe7\x81\x04f\x95\x90\x81\x04\x95i\x9e\x90\x81\x04m\x9ai\x90\x81\x02\xb2\x8a\x00tt\xb2\x93\x04\x88\x06\x8f\x02\xc7\x9a\x01u\x8b\x02\x85\x0a\x8c\x02\xc4\x90l\x8b\x07\x85\x0f\x8c\x01w\xf0\xc0c\x89\x06\x8e\x07\x92i\xba\x06\xb7m\x99\x03ntyc\x89\x06h\x99gzhou\x00tw\xadty\x8e\x07\x92i\xba\x02o\x81\x03\x8a\x03c\x89\x09\x95v\x92\xbfs\x99\xdd\x92if\x03\x93\x04\xe1\xafad\x92\x0cv\x92\x9ec\x90\x03\x88\x05\x8f\x03h\xaak\x81\x05\xc6\x03\xa8\x8e\x03m\x83\x03n\xed\x92\xa9\xbb\x8a\x00two\x8e\x04\x92i\xba\x06\xb5\x99\x03\xb7m\x99\x03\xab\x03\x91\x00\xe8\xa6\x02\xadg\x90i\x02\x96\x02\xa1\xc0\x01c\xa6\x9f\x02\x89\x05\x9d\x8d\x02\xae\x01d\xa9\xcd\x93\x02\xc8\x98\x00ud\xc8\xa2\x02e\xdc\x02i\x92\xa7\x98\x09\x8d\x09c\xa6\x9f\x0a\xae\x09\xa2\x09m\xaar\x9f\x02\xe1\x8d\x01\x88\x03\x8f\x01\x8b\x02\xcb\xa0\x03rn\x05\x98\x05\xb3\x8d\x00uh\xbbn\xa2\x05\xa3\xa0\x05\x9el\xb4\x02\xe0g\xa6\xed\xb8\xdf\x80\x01\x95v\x92t\x9ab\xa1\xc0\x01\x85\x09\x8c\x02\x80\x02o\xa1\x99\x01m\xaar\x9f\x80\x07\xd0\x92\xa7\x03t\xb6\xd3\x8f\x02\x83\x01nd\x92\xd7\xbb\x9a\xc8\x0am\x83\x0av\x92\x9ec\x90\x00\xe0d\x92\xd7o\xa1wavy\x02i\x9f\xb1\x05mul\x9e\xb1\x0a\xb9\xe6\x05sq\xb1\x07\xb9\xe6\x05\xb9\xe6\x01\x8e\x02\xc8\xe4k\x02p\x92\xb3\x82\x02\xb5l\x9f\xd0\x92\xa7\x0ft\x9f\xc3\x07l\xa9\x95\x07t\x9f\xc3\x02t\xaak\x8d\xa4\x06m\xba\x00u\xb6\xd3\x8f\x01\xbe\xbbt\x80\x02m\x90l\x8b\x06\x85\x0e\x8c\x02t\xb6\xbc\xf6\x80\x09\xab\x80\x01\x9el\xb4\x06\x98\x06\x8d\x01u\x8a\x02\x93\x02\x88\x04\x8f\x02m\xa9\xb6\xd3\x8f\x02\xa5\x8a\x00uu\xa5\x93\x0b\x88\x01\xa5\x8a\x0a\x93\x0a\x88\x00\xdc\x93\x02\x88\x04\x8f\x02\x85\x02v\x03\xac\x0965\x09\x82\x03\x82\x04o\xb8m\x08\x82\x00\xdcv\xdcv\x82\x03y\xba\x82\x01c\x89\x01\xb3\x8d\x01e\x80\x02h\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x02\x85\x02nus\x02r\x9ec\x90b\xa6\x08l\x95e\xa0\xa4\x0c\x8d\xa4\x0cl\xcam\xba\x0cm\xba\x00\xc0w\x87\x02\xe6\xadddo\xe8\xaf\x07s\x95g\xaf\x01\xa3\x01i\x85\x02\xb6\xda\x8a\x06\x93\x06\x88\x02s\xa6ga\x8a\x07\x93\x07\x88\x01m\x83\x01o\x87\x02\xc2\x9a\xeb\x92a\x9e\x9f\x8b\x0f\x85\x06m\xa6kk\x99a\x00vo\xc2\x9am\xa6kk\x99a\x8c\x02\x85\x01\x8e\x01\x9el\xb4\x02\xc7\x9a\x01u\x8b\x02\x85\x00w\x98\x02e\x84\x02\x8b\x02\x85\x0a\x8c\x03o\xa1\x99\x02\xc4\x90l\x8b\x07\x85\x02tto\x86\x00wav\x9aa\xbe\x03y\xe0d\x92\xd7o\xa1v\x92\x9ec\x90\x02w\x81\x03f\x95\x90\x81\x03\xb2\xef\xa0\x81\x0df\x95\x90\x81\x01b\x86\x01c\x89\x05\x9d\x01\xd0\x92\xa7\x02\x94\x04\x8d\x01e\x8b\x02\x85\x03o\xa1\x99\x02o\x84\x00w\xa2\x01hi\xb9bul\xaft\x05c\x89\x0b\x95v\x92\xbf\x06\xbbn\x92\xfb\xaa\xd8t\x97\x16v\x92\x9ec\x90\x12\x9b\x17v\x92\x9ec\x90\x05\xd0\xd9\x9fd\x0cc\x9f\xcd\x95\x95g\xc8\xaak\xc4\x90l\xd0\xd9\x9fd\x06\xcan\xfe\x95t\x95g\xc4\x90l\xcc\x99g\xaf\x11\xcc\x99g\xaf\x05\x97\xfe\x95t\x95g\xc4\x90l\xcc\x99g\xaf\x11\xcc\x99g\xaf\x07n\x9ecul\xa6\xfb\xaa\xd8t\x97\x16\x9b\x00\xd5t\x92\xbc\xf6\xfe\x95t\x95g\xc4\x90l\xcc\x99g\xaf\x12\xcc\x99g\xaf\x05\xc4\x90l\x86\x07\xc5\x95gf\xaae\x06\xc9a\xa1\x06t\xa6\x05\xb9\xaf\xe9\x9fe\x06\xbbtoi\xbf\xbe\xf0l\xfb\xaa\xd8t\x97\x19\x9b\x05\xce\xfe\x95t\x95g\xc4\x90l\xcc\x99g\xaf\x0f\xcc\x99g\xaf\x01i\x8b\x02\x85\x03o\xa1\x99\x01m\x83\x01o\x8b\x00wo\x85\x0a\x8c\x02n\x03m\x83\x02wa\xad\x91\x01\x8e\x01r\x95g\x01s\xce\x92i\xbb\x01t\xc7\x9a\x01ynn\x00x\xa0\xa4\x01b\x96\x01c\x89\x01\xd0\x92\xa7\x02\x94\x01\xe7\x87\x00xm\x83\x01\x8e\x01s\xce\x92i\xbb\x00yaado\x86\x02\x8a\x02\x93\x02e\x84\x02\x88\x04\x8f\x02\x8b\x02\x85\x0a\x8c\x03o\xa1\x99\x02\xdakk\x99\x91\x02\xc4\x90l\x8b\x07\x85\x00ya\xc4\x90l\x85\x8c\x02t\x80\x01c\x89\x05\x9d\x01d\x94\x04\x8d\x01\xe7\x81\x03b\xa6\xa1e\x81\x09f\x95\x90\x81\x03f\x95\x90\x81\x03\xb2\xef\xa0\x81\x0df\x95\x90\x81\x0d\x95i\x9e\x90\x81\x0dm\x9ai\x90\x81\x03\x95i\x9e\x90\x81\x03m\x9ai\x90\x81\x00y\xe7me\xdb\x95i\x9e\x90\x81\x08s\xcf\xa9\x9a\x81\x03\xea\x9ff\x95\x90\x81\x03\xd1\xa1\x9a\xe1s\x8d\x81\x02\x84\x02nm\x83\x02o\x84\x03r\x95h\xb0h\x84\x02\xb6hb\xady\xe2o\x82\x0c\x97\x82\x03i\x80\x03u\xd0\x92\xa7\x80\x02s\xb0\xf2\x84\x08p\x99\xb5\xc3\x84\x08\xb5\xc3\x84\x02\x9ev\x82\x00y\xa2\x01\xa3\x05\xa0\x01i\x87\x02\x80\x02\x84\x02ny\x99g\x02wn\x87\x01m\x83\x01\xba\x03\xac\x09\x82\x03\x82\x03y\xba\x82\x06p\xa9\xfc\x82\x02\x8b\x00yoi\x84\x02\x85\x0a\x8c\x03o\xa1\x99\x02\xc4\x90l\x8b\x07\x85\x0f\x8c\x02tg\xa1\xd4\x02yae\x84\x04\x84\x05\x91\x03\x95g\x91\x01\x8e\x02\xffeg\xb6mm\xadi\x0dg\xa1\xd4\xa4\x01r\x00yr\x95g\x01s\xce\x92i\xbb\x01\x9el\xb4\x02\xc7\x9a\x01u\x8b\x02i\x84\x02\x85\x0a\x8c\x03o\xa1\x99\x02sb\xbc\x80\x06io\x9e\xf7\x9a\x80\x03l\xebt\xaf\x80\x09io\x9e\xf7\x9a\x80\x03m\x90l\x8b\x07\x85\x0f\x8c\x00yu\xfd\x84\x04o\x84\x01ya\x8a\x03\x93\x00za\x87\x02\x93\x02\xd3\x8f\x02h\x81\x03f\x95\x90\x81\x03\x95i\x9e\x90\x81\x04\xb6g\x99a\x03m\x9ai\x90\x81\x02\x95\x81\x04f\x95\x90\x81\x02\x85\x02qefgad\xcf\x82\x00zaqefq\xa9\x99\x82\x02rqa\x82\x02y\x95\x05\xac\x0b\x82\x05\x82\x01b\x96\x01c\x89\x05\x9d\x02\xf3\x01\xb3\x04\x8d\x01e\x80\x02\xb4\xd7\xadd\x92\x80\x03i\x92\xa7\x80\x02\x8b\x00ze\x85\x02\xb7\x81\x04\x8a\x04\x93\x04\x88\x06\x8f\x04h\xaak\x81\x04m\x83\x04p\x92\xb5\x99\x04\x91\x04\xe3d\xd1jo\x95\x92\x09n\x9fjo\x95\x92\x09sp\xaae\x01hb\x96\x02e\x87\x03b\xa1\xc0\x80\x00z\xd6\x80\x03\xb4\xd7\xadd\x92\x80\x04i\x92\xa7\x80\x01i\x8b\x02\x85\x02n\xbb\x82\x01l\x95e\x8d\x01m\x83\x01o\x8b\x02\x85\x01\x8e\x01\xa1t\xb7f\xafx\xa3\x01\xab\x01u\x8b\x02\x85"; +pub(super) static BLOCK_OFFSETS: [u16; 239] = [ + 0, 52, 120, 179, 251, 320, 376, 440, 503, 574, 632, 703, 759, 825, 890, 954, 1009, 1063, 1126, + 1177, 1253, 1313, 1387, 1454, 1522, 1585, 1659, 1725, 1787, 1853, 1922, 1969, 2019, 2091, 2158, + 2208, 2274, 2316, 2353, 2391, 2428, 2465, 2504, 2552, 2591, 2629, 2669, 2710, 2754, 2792, 2834, + 2885, 2937, 2999, 3085, 3147, 3238, 3303, 3382, 3456, 3522, 3588, 3647, 3717, 3804, 3923, 3980, + 4042, 4112, 4170, 4225, 4290, 4369, 4444, 4523, 4604, 4660, 4711, 4790, 4856, 4907, 4969, 5050, + 5117, 5179, 5251, 5318, 5385, 5441, 5496, 5550, 5616, 5689, 5762, 5831, 5902, 5962, 6035, 6091, + 6156, 6207, 6266, 6330, 6397, 6467, 6526, 6597, 6667, 6730, 6807, 6873, 6940, 7006, 7096, 7145, + 7210, 7301, 7356, 7459, 7544, 7633, 7704, 7754, 7828, 7896, 7951, 8013, 8071, 8136, 8235, 8290, + 8361, 8430, 8493, 8561, 8635, 8734, 8820, 8875, 8947, 9014, 9093, 9157, 9221, 9284, 9351, 9403, + 9459, 9512, 9566, 9629, 9688, 9741, 9825, 9921, 9977, 10028, 10077, 10137, 10202, 10263, 10331, + 10405, 10476, 10536, 10592, 10668, 10729, 10799, 10871, 10947, 11022, 11082, 11156, 11215, + 11283, 11349, 11415, 11481, 11539, 11611, 11686, 11754, 11823, 11882, 11939, 12010, 12065, + 12131, 12203, 12261, 12329, 12394, 12450, 12508, 12572, 12638, 12722, 12806, 12902, 12986, + 13050, 13113, 13188, 13281, 13327, 13408, 13486, 13546, 13609, 13677, 13763, 13819, 13882, + 13940, 14008, 14060, 14108, 14182, 14255, 14309, 14349, 14418, 14482, 14538, 14604, 14722, + 14820, 14875, 14928, 15002, 15090, 15135, 15193, 15259, 15326, 15381, 15443, 15498, +]; + +pub(super) static GLYPH_VALUES: [u16; 3807] = [ + 0x01FC, 0x01E2, 0x1EAE, 0x04D0, 0x1EB6, 0x1EB0, 0x1EB2, 0x1EB4, 0x01CD, 0x24B6, 0x1EA4, 0x1EAC, + 0x1EA6, 0x1EA8, 0x1EAA, 0xF6C9, 0x0410, 0x0200, 0x04D2, 0x01DE, 0x1EA0, 0x01E0, 0x1EA2, 0x04D4, + 0x0202, 0x0386, 0xFF21, 0x01FA, 0x1E00, 0x0531, 0x24B7, 0x1E02, 0x1E04, 0x0411, 0x0532, 0x0181, + 0x1E06, 0xFF22, 0x0182, 0x053E, 0xF6CA, 0x1E08, 0x24B8, 0x0108, 0x010A, 0x010A, 0x0549, 0x04BC, + 0x0427, 0x04BE, 0x04B6, 0x04F4, 0x0543, 0x04CB, 0x04B8, 0x0187, 0xFF23, 0x0551, 0x01F1, 0x01C4, + 0x0534, 0x0189, 0x1E10, 0x24B9, 0x1E12, 0x1E0A, 0x1E0C, 0x0414, 0x03EE, 0x0394, 0x018A, 0xF6CB, + 0xF6CC, 0xF6CD, 0x03DC, 0x0402, 0x1E0E, 0xFF24, 0x0110, 0x018B, 0x01F2, 0x01C5, 0x04E0, 0x0405, + 0x040F, 0x0114, 0x1E1C, 0x0535, 0x24BA, 0x1EBE, 0x1E18, 0x1EC6, 0x1EC0, 0x1EC2, 0x1EC4, 0x0404, + 0x0204, 0x0116, 0x1EB8, 0x0424, 0x0537, 0x1EBA, 0x2167, 0x0206, 0x0464, 0x041B, 0x216A, 0x1E16, + 0x1E14, 0x041C, 0xFF25, 0x041D, 0x04A2, 0x014A, 0x04A4, 0x04C7, 0x0190, 0x0388, 0x0420, 0x018E, + 0x042D, 0x0421, 0x04AA, 0x01A9, 0x0538, 0x0389, 0x1EBC, 0x1E1A, 0x01B7, 0x01EE, 0x01B8, 0x24BB, + 0x1E1E, 0x0556, 0x03E4, 0x0191, 0x0472, 0x2164, 0xFF26, 0x2163, 0x3387, 0x01F4, 0x0194, 0x03EA, + 0x01E6, 0x0122, 0x24BC, 0x011C, 0x0120, 0x0120, 0x0413, 0x0542, 0x0494, 0x0492, 0x0490, 0x0193, + 0x0533, 0x0403, 0x1E20, 0xFF27, 0xF6CE, 0x029B, 0x01E4, 0x25CF, 0x25AA, 0x25AB, 0x25A1, 0x33CB, + 0x04A8, 0x04B2, 0x042A, 0x0126, 0x1E2A, 0x1E28, 0x24BD, 0x0124, 0x1E26, 0x1E22, 0x1E24, 0xFF28, + 0x0540, 0x03E8, 0xF6CF, 0x3390, 0x042F, 0x0132, 0x042E, 0x012C, 0x01CF, 0x24BE, 0x0406, 0x0208, + 0x1E2E, 0x04E4, 0x0130, 0x1ECA, 0x04D6, 0x0415, 0x1EC8, 0x0418, 0x020A, 0x0419, 0x04E2, 0xFF29, + 0x053B, 0x0401, 0x0196, 0x03AA, 0x038A, 0x0197, 0x0128, 0x1E2C, 0x0474, 0x0476, 0x0541, 0x24BF, + 0x0134, 0x0408, 0x054B, 0xFF2A, 0x3385, 0x33CD, 0x04A0, 0x1E30, 0x041A, 0x049A, 0x04C3, 0x049E, + 0x049C, 0x01E8, 0x0136, 0x24C0, 0x1E32, 0x0554, 0x053F, 0x0425, 0x03E6, 0x0198, 0x040C, 0x1E34, + 0xFF2B, 0x0480, 0x03DE, 0x046E, 0x01C7, 0xF6BF, 0x013B, 0x24C1, 0x1E3C, 0x013F, 0x013F, 0x1E36, + 0x1E38, 0x053C, 0x01C8, 0x0409, 0x1E3A, 0xFF2C, 0x3386, 0xF6D0, 0x1E3E, 0x24C2, 0x1E40, 0x1E42, + 0x0544, 0xFF2D, 0x019C, 0x01CA, 0x0145, 0x24C3, 0x1E4A, 0x1E44, 0x1E46, 0x019D, 0x2168, 0x01CB, + 0x040A, 0x1E48, 0xFF2E, 0x0546, 0x04E8, 0x04EA, 0x014E, 0x01D1, 0x019F, 0x24C4, 0x1ED0, 0x1ED8, + 0x1ED2, 0x1ED4, 0x1ED6, 0x041E, 0x0150, 0x020C, 0x04E6, 0x1ECC, 0x0555, 0x2126, 0x1ECE, 0x01A0, + 0x1EDA, 0x1EE2, 0x1EDC, 0x1EDE, 0x1EE0, 0x01A2, 0x020E, 0x1E52, 0x1E50, 0x0460, 0x03A9, 0x047A, + 0x047C, 0x038F, 0x038C, 0xFF2F, 0x2160, 0x01EA, 0x01EC, 0x0186, 0x01FE, 0x01FE, 0x047E, 0x1E4C, + 0x1E4E, 0x1E54, 0x24C5, 0x1E56, 0x041F, 0x054A, 0x04A6, 0x01A4, 0x0553, 0xFF30, 0x0470, 0x24C6, + 0xFF31, 0x054C, 0x0156, 0x24C7, 0x0210, 0x1E58, 0x1E5A, 0x1E5C, 0x0550, 0x0212, 0x1E5E, 0xFF32, + 0x0281, 0x02B6, 0x250C, 0x2514, 0x2510, 0x2518, 0x253C, 0x252C, 0x2534, 0x251C, 0x2524, 0x2500, + 0x2502, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x255E, + 0x255F, 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, + 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x1E64, 0x03E0, 0x1E66, 0x018F, 0x04D8, 0x04DA, + 0x24C8, 0x015C, 0x1E60, 0x1E62, 0x1E68, 0x054D, 0x2166, 0x0547, 0x0428, 0x0429, 0x03E2, 0x04BA, + 0x03EC, 0x2165, 0xFF33, 0x042C, 0x03DA, 0x0166, 0x0162, 0x24C9, 0x1E70, 0x1E6A, 0x1E6C, 0x0422, + 0x04AC, 0x2169, 0x04B4, 0x01AC, 0x2162, 0x054F, 0x1E6E, 0xFF34, 0x0539, 0x01BC, 0x0184, 0x01A7, + 0x01AE, 0x0426, 0x040B, 0x216B, 0x2161, 0x016C, 0x01D3, 0x24CA, 0x1E76, 0x0423, 0x0170, 0x0214, + 0x01D7, 0x1E72, 0x01D9, 0x04F0, 0x01DB, 0x01D5, 0x1EE4, 0x1EE6, 0x01AF, 0x1EE8, 0x1EF0, 0x1EEA, + 0x1EEC, 0x1EEE, 0x04F2, 0x0216, 0x0478, 0x04EE, 0x1E7A, 0xFF35, 0x03D3, 0x01B1, 0x03AB, 0x03D4, + 0x03D2, 0x038E, 0x040E, 0x04AE, 0x04B0, 0x0168, 0x1E78, 0x1E74, 0x24CB, 0x1E7E, 0x0412, 0x054E, + 0x01B2, 0xFF36, 0x0548, 0x1E7C, 0x1E82, 0x24CC, 0x0174, 0x1E84, 0x1E86, 0x1E88, 0x1E80, 0xFF37, + 0x24CD, 0x1E8C, 0x1E8A, 0x053D, 0xFF38, 0x0462, 0x24CE, 0x0176, 0x1E8E, 0x1EF4, 0x042B, 0x04F8, + 0x1EF2, 0x01B3, 0x1EF6, 0x0545, 0x0407, 0x0552, 0xFF39, 0x1EF8, 0x046A, 0x046C, 0x0466, 0x0468, + 0x0536, 0x24CF, 0x1E90, 0x017B, 0x1E92, 0x0417, 0x0498, 0x04DE, 0x053A, 0x04C1, 0x0416, 0x0496, + 0x04DC, 0x1E94, 0xFF3A, 0x01B5, 0x0986, 0x0906, 0x0A86, 0x0A06, 0x0A3E, 0x3303, 0x09BE, 0x093E, + 0x0ABE, 0x055F, 0x0970, 0x0985, 0x311A, 0x1EAF, 0x04D1, 0x1EB7, 0x1EB1, 0x1EB3, 0x1EB5, 0x01CE, + 0x24D0, 0x1EA5, 0x1EAD, 0x1EA7, 0x1EA9, 0x1EAB, 0x0317, 0x0301, 0x0301, 0x0954, 0x02CF, 0x0341, + 0x0430, 0x0201, 0x0A71, 0x0905, 0x04D3, 0x01DF, 0x1EA1, 0x01E1, 0x01FD, 0x3150, 0x01E3, 0x2015, + 0x20A4, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0401, 0x0416, 0x0417, 0x0418, 0x0419, + 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, + 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0490, 0x0402, + 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040A, 0x040B, 0x040C, 0x040E, 0xF6C4, + 0xF6C5, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0451, 0x0436, 0x0437, 0x0438, 0x0439, + 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, + 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, 0x0491, 0x0452, + 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045A, 0x045B, 0x045C, 0x045E, 0x040F, + 0x0462, 0x0472, 0x0474, 0xF6C6, 0x045F, 0x0463, 0x0473, 0x0475, 0xF6C7, 0xF6C8, 0x04D9, 0x200E, + 0x200F, 0x200D, 0x066A, 0x060C, 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0x061B, 0x061F, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, + 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, + 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, + 0x0646, 0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650, 0x0651, 0x0652, + 0x0647, 0x06A4, 0x067E, 0x0686, 0x0698, 0x06AF, 0x0679, 0x0688, 0x0691, 0x06BA, 0x06D2, 0x06D5, + 0x20AA, 0x05BE, 0x05C3, 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, + 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, + 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFB2A, 0xFB2B, 0xFB4B, 0xFB1F, 0x05F0, 0x05F1, + 0x05F2, 0xFB35, 0x05B4, 0x05B5, 0x05B6, 0x05BB, 0x05B8, 0x05B7, 0x05B0, 0x05B2, 0x05B1, 0x05B3, + 0x05C2, 0x05C1, 0x05B9, 0x05BC, 0x05BD, 0x05BF, 0x05C0, 0x02BC, 0x2105, 0x2113, 0x2116, 0x202C, + 0x202D, 0x202E, 0x200C, 0x066D, 0x02BD, 0x0A85, 0x0A05, 0x3042, 0x1EA3, 0x0990, 0x311E, 0x0910, + 0x04D5, 0x0A90, 0x0A10, 0x0A48, 0x0639, 0xFECA, 0xFECB, 0xFECC, 0x0203, 0x09C8, 0x0948, 0x0AC8, + 0x30A2, 0xFF71, 0x314F, 0x05D0, 0x0627, 0xFB30, 0xFE8E, 0x0623, 0xFE84, 0x0625, 0xFE88, 0x05D0, + 0xFB4F, 0x0622, 0xFE82, 0x0649, 0xFEF0, 0xFEF3, 0xFEF4, 0xFB2E, 0xFB2F, 0x224C, 0x03AC, 0xFF41, + 0xFF06, 0x33C2, 0x3122, 0x3124, 0x3008, 0x2329, 0x2329, 0x2329, 0x2329, 0x3009, 0x232A, 0x232A, + 0x232A, 0x232A, 0x0E5A, 0x3008, 0xFE3F, 0x3009, 0xFE40, 0x212B, 0x0387, 0x0952, 0x0982, 0x0902, + 0x0A82, 0x3300, 0x249C, 0x055A, 0x02BC, 0x2250, 0x2252, 0x2245, 0x318E, 0x318D, 0x2312, 0x1E9A, + 0x01FB, 0x1E01, 0x21E3, 0x21E0, 0x21E2, 0x21E1, 0x2199, 0x2198, 0x21E9, 0x02C5, 0x02C2, 0x02C3, + 0x02C4, 0x21AA, 0x21A9, 0x21BD, 0x21D0, 0x21CD, 0x21C6, 0x21BC, 0x21E6, 0x2197, 0x2196, 0x21C1, + 0x21CF, 0x279E, 0x21C4, 0x21C0, 0x21E8, 0x2198, 0x2199, 0x21E4, 0x21E5, 0x2195, 0x21A8, 0x21A8, + 0x2196, 0x21C5, 0x2197, 0x21E7, 0xFF3E, 0xFF5E, 0x0251, 0x0252, 0x3041, 0x30A1, 0xFF67, 0x066D, + 0x066D, 0xFF0A, 0xFE61, 0x2042, 0x2243, 0xFF20, 0xFE6B, 0x0250, 0x0994, 0x3120, 0x0914, 0x0A94, + 0x0A14, 0x09D7, 0x0A4C, 0x09CC, 0x094C, 0x0ACC, 0x093D, 0x0561, 0x05E2, 0xFB20, 0x05E2, 0x09AC, + 0x2216, 0x2216, 0x2216, 0x2216, 0xFF3C, 0x092C, 0x0AAC, 0x0A2C, 0x3070, 0x0E3F, 0x30D0, 0x2016, + 0xFF5C, 0x3105, 0x24D1, 0x1E03, 0x1E05, 0x266C, 0x2235, 0x0431, 0x0628, 0xFE90, 0xFE91, 0x3079, + 0xFE92, 0xFC9F, 0xFC08, 0xFC6D, 0x30D9, 0x0562, 0x05D1, 0x03D0, 0xFB31, 0xFB31, 0x05D1, 0xFB4C, + 0x09AD, 0x092D, 0x0AAD, 0x0A2D, 0x0253, 0x3073, 0x30D3, 0x0298, 0x0A02, 0x3331, 0x25CF, 0x25C6, + 0x25BC, 0x25C4, 0x25C0, 0x3010, 0xFE3B, 0x3011, 0xFE3C, 0x25E3, 0x25E2, 0x25AC, 0x25BA, 0x25B6, + 0x25AA, 0x263B, 0x25A0, 0x2605, 0x25E4, 0x25E5, 0x25B4, 0x25B2, 0x2423, 0x1E07, 0x2588, 0xFF42, + 0x0E1A, 0x307C, 0x30DC, 0x249D, 0x33C3, 0xFE37, 0xFE37, 0xFE38, 0xFE38, 0x007B, 0x007B, 0x007B, + 0x007B, 0xFF5B, 0xFE5B, 0xFE37, 0x007D, 0x007D, 0x007D, 0x007D, 0xFF5D, 0xFE5C, 0xFE38, 0x005B, + 0x005B, 0x005B, 0x005B, 0x005B, 0xFF3B, 0x005D, 0x005D, 0x005D, 0x005D, 0x005D, 0xFF3D, 0x032E, + 0x0306, 0x032F, 0x0311, 0x0361, 0x032A, 0x033A, 0x0180, 0x0183, 0x3076, 0x30D6, 0x25D8, 0x2219, + 0x25CE, 0x056E, 0x099A, 0x091A, 0x0A9A, 0x0A1A, 0x3388, 0x0981, 0x0310, 0x0901, 0x0A81, 0x21EA, + 0x2105, 0x032C, 0x030C, 0x3118, 0x1E09, 0x24D2, 0x0109, 0x0255, 0x010B, 0x010B, 0x33C5, 0x0327, + 0x2308, 0x2308, 0x2308, 0x2308, 0x2309, 0x2309, 0x2309, 0x2309, 0x2103, 0xFFE0, 0x0579, 0x099B, + 0x091B, 0x0A9B, 0x0A1B, 0x3114, 0x04BD, 0x2713, 0x0447, 0x04BF, 0x04B7, 0x04F5, 0x0573, 0x04CC, + 0x04B9, 0x3277, 0x3217, 0x3269, 0x314A, 0x3209, 0x0E0A, 0x0E08, 0x0E09, 0x0E0C, 0x0188, 0x3276, + 0x3216, 0x3268, 0x3148, 0x3208, 0x321C, 0x25CB, 0x00A9, 0x2299, 0x2299, 0x2297, 0x2297, 0x2299, + 0x2295, 0x2295, 0x3036, 0x25D0, 0x25D1, 0x032D, 0x0302, 0x2327, 0x01C2, 0x01C0, 0x01C1, 0x01C3, + 0x2663, 0x2667, 0x33A4, 0xFF43, 0x33A0, 0x0581, 0xFF1A, 0x20A1, 0xFE55, 0x02D1, 0x02D0, 0x0313, + 0x0315, 0x060C, 0x055D, 0xFF0C, 0x0314, 0x02BD, 0xFE50, 0x0312, 0x02BB, 0x263C, 0x222E, 0x222E, + 0x222E, 0x2303, 0x0006, 0x0007, 0x0008, 0x0018, 0x000D, 0x0011, 0x0012, 0x0013, 0x0014, 0x007F, + 0x0010, 0x0019, 0x0005, 0x0004, 0x001B, 0x0017, 0x0003, 0x000C, 0x001C, 0x001D, 0x0009, 0x000A, + 0x0015, 0x001E, 0x000F, 0x000E, 0x0002, 0x0001, 0x001A, 0x0016, 0x001F, 0x000B, 0x2210, 0x2210, + 0x300C, 0xFF62, 0xFE41, 0x300D, 0xFF63, 0xFE42, 0x337F, 0x33C7, 0x33C6, 0x249E, 0x20A2, 0x0297, + 0x22CF, 0x22CE, 0xF6D1, 0xF6D2, 0xF6D4, 0xF6D5, 0x0564, 0x09A6, 0x0636, 0x0926, 0xFEBE, 0xFEBF, + 0xFEC0, 0x05BC, 0x05BC, 0x0AA6, 0x0A26, 0x3060, 0x30C0, 0x062F, 0x05D3, 0xFB33, 0xFB33, 0xD800, + 0xD800, 0xD801, 0xD801, 0x05D3, 0xD802, 0xD802, 0xD803, 0xD803, 0xD804, 0xD804, 0xD805, 0xD805, + 0xD806, 0xD806, 0xD807, 0xD807, 0xD808, 0xD808, 0xD809, 0xD809, 0xFEAA, 0x064F, 0x064F, 0x064C, + 0x064C, 0x0964, 0x05A7, 0x05A7, 0x0485, 0xF6D3, 0x300A, 0xFE3D, 0x300B, 0xFE3E, 0x032B, 0x21D4, + 0x21D2, 0x0965, 0xF6D6, 0x030F, 0x222C, 0x2017, 0x0333, 0x033F, 0x02BA, 0x2016, 0x030E, 0x3109, + 0x33C8, 0x1E11, 0x24D3, 0x1E13, 0x09A1, 0x0921, 0x0AA1, 0x0A21, 0x0688, 0xFB89, 0x095C, 0x09A2, + 0x0922, 0x0AA2, 0x0A22, 0x1E0B, 0x1E0D, 0x066B, 0x066B, 0x0434, 0x05AD, 0x3067, 0x03EF, 0x30C7, + 0x232B, 0x2326, 0x018D, 0x09F8, 0x02A4, 0x09A7, 0x0927, 0x0AA7, 0x0A27, 0x0257, 0x0385, 0x0344, + 0x2662, 0xF6D7, 0x0324, 0x0308, 0xF6D8, 0x0385, 0x3062, 0x30C2, 0x3003, 0x2223, 0x2215, 0x0452, + 0x2593, 0x1E0F, 0x3397, 0x0111, 0xFF44, 0x2584, 0x0E0E, 0x0E14, 0x3069, 0x30C9, 0xFF04, 0xFE69, + 0x20AB, 0x3326, 0x0307, 0x0323, 0x0323, 0x30FB, 0xF6BE, 0x0284, 0x25CC, 0xFB1F, 0xFB1F, 0x031E, + 0x02D5, 0x249F, 0x0256, 0x018C, 0x3065, 0x30C5, 0x01F3, 0x02A3, 0x01C6, 0x02A5, 0x04E1, 0x0455, + 0x045F, 0x2641, 0x098F, 0x311C, 0x0115, 0x090D, 0x0A8D, 0x0945, 0x0AC5, 0x1E1D, 0x0565, 0x0587, + 0x24D4, 0x1EBF, 0x1E19, 0x1EC7, 0x1EC1, 0x1EC3, 0x1EC5, 0x0454, 0x0205, 0x090F, 0x0117, 0x1EB9, + 0x0A0F, 0x0A47, 0x0444, 0x0A8F, 0x0567, 0x311D, 0x3048, 0x1EBB, 0x311F, 0x0668, 0x09EE, 0x2467, + 0x2791, 0x096E, 0x2471, 0x2485, 0x2499, 0x0AEE, 0x0A6E, 0x0668, 0x3028, 0x266B, 0x3227, 0xFF18, + 0x247B, 0x248F, 0x06F8, 0x2177, 0x0E58, 0x0207, 0x0465, 0x30A8, 0xFF74, 0x0A74, 0x3154, 0x043B, + 0x246A, 0x247E, 0x2492, 0x217A, 0x22EE, 0x1E17, 0x1E15, 0x043C, 0xFE31, 0xFF45, 0x055B, 0x3123, + 0x043D, 0xFE32, 0x04A3, 0x014B, 0x3125, 0x04A5, 0x04C8, 0x2002, 0x3153, 0x025B, 0x029A, 0x025C, + 0x025E, 0x025D, 0x24A0, 0x03B5, 0x03AD, 0x003D, 0xFF1D, 0xFE66, 0x207C, 0x3126, 0x0440, 0x0258, + 0x044D, 0x0441, 0x04AB, 0x0283, 0x0286, 0x090E, 0x0946, 0x01AA, 0x0285, 0x3047, 0x30A7, 0xFF6A, + 0x212E, 0x0568, 0x03AE, 0x1EBD, 0x1E1B, 0x0591, 0x0591, 0x0591, 0x0591, 0x01DD, 0x3161, 0x20AC, + 0x09C7, 0x0947, 0x0AC7, 0x055C, 0x203C, 0xFF01, 0x0292, 0x01EF, 0x0293, 0x01B9, 0x01BA, 0x095E, + 0x0A5E, 0x2109, 0x064E, 0x064E, 0x064B, 0x3108, 0x24D5, 0x1E1F, 0x0641, 0x0586, 0xFED2, 0xFED3, + 0xFED4, 0x03E5, 0x2640, 0x246E, 0x2482, 0x2496, 0x25A0, 0x25AC, 0x05DA, 0xFB3A, 0xFB3A, 0x05DA, + 0xD80A, 0xD80A, 0xD80B, 0xD80B, 0x05DD, 0x05DD, 0x05DF, 0x05DF, 0x05E3, 0x05E3, 0x05E5, 0x05E5, + 0x02C9, 0x25C9, 0x0473, 0x0665, 0x09EB, 0x2464, 0x278E, 0x096B, 0x0AEB, 0x0A6B, 0x0665, 0x3025, + 0x3224, 0xFF15, 0x2478, 0x248C, 0x06F5, 0x2174, 0x0E55, 0x230A, 0x230A, 0x230A, 0x230A, 0x230B, + 0x230B, 0x230B, 0x230B, 0xFF46, 0x3399, 0x0E1F, 0x0E1D, 0x0E4F, 0x2200, 0x0664, 0x09EA, 0x2463, + 0x278D, 0x096A, 0x0AEA, 0x0A6A, 0x0664, 0x3024, 0x3223, 0xFF14, 0x09F7, 0x2477, 0x248B, 0x06F4, + 0x2173, 0x246D, 0x2481, 0x2495, 0x0E54, 0x02CB, 0x24A1, 0x20A3, 0x0997, 0x01F5, 0x0917, 0x06AF, + 0xFB93, 0xFB94, 0xFB95, 0x0A97, 0x0A17, 0x304C, 0x30AC, 0x0263, 0x02E0, 0x03EB, 0x310D, 0x01E7, + 0x0123, 0x24D6, 0x011D, 0x0121, 0x0121, 0x0433, 0x3052, 0x30B2, 0x2251, 0x059C, 0x05F3, 0x059D, + 0x059E, 0x05F4, 0x3013, 0x0998, 0x0572, 0x0918, 0x0A98, 0x0A18, 0x063A, 0xFECE, 0xFECF, 0xFED0, + 0x0495, 0x0493, 0x0491, 0x095A, 0x0A5A, 0x0260, 0x3393, 0x304E, 0x30AE, 0x0563, 0x05D2, 0xFB32, + 0xFB32, 0x05D2, 0x0453, 0x01BE, 0x0294, 0x0296, 0x02C0, 0x0295, 0x02C1, 0x02E4, 0x02A1, 0x02A2, + 0x1E21, 0xFF47, 0x3054, 0x30B4, 0x24A2, 0x33AC, 0x0316, 0x0300, 0x0300, 0x0953, 0x02CE, 0xFF40, + 0x0340, 0x22DB, 0xFF1E, 0x2273, 0x2277, 0x2267, 0xFE65, 0x0261, 0x01E5, 0x3050, 0x30B0, 0x3318, + 0x33C9, 0x04A9, 0x06C1, 0x09B9, 0x04B3, 0x0939, 0x0AB9, 0x0A39, 0x062D, 0xFEA2, 0xFEA3, 0x306F, + 0xFEA4, 0x332A, 0x30CF, 0xFF8A, 0x0A4D, 0x0621, 0xD80C, 0xD80D, 0xD80E, 0xD80F, 0x0621, 0xD810, + 0xD811, 0xD812, 0x3164, 0x044A, 0x21BC, 0x21C0, 0x33CA, 0x05B2, 0x05B2, 0x05B2, 0x05B2, 0x05B2, + 0x05B2, 0x05B2, 0x05B2, 0x05B3, 0x05B3, 0x05B3, 0x05B3, 0x05B3, 0x05B3, 0x05B3, 0x05B3, 0x05B1, + 0x05B1, 0x05B1, 0x05B1, 0x05B1, 0x05B1, 0x05B1, 0x05B1, 0x0302, 0x0302, 0x0302, 0x0127, 0x310F, + 0x1E2B, 0x1E29, 0x24D7, 0x0125, 0x1E27, 0x1E23, 0x1E25, 0x05D4, 0x2665, 0x2661, 0xFB34, 0xFB34, + 0x06C1, 0x0647, 0x05D4, 0xFBA7, 0xFEEA, 0xFEEA, 0xFBA5, 0xFBA4, 0xFBA8, 0xFEEB, 0x3078, 0xFBA9, + 0xFEEC, 0x337B, 0x30D8, 0xFF8D, 0x3336, 0x0267, 0x3339, 0x05D7, 0x05D7, 0x0266, 0x02B1, 0x327B, + 0x321B, 0x326D, 0x314E, 0x320D, 0x3072, 0x30D2, 0xFF8B, 0x05B4, 0x05B4, 0x05B4, 0x05B4, 0x05B4, + 0x05B4, 0x05B4, 0x05B4, 0x1E96, 0xFF48, 0x0570, 0x0E2B, 0x307B, 0x30DB, 0xFF8E, 0x05B9, 0x05B9, + 0x05B9, 0x05B9, 0x05B9, 0x05B9, 0x05B9, 0x05B9, 0x0E2E, 0x0309, 0x0309, 0x0321, 0x0322, 0x3342, + 0x03E9, 0x2015, 0x031B, 0x2668, 0x2302, 0x24A3, 0x02B0, 0x0265, 0x3075, 0x3333, 0x30D5, 0xFF8C, + 0x030B, 0x0195, 0xFF0D, 0xFE63, 0x2010, 0x044F, 0x0987, 0x3127, 0x012D, 0x01D0, 0x24D8, 0x0456, + 0x0209, 0x328F, 0x328B, 0x323F, 0x323A, 0x32A5, 0x3006, 0x3001, 0xFF64, 0x3237, 0x32A3, 0x322F, + 0x323D, 0x329D, 0x3240, 0x3296, 0x3236, 0x322B, 0x3232, 0x32A4, 0x3005, 0x3298, 0x3238, 0x32A7, + 0x32A6, 0x32A9, 0x322E, 0x322A, 0x3234, 0x3002, 0x329E, 0x3243, 0x3239, 0x323E, 0x32A8, 0x3299, + 0x3242, 0x3233, 0x3000, 0x3235, 0x3231, 0x323B, 0x3230, 0x323C, 0x322C, 0x322D, 0x3007, 0x328E, + 0x328A, 0x3294, 0x3290, 0x328C, 0x328D, 0x0907, 0x1E2F, 0x04E5, 0x1ECB, 0x04D7, 0x0435, 0x3275, + 0x3215, 0x3267, 0x3147, 0x3207, 0x0A87, 0x0A07, 0x3044, 0x1EC9, 0x0988, 0x0438, 0x0908, 0x0A88, + 0x0A08, 0x0A40, 0x020B, 0x0439, 0x09C0, 0x0940, 0x0AC0, 0x0133, 0x30A4, 0xFF72, 0x3163, 0x02DC, + 0x05AC, 0x04E3, 0x2253, 0x0A3F, 0xFF49, 0x2206, 0x056B, 0x2321, 0x222B, 0x222B, 0x2320, 0x22BA, + 0x22C2, 0x22C2, 0x3305, 0x25D8, 0x25D9, 0x263B, 0x0451, 0x03CA, 0x0390, 0x0269, 0x03AF, 0x24A4, + 0x0A72, 0x3043, 0x30A3, 0xFF68, 0x09FA, 0x0268, 0x309D, 0x30FD, 0x0129, 0x1E2D, 0x3129, 0x044E, + 0x09BF, 0x093F, 0x0ABF, 0x0475, 0x0477, 0x0571, 0x099C, 0x091C, 0x0A9C, 0x0A1C, 0x3110, 0x01F0, + 0x24D9, 0x0135, 0x029D, 0x025F, 0x0458, 0x062C, 0xFE9E, 0xFE9F, 0xFEA0, 0x0698, 0xFB8B, 0x099D, + 0x091D, 0x0A9D, 0x0A1D, 0x057B, 0x3004, 0xFF4A, 0x24A5, 0x02B2, 0x04A1, 0x0995, 0x1E31, 0x043A, + 0x049B, 0x0915, 0x05DB, 0x0643, 0xFB3B, 0xFB3B, 0xFEDA, 0x05DB, 0xFEDB, 0xFEDC, 0xFB4D, 0x0A95, + 0x0A15, 0x304B, 0x04C4, 0x30AB, 0xFF76, 0x03F0, 0x3171, 0x3184, 0x3178, 0x3179, 0x330D, 0x0640, + 0x0640, 0x30F5, 0x3384, 0x0650, 0x064D, 0x049F, 0xFF70, 0x049D, 0x310E, 0x3389, 0x01E9, 0x0137, + 0x24DA, 0x1E33, 0x0584, 0x3051, 0x30B1, 0xFF79, 0x056F, 0x30F6, 0x0138, 0x0996, 0x0445, 0x0916, + 0x0A96, 0x0A16, 0x062E, 0xFEA6, 0xFEA7, 0xFEA8, 0x03E7, 0x0959, 0x0A59, 0x3278, 0x3218, 0x326A, + 0x314B, 0x320A, 0x0E02, 0x0E05, 0x0E03, 0x0E04, 0x0E5B, 0x0199, 0x0E06, 0x3391, 0x304D, 0x30AD, + 0xFF77, 0x3315, 0x3316, 0x3314, 0x326E, 0x320E, 0x3260, 0x3131, 0x3200, 0x3133, 0x045C, 0x1E35, + 0x3398, 0x33A6, 0xFF4B, 0x33A2, 0x3053, 0x33C0, 0x0E01, 0x30B3, 0xFF7A, 0x331E, 0x0481, 0x327F, + 0x0343, 0x24A6, 0x33AA, 0x046F, 0x33CF, 0x029E, 0x304F, 0x30AF, 0xFF78, 0x33B8, 0x33BE, 0x09B2, + 0x0932, 0x0AB2, 0x0A32, 0x0E45, 0xFEFC, 0xFEF8, 0xFEF7, 0xFEFA, 0xFEF9, 0xFEFB, 0xFEF6, 0xFEF5, + 0x0644, 0x019B, 0x05DC, 0xFB3C, 0xFB3C, 0x05DC, 0xD813, 0xD814, 0xD814, 0xD813, 0xFEDE, 0xFCCA, + 0xFEDF, 0xFCC9, 0xFCCB, 0xFDF2, 0xFEE0, 0xFD88, 0xFCCC, 0xD815, 0xD816, 0x25EF, 0x019A, 0x026C, + 0x310C, 0x013C, 0x24DB, 0x1E3D, 0x0140, 0x0140, 0x1E37, 0x1E39, 0x031A, 0x0318, 0x22DA, 0xFF1C, + 0x2272, 0x2276, 0x2266, 0xFE64, 0x026E, 0x258C, 0x026D, 0x20A4, 0x056C, 0x01C9, 0x0459, 0xF6C0, + 0x0933, 0x0AB3, 0x1E3B, 0x0934, 0x09E1, 0x0961, 0x09E3, 0x0963, 0x026B, 0xFF4C, 0x33D0, 0x0E2C, + 0x2227, 0x2227, 0x2310, 0x2228, 0x2228, 0x0E25, 0x017F, 0xFE4E, 0x0332, 0xFE4D, 0x24A7, 0x2113, + 0x2591, 0x0E26, 0x098C, 0x090C, 0x09E2, 0x0962, 0x33D3, 0x09AE, 0x0331, 0x0304, 0x02CD, 0xFFE3, + 0x1E3F, 0x092E, 0x0AAE, 0x0A2E, 0x05A4, 0x05A4, 0x307E, 0xF895, 0xF894, 0x0E4B, 0xF893, 0xF88C, + 0xF88B, 0x0E48, 0xF88A, 0xF884, 0x0E31, 0xF889, 0x0E47, 0xF88F, 0xF88E, 0x0E49, 0xF88D, 0xF892, + 0xF891, 0x0E4A, 0xF890, 0x0E46, 0x30DE, 0xFF8F, 0x2642, 0x3347, 0x05BE, 0x2642, 0x05AF, 0x3383, + 0x3107, 0x33D4, 0x24DC, 0x33A5, 0x1E41, 0x1E43, 0x0645, 0xFEE2, 0xFEE3, 0xFEE4, 0xFCD1, 0xFC48, + 0x334D, 0x3081, 0x337E, 0x30E1, 0xFF92, 0x05DE, 0xFB3E, 0xFB3E, 0x05DE, 0x0574, 0x05A5, 0x05A6, + 0x05A6, 0x05A5, 0x0271, 0x3392, 0xFF65, 0x00B7, 0x3272, 0x3212, 0x3264, 0x3141, 0x3170, 0x3204, + 0x316E, 0x316F, 0x307F, 0x30DF, 0xFF90, 0x0320, 0x2296, 0x02D7, 0x2213, 0x334A, 0x3349, 0x0270, + 0x3396, 0x33A3, 0xFF4D, 0x339F, 0x3082, 0x33C1, 0x30E2, 0xFF93, 0x33D6, 0x0E21, 0x33A7, 0x33A8, + 0x24A8, 0x33AB, 0x33B3, 0x026F, 0x00B5, 0x3382, 0x226B, 0x226A, 0x338C, 0x03BC, 0x338D, 0x3080, + 0x30E0, 0xFF91, 0x3395, 0x339B, 0x05A3, 0x05A3, 0x266A, 0x266B, 0x266D, 0x266F, 0x33B2, 0x33B6, + 0x33BC, 0x33B9, 0x33B7, 0x33BF, 0x33BD, 0x09A8, 0x2207, 0x0928, 0x0AA8, 0x0A28, 0x306A, 0x30CA, + 0xFF85, 0x0149, 0x3381, 0x310B, 0x00A0, 0x0146, 0x24DD, 0x1E4B, 0x1E45, 0x1E47, 0x306D, 0x30CD, + 0xFF88, 0x20AA, 0x338B, 0x0999, 0x0919, 0x0A99, 0x0A19, 0x0E07, 0x3093, 0x0272, 0x0273, 0x326F, + 0x320F, 0x3135, 0x3261, 0x3136, 0x3134, 0x3168, 0x3201, 0x3167, 0x3166, 0x306B, 0x30CB, 0xFF86, + 0xF899, 0x0E4D, 0x0669, 0x09EF, 0x2468, 0x2792, 0x096F, 0x0AEF, 0x0A6F, 0x0669, 0x3029, 0x3228, + 0xFF19, 0x247C, 0x2490, 0x06F9, 0x2178, 0x2472, 0x2486, 0x249A, 0x0E59, 0x01CC, 0x045A, 0x30F3, + 0xFF9D, 0x019E, 0x1E49, 0xFF4E, 0x339A, 0x09A3, 0x0923, 0x0AA3, 0x0A23, 0x0929, 0x306E, 0x30CE, + 0xFF89, 0x00A0, 0x0E13, 0x0E19, 0x0646, 0xFEE6, 0x06BA, 0xFB9F, 0xD817, 0xFEE7, 0xFCD2, 0xFC4B, + 0xFEE8, 0xFCD5, 0xFC4E, 0xFC8D, 0x220C, 0x2209, 0x226F, 0x2271, 0x2279, 0x2262, 0x226E, 0x2270, + 0x2226, 0x2280, 0x2281, 0x2285, 0x0576, 0x24A9, 0x33B1, 0x306C, 0x30CC, 0xFF87, 0x09BC, 0x093C, + 0x0ABC, 0x0A3C, 0xFF03, 0xFE5F, 0x0374, 0x0375, 0x2116, 0x05E0, 0xFB40, 0xFB40, 0x05E0, 0x33B5, + 0x33BB, 0x099E, 0x091E, 0x0A9E, 0x0A1E, 0x0E2D, 0x0275, 0x04E9, 0x04EB, 0x0993, 0x311B, 0x014F, + 0x0911, 0x0A91, 0x0949, 0x0AC9, 0x01D2, 0x24DE, 0x1ED1, 0x1ED9, 0x1ED3, 0x1ED5, 0x1ED7, 0x043E, + 0x0151, 0x020D, 0x0913, 0x04E7, 0x1ECD, 0x315A, 0x0328, 0x0A93, 0x0585, 0x304A, 0x1ECF, 0x01A1, + 0x1EDB, 0x1EE3, 0x1EDD, 0x1EDF, 0x1EE1, 0x01A3, 0x020F, 0x30AA, 0xFF75, 0x3157, 0x05AB, 0x1E53, + 0x1E51, 0x0950, 0x0461, 0x0277, 0x047B, 0x047D, 0x03CE, 0x0AD0, 0x03CC, 0xFF4F, 0x0661, 0x09E7, + 0x2460, 0x278A, 0x0967, 0x0AE7, 0x0A67, 0x0661, 0x3021, 0x3220, 0xFF11, 0x09F4, 0x2474, 0x2488, + 0x06F1, 0x2170, 0x0E51, 0x01EB, 0x01ED, 0x0A13, 0x0A4B, 0x0254, 0x24AA, 0x25E6, 0x2325, 0x221F, + 0x0912, 0x094A, 0x01FF, 0x3049, 0x30A9, 0xFF6B, 0x01FF, 0x047F, 0x1E4D, 0x1E4F, 0x3121, 0x203E, + 0xFE4A, 0x0305, 0xFE49, 0xFE4C, 0xFE4B, 0x00AF, 0x09CB, 0x094B, 0x0ACB, 0x3380, 0x332B, 0x09AA, + 0x1E55, 0x092A, 0x21DF, 0x21DE, 0x0AAA, 0x0A2A, 0x3071, 0x0E2F, 0x30D1, 0x0484, 0x04C0, 0x317F, + 0x2225, 0x0028, 0x0028, 0xFD3E, 0x0028, 0x0028, 0x0028, 0xFF08, 0xFE59, 0xFE35, 0x0029, 0x0029, + 0xFD3F, 0x0029, 0x0029, 0x0029, 0xFF09, 0xFE5A, 0xFE36, 0x05C0, 0x0599, 0x33A9, 0x05B7, 0x05B7, + 0x05B7, 0x05B7, 0x05B7, 0x05B7, 0x05B7, 0x05B7, 0x05A1, 0x3106, 0x24DF, 0x1E57, 0x05E4, 0x043F, + 0xFB44, 0xFB44, 0x333B, 0xFB43, 0x067E, 0x057A, 0x05E4, 0xFB57, 0xFB58, 0x307A, 0xFB59, 0x30DA, + 0x04A7, 0xFB4E, 0x066A, 0xFF05, 0xFE6A, 0x0589, 0xFF61, 0xFF0E, 0xFE52, 0x0342, 0x20A7, 0x338A, + 0x09AB, 0x092B, 0x0AAB, 0x0A2B, 0x327A, 0x321A, 0x326C, 0x314D, 0x320C, 0x0278, 0x0E3A, 0x03D5, + 0x01A5, 0x0E1E, 0x0E1C, 0x0E20, 0x3273, 0x3213, 0x3176, 0x3265, 0x3172, 0x3142, 0x3205, 0x3174, + 0x3144, 0x3175, 0x3177, 0x3173, 0x3074, 0x30D4, 0x03D6, 0x0583, 0x210F, 0x210F, 0x031F, 0x2295, + 0x002B, 0x02D6, 0xFF0B, 0xFE62, 0x207A, 0xFF50, 0x33D8, 0x307D, 0x261F, 0x261C, 0x261E, 0x261D, + 0x30DD, 0x0E1B, 0x3012, 0x3020, 0x24AB, 0x227A, 0x211E, 0x2032, 0x02B9, 0x2035, 0x220F, 0x220F, + 0x2305, 0x30FC, 0x2318, 0x2237, 0x0471, 0x0486, 0x33B0, 0x3077, 0x30D7, 0x33B4, 0x33BA, 0x0958, + 0x05A8, 0x0642, 0xFED6, 0xFED7, 0xFED8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, + 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x05B8, 0x059F, 0x3111, 0x24E0, + 0x02A0, 0xFF51, 0x05E7, 0xFB47, 0xFB47, 0xD818, 0xD818, 0xD819, 0xD819, 0x05E7, 0xD81A, 0xD81A, + 0xD81B, 0xD81B, 0xD81C, 0xD81C, 0xD81D, 0xD81D, 0xD81E, 0xD81E, 0xD81F, 0xD81F, 0xD820, 0xD820, + 0xD821, 0xD821, 0x24AC, 0x2669, 0x05BB, 0x05BB, 0x05BB, 0x05BB, 0x05BB, 0x05BB, 0x05BB, 0x05BB, + 0x061F, 0x055E, 0x037E, 0xFF1F, 0xFF02, 0x301E, 0x301D, 0x201B, 0x201B, 0x0149, 0xFF07, 0x057C, + 0x09B0, 0x0930, 0x221A, 0x221A, 0x221A, 0x221A, 0x221A, 0x221A, 0x221A, 0x33AE, 0x33AF, 0x33AD, + 0x05BF, 0x05BF, 0x0AB0, 0x0A30, 0x3089, 0x30E9, 0xFF97, 0x09F1, 0x09F0, 0x0264, 0x2236, 0x3116, + 0x0157, 0x24E1, 0x0211, 0x1E59, 0x1E5B, 0x1E5D, 0x203B, 0x0631, 0x0580, 0xFEAE, 0x308C, 0xD822, + 0x30EC, 0xFF9A, 0x05E8, 0xFB48, 0xD823, 0xD823, 0xD824, 0xD824, 0x05E8, 0xD825, 0xD825, 0xD826, + 0xD826, 0xD827, 0xD827, 0xD828, 0xD828, 0xD829, 0xD829, 0xD82A, 0xD82A, 0xD82B, 0xD82B, 0xD82C, + 0xD82C, 0x223D, 0x0597, 0x0597, 0x2310, 0x027E, 0x027F, 0x09DD, 0x095D, 0x027D, 0x027B, 0x02B5, + 0x03F1, 0x02DE, 0x3271, 0x3211, 0x3263, 0x3140, 0x313A, 0x3169, 0x3139, 0x313B, 0x316C, 0x3203, + 0x313F, 0x313C, 0x316B, 0x313D, 0x313E, 0x316A, 0x316D, 0x221F, 0x0319, 0x22BF, 0x308A, 0x30EA, + 0xFF98, 0x0325, 0x030A, 0x02BF, 0x0559, 0x031C, 0x02D3, 0x02BE, 0x0339, 0x02D2, 0x0213, 0x3351, + 0x1E5F, 0x027C, 0x027A, 0xFF52, 0x308D, 0x30ED, 0xFF9B, 0x0E23, 0x24AD, 0x09DC, 0x0931, 0x0A5C, + 0x0691, 0xFB8D, 0x09E0, 0x0960, 0x0AE0, 0x09C4, 0x0944, 0x0AC4, 0x2590, 0x0279, 0x02B4, 0x308B, + 0x30EB, 0xFF99, 0x09F2, 0x09F3, 0x0E24, 0x098B, 0x090B, 0x0A8B, 0x09C3, 0x0943, 0x0AC3, 0x09B8, + 0x1E65, 0x0635, 0x0938, 0xFEBA, 0xFEBB, 0xFEBC, 0x0AB8, 0x0A38, 0x3055, 0x30B5, 0xFF7B, 0xFDFA, + 0x05E1, 0xFB41, 0xFB41, 0x05E1, 0x0E32, 0x0E41, 0x0E44, 0x0E43, 0x0E33, 0x0E30, 0x0E40, 0xF886, + 0x0E35, 0xF885, 0x0E34, 0x0E42, 0xF888, 0x0E37, 0xF887, 0x0E36, 0x0E38, 0x0E39, 0x3119, 0x1E67, + 0x0259, 0x04D9, 0x04DB, 0x025A, 0x24E2, 0x015D, 0x1E61, 0x1E63, 0x1E69, 0x033C, 0x02CA, 0x0633, + 0xFEB2, 0xFEB3, 0xFEB4, 0x05B6, 0x05B6, 0x05B6, 0x05B6, 0x05B6, 0x05B6, 0x05B6, 0x0592, 0x05B6, + 0x057D, 0x305B, 0x30BB, 0xFF7E, 0x061B, 0xFF1B, 0xFE54, 0x309C, 0xFF9F, 0x3322, 0x3323, 0x0667, + 0x09ED, 0x2466, 0x2790, 0x096D, 0x0AED, 0x0A6D, 0x0667, 0x3027, 0x3226, 0xFF17, 0x247A, 0x248E, + 0x06F7, 0x2176, 0x2470, 0x2484, 0x2498, 0x0E57, 0x00AD, 0x0577, 0x09B6, 0x0448, 0x0651, 0xFC61, + 0xFC5E, 0xFC60, 0xD82D, 0xFC62, 0xFC5F, 0x2592, 0x2593, 0x2591, 0x2592, 0x0936, 0x0AB6, 0x0A36, + 0x0593, 0x3115, 0x0449, 0x0634, 0xFEB6, 0xFEB7, 0xFEB8, 0x03E3, 0x20AA, 0x20AA, 0x05B0, 0x05B0, + 0x05B0, 0x05B0, 0x05B0, 0x05B0, 0x05B0, 0x05B0, 0x05B0, 0x04BB, 0x03ED, 0x05E9, 0xFB49, 0xFB49, + 0xFB2C, 0xFB2C, 0xFB2D, 0xFB2D, 0x05C1, 0x05E9, 0xFB2A, 0xFB2A, 0xFB2B, 0xFB2B, 0x0282, 0x03C2, + 0x03F2, 0x3057, 0x30B7, 0xFF7C, 0x05BD, 0x05BD, 0x05C2, 0x3274, 0x3214, 0x317E, 0x3266, 0x317A, + 0x3145, 0x317B, 0x3206, 0x317D, 0x317C, 0x0666, 0x09EC, 0x2465, 0x278F, 0x096C, 0x0AEC, 0x0A6C, + 0x0666, 0x3026, 0x3225, 0xFF16, 0x2479, 0x248D, 0x06F6, 0x2175, 0x246F, 0x09F9, 0x2483, 0x2497, + 0x0E56, 0x002F, 0x002F, 0x002F, 0x002F, 0xFF0F, 0x017F, 0x1E9B, 0x263A, 0xFF53, 0x05C3, 0x00AD, + 0x044C, 0x305D, 0x30BD, 0xFF7F, 0x0338, 0x0337, 0x0E29, 0x0E28, 0x0E0B, 0x0E2A, 0x0020, 0x2660, + 0x2664, 0x24AE, 0x033B, 0x33C4, 0x339D, 0x25A9, 0x25A4, 0x338F, 0x339E, 0x33CE, 0x33D1, 0x33D2, + 0x338E, 0x33D5, 0x339C, 0x33A1, 0x25A6, 0x25A7, 0x25A8, 0x25A5, 0x25A3, 0x33DB, 0x09B7, 0x0937, + 0x0AB7, 0x3149, 0x3185, 0x3180, 0x3132, 0x3165, 0x3143, 0x3146, 0x3138, 0xFFE1, 0x0336, 0x0335, + 0x2282, 0x228A, 0x2286, 0x227B, 0x3059, 0x30B9, 0xFF7D, 0x0652, 0x2211, 0x2211, 0x263C, 0x2283, + 0x228B, 0x2287, 0x33DC, 0x337C, 0x09A4, 0x22A4, 0x22A3, 0x0924, 0x0AA4, 0x0A24, 0x0637, 0xFEC2, + 0xFEC3, 0x305F, 0xFEC4, 0x337D, 0x30BF, 0xFF80, 0x0640, 0x05EA, 0xFB4A, 0xFB4A, 0xFB4A, 0x05EA, + 0x0167, 0x310A, 0x02A8, 0x0163, 0x0686, 0xFB7B, 0xFB7C, 0xFB7D, 0xD82E, 0x24E3, 0x1E71, 0x1E97, + 0x1E6B, 0x1E6D, 0x0442, 0x04AD, 0x062A, 0xFE96, 0xFCA2, 0xFC0C, 0xFE97, 0x3066, 0xFCA1, 0xFC0B, + 0x0629, 0xFE94, 0xFE98, 0xFCA4, 0xFC0E, 0xFC73, 0x30C6, 0xFF83, 0x2121, 0x260E, 0x05A0, 0x05A9, + 0x2469, 0x3229, 0x247D, 0x2491, 0x2179, 0x02A7, 0x05D8, 0xFB38, 0xFB38, 0x05D8, 0x04B5, 0x059B, + 0x059B, 0x09A5, 0x0925, 0x0AA5, 0x0A25, 0x0630, 0xFEAC, 0xF898, 0xF897, 0x0E4C, 0xF896, 0x062B, + 0xFE9A, 0xFE9B, 0xFE9C, 0x2203, 0x03D1, 0x3279, 0x3219, 0x326B, 0x314C, 0x320B, 0x246C, 0x2480, + 0x2494, 0x0E11, 0x01AD, 0x0E12, 0x0E17, 0x0E10, 0x0E18, 0x0E16, 0x0482, 0x066C, 0x066C, 0x0663, + 0x09E9, 0x2462, 0x278C, 0x0969, 0x0AE9, 0x0A69, 0x0663, 0x3023, 0x3222, 0xFF13, 0x09F6, 0x2476, + 0x248A, 0x06F3, 0x2172, 0x0E53, 0x3394, 0x3061, 0x30C1, 0xFF81, 0x3270, 0x3210, 0x3262, 0x3137, + 0x3202, 0x0330, 0x0303, 0x0303, 0x0360, 0x223C, 0x0334, 0x033E, 0x02DC, 0x02DC, 0x02DC, 0x2297, + 0x0596, 0x0596, 0x0A70, 0x0483, 0x057F, 0x1E6F, 0xFF54, 0x0569, 0x3068, 0x30C8, 0xFF84, 0x02E5, + 0x02E9, 0x02E6, 0x02E8, 0x02E7, 0x01BD, 0x0185, 0x01A8, 0x0384, 0x3327, 0x0E0F, 0x3014, 0xFE5D, + 0xFE39, 0x3015, 0xFE5E, 0xFE3A, 0x0E15, 0x01AB, 0x24AF, 0x0288, 0x25BC, 0x25C4, 0x25BA, 0x25B2, + 0x02A6, 0x05E6, 0xFB46, 0xFB46, 0x05E6, 0x0446, 0x05B5, 0x05B5, 0x05B5, 0x05B5, 0x05B5, 0x05B5, + 0x05B5, 0x05B5, 0x045B, 0x099F, 0x091F, 0x0A9F, 0x0A1F, 0x0679, 0xFB67, 0xFB68, 0xFB69, 0x09A0, + 0x0920, 0x0AA0, 0x0A20, 0x0287, 0x3064, 0x30C4, 0xFF82, 0x3063, 0x30C3, 0xFF6F, 0x246B, 0x247F, + 0x2493, 0x217B, 0x2473, 0x5344, 0x2487, 0x249B, 0x0662, 0x09E8, 0x2461, 0x278B, 0x0968, 0x2025, + 0xFE30, 0x0AE8, 0x0A68, 0x0662, 0x3022, 0x3221, 0xFF12, 0x09F5, 0x2475, 0x2489, 0x06F2, 0x2171, + 0x01BB, 0x0E52, 0x0289, 0x0989, 0x3128, 0x016D, 0x01D4, 0x24E4, 0x1E77, 0x0443, 0x0951, 0x0171, + 0x0215, 0x0909, 0x01D8, 0x1E73, 0x01DA, 0x04F1, 0x01DC, 0x01D6, 0x1EE5, 0x0A89, 0x0A09, 0x3046, + 0x1EE7, 0x01B0, 0x1EE9, 0x1EF1, 0x1EEB, 0x1EED, 0x1EEF, 0x04F3, 0x0217, 0x30A6, 0xFF73, 0x0479, + 0x315C, 0x04EF, 0x1E7B, 0x0A41, 0xFF55, 0x2017, 0xFF3F, 0xFE33, 0xFE4F, 0x22C3, 0x228E, 0x228E, + 0x2294, 0x2294, 0x22C3, 0x24B0, 0x2580, 0x05C4, 0x03CB, 0x03B0, 0x028A, 0x03CD, 0x031D, 0x02D4, + 0x0A73, 0x045E, 0x3045, 0x30A5, 0xFF69, 0x04AF, 0x04B1, 0x0169, 0x1E79, 0x1E75, 0x098A, 0x090A, + 0x0A8A, 0x0A0A, 0x0A42, 0x09C2, 0x0942, 0x0AC2, 0x09C1, 0x0941, 0x0AC1, 0x0935, 0x0AB5, 0x0A35, + 0x30F7, 0x05D5, 0xFB35, 0xFB35, 0xFB35, 0x05D5, 0xFB4B, 0xFB4B, 0x05F0, 0x05F1, 0x24E5, 0x1E7F, + 0x0432, 0x06A4, 0xFB6B, 0xFB6C, 0xFB6D, 0x30F9, 0x2640, 0x007C, 0x030D, 0x0329, 0x02CC, 0x02C8, + 0x057E, 0x2225, 0x2223, 0x028B, 0x30F8, 0x09CD, 0x094D, 0x0ACD, 0x0983, 0x0903, 0x0A83, 0xFF56, + 0x0578, 0x309E, 0x30FE, 0x309B, 0xFF9E, 0x30FA, 0x24B1, 0x1E7D, 0x028C, 0x3094, 0x30F4, 0x1E83, + 0x3159, 0x308F, 0x30EF, 0xFF9C, 0x3158, 0x308E, 0x30EE, 0x3357, 0x301C, 0xFE34, 0x0648, 0xFEEE, + 0x0624, 0xFE86, 0x33DD, 0x24E6, 0x0175, 0x1E85, 0x1E87, 0x1E89, 0x3091, 0x30F1, 0x315E, 0x315D, + 0x1E81, 0x25E6, 0x25CB, 0x25D9, 0x300E, 0xFE43, 0x300F, 0xFE44, 0x25C7, 0x25C8, 0x25BF, 0x25BD, + 0x25C3, 0x25C1, 0x3016, 0x3017, 0x25B9, 0x25B7, 0x25AB, 0x263A, 0x25A1, 0x2606, 0x260F, 0x3018, + 0x3019, 0x25B5, 0x25B3, 0x3090, 0x30F0, 0x315F, 0xFF57, 0x3092, 0x30F2, 0xFF66, 0x20A9, 0xFFE6, + 0x0E27, 0x24B2, 0x1E98, 0x02B7, 0x028D, 0x01BF, 0x033D, 0x3112, 0x24E7, 0x1E8D, 0x1E8B, 0x056D, + 0xFF58, 0x24B3, 0x02E3, 0x334E, 0x09AF, 0x092F, 0x3152, 0x0AAF, 0x0A2F, 0x3084, 0x30E4, 0xFF94, + 0x3151, 0x0E4E, 0x3083, 0x30E3, 0xFF6C, 0x0463, 0x24E8, 0x0177, 0x1E8F, 0x1EF5, 0x064A, 0x06D2, + 0xFBAF, 0xFEF2, 0x0626, 0xFE8A, 0xFE8B, 0xFE8C, 0xFEF3, 0xFEF4, 0xFCDD, 0xFC58, 0xFC94, 0x06D1, + 0x3156, 0xFFE5, 0x3155, 0x3186, 0x05AA, 0x05AA, 0x044B, 0x04F9, 0x3181, 0x3183, 0x3182, 0x059A, + 0x1EF3, 0x01B4, 0x1EF7, 0x0575, 0x0457, 0x3162, 0x262F, 0x0582, 0xFF59, 0x05D9, 0xFB39, 0xFB39, + 0x05D9, 0x05F2, 0xFB1F, 0x3088, 0x3189, 0x30E8, 0xFF96, 0x315B, 0x3087, 0x30E7, 0xFF6E, 0x03F3, + 0x3188, 0x3187, 0x0E22, 0x0E0D, 0x24B4, 0x037A, 0x0345, 0x01A6, 0x1E99, 0x02B8, 0x1EF9, 0x028E, + 0x3086, 0x318C, 0x30E6, 0xFF95, 0x3160, 0x046B, 0x046D, 0x0467, 0x0469, 0x3085, 0x30E5, 0xFF6D, + 0x318B, 0x318A, 0x09DF, 0x095F, 0x0566, 0x095B, 0x0A5B, 0x0638, 0xFEC6, 0xFEC7, 0x3056, 0xFEC8, + 0x0632, 0xFEB0, 0x30B6, 0x0595, 0x0594, 0x0598, 0x05D6, 0xFB36, 0xFB36, 0x05D6, 0x3117, 0x24E9, + 0x1E91, 0x0291, 0x017C, 0x1E93, 0x0437, 0x0499, 0x04DF, 0x305C, 0x30BC, 0x0660, 0x09E6, 0x0966, + 0x0AE6, 0x0A66, 0x0660, 0xFF10, 0x06F0, 0x0E50, 0xFEFF, 0x200C, 0x200B, 0x3113, 0x056A, 0x04C2, + 0x0436, 0x0497, 0x04DD, 0x3058, 0x30B8, 0x05AE, 0x1E95, 0xFF5A, 0x305E, 0x30BE, 0x24B5, 0x0290, + 0x01B6, 0x305A, 0x30BA, +]; + +pub(super) static ZAPF_VALUES: [u16; 206] = [ + 0x2701, 0x2702, 0x2704, 0x260E, 0x2706, 0x271D, 0x271E, 0x271F, 0x2720, 0x2721, 0x261B, 0x261E, + 0x270C, 0x270D, 0x270E, 0x270F, 0x2711, 0x2712, 0x2713, 0x2714, 0x2715, 0x2716, 0x2717, 0x2718, + 0x2719, 0x271A, 0x271B, 0x271C, 0x2722, 0x2723, 0x2724, 0x2725, 0x2726, 0x2727, 0x2605, 0x2729, + 0x272A, 0x272B, 0x272C, 0x272D, 0x272E, 0x272F, 0x2730, 0x2731, 0x2732, 0x2733, 0x2734, 0x2735, + 0x2736, 0x2737, 0x2738, 0x2739, 0x273A, 0x273B, 0x273C, 0x273D, 0x273E, 0x273F, 0x2740, 0x2741, + 0x2742, 0x2743, 0x2744, 0x2745, 0x2746, 0x2747, 0x2748, 0x2749, 0x274A, 0x274B, 0x25CF, 0x274D, + 0x25A0, 0x274F, 0x2751, 0x25B2, 0x25BC, 0x25C6, 0x2756, 0x0000, 0x25D7, 0x2758, 0x2759, 0x275A, + 0x276F, 0x2771, 0x2772, 0x2773, 0x2768, 0x2769, 0x276C, 0x276D, 0x276A, 0x276B, 0x2774, 0x2775, + 0x275B, 0x275C, 0x275D, 0x275E, 0x2761, 0x2762, 0x2763, 0x2764, 0x2710, 0x2765, 0x2766, 0x2767, + 0x2660, 0x2665, 0x2666, 0x2663, 0x0000, 0x0000, 0x0000, 0x0000, 0x2709, 0x2708, 0x2707, 0x2460, + 0x2461, 0x2462, 0x2463, 0x2464, 0x2465, 0x2466, 0x2467, 0x2468, 0x2469, 0x2776, 0x2777, 0x2778, + 0x2779, 0x277A, 0x277B, 0x277C, 0x277D, 0x277E, 0x277F, 0x2780, 0x2781, 0x2782, 0x2783, 0x2784, + 0x2785, 0x2786, 0x2787, 0x2788, 0x2789, 0x278A, 0x278B, 0x278C, 0x278D, 0x278E, 0x278F, 0x2790, + 0x2791, 0x2792, 0x2793, 0x2794, 0x2192, 0x27A3, 0x2194, 0x2195, 0x2799, 0x279B, 0x279C, 0x279D, + 0x279E, 0x279F, 0x27A0, 0x27A1, 0x27A2, 0x27A4, 0x27A5, 0x27A6, 0x27A7, 0x27A8, 0x27A9, 0x27AB, + 0x27AD, 0x27AF, 0x27B2, 0x27B3, 0x27B5, 0x27B8, 0x27BA, 0x27BB, 0x27BC, 0x27BD, 0x27BE, 0x279A, + 0x27AA, 0x27B6, 0x27B9, 0x2798, 0x27B4, 0x27B7, 0x27AC, 0x27AE, 0x27B1, 0x2703, 0x2750, 0x2752, + 0x276E, 0x2770, +]; + +pub(super) static MULTI_TEXT: &str = "\u{5d3}\u{5b2}\u{5d3}\u{5b1}\u{5d3}\u{5b4}\u{5d3}\u{5b9}\u{5d3}\u{5b7}\u{5d3}\u{5b8}\u{5d3}\u{5bb}\u{5d3}\u{5b6}\u{5d3}\u{5b0}\u{5d3}\u{5b5}\u{5da}\u{5b8}\u{5da}\u{5b0}\u{621}\u{64f}\u{621}\u{64c}\u{621}\u{64e}\u{621}\u{64b}\u{621}\u{650}\u{621}\u{64d}\u{621}\u{652}\u{5dc}\u{5b9}\u{5dc}\u{5b9}\u{5bc}\u{fedf}\u{fee4}\u{fea0}\u{fedf}\u{fee4}\u{fea8}\u{fee7}\u{feec}\u{5e7}\u{5b2}\u{5e7}\u{5b1}\u{5e7}\u{5b4}\u{5e7}\u{5b9}\u{5e7}\u{5b7}\u{5e7}\u{5b8}\u{5e7}\u{5bb}\u{5e7}\u{5b6}\u{5e7}\u{5b0}\u{5e7}\u{5b5}\u{631}\u{fef3}\u{fe8e}\u{644}\u{5e8}\u{5b2}\u{5e8}\u{5b1}\u{5e8}\u{5b4}\u{5e8}\u{5b9}\u{5e8}\u{5b7}\u{5e8}\u{5b8}\u{5e8}\u{5bb}\u{5e8}\u{5b6}\u{5e8}\u{5b0}\u{5e8}\u{5b5}\u{651}\u{64b}\u{fb7c}\u{fee4}"; +pub(super) static MULTI_OFFSETS: [u16; 48] = [ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 86, 95, 104, + 110, 114, 118, 122, 126, 130, 134, 138, 142, 146, 150, 160, 164, 168, 172, 176, 180, 184, 188, + 192, 196, 200, 204, 210, +]; diff --git a/crates/text-extract/src/pdf/tables/glyphs.rs b/crates/text-extract/src/pdf/tables/glyphs.rs new file mode 100644 index 00000000..f79f8f4f --- /dev/null +++ b/crates/text-extract/src/pdf/tables/glyphs.rs @@ -0,0 +1,301 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::cmp::Ordering; +use std::ops::RangeInclusive; + +use super::encoding::std_glyph_char; +use super::glyph_data::{ + BLOCK_OFFSETS, BLOCK_SIZE, GLYPH_VALUES, MAX_NAME, MULTI_BASE, MULTI_OFFSETS, MULTI_TEXT, + NAME_BLOCKS, PREFIX_LIMIT, TOKEN_BASE, TOKEN_OFFSETS, TOKEN_TEXT, ZAPF_VALUES, +}; +use super::names::NameTable; + +static TOKENS: NameTable = NameTable::new(TOKEN_TEXT, &TOKEN_OFFSETS); +static MULTI: NameTable = NameTable::new(MULTI_TEXT, &MULTI_OFFSETS); + +const MAX_SCALAR: u32 = 0x10FFFF; +const MIN_NAMED_CODE: u32 = 0x20; +const C1_CONTROLS: RangeInclusive = 0x7F..=0x9F; +const MAX_ZAPF_DIGITS: usize = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CodeRadix { + Decimal, + Hex, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GlyphCode { + Code(u32), + NeedsHex, +} + +#[derive(Clone, Copy)] +enum HexCase { + Upper, + Any, +} + +pub(crate) fn glyph_unicode(name: &[u8], out: &mut String) -> bool { + let base = strip_suffix(name); + if base.is_empty() { + return false; + } + let mark = out.len(); + for component in base.split(|byte| *byte == b'_') { + if !append_component(component, out) { + out.truncate(mark); + return false; + } + } + true +} + +pub(crate) fn zapf_dingbats_unicode(name: &[u8], out: &mut String) -> bool { + match zapf_char(strip_suffix(name)) { + Some(ch) => { + out.push(ch); + true + } + None => glyph_unicode(name, out), + } +} + +fn strip_suffix(name: &[u8]) -> &[u8] { + name.split(|byte| *byte == b'.').next().unwrap_or_default() +} + +fn zapf_digits(name: &[u8]) -> Option<&[u8]> { + name.strip_prefix(b"a") + .filter(|digits| !digits.is_empty() && digits.iter().all(u8::is_ascii_digit)) +} + +fn zapf_char(name: &[u8]) -> Option { + let digits = zapf_digits(name) + .filter(|digits| digits.len() <= MAX_ZAPF_DIGITS && digits.first() != Some(&b'0'))?; + let index = usize::try_from(parse_decimal(digits)?) + .ok()? + .checked_sub(1)?; + let value = *ZAPF_VALUES.get(index)?; + (value != 0) + .then(|| char::from_u32(u32::from(value))) + .flatten() +} + +pub(crate) fn glyph_name_code(name: &[u8], c_radix: CodeRadix) -> Option { + let (&first, digits) = name.split_first()?; + let code = match (first, digits.len()) { + (b'G', 2) | (b'g', 4) => parse_hex(digits, HexCase::Any)?, + (b'C' | b'c', 2..=3) => match c_radix { + CodeRadix::Hex => parse_hex(digits, HexCase::Any)?, + CodeRadix::Decimal => match parse_decimal(digits) { + Some(code) => code, + None => { + return parse_hex(digits, HexCase::Any).map(|_| GlyphCode::NeedsHex); + } + }, + }, + _ => return None, + }; + ((MIN_NAMED_CODE..=MAX_SCALAR).contains(&code) && !C1_CONTROLS.contains(&code)) + .then_some(GlyphCode::Code(code)) +} + +fn append_component(component: &[u8], out: &mut String) -> bool { + if zapf_digits(component).is_some() { + return false; + } + if let Some(ch) = std_glyph_char(component) { + out.push(ch); + return true; + } + if let Some(index) = glyph_index(component) { + return append_value(index, out).is_some(); + } + if let Some(hex) = component.strip_prefix(b"uni") + && append_uni(hex, out) + { + return true; + } + component + .strip_prefix(b"u") + .is_some_and(|hex| append_u(hex, out)) +} + +fn append_uni(hex: &[u8], out: &mut String) -> bool { + if hex.is_empty() || !hex.len().is_multiple_of(4) { + return false; + } + let mark = out.len(); + for group in hex.chunks(4) { + match parse_hex(group, HexCase::Any).and_then(char::from_u32) { + Some(ch) => out.push(ch), + None => { + out.truncate(mark); + return false; + } + } + } + true +} + +fn append_u(hex: &[u8], out: &mut String) -> bool { + if !(4..=6).contains(&hex.len()) { + return false; + } + match parse_hex(hex, HexCase::Upper).and_then(char::from_u32) { + Some(ch) => { + out.push(ch); + true + } + None => false, + } +} + +fn parse_hex(digits: &[u8], case: HexCase) -> Option { + digits.iter().try_fold(0u32, |acc, &byte| { + let digit = match (byte, case) { + (b'0'..=b'9', _) => byte - b'0', + (b'A'..=b'F', _) => byte - b'A' + 10, + (b'a'..=b'f', HexCase::Any) => byte - b'a' + 10, + _ => return None, + }; + Some((acc << 4) | u32::from(digit)) + }) +} + +fn parse_decimal(digits: &[u8]) -> Option { + digits.iter().try_fold(0u32, |acc, &byte| { + if byte.is_ascii_digit() { + acc.checked_mul(10)?.checked_add(u32::from(byte - b'0')) + } else { + None + } + }) +} + +fn append_value(index: usize, out: &mut String) -> Option<()> { + let value = *GLYPH_VALUES.get(index)?; + match char::from_u32(u32::from(value)) { + Some(ch) => out.push(ch), + None => out.push_str(MULTI.get(usize::from(value.checked_sub(MULTI_BASE)?))?), + } + Some(()) +} + +pub(super) fn glyph_index(name: &[u8]) -> Option { + if name.is_empty() || name.len() > MAX_NAME { + return None; + } + let mut buf = NameBuf::new(); + let (mut low, mut high) = (0, block_count()); + while low < high { + let mid = low + (high - low) / 2; + let (_, body) = Entries::new(block(mid)?).next()?; + buf.load(0, body)?; + if buf.as_slice() <= name { + low = mid + 1; + } else { + high = mid; + } + } + let block_index = low.checked_sub(1)?; + for (position, (prefix, body)) in Entries::new(block(block_index)?).enumerate() { + buf.load(prefix, body)?; + match buf.as_slice().cmp(name) { + Ordering::Less => {} + Ordering::Equal => return Some(block_index * BLOCK_SIZE + position), + Ordering::Greater => return None, + } + } + None +} + +fn block_count() -> usize { + BLOCK_OFFSETS.len().saturating_sub(1) +} + +fn block(index: usize) -> Option<&'static [u8]> { + let &[start, end] = BLOCK_OFFSETS.get(index..index + 2)? else { + return None; + }; + NAME_BLOCKS.get(usize::from(start)..usize::from(end)) +} + +struct Entries { + rest: &'static [u8], +} + +impl Entries { + fn new(block: &'static [u8]) -> Self { + Self { rest: block } + } +} + +impl Iterator for Entries { + type Item = (usize, &'static [u8]); + + fn next(&mut self) -> Option { + let (&prefix, tail) = self.rest.split_first()?; + let body = tail + .split(|byte| *byte < PREFIX_LIMIT) + .next() + .unwrap_or_default(); + self.rest = tail.get(body.len()..).unwrap_or_default(); + Some((usize::from(prefix), body)) + } +} + +struct NameBuf { + bytes: [u8; MAX_NAME], + len: usize, +} + +impl NameBuf { + fn new() -> Self { + Self { + bytes: [0; MAX_NAME], + len: 0, + } + } + + fn as_slice(&self) -> &[u8] { + self.bytes.get(..self.len).unwrap_or_default() + } + + fn load(&mut self, prefix: usize, body: &[u8]) -> Option<()> { + self.len = prefix.min(self.len); + for &byte in body { + match byte.checked_sub(TOKEN_BASE) { + Some(token) => self.extend(TOKENS.get(usize::from(token))?.as_bytes())?, + None => self.extend(&[byte])?, + } + } + Some(()) + } + + fn extend(&mut self, bytes: &[u8]) -> Option<()> { + let end = self.len + bytes.len(); + self.bytes.get_mut(self.len..end)?.copy_from_slice(bytes); + self.len = end; + Some(()) + } +} + +#[cfg(all(test, stalwart_tests))] +pub(super) fn all_names() -> Vec { + let mut names = Vec::with_capacity(GLYPH_VALUES.len()); + let mut buf = NameBuf::new(); + for index in 0..block_count() { + for (prefix, body) in Entries::new(block(index).unwrap_or_default()) { + if buf.load(prefix, body).is_some() { + names.push(String::from_utf8_lossy(buf.as_slice()).into_owned()); + } + } + } + names +} diff --git a/crates/text-extract/src/pdf/tables/mod.rs b/crates/text-extract/src/pdf/tables/mod.rs new file mode 100644 index 00000000..778e6395 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/mod.rs @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod cid; +mod cid_data; +mod cmap_data; +mod cmaps; +mod encoding; +mod encoding_data; +mod font_data; +mod fonts; +mod glyph_data; +mod glyphs; +mod names; +mod normalize; +mod presentation_data; + + +pub(crate) use self::cid::CidCollection; +pub(crate) use self::cmaps::{CmapDecoder, CodeRange, PredefinedCmap}; +pub(crate) use self::encoding::{BaseEncoding, pdfdoc_char}; +pub(crate) use self::fonts::StandardFont; +pub(crate) use self::glyphs::{ + CodeRadix, GlyphCode, glyph_name_code, glyph_unicode, zapf_dingbats_unicode, +}; +pub(crate) use self::normalize::{combining_accent, presentation_form}; diff --git a/crates/text-extract/src/pdf/tables/names.rs b/crates/text-extract/src/pdf/tables/names.rs new file mode 100644 index 00000000..95c1f540 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/names.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::cmp::Ordering; + +pub(super) struct NameTable { + text: &'static str, + offsets: &'static [u16], +} + +impl NameTable { + pub(super) const fn new(text: &'static str, offsets: &'static [u16]) -> Self { + Self { text, offsets } + } + + pub(super) fn len(&self) -> usize { + self.offsets.len().saturating_sub(1) + } + + pub(super) fn get(&self, index: usize) -> Option<&'static str> { + let &[start, end] = self.offsets.get(index..index + 2)? else { + return None; + }; + self.text.get(usize::from(start)..usize::from(end)) + } + + pub(super) fn find(&self, name: &[u8]) -> Option { + let (mut low, mut high) = (0, self.len()); + while low < high { + let mid = low + (high - low) / 2; + match self.get(mid)?.as_bytes().cmp(name) { + Ordering::Less => low = mid + 1, + Ordering::Greater => high = mid, + Ordering::Equal => return Some(mid), + } + } + None + } +} diff --git a/crates/text-extract/src/pdf/tables/normalize.rs b/crates/text-extract/src/pdf/tables/normalize.rs new file mode 100644 index 00000000..5ebd9cf2 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/normalize.rs @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::names::NameTable; +use super::presentation_data::{FORM_KEYS, FORM_OFFSETS, FORM_TEXT, FORM_VALUES}; + +static FORMS: NameTable = NameTable::new(FORM_TEXT, &FORM_OFFSETS); + +pub(crate) fn presentation_form(c: char) -> Option<&'static str> { + let key = u16::try_from(u32::from(c)).ok()?; + let position = FORM_KEYS.binary_search(&key).ok()?; + FORMS.get(usize::from(*FORM_VALUES.get(position)?)) +} + +pub(crate) fn combining_accent(c: char) -> Option { + match c { + '\u{60}' => Some('\u{300}'), + '\u{A8}' => Some('\u{308}'), + '\u{AF}' => Some('\u{304}'), + '\u{B4}' => Some('\u{301}'), + '\u{B8}' => Some('\u{327}'), + '\u{2C6}' => Some('\u{302}'), + '\u{2C7}' => Some('\u{30C}'), + '\u{2D8}' => Some('\u{306}'), + '\u{2D9}' => Some('\u{307}'), + '\u{2DA}' => Some('\u{30A}'), + '\u{2DB}' => Some('\u{328}'), + '\u{2DC}' => Some('\u{303}'), + '\u{2DD}' => Some('\u{30B}'), + _ => None, + } +} diff --git a/crates/text-extract/src/pdf/tables/presentation_data.rs b/crates/text-extract/src/pdf/tables/presentation_data.rs new file mode 100644 index 00000000..033ab303 --- /dev/null +++ b/crates/text-extract/src/pdf/tables/presentation_data.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) static FORM_KEYS: [u16; 788] = [ + 0xFB00, 0xFB01, 0xFB02, 0xFB03, 0xFB04, 0xFB05, 0xFB06, 0xFB13, 0xFB14, 0xFB15, 0xFB16, 0xFB17, + 0xFB1D, 0xFB1F, 0xFB20, 0xFB21, 0xFB22, 0xFB23, 0xFB24, 0xFB25, 0xFB26, 0xFB27, 0xFB28, 0xFB29, + 0xFB2A, 0xFB2B, 0xFB2C, 0xFB2D, 0xFB2E, 0xFB2F, 0xFB30, 0xFB31, 0xFB32, 0xFB33, 0xFB34, 0xFB35, + 0xFB36, 0xFB38, 0xFB39, 0xFB3A, 0xFB3B, 0xFB3C, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, + 0xFB47, 0xFB48, 0xFB49, 0xFB4A, 0xFB4B, 0xFB4C, 0xFB4D, 0xFB4E, 0xFB4F, 0xFB50, 0xFB51, 0xFB52, + 0xFB53, 0xFB54, 0xFB55, 0xFB56, 0xFB57, 0xFB58, 0xFB59, 0xFB5A, 0xFB5B, 0xFB5C, 0xFB5D, 0xFB5E, + 0xFB5F, 0xFB60, 0xFB61, 0xFB62, 0xFB63, 0xFB64, 0xFB65, 0xFB66, 0xFB67, 0xFB68, 0xFB69, 0xFB6A, + 0xFB6B, 0xFB6C, 0xFB6D, 0xFB6E, 0xFB6F, 0xFB70, 0xFB71, 0xFB72, 0xFB73, 0xFB74, 0xFB75, 0xFB76, + 0xFB77, 0xFB78, 0xFB79, 0xFB7A, 0xFB7B, 0xFB7C, 0xFB7D, 0xFB7E, 0xFB7F, 0xFB80, 0xFB81, 0xFB82, + 0xFB83, 0xFB84, 0xFB85, 0xFB86, 0xFB87, 0xFB88, 0xFB89, 0xFB8A, 0xFB8B, 0xFB8C, 0xFB8D, 0xFB8E, + 0xFB8F, 0xFB90, 0xFB91, 0xFB92, 0xFB93, 0xFB94, 0xFB95, 0xFB96, 0xFB97, 0xFB98, 0xFB99, 0xFB9A, + 0xFB9B, 0xFB9C, 0xFB9D, 0xFB9E, 0xFB9F, 0xFBA0, 0xFBA1, 0xFBA2, 0xFBA3, 0xFBA4, 0xFBA5, 0xFBA6, + 0xFBA7, 0xFBA8, 0xFBA9, 0xFBAA, 0xFBAB, 0xFBAC, 0xFBAD, 0xFBAE, 0xFBAF, 0xFBB0, 0xFBB1, 0xFBD3, + 0xFBD4, 0xFBD5, 0xFBD6, 0xFBD7, 0xFBD8, 0xFBD9, 0xFBDA, 0xFBDB, 0xFBDC, 0xFBDD, 0xFBDE, 0xFBDF, + 0xFBE0, 0xFBE1, 0xFBE2, 0xFBE3, 0xFBE4, 0xFBE5, 0xFBE6, 0xFBE7, 0xFBE8, 0xFBE9, 0xFBEA, 0xFBEB, + 0xFBEC, 0xFBED, 0xFBEE, 0xFBEF, 0xFBF0, 0xFBF1, 0xFBF2, 0xFBF3, 0xFBF4, 0xFBF5, 0xFBF6, 0xFBF7, + 0xFBF8, 0xFBF9, 0xFBFA, 0xFBFB, 0xFBFC, 0xFBFD, 0xFBFE, 0xFBFF, 0xFC00, 0xFC01, 0xFC02, 0xFC03, + 0xFC04, 0xFC05, 0xFC06, 0xFC07, 0xFC08, 0xFC09, 0xFC0A, 0xFC0B, 0xFC0C, 0xFC0D, 0xFC0E, 0xFC0F, + 0xFC10, 0xFC11, 0xFC12, 0xFC13, 0xFC14, 0xFC15, 0xFC16, 0xFC17, 0xFC18, 0xFC19, 0xFC1A, 0xFC1B, + 0xFC1C, 0xFC1D, 0xFC1E, 0xFC1F, 0xFC20, 0xFC21, 0xFC22, 0xFC23, 0xFC24, 0xFC25, 0xFC26, 0xFC27, + 0xFC28, 0xFC29, 0xFC2A, 0xFC2B, 0xFC2C, 0xFC2D, 0xFC2E, 0xFC2F, 0xFC30, 0xFC31, 0xFC32, 0xFC33, + 0xFC34, 0xFC35, 0xFC36, 0xFC37, 0xFC38, 0xFC39, 0xFC3A, 0xFC3B, 0xFC3C, 0xFC3D, 0xFC3E, 0xFC3F, + 0xFC40, 0xFC41, 0xFC42, 0xFC43, 0xFC44, 0xFC45, 0xFC46, 0xFC47, 0xFC48, 0xFC49, 0xFC4A, 0xFC4B, + 0xFC4C, 0xFC4D, 0xFC4E, 0xFC4F, 0xFC50, 0xFC51, 0xFC52, 0xFC53, 0xFC54, 0xFC55, 0xFC56, 0xFC57, + 0xFC58, 0xFC59, 0xFC5A, 0xFC5B, 0xFC5C, 0xFC5D, 0xFC5E, 0xFC5F, 0xFC60, 0xFC61, 0xFC62, 0xFC63, + 0xFC64, 0xFC65, 0xFC66, 0xFC67, 0xFC68, 0xFC69, 0xFC6A, 0xFC6B, 0xFC6C, 0xFC6D, 0xFC6E, 0xFC6F, + 0xFC70, 0xFC71, 0xFC72, 0xFC73, 0xFC74, 0xFC75, 0xFC76, 0xFC77, 0xFC78, 0xFC79, 0xFC7A, 0xFC7B, + 0xFC7C, 0xFC7D, 0xFC7E, 0xFC7F, 0xFC80, 0xFC81, 0xFC82, 0xFC83, 0xFC84, 0xFC85, 0xFC86, 0xFC87, + 0xFC88, 0xFC89, 0xFC8A, 0xFC8B, 0xFC8C, 0xFC8D, 0xFC8E, 0xFC8F, 0xFC90, 0xFC91, 0xFC92, 0xFC93, + 0xFC94, 0xFC95, 0xFC96, 0xFC97, 0xFC98, 0xFC99, 0xFC9A, 0xFC9B, 0xFC9C, 0xFC9D, 0xFC9E, 0xFC9F, + 0xFCA0, 0xFCA1, 0xFCA2, 0xFCA3, 0xFCA4, 0xFCA5, 0xFCA6, 0xFCA7, 0xFCA8, 0xFCA9, 0xFCAA, 0xFCAB, + 0xFCAC, 0xFCAD, 0xFCAE, 0xFCAF, 0xFCB0, 0xFCB1, 0xFCB2, 0xFCB3, 0xFCB4, 0xFCB5, 0xFCB6, 0xFCB7, + 0xFCB8, 0xFCB9, 0xFCBA, 0xFCBB, 0xFCBC, 0xFCBD, 0xFCBE, 0xFCBF, 0xFCC0, 0xFCC1, 0xFCC2, 0xFCC3, + 0xFCC4, 0xFCC5, 0xFCC6, 0xFCC7, 0xFCC8, 0xFCC9, 0xFCCA, 0xFCCB, 0xFCCC, 0xFCCD, 0xFCCE, 0xFCCF, + 0xFCD0, 0xFCD1, 0xFCD2, 0xFCD3, 0xFCD4, 0xFCD5, 0xFCD6, 0xFCD7, 0xFCD8, 0xFCD9, 0xFCDA, 0xFCDB, + 0xFCDC, 0xFCDD, 0xFCDE, 0xFCDF, 0xFCE0, 0xFCE1, 0xFCE2, 0xFCE3, 0xFCE4, 0xFCE5, 0xFCE6, 0xFCE7, + 0xFCE8, 0xFCE9, 0xFCEA, 0xFCEB, 0xFCEC, 0xFCED, 0xFCEE, 0xFCEF, 0xFCF0, 0xFCF1, 0xFCF2, 0xFCF3, + 0xFCF4, 0xFCF5, 0xFCF6, 0xFCF7, 0xFCF8, 0xFCF9, 0xFCFA, 0xFCFB, 0xFCFC, 0xFCFD, 0xFCFE, 0xFCFF, + 0xFD00, 0xFD01, 0xFD02, 0xFD03, 0xFD04, 0xFD05, 0xFD06, 0xFD07, 0xFD08, 0xFD09, 0xFD0A, 0xFD0B, + 0xFD0C, 0xFD0D, 0xFD0E, 0xFD0F, 0xFD10, 0xFD11, 0xFD12, 0xFD13, 0xFD14, 0xFD15, 0xFD16, 0xFD17, + 0xFD18, 0xFD19, 0xFD1A, 0xFD1B, 0xFD1C, 0xFD1D, 0xFD1E, 0xFD1F, 0xFD20, 0xFD21, 0xFD22, 0xFD23, + 0xFD24, 0xFD25, 0xFD26, 0xFD27, 0xFD28, 0xFD29, 0xFD2A, 0xFD2B, 0xFD2C, 0xFD2D, 0xFD2E, 0xFD2F, + 0xFD30, 0xFD31, 0xFD32, 0xFD33, 0xFD34, 0xFD35, 0xFD36, 0xFD37, 0xFD38, 0xFD39, 0xFD3A, 0xFD3B, + 0xFD3C, 0xFD3D, 0xFD50, 0xFD51, 0xFD52, 0xFD53, 0xFD54, 0xFD55, 0xFD56, 0xFD57, 0xFD58, 0xFD59, + 0xFD5A, 0xFD5B, 0xFD5C, 0xFD5D, 0xFD5E, 0xFD5F, 0xFD60, 0xFD61, 0xFD62, 0xFD63, 0xFD64, 0xFD65, + 0xFD66, 0xFD67, 0xFD68, 0xFD69, 0xFD6A, 0xFD6B, 0xFD6C, 0xFD6D, 0xFD6E, 0xFD6F, 0xFD70, 0xFD71, + 0xFD72, 0xFD73, 0xFD74, 0xFD75, 0xFD76, 0xFD77, 0xFD78, 0xFD79, 0xFD7A, 0xFD7B, 0xFD7C, 0xFD7D, + 0xFD7E, 0xFD7F, 0xFD80, 0xFD81, 0xFD82, 0xFD83, 0xFD84, 0xFD85, 0xFD86, 0xFD87, 0xFD88, 0xFD89, + 0xFD8A, 0xFD8B, 0xFD8C, 0xFD8D, 0xFD8E, 0xFD8F, 0xFD92, 0xFD93, 0xFD94, 0xFD95, 0xFD96, 0xFD97, + 0xFD98, 0xFD99, 0xFD9A, 0xFD9B, 0xFD9C, 0xFD9D, 0xFD9E, 0xFD9F, 0xFDA0, 0xFDA1, 0xFDA2, 0xFDA3, + 0xFDA4, 0xFDA5, 0xFDA6, 0xFDA7, 0xFDA8, 0xFDA9, 0xFDAA, 0xFDAB, 0xFDAC, 0xFDAD, 0xFDAE, 0xFDAF, + 0xFDB0, 0xFDB1, 0xFDB2, 0xFDB3, 0xFDB4, 0xFDB5, 0xFDB6, 0xFDB7, 0xFDB8, 0xFDB9, 0xFDBA, 0xFDBB, + 0xFDBC, 0xFDBD, 0xFDBE, 0xFDBF, 0xFDC0, 0xFDC1, 0xFDC2, 0xFDC3, 0xFDC4, 0xFDC5, 0xFDC6, 0xFDC7, + 0xFDF0, 0xFDF1, 0xFDF2, 0xFDF3, 0xFDF4, 0xFDF5, 0xFDF6, 0xFDF7, 0xFDF8, 0xFDF9, 0xFDFA, 0xFDFB, + 0xFDFC, 0xFE70, 0xFE71, 0xFE72, 0xFE74, 0xFE76, 0xFE77, 0xFE78, 0xFE79, 0xFE7A, 0xFE7B, 0xFE7C, + 0xFE7D, 0xFE7E, 0xFE7F, 0xFE80, 0xFE81, 0xFE82, 0xFE83, 0xFE84, 0xFE85, 0xFE86, 0xFE87, 0xFE88, + 0xFE89, 0xFE8A, 0xFE8B, 0xFE8C, 0xFE8D, 0xFE8E, 0xFE8F, 0xFE90, 0xFE91, 0xFE92, 0xFE93, 0xFE94, + 0xFE95, 0xFE96, 0xFE97, 0xFE98, 0xFE99, 0xFE9A, 0xFE9B, 0xFE9C, 0xFE9D, 0xFE9E, 0xFE9F, 0xFEA0, + 0xFEA1, 0xFEA2, 0xFEA3, 0xFEA4, 0xFEA5, 0xFEA6, 0xFEA7, 0xFEA8, 0xFEA9, 0xFEAA, 0xFEAB, 0xFEAC, + 0xFEAD, 0xFEAE, 0xFEAF, 0xFEB0, 0xFEB1, 0xFEB2, 0xFEB3, 0xFEB4, 0xFEB5, 0xFEB6, 0xFEB7, 0xFEB8, + 0xFEB9, 0xFEBA, 0xFEBB, 0xFEBC, 0xFEBD, 0xFEBE, 0xFEBF, 0xFEC0, 0xFEC1, 0xFEC2, 0xFEC3, 0xFEC4, + 0xFEC5, 0xFEC6, 0xFEC7, 0xFEC8, 0xFEC9, 0xFECA, 0xFECB, 0xFECC, 0xFECD, 0xFECE, 0xFECF, 0xFED0, + 0xFED1, 0xFED2, 0xFED3, 0xFED4, 0xFED5, 0xFED6, 0xFED7, 0xFED8, 0xFED9, 0xFEDA, 0xFEDB, 0xFEDC, + 0xFEDD, 0xFEDE, 0xFEDF, 0xFEE0, 0xFEE1, 0xFEE2, 0xFEE3, 0xFEE4, 0xFEE5, 0xFEE6, 0xFEE7, 0xFEE8, + 0xFEE9, 0xFEEA, 0xFEEB, 0xFEEC, 0xFEED, 0xFEEE, 0xFEEF, 0xFEF0, 0xFEF1, 0xFEF2, 0xFEF3, 0xFEF4, + 0xFEF5, 0xFEF6, 0xFEF7, 0xFEF8, 0xFEF9, 0xFEFA, 0xFEFB, 0xFEFC, +]; +pub(super) static FORM_VALUES: [u16; 788] = [ + 0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59, 60, 60, 60, + 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, + 66, 67, 67, 67, 67, 68, 68, 68, 68, 69, 69, 70, 70, 71, 71, 72, 72, 73, 73, 74, 74, 75, 75, 75, + 75, 76, 76, 76, 76, 77, 77, 77, 77, 78, 78, 78, 78, 79, 79, 80, 80, 80, 80, 81, 81, 82, 82, 82, + 82, 83, 83, 83, 83, 84, 84, 85, 85, 86, 86, 86, 86, 87, 87, 88, 88, 89, 89, 90, 91, 91, 92, 92, + 93, 93, 94, 94, 94, 94, 95, 95, 96, 96, 97, 97, 98, 98, 99, 99, 100, 100, 101, 101, 102, 102, + 102, 103, 103, 103, 104, 104, 104, 104, 105, 106, 107, 103, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, + 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 107, 206, 103, 108, + 207, 208, 112, 209, 113, 114, 210, 211, 118, 212, 119, 120, 213, 214, 122, 215, 123, 124, 153, + 154, 157, 158, 159, 163, 164, 165, 166, 170, 171, 172, 216, 176, 217, 218, 182, 219, 183, 184, + 197, 220, 221, 192, 222, 193, 194, 105, 106, 223, 107, 224, 109, 110, 111, 112, 225, 115, 116, + 117, 118, 226, 122, 125, 126, 127, 128, 129, 131, 132, 133, 134, 135, 136, 227, 137, 138, 139, + 140, 141, 142, 144, 145, 146, 147, 148, 149, 150, 151, 152, 155, 156, 160, 161, 162, 163, 164, + 167, 168, 169, 170, 228, 173, 174, 175, 176, 179, 180, 181, 182, 229, 185, 186, 230, 189, 190, + 191, 192, 231, 107, 224, 112, 225, 118, 226, 122, 232, 135, 233, 234, 235, 163, 164, 170, 182, + 229, 192, 231, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, + 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 234, 262, 263, 264, 265, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, + 234, 262, 263, 264, 265, 259, 260, 261, 234, 233, 235, 143, 132, 133, 134, 259, 260, 261, 143, + 144, 266, 266, 267, 268, 268, 269, 270, 271, 272, 273, 274, 274, 275, 276, 277, 278, 279, 280, + 280, 281, 282, 282, 283, 283, 284, 285, 285, 286, 287, 287, 288, 288, 289, 290, 290, 291, 291, + 292, 293, 294, 295, 295, 296, 297, 298, 299, 300, 300, 301, 302, 303, 304, 305, 306, 306, 307, + 307, 308, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 321, 322, 323, + 324, 325, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, + 342, 343, 344, 345, 346, 347, 301, 303, 348, 349, 350, 351, 352, 353, 352, 350, 354, 355, 356, + 357, 358, 353, 294, 284, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 389, 390, + 390, 391, 391, 392, 392, 393, 393, 393, 393, 394, 394, 395, 395, 395, 395, 396, 396, 397, 397, + 397, 397, 398, 398, 398, 398, 399, 399, 399, 399, 400, 400, 400, 400, 401, 401, 401, 401, 402, + 402, 403, 403, 404, 404, 405, 405, 406, 406, 406, 406, 407, 407, 407, 407, 408, 408, 408, 408, + 409, 409, 409, 409, 410, 410, 410, 410, 411, 411, 411, 411, 412, 412, 412, 412, 413, 413, 413, + 413, 414, 414, 414, 414, 415, 415, 415, 415, 416, 416, 416, 416, 417, 417, 417, 417, 418, 418, + 418, 418, 419, 419, 419, 419, 420, 420, 420, 420, 421, 421, 95, 95, 422, 422, 422, 422, 423, + 423, 424, 424, 425, 425, 426, 426, +]; + +pub(super) static FORM_TEXT: &str = "fffiflffifflst\u{574}\u{576}\u{574}\u{565}\u{574}\u{56b}\u{57e}\u{576}\u{574}\u{56d}\u{5d9}\u{5b4}\u{5f2}\u{5b7}\u{5e2}\u{5d0}\u{5d3}\u{5d4}\u{5db}\u{5dc}\u{5dd}\u{5e8}\u{5ea}+\u{5e9}\u{5c1}\u{5e9}\u{5c2}\u{5e9}\u{5bc}\u{5c1}\u{5e9}\u{5bc}\u{5c2}\u{5d0}\u{5b7}\u{5d0}\u{5b8}\u{5d0}\u{5bc}\u{5d1}\u{5bc}\u{5d2}\u{5bc}\u{5d3}\u{5bc}\u{5d4}\u{5bc}\u{5d5}\u{5bc}\u{5d6}\u{5bc}\u{5d8}\u{5bc}\u{5d9}\u{5bc}\u{5da}\u{5bc}\u{5db}\u{5bc}\u{5dc}\u{5bc}\u{5de}\u{5bc}\u{5e0}\u{5bc}\u{5e1}\u{5bc}\u{5e3}\u{5bc}\u{5e4}\u{5bc}\u{5e6}\u{5bc}\u{5e7}\u{5bc}\u{5e8}\u{5bc}\u{5e9}\u{5bc}\u{5ea}\u{5bc}\u{5d5}\u{5b9}\u{5d1}\u{5bf}\u{5db}\u{5bf}\u{5e4}\u{5bf}\u{5d0}\u{5dc}\u{671}\u{67b}\u{67e}\u{680}\u{67a}\u{67f}\u{679}\u{6a4}\u{6a6}\u{684}\u{683}\u{686}\u{687}\u{68d}\u{68c}\u{68e}\u{688}\u{698}\u{691}\u{6a9}\u{6af}\u{6b3}\u{6b1}\u{6ba}\u{6bb}\u{6c0}\u{6c1}\u{6be}\u{6d2}\u{6d3}\u{6ad}\u{6c7}\u{6c6}\u{6c8}\u{6c7}\u{674}\u{6cb}\u{6c5}\u{6c9}\u{6d0}\u{649}\u{626}\u{627}\u{626}\u{6d5}\u{626}\u{648}\u{626}\u{6c7}\u{626}\u{6c6}\u{626}\u{6c8}\u{626}\u{6d0}\u{626}\u{649}\u{6cc}\u{626}\u{62c}\u{626}\u{62d}\u{626}\u{645}\u{626}\u{64a}\u{628}\u{62c}\u{628}\u{62d}\u{628}\u{62e}\u{628}\u{645}\u{628}\u{649}\u{628}\u{64a}\u{62a}\u{62c}\u{62a}\u{62d}\u{62a}\u{62e}\u{62a}\u{645}\u{62a}\u{649}\u{62a}\u{64a}\u{62b}\u{62c}\u{62b}\u{645}\u{62b}\u{649}\u{62b}\u{64a}\u{62c}\u{62d}\u{62c}\u{645}\u{62d}\u{62c}\u{62d}\u{645}\u{62e}\u{62c}\u{62e}\u{62d}\u{62e}\u{645}\u{633}\u{62c}\u{633}\u{62d}\u{633}\u{62e}\u{633}\u{645}\u{635}\u{62d}\u{635}\u{645}\u{636}\u{62c}\u{636}\u{62d}\u{636}\u{62e}\u{636}\u{645}\u{637}\u{62d}\u{637}\u{645}\u{638}\u{645}\u{639}\u{62c}\u{639}\u{645}\u{63a}\u{62c}\u{63a}\u{645}\u{641}\u{62c}\u{641}\u{62d}\u{641}\u{62e}\u{641}\u{645}\u{641}\u{649}\u{641}\u{64a}\u{642}\u{62d}\u{642}\u{645}\u{642}\u{649}\u{642}\u{64a}\u{643}\u{627}\u{643}\u{62c}\u{643}\u{62d}\u{643}\u{62e}\u{643}\u{644}\u{643}\u{645}\u{643}\u{649}\u{643}\u{64a}\u{644}\u{62c}\u{644}\u{62d}\u{644}\u{62e}\u{644}\u{645}\u{644}\u{649}\u{644}\u{64a}\u{645}\u{62c}\u{645}\u{62d}\u{645}\u{62e}\u{645}\u{645}\u{645}\u{649}\u{645}\u{64a}\u{646}\u{62c}\u{646}\u{62d}\u{646}\u{62e}\u{646}\u{645}\u{646}\u{649}\u{646}\u{64a}\u{647}\u{62c}\u{647}\u{645}\u{647}\u{649}\u{647}\u{64a}\u{64a}\u{62c}\u{64a}\u{62d}\u{64a}\u{62e}\u{64a}\u{645}\u{64a}\u{649}\u{64a}\u{64a}\u{630}\u{670}\u{631}\u{670}\u{649}\u{670} \u{64c}\u{651} \u{64d}\u{651} \u{64e}\u{651} \u{64f}\u{651} \u{650}\u{651} \u{651}\u{670}\u{626}\u{631}\u{626}\u{632}\u{626}\u{646}\u{628}\u{631}\u{628}\u{632}\u{628}\u{646}\u{62a}\u{631}\u{62a}\u{632}\u{62a}\u{646}\u{62b}\u{631}\u{62b}\u{632}\u{62b}\u{646}\u{645}\u{627}\u{646}\u{631}\u{646}\u{632}\u{646}\u{646}\u{64a}\u{631}\u{64a}\u{632}\u{64a}\u{646}\u{626}\u{62e}\u{626}\u{647}\u{628}\u{647}\u{62a}\u{647}\u{635}\u{62e}\u{644}\u{647}\u{646}\u{647}\u{647}\u{670}\u{64a}\u{647}\u{62b}\u{647}\u{633}\u{647}\u{634}\u{645}\u{634}\u{647}\u{640}\u{64e}\u{651}\u{640}\u{64f}\u{651}\u{640}\u{650}\u{651}\u{637}\u{649}\u{637}\u{64a}\u{639}\u{649}\u{639}\u{64a}\u{63a}\u{649}\u{63a}\u{64a}\u{633}\u{649}\u{633}\u{64a}\u{634}\u{649}\u{634}\u{64a}\u{62d}\u{649}\u{62d}\u{64a}\u{62c}\u{649}\u{62c}\u{64a}\u{62e}\u{649}\u{62e}\u{64a}\u{635}\u{649}\u{635}\u{64a}\u{636}\u{649}\u{636}\u{64a}\u{634}\u{62c}\u{634}\u{62d}\u{634}\u{62e}\u{634}\u{631}\u{633}\u{631}\u{635}\u{631}\u{636}\u{631}\u{627}\u{64b}\u{62a}\u{62c}\u{645}\u{62a}\u{62d}\u{62c}\u{62a}\u{62d}\u{645}\u{62a}\u{62e}\u{645}\u{62a}\u{645}\u{62c}\u{62a}\u{645}\u{62d}\u{62a}\u{645}\u{62e}\u{62c}\u{645}\u{62d}\u{62d}\u{645}\u{64a}\u{62d}\u{645}\u{649}\u{633}\u{62d}\u{62c}\u{633}\u{62c}\u{62d}\u{633}\u{62c}\u{649}\u{633}\u{645}\u{62d}\u{633}\u{645}\u{62c}\u{633}\u{645}\u{645}\u{635}\u{62d}\u{62d}\u{635}\u{645}\u{645}\u{634}\u{62d}\u{645}\u{634}\u{62c}\u{64a}\u{634}\u{645}\u{62e}\u{634}\u{645}\u{645}\u{636}\u{62d}\u{649}\u{636}\u{62e}\u{645}\u{637}\u{645}\u{62d}\u{637}\u{645}\u{645}\u{637}\u{645}\u{64a}\u{639}\u{62c}\u{645}\u{639}\u{645}\u{645}\u{639}\u{645}\u{649}\u{63a}\u{645}\u{645}\u{63a}\u{645}\u{64a}\u{63a}\u{645}\u{649}\u{641}\u{62e}\u{645}\u{642}\u{645}\u{62d}\u{642}\u{645}\u{645}\u{644}\u{62d}\u{645}\u{644}\u{62d}\u{64a}\u{644}\u{62d}\u{649}\u{644}\u{62c}\u{62c}\u{644}\u{62e}\u{645}\u{644}\u{645}\u{62d}\u{645}\u{62d}\u{62c}\u{645}\u{62d}\u{645}\u{645}\u{62d}\u{64a}\u{645}\u{62c}\u{62d}\u{645}\u{62c}\u{645}\u{645}\u{62e}\u{62c}\u{645}\u{62e}\u{645}\u{645}\u{62c}\u{62e}\u{647}\u{645}\u{62c}\u{647}\u{645}\u{645}\u{646}\u{62d}\u{645}\u{646}\u{62d}\u{649}\u{646}\u{62c}\u{645}\u{646}\u{62c}\u{649}\u{646}\u{645}\u{64a}\u{646}\u{645}\u{649}\u{64a}\u{645}\u{645}\u{628}\u{62e}\u{64a}\u{62a}\u{62c}\u{64a}\u{62a}\u{62c}\u{649}\u{62a}\u{62e}\u{64a}\u{62a}\u{62e}\u{649}\u{62a}\u{645}\u{64a}\u{62a}\u{645}\u{649}\u{62c}\u{645}\u{64a}\u{62c}\u{62d}\u{649}\u{62c}\u{645}\u{649}\u{633}\u{62e}\u{649}\u{635}\u{62d}\u{64a}\u{634}\u{62d}\u{64a}\u{636}\u{62d}\u{64a}\u{644}\u{62c}\u{64a}\u{644}\u{645}\u{64a}\u{64a}\u{62d}\u{64a}\u{64a}\u{62c}\u{64a}\u{64a}\u{645}\u{64a}\u{645}\u{645}\u{64a}\u{642}\u{645}\u{64a}\u{646}\u{62d}\u{64a}\u{639}\u{645}\u{64a}\u{643}\u{645}\u{64a}\u{646}\u{62c}\u{62d}\u{645}\u{62e}\u{64a}\u{644}\u{62c}\u{645}\u{643}\u{645}\u{645}\u{62c}\u{62d}\u{64a}\u{62d}\u{62c}\u{64a}\u{645}\u{62c}\u{64a}\u{641}\u{645}\u{64a}\u{628}\u{62d}\u{64a}\u{633}\u{62e}\u{64a}\u{646}\u{62c}\u{64a}\u{635}\u{644}\u{6d2}\u{642}\u{644}\u{6d2}\u{627}\u{644}\u{644}\u{647}\u{627}\u{643}\u{628}\u{631}\u{645}\u{62d}\u{645}\u{62f}\u{635}\u{644}\u{639}\u{645}\u{631}\u{633}\u{648}\u{644}\u{639}\u{644}\u{64a}\u{647}\u{648}\u{633}\u{644}\u{645}\u{635}\u{644}\u{649}\u{635}\u{644}\u{649} \u{627}\u{644}\u{644}\u{647} \u{639}\u{644}\u{64a}\u{647} \u{648}\u{633}\u{644}\u{645}\u{62c}\u{644} \u{62c}\u{644}\u{627}\u{644}\u{647}\u{631}\u{6cc}\u{627}\u{644} \u{64b}\u{640}\u{64b} \u{64c} \u{64d} \u{64e}\u{640}\u{64e} \u{64f}\u{640}\u{64f} \u{650}\u{640}\u{650} \u{651}\u{640}\u{651} \u{652}\u{640}\u{652}\u{621}\u{622}\u{623}\u{624}\u{625}\u{626}\u{627}\u{628}\u{629}\u{62a}\u{62b}\u{62c}\u{62d}\u{62e}\u{62f}\u{630}\u{631}\u{632}\u{633}\u{634}\u{635}\u{636}\u{637}\u{638}\u{639}\u{63a}\u{641}\u{642}\u{643}\u{644}\u{645}\u{646}\u{647}\u{648}\u{64a}\u{644}\u{622}\u{644}\u{623}\u{644}\u{625}\u{644}\u{627}"; +pub(super) static FORM_OFFSETS: [u16; 428] = [ + 0, 2, 4, 6, 9, 12, 14, 18, 22, 26, 30, 34, 38, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 61, 65, + 69, 75, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, + 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 199, 201, 203, 205, 207, 209, 211, + 213, 215, 217, 219, 221, 223, 225, 227, 229, 231, 233, 235, 237, 239, 241, 243, 245, 247, 249, + 251, 253, 255, 257, 259, 261, 263, 265, 269, 271, 273, 275, 277, 279, 283, 287, 291, 295, 299, + 303, 307, 311, 313, 317, 321, 325, 329, 333, 337, 341, 345, 349, 353, 357, 361, 365, 369, 373, + 377, 381, 385, 389, 393, 397, 401, 405, 409, 413, 417, 421, 425, 429, 433, 437, 441, 445, 449, + 453, 457, 461, 465, 469, 473, 477, 481, 485, 489, 493, 497, 501, 505, 509, 513, 517, 521, 525, + 529, 533, 537, 541, 545, 549, 553, 557, 561, 565, 569, 573, 577, 581, 585, 589, 593, 597, 601, + 605, 609, 613, 617, 621, 625, 629, 633, 637, 641, 645, 649, 653, 657, 661, 665, 669, 673, 677, + 681, 685, 690, 695, 700, 705, 710, 715, 719, 723, 727, 731, 735, 739, 743, 747, 751, 755, 759, + 763, 767, 771, 775, 779, 783, 787, 791, 795, 799, 803, 807, 811, 815, 819, 823, 827, 831, 835, + 839, 843, 849, 855, 861, 865, 869, 873, 877, 881, 885, 889, 893, 897, 901, 905, 909, 913, 917, + 921, 925, 929, 933, 937, 941, 945, 949, 953, 957, 961, 965, 969, 973, 979, 985, 991, 997, 1003, + 1009, 1015, 1021, 1027, 1033, 1039, 1045, 1051, 1057, 1063, 1069, 1075, 1081, 1087, 1093, 1099, + 1105, 1111, 1117, 1123, 1129, 1135, 1141, 1147, 1153, 1159, 1165, 1171, 1177, 1183, 1189, 1195, + 1201, 1207, 1213, 1219, 1225, 1231, 1237, 1243, 1249, 1255, 1261, 1267, 1273, 1279, 1285, 1291, + 1297, 1303, 1309, 1315, 1321, 1327, 1333, 1339, 1345, 1351, 1357, 1363, 1369, 1375, 1381, 1387, + 1393, 1399, 1405, 1411, 1417, 1423, 1429, 1435, 1441, 1447, 1453, 1459, 1465, 1471, 1477, 1483, + 1489, 1495, 1501, 1507, 1513, 1519, 1525, 1531, 1537, 1543, 1549, 1557, 1565, 1573, 1581, 1589, + 1597, 1605, 1611, 1644, 1659, 1667, 1670, 1674, 1677, 1680, 1683, 1687, 1690, 1694, 1697, 1701, + 1704, 1708, 1711, 1715, 1717, 1719, 1721, 1723, 1725, 1727, 1729, 1731, 1733, 1735, 1737, 1739, + 1741, 1743, 1745, 1747, 1749, 1751, 1753, 1755, 1757, 1759, 1761, 1763, 1765, 1767, 1769, 1771, + 1773, 1775, 1777, 1779, 1781, 1783, 1785, 1789, 1793, 1797, 1801, +]; diff --git a/crates/text-extract/src/pdf/text_string.rs b/crates/text-extract/src/pdf/text_string.rs new file mode 100644 index 00000000..b31c2ae8 --- /dev/null +++ b/crates/text-extract/src/pdf/text_string.rs @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::tables::pdfdoc_char; + +const UTF16_BE_BOM: &[u8] = b"\xFE\xFF"; +const UTF16_LE_BOM: &[u8] = b"\xFF\xFE"; +const UTF8_BOM: &[u8] = b"\xEF\xBB\xBF"; +const ESCAPE: u8 = 0x1B; +const ESCAPE_UNIT: u16 = 0x1B; + +pub(crate) fn decode_text_string(bytes: &[u8], out: &mut String) { + if let Some(rest) = bytes.strip_prefix(UTF16_BE_BOM) { + push_utf16( + rest.as_chunks::<2>() + .0 + .iter() + .map(|&pair| u16::from_be_bytes(pair)), + out, + ); + } else if let Some(rest) = bytes.strip_prefix(UTF16_LE_BOM) { + push_utf16( + rest.as_chunks::<2>() + .0 + .iter() + .map(|&pair| u16::from_le_bytes(pair)), + out, + ); + } else if let Some(rest) = bytes.strip_prefix(UTF8_BOM) { + let mut skipping = false; + for chunk in rest.utf8_chunks() { + for ch in chunk.valid().chars() { + if ch == char::from(ESCAPE) { + skipping = !skipping; + } else if !skipping { + out.push(ch); + } + } + if !chunk.invalid().is_empty() && !skipping { + out.push(char::REPLACEMENT_CHARACTER); + } + } + } else { + let mut skipping = false; + for &byte in bytes { + if byte == ESCAPE { + skipping = !skipping; + } else if !skipping && let Some(ch) = pdfdoc_char(byte) { + out.push(ch); + } + } + } +} + +fn push_utf16(units: impl Iterator, out: &mut String) { + let mut skipping = false; + let filtered = units.filter(|&unit| { + if unit == ESCAPE_UNIT { + skipping = !skipping; + return false; + } + !skipping + }); + out.extend(char::decode_utf16(filtered).map(|ch| ch.unwrap_or(char::REPLACEMENT_CHARACTER))); +} + diff --git a/crates/text-extract/src/pdf/xref/mod.rs b/crates/text-extract/src/pdf/xref/mod.rs new file mode 100644 index 00000000..61acd8bd --- /dev/null +++ b/crates/text-extract/src/pdf/xref/mod.rs @@ -0,0 +1,385 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod stream; +mod table; + +pub(crate) use stream::read_stream_section; + +use super::{ + lexer::{Lexer, is_regular}, + object::{Dict, Indirect, MAX_OBJECT_NUMBER, keyword_at}, + source::Source, +}; +use memchr::memmem; +use std::collections::VecDeque; + +pub(crate) const MAX_SECTIONS: usize = 256; +pub(crate) const NO_SLOT: u32 = u32::MAX; +const MAX_STARTXREF_CANDIDATES: usize = 8; +const MAX_OFFSET_DIGITS: usize = 10; +const CORRECTION_WINDOW: usize = 64; +const DENSE_BASE: usize = 1 << 14; +const STARTXREF: &[u8] = b"startxref"; +const XREF: &[u8] = b"xref"; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum Entry { + #[default] + Missing, + Free { + section: u16, + }, + Offset { + offset: u32, + generation: u16, + slot: u32, + }, + Compressed { + stream: u32, + index: u32, + }, +} + +#[derive(Debug, Clone, Copy)] +struct Sparse { + num: u32, + section: u16, + entry: Entry, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct Entries { + dense: Vec, + sparse: Vec, +} + +pub(crate) struct Builder<'e> { + entries: &'e mut Entries, + dense_cap: usize, + sparse_cap: usize, +} + +pub(crate) struct Sections<'a> { + pub(crate) trailers: Vec>, + pub(crate) bias: usize, + pub(crate) complete: bool, +} + +impl Entry { + #[inline] + pub(crate) fn is_used(&self) -> bool { + matches!(self, Entry::Offset { .. } | Entry::Compressed { .. }) + } +} + +impl Entries { + pub(crate) fn clear(&mut self) { + self.dense.clear(); + self.sparse.clear(); + } + + pub(crate) fn shrink(&mut self, max_bytes: usize) { + self.clear(); + self.dense + .shrink_to(max_bytes / std::mem::size_of::()); + self.sparse + .shrink_to(max_bytes / std::mem::size_of::()); + } + + #[inline] + pub(crate) fn get(&self, num: u32) -> Entry { + match self.dense.get(num as usize) { + Some(entry) => *entry, + None => self + .sparse + .binary_search_by_key(&num, |sparse| sparse.num) + .ok() + .and_then(|index| self.sparse.get(index)) + .map_or(Entry::Missing, |sparse| sparse.entry), + } + } + + pub(crate) fn get_mut(&mut self, num: u32) -> Option<&mut Entry> { + if let Some(entry) = self.dense.get_mut(num as usize) { + return Some(entry); + } + let index = self + .sparse + .binary_search_by_key(&num, |sparse| sparse.num) + .ok()?; + self.sparse.get_mut(index).map(|sparse| &mut sparse.entry) + } + + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.dense + .iter() + .zip(0u32..) + .map(|(entry, num)| (num, *entry)) + .chain(self.sparse.iter().map(|sparse| (sparse.num, sparse.entry))) + } + + pub(crate) fn put_sorted(&mut self, num: u32, entry: Entry, dense_cap: usize) { + let index = num as usize; + if index < dense_cap { + if index >= self.dense.len() { + self.dense.resize(index + 1, Entry::Missing); + } + if let Some(slot) = self.dense.get_mut(index) { + *slot = entry; + } + } else { + self.sparse.push(Sparse { + num, + section: 0, + entry, + }); + } + } +} + +impl<'e> Builder<'e> { + pub(crate) fn new(entries: &'e mut Entries, input_len: usize, max_objects: usize) -> Self { + entries.clear(); + let dense_cap = Builder::dense_cap(input_len, max_objects); + Builder { + entries, + dense_cap, + sparse_cap: dense_cap, + } + } + + pub(crate) fn sparse(entries: &'e mut Entries, cap: usize) -> Self { + entries.clear(); + Builder { + entries, + dense_cap: 0, + sparse_cap: cap, + } + } + + pub(crate) fn dense_cap(input_len: usize, max_objects: usize) -> usize { + max_objects + .min(MAX_OBJECT_NUMBER as usize + 1) + .min(input_len / 4 + DENSE_BASE) + } + + pub(crate) fn merge(&mut self, num: u64, entry: Entry, section: u16) { + let Some(num) = u32::try_from(num) + .ok() + .filter(|&num| num <= MAX_OBJECT_NUMBER) + else { + return; + }; + let index = num as usize; + if index >= self.dense_cap { + if self.entries.sparse.len() < self.sparse_cap { + self.entries.sparse.push(Sparse { + num, + section, + entry, + }); + } + return; + } + if index >= self.entries.dense.len() { + self.entries.dense.resize(index + 1, Entry::Missing); + } + if let Some(slot) = self.entries.dense.get_mut(index) { + slot.merge(entry, section); + } + } + + pub(crate) fn finish(self) { + let sparse = &mut self.entries.sparse; + sparse.sort_by_key(|sparse| sparse.num); + sparse.dedup_by(|later, kept| { + if later.num != kept.num { + return false; + } + kept.entry.merge(later.entry, later.section); + true + }); + } +} + +impl Entry { + fn merge(&mut self, entry: Entry, section: u16) { + match *self { + Entry::Missing => *self = entry, + Entry::Free { section: owner } if owner == section && entry.is_used() => *self = entry, + _ => {} + } + } +} + +pub(crate) fn looks_like_section(data: &[u8], pos: usize) -> bool { + let mut lexer = Lexer::at(data, pos); + lexer.skip_whitespace(); + keyword_at(data, lexer.pos(), XREF) || Indirect::header(data, lexer.pos()).is_some() +} + +pub(crate) fn locate_section(data: &[u8], pos: usize) -> Option { + if pos >= data.len() { + return None; + } + if looks_like_section(data, pos) { + return Some(pos); + } + let from = pos.saturating_sub(CORRECTION_WINDOW); + let to = pos.saturating_add(CORRECTION_WINDOW).min(data.len()); + let window = data.get(from..to)?; + let xref = memmem::find_iter(window, XREF) + .map(|offset| from + offset) + .filter(|&found| { + found + .checked_sub(1) + .and_then(|before| data.get(before)) + .is_none_or(|&byte| !byte.is_ascii_alphanumeric()) + && keyword_at(data, found, XREF) + }); + let headers = (from..to).filter(|&candidate| { + data.get(candidate).is_some_and(u8::is_ascii_digit) + && candidate + .checked_sub(1) + .and_then(|before| data.get(before)) + .is_none_or(|&byte| !is_regular(byte)) + && Indirect::header(data, candidate).is_some() + }); + xref.chain(headers).min_by_key(|&found| found.abs_diff(pos)) +} + +pub(crate) fn startxref_candidates(data: &[u8]) -> impl Iterator + '_ { + let finder = memmem::FinderRev::new(STARTXREF); + let mut end = data.len(); + std::iter::from_fn(move || { + let found = finder.rfind(data.get(..end)?)?; + end = found; + Some(found) + }) + .take(MAX_STARTXREF_CANDIDATES) + .filter_map(|found| { + let mut lexer = Lexer::at(data, found + STARTXREF.len()); + lexer.skip_whitespace(); + let digits = lexer + .rest() + .iter() + .take_while(|byte| byte.is_ascii_digit()) + .take(MAX_OFFSET_DIGITS + 1) + .try_fold(0usize, |value, &digit| { + value + .checked_mul(10)? + .checked_add(usize::from(digit - b'0')) + })?; + Some(digits).filter(|&offset| offset != 0 && offset < data.len()) + }) +} + +pub(crate) fn read_sections<'a>( + source: &Source<'a>, + builder: &mut Builder<'_>, +) -> Option> { + let data = source.data; + let header = source.header; + let biases: &[usize] = if header > 0 { &[header, 0] } else { &[0] }; + let (start, bias) = startxref_candidates(data).find_map(|offset| { + biases + .iter() + .find_map(|&bias| { + let pos = bias.checked_add(offset)?; + (pos < data.len() && looks_like_section(data, pos)).then_some((pos, bias)) + }) + .or_else(|| { + biases.iter().find_map(|&bias| { + locate_section(data, bias.checked_add(offset)?).map(|pos| (pos, bias)) + }) + }) + })?; + let mut sections = Sections { + trailers: Vec::new(), + bias, + complete: true, + }; + let mut queue = VecDeque::from([start]); + let mut visited: Vec = Vec::new(); + let mut section = 0u16; + while let Some(target) = queue.pop_front() { + if usize::from(section) >= MAX_SECTIONS { + sections.complete = false; + break; + } + if visited.contains(&target) { + continue; + } + visited.push(target); + let Some(pos) = locate_section(data, target) else { + sections.complete = false; + continue; + }; + if pos != target { + if visited.contains(&pos) { + continue; + } + visited.push(pos); + } + let trailer = read_section(source, pos, builder, section, &mut visited, bias); + section = section.saturating_add(1); + let Some(trailer) = trailer else { + sections.complete = false; + continue; + }; + sections.trailers.push(trailer); + if !source.charge_scan(trailer.body().len()) { + sections.complete = false; + break; + } + if let Some(previous) = trailer + .get(b"Prev") + .and_then(|prev| { + prev.as_int() + .or_else(|| prev.as_ref().map(|id| i64::from(id.num))) + }) + .and_then(|prev| usize::try_from(prev).ok()) + .filter(|&prev| prev > 0) + .and_then(|prev| prev.checked_add(bias)) + .filter(|&prev| prev < data.len()) + { + queue.push_back(previous); + } + } + (!sections.trailers.is_empty()).then_some(sections) +} + +fn read_section<'a>( + source: &Source<'a>, + pos: usize, + builder: &mut Builder<'_>, + section: u16, + visited: &mut Vec, + bias: usize, +) -> Option> { + let data = source.data; + let mut lexer = Lexer::at(data, pos); + lexer.skip_whitespace(); + if !keyword_at(data, lexer.pos(), XREF) { + return read_stream_section(source, lexer.pos(), builder, section); + } + let trailer = table::read_table(data, lexer.pos() + XREF.len(), builder, section)?; + if let Some(stream) = trailer + .get(b"XRefStm") + .and_then(|value| value.as_int()) + .and_then(|value| usize::try_from(value).ok()) + .filter(|&value| value > 0) + .and_then(|value| value.checked_add(bias)) + .and_then(|value| locate_section(data, value)) + .filter(|value| !visited.contains(value)) + { + visited.push(stream); + read_stream_section(source, stream, builder, section); + } + Some(trailer) +} + diff --git a/crates/text-extract/src/pdf/xref/stream.rs b/crates/text-extract/src/pdf/xref/stream.rs new file mode 100644 index 00000000..3bbbed67 --- /dev/null +++ b/crates/text-extract/src/pdf/xref/stream.rs @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Builder, Entry, NO_SLOT}; +use crate::pdf::{ + decode::Chain, + object::{Array, Dict, Indirect}, + source::Source, +}; + +const FIELDS: usize = 3; +const MAX_FIELD_WIDTH: usize = 8; + +pub(crate) fn read_stream_section<'a>( + source: &Source<'a>, + pos: usize, + builder: &mut Builder<'_>, + section: u16, +) -> Option> { + let object = Indirect::parse(source.data, pos, true)?; + let dict = object.value.as_dict()?; + let start = object.stream_start?; + if !dict.is_type(b"XRef") { + return None; + } + let widths = widths(dict.get(b"W")?.as_array()?)?; + let length = dict.get(b"Length").and_then(|length| length.as_int()); + let raw = source.stream_data(start, length); + let chain = Chain::parse(dict, &|object| object); + let mut decoded = Vec::new(); + source.decode(&chain, raw, None, &mut decoded); + read_entries(&decoded, widths, dict, builder, section); + Some(dict) +} + +fn widths(array: Array<'_>) -> Option<[usize; FIELDS]> { + let mut widths = [0usize; FIELDS]; + let mut values = array.iter(); + for width in &mut widths { + *width = values + .next()? + .as_int() + .and_then(|value| usize::try_from(value).ok()) + .filter(|&value| value <= MAX_FIELD_WIDTH)?; + } + (widths.iter().sum::() > 0).then_some(widths) +} + +pub(crate) fn read_entries( + data: &[u8], + widths: [usize; FIELDS], + dict: Dict<'_>, + builder: &mut Builder<'_>, + section: u16, +) { + let row = widths.iter().sum::(); + let mut rows = data.chunks_exact(row); + let default_count = || { + dict.get(b"Size") + .and_then(|size| size.as_int()) + .and_then(|size| u64::try_from(size).ok()) + .unwrap_or((data.len() / row) as u64) + }; + let pairs: Vec<(u64, u64)> = match dict.get(b"Index").and_then(|index| index.as_array()) { + Some(index) => { + let mut values = index.iter().map(|value| value.as_int()); + std::iter::from_fn(|| Some((values.next()?, values.next()?))) + .filter_map(|(start, count)| { + Some((u64::try_from(start?).ok()?, u64::try_from(count?).ok()?)) + }) + .take(rows.len()) + .collect() + } + None => vec![(0, default_count())], + }; + for (start, count) in pairs { + for num in (0..count).map_while(|offset| start.checked_add(offset)) { + let Some(bytes) = rows.next() else { + return; + }; + let [type_width, second_width, _] = widths; + let (kind, rest) = bytes.split_at(type_width.min(bytes.len())); + let (second, third) = rest.split_at(second_width.min(rest.len())); + let kind = if type_width == 0 { 1 } else { big_endian(kind) }; + let (second, third) = (big_endian(second), big_endian(third)); + let entry = match kind { + 0 => Entry::Free { section }, + 1 => match u32::try_from(second) { + Ok(offset) if offset > 0 => Entry::Offset { + offset, + generation: u16::try_from(third).unwrap_or(u16::MAX), + slot: NO_SLOT, + }, + _ => Entry::Free { section }, + }, + 2 => match (u32::try_from(second), u32::try_from(third)) { + (Ok(stream), Ok(index)) => Entry::Compressed { stream, index }, + _ => continue, + }, + _ => continue, + }; + builder.merge(num, entry, section); + } + } +} + +fn big_endian(bytes: &[u8]) -> u64 { + bytes + .iter() + .fold(0u64, |value, &byte| value << 8 | u64::from(byte)) +} + diff --git a/crates/text-extract/src/pdf/xref/table.rs b/crates/text-extract/src/pdf/xref/table.rs new file mode 100644 index 00000000..b2c5cec5 --- /dev/null +++ b/crates/text-extract/src/pdf/xref/table.rs @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Builder, Entry, NO_SLOT}; +use crate::pdf::{ + lexer::{Lexer, Token}, + object::{Dict, Object}, +}; + +const MIN_ENTRY_BYTES: usize = 18; +const FREE_HEAD_GEN: i64 = 65535; + +enum Flag { + Used, + Free, +} + +pub(super) fn read_table<'a>( + data: &'a [u8], + pos: usize, + builder: &mut Builder<'_>, + section: u16, +) -> Option> { + let mut lexer = Lexer::at(data, pos); + let mut current = 0u64; + let mut first_subsection = true; + loop { + let before = lexer.pos(); + match lexer.next()? { + Token::Keyword(b"trailer") => { + return Object::read(&mut lexer, None).and_then(|object| object.as_dict()); + } + Token::Int(first) => { + let Token::Int(second) = lexer.next()? else { + return None; + }; + if let Some(flag) = flag(&lexer) { + lexer.next(); + emit(builder, current, first, second, flag, section); + current = current.saturating_add(1); + continue; + } + let Ok(start) = u64::try_from(first) else { + return None; + }; + let room = data.len().saturating_sub(lexer.pos()) / MIN_ENTRY_BYTES + 1; + let count = usize::try_from(second).unwrap_or(0).min(room); + current = start; + for index in 0..count { + let mut probe = lexer; + let (Some(Token::Int(offset)), Some(Token::Int(generation))) = + (probe.next(), probe.next()) + else { + break; + }; + lexer = probe; + let flag = match flag(&lexer) { + Some(flag) => { + lexer.next(); + flag + } + None => Flag::Free, + }; + if first_subsection + && index == 0 + && start == 1 + && offset == 0 + && generation == FREE_HEAD_GEN + && matches!(flag, Flag::Free) + { + current = 0; + } + emit(builder, current, offset, generation, flag, section); + current = current.saturating_add(1); + } + first_subsection = false; + } + _ => { + lexer.set_pos(before); + return None; + } + } + } +} + +fn flag(lexer: &Lexer<'_>) -> Option { + match lexer.peek()? { + Token::Keyword(b"n" | b"N") => Some(Flag::Used), + Token::Keyword(b"f" | b"F") => Some(Flag::Free), + Token::Keyword(word) if word.len() == 1 => Some(Flag::Free), + _ => None, + } +} + +fn emit( + builder: &mut Builder<'_>, + num: u64, + offset: i64, + generation: i64, + flag: Flag, + section: u16, +) { + let offset = u32::try_from(offset).ok().filter(|&offset| offset > 0); + let entry = match (flag, offset) { + (Flag::Used, Some(offset)) => Entry::Offset { + offset, + generation: u16::try_from(generation.clamp(0, i64::from(u16::MAX))).unwrap_or(u16::MAX), + slot: NO_SLOT, + }, + _ => Entry::Free { section }, + }; + builder.merge(num, entry, section); +} + diff --git a/crates/text-extract/src/rtf/codepage.rs b/crates/text-extract/src/rtf/codepage.rs new file mode 100644 index 00000000..74e8e818 --- /dev/null +++ b/crates/text-extract/src/rtf/codepage.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use encoding_rs::*; + +pub(crate) fn from_codepage(codepage: i64) -> Option<&'static Encoding> { + Some(match codepage { + 708 | 28596 => ISO_8859_6, + 866 => IBM866, + 874 => WINDOWS_874, + 932 => SHIFT_JIS, + 936 => GBK, + 949 => EUC_KR, + 950 => BIG5, + 1250 => WINDOWS_1250, + 1251 => WINDOWS_1251, + 1252 | 28591 => WINDOWS_1252, + 1253 => WINDOWS_1253, + 1254 => WINDOWS_1254, + 1255 => WINDOWS_1255, + 1256 => WINDOWS_1256, + 1257 => WINDOWS_1257, + 1258 => WINDOWS_1258, + 10000 => MACINTOSH, + 10007 => X_MAC_CYRILLIC, + 20866 => KOI8_R, + 21866 => KOI8_U, + 20932 | 51932 => EUC_JP, + 28592 => ISO_8859_2, + 28593 => ISO_8859_3, + 28594 => ISO_8859_4, + 28595 => ISO_8859_5, + 28597 => ISO_8859_7, + 28598 => ISO_8859_8, + 28603 => ISO_8859_13, + 28605 => ISO_8859_15, + 50220..=50222 => ISO_2022_JP, + 54936 => GB18030, + 65001 => UTF_8, + _ => return None, + }) +} + +pub(crate) fn from_charset(charset: i64) -> Option<&'static Encoding> { + Some(match charset { + 0 => WINDOWS_1252, + 77 => MACINTOSH, + 128 => SHIFT_JIS, + 129 => EUC_KR, + 134 => GBK, + 136 => BIG5, + 161 => WINDOWS_1253, + 162 => WINDOWS_1254, + 163 => WINDOWS_1258, + 177 => WINDOWS_1255, + 178 => WINDOWS_1256, + 186 => WINDOWS_1257, + 204 => WINDOWS_1251, + 222 => WINDOWS_874, + 238 => WINDOWS_1250, + _ => return None, + }) +} diff --git a/crates/text-extract/src/rtf/mod.rs b/crates/text-extract/src/rtf/mod.rs new file mode 100644 index 00000000..cb4b2ff6 --- /dev/null +++ b/crates/text-extract/src/rtf/mod.rs @@ -0,0 +1,586 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod codepage; +mod words; + +use crate::{ + html::tag_separator, + output::{Mark, Output, Separator}, + utf8::LazyUtf8, +}; +use encoding_rs::{CoderResult, Decoder, Encoding, WINDOWS_1252}; +use memchr::{memchr, memchr3, memmem}; +use words::Word; + +const SIGNATURE: &[u8] = b"{\\rtf"; +const UTF8_BOM: &[u8] = b"\xEF\xBB\xBF"; +const MAX_WORD: usize = 32; +const MAX_DIGITS: usize = 10; +const MAX_PENDING: usize = 4096; +const MAX_FONTS: usize = 4096; +const MAX_SKIP: i64 = 64; +const DECODE_CHUNK: usize = 256; +const MAX_NOTE: usize = 1 << 16; + +pub(crate) fn is_rtf(data: &[u8]) -> bool { + body(data).starts_with(SIGNATURE) +} + +fn body(data: &[u8]) -> &[u8] { + data.strip_prefix(UTF8_BOM) + .unwrap_or(data) + .trim_ascii_start() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Destination { + Text, + FontTable, + HtmlTag, + Ignored, +} + +#[derive(Debug, Clone, Copy)] +struct Group { + destination: Destination, + skip_after_unicode: u32, + encoding: Option<&'static Encoding>, + html_rtf: bool, + annotation: bool, +} + +#[derive(Default)] +pub(crate) struct Scratch { + stack: Vec, + fonts: Vec<(i64, &'static Encoding)>, + pending: Vec, + note: String, +} + +struct Parser<'s, 'o, 'b, 'd> { + scratch: &'s mut Scratch, + out: &'o mut Output<'b>, + source_len: usize, + utf8: LazyUtf8<'d>, + group_open: usize, + group: Group, + overflow: u64, + max_depth: usize, + document_encoding: &'static Encoding, + default_font: Option, + table_font: Option, + from_html: bool, + html_in_tag: bool, + note_mark: Option, + ignorable: bool, + skip: u32, + high_surrogate: Option, + decoder: Option, +} + +pub(crate) fn extract(data: &[u8], scratch: &mut Scratch, max_depth: usize, out: &mut Output<'_>) { + scratch.stack.clear(); + scratch.fonts.clear(); + scratch.pending.clear(); + scratch.note.clear(); + let body = body(data); + let mut parser = Parser { + scratch, + out, + source_len: body.len(), + utf8: LazyUtf8::new(body), + group_open: usize::MAX, + group: Group { + destination: Destination::Text, + skip_after_unicode: 1, + encoding: None, + html_rtf: false, + annotation: false, + }, + overflow: 0, + max_depth: max_depth.max(1), + document_encoding: WINDOWS_1252, + default_font: None, + table_font: None, + from_html: false, + html_in_tag: false, + note_mark: None, + ignorable: false, + skip: 0, + high_surrogate: None, + decoder: None, + }; + parser.run(body); + parser.flush(); + parser.emit_note(); +} + +impl<'d> Parser<'_, '_, '_, 'd> { + fn run(&mut self, mut rest: &[u8]) { + while let Some((&byte, tail)) = rest.split_first() { + if self.out.is_full() { + return; + } + rest = match byte { + b'{' => { + self.open_group(); + self.group_open = self.source_len - tail.len(); + tail + } + b'}' => { + if !self.close_group() { + return; + } + tail + } + b'\\' => { + let group_start = self.source_len - rest.len() == self.group_open; + self.control(tail, group_start) + } + b'\r' | b'\n' => { + if self.source_len - rest.len() == self.group_open { + self.group_open += 1; + } + tail + } + _ => self.text(rest), + }; + } + } + + fn open_group(&mut self) { + self.flush(); + self.skip = 0; + self.ignorable = false; + if self.scratch.stack.len() < self.max_depth { + self.scratch.stack.push(self.group); + self.group.annotation = false; + } else { + self.overflow = self.overflow.saturating_add(1); + } + } + + fn close_group(&mut self) -> bool { + self.flush(); + self.skip = 0; + self.ignorable = false; + if self.overflow > 0 { + self.overflow -= 1; + return true; + } + let Some(parent) = self.scratch.stack.pop() else { + return false; + }; + let closed = std::mem::replace(&mut self.group, parent); + if closed.annotation + && let Some(mark) = self.note_mark.take() + { + self.out.cut(mark, &mut self.scratch.note, MAX_NOTE); + } + !self.scratch.stack.is_empty() + } + + fn suppressed(&self) -> bool { + self.group.destination != Destination::Text || (self.from_html && self.group.html_rtf) + } + + fn text<'a>(&mut self, rest: &'a [u8]) -> &'a [u8] { + self.ignorable = false; + let run_len = memchr3(b'\\', b'{', b'}', rest).unwrap_or(rest.len()); + let (run, tail) = rest.split_at(run_len); + let mut run = run; + while self.skip > 0 { + match run.split_first() { + Some((b'\r' | b'\n', remaining)) => run = remaining, + Some((_, remaining)) => { + run = remaining; + self.skip -= 1; + } + None => break, + } + } + if self.group.destination == Destination::HtmlTag { + for line in run.split(|&byte| byte == b'\r' || byte == b'\n') { + if let Some(separator) = html_tag_separator(line, &mut self.html_in_tag) { + self.out.separator(separator); + } + } + return tail; + } + if self.suppressed() { + return tail; + } + let mut offset = self.source_len - run.len() - tail.len(); + for line in run.split(|&byte| byte == b'\r' || byte == b'\n') { + let line_start = offset; + offset += line.len() + 1; + if line.is_empty() { + continue; + } + if self.scratch.pending.is_empty() && line.is_ascii() { + match self.utf8.get(line_start, line.len()) { + Some(text) => self.out.push_str(text), + None => self.out.push_utf8(line), + } + } else { + self.scratch.pending.extend_from_slice(line); + if self.scratch.pending.len() > MAX_PENDING { + self.decode_pending(false); + } + } + } + tail + } + + fn control<'a>(&mut self, rest: &'a [u8], group_start: bool) -> &'a [u8] { + let Some((&first, tail)) = rest.split_first() else { + return rest; + }; + if first.is_ascii_alphabetic() { + return self.control_word(rest); + } + if first == b'\'' { + return self.hex_byte(tail); + } + self.flush(); + if self.skip > 0 { + self.skip -= 1; + return tail; + } + match first { + b'*' => { + self.ignorable = group_start; + return tail; + } + b'\\' | b'{' | b'}' => { + if !self.suppressed() { + self.out.push_char(char::from(first)); + } + } + b'~' => self.push_char('\u{a0}'), + b'_' => self.push_char('-'), + b'\r' | b'\n' => self.separator(Separator::Newline), + _ => {} + } + self.ignorable = false; + tail + } + + fn hex_byte<'a>(&mut self, rest: &'a [u8]) -> &'a [u8] { + self.ignorable = false; + let value = rest.get(..2).and_then(|hex| { + let high = char::from(*hex.first()?).to_digit(16)?; + let low = char::from(*hex.get(1)?).to_digit(16)?; + u8::try_from(high * 16 + low).ok() + }); + let Some(value) = value else { + return rest; + }; + let tail = rest.get(2..).unwrap_or_default(); + if self.skip > 0 { + self.skip -= 1; + } else if !self.suppressed() { + self.scratch.pending.push(value); + if self.scratch.pending.len() > MAX_PENDING { + self.decode_pending(false); + } + } + tail + } + + fn control_word<'a>(&mut self, rest: &'a [u8]) -> &'a [u8] { + let letters = rest + .iter() + .position(|byte| !byte.is_ascii_alphabetic()) + .unwrap_or(rest.len()); + let (name, mut tail) = rest.split_at(letters); + let negative = tail.first() == Some(&b'-'); + if negative { + tail = tail.get(1..).unwrap_or_default(); + } + let digits = tail + .iter() + .position(|byte| !byte.is_ascii_digit()) + .unwrap_or(tail.len()); + let (number, after) = tail.split_at(digits); + let parameter = (!number.is_empty()).then(|| { + let magnitude = number + .iter() + .take(MAX_DIGITS) + .fold(0i64, |value, &digit| value * 10 + i64::from(digit - b'0')); + if negative { -magnitude } else { magnitude } + }); + let tail = match after.split_first() { + Some((b' ', remaining)) => remaining, + _ => after, + }; + let word = if name.len() <= MAX_WORD { + Word::parse(name) + } else { + None + }; + if word == Some(Word::Bin) { + let skip = usize::try_from(parameter.unwrap_or(0).max(0)).unwrap_or(usize::MAX); + let tail = tail.get(skip..).unwrap_or_default(); + if self.skip > 0 { + self.skip -= 1; + } + self.ignorable = false; + return tail; + } + self.flush(); + if self.skip > 0 { + self.skip -= 1; + self.ignorable = false; + return tail; + } + let ignorable = std::mem::replace(&mut self.ignorable, false); + self.apply(word, parameter, ignorable); + tail + } + + fn apply(&mut self, word: Option, parameter: Option, ignorable: bool) { + let Some(word) = word.filter(|word| { + !ignorable + || matches!( + word, + Word::UnicodeDestination + | Word::ShapeInstructions + | Word::Annotation + | Word::HtmlTag + ) + }) else { + if ignorable { + self.group.destination = Destination::Ignored; + } + return; + }; + match word { + Word::AnsiCodePage => { + if let Some(encoding) = parameter.and_then(codepage::from_codepage) { + self.document_encoding = encoding; + } + } + Word::Mac => self.document_encoding = encoding_rs::MACINTOSH, + Word::Ansi => self.document_encoding = WINDOWS_1252, + Word::DefaultFont => self.default_font = parameter, + Word::FromHtml => self.from_html = true, + Word::HtmlRtf => self.group.html_rtf = parameter != Some(0), + Word::FontTable => self.group.destination = Destination::FontTable, + Word::Font => { + if self.group.destination == Destination::FontTable { + self.table_font = parameter; + } else if let Some(font) = parameter { + self.group.encoding = self.font_encoding(font); + } + } + Word::FontCharset | Word::FontCodePage => { + if self.group.destination == Destination::FontTable + && let (Some(font), Some(value)) = (self.table_font, parameter) + { + let encoding = if word == Word::FontCharset { + codepage::from_charset(value) + } else { + codepage::from_codepage(value) + }; + if let Some(encoding) = encoding { + self.insert_font(font, encoding); + } + } + } + Word::Plain => self.group.encoding = None, + Word::UnicodeSkip => { + self.group.skip_after_unicode = + u32::try_from(parameter.unwrap_or(1).clamp(0, MAX_SKIP)).unwrap_or(1) + } + Word::Unicode => self.unicode(parameter.unwrap_or(0)), + Word::Paragraph => { + if self.note_mark.is_none() { + self.emit_note(); + } + self.separator(Separator::Newline) + } + Word::NestedRow => self.out.separator(Separator::Newline), + Word::Tab => { + if self.note_mark.is_none() { + self.emit_note(); + } + self.separator(Separator::Space) + } + Word::Character(ch) => self.push_char(ch), + Word::UnicodeDestination => { + if self.group.destination == Destination::Ignored { + self.group.destination = Destination::Text; + } + } + Word::Annotation => { + if self.note_mark.is_none() { + self.note_mark = Some(self.out.mark()); + self.group.annotation = true; + } + } + Word::HtmlTag => { + self.html_in_tag = false; + self.group.destination = + if self.from_html && self.group.destination == Destination::Text { + Destination::HtmlTag + } else { + Destination::Ignored + }; + } + Word::ShapeInstructions | Word::Bin => {} + Word::Destination => self.group.destination = Destination::Ignored, + } + } + + fn unicode(&mut self, parameter: i64) { + self.skip = self.group.skip_after_unicode; + let Ok(unit) = u16::try_from(if parameter < 0 { + parameter + 0x10000 + } else { + parameter + }) else { + return; + }; + let ch = match unit { + 0xD800..=0xDBFF => { + self.high_surrogate = Some(unit); + return; + } + 0xDC00..=0xDFFF => self.high_surrogate.take().and_then(|high| { + char::from_u32( + 0x10000 + ((u32::from(high) - 0xD800) << 10) + (u32::from(unit) - 0xDC00), + ) + }), + _ => { + self.high_surrogate = None; + char::from_u32(u32::from(unit)) + } + }; + if let Some(ch) = ch { + self.flush(); + self.push_char(ch); + } + } + + fn emit_note(&mut self) { + if !self.scratch.note.is_empty() { + self.out.separator(Separator::Space); + self.out.push_str(&self.scratch.note); + self.out.separator(Separator::Space); + self.scratch.note.clear(); + } + } + + fn push_char(&mut self, ch: char) { + if !self.suppressed() { + self.out.push_char(ch); + } + } + + fn separator(&mut self, separator: Separator) { + if self.group.destination == Destination::Text { + self.out.separator(separator); + } + } + + fn font_encoding(&self, font: i64) -> Option<&'static Encoding> { + self.scratch + .fonts + .binary_search_by_key(&font, |&(number, _)| number) + .ok() + .and_then(|index| self.scratch.fonts.get(index)) + .map(|&(_, encoding)| encoding) + } + + fn insert_font(&mut self, font: i64, encoding: &'static Encoding) { + match self + .scratch + .fonts + .binary_search_by_key(&font, |&(number, _)| number) + { + Ok(index) => { + if let Some(slot) = self.scratch.fonts.get_mut(index) { + slot.1 = encoding; + } + } + Err(index) if self.scratch.fonts.len() < MAX_FONTS => { + self.scratch.fonts.insert(index, (font, encoding)) + } + Err(_) => {} + } + } + + fn flush(&mut self) { + if !self.scratch.pending.is_empty() || self.decoder.is_some() { + self.decode_pending(true); + } + } + + fn decode_pending(&mut self, last: bool) { + let encoding = self + .group + .encoding + .or_else(|| self.default_font.and_then(|font| self.font_encoding(font))) + .unwrap_or(self.document_encoding); + let decoder = self + .decoder + .get_or_insert_with(|| encoding.new_decoder_without_bom_handling()); + let mut buffer = [0u8; DECODE_CHUNK]; + let mut source = self.scratch.pending.as_slice(); + loop { + let (result, read, written, _) = decoder.decode_to_utf8(source, &mut buffer, last); + self.out + .push_utf8(buffer.get(..written).unwrap_or_default()); + source = source.get(read..).unwrap_or_default(); + if result == CoderResult::InputEmpty { + break; + } + } + self.scratch.pending.clear(); + if last { + self.decoder = None; + } + } +} + +fn html_tag_separator(line: &[u8], in_tag: &mut bool) -> Option { + let mut separator = None; + let mut rest = line; + loop { + if *in_tag { + match memchr(b'>', rest) { + Some(end) => { + *in_tag = false; + rest = rest.get(end + 1..).unwrap_or_default(); + } + None => return separator, + } + } + let (outside, tag) = match memchr(b'<', rest) { + Some(at) => rest.split_at(at), + None => (rest, b"".as_slice()), + }; + if outside.iter().any(u8::is_ascii_whitespace) + || memmem::find(outside, b" ").is_some() + || memmem::find(outside, b" ").is_some() + { + separator = separator.max(Some(Separator::Space)); + } + let Some(tag) = tag.get(1..) else { + return separator; + }; + let name = tag.strip_prefix(b"/").unwrap_or(tag); + let name_len = name + .iter() + .position(|byte| !byte.is_ascii_alphanumeric()) + .unwrap_or(name.len()); + separator = separator.max(tag_separator(name.get(..name_len).unwrap_or_default())); + *in_tag = true; + rest = tag; + } +} diff --git a/crates/text-extract/src/rtf/words.rs b/crates/text-extract/src/rtf/words.rs new file mode 100644 index 00000000..383bc21a --- /dev/null +++ b/crates/text-extract/src/rtf/words.rs @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Word { + Ansi, + AnsiCodePage, + Mac, + DefaultFont, + FromHtml, + HtmlRtf, + FontTable, + Font, + FontCharset, + FontCodePage, + Plain, + UnicodeSkip, + Unicode, + UnicodeDestination, + Annotation, + HtmlTag, + ShapeInstructions, + Bin, + Paragraph, + NestedRow, + Tab, + Character(char), + Destination, +} + +impl Word { + pub(crate) fn parse(name: &[u8]) -> Option { + hashify::map!(name, Word, + b"ansi" => Word::Ansi, + b"ansicpg" => Word::AnsiCodePage, + b"mac" => Word::Mac, + b"deff" => Word::DefaultFont, + b"fromhtml" => Word::FromHtml, + b"htmlrtf" => Word::HtmlRtf, + b"fonttbl" => Word::FontTable, + b"f" => Word::Font, + b"fcharset" => Word::FontCharset, + b"cpg" => Word::FontCodePage, + b"plain" => Word::Plain, + b"uc" => Word::UnicodeSkip, + b"u" => Word::Unicode, + b"ud" => Word::UnicodeDestination, + b"annotation" => Word::Annotation, + b"htmltag" => Word::HtmlTag, + b"shpinst" => Word::ShapeInstructions, + b"bin" => Word::Bin, + b"par" => Word::Paragraph, + b"line" => Word::Paragraph, + b"lbr" => Word::Paragraph, + b"sect" => Word::Paragraph, + b"page" => Word::Paragraph, + b"row" => Word::Paragraph, + b"nestrow" => Word::NestedRow, + b"column" => Word::Paragraph, + b"tab" => Word::Tab, + b"cell" => Word::Tab, + b"nestcell" => Word::Tab, + b"emdash" => Word::Character('\u{2014}'), + b"endash" => Word::Character('\u{2013}'), + b"lquote" => Word::Character('\u{2018}'), + b"rquote" => Word::Character('\u{2019}'), + b"ldblquote" => Word::Character('\u{201C}'), + b"rdblquote" => Word::Character('\u{201D}'), + b"bullet" => Word::Character('\u{2022}'), + b"emspace" => Word::Character(' '), + b"enspace" => Word::Character(' '), + b"qmspace" => Word::Character(' '), + b"aftnsep" => Word::Destination, + b"aftnsepc" => Word::Destination, + b"author" => Word::Destination, + b"bkmkend" => Word::Destination, + b"bkmkstart" => Word::Destination, + b"blipuid" => Word::Destination, + b"colortbl" => Word::Destination, + b"colorschememapping" => Word::Destination, + b"datastore" => Word::Destination, + b"defchp" => Word::Destination, + b"defpap" => Word::Destination, + b"docvar" => Word::Destination, + b"falt" => Word::Destination, + b"fchars" => Word::Destination, + b"filetbl" => Word::Destination, + b"fldinst" => Word::Destination, + b"ftnsep" => Word::Destination, + b"ftnsepc" => Word::Destination, + b"generator" => Word::Destination, + b"info" => Word::Destination, + b"latentstyles" => Word::Destination, + b"lchars" => Word::Destination, + b"levelnumbers" => Word::Destination, + b"leveltext" => Word::Destination, + b"listoverridetable" => Word::Destination, + b"listtable" => Word::Destination, + b"listtext" => Word::Destination, + b"mhtmltag" => Word::Destination, + b"mmathPr" => Word::Destination, + b"nonesttables" => Word::Destination, + b"nonshppict" => Word::Destination, + b"objclass" => Word::Destination, + b"objdata" => Word::Destination, + b"operator" => Word::Destination, + b"panose" => Word::Destination, + b"pgdsctbl" => Word::Destination, + b"pict" => Word::Destination, + b"pntext" => Word::Destination, + b"pntxta" => Word::Destination, + b"pntxtb" => Word::Destination, + b"protusertbl" => Word::Destination, + b"revtbl" => Word::Destination, + b"rsidtbl" => Word::Destination, + b"shprslt" => Word::Destination, + b"sp" => Word::Destination, + b"stylesheet" => Word::Destination, + b"tc" => Word::Destination, + b"template" => Word::Destination, + b"themedata" => Word::Destination, + b"upr" => Word::Destination, + b"userprops" => Word::Destination, + b"wgrffmtfilter" => Word::Destination, + b"xe" => Word::Destination, + b"xmlnstbl" => Word::Destination, + ) + .copied() + } +} diff --git a/crates/text-extract/src/utf8.rs b/crates/text-extract/src/utf8.rs new file mode 100644 index 00000000..963eda9a --- /dev/null +++ b/crates/text-extract/src/utf8.rs @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::xml::scan::incomplete_utf8_tail; + +const BLOCK: usize = 4096; +const SPARSE_RATIO: usize = 4; + +pub(crate) struct LazyUtf8<'a> { + source: &'a [u8], + block: &'a str, + block_start: usize, + validated: usize, + requested: usize, + sparse: bool, +} + +impl<'a> LazyUtf8<'a> { + pub(crate) fn new(source: &'a [u8]) -> Self { + LazyUtf8 { + source, + block: "", + block_start: 0, + validated: 0, + requested: 0, + sparse: false, + } + } + + #[inline] + pub(crate) fn get(&mut self, start: usize, len: usize) -> Option<&'a str> { + self.requested += len; + let offset = match start + .checked_sub(self.block_start) + .filter(|offset| offset + len <= self.block.len()) + { + Some(offset) => offset, + None if self.sparse => return None, + None => { + self.validate_from(start, len)?; + 0 + } + }; + self.block.get(offset..offset + len) + } + + #[inline(never)] + fn validate_from(&mut self, start: usize, len: usize) -> Option<()> { + if self.validated > self.requested.saturating_mul(SPARSE_RATIO) { + self.sparse = true; + return None; + } + let end = (start + len).max(start + BLOCK).min(self.source.len()); + self.block = valid_prefix(self.source.get(start..end)?); + self.block_start = start; + self.validated += end - start; + Some(()) + } +} + +fn valid_prefix(bytes: &[u8]) -> &str { + let complete = bytes + .get(..bytes.len() - incomplete_utf8_tail(bytes)) + .unwrap_or(bytes); + match std::str::from_utf8(complete) { + Ok(text) => text, + Err(error) => complete + .get(..error.valid_up_to()) + .and_then(|prefix| std::str::from_utf8(prefix).ok()) + .unwrap_or_default(), + } +} + diff --git a/crates/text-extract/src/xml/attr.rs b/crates/text-extract/src/xml/attr.rs new file mode 100644 index 00000000..a51e4b81 --- /dev/null +++ b/crates/text-extract/src/xml/attr.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{MAX_ENTITY, Tag, local_name}; +use crate::output::Output; +use memchr::memchr; + +impl<'a> Tag<'a> { + pub(crate) fn attributes(&self) -> Attributes<'a> { + Attributes { + rest: self.attrs.unwrap_or_default(), + } + } +} + +#[inline] +pub(crate) fn split_prefix(name: &[u8]) -> (bool, &[u8]) { + let local = local_name(name); + (local.len() < name.len(), local) +} + +pub(crate) struct Attributes<'a> { + rest: &'a [u8], +} + +impl<'a> Iterator for Attributes<'a> { + type Item = (&'a [u8], &'a [u8]); + + fn next(&mut self) -> Option { + let rest = self.rest.trim_ascii_start(); + let name_len = rest + .iter() + .position(|&byte| byte == b'=' || byte.is_ascii_whitespace())?; + let (name, rest) = rest.split_at(name_len); + let rest = rest + .trim_ascii_start() + .strip_prefix(b"=")? + .trim_ascii_start(); + let ("e, rest) = rest.split_first()?; + if quote != b'"' && quote != b'\'' { + self.rest = b""; + return None; + } + let value_len = rest.iter().position(|&byte| byte == quote)?; + let (value, rest) = rest.split_at(value_len); + self.rest = rest.get(1..).unwrap_or_default(); + Some((name, value)) + } +} + +pub(crate) fn resolve_entity(body: &[u8]) -> Option { + match body { + [b'#', b'x' | b'X', hex @ ..] => parse_code_point(hex, 16), + [b'#', decimal @ ..] => parse_code_point(decimal, 10), + _ => hashify::map!(body, char, + b"amp" => '&', + b"lt" => '<', + b"gt" => '>', + b"quot" => '"', + b"apos" => '\'', + ) + .copied(), + } +} + +fn parse_code_point(digits: &[u8], radix: u32) -> Option { + if digits.is_empty() { + return None; + } + digits + .iter() + .try_fold(0u32, |value, &digit| { + value + .checked_mul(radix)? + .checked_add(char::from(digit).to_digit(radix)?) + }) + .and_then(char::from_u32) +} + +pub(crate) fn split_entity(text: &[u8]) -> Option<(&[u8], &[u8])> { + let window = text.get(1..text.len().min(MAX_ENTITY))?; + let semicolon = memchr(b';', window)?; + Some((window.get(..semicolon)?, text.get(semicolon + 2..)?)) +} + +pub(crate) fn decode_into(value: &[u8], dst: &mut Vec) { + let mut rest = value; + while let Some(amp) = memchr(b'&', rest) { + let (plain, tail) = rest.split_at(amp); + dst.extend_from_slice(plain); + match split_entity(tail).and_then(|(body, after)| Some((resolve_entity(body)?, after))) { + Some((ch, after)) => { + let mut encoded = [0u8; 4]; + dst.extend_from_slice(ch.encode_utf8(&mut encoded).as_bytes()); + rest = after; + } + None => { + dst.push(b'&'); + rest = tail.get(1..).unwrap_or_default(); + } + } + } + dst.extend_from_slice(rest); +} + +pub(crate) fn push_decoded(value: &[u8], out: &mut Output<'_>) { + let mut rest = value; + while let Some(amp) = memchr(b'&', rest) { + let (plain, tail) = rest.split_at(amp); + out.push_utf8(plain); + match split_entity(tail).and_then(|(body, after)| Some((resolve_entity(body)?, after))) { + Some((ch, after)) => { + out.push_char(ch); + rest = after; + } + None => { + out.push_str("&"); + rest = tail.get(1..).unwrap_or_default(); + } + } + } + out.push_utf8(rest); +} + diff --git a/crates/text-extract/src/xml/mod.rs b/crates/text-extract/src/xml/mod.rs new file mode 100644 index 00000000..0ca70bfc --- /dev/null +++ b/crates/text-extract/src/xml/mod.rs @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) mod attr; +pub(crate) mod scan; +pub(crate) mod stream; + +use crate::output::Output; + +pub(crate) const MAX_TAG_CARRY: usize = 1 << 14; +pub(crate) const MAX_NAME: usize = 64; +const MAX_ENTITY: usize = 32; + +pub(crate) struct Tag<'a> { + pub(crate) name: &'a [u8], + pub(crate) attrs: Option<&'a [u8]>, +} + +impl<'a> Tag<'a> { + #[inline] + pub(crate) fn local(&self) -> &'a [u8] { + local_name(self.name) + } +} + +#[derive(Debug, Clone, Copy)] +enum Chunk<'a> { + Valid(&'a str), + Raw(&'a [u8]), +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Text<'a> { + chunk: Chunk<'a>, + closed_by_tag: bool, +} + +impl<'a> Text<'a> { + #[inline] + pub(crate) fn valid(text: &'a str) -> Self { + Text { + chunk: Chunk::Valid(text), + closed_by_tag: false, + } + } + + #[inline] + pub(crate) fn raw(bytes: &'a [u8]) -> Self { + Text { + chunk: Chunk::Raw(bytes), + closed_by_tag: false, + } + } + + #[inline] + pub(crate) fn closed_by_tag(mut self, closed: bool) -> Self { + self.closed_by_tag = closed; + self + } + + #[inline] + pub(crate) fn is_closed_by_tag(self) -> bool { + self.closed_by_tag + } + + #[inline] + pub(crate) fn as_bytes(self) -> &'a [u8] { + match self.chunk { + Chunk::Valid(text) => text.as_bytes(), + Chunk::Raw(bytes) => bytes, + } + } + + #[inline] + pub(crate) fn push(self, out: &mut Output<'_>) { + match self.chunk { + Chunk::Valid(text) => out.push_str(text), + Chunk::Raw(bytes) => out.push_utf8(bytes), + } + } +} + +pub(crate) trait Handler { + fn start(&mut self, tag: &Tag<'_>, out: &mut Output<'_>); + fn end(&mut self, name: &[u8], out: &mut Output<'_>); + fn text(&mut self, text: Text<'_>, out: &mut Output<'_>); + fn entity(&mut self, _name: &[u8], _out: &mut Output<'_>) {} + fn aborted(&self) -> bool { + false + } +} + +#[inline] +pub(crate) fn local_name(name: &[u8]) -> &[u8] { + match name.iter().rposition(|&byte| byte == b':') { + Some(colon) => name.get(colon + 1..).unwrap_or_default(), + None => name, + } +} + +pub(crate) struct Skip { + element: Option, + depth: u32, +} + +impl Default for Skip { + fn default() -> Self { + Skip { + element: None, + depth: 0, + } + } +} + +impl Skip { + #[inline] + pub(crate) fn active(&self) -> bool { + self.depth > 0 + } + + #[inline] + pub(crate) fn begin(&mut self, element: E) { + self.element = Some(element); + self.depth = 1; + } + + #[inline] + pub(crate) fn on_start(&mut self, element: Option) -> bool { + if self.depth == 0 { + return false; + } + if element.is_some() && element == self.element { + self.depth = self.depth.saturating_add(1); + } + true + } + + #[inline] + pub(crate) fn on_end(&mut self, element: Option) -> bool { + if self.depth == 0 { + return false; + } + if element.is_some() && element == self.element { + self.depth -= 1; + } + true + } + + pub(crate) fn reset(&mut self) { + self.depth = 0; + } +} diff --git a/crates/text-extract/src/xml/scan.rs b/crates/text-extract/src/xml/scan.rs new file mode 100644 index 00000000..6f30a938 --- /dev/null +++ b/crates/text-extract/src/xml/scan.rs @@ -0,0 +1,403 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Handler, MAX_ENTITY, MAX_NAME, MAX_TAG_CARRY, Tag, Text, + attr::{resolve_entity, split_entity}, +}; +use crate::{output::Output, utf8::LazyUtf8}; +use memchr::{memchr2, memmem}; + +const CDATA_OPEN: &[u8] = b" Self { + let mut stored = [0u8; MAX_NAME]; + let name_len = match stored.get_mut(..name.len()) { + Some(slot) => { + slot.copy_from_slice(name); + name.len() + } + None => 0, + }; + LongTag { + closing, + quote: 0, + previous: 0, + name: stored, + name_len, + } + } + + fn name(&self) -> &[u8] { + self.name.get(..self.name_len).unwrap_or_default() + } +} + +pub(crate) struct Scanner { + state: State, +} + +impl Default for Scanner { + fn default() -> Self { + Scanner { state: State::Text } + } +} + +struct Input<'a> { + len: usize, + utf8: LazyUtf8<'a>, +} + +impl<'a> Input<'a> { + #[inline] + fn text(&mut self, rest: &'a [u8], len: usize) -> Text<'a> { + match self.utf8.get(self.len - rest.len(), len) { + Some(valid) => Text::valid(valid), + None => Text::raw(rest.get(..len).unwrap_or_default()), + } + } +} + +impl Scanner { + pub(crate) fn feed( + &mut self, + bytes: &[u8], + last: bool, + handler: &mut H, + out: &mut Output<'_>, + ) -> usize { + let input_len = bytes.len(); + let mut input = Input { + len: input_len, + utf8: LazyUtf8::new(bytes), + }; + let mut rest = bytes; + while !out.is_full() && !handler.aborted() { + match self.state { + State::Text => match if rest.first() == Some(&b'<') { + Some(0) + } else { + memchr2(b'<', b'&', rest) + } { + None => { + let keep = if last { 0 } else { incomplete_utf8_tail(rest) }; + let len = rest.len() - keep; + if len > 0 { + handler.text(input.text(rest, len), out); + } + return input_len - keep; + } + Some(at) => { + let markup = rest.get(at..).unwrap_or_default(); + if at > 0 { + let text = input.text(rest, at); + handler.text(text.closed_by_tag(markup.first() == Some(&b'<')), out); + } + let step = if markup.first() == Some(&b'&') { + entity(markup, last, handler, out) + } else { + self.markup(markup, last, handler, out) + }; + match step { + Some(consumed) => rest = markup.get(consumed..).unwrap_or_default(), + None => return input_len - markup.len(), + } + } + }, + State::Comment => match memmem::find(rest, b"-->") { + Some(at) => { + self.state = State::Text; + rest = rest.get(at + 3..).unwrap_or_default(); + } + None => return input_len - if last { 0 } else { rest.len().min(2) }, + }, + State::Instruction => match memmem::find(rest, b"?>") { + Some(at) => { + self.state = State::Text; + rest = rest.get(at + 2..).unwrap_or_default(); + } + None => return input_len - if last { 0 } else { rest.len().min(1) }, + }, + State::CData => match memmem::find(rest, b"]]>") { + Some(at) => { + if at > 0 { + handler.text(input.text(rest, at), out); + } + self.state = State::Text; + rest = rest.get(at + 3..).unwrap_or_default(); + } + None => { + let mut end = if last { + rest.len() + } else { + rest.len().saturating_sub(2) + }; + if !last { + end -= incomplete_utf8_tail(rest.get(..end).unwrap_or_default()); + } + if end > 0 { + handler.text(input.text(rest, end), out); + } + return input_len - (rest.len() - end); + } + }, + State::Declaration { depth, quote } => match skip_declaration(rest, depth, quote) { + Ok(consumed) => { + self.state = State::Text; + rest = rest.get(consumed..).unwrap_or_default(); + } + Err((depth, quote)) => { + self.state = State::Declaration { depth, quote }; + return input_len; + } + }, + State::LongTag(mut long) => match find_tag_end(rest, &mut long.quote) { + Some(at) => { + let self_closing = match at.checked_sub(1) { + Some(before) => rest.get(before) == Some(&b'/'), + None => long.previous == b'/', + }; + let name = long.name(); + if long.closing { + handler.end(name, out); + } else { + handler.start(&Tag { name, attrs: None }, out); + if self_closing { + handler.end(name, out); + } + } + self.state = State::Text; + rest = rest.get(at + 1..).unwrap_or_default(); + } + None => { + if let Some(&previous) = rest.last() { + long.previous = previous; + } + self.state = State::LongTag(long); + return input_len; + } + }, + } + } + input_len + } + + fn markup( + &mut self, + rest: &[u8], + last: bool, + handler: &mut H, + out: &mut Output<'_>, + ) -> Option { + match rest { + [b'<', b'/', tail @ ..] => match tail.iter().position(|&byte| byte == b'>') { + Some(gt) => { + handler.end(tail.get(..gt).unwrap_or_default().trim_ascii_end(), out); + Some(gt + 3) + } + None if !last && rest.len() < MAX_TAG_CARRY => None, + None => { + let name_len = tail + .iter() + .position(|byte| byte.is_ascii_whitespace()) + .unwrap_or(tail.len()); + self.state = State::LongTag(LongTag::new( + true, + tail.get(..name_len).unwrap_or_default(), + )); + Some(rest.len()) + } + }, + [b'<', b'!', b'-', b'-', ..] => { + self.state = State::Comment; + Some(COMMENT_OPEN.len()) + } + [b'<', b'!', ..] if rest.starts_with(CDATA_OPEN) => { + self.state = State::CData; + Some(CDATA_OPEN.len()) + } + [b'<', b'!', ..] + if !last && (CDATA_OPEN.starts_with(rest) || COMMENT_OPEN.starts_with(rest)) => + { + None + } + [b'<', b'!', ..] => { + self.state = State::Declaration { depth: 0, quote: 0 }; + Some(2) + } + [b'<', b'?', ..] => { + self.state = State::Instruction; + Some(2) + } + [b'<', first, ..] if is_name_start(*first) => self.start_tag(rest, last, handler, out), + [b'<'] if !last => None, + _ => { + handler.text(Text::valid("<"), out); + Some(1) + } + } + } + + fn start_tag( + &mut self, + rest: &[u8], + last: bool, + handler: &mut H, + out: &mut Output<'_>, + ) -> Option { + let body = rest.get(1..).unwrap_or_default(); + let name_len = body + .iter() + .position(|&byte| byte.is_ascii_whitespace() || byte == b'/' || byte == b'>'); + let mut quote = 0; + let end = name_len.and_then(|name_len| { + find_tag_end(body.get(name_len..)?, &mut quote).map(|at| name_len + at) + }); + match (name_len, end) { + (Some(name_len), Some(at)) => { + let (name, _) = body.split_at(name_len); + let self_closing = at > name_len && body.get(at - 1) == Some(&b'/'); + let attrs_end = if self_closing { at - 1 } else { at }; + handler.start( + &Tag { + name, + attrs: body.get(name_len..attrs_end.max(name_len)), + }, + out, + ); + if self_closing { + handler.end(name, out); + } + Some(at + 2) + } + _ if !last && rest.len() < MAX_TAG_CARRY => None, + (name_len, _) => { + let name_len = name_len.unwrap_or(body.len()); + let mut long = LongTag::new(false, body.get(..name_len).unwrap_or_default()); + long.quote = quote; + long.previous = rest.last().copied().unwrap_or(0); + self.state = State::LongTag(long); + Some(rest.len()) + } + } + } +} + +fn entity( + rest: &[u8], + last: bool, + handler: &mut H, + out: &mut Output<'_>, +) -> Option { + match split_entity(rest) { + Some((body, after)) => { + match resolve_entity(body) { + Some(ch) => { + let mut encoded = [0u8; 4]; + handler.text(Text::valid(ch.encode_utf8(&mut encoded)), out); + } + None if body.first() == Some(&b'#') => (), + None if !body.is_empty() && body.iter().all(u8::is_ascii_alphanumeric) => { + handler.entity(body, out); + } + None => { + handler.text(Text::valid("&"), out); + return Some(1); + } + } + Some(rest.len() - after.len()) + } + None if !last && rest.len() < MAX_ENTITY => None, + None => { + handler.text(Text::valid("&"), out); + Some(1) + } + } +} + +fn find_tag_end(bytes: &[u8], quote: &mut u8) -> Option { + let mut cursor = 0; + loop { + let tail = bytes.get(cursor..)?; + if *quote != 0 { + cursor += tail.iter().position(|&byte| byte == *quote)? + 1; + *quote = 0; + continue; + } + let at = tail + .iter() + .position(|&byte| byte == b'>' || byte == b'"' || byte == b'\'')?; + match tail.get(at) { + Some(b'>') => return Some(cursor + at), + Some(&found) => { + *quote = found; + cursor += at + 1; + } + None => return None, + } + } +} + +fn skip_declaration(bytes: &[u8], mut depth: u32, mut quote: u8) -> Result { + for (at, &byte) in bytes.iter().enumerate() { + if quote != 0 { + if byte == quote { + quote = 0; + } + continue; + } + match byte { + b'"' | b'\'' => quote = byte, + b'[' => depth = depth.saturating_add(1), + b']' => depth = depth.saturating_sub(1), + b'>' if depth == 0 => return Ok(at + 1), + _ => {} + } + } + Err((depth, quote)) +} + +#[inline] +fn is_name_start(byte: u8) -> bool { + byte.is_ascii_alphabetic() || byte == b'_' || byte == b':' || byte >= 0x80 +} + +pub(crate) fn incomplete_utf8_tail(bytes: &[u8]) -> usize { + for (back, &byte) in bytes.iter().rev().take(3).enumerate() { + if byte & 0xC0 != 0x80 { + let needed = match byte { + 0xF0.. => 4, + 0xE0.. => 3, + 0xC0.. => 2, + _ => 1, + }; + return if needed > back + 1 { back + 1 } else { 0 }; + } + } + 0 +} + diff --git a/crates/text-extract/src/xml/stream.rs b/crates/text-extract/src/xml/stream.rs new file mode 100644 index 00000000..c542dfcf --- /dev/null +++ b/crates/text-extract/src/xml/stream.rs @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Handler, Tag, scan::Scanner}; +use crate::{output::Output, zip::MemberData}; +use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE}; +use flate2::{Decompress, FlushDecompress, Status}; +use memchr::memmem; + +pub(crate) const WINDOW: usize = 1 << 16; +const SNIFF_LEN: usize = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Part { + Content, + Plumbing, +} + +pub(crate) struct Budget { + pub(crate) part_bytes: u64, + pub(crate) total_bytes: u64, + pub(crate) parts: usize, + pub(crate) used_bytes: u64, + pub(crate) used_parts: usize, + pub(crate) truncated: bool, +} + +impl Budget { + pub(crate) fn part_cap(&self) -> u64 { + self.part_bytes + .min(self.total_bytes.saturating_sub(self.used_bytes)) + } + + pub(crate) fn charge(&mut self, bytes: usize) { + self.used_bytes = self.used_bytes.saturating_add(bytes as u64); + } + + pub(crate) fn exhausted(&self) -> bool { + self.used_parts >= self.parts || self.used_bytes >= self.total_bytes + } +} + +#[derive(Default)] +pub(crate) struct Buffers { + window: Vec, + raw: Vec, + inflater: Option, +} + +enum Sniff { + Utf8 { + skip: usize, + }, + Encoded { + encoding: &'static Encoding, + skip: usize, + }, + NotXml, +} + +enum Source<'a> { + Stored { + data: &'a [u8], + }, + Inflate { + data: &'a [u8], + inflater: &'a mut Decompress, + done: bool, + }, +} + +impl Source<'_> { + fn fill(&mut self, dst: &mut [u8]) -> usize { + match self { + Source::Stored { data } => { + let len = data.len().min(dst.len()); + let (head, tail) = data.split_at(len); + if let Some(slot) = dst.get_mut(..len) { + slot.copy_from_slice(head); + } + *data = tail; + len + } + Source::Inflate { + data, + inflater, + done, + } => { + if *done || dst.is_empty() { + return 0; + } + let before_in = inflater.total_in(); + let before_out = inflater.total_out(); + let status = inflater.decompress(data, dst, FlushDecompress::None); + let consumed = usize::try_from(inflater.total_in().saturating_sub(before_in)) + .unwrap_or(usize::MAX); + let produced = usize::try_from(inflater.total_out().saturating_sub(before_out)) + .unwrap_or(usize::MAX); + *data = data.get(consumed..).unwrap_or_default(); + if !matches!(status, Ok(Status::Ok | Status::BufError)) + || (produced == 0 && (consumed == 0 || data.is_empty())) + { + *done = true; + } + produced.min(dst.len()) + } + } + } + + fn finished(&self) -> bool { + match self { + Source::Stored { data } => data.is_empty(), + Source::Inflate { done, .. } => *done, + } + } +} + +impl Buffers { + pub(crate) fn scan( + &mut self, + member: MemberData<'_>, + part: Part, + budget: &mut Budget, + handler: &mut H, + out: &mut Output<'_>, + ) { + if budget.exhausted() { + budget.truncated = true; + return; + } + if part == Part::Content { + budget.used_parts += 1; + } + let cap = usize::try_from(budget.part_cap()).unwrap_or(usize::MAX); + match member { + MemberData::Stored(data) => { + let bounded = data.get(..cap).unwrap_or(data); + if bounded.len() < data.len() { + budget.truncated = true; + } + match sniff(bounded) { + Sniff::Utf8 { skip } => { + budget.charge(bounded.len()); + let body = bounded.get(skip..).unwrap_or_default(); + Scanner::default().feed(body, true, handler, out); + } + Sniff::Encoded { encoding, skip } => { + let source = Source::Stored { + data: bounded.get(skip..).unwrap_or_default(), + }; + self.decode(source, encoding, cap, budget, handler, out); + } + Sniff::NotXml => (), + } + } + MemberData::Deflated(data) => { + let inflater = match &mut self.inflater { + Some(inflater) => { + inflater.reset(false); + inflater + } + None => self.inflater.insert(Decompress::new(false)), + }; + let source = Source::Inflate { + data, + inflater, + done: false, + }; + Self::stream( + &mut self.window, + &mut self.raw, + source, + cap, + budget, + handler, + out, + ); + } + } + } + + fn decode( + &mut self, + source: Source<'_>, + encoding: &'static Encoding, + cap: usize, + budget: &mut Budget, + handler: &mut H, + out: &mut Output<'_>, + ) { + let mut decoder = encoding.new_decoder_without_bom_handling(); + prepare(&mut self.raw); + prepare(&mut self.window); + transcode( + &mut self.window, + &mut self.raw, + 0, + source, + &mut decoder, + cap, + budget, + handler, + out, + ); + } + + #[allow(clippy::too_many_arguments)] + fn stream( + window: &mut Vec, + raw: &mut Vec, + mut source: Source<'_>, + cap: usize, + budget: &mut Budget, + handler: &mut H, + out: &mut Output<'_>, + ) { + prepare(window); + let mut filled = 0; + let mut remaining = cap; + while filled < SNIFF_LEN && !source.finished() && remaining > 0 { + let room = window.len().min(filled.saturating_add(remaining)); + let produced = source.fill(window.get_mut(filled..room).unwrap_or_default()); + filled += produced; + remaining -= produced.min(remaining); + } + budget.charge(filled); + match sniff(window.get(..filled).unwrap_or_default()) { + Sniff::NotXml => {} + Sniff::Utf8 { skip } => { + let mut scanner = Scanner::default(); + let mut start = skip.min(filled); + loop { + let exhausted = source.finished() || remaining == 0; + let chunk = window.get(start..filled).unwrap_or_default(); + let consumed = scanner.feed(chunk, exhausted, handler, out); + if exhausted || out.is_full() || handler.aborted() { + if remaining == 0 && !source.finished() { + budget.truncated = true; + } + return; + } + let begin = start.saturating_add(consumed).min(filled); + window.copy_within(begin..filled, 0); + filled -= begin; + start = 0; + let room = window.len().min(filled.saturating_add(remaining)); + let produced = source.fill(window.get_mut(filled..room).unwrap_or_default()); + filled += produced; + remaining -= produced.min(remaining); + budget.charge(produced); + } + } + Sniff::Encoded { encoding, skip } => { + let mut decoder = encoding.new_decoder_without_bom_handling(); + prepare(raw); + std::mem::swap(window, raw); + let start = skip.min(filled); + raw.copy_within(start..filled, 0); + transcode( + window, + raw, + filled - start, + source, + &mut decoder, + remaining, + budget, + handler, + out, + ); + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn transcode( + window: &mut [u8], + raw: &mut [u8], + mut raw_filled: usize, + mut source: Source<'_>, + decoder: &mut Decoder, + mut remaining: usize, + budget: &mut Budget, + handler: &mut H, + out: &mut Output<'_>, +) { + let mut scanner = Scanner::default(); + let mut filled = 0; + loop { + let room = raw.len().min(raw_filled.saturating_add(remaining)); + let produced = source.fill(raw.get_mut(raw_filled..room).unwrap_or_default()); + raw_filled += produced; + remaining -= produced.min(remaining); + budget.charge(produced); + let source_done = source.finished() || remaining == 0; + let (result, read, written, _) = decoder.decode_to_utf8( + raw.get(..raw_filled).unwrap_or_default(), + window.get_mut(filled..).unwrap_or_default(), + source_done, + ); + raw.copy_within(read.min(raw_filled)..raw_filled, 0); + raw_filled -= read.min(raw_filled); + filled += written; + let exhausted = source_done && raw_filled == 0 && result == CoderResult::InputEmpty; + let chunk = window.get(..filled).unwrap_or_default(); + let consumed = scanner.feed(chunk, exhausted, handler, out); + if exhausted || out.is_full() || handler.aborted() { + if remaining == 0 && !source.finished() { + budget.truncated = true; + } + return; + } + let consumed = consumed.min(filled); + window.copy_within(consumed..filled, 0); + filled -= consumed; + if produced == 0 && read == 0 && written == 0 && consumed == 0 { + return; + } + } +} + +fn prepare(buffer: &mut Vec) { + if buffer.len() != WINDOW { + buffer.resize(WINDOW, 0); + } +} + +fn sniff(prefix: &[u8]) -> Sniff { + match prefix { + [0xEF, 0xBB, 0xBF, rest @ ..] => utf8_or_declared(rest, 3), + [0xFF, 0xFE, ..] => Sniff::Encoded { + encoding: UTF_16LE, + skip: 2, + }, + [0xFE, 0xFF, ..] => Sniff::Encoded { + encoding: UTF_16BE, + skip: 2, + }, + [b'<', 0, ..] => Sniff::Encoded { + encoding: UTF_16LE, + skip: 0, + }, + [0, b'<', ..] => Sniff::Encoded { + encoding: UTF_16BE, + skip: 0, + }, + _ => utf8_or_declared(prefix, 0), + } +} + +fn utf8_or_declared(rest: &[u8], skip: usize) -> Sniff { + let trimmed = rest.trim_ascii_start(); + if trimmed.first() != Some(&b'<') { + return Sniff::NotXml; + } + let Some(declaration) = trimmed.strip_prefix(b"").unwrap_or(declaration.len())) + .unwrap_or_default(); + let declaration = Tag { + name: b"", + attrs: Some(declaration), + }; + let Some(label) = declaration + .attributes() + .find_map(|(name, value)| hashify::set!(name, b"encoding").then_some(value)) + else { + return Sniff::Utf8 { skip }; + }; + match Encoding::for_label(label) { + Some(encoding) if encoding != UTF_8 && encoding.is_ascii_compatible() => { + Sniff::Encoded { encoding, skip } + } + _ => Sniff::Utf8 { skip }, + } +} diff --git a/crates/text-extract/src/zip.rs b/crates/text-extract/src/zip.rs new file mode 100644 index 00000000..02bab549 --- /dev/null +++ b/crates/text-extract/src/zip.rs @@ -0,0 +1,351 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::memmem; +use std::cmp::Ordering; + +const EOCD_SIGNATURE: &[u8; 4] = b"PK\x05\x06"; +const EOCD_LEN: usize = 22; +const MAX_EOCD_COMMENT: usize = 0xFFFF; +const ZIP64_LOCATOR_SIGNATURE: &[u8; 4] = b"PK\x06\x07"; +const ZIP64_LOCATOR_LEN: usize = 20; +const ZIP64_EOCD_SIGNATURE: &[u8; 4] = b"PK\x06\x06"; +const ZIP64_EOCD_LEN: usize = 56; +const CENTRAL_SIGNATURE: &[u8; 4] = b"PK\x01\x02"; +const CENTRAL_LEN: usize = 46; +const LOCAL_SIGNATURE: &[u8; 4] = b"PK\x03\x04"; +const LOCAL_LEN: usize = 30; +const ZIP64_EXTRA_ID: u16 = 0x0001; +const FLAG_ENCRYPTED: u16 = 0x0001; +const METHOD_STORED: u16 = 0; +const METHOD_DEFLATED: u16 = 8; +const U16_MAX: u64 = 0xFFFF; +const U32_MAX: u64 = 0xFFFF_FFFF; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Entry { + record: usize, + name_len: u16, +} + +pub(crate) struct Archive<'a> { + data: &'a [u8], + entries: &'a [Entry], + central_start: usize, + shift: usize, + pub(crate) entries_truncated: bool, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Member<'a> { + pub(crate) name: &'a [u8], + flags: u16, + method: u16, + compressed_size: u64, + local_offset: u64, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum MemberData<'a> { + Stored(&'a [u8]), + Deflated(&'a [u8]), +} + +#[derive(Default)] +pub(crate) struct ReadRanges { + ranges: Vec<(usize, usize)>, +} + +impl ReadRanges { + pub(crate) fn clear(&mut self) { + self.ranges.clear(); + } + + fn claim(&mut self, start: usize, end: usize) -> bool { + let index = self + .ranges + .partition_point(|&(range_start, _)| range_start < start); + let overlaps_previous = index + .checked_sub(1) + .and_then(|previous| self.ranges.get(previous)) + .is_some_and(|&(_, previous_end)| previous_end > start); + let overlaps_next = self + .ranges + .get(index) + .is_some_and(|&(next_start, _)| next_start < end); + if overlaps_previous || overlaps_next { + return false; + } + self.ranges.insert(index, (start, end)); + true + } +} + +impl Member<'_> { + #[inline] + pub(crate) fn is_readable(&self) -> bool { + self.flags & FLAG_ENCRYPTED == 0 && matches!(self.method, METHOD_STORED | METHOD_DEFLATED) + } +} + +impl<'a> Archive<'a> { + pub(crate) fn open( + data: &'a [u8], + entries: &'a mut Vec, + max_entries: usize, + ) -> Option> { + let (central_start, central_end, shift) = locate_central_directory(data)?; + entries.clear(); + let mut entries_truncated = false; + let mut cursor = central_start; + while cursor < central_end { + if entries.len() >= max_entries { + entries_truncated = true; + break; + } + let Some(record) = data.get(cursor..central_end) else { + break; + }; + if record.get(..4) != Some(CENTRAL_SIGNATURE.as_slice()) { + break; + } + let (Some(name_len), Some(extra_len), Some(comment_len)) = + (le_u16(record, 28), le_u16(record, 30), le_u16(record, 32)) + else { + break; + }; + let Some(record_len) = CENTRAL_LEN + .checked_add(usize::from(name_len)) + .and_then(|len| len.checked_add(usize::from(extra_len))) + .and_then(|len| len.checked_add(usize::from(comment_len))) + .filter(|&len| len <= record.len()) + else { + break; + }; + entries.push(Entry { + record: cursor, + name_len, + }); + cursor += record_len; + } + if entries.is_empty() { + return None; + } + entries + .sort_by(|left, right| compare_names(entry_name(data, left), entry_name(data, right))); + Some(Archive { + data, + entries, + central_start, + shift, + entries_truncated, + }) + } + + pub(crate) fn find(&self, name: &[u8]) -> Option> { + let index = self + .entries + .partition_point(|entry| compare_names(entry_name(self.data, entry), name).is_lt()); + let entry = self.entries.get(index)?; + if compare_names(entry_name(self.data, entry), name).is_eq() { + self.member(entry) + } else { + None + } + } + + pub(crate) fn contains(&self, name: &[u8]) -> bool { + self.find(name).is_some() + } + + pub(crate) fn members(&self) -> impl Iterator> + '_ { + self.entries.iter().filter_map(|entry| self.member(entry)) + } + + fn member(&self, entry: &Entry) -> Option> { + let record = self.data.get(entry.record..)?; + let flags = le_u16(record, 8)?; + let method = le_u16(record, 10)?; + let mut compressed_size = u64::from(le_u32(record, 20)?); + let mut uncompressed_size = u64::from(le_u32(record, 24)?); + let extra_len = usize::from(le_u16(record, 30)?); + let mut local_offset = u64::from(le_u32(record, 42)?); + let name_end = CENTRAL_LEN.checked_add(usize::from(entry.name_len))?; + let name = record.get(CENTRAL_LEN..name_end)?; + let extra = record.get(name_end..name_end.checked_add(extra_len)?)?; + if compressed_size == U32_MAX || uncompressed_size == U32_MAX || local_offset == U32_MAX { + let mut fields = zip64_extra(extra)?; + for value in [ + &mut uncompressed_size, + &mut compressed_size, + &mut local_offset, + ] { + if *value == U32_MAX { + *value = fields.next()?; + } + } + } + Some(Member { + name, + flags, + method, + compressed_size, + local_offset, + }) + } + + pub(crate) fn read( + &self, + member: &Member<'a>, + claimed: &mut ReadRanges, + ) -> Option> { + if !member.is_readable() { + return None; + } + let header_start = usize::try_from(member.local_offset) + .ok()? + .checked_add(self.shift)?; + let header = self.data.get(header_start..self.central_start)?; + if header.get(..4) != Some(LOCAL_SIGNATURE.as_slice()) { + return None; + } + let data_start = LOCAL_LEN + .checked_add(usize::from(le_u16(header, 26)?))? + .checked_add(usize::from(le_u16(header, 28)?))?; + let data_end = data_start.checked_add(usize::try_from(member.compressed_size).ok()?)?; + let body = header.get(data_start..data_end)?; + if !claimed.claim(header_start, header_start.checked_add(data_end)?) { + return None; + } + Some(if member.method == METHOD_STORED { + MemberData::Stored(body) + } else { + MemberData::Deflated(body) + }) + } +} + +fn locate_central_directory(data: &[u8]) -> Option<(usize, usize, usize)> { + let tail_start = data + .len() + .saturating_sub(EOCD_LEN.saturating_add(MAX_EOCD_COMMENT)); + let tail = data.get(tail_start..)?; + memmem::rfind_iter(tail, EOCD_SIGNATURE) + .find_map(|offset| central_directory_from_eocd(data, tail_start.checked_add(offset)?)) +} + +fn central_directory_from_eocd(data: &[u8], eocd: usize) -> Option<(usize, usize, usize)> { + let record = data.get(eocd..eocd.checked_add(EOCD_LEN)?)?; + let comment_len = usize::from(le_u16(record, 20)?); + if eocd.checked_add(EOCD_LEN)?.checked_add(comment_len)? > data.len() { + return None; + } + let entries = u64::from(le_u16(record, 10)?); + let mut central_size = u64::from(le_u32(record, 12)?); + let mut central_offset = u64::from(le_u32(record, 16)?); + let mut central_end = eocd; + if (entries == U16_MAX || central_size == U32_MAX || central_offset == U32_MAX) + && let Some((zip64_eocd, size, offset)) = zip64_directory(data, eocd) + { + central_end = zip64_eocd; + central_size = size; + central_offset = offset; + } + let central_size = usize::try_from(central_size).ok()?; + let central_start = central_end.checked_sub(central_size)?; + let central_offset = usize::try_from(central_offset).ok()?; + let shift = central_start.checked_sub(central_offset)?; + if data.get(central_start..central_start.checked_add(4)?)? != CENTRAL_SIGNATURE { + return None; + } + Some((central_start, central_end, shift)) +} + +fn zip64_directory(data: &[u8], eocd: usize) -> Option<(usize, u64, u64)> { + let locator_start = eocd.checked_sub(ZIP64_LOCATOR_LEN)?; + let locator = data.get(locator_start..eocd)?; + if locator.get(..4)? != ZIP64_LOCATOR_SIGNATURE { + return None; + } + let record_start = locator_start.checked_sub(ZIP64_EOCD_LEN)?; + let record = data.get(record_start..locator_start)?; + if record.get(..4)? != ZIP64_EOCD_SIGNATURE { + return None; + } + Some((record_start, le_u64(record, 40)?, le_u64(record, 48)?)) +} + +fn zip64_extra(mut extra: &[u8]) -> Option> { + while let [id_low, id_high, len_low, len_high, rest @ ..] = extra { + let len = usize::from(u16::from_le_bytes([*len_low, *len_high])); + let body = rest.get(..len)?; + if u16::from_le_bytes([*id_low, *id_high]) == ZIP64_EXTRA_ID { + return Some( + body.as_chunks::<8>() + .0 + .iter() + .map(|chunk| u64::from_le_bytes(*chunk)), + ); + } + extra = rest.get(len..)?; + } + None +} + +fn entry_name<'a>(data: &'a [u8], entry: &Entry) -> &'a [u8] { + entry + .record + .checked_add(CENTRAL_LEN) + .and_then(|start| data.get(start..start.checked_add(usize::from(entry.name_len))?)) + .unwrap_or_default() +} + +#[inline] +fn normalize(byte: u8) -> u8 { + if byte == b'\\' { + b'/' + } else { + byte.to_ascii_lowercase() + } +} + +fn trim_root(name: &[u8]) -> &[u8] { + match name { + [b'/' | b'\\', rest @ ..] => rest, + _ => name, + } +} + +pub(crate) fn compare_names(left: &[u8], right: &[u8]) -> Ordering { + trim_root(left) + .iter() + .map(|&byte| normalize(byte)) + .cmp(trim_root(right).iter().map(|&byte| normalize(byte))) +} + +fn le_u16(bytes: &[u8], at: usize) -> Option { + bytes + .get(at..at.checked_add(2)?)? + .try_into() + .ok() + .map(u16::from_le_bytes) +} + +fn le_u32(bytes: &[u8], at: usize) -> Option { + bytes + .get(at..at.checked_add(4)?)? + .try_into() + .ok() + .map(u32::from_le_bytes) +} + +fn le_u64(bytes: &[u8], at: usize) -> Option { + bytes + .get(at..at.checked_add(8)?)? + .try_into() + .ok() + .map(u64::from_le_bytes) +} diff --git a/crates/trc/Cargo.toml b/crates/trc/Cargo.toml index 1bb3b838..7af021b8 100644 --- a/crates/trc/Cargo.toml +++ b/crates/trc/Cargo.toml @@ -1,26 +1,26 @@ [package] name = "trc" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] event_macro = { path = "./event-macro" } -mail-auth = { version = "0.13", features = ["arc"] } -mail-parser = { version = "0.11", features = ["full_encoding"] } -base64 = "0.23.1" +mail-auth = { version = "0.14", features = ["arc"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +encodify = "1.0" serde = "1.0" -serde_json = "1.0.151" +serde_json = "1.0.154" reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"]} -rtrb = "0.4.0" -parking_lot = "0.12.5" -tokio = { version = "1.53", features = ["net", "macros"] } +rtrb = "0.4.1" +parking_lot = "0.13.0" +tokio = { version = "1.54", features = ["net", "macros"] } ahash = "0.8.12" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = "0.10.0" -hashify = "0.2.9" +hashify = "0.3" +itoa = "1" [features] -test_mode = [] dev_mode = [] enterprise = [] diff --git a/crates/trc/event-macro/Cargo.toml b/crates/trc/event-macro/Cargo.toml index c3b0c73c..c609b215 100644 --- a/crates/trc/event-macro/Cargo.toml +++ b/crates/trc/event-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "event_macro" -version = "0.16.27" +version = "1.0.0" edition = "2024" [lib] @@ -9,3 +9,6 @@ proc-macro = true [dependencies] syn = { version = "3.0", features = ["full"] } quote = "1.0" + +[lints] +workspace = true diff --git a/crates/trc/event-macro/src/lib.rs b/crates/trc/event-macro/src/lib.rs index 2f032d7c..564bded5 100644 --- a/crates/trc/event-macro/src/lib.rs +++ b/crates/trc/event-macro/src/lib.rs @@ -8,8 +8,6 @@ use proc_macro::TokenStream; use quote::quote; use syn::{Data, DeriveInput, Expr, ExprPath, Ident, Token, parse::Parse, parse_macro_input}; -static mut GLOBAL_ID_COUNTER: usize = 0; - #[proc_macro_attribute] pub fn key_names(_attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as DeriveInput); @@ -75,15 +73,6 @@ pub fn key_names(_attr: TokenStream, item: TokenStream) -> TokenStream { TokenStream::from(expanded) } -#[proc_macro] -pub fn total_event_count(_item: TokenStream) -> TokenStream { - let count = unsafe { GLOBAL_ID_COUNTER }; - let expanded = quote! { - #count - }; - TokenStream::from(expanded) -} - fn to_snake_case(name: &str) -> String { let mut out = String::with_capacity(name.len()); for (idx, ch) in name.char_indices() { @@ -215,6 +204,8 @@ impl IsMetricKey for Ident { | "SpfNone" | "Protocol" | "Code" + | "InputTokens" + | "OutputTokens" ) } } diff --git a/crates/trc/src/atomics/bitset.rs b/crates/trc/src/atomics/bitset.rs index 7934e8fa..10c9d4d2 100644 --- a/crates/trc/src/atomics/bitset.rs +++ b/crates/trc/src/atomics/bitset.rs @@ -74,67 +74,3 @@ impl AtomicBitset { } } -#[cfg(test)] -mod tests { - use super::*; - - const TEST_SIZE: usize = 1000; - type TestBitset = AtomicBitset<{ TEST_SIZE.div_ceil(USIZE_BITS) }>; - static BITSET: TestBitset = TestBitset::new(); - - #[test] - fn test_atomic_bitset() { - for i in 0..TEST_SIZE { - assert!(!BITSET.get(i), "Bit {} should be unset in new BITSET", i); - } - - for i in 0..TEST_SIZE { - assert!(!BITSET.get(i), "Bit {} should be initially unset", i); - BITSET.set(i); - assert!(BITSET.get(i), "Bit {} should be set after setting", i); - } - - BITSET.clear_all(); - - for i in 0..TEST_SIZE { - BITSET.set(i); - assert!(BITSET.get(i), "Bit {} should be set before clearing", i); - BITSET.clear(i); - assert!(!BITSET.get(i), "Bit {} should be unset after clearing", i); - } - - BITSET.clear_all(); - - // Set even bits - for i in (0..TEST_SIZE).step_by(2) { - BITSET.set(i); - } - - // Check all bits - for i in 0..TEST_SIZE { - if i % 2 == 0 { - assert!(BITSET.get(i), "Even bit {} should be set", i); - } else { - assert!(!BITSET.get(i), "Odd bit {} should be unset", i); - } - } - - // Clear even bits and set odd bits - for i in 0..TEST_SIZE { - if i % 2 == 0 { - BITSET.clear(i); - } else { - BITSET.set(i); - } - } - - // Check all bits again - for i in 0..TEST_SIZE { - if i % 2 == 0 { - assert!(!BITSET.get(i), "Even bit {} should now be unset", i); - } else { - assert!(BITSET.get(i), "Odd bit {} should now be set", i); - } - } - } -} diff --git a/crates/trc/src/atomics/histogram.rs b/crates/trc/src/atomics/histogram.rs index b6bc00b5..68f1b69c 100644 --- a/crates/trc/src/atomics/histogram.rs +++ b/crates/trc/src/atomics/histogram.rs @@ -36,17 +36,19 @@ impl AtomicHistogram { pub fn observe(&self, value: u64) { self.sum.fetch_add(value, Ordering::Relaxed); self.count.fetch_add(1, Ordering::Relaxed); - self.min.fetch_min(value, Ordering::Relaxed); - self.max.fetch_max(value, Ordering::Relaxed); - - for (idx, upper_bound) in self.upper_bounds.iter().enumerate() { - if value < *upper_bound { - self.buckets.add(idx, 1); - return; - } + if value < self.min.load(Ordering::Relaxed) { + self.min.fetch_min(value, Ordering::Relaxed); + } + if value > self.max.load(Ordering::Relaxed) { + self.max.fetch_max(value, Ordering::Relaxed); } - unreachable!() + let bucket = self + .upper_bounds + .iter() + .position(|upper_bound| value < *upper_bound) + .unwrap_or(N - 1); + self.buckets.add(bucket, 1); } pub fn id(&self) -> MetricType { @@ -136,6 +138,26 @@ impl AtomicHistogram { ) } + pub const fn new_token_counts(id: MetricType) -> AtomicHistogram<12> { + AtomicHistogram::new( + id, + [ + 10, + 50, + 100, + 250, + 500, + 1_000, + 2_000, + 4_000, + 8_000, + 16_000, + 64_000, + u64::MAX, + ], + ) + } + pub const fn new_short_durations(id: MetricType) -> AtomicHistogram<12> { AtomicHistogram::new( id, diff --git a/crates/trc/src/event/conv.rs b/crates/trc/src/event/conv.rs index adffa8c9..758f941a 100644 --- a/crates/trc/src/event/conv.rs +++ b/crates/trc/src/event/conv.rs @@ -7,9 +7,9 @@ use std::{borrow::Cow, fmt::Debug, str::FromStr, time::Duration}; use compact_str::{CompactString, ToCompactString, format_compact}; -use mail_auth::common::verify::VerifySignature; use crate::*; +use mail_auth::{arc::ArcError, dkim::DkimError, dkim2::Dkim2Error}; impl AsRef for Error { fn as_ref(&self) -> &EventType { @@ -203,10 +203,6 @@ impl EventType { self.reason(err).details("JSON deserialization failed") } - pub fn from_base64_error(self, err: base64::DecodeError) -> Error { - self.reason(err).details("Base64 decoding failed") - } - pub fn from_http_error(self, err: reqwest::Error) -> Error { self.into_err() .ctx_opt( @@ -226,9 +222,7 @@ impl EventType { impl From for Error { fn from(err: mail_auth::Error) -> Self { match err { - mail_auth::Error::ParseError => { - EventType::MailAuth(MailAuthEvent::ParseError).into_err() - } + mail_auth::Error::Parse => EventType::MailAuth(MailAuthEvent::ParseError).into_err(), mail_auth::Error::MissingParameters => { EventType::MailAuth(MailAuthEvent::MissingParameters).into_err() } @@ -243,16 +237,20 @@ impl From for Error { EventType::MailAuth(MailAuthEvent::PolicyNotAligned).into_err() } mail_auth::Error::Crypto(err) => match err { - mail_auth::common::crypto::CryptoError::Library(details) => { + mail_auth::crypto::CryptoError::Library(details) => { EventType::MailAuth(MailAuthEvent::Crypto) .into_err() .details(CompactString::from(details)) } - mail_auth::common::crypto::CryptoError::FailedVerification => { - EventType::Dkim(DkimEvent::FailedVerification).into_err() + mail_auth::crypto::CryptoError::FailedVerification => { + EventType::Dkim(DkimEvent::VerificationFailed) + .into_err() + .details(err.to_compact_string()) } - mail_auth::common::crypto::CryptoError::IncompatibleAlgorithms => { - EventType::Dkim(DkimEvent::IncompatibleAlgorithms).into_err() + mail_auth::crypto::CryptoError::IncompatibleAlgorithms => { + EventType::Dkim(DkimEvent::PublicKeyError) + .into_err() + .details(err.to_compact_string()) } }, mail_auth::Error::Dns(err) => match err { @@ -270,159 +268,79 @@ impl From for Error { EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType).into_err() } }, - mail_auth::Error::Dkim(err) => match err { - mail_auth::dkim::DkimError::UnsupportedVersion => { - EventType::Dkim(DkimEvent::UnsupportedVersion).into_err() + mail_auth::Error::Dkim(err) => EventType::Dkim(match err { + DkimError::UnsupportedVersion + | DkimError::UnsupportedAlgorithm + | DkimError::UnsupportedCanonicalization + | DkimError::AuidMismatch + | DkimError::SignatureExpired + | DkimError::BodyLengthTag => DkimEvent::SignatureInvalid, + DkimError::UnsupportedKeyType | DkimError::PublicKeyRevoked => { + DkimEvent::PublicKeyError } - mail_auth::dkim::DkimError::UnsupportedAlgorithm => { - EventType::Dkim(DkimEvent::UnsupportedAlgorithm).into_err() + DkimError::BodyHashMismatch => DkimEvent::HashMismatch, + }) + .into_err() + .details(err.to_compact_string()), + mail_auth::Error::Arc(err) => EventType::Arc(match err { + ArcError::ChainTooLong + | ArcError::InvalidInstance(_) + | ArcError::InvalidChainValidation + | ArcError::BrokenChain => ArcEvent::ChainInvalid, + ArcError::HasHeaderTag | ArcError::SignatureExpired | ArcError::BodyLengthTag => { + ArcEvent::SignatureInvalid } - mail_auth::dkim::DkimError::UnsupportedCanonicalization => { - EventType::Dkim(DkimEvent::UnsupportedCanonicalization).into_err() + ArcError::BodyHashMismatch => ArcEvent::HashMismatch, + }) + .into_err() + .details(err.to_compact_string()), + mail_auth::Error::Dkim2(err) => EventType::Dkim2(match &err { + Dkim2Error::InstanceMissing(_) + | Dkim2Error::InstanceSyntax(_) + | Dkim2Error::InstanceTagMissing { .. } + | Dkim2Error::InstanceNotSigned(_) + | Dkim2Error::InstanceDuplicateHash(_) + | Dkim2Error::NoValidHash(_) => Dkim2Event::InstanceInvalid, + Dkim2Error::SignatureMissing(_) + | Dkim2Error::SignatureSyntax(_) + | Dkim2Error::SignatureTagMissing { .. } + | Dkim2Error::SignatureTagUnexpected { .. } + | Dkim2Error::DuplicateSelector(_) + | Dkim2Error::TooManySelectors(_) + | Dkim2Error::NoValidAlgorithm(_) => Dkim2Event::SignatureInvalid, + Dkim2Error::SequenceGap(_) + | Dkim2Error::SequenceOverflow + | Dkim2Error::ChainTooLong + | Dkim2Error::NotValidlySigned => Dkim2Event::ChainInvalid, + Dkim2Error::SignatureExpired(_) | Dkim2Error::SignatureFromFuture(_) => { + Dkim2Event::TimestampInvalid } - mail_auth::dkim::DkimError::UnsupportedKeyType => { - EventType::Dkim(DkimEvent::UnsupportedKeyType).into_err() + Dkim2Error::MailFromMismatch { .. } | Dkim2Error::RcptToMismatch { .. } => { + Dkim2Event::EnvelopeMismatch } - mail_auth::dkim::DkimError::FailedBodyHashMatch => { - EventType::Dkim(DkimEvent::FailedBodyHashMatch).into_err() + Dkim2Error::MailFromDomainMismatch(_) + | Dkim2Error::NextDomainMismatch(_) + | Dkim2Error::CustodyMailFromMismatch { .. } + | Dkim2Error::CustodyDomainMismatch { .. } + | Dkim2Error::FromNotAuthentic(_) => Dkim2Event::CustodyBroken, + Dkim2Error::PublicKeyFetch { .. } + | Dkim2Error::PublicKeyMissing { .. } + | Dkim2Error::PublicKeyMultiple { .. } + | Dkim2Error::PublicKeySyntax { .. } + | Dkim2Error::PublicKeyAlgorithmMismatch { .. } + | Dkim2Error::PublicKeyRevoked { .. } => Dkim2Event::PublicKeyError, + Dkim2Error::IncorrectSignature { .. } + | Dkim2Error::PartialSignatureFailure { .. } => Dkim2Event::VerificationFailed, + Dkim2Error::HeaderHashMismatch { .. } | Dkim2Error::BodyHashMismatch { .. } => { + Dkim2Event::HashMismatch } - mail_auth::dkim::DkimError::FailedAuidMatch => { - EventType::Dkim(DkimEvent::FailedAuidMatch).into_err() - } - mail_auth::dkim::DkimError::RevokedPublicKey => { - EventType::Dkim(DkimEvent::RevokedPublicKey).into_err() - } - mail_auth::dkim::DkimError::SignatureExpired => { - EventType::Dkim(DkimEvent::SignatureExpired).into_err() - } - mail_auth::dkim::DkimError::SignatureLength => { - EventType::Dkim(DkimEvent::SignatureLength).into_err() - } - }, - mail_auth::Error::Arc(err) => match err { - mail_auth::arc::ArcError::ChainTooLong => { - EventType::Arc(ArcEvent::ChainTooLong).into_err() - } - mail_auth::arc::ArcError::InvalidInstance(instance) => { - EventType::Arc(ArcEvent::InvalidInstance).ctx(Key::Id, instance) - } - mail_auth::arc::ArcError::InvalidCV => { - EventType::Arc(ArcEvent::InvalidCv).into_err() - } - mail_auth::arc::ArcError::HasHeaderTag => { - EventType::Arc(ArcEvent::HasHeaderTag).into_err() - } - mail_auth::arc::ArcError::BrokenChain => { - EventType::Arc(ArcEvent::BrokenChain).into_err() - } - mail_auth::arc::ArcError::FailedBodyHashMatch => { - EventType::Dkim(DkimEvent::FailedBodyHashMatch).into_err() - } - mail_auth::arc::ArcError::SignatureExpired => { - EventType::Dkim(DkimEvent::SignatureExpired).into_err() - } - mail_auth::arc::ArcError::SignatureLength => { - EventType::Dkim(DkimEvent::SignatureLength).into_err() - } - }, - mail_auth::Error::Dkim2(err) => match err { - mail_auth::dkim2::Dkim2Error::InstanceMissing(m) => { - EventType::Dkim(DkimEvent::InstanceMissing).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::InstanceSyntax(m) => { - EventType::Dkim(DkimEvent::InstanceSyntax).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::InstanceTagMissing { m, tag } => { - EventType::Dkim(DkimEvent::InstanceTagMissing) - .ctx(Key::Id, m) - .details(tag) - } - mail_auth::dkim2::Dkim2Error::InstanceNotSigned(m) => { - EventType::Dkim(DkimEvent::InstanceNotSigned).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::InstanceAboveSignature(m) => { - EventType::Dkim(DkimEvent::InstanceAboveSignature).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::SignatureMissing(i) => { - EventType::Dkim(DkimEvent::SignatureMissing).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::SignatureSyntax(i) => { - EventType::Dkim(DkimEvent::SignatureSyntax).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::SignatureTagMissing { i, tag } => { - EventType::Dkim(DkimEvent::SignatureTagMissing) - .ctx(Key::Id, i) - .details(tag) - } - mail_auth::dkim2::Dkim2Error::SignatureTagUnexpected { i, tag } => { - EventType::Dkim(DkimEvent::SignatureTagUnexpected) - .ctx(Key::Id, i) - .details(tag) - } - mail_auth::dkim2::Dkim2Error::SequenceGap => { - EventType::Dkim(DkimEvent::SequenceGap).into_err() - } - mail_auth::dkim2::Dkim2Error::SequenceOverflow => { - EventType::Dkim(DkimEvent::SequenceOverflow).into_err() - } - mail_auth::dkim2::Dkim2Error::ChainTooLong => { - EventType::Dkim(DkimEvent::ChainTooLong).into_err() - } - mail_auth::dkim2::Dkim2Error::SignatureExpired(i) => { - EventType::Dkim(DkimEvent::SignatureExpired).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::MailFromMismatch(i) => { - EventType::Dkim(DkimEvent::MailFromMismatch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::RcptToMismatch(i) => { - EventType::Dkim(DkimEvent::RcptToMismatch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::MailFromDomainMismatch(i) => { - EventType::Dkim(DkimEvent::MailFromDomainMismatch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::NextDomainMismatch(i) => { - EventType::Dkim(DkimEvent::NextDomainMismatch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::CustodyBreak(i) => { - EventType::Dkim(DkimEvent::CustodyBreak).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeyFetch(i) => { - EventType::Dkim(DkimEvent::PublicKeyFetch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeyMissing(i) => { - EventType::Dkim(DkimEvent::PublicKeyMissing).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeyMultiple(i) => { - EventType::Dkim(DkimEvent::PublicKeyMultiple).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeySyntax(i) => { - EventType::Dkim(DkimEvent::PublicKeySyntax).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeyAlgorithmMismatch(i) => { - EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::PublicKeyRevoked(i) => { - EventType::Dkim(DkimEvent::RevokedPublicKey).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::IncorrectSignature(i) => { - EventType::Dkim(DkimEvent::FailedVerification).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::NoValidAlgorithm(i) => { - EventType::Dkim(DkimEvent::NoValidAlgorithm).ctx(Key::Id, i) - } - mail_auth::dkim2::Dkim2Error::HeaderHashMismatch(m) => { - EventType::Dkim(DkimEvent::HeaderHashMismatch).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::BodyHashMismatch(m) => { - EventType::Dkim(DkimEvent::FailedBodyHashMatch).ctx(Key::Id, m) - } - mail_auth::dkim2::Dkim2Error::Modified => { - EventType::Dkim(DkimEvent::Modified).into_err() - } - mail_auth::dkim2::Dkim2Error::Exploded => { - EventType::Dkim(DkimEvent::Exploded).into_err() - } - }, + Dkim2Error::InvalidJson(_) + | Dkim2Error::RecipeSyntax + | Dkim2Error::HeaderNotRepresentable + | Dkim2Error::BodyUnreconstructable => Dkim2Event::RecipeError, + }) + .into_err() + .details(err.to_compact_string()), } } } @@ -500,7 +418,7 @@ impl From<&mail_auth::DkimOutput<'_>> for Error { fn from(value: &mail_auth::DkimOutput<'_>) -> Self { Error::from(value.result()).ctx_opt( Key::Domain, - value.signature().map(|s| s.domain().to_compact_string()), + value.signature().map(|s| s.d.to_compact_string()), ) } } @@ -596,3 +514,4 @@ impl FromStr for Key { Key::parse(s).ok_or(()) } } + diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index cad2503a..1e9bf958 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -6,20 +6,21 @@ // This file is auto-generated. Do not edit directly. -pub const TOTAL_EVENT_COUNT: usize = 637; -pub const TOTAL_METRIC_COUNT: usize = 369; +pub const TOTAL_EVENT_COUNT: usize = 655; +pub const TOTAL_METRIC_COUNT: usize = 385; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EventType { Acme(AcmeEvent), - Ai(AiEvent), Arc(ArcEvent), Auth(AuthEvent), + Cache(CacheEvent), Calendar(CalendarEvent), Cluster(ClusterEvent), Dane(DaneEvent), Delivery(DeliveryEvent), Dkim(DkimEvent), + Dkim2(Dkim2Event), Dmarc(DmarcEvent), Dns(DnsEvent), Eval(EvalEvent), @@ -42,6 +43,7 @@ pub enum EventType { Queue(QueueEvent), Registry(RegistryEvent), Resource(ResourceEvent), + SearchIndex(SearchIndexEvent), Security(SecurityEvent), Server(ServerEvent), Sieve(SieveEvent), @@ -81,190 +83,180 @@ pub enum AcmeEvent { Error = 15, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum AiEvent { - LlmResponse = 556, - ApiError = 557, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ArcEvent { - ChainTooLong = 28, - InvalidInstance = 31, - InvalidCv = 30, - HasHeaderTag = 29, - BrokenChain = 27, - SealerNotFound = 32, + ChainInvalid = 647, + SignatureInvalid = 648, + HashMismatch = 649, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum AuthEvent { - Success = 37, - Failed = 35, - TokenExpired = 554, - MfaRequired = 36, - TooManyAttempts = 38, - ClientRegistration = 555, - Error = 34, - Warning = 595, - CredentialExpired = 276, + Success = 32, + Failed = 30, + TokenExpired = 537, + MfaRequired = 31, + TooManyAttempts = 33, + ClientRegistration = 538, + Error = 29, + Warning = 578, + CredentialExpired = 260, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum CacheEvent { + Miss = 45, + Hit = 46, + Stale = 47, + Update = 560, + EntryTooLarge = 592, + Invalidate = 600, + SwapHit = 594, + SwapMiss = 595, + SwapWrite = 596, + SwapError = 597, + SwapMetrics = 598, + SwapFlush = 599, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum CalendarEvent { - RuleExpansionError = 576, - AlarmSent = 579, - AlarmSkipped = 580, - AlarmRecipientOverride = 581, - AlarmFailed = 582, - ItipMessageSent = 583, - ItipMessageReceived = 584, - ItipMessageError = 585, + RuleExpansionError = 559, + AlarmSent = 562, + AlarmSkipped = 563, + AlarmRecipientOverride = 564, + AlarmFailed = 565, + ItipMessageSent = 566, + ItipMessageReceived = 567, + ItipMessageError = 568, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ClusterEvent { - Startup = 278, - SubscriberStart = 39, - SubscriberStop = 40, - SubscriberError = 41, - SubscriberDisconnected = 42, - PublisherStart = 43, - PublisherStop = 44, - PublisherError = 45, - MessageReceived = 46, - MessageSkipped = 47, - MessageInvalid = 49, - NodeIdRenewed = 275, + Startup = 262, + SubscriberStart = 34, + SubscriberStop = 35, + SubscriberError = 36, + SubscriberDisconnected = 37, + PublisherStart = 38, + PublisherStop = 39, + PublisherError = 40, + MessageReceived = 41, + MessageSkipped = 42, + MessageInvalid = 44, + NodeIdRenewed = 259, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum DaneEvent { - AuthenticationSuccess = 67, - AuthenticationFailure = 66, - NoCertificatesFound = 69, - CertificateParseError = 68, - TlsaRecordMatch = 73, - TlsaRecordFetch = 70, - TlsaRecordFetchError = 71, - TlsaRecordNotFound = 75, - TlsaRecordNotDnssecSigned = 74, - TlsaRecordInvalid = 72, - BogusDnssecRecord = 605, + AuthenticationSuccess = 62, + AuthenticationFailure = 61, + NoCertificatesFound = 64, + CertificateParseError = 63, + TlsaRecordMatch = 68, + TlsaRecordFetch = 65, + TlsaRecordFetchError = 66, + TlsaRecordNotFound = 70, + TlsaRecordNotDnssecSigned = 69, + TlsaRecordInvalid = 67, + BogusDnssecRecord = 588, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum DeliveryEvent { - AttemptStart = 77, - AttemptEnd = 76, - Completed = 80, - Failed = 92, - DomainDeliveryStart = 85, - MxLookup = 101, - MxLookupFailed = 102, - IpLookup = 95, - IpLookupFailed = 96, - NullMx = 103, - Connect = 82, - ConnectError = 83, - MissingOutboundHostname = 100, - GreetingFailed = 93, - Ehlo = 90, - EhloRejected = 91, - Auth = 78, - AuthFailed = 79, - MailFrom = 97, - MailFromRejected = 98, - Delivered = 84, - RcptTo = 107, - RcptToRejected = 109, - RcptToFailed = 108, - MessageRejected = 99, - StartTls = 110, - StartTlsUnavailable = 113, - StartTlsError = 112, - StartTlsDisabled = 111, - ImplicitTlsError = 94, - ConcurrencyLimitExceeded = 81, - RateLimitExceeded = 104, - DoubleBounce = 86, - DsnSuccess = 88, - DsnTempFail = 89, - DsnPermFail = 87, - RawInput = 105, - RawOutput = 106, + AttemptStart = 72, + AttemptEnd = 71, + Completed = 75, + Failed = 87, + DomainDeliveryStart = 80, + MxLookup = 96, + MxLookupFailed = 97, + IpLookup = 90, + IpLookupFailed = 91, + NullMx = 98, + Connect = 77, + ConnectError = 78, + MissingOutboundHostname = 95, + GreetingFailed = 88, + Ehlo = 85, + EhloRejected = 86, + Auth = 73, + AuthFailed = 74, + MailFrom = 92, + MailFromRejected = 93, + Delivered = 79, + RcptTo = 102, + RcptToRejected = 104, + RcptToFailed = 103, + MessageRejected = 94, + StartTls = 105, + StartTlsUnavailable = 108, + StartTlsError = 107, + StartTlsDisabled = 106, + ImplicitTlsError = 89, + ConcurrencyLimitExceeded = 76, + RateLimitExceeded = 99, + DoubleBounce = 81, + DsnSuccess = 83, + DsnTempFail = 84, + DsnPermFail = 82, + RawInput = 100, + RawOutput = 101, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum DkimEvent { - Pass = 121, - Neutral = 119, - Fail = 114, - PermError = 122, - TempError = 127, - None = 120, - UnsupportedVersion = 131, - UnsupportedAlgorithm = 128, - UnsupportedCanonicalization = 129, - UnsupportedKeyType = 130, - FailedBodyHashMatch = 116, - FailedVerification = 117, - FailedAuidMatch = 115, - RevokedPublicKey = 123, - IncompatibleAlgorithms = 118, - SignatureExpired = 124, - SignatureLength = 125, - SignerNotFound = 126, - BuildError = 592, - SignatureCreated = 596, - SignaturePublished = 597, - SignatureRetiring = 598, - SignatureRetired = 599, - SignatureDeleted = 600, - InstanceMissing = 606, - InstanceSyntax = 607, - InstanceTagMissing = 608, - InstanceNotSigned = 609, - InstanceAboveSignature = 610, - SignatureMissing = 611, - SignatureSyntax = 612, - SignatureTagMissing = 613, - SignatureTagUnexpected = 614, - SequenceGap = 615, - SequenceOverflow = 616, - ChainTooLong = 635, - MailFromMismatch = 617, - RcptToMismatch = 618, - MailFromDomainMismatch = 619, - NextDomainMismatch = 620, - CustodyBreak = 636, - PublicKeyFetch = 621, - PublicKeyMissing = 622, - PublicKeyMultiple = 623, - PublicKeySyntax = 624, - PublicKeyAlgorithmMismatch = 625, - NoValidAlgorithm = 626, - HeaderHashMismatch = 627, - Modified = 628, - Exploded = 629, + Pass = 112, + Neutral = 110, + Fail = 109, + PermError = 113, + TempError = 115, + None = 111, + SignatureInvalid = 643, + PublicKeyError = 644, + VerificationFailed = 645, + HashMismatch = 646, + SignerNotFound = 114, + BuildError = 575, + SignatureCreated = 579, + SignaturePublished = 580, + SignatureRetiring = 581, + SignatureRetired = 582, + SignatureDeleted = 583, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum Dkim2Event { + InstanceInvalid = 633, + SignatureInvalid = 634, + ChainInvalid = 635, + TimestampInvalid = 636, + EnvelopeMismatch = 637, + CustodyBroken = 638, + PublicKeyError = 639, + VerificationFailed = 640, + HashMismatch = 641, + RecipeError = 642, + SigningFailed = 650, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum DmarcEvent { - Pass = 134, - Fail = 132, - PermError = 135, - TempError = 136, - None = 133, + Pass = 118, + Fail = 116, + PermError = 119, + TempError = 120, + None = 117, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -277,635 +269,650 @@ pub enum DnsEvent { RecordLookupFailed = 11, RecordPropagated = 13, RecordPropagationTimeout = 14, - BuildError = 591, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum EvalEvent { - Result = 139, - Error = 138, - DirectoryNotFound = 137, - StoreNotFound = 140, + Result = 123, + Error = 122, + DirectoryNotFound = 121, + StoreNotFound = 124, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum HttpEvent { - ConnectionStart = 153, - ConnectionEnd = 152, - Error = 154, - RequestUrl = 156, - RequestBody = 155, - ResponseBody = 157, - XForwardedMissing = 158, + ConnectionStart = 137, + ConnectionEnd = 136, + Error = 138, + RequestUrl = 140, + RequestBody = 139, + ResponseBody = 141, + XForwardedMissing = 142, + XForwardedInvalid = 625, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ImapEvent { - ConnectionStart = 163, - ConnectionEnd = 162, - GetAcl = 171, - SetAcl = 188, - MyRights = 180, - ListRights = 176, - Append = 159, - Capabilities = 160, - Id = 172, - Close = 161, - Copy = 164, - Move = 179, - CreateMailbox = 165, - DeleteMailbox = 166, - RenameMailbox = 185, - Enable = 167, - Expunge = 169, - Fetch = 170, - IdleStart = 173, - IdleStop = 174, - List = 175, - Lsub = 178, - Logout = 177, - Namespace = 181, - Noop = 182, - Search = 186, - Sort = 189, - Select = 187, - Status = 190, - Store = 191, - Subscribe = 192, - Unsubscribe = 194, - Thread = 193, - UidBatches = 634, - GetQuota = 57, - Error = 168, - RawInput = 183, - RawOutput = 184, + ConnectionStart = 147, + ConnectionEnd = 146, + GetAcl = 155, + SetAcl = 172, + MyRights = 164, + ListRights = 160, + Append = 143, + Capabilities = 144, + Id = 156, + Close = 145, + Copy = 148, + Move = 163, + CreateMailbox = 149, + DeleteMailbox = 150, + RenameMailbox = 169, + Enable = 151, + Expunge = 153, + Fetch = 154, + IdleStart = 157, + IdleStop = 158, + List = 159, + Lsub = 162, + Logout = 161, + Namespace = 165, + Noop = 166, + Search = 170, + Sort = 173, + Select = 171, + Status = 174, + Store = 175, + Subscribe = 176, + Unsubscribe = 178, + Thread = 177, + UidBatches = 593, + GetQuota = 52, + GetMetadata = 607, + SetMetadata = 608, + Error = 152, + RawInput = 167, + RawOutput = 168, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum IncomingReportEvent { - DmarcReport = 200, - DmarcReportWithWarnings = 201, - TlsReport = 206, - TlsReportWithWarnings = 207, - AbuseReport = 195, - AuthFailureReport = 197, - FraudReport = 202, - NotSpamReport = 204, - VirusReport = 209, - OtherReport = 205, - MessageParseFailed = 203, - DmarcParseFailed = 199, - TlsRpcParseFailed = 208, - ArfParseFailed = 196, - DecompressError = 198, + DmarcReport = 184, + DmarcReportWithWarnings = 185, + TlsReport = 190, + TlsReportWithWarnings = 191, + AbuseReport = 179, + AuthFailureReport = 181, + FraudReport = 186, + NotSpamReport = 188, + VirusReport = 193, + OtherReport = 189, + MessageParseFailed = 187, + DmarcParseFailed = 183, + TlsRpcParseFailed = 192, + ArfParseFailed = 180, + DecompressError = 182, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum IprevEvent { - Pass = 212, - Fail = 210, - PermError = 213, - TempError = 214, - None = 211, + Pass = 196, + Fail = 194, + PermError = 197, + TempError = 198, + None = 195, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum JmapEvent { - MethodCall = 223, - InvalidArguments = 221, - RequestTooLarge = 227, - StateMismatch = 228, - AnchorNotFound = 218, - UnsupportedFilter = 232, - UnsupportedSort = 233, - UnknownMethod = 231, - InvalidResultReference = 222, - Forbidden = 220, - AccountNotFound = 215, - AccountNotSupportedByMethod = 216, - AccountReadOnly = 217, - NotFound = 224, - CannotCalculateChanges = 219, - UnknownDataType = 230, - UnknownCapability = 229, - NotJson = 225, - NotRequest = 226, - WebsocketStart = 235, - WebsocketStop = 236, - WebsocketError = 234, + MethodCall = 207, + InvalidArguments = 205, + RequestTooLarge = 211, + StateMismatch = 212, + AnchorNotFound = 202, + UnsupportedFilter = 216, + UnsupportedSort = 217, + UnknownMethod = 215, + InvalidResultReference = 206, + Forbidden = 204, + AccountNotFound = 199, + AccountNotSupportedByMethod = 200, + AccountReadOnly = 201, + FromAccountNotFound = 653, + FromAccountNotSupportedByMethod = 654, + NotFound = 208, + CannotCalculateChanges = 203, + UnknownDataType = 214, + UnknownCapability = 213, + NotJson = 209, + NotRequest = 210, + WebsocketStart = 219, + WebsocketStop = 220, + WebsocketError = 218, + ExpandDurationTooLarge = 602, + CannotCalculateOccurrences = 603, + TooLarge = 604, + RateLimit = 605, + ObjectNotFound = 606, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum LimitEvent { - SizeRequest = 243, - SizeUpload = 244, - CallsIn = 238, - ConcurrentRequest = 240, - ConcurrentUpload = 241, - ConcurrentConnection = 239, - Quota = 242, - BlobQuota = 237, - TenantQuota = 553, - TooManyRequests = 245, + SizeRequest = 227, + SizeUpload = 228, + CallsIn = 222, + ConcurrentRequest = 224, + ConcurrentUpload = 225, + ConcurrentConnection = 223, + Quota = 226, + BlobQuota = 221, + TooManyRequests = 229, + MemoryBudget = 627, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MailAuthEvent { - ParseError = 254, - MissingParameters = 252, - NoHeadersFound = 253, - Crypto = 247, - Io = 251, - Base64 = 246, - DnsError = 248, - DnsRecordNotFound = 250, - DnsInvalidRecordType = 249, - PolicyNotAligned = 255, + ParseError = 238, + MissingParameters = 236, + NoHeadersFound = 237, + Crypto = 231, + Io = 235, + Base64 = 230, + DnsError = 232, + DnsRecordNotFound = 234, + DnsInvalidRecordType = 233, + PolicyNotAligned = 239, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ManageSieveEvent { - ConnectionStart = 259, - ConnectionEnd = 258, - CreateScript = 260, - UpdateScript = 274, - GetScript = 263, - DeleteScript = 261, - RenameScript = 270, - CheckScript = 257, - HaveSpace = 264, - ListScripts = 265, - SetActive = 271, - Capabilities = 256, - StartTls = 272, - Unauthenticate = 273, - Logout = 266, - Noop = 267, - Error = 262, - RawInput = 268, - RawOutput = 269, + ConnectionStart = 243, + ConnectionEnd = 242, + CreateScript = 244, + UpdateScript = 258, + GetScript = 247, + DeleteScript = 245, + RenameScript = 254, + CheckScript = 241, + HaveSpace = 248, + ListScripts = 249, + SetActive = 255, + Capabilities = 240, + StartTls = 256, + Unauthenticate = 257, + Logout = 250, + Noop = 251, + Error = 246, + RawInput = 252, + RawOutput = 253, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MessageIngestEvent { - Ham = 283, - Spam = 286, - ImapAppend = 284, - JmapAppend = 285, - Duplicate = 281, - Error = 282, - SearchIndex = 142, + Ham = 267, + Spam = 270, + ImapAppend = 268, + JmapAppend = 269, + Duplicate = 265, + Error = 266, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MilterEvent { - Read = 299, - Write = 303, - ActionAccept = 287, - ActionDiscard = 289, - ActionReject = 290, - ActionTempFail = 293, - ActionReplyCode = 291, - ActionConnectionFailure = 288, - ActionShutdown = 292, - IoError = 297, - FrameTooLarge = 296, - FrameInvalid = 295, - UnexpectedResponse = 302, - Timeout = 300, - TlsInvalidName = 301, - Disconnected = 294, - ParseError = 298, + Read = 283, + Write = 287, + ActionAccept = 271, + ActionDiscard = 273, + ActionReject = 274, + ActionTempFail = 277, + ActionReplyCode = 275, + ActionConnectionFailure = 272, + ActionShutdown = 276, + IoError = 281, + FrameTooLarge = 280, + FrameInvalid = 279, + UnexpectedResponse = 286, + Timeout = 284, + TlsInvalidName = 285, + Disconnected = 278, + ParseError = 282, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MtaHookEvent { - ActionAccept = 304, - ActionDiscard = 305, - ActionReject = 307, - ActionQuarantine = 306, - Error = 308, + ActionAccept = 288, + ActionDiscard = 289, + ActionReject = 291, + ActionQuarantine = 290, + Error = 292, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum MtaStsEvent { - Authorized = 309, - NotAuthorized = 311, - PolicyFetch = 312, - PolicyNotFound = 314, - PolicyFetchError = 313, - InvalidPolicy = 310, + Authorized = 293, + NotAuthorized = 295, + PolicyFetch = 296, + PolicyNotFound = 298, + PolicyFetchError = 297, + InvalidPolicy = 294, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum NetworkEvent { - ListenStart = 321, - ListenStop = 322, - ListenError = 320, - BindError = 316, - ReadError = 324, - WriteError = 328, - FlushError = 319, - AcceptError = 315, - SplitError = 326, - Timeout = 327, - Closed = 317, - ProxyError = 323, - SetOptError = 325, + ListenStart = 305, + ListenStop = 306, + ListenError = 304, + BindError = 300, + ReadError = 308, + WriteError = 312, + FlushError = 303, + AcceptError = 299, + SplitError = 310, + Timeout = 311, + Closed = 301, + ProxyError = 307, + SetOptError = 309, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum OutgoingReportEvent { - SpfReport = 342, - SpfRateLimited = 341, - DkimReport = 330, - DkimRateLimited = 329, - DmarcReport = 333, - DmarcRateLimited = 332, - DmarcAggregateReport = 331, - TlsAggregate = 344, - HttpSubmission = 334, - UnauthorizedReportingAddress = 345, - ReportingAddressValidationError = 340, - NotFound = 339, - SubmissionError = 343, - NoRecipientsFound = 338, - Locked = 337, - MaxSizeExceeded = 59, + SpfReport = 326, + SpfRateLimited = 325, + DkimReport = 314, + DkimRateLimited = 313, + DmarcReport = 317, + DmarcRateLimited = 316, + DmarcAggregateReport = 315, + TlsAggregate = 328, + HttpSubmission = 318, + UnauthorizedReportingAddress = 329, + ReportingAddressValidationError = 324, + NotFound = 323, + SubmissionError = 327, + NoRecipientsFound = 322, + Locked = 321, + MaxSizeExceeded = 54, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum Pop3Event { - ConnectionStart = 348, - ConnectionEnd = 347, - Delete = 349, - Reset = 358, - Quit = 355, - Fetch = 351, - List = 352, - ListMessage = 353, - Uidl = 361, - UidlMessage = 362, - Stat = 360, - Noop = 354, - Capabilities = 346, - StartTls = 359, - Utf8 = 363, - Error = 350, - RawInput = 356, - RawOutput = 357, + ConnectionStart = 332, + ConnectionEnd = 331, + Delete = 333, + Reset = 342, + Quit = 339, + Fetch = 335, + List = 336, + ListMessage = 337, + Uidl = 345, + UidlMessage = 346, + Stat = 344, + Noop = 338, + Capabilities = 330, + StartTls = 343, + Utf8 = 347, + Error = 334, + RawInput = 340, + RawOutput = 341, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum PushSubscriptionEvent { - Success = 373, - Error = 371, - NotFound = 372, + Success = 357, + Error = 355, + NotFound = 356, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum QueueEvent { - Started = 390, - MessageQueued = 380, - AuthenticatedMessageQueued = 381, - ReportQueued = 382, - DsnQueued = 379, - AutogeneratedQueued = 378, - Rescheduled = 385, - Locked = 377, - BlobNotFound = 374, - RateLimitExceeded = 384, - ConcurrencyLimitExceeded = 375, - QuotaExceeded = 383, - BackPressure = 48, + Started = 374, + MessageQueued = 364, + AuthenticatedMessageQueued = 365, + ReportQueued = 366, + DsnQueued = 363, + AutogeneratedQueued = 362, + Rescheduled = 369, + Locked = 361, + BlobNotFound = 358, + RateLimitExceeded = 368, + ConcurrencyLimitExceeded = 359, + QuotaExceeded = 367, + BackPressure = 43, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum RegistryEvent { - LocalReadError = 62, - LocalWriteError = 54, - LocalParseError = 60, - ReadError = 65, - WriteError = 58, - DeserializationError = 56, - BuildError = 61, - BuildWarning = 55, - NotSupported = 64, - ValidationError = 63, + LocalReadError = 57, + LocalWriteError = 49, + LocalParseError = 55, + ReadError = 60, + WriteError = 53, + DeserializationError = 51, + BuildError = 56, + BuildWarning = 50, + NotSupported = 59, + ValidationError = 58, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ResourceEvent { - NotFound = 389, - BadParameters = 386, - Error = 388, - DownloadExternal = 387, - ApplicationUpdated = 601, - ApplicationUnpacked = 602, + NotFound = 373, + BadParameters = 370, + Error = 372, + DownloadExternal = 371, + ApplicationUpdated = 584, + ApplicationUnpacked = 585, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum SearchIndexEvent { + ManagerStarted = 629, + PartitionsAcquired = 630, + PartitionLocked = 631, + PartitionRetry = 632, + BatchIndexed = 126, + DocumentIgnored = 574, + DocumentTruncated = 628, + MetadataNotFound = 27, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SecurityEvent { - AuthenticationBan = 33, - AbuseBan = 549, - ScanBan = 558, - LoiterBan = 550, - IpBlocked = 318, - IpBlockExpired = 593, - IpAllowExpired = 594, - IpUnauthorized = 279, - Unauthorized = 552, + AuthenticationBan = 28, + AbuseBan = 532, + ScanBan = 541, + LoiterBan = 533, + IpBlocked = 302, + IpBlockExpired = 576, + IpAllowExpired = 577, + IpUnauthorized = 263, + Unauthorized = 535, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ServerEvent { - Startup = 393, - Shutdown = 392, - StartupError = 394, - ThreadError = 395, - Licensing = 391, - RecoveryMode = 603, - BootstrapMode = 604, + Startup = 377, + Shutdown = 376, + StartupError = 378, + ThreadError = 379, + RecoveryMode = 586, + BootstrapMode = 587, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SieveEvent { - ActionAccept = 396, - ActionAcceptReplace = 397, - ActionDiscard = 398, - ActionReject = 399, - SendMessage = 406, - MessageTooLarge = 401, - ScriptNotFound = 405, - ListNotFound = 400, - RuntimeError = 404, - UnexpectedError = 407, - NotSupported = 402, - QuotaExceeded = 403, + ActionAccept = 380, + ActionAcceptReplace = 381, + ActionDiscard = 382, + ActionReject = 383, + SendMessage = 390, + MessageTooLarge = 385, + ScriptNotFound = 389, + ListNotFound = 384, + RuntimeError = 388, + UnexpectedError = 391, + NotSupported = 386, + QuotaExceeded = 387, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SmtpEvent { - ConnectionStart = 417, - ConnectionEnd = 416, - Error = 428, - IdNotFound = 469, - ConcurrencyLimitExceeded = 415, - TransferLimitExceeded = 485, - RateLimitExceeded = 461, - TimeLimitExceeded = 481, - MissingAuthDirectory = 452, - MessageParseFailed = 450, - MessageTooLarge = 451, - LoopDetected = 443, - DkimPass = 422, - DkimFail = 421, - Dkim2Pass = 630, - Dkim2Fail = 631, - Dkim2DsnDiscarded = 632, - ArcPass = 410, - ArcFail = 409, - SpfEhloPass = 474, - SpfEhloFail = 473, - SpfFromPass = 476, - SpfFromFail = 475, - DmarcPass = 424, - DmarcFail = 423, - IprevPass = 441, - IprevFail = 440, - TooManyMessages = 483, - Ehlo = 426, - InvalidEhlo = 436, - DidNotSayEhlo = 420, - EhloExpected = 427, - LhloExpected = 442, - MailFromUnauthenticated = 447, - MailFromUnauthorized = 448, - MailFromNotAllowed = 551, - MailFromRewritten = 446, - MailFromMissing = 445, - MailFrom = 444, - MultipleMailFrom = 456, - MailboxDoesNotExist = 449, - RelayNotAllowed = 468, - RcptTo = 464, - RcptToDuplicate = 465, - RcptToRewritten = 467, - RcptToMissing = 466, - RcptToGreylisted = 561, - TooManyRecipients = 484, - TooManyInvalidRcpt = 482, - RawInput = 462, - RawOutput = 463, - MissingLocalHostname = 453, - Vrfy = 487, - VrfyNotFound = 489, - VrfyDisabled = 488, - Expn = 429, - ExpnNotFound = 431, - ExpnDisabled = 430, - RequireTlsDisabled = 471, - DeliverByDisabled = 418, - DeliverByInvalid = 419, - FutureReleaseDisabled = 432, - FutureReleaseInvalid = 433, - MtPriorityDisabled = 454, - MtPriorityInvalid = 455, - DsnDisabled = 425, - AuthNotAllowed = 413, - AuthMechanismNotSupported = 412, - AuthExchangeTooLong = 411, - AlreadyAuthenticated = 408, - Noop = 457, - StartTls = 477, - StartTlsUnavailable = 479, - StartTlsAlready = 478, - Rset = 472, - Quit = 460, - Help = 434, - CommandNotImplemented = 414, - InvalidCommand = 435, - InvalidSenderAddress = 439, - InvalidRecipientAddress = 438, - InvalidParameter = 437, - UnsupportedParameter = 486, - SyntaxError = 480, - RequestTooLarge = 470, + ConnectionStart = 401, + ConnectionEnd = 400, + Error = 412, + IdNotFound = 452, + ConcurrencyLimitExceeded = 399, + TransferLimitExceeded = 468, + RateLimitExceeded = 444, + TimeLimitExceeded = 464, + MessageParseFailed = 434, + MessageTooLarge = 435, + LoopDetected = 427, + MultipleFromDomains = 626, + DkimPass = 406, + DkimFail = 405, + Dkim2Pass = 589, + Dkim2Fail = 590, + Dkim2DsnRejected = 591, + ArcPass = 394, + ArcFail = 393, + SpfEhloPass = 457, + SpfEhloFail = 456, + SpfFromPass = 459, + SpfFromFail = 458, + DmarcPass = 408, + DmarcFail = 407, + IprevPass = 425, + IprevFail = 424, + TooManyMessages = 466, + Ehlo = 410, + InvalidEhlo = 420, + DidNotSayEhlo = 404, + EhloExpected = 411, + LhloExpected = 426, + MailFromUnauthenticated = 431, + MailFromUnauthorized = 432, + MailFromNotAllowed = 534, + MailFromRewritten = 430, + MailFromMissing = 429, + MailFrom = 428, + MultipleMailFrom = 439, + MailboxDoesNotExist = 433, + RelayNotAllowed = 451, + RcptTo = 447, + RcptToDuplicate = 448, + RcptToRewritten = 450, + RcptToMissing = 449, + RcptToGreylisted = 544, + TooManyRecipients = 467, + TooManyInvalidRcpt = 465, + RawInput = 445, + RawOutput = 446, + MissingLocalHostname = 436, + Vrfy = 470, + VrfyNotFound = 472, + VrfyDisabled = 471, + Expn = 413, + ExpnNotFound = 415, + ExpnDisabled = 414, + RequireTlsDisabled = 454, + DeliverByDisabled = 402, + DeliverByInvalid = 403, + FutureReleaseDisabled = 416, + FutureReleaseInvalid = 417, + MtPriorityDisabled = 437, + MtPriorityInvalid = 438, + DsnDisabled = 409, + AuthNotAllowed = 397, + AuthMechanismNotSupported = 396, + AuthExchangeTooLong = 395, + AlreadyAuthenticated = 392, + Noop = 440, + StartTls = 460, + StartTlsUnavailable = 462, + StartTlsAlready = 461, + Rset = 455, + Quit = 443, + Help = 418, + CommandNotImplemented = 398, + InvalidCommand = 419, + InvalidSenderAddress = 423, + InvalidRecipientAddress = 422, + InvalidParameter = 421, + UnsupportedParameter = 469, + SyntaxError = 463, + RequestTooLarge = 453, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SpamEvent { - Pyzor = 564, - PyzorError = 494, - Dnsbl = 562, - DnsblError = 563, - TrainStarted = 588, - TrainCompleted = 495, - TrainSampleAdded = 143, - TrainSampleNotFound = 491, - Classify = 490, - ModelLoaded = 589, - ModelNotReady = 496, - ModelNotFound = 497, - RulesUpdated = 280, + Pyzor = 547, + PyzorError = 477, + Dnsbl = 545, + DnsblError = 546, + TrainStarted = 571, + TrainCompleted = 478, + TrainSampleAdded = 127, + TrainSampleNotFound = 474, + Classify = 473, + ModelLoaded = 572, + ModelNotReady = 479, + ModelNotFound = 480, + RulesUpdated = 264, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum SpfEvent { - Pass = 501, - Fail = 498, - SoftFail = 503, - Neutral = 499, - TempError = 504, - PermError = 502, - None = 500, + Pass = 484, + Fail = 481, + SoftFail = 486, + Neutral = 482, + TempError = 487, + PermError = 485, + None = 483, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum StoreEvent { - AssertValueFailed = 505, - FoundationdbError = 518, - MysqlError = 522, - PostgresqlError = 527, - RocksdbError = 529, - SqliteError = 532, - LdapError = 520, - ElasticsearchError = 516, - MeilisearchError = 590, - RedisError = 528, - S3Error = 530, - AzureError = 559, - FilesystemError = 517, - PoolError = 526, - DataCorruption = 511, - DecompressError = 514, - DeserializeError = 515, - NotFound = 524, - NotConfigured = 523, - NotSupported = 525, - UnexpectedError = 533, - CryptoError = 510, - HttpStoreError = 493, - CacheMiss = 50, - CacheHit = 51, - CacheStale = 52, - CacheUpdate = 577, - CacheEntryTooLarge = 633, - BlobMissingMarker = 507, - DataWrite = 513, - DataIterate = 512, - BlobRead = 508, - BlobWrite = 509, - BlobDelete = 506, - SqlQuery = 531, - LdapQuery = 521, - LdapWarning = 519, - HttpStoreFetch = 492, - AutoExpunge = 364, - BlobStorePurged = 369, - DataStorePurged = 368, + AssertValueFailed = 488, + FoundationdbError = 501, + MysqlError = 505, + PostgresqlError = 510, + RocksdbError = 512, + SqliteError = 515, + LdapError = 503, + ElasticsearchError = 499, + MeilisearchError = 573, + RedisError = 511, + S3Error = 513, + AzureError = 542, + FilesystemError = 500, + PoolError = 509, + DataCorruption = 494, + DecompressError = 497, + DeserializeError = 498, + NotFound = 507, + NotConfigured = 506, + NotSupported = 508, + UnexpectedError = 516, + CryptoError = 493, + HttpStoreError = 476, + BlobMissingMarker = 490, + DataWrite = 496, + DataIterate = 495, + BlobRead = 491, + BlobWrite = 492, + BlobDelete = 489, + SqlQuery = 514, + LdapQuery = 504, + LdapWarning = 502, + HttpStoreFetch = 475, + AutoExpunge = 348, + BlobStorePurged = 353, + DataStorePurged = 352, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TaskManagerEvent { - TaskAcquired = 578, - TaskQueued = 149, - TaskScheduled = 370, - TaskLocked = 144, - TaskIgnored = 586, - TaskFailed = 587, - TaskRetry = 53, - BlobNotFound = 141, - MetadataNotFound = 145, - SchedulerStarted = 150, - ManagerStarted = 367, + TaskAcquired = 561, + TaskQueued = 133, + TaskScheduled = 354, + TaskLocked = 128, + TaskIgnored = 569, + TaskFailed = 570, + TaskRetry = 48, + TaskStalled = 651, + TaskExpired = 652, + BlobNotFound = 125, + MetadataNotFound = 129, + SchedulerStarted = 134, + ManagerStarted = 351, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TelemetryEvent { - AlertEvent = 548, - AlertMessage = 365, - LogError = 535, - WebhookError = 539, - OtelExporterError = 536, - OtelMetricsExporterError = 537, - PrometheusExporterError = 538, - JournalError = 534, - MetricsCollected = 151, - MetricsStored = 366, - MetricsPushed = 146, + LogError = 518, + WebhookError = 522, + OtelExporterError = 519, + OtelMetricsExporterError = 520, + PrometheusExporterError = 521, + JournalError = 517, + EventsDropped = 601, + MetricsCollected = 135, + MetricsPushed = 130, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TlsEvent { - Handshake = 543, - HandshakeError = 544, - NotConfigured = 547, - CertificateNotFound = 542, - NoCertificatesAvailable = 546, - MultipleCertificatesAvailable = 545, - ExpiredCertificateRemoved = 277, + Handshake = 526, + HandshakeError = 527, + NotConfigured = 530, + CertificateNotFound = 525, + NoCertificatesAvailable = 529, + MultipleCertificatesAvailable = 528, + ExpiredCertificateRemoved = 261, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TlsRptEvent { - RecordFetch = 540, - RecordFetchError = 541, - RecordNotFound = 560, + RecordFetch = 523, + RecordFetchError = 524, + RecordNotFound = 543, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum WebDavEvent { - Propfind = 147, - Proppatch = 148, - Get = 335, - Head = 574, - Report = 336, - Mkcol = 376, - Mkcalendar = 575, - Delete = 458, - Put = 459, - Post = 565, - Patch = 566, - Copy = 567, - Move = 568, - Lock = 569, - Unlock = 570, - Acl = 571, - Options = 573, - Error = 572, + Propfind = 131, + Proppatch = 132, + Get = 319, + Head = 557, + Report = 320, + Mkcol = 360, + Mkcalendar = 558, + Delete = 441, + Put = 442, + Post = 548, + Patch = 549, + Copy = 550, + Move = 551, + Lock = 552, + Unlock = 553, + Acl = 554, + Options = 556, + Error = 555, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -919,366 +926,370 @@ pub enum MetricType { AcmeTlsAlpnError = 32, AcmeTokenNotFound = 33, AcmeError = 34, - ArcChainTooLong = 39, - ArcInvalidInstance = 40, - ArcInvalidCv = 41, - ArcHasHeaderTag = 42, - ArcBrokenChain = 43, - AuthSuccess = 44, - AuthFailed = 45, - AuthTooManyAttempts = 46, - AuthError = 47, - CalendarAlarmSent = 48, - CalendarAlarmFailed = 49, - CalendarItipMessageSent = 50, - CalendarItipMessageReceived = 51, - CalendarItipMessageError = 52, - ClusterSubscriberError = 53, - ClusterSubscriberDisconnected = 54, - ClusterPublisherError = 55, - DaneAuthenticationSuccess = 56, - DaneAuthenticationFailure = 57, - DaneNoCertificatesFound = 58, - DaneCertificateParseError = 59, - DaneTlsaRecordFetchError = 60, - DaneTlsaRecordNotFound = 61, - DaneTlsaRecordNotDnssecSigned = 62, - DaneTlsaRecordInvalid = 63, - DaneBogusDnssecRecord = 339, + ArcChainInvalid = 379, + ArcSignatureInvalid = 380, + ArcHashMismatch = 381, + AuthSuccess = 39, + AuthFailed = 40, + AuthTooManyAttempts = 41, + AuthError = 42, + CacheMessagesSize = 348, + CacheMessagesLargeSize = 349, + CacheSize = 350, + CacheSwapPending = 351, + CacheMiss = 339, + CacheHit = 340, + CacheStale = 341, + CacheEntryTooLarge = 342, + CacheSwapHit = 343, + CacheSwapMiss = 344, + CacheSwapWrite = 345, + CacheSwapError = 346, + CalendarAlarmSent = 43, + CalendarAlarmFailed = 44, + CalendarItipMessageSent = 45, + CalendarItipMessageReceived = 46, + CalendarItipMessageError = 47, + ClusterSubscriberError = 48, + ClusterSubscriberDisconnected = 49, + ClusterPublisherError = 50, + DaneAuthenticationSuccess = 51, + DaneAuthenticationFailure = 52, + DaneNoCertificatesFound = 53, + DaneCertificateParseError = 54, + DaneTlsaRecordFetchError = 55, + DaneTlsaRecordNotFound = 56, + DaneTlsaRecordNotDnssecSigned = 57, + DaneTlsaRecordInvalid = 58, + DaneBogusDnssecRecord = 323, DeliveryTotalTime = 2, DeliveryAttemptTime = 3, DeliveryActiveConnections = 22, - DeliveryAttemptStart = 64, - DeliveryAttemptEnd = 65, - DeliveryCompleted = 66, - DeliveryMxLookupFailed = 67, - DeliveryIpLookupFailed = 68, - DeliveryNullMx = 69, - DeliveryGreetingFailed = 70, - DeliveryEhloRejected = 71, - DeliveryAuthFailed = 72, - DeliveryMailFromRejected = 73, - DeliveryDelivered = 74, - DeliveryRcptToRejected = 75, - DeliveryRcptToFailed = 76, - DeliveryMessageRejected = 77, - DeliveryStartTlsUnavailable = 78, - DeliveryStartTlsError = 79, - DeliveryStartTlsDisabled = 80, - DeliveryImplicitTlsError = 81, - DeliveryConcurrencyLimitExceeded = 82, - DeliveryRateLimitExceeded = 83, - DeliveryDoubleBounce = 84, - DeliveryDsnSuccess = 85, - DeliveryDsnTempFail = 86, - DeliveryDsnPermFail = 87, - DkimPass = 88, - DkimNeutral = 89, - DkimFail = 90, - DkimPermError = 91, - DkimTempError = 92, - DkimNone = 93, - DkimUnsupportedVersion = 94, - DkimUnsupportedAlgorithm = 95, - DkimUnsupportedCanonicalization = 96, - DkimUnsupportedKeyType = 97, - DkimFailedBodyHashMatch = 98, - DkimFailedVerification = 99, - DkimFailedAuidMatch = 100, - DkimRevokedPublicKey = 101, - DkimIncompatibleAlgorithms = 102, - DkimSignatureExpired = 103, - DkimSignatureLength = 104, - DkimSignerNotFound = 105, - DkimInstanceMissing = 340, - DkimInstanceSyntax = 341, - DkimInstanceTagMissing = 342, - DkimInstanceNotSigned = 343, - DkimInstanceAboveSignature = 344, - DkimSignatureMissing = 345, - DkimSignatureSyntax = 346, - DkimSignatureTagMissing = 347, - DkimSignatureTagUnexpected = 348, - DkimSequenceGap = 349, - DkimSequenceOverflow = 350, - DkimChainTooLong = 367, - DkimMailFromMismatch = 351, - DkimRcptToMismatch = 352, - DkimMailFromDomainMismatch = 353, - DkimNextDomainMismatch = 354, - DkimCustodyBreak = 368, - DkimPublicKeyFetch = 355, - DkimPublicKeyMissing = 356, - DkimPublicKeyMultiple = 357, - DkimPublicKeySyntax = 358, - DkimPublicKeyAlgorithmMismatch = 359, - DkimNoValidAlgorithm = 360, - DkimHeaderHashMismatch = 361, - DkimModified = 362, - DkimExploded = 363, - DmarcPass = 106, - DmarcFail = 107, - DmarcPermError = 108, - DmarcTempError = 109, - DmarcNone = 110, + DeliveryAttemptStart = 59, + DeliveryAttemptEnd = 60, + DeliveryCompleted = 61, + DeliveryMxLookupFailed = 62, + DeliveryIpLookupFailed = 63, + DeliveryNullMx = 64, + DeliveryGreetingFailed = 65, + DeliveryEhloRejected = 66, + DeliveryAuthFailed = 67, + DeliveryMailFromRejected = 68, + DeliveryDelivered = 69, + DeliveryRcptToRejected = 70, + DeliveryRcptToFailed = 71, + DeliveryMessageRejected = 72, + DeliveryStartTlsUnavailable = 73, + DeliveryStartTlsError = 74, + DeliveryStartTlsDisabled = 75, + DeliveryImplicitTlsError = 76, + DeliveryConcurrencyLimitExceeded = 77, + DeliveryRateLimitExceeded = 78, + DeliveryDoubleBounce = 79, + DeliveryDsnSuccess = 80, + DeliveryDsnTempFail = 81, + DeliveryDsnPermFail = 82, + DkimPass = 83, + DkimNeutral = 84, + DkimFail = 85, + DkimPermError = 86, + DkimTempError = 87, + DkimNone = 88, + DkimSignatureInvalid = 375, + DkimPublicKeyError = 376, + DkimVerificationFailed = 377, + DkimHashMismatch = 378, + DkimSignerNotFound = 89, + Dkim2InstanceInvalid = 365, + Dkim2SignatureInvalid = 366, + Dkim2ChainInvalid = 367, + Dkim2TimestampInvalid = 368, + Dkim2EnvelopeMismatch = 369, + Dkim2CustodyBroken = 370, + Dkim2PublicKeyError = 371, + Dkim2VerificationFailed = 372, + Dkim2HashMismatch = 373, + Dkim2RecipeError = 374, + Dkim2SigningFailed = 382, + DmarcPass = 90, + DmarcFail = 91, + DmarcPermError = 92, + DmarcTempError = 93, + DmarcNone = 94, DnsLookupTime = 11, DnsRecordCreationFailed = 35, DnsRecordDeletionFailed = 36, DnsRecordLookupFailed = 37, DnsRecordPropagationTimeout = 38, DomainCount = 26, - EvalError = 111, - EvalDirectoryNotFound = 112, - EvalStoreNotFound = 113, + EvalError = 95, + EvalDirectoryNotFound = 96, + EvalStoreNotFound = 97, HttpRequestTime = 12, HttpActiveConnections = 17, - HttpConnectionStart = 337, - HttpError = 114, - HttpRequestBody = 115, - HttpResponseBody = 116, - HttpXForwardedMissing = 117, + HttpConnectionStart = 321, + HttpError = 98, + HttpRequestBody = 99, + HttpResponseBody = 100, + HttpXForwardedMissing = 101, + HttpXForwardedInvalid = 336, ImapRequestTime = 13, ImapActiveConnections = 18, - ImapConnectionStart = 118, - ImapConnectionEnd = 119, - IncomingReportDmarcReport = 120, - IncomingReportDmarcReportWithWarnings = 121, - IncomingReportTlsReport = 122, - IncomingReportTlsReportWithWarnings = 123, - IncomingReportAbuseReport = 124, - IncomingReportAuthFailureReport = 125, - IncomingReportFraudReport = 126, - IncomingReportNotSpamReport = 127, - IncomingReportVirusReport = 128, - IncomingReportOtherReport = 129, - IncomingReportMessageParseFailed = 130, - IncomingReportDmarcParseFailed = 131, - IncomingReportTlsRpcParseFailed = 132, - IncomingReportArfParseFailed = 133, - IncomingReportDecompressError = 134, - IprevPass = 135, - IprevFail = 136, - IprevPermError = 137, - IprevTempError = 138, - IprevNone = 139, - JmapMethodCall = 140, - JmapInvalidArguments = 141, - JmapRequestTooLarge = 142, - JmapUnsupportedFilter = 143, - JmapUnsupportedSort = 144, - JmapUnknownMethod = 145, - JmapForbidden = 146, - JmapNotJson = 147, - JmapNotRequest = 148, - JmapWebsocketStart = 149, - JmapWebsocketError = 150, - LimitSizeRequest = 151, - LimitSizeUpload = 152, - LimitCallsIn = 153, - LimitConcurrentRequest = 154, - LimitConcurrentUpload = 155, - LimitConcurrentConnection = 156, - LimitQuota = 157, - LimitBlobQuota = 158, - LimitTenantQuota = 159, - LimitTooManyRequests = 160, - MailAuthParseError = 161, - MailAuthMissingParameters = 162, - MailAuthNoHeadersFound = 163, - MailAuthCrypto = 164, - MailAuthIo = 165, - MailAuthBase64 = 166, - MailAuthDnsError = 167, - MailAuthDnsRecordNotFound = 168, - MailAuthDnsInvalidRecordType = 169, - MailAuthPolicyNotAligned = 170, - ManageSieveConnectionStart = 171, - ManageSieveConnectionEnd = 172, + ImapConnectionStart = 102, + ImapConnectionEnd = 103, + IncomingReportDmarcReport = 104, + IncomingReportDmarcReportWithWarnings = 105, + IncomingReportTlsReport = 106, + IncomingReportTlsReportWithWarnings = 107, + IncomingReportAbuseReport = 108, + IncomingReportAuthFailureReport = 109, + IncomingReportFraudReport = 110, + IncomingReportNotSpamReport = 111, + IncomingReportVirusReport = 112, + IncomingReportOtherReport = 113, + IncomingReportMessageParseFailed = 114, + IncomingReportDmarcParseFailed = 115, + IncomingReportTlsRpcParseFailed = 116, + IncomingReportArfParseFailed = 117, + IncomingReportDecompressError = 118, + IprevPass = 119, + IprevFail = 120, + IprevPermError = 121, + IprevTempError = 122, + IprevNone = 123, + JmapMethodCall = 124, + JmapInvalidArguments = 125, + JmapRequestTooLarge = 126, + JmapUnsupportedFilter = 127, + JmapUnsupportedSort = 128, + JmapUnknownMethod = 129, + JmapForbidden = 130, + JmapNotJson = 131, + JmapNotRequest = 132, + JmapWebsocketStart = 133, + JmapWebsocketError = 134, + LimitSizeRequest = 135, + LimitSizeUpload = 136, + LimitCallsIn = 137, + LimitConcurrentRequest = 138, + LimitConcurrentUpload = 139, + LimitConcurrentConnection = 140, + LimitQuota = 141, + LimitBlobQuota = 142, + LimitTooManyRequests = 144, + LimitMemoryBudget = 338, + MailAuthParseError = 145, + MailAuthMissingParameters = 146, + MailAuthNoHeadersFound = 147, + MailAuthCrypto = 148, + MailAuthIo = 149, + MailAuthBase64 = 150, + MailAuthDnsError = 151, + MailAuthDnsRecordNotFound = 152, + MailAuthDnsInvalidRecordType = 153, + MailAuthPolicyNotAligned = 154, + ManageSieveConnectionStart = 155, + ManageSieveConnectionEnd = 156, MessageSize = 4, MessageAuthenticatedSize = 5, MessageIngestTime = 0, - MessageIngestIndexTime = 1, - MessageIngestHam = 173, - MessageIngestSpam = 174, - MessageIngestImapAppend = 175, - MessageIngestJmapAppend = 176, - MessageIngestDuplicate = 177, - MessageIngestError = 178, - MessageIngestSearchIndex = 179, - MilterActionAccept = 180, - MilterActionDiscard = 181, - MilterActionReject = 182, - MilterActionTempFail = 183, - MilterActionReplyCode = 184, - MilterActionConnectionFailure = 185, - MilterActionShutdown = 186, - MtaHookActionAccept = 187, - MtaHookActionDiscard = 188, - MtaHookActionReject = 189, - MtaHookActionQuarantine = 190, - MtaHookError = 191, - MtaStsAuthorized = 192, - MtaStsNotAuthorized = 193, - MtaStsInvalidPolicy = 194, - NetworkTimeout = 195, + MessageIngestHam = 157, + MessageIngestSpam = 158, + MessageIngestImapAppend = 159, + MessageIngestJmapAppend = 160, + MessageIngestDuplicate = 161, + MessageIngestError = 162, + MilterActionAccept = 164, + MilterActionDiscard = 165, + MilterActionReject = 166, + MilterActionTempFail = 167, + MilterActionReplyCode = 168, + MilterActionConnectionFailure = 169, + MilterActionShutdown = 170, + MtaHookActionAccept = 171, + MtaHookActionDiscard = 172, + MtaHookActionReject = 173, + MtaHookActionQuarantine = 174, + MtaHookError = 175, + MtaStsAuthorized = 176, + MtaStsNotAuthorized = 177, + MtaStsInvalidPolicy = 178, + NetworkTimeout = 179, OutgoingReportSize = 6, - OutgoingReportSpfReport = 196, - OutgoingReportSpfRateLimited = 197, - OutgoingReportDkimReport = 198, - OutgoingReportDkimRateLimited = 199, - OutgoingReportDmarcReport = 200, - OutgoingReportDmarcRateLimited = 201, - OutgoingReportDmarcAggregateReport = 202, - OutgoingReportTlsAggregate = 203, - OutgoingReportHttpSubmission = 204, - OutgoingReportUnauthorizedReportingAddress = 205, - OutgoingReportReportingAddressValidationError = 206, - OutgoingReportNotFound = 207, - OutgoingReportSubmissionError = 208, - OutgoingReportNoRecipientsFound = 209, + OutgoingReportSpfReport = 180, + OutgoingReportSpfRateLimited = 181, + OutgoingReportDkimReport = 182, + OutgoingReportDkimRateLimited = 183, + OutgoingReportDmarcReport = 184, + OutgoingReportDmarcRateLimited = 185, + OutgoingReportDmarcAggregateReport = 186, + OutgoingReportTlsAggregate = 187, + OutgoingReportHttpSubmission = 188, + OutgoingReportUnauthorizedReportingAddress = 189, + OutgoingReportReportingAddressValidationError = 190, + OutgoingReportNotFound = 191, + OutgoingReportSubmissionError = 192, + OutgoingReportNoRecipientsFound = 193, Pop3RequestTime = 14, Pop3ActiveConnections = 19, - Pop3ConnectionStart = 210, - Pop3ConnectionEnd = 211, - PushSubscriptionSuccess = 213, - PushSubscriptionError = 214, - PushSubscriptionNotFound = 215, + Pop3ConnectionStart = 194, + Pop3ConnectionEnd = 195, + PushSubscriptionSuccess = 197, + PushSubscriptionError = 198, + PushSubscriptionNotFound = 199, QueueCount = 24, - QueueMessageQueued = 216, - QueueAuthenticatedMessageQueued = 217, - QueueReportQueued = 218, - QueueDsnQueued = 219, - QueueAutogeneratedQueued = 220, - QueueRescheduled = 221, - QueueBlobNotFound = 222, - QueueRateLimitExceeded = 223, - QueueConcurrencyLimitExceeded = 224, - QueueQuotaExceeded = 225, - ResourceNotFound = 226, - ResourceBadParameters = 227, - ResourceError = 228, - SecurityAuthenticationBan = 229, - SecurityAbuseBan = 230, - SecurityScanBan = 231, - SecurityLoiterBan = 232, - SecurityIpBlocked = 233, - SecurityUnauthorized = 234, + QueueMessageQueued = 200, + QueueAuthenticatedMessageQueued = 201, + QueueReportQueued = 202, + QueueDsnQueued = 203, + QueueAutogeneratedQueued = 204, + QueueRescheduled = 205, + QueueBlobNotFound = 206, + QueueRateLimitExceeded = 207, + QueueConcurrencyLimitExceeded = 208, + QueueQuotaExceeded = 209, + QueueBackPressure = 347, + ResourceNotFound = 210, + ResourceBadParameters = 211, + ResourceError = 212, + SearchIndexIndexTime = 1, + SearchIndexPendingEmail = 354, + SearchIndexPendingCalendar = 355, + SearchIndexPendingContacts = 356, + SearchIndexPendingFile = 357, + SearchIndexPartitionRetry = 360, + SearchIndexBatchIndexed = 163, + SearchIndexDocumentTruncated = 359, + SecurityAuthenticationBan = 213, + SecurityAbuseBan = 214, + SecurityScanBan = 215, + SecurityLoiterBan = 216, + SecurityIpBlocked = 217, + SecurityUnauthorized = 218, ServerMemory = 23, - ServerThreadError = 235, + ServerBufferedMemorySmtp = 352, + ServerBufferedMemoryRead = 353, + ServerThreadError = 219, SieveRequestTime = 16, SieveActiveConnections = 21, - SieveActionAccept = 236, - SieveActionAcceptReplace = 237, - SieveActionDiscard = 238, - SieveActionReject = 239, - SieveSendMessage = 240, - SieveMessageTooLarge = 241, - SieveRuntimeError = 242, - SieveUnexpectedError = 243, - SieveNotSupported = 244, - SieveQuotaExceeded = 245, + SieveActionAccept = 220, + SieveActionAcceptReplace = 221, + SieveActionDiscard = 222, + SieveActionReject = 223, + SieveSendMessage = 224, + SieveMessageTooLarge = 225, + SieveRuntimeError = 226, + SieveUnexpectedError = 227, + SieveNotSupported = 228, + SieveQuotaExceeded = 229, SmtpRequestTime = 15, SmtpActiveConnections = 20, - SmtpConnectionStart = 246, - SmtpConnectionEnd = 247, - SmtpError = 248, - SmtpConcurrencyLimitExceeded = 249, - SmtpTransferLimitExceeded = 250, - SmtpRateLimitExceeded = 251, - SmtpTimeLimitExceeded = 252, - SmtpMessageParseFailed = 253, - SmtpMessageTooLarge = 254, - SmtpLoopDetected = 255, - SmtpDkimPass = 256, - SmtpDkimFail = 257, - SmtpDkim2Pass = 364, - SmtpDkim2Fail = 365, - SmtpDkim2DsnDiscarded = 366, - SmtpArcPass = 258, - SmtpArcFail = 259, - SmtpSpfEhloPass = 260, - SmtpSpfEhloFail = 261, - SmtpSpfFromPass = 262, - SmtpSpfFromFail = 263, - SmtpDmarcPass = 264, - SmtpDmarcFail = 265, - SmtpIprevPass = 266, - SmtpIprevFail = 267, - SmtpTooManyMessages = 268, - SmtpInvalidEhlo = 269, - SmtpDidNotSayEhlo = 270, - SmtpMailFromUnauthenticated = 271, - SmtpMailFromUnauthorized = 272, - SmtpMailFromMissing = 273, - SmtpMultipleMailFrom = 274, - SmtpMailboxDoesNotExist = 275, - SmtpRelayNotAllowed = 276, - SmtpRcptToDuplicate = 277, - SmtpRcptToMissing = 278, - SmtpTooManyRecipients = 279, - SmtpTooManyInvalidRcpt = 280, - SmtpAuthMechanismNotSupported = 281, - SmtpAuthExchangeTooLong = 282, - SmtpCommandNotImplemented = 283, - SmtpInvalidCommand = 284, - SmtpSyntaxError = 285, - SmtpRequestTooLarge = 286, - SpamPyzorError = 287, - SpamDnsblError = 288, - SpamTrainCompleted = 289, - SpamTrainSampleAdded = 290, - SpamClassify = 291, - SpamModelNotReady = 292, - SpfPass = 293, - SpfFail = 294, - SpfSoftFail = 295, - SpfNeutral = 296, - SpfTempError = 297, - SpfPermError = 298, - SpfNone = 299, + SmtpConnectionStart = 230, + SmtpConnectionEnd = 231, + SmtpError = 232, + SmtpConcurrencyLimitExceeded = 233, + SmtpTransferLimitExceeded = 234, + SmtpRateLimitExceeded = 235, + SmtpTimeLimitExceeded = 236, + SmtpMessageParseFailed = 237, + SmtpMessageTooLarge = 238, + SmtpLoopDetected = 239, + SmtpMultipleFromDomains = 337, + SmtpDkimPass = 240, + SmtpDkimFail = 241, + SmtpDkim2Pass = 324, + SmtpDkim2Fail = 325, + SmtpDkim2DsnRejected = 326, + SmtpArcPass = 242, + SmtpArcFail = 243, + SmtpSpfEhloPass = 244, + SmtpSpfEhloFail = 245, + SmtpSpfFromPass = 246, + SmtpSpfFromFail = 247, + SmtpDmarcPass = 248, + SmtpDmarcFail = 249, + SmtpIprevPass = 250, + SmtpIprevFail = 251, + SmtpTooManyMessages = 252, + SmtpInvalidEhlo = 253, + SmtpDidNotSayEhlo = 254, + SmtpMailFromUnauthenticated = 255, + SmtpMailFromUnauthorized = 256, + SmtpMailFromMissing = 257, + SmtpMultipleMailFrom = 258, + SmtpMailboxDoesNotExist = 259, + SmtpRelayNotAllowed = 260, + SmtpRcptToDuplicate = 261, + SmtpRcptToMissing = 262, + SmtpTooManyRecipients = 263, + SmtpTooManyInvalidRcpt = 264, + SmtpAuthMechanismNotSupported = 265, + SmtpAuthExchangeTooLong = 266, + SmtpCommandNotImplemented = 267, + SmtpInvalidCommand = 268, + SmtpSyntaxError = 269, + SmtpRequestTooLarge = 270, + SpamPyzorError = 271, + SpamDnsblError = 272, + SpamTrainCompleted = 273, + SpamTrainSampleAdded = 274, + SpamClassify = 275, + SpamModelNotReady = 276, + SpfPass = 277, + SpfFail = 278, + SpfSoftFail = 279, + SpfNeutral = 280, + SpfTempError = 281, + SpfPermError = 282, + SpfNone = 283, StoreDataReadTime = 7, StoreDataWriteTime = 8, StoreBlobReadTime = 9, StoreBlobWriteTime = 10, - StoreAssertValueFailed = 300, - StoreFoundationdbError = 301, - StoreMysqlError = 302, - StorePostgresqlError = 303, - StoreRocksdbError = 304, - StoreSqliteError = 305, - StoreLdapError = 306, - StoreElasticsearchError = 307, - StoreRedisError = 308, - StoreS3Error = 309, - StoreAzureError = 310, - StoreFilesystemError = 311, - StorePoolError = 312, - StoreDataCorruption = 313, - StoreDecompressError = 314, - StoreDeserializeError = 315, - StoreNotFound = 316, - StoreNotConfigured = 317, - StoreNotSupported = 318, - StoreUnexpectedError = 319, - StoreCryptoError = 320, - StoreHttpStoreError = 321, - StoreBlobMissingMarker = 322, - StoreDataWrite = 323, - StoreDataIterate = 324, - StoreBlobRead = 325, - StoreBlobWrite = 326, - StoreBlobDelete = 327, - TaskManagerBlobNotFound = 328, - TaskManagerMetadataNotFound = 329, - TelemetryAlertEvent = 338, - TelemetryAlertMessage = 212, - TelemetryLogError = 330, - TelemetryWebhookError = 331, - TelemetryOtelExporterError = 332, - TelemetryOtelMetricsExporterError = 333, - TelemetryPrometheusExporterError = 334, - TelemetryJournalError = 335, - TlsHandshakeError = 336, + StoreAssertValueFailed = 284, + StoreFoundationdbError = 285, + StoreMysqlError = 286, + StorePostgresqlError = 287, + StoreRocksdbError = 288, + StoreSqliteError = 289, + StoreLdapError = 290, + StoreElasticsearchError = 291, + StoreRedisError = 292, + StoreS3Error = 293, + StoreAzureError = 294, + StoreFilesystemError = 295, + StorePoolError = 296, + StoreDataCorruption = 297, + StoreDecompressError = 298, + StoreDeserializeError = 299, + StoreNotFound = 300, + StoreNotConfigured = 301, + StoreNotSupported = 302, + StoreUnexpectedError = 303, + StoreCryptoError = 304, + StoreHttpStoreError = 305, + StoreBlobMissingMarker = 306, + StoreDataWrite = 307, + StoreDataIterate = 308, + StoreBlobRead = 309, + StoreBlobWrite = 310, + StoreBlobDelete = 311, + TaskManagerPending = 363, + TaskManagerDue = 364, + TaskManagerTaskFailed = 361, + TaskManagerTaskRetry = 362, + TaskManagerTaskStalled = 383, + TaskManagerTaskExpired = 384, + TaskManagerBlobNotFound = 312, + TaskManagerMetadataNotFound = 313, + TelemetryLogError = 314, + TelemetryWebhookError = 315, + TelemetryOtelExporterError = 316, + TelemetryOtelMetricsExporterError = 317, + TelemetryPrometheusExporterError = 318, + TelemetryJournalError = 319, + TelemetryEventsDropped = 327, + TlsHandshakeError = 320, UserCount = 25, } @@ -1352,4 +1363,25 @@ pub enum Key { Value = 63, Version = 64, QueueName = 65, + Name = 66, + ObjectType = 67, + Property = 68, + NodeId = 69, + RemoteNodeId = 70, + TaskId = 71, + Partition = 72, + Index = 73, + Attempt = 74, + Used = 75, + Period = 76, + RetryAfter = 77, + Command = 78, + Tag = 79, + Address = 80, + Ip = 81, + Headers = 82, + Query = 83, + Cipher = 84, + ResourceUid = 85, + Script = 86, } diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index e84c5aba..288d011a 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -10,6 +10,7 @@ use crate::{Level, event::enums::*}; use std::borrow::Cow; impl EventType { + #[cfg(not(feature = "enterprise"))] pub fn parse(value: &str) -> Option { hashify::map! { value.as_bytes(), @@ -34,14 +35,9 @@ impl EventType { b"acme.tls-alpn-error" => EventType::Acme(AcmeEvent::TlsAlpnError), b"acme.token-not-found" => EventType::Acme(AcmeEvent::TokenNotFound), b"acme.error" => EventType::Acme(AcmeEvent::Error), - b"ai.llm-response" => EventType::Ai(AiEvent::LlmResponse), - b"ai.api-error" => EventType::Ai(AiEvent::ApiError), - b"arc.chain-too-long" => EventType::Arc(ArcEvent::ChainTooLong), - b"arc.invalid-instance" => EventType::Arc(ArcEvent::InvalidInstance), - b"arc.invalid-cv" => EventType::Arc(ArcEvent::InvalidCv), - b"arc.has-header-tag" => EventType::Arc(ArcEvent::HasHeaderTag), - b"arc.broken-chain" => EventType::Arc(ArcEvent::BrokenChain), - b"arc.sealer-not-found" => EventType::Arc(ArcEvent::SealerNotFound), + b"arc.chain-invalid" => EventType::Arc(ArcEvent::ChainInvalid), + b"arc.signature-invalid" => EventType::Arc(ArcEvent::SignatureInvalid), + b"arc.hash-mismatch" => EventType::Arc(ArcEvent::HashMismatch), b"auth.success" => EventType::Auth(AuthEvent::Success), b"auth.failed" => EventType::Auth(AuthEvent::Failed), b"auth.token-expired" => EventType::Auth(AuthEvent::TokenExpired), @@ -51,6 +47,18 @@ impl EventType { b"auth.error" => EventType::Auth(AuthEvent::Error), b"auth.warning" => EventType::Auth(AuthEvent::Warning), b"auth.credential-expired" => EventType::Auth(AuthEvent::CredentialExpired), + b"cache.miss" => EventType::Cache(CacheEvent::Miss), + b"cache.hit" => EventType::Cache(CacheEvent::Hit), + b"cache.stale" => EventType::Cache(CacheEvent::Stale), + b"cache.update" => EventType::Cache(CacheEvent::Update), + b"cache.entry-too-large" => EventType::Cache(CacheEvent::EntryTooLarge), + b"cache.invalidate" => EventType::Cache(CacheEvent::Invalidate), + b"cache.swap-hit" => EventType::Cache(CacheEvent::SwapHit), + b"cache.swap-miss" => EventType::Cache(CacheEvent::SwapMiss), + b"cache.swap-write" => EventType::Cache(CacheEvent::SwapWrite), + b"cache.swap-error" => EventType::Cache(CacheEvent::SwapError), + b"cache.swap-metrics" => EventType::Cache(CacheEvent::SwapMetrics), + b"cache.swap-flush" => EventType::Cache(CacheEvent::SwapFlush), b"calendar.rule-expansion-error" => EventType::Calendar(CalendarEvent::RuleExpansionError), b"calendar.alarm-sent" => EventType::Calendar(CalendarEvent::AlarmSent), b"calendar.alarm-skipped" => EventType::Calendar(CalendarEvent::AlarmSkipped), @@ -126,17 +134,10 @@ impl EventType { b"dkim.perm-error" => EventType::Dkim(DkimEvent::PermError), b"dkim.temp-error" => EventType::Dkim(DkimEvent::TempError), b"dkim.none" => EventType::Dkim(DkimEvent::None), - b"dkim.unsupported-version" => EventType::Dkim(DkimEvent::UnsupportedVersion), - b"dkim.unsupported-algorithm" => EventType::Dkim(DkimEvent::UnsupportedAlgorithm), - b"dkim.unsupported-canonicalization" => EventType::Dkim(DkimEvent::UnsupportedCanonicalization), - b"dkim.unsupported-key-type" => EventType::Dkim(DkimEvent::UnsupportedKeyType), - b"dkim.failed-body-hash-match" => EventType::Dkim(DkimEvent::FailedBodyHashMatch), - b"dkim.failed-verification" => EventType::Dkim(DkimEvent::FailedVerification), - b"dkim.failed-auid-match" => EventType::Dkim(DkimEvent::FailedAuidMatch), - b"dkim.revoked-public-key" => EventType::Dkim(DkimEvent::RevokedPublicKey), - b"dkim.incompatible-algorithms" => EventType::Dkim(DkimEvent::IncompatibleAlgorithms), - b"dkim.signature-expired" => EventType::Dkim(DkimEvent::SignatureExpired), - b"dkim.signature-length" => EventType::Dkim(DkimEvent::SignatureLength), + b"dkim.signature-invalid" => EventType::Dkim(DkimEvent::SignatureInvalid), + b"dkim.public-key-error" => EventType::Dkim(DkimEvent::PublicKeyError), + b"dkim.verification-failed" => EventType::Dkim(DkimEvent::VerificationFailed), + b"dkim.hash-mismatch" => EventType::Dkim(DkimEvent::HashMismatch), b"dkim.signer-not-found" => EventType::Dkim(DkimEvent::SignerNotFound), b"dkim.build-error" => EventType::Dkim(DkimEvent::BuildError), b"dkim.signature-created" => EventType::Dkim(DkimEvent::SignatureCreated), @@ -144,32 +145,17 @@ impl EventType { b"dkim.signature-retiring" => EventType::Dkim(DkimEvent::SignatureRetiring), b"dkim.signature-retired" => EventType::Dkim(DkimEvent::SignatureRetired), b"dkim.signature-deleted" => EventType::Dkim(DkimEvent::SignatureDeleted), - b"dkim.instance-missing" => EventType::Dkim(DkimEvent::InstanceMissing), - b"dkim.instance-syntax" => EventType::Dkim(DkimEvent::InstanceSyntax), - b"dkim.instance-tag-missing" => EventType::Dkim(DkimEvent::InstanceTagMissing), - b"dkim.instance-not-signed" => EventType::Dkim(DkimEvent::InstanceNotSigned), - b"dkim.instance-above-signature" => EventType::Dkim(DkimEvent::InstanceAboveSignature), - b"dkim.signature-missing" => EventType::Dkim(DkimEvent::SignatureMissing), - b"dkim.signature-syntax" => EventType::Dkim(DkimEvent::SignatureSyntax), - b"dkim.signature-tag-missing" => EventType::Dkim(DkimEvent::SignatureTagMissing), - b"dkim.signature-tag-unexpected" => EventType::Dkim(DkimEvent::SignatureTagUnexpected), - b"dkim.sequence-gap" => EventType::Dkim(DkimEvent::SequenceGap), - b"dkim.sequence-overflow" => EventType::Dkim(DkimEvent::SequenceOverflow), - b"dkim.chain-too-long" => EventType::Dkim(DkimEvent::ChainTooLong), - b"dkim.mail-from-mismatch" => EventType::Dkim(DkimEvent::MailFromMismatch), - b"dkim.rcpt-to-mismatch" => EventType::Dkim(DkimEvent::RcptToMismatch), - b"dkim.mail-from-domain-mismatch" => EventType::Dkim(DkimEvent::MailFromDomainMismatch), - b"dkim.next-domain-mismatch" => EventType::Dkim(DkimEvent::NextDomainMismatch), - b"dkim.custody-break" => EventType::Dkim(DkimEvent::CustodyBreak), - b"dkim.public-key-fetch" => EventType::Dkim(DkimEvent::PublicKeyFetch), - b"dkim.public-key-missing" => EventType::Dkim(DkimEvent::PublicKeyMissing), - b"dkim.public-key-multiple" => EventType::Dkim(DkimEvent::PublicKeyMultiple), - b"dkim.public-key-syntax" => EventType::Dkim(DkimEvent::PublicKeySyntax), - b"dkim.public-key-algorithm-mismatch" => EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch), - b"dkim.no-valid-algorithm" => EventType::Dkim(DkimEvent::NoValidAlgorithm), - b"dkim.header-hash-mismatch" => EventType::Dkim(DkimEvent::HeaderHashMismatch), - b"dkim.modified" => EventType::Dkim(DkimEvent::Modified), - b"dkim.exploded" => EventType::Dkim(DkimEvent::Exploded), + b"dkim2.instance-invalid" => EventType::Dkim2(Dkim2Event::InstanceInvalid), + b"dkim2.signature-invalid" => EventType::Dkim2(Dkim2Event::SignatureInvalid), + b"dkim2.chain-invalid" => EventType::Dkim2(Dkim2Event::ChainInvalid), + b"dkim2.timestamp-invalid" => EventType::Dkim2(Dkim2Event::TimestampInvalid), + b"dkim2.envelope-mismatch" => EventType::Dkim2(Dkim2Event::EnvelopeMismatch), + b"dkim2.custody-broken" => EventType::Dkim2(Dkim2Event::CustodyBroken), + b"dkim2.public-key-error" => EventType::Dkim2(Dkim2Event::PublicKeyError), + b"dkim2.verification-failed" => EventType::Dkim2(Dkim2Event::VerificationFailed), + b"dkim2.hash-mismatch" => EventType::Dkim2(Dkim2Event::HashMismatch), + b"dkim2.recipe-error" => EventType::Dkim2(Dkim2Event::RecipeError), + b"dkim2.signing-failed" => EventType::Dkim2(Dkim2Event::SigningFailed), b"dmarc.pass" => EventType::Dmarc(DmarcEvent::Pass), b"dmarc.fail" => EventType::Dmarc(DmarcEvent::Fail), b"dmarc.perm-error" => EventType::Dmarc(DmarcEvent::PermError), @@ -182,7 +168,6 @@ impl EventType { b"dns.record-lookup-failed" => EventType::Dns(DnsEvent::RecordLookupFailed), b"dns.record-propagated" => EventType::Dns(DnsEvent::RecordPropagated), b"dns.record-propagation-timeout" => EventType::Dns(DnsEvent::RecordPropagationTimeout), - b"dns.build-error" => EventType::Dns(DnsEvent::BuildError), b"eval.result" => EventType::Eval(EvalEvent::Result), b"eval.error" => EventType::Eval(EvalEvent::Error), b"eval.directory-not-found" => EventType::Eval(EvalEvent::DirectoryNotFound), @@ -194,6 +179,7 @@ impl EventType { b"http.request-body" => EventType::Http(HttpEvent::RequestBody), b"http.response-body" => EventType::Http(HttpEvent::ResponseBody), b"http.x-forwarded-missing" => EventType::Http(HttpEvent::XForwardedMissing), + b"http.x-forwarded-invalid" => EventType::Http(HttpEvent::XForwardedInvalid), b"imap.connection-start" => EventType::Imap(ImapEvent::ConnectionStart), b"imap.connection-end" => EventType::Imap(ImapEvent::ConnectionEnd), b"imap.get-acl" => EventType::Imap(ImapEvent::GetAcl), @@ -229,6 +215,8 @@ impl EventType { b"imap.thread" => EventType::Imap(ImapEvent::Thread), b"imap.uid-batches" => EventType::Imap(ImapEvent::UidBatches), b"imap.get-quota" => EventType::Imap(ImapEvent::GetQuota), + b"imap.get-metadata" => EventType::Imap(ImapEvent::GetMetadata), + b"imap.set-metadata" => EventType::Imap(ImapEvent::SetMetadata), b"imap.error" => EventType::Imap(ImapEvent::Error), b"imap.raw-input" => EventType::Imap(ImapEvent::RawInput), b"imap.raw-output" => EventType::Imap(ImapEvent::RawOutput), @@ -265,6 +253,8 @@ impl EventType { b"jmap.account-not-found" => EventType::Jmap(JmapEvent::AccountNotFound), b"jmap.account-not-supported-by-method" => EventType::Jmap(JmapEvent::AccountNotSupportedByMethod), b"jmap.account-read-only" => EventType::Jmap(JmapEvent::AccountReadOnly), + b"jmap.from-account-not-found" => EventType::Jmap(JmapEvent::FromAccountNotFound), + b"jmap.from-account-not-supported-by-method" => EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod), b"jmap.not-found" => EventType::Jmap(JmapEvent::NotFound), b"jmap.cannot-calculate-changes" => EventType::Jmap(JmapEvent::CannotCalculateChanges), b"jmap.unknown-data-type" => EventType::Jmap(JmapEvent::UnknownDataType), @@ -274,6 +264,11 @@ impl EventType { b"jmap.websocket-start" => EventType::Jmap(JmapEvent::WebsocketStart), b"jmap.websocket-stop" => EventType::Jmap(JmapEvent::WebsocketStop), b"jmap.websocket-error" => EventType::Jmap(JmapEvent::WebsocketError), + b"jmap.expand-duration-too-large" => EventType::Jmap(JmapEvent::ExpandDurationTooLarge), + b"jmap.cannot-calculate-occurrences" => EventType::Jmap(JmapEvent::CannotCalculateOccurrences), + b"jmap.too-large" => EventType::Jmap(JmapEvent::TooLarge), + b"jmap.rate-limit" => EventType::Jmap(JmapEvent::RateLimit), + b"jmap.object-not-found" => EventType::Jmap(JmapEvent::ObjectNotFound), b"limit.size-request" => EventType::Limit(LimitEvent::SizeRequest), b"limit.size-upload" => EventType::Limit(LimitEvent::SizeUpload), b"limit.calls-in" => EventType::Limit(LimitEvent::CallsIn), @@ -282,8 +277,8 @@ impl EventType { b"limit.concurrent-connection" => EventType::Limit(LimitEvent::ConcurrentConnection), b"limit.quota" => EventType::Limit(LimitEvent::Quota), b"limit.blob-quota" => EventType::Limit(LimitEvent::BlobQuota), - b"limit.tenant-quota" => EventType::Limit(LimitEvent::TenantQuota), b"limit.too-many-requests" => EventType::Limit(LimitEvent::TooManyRequests), + b"limit.memory-budget" => EventType::Limit(LimitEvent::MemoryBudget), b"mail-auth.parse-error" => EventType::MailAuth(MailAuthEvent::ParseError), b"mail-auth.missing-parameters" => EventType::MailAuth(MailAuthEvent::MissingParameters), b"mail-auth.no-headers-found" => EventType::MailAuth(MailAuthEvent::NoHeadersFound), @@ -319,7 +314,6 @@ impl EventType { b"message-ingest.jmap-append" => EventType::MessageIngest(MessageIngestEvent::JmapAppend), b"message-ingest.duplicate" => EventType::MessageIngest(MessageIngestEvent::Duplicate), b"message-ingest.error" => EventType::MessageIngest(MessageIngestEvent::Error), - b"message-ingest.search-index" => EventType::MessageIngest(MessageIngestEvent::SearchIndex), b"milter.read" => EventType::Milter(MilterEvent::Read), b"milter.write" => EventType::Milter(MilterEvent::Write), b"milter.action-accept" => EventType::Milter(MilterEvent::ActionAccept), @@ -427,6 +421,14 @@ impl EventType { b"resource.download-external" => EventType::Resource(ResourceEvent::DownloadExternal), b"resource.application-updated" => EventType::Resource(ResourceEvent::ApplicationUpdated), b"resource.application-unpacked" => EventType::Resource(ResourceEvent::ApplicationUnpacked), + b"search-index.manager-started" => EventType::SearchIndex(SearchIndexEvent::ManagerStarted), + b"search-index.partitions-acquired" => EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired), + b"search-index.partition-locked" => EventType::SearchIndex(SearchIndexEvent::PartitionLocked), + b"search-index.partition-retry" => EventType::SearchIndex(SearchIndexEvent::PartitionRetry), + b"search-index.batch-indexed" => EventType::SearchIndex(SearchIndexEvent::BatchIndexed), + b"search-index.document-ignored" => EventType::SearchIndex(SearchIndexEvent::DocumentIgnored), + b"search-index.document-truncated" => EventType::SearchIndex(SearchIndexEvent::DocumentTruncated), + b"search-index.metadata-not-found" => EventType::SearchIndex(SearchIndexEvent::MetadataNotFound), b"security.authentication-ban" => EventType::Security(SecurityEvent::AuthenticationBan), b"security.abuse-ban" => EventType::Security(SecurityEvent::AbuseBan), b"security.scan-ban" => EventType::Security(SecurityEvent::ScanBan), @@ -440,7 +442,6 @@ impl EventType { b"server.shutdown" => EventType::Server(ServerEvent::Shutdown), b"server.startup-error" => EventType::Server(ServerEvent::StartupError), b"server.thread-error" => EventType::Server(ServerEvent::ThreadError), - b"server.licensing" => EventType::Server(ServerEvent::Licensing), b"server.recovery-mode" => EventType::Server(ServerEvent::RecoveryMode), b"server.bootstrap-mode" => EventType::Server(ServerEvent::BootstrapMode), b"sieve.action-accept" => EventType::Sieve(SieveEvent::ActionAccept), @@ -463,15 +464,15 @@ impl EventType { b"smtp.transfer-limit-exceeded" => EventType::Smtp(SmtpEvent::TransferLimitExceeded), b"smtp.rate-limit-exceeded" => EventType::Smtp(SmtpEvent::RateLimitExceeded), b"smtp.time-limit-exceeded" => EventType::Smtp(SmtpEvent::TimeLimitExceeded), - b"smtp.missing-auth-directory" => EventType::Smtp(SmtpEvent::MissingAuthDirectory), b"smtp.message-parse-failed" => EventType::Smtp(SmtpEvent::MessageParseFailed), b"smtp.message-too-large" => EventType::Smtp(SmtpEvent::MessageTooLarge), b"smtp.loop-detected" => EventType::Smtp(SmtpEvent::LoopDetected), + b"smtp.multiple-from-domains" => EventType::Smtp(SmtpEvent::MultipleFromDomains), b"smtp.dkim-pass" => EventType::Smtp(SmtpEvent::DkimPass), b"smtp.dkim-fail" => EventType::Smtp(SmtpEvent::DkimFail), b"smtp.dkim2-pass" => EventType::Smtp(SmtpEvent::Dkim2Pass), b"smtp.dkim2-fail" => EventType::Smtp(SmtpEvent::Dkim2Fail), - b"smtp.dkim2-dsn-discarded" => EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded), + b"smtp.dkim2-dsn-rejected" => EventType::Smtp(SmtpEvent::Dkim2DsnRejected), b"smtp.arc-pass" => EventType::Smtp(SmtpEvent::ArcPass), b"smtp.arc-fail" => EventType::Smtp(SmtpEvent::ArcFail), b"smtp.spf-ehlo-pass" => EventType::Smtp(SmtpEvent::SpfEhloPass), @@ -583,11 +584,6 @@ impl EventType { b"store.unexpected-error" => EventType::Store(StoreEvent::UnexpectedError), b"store.crypto-error" => EventType::Store(StoreEvent::CryptoError), b"store.http-store-error" => EventType::Store(StoreEvent::HttpStoreError), - b"store.cache-miss" => EventType::Store(StoreEvent::CacheMiss), - b"store.cache-hit" => EventType::Store(StoreEvent::CacheHit), - b"store.cache-stale" => EventType::Store(StoreEvent::CacheStale), - b"store.cache-update" => EventType::Store(StoreEvent::CacheUpdate), - b"store.cache-entry-too-large" => EventType::Store(StoreEvent::CacheEntryTooLarge), b"store.blob-missing-marker" => EventType::Store(StoreEvent::BlobMissingMarker), b"store.data-write" => EventType::Store(StoreEvent::DataWrite), b"store.data-iterate" => EventType::Store(StoreEvent::DataIterate), @@ -608,20 +604,20 @@ impl EventType { b"task-manager.task-ignored" => EventType::TaskManager(TaskManagerEvent::TaskIgnored), b"task-manager.task-failed" => EventType::TaskManager(TaskManagerEvent::TaskFailed), b"task-manager.task-retry" => EventType::TaskManager(TaskManagerEvent::TaskRetry), + b"task-manager.task-stalled" => EventType::TaskManager(TaskManagerEvent::TaskStalled), + b"task-manager.task-expired" => EventType::TaskManager(TaskManagerEvent::TaskExpired), b"task-manager.blob-not-found" => EventType::TaskManager(TaskManagerEvent::BlobNotFound), b"task-manager.metadata-not-found" => EventType::TaskManager(TaskManagerEvent::MetadataNotFound), b"task-manager.scheduler-started" => EventType::TaskManager(TaskManagerEvent::SchedulerStarted), b"task-manager.manager-started" => EventType::TaskManager(TaskManagerEvent::ManagerStarted), - b"telemetry.alert-event" => EventType::Telemetry(TelemetryEvent::AlertEvent), - b"telemetry.alert-message" => EventType::Telemetry(TelemetryEvent::AlertMessage), b"telemetry.log-error" => EventType::Telemetry(TelemetryEvent::LogError), b"telemetry.webhook-error" => EventType::Telemetry(TelemetryEvent::WebhookError), b"telemetry.otel-exporter-error" => EventType::Telemetry(TelemetryEvent::OtelExporterError), b"telemetry.otel-metrics-exporter-error" => EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError), b"telemetry.prometheus-exporter-error" => EventType::Telemetry(TelemetryEvent::PrometheusExporterError), b"telemetry.journal-error" => EventType::Telemetry(TelemetryEvent::JournalError), + b"telemetry.events-dropped" => EventType::Telemetry(TelemetryEvent::EventsDropped), b"telemetry.metrics-collected" => EventType::Telemetry(TelemetryEvent::MetricsCollected), - b"telemetry.metrics-stored" => EventType::Telemetry(TelemetryEvent::MetricsStored), b"telemetry.metrics-pushed" => EventType::Telemetry(TelemetryEvent::MetricsPushed), b"tls.handshake" => EventType::Tls(TlsEvent::Handshake), b"tls.handshake-error" => EventType::Tls(TlsEvent::HandshakeError), @@ -677,14 +673,9 @@ impl EventType { EventType::Acme(AcmeEvent::TlsAlpnError) => "acme.tls-alpn-error", EventType::Acme(AcmeEvent::TokenNotFound) => "acme.token-not-found", EventType::Acme(AcmeEvent::Error) => "acme.error", - EventType::Ai(AiEvent::LlmResponse) => "ai.llm-response", - EventType::Ai(AiEvent::ApiError) => "ai.api-error", - EventType::Arc(ArcEvent::ChainTooLong) => "arc.chain-too-long", - EventType::Arc(ArcEvent::InvalidInstance) => "arc.invalid-instance", - EventType::Arc(ArcEvent::InvalidCv) => "arc.invalid-cv", - EventType::Arc(ArcEvent::HasHeaderTag) => "arc.has-header-tag", - EventType::Arc(ArcEvent::BrokenChain) => "arc.broken-chain", - EventType::Arc(ArcEvent::SealerNotFound) => "arc.sealer-not-found", + EventType::Arc(ArcEvent::ChainInvalid) => "arc.chain-invalid", + EventType::Arc(ArcEvent::SignatureInvalid) => "arc.signature-invalid", + EventType::Arc(ArcEvent::HashMismatch) => "arc.hash-mismatch", EventType::Auth(AuthEvent::Success) => "auth.success", EventType::Auth(AuthEvent::Failed) => "auth.failed", EventType::Auth(AuthEvent::TokenExpired) => "auth.token-expired", @@ -694,6 +685,18 @@ impl EventType { EventType::Auth(AuthEvent::Error) => "auth.error", EventType::Auth(AuthEvent::Warning) => "auth.warning", EventType::Auth(AuthEvent::CredentialExpired) => "auth.credential-expired", + EventType::Cache(CacheEvent::Miss) => "cache.miss", + EventType::Cache(CacheEvent::Hit) => "cache.hit", + EventType::Cache(CacheEvent::Stale) => "cache.stale", + EventType::Cache(CacheEvent::Update) => "cache.update", + EventType::Cache(CacheEvent::EntryTooLarge) => "cache.entry-too-large", + EventType::Cache(CacheEvent::Invalidate) => "cache.invalidate", + EventType::Cache(CacheEvent::SwapHit) => "cache.swap-hit", + EventType::Cache(CacheEvent::SwapMiss) => "cache.swap-miss", + EventType::Cache(CacheEvent::SwapWrite) => "cache.swap-write", + EventType::Cache(CacheEvent::SwapError) => "cache.swap-error", + EventType::Cache(CacheEvent::SwapMetrics) => "cache.swap-metrics", + EventType::Cache(CacheEvent::SwapFlush) => "cache.swap-flush", EventType::Calendar(CalendarEvent::RuleExpansionError) => { "calendar.rule-expansion-error" } @@ -787,19 +790,10 @@ impl EventType { EventType::Dkim(DkimEvent::PermError) => "dkim.perm-error", EventType::Dkim(DkimEvent::TempError) => "dkim.temp-error", EventType::Dkim(DkimEvent::None) => "dkim.none", - EventType::Dkim(DkimEvent::UnsupportedVersion) => "dkim.unsupported-version", - EventType::Dkim(DkimEvent::UnsupportedAlgorithm) => "dkim.unsupported-algorithm", - EventType::Dkim(DkimEvent::UnsupportedCanonicalization) => { - "dkim.unsupported-canonicalization" - } - EventType::Dkim(DkimEvent::UnsupportedKeyType) => "dkim.unsupported-key-type", - EventType::Dkim(DkimEvent::FailedBodyHashMatch) => "dkim.failed-body-hash-match", - EventType::Dkim(DkimEvent::FailedVerification) => "dkim.failed-verification", - EventType::Dkim(DkimEvent::FailedAuidMatch) => "dkim.failed-auid-match", - EventType::Dkim(DkimEvent::RevokedPublicKey) => "dkim.revoked-public-key", - EventType::Dkim(DkimEvent::IncompatibleAlgorithms) => "dkim.incompatible-algorithms", - EventType::Dkim(DkimEvent::SignatureExpired) => "dkim.signature-expired", - EventType::Dkim(DkimEvent::SignatureLength) => "dkim.signature-length", + EventType::Dkim(DkimEvent::SignatureInvalid) => "dkim.signature-invalid", + EventType::Dkim(DkimEvent::PublicKeyError) => "dkim.public-key-error", + EventType::Dkim(DkimEvent::VerificationFailed) => "dkim.verification-failed", + EventType::Dkim(DkimEvent::HashMismatch) => "dkim.hash-mismatch", EventType::Dkim(DkimEvent::SignerNotFound) => "dkim.signer-not-found", EventType::Dkim(DkimEvent::BuildError) => "dkim.build-error", EventType::Dkim(DkimEvent::SignatureCreated) => "dkim.signature-created", @@ -807,34 +801,17 @@ impl EventType { EventType::Dkim(DkimEvent::SignatureRetiring) => "dkim.signature-retiring", EventType::Dkim(DkimEvent::SignatureRetired) => "dkim.signature-retired", EventType::Dkim(DkimEvent::SignatureDeleted) => "dkim.signature-deleted", - EventType::Dkim(DkimEvent::InstanceMissing) => "dkim.instance-missing", - EventType::Dkim(DkimEvent::InstanceSyntax) => "dkim.instance-syntax", - EventType::Dkim(DkimEvent::InstanceTagMissing) => "dkim.instance-tag-missing", - EventType::Dkim(DkimEvent::InstanceNotSigned) => "dkim.instance-not-signed", - EventType::Dkim(DkimEvent::InstanceAboveSignature) => "dkim.instance-above-signature", - EventType::Dkim(DkimEvent::SignatureMissing) => "dkim.signature-missing", - EventType::Dkim(DkimEvent::SignatureSyntax) => "dkim.signature-syntax", - EventType::Dkim(DkimEvent::SignatureTagMissing) => "dkim.signature-tag-missing", - EventType::Dkim(DkimEvent::SignatureTagUnexpected) => "dkim.signature-tag-unexpected", - EventType::Dkim(DkimEvent::SequenceGap) => "dkim.sequence-gap", - EventType::Dkim(DkimEvent::SequenceOverflow) => "dkim.sequence-overflow", - EventType::Dkim(DkimEvent::ChainTooLong) => "dkim.chain-too-long", - EventType::Dkim(DkimEvent::MailFromMismatch) => "dkim.mail-from-mismatch", - EventType::Dkim(DkimEvent::RcptToMismatch) => "dkim.rcpt-to-mismatch", - EventType::Dkim(DkimEvent::MailFromDomainMismatch) => "dkim.mail-from-domain-mismatch", - EventType::Dkim(DkimEvent::NextDomainMismatch) => "dkim.next-domain-mismatch", - EventType::Dkim(DkimEvent::CustodyBreak) => "dkim.custody-break", - EventType::Dkim(DkimEvent::PublicKeyFetch) => "dkim.public-key-fetch", - EventType::Dkim(DkimEvent::PublicKeyMissing) => "dkim.public-key-missing", - EventType::Dkim(DkimEvent::PublicKeyMultiple) => "dkim.public-key-multiple", - EventType::Dkim(DkimEvent::PublicKeySyntax) => "dkim.public-key-syntax", - EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch) => { - "dkim.public-key-algorithm-mismatch" - } - EventType::Dkim(DkimEvent::NoValidAlgorithm) => "dkim.no-valid-algorithm", - EventType::Dkim(DkimEvent::HeaderHashMismatch) => "dkim.header-hash-mismatch", - EventType::Dkim(DkimEvent::Modified) => "dkim.modified", - EventType::Dkim(DkimEvent::Exploded) => "dkim.exploded", + EventType::Dkim2(Dkim2Event::InstanceInvalid) => "dkim2.instance-invalid", + EventType::Dkim2(Dkim2Event::SignatureInvalid) => "dkim2.signature-invalid", + EventType::Dkim2(Dkim2Event::ChainInvalid) => "dkim2.chain-invalid", + EventType::Dkim2(Dkim2Event::TimestampInvalid) => "dkim2.timestamp-invalid", + EventType::Dkim2(Dkim2Event::EnvelopeMismatch) => "dkim2.envelope-mismatch", + EventType::Dkim2(Dkim2Event::CustodyBroken) => "dkim2.custody-broken", + EventType::Dkim2(Dkim2Event::PublicKeyError) => "dkim2.public-key-error", + EventType::Dkim2(Dkim2Event::VerificationFailed) => "dkim2.verification-failed", + EventType::Dkim2(Dkim2Event::HashMismatch) => "dkim2.hash-mismatch", + EventType::Dkim2(Dkim2Event::RecipeError) => "dkim2.recipe-error", + EventType::Dkim2(Dkim2Event::SigningFailed) => "dkim2.signing-failed", EventType::Dmarc(DmarcEvent::Pass) => "dmarc.pass", EventType::Dmarc(DmarcEvent::Fail) => "dmarc.fail", EventType::Dmarc(DmarcEvent::PermError) => "dmarc.perm-error", @@ -847,7 +824,6 @@ impl EventType { EventType::Dns(DnsEvent::RecordLookupFailed) => "dns.record-lookup-failed", EventType::Dns(DnsEvent::RecordPropagated) => "dns.record-propagated", EventType::Dns(DnsEvent::RecordPropagationTimeout) => "dns.record-propagation-timeout", - EventType::Dns(DnsEvent::BuildError) => "dns.build-error", EventType::Eval(EvalEvent::Result) => "eval.result", EventType::Eval(EvalEvent::Error) => "eval.error", EventType::Eval(EvalEvent::DirectoryNotFound) => "eval.directory-not-found", @@ -859,6 +835,7 @@ impl EventType { EventType::Http(HttpEvent::RequestBody) => "http.request-body", EventType::Http(HttpEvent::ResponseBody) => "http.response-body", EventType::Http(HttpEvent::XForwardedMissing) => "http.x-forwarded-missing", + EventType::Http(HttpEvent::XForwardedInvalid) => "http.x-forwarded-invalid", EventType::Imap(ImapEvent::ConnectionStart) => "imap.connection-start", EventType::Imap(ImapEvent::ConnectionEnd) => "imap.connection-end", EventType::Imap(ImapEvent::GetAcl) => "imap.get-acl", @@ -894,6 +871,8 @@ impl EventType { EventType::Imap(ImapEvent::Thread) => "imap.thread", EventType::Imap(ImapEvent::UidBatches) => "imap.uid-batches", EventType::Imap(ImapEvent::GetQuota) => "imap.get-quota", + EventType::Imap(ImapEvent::GetMetadata) => "imap.get-metadata", + EventType::Imap(ImapEvent::SetMetadata) => "imap.set-metadata", EventType::Imap(ImapEvent::Error) => "imap.error", EventType::Imap(ImapEvent::RawInput) => "imap.raw-input", EventType::Imap(ImapEvent::RawOutput) => "imap.raw-output", @@ -962,6 +941,10 @@ impl EventType { "jmap.account-not-supported-by-method" } EventType::Jmap(JmapEvent::AccountReadOnly) => "jmap.account-read-only", + EventType::Jmap(JmapEvent::FromAccountNotFound) => "jmap.from-account-not-found", + EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod) => { + "jmap.from-account-not-supported-by-method" + } EventType::Jmap(JmapEvent::NotFound) => "jmap.not-found", EventType::Jmap(JmapEvent::CannotCalculateChanges) => "jmap.cannot-calculate-changes", EventType::Jmap(JmapEvent::UnknownDataType) => "jmap.unknown-data-type", @@ -971,6 +954,13 @@ impl EventType { EventType::Jmap(JmapEvent::WebsocketStart) => "jmap.websocket-start", EventType::Jmap(JmapEvent::WebsocketStop) => "jmap.websocket-stop", EventType::Jmap(JmapEvent::WebsocketError) => "jmap.websocket-error", + EventType::Jmap(JmapEvent::ExpandDurationTooLarge) => "jmap.expand-duration-too-large", + EventType::Jmap(JmapEvent::CannotCalculateOccurrences) => { + "jmap.cannot-calculate-occurrences" + } + EventType::Jmap(JmapEvent::TooLarge) => "jmap.too-large", + EventType::Jmap(JmapEvent::RateLimit) => "jmap.rate-limit", + EventType::Jmap(JmapEvent::ObjectNotFound) => "jmap.object-not-found", EventType::Limit(LimitEvent::SizeRequest) => "limit.size-request", EventType::Limit(LimitEvent::SizeUpload) => "limit.size-upload", EventType::Limit(LimitEvent::CallsIn) => "limit.calls-in", @@ -979,8 +969,8 @@ impl EventType { EventType::Limit(LimitEvent::ConcurrentConnection) => "limit.concurrent-connection", EventType::Limit(LimitEvent::Quota) => "limit.quota", EventType::Limit(LimitEvent::BlobQuota) => "limit.blob-quota", - EventType::Limit(LimitEvent::TenantQuota) => "limit.tenant-quota", EventType::Limit(LimitEvent::TooManyRequests) => "limit.too-many-requests", + EventType::Limit(LimitEvent::MemoryBudget) => "limit.memory-budget", EventType::MailAuth(MailAuthEvent::ParseError) => "mail-auth.parse-error", EventType::MailAuth(MailAuthEvent::MissingParameters) => "mail-auth.missing-parameters", EventType::MailAuth(MailAuthEvent::NoHeadersFound) => "mail-auth.no-headers-found", @@ -1030,9 +1020,6 @@ impl EventType { } EventType::MessageIngest(MessageIngestEvent::Duplicate) => "message-ingest.duplicate", EventType::MessageIngest(MessageIngestEvent::Error) => "message-ingest.error", - EventType::MessageIngest(MessageIngestEvent::SearchIndex) => { - "message-ingest.search-index" - } EventType::Milter(MilterEvent::Read) => "milter.read", EventType::Milter(MilterEvent::Write) => "milter.write", EventType::Milter(MilterEvent::ActionAccept) => "milter.action-accept", @@ -1184,6 +1171,28 @@ impl EventType { EventType::Resource(ResourceEvent::ApplicationUnpacked) => { "resource.application-unpacked" } + EventType::SearchIndex(SearchIndexEvent::ManagerStarted) => { + "search-index.manager-started" + } + EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired) => { + "search-index.partitions-acquired" + } + EventType::SearchIndex(SearchIndexEvent::PartitionLocked) => { + "search-index.partition-locked" + } + EventType::SearchIndex(SearchIndexEvent::PartitionRetry) => { + "search-index.partition-retry" + } + EventType::SearchIndex(SearchIndexEvent::BatchIndexed) => "search-index.batch-indexed", + EventType::SearchIndex(SearchIndexEvent::DocumentIgnored) => { + "search-index.document-ignored" + } + EventType::SearchIndex(SearchIndexEvent::DocumentTruncated) => { + "search-index.document-truncated" + } + EventType::SearchIndex(SearchIndexEvent::MetadataNotFound) => { + "search-index.metadata-not-found" + } EventType::Security(SecurityEvent::AuthenticationBan) => "security.authentication-ban", EventType::Security(SecurityEvent::AbuseBan) => "security.abuse-ban", EventType::Security(SecurityEvent::ScanBan) => "security.scan-ban", @@ -1197,7 +1206,6 @@ impl EventType { EventType::Server(ServerEvent::Shutdown) => "server.shutdown", EventType::Server(ServerEvent::StartupError) => "server.startup-error", EventType::Server(ServerEvent::ThreadError) => "server.thread-error", - EventType::Server(ServerEvent::Licensing) => "server.licensing", EventType::Server(ServerEvent::RecoveryMode) => "server.recovery-mode", EventType::Server(ServerEvent::BootstrapMode) => "server.bootstrap-mode", EventType::Sieve(SieveEvent::ActionAccept) => "sieve.action-accept", @@ -1222,15 +1230,15 @@ impl EventType { EventType::Smtp(SmtpEvent::TransferLimitExceeded) => "smtp.transfer-limit-exceeded", EventType::Smtp(SmtpEvent::RateLimitExceeded) => "smtp.rate-limit-exceeded", EventType::Smtp(SmtpEvent::TimeLimitExceeded) => "smtp.time-limit-exceeded", - EventType::Smtp(SmtpEvent::MissingAuthDirectory) => "smtp.missing-auth-directory", EventType::Smtp(SmtpEvent::MessageParseFailed) => "smtp.message-parse-failed", EventType::Smtp(SmtpEvent::MessageTooLarge) => "smtp.message-too-large", EventType::Smtp(SmtpEvent::LoopDetected) => "smtp.loop-detected", + EventType::Smtp(SmtpEvent::MultipleFromDomains) => "smtp.multiple-from-domains", EventType::Smtp(SmtpEvent::DkimPass) => "smtp.dkim-pass", EventType::Smtp(SmtpEvent::DkimFail) => "smtp.dkim-fail", EventType::Smtp(SmtpEvent::Dkim2Pass) => "smtp.dkim2-pass", EventType::Smtp(SmtpEvent::Dkim2Fail) => "smtp.dkim2-fail", - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded) => "smtp.dkim2-dsn-discarded", + EventType::Smtp(SmtpEvent::Dkim2DsnRejected) => "smtp.dkim2-dsn-rejected", EventType::Smtp(SmtpEvent::ArcPass) => "smtp.arc-pass", EventType::Smtp(SmtpEvent::ArcFail) => "smtp.arc-fail", EventType::Smtp(SmtpEvent::SpfEhloPass) => "smtp.spf-ehlo-pass", @@ -1344,11 +1352,6 @@ impl EventType { EventType::Store(StoreEvent::UnexpectedError) => "store.unexpected-error", EventType::Store(StoreEvent::CryptoError) => "store.crypto-error", EventType::Store(StoreEvent::HttpStoreError) => "store.http-store-error", - EventType::Store(StoreEvent::CacheMiss) => "store.cache-miss", - EventType::Store(StoreEvent::CacheHit) => "store.cache-hit", - EventType::Store(StoreEvent::CacheStale) => "store.cache-stale", - EventType::Store(StoreEvent::CacheUpdate) => "store.cache-update", - EventType::Store(StoreEvent::CacheEntryTooLarge) => "store.cache-entry-too-large", EventType::Store(StoreEvent::BlobMissingMarker) => "store.blob-missing-marker", EventType::Store(StoreEvent::DataWrite) => "store.data-write", EventType::Store(StoreEvent::DataIterate) => "store.data-iterate", @@ -1371,6 +1374,8 @@ impl EventType { EventType::TaskManager(TaskManagerEvent::TaskIgnored) => "task-manager.task-ignored", EventType::TaskManager(TaskManagerEvent::TaskFailed) => "task-manager.task-failed", EventType::TaskManager(TaskManagerEvent::TaskRetry) => "task-manager.task-retry", + EventType::TaskManager(TaskManagerEvent::TaskStalled) => "task-manager.task-stalled", + EventType::TaskManager(TaskManagerEvent::TaskExpired) => "task-manager.task-expired", EventType::TaskManager(TaskManagerEvent::BlobNotFound) => "task-manager.blob-not-found", EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => { "task-manager.metadata-not-found" @@ -1381,8 +1386,6 @@ impl EventType { EventType::TaskManager(TaskManagerEvent::ManagerStarted) => { "task-manager.manager-started" } - EventType::Telemetry(TelemetryEvent::AlertEvent) => "telemetry.alert-event", - EventType::Telemetry(TelemetryEvent::AlertMessage) => "telemetry.alert-message", EventType::Telemetry(TelemetryEvent::LogError) => "telemetry.log-error", EventType::Telemetry(TelemetryEvent::WebhookError) => "telemetry.webhook-error", EventType::Telemetry(TelemetryEvent::OtelExporterError) => { @@ -1395,8 +1398,8 @@ impl EventType { "telemetry.prometheus-exporter-error" } EventType::Telemetry(TelemetryEvent::JournalError) => "telemetry.journal-error", + EventType::Telemetry(TelemetryEvent::EventsDropped) => "telemetry.events-dropped", EventType::Telemetry(TelemetryEvent::MetricsCollected) => "telemetry.metrics-collected", - EventType::Telemetry(TelemetryEvent::MetricsStored) => "telemetry.metrics-stored", EventType::Telemetry(TelemetryEvent::MetricsPushed) => "telemetry.metrics-pushed", EventType::Tls(TlsEvent::Handshake) => "tls.handshake", EventType::Tls(TlsEvent::HandshakeError) => "tls.handshake-error", @@ -1455,147 +1458,132 @@ impl EventType { EventType::Acme(AcmeEvent::TlsAlpnError) => 24, EventType::Acme(AcmeEvent::TokenNotFound) => 26, EventType::Acme(AcmeEvent::Error) => 15, - EventType::Ai(AiEvent::LlmResponse) => 556, - EventType::Ai(AiEvent::ApiError) => 557, - EventType::Arc(ArcEvent::ChainTooLong) => 28, - EventType::Arc(ArcEvent::InvalidInstance) => 31, - EventType::Arc(ArcEvent::InvalidCv) => 30, - EventType::Arc(ArcEvent::HasHeaderTag) => 29, - EventType::Arc(ArcEvent::BrokenChain) => 27, - EventType::Arc(ArcEvent::SealerNotFound) => 32, - EventType::Auth(AuthEvent::Success) => 37, - EventType::Auth(AuthEvent::Failed) => 35, - EventType::Auth(AuthEvent::TokenExpired) => 554, - EventType::Auth(AuthEvent::MfaRequired) => 36, - EventType::Auth(AuthEvent::TooManyAttempts) => 38, - EventType::Auth(AuthEvent::ClientRegistration) => 555, - EventType::Auth(AuthEvent::Error) => 34, - EventType::Auth(AuthEvent::Warning) => 595, - EventType::Auth(AuthEvent::CredentialExpired) => 276, - EventType::Calendar(CalendarEvent::RuleExpansionError) => 576, - EventType::Calendar(CalendarEvent::AlarmSent) => 579, - EventType::Calendar(CalendarEvent::AlarmSkipped) => 580, - EventType::Calendar(CalendarEvent::AlarmRecipientOverride) => 581, - EventType::Calendar(CalendarEvent::AlarmFailed) => 582, - EventType::Calendar(CalendarEvent::ItipMessageSent) => 583, - EventType::Calendar(CalendarEvent::ItipMessageReceived) => 584, - EventType::Calendar(CalendarEvent::ItipMessageError) => 585, - EventType::Cluster(ClusterEvent::Startup) => 278, - EventType::Cluster(ClusterEvent::SubscriberStart) => 39, - EventType::Cluster(ClusterEvent::SubscriberStop) => 40, - EventType::Cluster(ClusterEvent::SubscriberError) => 41, - EventType::Cluster(ClusterEvent::SubscriberDisconnected) => 42, - EventType::Cluster(ClusterEvent::PublisherStart) => 43, - EventType::Cluster(ClusterEvent::PublisherStop) => 44, - EventType::Cluster(ClusterEvent::PublisherError) => 45, - EventType::Cluster(ClusterEvent::MessageReceived) => 46, - EventType::Cluster(ClusterEvent::MessageSkipped) => 47, - EventType::Cluster(ClusterEvent::MessageInvalid) => 49, - EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275, - EventType::Dane(DaneEvent::AuthenticationSuccess) => 67, - EventType::Dane(DaneEvent::AuthenticationFailure) => 66, - EventType::Dane(DaneEvent::NoCertificatesFound) => 69, - EventType::Dane(DaneEvent::CertificateParseError) => 68, - EventType::Dane(DaneEvent::TlsaRecordMatch) => 73, - EventType::Dane(DaneEvent::TlsaRecordFetch) => 70, - EventType::Dane(DaneEvent::TlsaRecordFetchError) => 71, - EventType::Dane(DaneEvent::TlsaRecordNotFound) => 75, - EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned) => 74, - EventType::Dane(DaneEvent::TlsaRecordInvalid) => 72, - EventType::Dane(DaneEvent::BogusDnssecRecord) => 605, - EventType::Delivery(DeliveryEvent::AttemptStart) => 77, - EventType::Delivery(DeliveryEvent::AttemptEnd) => 76, - EventType::Delivery(DeliveryEvent::Completed) => 80, - EventType::Delivery(DeliveryEvent::Failed) => 92, - EventType::Delivery(DeliveryEvent::DomainDeliveryStart) => 85, - EventType::Delivery(DeliveryEvent::MxLookup) => 101, - EventType::Delivery(DeliveryEvent::MxLookupFailed) => 102, - EventType::Delivery(DeliveryEvent::IpLookup) => 95, - EventType::Delivery(DeliveryEvent::IpLookupFailed) => 96, - EventType::Delivery(DeliveryEvent::NullMx) => 103, - EventType::Delivery(DeliveryEvent::Connect) => 82, - EventType::Delivery(DeliveryEvent::ConnectError) => 83, - EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => 100, - EventType::Delivery(DeliveryEvent::GreetingFailed) => 93, - EventType::Delivery(DeliveryEvent::Ehlo) => 90, - EventType::Delivery(DeliveryEvent::EhloRejected) => 91, - EventType::Delivery(DeliveryEvent::Auth) => 78, - EventType::Delivery(DeliveryEvent::AuthFailed) => 79, - EventType::Delivery(DeliveryEvent::MailFrom) => 97, - EventType::Delivery(DeliveryEvent::MailFromRejected) => 98, - EventType::Delivery(DeliveryEvent::Delivered) => 84, - EventType::Delivery(DeliveryEvent::RcptTo) => 107, - EventType::Delivery(DeliveryEvent::RcptToRejected) => 109, - EventType::Delivery(DeliveryEvent::RcptToFailed) => 108, - EventType::Delivery(DeliveryEvent::MessageRejected) => 99, - EventType::Delivery(DeliveryEvent::StartTls) => 110, - EventType::Delivery(DeliveryEvent::StartTlsUnavailable) => 113, - EventType::Delivery(DeliveryEvent::StartTlsError) => 112, - EventType::Delivery(DeliveryEvent::StartTlsDisabled) => 111, - EventType::Delivery(DeliveryEvent::ImplicitTlsError) => 94, - EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => 81, - EventType::Delivery(DeliveryEvent::RateLimitExceeded) => 104, - EventType::Delivery(DeliveryEvent::DoubleBounce) => 86, - EventType::Delivery(DeliveryEvent::DsnSuccess) => 88, - EventType::Delivery(DeliveryEvent::DsnTempFail) => 89, - EventType::Delivery(DeliveryEvent::DsnPermFail) => 87, - EventType::Delivery(DeliveryEvent::RawInput) => 105, - EventType::Delivery(DeliveryEvent::RawOutput) => 106, - EventType::Dkim(DkimEvent::Pass) => 121, - EventType::Dkim(DkimEvent::Neutral) => 119, - EventType::Dkim(DkimEvent::Fail) => 114, - EventType::Dkim(DkimEvent::PermError) => 122, - EventType::Dkim(DkimEvent::TempError) => 127, - EventType::Dkim(DkimEvent::None) => 120, - EventType::Dkim(DkimEvent::UnsupportedVersion) => 131, - EventType::Dkim(DkimEvent::UnsupportedAlgorithm) => 128, - EventType::Dkim(DkimEvent::UnsupportedCanonicalization) => 129, - EventType::Dkim(DkimEvent::UnsupportedKeyType) => 130, - EventType::Dkim(DkimEvent::FailedBodyHashMatch) => 116, - EventType::Dkim(DkimEvent::FailedVerification) => 117, - EventType::Dkim(DkimEvent::FailedAuidMatch) => 115, - EventType::Dkim(DkimEvent::RevokedPublicKey) => 123, - EventType::Dkim(DkimEvent::IncompatibleAlgorithms) => 118, - EventType::Dkim(DkimEvent::SignatureExpired) => 124, - EventType::Dkim(DkimEvent::SignatureLength) => 125, - EventType::Dkim(DkimEvent::SignerNotFound) => 126, - EventType::Dkim(DkimEvent::BuildError) => 592, - EventType::Dkim(DkimEvent::SignatureCreated) => 596, - EventType::Dkim(DkimEvent::SignaturePublished) => 597, - EventType::Dkim(DkimEvent::SignatureRetiring) => 598, - EventType::Dkim(DkimEvent::SignatureRetired) => 599, - EventType::Dkim(DkimEvent::SignatureDeleted) => 600, - EventType::Dkim(DkimEvent::InstanceMissing) => 606, - EventType::Dkim(DkimEvent::InstanceSyntax) => 607, - EventType::Dkim(DkimEvent::InstanceTagMissing) => 608, - EventType::Dkim(DkimEvent::InstanceNotSigned) => 609, - EventType::Dkim(DkimEvent::InstanceAboveSignature) => 610, - EventType::Dkim(DkimEvent::SignatureMissing) => 611, - EventType::Dkim(DkimEvent::SignatureSyntax) => 612, - EventType::Dkim(DkimEvent::SignatureTagMissing) => 613, - EventType::Dkim(DkimEvent::SignatureTagUnexpected) => 614, - EventType::Dkim(DkimEvent::SequenceGap) => 615, - EventType::Dkim(DkimEvent::SequenceOverflow) => 616, - EventType::Dkim(DkimEvent::ChainTooLong) => 635, - EventType::Dkim(DkimEvent::MailFromMismatch) => 617, - EventType::Dkim(DkimEvent::RcptToMismatch) => 618, - EventType::Dkim(DkimEvent::MailFromDomainMismatch) => 619, - EventType::Dkim(DkimEvent::NextDomainMismatch) => 620, - EventType::Dkim(DkimEvent::CustodyBreak) => 636, - EventType::Dkim(DkimEvent::PublicKeyFetch) => 621, - EventType::Dkim(DkimEvent::PublicKeyMissing) => 622, - EventType::Dkim(DkimEvent::PublicKeyMultiple) => 623, - EventType::Dkim(DkimEvent::PublicKeySyntax) => 624, - EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch) => 625, - EventType::Dkim(DkimEvent::NoValidAlgorithm) => 626, - EventType::Dkim(DkimEvent::HeaderHashMismatch) => 627, - EventType::Dkim(DkimEvent::Modified) => 628, - EventType::Dkim(DkimEvent::Exploded) => 629, - EventType::Dmarc(DmarcEvent::Pass) => 134, - EventType::Dmarc(DmarcEvent::Fail) => 132, - EventType::Dmarc(DmarcEvent::PermError) => 135, - EventType::Dmarc(DmarcEvent::TempError) => 136, - EventType::Dmarc(DmarcEvent::None) => 133, + EventType::Arc(ArcEvent::ChainInvalid) => 647, + EventType::Arc(ArcEvent::SignatureInvalid) => 648, + EventType::Arc(ArcEvent::HashMismatch) => 649, + EventType::Auth(AuthEvent::Success) => 32, + EventType::Auth(AuthEvent::Failed) => 30, + EventType::Auth(AuthEvent::TokenExpired) => 537, + EventType::Auth(AuthEvent::MfaRequired) => 31, + EventType::Auth(AuthEvent::TooManyAttempts) => 33, + EventType::Auth(AuthEvent::ClientRegistration) => 538, + EventType::Auth(AuthEvent::Error) => 29, + EventType::Auth(AuthEvent::Warning) => 578, + EventType::Auth(AuthEvent::CredentialExpired) => 260, + EventType::Cache(CacheEvent::Miss) => 45, + EventType::Cache(CacheEvent::Hit) => 46, + EventType::Cache(CacheEvent::Stale) => 47, + EventType::Cache(CacheEvent::Update) => 560, + EventType::Cache(CacheEvent::EntryTooLarge) => 592, + EventType::Cache(CacheEvent::Invalidate) => 600, + EventType::Cache(CacheEvent::SwapHit) => 594, + EventType::Cache(CacheEvent::SwapMiss) => 595, + EventType::Cache(CacheEvent::SwapWrite) => 596, + EventType::Cache(CacheEvent::SwapError) => 597, + EventType::Cache(CacheEvent::SwapMetrics) => 598, + EventType::Cache(CacheEvent::SwapFlush) => 599, + EventType::Calendar(CalendarEvent::RuleExpansionError) => 559, + EventType::Calendar(CalendarEvent::AlarmSent) => 562, + EventType::Calendar(CalendarEvent::AlarmSkipped) => 563, + EventType::Calendar(CalendarEvent::AlarmRecipientOverride) => 564, + EventType::Calendar(CalendarEvent::AlarmFailed) => 565, + EventType::Calendar(CalendarEvent::ItipMessageSent) => 566, + EventType::Calendar(CalendarEvent::ItipMessageReceived) => 567, + EventType::Calendar(CalendarEvent::ItipMessageError) => 568, + EventType::Cluster(ClusterEvent::Startup) => 262, + EventType::Cluster(ClusterEvent::SubscriberStart) => 34, + EventType::Cluster(ClusterEvent::SubscriberStop) => 35, + EventType::Cluster(ClusterEvent::SubscriberError) => 36, + EventType::Cluster(ClusterEvent::SubscriberDisconnected) => 37, + EventType::Cluster(ClusterEvent::PublisherStart) => 38, + EventType::Cluster(ClusterEvent::PublisherStop) => 39, + EventType::Cluster(ClusterEvent::PublisherError) => 40, + EventType::Cluster(ClusterEvent::MessageReceived) => 41, + EventType::Cluster(ClusterEvent::MessageSkipped) => 42, + EventType::Cluster(ClusterEvent::MessageInvalid) => 44, + EventType::Cluster(ClusterEvent::NodeIdRenewed) => 259, + EventType::Dane(DaneEvent::AuthenticationSuccess) => 62, + EventType::Dane(DaneEvent::AuthenticationFailure) => 61, + EventType::Dane(DaneEvent::NoCertificatesFound) => 64, + EventType::Dane(DaneEvent::CertificateParseError) => 63, + EventType::Dane(DaneEvent::TlsaRecordMatch) => 68, + EventType::Dane(DaneEvent::TlsaRecordFetch) => 65, + EventType::Dane(DaneEvent::TlsaRecordFetchError) => 66, + EventType::Dane(DaneEvent::TlsaRecordNotFound) => 70, + EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned) => 69, + EventType::Dane(DaneEvent::TlsaRecordInvalid) => 67, + EventType::Dane(DaneEvent::BogusDnssecRecord) => 588, + EventType::Delivery(DeliveryEvent::AttemptStart) => 72, + EventType::Delivery(DeliveryEvent::AttemptEnd) => 71, + EventType::Delivery(DeliveryEvent::Completed) => 75, + EventType::Delivery(DeliveryEvent::Failed) => 87, + EventType::Delivery(DeliveryEvent::DomainDeliveryStart) => 80, + EventType::Delivery(DeliveryEvent::MxLookup) => 96, + EventType::Delivery(DeliveryEvent::MxLookupFailed) => 97, + EventType::Delivery(DeliveryEvent::IpLookup) => 90, + EventType::Delivery(DeliveryEvent::IpLookupFailed) => 91, + EventType::Delivery(DeliveryEvent::NullMx) => 98, + EventType::Delivery(DeliveryEvent::Connect) => 77, + EventType::Delivery(DeliveryEvent::ConnectError) => 78, + EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => 95, + EventType::Delivery(DeliveryEvent::GreetingFailed) => 88, + EventType::Delivery(DeliveryEvent::Ehlo) => 85, + EventType::Delivery(DeliveryEvent::EhloRejected) => 86, + EventType::Delivery(DeliveryEvent::Auth) => 73, + EventType::Delivery(DeliveryEvent::AuthFailed) => 74, + EventType::Delivery(DeliveryEvent::MailFrom) => 92, + EventType::Delivery(DeliveryEvent::MailFromRejected) => 93, + EventType::Delivery(DeliveryEvent::Delivered) => 79, + EventType::Delivery(DeliveryEvent::RcptTo) => 102, + EventType::Delivery(DeliveryEvent::RcptToRejected) => 104, + EventType::Delivery(DeliveryEvent::RcptToFailed) => 103, + EventType::Delivery(DeliveryEvent::MessageRejected) => 94, + EventType::Delivery(DeliveryEvent::StartTls) => 105, + EventType::Delivery(DeliveryEvent::StartTlsUnavailable) => 108, + EventType::Delivery(DeliveryEvent::StartTlsError) => 107, + EventType::Delivery(DeliveryEvent::StartTlsDisabled) => 106, + EventType::Delivery(DeliveryEvent::ImplicitTlsError) => 89, + EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => 76, + EventType::Delivery(DeliveryEvent::RateLimitExceeded) => 99, + EventType::Delivery(DeliveryEvent::DoubleBounce) => 81, + EventType::Delivery(DeliveryEvent::DsnSuccess) => 83, + EventType::Delivery(DeliveryEvent::DsnTempFail) => 84, + EventType::Delivery(DeliveryEvent::DsnPermFail) => 82, + EventType::Delivery(DeliveryEvent::RawInput) => 100, + EventType::Delivery(DeliveryEvent::RawOutput) => 101, + EventType::Dkim(DkimEvent::Pass) => 112, + EventType::Dkim(DkimEvent::Neutral) => 110, + EventType::Dkim(DkimEvent::Fail) => 109, + EventType::Dkim(DkimEvent::PermError) => 113, + EventType::Dkim(DkimEvent::TempError) => 115, + EventType::Dkim(DkimEvent::None) => 111, + EventType::Dkim(DkimEvent::SignatureInvalid) => 643, + EventType::Dkim(DkimEvent::PublicKeyError) => 644, + EventType::Dkim(DkimEvent::VerificationFailed) => 645, + EventType::Dkim(DkimEvent::HashMismatch) => 646, + EventType::Dkim(DkimEvent::SignerNotFound) => 114, + EventType::Dkim(DkimEvent::BuildError) => 575, + EventType::Dkim(DkimEvent::SignatureCreated) => 579, + EventType::Dkim(DkimEvent::SignaturePublished) => 580, + EventType::Dkim(DkimEvent::SignatureRetiring) => 581, + EventType::Dkim(DkimEvent::SignatureRetired) => 582, + EventType::Dkim(DkimEvent::SignatureDeleted) => 583, + EventType::Dkim2(Dkim2Event::InstanceInvalid) => 633, + EventType::Dkim2(Dkim2Event::SignatureInvalid) => 634, + EventType::Dkim2(Dkim2Event::ChainInvalid) => 635, + EventType::Dkim2(Dkim2Event::TimestampInvalid) => 636, + EventType::Dkim2(Dkim2Event::EnvelopeMismatch) => 637, + EventType::Dkim2(Dkim2Event::CustodyBroken) => 638, + EventType::Dkim2(Dkim2Event::PublicKeyError) => 639, + EventType::Dkim2(Dkim2Event::VerificationFailed) => 640, + EventType::Dkim2(Dkim2Event::HashMismatch) => 641, + EventType::Dkim2(Dkim2Event::RecipeError) => 642, + EventType::Dkim2(Dkim2Event::SigningFailed) => 650, + EventType::Dmarc(DmarcEvent::Pass) => 118, + EventType::Dmarc(DmarcEvent::Fail) => 116, + EventType::Dmarc(DmarcEvent::PermError) => 119, + EventType::Dmarc(DmarcEvent::TempError) => 120, + EventType::Dmarc(DmarcEvent::None) => 117, EventType::Dns(DnsEvent::RecordCreated) => 8, EventType::Dns(DnsEvent::RecordCreationFailed) => 9, EventType::Dns(DnsEvent::RecordDeletionFailed) => 10, @@ -1603,475 +1591,485 @@ impl EventType { EventType::Dns(DnsEvent::RecordLookupFailed) => 11, EventType::Dns(DnsEvent::RecordPropagated) => 13, EventType::Dns(DnsEvent::RecordPropagationTimeout) => 14, - EventType::Dns(DnsEvent::BuildError) => 591, - EventType::Eval(EvalEvent::Result) => 139, - EventType::Eval(EvalEvent::Error) => 138, - EventType::Eval(EvalEvent::DirectoryNotFound) => 137, - EventType::Eval(EvalEvent::StoreNotFound) => 140, - EventType::Http(HttpEvent::ConnectionStart) => 153, - EventType::Http(HttpEvent::ConnectionEnd) => 152, - EventType::Http(HttpEvent::Error) => 154, - EventType::Http(HttpEvent::RequestUrl) => 156, - EventType::Http(HttpEvent::RequestBody) => 155, - EventType::Http(HttpEvent::ResponseBody) => 157, - EventType::Http(HttpEvent::XForwardedMissing) => 158, - EventType::Imap(ImapEvent::ConnectionStart) => 163, - EventType::Imap(ImapEvent::ConnectionEnd) => 162, - EventType::Imap(ImapEvent::GetAcl) => 171, - EventType::Imap(ImapEvent::SetAcl) => 188, - EventType::Imap(ImapEvent::MyRights) => 180, - EventType::Imap(ImapEvent::ListRights) => 176, - EventType::Imap(ImapEvent::Append) => 159, - EventType::Imap(ImapEvent::Capabilities) => 160, - EventType::Imap(ImapEvent::Id) => 172, - EventType::Imap(ImapEvent::Close) => 161, - EventType::Imap(ImapEvent::Copy) => 164, - EventType::Imap(ImapEvent::Move) => 179, - EventType::Imap(ImapEvent::CreateMailbox) => 165, - EventType::Imap(ImapEvent::DeleteMailbox) => 166, - EventType::Imap(ImapEvent::RenameMailbox) => 185, - EventType::Imap(ImapEvent::Enable) => 167, - EventType::Imap(ImapEvent::Expunge) => 169, - EventType::Imap(ImapEvent::Fetch) => 170, - EventType::Imap(ImapEvent::IdleStart) => 173, - EventType::Imap(ImapEvent::IdleStop) => 174, - EventType::Imap(ImapEvent::List) => 175, - EventType::Imap(ImapEvent::Lsub) => 178, - EventType::Imap(ImapEvent::Logout) => 177, - EventType::Imap(ImapEvent::Namespace) => 181, - EventType::Imap(ImapEvent::Noop) => 182, - EventType::Imap(ImapEvent::Search) => 186, - EventType::Imap(ImapEvent::Sort) => 189, - EventType::Imap(ImapEvent::Select) => 187, - EventType::Imap(ImapEvent::Status) => 190, - EventType::Imap(ImapEvent::Store) => 191, - EventType::Imap(ImapEvent::Subscribe) => 192, - EventType::Imap(ImapEvent::Unsubscribe) => 194, - EventType::Imap(ImapEvent::Thread) => 193, - EventType::Imap(ImapEvent::UidBatches) => 634, - EventType::Imap(ImapEvent::GetQuota) => 57, - EventType::Imap(ImapEvent::Error) => 168, - EventType::Imap(ImapEvent::RawInput) => 183, - EventType::Imap(ImapEvent::RawOutput) => 184, - EventType::IncomingReport(IncomingReportEvent::DmarcReport) => 200, - EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings) => 201, - EventType::IncomingReport(IncomingReportEvent::TlsReport) => 206, - EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings) => 207, - EventType::IncomingReport(IncomingReportEvent::AbuseReport) => 195, - EventType::IncomingReport(IncomingReportEvent::AuthFailureReport) => 197, - EventType::IncomingReport(IncomingReportEvent::FraudReport) => 202, - EventType::IncomingReport(IncomingReportEvent::NotSpamReport) => 204, - EventType::IncomingReport(IncomingReportEvent::VirusReport) => 209, - EventType::IncomingReport(IncomingReportEvent::OtherReport) => 205, - EventType::IncomingReport(IncomingReportEvent::MessageParseFailed) => 203, - EventType::IncomingReport(IncomingReportEvent::DmarcParseFailed) => 199, - EventType::IncomingReport(IncomingReportEvent::TlsRpcParseFailed) => 208, - EventType::IncomingReport(IncomingReportEvent::ArfParseFailed) => 196, - EventType::IncomingReport(IncomingReportEvent::DecompressError) => 198, - EventType::Iprev(IprevEvent::Pass) => 212, - EventType::Iprev(IprevEvent::Fail) => 210, - EventType::Iprev(IprevEvent::PermError) => 213, - EventType::Iprev(IprevEvent::TempError) => 214, - EventType::Iprev(IprevEvent::None) => 211, - EventType::Jmap(JmapEvent::MethodCall) => 223, - EventType::Jmap(JmapEvent::InvalidArguments) => 221, - EventType::Jmap(JmapEvent::RequestTooLarge) => 227, - EventType::Jmap(JmapEvent::StateMismatch) => 228, - EventType::Jmap(JmapEvent::AnchorNotFound) => 218, - EventType::Jmap(JmapEvent::UnsupportedFilter) => 232, - EventType::Jmap(JmapEvent::UnsupportedSort) => 233, - EventType::Jmap(JmapEvent::UnknownMethod) => 231, - EventType::Jmap(JmapEvent::InvalidResultReference) => 222, - EventType::Jmap(JmapEvent::Forbidden) => 220, - EventType::Jmap(JmapEvent::AccountNotFound) => 215, - EventType::Jmap(JmapEvent::AccountNotSupportedByMethod) => 216, - EventType::Jmap(JmapEvent::AccountReadOnly) => 217, - EventType::Jmap(JmapEvent::NotFound) => 224, - EventType::Jmap(JmapEvent::CannotCalculateChanges) => 219, - EventType::Jmap(JmapEvent::UnknownDataType) => 230, - EventType::Jmap(JmapEvent::UnknownCapability) => 229, - EventType::Jmap(JmapEvent::NotJson) => 225, - EventType::Jmap(JmapEvent::NotRequest) => 226, - EventType::Jmap(JmapEvent::WebsocketStart) => 235, - EventType::Jmap(JmapEvent::WebsocketStop) => 236, - EventType::Jmap(JmapEvent::WebsocketError) => 234, - EventType::Limit(LimitEvent::SizeRequest) => 243, - EventType::Limit(LimitEvent::SizeUpload) => 244, - EventType::Limit(LimitEvent::CallsIn) => 238, - EventType::Limit(LimitEvent::ConcurrentRequest) => 240, - EventType::Limit(LimitEvent::ConcurrentUpload) => 241, - EventType::Limit(LimitEvent::ConcurrentConnection) => 239, - EventType::Limit(LimitEvent::Quota) => 242, - EventType::Limit(LimitEvent::BlobQuota) => 237, - EventType::Limit(LimitEvent::TenantQuota) => 553, - EventType::Limit(LimitEvent::TooManyRequests) => 245, - EventType::MailAuth(MailAuthEvent::ParseError) => 254, - EventType::MailAuth(MailAuthEvent::MissingParameters) => 252, - EventType::MailAuth(MailAuthEvent::NoHeadersFound) => 253, - EventType::MailAuth(MailAuthEvent::Crypto) => 247, - EventType::MailAuth(MailAuthEvent::Io) => 251, - EventType::MailAuth(MailAuthEvent::Base64) => 246, - EventType::MailAuth(MailAuthEvent::DnsError) => 248, - EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) => 250, - EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType) => 249, - EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => 255, - EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => 259, - EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => 258, - EventType::ManageSieve(ManageSieveEvent::CreateScript) => 260, - EventType::ManageSieve(ManageSieveEvent::UpdateScript) => 274, - EventType::ManageSieve(ManageSieveEvent::GetScript) => 263, - EventType::ManageSieve(ManageSieveEvent::DeleteScript) => 261, - EventType::ManageSieve(ManageSieveEvent::RenameScript) => 270, - EventType::ManageSieve(ManageSieveEvent::CheckScript) => 257, - EventType::ManageSieve(ManageSieveEvent::HaveSpace) => 264, - EventType::ManageSieve(ManageSieveEvent::ListScripts) => 265, - EventType::ManageSieve(ManageSieveEvent::SetActive) => 271, - EventType::ManageSieve(ManageSieveEvent::Capabilities) => 256, - EventType::ManageSieve(ManageSieveEvent::StartTls) => 272, - EventType::ManageSieve(ManageSieveEvent::Unauthenticate) => 273, - EventType::ManageSieve(ManageSieveEvent::Logout) => 266, - EventType::ManageSieve(ManageSieveEvent::Noop) => 267, - EventType::ManageSieve(ManageSieveEvent::Error) => 262, - EventType::ManageSieve(ManageSieveEvent::RawInput) => 268, - EventType::ManageSieve(ManageSieveEvent::RawOutput) => 269, - EventType::MessageIngest(MessageIngestEvent::Ham) => 283, - EventType::MessageIngest(MessageIngestEvent::Spam) => 286, - EventType::MessageIngest(MessageIngestEvent::ImapAppend) => 284, - EventType::MessageIngest(MessageIngestEvent::JmapAppend) => 285, - EventType::MessageIngest(MessageIngestEvent::Duplicate) => 281, - EventType::MessageIngest(MessageIngestEvent::Error) => 282, - EventType::MessageIngest(MessageIngestEvent::SearchIndex) => 142, - EventType::Milter(MilterEvent::Read) => 299, - EventType::Milter(MilterEvent::Write) => 303, - EventType::Milter(MilterEvent::ActionAccept) => 287, - EventType::Milter(MilterEvent::ActionDiscard) => 289, - EventType::Milter(MilterEvent::ActionReject) => 290, - EventType::Milter(MilterEvent::ActionTempFail) => 293, - EventType::Milter(MilterEvent::ActionReplyCode) => 291, - EventType::Milter(MilterEvent::ActionConnectionFailure) => 288, - EventType::Milter(MilterEvent::ActionShutdown) => 292, - EventType::Milter(MilterEvent::IoError) => 297, - EventType::Milter(MilterEvent::FrameTooLarge) => 296, - EventType::Milter(MilterEvent::FrameInvalid) => 295, - EventType::Milter(MilterEvent::UnexpectedResponse) => 302, - EventType::Milter(MilterEvent::Timeout) => 300, - EventType::Milter(MilterEvent::TlsInvalidName) => 301, - EventType::Milter(MilterEvent::Disconnected) => 294, - EventType::Milter(MilterEvent::ParseError) => 298, - EventType::MtaHook(MtaHookEvent::ActionAccept) => 304, - EventType::MtaHook(MtaHookEvent::ActionDiscard) => 305, - EventType::MtaHook(MtaHookEvent::ActionReject) => 307, - EventType::MtaHook(MtaHookEvent::ActionQuarantine) => 306, - EventType::MtaHook(MtaHookEvent::Error) => 308, - EventType::MtaSts(MtaStsEvent::Authorized) => 309, - EventType::MtaSts(MtaStsEvent::NotAuthorized) => 311, - EventType::MtaSts(MtaStsEvent::PolicyFetch) => 312, - EventType::MtaSts(MtaStsEvent::PolicyNotFound) => 314, - EventType::MtaSts(MtaStsEvent::PolicyFetchError) => 313, - EventType::MtaSts(MtaStsEvent::InvalidPolicy) => 310, - EventType::Network(NetworkEvent::ListenStart) => 321, - EventType::Network(NetworkEvent::ListenStop) => 322, - EventType::Network(NetworkEvent::ListenError) => 320, - EventType::Network(NetworkEvent::BindError) => 316, - EventType::Network(NetworkEvent::ReadError) => 324, - EventType::Network(NetworkEvent::WriteError) => 328, - EventType::Network(NetworkEvent::FlushError) => 319, - EventType::Network(NetworkEvent::AcceptError) => 315, - EventType::Network(NetworkEvent::SplitError) => 326, - EventType::Network(NetworkEvent::Timeout) => 327, - EventType::Network(NetworkEvent::Closed) => 317, - EventType::Network(NetworkEvent::ProxyError) => 323, - EventType::Network(NetworkEvent::SetOptError) => 325, - EventType::OutgoingReport(OutgoingReportEvent::SpfReport) => 342, - EventType::OutgoingReport(OutgoingReportEvent::SpfRateLimited) => 341, - EventType::OutgoingReport(OutgoingReportEvent::DkimReport) => 330, - EventType::OutgoingReport(OutgoingReportEvent::DkimRateLimited) => 329, - EventType::OutgoingReport(OutgoingReportEvent::DmarcReport) => 333, - EventType::OutgoingReport(OutgoingReportEvent::DmarcRateLimited) => 332, - EventType::OutgoingReport(OutgoingReportEvent::DmarcAggregateReport) => 331, - EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate) => 344, - EventType::OutgoingReport(OutgoingReportEvent::HttpSubmission) => 334, - EventType::OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress) => 345, - EventType::OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError) => 340, - EventType::OutgoingReport(OutgoingReportEvent::NotFound) => 339, - EventType::OutgoingReport(OutgoingReportEvent::SubmissionError) => 343, - EventType::OutgoingReport(OutgoingReportEvent::NoRecipientsFound) => 338, - EventType::OutgoingReport(OutgoingReportEvent::Locked) => 337, - EventType::OutgoingReport(OutgoingReportEvent::MaxSizeExceeded) => 59, - EventType::Pop3(Pop3Event::ConnectionStart) => 348, - EventType::Pop3(Pop3Event::ConnectionEnd) => 347, - EventType::Pop3(Pop3Event::Delete) => 349, - EventType::Pop3(Pop3Event::Reset) => 358, - EventType::Pop3(Pop3Event::Quit) => 355, - EventType::Pop3(Pop3Event::Fetch) => 351, - EventType::Pop3(Pop3Event::List) => 352, - EventType::Pop3(Pop3Event::ListMessage) => 353, - EventType::Pop3(Pop3Event::Uidl) => 361, - EventType::Pop3(Pop3Event::UidlMessage) => 362, - EventType::Pop3(Pop3Event::Stat) => 360, - EventType::Pop3(Pop3Event::Noop) => 354, - EventType::Pop3(Pop3Event::Capabilities) => 346, - EventType::Pop3(Pop3Event::StartTls) => 359, - EventType::Pop3(Pop3Event::Utf8) => 363, - EventType::Pop3(Pop3Event::Error) => 350, - EventType::Pop3(Pop3Event::RawInput) => 356, - EventType::Pop3(Pop3Event::RawOutput) => 357, - EventType::PushSubscription(PushSubscriptionEvent::Success) => 373, - EventType::PushSubscription(PushSubscriptionEvent::Error) => 371, - EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372, - EventType::Queue(QueueEvent::Started) => 390, - EventType::Queue(QueueEvent::MessageQueued) => 380, - EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 381, - EventType::Queue(QueueEvent::ReportQueued) => 382, - EventType::Queue(QueueEvent::DsnQueued) => 379, - EventType::Queue(QueueEvent::AutogeneratedQueued) => 378, - EventType::Queue(QueueEvent::Rescheduled) => 385, - EventType::Queue(QueueEvent::Locked) => 377, - EventType::Queue(QueueEvent::BlobNotFound) => 374, - EventType::Queue(QueueEvent::RateLimitExceeded) => 384, - EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => 375, - EventType::Queue(QueueEvent::QuotaExceeded) => 383, - EventType::Queue(QueueEvent::BackPressure) => 48, - EventType::Registry(RegistryEvent::LocalReadError) => 62, - EventType::Registry(RegistryEvent::LocalWriteError) => 54, - EventType::Registry(RegistryEvent::LocalParseError) => 60, - EventType::Registry(RegistryEvent::ReadError) => 65, - EventType::Registry(RegistryEvent::WriteError) => 58, - EventType::Registry(RegistryEvent::DeserializationError) => 56, - EventType::Registry(RegistryEvent::BuildError) => 61, - EventType::Registry(RegistryEvent::BuildWarning) => 55, - EventType::Registry(RegistryEvent::NotSupported) => 64, - EventType::Registry(RegistryEvent::ValidationError) => 63, - EventType::Resource(ResourceEvent::NotFound) => 389, - EventType::Resource(ResourceEvent::BadParameters) => 386, - EventType::Resource(ResourceEvent::Error) => 388, - EventType::Resource(ResourceEvent::DownloadExternal) => 387, - EventType::Resource(ResourceEvent::ApplicationUpdated) => 601, - EventType::Resource(ResourceEvent::ApplicationUnpacked) => 602, - EventType::Security(SecurityEvent::AuthenticationBan) => 33, - EventType::Security(SecurityEvent::AbuseBan) => 549, - EventType::Security(SecurityEvent::ScanBan) => 558, - EventType::Security(SecurityEvent::LoiterBan) => 550, - EventType::Security(SecurityEvent::IpBlocked) => 318, - EventType::Security(SecurityEvent::IpBlockExpired) => 593, - EventType::Security(SecurityEvent::IpAllowExpired) => 594, - EventType::Security(SecurityEvent::IpUnauthorized) => 279, - EventType::Security(SecurityEvent::Unauthorized) => 552, - EventType::Server(ServerEvent::Startup) => 393, - EventType::Server(ServerEvent::Shutdown) => 392, - EventType::Server(ServerEvent::StartupError) => 394, - EventType::Server(ServerEvent::ThreadError) => 395, - EventType::Server(ServerEvent::Licensing) => 391, - EventType::Server(ServerEvent::RecoveryMode) => 603, - EventType::Server(ServerEvent::BootstrapMode) => 604, - EventType::Sieve(SieveEvent::ActionAccept) => 396, - EventType::Sieve(SieveEvent::ActionAcceptReplace) => 397, - EventType::Sieve(SieveEvent::ActionDiscard) => 398, - EventType::Sieve(SieveEvent::ActionReject) => 399, - EventType::Sieve(SieveEvent::SendMessage) => 406, - EventType::Sieve(SieveEvent::MessageTooLarge) => 401, - EventType::Sieve(SieveEvent::ScriptNotFound) => 405, - EventType::Sieve(SieveEvent::ListNotFound) => 400, - EventType::Sieve(SieveEvent::RuntimeError) => 404, - EventType::Sieve(SieveEvent::UnexpectedError) => 407, - EventType::Sieve(SieveEvent::NotSupported) => 402, - EventType::Sieve(SieveEvent::QuotaExceeded) => 403, - EventType::Smtp(SmtpEvent::ConnectionStart) => 417, - EventType::Smtp(SmtpEvent::ConnectionEnd) => 416, - EventType::Smtp(SmtpEvent::Error) => 428, - EventType::Smtp(SmtpEvent::IdNotFound) => 469, - EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded) => 415, - EventType::Smtp(SmtpEvent::TransferLimitExceeded) => 485, - EventType::Smtp(SmtpEvent::RateLimitExceeded) => 461, - EventType::Smtp(SmtpEvent::TimeLimitExceeded) => 481, - EventType::Smtp(SmtpEvent::MissingAuthDirectory) => 452, - EventType::Smtp(SmtpEvent::MessageParseFailed) => 450, - EventType::Smtp(SmtpEvent::MessageTooLarge) => 451, - EventType::Smtp(SmtpEvent::LoopDetected) => 443, - EventType::Smtp(SmtpEvent::DkimPass) => 422, - EventType::Smtp(SmtpEvent::DkimFail) => 421, - EventType::Smtp(SmtpEvent::Dkim2Pass) => 630, - EventType::Smtp(SmtpEvent::Dkim2Fail) => 631, - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded) => 632, - EventType::Smtp(SmtpEvent::ArcPass) => 410, - EventType::Smtp(SmtpEvent::ArcFail) => 409, - EventType::Smtp(SmtpEvent::SpfEhloPass) => 474, - EventType::Smtp(SmtpEvent::SpfEhloFail) => 473, - EventType::Smtp(SmtpEvent::SpfFromPass) => 476, - EventType::Smtp(SmtpEvent::SpfFromFail) => 475, - EventType::Smtp(SmtpEvent::DmarcPass) => 424, - EventType::Smtp(SmtpEvent::DmarcFail) => 423, - EventType::Smtp(SmtpEvent::IprevPass) => 441, - EventType::Smtp(SmtpEvent::IprevFail) => 440, - EventType::Smtp(SmtpEvent::TooManyMessages) => 483, - EventType::Smtp(SmtpEvent::Ehlo) => 426, - EventType::Smtp(SmtpEvent::InvalidEhlo) => 436, - EventType::Smtp(SmtpEvent::DidNotSayEhlo) => 420, - EventType::Smtp(SmtpEvent::EhloExpected) => 427, - EventType::Smtp(SmtpEvent::LhloExpected) => 442, - EventType::Smtp(SmtpEvent::MailFromUnauthenticated) => 447, - EventType::Smtp(SmtpEvent::MailFromUnauthorized) => 448, - EventType::Smtp(SmtpEvent::MailFromNotAllowed) => 551, - EventType::Smtp(SmtpEvent::MailFromRewritten) => 446, - EventType::Smtp(SmtpEvent::MailFromMissing) => 445, - EventType::Smtp(SmtpEvent::MailFrom) => 444, - EventType::Smtp(SmtpEvent::MultipleMailFrom) => 456, - EventType::Smtp(SmtpEvent::MailboxDoesNotExist) => 449, - EventType::Smtp(SmtpEvent::RelayNotAllowed) => 468, - EventType::Smtp(SmtpEvent::RcptTo) => 464, - EventType::Smtp(SmtpEvent::RcptToDuplicate) => 465, - EventType::Smtp(SmtpEvent::RcptToRewritten) => 467, - EventType::Smtp(SmtpEvent::RcptToMissing) => 466, - EventType::Smtp(SmtpEvent::RcptToGreylisted) => 561, - EventType::Smtp(SmtpEvent::TooManyRecipients) => 484, - EventType::Smtp(SmtpEvent::TooManyInvalidRcpt) => 482, - EventType::Smtp(SmtpEvent::RawInput) => 462, - EventType::Smtp(SmtpEvent::RawOutput) => 463, - EventType::Smtp(SmtpEvent::MissingLocalHostname) => 453, - EventType::Smtp(SmtpEvent::Vrfy) => 487, - EventType::Smtp(SmtpEvent::VrfyNotFound) => 489, - EventType::Smtp(SmtpEvent::VrfyDisabled) => 488, - EventType::Smtp(SmtpEvent::Expn) => 429, - EventType::Smtp(SmtpEvent::ExpnNotFound) => 431, - EventType::Smtp(SmtpEvent::ExpnDisabled) => 430, - EventType::Smtp(SmtpEvent::RequireTlsDisabled) => 471, - EventType::Smtp(SmtpEvent::DeliverByDisabled) => 418, - EventType::Smtp(SmtpEvent::DeliverByInvalid) => 419, - EventType::Smtp(SmtpEvent::FutureReleaseDisabled) => 432, - EventType::Smtp(SmtpEvent::FutureReleaseInvalid) => 433, - EventType::Smtp(SmtpEvent::MtPriorityDisabled) => 454, - EventType::Smtp(SmtpEvent::MtPriorityInvalid) => 455, - EventType::Smtp(SmtpEvent::DsnDisabled) => 425, - EventType::Smtp(SmtpEvent::AuthNotAllowed) => 413, - EventType::Smtp(SmtpEvent::AuthMechanismNotSupported) => 412, - EventType::Smtp(SmtpEvent::AuthExchangeTooLong) => 411, - EventType::Smtp(SmtpEvent::AlreadyAuthenticated) => 408, - EventType::Smtp(SmtpEvent::Noop) => 457, - EventType::Smtp(SmtpEvent::StartTls) => 477, - EventType::Smtp(SmtpEvent::StartTlsUnavailable) => 479, - EventType::Smtp(SmtpEvent::StartTlsAlready) => 478, - EventType::Smtp(SmtpEvent::Rset) => 472, - EventType::Smtp(SmtpEvent::Quit) => 460, - EventType::Smtp(SmtpEvent::Help) => 434, - EventType::Smtp(SmtpEvent::CommandNotImplemented) => 414, - EventType::Smtp(SmtpEvent::InvalidCommand) => 435, - EventType::Smtp(SmtpEvent::InvalidSenderAddress) => 439, - EventType::Smtp(SmtpEvent::InvalidRecipientAddress) => 438, - EventType::Smtp(SmtpEvent::InvalidParameter) => 437, - EventType::Smtp(SmtpEvent::UnsupportedParameter) => 486, - EventType::Smtp(SmtpEvent::SyntaxError) => 480, - EventType::Smtp(SmtpEvent::RequestTooLarge) => 470, - EventType::Spam(SpamEvent::Pyzor) => 564, - EventType::Spam(SpamEvent::PyzorError) => 494, - EventType::Spam(SpamEvent::Dnsbl) => 562, - EventType::Spam(SpamEvent::DnsblError) => 563, - EventType::Spam(SpamEvent::TrainStarted) => 588, - EventType::Spam(SpamEvent::TrainCompleted) => 495, - EventType::Spam(SpamEvent::TrainSampleAdded) => 143, - EventType::Spam(SpamEvent::TrainSampleNotFound) => 491, - EventType::Spam(SpamEvent::Classify) => 490, - EventType::Spam(SpamEvent::ModelLoaded) => 589, - EventType::Spam(SpamEvent::ModelNotReady) => 496, - EventType::Spam(SpamEvent::ModelNotFound) => 497, - EventType::Spam(SpamEvent::RulesUpdated) => 280, - EventType::Spf(SpfEvent::Pass) => 501, - EventType::Spf(SpfEvent::Fail) => 498, - EventType::Spf(SpfEvent::SoftFail) => 503, - EventType::Spf(SpfEvent::Neutral) => 499, - EventType::Spf(SpfEvent::TempError) => 504, - EventType::Spf(SpfEvent::PermError) => 502, - EventType::Spf(SpfEvent::None) => 500, - EventType::Store(StoreEvent::AssertValueFailed) => 505, - EventType::Store(StoreEvent::FoundationdbError) => 518, - EventType::Store(StoreEvent::MysqlError) => 522, - EventType::Store(StoreEvent::PostgresqlError) => 527, - EventType::Store(StoreEvent::RocksdbError) => 529, - EventType::Store(StoreEvent::SqliteError) => 532, - EventType::Store(StoreEvent::LdapError) => 520, - EventType::Store(StoreEvent::ElasticsearchError) => 516, - EventType::Store(StoreEvent::MeilisearchError) => 590, - EventType::Store(StoreEvent::RedisError) => 528, - EventType::Store(StoreEvent::S3Error) => 530, - EventType::Store(StoreEvent::AzureError) => 559, - EventType::Store(StoreEvent::FilesystemError) => 517, - EventType::Store(StoreEvent::PoolError) => 526, - EventType::Store(StoreEvent::DataCorruption) => 511, - EventType::Store(StoreEvent::DecompressError) => 514, - EventType::Store(StoreEvent::DeserializeError) => 515, - EventType::Store(StoreEvent::NotFound) => 524, - EventType::Store(StoreEvent::NotConfigured) => 523, - EventType::Store(StoreEvent::NotSupported) => 525, - EventType::Store(StoreEvent::UnexpectedError) => 533, - EventType::Store(StoreEvent::CryptoError) => 510, - EventType::Store(StoreEvent::HttpStoreError) => 493, - EventType::Store(StoreEvent::CacheMiss) => 50, - EventType::Store(StoreEvent::CacheHit) => 51, - EventType::Store(StoreEvent::CacheStale) => 52, - EventType::Store(StoreEvent::CacheUpdate) => 577, - EventType::Store(StoreEvent::CacheEntryTooLarge) => 633, - EventType::Store(StoreEvent::BlobMissingMarker) => 507, - EventType::Store(StoreEvent::DataWrite) => 513, - EventType::Store(StoreEvent::DataIterate) => 512, - EventType::Store(StoreEvent::BlobRead) => 508, - EventType::Store(StoreEvent::BlobWrite) => 509, - EventType::Store(StoreEvent::BlobDelete) => 506, - EventType::Store(StoreEvent::SqlQuery) => 531, - EventType::Store(StoreEvent::LdapQuery) => 521, - EventType::Store(StoreEvent::LdapWarning) => 519, - EventType::Store(StoreEvent::HttpStoreFetch) => 492, - EventType::Store(StoreEvent::AutoExpunge) => 364, - EventType::Store(StoreEvent::BlobStorePurged) => 369, - EventType::Store(StoreEvent::DataStorePurged) => 368, - EventType::TaskManager(TaskManagerEvent::TaskAcquired) => 578, - EventType::TaskManager(TaskManagerEvent::TaskQueued) => 149, - EventType::TaskManager(TaskManagerEvent::TaskScheduled) => 370, - EventType::TaskManager(TaskManagerEvent::TaskLocked) => 144, - EventType::TaskManager(TaskManagerEvent::TaskIgnored) => 586, - EventType::TaskManager(TaskManagerEvent::TaskFailed) => 587, - EventType::TaskManager(TaskManagerEvent::TaskRetry) => 53, - EventType::TaskManager(TaskManagerEvent::BlobNotFound) => 141, - EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => 145, - EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => 150, - EventType::TaskManager(TaskManagerEvent::ManagerStarted) => 367, - EventType::Telemetry(TelemetryEvent::AlertEvent) => 548, - EventType::Telemetry(TelemetryEvent::AlertMessage) => 365, - EventType::Telemetry(TelemetryEvent::LogError) => 535, - EventType::Telemetry(TelemetryEvent::WebhookError) => 539, - EventType::Telemetry(TelemetryEvent::OtelExporterError) => 536, - EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError) => 537, - EventType::Telemetry(TelemetryEvent::PrometheusExporterError) => 538, - EventType::Telemetry(TelemetryEvent::JournalError) => 534, - EventType::Telemetry(TelemetryEvent::MetricsCollected) => 151, - EventType::Telemetry(TelemetryEvent::MetricsStored) => 366, - EventType::Telemetry(TelemetryEvent::MetricsPushed) => 146, - EventType::Tls(TlsEvent::Handshake) => 543, - EventType::Tls(TlsEvent::HandshakeError) => 544, - EventType::Tls(TlsEvent::NotConfigured) => 547, - EventType::Tls(TlsEvent::CertificateNotFound) => 542, - EventType::Tls(TlsEvent::NoCertificatesAvailable) => 546, - EventType::Tls(TlsEvent::MultipleCertificatesAvailable) => 545, - EventType::Tls(TlsEvent::ExpiredCertificateRemoved) => 277, - EventType::TlsRpt(TlsRptEvent::RecordFetch) => 540, - EventType::TlsRpt(TlsRptEvent::RecordFetchError) => 541, - EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 560, - EventType::WebDav(WebDavEvent::Propfind) => 147, - EventType::WebDav(WebDavEvent::Proppatch) => 148, - EventType::WebDav(WebDavEvent::Get) => 335, - EventType::WebDav(WebDavEvent::Head) => 574, - EventType::WebDav(WebDavEvent::Report) => 336, - EventType::WebDav(WebDavEvent::Mkcol) => 376, - EventType::WebDav(WebDavEvent::Mkcalendar) => 575, - EventType::WebDav(WebDavEvent::Delete) => 458, - EventType::WebDav(WebDavEvent::Put) => 459, - EventType::WebDav(WebDavEvent::Post) => 565, - EventType::WebDav(WebDavEvent::Patch) => 566, - EventType::WebDav(WebDavEvent::Copy) => 567, - EventType::WebDav(WebDavEvent::Move) => 568, - EventType::WebDav(WebDavEvent::Lock) => 569, - EventType::WebDav(WebDavEvent::Unlock) => 570, - EventType::WebDav(WebDavEvent::Acl) => 571, - EventType::WebDav(WebDavEvent::Options) => 573, - EventType::WebDav(WebDavEvent::Error) => 572, + EventType::Eval(EvalEvent::Result) => 123, + EventType::Eval(EvalEvent::Error) => 122, + EventType::Eval(EvalEvent::DirectoryNotFound) => 121, + EventType::Eval(EvalEvent::StoreNotFound) => 124, + EventType::Http(HttpEvent::ConnectionStart) => 137, + EventType::Http(HttpEvent::ConnectionEnd) => 136, + EventType::Http(HttpEvent::Error) => 138, + EventType::Http(HttpEvent::RequestUrl) => 140, + EventType::Http(HttpEvent::RequestBody) => 139, + EventType::Http(HttpEvent::ResponseBody) => 141, + EventType::Http(HttpEvent::XForwardedMissing) => 142, + EventType::Http(HttpEvent::XForwardedInvalid) => 625, + EventType::Imap(ImapEvent::ConnectionStart) => 147, + EventType::Imap(ImapEvent::ConnectionEnd) => 146, + EventType::Imap(ImapEvent::GetAcl) => 155, + EventType::Imap(ImapEvent::SetAcl) => 172, + EventType::Imap(ImapEvent::MyRights) => 164, + EventType::Imap(ImapEvent::ListRights) => 160, + EventType::Imap(ImapEvent::Append) => 143, + EventType::Imap(ImapEvent::Capabilities) => 144, + EventType::Imap(ImapEvent::Id) => 156, + EventType::Imap(ImapEvent::Close) => 145, + EventType::Imap(ImapEvent::Copy) => 148, + EventType::Imap(ImapEvent::Move) => 163, + EventType::Imap(ImapEvent::CreateMailbox) => 149, + EventType::Imap(ImapEvent::DeleteMailbox) => 150, + EventType::Imap(ImapEvent::RenameMailbox) => 169, + EventType::Imap(ImapEvent::Enable) => 151, + EventType::Imap(ImapEvent::Expunge) => 153, + EventType::Imap(ImapEvent::Fetch) => 154, + EventType::Imap(ImapEvent::IdleStart) => 157, + EventType::Imap(ImapEvent::IdleStop) => 158, + EventType::Imap(ImapEvent::List) => 159, + EventType::Imap(ImapEvent::Lsub) => 162, + EventType::Imap(ImapEvent::Logout) => 161, + EventType::Imap(ImapEvent::Namespace) => 165, + EventType::Imap(ImapEvent::Noop) => 166, + EventType::Imap(ImapEvent::Search) => 170, + EventType::Imap(ImapEvent::Sort) => 173, + EventType::Imap(ImapEvent::Select) => 171, + EventType::Imap(ImapEvent::Status) => 174, + EventType::Imap(ImapEvent::Store) => 175, + EventType::Imap(ImapEvent::Subscribe) => 176, + EventType::Imap(ImapEvent::Unsubscribe) => 178, + EventType::Imap(ImapEvent::Thread) => 177, + EventType::Imap(ImapEvent::UidBatches) => 593, + EventType::Imap(ImapEvent::GetQuota) => 52, + EventType::Imap(ImapEvent::GetMetadata) => 607, + EventType::Imap(ImapEvent::SetMetadata) => 608, + EventType::Imap(ImapEvent::Error) => 152, + EventType::Imap(ImapEvent::RawInput) => 167, + EventType::Imap(ImapEvent::RawOutput) => 168, + EventType::IncomingReport(IncomingReportEvent::DmarcReport) => 184, + EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings) => 185, + EventType::IncomingReport(IncomingReportEvent::TlsReport) => 190, + EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings) => 191, + EventType::IncomingReport(IncomingReportEvent::AbuseReport) => 179, + EventType::IncomingReport(IncomingReportEvent::AuthFailureReport) => 181, + EventType::IncomingReport(IncomingReportEvent::FraudReport) => 186, + EventType::IncomingReport(IncomingReportEvent::NotSpamReport) => 188, + EventType::IncomingReport(IncomingReportEvent::VirusReport) => 193, + EventType::IncomingReport(IncomingReportEvent::OtherReport) => 189, + EventType::IncomingReport(IncomingReportEvent::MessageParseFailed) => 187, + EventType::IncomingReport(IncomingReportEvent::DmarcParseFailed) => 183, + EventType::IncomingReport(IncomingReportEvent::TlsRpcParseFailed) => 192, + EventType::IncomingReport(IncomingReportEvent::ArfParseFailed) => 180, + EventType::IncomingReport(IncomingReportEvent::DecompressError) => 182, + EventType::Iprev(IprevEvent::Pass) => 196, + EventType::Iprev(IprevEvent::Fail) => 194, + EventType::Iprev(IprevEvent::PermError) => 197, + EventType::Iprev(IprevEvent::TempError) => 198, + EventType::Iprev(IprevEvent::None) => 195, + EventType::Jmap(JmapEvent::MethodCall) => 207, + EventType::Jmap(JmapEvent::InvalidArguments) => 205, + EventType::Jmap(JmapEvent::RequestTooLarge) => 211, + EventType::Jmap(JmapEvent::StateMismatch) => 212, + EventType::Jmap(JmapEvent::AnchorNotFound) => 202, + EventType::Jmap(JmapEvent::UnsupportedFilter) => 216, + EventType::Jmap(JmapEvent::UnsupportedSort) => 217, + EventType::Jmap(JmapEvent::UnknownMethod) => 215, + EventType::Jmap(JmapEvent::InvalidResultReference) => 206, + EventType::Jmap(JmapEvent::Forbidden) => 204, + EventType::Jmap(JmapEvent::AccountNotFound) => 199, + EventType::Jmap(JmapEvent::AccountNotSupportedByMethod) => 200, + EventType::Jmap(JmapEvent::AccountReadOnly) => 201, + EventType::Jmap(JmapEvent::FromAccountNotFound) => 653, + EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod) => 654, + EventType::Jmap(JmapEvent::NotFound) => 208, + EventType::Jmap(JmapEvent::CannotCalculateChanges) => 203, + EventType::Jmap(JmapEvent::UnknownDataType) => 214, + EventType::Jmap(JmapEvent::UnknownCapability) => 213, + EventType::Jmap(JmapEvent::NotJson) => 209, + EventType::Jmap(JmapEvent::NotRequest) => 210, + EventType::Jmap(JmapEvent::WebsocketStart) => 219, + EventType::Jmap(JmapEvent::WebsocketStop) => 220, + EventType::Jmap(JmapEvent::WebsocketError) => 218, + EventType::Jmap(JmapEvent::ExpandDurationTooLarge) => 602, + EventType::Jmap(JmapEvent::CannotCalculateOccurrences) => 603, + EventType::Jmap(JmapEvent::TooLarge) => 604, + EventType::Jmap(JmapEvent::RateLimit) => 605, + EventType::Jmap(JmapEvent::ObjectNotFound) => 606, + EventType::Limit(LimitEvent::SizeRequest) => 227, + EventType::Limit(LimitEvent::SizeUpload) => 228, + EventType::Limit(LimitEvent::CallsIn) => 222, + EventType::Limit(LimitEvent::ConcurrentRequest) => 224, + EventType::Limit(LimitEvent::ConcurrentUpload) => 225, + EventType::Limit(LimitEvent::ConcurrentConnection) => 223, + EventType::Limit(LimitEvent::Quota) => 226, + EventType::Limit(LimitEvent::BlobQuota) => 221, + EventType::Limit(LimitEvent::TooManyRequests) => 229, + EventType::Limit(LimitEvent::MemoryBudget) => 627, + EventType::MailAuth(MailAuthEvent::ParseError) => 238, + EventType::MailAuth(MailAuthEvent::MissingParameters) => 236, + EventType::MailAuth(MailAuthEvent::NoHeadersFound) => 237, + EventType::MailAuth(MailAuthEvent::Crypto) => 231, + EventType::MailAuth(MailAuthEvent::Io) => 235, + EventType::MailAuth(MailAuthEvent::Base64) => 230, + EventType::MailAuth(MailAuthEvent::DnsError) => 232, + EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) => 234, + EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType) => 233, + EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => 239, + EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => 243, + EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => 242, + EventType::ManageSieve(ManageSieveEvent::CreateScript) => 244, + EventType::ManageSieve(ManageSieveEvent::UpdateScript) => 258, + EventType::ManageSieve(ManageSieveEvent::GetScript) => 247, + EventType::ManageSieve(ManageSieveEvent::DeleteScript) => 245, + EventType::ManageSieve(ManageSieveEvent::RenameScript) => 254, + EventType::ManageSieve(ManageSieveEvent::CheckScript) => 241, + EventType::ManageSieve(ManageSieveEvent::HaveSpace) => 248, + EventType::ManageSieve(ManageSieveEvent::ListScripts) => 249, + EventType::ManageSieve(ManageSieveEvent::SetActive) => 255, + EventType::ManageSieve(ManageSieveEvent::Capabilities) => 240, + EventType::ManageSieve(ManageSieveEvent::StartTls) => 256, + EventType::ManageSieve(ManageSieveEvent::Unauthenticate) => 257, + EventType::ManageSieve(ManageSieveEvent::Logout) => 250, + EventType::ManageSieve(ManageSieveEvent::Noop) => 251, + EventType::ManageSieve(ManageSieveEvent::Error) => 246, + EventType::ManageSieve(ManageSieveEvent::RawInput) => 252, + EventType::ManageSieve(ManageSieveEvent::RawOutput) => 253, + EventType::MessageIngest(MessageIngestEvent::Ham) => 267, + EventType::MessageIngest(MessageIngestEvent::Spam) => 270, + EventType::MessageIngest(MessageIngestEvent::ImapAppend) => 268, + EventType::MessageIngest(MessageIngestEvent::JmapAppend) => 269, + EventType::MessageIngest(MessageIngestEvent::Duplicate) => 265, + EventType::MessageIngest(MessageIngestEvent::Error) => 266, + EventType::Milter(MilterEvent::Read) => 283, + EventType::Milter(MilterEvent::Write) => 287, + EventType::Milter(MilterEvent::ActionAccept) => 271, + EventType::Milter(MilterEvent::ActionDiscard) => 273, + EventType::Milter(MilterEvent::ActionReject) => 274, + EventType::Milter(MilterEvent::ActionTempFail) => 277, + EventType::Milter(MilterEvent::ActionReplyCode) => 275, + EventType::Milter(MilterEvent::ActionConnectionFailure) => 272, + EventType::Milter(MilterEvent::ActionShutdown) => 276, + EventType::Milter(MilterEvent::IoError) => 281, + EventType::Milter(MilterEvent::FrameTooLarge) => 280, + EventType::Milter(MilterEvent::FrameInvalid) => 279, + EventType::Milter(MilterEvent::UnexpectedResponse) => 286, + EventType::Milter(MilterEvent::Timeout) => 284, + EventType::Milter(MilterEvent::TlsInvalidName) => 285, + EventType::Milter(MilterEvent::Disconnected) => 278, + EventType::Milter(MilterEvent::ParseError) => 282, + EventType::MtaHook(MtaHookEvent::ActionAccept) => 288, + EventType::MtaHook(MtaHookEvent::ActionDiscard) => 289, + EventType::MtaHook(MtaHookEvent::ActionReject) => 291, + EventType::MtaHook(MtaHookEvent::ActionQuarantine) => 290, + EventType::MtaHook(MtaHookEvent::Error) => 292, + EventType::MtaSts(MtaStsEvent::Authorized) => 293, + EventType::MtaSts(MtaStsEvent::NotAuthorized) => 295, + EventType::MtaSts(MtaStsEvent::PolicyFetch) => 296, + EventType::MtaSts(MtaStsEvent::PolicyNotFound) => 298, + EventType::MtaSts(MtaStsEvent::PolicyFetchError) => 297, + EventType::MtaSts(MtaStsEvent::InvalidPolicy) => 294, + EventType::Network(NetworkEvent::ListenStart) => 305, + EventType::Network(NetworkEvent::ListenStop) => 306, + EventType::Network(NetworkEvent::ListenError) => 304, + EventType::Network(NetworkEvent::BindError) => 300, + EventType::Network(NetworkEvent::ReadError) => 308, + EventType::Network(NetworkEvent::WriteError) => 312, + EventType::Network(NetworkEvent::FlushError) => 303, + EventType::Network(NetworkEvent::AcceptError) => 299, + EventType::Network(NetworkEvent::SplitError) => 310, + EventType::Network(NetworkEvent::Timeout) => 311, + EventType::Network(NetworkEvent::Closed) => 301, + EventType::Network(NetworkEvent::ProxyError) => 307, + EventType::Network(NetworkEvent::SetOptError) => 309, + EventType::OutgoingReport(OutgoingReportEvent::SpfReport) => 326, + EventType::OutgoingReport(OutgoingReportEvent::SpfRateLimited) => 325, + EventType::OutgoingReport(OutgoingReportEvent::DkimReport) => 314, + EventType::OutgoingReport(OutgoingReportEvent::DkimRateLimited) => 313, + EventType::OutgoingReport(OutgoingReportEvent::DmarcReport) => 317, + EventType::OutgoingReport(OutgoingReportEvent::DmarcRateLimited) => 316, + EventType::OutgoingReport(OutgoingReportEvent::DmarcAggregateReport) => 315, + EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate) => 328, + EventType::OutgoingReport(OutgoingReportEvent::HttpSubmission) => 318, + EventType::OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress) => 329, + EventType::OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError) => 324, + EventType::OutgoingReport(OutgoingReportEvent::NotFound) => 323, + EventType::OutgoingReport(OutgoingReportEvent::SubmissionError) => 327, + EventType::OutgoingReport(OutgoingReportEvent::NoRecipientsFound) => 322, + EventType::OutgoingReport(OutgoingReportEvent::Locked) => 321, + EventType::OutgoingReport(OutgoingReportEvent::MaxSizeExceeded) => 54, + EventType::Pop3(Pop3Event::ConnectionStart) => 332, + EventType::Pop3(Pop3Event::ConnectionEnd) => 331, + EventType::Pop3(Pop3Event::Delete) => 333, + EventType::Pop3(Pop3Event::Reset) => 342, + EventType::Pop3(Pop3Event::Quit) => 339, + EventType::Pop3(Pop3Event::Fetch) => 335, + EventType::Pop3(Pop3Event::List) => 336, + EventType::Pop3(Pop3Event::ListMessage) => 337, + EventType::Pop3(Pop3Event::Uidl) => 345, + EventType::Pop3(Pop3Event::UidlMessage) => 346, + EventType::Pop3(Pop3Event::Stat) => 344, + EventType::Pop3(Pop3Event::Noop) => 338, + EventType::Pop3(Pop3Event::Capabilities) => 330, + EventType::Pop3(Pop3Event::StartTls) => 343, + EventType::Pop3(Pop3Event::Utf8) => 347, + EventType::Pop3(Pop3Event::Error) => 334, + EventType::Pop3(Pop3Event::RawInput) => 340, + EventType::Pop3(Pop3Event::RawOutput) => 341, + EventType::PushSubscription(PushSubscriptionEvent::Success) => 357, + EventType::PushSubscription(PushSubscriptionEvent::Error) => 355, + EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 356, + EventType::Queue(QueueEvent::Started) => 374, + EventType::Queue(QueueEvent::MessageQueued) => 364, + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 365, + EventType::Queue(QueueEvent::ReportQueued) => 366, + EventType::Queue(QueueEvent::DsnQueued) => 363, + EventType::Queue(QueueEvent::AutogeneratedQueued) => 362, + EventType::Queue(QueueEvent::Rescheduled) => 369, + EventType::Queue(QueueEvent::Locked) => 361, + EventType::Queue(QueueEvent::BlobNotFound) => 358, + EventType::Queue(QueueEvent::RateLimitExceeded) => 368, + EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => 359, + EventType::Queue(QueueEvent::QuotaExceeded) => 367, + EventType::Queue(QueueEvent::BackPressure) => 43, + EventType::Registry(RegistryEvent::LocalReadError) => 57, + EventType::Registry(RegistryEvent::LocalWriteError) => 49, + EventType::Registry(RegistryEvent::LocalParseError) => 55, + EventType::Registry(RegistryEvent::ReadError) => 60, + EventType::Registry(RegistryEvent::WriteError) => 53, + EventType::Registry(RegistryEvent::DeserializationError) => 51, + EventType::Registry(RegistryEvent::BuildError) => 56, + EventType::Registry(RegistryEvent::BuildWarning) => 50, + EventType::Registry(RegistryEvent::NotSupported) => 59, + EventType::Registry(RegistryEvent::ValidationError) => 58, + EventType::Resource(ResourceEvent::NotFound) => 373, + EventType::Resource(ResourceEvent::BadParameters) => 370, + EventType::Resource(ResourceEvent::Error) => 372, + EventType::Resource(ResourceEvent::DownloadExternal) => 371, + EventType::Resource(ResourceEvent::ApplicationUpdated) => 584, + EventType::Resource(ResourceEvent::ApplicationUnpacked) => 585, + EventType::SearchIndex(SearchIndexEvent::ManagerStarted) => 629, + EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired) => 630, + EventType::SearchIndex(SearchIndexEvent::PartitionLocked) => 631, + EventType::SearchIndex(SearchIndexEvent::PartitionRetry) => 632, + EventType::SearchIndex(SearchIndexEvent::BatchIndexed) => 126, + EventType::SearchIndex(SearchIndexEvent::DocumentIgnored) => 574, + EventType::SearchIndex(SearchIndexEvent::DocumentTruncated) => 628, + EventType::SearchIndex(SearchIndexEvent::MetadataNotFound) => 27, + EventType::Security(SecurityEvent::AuthenticationBan) => 28, + EventType::Security(SecurityEvent::AbuseBan) => 532, + EventType::Security(SecurityEvent::ScanBan) => 541, + EventType::Security(SecurityEvent::LoiterBan) => 533, + EventType::Security(SecurityEvent::IpBlocked) => 302, + EventType::Security(SecurityEvent::IpBlockExpired) => 576, + EventType::Security(SecurityEvent::IpAllowExpired) => 577, + EventType::Security(SecurityEvent::IpUnauthorized) => 263, + EventType::Security(SecurityEvent::Unauthorized) => 535, + EventType::Server(ServerEvent::Startup) => 377, + EventType::Server(ServerEvent::Shutdown) => 376, + EventType::Server(ServerEvent::StartupError) => 378, + EventType::Server(ServerEvent::ThreadError) => 379, + EventType::Server(ServerEvent::RecoveryMode) => 586, + EventType::Server(ServerEvent::BootstrapMode) => 587, + EventType::Sieve(SieveEvent::ActionAccept) => 380, + EventType::Sieve(SieveEvent::ActionAcceptReplace) => 381, + EventType::Sieve(SieveEvent::ActionDiscard) => 382, + EventType::Sieve(SieveEvent::ActionReject) => 383, + EventType::Sieve(SieveEvent::SendMessage) => 390, + EventType::Sieve(SieveEvent::MessageTooLarge) => 385, + EventType::Sieve(SieveEvent::ScriptNotFound) => 389, + EventType::Sieve(SieveEvent::ListNotFound) => 384, + EventType::Sieve(SieveEvent::RuntimeError) => 388, + EventType::Sieve(SieveEvent::UnexpectedError) => 391, + EventType::Sieve(SieveEvent::NotSupported) => 386, + EventType::Sieve(SieveEvent::QuotaExceeded) => 387, + EventType::Smtp(SmtpEvent::ConnectionStart) => 401, + EventType::Smtp(SmtpEvent::ConnectionEnd) => 400, + EventType::Smtp(SmtpEvent::Error) => 412, + EventType::Smtp(SmtpEvent::IdNotFound) => 452, + EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded) => 399, + EventType::Smtp(SmtpEvent::TransferLimitExceeded) => 468, + EventType::Smtp(SmtpEvent::RateLimitExceeded) => 444, + EventType::Smtp(SmtpEvent::TimeLimitExceeded) => 464, + EventType::Smtp(SmtpEvent::MessageParseFailed) => 434, + EventType::Smtp(SmtpEvent::MessageTooLarge) => 435, + EventType::Smtp(SmtpEvent::LoopDetected) => 427, + EventType::Smtp(SmtpEvent::MultipleFromDomains) => 626, + EventType::Smtp(SmtpEvent::DkimPass) => 406, + EventType::Smtp(SmtpEvent::DkimFail) => 405, + EventType::Smtp(SmtpEvent::Dkim2Pass) => 589, + EventType::Smtp(SmtpEvent::Dkim2Fail) => 590, + EventType::Smtp(SmtpEvent::Dkim2DsnRejected) => 591, + EventType::Smtp(SmtpEvent::ArcPass) => 394, + EventType::Smtp(SmtpEvent::ArcFail) => 393, + EventType::Smtp(SmtpEvent::SpfEhloPass) => 457, + EventType::Smtp(SmtpEvent::SpfEhloFail) => 456, + EventType::Smtp(SmtpEvent::SpfFromPass) => 459, + EventType::Smtp(SmtpEvent::SpfFromFail) => 458, + EventType::Smtp(SmtpEvent::DmarcPass) => 408, + EventType::Smtp(SmtpEvent::DmarcFail) => 407, + EventType::Smtp(SmtpEvent::IprevPass) => 425, + EventType::Smtp(SmtpEvent::IprevFail) => 424, + EventType::Smtp(SmtpEvent::TooManyMessages) => 466, + EventType::Smtp(SmtpEvent::Ehlo) => 410, + EventType::Smtp(SmtpEvent::InvalidEhlo) => 420, + EventType::Smtp(SmtpEvent::DidNotSayEhlo) => 404, + EventType::Smtp(SmtpEvent::EhloExpected) => 411, + EventType::Smtp(SmtpEvent::LhloExpected) => 426, + EventType::Smtp(SmtpEvent::MailFromUnauthenticated) => 431, + EventType::Smtp(SmtpEvent::MailFromUnauthorized) => 432, + EventType::Smtp(SmtpEvent::MailFromNotAllowed) => 534, + EventType::Smtp(SmtpEvent::MailFromRewritten) => 430, + EventType::Smtp(SmtpEvent::MailFromMissing) => 429, + EventType::Smtp(SmtpEvent::MailFrom) => 428, + EventType::Smtp(SmtpEvent::MultipleMailFrom) => 439, + EventType::Smtp(SmtpEvent::MailboxDoesNotExist) => 433, + EventType::Smtp(SmtpEvent::RelayNotAllowed) => 451, + EventType::Smtp(SmtpEvent::RcptTo) => 447, + EventType::Smtp(SmtpEvent::RcptToDuplicate) => 448, + EventType::Smtp(SmtpEvent::RcptToRewritten) => 450, + EventType::Smtp(SmtpEvent::RcptToMissing) => 449, + EventType::Smtp(SmtpEvent::RcptToGreylisted) => 544, + EventType::Smtp(SmtpEvent::TooManyRecipients) => 467, + EventType::Smtp(SmtpEvent::TooManyInvalidRcpt) => 465, + EventType::Smtp(SmtpEvent::RawInput) => 445, + EventType::Smtp(SmtpEvent::RawOutput) => 446, + EventType::Smtp(SmtpEvent::MissingLocalHostname) => 436, + EventType::Smtp(SmtpEvent::Vrfy) => 470, + EventType::Smtp(SmtpEvent::VrfyNotFound) => 472, + EventType::Smtp(SmtpEvent::VrfyDisabled) => 471, + EventType::Smtp(SmtpEvent::Expn) => 413, + EventType::Smtp(SmtpEvent::ExpnNotFound) => 415, + EventType::Smtp(SmtpEvent::ExpnDisabled) => 414, + EventType::Smtp(SmtpEvent::RequireTlsDisabled) => 454, + EventType::Smtp(SmtpEvent::DeliverByDisabled) => 402, + EventType::Smtp(SmtpEvent::DeliverByInvalid) => 403, + EventType::Smtp(SmtpEvent::FutureReleaseDisabled) => 416, + EventType::Smtp(SmtpEvent::FutureReleaseInvalid) => 417, + EventType::Smtp(SmtpEvent::MtPriorityDisabled) => 437, + EventType::Smtp(SmtpEvent::MtPriorityInvalid) => 438, + EventType::Smtp(SmtpEvent::DsnDisabled) => 409, + EventType::Smtp(SmtpEvent::AuthNotAllowed) => 397, + EventType::Smtp(SmtpEvent::AuthMechanismNotSupported) => 396, + EventType::Smtp(SmtpEvent::AuthExchangeTooLong) => 395, + EventType::Smtp(SmtpEvent::AlreadyAuthenticated) => 392, + EventType::Smtp(SmtpEvent::Noop) => 440, + EventType::Smtp(SmtpEvent::StartTls) => 460, + EventType::Smtp(SmtpEvent::StartTlsUnavailable) => 462, + EventType::Smtp(SmtpEvent::StartTlsAlready) => 461, + EventType::Smtp(SmtpEvent::Rset) => 455, + EventType::Smtp(SmtpEvent::Quit) => 443, + EventType::Smtp(SmtpEvent::Help) => 418, + EventType::Smtp(SmtpEvent::CommandNotImplemented) => 398, + EventType::Smtp(SmtpEvent::InvalidCommand) => 419, + EventType::Smtp(SmtpEvent::InvalidSenderAddress) => 423, + EventType::Smtp(SmtpEvent::InvalidRecipientAddress) => 422, + EventType::Smtp(SmtpEvent::InvalidParameter) => 421, + EventType::Smtp(SmtpEvent::UnsupportedParameter) => 469, + EventType::Smtp(SmtpEvent::SyntaxError) => 463, + EventType::Smtp(SmtpEvent::RequestTooLarge) => 453, + EventType::Spam(SpamEvent::Pyzor) => 547, + EventType::Spam(SpamEvent::PyzorError) => 477, + EventType::Spam(SpamEvent::Dnsbl) => 545, + EventType::Spam(SpamEvent::DnsblError) => 546, + EventType::Spam(SpamEvent::TrainStarted) => 571, + EventType::Spam(SpamEvent::TrainCompleted) => 478, + EventType::Spam(SpamEvent::TrainSampleAdded) => 127, + EventType::Spam(SpamEvent::TrainSampleNotFound) => 474, + EventType::Spam(SpamEvent::Classify) => 473, + EventType::Spam(SpamEvent::ModelLoaded) => 572, + EventType::Spam(SpamEvent::ModelNotReady) => 479, + EventType::Spam(SpamEvent::ModelNotFound) => 480, + EventType::Spam(SpamEvent::RulesUpdated) => 264, + EventType::Spf(SpfEvent::Pass) => 484, + EventType::Spf(SpfEvent::Fail) => 481, + EventType::Spf(SpfEvent::SoftFail) => 486, + EventType::Spf(SpfEvent::Neutral) => 482, + EventType::Spf(SpfEvent::TempError) => 487, + EventType::Spf(SpfEvent::PermError) => 485, + EventType::Spf(SpfEvent::None) => 483, + EventType::Store(StoreEvent::AssertValueFailed) => 488, + EventType::Store(StoreEvent::FoundationdbError) => 501, + EventType::Store(StoreEvent::MysqlError) => 505, + EventType::Store(StoreEvent::PostgresqlError) => 510, + EventType::Store(StoreEvent::RocksdbError) => 512, + EventType::Store(StoreEvent::SqliteError) => 515, + EventType::Store(StoreEvent::LdapError) => 503, + EventType::Store(StoreEvent::ElasticsearchError) => 499, + EventType::Store(StoreEvent::MeilisearchError) => 573, + EventType::Store(StoreEvent::RedisError) => 511, + EventType::Store(StoreEvent::S3Error) => 513, + EventType::Store(StoreEvent::AzureError) => 542, + EventType::Store(StoreEvent::FilesystemError) => 500, + EventType::Store(StoreEvent::PoolError) => 509, + EventType::Store(StoreEvent::DataCorruption) => 494, + EventType::Store(StoreEvent::DecompressError) => 497, + EventType::Store(StoreEvent::DeserializeError) => 498, + EventType::Store(StoreEvent::NotFound) => 507, + EventType::Store(StoreEvent::NotConfigured) => 506, + EventType::Store(StoreEvent::NotSupported) => 508, + EventType::Store(StoreEvent::UnexpectedError) => 516, + EventType::Store(StoreEvent::CryptoError) => 493, + EventType::Store(StoreEvent::HttpStoreError) => 476, + EventType::Store(StoreEvent::BlobMissingMarker) => 490, + EventType::Store(StoreEvent::DataWrite) => 496, + EventType::Store(StoreEvent::DataIterate) => 495, + EventType::Store(StoreEvent::BlobRead) => 491, + EventType::Store(StoreEvent::BlobWrite) => 492, + EventType::Store(StoreEvent::BlobDelete) => 489, + EventType::Store(StoreEvent::SqlQuery) => 514, + EventType::Store(StoreEvent::LdapQuery) => 504, + EventType::Store(StoreEvent::LdapWarning) => 502, + EventType::Store(StoreEvent::HttpStoreFetch) => 475, + EventType::Store(StoreEvent::AutoExpunge) => 348, + EventType::Store(StoreEvent::BlobStorePurged) => 353, + EventType::Store(StoreEvent::DataStorePurged) => 352, + EventType::TaskManager(TaskManagerEvent::TaskAcquired) => 561, + EventType::TaskManager(TaskManagerEvent::TaskQueued) => 133, + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => 354, + EventType::TaskManager(TaskManagerEvent::TaskLocked) => 128, + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => 569, + EventType::TaskManager(TaskManagerEvent::TaskFailed) => 570, + EventType::TaskManager(TaskManagerEvent::TaskRetry) => 48, + EventType::TaskManager(TaskManagerEvent::TaskStalled) => 651, + EventType::TaskManager(TaskManagerEvent::TaskExpired) => 652, + EventType::TaskManager(TaskManagerEvent::BlobNotFound) => 125, + EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => 129, + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => 134, + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => 351, + EventType::Telemetry(TelemetryEvent::LogError) => 518, + EventType::Telemetry(TelemetryEvent::WebhookError) => 522, + EventType::Telemetry(TelemetryEvent::OtelExporterError) => 519, + EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError) => 520, + EventType::Telemetry(TelemetryEvent::PrometheusExporterError) => 521, + EventType::Telemetry(TelemetryEvent::JournalError) => 517, + EventType::Telemetry(TelemetryEvent::EventsDropped) => 601, + EventType::Telemetry(TelemetryEvent::MetricsCollected) => 135, + EventType::Telemetry(TelemetryEvent::MetricsPushed) => 130, + EventType::Tls(TlsEvent::Handshake) => 526, + EventType::Tls(TlsEvent::HandshakeError) => 527, + EventType::Tls(TlsEvent::NotConfigured) => 530, + EventType::Tls(TlsEvent::CertificateNotFound) => 525, + EventType::Tls(TlsEvent::NoCertificatesAvailable) => 529, + EventType::Tls(TlsEvent::MultipleCertificatesAvailable) => 528, + EventType::Tls(TlsEvent::ExpiredCertificateRemoved) => 261, + EventType::TlsRpt(TlsRptEvent::RecordFetch) => 523, + EventType::TlsRpt(TlsRptEvent::RecordFetchError) => 524, + EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 543, + EventType::WebDav(WebDavEvent::Propfind) => 131, + EventType::WebDav(WebDavEvent::Proppatch) => 132, + EventType::WebDav(WebDavEvent::Get) => 319, + EventType::WebDav(WebDavEvent::Head) => 557, + EventType::WebDav(WebDavEvent::Report) => 320, + EventType::WebDav(WebDavEvent::Mkcol) => 360, + EventType::WebDav(WebDavEvent::Mkcalendar) => 558, + EventType::WebDav(WebDavEvent::Delete) => 441, + EventType::WebDav(WebDavEvent::Put) => 442, + EventType::WebDav(WebDavEvent::Post) => 548, + EventType::WebDav(WebDavEvent::Patch) => 549, + EventType::WebDav(WebDavEvent::Copy) => 550, + EventType::WebDav(WebDavEvent::Move) => 551, + EventType::WebDav(WebDavEvent::Lock) => 552, + EventType::WebDav(WebDavEvent::Unlock) => 553, + EventType::WebDav(WebDavEvent::Acl) => 554, + EventType::WebDav(WebDavEvent::Options) => 556, + EventType::WebDav(WebDavEvent::Error) => 555, } } @@ -2097,147 +2095,132 @@ impl EventType { 24 => Some(EventType::Acme(AcmeEvent::TlsAlpnError)), 26 => Some(EventType::Acme(AcmeEvent::TokenNotFound)), 15 => Some(EventType::Acme(AcmeEvent::Error)), - 556 => Some(EventType::Ai(AiEvent::LlmResponse)), - 557 => Some(EventType::Ai(AiEvent::ApiError)), - 28 => Some(EventType::Arc(ArcEvent::ChainTooLong)), - 31 => Some(EventType::Arc(ArcEvent::InvalidInstance)), - 30 => Some(EventType::Arc(ArcEvent::InvalidCv)), - 29 => Some(EventType::Arc(ArcEvent::HasHeaderTag)), - 27 => Some(EventType::Arc(ArcEvent::BrokenChain)), - 32 => Some(EventType::Arc(ArcEvent::SealerNotFound)), - 37 => Some(EventType::Auth(AuthEvent::Success)), - 35 => Some(EventType::Auth(AuthEvent::Failed)), - 554 => Some(EventType::Auth(AuthEvent::TokenExpired)), - 36 => Some(EventType::Auth(AuthEvent::MfaRequired)), - 38 => Some(EventType::Auth(AuthEvent::TooManyAttempts)), - 555 => Some(EventType::Auth(AuthEvent::ClientRegistration)), - 34 => Some(EventType::Auth(AuthEvent::Error)), - 595 => Some(EventType::Auth(AuthEvent::Warning)), - 276 => Some(EventType::Auth(AuthEvent::CredentialExpired)), - 576 => Some(EventType::Calendar(CalendarEvent::RuleExpansionError)), - 579 => Some(EventType::Calendar(CalendarEvent::AlarmSent)), - 580 => Some(EventType::Calendar(CalendarEvent::AlarmSkipped)), - 581 => Some(EventType::Calendar(CalendarEvent::AlarmRecipientOverride)), - 582 => Some(EventType::Calendar(CalendarEvent::AlarmFailed)), - 583 => Some(EventType::Calendar(CalendarEvent::ItipMessageSent)), - 584 => Some(EventType::Calendar(CalendarEvent::ItipMessageReceived)), - 585 => Some(EventType::Calendar(CalendarEvent::ItipMessageError)), - 278 => Some(EventType::Cluster(ClusterEvent::Startup)), - 39 => Some(EventType::Cluster(ClusterEvent::SubscriberStart)), - 40 => Some(EventType::Cluster(ClusterEvent::SubscriberStop)), - 41 => Some(EventType::Cluster(ClusterEvent::SubscriberError)), - 42 => Some(EventType::Cluster(ClusterEvent::SubscriberDisconnected)), - 43 => Some(EventType::Cluster(ClusterEvent::PublisherStart)), - 44 => Some(EventType::Cluster(ClusterEvent::PublisherStop)), - 45 => Some(EventType::Cluster(ClusterEvent::PublisherError)), - 46 => Some(EventType::Cluster(ClusterEvent::MessageReceived)), - 47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), - 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), - 275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)), - 67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)), - 66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)), - 69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)), - 68 => Some(EventType::Dane(DaneEvent::CertificateParseError)), - 73 => Some(EventType::Dane(DaneEvent::TlsaRecordMatch)), - 70 => Some(EventType::Dane(DaneEvent::TlsaRecordFetch)), - 71 => Some(EventType::Dane(DaneEvent::TlsaRecordFetchError)), - 75 => Some(EventType::Dane(DaneEvent::TlsaRecordNotFound)), - 74 => Some(EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned)), - 72 => Some(EventType::Dane(DaneEvent::TlsaRecordInvalid)), - 605 => Some(EventType::Dane(DaneEvent::BogusDnssecRecord)), - 77 => Some(EventType::Delivery(DeliveryEvent::AttemptStart)), - 76 => Some(EventType::Delivery(DeliveryEvent::AttemptEnd)), - 80 => Some(EventType::Delivery(DeliveryEvent::Completed)), - 92 => Some(EventType::Delivery(DeliveryEvent::Failed)), - 85 => Some(EventType::Delivery(DeliveryEvent::DomainDeliveryStart)), - 101 => Some(EventType::Delivery(DeliveryEvent::MxLookup)), - 102 => Some(EventType::Delivery(DeliveryEvent::MxLookupFailed)), - 95 => Some(EventType::Delivery(DeliveryEvent::IpLookup)), - 96 => Some(EventType::Delivery(DeliveryEvent::IpLookupFailed)), - 103 => Some(EventType::Delivery(DeliveryEvent::NullMx)), - 82 => Some(EventType::Delivery(DeliveryEvent::Connect)), - 83 => Some(EventType::Delivery(DeliveryEvent::ConnectError)), - 100 => Some(EventType::Delivery(DeliveryEvent::MissingOutboundHostname)), - 93 => Some(EventType::Delivery(DeliveryEvent::GreetingFailed)), - 90 => Some(EventType::Delivery(DeliveryEvent::Ehlo)), - 91 => Some(EventType::Delivery(DeliveryEvent::EhloRejected)), - 78 => Some(EventType::Delivery(DeliveryEvent::Auth)), - 79 => Some(EventType::Delivery(DeliveryEvent::AuthFailed)), - 97 => Some(EventType::Delivery(DeliveryEvent::MailFrom)), - 98 => Some(EventType::Delivery(DeliveryEvent::MailFromRejected)), - 84 => Some(EventType::Delivery(DeliveryEvent::Delivered)), - 107 => Some(EventType::Delivery(DeliveryEvent::RcptTo)), - 109 => Some(EventType::Delivery(DeliveryEvent::RcptToRejected)), - 108 => Some(EventType::Delivery(DeliveryEvent::RcptToFailed)), - 99 => Some(EventType::Delivery(DeliveryEvent::MessageRejected)), - 110 => Some(EventType::Delivery(DeliveryEvent::StartTls)), - 113 => Some(EventType::Delivery(DeliveryEvent::StartTlsUnavailable)), - 112 => Some(EventType::Delivery(DeliveryEvent::StartTlsError)), - 111 => Some(EventType::Delivery(DeliveryEvent::StartTlsDisabled)), - 94 => Some(EventType::Delivery(DeliveryEvent::ImplicitTlsError)), - 81 => Some(EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded)), - 104 => Some(EventType::Delivery(DeliveryEvent::RateLimitExceeded)), - 86 => Some(EventType::Delivery(DeliveryEvent::DoubleBounce)), - 88 => Some(EventType::Delivery(DeliveryEvent::DsnSuccess)), - 89 => Some(EventType::Delivery(DeliveryEvent::DsnTempFail)), - 87 => Some(EventType::Delivery(DeliveryEvent::DsnPermFail)), - 105 => Some(EventType::Delivery(DeliveryEvent::RawInput)), - 106 => Some(EventType::Delivery(DeliveryEvent::RawOutput)), - 121 => Some(EventType::Dkim(DkimEvent::Pass)), - 119 => Some(EventType::Dkim(DkimEvent::Neutral)), - 114 => Some(EventType::Dkim(DkimEvent::Fail)), - 122 => Some(EventType::Dkim(DkimEvent::PermError)), - 127 => Some(EventType::Dkim(DkimEvent::TempError)), - 120 => Some(EventType::Dkim(DkimEvent::None)), - 131 => Some(EventType::Dkim(DkimEvent::UnsupportedVersion)), - 128 => Some(EventType::Dkim(DkimEvent::UnsupportedAlgorithm)), - 129 => Some(EventType::Dkim(DkimEvent::UnsupportedCanonicalization)), - 130 => Some(EventType::Dkim(DkimEvent::UnsupportedKeyType)), - 116 => Some(EventType::Dkim(DkimEvent::FailedBodyHashMatch)), - 117 => Some(EventType::Dkim(DkimEvent::FailedVerification)), - 115 => Some(EventType::Dkim(DkimEvent::FailedAuidMatch)), - 123 => Some(EventType::Dkim(DkimEvent::RevokedPublicKey)), - 118 => Some(EventType::Dkim(DkimEvent::IncompatibleAlgorithms)), - 124 => Some(EventType::Dkim(DkimEvent::SignatureExpired)), - 125 => Some(EventType::Dkim(DkimEvent::SignatureLength)), - 126 => Some(EventType::Dkim(DkimEvent::SignerNotFound)), - 592 => Some(EventType::Dkim(DkimEvent::BuildError)), - 596 => Some(EventType::Dkim(DkimEvent::SignatureCreated)), - 597 => Some(EventType::Dkim(DkimEvent::SignaturePublished)), - 598 => Some(EventType::Dkim(DkimEvent::SignatureRetiring)), - 599 => Some(EventType::Dkim(DkimEvent::SignatureRetired)), - 600 => Some(EventType::Dkim(DkimEvent::SignatureDeleted)), - 606 => Some(EventType::Dkim(DkimEvent::InstanceMissing)), - 607 => Some(EventType::Dkim(DkimEvent::InstanceSyntax)), - 608 => Some(EventType::Dkim(DkimEvent::InstanceTagMissing)), - 609 => Some(EventType::Dkim(DkimEvent::InstanceNotSigned)), - 610 => Some(EventType::Dkim(DkimEvent::InstanceAboveSignature)), - 611 => Some(EventType::Dkim(DkimEvent::SignatureMissing)), - 612 => Some(EventType::Dkim(DkimEvent::SignatureSyntax)), - 613 => Some(EventType::Dkim(DkimEvent::SignatureTagMissing)), - 614 => Some(EventType::Dkim(DkimEvent::SignatureTagUnexpected)), - 615 => Some(EventType::Dkim(DkimEvent::SequenceGap)), - 616 => Some(EventType::Dkim(DkimEvent::SequenceOverflow)), - 635 => Some(EventType::Dkim(DkimEvent::ChainTooLong)), - 617 => Some(EventType::Dkim(DkimEvent::MailFromMismatch)), - 618 => Some(EventType::Dkim(DkimEvent::RcptToMismatch)), - 619 => Some(EventType::Dkim(DkimEvent::MailFromDomainMismatch)), - 620 => Some(EventType::Dkim(DkimEvent::NextDomainMismatch)), - 636 => Some(EventType::Dkim(DkimEvent::CustodyBreak)), - 621 => Some(EventType::Dkim(DkimEvent::PublicKeyFetch)), - 622 => Some(EventType::Dkim(DkimEvent::PublicKeyMissing)), - 623 => Some(EventType::Dkim(DkimEvent::PublicKeyMultiple)), - 624 => Some(EventType::Dkim(DkimEvent::PublicKeySyntax)), - 625 => Some(EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch)), - 626 => Some(EventType::Dkim(DkimEvent::NoValidAlgorithm)), - 627 => Some(EventType::Dkim(DkimEvent::HeaderHashMismatch)), - 628 => Some(EventType::Dkim(DkimEvent::Modified)), - 629 => Some(EventType::Dkim(DkimEvent::Exploded)), - 134 => Some(EventType::Dmarc(DmarcEvent::Pass)), - 132 => Some(EventType::Dmarc(DmarcEvent::Fail)), - 135 => Some(EventType::Dmarc(DmarcEvent::PermError)), - 136 => Some(EventType::Dmarc(DmarcEvent::TempError)), - 133 => Some(EventType::Dmarc(DmarcEvent::None)), + 647 => Some(EventType::Arc(ArcEvent::ChainInvalid)), + 648 => Some(EventType::Arc(ArcEvent::SignatureInvalid)), + 649 => Some(EventType::Arc(ArcEvent::HashMismatch)), + 32 => Some(EventType::Auth(AuthEvent::Success)), + 30 => Some(EventType::Auth(AuthEvent::Failed)), + 537 => Some(EventType::Auth(AuthEvent::TokenExpired)), + 31 => Some(EventType::Auth(AuthEvent::MfaRequired)), + 33 => Some(EventType::Auth(AuthEvent::TooManyAttempts)), + 538 => Some(EventType::Auth(AuthEvent::ClientRegistration)), + 29 => Some(EventType::Auth(AuthEvent::Error)), + 578 => Some(EventType::Auth(AuthEvent::Warning)), + 260 => Some(EventType::Auth(AuthEvent::CredentialExpired)), + 45 => Some(EventType::Cache(CacheEvent::Miss)), + 46 => Some(EventType::Cache(CacheEvent::Hit)), + 47 => Some(EventType::Cache(CacheEvent::Stale)), + 560 => Some(EventType::Cache(CacheEvent::Update)), + 592 => Some(EventType::Cache(CacheEvent::EntryTooLarge)), + 600 => Some(EventType::Cache(CacheEvent::Invalidate)), + 594 => Some(EventType::Cache(CacheEvent::SwapHit)), + 595 => Some(EventType::Cache(CacheEvent::SwapMiss)), + 596 => Some(EventType::Cache(CacheEvent::SwapWrite)), + 597 => Some(EventType::Cache(CacheEvent::SwapError)), + 598 => Some(EventType::Cache(CacheEvent::SwapMetrics)), + 599 => Some(EventType::Cache(CacheEvent::SwapFlush)), + 559 => Some(EventType::Calendar(CalendarEvent::RuleExpansionError)), + 562 => Some(EventType::Calendar(CalendarEvent::AlarmSent)), + 563 => Some(EventType::Calendar(CalendarEvent::AlarmSkipped)), + 564 => Some(EventType::Calendar(CalendarEvent::AlarmRecipientOverride)), + 565 => Some(EventType::Calendar(CalendarEvent::AlarmFailed)), + 566 => Some(EventType::Calendar(CalendarEvent::ItipMessageSent)), + 567 => Some(EventType::Calendar(CalendarEvent::ItipMessageReceived)), + 568 => Some(EventType::Calendar(CalendarEvent::ItipMessageError)), + 262 => Some(EventType::Cluster(ClusterEvent::Startup)), + 34 => Some(EventType::Cluster(ClusterEvent::SubscriberStart)), + 35 => Some(EventType::Cluster(ClusterEvent::SubscriberStop)), + 36 => Some(EventType::Cluster(ClusterEvent::SubscriberError)), + 37 => Some(EventType::Cluster(ClusterEvent::SubscriberDisconnected)), + 38 => Some(EventType::Cluster(ClusterEvent::PublisherStart)), + 39 => Some(EventType::Cluster(ClusterEvent::PublisherStop)), + 40 => Some(EventType::Cluster(ClusterEvent::PublisherError)), + 41 => Some(EventType::Cluster(ClusterEvent::MessageReceived)), + 42 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), + 44 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), + 259 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)), + 62 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)), + 61 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)), + 64 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)), + 63 => Some(EventType::Dane(DaneEvent::CertificateParseError)), + 68 => Some(EventType::Dane(DaneEvent::TlsaRecordMatch)), + 65 => Some(EventType::Dane(DaneEvent::TlsaRecordFetch)), + 66 => Some(EventType::Dane(DaneEvent::TlsaRecordFetchError)), + 70 => Some(EventType::Dane(DaneEvent::TlsaRecordNotFound)), + 69 => Some(EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned)), + 67 => Some(EventType::Dane(DaneEvent::TlsaRecordInvalid)), + 588 => Some(EventType::Dane(DaneEvent::BogusDnssecRecord)), + 72 => Some(EventType::Delivery(DeliveryEvent::AttemptStart)), + 71 => Some(EventType::Delivery(DeliveryEvent::AttemptEnd)), + 75 => Some(EventType::Delivery(DeliveryEvent::Completed)), + 87 => Some(EventType::Delivery(DeliveryEvent::Failed)), + 80 => Some(EventType::Delivery(DeliveryEvent::DomainDeliveryStart)), + 96 => Some(EventType::Delivery(DeliveryEvent::MxLookup)), + 97 => Some(EventType::Delivery(DeliveryEvent::MxLookupFailed)), + 90 => Some(EventType::Delivery(DeliveryEvent::IpLookup)), + 91 => Some(EventType::Delivery(DeliveryEvent::IpLookupFailed)), + 98 => Some(EventType::Delivery(DeliveryEvent::NullMx)), + 77 => Some(EventType::Delivery(DeliveryEvent::Connect)), + 78 => Some(EventType::Delivery(DeliveryEvent::ConnectError)), + 95 => Some(EventType::Delivery(DeliveryEvent::MissingOutboundHostname)), + 88 => Some(EventType::Delivery(DeliveryEvent::GreetingFailed)), + 85 => Some(EventType::Delivery(DeliveryEvent::Ehlo)), + 86 => Some(EventType::Delivery(DeliveryEvent::EhloRejected)), + 73 => Some(EventType::Delivery(DeliveryEvent::Auth)), + 74 => Some(EventType::Delivery(DeliveryEvent::AuthFailed)), + 92 => Some(EventType::Delivery(DeliveryEvent::MailFrom)), + 93 => Some(EventType::Delivery(DeliveryEvent::MailFromRejected)), + 79 => Some(EventType::Delivery(DeliveryEvent::Delivered)), + 102 => Some(EventType::Delivery(DeliveryEvent::RcptTo)), + 104 => Some(EventType::Delivery(DeliveryEvent::RcptToRejected)), + 103 => Some(EventType::Delivery(DeliveryEvent::RcptToFailed)), + 94 => Some(EventType::Delivery(DeliveryEvent::MessageRejected)), + 105 => Some(EventType::Delivery(DeliveryEvent::StartTls)), + 108 => Some(EventType::Delivery(DeliveryEvent::StartTlsUnavailable)), + 107 => Some(EventType::Delivery(DeliveryEvent::StartTlsError)), + 106 => Some(EventType::Delivery(DeliveryEvent::StartTlsDisabled)), + 89 => Some(EventType::Delivery(DeliveryEvent::ImplicitTlsError)), + 76 => Some(EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded)), + 99 => Some(EventType::Delivery(DeliveryEvent::RateLimitExceeded)), + 81 => Some(EventType::Delivery(DeliveryEvent::DoubleBounce)), + 83 => Some(EventType::Delivery(DeliveryEvent::DsnSuccess)), + 84 => Some(EventType::Delivery(DeliveryEvent::DsnTempFail)), + 82 => Some(EventType::Delivery(DeliveryEvent::DsnPermFail)), + 100 => Some(EventType::Delivery(DeliveryEvent::RawInput)), + 101 => Some(EventType::Delivery(DeliveryEvent::RawOutput)), + 112 => Some(EventType::Dkim(DkimEvent::Pass)), + 110 => Some(EventType::Dkim(DkimEvent::Neutral)), + 109 => Some(EventType::Dkim(DkimEvent::Fail)), + 113 => Some(EventType::Dkim(DkimEvent::PermError)), + 115 => Some(EventType::Dkim(DkimEvent::TempError)), + 111 => Some(EventType::Dkim(DkimEvent::None)), + 643 => Some(EventType::Dkim(DkimEvent::SignatureInvalid)), + 644 => Some(EventType::Dkim(DkimEvent::PublicKeyError)), + 645 => Some(EventType::Dkim(DkimEvent::VerificationFailed)), + 646 => Some(EventType::Dkim(DkimEvent::HashMismatch)), + 114 => Some(EventType::Dkim(DkimEvent::SignerNotFound)), + 575 => Some(EventType::Dkim(DkimEvent::BuildError)), + 579 => Some(EventType::Dkim(DkimEvent::SignatureCreated)), + 580 => Some(EventType::Dkim(DkimEvent::SignaturePublished)), + 581 => Some(EventType::Dkim(DkimEvent::SignatureRetiring)), + 582 => Some(EventType::Dkim(DkimEvent::SignatureRetired)), + 583 => Some(EventType::Dkim(DkimEvent::SignatureDeleted)), + 633 => Some(EventType::Dkim2(Dkim2Event::InstanceInvalid)), + 634 => Some(EventType::Dkim2(Dkim2Event::SignatureInvalid)), + 635 => Some(EventType::Dkim2(Dkim2Event::ChainInvalid)), + 636 => Some(EventType::Dkim2(Dkim2Event::TimestampInvalid)), + 637 => Some(EventType::Dkim2(Dkim2Event::EnvelopeMismatch)), + 638 => Some(EventType::Dkim2(Dkim2Event::CustodyBroken)), + 639 => Some(EventType::Dkim2(Dkim2Event::PublicKeyError)), + 640 => Some(EventType::Dkim2(Dkim2Event::VerificationFailed)), + 641 => Some(EventType::Dkim2(Dkim2Event::HashMismatch)), + 642 => Some(EventType::Dkim2(Dkim2Event::RecipeError)), + 650 => Some(EventType::Dkim2(Dkim2Event::SigningFailed)), + 118 => Some(EventType::Dmarc(DmarcEvent::Pass)), + 116 => Some(EventType::Dmarc(DmarcEvent::Fail)), + 119 => Some(EventType::Dmarc(DmarcEvent::PermError)), + 120 => Some(EventType::Dmarc(DmarcEvent::TempError)), + 117 => Some(EventType::Dmarc(DmarcEvent::None)), 8 => Some(EventType::Dns(DnsEvent::RecordCreated)), 9 => Some(EventType::Dns(DnsEvent::RecordCreationFailed)), 10 => Some(EventType::Dns(DnsEvent::RecordDeletionFailed)), @@ -2245,517 +2228,527 @@ impl EventType { 11 => Some(EventType::Dns(DnsEvent::RecordLookupFailed)), 13 => Some(EventType::Dns(DnsEvent::RecordPropagated)), 14 => Some(EventType::Dns(DnsEvent::RecordPropagationTimeout)), - 591 => Some(EventType::Dns(DnsEvent::BuildError)), - 139 => Some(EventType::Eval(EvalEvent::Result)), - 138 => Some(EventType::Eval(EvalEvent::Error)), - 137 => Some(EventType::Eval(EvalEvent::DirectoryNotFound)), - 140 => Some(EventType::Eval(EvalEvent::StoreNotFound)), - 153 => Some(EventType::Http(HttpEvent::ConnectionStart)), - 152 => Some(EventType::Http(HttpEvent::ConnectionEnd)), - 154 => Some(EventType::Http(HttpEvent::Error)), - 156 => Some(EventType::Http(HttpEvent::RequestUrl)), - 155 => Some(EventType::Http(HttpEvent::RequestBody)), - 157 => Some(EventType::Http(HttpEvent::ResponseBody)), - 158 => Some(EventType::Http(HttpEvent::XForwardedMissing)), - 163 => Some(EventType::Imap(ImapEvent::ConnectionStart)), - 162 => Some(EventType::Imap(ImapEvent::ConnectionEnd)), - 171 => Some(EventType::Imap(ImapEvent::GetAcl)), - 188 => Some(EventType::Imap(ImapEvent::SetAcl)), - 180 => Some(EventType::Imap(ImapEvent::MyRights)), - 176 => Some(EventType::Imap(ImapEvent::ListRights)), - 159 => Some(EventType::Imap(ImapEvent::Append)), - 160 => Some(EventType::Imap(ImapEvent::Capabilities)), - 172 => Some(EventType::Imap(ImapEvent::Id)), - 161 => Some(EventType::Imap(ImapEvent::Close)), - 164 => Some(EventType::Imap(ImapEvent::Copy)), - 179 => Some(EventType::Imap(ImapEvent::Move)), - 165 => Some(EventType::Imap(ImapEvent::CreateMailbox)), - 166 => Some(EventType::Imap(ImapEvent::DeleteMailbox)), - 185 => Some(EventType::Imap(ImapEvent::RenameMailbox)), - 167 => Some(EventType::Imap(ImapEvent::Enable)), - 169 => Some(EventType::Imap(ImapEvent::Expunge)), - 170 => Some(EventType::Imap(ImapEvent::Fetch)), - 173 => Some(EventType::Imap(ImapEvent::IdleStart)), - 174 => Some(EventType::Imap(ImapEvent::IdleStop)), - 175 => Some(EventType::Imap(ImapEvent::List)), - 178 => Some(EventType::Imap(ImapEvent::Lsub)), - 177 => Some(EventType::Imap(ImapEvent::Logout)), - 181 => Some(EventType::Imap(ImapEvent::Namespace)), - 182 => Some(EventType::Imap(ImapEvent::Noop)), - 186 => Some(EventType::Imap(ImapEvent::Search)), - 189 => Some(EventType::Imap(ImapEvent::Sort)), - 187 => Some(EventType::Imap(ImapEvent::Select)), - 190 => Some(EventType::Imap(ImapEvent::Status)), - 191 => Some(EventType::Imap(ImapEvent::Store)), - 192 => Some(EventType::Imap(ImapEvent::Subscribe)), - 194 => Some(EventType::Imap(ImapEvent::Unsubscribe)), - 193 => Some(EventType::Imap(ImapEvent::Thread)), - 634 => Some(EventType::Imap(ImapEvent::UidBatches)), - 57 => Some(EventType::Imap(ImapEvent::GetQuota)), - 168 => Some(EventType::Imap(ImapEvent::Error)), - 183 => Some(EventType::Imap(ImapEvent::RawInput)), - 184 => Some(EventType::Imap(ImapEvent::RawOutput)), - 200 => Some(EventType::IncomingReport(IncomingReportEvent::DmarcReport)), - 201 => Some(EventType::IncomingReport( + 123 => Some(EventType::Eval(EvalEvent::Result)), + 122 => Some(EventType::Eval(EvalEvent::Error)), + 121 => Some(EventType::Eval(EvalEvent::DirectoryNotFound)), + 124 => Some(EventType::Eval(EvalEvent::StoreNotFound)), + 137 => Some(EventType::Http(HttpEvent::ConnectionStart)), + 136 => Some(EventType::Http(HttpEvent::ConnectionEnd)), + 138 => Some(EventType::Http(HttpEvent::Error)), + 140 => Some(EventType::Http(HttpEvent::RequestUrl)), + 139 => Some(EventType::Http(HttpEvent::RequestBody)), + 141 => Some(EventType::Http(HttpEvent::ResponseBody)), + 142 => Some(EventType::Http(HttpEvent::XForwardedMissing)), + 625 => Some(EventType::Http(HttpEvent::XForwardedInvalid)), + 147 => Some(EventType::Imap(ImapEvent::ConnectionStart)), + 146 => Some(EventType::Imap(ImapEvent::ConnectionEnd)), + 155 => Some(EventType::Imap(ImapEvent::GetAcl)), + 172 => Some(EventType::Imap(ImapEvent::SetAcl)), + 164 => Some(EventType::Imap(ImapEvent::MyRights)), + 160 => Some(EventType::Imap(ImapEvent::ListRights)), + 143 => Some(EventType::Imap(ImapEvent::Append)), + 144 => Some(EventType::Imap(ImapEvent::Capabilities)), + 156 => Some(EventType::Imap(ImapEvent::Id)), + 145 => Some(EventType::Imap(ImapEvent::Close)), + 148 => Some(EventType::Imap(ImapEvent::Copy)), + 163 => Some(EventType::Imap(ImapEvent::Move)), + 149 => Some(EventType::Imap(ImapEvent::CreateMailbox)), + 150 => Some(EventType::Imap(ImapEvent::DeleteMailbox)), + 169 => Some(EventType::Imap(ImapEvent::RenameMailbox)), + 151 => Some(EventType::Imap(ImapEvent::Enable)), + 153 => Some(EventType::Imap(ImapEvent::Expunge)), + 154 => Some(EventType::Imap(ImapEvent::Fetch)), + 157 => Some(EventType::Imap(ImapEvent::IdleStart)), + 158 => Some(EventType::Imap(ImapEvent::IdleStop)), + 159 => Some(EventType::Imap(ImapEvent::List)), + 162 => Some(EventType::Imap(ImapEvent::Lsub)), + 161 => Some(EventType::Imap(ImapEvent::Logout)), + 165 => Some(EventType::Imap(ImapEvent::Namespace)), + 166 => Some(EventType::Imap(ImapEvent::Noop)), + 170 => Some(EventType::Imap(ImapEvent::Search)), + 173 => Some(EventType::Imap(ImapEvent::Sort)), + 171 => Some(EventType::Imap(ImapEvent::Select)), + 174 => Some(EventType::Imap(ImapEvent::Status)), + 175 => Some(EventType::Imap(ImapEvent::Store)), + 176 => Some(EventType::Imap(ImapEvent::Subscribe)), + 178 => Some(EventType::Imap(ImapEvent::Unsubscribe)), + 177 => Some(EventType::Imap(ImapEvent::Thread)), + 593 => Some(EventType::Imap(ImapEvent::UidBatches)), + 52 => Some(EventType::Imap(ImapEvent::GetQuota)), + 607 => Some(EventType::Imap(ImapEvent::GetMetadata)), + 608 => Some(EventType::Imap(ImapEvent::SetMetadata)), + 152 => Some(EventType::Imap(ImapEvent::Error)), + 167 => Some(EventType::Imap(ImapEvent::RawInput)), + 168 => Some(EventType::Imap(ImapEvent::RawOutput)), + 184 => Some(EventType::IncomingReport(IncomingReportEvent::DmarcReport)), + 185 => Some(EventType::IncomingReport( IncomingReportEvent::DmarcReportWithWarnings, )), - 206 => Some(EventType::IncomingReport(IncomingReportEvent::TlsReport)), - 207 => Some(EventType::IncomingReport( + 190 => Some(EventType::IncomingReport(IncomingReportEvent::TlsReport)), + 191 => Some(EventType::IncomingReport( IncomingReportEvent::TlsReportWithWarnings, )), - 195 => Some(EventType::IncomingReport(IncomingReportEvent::AbuseReport)), - 197 => Some(EventType::IncomingReport( + 179 => Some(EventType::IncomingReport(IncomingReportEvent::AbuseReport)), + 181 => Some(EventType::IncomingReport( IncomingReportEvent::AuthFailureReport, )), - 202 => Some(EventType::IncomingReport(IncomingReportEvent::FraudReport)), - 204 => Some(EventType::IncomingReport( + 186 => Some(EventType::IncomingReport(IncomingReportEvent::FraudReport)), + 188 => Some(EventType::IncomingReport( IncomingReportEvent::NotSpamReport, )), - 209 => Some(EventType::IncomingReport(IncomingReportEvent::VirusReport)), - 205 => Some(EventType::IncomingReport(IncomingReportEvent::OtherReport)), - 203 => Some(EventType::IncomingReport( + 193 => Some(EventType::IncomingReport(IncomingReportEvent::VirusReport)), + 189 => Some(EventType::IncomingReport(IncomingReportEvent::OtherReport)), + 187 => Some(EventType::IncomingReport( IncomingReportEvent::MessageParseFailed, )), - 199 => Some(EventType::IncomingReport( + 183 => Some(EventType::IncomingReport( IncomingReportEvent::DmarcParseFailed, )), - 208 => Some(EventType::IncomingReport( + 192 => Some(EventType::IncomingReport( IncomingReportEvent::TlsRpcParseFailed, )), - 196 => Some(EventType::IncomingReport( + 180 => Some(EventType::IncomingReport( IncomingReportEvent::ArfParseFailed, )), - 198 => Some(EventType::IncomingReport( + 182 => Some(EventType::IncomingReport( IncomingReportEvent::DecompressError, )), - 212 => Some(EventType::Iprev(IprevEvent::Pass)), - 210 => Some(EventType::Iprev(IprevEvent::Fail)), - 213 => Some(EventType::Iprev(IprevEvent::PermError)), - 214 => Some(EventType::Iprev(IprevEvent::TempError)), - 211 => Some(EventType::Iprev(IprevEvent::None)), - 223 => Some(EventType::Jmap(JmapEvent::MethodCall)), - 221 => Some(EventType::Jmap(JmapEvent::InvalidArguments)), - 227 => Some(EventType::Jmap(JmapEvent::RequestTooLarge)), - 228 => Some(EventType::Jmap(JmapEvent::StateMismatch)), - 218 => Some(EventType::Jmap(JmapEvent::AnchorNotFound)), - 232 => Some(EventType::Jmap(JmapEvent::UnsupportedFilter)), - 233 => Some(EventType::Jmap(JmapEvent::UnsupportedSort)), - 231 => Some(EventType::Jmap(JmapEvent::UnknownMethod)), - 222 => Some(EventType::Jmap(JmapEvent::InvalidResultReference)), - 220 => Some(EventType::Jmap(JmapEvent::Forbidden)), - 215 => Some(EventType::Jmap(JmapEvent::AccountNotFound)), - 216 => Some(EventType::Jmap(JmapEvent::AccountNotSupportedByMethod)), - 217 => Some(EventType::Jmap(JmapEvent::AccountReadOnly)), - 224 => Some(EventType::Jmap(JmapEvent::NotFound)), - 219 => Some(EventType::Jmap(JmapEvent::CannotCalculateChanges)), - 230 => Some(EventType::Jmap(JmapEvent::UnknownDataType)), - 229 => Some(EventType::Jmap(JmapEvent::UnknownCapability)), - 225 => Some(EventType::Jmap(JmapEvent::NotJson)), - 226 => Some(EventType::Jmap(JmapEvent::NotRequest)), - 235 => Some(EventType::Jmap(JmapEvent::WebsocketStart)), - 236 => Some(EventType::Jmap(JmapEvent::WebsocketStop)), - 234 => Some(EventType::Jmap(JmapEvent::WebsocketError)), - 243 => Some(EventType::Limit(LimitEvent::SizeRequest)), - 244 => Some(EventType::Limit(LimitEvent::SizeUpload)), - 238 => Some(EventType::Limit(LimitEvent::CallsIn)), - 240 => Some(EventType::Limit(LimitEvent::ConcurrentRequest)), - 241 => Some(EventType::Limit(LimitEvent::ConcurrentUpload)), - 239 => Some(EventType::Limit(LimitEvent::ConcurrentConnection)), - 242 => Some(EventType::Limit(LimitEvent::Quota)), - 237 => Some(EventType::Limit(LimitEvent::BlobQuota)), - 553 => Some(EventType::Limit(LimitEvent::TenantQuota)), - 245 => Some(EventType::Limit(LimitEvent::TooManyRequests)), - 254 => Some(EventType::MailAuth(MailAuthEvent::ParseError)), - 252 => Some(EventType::MailAuth(MailAuthEvent::MissingParameters)), - 253 => Some(EventType::MailAuth(MailAuthEvent::NoHeadersFound)), - 247 => Some(EventType::MailAuth(MailAuthEvent::Crypto)), - 251 => Some(EventType::MailAuth(MailAuthEvent::Io)), - 246 => Some(EventType::MailAuth(MailAuthEvent::Base64)), - 248 => Some(EventType::MailAuth(MailAuthEvent::DnsError)), - 250 => Some(EventType::MailAuth(MailAuthEvent::DnsRecordNotFound)), - 249 => Some(EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType)), - 255 => Some(EventType::MailAuth(MailAuthEvent::PolicyNotAligned)), - 259 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionStart)), - 258 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionEnd)), - 260 => Some(EventType::ManageSieve(ManageSieveEvent::CreateScript)), - 274 => Some(EventType::ManageSieve(ManageSieveEvent::UpdateScript)), - 263 => Some(EventType::ManageSieve(ManageSieveEvent::GetScript)), - 261 => Some(EventType::ManageSieve(ManageSieveEvent::DeleteScript)), - 270 => Some(EventType::ManageSieve(ManageSieveEvent::RenameScript)), - 257 => Some(EventType::ManageSieve(ManageSieveEvent::CheckScript)), - 264 => Some(EventType::ManageSieve(ManageSieveEvent::HaveSpace)), - 265 => Some(EventType::ManageSieve(ManageSieveEvent::ListScripts)), - 271 => Some(EventType::ManageSieve(ManageSieveEvent::SetActive)), - 256 => Some(EventType::ManageSieve(ManageSieveEvent::Capabilities)), - 272 => Some(EventType::ManageSieve(ManageSieveEvent::StartTls)), - 273 => Some(EventType::ManageSieve(ManageSieveEvent::Unauthenticate)), - 266 => Some(EventType::ManageSieve(ManageSieveEvent::Logout)), - 267 => Some(EventType::ManageSieve(ManageSieveEvent::Noop)), - 262 => Some(EventType::ManageSieve(ManageSieveEvent::Error)), - 268 => Some(EventType::ManageSieve(ManageSieveEvent::RawInput)), - 269 => Some(EventType::ManageSieve(ManageSieveEvent::RawOutput)), - 283 => Some(EventType::MessageIngest(MessageIngestEvent::Ham)), - 286 => Some(EventType::MessageIngest(MessageIngestEvent::Spam)), - 284 => Some(EventType::MessageIngest(MessageIngestEvent::ImapAppend)), - 285 => Some(EventType::MessageIngest(MessageIngestEvent::JmapAppend)), - 281 => Some(EventType::MessageIngest(MessageIngestEvent::Duplicate)), - 282 => Some(EventType::MessageIngest(MessageIngestEvent::Error)), - 142 => Some(EventType::MessageIngest(MessageIngestEvent::SearchIndex)), - 299 => Some(EventType::Milter(MilterEvent::Read)), - 303 => Some(EventType::Milter(MilterEvent::Write)), - 287 => Some(EventType::Milter(MilterEvent::ActionAccept)), - 289 => Some(EventType::Milter(MilterEvent::ActionDiscard)), - 290 => Some(EventType::Milter(MilterEvent::ActionReject)), - 293 => Some(EventType::Milter(MilterEvent::ActionTempFail)), - 291 => Some(EventType::Milter(MilterEvent::ActionReplyCode)), - 288 => Some(EventType::Milter(MilterEvent::ActionConnectionFailure)), - 292 => Some(EventType::Milter(MilterEvent::ActionShutdown)), - 297 => Some(EventType::Milter(MilterEvent::IoError)), - 296 => Some(EventType::Milter(MilterEvent::FrameTooLarge)), - 295 => Some(EventType::Milter(MilterEvent::FrameInvalid)), - 302 => Some(EventType::Milter(MilterEvent::UnexpectedResponse)), - 300 => Some(EventType::Milter(MilterEvent::Timeout)), - 301 => Some(EventType::Milter(MilterEvent::TlsInvalidName)), - 294 => Some(EventType::Milter(MilterEvent::Disconnected)), - 298 => Some(EventType::Milter(MilterEvent::ParseError)), - 304 => Some(EventType::MtaHook(MtaHookEvent::ActionAccept)), - 305 => Some(EventType::MtaHook(MtaHookEvent::ActionDiscard)), - 307 => Some(EventType::MtaHook(MtaHookEvent::ActionReject)), - 306 => Some(EventType::MtaHook(MtaHookEvent::ActionQuarantine)), - 308 => Some(EventType::MtaHook(MtaHookEvent::Error)), - 309 => Some(EventType::MtaSts(MtaStsEvent::Authorized)), - 311 => Some(EventType::MtaSts(MtaStsEvent::NotAuthorized)), - 312 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetch)), - 314 => Some(EventType::MtaSts(MtaStsEvent::PolicyNotFound)), - 313 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetchError)), - 310 => Some(EventType::MtaSts(MtaStsEvent::InvalidPolicy)), - 321 => Some(EventType::Network(NetworkEvent::ListenStart)), - 322 => Some(EventType::Network(NetworkEvent::ListenStop)), - 320 => Some(EventType::Network(NetworkEvent::ListenError)), - 316 => Some(EventType::Network(NetworkEvent::BindError)), - 324 => Some(EventType::Network(NetworkEvent::ReadError)), - 328 => Some(EventType::Network(NetworkEvent::WriteError)), - 319 => Some(EventType::Network(NetworkEvent::FlushError)), - 315 => Some(EventType::Network(NetworkEvent::AcceptError)), - 326 => Some(EventType::Network(NetworkEvent::SplitError)), - 327 => Some(EventType::Network(NetworkEvent::Timeout)), - 317 => Some(EventType::Network(NetworkEvent::Closed)), - 323 => Some(EventType::Network(NetworkEvent::ProxyError)), - 325 => Some(EventType::Network(NetworkEvent::SetOptError)), - 342 => Some(EventType::OutgoingReport(OutgoingReportEvent::SpfReport)), - 341 => Some(EventType::OutgoingReport( + 196 => Some(EventType::Iprev(IprevEvent::Pass)), + 194 => Some(EventType::Iprev(IprevEvent::Fail)), + 197 => Some(EventType::Iprev(IprevEvent::PermError)), + 198 => Some(EventType::Iprev(IprevEvent::TempError)), + 195 => Some(EventType::Iprev(IprevEvent::None)), + 207 => Some(EventType::Jmap(JmapEvent::MethodCall)), + 205 => Some(EventType::Jmap(JmapEvent::InvalidArguments)), + 211 => Some(EventType::Jmap(JmapEvent::RequestTooLarge)), + 212 => Some(EventType::Jmap(JmapEvent::StateMismatch)), + 202 => Some(EventType::Jmap(JmapEvent::AnchorNotFound)), + 216 => Some(EventType::Jmap(JmapEvent::UnsupportedFilter)), + 217 => Some(EventType::Jmap(JmapEvent::UnsupportedSort)), + 215 => Some(EventType::Jmap(JmapEvent::UnknownMethod)), + 206 => Some(EventType::Jmap(JmapEvent::InvalidResultReference)), + 204 => Some(EventType::Jmap(JmapEvent::Forbidden)), + 199 => Some(EventType::Jmap(JmapEvent::AccountNotFound)), + 200 => Some(EventType::Jmap(JmapEvent::AccountNotSupportedByMethod)), + 201 => Some(EventType::Jmap(JmapEvent::AccountReadOnly)), + 653 => Some(EventType::Jmap(JmapEvent::FromAccountNotFound)), + 654 => Some(EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod)), + 208 => Some(EventType::Jmap(JmapEvent::NotFound)), + 203 => Some(EventType::Jmap(JmapEvent::CannotCalculateChanges)), + 214 => Some(EventType::Jmap(JmapEvent::UnknownDataType)), + 213 => Some(EventType::Jmap(JmapEvent::UnknownCapability)), + 209 => Some(EventType::Jmap(JmapEvent::NotJson)), + 210 => Some(EventType::Jmap(JmapEvent::NotRequest)), + 219 => Some(EventType::Jmap(JmapEvent::WebsocketStart)), + 220 => Some(EventType::Jmap(JmapEvent::WebsocketStop)), + 218 => Some(EventType::Jmap(JmapEvent::WebsocketError)), + 602 => Some(EventType::Jmap(JmapEvent::ExpandDurationTooLarge)), + 603 => Some(EventType::Jmap(JmapEvent::CannotCalculateOccurrences)), + 604 => Some(EventType::Jmap(JmapEvent::TooLarge)), + 605 => Some(EventType::Jmap(JmapEvent::RateLimit)), + 606 => Some(EventType::Jmap(JmapEvent::ObjectNotFound)), + 227 => Some(EventType::Limit(LimitEvent::SizeRequest)), + 228 => Some(EventType::Limit(LimitEvent::SizeUpload)), + 222 => Some(EventType::Limit(LimitEvent::CallsIn)), + 224 => Some(EventType::Limit(LimitEvent::ConcurrentRequest)), + 225 => Some(EventType::Limit(LimitEvent::ConcurrentUpload)), + 223 => Some(EventType::Limit(LimitEvent::ConcurrentConnection)), + 226 => Some(EventType::Limit(LimitEvent::Quota)), + 221 => Some(EventType::Limit(LimitEvent::BlobQuota)), + 229 => Some(EventType::Limit(LimitEvent::TooManyRequests)), + 627 => Some(EventType::Limit(LimitEvent::MemoryBudget)), + 238 => Some(EventType::MailAuth(MailAuthEvent::ParseError)), + 236 => Some(EventType::MailAuth(MailAuthEvent::MissingParameters)), + 237 => Some(EventType::MailAuth(MailAuthEvent::NoHeadersFound)), + 231 => Some(EventType::MailAuth(MailAuthEvent::Crypto)), + 235 => Some(EventType::MailAuth(MailAuthEvent::Io)), + 230 => Some(EventType::MailAuth(MailAuthEvent::Base64)), + 232 => Some(EventType::MailAuth(MailAuthEvent::DnsError)), + 234 => Some(EventType::MailAuth(MailAuthEvent::DnsRecordNotFound)), + 233 => Some(EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType)), + 239 => Some(EventType::MailAuth(MailAuthEvent::PolicyNotAligned)), + 243 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionStart)), + 242 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionEnd)), + 244 => Some(EventType::ManageSieve(ManageSieveEvent::CreateScript)), + 258 => Some(EventType::ManageSieve(ManageSieveEvent::UpdateScript)), + 247 => Some(EventType::ManageSieve(ManageSieveEvent::GetScript)), + 245 => Some(EventType::ManageSieve(ManageSieveEvent::DeleteScript)), + 254 => Some(EventType::ManageSieve(ManageSieveEvent::RenameScript)), + 241 => Some(EventType::ManageSieve(ManageSieveEvent::CheckScript)), + 248 => Some(EventType::ManageSieve(ManageSieveEvent::HaveSpace)), + 249 => Some(EventType::ManageSieve(ManageSieveEvent::ListScripts)), + 255 => Some(EventType::ManageSieve(ManageSieveEvent::SetActive)), + 240 => Some(EventType::ManageSieve(ManageSieveEvent::Capabilities)), + 256 => Some(EventType::ManageSieve(ManageSieveEvent::StartTls)), + 257 => Some(EventType::ManageSieve(ManageSieveEvent::Unauthenticate)), + 250 => Some(EventType::ManageSieve(ManageSieveEvent::Logout)), + 251 => Some(EventType::ManageSieve(ManageSieveEvent::Noop)), + 246 => Some(EventType::ManageSieve(ManageSieveEvent::Error)), + 252 => Some(EventType::ManageSieve(ManageSieveEvent::RawInput)), + 253 => Some(EventType::ManageSieve(ManageSieveEvent::RawOutput)), + 267 => Some(EventType::MessageIngest(MessageIngestEvent::Ham)), + 270 => Some(EventType::MessageIngest(MessageIngestEvent::Spam)), + 268 => Some(EventType::MessageIngest(MessageIngestEvent::ImapAppend)), + 269 => Some(EventType::MessageIngest(MessageIngestEvent::JmapAppend)), + 265 => Some(EventType::MessageIngest(MessageIngestEvent::Duplicate)), + 266 => Some(EventType::MessageIngest(MessageIngestEvent::Error)), + 283 => Some(EventType::Milter(MilterEvent::Read)), + 287 => Some(EventType::Milter(MilterEvent::Write)), + 271 => Some(EventType::Milter(MilterEvent::ActionAccept)), + 273 => Some(EventType::Milter(MilterEvent::ActionDiscard)), + 274 => Some(EventType::Milter(MilterEvent::ActionReject)), + 277 => Some(EventType::Milter(MilterEvent::ActionTempFail)), + 275 => Some(EventType::Milter(MilterEvent::ActionReplyCode)), + 272 => Some(EventType::Milter(MilterEvent::ActionConnectionFailure)), + 276 => Some(EventType::Milter(MilterEvent::ActionShutdown)), + 281 => Some(EventType::Milter(MilterEvent::IoError)), + 280 => Some(EventType::Milter(MilterEvent::FrameTooLarge)), + 279 => Some(EventType::Milter(MilterEvent::FrameInvalid)), + 286 => Some(EventType::Milter(MilterEvent::UnexpectedResponse)), + 284 => Some(EventType::Milter(MilterEvent::Timeout)), + 285 => Some(EventType::Milter(MilterEvent::TlsInvalidName)), + 278 => Some(EventType::Milter(MilterEvent::Disconnected)), + 282 => Some(EventType::Milter(MilterEvent::ParseError)), + 288 => Some(EventType::MtaHook(MtaHookEvent::ActionAccept)), + 289 => Some(EventType::MtaHook(MtaHookEvent::ActionDiscard)), + 291 => Some(EventType::MtaHook(MtaHookEvent::ActionReject)), + 290 => Some(EventType::MtaHook(MtaHookEvent::ActionQuarantine)), + 292 => Some(EventType::MtaHook(MtaHookEvent::Error)), + 293 => Some(EventType::MtaSts(MtaStsEvent::Authorized)), + 295 => Some(EventType::MtaSts(MtaStsEvent::NotAuthorized)), + 296 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetch)), + 298 => Some(EventType::MtaSts(MtaStsEvent::PolicyNotFound)), + 297 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetchError)), + 294 => Some(EventType::MtaSts(MtaStsEvent::InvalidPolicy)), + 305 => Some(EventType::Network(NetworkEvent::ListenStart)), + 306 => Some(EventType::Network(NetworkEvent::ListenStop)), + 304 => Some(EventType::Network(NetworkEvent::ListenError)), + 300 => Some(EventType::Network(NetworkEvent::BindError)), + 308 => Some(EventType::Network(NetworkEvent::ReadError)), + 312 => Some(EventType::Network(NetworkEvent::WriteError)), + 303 => Some(EventType::Network(NetworkEvent::FlushError)), + 299 => Some(EventType::Network(NetworkEvent::AcceptError)), + 310 => Some(EventType::Network(NetworkEvent::SplitError)), + 311 => Some(EventType::Network(NetworkEvent::Timeout)), + 301 => Some(EventType::Network(NetworkEvent::Closed)), + 307 => Some(EventType::Network(NetworkEvent::ProxyError)), + 309 => Some(EventType::Network(NetworkEvent::SetOptError)), + 326 => Some(EventType::OutgoingReport(OutgoingReportEvent::SpfReport)), + 325 => Some(EventType::OutgoingReport( OutgoingReportEvent::SpfRateLimited, )), - 330 => Some(EventType::OutgoingReport(OutgoingReportEvent::DkimReport)), - 329 => Some(EventType::OutgoingReport( + 314 => Some(EventType::OutgoingReport(OutgoingReportEvent::DkimReport)), + 313 => Some(EventType::OutgoingReport( OutgoingReportEvent::DkimRateLimited, )), - 333 => Some(EventType::OutgoingReport(OutgoingReportEvent::DmarcReport)), - 332 => Some(EventType::OutgoingReport( + 317 => Some(EventType::OutgoingReport(OutgoingReportEvent::DmarcReport)), + 316 => Some(EventType::OutgoingReport( OutgoingReportEvent::DmarcRateLimited, )), - 331 => Some(EventType::OutgoingReport( + 315 => Some(EventType::OutgoingReport( OutgoingReportEvent::DmarcAggregateReport, )), - 344 => Some(EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate)), - 334 => Some(EventType::OutgoingReport( + 328 => Some(EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate)), + 318 => Some(EventType::OutgoingReport( OutgoingReportEvent::HttpSubmission, )), - 345 => Some(EventType::OutgoingReport( + 329 => Some(EventType::OutgoingReport( OutgoingReportEvent::UnauthorizedReportingAddress, )), - 340 => Some(EventType::OutgoingReport( + 324 => Some(EventType::OutgoingReport( OutgoingReportEvent::ReportingAddressValidationError, )), - 339 => Some(EventType::OutgoingReport(OutgoingReportEvent::NotFound)), - 343 => Some(EventType::OutgoingReport( + 323 => Some(EventType::OutgoingReport(OutgoingReportEvent::NotFound)), + 327 => Some(EventType::OutgoingReport( OutgoingReportEvent::SubmissionError, )), - 338 => Some(EventType::OutgoingReport( + 322 => Some(EventType::OutgoingReport( OutgoingReportEvent::NoRecipientsFound, )), - 337 => Some(EventType::OutgoingReport(OutgoingReportEvent::Locked)), - 59 => Some(EventType::OutgoingReport( + 321 => Some(EventType::OutgoingReport(OutgoingReportEvent::Locked)), + 54 => Some(EventType::OutgoingReport( OutgoingReportEvent::MaxSizeExceeded, )), - 348 => Some(EventType::Pop3(Pop3Event::ConnectionStart)), - 347 => Some(EventType::Pop3(Pop3Event::ConnectionEnd)), - 349 => Some(EventType::Pop3(Pop3Event::Delete)), - 358 => Some(EventType::Pop3(Pop3Event::Reset)), - 355 => Some(EventType::Pop3(Pop3Event::Quit)), - 351 => Some(EventType::Pop3(Pop3Event::Fetch)), - 352 => Some(EventType::Pop3(Pop3Event::List)), - 353 => Some(EventType::Pop3(Pop3Event::ListMessage)), - 361 => Some(EventType::Pop3(Pop3Event::Uidl)), - 362 => Some(EventType::Pop3(Pop3Event::UidlMessage)), - 360 => Some(EventType::Pop3(Pop3Event::Stat)), - 354 => Some(EventType::Pop3(Pop3Event::Noop)), - 346 => Some(EventType::Pop3(Pop3Event::Capabilities)), - 359 => Some(EventType::Pop3(Pop3Event::StartTls)), - 363 => Some(EventType::Pop3(Pop3Event::Utf8)), - 350 => Some(EventType::Pop3(Pop3Event::Error)), - 356 => Some(EventType::Pop3(Pop3Event::RawInput)), - 357 => Some(EventType::Pop3(Pop3Event::RawOutput)), - 373 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), - 371 => Some(EventType::PushSubscription(PushSubscriptionEvent::Error)), - 372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), - 390 => Some(EventType::Queue(QueueEvent::Started)), - 380 => Some(EventType::Queue(QueueEvent::MessageQueued)), - 381 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)), - 382 => Some(EventType::Queue(QueueEvent::ReportQueued)), - 379 => Some(EventType::Queue(QueueEvent::DsnQueued)), - 378 => Some(EventType::Queue(QueueEvent::AutogeneratedQueued)), - 385 => Some(EventType::Queue(QueueEvent::Rescheduled)), - 377 => Some(EventType::Queue(QueueEvent::Locked)), - 374 => Some(EventType::Queue(QueueEvent::BlobNotFound)), - 384 => Some(EventType::Queue(QueueEvent::RateLimitExceeded)), - 375 => Some(EventType::Queue(QueueEvent::ConcurrencyLimitExceeded)), - 383 => Some(EventType::Queue(QueueEvent::QuotaExceeded)), - 48 => Some(EventType::Queue(QueueEvent::BackPressure)), - 62 => Some(EventType::Registry(RegistryEvent::LocalReadError)), - 54 => Some(EventType::Registry(RegistryEvent::LocalWriteError)), - 60 => Some(EventType::Registry(RegistryEvent::LocalParseError)), - 65 => Some(EventType::Registry(RegistryEvent::ReadError)), - 58 => Some(EventType::Registry(RegistryEvent::WriteError)), - 56 => Some(EventType::Registry(RegistryEvent::DeserializationError)), - 61 => Some(EventType::Registry(RegistryEvent::BuildError)), - 55 => Some(EventType::Registry(RegistryEvent::BuildWarning)), - 64 => Some(EventType::Registry(RegistryEvent::NotSupported)), - 63 => Some(EventType::Registry(RegistryEvent::ValidationError)), - 389 => Some(EventType::Resource(ResourceEvent::NotFound)), - 386 => Some(EventType::Resource(ResourceEvent::BadParameters)), - 388 => Some(EventType::Resource(ResourceEvent::Error)), - 387 => Some(EventType::Resource(ResourceEvent::DownloadExternal)), - 601 => Some(EventType::Resource(ResourceEvent::ApplicationUpdated)), - 602 => Some(EventType::Resource(ResourceEvent::ApplicationUnpacked)), - 33 => Some(EventType::Security(SecurityEvent::AuthenticationBan)), - 549 => Some(EventType::Security(SecurityEvent::AbuseBan)), - 558 => Some(EventType::Security(SecurityEvent::ScanBan)), - 550 => Some(EventType::Security(SecurityEvent::LoiterBan)), - 318 => Some(EventType::Security(SecurityEvent::IpBlocked)), - 593 => Some(EventType::Security(SecurityEvent::IpBlockExpired)), - 594 => Some(EventType::Security(SecurityEvent::IpAllowExpired)), - 279 => Some(EventType::Security(SecurityEvent::IpUnauthorized)), - 552 => Some(EventType::Security(SecurityEvent::Unauthorized)), - 393 => Some(EventType::Server(ServerEvent::Startup)), - 392 => Some(EventType::Server(ServerEvent::Shutdown)), - 394 => Some(EventType::Server(ServerEvent::StartupError)), - 395 => Some(EventType::Server(ServerEvent::ThreadError)), - 391 => Some(EventType::Server(ServerEvent::Licensing)), - 603 => Some(EventType::Server(ServerEvent::RecoveryMode)), - 604 => Some(EventType::Server(ServerEvent::BootstrapMode)), - 396 => Some(EventType::Sieve(SieveEvent::ActionAccept)), - 397 => Some(EventType::Sieve(SieveEvent::ActionAcceptReplace)), - 398 => Some(EventType::Sieve(SieveEvent::ActionDiscard)), - 399 => Some(EventType::Sieve(SieveEvent::ActionReject)), - 406 => Some(EventType::Sieve(SieveEvent::SendMessage)), - 401 => Some(EventType::Sieve(SieveEvent::MessageTooLarge)), - 405 => Some(EventType::Sieve(SieveEvent::ScriptNotFound)), - 400 => Some(EventType::Sieve(SieveEvent::ListNotFound)), - 404 => Some(EventType::Sieve(SieveEvent::RuntimeError)), - 407 => Some(EventType::Sieve(SieveEvent::UnexpectedError)), - 402 => Some(EventType::Sieve(SieveEvent::NotSupported)), - 403 => Some(EventType::Sieve(SieveEvent::QuotaExceeded)), - 417 => Some(EventType::Smtp(SmtpEvent::ConnectionStart)), - 416 => Some(EventType::Smtp(SmtpEvent::ConnectionEnd)), - 428 => Some(EventType::Smtp(SmtpEvent::Error)), - 469 => Some(EventType::Smtp(SmtpEvent::IdNotFound)), - 415 => Some(EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded)), - 485 => Some(EventType::Smtp(SmtpEvent::TransferLimitExceeded)), - 461 => Some(EventType::Smtp(SmtpEvent::RateLimitExceeded)), - 481 => Some(EventType::Smtp(SmtpEvent::TimeLimitExceeded)), - 452 => Some(EventType::Smtp(SmtpEvent::MissingAuthDirectory)), - 450 => Some(EventType::Smtp(SmtpEvent::MessageParseFailed)), - 451 => Some(EventType::Smtp(SmtpEvent::MessageTooLarge)), - 443 => Some(EventType::Smtp(SmtpEvent::LoopDetected)), - 422 => Some(EventType::Smtp(SmtpEvent::DkimPass)), - 421 => Some(EventType::Smtp(SmtpEvent::DkimFail)), - 630 => Some(EventType::Smtp(SmtpEvent::Dkim2Pass)), - 631 => Some(EventType::Smtp(SmtpEvent::Dkim2Fail)), - 632 => Some(EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded)), - 410 => Some(EventType::Smtp(SmtpEvent::ArcPass)), - 409 => Some(EventType::Smtp(SmtpEvent::ArcFail)), - 474 => Some(EventType::Smtp(SmtpEvent::SpfEhloPass)), - 473 => Some(EventType::Smtp(SmtpEvent::SpfEhloFail)), - 476 => Some(EventType::Smtp(SmtpEvent::SpfFromPass)), - 475 => Some(EventType::Smtp(SmtpEvent::SpfFromFail)), - 424 => Some(EventType::Smtp(SmtpEvent::DmarcPass)), - 423 => Some(EventType::Smtp(SmtpEvent::DmarcFail)), - 441 => Some(EventType::Smtp(SmtpEvent::IprevPass)), - 440 => Some(EventType::Smtp(SmtpEvent::IprevFail)), - 483 => Some(EventType::Smtp(SmtpEvent::TooManyMessages)), - 426 => Some(EventType::Smtp(SmtpEvent::Ehlo)), - 436 => Some(EventType::Smtp(SmtpEvent::InvalidEhlo)), - 420 => Some(EventType::Smtp(SmtpEvent::DidNotSayEhlo)), - 427 => Some(EventType::Smtp(SmtpEvent::EhloExpected)), - 442 => Some(EventType::Smtp(SmtpEvent::LhloExpected)), - 447 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthenticated)), - 448 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthorized)), - 551 => Some(EventType::Smtp(SmtpEvent::MailFromNotAllowed)), - 446 => Some(EventType::Smtp(SmtpEvent::MailFromRewritten)), - 445 => Some(EventType::Smtp(SmtpEvent::MailFromMissing)), - 444 => Some(EventType::Smtp(SmtpEvent::MailFrom)), - 456 => Some(EventType::Smtp(SmtpEvent::MultipleMailFrom)), - 449 => Some(EventType::Smtp(SmtpEvent::MailboxDoesNotExist)), - 468 => Some(EventType::Smtp(SmtpEvent::RelayNotAllowed)), - 464 => Some(EventType::Smtp(SmtpEvent::RcptTo)), - 465 => Some(EventType::Smtp(SmtpEvent::RcptToDuplicate)), - 467 => Some(EventType::Smtp(SmtpEvent::RcptToRewritten)), - 466 => Some(EventType::Smtp(SmtpEvent::RcptToMissing)), - 561 => Some(EventType::Smtp(SmtpEvent::RcptToGreylisted)), - 484 => Some(EventType::Smtp(SmtpEvent::TooManyRecipients)), - 482 => Some(EventType::Smtp(SmtpEvent::TooManyInvalidRcpt)), - 462 => Some(EventType::Smtp(SmtpEvent::RawInput)), - 463 => Some(EventType::Smtp(SmtpEvent::RawOutput)), - 453 => Some(EventType::Smtp(SmtpEvent::MissingLocalHostname)), - 487 => Some(EventType::Smtp(SmtpEvent::Vrfy)), - 489 => Some(EventType::Smtp(SmtpEvent::VrfyNotFound)), - 488 => Some(EventType::Smtp(SmtpEvent::VrfyDisabled)), - 429 => Some(EventType::Smtp(SmtpEvent::Expn)), - 431 => Some(EventType::Smtp(SmtpEvent::ExpnNotFound)), - 430 => Some(EventType::Smtp(SmtpEvent::ExpnDisabled)), - 471 => Some(EventType::Smtp(SmtpEvent::RequireTlsDisabled)), - 418 => Some(EventType::Smtp(SmtpEvent::DeliverByDisabled)), - 419 => Some(EventType::Smtp(SmtpEvent::DeliverByInvalid)), - 432 => Some(EventType::Smtp(SmtpEvent::FutureReleaseDisabled)), - 433 => Some(EventType::Smtp(SmtpEvent::FutureReleaseInvalid)), - 454 => Some(EventType::Smtp(SmtpEvent::MtPriorityDisabled)), - 455 => Some(EventType::Smtp(SmtpEvent::MtPriorityInvalid)), - 425 => Some(EventType::Smtp(SmtpEvent::DsnDisabled)), - 413 => Some(EventType::Smtp(SmtpEvent::AuthNotAllowed)), - 412 => Some(EventType::Smtp(SmtpEvent::AuthMechanismNotSupported)), - 411 => Some(EventType::Smtp(SmtpEvent::AuthExchangeTooLong)), - 408 => Some(EventType::Smtp(SmtpEvent::AlreadyAuthenticated)), - 457 => Some(EventType::Smtp(SmtpEvent::Noop)), - 477 => Some(EventType::Smtp(SmtpEvent::StartTls)), - 479 => Some(EventType::Smtp(SmtpEvent::StartTlsUnavailable)), - 478 => Some(EventType::Smtp(SmtpEvent::StartTlsAlready)), - 472 => Some(EventType::Smtp(SmtpEvent::Rset)), - 460 => Some(EventType::Smtp(SmtpEvent::Quit)), - 434 => Some(EventType::Smtp(SmtpEvent::Help)), - 414 => Some(EventType::Smtp(SmtpEvent::CommandNotImplemented)), - 435 => Some(EventType::Smtp(SmtpEvent::InvalidCommand)), - 439 => Some(EventType::Smtp(SmtpEvent::InvalidSenderAddress)), - 438 => Some(EventType::Smtp(SmtpEvent::InvalidRecipientAddress)), - 437 => Some(EventType::Smtp(SmtpEvent::InvalidParameter)), - 486 => Some(EventType::Smtp(SmtpEvent::UnsupportedParameter)), - 480 => Some(EventType::Smtp(SmtpEvent::SyntaxError)), - 470 => Some(EventType::Smtp(SmtpEvent::RequestTooLarge)), - 564 => Some(EventType::Spam(SpamEvent::Pyzor)), - 494 => Some(EventType::Spam(SpamEvent::PyzorError)), - 562 => Some(EventType::Spam(SpamEvent::Dnsbl)), - 563 => Some(EventType::Spam(SpamEvent::DnsblError)), - 588 => Some(EventType::Spam(SpamEvent::TrainStarted)), - 495 => Some(EventType::Spam(SpamEvent::TrainCompleted)), - 143 => Some(EventType::Spam(SpamEvent::TrainSampleAdded)), - 491 => Some(EventType::Spam(SpamEvent::TrainSampleNotFound)), - 490 => Some(EventType::Spam(SpamEvent::Classify)), - 589 => Some(EventType::Spam(SpamEvent::ModelLoaded)), - 496 => Some(EventType::Spam(SpamEvent::ModelNotReady)), - 497 => Some(EventType::Spam(SpamEvent::ModelNotFound)), - 280 => Some(EventType::Spam(SpamEvent::RulesUpdated)), - 501 => Some(EventType::Spf(SpfEvent::Pass)), - 498 => Some(EventType::Spf(SpfEvent::Fail)), - 503 => Some(EventType::Spf(SpfEvent::SoftFail)), - 499 => Some(EventType::Spf(SpfEvent::Neutral)), - 504 => Some(EventType::Spf(SpfEvent::TempError)), - 502 => Some(EventType::Spf(SpfEvent::PermError)), - 500 => Some(EventType::Spf(SpfEvent::None)), - 505 => Some(EventType::Store(StoreEvent::AssertValueFailed)), - 518 => Some(EventType::Store(StoreEvent::FoundationdbError)), - 522 => Some(EventType::Store(StoreEvent::MysqlError)), - 527 => Some(EventType::Store(StoreEvent::PostgresqlError)), - 529 => Some(EventType::Store(StoreEvent::RocksdbError)), - 532 => Some(EventType::Store(StoreEvent::SqliteError)), - 520 => Some(EventType::Store(StoreEvent::LdapError)), - 516 => Some(EventType::Store(StoreEvent::ElasticsearchError)), - 590 => Some(EventType::Store(StoreEvent::MeilisearchError)), - 528 => Some(EventType::Store(StoreEvent::RedisError)), - 530 => Some(EventType::Store(StoreEvent::S3Error)), - 559 => Some(EventType::Store(StoreEvent::AzureError)), - 517 => Some(EventType::Store(StoreEvent::FilesystemError)), - 526 => Some(EventType::Store(StoreEvent::PoolError)), - 511 => Some(EventType::Store(StoreEvent::DataCorruption)), - 514 => Some(EventType::Store(StoreEvent::DecompressError)), - 515 => Some(EventType::Store(StoreEvent::DeserializeError)), - 524 => Some(EventType::Store(StoreEvent::NotFound)), - 523 => Some(EventType::Store(StoreEvent::NotConfigured)), - 525 => Some(EventType::Store(StoreEvent::NotSupported)), - 533 => Some(EventType::Store(StoreEvent::UnexpectedError)), - 510 => Some(EventType::Store(StoreEvent::CryptoError)), - 493 => Some(EventType::Store(StoreEvent::HttpStoreError)), - 50 => Some(EventType::Store(StoreEvent::CacheMiss)), - 51 => Some(EventType::Store(StoreEvent::CacheHit)), - 52 => Some(EventType::Store(StoreEvent::CacheStale)), - 577 => Some(EventType::Store(StoreEvent::CacheUpdate)), - 633 => Some(EventType::Store(StoreEvent::CacheEntryTooLarge)), - 507 => Some(EventType::Store(StoreEvent::BlobMissingMarker)), - 513 => Some(EventType::Store(StoreEvent::DataWrite)), - 512 => Some(EventType::Store(StoreEvent::DataIterate)), - 508 => Some(EventType::Store(StoreEvent::BlobRead)), - 509 => Some(EventType::Store(StoreEvent::BlobWrite)), - 506 => Some(EventType::Store(StoreEvent::BlobDelete)), - 531 => Some(EventType::Store(StoreEvent::SqlQuery)), - 521 => Some(EventType::Store(StoreEvent::LdapQuery)), - 519 => Some(EventType::Store(StoreEvent::LdapWarning)), - 492 => Some(EventType::Store(StoreEvent::HttpStoreFetch)), - 364 => Some(EventType::Store(StoreEvent::AutoExpunge)), - 369 => Some(EventType::Store(StoreEvent::BlobStorePurged)), - 368 => Some(EventType::Store(StoreEvent::DataStorePurged)), - 578 => Some(EventType::TaskManager(TaskManagerEvent::TaskAcquired)), - 149 => Some(EventType::TaskManager(TaskManagerEvent::TaskQueued)), - 370 => Some(EventType::TaskManager(TaskManagerEvent::TaskScheduled)), - 144 => Some(EventType::TaskManager(TaskManagerEvent::TaskLocked)), - 586 => Some(EventType::TaskManager(TaskManagerEvent::TaskIgnored)), - 587 => Some(EventType::TaskManager(TaskManagerEvent::TaskFailed)), - 53 => Some(EventType::TaskManager(TaskManagerEvent::TaskRetry)), - 141 => Some(EventType::TaskManager(TaskManagerEvent::BlobNotFound)), - 145 => Some(EventType::TaskManager(TaskManagerEvent::MetadataNotFound)), - 150 => Some(EventType::TaskManager(TaskManagerEvent::SchedulerStarted)), - 367 => Some(EventType::TaskManager(TaskManagerEvent::ManagerStarted)), - 548 => Some(EventType::Telemetry(TelemetryEvent::AlertEvent)), - 365 => Some(EventType::Telemetry(TelemetryEvent::AlertMessage)), - 535 => Some(EventType::Telemetry(TelemetryEvent::LogError)), - 539 => Some(EventType::Telemetry(TelemetryEvent::WebhookError)), - 536 => Some(EventType::Telemetry(TelemetryEvent::OtelExporterError)), - 537 => Some(EventType::Telemetry( + 332 => Some(EventType::Pop3(Pop3Event::ConnectionStart)), + 331 => Some(EventType::Pop3(Pop3Event::ConnectionEnd)), + 333 => Some(EventType::Pop3(Pop3Event::Delete)), + 342 => Some(EventType::Pop3(Pop3Event::Reset)), + 339 => Some(EventType::Pop3(Pop3Event::Quit)), + 335 => Some(EventType::Pop3(Pop3Event::Fetch)), + 336 => Some(EventType::Pop3(Pop3Event::List)), + 337 => Some(EventType::Pop3(Pop3Event::ListMessage)), + 345 => Some(EventType::Pop3(Pop3Event::Uidl)), + 346 => Some(EventType::Pop3(Pop3Event::UidlMessage)), + 344 => Some(EventType::Pop3(Pop3Event::Stat)), + 338 => Some(EventType::Pop3(Pop3Event::Noop)), + 330 => Some(EventType::Pop3(Pop3Event::Capabilities)), + 343 => Some(EventType::Pop3(Pop3Event::StartTls)), + 347 => Some(EventType::Pop3(Pop3Event::Utf8)), + 334 => Some(EventType::Pop3(Pop3Event::Error)), + 340 => Some(EventType::Pop3(Pop3Event::RawInput)), + 341 => Some(EventType::Pop3(Pop3Event::RawOutput)), + 357 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), + 355 => Some(EventType::PushSubscription(PushSubscriptionEvent::Error)), + 356 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), + 374 => Some(EventType::Queue(QueueEvent::Started)), + 364 => Some(EventType::Queue(QueueEvent::MessageQueued)), + 365 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)), + 366 => Some(EventType::Queue(QueueEvent::ReportQueued)), + 363 => Some(EventType::Queue(QueueEvent::DsnQueued)), + 362 => Some(EventType::Queue(QueueEvent::AutogeneratedQueued)), + 369 => Some(EventType::Queue(QueueEvent::Rescheduled)), + 361 => Some(EventType::Queue(QueueEvent::Locked)), + 358 => Some(EventType::Queue(QueueEvent::BlobNotFound)), + 368 => Some(EventType::Queue(QueueEvent::RateLimitExceeded)), + 359 => Some(EventType::Queue(QueueEvent::ConcurrencyLimitExceeded)), + 367 => Some(EventType::Queue(QueueEvent::QuotaExceeded)), + 43 => Some(EventType::Queue(QueueEvent::BackPressure)), + 57 => Some(EventType::Registry(RegistryEvent::LocalReadError)), + 49 => Some(EventType::Registry(RegistryEvent::LocalWriteError)), + 55 => Some(EventType::Registry(RegistryEvent::LocalParseError)), + 60 => Some(EventType::Registry(RegistryEvent::ReadError)), + 53 => Some(EventType::Registry(RegistryEvent::WriteError)), + 51 => Some(EventType::Registry(RegistryEvent::DeserializationError)), + 56 => Some(EventType::Registry(RegistryEvent::BuildError)), + 50 => Some(EventType::Registry(RegistryEvent::BuildWarning)), + 59 => Some(EventType::Registry(RegistryEvent::NotSupported)), + 58 => Some(EventType::Registry(RegistryEvent::ValidationError)), + 373 => Some(EventType::Resource(ResourceEvent::NotFound)), + 370 => Some(EventType::Resource(ResourceEvent::BadParameters)), + 372 => Some(EventType::Resource(ResourceEvent::Error)), + 371 => Some(EventType::Resource(ResourceEvent::DownloadExternal)), + 584 => Some(EventType::Resource(ResourceEvent::ApplicationUpdated)), + 585 => Some(EventType::Resource(ResourceEvent::ApplicationUnpacked)), + 629 => Some(EventType::SearchIndex(SearchIndexEvent::ManagerStarted)), + 630 => Some(EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired)), + 631 => Some(EventType::SearchIndex(SearchIndexEvent::PartitionLocked)), + 632 => Some(EventType::SearchIndex(SearchIndexEvent::PartitionRetry)), + 126 => Some(EventType::SearchIndex(SearchIndexEvent::BatchIndexed)), + 574 => Some(EventType::SearchIndex(SearchIndexEvent::DocumentIgnored)), + 628 => Some(EventType::SearchIndex(SearchIndexEvent::DocumentTruncated)), + 27 => Some(EventType::SearchIndex(SearchIndexEvent::MetadataNotFound)), + 28 => Some(EventType::Security(SecurityEvent::AuthenticationBan)), + 532 => Some(EventType::Security(SecurityEvent::AbuseBan)), + 541 => Some(EventType::Security(SecurityEvent::ScanBan)), + 533 => Some(EventType::Security(SecurityEvent::LoiterBan)), + 302 => Some(EventType::Security(SecurityEvent::IpBlocked)), + 576 => Some(EventType::Security(SecurityEvent::IpBlockExpired)), + 577 => Some(EventType::Security(SecurityEvent::IpAllowExpired)), + 263 => Some(EventType::Security(SecurityEvent::IpUnauthorized)), + 535 => Some(EventType::Security(SecurityEvent::Unauthorized)), + 377 => Some(EventType::Server(ServerEvent::Startup)), + 376 => Some(EventType::Server(ServerEvent::Shutdown)), + 378 => Some(EventType::Server(ServerEvent::StartupError)), + 379 => Some(EventType::Server(ServerEvent::ThreadError)), + 586 => Some(EventType::Server(ServerEvent::RecoveryMode)), + 587 => Some(EventType::Server(ServerEvent::BootstrapMode)), + 380 => Some(EventType::Sieve(SieveEvent::ActionAccept)), + 381 => Some(EventType::Sieve(SieveEvent::ActionAcceptReplace)), + 382 => Some(EventType::Sieve(SieveEvent::ActionDiscard)), + 383 => Some(EventType::Sieve(SieveEvent::ActionReject)), + 390 => Some(EventType::Sieve(SieveEvent::SendMessage)), + 385 => Some(EventType::Sieve(SieveEvent::MessageTooLarge)), + 389 => Some(EventType::Sieve(SieveEvent::ScriptNotFound)), + 384 => Some(EventType::Sieve(SieveEvent::ListNotFound)), + 388 => Some(EventType::Sieve(SieveEvent::RuntimeError)), + 391 => Some(EventType::Sieve(SieveEvent::UnexpectedError)), + 386 => Some(EventType::Sieve(SieveEvent::NotSupported)), + 387 => Some(EventType::Sieve(SieveEvent::QuotaExceeded)), + 401 => Some(EventType::Smtp(SmtpEvent::ConnectionStart)), + 400 => Some(EventType::Smtp(SmtpEvent::ConnectionEnd)), + 412 => Some(EventType::Smtp(SmtpEvent::Error)), + 452 => Some(EventType::Smtp(SmtpEvent::IdNotFound)), + 399 => Some(EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded)), + 468 => Some(EventType::Smtp(SmtpEvent::TransferLimitExceeded)), + 444 => Some(EventType::Smtp(SmtpEvent::RateLimitExceeded)), + 464 => Some(EventType::Smtp(SmtpEvent::TimeLimitExceeded)), + 434 => Some(EventType::Smtp(SmtpEvent::MessageParseFailed)), + 435 => Some(EventType::Smtp(SmtpEvent::MessageTooLarge)), + 427 => Some(EventType::Smtp(SmtpEvent::LoopDetected)), + 626 => Some(EventType::Smtp(SmtpEvent::MultipleFromDomains)), + 406 => Some(EventType::Smtp(SmtpEvent::DkimPass)), + 405 => Some(EventType::Smtp(SmtpEvent::DkimFail)), + 589 => Some(EventType::Smtp(SmtpEvent::Dkim2Pass)), + 590 => Some(EventType::Smtp(SmtpEvent::Dkim2Fail)), + 591 => Some(EventType::Smtp(SmtpEvent::Dkim2DsnRejected)), + 394 => Some(EventType::Smtp(SmtpEvent::ArcPass)), + 393 => Some(EventType::Smtp(SmtpEvent::ArcFail)), + 457 => Some(EventType::Smtp(SmtpEvent::SpfEhloPass)), + 456 => Some(EventType::Smtp(SmtpEvent::SpfEhloFail)), + 459 => Some(EventType::Smtp(SmtpEvent::SpfFromPass)), + 458 => Some(EventType::Smtp(SmtpEvent::SpfFromFail)), + 408 => Some(EventType::Smtp(SmtpEvent::DmarcPass)), + 407 => Some(EventType::Smtp(SmtpEvent::DmarcFail)), + 425 => Some(EventType::Smtp(SmtpEvent::IprevPass)), + 424 => Some(EventType::Smtp(SmtpEvent::IprevFail)), + 466 => Some(EventType::Smtp(SmtpEvent::TooManyMessages)), + 410 => Some(EventType::Smtp(SmtpEvent::Ehlo)), + 420 => Some(EventType::Smtp(SmtpEvent::InvalidEhlo)), + 404 => Some(EventType::Smtp(SmtpEvent::DidNotSayEhlo)), + 411 => Some(EventType::Smtp(SmtpEvent::EhloExpected)), + 426 => Some(EventType::Smtp(SmtpEvent::LhloExpected)), + 431 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthenticated)), + 432 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthorized)), + 534 => Some(EventType::Smtp(SmtpEvent::MailFromNotAllowed)), + 430 => Some(EventType::Smtp(SmtpEvent::MailFromRewritten)), + 429 => Some(EventType::Smtp(SmtpEvent::MailFromMissing)), + 428 => Some(EventType::Smtp(SmtpEvent::MailFrom)), + 439 => Some(EventType::Smtp(SmtpEvent::MultipleMailFrom)), + 433 => Some(EventType::Smtp(SmtpEvent::MailboxDoesNotExist)), + 451 => Some(EventType::Smtp(SmtpEvent::RelayNotAllowed)), + 447 => Some(EventType::Smtp(SmtpEvent::RcptTo)), + 448 => Some(EventType::Smtp(SmtpEvent::RcptToDuplicate)), + 450 => Some(EventType::Smtp(SmtpEvent::RcptToRewritten)), + 449 => Some(EventType::Smtp(SmtpEvent::RcptToMissing)), + 544 => Some(EventType::Smtp(SmtpEvent::RcptToGreylisted)), + 467 => Some(EventType::Smtp(SmtpEvent::TooManyRecipients)), + 465 => Some(EventType::Smtp(SmtpEvent::TooManyInvalidRcpt)), + 445 => Some(EventType::Smtp(SmtpEvent::RawInput)), + 446 => Some(EventType::Smtp(SmtpEvent::RawOutput)), + 436 => Some(EventType::Smtp(SmtpEvent::MissingLocalHostname)), + 470 => Some(EventType::Smtp(SmtpEvent::Vrfy)), + 472 => Some(EventType::Smtp(SmtpEvent::VrfyNotFound)), + 471 => Some(EventType::Smtp(SmtpEvent::VrfyDisabled)), + 413 => Some(EventType::Smtp(SmtpEvent::Expn)), + 415 => Some(EventType::Smtp(SmtpEvent::ExpnNotFound)), + 414 => Some(EventType::Smtp(SmtpEvent::ExpnDisabled)), + 454 => Some(EventType::Smtp(SmtpEvent::RequireTlsDisabled)), + 402 => Some(EventType::Smtp(SmtpEvent::DeliverByDisabled)), + 403 => Some(EventType::Smtp(SmtpEvent::DeliverByInvalid)), + 416 => Some(EventType::Smtp(SmtpEvent::FutureReleaseDisabled)), + 417 => Some(EventType::Smtp(SmtpEvent::FutureReleaseInvalid)), + 437 => Some(EventType::Smtp(SmtpEvent::MtPriorityDisabled)), + 438 => Some(EventType::Smtp(SmtpEvent::MtPriorityInvalid)), + 409 => Some(EventType::Smtp(SmtpEvent::DsnDisabled)), + 397 => Some(EventType::Smtp(SmtpEvent::AuthNotAllowed)), + 396 => Some(EventType::Smtp(SmtpEvent::AuthMechanismNotSupported)), + 395 => Some(EventType::Smtp(SmtpEvent::AuthExchangeTooLong)), + 392 => Some(EventType::Smtp(SmtpEvent::AlreadyAuthenticated)), + 440 => Some(EventType::Smtp(SmtpEvent::Noop)), + 460 => Some(EventType::Smtp(SmtpEvent::StartTls)), + 462 => Some(EventType::Smtp(SmtpEvent::StartTlsUnavailable)), + 461 => Some(EventType::Smtp(SmtpEvent::StartTlsAlready)), + 455 => Some(EventType::Smtp(SmtpEvent::Rset)), + 443 => Some(EventType::Smtp(SmtpEvent::Quit)), + 418 => Some(EventType::Smtp(SmtpEvent::Help)), + 398 => Some(EventType::Smtp(SmtpEvent::CommandNotImplemented)), + 419 => Some(EventType::Smtp(SmtpEvent::InvalidCommand)), + 423 => Some(EventType::Smtp(SmtpEvent::InvalidSenderAddress)), + 422 => Some(EventType::Smtp(SmtpEvent::InvalidRecipientAddress)), + 421 => Some(EventType::Smtp(SmtpEvent::InvalidParameter)), + 469 => Some(EventType::Smtp(SmtpEvent::UnsupportedParameter)), + 463 => Some(EventType::Smtp(SmtpEvent::SyntaxError)), + 453 => Some(EventType::Smtp(SmtpEvent::RequestTooLarge)), + 547 => Some(EventType::Spam(SpamEvent::Pyzor)), + 477 => Some(EventType::Spam(SpamEvent::PyzorError)), + 545 => Some(EventType::Spam(SpamEvent::Dnsbl)), + 546 => Some(EventType::Spam(SpamEvent::DnsblError)), + 571 => Some(EventType::Spam(SpamEvent::TrainStarted)), + 478 => Some(EventType::Spam(SpamEvent::TrainCompleted)), + 127 => Some(EventType::Spam(SpamEvent::TrainSampleAdded)), + 474 => Some(EventType::Spam(SpamEvent::TrainSampleNotFound)), + 473 => Some(EventType::Spam(SpamEvent::Classify)), + 572 => Some(EventType::Spam(SpamEvent::ModelLoaded)), + 479 => Some(EventType::Spam(SpamEvent::ModelNotReady)), + 480 => Some(EventType::Spam(SpamEvent::ModelNotFound)), + 264 => Some(EventType::Spam(SpamEvent::RulesUpdated)), + 484 => Some(EventType::Spf(SpfEvent::Pass)), + 481 => Some(EventType::Spf(SpfEvent::Fail)), + 486 => Some(EventType::Spf(SpfEvent::SoftFail)), + 482 => Some(EventType::Spf(SpfEvent::Neutral)), + 487 => Some(EventType::Spf(SpfEvent::TempError)), + 485 => Some(EventType::Spf(SpfEvent::PermError)), + 483 => Some(EventType::Spf(SpfEvent::None)), + 488 => Some(EventType::Store(StoreEvent::AssertValueFailed)), + 501 => Some(EventType::Store(StoreEvent::FoundationdbError)), + 505 => Some(EventType::Store(StoreEvent::MysqlError)), + 510 => Some(EventType::Store(StoreEvent::PostgresqlError)), + 512 => Some(EventType::Store(StoreEvent::RocksdbError)), + 515 => Some(EventType::Store(StoreEvent::SqliteError)), + 503 => Some(EventType::Store(StoreEvent::LdapError)), + 499 => Some(EventType::Store(StoreEvent::ElasticsearchError)), + 573 => Some(EventType::Store(StoreEvent::MeilisearchError)), + 511 => Some(EventType::Store(StoreEvent::RedisError)), + 513 => Some(EventType::Store(StoreEvent::S3Error)), + 542 => Some(EventType::Store(StoreEvent::AzureError)), + 500 => Some(EventType::Store(StoreEvent::FilesystemError)), + 509 => Some(EventType::Store(StoreEvent::PoolError)), + 494 => Some(EventType::Store(StoreEvent::DataCorruption)), + 497 => Some(EventType::Store(StoreEvent::DecompressError)), + 498 => Some(EventType::Store(StoreEvent::DeserializeError)), + 507 => Some(EventType::Store(StoreEvent::NotFound)), + 506 => Some(EventType::Store(StoreEvent::NotConfigured)), + 508 => Some(EventType::Store(StoreEvent::NotSupported)), + 516 => Some(EventType::Store(StoreEvent::UnexpectedError)), + 493 => Some(EventType::Store(StoreEvent::CryptoError)), + 476 => Some(EventType::Store(StoreEvent::HttpStoreError)), + 490 => Some(EventType::Store(StoreEvent::BlobMissingMarker)), + 496 => Some(EventType::Store(StoreEvent::DataWrite)), + 495 => Some(EventType::Store(StoreEvent::DataIterate)), + 491 => Some(EventType::Store(StoreEvent::BlobRead)), + 492 => Some(EventType::Store(StoreEvent::BlobWrite)), + 489 => Some(EventType::Store(StoreEvent::BlobDelete)), + 514 => Some(EventType::Store(StoreEvent::SqlQuery)), + 504 => Some(EventType::Store(StoreEvent::LdapQuery)), + 502 => Some(EventType::Store(StoreEvent::LdapWarning)), + 475 => Some(EventType::Store(StoreEvent::HttpStoreFetch)), + 348 => Some(EventType::Store(StoreEvent::AutoExpunge)), + 353 => Some(EventType::Store(StoreEvent::BlobStorePurged)), + 352 => Some(EventType::Store(StoreEvent::DataStorePurged)), + 561 => Some(EventType::TaskManager(TaskManagerEvent::TaskAcquired)), + 133 => Some(EventType::TaskManager(TaskManagerEvent::TaskQueued)), + 354 => Some(EventType::TaskManager(TaskManagerEvent::TaskScheduled)), + 128 => Some(EventType::TaskManager(TaskManagerEvent::TaskLocked)), + 569 => Some(EventType::TaskManager(TaskManagerEvent::TaskIgnored)), + 570 => Some(EventType::TaskManager(TaskManagerEvent::TaskFailed)), + 48 => Some(EventType::TaskManager(TaskManagerEvent::TaskRetry)), + 651 => Some(EventType::TaskManager(TaskManagerEvent::TaskStalled)), + 652 => Some(EventType::TaskManager(TaskManagerEvent::TaskExpired)), + 125 => Some(EventType::TaskManager(TaskManagerEvent::BlobNotFound)), + 129 => Some(EventType::TaskManager(TaskManagerEvent::MetadataNotFound)), + 134 => Some(EventType::TaskManager(TaskManagerEvent::SchedulerStarted)), + 351 => Some(EventType::TaskManager(TaskManagerEvent::ManagerStarted)), + 518 => Some(EventType::Telemetry(TelemetryEvent::LogError)), + 522 => Some(EventType::Telemetry(TelemetryEvent::WebhookError)), + 519 => Some(EventType::Telemetry(TelemetryEvent::OtelExporterError)), + 520 => Some(EventType::Telemetry( TelemetryEvent::OtelMetricsExporterError, )), - 538 => Some(EventType::Telemetry( + 521 => Some(EventType::Telemetry( TelemetryEvent::PrometheusExporterError, )), - 534 => Some(EventType::Telemetry(TelemetryEvent::JournalError)), - 151 => Some(EventType::Telemetry(TelemetryEvent::MetricsCollected)), - 366 => Some(EventType::Telemetry(TelemetryEvent::MetricsStored)), - 146 => Some(EventType::Telemetry(TelemetryEvent::MetricsPushed)), - 543 => Some(EventType::Tls(TlsEvent::Handshake)), - 544 => Some(EventType::Tls(TlsEvent::HandshakeError)), - 547 => Some(EventType::Tls(TlsEvent::NotConfigured)), - 542 => Some(EventType::Tls(TlsEvent::CertificateNotFound)), - 546 => Some(EventType::Tls(TlsEvent::NoCertificatesAvailable)), - 545 => Some(EventType::Tls(TlsEvent::MultipleCertificatesAvailable)), - 277 => Some(EventType::Tls(TlsEvent::ExpiredCertificateRemoved)), - 540 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetch)), - 541 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetchError)), - 560 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)), - 147 => Some(EventType::WebDav(WebDavEvent::Propfind)), - 148 => Some(EventType::WebDav(WebDavEvent::Proppatch)), - 335 => Some(EventType::WebDav(WebDavEvent::Get)), - 574 => Some(EventType::WebDav(WebDavEvent::Head)), - 336 => Some(EventType::WebDav(WebDavEvent::Report)), - 376 => Some(EventType::WebDav(WebDavEvent::Mkcol)), - 575 => Some(EventType::WebDav(WebDavEvent::Mkcalendar)), - 458 => Some(EventType::WebDav(WebDavEvent::Delete)), - 459 => Some(EventType::WebDav(WebDavEvent::Put)), - 565 => Some(EventType::WebDav(WebDavEvent::Post)), - 566 => Some(EventType::WebDav(WebDavEvent::Patch)), - 567 => Some(EventType::WebDav(WebDavEvent::Copy)), - 568 => Some(EventType::WebDav(WebDavEvent::Move)), - 569 => Some(EventType::WebDav(WebDavEvent::Lock)), - 570 => Some(EventType::WebDav(WebDavEvent::Unlock)), - 571 => Some(EventType::WebDav(WebDavEvent::Acl)), - 573 => Some(EventType::WebDav(WebDavEvent::Options)), - 572 => Some(EventType::WebDav(WebDavEvent::Error)), + 517 => Some(EventType::Telemetry(TelemetryEvent::JournalError)), + 601 => Some(EventType::Telemetry(TelemetryEvent::EventsDropped)), + 135 => Some(EventType::Telemetry(TelemetryEvent::MetricsCollected)), + 130 => Some(EventType::Telemetry(TelemetryEvent::MetricsPushed)), + 526 => Some(EventType::Tls(TlsEvent::Handshake)), + 527 => Some(EventType::Tls(TlsEvent::HandshakeError)), + 530 => Some(EventType::Tls(TlsEvent::NotConfigured)), + 525 => Some(EventType::Tls(TlsEvent::CertificateNotFound)), + 529 => Some(EventType::Tls(TlsEvent::NoCertificatesAvailable)), + 528 => Some(EventType::Tls(TlsEvent::MultipleCertificatesAvailable)), + 261 => Some(EventType::Tls(TlsEvent::ExpiredCertificateRemoved)), + 523 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetch)), + 524 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetchError)), + 543 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)), + 131 => Some(EventType::WebDav(WebDavEvent::Propfind)), + 132 => Some(EventType::WebDav(WebDavEvent::Proppatch)), + 319 => Some(EventType::WebDav(WebDavEvent::Get)), + 557 => Some(EventType::WebDav(WebDavEvent::Head)), + 320 => Some(EventType::WebDav(WebDavEvent::Report)), + 360 => Some(EventType::WebDav(WebDavEvent::Mkcol)), + 558 => Some(EventType::WebDav(WebDavEvent::Mkcalendar)), + 441 => Some(EventType::WebDav(WebDavEvent::Delete)), + 442 => Some(EventType::WebDav(WebDavEvent::Put)), + 548 => Some(EventType::WebDav(WebDavEvent::Post)), + 549 => Some(EventType::WebDav(WebDavEvent::Patch)), + 550 => Some(EventType::WebDav(WebDavEvent::Copy)), + 551 => Some(EventType::WebDav(WebDavEvent::Move)), + 552 => Some(EventType::WebDav(WebDavEvent::Lock)), + 553 => Some(EventType::WebDav(WebDavEvent::Unlock)), + 554 => Some(EventType::WebDav(WebDavEvent::Acl)), + 556 => Some(EventType::WebDav(WebDavEvent::Options)), + 555 => Some(EventType::WebDav(WebDavEvent::Error)), _ => None, } } @@ -2768,7 +2761,6 @@ impl EventType { EventType::Cluster(ClusterEvent::PublisherError) => Level::Error, EventType::Cluster(ClusterEvent::MessageInvalid) => Level::Error, EventType::Dkim(DkimEvent::BuildError) => Level::Error, - EventType::Dns(DnsEvent::BuildError) => Level::Error, EventType::MessageIngest(MessageIngestEvent::Error) => Level::Error, EventType::Network(NetworkEvent::ListenError) => Level::Error, EventType::Network(NetworkEvent::BindError) => Level::Error, @@ -2838,18 +2830,12 @@ impl EventType { EventType::Dane(DaneEvent::TlsaRecordFetch) => Level::Info, EventType::Dane(DaneEvent::TlsaRecordFetchError) => Level::Info, EventType::Dane(DaneEvent::TlsaRecordNotFound) => Level::Info, - EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned) => Level::Info, EventType::Dane(DaneEvent::TlsaRecordInvalid) => Level::Info, EventType::Dane(DaneEvent::BogusDnssecRecord) => Level::Info, - EventType::Delivery(DeliveryEvent::AttemptStart) => Level::Info, - EventType::Delivery(DeliveryEvent::AttemptEnd) => Level::Info, - EventType::Delivery(DeliveryEvent::Completed) => Level::Info, EventType::Delivery(DeliveryEvent::Failed) => Level::Info, - EventType::Delivery(DeliveryEvent::DomainDeliveryStart) => Level::Info, EventType::Delivery(DeliveryEvent::MxLookupFailed) => Level::Info, EventType::Delivery(DeliveryEvent::IpLookupFailed) => Level::Info, EventType::Delivery(DeliveryEvent::NullMx) => Level::Info, - EventType::Delivery(DeliveryEvent::Connect) => Level::Info, EventType::Delivery(DeliveryEvent::ConnectError) => Level::Info, EventType::Delivery(DeliveryEvent::GreetingFailed) => Level::Info, EventType::Delivery(DeliveryEvent::EhloRejected) => Level::Info, @@ -2859,13 +2845,10 @@ impl EventType { EventType::Delivery(DeliveryEvent::RcptToRejected) => Level::Info, EventType::Delivery(DeliveryEvent::RcptToFailed) => Level::Info, EventType::Delivery(DeliveryEvent::MessageRejected) => Level::Info, - EventType::Delivery(DeliveryEvent::StartTls) => Level::Info, - EventType::Delivery(DeliveryEvent::StartTlsUnavailable) => Level::Info, EventType::Delivery(DeliveryEvent::StartTlsError) => Level::Info, EventType::Delivery(DeliveryEvent::StartTlsDisabled) => Level::Info, EventType::Delivery(DeliveryEvent::ImplicitTlsError) => Level::Info, EventType::Delivery(DeliveryEvent::DoubleBounce) => Level::Info, - EventType::Delivery(DeliveryEvent::DsnSuccess) => Level::Info, EventType::Delivery(DeliveryEvent::DsnTempFail) => Level::Info, EventType::Delivery(DeliveryEvent::DsnPermFail) => Level::Info, EventType::Dkim(DkimEvent::SignatureCreated) => Level::Info, @@ -2888,13 +2871,11 @@ impl EventType { EventType::IncomingReport(IncomingReportEvent::TlsRpcParseFailed) => Level::Info, EventType::IncomingReport(IncomingReportEvent::ArfParseFailed) => Level::Info, EventType::IncomingReport(IncomingReportEvent::DecompressError) => Level::Info, - EventType::Limit(LimitEvent::TenantQuota) => Level::Info, EventType::MessageIngest(MessageIngestEvent::Ham) => Level::Info, EventType::MessageIngest(MessageIngestEvent::Spam) => Level::Info, EventType::MessageIngest(MessageIngestEvent::ImapAppend) => Level::Info, EventType::MessageIngest(MessageIngestEvent::JmapAppend) => Level::Info, EventType::MessageIngest(MessageIngestEvent::Duplicate) => Level::Info, - EventType::MessageIngest(MessageIngestEvent::SearchIndex) => Level::Info, EventType::Milter(MilterEvent::ActionAccept) => Level::Info, EventType::Milter(MilterEvent::ActionDiscard) => Level::Info, EventType::Milter(MilterEvent::ActionReject) => Level::Info, @@ -2906,10 +2887,7 @@ impl EventType { EventType::MtaHook(MtaHookEvent::ActionDiscard) => Level::Info, EventType::MtaHook(MtaHookEvent::ActionReject) => Level::Info, EventType::MtaHook(MtaHookEvent::ActionQuarantine) => Level::Info, - EventType::MtaSts(MtaStsEvent::Authorized) => Level::Info, EventType::MtaSts(MtaStsEvent::NotAuthorized) => Level::Info, - EventType::MtaSts(MtaStsEvent::PolicyFetch) => Level::Info, - EventType::MtaSts(MtaStsEvent::PolicyNotFound) => Level::Info, EventType::MtaSts(MtaStsEvent::PolicyFetchError) => Level::Info, EventType::MtaSts(MtaStsEvent::InvalidPolicy) => Level::Info, EventType::Network(NetworkEvent::ListenStart) => Level::Info, @@ -2945,6 +2923,7 @@ impl EventType { EventType::Queue(QueueEvent::QuotaExceeded) => Level::Info, EventType::Resource(ResourceEvent::DownloadExternal) => Level::Info, EventType::Resource(ResourceEvent::ApplicationUpdated) => Level::Info, + EventType::SearchIndex(SearchIndexEvent::ManagerStarted) => Level::Info, EventType::Security(SecurityEvent::AuthenticationBan) => Level::Info, EventType::Security(SecurityEvent::AbuseBan) => Level::Info, EventType::Security(SecurityEvent::ScanBan) => Level::Info, @@ -2956,21 +2935,20 @@ impl EventType { EventType::Security(SecurityEvent::Unauthorized) => Level::Info, EventType::Server(ServerEvent::Startup) => Level::Info, EventType::Server(ServerEvent::Shutdown) => Level::Info, - EventType::Server(ServerEvent::Licensing) => Level::Info, EventType::Sieve(SieveEvent::SendMessage) => Level::Info, EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded) => Level::Info, EventType::Smtp(SmtpEvent::TransferLimitExceeded) => Level::Info, EventType::Smtp(SmtpEvent::RateLimitExceeded) => Level::Info, EventType::Smtp(SmtpEvent::TimeLimitExceeded) => Level::Info, - EventType::Smtp(SmtpEvent::MissingAuthDirectory) => Level::Info, EventType::Smtp(SmtpEvent::MessageParseFailed) => Level::Info, EventType::Smtp(SmtpEvent::MessageTooLarge) => Level::Info, EventType::Smtp(SmtpEvent::LoopDetected) => Level::Info, + EventType::Smtp(SmtpEvent::MultipleFromDomains) => Level::Info, EventType::Smtp(SmtpEvent::DkimPass) => Level::Info, EventType::Smtp(SmtpEvent::DkimFail) => Level::Info, EventType::Smtp(SmtpEvent::Dkim2Pass) => Level::Info, EventType::Smtp(SmtpEvent::Dkim2Fail) => Level::Info, - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded) => Level::Info, + EventType::Smtp(SmtpEvent::Dkim2DsnRejected) => Level::Info, EventType::Smtp(SmtpEvent::ArcPass) => Level::Info, EventType::Smtp(SmtpEvent::ArcFail) => Level::Info, EventType::Smtp(SmtpEvent::SpfEhloPass) => Level::Info, @@ -3010,17 +2988,15 @@ impl EventType { EventType::Store(StoreEvent::DataStorePurged) => Level::Info, EventType::TaskManager(TaskManagerEvent::TaskQueued) => Level::Info, EventType::TaskManager(TaskManagerEvent::TaskScheduled) => Level::Info, + EventType::TaskManager(TaskManagerEvent::TaskExpired) => Level::Info, EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => Level::Info, EventType::TaskManager(TaskManagerEvent::ManagerStarted) => Level::Info, - EventType::Telemetry(TelemetryEvent::AlertMessage) => Level::Info, EventType::Telemetry(TelemetryEvent::MetricsCollected) => Level::Info, EventType::Tls(TlsEvent::Handshake) => Level::Info, EventType::Tls(TlsEvent::ExpiredCertificateRemoved) => Level::Info, - EventType::TlsRpt(TlsRptEvent::RecordFetch) => Level::Info, EventType::TlsRpt(TlsRptEvent::RecordFetchError) => Level::Info, - EventType::TlsRpt(TlsRptEvent::RecordNotFound) => Level::Info, - EventType::Ai(AiEvent::LlmResponse) => Level::Trace, EventType::Auth(AuthEvent::MfaRequired) => Level::Trace, + EventType::Cache(CacheEvent::Invalidate) => Level::Trace, EventType::Cluster(ClusterEvent::MessageReceived) => Level::Trace, EventType::Cluster(ClusterEvent::MessageSkipped) => Level::Trace, EventType::Delivery(DeliveryEvent::RawInput) => Level::Trace, @@ -3055,23 +3031,25 @@ impl EventType { EventType::Acme(AcmeEvent::OrderInvalid) => Level::Warn, EventType::Acme(AcmeEvent::TlsAlpnError) => Level::Warn, EventType::Acme(AcmeEvent::TokenNotFound) => Level::Warn, - EventType::Ai(AiEvent::ApiError) => Level::Warn, - EventType::Arc(ArcEvent::SealerNotFound) => Level::Warn, EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn, + EventType::Cache(CacheEvent::SwapError) => Level::Warn, EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn, EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn, EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn, EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn, EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn, EventType::Dkim(DkimEvent::SignerNotFound) => Level::Warn, + EventType::Dkim2(Dkim2Event::SigningFailed) => Level::Warn, EventType::Dns(DnsEvent::RecordCreationFailed) => Level::Warn, EventType::Dns(DnsEvent::RecordPropagationTimeout) => Level::Warn, EventType::Eval(EvalEvent::DirectoryNotFound) => Level::Warn, EventType::Http(HttpEvent::XForwardedMissing) => Level::Warn, + EventType::Http(HttpEvent::XForwardedInvalid) => Level::Warn, EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings) => Level::Warn, EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings) => Level::Warn, EventType::Limit(LimitEvent::ConcurrentConnection) => Level::Warn, EventType::Limit(LimitEvent::TooManyRequests) => Level::Warn, + EventType::Limit(LimitEvent::MemoryBudget) => Level::Warn, EventType::Milter(MilterEvent::IoError) => Level::Warn, EventType::Milter(MilterEvent::FrameTooLarge) => Level::Warn, EventType::Milter(MilterEvent::FrameInvalid) => Level::Warn, @@ -3095,16 +3073,16 @@ impl EventType { EventType::Smtp(SmtpEvent::MissingLocalHostname) => Level::Warn, EventType::Spam(SpamEvent::TrainSampleNotFound) => Level::Warn, EventType::Store(StoreEvent::HttpStoreError) => Level::Warn, - EventType::Store(StoreEvent::CacheEntryTooLarge) => Level::Warn, EventType::Store(StoreEvent::BlobMissingMarker) => Level::Warn, EventType::TaskManager(TaskManagerEvent::TaskFailed) => Level::Warn, - EventType::Telemetry(TelemetryEvent::AlertEvent) => Level::Warn, + EventType::TaskManager(TaskManagerEvent::TaskStalled) => Level::Warn, EventType::Telemetry(TelemetryEvent::LogError) => Level::Warn, EventType::Telemetry(TelemetryEvent::WebhookError) => Level::Warn, EventType::Telemetry(TelemetryEvent::OtelExporterError) => Level::Warn, EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError) => Level::Warn, EventType::Telemetry(TelemetryEvent::PrometheusExporterError) => Level::Warn, EventType::Telemetry(TelemetryEvent::JournalError) => Level::Warn, + EventType::Telemetry(TelemetryEvent::EventsDropped) => Level::Warn, EventType::Tls(TlsEvent::NoCertificatesAvailable) => Level::Warn, EventType::Tls(TlsEvent::MultipleCertificatesAvailable) => Level::Warn, _ => Level::Debug, @@ -3135,14 +3113,9 @@ impl EventType { EventType::Acme(AcmeEvent::TlsAlpnError) => "ACME TLS ALPN error", EventType::Acme(AcmeEvent::TokenNotFound) => "ACME token not found", EventType::Acme(AcmeEvent::Error) => "ACME error", - EventType::Ai(AiEvent::LlmResponse) => "LLM response", - EventType::Ai(AiEvent::ApiError) => "AI API error", - EventType::Arc(ArcEvent::ChainTooLong) => "ARC chain too long", - EventType::Arc(ArcEvent::InvalidInstance) => "Invalid ARC instance", - EventType::Arc(ArcEvent::InvalidCv) => "Invalid ARC CV", - EventType::Arc(ArcEvent::HasHeaderTag) => "ARC has header tag", - EventType::Arc(ArcEvent::BrokenChain) => "Broken ARC chain", - EventType::Arc(ArcEvent::SealerNotFound) => "ARC sealer not found", + EventType::Arc(ArcEvent::ChainInvalid) => "ARC chain invalid", + EventType::Arc(ArcEvent::SignatureInvalid) => "ARC signature invalid", + EventType::Arc(ArcEvent::HashMismatch) => "ARC body hash mismatch", EventType::Auth(AuthEvent::Success) => "Authentication successful", EventType::Auth(AuthEvent::Failed) => "Authentication failed", EventType::Auth(AuthEvent::TokenExpired) => "OAuth token expired", @@ -3152,6 +3125,18 @@ impl EventType { EventType::Auth(AuthEvent::Error) => "Authentication error", EventType::Auth(AuthEvent::Warning) => "Authentication warning", EventType::Auth(AuthEvent::CredentialExpired) => "Credential expired", + EventType::Cache(CacheEvent::Miss) => "Cache miss", + EventType::Cache(CacheEvent::Hit) => "Cache hit", + EventType::Cache(CacheEvent::Stale) => "Cache is stale", + EventType::Cache(CacheEvent::Update) => "Cache update", + EventType::Cache(CacheEvent::EntryTooLarge) => "Cache entry too large", + EventType::Cache(CacheEvent::Invalidate) => "Cache invalidation", + EventType::Cache(CacheEvent::SwapHit) => "Cache snapshot loaded", + EventType::Cache(CacheEvent::SwapMiss) => "Cache snapshot unavailable", + EventType::Cache(CacheEvent::SwapWrite) => "Cache snapshot written", + EventType::Cache(CacheEvent::SwapError) => "Cache swap tier error", + EventType::Cache(CacheEvent::SwapMetrics) => "Cache swap metrics", + EventType::Cache(CacheEvent::SwapFlush) => "Cache swap flushed", EventType::Calendar(CalendarEvent::RuleExpansionError) => { "Calendar rule expansion error" } @@ -3249,19 +3234,10 @@ impl EventType { EventType::Dkim(DkimEvent::PermError) => "DKIM permanent error", EventType::Dkim(DkimEvent::TempError) => "DKIM temporary error", EventType::Dkim(DkimEvent::None) => "No DKIM signature", - EventType::Dkim(DkimEvent::UnsupportedVersion) => "Unsupported DKIM version", - EventType::Dkim(DkimEvent::UnsupportedAlgorithm) => "Unsupported DKIM algorithm", - EventType::Dkim(DkimEvent::UnsupportedCanonicalization) => { - "Unsupported DKIM canonicalization" - } - EventType::Dkim(DkimEvent::UnsupportedKeyType) => "Unsupported DKIM key type", - EventType::Dkim(DkimEvent::FailedBodyHashMatch) => "DKIM body hash mismatch", - EventType::Dkim(DkimEvent::FailedVerification) => "DKIM verification failed", - EventType::Dkim(DkimEvent::FailedAuidMatch) => "DKIM AUID mismatch", - EventType::Dkim(DkimEvent::RevokedPublicKey) => "DKIM public key revoked", - EventType::Dkim(DkimEvent::IncompatibleAlgorithms) => "Incompatible DKIM algorithms", - EventType::Dkim(DkimEvent::SignatureExpired) => "DKIM signature expired", - EventType::Dkim(DkimEvent::SignatureLength) => "DKIM signature length issue", + EventType::Dkim(DkimEvent::SignatureInvalid) => "DKIM signature invalid", + EventType::Dkim(DkimEvent::PublicKeyError) => "DKIM public key error", + EventType::Dkim(DkimEvent::VerificationFailed) => "DKIM signature verification failed", + EventType::Dkim(DkimEvent::HashMismatch) => "DKIM body hash mismatch", EventType::Dkim(DkimEvent::SignerNotFound) => "DKIM signer not found", EventType::Dkim(DkimEvent::BuildError) => "DKIM build error", EventType::Dkim(DkimEvent::SignatureCreated) => "DKIM signature created", @@ -3269,36 +3245,19 @@ impl EventType { EventType::Dkim(DkimEvent::SignatureRetiring) => "DKIM signature retiring", EventType::Dkim(DkimEvent::SignatureRetired) => "DKIM signature retired", EventType::Dkim(DkimEvent::SignatureDeleted) => "DKIM signature deleted", - EventType::Dkim(DkimEvent::InstanceMissing) => "DKIM2 message-instance missing", - EventType::Dkim(DkimEvent::InstanceSyntax) => "DKIM2 message-instance syntax error", - EventType::Dkim(DkimEvent::InstanceTagMissing) => "DKIM2 message-instance tag missing", - EventType::Dkim(DkimEvent::InstanceNotSigned) => "DKIM2 message-instance not signed", - EventType::Dkim(DkimEvent::InstanceAboveSignature) => { - "DKIM2 message-instance above signature" + EventType::Dkim2(Dkim2Event::InstanceInvalid) => "DKIM2 Message-Instance invalid", + EventType::Dkim2(Dkim2Event::SignatureInvalid) => "DKIM2 signature invalid", + EventType::Dkim2(Dkim2Event::ChainInvalid) => "DKIM2 chain invalid", + EventType::Dkim2(Dkim2Event::TimestampInvalid) => "DKIM2 timestamp invalid", + EventType::Dkim2(Dkim2Event::EnvelopeMismatch) => "DKIM2 envelope mismatch", + EventType::Dkim2(Dkim2Event::CustodyBroken) => "DKIM2 chain of custody broken", + EventType::Dkim2(Dkim2Event::PublicKeyError) => "DKIM2 public key error", + EventType::Dkim2(Dkim2Event::VerificationFailed) => { + "DKIM2 signature verification failed" } - EventType::Dkim(DkimEvent::SignatureMissing) => "DKIM2 signature missing", - EventType::Dkim(DkimEvent::SignatureSyntax) => "DKIM2 signature syntax error", - EventType::Dkim(DkimEvent::SignatureTagMissing) => "DKIM2 signature tag missing", - EventType::Dkim(DkimEvent::SignatureTagUnexpected) => "DKIM2 signature unexpected tag", - EventType::Dkim(DkimEvent::SequenceGap) => "DKIM2 sequence gap", - EventType::Dkim(DkimEvent::SequenceOverflow) => "DKIM2 sequence overflow", - EventType::Dkim(DkimEvent::ChainTooLong) => "DKIM2 chain too long", - EventType::Dkim(DkimEvent::MailFromMismatch) => "DKIM2 MAIL FROM mismatch", - EventType::Dkim(DkimEvent::RcptToMismatch) => "DKIM2 RCPT TO mismatch", - EventType::Dkim(DkimEvent::MailFromDomainMismatch) => "DKIM2 MAIL FROM domain mismatch", - EventType::Dkim(DkimEvent::NextDomainMismatch) => "DKIM2 next domain mismatch", - EventType::Dkim(DkimEvent::CustodyBreak) => "DKIM2 custody break", - EventType::Dkim(DkimEvent::PublicKeyFetch) => "DKIM2 public key fetch error", - EventType::Dkim(DkimEvent::PublicKeyMissing) => "DKIM2 public key missing", - EventType::Dkim(DkimEvent::PublicKeyMultiple) => "DKIM2 multiple public keys", - EventType::Dkim(DkimEvent::PublicKeySyntax) => "DKIM2 public key syntax error", - EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch) => { - "DKIM2 public key algorithm mismatch" - } - EventType::Dkim(DkimEvent::NoValidAlgorithm) => "DKIM2 no valid algorithm", - EventType::Dkim(DkimEvent::HeaderHashMismatch) => "DKIM2 header hash mismatch", - EventType::Dkim(DkimEvent::Modified) => "DKIM2 message modified", - EventType::Dkim(DkimEvent::Exploded) => "DKIM2 message exploded", + EventType::Dkim2(Dkim2Event::HashMismatch) => "DKIM2 hash mismatch", + EventType::Dkim2(Dkim2Event::RecipeError) => "DKIM2 recipe error", + EventType::Dkim2(Dkim2Event::SigningFailed) => "DKIM2 signing failed", EventType::Dmarc(DmarcEvent::Pass) => "DMARC check passed", EventType::Dmarc(DmarcEvent::Fail) => "DMARC check failed", EventType::Dmarc(DmarcEvent::PermError) => "DMARC permanent error", @@ -3311,7 +3270,6 @@ impl EventType { EventType::Dns(DnsEvent::RecordLookupFailed) => "DNS record lookup failed", EventType::Dns(DnsEvent::RecordPropagated) => "DNS record propagated", EventType::Dns(DnsEvent::RecordPropagationTimeout) => "DNS record propagation timeout", - EventType::Dns(DnsEvent::BuildError) => "DNS updater build error", EventType::Eval(EvalEvent::Result) => "Expression evaluation result", EventType::Eval(EvalEvent::Error) => "Expression evaluation error", EventType::Eval(EvalEvent::DirectoryNotFound) => { @@ -3327,6 +3285,7 @@ impl EventType { EventType::Http(HttpEvent::RequestBody) => "HTTP request body", EventType::Http(HttpEvent::ResponseBody) => "HTTP response body", EventType::Http(HttpEvent::XForwardedMissing) => "X-Forwarded-For header is missing", + EventType::Http(HttpEvent::XForwardedInvalid) => "Invalid forwarding header", EventType::Imap(ImapEvent::ConnectionStart) => "IMAP connection started", EventType::Imap(ImapEvent::ConnectionEnd) => "IMAP connection ended", EventType::Imap(ImapEvent::GetAcl) => "IMAP GET ACL command", @@ -3362,6 +3321,8 @@ impl EventType { EventType::Imap(ImapEvent::Thread) => "IMAP THREAD command", EventType::Imap(ImapEvent::UidBatches) => "IMAP UIDBATCHES command", EventType::Imap(ImapEvent::GetQuota) => "IMAP GETQUOTA command", + EventType::Imap(ImapEvent::GetMetadata) => "IMAP GETMETADATA command", + EventType::Imap(ImapEvent::SetMetadata) => "IMAP SETMETADATA command", EventType::Imap(ImapEvent::Error) => "IMAP error occurred", EventType::Imap(ImapEvent::RawInput) => "Raw IMAP input received", EventType::Imap(ImapEvent::RawOutput) => "Raw IMAP output sent", @@ -3420,6 +3381,10 @@ impl EventType { "JMAP account not supported by method" } EventType::Jmap(JmapEvent::AccountReadOnly) => "JMAP account is read-only", + EventType::Jmap(JmapEvent::FromAccountNotFound) => "JMAP source account not found", + EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod) => { + "JMAP source account not supported by method" + } EventType::Jmap(JmapEvent::NotFound) => "JMAP resource not found", EventType::Jmap(JmapEvent::CannotCalculateChanges) => "Cannot calculate JMAP changes", EventType::Jmap(JmapEvent::UnknownDataType) => "Unknown JMAP data type", @@ -3429,6 +3394,15 @@ impl EventType { EventType::Jmap(JmapEvent::WebsocketStart) => "JMAP WebSocket connection started", EventType::Jmap(JmapEvent::WebsocketStop) => "JMAP WebSocket connection stopped", EventType::Jmap(JmapEvent::WebsocketError) => "JMAP WebSocket error", + EventType::Jmap(JmapEvent::ExpandDurationTooLarge) => { + "JMAP expansion duration too large" + } + EventType::Jmap(JmapEvent::CannotCalculateOccurrences) => { + "Cannot calculate JMAP occurrences" + } + EventType::Jmap(JmapEvent::TooLarge) => "JMAP time span too large", + EventType::Jmap(JmapEvent::RateLimit) => "JMAP rate limit exceeded", + EventType::Jmap(JmapEvent::ObjectNotFound) => "JMAP object not found", EventType::Limit(LimitEvent::SizeRequest) => "Request size limit reached", EventType::Limit(LimitEvent::SizeUpload) => "Upload size limit reached", EventType::Limit(LimitEvent::CallsIn) => "Incoming calls limit reached", @@ -3439,8 +3413,8 @@ impl EventType { } EventType::Limit(LimitEvent::Quota) => "Quota limit reached", EventType::Limit(LimitEvent::BlobQuota) => "Blob quota limit reached", - EventType::Limit(LimitEvent::TenantQuota) => "Tenant quota limit reached", EventType::Limit(LimitEvent::TooManyRequests) => "Too many requests", + EventType::Limit(LimitEvent::MemoryBudget) => "Memory budget exceeded", EventType::MailAuth(MailAuthEvent::ParseError) => "Mail authentication parse error", EventType::MailAuth(MailAuthEvent::MissingParameters) => { "Missing mail authentication parameters" @@ -3498,7 +3472,6 @@ impl EventType { EventType::MessageIngest(MessageIngestEvent::JmapAppend) => "Message appended via JMAP", EventType::MessageIngest(MessageIngestEvent::Duplicate) => "Skipping duplicate message", EventType::MessageIngest(MessageIngestEvent::Error) => "Message ingestion error", - EventType::MessageIngest(MessageIngestEvent::SearchIndex) => "Search index updated", EventType::Milter(MilterEvent::Read) => "Reading from Milter", EventType::Milter(MilterEvent::Write) => "Writing to Milter", EventType::Milter(MilterEvent::ActionAccept) => "Milter action: Accept", @@ -3650,6 +3623,28 @@ impl EventType { EventType::Resource(ResourceEvent::ApplicationUnpacked) => { "Application resource unpacked" } + EventType::SearchIndex(SearchIndexEvent::ManagerStarted) => { + "Search index manager started" + } + EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired) => { + "Index partitions acquired from queue" + } + EventType::SearchIndex(SearchIndexEvent::PartitionLocked) => { + "Index partition is locked by another process" + } + EventType::SearchIndex(SearchIndexEvent::PartitionRetry) => { + "Index partition will be retried" + } + EventType::SearchIndex(SearchIndexEvent::BatchIndexed) => "Search index batch written", + EventType::SearchIndex(SearchIndexEvent::DocumentIgnored) => { + "Document ignored by the indexer" + } + EventType::SearchIndex(SearchIndexEvent::DocumentTruncated) => { + "Search document truncated" + } + EventType::SearchIndex(SearchIndexEvent::MetadataNotFound) => { + "Metadata not found for indexed document" + } EventType::Security(SecurityEvent::AuthenticationBan) => { "Banned due to authentication errors" } @@ -3665,7 +3660,6 @@ impl EventType { EventType::Server(ServerEvent::Shutdown) => "Shutting down Stalwart Server", EventType::Server(ServerEvent::StartupError) => "Server startup error", EventType::Server(ServerEvent::ThreadError) => "Server thread error", - EventType::Server(ServerEvent::Licensing) => "Server licensing event", EventType::Server(ServerEvent::RecoveryMode) => "Server started in recovery mode", EventType::Server(ServerEvent::BootstrapMode) => "Server started in bootstrap mode", EventType::Sieve(SieveEvent::ActionAccept) => "Sieve action: Accept", @@ -3688,15 +3682,15 @@ impl EventType { EventType::Smtp(SmtpEvent::TransferLimitExceeded) => "Transfer limit exceeded", EventType::Smtp(SmtpEvent::RateLimitExceeded) => "Rate limit exceeded", EventType::Smtp(SmtpEvent::TimeLimitExceeded) => "Time limit exceeded", - EventType::Smtp(SmtpEvent::MissingAuthDirectory) => "Missing auth directory", EventType::Smtp(SmtpEvent::MessageParseFailed) => "Message parsing failed", EventType::Smtp(SmtpEvent::MessageTooLarge) => "Message too large", EventType::Smtp(SmtpEvent::LoopDetected) => "Mail loop detected", + EventType::Smtp(SmtpEvent::MultipleFromDomains) => "Multiple From domains", EventType::Smtp(SmtpEvent::DkimPass) => "DKIM verification passed", EventType::Smtp(SmtpEvent::DkimFail) => "DKIM verification failed", EventType::Smtp(SmtpEvent::Dkim2Pass) => "DKIM2 verification passed", EventType::Smtp(SmtpEvent::Dkim2Fail) => "DKIM2 verification failed", - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded) => "DKIM2 DSN discarded", + EventType::Smtp(SmtpEvent::Dkim2DsnRejected) => "DKIM2 DSN rejected", EventType::Smtp(SmtpEvent::ArcPass) => "ARC verification passed", EventType::Smtp(SmtpEvent::ArcFail) => "ARC verification failed", EventType::Smtp(SmtpEvent::SpfEhloPass) => "SPF EHLO check passed", @@ -3814,11 +3808,6 @@ impl EventType { EventType::Store(StoreEvent::UnexpectedError) => "Unexpected store error", EventType::Store(StoreEvent::CryptoError) => "Store crypto error", EventType::Store(StoreEvent::HttpStoreError) => "Error updating HTTP store", - EventType::Store(StoreEvent::CacheMiss) => "Cache miss", - EventType::Store(StoreEvent::CacheHit) => "Cache hit", - EventType::Store(StoreEvent::CacheStale) => "Cache is stale", - EventType::Store(StoreEvent::CacheUpdate) => "Cache update", - EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large", EventType::Store(StoreEvent::BlobMissingMarker) => "Blob missing marker", EventType::Store(StoreEvent::DataWrite) => "Write batch operation", EventType::Store(StoreEvent::DataIterate) => "Data store iteration operation", @@ -3840,19 +3829,17 @@ impl EventType { EventType::TaskManager(TaskManagerEvent::TaskLocked) => { "Task is locked by another process" } - EventType::TaskManager(TaskManagerEvent::TaskIgnored) => { - "Task ignored based on current server roles" - } - EventType::TaskManager(TaskManagerEvent::TaskFailed) => "Task failed during processing", + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => "Task ignored", + EventType::TaskManager(TaskManagerEvent::TaskFailed) => "Task failed permanently", EventType::TaskManager(TaskManagerEvent::TaskRetry) => "Task will be retried", + EventType::TaskManager(TaskManagerEvent::TaskStalled) => "Task stalled", + EventType::TaskManager(TaskManagerEvent::TaskExpired) => "Task expired", EventType::TaskManager(TaskManagerEvent::BlobNotFound) => "Blob not found for task", EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => { "Metadata not found for task" } EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => "Task scheduler started", EventType::TaskManager(TaskManagerEvent::ManagerStarted) => "Task manager started", - EventType::Telemetry(TelemetryEvent::AlertEvent) => "Alert event triggered", - EventType::Telemetry(TelemetryEvent::AlertMessage) => "Alert message sent", EventType::Telemetry(TelemetryEvent::LogError) => "Log collector error", EventType::Telemetry(TelemetryEvent::WebhookError) => "Webhook collector error", EventType::Telemetry(TelemetryEvent::OtelExporterError) => { @@ -3865,8 +3852,8 @@ impl EventType { "Prometheus exporter error" } EventType::Telemetry(TelemetryEvent::JournalError) => "Journal collector error", + EventType::Telemetry(TelemetryEvent::EventsDropped) => "Telemetry events dropped", EventType::Telemetry(TelemetryEvent::MetricsCollected) => "Metrics collected", - EventType::Telemetry(TelemetryEvent::MetricsStored) => "Metric store", EventType::Telemetry(TelemetryEvent::MetricsPushed) => "Metrics pushed", EventType::Tls(TlsEvent::Handshake) => "TLS handshake", EventType::Tls(TlsEvent::HandshakeError) => "TLS handshake error", @@ -3915,6 +3902,18 @@ impl EventType { EventType::Auth(AuthEvent::ClientRegistration) => "Authentication error", EventType::Auth(AuthEvent::Error) => "Authentication error", EventType::Auth(AuthEvent::CredentialExpired) => "Credential expired", + EventType::Cache(CacheEvent::Miss) => "Cache miss", + EventType::Cache(CacheEvent::Hit) => "Cache hit", + EventType::Cache(CacheEvent::Stale) => "Cache stale", + EventType::Cache(CacheEvent::Update) => "Cache updated", + EventType::Cache(CacheEvent::EntryTooLarge) => "Cache entry too large", + EventType::Cache(CacheEvent::Invalidate) => "Cache invalidated", + EventType::Cache(CacheEvent::SwapHit) => "Cache snapshot loaded", + EventType::Cache(CacheEvent::SwapMiss) => "Cache snapshot unavailable", + EventType::Cache(CacheEvent::SwapWrite) => "Cache snapshot written", + EventType::Cache(CacheEvent::SwapError) => "Cache swap error", + EventType::Cache(CacheEvent::SwapMetrics) => "Cache swap metrics", + EventType::Cache(CacheEvent::SwapFlush) => "Cache swap flushed", EventType::Imap(ImapEvent::ConnectionStart) => "IMAP error", EventType::Imap(ImapEvent::ConnectionEnd) => "IMAP error", EventType::Imap(ImapEvent::GetAcl) => "IMAP error", @@ -3950,6 +3949,8 @@ impl EventType { EventType::Imap(ImapEvent::Thread) => "IMAP error", EventType::Imap(ImapEvent::UidBatches) => "IMAP error", EventType::Imap(ImapEvent::GetQuota) => "IMAP error", + EventType::Imap(ImapEvent::GetMetadata) => "IMAP error", + EventType::Imap(ImapEvent::SetMetadata) => "IMAP error", EventType::Imap(ImapEvent::Error) => "IMAP error", EventType::Imap(ImapEvent::RawInput) => "IMAP error", EventType::Imap(ImapEvent::RawOutput) => "IMAP error", @@ -3968,6 +3969,10 @@ impl EventType { "Account not supported by method" } EventType::Jmap(JmapEvent::AccountReadOnly) => "Account read-only", + EventType::Jmap(JmapEvent::FromAccountNotFound) => "Source account not found", + EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod) => { + "Source account not supported by method" + } EventType::Jmap(JmapEvent::NotFound) => "Not found", EventType::Jmap(JmapEvent::CannotCalculateChanges) => "Cannot calculate changes", EventType::Jmap(JmapEvent::UnknownDataType) => "Unknown data type", @@ -3977,6 +3982,13 @@ impl EventType { EventType::Jmap(JmapEvent::WebsocketStart) => "Other message", EventType::Jmap(JmapEvent::WebsocketStop) => "Other message", EventType::Jmap(JmapEvent::WebsocketError) => "Other message", + EventType::Jmap(JmapEvent::ExpandDurationTooLarge) => "Expand duration too large", + EventType::Jmap(JmapEvent::CannotCalculateOccurrences) => { + "Cannot calculate occurrences" + } + EventType::Jmap(JmapEvent::TooLarge) => "Too large", + EventType::Jmap(JmapEvent::RateLimit) => "Rate limit", + EventType::Jmap(JmapEvent::ObjectNotFound) => "Object not found", EventType::Limit(LimitEvent::SizeRequest) => "Request too large", EventType::Limit(LimitEvent::SizeUpload) => "Upload too large", EventType::Limit(LimitEvent::CallsIn) => "Too many calls in", @@ -3985,8 +3997,8 @@ impl EventType { EventType::Limit(LimitEvent::ConcurrentConnection) => "Too many concurrent connections", EventType::Limit(LimitEvent::Quota) => "Quota exceeded", EventType::Limit(LimitEvent::BlobQuota) => "Blob quota exceeded", - EventType::Limit(LimitEvent::TenantQuota) => "Tenant quota exceeded", EventType::Limit(LimitEvent::TooManyRequests) => "Too many requests", + EventType::Limit(LimitEvent::MemoryBudget) => "Insufficient memory available", EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => "ManageSieve error", EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => "ManageSieve error", EventType::ManageSieve(ManageSieveEvent::CreateScript) => "ManageSieve error", @@ -4041,6 +4053,9 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters) => "Bad parameters", EventType::Resource(ResourceEvent::Error) => "Resource error", EventType::Resource(ResourceEvent::DownloadExternal) => "Other status", + EventType::SearchIndex(SearchIndexEvent::DocumentTruncated) => { + "Search document truncated" + } EventType::Security(SecurityEvent::AuthenticationBan) => "Insufficient permissions", EventType::Security(SecurityEvent::AbuseBan) => "Insufficient permissions", EventType::Security(SecurityEvent::ScanBan) => "Insufficient permissions", @@ -4058,15 +4073,15 @@ impl EventType { EventType::Smtp(SmtpEvent::TransferLimitExceeded) => "SMTP error", EventType::Smtp(SmtpEvent::RateLimitExceeded) => "SMTP error", EventType::Smtp(SmtpEvent::TimeLimitExceeded) => "SMTP error", - EventType::Smtp(SmtpEvent::MissingAuthDirectory) => "SMTP error", EventType::Smtp(SmtpEvent::MessageParseFailed) => "SMTP error", EventType::Smtp(SmtpEvent::MessageTooLarge) => "SMTP error", EventType::Smtp(SmtpEvent::LoopDetected) => "SMTP error", + EventType::Smtp(SmtpEvent::MultipleFromDomains) => "SMTP error", EventType::Smtp(SmtpEvent::DkimPass) => "SMTP error", EventType::Smtp(SmtpEvent::DkimFail) => "SMTP error", EventType::Smtp(SmtpEvent::Dkim2Pass) => "SMTP error", EventType::Smtp(SmtpEvent::Dkim2Fail) => "SMTP error", - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded) => "SMTP error", + EventType::Smtp(SmtpEvent::Dkim2DsnRejected) => "SMTP error", EventType::Smtp(SmtpEvent::ArcPass) => "SMTP error", EventType::Smtp(SmtpEvent::ArcFail) => "SMTP error", EventType::Smtp(SmtpEvent::SpfEhloPass) => "SMTP error", @@ -4160,11 +4175,6 @@ impl EventType { EventType::Store(StoreEvent::UnexpectedError) => "Unexpected error", EventType::Store(StoreEvent::CryptoError) => "Crypto error", EventType::Store(StoreEvent::HttpStoreError) => "Store error", - EventType::Store(StoreEvent::CacheMiss) => "Store error", - EventType::Store(StoreEvent::CacheHit) => "Store error", - EventType::Store(StoreEvent::CacheStale) => "Store error", - EventType::Store(StoreEvent::CacheUpdate) => "Store error", - EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large to store", EventType::Store(StoreEvent::BlobMissingMarker) => "Blob is missing marker", EventType::Store(StoreEvent::DataWrite) => "Store error", EventType::Store(StoreEvent::DataIterate) => "Store error", @@ -4201,14 +4211,9 @@ impl EventType { EventType::Acme(AcmeEvent::TlsAlpnError), EventType::Acme(AcmeEvent::TokenNotFound), EventType::Acme(AcmeEvent::Error), - EventType::Ai(AiEvent::LlmResponse), - EventType::Ai(AiEvent::ApiError), - EventType::Arc(ArcEvent::ChainTooLong), - EventType::Arc(ArcEvent::InvalidInstance), - EventType::Arc(ArcEvent::InvalidCv), - EventType::Arc(ArcEvent::HasHeaderTag), - EventType::Arc(ArcEvent::BrokenChain), - EventType::Arc(ArcEvent::SealerNotFound), + EventType::Arc(ArcEvent::ChainInvalid), + EventType::Arc(ArcEvent::SignatureInvalid), + EventType::Arc(ArcEvent::HashMismatch), EventType::Auth(AuthEvent::Success), EventType::Auth(AuthEvent::Failed), EventType::Auth(AuthEvent::TokenExpired), @@ -4218,6 +4223,18 @@ impl EventType { EventType::Auth(AuthEvent::Error), EventType::Auth(AuthEvent::Warning), EventType::Auth(AuthEvent::CredentialExpired), + EventType::Cache(CacheEvent::Miss), + EventType::Cache(CacheEvent::Hit), + EventType::Cache(CacheEvent::Stale), + EventType::Cache(CacheEvent::Update), + EventType::Cache(CacheEvent::EntryTooLarge), + EventType::Cache(CacheEvent::Invalidate), + EventType::Cache(CacheEvent::SwapHit), + EventType::Cache(CacheEvent::SwapMiss), + EventType::Cache(CacheEvent::SwapWrite), + EventType::Cache(CacheEvent::SwapError), + EventType::Cache(CacheEvent::SwapMetrics), + EventType::Cache(CacheEvent::SwapFlush), EventType::Calendar(CalendarEvent::RuleExpansionError), EventType::Calendar(CalendarEvent::AlarmSent), EventType::Calendar(CalendarEvent::AlarmSkipped), @@ -4293,17 +4310,10 @@ impl EventType { EventType::Dkim(DkimEvent::PermError), EventType::Dkim(DkimEvent::TempError), EventType::Dkim(DkimEvent::None), - EventType::Dkim(DkimEvent::UnsupportedVersion), - EventType::Dkim(DkimEvent::UnsupportedAlgorithm), - EventType::Dkim(DkimEvent::UnsupportedCanonicalization), - EventType::Dkim(DkimEvent::UnsupportedKeyType), - EventType::Dkim(DkimEvent::FailedBodyHashMatch), - EventType::Dkim(DkimEvent::FailedVerification), - EventType::Dkim(DkimEvent::FailedAuidMatch), - EventType::Dkim(DkimEvent::RevokedPublicKey), - EventType::Dkim(DkimEvent::IncompatibleAlgorithms), - EventType::Dkim(DkimEvent::SignatureExpired), - EventType::Dkim(DkimEvent::SignatureLength), + EventType::Dkim(DkimEvent::SignatureInvalid), + EventType::Dkim(DkimEvent::PublicKeyError), + EventType::Dkim(DkimEvent::VerificationFailed), + EventType::Dkim(DkimEvent::HashMismatch), EventType::Dkim(DkimEvent::SignerNotFound), EventType::Dkim(DkimEvent::BuildError), EventType::Dkim(DkimEvent::SignatureCreated), @@ -4311,32 +4321,17 @@ impl EventType { EventType::Dkim(DkimEvent::SignatureRetiring), EventType::Dkim(DkimEvent::SignatureRetired), EventType::Dkim(DkimEvent::SignatureDeleted), - EventType::Dkim(DkimEvent::InstanceMissing), - EventType::Dkim(DkimEvent::InstanceSyntax), - EventType::Dkim(DkimEvent::InstanceTagMissing), - EventType::Dkim(DkimEvent::InstanceNotSigned), - EventType::Dkim(DkimEvent::InstanceAboveSignature), - EventType::Dkim(DkimEvent::SignatureMissing), - EventType::Dkim(DkimEvent::SignatureSyntax), - EventType::Dkim(DkimEvent::SignatureTagMissing), - EventType::Dkim(DkimEvent::SignatureTagUnexpected), - EventType::Dkim(DkimEvent::SequenceGap), - EventType::Dkim(DkimEvent::SequenceOverflow), - EventType::Dkim(DkimEvent::ChainTooLong), - EventType::Dkim(DkimEvent::MailFromMismatch), - EventType::Dkim(DkimEvent::RcptToMismatch), - EventType::Dkim(DkimEvent::MailFromDomainMismatch), - EventType::Dkim(DkimEvent::NextDomainMismatch), - EventType::Dkim(DkimEvent::CustodyBreak), - EventType::Dkim(DkimEvent::PublicKeyFetch), - EventType::Dkim(DkimEvent::PublicKeyMissing), - EventType::Dkim(DkimEvent::PublicKeyMultiple), - EventType::Dkim(DkimEvent::PublicKeySyntax), - EventType::Dkim(DkimEvent::PublicKeyAlgorithmMismatch), - EventType::Dkim(DkimEvent::NoValidAlgorithm), - EventType::Dkim(DkimEvent::HeaderHashMismatch), - EventType::Dkim(DkimEvent::Modified), - EventType::Dkim(DkimEvent::Exploded), + EventType::Dkim2(Dkim2Event::InstanceInvalid), + EventType::Dkim2(Dkim2Event::SignatureInvalid), + EventType::Dkim2(Dkim2Event::ChainInvalid), + EventType::Dkim2(Dkim2Event::TimestampInvalid), + EventType::Dkim2(Dkim2Event::EnvelopeMismatch), + EventType::Dkim2(Dkim2Event::CustodyBroken), + EventType::Dkim2(Dkim2Event::PublicKeyError), + EventType::Dkim2(Dkim2Event::VerificationFailed), + EventType::Dkim2(Dkim2Event::HashMismatch), + EventType::Dkim2(Dkim2Event::RecipeError), + EventType::Dkim2(Dkim2Event::SigningFailed), EventType::Dmarc(DmarcEvent::Pass), EventType::Dmarc(DmarcEvent::Fail), EventType::Dmarc(DmarcEvent::PermError), @@ -4349,7 +4344,6 @@ impl EventType { EventType::Dns(DnsEvent::RecordLookupFailed), EventType::Dns(DnsEvent::RecordPropagated), EventType::Dns(DnsEvent::RecordPropagationTimeout), - EventType::Dns(DnsEvent::BuildError), EventType::Eval(EvalEvent::Result), EventType::Eval(EvalEvent::Error), EventType::Eval(EvalEvent::DirectoryNotFound), @@ -4361,6 +4355,7 @@ impl EventType { EventType::Http(HttpEvent::RequestBody), EventType::Http(HttpEvent::ResponseBody), EventType::Http(HttpEvent::XForwardedMissing), + EventType::Http(HttpEvent::XForwardedInvalid), EventType::Imap(ImapEvent::ConnectionStart), EventType::Imap(ImapEvent::ConnectionEnd), EventType::Imap(ImapEvent::GetAcl), @@ -4396,6 +4391,8 @@ impl EventType { EventType::Imap(ImapEvent::Thread), EventType::Imap(ImapEvent::UidBatches), EventType::Imap(ImapEvent::GetQuota), + EventType::Imap(ImapEvent::GetMetadata), + EventType::Imap(ImapEvent::SetMetadata), EventType::Imap(ImapEvent::Error), EventType::Imap(ImapEvent::RawInput), EventType::Imap(ImapEvent::RawOutput), @@ -4432,6 +4429,8 @@ impl EventType { EventType::Jmap(JmapEvent::AccountNotFound), EventType::Jmap(JmapEvent::AccountNotSupportedByMethod), EventType::Jmap(JmapEvent::AccountReadOnly), + EventType::Jmap(JmapEvent::FromAccountNotFound), + EventType::Jmap(JmapEvent::FromAccountNotSupportedByMethod), EventType::Jmap(JmapEvent::NotFound), EventType::Jmap(JmapEvent::CannotCalculateChanges), EventType::Jmap(JmapEvent::UnknownDataType), @@ -4441,6 +4440,11 @@ impl EventType { EventType::Jmap(JmapEvent::WebsocketStart), EventType::Jmap(JmapEvent::WebsocketStop), EventType::Jmap(JmapEvent::WebsocketError), + EventType::Jmap(JmapEvent::ExpandDurationTooLarge), + EventType::Jmap(JmapEvent::CannotCalculateOccurrences), + EventType::Jmap(JmapEvent::TooLarge), + EventType::Jmap(JmapEvent::RateLimit), + EventType::Jmap(JmapEvent::ObjectNotFound), EventType::Limit(LimitEvent::SizeRequest), EventType::Limit(LimitEvent::SizeUpload), EventType::Limit(LimitEvent::CallsIn), @@ -4449,8 +4453,8 @@ impl EventType { EventType::Limit(LimitEvent::ConcurrentConnection), EventType::Limit(LimitEvent::Quota), EventType::Limit(LimitEvent::BlobQuota), - EventType::Limit(LimitEvent::TenantQuota), EventType::Limit(LimitEvent::TooManyRequests), + EventType::Limit(LimitEvent::MemoryBudget), EventType::MailAuth(MailAuthEvent::ParseError), EventType::MailAuth(MailAuthEvent::MissingParameters), EventType::MailAuth(MailAuthEvent::NoHeadersFound), @@ -4486,7 +4490,6 @@ impl EventType { EventType::MessageIngest(MessageIngestEvent::JmapAppend), EventType::MessageIngest(MessageIngestEvent::Duplicate), EventType::MessageIngest(MessageIngestEvent::Error), - EventType::MessageIngest(MessageIngestEvent::SearchIndex), EventType::Milter(MilterEvent::Read), EventType::Milter(MilterEvent::Write), EventType::Milter(MilterEvent::ActionAccept), @@ -4594,6 +4597,14 @@ impl EventType { EventType::Resource(ResourceEvent::DownloadExternal), EventType::Resource(ResourceEvent::ApplicationUpdated), EventType::Resource(ResourceEvent::ApplicationUnpacked), + EventType::SearchIndex(SearchIndexEvent::ManagerStarted), + EventType::SearchIndex(SearchIndexEvent::PartitionsAcquired), + EventType::SearchIndex(SearchIndexEvent::PartitionLocked), + EventType::SearchIndex(SearchIndexEvent::PartitionRetry), + EventType::SearchIndex(SearchIndexEvent::BatchIndexed), + EventType::SearchIndex(SearchIndexEvent::DocumentIgnored), + EventType::SearchIndex(SearchIndexEvent::DocumentTruncated), + EventType::SearchIndex(SearchIndexEvent::MetadataNotFound), EventType::Security(SecurityEvent::AuthenticationBan), EventType::Security(SecurityEvent::AbuseBan), EventType::Security(SecurityEvent::ScanBan), @@ -4607,7 +4618,6 @@ impl EventType { EventType::Server(ServerEvent::Shutdown), EventType::Server(ServerEvent::StartupError), EventType::Server(ServerEvent::ThreadError), - EventType::Server(ServerEvent::Licensing), EventType::Server(ServerEvent::RecoveryMode), EventType::Server(ServerEvent::BootstrapMode), EventType::Sieve(SieveEvent::ActionAccept), @@ -4630,15 +4640,15 @@ impl EventType { EventType::Smtp(SmtpEvent::TransferLimitExceeded), EventType::Smtp(SmtpEvent::RateLimitExceeded), EventType::Smtp(SmtpEvent::TimeLimitExceeded), - EventType::Smtp(SmtpEvent::MissingAuthDirectory), EventType::Smtp(SmtpEvent::MessageParseFailed), EventType::Smtp(SmtpEvent::MessageTooLarge), EventType::Smtp(SmtpEvent::LoopDetected), + EventType::Smtp(SmtpEvent::MultipleFromDomains), EventType::Smtp(SmtpEvent::DkimPass), EventType::Smtp(SmtpEvent::DkimFail), EventType::Smtp(SmtpEvent::Dkim2Pass), EventType::Smtp(SmtpEvent::Dkim2Fail), - EventType::Smtp(SmtpEvent::Dkim2DsnDiscarded), + EventType::Smtp(SmtpEvent::Dkim2DsnRejected), EventType::Smtp(SmtpEvent::ArcPass), EventType::Smtp(SmtpEvent::ArcFail), EventType::Smtp(SmtpEvent::SpfEhloPass), @@ -4750,11 +4760,6 @@ impl EventType { EventType::Store(StoreEvent::UnexpectedError), EventType::Store(StoreEvent::CryptoError), EventType::Store(StoreEvent::HttpStoreError), - EventType::Store(StoreEvent::CacheMiss), - EventType::Store(StoreEvent::CacheHit), - EventType::Store(StoreEvent::CacheStale), - EventType::Store(StoreEvent::CacheUpdate), - EventType::Store(StoreEvent::CacheEntryTooLarge), EventType::Store(StoreEvent::BlobMissingMarker), EventType::Store(StoreEvent::DataWrite), EventType::Store(StoreEvent::DataIterate), @@ -4775,20 +4780,20 @@ impl EventType { EventType::TaskManager(TaskManagerEvent::TaskIgnored), EventType::TaskManager(TaskManagerEvent::TaskFailed), EventType::TaskManager(TaskManagerEvent::TaskRetry), + EventType::TaskManager(TaskManagerEvent::TaskStalled), + EventType::TaskManager(TaskManagerEvent::TaskExpired), EventType::TaskManager(TaskManagerEvent::BlobNotFound), EventType::TaskManager(TaskManagerEvent::MetadataNotFound), EventType::TaskManager(TaskManagerEvent::SchedulerStarted), EventType::TaskManager(TaskManagerEvent::ManagerStarted), - EventType::Telemetry(TelemetryEvent::AlertEvent), - EventType::Telemetry(TelemetryEvent::AlertMessage), EventType::Telemetry(TelemetryEvent::LogError), EventType::Telemetry(TelemetryEvent::WebhookError), EventType::Telemetry(TelemetryEvent::OtelExporterError), EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError), EventType::Telemetry(TelemetryEvent::PrometheusExporterError), EventType::Telemetry(TelemetryEvent::JournalError), + EventType::Telemetry(TelemetryEvent::EventsDropped), EventType::Telemetry(TelemetryEvent::MetricsCollected), - EventType::Telemetry(TelemetryEvent::MetricsStored), EventType::Telemetry(TelemetryEvent::MetricsPushed), EventType::Tls(TlsEvent::Handshake), EventType::Tls(TlsEvent::HandshakeError), @@ -4823,6 +4828,7 @@ impl EventType { } impl MetricType { + #[cfg(not(feature = "enterprise"))] pub fn parse(value: &str) -> Option { hashify::map! { value.as_bytes(), @@ -4835,15 +4841,25 @@ impl MetricType { b"acme.tls-alpn-error" => MetricType::AcmeTlsAlpnError, b"acme.token-not-found" => MetricType::AcmeTokenNotFound, b"acme.error" => MetricType::AcmeError, - b"arc.chain-too-long" => MetricType::ArcChainTooLong, - b"arc.invalid-instance" => MetricType::ArcInvalidInstance, - b"arc.invalid-cv" => MetricType::ArcInvalidCv, - b"arc.has-header-tag" => MetricType::ArcHasHeaderTag, - b"arc.broken-chain" => MetricType::ArcBrokenChain, + b"arc.chain-invalid" => MetricType::ArcChainInvalid, + b"arc.signature-invalid" => MetricType::ArcSignatureInvalid, + b"arc.hash-mismatch" => MetricType::ArcHashMismatch, b"auth.success" => MetricType::AuthSuccess, b"auth.failed" => MetricType::AuthFailed, b"auth.too-many-attempts" => MetricType::AuthTooManyAttempts, b"auth.error" => MetricType::AuthError, + b"cache.messages-size" => MetricType::CacheMessagesSize, + b"cache.messages-large-size" => MetricType::CacheMessagesLargeSize, + b"cache.size" => MetricType::CacheSize, + b"cache.swap-pending" => MetricType::CacheSwapPending, + b"cache.miss" => MetricType::CacheMiss, + b"cache.hit" => MetricType::CacheHit, + b"cache.stale" => MetricType::CacheStale, + b"cache.entry-too-large" => MetricType::CacheEntryTooLarge, + b"cache.swap-hit" => MetricType::CacheSwapHit, + b"cache.swap-miss" => MetricType::CacheSwapMiss, + b"cache.swap-write" => MetricType::CacheSwapWrite, + b"cache.swap-error" => MetricType::CacheSwapError, b"calendar.alarm-sent" => MetricType::CalendarAlarmSent, b"calendar.alarm-failed" => MetricType::CalendarAlarmFailed, b"calendar.itip-message-sent" => MetricType::CalendarItipMessageSent, @@ -4894,44 +4910,22 @@ impl MetricType { b"dkim.perm-error" => MetricType::DkimPermError, b"dkim.temp-error" => MetricType::DkimTempError, b"dkim.none" => MetricType::DkimNone, - b"dkim.unsupported-version" => MetricType::DkimUnsupportedVersion, - b"dkim.unsupported-algorithm" => MetricType::DkimUnsupportedAlgorithm, - b"dkim.unsupported-canonicalization" => MetricType::DkimUnsupportedCanonicalization, - b"dkim.unsupported-key-type" => MetricType::DkimUnsupportedKeyType, - b"dkim.failed-body-hash-match" => MetricType::DkimFailedBodyHashMatch, - b"dkim.failed-verification" => MetricType::DkimFailedVerification, - b"dkim.failed-auid-match" => MetricType::DkimFailedAuidMatch, - b"dkim.revoked-public-key" => MetricType::DkimRevokedPublicKey, - b"dkim.incompatible-algorithms" => MetricType::DkimIncompatibleAlgorithms, - b"dkim.signature-expired" => MetricType::DkimSignatureExpired, - b"dkim.signature-length" => MetricType::DkimSignatureLength, + b"dkim.signature-invalid" => MetricType::DkimSignatureInvalid, + b"dkim.public-key-error" => MetricType::DkimPublicKeyError, + b"dkim.verification-failed" => MetricType::DkimVerificationFailed, + b"dkim.hash-mismatch" => MetricType::DkimHashMismatch, b"dkim.signer-not-found" => MetricType::DkimSignerNotFound, - b"dkim.instance-missing" => MetricType::DkimInstanceMissing, - b"dkim.instance-syntax" => MetricType::DkimInstanceSyntax, - b"dkim.instance-tag-missing" => MetricType::DkimInstanceTagMissing, - b"dkim.instance-not-signed" => MetricType::DkimInstanceNotSigned, - b"dkim.instance-above-signature" => MetricType::DkimInstanceAboveSignature, - b"dkim.signature-missing" => MetricType::DkimSignatureMissing, - b"dkim.signature-syntax" => MetricType::DkimSignatureSyntax, - b"dkim.signature-tag-missing" => MetricType::DkimSignatureTagMissing, - b"dkim.signature-tag-unexpected" => MetricType::DkimSignatureTagUnexpected, - b"dkim.sequence-gap" => MetricType::DkimSequenceGap, - b"dkim.sequence-overflow" => MetricType::DkimSequenceOverflow, - b"dkim.chain-too-long" => MetricType::DkimChainTooLong, - b"dkim.mail-from-mismatch" => MetricType::DkimMailFromMismatch, - b"dkim.rcpt-to-mismatch" => MetricType::DkimRcptToMismatch, - b"dkim.mail-from-domain-mismatch" => MetricType::DkimMailFromDomainMismatch, - b"dkim.next-domain-mismatch" => MetricType::DkimNextDomainMismatch, - b"dkim.custody-break" => MetricType::DkimCustodyBreak, - b"dkim.public-key-fetch" => MetricType::DkimPublicKeyFetch, - b"dkim.public-key-missing" => MetricType::DkimPublicKeyMissing, - b"dkim.public-key-multiple" => MetricType::DkimPublicKeyMultiple, - b"dkim.public-key-syntax" => MetricType::DkimPublicKeySyntax, - b"dkim.public-key-algorithm-mismatch" => MetricType::DkimPublicKeyAlgorithmMismatch, - b"dkim.no-valid-algorithm" => MetricType::DkimNoValidAlgorithm, - b"dkim.header-hash-mismatch" => MetricType::DkimHeaderHashMismatch, - b"dkim.modified" => MetricType::DkimModified, - b"dkim.exploded" => MetricType::DkimExploded, + b"dkim2.instance-invalid" => MetricType::Dkim2InstanceInvalid, + b"dkim2.signature-invalid" => MetricType::Dkim2SignatureInvalid, + b"dkim2.chain-invalid" => MetricType::Dkim2ChainInvalid, + b"dkim2.timestamp-invalid" => MetricType::Dkim2TimestampInvalid, + b"dkim2.envelope-mismatch" => MetricType::Dkim2EnvelopeMismatch, + b"dkim2.custody-broken" => MetricType::Dkim2CustodyBroken, + b"dkim2.public-key-error" => MetricType::Dkim2PublicKeyError, + b"dkim2.verification-failed" => MetricType::Dkim2VerificationFailed, + b"dkim2.hash-mismatch" => MetricType::Dkim2HashMismatch, + b"dkim2.recipe-error" => MetricType::Dkim2RecipeError, + b"dkim2.signing-failed" => MetricType::Dkim2SigningFailed, b"dmarc.pass" => MetricType::DmarcPass, b"dmarc.fail" => MetricType::DmarcFail, b"dmarc.perm-error" => MetricType::DmarcPermError, @@ -4953,6 +4947,7 @@ impl MetricType { b"http.request-body" => MetricType::HttpRequestBody, b"http.response-body" => MetricType::HttpResponseBody, b"http.x-forwarded-missing" => MetricType::HttpXForwardedMissing, + b"http.x-forwarded-invalid" => MetricType::HttpXForwardedInvalid, b"imap.request-time" => MetricType::ImapRequestTime, b"imap.active-connections" => MetricType::ImapActiveConnections, b"imap.connection-start" => MetricType::ImapConnectionStart, @@ -4996,8 +4991,8 @@ impl MetricType { b"limit.concurrent-connection" => MetricType::LimitConcurrentConnection, b"limit.quota" => MetricType::LimitQuota, b"limit.blob-quota" => MetricType::LimitBlobQuota, - b"limit.tenant-quota" => MetricType::LimitTenantQuota, b"limit.too-many-requests" => MetricType::LimitTooManyRequests, + b"limit.memory-budget" => MetricType::LimitMemoryBudget, b"mail-auth.parse-error" => MetricType::MailAuthParseError, b"mail-auth.missing-parameters" => MetricType::MailAuthMissingParameters, b"mail-auth.no-headers-found" => MetricType::MailAuthNoHeadersFound, @@ -5013,14 +5008,12 @@ impl MetricType { b"message.size" => MetricType::MessageSize, b"message.authenticated-size" => MetricType::MessageAuthenticatedSize, b"message-ingest.time" => MetricType::MessageIngestTime, - b"message-ingest.index-time" => MetricType::MessageIngestIndexTime, b"message-ingest.ham" => MetricType::MessageIngestHam, b"message-ingest.spam" => MetricType::MessageIngestSpam, b"message-ingest.imap-append" => MetricType::MessageIngestImapAppend, b"message-ingest.jmap-append" => MetricType::MessageIngestJmapAppend, b"message-ingest.duplicate" => MetricType::MessageIngestDuplicate, b"message-ingest.error" => MetricType::MessageIngestError, - b"message-ingest.search-index" => MetricType::MessageIngestSearchIndex, b"milter.action-accept" => MetricType::MilterActionAccept, b"milter.action-discard" => MetricType::MilterActionDiscard, b"milter.action-reject" => MetricType::MilterActionReject, @@ -5070,9 +5063,18 @@ impl MetricType { b"queue.rate-limit-exceeded" => MetricType::QueueRateLimitExceeded, b"queue.concurrency-limit-exceeded" => MetricType::QueueConcurrencyLimitExceeded, b"queue.quota-exceeded" => MetricType::QueueQuotaExceeded, + b"queue.back-pressure" => MetricType::QueueBackPressure, b"resource.not-found" => MetricType::ResourceNotFound, b"resource.bad-parameters" => MetricType::ResourceBadParameters, b"resource.error" => MetricType::ResourceError, + b"search-index.index-time" => MetricType::SearchIndexIndexTime, + b"search-index.pending-email" => MetricType::SearchIndexPendingEmail, + b"search-index.pending-calendar" => MetricType::SearchIndexPendingCalendar, + b"search-index.pending-contacts" => MetricType::SearchIndexPendingContacts, + b"search-index.pending-file" => MetricType::SearchIndexPendingFile, + b"search-index.partition-retry" => MetricType::SearchIndexPartitionRetry, + b"search-index.batch-indexed" => MetricType::SearchIndexBatchIndexed, + b"search-index.document-truncated" => MetricType::SearchIndexDocumentTruncated, b"security.authentication-ban" => MetricType::SecurityAuthenticationBan, b"security.abuse-ban" => MetricType::SecurityAbuseBan, b"security.scan-ban" => MetricType::SecurityScanBan, @@ -5080,6 +5082,8 @@ impl MetricType { b"security.ip-blocked" => MetricType::SecurityIpBlocked, b"security.unauthorized" => MetricType::SecurityUnauthorized, b"server.memory" => MetricType::ServerMemory, + b"server.buffered-memory-smtp" => MetricType::ServerBufferedMemorySmtp, + b"server.buffered-memory-read" => MetricType::ServerBufferedMemoryRead, b"server.thread-error" => MetricType::ServerThreadError, b"sieve.request-time" => MetricType::SieveRequestTime, b"sieve.active-connections" => MetricType::SieveActiveConnections, @@ -5105,11 +5109,12 @@ impl MetricType { b"smtp.message-parse-failed" => MetricType::SmtpMessageParseFailed, b"smtp.message-too-large" => MetricType::SmtpMessageTooLarge, b"smtp.loop-detected" => MetricType::SmtpLoopDetected, + b"smtp.multiple-from-domains" => MetricType::SmtpMultipleFromDomains, b"smtp.dkim-pass" => MetricType::SmtpDkimPass, b"smtp.dkim-fail" => MetricType::SmtpDkimFail, b"smtp.dkim2-pass" => MetricType::SmtpDkim2Pass, b"smtp.dkim2-fail" => MetricType::SmtpDkim2Fail, - b"smtp.dkim2-dsn-discarded" => MetricType::SmtpDkim2DsnDiscarded, + b"smtp.dkim2-dsn-rejected" => MetricType::SmtpDkim2DsnRejected, b"smtp.arc-pass" => MetricType::SmtpArcPass, b"smtp.arc-fail" => MetricType::SmtpArcFail, b"smtp.spf-ehlo-pass" => MetricType::SmtpSpfEhloPass, @@ -5184,16 +5189,21 @@ impl MetricType { b"store.blob-read" => MetricType::StoreBlobRead, b"store.blob-write" => MetricType::StoreBlobWrite, b"store.blob-delete" => MetricType::StoreBlobDelete, + b"task-manager.pending" => MetricType::TaskManagerPending, + b"task-manager.due" => MetricType::TaskManagerDue, + b"task-manager.task-failed" => MetricType::TaskManagerTaskFailed, + b"task-manager.task-retry" => MetricType::TaskManagerTaskRetry, + b"task-manager.task-stalled" => MetricType::TaskManagerTaskStalled, + b"task-manager.task-expired" => MetricType::TaskManagerTaskExpired, b"task-manager.blob-not-found" => MetricType::TaskManagerBlobNotFound, b"task-manager.metadata-not-found" => MetricType::TaskManagerMetadataNotFound, - b"telemetry.alert-event" => MetricType::TelemetryAlertEvent, - b"telemetry.alert-message" => MetricType::TelemetryAlertMessage, b"telemetry.log-error" => MetricType::TelemetryLogError, b"telemetry.webhook-error" => MetricType::TelemetryWebhookError, b"telemetry.otel-exporter-error" => MetricType::TelemetryOtelExporterError, b"telemetry.otel-metrics-exporter-error" => MetricType::TelemetryOtelMetricsExporterError, b"telemetry.prometheus-exporter-error" => MetricType::TelemetryPrometheusExporterError, b"telemetry.journal-error" => MetricType::TelemetryJournalError, + b"telemetry.events-dropped" => MetricType::TelemetryEventsDropped, b"tls.handshake-error" => MetricType::TlsHandshakeError, b"user.count" => MetricType::UserCount, } @@ -5210,15 +5220,25 @@ impl MetricType { MetricType::AcmeTlsAlpnError => "acme.tls-alpn-error", MetricType::AcmeTokenNotFound => "acme.token-not-found", MetricType::AcmeError => "acme.error", - MetricType::ArcChainTooLong => "arc.chain-too-long", - MetricType::ArcInvalidInstance => "arc.invalid-instance", - MetricType::ArcInvalidCv => "arc.invalid-cv", - MetricType::ArcHasHeaderTag => "arc.has-header-tag", - MetricType::ArcBrokenChain => "arc.broken-chain", + MetricType::ArcChainInvalid => "arc.chain-invalid", + MetricType::ArcSignatureInvalid => "arc.signature-invalid", + MetricType::ArcHashMismatch => "arc.hash-mismatch", MetricType::AuthSuccess => "auth.success", MetricType::AuthFailed => "auth.failed", MetricType::AuthTooManyAttempts => "auth.too-many-attempts", MetricType::AuthError => "auth.error", + MetricType::CacheMessagesSize => "cache.messages-size", + MetricType::CacheMessagesLargeSize => "cache.messages-large-size", + MetricType::CacheSize => "cache.size", + MetricType::CacheSwapPending => "cache.swap-pending", + MetricType::CacheMiss => "cache.miss", + MetricType::CacheHit => "cache.hit", + MetricType::CacheStale => "cache.stale", + MetricType::CacheEntryTooLarge => "cache.entry-too-large", + MetricType::CacheSwapHit => "cache.swap-hit", + MetricType::CacheSwapMiss => "cache.swap-miss", + MetricType::CacheSwapWrite => "cache.swap-write", + MetricType::CacheSwapError => "cache.swap-error", MetricType::CalendarAlarmSent => "calendar.alarm-sent", MetricType::CalendarAlarmFailed => "calendar.alarm-failed", MetricType::CalendarItipMessageSent => "calendar.itip-message-sent", @@ -5269,44 +5289,22 @@ impl MetricType { MetricType::DkimPermError => "dkim.perm-error", MetricType::DkimTempError => "dkim.temp-error", MetricType::DkimNone => "dkim.none", - MetricType::DkimUnsupportedVersion => "dkim.unsupported-version", - MetricType::DkimUnsupportedAlgorithm => "dkim.unsupported-algorithm", - MetricType::DkimUnsupportedCanonicalization => "dkim.unsupported-canonicalization", - MetricType::DkimUnsupportedKeyType => "dkim.unsupported-key-type", - MetricType::DkimFailedBodyHashMatch => "dkim.failed-body-hash-match", - MetricType::DkimFailedVerification => "dkim.failed-verification", - MetricType::DkimFailedAuidMatch => "dkim.failed-auid-match", - MetricType::DkimRevokedPublicKey => "dkim.revoked-public-key", - MetricType::DkimIncompatibleAlgorithms => "dkim.incompatible-algorithms", - MetricType::DkimSignatureExpired => "dkim.signature-expired", - MetricType::DkimSignatureLength => "dkim.signature-length", + MetricType::DkimSignatureInvalid => "dkim.signature-invalid", + MetricType::DkimPublicKeyError => "dkim.public-key-error", + MetricType::DkimVerificationFailed => "dkim.verification-failed", + MetricType::DkimHashMismatch => "dkim.hash-mismatch", MetricType::DkimSignerNotFound => "dkim.signer-not-found", - MetricType::DkimInstanceMissing => "dkim.instance-missing", - MetricType::DkimInstanceSyntax => "dkim.instance-syntax", - MetricType::DkimInstanceTagMissing => "dkim.instance-tag-missing", - MetricType::DkimInstanceNotSigned => "dkim.instance-not-signed", - MetricType::DkimInstanceAboveSignature => "dkim.instance-above-signature", - MetricType::DkimSignatureMissing => "dkim.signature-missing", - MetricType::DkimSignatureSyntax => "dkim.signature-syntax", - MetricType::DkimSignatureTagMissing => "dkim.signature-tag-missing", - MetricType::DkimSignatureTagUnexpected => "dkim.signature-tag-unexpected", - MetricType::DkimSequenceGap => "dkim.sequence-gap", - MetricType::DkimSequenceOverflow => "dkim.sequence-overflow", - MetricType::DkimChainTooLong => "dkim.chain-too-long", - MetricType::DkimMailFromMismatch => "dkim.mail-from-mismatch", - MetricType::DkimRcptToMismatch => "dkim.rcpt-to-mismatch", - MetricType::DkimMailFromDomainMismatch => "dkim.mail-from-domain-mismatch", - MetricType::DkimNextDomainMismatch => "dkim.next-domain-mismatch", - MetricType::DkimCustodyBreak => "dkim.custody-break", - MetricType::DkimPublicKeyFetch => "dkim.public-key-fetch", - MetricType::DkimPublicKeyMissing => "dkim.public-key-missing", - MetricType::DkimPublicKeyMultiple => "dkim.public-key-multiple", - MetricType::DkimPublicKeySyntax => "dkim.public-key-syntax", - MetricType::DkimPublicKeyAlgorithmMismatch => "dkim.public-key-algorithm-mismatch", - MetricType::DkimNoValidAlgorithm => "dkim.no-valid-algorithm", - MetricType::DkimHeaderHashMismatch => "dkim.header-hash-mismatch", - MetricType::DkimModified => "dkim.modified", - MetricType::DkimExploded => "dkim.exploded", + MetricType::Dkim2InstanceInvalid => "dkim2.instance-invalid", + MetricType::Dkim2SignatureInvalid => "dkim2.signature-invalid", + MetricType::Dkim2ChainInvalid => "dkim2.chain-invalid", + MetricType::Dkim2TimestampInvalid => "dkim2.timestamp-invalid", + MetricType::Dkim2EnvelopeMismatch => "dkim2.envelope-mismatch", + MetricType::Dkim2CustodyBroken => "dkim2.custody-broken", + MetricType::Dkim2PublicKeyError => "dkim2.public-key-error", + MetricType::Dkim2VerificationFailed => "dkim2.verification-failed", + MetricType::Dkim2HashMismatch => "dkim2.hash-mismatch", + MetricType::Dkim2RecipeError => "dkim2.recipe-error", + MetricType::Dkim2SigningFailed => "dkim2.signing-failed", MetricType::DmarcPass => "dmarc.pass", MetricType::DmarcFail => "dmarc.fail", MetricType::DmarcPermError => "dmarc.perm-error", @@ -5328,6 +5326,7 @@ impl MetricType { MetricType::HttpRequestBody => "http.request-body", MetricType::HttpResponseBody => "http.response-body", MetricType::HttpXForwardedMissing => "http.x-forwarded-missing", + MetricType::HttpXForwardedInvalid => "http.x-forwarded-invalid", MetricType::ImapRequestTime => "imap.request-time", MetricType::ImapActiveConnections => "imap.active-connections", MetricType::ImapConnectionStart => "imap.connection-start", @@ -5375,8 +5374,8 @@ impl MetricType { MetricType::LimitConcurrentConnection => "limit.concurrent-connection", MetricType::LimitQuota => "limit.quota", MetricType::LimitBlobQuota => "limit.blob-quota", - MetricType::LimitTenantQuota => "limit.tenant-quota", MetricType::LimitTooManyRequests => "limit.too-many-requests", + MetricType::LimitMemoryBudget => "limit.memory-budget", MetricType::MailAuthParseError => "mail-auth.parse-error", MetricType::MailAuthMissingParameters => "mail-auth.missing-parameters", MetricType::MailAuthNoHeadersFound => "mail-auth.no-headers-found", @@ -5392,14 +5391,12 @@ impl MetricType { MetricType::MessageSize => "message.size", MetricType::MessageAuthenticatedSize => "message.authenticated-size", MetricType::MessageIngestTime => "message-ingest.time", - MetricType::MessageIngestIndexTime => "message-ingest.index-time", MetricType::MessageIngestHam => "message-ingest.ham", MetricType::MessageIngestSpam => "message-ingest.spam", MetricType::MessageIngestImapAppend => "message-ingest.imap-append", MetricType::MessageIngestJmapAppend => "message-ingest.jmap-append", MetricType::MessageIngestDuplicate => "message-ingest.duplicate", MetricType::MessageIngestError => "message-ingest.error", - MetricType::MessageIngestSearchIndex => "message-ingest.search-index", MetricType::MilterActionAccept => "milter.action-accept", MetricType::MilterActionDiscard => "milter.action-discard", MetricType::MilterActionReject => "milter.action-reject", @@ -5455,9 +5452,18 @@ impl MetricType { MetricType::QueueRateLimitExceeded => "queue.rate-limit-exceeded", MetricType::QueueConcurrencyLimitExceeded => "queue.concurrency-limit-exceeded", MetricType::QueueQuotaExceeded => "queue.quota-exceeded", + MetricType::QueueBackPressure => "queue.back-pressure", MetricType::ResourceNotFound => "resource.not-found", MetricType::ResourceBadParameters => "resource.bad-parameters", MetricType::ResourceError => "resource.error", + MetricType::SearchIndexIndexTime => "search-index.index-time", + MetricType::SearchIndexPendingEmail => "search-index.pending-email", + MetricType::SearchIndexPendingCalendar => "search-index.pending-calendar", + MetricType::SearchIndexPendingContacts => "search-index.pending-contacts", + MetricType::SearchIndexPendingFile => "search-index.pending-file", + MetricType::SearchIndexPartitionRetry => "search-index.partition-retry", + MetricType::SearchIndexBatchIndexed => "search-index.batch-indexed", + MetricType::SearchIndexDocumentTruncated => "search-index.document-truncated", MetricType::SecurityAuthenticationBan => "security.authentication-ban", MetricType::SecurityAbuseBan => "security.abuse-ban", MetricType::SecurityScanBan => "security.scan-ban", @@ -5465,6 +5471,8 @@ impl MetricType { MetricType::SecurityIpBlocked => "security.ip-blocked", MetricType::SecurityUnauthorized => "security.unauthorized", MetricType::ServerMemory => "server.memory", + MetricType::ServerBufferedMemorySmtp => "server.buffered-memory-smtp", + MetricType::ServerBufferedMemoryRead => "server.buffered-memory-read", MetricType::ServerThreadError => "server.thread-error", MetricType::SieveRequestTime => "sieve.request-time", MetricType::SieveActiveConnections => "sieve.active-connections", @@ -5490,11 +5498,12 @@ impl MetricType { MetricType::SmtpMessageParseFailed => "smtp.message-parse-failed", MetricType::SmtpMessageTooLarge => "smtp.message-too-large", MetricType::SmtpLoopDetected => "smtp.loop-detected", + MetricType::SmtpMultipleFromDomains => "smtp.multiple-from-domains", MetricType::SmtpDkimPass => "smtp.dkim-pass", MetricType::SmtpDkimFail => "smtp.dkim-fail", MetricType::SmtpDkim2Pass => "smtp.dkim2-pass", MetricType::SmtpDkim2Fail => "smtp.dkim2-fail", - MetricType::SmtpDkim2DsnDiscarded => "smtp.dkim2-dsn-discarded", + MetricType::SmtpDkim2DsnRejected => "smtp.dkim2-dsn-rejected", MetricType::SmtpArcPass => "smtp.arc-pass", MetricType::SmtpArcFail => "smtp.arc-fail", MetricType::SmtpSpfEhloPass => "smtp.spf-ehlo-pass", @@ -5569,10 +5578,14 @@ impl MetricType { MetricType::StoreBlobRead => "store.blob-read", MetricType::StoreBlobWrite => "store.blob-write", MetricType::StoreBlobDelete => "store.blob-delete", + MetricType::TaskManagerPending => "task-manager.pending", + MetricType::TaskManagerDue => "task-manager.due", + MetricType::TaskManagerTaskFailed => "task-manager.task-failed", + MetricType::TaskManagerTaskRetry => "task-manager.task-retry", + MetricType::TaskManagerTaskStalled => "task-manager.task-stalled", + MetricType::TaskManagerTaskExpired => "task-manager.task-expired", MetricType::TaskManagerBlobNotFound => "task-manager.blob-not-found", MetricType::TaskManagerMetadataNotFound => "task-manager.metadata-not-found", - MetricType::TelemetryAlertEvent => "telemetry.alert-event", - MetricType::TelemetryAlertMessage => "telemetry.alert-message", MetricType::TelemetryLogError => "telemetry.log-error", MetricType::TelemetryWebhookError => "telemetry.webhook-error", MetricType::TelemetryOtelExporterError => "telemetry.otel-exporter-error", @@ -5581,6 +5594,7 @@ impl MetricType { } MetricType::TelemetryPrometheusExporterError => "telemetry.prometheus-exporter-error", MetricType::TelemetryJournalError => "telemetry.journal-error", + MetricType::TelemetryEventsDropped => "telemetry.events-dropped", MetricType::TlsHandshakeError => "tls.handshake-error", MetricType::UserCount => "user.count", } @@ -5596,366 +5610,370 @@ impl MetricType { MetricType::AcmeTlsAlpnError => 32, MetricType::AcmeTokenNotFound => 33, MetricType::AcmeError => 34, - MetricType::ArcChainTooLong => 39, - MetricType::ArcInvalidInstance => 40, - MetricType::ArcInvalidCv => 41, - MetricType::ArcHasHeaderTag => 42, - MetricType::ArcBrokenChain => 43, - MetricType::AuthSuccess => 44, - MetricType::AuthFailed => 45, - MetricType::AuthTooManyAttempts => 46, - MetricType::AuthError => 47, - MetricType::CalendarAlarmSent => 48, - MetricType::CalendarAlarmFailed => 49, - MetricType::CalendarItipMessageSent => 50, - MetricType::CalendarItipMessageReceived => 51, - MetricType::CalendarItipMessageError => 52, - MetricType::ClusterSubscriberError => 53, - MetricType::ClusterSubscriberDisconnected => 54, - MetricType::ClusterPublisherError => 55, - MetricType::DaneAuthenticationSuccess => 56, - MetricType::DaneAuthenticationFailure => 57, - MetricType::DaneNoCertificatesFound => 58, - MetricType::DaneCertificateParseError => 59, - MetricType::DaneTlsaRecordFetchError => 60, - MetricType::DaneTlsaRecordNotFound => 61, - MetricType::DaneTlsaRecordNotDnssecSigned => 62, - MetricType::DaneTlsaRecordInvalid => 63, - MetricType::DaneBogusDnssecRecord => 339, + MetricType::ArcChainInvalid => 379, + MetricType::ArcSignatureInvalid => 380, + MetricType::ArcHashMismatch => 381, + MetricType::AuthSuccess => 39, + MetricType::AuthFailed => 40, + MetricType::AuthTooManyAttempts => 41, + MetricType::AuthError => 42, + MetricType::CacheMessagesSize => 348, + MetricType::CacheMessagesLargeSize => 349, + MetricType::CacheSize => 350, + MetricType::CacheSwapPending => 351, + MetricType::CacheMiss => 339, + MetricType::CacheHit => 340, + MetricType::CacheStale => 341, + MetricType::CacheEntryTooLarge => 342, + MetricType::CacheSwapHit => 343, + MetricType::CacheSwapMiss => 344, + MetricType::CacheSwapWrite => 345, + MetricType::CacheSwapError => 346, + MetricType::CalendarAlarmSent => 43, + MetricType::CalendarAlarmFailed => 44, + MetricType::CalendarItipMessageSent => 45, + MetricType::CalendarItipMessageReceived => 46, + MetricType::CalendarItipMessageError => 47, + MetricType::ClusterSubscriberError => 48, + MetricType::ClusterSubscriberDisconnected => 49, + MetricType::ClusterPublisherError => 50, + MetricType::DaneAuthenticationSuccess => 51, + MetricType::DaneAuthenticationFailure => 52, + MetricType::DaneNoCertificatesFound => 53, + MetricType::DaneCertificateParseError => 54, + MetricType::DaneTlsaRecordFetchError => 55, + MetricType::DaneTlsaRecordNotFound => 56, + MetricType::DaneTlsaRecordNotDnssecSigned => 57, + MetricType::DaneTlsaRecordInvalid => 58, + MetricType::DaneBogusDnssecRecord => 323, MetricType::DeliveryTotalTime => 2, MetricType::DeliveryAttemptTime => 3, MetricType::DeliveryActiveConnections => 22, - MetricType::DeliveryAttemptStart => 64, - MetricType::DeliveryAttemptEnd => 65, - MetricType::DeliveryCompleted => 66, - MetricType::DeliveryMxLookupFailed => 67, - MetricType::DeliveryIpLookupFailed => 68, - MetricType::DeliveryNullMx => 69, - MetricType::DeliveryGreetingFailed => 70, - MetricType::DeliveryEhloRejected => 71, - MetricType::DeliveryAuthFailed => 72, - MetricType::DeliveryMailFromRejected => 73, - MetricType::DeliveryDelivered => 74, - MetricType::DeliveryRcptToRejected => 75, - MetricType::DeliveryRcptToFailed => 76, - MetricType::DeliveryMessageRejected => 77, - MetricType::DeliveryStartTlsUnavailable => 78, - MetricType::DeliveryStartTlsError => 79, - MetricType::DeliveryStartTlsDisabled => 80, - MetricType::DeliveryImplicitTlsError => 81, - MetricType::DeliveryConcurrencyLimitExceeded => 82, - MetricType::DeliveryRateLimitExceeded => 83, - MetricType::DeliveryDoubleBounce => 84, - MetricType::DeliveryDsnSuccess => 85, - MetricType::DeliveryDsnTempFail => 86, - MetricType::DeliveryDsnPermFail => 87, - MetricType::DkimPass => 88, - MetricType::DkimNeutral => 89, - MetricType::DkimFail => 90, - MetricType::DkimPermError => 91, - MetricType::DkimTempError => 92, - MetricType::DkimNone => 93, - MetricType::DkimUnsupportedVersion => 94, - MetricType::DkimUnsupportedAlgorithm => 95, - MetricType::DkimUnsupportedCanonicalization => 96, - MetricType::DkimUnsupportedKeyType => 97, - MetricType::DkimFailedBodyHashMatch => 98, - MetricType::DkimFailedVerification => 99, - MetricType::DkimFailedAuidMatch => 100, - MetricType::DkimRevokedPublicKey => 101, - MetricType::DkimIncompatibleAlgorithms => 102, - MetricType::DkimSignatureExpired => 103, - MetricType::DkimSignatureLength => 104, - MetricType::DkimSignerNotFound => 105, - MetricType::DkimInstanceMissing => 340, - MetricType::DkimInstanceSyntax => 341, - MetricType::DkimInstanceTagMissing => 342, - MetricType::DkimInstanceNotSigned => 343, - MetricType::DkimInstanceAboveSignature => 344, - MetricType::DkimSignatureMissing => 345, - MetricType::DkimSignatureSyntax => 346, - MetricType::DkimSignatureTagMissing => 347, - MetricType::DkimSignatureTagUnexpected => 348, - MetricType::DkimSequenceGap => 349, - MetricType::DkimSequenceOverflow => 350, - MetricType::DkimChainTooLong => 367, - MetricType::DkimMailFromMismatch => 351, - MetricType::DkimRcptToMismatch => 352, - MetricType::DkimMailFromDomainMismatch => 353, - MetricType::DkimNextDomainMismatch => 354, - MetricType::DkimCustodyBreak => 368, - MetricType::DkimPublicKeyFetch => 355, - MetricType::DkimPublicKeyMissing => 356, - MetricType::DkimPublicKeyMultiple => 357, - MetricType::DkimPublicKeySyntax => 358, - MetricType::DkimPublicKeyAlgorithmMismatch => 359, - MetricType::DkimNoValidAlgorithm => 360, - MetricType::DkimHeaderHashMismatch => 361, - MetricType::DkimModified => 362, - MetricType::DkimExploded => 363, - MetricType::DmarcPass => 106, - MetricType::DmarcFail => 107, - MetricType::DmarcPermError => 108, - MetricType::DmarcTempError => 109, - MetricType::DmarcNone => 110, + MetricType::DeliveryAttemptStart => 59, + MetricType::DeliveryAttemptEnd => 60, + MetricType::DeliveryCompleted => 61, + MetricType::DeliveryMxLookupFailed => 62, + MetricType::DeliveryIpLookupFailed => 63, + MetricType::DeliveryNullMx => 64, + MetricType::DeliveryGreetingFailed => 65, + MetricType::DeliveryEhloRejected => 66, + MetricType::DeliveryAuthFailed => 67, + MetricType::DeliveryMailFromRejected => 68, + MetricType::DeliveryDelivered => 69, + MetricType::DeliveryRcptToRejected => 70, + MetricType::DeliveryRcptToFailed => 71, + MetricType::DeliveryMessageRejected => 72, + MetricType::DeliveryStartTlsUnavailable => 73, + MetricType::DeliveryStartTlsError => 74, + MetricType::DeliveryStartTlsDisabled => 75, + MetricType::DeliveryImplicitTlsError => 76, + MetricType::DeliveryConcurrencyLimitExceeded => 77, + MetricType::DeliveryRateLimitExceeded => 78, + MetricType::DeliveryDoubleBounce => 79, + MetricType::DeliveryDsnSuccess => 80, + MetricType::DeliveryDsnTempFail => 81, + MetricType::DeliveryDsnPermFail => 82, + MetricType::DkimPass => 83, + MetricType::DkimNeutral => 84, + MetricType::DkimFail => 85, + MetricType::DkimPermError => 86, + MetricType::DkimTempError => 87, + MetricType::DkimNone => 88, + MetricType::DkimSignatureInvalid => 375, + MetricType::DkimPublicKeyError => 376, + MetricType::DkimVerificationFailed => 377, + MetricType::DkimHashMismatch => 378, + MetricType::DkimSignerNotFound => 89, + MetricType::Dkim2InstanceInvalid => 365, + MetricType::Dkim2SignatureInvalid => 366, + MetricType::Dkim2ChainInvalid => 367, + MetricType::Dkim2TimestampInvalid => 368, + MetricType::Dkim2EnvelopeMismatch => 369, + MetricType::Dkim2CustodyBroken => 370, + MetricType::Dkim2PublicKeyError => 371, + MetricType::Dkim2VerificationFailed => 372, + MetricType::Dkim2HashMismatch => 373, + MetricType::Dkim2RecipeError => 374, + MetricType::Dkim2SigningFailed => 382, + MetricType::DmarcPass => 90, + MetricType::DmarcFail => 91, + MetricType::DmarcPermError => 92, + MetricType::DmarcTempError => 93, + MetricType::DmarcNone => 94, MetricType::DnsLookupTime => 11, MetricType::DnsRecordCreationFailed => 35, MetricType::DnsRecordDeletionFailed => 36, MetricType::DnsRecordLookupFailed => 37, MetricType::DnsRecordPropagationTimeout => 38, MetricType::DomainCount => 26, - MetricType::EvalError => 111, - MetricType::EvalDirectoryNotFound => 112, - MetricType::EvalStoreNotFound => 113, + MetricType::EvalError => 95, + MetricType::EvalDirectoryNotFound => 96, + MetricType::EvalStoreNotFound => 97, MetricType::HttpRequestTime => 12, MetricType::HttpActiveConnections => 17, - MetricType::HttpConnectionStart => 337, - MetricType::HttpError => 114, - MetricType::HttpRequestBody => 115, - MetricType::HttpResponseBody => 116, - MetricType::HttpXForwardedMissing => 117, + MetricType::HttpConnectionStart => 321, + MetricType::HttpError => 98, + MetricType::HttpRequestBody => 99, + MetricType::HttpResponseBody => 100, + MetricType::HttpXForwardedMissing => 101, + MetricType::HttpXForwardedInvalid => 336, MetricType::ImapRequestTime => 13, MetricType::ImapActiveConnections => 18, - MetricType::ImapConnectionStart => 118, - MetricType::ImapConnectionEnd => 119, - MetricType::IncomingReportDmarcReport => 120, - MetricType::IncomingReportDmarcReportWithWarnings => 121, - MetricType::IncomingReportTlsReport => 122, - MetricType::IncomingReportTlsReportWithWarnings => 123, - MetricType::IncomingReportAbuseReport => 124, - MetricType::IncomingReportAuthFailureReport => 125, - MetricType::IncomingReportFraudReport => 126, - MetricType::IncomingReportNotSpamReport => 127, - MetricType::IncomingReportVirusReport => 128, - MetricType::IncomingReportOtherReport => 129, - MetricType::IncomingReportMessageParseFailed => 130, - MetricType::IncomingReportDmarcParseFailed => 131, - MetricType::IncomingReportTlsRpcParseFailed => 132, - MetricType::IncomingReportArfParseFailed => 133, - MetricType::IncomingReportDecompressError => 134, - MetricType::IprevPass => 135, - MetricType::IprevFail => 136, - MetricType::IprevPermError => 137, - MetricType::IprevTempError => 138, - MetricType::IprevNone => 139, - MetricType::JmapMethodCall => 140, - MetricType::JmapInvalidArguments => 141, - MetricType::JmapRequestTooLarge => 142, - MetricType::JmapUnsupportedFilter => 143, - MetricType::JmapUnsupportedSort => 144, - MetricType::JmapUnknownMethod => 145, - MetricType::JmapForbidden => 146, - MetricType::JmapNotJson => 147, - MetricType::JmapNotRequest => 148, - MetricType::JmapWebsocketStart => 149, - MetricType::JmapWebsocketError => 150, - MetricType::LimitSizeRequest => 151, - MetricType::LimitSizeUpload => 152, - MetricType::LimitCallsIn => 153, - MetricType::LimitConcurrentRequest => 154, - MetricType::LimitConcurrentUpload => 155, - MetricType::LimitConcurrentConnection => 156, - MetricType::LimitQuota => 157, - MetricType::LimitBlobQuota => 158, - MetricType::LimitTenantQuota => 159, - MetricType::LimitTooManyRequests => 160, - MetricType::MailAuthParseError => 161, - MetricType::MailAuthMissingParameters => 162, - MetricType::MailAuthNoHeadersFound => 163, - MetricType::MailAuthCrypto => 164, - MetricType::MailAuthIo => 165, - MetricType::MailAuthBase64 => 166, - MetricType::MailAuthDnsError => 167, - MetricType::MailAuthDnsRecordNotFound => 168, - MetricType::MailAuthDnsInvalidRecordType => 169, - MetricType::MailAuthPolicyNotAligned => 170, - MetricType::ManageSieveConnectionStart => 171, - MetricType::ManageSieveConnectionEnd => 172, + MetricType::ImapConnectionStart => 102, + MetricType::ImapConnectionEnd => 103, + MetricType::IncomingReportDmarcReport => 104, + MetricType::IncomingReportDmarcReportWithWarnings => 105, + MetricType::IncomingReportTlsReport => 106, + MetricType::IncomingReportTlsReportWithWarnings => 107, + MetricType::IncomingReportAbuseReport => 108, + MetricType::IncomingReportAuthFailureReport => 109, + MetricType::IncomingReportFraudReport => 110, + MetricType::IncomingReportNotSpamReport => 111, + MetricType::IncomingReportVirusReport => 112, + MetricType::IncomingReportOtherReport => 113, + MetricType::IncomingReportMessageParseFailed => 114, + MetricType::IncomingReportDmarcParseFailed => 115, + MetricType::IncomingReportTlsRpcParseFailed => 116, + MetricType::IncomingReportArfParseFailed => 117, + MetricType::IncomingReportDecompressError => 118, + MetricType::IprevPass => 119, + MetricType::IprevFail => 120, + MetricType::IprevPermError => 121, + MetricType::IprevTempError => 122, + MetricType::IprevNone => 123, + MetricType::JmapMethodCall => 124, + MetricType::JmapInvalidArguments => 125, + MetricType::JmapRequestTooLarge => 126, + MetricType::JmapUnsupportedFilter => 127, + MetricType::JmapUnsupportedSort => 128, + MetricType::JmapUnknownMethod => 129, + MetricType::JmapForbidden => 130, + MetricType::JmapNotJson => 131, + MetricType::JmapNotRequest => 132, + MetricType::JmapWebsocketStart => 133, + MetricType::JmapWebsocketError => 134, + MetricType::LimitSizeRequest => 135, + MetricType::LimitSizeUpload => 136, + MetricType::LimitCallsIn => 137, + MetricType::LimitConcurrentRequest => 138, + MetricType::LimitConcurrentUpload => 139, + MetricType::LimitConcurrentConnection => 140, + MetricType::LimitQuota => 141, + MetricType::LimitBlobQuota => 142, + MetricType::LimitTooManyRequests => 144, + MetricType::LimitMemoryBudget => 338, + MetricType::MailAuthParseError => 145, + MetricType::MailAuthMissingParameters => 146, + MetricType::MailAuthNoHeadersFound => 147, + MetricType::MailAuthCrypto => 148, + MetricType::MailAuthIo => 149, + MetricType::MailAuthBase64 => 150, + MetricType::MailAuthDnsError => 151, + MetricType::MailAuthDnsRecordNotFound => 152, + MetricType::MailAuthDnsInvalidRecordType => 153, + MetricType::MailAuthPolicyNotAligned => 154, + MetricType::ManageSieveConnectionStart => 155, + MetricType::ManageSieveConnectionEnd => 156, MetricType::MessageSize => 4, MetricType::MessageAuthenticatedSize => 5, MetricType::MessageIngestTime => 0, - MetricType::MessageIngestIndexTime => 1, - MetricType::MessageIngestHam => 173, - MetricType::MessageIngestSpam => 174, - MetricType::MessageIngestImapAppend => 175, - MetricType::MessageIngestJmapAppend => 176, - MetricType::MessageIngestDuplicate => 177, - MetricType::MessageIngestError => 178, - MetricType::MessageIngestSearchIndex => 179, - MetricType::MilterActionAccept => 180, - MetricType::MilterActionDiscard => 181, - MetricType::MilterActionReject => 182, - MetricType::MilterActionTempFail => 183, - MetricType::MilterActionReplyCode => 184, - MetricType::MilterActionConnectionFailure => 185, - MetricType::MilterActionShutdown => 186, - MetricType::MtaHookActionAccept => 187, - MetricType::MtaHookActionDiscard => 188, - MetricType::MtaHookActionReject => 189, - MetricType::MtaHookActionQuarantine => 190, - MetricType::MtaHookError => 191, - MetricType::MtaStsAuthorized => 192, - MetricType::MtaStsNotAuthorized => 193, - MetricType::MtaStsInvalidPolicy => 194, - MetricType::NetworkTimeout => 195, + MetricType::MessageIngestHam => 157, + MetricType::MessageIngestSpam => 158, + MetricType::MessageIngestImapAppend => 159, + MetricType::MessageIngestJmapAppend => 160, + MetricType::MessageIngestDuplicate => 161, + MetricType::MessageIngestError => 162, + MetricType::MilterActionAccept => 164, + MetricType::MilterActionDiscard => 165, + MetricType::MilterActionReject => 166, + MetricType::MilterActionTempFail => 167, + MetricType::MilterActionReplyCode => 168, + MetricType::MilterActionConnectionFailure => 169, + MetricType::MilterActionShutdown => 170, + MetricType::MtaHookActionAccept => 171, + MetricType::MtaHookActionDiscard => 172, + MetricType::MtaHookActionReject => 173, + MetricType::MtaHookActionQuarantine => 174, + MetricType::MtaHookError => 175, + MetricType::MtaStsAuthorized => 176, + MetricType::MtaStsNotAuthorized => 177, + MetricType::MtaStsInvalidPolicy => 178, + MetricType::NetworkTimeout => 179, MetricType::OutgoingReportSize => 6, - MetricType::OutgoingReportSpfReport => 196, - MetricType::OutgoingReportSpfRateLimited => 197, - MetricType::OutgoingReportDkimReport => 198, - MetricType::OutgoingReportDkimRateLimited => 199, - MetricType::OutgoingReportDmarcReport => 200, - MetricType::OutgoingReportDmarcRateLimited => 201, - MetricType::OutgoingReportDmarcAggregateReport => 202, - MetricType::OutgoingReportTlsAggregate => 203, - MetricType::OutgoingReportHttpSubmission => 204, - MetricType::OutgoingReportUnauthorizedReportingAddress => 205, - MetricType::OutgoingReportReportingAddressValidationError => 206, - MetricType::OutgoingReportNotFound => 207, - MetricType::OutgoingReportSubmissionError => 208, - MetricType::OutgoingReportNoRecipientsFound => 209, + MetricType::OutgoingReportSpfReport => 180, + MetricType::OutgoingReportSpfRateLimited => 181, + MetricType::OutgoingReportDkimReport => 182, + MetricType::OutgoingReportDkimRateLimited => 183, + MetricType::OutgoingReportDmarcReport => 184, + MetricType::OutgoingReportDmarcRateLimited => 185, + MetricType::OutgoingReportDmarcAggregateReport => 186, + MetricType::OutgoingReportTlsAggregate => 187, + MetricType::OutgoingReportHttpSubmission => 188, + MetricType::OutgoingReportUnauthorizedReportingAddress => 189, + MetricType::OutgoingReportReportingAddressValidationError => 190, + MetricType::OutgoingReportNotFound => 191, + MetricType::OutgoingReportSubmissionError => 192, + MetricType::OutgoingReportNoRecipientsFound => 193, MetricType::Pop3RequestTime => 14, MetricType::Pop3ActiveConnections => 19, - MetricType::Pop3ConnectionStart => 210, - MetricType::Pop3ConnectionEnd => 211, - MetricType::PushSubscriptionSuccess => 213, - MetricType::PushSubscriptionError => 214, - MetricType::PushSubscriptionNotFound => 215, + MetricType::Pop3ConnectionStart => 194, + MetricType::Pop3ConnectionEnd => 195, + MetricType::PushSubscriptionSuccess => 197, + MetricType::PushSubscriptionError => 198, + MetricType::PushSubscriptionNotFound => 199, MetricType::QueueCount => 24, - MetricType::QueueMessageQueued => 216, - MetricType::QueueAuthenticatedMessageQueued => 217, - MetricType::QueueReportQueued => 218, - MetricType::QueueDsnQueued => 219, - MetricType::QueueAutogeneratedQueued => 220, - MetricType::QueueRescheduled => 221, - MetricType::QueueBlobNotFound => 222, - MetricType::QueueRateLimitExceeded => 223, - MetricType::QueueConcurrencyLimitExceeded => 224, - MetricType::QueueQuotaExceeded => 225, - MetricType::ResourceNotFound => 226, - MetricType::ResourceBadParameters => 227, - MetricType::ResourceError => 228, - MetricType::SecurityAuthenticationBan => 229, - MetricType::SecurityAbuseBan => 230, - MetricType::SecurityScanBan => 231, - MetricType::SecurityLoiterBan => 232, - MetricType::SecurityIpBlocked => 233, - MetricType::SecurityUnauthorized => 234, + MetricType::QueueMessageQueued => 200, + MetricType::QueueAuthenticatedMessageQueued => 201, + MetricType::QueueReportQueued => 202, + MetricType::QueueDsnQueued => 203, + MetricType::QueueAutogeneratedQueued => 204, + MetricType::QueueRescheduled => 205, + MetricType::QueueBlobNotFound => 206, + MetricType::QueueRateLimitExceeded => 207, + MetricType::QueueConcurrencyLimitExceeded => 208, + MetricType::QueueQuotaExceeded => 209, + MetricType::QueueBackPressure => 347, + MetricType::ResourceNotFound => 210, + MetricType::ResourceBadParameters => 211, + MetricType::ResourceError => 212, + MetricType::SearchIndexIndexTime => 1, + MetricType::SearchIndexPendingEmail => 354, + MetricType::SearchIndexPendingCalendar => 355, + MetricType::SearchIndexPendingContacts => 356, + MetricType::SearchIndexPendingFile => 357, + MetricType::SearchIndexPartitionRetry => 360, + MetricType::SearchIndexBatchIndexed => 163, + MetricType::SearchIndexDocumentTruncated => 359, + MetricType::SecurityAuthenticationBan => 213, + MetricType::SecurityAbuseBan => 214, + MetricType::SecurityScanBan => 215, + MetricType::SecurityLoiterBan => 216, + MetricType::SecurityIpBlocked => 217, + MetricType::SecurityUnauthorized => 218, MetricType::ServerMemory => 23, - MetricType::ServerThreadError => 235, + MetricType::ServerBufferedMemorySmtp => 352, + MetricType::ServerBufferedMemoryRead => 353, + MetricType::ServerThreadError => 219, MetricType::SieveRequestTime => 16, MetricType::SieveActiveConnections => 21, - MetricType::SieveActionAccept => 236, - MetricType::SieveActionAcceptReplace => 237, - MetricType::SieveActionDiscard => 238, - MetricType::SieveActionReject => 239, - MetricType::SieveSendMessage => 240, - MetricType::SieveMessageTooLarge => 241, - MetricType::SieveRuntimeError => 242, - MetricType::SieveUnexpectedError => 243, - MetricType::SieveNotSupported => 244, - MetricType::SieveQuotaExceeded => 245, + MetricType::SieveActionAccept => 220, + MetricType::SieveActionAcceptReplace => 221, + MetricType::SieveActionDiscard => 222, + MetricType::SieveActionReject => 223, + MetricType::SieveSendMessage => 224, + MetricType::SieveMessageTooLarge => 225, + MetricType::SieveRuntimeError => 226, + MetricType::SieveUnexpectedError => 227, + MetricType::SieveNotSupported => 228, + MetricType::SieveQuotaExceeded => 229, MetricType::SmtpRequestTime => 15, MetricType::SmtpActiveConnections => 20, - MetricType::SmtpConnectionStart => 246, - MetricType::SmtpConnectionEnd => 247, - MetricType::SmtpError => 248, - MetricType::SmtpConcurrencyLimitExceeded => 249, - MetricType::SmtpTransferLimitExceeded => 250, - MetricType::SmtpRateLimitExceeded => 251, - MetricType::SmtpTimeLimitExceeded => 252, - MetricType::SmtpMessageParseFailed => 253, - MetricType::SmtpMessageTooLarge => 254, - MetricType::SmtpLoopDetected => 255, - MetricType::SmtpDkimPass => 256, - MetricType::SmtpDkimFail => 257, - MetricType::SmtpDkim2Pass => 364, - MetricType::SmtpDkim2Fail => 365, - MetricType::SmtpDkim2DsnDiscarded => 366, - MetricType::SmtpArcPass => 258, - MetricType::SmtpArcFail => 259, - MetricType::SmtpSpfEhloPass => 260, - MetricType::SmtpSpfEhloFail => 261, - MetricType::SmtpSpfFromPass => 262, - MetricType::SmtpSpfFromFail => 263, - MetricType::SmtpDmarcPass => 264, - MetricType::SmtpDmarcFail => 265, - MetricType::SmtpIprevPass => 266, - MetricType::SmtpIprevFail => 267, - MetricType::SmtpTooManyMessages => 268, - MetricType::SmtpInvalidEhlo => 269, - MetricType::SmtpDidNotSayEhlo => 270, - MetricType::SmtpMailFromUnauthenticated => 271, - MetricType::SmtpMailFromUnauthorized => 272, - MetricType::SmtpMailFromMissing => 273, - MetricType::SmtpMultipleMailFrom => 274, - MetricType::SmtpMailboxDoesNotExist => 275, - MetricType::SmtpRelayNotAllowed => 276, - MetricType::SmtpRcptToDuplicate => 277, - MetricType::SmtpRcptToMissing => 278, - MetricType::SmtpTooManyRecipients => 279, - MetricType::SmtpTooManyInvalidRcpt => 280, - MetricType::SmtpAuthMechanismNotSupported => 281, - MetricType::SmtpAuthExchangeTooLong => 282, - MetricType::SmtpCommandNotImplemented => 283, - MetricType::SmtpInvalidCommand => 284, - MetricType::SmtpSyntaxError => 285, - MetricType::SmtpRequestTooLarge => 286, - MetricType::SpamPyzorError => 287, - MetricType::SpamDnsblError => 288, - MetricType::SpamTrainCompleted => 289, - MetricType::SpamTrainSampleAdded => 290, - MetricType::SpamClassify => 291, - MetricType::SpamModelNotReady => 292, - MetricType::SpfPass => 293, - MetricType::SpfFail => 294, - MetricType::SpfSoftFail => 295, - MetricType::SpfNeutral => 296, - MetricType::SpfTempError => 297, - MetricType::SpfPermError => 298, - MetricType::SpfNone => 299, + MetricType::SmtpConnectionStart => 230, + MetricType::SmtpConnectionEnd => 231, + MetricType::SmtpError => 232, + MetricType::SmtpConcurrencyLimitExceeded => 233, + MetricType::SmtpTransferLimitExceeded => 234, + MetricType::SmtpRateLimitExceeded => 235, + MetricType::SmtpTimeLimitExceeded => 236, + MetricType::SmtpMessageParseFailed => 237, + MetricType::SmtpMessageTooLarge => 238, + MetricType::SmtpLoopDetected => 239, + MetricType::SmtpMultipleFromDomains => 337, + MetricType::SmtpDkimPass => 240, + MetricType::SmtpDkimFail => 241, + MetricType::SmtpDkim2Pass => 324, + MetricType::SmtpDkim2Fail => 325, + MetricType::SmtpDkim2DsnRejected => 326, + MetricType::SmtpArcPass => 242, + MetricType::SmtpArcFail => 243, + MetricType::SmtpSpfEhloPass => 244, + MetricType::SmtpSpfEhloFail => 245, + MetricType::SmtpSpfFromPass => 246, + MetricType::SmtpSpfFromFail => 247, + MetricType::SmtpDmarcPass => 248, + MetricType::SmtpDmarcFail => 249, + MetricType::SmtpIprevPass => 250, + MetricType::SmtpIprevFail => 251, + MetricType::SmtpTooManyMessages => 252, + MetricType::SmtpInvalidEhlo => 253, + MetricType::SmtpDidNotSayEhlo => 254, + MetricType::SmtpMailFromUnauthenticated => 255, + MetricType::SmtpMailFromUnauthorized => 256, + MetricType::SmtpMailFromMissing => 257, + MetricType::SmtpMultipleMailFrom => 258, + MetricType::SmtpMailboxDoesNotExist => 259, + MetricType::SmtpRelayNotAllowed => 260, + MetricType::SmtpRcptToDuplicate => 261, + MetricType::SmtpRcptToMissing => 262, + MetricType::SmtpTooManyRecipients => 263, + MetricType::SmtpTooManyInvalidRcpt => 264, + MetricType::SmtpAuthMechanismNotSupported => 265, + MetricType::SmtpAuthExchangeTooLong => 266, + MetricType::SmtpCommandNotImplemented => 267, + MetricType::SmtpInvalidCommand => 268, + MetricType::SmtpSyntaxError => 269, + MetricType::SmtpRequestTooLarge => 270, + MetricType::SpamPyzorError => 271, + MetricType::SpamDnsblError => 272, + MetricType::SpamTrainCompleted => 273, + MetricType::SpamTrainSampleAdded => 274, + MetricType::SpamClassify => 275, + MetricType::SpamModelNotReady => 276, + MetricType::SpfPass => 277, + MetricType::SpfFail => 278, + MetricType::SpfSoftFail => 279, + MetricType::SpfNeutral => 280, + MetricType::SpfTempError => 281, + MetricType::SpfPermError => 282, + MetricType::SpfNone => 283, MetricType::StoreDataReadTime => 7, MetricType::StoreDataWriteTime => 8, MetricType::StoreBlobReadTime => 9, MetricType::StoreBlobWriteTime => 10, - MetricType::StoreAssertValueFailed => 300, - MetricType::StoreFoundationdbError => 301, - MetricType::StoreMysqlError => 302, - MetricType::StorePostgresqlError => 303, - MetricType::StoreRocksdbError => 304, - MetricType::StoreSqliteError => 305, - MetricType::StoreLdapError => 306, - MetricType::StoreElasticsearchError => 307, - MetricType::StoreRedisError => 308, - MetricType::StoreS3Error => 309, - MetricType::StoreAzureError => 310, - MetricType::StoreFilesystemError => 311, - MetricType::StorePoolError => 312, - MetricType::StoreDataCorruption => 313, - MetricType::StoreDecompressError => 314, - MetricType::StoreDeserializeError => 315, - MetricType::StoreNotFound => 316, - MetricType::StoreNotConfigured => 317, - MetricType::StoreNotSupported => 318, - MetricType::StoreUnexpectedError => 319, - MetricType::StoreCryptoError => 320, - MetricType::StoreHttpStoreError => 321, - MetricType::StoreBlobMissingMarker => 322, - MetricType::StoreDataWrite => 323, - MetricType::StoreDataIterate => 324, - MetricType::StoreBlobRead => 325, - MetricType::StoreBlobWrite => 326, - MetricType::StoreBlobDelete => 327, - MetricType::TaskManagerBlobNotFound => 328, - MetricType::TaskManagerMetadataNotFound => 329, - MetricType::TelemetryAlertEvent => 338, - MetricType::TelemetryAlertMessage => 212, - MetricType::TelemetryLogError => 330, - MetricType::TelemetryWebhookError => 331, - MetricType::TelemetryOtelExporterError => 332, - MetricType::TelemetryOtelMetricsExporterError => 333, - MetricType::TelemetryPrometheusExporterError => 334, - MetricType::TelemetryJournalError => 335, - MetricType::TlsHandshakeError => 336, + MetricType::StoreAssertValueFailed => 284, + MetricType::StoreFoundationdbError => 285, + MetricType::StoreMysqlError => 286, + MetricType::StorePostgresqlError => 287, + MetricType::StoreRocksdbError => 288, + MetricType::StoreSqliteError => 289, + MetricType::StoreLdapError => 290, + MetricType::StoreElasticsearchError => 291, + MetricType::StoreRedisError => 292, + MetricType::StoreS3Error => 293, + MetricType::StoreAzureError => 294, + MetricType::StoreFilesystemError => 295, + MetricType::StorePoolError => 296, + MetricType::StoreDataCorruption => 297, + MetricType::StoreDecompressError => 298, + MetricType::StoreDeserializeError => 299, + MetricType::StoreNotFound => 300, + MetricType::StoreNotConfigured => 301, + MetricType::StoreNotSupported => 302, + MetricType::StoreUnexpectedError => 303, + MetricType::StoreCryptoError => 304, + MetricType::StoreHttpStoreError => 305, + MetricType::StoreBlobMissingMarker => 306, + MetricType::StoreDataWrite => 307, + MetricType::StoreDataIterate => 308, + MetricType::StoreBlobRead => 309, + MetricType::StoreBlobWrite => 310, + MetricType::StoreBlobDelete => 311, + MetricType::TaskManagerPending => 363, + MetricType::TaskManagerDue => 364, + MetricType::TaskManagerTaskFailed => 361, + MetricType::TaskManagerTaskRetry => 362, + MetricType::TaskManagerTaskStalled => 383, + MetricType::TaskManagerTaskExpired => 384, + MetricType::TaskManagerBlobNotFound => 312, + MetricType::TaskManagerMetadataNotFound => 313, + MetricType::TelemetryLogError => 314, + MetricType::TelemetryWebhookError => 315, + MetricType::TelemetryOtelExporterError => 316, + MetricType::TelemetryOtelMetricsExporterError => 317, + MetricType::TelemetryPrometheusExporterError => 318, + MetricType::TelemetryJournalError => 319, + MetricType::TelemetryEventsDropped => 327, + MetricType::TlsHandshakeError => 320, MetricType::UserCount => 25, } } @@ -5970,366 +5988,370 @@ impl MetricType { 32 => Some(MetricType::AcmeTlsAlpnError), 33 => Some(MetricType::AcmeTokenNotFound), 34 => Some(MetricType::AcmeError), - 39 => Some(MetricType::ArcChainTooLong), - 40 => Some(MetricType::ArcInvalidInstance), - 41 => Some(MetricType::ArcInvalidCv), - 42 => Some(MetricType::ArcHasHeaderTag), - 43 => Some(MetricType::ArcBrokenChain), - 44 => Some(MetricType::AuthSuccess), - 45 => Some(MetricType::AuthFailed), - 46 => Some(MetricType::AuthTooManyAttempts), - 47 => Some(MetricType::AuthError), - 48 => Some(MetricType::CalendarAlarmSent), - 49 => Some(MetricType::CalendarAlarmFailed), - 50 => Some(MetricType::CalendarItipMessageSent), - 51 => Some(MetricType::CalendarItipMessageReceived), - 52 => Some(MetricType::CalendarItipMessageError), - 53 => Some(MetricType::ClusterSubscriberError), - 54 => Some(MetricType::ClusterSubscriberDisconnected), - 55 => Some(MetricType::ClusterPublisherError), - 56 => Some(MetricType::DaneAuthenticationSuccess), - 57 => Some(MetricType::DaneAuthenticationFailure), - 58 => Some(MetricType::DaneNoCertificatesFound), - 59 => Some(MetricType::DaneCertificateParseError), - 60 => Some(MetricType::DaneTlsaRecordFetchError), - 61 => Some(MetricType::DaneTlsaRecordNotFound), - 62 => Some(MetricType::DaneTlsaRecordNotDnssecSigned), - 63 => Some(MetricType::DaneTlsaRecordInvalid), - 339 => Some(MetricType::DaneBogusDnssecRecord), + 379 => Some(MetricType::ArcChainInvalid), + 380 => Some(MetricType::ArcSignatureInvalid), + 381 => Some(MetricType::ArcHashMismatch), + 39 => Some(MetricType::AuthSuccess), + 40 => Some(MetricType::AuthFailed), + 41 => Some(MetricType::AuthTooManyAttempts), + 42 => Some(MetricType::AuthError), + 348 => Some(MetricType::CacheMessagesSize), + 349 => Some(MetricType::CacheMessagesLargeSize), + 350 => Some(MetricType::CacheSize), + 351 => Some(MetricType::CacheSwapPending), + 339 => Some(MetricType::CacheMiss), + 340 => Some(MetricType::CacheHit), + 341 => Some(MetricType::CacheStale), + 342 => Some(MetricType::CacheEntryTooLarge), + 343 => Some(MetricType::CacheSwapHit), + 344 => Some(MetricType::CacheSwapMiss), + 345 => Some(MetricType::CacheSwapWrite), + 346 => Some(MetricType::CacheSwapError), + 43 => Some(MetricType::CalendarAlarmSent), + 44 => Some(MetricType::CalendarAlarmFailed), + 45 => Some(MetricType::CalendarItipMessageSent), + 46 => Some(MetricType::CalendarItipMessageReceived), + 47 => Some(MetricType::CalendarItipMessageError), + 48 => Some(MetricType::ClusterSubscriberError), + 49 => Some(MetricType::ClusterSubscriberDisconnected), + 50 => Some(MetricType::ClusterPublisherError), + 51 => Some(MetricType::DaneAuthenticationSuccess), + 52 => Some(MetricType::DaneAuthenticationFailure), + 53 => Some(MetricType::DaneNoCertificatesFound), + 54 => Some(MetricType::DaneCertificateParseError), + 55 => Some(MetricType::DaneTlsaRecordFetchError), + 56 => Some(MetricType::DaneTlsaRecordNotFound), + 57 => Some(MetricType::DaneTlsaRecordNotDnssecSigned), + 58 => Some(MetricType::DaneTlsaRecordInvalid), + 323 => Some(MetricType::DaneBogusDnssecRecord), 2 => Some(MetricType::DeliveryTotalTime), 3 => Some(MetricType::DeliveryAttemptTime), 22 => Some(MetricType::DeliveryActiveConnections), - 64 => Some(MetricType::DeliveryAttemptStart), - 65 => Some(MetricType::DeliveryAttemptEnd), - 66 => Some(MetricType::DeliveryCompleted), - 67 => Some(MetricType::DeliveryMxLookupFailed), - 68 => Some(MetricType::DeliveryIpLookupFailed), - 69 => Some(MetricType::DeliveryNullMx), - 70 => Some(MetricType::DeliveryGreetingFailed), - 71 => Some(MetricType::DeliveryEhloRejected), - 72 => Some(MetricType::DeliveryAuthFailed), - 73 => Some(MetricType::DeliveryMailFromRejected), - 74 => Some(MetricType::DeliveryDelivered), - 75 => Some(MetricType::DeliveryRcptToRejected), - 76 => Some(MetricType::DeliveryRcptToFailed), - 77 => Some(MetricType::DeliveryMessageRejected), - 78 => Some(MetricType::DeliveryStartTlsUnavailable), - 79 => Some(MetricType::DeliveryStartTlsError), - 80 => Some(MetricType::DeliveryStartTlsDisabled), - 81 => Some(MetricType::DeliveryImplicitTlsError), - 82 => Some(MetricType::DeliveryConcurrencyLimitExceeded), - 83 => Some(MetricType::DeliveryRateLimitExceeded), - 84 => Some(MetricType::DeliveryDoubleBounce), - 85 => Some(MetricType::DeliveryDsnSuccess), - 86 => Some(MetricType::DeliveryDsnTempFail), - 87 => Some(MetricType::DeliveryDsnPermFail), - 88 => Some(MetricType::DkimPass), - 89 => Some(MetricType::DkimNeutral), - 90 => Some(MetricType::DkimFail), - 91 => Some(MetricType::DkimPermError), - 92 => Some(MetricType::DkimTempError), - 93 => Some(MetricType::DkimNone), - 94 => Some(MetricType::DkimUnsupportedVersion), - 95 => Some(MetricType::DkimUnsupportedAlgorithm), - 96 => Some(MetricType::DkimUnsupportedCanonicalization), - 97 => Some(MetricType::DkimUnsupportedKeyType), - 98 => Some(MetricType::DkimFailedBodyHashMatch), - 99 => Some(MetricType::DkimFailedVerification), - 100 => Some(MetricType::DkimFailedAuidMatch), - 101 => Some(MetricType::DkimRevokedPublicKey), - 102 => Some(MetricType::DkimIncompatibleAlgorithms), - 103 => Some(MetricType::DkimSignatureExpired), - 104 => Some(MetricType::DkimSignatureLength), - 105 => Some(MetricType::DkimSignerNotFound), - 340 => Some(MetricType::DkimInstanceMissing), - 341 => Some(MetricType::DkimInstanceSyntax), - 342 => Some(MetricType::DkimInstanceTagMissing), - 343 => Some(MetricType::DkimInstanceNotSigned), - 344 => Some(MetricType::DkimInstanceAboveSignature), - 345 => Some(MetricType::DkimSignatureMissing), - 346 => Some(MetricType::DkimSignatureSyntax), - 347 => Some(MetricType::DkimSignatureTagMissing), - 348 => Some(MetricType::DkimSignatureTagUnexpected), - 349 => Some(MetricType::DkimSequenceGap), - 350 => Some(MetricType::DkimSequenceOverflow), - 367 => Some(MetricType::DkimChainTooLong), - 351 => Some(MetricType::DkimMailFromMismatch), - 352 => Some(MetricType::DkimRcptToMismatch), - 353 => Some(MetricType::DkimMailFromDomainMismatch), - 354 => Some(MetricType::DkimNextDomainMismatch), - 368 => Some(MetricType::DkimCustodyBreak), - 355 => Some(MetricType::DkimPublicKeyFetch), - 356 => Some(MetricType::DkimPublicKeyMissing), - 357 => Some(MetricType::DkimPublicKeyMultiple), - 358 => Some(MetricType::DkimPublicKeySyntax), - 359 => Some(MetricType::DkimPublicKeyAlgorithmMismatch), - 360 => Some(MetricType::DkimNoValidAlgorithm), - 361 => Some(MetricType::DkimHeaderHashMismatch), - 362 => Some(MetricType::DkimModified), - 363 => Some(MetricType::DkimExploded), - 106 => Some(MetricType::DmarcPass), - 107 => Some(MetricType::DmarcFail), - 108 => Some(MetricType::DmarcPermError), - 109 => Some(MetricType::DmarcTempError), - 110 => Some(MetricType::DmarcNone), + 59 => Some(MetricType::DeliveryAttemptStart), + 60 => Some(MetricType::DeliveryAttemptEnd), + 61 => Some(MetricType::DeliveryCompleted), + 62 => Some(MetricType::DeliveryMxLookupFailed), + 63 => Some(MetricType::DeliveryIpLookupFailed), + 64 => Some(MetricType::DeliveryNullMx), + 65 => Some(MetricType::DeliveryGreetingFailed), + 66 => Some(MetricType::DeliveryEhloRejected), + 67 => Some(MetricType::DeliveryAuthFailed), + 68 => Some(MetricType::DeliveryMailFromRejected), + 69 => Some(MetricType::DeliveryDelivered), + 70 => Some(MetricType::DeliveryRcptToRejected), + 71 => Some(MetricType::DeliveryRcptToFailed), + 72 => Some(MetricType::DeliveryMessageRejected), + 73 => Some(MetricType::DeliveryStartTlsUnavailable), + 74 => Some(MetricType::DeliveryStartTlsError), + 75 => Some(MetricType::DeliveryStartTlsDisabled), + 76 => Some(MetricType::DeliveryImplicitTlsError), + 77 => Some(MetricType::DeliveryConcurrencyLimitExceeded), + 78 => Some(MetricType::DeliveryRateLimitExceeded), + 79 => Some(MetricType::DeliveryDoubleBounce), + 80 => Some(MetricType::DeliveryDsnSuccess), + 81 => Some(MetricType::DeliveryDsnTempFail), + 82 => Some(MetricType::DeliveryDsnPermFail), + 83 => Some(MetricType::DkimPass), + 84 => Some(MetricType::DkimNeutral), + 85 => Some(MetricType::DkimFail), + 86 => Some(MetricType::DkimPermError), + 87 => Some(MetricType::DkimTempError), + 88 => Some(MetricType::DkimNone), + 375 => Some(MetricType::DkimSignatureInvalid), + 376 => Some(MetricType::DkimPublicKeyError), + 377 => Some(MetricType::DkimVerificationFailed), + 378 => Some(MetricType::DkimHashMismatch), + 89 => Some(MetricType::DkimSignerNotFound), + 365 => Some(MetricType::Dkim2InstanceInvalid), + 366 => Some(MetricType::Dkim2SignatureInvalid), + 367 => Some(MetricType::Dkim2ChainInvalid), + 368 => Some(MetricType::Dkim2TimestampInvalid), + 369 => Some(MetricType::Dkim2EnvelopeMismatch), + 370 => Some(MetricType::Dkim2CustodyBroken), + 371 => Some(MetricType::Dkim2PublicKeyError), + 372 => Some(MetricType::Dkim2VerificationFailed), + 373 => Some(MetricType::Dkim2HashMismatch), + 374 => Some(MetricType::Dkim2RecipeError), + 382 => Some(MetricType::Dkim2SigningFailed), + 90 => Some(MetricType::DmarcPass), + 91 => Some(MetricType::DmarcFail), + 92 => Some(MetricType::DmarcPermError), + 93 => Some(MetricType::DmarcTempError), + 94 => Some(MetricType::DmarcNone), 11 => Some(MetricType::DnsLookupTime), 35 => Some(MetricType::DnsRecordCreationFailed), 36 => Some(MetricType::DnsRecordDeletionFailed), 37 => Some(MetricType::DnsRecordLookupFailed), 38 => Some(MetricType::DnsRecordPropagationTimeout), 26 => Some(MetricType::DomainCount), - 111 => Some(MetricType::EvalError), - 112 => Some(MetricType::EvalDirectoryNotFound), - 113 => Some(MetricType::EvalStoreNotFound), + 95 => Some(MetricType::EvalError), + 96 => Some(MetricType::EvalDirectoryNotFound), + 97 => Some(MetricType::EvalStoreNotFound), 12 => Some(MetricType::HttpRequestTime), 17 => Some(MetricType::HttpActiveConnections), - 337 => Some(MetricType::HttpConnectionStart), - 114 => Some(MetricType::HttpError), - 115 => Some(MetricType::HttpRequestBody), - 116 => Some(MetricType::HttpResponseBody), - 117 => Some(MetricType::HttpXForwardedMissing), + 321 => Some(MetricType::HttpConnectionStart), + 98 => Some(MetricType::HttpError), + 99 => Some(MetricType::HttpRequestBody), + 100 => Some(MetricType::HttpResponseBody), + 101 => Some(MetricType::HttpXForwardedMissing), + 336 => Some(MetricType::HttpXForwardedInvalid), 13 => Some(MetricType::ImapRequestTime), 18 => Some(MetricType::ImapActiveConnections), - 118 => Some(MetricType::ImapConnectionStart), - 119 => Some(MetricType::ImapConnectionEnd), - 120 => Some(MetricType::IncomingReportDmarcReport), - 121 => Some(MetricType::IncomingReportDmarcReportWithWarnings), - 122 => Some(MetricType::IncomingReportTlsReport), - 123 => Some(MetricType::IncomingReportTlsReportWithWarnings), - 124 => Some(MetricType::IncomingReportAbuseReport), - 125 => Some(MetricType::IncomingReportAuthFailureReport), - 126 => Some(MetricType::IncomingReportFraudReport), - 127 => Some(MetricType::IncomingReportNotSpamReport), - 128 => Some(MetricType::IncomingReportVirusReport), - 129 => Some(MetricType::IncomingReportOtherReport), - 130 => Some(MetricType::IncomingReportMessageParseFailed), - 131 => Some(MetricType::IncomingReportDmarcParseFailed), - 132 => Some(MetricType::IncomingReportTlsRpcParseFailed), - 133 => Some(MetricType::IncomingReportArfParseFailed), - 134 => Some(MetricType::IncomingReportDecompressError), - 135 => Some(MetricType::IprevPass), - 136 => Some(MetricType::IprevFail), - 137 => Some(MetricType::IprevPermError), - 138 => Some(MetricType::IprevTempError), - 139 => Some(MetricType::IprevNone), - 140 => Some(MetricType::JmapMethodCall), - 141 => Some(MetricType::JmapInvalidArguments), - 142 => Some(MetricType::JmapRequestTooLarge), - 143 => Some(MetricType::JmapUnsupportedFilter), - 144 => Some(MetricType::JmapUnsupportedSort), - 145 => Some(MetricType::JmapUnknownMethod), - 146 => Some(MetricType::JmapForbidden), - 147 => Some(MetricType::JmapNotJson), - 148 => Some(MetricType::JmapNotRequest), - 149 => Some(MetricType::JmapWebsocketStart), - 150 => Some(MetricType::JmapWebsocketError), - 151 => Some(MetricType::LimitSizeRequest), - 152 => Some(MetricType::LimitSizeUpload), - 153 => Some(MetricType::LimitCallsIn), - 154 => Some(MetricType::LimitConcurrentRequest), - 155 => Some(MetricType::LimitConcurrentUpload), - 156 => Some(MetricType::LimitConcurrentConnection), - 157 => Some(MetricType::LimitQuota), - 158 => Some(MetricType::LimitBlobQuota), - 159 => Some(MetricType::LimitTenantQuota), - 160 => Some(MetricType::LimitTooManyRequests), - 161 => Some(MetricType::MailAuthParseError), - 162 => Some(MetricType::MailAuthMissingParameters), - 163 => Some(MetricType::MailAuthNoHeadersFound), - 164 => Some(MetricType::MailAuthCrypto), - 165 => Some(MetricType::MailAuthIo), - 166 => Some(MetricType::MailAuthBase64), - 167 => Some(MetricType::MailAuthDnsError), - 168 => Some(MetricType::MailAuthDnsRecordNotFound), - 169 => Some(MetricType::MailAuthDnsInvalidRecordType), - 170 => Some(MetricType::MailAuthPolicyNotAligned), - 171 => Some(MetricType::ManageSieveConnectionStart), - 172 => Some(MetricType::ManageSieveConnectionEnd), + 102 => Some(MetricType::ImapConnectionStart), + 103 => Some(MetricType::ImapConnectionEnd), + 104 => Some(MetricType::IncomingReportDmarcReport), + 105 => Some(MetricType::IncomingReportDmarcReportWithWarnings), + 106 => Some(MetricType::IncomingReportTlsReport), + 107 => Some(MetricType::IncomingReportTlsReportWithWarnings), + 108 => Some(MetricType::IncomingReportAbuseReport), + 109 => Some(MetricType::IncomingReportAuthFailureReport), + 110 => Some(MetricType::IncomingReportFraudReport), + 111 => Some(MetricType::IncomingReportNotSpamReport), + 112 => Some(MetricType::IncomingReportVirusReport), + 113 => Some(MetricType::IncomingReportOtherReport), + 114 => Some(MetricType::IncomingReportMessageParseFailed), + 115 => Some(MetricType::IncomingReportDmarcParseFailed), + 116 => Some(MetricType::IncomingReportTlsRpcParseFailed), + 117 => Some(MetricType::IncomingReportArfParseFailed), + 118 => Some(MetricType::IncomingReportDecompressError), + 119 => Some(MetricType::IprevPass), + 120 => Some(MetricType::IprevFail), + 121 => Some(MetricType::IprevPermError), + 122 => Some(MetricType::IprevTempError), + 123 => Some(MetricType::IprevNone), + 124 => Some(MetricType::JmapMethodCall), + 125 => Some(MetricType::JmapInvalidArguments), + 126 => Some(MetricType::JmapRequestTooLarge), + 127 => Some(MetricType::JmapUnsupportedFilter), + 128 => Some(MetricType::JmapUnsupportedSort), + 129 => Some(MetricType::JmapUnknownMethod), + 130 => Some(MetricType::JmapForbidden), + 131 => Some(MetricType::JmapNotJson), + 132 => Some(MetricType::JmapNotRequest), + 133 => Some(MetricType::JmapWebsocketStart), + 134 => Some(MetricType::JmapWebsocketError), + 135 => Some(MetricType::LimitSizeRequest), + 136 => Some(MetricType::LimitSizeUpload), + 137 => Some(MetricType::LimitCallsIn), + 138 => Some(MetricType::LimitConcurrentRequest), + 139 => Some(MetricType::LimitConcurrentUpload), + 140 => Some(MetricType::LimitConcurrentConnection), + 141 => Some(MetricType::LimitQuota), + 142 => Some(MetricType::LimitBlobQuota), + 144 => Some(MetricType::LimitTooManyRequests), + 338 => Some(MetricType::LimitMemoryBudget), + 145 => Some(MetricType::MailAuthParseError), + 146 => Some(MetricType::MailAuthMissingParameters), + 147 => Some(MetricType::MailAuthNoHeadersFound), + 148 => Some(MetricType::MailAuthCrypto), + 149 => Some(MetricType::MailAuthIo), + 150 => Some(MetricType::MailAuthBase64), + 151 => Some(MetricType::MailAuthDnsError), + 152 => Some(MetricType::MailAuthDnsRecordNotFound), + 153 => Some(MetricType::MailAuthDnsInvalidRecordType), + 154 => Some(MetricType::MailAuthPolicyNotAligned), + 155 => Some(MetricType::ManageSieveConnectionStart), + 156 => Some(MetricType::ManageSieveConnectionEnd), 4 => Some(MetricType::MessageSize), 5 => Some(MetricType::MessageAuthenticatedSize), 0 => Some(MetricType::MessageIngestTime), - 1 => Some(MetricType::MessageIngestIndexTime), - 173 => Some(MetricType::MessageIngestHam), - 174 => Some(MetricType::MessageIngestSpam), - 175 => Some(MetricType::MessageIngestImapAppend), - 176 => Some(MetricType::MessageIngestJmapAppend), - 177 => Some(MetricType::MessageIngestDuplicate), - 178 => Some(MetricType::MessageIngestError), - 179 => Some(MetricType::MessageIngestSearchIndex), - 180 => Some(MetricType::MilterActionAccept), - 181 => Some(MetricType::MilterActionDiscard), - 182 => Some(MetricType::MilterActionReject), - 183 => Some(MetricType::MilterActionTempFail), - 184 => Some(MetricType::MilterActionReplyCode), - 185 => Some(MetricType::MilterActionConnectionFailure), - 186 => Some(MetricType::MilterActionShutdown), - 187 => Some(MetricType::MtaHookActionAccept), - 188 => Some(MetricType::MtaHookActionDiscard), - 189 => Some(MetricType::MtaHookActionReject), - 190 => Some(MetricType::MtaHookActionQuarantine), - 191 => Some(MetricType::MtaHookError), - 192 => Some(MetricType::MtaStsAuthorized), - 193 => Some(MetricType::MtaStsNotAuthorized), - 194 => Some(MetricType::MtaStsInvalidPolicy), - 195 => Some(MetricType::NetworkTimeout), + 157 => Some(MetricType::MessageIngestHam), + 158 => Some(MetricType::MessageIngestSpam), + 159 => Some(MetricType::MessageIngestImapAppend), + 160 => Some(MetricType::MessageIngestJmapAppend), + 161 => Some(MetricType::MessageIngestDuplicate), + 162 => Some(MetricType::MessageIngestError), + 164 => Some(MetricType::MilterActionAccept), + 165 => Some(MetricType::MilterActionDiscard), + 166 => Some(MetricType::MilterActionReject), + 167 => Some(MetricType::MilterActionTempFail), + 168 => Some(MetricType::MilterActionReplyCode), + 169 => Some(MetricType::MilterActionConnectionFailure), + 170 => Some(MetricType::MilterActionShutdown), + 171 => Some(MetricType::MtaHookActionAccept), + 172 => Some(MetricType::MtaHookActionDiscard), + 173 => Some(MetricType::MtaHookActionReject), + 174 => Some(MetricType::MtaHookActionQuarantine), + 175 => Some(MetricType::MtaHookError), + 176 => Some(MetricType::MtaStsAuthorized), + 177 => Some(MetricType::MtaStsNotAuthorized), + 178 => Some(MetricType::MtaStsInvalidPolicy), + 179 => Some(MetricType::NetworkTimeout), 6 => Some(MetricType::OutgoingReportSize), - 196 => Some(MetricType::OutgoingReportSpfReport), - 197 => Some(MetricType::OutgoingReportSpfRateLimited), - 198 => Some(MetricType::OutgoingReportDkimReport), - 199 => Some(MetricType::OutgoingReportDkimRateLimited), - 200 => Some(MetricType::OutgoingReportDmarcReport), - 201 => Some(MetricType::OutgoingReportDmarcRateLimited), - 202 => Some(MetricType::OutgoingReportDmarcAggregateReport), - 203 => Some(MetricType::OutgoingReportTlsAggregate), - 204 => Some(MetricType::OutgoingReportHttpSubmission), - 205 => Some(MetricType::OutgoingReportUnauthorizedReportingAddress), - 206 => Some(MetricType::OutgoingReportReportingAddressValidationError), - 207 => Some(MetricType::OutgoingReportNotFound), - 208 => Some(MetricType::OutgoingReportSubmissionError), - 209 => Some(MetricType::OutgoingReportNoRecipientsFound), + 180 => Some(MetricType::OutgoingReportSpfReport), + 181 => Some(MetricType::OutgoingReportSpfRateLimited), + 182 => Some(MetricType::OutgoingReportDkimReport), + 183 => Some(MetricType::OutgoingReportDkimRateLimited), + 184 => Some(MetricType::OutgoingReportDmarcReport), + 185 => Some(MetricType::OutgoingReportDmarcRateLimited), + 186 => Some(MetricType::OutgoingReportDmarcAggregateReport), + 187 => Some(MetricType::OutgoingReportTlsAggregate), + 188 => Some(MetricType::OutgoingReportHttpSubmission), + 189 => Some(MetricType::OutgoingReportUnauthorizedReportingAddress), + 190 => Some(MetricType::OutgoingReportReportingAddressValidationError), + 191 => Some(MetricType::OutgoingReportNotFound), + 192 => Some(MetricType::OutgoingReportSubmissionError), + 193 => Some(MetricType::OutgoingReportNoRecipientsFound), 14 => Some(MetricType::Pop3RequestTime), 19 => Some(MetricType::Pop3ActiveConnections), - 210 => Some(MetricType::Pop3ConnectionStart), - 211 => Some(MetricType::Pop3ConnectionEnd), - 213 => Some(MetricType::PushSubscriptionSuccess), - 214 => Some(MetricType::PushSubscriptionError), - 215 => Some(MetricType::PushSubscriptionNotFound), + 194 => Some(MetricType::Pop3ConnectionStart), + 195 => Some(MetricType::Pop3ConnectionEnd), + 197 => Some(MetricType::PushSubscriptionSuccess), + 198 => Some(MetricType::PushSubscriptionError), + 199 => Some(MetricType::PushSubscriptionNotFound), 24 => Some(MetricType::QueueCount), - 216 => Some(MetricType::QueueMessageQueued), - 217 => Some(MetricType::QueueAuthenticatedMessageQueued), - 218 => Some(MetricType::QueueReportQueued), - 219 => Some(MetricType::QueueDsnQueued), - 220 => Some(MetricType::QueueAutogeneratedQueued), - 221 => Some(MetricType::QueueRescheduled), - 222 => Some(MetricType::QueueBlobNotFound), - 223 => Some(MetricType::QueueRateLimitExceeded), - 224 => Some(MetricType::QueueConcurrencyLimitExceeded), - 225 => Some(MetricType::QueueQuotaExceeded), - 226 => Some(MetricType::ResourceNotFound), - 227 => Some(MetricType::ResourceBadParameters), - 228 => Some(MetricType::ResourceError), - 229 => Some(MetricType::SecurityAuthenticationBan), - 230 => Some(MetricType::SecurityAbuseBan), - 231 => Some(MetricType::SecurityScanBan), - 232 => Some(MetricType::SecurityLoiterBan), - 233 => Some(MetricType::SecurityIpBlocked), - 234 => Some(MetricType::SecurityUnauthorized), + 200 => Some(MetricType::QueueMessageQueued), + 201 => Some(MetricType::QueueAuthenticatedMessageQueued), + 202 => Some(MetricType::QueueReportQueued), + 203 => Some(MetricType::QueueDsnQueued), + 204 => Some(MetricType::QueueAutogeneratedQueued), + 205 => Some(MetricType::QueueRescheduled), + 206 => Some(MetricType::QueueBlobNotFound), + 207 => Some(MetricType::QueueRateLimitExceeded), + 208 => Some(MetricType::QueueConcurrencyLimitExceeded), + 209 => Some(MetricType::QueueQuotaExceeded), + 347 => Some(MetricType::QueueBackPressure), + 210 => Some(MetricType::ResourceNotFound), + 211 => Some(MetricType::ResourceBadParameters), + 212 => Some(MetricType::ResourceError), + 1 => Some(MetricType::SearchIndexIndexTime), + 354 => Some(MetricType::SearchIndexPendingEmail), + 355 => Some(MetricType::SearchIndexPendingCalendar), + 356 => Some(MetricType::SearchIndexPendingContacts), + 357 => Some(MetricType::SearchIndexPendingFile), + 360 => Some(MetricType::SearchIndexPartitionRetry), + 163 => Some(MetricType::SearchIndexBatchIndexed), + 359 => Some(MetricType::SearchIndexDocumentTruncated), + 213 => Some(MetricType::SecurityAuthenticationBan), + 214 => Some(MetricType::SecurityAbuseBan), + 215 => Some(MetricType::SecurityScanBan), + 216 => Some(MetricType::SecurityLoiterBan), + 217 => Some(MetricType::SecurityIpBlocked), + 218 => Some(MetricType::SecurityUnauthorized), 23 => Some(MetricType::ServerMemory), - 235 => Some(MetricType::ServerThreadError), + 352 => Some(MetricType::ServerBufferedMemorySmtp), + 353 => Some(MetricType::ServerBufferedMemoryRead), + 219 => Some(MetricType::ServerThreadError), 16 => Some(MetricType::SieveRequestTime), 21 => Some(MetricType::SieveActiveConnections), - 236 => Some(MetricType::SieveActionAccept), - 237 => Some(MetricType::SieveActionAcceptReplace), - 238 => Some(MetricType::SieveActionDiscard), - 239 => Some(MetricType::SieveActionReject), - 240 => Some(MetricType::SieveSendMessage), - 241 => Some(MetricType::SieveMessageTooLarge), - 242 => Some(MetricType::SieveRuntimeError), - 243 => Some(MetricType::SieveUnexpectedError), - 244 => Some(MetricType::SieveNotSupported), - 245 => Some(MetricType::SieveQuotaExceeded), + 220 => Some(MetricType::SieveActionAccept), + 221 => Some(MetricType::SieveActionAcceptReplace), + 222 => Some(MetricType::SieveActionDiscard), + 223 => Some(MetricType::SieveActionReject), + 224 => Some(MetricType::SieveSendMessage), + 225 => Some(MetricType::SieveMessageTooLarge), + 226 => Some(MetricType::SieveRuntimeError), + 227 => Some(MetricType::SieveUnexpectedError), + 228 => Some(MetricType::SieveNotSupported), + 229 => Some(MetricType::SieveQuotaExceeded), 15 => Some(MetricType::SmtpRequestTime), 20 => Some(MetricType::SmtpActiveConnections), - 246 => Some(MetricType::SmtpConnectionStart), - 247 => Some(MetricType::SmtpConnectionEnd), - 248 => Some(MetricType::SmtpError), - 249 => Some(MetricType::SmtpConcurrencyLimitExceeded), - 250 => Some(MetricType::SmtpTransferLimitExceeded), - 251 => Some(MetricType::SmtpRateLimitExceeded), - 252 => Some(MetricType::SmtpTimeLimitExceeded), - 253 => Some(MetricType::SmtpMessageParseFailed), - 254 => Some(MetricType::SmtpMessageTooLarge), - 255 => Some(MetricType::SmtpLoopDetected), - 256 => Some(MetricType::SmtpDkimPass), - 257 => Some(MetricType::SmtpDkimFail), - 364 => Some(MetricType::SmtpDkim2Pass), - 365 => Some(MetricType::SmtpDkim2Fail), - 366 => Some(MetricType::SmtpDkim2DsnDiscarded), - 258 => Some(MetricType::SmtpArcPass), - 259 => Some(MetricType::SmtpArcFail), - 260 => Some(MetricType::SmtpSpfEhloPass), - 261 => Some(MetricType::SmtpSpfEhloFail), - 262 => Some(MetricType::SmtpSpfFromPass), - 263 => Some(MetricType::SmtpSpfFromFail), - 264 => Some(MetricType::SmtpDmarcPass), - 265 => Some(MetricType::SmtpDmarcFail), - 266 => Some(MetricType::SmtpIprevPass), - 267 => Some(MetricType::SmtpIprevFail), - 268 => Some(MetricType::SmtpTooManyMessages), - 269 => Some(MetricType::SmtpInvalidEhlo), - 270 => Some(MetricType::SmtpDidNotSayEhlo), - 271 => Some(MetricType::SmtpMailFromUnauthenticated), - 272 => Some(MetricType::SmtpMailFromUnauthorized), - 273 => Some(MetricType::SmtpMailFromMissing), - 274 => Some(MetricType::SmtpMultipleMailFrom), - 275 => Some(MetricType::SmtpMailboxDoesNotExist), - 276 => Some(MetricType::SmtpRelayNotAllowed), - 277 => Some(MetricType::SmtpRcptToDuplicate), - 278 => Some(MetricType::SmtpRcptToMissing), - 279 => Some(MetricType::SmtpTooManyRecipients), - 280 => Some(MetricType::SmtpTooManyInvalidRcpt), - 281 => Some(MetricType::SmtpAuthMechanismNotSupported), - 282 => Some(MetricType::SmtpAuthExchangeTooLong), - 283 => Some(MetricType::SmtpCommandNotImplemented), - 284 => Some(MetricType::SmtpInvalidCommand), - 285 => Some(MetricType::SmtpSyntaxError), - 286 => Some(MetricType::SmtpRequestTooLarge), - 287 => Some(MetricType::SpamPyzorError), - 288 => Some(MetricType::SpamDnsblError), - 289 => Some(MetricType::SpamTrainCompleted), - 290 => Some(MetricType::SpamTrainSampleAdded), - 291 => Some(MetricType::SpamClassify), - 292 => Some(MetricType::SpamModelNotReady), - 293 => Some(MetricType::SpfPass), - 294 => Some(MetricType::SpfFail), - 295 => Some(MetricType::SpfSoftFail), - 296 => Some(MetricType::SpfNeutral), - 297 => Some(MetricType::SpfTempError), - 298 => Some(MetricType::SpfPermError), - 299 => Some(MetricType::SpfNone), + 230 => Some(MetricType::SmtpConnectionStart), + 231 => Some(MetricType::SmtpConnectionEnd), + 232 => Some(MetricType::SmtpError), + 233 => Some(MetricType::SmtpConcurrencyLimitExceeded), + 234 => Some(MetricType::SmtpTransferLimitExceeded), + 235 => Some(MetricType::SmtpRateLimitExceeded), + 236 => Some(MetricType::SmtpTimeLimitExceeded), + 237 => Some(MetricType::SmtpMessageParseFailed), + 238 => Some(MetricType::SmtpMessageTooLarge), + 239 => Some(MetricType::SmtpLoopDetected), + 337 => Some(MetricType::SmtpMultipleFromDomains), + 240 => Some(MetricType::SmtpDkimPass), + 241 => Some(MetricType::SmtpDkimFail), + 324 => Some(MetricType::SmtpDkim2Pass), + 325 => Some(MetricType::SmtpDkim2Fail), + 326 => Some(MetricType::SmtpDkim2DsnRejected), + 242 => Some(MetricType::SmtpArcPass), + 243 => Some(MetricType::SmtpArcFail), + 244 => Some(MetricType::SmtpSpfEhloPass), + 245 => Some(MetricType::SmtpSpfEhloFail), + 246 => Some(MetricType::SmtpSpfFromPass), + 247 => Some(MetricType::SmtpSpfFromFail), + 248 => Some(MetricType::SmtpDmarcPass), + 249 => Some(MetricType::SmtpDmarcFail), + 250 => Some(MetricType::SmtpIprevPass), + 251 => Some(MetricType::SmtpIprevFail), + 252 => Some(MetricType::SmtpTooManyMessages), + 253 => Some(MetricType::SmtpInvalidEhlo), + 254 => Some(MetricType::SmtpDidNotSayEhlo), + 255 => Some(MetricType::SmtpMailFromUnauthenticated), + 256 => Some(MetricType::SmtpMailFromUnauthorized), + 257 => Some(MetricType::SmtpMailFromMissing), + 258 => Some(MetricType::SmtpMultipleMailFrom), + 259 => Some(MetricType::SmtpMailboxDoesNotExist), + 260 => Some(MetricType::SmtpRelayNotAllowed), + 261 => Some(MetricType::SmtpRcptToDuplicate), + 262 => Some(MetricType::SmtpRcptToMissing), + 263 => Some(MetricType::SmtpTooManyRecipients), + 264 => Some(MetricType::SmtpTooManyInvalidRcpt), + 265 => Some(MetricType::SmtpAuthMechanismNotSupported), + 266 => Some(MetricType::SmtpAuthExchangeTooLong), + 267 => Some(MetricType::SmtpCommandNotImplemented), + 268 => Some(MetricType::SmtpInvalidCommand), + 269 => Some(MetricType::SmtpSyntaxError), + 270 => Some(MetricType::SmtpRequestTooLarge), + 271 => Some(MetricType::SpamPyzorError), + 272 => Some(MetricType::SpamDnsblError), + 273 => Some(MetricType::SpamTrainCompleted), + 274 => Some(MetricType::SpamTrainSampleAdded), + 275 => Some(MetricType::SpamClassify), + 276 => Some(MetricType::SpamModelNotReady), + 277 => Some(MetricType::SpfPass), + 278 => Some(MetricType::SpfFail), + 279 => Some(MetricType::SpfSoftFail), + 280 => Some(MetricType::SpfNeutral), + 281 => Some(MetricType::SpfTempError), + 282 => Some(MetricType::SpfPermError), + 283 => Some(MetricType::SpfNone), 7 => Some(MetricType::StoreDataReadTime), 8 => Some(MetricType::StoreDataWriteTime), 9 => Some(MetricType::StoreBlobReadTime), 10 => Some(MetricType::StoreBlobWriteTime), - 300 => Some(MetricType::StoreAssertValueFailed), - 301 => Some(MetricType::StoreFoundationdbError), - 302 => Some(MetricType::StoreMysqlError), - 303 => Some(MetricType::StorePostgresqlError), - 304 => Some(MetricType::StoreRocksdbError), - 305 => Some(MetricType::StoreSqliteError), - 306 => Some(MetricType::StoreLdapError), - 307 => Some(MetricType::StoreElasticsearchError), - 308 => Some(MetricType::StoreRedisError), - 309 => Some(MetricType::StoreS3Error), - 310 => Some(MetricType::StoreAzureError), - 311 => Some(MetricType::StoreFilesystemError), - 312 => Some(MetricType::StorePoolError), - 313 => Some(MetricType::StoreDataCorruption), - 314 => Some(MetricType::StoreDecompressError), - 315 => Some(MetricType::StoreDeserializeError), - 316 => Some(MetricType::StoreNotFound), - 317 => Some(MetricType::StoreNotConfigured), - 318 => Some(MetricType::StoreNotSupported), - 319 => Some(MetricType::StoreUnexpectedError), - 320 => Some(MetricType::StoreCryptoError), - 321 => Some(MetricType::StoreHttpStoreError), - 322 => Some(MetricType::StoreBlobMissingMarker), - 323 => Some(MetricType::StoreDataWrite), - 324 => Some(MetricType::StoreDataIterate), - 325 => Some(MetricType::StoreBlobRead), - 326 => Some(MetricType::StoreBlobWrite), - 327 => Some(MetricType::StoreBlobDelete), - 328 => Some(MetricType::TaskManagerBlobNotFound), - 329 => Some(MetricType::TaskManagerMetadataNotFound), - 338 => Some(MetricType::TelemetryAlertEvent), - 212 => Some(MetricType::TelemetryAlertMessage), - 330 => Some(MetricType::TelemetryLogError), - 331 => Some(MetricType::TelemetryWebhookError), - 332 => Some(MetricType::TelemetryOtelExporterError), - 333 => Some(MetricType::TelemetryOtelMetricsExporterError), - 334 => Some(MetricType::TelemetryPrometheusExporterError), - 335 => Some(MetricType::TelemetryJournalError), - 336 => Some(MetricType::TlsHandshakeError), + 284 => Some(MetricType::StoreAssertValueFailed), + 285 => Some(MetricType::StoreFoundationdbError), + 286 => Some(MetricType::StoreMysqlError), + 287 => Some(MetricType::StorePostgresqlError), + 288 => Some(MetricType::StoreRocksdbError), + 289 => Some(MetricType::StoreSqliteError), + 290 => Some(MetricType::StoreLdapError), + 291 => Some(MetricType::StoreElasticsearchError), + 292 => Some(MetricType::StoreRedisError), + 293 => Some(MetricType::StoreS3Error), + 294 => Some(MetricType::StoreAzureError), + 295 => Some(MetricType::StoreFilesystemError), + 296 => Some(MetricType::StorePoolError), + 297 => Some(MetricType::StoreDataCorruption), + 298 => Some(MetricType::StoreDecompressError), + 299 => Some(MetricType::StoreDeserializeError), + 300 => Some(MetricType::StoreNotFound), + 301 => Some(MetricType::StoreNotConfigured), + 302 => Some(MetricType::StoreNotSupported), + 303 => Some(MetricType::StoreUnexpectedError), + 304 => Some(MetricType::StoreCryptoError), + 305 => Some(MetricType::StoreHttpStoreError), + 306 => Some(MetricType::StoreBlobMissingMarker), + 307 => Some(MetricType::StoreDataWrite), + 308 => Some(MetricType::StoreDataIterate), + 309 => Some(MetricType::StoreBlobRead), + 310 => Some(MetricType::StoreBlobWrite), + 311 => Some(MetricType::StoreBlobDelete), + 363 => Some(MetricType::TaskManagerPending), + 364 => Some(MetricType::TaskManagerDue), + 361 => Some(MetricType::TaskManagerTaskFailed), + 362 => Some(MetricType::TaskManagerTaskRetry), + 383 => Some(MetricType::TaskManagerTaskStalled), + 384 => Some(MetricType::TaskManagerTaskExpired), + 312 => Some(MetricType::TaskManagerBlobNotFound), + 313 => Some(MetricType::TaskManagerMetadataNotFound), + 314 => Some(MetricType::TelemetryLogError), + 315 => Some(MetricType::TelemetryWebhookError), + 316 => Some(MetricType::TelemetryOtelExporterError), + 317 => Some(MetricType::TelemetryOtelMetricsExporterError), + 318 => Some(MetricType::TelemetryPrometheusExporterError), + 319 => Some(MetricType::TelemetryJournalError), + 327 => Some(MetricType::TelemetryEventsDropped), + 320 => Some(MetricType::TlsHandshakeError), 25 => Some(MetricType::UserCount), _ => None, } @@ -6345,340 +6367,332 @@ impl MetricType { MetricType::AcmeTlsAlpnError => 24, MetricType::AcmeTokenNotFound => 26, MetricType::AcmeError => 15, - MetricType::ArcChainTooLong => 28, - MetricType::ArcInvalidInstance => 31, - MetricType::ArcInvalidCv => 30, - MetricType::ArcHasHeaderTag => 29, - MetricType::ArcBrokenChain => 27, - MetricType::AuthSuccess => 37, - MetricType::AuthFailed => 35, - MetricType::AuthTooManyAttempts => 38, - MetricType::AuthError => 34, - MetricType::CalendarAlarmSent => 579, - MetricType::CalendarAlarmFailed => 582, - MetricType::CalendarItipMessageSent => 583, - MetricType::CalendarItipMessageReceived => 584, - MetricType::CalendarItipMessageError => 585, - MetricType::ClusterSubscriberError => 41, - MetricType::ClusterSubscriberDisconnected => 42, - MetricType::ClusterPublisherError => 45, - MetricType::DaneAuthenticationSuccess => 67, - MetricType::DaneAuthenticationFailure => 66, - MetricType::DaneNoCertificatesFound => 69, - MetricType::DaneCertificateParseError => 68, - MetricType::DaneTlsaRecordFetchError => 71, - MetricType::DaneTlsaRecordNotFound => 75, - MetricType::DaneTlsaRecordNotDnssecSigned => 74, - MetricType::DaneTlsaRecordInvalid => 72, - MetricType::DaneBogusDnssecRecord => 605, - MetricType::DeliveryAttemptStart => 77, - MetricType::DeliveryAttemptEnd => 76, - MetricType::DeliveryCompleted => 80, - MetricType::DeliveryMxLookupFailed => 102, - MetricType::DeliveryIpLookupFailed => 96, - MetricType::DeliveryNullMx => 103, - MetricType::DeliveryGreetingFailed => 93, - MetricType::DeliveryEhloRejected => 91, - MetricType::DeliveryAuthFailed => 79, - MetricType::DeliveryMailFromRejected => 98, - MetricType::DeliveryDelivered => 84, - MetricType::DeliveryRcptToRejected => 109, - MetricType::DeliveryRcptToFailed => 108, - MetricType::DeliveryMessageRejected => 99, - MetricType::DeliveryStartTlsUnavailable => 113, - MetricType::DeliveryStartTlsError => 112, - MetricType::DeliveryStartTlsDisabled => 111, - MetricType::DeliveryImplicitTlsError => 94, - MetricType::DeliveryConcurrencyLimitExceeded => 81, - MetricType::DeliveryRateLimitExceeded => 104, - MetricType::DeliveryDoubleBounce => 86, - MetricType::DeliveryDsnSuccess => 88, - MetricType::DeliveryDsnTempFail => 89, - MetricType::DeliveryDsnPermFail => 87, - MetricType::DkimPass => 121, - MetricType::DkimNeutral => 119, - MetricType::DkimFail => 114, - MetricType::DkimPermError => 122, - MetricType::DkimTempError => 127, - MetricType::DkimNone => 120, - MetricType::DkimUnsupportedVersion => 131, - MetricType::DkimUnsupportedAlgorithm => 128, - MetricType::DkimUnsupportedCanonicalization => 129, - MetricType::DkimUnsupportedKeyType => 130, - MetricType::DkimFailedBodyHashMatch => 116, - MetricType::DkimFailedVerification => 117, - MetricType::DkimFailedAuidMatch => 115, - MetricType::DkimRevokedPublicKey => 123, - MetricType::DkimIncompatibleAlgorithms => 118, - MetricType::DkimSignatureExpired => 124, - MetricType::DkimSignatureLength => 125, - MetricType::DkimSignerNotFound => 126, - MetricType::DkimInstanceMissing => 606, - MetricType::DkimInstanceSyntax => 607, - MetricType::DkimInstanceTagMissing => 608, - MetricType::DkimInstanceNotSigned => 609, - MetricType::DkimInstanceAboveSignature => 610, - MetricType::DkimSignatureMissing => 611, - MetricType::DkimSignatureSyntax => 612, - MetricType::DkimSignatureTagMissing => 613, - MetricType::DkimSignatureTagUnexpected => 614, - MetricType::DkimSequenceGap => 615, - MetricType::DkimSequenceOverflow => 616, - MetricType::DkimChainTooLong => 635, - MetricType::DkimMailFromMismatch => 617, - MetricType::DkimRcptToMismatch => 618, - MetricType::DkimMailFromDomainMismatch => 619, - MetricType::DkimNextDomainMismatch => 620, - MetricType::DkimCustodyBreak => 636, - MetricType::DkimPublicKeyFetch => 621, - MetricType::DkimPublicKeyMissing => 622, - MetricType::DkimPublicKeyMultiple => 623, - MetricType::DkimPublicKeySyntax => 624, - MetricType::DkimPublicKeyAlgorithmMismatch => 625, - MetricType::DkimNoValidAlgorithm => 626, - MetricType::DkimHeaderHashMismatch => 627, - MetricType::DkimModified => 628, - MetricType::DkimExploded => 629, - MetricType::DmarcPass => 134, - MetricType::DmarcFail => 132, - MetricType::DmarcPermError => 135, - MetricType::DmarcTempError => 136, - MetricType::DmarcNone => 133, + MetricType::ArcChainInvalid => 647, + MetricType::ArcSignatureInvalid => 648, + MetricType::ArcHashMismatch => 649, + MetricType::AuthSuccess => 32, + MetricType::AuthFailed => 30, + MetricType::AuthTooManyAttempts => 33, + MetricType::AuthError => 29, + MetricType::CacheMiss => 45, + MetricType::CacheHit => 46, + MetricType::CacheStale => 47, + MetricType::CacheEntryTooLarge => 592, + MetricType::CacheSwapHit => 594, + MetricType::CacheSwapMiss => 595, + MetricType::CacheSwapWrite => 596, + MetricType::CacheSwapError => 597, + MetricType::CalendarAlarmSent => 562, + MetricType::CalendarAlarmFailed => 565, + MetricType::CalendarItipMessageSent => 566, + MetricType::CalendarItipMessageReceived => 567, + MetricType::CalendarItipMessageError => 568, + MetricType::ClusterSubscriberError => 36, + MetricType::ClusterSubscriberDisconnected => 37, + MetricType::ClusterPublisherError => 40, + MetricType::DaneAuthenticationSuccess => 62, + MetricType::DaneAuthenticationFailure => 61, + MetricType::DaneNoCertificatesFound => 64, + MetricType::DaneCertificateParseError => 63, + MetricType::DaneTlsaRecordFetchError => 66, + MetricType::DaneTlsaRecordNotFound => 70, + MetricType::DaneTlsaRecordNotDnssecSigned => 69, + MetricType::DaneTlsaRecordInvalid => 67, + MetricType::DaneBogusDnssecRecord => 588, + MetricType::DeliveryAttemptStart => 72, + MetricType::DeliveryAttemptEnd => 71, + MetricType::DeliveryCompleted => 75, + MetricType::DeliveryMxLookupFailed => 97, + MetricType::DeliveryIpLookupFailed => 91, + MetricType::DeliveryNullMx => 98, + MetricType::DeliveryGreetingFailed => 88, + MetricType::DeliveryEhloRejected => 86, + MetricType::DeliveryAuthFailed => 74, + MetricType::DeliveryMailFromRejected => 93, + MetricType::DeliveryDelivered => 79, + MetricType::DeliveryRcptToRejected => 104, + MetricType::DeliveryRcptToFailed => 103, + MetricType::DeliveryMessageRejected => 94, + MetricType::DeliveryStartTlsUnavailable => 108, + MetricType::DeliveryStartTlsError => 107, + MetricType::DeliveryStartTlsDisabled => 106, + MetricType::DeliveryImplicitTlsError => 89, + MetricType::DeliveryConcurrencyLimitExceeded => 76, + MetricType::DeliveryRateLimitExceeded => 99, + MetricType::DeliveryDoubleBounce => 81, + MetricType::DeliveryDsnSuccess => 83, + MetricType::DeliveryDsnTempFail => 84, + MetricType::DeliveryDsnPermFail => 82, + MetricType::DkimPass => 112, + MetricType::DkimNeutral => 110, + MetricType::DkimFail => 109, + MetricType::DkimPermError => 113, + MetricType::DkimTempError => 115, + MetricType::DkimNone => 111, + MetricType::DkimSignatureInvalid => 643, + MetricType::DkimPublicKeyError => 644, + MetricType::DkimVerificationFailed => 645, + MetricType::DkimHashMismatch => 646, + MetricType::DkimSignerNotFound => 114, + MetricType::Dkim2InstanceInvalid => 633, + MetricType::Dkim2SignatureInvalid => 634, + MetricType::Dkim2ChainInvalid => 635, + MetricType::Dkim2TimestampInvalid => 636, + MetricType::Dkim2EnvelopeMismatch => 637, + MetricType::Dkim2CustodyBroken => 638, + MetricType::Dkim2PublicKeyError => 639, + MetricType::Dkim2VerificationFailed => 640, + MetricType::Dkim2HashMismatch => 641, + MetricType::Dkim2RecipeError => 642, + MetricType::Dkim2SigningFailed => 650, + MetricType::DmarcPass => 118, + MetricType::DmarcFail => 116, + MetricType::DmarcPermError => 119, + MetricType::DmarcTempError => 120, + MetricType::DmarcNone => 117, MetricType::DnsRecordCreationFailed => 9, MetricType::DnsRecordDeletionFailed => 10, MetricType::DnsRecordLookupFailed => 11, MetricType::DnsRecordPropagationTimeout => 14, - MetricType::EvalError => 138, - MetricType::EvalDirectoryNotFound => 137, - MetricType::EvalStoreNotFound => 140, - MetricType::HttpConnectionStart => 153, - MetricType::HttpError => 154, - MetricType::HttpRequestBody => 155, - MetricType::HttpResponseBody => 157, - MetricType::HttpXForwardedMissing => 158, - MetricType::ImapConnectionStart => 163, - MetricType::ImapConnectionEnd => 162, - MetricType::IncomingReportDmarcReport => 200, - MetricType::IncomingReportDmarcReportWithWarnings => 201, - MetricType::IncomingReportTlsReport => 206, - MetricType::IncomingReportTlsReportWithWarnings => 207, - MetricType::IncomingReportAbuseReport => 195, - MetricType::IncomingReportAuthFailureReport => 197, - MetricType::IncomingReportFraudReport => 202, - MetricType::IncomingReportNotSpamReport => 204, - MetricType::IncomingReportVirusReport => 209, - MetricType::IncomingReportOtherReport => 205, - MetricType::IncomingReportMessageParseFailed => 203, - MetricType::IncomingReportDmarcParseFailed => 199, - MetricType::IncomingReportTlsRpcParseFailed => 208, - MetricType::IncomingReportArfParseFailed => 196, - MetricType::IncomingReportDecompressError => 198, - MetricType::IprevPass => 212, - MetricType::IprevFail => 210, - MetricType::IprevPermError => 213, - MetricType::IprevTempError => 214, - MetricType::IprevNone => 211, - MetricType::JmapMethodCall => 223, - MetricType::JmapInvalidArguments => 221, - MetricType::JmapRequestTooLarge => 227, - MetricType::JmapUnsupportedFilter => 232, - MetricType::JmapUnsupportedSort => 233, - MetricType::JmapUnknownMethod => 231, - MetricType::JmapForbidden => 220, - MetricType::JmapNotJson => 225, - MetricType::JmapNotRequest => 226, - MetricType::JmapWebsocketStart => 235, - MetricType::JmapWebsocketError => 234, - MetricType::LimitSizeRequest => 243, - MetricType::LimitSizeUpload => 244, - MetricType::LimitCallsIn => 238, - MetricType::LimitConcurrentRequest => 240, - MetricType::LimitConcurrentUpload => 241, - MetricType::LimitConcurrentConnection => 239, - MetricType::LimitQuota => 242, - MetricType::LimitBlobQuota => 237, - MetricType::LimitTenantQuota => 553, - MetricType::LimitTooManyRequests => 245, - MetricType::MailAuthParseError => 254, - MetricType::MailAuthMissingParameters => 252, - MetricType::MailAuthNoHeadersFound => 253, - MetricType::MailAuthCrypto => 247, - MetricType::MailAuthIo => 251, - MetricType::MailAuthBase64 => 246, - MetricType::MailAuthDnsError => 248, - MetricType::MailAuthDnsRecordNotFound => 250, - MetricType::MailAuthDnsInvalidRecordType => 249, - MetricType::MailAuthPolicyNotAligned => 255, - MetricType::ManageSieveConnectionStart => 259, - MetricType::ManageSieveConnectionEnd => 258, - MetricType::MessageIngestHam => 283, - MetricType::MessageIngestSpam => 286, - MetricType::MessageIngestImapAppend => 284, - MetricType::MessageIngestJmapAppend => 285, - MetricType::MessageIngestDuplicate => 281, - MetricType::MessageIngestError => 282, - MetricType::MessageIngestSearchIndex => 142, - MetricType::MilterActionAccept => 287, - MetricType::MilterActionDiscard => 289, - MetricType::MilterActionReject => 290, - MetricType::MilterActionTempFail => 293, - MetricType::MilterActionReplyCode => 291, - MetricType::MilterActionConnectionFailure => 288, - MetricType::MilterActionShutdown => 292, - MetricType::MtaHookActionAccept => 304, - MetricType::MtaHookActionDiscard => 305, - MetricType::MtaHookActionReject => 307, - MetricType::MtaHookActionQuarantine => 306, - MetricType::MtaHookError => 308, - MetricType::MtaStsAuthorized => 309, - MetricType::MtaStsNotAuthorized => 311, - MetricType::MtaStsInvalidPolicy => 310, - MetricType::NetworkTimeout => 327, - MetricType::OutgoingReportSpfReport => 342, - MetricType::OutgoingReportSpfRateLimited => 341, - MetricType::OutgoingReportDkimReport => 330, - MetricType::OutgoingReportDkimRateLimited => 329, - MetricType::OutgoingReportDmarcReport => 333, - MetricType::OutgoingReportDmarcRateLimited => 332, - MetricType::OutgoingReportDmarcAggregateReport => 331, - MetricType::OutgoingReportTlsAggregate => 344, - MetricType::OutgoingReportHttpSubmission => 334, - MetricType::OutgoingReportUnauthorizedReportingAddress => 345, - MetricType::OutgoingReportReportingAddressValidationError => 340, - MetricType::OutgoingReportNotFound => 339, - MetricType::OutgoingReportSubmissionError => 343, - MetricType::OutgoingReportNoRecipientsFound => 338, - MetricType::Pop3ConnectionStart => 348, - MetricType::Pop3ConnectionEnd => 347, - MetricType::PushSubscriptionSuccess => 373, - MetricType::PushSubscriptionError => 371, - MetricType::PushSubscriptionNotFound => 372, - MetricType::QueueMessageQueued => 380, - MetricType::QueueAuthenticatedMessageQueued => 381, - MetricType::QueueReportQueued => 382, - MetricType::QueueDsnQueued => 379, - MetricType::QueueAutogeneratedQueued => 378, - MetricType::QueueRescheduled => 385, - MetricType::QueueBlobNotFound => 374, - MetricType::QueueRateLimitExceeded => 384, - MetricType::QueueConcurrencyLimitExceeded => 375, - MetricType::QueueQuotaExceeded => 383, - MetricType::ResourceNotFound => 389, - MetricType::ResourceBadParameters => 386, - MetricType::ResourceError => 388, - MetricType::SecurityAuthenticationBan => 33, - MetricType::SecurityAbuseBan => 549, - MetricType::SecurityScanBan => 558, - MetricType::SecurityLoiterBan => 550, - MetricType::SecurityIpBlocked => 318, - MetricType::SecurityUnauthorized => 552, - MetricType::ServerThreadError => 395, - MetricType::SieveActionAccept => 396, - MetricType::SieveActionAcceptReplace => 397, - MetricType::SieveActionDiscard => 398, - MetricType::SieveActionReject => 399, - MetricType::SieveSendMessage => 406, - MetricType::SieveMessageTooLarge => 401, - MetricType::SieveRuntimeError => 404, - MetricType::SieveUnexpectedError => 407, - MetricType::SieveNotSupported => 402, - MetricType::SieveQuotaExceeded => 403, - MetricType::SmtpConnectionStart => 417, - MetricType::SmtpConnectionEnd => 416, - MetricType::SmtpError => 428, - MetricType::SmtpConcurrencyLimitExceeded => 415, - MetricType::SmtpTransferLimitExceeded => 485, - MetricType::SmtpRateLimitExceeded => 461, - MetricType::SmtpTimeLimitExceeded => 481, - MetricType::SmtpMessageParseFailed => 450, - MetricType::SmtpMessageTooLarge => 451, - MetricType::SmtpLoopDetected => 443, - MetricType::SmtpDkimPass => 422, - MetricType::SmtpDkimFail => 421, - MetricType::SmtpDkim2Pass => 630, - MetricType::SmtpDkim2Fail => 631, - MetricType::SmtpDkim2DsnDiscarded => 632, - MetricType::SmtpArcPass => 410, - MetricType::SmtpArcFail => 409, - MetricType::SmtpSpfEhloPass => 474, - MetricType::SmtpSpfEhloFail => 473, - MetricType::SmtpSpfFromPass => 476, - MetricType::SmtpSpfFromFail => 475, - MetricType::SmtpDmarcPass => 424, - MetricType::SmtpDmarcFail => 423, - MetricType::SmtpIprevPass => 441, - MetricType::SmtpIprevFail => 440, - MetricType::SmtpTooManyMessages => 483, - MetricType::SmtpInvalidEhlo => 436, - MetricType::SmtpDidNotSayEhlo => 420, - MetricType::SmtpMailFromUnauthenticated => 447, - MetricType::SmtpMailFromUnauthorized => 448, - MetricType::SmtpMailFromMissing => 445, - MetricType::SmtpMultipleMailFrom => 456, - MetricType::SmtpMailboxDoesNotExist => 449, - MetricType::SmtpRelayNotAllowed => 468, - MetricType::SmtpRcptToDuplicate => 465, - MetricType::SmtpRcptToMissing => 466, - MetricType::SmtpTooManyRecipients => 484, - MetricType::SmtpTooManyInvalidRcpt => 482, - MetricType::SmtpAuthMechanismNotSupported => 412, - MetricType::SmtpAuthExchangeTooLong => 411, - MetricType::SmtpCommandNotImplemented => 414, - MetricType::SmtpInvalidCommand => 435, - MetricType::SmtpSyntaxError => 480, - MetricType::SmtpRequestTooLarge => 470, - MetricType::SpamPyzorError => 494, - MetricType::SpamDnsblError => 563, - MetricType::SpamTrainCompleted => 495, - MetricType::SpamTrainSampleAdded => 143, - MetricType::SpamClassify => 490, - MetricType::SpamModelNotReady => 496, - MetricType::SpfPass => 501, - MetricType::SpfFail => 498, - MetricType::SpfSoftFail => 503, - MetricType::SpfNeutral => 499, - MetricType::SpfTempError => 504, - MetricType::SpfPermError => 502, - MetricType::SpfNone => 500, - MetricType::StoreAssertValueFailed => 505, - MetricType::StoreFoundationdbError => 518, - MetricType::StoreMysqlError => 522, - MetricType::StorePostgresqlError => 527, - MetricType::StoreRocksdbError => 529, - MetricType::StoreSqliteError => 532, - MetricType::StoreLdapError => 520, - MetricType::StoreElasticsearchError => 516, - MetricType::StoreRedisError => 528, - MetricType::StoreS3Error => 530, - MetricType::StoreAzureError => 559, - MetricType::StoreFilesystemError => 517, - MetricType::StorePoolError => 526, - MetricType::StoreDataCorruption => 511, - MetricType::StoreDecompressError => 514, - MetricType::StoreDeserializeError => 515, - MetricType::StoreNotFound => 524, - MetricType::StoreNotConfigured => 523, - MetricType::StoreNotSupported => 525, - MetricType::StoreUnexpectedError => 533, - MetricType::StoreCryptoError => 510, - MetricType::StoreHttpStoreError => 493, - MetricType::StoreBlobMissingMarker => 507, - MetricType::StoreDataWrite => 513, - MetricType::StoreDataIterate => 512, - MetricType::StoreBlobRead => 508, - MetricType::StoreBlobWrite => 509, - MetricType::StoreBlobDelete => 506, - MetricType::TaskManagerBlobNotFound => 141, - MetricType::TaskManagerMetadataNotFound => 145, - MetricType::TelemetryAlertEvent => 548, - MetricType::TelemetryAlertMessage => 365, - MetricType::TelemetryLogError => 535, - MetricType::TelemetryWebhookError => 539, - MetricType::TelemetryOtelExporterError => 536, - MetricType::TelemetryOtelMetricsExporterError => 537, - MetricType::TelemetryPrometheusExporterError => 538, - MetricType::TelemetryJournalError => 534, - MetricType::TlsHandshakeError => 544, + MetricType::EvalError => 122, + MetricType::EvalDirectoryNotFound => 121, + MetricType::EvalStoreNotFound => 124, + MetricType::HttpConnectionStart => 137, + MetricType::HttpError => 138, + MetricType::HttpRequestBody => 139, + MetricType::HttpResponseBody => 141, + MetricType::HttpXForwardedMissing => 142, + MetricType::HttpXForwardedInvalid => 625, + MetricType::ImapConnectionStart => 147, + MetricType::ImapConnectionEnd => 146, + MetricType::IncomingReportDmarcReport => 184, + MetricType::IncomingReportDmarcReportWithWarnings => 185, + MetricType::IncomingReportTlsReport => 190, + MetricType::IncomingReportTlsReportWithWarnings => 191, + MetricType::IncomingReportAbuseReport => 179, + MetricType::IncomingReportAuthFailureReport => 181, + MetricType::IncomingReportFraudReport => 186, + MetricType::IncomingReportNotSpamReport => 188, + MetricType::IncomingReportVirusReport => 193, + MetricType::IncomingReportOtherReport => 189, + MetricType::IncomingReportMessageParseFailed => 187, + MetricType::IncomingReportDmarcParseFailed => 183, + MetricType::IncomingReportTlsRpcParseFailed => 192, + MetricType::IncomingReportArfParseFailed => 180, + MetricType::IncomingReportDecompressError => 182, + MetricType::IprevPass => 196, + MetricType::IprevFail => 194, + MetricType::IprevPermError => 197, + MetricType::IprevTempError => 198, + MetricType::IprevNone => 195, + MetricType::JmapMethodCall => 207, + MetricType::JmapInvalidArguments => 205, + MetricType::JmapRequestTooLarge => 211, + MetricType::JmapUnsupportedFilter => 216, + MetricType::JmapUnsupportedSort => 217, + MetricType::JmapUnknownMethod => 215, + MetricType::JmapForbidden => 204, + MetricType::JmapNotJson => 209, + MetricType::JmapNotRequest => 210, + MetricType::JmapWebsocketStart => 219, + MetricType::JmapWebsocketError => 218, + MetricType::LimitSizeRequest => 227, + MetricType::LimitSizeUpload => 228, + MetricType::LimitCallsIn => 222, + MetricType::LimitConcurrentRequest => 224, + MetricType::LimitConcurrentUpload => 225, + MetricType::LimitConcurrentConnection => 223, + MetricType::LimitQuota => 226, + MetricType::LimitBlobQuota => 221, + MetricType::LimitTooManyRequests => 229, + MetricType::LimitMemoryBudget => 627, + MetricType::MailAuthParseError => 238, + MetricType::MailAuthMissingParameters => 236, + MetricType::MailAuthNoHeadersFound => 237, + MetricType::MailAuthCrypto => 231, + MetricType::MailAuthIo => 235, + MetricType::MailAuthBase64 => 230, + MetricType::MailAuthDnsError => 232, + MetricType::MailAuthDnsRecordNotFound => 234, + MetricType::MailAuthDnsInvalidRecordType => 233, + MetricType::MailAuthPolicyNotAligned => 239, + MetricType::ManageSieveConnectionStart => 243, + MetricType::ManageSieveConnectionEnd => 242, + MetricType::MessageIngestHam => 267, + MetricType::MessageIngestSpam => 270, + MetricType::MessageIngestImapAppend => 268, + MetricType::MessageIngestJmapAppend => 269, + MetricType::MessageIngestDuplicate => 265, + MetricType::MessageIngestError => 266, + MetricType::MilterActionAccept => 271, + MetricType::MilterActionDiscard => 273, + MetricType::MilterActionReject => 274, + MetricType::MilterActionTempFail => 277, + MetricType::MilterActionReplyCode => 275, + MetricType::MilterActionConnectionFailure => 272, + MetricType::MilterActionShutdown => 276, + MetricType::MtaHookActionAccept => 288, + MetricType::MtaHookActionDiscard => 289, + MetricType::MtaHookActionReject => 291, + MetricType::MtaHookActionQuarantine => 290, + MetricType::MtaHookError => 292, + MetricType::MtaStsAuthorized => 293, + MetricType::MtaStsNotAuthorized => 295, + MetricType::MtaStsInvalidPolicy => 294, + MetricType::NetworkTimeout => 311, + MetricType::OutgoingReportSpfReport => 326, + MetricType::OutgoingReportSpfRateLimited => 325, + MetricType::OutgoingReportDkimReport => 314, + MetricType::OutgoingReportDkimRateLimited => 313, + MetricType::OutgoingReportDmarcReport => 317, + MetricType::OutgoingReportDmarcRateLimited => 316, + MetricType::OutgoingReportDmarcAggregateReport => 315, + MetricType::OutgoingReportTlsAggregate => 328, + MetricType::OutgoingReportHttpSubmission => 318, + MetricType::OutgoingReportUnauthorizedReportingAddress => 329, + MetricType::OutgoingReportReportingAddressValidationError => 324, + MetricType::OutgoingReportNotFound => 323, + MetricType::OutgoingReportSubmissionError => 327, + MetricType::OutgoingReportNoRecipientsFound => 322, + MetricType::Pop3ConnectionStart => 332, + MetricType::Pop3ConnectionEnd => 331, + MetricType::PushSubscriptionSuccess => 357, + MetricType::PushSubscriptionError => 355, + MetricType::PushSubscriptionNotFound => 356, + MetricType::QueueMessageQueued => 364, + MetricType::QueueAuthenticatedMessageQueued => 365, + MetricType::QueueReportQueued => 366, + MetricType::QueueDsnQueued => 363, + MetricType::QueueAutogeneratedQueued => 362, + MetricType::QueueRescheduled => 369, + MetricType::QueueBlobNotFound => 358, + MetricType::QueueRateLimitExceeded => 368, + MetricType::QueueConcurrencyLimitExceeded => 359, + MetricType::QueueQuotaExceeded => 367, + MetricType::QueueBackPressure => 43, + MetricType::ResourceNotFound => 373, + MetricType::ResourceBadParameters => 370, + MetricType::ResourceError => 372, + MetricType::SearchIndexPartitionRetry => 632, + MetricType::SearchIndexBatchIndexed => 126, + MetricType::SearchIndexDocumentTruncated => 628, + MetricType::SecurityAuthenticationBan => 28, + MetricType::SecurityAbuseBan => 532, + MetricType::SecurityScanBan => 541, + MetricType::SecurityLoiterBan => 533, + MetricType::SecurityIpBlocked => 302, + MetricType::SecurityUnauthorized => 535, + MetricType::ServerThreadError => 379, + MetricType::SieveActionAccept => 380, + MetricType::SieveActionAcceptReplace => 381, + MetricType::SieveActionDiscard => 382, + MetricType::SieveActionReject => 383, + MetricType::SieveSendMessage => 390, + MetricType::SieveMessageTooLarge => 385, + MetricType::SieveRuntimeError => 388, + MetricType::SieveUnexpectedError => 391, + MetricType::SieveNotSupported => 386, + MetricType::SieveQuotaExceeded => 387, + MetricType::SmtpConnectionStart => 401, + MetricType::SmtpConnectionEnd => 400, + MetricType::SmtpError => 412, + MetricType::SmtpConcurrencyLimitExceeded => 399, + MetricType::SmtpTransferLimitExceeded => 468, + MetricType::SmtpRateLimitExceeded => 444, + MetricType::SmtpTimeLimitExceeded => 464, + MetricType::SmtpMessageParseFailed => 434, + MetricType::SmtpMessageTooLarge => 435, + MetricType::SmtpLoopDetected => 427, + MetricType::SmtpMultipleFromDomains => 626, + MetricType::SmtpDkimPass => 406, + MetricType::SmtpDkimFail => 405, + MetricType::SmtpDkim2Pass => 589, + MetricType::SmtpDkim2Fail => 590, + MetricType::SmtpDkim2DsnRejected => 591, + MetricType::SmtpArcPass => 394, + MetricType::SmtpArcFail => 393, + MetricType::SmtpSpfEhloPass => 457, + MetricType::SmtpSpfEhloFail => 456, + MetricType::SmtpSpfFromPass => 459, + MetricType::SmtpSpfFromFail => 458, + MetricType::SmtpDmarcPass => 408, + MetricType::SmtpDmarcFail => 407, + MetricType::SmtpIprevPass => 425, + MetricType::SmtpIprevFail => 424, + MetricType::SmtpTooManyMessages => 466, + MetricType::SmtpInvalidEhlo => 420, + MetricType::SmtpDidNotSayEhlo => 404, + MetricType::SmtpMailFromUnauthenticated => 431, + MetricType::SmtpMailFromUnauthorized => 432, + MetricType::SmtpMailFromMissing => 429, + MetricType::SmtpMultipleMailFrom => 439, + MetricType::SmtpMailboxDoesNotExist => 433, + MetricType::SmtpRelayNotAllowed => 451, + MetricType::SmtpRcptToDuplicate => 448, + MetricType::SmtpRcptToMissing => 449, + MetricType::SmtpTooManyRecipients => 467, + MetricType::SmtpTooManyInvalidRcpt => 465, + MetricType::SmtpAuthMechanismNotSupported => 396, + MetricType::SmtpAuthExchangeTooLong => 395, + MetricType::SmtpCommandNotImplemented => 398, + MetricType::SmtpInvalidCommand => 419, + MetricType::SmtpSyntaxError => 463, + MetricType::SmtpRequestTooLarge => 453, + MetricType::SpamPyzorError => 477, + MetricType::SpamDnsblError => 546, + MetricType::SpamTrainCompleted => 478, + MetricType::SpamTrainSampleAdded => 127, + MetricType::SpamClassify => 473, + MetricType::SpamModelNotReady => 479, + MetricType::SpfPass => 484, + MetricType::SpfFail => 481, + MetricType::SpfSoftFail => 486, + MetricType::SpfNeutral => 482, + MetricType::SpfTempError => 487, + MetricType::SpfPermError => 485, + MetricType::SpfNone => 483, + MetricType::StoreAssertValueFailed => 488, + MetricType::StoreFoundationdbError => 501, + MetricType::StoreMysqlError => 505, + MetricType::StorePostgresqlError => 510, + MetricType::StoreRocksdbError => 512, + MetricType::StoreSqliteError => 515, + MetricType::StoreLdapError => 503, + MetricType::StoreElasticsearchError => 499, + MetricType::StoreRedisError => 511, + MetricType::StoreS3Error => 513, + MetricType::StoreAzureError => 542, + MetricType::StoreFilesystemError => 500, + MetricType::StorePoolError => 509, + MetricType::StoreDataCorruption => 494, + MetricType::StoreDecompressError => 497, + MetricType::StoreDeserializeError => 498, + MetricType::StoreNotFound => 507, + MetricType::StoreNotConfigured => 506, + MetricType::StoreNotSupported => 508, + MetricType::StoreUnexpectedError => 516, + MetricType::StoreCryptoError => 493, + MetricType::StoreHttpStoreError => 476, + MetricType::StoreBlobMissingMarker => 490, + MetricType::StoreDataWrite => 496, + MetricType::StoreDataIterate => 495, + MetricType::StoreBlobRead => 491, + MetricType::StoreBlobWrite => 492, + MetricType::StoreBlobDelete => 489, + MetricType::TaskManagerTaskFailed => 570, + MetricType::TaskManagerTaskRetry => 48, + MetricType::TaskManagerTaskStalled => 651, + MetricType::TaskManagerTaskExpired => 652, + MetricType::TaskManagerBlobNotFound => 125, + MetricType::TaskManagerMetadataNotFound => 129, + MetricType::TelemetryLogError => 518, + MetricType::TelemetryWebhookError => 522, + MetricType::TelemetryOtelExporterError => 519, + MetricType::TelemetryOtelMetricsExporterError => 520, + MetricType::TelemetryPrometheusExporterError => 521, + MetricType::TelemetryJournalError => 517, + MetricType::TelemetryEventsDropped => 601, + MetricType::TlsHandshakeError => 527, _ => usize::MAX, } } @@ -6693,15 +6707,29 @@ impl MetricType { MetricType::AcmeTlsAlpnError => "ACME TLS ALPN error", MetricType::AcmeTokenNotFound => "ACME token not found", MetricType::AcmeError => "ACME error", - MetricType::ArcChainTooLong => "ARC chain too long", - MetricType::ArcInvalidInstance => "Invalid ARC instance", - MetricType::ArcInvalidCv => "Invalid ARC CV", - MetricType::ArcHasHeaderTag => "ARC has header tag", - MetricType::ArcBrokenChain => "Broken ARC chain", + MetricType::ArcChainInvalid => "ARC chain invalid", + MetricType::ArcSignatureInvalid => "ARC signature invalid", + MetricType::ArcHashMismatch => "ARC body hash mismatch", MetricType::AuthSuccess => "Authentication successful", MetricType::AuthFailed => "Authentication failed", MetricType::AuthTooManyAttempts => "Too many authentication attempts", MetricType::AuthError => "Authentication error", + MetricType::CacheMessagesSize => "Memory used by the e-mail data cache", + MetricType::CacheMessagesLargeSize => { + "Total size of the large-account e-mail data caches held in memory, both those retained by the large tier and those still referenced by in-flight requests" + } + MetricType::CacheSize => "Memory used by all other caches", + MetricType::CacheSwapPending => { + "Cache snapshots with changes not yet written to the swap tier" + } + MetricType::CacheMiss => "Cache miss", + MetricType::CacheHit => "Cache hit", + MetricType::CacheStale => "Cache is stale", + MetricType::CacheEntryTooLarge => "Cache entry too large", + MetricType::CacheSwapHit => "Cache snapshot loaded", + MetricType::CacheSwapMiss => "Cache snapshot unavailable", + MetricType::CacheSwapWrite => "Cache snapshot written", + MetricType::CacheSwapError => "Cache swap tier error", MetricType::CalendarAlarmSent => "Calendar alarm sent", MetricType::CalendarAlarmFailed => "Calendar alarm could not be sent", MetricType::CalendarItipMessageSent => "Calendar iTIP message sent", @@ -6754,44 +6782,22 @@ impl MetricType { MetricType::DkimPermError => "DKIM permanent error", MetricType::DkimTempError => "DKIM temporary error", MetricType::DkimNone => "No DKIM signature", - MetricType::DkimUnsupportedVersion => "Unsupported DKIM version", - MetricType::DkimUnsupportedAlgorithm => "Unsupported DKIM algorithm", - MetricType::DkimUnsupportedCanonicalization => "Unsupported DKIM canonicalization", - MetricType::DkimUnsupportedKeyType => "Unsupported DKIM key type", - MetricType::DkimFailedBodyHashMatch => "DKIM body hash mismatch", - MetricType::DkimFailedVerification => "DKIM verification failed", - MetricType::DkimFailedAuidMatch => "DKIM AUID mismatch", - MetricType::DkimRevokedPublicKey => "DKIM public key revoked", - MetricType::DkimIncompatibleAlgorithms => "Incompatible DKIM algorithms", - MetricType::DkimSignatureExpired => "DKIM signature expired", - MetricType::DkimSignatureLength => "DKIM signature length issue", + MetricType::DkimSignatureInvalid => "DKIM signature invalid", + MetricType::DkimPublicKeyError => "DKIM public key error", + MetricType::DkimVerificationFailed => "DKIM signature verification failed", + MetricType::DkimHashMismatch => "DKIM body hash mismatch", MetricType::DkimSignerNotFound => "DKIM signer not found", - MetricType::DkimInstanceMissing => "DKIM2 message-instance missing", - MetricType::DkimInstanceSyntax => "DKIM2 message-instance syntax error", - MetricType::DkimInstanceTagMissing => "DKIM2 message-instance tag missing", - MetricType::DkimInstanceNotSigned => "DKIM2 message-instance not signed", - MetricType::DkimInstanceAboveSignature => "DKIM2 message-instance above signature", - MetricType::DkimSignatureMissing => "DKIM2 signature missing", - MetricType::DkimSignatureSyntax => "DKIM2 signature syntax error", - MetricType::DkimSignatureTagMissing => "DKIM2 signature tag missing", - MetricType::DkimSignatureTagUnexpected => "DKIM2 signature unexpected tag", - MetricType::DkimSequenceGap => "DKIM2 sequence gap", - MetricType::DkimSequenceOverflow => "DKIM2 sequence overflow", - MetricType::DkimChainTooLong => "DKIM2 chain too long", - MetricType::DkimMailFromMismatch => "DKIM2 MAIL FROM mismatch", - MetricType::DkimRcptToMismatch => "DKIM2 RCPT TO mismatch", - MetricType::DkimMailFromDomainMismatch => "DKIM2 MAIL FROM domain mismatch", - MetricType::DkimNextDomainMismatch => "DKIM2 next domain mismatch", - MetricType::DkimCustodyBreak => "DKIM2 custody break", - MetricType::DkimPublicKeyFetch => "DKIM2 public key fetch error", - MetricType::DkimPublicKeyMissing => "DKIM2 public key missing", - MetricType::DkimPublicKeyMultiple => "DKIM2 multiple public keys", - MetricType::DkimPublicKeySyntax => "DKIM2 public key syntax error", - MetricType::DkimPublicKeyAlgorithmMismatch => "DKIM2 public key algorithm mismatch", - MetricType::DkimNoValidAlgorithm => "DKIM2 no valid algorithm", - MetricType::DkimHeaderHashMismatch => "DKIM2 header hash mismatch", - MetricType::DkimModified => "DKIM2 message modified", - MetricType::DkimExploded => "DKIM2 message exploded", + MetricType::Dkim2InstanceInvalid => "DKIM2 Message-Instance invalid", + MetricType::Dkim2SignatureInvalid => "DKIM2 signature invalid", + MetricType::Dkim2ChainInvalid => "DKIM2 chain invalid", + MetricType::Dkim2TimestampInvalid => "DKIM2 timestamp invalid", + MetricType::Dkim2EnvelopeMismatch => "DKIM2 envelope mismatch", + MetricType::Dkim2CustodyBroken => "DKIM2 chain of custody broken", + MetricType::Dkim2PublicKeyError => "DKIM2 public key error", + MetricType::Dkim2VerificationFailed => "DKIM2 signature verification failed", + MetricType::Dkim2HashMismatch => "DKIM2 hash mismatch", + MetricType::Dkim2RecipeError => "DKIM2 recipe error", + MetricType::Dkim2SigningFailed => "DKIM2 signing failed", MetricType::DmarcPass => "DMARC check passed", MetricType::DmarcFail => "DMARC check failed", MetricType::DmarcPermError => "DMARC permanent error", @@ -6806,14 +6812,15 @@ impl MetricType { MetricType::EvalError => "Expression evaluation error", MetricType::EvalDirectoryNotFound => "Directory not found while evaluating expression", MetricType::EvalStoreNotFound => "Store not found while evaluating expression", - MetricType::HttpRequestTime => "HTTP request duration", + MetricType::HttpRequestTime => "HTTP connection duration", MetricType::HttpActiveConnections => "Active HTTP connections", MetricType::HttpConnectionStart => "HTTP connection started", MetricType::HttpError => "HTTP error occurred", MetricType::HttpRequestBody => "HTTP request body", MetricType::HttpResponseBody => "HTTP response body", MetricType::HttpXForwardedMissing => "X-Forwarded-For header is missing", - MetricType::ImapRequestTime => "IMAP request duration", + MetricType::HttpXForwardedInvalid => "Invalid forwarding header", + MetricType::ImapRequestTime => "IMAP connection duration", MetricType::ImapActiveConnections => "Active IMAP connections", MetricType::ImapConnectionStart => "IMAP connection started", MetricType::ImapConnectionEnd => "IMAP connection ended", @@ -6860,8 +6867,8 @@ impl MetricType { MetricType::LimitConcurrentConnection => "Concurrent connection limit reached", MetricType::LimitQuota => "Quota limit reached", MetricType::LimitBlobQuota => "Blob quota limit reached", - MetricType::LimitTenantQuota => "Tenant quota limit reached", MetricType::LimitTooManyRequests => "Too many requests", + MetricType::LimitMemoryBudget => "Memory budget exceeded", MetricType::MailAuthParseError => "Mail authentication parse error", MetricType::MailAuthMissingParameters => "Missing mail authentication parameters", MetricType::MailAuthNoHeadersFound => "No headers found in message", @@ -6879,14 +6886,12 @@ impl MetricType { "Received message size from authenticated users" } MetricType::MessageIngestTime => "Message ingestion time", - MetricType::MessageIngestIndexTime => "Message full-text indexing time", MetricType::MessageIngestHam => "Message ingested", MetricType::MessageIngestSpam => "Possible spam message ingested", MetricType::MessageIngestImapAppend => "Message appended via IMAP", MetricType::MessageIngestJmapAppend => "Message appended via JMAP", MetricType::MessageIngestDuplicate => "Skipping duplicate message", MetricType::MessageIngestError => "Message ingestion error", - MetricType::MessageIngestSearchIndex => "Search index updated", MetricType::MilterActionAccept => "Milter action: Accept", MetricType::MilterActionDiscard => "Milter action: Discard", MetricType::MilterActionReject => "Milter action: Reject", @@ -6922,7 +6927,7 @@ impl MetricType { MetricType::OutgoingReportNotFound => "Report not found", MetricType::OutgoingReportSubmissionError => "Error submitting report", MetricType::OutgoingReportNoRecipientsFound => "No recipients found for report", - MetricType::Pop3RequestTime => "POP3 request duration", + MetricType::Pop3RequestTime => "POP3 connection duration", MetricType::Pop3ActiveConnections => "Active POP3 connections", MetricType::Pop3ConnectionStart => "POP3 connection started", MetricType::Pop3ConnectionEnd => "POP3 connection ended", @@ -6940,9 +6945,20 @@ impl MetricType { MetricType::QueueRateLimitExceeded => "Rate limit exceeded", MetricType::QueueConcurrencyLimitExceeded => "Concurrency limit exceeded", MetricType::QueueQuotaExceeded => "Quota exceeded", + MetricType::QueueBackPressure => "Queue backpressure detected", MetricType::ResourceNotFound => "Resource not found", MetricType::ResourceBadParameters => "Bad resource parameters", MetricType::ResourceError => "Resource error", + MetricType::SearchIndexIndexTime => { + "Time to build and write one batch of documents to the full-text index" + } + MetricType::SearchIndexPendingEmail => "E-mail messages waiting to be indexed", + MetricType::SearchIndexPendingCalendar => "Calendar events waiting to be indexed", + MetricType::SearchIndexPendingContacts => "Contacts waiting to be indexed", + MetricType::SearchIndexPendingFile => "Files waiting to be indexed", + MetricType::SearchIndexPartitionRetry => "Index partition will be retried", + MetricType::SearchIndexBatchIndexed => "Search index batch written", + MetricType::SearchIndexDocumentTruncated => "Search document truncated", MetricType::SecurityAuthenticationBan => "Banned due to authentication errors", MetricType::SecurityAbuseBan => "Banned due to abuse", MetricType::SecurityScanBan => "Banned due to scan", @@ -6950,8 +6966,14 @@ impl MetricType { MetricType::SecurityIpBlocked => "Blocked IP address", MetricType::SecurityUnauthorized => "Unauthorized access", MetricType::ServerMemory => "Server memory usage", + MetricType::ServerBufferedMemorySmtp => { + "Memory reserved for buffering inbound SMTP messages" + } + MetricType::ServerBufferedMemoryRead => { + "Memory reserved for blobs read from the blob store" + } MetricType::ServerThreadError => "Server thread error", - MetricType::SieveRequestTime => "ManageSieve request duration", + MetricType::SieveRequestTime => "ManageSieve connection duration", MetricType::SieveActiveConnections => "Active ManageSieve connections", MetricType::SieveActionAccept => "Sieve action: Accept", MetricType::SieveActionAcceptReplace => "Sieve action: Accept and replace", @@ -6963,7 +6985,7 @@ impl MetricType { MetricType::SieveUnexpectedError => "Unexpected Sieve error", MetricType::SieveNotSupported => "Sieve action not supported", MetricType::SieveQuotaExceeded => "Sieve quota exceeded", - MetricType::SmtpRequestTime => "SMTP request duration", + MetricType::SmtpRequestTime => "SMTP connection duration", MetricType::SmtpActiveConnections => "Active SMTP connections", MetricType::SmtpConnectionStart => "SMTP connection started", MetricType::SmtpConnectionEnd => "SMTP connection ended", @@ -6975,11 +6997,12 @@ impl MetricType { MetricType::SmtpMessageParseFailed => "Message parsing failed", MetricType::SmtpMessageTooLarge => "Message too large", MetricType::SmtpLoopDetected => "Mail loop detected", + MetricType::SmtpMultipleFromDomains => "Multiple From domains", MetricType::SmtpDkimPass => "DKIM verification passed", MetricType::SmtpDkimFail => "DKIM verification failed", MetricType::SmtpDkim2Pass => "DKIM2 verification passed", MetricType::SmtpDkim2Fail => "DKIM2 verification failed", - MetricType::SmtpDkim2DsnDiscarded => "DKIM2 DSN discarded", + MetricType::SmtpDkim2DsnRejected => "DKIM2 DSN rejected", MetricType::SmtpArcPass => "ARC verification passed", MetricType::SmtpArcFail => "ARC verification failed", MetricType::SmtpSpfEhloPass => "SPF EHLO check passed", @@ -7054,16 +7077,21 @@ impl MetricType { MetricType::StoreBlobRead => "Blob read operation", MetricType::StoreBlobWrite => "Blob write operation", MetricType::StoreBlobDelete => "Blob delete operation", + MetricType::TaskManagerPending => "Tasks queued in the task manager", + MetricType::TaskManagerDue => "Tasks due for execution that have not run yet", + MetricType::TaskManagerTaskFailed => "Task failed permanently", + MetricType::TaskManagerTaskRetry => "Task will be retried", + MetricType::TaskManagerTaskStalled => "Task stalled", + MetricType::TaskManagerTaskExpired => "Task expired", MetricType::TaskManagerBlobNotFound => "Blob not found for task", MetricType::TaskManagerMetadataNotFound => "Metadata not found for task", - MetricType::TelemetryAlertEvent => "Alert event triggered", - MetricType::TelemetryAlertMessage => "Alert message sent", MetricType::TelemetryLogError => "Log collector error", MetricType::TelemetryWebhookError => "Webhook collector error", MetricType::TelemetryOtelExporterError => "OpenTelemetry exporter error", MetricType::TelemetryOtelMetricsExporterError => "OpenTelemetry metrics exporter error", MetricType::TelemetryPrometheusExporterError => "Prometheus exporter error", MetricType::TelemetryJournalError => "Journal collector error", + MetricType::TelemetryEventsDropped => "Telemetry events dropped", MetricType::TlsHandshakeError => "TLS handshake error", MetricType::UserCount => "Total number of users", } @@ -7071,10 +7099,15 @@ impl MetricType { pub fn unit(&self) -> &'static str { match self { - MetricType::MessageSize + MetricType::CacheMessagesSize + | MetricType::CacheMessagesLargeSize + | MetricType::CacheSize + | MetricType::MessageSize | MetricType::MessageAuthenticatedSize | MetricType::OutgoingReportSize - | MetricType::ServerMemory => "bytes", + | MetricType::ServerMemory + | MetricType::ServerBufferedMemorySmtp + | MetricType::ServerBufferedMemoryRead => "bytes", MetricType::DeliveryActiveConnections | MetricType::HttpActiveConnections | MetricType::ImapActiveConnections @@ -7089,15 +7122,21 @@ impl MetricType { | MetricType::AcmeTlsAlpnError | MetricType::AcmeTokenNotFound | MetricType::AcmeError - | MetricType::ArcChainTooLong - | MetricType::ArcInvalidInstance - | MetricType::ArcInvalidCv - | MetricType::ArcHasHeaderTag - | MetricType::ArcBrokenChain + | MetricType::ArcChainInvalid + | MetricType::ArcSignatureInvalid + | MetricType::ArcHashMismatch | MetricType::AuthSuccess | MetricType::AuthFailed | MetricType::AuthTooManyAttempts | MetricType::AuthError + | MetricType::CacheMiss + | MetricType::CacheHit + | MetricType::CacheStale + | MetricType::CacheEntryTooLarge + | MetricType::CacheSwapHit + | MetricType::CacheSwapMiss + | MetricType::CacheSwapWrite + | MetricType::CacheSwapError | MetricType::CalendarAlarmSent | MetricType::CalendarAlarmFailed | MetricType::CalendarItipMessageSent @@ -7145,44 +7184,22 @@ impl MetricType { | MetricType::DkimPermError | MetricType::DkimTempError | MetricType::DkimNone - | MetricType::DkimUnsupportedVersion - | MetricType::DkimUnsupportedAlgorithm - | MetricType::DkimUnsupportedCanonicalization - | MetricType::DkimUnsupportedKeyType - | MetricType::DkimFailedBodyHashMatch - | MetricType::DkimFailedVerification - | MetricType::DkimFailedAuidMatch - | MetricType::DkimRevokedPublicKey - | MetricType::DkimIncompatibleAlgorithms - | MetricType::DkimSignatureExpired - | MetricType::DkimSignatureLength + | MetricType::DkimSignatureInvalid + | MetricType::DkimPublicKeyError + | MetricType::DkimVerificationFailed + | MetricType::DkimHashMismatch | MetricType::DkimSignerNotFound - | MetricType::DkimInstanceMissing - | MetricType::DkimInstanceSyntax - | MetricType::DkimInstanceTagMissing - | MetricType::DkimInstanceNotSigned - | MetricType::DkimInstanceAboveSignature - | MetricType::DkimSignatureMissing - | MetricType::DkimSignatureSyntax - | MetricType::DkimSignatureTagMissing - | MetricType::DkimSignatureTagUnexpected - | MetricType::DkimSequenceGap - | MetricType::DkimSequenceOverflow - | MetricType::DkimChainTooLong - | MetricType::DkimMailFromMismatch - | MetricType::DkimRcptToMismatch - | MetricType::DkimMailFromDomainMismatch - | MetricType::DkimNextDomainMismatch - | MetricType::DkimCustodyBreak - | MetricType::DkimPublicKeyFetch - | MetricType::DkimPublicKeyMissing - | MetricType::DkimPublicKeyMultiple - | MetricType::DkimPublicKeySyntax - | MetricType::DkimPublicKeyAlgorithmMismatch - | MetricType::DkimNoValidAlgorithm - | MetricType::DkimHeaderHashMismatch - | MetricType::DkimModified - | MetricType::DkimExploded + | MetricType::Dkim2InstanceInvalid + | MetricType::Dkim2SignatureInvalid + | MetricType::Dkim2ChainInvalid + | MetricType::Dkim2TimestampInvalid + | MetricType::Dkim2EnvelopeMismatch + | MetricType::Dkim2CustodyBroken + | MetricType::Dkim2PublicKeyError + | MetricType::Dkim2VerificationFailed + | MetricType::Dkim2HashMismatch + | MetricType::Dkim2RecipeError + | MetricType::Dkim2SigningFailed | MetricType::DmarcPass | MetricType::DmarcFail | MetricType::DmarcPermError @@ -7200,6 +7217,7 @@ impl MetricType { | MetricType::HttpRequestBody | MetricType::HttpResponseBody | MetricType::HttpXForwardedMissing + | MetricType::HttpXForwardedInvalid | MetricType::ImapConnectionStart | MetricType::ImapConnectionEnd | MetricType::IncomingReportDmarcReport @@ -7241,8 +7259,8 @@ impl MetricType { | MetricType::LimitConcurrentConnection | MetricType::LimitQuota | MetricType::LimitBlobQuota - | MetricType::LimitTenantQuota | MetricType::LimitTooManyRequests + | MetricType::LimitMemoryBudget | MetricType::MailAuthParseError | MetricType::MailAuthMissingParameters | MetricType::MailAuthNoHeadersFound @@ -7261,7 +7279,6 @@ impl MetricType { | MetricType::MessageIngestJmapAppend | MetricType::MessageIngestDuplicate | MetricType::MessageIngestError - | MetricType::MessageIngestSearchIndex | MetricType::MilterActionAccept | MetricType::MilterActionDiscard | MetricType::MilterActionReject @@ -7307,9 +7324,13 @@ impl MetricType { | MetricType::QueueRateLimitExceeded | MetricType::QueueConcurrencyLimitExceeded | MetricType::QueueQuotaExceeded + | MetricType::QueueBackPressure | MetricType::ResourceNotFound | MetricType::ResourceBadParameters | MetricType::ResourceError + | MetricType::SearchIndexPartitionRetry + | MetricType::SearchIndexBatchIndexed + | MetricType::SearchIndexDocumentTruncated | MetricType::SecurityAuthenticationBan | MetricType::SecurityAbuseBan | MetricType::SecurityScanBan @@ -7337,11 +7358,12 @@ impl MetricType { | MetricType::SmtpMessageParseFailed | MetricType::SmtpMessageTooLarge | MetricType::SmtpLoopDetected + | MetricType::SmtpMultipleFromDomains | MetricType::SmtpDkimPass | MetricType::SmtpDkimFail | MetricType::SmtpDkim2Pass | MetricType::SmtpDkim2Fail - | MetricType::SmtpDkim2DsnDiscarded + | MetricType::SmtpDkim2DsnRejected | MetricType::SmtpArcPass | MetricType::SmtpArcFail | MetricType::SmtpSpfEhloPass @@ -7412,17 +7434,24 @@ impl MetricType { | MetricType::StoreBlobRead | MetricType::StoreBlobWrite | MetricType::StoreBlobDelete + | MetricType::TaskManagerTaskFailed + | MetricType::TaskManagerTaskRetry + | MetricType::TaskManagerTaskStalled + | MetricType::TaskManagerTaskExpired | MetricType::TaskManagerBlobNotFound | MetricType::TaskManagerMetadataNotFound - | MetricType::TelemetryAlertEvent - | MetricType::TelemetryAlertMessage | MetricType::TelemetryLogError | MetricType::TelemetryWebhookError | MetricType::TelemetryOtelExporterError | MetricType::TelemetryOtelMetricsExporterError | MetricType::TelemetryPrometheusExporterError | MetricType::TelemetryJournalError + | MetricType::TelemetryEventsDropped | MetricType::TlsHandshakeError => "count", + MetricType::SearchIndexPendingEmail + | MetricType::SearchIndexPendingCalendar + | MetricType::SearchIndexPendingContacts + | MetricType::SearchIndexPendingFile => "documents", MetricType::DomainCount => "domains", MetricType::QueueCount => "messages", MetricType::DeliveryTotalTime @@ -7431,14 +7460,16 @@ impl MetricType { | MetricType::HttpRequestTime | MetricType::ImapRequestTime | MetricType::MessageIngestTime - | MetricType::MessageIngestIndexTime | MetricType::Pop3RequestTime + | MetricType::SearchIndexIndexTime | MetricType::SieveRequestTime | MetricType::SmtpRequestTime | MetricType::StoreDataReadTime | MetricType::StoreDataWriteTime | MetricType::StoreBlobReadTime | MetricType::StoreBlobWriteTime => "milliseconds", + MetricType::CacheSwapPending => "snapshots", + MetricType::TaskManagerPending | MetricType::TaskManagerDue => "tasks", MetricType::UserCount => "users", } } @@ -7453,15 +7484,25 @@ impl MetricType { MetricType::AcmeTlsAlpnError, MetricType::AcmeTokenNotFound, MetricType::AcmeError, - MetricType::ArcChainTooLong, - MetricType::ArcInvalidInstance, - MetricType::ArcInvalidCv, - MetricType::ArcHasHeaderTag, - MetricType::ArcBrokenChain, + MetricType::ArcChainInvalid, + MetricType::ArcSignatureInvalid, + MetricType::ArcHashMismatch, MetricType::AuthSuccess, MetricType::AuthFailed, MetricType::AuthTooManyAttempts, MetricType::AuthError, + MetricType::CacheMessagesSize, + MetricType::CacheMessagesLargeSize, + MetricType::CacheSize, + MetricType::CacheSwapPending, + MetricType::CacheMiss, + MetricType::CacheHit, + MetricType::CacheStale, + MetricType::CacheEntryTooLarge, + MetricType::CacheSwapHit, + MetricType::CacheSwapMiss, + MetricType::CacheSwapWrite, + MetricType::CacheSwapError, MetricType::CalendarAlarmSent, MetricType::CalendarAlarmFailed, MetricType::CalendarItipMessageSent, @@ -7512,44 +7553,22 @@ impl MetricType { MetricType::DkimPermError, MetricType::DkimTempError, MetricType::DkimNone, - MetricType::DkimUnsupportedVersion, - MetricType::DkimUnsupportedAlgorithm, - MetricType::DkimUnsupportedCanonicalization, - MetricType::DkimUnsupportedKeyType, - MetricType::DkimFailedBodyHashMatch, - MetricType::DkimFailedVerification, - MetricType::DkimFailedAuidMatch, - MetricType::DkimRevokedPublicKey, - MetricType::DkimIncompatibleAlgorithms, - MetricType::DkimSignatureExpired, - MetricType::DkimSignatureLength, + MetricType::DkimSignatureInvalid, + MetricType::DkimPublicKeyError, + MetricType::DkimVerificationFailed, + MetricType::DkimHashMismatch, MetricType::DkimSignerNotFound, - MetricType::DkimInstanceMissing, - MetricType::DkimInstanceSyntax, - MetricType::DkimInstanceTagMissing, - MetricType::DkimInstanceNotSigned, - MetricType::DkimInstanceAboveSignature, - MetricType::DkimSignatureMissing, - MetricType::DkimSignatureSyntax, - MetricType::DkimSignatureTagMissing, - MetricType::DkimSignatureTagUnexpected, - MetricType::DkimSequenceGap, - MetricType::DkimSequenceOverflow, - MetricType::DkimChainTooLong, - MetricType::DkimMailFromMismatch, - MetricType::DkimRcptToMismatch, - MetricType::DkimMailFromDomainMismatch, - MetricType::DkimNextDomainMismatch, - MetricType::DkimCustodyBreak, - MetricType::DkimPublicKeyFetch, - MetricType::DkimPublicKeyMissing, - MetricType::DkimPublicKeyMultiple, - MetricType::DkimPublicKeySyntax, - MetricType::DkimPublicKeyAlgorithmMismatch, - MetricType::DkimNoValidAlgorithm, - MetricType::DkimHeaderHashMismatch, - MetricType::DkimModified, - MetricType::DkimExploded, + MetricType::Dkim2InstanceInvalid, + MetricType::Dkim2SignatureInvalid, + MetricType::Dkim2ChainInvalid, + MetricType::Dkim2TimestampInvalid, + MetricType::Dkim2EnvelopeMismatch, + MetricType::Dkim2CustodyBroken, + MetricType::Dkim2PublicKeyError, + MetricType::Dkim2VerificationFailed, + MetricType::Dkim2HashMismatch, + MetricType::Dkim2RecipeError, + MetricType::Dkim2SigningFailed, MetricType::DmarcPass, MetricType::DmarcFail, MetricType::DmarcPermError, @@ -7571,6 +7590,7 @@ impl MetricType { MetricType::HttpRequestBody, MetricType::HttpResponseBody, MetricType::HttpXForwardedMissing, + MetricType::HttpXForwardedInvalid, MetricType::ImapRequestTime, MetricType::ImapActiveConnections, MetricType::ImapConnectionStart, @@ -7614,8 +7634,8 @@ impl MetricType { MetricType::LimitConcurrentConnection, MetricType::LimitQuota, MetricType::LimitBlobQuota, - MetricType::LimitTenantQuota, MetricType::LimitTooManyRequests, + MetricType::LimitMemoryBudget, MetricType::MailAuthParseError, MetricType::MailAuthMissingParameters, MetricType::MailAuthNoHeadersFound, @@ -7631,14 +7651,12 @@ impl MetricType { MetricType::MessageSize, MetricType::MessageAuthenticatedSize, MetricType::MessageIngestTime, - MetricType::MessageIngestIndexTime, MetricType::MessageIngestHam, MetricType::MessageIngestSpam, MetricType::MessageIngestImapAppend, MetricType::MessageIngestJmapAppend, MetricType::MessageIngestDuplicate, MetricType::MessageIngestError, - MetricType::MessageIngestSearchIndex, MetricType::MilterActionAccept, MetricType::MilterActionDiscard, MetricType::MilterActionReject, @@ -7688,9 +7706,18 @@ impl MetricType { MetricType::QueueRateLimitExceeded, MetricType::QueueConcurrencyLimitExceeded, MetricType::QueueQuotaExceeded, + MetricType::QueueBackPressure, MetricType::ResourceNotFound, MetricType::ResourceBadParameters, MetricType::ResourceError, + MetricType::SearchIndexIndexTime, + MetricType::SearchIndexPendingEmail, + MetricType::SearchIndexPendingCalendar, + MetricType::SearchIndexPendingContacts, + MetricType::SearchIndexPendingFile, + MetricType::SearchIndexPartitionRetry, + MetricType::SearchIndexBatchIndexed, + MetricType::SearchIndexDocumentTruncated, MetricType::SecurityAuthenticationBan, MetricType::SecurityAbuseBan, MetricType::SecurityScanBan, @@ -7698,6 +7725,8 @@ impl MetricType { MetricType::SecurityIpBlocked, MetricType::SecurityUnauthorized, MetricType::ServerMemory, + MetricType::ServerBufferedMemorySmtp, + MetricType::ServerBufferedMemoryRead, MetricType::ServerThreadError, MetricType::SieveRequestTime, MetricType::SieveActiveConnections, @@ -7723,11 +7752,12 @@ impl MetricType { MetricType::SmtpMessageParseFailed, MetricType::SmtpMessageTooLarge, MetricType::SmtpLoopDetected, + MetricType::SmtpMultipleFromDomains, MetricType::SmtpDkimPass, MetricType::SmtpDkimFail, MetricType::SmtpDkim2Pass, MetricType::SmtpDkim2Fail, - MetricType::SmtpDkim2DsnDiscarded, + MetricType::SmtpDkim2DsnRejected, MetricType::SmtpArcPass, MetricType::SmtpArcFail, MetricType::SmtpSpfEhloPass, @@ -7802,16 +7832,21 @@ impl MetricType { MetricType::StoreBlobRead, MetricType::StoreBlobWrite, MetricType::StoreBlobDelete, + MetricType::TaskManagerPending, + MetricType::TaskManagerDue, + MetricType::TaskManagerTaskFailed, + MetricType::TaskManagerTaskRetry, + MetricType::TaskManagerTaskStalled, + MetricType::TaskManagerTaskExpired, MetricType::TaskManagerBlobNotFound, MetricType::TaskManagerMetadataNotFound, - MetricType::TelemetryAlertEvent, - MetricType::TelemetryAlertMessage, MetricType::TelemetryLogError, MetricType::TelemetryWebhookError, MetricType::TelemetryOtelExporterError, MetricType::TelemetryOtelMetricsExporterError, MetricType::TelemetryPrometheusExporterError, MetricType::TelemetryJournalError, + MetricType::TelemetryEventsDropped, MetricType::TlsHandshakeError, MetricType::UserCount, ] @@ -7819,6 +7854,7 @@ impl MetricType { } impl Key { + #[cfg(not(feature = "enterprise"))] pub fn parse(value: &str) -> Option { hashify::map! { value.as_bytes(), @@ -7889,6 +7925,27 @@ impl Key { b"value" => Key::Value, b"version" => Key::Version, b"queueName" => Key::QueueName, + b"name" => Key::Name, + b"objectType" => Key::ObjectType, + b"property" => Key::Property, + b"nodeId" => Key::NodeId, + b"remoteNodeId" => Key::RemoteNodeId, + b"taskId" => Key::TaskId, + b"partition" => Key::Partition, + b"index" => Key::Index, + b"attempt" => Key::Attempt, + b"used" => Key::Used, + b"period" => Key::Period, + b"retryAfter" => Key::RetryAfter, + b"command" => Key::Command, + b"tag" => Key::Tag, + b"address" => Key::Address, + b"ip" => Key::Ip, + b"headers" => Key::Headers, + b"query" => Key::Query, + b"cipher" => Key::Cipher, + b"resourceUid" => Key::ResourceUid, + b"script" => Key::Script, } .copied() } @@ -7961,6 +8018,27 @@ impl Key { Key::Value => "value", Key::Version => "version", Key::QueueName => "queueName", + Key::Name => "name", + Key::ObjectType => "objectType", + Key::Property => "property", + Key::NodeId => "nodeId", + Key::RemoteNodeId => "remoteNodeId", + Key::TaskId => "taskId", + Key::Partition => "partition", + Key::Index => "index", + Key::Attempt => "attempt", + Key::Used => "used", + Key::Period => "period", + Key::RetryAfter => "retryAfter", + Key::Command => "command", + Key::Tag => "tag", + Key::Address => "address", + Key::Ip => "ip", + Key::Headers => "headers", + Key::Query => "query", + Key::Cipher => "cipher", + Key::ResourceUid => "resourceUid", + Key::Script => "script", } } @@ -8036,11 +8114,32 @@ impl Key { 63 => Some(Key::Value), 64 => Some(Key::Version), 65 => Some(Key::QueueName), + 66 => Some(Key::Name), + 67 => Some(Key::ObjectType), + 68 => Some(Key::Property), + 69 => Some(Key::NodeId), + 70 => Some(Key::RemoteNodeId), + 71 => Some(Key::TaskId), + 72 => Some(Key::Partition), + 73 => Some(Key::Index), + 74 => Some(Key::Attempt), + 75 => Some(Key::Used), + 76 => Some(Key::Period), + 77 => Some(Key::RetryAfter), + 78 => Some(Key::Command), + 79 => Some(Key::Tag), + 80 => Some(Key::Address), + 81 => Some(Key::Ip), + 82 => Some(Key::Headers), + 83 => Some(Key::Query), + 84 => Some(Key::Cipher), + 85 => Some(Key::ResourceUid), + 86 => Some(Key::Script), _ => None, } } - pub const COUNT: usize = 66; + pub const COUNT: usize = 96; } impl serde::Serialize for Key { diff --git a/crates/trc/src/event/mod.rs b/crates/trc/src/event/mod.rs index 2c053560..5332a22d 100644 --- a/crates/trc/src/event/mod.rs +++ b/crates/trc/src/event/mod.rs @@ -12,9 +12,12 @@ pub mod enums; pub mod enums_impl; use compact_str::ToCompactString; -use std::fmt::Display; +use std::{fmt::Display, mem}; -use crate::*; +use crate::{ + ipc::{USIZE_BITS, USIZE_BITS_MASK, bitset::Bitset}, + *, +}; impl Event { pub fn with_capacity(inner: T, capacity: usize) -> Self { @@ -31,7 +34,7 @@ impl Event { pub fn new(inner: T) -> Self { Self { inner, - keys: Vec::with_capacity(5), + keys: Vec::new(), } } @@ -50,13 +53,9 @@ impl Event { } pub fn take_value(&mut self, key: Key) -> Option { - self.keys.iter_mut().find_map(|(k, v)| { - if *k == key { - Some(std::mem::take(v)) - } else { - None - } - }) + self.keys + .iter_mut() + .find_map(|(k, v)| if *k == key { Some(mem::take(v)) } else { None }) } pub fn into_boxed(self) -> Box { @@ -64,27 +63,57 @@ impl Event { } } +const ERROR_KEYS_CAPACITY: usize = 5; + +#[inline(never)] +fn push_key_at_capacity(keys: &mut Vec<(Key, Value)>, key: Key, value: Value) { + if keys.capacity() == 0 { + let mut first = Vec::with_capacity(ERROR_KEYS_CAPACITY); + first.push((key, value)); + mem::forget(mem::replace(keys, first)); + } else { + push_key_slow(keys, key, value); + } +} + +#[cold] +#[inline(never)] +fn push_key_slow(keys: &mut Vec<(Key, Value)>, key: Key, value: Value) { + keys.reserve(ERROR_KEYS_CAPACITY); + keys.push((key, value)); +} + impl Error { #[inline(always)] pub fn new(inner: EventType) -> Self { Error(Box::new(Event::new(inner))) } + #[inline(always)] + fn push_key(&mut self, key: Key, value: Value) { + let keys = &mut self.0.keys; + if keys.len() == keys.capacity() { + push_key_at_capacity(keys, key, value); + } else { + keys.push((key, value)); + } + } + #[inline(always)] pub fn set_ctx(&mut self, key: Key, value: impl Into) { - self.0.keys.push((key, value.into())); + self.push_key(key, value.into()); } #[inline(always)] pub fn ctx(mut self, key: Key, value: impl Into) -> Self { - self.0.keys.push((key, value.into())); + self.push_key(key, value.into()); self } #[inline(always)] pub fn ctx_unique(mut self, key: Key, value: impl Into) -> Self { if self.0.keys.iter().all(|(k, _)| *k != key) { - self.0.keys.push((key, value.into())); + self.push_key(key, value.into()); } self } @@ -122,6 +151,15 @@ impl Error { self.ctx(Key::Details, error) } + pub fn details_to_reason(mut self) -> Self { + for (key, _) in &mut self.0.keys { + if *key == Key::Details { + *key = Key::Reason; + } + } + self + } + #[inline(always)] pub fn code(self, error: impl Into) -> Self { self.ctx(Key::Code, error) @@ -242,43 +280,146 @@ impl Event { } } +const KEY_SET_WORDS: usize = 4; + +#[derive(Default, Clone, Copy)] +pub struct KeySet([u64; KEY_SET_WORDS]); + +impl KeySet { + #[inline(always)] + pub fn insert(&mut self, key: Key) -> bool { + let id = key as usize; + match self.0.get_mut(id >> 6) { + Some(word) => { + let bit = 1u64 << (id & 63); + let is_new = *word & bit == 0; + *word |= bit; + is_new + } + None => true, + } + } + + #[inline(always)] + pub fn contains(&self, key: Key) -> bool { + let id = key as usize; + self.0 + .get(id >> 6) + .is_some_and(|word| word & (1u64 << (id & 63)) != 0) + } +} + +pub(crate) type EventSet = Bitset<{ TOTAL_EVENT_COUNT.div_ceil(USIZE_BITS) }>; + +pub(crate) static SPAN_START_EVENTS: EventSet = EventClass::SpanStart.events(); +pub(crate) static SPAN_END_EVENTS: EventSet = EventClass::SpanEnd.events(); +pub(crate) static SPAN_EVENTS: EventSet = EventClass::Span.events(); +pub(crate) static UNCOUNTED_EVENTS: EventSet = EventClass::Uncounted.events(); +static RAW_IO_EVENTS: EventSet = EventClass::RawIo.events(); + +#[derive(Clone, Copy)] +enum EventClass { + SpanStart, + SpanEnd, + Span, + RawIo, + Uncounted, +} + +impl EventClass { + const fn contains(self, event: EventType) -> bool { + match self { + EventClass::SpanStart => matches!( + event, + EventType::Smtp(SmtpEvent::ConnectionStart) + | EventType::Imap(ImapEvent::ConnectionStart) + | EventType::ManageSieve(ManageSieveEvent::ConnectionStart) + | EventType::Pop3(Pop3Event::ConnectionStart) + | EventType::Http(HttpEvent::ConnectionStart) + | EventType::Delivery(DeliveryEvent::AttemptStart) + ), + EventClass::SpanEnd => matches!( + event, + EventType::Smtp(SmtpEvent::ConnectionEnd) + | EventType::Imap(ImapEvent::ConnectionEnd) + | EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) + | EventType::Pop3(Pop3Event::ConnectionEnd) + | EventType::Http(HttpEvent::ConnectionEnd) + | EventType::Delivery(DeliveryEvent::AttemptEnd) + ), + EventClass::RawIo => matches!( + event, + EventType::Imap(ImapEvent::RawInput | ImapEvent::RawOutput) + | EventType::Smtp(SmtpEvent::RawInput | SmtpEvent::RawOutput) + | EventType::Pop3(Pop3Event::RawInput | Pop3Event::RawOutput) + | EventType::ManageSieve( + ManageSieveEvent::RawInput | ManageSieveEvent::RawOutput + ) + | EventType::Delivery(DeliveryEvent::RawInput | DeliveryEvent::RawOutput) + | EventType::Milter(MilterEvent::Read | MilterEvent::Write) + ), + EventClass::Span => { + EventClass::SpanStart.contains(event) || EventClass::SpanEnd.contains(event) + } + EventClass::Uncounted => { + EventClass::SpanEnd.contains(event) || EventClass::RawIo.contains(event) + } + } + } + + const fn events(self) -> EventSet { + let mut set = [0usize; TOTAL_EVENT_COUNT.div_ceil(USIZE_BITS)]; + let variants = EventType::variants(); + let mut idx = 0; + while idx < variants.len() { + if self.contains(variants[idx]) { + let id = variants[idx].to_id() as usize; + set[id / USIZE_BITS] |= 1 << (id & USIZE_BITS_MASK); + } + idx += 1; + } + Bitset(set) + } +} + impl EventType { #[inline(always)] pub fn is_span_start(&self) -> bool { - matches!( - self, - EventType::Smtp(SmtpEvent::ConnectionStart) - | EventType::Imap(ImapEvent::ConnectionStart) - | EventType::ManageSieve(ManageSieveEvent::ConnectionStart) - | EventType::Pop3(Pop3Event::ConnectionStart) - | EventType::Http(HttpEvent::ConnectionStart) - | EventType::Delivery(DeliveryEvent::AttemptStart) - ) + SPAN_START_EVENTS.get(self.to_id() as usize) } #[inline(always)] pub fn is_span_end(&self) -> bool { - matches!( - self, - EventType::Smtp(SmtpEvent::ConnectionEnd) - | EventType::Imap(ImapEvent::ConnectionEnd) - | EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) - | EventType::Pop3(Pop3Event::ConnectionEnd) - | EventType::Http(HttpEvent::ConnectionEnd) - | EventType::Delivery(DeliveryEvent::AttemptEnd) - ) + SPAN_END_EVENTS.get(self.to_id() as usize) } + pub const fn span_end(&self) -> Option { + match self { + EventType::Smtp(SmtpEvent::ConnectionStart) => { + Some(EventType::Smtp(SmtpEvent::ConnectionEnd)) + } + EventType::Imap(ImapEvent::ConnectionStart) => { + Some(EventType::Imap(ImapEvent::ConnectionEnd)) + } + EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => { + Some(EventType::ManageSieve(ManageSieveEvent::ConnectionEnd)) + } + EventType::Pop3(Pop3Event::ConnectionStart) => { + Some(EventType::Pop3(Pop3Event::ConnectionEnd)) + } + EventType::Http(HttpEvent::ConnectionStart) => { + Some(EventType::Http(HttpEvent::ConnectionEnd)) + } + EventType::Delivery(DeliveryEvent::AttemptStart) => { + Some(EventType::Delivery(DeliveryEvent::AttemptEnd)) + } + _ => None, + } + } + + #[inline(always)] pub fn is_raw_io(&self) -> bool { - matches!( - self, - EventType::Imap(ImapEvent::RawInput | ImapEvent::RawOutput) - | EventType::Smtp(SmtpEvent::RawInput | SmtpEvent::RawOutput) - | EventType::Pop3(Pop3Event::RawInput | Pop3Event::RawOutput) - | EventType::ManageSieve(ManageSieveEvent::RawInput | ManageSieveEvent::RawOutput) - | EventType::Delivery(DeliveryEvent::RawInput | DeliveryEvent::RawOutput) - | EventType::Milter(MilterEvent::Read | MilterEvent::Write) - ) + RAW_IO_EVENTS.get(self.to_id() as usize) } #[inline(always)] diff --git a/crates/trc/src/ipc/bitset.rs b/crates/trc/src/ipc/bitset.rs index 716ccd09..9d38da23 100644 --- a/crates/trc/src/ipc/bitset.rs +++ b/crates/trc/src/ipc/bitset.rs @@ -19,6 +19,10 @@ impl Bitset { Self([usize::MAX; N]) } + pub const fn insert(&mut self, index: usize) { + self.0[index / USIZE_BITS] |= 1 << (index & USIZE_BITS_MASK); + } + #[inline(always)] pub fn set(&mut self, index: impl Into) { let index = index.into(); diff --git a/crates/trc/src/ipc/channel.rs b/crates/trc/src/ipc/channel.rs index 9e85a8b8..bb4eef6e 100644 --- a/crates/trc/src/ipc/channel.rs +++ b/crates/trc/src/ipc/channel.rs @@ -6,6 +6,7 @@ use std::{ cell::UnsafeCell, + collections::VecDeque, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -24,25 +25,26 @@ use super::collector::{Collector, CollectorThread}; pub(crate) static CHANNEL_FLAGS: AtomicU64 = AtomicU64::new(0); pub(crate) const CHANNEL_SIZE: usize = 10240; pub(crate) const CHANNEL_UPDATE_MARKER: u64 = 1 << 63; +const OVERFLOW_SIZE: usize = CHANNEL_SIZE * 2; thread_local! { static EVENT_TX: UnsafeCell = { - // Create channel. let (tx, rx) = RingBuffer::new(CHANNEL_SIZE); + let dropped = Arc::new(AtomicU64::new(0)); - // Register receiver with collector. - COLLECTOR_UPDATES.lock().push(Update::RegisterReceiver { receiver: Receiver { rx } }); + COLLECTOR_UPDATES.lock().push(Update::RegisterReceiver { + receiver: Receiver { rx, dropped: dropped.clone() }, + }); - // Spawn collector thread. let collector = COLLECTOR_THREAD.clone(); CHANNEL_FLAGS.fetch_or(CHANNEL_UPDATE_MARKER, Ordering::Relaxed); collector.thread().unpark(); - // Return sender. UnsafeCell::new(Sender { tx, collector, - overflow: Vec::with_capacity(0), + overflow: VecDeque::new(), + dropped, }) }; } @@ -50,11 +52,13 @@ thread_local! { pub struct Sender { tx: Producer>, collector: Arc, - overflow: Vec>, + overflow: VecDeque>, + dropped: Arc, } pub struct Receiver { rx: Consumer>, + dropped: Arc, } #[derive(Debug)] @@ -62,22 +66,58 @@ pub struct ChannelError; impl Sender { pub fn send(&mut self, event: Event) -> Result<(), ChannelError> { - while let Some(event) = self.overflow.pop() { - if let Err(PushError::Full(event)) = self.tx.push(event) { - self.overflow.push(event); + if self.overflow.is_empty() { + match self.tx.push(event) { + Ok(()) => Ok(()), + Err(PushError::Full(event)) => self.spill(event), + } + } else { + self.send_with_overflow(event) + } + } + + #[cold] + fn send_with_overflow(&mut self, event: Event) -> Result<(), ChannelError> { + while let Some(pending) = self.overflow.pop_front() { + if let Err(PushError::Full(pending)) = self.tx.push(pending) { + self.overflow.push_front(pending); + return self.spill(event); + } + } + + match self.tx.push(event) { + Ok(()) => Ok(()), + Err(PushError::Full(event)) => self.spill(event), + } + } + + #[cold] + fn spill(&mut self, event: Event) -> Result<(), ChannelError> { + if self.overflow.len() < OVERFLOW_SIZE { + self.overflow.push_back(event); + Ok(()) + } else { + self.dropped.fetch_add(1, Ordering::Relaxed); + Err(ChannelError) + } + } +} + +impl Drop for Sender { + fn drop(&mut self) { + if self.overflow.is_empty() { + return; + } + + while let Some(event) = self.overflow.pop_front() { + if self.tx.push(event).is_err() { + self.dropped + .fetch_add(self.overflow.len() as u64 + 1, Ordering::Relaxed); break; } } - - if let Err(PushError::Full(event)) = self.tx.push(event) { - if self.overflow.len() <= CHANNEL_SIZE * 2 { - self.overflow.push(event); - } else { - return Err(ChannelError); - } - } - - Ok(()) + CHANNEL_FLAGS.fetch_or(1, Ordering::Release); + self.collector.thread().unpark(); } } @@ -94,6 +134,14 @@ impl Receiver { } } } + + pub fn is_closed(&self) -> bool { + self.rx.is_abandoned() && self.rx.is_empty() + } + + pub fn take_dropped(&self) -> u64 { + self.dropped.swap(0, Ordering::Relaxed) + } } impl Event { @@ -101,8 +149,7 @@ impl Event { // SAFETY: EVENT_TX is thread-local. let _ = EVENT_TX.try_with(|tx| unsafe { let tx = &mut *tx.get(); - if tx.send(self).is_ok() { - CHANNEL_FLAGS.fetch_add(1, Ordering::Relaxed); + if tx.send(self).is_ok() && CHANNEL_FLAGS.fetch_or(1, Ordering::Release) == 0 { tx.collector.thread().unpark(); } }); diff --git a/crates/trc/src/ipc/collector.rs b/crates/trc/src/ipc/collector.rs index a031d002..e82d2baa 100644 --- a/crates/trc/src/ipc/collector.rs +++ b/crates/trc/src/ipc/collector.rs @@ -6,12 +6,13 @@ use std::{ sync::{Arc, LazyLock, atomic::Ordering}, - thread::{Builder, JoinHandle, park}, - time::SystemTime, + thread::{Builder, JoinHandle, park_timeout}, + time::{Duration, SystemTime}, }; use ahash::AHashMap; use atomics::bitset::AtomicBitset; +use event::SPAN_EVENTS; use ipc::{ USIZE_BITS, channel::{CHANNEL_FLAGS, CHANNEL_UPDATE_MARKER, Receiver}, @@ -54,9 +55,17 @@ pub(crate) enum Update { pub struct Collector { receivers: Vec, + dispatcher: Dispatcher, + dropped: u64, + next_maintenance: u64, +} + +struct Dispatcher { subscribers: Vec, levels: [Level; TOTAL_EVENT_COUNT], active_spans: AHashMap>>, + discarded_spans: u64, + next_span_pressure_sweep: u64, } const HTTP_CONN_START: usize = EventType::Http(HttpEvent::ConnectionStart).to_id() as usize; @@ -74,8 +83,14 @@ const MANAGE_SIEVE_CONN_END: usize = const EV_ATTEMPT_START: usize = EventType::Delivery(DeliveryEvent::AttemptStart).to_id() as usize; const EV_ATTEMPT_END: usize = EventType::Delivery(DeliveryEvent::AttemptEnd).to_id() as usize; -const STALE_SPAN_CHECK_WATERMARK: usize = 8000; -const SPAN_MAX_HOLD: u64 = 60 * 60 * 24; // 1 day +const EVENTS_DROPPED: EventType = EventType::Telemetry(TelemetryEvent::EventsDropped); + +pub const SPAN_MAX_HOLD: u64 = 60 * 60 * 24; +const SPAN_PRESSURE_HOLD: u64 = 60 * 60; +const MAX_ACTIVE_SPANS: usize = 1 << 17; +const MAINTENANCE_INTERVAL: u64 = 60; +const FLUSH_BATCH_SIZE: usize = 1024; +const MAX_COALESCED_PASSES: u32 = 64; pub(crate) static COLLECTOR_THREAD: LazyLock> = LazyLock::new(|| { Arc::new( @@ -90,15 +105,13 @@ pub(crate) static COLLECTOR_THREAD: LazyLock> = LazyLock::n impl Collector { fn collect(&mut self) { - let mut do_continue = true; - - // Update - self.update(); + let mut do_continue = self.update(); + let mut coalesced_passes = 0u32; while do_continue { - match CHANNEL_FLAGS.swap(0, Ordering::Relaxed) { - 0 => { - park(); + match CHANNEL_FLAGS.swap(0, Ordering::Acquire) { + 0 if coalesced_passes == 0 => { + park_timeout(Duration::from_secs(MAINTENANCE_INTERVAL)); } CHANNEL_UPDATE_MARKER..=u64::MAX => { do_continue = self.update(); @@ -106,105 +119,22 @@ impl Collector { _ => {} } - // Collect all events - let mut closed_rxs = Vec::new(); - for (rx_idx, rx) in self.receivers.iter_mut().enumerate() { - let timestamp = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let mut has_closed = false; + let mut drained = false; + for rx in self.receivers.iter_mut() { loop { match rx.try_recv() { Ok(Some(event)) => { - // Build event - let event_id = event.inner.to_id() as usize; - let mut event = Event { - inner: EventDetails { - level: self.levels[event_id], - typ: event.inner, - timestamp, - span: None, - }, - keys: event.keys, - }; - - // Track spans - let event = match event_id { - HTTP_CONN_START - | IMAP_CONN_START - | POP3_CONN_START - | SMTP_CONN_START - | MANAGE_SIEVE_CONN_START - | EV_ATTEMPT_START => { - let event = Arc::new(event); - self.active_spans.insert( - event.span_id().unwrap_or_else(|| { - panic!("Missing span ID: {event:?}") - }), - event.clone(), - ); - - if self.active_spans.len() > STALE_SPAN_CHECK_WATERMARK { - self.active_spans.retain(|_, span| { - timestamp.saturating_sub(span.inner.timestamp) - < SPAN_MAX_HOLD - }); - } - event - } - - HTTP_CONN_END - | IMAP_CONN_END - | POP3_CONN_END - | SMTP_CONN_END - | MANAGE_SIEVE_CONN_END - | EV_ATTEMPT_END => { - if let Some(span) = self - .active_spans - .remove(&event.span_id().expect("Missing span ID")) - { - event.inner.span = Some(span.clone()); - } else { - #[cfg(any(feature = "dev_mode", feature = "test_mode"))] - { - if event.span_id().unwrap() != 0 { - eprintln!("Unregistered span ID: {event:?}"); - } - } - } - Arc::new(event) - } - _ => { - if let Some(span_id) = event.span_id() { - if let Some(span) = self.active_spans.get(&span_id) { - event.inner.span = Some(span.clone()); - } else { - #[cfg(any( - feature = "dev_mode", - feature = "test_mode" - ))] - { - if span_id != 0 { - eprintln!("Unregistered span ID: {event:?}"); - } - } - } - } - - Arc::new(event) - } - }; - - // Send to subscribers - for subscriber in self.subscribers.iter_mut() { - subscriber.push_event(event_id, event.clone()); - } - } - Ok(None) => { - break; + drained = true; + self.dispatcher.dispatch(event, timestamp); } + Ok(None) => break, Err(_) => { - closed_rxs.push(rx_idx); // Channel is closed, remove. + has_closed = true; break; } } @@ -212,31 +142,71 @@ impl Collector { } if do_continue { - // Remove closed receivers (should be rare in Tokio) - if !closed_rxs.is_empty() { - let mut receivers = Vec::with_capacity(self.receivers.len() - closed_rxs.len()); - for (rx_idx, rx) in self.receivers.drain(..).enumerate() { - if !closed_rxs.contains(&rx_idx) { - receivers.push(rx); + if has_closed { + let dropped = &mut self.dropped; + self.receivers.retain(|rx| { + if rx.is_closed() { + *dropped += rx.take_dropped(); + false + } else { + true } - } - self.receivers = receivers; + }); } - // Send batched events - if !self.subscribers.is_empty() { - self.subscribers - .retain_mut(|subscriber| subscriber.send_batch().is_ok()); + if timestamp >= self.next_maintenance { + self.maintenance(timestamp); } + + if drained + && coalesced_passes < MAX_COALESCED_PASSES + && self + .dispatcher + .subscribers + .iter() + .all(|subscriber| subscriber.batch.len() < FLUSH_BATCH_SIZE) + { + coalesced_passes += 1; + continue; + } + coalesced_passes = 0; + + let dropped = &mut self.dropped; + self.dispatcher.subscribers.retain_mut(|subscriber| { + match subscriber.send_batch() { + Ok(lost) => { + *dropped += lost; + true + } + Err(_) => false, + } + }); } } - // Send remaining events - for mut subscriber in self.subscribers.drain(..) { + for mut subscriber in self.dispatcher.subscribers.drain(..) { let _ = subscriber.send_batch(); } } + fn maintenance(&mut self, timestamp: u64) { + self.next_maintenance = timestamp + MAINTENANCE_INTERVAL; + + self.dispatcher.retain_spans(SPAN_MAX_HOLD, timestamp); + let discarded_spans = std::mem::take(&mut self.dispatcher.discarded_spans); + + let dropped = self + .receivers + .iter() + .map(Receiver::take_dropped) + .sum::() + + std::mem::take(&mut self.dropped); + if dropped > 0 || discarded_spans > 0 { + self.dispatcher + .report_dropped(dropped + discarded_spans, timestamp); + } + } + fn update(&mut self) -> bool { for update in COLLECTOR_UPDATES.lock().drain(..) { match update { @@ -245,33 +215,32 @@ impl Collector { } Update::RegisterSubscriber { subscriber } => { ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone()); - self.subscribers.push(subscriber); + self.dispatcher.subscribers.push(subscriber); } Update::UnregisterSubscriber { id } => { ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id); - self.subscribers.retain(|s| s.id != id); + self.dispatcher.subscribers.retain(|s| s.id != id); } Update::UpdateSubscriber { id, interests, lossy, } => { - for subscriber in self.subscribers.iter_mut() { - if subscriber.id == id { - subscriber.interests = interests; - subscriber.lossy = lossy; - break; - } + if let Some(subscriber) = self + .dispatcher + .subscribers + .iter_mut() + .find(|subscriber| subscriber.id == id) + { + subscriber.shared_interests.update(&interests); + subscriber.interests = interests; + subscriber.lossy = lossy; } } Update::UpdateLevels { levels } => { for event in EVENT_TYPES.iter() { - let event_id = event.to_id() as usize; - if let Some(level) = levels.get(event) { - self.levels[event_id] = *level; - } else { - self.levels[event_id] = event.level(); - } + self.dispatcher.levels[event.to_id() as usize] = + levels.get(event).copied().unwrap_or_else(|| event.level()); } } Update::Shutdown => return false, @@ -283,11 +252,7 @@ impl Collector { pub fn set_interests(mut interests: Interests) { if !interests.is_empty() { - for event_type in EVENT_TYPES.iter() { - if event_type.is_span_start() || event_type.is_span_end() { - interests.set(*event_type); - } - } + interests.union(&SPAN_EVENTS); } TRACE_INTERESTS.update(interests); @@ -341,20 +306,140 @@ impl Collector { } } -impl Default for Collector { - fn default() -> Self { - let mut c = Collector { - subscribers: Vec::new(), - levels: [Level::Disable; TOTAL_EVENT_COUNT], - active_spans: AHashMap::new(), - receivers: Vec::new(), +impl Dispatcher { + #[inline(always)] + fn dispatch(&mut self, event: Event, timestamp: u64) { + let event_id = event.inner.to_id() as usize; + let mut event = Event { + inner: EventDetails { + level: self.levels[event_id], + typ: event.inner, + timestamp, + span: None, + }, + keys: event.keys, }; - for event in EVENT_TYPES.iter() { - let event_id = event.to_id() as usize; - c.levels[event_id] = event.level(); + let event = match event_id { + HTTP_CONN_START + | IMAP_CONN_START + | POP3_CONN_START + | SMTP_CONN_START + | MANAGE_SIEVE_CONN_START + | EV_ATTEMPT_START => { + let event = Arc::new(event); + match event.span_id() { + Some(span_id) => self.track_span(span_id, &event, timestamp), + None => missing_span_id(&event), + } + event + } + HTTP_CONN_END + | IMAP_CONN_END + | POP3_CONN_END + | SMTP_CONN_END + | MANAGE_SIEVE_CONN_END + | EV_ATTEMPT_END => { + match event.span_id() { + Some(span_id) => match self.active_spans.remove(&span_id) { + Some(span) => event.inner.span = Some(span), + None => unregistered_span_id(span_id, &event), + }, + None => missing_span_id(&event), + } + Arc::new(event) + } + _ => { + if let Some(span_id) = event.span_id() { + match self.active_spans.get(&span_id) { + Some(span) => event.inner.span = Some(span.clone()), + None => unregistered_span_id(span_id, &event), + } + } + Arc::new(event) + } + }; + + for subscriber in self.subscribers.iter_mut() { + subscriber.push_event(event_id, &event); + } + } + + fn track_span(&mut self, span_id: u64, event: &Arc>, timestamp: u64) { + if self.active_spans.len() >= MAX_ACTIVE_SPANS { + if timestamp < self.next_span_pressure_sweep { + self.discarded_spans += 1; + return; + } + self.next_span_pressure_sweep = timestamp + MAINTENANCE_INTERVAL; + self.retain_spans(SPAN_PRESSURE_HOLD, timestamp); + if self.active_spans.len() >= MAX_ACTIVE_SPANS { + self.discarded_spans += 1; + return; + } } - c + self.active_spans.insert(span_id, event.clone()); + } + + fn retain_spans(&mut self, max_age: u64, timestamp: u64) { + let discarded = &mut self.discarded_spans; + self.active_spans.retain(|_, span| { + let keep = timestamp.saturating_sub(span.inner.timestamp) < max_age; + *discarded += u64::from(!keep); + keep + }); + } + + fn report_dropped(&mut self, dropped: u64, timestamp: u64) { + let event_id = EVENTS_DROPPED.to_id() as usize; + if Collector::is_metric(event_id) { + Collector::update_event_counter( + EVENTS_DROPPED, + u32::try_from(dropped).unwrap_or(u32::MAX), + ); + } + if Collector::has_interest(event_id) { + self.dispatch( + Event::with_keys(EVENTS_DROPPED, vec![(Key::Total, Value::UInt(dropped))]), + timestamp, + ); + } + } +} + +#[inline(always)] +fn missing_span_id(_event: &Event) { + #[cfg(any(feature = "dev_mode", stalwart_tests))] + eprintln!("Missing span ID: {_event:?}"); +} + +#[inline(always)] +fn unregistered_span_id(_span_id: u64, _event: &Event) { + #[cfg(any(feature = "dev_mode", stalwart_tests))] + if _span_id != 0 { + eprintln!("Unregistered span ID: {_event:?}"); + } +} + +impl Default for Collector { + fn default() -> Self { + let mut levels = [Level::Disable; TOTAL_EVENT_COUNT]; + for event in EVENT_TYPES.iter() { + levels[event.to_id() as usize] = event.level(); + } + + Collector { + receivers: Vec::new(), + dispatcher: Dispatcher { + subscribers: Vec::new(), + levels, + active_spans: AHashMap::new(), + discarded_spans: 0, + next_span_pressure_sweep: 0, + }, + dropped: 0, + next_maintenance: 0, + } } } diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index bef53026..916bc240 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -7,6 +7,7 @@ use std::sync::atomic::Ordering; use atomics::{array::AtomicU32Array, gauge::AtomicGauge, histogram::AtomicHistogram}; +use event::UNCOUNTED_EVENTS; use ipc::{ collector::{Collector, GlobalInterests}, subscriber::Interests, @@ -21,8 +22,8 @@ static CONNECTION_METRICS: [ConnectionMetrics; TOTAL_CONN_TYPES] = init_conn_met static MESSAGE_INGESTION_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::MessageIngestTime); -static MESSAGE_INDEX_TIME: AtomicHistogram<12> = - AtomicHistogram::<10>::new_short_durations(MetricType::MessageIngestIndexTime); +static SEARCH_INDEX_TIME: AtomicHistogram<12> = + AtomicHistogram::<10>::new_short_durations(MetricType::SearchIndexIndexTime); static MESSAGE_DELIVERY_TIME: AtomicHistogram<12> = AtomicHistogram::<18>::new_long_durations(MetricType::DeliveryTotalTime); @@ -45,10 +46,30 @@ static STORE_BLOB_WRITE_TIME: AtomicHistogram<12> = static DNS_LOOKUP_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::DnsLookupTime); + static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory); static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount); static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount); static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount); +static CACHE_MESSAGES_SIZE: AtomicGauge = AtomicGauge::new(MetricType::CacheMessagesSize); +static CACHE_MESSAGES_LARGE_SIZE: AtomicGauge = + AtomicGauge::new(MetricType::CacheMessagesLargeSize); +static CACHE_SIZE: AtomicGauge = AtomicGauge::new(MetricType::CacheSize); +static CACHE_SWAP_PENDING: AtomicGauge = AtomicGauge::new(MetricType::CacheSwapPending); +static SERVER_BUFFERED_MEMORY_SMTP: AtomicGauge = + AtomicGauge::new(MetricType::ServerBufferedMemorySmtp); +static SERVER_BUFFERED_MEMORY_READ: AtomicGauge = + AtomicGauge::new(MetricType::ServerBufferedMemoryRead); +static SEARCH_INDEX_PENDING_EMAIL: AtomicGauge = + AtomicGauge::new(MetricType::SearchIndexPendingEmail); +static SEARCH_INDEX_PENDING_CALENDAR: AtomicGauge = + AtomicGauge::new(MetricType::SearchIndexPendingCalendar); +static SEARCH_INDEX_PENDING_CONTACTS: AtomicGauge = + AtomicGauge::new(MetricType::SearchIndexPendingContacts); +static SEARCH_INDEX_PENDING_FILE: AtomicGauge = + AtomicGauge::new(MetricType::SearchIndexPendingFile); +static TASK_MANAGER_PENDING: AtomicGauge = AtomicGauge::new(MetricType::TaskManagerPending); +static TASK_MANAGER_DUE: AtomicGauge = AtomicGauge::new(MetricType::TaskManagerDue); const CONN_SMTP_IN: usize = 0; const CONN_SMTP_OUT: usize = 1; @@ -71,7 +92,7 @@ pub struct EventCounter { impl Collector { pub fn record_metric(event: EventType, event_id: usize, keys: &[(Key, Value)]) { // Increment the event counter - if !event.is_span_end() && !event.is_raw_io() { + if !UNCOUNTED_EVENTS.get(event_id) { EVENT_COUNTERS.add(event_id, 1); } @@ -178,8 +199,8 @@ impl Collector { EventType::Queue(QueueEvent::AutogeneratedQueued | QueueEvent::DsnQueued) => { QUEUE_COUNT.increment(); } - EventType::MessageIngest(MessageIngestEvent::SearchIndex) => { - MESSAGE_INDEX_TIME.observe(elapsed); + EventType::SearchIndex(SearchIndexEvent::BatchIndexed) => { + SEARCH_INDEX_TIME.observe(elapsed); } EventType::Store(StoreEvent::BlobWrite) => { STORE_BLOB_WRITE_TIME.observe(elapsed); @@ -193,7 +214,6 @@ impl Collector { EventType::Store(StoreEvent::DataIterate) => { STORE_DATA_READ_TIME.observe(elapsed); } - _ => {} } } @@ -226,8 +246,25 @@ impl Collector { } pub fn collect_gauges(is_enterprise: bool) -> impl Iterator { - static E_GAUGES: &[&AtomicGauge] = - &[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT]; + #[cfg(not(feature = "enterprise"))] + static E_GAUGES: &[&AtomicGauge] = &[ + &SERVER_MEMORY, + &QUEUE_COUNT, + &USER_COUNT, + &DOMAIN_COUNT, + &CACHE_MESSAGES_SIZE, + &CACHE_MESSAGES_LARGE_SIZE, + &CACHE_SIZE, + &CACHE_SWAP_PENDING, + &SERVER_BUFFERED_MEMORY_SMTP, + &SERVER_BUFFERED_MEMORY_READ, + &SEARCH_INDEX_PENDING_EMAIL, + &SEARCH_INDEX_PENDING_CALENDAR, + &SEARCH_INDEX_PENDING_CONTACTS, + &SEARCH_INDEX_PENDING_FILE, + &TASK_MANAGER_PENDING, + &TASK_MANAGER_DUE, + ]; static C_GAUGES: &[&AtomicGauge] = &[&SERVER_MEMORY, &USER_COUNT, &DOMAIN_COUNT]; if is_enterprise { E_GAUGES } else { C_GAUGES } @@ -239,9 +276,10 @@ impl Collector { pub fn collect_histograms( is_enterprise: bool, ) -> impl Iterator> { + #[cfg(not(feature = "enterprise"))] static E_HISTOGRAMS: &[&AtomicHistogram<12>] = &[ &MESSAGE_INGESTION_TIME, - &MESSAGE_INDEX_TIME, + &SEARCH_INDEX_TIME, &MESSAGE_DELIVERY_TIME, &MESSAGE_INCOMING_SIZE, &MESSAGE_SUBMISSION_SIZE, @@ -278,7 +316,7 @@ impl Collector { match metric_type { MetricType::ServerMemory => SERVER_MEMORY.get() as f64, MetricType::MessageIngestTime => MESSAGE_INGESTION_TIME.average(), - MetricType::MessageIngestIndexTime => MESSAGE_INDEX_TIME.average(), + MetricType::SearchIndexIndexTime => SEARCH_INDEX_TIME.average(), MetricType::MessageSize => MESSAGE_INCOMING_SIZE.average(), MetricType::MessageAuthenticatedSize => MESSAGE_SUBMISSION_SIZE.average(), MetricType::DeliveryTotalTime => MESSAGE_DELIVERY_TIME.average(), @@ -315,6 +353,18 @@ impl Collector { MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(), MetricType::UserCount => USER_COUNT.get() as f64, MetricType::DomainCount => DOMAIN_COUNT.get() as f64, + MetricType::CacheMessagesSize => CACHE_MESSAGES_SIZE.get() as f64, + MetricType::CacheMessagesLargeSize => CACHE_MESSAGES_LARGE_SIZE.get() as f64, + MetricType::CacheSize => CACHE_SIZE.get() as f64, + MetricType::CacheSwapPending => CACHE_SWAP_PENDING.get() as f64, + MetricType::ServerBufferedMemorySmtp => SERVER_BUFFERED_MEMORY_SMTP.get() as f64, + MetricType::ServerBufferedMemoryRead => SERVER_BUFFERED_MEMORY_READ.get() as f64, + MetricType::SearchIndexPendingEmail => SEARCH_INDEX_PENDING_EMAIL.get() as f64, + MetricType::SearchIndexPendingCalendar => SEARCH_INDEX_PENDING_CALENDAR.get() as f64, + MetricType::SearchIndexPendingContacts => SEARCH_INDEX_PENDING_CONTACTS.get() as f64, + MetricType::SearchIndexPendingFile => SEARCH_INDEX_PENDING_FILE.get() as f64, + MetricType::TaskManagerPending => TASK_MANAGER_PENDING.get() as f64, + MetricType::TaskManagerDue => TASK_MANAGER_DUE.get() as f64, _ => EVENT_COUNTERS.get(metric_type.event_id()) as f64, } } @@ -325,6 +375,18 @@ impl Collector { MetricType::QueueCount => QUEUE_COUNT.set(value), MetricType::UserCount => USER_COUNT.set(value), MetricType::DomainCount => DOMAIN_COUNT.set(value), + MetricType::CacheMessagesSize => CACHE_MESSAGES_SIZE.set(value), + MetricType::CacheMessagesLargeSize => CACHE_MESSAGES_LARGE_SIZE.set(value), + MetricType::CacheSize => CACHE_SIZE.set(value), + MetricType::CacheSwapPending => CACHE_SWAP_PENDING.set(value), + MetricType::ServerBufferedMemorySmtp => SERVER_BUFFERED_MEMORY_SMTP.set(value), + MetricType::ServerBufferedMemoryRead => SERVER_BUFFERED_MEMORY_READ.set(value), + MetricType::SearchIndexPendingEmail => SEARCH_INDEX_PENDING_EMAIL.set(value), + MetricType::SearchIndexPendingCalendar => SEARCH_INDEX_PENDING_CALENDAR.set(value), + MetricType::SearchIndexPendingContacts => SEARCH_INDEX_PENDING_CONTACTS.set(value), + MetricType::SearchIndexPendingFile => SEARCH_INDEX_PENDING_FILE.set(value), + MetricType::TaskManagerPending => TASK_MANAGER_PENDING.set(value), + MetricType::TaskManagerDue => TASK_MANAGER_DUE.set(value), _ => {} } } @@ -336,7 +398,7 @@ impl Collector { pub fn update_histogram(metric_type: MetricType, value: u64) { match metric_type { MetricType::MessageIngestTime => MESSAGE_INGESTION_TIME.observe(value), - MetricType::MessageIngestIndexTime => MESSAGE_INDEX_TIME.observe(value), + MetricType::SearchIndexIndexTime => SEARCH_INDEX_TIME.observe(value), MetricType::DeliveryTotalTime => MESSAGE_DELIVERY_TIME.observe(value), MetricType::DeliveryAttemptTime => { CONNECTION_METRICS[CONN_SMTP_OUT].elapsed.observe(value) diff --git a/crates/trc/src/ipc/subscriber.rs b/crates/trc/src/ipc/subscriber.rs index 7a9c588f..dd1c6aed 100644 --- a/crates/trc/src/ipc/subscriber.rs +++ b/crates/trc/src/ipc/subscriber.rs @@ -8,7 +8,9 @@ use std::sync::Arc; use tokio::sync::mpsc::{self, error::TrySendError}; -use crate::{Event, EventDetails, EventType, Level, TOTAL_EVENT_COUNT}; +use crate::{Event, EventDetails, Level, TOTAL_EVENT_COUNT}; + +use crate::atomics::bitset::AtomicBitset; use super::{ USIZE_BITS, @@ -18,14 +20,16 @@ use super::{ }; const MAX_BATCH_SIZE: usize = 32768; +const MAX_LOSSLESS_BATCH_SIZE: usize = MAX_BATCH_SIZE * 16; pub type Interests = Box>; +pub type SharedInterests = Arc>; pub type EventBatch = Vec>>; -#[derive(Debug)] pub(crate) struct Subscriber { pub id: String, pub interests: Interests, + pub shared_interests: SharedInterests, pub tx: mpsc::Sender, pub lossy: bool, pub batch: EventBatch, @@ -39,33 +43,37 @@ pub struct SubscriberBuilder { impl Subscriber { #[inline(always)] - pub fn push_event(&mut self, event_id: usize, trace: Arc>) { + pub fn push_event(&mut self, event_id: usize, trace: &Arc>) { if self.interests.get(event_id) { - self.batch.push(trace); + self.batch.push(trace.clone()); } } - pub fn send_batch(&mut self) -> Result<(), ChannelError> { - if !self.batch.is_empty() { - match self - .tx - .try_send(std::mem::replace(&mut self.batch, Vec::with_capacity(128))) - { - Ok(_) => Ok(()), - Err(TrySendError::Full(mut events)) => { - if self.lossy && events.len() > MAX_BATCH_SIZE { + pub fn send_batch(&mut self) -> Result { + if self.batch.is_empty() { + return Ok(0); + } + + match self + .tx + .try_send(std::mem::replace(&mut self.batch, Vec::with_capacity(128))) + { + Ok(_) => Ok(0), + Err(TrySendError::Full(mut events)) => { + let total = events.len(); + if self.lossy { + if total > MAX_BATCH_SIZE { events.retain(|e| e.inner.level == Level::Error); - if events.len() > MAX_BATCH_SIZE { - events.truncate(MAX_BATCH_SIZE); - } + events.truncate(MAX_BATCH_SIZE); } - self.batch = events; - Ok(()) + } else if total > MAX_LOSSLESS_BATCH_SIZE { + events.truncate(MAX_LOSSLESS_BATCH_SIZE); } - Err(TrySendError::Closed(_)) => Err(ChannelError), + let dropped = (total - events.len()) as u64; + self.batch = events; + Ok(dropped) } - } else { - Ok(()) + Err(TrySendError::Closed(_)) => Err(ChannelError), } } } @@ -79,15 +87,6 @@ impl SubscriberBuilder { } } - pub fn with_default_interests(mut self, level: Level) -> Self { - for event in EventType::variants() { - if event.level() >= level { - self.interests.set(*event); - } - } - self - } - pub fn with_interests(mut self, interests: Interests) -> Self { self.interests = interests; self @@ -105,13 +104,22 @@ impl SubscriberBuilder { self } - pub fn register(self) -> (mpsc::Sender, mpsc::Receiver) { + pub fn register( + self, + ) -> ( + mpsc::Sender, + mpsc::Receiver, + SharedInterests, + ) { let (tx, rx) = mpsc::channel(8192); + let shared_interests = SharedInterests::new(AtomicBitset::new()); + shared_interests.update(&self.interests); COLLECTOR_UPDATES.lock().push(Update::RegisterSubscriber { subscriber: Subscriber { id: self.id, interests: self.interests, + shared_interests: shared_interests.clone(), tx: tx.clone(), lossy: self.lossy, batch: Vec::new(), @@ -121,6 +129,6 @@ impl SubscriberBuilder { // Notify collector Collector::reload(); - (tx, rx) + (tx, rx, shared_interests) } } diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index cefa91b1..e0a790d1 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -42,7 +42,7 @@ pub struct EventDetails { } #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] -#[repr(usize)] +#[repr(u8)] pub enum Level { Trace = 0, Debug = 1, diff --git a/crates/trc/src/serializers/escape.rs b/crates/trc/src/serializers/escape.rs new file mode 100644 index 00000000..e8a91743 --- /dev/null +++ b/crates/trc/src/serializers/escape.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const WORD: usize = 8; +const LSB: u64 = 0x0101_0101_0101_0101; +const MSB: u64 = 0x8080_8080_8080_8080; +const HEX: &[u8; 16] = b"0123456789abcdef"; +const HEX_ESCAPE: u8 = b'x'; + +static ESCAPES: [u8; 256] = { + let mut table = [0u8; 256]; + let mut byte = 0; + while byte < 0x20 { + table[byte] = HEX_ESCAPE; + byte += 1; + } + table[0x7f] = HEX_ESCAPE; + table[b'\r' as usize] = b'r'; + table[b'\n' as usize] = b'n'; + table[b'\t' as usize] = b't'; + table[b'\\' as usize] = b'\\'; + table[b'"' as usize] = b'"'; + table +}; + +#[inline(always)] +const fn has_zero_byte(word: u64) -> u64 { + word.wrapping_sub(LSB) & !word & MSB +} + +#[inline(always)] +const fn may_need_escape(word: u64) -> bool { + let below_space = word.wrapping_sub(LSB * 0x20) & !word & MSB; + (below_space + | has_zero_byte(word ^ (LSB * b'"' as u64)) + | has_zero_byte(word ^ (LSB * b'\\' as u64)) + | has_zero_byte(word ^ (LSB * 0x7f))) + != 0 +} + +pub fn escape_into(out: &mut Vec, input: &[u8]) { + let mut clean_start = 0; + let mut offset = 0; + let (chunks, remainder) = input.as_chunks::(); + + for chunk in chunks { + if may_need_escape(u64::from_le_bytes(*chunk)) { + clean_start = escape_bytes(out, input, chunk, offset, clean_start); + } + offset += WORD; + } + + clean_start = escape_bytes(out, input, remainder, offset, clean_start); + out.extend_from_slice(&input[clean_start..]); +} + +#[inline(always)] +fn escape_bytes( + out: &mut Vec, + input: &[u8], + bytes: &[u8], + offset: usize, + mut clean_start: usize, +) -> usize { + for (pos, &byte) in bytes.iter().enumerate() { + let escape = ESCAPES[byte as usize]; + if escape != 0 { + let pos = offset + pos; + out.extend_from_slice(&input[clean_start..pos]); + if escape == HEX_ESCAPE { + out.extend_from_slice(&[ + b'\\', + b'x', + HEX[(byte >> 4) as usize], + HEX[(byte & 0x0f) as usize], + ]); + } else { + out.extend_from_slice(&[b'\\', escape]); + } + clean_start = pos + 1; + } + } + clean_start +} + diff --git a/crates/trc/src/serializers/json.rs b/crates/trc/src/serializers/json.rs index 96f79f4e..221bbbe5 100644 --- a/crates/trc/src/serializers/json.rs +++ b/crates/trc/src/serializers/json.rs @@ -4,23 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Error, Event, EventDetails, EventType, Key, MetricType, Value}; -use ahash::AHashSet; -use base64::{Engine, engine::general_purpose::STANDARD}; -use mail_parser::DateTime; +use crate::{ + Error, Event, EventDetails, EventType, Key, MetricType, Value, event::KeySet, + serializers::timestamp::TimestampCache, +}; +use encodify::base64::STANDARD; use serde::{ Serialize, Serializer, ser::{SerializeMap, SerializeSeq}, }; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::{ + cell::{Cell, RefCell}, + sync::atomic::{AtomicU64, Ordering}, +}; static EVENT_ID_COUNTER: AtomicU64 = AtomicU64::new(0); -struct Keys<'x> { - keys: &'x [(Key, Value)], - span_keys: &'x [(Key, Value)], -} - pub struct JsonEventSerializer { inner: T, with_id: bool, @@ -28,6 +27,25 @@ pub struct JsonEventSerializer { with_description: bool, } +struct Context { + with_id: bool, + with_spans: bool, + with_description: bool, + next_id: Cell, + timestamps: RefCell, + value_timestamps: RefCell, +} + +struct Serialized<'x, T> { + ctx: &'x Context, + inner: T, +} + +struct Keys<'x> { + keys: &'x [(Key, Value)], + span_keys: &'x [(Key, Value)], +} + impl JsonEventSerializer { pub fn new(inner: T) -> Self { Self { @@ -56,6 +74,21 @@ impl JsonEventSerializer { pub fn into_inner(self) -> T { self.inner } + + fn context(&self, num_events: usize) -> Context { + Context { + with_id: self.with_id, + with_spans: self.with_spans, + with_description: self.with_description, + next_id: Cell::new(if self.with_id { + EVENT_ID_COUNTER.fetch_add(num_events as u64, Ordering::Relaxed) + } else { + 0 + }), + timestamps: RefCell::new(TimestampCache::new()), + value_timestamps: RefCell::new(TimestampCache::new()), + } + } } impl>> Serialize for JsonEventSerializer> { @@ -63,13 +96,12 @@ impl>> Serialize for JsonEventSerializer> { where S: Serializer, { + let ctx = self.context(self.inner.len()); let mut seq = serializer.serialize_seq(Some(self.inner.len()))?; for event in &self.inner { - seq.serialize_element(&JsonEventSerializer { - inner: event, - with_id: self.with_id, - with_spans: self.with_spans, - with_description: self.with_description, + seq.serialize_element(&Serialized { + ctx: &ctx, + inner: event.as_ref(), })?; } seq.end() @@ -81,68 +113,74 @@ impl>> Serialize for JsonEventSerializer { where S: Serializer, { - let event = self.inner.as_ref(); + Serialized { + ctx: &self.context(1), + inner: self.inner.as_ref(), + } + .serialize(serializer) + } +} + +impl Serialize for Serialized<'_, &Event> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let event = self.inner; let mut map = serializer.serialize_map(None)?; - if self.with_id { + if self.ctx.with_id { + let counter = self.ctx.next_id.get(); + self.ctx.next_id.set(counter.wrapping_add(1)); + let mut id = EventId::default(); map.serialize_entry( "id", - &format!( - "{}{}{}", - event.inner.timestamp, - EVENT_ID_COUNTER.fetch_add(1, Ordering::Relaxed), - event.inner.typ.to_id() - ), + id.format(event.inner.timestamp, counter, event.inner.typ.to_id()), )?; } - if self.with_description { + if self.ctx.with_description { map.serialize_entry("text", event.inner.typ.description())?; } map.serialize_entry( "createdAt", - &DateTime::from_timestamp(event.inner.timestamp as i64).to_rfc3339(), + self.ctx.timestamps.borrow_mut().get(event.inner.timestamp), )?; map.serialize_entry("type", event.inner.typ.as_str())?; map.serialize_entry( "data", - &JsonEventSerializer { + &Serialized { + ctx: self.ctx, inner: Keys { keys: event.keys.as_slice(), span_keys: event .inner .span .as_ref() - .map(|s| &s.keys[..]) - .unwrap_or(&[]), + .map_or(&[][..], |span| span.keys.as_slice()), }, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, }, )?; map.end() } } -impl Serialize for JsonEventSerializer> { +impl Serialize for Serialized<'_, Keys<'_>> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let keys_len = self.inner.keys.len() + self.inner.span_keys.len(); - let mut seen_keys = AHashSet::with_capacity(keys_len); + let mut seen_keys = KeySet::default(); let mut keys = serializer.serialize_map(Some(keys_len))?; for (key, value) in self.inner.keys.iter().chain(self.inner.span_keys.iter()) { if !matches!(value, Value::None) - && (self.with_spans || !matches!(key, Key::SpanId)) + && (self.ctx.with_spans || !matches!(key, Key::SpanId)) && seen_keys.insert(*key) { keys.serialize_entry( key.as_str(), - &JsonEventSerializer { + &Serialized { + ctx: self.ctx, inner: value, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, }, )?; } @@ -151,84 +189,90 @@ impl Serialize for JsonEventSerializer> { } } -impl Serialize for JsonEventSerializer<&Error> { +impl Serialize for Serialized<'_, &Error> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(None)?; map.serialize_entry("type", self.inner.0.inner.as_str())?; - if self.with_description { + if self.ctx.with_description { map.serialize_entry("text", self.inner.0.inner.description())?; } map.serialize_entry( "data", - &JsonEventSerializer { + &Serialized { + ctx: self.ctx, inner: Keys { keys: self.inner.0.keys.as_slice(), span_keys: &[], }, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, }, )?; map.end() } } -impl Serialize for JsonEventSerializer<&Value> { +impl Serialize for Serialized<'_, &Value> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { - match &self.inner { - Value::String(value) => value.serialize(serializer), - Value::UInt(value) => value.serialize(serializer), - Value::Int(value) => value.serialize(serializer), - Value::Float(value) => value.serialize(serializer), - Value::Timestamp(value) => DateTime::from_timestamp(*value as i64) - .to_rfc3339() - .serialize(serializer), - Value::Duration(value) => value.serialize(serializer), - Value::Bytes(value) => STANDARD.encode(value).serialize(serializer), - Value::Bool(value) => value.serialize(serializer), + match self.inner { + Value::String(value) => serializer.serialize_str(value.as_str()), + Value::UInt(value) => serializer.serialize_u64(*value), + Value::Int(value) => serializer.serialize_i64(*value), + Value::Float(value) => serializer.serialize_f64(*value), + Value::Timestamp(value) => { + serializer.serialize_str(self.ctx.value_timestamps.borrow_mut().get(*value)) + } + Value::Duration(value) => serializer.serialize_u64(*value), + Value::Bytes(value) => serializer.collect_str(&STANDARD.display(value)), + Value::Bool(value) => serializer.serialize_bool(*value), Value::Ipv4(value) => value.serialize(serializer), Value::Ipv6(value) => value.serialize(serializer), - Value::Event(value) => JsonEventSerializer { + Value::Event(value) => Serialized { + ctx: self.ctx, inner: value, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, } .serialize(serializer), - Value::Array(value) => JsonEventSerializer { - inner: value, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, + Value::Array(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&Serialized { + ctx: self.ctx, + inner: value, + })?; + } + seq.end() } - .serialize(serializer), - Value::None => unreachable!(), + Value::None => serializer.serialize_unit(), } } } -impl Serialize for JsonEventSerializer<&Vec> { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.inner.len()))?; - for value in self.inner { - seq.serialize_element(&JsonEventSerializer { - inner: value, - with_spans: self.with_spans, - with_description: self.with_description, - with_id: self.with_id, - })?; +struct EventId { + buf: [u8; 64], +} + +impl Default for EventId { + fn default() -> Self { + Self { buf: [0; 64] } + } +} + +impl EventId { + fn format(&mut self, timestamp: u64, counter: u64, type_id: u16) -> &str { + let mut len = 0; + for part in [ + itoa::Buffer::new().format(timestamp).as_bytes(), + itoa::Buffer::new().format(counter).as_bytes(), + itoa::Buffer::new().format(type_id).as_bytes(), + ] { + self.buf[len..len + part.len()].copy_from_slice(part); + len += part.len(); } - seq.end() + std::str::from_utf8(&self.buf[..len]).unwrap_or_default() } } @@ -269,3 +313,4 @@ impl<'de> serde::Deserialize<'de> for MetricType { Self::parse(s).ok_or_else(|| serde::de::Error::unknown_variant(s, &[])) } } + diff --git a/crates/trc/src/serializers/mod.rs b/crates/trc/src/serializers/mod.rs index 6c744fdf..220cb6a9 100644 --- a/crates/trc/src/serializers/mod.rs +++ b/crates/trc/src/serializers/mod.rs @@ -4,6 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod escape; pub mod json; pub mod text; +pub mod timestamp; +pub fn write_uint(out: &mut Vec, value: u64) { + out.extend_from_slice(itoa::Buffer::new().format(value).as_bytes()); +} + +pub fn write_int(out: &mut Vec, value: i64) { + out.extend_from_slice(itoa::Buffer::new().format(value).as_bytes()); +} diff --git a/crates/trc/src/serializers/text.rs b/crates/trc/src/serializers/text.rs index 1e86b8bd..89cb4d8c 100644 --- a/crates/trc/src/serializers/text.rs +++ b/crates/trc/src/serializers/text.rs @@ -4,18 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::fmt::Display; +use std::{fmt::Display, io::Write}; -use mail_parser::DateTime; -use tokio::io::{AsyncWrite, AsyncWriteExt}; +use crate::{ + Error, Event, EventDetails, Key, Level, Value, + event::KeySet, + serializers::{escape::escape_into, timestamp::TimestampCache, write_int, write_uint}, +}; +use encodify::base64::STANDARD; -use crate::{Error, Event, EventDetails, Key, Level, Value}; -use base64::{Engine, engine::general_purpose::STANDARD}; - -pub struct FmtWriter { - writer: T, +#[derive(Default)] +pub struct FmtWriter { ansi: bool, multiline: bool, + timestamps: TimestampCache, + value_timestamps: TimestampCache, } #[allow(dead_code)] @@ -30,13 +33,9 @@ enum Color { White, } -impl FmtWriter { - pub fn new(writer: T) -> Self { - Self { - writer, - ansi: false, - multiline: false, - } +impl FmtWriter { + pub fn new() -> Self { + Self::default() } pub fn with_ansi(self, ansi: bool) -> Self { @@ -47,241 +46,163 @@ impl FmtWriter { Self { multiline, ..self } } - pub async fn write(&mut self, event: &Event) -> std::io::Result<()> { - // Write timestamp - if self.ansi { - self.writer - .write_all(Color::White.as_code().as_bytes()) - .await?; - } - self.writer - .write_all( - DateTime::from_timestamp(event.inner.timestamp as i64) - .to_rfc3339() - .as_bytes(), - ) - .await?; - if self.ansi { - self.writer.write_all(Color::reset().as_bytes()).await?; - } - self.writer.write_all(" ".as_bytes()).await?; + pub fn write(&mut self, event: &Event, out: &mut Vec) { + let level_color = match event.inner.level { + Level::Error => Color::Red, + Level::Warn => Color::Yellow, + Level::Info => Color::Green, + Level::Debug => Color::Blue, + Level::Trace => Color::Magenta, + Level::Disable => return, + }; - // Write level if self.ansi { - self.writer - .write_all( - match event.inner.level { - Level::Error => Color::Red, - Level::Warn => Color::Yellow, - Level::Info => Color::Green, - Level::Debug => Color::Blue, - Level::Trace => Color::Magenta, - Level::Disable => return Ok(()), - } - .as_code_bold() - .as_bytes(), - ) - .await?; + out.extend_from_slice(Color::White.as_code().as_bytes()); } - self.writer - .write_all(event.inner.level.as_str().as_bytes()) - .await?; + out.extend_from_slice(self.timestamps.get(event.inner.timestamp).as_bytes()); if self.ansi { - self.writer.write_all(Color::reset().as_bytes()).await?; + out.extend_from_slice(Color::reset().as_bytes()); } - self.writer.write_all(" ".as_bytes()).await?; + out.push(b' '); - // Write message if self.ansi { - self.writer - .write_all(Color::White.as_code_bold().as_bytes()) - .await?; + out.extend_from_slice(level_color.as_code_bold().as_bytes()); } - self.writer - .write_all(event.inner.typ.description().as_bytes()) - .await?; + out.extend_from_slice(event.inner.level.as_str().as_bytes()); if self.ansi { - self.writer.write_all(Color::reset().as_bytes()).await?; + out.extend_from_slice(Color::reset().as_bytes()); } - self.writer.write_all(" (".as_bytes()).await?; - self.writer - .write_all(event.inner.typ.as_str().as_bytes()) - .await?; + out.push(b' '); - self.writer - .write_all(if self.multiline { ")\n" } else { ") " }.as_bytes()) - .await?; - - // Write keys - if let Some(parent_event) = &event.inner.span { - self.write_keys(&parent_event.keys, &event.keys, 1).await?; - } else { - self.write_keys(&[], &event.keys, 1).await?; + if self.ansi { + out.extend_from_slice(Color::White.as_code_bold().as_bytes()); } + out.extend_from_slice(event.inner.typ.description().as_bytes()); + if self.ansi { + out.extend_from_slice(Color::reset().as_bytes()); + } + out.extend_from_slice(b" ("); + out.extend_from_slice(event.inner.typ.as_str().as_bytes()); + out.extend_from_slice(if self.multiline { b")\n" } else { b") " }); + + let span_keys = event + .inner + .span + .as_ref() + .map_or(&[][..], |span| span.keys.as_slice()); + self.write_keys(out, span_keys, &event.keys, 1); if !self.multiline { - self.writer.write_all("\n".as_bytes()).await?; + out.push(b'\n'); } - - Ok(()) } - async fn write_keys( + fn write_keys( &mut self, + out: &mut Vec, span_keys: &[(Key, Value)], keys: &[(Key, Value)], indent: usize, - ) -> std::io::Result<()> { - Box::pin(async move { - let mut is_first = true; - for (key, value) in span_keys.iter().chain(keys.iter()) { - if matches!(key, Key::SpanId) { - continue; - } else if is_first { - is_first = false; - } else if !self.multiline { - self.writer.write_all(", ".as_bytes()).await?; - } + ) { + let mut event_keys = KeySet::default(); + if !span_keys.is_empty() { + for (key, _) in keys { + event_keys.insert(*key); + } + } - // Write key - if self.multiline { - for _ in 0..indent { - self.writer.write_all("\t".as_bytes()).await?; - } - } - if self.ansi { - self.writer - .write_all(Color::Cyan.as_code().as_bytes()) - .await?; - } - self.writer.write_all(key.as_str().as_bytes()).await?; - if self.ansi { - self.writer.write_all(Color::reset().as_bytes()).await?; - } - - // Write value - self.writer.write_all(" = ".as_bytes()).await?; - self.write_value(value, indent).await?; - - if self.multiline && !matches!(value, Value::Event(_)) { - self.writer.write_all("\n".as_bytes()).await?; - } + let mut is_first = true; + for (key, value) in span_keys + .iter() + .filter(|(key, _)| !event_keys.contains(*key)) + .chain(keys.iter()) + { + if matches!(key, Key::SpanId) { + continue; + } else if is_first { + is_first = false; + } else if !self.multiline { + out.extend_from_slice(b", "); } - Ok(()) - }) - .await - } - - async fn write_value(&mut self, value: &Value, indent: usize) -> std::io::Result<()> { - Box::pin(async move { - match value { - Value::String(v) => { - self.writer.write_all("\"".as_bytes()).await?; - for ch in v.as_bytes() { - match ch { - b'\r' => { - self.writer.write_all("\\r".as_bytes()).await?; - } - b'\n' => { - self.writer.write_all("\\n".as_bytes()).await?; - } - b'\t' => { - self.writer.write_all("\\t".as_bytes()).await?; - } - b'\\' => { - self.writer.write_all("\\\\".as_bytes()).await?; - } - _ => { - self.writer.write_all(&[*ch]).await?; - } - } - } - self.writer.write_all("\"".as_bytes()).await?; - } - Value::UInt(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - } - Value::Int(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - } - Value::Float(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - } - Value::Timestamp(v) => { - self.writer - .write_all(DateTime::from_timestamp(*v as i64).to_rfc3339().as_bytes()) - .await?; - } - Value::Duration(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - self.writer.write_all("ms".as_bytes()).await?; - } - Value::Bytes(bytes) => { - self.writer.write_all("base64:".as_bytes()).await?; - self.writer - .write_all(STANDARD.encode(bytes).as_bytes()) - .await?; - } - Value::Bool(true) => { - self.writer.write_all("true".as_bytes()).await?; - } - Value::Bool(false) => { - self.writer.write_all("false".as_bytes()).await?; - } - Value::Ipv4(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - } - Value::Ipv6(v) => { - self.writer.write_all(v.to_string().as_bytes()).await?; - } - Value::Event(e) => { - self.writer - .write_all(e.0.inner.description().as_bytes()) - .await?; - self.writer.write_all(" (".as_bytes()).await?; - self.writer.write_all(e.0.inner.as_str().as_bytes()).await?; - self.writer.write_all(")".as_bytes()).await?; - if !e.0.keys.is_empty() { - self.writer - .write_all(if self.multiline { "\n" } else { " { " }.as_bytes()) - .await?; - - self.write_keys(&e.0.keys, &[], indent + 1).await?; - - if !self.multiline { - self.writer.write_all(" }".as_bytes()).await?; - } - } else if self.multiline { - self.writer.write_all("\n".as_bytes()).await?; - } - } - Value::Array(arr) => { - self.writer.write_all("[".as_bytes()).await?; - for (pos, value) in arr.iter().enumerate() { - if pos > 0 { - self.writer.write_all(", ".as_bytes()).await?; - } - self.write_value(value, indent).await?; - } - self.writer.write_all("]".as_bytes()).await?; - } - Value::None => { - self.writer.write_all("(null)".as_bytes()).await?; - } + if self.multiline { + out.resize(out.len() + indent, b'\t'); + } + if self.ansi { + out.extend_from_slice(Color::Cyan.as_code().as_bytes()); + } + out.extend_from_slice(key.as_str().as_bytes()); + if self.ansi { + out.extend_from_slice(Color::reset().as_bytes()); } - Ok(()) - }) - .await + out.extend_from_slice(b" = "); + self.write_value(out, value, indent); + + if self.multiline && !matches!(value, Value::Event(_)) { + out.push(b'\n'); + } + } } - pub async fn flush(&mut self) -> std::io::Result<()> { - self.writer.flush().await - } - - pub fn update_writer(&mut self, writer: T) { - self.writer = writer; + fn write_value(&mut self, out: &mut Vec, value: &Value, indent: usize) { + match value { + Value::String(v) => { + out.push(b'"'); + escape_into(out, v.as_bytes()); + out.push(b'"'); + } + Value::UInt(v) => write_uint(out, *v), + Value::Int(v) => write_int(out, *v), + Value::Float(v) => { + let _ = write!(out, "{v}"); + } + Value::Timestamp(v) => { + out.extend_from_slice(self.value_timestamps.get(*v).as_bytes()); + } + Value::Duration(v) => { + write_uint(out, *v); + out.extend_from_slice(b"ms"); + } + Value::Bytes(bytes) => { + out.extend_from_slice(b"base64:"); + STANDARD.encode_append(bytes, out); + } + Value::Bool(true) => out.extend_from_slice(b"true"), + Value::Bool(false) => out.extend_from_slice(b"false"), + Value::Ipv4(v) => { + let _ = write!(out, "{v}"); + } + Value::Ipv6(v) => { + let _ = write!(out, "{v}"); + } + Value::Event(e) => { + out.extend_from_slice(e.0.inner.description().as_bytes()); + out.extend_from_slice(b" ("); + out.extend_from_slice(e.0.inner.as_str().as_bytes()); + out.push(b')'); + if !e.0.keys.is_empty() { + out.extend_from_slice(if self.multiline { b"\n" } else { b" { " }); + self.write_keys(out, &e.0.keys, &[], indent + 1); + if !self.multiline { + out.extend_from_slice(b" }"); + } + } else if self.multiline { + out.push(b'\n'); + } + } + Value::Array(arr) => { + out.push(b'['); + for (pos, value) in arr.iter().enumerate() { + if pos > 0 { + out.extend_from_slice(b", "); + } + self.write_value(out, value, indent); + } + out.push(b']'); + } + Value::None => out.extend_from_slice(b"(null)"), + } } } @@ -317,6 +238,69 @@ impl Color { } } +impl Value { + pub fn write_display(&self, out: &mut Vec) { + match self { + Value::String(value) => out.extend_from_slice(value.as_bytes()), + Value::UInt(value) | Value::Timestamp(value) | Value::Duration(value) => { + write_uint(out, *value) + } + Value::Int(value) => write_int(out, *value), + Value::Float(value) => { + let _ = write!(out, "{value}"); + } + Value::Bytes(value) => { + STANDARD.encode_append(value, out); + } + Value::Bool(true) => out.extend_from_slice(b"true"), + Value::Bool(false) => out.extend_from_slice(b"false"), + Value::Ipv4(value) => { + let _ = write!(out, "{value}"); + } + Value::Ipv6(value) => { + let _ = write!(out, "{value}"); + } + Value::Event(value) => { + out.push(b'{'); + value.write_display(out); + out.push(b'}'); + } + Value::Array(values) => { + out.push(b'['); + for (pos, value) in values.iter().enumerate() { + if pos > 0 { + out.extend_from_slice(b", "); + } + value.write_display(out); + } + out.push(b']'); + } + Value::None => out.extend_from_slice(b"(null)"), + } + } +} + +impl Error { + pub fn write_display(&self, out: &mut Vec) { + out.extend_from_slice(self.0.inner.description().as_bytes()); + out.extend_from_slice(b" ("); + out.extend_from_slice(self.0.inner.as_str().as_bytes()); + out.push(b')'); + + if !self.0.keys.is_empty() { + out.extend_from_slice(b": "); + for (pos, (key, value)) in self.0.keys.iter().enumerate() { + if pos > 0 { + out.extend_from_slice(b", "); + } + out.extend_from_slice(key.as_str().as_bytes()); + out.extend_from_slice(b" = "); + value.write_display(out); + } + } + } +} + impl Display for Value { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -326,7 +310,7 @@ impl Display for Value { Value::Float(value) => value.fmt(f), Value::Timestamp(value) => value.fmt(f), Value::Duration(value) => value.fmt(f), - Value::Bytes(value) => STANDARD.encode(value).fmt(f), + Value::Bytes(value) => STANDARD.display(value).fmt(f), Value::Bool(value) => value.fmt(f), Value::Ipv4(value) => value.fmt(f), Value::Ipv6(value) => value.fmt(f), @@ -372,3 +356,4 @@ impl Display for Error { Ok(()) } } + diff --git a/crates/trc/src/serializers/timestamp.rs b/crates/trc/src/serializers/timestamp.rs new file mode 100644 index 00000000..66c2b25e --- /dev/null +++ b/crates/trc/src/serializers/timestamp.rs @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Write; + +use mail_parser::DateTime; + +pub struct TimestampCache { + timestamp: u64, + text: String, +} + +impl TimestampCache { + pub fn new() -> Self { + Self { + timestamp: u64::MAX, + text: String::with_capacity(20), + } + } + + #[inline(always)] + pub fn get(&mut self, timestamp: u64) -> &str { + if self.timestamp != timestamp { + self.timestamp = timestamp; + self.text.clear(); + write_rfc3339(&mut self.text, timestamp); + } + &self.text + } +} + +impl Default for TimestampCache { + fn default() -> Self { + Self::new() + } +} + +pub fn write_rfc3339(out: &mut impl Write, timestamp: u64) { + let dt = DateTime::from_timestamp(timestamp as i64); + let _ = write!( + out, + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second + ); +} + diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 4921f70e..a42fbb59 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -1,21 +1,23 @@ [package] name = "types" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] utils = { path = "../utils" } trc = { path = "../trc" } -jmap-tools = { version = "0.1" } -hashify = "0.2" +jmap-tools = { path = "../jmap-tools" } +hashify = "0.3" +memchr = "2.8.3" serde = { version = "1.0", features = ["derive"]} -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = { version = "0.10.0", features = ["rkyv", "serde"] } blake3 = "1.8.7" +encodify = "1.0" +unicode-normalization = "0.1.26" [features] -test_mode = [] [lints] diff --git a/crates/types/src/acl.rs b/crates/types/src/acl.rs index a73bbdec..a8ec06a0 100644 --- a/crates/types/src/acl.rs +++ b/crates/types/src/acl.rs @@ -39,7 +39,8 @@ pub enum Acl { ModifyItemsOwn = 13, ModifyPrivateProperties = 14, ModifyRSVP = 15, - None = 16, + ModifyMetadata = 16, + None = 17, } #[derive( @@ -60,6 +61,20 @@ pub struct AclGrant { } impl Acl { + pub const WRITE: Bitmap = Bitmap::from_bits( + 1 << Acl::Modify as u64 + | 1 << Acl::Delete as u64 + | 1 << Acl::AddItems as u64 + | 1 << Acl::ModifyItems as u64 + | 1 << Acl::RemoveItems as u64 + | 1 << Acl::CreateChild as u64 + | 1 << Acl::ModifyItemsOwn as u64 + | 1 << Acl::ModifyRSVP as u64 + | 1 << Acl::ModifyMetadata as u64 + | 1 << Acl::Share as u64 + | 1 << Acl::Submit as u64, + ); + fn as_str(&self) -> &'static str { match self { Acl::Read => "read", @@ -79,6 +94,7 @@ impl Acl { Acl::SchedulingInvite => "schedulingInvite", Acl::SchedulingReply => "schedulingReply", Acl::ModifyRSVP => "modifyRSVP", + Acl::ModifyMetadata => "modifyMetadata", } } } @@ -133,6 +149,7 @@ impl From for Acl { 13 => Acl::ModifyItemsOwn, 14 => Acl::ModifyPrivateProperties, 15 => Acl::ModifyRSVP, + 16 => Acl::ModifyMetadata, _ => Acl::None, } } diff --git a/crates/types/src/blob.rs b/crates/types/src/blob.rs index 16619fa5..75c84d2b 100644 --- a/crates/types/src/blob.rs +++ b/crates/types/src/blob.rs @@ -4,17 +4,38 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use encodify::base32::STALWART; use jmap_tools::{Element, Property, Value}; -use std::{borrow::Borrow, str::FromStr, time::SystemTime}; -use utils::codec::{ - base32_custom::{Base32Reader, Base32Writer}, - leb128::{Leb128Iterator, Leb128Writer}, +use std::{ + io::{Cursor, Write}, + ops::Range, + str::FromStr, + time::SystemTime, }; +use utils::codec::leb128::{Leb128, Leb128Iterator}; -use crate::blob_hash::BlobHash; +use crate::blob_hash::{BLOB_HASH_LEN, BlobHash}; const B_LINKED: u8 = 0x10; const B_RESERVED: u8 = 0x20; +const B_EMBEDDED: u8 = 0x40; +const B_NESTED: u8 = 0x80; +const ENCODING_MASK: u8 = 0x0F; +const ENCODING_NONE: u8 = 0; +const ENCODING_QUOTED_PRINTABLE: u8 = 1; +const ENCODING_BASE64: u8 = 2; +const MAX_LEB128_LEN: usize = 10; +const MAX_CLASS_LEN: usize = 2 * MAX_LEB128_LEN; +const MAX_RANGE_LEN: usize = 2 * MAX_LEB128_LEN; +const MAX_CONTAINER_LEN: usize = 1 + MAX_RANGE_LEN; +const MAX_SERIALIZED_LEN: usize = 1 + + BLOB_HASH_LEN + + MAX_CLASS_LEN + + MAX_RANGE_LEN + + 1 + + MAX_SECTION_CONTAINERS * MAX_CONTAINER_LEN; + +pub const MAX_SECTION_CONTAINERS: usize = 8; #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum BlobClass { @@ -27,6 +48,11 @@ pub enum BlobClass { collection: u8, document_id: u32, }, + Embedded { + account_id: u32, + collection: u8, + document_id: u32, + }, } impl Default for BlobClass { @@ -47,9 +73,9 @@ impl AsRef for BlobClass { impl BlobClass { pub fn account_id(&self) -> u32 { match self { - BlobClass::Reserved { account_id, .. } | BlobClass::Linked { account_id, .. } => { - *account_id - } + BlobClass::Reserved { account_id, .. } + | BlobClass::Linked { account_id, .. } + | BlobClass::Embedded { account_id, .. } => *account_id, } } @@ -61,7 +87,7 @@ impl BlobClass { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) } - BlobClass::Linked { .. } => true, + BlobClass::Linked { .. } | BlobClass::Embedded { .. } => true, } } @@ -77,13 +103,156 @@ pub struct BlobId { pub section: Option, } -#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct BlobSection { +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct EncodedRange { pub offset_start: usize, pub size: usize, pub encoding: u8, } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BlobSection { + Single(EncodedRange), + Nested(Box), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct NestedSection { + containers: Vec, + part: EncodedRange, +} + +impl EncodedRange { + pub fn new(offset_start: usize, size: usize, encoding: u8) -> Self { + EncodedRange { + offset_start, + size, + encoding, + } + } + + pub fn range(&self) -> Range { + self.offset_start..self.offset_start.saturating_add(self.size) + } + + pub fn is_encoded(&self) -> bool { + self.encoding != ENCODING_NONE + } + + pub fn is_container(&self) -> bool { + matches!(self.encoding, ENCODING_QUOTED_PRINTABLE | ENCODING_BASE64) + } + + fn shifted(self, shift: usize) -> Option { + Some(EncodedRange { + offset_start: self.offset_start.checked_add(shift)?, + ..self + }) + } + + fn marker(&self) -> u8 { + self.encoding.saturating_add(1) & ENCODING_MASK + } + + fn from_leb128(it: &mut T, encoding: u8) -> Option + where + T: Leb128Iterator, + { + Some(EncodedRange { + offset_start: it.next_leb128()?, + size: it.next_leb128()?, + encoding, + }) + } + + fn write_range(&self, writer: &mut impl Write) { + let _ = writer.write_all(&self.offset_start.to_leb128()); + let _ = writer.write_all(&self.size.to_leb128()); + } +} + +impl BlobSection { + pub fn new(offset_start: usize, size: usize, encoding: u8) -> Self { + BlobSection::Single(EncodedRange::new(offset_start, size, encoding)) + } + + pub fn nested(containers: Vec, part: EncodedRange) -> Option { + match containers.len() { + 0 => Some(BlobSection::Single(part)), + 1..=MAX_SECTION_CONTAINERS if containers.iter().all(EncodedRange::is_container) => { + Some(BlobSection::Nested(Box::new(NestedSection { + containers, + part, + }))) + } + _ => None, + } + } + + pub fn part(&self) -> &EncodedRange { + match self { + BlobSection::Single(part) => part, + BlobSection::Nested(nested) => &nested.part, + } + } + + pub fn containers(&self) -> &[EncodedRange] { + match self { + BlobSection::Single(_) => &[], + BlobSection::Nested(nested) => &nested.containers, + } + } + + pub fn outermost(&self) -> &EncodedRange { + self.containers().first().unwrap_or(self.part()) + } + + fn marker(&self) -> u8 { + match self { + BlobSection::Single(part) => part.marker(), + BlobSection::Nested(nested) => B_NESTED | nested.part.marker(), + } + } + + fn from_leb128(it: &mut T, marker: u8) -> Option> + where + T: Leb128Iterator, + { + let Some(encoding) = (marker & ENCODING_MASK).checked_sub(1) else { + return (marker & B_NESTED == 0).then_some(None); + }; + let part = EncodedRange::from_leb128(it, encoding)?; + if marker & B_NESTED == 0 { + return Some(Some(BlobSection::Single(part))); + } + let count = usize::from(it.next()?); + if !(1..=MAX_SECTION_CONTAINERS).contains(&count) { + return None; + } + let mut containers = Vec::with_capacity(count); + for _ in 0..count { + let encoding = it.next()?; + let container = EncodedRange::from_leb128(it, encoding)?; + if !container.is_container() { + return None; + } + containers.push(container); + } + BlobSection::nested(containers, part).map(Some) + } + + fn write_levels(&self, writer: &mut impl Write) { + self.part().write_range(writer); + if let BlobSection::Nested(nested) = self { + let _ = writer.write_all(&[nested.containers.len() as u8]); + for container in &nested.containers { + let _ = writer.write_all(&[container.encoding]); + container.write_range(writer); + } + } + } +} + impl FromStr for BlobId { type Err = (); @@ -111,47 +280,74 @@ impl BlobId { BlobId { hash, class, - section: BlobSection { - offset_start, - size: offset_end - offset_start, - encoding: encoding.into(), - } - .into(), + section: BlobSection::new(offset_start, offset_end - offset_start, encoding.into()) + .into(), } } - pub fn with_section_size(mut self, size: usize) -> Self { - self.section.get_or_insert_with(Default::default).size = size; - self + pub fn inner_part(&self, containers: &[EncodedRange], part: EncodedRange) -> Option { + let (mut levels, shift) = match &self.section { + None => (Vec::new(), 0), + Some(section) => { + let base = section.part(); + let mut levels = Vec::with_capacity( + section.containers().len() + usize::from(base.is_encoded()) + containers.len(), + ); + levels.extend_from_slice(section.containers()); + if base.is_encoded() { + levels.push(*base); + (levels, 0) + } else { + (levels, base.offset_start) + } + } + }; + let part = match containers.split_first() { + Some((first, rest)) => { + levels.push(first.shifted(shift)?); + levels.extend_from_slice(rest); + part + } + None => part.shifted(shift)?, + }; + Some(BlobId { + hash: self.hash.clone(), + class: self.class.clone(), + section: Some(BlobSection::nested(levels, part)?), + }) } #[inline] pub fn from_base32(value: impl AsRef<[u8]>) -> Option { - BlobId::from_iter(&mut Base32Reader::new(value.as_ref())) + BlobId::from_iter(&mut STALWART.decoder(value.as_ref())) } #[allow(clippy::should_implement_trait)] - pub fn from_iter(it: &mut T) -> Option + pub fn from_iter(it: &mut T) -> Option where - T: Iterator + Leb128Iterator, - U: Borrow, + T: Leb128Iterator, { - let class = *it.next()?.borrow(); - let encoding = class & 0x0F; + let class = it.next()?; let mut hash = BlobHash::default(); for byte in hash.as_mut().iter_mut() { - *byte = *it.next()?.borrow(); + *byte = it.next()?; } let account_id: u32 = it.next_leb128()?; BlobId { hash, - class: if (class & B_LINKED) != 0 { + class: if (class & B_EMBEDDED) != 0 { + BlobClass::Embedded { + account_id, + collection: it.next()?, + document_id: it.next_leb128()?, + } + } else if (class & B_LINKED) != 0 { BlobClass::Linked { account_id, - collection: *it.next()?.borrow(), + collection: it.next()?, document_id: it.next_leb128()?, } } else { @@ -160,70 +356,48 @@ impl BlobId { expires: it.next_leb128()?, } }, - section: if encoding != 0 { - BlobSection { - offset_start: it.next_leb128()?, - size: it.next_leb128()?, - encoding: encoding - 1, - } - .into() - } else { - None - }, + section: BlobSection::from_leb128(it, class)?, } .into() } - fn serialize_as(&self, writer: &mut impl Leb128Writer) { - let marker = self - .section - .as_ref() - .map_or(0, |section| section.encoding + 1) - | if matches!( - self, - BlobId { - class: BlobClass::Linked { .. }, - .. - } - ) { - B_LINKED - } else { - B_RESERVED + fn serialize_as(&self, writer: &mut impl Write) { + let marker = self.section.as_ref().map_or(0, BlobSection::marker) + | match self.class { + BlobClass::Linked { .. } => B_LINKED, + BlobClass::Embedded { .. } => B_EMBEDDED, + BlobClass::Reserved { .. } => B_RESERVED, }; - let _ = writer.write(&[marker]); - let _ = writer.write(self.hash.as_ref()); + let _ = writer.write_all(&[marker]); + let _ = writer.write_all(self.hash.as_ref()); match &self.class { BlobClass::Reserved { account_id, expires, } => { - let _ = writer.write_leb128(*account_id); - let _ = writer.write_leb128(*expires); + let _ = writer.write_all(&account_id.to_leb128()); + let _ = writer.write_all(&expires.to_leb128()); } BlobClass::Linked { account_id, collection, document_id, + } + | BlobClass::Embedded { + account_id, + collection, + document_id, } => { - let _ = writer.write_leb128(*account_id); - let _ = writer.write(&[*collection]); - let _ = writer.write_leb128(*document_id); + let _ = writer.write_all(&account_id.to_leb128()); + let _ = writer.write_all(&[*collection]); + let _ = writer.write_all(&document_id.to_leb128()); } } if let Some(section) = &self.section { - let _ = writer.write_leb128(section.offset_start); - let _ = writer.write_leb128(section.size); - } - } - - pub fn start_offset(&self) -> usize { - if let Some(section) = &self.section { - section.offset_start - } else { - 0 + section.write_levels(writer); } } @@ -237,7 +411,7 @@ impl serde::Serialize for BlobId { where S: serde::Serializer, { - serializer.serialize_str(self.to_string().as_str()) + serializer.collect_str(self) } } @@ -252,11 +426,15 @@ impl<'de> serde::Deserialize<'de> for BlobId { } impl std::fmt::Display for BlobId { - #[allow(clippy::unused_io_amount)] fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut writer = Base32Writer::with_capacity(std::mem::size_of::() * 2); - self.serialize_as(&mut writer); - f.write_str(&writer.finalize()) + let mut bytes = Cursor::new([0u8; MAX_SERIALIZED_LEN]); + self.serialize_as(&mut bytes); + let len = bytes.position() as usize; + write!( + f, + "{}", + STALWART.display(bytes.get_ref().get(..len).unwrap_or_default()) + ) } } @@ -265,3 +443,4 @@ impl<'x, P: Property, E: Element + From> From for Value<'x, P, E Value::Element(E::from(id)) } } + diff --git a/crates/types/src/blob_hash.rs b/crates/types/src/blob_hash.rs index c756e88b..30105470 100644 --- a/crates/types/src/blob_hash.rs +++ b/crates/types/src/blob_hash.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use utils::text::hex_encode; + pub const BLOB_HASH_LEN: usize = 32; #[derive( @@ -43,11 +45,7 @@ impl BlobHash { } pub fn to_hex(&self) -> String { - let mut hex = String::with_capacity(BLOB_HASH_LEN * 2); - for byte in self.0.iter() { - hex.push_str(&format!("{:02x}", byte)); - } - hex + hex_encode(&self.0) } pub fn is_empty(&self) -> bool { diff --git a/crates/types/src/collation.rs b/crates/types/src/collation.rs new file mode 100644 index 00000000..7df92cc7 --- /dev/null +++ b/crates/types/src/collation.rs @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use unicode_normalization::UnicodeNormalization; + +const GREEK_TITLECASE_OFFSET: u32 = 8; + +pub fn unicode_casemap(text: &str, out: &mut String) { + if text.is_ascii() { + let start = out.len(); + out.push_str(text); + if let Some(appended) = out.get_mut(start..) { + appended.make_ascii_uppercase(); + } + } else { + out.extend(text.chars().map(simple_titlecase).nfkd()); + } +} + +fn simple_titlecase(ch: char) -> char { + match ch { + '\u{01C4}'..='\u{01C6}' => '\u{01C5}', + '\u{01C7}'..='\u{01C9}' => '\u{01C8}', + '\u{01CA}'..='\u{01CC}' => '\u{01CB}', + '\u{01F1}'..='\u{01F3}' => '\u{01F2}', + '\u{10D0}'..='\u{10FA}' | '\u{10FD}'..='\u{10FF}' => ch, + '\u{1F80}'..='\u{1F87}' | '\u{1F90}'..='\u{1F97}' | '\u{1FA0}'..='\u{1FA7}' => { + char::from_u32(u32::from(ch) + GREEK_TITLECASE_OFFSET).unwrap_or(ch) + } + '\u{1FB3}' => '\u{1FBC}', + '\u{1FC3}' => '\u{1FCC}', + '\u{1FF3}' => '\u{1FFC}', + _ => { + let mut upper = ch.to_uppercase(); + match (upper.next(), upper.next()) { + (Some(upper), None) => upper, + _ => ch, + } + } + } +} + diff --git a/crates/types/src/collection.rs b/crates/types/src/collection.rs index d40416f5..867487ad 100644 --- a/crates/types/src/collection.rs +++ b/crates/types/src/collection.rs @@ -12,6 +12,43 @@ use std::{ }; use utils::map::bitmap::BitmapItem; +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Default)] +#[repr(transparent)] +pub struct ChangeGroup(u8); + +impl ChangeGroup { + #[inline(always)] + pub const fn from_u8(group: u8) -> Self { + ChangeGroup(group) + } + + #[inline(always)] + pub const fn to_u8(self) -> u8 { + self.0 + } +} + +impl From for ChangeGroup { + #[inline(always)] + fn from(collection: SyncCollection) -> Self { + collection.change_group() + } +} + +impl From for ChangeGroup { + #[inline(always)] + fn from(collection: Collection) -> Self { + collection.change_group() + } +} + +impl From for ChangeGroup { + #[inline(always)] + fn from(collection: VanishedCollection) -> Self { + collection.change_group() + } +} + #[derive( rkyv::Archive, rkyv::Deserialize, @@ -40,8 +77,9 @@ pub enum Collection { ContactCard = 11, FileNode = 12, CalendarEventNotification = 13, + Metadata = 14, #[default] - None = 14, + None = 15, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Default)] @@ -70,6 +108,30 @@ pub enum VanishedCollection { FileNode = 254, } +impl VanishedCollection { + pub fn is_named(&self) -> bool { + match self { + VanishedCollection::Email => false, + VanishedCollection::Calendar + | VanishedCollection::AddressBook + | VanishedCollection::FileNode => true, + } + } + + pub const fn sync_collection(&self) -> SyncCollection { + match self { + VanishedCollection::Email => SyncCollection::Email, + VanishedCollection::Calendar => SyncCollection::Calendar, + VanishedCollection::AddressBook => SyncCollection::AddressBook, + VanishedCollection::FileNode => SyncCollection::FileNode, + } + } + + pub const fn change_group(&self) -> ChangeGroup { + self.sync_collection().change_group() + } +} + impl Collection { pub const MAX: usize = Collection::None as usize; @@ -93,6 +155,10 @@ impl Collection { } } + pub fn change_group(&self) -> ChangeGroup { + SyncCollection::from(*self).change_group() + } + pub fn child_collection(&self) -> Option { match self { Collection::Mailbox => Some(Collection::Email), @@ -148,6 +214,19 @@ impl SyncCollection { _ => None, } } + + pub fn is_prefixed(&self) -> bool { + matches!(self, SyncCollection::Email) + } + + pub const fn change_group(&self) -> ChangeGroup { + match self { + SyncCollection::Thread => ChangeGroup(SyncCollection::Email as u8), + other => ChangeGroup(*other as u8), + } + } + + pub const MAX_CHANGE_GROUP: u8 = SyncCollection::None as u8; } impl From for SyncCollection { @@ -189,6 +268,7 @@ impl From for Collection { 11 => Collection::ContactCard, 12 => Collection::FileNode, 13 => Collection::CalendarEventNotification, + 14 => Collection::Metadata, _ => Collection::None, } } @@ -247,6 +327,7 @@ impl From for Collection { 11 => Collection::ContactCard, 12 => Collection::FileNode, 13 => Collection::CalendarEventNotification, + 14 => Collection::Metadata, _ => Collection::None, } } @@ -353,6 +434,7 @@ impl Collection { Collection::ContactCard => "contactCard", Collection::FileNode => "fileNode", Collection::CalendarEventNotification => "calendarEventNotification", + Collection::Metadata => "metadata", Collection::None => "", } } @@ -373,6 +455,7 @@ impl Collection { Collection::ContactCard => "contact-card", Collection::FileNode => "file-node", Collection::CalendarEventNotification => "calendar-event-notification", + Collection::Metadata => "metadata", Collection::None => "", } } @@ -382,7 +465,7 @@ impl FromStr for Collection { type Err = (); fn from_str(s: &str) -> Result { - hashify::tiny_map!(s.as_bytes(), + hashify::map!(s.as_bytes(), Collection, "pushSubscription" => Collection::PushSubscription, "email" => Collection::Email, "mailbox" => Collection::Mailbox, @@ -397,7 +480,9 @@ impl FromStr for Collection { "contactCard" => Collection::ContactCard, "fileNode" => Collection::FileNode, "calendarEventNotification" => Collection::CalendarEventNotification, + "metadata" => Collection::Metadata, ) + .copied() .ok_or(()) } } diff --git a/crates/types/src/dead_property.rs b/crates/types/src/dead_property.rs deleted file mode 100644 index 73aaf4e6..00000000 --- a/crates/types/src/dead_property.rs +++ /dev/null @@ -1,191 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -#[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -#[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "test_mode", serde(tag = "type", content = "data"))] -#[rkyv(derive(Debug))] -pub enum DeadPropertyTag { - ElementStart(DeadElementTag), - ElementEnd, - Text(String), -} - -#[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -#[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] -#[rkyv(derive(Debug))] -pub struct DeadElementTag { - pub name: String, - pub attrs: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -#[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "test_mode", serde(transparent))] -#[rkyv(derive(Debug))] -#[repr(transparent)] -pub struct DeadProperty(pub Vec); - -impl From<&ArchivedDeadProperty> for DeadProperty { - fn from(value: &ArchivedDeadProperty) -> Self { - DeadProperty(value.0.iter().map(|tag| tag.into()).collect::>()) - } -} - -impl From<&ArchivedDeadPropertyTag> for DeadPropertyTag { - fn from(tag: &ArchivedDeadPropertyTag) -> Self { - match tag { - ArchivedDeadPropertyTag::ElementStart(tag) => DeadPropertyTag::ElementStart(tag.into()), - ArchivedDeadPropertyTag::ElementEnd => DeadPropertyTag::ElementEnd, - ArchivedDeadPropertyTag::Text(tag) => DeadPropertyTag::Text(tag.to_string()), - } - } -} - -impl From<&ArchivedDeadElementTag> for DeadElementTag { - fn from(tag: &ArchivedDeadElementTag) -> Self { - DeadElementTag { - name: tag.name.to_string(), - attrs: tag.attrs.as_ref().map(|s| s.to_string()), - } - } -} - -impl ArchivedDeadProperty { - pub fn find_tag(&self, needle: &str) -> Option { - let mut depth: u32 = 0; - let mut tags = Vec::new(); - let mut found_tag = false; - - for tag in self.0.iter() { - match tag { - ArchivedDeadPropertyTag::ElementStart(start) => { - if depth == 0 && start.name == needle { - found_tag = true; - } else if found_tag { - tags.push(tag.into()); - } - - depth += 1; - } - ArchivedDeadPropertyTag::ElementEnd => { - if found_tag { - if depth == 1 { - break; - } else { - tags.push(tag.into()); - } - } - depth = depth.saturating_sub(1); - } - ArchivedDeadPropertyTag::Text(_) => { - if found_tag { - tags.push(tag.into()); - } - } - } - } - - if found_tag { - Some(DeadProperty(tags)) - } else { - None - } - } -} - -impl DeadProperty { - pub fn remove_element(&mut self, element: &DeadElementTag) { - let mut depth = 0; - let mut remove = false; - self.0.retain(|item| match item { - DeadPropertyTag::ElementStart(tag) => { - if depth == 0 && !remove && tag.name == element.name { - remove = true; - } - depth += 1; - - !remove - } - DeadPropertyTag::ElementEnd => { - depth -= 1; - if remove && depth == 0 { - remove = false; - false - } else { - !remove - } - } - _ => !remove, - }); - } - - pub fn add_element(&mut self, element: DeadElementTag, values: Vec) { - self.0.push(DeadPropertyTag::ElementStart(element)); - self.0.extend(values); - self.0.push(DeadPropertyTag::ElementEnd); - } - - pub fn size(&self) -> usize { - let mut size = 0; - for item in &self.0 { - match item { - DeadPropertyTag::ElementStart(tag) => { - size += tag.size(); - } - DeadPropertyTag::ElementEnd => { - size += 1; - } - DeadPropertyTag::Text(text) => { - size += text.len(); - } - } - } - size - } -} - -impl ArchivedDeadProperty { - pub fn size(&self) -> usize { - let mut size = 0; - for item in self.0.iter() { - match item { - ArchivedDeadPropertyTag::ElementStart(tag) => { - size += tag.size(); - } - ArchivedDeadPropertyTag::ElementEnd => { - size += 1; - } - ArchivedDeadPropertyTag::Text(text) => { - size += text.len(); - } - } - } - size - } -} - -impl DeadElementTag { - pub fn new(name: String, attrs: Option) -> Self { - DeadElementTag { name, attrs } - } - - pub fn size(&self) -> usize { - self.name.len() + self.attrs.as_ref().map_or(0, |attrs| attrs.len()) - } -} - -impl ArchivedDeadElementTag { - pub fn size(&self) -> usize { - self.name.len() + self.attrs.as_ref().map_or(0, |attrs| attrs.len()) - } -} - -impl Default for DeadProperty { - fn default() -> Self { - DeadProperty(Vec::with_capacity(4)) - } -} diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index 7cb59d5e..56a19475 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -15,10 +15,9 @@ pub struct Field(u8); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum ContactField { - Uid, Email, Archive, - CreatedToUpdated, + Content, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -26,13 +25,15 @@ pub enum ContactField { pub enum CalendarEventField { Uid, Archive, + Content, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum CalendarNotificationField { - CreatedToId, + Created, Archive, + Content, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -40,6 +41,7 @@ pub enum CalendarNotificationField { pub enum EmailField { Archive, Metadata, + SortKeys, Threading, DeletedAt, } @@ -47,6 +49,7 @@ pub enum EmailField { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum MailboxField { + Name = 1, UidCounter = 84, Archive = ARCHIVE_FIELD, } @@ -55,8 +58,8 @@ pub enum MailboxField { #[repr(u8)] pub enum SieveField { Name, - Ids, Archive, + SeenId, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -64,13 +67,13 @@ pub enum SieveField { pub enum EmailSubmissionField { Archive, Metadata, + DeliveryResult, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum IdentityField { Archive, - DocumentId, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -85,32 +88,63 @@ pub enum PrincipalField { PushSubscriptions = 44, } +impl ContactField { + pub const fn as_u8(self) -> u8 { + match self { + ContactField::Email => 1, + ContactField::Archive => ARCHIVE_FIELD, + ContactField::Content => 2, + } + } + + pub const fn field(self) -> Field { + Field(self.as_u8()) + } +} + +impl CalendarEventField { + pub const fn as_u8(self) -> u8 { + match self { + CalendarEventField::Uid => 0, + CalendarEventField::Archive => ARCHIVE_FIELD, + CalendarEventField::Content => 1, + } + } + + pub const fn field(self) -> Field { + Field(self.as_u8()) + } +} + +impl CalendarNotificationField { + pub const fn as_u8(self) -> u8 { + match self { + CalendarNotificationField::Created => 0, + CalendarNotificationField::Archive => ARCHIVE_FIELD, + CalendarNotificationField::Content => 1, + } + } + + pub const fn field(self) -> Field { + Field(self.as_u8()) + } +} + impl From for u8 { fn from(value: ContactField) -> Self { - match value { - ContactField::Uid => 0, - ContactField::Email => 1, - ContactField::CreatedToUpdated => 2, - ContactField::Archive => ARCHIVE_FIELD, - } + value.as_u8() } } impl From for u8 { fn from(value: CalendarEventField) -> Self { - match value { - CalendarEventField::Uid => 0, - CalendarEventField::Archive => ARCHIVE_FIELD, - } + value.as_u8() } } impl From for u8 { fn from(value: CalendarNotificationField) -> Self { - match value { - CalendarNotificationField::CreatedToId => 0, - CalendarNotificationField::Archive => ARCHIVE_FIELD, - } + value.as_u8() } } @@ -118,6 +152,7 @@ impl From for u8 { fn from(value: EmailField) -> Self { match value { EmailField::Metadata => 71, + EmailField::SortKeys => 72, EmailField::Threading => 90, EmailField::DeletedAt => 91, EmailField::Archive => ARCHIVE_FIELD, @@ -128,6 +163,7 @@ impl From for u8 { impl From for u8 { fn from(value: MailboxField) -> Self { match value { + MailboxField::Name => 1, MailboxField::UidCounter => 84, MailboxField::Archive => ARCHIVE_FIELD, } @@ -138,7 +174,7 @@ impl From for u8 { fn from(value: SieveField) -> Self { match value { SieveField::Name => 13, - SieveField::Ids => 84, + SieveField::SeenId => 14, SieveField::Archive => ARCHIVE_FIELD, } } @@ -148,6 +184,7 @@ impl From for u8 { fn from(value: EmailSubmissionField) -> Self { match value { EmailSubmissionField::Metadata => 49, + EmailSubmissionField::DeliveryResult => 50, EmailSubmissionField::Archive => ARCHIVE_FIELD, } } @@ -171,7 +208,6 @@ impl From for u8 { fn from(value: IdentityField) -> Self { match value { IdentityField::Archive => ARCHIVE_FIELD, - IdentityField::DocumentId => 51, } } } @@ -239,7 +275,7 @@ impl From for Field { impl Field { pub const ARCHIVE: Field = Field(ARCHIVE_FIELD); - pub fn new(value: u8) -> Self { + pub const fn new(value: u8) -> Self { Field(value) } diff --git a/crates/types/src/id.rs b/crates/types/src/id.rs index 2befedc2..c6a60e15 100644 --- a/crates/types/src/id.rs +++ b/crates/types/src/id.rs @@ -5,9 +5,9 @@ */ use crate::DocumentId; +use encodify::base32::{STALWART, U64Text}; use jmap_tools::{Element, Property, Value}; use std::{ops::Deref, str::FromStr}; -use utils::codec::base32_custom::{BASE32_ALPHABET, BASE32_INVERSE}; #[derive( rkyv::Archive, @@ -36,18 +36,7 @@ impl FromStr for Id { type Err = (); fn from_str(s: &str) -> Result { - let mut id = 0; - - for &ch in s.as_bytes() { - let i = BASE32_INVERSE[ch as usize]; - if i != u8::MAX { - id = (id << 5) | i as u64; - } else { - return Err(()); - } - } - - Ok(Id(id)) + STALWART.decode_u64(s).map(Id).map_err(|_| ()) } } @@ -66,57 +55,12 @@ impl Id { Self::new(20080258862541) } - // From https://github.com/archer884/crockford by J/A - // License: MIT/Apache 2.0 pub fn as_string(&self) -> String { - match self.0 { - 0 => "a".to_string(), - mut n => { - // Used for the initial shift. - const QUAD_SHIFT: usize = 60; - const QUAD_RESET: usize = 4; + self.text().as_str().to_string() + } - // Used for all subsequent shifts. - const FIVE_SHIFT: usize = 59; - const FIVE_RESET: usize = 5; - - // After we clear the four most significant bits, the four least significant bits will be - // replaced with 0001. We can then know to stop once the four most significant bits are, - // likewise, 0001. - const STOP_BIT: u64 = 1 << QUAD_SHIFT; - - let mut buf = String::with_capacity(7); - - // Start by getting the most significant four bits. We get four here because these would be - // leftovers when starting from the least significant bits. In either case, tag the four least - // significant bits with our stop bit. - match (n >> QUAD_SHIFT) as usize { - // Eat leading zero-bits. This should not be done if the first four bits were non-zero. - // Additionally, we *must* do this in increments of five bits. - 0 => { - n <<= QUAD_RESET; - n |= 1; - n <<= n.leading_zeros() / 5 * 5; - } - - // Write value of first four bytes. - i => { - n <<= QUAD_RESET; - n |= 1; - buf.push(char::from(BASE32_ALPHABET[i])); - } - } - - // From now until we reach the stop bit, take the five most significant bits and then shift - // left by five bits. - while n != STOP_BIT { - buf.push(char::from(BASE32_ALPHABET[(n >> FIVE_SHIFT) as usize])); - n <<= FIVE_RESET; - } - - buf - } - } + pub fn text(&self) -> U64Text { + STALWART.encode_u64(self.0) } #[inline(always)] @@ -211,7 +155,7 @@ impl serde::Serialize for Id { where S: serde::Serializer, { - serializer.serialize_str(self.as_string().as_str()) + serializer.serialize_str(self.text().as_str()) } } @@ -227,7 +171,7 @@ impl<'de> serde::Deserialize<'de> for Id { impl std::fmt::Display for Id { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.as_string()) + f.write_str(self.text().as_str()) } } @@ -237,28 +181,3 @@ impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { } } -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use crate::id::Id; - - #[test] - fn parse_jmap_id() { - for number in [ - 0, - 1, - 10, - 1000, - Id::singleton().id(), - u64::MAX / 2, - u64::MAX - 1, - u64::MAX, - ] { - let id = Id::from(number); - assert_eq!(Id::from_str(&id.to_string()).unwrap(), id); - } - - Id::from_str("p333333333333p333333333333").unwrap(); - } -} diff --git a/crates/types/src/keyword.rs b/crates/types/src/keyword.rs index 88427e97..c47c2569 100644 --- a/crates/types/src/keyword.rs +++ b/crates/types/src/keyword.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use jmap_tools::{Element, Property, Value}; use std::{fmt::Display, str::FromStr}; @@ -54,8 +55,9 @@ pub const OTHER: usize = 29; )] #[serde(untagged)] #[rkyv(derive(PartialEq), compare(PartialEq))] +#[repr(u8)] pub enum Keyword { - Other(Box), + Other(CompactString), #[serde(rename(serialize = "$seen"))] Seen, #[serde(rename(serialize = "$draft"))] @@ -97,11 +99,11 @@ pub enum Keyword { Imported, #[serde(rename(serialize = "$istrusted"))] IsTrusted, - #[serde(rename(serialize = "$MailFlagBit0"))] + #[serde(rename(serialize = "$mailflagbit0"))] MailFlagBit0, - #[serde(rename(serialize = "$MailFlagBit1"))] + #[serde(rename(serialize = "$mailflagbit1"))] MailFlagBit1, - #[serde(rename(serialize = "$MailFlagBit2"))] + #[serde(rename(serialize = "$mailflagbit2"))] MailFlagBit2, #[serde(rename(serialize = "$maskedemail"))] MaskedEmail, @@ -117,69 +119,172 @@ pub enum Keyword { Unsubscribed, } +const UNIQUE_SCAN_THRESHOLD: usize = 16; + +const ATOM_CHARS: [bool; 256] = { + let mut table = [false; 256]; + let mut ch = 0x21; + while ch < 0x7f { + table[ch] = !matches!( + ch as u8, + b'(' | b')' | b'{' | b']' | b'%' | b'*' | b'"' | b'\\' + ); + ch += 1; + } + table +}; + impl Keyword { - pub const MAX_LENGTH: usize = 128; + pub const MAX_LENGTH: usize = 1024; + pub const MAX_JMAP_LENGTH: usize = 255; pub fn parse(value: &str) -> Self { + Self::try_parse(value).unwrap_or_else(|| Keyword::from_str(value)) + } + + pub fn try_from_jmap(value: &str) -> Option { Self::try_parse(value) - .unwrap_or_else(|| Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect())) + .or_else(|| Self::is_valid_jmap_name(value).then(|| Self::other(value.into()))) } - pub fn from_other(value: String) -> Self { - if value.len() <= Keyword::MAX_LENGTH { - Keyword::Other(value.into_boxed_str()) - } else { - Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect()) + pub fn is_jmap_valid(&self) -> bool { + match self { + Keyword::Other(name) => Self::is_valid_jmap_name(name), + _ => true, } } - pub fn from_boxed_other(value: Box) -> Self { + pub fn is_valid_jmap_name(value: &str) -> bool { + value.len() <= Self::MAX_JMAP_LENGTH && Self::is_atom(value.as_bytes()) + } + + pub fn is_atom(value: &[u8]) -> bool { + !value.is_empty() && value.iter().all(|&ch| ATOM_CHARS[usize::from(ch)]) + } + + pub fn from_string(value: String) -> Self { if value.len() <= Keyword::MAX_LENGTH { - Keyword::Other(value) + Self::other(value.into()) } else { - Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect()) + Self::other(truncate(&value)) } } + #[allow(clippy::should_implement_trait)] + pub fn from_str(value: &str) -> Self { + if value.len() <= Keyword::MAX_LENGTH { + Self::other(value.into()) + } else { + Self::other(truncate(value)) + } + } + + pub fn from_compact_string(value: CompactString) -> Self { + if value.len() <= Keyword::MAX_LENGTH { + Self::other(value) + } else { + Self::other(truncate(&value)) + } + } + + fn other(mut name: CompactString) -> Self { + name.make_ascii_lowercase(); + Keyword::Other(name) + } + + pub fn unique(keywords: impl IntoIterator) -> Vec { + let mut keywords = keywords.into_iter().collect::>(); + if keywords.len() <= UNIQUE_SCAN_THRESHOLD { + let mut idx = 0; + while idx < keywords.len() { + if keywords[..idx].contains(&keywords[idx]) { + keywords.remove(idx); + } else { + idx += 1; + } + } + } else { + let mut seen = std::collections::HashSet::with_capacity(keywords.len()); + keywords.retain(|keyword| seen.insert(keyword.clone())); + } + keywords + } + pub fn try_parse(value: &str) -> Option { value .split_at_checked(1) .filter(|(prefix, _)| matches!(*prefix, "$" | "\\")) .and_then(|(_, rest)| { - hashify::tiny_map_ignore_case!(rest.as_bytes(), - "seen" => Keyword::Seen, - "draft" => Keyword::Draft, - "flagged" => Keyword::Flagged, - "answered" => Keyword::Answered, - "recent" => Keyword::Recent, - "important" => Keyword::Important, - "phishing" => Keyword::Phishing, - "junk" => Keyword::Junk, - "notjunk" => Keyword::NotJunk, - "deleted" => Keyword::Deleted, - "forwarded" => Keyword::Forwarded, - "mdnsent" => Keyword::MdnSent, - "autosent" => Keyword::Autosent, - "canunsubscribe" => Keyword::CanUnsubscribe, - "followed" => Keyword::Followed, - "hasattachment" => Keyword::HasAttachment, - "hasmemo" => Keyword::HasMemo, - "hasnoattachment" => Keyword::HasNoAttachment, - "imported" => Keyword::Imported, - "istrusted" => Keyword::IsTrusted, - "mailflagbit0" => Keyword::MailFlagBit0, - "mailflagbit1" => Keyword::MailFlagBit1, - "mailflagbit2" => Keyword::MailFlagBit2, - "maskedemail" => Keyword::MaskedEmail, - "memo" => Keyword::Memo, - "muted" => Keyword::Muted, - "new" => Keyword::New, - "notify" => Keyword::Notify, - "unsubscribed" => Keyword::Unsubscribed, + hashify::fnc_map_ignore_case!(rest.as_bytes(), + "seen" => Some(Keyword::Seen), + "draft" => Some(Keyword::Draft), + "flagged" => Some(Keyword::Flagged), + "answered" => Some(Keyword::Answered), + "recent" => Some(Keyword::Recent), + "important" => Some(Keyword::Important), + "phishing" => Some(Keyword::Phishing), + "junk" => Some(Keyword::Junk), + "notjunk" => Some(Keyword::NotJunk), + "deleted" => Some(Keyword::Deleted), + "forwarded" => Some(Keyword::Forwarded), + "mdnsent" => Some(Keyword::MdnSent), + "autosent" => Some(Keyword::Autosent), + "canunsubscribe" => Some(Keyword::CanUnsubscribe), + "followed" => Some(Keyword::Followed), + "hasattachment" => Some(Keyword::HasAttachment), + "hasmemo" => Some(Keyword::HasMemo), + "hasnoattachment" => Some(Keyword::HasNoAttachment), + "imported" => Some(Keyword::Imported), + "istrusted" => Some(Keyword::IsTrusted), + "mailflagbit0" => Some(Keyword::MailFlagBit0), + "mailflagbit1" => Some(Keyword::MailFlagBit1), + "mailflagbit2" => Some(Keyword::MailFlagBit2), + "maskedemail" => Some(Keyword::MaskedEmail), + "memo" => Some(Keyword::Memo), + "muted" => Some(Keyword::Muted), + "new" => Some(Keyword::New), + "notify" => Some(Keyword::Notify), + "unsubscribed" => Some(Keyword::Unsubscribed), + _ => None, ) }) } + pub fn as_str(&self) -> &str { + match self { + Keyword::Seen => "$seen", + Keyword::Draft => "$draft", + Keyword::Flagged => "$flagged", + Keyword::Answered => "$answered", + Keyword::Recent => "$recent", + Keyword::Important => "$important", + Keyword::Phishing => "$phishing", + Keyword::Junk => "$junk", + Keyword::NotJunk => "$notjunk", + Keyword::Deleted => "$deleted", + Keyword::Forwarded => "$forwarded", + Keyword::MdnSent => "$mdnsent", + Keyword::Autosent => "$autosent", + Keyword::CanUnsubscribe => "$canunsubscribe", + Keyword::Followed => "$followed", + Keyword::HasAttachment => "$hasattachment", + Keyword::HasMemo => "$hasmemo", + Keyword::HasNoAttachment => "$hasnoattachment", + Keyword::Imported => "$imported", + Keyword::IsTrusted => "$istrusted", + Keyword::MailFlagBit0 => "$mailflagbit0", + Keyword::MailFlagBit1 => "$mailflagbit1", + Keyword::MailFlagBit2 => "$mailflagbit2", + Keyword::MaskedEmail => "$maskedemail", + Keyword::Memo => "$memo", + Keyword::Muted => "$muted", + Keyword::New => "$new", + Keyword::Notify => "$notify", + Keyword::Unsubscribed => "$unsubscribed", + Keyword::Other(string) => string.as_str(), + } + } + pub fn id(&self) -> Result { match self { Keyword::Seen => Ok(SEEN as u32), @@ -215,7 +320,7 @@ impl Keyword { } } - pub fn into_id(self) -> Result> { + pub fn into_id(self) -> Result { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), @@ -288,44 +393,13 @@ impl Keyword { impl From for Keyword { fn from(value: String) -> Self { - Keyword::try_parse(&value).unwrap_or_else(|| Keyword::from_other(value)) + Keyword::try_parse(&value).unwrap_or_else(|| Keyword::from_string(value)) } } impl Display for Keyword { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Keyword::Seen => write!(f, "$seen"), - Keyword::Draft => write!(f, "$draft"), - Keyword::Flagged => write!(f, "$flagged"), - Keyword::Answered => write!(f, "$answered"), - Keyword::Recent => write!(f, "$recent"), - Keyword::Important => write!(f, "$important"), - Keyword::Phishing => write!(f, "$phishing"), - Keyword::Junk => write!(f, "$junk"), - Keyword::NotJunk => write!(f, "$notjunk"), - Keyword::Deleted => write!(f, "$deleted"), - Keyword::Forwarded => write!(f, "$forwarded"), - Keyword::MdnSent => write!(f, "$mdnsent"), - Keyword::Autosent => write!(f, "$autosent"), - Keyword::CanUnsubscribe => write!(f, "$canunsubscribe"), - Keyword::Followed => write!(f, "$followed"), - Keyword::HasAttachment => write!(f, "$hasattachment"), - Keyword::HasMemo => write!(f, "$hasmemo"), - Keyword::HasNoAttachment => write!(f, "$hasnoattachment"), - Keyword::Imported => write!(f, "$imported"), - Keyword::IsTrusted => write!(f, "$istrusted"), - Keyword::MailFlagBit0 => write!(f, "$MailFlagBit0"), - Keyword::MailFlagBit1 => write!(f, "$MailFlagBit1"), - Keyword::MailFlagBit2 => write!(f, "$MailFlagBit2"), - Keyword::MaskedEmail => write!(f, "$maskedemail"), - Keyword::Memo => write!(f, "$memo"), - Keyword::Muted => write!(f, "$muted"), - Keyword::New => write!(f, "$new"), - Keyword::Notify => write!(f, "$notify"), - Keyword::Unsubscribed => write!(f, "$unsubscribed"), - Keyword::Other(s) => write!(f, "{}", s), - } + f.write_str(self.as_str()) } } @@ -352,9 +426,9 @@ impl Display for ArchivedKeyword { ArchivedKeyword::HasNoAttachment => write!(f, "$hasnoattachment"), ArchivedKeyword::Imported => write!(f, "$imported"), ArchivedKeyword::IsTrusted => write!(f, "$istrusted"), - ArchivedKeyword::MailFlagBit0 => write!(f, "$MailFlagBit0"), - ArchivedKeyword::MailFlagBit1 => write!(f, "$MailFlagBit1"), - ArchivedKeyword::MailFlagBit2 => write!(f, "$MailFlagBit2"), + ArchivedKeyword::MailFlagBit0 => write!(f, "$mailflagbit0"), + ArchivedKeyword::MailFlagBit1 => write!(f, "$mailflagbit1"), + ArchivedKeyword::MailFlagBit2 => write!(f, "$mailflagbit2"), ArchivedKeyword::MaskedEmail => write!(f, "$maskedemail"), ArchivedKeyword::Memo => write!(f, "$memo"), ArchivedKeyword::Muted => write!(f, "$muted"), @@ -411,115 +485,6 @@ impl FromStr for Keyword { } } -impl ArchivedKeyword { - pub fn id(&self) -> Result { - match self { - ArchivedKeyword::Seen => Ok(SEEN as u32), - ArchivedKeyword::Draft => Ok(DRAFT as u32), - ArchivedKeyword::Flagged => Ok(FLAGGED as u32), - ArchivedKeyword::Answered => Ok(ANSWERED as u32), - ArchivedKeyword::Recent => Ok(RECENT as u32), - ArchivedKeyword::Important => Ok(IMPORTANT as u32), - ArchivedKeyword::Phishing => Ok(PHISHING as u32), - ArchivedKeyword::Junk => Ok(JUNK as u32), - ArchivedKeyword::NotJunk => Ok(NOTJUNK as u32), - ArchivedKeyword::Deleted => Ok(DELETED as u32), - ArchivedKeyword::Forwarded => Ok(FORWARDED as u32), - ArchivedKeyword::MdnSent => Ok(MDN_SENT as u32), - ArchivedKeyword::Autosent => Ok(AUTOSENT as u32), - ArchivedKeyword::CanUnsubscribe => Ok(CANUNSUBSCRIBE as u32), - ArchivedKeyword::Followed => Ok(FOLLOWED as u32), - ArchivedKeyword::HasAttachment => Ok(HASATTACHMENT as u32), - ArchivedKeyword::HasMemo => Ok(HASMEMO as u32), - ArchivedKeyword::HasNoAttachment => Ok(HASNOATTACHMENT as u32), - ArchivedKeyword::Imported => Ok(IMPORTED as u32), - ArchivedKeyword::IsTrusted => Ok(ISTRUSTED as u32), - ArchivedKeyword::MailFlagBit0 => Ok(MAILFLAGBIT0 as u32), - ArchivedKeyword::MailFlagBit1 => Ok(MAILFLAGBIT1 as u32), - ArchivedKeyword::MailFlagBit2 => Ok(MAILFLAGBIT2 as u32), - ArchivedKeyword::MaskedEmail => Ok(MASKEDEMAIL as u32), - ArchivedKeyword::Memo => Ok(MEMO as u32), - ArchivedKeyword::Muted => Ok(MUTED as u32), - ArchivedKeyword::New => Ok(NEW as u32), - ArchivedKeyword::Notify => Ok(NOTIFY as u32), - ArchivedKeyword::Unsubscribed => Ok(UNSUBSCRIBED as u32), - ArchivedKeyword::Other(string) => Err(string.as_ref()), - } - } - - pub fn to_native(&self) -> Keyword { - match self { - ArchivedKeyword::Seen => Keyword::Seen, - ArchivedKeyword::Draft => Keyword::Draft, - ArchivedKeyword::Flagged => Keyword::Flagged, - ArchivedKeyword::Answered => Keyword::Answered, - ArchivedKeyword::Recent => Keyword::Recent, - ArchivedKeyword::Important => Keyword::Important, - ArchivedKeyword::Phishing => Keyword::Phishing, - ArchivedKeyword::Junk => Keyword::Junk, - ArchivedKeyword::NotJunk => Keyword::NotJunk, - ArchivedKeyword::Deleted => Keyword::Deleted, - ArchivedKeyword::Forwarded => Keyword::Forwarded, - ArchivedKeyword::MdnSent => Keyword::MdnSent, - ArchivedKeyword::Autosent => Keyword::Autosent, - ArchivedKeyword::CanUnsubscribe => Keyword::CanUnsubscribe, - ArchivedKeyword::Followed => Keyword::Followed, - ArchivedKeyword::HasAttachment => Keyword::HasAttachment, - ArchivedKeyword::HasMemo => Keyword::HasMemo, - ArchivedKeyword::HasNoAttachment => Keyword::HasNoAttachment, - ArchivedKeyword::Imported => Keyword::Imported, - ArchivedKeyword::IsTrusted => Keyword::IsTrusted, - ArchivedKeyword::MailFlagBit0 => Keyword::MailFlagBit0, - ArchivedKeyword::MailFlagBit1 => Keyword::MailFlagBit1, - ArchivedKeyword::MailFlagBit2 => Keyword::MailFlagBit2, - ArchivedKeyword::MaskedEmail => Keyword::MaskedEmail, - ArchivedKeyword::Memo => Keyword::Memo, - ArchivedKeyword::Muted => Keyword::Muted, - ArchivedKeyword::New => Keyword::New, - ArchivedKeyword::Notify => Keyword::Notify, - ArchivedKeyword::Unsubscribed => Keyword::Unsubscribed, - ArchivedKeyword::Other(other) => Keyword::Other(other.as_ref().into()), - } - } -} - -impl From<&ArchivedKeyword> for Keyword { - fn from(value: &ArchivedKeyword) -> Self { - match value { - ArchivedKeyword::Seen => Keyword::Seen, - ArchivedKeyword::Draft => Keyword::Draft, - ArchivedKeyword::Flagged => Keyword::Flagged, - ArchivedKeyword::Answered => Keyword::Answered, - ArchivedKeyword::Recent => Keyword::Recent, - ArchivedKeyword::Important => Keyword::Important, - ArchivedKeyword::Phishing => Keyword::Phishing, - ArchivedKeyword::Junk => Keyword::Junk, - ArchivedKeyword::NotJunk => Keyword::NotJunk, - ArchivedKeyword::Deleted => Keyword::Deleted, - ArchivedKeyword::Forwarded => Keyword::Forwarded, - ArchivedKeyword::MdnSent => Keyword::MdnSent, - ArchivedKeyword::Autosent => Keyword::Autosent, - ArchivedKeyword::CanUnsubscribe => Keyword::CanUnsubscribe, - ArchivedKeyword::Followed => Keyword::Followed, - ArchivedKeyword::HasAttachment => Keyword::HasAttachment, - ArchivedKeyword::HasMemo => Keyword::HasMemo, - ArchivedKeyword::HasNoAttachment => Keyword::HasNoAttachment, - ArchivedKeyword::Imported => Keyword::Imported, - ArchivedKeyword::IsTrusted => Keyword::IsTrusted, - ArchivedKeyword::MailFlagBit0 => Keyword::MailFlagBit0, - ArchivedKeyword::MailFlagBit1 => Keyword::MailFlagBit1, - ArchivedKeyword::MailFlagBit2 => Keyword::MailFlagBit2, - ArchivedKeyword::MaskedEmail => Keyword::MaskedEmail, - ArchivedKeyword::Memo => Keyword::Memo, - ArchivedKeyword::Muted => Keyword::Muted, - ArchivedKeyword::New => Keyword::New, - ArchivedKeyword::Notify => Keyword::Notify, - ArchivedKeyword::Unsubscribed => Keyword::Unsubscribed, - ArchivedKeyword::Other(string) => Keyword::Other(string.as_ref().into()), - } - } -} - impl<'de> serde::Deserialize<'de> for Keyword { fn deserialize(deserializer: D) -> Result where @@ -536,3 +501,13 @@ impl<'x, P: Property, E: Element + From> From for Value<'x, P, Value::Element(E::from(id)) } } + +fn truncate(value: &str) -> CompactString { + let end = value + .char_indices() + .map(|(offset, ch)| offset + ch.len_utf8()) + .find(|&end| end > Keyword::MAX_LENGTH) + .unwrap_or(value.len()); + value[..end].into() +} + diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 422c41aa..3ec8969a 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -7,37 +7,120 @@ pub mod acl; pub mod blob; pub mod blob_hash; +pub mod collation; pub mod collection; -pub mod dead_property; pub mod field; pub mod id; pub mod keyword; +pub mod media_type; +pub mod metadata; pub mod semver; pub mod special_use; +pub mod text; pub mod type_state; pub type DocumentId = u32; pub type ChangeId = u64; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] pub struct TimeRange { pub start: i64, pub end: i64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OverlapRule { + CalDav, + Jmap, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum OverlapCondition { + #[default] + Event, + TodoStartDuration, + TodoStartDue, + TodoStart, + TodoDue, + TodoCreatedCompleted, + TodoCompleted, + TodoCreated, +} + +impl OverlapCondition { + pub const ALL: [OverlapCondition; 8] = [ + OverlapCondition::Event, + OverlapCondition::TodoStartDuration, + OverlapCondition::TodoStartDue, + OverlapCondition::TodoStart, + OverlapCondition::TodoDue, + OverlapCondition::TodoCreatedCompleted, + OverlapCondition::TodoCompleted, + OverlapCondition::TodoCreated, + ]; + + pub fn for_instance(self, start: i64, end: i64) -> Self { + match self { + OverlapCondition::TodoStart if start != end => OverlapCondition::TodoStartDuration, + condition => condition, + } + } +} + impl TimeRange { pub fn new(start: i64, end: i64) -> Self { Self { start, end } } - pub fn is_in_range(&self, match_overlap: bool, start: i64, end: i64) -> bool { - if !match_overlap { + pub fn is_in_range(&self, condition: OverlapCondition, start: i64, end: i64) -> bool { + match condition { + OverlapCondition::Event if start == end => self.start <= start && self.end > start, // RFC4791#9.9: (start < DTEND AND end > DTSTART) - self.start < end && self.end > start - } else { + OverlapCondition::Event => self.start < end && self.end > start, + OverlapCondition::TodoStartDuration => { + self.start <= end && (self.end > start || self.end >= end) + } // RFC4791#9.9: ((start < DUE) OR (start <= DTSTART)) AND ((end > DTSTART) OR (end >= DUE)) - ((start < self.end) || (start <= self.start)) && (end > self.start || end >= self.end) + OverlapCondition::TodoStartDue => { + (self.start < end || self.start <= start) && (self.end > start || self.end >= end) + } + OverlapCondition::TodoStart => self.start <= start && self.end > start, + OverlapCondition::TodoDue => self.start < start && self.end >= start, + OverlapCondition::TodoCreatedCompleted => { + (self.start <= start || self.start <= end) && (self.end >= start || self.end >= end) + } + OverlapCondition::TodoCompleted => self.start <= start && self.end >= start, + OverlapCondition::TodoCreated => self.end > start, + } + } + + pub fn overlaps(&self, start: i64, end: i64) -> bool { + self.start < end && self.end > start + } + + pub fn union(self, other: TimeRange) -> Self { + TimeRange { + start: self.start.min(other.start), + end: self.end.max(other.end), + } + } + + pub fn touches(&self, start: i64, end: i64) -> bool { + self.start <= end && self.end >= start + } + + pub fn matches( + &self, + rule: OverlapRule, + condition: OverlapCondition, + start: i64, + end: i64, + ) -> bool { + match rule { + OverlapRule::CalDav => self.is_in_range(condition, start, end), + OverlapRule::Jmap => self.overlaps(start, end), } } } @@ -50,3 +133,4 @@ impl Default for TimeRange { } } } + diff --git a/crates/types/src/media_type/catalogue.rs b/crates/types/src/media_type/catalogue.rs new file mode 100644 index 00000000..2893d9cc --- /dev/null +++ b/crates/types/src/media_type/catalogue.rs @@ -0,0 +1,2928 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(super) static MEDIA_TYPES: [&str; 2744] = [ + "application/1d-interleaved-parityfec", + "application/3gpdash-qoe-report+xml", + "application/3gpp-ims+xml", + "application/3gpp-mbs-object-manifest+json", + "application/3gpp-mbs-user-service-descriptions+json", + "application/3gpp-media-delivery-metrics-report+json", + "application/3gpphal+json", + "application/3gpphalforms+json", + "application/a2l", + "application/aas+zip", + "application/acad", + "application/ace+cbor", + "application/ace+json", + "application/ace-groupcomm+cbor", + "application/ace-trl+cbor", + "application/activemessage", + "application/activity+json", + "application/aif+cbor", + "application/aif+json", + "application/alto-cdni+json", + "application/alto-cdnifilter+json", + "application/alto-costmap+json", + "application/alto-costmapfilter+json", + "application/alto-directory+json", + "application/alto-endpointcost+json", + "application/alto-endpointcostparams+json", + "application/alto-endpointprop+json", + "application/alto-endpointpropparams+json", + "application/alto-error+json", + "application/alto-networkmap+json", + "application/alto-networkmapfilter+json", + "application/alto-propmap+json", + "application/alto-propmapparams+json", + "application/alto-tips+json", + "application/alto-tipsparams+json", + "application/alto-updatestreamcontrol+json", + "application/alto-updatestreamparams+json", + "application/aml", + "application/andrew-inset", + "application/annodex", + "application/applefile", + "application/applixware", + "application/asyncapi+json", + "application/asyncapi+yaml", + "application/at+jwt", + "application/atf", + "application/atfx", + "application/atom+xml", + "application/atomcat+xml", + "application/atomdeleted+xml", + "application/atomicmail", + "application/atomsvc+xml", + "application/atsc-dwd+xml", + "application/atsc-dynamic-event-message", + "application/atsc-held+xml", + "application/atsc-rdt+json", + "application/atsc-rsat+xml", + "application/atxml", + "application/auth-policy+xml", + "application/automationml-aml+xml", + "application/automationml-amlx+zip", + "application/bacnet-xdd+zip", + "application/batch-smtp", + "application/bdoc", + "application/beep+xml", + "application/bufr", + "application/c2pa", + "application/calendar+json", + "application/calendar+xml", + "application/call-completion", + "application/cals-1840", + "application/captive+json", + "application/cbor", + "application/cbor-seq", + "application/cccex", + "application/ccmp+xml", + "application/ccxml+xml", + "application/cda+xml", + "application/cdfx+xml", + "application/cdmi-capability", + "application/cdmi-container", + "application/cdmi-domain", + "application/cdmi-object", + "application/cdmi-queue", + "application/cdni", + "application/ce+cbor", + "application/cea", + "application/cea-2018+xml", + "application/cellml+xml", + "application/cfw", + "application/cid", + "application/cid-edhoc+cbor-seq", + "application/city+json", + "application/city+json-seq", + "application/client-authentication+jwt", + "application/cloudevents+json", + "application/cloudevents-batch+json", + "application/clr", + "application/clue+xml", + "application/clue_info+xml", + "application/cmcd", + "application/cms", + "application/cmw+cbor", + "application/cmw+cose", + "application/cmw+json", + "application/cmw+jws", + "application/cnrp+xml", + "application/coap-eap", + "application/coap-group+json", + "application/coap-payload", + "application/commonground", + "application/concise-problem-details+cbor", + "application/conference-info+xml", + "application/cose", + "application/cose-c509+cbor", + "application/cose-c509-cert+cbor", + "application/cose-c509-crtemplate+cbor", + "application/cose-c509-pem+cbor", + "application/cose-c509-pkcs10+cbor", + "application/cose-c509-privkey+cbor", + "application/cose-certhash+cbor", + "application/cose-key", + "application/cose-key-set", + "application/cose-x509", + "application/cpl+xml", + "application/csrattrs", + "application/csta+xml", + "application/cstadata+xml", + "application/csvm+json", + "application/cu-seeme", + "application/cwl", + "application/cwl+json", + "application/cwl+yaml", + "application/cwt", + "application/cybercash", + "application/dash+xml", + "application/dash-patch+xml", + "application/dashdelta", + "application/davmount+xml", + "application/dca-rft", + "application/dcd", + "application/dec-dx", + "application/dialog-info+xml", + "application/dicom", + "application/dicom+json", + "application/dicom+xml", + "application/did", + "application/dii", + "application/directx", + "application/dit", + "application/dns", + "application/dns+json", + "application/dns-message", + "application/docbook+xml", + "application/dots+cbor", + "application/dpop+jwt", + "application/dskpp+xml", + "application/dssc+der", + "application/dssc+xml", + "application/dvcs", + "application/eat+cwt", + "application/eat+jwt", + "application/eat-bun+cbor", + "application/eat-bun+json", + "application/eat-ucs+cbor", + "application/eat-ucs+json", + "application/ecmascript", + "application/edhoc+cbor-seq", + "application/edi-consent", + "application/edi-x12", + "application/edifact", + "application/efi", + "application/elm+json", + "application/elm+xml", + "application/emergencycalldata.cap+xml", + "application/emergencycalldata.comment+xml", + "application/emergencycalldata.control+xml", + "application/emergencycalldata.deviceinfo+xml", + "application/emergencycalldata.ecall.msd", + "application/emergencycalldata.legacyesn+json", + "application/emergencycalldata.providerinfo+xml", + "application/emergencycalldata.serviceinfo+xml", + "application/emergencycalldata.subscriberinfo+xml", + "application/emergencycalldata.veds+xml", + "application/emma+xml", + "application/emotionml+xml", + "application/encaprtp", + "application/entity-statement+jwt", + "application/envoy", + "application/epp+xml", + "application/epub+zip", + "application/eshop", + "application/etl", + "application/example", + "application/exi", + "application/expect-ct-report+json", + "application/explicit-registration-response+jwt", + "application/express", + "application/fastinfoset", + "application/fastsoap", + "application/fdf", + "application/fdt+xml", + "application/fhir+json", + "application/fhir+xml", + "application/fits", + "application/flexfec", + "application/font-sfnt", + "application/font-tdpfr", + "application/font-woff", + "application/fractals", + "application/framework-attributes+xml", + "application/fsharp-script", + "application/futuresplash", + "application/geo+json", + "application/geo+json-seq", + "application/geofeed+csv", + "application/geopackage+sqlite3", + "application/geopose+json", + "application/geoxacml+json", + "application/geoxacml+xml", + "application/gltf-buffer", + "application/gml+xml", + "application/gnap-binding-jws", + "application/gnap-binding-jwsd", + "application/gnap-binding-rotation-jws", + "application/gnap-binding-rotation-jwsd", + "application/gpx+xml", + "application/grib", + "application/gxf", + "application/gzip", + "application/h224", + "application/held+xml", + "application/hjson", + "application/hl7v2+xml", + "application/hta", + "application/http", + "application/hyperstudio", + "application/ibe-key-request+xml", + "application/ibe-pkg-reply+xml", + "application/ibe-pp-data", + "application/iges", + "application/im-iscomposing+xml", + "application/index", + "application/index.cmd", + "application/index.obj", + "application/index.response", + "application/index.vnd", + "application/inkml+xml", + "application/internet-property-stream", + "application/iotp", + "application/ipfix", + "application/ipp", + "application/isup", + "application/its+xml", + "application/java-archive", + "application/java-serialized-object", + "application/javascript", + "application/jf2feed+json", + "application/jose", + "application/jose+json", + "application/jrd+json", + "application/jscalendar+json", + "application/jscontact+json", + "application/json", + "application/json-patch+json", + "application/json-patch-query+json", + "application/json-seq", + "application/json5", + "application/jsonml+json", + "application/jsonpath", + "application/jumbf", + "application/jwk+json", + "application/jwk-set+json", + "application/jwk-set+jwt", + "application/jwt", + "application/kb+jwt", + "application/kbl+xml", + "application/kpml-request+xml", + "application/kpml-response+xml", + "application/ld+json", + "application/lgr+xml", + "application/link-format", + "application/linkset", + "application/linkset+json", + "application/liquidmotion", + "application/load-control+xml", + "application/logout+jwt", + "application/lost+xml", + "application/lostsync+xml", + "application/lpf+zip", + "application/lxf", + "application/mac-binhex40", + "application/mac-compactpro", + "application/macwriteii", + "application/mads+xml", + "application/manifest+json", + "application/marc", + "application/marcxml+xml", + "application/mathematica", + "application/mathml+xml", + "application/mathml-content+xml", + "application/mathml-presentation+xml", + "application/mbms-associated-procedure-description+xml", + "application/mbms-deregister+xml", + "application/mbms-envelope+xml", + "application/mbms-msk+xml", + "application/mbms-msk-response+xml", + "application/mbms-protection-description+xml", + "application/mbms-reception-report+xml", + "application/mbms-register+xml", + "application/mbms-register-response+xml", + "application/mbms-schedule+xml", + "application/mbms-user-service-description+xml", + "application/mbox", + "application/measured-component+cbor", + "application/measured-component+json", + "application/media-policy-dataset+xml", + "application/media_control+xml", + "application/mediaservercontrol+xml", + "application/merge-patch+json", + "application/metalink+xml", + "application/metalink4+xml", + "application/mets+xml", + "application/mf4", + "application/mikey", + "application/mipc", + "application/missing-blocks+cbor-seq", + "application/mmt-aei+xml", + "application/mmt-usd+xml", + "application/mods+xml", + "application/moss-keys", + "application/moss-signature", + "application/mosskey-data", + "application/mosskey-request", + "application/mp21", + "application/mp4", + "application/mpeg", + "application/mpeg4-generic", + "application/mpeg4-iod", + "application/mpeg4-iod-xmt", + "application/mrb-consumer+xml", + "application/mrb-publish+xml", + "application/ms-vsi", + "application/msaccess", + "application/msaccess.addin", + "application/msaccess.cab", + "application/msaccess.ftemplate", + "application/msaccess.runtime", + "application/msaccess.webapplication", + "application/msc-ivr+xml", + "application/msc-mixer+xml", + "application/msword", + "application/mud+json", + "application/multipart-core", + "application/mxf", + "application/n-quads", + "application/n-triples", + "application/nasdata", + "application/news-checkgroups", + "application/news-groupinfo", + "application/news-transmission", + "application/nlsml+xml", + "application/node", + "application/nss", + "application/oauth-authz-req+jwt", + "application/oblivious-dns-message", + "application/ocsp-request", + "application/ocsp-response", + "application/octet-stream", + "application/oda", + "application/odm+xml", + "application/odx", + "application/oebps-package+xml", + "application/ogg", + "application/ohttp-keys", + "application/olescript", + "application/omdoc+xml", + "application/onenote", + "application/opc-nodeset+xml", + "application/opensearchdescription+xml", + "application/oscore", + "application/oxps", + "application/p21", + "application/p21+zip", + "application/p2p-overlay+xml", + "application/parityfec", + "application/passport", + "application/patch-ops-error+xml", + "application/pdf", + "application/pdx", + "application/pem-certificate-chain", + "application/pgp-encrypted", + "application/pgp-keys", + "application/pgp-signature", + "application/pics-rules", + "application/pidf+xml", + "application/pidf-diff+xml", + "application/pkcs10", + "application/pkcs12", + "application/pkcs7-mime", + "application/pkcs7-signature", + "application/pkcs8", + "application/pkcs8-encrypted", + "application/pkix-attr-cert", + "application/pkix-cert", + "application/pkix-crl", + "application/pkix-pkipath", + "application/pkixcmp", + "application/pls+xml", + "application/poc-settings+xml", + "application/postscript", + "application/powershell", + "application/ppsp-tracker+json", + "application/private-token-issuer-directory", + "application/private-token-request", + "application/private-token-response", + "application/problem+json", + "application/problem+xml", + "application/protobuf", + "application/protobuf+json", + "application/provenance+xml", + "application/provided-claims+jwt", + "application/prs.alvestrand.titrax-sheet", + "application/prs.archive-markdown-format", + "application/prs.bwtc32key", + "application/prs.cww", + "application/prs.cyn", + "application/prs.hpub+zip", + "application/prs.implied-document+xml", + "application/prs.implied-executable", + "application/prs.implied-object+json", + "application/prs.implied-object+json-seq", + "application/prs.implied-object+yaml", + "application/prs.implied-structure", + "application/prs.mayfile", + "application/prs.nprend", + "application/prs.plucker", + "application/prs.rdf-xml-crypt", + "application/prs.sclt", + "application/prs.vcfbzip2", + "application/prs.xsf+xml", + "application/pskc+xml", + "application/pvd+json", + "application/qsig", + "application/raml+yaml", + "application/raptorfec", + "application/rat-file", + "application/rdap+json", + "application/rdf+xml", + "application/reginfo+xml", + "application/relax-ng-compact-syntax", + "application/remote-printing", + "application/reputon+json", + "application/resolve-response+jwt", + "application/resource-lists+xml", + "application/resource-lists-diff+xml", + "application/rfc+xml", + "application/riscos", + "application/rlmi+xml", + "application/rls-services+xml", + "application/roughtime-malfeasance+json", + "application/roughtime-server+json", + "application/route-apd+xml", + "application/route-s-tsid+xml", + "application/route-usd+xml", + "application/rpki-ccr", + "application/rpki-ccr+gzip", + "application/rpki-checklist", + "application/rpki-ghostbusters", + "application/rpki-manifest", + "application/rpki-publication", + "application/rpki-roa", + "application/rpki-signed-tal", + "application/rpki-updown", + "application/rs-metadata+xml", + "application/rsd+xml", + "application/rss+xml", + "application/rtf", + "application/rtploopback", + "application/rtx", + "application/samlassertion+xml", + "application/samlmetadata+xml", + "application/sarif+json", + "application/sarif-external-properties+json", + "application/sbe", + "application/sbml+xml", + "application/scaip+xml", + "application/scim+json", + "application/scitt-receipt+cose", + "application/scitt-statement+cose", + "application/scvp-cv-request", + "application/scvp-cv-response", + "application/scvp-vp-request", + "application/scvp-vp-response", + "application/sd-jwt", + "application/sd-jwt+json", + "application/sdf+json", + "application/sdp", + "application/secevent+jwt", + "application/senml+cbor", + "application/senml+json", + "application/senml+xml", + "application/senml-etch+cbor", + "application/senml-etch+json", + "application/senml-exi", + "application/sensml+cbor", + "application/sensml+json", + "application/sensml+xml", + "application/sensml-exi", + "application/sep+xml", + "application/sep-exi", + "application/session-info", + "application/set-payment", + "application/set-payment-initiation", + "application/set-registration", + "application/set-registration-initiation", + "application/sgml", + "application/sgml-open-catalog", + "application/shf+xml", + "application/sieve", + "application/simple-filter+xml", + "application/simple-message-summary", + "application/simplesymbolcontainer", + "application/sipc", + "application/slate", + "application/smil", + "application/smil+xml", + "application/smpte336m", + "application/soap+fastinfoset", + "application/soap+xml", + "application/sparql-query", + "application/sparql-results+xml", + "application/spdx+json", + "application/spdx3+json", + "application/spirits-event+xml", + "application/sql", + "application/srgs", + "application/srgs+xml", + "application/sru+xml", + "application/ssdl+xml", + "application/sslkeylogfile", + "application/ssml+xml", + "application/st2110-41", + "application/statuslist+cwt", + "application/statuslist+jwt", + "application/step", + "application/stix+json", + "application/stratum", + "application/streamingmedia", + "application/suit-envelope+cose", + "application/suit-report+cose", + "application/swid+cbor", + "application/swid+xml", + "application/syslog-msg", + "application/tamp-apex-update", + "application/tamp-apex-update-confirm", + "application/tamp-community-update", + "application/tamp-community-update-confirm", + "application/tamp-error", + "application/tamp-sequence-adjust", + "application/tamp-sequence-adjust-confirm", + "application/tamp-status-query", + "application/tamp-status-response", + "application/tamp-update", + "application/tamp-update-confirm", + "application/taxii+json", + "application/td+json", + "application/teep+cbor", + "application/tei+xml", + "application/tetra_isi", + "application/texinfo", + "application/thraud+xml", + "application/timestamp-query", + "application/timestamp-reply", + "application/timestamped-data", + "application/tlsrpt+gzip", + "application/tlsrpt+json", + "application/tm+json", + "application/tnauthlist", + "application/toc+cbor", + "application/token-introspection+jwt", + "application/toml", + "application/trickle-ice-sdpfrag", + "application/trig", + "application/trust-chain+json", + "application/trust-mark+jwt", + "application/trust-mark-delegation+jwt", + "application/trust-mark-status-response+jwt", + "application/ttml+xml", + "application/tve-trigger", + "application/tzif", + "application/tzif-leap", + "application/uccs+cbor", + "application/ujcs+json", + "application/ulpfec", + "application/urc-grpsheet+xml", + "application/urc-ressheet+xml", + "application/urc-targetdesc+xml", + "application/urc-uisocketdesc+xml", + "application/v3c", + "application/vc", + "application/vc+cose", + "application/vc+jwt", + "application/vc+sd-jwt", + "application/vcard+json", + "application/vcard+xml", + "application/vec+xml", + "application/vec-package+gzip", + "application/vec-package+zip", + "application/vemmi", + "application/vnd.1000minds.decision-model+xml", + "application/vnd.1ob", + "application/vnd.3gpp-prose+xml", + "application/vnd.3gpp-prose-pc3a+xml", + "application/vnd.3gpp-prose-pc3ach+xml", + "application/vnd.3gpp-prose-pc3ch+xml", + "application/vnd.3gpp-prose-pc8+xml", + "application/vnd.3gpp-v2x-local-service-information", + "application/vnd.3gpp.5gnas", + "application/vnd.3gpp.5gsa2x", + "application/vnd.3gpp.5gsa2x-local-service-information", + "application/vnd.3gpp.5gsv2x", + "application/vnd.3gpp.5gsv2x-local-service-information", + "application/vnd.3gpp.access-transfer-events+xml", + "application/vnd.3gpp.bsf+xml", + "application/vnd.3gpp.crs+xml", + "application/vnd.3gpp.current-location-discovery+xml", + "application/vnd.3gpp.gmop+xml", + "application/vnd.3gpp.gtpc", + "application/vnd.3gpp.interworking-data", + "application/vnd.3gpp.lpp", + "application/vnd.3gpp.mc-signalling-ear", + "application/vnd.3gpp.mcdata-affiliation-command+xml", + "application/vnd.3gpp.mcdata-info+xml", + "application/vnd.3gpp.mcdata-msgstore-ctrl-request+xml", + "application/vnd.3gpp.mcdata-payload", + "application/vnd.3gpp.mcdata-regroup+xml", + "application/vnd.3gpp.mcdata-service-config+xml", + "application/vnd.3gpp.mcdata-signalling", + "application/vnd.3gpp.mcdata-ue-config+xml", + "application/vnd.3gpp.mcdata-user-profile+xml", + "application/vnd.3gpp.mcptt-affiliation-command+xml", + "application/vnd.3gpp.mcptt-floor-request+xml", + "application/vnd.3gpp.mcptt-info+xml", + "application/vnd.3gpp.mcptt-location-info+xml", + "application/vnd.3gpp.mcptt-mbms-usage-info+xml", + "application/vnd.3gpp.mcptt-regroup+xml", + "application/vnd.3gpp.mcptt-service-config+xml", + "application/vnd.3gpp.mcptt-signed+xml", + "application/vnd.3gpp.mcptt-ue-config+xml", + "application/vnd.3gpp.mcptt-ue-init-config+xml", + "application/vnd.3gpp.mcptt-user-profile+xml", + "application/vnd.3gpp.mcs-location-user-config+xml", + "application/vnd.3gpp.mcvideo-affiliation-command+xml", + "application/vnd.3gpp.mcvideo-affiliation-info+xml", + "application/vnd.3gpp.mcvideo-info+xml", + "application/vnd.3gpp.mcvideo-location-info+xml", + "application/vnd.3gpp.mcvideo-mbms-usage-info+xml", + "application/vnd.3gpp.mcvideo-regroup+xml", + "application/vnd.3gpp.mcvideo-service-config+xml", + "application/vnd.3gpp.mcvideo-transmission-request+xml", + "application/vnd.3gpp.mcvideo-ue-config+xml", + "application/vnd.3gpp.mcvideo-user-profile+xml", + "application/vnd.3gpp.mid-call+xml", + "application/vnd.3gpp.ngap", + "application/vnd.3gpp.pfcp", + "application/vnd.3gpp.pic-bw-large", + "application/vnd.3gpp.pic-bw-small", + "application/vnd.3gpp.pic-bw-var", + "application/vnd.3gpp.pinapp-info+xml", + "application/vnd.3gpp.s1ap", + "application/vnd.3gpp.seal-app-comm-requirements-info+xml", + "application/vnd.3gpp.seal-data-delivery-info+cbor", + "application/vnd.3gpp.seal-data-delivery-info+xml", + "application/vnd.3gpp.seal-group-doc+xml", + "application/vnd.3gpp.seal-info+xml", + "application/vnd.3gpp.seal-location-info+cbor", + "application/vnd.3gpp.seal-location-info+xml", + "application/vnd.3gpp.seal-mbms-usage-info+xml", + "application/vnd.3gpp.seal-mbs-usage-info+xml", + "application/vnd.3gpp.seal-network-qos-management-info+xml", + "application/vnd.3gpp.seal-network-resource-info+cbor", + "application/vnd.3gpp.seal-store-forward-events-info+xml", + "application/vnd.3gpp.seal-ue-config-info+xml", + "application/vnd.3gpp.seal-unicast-info+xml", + "application/vnd.3gpp.seal-user-profile-info+xml", + "application/vnd.3gpp.sms", + "application/vnd.3gpp.sms+xml", + "application/vnd.3gpp.srvcc-ext+xml", + "application/vnd.3gpp.srvcc-info+xml", + "application/vnd.3gpp.state-and-event-info+xml", + "application/vnd.3gpp.ussd+xml", + "application/vnd.3gpp.v2x", + "application/vnd.3gpp.vae-info+xml", + "application/vnd.3gpp2.bcmcsinfo+xml", + "application/vnd.3gpp2.sms", + "application/vnd.3gpp2.tcap", + "application/vnd.3lightssoftware.imagescal", + "application/vnd.3m.post-it-notes", + "application/vnd.abdalsecuritygroup.lockbox", + "application/vnd.accpac.simply.aso", + "application/vnd.accpac.simply.imp", + "application/vnd.acm.addressxfer+json", + "application/vnd.acm.chatbot+json", + "application/vnd.acucobol", + "application/vnd.acucorp", + "application/vnd.adobe.air-application-installer-package+zip", + "application/vnd.adobe.flash.movie", + "application/vnd.adobe.formscentral.fcdt", + "application/vnd.adobe.fxp", + "application/vnd.adobe.partial-upload", + "application/vnd.adobe.xdp+xml", + "application/vnd.adobe.xfdf", + "application/vnd.aep+zip", + "application/vnd.aether.imp", + "application/vnd.afpc.afplinedata", + "application/vnd.afpc.afplinedata-pagedef", + "application/vnd.afpc.cmoca-cmresource", + "application/vnd.afpc.foca-charset", + "application/vnd.afpc.foca-codedfont", + "application/vnd.afpc.foca-codepage", + "application/vnd.afpc.modca", + "application/vnd.afpc.modca-cmtable", + "application/vnd.afpc.modca-formdef", + "application/vnd.afpc.modca-mediummap", + "application/vnd.afpc.modca-objectcontainer", + "application/vnd.afpc.modca-overlay", + "application/vnd.afpc.modca-pagesegment", + "application/vnd.age", + "application/vnd.agentmug.agent+json", + "application/vnd.agtp.identity+json", + "application/vnd.agtp.identity+yaml", + "application/vnd.ah-barcode", + "application/vnd.ahead.space", + "application/vnd.aia", + "application/vnd.airzip.filesecure.azf", + "application/vnd.airzip.filesecure.azs", + "application/vnd.amadeus+json", + "application/vnd.amazon.ebook", + "application/vnd.amazon.mobi8-ebook", + "application/vnd.americandynamics.acc", + "application/vnd.amiga.ami", + "application/vnd.amundsen.maze+xml", + "application/vnd.android.ota", + "application/vnd.android.package-archive", + "application/vnd.anki", + "application/vnd.anser-web-certificate-issue-initiation", + "application/vnd.anser-web-funds-transfer-initiation", + "application/vnd.antix.game-component", + "application/vnd.apache.arrow.file", + "application/vnd.apache.arrow.stream", + "application/vnd.apache.parquet", + "application/vnd.apache.thrift.binary", + "application/vnd.apache.thrift.compact", + "application/vnd.apache.thrift.json", + "application/vnd.apexlang", + "application/vnd.api+json", + "application/vnd.aplextor.warrp+json", + "application/vnd.apothekende.reservation+json", + "application/vnd.apple.installer+xml", + "application/vnd.apple.keynote", + "application/vnd.apple.mpegurl", + "application/vnd.apple.numbers", + "application/vnd.apple.pages", + "application/vnd.apple.pkpass", + "application/vnd.apple.steering-list", + "application/vnd.arastra.swi", + "application/vnd.aristanetworks.swi", + "application/vnd.artisan+json", + "application/vnd.artsquare", + "application/vnd.as207960.vas.config+jer", + "application/vnd.as207960.vas.config+uper", + "application/vnd.as207960.vas.tap+jer", + "application/vnd.as207960.vas.tap+uper", + "application/vnd.astraea-software.iota", + "application/vnd.audiograph", + "application/vnd.aumtrix.aum", + "application/vnd.autopackage", + "application/vnd.avalon+json", + "application/vnd.avistar+xml", + "application/vnd.balsamiq.bmml+xml", + "application/vnd.balsamiq.bmpr", + "application/vnd.banana-accounting", + "application/vnd.bbf.usp.error", + "application/vnd.bbf.usp.msg", + "application/vnd.bbf.usp.msg+json", + "application/vnd.bekitzur-stech+json", + "application/vnd.belightsoft.lhzd+zip", + "application/vnd.belightsoft.lhzl+zip", + "application/vnd.bint.med-content", + "application/vnd.biopax.rdf+xml", + "application/vnd.blink-idb-value-wrapper", + "application/vnd.blockfact.facts", + "application/vnd.blueice.multipass", + "application/vnd.bluetooth.ep.oob", + "application/vnd.bluetooth.le.oob", + "application/vnd.bmi", + "application/vnd.bpf", + "application/vnd.bpf3", + "application/vnd.businessobjects", + "application/vnd.byu.uapi+json", + "application/vnd.bzip3", + "application/vnd.c3voc.schedule+xml", + "application/vnd.cab-jscript", + "application/vnd.canon-cpdl", + "application/vnd.canon-lips", + "application/vnd.capasystems-pg+json", + "application/vnd.cel", + "application/vnd.cendio.thinlinc.clientconf", + "application/vnd.century-systems.tcp_stream", + "application/vnd.chemdraw+xml", + "application/vnd.chess-pgn", + "application/vnd.chipnuts.karaoke-mmd", + "application/vnd.ciedi", + "application/vnd.cinderella", + "application/vnd.cirpack.isdn-ext", + "application/vnd.citationstyles.style+xml", + "application/vnd.claymore", + "application/vnd.cloanto.rp9", + "application/vnd.clonk.c4group", + "application/vnd.cluetrust.cartomobile-config", + "application/vnd.cluetrust.cartomobile-config-pkg", + "application/vnd.cmmf-configuration-information+json", + "application/vnd.cmmf-efd+xml", + "application/vnd.cmmf-encoder-configuration+json", + "application/vnd.cncf.helm.chart.content.v1.tar+gzip", + "application/vnd.cncf.helm.chart.provenance.v1.prov", + "application/vnd.cncf.helm.config.v1+json", + "application/vnd.coffeescript", + "application/vnd.collabio.xodocuments.document", + "application/vnd.collabio.xodocuments.document-template", + "application/vnd.collabio.xodocuments.presentation", + "application/vnd.collabio.xodocuments.presentation-template", + "application/vnd.collabio.xodocuments.spreadsheet", + "application/vnd.collabio.xodocuments.spreadsheet-template", + "application/vnd.collection+json", + "application/vnd.collection.doc+json", + "application/vnd.collection.next+json", + "application/vnd.comicbook+zip", + "application/vnd.comicbook-rar", + "application/vnd.commerce-battelle", + "application/vnd.commonspace", + "application/vnd.contact.cmsg", + "application/vnd.coreos.ignition+json", + "application/vnd.cosmocaller", + "application/vnd.crick.clicker", + "application/vnd.crick.clicker.keyboard", + "application/vnd.crick.clicker.palette", + "application/vnd.crick.clicker.template", + "application/vnd.crick.clicker.wordbank", + "application/vnd.criticaltools.wbs+xml", + "application/vnd.cryptii.pipe+json", + "application/vnd.crypto-shade-file", + "application/vnd.cryptomator.encrypted", + "application/vnd.cryptomator.vault", + "application/vnd.ctc-posml", + "application/vnd.ctct.ws+xml", + "application/vnd.cups-pdf", + "application/vnd.cups-postscript", + "application/vnd.cups-ppd", + "application/vnd.cups-raster", + "application/vnd.cups-raw", + "application/vnd.curl", + "application/vnd.curl.car", + "application/vnd.curl.pcurl", + "application/vnd.cxtf", + "application/vnd.cxzip", + "application/vnd.cyan.dean.root+xml", + "application/vnd.cybank", + "application/vnd.cyclonedx+json", + "application/vnd.cyclonedx+xml", + "application/vnd.d2l.coursepackage1p0+zip", + "application/vnd.d3m-dataset", + "application/vnd.d3m-problem", + "application/vnd.dart", + "application/vnd.data-vision.rdz", + "application/vnd.datalog", + "application/vnd.datapackage+json", + "application/vnd.dataresource+json", + "application/vnd.dbf", + "application/vnd.dcmp+xml", + "application/vnd.debian.binary-package", + "application/vnd.dece.data", + "application/vnd.dece.ttml+xml", + "application/vnd.dece.unspecified", + "application/vnd.dece.zip", + "application/vnd.deckyard.deck", + "application/vnd.denovo.fcselayout-link", + "application/vnd.desmume.movie", + "application/vnd.deut+json", + "application/vnd.dgl", + "application/vnd.digitalstack.document+zip", + "application/vnd.dir-bi.plate-dl-nosuffix", + "application/vnd.dm.delegation+xml", + "application/vnd.dna", + "application/vnd.document+json", + "application/vnd.dolby.mlp", + "application/vnd.dolby.mobile.1", + "application/vnd.dolby.mobile.2", + "application/vnd.doremir.scorecloud-binary-document", + "application/vnd.dpgraph", + "application/vnd.dreamfactory", + "application/vnd.drive+json", + "application/vnd.ds-keypoint", + "application/vnd.dtg.local", + "application/vnd.dtg.local.flash", + "application/vnd.dtg.local.html", + "application/vnd.dvb.ait", + "application/vnd.dvb.dash-playlist+xml", + "application/vnd.dvb.dvbisl+xml", + "application/vnd.dvb.dvbj", + "application/vnd.dvb.esgcontainer", + "application/vnd.dvb.ipdcdftnotifaccess", + "application/vnd.dvb.ipdcesgaccess", + "application/vnd.dvb.ipdcesgaccess2", + "application/vnd.dvb.ipdcesgpdd", + "application/vnd.dvb.ipdcroaming", + "application/vnd.dvb.iptv.alfec-base", + "application/vnd.dvb.iptv.alfec-enhancement", + "application/vnd.dvb.notif-aggregate-root+xml", + "application/vnd.dvb.notif-container+xml", + "application/vnd.dvb.notif-generic+xml", + "application/vnd.dvb.notif-ia-msglist+xml", + "application/vnd.dvb.notif-ia-registration-request+xml", + "application/vnd.dvb.notif-ia-registration-response+xml", + "application/vnd.dvb.notif-init+xml", + "application/vnd.dvb.pfr", + "application/vnd.dvb.service", + "application/vnd.dxr", + "application/vnd.dynageo", + "application/vnd.dzr", + "application/vnd.easykaraoke.cdgdownload", + "application/vnd.ecdis-update", + "application/vnd.ecip.rlp", + "application/vnd.eclipse.ditto+json", + "application/vnd.ecowin.chart", + "application/vnd.ecowin.filerequest", + "application/vnd.ecowin.fileupdate", + "application/vnd.ecowin.series", + "application/vnd.ecowin.seriesrequest", + "application/vnd.ecowin.seriesupdate", + "application/vnd.edulith.edux+json", + "application/vnd.efi.img", + "application/vnd.efi.iso", + "application/vnd.eln+zip", + "application/vnd.emclient.accessrequest+xml", + "application/vnd.enliven", + "application/vnd.enphase.envoy", + "application/vnd.eprints.data+xml", + "application/vnd.epson.esf", + "application/vnd.epson.msf", + "application/vnd.epson.quickanime", + "application/vnd.epson.salt", + "application/vnd.epson.ssf", + "application/vnd.ericsson.quickcall", + "application/vnd.erofs", + "application/vnd.espass-espass+zip", + "application/vnd.eszigno3+xml", + "application/vnd.etsi.aoc+xml", + "application/vnd.etsi.asic-e+zip", + "application/vnd.etsi.asic-s+zip", + "application/vnd.etsi.cug+xml", + "application/vnd.etsi.iptvcommand+xml", + "application/vnd.etsi.iptvdiscovery+xml", + "application/vnd.etsi.iptvprofile+xml", + "application/vnd.etsi.iptvsad-bc+xml", + "application/vnd.etsi.iptvsad-cod+xml", + "application/vnd.etsi.iptvsad-npvr+xml", + "application/vnd.etsi.iptvservice+xml", + "application/vnd.etsi.iptvsync+xml", + "application/vnd.etsi.iptvueprofile+xml", + "application/vnd.etsi.mcid+xml", + "application/vnd.etsi.mheg5", + "application/vnd.etsi.overload-control-policy-dataset+xml", + "application/vnd.etsi.pstn+xml", + "application/vnd.etsi.sci+xml", + "application/vnd.etsi.simservs+xml", + "application/vnd.etsi.timestamp-token", + "application/vnd.etsi.tsl+xml", + "application/vnd.etsi.tsl.der", + "application/vnd.eu.kasparian.car+json", + "application/vnd.eudora.data", + "application/vnd.evolv.ecig.profile", + "application/vnd.evolv.ecig.settings", + "application/vnd.evolv.ecig.theme", + "application/vnd.exstream-empower+zip", + "application/vnd.exstream-package", + "application/vnd.ezpix-album", + "application/vnd.ezpix-package", + "application/vnd.f-secure.mobile", + "application/vnd.faf+yaml", + "application/vnd.fafa+yaml", + "application/vnd.fafm+yaml", + "application/vnd.familysearch.gedcom+zip", + "application/vnd.fastcopy-disk-image", + "application/vnd.fdf", + "application/vnd.fdsn.mseed", + "application/vnd.fdsn.seed", + "application/vnd.fdsn.stationxml+xml", + "application/vnd.ffsns", + "application/vnd.fgb", + "application/vnd.ficlab.flb+zip", + "application/vnd.fiduswriter+zip", + "application/vnd.fiduswriter.book+zip", + "application/vnd.fiduswriter.template+zip", + "application/vnd.filmit.zfc", + "application/vnd.fints", + "application/vnd.firemonkeys.cloudcell", + "application/vnd.flographit", + "application/vnd.fluxtime.clip", + "application/vnd.font-fontforge-sfd", + "application/vnd.foritech.container", + "application/vnd.framemaker", + "application/vnd.freelog.comic", + "application/vnd.frogans.fnc", + "application/vnd.frogans.ltf", + "application/vnd.fsc.weblaunch", + "application/vnd.fujifilm.fb.docuworks", + "application/vnd.fujifilm.fb.docuworks.binder", + "application/vnd.fujifilm.fb.docuworks.container", + "application/vnd.fujifilm.fb.jfi+xml", + "application/vnd.fujitsu.oasys", + "application/vnd.fujitsu.oasys2", + "application/vnd.fujitsu.oasys3", + "application/vnd.fujitsu.oasysgp", + "application/vnd.fujitsu.oasysprs", + "application/vnd.fujixerox.art-ex", + "application/vnd.fujixerox.art4", + "application/vnd.fujixerox.ddd", + "application/vnd.fujixerox.docuworks", + "application/vnd.fujixerox.docuworks.binder", + "application/vnd.fujixerox.docuworks.container", + "application/vnd.fujixerox.hbpl", + "application/vnd.fut-misnet", + "application/vnd.futoin+cbor", + "application/vnd.futoin+json", + "application/vnd.fuzzysheet", + "application/vnd.g3pix.g3fc", + "application/vnd.ga4gh.passport+jwt", + "application/vnd.genomatix.tuxedo", + "application/vnd.genozip", + "application/vnd.gentics.grd+json", + "application/vnd.gentoo.catmetadata+xml", + "application/vnd.gentoo.ebuild", + "application/vnd.gentoo.eclass", + "application/vnd.gentoo.gpkg", + "application/vnd.gentoo.manifest", + "application/vnd.gentoo.pkgmetadata+xml", + "application/vnd.gentoo.xpak", + "application/vnd.geo+json", + "application/vnd.geocube+xml", + "application/vnd.geogebra.file", + "application/vnd.geogebra.pinboard", + "application/vnd.geogebra.slides", + "application/vnd.geogebra.tool", + "application/vnd.geometry-explorer", + "application/vnd.geonext", + "application/vnd.geoplan", + "application/vnd.geospace", + "application/vnd.gerber", + "application/vnd.globalplatform.card-content-mgt", + "application/vnd.globalplatform.card-content-mgt-response", + "application/vnd.gmx", + "application/vnd.gnu.taler.exchange+json", + "application/vnd.gnu.taler.merchant+json", + "application/vnd.google-apps.document", + "application/vnd.google-apps.presentation", + "application/vnd.google-apps.spreadsheet", + "application/vnd.google-earth.kml+xml", + "application/vnd.google-earth.kmz", + "application/vnd.gov.sk.e-form+xml", + "application/vnd.gov.sk.e-form+zip", + "application/vnd.gov.sk.xmldatacontainer+xml", + "application/vnd.gp3", + "application/vnd.gpxsee.map+xml", + "application/vnd.grafeq", + "application/vnd.gridmp", + "application/vnd.groove-account", + "application/vnd.groove-help", + "application/vnd.groove-identity-message", + "application/vnd.groove-injector", + "application/vnd.groove-tool-message", + "application/vnd.groove-tool-template", + "application/vnd.groove-vcard", + "application/vnd.hal+json", + "application/vnd.hal+xml", + "application/vnd.handheld-entertainment+xml", + "application/vnd.hbci", + "application/vnd.hc+json", + "application/vnd.hcl-bireports", + "application/vnd.hdfgroup.hdf4", + "application/vnd.hdfgroup.hdf5", + "application/vnd.hdt", + "application/vnd.heroku+json", + "application/vnd.hhe.lesson-player", + "application/vnd.hp-hpgl", + "application/vnd.hp-hpid", + "application/vnd.hp-hps", + "application/vnd.hp-jlyt", + "application/vnd.hp-pcl", + "application/vnd.hp-pclxl", + "application/vnd.hsl", + "application/vnd.httphone", + "application/vnd.hydrostatix.sof-data", + "application/vnd.hyper+json", + "application/vnd.hyper-item+json", + "application/vnd.hyperdrive+json", + "application/vnd.hzn-3d-crossword", + "application/vnd.ibm.afplinedata", + "application/vnd.ibm.electronic-media", + "application/vnd.ibm.minipay", + "application/vnd.ibm.modcap", + "application/vnd.ibm.rights-management", + "application/vnd.ibm.secure-container", + "application/vnd.iccprofile", + "application/vnd.ieee.1905", + "application/vnd.igloader", + "application/vnd.imagemeter.folder+zip", + "application/vnd.imagemeter.image+zip", + "application/vnd.immervision-ivp", + "application/vnd.immervision-ivu", + "application/vnd.ims.imsccv1p1", + "application/vnd.ims.imsccv1p2", + "application/vnd.ims.imsccv1p3", + "application/vnd.ims.lis.v2.result+json", + "application/vnd.ims.lti.v2.toolconsumerprofile+json", + "application/vnd.ims.lti.v2.toolproxy+json", + "application/vnd.ims.lti.v2.toolproxy.id+json", + "application/vnd.ims.lti.v2.toolsettings+json", + "application/vnd.ims.lti.v2.toolsettings.simple+json", + "application/vnd.informedcontrol.rms+xml", + "application/vnd.informix-visionary", + "application/vnd.infotech.project", + "application/vnd.infotech.project+xml", + "application/vnd.innopath.wamp.notification", + "application/vnd.insors.igm", + "application/vnd.intercon.formnet", + "application/vnd.intergeo", + "application/vnd.intertrust.digibox", + "application/vnd.intertrust.nncp", + "application/vnd.intu.qbo", + "application/vnd.intu.qfx", + "application/vnd.ipfs.ipns-record", + "application/vnd.ipld.car", + "application/vnd.ipld.dag-cbor", + "application/vnd.ipld.dag-json", + "application/vnd.ipld.raw", + "application/vnd.iptc.g2.catalogitem+xml", + "application/vnd.iptc.g2.conceptitem+xml", + "application/vnd.iptc.g2.knowledgeitem+xml", + "application/vnd.iptc.g2.newsitem+xml", + "application/vnd.iptc.g2.newsmessage+xml", + "application/vnd.iptc.g2.packageitem+xml", + "application/vnd.iptc.g2.planningitem+xml", + "application/vnd.ipunplugged.rcprofile", + "application/vnd.irepository.package+xml", + "application/vnd.is-xpr", + "application/vnd.isac.fcs", + "application/vnd.iso11783-10+zip", + "application/vnd.jam", + "application/vnd.japannet-directory-service", + "application/vnd.japannet-jpnstore-wakeup", + "application/vnd.japannet-payment-wakeup", + "application/vnd.japannet-registration", + "application/vnd.japannet-registration-wakeup", + "application/vnd.japannet-setstore-wakeup", + "application/vnd.japannet-verification", + "application/vnd.japannet-verification-wakeup", + "application/vnd.jcp.javame.midlet-rms", + "application/vnd.jisp", + "application/vnd.joost.joda-archive", + "application/vnd.jsk.isdn-ngn", + "application/vnd.jupyter", + "application/vnd.kahootz", + "application/vnd.kde.karbon", + "application/vnd.kde.kchart", + "application/vnd.kde.kformula", + "application/vnd.kde.kivio", + "application/vnd.kde.kontour", + "application/vnd.kde.kpresenter", + "application/vnd.kde.kspread", + "application/vnd.kde.kword", + "application/vnd.kdl", + "application/vnd.kenameaapp", + "application/vnd.keyman.kmp+zip", + "application/vnd.keyman.kmx", + "application/vnd.kidspiration", + "application/vnd.kinar", + "application/vnd.klypix+zip", + "application/vnd.koan", + "application/vnd.kodak-descriptor", + "application/vnd.las", + "application/vnd.las.las+json", + "application/vnd.las.las+xml", + "application/vnd.laszip", + "application/vnd.ldev.productlicensing", + "application/vnd.leap+json", + "application/vnd.liberty-request+xml", + "application/vnd.llamagraphics.life-balance.desktop", + "application/vnd.llamagraphics.life-balance.exchange+xml", + "application/vnd.logipipe.circuit+zip", + "application/vnd.loom", + "application/vnd.lotus-1-2-3", + "application/vnd.lotus-approach", + "application/vnd.lotus-freelance", + "application/vnd.lotus-notes", + "application/vnd.lotus-organizer", + "application/vnd.lotus-screencam", + "application/vnd.lotus-wordpro", + "application/vnd.lukuid.package+zip", + "application/vnd.macports.portpkg", + "application/vnd.majikah.bundle", + "application/vnd.majikah.mjksig", + "application/vnd.maml", + "application/vnd.mapbox-vector-tile", + "application/vnd.marlin.drm.actiontoken+xml", + "application/vnd.marlin.drm.conftoken+xml", + "application/vnd.marlin.drm.license+xml", + "application/vnd.marlin.drm.mdcf", + "application/vnd.mason+json", + "application/vnd.maxar.archive.3tz+zip", + "application/vnd.maxmind.maxmind-db", + "application/vnd.mcd", + "application/vnd.mdl", + "application/vnd.mdl-mbsdf", + "application/vnd.medcalcdata", + "application/vnd.mediastation.cdkey", + "application/vnd.medicalholodeck.recordxr", + "application/vnd.meridian-slingshot", + "application/vnd.mermaid", + "application/vnd.mfer", + "application/vnd.mfmp", + "application/vnd.micro+json", + "application/vnd.micrografx.flo", + "application/vnd.micrografx.igx", + "application/vnd.microsoft.portable-executable", + "application/vnd.microsoft.windows.thumbnail-cache", + "application/vnd.miele+json", + "application/vnd.mif", + "application/vnd.minisoft-hp3000-save", + "application/vnd.mitsubishi.misty-guard.trustweb", + "application/vnd.mobius.daf", + "application/vnd.mobius.dis", + "application/vnd.mobius.mbk", + "application/vnd.mobius.mqy", + "application/vnd.mobius.msl", + "application/vnd.mobius.plc", + "application/vnd.mobius.txf", + "application/vnd.modl", + "application/vnd.mohnetic", + "application/vnd.mophun.application", + "application/vnd.mophun.certificate", + "application/vnd.motorola.flexsuite", + "application/vnd.motorola.flexsuite.adsi", + "application/vnd.motorola.flexsuite.fis", + "application/vnd.motorola.flexsuite.gotap", + "application/vnd.motorola.flexsuite.kmr", + "application/vnd.motorola.flexsuite.ttc", + "application/vnd.motorola.flexsuite.wem", + "application/vnd.motorola.iprm", + "application/vnd.mozilla.xul+xml", + "application/vnd.ms-3mfdocument", + "application/vnd.ms-artgalry", + "application/vnd.ms-asf", + "application/vnd.ms-cab-compressed", + "application/vnd.ms-excel", + "application/vnd.ms-excel.addin.macroenabled.12", + "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "application/vnd.ms-excel.sheet.macroenabled.12", + "application/vnd.ms-excel.template.macroenabled.12", + "application/vnd.ms-fontobject", + "application/vnd.ms-htmlhelp", + "application/vnd.ms-ims", + "application/vnd.ms-lrm", + "application/vnd.ms-mediapackage", + "application/vnd.ms-office.activex+xml", + "application/vnd.ms-office.calx", + "application/vnd.ms-officetheme", + "application/vnd.ms-outlook", + "application/vnd.ms-pki.certstore", + "application/vnd.ms-pki.pko", + "application/vnd.ms-pki.seccat", + "application/vnd.ms-pki.stl", + "application/vnd.ms-playready.initiator+xml", + "application/vnd.ms-powerpoint", + "application/vnd.ms-powerpoint.addin.macroenabled.12", + "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "application/vnd.ms-powerpoint.slide.macroenabled.12", + "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "application/vnd.ms-powerpoint.template.macroenabled.12", + "application/vnd.ms-printdevicecapabilities+xml", + "application/vnd.ms-printschematicket+xml", + "application/vnd.ms-project", + "application/vnd.ms-tnef", + "application/vnd.ms-visio.viewer", + "application/vnd.ms-windows.devicepairing", + "application/vnd.ms-windows.nwprinting.oob", + "application/vnd.ms-windows.printerpairing", + "application/vnd.ms-windows.wsd.oob", + "application/vnd.ms-wmdrm.lic-chlg-req", + "application/vnd.ms-wmdrm.lic-resp", + "application/vnd.ms-wmdrm.meter-chlg-req", + "application/vnd.ms-wmdrm.meter-resp", + "application/vnd.ms-word.document.macroenabled.12", + "application/vnd.ms-word.template.macroenabled.12", + "application/vnd.ms-works", + "application/vnd.ms-wpl", + "application/vnd.ms-xpsdocument", + "application/vnd.msa-disk-image", + "application/vnd.mseq", + "application/vnd.msgpack", + "application/vnd.msign", + "application/vnd.multiad.creator", + "application/vnd.multiad.creator.cif", + "application/vnd.music-niff", + "application/vnd.musician", + "application/vnd.muvee.style", + "application/vnd.mynfc", + "application/vnd.nacamar.ybrid+json", + "application/vnd.nato.bindingdataobject+cbor", + "application/vnd.nato.bindingdataobject+json", + "application/vnd.nato.bindingdataobject+xml", + "application/vnd.nato.openxmlformats-package.iepd+zip", + "application/vnd.ncd.control", + "application/vnd.ncd.reference", + "application/vnd.nearst.inv+json", + "application/vnd.nebumind.line", + "application/vnd.nervana", + "application/vnd.netfpx", + "application/vnd.neurolanguage.nlu", + "application/vnd.nila.protobuf-bundle+zip", + "application/vnd.nimn", + "application/vnd.nintendo.nitro.rom", + "application/vnd.nintendo.snes.rom", + "application/vnd.nitf", + "application/vnd.noblenet-directory", + "application/vnd.noblenet-sealer", + "application/vnd.noblenet-web", + "application/vnd.nokia.catalogs", + "application/vnd.nokia.conml+wbxml", + "application/vnd.nokia.conml+xml", + "application/vnd.nokia.iptv.config+xml", + "application/vnd.nokia.isds-radio-presets", + "application/vnd.nokia.landmark+wbxml", + "application/vnd.nokia.landmark+xml", + "application/vnd.nokia.landmarkcollection+xml", + "application/vnd.nokia.n-gage.ac+xml", + "application/vnd.nokia.n-gage.data", + "application/vnd.nokia.n-gage.symbian.install", + "application/vnd.nokia.ncd", + "application/vnd.nokia.pcd+wbxml", + "application/vnd.nokia.pcd+xml", + "application/vnd.nokia.radio-preset", + "application/vnd.nokia.radio-presets", + "application/vnd.nomos+json", + "application/vnd.novadigm.edm", + "application/vnd.novadigm.edx", + "application/vnd.novadigm.ext", + "application/vnd.ntt-local.content-share", + "application/vnd.ntt-local.file-transfer", + "application/vnd.ntt-local.ogw_remote-access", + "application/vnd.ntt-local.sip-ta_remote", + "application/vnd.ntt-local.sip-ta_tcp_stream", + "application/vnd.nubaltec.nudoku-game", + "application/vnd.oai.workflows", + "application/vnd.oai.workflows+json", + "application/vnd.oai.workflows+yaml", + "application/vnd.oasis.opendocument.base", + "application/vnd.oasis.opendocument.chart", + "application/vnd.oasis.opendocument.chart-template", + "application/vnd.oasis.opendocument.database", + "application/vnd.oasis.opendocument.formula", + "application/vnd.oasis.opendocument.formula-template", + "application/vnd.oasis.opendocument.graphics", + "application/vnd.oasis.opendocument.graphics-template", + "application/vnd.oasis.opendocument.image", + "application/vnd.oasis.opendocument.image-template", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.presentation-template", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.spreadsheet-template", + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.text-master", + "application/vnd.oasis.opendocument.text-master-template", + "application/vnd.oasis.opendocument.text-template", + "application/vnd.oasis.opendocument.text-web", + "application/vnd.obn", + "application/vnd.ocf+cbor", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oftn.l10n+json", + "application/vnd.oipf.contentaccessdownload+xml", + "application/vnd.oipf.contentaccessstreaming+xml", + "application/vnd.oipf.cspg-hexbinary", + "application/vnd.oipf.dae.svg+xml", + "application/vnd.oipf.dae.xhtml+xml", + "application/vnd.oipf.mippvcontrolmessage+xml", + "application/vnd.oipf.pae.gem", + "application/vnd.oipf.spdiscovery+xml", + "application/vnd.oipf.spdlist+xml", + "application/vnd.oipf.ueprofile+xml", + "application/vnd.oipf.userprofile+xml", + "application/vnd.olpc-sugar", + "application/vnd.oma-scws-config", + "application/vnd.oma-scws-http-request", + "application/vnd.oma-scws-http-response", + "application/vnd.oma.bcast.associated-procedure-parameter+xml", + "application/vnd.oma.bcast.drm-trigger+xml", + "application/vnd.oma.bcast.imd+xml", + "application/vnd.oma.bcast.ltkm", + "application/vnd.oma.bcast.notification+xml", + "application/vnd.oma.bcast.provisioningtrigger", + "application/vnd.oma.bcast.sgboot", + "application/vnd.oma.bcast.sgdd+xml", + "application/vnd.oma.bcast.sgdu", + "application/vnd.oma.bcast.simple-symbol-container", + "application/vnd.oma.bcast.smartcard-trigger+xml", + "application/vnd.oma.bcast.sprov+xml", + "application/vnd.oma.bcast.stkm", + "application/vnd.oma.cab-address-book+xml", + "application/vnd.oma.cab-feature-handler+xml", + "application/vnd.oma.cab-pcc+xml", + "application/vnd.oma.cab-subs-invite+xml", + "application/vnd.oma.cab-user-prefs+xml", + "application/vnd.oma.dcd", + "application/vnd.oma.dcdc", + "application/vnd.oma.dd2+xml", + "application/vnd.oma.drm.risd+xml", + "application/vnd.oma.group-usage-list+xml", + "application/vnd.oma.lwm2m+cbor", + "application/vnd.oma.lwm2m+json", + "application/vnd.oma.lwm2m+tlv", + "application/vnd.oma.pal+xml", + "application/vnd.oma.poc.detailed-progress-report+xml", + "application/vnd.oma.poc.final-report+xml", + "application/vnd.oma.poc.groups+xml", + "application/vnd.oma.poc.invocation-descriptor+xml", + "application/vnd.oma.poc.optimized-progress-report+xml", + "application/vnd.oma.push", + "application/vnd.oma.scidm.messages+xml", + "application/vnd.oma.xcap-directory+xml", + "application/vnd.omads-email+xml", + "application/vnd.omads-file+xml", + "application/vnd.omads-folder+xml", + "application/vnd.omaloc-supl-init", + "application/vnd.oms.cellular-cose-content+cbor", + "application/vnd.onepager", + "application/vnd.onepagertamp", + "application/vnd.onepagertamx", + "application/vnd.onepagertat", + "application/vnd.onepagertatp", + "application/vnd.onepagertatx", + "application/vnd.onvif.metadata", + "application/vnd.ootmm.patch+zip", + "application/vnd.openblox.game+xml", + "application/vnd.openblox.game-binary", + "application/vnd.openeye.oeb", + "application/vnd.openofficeorg.extension", + "application/vnd.openprinttag", + "application/vnd.openstreetmap.data+xml", + "application/vnd.opentimestamps.ots", + "application/vnd.openvpi.dspx+json", + "application/vnd.openxmlformats-officedocument.custom-properties+xml", + "application/vnd.openxmlformats-officedocument.customxmlproperties+xml", + "application/vnd.openxmlformats-officedocument.drawing+xml", + "application/vnd.openxmlformats-officedocument.drawingml.chart+xml", + "application/vnd.openxmlformats-officedocument.drawingml.chartshapes+xml", + "application/vnd.openxmlformats-officedocument.drawingml.diagramcolors+xml", + "application/vnd.openxmlformats-officedocument.drawingml.diagramdata+xml", + "application/vnd.openxmlformats-officedocument.drawingml.diagramlayout+xml", + "application/vnd.openxmlformats-officedocument.drawingml.diagramstyle+xml", + "application/vnd.openxmlformats-officedocument.extended-properties+xml", + "application/vnd.openxmlformats-officedocument.presentationml.commentauthors+xml", + "application/vnd.openxmlformats-officedocument.presentationml.comments+xml", + "application/vnd.openxmlformats-officedocument.presentationml.handoutmaster+xml", + "application/vnd.openxmlformats-officedocument.presentationml.notesmaster+xml", + "application/vnd.openxmlformats-officedocument.presentationml.notesslide+xml", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml", + "application/vnd.openxmlformats-officedocument.presentationml.presprops+xml", + "application/vnd.openxmlformats-officedocument.presentationml.slide", + "application/vnd.openxmlformats-officedocument.presentationml.slide+xml", + "application/vnd.openxmlformats-officedocument.presentationml.slidelayout+xml", + "application/vnd.openxmlformats-officedocument.presentationml.slidemaster+xml", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow.main+xml", + "application/vnd.openxmlformats-officedocument.presentationml.slideupdateinfo+xml", + "application/vnd.openxmlformats-officedocument.presentationml.tablestyles+xml", + "application/vnd.openxmlformats-officedocument.presentationml.tags+xml", + "application/vnd.openxmlformats-officedocument.presentationml.template", + "application/vnd.openxmlformats-officedocument.presentationml.template.main+xml", + "application/vnd.openxmlformats-officedocument.presentationml.viewprops+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.calcchain+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.connections+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.dialogsheet+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.externallink+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotcachedefinition+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotcacherecords+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.pivottable+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.querytable+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.revisionheaders+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.revisionlog+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sharedstrings+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheetmetadata+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.table+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.tablesinglecells+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.usernames+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.volatiledependencies+xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml", + "application/vnd.openxmlformats-officedocument.theme+xml", + "application/vnd.openxmlformats-officedocument.themeoverride+xml", + "application/vnd.openxmlformats-officedocument.vmldrawing", + "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document.glossary+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.fonttable+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.websettings+xml", + "application/vnd.openxmlformats-package.core-properties+xml", + "application/vnd.openxmlformats-package.digital-signature-xmlsignature+xml", + "application/vnd.openxmlformats-package.relationships+xml", + "application/vnd.oracle.resource+json", + "application/vnd.orange.indata", + "application/vnd.osa.netdeploy", + "application/vnd.osgeo.mapguide.package", + "application/vnd.osgi.bundle", + "application/vnd.osgi.dp", + "application/vnd.osgi.subsystem", + "application/vnd.otps.ct-kip+xml", + "application/vnd.oxli.countgraph", + "application/vnd.pagerduty+json", + "application/vnd.palm", + "application/vnd.panoply", + "application/vnd.paos.xml", + "application/vnd.pasti-stx-disk-image", + "application/vnd.patentdive", + "application/vnd.patientecommsdoc", + "application/vnd.pawaafile", + "application/vnd.pcos", + "application/vnd.pg.format", + "application/vnd.pg.osasli", + "application/vnd.phbk+xml", + "application/vnd.piaccess.application-licence", + "application/vnd.picsel", + "application/vnd.pmi.widget", + "application/vnd.pmtiles", + "application/vnd.poc.group-advertisement+xml", + "application/vnd.pocketlearn", + "application/vnd.portableweb+zip", + "application/vnd.powerbuilder6", + "application/vnd.powerbuilder6-s", + "application/vnd.powerbuilder7", + "application/vnd.powerbuilder7-s", + "application/vnd.powerbuilder75", + "application/vnd.powerbuilder75-s", + "application/vnd.pp.systemverify+xml", + "application/vnd.preminet", + "application/vnd.previewsystems.box", + "application/vnd.prismatic", + "application/vnd.prml+yaml", + "application/vnd.project-graph", + "application/vnd.proteus.magazine", + "application/vnd.psfs", + "application/vnd.pt.mundusmundi", + "application/vnd.publishare-delta-tree", + "application/vnd.pvi.ptid1", + "application/vnd.pwg-multiplexed", + "application/vnd.pwg-xhtml-print+xml", + "application/vnd.pyon+json", + "application/vnd.qualcomm.brew-app-res", + "application/vnd.quarantainenet", + "application/vnd.quark.quarkxpress", + "application/vnd.quobject-quoxdocument", + "application/vnd.r74n.sandboxels+json", + "application/vnd.radisys.moml+xml", + "application/vnd.radisys.msml+xml", + "application/vnd.radisys.msml-audit+xml", + "application/vnd.radisys.msml-audit-conf+xml", + "application/vnd.radisys.msml-audit-conn+xml", + "application/vnd.radisys.msml-audit-dialog+xml", + "application/vnd.radisys.msml-audit-stream+xml", + "application/vnd.radisys.msml-conf+xml", + "application/vnd.radisys.msml-dialog+xml", + "application/vnd.radisys.msml-dialog-base+xml", + "application/vnd.radisys.msml-dialog-fax-detect+xml", + "application/vnd.radisys.msml-dialog-fax-sendrecv+xml", + "application/vnd.radisys.msml-dialog-group+xml", + "application/vnd.radisys.msml-dialog-speech+xml", + "application/vnd.radisys.msml-dialog-transform+xml", + "application/vnd.rainstor.data", + "application/vnd.rapid", + "application/vnd.rar", + "application/vnd.realvnc.bed", + "application/vnd.recordare.musicxml", + "application/vnd.recordare.musicxml+xml", + "application/vnd.rego", + "application/vnd.relpipe", + "application/vnd.renlearn.rlprint", + "application/vnd.resilient.logic", + "application/vnd.restful+json", + "application/vnd.rig.cryptonote", + "application/vnd.rn-realmedia", + "application/vnd.rn-realmedia-vbr", + "application/vnd.rn-rn_music_package", + "application/vnd.route66.link66+xml", + "application/vnd.rs-274x", + "application/vnd.ruckus.download", + "application/vnd.s3sms", + "application/vnd.sailingtracker.track", + "application/vnd.salvanote.sal", + "application/vnd.sar", + "application/vnd.sbm.cid", + "application/vnd.sbm.mid2", + "application/vnd.scribus", + "application/vnd.sealed.3df", + "application/vnd.sealed.csf", + "application/vnd.sealed.doc", + "application/vnd.sealed.eml", + "application/vnd.sealed.mht", + "application/vnd.sealed.net", + "application/vnd.sealed.ppt", + "application/vnd.sealed.tiff", + "application/vnd.sealed.xls", + "application/vnd.sealedmedia.softseal.html", + "application/vnd.sealedmedia.softseal.pdf", + "application/vnd.seemail", + "application/vnd.seis+json", + "application/vnd.sema", + "application/vnd.semd", + "application/vnd.semf", + "application/vnd.shade-save-file", + "application/vnd.shana.informed.formdata", + "application/vnd.shana.informed.formtemplate", + "application/vnd.shana.informed.interchange", + "application/vnd.shana.informed.package", + "application/vnd.shootproof+json", + "application/vnd.shopkick+json", + "application/vnd.shp", + "application/vnd.shx", + "application/vnd.sigrok.session", + "application/vnd.simtech-mindmapper", + "application/vnd.siren+json", + "application/vnd.sirtx.vmv0", + "application/vnd.sketchometry", + "application/vnd.smaf", + "application/vnd.smart.notebook", + "application/vnd.smart.teacher", + "application/vnd.smintio.portals.archive", + "application/vnd.snesdev-page-table", + "application/vnd.softpres-ipf-disk-image", + "application/vnd.software602.filler.form+xml", + "application/vnd.software602.filler.form-xml-zip", + "application/vnd.solent.sdkm+xml", + "application/vnd.spotfire.dxp", + "application/vnd.spotfire.sfs", + "application/vnd.sqlite3", + "application/vnd.sri", + "application/vnd.sss-cod", + "application/vnd.sss-dtf", + "application/vnd.sss-ntf", + "application/vnd.stardivision.calc", + "application/vnd.stardivision.draw", + "application/vnd.stardivision.impress", + "application/vnd.stardivision.math", + "application/vnd.stardivision.writer", + "application/vnd.stardivision.writer-global", + "application/vnd.stepmania.package", + "application/vnd.stepmania.stepchart", + "application/vnd.street-stream", + "application/vnd.sun.wadl+xml", + "application/vnd.sun.xml.calc", + "application/vnd.sun.xml.calc.template", + "application/vnd.sun.xml.draw", + "application/vnd.sun.xml.draw.template", + "application/vnd.sun.xml.impress", + "application/vnd.sun.xml.impress.template", + "application/vnd.sun.xml.math", + "application/vnd.sun.xml.writer", + "application/vnd.sun.xml.writer.global", + "application/vnd.sun.xml.writer.template", + "application/vnd.supercard-pro-disk-image", + "application/vnd.superfile.super", + "application/vnd.sus-calendar", + "application/vnd.svd", + "application/vnd.svr.receipt+json", + "application/vnd.swiftview-ics", + "application/vnd.sybyl.mol2", + "application/vnd.sycle+xml", + "application/vnd.syft+json", + "application/vnd.symbian.install", + "application/vnd.syncml+xml", + "application/vnd.syncml.dm+wbxml", + "application/vnd.syncml.dm+xml", + "application/vnd.syncml.dm.notification", + "application/vnd.syncml.dmddf+wbxml", + "application/vnd.syncml.dmddf+xml", + "application/vnd.syncml.dmtnds+wbxml", + "application/vnd.syncml.dmtnds+xml", + "application/vnd.syncml.ds.notification", + "application/vnd.tablafocus.notation", + "application/vnd.tableschema+json", + "application/vnd.tao.intent-module-archive", + "application/vnd.tcpdump.pcap", + "application/vnd.think-cell.ppttc+json", + "application/vnd.tmd.mediaflex.api+xml", + "application/vnd.tml", + "application/vnd.tmobile-livetv", + "application/vnd.tri.onesource", + "application/vnd.trid.tpt", + "application/vnd.triscape.mxs", + "application/vnd.trueapp", + "application/vnd.truedoc", + "application/vnd.ubisoft.webplayer", + "application/vnd.ufdl", + "application/vnd.uic.dosipas.v1", + "application/vnd.uic.dosipas.v2", + "application/vnd.uic.osdm+json", + "application/vnd.uic.tlb-fcb", + "application/vnd.uiq.theme", + "application/vnd.umajin", + "application/vnd.unity", + "application/vnd.uoml+xml", + "application/vnd.uplanet.alert", + "application/vnd.uplanet.alert-wbxml", + "application/vnd.uplanet.bearer-choice", + "application/vnd.uplanet.bearer-choice-wbxml", + "application/vnd.uplanet.cacheop", + "application/vnd.uplanet.cacheop-wbxml", + "application/vnd.uplanet.channel", + "application/vnd.uplanet.channel-wbxml", + "application/vnd.uplanet.list", + "application/vnd.uplanet.list-wbxml", + "application/vnd.uplanet.listcmd", + "application/vnd.uplanet.listcmd-wbxml", + "application/vnd.uplanet.signal", + "application/vnd.uri-map", + "application/vnd.valve.source.material", + "application/vnd.vcx", + "application/vnd.vd-study", + "application/vnd.vectorworks", + "application/vnd.vel+json", + "application/vnd.veraison.nvidia-gpu-evidence+json", + "application/vnd.veraison.tsm-report+cbor", + "application/vnd.veraison.tsm-report+json", + "application/vnd.verifier-attestation+jwt", + "application/vnd.verimatrix.vcas", + "application/vnd.veritone.aion+json", + "application/vnd.vertifile.pvf", + "application/vnd.veryant.thin", + "application/vnd.ves.encrypted", + "application/vnd.vidsoft.vidconference", + "application/vnd.vimina.vma", + "application/vnd.visio", + "application/vnd.visionary", + "application/vnd.vividence.scriptfile", + "application/vnd.vocalshaper.vsp4", + "application/vnd.vsf", + "application/vnd.vuq", + "application/vnd.wantverse", + "application/vnd.wap.sic", + "application/vnd.wap.slc", + "application/vnd.wap.wbxml", + "application/vnd.wap.wmlc", + "application/vnd.wap.wmlscriptc", + "application/vnd.wasmflow.wafl", + "application/vnd.webturbo", + "application/vnd.wfa.dpp", + "application/vnd.wfa.p2p", + "application/vnd.wfa.wsc", + "application/vnd.windows.devicepairing", + "application/vnd.wmap", + "application/vnd.wmc", + "application/vnd.wmf.bootstrap", + "application/vnd.wolfram.mathematica", + "application/vnd.wolfram.mathematica.package", + "application/vnd.wolfram.player", + "application/vnd.wordlift", + "application/vnd.wordperfect", + "application/vnd.wqd", + "application/vnd.wrq-hp3000-labelled", + "application/vnd.wt.stf", + "application/vnd.wv.csp+wbxml", + "application/vnd.wv.csp+xml", + "application/vnd.wv.ssp+xml", + "application/vnd.xacml+json", + "application/vnd.xara", + "application/vnd.xarin.cpj", + "application/vnd.xcdn", + "application/vnd.xecrets-encrypted", + "application/vnd.xfdl", + "application/vnd.xfdl.webform", + "application/vnd.xmi+xml", + "application/vnd.xmpie.cpkg", + "application/vnd.xmpie.dpkg", + "application/vnd.xmpie.plan", + "application/vnd.xmpie.ppkg", + "application/vnd.xmpie.xlim", + "application/vnd.yamaha.hv-dic", + "application/vnd.yamaha.hv-script", + "application/vnd.yamaha.hv-voice", + "application/vnd.yamaha.openscoreformat", + "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "application/vnd.yamaha.remote-setup", + "application/vnd.yamaha.smaf-audio", + "application/vnd.yamaha.smaf-phrase", + "application/vnd.yamaha.through-ngn", + "application/vnd.yamaha.tunnel-udpencap", + "application/vnd.yaoweme", + "application/vnd.yellowriver-custom-menu", + "application/vnd.youtube.yt", + "application/vnd.zoho-document.writer", + "application/vnd.zoho-presentation.show", + "application/vnd.zoho.spreadsheetml.sheet", + "application/vnd.zul", + "application/vnd.zzazz.deck+xml", + "application/voicexml+xml", + "application/voucher-cms+json", + "application/voucher-jws+json", + "application/vp", + "application/vp+cose", + "application/vp+jwt", + "application/vp+sd-jwt", + "application/vq-rtcpxr", + "application/vsix", + "application/wasm", + "application/watcherinfo+xml", + "application/webpush-options+json", + "application/whoispp-query", + "application/whoispp-response", + "application/widget", + "application/windows-library+xml", + "application/windows-search-connector+xml", + "application/winhlp", + "application/wita", + "application/wlmoviemaker", + "application/wordperfect5.1", + "application/wsdl+xml", + "application/wspolicy+xml", + "application/x-7z-compressed", + "application/x-abiword", + "application/x-ace-compressed", + "application/x-arj", + "application/x-authorware-bin", + "application/x-authorware-map", + "application/x-authorware-seg", + "application/x-bcpio", + "application/x-bittorrent", + "application/x-blorb", + "application/x-bridge-url", + "application/x-bzip", + "application/x-bzip2", + "application/x-cbr", + "application/x-cdf", + "application/x-cdlink", + "application/x-cfs-compressed", + "application/x-chat", + "application/x-chess-pgn", + "application/x-chrome-extension", + "application/x-cocoa", + "application/x-compress", + "application/x-compressed", + "application/x-cpio", + "application/x-csh", + "application/x-debian-package", + "application/x-dgc-compressed", + "application/x-director", + "application/x-doom", + "application/x-dtbncx+xml", + "application/x-dtbook+xml", + "application/x-dtbresource+xml", + "application/x-dvi", + "application/x-dxf", + "application/x-endace-erf", + "application/x-eva", + "application/x-font-bdf", + "application/x-font-ghostscript", + "application/x-font-linux-psf", + "application/x-font-pcf", + "application/x-font-snf", + "application/x-font-ttf", + "application/x-font-type1", + "application/x-freearc", + "application/x-gca-compressed", + "application/x-glulx", + "application/x-gnumeric", + "application/x-gramps-xml", + "application/x-gtar", + "application/x-gzip", + "application/x-hdf", + "application/x-httpd-php", + "application/x-install-instructions", + "application/x-internet-signup", + "application/x-iphone", + "application/x-itunes-ipa", + "application/x-itunes-ipg", + "application/x-itunes-ipsw", + "application/x-itunes-ite", + "application/x-itunes-itlp", + "application/x-itunes-itms", + "application/x-itunes-itpc", + "application/x-java-applet", + "application/x-java-archive-diff", + "application/x-java-jnlp-file", + "application/x-koan", + "application/x-latex", + "application/x-lua-bytecode", + "application/x-lzh-compressed", + "application/x-makeself", + "application/x-mie", + "application/x-miva-compiled", + "application/x-mmxp", + "application/x-mobipocket-ebook", + "application/x-ms-application", + "application/x-ms-license", + "application/x-ms-manifest", + "application/x-ms-reader", + "application/x-ms-shortcut", + "application/x-ms-vsto", + "application/x-ms-wmd", + "application/x-ms-wmz", + "application/x-ms-xbap", + "application/x-msaccess", + "application/x-msbinder", + "application/x-mscardfile", + "application/x-msclip", + "application/x-msdownload", + "application/x-msmediaview", + "application/x-msmetafile", + "application/x-msmoney", + "application/x-mspublisher", + "application/x-msschedule", + "application/x-msterminal", + "application/x-mswrite", + "application/x-netcdf", + "application/x-ns-proxy-autoconfig", + "application/x-nzb", + "application/x-oleobject", + "application/x-parquet", + "application/x-perfmon", + "application/x-perl", + "application/x-pkcs12", + "application/x-pkcs7-certificates", + "application/x-pkcs7-certreqresp", + "application/x-pki-message", + "application/x-podcast", + "application/x-quicktimeplayer", + "application/x-rar-compressed", + "application/x-research-info-systems", + "application/x-safari-safariextz", + "application/x-safari-webarchive", + "application/x-sgimb", + "application/x-sh", + "application/x-shar", + "application/x-shockwave-flash", + "application/x-silverlight-app", + "application/x-smaf", + "application/x-sql", + "application/x-stuffit", + "application/x-stuffitx", + "application/x-subrip", + "application/x-sv4cpio", + "application/x-sv4crc", + "application/x-t3vm-image", + "application/x-tads", + "application/x-tar", + "application/x-tcl", + "application/x-tex", + "application/x-tex-tfm", + "application/x-texinfo", + "application/x-tgif", + "application/x-troff", + "application/x-troff-man", + "application/x-troff-me", + "application/x-troff-ms", + "application/x-ustar", + "application/x-virtualbox-hdd", + "application/x-virtualbox-ova", + "application/x-virtualbox-ovf", + "application/x-virtualbox-vbox", + "application/x-virtualbox-vbox-extpack", + "application/x-virtualbox-vdi", + "application/x-virtualbox-vhd", + "application/x-virtualbox-vmdk", + "application/x-wais-source", + "application/x-web-app-manifest+json", + "application/x-wlpg-detect", + "application/x-wlpg3-detect", + "application/x-www-form-urlencoded", + "application/x-x509-ca-cert", + "application/x-x509-ca-ra-cert", + "application/x-x509-next-ca-cert", + "application/x-xfig", + "application/x-xliff+xml", + "application/x-xpinstall", + "application/x-xz", + "application/x-zmachine", + "application/x400-bp", + "application/xacml+xml", + "application/xaml+xml", + "application/xcap-att+xml", + "application/xcap-caps+xml", + "application/xcap-diff+xml", + "application/xcap-el+xml", + "application/xcap-error+xml", + "application/xcap-ns+xml", + "application/xcon-conference-info+xml", + "application/xcon-conference-info-diff+xml", + "application/xenc+xml", + "application/xfdf", + "application/xhtml+xml", + "application/xliff+xml", + "application/xml", + "application/xml-dtd", + "application/xml-external-parsed-entity", + "application/xml-patch+xml", + "application/xmpp+xml", + "application/xop+xml", + "application/xproc+xml", + "application/xslt+xml", + "application/xspf+xml", + "application/xv+xml", + "application/yaml", + "application/yang", + "application/yang-data+cbor", + "application/yang-data+json", + "application/yang-data+xml", + "application/yang-patch+json", + "application/yang-patch+xml", + "application/yang-sid+json", + "application/yin+xml", + "application/zip", + "application/zlib", + "application/zstd", + "audio/1d-interleaved-parityfec", + "audio/32kadpcm", + "audio/3gpp", + "audio/3gpp2", + "audio/aac", + "audio/ac3", + "audio/aiff", + "audio/amr", + "audio/amr-wb", + "audio/amr-wb+", + "audio/annodex", + "audio/aptx", + "audio/asc", + "audio/atrac-advanced-lossless", + "audio/atrac-x", + "audio/atrac3", + "audio/audible", + "audio/basic", + "audio/bv16", + "audio/bv32", + "audio/clearmode", + "audio/cn", + "audio/dat12", + "audio/dls", + "audio/dsr-es201108", + "audio/dsr-es202050", + "audio/dsr-es202211", + "audio/dsr-es202212", + "audio/dv", + "audio/dvi4", + "audio/eac3", + "audio/encaprtp", + "audio/evrc", + "audio/evrc-qcp", + "audio/evrc0", + "audio/evrc1", + "audio/evrcb", + "audio/evrcb0", + "audio/evrcb1", + "audio/evrcnw", + "audio/evrcnw0", + "audio/evrcnw1", + "audio/evrcwb", + "audio/evrcwb0", + "audio/evrcwb1", + "audio/evs", + "audio/example", + "audio/flac", + "audio/flexfec", + "audio/fwdred", + "audio/g711-0", + "audio/g719", + "audio/g722", + "audio/g7221", + "audio/g723", + "audio/g726-16", + "audio/g726-24", + "audio/g726-32", + "audio/g726-40", + "audio/g728", + "audio/g729", + "audio/g7291", + "audio/g729d", + "audio/g729e", + "audio/gsm", + "audio/gsm-efr", + "audio/gsm-hr-08", + "audio/ilbc", + "audio/ip-mr_v2.5", + "audio/l16", + "audio/l20", + "audio/l24", + "audio/l8", + "audio/lpc", + "audio/m4a", + "audio/m4b", + "audio/m4p", + "audio/matroska", + "audio/melp", + "audio/melp1200", + "audio/melp2400", + "audio/melp600", + "audio/mhas", + "audio/mid", + "audio/midi", + "audio/midi-clip", + "audio/mobile-xmf", + "audio/mp4", + "audio/mp4a-latm", + "audio/mpa", + "audio/mpa-robust", + "audio/mpeg", + "audio/mpeg4-generic", + "audio/ogg", + "audio/opus", + "audio/parityfec", + "audio/pcma", + "audio/pcma-wb", + "audio/pcmu", + "audio/pcmu-wb", + "audio/prs.aaud", + "audio/prs.sid", + "audio/qcelp", + "audio/raptorfec", + "audio/red", + "audio/rtp-enc-aescm128", + "audio/rtp-midi", + "audio/rtploopback", + "audio/rtx", + "audio/s3m", + "audio/scip", + "audio/scpls", + "audio/silk", + "audio/smv", + "audio/smv-qcp", + "audio/smv0", + "audio/sofa", + "audio/soundfont", + "audio/sp-midi", + "audio/speex", + "audio/t140c", + "audio/t38", + "audio/telephone-event", + "audio/tetra_acelp", + "audio/tetra_acelp_bb", + "audio/tone", + "audio/tsvcis", + "audio/uemclip", + "audio/ulpfec", + "audio/usac", + "audio/vdvi", + "audio/vmr-wb", + "audio/vnd.3gpp.iufp", + "audio/vnd.4sb", + "audio/vnd.audible.aax", + "audio/vnd.audiokoz", + "audio/vnd.blockfact.facta", + "audio/vnd.celp", + "audio/vnd.cisco.nse", + "audio/vnd.cmles.radio-events", + "audio/vnd.cns.anp1", + "audio/vnd.cns.inf1", + "audio/vnd.dece.audio", + "audio/vnd.digital-winds", + "audio/vnd.dlna.adts", + "audio/vnd.dolby.heaac.1", + "audio/vnd.dolby.heaac.2", + "audio/vnd.dolby.mlp", + "audio/vnd.dolby.mps", + "audio/vnd.dolby.pl2", + "audio/vnd.dolby.pl2x", + "audio/vnd.dolby.pl2z", + "audio/vnd.dolby.pulse.1", + "audio/vnd.dra", + "audio/vnd.dts", + "audio/vnd.dts.hd", + "audio/vnd.dts.uhd", + "audio/vnd.dvb.file", + "audio/vnd.everad.plj", + "audio/vnd.hns.audio", + "audio/vnd.lucent.voice", + "audio/vnd.ms-playready.media.pya", + "audio/vnd.nokia.mobile-xmf", + "audio/vnd.nortel.vbk", + "audio/vnd.nuera.ecelp4800", + "audio/vnd.nuera.ecelp7470", + "audio/vnd.nuera.ecelp9600", + "audio/vnd.octel.sbc", + "audio/vnd.opennbs.nbs", + "audio/vnd.presonus.multitrack", + "audio/vnd.qcelp", + "audio/vnd.rhetorex.32kadpcm", + "audio/vnd.rip", + "audio/vnd.sealedmedia.softseal.mpeg", + "audio/vnd.vmx.cvsd", + "audio/vorbis", + "audio/vorbis-config", + "audio/wav", + "audio/webm", + "audio/x-caf", + "audio/x-gsm", + "audio/x-m4r", + "audio/x-matroska", + "audio/x-mpegurl", + "audio/x-ms-wax", + "audio/x-ms-wma", + "audio/x-pn-realaudio", + "audio/x-pn-realaudio-plugin", + "audio/x-sd2", + "audio/x-smd", + "audio/xm", + "chemical/x-cdx", + "chemical/x-cif", + "chemical/x-cmdf", + "chemical/x-cml", + "chemical/x-csml", + "chemical/x-xyz", + "drawing/x-dwf", + "font/collection", + "font/otf", + "font/sfnt", + "font/ttf", + "font/woff", + "font/woff2", + "haptics/hjif", + "haptics/hmpg", + "haptics/ivs", + "image/aces", + "image/apng", + "image/avci", + "image/avcs", + "image/avif", + "image/avif-sequence", + "image/bmp", + "image/cgm", + "image/cis-cod", + "image/dicom-rle", + "image/dpx", + "image/emf", + "image/example", + "image/fits", + "image/g3fax", + "image/gif", + "image/heic", + "image/heic-sequence", + "image/heif", + "image/heif-sequence", + "image/hej2k", + "image/hsj2", + "image/ief", + "image/j2c", + "image/jaii", + "image/jais", + "image/jls", + "image/jp2", + "image/jpeg", + "image/jph", + "image/jphc", + "image/jpm", + "image/jpx", + "image/jxl", + "image/jxr", + "image/jxra", + "image/jxrs", + "image/jxs", + "image/jxsc", + "image/jxsi", + "image/jxss", + "image/ktx", + "image/ktx2", + "image/naplps", + "image/pict", + "image/png", + "image/prs.aimg", + "image/prs.btif", + "image/prs.pti", + "image/pwg-raster", + "image/sgi", + "image/svg+xml", + "image/t38", + "image/tiff", + "image/tiff-fx", + "image/vnd.adobe.photoshop", + "image/vnd.airzip.accelerator.azv", + "image/vnd.blockfact.facti", + "image/vnd.clip", + "image/vnd.cns.inf2", + "image/vnd.dece.graphic", + "image/vnd.djvu", + "image/vnd.dvb.subtitle", + "image/vnd.dwg", + "image/vnd.dxf", + "image/vnd.fastbidsheet", + "image/vnd.fpx", + "image/vnd.fst", + "image/vnd.fujixerox.edmics-mmr", + "image/vnd.fujixerox.edmics-rlc", + "image/vnd.globalgraphics.pgb", + "image/vnd.microsoft.icon", + "image/vnd.mix", + "image/vnd.mozilla.apng", + "image/vnd.ms-modi", + "image/vnd.ms-photo", + "image/vnd.net-fpx", + "image/vnd.pco.b16", + "image/vnd.radiance", + "image/vnd.rn-realflash", + "image/vnd.sealed.png", + "image/vnd.sealedmedia.softseal.gif", + "image/vnd.sealedmedia.softseal.jpg", + "image/vnd.sld", + "image/vnd.svf", + "image/vnd.tencent.tap", + "image/vnd.valve.source.texture", + "image/vnd.wap.wbmp", + "image/vnd.xiff", + "image/vnd.zbrush.pcx", + "image/webp", + "image/wmf", + "image/x-3ds", + "image/x-adobe-dng", + "image/x-canon-cr2", + "image/x-canon-cr3", + "image/x-canon-crw", + "image/x-cmu-raster", + "image/x-cmx", + "image/x-emf", + "image/x-epson-erf", + "image/x-freehand", + "image/x-fuji-raf", + "image/x-icon", + "image/x-jg", + "image/x-jng", + "image/x-kodak-dcr", + "image/x-kodak-k25", + "image/x-kodak-kdc", + "image/x-macpaint", + "image/x-minolta-mrw", + "image/x-mrsid-image", + "image/x-nikon-nef", + "image/x-nikon-nrw", + "image/x-olympus-orf", + "image/x-panasonic-rw", + "image/x-panasonic-rw2", + "image/x-pentax-pef", + "image/x-portable-anymap", + "image/x-portable-bitmap", + "image/x-portable-graymap", + "image/x-portable-pixmap", + "image/x-quicktime", + "image/x-rgb", + "image/x-sigma-x3f", + "image/x-sony-arw", + "image/x-sony-sr2", + "image/x-sony-srf", + "image/x-tga", + "image/x-wmf", + "image/x-xbitmap", + "image/x-xpixmap", + "image/x-xwindowdump", + "message/bhttp", + "message/cpim", + "message/delivery-status", + "message/disposition-notification", + "message/example", + "message/external-body", + "message/feedback-report", + "message/global", + "message/global-delivery-status", + "message/global-disposition-notification", + "message/global-headers", + "message/http", + "message/imdn+xml", + "message/mls", + "message/news", + "message/ohttp-chunked-req", + "message/ohttp-chunked-res", + "message/ohttp-req", + "message/ohttp-res", + "message/partial", + "message/rfc822", + "message/s-http", + "message/sip", + "message/sipfrag", + "message/tracking-status", + "message/vnd.si.simp", + "message/vnd.wfa.wsc", + "model/3mf", + "model/e57", + "model/example", + "model/gltf+json", + "model/gltf-binary", + "model/iges", + "model/jt", + "model/mesh", + "model/mtl", + "model/obj", + "model/prc", + "model/step", + "model/step+xml", + "model/step+zip", + "model/step-xml+zip", + "model/stl", + "model/u3d", + "model/vnd.bary", + "model/vnd.cld", + "model/vnd.collada+xml", + "model/vnd.dwf", + "model/vnd.flatland.3dml", + "model/vnd.gdl", + "model/vnd.gs-gdl", + "model/vnd.gtw", + "model/vnd.moml+xml", + "model/vnd.mts", + "model/vnd.opengex", + "model/vnd.parasolid.transmit.binary", + "model/vnd.parasolid.transmit.text", + "model/vnd.pytha.pyox", + "model/vnd.rosette.annotated-data-model", + "model/vnd.sap.vds", + "model/vnd.usda", + "model/vnd.usdz+zip", + "model/vnd.valve.source.compiled-map", + "model/vnd.vtu", + "model/vrml", + "model/x3d+binary", + "model/x3d+fastinfoset", + "model/x3d+vrml", + "model/x3d+xml", + "model/x3d-vrml", + "multipart/alternative", + "multipart/appledouble", + "multipart/byteranges", + "multipart/digest", + "multipart/encrypted", + "multipart/example", + "multipart/form-data", + "multipart/header-set", + "multipart/mixed", + "multipart/multilingual", + "multipart/parallel", + "multipart/related", + "multipart/report", + "multipart/signed", + "multipart/vnd.bint.med-plus", + "multipart/voice-message", + "multipart/x-mixed-replace", + "text/1d-interleaved-parityfec", + "text/cache-manifest", + "text/calendar", + "text/coffeescript", + "text/cql", + "text/cql-expression", + "text/cql-identifier", + "text/css", + "text/csv", + "text/csv-schema", + "text/directory", + "text/dlm", + "text/dns", + "text/ecmascript", + "text/encaprtp", + "text/enriched", + "text/example", + "text/fhirpath", + "text/flexfec", + "text/fwdred", + "text/gemini", + "text/gff3", + "text/grammar-ref-list", + "text/h323", + "text/hl7v2", + "text/html", + "text/iuls", + "text/jade", + "text/javascript", + "text/jcr-cnd", + "text/less", + "text/markdown", + "text/mathml", + "text/mizar", + "text/n3", + "text/org", + "text/parameters", + "text/parityfec", + "text/plain", + "text/provenance-notation", + "text/prs.fallenstein.rst", + "text/prs.lines.tag", + "text/prs.prop.logic", + "text/prs.texi", + "text/qml", + "text/raptorfec", + "text/red", + "text/rfc822-headers", + "text/richtext", + "text/rtf", + "text/rtp-enc-aescm128", + "text/rtploopback", + "text/rtx", + "text/scriptlet", + "text/sgml", + "text/shaclc", + "text/shex", + "text/slim", + "text/spdx", + "text/strings", + "text/stylus", + "text/t140", + "text/tab-separated-values", + "text/troff", + "text/turtle", + "text/ulpfec", + "text/uri-list", + "text/vbscript", + "text/vcard", + "text/vnd.a", + "text/vnd.abc", + "text/vnd.ascii-art", + "text/vnd.bovnar", + "text/vnd.curl", + "text/vnd.curl.dcurl", + "text/vnd.curl.mcurl", + "text/vnd.curl.scurl", + "text/vnd.debian.copyright", + "text/vnd.dmclientscript", + "text/vnd.dvb.subtitle", + "text/vnd.esmertec.theme-descriptor", + "text/vnd.exchangeable", + "text/vnd.familysearch.gedcom", + "text/vnd.ficlab.flt", + "text/vnd.fly", + "text/vnd.fmi.flexstor", + "text/vnd.gist.mx", + "text/vnd.gml", + "text/vnd.graphviz", + "text/vnd.hans", + "text/vnd.hekaya", + "text/vnd.hgl", + "text/vnd.in3d.3dml", + "text/vnd.in3d.spot", + "text/vnd.iptc.newsml", + "text/vnd.iptc.nitf", + "text/vnd.latex-z", + "text/vnd.longform", + "text/vnd.motorola.reflex", + "text/vnd.ms-mediapackage", + "text/vnd.net2phone.commcenter.command", + "text/vnd.radisys.msml-basic-layout", + "text/vnd.senx.warpscript", + "text/vnd.si.uricatalogue", + "text/vnd.sosi", + "text/vnd.sun.j2me.app-descriptor", + "text/vnd.tps", + "text/vnd.trolltech.linguist", + "text/vnd.typst", + "text/vnd.vcf", + "text/vnd.vri", + "text/vnd.wap.si", + "text/vnd.wap.sl", + "text/vnd.wap.wml", + "text/vnd.wap.wmlscript", + "text/vnd.zoo.kcl", + "text/vtt", + "text/webviewhtml", + "text/wgsl", + "text/x-c", + "text/x-component", + "text/x-fortran", + "text/x-gherkin", + "text/x-handlebars-template", + "text/x-hdml", + "text/x-html-insertion", + "text/x-lua", + "text/x-markdown", + "text/x-ms-contact", + "text/x-ms-group", + "text/x-ms-iqy", + "text/x-ms-rqy", + "text/x-nfo", + "text/x-opml", + "text/x-pascal", + "text/x-processing", + "text/x-rust", + "text/x-sass", + "text/x-scss", + "text/x-setext", + "text/x-sfv", + "text/x-suse-ymp", + "text/x-toml", + "text/x-uuencode", + "text/x-vcard", + "text/x-yaml", + "text/xml", + "text/xml-external-parsed-entity", + "video/1d-interleaved-parityfec", + "video/3gpp", + "video/3gpp-tt", + "video/3gpp2", + "video/annodex", + "video/av1", + "video/bmpeg", + "video/bt656", + "video/celb", + "video/divx", + "video/dv", + "video/encaprtp", + "video/evc", + "video/example", + "video/ffv1", + "video/flexfec", + "video/h261", + "video/h263", + "video/h263-1998", + "video/h263-2000", + "video/h264", + "video/h264-rcdo", + "video/h264-svc", + "video/h265", + "video/h266", + "video/iso.segment", + "video/jpeg", + "video/jpeg2000", + "video/jpeg2000-scl", + "video/jpm", + "video/jxsv", + "video/lottie+json", + "video/matroska", + "video/matroska-3d", + "video/mj2", + "video/mp1s", + "video/mp2p", + "video/mp2t", + "video/mp4", + "video/mp4v-es", + "video/mpeg", + "video/mpeg4-generic", + "video/mpv", + "video/nv", + "video/ogg", + "video/parityfec", + "video/pointer", + "video/prs.avid", + "video/quicktime", + "video/raptorfec", + "video/raw", + "video/rtp-enc-aescm128", + "video/rtploopback", + "video/rtx", + "video/scip", + "video/smpte291", + "video/smpte292m", + "video/ulpfec", + "video/vc1", + "video/vc2", + "video/vnd.blockfact.factv", + "video/vnd.cctv", + "video/vnd.dece.hd", + "video/vnd.dece.mobile", + "video/vnd.dece.mp4", + "video/vnd.dece.pd", + "video/vnd.dece.sd", + "video/vnd.dece.video", + "video/vnd.directv.mpeg", + "video/vnd.directv.mpeg-tts", + "video/vnd.dlna.mpeg-tts", + "video/vnd.dvb.file", + "video/vnd.fvt", + "video/vnd.hns.video", + "video/vnd.iptvforum.1dparityfec-1010", + "video/vnd.iptvforum.1dparityfec-2005", + "video/vnd.iptvforum.2dparityfec-1010", + "video/vnd.iptvforum.2dparityfec-2005", + "video/vnd.iptvforum.ttsavc", + "video/vnd.iptvforum.ttsmpeg2", + "video/vnd.motorola.video", + "video/vnd.motorola.videop", + "video/vnd.mpegurl", + "video/vnd.ms-playready.media.pyv", + "video/vnd.nokia.interleaved-multimedia", + "video/vnd.nokia.mp4vr", + "video/vnd.nokia.videovoip", + "video/vnd.objectvideo", + "video/vnd.planar", + "video/vnd.radgamettools.bink", + "video/vnd.radgamettools.smacker", + "video/vnd.sealed.mpeg1", + "video/vnd.sealed.mpeg4", + "video/vnd.sealed.swf", + "video/vnd.sealedmedia.softseal.mov", + "video/vnd.uvvu.mp4", + "video/vnd.vivo", + "video/vnd.youtube.yt", + "video/vp8", + "video/vp9", + "video/webm", + "video/x-dv", + "video/x-f4v", + "video/x-fli", + "video/x-flv", + "video/x-ivf", + "video/x-la-asf", + "video/x-m4v", + "video/x-matroska", + "video/x-mng", + "video/x-ms-asf", + "video/x-ms-vob", + "video/x-ms-wm", + "video/x-ms-wmp", + "video/x-ms-wmv", + "video/x-ms-wmx", + "video/x-ms-wvx", + "video/x-msvideo", + "video/x-sgi-movie", + "video/x-smv", + "x-conference/x-cooltalk", + "x-world/x-vrml", +]; + +#[rustfmt::skip] +pub(super) static BY_NAME: [u16; 2744] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, + 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, + 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, + 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, + 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, + 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, + 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, + 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, + 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, + 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, + 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, + 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, + 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, + 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, + 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, + 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, + 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, + 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, + 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, + 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, + 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, + 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, + 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, + 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, + 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, + 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, + 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, + 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, + 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, + 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, + 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, + 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, + 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, + 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, + 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, + 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, + 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, + 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, + 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, + 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, + 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, + 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, + 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, + 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, + 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, + 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, + 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, + 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1263, + 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, + 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, + 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, + 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, + 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, + 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, + 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, + 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, + 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, + 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, + 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, + 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, 1470, 1471, + 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, + 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, 1502, 1503, + 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, + 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, + 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, + 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1567, + 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, + 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, + 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, + 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, + 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, + 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, + 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, + 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, + 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1711, + 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, + 1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, + 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1758, 1759, + 1760, 1761, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, + 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, + 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, + 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, + 1824, 1825, 1826, 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838, 1839, + 1840, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, + 1856, 1857, 1858, 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, + 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, + 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, + 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, + 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, + 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, + 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, + 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, + 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, + 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, + 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, + 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, + 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, + 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, + 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2095, + 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, + 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, + 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, + 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, + 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, + 2176, 2177, 2178, 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2189, 2190, 2191, + 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, + 2208, 2209, 2210, 2211, 2212, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, + 2224, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, + 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, + 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, + 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, + 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, + 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, + 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, + 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, + 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, + 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, + 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, + 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, + 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, + 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, + 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, + 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, + 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, + 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, + 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, + 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, + 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, + 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, + 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, + 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, + 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, + 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, + 2640, 2641, 2642, 2643, 2644, 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2655, + 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2671, + 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, + 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, 2700, 2701, 2702, 2703, + 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, + 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, + 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, +]; diff --git a/crates/types/src/media_type/mod.rs b/crates/types/src/media_type/mod.rs new file mode 100644 index 00000000..a1101627 --- /dev/null +++ b/crates/types/src/media_type/mod.rs @@ -0,0 +1,103 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod catalogue; + +use catalogue::{BY_NAME, MEDIA_TYPES}; +use std::borrow::Cow; + +const MAX_RESTRICTED_NAME: usize = 127; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] +pub struct MediaTypeId(u16); + +impl MediaTypeId { + pub const NONE: Self = Self(0); + pub const UNCATALOGUED: Self = Self((1 << Self::BITS) - 1); + pub const BITS: u32 = 12; + + #[inline(always)] + pub fn from_raw(raw: u16) -> Self { + Self(raw & Self::UNCATALOGUED.0) + } + + #[inline(always)] + pub fn raw(self) -> u16 { + self.0 + } + + #[inline(always)] + pub fn is_none(self) -> bool { + self == Self::NONE + } + + #[inline(always)] + pub fn is_uncatalogued(self) -> bool { + self == Self::UNCATALOGUED + } + + pub fn lookup(essence: &str) -> Self { + BY_NAME + .binary_search_by(|&idx| { + MEDIA_TYPES + .get(idx as usize) + .copied() + .unwrap_or_default() + .cmp(essence) + }) + .ok() + .and_then(|pos| BY_NAME.get(pos)) + .map_or(Self::UNCATALOGUED, |&idx| Self(idx + 1)) + } + + pub fn as_str(self) -> Option<&'static str> { + self.0 + .checked_sub(1) + .and_then(|idx| MEDIA_TYPES.get(idx as usize)) + .copied() + } + + pub fn catalogue() -> impl Iterator { + MEDIA_TYPES + .iter() + .zip(1u16..) + .map(|(name, id)| (MediaTypeId(id), *name)) + } +} + +pub fn is_valid_media_type(value: &str) -> bool { + value + .split_once('/') + .is_some_and(|(top, sub)| is_restricted_name(top) && is_restricted_name(sub)) +} + +pub fn media_type_essence(value: &str) -> Option> { + let essence = value + .split_once(';') + .map_or(value, |(essence, _)| essence) + .trim_matches(|c: char| c.is_ascii_whitespace()); + if !is_valid_media_type(essence) { + None + } else if essence.bytes().any(|b| b.is_ascii_uppercase()) { + Some(Cow::Owned(essence.to_ascii_lowercase())) + } else { + Some(Cow::Borrowed(essence)) + } +} + +fn is_restricted_name(name: &str) -> bool { + let bytes = name.as_bytes(); + matches!(bytes.first(), Some(b) if b.is_ascii_alphanumeric()) + && bytes.len() <= MAX_RESTRICTED_NAME + && bytes.iter().all(|b| { + b.is_ascii_alphanumeric() + || matches!( + b, + b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+' + ) + }) +} + diff --git a/crates/types/src/metadata/builder.rs b/crates/types/src/metadata/builder.rs new file mode 100644 index 00000000..f342fbfa --- /dev/null +++ b/crates/types/src/metadata/builder.rs @@ -0,0 +1,342 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + EntryKind, FORMAT_VERSION, MetadataKinds, + codec::{Reader, bytes_len, varint_len, write_bytes, write_varint}, + entries::{Entries, Entry}, + json::{EncodedJson, JsonView}, + registry::Namespace, + view::{MetadataView, RawKey}, + xml::{DavValueView, XmlError, XmlName, XmlValue}, +}; +use std::{borrow::Cow, collections::BTreeMap}; + +pub const STORAGE_TRAILER_CAPACITY: usize = 5; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum EntryKey<'x> { + Vendor(Cow<'x, str>), + Registered(u16), + Dav(XmlName<'x>), + Imap(Cow<'x, str>), +} + +#[derive(Debug, Clone, Default)] +pub struct MetadataBuilder<'x> { + entries: Entries<'x>, + written: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetadataEdit { + RemovalOnly, + Write, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedMetadata { + bytes: Vec, + kinds: MetadataKinds, + entries: usize, +} + +impl<'x> EntryKey<'x> { + fn kind(&self) -> EntryKind { + match self { + EntryKey::Vendor(_) => EntryKind::JmapVendor, + EntryKey::Registered(_) => EntryKind::JmapRegistered, + EntryKey::Dav(_) => EntryKind::Dav, + EntryKey::Imap(_) => EntryKind::Imap, + } + } + + fn from_raw(key: RawKey<'x>) -> Self { + match key { + RawKey::Vendor(name) => EntryKey::Vendor(Cow::Borrowed(name)), + RawKey::Registered(id) => EntryKey::Registered(id), + RawKey::Dav(namespace, name) => EntryKey::Dav(XmlName::borrowed(namespace, name)), + RawKey::Imap(name) => EntryKey::Imap(Cow::Borrowed(name)), + } + } + + fn from_namespace(namespace: &Namespace<'x>) -> Self { + match namespace { + Namespace::Registered(namespace) => EntryKey::Registered(namespace.id), + Namespace::Vendor(name) => EntryKey::Vendor(Cow::Borrowed(name)), + } + } + + #[inline(always)] + fn code(&self) -> u64 { + let value = match self { + EntryKey::Vendor(name) | EntryKey::Imap(name) => name.len() as u64, + EntryKey::Registered(id) => u64::from(*id), + EntryKey::Dav(name) => name.name.len() as u64, + }; + self.kind().code(value) + } + + #[inline(always)] + fn encoded_len(&self) -> usize { + varint_len(self.code()) + + match self { + EntryKey::Vendor(name) | EntryKey::Imap(name) => name.len(), + EntryKey::Registered(_) => 0, + EntryKey::Dav(name) => name.key_len(), + } + } + + #[inline(always)] + fn write(&self, out: &mut Vec) { + write_varint(out, self.code()); + match self { + EntryKey::Vendor(name) | EntryKey::Imap(name) => out.extend_from_slice(name.as_bytes()), + EntryKey::Registered(_) => {} + EntryKey::Dav(name) => name.write_key(out), + } + } + + pub(super) fn into_owned(self) -> EntryKey<'static> { + match self { + EntryKey::Vendor(name) => EntryKey::Vendor(Cow::Owned(name.into_owned())), + EntryKey::Registered(id) => EntryKey::Registered(id), + EntryKey::Dav(name) => EntryKey::Dav(name.into_owned()), + EntryKey::Imap(name) => EntryKey::Imap(Cow::Owned(name.into_owned())), + } + } +} + +impl<'x> MetadataBuilder<'x> { + pub fn new() -> Self { + MetadataBuilder::default() + } + + pub fn from_view(view: &MetadataView<'x>) -> Self { + let mut entries = Vec::with_capacity(view.len()); + entries.extend(view.raw_entries().map(|entry| Entry { + key: EntryKey::from_raw(entry.key), + value: Cow::Borrowed(entry.value.remaining()), + })); + MetadataBuilder { + entries: Entries::from_list(entries), + written: false, + } + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.len() == 0 + } + + pub fn edit(&self) -> MetadataEdit { + if self.written { + MetadataEdit::Write + } else { + MetadataEdit::RemovalOnly + } + } + + pub fn kinds(&self) -> MetadataKinds { + match &self.entries { + Entries::List(list) => kinds_of(list.iter().map(|entry| &entry.key)), + Entries::Tree(tree) => kinds_of(tree.keys()), + } + } + + fn value(&self, key: &EntryKey<'_>) -> Option<&[u8]> { + self.entries.get(key) + } + + fn upsert(&mut self, key: EntryKey<'x>, value: Cow<'x, [u8]>) { + self.written |= self.entries.upsert(key, value); + } + + fn remove_key(&mut self, key: &EntryKey<'_>) -> bool { + self.entries.remove(key) + } + + pub fn jmap(&self, namespace: &Namespace<'_>) -> Option> { + self.value(&EntryKey::from_namespace(namespace)) + .map(|value| JsonView::new(unsafe { Reader::trusted(value) })) + } + + pub fn set_jmap(&mut self, namespace: Namespace<'x>, value: EncodedJson) { + self.upsert( + EntryKey::from_namespace(&namespace), + Cow::Owned(value.into_bytes()), + ); + } + + pub fn remove_jmap(&mut self, namespace: &Namespace<'_>) -> bool { + self.remove_key(&EntryKey::from_namespace(namespace)) + } + + pub fn clear_jmap(&mut self) { + self.entries + .retain(|key| key.kind().kinds() != MetadataKinds::JMAP); + } + + pub fn dav(&self, name: &XmlName<'_>) -> Option> { + self.value(&EntryKey::Dav(XmlName::borrowed( + name.namespace(), + name.name(), + ))) + .map(|value| DavValueView::new(unsafe { Reader::trusted(value) })) + } + + pub fn set_dav(&mut self, name: XmlName<'x>, value: &XmlValue<'_>) -> Result { + name.validate_property()?; + let value = value.encode()?; + let len = value.len(); + self.upsert(EntryKey::Dav(name), Cow::Owned(value)); + Ok(len) + } + + pub fn remove_dav(&mut self, name: &XmlName<'_>) -> bool { + self.remove_key(&EntryKey::Dav(XmlName::borrowed( + name.namespace(), + name.name(), + ))) + } + + pub fn imap(&self, name: &str) -> Option<&[u8]> { + self.value(&EntryKey::Imap(Cow::Borrowed(name))) + .and_then(|value| unsafe { Reader::trusted(value) }.bytes()) + } + + pub fn set_imap(&mut self, name: Cow<'x, str>, value: &[u8]) { + let mut encoded = Vec::with_capacity(bytes_len(value.len())); + write_bytes(&mut encoded, value); + self.upsert(EntryKey::Imap(name), Cow::Owned(encoded)); + } + + pub fn remove_imap(&mut self, name: &str) -> bool { + self.remove_key(&EntryKey::Imap(Cow::Borrowed(name))) + } + + pub fn encode(&self) -> Option { + match &self.entries { + Entries::List(list) => encode_list(list), + Entries::Tree(tree) => encode_tree(tree), + } + } + + pub fn into_owned(self) -> MetadataBuilder<'static> { + MetadataBuilder { + entries: self.entries.into_owned(), + written: self.written, + } + } +} + +fn kinds_of<'a, 'x: 'a>(keys: impl Iterator>) -> MetadataKinds { + keys.fold(MetadataKinds::NONE, |kinds, key| { + kinds.union(key.kind().kinds()) + }) +} + +fn encode_list(list: &[Entry<'_>]) -> Option { + if list.is_empty() { + return None; + } + let len = 2 + + varint_len(list.len() as u64) + + list + .iter() + .map(|entry| entry.key.encoded_len() + entry.value.len()) + .sum::(); + let kinds = kinds_of(list.iter().map(|entry| &entry.key)); + let mut bytes = Vec::with_capacity(len + STORAGE_TRAILER_CAPACITY); + bytes.push(FORMAT_VERSION); + bytes.push(kinds.bits()); + write_varint(&mut bytes, list.len() as u64); + for entry in list { + entry.key.write(&mut bytes); + bytes.extend_from_slice(&entry.value); + } + debug_assert_eq!(bytes.len(), len, "metadata container length mismatch"); + Some(EncodedMetadata { + bytes, + kinds, + entries: list.len(), + }) +} + +#[cold] +fn encode_tree(tree: &BTreeMap, Cow<'_, [u8]>>) -> Option { + if tree.is_empty() { + return None; + } + let len = 2 + + varint_len(tree.len() as u64) + + tree + .iter() + .map(|(key, value)| key.encoded_len() + value.len()) + .sum::(); + let kinds = kinds_of(tree.keys()); + let mut bytes = Vec::with_capacity(len + STORAGE_TRAILER_CAPACITY); + bytes.push(FORMAT_VERSION); + bytes.push(kinds.bits()); + write_varint(&mut bytes, tree.len() as u64); + for (key, value) in tree { + key.write(&mut bytes); + bytes.extend_from_slice(value); + } + debug_assert_eq!(bytes.len(), len, "metadata container length mismatch"); + Some(EncodedMetadata { + bytes, + kinds, + entries: tree.len(), + }) +} + +impl EncodedMetadata { + pub fn from_view(view: &MetadataView<'_>) -> Option { + if view.is_empty() { + return None; + } + let contents = view.as_bytes(); + let mut bytes = Vec::with_capacity(contents.len() + STORAGE_TRAILER_CAPACITY); + bytes.extend_from_slice(contents); + Some(EncodedMetadata { + bytes, + kinds: view.kinds(), + entries: view.len(), + }) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.entries == 0 + } + + pub fn kinds(&self) -> MetadataKinds { + self.kinds + } + + pub fn entries(&self) -> usize { + self.entries + } + + pub fn view(&self) -> MetadataView<'_> { + unsafe { MetadataView::from_trusted(&self.bytes) }.unwrap_or_else(MetadataView::empty) + } +} diff --git a/crates/types/src/metadata/codec.rs b/crates/types/src/metadata/codec.rs new file mode 100644 index 00000000..f5872198 --- /dev/null +++ b/crates/types/src/metadata/codec.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) const MAX_NESTING: u32 = 128; + +#[inline] +pub(crate) const fn varint_len(value: u64) -> usize { + ((u64::BITS - (value | 1).leading_zeros()) as usize).div_ceil(7) +} + +#[inline] +pub(crate) fn write_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +#[inline] +pub(crate) const fn bytes_len(len: usize) -> usize { + varint_len(len as u64) + len +} + +#[inline] +pub(crate) fn write_bytes(out: &mut Vec, bytes: &[u8]) { + write_varint(out, bytes.len() as u64); + out.extend_from_slice(bytes); +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Reader<'x, const TRUSTED: bool> { + bytes: &'x [u8], +} + +pub(crate) type TrustedReader<'x> = Reader<'x, true>; +pub(crate) type CheckedReader<'x> = Reader<'x, false>; + +impl<'x> Reader<'x, true> { + #[inline] + pub(crate) const unsafe fn trusted(bytes: &'x [u8]) -> Self { + Reader { bytes } + } + + #[inline] + pub(crate) const fn empty() -> Self { + Reader { bytes: &[] } + } +} + +impl<'x> Reader<'x, false> { + #[inline] + pub(crate) const fn checked(bytes: &'x [u8]) -> Self { + Reader { bytes } + } +} + +impl<'x, const TRUSTED: bool> Reader<'x, TRUSTED> { + #[inline] + pub(crate) const fn remaining(&self) -> &'x [u8] { + self.bytes + } + + #[inline] + pub(crate) const fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + #[inline] + pub(crate) fn peek(&self) -> Option { + self.bytes.first().copied() + } + + #[inline] + pub(crate) fn u8(&mut self) -> Option { + let (first, rest) = self.bytes.split_first()?; + self.bytes = rest; + Some(*first) + } + + #[inline] + pub(crate) fn varint(&mut self) -> Option { + let (&first, rest) = self.bytes.split_first()?; + if first < 0x80 { + self.bytes = rest; + return Some(u64::from(first)); + } + self.varint_continued(first, rest) + } + + #[inline(never)] + fn varint_continued(&mut self, first: u8, mut rest: &'x [u8]) -> Option { + let mut value = u64::from(first & 0x7F); + let mut shift = 7u32; + loop { + let (&byte, next) = rest.split_first()?; + rest = next; + let chunk = u64::from(byte & 0x7F); + if shift == 63 && (chunk > 1 || byte & 0x80 != 0) { + return None; + } + value |= chunk << shift; + if byte & 0x80 == 0 { + self.bytes = rest; + return Some(value); + } + shift += 7; + } + } + + #[inline] + pub(crate) fn len(&mut self) -> Option { + self.varint().and_then(|len| usize::try_from(len).ok()) + } + + #[inline] + pub(crate) fn take(&mut self, len: usize) -> Option<&'x [u8]> { + let (taken, rest) = self.bytes.split_at_checked(len)?; + self.bytes = rest; + Some(taken) + } + + #[inline] + pub(crate) fn take_reader(&mut self, len: usize) -> Option { + self.take(len).map(|bytes| Reader { bytes }) + } + + #[inline] + pub(crate) fn bytes(&mut self) -> Option<&'x [u8]> { + let len = self.len()?; + self.take(len) + } + + #[inline] + pub(crate) fn skip(&mut self, len: usize) -> Option<()> { + self.take(len).map(|_| ()) + } + + #[inline] + pub(crate) fn consumed_since(&self, start: &Self) -> Option { + let len = start.bytes.len().checked_sub(self.bytes.len())?; + start.bytes.get(..len).map(|bytes| Reader { bytes }) + } + + #[inline] + pub(crate) fn text(&mut self, len: usize) -> Option<&'x str> { + let bytes = self.take(len)?; + if TRUSTED { + Some(unsafe { std::str::from_utf8_unchecked(bytes) }) + } else { + std::str::from_utf8(bytes).ok() + } + } + + #[inline] + pub(crate) fn str(&mut self) -> Option<&'x str> { + let len = self.len()?; + self.text(len) + } +} diff --git a/crates/types/src/metadata/entries.rs b/crates/types/src/metadata/entries.rs new file mode 100644 index 00000000..6a884fa2 --- /dev/null +++ b/crates/types/src/metadata/entries.rs @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::builder::EntryKey; +use std::{ + borrow::Cow, + collections::{BTreeMap, btree_map}, + mem, +}; + +const TREE_THRESHOLD: usize = 256; + +type Value<'x> = Cow<'x, [u8]>; + +#[derive(Debug, Clone)] +pub(super) struct Entry<'x> { + pub(super) key: EntryKey<'x>, + pub(super) value: Value<'x>, +} + +#[derive(Debug, Clone)] +pub(super) enum Entries<'x> { + List(Vec>), + Tree(BTreeMap, Value<'x>>), +} + +impl Default for Entries<'_> { + fn default() -> Self { + Entries::List(Vec::new()) + } +} + +impl<'x> Entries<'x> { + pub(super) fn from_list(mut list: Vec>) -> Self { + if !list.is_sorted_by(|a, b| a.key < b.key) { + list.sort_unstable_by(|a, b| a.key.cmp(&b.key)); + list.dedup_by(|a, b| a.key == b.key); + } + if list.len() > TREE_THRESHOLD { + Entries::Tree(into_tree(list)) + } else { + Entries::List(list) + } + } + + pub(super) fn len(&self) -> usize { + match self { + Entries::List(list) => list.len(), + Entries::Tree(tree) => tree.len(), + } + } + + pub(super) fn get(&self, key: &EntryKey<'_>) -> Option<&[u8]> { + match self { + Entries::List(list) => list + .binary_search_by(|entry| entry.key.cmp(key)) + .ok() + .and_then(|position| list.get(position)) + .map(|entry| entry.value.as_ref()), + Entries::Tree(tree) => tree.get(&key.clone().into_owned()).map(AsRef::as_ref), + } + } + + pub(super) fn upsert(&mut self, key: EntryKey<'x>, value: Value<'x>) -> bool { + match self { + Entries::List(list) => match list.binary_search_by(|entry| entry.key.cmp(&key)) { + Ok(position) => list + .get_mut(position) + .is_some_and(|entry| replace(&mut entry.value, value)), + Err(position) => { + list.insert(position, Entry { key, value }); + if list.len() > TREE_THRESHOLD { + *self = Entries::Tree(into_tree(mem::take(list))); + } + true + } + }, + Entries::Tree(tree) => match tree.entry(key) { + btree_map::Entry::Occupied(mut entry) => replace(entry.get_mut(), value), + btree_map::Entry::Vacant(entry) => { + entry.insert(value); + true + } + }, + } + } + + pub(super) fn remove(&mut self, key: &EntryKey<'_>) -> bool { + match self { + Entries::List(list) => match list.binary_search_by(|entry| entry.key.cmp(key)) { + Ok(position) => { + list.remove(position); + true + } + Err(_) => false, + }, + Entries::Tree(tree) => tree.remove(&key.clone().into_owned()).is_some(), + } + } + + pub(super) fn retain(&mut self, mut keep: impl FnMut(&EntryKey<'x>) -> bool) { + match self { + Entries::List(list) => list.retain(|entry| keep(&entry.key)), + Entries::Tree(tree) => tree.retain(|key, _| keep(key)), + } + } + + pub(super) fn into_owned(self) -> Entries<'static> { + match self { + Entries::List(list) => Entries::List( + list.into_iter() + .map(|entry| Entry { + key: entry.key.into_owned(), + value: Cow::Owned(entry.value.into_owned()), + }) + .collect(), + ), + Entries::Tree(tree) => Entries::Tree( + tree.into_iter() + .map(|(key, value)| (key.into_owned(), Cow::Owned(value.into_owned()))) + .collect(), + ), + } + } +} + +#[cold] +#[inline(never)] +fn into_tree<'x>(list: Vec>) -> BTreeMap, Value<'x>> { + list.into_iter() + .map(|entry| (entry.key, entry.value)) + .collect() +} + +fn replace<'x>(current: &mut Value<'x>, value: Value<'x>) -> bool { + if *current != value { + *current = value; + true + } else { + false + } +} diff --git a/crates/types/src/metadata/json.rs b/crates/types/src/metadata/json.rs new file mode 100644 index 00000000..15ddd7d7 --- /dev/null +++ b/crates/types/src/metadata/json.rs @@ -0,0 +1,561 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::codec::{ + CheckedReader, MAX_NESTING, Reader, TrustedReader, bytes_len, varint_len, write_bytes, + write_varint, +}; +use jmap_tools::{Element, Key, Map, Property, Value}; +use std::{borrow::Cow, slice}; + +const NULL: u8 = 0x00; +const FALSE: u8 = 0x01; +const TRUE: u8 = 0x02; +const UINT: u8 = 0x03; +const NEG_INT: u8 = 0x04; +const FLOAT: u8 = 0x05; +const STR: u8 = 0x06; +const ARRAY: u8 = 0x07; +const OBJECT: u8 = 0x08; +const SMALL_UINT: u8 = 0x40; +const SMALL_UINT_MAX: u64 = 0x3F; +const SHORT_STR: u8 = 0x80; +const SHORT_STR_MAX: usize = 0x7F; +const FLOAT_LEN: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum JsonKind { + Null, + Bool, + Number, + String, + Array, + Object, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum JsonError { + NotAnObject, + ControlCharacter, + NestingTooDeep, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedJson { + bytes: Vec, + depth: u32, +} + +#[derive(Debug, Clone, Copy)] +pub struct JsonView<'x> { + reader: TrustedReader<'x>, +} + +#[derive(Debug, Clone)] +pub struct JsonMembers<'x> { + reader: TrustedReader<'x>, + remaining: usize, +} + +#[derive(Debug, Clone)] +pub struct JsonItems<'x> { + reader: TrustedReader<'x>, + remaining: usize, +} + +impl EncodedJson { + pub fn encode_namespace>( + value: &Value<'_, P, E>, + ) -> Result { + if matches!(value, Value::Object(_)) { + Self::encode(value) + } else { + Err(JsonError::NotAnObject) + } + } + + pub fn encode>( + value: &Value<'_, P, E>, + ) -> Result { + let mut sizes = Vec::new(); + let (len, depth) = measure(value, 0, &mut sizes)?; + let mut bytes = Vec::with_capacity(len); + write(value, &mut sizes.iter(), &mut bytes); + debug_assert_eq!(bytes.len(), len, "measured and written JSON lengths differ"); + Ok(EncodedJson { + bytes, + depth: depth.max(1), + }) + } + + pub fn depth(&self) -> u32 { + self.depth + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } + + pub fn view(&self) -> JsonView<'_> { + JsonView { + reader: unsafe { Reader::trusted(&self.bytes) }, + } + } +} + +fn is_control(byte: u8) -> bool { + (byte < 0x20 && !matches!(byte, b'\t' | b'\n' | b'\r')) || byte == 0x7F +} + +fn check_text(text: &str) -> Result<(), JsonError> { + if text.bytes().any(is_control) { + Err(JsonError::ControlCharacter) + } else { + Ok(()) + } +} + +fn str_encoded_len(len: usize) -> usize { + if len <= SHORT_STR_MAX { + 1 + len + } else { + 1 + bytes_len(len) + } +} + +#[derive(Clone, Copy)] +enum NumberRepr { + Small(u8), + Unsigned(u64), + Negative(u64), + Float(f64), +} + +impl NumberRepr { + fn new(unsigned: Option, signed: Option, float: Option) -> Self { + match (unsigned, signed) { + (Some(value), _) if value <= SMALL_UINT_MAX => NumberRepr::Small(value as u8), + (Some(value), _) => NumberRepr::Unsigned(value), + (None, Some(value)) => NumberRepr::Negative(!value as u64), + (None, None) => NumberRepr::Float(float.unwrap_or_default()), + } + } + + fn encoded_len(self) -> usize { + match self { + NumberRepr::Small(_) => 1, + NumberRepr::Unsigned(value) | NumberRepr::Negative(value) => 1 + varint_len(value), + NumberRepr::Float(_) => 1 + FLOAT_LEN, + } + } + + fn write(self, out: &mut Vec) { + match self { + NumberRepr::Small(value) => out.push(SMALL_UINT | value), + NumberRepr::Unsigned(value) => { + out.push(UINT); + write_varint(out, value); + } + NumberRepr::Negative(value) => { + out.push(NEG_INT); + write_varint(out, value); + } + NumberRepr::Float(value) => { + out.push(FLOAT); + out.extend_from_slice(&value.to_le_bytes()); + } + } + } +} + +fn measure>( + value: &Value<'_, P, E>, + nesting: u32, + sizes: &mut Vec, +) -> Result<(usize, u32), JsonError> { + match value { + Value::Null | Value::Bool(_) => Ok((1, 0)), + Value::Number(number) => Ok(( + NumberRepr::new(number.as_u64(), number.as_i64(), number.as_f64()).encoded_len(), + 0, + )), + Value::Str(text) => { + check_text(text)?; + Ok((str_encoded_len(text.len()), 0)) + } + Value::Element(element) => { + let text = element.to_cow(); + check_text(&text)?; + Ok((str_encoded_len(text.len()), 0)) + } + Value::Array(items) => { + if nesting >= MAX_NESTING { + return Err(JsonError::NestingTooDeep); + } + let slot = sizes.len(); + sizes.push(0); + let mut body = varint_len(items.len() as u64); + let mut depth = 0; + for item in items { + let (len, item_depth) = measure(item, nesting + 1, sizes)?; + body += len; + depth = depth.max(item_depth); + } + if let Some(size) = sizes.get_mut(slot) { + *size = body; + } + Ok((1 + bytes_len(body), depth)) + } + Value::Object(object) => { + if nesting >= MAX_NESTING { + return Err(JsonError::NestingTooDeep); + } + let slot = sizes.len(); + sizes.push(0); + let mut body = varint_len(object.len() as u64); + let mut depth = 0; + for (key, item) in object.iter() { + let key = key.to_string(); + check_text(&key)?; + let (len, item_depth) = measure(item, nesting + 1, sizes)?; + body += bytes_len(key.len()) + len; + depth = depth.max(item_depth); + } + if let Some(size) = sizes.get_mut(slot) { + *size = body; + } + Ok((1 + bytes_len(body), depth + 1)) + } + } +} + +fn write_str(out: &mut Vec, text: &str) { + if text.len() <= SHORT_STR_MAX { + out.push(SHORT_STR | text.len() as u8); + out.extend_from_slice(text.as_bytes()); + } else { + out.push(STR); + write_bytes(out, text.as_bytes()); + } +} + +fn write>( + value: &Value<'_, P, E>, + sizes: &mut slice::Iter<'_, usize>, + out: &mut Vec, +) { + match value { + Value::Null => out.push(NULL), + Value::Bool(false) => out.push(FALSE), + Value::Bool(true) => out.push(TRUE), + Value::Number(number) => { + NumberRepr::new(number.as_u64(), number.as_i64(), number.as_f64()).write(out) + } + Value::Str(text) => write_str(out, text), + Value::Element(element) => write_str(out, &element.to_cow()), + Value::Array(items) => { + out.push(ARRAY); + write_varint(out, sizes.next().copied().unwrap_or_default() as u64); + write_varint(out, items.len() as u64); + for item in items { + write(item, sizes, out); + } + } + Value::Object(object) => { + out.push(OBJECT); + write_varint(out, sizes.next().copied().unwrap_or_default() as u64); + write_varint(out, object.len() as u64); + for (key, item) in object.iter() { + write_bytes(out, key.to_string().as_bytes()); + write(item, sizes, out); + } + } + } +} + +pub(crate) fn skip_value(reader: &mut Reader<'_, TRUSTED>) -> Option<()> { + match reader.u8()? { + NULL | FALSE | TRUE => Some(()), + UINT | NEG_INT => reader.varint().map(drop), + FLOAT => reader.skip(FLOAT_LEN), + STR | ARRAY | OBJECT => { + let len = reader.len()?; + reader.skip(len) + } + tag if tag >= SHORT_STR => reader.skip(usize::from(tag & !SHORT_STR)), + tag if tag >= SMALL_UINT => Some(()), + _ => None, + } +} + +pub(crate) fn value_span<'x, const TRUSTED: bool>( + reader: &mut Reader<'x, TRUSTED>, +) -> Option> { + let start = *reader; + skip_value(reader)?; + reader.consumed_since(&start) +} + +pub(crate) fn validate_value(reader: &mut CheckedReader<'_>, nesting: u32) -> Option<()> { + match reader.u8()? { + NULL | FALSE | TRUE => Some(()), + UINT => reader.varint().map(drop), + NEG_INT => reader + .varint() + .filter(|value| i64::try_from(*value).is_ok()) + .map(drop), + FLOAT => reader.skip(FLOAT_LEN), + STR => reader.str().map(drop), + tag @ (ARRAY | OBJECT) => { + if nesting >= MAX_NESTING { + return None; + } + let len = reader.len()?; + let mut body = reader.take_reader(len)?; + let count = body.len()?; + for _ in 0..count { + if tag == OBJECT { + body.str()?; + } + validate_value(&mut body, nesting + 1)?; + } + body.is_empty().then_some(()) + } + tag if tag >= SHORT_STR => reader.text(usize::from(tag & !SHORT_STR)).map(drop), + tag if tag >= SMALL_UINT => Some(()), + _ => None, + } +} + +pub(crate) fn validate_namespace_value(reader: &mut CheckedReader<'_>) -> Option<()> { + (reader.peek()? == OBJECT).then_some(())?; + validate_value(reader, 0) +} + +impl<'x> JsonView<'x> { + pub(crate) const fn new(reader: TrustedReader<'x>) -> Self { + JsonView { reader } + } + + pub fn as_bytes(&self) -> &'x [u8] { + self.reader.remaining() + } + + pub fn encoded_len(&self) -> usize { + self.reader.remaining().len() + } + + pub fn kind(&self) -> JsonKind { + match self.reader.peek() { + Some(FALSE | TRUE) => JsonKind::Bool, + Some(UINT | NEG_INT | FLOAT) => JsonKind::Number, + Some(STR) => JsonKind::String, + Some(ARRAY) => JsonKind::Array, + Some(OBJECT) => JsonKind::Object, + Some(tag) if tag >= SHORT_STR => JsonKind::String, + Some(tag) if tag >= SMALL_UINT => JsonKind::Number, + _ => JsonKind::Null, + } + } + + pub fn is_null(&self) -> bool { + self.kind() == JsonKind::Null + } + + pub fn as_bool(&self) -> Option { + match self.reader.peek()? { + FALSE => Some(false), + TRUE => Some(true), + _ => None, + } + } + + pub fn as_u64(&self) -> Option { + let mut reader = self.reader; + match reader.u8()? { + UINT => reader.varint(), + tag if (SMALL_UINT..SHORT_STR).contains(&tag) => Some(u64::from(tag & !SMALL_UINT)), + _ => None, + } + } + + pub fn as_i64(&self) -> Option { + let mut reader = self.reader; + match reader.peek()? { + NEG_INT => { + reader.u8()?; + reader.varint().map(|value| !(value as i64)) + } + _ => self.as_u64().and_then(|value| i64::try_from(value).ok()), + } + } + + pub fn as_f64(&self) -> Option { + let mut reader = self.reader; + match reader.peek()? { + FLOAT => { + reader.u8()?; + reader + .take(FLOAT_LEN)? + .try_into() + .ok() + .map(f64::from_le_bytes) + } + NEG_INT => self.as_i64().map(|value| value as f64), + _ => self.as_u64().map(|value| value as f64), + } + } + + pub fn as_str(&self) -> Option<&'x str> { + let mut reader = self.reader; + match reader.u8()? { + STR => reader.str(), + tag if tag >= SHORT_STR => reader.text(usize::from(tag & !SHORT_STR)), + _ => None, + } + } + + fn container(&self, tag: u8) -> Option<(TrustedReader<'x>, usize)> { + let mut reader = self.reader; + (reader.u8()? == tag).then_some(())?; + let len = reader.len()?; + let mut body = reader.take_reader(len)?; + let count = body.len()?; + Some((body, count)) + } + + pub fn len(&self) -> usize { + self.container(OBJECT) + .or_else(|| self.container(ARRAY)) + .map_or(0, |(_, count)| count) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn is_object(&self) -> bool { + self.reader.peek() == Some(OBJECT) + } + + pub fn is_empty_object(&self) -> bool { + self.container(OBJECT).is_some_and(|(_, count)| count == 0) + } + + pub fn members(&self) -> JsonMembers<'x> { + let (reader, remaining) = self.container(OBJECT).unwrap_or((Reader::empty(), 0)); + JsonMembers { reader, remaining } + } + + pub fn items(&self) -> JsonItems<'x> { + let (reader, remaining) = self.container(ARRAY).unwrap_or((Reader::empty(), 0)); + JsonItems { reader, remaining } + } + + pub fn get(&self, key: &str) -> Option> { + let (mut body, count) = self.container(OBJECT)?; + let key = key.as_bytes(); + for _ in 0..count { + if body.bytes()? == key { + return value_span(&mut body).map(JsonView::new); + } + skip_value(&mut body)?; + } + None + } + + pub fn to_value>(&self) -> Option> { + let mut reader = self.reader; + decode(&mut reader, 0) + } +} + +fn decode<'x, P: Property, E: Element>( + reader: &mut TrustedReader<'x>, + nesting: u32, +) -> Option> { + Some(match reader.u8()? { + NULL => Value::Null, + FALSE => Value::Bool(false), + TRUE => Value::Bool(true), + UINT => Value::Number(reader.varint()?.into()), + NEG_INT => Value::Number((!(reader.varint()? as i64)).into()), + FLOAT => Value::Number(f64::from_le_bytes(reader.take(FLOAT_LEN)?.try_into().ok()?).into()), + STR => Value::Str(Cow::Borrowed(reader.str()?)), + ARRAY => { + if nesting >= MAX_NESTING { + return None; + } + let len = reader.len()?; + let mut body = reader.take_reader(len)?; + let count = body.len()?; + let mut items = Vec::with_capacity(count.min(body.remaining().len())); + for _ in 0..count { + items.push(decode(&mut body, nesting + 1)?); + } + Value::Array(items) + } + OBJECT => { + if nesting >= MAX_NESTING { + return None; + } + let len = reader.len()?; + let mut body = reader.take_reader(len)?; + let count = body.len()?; + let mut object = Map::with_capacity(count.min(body.remaining().len())); + for _ in 0..count { + let key = body.str()?; + object.insert_unchecked(Key::Borrowed(key), decode(&mut body, nesting + 1)?); + } + Value::Object(object) + } + tag if tag >= SHORT_STR => { + Value::Str(Cow::Borrowed(reader.text(usize::from(tag & !SHORT_STR))?)) + } + tag if tag >= SMALL_UINT => Value::Number(u64::from(tag & !SMALL_UINT).into()), + _ => return None, + }) +} + +impl<'x> Iterator for JsonMembers<'x> { + type Item = (&'x str, JsonView<'x>); + + fn next(&mut self) -> Option { + self.remaining = self.remaining.checked_sub(1)?; + let key = self.reader.str()?; + let value = value_span(&mut self.reader)?; + Some((key, JsonView::new(value))) + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.remaining)) + } +} + +impl<'x> Iterator for JsonItems<'x> { + type Item = JsonView<'x>; + + fn next(&mut self) -> Option { + self.remaining = self.remaining.checked_sub(1)?; + value_span(&mut self.reader).map(JsonView::new) + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.remaining)) + } +} diff --git a/crates/types/src/metadata/limits.rs b/crates/types/src/metadata/limits.rs new file mode 100644 index 00000000..6c44e88b --- /dev/null +++ b/crates/types/src/metadata/limits.rs @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::builder::EncodedMetadata; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetadataScope { + Shared, + Private, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MetadataLimits { + pub max_depth: Option, + pub max_entry_size: usize, + pub max_size: usize, + pub max_private_size: usize, + pub max_entries: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LimitViolation { + Depth { depth: u32, max: u32 }, + EntrySize { size: usize, max: usize }, + ContainerSize { size: usize, max: usize }, + Entries { count: usize, max: usize }, +} + +#[derive(Debug, Clone, Copy)] +pub struct EntryBound { + bound: usize, + max_entries: usize, +} + +impl Default for MetadataLimits { + fn default() -> Self { + MetadataLimits { + max_depth: Some(8), + max_entry_size: 8 * 1024, + max_size: 64 * 1024, + max_private_size: 16 * 1024, + max_entries: 128, + } + } +} + +impl MetadataLimits { + pub fn max_container_size(&self, scope: MetadataScope) -> usize { + match scope { + MetadataScope::Shared => self.max_size, + MetadataScope::Private => self.max_private_size, + } + } + + pub fn check_depth(&self, depth: u32) -> Result<(), LimitViolation> { + match self.max_depth { + Some(max) if depth > max => Err(LimitViolation::Depth { depth, max }), + _ => Ok(()), + } + } + + pub fn check_entry_size(&self, size: usize) -> Result<(), LimitViolation> { + if size > self.max_entry_size { + Err(LimitViolation::EntrySize { + size, + max: self.max_entry_size, + }) + } else { + Ok(()) + } + } + + pub fn entry_bound(&self, previous_entries: usize) -> EntryBound { + EntryBound { + bound: self.max_entries.max(previous_entries), + max_entries: self.max_entries, + } + } + + pub fn check_edit( + &self, + scope: MetadataScope, + previous_len: usize, + previous_entries: usize, + next: &EncodedMetadata, + ) -> Result<(), LimitViolation> { + let count = next.entries(); + if count > self.max_entries && count > previous_entries { + return Err(LimitViolation::Entries { + count, + max: self.max_entries, + }); + } + let size = next.len(); + let max = self.max_container_size(scope); + if size > max && size > previous_len { + Err(LimitViolation::ContainerSize { size, max }) + } else { + Ok(()) + } + } +} + +impl EntryBound { + pub fn check(self, entries: usize, pending_removals: usize) -> Result<(), LimitViolation> { + let count = entries.saturating_sub(pending_removals); + if count > self.bound { + Err(LimitViolation::Entries { + count, + max: self.max_entries, + }) + } else { + Ok(()) + } + } +} diff --git a/crates/types/src/metadata/mod.rs b/crates/types/src/metadata/mod.rs new file mode 100644 index 00000000..39a26696 --- /dev/null +++ b/crates/types/src/metadata/mod.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod builder; +mod codec; +mod entries; +mod json; +mod limits; +mod registry; +mod view; +mod xml; + + +pub use builder::{EncodedMetadata, MetadataBuilder, MetadataEdit, STORAGE_TRAILER_CAPACITY}; +pub use json::{EncodedJson, JsonError, JsonItems, JsonKind, JsonMembers, JsonView}; +pub use limits::{EntryBound, LimitViolation, MetadataLimits, MetadataScope}; +pub use registry::{ + Namespace, NamespaceError, NamespaceScope, NamespaceUsage, REGISTERED_NAMESPACES, + RegisteredNamespace, +}; +pub use view::{DavEntries, ImapEntries, JmapEntries, MetadataView}; +pub use xml::{ + DavValueView, EncodedDavValue, XmlAttribute, XmlElement, XmlError, XmlName, XmlNode, XmlValue, +}; + +pub const FORMAT_VERSION: u8 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[repr(transparent)] +pub struct MetadataKinds(u8); + +impl MetadataKinds { + pub const NONE: MetadataKinds = MetadataKinds(0); + pub const JMAP: MetadataKinds = MetadataKinds(1); + pub const DAV: MetadataKinds = MetadataKinds(1 << 1); + pub const IMAP: MetadataKinds = MetadataKinds(1 << 2); + const ALL: u8 = Self::JMAP.0 | Self::DAV.0 | Self::IMAP.0; + + pub const fn from_bits(bits: u8) -> Option { + if bits & !Self::ALL == 0 { + Some(MetadataKinds(bits)) + } else { + None + } + } + + pub const fn bits(self) -> u8 { + self.0 + } + + pub const fn contains(self, other: MetadataKinds) -> bool { + self.0 & other.0 == other.0 && other.0 != 0 + } + + pub const fn intersects(self, other: MetadataKinds) -> bool { + self.0 & other.0 != 0 + } + + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + pub const fn union(self, other: MetadataKinds) -> Self { + MetadataKinds(self.0 | other.0) + } + + pub fn insert(&mut self, other: MetadataKinds) { + self.0 |= other.0; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(u8)] +pub(crate) enum EntryKind { + JmapVendor = 0, + JmapRegistered = 1, + Dav = 2, + Imap = 3, +} + +impl EntryKind { + const MASK: u64 = 0b111; + const SHIFT: u32 = 3; + + pub(crate) const fn split_code(code: u64) -> Option<(Self, u64)> { + let kind = match code & Self::MASK { + 0 => EntryKind::JmapVendor, + 1 => EntryKind::JmapRegistered, + 2 => EntryKind::Dav, + 3 => EntryKind::Imap, + _ => return None, + }; + Some((kind, code >> Self::SHIFT)) + } + + pub(crate) const fn code(self, value: u64) -> u64 { + (value << Self::SHIFT) | self as u64 + } + + pub(crate) const fn kinds(self) -> MetadataKinds { + match self { + EntryKind::JmapVendor | EntryKind::JmapRegistered => MetadataKinds::JMAP, + EntryKind::Dav => MetadataKinds::DAV, + EntryKind::Imap => MetadataKinds::IMAP, + } + } +} diff --git a/crates/types/src/metadata/registry.rs b/crates/types/src/metadata/registry.rs new file mode 100644 index 00000000..879ae0bc --- /dev/null +++ b/crates/types/src/metadata/registry.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::MetadataScope; +use crate::type_state::DataType; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NamespaceScope { + Metadata, + PrivateMetadata, + Both, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NamespaceUsage { + Common, + Reserved, + Obsolete, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct RegisteredNamespace { + pub id: u16, + pub name: &'static str, + pub scope: NamespaceScope, + pub data_types: &'static [DataType], + pub usage: NamespaceUsage, +} + +macro_rules! registered_namespaces { + ($($id:literal => $name:literal { + scope: $scope:ident, + data_types: [$($data_type:ident),* $(,)?], + usage: $usage:ident $(,)? + }),* $(,)?) => { + pub const REGISTERED_NAMESPACES: &[RegisteredNamespace] = &[$( + RegisteredNamespace { + id: $id, + name: $name, + scope: NamespaceScope::$scope, + data_types: &[$(DataType::$data_type),*], + usage: NamespaceUsage::$usage, + }, + )*]; + + impl RegisteredNamespace { + pub fn by_name(name: &str) -> Option<&'static RegisteredNamespace> { + let id: Option<&'static u16> = hashify::map!(name.as_bytes(), u16, $($name => $id,)*); + id.and_then(|id| RegisteredNamespace::by_id(*id)) + } + } + }; +} + +registered_namespaces!(); + +impl RegisteredNamespace { + pub fn by_id(id: u16) -> Option<&'static RegisteredNamespace> { + REGISTERED_NAMESPACES + .binary_search_by_key(&id, |namespace| namespace.id) + .ok() + .and_then(|position| REGISTERED_NAMESPACES.get(position)) + } + + pub fn is_advertised(&self) -> bool { + self.usage == NamespaceUsage::Common + } + + pub fn is_writable(&self) -> bool { + self.usage == NamespaceUsage::Common + } + + pub fn supports(&self, data_type: DataType, scope: MetadataScope) -> bool { + self.data_types.contains(&data_type) + && matches!( + (self.scope, scope), + (NamespaceScope::Both, _) + | (NamespaceScope::Metadata, MetadataScope::Shared) + | (NamespaceScope::PrivateMetadata, MetadataScope::Private) + ) + } +} + +macro_rules! known_xml_namespaces { + ($($id:literal => $uri:literal),* $(,)?) => { + const KNOWN_XML_NAMESPACES: &[&str] = &[$($uri),*]; + + pub(crate) fn known_xml_namespace_id(uri: &str) -> Option { + hashify::map!(uri.as_bytes(), u8, $($uri => $id,)*).copied() + } + }; +} + +known_xml_namespaces!( + 1 => "DAV:", + 2 => "urn:ietf:params:xml:ns:caldav", + 3 => "urn:ietf:params:xml:ns:carddav", + 4 => "http://calendarserver.org/ns/", + 5 => "http://calendarserver.org/ns", + 6 => "http://apple.com/ns/ical/", + 7 => "http://www.apple.com/webdav_fs/props/", + 8 => "http://me.com/_namespace/", + 9 => "urn:schemas-microsoft-com:", + 10 => "http://owncloud.org/ns", + 11 => "http://nextcloud.org/ns", + 12 => "http://sabredav.org/ns", + 13 => "http://www.w3.org/XML/1998/namespace", + 14 => "http://www.w3.org/1999/xhtml", +); + +pub(crate) const XML_NAMESPACE: &str = "http://www.w3.org/XML/1998/namespace"; +pub(crate) const XMLNS_NAMESPACE: &str = "http://www.w3.org/2000/xmlns/"; + +pub(crate) fn known_xml_namespace(id: u8) -> Option<&'static str> { + KNOWN_XML_NAMESPACES + .get(usize::from(id).checked_sub(1)?) + .copied() +} + +#[cfg(all(test, stalwart_tests))] +pub(crate) fn known_xml_namespace_count() -> usize { + KNOWN_XML_NAMESPACES.len() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Namespace<'x> { + Registered(&'static RegisteredNamespace), + Vendor(&'x str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NamespaceError { + Invalid, + Unregistered, +} + +const MAX_REGISTERED_NAME: usize = 64; +const MAX_DOMAIN_NAME: usize = 253; +const MAX_DOMAIN_LABEL: usize = 63; + +impl<'x> Namespace<'x> { + pub fn parse(name: &'x str) -> Result { + if name.as_bytes().contains(&b'.') { + if is_vendor_name(name) { + Ok(Namespace::Vendor(name)) + } else { + Err(NamespaceError::Invalid) + } + } else if is_registered_name(name) { + RegisteredNamespace::by_name(name) + .map(Namespace::Registered) + .ok_or(NamespaceError::Unregistered) + } else { + Err(NamespaceError::Invalid) + } + } + + pub fn name(&self) -> &'x str { + match self { + Namespace::Registered(namespace) => namespace.name, + Namespace::Vendor(name) => name, + } + } +} + +fn is_registered_name(name: &str) -> bool { + (1..=MAX_REGISTERED_NAME).contains(&name.len()) + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') +} + +fn is_vendor_name(name: &str) -> bool { + name.len() <= MAX_DOMAIN_NAME + && name.split('.').all(|label| { + let bytes = label.as_bytes(); + (1..=MAX_DOMAIN_LABEL).contains(&bytes.len()) + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') + && bytes.first() != Some(&b'-') + && bytes.last() != Some(&b'-') + }) +} diff --git a/crates/types/src/metadata/view.rs b/crates/types/src/metadata/view.rs new file mode 100644 index 00000000..b770161f --- /dev/null +++ b/crates/types/src/metadata/view.rs @@ -0,0 +1,333 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + EntryKind, FORMAT_VERSION, MetadataKinds, + codec::{Reader, TrustedReader}, + json::{JsonView, validate_namespace_value, value_span}, + registry::{Namespace, RegisteredNamespace}, + xml::{ + DavValueView, XmlName, check_property_name, dav_value_span, read_ns_ref, validate_dav_value, + }, +}; + +const EMPTY_CONTAINER: [u8; 3] = [FORMAT_VERSION, 0, 0]; + +#[derive(Debug, Clone, Copy)] +pub struct MetadataView<'x> { + bytes: &'x [u8], + kinds: MetadataKinds, + count: usize, + entries: TrustedReader<'x>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RawKey<'x> { + Vendor(&'x str), + Registered(u16), + Dav(Option<&'x str>, &'x str), + Imap(&'x str), +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct RawEntry<'x> { + pub key: RawKey<'x>, + pub value: TrustedReader<'x>, +} + +#[derive(Debug, Clone)] +pub(crate) struct RawEntries<'x> { + reader: TrustedReader<'x>, + remaining: usize, +} + +#[derive(Debug, Clone)] +pub struct JmapEntries<'x> { + entries: RawEntries<'x>, +} + +#[derive(Debug, Clone)] +pub struct DavEntries<'x> { + entries: RawEntries<'x>, +} + +#[derive(Debug, Clone)] +pub struct ImapEntries<'x> { + entries: RawEntries<'x>, +} + +fn header( + bytes: &[u8], + reader: &mut Reader<'_, TRUSTED>, +) -> Option<(MetadataKinds, usize)> { + (reader.u8()? == FORMAT_VERSION).then_some(())?; + let kinds = MetadataKinds::from_bits(reader.u8()?)?; + let count = reader.len()?; + (count <= bytes.len()).then_some((kinds, count)) +} + +fn validate(bytes: &[u8]) -> Option<()> { + let mut reader = Reader::checked(bytes); + let (kinds, count) = header(bytes, &mut reader)?; + let mut found = MetadataKinds::NONE; + let mut previous = EntryKind::JmapVendor; + + for _ in 0..count { + let (kind, value) = EntryKind::split_code(reader.varint()?)?; + if kind < previous { + return None; + } + previous = kind; + found.insert(kind.kinds()); + + match kind { + EntryKind::JmapVendor => { + reader.text(usize::try_from(value).ok()?)?; + validate_namespace_value(&mut reader)?; + } + EntryKind::JmapRegistered => { + u16::try_from(value).ok()?; + validate_namespace_value(&mut reader)?; + } + EntryKind::Dav => { + let namespace = read_ns_ref(&mut reader)?; + let name = reader.text(usize::try_from(value).ok()?)?; + check_property_name(namespace, name).ok()?; + validate_dav_value(&mut reader)?; + } + EntryKind::Imap => { + reader.text(usize::try_from(value).ok()?)?; + reader.bytes()?; + } + } + } + + (reader.is_empty() && found == kinds).then_some(()) +} + +fn parse_entry<'x>(reader: &mut TrustedReader<'x>) -> Option> { + let (kind, value) = EntryKind::split_code(reader.varint()?)?; + let key = match kind { + EntryKind::JmapVendor => RawKey::Vendor(reader.text(usize::try_from(value).ok()?)?), + EntryKind::JmapRegistered => RawKey::Registered(u16::try_from(value).ok()?), + EntryKind::Dav => { + let namespace = read_ns_ref(reader)?; + RawKey::Dav(namespace, reader.text(usize::try_from(value).ok()?)?) + } + EntryKind::Imap => RawKey::Imap(reader.text(usize::try_from(value).ok()?)?), + }; + let value = match key { + RawKey::Vendor(_) | RawKey::Registered(_) => value_span(reader)?, + RawKey::Dav(..) => dav_value_span(reader)?, + RawKey::Imap(_) => { + let start = *reader; + reader.bytes()?; + reader.consumed_since(&start)? + } + }; + Some(RawEntry { key, value }) +} + +impl RawKey<'_> { + pub(crate) const fn kind(&self) -> EntryKind { + match self { + RawKey::Vendor(_) => EntryKind::JmapVendor, + RawKey::Registered(_) => EntryKind::JmapRegistered, + RawKey::Dav(..) => EntryKind::Dav, + RawKey::Imap(_) => EntryKind::Imap, + } + } +} + +impl<'x> Iterator for RawEntries<'x> { + type Item = RawEntry<'x>; + + fn next(&mut self) -> Option { + self.remaining = self.remaining.checked_sub(1)?; + let entry = parse_entry(&mut self.reader); + if entry.is_none() { + self.remaining = 0; + } + entry + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.remaining)) + } +} + +impl RawEntries<'_> { + fn stop(&mut self) { + self.remaining = 0; + } +} + +impl<'x> MetadataView<'x> { + pub fn new(bytes: &'x [u8]) -> Option { + validate(bytes)?; + unsafe { Self::from_trusted(bytes) } + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_trusted(bytes: &'x [u8]) -> Option { + let mut entries = unsafe { Reader::trusted(bytes) }; + let (kinds, count) = header(bytes, &mut entries)?; + Some(MetadataView { + bytes, + kinds, + count, + entries, + }) + } + + pub fn empty() -> Self { + MetadataView { + bytes: &EMPTY_CONTAINER, + kinds: MetadataKinds::NONE, + count: 0, + entries: Reader::empty(), + } + } + + pub fn kinds(&self) -> MetadataKinds { + self.kinds + } + + pub fn len(&self) -> usize { + self.count + } + + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + pub fn as_bytes(&self) -> &'x [u8] { + self.bytes + } + + pub(crate) fn raw_entries(&self) -> RawEntries<'x> { + RawEntries { + reader: self.entries, + remaining: self.count, + } + } + + fn entries_of(&self, kinds: MetadataKinds) -> RawEntries<'x> { + if self.kinds.intersects(kinds) { + self.raw_entries() + } else { + RawEntries { + reader: Reader::empty(), + remaining: 0, + } + } + } + + pub fn jmap(&self) -> JmapEntries<'x> { + JmapEntries { + entries: self.entries_of(MetadataKinds::JMAP), + } + } + + pub fn jmap_namespace(&self, namespace: &Namespace<'_>) -> Option> { + let wanted = match namespace { + Namespace::Registered(namespace) => RawKey::Registered(namespace.id), + Namespace::Vendor(name) => RawKey::Vendor(name), + }; + let mut entries = self.entries_of(MetadataKinds::JMAP); + while let Some(entry) = entries.next() { + if entry.key == wanted { + return Some(JsonView::new(entry.value)); + } else if entry.key.kind() > wanted.kind() { + entries.stop(); + } + } + None + } + + pub fn dav(&self) -> DavEntries<'x> { + DavEntries { + entries: self.entries_of(MetadataKinds::DAV), + } + } + + pub fn dav_property(&self, name: &XmlName<'_>) -> Option> { + self.dav() + .find(|(entry, _)| entry.matches(name.namespace(), name.name())) + .map(|(_, value)| value) + } + + pub fn imap(&self) -> ImapEntries<'x> { + ImapEntries { + entries: self.entries_of(MetadataKinds::IMAP), + } + } + + pub fn imap_entry(&self, name: &str) -> Option<&'x [u8]> { + self.imap() + .find(|(entry, _)| *entry == name) + .map(|(_, value)| value) + } +} + +impl<'x> Iterator for JmapEntries<'x> { + type Item = (Namespace<'x>, JsonView<'x>); + + fn next(&mut self) -> Option { + loop { + let entry = self.entries.next()?; + let namespace = match entry.key { + RawKey::Vendor(name) => Namespace::Vendor(name), + RawKey::Registered(id) => match RegisteredNamespace::by_id(id) { + Some(namespace) => Namespace::Registered(namespace), + None => continue, + }, + RawKey::Dav(..) | RawKey::Imap(_) => { + self.entries.stop(); + return None; + } + }; + return Some((namespace, JsonView::new(entry.value))); + } + } +} + +impl<'x> Iterator for DavEntries<'x> { + type Item = (XmlName<'x>, DavValueView<'x>); + + fn next(&mut self) -> Option { + loop { + let entry = self.entries.next()?; + match entry.key { + RawKey::Dav(namespace, name) => { + return Some(( + XmlName::borrowed(namespace, name), + DavValueView::new(entry.value), + )); + } + RawKey::Imap(_) => { + self.entries.stop(); + return None; + } + RawKey::Vendor(_) | RawKey::Registered(_) => {} + } + } + } +} + +impl<'x> Iterator for ImapEntries<'x> { + type Item = (&'x str, &'x [u8]); + + fn next(&mut self) -> Option { + loop { + let entry = self.entries.next()?; + if let RawKey::Imap(name) = entry.key { + let mut value = entry.value; + return value.bytes().map(|value| (name, value)); + } + } + } +} diff --git a/crates/types/src/metadata/xml/mod.rs b/crates/types/src/metadata/xml/mod.rs new file mode 100644 index 00000000..f86fd685 --- /dev/null +++ b/crates/types/src/metadata/xml/mod.rs @@ -0,0 +1,406 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod validate; +mod view; +mod writer; + + +pub(crate) use validate::check_property_name; +pub(crate) use view::{dav_value_span, validate_dav_value}; + +use super::{ + codec::{Reader, TrustedReader, bytes_len, varint_len, write_bytes, write_varint}, + registry::{known_xml_namespace, known_xml_namespace_id}, +}; +use std::{borrow::Cow, fmt}; +use validate::{AttributeScope, check_attribute, check_element, check_text, has_duplicates}; + +const NS_NONE: u8 = 0x00; +const NS_INLINE: u8 = 0xFF; +const NODE_TEXT: u8 = 0x01; +const NODE_ELEMENT: u8 = 0x02; +const VALUE_HAS_LANG: u8 = 1; +const ELEMENT_HAS_PREFIX: u8 = 1; +const MAX_XML_NESTING: u32 = 64; +const XML_PREFIX: &str = "xml"; +const XMLNS_PREFIX: &str = "xmlns"; +const GENERATED_PREFIX: &str = "a"; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] +pub struct XmlName<'x> { + pub namespace: Option>, + pub name: Cow<'x, str>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] +pub struct XmlValue<'x> { + pub lang: Option>, + pub attributes: Vec>, + pub children: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] +pub struct XmlElement<'x> { + pub name: XmlName<'x>, + pub prefix: Option>, + pub attributes: Vec>, + pub children: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] +pub struct XmlAttribute<'x> { + pub name: XmlName<'x>, + pub value: Cow<'x, str>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(stalwart_tests, derive(serde::Serialize, serde::Deserialize))] +pub enum XmlNode<'x> { + Text(Cow<'x, str>), + Element(XmlElement<'x>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum XmlError { + NestingTooDeep, + InvalidName, + InvalidPrefix, + InvalidNamespace, + InvalidCharacter, + ReservedAttribute, + DuplicateAttribute, +} + +#[derive(Debug, Clone, Copy)] +pub struct DavValueView<'x> { + reader: TrustedReader<'x>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedDavValue(Box<[u8]>); + +impl<'x> XmlName<'x> { + pub const fn borrowed(namespace: Option<&'x str>, name: &'x str) -> Self { + XmlName { + namespace: match namespace { + Some(namespace) => Some(Cow::Borrowed(namespace)), + None => None, + }, + name: Cow::Borrowed(name), + } + } + + pub fn owned(namespace: Option, name: String) -> XmlName<'static> { + XmlName { + namespace: namespace.map(Cow::Owned), + name: Cow::Owned(name), + } + } + + pub fn namespace(&self) -> Option<&str> { + self.namespace.as_deref() + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn matches(&self, namespace: Option<&str>, name: &str) -> bool { + self.name() == name && self.namespace() == namespace + } + + pub fn into_owned(self) -> XmlName<'static> { + XmlName { + namespace: self + .namespace + .map(|namespace| Cow::Owned(namespace.into_owned())), + name: Cow::Owned(self.name.into_owned()), + } + } + + pub fn validate_property(&self) -> Result<(), XmlError> { + check_property_name(self.namespace(), self.name()) + } + + pub fn write_empty(&self, out: &mut dyn fmt::Write) -> fmt::Result { + writer::write_empty_element(out, self.namespace(), self.name()) + } + + pub(crate) fn key_len(&self) -> usize { + ns_ref_len(self.namespace()) + self.name.len() + } + + pub(crate) fn write_key(&self, out: &mut Vec) { + write_ns_ref(out, self.namespace()); + out.extend_from_slice(self.name.as_bytes()); + } +} + +fn ns_ref_len(namespace: Option<&str>) -> usize { + match namespace { + None => 1, + Some(uri) if known_xml_namespace_id(uri).is_some() => 1, + Some(uri) => 1 + bytes_len(uri.len()), + } +} + +fn write_ns_ref(out: &mut Vec, namespace: Option<&str>) { + match namespace { + None => out.push(NS_NONE), + Some(uri) => match known_xml_namespace_id(uri) { + Some(id) => out.push(id), + None => { + out.push(NS_INLINE); + write_bytes(out, uri.as_bytes()); + } + }, + } +} + +pub(crate) fn read_ns_ref<'x, const TRUSTED: bool>( + reader: &mut Reader<'x, TRUSTED>, +) -> Option> { + match reader.u8()? { + NS_NONE => Some(None), + NS_INLINE => reader.str().map(Some), + id => known_xml_namespace(id).map(Some), + } +} + +fn element_prefix<'y>(namespace: Option<&str>, prefix: Option<&'y str>) -> Option<&'y str> { + namespace.and(prefix) +} + +fn attributes_len( + attributes: &[XmlAttribute<'_>], + scope: AttributeScope, +) -> Result { + let mut len = varint_len(attributes.len() as u64); + for attribute in attributes { + check_attribute( + attribute.name.namespace(), + attribute.name.name(), + &attribute.value, + scope, + )?; + len += ns_ref_len(attribute.name.namespace()) + + bytes_len(attribute.name.name.len()) + + bytes_len(attribute.value.len()); + } + if has_duplicates(attributes, |attribute| &attribute.name) { + return Err(XmlError::DuplicateAttribute); + } + Ok(len) +} + +fn children_len(children: &[XmlNode<'_>], nesting: u32) -> Result { + if nesting > MAX_XML_NESTING { + return Err(XmlError::NestingTooDeep); + } + let mut len = varint_len(children.len() as u64); + for child in children { + len += 1 + match child { + XmlNode::Text(text) => { + check_text(text)?; + bytes_len(text.len()) + } + XmlNode::Element(element) => element.encoded_len(nesting)?, + }; + } + Ok(len) +} + +fn write_attributes(out: &mut Vec, attributes: &[XmlAttribute<'_>]) { + write_varint(out, attributes.len() as u64); + for attribute in attributes { + write_ns_ref(out, attribute.name.namespace()); + write_bytes(out, attribute.name.name.as_bytes()); + write_bytes(out, attribute.value.as_bytes()); + } +} + +fn write_children(out: &mut Vec, children: &[XmlNode<'_>]) { + write_varint(out, children.len() as u64); + for child in children { + match child { + XmlNode::Text(text) => { + out.push(NODE_TEXT); + write_bytes(out, text.as_bytes()); + } + XmlNode::Element(element) => { + out.push(NODE_ELEMENT); + element.write(out); + } + } + } +} + +impl XmlElement<'_> { + fn stored_prefix(&self) -> Option<&str> { + element_prefix(self.name.namespace(), self.prefix.as_deref()) + } + + fn encoded_len(&self, nesting: u32) -> Result { + check_element( + self.name.namespace(), + self.name.name(), + self.prefix.as_deref(), + )?; + Ok(ns_ref_len(self.name.namespace()) + + bytes_len(self.name.name.len()) + + 1 + + self + .stored_prefix() + .map_or(0, |prefix| bytes_len(prefix.len())) + + attributes_len(&self.attributes, AttributeScope::Element)? + + children_len(&self.children, nesting + 1)?) + } + + fn write(&self, out: &mut Vec) { + write_ns_ref(out, self.name.namespace()); + write_bytes(out, self.name.name.as_bytes()); + match self.stored_prefix() { + Some(prefix) => { + out.push(ELEMENT_HAS_PREFIX); + write_bytes(out, prefix.as_bytes()); + } + None => out.push(0), + } + write_attributes(out, &self.attributes); + write_children(out, &self.children); + } +} + +impl<'x> XmlValue<'x> { + pub const MAX_DEPTH: usize = MAX_XML_NESTING as usize - 1; + + fn rest_len(&self) -> Result { + let lang_len = match &self.lang { + Some(lang) => { + check_text(lang)?; + bytes_len(lang.len()) + } + None => 0, + }; + Ok(1 + lang_len + + attributes_len(&self.attributes, AttributeScope::Property)? + + children_len(&self.children, 1)?) + } + + pub fn validate(&self) -> Result<(), XmlError> { + self.rest_len().map(|_| ()) + } + + pub fn encoded_len(&self) -> Result { + self.rest_len().map(bytes_len) + } + + pub fn encode(&self) -> Result, XmlError> { + let rest_len = self.rest_len()?; + let mut out = Vec::with_capacity(bytes_len(rest_len)); + write_varint(&mut out, rest_len as u64); + match &self.lang { + Some(lang) => { + out.push(VALUE_HAS_LANG); + write_bytes(&mut out, lang.as_bytes()); + } + None => out.push(0), + } + write_attributes(&mut out, &self.attributes); + write_children(&mut out, &self.children); + debug_assert_eq!(out.len(), bytes_len(rest_len), "XML value length mismatch"); + Ok(out) + } + + pub fn to_encoded(&self) -> Result { + self.encode() + .map(|bytes| EncodedDavValue(bytes.into_boxed_slice())) + } + + pub fn is_empty(&self) -> bool { + self.lang.is_none() && self.attributes.is_empty() && self.children.is_empty() + } + + pub fn into_owned(self) -> XmlValue<'static> { + XmlValue { + lang: self.lang.map(|lang| Cow::Owned(lang.into_owned())), + attributes: self + .attributes + .into_iter() + .map(XmlAttribute::into_owned) + .collect(), + children: self.children.into_iter().map(XmlNode::into_owned).collect(), + } + } +} + +impl XmlAttribute<'_> { + pub fn into_owned(self) -> XmlAttribute<'static> { + XmlAttribute { + name: self.name.into_owned(), + value: Cow::Owned(self.value.into_owned()), + } + } +} + +impl XmlNode<'_> { + pub fn into_owned(self) -> XmlNode<'static> { + match self { + XmlNode::Text(text) => XmlNode::Text(Cow::Owned(text.into_owned())), + XmlNode::Element(element) => XmlNode::Element(XmlElement { + name: element.name.into_owned(), + prefix: element.prefix.map(|prefix| Cow::Owned(prefix.into_owned())), + attributes: element + .attributes + .into_iter() + .map(XmlAttribute::into_owned) + .collect(), + children: element + .children + .into_iter() + .map(XmlNode::into_owned) + .collect(), + }), + } + } +} + +impl EncodedDavValue { + pub fn view(&self) -> DavValueView<'_> { + DavValueView::new(unsafe { Reader::trusted(&self.0) }) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + pub fn into_bytes(self) -> Box<[u8]> { + self.0 + } + + pub fn encoded_len(&self) -> usize { + self.0.len() + } +} + +impl fmt::Display for XmlError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + XmlError::NestingTooDeep => "XML value is nested too deeply", + XmlError::InvalidName => "invalid XML name", + XmlError::InvalidPrefix => "invalid XML namespace prefix", + XmlError::InvalidNamespace => "invalid XML namespace", + XmlError::InvalidCharacter => "character not allowed in XML", + XmlError::ReservedAttribute => "reserved XML attribute", + XmlError::DuplicateAttribute => "duplicate XML attribute", + }) + } +} diff --git a/crates/types/src/metadata/xml/validate.rs b/crates/types/src/metadata/xml/validate.rs new file mode 100644 index 00000000..2426fb7c --- /dev/null +++ b/crates/types/src/metadata/xml/validate.rs @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{XML_PREFIX, XMLNS_PREFIX, XmlError}; +use crate::metadata::registry::{XML_NAMESPACE, XMLNS_NAMESPACE}; +use memchr::memchr; + +const LINEAR_SCAN_MAX: usize = 8; +const LANG_ATTRIBUTE: &str = "lang"; +const NONCHARACTER_LEAD_BYTE: u8 = 0xEF; + +pub(crate) fn is_nc_name(name: &str) -> bool { + let mut chars = name.chars(); + chars.next().is_some_and(is_name_start) && chars.all(is_name_char) +} + +fn is_name_start(ch: char) -> bool { + if ch.is_ascii() { + ch.is_ascii_alphabetic() || ch == '_' + } else { + matches!( + ch, + '\u{C0}'..='\u{D6}' + | '\u{D8}'..='\u{F6}' + | '\u{F8}'..='\u{2FF}' + | '\u{370}'..='\u{37D}' + | '\u{37F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' + | '\u{2070}'..='\u{218F}' + | '\u{2C00}'..='\u{2FEF}' + | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' + | '\u{FDF0}'..='\u{FFFD}' + | '\u{10000}'..='\u{EFFFF}' + ) + } +} + +fn is_name_char(ch: char) -> bool { + is_name_start(ch) + || ch.is_ascii_digit() + || matches!( + ch, + '-' | '.' | '\u{B7}' | '\u{300}'..='\u{36F}' | '\u{203F}'..='\u{2040}' + ) +} + +pub(crate) fn is_xml_text(text: &str) -> bool { + let bytes = text.as_bytes(); + bytes + .iter() + .all(|&byte| byte >= 0x20 || matches!(byte, b'\t' | b'\n' | b'\r')) + && (memchr(NONCHARACTER_LEAD_BYTE, bytes).is_none() + || !text.contains(['\u{FFFE}', '\u{FFFF}'])) +} + +pub(crate) fn check_text(text: &str) -> Result<(), XmlError> { + if is_xml_text(text) { + Ok(()) + } else { + Err(XmlError::InvalidCharacter) + } +} + +fn check_local_name(name: &str) -> Result<(), XmlError> { + if is_nc_name(name) { + Ok(()) + } else { + Err(XmlError::InvalidName) + } +} + +fn check_namespace(uri: &str) -> Result<(), XmlError> { + if uri.is_empty() || uri == XMLNS_NAMESPACE { + Err(XmlError::InvalidNamespace) + } else { + check_text(uri) + } +} + +pub(crate) fn check_property_name(namespace: Option<&str>, name: &str) -> Result<(), XmlError> { + check_local_name(name)?; + match namespace { + Some(XML_NAMESPACE) => Err(XmlError::InvalidNamespace), + Some(uri) => check_namespace(uri), + None => Ok(()), + } +} + +pub(crate) fn check_element( + namespace: Option<&str>, + name: &str, + prefix: Option<&str>, +) -> Result<(), XmlError> { + check_local_name(name)?; + let Some(uri) = namespace else { + return Ok(()); + }; + check_namespace(uri)?; + let is_xml_namespace = uri == XML_NAMESPACE; + match prefix { + Some(prefix) + if !is_nc_name(prefix) + || prefix == XMLNS_PREFIX + || (prefix == XML_PREFIX) != is_xml_namespace => + { + Err(XmlError::InvalidPrefix) + } + None if is_xml_namespace => Err(XmlError::InvalidPrefix), + _ => Ok(()), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AttributeScope { + Property, + Element, +} + +pub(crate) fn check_attribute( + namespace: Option<&str>, + name: &str, + value: &str, + scope: AttributeScope, +) -> Result<(), XmlError> { + check_local_name(name)?; + match namespace { + None if name == XMLNS_PREFIX => return Err(XmlError::ReservedAttribute), + Some(XML_NAMESPACE) if scope == AttributeScope::Property && name == LANG_ATTRIBUTE => { + return Err(XmlError::ReservedAttribute); + } + Some(XMLNS_NAMESPACE) => return Err(XmlError::ReservedAttribute), + Some(uri) => check_namespace(uri)?, + None => {} + } + check_text(value) +} + +pub(crate) fn has_duplicates<'i, T, K: Ord>(items: &'i [T], key: impl Fn(&'i T) -> K) -> bool { + if items.len() <= LINEAR_SCAN_MAX { + let mut rest = items; + while let [first, tail @ ..] = rest { + let first = key(first); + if tail.iter().any(|item| key(item) == first) { + return true; + } + rest = tail; + } + false + } else { + let mut keys = items.iter().map(key).collect::>(); + keys.sort_unstable(); + keys.windows(2).any(|pair| matches!(pair, [a, b] if a == b)) + } +} + diff --git a/crates/types/src/metadata/xml/view.rs b/crates/types/src/metadata/xml/view.rs new file mode 100644 index 00000000..3147eac9 --- /dev/null +++ b/crates/types/src/metadata/xml/view.rs @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + DavValueView, ELEMENT_HAS_PREFIX, EncodedDavValue, MAX_XML_NESTING, NODE_ELEMENT, NODE_TEXT, + VALUE_HAS_LANG, XmlAttribute, XmlElement, XmlName, XmlNode, XmlValue, read_ns_ref, + validate::{AttributeScope, check_attribute, check_element, has_duplicates, is_xml_text}, + writer::write_element, +}; +use crate::metadata::codec::{CheckedReader, Reader, TrustedReader}; +use std::{borrow::Cow, fmt}; + +pub(crate) fn dav_value_span<'x, const TRUSTED: bool>( + reader: &mut Reader<'x, TRUSTED>, +) -> Option> { + let start = *reader; + let len = reader.len()?; + reader.skip(len)?; + reader.consumed_since(&start) +} + +fn validate_attributes(reader: &mut CheckedReader<'_>, scope: AttributeScope) -> Option<()> { + let count = reader.len()?; + let mut names = Vec::with_capacity(count.min(reader.remaining().len())); + for _ in 0..count { + let namespace = read_ns_ref(reader)?; + let name = reader.str()?; + check_attribute(namespace, name, reader.str()?, scope).ok()?; + names.push((namespace, name)); + } + (!has_duplicates(&names, |name| *name)).then_some(()) +} + +fn validate_children(reader: &mut CheckedReader<'_>, nesting: u32) -> Option<()> { + if nesting > MAX_XML_NESTING { + return None; + } + let count = reader.len()?; + for _ in 0..count { + match reader.u8()? { + NODE_TEXT => { + is_xml_text(reader.str()?).then_some(())?; + } + NODE_ELEMENT => { + let namespace = read_ns_ref(reader)?; + let name = reader.str()?; + let prefix = match reader.u8()? { + 0 => None, + ELEMENT_HAS_PREFIX => Some(reader.str()?), + _ => return None, + }; + if prefix.is_some() && namespace.is_none() { + return None; + } + check_element(namespace, name, prefix).ok()?; + validate_attributes(reader, AttributeScope::Element)?; + validate_children(reader, nesting + 1)?; + } + _ => return None, + } + } + Some(()) +} + +pub(crate) fn validate_dav_value(reader: &mut CheckedReader<'_>) -> Option<()> { + let len = reader.len()?; + let mut rest = reader.take_reader(len)?; + match rest.u8()? { + 0 => {} + VALUE_HAS_LANG => { + is_xml_text(rest.str()?).then_some(())?; + } + _ => return None, + } + validate_attributes(&mut rest, AttributeScope::Property)?; + validate_children(&mut rest, 1)?; + rest.is_empty().then_some(()) +} + +struct ValueParts<'x> { + lang: Option<&'x str>, + body: TrustedReader<'x>, +} + +impl<'x> DavValueView<'x> { + pub(crate) const fn new(reader: TrustedReader<'x>) -> Self { + DavValueView { reader } + } + + pub fn parse(bytes: &'x [u8]) -> Option { + let mut reader = Reader::checked(bytes); + validate_dav_value(&mut reader)?; + reader.is_empty().then_some(())?; + Some(DavValueView { + reader: unsafe { Reader::trusted(bytes) }, + }) + } + + pub fn as_bytes(&self) -> &'x [u8] { + self.reader.remaining() + } + + pub fn encoded_len(&self) -> usize { + self.reader.remaining().len() + } + + fn parts(&self) -> Option> { + let mut reader = self.reader; + let len = reader.len()?; + let mut body = reader.take_reader(len)?; + let lang = match body.u8()? { + VALUE_HAS_LANG => Some(body.str()?), + _ => None, + }; + Some(ValueParts { lang, body }) + } + + pub fn lang(&self) -> Option<&'x str> { + self.parts()?.lang + } + + pub fn to_value(&self) -> Option> { + let mut parts = self.parts()?; + Some(XmlValue { + lang: parts.lang.map(Cow::Borrowed), + attributes: decode_attributes(&mut parts.body)?, + children: decode_children(&mut parts.body, 1)?, + }) + } + + pub fn text(&self) -> Option> { + let mut body = self.parts()?.body; + if body.len()? == 0 && body.len()? == 1 && body.u8()? == NODE_TEXT { + return body.str().map(Cow::Borrowed); + } + let mut text = String::new(); + for child in self.to_value()?.children { + if let XmlNode::Text(part) = child { + text.push_str(&part); + } + } + Some(Cow::Owned(text)) + } + + pub fn to_encoded(&self) -> EncodedDavValue { + EncodedDavValue(self.as_bytes().into()) + } + + pub fn write_property(&self, name: &XmlName<'_>, out: &mut dyn fmt::Write) -> fmt::Result { + let mut parts = self.parts().ok_or(fmt::Error)?; + write_element( + out, + name.namespace(), + name.name(), + None, + parts.lang, + &mut parts.body, + ) + } + + pub fn write_property_prefixed( + &self, + name: &XmlName<'_>, + bound_prefix: &str, + out: &mut dyn fmt::Write, + ) -> fmt::Result { + let mut parts = self.parts().ok_or(fmt::Error)?; + write_element( + out, + name.namespace(), + name.name(), + Some(bound_prefix), + parts.lang, + &mut parts.body, + ) + } +} + +fn decode_attributes<'x>(reader: &mut TrustedReader<'x>) -> Option>> { + let count = reader.len()?; + let mut attributes = Vec::with_capacity(count.min(reader.remaining().len())); + for _ in 0..count { + let namespace = read_ns_ref(reader)?; + let name = reader.str()?; + let value = reader.str()?; + attributes.push(XmlAttribute { + name: XmlName::borrowed(namespace, name), + value: Cow::Borrowed(value), + }); + } + Some(attributes) +} + +fn decode_children<'x>(reader: &mut TrustedReader<'x>, nesting: u32) -> Option>> { + if nesting > MAX_XML_NESTING { + return None; + } + let count = reader.len()?; + let mut children = Vec::with_capacity(count.min(reader.remaining().len())); + for _ in 0..count { + children.push(match reader.u8()? { + NODE_TEXT => XmlNode::Text(Cow::Borrowed(reader.str()?)), + NODE_ELEMENT => { + let namespace = read_ns_ref(reader)?; + let name = reader.str()?; + let prefix = match reader.u8()? { + ELEMENT_HAS_PREFIX => Some(Cow::Borrowed(reader.str()?)), + _ => None, + }; + XmlNode::Element(XmlElement { + name: XmlName::borrowed(namespace, name), + prefix, + attributes: decode_attributes(reader)?, + children: decode_children(reader, nesting + 1)?, + }) + } + _ => return None, + }); + } + Some(children) +} diff --git a/crates/types/src/metadata/xml/writer.rs b/crates/types/src/metadata/xml/writer.rs new file mode 100644 index 00000000..477f8ff8 --- /dev/null +++ b/crates/types/src/metadata/xml/writer.rs @@ -0,0 +1,349 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + ELEMENT_HAS_PREFIX, GENERATED_PREFIX, MAX_XML_NESTING, NODE_ELEMENT, NODE_TEXT, XML_PREFIX, + element_prefix, read_ns_ref, +}; +use crate::metadata::{codec::TrustedReader, registry::XML_NAMESPACE}; +use memchr::{memchr, memchr3}; +use std::fmt; + +pub(super) fn write_element<'a>( + out: &mut dyn fmt::Write, + namespace: Option<&'a str>, + name: &'a str, + bound_prefix: Option<&'a str>, + lang: Option<&'a str>, + reader: &mut TrustedReader<'a>, +) -> fmt::Result { + let mut writer = XmlWriter { + out, + bindings: Vec::new(), + next_generated: 0, + }; + let prefix = match (bound_prefix, namespace) { + (Some(prefix), Some(uri)) => { + writer.bindings.push(Binding { + prefix: Prefix::Stored(prefix), + uri, + }); + Some(prefix) + } + _ => None, + }; + writer.element_at( + ElementStart { + namespace, + name, + prefix, + lang, + }, + reader, + DefaultNamespace::Unknown, + 0, + ) +} + +pub(super) fn write_empty_element( + out: &mut dyn fmt::Write, + namespace: Option<&str>, + name: &str, +) -> fmt::Result { + out.write_char('<')?; + out.write_str(name)?; + out.write_str(" xmlns=\"")?; + escape(out, namespace.unwrap_or_default(), Escape::Attribute)?; + out.write_str("\"/>") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DefaultNamespace<'a> { + Unknown, + Namespace(Option<&'a str>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Prefix<'a> { + Stored(&'a str), + Generated(u32), +} + +struct Binding<'a> { + prefix: Prefix<'a>, + uri: &'a str, +} + +struct ElementStart<'a> { + namespace: Option<&'a str>, + name: &'a str, + prefix: Option<&'a str>, + lang: Option<&'a str>, +} + +struct XmlWriter<'w, 'a> { + out: &'w mut dyn fmt::Write, + bindings: Vec>, + next_generated: u32, +} + +impl Prefix<'_> { + fn write(self, out: &mut dyn fmt::Write) -> fmt::Result { + match self { + Prefix::Stored(prefix) => out.write_str(prefix), + Prefix::Generated(number) => write!(out, "{GENERATED_PREFIX}{number}"), + } + } + + fn renders_as(self, other: Prefix<'_>) -> bool { + match (self, other) { + (Prefix::Stored(left), Prefix::Stored(right)) => left == right, + (Prefix::Generated(left), Prefix::Generated(right)) => left == right, + (Prefix::Stored(stored), Prefix::Generated(number)) + | (Prefix::Generated(number), Prefix::Stored(stored)) => { + renders_generated(stored, number) + } + } + } +} + +fn renders_generated(stored: &str, number: u32) -> bool { + stored.strip_prefix(GENERATED_PREFIX).is_some_and(|digits| { + digits.bytes().all(|byte| byte.is_ascii_digit()) + && (digits.len() == 1 || !digits.starts_with('0')) + && digits.parse::() == Ok(number) + }) +} + +impl<'a> XmlWriter<'_, 'a> { + fn resolve(&self, prefix: Prefix<'_>) -> Option<&'a str> { + self.bindings + .iter() + .rev() + .find(|binding| binding.prefix.renders_as(prefix)) + .map(|binding| binding.uri) + } + + fn is_bound(&self, prefix: &str, uri: &str) -> bool { + (prefix == XML_PREFIX && uri == XML_NAMESPACE) + || self.resolve(Prefix::Stored(prefix)) == Some(uri) + } + + fn prefix_for(&self, uri: &str) -> Option> { + self.bindings + .iter() + .rev() + .filter(|binding| binding.uri == uri) + .map(|binding| binding.prefix) + .find(|prefix| self.resolve(*prefix) == Some(uri)) + } + + fn generate_prefix(&mut self) -> Result, fmt::Error> { + loop { + let candidate = Prefix::Generated(self.next_generated); + self.next_generated = self.next_generated.checked_add(1).ok_or(fmt::Error)?; + if self.resolve(candidate).is_none() { + return Ok(candidate); + } + } + } + + fn element_at( + &mut self, + start: ElementStart<'a>, + reader: &mut TrustedReader<'a>, + parent_default: DefaultNamespace<'a>, + nesting: u32, + ) -> fmt::Result { + if nesting > MAX_XML_NESTING { + return Err(fmt::Error); + } + let mark = self.bindings.len(); + let prefix = element_prefix(start.namespace, start.prefix); + let mut child_default = parent_default; + + self.out.write_char('<')?; + match (prefix, start.namespace) { + (Some(prefix), Some(uri)) => { + self.out.write_str(prefix)?; + self.out.write_char(':')?; + self.out.write_str(start.name)?; + if !self.is_bound(prefix, uri) { + self.out.write_str(" xmlns:")?; + self.out.write_str(prefix)?; + self.out.write_str("=\"")?; + escape(self.out, uri, Escape::Attribute)?; + self.out.write_char('"')?; + self.bindings.push(Binding { + prefix: Prefix::Stored(prefix), + uri, + }); + } + } + _ => { + self.out.write_str(start.name)?; + let own = DefaultNamespace::Namespace(start.namespace); + if parent_default != own { + self.out.write_str(" xmlns=\"")?; + escape( + self.out, + start.namespace.unwrap_or_default(), + Escape::Attribute, + )?; + self.out.write_char('"')?; + } + child_default = own; + } + } + + if let Some(lang) = start.lang { + self.out.write_str(" xml:lang=\"")?; + escape(self.out, lang, Escape::Attribute)?; + self.out.write_char('"')?; + } + + let attributes = reader.len().ok_or(fmt::Error)?; + for _ in 0..attributes { + let namespace = read_ns_ref(reader).ok_or(fmt::Error)?; + let name = reader.str().ok_or(fmt::Error)?; + let value = reader.str().ok_or(fmt::Error)?; + self.out.write_char(' ')?; + match namespace { + None => {} + Some(XML_NAMESPACE) => self.out.write_str("xml:")?, + Some(uri) => { + let prefix = match self.prefix_for(uri) { + Some(prefix) => prefix, + None => { + let prefix = self.generate_prefix()?; + self.out.write_str("xmlns:")?; + prefix.write(self.out)?; + self.out.write_str("=\"")?; + escape(self.out, uri, Escape::Attribute)?; + self.out.write_str("\" ")?; + self.bindings.push(Binding { prefix, uri }); + prefix + } + }; + prefix.write(self.out)?; + self.out.write_char(':')?; + } + } + self.out.write_str(name)?; + self.out.write_str("=\"")?; + escape(self.out, value, Escape::Attribute)?; + self.out.write_char('"')?; + } + + let children = reader.len().ok_or(fmt::Error)?; + if children == 0 { + self.out.write_str("/>")?; + } else { + self.out.write_char('>')?; + for _ in 0..children { + match reader.u8().ok_or(fmt::Error)? { + NODE_TEXT => escape(self.out, reader.str().ok_or(fmt::Error)?, Escape::Text)?, + NODE_ELEMENT => { + let namespace = read_ns_ref(reader).ok_or(fmt::Error)?; + let name = reader.str().ok_or(fmt::Error)?; + let prefix = match reader.u8().ok_or(fmt::Error)? { + ELEMENT_HAS_PREFIX => Some(reader.str().ok_or(fmt::Error)?), + _ => None, + }; + self.element_at( + ElementStart { + namespace, + name, + prefix, + lang: None, + }, + reader, + child_default, + nesting + 1, + )?; + } + _ => return Err(fmt::Error), + } + } + self.out.write_str("')?; + } + + self.bindings.truncate(mark); + Ok(()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Escape { + Text, + Attribute, +} + +fn escape_for(byte: u8, context: Escape) -> Option<&'static str> { + match (byte, context) { + (b'<', _) => Some("<"), + (b'&', _) => Some("&"), + (b'\r', _) => Some(" "), + (b'>', Escape::Text) => Some(">"), + (b'"', Escape::Attribute) => Some("""), + (b'\n', Escape::Attribute) => Some(" "), + (b'\t', Escape::Attribute) => Some(" "), + _ => None, + } +} + +fn find_markup(bytes: &[u8], from: usize, context: Escape) -> Option { + let rest = bytes.get(from..)?; + match context { + Escape::Text => memchr3(b'<', b'&', b'>', rest), + Escape::Attribute => memchr3(b'<', b'&', b'"', rest), + } + .map(|position| position + from) +} + +fn find_whitespace(bytes: &[u8], from: usize, context: Escape) -> Option { + let rest = bytes.get(from..)?; + match context { + Escape::Text => memchr(b'\r', rest), + Escape::Attribute => memchr3(b'\t', b'\n', b'\r', rest), + } + .map(|position| position + from) +} + +fn escape(out: &mut dyn fmt::Write, text: &str, context: Escape) -> fmt::Result { + let bytes = text.as_bytes(); + let mut markup = find_markup(bytes, 0, context); + let mut whitespace = find_whitespace(bytes, 0, context); + let mut last = 0; + + loop { + let index = match (markup, whitespace) { + (Some(markup), Some(whitespace)) => markup.min(whitespace), + (Some(index), None) | (None, Some(index)) => index, + (None, None) => break, + }; + if let Some(replacement) = bytes.get(index).and_then(|byte| escape_for(*byte, context)) { + out.write_str(text.get(last..index).unwrap_or_default())?; + out.write_str(replacement)?; + } + last = index + 1; + if markup == Some(index) { + markup = find_markup(bytes, last, context); + } + if whitespace == Some(index) { + whitespace = find_whitespace(bytes, last, context); + } + } + + out.write_str(text.get(last..).unwrap_or_default()) +} + diff --git a/crates/types/src/special_use.rs b/crates/types/src/special_use.rs index 4e9cda04..1b6c5110 100644 --- a/crates/types/src/special_use.rs +++ b/crates/types/src/special_use.rs @@ -6,20 +6,7 @@ use jmap_tools::{Element, Property, Value}; -#[derive( - rkyv::Archive, - rkyv::Deserialize, - rkyv::Serialize, - Clone, - Copy, - PartialEq, - Eq, - Hash, - Debug, - PartialOrd, - Ord, -)] -#[rkyv(derive(Debug))] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] pub enum SpecialUse { Inbox, Trash, @@ -37,7 +24,7 @@ pub enum SpecialUse { impl SpecialUse { pub fn parse(s: &str) -> Option { - hashify::tiny_map_ignore_case!(s.as_bytes(), + hashify::map_ignore_case!(s.as_bytes(), SpecialUse, b"inbox" => SpecialUse::Inbox, b"trash" => SpecialUse::Trash, b"junk" => SpecialUse::Junk, @@ -50,6 +37,7 @@ impl SpecialUse { b"scheduled" => SpecialUse::Scheduled, b"snoozed" => SpecialUse::Snoozed, ) + .copied() } #[inline(always)] @@ -73,42 +61,38 @@ impl SpecialUse { SpecialUse::None => None, } } -} -impl ArchivedSpecialUse { - pub fn as_str(&self) -> Option<&'static str> { + pub const fn id(self) -> u8 { match self { - ArchivedSpecialUse::Inbox => Some("inbox"), - ArchivedSpecialUse::Trash => Some("trash"), - ArchivedSpecialUse::Junk => Some("junk"), - ArchivedSpecialUse::Drafts => Some("drafts"), - ArchivedSpecialUse::Archive => Some("archive"), - ArchivedSpecialUse::Sent => Some("sent"), - ArchivedSpecialUse::Shared => Some("shared"), - ArchivedSpecialUse::Important => Some("important"), - ArchivedSpecialUse::Memos => Some("memos"), - ArchivedSpecialUse::Scheduled => Some("scheduled"), - ArchivedSpecialUse::Snoozed => Some("snoozed"), - ArchivedSpecialUse::None => None, + SpecialUse::None => 0, + SpecialUse::Inbox => 1, + SpecialUse::Trash => 2, + SpecialUse::Junk => 3, + SpecialUse::Drafts => 4, + SpecialUse::Archive => 5, + SpecialUse::Sent => 6, + SpecialUse::Shared => 7, + SpecialUse::Important => 8, + SpecialUse::Memos => 9, + SpecialUse::Scheduled => 10, + SpecialUse::Snoozed => 11, } } -} -impl From<&ArchivedSpecialUse> for SpecialUse { - fn from(value: &ArchivedSpecialUse) -> Self { - match value { - ArchivedSpecialUse::Inbox => SpecialUse::Inbox, - ArchivedSpecialUse::Trash => SpecialUse::Trash, - ArchivedSpecialUse::Junk => SpecialUse::Junk, - ArchivedSpecialUse::Drafts => SpecialUse::Drafts, - ArchivedSpecialUse::Archive => SpecialUse::Archive, - ArchivedSpecialUse::Sent => SpecialUse::Sent, - ArchivedSpecialUse::Shared => SpecialUse::Shared, - ArchivedSpecialUse::Important => SpecialUse::Important, - ArchivedSpecialUse::Memos => SpecialUse::Memos, - ArchivedSpecialUse::Scheduled => SpecialUse::Scheduled, - ArchivedSpecialUse::Snoozed => SpecialUse::Snoozed, - ArchivedSpecialUse::None => SpecialUse::None, + pub const fn from_id(id: u8) -> Self { + match id { + 1 => SpecialUse::Inbox, + 2 => SpecialUse::Trash, + 3 => SpecialUse::Junk, + 4 => SpecialUse::Drafts, + 5 => SpecialUse::Archive, + 6 => SpecialUse::Sent, + 7 => SpecialUse::Shared, + 8 => SpecialUse::Important, + 9 => SpecialUse::Memos, + 10 => SpecialUse::Scheduled, + 11 => SpecialUse::Snoozed, + _ => SpecialUse::None, } } } diff --git a/crates/types/src/text.rs b/crates/types/src/text.rs new file mode 100644 index 00000000..0b9f08ac --- /dev/null +++ b/crates/types/src/text.rs @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::id::Id; +use serde::{Serialize, Serializer}; +use std::{ + borrow::Cow, + fmt::{self, Display, Write}, +}; + +#[derive(Clone, Copy)] +pub enum Text<'x> { + Static(&'static str), + Str(&'x str), + Id(Id), + Reference(&'x str), + Display(&'x dyn Display), +} + +struct Expect<'x>(&'x str); + +impl<'x> Text<'x> { + #[inline] + pub fn to_cow(self) -> Cow<'static, str> { + match self { + Text::Static(text) => Cow::Borrowed(text), + Text::Str(text) => Cow::Owned(text.to_string()), + Text::Id(id) => Cow::Owned(id.as_string()), + Text::Reference(reference) => Cow::Owned(format!("#{reference}")), + Text::Display(value) => Cow::Owned(value.to_string()), + } + } + + #[inline] + pub fn eq_str(self, text: &str) -> bool { + match self { + Text::Static(value) => value == text, + Text::Str(value) => value == text, + Text::Id(id) => id.text().as_str() == text, + Text::Reference(reference) => text.strip_prefix('#') == Some(reference), + Text::Display(value) => { + let mut expect = Expect(text); + write!(expect, "{value}").is_ok() && expect.0.is_empty() + } + } + } + + #[inline] + pub fn eq_text(self, other: Text<'_>) -> bool { + match (self, other) { + (Text::Id(a), Text::Id(b)) => a == b, + (Text::Static(text) | Text::Str(text), other) + | (other, Text::Static(text) | Text::Str(text)) => other.eq_str(text), + (Text::Id(id), other) | (other, Text::Id(id)) => other.eq_str(id.text().as_str()), + (Text::Reference(a), Text::Reference(b)) => a == b, + (Text::Reference(_) | Text::Display(_), Text::Reference(_) | Text::Display(_)) => { + other.eq_str(&self.to_cow()) + } + } + } +} + +impl Serialize for Text<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Text::Static(text) => serializer.serialize_str(text), + Text::Str(text) => serializer.serialize_str(text), + Text::Id(id) => serializer.serialize_str(id.text().as_str()), + Text::Reference(reference) => serializer.collect_str(&format_args!("#{reference}")), + Text::Display(value) => serializer.collect_str(value), + } + } +} + +impl Write for Expect<'_> { + fn write_str(&mut self, text: &str) -> fmt::Result { + self.0 = self.0.strip_prefix(text).ok_or(fmt::Error)?; + Ok(()) + } +} diff --git a/crates/types/src/type_state.rs b/crates/types/src/type_state.rs index b4f8b703..abb41c2a 100644 --- a/crates/types/src/type_state.rs +++ b/crates/types/src/type_state.rs @@ -69,6 +69,8 @@ pub struct StateChange { pub types: Bitmap, } +const _: () = assert!(size_of::() <= 24); + impl StateChange { pub fn new(account_id: u32) -> Self { Self { @@ -161,6 +163,9 @@ impl DataType { (SyncCollection::Identity, _) => DataType::Identity.into(), (SyncCollection::EmailSubmission, _) => DataType::EmailSubmission.into(), (SyncCollection::SieveScript, _) => DataType::SieveScript.into(), + (SyncCollection::CalendarEventNotification, _) => { + DataType::CalendarEventNotification.into() + } _ => None, } } @@ -168,7 +173,7 @@ impl DataType { impl DataType { pub fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), DataType, b"Email" => DataType::Email, b"EmailDelivery" => DataType::EmailDelivery, b"EmailSubmission" => DataType::EmailSubmission, @@ -193,6 +198,7 @@ impl DataType { b"ParticipantIdentity" => DataType::ParticipantIdentity, b"CalendarAlert" => DataType::CalendarAlert, ) + .copied() } pub fn as_str(&self) -> &'static str { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 7b8afc55..ffe31f09 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,42 +1,50 @@ [package] name = "utils" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] trc = { path = "../trc" } -rustls = { version = "0.23.43", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +rustls = { version = "0.23.46", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } rustls-pki-types = { version = "1" } rustls-platform-verifier = "0.7" -tokio = { version = "1.53", features = ["net", "macros", "signal"] } +tokio = { version = "1.54", features = ["net", "macros", "signal", "rt-multi-thread"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } serde = { version = "1.0", features = ["derive"]} -mail-auth = { version = "0.13" } +mail-auth = { version = "0.14" } ahash = { version = "0.8", features = ["serde"] } -chrono = "0.4" +jiff = "0.2.38" aws-lc-rs = { version = "1" } -base64 = "0.23" +encodify = "1.0" reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "stream"]} futures = "0.3" form_urlencoded = "1.2.2" psl = "2" +url = "2.5" +hashify = "0.3" idna = "1.1" quick_cache = "0.7" -rkyv = { version = "0.8.18", features = ["little_endian"] } +rkyv.workspace = true compact_str = "0.10.0" -xxhash-rust = { version = "0.8.18", features = ["xxh3"] } +xxhash-rust = { version = "0.8.19", features = ["xxh3"] } farmhash = "1.1.5" nohash-hasher = "0.2.0" arcstr = "1.2.0" +memchr = "2.8" +simdutf8 = "0.1.5" +libc = "0.2.191" [target.'cfg(unix)'.dependencies] -privdrop = "0.5.6" +privdrop = "0.5.7" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_System_SystemInformation"] } [features] -test_mode = [] +enterprise = ["trc/enterprise"] [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } [lints] workspace = true diff --git a/crates/utils/proc-macros/Cargo.toml b/crates/utils/proc-macros/Cargo.toml index 3d447c40..a7b3694a 100644 --- a/crates/utils/proc-macros/Cargo.toml +++ b/crates/utils/proc-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "proc_macros" -version = "0.16.27" +version = "1.0.0" edition = "2024" [lib] diff --git a/crates/utils/src/cache.rs b/crates/utils/src/cache.rs index d4a7ccaf..8b961ac1 100644 --- a/crates/utils/src/cache.rs +++ b/crates/utils/src/cache.rs @@ -5,54 +5,159 @@ */ use arcstr::ArcStr; -use mail_auth::{DnssecStatus, MX, RecordSet, ResolverCache, Txt}; +use compact_str::CompactString; +use mail_auth::{DnssecStatus, Mx, Negative, RecordSet, ResolverCache, TxtRecord}; use quick_cache::{ - Equivalent, Options, OptionsBuilder, Weighter, + Equivalent, Lifecycle, Options, OptionsBuilder, Weighter, sync::{DefaultLifecycle, PlaceholderGuard}, }; use std::{ borrow::Borrow, - hash::Hash, + hash::{BuildHasher, Hash}, net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, time::{Duration, Instant}, }; -pub struct Cache( - quick_cache::sync::Cache, -); +const HOT_ALLOCATION: f64 = 0.97; +const SPILL_FRACTION: u64 = 8; -pub struct CacheWithTtl( - quick_cache::sync::Cache, CacheItemWeighter, ahash::RandomState>, -); +pub const ENTRY_SLOT_SIZE: u64 = 48; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryCharge { + Payload, + PayloadAndSlot, +} + +pub struct Cache< + K: Eq + Hash + CacheItemWeight, + V: Clone + CacheItemWeight, + L: Lifecycle + Clone = DefaultLifecycle, + S: BuildHasher + Clone = ahash::RandomState, +> { + inner: quick_cache::sync::Cache, + weighter: CacheItemWeighter, + admission_limit: u64, + name: &'static str, +} + +pub struct SpillCache { + main: Cache, + spill: Cache, +} + +pub struct SpillGuard<'a, K: Eq + Hash + CacheItemWeight, V: Clone + CacheItemWeight> { + guard: CacheGuard<'a, K, V, DefaultLifecycle>, + key: K, + cache: &'a SpillCache, +} + +pub type CacheGuard<'a, K, V, L, S = ahash::RandomState> = + PlaceholderGuard<'a, K, V, CacheItemWeighter, S, L>; + +pub struct CacheWithTtl { + inner: quick_cache::sync::Cache, CacheItemWeighter, ahash::RandomState>, + clock: TtlClock, +} + +pub type TtlPlaceholder<'a, K, V> = PlaceholderGuard< + 'a, + K, + TtlEntry, + CacheItemWeighter, + ahash::RandomState, + DefaultLifecycle>, +>; + +pub struct TtlGuard<'a, K: Eq + Hash + CacheItemWeight, V: Clone + CacheItemWeight> { + guard: TtlPlaceholder<'a, K, V>, + clock: &'a TtlClock, +} #[derive(Clone)] pub struct TtlEntry { value: V, - expires: Instant, + expires: u32, } +#[derive(Debug, Clone, Copy)] +pub struct TtlClock(Instant); + impl Cache { pub fn new(weight: u64, estimated_weight: u64) -> Self { - Self::new_estimated(weight as usize / estimated_weight as usize, weight) + Self::new_with_charge(weight, estimated_weight, EntryCharge::PayloadAndSlot) + } + + pub fn new_with_charge(weight: u64, estimated_weight: u64, charge: EntryCharge) -> Self { + let weighter = CacheItemWeighter::from(charge); + Self::build( + cache_options(weighter.items(weight, estimated_weight), weight, None), + DefaultLifecycle::default(), + weighter, + ) } pub fn new_estimated(estimated_items_capacity: usize, weight_capacity: u64) -> Self { - Self(quick_cache::sync::Cache::with_options( + Self::build( cache_options(estimated_items_capacity, weight_capacity, None), - CacheItemWeighter, - ahash::RandomState::default(), DefaultLifecycle::default(), - )) + EntryCharge::PayloadAndSlot.into(), + ) } pub fn new_single_shard(weight: u64, estimated_weight: u64) -> Self { - Self(quick_cache::sync::Cache::with_options( - cache_options(weight as usize / estimated_weight as usize, weight, Some(1)), - CacheItemWeighter, - ahash::RandomState::default(), + Self::new_single_shard_with(weight, estimated_weight, DefaultLifecycle::default()) + } +} + +impl + Cache, S> +{ + pub fn new_with_hasher(weight: u64, estimated_weight: u64) -> Self { + let weighter = CacheItemWeighter::from(EntryCharge::PayloadAndSlot); + Self::build( + cache_options(weighter.items(weight, estimated_weight), weight, None), DefaultLifecycle::default(), - )) + weighter, + ) + } +} + +impl< + K: Eq + Hash + CacheItemWeight, + V: Clone + CacheItemWeight, + L: Lifecycle + Clone, + S: BuildHasher + Clone + Default, +> Cache +{ + pub fn new_single_shard_with(weight: u64, estimated_weight: u64, lifecycle: L) -> Self { + let weighter = CacheItemWeighter::from(EntryCharge::Payload); + Self::build( + cache_options(weighter.items(weight, estimated_weight), weight, Some(1)), + lifecycle, + weighter, + ) + } + + pub fn with_name(mut self, name: &'static str) -> Self { + self.name = name; + self + } + + fn build(options: Options, lifecycle: L, weighter: CacheItemWeighter) -> Self { + let inner = + quick_cache::sync::Cache::with_options(options, weighter, S::default(), lifecycle); + let shard_capacity = inner.shard_capacity(); + let admission_limit = ((shard_capacity as f64 * HOT_ALLOCATION) as u64) + .clamp(shard_capacity.min(1), shard_capacity); + + Self { + inner, + weighter, + admission_limit, + name: "", + } } #[inline(always)] @@ -60,7 +165,7 @@ impl Cache { where Q: Hash + Equivalent + ?Sized, { - self.0.get(key) + self.inner.get(key) } #[inline(always)] @@ -68,33 +173,57 @@ impl Cache { where Q: Hash + Equivalent + ?Sized, { - self.0.peek(key) + self.inner.peek(key) } #[inline(always)] - pub async fn get_value_or_guard_async<'a, Q>( + pub fn get_value_or_guard_async<'a, Q>( &'a self, key: &Q, - ) -> Result< - V, - PlaceholderGuard<'a, K, V, CacheItemWeighter, ahash::RandomState, DefaultLifecycle>, - > + ) -> impl Future>> where Q: Hash + Equivalent + ToOwned + ?Sized, { - self.0.get_value_or_guard_async(key).await + self.inner.get_value_or_guard_async(key) } #[inline(always)] pub fn insert(&self, key: K, value: V) { - self.0.insert(key, value); + if self.reject_oversized(&key, &value) { + return; + } + self.inner.insert(key, value); } #[inline(always)] pub fn update(&self, key: K, value: V) { - if let Err((key, value)) = self.0.replace(key, value, true) { - self.0.insert(key, value); + if self.reject_oversized(&key, &value) { + return; } + if let Err((key, value)) = self.inner.replace(key, value, true) { + self.inner.insert(key, value); + } + } + + #[inline(always)] + fn reject_oversized(&self, key: &K, value: &V) -> bool { + let weight = self.weighter.charge(key, value); + if weight <= self.admission_limit { + return false; + } + + trc::event!( + Cache(trc::CacheEvent::EntryTooLarge), + Name = self.name, + Size = weight, + Limit = self.admission_limit, + ); + true + } + + #[inline(always)] + pub fn is_oversized(&self, key: &K, value: &V) -> bool { + self.weighter.charge(key, value) > self.admission_limit } #[inline(always)] @@ -102,37 +231,58 @@ impl Cache { where Q: Hash + Equivalent + ?Sized, { - self.0.remove(key).map(|(_, v)| v) + self.inner.remove(key).map(|(_, v)| v) } #[inline(always)] pub fn clear(&self) { - self.0.clear(); + self.inner.clear(); } #[inline(always)] - pub fn inner(&self) -> &quick_cache::sync::Cache { - &self.0 + pub fn inner(&self) -> &quick_cache::sync::Cache { + &self.inner } #[inline(always)] pub fn weight_capacity(&self) -> u64 { - self.0.capacity() + self.inner.capacity() + } + + #[inline(always)] + pub fn weight(&self) -> u64 { + self.inner.weight() + } + + #[inline(always)] + pub fn admission_limit(&self) -> u64 { + self.admission_limit } } impl CacheWithTtl { pub fn new(weight: u64, estimated_weight: u64) -> Self { - Self::new_estimated(weight as usize / estimated_weight as usize, weight) + Self::new_estimated( + CacheItemWeighter::from(EntryCharge::PayloadAndSlot).items(weight, estimated_weight), + weight, + ) + } + + #[inline(always)] + pub fn weight(&self) -> u64 { + self.inner.weight() } pub fn new_estimated(estimated_items_capacity: usize, weight_capacity: u64) -> Self { - Self(quick_cache::sync::Cache::with_options( - cache_options(estimated_items_capacity, weight_capacity, None), - CacheItemWeighter, - ahash::RandomState::default(), - DefaultLifecycle::default(), - )) + Self { + inner: quick_cache::sync::Cache::with_options( + cache_options(estimated_items_capacity, weight_capacity, None), + CacheItemWeighter::from(EntryCharge::PayloadAndSlot), + ahash::RandomState::default(), + DefaultLifecycle::default(), + ), + clock: TtlClock::new(), + } } #[inline(always)] @@ -140,55 +290,47 @@ impl CacheWithTtl + ?Sized, { - self.0.get(key).and_then(|v| { - if v.expires > Instant::now() { + self.inner.get(key).and_then(|v| { + if self.clock.is_live(v.expires) { Some(v.value) } else { - self.0.remove(key); + self.inner + .remove_if(key, |entry| !self.clock.is_live(entry.expires)); None } }) } #[inline(always)] - pub async fn get_value_or_guard_async<'a, Q>( - &'a self, - key: &Q, - ) -> Result< - V, - PlaceholderGuard< - 'a, - K, - TtlEntry, - CacheItemWeighter, - ahash::RandomState, - DefaultLifecycle>, - >, - > + pub async fn get_value_or_guard_async<'a, Q>(&'a self, key: &Q) -> Result> where Q: Hash + Equivalent + ToOwned + ?Sized, { - match self.0.get_value_or_guard_async(key).await { - Ok(value) => { - if value.expires > Instant::now() { - Ok(value.value) - } else { - self.0.remove(key); - self.0.get_value_or_guard_async(key).await.map(|v| v.value) - } + let result = match self.inner.get_value_or_guard_async(key).await { + Ok(value) if self.clock.is_live(value.expires) => return Ok(value.value), + Ok(_) => { + self.inner + .remove_if(key, |entry| !self.clock.is_live(entry.expires)); + self.inner.get_value_or_guard_async(key).await } - Err(err) => Err(err), - } + Err(guard) => Err(guard), + }; + result.map(|v| v.value).map_err(|guard| TtlGuard { + guard, + clock: &self.clock, + }) } #[inline(always)] pub fn insert(&self, key: K, value: V, expires: Duration) { - self.0.insert(key, TtlEntry::new(value, expires)); + self.inner + .insert(key, TtlEntry::new(value, self.clock.expires_in(expires))); } #[inline(always)] pub fn insert_with_expiry(&self, key: K, value: V, expires: Instant) { - self.0.insert(key, TtlEntry::with_expiry(value, expires)); + self.inner + .insert(key, TtlEntry::new(value, self.clock.expires_at(expires))); } #[inline(always)] @@ -196,41 +338,251 @@ impl CacheWithTtl + ?Sized, { - self.0.remove(key).map(|(_, v)| v.value) + self.inner.remove(key).map(|(_, v)| v.value) } #[inline(always)] pub fn retain(&self, f: impl Fn(&K) -> bool) { - self.0.retain(|key, _| f(key)); + self.inner.retain(|key, _| f(key)); } #[inline(always)] pub fn clear(&self) { - self.0.clear(); + self.inner.clear(); } } +impl<'a, K: Eq + Hash + CacheItemWeight, V: Clone + CacheItemWeight> TtlGuard<'a, K, V> { + pub fn insert(self, value: V, expires: Duration) { + let _ = self + .guard + .insert(TtlEntry::new(value, self.clock.expires_in(expires))); + } + + pub fn insert_with_expiry(self, value: V, expires: Instant) { + let _ = self + .guard + .insert(TtlEntry::new(value, self.clock.expires_at(expires))); + } +} + +impl TtlClock { + pub fn new() -> Self { + TtlClock(Instant::now()) + } + + #[inline(always)] + pub fn now(&self) -> u32 { + clamp_secs(self.0.elapsed().as_secs()) + } + + #[inline(always)] + pub fn is_live(&self, expires: u32) -> bool { + expires > self.now() + } + + #[inline(always)] + pub fn expires_in(&self, ttl: Duration) -> u32 { + if !ttl.is_zero() { + ceil_secs(self.0.elapsed().saturating_add(ttl)) + } else { + 0 + } + } + + #[inline(always)] + pub fn expires_at(&self, expires: Instant) -> u32 { + if expires > Instant::now() { + ceil_secs(expires.saturating_duration_since(self.0)) + } else { + 0 + } + } +} + +impl Default for TtlClock { + fn default() -> Self { + TtlClock::new() + } +} + +#[inline(always)] +fn ceil_secs(elapsed: Duration) -> u32 { + clamp_secs(elapsed.as_secs() + u64::from(elapsed.subsec_nanos() > 0)) +} + +#[inline(always)] +fn clamp_secs(secs: u64) -> u32 { + u32::try_from(secs).unwrap_or(u32::MAX) +} + +impl SpillCache { + pub fn new(weight: u64, estimated_weight: u64) -> Self { + let weighter = CacheItemWeighter::from(EntryCharge::PayloadAndSlot); + let spill_weight = weight / SPILL_FRACTION; + let main_weight = weight - spill_weight; + SpillCache { + main: Cache::build( + cache_options( + weighter.items(main_weight, estimated_weight), + main_weight, + None, + ), + DefaultLifecycle::default(), + weighter, + ), + spill: Cache::build( + cache_options( + weighter.items(spill_weight, estimated_weight), + spill_weight, + Some(1), + ), + DefaultLifecycle::default(), + weighter, + ), + } + } + + pub fn with_name(self, name: &'static str) -> Self { + SpillCache { + main: self.main.with_name(name), + spill: self.spill.with_name(name), + } + } + + #[inline(always)] + pub fn get(&self, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + self.main.get(key).or_else(|| self.spill.get(key)) + } + + pub async fn get_value_or_guard_async<'a, Q>( + &'a self, + key: &Q, + ) -> Result> + where + Q: Hash + Equivalent + ToOwned + ?Sized, + { + if let Some(value) = self.get(key) { + return Ok(value); + } + match self.main.get_value_or_guard_async(key).await { + Ok(value) => Ok(value), + Err(guard) => match self.spill.get(key) { + Some(value) => Ok(value), + None => Err(SpillGuard { + guard, + key: key.to_owned(), + cache: self, + }), + }, + } + } + + pub fn update(&self, key: K, value: V) { + if self.main.is_oversized(&key, &value) { + self.main.remove(&key); + self.spill.update(key, value); + } else { + self.spill.remove(&key); + self.main.update(key, value); + } + } + + pub fn remove(&self, key: &Q) + where + Q: Hash + Equivalent + ?Sized, + { + self.main.remove(key); + self.spill.remove(key); + } + + pub fn clear(&self) { + self.main.clear(); + self.spill.clear(); + } + + pub fn weight(&self) -> u64 { + self.main.weight() + self.spill.weight() + } +} + +impl SpillGuard<'_, K, V> { + pub fn insert(self, value: V) { + if self.cache.main.is_oversized(&self.key, &value) { + self.cache.spill.insert(self.key, value); + drop(self.guard); + } else { + let _ = self.guard.insert(value); + } + } +} + +const MIN_ENTRIES_PER_SHARD: usize = 256; + fn cache_options( estimated_items_capacity: usize, weight_capacity: u64, shards: Option, ) -> Options { + let estimated_items_capacity = estimated_items_capacity.max(1); let mut builder = OptionsBuilder::new(); builder - .estimated_items_capacity(estimated_items_capacity.max(1)) + .estimated_items_capacity(estimated_items_capacity) .weight_capacity(weight_capacity); - if let Some(shards) = shards { - builder.shards(shards.max(1)); - } + builder.shards(match shards { + Some(shards) => shards.max(1), + None => { + let capped = default_shards().min(estimated_items_capacity / MIN_ENTRIES_PER_SHARD); + if capped <= 1 { + 1 + } else { + 1 << (usize::BITS - 1 - capped.leading_zeros()) + } + } + }); builder.build().unwrap() } -#[derive(Clone)] -pub struct CacheItemWeighter; +fn default_shards() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 4 +} + +#[derive(Debug, Clone, Copy)] +pub struct CacheItemWeighter { + entry_overhead: u64, +} + +impl CacheItemWeighter { + #[inline(always)] + fn charge(&self, key: &K, val: &V) -> u64 { + key.weight() + val.weight() + self.entry_overhead + } + + fn items(&self, weight: u64, estimated_weight: u64) -> usize { + (weight / (estimated_weight + self.entry_overhead).max(1)) as usize + } +} + +impl From for CacheItemWeighter { + fn from(charge: EntryCharge) -> Self { + CacheItemWeighter { + entry_overhead: match charge { + EntryCharge::Payload => 0, + EntryCharge::PayloadAndSlot => ENTRY_SLOT_SIZE, + }, + } + } +} impl Weighter for CacheItemWeighter { fn weight(&self, key: &K, val: &V) -> u64 { - key.weight() + val.weight() + self.charge(key, val) } } @@ -240,7 +592,7 @@ pub trait CacheItemWeight { impl CacheItemWeight for TtlEntry { fn weight(&self) -> u64 { - self.value.weight() + std::mem::size_of::() as u64 + self.value.weight() + std::mem::size_of::() as u64 } } @@ -291,7 +643,16 @@ impl CacheItemWeight for Arc<[T]> { impl CacheItemWeight for RecordSet { fn weight(&self) -> u64 { - self.rrset.weight() + std::mem::size_of::() as u64 + self.records.weight() + std::mem::size_of::() as u64 + } +} + +impl CacheItemWeight for Result, Negative> { + fn weight(&self) -> u64 { + match self { + Ok(records) => records.weight(), + Err(_) => std::mem::size_of::() as u64, + } } } @@ -319,19 +680,19 @@ impl CacheItemWeight for Ipv6Addr { } } -impl CacheItemWeight for MX { +impl CacheItemWeight for Mx { fn weight(&self) -> u64 { self.exchanges .iter() .map(|e| e.len() as u64 + std::mem::size_of::>() as u64) .sum::() - + std::mem::size_of::() as u64 + + std::mem::size_of::() as u64 } } -impl CacheItemWeight for Txt { +impl CacheItemWeight for TxtRecord { fn weight(&self) -> u64 { - std::mem::size_of::() as u64 + (std::mem::size_of::() + self.heap_size()) as u64 } } @@ -341,6 +702,17 @@ impl CacheItemWeight for bool { } } +impl CacheItemWeight for CompactString { + fn weight(&self) -> u64 { + let heap = if self.is_heap_allocated() { + self.capacity() as u64 + } else { + 0 + }; + heap + std::mem::size_of::() as u64 + } +} + impl CacheItemWeight for ArcStr { fn weight(&self) -> u64 { self.len() as u64 + std::mem::size_of::() as u64 @@ -354,14 +726,7 @@ impl CacheItemWeight for () { } impl TtlEntry { - pub fn new(value: T, expires: Duration) -> Self { - Self { - value, - expires: Instant::now() + expires, - } - } - - pub fn with_expiry(value: T, expires: Instant) -> Self { + fn new(value: T, expires: u32) -> Self { Self { value, expires } } } @@ -386,35 +751,8 @@ impl ResolverCache::new_single_shard(capacity, 1000); - assert_eq!(cache.inner().num_shards(), 1); - - let value = "x".repeat(9_000_000); - cache.insert(0, value.clone()); - assert_eq!(cache.get(&0), Some(value)); - } - - #[test] - fn sharded_cache_drops_entry_larger_than_a_shard() { - let capacity = 10_000_000u64; - let cache = Cache::::new_estimated(10_000, capacity); - - let value = "x".repeat((capacity / 2) as usize); - cache.insert(0, value); - - if cache.inner().num_shards() > 1 { - assert_eq!(cache.get(&0), None); - } - } -} diff --git a/crates/utils/src/chained_bytes.rs b/crates/utils/src/chained_bytes.rs index 0942f498..86defe22 100644 --- a/crates/utils/src/chained_bytes.rs +++ b/crates/utils/src/chained_bytes.rs @@ -4,153 +4,125 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use memchr::memchr_iter; use std::{borrow::Cow, ops::Range}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy, Default)] pub struct ChainedBytes<'x> { - first: &'x [u8], - last: &'x [u8], -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SliceRange<'x> { - Single(&'x [u8]), - Split(&'x [u8], &'x [u8]), - None, + head: &'x [u8], + tail: &'x [u8], } impl<'x> ChainedBytes<'x> { - pub fn new(first: &'x [u8]) -> Self { - Self { first, last: &[] } + pub const fn new(bytes: &'x [u8]) -> Self { + Self { + head: bytes, + tail: &[], + } } - pub fn append(&mut self, bytes: &'x [u8]) { - self.last = bytes; + pub const fn chain(head: &'x [u8], tail: &'x [u8]) -> Self { + Self { head, tail } } - pub fn with_last(mut self, bytes: &'x [u8]) -> Self { - self.last = bytes; - self + pub fn from_blob(head: &'x [u8], blob: &'x [u8], body_offset: usize) -> Self { + Self { + head, + tail: blob.get(body_offset..).unwrap_or_default(), + } } - pub fn get(&self, index: Range) -> Option> { - let start = index.start; - let end = index.end; + pub const fn len(&self) -> usize { + self.head.len() + self.tail.len() + } - if let Some(bytes) = self.first.get(start..end) { - Some(Cow::Borrowed(bytes)) - } else if start >= self.first.len() { - self.last - .get(start - self.first.len()..end - self.first.len()) + pub const fn is_empty(&self) -> bool { + self.head.is_empty() && self.tail.is_empty() + } + + pub const fn segments(&self) -> [&'x [u8]; 2] { + [self.head, self.tail] + } + + #[inline] + pub fn view(&self, range: Range) -> Option { + let split = self.head.len(); + if range.end <= split { + self.head.get(range).map(Self::new) + } else if range.start >= split { + self.tail + .get(range.start - split..range.end - split) + .map(Self::new) + } else { + Some(Self::chain( + self.head.get(range.start..)?, + self.tail.get(..range.end - split)?, + )) + } + } + + #[inline] + pub fn get(&self, range: Range) -> Option> { + let split = self.head.len(); + if range.end <= split { + self.head.get(range).map(Cow::Borrowed) + } else if range.start >= split { + self.tail + .get(range.start - split..range.end - split) .map(Cow::Borrowed) - } else if let (Some(first), Some(last)) = ( - self.first.get(start..), - self.last.get(..end - self.first.len()), - ) { - let mut vec = vec![0u8; first.len() + last.len()]; - vec[..first.len()].copy_from_slice(first); - vec[first.len()..].copy_from_slice(last); - Some(Cow::Owned(vec)) } else { - None + self.concat(range) } } - pub fn get_slice_range(&self, index: Range) -> SliceRange<'x> { - let start = index.start; - let end = index.end; - - if let Some(bytes) = self.first.get(start..end) { - SliceRange::Single(bytes) - } else if start >= self.first.len() { - self.last - .get(start - self.first.len()..end - self.first.len()) - .map(SliceRange::Single) - .unwrap_or(SliceRange::None) - } else if let (Some(first), Some(last)) = ( - self.first.get(start..), - self.last.get(..end - self.first.len()), - ) { - SliceRange::Split(first, last) - } else { - SliceRange::None - } + pub fn to_vec(&self) -> Vec { + self.segments().concat() } - pub fn get_full_range(&self) -> SliceRange<'x> { - if self.last.is_empty() { - SliceRange::Single(self.first) - } else { - SliceRange::Split(self.first, self.last) - } + #[inline] + pub fn extend_into(&self, buf: &mut Vec) { + buf.reserve(self.len()); + buf.extend_from_slice(self.head); + buf.extend_from_slice(self.tail); } - pub fn to_bytes(&self) -> Vec { - let mut bytes = vec![0u8; self.first.len() + self.last.len()]; - bytes[..self.first.len()].copy_from_slice(self.first); - bytes[self.first.len()..].copy_from_slice(self.last); - bytes + pub fn line_prefix_len(&self, lines: usize) -> usize { + let Some(nth) = lines.checked_sub(1) else { + return 0; + }; + let [head, tail] = self.segments(); + memchr_iter(b'\n', head) + .chain(memchr_iter(b'\n', tail).map(|pos| pos + head.len())) + .nth(nth) + .map_or(self.len(), |pos| pos + 1) } - pub fn len(&self) -> usize { - self.first.len() + self.last.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 + #[cold] + fn concat(&self, range: Range) -> Option> { + self.view(range).map(|bytes| Cow::Owned(bytes.to_vec())) } } -impl<'x> SliceRange<'x> { - pub fn len(&self) -> usize { - match self { - SliceRange::Single(bytes) => bytes.len(), - SliceRange::Split(first, last) => first.len() + last.len(), - SliceRange::None => 0, +impl PartialEq for ChainedBytes<'_> { + fn eq(&self, other: &Self) -> bool { + let (short, long) = if self.head.len() <= other.head.len() { + (self, other) + } else { + (other, self) + }; + if short.len() != long.len() { + return false; } - } - - pub fn try_into_bytes(self) -> Option> { - match self { - SliceRange::Single(bytes) => Some(Cow::Borrowed(bytes)), - SliceRange::Split(first, last) => { - let mut vec = vec![0u8; first.len() + last.len()]; - vec[..first.len()].copy_from_slice(first); - vec[first.len()..].copy_from_slice(last); - Some(Cow::Owned(vec)) - } - SliceRange::None => None, - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - fn into_pairs(self) -> (&'x [u8], &'x [u8]) { - match self { - SliceRange::Single(bytes) => (bytes, &[][..]), - SliceRange::Split(first, last) => (first, last), - SliceRange::None => (&[][..], &[][..]), - } - } - - pub fn is_none(&self) -> bool { - matches!(self, SliceRange::None) - } - - pub fn is_some(&self) -> bool { - !self.is_none() + let Some((long_head, long_middle)) = long.head.split_at_checked(short.head.len()) else { + return false; + }; + let Some((short_middle, short_tail)) = short.tail.split_at_checked(long_middle.len()) + else { + return false; + }; + short.head == long_head && short_middle == long_middle && short_tail == long.tail } } -impl<'x> IntoIterator for SliceRange<'x> { - type Item = &'x u8; - type IntoIter = std::iter::Chain, std::slice::Iter<'x, u8>>; +impl Eq for ChainedBytes<'_> {} - fn into_iter(self) -> Self::IntoIter { - let (first, last) = self.into_pairs(); - - first.iter().chain(last.iter()) - } -} diff --git a/crates/utils/src/cheeky_hash.rs b/crates/utils/src/cheeky_hash.rs index bac96117..701a1bd9 100644 --- a/crates/utils/src/cheeky_hash.rs +++ b/crates/utils/src/cheeky_hash.rs @@ -9,7 +9,6 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, hash::Hash, - str::FromStr, }; // A hash that can cheekily store small inputs directly without hashing them. @@ -17,10 +16,9 @@ use std::{ Copy, Clone, PartialEq, Eq, PartialOrd, Ord, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, )] #[repr(transparent)] -pub struct CheekyHash([u8; HASH_SIZE]); +pub struct CheekyHash([u8; HASH_SIZE + 1]); -const HASH_SIZE: usize = std::mem::size_of::() * 2; -const HASH_PAYLOAD: usize = HASH_SIZE - 1; +const HASH_SIZE: usize = std::mem::size_of::(); pub type CheekyHashSet = HashSet>; pub type CheekyHashMap = HashMap>; @@ -28,87 +26,60 @@ pub type CheekyBTreeMap = BTreeMap; impl CheekyHash { pub const HASH_SIZE: usize = HASH_SIZE; - pub const NULL: CheekyHash = CheekyHash([0u8; HASH_SIZE]); - pub const FULL: CheekyHash = CheekyHash([u8::MAX; HASH_SIZE]); + pub const EMPTY: CheekyHash = CheekyHash([0u8; HASH_SIZE + 1]); + pub const FULL: CheekyHash = CheekyHash([u8::MAX; HASH_SIZE + 1]); pub fn new(bytes: impl AsRef<[u8]>) -> Self { - let mut hash = [0u8; HASH_SIZE]; let bytes = bytes.as_ref(); + let mut hash = [0u8; HASH_SIZE + 1]; - if bytes.len() <= HASH_PAYLOAD { - hash[0] = bytes.len() as u8; - hash[1..1 + bytes.len()].copy_from_slice(bytes); + hash[HASH_SIZE] = bytes.len().min(u8::MAX as usize) as u8; + + if bytes.len() <= HASH_SIZE { + hash[..bytes.len()].copy_from_slice(bytes); } else { - let h1 = xxhash_rust::xxh3::xxh3_64(bytes).to_be_bytes(); - let h2 = farmhash::fingerprint64(bytes).to_be_bytes(); - hash[0] = bytes.len().min(u8::MAX as usize) as u8; - hash[1..1 + std::mem::size_of::()].copy_from_slice(&h1); - hash[1 + std::mem::size_of::()..] - .copy_from_slice(&h2[..std::mem::size_of::() - 1]); + let h1 = xxhash_rust::xxh3::xxh3_128(bytes).to_be_bytes(); + hash[..HASH_SIZE].copy_from_slice(&h1); } CheekyHash(hash) } - pub fn deserialize(bytes: &[u8]) -> Option { - let len = *bytes.first()?; - let mut hash = [0u8; HASH_SIZE]; - let hash_len = 1 + (len as usize).min(HASH_PAYLOAD); - - hash[0] = len; - hash[1..hash_len].copy_from_slice(bytes.get(1..hash_len)?); - Some(CheekyHash(hash)) - } - - #[allow(clippy::len_without_is_empty)] #[inline(always)] - pub fn len(&self) -> usize { - (self.0[0] as usize).min(HASH_PAYLOAD) + 1 + pub fn as_key(&self) -> &[u8] { + &self.0[..(self.0[HASH_SIZE] as usize).min(HASH_SIZE)] } #[inline(always)] - pub fn as_bytes(&self) -> &[u8] { - &self.0[..self.len()] + pub fn key_len(&self) -> usize { + (self.0[HASH_SIZE] as usize).min(HASH_SIZE) } #[inline(always)] - pub fn as_raw_bytes(&self) -> &[u8; HASH_SIZE] { - &self.0 + pub fn as_payload(&self) -> &[u8] { + &self.0[..HASH_SIZE] } - pub fn into_inner(self) -> [u8; HASH_SIZE] { - self.0 - } - - pub fn payload(&self) -> &[u8] { - let len = self.0[0] as usize; - if len <= HASH_PAYLOAD { - &self.0[1..1 + len] + pub fn from_key_bytes(key: &[u8], len: u8) -> Option { + if key.len() == (len as usize).min(HASH_SIZE) { + let mut hash = [0u8; HASH_SIZE + 1]; + hash[..key.len()].copy_from_slice(key); + hash[HASH_SIZE] = len; + Some(CheekyHash(hash)) } else { - &self.0[1..] + None } } - pub fn payload_len(&self) -> u8 { - self.0[0] + #[inline(always)] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.0[HASH_SIZE] as usize } + #[inline(always)] fn as_u128(&self) -> u128 { - u128::from_be_bytes(self.0) - } -} - -impl AsRef<[u8]> for CheekyHash { - fn as_ref(&self) -> &[u8] { - self.as_bytes() - } -} - -impl FromStr for CheekyHash { - type Err = std::num::ParseIntError; - - fn from_str(s: &str) -> Result { - u128::from_str_radix(s, 16).map(|n| CheekyHash(n.to_be_bytes())) + u128::from_be_bytes(self.as_payload().try_into().unwrap()) } } @@ -120,14 +91,12 @@ impl std::fmt::Display for CheekyHash { impl Hash for CheekyHash { fn hash(&self, state: &mut H) { - let len = self.0[0] as usize; - if len <= HASH_PAYLOAD { - state.write_u64(xxhash_rust::xxh3::xxh3_64(&self.0[1..1 + len])); + let len = self.0[HASH_SIZE] as usize; + if len <= HASH_SIZE { + state.write_u64(xxhash_rust::xxh3::xxh3_64(&self.0[..len])); } else { state.write_u64(u64::from_be_bytes( - self.0[1..1 + std::mem::size_of::()] - .try_into() - .unwrap(), + self.0[..std::mem::size_of::()].try_into().unwrap(), )); } } @@ -135,30 +104,12 @@ impl Hash for CheekyHash { impl IsEnabled for CheekyHash {} -impl ArchivedCheekyHash { - #[inline(always)] - pub fn as_raw_bytes(&self) -> &[u8; HASH_SIZE] { - &self.0 - } - - #[inline(always)] - pub fn as_bytes(&self) -> &[u8] { - let len = self.0[0] as usize; - &self.0[..1 + len.min(HASH_PAYLOAD)] - } - - #[inline(always)] - pub fn to_native(&self) -> CheekyHash { - CheekyHash(self.0) - } -} - impl Debug for CheekyHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let len = self.payload_len(); - let payload = self.payload(); - let payload_str = if len <= HASH_PAYLOAD as u8 { - std::str::from_utf8(payload).unwrap_or("") + let len = self.0[HASH_SIZE] as usize; + let payload = self.as_payload(); + let payload_str = if len <= HASH_SIZE { + std::str::from_utf8(&payload[..len]).unwrap_or("") } else { "" }; @@ -170,150 +121,3 @@ impl Debug for CheekyHash { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_cheeky_hash_all() { - // Test 1: Empty input - let hash_empty = CheekyHash::new([]); - assert_eq!( - hash_empty.as_bytes()[0], - 0, - "Empty input should have length 0" - ); - assert_eq!( - hash_empty.as_bytes().len(), - 1, - "Empty input should only have length byte" - ); - - // Test 2: Single byte input - let hash_single = CheekyHash::new([42]); - assert_eq!( - hash_single.as_bytes()[0], - 1, - "Single byte should have length 1" - ); - assert_eq!( - hash_single.as_bytes()[1], - 42, - "Single byte value should be preserved" - ); - assert_eq!(hash_single.as_bytes().len(), 2); - - // Test 3: Small input (less than HASH_LEN) - let small_data = b"hello"; - let hash_small = CheekyHash::new(small_data); - assert_eq!(hash_small.as_bytes()[0], 5, "Length should be 5"); - assert_eq!( - &hash_small.as_bytes()[1..6], - small_data, - "Small data should be stored directly" - ); - assert_eq!(hash_small.as_bytes().len(), 6); - - // Test 4: Input exactly at HASH_PAYLOAD boundary - let boundary_data = vec![1u8; HASH_PAYLOAD - 1]; - let hash_boundary = CheekyHash::new(&boundary_data); - assert_eq!( - hash_boundary.as_bytes()[0], - (HASH_PAYLOAD - 1) as u8, - "Length should be HASH_LEN" - ); - assert_eq!( - &hash_boundary.as_bytes()[1..], - &boundary_data[..], - "Boundary data should be stored directly" - ); - - // Test 5: Large input (greater than HASH_LEN) - uses hashing - let large_data = vec![7u8; HASH_SIZE]; - let hash_large = CheekyHash::new(&large_data); - assert_eq!( - hash_large.as_bytes()[0], - HASH_SIZE as u8, - "Large data should have length byte set to HASH_LEN" - ); - assert_eq!( - hash_large.as_bytes().len(), - HASH_SIZE, - "Large data hash should be full length" - ); - // Verify it's actually hashed (not raw data) - assert_ne!( - &hash_large.as_bytes()[1..], - &large_data[..HASH_PAYLOAD], - "Large data should be hashed, not stored directly" - ); - - // Test 6: AsRef<[u8]> trait - let hash = CheekyHash::new(b"test"); - let bytes_ref: &[u8] = hash.as_ref(); - assert_eq!(bytes_ref, hash.as_bytes(), "AsRef should match as_bytes"); - - // Test 7: Copy, Clone, PartialEq traits - let hash1 = CheekyHash::new(b"identical"); - let hash2 = hash1; // Copy - assert_eq!(hash1, hash2, "Copied hashes should be equal"); - - // Test 8: Different inputs produce different hashes - let hash_a = CheekyHash::new(b"abc"); - let hash_b = CheekyHash::new(b"def"); - assert_ne!( - hash_a, hash_b, - "Different inputs should produce different hashes" - ); - - // Test 9: Same input produces same hash (deterministic) - let hash_x1 = CheekyHash::new(b"deterministic"); - let hash_x2 = CheekyHash::new(b"deterministic"); - assert_eq!( - hash_x1, hash_x2, - "Same input should produce identical hashes" - ); - - // Test 10: Large inputs with different content produce different hashes - let large1 = vec![1u8; 100]; - let large2 = vec![2u8; 100]; - let hash_large1 = CheekyHash::new(&large1); - let hash_large2 = CheekyHash::new(&large2); - assert_ne!( - hash_large1, hash_large2, - "Different large inputs should produce different hashes" - ); - - // Test 11: Hash trait (can be used in HashMap/HashSet) - use std::collections::HashMap; - let mut map = HashMap::new(); - let key = CheekyHash::new(b"key"); - map.insert(key, "value"); - assert_eq!( - map.get(&key), - Some(&"value"), - "CheekyHash should work as HashMap key" - ); - - // Test 12: Debug trait - let hash = CheekyHash::new(b"debug"); - let debug_str = format!("{:?}", hash); - assert!( - debug_str.contains("CheekyHash"), - "Debug output should contain type name" - ); - - // Test 13: CheekyHashSet and CheekyHashMap - let mut cheeky_set: CheekyHashSet = CheekyHashSet::default(); - cheeky_set.insert(CheekyHash::new(b"set_item")); - assert!(cheeky_set.contains(&CheekyHash::new(b"set_item"))); - let mut cheeky_map: CheekyHashMap<&str> = CheekyHashMap::default(); - cheeky_map.insert(CheekyHash::new(b"map_key"), "map_value"); - assert_eq!( - cheeky_map.get(&CheekyHash::new(b"map_key")), - Some(&"map_value") - ); - - println!("All CheekyHash tests passed!"); - } -} diff --git a/crates/utils/src/codec/base32_custom.rs b/crates/utils/src/codec/base32_custom.rs deleted file mode 100644 index ce19d36a..00000000 --- a/crates/utils/src/codec/base32_custom.rs +++ /dev/null @@ -1,221 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::leb128::{Leb128Iterator, Leb128Writer}; -use std::{io::Write, slice::Iter}; - -pub static BASE32_ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz792013"; -pub static BASE32_INVERSE: [u8; 256] = [ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 29, 30, 28, 31, 255, 255, 255, 26, 255, 27, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 10, 11, - 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 0, 1, 2, - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, -]; - -pub struct Base32Writer { - last_byte: u8, - pos: usize, - result: String, -} - -impl Base32Writer { - pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { - let bytes = bytes.as_ref(); - let mut writer = Base32Writer::with_capacity(bytes.len()); - writer.write_all(bytes).unwrap(); - writer - } - - pub fn with_capacity(capacity: usize) -> Self { - Self::with_raw_capacity(capacity.div_ceil(4) * 5) - } - - pub fn with_raw_capacity(capacity: usize) -> Self { - Base32Writer { - result: String::with_capacity(capacity), - last_byte: 0, - pos: 0, - } - } - - pub fn push_char(&mut self, ch: char) { - self.result.push(ch); - } - - pub fn push_string(&mut self, string: &str) { - self.result.push_str(string); - } - - fn push_byte(&mut self, byte: u8, is_remainder: bool) { - let (ch1, ch2) = match self.pos % 5 { - 0 => ((byte & 0xF8) >> 3, u8::MAX), - 1 => ( - (((self.last_byte & 0x07) << 2) | ((byte & 0xC0) >> 6)), - ((byte & 0x3E) >> 1), - ), - 2 => ( - (((self.last_byte & 0x01) << 4) | ((byte & 0xF0) >> 4)), - u8::MAX, - ), - 3 => ( - (((self.last_byte & 0x0F) << 1) | (byte >> 7)), - ((byte & 0x7C) >> 2), - ), - 4 => ( - (((self.last_byte & 0x03) << 3) | ((byte & 0xE0) >> 5)), - (byte & 0x1F), - ), - _ => unreachable!(), - }; - - self.result.push(char::from(BASE32_ALPHABET[ch1 as usize])); - if !is_remainder { - if ch2 != u8::MAX { - self.result.push(char::from(BASE32_ALPHABET[ch2 as usize])); - } - self.last_byte = byte; - self.pos += 1; - } - } - - pub fn finalize(mut self) -> String { - if !self.pos.is_multiple_of(5) { - self.push_byte(0, true); - } - - self.result - } -} - -impl std::io::Write for Base32Writer { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - let start_pos = self.pos; - - for &byte in bytes { - self.push_byte(byte, false); - } - - Ok(self.pos - start_pos) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -#[derive(Debug)] -pub struct Base32Reader<'x> { - bytes: Iter<'x, u8>, - last_byte: u8, - pos: usize, -} - -impl<'x> Base32Reader<'x> { - pub fn new(bytes: &'x [u8]) -> Self { - Base32Reader { - bytes: bytes.iter(), - pos: 0, - last_byte: 0, - } - } - - #[allow(clippy::should_implement_trait)] - pub fn from_iter(bytes: Iter<'x, u8>) -> Self { - Base32Reader { - bytes, - pos: 0, - last_byte: 0, - } - } - - #[inline(always)] - fn map_byte(&mut self) -> Option { - match self.bytes.next() { - Some(&byte) => match BASE32_INVERSE[byte as usize] { - byte if byte != u8::MAX => { - self.last_byte = byte; - Some(byte) - } - _ => None, - }, - _ => None, - } - } -} - -impl Iterator for Base32Reader<'_> { - type Item = u8; - fn next(&mut self) -> Option { - let pos = self.pos % 5; - let last_byte = self.last_byte; - let byte = self.map_byte()?; - self.pos += 1; - - match pos { - 0 => ((byte << 3) | (self.map_byte().unwrap_or(0) >> 2)).into(), - 1 => ((last_byte << 6) | (byte << 1) | (self.map_byte().unwrap_or(0) >> 4)).into(), - 2 => ((last_byte << 4) | (byte >> 1)).into(), - 3 => ((last_byte << 7) | (byte << 2) | (self.map_byte().unwrap_or(0) >> 3)).into(), - 4 => ((last_byte << 5) | byte).into(), - _ => None, - } - } -} - -impl Leb128Iterator for Base32Reader<'_> {} -impl Leb128Writer for Base32Writer {} - -#[cfg(test)] -mod tests { - use std::io::Write; - - use crate::codec::base32_custom::{Base32Reader, Base32Writer}; - - #[test] - fn base32_roundtrip() { - let mut bytes = Vec::with_capacity(100); - for byte in 0..100 { - bytes.push((100 - byte) as u8); - let mut writer = Base32Writer::with_capacity(10); - writer.write_all(&bytes).unwrap(); - let result = writer.finalize(); - - let mut bytes_result = Vec::new(); - for byte in Base32Reader::new(result.as_bytes()) { - bytes_result.push(byte); - } - - assert_eq!(bytes, bytes_result); - } - - for bytes in [ - vec![0], - vec![32, 43, 55, 99, 43, 55], - vec![84, 4, 43, 77, 62, 55, 92], - vec![84, 4, 43, 77, 62, 55, 92], - ] { - let mut writer = Base32Writer::with_capacity(10); - writer.write_all(&bytes).unwrap(); - let result = writer.finalize(); - - let mut bytes_result = Vec::new(); - for byte in Base32Reader::new(result.as_bytes()) { - bytes_result.push(byte); - } - - assert_eq!(bytes, bytes_result); - } - } -} diff --git a/crates/utils/src/codec/leb128.rs b/crates/utils/src/codec/leb128.rs deleted file mode 100644 index 078563d7..00000000 --- a/crates/utils/src/codec/leb128.rs +++ /dev/null @@ -1,189 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -#![allow(dead_code)] - -use std::{borrow::Borrow, io::Write}; - -pub trait Leb128_ { - fn to_leb128_writer(self, out: &mut impl Write) -> std::io::Result; - fn to_leb128_bytes(self, out: &mut Vec); - fn from_leb128_bytes_pos(slice: &[u8]) -> Option<(Self, usize)> - where - Self: std::marker::Sized; - fn from_leb128_bytes(slice: &[u8]) -> Option - where - Self: std::marker::Sized; - fn from_leb128_it(it: T) -> Option - where - Self: std::marker::Sized, - T: Iterator, - I: Borrow; -} - -pub trait Leb128Vec { - fn push_leb128(&mut self, value: T); -} - -pub trait Leb128Writer: Write + Sized { - #[inline(always)] - fn write_leb128(&mut self, value: T) -> std::io::Result { - T::to_leb128_writer(value, self) - } -} - -impl Leb128Vec for Vec { - #[inline(always)] - fn push_leb128(&mut self, value: T) { - T::to_leb128_bytes(value, self); - } -} - -pub trait Leb128Iterator: Iterator -where - I: Borrow, -{ - #[inline(always)] - fn next_leb128(&mut self) -> Option { - T::from_leb128_it(self) - } - - #[inline(always)] - fn skip_leb128(&mut self) -> Option<()> { - for byte in self { - if (byte.borrow() & 0x80) == 0 { - return Some(()); - } - } - None - } -} - -pub trait Leb128Reader: AsRef<[u8]> { - #[inline(always)] - fn read_leb128(&self) -> Option<(T, usize)> { - T::from_leb128_bytes_pos(self.as_ref()) - } - - #[inline(always)] - fn skip_leb128(&self) -> Option { - for (pos, byte) in self.as_ref().iter().enumerate() { - if (byte & 0x80) == 0 { - return (pos + 1).into(); - } - } - None - } -} - -impl Leb128Reader for &[u8] {} -impl Leb128Reader for Vec {} -impl Leb128Reader for Box<[u8]> {} -impl<'x> Leb128Iterator<&'x u8> for std::slice::Iter<'x, u8> {} - -// Based on leb128.rs from rustc -macro_rules! impl_unsigned_leb128 { - ($int_ty:ident, $shifts:expr) => { - impl Leb128_ for $int_ty { - #[inline(always)] - fn to_leb128_writer(self, out: &mut impl Write) -> std::io::Result { - let mut value = self; - let mut bytes_written = 0; - loop { - if value < 0x80 { - bytes_written += out.write(&[value as u8])?; - break; - } else { - bytes_written += out.write(&[((value & 0x7f) | 0x80) as u8])?; - value >>= 7; - } - } - Ok(bytes_written) - } - - #[inline(always)] - fn to_leb128_bytes(self, out: &mut Vec) { - let mut value = self; - loop { - if value < 0x80 { - out.push(value as u8); - break; - } else { - out.push(((value & 0x7f) | 0x80) as u8); - value >>= 7; - } - } - } - - #[inline(always)] - fn from_leb128_bytes_pos(slice: &[u8]) -> Option<($int_ty, usize)> { - let mut result = 0; - - for (shift, (pos, &byte)) in $shifts.into_iter().zip(slice.iter().enumerate()) { - if (byte & 0x80) == 0 { - result |= (byte as $int_ty) << shift; - return Some((result, pos + 1)); - } else { - result |= ((byte & 0x7F) as $int_ty) << shift; - } - } - - None - } - - #[inline(always)] - fn from_leb128_bytes(slice: &[u8]) -> Option<$int_ty> { - let mut result = 0; - - for (shift, &byte) in $shifts.into_iter().zip(slice.iter()) { - if (byte & 0x80) == 0 { - result |= (byte as $int_ty) << shift; - return Some(result); - } else { - result |= ((byte & 0x7F) as $int_ty) << shift; - } - } - - None - } - - #[inline(always)] - fn from_leb128_it(it: T) -> Option<$int_ty> - where - T: Iterator, - I: Borrow, - { - let mut result = 0; - - for (shift, byte_) in $shifts.into_iter().zip(it) { - let byte = byte_.borrow(); - - if (byte & 0x80) == 0 { - result |= (*byte as $int_ty) << shift; - return Some(result); - } else { - result |= ((byte & 0x7F) as $int_ty) << shift; - } - } - - None - } - } - }; -} - -impl_unsigned_leb128!(u8, [0]); -impl_unsigned_leb128!(u16, [0, 7, 14]); -impl_unsigned_leb128!(u32, [0, 7, 14, 21, 28]); -impl_unsigned_leb128!(u64, [0, 7, 14, 21, 28, 35, 42, 49, 56, 63]); -impl_unsigned_leb128!(usize, [0, 7, 14, 21, 28, 35, 42, 49, 56, 63]); - -impl Leb128Writer for Vec { - #[inline(always)] - fn write_leb128(&mut self, value: T) -> std::io::Result { - T::to_leb128_writer(value, self) - } -} diff --git a/crates/utils/src/codec/leb128/kernel.rs b/crates/utils/src/codec/leb128/kernel.rs new file mode 100644 index 00000000..1595a86c --- /dev/null +++ b/crates/utils/src/codec/leb128/kernel.rs @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +const MSB: u64 = 0x8080_8080_8080_8080; +const LOW7: u64 = 0x7F7F_7F7F_7F7F_7F7F; +const SHORT_LIMIT: u64 = 1 << 56; + +#[inline(always)] +const fn spread(value: u64) -> u64 { + let value = (value & 0x0000_0000_0FFF_FFFF) | ((value & 0x00FF_FFFF_F000_0000) << 4); + let value = (value & 0x0000_3FFF_0000_3FFF) | ((value & 0x0FFF_C000_0FFF_C000) << 2); + (value & 0x007F_007F_007F_007F) | ((value & 0x3F80_3F80_3F80_3F80) << 1) +} + +#[inline(always)] +const fn compact(value: u64) -> u64 { + let value = (value & 0x007F_007F_007F_007F) | ((value & 0x7F00_7F00_7F00_7F00) >> 1); + let value = (value & 0x0000_3FFF_0000_3FFF) | ((value & 0x3FFF_0000_3FFF_0000) >> 2); + (value & 0x0000_0000_0FFF_FFFF) | ((value & 0x0FFF_FFFF_0000_0000) >> 4) +} + +#[inline(always)] +pub(super) const fn encoded_len(value: u64) -> usize { + let bits = u64::BITS - (value | 1).leading_zeros(); + (bits as usize * 9 + 64) / 64 +} + +#[inline(always)] +pub(super) const fn encode(value: u64) -> (u64, u64, usize) { + if value < SHORT_LIMIT { + let spread = spread(value); + let top = (u64::BITS - 1 - (spread | 1).leading_zeros()) & !7; + ( + spread | (MSB & ((1 << top) - 1)), + 0, + (top as usize >> 3) + 1, + ) + } else { + let high = value >> 56; + let extra = high >> 7; + (spread(value) | MSB, high | (extra << 8), 9 + extra as usize) + } +} + +#[inline(always)] +pub(super) fn push(out: &mut Vec, mut value: u64) { + loop { + if value < 0x80 { + out.push(value as u8); + break; + } + out.push((value & 0x7F) as u8 | 0x80); + value >>= 7; + } +} + +#[inline(always)] +pub(super) fn decode(bytes: &[u8], max_len: usize) -> Option<(u64, usize)> { + let mut value = 0u64; + for (index, &byte) in bytes.iter().take(max_len).enumerate() { + let bits = u64::from(byte & 0x7F); + let shift = 7 * index as u32; + if (bits << shift) >> shift != bits { + return None; + } + value |= bits << shift; + if byte < 0x80 { + return Some((value, index + 1)); + } + } + None +} + +#[inline(always)] +pub(super) fn decode_wide(bytes: &[u8], max_len: usize) -> Option<(u64, usize)> { + match bytes.first_chunk::<8>() { + Some(chunk) if u64::from_le_bytes(*chunk) & MSB == MSB => { + decode_long(u64::from_le_bytes(*chunk), bytes) + } + _ => decode(bytes, max_len), + } +} + +#[inline] +fn decode_long(word: u64, bytes: &[u8]) -> Option<(u64, usize)> { + let &ninth = bytes.get(8)?; + let value = compact(word & LOW7) | (u64::from(ninth & 0x7F) << 56); + if ninth < 0x80 { + Some((value, 9)) + } else { + let &tenth = bytes.get(9)?; + (tenth <= 1).then_some((value | (u64::from(tenth) << 63), 10)) + } +} + +#[inline] +pub(super) fn skip(bytes: &[u8], mut count: usize) -> Option { + let mut offset = 0; + while count > 0 { + let rest = bytes.get(offset..)?; + if let Some(chunk) = rest.first_chunk::<8>() { + let mut stops = !u64::from_le_bytes(*chunk) & MSB; + let found = stops.count_ones() as usize; + if found < count { + offset += 8; + count -= found; + } else { + for _ in 1..count { + stops &= stops - 1; + } + return Some(offset + (stops.trailing_zeros() as usize >> 3) + 1); + } + } else { + for (index, &byte) in rest.iter().enumerate() { + if byte < 0x80 { + count -= 1; + if count == 0 { + return Some(offset + index + 1); + } + } + } + return None; + } + } + Some(offset) +} diff --git a/crates/utils/src/codec/leb128/mod.rs b/crates/utils/src/codec/leb128/mod.rs new file mode 100644 index 00000000..895edfc4 --- /dev/null +++ b/crates/utils/src/codec/leb128/mod.rs @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod kernel; + +use std::ops::Deref; + +mod sealed { + pub trait Sealed: Copy { + const MAX_LEN: usize; + + fn widen(self) -> u64; + fn narrow(value: u64) -> Option; + } +} + +use sealed::Sealed; + +pub trait Leb128: Sealed { + #[inline(always)] + fn leb128_len(self) -> usize { + kernel::encoded_len(self.widen()) + } + + #[inline(always)] + fn to_leb128(self) -> Leb128Bytes { + Leb128Bytes::new(self.widen()) + } + + #[inline(always)] + fn from_leb128(bytes: &[u8]) -> Option<(Self, usize)> { + let (value, len) = kernel::decode(bytes, Self::MAX_LEN)?; + Self::narrow(value).map(|value| (value, len)) + } + + #[inline(always)] + fn from_leb128_wide(bytes: &[u8]) -> Option<(Self, usize)> { + let (value, len) = kernel::decode_wide(bytes, Self::MAX_LEN)?; + Self::narrow(value).map(|value| (value, len)) + } +} + +pub trait Leb128Writer { + fn push_leb128(&mut self, value: T); +} + +pub trait Leb128Reader { + fn read_leb128(&mut self) -> Option; + fn read_leb128_wide(&mut self) -> Option; + fn skip_leb128(&mut self, count: usize) -> Option<()>; +} + +pub trait Leb128Iterator: Iterator { + #[inline] + fn next_leb128(&mut self) -> Option { + let mut value = 0u64; + for (shift, byte) in (0..T::MAX_LEN).map(|index| index * 7).zip(self) { + let bits = u64::from(byte & 0x7F); + if (bits << shift) >> shift != bits { + return None; + } + value |= bits << shift; + if byte < 0x80 { + return T::narrow(value); + } + } + None + } +} + +impl> Leb128Iterator for I {} + +impl Leb128Writer for Vec { + #[inline(always)] + fn push_leb128(&mut self, value: T) { + kernel::push(self, value.widen()); + } +} + +impl Leb128Reader for &[u8] { + #[inline(always)] + fn read_leb128(&mut self) -> Option { + let (value, len) = T::from_leb128(self)?; + *self = self.get(len..)?; + Some(value) + } + + #[inline(always)] + fn read_leb128_wide(&mut self) -> Option { + let (value, len) = T::from_leb128_wide(self)?; + *self = self.get(len..)?; + Some(value) + } + + #[inline] + fn skip_leb128(&mut self, count: usize) -> Option<()> { + *self = self.get(kernel::skip(self, count)?..)?; + Some(()) + } +} + +#[derive(Clone, Copy)] +pub struct Leb128Bytes { + bytes: [u8; 16], + len: usize, +} + +impl Leb128Bytes { + #[inline(always)] + fn new(value: u64) -> Self { + let (low, high, len) = kernel::encode(value); + Leb128Bytes { + bytes: ((u128::from(high) << 64) | u128::from(low)).to_le_bytes(), + len, + } + } +} + +impl Deref for Leb128Bytes { + type Target = [u8]; + + #[inline(always)] + fn deref(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + +impl AsRef<[u8]> for Leb128Bytes { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + self + } +} + +macro_rules! impl_leb128 { + ($($ty:ty),+) => {$( + impl Sealed for $ty { + const MAX_LEN: usize = <$ty>::BITS.div_ceil(7) as usize; + + #[inline(always)] + fn widen(self) -> u64 { + self as u64 + } + + #[inline(always)] + fn narrow(value: u64) -> Option { + <$ty>::try_from(value).ok() + } + } + + impl Leb128 for $ty {} + )+}; +} + +impl_leb128!(u8, u16, u32, u64, usize); diff --git a/crates/utils/src/codec/mod.rs b/crates/utils/src/codec/mod.rs index b693447e..90683b6d 100644 --- a/crates/utils/src/codec/mod.rs +++ b/crates/utils/src/codec/mod.rs @@ -4,5 +4,4 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod base32_custom; pub mod leb128; diff --git a/crates/utils/src/cron.rs b/crates/utils/src/cron.rs index b9fc94e4..832a2e2a 100644 --- a/crates/utils/src/cron.rs +++ b/crates/utils/src/cron.rs @@ -4,8 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use chrono::{Datelike, Local, TimeDelta, TimeZone, Timelike}; -use std::{str::FromStr, time::Duration}; +use jiff::{ + SignedDuration, Span, ToSpan, Zoned, + civil::{Date, Time}, +}; +use std::time::Duration; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SimpleCron { @@ -16,47 +19,61 @@ pub enum SimpleCron { impl SimpleCron { pub fn time_to_next(&self) -> Duration { - let now = Local::now(); - let next = match self { - SimpleCron::Day { hour, minute } => { - let next = Local - .with_ymd_and_hms(now.year(), now.month(), now.day(), *hour, *minute, 0) - .earliest() - .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); - if next <= now { - next + TimeDelta::try_days(1).unwrap_or_default() - } else { - next - } - } - SimpleCron::Week { day, hour, minute } => { - let next = Local - .with_ymd_and_hms(now.year(), now.month(), now.day(), *hour, *minute, 0) - .earliest() - .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); - if next <= now { - next + TimeDelta::try_days( - (7 - now.weekday().number_from_monday() + *day).into(), - ) - .unwrap_or_default() - } else { - next - } - } - SimpleCron::Hour { minute } => { - let next = Local - .with_ymd_and_hms(now.year(), now.month(), now.day(), now.hour(), *minute, 0) - .earliest() - .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); - if next <= now { - next + TimeDelta::try_hours(1).unwrap_or_default() - } else { - next - } - } - }; + let now = Zoned::now(); + self.next_after(&now) + .and_then(|next| Duration::try_from(now.duration_until(&next)).ok()) + .unwrap_or_else(|| self.as_duration()) + } - (next - now).to_std().unwrap_or_else(|_| self.as_duration()) + fn next_after(&self, now: &Zoned) -> Option { + match *self { + SimpleCron::Hour { minute } => { + let next = now + .with() + .minute(i8::try_from(minute).ok()?) + .second(0) + .subsec_nanosecond(0) + .build() + .ok()?; + if next > *now { + Some(next) + } else { + next.checked_add(SignedDuration::from_hours(1)).ok() + } + } + SimpleCron::Day { hour, minute } => { + Self::next_local_time(now, 0, 1.day(), hour, minute) + } + SimpleCron::Week { day, hour, minute } => Self::next_local_time( + now, + (i64::from(day) - i64::from(now.weekday().to_sunday_zero_offset())).rem_euclid(7), + 1.week(), + hour, + minute, + ), + } + } + + fn next_local_time( + now: &Zoned, + days_ahead: i64, + period: Span, + hour: u32, + minute: u32, + ) -> Option { + let time = Time::new(i8::try_from(hour).ok()?, i8::try_from(minute).ok()?, 0, 0).ok()?; + let date = now.date().checked_add(days_ahead.days()).ok()?; + let at = |date: Date| { + date.to_datetime(time) + .to_zoned(now.time_zone().clone()) + .ok() + }; + let next = at(date)?; + if next > *now { + Some(next) + } else { + at(date.checked_add(period).ok()?) + } } pub fn as_duration(&self) -> Duration { @@ -68,71 +85,9 @@ impl SimpleCron { } } -impl FromStr for SimpleCron { - type Err = String; - - fn from_str(value: &str) -> Result { - let mut hour = 0; - let mut minute = 0; - - for (pos, value) in value.split(' ').enumerate() { - if pos == 0 { - minute = value - .parse::() - .map_err(|_| "Invalid cron key: failed to parse cron minute".to_string())?; - if !(0..=59).contains(&minute) { - return Err(format!( - "Invalid cron key: failed to parse minute, invalid value: {minute}" - )); - } - } else if pos == 1 { - if value - .as_bytes() - .first() - .ok_or_else(|| "Invalid cron key: failed to parse cron hour".to_string())? - == &b'*' - { - return Ok(SimpleCron::Hour { minute }); - } else { - hour = value - .parse::() - .map_err(|_| "Invalid cron key: failed to parse cron hour".to_string())?; - if !(0..=23).contains(&hour) { - return Err(format!( - "Invalid cron key: failed to parse hour, invalid value: {hour}" - )); - } - } - } else if pos == 2 { - if value - .as_bytes() - .first() - .ok_or_else(|| "Invalid cron key: failed to parse cron weekday".to_string())? - == &b'*' - { - return Ok(SimpleCron::Day { hour, minute }); - } else { - let day = value.parse::().map_err(|_| { - "Invalid cron key: failed to parse cron weekday".to_string() - })?; - if !(1..=7).contains(&hour) { - return Err(format!( - "Invalid cron key: failed to parse weekday, invalid value: {}, range is 1 (Monday) to 7 (Sunday).", - hour, - )); - } - - return Ok(SimpleCron::Week { day, hour, minute }); - } - } - } - - Err("Invalid cron key: parse cron expression.".to_string()) - } -} - impl Default for SimpleCron { fn default() -> Self { SimpleCron::Hour { minute: 0 } } } + diff --git a/crates/utils/src/glob.rs b/crates/utils/src/glob.rs index 43beb9d3..ce167116 100644 --- a/crates/utils/src/glob.rs +++ b/crates/utils/src/glob.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::text::{ConstNeedle, IgnoreCaseNeedle, lowercase}; use ahash::{AHashMap, AHashSet}; use serde::Deserialize; use std::borrow::Cow; @@ -17,10 +18,17 @@ pub enum MatchType { All, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct GlobPattern { pattern: Vec, to_lower: bool, + literal: Option>, +} + +#[derive(Debug, Clone)] +enum Literal { + Contains(ConstNeedle), + ContainsIgnoreCase(IgnoreCaseNeedle), } #[derive(Debug, Clone, PartialEq, Eq)] @@ -69,6 +77,7 @@ impl GlobPattern { } GlobPattern { + literal: Literal::classify(&chars, to_lower).map(Box::new), pattern: chars, to_lower, } @@ -104,40 +113,47 @@ impl GlobPattern { } } - // Credits: Algorithm ported from https://research.swtch.com/glob pub fn matches(&self, value: &str) -> bool { - let value = if self.to_lower { - value.to_lowercase().chars().collect::>() - } else { - value.chars().collect::>() - }; + match self.literal.as_deref() { + Some(Literal::Contains(needle)) => needle.contains(value), + Some(Literal::ContainsIgnoreCase(needle)) => needle.contains(value), + None if !self.to_lower => self.matches_chars(value, identity), + None if value.is_ascii() => self.matches_chars(value, char::to_ascii_lowercase), + None => self.matches_chars(&lowercase(value), identity), + } + } + // Credits: Algorithm ported from https://research.swtch.com/glob + fn matches_chars(&self, value: &str, fold: impl Fn(&char) -> char) -> bool { let mut px = 0; let mut nx = 0; let mut next_px = 0; let mut next_nx = 0; while px < self.pattern.len() || nx < value.len() { + let next = value.get(nx..).and_then(|rest| rest.chars().next()); match self.pattern.get(px) { Some(PatternChar::Char { char, .. }) => { - if matches!(value.get(nx), Some(nc) if nc == char ) { + if let Some(next) = next.filter(|next| fold(next) == *char) { px += 1; - nx += 1; + nx += next.len_utf8(); continue; } } - Some(PatternChar::WildcardSingle { .. }) if nx < value.len() => { - px += 1; - nx += 1; - continue; + Some(PatternChar::WildcardSingle { .. }) => { + if let Some(next) = next { + px += 1; + nx += next.len_utf8(); + continue; + } } Some(PatternChar::WildcardMany { .. }) => { next_px = px; - next_nx = nx + 1; + next_nx = nx + next.map_or(1, char::len_utf8); px += 1; continue; } - _ => (), + None => (), } if 0 < next_nx && next_nx <= value.len() { px = next_px; @@ -150,6 +166,45 @@ impl GlobPattern { } } +impl Literal { + fn classify(pattern: &[PatternChar], to_lower: bool) -> Option { + let [ + PatternChar::WildcardMany { .. }, + middle @ .., + PatternChar::WildcardMany { .. }, + ] = pattern + else { + return None; + }; + let literal = middle + .iter() + .map(|item| match item { + PatternChar::Char { char, .. } => Some(*char), + _ => None, + }) + .collect::>()?; + if !to_lower { + Some(Literal::Contains(ConstNeedle::new(&literal))) + } else if literal.is_ascii() { + Some(Literal::ContainsIgnoreCase(IgnoreCaseNeedle::new(&literal))) + } else { + None + } + } +} + +impl PartialEq for GlobPattern { + fn eq(&self, other: &Self) -> bool { + self.pattern == other.pattern && self.to_lower == other.to_lower + } +} + +impl Eq for GlobPattern {} + +fn identity(char: &char) -> char { + *char +} + #[derive(Debug, Clone, Default)] pub struct GlobSet { entries: AHashSet, @@ -262,3 +317,4 @@ impl<'de> Deserialize<'de> for GlobPattern { )) } } + diff --git a/crates/utils/src/hash128.rs b/crates/utils/src/hash128.rs new file mode 100644 index 00000000..9c77c413 --- /dev/null +++ b/crates/utils/src/hash128.rs @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{fmt::Debug, num::ParseIntError, str::FromStr}; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[repr(transparent)] +pub struct Hash128([u8; Hash128::LEN]); + +impl Hash128 { + pub const LEN: usize = std::mem::size_of::(); + + #[inline(always)] + pub fn from_be_bytes(bytes: [u8; Hash128::LEN]) -> Self { + Hash128(bytes) + } + + #[inline(always)] + pub fn from_be_slice(bytes: &[u8]) -> Option { + bytes + .get(..Hash128::LEN) + .and_then(|bytes| bytes.try_into().ok()) + .map(Hash128) + } + + #[inline(always)] + pub fn as_be_bytes(&self) -> &[u8; Hash128::LEN] { + &self.0 + } + + #[inline(always)] + pub fn to_u128(self) -> u128 { + u128::from_be_bytes(self.0) + } +} + +impl From for Hash128 { + #[inline(always)] + fn from(value: u128) -> Self { + Hash128(value.to_be_bytes()) + } +} + +impl From for u128 { + #[inline(always)] + fn from(value: Hash128) -> Self { + value.to_u128() + } +} + +impl AsRef<[u8]> for Hash128 { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl std::fmt::Display for Hash128 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_u128()) + } +} + +impl Debug for Hash128 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("Hash128") + .field(&format_args!("{:032x}", self.to_u128())) + .finish() + } +} + +impl FromStr for Hash128 { + type Err = ParseIntError; + + fn from_str(s: &str) -> Result { + u128::from_str(s).map(Hash128::from) + } +} + diff --git a/crates/utils/src/http.rs b/crates/utils/src/http.rs index 6f784781..991f00ff 100644 --- a/crates/utils/src/http.rs +++ b/crates/utils/src/http.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{Engine, engine::general_purpose}; +use encodify::base64::STANDARD; use reqwest::{ Client, ClientBuilder, header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT}, @@ -145,7 +145,7 @@ pub fn build_http_client( allow_invalid_certs: bool, ) -> Result { let mut headers = build_http_headers(raw_headers, username, password, token, content_type)?; - headers.insert(USER_AGENT, "Stalwart/1.0.0".parse().unwrap()); + headers.insert(USER_AGENT, HeaderValue::from_static("Stalwart/1.0.0")); match http_client_builder(allow_invalid_certs) .connect_timeout(timeout) @@ -167,7 +167,11 @@ pub fn build_http_headers( let mut headers = HeaderMap::new(); if let Some(content_type) = content_type { - headers.insert(CONTENT_TYPE, HeaderValue::from_str(content_type).unwrap()); + headers.insert( + CONTENT_TYPE, + HeaderValue::from_str(content_type) + .map_err(|err| format!("Invalid content type {content_type:?}: {err}"))?, + ); } for (header, value) in raw_headers @@ -186,17 +190,19 @@ pub fn build_http_headers( } if let (Some(name), Some(secret)) = (username, password) { + let mut value = String::from("Basic "); + STANDARD.encode_append(format!("{name}:{secret}"), &mut value); headers.insert( AUTHORIZATION, - format!( - "Basic {}", - general_purpose::STANDARD.encode(format!("{}:{}", name, secret)) - ) - .parse() - .unwrap(), + HeaderValue::try_from(value) + .map_err(|err| format!("Invalid basic credentials: {err}"))?, ); } else if let Some(token) = token { - headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse().unwrap()); + headers.insert( + AUTHORIZATION, + HeaderValue::try_from(format!("Bearer {token}")) + .map_err(|err| format!("Invalid bearer token: {err}"))?, + ); } Ok(headers) diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 4629c3cc..331a7430 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -13,10 +13,15 @@ pub mod cheeky_hash; pub mod codec; pub mod cron; pub mod glob; +pub mod hash128; pub mod http; pub mod map; +pub mod memory; +pub mod outbound; +pub mod runtime; pub mod snowflake; pub mod template; +pub mod text; pub mod tls; pub mod topological; pub mod url_params; @@ -26,8 +31,10 @@ use futures::StreamExt; pub use reqwest::Client; use reqwest::Response; pub use reqwest::header::HeaderMap; -use std::borrow::Cow; -use std::fmt::Write; +use std::{borrow::Cow, time::Duration}; +use trc::{Collector, EventType, ServerEvent}; + +const STARTUP_ERROR_LOG_GRACE: Duration = Duration::from_millis(100); pub trait HttpLimitResponse: Sync + Send { fn bytes_with_limit( @@ -74,7 +81,7 @@ impl UnwrapFailure for Option { Details = message.to_compact_string() ); eprintln!("{message}"); - std::process::exit(1); + exit_failed(); } } } @@ -91,13 +98,13 @@ impl UnwrapFailure for Result { Reason = err.to_compact_string() ); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] panic!("{message}: {err}"); - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { eprintln!("{message}: {err}"); - std::process::exit(1); + exit_failed(); } } } @@ -110,7 +117,14 @@ pub fn failed(message: &str) -> ! { Details = message.to_compact_string(), ); eprintln!("{message}"); - std::process::exit(1); + exit_failed(); +} + +pub fn exit_failed() -> ! { + if Collector::has_interest(EventType::Server(ServerEvent::StartupError)) { + std::thread::sleep(STARTUP_ERROR_LOG_GRACE); + } + runtime::exit(1) } pub async fn wait_for_shutdown() { @@ -135,7 +149,7 @@ pub async fn wait_for_shutdown() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Unable to listen for shutdown signal", - Reason = err.to_string(), + Reason = err.to_compact_string(), ); "Error" } @@ -159,7 +173,7 @@ impl> DomainPart for T { let address = self.as_ref(); if let Some((local, domain)) = address.rsplit_once('@') { let mut address = String::with_capacity(address.len()); - if lower_local { + if lower_local && !local.is_ascii() { for ch in local.chars() { for ch in ch.to_lowercase() { address.push(ch); @@ -167,13 +181,16 @@ impl> DomainPart for T { } } else { address.push_str(local); + if lower_local { + address.make_ascii_lowercase(); + } } address.push('@'); if domain.is_ascii() { - for ch in domain.chars() { - for ch in ch.to_lowercase() { - address.push(ch); - } + let domain_start = address.len(); + address.push_str(domain); + if let Some(domain) = address.get_mut(domain_start..) { + domain.make_ascii_lowercase(); } } else { let domain = @@ -237,12 +254,7 @@ pub trait HexEncode { impl> HexEncode for T { fn hex_encode(&self) -> String { - let bytes = self.as_ref(); - let mut s = String::with_capacity(bytes.len() * 2); - for &b in bytes { - let _ = write!(&mut s, "{b:02x}"); - } - s + text::hex_encode(self.as_ref()) } } @@ -445,92 +457,3 @@ pub fn is_valid_domain(domain: &str) -> bool { .is_some_and(|(_, tld)| RESERVED_TLDS.contains(&tld)) } -#[cfg(test)] -mod tests { - use crate::DomainPart; - - use super::{sanitize_domain, sanitize_email}; - - #[test] - fn idn_domains_canonicalize_to_a_label() { - assert_eq!( - sanitize_domain("straß6.de").as_deref(), - Some("xn--stra6-oqa.de") - ); - assert_eq!( - sanitize_domain("STRASS.straß6.DE").as_deref(), - Some("strass.xn--stra6-oqa.de") - ); - assert_eq!( - sanitize_domain("münchen.de").as_deref(), - Some("xn--mnchen-3ya.de") - ); - } - - #[test] - fn a_label_and_ascii_domains_are_idempotent() { - assert_eq!( - sanitize_domain("xn--stra6-oqa.de").as_deref(), - Some("xn--stra6-oqa.de") - ); - assert_eq!( - sanitize_domain(&sanitize_domain("straß6.de").unwrap()).as_deref(), - Some("xn--stra6-oqa.de") - ); - assert_eq!( - sanitize_domain("Example.COM").as_deref(), - Some("example.com") - ); - } - - #[test] - fn email_domain_part_canonicalizes_local_part_preserved() { - assert_eq!( - sanitize_email("cornelius_strauss@straß6.de").as_deref(), - Some("cornelius_strauss@xn--stra6-oqa.de") - ); - assert_eq!( - sanitize_email("Foo.Bar@münchen.de").as_deref(), - Some("foo.bar@xn--mnchen-3ya.de") - ); - assert_eq!( - sanitize_email("user@example.com").as_deref(), - Some("user@example.com") - ); - } - - #[test] - fn bare_public_suffix_domains_are_accepted() { - assert_eq!( - sanitize_email("user@gov.in").as_deref(), - Some("user@gov.in") - ); - assert_eq!(sanitize_email("user@co.uk").as_deref(), Some("user@co.uk")); - assert_eq!(sanitize_email("user@com"), None); - assert_eq!(sanitize_email("user@example.invalidtld"), None); - } - - #[test] - fn a_label_email_domains_are_accepted_and_idempotent() { - assert_eq!( - sanitize_email("user@xn--fsqu00a.com").as_deref(), - Some("user@xn--fsqu00a.com") - ); - assert_eq!( - sanitize_email("User@例子.com").as_deref(), - sanitize_email("user@xn--fsqu00a.com").as_deref() - ); - } - - #[test] - fn to_ascii_domain_borrows_ascii_owns_idn() { - assert!(matches!( - "example.com".to_ascii_domain(), - Some(std::borrow::Cow::Borrowed(_)) - )); - assert!(matches!( - "straß6.de".to_ascii_domain(), - Some(std::borrow::Cow::Owned(_)) - )); - } -} diff --git a/crates/utils/src/map/bitmap.rs b/crates/utils/src/map/bitmap.rs index ced6e201..e7a7d612 100644 --- a/crates/utils/src/map/bitmap.rs +++ b/crates/utils/src/map/bitmap.rs @@ -45,6 +45,14 @@ impl Bitmap { Self::default() } + #[inline(always)] + pub const fn from_bits(bitmap: u64) -> Self { + Self { + bitmap, + _state: std::marker::PhantomData, + } + } + #[inline(always)] pub fn all() -> Self { Self { @@ -68,6 +76,16 @@ impl Bitmap { self.bitmap &= items.bitmap; } + #[inline(always)] + pub fn difference(&mut self, items: &Bitmap) { + self.bitmap &= !items.bitmap; + } + + #[inline(always)] + pub fn intersects(&self, items: &Bitmap) -> bool { + self.bitmap & items.bitmap != 0 + } + #[inline(always)] pub fn insert(&mut self, item: T) { debug_assert!(item.is_valid()); @@ -96,7 +114,7 @@ impl Bitmap { #[inline(always)] pub fn remove(&mut self, item: T) { debug_assert!(item.is_valid()); - self.bitmap ^= 1 << item.into(); + self.bitmap &= !(1 << item.into()); } #[inline(always)] @@ -314,3 +332,4 @@ impl Default for Bitmap { } } } + diff --git a/crates/utils/src/map/vec_map.rs b/crates/utils/src/map/vec_map.rs index 67b42b43..86ff2a75 100644 --- a/crates/utils/src/map/vec_map.rs +++ b/crates/utils/src/map/vec_map.rs @@ -291,6 +291,25 @@ impl ArchivedVecMap { > { self.inner.iter().map(|kv| (&kv.key, &kv.value)) } + + #[inline(always)] + pub fn values(&self) -> impl Iterator::Archived> { + self.inner.iter().map(|kv| &kv.value) + } + + #[inline(always)] + pub fn get(&self, key: &Q) -> Option<&::Archived> + where + ::Archived: PartialEq, + { + self.inner.iter().find_map(|kv| { + if kv.key == *key { + Some(&kv.value) + } else { + None + } + }) + } } impl IntoIterator for VecMap { diff --git a/crates/utils/src/memory.rs b/crates/utils/src/memory.rs new file mode 100644 index 00000000..3bf46e1d --- /dev/null +++ b/crates/utils/src/memory.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +const CGROUP_V1_ROOT: &str = "/sys/fs/cgroup/memory"; +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +const CGROUP_V1_FILES: &[&str] = &["memory.limit_in_bytes"]; +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +const CGROUP_V2_ROOT: &str = "/sys/fs/cgroup"; +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +const CGROUP_V2_FILES: &[&str] = &["memory.max", "memory.high"]; + +pub fn available_memory() -> Option { + physical_memory() + .into_iter() + .chain(cgroup_memory_limit()) + .filter(|memory| *memory > 0) + .min() +} + +#[cfg(unix)] +fn physical_memory() -> Option { + let (pages, page_size) = unsafe { + ( + libc::sysconf(libc::_SC_PHYS_PAGES), + libc::sysconf(libc::_SC_PAGESIZE), + ) + }; + u64::try_from(pages) + .ok()? + .checked_mul(u64::try_from(page_size).ok()?) +} + +#[cfg(windows)] +fn physical_memory() -> Option { + use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX}; + + let mut status = MEMORYSTATUSEX { + dwLength: size_of::() as u32, + ..Default::default() + }; + (unsafe { GlobalMemoryStatusEx(&mut status) } != 0).then_some(status.ullTotalPhys) +} + +#[cfg(not(any(unix, windows)))] +fn physical_memory() -> Option { + None +} + +#[cfg(target_os = "linux")] +fn cgroup_memory_limit() -> Option { + std::fs::read_to_string("/proc/self/cgroup") + .ok()? + .lines() + .filter_map(CgroupMemory::parse) + .flat_map(CgroupMemory::limit_files) + .filter_map(|file| std::fs::read_to_string(file).ok()) + .filter_map(|limit| parse_cgroup_limit(&limit)) + .min() +} + +#[cfg(not(target_os = "linux"))] +fn cgroup_memory_limit() -> Option { + None +} + +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +#[derive(Clone, Copy)] +struct CgroupMemory<'x> { + root: &'static str, + files: &'static [&'static str], + path: &'x str, +} + +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +impl<'x> CgroupMemory<'x> { + fn parse(line: &'x str) -> Option { + let (_, entry) = line.split_once(':')?; + let (controllers, path) = entry.split_once(':')?; + let (root, files) = if controllers.is_empty() { + (CGROUP_V2_ROOT, CGROUP_V2_FILES) + } else if controllers + .split(',') + .any(|controller| controller == "memory") + { + (CGROUP_V1_ROOT, CGROUP_V1_FILES) + } else { + return None; + }; + Some(CgroupMemory { + root, + files, + path: path.trim_end_matches('/'), + }) + } + + fn limit_files(self) -> impl Iterator + 'x { + let Self { root, files, path } = self; + std::iter::successors(Some(path), |path| { + path.rsplit_once('/').map(|(parent, _)| parent) + }) + .flat_map(move |dir| files.iter().map(move |file| format!("{root}{dir}/{file}"))) + } +} + +#[cfg(any(target_os = "linux", all(test, stalwart_tests)))] +fn parse_cgroup_limit(contents: &str) -> Option { + contents.trim().parse().ok() +} + diff --git a/crates/utils/src/outbound.rs b/crates/utils/src/outbound.rs new file mode 100644 index 00000000..958c2a15 --- /dev/null +++ b/crates/utils/src/outbound.rs @@ -0,0 +1,367 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use reqwest::{ + Client, ClientBuilder, Method, RequestBuilder, Url, + dns::{Addrs, Name, Resolve, Resolving}, + redirect::{Action, Attempt, Policy}, +}; +use std::{ + fmt::{self, Display}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, ToSocketAddrs}, +}; +use url::Host; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutboundPolicy { + Trusted, + PublicOnly, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Redirects { + None, + Limited(usize), + SameOrganization(usize), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutboundError { + InvalidUrl, + InsecureScheme, + Credentials, + MissingHost, + NonPublicHost, + NonPublicAddress, + TooManyRedirects, + Unavailable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RedirectDecision { + Follow, + Stop, + Reject(OutboundError), +} + +#[derive(Debug, Clone)] +pub struct OutboundClient { + client: Option, + policy: OutboundPolicy, +} + +impl Default for OutboundClient { + fn default() -> Self { + OutboundClient { + client: None, + policy: OutboundPolicy::PublicOnly, + } + } +} + +#[derive(Debug, Clone, Copy, Default)] +struct PublicResolver; + +impl OutboundPolicy { + + pub fn parse_url(self, url: &str) -> Result { + let url = Url::parse(url).map_err(|_| OutboundError::InvalidUrl)?; + self.check_url(&url).map(|_| url) + } + + pub fn check_url(self, url: &Url) -> Result<(), OutboundError> { + if self == OutboundPolicy::Trusted { + Ok(()) + } else if url.scheme() != "https" { + Err(OutboundError::InsecureScheme) + } else if !url.username().is_empty() || url.password().is_some() { + Err(OutboundError::Credentials) + } else { + match url.host() { + Some(Host::Domain(domain)) => self.check_domain(domain), + Some(Host::Ipv4(ip)) => self.check_ip(IpAddr::V4(ip)), + Some(Host::Ipv6(ip)) => self.check_ip(IpAddr::V6(ip)), + None => Err(OutboundError::MissingHost), + } + } + } + + pub fn check_domain(self, domain: &str) -> Result<(), OutboundError> { + if self == OutboundPolicy::Trusted || is_public_domain(domain) { + Ok(()) + } else { + Err(OutboundError::NonPublicHost) + } + } + + pub fn check_ip(self, ip: IpAddr) -> Result<(), OutboundError> { + if self == OutboundPolicy::Trusted || is_permitted_ip(ip) { + Ok(()) + } else { + Err(OutboundError::NonPublicAddress) + } + } +} + +impl Redirects { + pub fn decide(self, policy: OutboundPolicy, next: &Url, previous: &[Url]) -> RedirectDecision { + let (max_redirects, same_organization) = match self { + Redirects::None => return RedirectDecision::Stop, + Redirects::Limited(max_redirects) => (max_redirects, false), + Redirects::SameOrganization(max_redirects) => (max_redirects, true), + }; + + if same_organization + && !previous + .last() + .is_some_and(|previous| is_same_organization(previous, next)) + { + RedirectDecision::Stop + } else if previous.len() > max_redirects { + RedirectDecision::Reject(OutboundError::TooManyRedirects) + } else if let Err(err) = policy.check_url(next) { + RedirectDecision::Reject(err) + } else { + RedirectDecision::Follow + } + } + + fn into_policy(self, policy: OutboundPolicy) -> Policy { + match (self, policy) { + (Redirects::None, _) => Policy::none(), + (Redirects::Limited(max_redirects), OutboundPolicy::Trusted) => { + Policy::limited(max_redirects) + } + (redirects, policy) => { + Policy::custom(move |attempt| redirects.attempt(policy, attempt)) + } + } + } + + fn attempt(self, policy: OutboundPolicy, attempt: Attempt<'_>) -> Action { + match self.decide(policy, attempt.url(), attempt.previous()) { + RedirectDecision::Follow => attempt.follow(), + RedirectDecision::Stop => attempt.stop(), + RedirectDecision::Reject(err) => attempt.error(err), + } + } +} + +impl OutboundClient { + pub fn new(builder: ClientBuilder, policy: OutboundPolicy, redirects: Redirects) -> Self { + let builder = builder.redirect(redirects.into_policy(policy)); + let builder = match policy { + OutboundPolicy::Trusted => builder, + OutboundPolicy::PublicOnly => builder.no_proxy().dns_resolver(PublicResolver), + }; + + let client = builder + .build() + .map_err(|err| { + trc::event!( + Http(trc::HttpEvent::Error), + Details = "Failed to build outbound HTTP client", + Reason = err.to_string(), + ); + }) + .ok(); + + OutboundClient { client, policy } + } + + pub fn request(&self, method: Method, url: &str) -> Result { + self.request_url(method, self.policy.parse_url(url)?) + } + + pub fn request_url(&self, method: Method, url: Url) -> Result { + self.policy.check_url(&url)?; + self.client + .as_ref() + .map(|client| client.request(method, url)) + .ok_or(OutboundError::Unavailable) + } + + pub fn policy(&self) -> OutboundPolicy { + self.policy + } +} + +impl Resolve for PublicResolver { + fn resolve(&self, name: Name) -> Resolving { + Box::pin(async move { + let mut addrs = + tokio::task::spawn_blocking(move || (name.as_str(), 0).to_socket_addrs()) + .await?? + .filter(|addr| is_permitted_ip(addr.ip())) + .peekable(); + + if addrs.peek().is_some() { + Ok(Box::new(addrs) as Addrs) + } else { + Err(OutboundError::NonPublicAddress.into()) + } + }) + } +} + +impl OutboundError { + pub fn as_str(&self) -> &'static str { + match self { + OutboundError::InvalidUrl => "Invalid URL", + OutboundError::InsecureScheme => "URL must use the https scheme", + OutboundError::Credentials => "URL must not contain credentials", + OutboundError::MissingHost => "URL must contain a host", + OutboundError::NonPublicHost => "Host is not a public domain name", + OutboundError::NonPublicAddress => "Destination is a local or reserved IP address", + OutboundError::TooManyRedirects => "Too many redirects", + OutboundError::Unavailable => "HTTP client unavailable", + } + } +} + +impl Display for OutboundError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::error::Error for OutboundError {} + +fn is_public_domain(domain: &str) -> bool { + #[cfg(all(stalwart_tests, not(test)))] + if domain.eq_ignore_ascii_case("localhost") { + return true; + } + + psl::domain(domain.as_bytes()).is_some_and(|domain| { + let suffix = domain.suffix(); + suffix.is_known() && !is_special_use_suffix(suffix.as_bytes()) + }) +} + +fn is_special_use_suffix(suffix: &[u8]) -> bool { + hashify::set_ignore_case!( + suffix, + "arpa", + "home.arpa", + "in-addr.arpa", + "ip6.arpa", + "onion", + ) +} + +fn is_permitted_ip(ip: IpAddr) -> bool { + #[cfg(all(stalwart_tests, not(test)))] + if ip.is_loopback() { + return true; + } + + is_global_ip(&ip) +} + +pub fn is_same_organization(previous: &Url, next: &Url) -> bool { + if previous.scheme() != next.scheme() { + return false; + } + + match (previous.host(), next.host()) { + (Some(Host::Domain(previous)), Some(Host::Domain(next))) => { + match (psl::domain_str(previous), psl::domain_str(next)) { + (Some(previous), Some(next)) => previous == next, + _ => previous == next, + } + } + (Some(previous), Some(next)) => previous == next, + _ => false, + } +} + +pub fn is_global_ip(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => is_global_ipv4(ip), + IpAddr::V6(ip) => is_global_ipv6(ip), + } +} + +fn is_global_ipv4(ip: &Ipv4Addr) -> bool { + let [a, b, c, _] = ip.octets(); + let is_this_network = a == 0; + let is_shared = a == 100 && (64..128).contains(&b); + let is_protocol_assignment = a == 192 && b == 0 && c == 0; + let is_benchmarking = a == 198 && (b & 0xfe) == 18; + let is_relay_6to4 = a == 192 && b == 88 && c == 99; + let is_reserved = a >= 240; + + !(ip.is_unspecified() + || ip.is_loopback() + || ip.is_private() + || ip.is_link_local() + || ip.is_multicast() + || ip.is_broadcast() + || ip.is_documentation() + || is_this_network + || is_shared + || is_protocol_assignment + || is_benchmarking + || is_relay_6to4 + || is_reserved) +} + +fn is_global_ipv6(ip: &Ipv6Addr) -> bool { + if ip.is_unspecified() || ip.is_loopback() || ip.is_multicast() { + return false; + } + + if let Some(ip) = ip.to_ipv4() { + return is_global_ipv4(&ip); + } + + let segments = ip.segments(); + + if let [0, 0, 0, 0, 0xffff, 0, high, low] = segments { + return is_global_ipv4(&Ipv4Addr::from(((high as u32) << 16) | low as u32)); + } + + if segments[0] == 0x2002 { + return is_global_ipv4(&Ipv4Addr::from( + ((segments[1] as u32) << 16) | segments[2] as u32, + )); + } + + if segments[0] == 0x0064 && segments[1] == 0xff9b { + let is_well_known_prefix = + segments[2] == 0 && segments[3] == 0 && segments[4] == 0 && segments[5] == 0; + + return is_well_known_prefix + && is_global_ipv4(&Ipv4Addr::from( + ((segments[6] as u32) << 16) | segments[7] as u32, + )); + } + + let is_unique_local = (segments[0] & 0xfe00) == 0xfc00; + let is_link_local = (segments[0] & 0xffc0) == 0xfe80; + let is_site_local = (segments[0] & 0xffc0) == 0xfec0; + let is_discard_only = + segments[0] == 0x0100 && segments[1] == 0 && segments[2] == 0 && segments[3] == 0; + let is_documentation = segments[0] == 0x2001 && segments[1] == 0x0db8; + let is_teredo = segments[0] == 0x2001 && segments[1] == 0; + let is_orchid = segments[0] == 0x2001 && (segments[1] & 0xfff0) == 0x0020; + let is_benchmarking = segments[0] == 0x2001 && segments[1] == 0x0002 && segments[2] == 0; + let is_documentation_v2 = segments[0] == 0x3fff && (segments[1] & 0xf000) == 0; + let is_srv6_sid = segments[0] == 0x5f00; + + !(is_unique_local + || is_benchmarking + || is_documentation_v2 + || is_srv6_sid + || is_link_local + || is_site_local + || is_discard_only + || is_documentation + || is_teredo + || is_orchid) +} + diff --git a/crates/utils/src/runtime.rs b/crates/utils/src/runtime.rs new file mode 100644 index 00000000..50ef8fd8 --- /dev/null +++ b/crates/utils/src/runtime.rs @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{ + future::Future, + io::{self, Write}, + mem, +}; +use tokio::runtime::{Builder, Runtime}; + +pub fn run_server(future: F) -> io::Result { + let runtime = Builder::new_multi_thread().enable_all().build()?; + let output = runtime.block_on(future); + finish(runtime); + Ok(output) +} + +pub fn finish(runtime: Runtime) { + mem::forget(runtime); +} + +pub fn exit(code: i32) -> ! { + let _ = io::stdout().flush(); + unsafe { libc::_exit(code) } +} diff --git a/crates/utils/src/snowflake.rs b/crates/utils/src/snowflake.rs index b5e75388..4a4ae542 100644 --- a/crates/utils/src/snowflake.rs +++ b/crates/utils/src/snowflake.rs @@ -32,9 +32,12 @@ static SEQUENCE_ID: AtomicU64 = AtomicU64::new(0); ID characteristics: -- 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years (from year 2022 until 2300) -- 9 bits for a node id: 2^9 = 512 nodes +- 43 bits for milliseconds since DEFAULT_EPOCH (September 22nd, 2021): 2^43 ms = 278.92 years (until 2300) - 12 bits for a sequence number: 2^12 = 4096 ids per millisecond +- 9 bits for a node id: 2^9 = 512 nodes + +The two most significant bits stay zero only while the timestamp is below 2^41 ms, until around May 2091. +Task ids with either of those bits set belong to the derived task id partitions. */ @@ -83,6 +86,17 @@ impl SnowflakeIdGenerator { Self::from_timestamp(timestamp).map(|id| id | (sequence << NODE_ID_LEN) | node_id()) } + pub fn global_id_with_sequence_id(sequence: u64) -> Option { + (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH)) + .elapsed() + .ok() + .map(|elapsed| { + ((elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) + | (sequence << NODE_ID_LEN) + | node_id() + }) + } + pub fn global_id() -> Option { let sequence = SEQUENCE_ID.fetch_add(1, Ordering::Relaxed) & SEQUENCE_MASK; diff --git a/crates/utils/src/template.rs b/crates/utils/src/template.rs index ba28bb1e..279f4a9e 100644 --- a/crates/utils/src/template.rs +++ b/crates/utils/src/template.rs @@ -238,381 +238,3 @@ impl> Default for Variables { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_simple_variable_substitution() { - let template = Template::parse("Hello {{name}}!").unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("name".to_string(), "World".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "Hello World!"); - } - - #[test] - fn test_multiple_variables() { - let template = Template::parse("{{greeting}} {{name}}, today is {{day}}").unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("greeting".to_string(), "Hello".to_string()); - vars.insert_single("name".to_string(), "Alice".to_string()); - vars.insert_single("day".to_string(), "Monday".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "Hello Alice, today is Monday"); - } - - #[test] - fn test_missing_variable() { - let template = Template::parse("Hello {{name}}!").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, "Hello !"); - } - - #[test] - fn test_static_text_only() { - let template = Template::parse("This is just static text").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, "This is just static text"); - } - - #[test] - fn test_empty_template() { - let template = Template::parse("").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, ""); - } - - #[test] - fn test_if_block_with_existing_variable() { - let template = - Template::parse("{{#if show_message}}Hello World!{{/if show_message}}").unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("show_message".to_string(), "true".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "Hello World!"); - } - - #[test] - fn test_if_block_with_missing_variable() { - let template = - Template::parse("{{#if show_message}}Hello World!{{/if show_message}}").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, ""); - } - - #[test] - fn test_if_block_with_content_and_variables() { - let template = Template::parse( - "{{#if notifications}}You have notifications: {{count}}{{/if notifications}}", - ) - .unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("notifications".to_string(), "true".to_string()); - vars.insert_single("count".to_string(), "5".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "You have notifications: 5"); - } - - #[test] - fn test_foreach_block_basic() { - let template = Template::parse("{{#each items}}{{name}} {{/each items}}").unwrap(); - let mut vars = Variables::::new(); - - let items = vec![ - vec![("name".to_string(), "Item1".to_string())], - vec![("name".to_string(), "Item2".to_string())], - vec![("name".to_string(), "Item3".to_string())], - ]; - vars.insert_block("items".to_string(), items); - - let result = template.eval(&vars); - assert_eq!(result, "Item1 Item2 Item3 "); - } - - #[test] - fn test_foreach_block_multiple_variables() { - let template = Template::parse( - "{{#each notifications}}* {{name}} at {{time}}\n{{/each notifications}}", - ) - .unwrap(); - let mut vars = Variables::::new(); - - let notifications = vec![ - vec![ - ("name".to_string(), "Meeting".to_string()), - ("time".to_string(), "10:00".to_string()), - ], - vec![ - ("name".to_string(), "Call".to_string()), - ("time".to_string(), "14:30".to_string()), - ], - ]; - vars.insert_block("notifications".to_string(), notifications); - - let result = template.eval(&vars); - assert_eq!(result, "* Meeting at 10:00\n* Call at 14:30\n"); - } - - #[test] - fn test_foreach_block_empty() { - let template = Template::parse("{{#each items}}{{name}}{{/each items}}").unwrap(); - let mut vars = Variables::::new(); - vars.insert_block("items".to_string(), Vec::>::new()); - - let result = template.eval(&vars); - assert_eq!(result, ""); - } - - #[test] - fn test_foreach_block_missing_variable() { - let template = Template::parse("{{#each items}}{{name}}{{/each items}}").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, ""); - } - - #[test] - fn test_complex_template_example() { - let template_str = r#"Hello {{name}}, - -{{#if notifications}}You have the following notifications: -{{#each notifications}}* {{name}} at {{time}} -{{/each notifications}}{{/if notifications}} -Best regards"#; - - let template = Template::parse(template_str).unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("name".to_string(), "Alice".to_string()); - vars.insert_single("notifications".to_string(), "true".to_string()); - - let notifications = vec![ - vec![ - ("name".to_string(), "Team Meeting".to_string()), - ("time".to_string(), "09:00".to_string()), - ], - vec![ - ("name".to_string(), "Doctor Appointment".to_string()), - ("time".to_string(), "15:30".to_string()), - ], - ]; - vars.insert_block("notifications".to_string(), notifications); - - let result = template.eval(&vars); - let expected = r#"Hello Alice, - -You have the following notifications: -* Team Meeting at 09:00 -* Doctor Appointment at 15:30 - -Best regards"#; - - assert_eq!(result, expected); - } - - #[test] - fn test_complex_template_no_notifications() { - let template_str = r#"Hello {{name}}, - -{{#if notifications}} -You have the following notifications: -{{#each notifications}} -* {{name}} at {{time}} -{{/each notifications}}{{/if notifications}} -Best regards"#; - - let template = Template::parse(template_str).unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("name".to_string(), "Bob".to_string()); - - let result = template.eval(&vars); - let expected = r#"Hello Bob, - - -Best regards"#; - - assert_eq!(result, expected); - } - - #[test] - fn test_whitespace_handling() { - let template = Template::parse("{{ name }}").unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("name".to_string(), "Test".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "Test"); - } - - #[test] - fn test_whitespace_in_blocks() { - let template = Template::parse("{{# if condition }}Content{{/ if condition }}").unwrap(); - let mut vars = Variables::::new(); - vars.insert_single("condition".to_string(), "true".to_string()); - - let result = template.eval(&vars); - assert_eq!(result, "Content"); - } - - // Error handling tests - #[test] - fn test_unmatched_opening_brace() { - let result = Template::::parse("Hello {{name"); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("Unmatched {{")); - } - - #[test] - fn test_invalid_block_start() { - let result = Template::::parse("{{#invalid block}}{{/invalid block}}"); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("Invalid block start")); - } - - #[test] - fn test_invalid_block_end() { - let result = Template::::parse("{{#if test}}{{\\/invalid block}}"); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("Unmatched")); - } - - #[test] - fn test_mismatched_block_names() { - let result = Template::::parse("{{#if test}}{{/if different}}"); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .contains("Block end does not match start") - ); - } - - #[test] - fn test_mismatched_block_types() { - let result = Template::::parse("{{#if test}}{{/each test}}"); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .contains("Block end does not match start") - ); - } - - #[test] - fn test_consecutive_braces() { - let template = Template::parse("{{}}").unwrap(); - let vars = Variables::::new(); - - let result = template.eval(&vars); - assert_eq!(result, ""); - } - - #[test] - fn test_foreach_with_missing_inner_variables() { - let template = - Template::parse("{{#each items}}{{name}}: {{missing}}{{/each items}}").unwrap(); - let mut vars = Variables::::new(); - - let items = vec![ - vec![("name".to_string(), "Item1".to_string())], - vec![("name".to_string(), "Item2".to_string())], - ]; - vars.insert_block("items".to_string(), items); - - let result = template.eval(&vars); - assert_eq!(result, "Item1: Item2: "); - } - - /*#[test] - fn test_full() { - // Load static html in memory from resources/email-templates/calendar-alarm.html - let template_str = include_str!("../../../resources/email-templates/calendar-alarm.html"); - let template: Template = Template::parse(template_str).unwrap(); - - let mut vars = Variables::::new(); - vars.insert_single( - CalendarTemplateVariable::PageTitle, - "Test Event".to_string(), - ); - vars.insert_single(CalendarTemplateVariable::Header, "Event Header".to_string()); - vars.insert_single(CalendarTemplateVariable::Footer, "Event Footer".to_string()); - vars.insert_single( - CalendarTemplateVariable::EventTitle, - "Meeting with Team".to_string(), - ); - vars.insert_single( - CalendarTemplateVariable::EventDescription, - "Discuss project updates".to_string(), - ); - vars.insert_single( - CalendarTemplateVariable::EventDetails, - "Details about the event".to_string(), - ); - vars.insert_single( - CalendarTemplateVariable::ActionUrl, - "http://example.com/action".to_string(), - ); - vars.insert_single( - CalendarTemplateVariable::ActionName, - "Join Meeting".to_string(), - ); - vars.insert_single( - CalendarTemplateVariable::AttendeesTitle, - "Attendees".to_string(), - ); - vars.insert_block( - CalendarTemplateVariable::EventDetails, - vec![ - vec![ - (CalendarTemplateVariable::Key, "Location".to_string()), - ( - CalendarTemplateVariable::Value, - "Conference Room A".to_string(), - ), - ], - vec![ - (CalendarTemplateVariable::Key, "Time".to_string()), - ( - CalendarTemplateVariable::Value, - "10:00 AM - 11:00 AM".to_string(), - ), - ], - ], - ); - vars.insert_block( - CalendarTemplateVariable::Attendees, - vec![ - vec![ - (CalendarTemplateVariable::Key, "Alice".to_string()), - ( - CalendarTemplateVariable::Value, - "alice@domain.org".to_string(), - ), - ], - vec![ - (CalendarTemplateVariable::Key, "Bob".to_string()), - ( - CalendarTemplateVariable::Value, - "bob@domain.org".to_string(), - ), - ], - ], - ); - let result = template.eval(&vars); - // Write result to test.html - std::fs::write("test.html", result).expect("Unable to write file"); - }*/ -} diff --git a/crates/utils/src/text/case.rs b/crates/utils/src/text/case.rs new file mode 100644 index 00000000..f74b65b0 --- /dev/null +++ b/crates/utils/src/text/case.rs @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::segments::{Change, find_change, is_ascii}; +use std::borrow::Cow; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaseChange { + Unchanged, + AsciiFrom(usize), + Unicode, +} + +impl CaseChange { + pub fn lowercase(text: &str) -> Self { + Self::classify( + text, + find_change(text, |byte| byte.is_ascii_uppercase(), lower_fixed), + ) + } + + pub fn uppercase(text: &str) -> Self { + Self::classify( + text, + find_change(text, |byte| byte.is_ascii_lowercase(), upper_fixed), + ) + } + + fn classify(text: &str, change: Option) -> Self { + match change { + None => Self::Unchanged, + Some(Change::Ascii { position, checked }) + if text.as_bytes().get(checked..).is_none_or(is_ascii) => + { + Self::AsciiFrom(position) + } + Some(_) => Self::Unicode, + } + } +} + +pub fn lowercase(text: &str) -> Cow<'_, str> { + match CaseChange::lowercase(text) { + CaseChange::Unchanged => Cow::Borrowed(text), + CaseChange::AsciiFrom(position) => { + Cow::Owned(convert_ascii(text, position, str::make_ascii_lowercase)) + } + CaseChange::Unicode => Cow::Owned(text.to_lowercase()), + } +} + +pub fn uppercase(text: &str) -> Cow<'_, str> { + match CaseChange::uppercase(text) { + CaseChange::Unchanged => Cow::Borrowed(text), + CaseChange::AsciiFrom(position) => { + Cow::Owned(convert_ascii(text, position, str::make_ascii_uppercase)) + } + CaseChange::Unicode => Cow::Owned(text.to_uppercase()), + } +} + +fn convert_ascii(text: &str, position: usize, convert: fn(&mut str)) -> String { + let mut out = text.to_owned(); + if let Some(changed) = out.get_mut(position..) { + convert(changed); + } + out +} + +fn lower_fixed(c: char) -> bool { + let mut lower = c.to_lowercase(); + lower.next() == Some(c) && lower.next().is_none() +} + +fn upper_fixed(c: char) -> bool { + let mut upper = c.to_uppercase(); + upper.next() == Some(c) && upper.next().is_none() +} diff --git a/crates/utils/src/text/count.rs b/crates/utils/src/text/count.rs new file mode 100644 index 00000000..6e9c92b4 --- /dev/null +++ b/crates/utils/src/text/count.rs @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::segments::{CHUNK, SHORT_LEN, any_in_chunk, count_in_chunk, find_change}; + +const PROBE_LEN: usize = 16; + +pub fn count_whitespace(text: &str) -> usize { + if text.is_ascii() { + count_ascii(text.as_bytes(), is_ascii_space) + } else { + text.chars().filter(|c| c.is_whitespace()).count() + } +} + +pub fn count_uppercase(text: &str) -> usize { + if text.is_ascii() { + count_ascii(text.as_bytes(), |byte| byte.is_ascii_uppercase()) + } else { + text.chars() + .filter(|c| c.is_alphabetic() && c.is_uppercase()) + .count() + } +} + +pub fn count_lowercase(text: &str) -> usize { + if text.is_ascii() { + count_ascii(text.as_bytes(), |byte| byte.is_ascii_lowercase()) + } else { + text.chars() + .filter(|c| c.is_alphabetic() && c.is_lowercase()) + .count() + } +} + +pub fn count_chars(text: &str) -> usize { + if text.len() < SHORT_LEN { + text.chars().count() + } else { + count_ascii(text.as_bytes(), is_char_start) + } +} + +pub fn has_digits(text: &str) -> bool { + let bytes = text.as_bytes(); + if bytes.len() < SHORT_LEN { + bytes.iter().any(u8::is_ascii_digit) + } else { + any_ascii(bytes, |byte| byte.is_ascii_digit()) + } +} + +pub fn is_uppercase(text: &str) -> bool { + is_cased(text, |byte| byte.is_ascii_lowercase(), char::is_uppercase) +} + +pub fn is_lowercase(text: &str) -> bool { + is_cased(text, |byte| byte.is_ascii_uppercase(), char::is_lowercase) +} + +fn is_cased(text: &str, ascii_rejects: impl Fn(u8) -> bool, cased: fn(char) -> bool) -> bool { + let keeps = |c: char| !c.is_alphabetic() || cased(c); + let mut chars = text.chars(); + while chars.as_str().len() + PROBE_LEN > text.len() { + match chars.next() { + Some(c) if !keeps(c) => return false, + Some(_) => {} + None => return true, + } + } + find_change(chars.as_str(), ascii_rejects, keeps).is_none() +} + +fn count_ascii(bytes: &[u8], matches: impl Fn(u8) -> bool) -> usize { + bytes + .chunks(CHUNK) + .map(|chunk| count_in_chunk(chunk, &matches)) + .sum() +} + +fn any_ascii(bytes: &[u8], matches: impl Fn(u8) -> bool) -> bool { + bytes + .chunks(CHUNK) + .any(|chunk| any_in_chunk(chunk, &matches)) +} + +fn is_ascii_space(byte: u8) -> bool { + byte == b' ' || byte.wrapping_sub(b'\t') < 5 +} + +fn is_char_start(byte: u8) -> bool { + byte & 0xC0 != 0x80 +} diff --git a/crates/utils/src/text/encode.rs b/crates/utils/src/text/encode.rs new file mode 100644 index 00000000..8fdf3f14 --- /dev/null +++ b/crates/utils/src/text/encode.rs @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; +const SIMD_UTF8_MIN_LEN: usize = 64; + +pub fn hex_encode(bytes: &[u8]) -> String { + let mut out = vec![b'0'; bytes.len() * 2]; + hex_encode_into(bytes, &mut out); + String::from_utf8(out).unwrap_or_default() +} + +pub fn hex_encode_into<'a>(bytes: &[u8], out: &'a mut [u8]) -> &'a str { + let (pairs, _) = out.as_chunks_mut::<2>(); + let len = pairs.len().min(bytes.len()) * 2; + for (pair, &byte) in pairs.iter_mut().zip(bytes) { + *pair = [ + HEX_DIGITS[usize::from(byte >> 4)], + HEX_DIGITS[usize::from(byte & 0x0F)], + ]; + } + let out: &'a [u8] = out; + out.get(..len) + .and_then(|encoded| std::str::from_utf8(encoded).ok()) + .unwrap_or_default() +} + +pub fn validate_utf8(bytes: &[u8]) -> Option<&str> { + if bytes.len() >= SIMD_UTF8_MIN_LEN { + simdutf8::basic::from_utf8(bytes).ok() + } else if bytes.is_ascii() { + std::str::from_utf8(bytes).ok() + } else { + single_valid_chunk(bytes) + } +} + +pub fn utf8_lossy(bytes: &[u8]) -> Cow<'_, str> { + match validate_utf8(bytes) { + Some(text) => Cow::Borrowed(text), + None => Cow::Owned(replace_invalid(bytes)), + } +} + +fn single_valid_chunk(bytes: &[u8]) -> Option<&str> { + bytes + .utf8_chunks() + .next() + .filter(|chunk| chunk.invalid().is_empty()) + .map(|chunk| chunk.valid()) +} + +fn replace_invalid(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + out.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + } + } + out +} diff --git a/crates/utils/src/text/ignore_case.rs b/crates/utils/src/text/ignore_case.rs new file mode 100644 index 00000000..e14bd37b --- /dev/null +++ b/crates/utils/src/text/ignore_case.rs @@ -0,0 +1,240 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{case::lowercase, rank::BYTE_RANK}; +use memchr::{memchr_iter, memchr2_iter, memmem::Finder}; + +const KELVIN_SIGN: [u8; 3] = [0xE2, 0x84, 0xAA]; +const CAPITAL_I_WITH_DOT: [u8; 2] = [0xC4, 0xB0]; +const MAX_INLINE_NEEDLE_LEN: usize = 64; +const CANDIDATE_COST: usize = 8; +const EITHER_CASE_RANK: [u8; 256] = { + let mut ranks = [0u8; 256]; + let mut index = 0; + while index < ranks.len() { + let byte = index as u8; + let lower = BYTE_RANK[byte.to_ascii_lowercase() as usize]; + let upper = BYTE_RANK[byte.to_ascii_uppercase() as usize]; + ranks[index] = if lower > upper { lower } else { upper }; + index += 1; + } + ranks +}; + +pub fn contains_ignore_case(haystack: &str, needle: &str) -> bool { + let Some(plan) = AsciiPlan::new(needle.as_bytes()) else { + return haystack.to_lowercase().contains(&needle.to_lowercase()); + }; + let found = if haystack.len() < IgnoreCaseNeedle::LONG_HAYSTACK_LEN { + contains_ascii(haystack.as_bytes(), needle.as_bytes(), plan) + } else { + contains_long(haystack.as_bytes(), needle.as_bytes()) + }; + plan.settle(found, haystack) + .unwrap_or_else(|| lowercase(haystack).contains(needle.to_ascii_lowercase().as_str())) +} + +fn contains_long(haystack: &[u8], needle: &[u8]) -> Option { + let mut buffer = [0u8; MAX_INLINE_NEEDLE_LEN]; + let lowered = buffer.get_mut(..needle.len())?; + for (out, byte) in lowered.iter_mut().zip(needle) { + *out = byte.to_ascii_lowercase(); + } + contains_windowed(haystack, &Finder::new(lowered)) +} + +#[derive(Debug, Clone)] +pub struct IgnoreCaseNeedle { + lowered: Box, + plan: Option, + finder: Option>, +} + +impl IgnoreCaseNeedle { + pub const LONG_HAYSTACK_LEN: usize = 256; + pub const SHORT_WINDOW_LEN: usize = 1024; + pub const WINDOW_LEN: usize = 4096; + pub const MAX_WINDOW_NEEDLE_LEN: usize = Self::WINDOW_LEN / 2; + + pub fn new(needle: &str) -> Self { + let lowered = needle.to_lowercase().into_boxed_str(); + let plan = AsciiPlan::new(needle.as_bytes()); + let finder = (plan.is_some() && lowered.len() <= Self::MAX_WINDOW_NEEDLE_LEN) + .then(|| Finder::new(lowered.as_bytes()).into_owned()); + Self { + lowered, + plan, + finder, + } + } + + pub fn as_str(&self) -> &str { + &self.lowered + } + + pub fn contains(&self, haystack: &str) -> bool { + self.contains_with(haystack, lowercase) + } + + pub fn contains_with<'h, L: AsRef>( + &self, + haystack: &'h str, + lower: impl FnOnce(&'h str) -> L, + ) -> bool { + let Some(plan) = self.plan else { + return !(haystack.is_ascii() && !self.lowered.is_ascii()) + && lower(haystack).as_ref().contains(self.lowered.as_ref()); + }; + let found = if haystack.len() < Self::LONG_HAYSTACK_LEN { + contains_ascii(haystack.as_bytes(), self.lowered.as_bytes(), plan) + } else { + self.finder + .as_ref() + .and_then(|finder| contains_windowed(haystack.as_bytes(), finder)) + }; + plan.settle(found, haystack) + .unwrap_or_else(|| lower(haystack).as_ref().contains(self.lowered.as_ref())) + } +} + +#[derive(Debug, Clone, Copy)] +struct AsciiPlan { + offset: usize, + lower: u8, + upper: u8, + has_joinable: bool, +} + +impl AsciiPlan { + #[inline] + fn new(needle: &[u8]) -> Option { + if !needle.is_ascii() { + return None; + } + let mut anchor_rank = u8::MAX; + let mut offset = 0; + let mut has_joinable = false; + for (position, &byte) in needle.iter().enumerate() { + let rank = either_case_rank(byte); + if rank < anchor_rank { + anchor_rank = rank; + offset = position; + } + has_joinable |= matches!(byte, b'i' | b'I' | b'k' | b'K'); + } + let anchor = needle.get(offset).copied().unwrap_or_default(); + Some(Self { + offset, + lower: anchor.to_ascii_lowercase(), + upper: anchor.to_ascii_uppercase(), + has_joinable, + }) + } + + fn settle(self, found: Option, haystack: &str) -> Option { + match found { + Some(true) => Some(true), + Some(false) if !self.may_join(haystack) => Some(false), + _ => None, + } + } + + fn may_join(self, haystack: &str) -> bool { + self.has_joinable && has_ascii_lowering_char(haystack.as_bytes()) + } +} + +fn either_case_rank(byte: u8) -> u8 { + EITHER_CASE_RANK[usize::from(byte)] +} + +fn contains_ascii(haystack: &[u8], needle: &[u8], plan: AsciiPlan) -> Option { + if needle.is_empty() { + return Some(true); + } + let Some(last_start) = haystack.len().checked_sub(needle.len()) else { + return Some(false); + }; + let Some(window) = haystack.get(plan.offset..=last_start + plan.offset) else { + return Some(false); + }; + if plan.lower == plan.upper { + verify_candidates(haystack, needle, memchr_iter(plan.lower, window)) + } else { + verify_candidates( + haystack, + needle, + memchr2_iter(plan.lower, plan.upper, window), + ) + } +} + +fn verify_candidates( + haystack: &[u8], + needle: &[u8], + starts: impl Iterator, +) -> Option { + let mut budget = haystack.len(); + for start in starts { + let candidate = haystack.get(start..start + needle.len())?; + let matched = candidate + .iter() + .zip(needle) + .take(budget) + .take_while(|(left, right)| left.eq_ignore_ascii_case(right)) + .count(); + if matched == needle.len() { + return Some(true); + } + budget = budget.checked_sub(matched + CANDIDATE_COST)?; + } + Some(false) +} + +fn contains_windowed(haystack: &[u8], finder: &Finder<'_>) -> Option { + if haystack.len() <= IgnoreCaseNeedle::SHORT_WINDOW_LEN { + Some(search_windows::<{ IgnoreCaseNeedle::SHORT_WINDOW_LEN }>( + haystack, finder, + )) + } else if finder.needle().len() <= IgnoreCaseNeedle::MAX_WINDOW_NEEDLE_LEN { + Some(search_windows::<{ IgnoreCaseNeedle::WINDOW_LEN }>( + haystack, finder, + )) + } else { + None + } +} + +fn search_windows(haystack: &[u8], finder: &Finder<'_>) -> bool { + let mut buffer = [0u8; LEN]; + let step = LEN.saturating_sub(finder.needle().len().saturating_sub(1)); + let mut rest = haystack; + loop { + let window = rest.get(..LEN).unwrap_or(rest); + let Some(lowered) = buffer.get_mut(..window.len()) else { + return false; + }; + for (out, byte) in lowered.iter_mut().zip(window) { + *out = byte.to_ascii_lowercase(); + } + if finder.find(lowered).is_some() { + return true; + } + match rest.get(step..) { + Some(next) if step > 0 && window.len() < rest.len() => rest = next, + _ => return false, + } + } +} + +fn has_ascii_lowering_char(haystack: &[u8]) -> bool { + memchr2_iter(KELVIN_SIGN[0], CAPITAL_I_WITH_DOT[0], haystack).any(|position| { + haystack.get(position..).is_some_and(|rest| { + rest.starts_with(&KELVIN_SIGN) || rest.starts_with(&CAPITAL_I_WITH_DOT) + }) + }) +} + diff --git a/crates/utils/src/text/mod.rs b/crates/utils/src/text/mod.rs new file mode 100644 index 00000000..068482ac --- /dev/null +++ b/crates/utils/src/text/mod.rs @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod case; +mod count; +mod encode; +mod ignore_case; +mod needle; +mod rank; +mod segments; +mod set; +mod split; + + +pub use case::{CaseChange, lowercase, uppercase}; +pub use count::{ + count_chars, count_lowercase, count_uppercase, count_whitespace, has_digits, is_lowercase, + is_uppercase, +}; +pub use encode::{hex_encode, hex_encode_into, utf8_lossy, validate_utf8}; +pub use ignore_case::{IgnoreCaseNeedle, contains_ignore_case}; +pub use needle::ConstNeedle; +pub use set::ConstSet; +pub use split::{SplitWords, split_words, substring}; diff --git a/crates/utils/src/text/needle.rs b/crates/utils/src/text/needle.rs new file mode 100644 index 00000000..ba098f5f --- /dev/null +++ b/crates/utils/src/text/needle.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use memchr::{memchr, memmem::Finder}; + +#[derive(Debug, Clone)] +pub struct ConstNeedle { + needle: Box, + finder: Finder<'static>, + byte: Option, +} + +impl ConstNeedle { + pub fn new(needle: &str) -> Self { + let byte = match needle.as_bytes() { + [byte] => Some(*byte), + _ => None, + }; + Self { + needle: needle.into(), + finder: Finder::new(needle.as_bytes()).into_owned(), + byte, + } + } + + pub fn as_str(&self) -> &str { + &self.needle + } + + pub fn contains(&self, haystack: &str) -> bool { + match self.byte { + Some(byte) => memchr(byte, haystack.as_bytes()).is_some(), + None => self.finder.find(haystack.as_bytes()).is_some(), + } + } + + pub fn starts_with(&self, haystack: &str) -> bool { + haystack.starts_with(self.as_str()) + } + + pub fn ends_with(&self, haystack: &str) -> bool { + haystack.ends_with(self.as_str()) + } +} diff --git a/crates/utils/src/text/rank.rs b/crates/utils/src/text/rank.rs new file mode 100644 index 00000000..1779d669 --- /dev/null +++ b/crates/utils/src/text/rank.rs @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) const BYTE_RANK: [u8; 256] = [ + 55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, 42, 41, 40, 39, 38, 37, 36, + 35, 34, 33, 56, 32, 31, 30, 29, 28, 255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, + 232, 202, 215, 224, 208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, + 126, 120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185, 167, 186, 112, + 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114, 223, 151, 249, 216, 238, 236, + 253, 227, 218, 230, 247, 135, 180, 241, 233, 246, 244, 231, 139, 245, 243, 251, 235, 201, 196, + 240, 214, 152, 182, 205, 181, 127, 27, 212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, + 80, 98, 96, 97, 81, 207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, + 108, 118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, 166, 237, 163, + 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, +]; diff --git a/crates/utils/src/text/segments.rs b/crates/utils/src/text/segments.rs new file mode 100644 index 00000000..85615b03 --- /dev/null +++ b/crates/utils/src/text/segments.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub(crate) const CHUNK: usize = 192; +pub(crate) const SHORT_LEN: usize = 64; + +pub(crate) enum Change { + Ascii { position: usize, checked: usize }, + Unicode, +} + +pub(crate) fn find_change( + text: &str, + ascii_changes: impl Fn(u8) -> bool, + unicode_keeps: impl Fn(char) -> bool, +) -> Option { + let mut rest = text; + let mut offset = 0; + loop { + let bytes = rest.as_bytes(); + let head = bytes.get(..CHUNK).unwrap_or(bytes); + if head.is_empty() { + return None; + } + let (high, hit) = scan_chunk(head, &ascii_changes); + let end = if high < 0x80 { + if hit { + return head + .iter() + .position(|&byte| ascii_changes(byte)) + .map(|position| Change::Ascii { + position: offset + position, + checked: offset + head.len(), + }); + } + head.len() + } else { + let end = rest.ceil_char_boundary(head.len()); + if hit + || !rest + .get(..end) + .is_some_and(|segment| segment.chars().all(&unicode_keeps)) + { + return Some(Change::Unicode); + } + end + }; + let Some(tail) = rest.get(end..) else { + return Some(Change::Unicode); + }; + rest = tail; + offset += end; + } +} + +pub(crate) fn is_ascii(bytes: &[u8]) -> bool { + bytes.chunks(CHUNK).all(chunk_is_ascii) +} + +fn chunk_is_ascii(chunk: &[u8]) -> bool { + chunk.iter().fold(0u8, |acc, &byte| acc | byte) < 0x80 +} + +fn scan_chunk(chunk: &[u8], matches: impl Fn(u8) -> bool) -> (u8, bool) { + let (high, hit) = chunk.iter().fold((0u8, 0u8), |(high, hit), &byte| { + (high | byte, hit | u8::from(matches(byte))) + }); + (high, hit != 0) +} + +pub(crate) fn count_in_chunk(chunk: &[u8], matches: impl Fn(u8) -> bool) -> usize { + usize::from( + chunk + .iter() + .fold(0u8, |acc, &byte| acc.wrapping_add(u8::from(matches(byte)))), + ) +} + +pub(crate) fn any_in_chunk(chunk: &[u8], matches: impl Fn(u8) -> bool) -> bool { + chunk + .iter() + .fold(0u8, |acc, &byte| acc | u8::from(matches(byte))) + != 0 +} diff --git a/crates/utils/src/text/set.rs b/crates/utils/src/text/set.rs new file mode 100644 index 00000000..4410b15f --- /dev/null +++ b/crates/utils/src/text/set.rs @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashSet; + +pub struct ConstSet { + items: AHashSet>, +} + +impl ConstSet { + pub const LINEAR_MAX_ITEMS: usize = 15; + + pub fn new(items: I) -> Self + where + I: IntoIterator, + I::Item: AsRef, + { + Self { + items: items.into_iter().map(|item| item.as_ref().into()).collect(), + } + } + + pub fn contains(&self, value: &str) -> bool { + self.items.contains(value) + } +} diff --git a/crates/utils/src/text/split.rs b/crates/utils/src/text/split.rs new file mode 100644 index 00000000..828509c0 --- /dev/null +++ b/crates/utils/src/text/split.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{iter::FusedIterator, str::SplitWhitespace}; + +pub fn split_words(text: &str) -> SplitWords<'_> { + SplitWords { + words: if text.is_ascii() { + Words::Ascii(text) + } else { + Words::Unicode(text.split_whitespace()) + }, + } +} + +#[derive(Clone)] +pub struct SplitWords<'a> { + words: Words<'a>, +} + +#[derive(Clone)] +enum Words<'a> { + Ascii(&'a str), + Unicode(SplitWhitespace<'a>), +} + +impl<'a> Iterator for SplitWords<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + match &mut self.words { + Words::Ascii(rest) => next_ascii_word(rest), + Words::Unicode(words) => words.find(|word| word.chars().all(char::is_alphanumeric)), + } + } +} + +impl FusedIterator for SplitWords<'_> {} + +fn next_ascii_word<'a>(rest: &mut &'a str) -> Option<&'a str> { + loop { + let Some(start) = rest.bytes().position(|byte| !is_ascii_space(byte)) else { + *rest = ""; + return None; + }; + let text = rest.get(start..)?; + let mut valid = true; + let end = text + .bytes() + .position(|byte| { + let space = is_ascii_space(byte); + valid &= space || byte.is_ascii_alphanumeric(); + space + }) + .unwrap_or(text.len()); + let (word, tail) = text.split_at_checked(end)?; + *rest = tail; + if valid { + return Some(word); + } + } +} + +fn is_ascii_space(byte: u8) -> bool { + byte == b' ' || byte.wrapping_sub(b'\t') < 5 +} + +pub fn substring(text: &str, start: usize, len: usize) -> &str { + let end = start.saturating_add(len); + let bytes = text.as_bytes(); + if bytes.get(..end).unwrap_or(bytes).is_ascii() { + return text + .get(start.min(text.len())..end.min(text.len())) + .unwrap_or_default(); + } + let mut chars = text.chars(); + if start > 0 && chars.nth(start - 1).is_none() { + return ""; + } + let rest = chars.as_str(); + let Some(last) = len.checked_sub(1) else { + return ""; + }; + let mut taken = rest.chars(); + if taken.nth(last).is_none() { + return rest; + } + rest.get(..rest.len() - taken.as_str().len()) + .unwrap_or(rest) +} diff --git a/crates/utils/src/topological.rs b/crates/utils/src/topological.rs index eb174896..80a7a0be 100644 --- a/crates/utils/src/topological.rs +++ b/crates/utils/src/topological.rs @@ -81,40 +81,3 @@ impl TopologicalSortIterator { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_topological_sort() { - let mut sort = TopologicalSort::with_capacity(6); - sort.insert(1, 2); - sort.insert(1, 3); - sort.insert(2, 4); - sort.insert(3, 4); - sort.insert(4, 5); - sort.insert(5, 6); - - let mut iter = sort.into_iterator(); - assert_eq!(iter.next(), Some(1)); - assert_eq!(iter.next(), Some(2)); - assert_eq!(iter.next(), Some(3)); - assert_eq!(iter.next(), Some(4)); - assert_eq!(iter.next(), Some(5)); - assert_eq!(iter.next(), Some(6)); - assert_eq!(iter.next(), None); - assert!(iter.is_valid(), "{:?}", iter); - } - - #[test] - fn test_topological_sort_cycle() { - let mut sort = TopologicalSort::with_capacity(6); - sort.insert(1, 2); - sort.insert(2, 3); - sort.insert(3, 1); - - let mut iter = sort.into_iterator(); - assert_eq!(iter.next(), None); - assert!(!iter.is_valid()); - } -} diff --git a/install.sh b/install.sh index e362be2b..08029a5f 100644 --- a/install.sh +++ b/install.sh @@ -15,6 +15,16 @@ set -u readonly BASE_URL="https://github.com/stalwartlabs/stalwart/releases/latest/download" main() { + local _arg + for _arg in "$@"; do + case "$_arg" in + -h|--help) + print_usage + exit 0 + ;; + esac + done + downloader --check need_cmd uname need_cmd mktemp @@ -24,6 +34,7 @@ main() { need_cmd rm need_cmd tar need_cmd cp + need_cmd mv need_cmd hostname # Require root @@ -90,6 +101,11 @@ main() { _config_file="${_conf_dir}/config.json" _env_file="${_conf_dir}/stalwart.env" + local _upgrade="" + if [ -e "$_config_file" ]; then + _upgrade="1" + fi + # Detect architecture get_architecture || return 1 local _arch="$RETVAL" @@ -112,8 +128,9 @@ main() { if [ "$_component" = "stalwart-foundationdb" ]; then _src_name="stalwart-foundationdb" fi - ensure cp "${_tmp}/${_src_name}" "$_bin_file" - ensure chmod 0755 "$_bin_file" + ensure cp "${_tmp}/${_src_name}" "${_bin_file}.new" + ensure chmod 0755 "${_bin_file}.new" + ensure mv -f "${_bin_file}.new" "$_bin_file" ensure rm -rf "$_tmp" # Create env file if absent (preserve user edits on reinstall) @@ -138,7 +155,7 @@ main() { create_service_linux_systemd "$_bin_file" "$_config_file" "$_env_file" "$_account" _service_type="systemd" else - create_service_linux_initd "$_bin_file" "$_config_file" "$_env_file" "$_account" + create_service_linux_initd "$_bin_file" "$_config_file" "$_env_file" "$_account" "$_log_dir" _service_type="initd" fi ;; @@ -153,9 +170,15 @@ main() { esac # Completion message - local _host + local _host _port _host="$(hostname -f 2>/dev/null || hostname)" + _port="$(sed -n 's/^STALWART_RECOVERY_MODE_PORT=\([0-9][0-9]*\).*/\1/p' "$_env_file" 2>/dev/null | tail -n 1)" say "" + if [ -n "$_upgrade" ]; then + say "🎉 Upgrade complete! The service was restarted with the new binary." + say "" + return 0 + fi say "🎉 Installation complete!" say "" say "Stalwart is running in bootstrap mode. A temporary administrator" @@ -167,8 +190,7 @@ main() { say " journalctl -u stalwart -n 200 | grep -A8 'bootstrap mode'" ;; initd) - say " grep -A8 'bootstrap mode' /var/log/syslog 2>/dev/null \\" - say " || grep -A8 'bootstrap mode' /var/log/messages" + say " grep -A8 'bootstrap mode' ${_log_dir}/stalwart.log" ;; launchd) say " sudo log show --predicate 'process == \"stalwart\"' --last 5m" @@ -181,7 +203,7 @@ main() { say " Or set STALWART_RECOVERY_ADMIN=admin: in" say " ${_env_file} and restart the service to pin a credential." say "" - say " Finish setup at: http://${_host}:8080/admin" + say " Finish setup at: http://${_host}:${_port:-8080}/admin" say "" return 0 @@ -222,7 +244,8 @@ write_env_file() { #STALWART_HOSTNAME=mail.example.com # Override the public base URL published in OAuth, OIDC, and JMAP discovery -# documents. Accepts scheme, host, optional port, and optional path prefix. +# documents. Accepts scheme, host and optional port; serving Stalwart under a +# path prefix is not supported. #STALWART_PUBLIC_URL=https://mail.example.com # Enable bootstrap / recovery mode on startup. Accepted: 1, true. Default: false. @@ -317,7 +340,22 @@ EOF } create_service_linux_initd() { - local _bin="$1" _config="$2" _env="$3" _user="$4" + local _bin="$1" _config="$2" _env="$3" _user="$4" _log_dir="$5" + local _start_opts _start_env="" + if check_cmd setcap && setcap cap_net_bind_service=+ep "$_bin"; then + _start_opts="--chuid ${_user}:${_user}" + else + _start_opts="" + _start_env="export RUN_AS_USER=${_user} RUN_AS_GROUP=${_user}" + fi + local _output_opts _daemon_args + if start-stop-daemon --help 2>&1 | grep -q -- '--output'; then + _output_opts='--output $LOGFILE' + _daemon_args='$DAEMON_ARGS' + else + _output_opts='--startas /bin/sh' + _daemon_args='-c "exec $DAEMON $DAEMON_ARGS >> $LOGFILE 2>&1"' + fi cat > /etc/init.d/stalwart < /dev/null \\ || return 1 start-stop-daemon --start --quiet --pidfile \$PIDFILE --exec \$DAEMON \\ - --background --make-pidfile --chuid ${_user}:${_user} \\ - -- \$DAEMON_ARGS \\ + --background --make-pidfile ${_output_opts} ${_start_opts} \\ + -- ${_daemon_args} \\ || return 2 } @@ -418,8 +458,22 @@ esac exit 0 EOF chmod +x /etc/init.d/stalwart + if [ -d /etc/logrotate.d ]; then + cat > /etc/logrotate.d/stalwart < /usr/local/etc/newsyslog.conf.d/stalwart.conf < /dev/null 2>&1 || true service stalwart start diff --git a/resources/html-templates/login.html b/resources/html-templates/login.html index f0d1fad2..641662e6 100644 --- a/resources/html-templates/login.html +++ b/resources/html-templates/login.html @@ -242,6 +242,154 @@ transform: none; } + .btn.secondary { + background: transparent; + color: var(--text); + border: 1px solid var(--border); + } + + .actions { + display: flex; + gap: .625rem; + margin-top: 1.25rem; + } + + .actions .btn { + margin-top: 0; + } + + .client { + display: flex; + align-items: center; + gap: .75rem; + margin-bottom: 1rem; + } + + .tile { + flex-shrink: 0; + width: 44px; + height: 44px; + border-radius: 10px; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + color: #fff; + font-size: 1.25rem; + font-weight: 600; + background: var(--muted); + } + + .tile img { + width: 100%; + height: 100%; + object-fit: contain; + background: var(--card); + } + + .client-name { + font-size: .9375rem; + font-weight: 600; + word-break: break-word; + } + + .client-host { + font-size: .75rem; + color: var(--muted); + word-break: break-all; + } + + .badge { + display: inline-block; + margin-top: .25rem; + padding: .0625rem .4375rem; + font-size: .6875rem; + font-weight: 500; + border-radius: 999px; + color: #a0192f; + background: color-mix(in srgb, #db2d54 10%, transparent); + border: 1px solid color-mix(in srgb, #db2d54 30%, transparent); + } + + @media(prefers-color-scheme:dark) { + .badge { + color: #f4839a + } + } + + .consent-lead { + font-size: .8125rem; + color: var(--muted); + margin-bottom: .5rem; + } + + .scopes { + list-style: none; + font-size: .875rem; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--input-bg); + } + + .scopes li { + display: flex; + align-items: flex-start; + gap: .5rem; + padding: .5rem .75rem; + font-size: .8125rem; + line-height: 1.35; + color: color-mix(in srgb, var(--text) 78%, var(--card)); + } + + .scopes li::before { + content: ''; + flex-shrink: 0; + width: 16px; + height: 16px; + margin-top: .05rem; + border-radius: 50%; + background: + url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%23fff' stroke-width='3.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M19 7 10 16l-4-4'/%3E%3C/svg%3E") center / 10px no-repeat, + var(--success); + } + + @media(prefers-color-scheme:dark) { + .scopes li::before { + background: + url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%2318181b' stroke-width='3.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M19 7 10 16l-4-4'/%3E%3C/svg%3E") center / 10px no-repeat, + var(--success); + } + } + + .scopes li+li { + border-top: 1px solid var(--border); + } + + .redirect-note { + font-size: .75rem; + color: var(--muted); + margin-top: .75rem; + word-break: break-all; + } + + .app-header { + display: flex; + align-items: center; + gap: .625rem; + margin-bottom: .25rem; + } + + .app-header .tile { + width: 32px; + height: 32px; + font-size: 1rem; + border-radius: 8px; + } + + .app-header h1 { + margin-bottom: 0; + } + [hidden] { display: none !important; } @@ -258,7 +406,10 @@ d="M149.1 84.7h-4.8l-44.8 25.9v8.3l44.8 25.9h4.8l44.8-25.9v-8.3l-44.8-25.9zm32.9 30h-35.3V94.4l35.3 20.3zm-35.3 20.4-35.3-20.4 27-15.6v20.2l6.3 3.6h22.9l-20.9 12.2zM99.5 129.9v11l44.8 25.9h4.8l44.8-25.9v-11l-47.2 27.3zM187.3 166.8l6.6-3.8v-11l-25.7 14.8zM99.5 163l6.6 3.8h19.1L99.5 152z" /> -

Sign in

+
+ +

Sign in

+

Enter your credentials to continue

+ + diff --git a/resources/html-templates/login.html.min b/resources/html-templates/login.html.min index 1f439139..14068ef9 100644 --- a/resources/html-templates/login.html.min +++ b/resources/html-templates/login.html.min @@ -1 +1 @@ - Sign in

Sign in

Enter your credentials to continue

\ No newline at end of file + Sign in

Sign in

Enter your credentials to continue

\ No newline at end of file diff --git a/resources/html-templates/login.html.min.gz b/resources/html-templates/login.html.min.gz index 80d26e83376b4bc4d64a79f9f5fc9fb988a6bdc8..e38f11ebb408027112079044830744f7a42a4cbd 100644 GIT binary patch literal 6981 zcmV-L8@l8liwFP!000021MOV-a@$CHe-#LhC1MRg1nUP@K}r7H-eR)W+I=NA3yxj8JR_Q?61Yl+=b$Llcc$61WBBW z7}Ku9MZTIX#4Zek!wox(!#wmOCky;Y%)Em=Qn3_iDpGBaIC08eOwPkR63^a;m$4DX zPtRr3c$($6c>d0wO#EdoQatBknWW-wp4>QD_$7=lC-Wp-h}1#PbnT~?VLWM1H~wNl zW9|JUO_Kc1apsqkN6S8bdQ-;<{B(ila*21jeDX;2@zX^|E^hM4BhSar2+j327nnSl z<7bGr9|S-^n;<)!G=2&`~5sw+}ZD@_Pb=BTm9TBXauY z6}^U?O{rN56n~p121BQvYGk`hcOjPkHp(69Ey0*aNpPihhkXcIbcFH5AR<~*bqwuU z9OKl^p=mm{opue|PYXZyaiC8$`)t%f*o9d*52G-@ovgyeLc}`2;rh}IK#FAD1aEa> zhfd_Sf^|GVTaGcsmrFXt$@4$iYx2+xP(qlH<0$J`<713mMFTvts7n z&9-xGPj{sD)Rl;hg>|srbBA&RnEW)qU+AdA$#Or%Lc`q~~{%r>hjk8}3$- zBTqKOfxWv7MYPE362U7LVG894lbA5uu45?>1^IXxqLjIabb%ZmiFh9(-XNX_Qt!6A zSXTq$(D2bz;mk#TEkxX8DGwOkmC~?hH}@&O-H4f3`(gCi)_OL6mXVIvI!VDl-)R=M zGV*o#*AO3fwJzm7!*~V1mmmAR!t?%0&qLyV>@JgS4!5(JPUh=KA!P*4Y>&>xj)}VeoF0Z|47B~L9!4>2eHR^*h$>ObVG2yo@^lwY2at# z0U)^ZJie<%HKi$WmG~bziuS!aqM3&W>5L3<>+xb4_Lho7BHg4l+cd+fO%gH<8eA^X zs!|`tSk@dyd$LM)kb#onqmk_PVIg;->L2!}6>T}eYPsXF#gjk5y!rD;EIwEB`~3p8 zI3Y$w$+cK$Td<1KVmA{ogm4G9y03z!G+2ah8x^<)HtPpAUw8Mqk0>G!wa8#0&?dAI zT1O$+@ZCt3PZ;V46u}7}kcW{t@KsXQdWe{du#MmA{N>c_H%8IK)Fo)H0?xuQQ*Nkh zQ*+>UB%;zPNK7bT)peyxXdX9&ozv#Tq#&(BsrlV^G>JeWqvp(0;a^SoPlr0nT~sPq4xc&Wgig{V*tx7(k7JA7aRXADP=ZB0#dFw3e_q3LI5(O*lgLT%HiWl}af3D^FmI7guU#m!m zm*<^!yG@hN3@X;v$MyYF6>d!mTB(ntn2p6 zE}~)JZ)Rrdl>Zi5$pQIcA zT!Ksza1ySmCXT3f|MXlIsg0-Ss-{jAPdpak&Ilr81v8T@idpX}t5Bv9E^1A^oJA#Y zNUU0;SBFd`#^(KEY?fkdk{Az0ZMSc9`W@FZ4GPH}&a`LdYuXMcvFNFziW(KbnVD}o zouNB2yk6HG1a8;pxg7($7(;hp^zroC_@j&84h|UdJ7#y(OAjmB=pH-nn5Oof4thK8 z(DBd#(Z-^lLkF}m(sM9?C3x*PeV)|EZ_nxC7yCN{ymj~ug9AC-b_eL`V%@-TN6yHl z{;}iHA2hp*F*pKV+r_Zw4Dj3G-y!;_-Q_2?8Ul_UVTI-pE!qIx;|!=0M8+81y#T0T z&yg`=XaKp=9plJ}YLlBP89W$Lq8R%Kj3rMaASNLvrw~rW2|!2N5Ciy0?1D9OWgl^?17K+${USWUFBpKn zvEpVAGpR+R9!n1+FkrL?j1tipv%CCD^9H;RsDif+??OFjF`gZ{Pn%>2-sEKU8dHM1dX4N z&wfPYcDrBRbcQ1$LK^_1k0fJ<9S18+8tlgIaOLnD)Swf9ga%WKs7B-f_n7LyO(_SS ztC8DND*<8P0B0qaEKLMMpV`?_?CdiF%+}$~>4QpN-gNr|mJEQhgWh&lpvf@eJ)lnk z%U_34B{+Z^s7yN&#|97-Qd3AiGo9oJFpP=ngw0TC#*p=3fM*}=5g;2gFF;D3F=ALd zMBxtB_nFP4C{i(ctQI8Law#dskXboogt}x1Snn`@%!UAGOOw%I0LLtVeP(_i2#iTo zA`%F2aRr3GtR0V(sylLrK}Uka@-vj=CTw~PT8Aygn011?2acQ%)d#s*B0PTdSa?Rv zD=@_~daPX|Mykiq^Ckp6iEE!{_6(9q)y?9lM7vMx4Y^G=VmP24bkqo39FUfchb(U_ zTSF2Mocd?#!~|I8u4MBH5;=&ZQjL{*IrO$7iqSw%4Y1ts0K@?@LBxEV*S2XNYD=0L%4dxLJ0`_Za_$4 z(<3ixu&*-)()Y3P-E{_1baoC4(HJ7+F{GowlNQvFzJ_cw8DA2P9+3oXqJ&2-kDjXz z$#fs4tNRbPsM&p{)I2{3aOB;Z3~#*Nm^@LB_U@7H^oY_OY_l7=-KcD>Xt2l3q%nsE zowEI9-6eVJwpXNh-9L{(tBBr=4mbiiE8ctj=Fpqb8It&c4s9s$faQ3w>R_LU##zlf zU*3!8f|ryf7`MqbH3B3oNE$*v%8WeWFHFLCE1sTj4xpvWD+QH&l_05DINXVuxj{md z3sZ&NGM6*GKBjU;A>V6JH7AFrOb$&-4a_1rG=ptQWu6z5Fq!rI(CnCp*`die}LVxHUf)YO5_`fTIJ z72%w9c4>l<)LU_jO3MYQ<^5lo*&v;wQ4?wE4!Ir@98qTt&zIBr)!AkurTQeH%~^_p z4Q6K5{ZYL=J?CLehB2w$V;b5u%*=!!{a)Po>rEuwAOWOwm5&#+ z^V$yF?{C5!czh8B$K~vlvgVnYESKQoMnqAt62TQvf@ES+D*m=bV!dcor6URy*GUHy zsJh>(wr~o>8kYKmXFqVmphgp} zwap~DwXS=4{_ptDp+64MV+I~Fn!b_q0~npGiMbEMJIOcA_|m|$pJD-1xxOD$UM?}@ z*LyK*^M9tX_HQB)Uy{9nhVw>i>FYp=^gNGCTaA8)+Ld{iD`_V(2#Q=ZTNSi5^62FgzG}GF$EceK?s%s0p=lFG zs1NI)g~+LCR&qmR`>u_zaQEV~T)9xe=gx(Yiqs)wFX zult+NUM}0u(8RC|8N4B4QatQpUJ3lqVtQ-~AT`KZz8XJowfJ()=dnxjlk)YQ<+`d|1 z#@>cY_#miU2&8~s3cedlKa94ia1F39UkPKACg9N8*ucJoS)SP(#(-=x#PJvFFh+ms z=Sk|C(*gnmFxa_uCtR`uFVwfS@fNYjEWa?#$$J1EU$$~OsaPjF!OfOFrY#Nt$Q{uzoQakS!mMfNGTUAQ zKeb8ARP}4qvVmbs3p*^JYbUKx+tmk`!dA7RwWlMpSuC!NkMCZ$uEThdT)R;sb;4bx zVrkjiS>|TKPlMH481{8$N3-*v&!y*R{e1rQU#xQwHOpI>OKAgH3Co5^U6`|3xzr`E z@H}s|t&6fNi)aVWniM_ua<(P8Xvt9Mud5KY%(6GkCwm3*+dKJwsiGy*UQ_R_?48TU zH9w|HVoj@xNGum2)fPf&v2?zeeKLRXcfOS5iD}Dw5IlGDst?hspyxbF=JfRE$rb~j zE5UkAd^R%{A7D&smMjg(g}6A;-+Q>#z1z$*?U${@k~$u+0=^=MrwYQr^`o3-1ujC1G)3s6Sz37p z%R1B>bFws;CH5`}7lxoq8WAs~c?qP7`?Ke1>fgG{G+DP=`PZ+Yw+p(iAzYu@pV7WB zr%hn4vZ!S~(Z!f78f}@++IBo^PvfV;m7&u#e)7c1XI<}i`4hqV{sO@ebUwD8J$u$J zP|F{82km_;{Q6bG@x-)y{m1lcnIsps59$9%-$fPDU()ttE4KsNmsBI4e4uo>ONxGa zDxVkfF@Xfqvlcy=QjHGz!x;RGBW9(UT$wLYnYcBZ3}^OxKc78v~qu%NudlQNmh{rOjQU-W5@~gZd+3Y7?(x{=>#Aa>Fqj` z%IFRI`~?^kAtM<=gS3W>ocW}_#S^3oOI3j}G8x^&3fGuhZG6CBZNR?I65nfiqYLx- zHebPJeG!Iq*XBg)@@s*EW{hc}T0riDGFG^EkWDQiG-a?QS?AKWam@`qY?%d*9l%}t zI>OAE9K^st8(Sb&wa8*O^j-^4aF}^hg#r(rj&%*)pe?bh0VFV0xzt2ar)EmZkGl zn=A3N;954U@fISW=XDu@-;iT=VPG`*Y4FS6uO5;`Y=);nv=C~4e@BJKS!tHiD?q|gET*#nsW=th z|DBGe{T?C0O2jqei;$+%Zl6%RmF0fE&8&QtBBiDP<|TzGE&j>^>?EfCUBdfQ41XMUhD#j4CLKHEihWlOZpbAJ<_`!Hl?MS!1kwZ5_@>^Al(} zJQ{yVZO9!F*!=k6!&}oX_9V^z20!+Y%aRob$ATLtdIRh}NfD*VPiCtaoca#q^3^da=s-5oWyLgk>OdJK-5o1lbuXKwO zbsBH+8Vid^-J0^mFy<yR@g^)QP)}Ef$nisX!gfp&;zruR>(DI_S(NiMhMA@l1#{9 zmF7^`I4h1V;CD^CwbhCcmR5s6j`__3M5dUVP~^x|fpY#`GRS@y0R&)p{gwiEz4QyWTf)31Llid04qlFj$W#g& zI1AXeoU7mlrFJU3sGSu15!GNtT!h4if9Z2zEKgNczk5zafci?Rn^92TNGdRF zYUGt5FnwyEy^VxVyiae7b0^>2v1x@hRUa8nYPRlUd*>XVGBx3tK~^5pZ~v;qYFEMjX+O<}^Ls-@cbcP&z#>JZy9k&+a66Bso(3OM^SBJ>a#`x4 zwVTEEW+v>9kS$FHR|FKruX^X(R)eG+$_q&>a*Cj(8*|p-%FU8;>Kbi6K#i1Ri<0DO zyLpHjB?oFGA}BJzQno;phFF7)pG;AfLF+7d~tih_VI1&ee2eaDJ|12^?Nu@0_xBR9vWb4Qmm-q$*v{QFJ1{% zDVa@Wb{#~UZ!wN@DWtf`>1lYu5YO&(8!$=iy4Fv^Jy$%XWE_K5nr%N`RCXI6O141@ z_xfd(v;s2vP@j`q`ehN~Z*|4;JO>io2XvyqI-o~iE%;S*mnpR;rH885;dr^14AYQ?A)4d5YVQSrzPb!m~%s#$*t*UuH zvHg*EB9NV`I<79j{=_%?+P_C(+Mz6%zgCSTCN}9c^Ax&h=e3GpNFCo}GH-TNfhSm@ z&+su7tO*5cynxH`Qto7y^&QLff-y1ppR(A-{tl*z%#JipGG&Gm9fZmheZM4&LZP?- zh=qGF?N>ZmFOMD1g^LKTOx-w>J({1M&gndxeUZ9Qe)#h1)6?r>xinYDEBWJ!+DZz+ zm9MK_D_{Kc57G(S1M*BQ3Q(CkZheSihRAf zpon~Y#WniPra9SyN8b3C?3oiN7G1JzA<8~wAC!pBZzW3f4+7W*{e^7$Vu8H3y0zzz zFsP#EAJnCrs>Ow4;&Y1}oX-#Zm{PsDAV07$WO*REt(+YF{Ya*Fu}sUC06-W+JbljI zkFb0#ZGr0&2u)#@`DD>-+D=46?L-`x%+_Unq{tTl$Ndo(MO)&qEPU~h_zMQrCAEHC zlu7p1qbX~+Lh&(&us X#?y24SIf@%Uv>Lmm`UBbC>i-j!2elBmEGX0R|%U~{xL=+AvvvpJ~(>&fQw0yb2%ia>>un`4cr?dn zjCK^q5TG?c5uC-cUNOxV68)3mA{wGuZZV)NgNMDvIO@ls6j73H%fRbxZg}`+TPgk` ze5QwQ*CI(HYs2~}omghFfMQo-ErKK}-gowGp}Yw`^P^DS!dv+LmR!E@MZb}2F~G-f z^5+s%_b90 zd0SRuZGWGo`TLhq{Dzx9Vx0Ze8?ltac>DY-`xiNvm9&2n*-lhx9NFI$K$%@edFhm* zNEfqt6u(~E&< zIVl(@0}@&kvp=?Fl`d`_4cMAo1IsJK`rwkv_jfoTYG|3Qf6slX!e*~ch^u* zeX7wpZsHEHi{ZPzdg7lMdmD-HCxHD%IbYD93^q3hHk@nXlsv>L{k_P;ZoC%Qj1fdkxb=(D(aAnq_U2o)0W<*K?VvVGonJp9}|w zm8YBu!A}I9>#?F{OuANFjC#FUCMp=zDK}9}KyuukH)JJ}dV#?TGHYWV+;Tf_t?5qm znr$cIV9`07@3~{O078CI-48kqJPcq83vbyRXg7C5+z zRAfnciU^*Oqy?NKl{rbaUFQlAwfcA(s+76N+JGG2f%p&*zW~oO)VsYt=AAHc9{6ae zaOR@A79zhCDbEbuS77Zc#Q-S(1;R5Xr%R-?t)c6XZ|v-x7S< zZ;^ENO!F1;UUfI`)tUEi%sd45yXjJHE97MX(=x_VHdz~)h|0rq27Aokrxj&%5n*5~ zf?Vd}u6lxnM4-K>q`O_LDn;IZ#v5wU7qVlF?s^hIG)m-k&@<@MhekfVhUcHIaY2#* zXo|&3X5j0xK=+MMauh&vl`#`vT`G94)(`knX|BCPy%`=hw-i#j(Yu<86hRWo%NLov< zi~g$0q2G@|ny3%bH68U}ti$83I(r$9yZWd|g*+Zfx-;SoU=K|ybj?%JvA8mgG&yOu zt28>azt`(^G0imSNcz3J2~Be9(WK`N z4S(pno@o%$98Msi`GS_iPMfG&)mzo27uX5Sm%cxCCx$oZyQA3c8w1xzvF#aScVrCl z^m_QGk3Sz9jQJg-ef=`Pj2_5S$DPv9q2q()cKL3qD zqjsO4SZWMD1|$`lW3*@ikf$X?C60_K$b%T7Va==q8ai^@ONeO*6`ht)vVtE)lEbiFx!PW@8kVI3`8Y@MyI#5#63o+XRPeTlvVi%H$ zECKFl88QGr$y`13DjHeK2eUMe{!s7X4+a2js;xP|NNQ29$LL`JMlAM-MIs$zbf5od z+=$nKR`BNYDnvnx<@D7$J;D&w=NZJ}fL39P(<;z0^8i@}q$P)SM@BVB1zM#O41+jI z3$WXOP3Gz)#ttivRv0we zP2KUz;WxN}A47!(Ym2l->VWlF>yS-52cD~mJJ2)1VeCL=jY~!o@i1g_`r4dB7J$t< z-Z?{P>BGzZaKy*}D?9kDzk*K28Lt6I1eg+3G6DpwxPrhR){aL`)t|WI*jMZ@e#T1OB+YySqd48o))GaNOv zhcw?%%ajx51MWdvO<=_ldD(Q#cw=mh2_V?@&qTxkMCQIS^9qO@Wy-0h+PxflTam`7 zW1u@2H{1a@028DPg%J=Y8421SQgGxc`fW;AeEp^n(!#)(R1Y!riTb!{eOByE$<7ZX7{YXQCCK)yzvH8%0vTNdqBQ3 zAWi#NW;b#B*|D{v&H)>f`W)(Xj_nWYKH;t3Tan}S|2&1RGI}$7SOhq$-v|8W(3|0p z34YK+4^BK{9FJB$*2$=!-Q551a!Lh*=fi+I&%2)=^f?3a7HAdz?M(>7-UvA)8?l>A z)EJ+t7>|LYRvz_f0F@yt52|>fs6$_=jcQ4Hpn9e6Lp&~6`HKg$`66QFk%Euf-Og2A zix)le85RG=t=twyTnJRu^fgFnRFc2yNb{|D+})hk$b5^U*3qx37T47)YX8vO;NV^f zQy0qW3m4NHVkjSR)cZ*t>_h7V)1b>j`gSE3rcqXfe2>L&8lcncQ#$3;R7bDr9|?5e zkE2a!@(JmL{0GiBH61L)X%T0_h_SPW`SGoK7I^enwKT`L@f7(M&k5>!UpLkB;xKPi zkHhIn4;{6Qh0|5uD9%}zOY)1ysVL5>fiwzwKy&?Q@F%Eba!+=wiTS0a87k$_ALD#Ty6 zDX`k8>{A-3?={Xe(2!s2w@Vs0j?x-v+DZdW`$P->{h$AJ@qg#)(7sX{@80Oyo<7H}G&?XpOajT6HCaX%XF?Ajtfd>_>+? ziqITsLmXwCZ>vh?dVj>^bxL1TE1~CAevH+iJJfFCx|&HVQ9x)`EaM{GRNp*UY;(S+ zwmN%qItqqqT$!_7R2U*W_KMEF5Y@KGjYP(fC9mA2sGenlUcbM6o?ygbRu4BSOW0Ls zFI>{qQ~gzVzD2|it0$%zyumK{(yBU#;}l7Ih)0hs;VOfL54=`)dbhjR>MpFbvnQU5 zP*^(KbY_LiI(m_om0O6l+@Vij`I{XZOj@n@+OgA)Ev=%pDrqIPngu%t+9C>bac#W) z<%M;f=83#^GvJoSyQ@MhI(8gJZYiQ7UcCl_*QH&C-QT*(MXcYtAOEG(g#84t}H zY8DgaqYFe2d&MEAiAo!*vL!rvVU;Raq~V@IYG9HnRn{|YZM~l%cu={z^^WvVcfAp) zfU8z#$pm(b;2y@vfjch3-r54#HL4}#NZs`!xTzJHI1=6#X?W(4cII+i;c@&HY6^EzbWvUO7HSH0ag5yV|BJn!K^IVusm+ z7>ZWElQ18IWKy{>h*om`YtdYKse>H?O({_q!!0fIWt36GgKM*w!qdwQ`T-6*KE@aG zO3)V?QE_X)XLoS-1;W?15U%lj!9fIGH%GQMHmIkN4okZZXF8%J>ofmYA!j85 zX}q#DV|)6Ni_u8uK4d$Qn1SNlmB3A&$9s9g?LY;;zVM(`=x;jL; zWu1V@j$N$NnK2l5^sBhE!Qtmi1bwVIrtRcXs``uE8mkBiO4o^tEfodLoRh?5(CBq^ zi(~dRFfB`+*R*U@Wmz428`fLbJiDfCC9OiW)2dUWB>7~Ky44tvR-!e5AoPn)|3_sY z|Ml#&NvcgM@3hIs^CZY9zS8@$a63QKN$8Gludb!D0Js3oaB@swRP69Mk zd;n|LH_Z}~tDRbvEI9ymg)@@eR#xluf<2fsmw!)zBy0Dyeza#axsl!h#JH??|;A0tz z$Y?~%D9tG;=r6E?lPLB!e-HTQXX(jM1q{XWe?+w6^3_h7l)JWcFpsE!tsXt9+)W{N zP{9w1X-gq!p+?1_b!wn?eO4Dn@srgPR@TzmSW&owFrDDes(kX#c zO#-KrXQpP|NiFH&#tPq(#0)Z zoj~!3lGcSg8o$?j5Qc&i^0s`Uu3@b6@z94Va8BI$1-mg0z5IJ0L#>Fs_tVwYYt#aI zO_WY{t1hA+u^8spbuV}egiG9GQ&j@BxxBFJkBA49TqT`YVcN4`Lc#}JiHO5R!FP23 zbZM^(ve8*tloiJyk*H<=5i#nV&Cb3JG!KHbZ{G--Rj5iB{ZpdOtaUpEwdp&o?<$~R zzOBe5Tfa5Zj-Q5Rmw#Zy2l@I^Z0n-HsGjGExOug(%x|%_;{hYZqesQ#tv>BI9-T+^ z{3nTANAzvT*CHpEef!JvrxF^RhQofMaqIe3IL==F_u>+-z+x*+J zPkGHJa4cN;{s}>uZvB)i)%O6(n<&Ys%ds;y+_V-oT}MmNH9Jozh5(iGOwO%NK&?wV zLusPE96&+uf^8JFulNi{1wLVgtNrJ+Zhgb}Gv5)neqwLhg&Sp+{&5x>GAA>IErX*- zwPuyONQ<(1x=OQT)~PMbO{?A4!ASb?X`ZOQ0u{n%;lSGp^3e9aBo#6vj;)PUeDlc@ z$|4R^6=;d_S(kdsG8_Cm{%s!Z;4=hp@*Q~~J{t9gZdd_8YqI!YpXSSm0GA6(W=@07khFW5@G+ zny*SiB%0*&aZ*Hi5zS-3F-igZY6f4*by+2gvIRkj6ySGLq}eE4#`As{W)b{1t=3UB zi*e^*?1v0-DQ9t!M9?l@6{7^OE?1*j6yu~;Wm4=%(;{Cj_f?C1nm|5^agY|2HJ*U< zAcegd5Nh3Mp8=FCK?JBlm?ssWS4Z|V&nD?~RV1{=@v6!f0AvKqwVbabT>R)^mSh?| zT^@B%00QIp5v~n<(-W||{s<^;)bl1T%ZI!e z2koG2d~g?+up_nEk6@SZ&v@QBZ#Alk${?vK7`p6xn8wK@Ud=1DF{OY*JA~H{jt%Wh zHY(Q3mb<9&Q>Y017RgPK->2iG=y7hc=;ZY394{dClm|wf`r+ju8YM-QPH_E_XaOW} znk*9UH6-i|Le}WUT z0CGK9WuuPMDKOvPgPlu?WsxH3t%_B`6Pv?%>NTwxdql+w32Tu~r$lIBQi}xs9N}V* z)2@xCLA@Bw^M_=7vFwQz^@=WT)c(WO$&w04#^0Vu&j~R<#I0QbDtOQA-RiU-Id7( zoeVnBL_}J#P=hiS#b^ev`1k^Nx4yhsEc!+%bk;1{9Wi1+1d?$JBIdZhBU(((zrc53 zKw;oKEOR(R zDJ}C@J@G{j4^j)3HgbgUyLo;`8`c+BJn47PSv@tl$w2;T?KF>yAJSH=GzJnOfYXmA zA|lvnGm@xBWCY+rFRLP62Cc|NmJ%(3y#nSJxB-xbaA_uKQG$q(F5o4r;`u{d;G+!x zTOLNWRb+bc4Q%t6C98e8ru)oW3(gIg4iExS9NnjNJ^MHaVvF>IXnv>2r1ahGG&@)% zi@aD19;Zxy71sps%|77*Hyci7$;jbR&WrR{KFL)MN2BEoFirx!h72Esp_1JlwZ-zy@e>0<$}j;=OgTaEnEh$cH`9?FxY&}6o~yO z9l|Z*3#^XXhvD!hx*dGF31BE4Xfe8)o9}bA>w*BHLt9vu`6!K%uY@=29)AJ`r3Ecx zc*bYG0Gx= zW@kWNZj%ZLWB>P*-tl%Mkh&hZ3xlpVWvjUZ2wiv%H%sfzJMLaaG4;0?*HiG`7T;PmySJ zTu%og3c*1-%}BoQL)8ecQubu60jRj~OO&swX^zh~Ka`TEY{yPx!HB_;7;P%RL}yng zx2I7&ofgRyADFF07#BJk&x>qRK?|1Z4i4Mchw&0_0j{J_7q2@otRbQ?-0~3CkR#z> zgG1=Rgi*{&P-f@z`(&e6jCok@&nJ=iTTn>CTi<O4f96@h6J+)-vn@}exCbbk zRbW;7XfWZ?v$Mg+=e-VO10BNh99OX~W)`gz;>y0Eo8L!H3VG3z{xAg@VHND&2u4if z_=5iTjKP$hJK^qru29?%z$MLf!_EU1n0xg$)LlLINcDsb8i8Od_5XLEX&^cqum^f{ z#s)3U!y+C*w@c7`&wI%Rz<$F^B(ww39Q#K(9Pk2E3QR~q&z_9%jI(V!LeMSPHLxKh z--9Nnxza&vhDso%D&k}9AKn~?2L4c0%f2I$%qnQ9Lnlxm(IP}h%00cl9q@H3lhLX` z`}Qn}$E{)%VWseUf&ViQojVVr-S>UoUz;^}Q~*;vAh zG0d(=?X|H~QdgX5_Mc^x79^~p1vca4%O6+CD&cplm)?S9B=UA35>??!fR71~h6Dv) zKoy`ze?jnVSk+J<#D3UdBY&_qQYhw-Ruu{0P~wKImcrPItF~N~vrui>4{8JPUoY@~ z0-qO0&l?zclYT!XcO`0;s~(*%^J!0Il%Q}x`|(|BW9<<(4T3&ENZ8@7_IYI%+>P!y zTbjo3uOhu$Rq(q~`ICnd`+~sf0=_j}riiN_F%utwUW?*qRWa|AKr!2ivgI`g<+#1 z0JslfFxa7O8`&Bf>JE2~+WPwu^<00uw zY#CW}T%hjXG4+*y?ngkY&Q3moC=sJN9X(hoXr*U)*iWQ~$q#wcm2>zHQtFjB( z(p;wU#VHSx7uS`={QC}U8wicWMu0YEcHE|5UPRo*&L!cIeY*(C?>x(RCk7RAa?^D$ zF-i4~O3|HARN6Bsc6Y1E+_*?IrFub_z#~Hu7{!7z(pd7YRC*b z^u8B^tD(WUB?=C__Y-oF0v2F2?|Sqt3?xz?f}F_>8hbT7iDu~iraaHg71I1}Dp`4Ertx?G^lnBDkZJ;D5CrQyMg*0)gp*Wnu9#r$bj}jxO1tE5D z!(}6+^(_g)O(7wXyCNs+KKGJ)Nbl@zHd2f&QMncu+d(YCUQHii00Zob^#-QpT&hI{ zaY>W*1#DnfM|-3FRXDpgxEJMUpds-i@$ZkT++ssIclj^ap@|B~o9zo_j3-gJg=zLO zG>Hw<<8Mavd0dGFh``eXy{a~%pfz(GioL+EE`(e{!{s%3IFN>^zK`&3r{fM2f;GB@&(#J-gsYQ_%jo^>^%XCJxn8W{%+OHK2d^uo zHkV&+N6X#^>-O|!7!9x8SSd|GFT@B9jaaz}+X_6y!C*LG|B$shOkIAwVn8|ga9(c0 zl+uGCRnBgI`_QhEw-Vhxw)cyQeI1pAh5V4bv`l;+d?~I+4o{qJP+8`S}Lkw@ezd?`V z!HcVltMli6oHPc65Z#6G72l$sieI{! z3R~ha26kPh{HAIymw)U&4K53PUmr{>HdhZ*T5%T`$u!Ma3YMPR9BqI)@iq*VJLZ54 zGQCZ#tNHw(0&(0j=-_Ek=N3#U(&rcd$<{;o7rE%~OYwBn%5Dne3`9597d0^Mbl5Wo z6c{uNuA!k8!m|dvo$o-sX)}}Y9XJ5(dpzc(L5dFEi2JsoEHD)KBJZ_{%pkb78cJ}< zO*N4;g0i2^;O6~4dtEWc@Q*e;6=sYngmZ7;Q(kg6JIvHLbY{q+Sjdh55s3(6MUmDN z=6=CS!lc^liQ!oJ19q0+XhRVgo@&5)8thu#uCehTS4X^m_&VciW0`z`gH?rxoi!2A zcwv0>mdlBeD!X6II$}1;{TFfy!A5Zn!}uPqHG0w0QRV3Q1zzqDKr$ zY%LD_r&l~>SX0Ki{Q6?Pll`C70s1ubJNH`pFIybfYX@u2A_ zrW_7+Tx4NhT{R39*5HzPH{Zw`^xdVjhAEF!nglCL$sFJ1nOaAnj@5=Pl!O~Fp=jRJ z&KovzK}eY#8Gc6ZFJ&6&vs~n3rpys)pQV0HYX175Xyc+#yjlOcS)>egB!gmnm!n_l zy*QM7H)Jqz+U7*A5m5NtuKF>u#u79&vi7>Q<8iAjO8D2NXGu_H6Dy!!m|J0v-anv! zT%(+?lhgTw{ALW49FNT zGqvI3(6RFSrz$-=8H0tkOVEZkByF4=c)k$Yu7QBnDcJV`*;tBjkVA01>Zt-jaS^5J zjN>0wI3I5)Gsvw+i^f!=zrg=_4+k_Ech11>c|~CvykFWK#D1Zm@7kz zF)0zH0NOqyi4CI%5u*Jxm=*QlCjkNi(gX1!$v`QbXKdJ@;L{Z1T_^%qCHE_6c5a^S7ylPET zJ1V3^u(J{K%^CVyu%TgnM+|>_AsiT_gG~38Cy)jJX&~PiLN8%JMt2xWIy_9v6}k1z z@^Qyv6nbEM=XdGjCDCd3@v=tt>1=|mg3aa@_QM7-lo%;!(iG1K=g2D+I06e z(`U$k2M%0o?#!s^?r9{s6#ony(r)d^k?HOlkfqYyv2X6kdjHToj`WrmU(a5P5U8xx9?;>t&+OaKE zhDH)c3M)52lHCN>!Orv8TyW}VwlW70%D^>>*amFU?Y<{X$9IuzqnV%PAckUbZNk6E zgtJUSDU{45p%33hx&>)-N8HQ_;v@2Wz{v!F%0yEKyqg$AYU#CS00A$X?;`q9IF)Gs zANzEatA)2e|LOqnOJ^!g$93&aec5qc(ezLHYKbI$xS$wqSf4xRs62@1YRvp3L#6#P zH4mZr%?(GlFbOdz{4*1;@ADj%AX4Amu*A;>pleW6E@E?n@`Gnq{Ihw*U?B(b#N0{a zlN*X^Ah)$X)quTiu1;f~!BEt?G$tAhLM=?=1cZZ-C&sZ*!9euVx>BNgJ5xK|r5>8s zaZS$BlCo5p)Kz{mj>>sfZ>7Ju5F#vYRJ4*hEw0}J=hy(L0p2Hl*r~^wGGgJIa_zU! zDc3>zGSm8@w5OEy_Up_C>+czq$F-%qI{~YdIO==t1?22!gJd6@(>QEmj$RfxcTN#M zAS2&}{Dda0TGG2ND5^_ay4sElAtP$l%O(@oo|_JA-7`@D6;ckE_*>9)G^1v1p#!8! zAt_q~o&`OEv8OKhB9(*qJ?LO4*rp8WfX(g>>UR%pOA!6V+Uh>?tOsCMGGRY3ch=ux z@?K;X?gOgU$B{sIH+210D8x;P`t~w1KG$)>975tp0}hzT&55#wdEmEhBFFyk!^*q%joByyFXzLD&?V z_?O7|$p^d+#DM=3SocsgcMKqZ3K^jRMjlt^`~pspXIylmwmQ;C>}lOsScS1gkHr^MBHcyO+ECZ0 za2qrCxQ6)31orBnO_(3NYILXho3AkbZKX=@STl>mVN1CI4nLOiV-1Y%(1eACDx$Mz zcP2Fb`JDrv(WwFP1{@T?OgQ+9$<@h^f4po$(;XVHuGTh~#o7shnu*T8*oW&~T=qC% zPh0$VxW$8qI7A$^+iMW;*G|Zg+U6&s@%MY88YL!LPE%NM2uIhDx(m&2RxJehXYJVU zMn1X3x?e&{mhDg^KA?hN;6PN+%LM1+8AG>GNAj-4(`nk(u1wYn7 z>X%uAeW!0;*RB7hy_SS))x*m?_kWs=x#tYXA6Lx9BNW%+S(}vcUEOhnA zX3MxH?y}ukeC+PcqYNN9RCDMOzB+n07sxSpYklgt)`pYF?rOYWygSlw-@B&etr*e) zv8(V$M|Yz}2Y)~Z&)pt4ay#vH$QnI54;J_EO;XKi%vatp$yshWx?~;aRY{>fE`-^EC#L0;(-e{t zjCX{BF23C zDm6EC-AVG<0pI~XyCH}U28a(}7IA=>!DlxJ(LpVxL@Q+J^b~AhVSVMu`}SnQI;)xE z-&BME`IVrj(`X$~=*!`D7gj#1%ij{D`_fWw@E3<(Lo{uwi~ll*+NQ4r zCk)I#RC5i~iHw?Hu(<)}7kdd@il@$Jh|T6T7LltG)Jv9G%AtR>MTELwN3QZ@-m_?T zP(Y6R0y%a7*+c{5)S;*hHdI8dO?zSw>lU(nN}f7!V@)?yv9w?lR_o3UvQJ&QE)D|s z|9pJ>=FQRX!BTkq=B>$+!Gat`&&1Es;KRuQG@ERzvxNv|%f-8qvWYC}}~T!=*cDAXZdKAe>qxit95vdNhy=P$twA2^JG;J9mmd zyYM@Mf)S9E&f*#JNmT=8eA)ouFj*{_r!)c(jBgu&L>aXX()_jo$dAbim^*XU`pYKZ zu%e=A*~@I&=jy(}`O+hEy-Oj~`D$=FAU66)P`k8Kq1$8++mz<3?0zon+l*1ZpM#dA zg+X#wIqhH?<68+s52nS0wqo+D&C)Vkg`aX#tf;PUf=*|`F2THDbbJo4!42`Sa zNM_b?0Edt?$<(sEKRrHr^)`Ao99(?=+&LfncOA92Uk)?8-rd2$@9zj)gOYdrZ zZiQ)EWd`S`CLHm1Fo5fO(!3EJs$mM-n+S@q2aOSduA-&nojjp?a%^3x#FUwy!2 ziw2FL^|O8S1+Ac8qW0CV!7$$?2(z`I!iPK1oCJXtolS61J1FD5Hlss~kHH57Uq>4? z5*r`^)W1jgTL)^Qcir>NUc#$%ROBUaeG$9Cb-Ay})!M6W%hm z4Va^w*lD=baJYv_3Shsj7#5h-;BcJuqN>Objg#}ZT-Tyu@N*=-w~!Up+XM5j3fO%`Be^{UvxlA9Bk3d1fxkjKB3HA_9_)knt@X&?O=6PE?2Sr z0N{t#+aY>bHfV4g8odMEvE^~bpCk3%2hi$nIj}7f=wjJA&2K(t(inFFUap^4jz)@-rr=p?{B(2==C@6XWv(y@HokSmKh9blI-rrNH&2S@Y z!8`3YPg>>DnDtJpR~$PTIDCL@{?BiNvr&%ResVcKWUYpw8~Ff z&eSbc(6516+6=9a*Efzx7}Qv~Y)%Mja~l2;#4T3Tz!O})T8)IqS| z(gTK+bld_<4?w1rjBC%?|H?J9`I`7ic9+F15F7m4O4a>C~W$63NSsu7($Ojis=DJ$vqz7r4J}mL3asiaPuk3Y_lt;(kn);XX!2y4% zb35?USq%So{1UP~tdCy3 zH0@UMrvv%hCL{e)+p`{L--$SV*X5@JfT+gY8H+oKde)avLR2?^qg zkW*6YKtQbW#N8n5YQ_lQLsf-5@YGCo=vVvb$IQs$Y0;#UEYH@9d{y&cDXm-w^mkON z!{kH|+i-6a96SP+afv_piX|yH+)dnS8+zDsnbIJ4GzXXfZd;4T(Hf+aQM z4%rpuK;F9C2NgWPoXVoouuQ7VC`4$rY^GNB7bI%0MPy^$mEHKor+EI^Ro(3GILWcK z0{sg5{1dGa$PiQ-djK|!Rmj`c=%N8NV00}rIRA1Hbe&)YD{;eQ?h;HQEiMWi_6%v! z+)j&&S)8n!aS5^n+%LccJI56;1j{O#$Y!pIFvfx!e~LihjEvuXz#E{%verirSe6F9t>g3n*u-() zm7l&Fq9dh!BZp}>lzPt7Ms=J35VdML^l2K@aVneRyWI1Y`^w3Icb7NW|IYHdJUzD^ zPstUKz4I@lSxl~q-RI73uRx$U8&qT>!^w^P?y{79?Bz})+-o%AFrmC zr!oHB7sv-VAXLD?bXk?(I)DvU3w6E?=s^o;;9|iG?M!3qu4Qn?(&~0yCG(xDBT30% zEaz}|B0N+FS`f0E+pOAwgfqU<@y<7Y=2VR!7>?vOM5ip{xxDZ5={cSu+V!Ir9KpBkv)tjMb>^vu>fNnls$yb2RuR9o5gdQ~xg zy=Z2mR~=lxc{O@TaHLoJ<0=nh{;l>B1Ubg^NW$>Nk}4NDlw?(O_Sx0HKX;BxgIs;F zCDpkVE35(4+B7btG1;w3(QKz0u1+Sj0!w7~4!vkV7t^~LVY6AbS!7lc27 zo^qPhl#DG9baM-UteO)JL-w%1oW-GsQa85%oX;TOC>#Jaw*XuPDh$l#7J%E)vd=P5 za}A&j?W)Nxq|WX~XUq-~$fvZZAT0`A^)M(_4+ca0Xt?`?-QEBZs&FxNe~yBn-4L23 zMDZgR4wmJ5L%`H}5SVMcB&YK<2~rW5t#$YB!7VSiH#g?aR;j9h{&s0A2?FG~`!vi6 z%4*I48vX{}zM>^{bCd$@hTyCpir;rrVpDtp^{-jR(iGT|!ctA$nB_P@WQB;zJJ6Ai zqldJb0kh@;*PK+JTASvGg;95N)3JW3uV0`Z)>&iI5Ou?;gQHU&SRkEfO4|4lDvhM0 zFpaEkYjgz?3eB&{{i{#u+(!MB=w4`LbtD~qFe2&1-%hj`k56?N=h=M8$y&#llvf2;j)j@Wxs^i)H)#9SQsNSp&HAJ!WyfQFX`So3il zB@CxAvN1@v>PLJE2U*woxPIzh9Vfv3Ci1-?DP0bI8()E>9Oi;)w>LmQS+jH*$j;Pz z5SZJz3PFIHaR=GIuP=#Ibc|hkJuqoPrOz??npl~rFap!|a}91 ztBx4>yJc1PU`yFl4Tie*umOJ&`&2hB2kf%LAni6O2g0h^q4#YAdMJ>3=P-W@-PZ29 zwU;0&2Ej*gL1tX(BFJX>hI8OQjdKC$FYTV9%fzSHz1|!#eg)jSl5Dr%QZ^irnTGd?nbZU35r4H| zo=gUy?wgrSaN!`YDE%E==pEST=J&d*x_m?b*fLF`abop3YU$IGl^bbvOrc{jis@>I4Wmt|f{LyNDvMBw{WBk!-KA0|#yv9dHjBgo%B(3@sY`&V&}aP zy?ocl=ylxM>$u*_Gb87PlI@TE?Q8Hov{!WUlVGdoaYxU_m`>uVdwOWB)$_Qc=Wzgs z6383~CWTxzLl_KhXmr{MU+?xzvSOf3H?P|FkkDy^I4ajvOT2}zK7bBy)_-*GDNuIh zfMT)kWqWY-lHgc(7Jg9bieCodQ#q;udTJa6)Za+k*ch#crm=O%psi`cxG`!Eh>WEL zVi%@>l7O|@0-lb9HNzfTW~iR|+g7%KkbT&|8u0hF5je%iIrvZO?OX4DC%fLOB{V+3n zFqmb$I7xd4A_PTbgal;p_{}H`{(g!MSz+!vwGIZ5OJLZuuRTLT!Z*$OkD;?|0Nr=3 zXecx{pg}s#!gzkOxdC99hMjcX*yxNo!RjLE7W=0c{S;)wGM$9uW^`}>!F)ESqsgo{ zFxAlm^!;Wa-#YRnSl>Rxki?JD6oesW zjX<;ak8uiBbMkzXp~SL=MUyp7#=76SuN&*V9>&w2FtCRL19|N3=KSRMNOfSUP!Y==UPjKf*mxUl2Z*lo`^y`eF7``*p$ z4bV)y=-k%c5RLG4_XhQbs5aJla(aWiSUN%Bt?CVnucG}u*nZ#aqd=j^rDB@`LD_v2 zsGIe#dFY~ZTh;Z}j=0+mB}AZ`kwEOZ4ei7J+Yvp4uo<<(o?DPP?7jtk!|rv$Cifxi zWawS1gADI)6E9;o!a>x@mEMqW7&QWAENwW9I!O}S8xFuELh@XP_qNDRPm0qdHa6ST z>C|Xf+B*S0YMMm@cfmaYo`1DD!%jkF*gwvu!02621>s)CobRxte+DL{DHNQJuaZ#fq)EStMWltVk4+RgNEM|D0Z}{ z^D^>B2*QG2qr}77;D`=5mdBs*>&LtTkzeJ~^$eETt-6M^fI3FLfJKnu0FE)!;l&N) zWRbtOjAjC=2hTx}Iy~%Fu|)hFCAc0@rG9mQxB8n$askRTCbE_Z1`-#ZU7g&XVw@cO z7F9X?i$c;%)<8H%F`=1Os(I-iVt6^|H-Ls@y|hHYPe3aOZujF0II`BW%j7*4KSwG5 z?ttzBWaZ8&t0G^^6w{`QF#Qn4{0pUh#JECG;^d5`0Na%Uq*C`&<+idl!~p-*iQJ?<}SeLI)gHfbH}hJr;wk2gvxg!Iv9gSO(Xa5FZPP2x9xyf0F%8ap9tfyL8+`OB5L%P0NnVP+6@hSMYu(*s#R6g2+?(q%h5@RPjuyTr6rPJ? zQa@Jc$;_;3zWe^SEDWgg5p3mq0^xdiCdJP6i*LRW@yjOHL0_T@ zphfXRoT69@A?)D79FdyJ!s>c8E#E=189Q4OC%)g zH0D_bO!Sx9J_pwkaG>wpxjvBeZaYxz14!(OFA~}8{ z$RT~XA$OXWXc~k91M~2)8+{}w5zT)F8M)J-yM}q1D!r2w-x22-=>qjtIr6<`bP$BlL89?tA`dkv!AzsrY{upi4o^{-=SnLTW;*U(Lj#a0yRVA6##nw5X@Jr!TEMdh z@m?6qRJM(wfLV5URLWbn_(*8{1Gb4f`^$fE`lxy8%JHee)KjGKB{O0SfSt#rKKJa{ z-+`38qB>#$^1hhR;7^&i>zCR!lqV(dHrGyfp#27oJCdKg1IYW3{v&}*e8{(wMQdz^Z6f>1 zlhBiq^3Ft5vDIHP;N&=*yed`+%w@G4Q=URNI+Y9*PfMf7gN^o2Gpv160FIE&4E~RB z zjk&w|kr|0(6bTN@+NS3CBI@=WKnzv4WV`y@z~qHc&druK5LVTtdt z>@YZzI1U`vCEecx6FSB4G?x2fTwm}Tm?dQJa!Yd(@KM}xS{VD%B2BVp`4ZbV*QY^hfqH-iD5)H$_$sCZrb(5A zT&1qd9ItP|bmtyrl2oG^-(xWNf|5Q$N8^$y$PPw^%W9S}Y$CA8$MUd?cVZ#h^UbYa z8@wUnN;fc0tBp`3hl?QO@PM31U_`DH`vQ@De09m!3%DBcDcSj^ z;n|hu`sR@x<@3Dd9@EccdLkVT(lbCNIvotePfU>EnMl4sh`@+qDsR7kr<6UNRGAS7 zWC^z6WkzBRetSM98_ic^(4(B(gDn73Y|TX`h&pSK!oB5XR{JiIregu*X@&G7k75H7 zwi&JF%6Y7YeF78`zT$sAO6{%-pT}at7Hr`#m= z;N>h|&BwGTd?AzZVRSKtoN(w@H3MyhIb+;}2^1Jh4!;h2DA5jO4JB3DF<c^qa?j`Fn_L4X4)+yBJWNKO0&3A;G?c*4e@Uy*jDlyDA{02mJ=p$|$d zK`iozO!o*O%0r1o(4$gkU)?XvUQFf_7jtV+B{#QQ+x=}=UH^RyU~>nbZrmeqhiDlC zN_f2KsV~V%0`HF86}WtfCiuun=`LBdDd+vZNRoG}a_vTqGEL!_6KMA04%)#h!pBm2 zKySY=Fa~e0JjCCIE>Iv7 z%Ik^c*%hnDG&E65@+>)=9!5X!VOiNPKkor!aRo|#WSm!7`WNtIh{4P@jSvF*9T71g z&rKu@2oN3xnC4**p3$?o4CunK)fbv?5QLB0%1O|H#K%8MCrYRSFVB2=Fo$zG?^^OE z>vyF(1PQ7|?vhfd=rN?!h7&;c6v_Xr<@yP4p@^b()HZ+^aZ1Gi=8YZuDR z-HS1kugKj)+}WH$b{ynxh(AUQ8}B5=y=q>`d*AwDV-YyQ#|YN25&Ec55y7{-Zs6cH z$8_v;J2*J^F>zdFIJR%mK6WgZDG5dY10Jso>DvDo7vR;6p@3L2^byeQv8Hu|$>*K) zuJ8*(6LZ@ko*P&k>>KM6!N4ZbA_RcaSnd(=kH(bm1h<4V90Eb{V3mT;;}8Zgwc|hN z`?HIylN$&jy@Urgg~xRh&W$4mPiN_T+{U~`BWxkjlyt*F9e5D&Zkl*~-Vm=)AyzRO z#+Y9=9J51qGtOV*sN!pb4{=I7I9<^Q*qb7)ZPUEi7&m0~Td^ZqK;Wl!md42vhY0Tf z0;6I~PF#Ig$d-U@m28Zyn_*-`iBq(YladT9-zuY53>2%ANkYfTajG`m;Q=L*MoQOm zH7TBl&=FLz6wC$CqfAoH%y$w@G#@rcb8CU#Jeg9SR~V=Ea%P+3Ie^s&SxWurh(!%t z!g5p8R(FfEa${D~Z!%2>m_#TC4)8g3YHT91?2ib(6`CvD!~w`*_tJSx}ly4w;K zItL0n+C14k(r&^ed1%vKp;4An><~jWMlih2%HNYC%4yb%incg6d)tax9UPuTXghSyr=QgoYcV~z#LKN--6yT8F zr{fjMV$gy!v#V_B_AR`FwDw_I0cP6+V9RpPD};8V=W8_c2$_BlX~>CCArh6qm(iNztf7l3fh(tNBh9KId#XLw|RZ2>ot zbg_VvtpIPnZq8>_jf@zlpc^-<7IT^B(njyY@ahu$PsVGt!U{*YZm!{*%Zbp#-9Kf% zHUC*fKl5kWHZ(M->84oI`8Xk~0pz}dWo0M}Yxt^_#^9Zjysacx)$Ui{v zRF-(w>fh)Af>qdOR7gw&*(c8tcPw4F#8Xq?#zsyH;kYB#L7NhxYlxO({{w3S>idg$?MmG0gsm~mK^pEr6AQv+#)-=>fg}GB)7f7}^ zh^qr2kkt-0^1ME*2fcZY7D5M&DH0I6QJ&-$-Iz4Fwo8WHo6Xf@YhHJQ;kRp6hdJgXvIgBsqykY)bSc)i4=L@!?Z+Z*#|>5B)&$T?r4?FS<)9p-Aw0d*?8UW0pXiq>ZM57^G#nLNx*9up&OY|5;kD zz*a;@5`I%a+l~%V?NQCTrwnyCCsA?y$mNUAoNmb%qa z@?}6R)2xkeGxOMvW+biVWECSKAcK$jClvmZ$JFyuBHee%BzL(We#jppTuWF|YyfuE zWh!#OkaLwW5$)1I*x^0iQgm%r1%}P@XZuoA_=J(*pd;W)pxU)TeRb z^l#|Godf@^;6IuQlVT?&4xf!5d%;@vUV1x7@hdI;6)L_5>92`@!O7bS7WY@BHzI9$ z^lUlHs~nil>y@`fl?M9+5|5KjXH@R_XJ*X>FAm!4dHO597WQ#L zLc^ZRC1XG4Of3bt2cmi^ma6%Y1RL@celoEPd3Ritp7N7EpczI6slVfZ+Xi)&U-lei z?D)+~KW#XASwFnTFMt2Lqc_L(Gx6s5_3^jg)|Y?p{XWocG#wTbc(9#J$jQwGu^4Eul}A@GqpOnT zxe{I}G9uQr(lc2`E>Lx;%Di9G@a#tGRIPpiWz0=nKu6$J9c%Z)09Qb$zu@{~#L(f$ zqu(=g;mmR_FvT{dV$3nhpjfN(eaCg{c4|IB=7@{5xxD(vhIDuJN zU;G~4%5Czr(@8l@$Z0%R!i;h&Fb;WZ)$H{7Lf;#_Jm^`gLI5hZ3T^Rn0g8Lyy?*)o zxBe+yud3y$8bYe;EC~6ETJkeM7^6jf3Y=|0p)S0McthafEpzr4pe~2ekc^yAZe*mY z79)Kw=`vw}V!1%hy8uJw!s(jcd+J9o>Hqm=FN;Ih=1kKvyiHtWV0#6Q+kId#LpOBd zf2CH5#LNV1vgmWN{qVS5m#N29tj~!x{1^%ffE311*y)i-wK-oOMnB;joGg|VC|
  • BOtJdSg@;~WNQQmTjtjwTR6shXhy(Q<;k!zkInd;5u@d}1h{7|JJx z@`<5*VkrN<8Om$GX%Rm$lur!hKQcr4IZ2i$b7%O9vb6I}h_I>2~kHPYmS~ zL%Bg-$0Dcdd5-8*k{HGBGA@ThQF!0QRU>Xtu{T281d266MhK#u>0m{Sj^cO%X}ODA z<45mC;QhoWKk>;=eDXgRpR8!so?w7CV4X24V-Qc0C+2y#%=44S=HCUw_oPGnr{E2q zo~+;c$#On9Ak0**8f6)iLr-QM`7=k80P@o*QA}i?5n_)sbN&Sr$CWJ`!cp!LQCCJn zh-FbJdsMEKa1;nOLGq%I#m;(CoG**At#UbJy#&1$_S&Jhz=!b(hF7WP)u^DFppaEE^) z0%q39@NxSEb$ZjHkv2o5M(exm9VspI9_tP9!RZo%&z_aff4f{4=8?rS-*#l_>RoZ} zI`_mxVmqsT4n4U{`gW^2m8fHa|MA7rAqTXb`HLI90g#a|EJ!PCf0S3nPC*>A6(D*TOa1SY@v}1rf z$>_)upBpj$jrkcwfYXgym_0_8OgK(TR&&Udc+4Ef=`t;ea)>qtj-0#x^y2Imn8s|H zltzs$P^}@C9?v(w?8YJVef$O3+>|Ma?~t9lV4T zHwk9j9#Jg|aa>s3*WFCG(^&M+FtIOdGf+4*#)BT*dKY?&j?``?>98DC2Og3;6R90s zC6BOd7T&9^0(W6+icZT!M?j?m^QG445UX=kFP~EK3s;&h{>4-*>Vz7UwTE+*VTN{v z4W#}^WQH~F2>I}pMo1*s0D6rnHpRh}u)77|PxvA}K-c7bvd#-bX8Xks$_K78(20INYtJ@F>0`kHh|3R| z&!|h|APYKB<*=0#s2?_|Or@aNk|Yi)m56|thQFv5%STvqP-nJfuf>Jt47N$h+1-o# zCZWRosPD0O=NGh;q7%gg6;RmW=e-y47=-bk_x#i33Vl>C0FEx&VIZp*-)HW;gG7A+ z8Ub6s<$5xH_q*S{WrvJ?m;S4OT3J8?WTj@-rTOv(`;s%tG@z*3CdUO@H31sl7H|R* z=>YrW0u-4Zy_Pxw-zhJ{(*$Ll7fVg`2n!O@MZa*4&N1kZKq2RCAHoVW9p&4E$MB#;9`W6D|%TBAoQ~_twI&H{Xmssk8ns};U5QDPK z(4pE&Dj4oq1DUN~6cnE{PePC@i0FWnV8BCp4N#SfOBv$l-V4$1(W#LW&ebIo**y`H zCt~u5#H2N&UJxm9+sNNzT@cgU5jf=5){nmpdSYMcb6G#_=`ufE=FZEEf{aV6LiIB1 z1F(rr*rRsfagvM@-}c{%-`td>1CfB`E!~+ojohK3g9W75w5&&eGfYTlNvlWQ{+dL@ z`|;8yW$(eA@gw*YJyEkKYWBFS2|3R0k|p`*;zrz~^d*~rlUzq@HO!~m`gB`8HE^hE ztFs6W9_9KdrBM`h`-`+OwP;6U*HmM-ll~i{9urwF<~BjQfx|z_ZyZ(Dhl2W&p=1OV zr&e=O#!TIijc$RPH9mK{nGpiJ_3qoU%8S71>y&qCW9{qP#%v|hR@rN#?JCp(s#TJS z3$!eq+`f+{Nix1e>xmfITUOcY##&6?3mD?OZdESUK0S}z8)I|;~ySvrwK(rGfM(i3aOEzSXD;`bvp-y1MJH#28!Y)HYcqNY z$JTXy0F~1?%dwgtl+#GY?yAkgUzokxIm;E{InrJGI#g_mAV;{X!-KB0u}OdO#5D7c8Cn`_@A4n7vYi=#)nTfqLCcS^-%>dsz$ghz`Gz1 zE8M*l@4QK{sh5}c1b8$yP@Lu8as%b#dY;{bu&_*A!iK_ga0RvE6VVdA$)25m{B&{l z{PEDb0*>gtQx;Q5DSTK0Q{vgPtCNe%=>6^W)$`s`Sx+o)M@xvu55ZA0;R?;$)0<&5 zynZw!B}wA9uk<~NWhVvO5f3N_fMVJXWT^a{N_|VYsO)9~$xycMc1<>UoeD^o?84Mj^H@PQAnT-@bYC>P;xN%ZZbb z7~H)7P1px4lz1|AzKDZ-QaxZ^G@|n|L)7?jPu0DztRE>yI2Dn%2;YWpU(L>EJkBo- zgpP!cpv=UE*@^)uA)S1IAKmtYWnQMHSNX}{qx7H0T^|1XQ0(;K4qtSMl0W{|g%&Az zcZXXjhlaSGqBX?lV_e5|#Es#<>7|KjIe`~pz+Dj#yMUvs5zW6@Li3rdkIo!5}FvN5KQ#or7hXV#E$wWFMY~8@+ za2Jtj-c&zm6c9>DZDyCl%U~gKdoaKeViPgI3B5Q#{_0yKATWInZ`#qz*WbRuU=YVj ze*5;dh!IgUd-c-Dsbk$2Z^8Q$41Bo-8h9~w34E;x_B{xX2hrMZ$029avT%%U{b>FctOLGQ?*ig$(`f3kY_K6dmoqf=<5 zd(M{O$*>}GdMz3$*+k+)u(TlV1yzY>50~wv?r^kK8iL-q1Iqycj-vC|nt;UqGRdYc z=0v`obNZdm58qH)?_SZ$e8}~s~bYNeK7QOJ_#|Wy?YEJqU z&H#!HfBiDLzso>$n1ZsZnz_Pxn|6o>s=ka0qMe|Ql*VE0C!UE;^FXn1{iavvVyLI| z5k&qpE!dI{;xZ)jz&5D{{V3IrB=8S$IXjUn?DEawW?=6xXSgwHHzWzhl|WBM{Ber| z>)%Nm!$3Ywk?1!L()Y$mO^%MgbzanOJy&3_niUCBKF?Buv&=Nlj?=M}rhf&a?1{=Z zFi`n2YS|UN*$9jr7tu3r_8jdyDuY~cpN=rnA60nEUF1C()iCwrBeD^QR2}`(^ZGM$ z`<&~p(ovC@AfrU*AR5QG5P~JUINMPOrrfZrI0Fs&*z^$y-7}rj0UuGiAO|Wp%Yxym zG!c&SXKTo|u+TkJLJ5(M;gPa|$%zl*k}_B~XIvV;)b!>j1Ce2W3Ym#2QAz=_%Ra!p z;IwP(^J(R)$uptVB;^NYuAsyU&=CQA^Z$QCY1L&8=Uttz`gB2AnT*|u(3nwnCbTAv zCv~N~)>3FksL=q(9a>kUulplm{u88X!c-WW98nKhX&%Ot|NQax{7`e}WTj zh(6$~R-}~T^pOtcrj)=pi{bglldm!hyczGEg)44E{t!fR0{M}Ixrwe4DSbRsd4GO( zetR-JKl{`0`VZ$H|MdRi^87cQm2npFxF^m8OtUWcN6b4j%f~8*Hm3!ld2W1()ct4+ za3FfqouE}Um26EXR{c{WsXcQn7Lk+DJYS8?&D~e)1qUV-$yuZr$GNElTY$M-WIYPU zjlnHK&Sf3s?1RXh8|Zoqgsy$*Wllt588)w~TrlIYb}1!UnX})Cs#PUt+8Q^LZ|(l3 z0*82;K##;DD>r3i6bwq%m!htowUA2vd>9QlC(cXSEs+tr5JPMxz_H-wjU%D+DUYXf z63#~oW2y>XT=0BJ4$PoZNNP4PqFvxm(7zr&R~zVyY3l7G@6!t3%6eJcfW!};#XCF< zaw<7hYN32}kCpZP6tZ!F<7NGl?Gs8%^pmf2?s-brDA@w!gOUZxyLn+#b5g$woE};2`ey@&w{)cxWCdUp(I~IVH@R)&+8Re%x}D* ziWGQBQqcrfp?XNrnDdb9Wgv#URG~>!!Xx%QX8kV>iroy&ZRw(Q2P73f)=RKN#}lmh zbSTMEDQXsK{3#V+@@14rw;XiWDLsqEl`YuA3=GVF?5b+hN2PtB_OS`QNExe z2^fed5x|+r(r0KJo8!V&7Q}Dxlpa6`8|vHnu8_?UY0F}TCDTMTJ3A4Y;Ti9A1|l(laZWLPjLk#_g>#M%?nT>M0^{hhoZRZk`^%K1n+{?5*+tS3^rTj z1ZgMdzLY$0tw@N*4>0Df5z8*7B+jqr1@-Z`L|lQcGV2{e$ZdLiOt#7zMR_YsnDsk+ z^zwJFe)sz5+vC^GYaxuH)<(XCZ!fKf023PSC~N&4Li?M_T7Sp$+c4sI`>gde{@2P_ zPlE?#tZxbwbTGI~U(gYjD8T{ER#?=Fk5>+t2;>>Z@w$^Nh|^0+5QQRsP~QFUyeBor zH?AO4)r>5?`?L``c{k08e?SSHh%(>b!=JOuB*yw>y2=kp2GVLtz8ut+jGDwGlGvNc z3cZr)le}e#L>a&hUeR*Xuvk%WInUQV^K&cF@DmJJY(TGdy^KnkG!>B+`8&00oJ4kC zCC7s`6S=RFwsC2Ds%@P)aN zS7(tfVt9Xf(L_37xX&7{*lO;hZ8oRtskzGY7km*vo&<&VOTl)_wKqZ2wV`&h4a znj`tC1zPd8*XkQ;seA{M_a_JDt3p;WC0`&mI{!IwRa~nK<~DR%%Ul;A!IES7gm;j6 zHZmqGI5G*GYNl*z{3*Hn@nRpc4_7)Z_4UQsDH5H_vy&S&3@flQr~loI0L$s0fJl$nt>oxqmUaGdNU|K zze>QH6KAhFz&_&X7kR=p(NU28P2}oFDR%x4uQA9*r9VeOx9GE~a;-X7RR%YlL42p% zvE_TwLWh{*VvvGPijVS@W72ad0*Q4HGPZoFBmLJ-z`G66}R^YFmVM9X!~M@=-n~ahz>qd_4t8K!Z2%K#0Lgg;v1{Y@E2-VH2-ArtvrMWU?_|_djuE zf_Z>C8Qcl5`v+KREaDCkg#bw2jV6ioou??8#Aqrg)vF2#4e+JF18DHWo3p#nd`z<+ z%Kmgk@A9A_(+>D#v<&$=aK&(CBwY7APNw$)~NGeVigXlyank~;sl zfGo>=*QG4^JBvMcC$iRhNJND+E>o9qHGQDM zSn~S8xUgifYc!;|9f*e1A<;D^DMj?(Ib!PQ-b&6b5J%RR5+5j8dRC+=7EmOvz&vHd zNTNw*es3Fyk}us3}CD2TibM&#`yB65XKO5g@T!i0mtaPXMedA=zuuc29XtUO03EYY8nMYZGnM`t73!#Yfvq7R5q&A5mloSu^gq zb9_hw3`2)v)uoAwr2AbXLy@7FfrT7J`Yn>OHSIH^q+#UD!>He3-I^5v39>*gO?fQu zr=%(tia7S(OG#((Yf|K<4_{gdZw<@`25=3Q9t}*SY!v?C$`5CzO%Ph=w1^l2yW&)A zqTR};!d^osBP#u?R1H$!8lpX*wuBtiFgG2j%=)N8$;es&SUWV4Lp*6k1_oU&6+(+h z*^-R+4KAOI9&@yFjBABbbCsC4OypLgYctJ%?Aj@z12x)n`gU4tD6&YSp)Ej0oo1$K zcZWVv{=^Ed)2tFKfOUqgDk{&xt}h!BKG;@FLmWr|L=R9~mybY+q!ufXZnKZ-qmUtq zNe?R2K&OM!Fr8`i`a7jL)`EqRoaTak6~YobHsSAuvPd4qx!~ySb5P}EZ6Y*mjs{GO zu*jj0NGW_3pY&IdfcvcUQ5ZY|w1hNFf3MMzWmlu25L}JE2lhD4fUAy z?Tg)u^hIU^*4$WMP=%t$HkBE=nM<9Z$B}Zp`cfCF6?mUWt9uGL^3BUvua4fn{#HqH z$8V0_z9Dbjx-|9X)!T1>_wx5g$4(*Pm0WJku&TjqR#AUe9Gl$gmks907_`YHs4=al zOnnvfU#+f5j^7^t{^<3aw>Du^AJwH2eWfjl#T(7rB(n+CQ|#Uko?Gm9ZolHImJp?l6K*Bn3mvduy!ZE94z5-O;X`sqda%G3%~co ztZ1bJxf^qIl5mxNg!J;lixA=h`=AY37$#p0XLOL)ZV>G$-KreynYPVMca5-?5E;$M zk41Ribyb$`$(a+f=tSt^r0msoqyUm#S?Ummevxg@Mmv?4iyd9Ejq?i2v^xIeG%zLO zN}Wk2(!UAFt^%3JtXT-T2&O@YhHkBV{mm;26=cA{=*8WtTt8R@0~%+SFKn(&%D?nGL^b2WNjTw2bdGKA9LZzKtZED7e6yQH#dMN zlRdBUx7n0)kA^wyEJ;1{S2|IUlqp7!=Re|g>~pnEl?N*cPy`URa$q=COmgW0*Nz)M z*hCu=8lHTwsr23`DR*?qFXUbE1Qa?cv3xi%8c)$z(#)EaY_&x3M+edS{F{2e98jeV zyvKmjVG<5mCZzAdT52F>v5~KIzHZgV(n%}bH^1v$luw-fE^0?Jk{DxgipXL9wD+sN zNg{VI8!Pew93fW?Y;T%FPL;i*g^zjM8wfY7NU_WgUQK@4bMr)Rj$a>t`|Zm*4|4pn zvH|OYO8||pWtldOLv+cYeiH`2G^Gby}O1LMW>05ZRXRFGZ zP?^M)-Wc1IhT#uEm7;EDnXm(`au3ojQ2QJ*fJe=Kmn!Mr3(m@zE4P(Eba)(dRDvU! zCw(t#CQHVAl91HC&)L9b!Z^YUd_+~NT)#V{gQAj?wW-{vvC!4_#vJeCi7(pJzU)F9 zEM4PtfI1exi&Z9=Z?O2yH$aP%`Ph0sX+{DHZ3_e~Gg}!~hNm@DlbE9yKbqKt#zbt* zM%79~OJ@PnA##=o*4l3B9O05mf46*!aYAUw5cPnR{6?#?eNYxW1w$^))RHE_ChWo} zp>ifzJR6_Q`WA{4iyBER>1-5(0u#@KgbMUR+)0{ji1wH}BO&Z*d>CF`MzXPvequj3vny-wu55So$_P2viHxgwVV*?Er+Azaq z4p`&k7Gp>S6Qc>35f3*;`!dZw`?A}DJySe%I=!tVM1;AX5A4I|vXRMl&2K`MlP@#; z0-Onz_&D5^w**eXf6}2&@jM=h#YdY>bs)<=5Y|gjF>@-(aXuocFOosWV=Sa}2jeg3 zq>xn@m%mK$SrXeemr9@)@iKiO_whvmNU79Lm>b0%Cb;WcBiG}{qV+Y41qmmsopdqK zYa;R97kR!+K7`L7CL(qm*~6V?mtyw2N|zoPSRPF&aA~PY=0`wz7^kO6?GOl=G%OtU zwTK8+ zdROhp%~~eXwy@56=qB9eoAWC#ui-P8+)%E8AqYp_7~Qg5xkBqs&B3|NX3VG+$Onwm zCnSp}NIqQ^5H$J4fu%R2u~M6*l#%1UBh{CqRw2v~br(AxR%E|(X3M=1nei}H?uNU| zxpCWew_H(=lQ{{;XiW5Ynh&^PWVECbaF`U}J!T&a`}H`WcR zN53tS8<9Z3l^&P17w|F^YBn;qnfh?#J8}L-NiNYA!^Sy(NspQuCk94a?~qyKNd#`! z%RJRKh?3;A?rZrR!e7tvfi+-~&)l=*5VE!Xc~Qj&VBY@oe)MPL|3N69q|-k?-#5Me zG2a}>!jzIDHU$6-{?_=B^t6O1Ywm_3r0`n98s}KcM2soZoIno}wVa%EAR!}66*cXbiHcF&Og)C0}v8slN(>uqwv;v(2PytGHf!Yr5Tr_l6 zu0bedUE%Cqk!(!+&yN~TRQg1!6H5*a+gG#^NyFq^LgxJuk|Q)HIrHYlsaLSY zKhPEO7EpeOJ-&dTuwE?XnB+J~{*WjWVkhb3gszK#keOq+!kq2kf1PT4Iq4OthDH?& zTO!3CElKzjVtO+ITo*0-F-zeV2-AT@(lVnCrg!9xYl&i>s#WRA0gF1|D&pVGUF^iA znFP%^LjxW!gt?PTbz~hxs>b9fE@mL#EOny(G94**?=zvUv9An$^S+~oP3FN-*IX>E z2TQLHo`XVpkD>F@`VV2x_3~yT#H^Iw)uA6BVMT@jw`Db~N zSruD&<{ZEBYu{Hc>F$cZZ$9Fi--nPyp6pAF$4k9FIAzxYePxs#nEsR`pIw-aV^^K3 zc)bOVZ+!~5f2LDI>-uuve|ku%b}PdW>`f_r)fXj6;Ftte$cHQeK;)GB*yC1UI6h4`L_h=n#Dm zql*a@cieA6GqQr$yFsqF1wRZ6kTZkC8a2`zj@sh1cY z8$kZH77p_vopR=FVRCb)E6Z5epej*ZhJwwlWM`G825>z|$N>X`Ktx_D&U!o+UQnej zl)zbiIl@UyU#VXsP`^=e5m1~1-9;b;C*kXnvj|C6e74EYq!= z0mMV~Q_uG;zbI#gA)4fQy7tY;_IWprb9z!gTtOmN3iq^4^zg#y1hQ1UXsiojkOIq^ zJ)OGoqt17ACmCbUd>b^A9G!$CPN};qZ%%#iSjc)T)FCS!ly?k7T4gB}G@d#WHTq$d zeU7ehG7~stDBJ#K6Tv9ID=gJA3Vf~>+*lnW>*%&0Vj5v4y2-XYG)3FyL;4_ov%wpHk& z$%Q69FkM|%z?WB(E6r95hRq6SNTsAS+Fq*uS z?o(&Jn2RiIG8ja_*w8>5W(4OucA7)2l@TsUh`-v*Vp57ww~FlDa+;7DEBE_Hw~jA*C=yXH-DW z;pE@0AjYP`(Ap#EX^}C0kXF4M?deqKK$L`(a(F&cfY+7y$`{5KWi~%f>UzFdW@h|iJ$i# z*!wP-jL@y$MKxK>N_?c^{9RtTsjHOnluGxo7rAMmF{4zQnMp-frQI(yIMqRo3=pn_ zH?_lnSZe`Zp`w61Iv31S9mpj!KT#xGK2|KAD8>*(6<|^819~gt`U_&6;!ky7C3{6T z^c!Lqi9jb7vszAVKU31Q=A)DXu&^9lANLheu*WYBrY={ip)RW<1@k6iqQn>HqhkTY z@|_)_|LK@sL~^5j6d8jLl3!I_E1=_Ww&~jJoj8;ruYz+OdJBI$4uvTE?OJ;dA}*a# z#Qwv;qi-$&VRk8C(qLN=Y1ef+Q|>#2^=Ii+U8!D@1W=|bs7S%L4<5BCd-@=1=#c?O z$>@Y`8G=0?@1zb~t5@Io)dBJ-W2$DOJ^Z9>hTG66(Nylz<&VotIgxt@zAbNGYy7wn z6{1J~ENA**SpF}tuTK?GQ?-Nkn?-qzX@&m~V-G*v@s{9gOLSw~YonckC_i>`r z-|CPvwcjQ&I2JNhS7@G2lF@qPa+Ywvp0}dsfJXO)YR%GOjNzkecg=gu=ExOmi}3+Q zM6b>5i{~rpaiw+^?@W$i#j|)m;TUjEBj@wY+?H|CQ}_U4tAl@quN2IuxD~nebo|!R zCeS`teZ4~0pRn1acy*H#gDW?Wza--q;=g;=%XnrKw7?KY`ES88A6%TtFT&GQ{yEGQ zQu0oVvKV||_t;5pPqVqlEo|-+7v`;i9ZwxB%zOMSiqFUd2ofJ{C5$yKg;`7>J0sQ{ zisCyg;xwZ=&06ZgqHVygIUmtb{~g+9#Ldg<$Rq-Iz+DbgAIN?nZ(FIu__S!CC4(6O zQK|LikrhM8<`lCl7t16yTh0?vJ&~9SzYemIx8z`XV)ULEy?+iy?}>{F;-YHLcauDV zwfp$ZTXl@RuWMiojiRi6u~;%<1{D?|7IddY!WYWAkv!-+lG{;J#T&Hvu;NmoQ7^za z&0YZH-k=Lz3h_Xe;w2RZ=fBt2f^sR}VNNAf{@rVHDGjZ1NgI~NL#Q12mlbO}q?Z8! z)P-ty5+n>$fgCnT3kvI_}~4U9YlZr zZ*@Qa`seSOmzK&E=a8LxgaI$l{#=wXMtLRU<2P@PevkiL09O>hhF3&z(!c!2zbyV`{8tGbM3K>~4tSNNUR@j%=V*a|oU z*pmMheR}{_aIi8`7Pa{jcwva62NPnX-kPo6pnB3Oix86);GFkiU=aL${^hoe2lg{y z5kk`-YEg-|kOx*}l{M~|X?ty+-o1OZr3c0#JFwQFv z$ZyIS9l4LD$!E>P?Eu{~WsbFlm`p&L#c4s*H zXy-Kg%6MYBlJ}e~Xyo8(Z>MG5v% zmYA#x(k2n;hI+9!1bgDDR8zZK1gS^yg=lB3d-jMMRF4yI+*?k42b69NenX3A-!LM7 z5|{V6^8(2}-()!@06Rdy|NEQhU!x8v`baAfOlvUenfrd@Iaad6|2f)^j>f+<)?ERl zRmY8jC4Zg*3Y2}h%A=K}U z*XION9#XPIS8(bCQn0zrPKbwsd-Ydq4=co z#YfB}g|^f`f~yx!o*RW;Pe|L{{=%ypLI`Y_rtooefdZ$mqXQJ_#>OGr#!oa>E|R@A z+Ag2r@a*d3_Ee4~E=~B8NM}@>%}w9yd734)lj3=p``%dCRDYbEGIG|6Id1NyV4vEq zn=MPQ`)+oNNvD29@Njr57X+LQ;X}AN5y9S@j@|X>>$@6aC+d4dmN}?O*P2!Y(Gk8g z+?Cl9oyFNec!zlB7w=dfXdjGu_Sod}-Xd8~XJcPHO?L z^)2KwaWNR^dh9GqV5Ar_`9OIrashYute}M4?g}gpq--o51B3X&=IV5^CO*Q2U2f5* z{M_Z-4Igb{kH<)47vG3}$kD(^;JP97d8Z@t1EFCG~?Numw)pe?nes5mbrv8QdXgHUuuwLRX@=GW7)3eQ{`XT zttnP9ZPBO+#LSi2339EE{0Zt!JR&Xl`%E)3cP!BNUnk_5vmLp1M%E6*wiXK`FQrv+SBG8b;1s8Jz4W9c2inK<4J*=)l~a0M zyQ`yM)fL^`VC_cE`-Jd}J}$T2bLLLLzYoHvDXUk7f#R~h;?57mK*HYPvE?Z|Ce}kh z21dMc#(60(V?~1S@FGp(UPP^WI}9_VPyA>#2mNHfX4(MXA!BTUMkN(_d1%lKL=_*fqE30fnNc(lF>CnM&GV1YMN0Ppx|GeeZnd5;~QBK4S%u z>I(;}3E7}(Ivu1{#Pd+bq2L(jblJ2176^%6y52nYyCmxaIdaf*ES|>ZTk3ahiX|)6 zs)0Fms9o6fP!HGMy*rb_@a&Piu!x$eQ66&!?;puC%Q=JHJ3m?T*p;`|Z;ktPf3?_e zeb5K(PG;&S?P=y;N!u%o3oPYJQT!P5DNz0x2&KSOJWywdGTH8%S)Pq_T1vyZQ`bLM zMLh29wh6SYT6u)YJnltro6IA)X6h;J^v*vrzJp$#ng&KYchw}=U&~ihuEv%wn_uJL zVYOjN%>*0ctUq>DtGeCttiW{c)>TqIn#ZZ1os#~VlXzZs^EClps(!Ab>8^^LV3)tD z=XGy)Ty;OIeXt++U0>-<>UbhT+<84Td9!75lEd~z@>3*-ldtQ-YF}SE)?LoN?N>(v z6Lr2VhXO5b^mPpekEGFO;fo3)(LbF_Fp7a2dIZCw2RHQ{)x9`GlL@OW7DubeeEL>l?W^UQW1iF-h|$v)*~BUE9P2fBfHE>rUTmIpB`4#Sfi^c|*{5vykr3LQ7(JeT{Jxr7MKNJLPCrP2uH zp+?KfWdpoDy%|QsYr~njW%2fnInXOAt&1Jc=MEj8ksHK=cj5GlsU1nV4?Xu^+baCF ziSrhkDA;qgR>-IFj&NLaUFv0^R4SwlGbJra z#a*tm(H6O^@Xq1bvKT>EQoy=TU}K7i^C{J3mpNhdmrKx5qRgwmOK(E?Z4}4ypBz3Q z{*8}{`~!->4{&tx*B6z^6Wy1@KZ|?$&uU)YE-P6jMc#c$Xs{@Tv*seC^nqhkwz(cs zh8{{<39we;sp9;V(yUToRgZdcFC&@wgvwg%&?}MiMxBqdvNdP1DDQiYXYZK~slQ6F zvQAnqN=d(;gY)y!Z8z=kGR9ITc_z{t39swYZ_mlpveGqW3R;TigD3-JsfBBh%yq>f zA;~^_i5Bc5`P6X6GrZVnWTRbrzQYwp3lSnm@x-D~WBX#u9$e?n=w?q6JIQE_;LAZ<^6kp^Ty@6C% zO;SeO>k}z^B4z(zq)b5d=av|;U$DV>|9E>@nv#gdqNgdLArywBO@5-i|7^5(=+bXI z5#A@l`@};&(J>#o@Hh?U?c0v;*YI+nv}tYcSnJ~aEl^E8V_C7(&}2c#ydXF=B09p7 z<&ah0rG7*IX6Seq-^J1CJYS6?ZcaIgp%lp0+IefZh8NHbRj|f4R>!7gPX%y03Yf0T zp~zc`92pmb8_wuu#GXFWi2}i@R~;BjpAECP9w#6be_x5#?#DUAj++XC^%ImES z(xyI>$aKDp(GPDI^g#l{phB>+IDADeFfStaq?a4H)wp;Sf{{Xh2Jg{^TUOB4Pp z8uoc6VFoDfNmfnIHVI@E-P{_JS=~F+U#+kO)?iz1OD2s9+3?N0YVq7LmNTlH?))EO5hZ;!! zgvvqnWh<|w18d#|oRTSuJ~vKH;ykrX#}W1L^G?DEwWD}eC$|aa{X4heGHhEqX|Aj< zQF6cc!`_q}2sK@<^x$vS+1iS)R(!p7@m1m-G!G}=W^ax=w}#ug+LpzSp=A5-onaPQ1y$-qs#d4OJPII@GEhj0=n^@j-~4)%PFfnMhOq zL%?bdgU{s>2d5YB^1dmDC8|bNe49WMweKL>x?2z!_~1}6H3 z{JDg9a4U(ID2X#G+=}5=49_Emr*}s)7>ME!^*kjNC#5yZmMAZJw#xOEA-A5YmMvED zRM}aBxTkPyQo@+3MK@qa1L`?ClV7#YtXIi3?+1W5-LlS>b+)XtWt}bStjao}&?pH8 zgYb*RM==f^LD}>v@96rB0^H@a$Nnf7v}$b25?hw| z4QBh>3)cYW8xA$8N_Z=j zWjyv7P|^s}I@~Tam>T{{%c@&e-LmT6tI0sP8_Hgoi(Py>CdJEf@CXW?ut~tUb+EMD zvgMXdDz^G){Wi5)JD1Llnwj+~ex6Gzy}l3F|H>tzL5bEO?=wUwO&v5%=WMNk+ScDJ z14paLnoj)+xc8$i4nqXPN1Bbkw1T1)6s=l455rdHaWQTUr@3X*Wo)|lU`OfU0)7X) z-_fHFZF5wSf75{yalUk55xaeB%)Edl7t_7uLeI1$8o~^QU~_43)8-TGXJ8BQOf3sm zh~HbjO7d;TDt&{lDhx_FM~m!&ZfI@1R&=+byNw{sDHp#)F|<|->V$OFGi@}tWr!_9 zd|ghj%-L)Mx!(#K{t+4lKgTX6TpTpq-}zfdG2q`8erL;mTlU-Pqu&mMv%!nhP3*!_ zPK$`zK;s!Bq?EO9Cgt(Rs0WR;Yo$B>7@goFt!muz=9V|VF4fpPnyqI1&1l9^^e_mh zoc^_jSs~jA4jB09lK^QByTK7lz7mIi%r|No8GuiBfJec2*%JDb5gJ?X|3tQaY7EuZ zBP4BZskl1f-kv?o@`~VtkmWTP026wq6OG$Cp5EU>MIT&)umtrAMDKsn_E!5AL{Inx zKiW5g(SQ*)0W1_KZ?rLtMtVdLy@M{r&@6QDTpLJHR6|#9Lq7ik=&RJo0s{Qd*t?4# z=vF8%K`5Jy@lUapTjzMn|(Tf13KAgK!@7eSWfq z-*3fSYhk^z7NVJ>DK{mXQTV*ZV-KW zl@SZ*X#WWsm)OBTbW0O&RZSIc_R}v3F2P_4u6=u=yP830LJA#5hG0nm|7}y=%UgNq zN&8)O)q*oq5^KZD*}c5Cw_8cDOaWizVHEUmwA{rAj6dKL7)gVZeXzg3yI)`Cv=XzG zn2k%!{JC$g5!Pog8a~0svdXKU(O!7stGK|}8!B_ub~d-S8y6WbD8)a2X@NOjR)Ssr z{MWW8WGfjKiZ=VB-W}`SSxP3Tp`?a#FwPIdd6y!X7=qE9x)z(ZfI{LZu z8_X-}LH=$yDp5-Gba{L$y8vOg$6c5@S$+>^k=Dh|3fm;#mSO*1v!9S}@U^T5WEyX9 z8q5+r=OxKeNRC(-DK~4-j*j7#xGr`~9;4797T*v;ThXBnhTxI)Cs2t4B$_!rt%39H z2-m6qTFY@-8Pdv-R_UJyiVzRAiA4B~HwALKdSsm0+B=}(oAdfyJOj(>zDM<<<;5=> zrr@dfc%BExg%{ zhQEFFSda9ZjfUa~;5_-`>Jz&9cBU^45YIL(W5c!t3~8{04)LTJDZm2Qw_ zAYj>N*B)TvOwc=su1id@RjBgm9JXcEt=wznUTX)uqzo!_K1iVj&De@+RD@T*-7M3~ zAn8uWbzlbhiKUxraZhWFh$ zV35>fEd)}ltGDd%D}l_~JqhY$wunUE5+A;r_+0^-wIB~g!?8a}ptWjp)eS8?={UW? zyE-o5Q{s#?gH8Dg`13tnBc5kNxFvLDD4NBtbkwHKxVe=KuM`6o_|3J`c1Y`};g)fK z0#!IA{zT_|sv8=Z%ofV* z>#0D|Q~z|-8%I{<5h9e(MgW&Dkb4AEqDWx?Y>B=w9SUt%#+zG5W$UQ?eHzG_^M!wH z1k;uEB?`!EXkGYNJ$K&LEq8noV`O6aa~TWuA|Sv#9C(U)UN4?%eFIk%cEN!{nZ-X0d8ooNZJ{<#F?kk$~e8v<&Pe!9KArgh1z2pR=o6X=Ec{&4GX zyRu>My0VUM;I?3Yw`SfMuF-E3Aq&@&d*fQ$4$@Wbfr{xJ0(I=}kW0zxZAaX1v?J~n zwzB=YU2$8<{4%KqileyX zw!*cQXs^N_h+C65e0j#qQnF=QCch-h4R}Kr)^MKg>GG&_SK}o->v6Wa=JK(wtF@{o zEjMqudCSdTE)`m-?0f_KLnX((D4uYZb1bE{siNu9x$TsiFRkhFZA!I?5AF8GEo|EF ztnkQI+CzH*z=_(jo;Jz7wb^i4))>55~uhlnSsf369c{F&4CNRFCL)LfBe)N2G&jNg}I6b$S zuYXgP8nKZ5-BRr!-ES+?-q;pu%Fg~uf9Q8NpsVTr_RdO~GVyVcOxnINFOL4k({6OU zXffbgIn-Kp^X3d{N94_E=a#}GcRGs@hB|>=(P&0snQ>V-+TR>j_orae4F;`5Xro3g zPo9S<2LAA@47CYwkc?ICV?HgW>J4XYE`hmkTsF`Sx zKKL6=6)!n*c*5|OTIxWX``wDBHld;Mga%lS(*6RdwewBhDpvQP^PWooU$$Gb!7f|a z*l$Td{v6zOreo!qubNz)KDCq{4z2Cf3aC~<&7D7;JDrU;b#0mPl9=(|{82CX;>wG5 zU?yM*@WI{&q&G%A2oaqfUC>;I-9S6lkU#&ab9LEd*3oMK@{R^Qd#yT8q~v_p#|ZB#s~M*|VS)HoUbSenL1+@VZs8c4a>nbP95)zR z4EW3_2nHeSBj_&=Mxs9nqqufKIeoj+4TslwbTN0nlv)s67)OF+`{T0<=@uG$LtqtS z^}LdfG16&F^ri-Xne_r7d7!fn zX(s})0gMqQ4Sw<;aO`}U(@>_P#}Pf@G%dc9PfH+#g%ip`?#QMWrb{LZ4xggY@j# zRu}p^PVoF9mjYrG#dii zdy*y3wR)qt(ijY-X8u!1uc?0qPck&R2ZER>AHspf(|SCeOt5ZDSqpdy{d7lGJ{d{g zmdcz+mNq5Q*O)Z;mk6iYQWSFcBTEtnW`0a43&b!a!y!x?tlSvbG@eA%RhPvJtcPSl zt4e0;>sVCwY?qT%-&@@1gp~m!Tg2!`NX99A?lddaBPG+llZ&I9<8>608#mXtq^cYr z9eMo5!YR$LP=te0uaWzg5NMd*$+$|DJh*Mc*Y|m=nmeGMF&$C(}s~D@pko)nL`2iY!8<#&!#ka zuk)LVuHx~2h7CGgeZw^TB0Fo%x|ZshHy1}|=chLtCr76jSC@Z$g;g^BRJ&Sy-w#mV`tQs z$QY{yzR8@viLFDWb1 zm$WfV)mHSR7i^GVMf0Xg4bgG-i|#NHjrIe@U}BrMfv$t25L`(9U}H+%IR}cvuv%v`!k!2_6WC(;;Z1zlzs&q>sh#qmXhbtXUtfbdE#)B+y2O z9=8Sa;>9*{wd7#`3+nZE(U+58Y&(ND0$!YD4n4dX$p_TpgdB?Wj;04OlV}teF79y} z;{##b&oMUPCcGK=H4C$inS|Eg7)zSc+WR7?#pz6{O@1cKPQviZn(Fuw!a5Q$$Qf&P8*63o>)uL1@n^5DMbF*e;{R9B zGq-mRZF=0c+?vsn_O~*>%A0rgb`B2L>bwm{Hv0C&9UuH5IE)X}vF~-&t zbH9F;mn5C4`<9iwC7iI#t0^uNa%&1vW^}!w61QyaRU*Pc12QF)T!My-z5?<%9DhNU~1)^<3KhFee#943v4 zs>46EgmP>QS?p9Pu9~l}3Z{@k_(Cv>F_I4ZKLBqe#p3mbh2O#J2{^!;-ql@#BI6li z61wCH+%>Apue@jz%*(;&P2icEOMniRk61_bar`g=T|^zd)LQVsWgd$Z?jUu<(4YAZ zTpAjZng|>grgmiqtge;~_`AF5^D@R%{plJwOkE#qe$dxCCbgX z$8aB%F0GnJW=)}CGuq@d)`qP(nWG(u}b1$6W1oLGqaa@f=p;<=5PVi)f_V%I|df{_l{u;3bvX6-# zqXK5ryTza3e-xT{Plaer?|ZiS1ifS({CQY25YqY%*~5Xp-akswY=fZ*vpRG}_J5Yd zs+7&BNOc!!Nm22Pl|Zj73S+4r)uqUpvR^Jq2A?%ds*hF^l^ILo&2vemTk~~R6os*= zzmwn|@c&>Se8;SlU1VtI{O0ALX5=^b*Dvp)l8)?_1WKfcG0j+%y_*KVHWzn!jPVq^ zBJJ1d#dLF$U8AP4y5rP%my4EvOryk>^8Oe4oz@Zx-C=9t0pQnDp+ihzj12EuEY+8N zGxS+!vhQqN+7)`6L{|6CO+*fy1*e(bQU>LL1*>I8mLLbHUR9kPrzo_zOVtOe<9qi` z!uXM=x)|{o5(*KVj2_&C!7C`M=yF}{K;ku&VGQ%6Ad6NhXL4;W&x^sd^FCc+buieC zj~&r|urS6EqYHYc0_|w4i9z13oSp1N>ZKV^pCvLT=&=~b8)=|Ol%w(il*xhlqPQPT z2R*!-${k{^_`(PSXp&&Zd3#&C>t2v_H`ki1RAYg3Hg?s?q^>lsy=SKgJ_8ONqh%G0 zETYg22H~J+OW-u9WEx^WErt#F%mdDLSal%vm9aVHwMSRxqxKC zJ#dO1xZ??INAh!xQQ;2&bObW{5b|pDtYj2^AL3G}7vfH#M<(Gz-zE_2 z`#fBgM&N`9$OCf(;5E;{n&Mf0W2rdu@JaT`&>~DnW-p$et2o8U;@LM|iNU5cn_rxh z?w6xF;6ry4MUK8-7)PbLbiOgIj4`qYgn#0A;5{CCo|Tei9sv(xNLZW>i8bqnx0$PJ z!JRgpAQRf#+QZ^m_D*^n7K12q^ER7$^}-dE{F>KQQVGkX^uVX!IGR*taGc44fJ5M# zfq<^A-#EUy0d$WDg8Ekewph}RU6QL3Vi2h|@dp~UVmBtE+;;H~@tsgYBxn&~1gCRl zdsu#W(}yfiWe(5EcS?OrbryUbfIkA9KtXwNA%k8^~cppFm3pSfTb0#9(Z9RKt?%-FU4U&&(ol41zr(&zVFogtea2vV( zuk1<4t}(-S$iu9RMI(qKk_v$Jznk;tq-MQk^JW-peK$jxk?}u=R*OY}jw1c?A(amfJO`}9=YQqZ{T%bg6=@1{RG?R`P!v8)fkEL_L7&ek8cG@OS_l=0y&rT zE=uBMjl_NM=Uor;x#g}xDPx$Ie8ouQQK^2~Tt>kDON9beA;WPDb^Yq*Vp%YT9#o}f z&&3on**Zg?Rb`*tUH2GqW3adf;Cds&E3vs81BWnR;8!=!lIC^lR!jW{@8CCa)<3+t zn9^(05LQwwsXvm;pgur30lku{wMFp=s1Md(Owc9k6two3360+8S((zDedIMs1 z@tlTOra%FLYj6Vl6+0!!-6dypo|a^=K`S&9kjgK*J>=i@)=2a@g*XT{{>2P#P&^qiB9eIPS_U^%!ttUZV zOFE99EX5WG-exV>LUs-MU})wqQCXU4$j~yKS7n5S$hixU4`}*?GPD|1ByO9et8%+L zOyUEMVkc8yh9)EMk8VQPSR50BD*`pHwZ8^jLCAGKV2hLTGFZ4vOj7KIf*id35Y#a% z(*3lRnIL$p>_-UegC;l%ARUhwv_Giso<2d4ZB-kI1l!y5%DQbR-q9uqh=DZb#~! z)q6u(gtuj^9Tlj2NHi9O743CMWI){t^TwY7C86#Ds?Ie*=9=p{lc8lM?QY5^0pqUs z27w7hAOS|s_@E{eVIx8wzcYhcV>ykvMf!w6A}k*iW1OE-{)mn2J4mZ|jFsm{Ue*Np z2zmlj`GHl6(&U2NmIW-oU^q@PHDp6b7|3oG$RQGf=h-v$5uCu~A-PJ&#u=*S!jcDb zdy5vwIFQ~rn7GrkmI9+C@JQDPDKmsjj9w)iErk>bjH}C?V947hm|7oBZ;yY1emSQA zE|34cy}CJF_b#qJono!@M<@+C{p>(>eO_ z^!)7N>=yRJ*uM**q0sZDcMb;VMB6vY-i%dwD@l;_g_xSf*!AIXJRo3p@8-j?_wMcP z;bH)RTa_lc<39zpW*D;`#z}61kFT!(hEsWTeSLa)Qezf7N1rTH2uchUC(~PEhG_uc zfv^j?R8hu)RM{hBUQv(>(lj4_08t%`$s|muX~!M(3oou3sc%c0Wa(a3&71~)LT67U z#?Z#d=Zf&uFY4?D(msa0_ZVTR=3Bw@0m)tuw1+WUCAau+{rwRfu+xrOvLAq63_L0` zidIlt_yh^!d$(6t7e|+W(}^UMNVp6}8III=9i_NXlrz9dzvHMAE{U%)#IP|JKuFQ4 zC4atcv@yM5_IN*>{+u1|rgw}-M7g27eaxcN^$fWF22BAsD&q}7E|Y&u!wH|QSUM#y z?z-A{?&(@ll5{F`J2RR~@XCY9q|>PbP61=rySJL^vveHRz3Z#%U2+L=---@EkpyeY z2$|$Fa|(*3blFz0O66Tkilw( z&|WSjpF!bAx>lDyqK56!<_L=QUbA<`5^WH_&&Gy$8)zWaxd?49+1HgidZv4A%d)Op zg_+zk4iSF`Sw&!JwLAiQ`o`{R#iBv>o}(c~o@O=ax{(W`IrXX+9cDbder!8Fr;|xE zksdFp-Gl83((jA?G*>sWoQvX5Ru=`~?Qn+VEvT2!6LmJ`27s2K%%bUnLedh~3C(4M9`{L`x)Hpas_3+a{YEd43MB=P@5yUHRfkTXJ@F5qzP4=Z^G12g zXTV`w4bW6k$eGrXz)x`gi(ApvEzopa9|>}0zF`p>2V=Hmh6Z~UX?>ZY{1_zjZ?eqp zlC+ll05yjmYr>-b>+LpJ5)A(Zec%}0Z>{tmhKXwk25H%GNEkjl5ZQ^#8N~H8?jMO4 zsfzwQhWk^ulen4u3_C&_09{U$1in0~>ogLeA`Zln^YFDoFOIZybM)gBX zs)TM$kWnn=!-AVZB87-au1Zd2O{x2I3=#>PufZ^TVClUtC34e2II`|bhahS7;`DwM zqp-uDmX0C>oe2V%{^YAKMymY$?1!${VVWYuwpd05YQprw7m5fik78$dE#b&w}6^pt60IGO%a#&WgHjvzSIfhcP9~4 zsSH8Tgb?h4`+9R@K+f84ZZ|+~7HO$5)14@!W_~&f!|c^u^QPwa69}05=kjf+XjC~T znZ1XZ`}7Vxz=m{gBxQ}nivu&m6j2Fn1f^xZ5)sQEyiEqPlmwjQjoWDB9Me8lZ*R}n z7@QsRS2f%!4r*O`UT|cFGnfXHwn}bS!ISXt1V#ke5HrBbCSW7rT#&n1O*&=a1IP?z zn?QCeg({Um2h|4Tjs~YE(3AiJs5M~>b$rr^82352LmSK~;uZY3u6Ox(a|Qt#Pnv(D zx115rRawYdYE}EXnn>_0U~_x|Z1@wdRgE!de^5H zYdYr+cdv|p6=VB{JMVVZip2>uGq?ej{ahOu5ysOl>cz?sPNk)&I9~#f>HA$-L0fr z7ZZBWJGsDKB+`$hVe|t{nnH?D;1xw8$Ho;mnJPg--~AFW7d8U)-Hl`u|JlEP^pB$m z;g?jldhKe-qfCk9?Rh6tBFS-1#W1;^1jP}38Zt?7jP=us2@=l+{0K)nhRF6rARa-W zLz&Wj_+`^OMYRR(8Sj50f{y{K9O_z-G~RXra~U#Wzm zx1OD>YpsbBf#_$zVF^HZ#QL{Xx-Fj7k43na6_L3G^cQ&CAFEkjN3{xbtTWCe2pH9?|Rl1y|ZZO-X-0wHO<9^Orp6vh&n z=bIEK5@zwq8b>B(Vn%8-NkNEMR5GM;+=mJECTK*ubWR-{s&!uOvFJF; zyk_h)g;CZA@hphtF}G68$CF`t1&;SFfXcwdl|7uy9A?}s^asoQ%lb$|-Xh%PREkfTgPcbXAuWW& z$j$Sn*@39Rx(EyJ=PxT(rHvO=6QVhb+DAeb%1TnyXT6;bxnq{Wdy%G`{!1Zo^wgZYJ3jJ_Q- zf!?Bv_2bZ|@GJG?7y}o-AP0HPB3)gEn}wDOM}lG>hZsr+NCU>nNiqIZ_yE7WK##@4 z?RTyeN_?{z`Nxh z-}a#~RA!3xeU|`s5hX!0!0V=SkGZ}4Db&sKof)@jrxpJG-fq>-k=EryLg{@mlgsGdvRYP0YdBr2m@uB`s+sxc*h07;WvO&*}k6w1HWR&f}A z_}^>jyUJ^P`6o#WxVlF<8pDHhU*=Im7g|~UXZd&kslL3bClcSaR{BlXLr8+D;;fjD zt=J_t#b%3AZ&yOF7GYAn#O!(32AU--Wn}|F#d zkR!D;pJ87G2L0ql5(ENvpQS*Gb(6|>06)$R#85gAjl*D2Hh~=?LjY;B3C#^K1jNWH zBogkX;Q%qDu!3;F?O1%?G_))dG!0A(Q$a-PmXqXi8Yvi4)Tb#`d{!7i0`!j|_E1#d5JVWBAM-;}J*E1QqYrRVd=GmAbRBua&+&kSSQ)FV zI);9F4fj6JGA70l$Q``{11Z7WBPkkZasMbJQ{jydwq_JyC|RopqX)%t7OP2K=R7J) zf*=(UD>P!Nh>jlEDNzN{fF3U(8kjwa;k!1Lg?Rwji;J$$^=WAA)rIk&(+5h_b@JZZ z6?S&)&;Iz#s&ii33s}U{FD$MsNhByGM@}SvbyW)*beII{ZR;p+(5zQJ?Zl{s(l2h@ z6$GjV%$(SfIVe}$CwPV#>anrlV+#9E^EfP!m%}P9|E_Vp<1K96_=O)mP z5BB$W_bUM<=xNyAN|zcpUl=#n?s>@XEQKSVx9fA=KFsB`e6@DWCRvMiWj5dSYML|O zR-<;=eVP|^WIC4Vb@dm{&ge_E8P-1W+3X>0LqySsNuZ(%6%T21y@(XgiywouIZ%%0 zvBpzzv!tOt+Aa9r+@#O9YmQWlHx~f8xMh!AjjG_{hWH%w2N6U3^7v+T6(Mu1cs9|OUOz4)o30BR2 zuN)m;oO-$k9)q1t{(EH!QK_EX8IutAMCG8e!+%rxXmM=Jq2|ssb z@9557ywxdoF9Jy3NwBt@d7ag6$H|3}-*Add{T@GG#$d|FHcoqrm$;g&XP}+UI5{SV ztvqy?GxyQax@D#tT9`=NAq(S20j7B&mlb~ZUGT+6NZd{EmjEZ!swg_+REKBSFz~jZ zm*>aNX|{+&o&1H)G8%Nv6`)bJ&aDQN&$~14{Pfw_P>C$>wXS` z_VFE}00q78=o4uc7HCM76O559SJb`Jc|dSN-g$S!d*_{BoV!uS? z2x39?9!m>I#=+J5Fg>PpcCz3oxT>S5I%wwN$b55j>Qsud8WA_a(Aw$zJvkO?x>*3s zBqQp3HUGJ3UuUq>lcP_tQhrZyf@ey^mX01}+h|rxN~a=-DhP9->fJgOOijwmxKcn` zr(l<(I`dzhQUdJgK=yK#?IF{JZYTu~%Txak0$K&MTZU~3Z~@3il@5rSjRD_dTV#{4 zS*a;-KQ^GVs9BNL8m1TQxe#)L>82%HkpHOz z+EOt&w9aS^<5us~(vuN`UoGr9+K3aSl#dc5#$3maGz~{irF_9b1_oJfRSzf|CZ=&& ze_z2hIFx*s!(G4Gzs@|>6I zk@Q)f^2(oHz)4eR(49&4Hl}ym^pflCf&r;l7CUUCam$XzwcOwv6XTdbH zoB)nT8D5AXAeCTz6Pw5(@BJ85wOKG23qyza22f}nN);Ao4O$n>?BF#>{Iu-xZ2VJX z%ft`*gXkw`*~JyO&nJWcg`4a2r}L|{3L{gTHr0q?25g@k2yoV&T`w=;kdlwj#)LlkPaVo%1~09ADpV+%Drt z6kJYO@v$p~xf#x_s&zBxTisbWVY|AvH@7X+h1|sK8Yu9v>rQw(Kd0HvW*^O*+Y8Ev zYc$!Elk!|>v}qs`P~A67HEa%y&fRlk=B7i1d7p|oj{TCUkS;el?Tk{y9i1ogoov)F zm9*f8dzk7PCJJOtCaMV~AfG=hwC?P`+p({E2RqJA2Z61;yF_#!AwH&Ec*pX_bzKNQ z(sI&7F~jk5mcl@ouzGSC6)-pv&!yfBo~slpgO}1T-Oy-k(rv2k60sL*c2vVLXxn;C zaagsy4RBb3zkhbiP?#S785UVj(@Ezt*HTx^)OfDRQCn4|gT_|?B1C)LHCfaM-WH?D z*R4H;lLXMhxpln~_dJKF!Z|0=UDF42f8v+^1>fl|thBnq)s%AgIX!I|jD0oDFEW=- zzF{Uz=lF+WRCZ3OQD5!}lZMllD7C85mhkxm15MQKK{-RzGvjuum*|r-ce+yzbWZT> zUJZIIrvU#miWL-M&VubvM~`^Uiv>yIZ?kke`V4#pg=B?{BL)8L@SCEvn;}q79#%o}Ya>y?OsPp0J(6eds@?CrgPznJ&&2rA!*K z8@LrBB*ZZR-Lh{sIv4TCy#q82QjL*q>&4q*?_hha;@HYw2LRk1g}}|FyHaWXh70nY zfOPW@9R&TTYzKTcbOM?XwweEWeW?tDO5w+M`+fzUiF_$QP%^O3|I5XtU%| z21yc8V4m`4$3EN*K?j;LNNRhMY}Q=;eUmeGKZP9M;0Aoyh@e#Q@T6|~^w5Yd*bfE_ zAE|dndYV3by#07{dUJYydek}Py|#CFuy35GCPy|*uAN{p*&9a8DOg-{nZR30JJd$56U!z0#!2%zaWGS zoHM9g+}qnaAXW;N?3kcXhCg3kb*|3Oj?Zq- zE`JoJLmNRRL8G5=hXYkuV5H;g8bSbWfb2a;V2qijphqVwm_Xv&Kn~98$Kz81KZoZ-7PTk#CSc8(dG44l8ziHV{Fk%2O#b>4?#d$T{ zUJBfh3f?AXKw5WL0ZG*i$UOv3%56Lk4&RmO9V$^(IIemY+PO}i*&fZf6`g&d<{#G6 zz79Qkzf;cu1Xi1Z((8uKp*GC1l6sT6^P)o(_Cb77g@eq+c$4)j4;3Qc{Y*1uFhM7ldUbLpD4-vRi(qY z@GsKS#{5u&hS@nzh;0L5a1AoPHc{sV%I0q*1~I$BWgW+I_Y8Y%o#lhhez7FUh$@t( z0bD@j7k9r??$^Ud1=aJxUSxuTjzw{zJOz{KDBYo6_4ap`B-q67;A@>TiVoWkRBwJd znM9M)gUs_M^$@<%cpk$O*f?y}z+Ln9zI@T+LhH5**}Q8UE7VgDvKzCP2u24K8l(V4 zzo>Ddl3|N3uE40WEHk*r>x2-Xbyit8O(sB@%rY-G3WsTbXK&~5kosw+yHyw3*ft9n z;8=kN68~&;2a5~cm6&N3-G#k@yL&1o-XUgOCMM}?Ahn1TLTbmriku3eL8To$WNlr( zfHe$dPHwNQ=@^r{q?ogL-C8*iseN}jEopp)cwnZRi|L0P2b1P0&FA6ems;(`{h@V! z)#dH2p<~qQj-7*@Z7}G!whT9~Y`nDdcdZa!G5bb?VyYJ|j7%^S!z-((xIa7((t}DJ z)bT|u+DBKzlMfJ1WOZg0C}{zhAtmhti0DrU%C?Y@$}JhS7Iz*-(rfJoGb3JsT#7?= z4qkz{z9b!{#;25Xd)OsqYN(&DiY0pXE@De2=ygbIMC zG>m%1COYY8RR14Rx2w~9gA~=p+{&y%R<7t%ctJ%x3mFeuPms-Kv}W-=e!r>}64w^Z zOW!AT!qKw1_8}4xGh0+y(yohwN>auzOtPC$F{)-@R?HVi4y3X$vt5r2U99ObR7%v$ z4WLG@d*k#rXNnH_urwK)82AtIQP(cxvIRBi04g_zXIDY_TC){;s34mjnKaza9;C%kQ}T+N zbUd>r8{2Y?*XMPgFx%MP-re1?oY0l0gOz$N7OhiXO%wL1%qqmOC#Pb}Fsqp`N?Gix zE^%e}n`uooa{TAAm|#7E+GC-oIF->c1WIY_>5()^a%|)d*P>b;MDnaFKs1~4->7TJ zcFt!IjHcDA*(>&%f`U)!jr{XXW_L;O?9K4Yn_}z?LT6M6^9zGJtzzkb{M!hgy`g*F zaL~`(1M1y8`o-VBE{}1)LhlcM@ZLZW_lyNOz2rUsxXDci=_lN5mB0pr6h7qGD*RukPFN)#cZ$b=H0R&N@oQ z|I0M8LfFoC0{=m8YrbNo-f?tL@#1HZ=xF#Lri=EQJ%U=%sZdv^6%EOgl%f?re0mAW zj$Pv*wQ;GlBLR8_8jN!pA6%INo#KSZN5avFKq*(VZdxlrNXh6Y1h^wT-~^g{6~ zS5I;krkd?rbPiSyQ4n%Y2@caaHmf3`1Iv}Vi__xl?3+VdiHCqHsRac^n^dBB0ZQZp z$A&5i?pzu<`QKs|A3@KA6_Er#TzII1k&|FEKNm370Mz*cZOsmq>Qt%#!rp;qH3IjB zpgtC=4}CZ>q1c0sTsLL>sz)ZSStgX}LAit~VJXR|GRE8zA+$-*MT#cRIy)#sXS1OQ zKOS5J(5)2O?+k7JeJwybM^KoNBX+2tggfpDi3UiP3bS?TH^SXt=ru-_RrXDrMu?dk z9L3)C`O(?sx_5r{BmU>==;Q5A?@y1Q#Fd!PU+IUPP1gk~>l_77wAf5qG|I4e6!e(T z4blKP0Zh%4>W*rJG$fkQXgxh9Azh6=*n10SFdkHLvtX)>@(Unw;-$q%3hYL?2X~z( z+EJa9y~xoFl1BgX2M{p!tyo^Llz#_C86Z%SFm?n77XE8IfEdX-q~AT@eN7hwexq5Y~R7kDsAg=8<&)TZ z42K=C+gvTx`F9ib>*UX87ap2=)}+t{kY^~`(18N{54vbzc62YGl6{qOiS+4!>@rko zbo%jYt2Xn~SdNm;x*Tfi#m3Akuy}e0QN2WJz?KCEL3wf}GbtgM8|5j~D@}clx5Ub_ z2fUndY+^X586QN^IR3iY7pwFJh6q}E*jLcRlwO*~<5C$atK5GewcUckP!R6Tys{{n zId#DleviOUOVxat;`D8VAxu|qCw9+a;O=KPu)DWu}P#Sj0 zOC8)fts>n!+<$w3HKG>Anch{#t+c(kHN!~AD)gs-Ra{i)xfAGRH2Uz5-e^V!IrR)R zfyq^tNkkA%3bw)b?!le^yzvh})TJjD6!`-WF`3%_;<1t45RY4<>HnVMo2C@ zu8JFFA=kxW38~Q&G8*AJNhPZP^}9*KlBqeSo*l>1xO!pVEPnc}?i=OXsl3fE#IKTX zY|oCPBlbtnu9MTL#qfQ0Ul=~!Fm-H;-?Z6%{22-sX4LZGd8FG+x}42IUWK-Sxb{om zmzIT=omnCZynp`F`xi*G`-M=v;?*~(&2QYnrhR;~?q#KlvA6%8U#y~&rs}S}>5dae zsGh?3RKwGVkZ ziLd*dW2=Vd&v|()<0w~3o(*Q?_uukto#KI2$NpbtDvkE=$G&3&r@G?S$|8KIay-J( zZQ>!A-N~a18C+7=dwaCex$OunYZHa<)S0#1NdxcdLLLjaJrar(rtpii2`Wb6xzFb* zzZEO5os_w<-c;iv4myh>p%dS98<-4b!o&ek@}9$->wJ@X3S*Eu+q-A<1EydA)|R6h z8&^&8t_g=`>E40+;sC52)%%2&X(eGG%QOK2l4#r&gs(PLt>=$QHr>h5a8advYbm(g7GyL5Tf_R;gD^I-IVNp^?M^8yu;;N*+1gUO{;qbV9G@`M=`(^>K=ZGIG` zVC~?Yi<8ArK&ouo{6Kb8TKaS6AA^q4Luz%iy$0n3U`{yYD+S)-{&Z2Dq@;#2ygwS* z-PH(MIENe3G48Cb9WIR~N&2pZlbmNITW7!5SZ=jFpZJ>4Z7P1U!JOc{b(h%_&|~Os z`UML*)3r%|il#SROM3^ZJ;vQmk2^Xxv$@t4TX;{Aqdarf~s%qx7TAEc= z+X9SKw!N?Y?qiTtNuwCkoh4Zm)tS%M&Ytj}?X9=FZ}+wj@rO#3kKz0RZD}{!N zwL~e`_2nn+#(VI|pF74>DHGXPP59;#_>kaSRpLV$6RYuAhO}k<(F*gd=a-6{`B`i} zy6K&M=|TWuK%Tz?K<3&EtOIBF;Q`T7C8!kx%JW2ERkynrSAyz4LZ+3;zH1_j@}#JDRO17Q#us+0!f7tG2^Ligs}yZj1BX zx-sw>r3o@{^Eq`5wDl&qgQl7GLN)`;MI7e%2HrLJ6s*DmG9y9@_ec^u`q6Yi~KDLBx`*S%}J$o;=3xhHQQa>R9lC?}W z9Pd|_ARP|3E2~Ga)#4Yx_o^G%P(yDb)+b750O$l?o^K}@X6VTVclHxSG4bWuz@?8S zne|ZuIo3VL7tn;lv5@10rmG14Kq60SM%^=5j7;W^dpV55x}3uIz~Z5txavmNB}dt9 zo)+1FPmV4o&|7Qs z5`Ikb>zTLPYpDo%NF0DPYliN_$_2gxrC(C6`_&#GxKvWxM;a@K z_6X%Pqo8}n{eiq1c_45)e9wZD3~#%(`oa{Qx@9sy5f`Y{nz^DGH6QY>%tsR@2`{qh zu^gorUGv*0{N=JALSv zp0{OX=B zA1&l%h|PK&j^RjjreWglK+&apAbTcE*a#Q4I6sCfC~wAx-90)rnM#B?X++aN$<4-S z)O|bL^qoD7U>SeFMjA7C<79RtdboC1&yuvFT;`D6wuIPLg;M zUKPb!7$?0?_7b&!$C&6H?=WoU+VP~>s;}I=XD5KDkoYJ{yc6l>B$dP)6G*I#6(G-; zVls)YQBvM31Ve@}-kGK)_8xE<38spaRajezz%ot|<%Y3a8T&KG-+%Yr=Hek^a~kD1GFCJYa_yFCF1KdN84|yFcEXnTf0b7&fAAN|59lOz54c zX9(jBlN5;zD33k^N>XrV4m(zbwW9zD=0i+=h3}EdXUh3Umpg!9{A#$c!wC%={1zeH$k7aY;$hhLU~dE4T!9J*o-0IifVeoE9ptkT zMOyAz9R{cg+`^WzOQEx(UnilPdApdz!T!h2pt03E$PuoyUNP9k->yWjHJT4_{>eE^ z^AtokACr%8-8SiGg>3`}kE2Rhic2;tm2(N|3wGA_R&ztj$A8Lf|3|Ev5K25zyIZ4yPv5Y$AeYU^_S(Al@ zk=Z1#Df$m0P(DZrV>U_beFMWm&kCVYl8lVXC4~hC3qX88y%Se=13VC`m_ioz))wUg z9UPl`mechA%FAj^F-?DLqCs#(FE1oyy=`Xn8dPMa#V#Q!WQCM+#0 zvJ<(R6;2{6=xMN2O;jC@a>Q?Rdz0Nt_gSg7tqS)Apr-QyYmlc7)N1fJgjB;5O*X4| zr48GMhBRZ?m(~!2ne}RngHjn_=RJkJ)MYLUh4@%F4wK-9glCp)x2{h>e}yeV>m|E< zPGw&#FRQM9!{-$kCLCBVy>nfM($Q7b#Z8P3SoG(i>Im1U<%99?ToENF&Db#w!mKkU z%GlQU01z|^uyU1_k|e>LQzmhVayRw>v)hvFEKqxOJPFSM>NmkC6Q3~%uBQHGh+?3a zv$3@;i1I>{%FRy3ntRC>qdL-M{T26jCApnvM-RXqN2cihV2JAsBkKF$UL})C-)+i- zB}istba{zr94jeVy9#^V6VjKW$+K0?K}Z)i^|S{}r)!%hnD9^zqmV#m5pulZS>q#dOhZBpBy=AIsbkZT0%hAcWDe@jC?I6!tckR0X~+>8m|b zNRDy3Y?IDyvN8QGGg1|*M5`X;9v^U;bC~BTs_J;war$sL$uDsB`dTB|%Ib%poNNoaX z)_;VU&)AV=!AF78Y?b7~Hk^bKq8uo1ko&3v`@D^<{daiTGLoO+oio-}Jb# zmp+q|fZr73{ga5(`cyDm5E|83(S09{?!yQ1m}m1fm>~Qb?q}kN=;6WZxXRIZied)z zw96=whG#Lf+A+3Pgs71G8rhqI);H*1dqLKnaVSvc+P@caYrs01e?V*<40y zG*ENt1S+q%jWAmpVg!8SygNwsh>@55f`40*dCX5a_Qo<+#@N2)iv$@;5&9mVNU~7T z4*cH%hw;<*C@{WQN~^QHP~;J0Fi(7Kc!x^*;o>c8FO)Y;s<6 zHXK)j&4*3zf1*ChCdm1LG2tJ76fI#Bep<6qr4l?O6P`rbA64@>nsGng2;Ms63xd> zXU?(KW7SzViLOiY!rdT?M7v6!juqS)xd@vee&LcR3XDxYP8{ird|3saOEg7cEh;7q z&5kw=rt%$dkHIkhZum_9SflVZo%G?DvS!7Ho;J3V4|8Mp)G}9}BVk5C1LQ&KX+I~* z`01F0AB!~xWPQdYrEq|M!~-%0PGyD#4hzu)3CWqo#CMyYgTY|qaRjlp@2DfoM#f;@ zkh4SXYbDl4wr9yRy^)gw*rM|y?81Ua_W+YSLfLwDLarSnFtee174zIw;!@jjF5c z9vQ#tlf|Eu(njDi88&Ji;?0L+@9n|Cj)Uc{^B9eR;Wkwxqt@aaw2yUMDy;T!K7w z_Yr!3Vk5u>lZW3XZb#4>l={_8qpN3`LBz`cOYY~`aNoLzM|>N{XEf#vMwIq@PaB3b zbjHNhc-VSTg_F?urD}tx3FM;TK!(#4DWV_^7cwd++0D$@ksU-HKjt-bRJj0vJSZuE z*aL?Lh_U#HUKhgb=_5`@zVgZZXznM*_*`*kR~ojCUF9+%?{P?E$tk^1HwV@esRPX! zE7@=(u*F6O8(?}4@1Pcibu~!|;thv+ovFSiM-AOgWGseQS8lfywG*VerIr$}hr_=Y ziMH8qje8h!0fU^oVWnvCuauZq67ggO{JkHC-4&-?UqNoNQshX2k-w5gE3TD8)*7sc zV|tR3@SaPP^9Ys@Q%Fa`A2R;vOi)~>qah#WoVD#OkOU*_(M(4R@&S?Qq?aK#emdE# znU+k+?v!K;C6*SC3cjptET`82t$4YdK z($okw&E-NN?eY!wx~=F@Hvxx6114r8eQ6G#fE&1L9yXHo2dMa0Ge`t!l9G!kRMyZ^ zGdP-|L1GY&P{pPrj_;dAI-KHw$$()0BjlyZ5FjRj&O^vj$fXUiI7-VN;LB{^NNy?t z*>iTPfZU_@j2a2x_JFK-g1GIb_le!fXk&i%cDEp9`dO#!vI2Gg}za zAW>#9o;3`Y9QqG<+=e(0;BK&5AhTK6;lW-x6T`3hpg;sisuZ)hb{NRj_Zc;t<`&QF zu1e?cPoI-c@#C||v>rVyKA<@kg&&OdCv?)`IlMl-&>lRRTC%z;(L5U-rc8gVD+f8Q zVUd1DinZCT8FpXeaKsiRW%oePhV!yr{q7WlD+aK0j%B^uQm_j}&at!qZf8p&99wqh zN7`d*~{kcLwyA24u$&SdeSgPnc z5@~hyB8|CyfZ5JDimN&tt0~%fvL$ZrLK0`Y5kg=vhUAPvGaZvZ=W2Ars~5sM8@G`FSkXAqMbg|r%uM_s6e zma)AsWBYw!p94bGa~E?qT!~-&yYjXW@0zG*GUJwt27GC?Hg}5A=1FML&TVMt79H{1 zggno)WI;H$3uns)g~pOJKLyE64o~sWHB*ur#>$O~zP$O8eP)-V)&vYmDS9Eekg^Zf zW%DM4imhVmGzZ`U8cfvXYI~jDQ%X;hP%iboB30L>B7=Ukxtq~(eEs7!8sMGpF3v7a z%{^+ziz|*lak`Jq!^FVm6o)7tsweexmOQ?D!Hr@Pfu8PqB;XV5rDd5;*1 z++v;Ut&-j_OJ~r%)CAx>)3jRfmOXjzqSd%A9$g@Nt1xB($a4(WqOaY>(O{Y|{8*a9 zF>^pJ)@56b-i0*GE}|L_qB|R#zX?<(swv=lnyfCza4Zs!UpP+*_|N&VbFsd_2cX7B z$f)xcf(N4~+ekYX;4(eE`~>0D;!i2}lq_tl#YX02Q0B5VB6 z4OAh`Bqq-=8X_hGQp_NoHYZl>;kLoA=y$b9t%xdTx+tl@$`&(@}RdhaVAAr~ymG&yEVIX}#l@=6ZDm)VR z-YluKFRA`#%07j@9l7>A>SC({vTjBO1b~f~EW^;rQ=91xZ!>g&9f5v4Cd4qgS>^PH z1?E$vc<3XLBd9RPHFdXj)&la6xjb1cKXkME5sP4IUxo3|p4XyR)yTHT4H$mR$;jo6*n>&zVwc z9hucTthxi!P-s@_39JrFOH)vllTcMw>Diejtw>TGlMz|;0cv$JFREgKR&GpjRCJt7 zgCz=0mveB^9a{!}1uaZMbPVd%NRJ|^L031&*SD!uT5Dd*Aq~!6fKFRZs)(y>oBI8o zb37VUh?%>jXmkwyUeMq!Au=j#k_D6*SJa**OHt#L<{UFckc)TdgUYO+l3|rOv|HKL z5gvI%Zzv!F+gI?|o@7v4&Tn&WlS}`Dc)!gDe9gblkpZ~vNd3zv)9Z;+B&bOO#KQQP z8;iemK#soM_B4ELaXC!4Eym`|#ab{hSJuG}!6M=^2FrL7v1KNi!Y=RSMvijjhQ31j z^UpAg73W&Trer*Go_&K<{G%GN;i!yQ>Baj>h;24p*iiVdgjP9l6j%t#yV{Xwi#uy( zkK8h%+y-qN@CjzM+*2V)|K%vN??{s@+>J<7A%j4>OYGs%2FR4A2&#caQRt`^Q|gWi zF9`<7CQt+ z9t?gD+7}hOjUWp@eoZP^8CAbSJ**YEt;qeIi`-c8^Vjr+%3 z{p@xS8Z3lw#k}6>kU^Fi-hfaTb-HgChG8^ljng4f;7&%g>7DzN2c0AwbMrKuGl+zf zVg!YQ2%GRd0lLNEmjpbMkhz)-*)!C*z@X{ec_LvE&Q>;K1mX4M4;)Cd1ko(Fq;QR& zqjVZ@28^DH+%DNw{yLX`U2>)E+l}TO8#pO?C`m`5@?skkYV8KyxYW`xL|sZh6}ARn zy1LyR)_Wod-ng2CGdJkY*50Apfn7R~4g$7kMgqDzLU`~HfV*sZ?*oSS2xDm@csC7! z_%0Ly0G^06)#gn@%R=xXX7W06FdmuI=gOO=*u;HI+10(2034Zt%hny>#AUm2yun?% z0mzw%d&y^o;O7Zg1ffd^hn~Pp{1KfW^Fw&2Y;t$0l4Hi*k!DnaUC5UV_hJlj3X0+r zpEc(23JsDQH|i5^z`xYUzMZwZW2If@^HPx#zycpc&jeK$jx=^AO|g(KdzDO*I+1JP z>#W;W5>hF@--a!||MnoyaCb-u2>;g~JpXF#IMhswm&_Oe^B5Mlym^zNSRfqlDBb2W^;H)Z79z4{E>Gy5 zn;yep;&2WCVgJjSNvECBD{CeeMM;LHDP}XvHN)#Vtq*}a`H2C_z=`7hp>zPzYsEbh z>xwcAc)Aj=rin317arw_fkmkrWC>4EOh1~&5B^A@>Z#gN$Q_fb@?|wy>qaA2 zcqedXe39T+dH*+l2H8JceZzPE!f0`C*8VSnPY#0n#B2?zXzS@-0!Z}|T2f@EuOs=y z6Pp;IOkb68oK6(zTHK6-$X8XE<{fMYuY|2HX-Wm#yE#6OG zsO<2_pK6`aK!Vn>N$;kJ(r%xY+X&!VJ%30lB}|K`e{rAS_yr-1XNX?|k_PUfceAsS z+tGmQZaM_h9(q$k3taD~Ohv#QHOeKZIWaL1d5*u7mPBzoYEnC-m0=h^)}55Y-NVC! ztwWiYy`6V^?+)JXymL(r>0k@Y46GhN+tOQN$7H0f8Cl_@U>`jfqX6gyUj#-Kegh#5 zn=QdvPUGk}4cRhtv^xWsVbka&;!+N(JZbFiTc!SofQ&`v^0*Ho5sMGhQv%E{XuEKe zT%cCxYiSuf;G#i;g7@^mq4#6>{s$~w=#$Ap#haeIEmVOx~{~BcB+~l*EtIg^CcyF^S~;nEP8M2-9eT+ZSU^w zY`=ZGbEtUzz{2C}RyQM>RRf)TT41i<7BJVsO+s)nzEXb>{PbC`43vU4N=haq8!Ks3bPURC&ZU+Q+>qn<5=L1$icWHG$bOswK}_QSENf1Hf!8xChsgO z3J8Qt2(z{kdN=Jp+M@rr{93+@-<<@amaYaHgs-5S&;Z9zz3=#G_!F7{Zmwy2(&cWd z%N$|dt^sQ`{9>6j>Xa9L$vShv`q9Qau6iusogS}Xoa)vymE=WDxU#HooakX8Vu>Nw zShW^}LRpybpr^VZ0#gR}4qn|79hUU58i~smsmnUPJ4*#E6l)qrPvQAc4@3g=N!!w} zdg&{bHgg4DNS3uUn?g}#=##XJ%1-sov%9Xknx(&Tdz_dJQo+w&1(Bzf)io;!2zQxQ zgKK=*5qdrvInNefma|1$kPq90n>W`;t6_e4!=Ed~Beh~r-_oq-rc}$LdAMSYL?2o* zLQl3~rJ4&$1c3`9n)s;2r(0>AsuWdAwl5q0@K4x-R2QAhXh+TL9i7R>H zMmm-zMz|WLlG@)m&=Tqg1gIau^AY+Xe@11_!W+p)$FZ`@TE-7ylbqcM7)dh%LzY*7IF-wIG7u) zj(S!jaj^it-?RJN=1BOm)t5_OUPww8~BrM-)1S%WS&D@XDVPSd25MXj03qN z0|KXiL`%G@>U?oafLkyQ?`&MTsD~5GLE!H63I31w=pM2=VN#nH7I;WP7lufi6n(^a za@e^9hY_V4y^lJS)jG;GqwVySV`+S4TU|P$*qCbrv4?;I;`*e-QAUwG?f`U}p&4c< z_&`l=-Nu0a)IE2@g-N96=Qpl+!)3l-67E4#xnI~!O8K_N2{pc6iOzW5MU}@FSX9DC zoAdM1TZ@s7IBqX(6BeLl)|&B08(y*W%jN6!0sGC(Ia}0C1&s;=9?Dy#Uy=1^+YHh3 zq^!<2mA3&zm+crpm52rT+Qom@Ureh5VMys{T`rtIBLbmGOr?BHO;&Mo6x zG?awHWn?)4)u*Uwbdw{PAg~XxN3PXyE0XS9!z-9xDaUdbdc(4erdB!591{J5_DS)u>TB3K3}Q;!YI zyw$&6oYTM7d2azn8B=Uz;$5G7!1j}YKU{+spp9uv&T__8>DVB(aY{Iva|1Mh;lKSL zNXQv2!+>yL8nl{g&&_>?u6y@a>^Q12Cgo$I#*#Sk*m67Gn4}1T{@s!f{Y`Rz0xYQS zCYLNW>yhpvwZWX(NR8_YUdlr)4Ca!a%$u0m{Y)%y;GvJ zDBFe`o{9iQoggQK=htJD-BZQTE!b)BqK%K8m7;w_n0Pt!gVQb?76F3qz?DGVmYELz z2}04n`8RYy4D@Rg2*(6Q>W592msA$@NWdB_|M3Wq=*$p`x2V6kbWHvob{{i;M~VJ- zMClbBaiOapjnpFb+qjQiRBfcjnMpb|>9gs%sq9gA-F3VApHc5w{X6{?x4HT^D?MzhEcQi}I`W64T- z0s0&e?eVMr+uBb!P5- z4zAI&8q{$aRg;Rk8+$dZH58tu%3GZT1OJ))?-LalCxp)1D}B?NiGmOEHN^bT(Wq9{ z3sP8hQ~=5{=akC;2GIjrip!jNrm@b%TWNdQnH@wBlbOspxBPIf!_ek3CRdYLXL?ie zAc4cIP=C=Gjv*Ts=NEXqBRK`xe{7`I0LyxKpt=_iTF^jo80scC=a_Fn!CfpDq%P+q zInt6E=vlH$=Q&*vv8W!z5B&`^V|2|HaEZfH)Z4g+om6njla4Nq4C$aZ(RFsbJO?( zZrr#FN5(JNn?H{?(!Y9>>DV8nfBz4p*bn^lZ`1^6!>2!e84f1nZd&e=+X-qC^TSoL zAFH73ox#DM^oddy+?8%eH^-d!VFGKzY3fp!@PYp&AiYJU4{tVC5(GQvF&uX2Tc!AZ zT!Mx^YE*Lq@8r*C7oP5;9_zx&gi5oeq=2z01_W`PqIK7fq#=p3rjY2VF^7##;jqFQ zpIp>jbvK{=cke3a=rT#CBPc0@ z;uZ(xfm~L7uV?_P!REu}y7%8Jdz)`JxBq)(Z5b)VOrC0;JG-fFW-=vpf?m?Am4#oE zKBJdf*}AJ|CO^w5IaRkGfh_$TSbs2a6?Kx@qjb7DPM8Ketsyf zgc4_nv~AW_gZj?>hK5N$x94OW>nKuiUCa`5UfMTtg>4-%OJ!|y1tjDhY^n1pq0oHVeiKP&PnoalF=z@w=zY^`4&)2435H>8KuoG3k z3jJ*C*;MMpp_z#O`z}>#z-XXO8LT5g%cQ(;^)ABB{5iNgIr>D2h<~~`x}FDw14*|| zs??=$SwYblr<#^PTq8)MS#+fH5v1}3bBn!Il_tBcu^E88M^7>Z*BVc?C}z8>!EOsU z4%KdT2nykhbNB*5O0ed6qjsUXra4o+1GE2bzI2#aKc#X~dNtizY&oTpI?U3tM$M>n zIQB@58jA|fyxJk$l=PTWgW%!|9mv74d`-ep(}z9mLW233q@19y#0Bf$tRKLimC5+E zhJ%?Dz zL{!FV>2%ZNA;N`LMW+om8>12+?nU?$Rf$!iNVnka%QH)vCd6KwRLwb@=$jjz9Ef#S zL#Fa|NP8KR!3%%Zb6Y}Jpl41ntJOp{A_F&7SWV{ffU2>A+?Cz9an^Iz3NC6ZjiO_J z?B9h02;3g`gYKgR4&YwF60E1SQB+vZxfh~k5ZX%?dC)|D2q1nkA?+!3r~P9Z5}XiF zWl!p?Q#I#O1{Ac?8+1aQ-fWy4onBmB{_&0X@18?z-ClXeNe#?E#kA zNI3J@xS&+imb^gjE&dToYG$Vpe!MwLA$7THWXd&`7G#x!#|?2$rqmoLDN&_^G@p$S z<&ogC6OF?la92I`qI3kT!J#dvKqi`~oz9T|$yW)=i&6h1?G)tX~(6k`|0x zLldHWZw)k-$dchgu*c{8Ei9Oqz%Z5Q|m1=sIiJT!2Ahg3m<801fVTdO^<^1j= z)VwLAvB5>X;l1h8;NTUe6_i5M>XiZaOSC~<)#?CHeCtLMOZKW-u1h~`f|5Fh+Hx&4 zDlP$yrV=TvWh?c6NfcdR=&CA36z5ttMXW-;C!oa|rZpooE|VmHOg%-I$Qndp1`%N$ zES2XfRirk?@~gF)N@rcdLV~=jySXxjB}=2~ z75D_Pp<93@z)sGj6n|kuI#j3ee+NOrnNS?KOFd@5EfRie>6UapWy)s%V{F5y@^qQr zy7Pt8yxrMH5F?RFH?Do!KUYp$AvS@SL`&NBw*Q&_u7WIg2 zi$8ILoPCv-JR^A=KE}?3)g)^;pyw5>no@T zz}6TWAB+-Z{bMIvtE@(!Evi+XvDOdzd+>vf{ozq~4A&l_$#YHg?Vpi_YbOU|Bd4XP z>YWSwl|rYHa{9#|jt7jrqUi`nGYM$M>1tmoHJ2pvr5k`awG#h zExUvux8d@g@_#v7WdKv{h2$jEH>+S(&&(oIiG!GThd@=F?;q1jRM9N%p#@Z;^a06uc_Ol~_ZjI(L zmaQNtM9lbSWJhvI@LxCZjT=X2qH3iWyRyRdfvG{|hYkeCa90oDLhOA&VKWRN6L(;A zs*|ovl&1qq)0FC>z&qKGFkOe>O}h;>qZJZw)$7Z!CA$$-vmkd?bj3m{XLTAQLGAP( zlqd%pDdfTpq;&?*M4g6mGrB7vo<=DyIQCd^A6~`mGN)ICp?*61@h4Z<758aq0s2@#5j{6v)kho?8nE3Zx}S~dQSD|X(}7CR>_v300Obndh~ag> zt8j8KO*=ES05ph*!;nZpag{L9*u|))HCqn{yb_UzX0_f10)bi>>q9`@_ploWZDa&S z7QwSFPEu||DIkQ{2@cb-!-_t*SsV|uOrW)#OEXKwLj0)HYk=VqXN#pvbBv8*J>yME z#hh1Ch^ls^A(i4%+R(H#u>TYg=cuOvI`Omd++<8`ewzIE30VF5!#aiZ)}>&NP9OYt zLMW>W)HKGmJe`WKh&T(-iGREh|cA5o&yIR{$&a-zH(841$w}p|ABY1TUaR;MjnbT z=c9uzXcQ=92rs|BMA!309n6dLeNudytV~~X@GdvYlAh;`3ufx^u(Lj19-rRa9-Up% zRzs16s8$rI&|;yBK2 zE8`vpF3T$%h0BI?4`fJBysoi)4(`$h9l5g{+$pqopyMe-=ZsQq_)bhHA3cuHwICgq zIqVYG)ogy&R8)#{r4A;6JV-z}Du$e%`p=vMe)jeQfW8 zjQI|ue^{{wf>o7FM8pH@3lsQ$S;ByR7=Y(}KmQFabQG`;D^>{SV)LZB_4FBT9 zU_m!ztv*nQF&%mOP8{~G4M|lRCt(k^2vEBw&Sl zEefm}4Y_axz{wi>6djo+kg&1lbrk%yvX&FG24Y`Yeur&o+L2AB+&T02d3!!C)Xrf@o<*L}MHQa!?=s zMdj?E+6lA}sAxmwN|KWPa5!fEkx=KW%!a^s(#70Ng3o6V+vaWz_R-Mq22(OHu|gP= zWUvvdKfEI{W8kuQ8hTHNMX_-o1ahEo<)4W_oX>mo4#6RWtX)(EnYa9E*dbKl4)+310iP< zzLG)8QB4+&nIv{b6bX^Ag8`q}yw%tWJ#cP`t3S>R-JJ0;XsCUOwH>%7{4)Vup)AH@ zGsZu{xODU*)3b#DKOjg!ndnKs)6qS3CE#$v5KMT|JIo1U(viQdtFSV>2KMD7GF=Vu z6dTE?MW}E1j20gAhZ*r(boA*69DbBdpt!G^UX9luGlD>l`KL2B#6CF_)+2HbV_g6S zJNOr}i@pE!Hg>#!{a4TU=|A|7G#d2p-T~gSNBnG%6Nvkb%Ss=AW!1B4r1p5C+%Y8?v@62^j;Hzc*poIMFImRb1> zu0WPr9Q^lxPvL(()Gz!1f9xlN`9cC0Mh2fMnX56ZQfcXV3I1JaDe$nMZ91{v)(7;d z4mz#?yTa;zVjS_^&BkrxH0L081h~m;z0Ijm6d#6W!3C_>4O`+KGXHN_SKy^cq%-C- zh>RXFPn}~JY6C}H%(;&e{*s0MiVX}O^TAVC1tm|pM@a1YQ;X?XaVXYy+@Z2|-|NDmLtFE(0ON={IO%*fGdx^s?V*nh3 zJ67`~oUdHGQ)kFBBXl~L3^9M}CncoH?&R?bo%;yyD>~)#dvrWz-J*y+fu=?9ro|&E zh4UoLF#A!@pBU&Zr}*p~C14fq59l987t>EF@tWg4Loi?legxhjF%LWx01C=uX{x!b zGU*jQw{KDSQz(fkG*caXys}xWPfN;gL1Vuvh!uC=$|<6^?|_rC5?Hg|MR(#q^lMG}r(Y`#-GZ6P#)!g0 z^1jr4&OxtL=q|(2EoIAAlH_1;&-oZ-w92~Z>yzoec^LLoxF$3Q$V!S7#yU*i<1^#w zr%wQs&hEeSj)DIm#>JGn^3CqbI-ehWYfB2jipLVUJwJH;+Lc~CKlr0**CV2C&!#7A zJjSIPOvfH>O;@j>`|-a@j_hB<*6ZJ4PM4WdBS308l{26Fo9ji{rezz+&%KEg`Fs|U zNB0&yVN*nGW2wJ7XALnSF&w$51@h7pRX>VkQ1$#`x<NBY&hICu;RaBZD(|;%}Fthcs z2F-5L>HDyL(p`Y;k0Ef?nM#&t-4z>l8R>xfd<7?UP+2c_XtiQ_gysWNkoN(u@tadN z&iWdU*imgT`9Hlkr$7IXa+@bP7`zRL3A{p89PIUPWRUve(i&p2Jf+9q1fL`7)hEvZ zwBDz#*X0fp2z7(`&kcw8hbg+-%-LX`hGJt4$ksv+!CEzZg9+TeMY)s^;O-gb{O>j>Gur#Wg;kZ0jU@ zX15=)!nq#ZYwpGC*RQo}nSb?2AV~kLH!{2v^=0_jkPwFUPUgvVHVY*ND1bf1oQZz* zE%!fk%^z3uad&tfh`Df`krl-0%X6I8anLQ&xW7(c?oU<0MCvmCr*GAjE4ku`Q#~qf z`V=6v@j9K&SiXVsO4PML4bjv*PUp0o=u*XK2%Kqep{cnJ;!tB7>B_2ex?9JC-syg~ zd;B(Lsuep)aAA$E#9U1`=qz>VaXRE|#SjTf&`_5YbWG6DBESi8Ec1#XN51z*ZkKrZ z+8-x$%YO-Z5fcKLA{27FM+@}$8qO*i4aSg*kzGMA|IXl+y6bT!to zBGHbLaHI0FE|=y%2K~ptCz#}a!f99?1Z{7X{QxlqK%?>BUm0?Qx&&GjihIx=i_$(S zczRHqpyUwbrC3NjqPFCfx^l2$l@~SF3y)X!3im`6aox#2(GW~QdZ!H9(U_VYrTJ35 zrdS6Yre=?#p_%P%-#0je2qH1yFbY;9S`iV|YK%Ign_KXcEeGSW*?gJ5r{frq?J@;+ zl1DFuP1pdo9inNIx4Cx^==e8ED)GSalek&IoUZ6j6Him;u^CNm)8%!!t+BQ`Cymdx z#W{(^>*%E)MSG)U3#K}SA`7q;%7Z*I=EGSvDB^e>I-sWxjJ@6>wiH9!OB;PZVu+HH zSzxX>cu5_=J8%%RnW=oXg1lIT97a4w|1OKsu++|)FHj4b%hAx;WLXgkoq!Su zvbxYAyK50uv@bDvcl7fpK#NlmiPunZ2x^#xvOn_}v!UgL7VX<4ok8M{*Q50ev^c`R8E{|QSi`bZ04M^Kaz*UQ5?6HTCtVNQpZ7E}h! z*3j8?1k8llCi1O%@ zhKsPqr@1p50f=-!@1Fy!%62L|YUfKjl#vodZNiU099lzq#)KYqBbJa@;?gT>PV!kr znVoj{5?7^dy8HfGgj{`_x`yEov6j|#9Xwj3K1D4CJh@HsZ@|WANAo+H|2@%s{k+;y z{f_E?FH}#)=R2a`5&iFp=xea69o6rs{`W%lEY{jl{Ep&(FBJc7zGH26B>%f1`3mLU z0eDZ=4W5FBb=3wDi5Gr|5$xF3Ky-@{WZ4HpyOU;S^WGipn<0oFrHA=!^X}Ceqeto5 zV*Y@sd)GTHA3F}UMSO^UZZ0?e){~1r>fyXLgz?AV#dxHV`D1|L1A_7k8hoCeoE
  • { + let len = properties.size_hint().1.unwrap_or(usize::MAX); + if len <= LINEAR_DEDUP_LIMIT { + let mut unique = Vec::with_capacity(len); + for property in properties { + if !unique.contains(&property) { + unique.push(property); + } + } + unique + } else { + let mut indexed = properties + .enumerate() + .map(|(position, property)| (property, position)) + .collect::>(); + indexed.sort_unstable(); + indexed.dedup_by(|next, first| next.0 == first.0); + indexed.sort_unstable_by_key(|(_, position)| *position); + indexed.into_iter().map(|(property, _)| property).collect() + } +} + +pub fn all_ids(ids: impl Iterator, max_objects_in_get: usize) -> trc::Result> { + let ids = ids + .take(max_objects_in_get.saturating_add(1)) + .collect::>(); + if ids.len() <= max_objects_in_get { + Ok(ids) + } else { + Err(trc::JmapEvent::RequestTooLarge.into_err()) + } +} + #[derive(Debug, Clone)] pub struct GetRequest { pub account_id: Id, @@ -98,24 +133,11 @@ impl Default for GetRequest { impl GetRequest { pub fn unwrap_properties(&mut self, default: &[T::Property]) -> Vec { - if let Some(properties_) = self.properties.take().map(|p| p.unwrap()) { - let mut properties = Vec::with_capacity(properties_.len()); - let id_prop = T::ID_PROPERTY; - let mut has_id = false; - - for prop in properties_ { - if let MaybeInvalid::Value(p) = prop { - if p == id_prop { - has_id = true; - } - properties.push(p); - } + if let Some(properties) = self.properties.take().map(|p| p.unwrap()) { + let mut properties = unique_properties(properties.into_valid()); + if !properties.contains(&T::ID_PROPERTY) { + properties.push(T::ID_PROPERTY); } - - if !has_id { - properties.push(id_prop); - } - properties } else { default.to_vec() @@ -149,3 +171,4 @@ impl GetRequest { } } } + diff --git a/crates/jmap-proto/src/method/import.rs b/crates/jmap-proto/src/method/import.rs index dbeb4886..6c477df7 100644 --- a/crates/jmap-proto/src/method/import.rs +++ b/crates/jmap-proto/src/method/import.rs @@ -16,7 +16,10 @@ use crate::{ reference::{MaybeIdReference, MaybeResultReference, ResultReference}, }, response::Response, - types::{date::UTCDate, state::State}, + types::{ + date::UTCDate, + state::{ExpectedState, State}, + }, }; use jmap_tools::{Key, Value}; use serde::{Deserialize, Deserializer}; @@ -69,7 +72,7 @@ impl<'de> DeserializeArguments<'de> for ImportEmailRequest { self.account_id = crate::request::deserialize_account_id(map)?; }, b"ifInState" => { - self.if_in_state = map.next_value()?; + self.if_in_state = map.next_value::()?.0; }, b"emails" => { self.emails = map.next_value()?; diff --git a/crates/jmap-proto/src/method/mod.rs b/crates/jmap-proto/src/method/mod.rs index 4aa8f47b..c0ca39b8 100644 --- a/crates/jmap-proto/src/method/mod.rs +++ b/crates/jmap-proto/src/method/mod.rs @@ -27,7 +27,7 @@ pub mod upload; pub mod validate; #[inline(always)] -fn ahash_is_empty(map: &AHashMap) -> bool { +pub(crate) fn ahash_is_empty(map: &AHashMap) -> bool { map.is_empty() } @@ -46,7 +46,7 @@ impl Serialize for PropertyWrapper { where S: Serializer, { - serializer.serialize_str(self.0.to_cow().as_ref()) + self.0.serialize_text(serializer) } } diff --git a/crates/jmap-proto/src/method/query.rs b/crates/jmap-proto/src/method/query.rs index 9445f754..67b968c3 100644 --- a/crates/jmap-proto/src/method/query.rs +++ b/crates/jmap-proto/src/method/query.rs @@ -223,11 +223,12 @@ where while let Some(key) = map.next_key::>()? { match key.len() { 8 if key == "operator" => { - let op_ = hashify::tiny_map!( + let op_ = hashify::fnc_map!( map.next_value::<&str>()?.as_bytes(), - "AND" => Filter::And, - "OR" => Filter::Or, - "NOT" => Filter::Not, + "AND" => Some(Filter::And), + "OR" => Some(Filter::Or), + "NOT" => Some(Filter::Not), + _ => None, ) .ok_or_else(|| { de::Error::custom(format!("Unknown filter operator: {}", key)) diff --git a/crates/jmap-proto/src/method/set.rs b/crates/jmap-proto/src/method/set.rs index 445f91f0..4df21dfc 100644 --- a/crates/jmap-proto/src/method/set.rs +++ b/crates/jmap-proto/src/method/set.rs @@ -14,7 +14,7 @@ use crate::{ reference::{MaybeResultReference, ResultReference}, }, response::Response, - types::state::State, + types::state::{ExpectedState, State}, }; use ahash::AHashMap; use jmap_tools::{Key, Map, Value}; @@ -83,7 +83,7 @@ impl<'de, T: JmapObject> DeserializeArguments<'de> for SetRequest<'de, T> { self.account_id = crate::request::deserialize_account_id(map)?; }, b"ifInState" => { - self.if_in_state = map.next_value()?; + self.if_in_state = map.next_value::()?.0; }, b"create" => { self.create = map.next_value()?; @@ -273,13 +273,12 @@ impl SetResponse { &mut self, id: Id, ) -> Option<&mut Value<'static, T::Property, T::Element>> { - if let Some(obj) = self.updated.get_mut(&id) { - if let Some(obj) = obj { - return Some(obj); - } else { - *obj = Some(Value::Object(Map::with_capacity(1))); - return obj.as_mut().unwrap().into(); - } + if let Some(entry) = self.updated.inner.iter_mut().rev().find(|kv| kv.key == id) { + return Some( + entry + .value + .get_or_insert_with(|| Value::Object(Map::with_capacity(1))), + ); } (&mut self.created) @@ -296,4 +295,47 @@ impl SetResponse { pub fn has_changes(&self) -> bool { !self.created.is_empty() || !self.updated.is_empty() || !self.destroyed.is_empty() } + + pub fn add_created_properties( + &mut self, + create_id: &str, + properties: impl IntoIterator)>, + ) { + if let Some(Value::Object(object)) = self.created.get_mut(create_id) { + for (property, value) in properties { + object.insert(property, value); + } + } + } + + pub fn add_server_set_property( + &mut self, + id: Id, + property: T::Property, + value: impl Into>, + ) { + self.add_server_set_properties(id, [(property, value.into())]); + } + + pub fn add_server_set_properties( + &mut self, + id: Id, + properties: impl IntoIterator)>, + ) { + let mut properties = properties.into_iter().peekable(); + if properties.peek().is_none() { + return; + } + if let Some(Value::Object(object)) = self.get_object_by_id(id) { + for (property, value) in properties { + object.insert(property, value); + } + } else { + let mut object = Map::with_capacity(properties.size_hint().0); + for (property, value) in properties { + object.insert(property, value); + } + self.updated.append(id, Some(Value::Object(object))); + } + } } diff --git a/crates/jmap-proto/src/method/upload.rs b/crates/jmap-proto/src/method/upload.rs index a751f590..c81e1373 100644 --- a/crates/jmap-proto/src/method/upload.rs +++ b/crates/jmap-proto/src/method/upload.rs @@ -4,20 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; - use super::ahash_is_empty; use crate::{ error::set::SetError, object::{AnyId, blob::BlobProperty}, request::{ - deserialize::{DeserializeArguments, deserialize_request}, + deserialize::{CowStr, DeserializeArguments, deserialize_request}, reference::MaybeIdReference, }, response::Response, }; use ahash::AHashMap; -use mail_parser::decoders::base64::base64_decode; +use encodify::base64::LENIENT; use serde::{Deserialize, Deserializer}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; @@ -121,7 +119,7 @@ impl<'de> DeserializeArguments<'de> for DataSourceObject { *self = DataSourceObject::Value(map.next_value::().map(|v| v.into_bytes())?); }, b"data:asBase64" => { - *self = DataSourceObject::Value(base64_decode(map.next_value::>()?.as_bytes()).ok_or_else(|| serde::de::Error::custom("Failed to decode base64 data"))?); + *self = DataSourceObject::Value(LENIENT.decode(map.next_value::()?.0.as_bytes()).map_err(|_| serde::de::Error::custom("Failed to decode base64 data"))?); }, b"blobId" => { match self { diff --git a/crates/jmap-proto/src/object/addressbook.rs b/crates/jmap-proto/src/object/addressbook.rs index ffff813d..93786a96 100644 --- a/crates/jmap-proto/src/object/addressbook.rs +++ b/crates/jmap-proto/src/object/addressbook.rs @@ -6,13 +6,16 @@ use crate::{ object::{ - AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, + AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, }, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{acl::Acl, id::Id, special_use::SpecialUse}; +use types::{acl::Acl, id::Id, special_use::SpecialUse, text::Text}; #[derive(Debug, Clone, Default)] pub struct AddressBook; @@ -27,6 +30,8 @@ pub enum AddressBookProperty { IsSubscribed, ShareWith, MyRights, + Metadata, + PrivateMetadata, // Other IdValue(Id), @@ -40,6 +45,7 @@ pub enum AddressBookRight { MayWrite, MayShare, MayDelete, + MaySetMetadata, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -51,34 +57,48 @@ pub enum AddressBookValue { impl Property for AddressBookProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - let allow_patch = key.is_none(); - if let Some(Key::Property(key)) = key { - match key.patch_or_prop() { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) => match key.patch_or_prop() { AddressBookProperty::ShareWith => { Id::from_str(value).ok().map(AddressBookProperty::IdValue) } - _ => AddressBookProperty::parse(value, allow_patch), - } - } else { - AddressBookProperty::parse(value, allow_patch) + _ => AddressBookProperty::parse_nested_name(value), + }, + Some(_) => AddressBookProperty::parse_nested_name(value), + None => AddressBookProperty::parse(value, Some(depth)), } } + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() + } + fn to_cow(&self) -> Cow<'static, str> { - match self { - AddressBookProperty::Id => "id", - AddressBookProperty::Name => "name", - AddressBookProperty::Description => "description", - AddressBookProperty::SortOrder => "sortOrder", - AddressBookProperty::IsDefault => "isDefault", - AddressBookProperty::IsSubscribed => "isSubscribed", - AddressBookProperty::ShareWith => "shareWith", - AddressBookProperty::MyRights => "myRights", - AddressBookProperty::Rights(addressbook_right) => addressbook_right.as_str(), - AddressBookProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), - AddressBookProperty::IdValue(id) => return id.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -89,6 +109,7 @@ impl AddressBookRight { AddressBookRight::MayWrite => "mayWrite", AddressBookRight::MayShare => "mayShare", AddressBookRight::MayDelete => "mayDelete", + AddressBookRight::MaySetMetadata => "maySetMetadata", } } } @@ -112,36 +133,75 @@ impl Element for AddressBookValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl AddressBookValue { + pub fn text(&self) -> Text<'_> { match self { - AddressBookValue::Id(id) => id.to_string().into(), - AddressBookValue::IdReference(r) => format!("#{r}").into(), - AddressBookValue::Role(special_use) => special_use.as_str().unwrap_or_default().into(), + AddressBookValue::Id(id) => Text::Id(*id), + AddressBookValue::IdReference(r) => Text::Reference(r), + AddressBookValue::Role(special_use) => { + Text::Static(special_use.as_str().unwrap_or_default()) + } } } } impl AddressBookProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => AddressBookProperty::Id, - b"name" => AddressBookProperty::Name, - b"description" => AddressBookProperty::Description, - b"sortOrder" => AddressBookProperty::SortOrder, - b"isDefault" => AddressBookProperty::IsDefault, - b"isSubscribed" => AddressBookProperty::IsSubscribed, - b"shareWith" => AddressBookProperty::ShareWith, - b"myRights" => AddressBookProperty::MyRights, - b"mayRead" => AddressBookProperty::Rights(AddressBookRight::MayRead), - b"mayWrite" => AddressBookProperty::Rights(AddressBookRight::MayWrite), - b"mayShare" => AddressBookProperty::Rights(AddressBookRight::MayShare), - b"mayDelete" => AddressBookProperty::Rights(AddressBookRight::MayDelete) + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + AddressBookProperty::Id => "id", + AddressBookProperty::Name => "name", + AddressBookProperty::Description => "description", + AddressBookProperty::SortOrder => "sortOrder", + AddressBookProperty::IsDefault => "isDefault", + AddressBookProperty::IsSubscribed => "isSubscribed", + AddressBookProperty::ShareWith => "shareWith", + AddressBookProperty::MyRights => "myRights", + AddressBookProperty::Metadata => "metadata", + AddressBookProperty::PrivateMetadata => "privateMetadata", + AddressBookProperty::Rights(addressbook_right) => addressbook_right.as_str(), + AddressBookProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + AddressBookProperty::IdValue(id) => return Text::Id(*id), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!( + self, + AddressBookProperty::Pointer(_) | AddressBookProperty::IdValue(_) ) - .or_else(|| { - if allow_patch && value.contains('/') { - AddressBookProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + } + + property_names!( + AddressBookProperty, + "id" => AddressBookProperty::Id, + "name" => AddressBookProperty::Name, + "description" => AddressBookProperty::Description, + "sortOrder" => AddressBookProperty::SortOrder, + "isDefault" => AddressBookProperty::IsDefault, + "isSubscribed" => AddressBookProperty::IsSubscribed, + "shareWith" => AddressBookProperty::ShareWith, + "myRights" => AddressBookProperty::MyRights, + "mayRead" => AddressBookProperty::Rights(AddressBookRight::MayRead), + "mayWrite" => AddressBookProperty::Rights(AddressBookRight::MayWrite), + "mayShare" => AddressBookProperty::Rights(AddressBookRight::MayShare), + "mayDelete" => AddressBookProperty::Rights(AddressBookRight::MayDelete), + "maySetMetadata" => AddressBookProperty::Rights(AddressBookRight::MaySetMetadata), + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + AddressBookProperty::parse_name(value).or_else(|| { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(AddressBookProperty::Pointer) }) } @@ -156,6 +216,30 @@ impl AddressBookProperty { } } +impl MetadataProperty for AddressBookProperty { + fn as_metadata_root(&self) -> Option { + match self { + AddressBookProperty::Metadata => Some(MetadataRoot::Shared), + AddressBookProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + AddressBookProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => AddressBookProperty::Metadata, + MetadataRoot::Private => AddressBookProperty::PrivateMetadata, + } + } +} + #[derive(Debug, Clone, Default)] pub struct AddressBookSetArguments { pub on_destroy_remove_contents: Option, @@ -187,7 +271,41 @@ impl FromStr for AddressBookProperty { type Err = (); fn from_str(s: &str) -> Result { - AddressBookProperty::parse(s, false).ok_or(()) + AddressBookProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| AddressBookProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AddressBookFilter { + Metadata(MetadataFilter), + _T(String), +} + +impl<'de> DeserializeArguments<'de> for AddressBookFilter { + fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> + where + A: serde::de::MapAccess<'de>, + { + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => AddressBookFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + AddressBookFilter::_T(key.to_string()) + } + }; + + Ok(()) + } +} + +impl Default for AddressBookFilter { + fn default() -> Self { + AddressBookFilter::_T("".to_string()) } } @@ -198,7 +316,7 @@ impl JmapObject for AddressBook { type Id = Id; - type Filter = (); + type Filter = AddressBookFilter; type Comparator = (); @@ -294,10 +412,16 @@ impl JmapObjectId for AddressBookValue { impl JmapRight for AddressBookRight { fn to_acl(&self) -> &'static [Acl] { match self { - AddressBookRight::MayDelete => &[Acl::Delete, Acl::RemoveItems], + AddressBookRight::MayDelete => &[Acl::Delete], AddressBookRight::MayShare => &[Acl::Share], + AddressBookRight::MaySetMetadata => &[Acl::ModifyMetadata], AddressBookRight::MayRead => &[Acl::Read, Acl::ReadItems], - AddressBookRight::MayWrite => &[Acl::Modify, Acl::AddItems, Acl::ModifyItems], + AddressBookRight::MayWrite => &[ + Acl::Modify, + Acl::AddItems, + Acl::ModifyItems, + Acl::RemoveItems, + ], } } @@ -307,6 +431,7 @@ impl JmapRight for AddressBookRight { AddressBookRight::MayWrite, AddressBookRight::MayDelete, AddressBookRight::MayShare, + AddressBookRight::MaySetMetadata, ] } } diff --git a/crates/jmap-proto/src/object/blob.rs b/crates/jmap-proto/src/object/blob.rs index 3ede7363..69ac3a01 100644 --- a/crates/jmap-proto/src/object/blob.rs +++ b/crates/jmap-proto/src/object/blob.rs @@ -9,8 +9,9 @@ use crate::{ request::deserialize::DeserializeArguments, }; use jmap_tools::{Element, Key, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{blob::BlobId, id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct Blob; @@ -73,6 +74,10 @@ impl Property for BlobProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for BlobValue { @@ -94,28 +99,39 @@ impl Element for BlobValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl BlobValue { + pub fn text(&self) -> Text<'_> { match self { - BlobValue::BlobId(blob_id) => blob_id.to_string().into(), - BlobValue::IdReference(r) => format!("#{r}").into(), + BlobValue::BlobId(blob_id) => Text::Display(blob_id), + BlobValue::IdReference(r) => Text::Reference(r), } } } impl BlobProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"blobId" => BlobProperty::BlobId, - b"id" => BlobProperty::Id, - b"size" => BlobProperty::Size, - b"type" => BlobProperty::Type, - b"isEncodingProblem" => BlobProperty::IsEncodingProblem, - b"isTruncated" => BlobProperty::IsTruncated, - b"data:asText" => BlobProperty::Data(DataProperty::AsText), - b"data:asBase64" => BlobProperty::Data(DataProperty::AsBase64), - b"data" => BlobProperty::Data(DataProperty::Default), - b"digest:sha" => BlobProperty::Digest(DigestProperty::Sha), - b"digest:sha-256" => BlobProperty::Digest(DigestProperty::Sha256), - b"digest:sha-512" => BlobProperty::Digest(DigestProperty::Sha512), + hashify::fnc_map!(value.as_bytes(), + b"blobId" => Some(BlobProperty::BlobId), + b"id" => Some(BlobProperty::Id), + b"size" => Some(BlobProperty::Size), + b"type" => Some(BlobProperty::Type), + b"isEncodingProblem" => Some(BlobProperty::IsEncodingProblem), + b"isTruncated" => Some(BlobProperty::IsTruncated), + b"data:asText" => Some(BlobProperty::Data(DataProperty::AsText)), + b"data:asBase64" => Some(BlobProperty::Data(DataProperty::AsBase64)), + b"data" => Some(BlobProperty::Data(DataProperty::Default)), + b"digest:sha" => Some(BlobProperty::Digest(DigestProperty::Sha)), + b"digest:sha-256" => Some(BlobProperty::Digest(DigestProperty::Sha256)), + b"digest:sha-512" => Some(BlobProperty::Digest(DigestProperty::Sha512)), + _ => None, ) } } diff --git a/crates/jmap-proto/src/object/calendar.rs b/crates/jmap-proto/src/object/calendar.rs index e17c6886..ada51fca 100644 --- a/crates/jmap-proto/src/object/calendar.rs +++ b/crates/jmap-proto/src/object/calendar.rs @@ -6,7 +6,9 @@ use crate::{ object::{ - AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, + AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, }, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, types::date::UTCDate, @@ -16,9 +18,10 @@ use calcard::{ icalendar::ICalendarDuration, jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::{acl::Acl, id::Id}; +use types::{acl::Acl, id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct Calendar; @@ -39,6 +42,8 @@ pub enum CalendarProperty { TimeZone, ShareWith, MyRights, + Metadata, + PrivateMetadata, // Alert object properties When, @@ -64,6 +69,7 @@ pub enum CalendarRight { MayRSVP, MayShare, MayDelete, + MaySetMetadata, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -89,46 +95,48 @@ pub enum IncludeInAvailability { impl Property for CalendarProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - let allow_patch = key.is_none(); - if let Some(Key::Property(key)) = key { - match key.patch_or_prop() { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) => match key.patch_or_prop() { CalendarProperty::ShareWith => { Id::from_str(value).ok().map(CalendarProperty::IdValue) } - _ => CalendarProperty::parse(value, allow_patch), - } - } else { - CalendarProperty::parse(value, allow_patch) + _ => CalendarProperty::parse_nested_name(value), + }, + Some(_) => CalendarProperty::parse_nested_name(value), + None => CalendarProperty::parse(value, Some(depth)), } } + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() + } + fn to_cow(&self) -> Cow<'static, str> { - match self { - CalendarProperty::Id => "id", - CalendarProperty::Name => "name", - CalendarProperty::Description => "description", - CalendarProperty::Color => "color", - CalendarProperty::SortOrder => "sortOrder", - CalendarProperty::IsSubscribed => "isSubscribed", - CalendarProperty::IsVisible => "isVisible", - CalendarProperty::IsDefault => "isDefault", - CalendarProperty::IncludeInAvailability => "includeInAvailability", - CalendarProperty::DefaultAlertsWithTime => "defaultAlertsWithTime", - CalendarProperty::DefaultAlertsWithoutTime => "defaultAlertsWithoutTime", - CalendarProperty::TimeZone => "timeZone", - CalendarProperty::ShareWith => "shareWith", - CalendarProperty::MyRights => "myRights", - CalendarProperty::When => "when", - CalendarProperty::Trigger => "trigger", - CalendarProperty::Offset => "offset", - CalendarProperty::RelativeTo => "relativeTo", - CalendarProperty::Action => "action", - CalendarProperty::Type => "@type", - CalendarProperty::Rights(calendar_right) => calendar_right.as_str(), - CalendarProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), - CalendarProperty::IdValue(id) => return id.to_string().into(), + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other } - .into() + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) } } @@ -143,17 +151,19 @@ impl CalendarRight { CalendarRight::MayRSVP => "mayRSVP", CalendarRight::MayShare => "mayShare", CalendarRight::MayDelete => "mayDelete", + CalendarRight::MaySetMetadata => "maySetMetadata", } } } impl IncludeInAvailability { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), IncludeInAvailability, b"all" => IncludeInAvailability::All, b"attending" => IncludeInAvailability::Attending, b"none" => IncludeInAvailability::None, ) + .copied() } pub fn as_str(&self) -> &'static str { @@ -199,7 +209,7 @@ impl Element for CalendarValue { fn to_cow(&self) -> Cow<'static, str> { match self { - CalendarValue::Id(id) => id.to_string().into(), + CalendarValue::Id(id) => id.as_string().into(), CalendarValue::IdReference(r) => format!("#{r}").into(), CalendarValue::IncludeInAvailability(include) => include.as_str().into(), CalendarValue::Date(date) => date.to_string().into(), @@ -210,46 +220,103 @@ impl Element for CalendarValue { CalendarValue::Timezone(tz) => tz.name().unwrap_or_default(), } } + + fn serialize_text(&self, serializer: S) -> Result { + match self { + CalendarValue::Id(id) => serializer.serialize_str(id.text().as_str()), + CalendarValue::IdReference(reference) => { + serializer.collect_str(&format_args!("#{reference}")) + } + CalendarValue::Date(date) => date.serialize(serializer), + CalendarValue::Duration(duration) => serializer.collect_str(duration), + CalendarValue::IncludeInAvailability(include) => { + serializer.serialize_str(include.as_str()) + } + CalendarValue::Action(action) => serializer.serialize_str(action.as_str()), + CalendarValue::RelativeTo(relative) => serializer.serialize_str(relative.as_str()), + CalendarValue::Type(typ) => serializer.serialize_str(typ.as_str()), + CalendarValue::Timezone(tz) => serializer.serialize_str(&tz.name().unwrap_or_default()), + } + } } impl CalendarProperty { - fn parse(value: &str, allow_patch: bool) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => CalendarProperty::Id, - b"name" => CalendarProperty::Name, - b"description" => CalendarProperty::Description, - b"color" => CalendarProperty::Color, - b"sortOrder" => CalendarProperty::SortOrder, - b"isSubscribed" => CalendarProperty::IsSubscribed, - b"isVisible" => CalendarProperty::IsVisible, - b"isDefault" => CalendarProperty::IsDefault, - b"includeInAvailability" => CalendarProperty::IncludeInAvailability, - b"defaultAlertsWithTime" => CalendarProperty::DefaultAlertsWithTime, - b"defaultAlertsWithoutTime" => CalendarProperty::DefaultAlertsWithoutTime, - b"timeZone" => CalendarProperty::TimeZone, - b"shareWith" => CalendarProperty::ShareWith, - b"myRights" => CalendarProperty::MyRights, - b"mayReadFreeBusy" => CalendarProperty::Rights(CalendarRight::MayReadFreeBusy), - b"mayReadItems" => CalendarProperty::Rights(CalendarRight::MayReadItems), - b"mayWriteAll" => CalendarProperty::Rights(CalendarRight::MayWriteAll), - b"mayWriteOwn" => CalendarProperty::Rights(CalendarRight::MayWriteOwn), - b"mayUpdatePrivate" => CalendarProperty::Rights(CalendarRight::MayUpdatePrivate), - b"mayRSVP" => CalendarProperty::Rights(CalendarRight::MayRSVP), - b"mayShare" => CalendarProperty::Rights(CalendarRight::MayShare), - b"mayDelete" => CalendarProperty::Rights(CalendarRight::MayDelete), - b"@type" => CalendarProperty::Type, - b"when" => CalendarProperty::When, - b"trigger" => CalendarProperty::Trigger, - b"offset" => CalendarProperty::Offset, - b"relativeTo" => CalendarProperty::RelativeTo, - b"action" => CalendarProperty::Action, + pub fn text(&self) -> Text<'_> { + Text::Static(match self { + CalendarProperty::Id => "id", + CalendarProperty::Name => "name", + CalendarProperty::Description => "description", + CalendarProperty::Color => "color", + CalendarProperty::SortOrder => "sortOrder", + CalendarProperty::IsSubscribed => "isSubscribed", + CalendarProperty::IsVisible => "isVisible", + CalendarProperty::IsDefault => "isDefault", + CalendarProperty::IncludeInAvailability => "includeInAvailability", + CalendarProperty::DefaultAlertsWithTime => "defaultAlertsWithTime", + CalendarProperty::DefaultAlertsWithoutTime => "defaultAlertsWithoutTime", + CalendarProperty::TimeZone => "timeZone", + CalendarProperty::ShareWith => "shareWith", + CalendarProperty::MyRights => "myRights", + CalendarProperty::Metadata => "metadata", + CalendarProperty::PrivateMetadata => "privateMetadata", + CalendarProperty::When => "when", + CalendarProperty::Trigger => "trigger", + CalendarProperty::Offset => "offset", + CalendarProperty::RelativeTo => "relativeTo", + CalendarProperty::Action => "action", + CalendarProperty::Type => "@type", + CalendarProperty::Rights(calendar_right) => calendar_right.as_str(), + CalendarProperty::Pointer(json_pointer) => return Text::Display(json_pointer), + CalendarProperty::IdValue(id) => return Text::Id(*id), + }) + } + + fn has_dynamic_text(&self) -> bool { + matches!( + self, + CalendarProperty::Pointer(_) | CalendarProperty::IdValue(_) ) - .or_else(|| { - if allow_patch && value.contains('/') { - CalendarProperty::Pointer(JsonPointer::parse(value)).into() - } else { - None - } + } + + property_names!( + CalendarProperty, + "id" => CalendarProperty::Id, + "name" => CalendarProperty::Name, + "description" => CalendarProperty::Description, + "color" => CalendarProperty::Color, + "sortOrder" => CalendarProperty::SortOrder, + "isSubscribed" => CalendarProperty::IsSubscribed, + "isVisible" => CalendarProperty::IsVisible, + "isDefault" => CalendarProperty::IsDefault, + "includeInAvailability" => CalendarProperty::IncludeInAvailability, + "defaultAlertsWithTime" => CalendarProperty::DefaultAlertsWithTime, + "defaultAlertsWithoutTime" => CalendarProperty::DefaultAlertsWithoutTime, + "timeZone" => CalendarProperty::TimeZone, + "shareWith" => CalendarProperty::ShareWith, + "myRights" => CalendarProperty::MyRights, + "mayReadFreeBusy" => CalendarProperty::Rights(CalendarRight::MayReadFreeBusy), + "mayReadItems" => CalendarProperty::Rights(CalendarRight::MayReadItems), + "mayWriteAll" => CalendarProperty::Rights(CalendarRight::MayWriteAll), + "mayWriteOwn" => CalendarProperty::Rights(CalendarRight::MayWriteOwn), + "mayUpdatePrivate" => CalendarProperty::Rights(CalendarRight::MayUpdatePrivate), + "mayRSVP" => CalendarProperty::Rights(CalendarRight::MayRSVP), + "mayShare" => CalendarProperty::Rights(CalendarRight::MayShare), + "mayDelete" => CalendarProperty::Rights(CalendarRight::MayDelete), + "maySetMetadata" => CalendarProperty::Rights(CalendarRight::MaySetMetadata), + "@type" => CalendarProperty::Type, + "when" => CalendarProperty::When, + "trigger" => CalendarProperty::Trigger, + "offset" => CalendarProperty::Offset, + "relativeTo" => CalendarProperty::RelativeTo, + "action" => CalendarProperty::Action, + ); + + fn parse(value: &str, patch_depth: Option) -> Option { + CalendarProperty::parse_name(value).or_else(|| { + patch_depth + .filter(|_| value.contains('/')) + .and_then(|depth| JsonPointer::parse_nested(value, depth)) + .map(CalendarProperty::Pointer) }) } @@ -264,6 +331,30 @@ impl CalendarProperty { } } +impl MetadataProperty for CalendarProperty { + fn as_metadata_root(&self) -> Option { + match self { + CalendarProperty::Metadata => Some(MetadataRoot::Shared), + CalendarProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + CalendarProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => CalendarProperty::Metadata, + MetadataRoot::Private => CalendarProperty::PrivateMetadata, + } + } +} + #[derive(Debug, Clone, Default)] pub struct CalendarSetArguments { pub on_destroy_remove_events: Option, @@ -295,7 +386,41 @@ impl FromStr for CalendarProperty { type Err = (); fn from_str(s: &str) -> Result { - CalendarProperty::parse(s, false).ok_or(()) + CalendarProperty::parse(s, None) + .or_else(|| { + MetadataRoot::from_selector(s) + .map(|_| CalendarProperty::Pointer(JsonPointer::parse(s))) + }) + .ok_or(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CalendarFilter { + Metadata(MetadataFilter), + _T(String), +} + +impl<'de> DeserializeArguments<'de> for CalendarFilter { + fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> + where + A: serde::de::MapAccess<'de>, + { + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => CalendarFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + CalendarFilter::_T(key.to_string()) + } + }; + + Ok(()) + } +} + +impl Default for CalendarFilter { + fn default() -> Self { + CalendarFilter::_T("".to_string()) } } @@ -306,7 +431,7 @@ impl JmapObject for Calendar { type Id = Id; - type Filter = (); + type Filter = CalendarFilter; type Comparator = (); @@ -414,7 +539,16 @@ impl JmapRight for CalendarRight { CalendarRight::MayUpdatePrivate => &[Acl::ModifyPrivateProperties], CalendarRight::MayRSVP => &[Acl::ModifyRSVP], CalendarRight::MayShare => &[Acl::Share], - CalendarRight::MayDelete => &[Acl::Delete, Acl::RemoveItems], + CalendarRight::MayDelete => &[Acl::Delete], + CalendarRight::MaySetMetadata => &[Acl::ModifyMetadata], + } + } + + fn implied_by_acl(&self) -> &'static [Acl] { + match self { + CalendarRight::MayWriteOwn => &[Acl::AddItems, Acl::ModifyItems, Acl::RemoveItems], + CalendarRight::MayUpdatePrivate | CalendarRight::MayRSVP => &[Acl::ModifyItems], + _ => &[], } } @@ -428,6 +562,7 @@ impl JmapRight for CalendarRight { CalendarRight::MayRSVP, CalendarRight::MayShare, CalendarRight::MayDelete, + CalendarRight::MaySetMetadata, ] } } diff --git a/crates/jmap-proto/src/object/calendar_event.rs b/crates/jmap-proto/src/object/calendar_event.rs index 591f9f60..acfe3774 100644 --- a/crates/jmap-proto/src/object/calendar_event.rs +++ b/crates/jmap-proto/src/object/calendar_event.rs @@ -5,16 +5,22 @@ */ use crate::{ - object::{AnyId, JmapObject, JmapObjectId}, - request::{MaybeInvalid, deserialize::DeserializeArguments}, + object::{ + AnyId, JmapObject, JmapObjectId, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot}, + }, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments, next_lowercase_value}, + }, }; use calcard::{ common::timezone::Tz, jscalendar::{JSCalendarDateTime, JSCalendarProperty, JSCalendarValue}, }; -use jmap_tools::{JsonPointerItem, Key}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; use mail_parser::DateTime; -use std::{borrow::Cow, str::FromStr}; +use std::borrow::Cow; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] @@ -69,12 +75,11 @@ impl JmapObjectId for JSCalendarValue { } fn try_set_id(&mut self, new_id: AnyId) -> bool { - if let AnyId::Id(id) = new_id { - *self = JSCalendarValue::Id(id); - true - } else { - false - } + *self = match new_id { + AnyId::Id(id) => JSCalendarValue::Id(id), + AnyId::BlobId(id) => JSCalendarValue::BlobId(id), + }; + true } } @@ -88,16 +93,17 @@ pub enum CalendarEventFilter { Description(String), Location(String), Owner(String), - Attendee(String), + Participant(String), Uid(String), + Metadata(MetadataFilter), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CalendarEventComparator { - Start, + Id, Uid, - RecurrenceId, + Start, Created, Updated, _T(String), @@ -108,7 +114,7 @@ pub struct CalendarEventGetArguments { pub recurrence_overrides_before: Option, pub recurrence_overrides_after: Option, pub reduce_participants: Option, - pub time_zone: Option, + pub time_zone: Option>, } #[derive(Debug, Clone, Default)] @@ -119,7 +125,34 @@ pub struct CalendarEventSetArguments { #[derive(Debug, Clone, Default)] pub struct CalendarEventQueryArguments { pub expand_recurrences: Option, - pub time_zone: Option, + pub time_zone: Option>, +} + +impl CalendarEventGetArguments { + pub fn resolved_time_zone(&self) -> trc::Result { + self.time_zone + .as_ref() + .map_or(Ok(Tz::UTC), MaybeInvalid::to_time_zone) + } +} + +impl CalendarEventQueryArguments { + pub fn resolved_time_zone(&self) -> trc::Result { + self.time_zone + .as_ref() + .map_or(Ok(Tz::UTC), MaybeInvalid::to_time_zone) + } +} + +impl MaybeInvalid { + fn to_time_zone(&self) -> trc::Result { + match self { + MaybeInvalid::Value(tz) => Ok(*tz), + MaybeInvalid::Invalid(tz) => Err(trc::JmapEvent::InvalidArguments + .into_err() + .details(format!("Invalid timeZone {tz:?}"))), + } + } } impl<'de> DeserializeArguments<'de> for CalendarEventFilter { @@ -138,29 +171,34 @@ impl<'de> DeserializeArguments<'de> for CalendarEventFilter { *self = CalendarEventFilter::Before(map.next_value::()?.0); }, b"text" => { - *self = CalendarEventFilter::Text(map.next_value::>()?.to_lowercase()); + *self = CalendarEventFilter::Text(next_lowercase_value(map)?); }, b"title" => { - *self = CalendarEventFilter::Title(map.next_value::>()?.to_lowercase()); + *self = CalendarEventFilter::Title(next_lowercase_value(map)?); }, b"description" => { - *self = CalendarEventFilter::Description(map.next_value::>()?.to_lowercase()); + *self = CalendarEventFilter::Description(next_lowercase_value(map)?); }, b"location" => { - *self = CalendarEventFilter::Location(map.next_value::>()?.to_lowercase()); + *self = CalendarEventFilter::Location(next_lowercase_value(map)?); }, b"owner" => { - *self = CalendarEventFilter::Owner(map.next_value::>()?.to_lowercase()); + *self = CalendarEventFilter::Owner(next_lowercase_value(map)?); }, - b"attendee" => { - *self = CalendarEventFilter::Attendee(map.next_value::>()?.to_lowercase()); + b"participant" => { + *self = CalendarEventFilter::Participant(next_lowercase_value(map)?); }, b"uid" => { *self = CalendarEventFilter::Uid(map.next_value()?); }, _ => { - *self = CalendarEventFilter::_T(key.to_string()); - let _ = map.next_value::()?; + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => CalendarEventFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + CalendarEventFilter::_T(key.to_string()) + } + }; } ); Ok(()) @@ -173,16 +211,16 @@ impl<'de> DeserializeArguments<'de> for CalendarEventComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), - b"start" => { - *self = CalendarEventComparator::Start; + b"id" => { + *self = CalendarEventComparator::Id; }, b"uid" => { *self = CalendarEventComparator::Uid; }, - b"recurrenceId" => { - *self = CalendarEventComparator::RecurrenceId; + b"start" => { + *self = CalendarEventComparator::Start; }, b"created" => { *self = CalendarEventComparator::Created; @@ -191,7 +229,7 @@ impl<'de> DeserializeArguments<'de> for CalendarEventComparator { *self = CalendarEventComparator::Updated; }, _ => { - *self = CalendarEventComparator::_T(value.to_string()); + *self = CalendarEventComparator::_T(value.into_owned()); } ); } else { @@ -217,7 +255,7 @@ impl<'de> DeserializeArguments<'de> for CalendarEventGetArguments { self.reduce_participants = map.next_value()?; }, b"timeZone" => { - self.time_zone = map.next_value::>()?.and_then(|s| Tz::from_str(s).ok()); + self.time_zone = map.next_value()?; }, _ => { let _ = map.next_value::()?; @@ -254,7 +292,7 @@ impl<'de> DeserializeArguments<'de> for CalendarEventQueryArguments { self.expand_recurrences = map.next_value()?; }, b"timeZone" => { - self.time_zone = map.next_value::>()?.and_then(|s| Tz::from_str(s).ok()); + self.time_zone = map.next_value()?; }, _ => { let _ = map.next_value::()?; @@ -275,8 +313,9 @@ impl CalendarEventFilter { CalendarEventFilter::Description(_) => "description", CalendarEventFilter::Location(_) => "location", CalendarEventFilter::Owner(_) => "owner", - CalendarEventFilter::Attendee(_) => "attendee", + CalendarEventFilter::Participant(_) => "participant", CalendarEventFilter::Uid(_) => "uid", + CalendarEventFilter::Metadata(filter) => filter.as_str(), CalendarEventFilter::_T(s) => return Cow::Owned(s), } .into() @@ -286,9 +325,9 @@ impl CalendarEventFilter { impl CalendarEventComparator { pub fn into_string(self) -> Cow<'static, str> { match self { - CalendarEventComparator::Start => "start", + CalendarEventComparator::Id => "id", CalendarEventComparator::Uid => "uid", - CalendarEventComparator::RecurrenceId => "recurrenceId", + CalendarEventComparator::Start => "start", CalendarEventComparator::Created => "created", CalendarEventComparator::Updated => "updated", CalendarEventComparator::_T(s) => return Cow::Owned(s), @@ -332,6 +371,30 @@ impl<'de> serde::Deserialize<'de> for LocalTime { } } +impl MetadataProperty for JSCalendarProperty { + fn as_metadata_root(&self) -> Option { + match self { + JSCalendarProperty::Metadata => Some(MetadataRoot::Shared), + JSCalendarProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + JSCalendarProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => JSCalendarProperty::Metadata, + MetadataRoot::Private => JSCalendarProperty::PrivateMetadata, + } + } +} + impl JmapObjectId for JSCalendarProperty { fn as_id(&self) -> Option { if let JSCalendarProperty::IdValue(id) = self { diff --git a/crates/jmap-proto/src/object/calendar_event_notification.rs b/crates/jmap-proto/src/object/calendar_event_notification.rs index 64c2db3c..9ab1d019 100644 --- a/crates/jmap-proto/src/object/calendar_event_notification.rs +++ b/crates/jmap-proto/src/object/calendar_event_notification.rs @@ -6,14 +6,17 @@ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, - request::{MaybeInvalid, deserialize::DeserializeArguments}, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments}, + }, types::{date::UTCDate, state::State}, }; use calcard::jscalendar::JSCalendar; use jmap_tools::{Element, Key, Property}; -use serde::Serialize; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{blob::BlobId, id::Id, text::Text}; #[derive(Debug, Clone, Default)] pub struct CalendarEventNotification; @@ -23,41 +26,73 @@ pub struct CalendarEventNotification; pub struct CalendarEventNotificationObject { pub id: Id, - #[serde(skip_serializing_if = "Option::is_none")] - pub created: Option, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub created: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] - pub changed_by: Option, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub changed_by: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] - pub comment: Option, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub comment: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] #[serde(rename = "type")] - pub notification_type: Option, + pub notification_type: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] - pub calendar_event_id: Option, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub calendar_event_id: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_draft: Option, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub is_draft: ResponseProperty, - #[serde(skip_serializing_if = "Option::is_none")] - pub event: Option>, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub event: ResponseProperty>, - #[serde(skip_serializing_if = "Option::is_none")] - pub event_patch: Option>, + #[serde(skip_serializing_if = "ResponseProperty::is_not_requested")] + pub event_patch: ResponseProperty>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum ResponseProperty { + #[default] + NotRequested, + Requested(Option), +} + +impl ResponseProperty { + pub fn is_not_requested(&self) -> bool { + matches!(self, ResponseProperty::NotRequested) + } + + pub fn value(&self) -> Option<&T> { + match self { + ResponseProperty::Requested(Some(value)) => Some(value), + _ => None, + } + } +} + +impl From> for ResponseProperty { + fn from(value: Option) -> Self { + ResponseProperty::Requested(value) + } +} + +impl Serialize for ResponseProperty { + fn serialize(&self, serializer: S) -> Result { + match self.value() { + Some(value) => value.serialize(serializer), + None => serializer.serialize_none(), + } + } } #[derive(Debug, Serialize, Clone, Default)] #[serde(rename_all = "camelCase")] pub struct PersonObject { pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] pub email: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub principal_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub calendar_address: Option, } @@ -128,6 +163,10 @@ impl Property for CalendarEventNotificationProperty { } .into() } + + fn key_eq(&self, other: &Self) -> bool { + self == other + } } impl Element for CalendarEventNotificationValue { @@ -155,21 +194,35 @@ impl Element for CalendarEventNotificationValue { } fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn serialize_text(&self, serializer: S) -> Result { match self { - CalendarEventNotificationValue::Id(id) => id.to_string().into(), - CalendarEventNotificationValue::Date(date) => date.to_string().into(), - CalendarEventNotificationValue::Type(t) => t.as_str().into(), + CalendarEventNotificationValue::Date(date) => date.serialize(serializer), + value => value.text().serialize(serializer), + } + } +} + +impl CalendarEventNotificationValue { + pub fn text(&self) -> Text<'_> { + match self { + CalendarEventNotificationValue::Id(id) => Text::Id(*id), + CalendarEventNotificationValue::Date(date) => Text::Display(date), + CalendarEventNotificationValue::Type(t) => Text::Static(t.as_str()), } } } impl CalendarEventNotificationType { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), + hashify::map!(value.as_bytes(), CalendarEventNotificationType, b"created" => CalendarEventNotificationType::Created, b"updated" => CalendarEventNotificationType::Updated, b"destroyed" => CalendarEventNotificationType::Destroyed, ) + .copied() } pub fn as_str(&self) -> &'static str { @@ -183,16 +236,17 @@ impl CalendarEventNotificationType { impl CalendarEventNotificationProperty { fn parse(value: &str) -> Option { - hashify::tiny_map!(value.as_bytes(), - b"id" => CalendarEventNotificationProperty::Id, - b"created" => CalendarEventNotificationProperty::Created, - b"changedBy" => CalendarEventNotificationProperty::ChangedBy, - b"comment" => CalendarEventNotificationProperty::Comment, - b"type" => CalendarEventNotificationProperty::Type, - b"calendarEventId" => CalendarEventNotificationProperty::CalendarEventId, - b"isDraft" => CalendarEventNotificationProperty::IsDraft, - b"event" => CalendarEventNotificationProperty::Event, - b"eventPatch" => CalendarEventNotificationProperty::EventPatch + hashify::fnc_map!(value.as_bytes(), + b"id" => Some(CalendarEventNotificationProperty::Id), + b"created" => Some(CalendarEventNotificationProperty::Created), + b"changedBy" => Some(CalendarEventNotificationProperty::ChangedBy), + b"comment" => Some(CalendarEventNotificationProperty::Comment), + b"type" => Some(CalendarEventNotificationProperty::Type), + b"calendarEventId" => Some(CalendarEventNotificationProperty::CalendarEventId), + b"isDraft" => Some(CalendarEventNotificationProperty::IsDraft), + b"event" => Some(CalendarEventNotificationProperty::Event), + b"eventPatch" => Some(CalendarEventNotificationProperty::EventPatch), + _ => None ) } } @@ -277,13 +331,13 @@ impl<'de> DeserializeArguments<'de> for CalendarEventNotificationComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = CalendarEventNotificationComparator::Created; }, _ => { - *self = CalendarEventNotificationComparator::_T(value.to_string()); + *self = CalendarEventNotificationComparator::_T(value.into_owned()); } ); } else { diff --git a/crates/jmap-proto/src/object/contact.rs b/crates/jmap-proto/src/object/contact.rs index a399a36a..840f84c4 100644 --- a/crates/jmap-proto/src/object/contact.rs +++ b/crates/jmap-proto/src/object/contact.rs @@ -5,12 +5,18 @@ */ use crate::{ - object::{AnyId, JmapObject, JmapObjectId}, - request::{MaybeInvalid, deserialize::DeserializeArguments}, + object::{ + AnyId, JmapObject, JmapObjectId, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot}, + }, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments, next_lowercase_value}, + }, types::date::UTCDate, }; use calcard::jscontact::{JSContactProperty, JSContactValue}; -use jmap_tools::{JsonPointerItem, Key}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; use std::borrow::Cow; use types::{blob::BlobId, id::Id}; @@ -101,6 +107,7 @@ pub enum ContactCardFilter { OnlineService(String), Address(String), Note(String), + Metadata(MetadataFilter), _T(String), } @@ -145,44 +152,49 @@ impl<'de> DeserializeArguments<'de> for ContactCardFilter { *self = ContactCardFilter::UpdatedAfter(map.next_value()?); }, b"text" => { - *self = ContactCardFilter::Text(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Text(next_lowercase_value(map)?); }, b"name" => { - *self = ContactCardFilter::Name(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Name(next_lowercase_value(map)?); }, b"name/given" => { - *self = ContactCardFilter::NameGiven(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::NameGiven(next_lowercase_value(map)?); }, b"name/surname" => { - *self = ContactCardFilter::NameSurname(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::NameSurname(next_lowercase_value(map)?); }, b"name/surname2" => { - *self = ContactCardFilter::NameSurname2(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::NameSurname2(next_lowercase_value(map)?); }, b"nickname" => { - *self = ContactCardFilter::Nickname(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Nickname(next_lowercase_value(map)?); }, b"organization" => { - *self = ContactCardFilter::Organization(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Organization(next_lowercase_value(map)?); }, b"email" => { - *self = ContactCardFilter::Email(map.next_value()?); + *self = ContactCardFilter::Email(next_lowercase_value(map)?); }, b"phone" => { - *self = ContactCardFilter::Phone(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Phone(next_lowercase_value(map)?); }, b"onlineService" => { - *self = ContactCardFilter::OnlineService(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::OnlineService(next_lowercase_value(map)?); }, b"address" => { - *self = ContactCardFilter::Address(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Address(next_lowercase_value(map)?); }, b"note" => { - *self = ContactCardFilter::Note(map.next_value::>()?.to_lowercase()); + *self = ContactCardFilter::Note(next_lowercase_value(map)?); }, _ => { - *self = ContactCardFilter::_T(key.to_string()); - let _ = map.next_value::()?; + *self = match MetadataFilter::try_deserialize(key, map)? { + Some(filter) => ContactCardFilter::Metadata(filter), + None => { + let _ = map.next_value::()?; + ContactCardFilter::_T(key.to_string()) + } + }; } ); Ok(()) @@ -195,7 +207,7 @@ impl<'de> DeserializeArguments<'de> for ContactCardComparator { A: serde::de::MapAccess<'de>, { if key == "property" { - let value = map.next_value::>()?; + let value = map.next_value::()?.0; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = ContactCardComparator::Created; @@ -213,7 +225,7 @@ impl<'de> DeserializeArguments<'de> for ContactCardComparator { *self = ContactCardComparator::NameSurname2; }, _ => { - *self = ContactCardComparator::_T(value.to_string()); + *self = ContactCardComparator::_T(value.into_owned()); } ); } else { @@ -246,6 +258,7 @@ impl ContactCardFilter { ContactCardFilter::OnlineService(_) => "onlineService", ContactCardFilter::Address(_) => "address", ContactCardFilter::Note(_) => "note", + ContactCardFilter::Metadata(filter) => filter.as_str(), ContactCardFilter::_T(s) => return Cow::Owned(s), } .into() @@ -278,6 +291,30 @@ impl Default for ContactCardComparator { } } +impl MetadataProperty for JSContactProperty { + fn as_metadata_root(&self) -> Option { + match self { + JSContactProperty::Metadata => Some(MetadataRoot::Shared), + JSContactProperty::PrivateMetadata => Some(MetadataRoot::Private), + _ => None, + } + } + + fn as_pointer(&self) -> Option<&JsonPointer> { + match self { + JSContactProperty::Pointer(pointer) => Some(pointer), + _ => None, + } + } + + fn from_metadata_root(root: MetadataRoot) -> Self { + match root { + MetadataRoot::Shared => JSContactProperty::Metadata, + MetadataRoot::Private => JSContactProperty::PrivateMetadata, + } + } +} + impl JmapObjectId for JSContactProperty { fn as_id(&self) -> Option { if let JSContactProperty::IdValue(id) = self { diff --git a/crates/jmap-proto/src/object/email.rs b/crates/jmap-proto/src/object/email.rs index 9bf62a64..2ba45521 100644 --- a/crates/jmap-proto/src/object/email.rs +++ b/crates/jmap-proto/src/object/email.rs @@ -6,15 +6,22 @@ use crate::{ method::query::{Comparator, Filter}, - object::{AnyId, JmapObject, JmapObjectId, MaybeReference, parse_ref}, - request::{MaybeInvalid, deserialize::DeserializeArguments}, + object::{ + AnyId, JmapObject, JmapObjectId, MaybeReference, + metadata::{MetadataFilter, MetadataProperty, MetadataRoot, property_names}, + parse_ref, + }, + request::{ + MaybeInvalid, + deserialize::{CowStr, DeserializeArguments}, + }, types::date::UTCDate, }; -use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; +use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, PointerDepth, Property}; use mail_parser::HeaderName; -use serde::Serialize; +use serde::{Serialize, Serializer}; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use types::{blob::BlobId, id::Id, keyword::Keyword}; +use types::{blob::BlobId, id::Id, keyword::Keyword, text::Text}; #[derive(Debug, Clone, Default)] pub struct Email; @@ -73,6 +80,8 @@ pub enum EmailProperty { IsTruncated, HasAttachment, Preview, + Metadata, + PrivateMetadata, // Other Keyword(Keyword), @@ -86,6 +95,7 @@ pub struct HeaderProperty { pub form: HeaderForm, pub header: String, pub all: bool, + pub as_raw: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -109,24 +119,113 @@ pub enum EmailValue { impl Property for EmailProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { - let allow_patch = key.is_none(); - if let Some(Key::Property(key)) = key { - match key.patch_or_prop() { - EmailProperty::Keywords => EmailProperty::Keyword(Keyword::parse(value)).into(), + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) => match key.patch_or_prop() { + EmailProperty::Keywords => { + Keyword::try_from_jmap(value).map(EmailProperty::Keyword) + } EmailProperty::MailboxIds => match parse_ref(value) { MaybeReference::Value(v) => Some(EmailProperty::IdValue(v)), MaybeReference::Reference(v) => Some(EmailProperty::IdReference(v)), MaybeReference::ParseError => None, }, - _ => EmailProperty::parse(value, allow_patch), + _ => EmailProperty::parse_nested(value), + }, + Some(_) => EmailProperty::parse_nested(value), + None => EmailProperty::parse(value, Some(depth)), + } + } + + fn is_opaque(&self) -> bool { + self.metadata_root().is_some() + } + + fn to_cow(&self) -> Cow<'static, str> { + self.text().to_cow() + } + + fn key_eq(&self, other: &Self) -> bool { + if self.has_dynamic_text() || other.has_dynamic_text() { + self.text().eq_text(other.text()) + } else { + self == other + } + } + + fn key_eq_str(&self, other: &str) -> bool { + self.text().eq_str(other) + } + + fn serialize_text(&self, serializer: S) -> Result { + self.text().serialize(serializer) + } +} + +impl Element for EmailValue { + type Property = EmailProperty; + + fn try_parse

    (key: &Key<'_, Self::Property>, value: &str) -> Option { + if let Key::Property(prop) = key { + match prop.patch_or_prop() { + JSContactProperty::Type => JSContactType::from_str(value) + .ok() + .map(JSContactValue::Type), + JSContactProperty::CalendarScale => { + CalendarScale::parse(value.as_bytes()).map(JSContactValue::CalendarScale) + } + JSContactProperty::Created + | JSContactProperty::Updated + | JSContactProperty::Utc => value + .parse::() + .map(|dt| JSContactValue::Timestamp(dt.as_second())) + .ok(), + JSContactProperty::Kind => JSContactKind::from_str(value) + .ok() + .map(JSContactValue::Kind), + JSContactProperty::GrammaticalGender => JSContactGrammaticalGender::from_str(value) + .ok() + .map(JSContactValue::GrammaticalGender), + JSContactProperty::PhoneticSystem => JSContactPhoneticSystem::from_str(value) + .ok() + .map(JSContactValue::PhoneticSystem), + JSContactProperty::Relation => JSContactRelation::from_str(value) + .ok() + .map(JSContactValue::Relation), + JSContactProperty::Level => JSContactLevel::from_str(value) + .ok() + .map(JSContactValue::Level), + JSContactProperty::BlobId => match IdReference::parse(value) { + IdReference::Value(value) => JSContactValue::BlobId(value).into(), + IdReference::Reference(value) => JSContactValue::IdReference(value).into(), + IdReference::Error => None, + }, + JSContactProperty::Id => match IdReference::parse(value) { + IdReference::Value(value) => JSContactValue::Id(value).into(), + IdReference::Reference(value) => JSContactValue::IdReference(value).into(), + IdReference::Error => None, + }, + _ => None, + } + } else if key + .as_string_key() + .is_some_and(|name| name.ends_with(BLOB_ID_SUFFIX)) + { + match IdReference::parse(value) { + IdReference::Value(value) => JSContactValue::BlobId(value).into(), + IdReference::Reference(value) => JSContactValue::IdReference(value).into(), + IdReference::Error => None, + } + } else { + None + } + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + JSContactValue::Type(v) => v.as_str().into(), + JSContactValue::GrammaticalGender(v) => v.as_str().into(), + JSContactValue::Kind(v) => v.as_str().into(), + JSContactValue::Level(v) => v.as_str().into(), + JSContactValue::Relation(v) => v.as_str().into(), + JSContactValue::PhoneticSystem(v) => v.as_str().into(), + JSContactValue::Timestamp(v) => JSCalendarDateTime::new(*v, false).to_rfc3339().into(), + JSContactValue::CalendarScale(v) => v.as_js_str().into(), + JSContactValue::Id(v) => v.to_string().into(), + JSContactValue::BlobId(v) => v.to_string().into(), + JSContactValue::IdReference(s) => s.to_id_reference().into(), + } + } + + #[inline] + fn serialize_text(&self, serializer: S) -> Result { + match self { + JSContactValue::Timestamp(_) + | JSContactValue::Id(_) + | JSContactValue::BlobId(_) + | JSContactValue::IdReference(_) => self.serialize_data_text(serializer), + _ => serializer.serialize_str(&self.to_cow()), + } + } +} + +impl JSContactValue { + #[inline(never)] + fn serialize_data_text(&self, serializer: S) -> Result { + StackText::::serialize( + serializer, + |text| match self { + JSContactValue::Timestamp(timestamp) => { + JSCalendarDateTime::new(*timestamp, false).push_rfc3339(text) + } + JSContactValue::Id(id) => write!(text, "{id}"), + JSContactValue::BlobId(id) => write!(text, "{id}"), + JSContactValue::IdReference(id) => text.push_id_reference(id), + _ => Err(fmt::Error), + }, + || self.to_cow(), + ) + } +} + +impl jmap_tools::Property for JSContactProperty { + fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { + Self::try_parse_nested(key, value, PointerDepth::default()) + } + + fn try_parse_nested( + key: Option<&Key<'_, Self>>, + value: &str, + depth: PointerDepth, + ) -> Option { + match key { + Some(Key::Property(key)) if key.is_metadata() => None, + Some(Key::Property(key)) => match key.patch_or_prop() { + JSContactProperty::Contexts => Context::from_str(value) + .ok() + .map(JSContactProperty::Context), + JSContactProperty::Features => Feature::from_str(value) + .ok() + .map(JSContactProperty::Feature), + JSContactProperty::SortAs => JSContactKind::from_str(value) + .ok() + .map(JSContactProperty::SortAsKind), + JSContactProperty::ConvertedProperties | JSContactProperty::Localizations => { + JsonPointer::parse_nested(value, depth).map(JSContactProperty::Pointer) + } + JSContactProperty::AddressBookIds => match IdReference::parse(value) { + IdReference::Value(value) => JSContactProperty::IdValue(value).into(), + IdReference::Reference(value) => JSContactProperty::IdReference(value).into(), + IdReference::Error => None, + }, + JSContactProperty::Pointer(language) + if matches!( + language.as_slice(), + [JsonPointerItem::Key(Key::Borrowed(_) | Key::Owned(_))] + ) => + { + JSContactProperty::parse_name(value) + } + _ => JSContactProperty::parse_nested_name(value), + }, + None if value.contains('/') => { + JsonPointer::parse_nested(value, depth).map(JSContactProperty::Pointer) + } + None => JSContactProperty::parse_name(value), + Some(_) => JSContactProperty::parse_nested_name(value), + } + } + + fn is_opaque(&self) -> bool { + self.is_metadata() + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + JSContactProperty::Pointer(pointer) => pointer.to_text().into(), + JSContactProperty::IdReference(id) => id.to_id_reference().into(), + _ => self.to_string(), + } + } + + #[inline] + fn key_eq(&self, other: &Self) -> bool { + match (self, other) { + (a, b) if !a.has_data() && !b.has_data() => discriminant(a) == discriminant(b), + _ => self.data_key_eq(other), + } + } + + #[inline] + fn key_eq_str(&self, other: &str) -> bool { + if !self.has_data() { + self.to_string() == other + } else { + self.data_key_eq_str(other) + } + } + + #[inline] + fn serialize_text(&self, serializer: S) -> Result { + match self { + JSContactProperty::Pointer(_) + | JSContactProperty::IdValue(_) + | JSContactProperty::IdReference(_) => self.serialize_data_text(serializer), + _ => serializer.serialize_str(&self.to_cow()), + } + } +} + +impl JSContactProperty { + #[inline(never)] + fn serialize_data_text(&self, serializer: S) -> Result { + StackText::::serialize( + serializer, + |text| match self { + JSContactProperty::Pointer(pointer) => { + text.push_pointer(pointer, |key, text| text.push_pointer_token(&key.to_cow())) + } + JSContactProperty::IdValue(id) => write!(text, "{id}"), + JSContactProperty::IdReference(id) => text.push_id_reference(id), + _ => Err(fmt::Error), + }, + || self.to_cow(), + ) + } + + #[inline(never)] + fn data_key_eq(&self, other: &Self) -> bool { + match (self, other) { + (JSContactProperty::Pointer(a), JSContactProperty::Pointer(b)) if a == b => true, + _ => match (self.static_name(), other.static_name()) { + (Some(a), Some(b)) => a == b, + (Some(a), None) => other.data_key_eq_str(a), + (None, Some(b)) => self.data_key_eq_str(b), + (None, None) => other.data_key_eq_str(&self.to_string()), + }, + } + } + + #[inline(never)] + fn data_key_eq_str(&self, other: &str) -> bool { + match self { + JSContactProperty::IdValue(id) => id.display_eq(other), + JSContactProperty::Pointer(pointer) => pointer.text_eq(other), + JSContactProperty::IdReference(reference) => { + other.strip_prefix('#') == Some(reference.as_str()) + } + _ => self.to_string() == other, + } + } + + #[inline] + fn has_data(&self) -> bool { + match self { + JSContactProperty::IdValue(_) + | JSContactProperty::IdReference(_) + | JSContactProperty::Context(_) + | JSContactProperty::Feature(_) + | JSContactProperty::SortAsKind(_) + | JSContactProperty::Pointer(_) => true, + JSContactProperty::Type + | JSContactProperty::Address + | JSContactProperty::AddressBookIds + | JSContactProperty::Addresses + | JSContactProperty::Anniversaries + | JSContactProperty::Author + | JSContactProperty::BlobId + | JSContactProperty::Calendars + | JSContactProperty::CalendarScale + | JSContactProperty::Components + | JSContactProperty::Contexts + | JSContactProperty::ConvertedProperties + | JSContactProperty::Coordinates + | JSContactProperty::CountryCode + | JSContactProperty::Created + | JSContactProperty::CryptoKeys + | JSContactProperty::Date + | JSContactProperty::Day + | JSContactProperty::DefaultSeparator + | JSContactProperty::Directories + | JSContactProperty::Emails + | JSContactProperty::Extra + | JSContactProperty::Features + | JSContactProperty::Full + | JSContactProperty::GrammaticalGender + | JSContactProperty::Id + | JSContactProperty::IsOrdered + | JSContactProperty::Keywords + | JSContactProperty::Kind + | JSContactProperty::Label + | JSContactProperty::Language + | JSContactProperty::Level + | JSContactProperty::Links + | JSContactProperty::ListAs + | JSContactProperty::Localizations + | JSContactProperty::Media + | JSContactProperty::MediaType + | JSContactProperty::Members + | JSContactProperty::Month + | JSContactProperty::Name + | JSContactProperty::Nicknames + | JSContactProperty::Note + | JSContactProperty::Notes + | JSContactProperty::Number + | JSContactProperty::OnlineServices + | JSContactProperty::OrganizationId + | JSContactProperty::Organizations + | JSContactProperty::Parameters + | JSContactProperty::PersonalInfo + | JSContactProperty::Phones + | JSContactProperty::Phonetic + | JSContactProperty::PhoneticScript + | JSContactProperty::PhoneticSystem + | JSContactProperty::Place + | JSContactProperty::Pref + | JSContactProperty::PreferredLanguages + | JSContactProperty::ProdId + | JSContactProperty::Pronouns + | JSContactProperty::Properties + | JSContactProperty::RelatedTo + | JSContactProperty::Relation + | JSContactProperty::SchedulingAddresses + | JSContactProperty::Service + | JSContactProperty::SortAs + | JSContactProperty::SpeakToAs + | JSContactProperty::TimeZone + | JSContactProperty::Titles + | JSContactProperty::Uid + | JSContactProperty::Units + | JSContactProperty::Updated + | JSContactProperty::Uri + | JSContactProperty::User + | JSContactProperty::Utc + | JSContactProperty::VCard + | JSContactProperty::Value + | JSContactProperty::Version + | JSContactProperty::Year + | JSContactProperty::Metadata + | JSContactProperty::PrivateMetadata => false, + } + } + + fn static_name(&self) -> Option<&'static str> { + match self { + JSContactProperty::Pointer(_) + | JSContactProperty::IdValue(_) + | JSContactProperty::IdReference(_) => None, + _ => match self.to_string() { + Cow::Borrowed(name) => Some(name), + Cow::Owned(_) => None, + }, + } + } + + fn patch_or_prop(&self) -> &JSContactProperty { + if let JSContactProperty::Pointer(ptr) = self + && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() + { + prop + } else { + self + } + } +} + diff --git a/crates/calcard/src/jscontact/types.rs b/crates/calcard/src/jscontact/types.rs new file mode 100644 index 00000000..8a83e025 --- /dev/null +++ b/crates/calcard/src/jscontact/types.rs @@ -0,0 +1,602 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::jsprop::{property_names, text::PointerProperty}, + jscontact::*, +}; +use jmap_tools::JsonPointerItem; +use std::str::FromStr; + +impl PointerProperty for JSContactProperty { + fn into_pointer(self) -> Result, Self> { + match self { + JSContactProperty::Pointer(pointer) => Ok(pointer), + property => Err(property), + } + } +} + +impl FromStr for JSContactProperty { + type Err = (); + fn from_str(s: &str) -> Result { + JSContactProperty::parse_name(s) + .or_else(|| JSContactProperty::parse_metadata_selector(s)) + .ok_or(()) + } +} + +impl JSContactProperty { + property_names!( + JSContactProperty, + "@type" => Type, + "address" => Address, + "addressBookIds" => AddressBookIds, + "addresses" => Addresses, + "anniversaries" => Anniversaries, + "author" => Author, + "blobId" => BlobId, + "calendars" => Calendars, + "calendarScale" => CalendarScale, + "components" => Components, + "contexts" => Contexts, + "convertedProperties" => ConvertedProperties, + "coordinates" => Coordinates, + "countryCode" => CountryCode, + "created" => Created, + "cryptoKeys" => CryptoKeys, + "date" => Date, + "day" => Day, + "defaultSeparator" => DefaultSeparator, + "directories" => Directories, + "emails" => Emails, + "extra" => Extra, + "features" => Features, + "full" => Full, + "grammaticalGender" => GrammaticalGender, + "id" => Id, + "isOrdered" => IsOrdered, + "keywords" => Keywords, + "kind" => Kind, + "label" => Label, + "language" => Language, + "level" => Level, + "links" => Links, + "listAs" => ListAs, + "localizations" => Localizations, + "media" => Media, + "mediaType" => MediaType, + "members" => Members, + "month" => Month, + "name" => Name, + "nicknames" => Nicknames, + "note" => Note, + "notes" => Notes, + "number" => Number, + "onlineServices" => OnlineServices, + "organizationId" => OrganizationId, + "organizations" => Organizations, + "parameters" => Parameters, + "personalInfo" => PersonalInfo, + "phones" => Phones, + "phonetic" => Phonetic, + "phoneticScript" => PhoneticScript, + "phoneticSystem" => PhoneticSystem, + "place" => Place, + "pref" => Pref, + "preferredLanguages" => PreferredLanguages, + "prodId" => ProdId, + "pronouns" => Pronouns, + "properties" => Properties, + "relatedTo" => RelatedTo, + "relation" => Relation, + "schedulingAddresses" => SchedulingAddresses, + "service" => Service, + "sortAs" => SortAs, + "speakToAs" => SpeakToAs, + "timeZone" => TimeZone, + "titles" => Titles, + "uid" => Uid, + "units" => Units, + "updated" => Updated, + "uri" => Uri, + "user" => User, + "utc" => Utc, + "value" => Value, + "vCard" => VCard, + "version" => Version, + "year" => Year, + ); + + fn parse_metadata_selector(s: &str) -> Option { + s.split_once('/') + .and_then(|(root, _)| JSContactProperty::::parse_name(root)) + .filter(JSContactProperty::is_metadata) + .map(|_| JSContactProperty::Pointer(JsonPointer::parse(s))) + } + + pub(crate) fn is_metadata(&self) -> bool { + match self { + JSContactProperty::Metadata | JSContactProperty::PrivateMetadata => true, + JSContactProperty::Pointer(pointer) => JSContactProperty::is_metadata_pointer(pointer), + _ => false, + } + } + + pub(crate) fn is_metadata_pointer(pointer: &JsonPointer) -> bool { + pointer + .first() + .and_then(JsonPointerItem::as_property_key) + .is_some_and(JSContactProperty::is_metadata) + } + + pub fn to_string(&self) -> Cow<'static, str> { + match self { + JSContactProperty::Type => "@type", + JSContactProperty::Address => "address", + JSContactProperty::AddressBookIds => "addressBookIds", + JSContactProperty::Addresses => "addresses", + JSContactProperty::Anniversaries => "anniversaries", + JSContactProperty::Author => "author", + JSContactProperty::BlobId => "blobId", + JSContactProperty::Calendars => "calendars", + JSContactProperty::CalendarScale => "calendarScale", + JSContactProperty::Components => "components", + JSContactProperty::Contexts => "contexts", + JSContactProperty::ConvertedProperties => "convertedProperties", + JSContactProperty::Coordinates => "coordinates", + JSContactProperty::CountryCode => "countryCode", + JSContactProperty::Created => "created", + JSContactProperty::CryptoKeys => "cryptoKeys", + JSContactProperty::Date => "date", + JSContactProperty::Day => "day", + JSContactProperty::DefaultSeparator => "defaultSeparator", + JSContactProperty::Directories => "directories", + JSContactProperty::Emails => "emails", + JSContactProperty::Extra => "extra", + JSContactProperty::Features => "features", + JSContactProperty::Full => "full", + JSContactProperty::GrammaticalGender => "grammaticalGender", + JSContactProperty::Id => "id", + JSContactProperty::IsOrdered => "isOrdered", + JSContactProperty::Keywords => "keywords", + JSContactProperty::Kind => "kind", + JSContactProperty::Label => "label", + JSContactProperty::Language => "language", + JSContactProperty::Level => "level", + JSContactProperty::Links => "links", + JSContactProperty::ListAs => "listAs", + JSContactProperty::Localizations => "localizations", + JSContactProperty::Media => "media", + JSContactProperty::MediaType => "mediaType", + JSContactProperty::Members => "members", + JSContactProperty::Month => "month", + JSContactProperty::Name => "name", + JSContactProperty::Nicknames => "nicknames", + JSContactProperty::Note => "note", + JSContactProperty::Notes => "notes", + JSContactProperty::Number => "number", + JSContactProperty::OnlineServices => "onlineServices", + JSContactProperty::OrganizationId => "organizationId", + JSContactProperty::Organizations => "organizations", + JSContactProperty::Parameters => "parameters", + JSContactProperty::PersonalInfo => "personalInfo", + JSContactProperty::Phones => "phones", + JSContactProperty::Phonetic => "phonetic", + JSContactProperty::PhoneticScript => "phoneticScript", + JSContactProperty::PhoneticSystem => "phoneticSystem", + JSContactProperty::Place => "place", + JSContactProperty::Pref => "pref", + JSContactProperty::PreferredLanguages => "preferredLanguages", + JSContactProperty::ProdId => "prodId", + JSContactProperty::Pronouns => "pronouns", + JSContactProperty::Properties => "properties", + JSContactProperty::RelatedTo => "relatedTo", + JSContactProperty::Relation => "relation", + JSContactProperty::SchedulingAddresses => "schedulingAddresses", + JSContactProperty::Service => "service", + JSContactProperty::SortAs => "sortAs", + JSContactProperty::SpeakToAs => "speakToAs", + JSContactProperty::TimeZone => "timeZone", + JSContactProperty::Titles => "titles", + JSContactProperty::Uid => "uid", + JSContactProperty::Units => "units", + JSContactProperty::Updated => "updated", + JSContactProperty::Uri => "uri", + JSContactProperty::User => "user", + JSContactProperty::Utc => "utc", + JSContactProperty::Value => "value", + JSContactProperty::VCard => "vCard", + JSContactProperty::Version => "version", + JSContactProperty::Year => "year", + JSContactProperty::Metadata => "metadata", + JSContactProperty::PrivateMetadata => "privateMetadata", + JSContactProperty::Context(context) => context.as_str(), + JSContactProperty::Feature(feature) => feature.as_str(), + JSContactProperty::SortAsKind(kind) => kind.as_str(), + JSContactProperty::Pointer(pointer) => return Cow::Owned(pointer.to_string()), + JSContactProperty::IdValue(id) => return Cow::Owned(id.to_string()), + JSContactProperty::IdReference(s) => return Cow::Owned(format!("#{}", s)), + } + .into() + } +} + +impl FromStr for Context { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), Context, + "billing" => Context::Billing, + "delivery" => Context::Delivery, + "private" => Context::Private, + "work" => Context::Work, + ) + .copied() + .ok_or(()) + } +} + +impl Context { + pub fn as_str(&self) -> &'static str { + match self { + Context::Billing => "billing", + Context::Delivery => "delivery", + Context::Private => "private", + Context::Work => "work", + } + } +} + +impl FromStr for Feature { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), Feature, + "fax" => Feature::Fax, + "main-number" => Feature::MainNumber, + "mobile" => Feature::Mobile, + "pager" => Feature::Pager, + "text" => Feature::Text, + "textphone" => Feature::TextPhone, + "video" => Feature::Video, + "voice" => Feature::Voice, + ) + .copied() + .ok_or(()) + } +} + +impl Feature { + pub fn as_str(&self) -> &'static str { + match self { + Feature::Fax => "fax", + Feature::MainNumber => "main-number", + Feature::Mobile => "mobile", + Feature::Pager => "pager", + Feature::Text => "text", + Feature::TextPhone => "textphone", + Feature::Video => "video", + Feature::Voice => "voice", + } + } +} + +impl FromStr for JSContactGrammaticalGender { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactGrammaticalGender, + "animate" => JSContactGrammaticalGender::Animate, + "common" => JSContactGrammaticalGender::Common, + "feminine" => JSContactGrammaticalGender::Feminine, + "inanimate" => JSContactGrammaticalGender::Inanimate, + "masculine" => JSContactGrammaticalGender::Masculine, + "neuter" => JSContactGrammaticalGender::Neuter, + ) + .copied() + .ok_or(()) + } +} + +impl JSContactGrammaticalGender { + pub fn as_str(&self) -> &'static str { + match self { + JSContactGrammaticalGender::Animate => "animate", + JSContactGrammaticalGender::Common => "common", + JSContactGrammaticalGender::Feminine => "feminine", + JSContactGrammaticalGender::Inanimate => "inanimate", + JSContactGrammaticalGender::Masculine => "masculine", + JSContactGrammaticalGender::Neuter => "neuter", + } + } +} + +impl FromStr for JSContactKind { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactKind, + "apartment" => JSContactKind::Apartment, + "block" => JSContactKind::Block, + "building" => JSContactKind::Building, + "country" => JSContactKind::Country, + "direction" => JSContactKind::Direction, + "district" => JSContactKind::District, + "floor" => JSContactKind::Floor, + "landmark" => JSContactKind::Landmark, + "locality" => JSContactKind::Locality, + "name" => JSContactKind::Name, + "number" => JSContactKind::Number, + "postcode" => JSContactKind::Postcode, + "postOfficeBox" => JSContactKind::PostOfficeBox, + "region" => JSContactKind::Region, + "room" => JSContactKind::Room, + "separator" => JSContactKind::Separator, + "subdistrict" => JSContactKind::Subdistrict, + "birth" => JSContactKind::Birth, + "death" => JSContactKind::Death, + "wedding" => JSContactKind::Wedding, + "calendar" => JSContactKind::Calendar, + "freeBusy" => JSContactKind::FreeBusy, + "application" => JSContactKind::Application, + "device" => JSContactKind::Device, + "group" => JSContactKind::Group, + "individual" => JSContactKind::Individual, + "location" => JSContactKind::Location, + "org" => JSContactKind::Org, + "directory" => JSContactKind::Directory, + "entry" => JSContactKind::Entry, + "contact" => JSContactKind::Contact, + "logo" => JSContactKind::Logo, + "photo" => JSContactKind::Photo, + "sound" => JSContactKind::Sound, + "credential" => JSContactKind::Credential, + "generation" => JSContactKind::Generation, + "given" => JSContactKind::Given, + "given2" => JSContactKind::Given2, + "surname" => JSContactKind::Surname, + "surname2" => JSContactKind::Surname2, + "title" => JSContactKind::Title, + "expertise" => JSContactKind::Expertise, + "hobby" => JSContactKind::Hobby, + "interest" => JSContactKind::Interest, + "role" => JSContactKind::Role, + ) + .copied() + .ok_or(()) + } +} + +#[allow(clippy::wrong_self_convention)] +impl JSContactKind { + pub fn as_str(&self) -> &'static str { + match self { + JSContactKind::Apartment => "apartment", + JSContactKind::Block => "block", + JSContactKind::Building => "building", + JSContactKind::Country => "country", + JSContactKind::Direction => "direction", + JSContactKind::District => "district", + JSContactKind::Floor => "floor", + JSContactKind::Landmark => "landmark", + JSContactKind::Locality => "locality", + JSContactKind::Name => "name", + JSContactKind::Number => "number", + JSContactKind::Postcode => "postcode", + JSContactKind::PostOfficeBox => "postOfficeBox", + JSContactKind::Region => "region", + JSContactKind::Room => "room", + JSContactKind::Separator => "separator", + JSContactKind::Subdistrict => "subdistrict", + JSContactKind::Birth => "birth", + JSContactKind::Death => "death", + JSContactKind::Wedding => "wedding", + JSContactKind::Calendar => "calendar", + JSContactKind::FreeBusy => "freeBusy", + JSContactKind::Application => "application", + JSContactKind::Device => "device", + JSContactKind::Group => "group", + JSContactKind::Individual => "individual", + JSContactKind::Location => "location", + JSContactKind::Org => "org", + JSContactKind::Directory => "directory", + JSContactKind::Entry => "entry", + JSContactKind::Contact => "contact", + JSContactKind::Logo => "logo", + JSContactKind::Photo => "photo", + JSContactKind::Sound => "sound", + JSContactKind::Credential => "credential", + JSContactKind::Generation => "generation", + JSContactKind::Given => "given", + JSContactKind::Given2 => "given2", + JSContactKind::Surname => "surname", + JSContactKind::Surname2 => "surname2", + JSContactKind::Title => "title", + JSContactKind::Expertise => "expertise", + JSContactKind::Hobby => "hobby", + JSContactKind::Interest => "interest", + JSContactKind::Role => "role", + } + } + + pub(crate) fn from_vcard_adr_pos(pos: usize) -> Option { + match pos { + 0 => JSContactKind::PostOfficeBox, + 1 => JSContactKind::Apartment, + 2 => JSContactKind::Name, + 3 => JSContactKind::Locality, + 4 => JSContactKind::Region, + 5 => JSContactKind::Postcode, + 6 => JSContactKind::Country, + 7 => JSContactKind::Room, + 8 => JSContactKind::Apartment, + 9 => JSContactKind::Floor, + 10 => JSContactKind::Number, + 11 => JSContactKind::Name, + 12 => JSContactKind::Building, + 13 => JSContactKind::Block, + 14 => JSContactKind::Subdistrict, + 15 => JSContactKind::District, + 16 => JSContactKind::Landmark, + 17 => JSContactKind::Direction, + _ => return None, + } + .into() + } + + pub(crate) fn to_vcard_adr_pos(&self) -> Option { + match self { + JSContactKind::PostOfficeBox => 0, + JSContactKind::Apartment => 8, + JSContactKind::Name => 11, + JSContactKind::Locality => 3, + JSContactKind::Region => 4, + JSContactKind::Postcode => 5, + JSContactKind::Country => 6, + JSContactKind::Room => 7, + JSContactKind::Floor => 9, + JSContactKind::Number => 10, + JSContactKind::Building => 12, + JSContactKind::Block => 13, + JSContactKind::Subdistrict => 14, + JSContactKind::District => 15, + JSContactKind::Landmark => 16, + JSContactKind::Direction => 17, + _ => return None, + } + .into() + } + + pub(crate) fn from_vcard_n_pos(pos: usize) -> Option { + match pos { + 0 => JSContactKind::Surname, + 1 => JSContactKind::Given, + 2 => JSContactKind::Given2, + 3 => JSContactKind::Title, + 4 => JSContactKind::Credential, + 5 => JSContactKind::Surname2, + 6 => JSContactKind::Generation, + _ => return None, + } + .into() + } + + pub(crate) fn to_vcard_n_pos(&self) -> Option { + match self { + JSContactKind::Surname => 0, + JSContactKind::Given => 1, + JSContactKind::Given2 => 2, + JSContactKind::Title => 3, + JSContactKind::Credential => 4, + JSContactKind::Surname2 => 5, + JSContactKind::Generation => 6, + _ => return None, + } + .into() + } +} + +impl FromStr for JSContactLevel { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactLevel, + "high" => JSContactLevel::High, + "low" => JSContactLevel::Low, + "medium" => JSContactLevel::Medium, + ) + .copied() + .ok_or(()) + } +} + +impl JSContactLevel { + pub fn as_str(&self) -> &'static str { + match self { + JSContactLevel::High => "high", + JSContactLevel::Low => "low", + JSContactLevel::Medium => "medium", + } + } +} + +impl FromStr for JSContactRelation { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactRelation, + "acquaintance" => JSContactRelation::Acquaintance, + "agent" => JSContactRelation::Agent, + "child" => JSContactRelation::Child, + "colleague" => JSContactRelation::Colleague, + "contact" => JSContactRelation::Contact, + "co-resident" => JSContactRelation::CoResident, + "co-worker" => JSContactRelation::CoWorker, + "crush" => JSContactRelation::Crush, + "date" => JSContactRelation::Date, + "emergency" => JSContactRelation::Emergency, + "friend" => JSContactRelation::Friend, + "kin" => JSContactRelation::Kin, + "me" => JSContactRelation::Me, + "met" => JSContactRelation::Met, + "muse" => JSContactRelation::Muse, + "neighbor" => JSContactRelation::Neighbor, + "parent" => JSContactRelation::Parent, + "sibling" => JSContactRelation::Sibling, + "spouse" => JSContactRelation::Spouse, + "sweetheart" => JSContactRelation::Sweetheart, + ) + .copied() + .ok_or(()) + } +} + +impl JSContactRelation { + pub fn as_str(&self) -> &'static str { + match self { + JSContactRelation::Acquaintance => "acquaintance", + JSContactRelation::Agent => "agent", + JSContactRelation::Child => "child", + JSContactRelation::Colleague => "colleague", + JSContactRelation::Contact => "contact", + JSContactRelation::CoResident => "co-resident", + JSContactRelation::CoWorker => "co-worker", + JSContactRelation::Crush => "crush", + JSContactRelation::Date => "date", + JSContactRelation::Emergency => "emergency", + JSContactRelation::Friend => "friend", + JSContactRelation::Kin => "kin", + JSContactRelation::Me => "me", + JSContactRelation::Met => "met", + JSContactRelation::Muse => "muse", + JSContactRelation::Neighbor => "neighbor", + JSContactRelation::Parent => "parent", + JSContactRelation::Sibling => "sibling", + JSContactRelation::Spouse => "spouse", + JSContactRelation::Sweetheart => "sweetheart", + } + } +} + +impl FromStr for JSContactPhoneticSystem { + type Err = (); + fn from_str(s: &str) -> Result { + hashify::map!(s.as_bytes(), JSContactPhoneticSystem, + "ipa" => JSContactPhoneticSystem::Ipa, + "jyut" => JSContactPhoneticSystem::Jyut, + "piny" => JSContactPhoneticSystem::Piny, + "script" => JSContactPhoneticSystem::Script, + ) + .copied() + .ok_or(()) + } +} + +impl JSContactPhoneticSystem { + pub fn as_str(&self) -> &'static str { + match self { + JSContactPhoneticSystem::Ipa => "ipa", + JSContactPhoneticSystem::Jyut => "jyut", + JSContactPhoneticSystem::Piny => "piny", + JSContactPhoneticSystem::Script => "script", + } + } +} diff --git a/crates/calcard/src/lib.rs b/crates/calcard/src/lib.rs new file mode 100644 index 00000000..8ab05b5a --- /dev/null +++ b/crates/calcard/src/lib.rs @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ +#![deny(rust_2018_idioms)] +#![forbid(unsafe_code)] +use common::tokenizer::{Mode, StopChar, Token}; +use icalendar::{ICalendar, ICalendarComponentType}; +use std::borrow::Cow; +use vcard::VCard; + +pub mod common; +pub mod datecalc; +pub mod icalendar; +#[cfg(feature = "jmap")] +pub mod jscalendar; +#[cfg(feature = "jmap")] +pub mod jscontact; +pub mod vcard; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Entry { + VCard(VCard), + ICalendar(ICalendar), + InvalidLine(String), + UnexpectedComponentEnd { + expected: ICalendarComponentType, + found: ICalendarComponentType, + }, + UnterminatedComponent(Cow<'static, str>), + TooManyComponents, + Eof, +} + +const PRESIZE_BYTES_PER_SLOT: usize = 8; + +pub struct Parser<'x> { + pub(crate) input: &'x [u8], + pub(crate) source: &'x str, + pub(crate) pos: usize, + pub(crate) strict: bool, + pub(crate) mode: Mode, + pub(crate) token_buf: Vec>, + pub(crate) last_token_end: usize, + presize_budget: usize, +} + +impl<'x> Parser<'x> { + pub fn new(input: &'x str) -> Self { + Self { + input: input.as_bytes(), + source: input, + pos: 0, + strict: false, + mode: Mode::INITIAL, + token_buf: Vec::new(), + last_token_end: usize::MAX, + presize_budget: input.len() / PRESIZE_BYTES_PER_SLOT, + } + } + + pub fn strict(mut self) -> Self { + self.strict = true; + self + } + + pub(crate) fn presize(&mut self, items: &mut Vec, hint: usize) { + let slots = hint.min(self.presize_budget); + self.presize_budget -= slots; + items.reserve_exact(slots); + } + + pub fn entry(&mut self) -> Entry { + self.expect_iana_token(); + + loop { + if let Some(token) = self.token() { + if (token.text.eq_ignore_ascii_case(b"BEGIN") + || token.text.eq_ignore_ascii_case("\u{feff}BEGIN".as_bytes())) + && token.stop_char == StopChar::Colon + { + if let Some(token) = self.token() { + if token.stop_char == StopChar::Lf { + hashify::fnc_map_ignore_case!(token.text.as_ref(), + b"VCARD" => { return self.vcard(); }, + b"VCALENDAR" => { return self.icalendar(ICalendarComponentType::VCalendar); }, + b"VEVENT" => { return self.icalendar(ICalendarComponentType::VEvent); }, + b"VTODO" => { return self.icalendar(ICalendarComponentType::VTodo); }, + b"VJOURNAL" => { return self.icalendar(ICalendarComponentType::VJournal); }, + b"VFREEBUSY" => { return self.icalendar(ICalendarComponentType::VFreebusy); }, + b"VTIMEZONE" => { return self.icalendar(ICalendarComponentType::VTimezone); }, + b"VALARM" => { return self.icalendar(ICalendarComponentType::VAlarm); }, + b"STANDARD" => { return self.icalendar(ICalendarComponentType::Standard); }, + b"DAYLIGHT" => { return self.icalendar(ICalendarComponentType::Daylight); }, + b"VAVAILABILITY" => { return self.icalendar(ICalendarComponentType::VAvailability); }, + b"AVAILABLE" => { return self.icalendar(ICalendarComponentType::Available); }, + b"PARTICIPANT" => { return self.icalendar(ICalendarComponentType::Participant); }, + b"VLOCATION" => { return self.icalendar(ICalendarComponentType::VLocation); }, + b"VRESOURCE" => { return self.icalendar(ICalendarComponentType::VResource); }, + _ => { + return self.icalendar(ICalendarComponentType::Other(token.into_string())); + } + ) + } + } else { + return Entry::Eof; + } + } + + let token_start = token.start; + let mut token_end = token.end; + + if token.stop_char != StopChar::Lf { + self.expect_single_value(); + while let Some(token) = self.token() { + token_end = token.end; + if token.stop_char == StopChar::Lf { + break; + } + } + } else if token.text.is_empty() { + continue; + } + + return Entry::InvalidLine( + self.source + .get(token_start..=token_end) + .unwrap_or_default() + .to_string(), + ); + } else { + return Entry::Eof; + } + } + } +} diff --git a/crates/calcard/src/vcard/builder.rs b/crates/calcard/src/vcard/builder.rs new file mode 100644 index 00000000..81cfe9d4 --- /dev/null +++ b/crates/calcard/src/vcard/builder.rs @@ -0,0 +1,348 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::parser::{Boolean, Timestamp}, + vcard::*, +}; +use smallvec::SmallVec; + +impl VCardEntry { + pub fn new(name: VCardProperty) -> Self { + Self { + group: None, + name, + params: vec![], + values: SmallVec::new(), + } + } + + pub fn with_params(mut self, params: Vec) -> Self { + self.params = params; + self + } + + pub fn with_group(mut self, group: Option) -> Self { + self.group = group; + self + } + + pub fn with_value(mut self, value: impl Into) -> Self { + self.values.push(value.into()); + self + } + + pub fn with_values(mut self, values: impl IntoIterator) -> Self { + self.values = values.into_iter().collect(); + self + } + + pub fn with_param(mut self, param: impl Into) -> Self { + self.params.push(param.into()); + self + } + + pub fn add_param(&mut self, param: impl Into) { + self.params.push(param.into()); + } + + pub fn with_param_opt(mut self, param: Option>) -> Self { + if let Some(param) = param { + self.params.push(param.into()); + } + self + } + + pub fn is_type(&self, typ: &VCardValueType) -> bool { + self.parameters(&VCardParameterName::Value) + .any(|p| matches!(p, VCardParameterValue::ValueType(v) if v == typ)) + } +} + +impl VCardParameter { + pub fn new(name: VCardParameterName, value: impl Into) -> Self { + VCardParameter { + name, + value: value.into(), + } + } + + pub fn language(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Language, + value: value.into(), + } + } + + pub fn value(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Value, + value: value.into(), + } + } + + pub fn pref(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Pref, + value: value.into(), + } + } + + pub fn altid(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Altid, + value: value.into(), + } + } + + pub fn pid(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Pid, + value: value.into(), + } + } + + pub fn typ(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Type, + value: value.into(), + } + } + + pub fn mediatype(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Mediatype, + value: value.into(), + } + } + + pub fn calscale(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Calscale, + value: value.into(), + } + } + + pub fn sort_as(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::SortAs, + value: value.into(), + } + } + + pub fn geo(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Geo, + value: value.into(), + } + } + + pub fn tz(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Tz, + value: value.into(), + } + } + + pub fn index(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Index, + value: value.into(), + } + } + + pub fn level(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Level, + value: value.into(), + } + } + + pub fn group(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Group, + value: value.into(), + } + } + + pub fn cc(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Cc, + value: value.into(), + } + } + + pub fn author(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Author, + value: value.into(), + } + } + + pub fn author_name(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::AuthorName, + value: value.into(), + } + } + + pub fn created(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Created, + value: value.into(), + } + } + + pub fn derived(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Derived, + value: value.into(), + } + } + + pub fn label(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Label, + value: value.into(), + } + } + + pub fn phonetic(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Phonetic, + value: value.into(), + } + } + + pub fn prop_id(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::PropId, + value: value.into(), + } + } + + pub fn script(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Script, + value: value.into(), + } + } + + pub fn service_type(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::ServiceType, + value: value.into(), + } + } + + pub fn username(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Username, + value: value.into(), + } + } + + pub fn jsptr(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Jsptr, + value: value.into(), + } + } + + pub fn jscomps(value: impl Into) -> Self { + VCardParameter { + name: VCardParameterName::Jscomps, + value: value.into(), + } + } +} + +impl From for VCardValue { + fn from(text: String) -> Self { + VCardValue::Text(text) + } +} + +impl From for VCardValue { + fn from(kind: VCardKind) -> Self { + VCardValue::Kind(kind) + } +} + +impl From for VCardParameterValue { + fn from(value: String) -> Self { + VCardParameterValue::Text(value) + } +} + +impl From for VCardParameterValue { + fn from(value: u32) -> Self { + VCardParameterValue::Integer(value) + } +} + +impl From for VCardParameterValue { + fn from(value: VCardType) -> Self { + VCardParameterValue::Type(value) + } +} + +impl From for VCardParameterValue { + fn from(value: VCardValueType) -> Self { + VCardParameterValue::ValueType(value) + } +} + +impl From for VCardParameterValue { + fn from(value: CalendarScale) -> Self { + VCardParameterValue::Calscale(value) + } +} + +impl From for VCardParameterValue { + fn from(value: VCardPhonetic) -> Self { + VCardParameterValue::Phonetic(value) + } +} + +impl From for VCardParameterValue { + fn from(value: VCardLevel) -> Self { + VCardParameterValue::Level(value) + } +} + +impl From for VCardParameterValue { + fn from(value: bool) -> Self { + VCardParameterValue::Bool(value) + } +} + +impl From> for VCardParameterValue { + fn from(value: Vec) -> Self { + VCardParameterValue::Jscomps(value) + } +} + +impl From for VCardParameterValue { + fn from(value: Timestamp) -> Self { + VCardParameterValue::Timestamp(value.0) + } +} + +impl From for VCardParameterValue { + fn from(value: Boolean) -> Self { + VCardParameterValue::Bool(value.0) + } +} + +impl> From> for VCardParameterValue { + fn from(value: IanaType) -> Self { + match value { + IanaType::Iana(v) => v.into(), + IanaType::Other(s) => s.into(), + } + } +} diff --git a/crates/calcard/src/vcard/media_type.rs b/crates/calcard/src/vcard/media_type.rs new file mode 100644 index 00000000..e474616a --- /dev/null +++ b/crates/calcard/src/vcard/media_type.rs @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +pub(crate) fn media_type_from_legacy(property: &str, token: &str) -> Option> { + if token.contains('/') { + Some(Cow::Owned(token.to_ascii_lowercase())) + } else if let Some(media_type) = legacy_alias(property, token.as_bytes()) { + Some(Cow::Borrowed(media_type)) + } else { + let subtype = token.to_ascii_lowercase(); + match property { + "PHOTO" | "LOGO" if is_image_subtype(subtype.as_bytes()) => { + Some(Cow::Owned(format!("image/{subtype}"))) + } + "SOUND" if is_audio_subtype(subtype.as_bytes()) => { + Some(Cow::Owned(format!("audio/{subtype}"))) + } + _ => None, + } + } +} + +pub(crate) fn legacy_media_type(property: &str, media_type: &str) -> Option { + let token = match media_type { + "application/pgp-keys" => "PGP".to_string(), + "application/x-x509-user-cert" => "X509".to_string(), + _ => media_type.split_once('/')?.1.to_ascii_uppercase(), + }; + + (media_type_from_legacy(property, &token).as_deref() == Some(media_type)).then_some(token) +} + +fn legacy_alias(property: &str, token: &[u8]) -> Option<&'static str> { + match property { + "PHOTO" | "LOGO" => hashify::map_ignore_case!(token, &'static str, + b"JPG" => "image/jpeg", + b"TIF" => "image/tiff", + ) + .copied(), + "SOUND" => hashify::map_ignore_case!(token, &'static str, + b"AIF" => "audio/aiff", + b"AIFF" => "audio/aiff", + b"MP3" => "audio/mpeg", + b"PCM" => "audio/l16", + b"WAV" => "audio/wav", + b"WAVE" => "audio/wav", + ) + .copied(), + "KEY" => hashify::map_ignore_case!(token, &'static str, + b"GPG" => "application/pgp-keys", + b"PGP" => "application/pgp-keys", + b"X509" => "application/x-x509-user-cert", + ) + .copied(), + _ => None, + } +} + +fn is_image_subtype(subtype: &[u8]) -> bool { + hashify::set!( + subtype, + "aces", + "apng", + "avci", + "avcs", + "avif", + "bmp", + "cgm", + "dicom-rle", + "dpx", + "emf", + "example", + "fits", + "g3fax", + "gif", + "heic", + "heic-sequence", + "heif", + "heif-sequence", + "hej2k", + "hsj2", + "ief", + "j2c", + "jaii", + "jais", + "jls", + "jp2", + "jpeg", + "jph", + "jphc", + "jpm", + "jpx", + "jxl", + "jxr", + "jxra", + "jxrs", + "jxs", + "jxsc", + "jxsi", + "jxss", + "ktx", + "ktx2", + "naplps", + "png", + "prs.aimg", + "prs.btif", + "prs.pti", + "pwg-raster", + "svg+xml", + "t38", + "tiff", + "tiff-fx", + "vnd.adobe.photoshop", + "vnd.airzip.accelerator.azv", + "vnd.blockfact.facti", + "vnd.clip", + "vnd.cns.inf2", + "vnd.dece.graphic", + "vnd.djvu", + "vnd.dvb.subtitle", + "vnd.dwg", + "vnd.dxf", + "vnd.fastbidsheet", + "vnd.fpx", + "vnd.fst", + "vnd.fujixerox.edmics-mmr", + "vnd.fujixerox.edmics-rlc", + "vnd.globalgraphics.pgb", + "vnd.microsoft.icon", + "vnd.mix", + "vnd.mozilla.apng", + "vnd.ms-modi", + "vnd.net-fpx", + "vnd.pco.b16", + "vnd.radiance", + "vnd.sealed.png", + "vnd.sealedmedia.softseal.gif", + "vnd.sealedmedia.softseal.jpg", + "vnd.sld", + "vnd.svf", + "vnd.tencent.tap", + "vnd.valve.source.texture", + "vnd.wap.wbmp", + "vnd.xiff", + "vnd.zbrush.pcx", + "webp", + "wmf", + "x-emf", + "x-wmf", + ) +} + +fn is_audio_subtype(subtype: &[u8]) -> bool { + hashify::set!( + subtype, + "1d-interleaved-parityfec", + "32kadpcm", + "3gpp", + "3gpp2", + "amr", + "amr-wb", + "atrac-advanced-lossless", + "atrac-x", + "atrac3", + "bv16", + "bv32", + "cn", + "dat12", + "dv", + "dvi4", + "evrc", + "evrc-qcp", + "evrc0", + "evrc1", + "evrcb", + "evrcb0", + "evrcb1", + "evrcnw", + "evrcnw0", + "evrcnw1", + "evrcwb", + "evrcwb0", + "evrcwb1", + "evs", + "g711-0", + "g719", + "g722", + "g7221", + "g723", + "g726-16", + "g726-24", + "g726-32", + "g726-40", + "g728", + "g729", + "g7291", + "g729d", + "g729e", + "gsm", + "gsm-efr", + "gsm-hr-08", + "l16", + "l20", + "l24", + "l8", + "lpc", + "melp", + "melp1200", + "melp2400", + "melp600", + "mp4a-latm", + "mpa", + "pcma", + "pcma-wb", + "pcmu", + "pcmu-wb", + "qcelp", + "red", + "smv", + "smv-qcp", + "smv0", + "tetra_acelp", + "tetra_acelp_bb", + "tsvcis", + "uemclip", + "vdvi", + "vmr-wb", + "aac", + "ac3", + "amr-wb+", + "aptx", + "asc", + "basic", + "clearmode", + "dls", + "dsr-es201108", + "dsr-es202050", + "dsr-es202211", + "dsr-es202212", + "eac3", + "encaprtp", + "example", + "flac", + "flexfec", + "fwdred", + "ilbc", + "ip-mr_v2.5", + "matroska", + "mhas", + "midi-clip", + "mobile-xmf", + "mp4", + "mpa-robust", + "mpeg", + "mpeg4-generic", + "ogg", + "opus", + "parityfec", + "prs.aaud", + "prs.sid", + "raptorfec", + "rtp-enc-aescm128", + "rtp-midi", + "rtploopback", + "rtx", + "scip", + "sofa", + "soundfont", + "sp-midi", + "speex", + "t140c", + "t38", + "telephone-event", + "tone", + "ulpfec", + "usac", + "vnd.3gpp.iufp", + "vnd.4sb", + "vnd.celp", + "vnd.audiokoz", + "vnd.blockfact.facta", + "vnd.cisco.nse", + "vnd.cmles.radio-events", + "vnd.cns.anp1", + "vnd.cns.inf1", + "vnd.dece.audio", + "vnd.digital-winds", + "vnd.dlna.adts", + "vnd.dolby.heaac.1", + "vnd.dolby.heaac.2", + "vnd.dolby.mlp", + "vnd.dolby.mps", + "vnd.dolby.pl2", + "vnd.dolby.pl2x", + "vnd.dolby.pl2z", + "vnd.dolby.pulse.1", + "vnd.dra", + "vnd.dts", + "vnd.dts.hd", + "vnd.dts.uhd", + "vnd.dvb.file", + "vnd.everad.plj", + "vnd.hns.audio", + "vnd.lucent.voice", + "vnd.ms-playready.media.pya", + "vnd.nokia.mobile-xmf", + "vnd.nortel.vbk", + "vnd.nuera.ecelp4800", + "vnd.nuera.ecelp7470", + "vnd.nuera.ecelp9600", + "vnd.octel.sbc", + "vnd.presonus.multitrack", + "vnd.qcelp", + "vnd.rhetorex.32kadpcm", + "vnd.rip", + "vnd.sealedmedia.softseal.mpeg", + "vnd.vmx.cvsd", + "vorbis", + "vorbis-config", + ) +} + diff --git a/crates/calcard/src/vcard/mod.rs b/crates/calcard/src/vcard/mod.rs new file mode 100644 index 00000000..6911cd09 --- /dev/null +++ b/crates/calcard/src/vcard/mod.rs @@ -0,0 +1,474 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[cfg(feature = "rkyv")] +use crate::common::archive::{InlineVec, Unboxed}; +use crate::{ + Entry, Parser, + common::{CalendarScale, Data, IanaParse, IanaString, IanaType, PartialDateTime}, +}; +use smallvec::SmallVec; + +pub mod builder; +pub(crate) mod media_type; +pub mod parser; +pub mod types; +pub mod utils; +pub mod writer; + +#[cfg(feature = "rkyv")] +pub mod rkyv_types; +#[cfg(feature = "rkyv")] +pub mod rkyv_utils; +#[cfg(feature = "rkyv")] +pub mod rkyv_writer; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct VCard { + pub entries: Vec, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum VCardVersion { + V2_0 = 20, + V2_1 = 21, + V3_0 = 30, + #[default] + V4_0 = 40, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct VCardEntry { + pub group: Option, + pub name: VCardProperty, + pub params: Vec, + #[cfg_attr(feature = "rkyv", rkyv(with = InlineVec))] + pub values: SmallVec<[VCardValue; 1]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardProperty { + Other(String), + Begin, + End, + Source, // [RFC6350, Section 6.1.3] + Kind, // [RFC6350, Section 6.1.4] + Xml, // [RFC6350, Section 6.1.5] + Fn, // [RFC6350, Section 6.2.1] + N, // [RFC6350, Section 6.2.2][RFC9554, Section 2.2] + Nickname, // [RFC6350, Section 6.2.3] + Photo, // [RFC6350, Section 6.2.4] + Bday, // [RFC6350, Section 6.2.5] + Anniversary, // [RFC6350, Section 6.2.6] + Gender, // [RFC6350, Section 6.2.7] + Adr, // [RFC6350, Section 6.3.1][RFC9554, Section 2.1] + Tel, // [RFC6350, Section 6.4.1] + Email, // [RFC6350, Section 6.4.2] + Impp, // [RFC6350, Section 6.4.3] + Lang, // [RFC6350, Section 6.4.4] + Tz, // [RFC6350, Section 6.5.1] + Geo, // [RFC6350, Section 6.5.2] + Title, // [RFC6350, Section 6.6.1] + Role, // [RFC6350, Section 6.6.2] + Logo, // [RFC6350, Section 6.6.3] + Org, // [RFC6350, Section 6.6.4] + Member, // [RFC6350, Section 6.6.5] + Related, // [RFC6350, Section 6.6.6] + Categories, // [RFC6350, Section 6.7.1] + Note, // [RFC6350, Section 6.7.2] + Prodid, // [RFC6350, Section 6.7.3] + Rev, // [RFC6350, Section 6.7.4] + Sound, // [RFC6350, Section 6.7.5] + Uid, // [RFC6350, Section 6.7.6] + Clientpidmap, // [RFC6350, Section 6.7.7] + Url, // [RFC6350, Section 6.7.8] + Version, // [RFC6350, Section 6.7.9] + Key, // [RFC6350, Section 6.8.1] + Fburl, // [RFC6350, Section 6.9.1] + Caladruri, // [RFC6350, Section 6.9.2] + Caluri, // [RFC6350, Section 6.9.3] + Birthplace, // [RFC6474, Section 2.1] + Deathplace, // [RFC6474, Section 2.2] + Deathdate, // [RFC6474, Section 2.3] + Expertise, // [RFC6715, Section 2.1] + Hobby, // [RFC6715, Section 2.2] + Interest, // [RFC6715, Section 2.3] + OrgDirectory, // [RFC6715, Section 2.4][RFC Errata 3341] + ContactUri, // [RFC8605, Section 2.1] + Created, // [RFC9554, Section 3.1] + Gramgender, // [RFC9554, Section 3.2] + Language, // [RFC9554, Section 3.3] + Pronouns, // [RFC9554, Section 3.4] + Socialprofile, // [RFC9554, Section 3.5] + Jsprop, // [RFC9555, Section 3.2.1] +} + +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub enum VCardValue { + Text(String), + Integer(i64), + Float(f64), + Boolean(bool), + PartialDateTime(PartialDateTime), + Binary(#[cfg_attr(feature = "rkyv", rkyv(with = Unboxed))] Box), + Sex(VCardSex), + GramGender(VCardGramGender), + Kind(VCardKind), + Component(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub struct VCardParameter { + pub name: VCardParameterName, + pub value: VCardParameterValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq)))] +pub enum VCardParameterValue { + Text(String), + Integer(u32), + Timestamp(i64), + Bool(bool), + ValueType(VCardValueType), + Type(VCardType), + Calscale(CalendarScale), + Level(VCardLevel), + Phonetic(VCardPhonetic), + Jscomps(Vec), + Null, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum Jscomp { + Entry { position: u32, value: u32 }, + Separator(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(PartialEq)))] +pub enum VCardParameterName { + Other(String), + Language, // [RFC6350, Section 5.1] + Value, // [RFC6350, Section 5.2] + Pref, // [RFC6350, Section 5.3] + Altid, // [RFC6350, Section 5.4] + Pid, // [RFC6350, Section 5.5] + Type, // [RFC6350, Section 5.6] + Mediatype, // [RFC6350, Section 5.7] + Calscale, // [RFC6350, Section 5.8] + SortAs, // [RFC6350, Section 5.9] + Geo, // [RFC6350, Section 5.10] + Tz, // [RFC6350, Section 5.11] + Index, // [RFC6715, Section 3.1] + Level, // [RFC6715, Section 3.2] + Group, // [RFC7095, Section 8.1] + Cc, // [RFC8605, Section 3.1] + Author, // [RFC9554, Section 4.1] + AuthorName, // [RFC9554, Section 4.2] + Created, // [RFC9554, Section 4.3] + Derived, // [RFC9554, Section 4.4] + Label, // [RFC6350, Section 6.3.1][RFC9554, Section 4.5] + Phonetic, // [RFC9554, Section 4.6] + PropId, // [RFC9554, Section 4.7] + Script, // [RFC9554, Section 4.8] + ServiceType, // [RFC9554, Section 4.9] + Username, // [RFC9554, Section 4.10] + Jsptr, // [RFC9555, Section 3.3.2] + Jscomps, // [RFC9555, Section 3.3.2] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardValueType { + Boolean, // [RFC6350, Section 4.4] + Date, // [RFC6350, Section 4.3.1] + DateAndOrTime, // [RFC6350, Section 4.3.4] + DateTime, // [RFC6350, Section 4.3.3] + Float, // [RFC6350, Section 4.6] + Integer, // [RFC6350, Section 4.5] + LanguageTag, // [RFC6350, Section 4.8] + Text, // [RFC6350, Section 4.1] + Time, // [RFC6350, Section 4.3.2] + Timestamp, // [RFC6350, Section 4.3.5] + Uri, // [RFC6350, Section 4.2] + UtcOffset, // [RFC6350, Section 4.7] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardLevel { + Beginner, // [RFC6715, Section 3.2] + Average, // [RFC6715, Section 3.2] + Expert, // [RFC6715, Section 3.2] + High, // [RFC6715, Section 3.2] + Medium, // [RFC6715, Section 3.2] + Low, // [RFC6715, Section 3.2] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardPhonetic { + Ipa, // [RFC9554, Section 4.6] + Jyut, // [RFC9554, Section 4.6] + Piny, // [RFC9554, Section 4.6] + Script, // [RFC9554, Section 4.6] +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + serde(tag = "type", content = "data") +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardType { + Work, // [RFC6350, Section 5.6] + Home, // [RFC6350, Section 5.6] + Billing, // [RFC9554, Section 5.1] + Delivery, // [RFC9554, Section 5.2] + Contact, // [RFC6350, Section 6.6.6] + Acquaintance, // [RFC6350, Section 6.6.6] + Friend, // [RFC6350, Section 6.6.6] + Met, // [RFC6350, Section 6.6.6] + CoWorker, // [RFC6350, Section 6.6.6] + Colleague, // [RFC6350, Section 6.6.6] + CoResident, // [RFC6350, Section 6.6.6] + Neighbor, // [RFC6350, Section 6.6.6] + Child, // [RFC6350, Section 6.6.6] + Parent, // [RFC6350, Section 6.6.6] + Sibling, // [RFC6350, Section 6.6.6] + Spouse, // [RFC6350, Section 6.6.6] + Kin, // [RFC6350, Section 6.6.6] + Muse, // [RFC6350, Section 6.6.6] + Crush, // [RFC6350, Section 6.6.6] + Date, // [RFC6350, Section 6.6.6] + Sweetheart, // [RFC6350, Section 6.6.6] + Me, // [RFC6350, Section 6.6.6] + Agent, // [RFC6350, Section 6.6.6] + Emergency, // [RFC6350, Section 6.6.6] + Text, // [RFC6350, Section 6.4.1] + Voice, // [RFC6350, Section 6.4.1] + Fax, // [RFC6350, Section 6.4.1] + Cell, // [RFC6350, Section 6.4.1] + Video, // [RFC6350, Section 6.4.1] + Pager, // [RFC6350, Section 6.4.1] + Textphone, // [RFC6350, Section 6.4.1] + MainNumber, // [RFC7852] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardGramGender { + Animate, // [RFC9554, Section 3.2] + Common, // [RFC9554, Section 3.2] + Feminine, // [RFC9554, Section 3.2] + Inanimate, // [RFC9554, Section 3.2] + Masculine, // [RFC9554, Section 3.2] + Neuter, // [RFC9554, Section 3.2] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardSex { + Male, + Female, + Other, + NoneOrNotApplicable, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + any(all(test, stalwart_tests), feature = "serde"), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr( + feature = "rkyv", + derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive) +)] +#[cfg_attr(feature = "rkyv", rkyv(compare(PartialEq), derive(Debug)))] +pub enum VCardKind { + Individual, // [RFC6350, Section 6.1.4] + Group, // [RFC6350, Section 6.1.4] + Org, // [RFC6350, Section 6.1.4] + Location, // [RFC6350, Section 6.1.4] + Application, // [RFC6473, Section 3] + Device, // [RFC6869, Section 3] +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ValueSeparator { + None, + Comma, + Semicolon, + SemicolonAndComma, + Skip, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ValueType { + Vcard(VCardValueType), + Kind, + Sex, + GramGender, +} + +impl ValueType { + pub fn unwrap_vcard(self) -> VCardValueType { + match self { + ValueType::Vcard(v) => v, + _ => VCardValueType::Text, + } + } +} + +impl VCard { + pub fn parse(value: impl AsRef) -> Result { + let mut parser = Parser::new(value.as_ref()); + match parser.entry() { + Entry::VCard(vcard) => Ok(vcard), + other => Err(other), + } + } +} diff --git a/crates/calcard/src/vcard/parser.rs b/crates/calcard/src/vcard/parser.rs new file mode 100644 index 00000000..1fd04930 --- /dev/null +++ b/crates/calcard/src/vcard/parser.rs @@ -0,0 +1,901 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + PartialDateTime, VCard, VCardEntry, VCardParameter, VCardParameterName, VCardType, VCardValue, + VCardValueType, ValueSeparator, ValueType, +}; +use crate::{ + Entry, Parser, Token, + common::{ + CalendarScale, Data, Encoding, IanaParse, IanaType, + decode::{Decoded, ValueDecoder}, + parser::{Boolean, Integer, Timestamp, parse_digits, parse_small_digits}, + tokenizer::{Mode, StopChar}, + }, + vcard::{ + Jscomp, VCardGramGender, VCardKind, VCardLevel, VCardParameterValue, VCardPhonetic, + VCardProperty, VCardSex, + }, +}; +use smallvec::{SmallVec, smallvec}; +use std::{iter::Peekable, mem::take, slice::Iter}; + +struct Params { + params: Vec, + stop_char: StopChar, + data_types: Vec>, + charset: Option, + encoding: Option, + group_name: Option, +} + +impl VCardProperty { + pub(crate) fn value_decoder<'x>( + &self, + encoding: Encoding, + charset: Option<&'x str>, + ) -> ValueDecoder<'x> { + let binary = matches!(encoding, Encoding::Base64) + && match self { + VCardProperty::Photo + | VCardProperty::Logo + | VCardProperty::Sound + | VCardProperty::Key => true, + VCardProperty::Other(_) => charset.is_none(), + _ => false, + }; + ValueDecoder { + encoding, + charset, + binary, + } + } +} + +struct ValueText<'x> { + token: Token<'x>, + decoded: Option, +} + +impl<'x> ValueText<'x> { + #[inline(always)] + fn bytes(&self) -> &[u8] { + match &self.decoded { + Some(decoded) => decoded.as_bytes(), + None => self.token.text.as_ref(), + } + } + + #[inline(always)] + fn into_string(self) -> String { + match self.decoded { + Some(decoded) => decoded, + None => self.token.into_string(), + } + } + + #[inline(always)] + fn into_token(self) -> Token<'x> { + let mut token = self.token; + if let Some(decoded) = self.decoded { + token.text = decoded.into(); + } + token + } +} + +const BYTES_PER_ENTRY: usize = 32; +const MIN_ENTRIES: usize = 4; +const MAX_ENTRIES: usize = 32; +const MULTI_VALUES: usize = 4; +const STRUCTURED_VALUES: usize = 8; + +impl Parser<'_> { + pub fn vcard(&mut self) -> Entry { + let mut vcard = VCard::default(); + let entries_hint = self.entries_hint(); + let mut is_v4 = true; + let mut is_valid = false; + + 'outer: loop { + self.expect_iana_token(); + self.mode.set(Mode::STOP_DOT, true); + let Some(mut token) = self.token() else { + break; + }; + self.mode.set(Mode::STOP_DOT, false); + + let mut params = Params { + params: Vec::new(), + stop_char: token.stop_char, + data_types: Vec::new(), + group_name: None, + encoding: None, + charset: None, + }; + + if matches!(token.stop_char, StopChar::Dot) { + params.group_name = token.into_string().into(); + token = match self.token() { + Some(token) => token, + None => break, + }; + params.stop_char = token.stop_char; + } + + match params.stop_char { + StopChar::Semicolon => { + self.vcard_parameters(&mut params); + } + StopChar::Lf => { + if token.text.is_empty() || !self.strict { + continue; + } else { + return Entry::InvalidLine(token.into_string()); + } + } + _ => {} + } + + if !matches!(params.stop_char, StopChar::Colon | StopChar::Lf) { + params.stop_char = self.seek_value_or_eol(); + } + + let name = match VCardProperty::parse(token.text.as_ref()) { + Some(name) => name, + None if !token.text.is_empty() => VCardProperty::Other(token.into_string()), + None => { + if params.stop_char != StopChar::Lf { + self.seek_lf(); + } + continue; + } + }; + let mut entry = VCardEntry { + group: params.group_name, + name, + params: params.params, + values: SmallVec::new(), + }; + + if params.stop_char != StopChar::Lf { + let (default_type, multi_value) = entry.name.default_types(); + + let (is_structured, value_capacity) = match multi_value { + ValueSeparator::None => { + self.expect_single_value(); + (false, 1) + } + ValueSeparator::Comma => { + self.expect_multi_value_comma(); + (false, MULTI_VALUES) + } + ValueSeparator::Semicolon => { + self.expect_multi_value_semicolon(); + (false, MULTI_VALUES) + } + ValueSeparator::SemicolonAndComma => { + self.expect_multi_value_semicolon_and_comma(); + (true, STRUCTURED_VALUES) + } + ValueSeparator::Skip => { + is_valid = entry.name == VCardProperty::End; + self.expect_single_value(); + self.token(); + break 'outer; + } + }; + entry.values.reserve_exact(value_capacity); + let charset = params.charset.as_deref(); + let decoder = match params.encoding { + Some(Encoding::Base64) => { + if multi_value != ValueSeparator::None { + self.expect_single_value(); + } + self.mode.set(Mode::UNFOLD_B64, true); + Some(entry.name.value_decoder(Encoding::Base64, charset)) + } + Some(Encoding::QuotedPrintable) => { + self.mode.set(Mode::UNFOLD_QP, true); + Some(entry.name.value_decoder(Encoding::QuotedPrintable, charset)) + } + None => None, + }; + + let mut data_types = params.data_types.iter(); + let mut token_idx = 0; + let mut last_is_comma = false; + + let mut payload = None; + while let Some(token) = self.value_token(params.encoding, &mut payload) { + let (is_eol, is_comma) = match token.stop_char { + StopChar::Lf => (true, false), + StopChar::Comma => (false, true), + _ => (false, false), + }; + + let mut text = ValueText { + token, + decoded: None, + }; + if let Some(decoder) = &decoder { + match decoder.decode( + text.token.text.as_ref(), + text.token.stop_char, + payload.take(), + ) { + Decoded::Binary(data) => { + entry.values.push(VCardValue::Binary(Box::new(Data { + data, + content_type: None, + }))); + if is_eol { + break; + } else { + continue; + } + } + Decoded::Text(decoded) => text.decoded = Some(decoded), + Decoded::Undecodable => {} + } + } + + let default_type = match &default_type { + ValueType::Vcard(default_type) => default_type, + ValueType::Kind if token_idx == 0 => { + if let Some(value) = VCardKind::parse(text.bytes()) { + entry.values.push(VCardValue::Kind(value)); + if is_eol { + break; + } else { + continue; + } + } + &VCardValueType::Text + } + ValueType::Sex if token_idx == 0 => { + if let Some(value) = VCardSex::parse(text.bytes()) { + entry.values.push(VCardValue::Sex(value)); + if is_eol { + break; + } else { + continue; + } + } + &VCardValueType::Text + } + ValueType::GramGender if token_idx == 0 => { + if let Some(value) = VCardGramGender::parse(text.bytes()) { + entry.values.push(VCardValue::GramGender(value)); + if is_eol { + break; + } else { + continue; + } + } + &VCardValueType::Text + } + _ => &VCardValueType::Text, + }; + + let value = match data_types.next().unwrap_or(&IanaType::Iana(*default_type)) { + IanaType::Iana(VCardValueType::Text) => { + if is_v4 + && entry.name == VCardProperty::Version + && matches!(text.bytes().first(), Some(b'1'..=b'3')) + { + is_v4 = false; + } + VCardValue::Text(text.into_string()) + } + IanaType::Iana(VCardValueType::LanguageTag) | IanaType::Other(_) => { + VCardValue::Text(text.into_string()) + } + IanaType::Iana(value_type) => { + text.into_token().into_vcard_value(*value_type, is_v4) + } + }; + + if is_structured { + match (last_is_comma, entry.values.last_mut(), value) { + (true, Some(last), VCardValue::Text(item)) => { + if let Some(item) = last.extend_component(item) { + entry.values.push(VCardValue::Text(item)); + } + } + (_, _, value) => { + entry.values.push(value); + } + } + + last_is_comma = is_comma; + } else { + entry.values.push(value); + } + + if is_eol { + break; + } + + token_idx += 1; + } + } else { + entry.values = smallvec![VCardValue::Text(String::new())]; + } + + if !params.data_types.is_empty() { + entry + .params + .extend(params.data_types.into_iter().map(|dt| VCardParameter { + name: VCardParameterName::Value, + value: match dt { + IanaType::Iana(v) => VCardParameterValue::ValueType(v), + IanaType::Other(v) => VCardParameterValue::Text(v), + }, + })); + } + + if !is_v4 { + entry.normalize_legacy_media_type(); + } + + if vcard.entries.capacity() == 0 { + self.presize(&mut vcard.entries, entries_hint); + } + vcard.entries.push(entry); + } + + if is_valid || !self.strict { + Entry::VCard(vcard) + } else { + Entry::UnterminatedComponent("BEGIN".into()) + } + } + + fn entries_hint(&self) -> usize { + (self.input.len().saturating_sub(self.pos) / BYTES_PER_ENTRY) + .clamp(MIN_ENTRIES, MAX_ENTRIES) + } + + fn vcard_parameters(&mut self, params: &mut Params) { + while params.stop_char == StopChar::Semicolon { + self.expect_iana_token(); + let Some(name_token) = self.token() else { + params.stop_char = StopChar::Lf; + break; + }; + + params.stop_char = name_token.stop_char; + if !matches!( + params.stop_char, + StopChar::Lf | StopChar::Colon | StopChar::Semicolon + ) { + if params.stop_char != StopChar::Equal { + params.stop_char = self.seek_param_value_or_eol(); + } + if params.stop_char == StopChar::Equal { + self.expect_param_value(); + while !matches!( + params.stop_char, + StopChar::Lf | StopChar::Colon | StopChar::Semicolon + ) { + match self.token() { + Some(token) => { + params.stop_char = token.stop_char; + self.token_buf.push(token); + } + None => { + params.stop_char = StopChar::Lf; + break; + } + } + } + } + } + + let param_name = name_token.text.as_ref(); + let param_values = &mut params.params; + if let Some(param_name) = VCardParameterName::try_parse(param_name) { + if self.token_buf.is_empty() { + param_values.push(VCardParameter::new(param_name, VCardParameterValue::Null)); + continue; + } + + match param_name { + VCardParameterName::Value => { + params + .data_types + .extend(self.token_buf.drain(..).map(Into::into)); + } + VCardParameterName::Pref | VCardParameterName::Index => { + if let Some(value) = self.buf_parse_one::>() { + param_values.push(VCardParameter { + name: param_name, + value: match value { + IanaType::Iana(value) => { + VCardParameterValue::Integer(value.0 as u32) + } + IanaType::Other(value) => VCardParameterValue::Text(value), + }, + }); + } + } + VCardParameterName::Calscale => { + if let Some(value) = self.buf_parse_one::>() + { + param_values.push(VCardParameter::new(param_name, value)); + } + } + VCardParameterName::Phonetic => { + if let Some(value) = self.buf_parse_one::>() + { + param_values.push(VCardParameter::new(param_name, value)); + } + } + VCardParameterName::Level => { + if let Some(value) = self.buf_parse_one::>() { + param_values.push(VCardParameter::new(param_name, value)); + } + } + VCardParameterName::Created => { + if let Some(value) = self.buf_parse_one::>() { + param_values.push(VCardParameter::new(param_name, value)); + } + } + VCardParameterName::Derived => { + if let Some(value) = self.buf_parse_one::>() { + param_values.push(VCardParameter::new(param_name, value)); + } + } + VCardParameterName::Type => { + for token in self.token_buf.drain(..) { + token.into_vcard_types(param_values); + } + } + VCardParameterName::Jscomps => { + if let Some(text) = self.raw_token() { + param_values.push(VCardParameter::jscomps( + VCardParameterValue::Jscomps(Jscomp::parse_list(&text)), + )); + } + self.token_buf.clear(); + } + _ => { + param_values.extend(self.token_buf.drain(..).map(|token| { + VCardParameter::new( + param_name.clone(), + VCardParameterValue::Text(token.into_string()), + ) + })); + } + } + } else if !param_name.is_empty() { + match VCardType::parse(param_name) { + Some(typ) if self.token_buf.is_empty() => { + param_values.push(VCardParameter::typ(VCardParameterValue::Type(typ))); + } + _ => match param_name.first() { + Some(b'c' | b'C') if param_name.eq_ignore_ascii_case(b"charset") => { + if let Some(token) = self.token_buf.drain(..).next_back() { + params.charset = token.into_string().into(); + } + } + Some(b'e' | b'E') if param_name.eq_ignore_ascii_case(b"encoding") => { + if let Some(token) = self.token_buf.drain(..).next_back() { + params.encoding = Encoding::parse(token.text.as_ref()); + } + } + _ => { + if params.encoding.is_none() + && param_name.eq_ignore_ascii_case(b"base64") + { + params.encoding = Some(Encoding::Base64); + } else { + let name = VCardParameterName::Other(name_token.into_string()); + let mut tokens = self.token_buf.drain(..); + match tokens.next_back() { + Some(last) => { + param_values.extend(tokens.map(|token| { + VCardParameter::new( + name.clone(), + VCardParameterValue::Text(token.into_string()), + ) + })); + param_values.push(VCardParameter::new( + name, + VCardParameterValue::Text(last.into_string()), + )); + } + None => param_values + .push(VCardParameter::new(name, VCardParameterValue::Null)), + } + } + } + }, + } + } + } + } +} + +impl VCardValue { + fn extend_component(&mut self, item: String) -> Option { + match self { + VCardValue::Component(items) => items.push(item), + VCardValue::Text(text) => *self = VCardValue::Component(vec![take(text), item]), + _ => return Some(item), + } + None + } +} + +impl VCardParameter { + fn type_list(text: &str) -> impl Iterator + '_ { + text.split(',') + .map(str::trim) + .filter(|text| !text.is_empty()) + .map(|text| { + VCardParameter::typ(match VCardType::parse(text.as_bytes()) { + Some(typ) => VCardParameterValue::Type(typ), + None => VCardParameterValue::Text(text.to_string()), + }) + }) + } +} + +impl Jscomp { + fn parse_list(text: &str) -> Vec { + let mut jscomps = Vec::with_capacity(4); + for (item_pos, item) in text.split(';').enumerate() { + if let Some(item) = item.strip_prefix("s,") { + let mut sep = String::with_capacity(item.len()); + let mut last_is_escape = false; + + for ch in item.chars() { + if ch == '\\' && !last_is_escape { + last_is_escape = true; + continue; + } + last_is_escape = false; + sep.push(ch); + } + + jscomps.push(Jscomp::Separator(sep)); + } else if item_pos != 0 { + let mut position = None; + let mut value = None; + + for (pos, item) in item.split(',').enumerate() { + if pos == 0 { + position = item.parse::().ok(); + } else if pos == 1 { + value = item.parse::().ok(); + } + } + + if let Some(position) = position { + jscomps.push(Jscomp::Entry { + position, + value: value.unwrap_or_default(), + }); + } + } else { + jscomps.push(Jscomp::Separator(item.to_string())); + } + } + jscomps + } +} + +impl Token<'_> { + fn into_vcard_types(self, param_values: &mut Vec) { + if let Some(typ) = VCardType::parse(self.text.as_ref()) { + param_values.push(VCardParameter::typ(VCardParameterValue::Type(typ))); + } else if !self.text.contains(&b',') { + param_values.push(VCardParameter::typ(VCardParameterValue::Text( + self.into_string(), + ))); + } else { + param_values.extend(VCardParameter::type_list(self.text.as_str())); + } + } + + fn into_vcard_value(self, value_type: VCardValueType, is_v4: bool) -> VCardValue { + match value_type { + VCardValueType::Date if is_v4 => self + .into_vcard_date() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::DateAndOrTime if is_v4 => self + .into_vcard_date_and_or_datetime() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::DateTime if is_v4 => self + .into_vcard_date_time() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Time if is_v4 => self + .into_vcard_time() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Timestamp if is_v4 => self + .into_timestamp(true) + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::UtcOffset if is_v4 => self + .into_offset() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Boolean => VCardValue::Boolean(self.into_boolean()), + VCardValueType::Float => self + .into_float() + .map(VCardValue::Float) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Integer => self + .into_integer() + .map(VCardValue::Integer) + .unwrap_or_else(VCardValue::Text), + VCardValueType::LanguageTag | VCardValueType::Text => { + VCardValue::Text(self.into_string()) + } + VCardValueType::Uri => self + .into_uri_bytes() + .map(|data| VCardValue::Binary(Box::new(data))) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Date + | VCardValueType::DateAndOrTime + | VCardValueType::DateTime + | VCardValueType::Time => self + .into_vcard_datetime_or_legacy() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::Timestamp => self + .into_vcard_timestamp_or_legacy() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + VCardValueType::UtcOffset => self + .into_vcard_offset_or_legacy() + .map(VCardValue::PartialDateTime) + .unwrap_or_else(VCardValue::Text), + } + } + + pub(crate) fn into_vcard_date(self) -> std::result::Result { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date(&mut self.text.iter().peekable()); + if !dt.is_null() { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_vcard_date_and_or_datetime( + self, + ) -> std::result::Result { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date_and_or_time(&mut self.text.iter().peekable()); + if !dt.is_null() { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_vcard_date_time(self) -> std::result::Result { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_date_time(&mut self.text.iter().peekable()); + if !dt.is_null() { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_vcard_time(self) -> std::result::Result { + let mut dt = PartialDateTime::default(); + dt.parse_vcard_time(&mut self.text.iter().peekable(), false); + if !dt.is_null() { + Ok(dt) + } else { + Err(self.into_string()) + } + } + + pub(crate) fn into_vcard_timestamp_or_legacy( + self, + ) -> std::result::Result { + let mut dt = PartialDateTime::default(); + if dt.parse_timestamp(&mut self.text.iter().peekable(), false) { + Ok(dt) + } else { + let mut dt = PartialDateTime::default(); + if dt.parse_vcard_date_legacy(&mut self.text.iter().peekable()) { + Ok(dt) + } else { + Err(self.into_string()) + } + } + } + + pub(crate) fn into_vcard_datetime_or_legacy( + self, + ) -> std::result::Result { + let mut dt = PartialDateTime::default(); + if dt.parse_vcard_date_legacy(&mut self.text.iter().peekable()) { + Ok(dt) + } else { + self.into_vcard_date_and_or_datetime() + } + } + + pub(crate) fn into_vcard_offset_or_legacy( + self, + ) -> std::result::Result { + let mut dt = PartialDateTime::default(); + if dt.parse_vcard_zone_legacy(&mut self.text.iter().peekable()) { + Ok(dt) + } else { + self.into_offset() + } + } +} + +impl PartialDateTime { + pub fn parse_vcard_date_legacy(&mut self, iter: &mut Peekable>) -> bool { + let mut idx = 0; + + for ch in iter { + match ch { + b'0'..=b'9' => { + let value = match idx { + 0 => { + if let Some(value) = &mut self.year { + *value = + value.saturating_mul(10).saturating_add((ch - b'0') as u16); + } else { + self.year = Some((ch - b'0') as u16); + } + continue; + } + 1 => &mut self.month, + 2 => &mut self.day, + 3 => &mut self.hour, + 4 => &mut self.minute, + 5 => &mut self.second, + 6 => &mut self.tz_hour, + 7 => &mut self.tz_minute, + _ => return false, + }; + + if let Some(value) = value { + *value = value.saturating_mul(10).saturating_add(ch - b'0'); + } else { + *value = Some(ch - b'0'); + } + } + b'T' | b't' if idx < 3 => { + idx = 3; + } + b'+' if idx <= 5 => { + idx = 6; + } + b'Z' | b'z' if idx == 5 => { + self.tz_hour = Some(0); + self.tz_minute = Some(0); + break; + } + b'-' if idx <= 2 => { + idx += 1; + } + b'-' if idx <= 5 => { + self.tz_minus = true; + idx = 6; + } + b':' if (3..=6).contains(&idx) => { + idx += 1; + } + b' ' | b'\t' | b'\r' | b'\n' => { + continue; + } + _ => return false, + } + } + + self.has_date() || self.has_zone() + } + + pub fn parse_vcard_zone_legacy(&mut self, iter: &mut Peekable>) -> bool { + let mut idx = 0; + + for ch in iter { + match ch { + b'0'..=b'9' => { + let value = match idx { + 0 => &mut self.tz_hour, + 1 => &mut self.tz_minute, + _ => return false, + }; + + if let Some(value) = value { + *value = value.saturating_mul(10).saturating_add(ch - b'0'); + } else { + *value = Some(ch - b'0'); + } + } + b'+' if self.tz_hour.is_none() => {} + b'-' if self.tz_hour.is_none() => { + self.tz_minus = true; + } + b'Z' | b'z' if self.tz_hour.is_none() => { + self.tz_hour = Some(0); + self.tz_minute = Some(0); + break; + } + b':' => { + idx += 1; + } + b' ' | b'\t' | b'\r' | b'\n' => { + continue; + } + _ => return false, + } + } + + self.tz_hour.is_some() && self.tz_minute.is_some() + } + + pub fn parse_vcard_date_time(&mut self, iter: &mut Peekable>) { + self.parse_vcard_date_noreduc(iter); + if matches!(iter.peek(), Some(&&b'T' | &&b't')) { + iter.next(); + self.parse_vcard_time(iter, true); + } + } + + pub fn parse_vcard_date_and_or_time(&mut self, iter: &mut Peekable>) { + self.parse_vcard_date(iter); + if matches!(iter.peek(), Some(&&b'T' | &&b't')) { + iter.next(); + self.parse_vcard_time(iter, false); + } + } + + pub fn parse_vcard_date(&mut self, iter: &mut Peekable>) { + parse_digits(iter, &mut self.year, 4, true); + if self.year.is_some() && iter.peek() == Some(&&b'-') { + iter.next(); + parse_small_digits(iter, &mut self.month, 2, true); + } else { + parse_small_digits(iter, &mut self.month, 2, true); + parse_small_digits(iter, &mut self.day, 2, false); + } + } + + pub fn parse_vcard_date_noreduc(&mut self, iter: &mut Peekable>) { + parse_digits(iter, &mut self.year, 4, true); + parse_small_digits(iter, &mut self.month, 2, true); + parse_small_digits(iter, &mut self.day, 2, false); + } + + pub fn parse_vcard_time(&mut self, iter: &mut Peekable>, mut notrunc: bool) { + for part in [&mut self.hour, &mut self.minute, &mut self.second] { + match iter.peek() { + Some(b'0'..=b'9') => { + notrunc = true; + parse_small_digits(iter, part, 2, false); + } + Some(b'-') if !notrunc => { + iter.next(); + } + _ => break, + } + } + self.parse_zone(iter); + } +} + diff --git a/crates/calcard/src/vcard/rkyv_types.rs b/crates/calcard/src/vcard/rkyv_types.rs new file mode 100644 index 00000000..8ee7cd2e --- /dev/null +++ b/crates/calcard/src/vcard/rkyv_types.rs @@ -0,0 +1,425 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + types::{LEGACY_ADR_COMPONENTS, LEGACY_N_COMPONENTS}, + *, +}; + +impl ArchivedVCardProperty { + pub fn as_str(&self) -> &str { + match self { + ArchivedVCardProperty::Source => "SOURCE", + ArchivedVCardProperty::Kind => "KIND", + ArchivedVCardProperty::Xml => "XML", + ArchivedVCardProperty::Fn => "FN", + ArchivedVCardProperty::N => "N", + ArchivedVCardProperty::Nickname => "NICKNAME", + ArchivedVCardProperty::Photo => "PHOTO", + ArchivedVCardProperty::Bday => "BDAY", + ArchivedVCardProperty::Anniversary => "ANNIVERSARY", + ArchivedVCardProperty::Gender => "GENDER", + ArchivedVCardProperty::Adr => "ADR", + ArchivedVCardProperty::Tel => "TEL", + ArchivedVCardProperty::Email => "EMAIL", + ArchivedVCardProperty::Impp => "IMPP", + ArchivedVCardProperty::Lang => "LANG", + ArchivedVCardProperty::Tz => "TZ", + ArchivedVCardProperty::Geo => "GEO", + ArchivedVCardProperty::Title => "TITLE", + ArchivedVCardProperty::Role => "ROLE", + ArchivedVCardProperty::Logo => "LOGO", + ArchivedVCardProperty::Org => "ORG", + ArchivedVCardProperty::Member => "MEMBER", + ArchivedVCardProperty::Related => "RELATED", + ArchivedVCardProperty::Categories => "CATEGORIES", + ArchivedVCardProperty::Note => "NOTE", + ArchivedVCardProperty::Prodid => "PRODID", + ArchivedVCardProperty::Rev => "REV", + ArchivedVCardProperty::Sound => "SOUND", + ArchivedVCardProperty::Uid => "UID", + ArchivedVCardProperty::Clientpidmap => "CLIENTPIDMAP", + ArchivedVCardProperty::Url => "URL", + ArchivedVCardProperty::Version => "VERSION", + ArchivedVCardProperty::Key => "KEY", + ArchivedVCardProperty::Fburl => "FBURL", + ArchivedVCardProperty::Caladruri => "CALADRURI", + ArchivedVCardProperty::Caluri => "CALURI", + ArchivedVCardProperty::Birthplace => "BIRTHPLACE", + ArchivedVCardProperty::Deathplace => "DEATHPLACE", + ArchivedVCardProperty::Deathdate => "DEATHDATE", + ArchivedVCardProperty::Expertise => "EXPERTISE", + ArchivedVCardProperty::Hobby => "HOBBY", + ArchivedVCardProperty::Interest => "INTEREST", + ArchivedVCardProperty::OrgDirectory => "ORG-DIRECTORY", + ArchivedVCardProperty::ContactUri => "CONTACT-URI", + ArchivedVCardProperty::Created => "CREATED", + ArchivedVCardProperty::Gramgender => "GRAMGENDER", + ArchivedVCardProperty::Language => "LANGUAGE", + ArchivedVCardProperty::Pronouns => "PRONOUNS", + ArchivedVCardProperty::Socialprofile => "SOCIALPROFILE", + ArchivedVCardProperty::Jsprop => "JSPROP", + ArchivedVCardProperty::Begin => "BEGIN", + ArchivedVCardProperty::End => "END", + ArchivedVCardProperty::Other(s) => s.as_str(), + } + } + + pub(crate) fn legacy_max_values(&self) -> usize { + match self { + ArchivedVCardProperty::N => LEGACY_N_COMPONENTS, + ArchivedVCardProperty::Adr => LEGACY_ADR_COMPONENTS, + _ => usize::MAX, + } + } +} + +impl ArchivedVCardProperty { + // Returns the default value type and whether the property is multi-valued. + pub(crate) fn default_types(&self) -> (ArchivedValueType, ValueSeparator) { + match self { + ArchivedVCardProperty::Source => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Kind => (ArchivedValueType::Kind, ValueSeparator::None), + ArchivedVCardProperty::Xml => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Fn => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::N => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::SemicolonAndComma, + ), + ArchivedVCardProperty::Nickname => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::Comma, + ), + ArchivedVCardProperty::Photo => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Bday => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + ArchivedVCardProperty::Anniversary => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + ArchivedVCardProperty::Gender => (ArchivedValueType::Sex, ValueSeparator::Semicolon), + ArchivedVCardProperty::Adr => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::SemicolonAndComma, + ), + ArchivedVCardProperty::Tel => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Email => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Impp => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Lang => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Tz => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::UtcOffset), + ValueSeparator::None, + ), + ArchivedVCardProperty::Geo => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Title => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Role => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Logo => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Org => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::Semicolon, + ), + ArchivedVCardProperty::Member => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Related => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Categories => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::Comma, + ), + ArchivedVCardProperty::Note => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Prodid => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Rev => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Timestamp), + ValueSeparator::None, + ), + ArchivedVCardProperty::Sound => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Uid => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Clientpidmap => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::Semicolon, + ), + ArchivedVCardProperty::Url => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Version => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Key => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Fburl => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Caladruri => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Caluri => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Birthplace => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Deathplace => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Deathdate => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + ArchivedVCardProperty::Expertise => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Hobby => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Interest => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::OrgDirectory => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::ContactUri => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Created => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Timestamp), + ValueSeparator::None, + ), + ArchivedVCardProperty::Gramgender => { + (ArchivedValueType::GramGender, ValueSeparator::None) + } + ArchivedVCardProperty::Language => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::LanguageTag), + ValueSeparator::None, + ), + ArchivedVCardProperty::Pronouns => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Socialprofile => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Uri), + ValueSeparator::None, + ), + ArchivedVCardProperty::Jsprop => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + ArchivedVCardProperty::Begin | ArchivedVCardProperty::End => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::Skip, + ), + ArchivedVCardProperty::Other(_) => ( + ArchivedValueType::Vcard(ArchivedVCardValueType::Text), + ValueSeparator::None, + ), + } + } +} + +impl IanaString for ArchivedVCardValueType { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardValueType::Boolean => "BOOLEAN", + ArchivedVCardValueType::Date => "DATE", + ArchivedVCardValueType::DateAndOrTime => "DATE-AND-OR-TIME", + ArchivedVCardValueType::DateTime => "DATE-TIME", + ArchivedVCardValueType::Float => "FLOAT", + ArchivedVCardValueType::Integer => "INTEGER", + ArchivedVCardValueType::LanguageTag => "LANGUAGE-TAG", + ArchivedVCardValueType::Text => "TEXT", + ArchivedVCardValueType::Time => "TIME", + ArchivedVCardValueType::Timestamp => "TIMESTAMP", + ArchivedVCardValueType::Uri => "URI", + ArchivedVCardValueType::UtcOffset => "UTC-OFFSET", + } + } +} + +impl IanaString for ArchivedVCardLevel { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardLevel::Beginner => "BEGINNER", + ArchivedVCardLevel::Average => "AVERAGE", + ArchivedVCardLevel::Expert => "EXPERT", + ArchivedVCardLevel::High => "HIGH", + ArchivedVCardLevel::Medium => "MEDIUM", + ArchivedVCardLevel::Low => "LOW", + } + } +} + +impl IanaString for ArchivedVCardPhonetic { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardPhonetic::Ipa => "IPA", + ArchivedVCardPhonetic::Jyut => "JYUT", + ArchivedVCardPhonetic::Piny => "PINY", + ArchivedVCardPhonetic::Script => "SCRIPT", + } + } +} + +impl IanaString for ArchivedVCardType { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardType::Work => "WORK", + ArchivedVCardType::Home => "HOME", + ArchivedVCardType::Billing => "BILLING", + ArchivedVCardType::Delivery => "DELIVERY", + ArchivedVCardType::Contact => "CONTACT", + ArchivedVCardType::Acquaintance => "ACQUAINTANCE", + ArchivedVCardType::Friend => "FRIEND", + ArchivedVCardType::Met => "MET", + ArchivedVCardType::CoWorker => "CO-WORKER", + ArchivedVCardType::Colleague => "COLLEAGUE", + ArchivedVCardType::CoResident => "CO-RESIDENT", + ArchivedVCardType::Neighbor => "NEIGHBOR", + ArchivedVCardType::Child => "CHILD", + ArchivedVCardType::Parent => "PARENT", + ArchivedVCardType::Sibling => "SIBLING", + ArchivedVCardType::Spouse => "SPOUSE", + ArchivedVCardType::Kin => "KIN", + ArchivedVCardType::Muse => "MUSE", + ArchivedVCardType::Crush => "CRUSH", + ArchivedVCardType::Date => "DATE", + ArchivedVCardType::Sweetheart => "SWEETHEART", + ArchivedVCardType::Me => "ME", + ArchivedVCardType::Agent => "AGENT", + ArchivedVCardType::Emergency => "EMERGENCY", + ArchivedVCardType::Text => "TEXT", + ArchivedVCardType::Voice => "VOICE", + ArchivedVCardType::Fax => "FAX", + ArchivedVCardType::Cell => "CELL", + ArchivedVCardType::Video => "VIDEO", + ArchivedVCardType::Pager => "PAGER", + ArchivedVCardType::Textphone => "TEXTPHONE", + ArchivedVCardType::MainNumber => "MAIN-NUMBER", + } + } +} + +impl IanaString for ArchivedVCardGramGender { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardGramGender::Animate => "ANIMATE", + ArchivedVCardGramGender::Common => "COMMON", + ArchivedVCardGramGender::Feminine => "FEMININE", + ArchivedVCardGramGender::Inanimate => "INANIMATE", + ArchivedVCardGramGender::Masculine => "MASCULINE", + ArchivedVCardGramGender::Neuter => "NEUTER", + } + } +} + +impl IanaString for ArchivedVCardSex { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardSex::Male => "M", + ArchivedVCardSex::Female => "F", + ArchivedVCardSex::Other => "O", + ArchivedVCardSex::NoneOrNotApplicable => "N", + ArchivedVCardSex::Unknown => "U", + } + } +} + +impl IanaString for ArchivedVCardKind { + fn as_str(&self) -> &'static str { + match self { + ArchivedVCardKind::Individual => "INDIVIDUAL", + ArchivedVCardKind::Group => "GROUP", + ArchivedVCardKind::Org => "ORG", + ArchivedVCardKind::Location => "LOCATION", + ArchivedVCardKind::Application => "APPLICATION", + ArchivedVCardKind::Device => "DEVICE", + } + } +} + +#[derive(Debug)] + +pub(crate) enum ArchivedValueType { + Vcard(ArchivedVCardValueType), + Kind, + Sex, + GramGender, +} + +impl ArchivedValueType { + pub fn unwrap_vcard(self) -> ArchivedVCardValueType { + match self { + ArchivedValueType::Vcard(v) => v, + _ => ArchivedVCardValueType::Text, + } + } +} diff --git a/crates/calcard/src/vcard/rkyv_utils.rs b/crates/calcard/src/vcard/rkyv_utils.rs new file mode 100644 index 00000000..7c8a4654 --- /dev/null +++ b/crates/calcard/src/vcard/rkyv_utils.rs @@ -0,0 +1,225 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::*; +use crate::common::{ArchivedCalendarScale, ArchivedData, ArchivedPartialDateTime}; + +impl ArchivedVCard { + pub fn uid(&self) -> Option<&str> { + self.property(&VCardProperty::Uid) + .and_then(|e| e.values.first()) + .and_then(|v| v.as_text()) + } + + pub fn property(&self, prop: &VCardProperty) -> Option<&ArchivedVCardEntry> { + self.entries.iter().find(|entry| &entry.name == prop) + } + + pub fn properties<'x, 'y: 'x>( + &'x self, + prop: &'y VCardProperty, + ) -> impl Iterator + 'x { + self.entries.iter().filter(move |entry| &entry.name == prop) + } + + pub fn version(&self) -> Option { + self.entries + .iter() + .find(|e| e.name == VCardProperty::Version) + .and_then(|e| { + e.values + .first() + .and_then(|v| v.as_text()) + .and_then(VCardVersion::try_parse) + }) + } +} + +impl ArchivedVCardEntry { + #[inline] + pub fn parameters( + &self, + prop: &VCardParameterName, + ) -> impl Iterator { + self.params.iter().filter_map(move |param| { + if ¶m.name == prop { + Some(¶m.value) + } else { + None + } + }) + } +} + +impl ArchivedVCardValue { + pub fn as_text(&self) -> Option<&str> { + match self { + ArchivedVCardValue::Text(v) => Some(v.as_str()), + ArchivedVCardValue::Sex(v) => Some(v.as_str()), + ArchivedVCardValue::GramGender(v) => Some(v.as_str()), + ArchivedVCardValue::Kind(v) => Some(v.as_str()), + ArchivedVCardValue::Component(v) => v.first().map(|s| s.as_str()), + ArchivedVCardValue::Integer(_) + | ArchivedVCardValue::Float(_) + | ArchivedVCardValue::Boolean(_) + | ArchivedVCardValue::PartialDateTime(_) + | ArchivedVCardValue::Binary(_) => None, + } + } + + pub fn as_integer(&self) -> Option { + match self { + ArchivedVCardValue::Integer(i) => Some(i.to_native()), + _ => None, + } + } + + pub fn as_float(&self) -> Option { + match self { + ArchivedVCardValue::Float(f) => Some(f.to_native()), + _ => None, + } + } + + pub fn as_boolean(&self) -> Option { + match self { + ArchivedVCardValue::Boolean(b) => Some(*b), + _ => None, + } + } + + pub fn as_partial_date_time(&self) -> Option<&ArchivedPartialDateTime> { + match self { + ArchivedVCardValue::PartialDateTime(dt) => Some(dt), + _ => None, + } + } + + pub fn as_binary(&self) -> Option<&ArchivedData> { + match self { + ArchivedVCardValue::Binary(d) => Some(d), + _ => None, + } + } +} + +impl ArchivedVCardParameterName { + pub fn as_str(&self) -> &str { + match self { + ArchivedVCardParameterName::Language => "LANGUAGE", + ArchivedVCardParameterName::Value => "VALUE", + ArchivedVCardParameterName::Pref => "PREF", + ArchivedVCardParameterName::Altid => "ALTID", + ArchivedVCardParameterName::Pid => "PID", + ArchivedVCardParameterName::Type => "TYPE", + ArchivedVCardParameterName::Mediatype => "MEDIATYPE", + ArchivedVCardParameterName::Calscale => "CALSCALE", + ArchivedVCardParameterName::SortAs => "SORT-AS", + ArchivedVCardParameterName::Geo => "GEO", + ArchivedVCardParameterName::Tz => "TZ", + ArchivedVCardParameterName::Index => "INDEX", + ArchivedVCardParameterName::Level => "LEVEL", + ArchivedVCardParameterName::Group => "GROUP", + ArchivedVCardParameterName::Cc => "CC", + ArchivedVCardParameterName::Author => "AUTHOR", + ArchivedVCardParameterName::AuthorName => "AUTHOR-NAME", + ArchivedVCardParameterName::Created => "CREATED", + ArchivedVCardParameterName::Derived => "DERIVED", + ArchivedVCardParameterName::Label => "LABEL", + ArchivedVCardParameterName::Phonetic => "PHONETIC", + ArchivedVCardParameterName::PropId => "PROP-ID", + ArchivedVCardParameterName::Script => "SCRIPT", + ArchivedVCardParameterName::ServiceType => "SERVICE-TYPE", + ArchivedVCardParameterName::Username => "USERNAME", + ArchivedVCardParameterName::Jsptr => "JSPTR", + ArchivedVCardParameterName::Jscomps => "JSCOMPS", + ArchivedVCardParameterName::Other(name) => name.as_str(), + } + } +} + +impl ArchivedVCardParameterValue { + pub fn as_text(&self) -> Option<&str> { + match self { + ArchivedVCardParameterValue::Text(v) => Some(v.as_str()), + ArchivedVCardParameterValue::ValueType(v) => Some(v.as_str()), + ArchivedVCardParameterValue::Type(v) => Some(v.as_str()), + ArchivedVCardParameterValue::Calscale(v) => Some(v.as_str()), + ArchivedVCardParameterValue::Level(v) => Some(v.as_str()), + ArchivedVCardParameterValue::Phonetic(v) => Some(v.as_str()), + ArchivedVCardParameterValue::Jscomps(_) + | ArchivedVCardParameterValue::Integer(_) + | ArchivedVCardParameterValue::Timestamp(_) + | ArchivedVCardParameterValue::Bool(_) + | ArchivedVCardParameterValue::Null => None, + } + } + + pub fn as_phonetic(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Phonetic(v) => Some(IanaType::Iana(v)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_calscale(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Calscale(v) => Some(IanaType::Iana(v)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_level(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Level(v) => Some(IanaType::Iana(v)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_type(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Type(v) => Some(IanaType::Iana(v)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_value_type(&self) -> Option> { + match self { + ArchivedVCardParameterValue::ValueType(v) => Some(IanaType::Iana(v)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_bool(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Bool(b) => Some(IanaType::Iana(*b)), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_integer(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Integer(i) => Some(IanaType::Iana(i.to_native())), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_timestamp(&self) -> Option> { + match self { + ArchivedVCardParameterValue::Timestamp(t) => Some(IanaType::Iana(t.to_native())), + ArchivedVCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } +} + diff --git a/crates/calcard/src/vcard/rkyv_writer.rs b/crates/calcard/src/vcard/rkyv_writer.rs new file mode 100644 index 00000000..c525d2ae --- /dev/null +++ b/crates/calcard/src/vcard/rkyv_writer.rs @@ -0,0 +1,365 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + common::{ + ArchivedPartialDateTime, PartialDateTime, + format::{AsciiPush, BufferedWrite}, + parser::Timestamp, + writer::{ + DOCUMENT_BUFFER, ENTRY_BUFFER, FoldingWriter, JscompRef, LineWriter, NeedsQuotes, + write_jscomps, write_param_value, write_text, + }, + }, + vcard::{media_type::legacy_media_type, *}, +}; +use std::fmt::{self, Display, Write}; + +impl ArchivedVCard { + pub fn write_to(&self, out: &mut impl Write, version: VCardVersion) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_entries(&mut FoldingWriter::new(buf), version) + }) + } + + fn write_entries( + &self, + out: &mut FoldingWriter<'_, W>, + version: VCardVersion, + ) -> fmt::Result { + out.write_raw(version.begin_and_version())?; + for entry in self.entries.iter() { + if !matches!( + entry.name, + ArchivedVCardProperty::Version + | ArchivedVCardProperty::Begin + | ArchivedVCardProperty::End + ) { + entry.write_line(out, true, version)?; + } + } + + out.write_raw("END:VCARD\r\n") + } +} + +impl ArchivedVCardEntry { + #[deprecated(since = "0.4.0", note = "use write_with_version")] + pub fn write_to(&self, out: &mut impl Write, with_value: bool, is_v4: bool) -> fmt::Result { + self.write_with_version( + out, + with_value, + if is_v4 { + VCardVersion::V4_0 + } else { + VCardVersion::V3_0 + }, + ) + } + + pub fn write_with_version( + &self, + out: &mut impl Write, + with_value: bool, + version: VCardVersion, + ) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_line(&mut FoldingWriter::new(buf), with_value, version) + }) + } + + pub(crate) fn write_line( + &self, + out: &mut FoldingWriter<'_, W>, + with_value: bool, + version: VCardVersion, + ) -> fmt::Result { + let is_v4 = matches!(version, VCardVersion::V4_0); + + if let Some(group_name) = self.group.as_ref() { + out.write_atomic(group_name)?; + out.write_atomic(".")?; + } + + out.write_atomic(self.name.as_str())?; + let mut types = None; + let mut last_param: Option<&ArchivedVCardParameterName> = None; + let legacy_binary = if is_v4 { + None + } else { + self.values.iter().find_map(|v| match v { + ArchivedVCardValue::Binary(data) => Some(data), + _ => None, + }) + }; + + for param in self.params.iter() { + if !is_v4 + && match param.name { + ArchivedVCardParameterName::PropId | ArchivedVCardParameterName::Jscomps => { + true + } + ArchivedVCardParameterName::Mediatype => legacy_binary.is_some(), + _ => false, + } + { + continue; + } + + if last_param.is_some_and(|last_param| last_param == ¶m.name) { + out.write_atomic(",")?; + } else { + out.write_atomic(";")?; + out.write_atomic(param.name.as_str())?; + if !matches!(param.value, ArchivedVCardParameterValue::Null) { + out.write_atomic("=")?; + } + last_param = Some(¶m.name); + } + + match ¶m.value { + ArchivedVCardParameterValue::Text(v) + if param.name == VCardParameterName::Jsptr && !v.as_str().needs_quotes() => + { + out.write_atomic("\"")?; + write_param_value(out, v, is_v4)?; + out.write_atomic("\"")?; + } + ArchivedVCardParameterValue::Text(v) => { + write_param_value(out, v, is_v4)?; + } + ArchivedVCardParameterValue::Integer(i) => { + out.push_u64(i.to_native().into())?; + } + ArchivedVCardParameterValue::Timestamp(v) => { + Timestamp(v.to_native()).push_to(out)?; + } + ArchivedVCardParameterValue::Bool(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + } + ArchivedVCardParameterValue::ValueType(v) => { + if types.is_none() { + types = Some(v); + } + write_param_value(out, v.as_str(), is_v4)?; + } + ArchivedVCardParameterValue::Type(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + ArchivedVCardParameterValue::Calscale(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + ArchivedVCardParameterValue::Level(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + ArchivedVCardParameterValue::Phonetic(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + ArchivedVCardParameterValue::Jscomps(v) => { + out.write_atomic("\"")?; + write_jscomps(out, v)?; + out.write_atomic("\"")?; + last_param = None; + } + ArchivedVCardParameterValue::Null => { + last_param = None; + } + } + } + + let is_v21_base64 = matches!(version, VCardVersion::V2_1) && legacy_binary.is_some(); + let max_values = if is_v4 { + usize::MAX + } else { + self.name.legacy_max_values() + }; + + if !is_v4 { + if let Some(data) = legacy_binary { + out.write_atomic(if is_v21_base64 { + ";ENCODING=BASE64" + } else { + ";ENCODING=b" + })?; + + if let Some(media_type) = data.content_type.as_deref() + && !self + .params + .iter() + .any(|param| param.name == VCardParameterName::Type) + && let Some(token) = legacy_media_type(self.name.as_str(), media_type) + { + out.write_atomic(";TYPE=")?; + out.write_atomic(&token)?; + } + } + + if self.values.iter().take(max_values).any(|v| match v { + ArchivedVCardValue::Text(s) => !s.is_ascii(), + ArchivedVCardValue::Component(items) => items.iter().any(|s| !s.is_ascii()), + _ => false, + }) { + out.write_atomic(";CHARSET=UTF-8")?; + } + } + + out.write_atomic(":")?; + + if with_value { + let (default_type, value_separator) = self.name.default_types(); + let default_type = default_type.unwrap_vcard(); + + let mut separator = ";"; + let mut escape_semicolon = + matches!(types.unwrap_or(&default_type), ArchivedVCardValueType::Text); + let mut escape_comma = escape_semicolon; + + match value_separator { + ValueSeparator::Comma => { + escape_comma = true; + separator = ","; + } + ValueSeparator::Semicolon => escape_semicolon = true, + ValueSeparator::SemicolonAndComma => { + escape_semicolon = true; + escape_comma = true; + } + _ => {} + } + + for (pos, value) in self.values.iter().take(max_values).enumerate() { + if pos > 0 { + out.write_atomic(separator)?; + } + + match value { + ArchivedVCardValue::Text(v) => { + write_text(out, v, escape_semicolon, escape_comma)?; + } + ArchivedVCardValue::Component(v) => { + for (pos, item) in v.iter().enumerate() { + if pos > 0 { + out.write_atomic(",")?; + } + write_text(out, item, true, true)?; + } + } + ArchivedVCardValue::Integer(v) => { + out.push_i64(v.to_native())?; + } + ArchivedVCardValue::Float(v) => { + write!(out, "{v}")?; + } + ArchivedVCardValue::Boolean(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + } + ArchivedVCardValue::PartialDateTime(v) => { + let typ = if pos == 0 { + types + } else { + self.parameters(&VCardParameterName::Value) + .nth(pos) + .and_then(|v| v.as_value_type()) + .and_then(|v| v.iana().copied()) + } + .unwrap_or(&default_type); + let date = PartialDateTime::from(v); + let typ = VCardValueType::from(typ); + if is_v4 { + date.push_vcard(out, &typ)?; + } else { + date.push_legacy_vcard(out, &typ)?; + } + } + ArchivedVCardValue::Binary(v) => { + if is_v4 { + let media_type = v.content_type.as_deref().unwrap_or_default(); + out.write_str("data:")?; + out.write_str(media_type)?; + if escape_semicolon { + out.write_atomic("\\;")?; + } else { + out.write_str(";")?; + } + out.write_atomic("base64\\,")?; + } + out.write_base64(&v.data)?; + } + ArchivedVCardValue::Sex(v) => { + out.write_atomic(v.as_str())?; + } + ArchivedVCardValue::GramGender(v) => { + out.write_atomic(v.as_str())?; + } + ArchivedVCardValue::Kind(v) => { + out.write_atomic(v.as_str())?; + } + } + } + } + out.end_line()?; + if is_v21_base64 { + out.end_line()?; + } + Ok(()) + } +} + +impl ArchivedPartialDateTime { + pub fn format_as_vcard( + &self, + out: &mut impl Write, + fmt: &ArchivedVCardValueType, + ) -> fmt::Result { + PartialDateTime::from(self).format_as_vcard(out, &VCardValueType::from(fmt)) + } + + pub fn format_as_legacy_vcard( + &self, + out: &mut impl Write, + fmt: &ArchivedVCardValueType, + ) -> fmt::Result { + PartialDateTime::from(self).format_as_legacy_vcard(out, &VCardValueType::from(fmt)) + } +} + +impl From<&ArchivedVCardValueType> for VCardValueType { + fn from(value_type: &ArchivedVCardValueType) -> Self { + match value_type { + ArchivedVCardValueType::Boolean => VCardValueType::Boolean, + ArchivedVCardValueType::Date => VCardValueType::Date, + ArchivedVCardValueType::DateAndOrTime => VCardValueType::DateAndOrTime, + ArchivedVCardValueType::DateTime => VCardValueType::DateTime, + ArchivedVCardValueType::Float => VCardValueType::Float, + ArchivedVCardValueType::Integer => VCardValueType::Integer, + ArchivedVCardValueType::LanguageTag => VCardValueType::LanguageTag, + ArchivedVCardValueType::Text => VCardValueType::Text, + ArchivedVCardValueType::Time => VCardValueType::Time, + ArchivedVCardValueType::Timestamp => VCardValueType::Timestamp, + ArchivedVCardValueType::Uri => VCardValueType::Uri, + ArchivedVCardValueType::UtcOffset => VCardValueType::UtcOffset, + } + } +} + +impl Display for ArchivedVCard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_to(f, self.version().unwrap_or_default()) + } +} + +impl<'x> From<&'x ArchivedJscomp> for JscompRef<'x> { + fn from(jscomp: &'x ArchivedJscomp) -> Self { + match jscomp { + ArchivedJscomp::Entry { position, value } => JscompRef::Entry { + position: position.to_native(), + value: value.to_native(), + }, + ArchivedJscomp::Separator(separator) => JscompRef::Separator(separator), + } + } +} + diff --git a/crates/calcard/src/vcard/types.rs b/crates/calcard/src/vcard/types.rs new file mode 100644 index 00000000..95da14ff --- /dev/null +++ b/crates/calcard/src/vcard/types.rs @@ -0,0 +1,629 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +use super::*; + +impl IanaParse for VCardProperty { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "BEGIN" => Some(VCardProperty::Begin), + "END" => Some(VCardProperty::End), + "SOURCE" => Some(VCardProperty::Source), + "KIND" => Some(VCardProperty::Kind), + "XML" => Some(VCardProperty::Xml), + "FN" => Some(VCardProperty::Fn), + "N" => Some(VCardProperty::N), + "NICKNAME" => Some(VCardProperty::Nickname), + "PHOTO" => Some(VCardProperty::Photo), + "BDAY" => Some(VCardProperty::Bday), + "ANNIVERSARY" => Some(VCardProperty::Anniversary), + "GENDER" => Some(VCardProperty::Gender), + "ADR" => Some(VCardProperty::Adr), + "TEL" => Some(VCardProperty::Tel), + "EMAIL" => Some(VCardProperty::Email), + "IMPP" => Some(VCardProperty::Impp), + "LANG" => Some(VCardProperty::Lang), + "TZ" => Some(VCardProperty::Tz), + "GEO" => Some(VCardProperty::Geo), + "TITLE" => Some(VCardProperty::Title), + "ROLE" => Some(VCardProperty::Role), + "LOGO" => Some(VCardProperty::Logo), + "ORG" => Some(VCardProperty::Org), + "MEMBER" => Some(VCardProperty::Member), + "RELATED" => Some(VCardProperty::Related), + "CATEGORIES" => Some(VCardProperty::Categories), + "NOTE" => Some(VCardProperty::Note), + "PRODID" => Some(VCardProperty::Prodid), + "REV" => Some(VCardProperty::Rev), + "SOUND" => Some(VCardProperty::Sound), + "UID" => Some(VCardProperty::Uid), + "CLIENTPIDMAP" => Some(VCardProperty::Clientpidmap), + "URL" => Some(VCardProperty::Url), + "VERSION" => Some(VCardProperty::Version), + "KEY" => Some(VCardProperty::Key), + "FBURL" => Some(VCardProperty::Fburl), + "CALADRURI" => Some(VCardProperty::Caladruri), + "CALURI" => Some(VCardProperty::Caluri), + "BIRTHPLACE" => Some(VCardProperty::Birthplace), + "DEATHPLACE" => Some(VCardProperty::Deathplace), + "DEATHDATE" => Some(VCardProperty::Deathdate), + "EXPERTISE" => Some(VCardProperty::Expertise), + "HOBBY" => Some(VCardProperty::Hobby), + "INTEREST" => Some(VCardProperty::Interest), + "ORG-DIRECTORY" => Some(VCardProperty::OrgDirectory), + "CONTACT-URI" => Some(VCardProperty::ContactUri), + "CREATED" => Some(VCardProperty::Created), + "GRAMGENDER" => Some(VCardProperty::Gramgender), + "LANGUAGE" => Some(VCardProperty::Language), + "PRONOUNS" => Some(VCardProperty::Pronouns), + "SOCIALPROFILE" => Some(VCardProperty::Socialprofile), + "JSPROP" => Some(VCardProperty::Jsprop), + _ => None, + ) + } +} + +impl VCardProperty { + pub fn as_str(&self) -> &str { + match self { + VCardProperty::Source => "SOURCE", + VCardProperty::Kind => "KIND", + VCardProperty::Xml => "XML", + VCardProperty::Fn => "FN", + VCardProperty::N => "N", + VCardProperty::Nickname => "NICKNAME", + VCardProperty::Photo => "PHOTO", + VCardProperty::Bday => "BDAY", + VCardProperty::Anniversary => "ANNIVERSARY", + VCardProperty::Gender => "GENDER", + VCardProperty::Adr => "ADR", + VCardProperty::Tel => "TEL", + VCardProperty::Email => "EMAIL", + VCardProperty::Impp => "IMPP", + VCardProperty::Lang => "LANG", + VCardProperty::Tz => "TZ", + VCardProperty::Geo => "GEO", + VCardProperty::Title => "TITLE", + VCardProperty::Role => "ROLE", + VCardProperty::Logo => "LOGO", + VCardProperty::Org => "ORG", + VCardProperty::Member => "MEMBER", + VCardProperty::Related => "RELATED", + VCardProperty::Categories => "CATEGORIES", + VCardProperty::Note => "NOTE", + VCardProperty::Prodid => "PRODID", + VCardProperty::Rev => "REV", + VCardProperty::Sound => "SOUND", + VCardProperty::Uid => "UID", + VCardProperty::Clientpidmap => "CLIENTPIDMAP", + VCardProperty::Url => "URL", + VCardProperty::Version => "VERSION", + VCardProperty::Key => "KEY", + VCardProperty::Fburl => "FBURL", + VCardProperty::Caladruri => "CALADRURI", + VCardProperty::Caluri => "CALURI", + VCardProperty::Birthplace => "BIRTHPLACE", + VCardProperty::Deathplace => "DEATHPLACE", + VCardProperty::Deathdate => "DEATHDATE", + VCardProperty::Expertise => "EXPERTISE", + VCardProperty::Hobby => "HOBBY", + VCardProperty::Interest => "INTEREST", + VCardProperty::OrgDirectory => "ORG-DIRECTORY", + VCardProperty::ContactUri => "CONTACT-URI", + VCardProperty::Created => "CREATED", + VCardProperty::Gramgender => "GRAMGENDER", + VCardProperty::Language => "LANGUAGE", + VCardProperty::Pronouns => "PRONOUNS", + VCardProperty::Socialprofile => "SOCIALPROFILE", + VCardProperty::Jsprop => "JSPROP", + VCardProperty::Begin => "BEGIN", + VCardProperty::End => "END", + VCardProperty::Other(v) => v.as_str(), + } + } + + pub(crate) fn legacy_max_values(&self) -> usize { + match self { + VCardProperty::N => LEGACY_N_COMPONENTS, + VCardProperty::Adr => LEGACY_ADR_COMPONENTS, + _ => usize::MAX, + } + } +} + +pub(crate) const LEGACY_N_COMPONENTS: usize = 5; +pub(crate) const LEGACY_ADR_COMPONENTS: usize = 7; + +impl VCardProperty { + // Returns the default value type and whether the property is multi-valued. + pub(crate) fn default_types(&self) -> (ValueType, ValueSeparator) { + match self { + VCardProperty::Source => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Kind => (ValueType::Kind, ValueSeparator::None), + VCardProperty::Xml => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Fn => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::N => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::SemicolonAndComma, + ), + VCardProperty::Nickname => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::Comma, + ), + VCardProperty::Photo => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Bday => ( + ValueType::Vcard(VCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + VCardProperty::Anniversary => ( + ValueType::Vcard(VCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + VCardProperty::Gender => (ValueType::Sex, ValueSeparator::Semicolon), + VCardProperty::Adr => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::SemicolonAndComma, + ), + VCardProperty::Tel => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Email => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Impp => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Lang => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Tz => ( + ValueType::Vcard(VCardValueType::UtcOffset), + ValueSeparator::None, + ), + VCardProperty::Geo => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Title => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Role => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Logo => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Org => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::Semicolon, + ), + VCardProperty::Member => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Related => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Categories => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::Comma, + ), + VCardProperty::Note => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Prodid => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Rev => ( + ValueType::Vcard(VCardValueType::Timestamp), + ValueSeparator::None, + ), + VCardProperty::Sound => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Uid => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Clientpidmap => ( + ValueType::Vcard(VCardValueType::Text), + ValueSeparator::Semicolon, + ), + VCardProperty::Url => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Version => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Key => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Fburl => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Caladruri => { + (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None) + } + VCardProperty::Caluri => (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None), + VCardProperty::Birthplace => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Deathplace => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Deathdate => ( + ValueType::Vcard(VCardValueType::DateAndOrTime), + ValueSeparator::None, + ), + VCardProperty::Expertise => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Hobby => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Interest => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::OrgDirectory => { + (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None) + } + VCardProperty::ContactUri => { + (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None) + } + VCardProperty::Created => ( + ValueType::Vcard(VCardValueType::Timestamp), + ValueSeparator::None, + ), + VCardProperty::Gramgender => (ValueType::GramGender, ValueSeparator::None), + VCardProperty::Language => ( + ValueType::Vcard(VCardValueType::LanguageTag), + ValueSeparator::None, + ), + VCardProperty::Pronouns => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Socialprofile => { + (ValueType::Vcard(VCardValueType::Uri), ValueSeparator::None) + } + VCardProperty::Jsprop => (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None), + VCardProperty::Other(_) => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::None) + } + VCardProperty::Begin | VCardProperty::End => { + (ValueType::Vcard(VCardValueType::Text), ValueSeparator::Skip) + } + } + } +} + +impl Eq for VCardValue {} + +impl IanaString for VCardValueType { + fn as_str(&self) -> &'static str { + match self { + VCardValueType::Boolean => "BOOLEAN", + VCardValueType::Date => "DATE", + VCardValueType::DateAndOrTime => "DATE-AND-OR-TIME", + VCardValueType::DateTime => "DATE-TIME", + VCardValueType::Float => "FLOAT", + VCardValueType::Integer => "INTEGER", + VCardValueType::LanguageTag => "LANGUAGE-TAG", + VCardValueType::Text => "TEXT", + VCardValueType::Time => "TIME", + VCardValueType::Timestamp => "TIMESTAMP", + VCardValueType::Uri => "URI", + VCardValueType::UtcOffset => "UTC-OFFSET", + } + } +} + +impl IanaParse for VCardValueType { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, VCardValueType, + "BOOLEAN" => VCardValueType::Boolean, + "DATE" => VCardValueType::Date, + "DATE-AND-OR-TIME" => VCardValueType::DateAndOrTime, + "DATE-TIME" => VCardValueType::DateTime, + "FLOAT" => VCardValueType::Float, + "INTEGER" => VCardValueType::Integer, + "LANGUAGE-TAG" => VCardValueType::LanguageTag, + "TEXT" => VCardValueType::Text, + "TIME" => VCardValueType::Time, + "TIMESTAMP" => VCardValueType::Timestamp, + "URI" => VCardValueType::Uri, + "UTC-OFFSET" => VCardValueType::UtcOffset, + ) + .copied() + } +} + +impl IanaParse for VCardLevel { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, VCardLevel, + "beginner" => VCardLevel::Beginner, + "average" => VCardLevel::Average, + "expert" => VCardLevel::Expert, + "high" => VCardLevel::High, + "medium" => VCardLevel::Medium, + "low" => VCardLevel::Low, + ) + .copied() + } +} + +impl IanaString for VCardLevel { + fn as_str(&self) -> &'static str { + match self { + VCardLevel::Beginner => "BEGINNER", + VCardLevel::Average => "AVERAGE", + VCardLevel::Expert => "EXPERT", + VCardLevel::High => "HIGH", + VCardLevel::Medium => "MEDIUM", + VCardLevel::Low => "LOW", + } + } +} + +impl IanaParse for VCardPhonetic { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "ipa" => Some(VCardPhonetic::Ipa), + "jyut" => Some(VCardPhonetic::Jyut), + "piny" => Some(VCardPhonetic::Piny), + "script" => Some(VCardPhonetic::Script), + _ => None, + ) + } +} + +impl IanaString for VCardPhonetic { + fn as_str(&self) -> &'static str { + match self { + VCardPhonetic::Ipa => "IPA", + VCardPhonetic::Jyut => "JYUT", + VCardPhonetic::Piny => "PINY", + VCardPhonetic::Script => "SCRIPT", + } + } +} + +impl IanaParse for VCardType { + fn parse(value: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(value, + "work" => Some(VCardType::Work), + "home" => Some(VCardType::Home), + "billing" => Some(VCardType::Billing), + "delivery" => Some(VCardType::Delivery), + "contact" => Some(VCardType::Contact), + "acquaintance" => Some(VCardType::Acquaintance), + "friend" => Some(VCardType::Friend), + "met" => Some(VCardType::Met), + "co-worker" => Some(VCardType::CoWorker), + "colleague" => Some(VCardType::Colleague), + "co-resident" => Some(VCardType::CoResident), + "neighbor" => Some(VCardType::Neighbor), + "child" => Some(VCardType::Child), + "parent" => Some(VCardType::Parent), + "sibling" => Some(VCardType::Sibling), + "spouse" => Some(VCardType::Spouse), + "kin" => Some(VCardType::Kin), + "muse" => Some(VCardType::Muse), + "crush" => Some(VCardType::Crush), + "date" => Some(VCardType::Date), + "sweetheart" => Some(VCardType::Sweetheart), + "me" => Some(VCardType::Me), + "agent" => Some(VCardType::Agent), + "emergency" => Some(VCardType::Emergency), + "text" => Some(VCardType::Text), + "voice" => Some(VCardType::Voice), + "fax" => Some(VCardType::Fax), + "cell" => Some(VCardType::Cell), + "video" => Some(VCardType::Video), + "pager" => Some(VCardType::Pager), + "textphone" => Some(VCardType::Textphone), + "main-number" => Some(VCardType::MainNumber), + _ => None, + ) + } +} + +impl IanaString for VCardType { + fn as_str(&self) -> &'static str { + match self { + VCardType::Work => "WORK", + VCardType::Home => "HOME", + VCardType::Billing => "BILLING", + VCardType::Delivery => "DELIVERY", + VCardType::Contact => "CONTACT", + VCardType::Acquaintance => "ACQUAINTANCE", + VCardType::Friend => "FRIEND", + VCardType::Met => "MET", + VCardType::CoWorker => "CO-WORKER", + VCardType::Colleague => "COLLEAGUE", + VCardType::CoResident => "CO-RESIDENT", + VCardType::Neighbor => "NEIGHBOR", + VCardType::Child => "CHILD", + VCardType::Parent => "PARENT", + VCardType::Sibling => "SIBLING", + VCardType::Spouse => "SPOUSE", + VCardType::Kin => "KIN", + VCardType::Muse => "MUSE", + VCardType::Crush => "CRUSH", + VCardType::Date => "DATE", + VCardType::Sweetheart => "SWEETHEART", + VCardType::Me => "ME", + VCardType::Agent => "AGENT", + VCardType::Emergency => "EMERGENCY", + VCardType::Text => "TEXT", + VCardType::Voice => "VOICE", + VCardType::Fax => "FAX", + VCardType::Cell => "CELL", + VCardType::Video => "VIDEO", + VCardType::Pager => "PAGER", + VCardType::Textphone => "TEXTPHONE", + VCardType::MainNumber => "MAIN-NUMBER", + } + } +} + +impl IanaParse for VCardGramGender { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, VCardGramGender, + "animate" => VCardGramGender::Animate, + "common" => VCardGramGender::Common, + "feminine" => VCardGramGender::Feminine, + "inanimate" => VCardGramGender::Inanimate, + "masculine" => VCardGramGender::Masculine, + "neuter" => VCardGramGender::Neuter, + ) + .copied() + } +} + +impl IanaString for VCardGramGender { + fn as_str(&self) -> &'static str { + match self { + VCardGramGender::Animate => "ANIMATE", + VCardGramGender::Common => "COMMON", + VCardGramGender::Feminine => "FEMININE", + VCardGramGender::Inanimate => "INANIMATE", + VCardGramGender::Masculine => "MASCULINE", + VCardGramGender::Neuter => "NEUTER", + } + } +} + +impl IanaParse for VCardSex { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, VCardSex, + "M" => VCardSex::Male, + "F" => VCardSex::Female, + "O" => VCardSex::Other, + "N" => VCardSex::NoneOrNotApplicable, + "U" => VCardSex::Unknown, + ) + .copied() + } +} + +impl IanaString for VCardSex { + fn as_str(&self) -> &'static str { + match self { + VCardSex::Male => "M", + VCardSex::Female => "F", + VCardSex::Other => "O", + VCardSex::NoneOrNotApplicable => "N", + VCardSex::Unknown => "U", + } + } +} + +impl IanaParse for VCardKind { + fn parse(value: &[u8]) -> Option { + hashify::map_ignore_case!(value, VCardKind, + "individual" => VCardKind::Individual, + "group" => VCardKind::Group, + "org" => VCardKind::Org, + "location" => VCardKind::Location, + "application" => VCardKind::Application, + "device" => VCardKind::Device, + ) + .copied() + } +} + +impl IanaString for VCardKind { + fn as_str(&self) -> &'static str { + match self { + VCardKind::Individual => "INDIVIDUAL", + VCardKind::Group => "GROUP", + VCardKind::Org => "ORG", + VCardKind::Location => "LOCATION", + VCardKind::Application => "APPLICATION", + VCardKind::Device => "DEVICE", + } + } +} + +impl VCardParameterName { + pub fn try_parse(input: &[u8]) -> Option { + hashify::fnc_map_ignore_case!(input, + b"LANGUAGE" => Some(VCardParameterName::Language), + b"VALUE" => Some(VCardParameterName::Value), + b"PREF" => Some(VCardParameterName::Pref), + b"ALTID" => Some(VCardParameterName::Altid), + b"PID" => Some(VCardParameterName::Pid), + b"TYPE" => Some(VCardParameterName::Type), + b"MEDIATYPE" => Some(VCardParameterName::Mediatype), + b"CALSCALE" => Some(VCardParameterName::Calscale), + b"SORT-AS" => Some(VCardParameterName::SortAs), + b"GEO" => Some(VCardParameterName::Geo), + b"TZ" => Some(VCardParameterName::Tz), + b"INDEX" => Some(VCardParameterName::Index), + b"LEVEL" => Some(VCardParameterName::Level), + b"GROUP" => Some(VCardParameterName::Group), + b"CC" => Some(VCardParameterName::Cc), + b"AUTHOR" => Some(VCardParameterName::Author), + b"AUTHOR-NAME" => Some(VCardParameterName::AuthorName), + b"CREATED" => Some(VCardParameterName::Created), + b"DERIVED" => Some(VCardParameterName::Derived), + b"LABEL" => Some(VCardParameterName::Label), + b"PHONETIC" => Some(VCardParameterName::Phonetic), + b"PROP-ID" => Some(VCardParameterName::PropId), + b"SCRIPT" => Some(VCardParameterName::Script), + b"SERVICE-TYPE" => Some(VCardParameterName::ServiceType), + b"USERNAME" => Some(VCardParameterName::Username), + b"JSPTR" => Some(VCardParameterName::Jsptr), + b"JSCOMPS" => Some(VCardParameterName::Jscomps), + _ => None, + ) + } + + pub fn parse(input: &str) -> Self { + Self::try_parse(input.as_bytes()).unwrap_or_else(|| VCardParameterName::Other(input.into())) + } + + pub fn as_str(&self) -> &str { + match self { + VCardParameterName::Language => "LANGUAGE", + VCardParameterName::Value => "VALUE", + VCardParameterName::Pref => "PREF", + VCardParameterName::Altid => "ALTID", + VCardParameterName::Pid => "PID", + VCardParameterName::Type => "TYPE", + VCardParameterName::Mediatype => "MEDIATYPE", + VCardParameterName::Calscale => "CALSCALE", + VCardParameterName::SortAs => "SORT-AS", + VCardParameterName::Geo => "GEO", + VCardParameterName::Tz => "TZ", + VCardParameterName::Index => "INDEX", + VCardParameterName::Level => "LEVEL", + VCardParameterName::Group => "GROUP", + VCardParameterName::Cc => "CC", + VCardParameterName::Author => "AUTHOR", + VCardParameterName::AuthorName => "AUTHOR-NAME", + VCardParameterName::Created => "CREATED", + VCardParameterName::Derived => "DERIVED", + VCardParameterName::Label => "LABEL", + VCardParameterName::Phonetic => "PHONETIC", + VCardParameterName::PropId => "PROP-ID", + VCardParameterName::Script => "SCRIPT", + VCardParameterName::ServiceType => "SERVICE-TYPE", + VCardParameterName::Username => "USERNAME", + VCardParameterName::Jsptr => "JSPTR", + VCardParameterName::Jscomps => "JSCOMPS", + VCardParameterName::Other(name) => name, + } + } + + pub fn into_string(self) -> Cow<'static, str> { + match self { + VCardParameterName::Language => "LANGUAGE".into(), + VCardParameterName::Value => "VALUE".into(), + VCardParameterName::Pref => "PREF".into(), + VCardParameterName::Altid => "ALTID".into(), + VCardParameterName::Pid => "PID".into(), + VCardParameterName::Type => "TYPE".into(), + VCardParameterName::Mediatype => "MEDIATYPE".into(), + VCardParameterName::Calscale => "CALSCALE".into(), + VCardParameterName::SortAs => "SORT-AS".into(), + VCardParameterName::Geo => "GEO".into(), + VCardParameterName::Tz => "TZ".into(), + VCardParameterName::Index => "INDEX".into(), + VCardParameterName::Level => "LEVEL".into(), + VCardParameterName::Group => "GROUP".into(), + VCardParameterName::Cc => "CC".into(), + VCardParameterName::Author => "AUTHOR".into(), + VCardParameterName::AuthorName => "AUTHOR-NAME".into(), + VCardParameterName::Created => "CREATED".into(), + VCardParameterName::Derived => "DERIVED".into(), + VCardParameterName::Label => "LABEL".into(), + VCardParameterName::Phonetic => "PHONETIC".into(), + VCardParameterName::PropId => "PROP-ID".into(), + VCardParameterName::Script => "SCRIPT".into(), + VCardParameterName::ServiceType => "SERVICE-TYPE".into(), + VCardParameterName::Username => "USERNAME".into(), + VCardParameterName::Jsptr => "JSPTR".into(), + VCardParameterName::Jscomps => "JSCOMPS".into(), + VCardParameterName::Other(name) => name.into(), + } + } +} + +impl VCardVersion { + pub fn try_parse(input: &str) -> Option { + hashify::map!(input.as_bytes(), VCardVersion, + b"4.0" => VCardVersion::V4_0, + b"3.0" => VCardVersion::V3_0, + b"2.1" => VCardVersion::V2_1, + b"2.0" => VCardVersion::V2_0, + ) + .copied() + } +} diff --git a/crates/calcard/src/vcard/utils.rs b/crates/calcard/src/vcard/utils.rs new file mode 100644 index 00000000..736e9bbf --- /dev/null +++ b/crates/calcard/src/vcard/utils.rs @@ -0,0 +1,430 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{VCard, VCardEntry, VCardParameterName, VCardProperty, VCardValue, VCardVersion}; +use crate::{ + common::{CalendarScale, Data, IanaString, IanaType, PartialDateTime, writer::write_jscomps}, + vcard::{ + Jscomp, VCardLevel, VCardParameter, VCardParameterValue, VCardPhonetic, VCardType, + VCardValueType, media_type::media_type_from_legacy, + }, +}; +use encodify::base64; +use std::borrow::Cow; + +impl VCard { + pub fn uid(&self) -> Option<&str> { + self.property(&VCardProperty::Uid) + .and_then(|e| e.values.first()) + .and_then(|v| v.as_text()) + } + + pub fn property(&self, prop: &VCardProperty) -> Option<&VCardEntry> { + self.entries.iter().find(|entry| &entry.name == prop) + } + + pub fn properties<'x, 'y: 'x>( + &'x self, + prop: &'y VCardProperty, + ) -> impl Iterator + 'x { + self.entries.iter().filter(move |entry| &entry.name == prop) + } + + pub fn version(&self) -> Option { + self.entries + .iter() + .find(|e| e.name == VCardProperty::Version) + .and_then(|e| { + e.values + .first() + .and_then(|v| v.as_text()) + .and_then(VCardVersion::try_parse) + }) + } + + pub fn size(&self) -> usize { + self.entries.iter().map(|e| e.size()).sum() + } +} + +impl VCardEntry { + #[inline] + pub fn parameters( + &self, + prop: &VCardParameterName, + ) -> impl Iterator { + self.params.iter().filter_map(move |param| { + if ¶m.name == prop { + Some(¶m.value) + } else { + None + } + }) + } + + pub fn language(&self) -> Option<&str> { + self.parameters(&VCardParameterName::Language) + .find_map(|v| v.as_text()) + } + + pub fn alt_id(&self) -> Option<&str> { + self.parameters(&VCardParameterName::Altid) + .find_map(|v| v.as_text()) + } + + pub fn prop_id(&self) -> Option<&str> { + self.parameters(&VCardParameterName::PropId) + .find_map(|v| v.as_text()) + } + + pub fn phonetic_system(&self) -> Option> { + self.parameters(&VCardParameterName::Phonetic) + .find_map(|v| v.as_phonetic()) + } + + pub fn phonetic_script(&self) -> Option<&str> { + self.parameters(&VCardParameterName::Script) + .find_map(|v| v.as_text()) + } + + pub fn size(&self) -> usize { + self.group.as_ref().map_or(0, |g| g.len()) + + self.name.as_str().len() + + self.params.iter().map(|p| p.size()).sum::() + + self.values.iter().map(|v| v.size()).sum::() + } + + pub(crate) fn normalize_legacy_media_type(&mut self) { + if self + .values + .iter() + .any(|value| matches!(value, VCardValue::Binary(data) if data.content_type.is_none())) + { + let property = self.name.as_str(); + let media_type = + self.params + .iter() + .find_map(|param| match (¶m.name, ¶m.value) { + (VCardParameterName::Mediatype, VCardParameterValue::Text(text)) => { + Some(Cow::Owned(text.to_ascii_lowercase())) + } + (VCardParameterName::Type, VCardParameterValue::Text(text)) => { + media_type_from_legacy(property, text) + } + _ => None, + }); + + if let Some(media_type) = media_type { + for value in &mut self.values { + if let VCardValue::Binary(data) = value + && data.content_type.is_none() + { + data.content_type = Some(media_type.to_string()); + } + } + } + } + } +} + +impl VCardValue { + pub fn as_text(&self) -> Option<&str> { + match self { + VCardValue::Text(v) => v.as_str().into(), + VCardValue::Sex(v) => v.as_str().into(), + VCardValue::GramGender(v) => v.as_str().into(), + VCardValue::Kind(v) => v.as_str().into(), + VCardValue::Component(v) => v.first().map(|s| s.as_str()), + VCardValue::Integer(_) + | VCardValue::Float(_) + | VCardValue::Boolean(_) + | VCardValue::PartialDateTime(_) + | VCardValue::Binary(_) => None, + } + } + + pub fn into_text(self) -> Option> { + match self { + VCardValue::Text(s) => Some(Cow::Owned(s)), + VCardValue::Sex(v) => Some(Cow::Borrowed(v.as_str())), + VCardValue::GramGender(v) => Some(Cow::Borrowed(v.as_str())), + VCardValue::Kind(v) => Some(Cow::Borrowed(v.as_str())), + VCardValue::Component(v) => Some(Cow::Owned(v.join(","))), + VCardValue::Integer(_) + | VCardValue::Float(_) + | VCardValue::Boolean(_) + | VCardValue::PartialDateTime(_) + | VCardValue::Binary(_) => None, + } + } + + pub fn into_uri(self) -> Option { + match self { + VCardValue::Binary(v) => Some(v.to_unwrapped_string()), + VCardValue::Text(v) => Some(v), + _ => None, + } + } + + pub fn as_integer(&self) -> Option { + match self { + VCardValue::Integer(i) => Some(*i), + _ => None, + } + } + + pub fn as_float(&self) -> Option { + match self { + VCardValue::Float(f) => Some(*f), + _ => None, + } + } + + pub fn as_boolean(&self) -> Option { + match self { + VCardValue::Boolean(b) => Some(*b), + _ => None, + } + } + + pub fn as_partial_date_time(&self) -> Option<&PartialDateTime> { + match self { + VCardValue::PartialDateTime(dt) => Some(dt), + _ => None, + } + } + + pub fn as_binary(&self) -> Option<&Data> { + match self { + VCardValue::Binary(d) => Some(d), + _ => None, + } + } + + pub fn size(&self) -> usize { + match self { + VCardValue::Text(v) => v.len(), + VCardValue::Sex(v) => v.as_str().len(), + VCardValue::GramGender(v) => v.as_str().len(), + VCardValue::Kind(v) => v.as_str().len(), + VCardValue::Component(v) => v.iter().map(|s| s.len()).sum::(), + VCardValue::Integer(_) => std::mem::size_of::(), + VCardValue::Float(_) => std::mem::size_of::(), + VCardValue::Boolean(_) => std::mem::size_of::(), + VCardValue::PartialDateTime(_) => std::mem::size_of::(), + VCardValue::Binary(d) => { + d.content_type.as_ref().map_or(0, |ct| ct.len()) + d.data.len() + } + } + } +} + +impl Data { + pub fn to_unwrapped_string(&self) -> String { + const SCHEME: &str = "data:"; + const BASE64_MARKER: &str = ";base64,"; + let media_type = self.content_type.as_deref().unwrap_or_default(); + let mut out = String::with_capacity( + SCHEME.len() + + media_type.len() + + BASE64_MARKER.len() + + base64::STANDARD.encoded_len(self.data.len()), + ); + out.push_str(SCHEME); + out.push_str(media_type); + out.push_str(BASE64_MARKER); + base64::STANDARD.encode_append(&self.data, &mut out); + out + } +} + +impl VCardParameterValue { + pub fn into_text(self) -> Cow<'static, str> { + match self { + VCardParameterValue::Text(s) => Cow::Owned(s), + VCardParameterValue::ValueType(v) => Cow::Borrowed(v.as_str()), + VCardParameterValue::Type(v) => Cow::Borrowed(v.as_str()), + VCardParameterValue::Calscale(v) => Cow::Borrowed(v.as_str()), + VCardParameterValue::Level(v) => Cow::Borrowed(v.as_str()), + VCardParameterValue::Phonetic(v) => Cow::Borrowed(v.as_str()), + VCardParameterValue::Jscomps(v) => { + let mut jscomps = String::new(); + let _ = write_jscomps(&mut jscomps, &v); + Cow::Owned(jscomps) + } + VCardParameterValue::Integer(i) => Cow::Owned(i.to_string()), + VCardParameterValue::Timestamp(t) => Cow::Owned(t.to_string()), + VCardParameterValue::Bool(b) => Cow::Borrowed(if b { "true" } else { "false" }), + VCardParameterValue::Null => Cow::Borrowed(""), + } + } + + pub fn as_text(&self) -> Option<&str> { + match self { + VCardParameterValue::Text(v) => Some(v.as_str()), + VCardParameterValue::ValueType(v) => Some(v.as_str()), + VCardParameterValue::Type(v) => Some(v.as_str()), + VCardParameterValue::Calscale(v) => Some(v.as_str()), + VCardParameterValue::Level(v) => Some(v.as_str()), + VCardParameterValue::Phonetic(v) => Some(v.as_str()), + VCardParameterValue::Jscomps(_) + | VCardParameterValue::Integer(_) + | VCardParameterValue::Timestamp(_) + | VCardParameterValue::Bool(_) + | VCardParameterValue::Null => None, + } + } + + pub fn into_phonetic(self) -> Option> { + match self { + VCardParameterValue::Phonetic(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_phonetic(&self) -> Option> { + match self { + VCardParameterValue::Phonetic(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_calscale(self) -> Option> { + match self { + VCardParameterValue::Calscale(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_calscale(&self) -> Option> { + match self { + VCardParameterValue::Calscale(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_level(self) -> Option> { + match self { + VCardParameterValue::Level(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_level(&self) -> Option> { + match self { + VCardParameterValue::Level(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_type(self) -> Option> { + match self { + VCardParameterValue::Type(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_type(&self) -> Option> { + match self { + VCardParameterValue::Type(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_value_type(self) -> Option> { + match self { + VCardParameterValue::ValueType(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_value_type(&self) -> Option> { + match self { + VCardParameterValue::ValueType(v) => Some(IanaType::Iana(v)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_bool(&self) -> Option> { + match self { + VCardParameterValue::Bool(b) => Some(IanaType::Iana(*b)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn as_integer(&self) -> Option> { + match self { + VCardParameterValue::Integer(i) => Some(IanaType::Iana(*i)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_timestamp(self) -> Option> { + match self { + VCardParameterValue::Timestamp(t) => Some(IanaType::Iana(t)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v)), + _ => None, + } + } + + pub fn as_timestamp(&self) -> Option> { + match self { + VCardParameterValue::Timestamp(t) => Some(IanaType::Iana(*t)), + VCardParameterValue::Text(v) => Some(IanaType::Other(v.as_str())), + _ => None, + } + } + + pub fn into_jscomps(self) -> Option> { + match self { + VCardParameterValue::Jscomps(v) => Some(v), + _ => None, + } + } + + pub fn size(&self) -> usize { + match self { + VCardParameterValue::Text(v) => v.len(), + VCardParameterValue::ValueType(v) => v.as_str().len(), + VCardParameterValue::Type(v) => v.as_str().len(), + VCardParameterValue::Calscale(v) => v.as_str().len(), + VCardParameterValue::Level(v) => v.as_str().len(), + VCardParameterValue::Phonetic(v) => v.as_str().len(), + VCardParameterValue::Jscomps(v) => v.iter().map(|j| j.size()).sum::(), + VCardParameterValue::Integer(_) => std::mem::size_of::(), + VCardParameterValue::Timestamp(_) => std::mem::size_of::(), + VCardParameterValue::Bool(_) => std::mem::size_of::(), + VCardParameterValue::Null => 0, + } + } +} + +impl VCardParameter { + pub fn size(&self) -> usize { + self.name.as_str().len() + self.value.size() + } +} + +impl Jscomp { + pub fn size(&self) -> usize { + match self { + Jscomp::Entry { .. } => std::mem::size_of::(), + Jscomp::Separator(value) => value.len(), + } + } +} diff --git a/crates/calcard/src/vcard/writer.rs b/crates/calcard/src/vcard/writer.rs new file mode 100644 index 00000000..76bc4773 --- /dev/null +++ b/crates/calcard/src/vcard/writer.rs @@ -0,0 +1,573 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{PartialDateTime, VCard, VCardEntry, VCardValueType, VCardVersion}; +use crate::{ + common::{ + IanaString, + format::{AsciiPush, BufferedWrite}, + parser::Timestamp, + writer::{ + DOCUMENT_BUFFER, ENTRY_BUFFER, FoldingWriter, LineWriter, NeedsQuotes, write_jscomps, + write_param_value, write_text, + }, + }, + vcard::{ + VCardParameterName, VCardParameterValue, VCardProperty, VCardValue, ValueSeparator, + media_type::legacy_media_type, + }, +}; +use std::fmt::{self, Display, Write}; + +pub(crate) const DATE_BUFFER: usize = 48; + +impl VCardVersion { + pub(crate) fn begin_and_version(self) -> &'static str { + match self { + VCardVersion::V2_0 => "BEGIN:VCARD\r\nVERSION:2.0\r\n", + VCardVersion::V2_1 => "BEGIN:VCARD\r\nVERSION:2.1\r\n", + VCardVersion::V3_0 => "BEGIN:VCARD\r\nVERSION:3.0\r\n", + VCardVersion::V4_0 => "BEGIN:VCARD\r\nVERSION:4.0\r\n", + } + } +} + +impl VCard { + pub fn write_to(&self, out: &mut impl Write, version: VCardVersion) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_entries(&mut FoldingWriter::new(buf), version) + }) + } + + fn write_entries( + &self, + out: &mut FoldingWriter<'_, W>, + version: VCardVersion, + ) -> fmt::Result { + out.write_raw(version.begin_and_version())?; + for entry in &self.entries { + if !matches!( + entry.name, + VCardProperty::Begin | VCardProperty::End | VCardProperty::Version + ) { + entry.write_line(out, version)?; + } + } + out.write_raw("END:VCARD\r\n") + } +} + +impl VCardEntry { + #[deprecated(since = "0.4.0", note = "use write_with_version")] + pub fn write_to(&self, out: &mut impl Write, is_v4: bool) -> fmt::Result { + self.write_with_version( + out, + if is_v4 { + VCardVersion::V4_0 + } else { + VCardVersion::V3_0 + }, + ) + } + + pub fn write_with_version(&self, out: &mut impl Write, version: VCardVersion) -> fmt::Result { + out.write_buffered::(|buf| { + self.write_line(&mut FoldingWriter::new(buf), version) + }) + } + + pub(crate) fn write_line( + &self, + out: &mut FoldingWriter<'_, W>, + version: VCardVersion, + ) -> fmt::Result { + let is_v4 = matches!(version, VCardVersion::V4_0); + + if let Some(group_name) = &self.group { + out.write_atomic(group_name)?; + out.write_atomic(".")?; + } + + out.write_atomic(self.name.as_str())?; + let mut types = None; + let mut last_param: Option<&VCardParameterName> = None; + let legacy_binary = if is_v4 { + None + } else { + self.values.iter().find_map(|v| match v { + VCardValue::Binary(data) => Some(data), + _ => None, + }) + }; + + for param in &self.params { + if !is_v4 + && match param.name { + VCardParameterName::PropId | VCardParameterName::Jscomps => true, + VCardParameterName::Mediatype => legacy_binary.is_some(), + _ => false, + } + { + continue; + } + + if last_param.is_some_and(|last_param| last_param == ¶m.name) { + out.write_atomic(",")?; + } else { + out.write_atomic(";")?; + out.write_atomic(param.name.as_str())?; + if !matches!(param.value, VCardParameterValue::Null) { + out.write_atomic("=")?; + } + last_param = Some(¶m.name); + } + + match ¶m.value { + VCardParameterValue::Text(v) + if param.name == VCardParameterName::Jsptr && !v.needs_quotes() => + { + out.write_atomic("\"")?; + write_param_value(out, v, is_v4)?; + out.write_atomic("\"")?; + } + VCardParameterValue::Text(v) => { + write_param_value(out, v, is_v4)?; + } + VCardParameterValue::Integer(i) => { + out.push_u64((*i).into())?; + } + VCardParameterValue::Timestamp(v) => { + Timestamp(*v).push_to(out)?; + } + VCardParameterValue::Bool(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + } + VCardParameterValue::ValueType(v) => { + if types.is_none() { + types = Some(v); + } + write_param_value(out, v.as_str(), is_v4)?; + } + VCardParameterValue::Type(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + VCardParameterValue::Calscale(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + VCardParameterValue::Level(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + VCardParameterValue::Phonetic(v) => { + write_param_value(out, v.as_str(), is_v4)?; + } + VCardParameterValue::Jscomps(v) => { + out.write_atomic("\"")?; + write_jscomps(out, v)?; + out.write_atomic("\"")?; + last_param = None; + } + VCardParameterValue::Null => { + last_param = None; + } + } + } + + let is_v21_base64 = matches!(version, VCardVersion::V2_1) && legacy_binary.is_some(); + let max_values = if is_v4 { + usize::MAX + } else { + self.name.legacy_max_values() + }; + + if !is_v4 { + if let Some(data) = legacy_binary { + out.write_atomic(if is_v21_base64 { + ";ENCODING=BASE64" + } else { + ";ENCODING=b" + })?; + + if let Some(media_type) = data.content_type.as_deref() + && !self + .params + .iter() + .any(|param| param.name == VCardParameterName::Type) + && let Some(token) = legacy_media_type(self.name.as_str(), media_type) + { + out.write_atomic(";TYPE=")?; + out.write_atomic(&token)?; + } + } + + if self.values.iter().take(max_values).any(|v| match v { + VCardValue::Text(s) => !s.is_ascii(), + VCardValue::Component(items) => items.iter().any(|s| !s.is_ascii()), + _ => false, + }) { + out.write_atomic(";CHARSET=UTF-8")?; + } + } + + out.write_atomic(":")?; + + let (default_type, value_separator) = self.name.default_types(); + let default_type = default_type.unwrap_vcard(); + + let mut separator = ";"; + let mut escape_semicolon = matches!(types.unwrap_or(&default_type), VCardValueType::Text); + let mut escape_comma = escape_semicolon; + + match value_separator { + ValueSeparator::Comma => { + escape_comma = true; + separator = ","; + } + ValueSeparator::Semicolon => escape_semicolon = true, + ValueSeparator::SemicolonAndComma => { + escape_semicolon = true; + escape_comma = true; + } + _ => {} + } + + for (pos, value) in self.values.iter().take(max_values).enumerate() { + if pos > 0 { + out.write_atomic(separator)?; + } + + match value { + VCardValue::Text(v) => { + write_text(out, v, escape_semicolon, escape_comma)?; + } + VCardValue::Component(v) => { + for (pos, item) in v.iter().enumerate() { + if pos > 0 { + out.write_atomic(",")?; + } + write_text(out, item, true, true)?; + } + } + VCardValue::Integer(v) => { + out.push_i64(*v)?; + } + VCardValue::Float(v) => { + write!(out, "{v}")?; + } + VCardValue::Boolean(v) => { + out.write_atomic(if *v { "TRUE" } else { "FALSE" })?; + } + VCardValue::PartialDateTime(v) => { + let typ = if pos == 0 { + types + } else { + self.parameters(&VCardParameterName::Value) + .nth(pos) + .and_then(|v| v.as_value_type()) + .and_then(|v| v.iana().copied()) + } + .unwrap_or(&default_type); + if is_v4 { + v.push_vcard(out, typ)?; + } else { + v.push_legacy_vcard(out, typ)?; + } + } + VCardValue::Binary(v) => { + if is_v4 { + let media_type = v.content_type.as_deref().unwrap_or_default(); + out.write_str("data:")?; + out.write_str(media_type)?; + if escape_semicolon { + out.write_atomic("\\;")?; + } else { + out.write_str(";")?; + } + out.write_atomic("base64\\,")?; + } + out.write_base64(&v.data)?; + } + VCardValue::Sex(v) => { + out.write_atomic(v.as_str())?; + } + VCardValue::GramGender(v) => { + out.write_atomic(v.as_str())?; + } + VCardValue::Kind(v) => { + out.write_atomic(v.as_str())?; + } + } + } + + out.end_line()?; + if is_v21_base64 { + out.end_line()?; + } + Ok(()) + } +} + +impl PartialDateTime { + pub fn format_as_vcard(&self, out: &mut impl Write, fmt: &VCardValueType) -> fmt::Result { + out.write_buffered::(|buf| self.push_vcard(buf, fmt)) + } + + pub fn format_as_legacy_vcard( + &self, + out: &mut impl Write, + fmt: &VCardValueType, + ) -> fmt::Result { + out.write_buffered::(|buf| self.push_legacy_vcard(buf, fmt)) + } + + pub(crate) fn push_vcard(&self, buf: &mut impl AsciiPush, fmt: &VCardValueType) -> fmt::Result { + if matches!(fmt, VCardValueType::Timestamp) { + buf.push_4_digits(self.year.unwrap_or_default())?; + buf.push_2_digits(self.month.unwrap_or_default())?; + buf.push_2_digits(self.day.unwrap_or_default())?; + buf.push_byte(b'T')?; + buf.push_2_digits(self.hour.unwrap_or_default())?; + buf.push_2_digits(self.minute.unwrap_or_default())?; + buf.push_2_digits(self.second.unwrap_or_default())?; + return self.push_timestamp_zone(buf); + } + + let missing_time = self.hour.is_none() && self.minute.is_none() && self.second.is_none(); + let missing_tz = self.tz_hour.is_none(); + + if matches!( + fmt, + VCardValueType::Date | VCardValueType::DateAndOrTime | VCardValueType::DateTime + ) { + match (self.year, self.month, self.day) { + (Some(year), Some(month), Some(day)) => { + buf.push_4_digits(year)?; + buf.push_2_digits(month)?; + buf.push_2_digits(day)?; + } + (Some(year), Some(month), None) => { + buf.push_4_digits(year)?; + if missing_time && missing_tz { + buf.push_byte(b'-')?; + } + buf.push_2_digits(month)?; + } + (None, Some(month), Some(day)) => { + buf.push_ascii(b"--")?; + buf.push_2_digits(month)?; + buf.push_2_digits(day)?; + } + (None, None, Some(day)) => { + buf.push_ascii(b"---")?; + buf.push_2_digits(day)?; + } + (Some(year), None, None) => { + buf.push_4_digits(year)?; + } + (None, Some(month), None) => { + buf.push_ascii(b"--")?; + buf.push_u64(month.into())?; + } + _ => {} + } + } + + if matches!( + fmt, + VCardValueType::DateAndOrTime | VCardValueType::DateTime | VCardValueType::Time + ) && !missing_time + { + if matches!( + fmt, + VCardValueType::DateAndOrTime | VCardValueType::DateTime + ) { + buf.push_byte(b'T')?; + } + let mut last_is_some = false; + for value in [self.hour, self.minute, self.second] { + if let Some(value) = value { + buf.push_2_digits(value)?; + last_is_some = true; + } else if !last_is_some { + buf.push_byte(b'-')?; + } + } + } + + if matches!( + fmt, + VCardValueType::DateAndOrTime + | VCardValueType::DateTime + | VCardValueType::Time + | VCardValueType::UtcOffset + ) { + match (self.tz_hour, self.tz_minute) { + (Some(0), Some(0)) | (Some(0), _) => { + buf.push_byte(b'Z')?; + } + (Some(hour), Some(minute)) => { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(hour)?; + buf.push_2_digits(minute)?; + } + (Some(hour), None) => { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(hour)?; + } + _ => {} + } + } + + Ok(()) + } + + pub(crate) fn push_legacy_vcard( + &self, + buf: &mut impl AsciiPush, + fmt: &VCardValueType, + ) -> fmt::Result { + if matches!(fmt, VCardValueType::Timestamp) { + buf.push_4_digits(self.year.unwrap_or_default())?; + buf.push_2_digits(self.month.unwrap_or_default())?; + buf.push_2_digits(self.day.unwrap_or_default())?; + + if self.hour.is_some() { + buf.push_byte(b'T')?; + buf.push_2_digits(self.hour.unwrap_or_default())?; + buf.push_2_digits(self.minute.unwrap_or_default())?; + buf.push_2_digits(self.second.unwrap_or_default())?; + } + + return self.push_timestamp_zone(buf); + } + + let missing_time = self.hour.is_none() && self.minute.is_none() && self.second.is_none(); + + if matches!( + fmt, + VCardValueType::Date | VCardValueType::DateAndOrTime | VCardValueType::DateTime + ) { + match (self.year, self.month, self.day) { + (Some(year), Some(month), Some(day)) => { + buf.push_4_digits(year)?; + buf.push_byte(b'-')?; + buf.push_2_digits(month)?; + buf.push_byte(b'-')?; + buf.push_2_digits(day)?; + } + (Some(year), Some(month), None) => { + buf.push_4_digits(year)?; + buf.push_byte(b'-')?; + buf.push_2_digits(month)?; + } + (None, Some(month), Some(day)) => { + buf.push_ascii(b"--")?; + buf.push_2_digits(month)?; + buf.push_byte(b'-')?; + buf.push_2_digits(day)?; + } + (None, None, Some(day)) => { + buf.push_ascii(b"---")?; + buf.push_2_digits(day)?; + } + (Some(year), None, None) => { + buf.push_4_digits(year)?; + } + (None, Some(month), None) => { + buf.push_ascii(b"--")?; + buf.push_u64(month.into())?; + } + _ => {} + } + } + + if matches!( + fmt, + VCardValueType::DateAndOrTime | VCardValueType::DateTime | VCardValueType::Time + ) && !missing_time + { + if matches!( + fmt, + VCardValueType::DateAndOrTime | VCardValueType::DateTime + ) { + buf.push_byte(b'T')?; + } + let mut last_is_some = false; + for value in [self.hour, self.minute, self.second] { + if let Some(value) = value { + if last_is_some { + buf.push_byte(b':')?; + } + buf.push_2_digits(value)?; + last_is_some = true; + } else if !last_is_some { + buf.push_byte(b'-')?; + } + } + } + + if matches!( + fmt, + VCardValueType::DateAndOrTime + | VCardValueType::DateTime + | VCardValueType::Time + | VCardValueType::UtcOffset + ) { + match (self.tz_hour, self.tz_minute) { + (Some(0), Some(0)) | (Some(0), _) => { + buf.push_byte(b'Z')?; + } + (Some(hour), Some(minute)) => { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(hour)?; + buf.push_byte(b':')?; + buf.push_2_digits(minute)?; + } + (Some(hour), None) => { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(hour)?; + } + _ => {} + } + } + + Ok(()) + } + + fn push_timestamp_zone(&self, buf: &mut impl AsciiPush) -> fmt::Result { + if let Some(tz_hour) = self.tz_hour { + let tz_minute = self.tz_minute.unwrap_or_default(); + if tz_hour == 0 && tz_minute == 0 { + buf.push_byte(b'Z')?; + } else { + buf.push_byte(if self.tz_minus { b'-' } else { b'+' })?; + buf.push_2_digits(tz_hour)?; + + if let Some(tz_minute) = self.tz_minute { + buf.push_2_digits(tz_minute)?; + } + } + } + Ok(()) + } +} + +impl Display for VCard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_to(f, self.version().unwrap_or_default()) + } +} + +impl Display for VCardVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + VCardVersion::V2_0 => "2.0", + VCardVersion::V2_1 => "2.1", + VCardVersion::V3_0 => "3.0", + VCardVersion::V4_0 => "4.0", + }) + } +} + diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 216060fd..abc1fb0c 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "common" -version = "0.16.27" +version = "1.0.0" edition = "2024" build = "build.rs" @@ -14,34 +14,35 @@ coordinator = { path = "../coordinator" } types = { path = "../types" } registry = { path = "../registry" } jmap_proto = { path = "../jmap-proto" } -sieve-rs = { version = "0.7", features = ["rkyv", "serde"] } -mail-parser = { version = "0.11", features = ["full_encoding"] } -mail-builder = { version = "1.0" } -mail-auth = { version = "0.13", features = ["generate", "arc"] } -smtp-proto = { version = "0.2.3", features = ["rkyv"] } +sieve-rs = { path = "../sieve", features = ["serde"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +mail-builder = { version = "2.0" } +mail-auth = { version = "0.14", features = ["generate", "arc"] } +smtp-report = { path = "../smtp-report" } +smtp-proto = { path = "../smtp-proto", features = ["rkyv"] } dns-update = { version = "0.6" } -calcard = { version = "0.3", features = ["rkyv"] } +calcard = { path = "../calcard", features = ["rkyv"] } ahash = { version = "0.8.12", features = ["serde"] } -parking_lot = "0.12.5" -regex = "1.13.1" -proxy-header = { version = "0.1.2", features = ["tokio"] } -arc-swap = "1.9.2" -rustls = { version = "0.23.43", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +parking_lot = "0.13.0" +proxy-header = { version = "0.1.3", features = ["tokio"] } +arc-swap = "1.9.3" +rustls = { version = "0.23.46", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } rustls-pemfile = "2.2" rustls-pki-types = { version = "1" } aws-lc-rs = { version = "1" } -tokio = { version = "1.53", features = ["net", "macros"] } +tokio = { version = "1.54", features = ["net", "macros", "sync", "time"] } tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12"] } +socket2 = { version = "0.6", features = ["all"] } futures = "0.3" rcgen = { version = "0.14", default-features = false, features = ["crypto", "pem", "aws_lc_rs"] } reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "stream"]} serde = { version = "1.0", features = ["derive"]} -serde_json = "1.0" -base64 = "0.23" +serde_json = { version = "1.0", features = ["raw_value"] } +encodify = "1.0" x509-parser = "0.18" pem = "4.0" -chrono = { version = "0.4", features = ["serde"] } -hyper = { version = "1.11.1", features = ["server", "http1", "http2"] } +jiff = "0.2.38" +hyper = { version = "1.12.0", features = ["server", "http1", "http2"] } opentelemetry = { git = "https://github.com/stalwartlabs/opentelemetry-rust" } opentelemetry_sdk = { git = "https://github.com/stalwartlabs/opentelemetry-rust" } opentelemetry-otlp = { git = "https://github.com/stalwartlabs/opentelemetry-rust", default-features = false, features = ["reqwest-client", "http-proto", "trace", "metrics", "logs", "internal-logs", "grpc-tonic", "tls-aws-lc", "tls-roots", "reqwest-rustls"] } @@ -50,55 +51,64 @@ prometheus = { version = "0.14", default-features = false } imagesize = "0.15" sha1 = "0.11" sha2 = "0.11" +siphasher = "1" md5 = "0.8.1" whatlang = "0.18" idna = "1.1" -decancer = "3.3.3" +decancer = "4.0.0" unicode-security = "0.1.2" infer = "0.22" bincode = { version = "2.0.1", features = ["serde"] } hostname = "0.4.2" -zip = "8.6" -xxhash-rust = { version = "0.8.18", features = ["xxh3"] } +zip = "9.1" +xxhash-rust = { version = "0.8.19", features = ["xxh3"] } psl = "2" aes-gcm-siv = "0.12.1" -jsonwebtoken = { version = "11.0.0", features = ["aws_lc_rs"] } +jsonwebtoken = { version = "11.1.0", features = ["aws_lc_rs"] } rsa = "0.9.10" -p256 = { version = "0.13", features = ["ecdh", "ecdsa", "pkcs8", "pem"] } -p384 = { version = "0.13", features = ["ecdh"] } -hashify = "0.2" -rkyv = { version = "0.8.18", features = ["little_endian"] } -tinyvec = { version = "1.12.0", features = ["alloc"] } +p256 = { version = "0.14", features = ["ecdh", "ecdsa", "pkcs8", "pem"] } +p384 = { version = "0.14", features = ["ecdh"] } +hashify = "0.3" +rkyv.workspace = true +tinyvec = { version = "1.13.3", features = ["alloc"] } compact_str = { version = "0.10.0", features = ["rkyv", "serde"] } +bytes = "1" lz4_flex = { version = "0.14", features = ["frame"], default-features = false } hickory-proto = "0.26.3" nohash-hasher = "0.2.0" quick_cache = "0.7" -rasn = "0.28" -rasn-pkix = "0.28" +rasn = "0.29" +rasn-pkix = "0.29" sequoia-openpgp = { version = "2.4", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } zxcvbn = "3.1.1" -pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } -quick-xml = "0.41" +pkcs8 = { version = "0.11.0", features = ["alloc", "std"] } +quick-xml = "0.42" +percent-encoding = "2.3.2" +bumpalo = { version = "3.20.3", features = ["collections"] } +regex-automata = { version = "0.4.18", default-features = false, features = ["std", "syntax", "meta", "nfa-pikevm", "nfa-backtrack", "hybrid", "dfa-onepass", "perf-inline", "perf-literal", "unicode"] } +memchr = "2.8.3" +itoa = "1.0.18" +zmij = "1.0.23" [target.'cfg(unix)'.dependencies] -privdrop = "0.5.6" -libc = "0.2.189" +privdrop = "0.5.7" +libc = "0.2.191" [target.'cfg(target_os = "linux")'.dependencies] sd-notify = "0.5" -[target.'cfg(windows)'.dependencies] -socket2 = "0.6" +[target.'cfg(stalwart_tests)'.dependencies] +dns-update = { version = "0.6", features = ["test_provider"] } [features] -test_mode = [] dev_mode = [] -enterprise = [] +enterprise = ["store/enterprise", "utils/enterprise", "trc/enterprise", "directory/enterprise", "coordinator/enterprise", "registry/enterprise", "jmap_proto/enterprise"] foundation = [] +redis = ["store/redis"] [dev-dependencies] -tokio = { version = "1.53", features = ["full"] } +tokio = { version = "1.54", features = ["full"] } +regex = "1.13.1" [lints] workspace = true diff --git a/crates/common/build.rs b/crates/common/build.rs index c62f67d4..341e7a0d 100644 --- a/crates/common/build.rs +++ b/crates/common/build.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::env; use std::fs; use std::path::Path; @@ -23,8 +23,8 @@ fn main() { println!("cargo:rerun-if-changed={}", yaml_path.display()); } -fn parse_yaml(content: &str) -> HashMap> { - let mut result: HashMap> = HashMap::new(); +fn parse_yaml(content: &str) -> BTreeMap> { + let mut result: BTreeMap> = BTreeMap::new(); let mut current_key = None; for line in content.lines() { @@ -78,7 +78,7 @@ fn split_plural_forms(value: &str) -> Option> { .collect() } -fn plural_keys(locales: &HashMap>) -> HashSet { +fn plural_keys(locales: &BTreeMap>) -> HashSet { let mut keys = HashSet::new(); for (key, translations) in locales { @@ -132,7 +132,7 @@ fn plural_forms_literal(value: &str) -> String { literal } -fn generate_locale_code(locales: &HashMap>) -> String { +fn generate_locale_code(locales: &BTreeMap>) -> String { let mut code = String::new(); let plural = plural_keys(locales); @@ -159,7 +159,7 @@ fn generate_locale_code(locales: &HashMap>) -> S code.push_str("}\n\n"); - let mut languages = std::collections::HashSet::new(); + let mut languages = BTreeSet::new(); for translations in locales.values() { for lang in translations.keys() { languages.insert(lang.clone()); @@ -188,19 +188,17 @@ fn generate_locale_code(locales: &HashMap>) -> S code.push_str("};\n\n"); } - let mut sorted: Vec<&String> = languages.iter().collect(); - sorted.sort_unstable(); code.push_str(&format!( "pub static ALL_LOCALES: [&Locale; {}] = [\n", - sorted.len() + languages.len() )); - for lang in &sorted { + for lang in &languages { code.push_str(&format!(" &{}_LOCALES,\n", const_name(lang))); } code.push_str("];\n\n"); code.push_str("pub fn locale(name: &str) -> Option<&'static Locale> {\n"); - code.push_str(" hashify::tiny_map_ignore_case!(name.as_bytes(),\n"); + code.push_str(" hashify::map_ignore_case!(name.as_bytes(), &'static Locale,\n"); for lang in &languages { code.push_str(&format!( " \"{}\" => &{}_LOCALES,\n", @@ -208,7 +206,7 @@ fn generate_locale_code(locales: &HashMap>) -> S const_name(lang) )); } - code.push_str(" )\n"); + code.push_str(" )\n .copied()\n"); code.push_str("}\n\n"); // Maps a bare language tag onto the regional locale shipped for it @@ -220,7 +218,7 @@ fn generate_locale_code(locales: &HashMap>) -> S by_language.dedup_by_key(|(language, _)| *language); code.push_str("pub fn locale_by_language(language: &str) -> Option<&'static Locale> {\n"); - code.push_str(" hashify::tiny_map_ignore_case!(language.as_bytes(),\n"); + code.push_str(" hashify::map_ignore_case!(language.as_bytes(), &'static Locale,\n"); for (language, lang) in by_language { code.push_str(&format!( " \"{}\" => &{}_LOCALES,\n", @@ -228,7 +226,7 @@ fn generate_locale_code(locales: &HashMap>) -> S const_name(lang) )); } - code.push_str(" )\n"); + code.push_str(" )\n .copied()\n"); code.push_str("}\n"); code } diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index 4c6ce2d4..731abe65 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -8,28 +8,37 @@ use super::AccessToken; use crate::{ Server, auth::{ - AccessScope, AccessTo, AccessTokenInner, AccountTenantIds, Permissions, RECOVERY_ADMIN_ID, + AccessScope, AccessTo, AccessTokenInner, AccountCache, ChangedBy, GrantClient, Permissions, + RECOVERY_ADMIN_ID, TokenScope, + oauth::scope::{OAuthScopes, scope_data_types}, permissions::{BuildPermissions, PermissionsListBuilder}, }, - network::limiter::{ConcurrencyLimiter, LimiterResult}, + network::limiter::{AccountLimit, AccountLimits, AccountRate, LimiterResult, RateCounter}, }; use ahash::AHasher; +use compact_str::format_compact; use registry::{ schema::{ enums::Permission, structs::{self, Account, Roles, UserRoles}, }, - types::EnumImpl, + types::{EnumImpl, ipmask::IpAddrOrMask}, }; use std::{ + borrow::Cow, hash::{Hash, Hasher}, net::IpAddr, sync::Arc, }; -use store::{query::acl::AclQuery, rand, write::now}; +use store::{ + Deserialize, IterateParams, ValueKey, + query::acl::{AclItem, AclQuery}, + rand, + write::{ValueClass, key::DeserializeBigEndian, now}, +}; use tinyvec::TinyVec; -use trc::{AddContext, StoreEvent}; -use types::{acl::Acl, collection::Collection}; +use trc::{AddContext, CacheEvent}; +use types::{acl::Acl, collection::Collection, type_state::DataType}; use utils::map::bitmap::{Bitmap, BitmapItem}; use xxhash_rust::xxh3; @@ -43,83 +52,28 @@ impl Server { ) -> trc::Result { match account { Account::User(account) => { - let tenant_id = account.member_tenant_id.map(|t| t.id() as u32); + let role_ids = match &account.roles { + UserRoles::User => self.core.network.security.default_role_ids_user.as_slice(), + UserRoles::Admin => { + self.core.network.security.default_role_ids_admin.as_slice() + } + UserRoles::Custom(custom_roles) => custom_roles.role_ids.as_slice(), + }; + #[cfg(not(feature = "enterprise"))] let permissions = self - .effective_permissions( - &account.permissions, - match &account.roles { - UserRoles::User => { - self.core.network.security.default_role_ids_user.as_slice() - } - UserRoles::Admin => { - if tenant_id.is_none() { - self.core.network.security.default_role_ids_admin.as_slice() - } else { - self.core - .network - .security - .default_role_ids_tenant - .as_slice() - } - } - UserRoles::Custom(custom_roles) => custom_roles.role_ids.as_slice(), - }, - tenant_id, - ) + .effective_permissions(&account.permissions, role_ids) .await?; let member_of = account .member_group_ids .iter() .map(|m| m.id() as u32) - .collect::>(); - let mut access_to: Vec = Vec::new(); - for grant_account_id in [account_id].into_iter().chain(member_of.iter().copied()) { - for acl_item in self - .store() - .acl_query(AclQuery::HasAccess { grant_account_id }) - .await - .caused_by(trc::location!())? - { - if acl_item.to_account_id != account_id - && !member_of.contains(&acl_item.to_account_id) - { - let acl = Bitmap::::from(acl_item.permissions); - let collection = acl_item.to_collection; - if !collection.is_valid() { - return Err(trc::StoreEvent::DataCorruption - .ctx(trc::Key::Reason, "Corrupted collection found in ACL key.") - .details(format!("{acl_item:?}")) - .account_id(grant_account_id) - .caused_by(trc::location!())); - } - - let mut collections: Bitmap = Bitmap::new(); - if acl.contains(Acl::Read) { - collections.insert(collection); - } - if acl.contains(Acl::ReadItems) - && let Some(child_col) = collection.child_collection() - { - collections.insert(child_col); - } - - if !collections.is_empty() { - if let Some(idx) = access_to - .iter() - .position(|a| a.account_id == acl_item.to_account_id) - { - access_to[idx].collections.union(&collections); - } else { - access_to.push(AccessTo { - account_id: acl_item.to_account_id, - collections, - }); - } - } - } - } - } + .collect::>(); + let access_to = AccessTo::from_acl_items( + account_id, + &member_of, + self.acl_items_by_grantee(account_id, &member_of).await?, + )?; let now = now(); let mut credential_version = 0; @@ -184,29 +138,14 @@ impl Server { } Ok(AccessTokenInner { - concurrent_imap_requests: self - .core - .imap - .rate_concurrent - .map(ConcurrencyLimiter::new), - concurrent_http_requests: self - .core - .jmap - .request_max_concurrent - .map(ConcurrencyLimiter::new), - concurrent_uploads: self - .core - .jmap - .upload_max_concurrent - .map(ConcurrencyLimiter::new), + limits: self.account_limits(), obj_size: 0, revision, revision_account, credential_version, account_id, - tenant_id, member_of, - access_to: access_to.into_boxed_slice(), + access_to, scopes: [] .into_iter() .chain(credential_scopes) @@ -215,39 +154,22 @@ impl Server { .update_size()) } Account::Group(account) => { - let tenant_id = account.member_tenant_id.map(|t| t.id() as u32); + let role_ids = account + .roles + .role_ids() + .unwrap_or(self.core.network.security.default_role_ids_group.as_slice()); + #[cfg(not(feature = "enterprise"))] let permissions = self - .effective_permissions( - &account.permissions, - account.roles.role_ids().unwrap_or( - self.core.network.security.default_role_ids_group.as_slice(), - ), - tenant_id, - ) + .effective_permissions(&account.permissions, role_ids) .await?; Ok(AccessTokenInner { - concurrent_imap_requests: self - .core - .imap - .rate_concurrent - .map(ConcurrencyLimiter::new), - concurrent_http_requests: self - .core - .jmap - .request_max_concurrent - .map(ConcurrencyLimiter::new), - concurrent_uploads: self - .core - .jmap - .upload_max_concurrent - .map(ConcurrencyLimiter::new), + limits: self.account_limits(), obj_size: 0, revision, revision_account, credential_version: 0, account_id, - tenant_id, member_of: Default::default(), access_to: Default::default(), scopes: Box::new([AccessScope::new(permissions.finalize(), u32::MAX)]), @@ -257,6 +179,79 @@ impl Server { } } + async fn acl_items_by_grantee( + &self, + account_id: u32, + member_of: &[u32], + ) -> trc::Result> { + if member_of.is_empty() { + return self + .store() + .acl_query(AclQuery::HasAccess { + grant_account_id: account_id, + }) + .await + .caused_by(trc::location!()) + .map(|items| items.into_iter().map(|item| (account_id, item)).collect()); + } + + let grantees = [account_id] + .into_iter() + .chain(member_of.iter().copied()) + .collect::>(); + let mut ordered = grantees.clone(); + ordered.sort_unstable(); + let mut previous = None; + ordered.retain(|id| previous.replace(*id) != Some(*id)); + let mut items = Vec::new(); + self.store() + .iterate_many( + ordered + .iter() + .map(|&grant_account_id| { + IterateParams::new( + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Acl(grant_account_id), + }, + ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Acl(grant_account_id), + }, + ) + .ascending() + }) + .collect(), + |key, value| { + let grant_account_id = key.deserialize_be_u32(0)?; + let mut item = AclItem::deserialize(key)?; + item.permissions = u64::deserialize(value)?; + items.push((grant_account_id, item)); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + items.sort_unstable_by_key(|(grant_account_id, item)| { + ( + grantees + .iter() + .position(|id| id == grant_account_id) + .unwrap_or(usize::MAX), + item.to_account_id, + u8::from(item.to_collection), + item.to_document_id, + ) + }); + + Ok(items) + } + pub async fn access_token(&self, account_id: u32) -> trc::Result> { match self .inner @@ -267,18 +262,18 @@ impl Server { { Ok(token) => { trc::event!( - Store(StoreEvent::CacheHit), + Cache(CacheEvent::Hit), Key = account_id, - Collection = "accessToken", + Name = "accessToken", ); Ok(token) } Err(guard) => { trc::event!( - Store(StoreEvent::CacheMiss), + Cache(CacheEvent::Miss), Key = account_id, - Collection = "accessToken", + Name = "accessToken", ); let token: Arc = if let Some(account) = @@ -299,7 +294,7 @@ impl Server { .caused_by(trc::location!())); }; - let _ = guard.insert(token.clone()); + guard.insert(token.clone()); Ok(token) } } @@ -321,18 +316,18 @@ impl Server { Ok(token) => { if token.revision_account == revision_account { trc::event!( - Store(StoreEvent::CacheHit), + Cache(CacheEvent::Hit), Key = account_id, - Collection = "accessToken", + Name = "accessToken", ); Ok(token) } else { // Token is stale, rebuild it trc::event!( - Store(StoreEvent::CacheStale), + Cache(CacheEvent::Stale), Key = account_id, - Collection = "accessToken", + Name = "accessToken", ); debug_assert!( @@ -353,9 +348,9 @@ impl Server { } Err(guard) => { trc::event!( - Store(StoreEvent::CacheMiss), + Cache(CacheEvent::Miss), Key = account_id, - Collection = "accessToken", + Name = "accessToken", ); let revision = rand::random::(); @@ -363,7 +358,7 @@ impl Server { .build_access_token(account, account_id, revision, revision_account) .await? .into(); - let _ = guard.insert(token.clone()); + guard.insert(token.clone()); Ok(token) } } @@ -375,6 +370,7 @@ impl AccessToken { AccessToken { scope_idx: 0, inner, + token_scope: None, } .assert_is_valid(remote_ip) } @@ -383,6 +379,7 @@ impl AccessToken { AccessToken { scope_idx: 0, inner, + token_scope: None, } } @@ -402,26 +399,87 @@ impl AccessToken { .ctx(trc::Key::Id, credential_id) .reason("Credential expired or removed.") }) - .map(|scope_idx| AccessToken { scope_idx, inner }) + .map(|scope_idx| AccessToken { + scope_idx, + inner, + token_scope: None, + }) .and_then(|token| token.assert_is_valid(remote_ip)) } pub fn renew( inner: Arc, credential_id: Option, + token_scope: Option, remote_ip: IpAddr, ) -> trc::Result { - if let Some(credential_id) = credential_id { + if let Some(credential_id) = credential_id.filter(|id| *id != u32::MAX) { Self::new_scoped(inner, credential_id, remote_ip) } else { AccessToken { scope_idx: 0, inner, + token_scope, } .assert_is_valid(remote_ip) } } + pub fn is_built_from(&self, inner: &Arc) -> bool { + Arc::ptr_eq(&self.inner, inner) + } + + pub fn with_token_scope(mut self, token_scope: Option) -> Self { + self.token_scope = token_scope; + self + } + + #[inline(always)] + pub fn token_scope(&self) -> Option { + self.token_scope + } + + #[inline(always)] + pub fn grant_id(&self) -> Option { + self.token_scope.and_then(TokenScope::grant_id) + } + + pub fn oauth_scopes(&self) -> Option { + self.token_scope.and_then(TokenScope::restricted_scopes) + } + + pub fn allows_oauth_scopes(&self, required: OAuthScopes) -> bool { + self.oauth_scopes() + .is_none_or(|scopes| scopes.intersects(required)) + } + + pub fn assert_allows_oauth_scopes(self, required: OAuthScopes) -> trc::Result { + if self.allows_oauth_scopes(required) { + Ok(self) + } else { + Err(trc::SecurityEvent::Unauthorized + .into_err() + .details("The token scope does not allow this request") + .account_id(self.account_id())) + } + } + + pub fn restrict_data_types(&self, types: &mut Bitmap) { + if let Some(scopes) = self.oauth_scopes() { + types.intersection(&scope_data_types(scopes)); + } + } + + #[inline(always)] + pub fn credential_version(&self) -> u64 { + self.inner.credential_version + } + + #[inline(always)] + pub fn is_main_credential_expired(&self) -> bool { + self.inner.is_main_credential_expired() + } + pub fn state(&self) -> u32 { // Hash state let mut s = AHasher::default(); @@ -435,10 +493,6 @@ impl AccessToken { self.inner.account_id } - #[inline(always)] - pub fn tenant_id(&self) -> Option { - self.inner.tenant_id - } pub fn secondary_ids(&self) -> impl Iterator { self.inner @@ -487,10 +541,9 @@ impl AccessToken { let child_collection = collection.child_collection(); if self.is_account_id(account_id) || self.inner.member_of.contains(&account_id) - || self.inner.access_to.iter().any(|a| { - a.account_id == account_id - && (a.collections.contains(collection) - || child_collection.is_some_and(|child| a.collections.contains(child))) + || self.inner.shared_access(account_id).is_some_and(|a| { + a.collections.contains(collection) + || child_collection.is_some_and(|child| a.collections.contains(child)) }) { self.inner.account_id @@ -505,17 +558,27 @@ impl AccessToken { .scopes .get(self.scope_idx) .is_some_and(|scope| scope.permissions.get(permission as usize)) + && self + .token_scope + .and_then(TokenScope::permissions) + .is_none_or(|permissions| permissions.get(permission as usize)) } pub fn assert_is_valid(self, remote_ip: IpAddr) -> trc::Result { if let Some(scope) = self.inner.scopes.get(self.scope_idx) { - let has_expired = scope.expires_at <= now(); - let is_valid_ip = scope.allowed_ips.is_empty() - || scope + if !scope.allowed_ips.is_empty() + && !scope .allowed_ips .iter() - .any(|ip_mask| ip_mask.matches(&remote_ip)); + .any(|ip_mask| ip_mask.matches(&remote_ip)) + { + return Err(trc::SecurityEvent::IpUnauthorized + .into_err() + .ctx(trc::Key::AccountId, self.inner.account_id) + .reason("IP address not allowed.")); + } + let has_expired = scope.expires_at <= now(); let mut access_token = self; if has_expired { if access_token.scope_idx > 0 { @@ -523,6 +586,17 @@ impl AccessToken { .into_err() .ctx(trc::Key::AccountId, access_token.inner.account_id) .reason("Credential expired.")); + } else if matches!( + access_token.token_scope, + Some(TokenScope::Grant { + client: GrantClient::ThirdParty, + .. + }) + ) { + return Err(trc::AuthEvent::CredentialExpired + .into_err() + .ctx(trc::Key::AccountId, access_token.inner.account_id) + .reason("Main credential expired, third-party grant refused.")); } else { trc::event!( Auth(trc::AuthEvent::CredentialExpired), @@ -552,7 +626,7 @@ impl AccessToken { scopes.push(AccessScope { permissions, credential_id: scope.credential_id, - expires_at: u64::MAX, + expires_at: scope.expires_at, allowed_ips: scope.allowed_ips.clone(), }); } else { @@ -563,12 +637,9 @@ impl AccessToken { let inner = AccessTokenInner { scopes: scopes.into_boxed_slice(), account_id: old_inner.account_id, - tenant_id: old_inner.tenant_id, member_of: old_inner.member_of.clone(), access_to: old_inner.access_to.clone(), - concurrent_http_requests: old_inner.concurrent_http_requests.clone(), - concurrent_imap_requests: old_inner.concurrent_imap_requests.clone(), - concurrent_uploads: old_inner.concurrent_uploads.clone(), + limits: old_inner.limits.clone(), revision_account: old_inner.revision_account, revision: old_inner.revision, credential_version: old_inner.credential_version, @@ -578,17 +649,11 @@ impl AccessToken { access_token = AccessToken { scope_idx: access_token.scope_idx, inner: Arc::new(inner), + token_scope: access_token.token_scope, }; } - if is_valid_ip { - Ok(access_token) - } else { - Err(trc::SecurityEvent::IpUnauthorized - .into_err() - .ctx(trc::Key::AccountId, access_token.inner.account_id) - .reason("IP address not allowed.")) - } + Ok(access_token) } else { Err(trc::SecurityEvent::Unauthorized .into_err() @@ -646,11 +711,7 @@ impl AccessToken { } pub fn permissions(&self) -> Vec { - if let Some(scope) = self.inner.scopes.get(self.scope_idx) { - scope.permissions.build_permissions_list() - } else { - vec![] - } + self.effective_permissions().build_permissions_list() } #[inline(always)] @@ -658,13 +719,18 @@ impl AccessToken { self.inner.scopes.get(self.scope_idx) } - pub(crate) fn permissions_bits(&self) -> &Permissions { - &self - .inner - .scopes - .get(self.scope_idx) - .unwrap_or(&self.inner.scopes[0]) - .permissions + pub fn effective_permissions(&self) -> Cow<'_, Permissions> { + let Some(scope) = self.inner.scopes.get(self.scope_idx) else { + return Cow::Owned(Permissions::new()); + }; + match self.token_scope.and_then(TokenScope::permissions) { + Some(filter) => { + let mut permissions = scope.permissions.clone(); + permissions.intersection(filter); + Cow::Owned(permissions) + } + None => Cow::Borrowed(&scope.permissions), + } } pub fn account_permissions(&self) -> &Permissions { @@ -672,12 +738,7 @@ impl AccessToken { } pub fn is_shared(&self, account_id: u32) -> bool { - !self.is_member(account_id) - && self - .inner - .access_to - .iter() - .any(|a| a.account_id == account_id) + !self.is_member(account_id) && self.inner.shared_access(account_id).is_some() } pub fn shared_accounts(&self, collection: Collection) -> impl Iterator { @@ -698,61 +759,45 @@ impl AccessToken { self.is_member(to_account_id) || self .inner - .access_to - .iter() - .any(|a| a.account_id == to_account_id && a.collections.contains(to_collection)) + .shared_access(to_account_id) + .is_some_and(|a| a.collections.contains(to_collection)) + } + + pub fn is_read_only(&self, to_account_id: u32) -> bool { + !self.is_member(to_account_id) + && !self + .inner + .shared_access(to_account_id) + .is_some_and(|a| a.is_writable) } pub fn has_account_access(&self, to_account_id: u32) -> bool { - self.is_member(to_account_id) - || self - .inner - .access_to - .iter() - .any(|a| a.account_id == to_account_id) + self.is_member(to_account_id) || self.inner.shared_access(to_account_id).is_some() } pub fn is_http_request_allowed(&self) -> LimiterResult { - self.inner - .concurrent_http_requests - .as_ref() - .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) + self.inner.is_allowed(AccountLimit::HttpRequests) } pub fn concurrent_http_requests(&self) -> u64 { - self.inner - .concurrent_http_requests - .as_ref() - .map(|limiter| limiter.max_concurrent()) - .unwrap_or(0) + self.inner.max_concurrent(AccountLimit::HttpRequests) } pub fn is_imap_request_allowed(&self) -> LimiterResult { - self.inner - .concurrent_imap_requests - .as_ref() - .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) + self.inner.is_allowed(AccountLimit::ImapRequests) } pub fn is_upload_allowed(&self) -> LimiterResult { - self.inner - .concurrent_uploads - .as_ref() - .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) + self.inner.is_allowed(AccountLimit::Uploads) } pub fn concurrent_uploads(&self) -> u64 { - self.inner - .concurrent_uploads - .as_ref() - .map(|limiter| limiter.max_concurrent()) - .unwrap_or(0) + self.inner.max_concurrent(AccountLimit::Uploads) } - pub fn account_tenant_ids(&self) -> AccountTenantIds { - AccountTenantIds { + pub fn changed_by(&self) -> ChangedBy { + ChangedBy { account_id: self.account_id(), - tenant_id: self.tenant_id(), } } @@ -760,6 +805,7 @@ impl AccessToken { AccessToken { scope_idx: 0, inner: Arc::new(AccessTokenInner::new_admin()), + token_scope: None, } } @@ -773,15 +819,13 @@ impl AccessToken { } AccessToken { scope_idx: 0, + token_scope: None, inner: Arc::new(AccessTokenInner { account_id, - tenant_id: Default::default(), member_of: Default::default(), access_to: Default::default(), scopes: Box::new([AccessScope::new(permissions, u32::MAX)]), - concurrent_http_requests: Default::default(), - concurrent_imap_requests: Default::default(), - concurrent_uploads: Default::default(), + limits: Default::default(), revision: Default::default(), revision_account: Default::default(), credential_version: Default::default(), @@ -793,6 +837,14 @@ impl AccessToken { pub fn from_id_maybe_invalid(account_id: u32) -> Self { AccessToken::new_maybe_invalid(Arc::new(AccessTokenInner::from_id(account_id))) } + + pub fn from_account_maybe_invalid(account: &AccountCache) -> Self { + AccessToken::new_maybe_invalid(Arc::new(AccessTokenInner { + account_id: account.id, + member_of: account.id_member_of.clone(), + ..Default::default() + })) + } } impl AccessTokenInner { @@ -803,30 +855,60 @@ impl AccessTokenInner { } } - pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { - self.tenant_id = tenant_id; + + pub fn update_size(mut self) -> Self { + let size = std::mem::size_of::() + + std::mem::size_of_val::<[u32]>(&self.member_of) + + std::mem::size_of_val::<[AccessTo]>(&self.access_to) + + std::mem::size_of_val::<[AccessScope]>(&self.scopes) + + self + .scopes + .iter() + .map(|scope| std::mem::size_of_val::<[IpAddrOrMask]>(&scope.allowed_ips)) + .sum::() + + self + .limits + .as_ref() + .map_or(0, |_| std::mem::size_of::()); + self.obj_size = u32::try_from(size).unwrap_or(u32::MAX); self } - pub fn update_size(mut self) -> Self { - self.obj_size = (std::mem::size_of::() - + (self.member_of.len() * std::mem::size_of::()) - + (self.access_to.len() * (std::mem::size_of::() + std::mem::size_of::())) - + (self.scopes.len() * std::mem::size_of::())) - as u64; - self + fn shared_access(&self, account_id: u32) -> Option<&AccessTo> { + self.access_to + .binary_search_by_key(&account_id, |a| a.account_id) + .ok() + .and_then(|idx| self.access_to.get(idx)) + } + + #[inline] + fn is_allowed(&self, limit: AccountLimit) -> LimiterResult { + self.limits + .as_ref() + .map_or(LimiterResult::Disabled, |limits| limits.is_allowed(limit)) + } + + #[inline] + fn max_concurrent(&self, limit: AccountLimit) -> u64 { + self.limits + .as_ref() + .map_or(0, |limits| limits.max_concurrent(limit)) + } + + #[inline] + pub(crate) fn rate_counter(&self, rate: AccountRate) -> Option<&RateCounter> { + self.limits + .as_ref() + .and_then(|limits| limits.rate_counter(rate)) } pub fn new_admin() -> Self { AccessTokenInner { account_id: RECOVERY_ADMIN_ID, - tenant_id: Default::default(), member_of: Default::default(), access_to: Default::default(), scopes: Box::new([AccessScope::new(Permissions::all(), u32::MAX)]), - concurrent_http_requests: Default::default(), - concurrent_imap_requests: Default::default(), - concurrent_uploads: Default::default(), + limits: Default::default(), revision: Default::default(), revision_account: Default::default(), credential_version: Default::default(), @@ -845,6 +927,82 @@ impl AccessTokenInner { pub fn credential_version(&self) -> u64 { self.credential_version } + + pub fn is_main_credential_expired(&self) -> bool { + self.scopes + .first() + .is_some_and(|scope| scope.expires_at <= now()) + } +} + +impl AccessTo { + fn from_acl_items( + account_id: u32, + member_of: &[u32], + items: Vec<(u32, AclItem)>, + ) -> trc::Result> { + let mut access_to: Vec = Vec::new(); + let mut is_sorted = true; + for (grant_account_id, acl_item) in items { + if acl_item.to_account_id == account_id || member_of.contains(&acl_item.to_account_id) { + continue; + } + + let acl = Bitmap::::from(acl_item.permissions); + let collection = acl_item.to_collection; + if !collection.is_valid() { + return Err(trc::StoreEvent::DataCorruption + .ctx(trc::Key::Reason, "Corrupted collection found in ACL key.") + .details(format_compact!("{acl_item:?}")) + .account_id(grant_account_id) + .caused_by(trc::location!())); + } + + let mut collections: Bitmap = Bitmap::new(); + if acl.contains(Acl::Read) { + collections.insert(collection); + } + if acl.contains(Acl::ReadItems) + && let Some(child_col) = collection.child_collection() + { + collections.insert(child_col); + } + + if !collections.is_empty() { + let is_writable = acl.intersects(&Acl::WRITE); + match access_to.last_mut() { + Some(last) if last.account_id == acl_item.to_account_id => { + last.collections.union(&collections); + last.is_writable |= is_writable; + } + last => { + is_sorted &= + last.is_none_or(|last| last.account_id < acl_item.to_account_id); + access_to.push(AccessTo { + account_id: acl_item.to_account_id, + collections, + is_writable, + }); + } + } + } + } + + if !is_sorted { + access_to.sort_unstable_by_key(|a| a.account_id); + access_to.dedup_by(|next, kept| { + if next.account_id == kept.account_id { + kept.collections.union(&next.collections); + kept.is_writable |= next.is_writable; + true + } else { + false + } + }); + } + + Ok(access_to.into_boxed_slice()) + } } impl AccessScope { @@ -863,7 +1021,6 @@ fn hash_account(account: &Account) -> u64 { match account { Account::User(account) => { - account.member_tenant_id.hash(&mut s); match &account.roles { UserRoles::User => { 0u8.hash(&mut s); @@ -891,7 +1048,6 @@ fn hash_account(account: &Account) -> u64 { } } Account::Group(account) => { - account.member_tenant_id.hash(&mut s); match &account.roles { Roles::Default => {} Roles::Custom(custom_roles) => { @@ -938,3 +1094,4 @@ fn hash_credential_permissions(hasher: &mut AHasher, permissions: &structs::Cred } } } + diff --git a/crates/common/src/auth/active_script.rs b/crates/common/src/auth/active_script.rs new file mode 100644 index 00000000..0a1b90a8 --- /dev/null +++ b/crates/common/src/auth/active_script.rs @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::atomic::{AtomicU64, Ordering}; + +const VALUE_MASK: u64 = u32::MAX as u64; +const GENERATION: u64 = 1 << u32::BITS; +const UNKNOWN: u32 = u32::MAX; +const INACTIVE: u32 = u32::MAX - 1; + +#[derive(Debug)] +pub struct ActiveScriptSlot(AtomicU64); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CachedActiveScript { + Known(Option), + Unknown(ActiveScriptTicket), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ActiveScriptTicket(u64); + +impl ActiveScriptSlot { + pub fn get(&self) -> CachedActiveScript { + let state = self.0.load(Ordering::Relaxed); + match (state & VALUE_MASK) as u32 { + UNKNOWN => CachedActiveScript::Unknown(ActiveScriptTicket(state)), + INACTIVE => CachedActiveScript::Known(None), + document_id => CachedActiveScript::Known(Some(document_id)), + } + } + + pub fn fill(&self, ticket: ActiveScriptTicket, document_id: Option) { + let value = match document_id { + None => INACTIVE, + Some(document_id) if document_id < INACTIVE => document_id, + Some(_) => return, + }; + let _ = self.0.compare_exchange( + ticket.0, + (ticket.0 & !VALUE_MASK) | u64::from(value), + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + pub fn invalidate(&self) { + self.0 + .update(Ordering::Relaxed, Ordering::Relaxed, |state| { + (state & !VALUE_MASK).wrapping_add(GENERATION) | u64::from(UNKNOWN) + }); + } +} + +impl Default for ActiveScriptSlot { + fn default() -> Self { + ActiveScriptSlot(AtomicU64::new(u64::from(UNKNOWN))) + } +} + +impl Clone for ActiveScriptSlot { + fn clone(&self) -> Self { + ActiveScriptSlot::default() + } +} + diff --git a/crates/common/src/auth/authentication.rs b/crates/common/src/auth/authentication.rs index 9d81f989..41b13d5c 100644 --- a/crates/common/src/auth/authentication.rs +++ b/crates/common/src/auth/authentication.rs @@ -7,16 +7,22 @@ use crate::{ Server, auth::{ - AccessToken, AuthRequest, DomainCache, + AccessToken, AuthRequest, DomainCache, TokenScope, credential::{ApiKey, AppPassword}, - oauth::{GrantType, token::TOKEN_HEADER}, + oauth::{ + GrantType, + scope::OAuthScopes, + token::{TOKEN_HEADER, assert_credential_version}, + }, }, }; -use base64::{Engine, engine::general_purpose}; +use compact_str::CompactString; use directory::{ Credentials, Directory, Recipient, + backend::oidc::{ExternalToken, TokenAccess, TokenContext}, core::secret::{SecretVerificationResult, verify_mfa_secret_hash, verify_secret_hash}, }; +use encodify::base64::URL_SAFE_NO_PAD; use registry::schema::{ enums::Permission, structs::{self, Credential}, @@ -26,6 +32,11 @@ use std::{borrow::Cow, net::IpAddr, sync::Arc}; use store::write::now; use trc::AddContext; +pub struct Authenticated { + pub access_token: AccessToken, + pub expires_in: Option, +} + pub struct UsernameParts { pub account: Username, pub master_user: Option, @@ -38,15 +49,38 @@ pub struct Username { } impl Server { - pub async fn authenticate(&self, req: &AuthRequest) -> trc::Result { - match Box::pin(self.route_auth_request(req)) - .await - .and_then(|token| token.assert_has_permission(Permission::Authenticate)) - { - Ok(token) => Ok(token), + pub async fn authenticate(&self, req: &AuthRequest) -> trc::Result { + let result = match &req.credentials { + Credentials::Basic { + username, + secret, + mfa_token, + } => self + .authenticate_basic(req, username, secret, mfa_token.as_deref()) + .await + .map(|access_token| Authenticated { + access_token, + expires_in: None, + }), + Credentials::Bearer { username, token } => { + self.authenticate_bearer(req, username.as_deref(), token) + .await + } + }; + + match result.and_then(|authenticated| { + authenticated + .access_token + .assert_has_permission(Permission::Authenticate) + .map(|access_token| Authenticated { + access_token, + expires_in: authenticated.expires_in, + }) + }) { + Ok(authenticated) => Ok(authenticated), Err(err) => { // Random delay to mitigate user enumeration attacks - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { use store::rand::{self, RngExt}; @@ -54,314 +88,439 @@ impl Server { tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } - if matches!( - err.as_ref(), - trc::EventType::Auth(trc::AuthEvent::Failed) - | trc::EventType::Security(trc::SecurityEvent::IpUnauthorized) - ) && self.has_auth_fail2ban() - && self - .is_auth_fail2banned(req.remote_ip, req.username()) - .await? - { - Err(trc::SecurityEvent::AuthenticationBan - .into_err() - .ctx(trc::Key::RemoteIp, req.remote_ip) - .ctx_opt(trc::Key::AccountName, req.username().map(|s| s.to_string()))) - } else { - Err(err.ctx(trc::Key::RemoteIp, req.remote_ip)) - } + Err(self.auth_failure(err, req.remote_ip, req.username()).await) } } } - async fn route_auth_request(&self, req: &AuthRequest) -> trc::Result { - match &req.credentials { - Credentials::Basic { - username, - secret, - mfa_token, - } => { - let mut username = UsernameParts::new(username); - - // Try to authenticate as fallback admin if configured - if let Some((fallback_user, fallback_hash)) = &self.registry().recovery_admin() - && username.auth_as().address() == fallback_user - { - return if verify_secret_hash(fallback_hash, secret.as_bytes()).await? { - if username.is_master() { - let address = username.account().address(); - if let Some(account_id) = - self.impersonated_account_id(username.account()).await? - { - trc::event!( - Auth(trc::AuthEvent::Success), - AccountName = address.to_string(), - AccountId = account_id, - SpanId = req.session_id, - Details = fallback_user.to_string(), - ); - - self.access_token(account_id) - .await - .and_then(|token| AccessToken::new(token, req.remote_ip)) - } else { - Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, address.to_string()) - .reason("Master user account not found for fallback admin authentication")) - } - } else { - trc::event!( - Auth(trc::AuthEvent::Success), - AccountName = fallback_user.to_string(), - SpanId = req.session_id, - ); - - Ok(AccessToken::new_admin()) - } - } else { - Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, fallback_user.to_string()) - .ctx(trc::Key::SpanId, req.session_id) - .reason("Fallback admin authentication failed")) - }; - } - - // Add domain if missing, use the default domain - self.add_missing_domain(&mut username.account); - if let Some(master_user) = &mut username.master_user { - self.add_missing_domain(master_user); - } - - // Obtain domain - let auth_as = username.auth_as(); - let auth_as_address = auth_as.address(); - let auth_as_local = auth_as.local(); - let auth_as_domain = auth_as.domain().unwrap(); - let domain = self.resolve_domain(auth_as_domain).await?; - - // Authenticate app passwords - if let Some(app_pass) = AppPassword::parse(secret) { - if username.is_master() { - return Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::SpanId, req.session_id) - .reason("App passwords cannot be used for impersonation")); - } - return if let Some(account_id) = - self.account_id_from_parts(auth_as_local, domain.id).await? - { - self.validate_credential( - account_id, - app_pass.credential_id, - app_pass.secret.as_ref(), - req.remote_ip, - req.session_id, - ) - .await - } else { - Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .reason("App password authentication failed: account not found")) - }; - } - - // Obtain external directory, if any - let mut is_alias_login = false; - let token = if let Some(directory) = self.get_directory_for_cached_domain(&domain) { - let directory_account = if username.is_master() { - directory - .authenticate(&Credentials::Basic { - username: auth_as_address.to_string(), - secret: secret.clone(), - mfa_token: mfa_token.clone(), - }) - .await? - } else { - directory.authenticate(&req.credentials).await? - }; - - is_alias_login = directory_account.email != auth_as_address; - self.build_directory_token(directory_account, req.remote_ip) - .await - } else if let Some(account_id) = - self.account_id_from_parts(auth_as_local, domain.id).await? - { - if let Some(account) = self - .registry() - .object::(account_id.into()) - .await? - .and_then(|account| account.into_user()) - { - let Some(credential) = account.password_credential() else { - return Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::AccountId, account_id) - .ctx(trc::Key::SpanId, req.session_id) - .reason("Password credential not found for account")); - }; - - match verify_mfa_secret_hash( - credential.otp_auth.as_deref(), - mfa_token.as_deref(), - credential.secret.as_str(), - secret, - ) - .await? - { - SecretVerificationResult::Valid => { - is_alias_login = account.name != auth_as_local; - self.access_token(account_id) - .await - .and_then(|token| AccessToken::new(token, req.remote_ip)) - } - SecretVerificationResult::Invalid => Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::AccountId, account_id) - .ctx(trc::Key::SpanId, req.session_id) - .reason("Authentication failed")), - SecretVerificationResult::MissingMfaToken => { - Err(trc::AuthEvent::MfaRequired - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::AccountId, account_id) - .ctx(trc::Key::SpanId, req.session_id) - .reason("MFA token required")) - } - } - } else { - Err(trc::AuthEvent::Error - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::AccountId, account_id) - .reason("Account not found in registry")) - } - } else { - Err(trc::AuthEvent::Failed + pub async fn auth_failure( + &self, + err: trc::Error, + remote_ip: IpAddr, + username: Option<&str>, + ) -> trc::Error { + if matches!( + err.as_ref(), + trc::EventType::Auth(trc::AuthEvent::Failed) + | trc::EventType::Security(trc::SecurityEvent::IpUnauthorized) + ) && self.has_auth_fail2ban() + { + match self.is_auth_fail2banned(remote_ip, username).await { + Ok(true) => { + return trc::SecurityEvent::AuthenticationBan .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .reason("Account not found")) - }?; - - // Enforce alias login restrictions - if is_alias_login && !token.has_permission(Permission::AuthenticateWithAlias) { - return Err(trc::AuthEvent::Failed - .into_err() - .ctx(trc::Key::AccountName, auth_as_address.to_string()) - .ctx(trc::Key::AccountId, token.account_id()) - .ctx(trc::Key::SpanId, req.session_id) - .reason("Authenticated using an email alias but account does not have AuthenticateAlias permission")); + .ctx(trc::Key::RemoteIp, remote_ip) + .ctx_opt(trc::Key::AccountName, username.map(CompactString::from)); } + Ok(false) => {} + Err(err) => return err, + } + } - // Validate master user access + err.details_to_reason().ctx(trc::Key::RemoteIp, remote_ip) + } + + async fn authenticate_basic( + &self, + req: &AuthRequest, + username: &str, + secret: &str, + mfa_token: Option<&str>, + ) -> trc::Result { + let mut username = UsernameParts::new(username); + + // Try to authenticate as fallback admin if configured + if let Some((fallback_user, fallback_hash)) = &self.registry().recovery_admin() + && username.auth_as().address() == fallback_user + { + return if verify_secret_hash( + &self.inner.data.password_verifier, + fallback_hash, + secret.as_bytes(), + ) + .await? + { if username.is_master() { - token.assert_has_permissions(&[ - Permission::Impersonate, - Permission::Authenticate, - ])?; let address = username.account().address(); - let master_address = auth_as_address; if let Some(account_id) = self.impersonated_account_id(username.account()).await? { trc::event!( Auth(trc::AuthEvent::Success), - AccountName = address.to_string(), + AccountName = CompactString::from(address), AccountId = account_id, SpanId = req.session_id, - Details = master_address.to_string(), + Address = CompactString::from(fallback_user), ); self.access_token(account_id) .await - .map(AccessToken::new_maybe_invalid) + .and_then(|token| AccessToken::new(token, req.remote_ip)) } else { Err(trc::AuthEvent::Failed .into_err() - .ctx(trc::Key::AccountName, address.to_string()) - .details(master_address.to_string()) - .reason("Master user account not found")) + .ctx(trc::Key::AccountName, CompactString::from(address)) + .reason( + "Master user account not found for fallback admin authentication", + )) } } else { trc::event!( Auth(trc::AuthEvent::Success), - AccountName = auth_as_address.to_string(), - AccountId = token.account_id(), + AccountName = CompactString::from(fallback_user), SpanId = req.session_id, ); - Ok(token) + Ok(AccessToken::new_admin()) } + } else { + Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(fallback_user)) + .ctx(trc::Key::SpanId, req.session_id) + .reason("Fallback admin authentication failed")) + }; + } + + // Add domain if missing, use the default domain + self.add_missing_domain(&mut username.account); + if let Some(master_user) = &mut username.master_user { + self.add_missing_domain(master_user); + } + + // Obtain domain + let auth_as = username.auth_as(); + let auth_as_address = auth_as.address(); + let auth_as_local = auth_as.local(); + let auth_as_domain = auth_as.domain().unwrap(); + let domain = self.resolve_domain(auth_as_domain).await?; + + // Authenticate app passwords + if let Some(app_pass) = AppPassword::parse(secret) { + if username.is_master() { + return Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::SpanId, req.session_id) + .reason("App passwords cannot be used for impersonation")); } - Credentials::Bearer { username, token } => { - // Handle API key authentication - if let Some(key) = ApiKey::parse(token) { - return self - .validate_credential( - key.account_id, - key.credential_id, - key.secret.as_ref(), - req.remote_ip, - req.session_id, - ) - .await; - } + return if let Some(account_id) = + self.account_id_from_parts(auth_as_local, domain.id).await? + { + self.validate_credential( + account_id, + app_pass.credential_id, + app_pass.secret.as_ref(), + req.remote_ip, + req.session_id, + ) + .await + } else { + Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .reason("App password authentication failed: account not found")) + }; + } - #[cfg(feature = "dev_mode")] - if std::env::var("API_TOKEN_ADMIN").is_ok_and(|admin_token| &admin_token == token) { - return Ok(AccessToken::new_admin()); - } + // Obtain external directory, if any + let mut is_alias_login = false; + let token = if let Some(directory) = self.get_directory_for_cached_domain(&domain) { + let directory_account = if username.is_master() { + directory + .authenticate( + &Credentials::Basic { + username: auth_as_address.to_string(), + secret: secret.to_string(), + mfa_token: mfa_token.map(String::from), + }, + &self.inner.data.password_verifier, + ) + .await? + } else { + directory + .authenticate(&req.credentials, &self.inner.data.password_verifier) + .await? + }; - // Obtain external directory, if any. When no username is supplied - // (e.g. HTTP bearer auth), peek at the JWT claims to find the - // user's domain so per-domain OIDC directories are reachable. - let directory = match username.as_deref().map(UsernameParts::new) { - Some(username) => match username.auth_as().domain() { - Some(domain_name) => self.get_directory_for_domain(domain_name).await?, - None => self.get_directory_for_token(token).await?, - }, - None => self.get_directory_for_token(token).await?, + self.assert_account_directory(&directory_account.email, directory) + .await?; + is_alias_login = directory_account.email != auth_as_address; + self.build_directory_token(directory_account, req.remote_ip) + .await + } else if let Some(account_id) = + self.account_id_from_parts(auth_as_local, domain.id).await? + { + if let Some(account) = self + .registry() + .object::(account_id.into()) + .await? + .and_then(|account| account.into_user()) + { + let Some(credential) = account.password_credential() else { + return Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::AccountId, account_id) + .ctx(trc::Key::SpanId, req.session_id) + .reason("Password credential not found for account")); }; - // Try external directory authentication first if supported, then fallback to internal OAuth. - let mut external_error = None; - if let Some(directory) = directory - && directory.has_bearer_token_support() + match verify_mfa_secret_hash( + &self.inner.data.password_verifier, + credential.otp_url.as_deref(), + mfa_token, + credential.secret.as_str(), + secret, + ) + .await? { - match directory.authenticate(&req.credentials).await { - Ok(result) => { - return self.build_directory_token(result, req.remote_ip).await; - } - Err(err) => { - external_error = Some(err); - } + SecretVerificationResult::Valid => { + is_alias_login = account.name != auth_as_local; + self.access_token(account_id) + .await + .and_then(|token| AccessToken::new(token, req.remote_ip)) } + SecretVerificationResult::Invalid => Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::AccountId, account_id) + .ctx(trc::Key::SpanId, req.session_id) + .reason("Authentication failed")), + SecretVerificationResult::MissingMfaToken => Err(trc::AuthEvent::MfaRequired + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::AccountId, account_id) + .ctx(trc::Key::SpanId, req.session_id) + .reason("MFA token required")), } + } else { + Err(trc::AuthEvent::Error + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::AccountId, account_id) + .reason("Account not found in registry")) + } + } else { + Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .reason("Account not found")) + }?; - // Internal OAuth - match self - .validate_access_token(GrantType::AccessToken.into(), token) + // Enforce alias login restrictions + if is_alias_login && !token.has_permission(Permission::AuthenticateWithAlias) { + return Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(auth_as_address)) + .ctx(trc::Key::AccountId, token.account_id()) + .ctx(trc::Key::SpanId, req.session_id) + .reason("Authenticated using an email alias but account does not have AuthenticateAlias permission")); + } + + // Validate master user access + if username.is_master() { + token.assert_has_permissions(&[Permission::Impersonate, Permission::Authenticate])?; + let address = username.account().address(); + let master_address = auth_as_address; + if let Some(account_id) = self.impersonated_account_id(username.account()).await? { + trc::event!( + Auth(trc::AuthEvent::Success), + AccountName = CompactString::from(address), + AccountId = account_id, + SpanId = req.session_id, + Address = CompactString::from(master_address), + ); + + self.access_token(account_id) .await - { - Ok(token_info) => self - .access_token(token_info.account_id) + .map(AccessToken::new_maybe_invalid) + } else { + Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(address)) + .ctx(trc::Key::Address, CompactString::from(master_address)) + .reason("Master user account not found")) + } + } else { + trc::event!( + Auth(trc::AuthEvent::Success), + AccountName = CompactString::from(auth_as_address), + AccountId = token.account_id(), + SpanId = req.session_id, + ); + + Ok(token) + } + } + + async fn authenticate_bearer( + &self, + req: &AuthRequest, + username: Option<&str>, + token: &str, + ) -> trc::Result { + // Handle API key authentication + if let Some(key) = ApiKey::parse(token) { + return self + .validate_credential( + key.account_id, + key.credential_id, + key.secret.as_ref(), + req.remote_ip, + req.session_id, + ) + .await + .map(|access_token| Authenticated { + access_token, + expires_in: None, + }); + } + + #[cfg(feature = "dev_mode")] + if std::env::var("API_TOKEN_ADMIN").is_ok_and(|admin_token| admin_token == token) { + return Ok(Authenticated { + access_token: AccessToken::new_admin(), + expires_in: None, + }); + } + + if token.starts_with(TOKEN_HEADER) { + return self.authenticate_oauth_token(token, req.remote_ip).await; + } + + // Obtain external directory, if any. When no username is supplied + // (e.g. HTTP bearer auth), peek at the JWT claims to find the + // user's domain so per-domain OIDC directories are reachable. + let directory = self.bearer_directory(username, token).await?; + + // Try external directory authentication first if supported, then fallback to internal OAuth. + let mut external_error = None; + if let Some(directory) = directory + && directory.has_bearer_token_support() + { + match self.authenticate_external_token(directory, token).await { + Ok(external) => { + self.assert_account_directory(&external.account.email, directory) + .await?; + let token_scope = match external.access { + TokenAccess::Unrestricted => None, + TokenAccess::Scoped(scopes) => Some(TokenScope::External { + scopes: OAuthScopes::from_external(scopes), + }), + }; + let expires_in = external.expires_at.map(|exp| exp.saturating_sub(now())); + return self + .build_directory_token(external.account, req.remote_ip) .await - .and_then(|token| AccessToken::new(token, req.remote_ip)), - Err(err) => { - if let Some(external_error) = external_error { - Err(external_error) - } else { - Err(err) - } - } + .map(|access_token| Authenticated { + access_token: access_token.with_token_scope(token_scope), + expires_in, + }); + } + Err(err) => { + external_error = Some(err); } } } + + // Internal OAuth + match self.authenticate_oauth_token(token, req.remote_ip).await { + Ok(authenticated) => Ok(authenticated), + Err(err) => { + if let Some(external_error) = external_error { + Err(external_error) + } else { + Err(err) + } + } + } + } + + pub async fn authenticate_external_token( + &self, + directory: &Directory, + token: &str, + ) -> trc::Result { + let resources = &self.core.network.resources; + let applications = &self.inner.data.applications; + let app_policy = self.app_origin_policy(None); + directory + .authenticate_token( + token, + &TokenContext { + is_resource: &|audience| { + let audience = audience.strip_suffix('/').unwrap_or(audience); + resources + .identifiers + .iter() + .any(|identifier| identifier.eq_ignore_ascii_case(audience)) + }, + is_first_party: &|client_id| { + applications.is_first_party_client(client_id, app_policy) + }, + }, + ) + .await + } + + async fn assert_account_directory( + &self, + email: &str, + directory: &Arc, + ) -> trc::Result<()> { + let account_directory = match email.rsplit_once('@') { + Some((_, domain)) => self + .get_directory_for_domain(domain) + .await + .caused_by(trc::location!())?, + None => None, + }; + if account_directory + .is_some_and(|account_directory| Arc::ptr_eq(account_directory, directory)) + { + Ok(()) + } else { + Err(trc::AuthEvent::Failed + .into_err() + .ctx(trc::Key::AccountName, CompactString::from(email)) + .reason( + "The account's domain is not served by the directory that authenticated it", + )) + } + } + + async fn authenticate_oauth_token( + &self, + token: &str, + remote_ip: IpAddr, + ) -> trc::Result { + let token_info = self + .decode_access_token(GrantType::AccessToken.into(), token) + .map_err(|err| { + if err.matches(trc::EventType::Auth(trc::AuthEvent::Error)) { + trc::AuthEvent::Failed.into_err().caused_by(err) + } else { + err + } + })?; + let Some(grant) = token_info.grant else { + return Err(trc::AuthEvent::Failed + .into_err() + .details("Access token is not bound to an OAuth grant")); + }; + let inner = self.access_token(token_info.account_id).await?; + assert_credential_version(token_info.credential_version, inner.credential_version())?; + let token_scope = self + .oauth_grant_scope(grant.id, token_info.account_id) + .await?; + + AccessToken::renew(inner, None, Some(token_scope), remote_ip).map(|access_token| { + Authenticated { + access_token, + expires_in: Some(token_info.expires_in), + } + }) } async fn impersonated_account_id(&self, username: &Username) -> trc::Result> { @@ -376,7 +535,7 @@ impl Server { && let Some(directory) = self.get_directory_for_cached_domain(&domain_cache) && let Recipient::Account(account) = directory.recipient(address).await? { - return Ok(Some(Box::pin(self.synchronize_account(account)).await?.id)); + return Ok(Some(self.synchronize_account(account).await?.id)); } Ok(None) @@ -406,7 +565,13 @@ impl Server { }) { if credential.credential_id.document_id() == credential_id { - if !verify_secret_hash(&credential.secret, secret).await? { + if !verify_secret_hash( + &self.inner.data.password_verifier, + &credential.secret, + secret, + ) + .await? + { return Err(trc::AuthEvent::Failed .into_err() .ctx(trc::Key::AccountName, account.name) @@ -478,7 +643,7 @@ impl Server { } else { Err(trc::AuthEvent::Failed .into_err() - .ctx(trc::Key::Details, domain_name.to_string()) + .ctx(trc::Key::Domain, CompactString::from(domain_name)) .reason("Domain not found")) } } @@ -487,7 +652,7 @@ impl Server { if address.domain().is_none() { trc::event!( Auth(trc::AuthEvent::Warning), - AccountName = address.address().to_string(), + AccountName = CompactString::from(address.address()), Reason = "No domain in username", ); address.domain_start = address.name.len() + 1; @@ -500,12 +665,28 @@ impl Server { account: directory::Account, remote_ip: IpAddr, ) -> trc::Result { - let account = Box::pin(self.synchronize_account(account)).await?; + let account = self.synchronize_account(account).await?; self.access_token_from_account(account.id, account.account) .await .and_then(|token| AccessToken::new(token, remote_ip)) } + pub(crate) async fn bearer_directory( + &self, + username: Option<&str>, + token: &str, + ) -> trc::Result>> { + let username = username.map(UsernameParts::new); + match username + .as_ref() + .and_then(|username| username.auth_as().domain()) + { + Some(domain_name) => self.get_directory_for_domain(domain_name).await, + None => self.get_directory_for_token(token).await, + } + } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] pub async fn get_directory_for_domain( &self, domain_name: &str, @@ -514,6 +695,12 @@ impl Server { Ok(self.get_default_directory()) } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + pub async fn get_directory_for_host(&self, host: &str) -> trc::Result>> { + + Ok(self.get_default_directory()) + } + async fn get_directory_for_token(&self, token: &str) -> trc::Result>> { let Some(payload) = JwtClaims::decode_payload(token) else { return Ok(self.get_default_directory()); @@ -531,11 +718,13 @@ impl Server { } } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] fn get_directory_for_issuer(&self, issuer: &str) -> Option<&Arc> { None } + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] pub fn get_directory_for_cached_domain(&self, domain: &DomainCache) -> Option<&Arc> { self.get_default_directory() @@ -568,7 +757,7 @@ impl<'x> JwtClaims<'x> { return None; } - general_purpose::URL_SAFE_NO_PAD.decode(payload).ok() + URL_SAFE_NO_PAD.decode(payload).ok() } fn parse(payload: &'x [u8]) -> Option { @@ -684,75 +873,3 @@ impl AuthRequest { } } -#[cfg(test)] -mod tests { - use super::*; - - fn jwt(payload: &str) -> String { - format!( - "eyJhbGciOiJSUzI1NiJ9.{}.c2lnbmF0dXJl", - general_purpose::URL_SAFE_NO_PAD.encode(payload) - ) - } - - fn hints(token: &str) -> Option<(Option, Option)> { - let payload = JwtClaims::decode_payload(token)?; - let claims = JwtClaims::parse(&payload)?; - - Some(( - claims.domain().map(str::to_string), - claims.iss.as_deref().map(str::to_string), - )) - } - - #[test] - fn jwt_claims_are_extracted() { - for (payload, domain, issuer) in [ - ( - r#"{"iss":"https://idp.example.org","email":"John@Example.ORG"}"#, - Some("Example.ORG"), - Some("https://idp.example.org"), - ), - ( - r#"{"preferred_username":"jane@example.net","upn":"jane@example.com"}"#, - Some("example.net"), - None, - ), - ( - r#"{"email":"broken@","upn":"jane@example.com"}"#, - Some("example.com"), - None, - ), - ( - r#"{"iss":"https://idp.example.org","sub":"5db2d1b6","aud":["a","b"],"scope":"openid"}"#, - None, - Some("https://idp.example.org"), - ), - (r#"{"sub":"5db2d1b6"}"#, None, None), - (r#"{"email":"jane@example.net"}"#, Some("example.net"), None), - ] { - assert_eq!( - hints(&jwt(payload)), - Some((domain.map(str::to_string), issuer.map(str::to_string))), - "Unexpected claims for {payload}" - ); - } - } - - #[test] - fn non_jwt_tokens_are_ignored() { - for token in [ - "sw1.eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJodHRwczovL2lkcC5leGFtcGxlLm9yZyJ9", - "sw1.eyJhbGciOiJSUzI1NiJ9", - "opaque-token", - "one.two", - "one.two.three.four", - "", - ] { - assert!( - JwtClaims::decode_payload(token).is_none(), - "Token {token:?} was parsed as a JWT" - ); - } - } -} diff --git a/crates/common/src/auth/credential.rs b/crates/common/src/auth/credential.rs index 33123f46..78accca2 100644 --- a/crates/common/src/auth/credential.rs +++ b/crates/common/src/auth/credential.rs @@ -4,13 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use std::io::Write; +use encodify::{base32::STALWART, base64::URL_SAFE_NO_PAD}; use store::{ U32_LEN, rand::{self}, }; -use utils::codec::base32_custom::{Base32Reader, Base32Writer}; pub struct ApiKey { pub account_id: u32, @@ -18,6 +16,8 @@ pub struct ApiKey { pub secret: [u8; 20], } +const API_KEY_LEN: usize = U32_LEN * 2 + 20; + pub struct AppPassword { pub credential_id: u32, pub secret: [u8; 18], @@ -33,7 +33,11 @@ impl ApiKey { } pub fn parse(token: &str) -> Option { - let decoded = URL_SAFE_NO_PAD.decode(token.strip_prefix("API_")?).ok()?; + let mut decoded = [0u8; API_KEY_LEN]; + URL_SAFE_NO_PAD + .decode_slice(token.strip_prefix("API_")?, &mut decoded) + .ok() + .filter(|&len| len == API_KEY_LEN)?; Some(ApiKey { account_id: u32::from_be_bytes(decoded.get(0..U32_LEN)?.try_into().ok()?), @@ -43,11 +47,13 @@ impl ApiKey { } pub fn build(&self) -> String { - let mut bytes = Vec::with_capacity(U32_LEN * 2 + self.secret.len()); + let mut bytes = Vec::with_capacity(API_KEY_LEN); bytes.extend_from_slice(&self.account_id.to_be_bytes()); bytes.extend_from_slice(&self.credential_id.to_be_bytes()); bytes.extend_from_slice(&self.secret); - format!("API_{}", URL_SAFE_NO_PAD.encode(bytes)) + let mut token = String::from("API_"); + URL_SAFE_NO_PAD.encode_append(bytes, &mut token); + token } } @@ -61,7 +67,7 @@ impl AppPassword { pub fn parse(token: &str) -> Option { let token = token.strip_prefix("app")?; - let mut reader = Base32Reader::new(token.as_bytes().get(1..)?); + let mut reader = STALWART.decoder(token.as_bytes().get(1..)?); let mut credential_id = [0u8; 4]; let mut secret = [0u8; 18]; @@ -84,10 +90,11 @@ impl AppPassword { } pub fn build(&self) -> String { - let mut writer = Base32Writer::with_capacity(std::mem::size_of::().div_ceil(5) * 8); - writer.push_string("app_"); - let _ = writer.write(&self.credential_id.to_be_bytes()); - let _ = writer.write_all(&self.secret); - writer.finalize() + let mut token = String::from("app_"); + let mut encoder = STALWART.encoder(&mut token); + encoder.push(&self.credential_id.to_be_bytes()); + encoder.push(&self.secret); + encoder.finish(); + token } } diff --git a/crates/common/src/auth/mod.rs b/crates/common/src/auth/mod.rs index c4e98707..a4a164a3 100644 --- a/crates/common/src/auth/mod.rs +++ b/crates/common/src/auth/mod.rs @@ -5,10 +5,12 @@ */ use crate::{ + auth::{active_script::ActiveScriptSlot, oauth::scope::OAuthScopes}, expr::if_block::IfBlock, - network::limiter::ConcurrencyLimiter, - storage::{ObjectQuota, TenantQuota}, + network::limiter::AccountLimits, + storage::ObjectQuota, }; +use compact_str::CompactString; use directory::Credentials; use quick_cache::Equivalent; use registry::{ @@ -26,19 +28,21 @@ use types::collection::Collection; use utils::{cache::CacheItemWeight, map::bitmap::Bitmap}; pub mod access_token; +pub mod active_script; pub mod authentication; pub mod credential; pub mod oauth; pub mod permissions; pub mod rate_limit; +pub mod sasl; pub const RECOVERY_ADMIN_ID: u32 = u32::MAX; -const PERMISSIONS_BITSET_SIZE: usize = Permission::COUNT.div_ceil(std::mem::size_of::()); +const PERMISSIONS_BITSET_SIZE: usize = Permission::COUNT.div_ceil(std::mem::size_of::() * 8); pub type Permissions = Bitset; #[derive(Debug, PartialEq, Eq, Clone)] pub struct EmailAddress { - pub local_part: Box, + pub local_part: CompactString, pub domain_id: u32, } @@ -60,8 +64,6 @@ pub enum EmailCache { pub struct DomainCache { pub names: Box<[Box]>, pub id: u32, - pub id_directory: Option, - pub id_tenant: Option, pub catch_all: Option>, pub sub_addressing_custom: Option>, pub flags: u8, @@ -76,14 +78,35 @@ pub struct AccountCache { pub name: Box, pub id: u32, pub addresses: Box<[EmailAddress]>, - pub id_tenant: Option, - pub id_member_of: TinyVec<[u32; 3]>, + pub id_member_of: Box<[u32]>, pub quota_disk: u64, - pub quota_objects: Option>, pub description: Option>, - pub encryption_key: Option, + pub storage: Option>, pub locale: Locale, pub flags: u64, + pub active_script: ActiveScriptSlot, +} + +#[derive(Debug, Clone, Default)] +pub struct AccountStorage { + pub quota_objects: Option, + pub encryption_key: Option, +} + +const _: () = assert!(size_of::() <= 112); + +impl AccountStorage { + pub fn boxed( + quota_objects: Option, + encryption_key: Option, + ) -> Option> { + (quota_objects.is_some() || encryption_key.is_some()).then(|| { + Box::new(AccountStorage { + quota_objects, + encryption_key, + }) + }) + } } pub type EncryptionKeys = Box<[Box<[u8]>]>; @@ -110,13 +133,6 @@ pub struct MailingListCache { pub recipients: Arc<[Box]>, } -#[derive(Debug, Clone)] -pub struct TenantCache { - pub id_roles: TinyVec<[u32; 3]>, - pub quota_disk: u64, - pub quota_objects: Option>, - pub permissions: Option>, -} #[derive(Debug, Clone, Default)] pub struct PermissionsGroup { @@ -129,24 +145,45 @@ pub struct PermissionsGroup { pub struct AccessToken { scope_idx: usize, inner: Arc, + token_scope: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TokenScope { + Grant { + grant_id: u64, + scopes: OAuthScopes, + client: GrantClient, + }, + External { + scopes: OAuthScopes, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrantClient { + FirstParty, + ThirdParty, +} + +const _: () = assert!(size_of::() <= 16); +const _: () = assert!(size_of::>() <= 16); + #[derive(Debug, Default, Clone)] pub struct AccessTokenInner { pub(crate) account_id: u32, - pub(crate) tenant_id: Option, - pub(crate) member_of: TinyVec<[u32; 3]>, + pub(crate) obj_size: u32, + pub(crate) member_of: Box<[u32]>, pub(crate) access_to: Box<[AccessTo]>, pub(crate) scopes: Box<[AccessScope]>, - pub(crate) concurrent_http_requests: Option, - pub(crate) concurrent_imap_requests: Option, - pub(crate) concurrent_uploads: Option, + pub(crate) limits: Option>, pub(crate) revision_account: u64, pub(crate) revision: u64, pub(crate) credential_version: u64, - pub(crate) obj_size: u64, } +const _: () = assert!(size_of::() <= 96); + #[derive(Debug, Default, Hash, Clone)] pub struct AccessScope { pub permissions: Permissions, @@ -159,6 +196,7 @@ pub struct AccessScope { pub(crate) struct AccessTo { pub account_id: u32, pub collections: Bitmap, + pub is_writable: bool, } #[derive(Clone)] @@ -169,9 +207,8 @@ pub struct AccountInfo { } #[derive(Clone, Copy)] -pub struct AccountTenantIds { +pub struct ChangedBy { pub account_id: u32, - pub tenant_id: Option, } pub struct AuthRequest { @@ -182,13 +219,13 @@ pub struct AuthRequest { impl CacheItemWeight for AccessTokenInner { fn weight(&self) -> u64 { - self.obj_size + u64::from(self.obj_size) } } impl CacheItemWeight for EmailAddress { fn weight(&self) -> u64 { - std::mem::size_of::() as u64 + self.local_part.len() as u64 + std::mem::size_of::() as u64 + self.heap_size() } } @@ -216,13 +253,13 @@ impl CacheItemWeight for DomainCache { impl Equivalent for EmailAddressRef<'_> { fn equivalent(&self, key: &EmailAddress) -> bool { - self.local_part == &*key.local_part && self.domain_id == key.domain_id + self.local_part == key.local_part.as_str() && self.domain_id == key.domain_id } } impl Hash for EmailAddress { fn hash(&self, state: &mut H) { - self.local_part.as_ref().hash(state); + self.local_part.as_str().hash(state); self.domain_id.hash(state); } } @@ -236,14 +273,22 @@ impl Hash for EmailAddressRef<'_> { impl CacheItemWeight for AccountCache { fn weight(&self) -> u64 { - std::mem::size_of::() as u64 - + self.name.len() as u64 + (std::mem::size_of::() + + self.name.len() + + std::mem::size_of_val::<[u32]>(&self.id_member_of) + + self.description.as_ref().map_or(0, |s| s.len())) as u64 + self .addresses .iter() - .map(|s| s.local_part.len() as u64 + std::mem::size_of::() as u64) + .map(|address| address.weight()) .sum::() - + self.description.as_ref().map_or(0, |s| s.len() as u64) + + self.storage.as_ref().map_or(0, |storage| storage.weight()) + } +} + +impl CacheItemWeight for AccountStorage { + fn weight(&self) -> u64 { + std::mem::size_of::() as u64 + self.encryption_key.as_ref().map_or(0, |keys| { keys.iter() .map(|k| k.len() as u64 + std::mem::size_of::>() as u64) @@ -264,7 +309,7 @@ impl CacheItemWeight for MailingListCache { + self .addresses .iter() - .map(|s| s.local_part.len() as u64 + std::mem::size_of::() as u64) + .map(|address| address.weight()) .sum::() + self .recipients @@ -274,12 +319,6 @@ impl CacheItemWeight for MailingListCache { } } -impl CacheItemWeight for TenantCache { - fn weight(&self) -> u64 { - std::mem::size_of::() as u64 - + self.permissions.as_ref().map_or(0, |p| p.weight()) - } -} impl CacheItemWeight for PermissionsGroup { fn weight(&self) -> u64 { @@ -296,17 +335,27 @@ impl BuildAccessToken for Arc { AccessToken { scope_idx: 0, inner: self, + token_scope: None, } } } impl EmailAddress { - pub fn new(local_part: impl Into>, domain_id: u32) -> Self { + pub fn new(local_part: impl Into, domain_id: u32) -> Self { Self { local_part: local_part.into(), domain_id, } } + + #[inline] + pub fn heap_size(&self) -> u64 { + if self.local_part.is_heap_allocated() { + self.local_part.capacity() as u64 + } else { + 0 + } + } } impl<'x> EmailAddressRef<'x> { @@ -322,6 +371,13 @@ impl AccountCache { pub fn domain_id(&self) -> Option { self.addresses.first().map(|address| address.domain_id) } + + + #[cfg(not(feature = "enterprise"))] + #[inline] + pub fn is_same_tenant(&self, _access_token: &AccessToken) -> bool { + true + } } impl DomainCache { @@ -329,4 +385,11 @@ impl DomainCache { self.names.first().map(|s| s.as_ref()).unwrap_or_default() } + + #[cfg(not(feature = "enterprise"))] + #[inline] + pub fn is_same_tenant(&self, _other: &DomainCache) -> bool { + true + } + } diff --git a/crates/common/src/auth/oauth/cimd.rs b/crates/common/src/auth/oauth/cimd.rs new file mode 100644 index 00000000..6dfc273c --- /dev/null +++ b/crates/common/src/auth/oauth/cimd.rs @@ -0,0 +1,412 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + registration::{check_client_name, is_https_url, loopback_redirect_parts}, + scope::OAuthScopes, +}; +use crate::{KV_OAUTH_CIMD, KV_RATE_LIMIT_CIMD, Server, USER_AGENT}; +use registry::schema::{enums::OAuthClientMetadataPolicy, prelude::Duration, structs::Rate}; +use reqwest::{ + Method, StatusCode, Url, + header::{ACCEPT, CACHE_CONTROL}, +}; +use serde::{Deserialize, de::IgnoredAny}; +use std::time::Duration as StdDuration; +use store::{ + Serialize, + dispatch::lookup::KeyValue, + write::{Archive, ArchiveBytes, Archiver}, +}; +use trc::{AddContext, AuthEvent}; +use utils::{ + HttpLimitResponse, + http::http_client_builder, + outbound::{OutboundClient, OutboundError, OutboundPolicy, Redirects}, +}; + +pub const CIMD_MAX_DOCUMENT_SIZE: usize = 5 * 1024; +const CIMD_FETCH_TIMEOUT: StdDuration = StdDuration::from_secs(5); +const CIMD_MIN_TTL: u64 = 5 * 60; +const CIMD_MAX_TTL: u64 = 24 * 60 * 60; +const CIMD_MAX_REDIRECT_URIS: usize = 16; +const CIMD_MAX_REDIRECT_URI_LEN: usize = 2048; +const CIMD_DOMAIN_RATE: Rate = Rate { + count: 30, + period: Duration(StdDuration::from_secs(60)), +}; + +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq)] +pub struct CimdClient { + pub client_host: String, + pub client_name: Option, + pub redirect_uris: Vec, + pub post_logout_redirect_uris: Vec, + pub scopes: OAuthScopes, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CimdError { + Disabled, + InvalidUrl(&'static str), + DomainNotAllowed, + Outbound(OutboundError), + Fetch, + Status(u16), + TooLarge, + InvalidDocument(&'static str), +} + +#[derive(Deserialize)] +struct CimdDocument { + #[serde(default)] + client_id: Option, + #[serde(default)] + client_name: Option, + #[serde(default)] + redirect_uris: Option>, + #[serde(default)] + post_logout_redirect_uris: Option>, + #[serde(default)] + scope: Option, + #[serde(default)] + token_endpoint_auth_method: Option, + #[serde(default)] + client_secret: Option, + #[serde(default)] + client_secret_expires_at: Option, + #[serde(default)] + jwks: Option, +} + +impl Server { + pub fn check_cimd_policy(&self, client_id: &str) -> Result { + cimd_policy_allows( + self.core.oauth.client_metadata_documents, + client_id, + self.core.network.resources.registrable_domain.as_deref(), + ) + } + + pub async fn cimd_client( + &self, + client_id: &str, + url: Url, + ) -> trc::Result> { + if let Some(stored) = self + .in_memory_store() + .key_get::>(KeyValue::<()>::build_key( + KV_OAUTH_CIMD, + client_id.as_bytes(), + )) + .await + .caused_by(trc::location!())? + && let Ok(client) = stored.deserialize::() + { + return Ok(Ok(client)); + } + + let host = url.host_str().unwrap_or_default(); + let domain = psl::domain_str(host).unwrap_or(host); + if let Some(reset) = self + .core + .storage + .memory + .is_rate_allowed(KV_RATE_LIMIT_CIMD, domain.as_bytes(), &CIMD_DOMAIN_RATE) + .await + .caused_by(trc::location!())? + { + return Err(trc::LimitEvent::TooManyRequests + .into_err() + .ctx(trc::Key::Expires, reset) + .ctx(trc::Key::Limit, CIMD_DOMAIN_RATE.count) + .details("Too many client metadata document fetches for this domain")); + } + + let client_host = host.to_string(); + let (body, ttl) = match fetch_cimd_document(url).await { + Ok(result) => result, + Err(err) => return Ok(Err(err)), + }; + let client = match validate_cimd_document(client_id, client_host, &body) { + Ok(client) => client, + Err(err) => return Ok(Err(err)), + }; + + let value = Archiver::new(client.clone()) + .untrusted() + .serialize() + .caused_by(trc::location!())?; + self.in_memory_store() + .key_set(KeyValue::with_prefix(KV_OAUTH_CIMD, client_id.as_bytes(), value).expires(ttl)) + .await + .caused_by(trc::location!())?; + + Ok(Ok(client)) + } +} + +pub fn cimd_policy_allows( + policy: OAuthClientMetadataPolicy, + client_id: &str, + registrable_domain: Option<&str>, +) -> Result { + match policy { + OAuthClientMetadataPolicy::Disabled => Err(CimdError::Disabled), + OAuthClientMetadataPolicy::SameDomain => { + let url = check_cimd_url(client_id)?; + if registrable_domain.is_some_and(|expected| { + url.host_str() + .and_then(psl::domain_str) + .is_some_and(|domain| domain.eq_ignore_ascii_case(expected)) + }) { + Ok(url) + } else { + Err(CimdError::DomainNotAllowed) + } + } + OAuthClientMetadataPolicy::Any => check_cimd_url(client_id), + } +} + +pub fn check_cimd_url(client_id: &str) -> Result { + let rest = client_id + .strip_prefix("https://") + .ok_or(CimdError::InvalidUrl("Client id must use the https scheme"))?; + if client_id.contains(['#', '\\']) { + return Err(CimdError::InvalidUrl( + "Client id must not contain a fragment", + )); + } + if client_id.contains('?') { + return Err(CimdError::InvalidUrl( + "Client id must not contain a query component", + )); + } + let (authority, path) = rest + .split_once('/') + .ok_or(CimdError::InvalidUrl("Client id must contain a path"))?; + if authority.is_empty() || authority.contains('@') { + return Err(CimdError::InvalidUrl( + "Client id must not contain a userinfo component", + )); + } + if path.split('/').any(is_dot_segment) { + return Err(CimdError::InvalidUrl( + "Client id must not contain dot path segments", + )); + } + + let url = Url::parse(client_id).map_err(|_| CimdError::InvalidUrl("Invalid client id URL"))?; + if url.domain().is_some_and(|domain| !domain.is_empty()) { + Ok(url) + } else { + Err(CimdError::InvalidUrl( + "Client id must contain a domain name, not an IP address literal", + )) + } +} + +pub(crate) fn is_dot_segment(segment: &str) -> bool { + hashify::set_ignore_case!( + segment.as_bytes(), + ".", + "..", + "%2e", + "%2e%2e", + ".%2e", + "%2e.", + ) +} + +pub fn validate_cimd_document( + client_id: &str, + client_host: String, + body: &[u8], +) -> Result { + if body.len() > CIMD_MAX_DOCUMENT_SIZE { + return Err(CimdError::TooLarge); + } + let document = serde_json::from_slice::(body) + .map_err(|_| CimdError::InvalidDocument("Document is not a valid JSON object"))?; + + if document.client_id.as_deref() != Some(client_id) { + return Err(CimdError::InvalidDocument( + "client_id does not match the document URL", + )); + } + if document.client_secret.is_some() || document.client_secret_expires_at.is_some() { + return Err(CimdError::InvalidDocument( + "Document must not contain a client secret", + )); + } + if document.token_endpoint_auth_method.as_deref() != Some("none") { + return Err(CimdError::InvalidDocument( + "token_endpoint_auth_method must be \"none\"", + )); + } + if document.jwks.as_ref().is_some_and(has_private_key) { + return Err(CimdError::InvalidDocument( + "Document must not contain private key material", + )); + } + + let redirect_uris = document + .redirect_uris + .filter(|uris| !uris.is_empty() && uris.len() <= CIMD_MAX_REDIRECT_URIS) + .ok_or(CimdError::InvalidDocument( + "redirect_uris must list between 1 and 16 URIs", + ))?; + if !redirect_uris + .iter() + .all(|uri| is_valid_cimd_redirect_uri(uri)) + { + return Err(CimdError::InvalidDocument( + "redirect_uris must be https or loopback URIs without a fragment, of at most 2048 bytes", + )); + } + + let post_logout_redirect_uris = document.post_logout_redirect_uris.unwrap_or_default(); + if post_logout_redirect_uris.len() > CIMD_MAX_REDIRECT_URIS { + return Err(CimdError::InvalidDocument( + "post_logout_redirect_uris must not list more than 16 URIs", + )); + } + if !post_logout_redirect_uris + .iter() + .all(|uri| is_valid_cimd_redirect_uri(uri)) + { + return Err(CimdError::InvalidDocument( + "post_logout_redirect_uris must be https or loopback URIs without a fragment, of at most 2048 bytes", + )); + } + + let client_name = match document.client_name.as_deref().map(str::trim) { + Some(name) if !name.is_empty() => { + check_client_name(name).map_err(CimdError::InvalidDocument)?; + Some(name.to_string()) + } + _ => None, + }; + + Ok(CimdClient { + client_host, + client_name, + redirect_uris, + post_logout_redirect_uris, + scopes: document + .scope + .as_deref() + .map_or(OAuthScopes::INTEROPERABLE, |scope| { + OAuthScopes::parse(scope).interoperable() + }), + }) +} + +fn is_valid_cimd_redirect_uri(uri: &str) -> bool { + uri.len() <= CIMD_MAX_REDIRECT_URI_LEN + && !uri.contains(['#', '\\']) + && (is_https_url(uri) || loopback_redirect_parts(uri).is_some()) +} + +fn has_private_key(jwks: &serde_json::Value) -> bool { + jwks.get("keys") + .and_then(|keys| keys.as_array()) + .is_some_and(|keys| { + keys.iter().any(|key| { + key.as_object().is_some_and(|key| { + ["d", "p", "q", "dp", "dq", "qi", "oth", "k"] + .iter() + .any(|member| key.contains_key(*member)) + }) + }) + }) +} + +async fn fetch_cimd_document(url: Url) -> Result<(Vec, u64), CimdError> { + let display_url = url.to_string(); + let response = OutboundClient::new( + http_client_builder(cfg!(stalwart_tests)) + .user_agent(USER_AGENT) + .timeout(CIMD_FETCH_TIMEOUT), + OutboundPolicy::PublicOnly, + Redirects::None, + ) + .request_url(Method::GET, url) + .map_err(CimdError::Outbound)? + .header(ACCEPT, "application/json") + .send() + .await + .map_err(|err| { + trc::event!( + Auth(AuthEvent::Error), + Url = display_url.clone(), + Reason = err.to_string(), + Details = "Failed to fetch client metadata document", + ); + CimdError::Fetch + })?; + + if response.status() != StatusCode::OK { + return Err(CimdError::Status(response.status().as_u16())); + } + let ttl = cache_ttl( + response + .headers() + .get(CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + ); + let body = response + .bytes_with_limit(CIMD_MAX_DOCUMENT_SIZE) + .await + .map_err(|err| { + trc::event!( + Auth(AuthEvent::Error), + Url = display_url, + Reason = err.to_string(), + Details = "Failed to read client metadata document", + ); + CimdError::Fetch + })? + .ok_or(CimdError::TooLarge)?; + + Ok((body, ttl)) +} + +fn cache_ttl(cache_control: Option<&str>) -> u64 { + cache_control + .and_then(|value| { + value.split(',').find_map(|directive| { + let (name, value) = directive.split_once('=')?; + if name.trim().eq_ignore_ascii_case("max-age") { + value.trim().trim_matches('"').parse::().ok() + } else { + None + } + }) + }) + .unwrap_or(CIMD_MIN_TTL) + .clamp(CIMD_MIN_TTL, CIMD_MAX_TTL) +} + +impl CimdError { + pub fn as_str(&self) -> &'static str { + match self { + CimdError::Disabled => "Client metadata documents are disabled", + CimdError::InvalidUrl(reason) | CimdError::InvalidDocument(reason) => reason, + CimdError::DomainNotAllowed => "Client metadata document domain is not allowed", + CimdError::Outbound(err) => err.as_str(), + CimdError::Fetch => "Failed to fetch the client metadata document", + CimdError::Status(_) => "Client metadata document request did not return 200 OK", + CimdError::TooLarge => "Client metadata document is too large", + } + } +} + +impl store::write::ArchiveCompression for CimdClient { + const COMPRESSION: store::write::Compression = store::write::Compression::None; +} + diff --git a/crates/common/src/auth/oauth/client.rs b/crates/common/src/auth/oauth/client.rs new file mode 100644 index 00000000..a116d96a --- /dev/null +++ b/crates/common/src/auth/oauth/client.rs @@ -0,0 +1,420 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CLIENT_ID_MAX_LEN, + cimd::{CimdClient, CimdError}, + client_id::{CLIENT_ID_HEADER, ClientMeta, decode_client_id}, + registration::{ + is_safe_redirect_target, post_logout_redirect_uri_matches, redirect_uri_matches, + }, + scope::OAuthScopes, +}; +use crate::{ + Server, + config::resource::ResourceIdentifiers, + manager::{app_client::is_safe_authority, application::AppClientDocument}, +}; +use registry::schema::{ + enums::{OAuthClientKind, OAuthGrantType}, + prelude::{ObjectType, Property, UTCDateTime}, + structs::OAuthClient, +}; +use std::net::IpAddr; +use trc::AddContext; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolveMode { + Authorize, + Token, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RequestOrigin<'x> { + pub is_tls: bool, + pub authority: &'x str, +} + +#[derive(Debug, Clone, Copy)] +pub struct ResolveContext<'x> { + pub mode: ResolveMode, + pub origin: Option>, + pub remote_ip: IpAddr, +} + +#[derive(Debug, Clone, Copy)] +pub enum AppOriginPolicy<'x> { + TrustedHosts(&'x ResourceIdentifiers), + SameOrigin(RequestOrigin<'x>), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ApplicationClient { + pub client_name: Option, + pub logo_uri: Option, + pub redirect_uris: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResolvedClient { + Stateless(ClientMeta), + Registered(Box), + Application(ApplicationClient), + Cimd(Option>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientRejection { + TooLong, + Unknown, + Expired, + Metadata(CimdError), +} + +const LOGO_DATA_URI_PREFIX: &str = "data:image/"; + +impl Server { + pub async fn resolve_oauth_client( + &self, + client_id: &str, + ctx: &ResolveContext<'_>, + ) -> trc::Result> { + if client_id.len() > CLIENT_ID_MAX_LEN { + return Ok(Err(ClientRejection::TooLong)); + } else if client_id.starts_with(CLIENT_ID_HEADER) { + return Ok( + decode_client_id(self.core.oauth.oauth_key.as_bytes(), client_id) + .map(ResolvedClient::Stateless) + .ok_or(ClientRejection::Unknown), + ); + } + + let is_https = client_id.starts_with("https://"); + if is_https || client_id.starts_with("http://") { + if let Some(app) = self.app_origin_policy(ctx.origin).and_then(|policy| { + self.inner + .data + .applications + .recognize_client(client_id, policy) + }) { + return Ok(Ok(ResolvedClient::Application(app))); + } + + match self.registered_oauth_client(client_id).await? { + Err(ClientRejection::Unknown) if is_https => {} + result => return Ok(result.map(ResolvedClient::Registered)), + } + + let url = match self.check_cimd_policy(client_id) { + Ok(url) => url, + Err(err) => return Ok(Err(ClientRejection::Metadata(err))), + }; + + match ctx.mode { + ResolveMode::Authorize => Ok(self + .cimd_client(client_id, url) + .await? + .map(|client| ResolvedClient::Cimd(Some(Box::new(client)))) + .map_err(ClientRejection::Metadata)), + ResolveMode::Token => Ok(Ok(ResolvedClient::Cimd(None))), + } + } else { + Ok(self + .registered_oauth_client(client_id) + .await? + .map(ResolvedClient::Registered)) + } + } + + pub async fn is_post_logout_redirect_uri_allowed( + &self, + client_id: &str, + uri: &str, + origin: RequestOrigin<'_>, + ) -> trc::Result { + if client_id.len() > CLIENT_ID_MAX_LEN + || client_id.starts_with(CLIENT_ID_HEADER) + || !is_safe_redirect_target(uri) + { + return Ok(false); + } + + let is_https = client_id.starts_with("https://"); + if !is_https && !client_id.starts_with("http://") { + return Ok(self + .registered_oauth_client(client_id) + .await? + .is_ok_and(|client| { + post_logout_redirect_uri_matches(client.post_logout_redirect_uris.iter(), uri) + })); + } + + if let Some(matches) = self.app_origin_policy(Some(origin)).and_then(|policy| { + self.inner + .data + .applications + .post_logout_redirect_uri_matches(client_id, policy, uri) + }) { + return Ok(matches); + } + + match self.registered_oauth_client(client_id).await? { + Err(ClientRejection::Unknown) if is_https => {} + result => { + return Ok(result.is_ok_and(|client| { + post_logout_redirect_uri_matches(client.post_logout_redirect_uris.iter(), uri) + })); + } + } + + let Ok(url) = self.check_cimd_policy(client_id) else { + return Ok(false); + }; + Ok(self.cimd_client(client_id, url).await?.is_ok_and(|client| { + post_logout_redirect_uri_matches(&client.post_logout_redirect_uris, uri) + })) + } + + pub fn app_origin_policy<'x>( + &'x self, + origin: Option>, + ) -> Option> { + if self.registry().is_bootstrap_mode() || self.registry().is_recovery_mode() { + origin.map(AppOriginPolicy::SameOrigin) + } else { + Some(AppOriginPolicy::TrustedHosts(&self.core.network.resources)) + } + } + + pub fn application_client_document( + &self, + prefix: &str, + origin: RequestOrigin<'_>, + ) -> AppClientDocument { + let applications = &self.inner.data.applications; + if self.registry().is_bootstrap_mode() || self.registry().is_recovery_mode() { + applications.client_document( + prefix, + origin.is_tls, + origin.authority, + AppOriginPolicy::SameOrigin(origin), + ) + } else { + applications.client_document( + prefix, + origin.is_tls || !cfg!(feature = "dev_mode"), + origin.authority, + AppOriginPolicy::TrustedHosts(&self.core.network.resources), + ) + } + } + + pub async fn external_issuer_for_host(&self, authority: &str) -> trc::Result> { + if self.registry().is_bootstrap_mode() || self.registry().is_recovery_mode() { + return Ok(None); + } + let host = match authority.strip_prefix('[') { + Some(_) => return Ok(None), + None => authority + .rsplit_once(':') + .map_or(authority, |(host, _)| host), + }; + Ok(self + .get_directory_for_host(host) + .await + .caused_by(trc::location!())? + .and_then(|directory| directory.oidc_discovery_document()) + .map(|discovery| discovery.document.issuer.clone())) + } + + async fn registered_oauth_client( + &self, + client_id: &str, + ) -> trc::Result, ClientRejection>> { + let Some(id) = self + .registry() + .primary_key( + ObjectType::OAuthClient.into(), + Property::ClientId, + client_id.as_bytes().to_vec(), + ) + .await + .caused_by(trc::location!())? + else { + return Ok(Err(ClientRejection::Unknown)); + }; + let Some(client) = self + .registry() + .object::(id.id()) + .await + .caused_by(trc::location!())? + else { + return Ok(Err(ClientRejection::Unknown)); + }; + + if client + .expires_at + .is_some_and(|expires_at| expires_at.timestamp() <= UTCDateTime::now().timestamp()) + { + Ok(Err(ClientRejection::Expired)) + } else { + Ok(Ok(Box::new(client))) + } + } +} + +impl AppOriginPolicy<'_> { + #[cfg(feature = "dev_mode")] + pub fn accepts(&self, _is_tls: bool, authority: &str) -> bool { + is_safe_authority(authority) + } + + #[cfg(not(feature = "dev_mode"))] + pub fn accepts(&self, is_tls: bool, authority: &str) -> bool { + match self { + AppOriginPolicy::TrustedHosts(resources) => { + is_tls && resources.is_trusted_host(authority) + } + AppOriginPolicy::SameOrigin(origin) => { + origin.is_tls == is_tls + && is_safe_authority(authority) + && origin.authority.eq_ignore_ascii_case(authority) + } + } + } +} + +impl RequestOrigin<'_> { + pub fn scheme(&self) -> &'static str { + if self.is_tls { "https" } else { "http" } + } +} + +impl ResolvedClient { + pub fn kind(&self) -> OAuthClientKind { + match self { + ResolvedClient::Stateless(_) => OAuthClientKind::Dynamic, + ResolvedClient::Registered(client) if client.first_party => OAuthClientKind::FirstParty, + ResolvedClient::Registered(_) => OAuthClientKind::Registered, + ResolvedClient::Application(_) => OAuthClientKind::FirstParty, + ResolvedClient::Cimd(_) => OAuthClientKind::MetadataDocument, + } + } + + pub fn is_first_party(&self) -> bool { + match self { + ResolvedClient::Application(_) => true, + ResolvedClient::Registered(client) => client.first_party, + ResolvedClient::Stateless(_) | ResolvedClient::Cimd(_) => false, + } + } + + pub fn requires_consent(&self) -> bool { + !matches!(self, ResolvedClient::Application(_)) + } + + pub fn is_verified(&self) -> bool { + match self { + #[cfg(not(feature = "enterprise"))] + ResolvedClient::Registered(_) => true, + ResolvedClient::Application(_) => true, + ResolvedClient::Stateless(_) | ResolvedClient::Cimd(_) => false, + } + } + + + pub fn display_name(&self) -> Option<&str> { + match self { + ResolvedClient::Stateless(meta) => meta.client_name.as_deref(), + ResolvedClient::Registered(client) => client.description.as_deref(), + ResolvedClient::Application(app) => app.client_name.as_deref(), + ResolvedClient::Cimd(client) => client.as_ref()?.client_name.as_deref(), + } + } + + pub fn display_logo(&self) -> Option<&str> { + match self { + ResolvedClient::Registered(client) => client + .logo + .as_deref() + .filter(|logo| logo.starts_with(LOGO_DATA_URI_PREFIX)), + ResolvedClient::Application(app) => app.logo_uri.as_deref(), + ResolvedClient::Stateless(_) | ResolvedClient::Cimd(_) => None, + } + } + + pub fn client_host(&self) -> Option<&str> { + match self { + ResolvedClient::Cimd(Some(client)) => Some(client.client_host.as_str()), + _ => None, + } + } + + pub fn redirect_uris(&self) -> &[String] { + match self { + ResolvedClient::Stateless(meta) => &meta.redirect_uris, + ResolvedClient::Registered(client) => client.redirect_uris.as_slice(), + ResolvedClient::Application(app) => &app.redirect_uris, + ResolvedClient::Cimd(Some(client)) => &client.redirect_uris, + ResolvedClient::Cimd(None) => &[], + } + } + + pub fn redirect_uri_matches(&self, redirect_uri: &str) -> bool { + is_safe_redirect_target(redirect_uri) + && self + .redirect_uris() + .iter() + .any(|registered| redirect_uri_matches(registered, redirect_uri)) + } + + pub fn allowed_scopes(&self) -> Option { + if self.is_first_party() { + return None; + } + let scopes = match self { + ResolvedClient::Stateless(meta) => meta.scopes, + ResolvedClient::Registered(client) => OAuthScopes::from_registry(client.scopes.iter()), + ResolvedClient::Cimd(Some(client)) => client.scopes, + ResolvedClient::Cimd(None) | ResolvedClient::Application(_) => OAuthScopes::NONE, + }; + Some(scopes | OAuthScopes::ALWAYS_ALLOWED) + } + + pub fn secret_hash(&self) -> Option<&str> { + match self { + ResolvedClient::Registered(client) => { + client.secret.as_deref().filter(|secret| !secret.is_empty()) + } + _ => None, + } + } + + pub fn is_confidential(&self) -> bool { + self.secret_hash().is_some() + } + + pub fn allows_grant(&self, grant: OAuthGrantType) -> bool { + match self { + ResolvedClient::Registered(client) => client.grant_types.contains(&grant), + ResolvedClient::Stateless(_) + | ResolvedClient::Application(_) + | ResolvedClient::Cimd(_) => grant != OAuthGrantType::DeviceCode, + } + } +} + +impl ClientRejection { + pub fn as_str(&self) -> &'static str { + match self { + ClientRejection::TooLong => "Client id is too long", + ClientRejection::Unknown => "Unknown client id", + ClientRejection::Expired => "Client registration has expired", + ClientRejection::Metadata(err) => err.as_str(), + } + } +} + diff --git a/crates/common/src/auth/oauth/client_id.rs b/crates/common/src/auth/oauth/client_id.rs index 525fab3a..4ec62636 100644 --- a/crates/common/src/auth/oauth/client_id.rs +++ b/crates/common/src/auth/oauth/client_id.rs @@ -4,40 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - SCOPE_CALENDARS, SCOPE_CONTACTS, SCOPE_MAIL, SCOPE_OFFLINE_ACCESS, SCOPE_OPENID, - crypto::SymmetricEncrypt, -}; -use base64::{Engine, engine::general_purpose}; +use super::{crypto::SymmetricEncrypt, scope::OAuthScopes}; +use encodify::base64::URL_SAFE_NO_PAD; use store::blake3; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; +use utils::codec::leb128::{Leb128Reader, Leb128Writer}; -const CLIENT_ID_HEADER: &str = "swc1."; +pub const CLIENT_ID_HEADER: &str = "swc1."; const CLIENT_ID_KEY_CONTEXT: &str = "stalwart-oauth-client-id-sw1"; const CLIENT_ID_VERSION: u8 = 1; -const SCOPE_BITS: &[&str] = &[ - SCOPE_OPENID, - SCOPE_OFFLINE_ACCESS, - SCOPE_MAIL, - SCOPE_CONTACTS, - SCOPE_CALENDARS, -]; - -pub fn scopes_to_mask(scope: &str) -> u64 { - let mut mask = 0u64; - for scope in scope.split_ascii_whitespace() { - if let Some(bit) = SCOPE_BITS.iter().position(|known| *known == scope) { - mask |= 1 << bit; - } - } - mask -} - #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ClientMeta { pub redirect_uris: Vec, - pub scope_mask: u64, + pub scopes: OAuthScopes, pub client_name: Option, } @@ -58,7 +37,7 @@ pub fn encode_client_id(key: &[u8], meta: &ClientMeta) -> Result payload.push_leb128(uri.len()); payload.extend_from_slice(uri.as_bytes()); } - payload.push_leb128(meta.scope_mask); + payload.push_leb128(u64::from(meta.scopes.bits())); payload.push_leb128(client_name.len()); payload.extend_from_slice(client_name.as_bytes()); @@ -73,13 +52,13 @@ pub fn encode_client_id(key: &[u8], meta: &ClientMeta) -> Result let mut out = String::with_capacity(CLIENT_ID_HEADER.len() + body.len().div_ceil(3) * 4); out.push_str(CLIENT_ID_HEADER); - general_purpose::URL_SAFE_NO_PAD.encode_string(&body, &mut out); + URL_SAFE_NO_PAD.encode_append(&body, &mut out); Ok(out) } pub fn decode_client_id(key: &[u8], client_id: &str) -> Option { - let body = general_purpose::URL_SAFE_NO_PAD + let body = URL_SAFE_NO_PAD .decode(client_id.strip_prefix(CLIENT_ID_HEADER)?.as_bytes()) .ok()?; if body.len() < SymmetricEncrypt::NONCE_LEN + SymmetricEncrypt::ENCRYPT_TAG_LEN { @@ -90,12 +69,12 @@ pub fn decode_client_id(key: &[u8], client_id: &str) -> Option { .decrypt_with_aad(ciphertext, nonce, &[]) .ok()?; - let mut bytes = payload.iter(); - if bytes.next().copied()? != CLIENT_ID_VERSION { + let (&version, mut bytes) = payload.split_first()?; + if version != CLIENT_ID_VERSION { return None; } - let uri_count: usize = bytes.next_leb128()?; + let uri_count: usize = bytes.read_leb128()?; if uri_count > u8::MAX as usize { return None; } @@ -103,123 +82,21 @@ pub fn decode_client_id(key: &[u8], client_id: &str) -> Option { for _ in 0..uri_count { redirect_uris.push(take_string(&mut bytes)?); } - let scope_mask: u64 = bytes.next_leb128()?; + let scopes = OAuthScopes::from_bits(u8::try_from(bytes.read_leb128::()?).ok()?); let client_name = take_string(&mut bytes)?; Some(ClientMeta { redirect_uris, - scope_mask, + scopes, client_name: (!client_name.is_empty()).then_some(client_name), }) } -fn take_string(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - let len: usize = bytes.next_leb128()?; - let slice = bytes.as_slice(); - if slice.len() < len { - return None; - } - let value = String::from_utf8(slice[..len].to_vec()).ok()?; - if len > 0 { - bytes.nth(len - 1)?; - } +fn take_string(bytes: &mut &[u8]) -> Option { + let len: usize = bytes.read_leb128()?; + let (value, rest) = bytes.split_at_checked(len)?; + let value = String::from_utf8(value.to_vec()).ok()?; + *bytes = rest; Some(value) } -#[cfg(test)] -mod tests { - use super::*; - - const KEY: &[u8] = b"a-test-encryption-key-of-some-length"; - - fn sample() -> ClientMeta { - ClientMeta { - redirect_uris: vec![ - "http://127.0.0.1/cb".to_string(), - "com.example.app:/oauth".to_string(), - ], - scope_mask: scopes_to_mask(&format!("{SCOPE_OFFLINE_ACCESS} {SCOPE_MAIL}")), - client_name: Some("Example Client".to_string()), - } - } - - #[test] - fn round_trip_preserves_all_fields() { - for meta in [ - sample(), - ClientMeta { - redirect_uris: vec!["http://[::1]/".to_string()], - scope_mask: 0, - client_name: None, - }, - ClientMeta::default(), - ] { - let client_id = encode_client_id(KEY, &meta).unwrap(); - assert!(client_id.starts_with(CLIENT_ID_HEADER)); - assert_eq!(decode_client_id(KEY, &client_id), Some(meta)); - } - } - - #[test] - fn scope_mask_is_order_independent_and_drops_unknown() { - assert_eq!( - scopes_to_mask(&format!("{SCOPE_MAIL} {SCOPE_OFFLINE_ACCESS}")), - scopes_to_mask(&format!("{SCOPE_OFFLINE_ACCESS} {SCOPE_MAIL}")) - ); - assert_eq!( - scopes_to_mask(&format!("{SCOPE_MAIL} custom:unknown")), - scopes_to_mask(SCOPE_MAIL) - ); - assert_eq!(scopes_to_mask("totally unknown"), 0); - } - - #[test] - fn identical_input_is_deterministic() { - let meta = sample(); - assert_eq!( - encode_client_id(KEY, &meta).unwrap(), - encode_client_id(KEY, &meta).unwrap() - ); - } - - #[test] - fn wrong_key_is_rejected() { - let client_id = encode_client_id(KEY, &sample()).unwrap(); - assert_eq!( - decode_client_id(b"a-completely-different-key-value!", &client_id), - None - ); - } - - #[test] - fn tampering_is_rejected() { - let client_id = encode_client_id(KEY, &sample()).unwrap(); - let (header, body_b64) = client_id.split_at(CLIENT_ID_HEADER.len()); - let mut body = general_purpose::URL_SAFE_NO_PAD.decode(body_b64).unwrap(); - for idx in 0..body.len() { - let mut tampered = body.clone(); - tampered[idx] ^= 0x01; - let forged = format!( - "{header}{}", - general_purpose::URL_SAFE_NO_PAD.encode(&tampered) - ); - assert_eq!(decode_client_id(KEY, &forged), None, "byte {idx}"); - } - body[0] ^= 0x00; - assert!(decode_client_id(KEY, &client_id).is_some()); - } - - #[test] - fn malformed_input_never_panics() { - for case in [ - "", - "swc1.", - "swc1.!!!", - "swc1.AAAA", - "wrong.AAAA", - "swc1.AAAAAAAAAAAAAAAAAAAAAAAAAAAA", - ] { - assert_eq!(decode_client_id(KEY, case), None, "{case:?}"); - } - } -} diff --git a/crates/common/src/auth/oauth/config.rs b/crates/common/src/auth/oauth/config.rs index 598d4c93..fcf790e6 100644 --- a/crates/common/src/auth/oauth/config.rs +++ b/crates/common/src/auth/oauth/config.rs @@ -4,42 +4,54 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{crypto::SymmetricEncrypt, token::TOKEN_KEY_CONTEXT}; use crate::{ config::{EcKeyCurve, build_ecdsa_pem, build_rsa_keypair}, manager::application::Resource, }; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use aws_lc_rs::{ + rand::SystemRandom, + signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair}, +}; +use encodify::base64::URL_SAFE_NO_PAD; use jsonwebtoken::{ - Algorithm, EncodingKey, + Algorithm, DecodingKey, EncodingKey, jwk::{ AlgorithmParameters, CommonParameters, EllipticCurve, EllipticCurveKeyParameters, - EllipticCurveKeyType, Jwk, JwkSet, KeyAlgorithm, OctetKeyParameters, OctetKeyType, - PublicKeyUse, RSAKeyParameters, RSAKeyType, + EllipticCurveKeyType, Jwk, JwkSet, KeyAlgorithm, PublicKeyUse, RSAKeyParameters, + RSAKeyType, }, }; -use registry::schema::{enums::JwtSignatureAlgorithm, prelude::ObjectType, structs::OidcProvider}; +use registry::schema::{ + enums::{JwtSignatureAlgorithm, OAuthClientMetadataPolicy}, + prelude::ObjectType, + structs::OidcProvider, +}; use std::borrow::Cow; use store::{ + blake3, rand::{RngExt, distr::Alphanumeric, rng}, registry::bootstrap::Bootstrap, }; +const REFRESH_NONCE_KEY_CONTEXT: &str = "stalwart-oauth-refresh-nonce-sw1"; + #[derive(Clone)] pub struct OAuthConfig { pub oauth_key: String, + pub token_cipher: SymmetricEncrypt, + pub refresh_nonce_key: [u8; 32], pub oauth_expiry_user_code: u64, pub oauth_expiry_auth_code: u64, pub oauth_expiry_token: u64, pub oauth_expiry_refresh_token: u64, - pub oauth_expiry_refresh_token_renew: u64, - pub oauth_max_auth_attempts: u32, - - pub allow_anonymous_client_registration: bool, - pub require_client_authentication: bool, + pub client_metadata_documents: OAuthClientMetadataPolicy, pub oidc_expiry_id_token: u64, pub oidc_signing_secret: EncodingKey, + pub oidc_verifying_key: Option, pub oidc_signature_algorithm: Algorithm, + pub oidc_signature_algorithm_name: &'static str, pub oidc_jwks: Resource>, } @@ -47,76 +59,94 @@ impl OAuthConfig { pub async fn parse(bp: &mut Bootstrap) -> Self { let auth = bp.setting_infallible::().await; - let oidc_signature_algorithm = match auth.signature_algorithm { - JwtSignatureAlgorithm::Es256 => Algorithm::ES256, - JwtSignatureAlgorithm::Es384 => Algorithm::ES384, - JwtSignatureAlgorithm::Ps256 => Algorithm::PS256, - JwtSignatureAlgorithm::Ps384 => Algorithm::PS384, - JwtSignatureAlgorithm::Ps512 => Algorithm::PS512, - JwtSignatureAlgorithm::Rs256 => Algorithm::RS256, - JwtSignatureAlgorithm::Rs384 => Algorithm::RS384, - JwtSignatureAlgorithm::Rs512 => Algorithm::RS512, - JwtSignatureAlgorithm::Hs256 => Algorithm::HS256, - JwtSignatureAlgorithm::Hs384 => Algorithm::HS384, - JwtSignatureAlgorithm::Hs512 => Algorithm::HS512, - }; + let (mut oidc_signature_algorithm, mut oidc_signature_algorithm_name) = + match auth.signature_algorithm { + JwtSignatureAlgorithm::Es256 => (Algorithm::ES256, "ES256"), + JwtSignatureAlgorithm::Es384 => (Algorithm::ES384, "ES384"), + JwtSignatureAlgorithm::Ps256 => (Algorithm::PS256, "PS256"), + JwtSignatureAlgorithm::Ps384 => (Algorithm::PS384, "PS384"), + JwtSignatureAlgorithm::Ps512 => (Algorithm::PS512, "PS512"), + JwtSignatureAlgorithm::Rs256 => (Algorithm::RS256, "RS256"), + JwtSignatureAlgorithm::Rs384 => (Algorithm::RS384, "RS384"), + JwtSignatureAlgorithm::Rs512 => (Algorithm::RS512, "RS512"), + JwtSignatureAlgorithm::Hs256 => (Algorithm::HS256, "HS256"), + JwtSignatureAlgorithm::Hs384 => (Algorithm::HS384, "HS384"), + JwtSignatureAlgorithm::Hs512 => (Algorithm::HS512, "HS512"), + }; - let rand_key = rng() - .sample_iter(Alphanumeric) - .take(64) - .map(char::from) - .collect::(); - - let signature_key = auth - .signature_key - .secret() - .await - .map_err(|err| { - bp.build_error(ObjectType::OidcProvider.singleton(), err); - }) - .unwrap_or(Cow::Borrowed(rand_key.as_str())); - - let fallback_key = || { - ( - EncodingKey::from_secret(rand_key.as_bytes()), - AlgorithmParameters::OctetKey(OctetKeyParameters { - key_type: OctetKeyType::Octet, - value: URL_SAFE_NO_PAD.encode(&rand_key), - }) - .into(), - ) - }; - - let (oidc_signing_secret, algorithm) = match oidc_signature_algorithm { - Algorithm::HS256 | Algorithm::HS384 | Algorithm::HS512 => { - (EncodingKey::from_secret(signature_key.as_bytes()), None) + let is_recovery_mode = bp.registry.is_recovery_mode(); + let is_hmac = matches!( + oidc_signature_algorithm, + Algorithm::HS256 | Algorithm::HS384 | Algorithm::HS512 + ); + let mut signature_key = match auth.signature_key.secret().await { + Ok(key) if !key.is_empty() => key, + Ok(_) => { + if is_hmac && !is_recovery_mode { + bp.build_error( + ObjectType::OidcProvider.singleton(), + "OIDC signature key is empty, using a random key", + ); + } + Cow::Owned(random_secret()) } + Err(err) => { + bp.build_error(ObjectType::OidcProvider.singleton(), err); + Cow::Owned(random_secret()) + } + }; + + let asymmetric_key = match oidc_signature_algorithm { + Algorithm::HS256 | Algorithm::HS384 | Algorithm::HS512 => None, Algorithm::RS256 | Algorithm::RS384 | Algorithm::RS512 | Algorithm::PS256 | Algorithm::PS384 - | Algorithm::PS512 => parse_rsa_key(&auth) - .await - .map_err(|err| { - bp.build_error(ObjectType::OidcProvider.singleton(), err); - }) - .map(|(secret, alg)| (secret, Some(alg))) - .unwrap_or_else(|_| fallback_key()), - Algorithm::ES256 | Algorithm::ES384 => parse_ecdsa_key(&auth, oidc_signature_algorithm) - .await - .map_err(|err| { - bp.build_error(ObjectType::OidcProvider.singleton(), err); - }) - .map(|(secret, alg)| (secret, Some(alg))) - .unwrap_or_else(|_| fallback_key()), - _ => { - bp.build_error( - ObjectType::OidcProvider.singleton(), - format!("Unsupported OIDC signature algorithm {oidc_signature_algorithm:?}"), - ); - fallback_key() + | Algorithm::PS512 => Some(parse_rsa_key(&auth).await), + Algorithm::ES256 | Algorithm::ES384 => { + Some(parse_ecdsa_key(&auth, oidc_signature_algorithm).await) } + _ => Some(Err(format!( + "Unsupported OIDC signature algorithm {oidc_signature_algorithm:?}" + ))), + }; + + let (oidc_signing_secret, algorithm) = match asymmetric_key { + None => (EncodingKey::from_secret(signature_key.as_bytes()), None), + Some(Ok((secret, params))) => (secret, Some(params)), + Some(Err(err)) => match ephemeral_ecdsa_key() { + Ok((secret, params)) => { + bp.build_error( + ObjectType::OidcProvider.singleton(), + format!("{err}; using a temporary ES256 signing key"), + ); + oidc_signature_algorithm = Algorithm::ES256; + oidc_signature_algorithm_name = "ES256"; + (secret, Some(params)) + } + Err(ephemeral_err) => { + bp.build_error( + ObjectType::OidcProvider.singleton(), + format!( + "{err}; failed to generate a temporary signing key ({ephemeral_err}), using a random HS256 key" + ), + ); + oidc_signature_algorithm = Algorithm::HS256; + oidc_signature_algorithm_name = "HS256"; + signature_key = Cow::Owned(random_secret()); + (EncodingKey::from_secret(signature_key.as_bytes()), None) + } + }, + }; + + let oidc_verifying_key = match &algorithm { + Some(params) => DecodingKey::from_jwk(&Jwk { + common: CommonParameters::default(), + algorithm: params.clone(), + }) + .ok(), + None => Some(DecodingKey::from_secret(signature_key.as_bytes())), }; let oidc_jwks = Resource { @@ -139,24 +169,37 @@ impl OAuthConfig { .into_bytes(), }; + let oauth_key = match auth.encryption_key.secret().await { + Ok(key) if !key.is_empty() => key.into_owned(), + Ok(_) => { + if !is_recovery_mode { + bp.build_error( + ObjectType::OidcProvider.singleton(), + "OAuth encryption key is empty, using a random key", + ); + } + random_secret() + } + Err(err) => { + bp.build_error(ObjectType::OidcProvider.singleton(), err); + random_secret() + } + }; + OAuthConfig { - oauth_key: auth - .encryption_key - .secret() - .await - .map_err(|err| bp.build_error(ObjectType::OidcProvider.singleton(), err)) - .map_or_else(|_| rand_key.clone(), Cow::into_owned), + token_cipher: SymmetricEncrypt::new(oauth_key.as_bytes(), TOKEN_KEY_CONTEXT), + refresh_nonce_key: blake3::derive_key(REFRESH_NONCE_KEY_CONTEXT, oauth_key.as_bytes()), + oauth_key, oauth_expiry_user_code: auth.user_code_expiry.as_secs(), oauth_expiry_auth_code: auth.auth_code_expiry.as_secs(), oauth_expiry_token: auth.access_token_expiry.as_secs(), oauth_expiry_refresh_token: auth.refresh_token_expiry.as_secs(), - oauth_expiry_refresh_token_renew: auth.refresh_token_renewal.as_secs(), - oauth_max_auth_attempts: auth.auth_code_max_attempts as u32, + client_metadata_documents: auth.client_metadata_documents, oidc_expiry_id_token: auth.id_token_expiry.as_secs(), - allow_anonymous_client_registration: auth.anonymous_client_registration, - require_client_authentication: auth.require_client_registration, oidc_signing_secret, + oidc_verifying_key, oidc_signature_algorithm, + oidc_signature_algorithm_name, oidc_jwks, } } @@ -187,7 +230,26 @@ async fn parse_ecdsa_key( _ => unreachable!(), }; - let ecdsa_key = build_ecdsa_pem(ec_curve, auth.signature_key.secret().await?.as_ref())?; + ecdsa_key(curve, ec_curve, auth.signature_key.secret().await?.as_ref()) +} + +fn ephemeral_ecdsa_key() -> Result<(EncodingKey, AlgorithmParameters), String> { + let pkcs8 = + EcdsaKeyPair::generate_pkcs8(&ECDSA_P256_SHA256_FIXED_SIGNING, &SystemRandom::new()) + .map_err(|err| format!("Failed to generate ECDSA key: {err}"))?; + ecdsa_key( + EllipticCurve::P256, + EcKeyCurve::P256, + &pem::encode(&pem::Pem::new("PRIVATE KEY", pkcs8.as_ref())), + ) +} + +fn ecdsa_key( + curve: EllipticCurve, + ec_curve: EcKeyCurve, + pem: &str, +) -> Result<(EncodingKey, AlgorithmParameters), String> { + let ecdsa_key = build_ecdsa_pem(ec_curve, pem)?; let ecdsa_key_params = EllipticCurveKeyParameters { key_type: EllipticCurveKeyType::EC, @@ -201,3 +263,12 @@ async fn parse_ecdsa_key( AlgorithmParameters::EllipticCurve(ecdsa_key_params), )) } + +fn random_secret() -> String { + rng() + .sample_iter(Alphanumeric) + .take(64) + .map(char::from) + .collect() +} + diff --git a/crates/common/src/auth/oauth/crypto.rs b/crates/common/src/auth/oauth/crypto.rs index dd8927e9..5b40225c 100644 --- a/crates/common/src/auth/oauth/crypto.rs +++ b/crates/common/src/auth/oauth/crypto.rs @@ -10,6 +10,7 @@ use aes_gcm_siv::{ }; use store::blake3; +#[derive(Clone)] pub struct SymmetricEncrypt { aes: Aes256GcmSiv, } diff --git a/crates/common/src/auth/oauth/grant.rs b/crates/common/src/auth/oauth/grant.rs new file mode 100644 index 00000000..3da291c2 --- /dev/null +++ b/crates/common/src/auth/oauth/grant.rs @@ -0,0 +1,462 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{scope::OAuthScopes, token::TokenGrant}; +use crate::{ + Server, + auth::{AccessToken, GrantClient, RECOVERY_ADMIN_ID, TokenScope}, + cache::invalidate::CacheInvalidationBuilder, + ipc::CacheInvalidation, +}; +use registry::{ + schema::{ + enums::OAuthClientKind, + prelude::{Object, ObjectInner, ObjectType}, + structs::OAuthGrant, + }, + types::{datetime::UTCDateTime, id::ObjectId, ipaddr::IpAddr}, +}; +use std::net::IpAddr as StdIpAddr; +use store::{ + registry::{ + RegistryQuery, + write::{RegistryWrite, RegistryWriteResult}, + }, + write::now, +}; +use trc::AddContext; +use types::id::Id; + +pub const MAX_OAUTH_GRANTS: usize = 256; +pub const RECOVERY_ADMIN_GRANT_ID: u64 = 1 << 62; +const MAX_WRITE_ATTEMPTS: usize = 3; + +pub struct NewOAuthGrant<'x> { + pub account_id: u32, + pub client_id: &'x str, + pub client_name: Option<&'x str>, + pub client_kind: OAuthClientKind, + pub scopes: OAuthScopes, + pub remote_ip: StdIpAddr, +} + +pub struct StoredOAuthGrant { + pub id: u64, + pub revision: u64, + pub grant: Box, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrantDeletion { + Unconditional, + IfUnchanged, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrantRotation { + Issue { next: TokenGrant, rotated_at: u64 }, + Revoked, + Rejected, +} + +impl StoredOAuthGrant { + pub fn account_id(&self) -> u32 { + self.grant.account_id.document_id() + } + + fn object_id(&self) -> ObjectId { + ObjectId::new(ObjectType::OAuthGrant, Id::new(self.id)) + } + + pub fn is_current(&self, presented: u64) -> bool { + matches!( + self.rotation(presented), + RotationStep::Retry | RotationStep::Rotate { .. } + ) + } + + fn rotation(&self, presented: u64) -> RotationStep { + let current = self.grant.generation; + if presented == current { + if self.grant.pending { + RotationStep::Retry + } else { + RotationStep::Rotate { + stored: current, + next: current.saturating_add(1), + } + } + } else if self.grant.pending && presented == current.saturating_add(1) { + RotationStep::Rotate { + stored: presented, + next: presented.saturating_add(1), + } + } else if presented < current { + RotationStep::Replay + } else { + RotationStep::Invalid + } + } +} + +impl TokenScope { + pub fn from_grant(grant_id: u64, grant: &OAuthGrant) -> Self { + TokenScope::Grant { + grant_id, + scopes: OAuthScopes::from_registry(grant.scopes.iter()), + client: if grant.client_kind == OAuthClientKind::FirstParty { + GrantClient::FirstParty + } else { + GrantClient::ThirdParty + }, + } + } +} + +fn prune_selection(last_used: &[i64], now: i64, idle_timeout: i64) -> Vec { + let mut prune = last_used + .iter() + .map(|last_used| last_used.saturating_add(idle_timeout) <= now) + .collect::>(); + let mut live = last_used + .iter() + .zip(&prune) + .enumerate() + .filter(|(_, (_, expired))| !**expired) + .map(|(idx, (last_used, _))| (*last_used, idx)) + .collect::>(); + if live.len() >= MAX_OAUTH_GRANTS { + live.sort_unstable(); + let excess = live.len() + 1 - MAX_OAUTH_GRANTS; + for (_, idx) in live.into_iter().take(excess) { + prune[idx] = true; + } + } + prune +} + +enum RotationStep { + Retry, + Rotate { stored: u64, next: u64 }, + Replay, + Invalid, +} + +impl Server { + pub async fn oauth_grant(&self, grant_id: u64) -> trc::Result> { + self.registry() + .get(ObjectId::new(ObjectType::OAuthGrant, Id::new(grant_id))) + .await + .caused_by(trc::location!()) + .map(|object| { + object.and_then(|object| match object.inner { + ObjectInner::OAuthGrant(grant) => Some(StoredOAuthGrant { + id: grant_id, + revision: object.revision, + grant, + }), + _ => None, + }) + }) + } + + pub async fn oauth_grant_scope( + &self, + grant_id: u64, + account_id: u32, + ) -> trc::Result { + match self.oauth_grant(grant_id).await? { + Some(stored) if stored.account_id() == account_id => { + Ok(TokenScope::from_grant(grant_id, &stored.grant)) + } + None if account_id == RECOVERY_ADMIN_ID && grant_id == RECOVERY_ADMIN_GRANT_ID => { + Ok(TokenScope::Grant { + grant_id, + scopes: OAuthScopes::ALL, + client: GrantClient::FirstParty, + }) + } + _ => Err(trc::AuthEvent::TokenExpired + .into_err() + .account_id(account_id) + .id(grant_id) + .details("OAuth grant revoked")), + } + } + + pub async fn renew_access_token( + &self, + access_token: &AccessToken, + remote_ip: StdIpAddr, + ) -> trc::Result { + let account_id = access_token.account_id(); + let inner = self.access_token(account_id).await?; + let token_scope = match access_token.token_scope() { + Some(TokenScope::Grant { grant_id, .. }) => { + Some(self.oauth_grant_scope(grant_id, account_id).await?) + } + token_scope => token_scope, + }; + AccessToken::renew(inner, access_token.credential_id(), token_scope, remote_ip) + } + + pub async fn refresh_access_token( + &self, + access_token: &AccessToken, + remote_ip: StdIpAddr, + ) -> trc::Result> { + let inner = self.access_token(access_token.account_id()).await?; + if access_token.is_built_from(&inner) { + Ok(None) + } else { + AccessToken::renew( + inner, + access_token.credential_id(), + access_token.token_scope(), + remote_ip, + ) + .map(Some) + } + } + + pub async fn create_oauth_grant(&self, new: NewOAuthGrant<'_>) -> trc::Result { + let mut invalidator = CacheInvalidationBuilder::default(); + self.prune_oauth_grants(new.account_id, &mut invalidator) + .await?; + + let now = UTCDateTime::now(); + let grant = OAuthGrant { + account_id: Id::from(new.account_id), + client_id: new.client_id.to_string(), + client_name: new + .client_name + .filter(|name| !name.is_empty()) + .map(str::to_string), + client_kind: new.client_kind, + scopes: new.scopes.to_registry(), + remote_ip: Some(IpAddr(new.remote_ip)), + created_at: now, + last_used_at: now, + generation: 0, + pending: false, + }; + let object = Object::new(grant.into()); + + let result = self + .registry() + .write(RegistryWrite::insert(&object)) + .await + .caused_by(trc::location!())?; + self.invalidate_caches(invalidator).await?; + + match result { + RegistryWriteResult::Success(id) => Ok(id.id()), + failure => Err(trc::AuthEvent::Error + .into_err() + .account_id(new.account_id) + .details("Failed to create OAuth grant") + .reason(failure) + .caused_by(trc::location!())), + } + } + + async fn prune_oauth_grants( + &self, + account_id: u32, + invalidator: &mut CacheInvalidationBuilder, + ) -> trc::Result<()> { + let ids = self + .registry() + .query::>(RegistryQuery::new(ObjectType::OAuthGrant).with_account(account_id)) + .await + .caused_by(trc::location!())?; + if ids.is_empty() { + return Ok(()); + } + + let mut grants = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(stored) = self.oauth_grant(id.id()).await? { + grants.push(stored); + } + } + + let last_used = grants + .iter() + .map(|stored| stored.grant.last_used_at.timestamp()) + .collect::>(); + let prune = prune_selection( + &last_used, + now() as i64, + self.core.oauth.oauth_expiry_refresh_token as i64, + ); + for (stored, prune) in grants.into_iter().zip(prune) { + if prune { + self.delete_oauth_grant(stored, GrantDeletion::IfUnchanged, invalidator) + .await?; + } + } + + Ok(()) + } + + pub async fn revoke_oauth_grants( + &self, + account_id: u32, + invalidator: &mut CacheInvalidationBuilder, + ) -> trc::Result<()> { + for id in self + .registry() + .query::>(RegistryQuery::new(ObjectType::OAuthGrant).with_account(account_id)) + .await + .caused_by(trc::location!())? + { + if let Some(stored) = self.oauth_grant(id.id()).await? { + self.delete_oauth_grant(stored, GrantDeletion::Unconditional, invalidator) + .await?; + } + } + Ok(()) + } + + pub async fn delete_oauth_grant( + &self, + mut stored: StoredOAuthGrant, + mode: GrantDeletion, + invalidator: &mut CacheInvalidationBuilder, + ) -> trc::Result { + for _ in 0..MAX_WRITE_ATTEMPTS { + let account_id = stored.account_id(); + let object_id = stored.object_id(); + let object = + Object::with_revision(ObjectInner::OAuthGrant(stored.grant), stored.revision); + match self + .registry() + .write(RegistryWrite::delete_object(object_id, &object)) + .await + { + Ok(RegistryWriteResult::Success(_)) => { + invalidator.invalidate(CacheInvalidation::OAuthGrant { + account_id, + grant_id: stored.id, + }); + return Ok(true); + } + Ok(RegistryWriteResult::NotFound { .. }) => return Ok(false), + Ok(failure) => { + return Err(trc::AuthEvent::Error + .into_err() + .id(stored.id) + .details("Failed to delete OAuth grant") + .reason(failure) + .caused_by(trc::location!())); + } + Err(err) if err.is_assertion_failure() => { + if mode == GrantDeletion::IfUnchanged { + return Ok(false); + } + match self.oauth_grant(stored.id).await? { + Some(current) => stored = current, + None => return Ok(false), + } + } + Err(err) => return Err(err.caused_by(trc::location!())), + } + } + + Err(trc::StoreEvent::AssertValueFailed + .into_err() + .id(stored.id) + .details("OAuth grant changed concurrently") + .caused_by(trc::location!())) + } + + pub async fn rotate_oauth_grant( + &self, + mut stored: StoredOAuthGrant, + presented_generation: u64, + ) -> trc::Result { + for _ in 0..MAX_WRITE_ATTEMPTS { + let (stored_generation, next_generation) = match stored.rotation(presented_generation) { + RotationStep::Retry => { + return Ok(GrantRotation::Issue { + next: TokenGrant { + id: stored.id, + generation: stored.grant.generation.saturating_add(1), + }, + rotated_at: stored.grant.last_used_at.timestamp() as u64, + }); + } + RotationStep::Rotate { stored, next } => (stored, next), + RotationStep::Replay => { + trc::event!( + Auth(trc::AuthEvent::Failed), + AccountId = stored.account_id(), + Id = stored.id, + Reason = "Superseded refresh token reused, revoking OAuth grant", + ); + let mut invalidator = CacheInvalidationBuilder::default(); + self.delete_oauth_grant(stored, GrantDeletion::Unconditional, &mut invalidator) + .await?; + self.invalidate_caches(invalidator).await?; + return Ok(GrantRotation::Revoked); + } + RotationStep::Invalid => return Ok(GrantRotation::Rejected), + }; + + let now = UTCDateTime::now(); + let mut updated = stored.grant.clone(); + updated.generation = stored_generation; + updated.pending = true; + updated.last_used_at = now; + let old_object = + Object::with_revision(ObjectInner::OAuthGrant(stored.grant), stored.revision); + let new_object = Object::new(ObjectInner::OAuthGrant(updated)); + + match self + .registry() + .write(RegistryWrite::update( + Id::new(stored.id), + &new_object, + &old_object, + )) + .await + { + Ok(RegistryWriteResult::Success(_)) => { + return Ok(GrantRotation::Issue { + next: TokenGrant { + id: stored.id, + generation: next_generation, + }, + rotated_at: now.timestamp() as u64, + }); + } + Ok(failure) => { + return Err(trc::AuthEvent::Error + .into_err() + .id(stored.id) + .details("Failed to rotate OAuth grant") + .reason(failure) + .caused_by(trc::location!())); + } + Err(err) if err.is_assertion_failure() => { + match self.oauth_grant(stored.id).await? { + Some(current) => stored = current, + None => return Ok(GrantRotation::Rejected), + } + } + Err(err) => return Err(err.caused_by(trc::location!())), + } + } + + Err(trc::StoreEvent::AssertValueFailed + .into_err() + .id(stored.id) + .details("OAuth grant changed concurrently") + .caused_by(trc::location!())) + } +} + diff --git a/crates/common/src/auth/oauth/introspect.rs b/crates/common/src/auth/oauth/introspect.rs index 39a4c3ba..634a493f 100644 --- a/crates/common/src/auth/oauth/introspect.rs +++ b/crates/common/src/auth/oauth/introspect.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Server, auth::AccessToken}; +use super::{GrantType, scope::OAuthScopes}; +use crate::Server; +use registry::schema::enums::OAuthClientKind; use serde::{Deserialize, Serialize}; -use trc::{AddContext, AuthEvent, EventType}; +use trc::{AuthEvent, EventType}; #[derive(Debug, Default, Clone, Eq, PartialEq, Deserialize, Serialize)] pub struct OAuthIntrospect { @@ -47,26 +49,9 @@ pub struct OAuthIntrospect { } impl Server { - pub async fn introspect_access_token( - &self, - token: &str, - access_token: &AccessToken, - ) -> trc::Result { - match self.validate_access_token(None, token).await { - Ok(token_info) => Ok(OAuthIntrospect { - active: true, - username: self - .account(access_token.account_id()) - .await - .caused_by(trc::location!())? - .name() - .to_string() - .into(), - token_type: Some("bearer".into()), - exp: Some(token_info.expiry as i64), - iat: Some(token_info.issued_at as i64), - ..Default::default() - }), + pub async fn introspect_access_token(&self, token: &str) -> trc::Result { + match self.introspect_grant_token(token).await { + Ok(introspect) => Ok(introspect), Err(err) if matches!( err.event_type(), @@ -75,7 +60,51 @@ impl Server { { Ok(OAuthIntrospect::default()) } - Err(err) => Err(err), + Err(err) => Err(err.caused_by(trc::location!())), } } + + async fn introspect_grant_token(&self, token: &str) -> trc::Result { + let token_info = self.validate_access_token(None, token).await?; + let token_type = match token_info.grant_type { + GrantType::AccessToken => "bearer", + GrantType::RefreshToken => "refresh_token", + _ => return Ok(OAuthIntrospect::default()), + }; + let Some(presented) = token_info.grant else { + return Ok(OAuthIntrospect::default()); + }; + let Some(stored) = self + .oauth_grant(presented.id) + .await? + .filter(|stored| stored.account_id() == token_info.account_id) + else { + return Ok(OAuthIntrospect::default()); + }; + if (token_info.grant_type == GrantType::RefreshToken + && !stored.is_current(presented.generation)) + || (token_info.credential_expired + && stored.grant.client_kind != OAuthClientKind::FirstParty) + { + return Ok(OAuthIntrospect::default()); + } + + let scope = match token_info.claims.filter(|claims| !claims.is_empty()) { + Some(scope) if token_info.grant_type == GrantType::RefreshToken => scope, + _ => OAuthScopes::from_registry(stored.grant.scopes.iter()).to_wire(), + }; + let account = self.account(token_info.account_id).await?; + + Ok(OAuthIntrospect { + active: true, + scope: (!scope.is_empty()).then_some(scope), + client_id: stored.grant.client_id.into(), + username: account.name().to_string().into(), + sub: token_info.account_id.to_string().into(), + token_type: Some(token_type.into()), + exp: Some(token_info.expiry as i64), + iat: Some(token_info.issued_at as i64), + ..Default::default() + }) + } } diff --git a/crates/common/src/auth/oauth/mod.rs b/crates/common/src/auth/oauth/mod.rs index 2beb8192..4a2f28a0 100644 --- a/crates/common/src/auth/oauth/mod.rs +++ b/crates/common/src/auth/oauth/mod.rs @@ -4,12 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod cimd; +pub mod client; pub mod client_id; pub mod config; pub mod crypto; +pub mod grant; pub mod introspect; pub mod oidc; pub mod registration; +pub mod scope; pub mod token; pub const DEVICE_CODE_LEN: usize = 40; @@ -24,15 +28,24 @@ pub const SCOPE_OFFLINE_ACCESS: &str = "offline_access"; pub const SCOPE_MAIL: &str = "urn:ietf:params:oauth:scope:mail"; pub const SCOPE_CONTACTS: &str = "urn:ietf:params:oauth:scope:contacts"; pub const SCOPE_CALENDARS: &str = "urn:ietf:params:oauth:scope:calendars"; +pub const SCOPE_FILES: &str = "urn:ietf:params:oauth:scope:files"; -pub const SUPPORTED_SCOPES: &[&str] = &[ +const SUPPORTED_SCOPES: &[&str] = &[ SCOPE_OPENID, SCOPE_OFFLINE_ACCESS, SCOPE_MAIL, SCOPE_CONTACTS, SCOPE_CALENDARS, + SCOPE_FILES, ]; +pub fn supported_scopes(serves_files: bool) -> &'static [&'static str] { + match SUPPORTED_SCOPES.split_last() { + Some((_, without_files)) if !serves_files => without_files, + _ => SUPPORTED_SCOPES, + } +} + #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum GrantType { AccessToken, diff --git a/crates/common/src/auth/oauth/oidc.rs b/crates/common/src/auth/oauth/oidc.rs index 48b97dbc..8ebfc917 100644 --- a/crates/common/src/auth/oauth/oidc.rs +++ b/crates/common/src/auth/oauth/oidc.rs @@ -6,7 +6,7 @@ use std::fmt; -use jsonwebtoken::Header; +use jsonwebtoken::{Header, Validation}; use serde::{ Deserialize, Deserializer, Serialize, @@ -93,9 +93,13 @@ pub struct StandardClaims { #[serde(default)] pub email: Option, + #[serde(default, deserialize_with = "any_bool")] + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub email_verified: bool, + #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] - pub description: Option, + pub name: Option, } #[derive(Serialize)] @@ -111,7 +115,25 @@ struct IdTokenClaims { private: StandardClaims, } +#[derive(Debug, Deserialize)] +pub struct IdTokenHint { + pub sub: String, + pub aud: String, +} + impl Server { + pub fn verify_id_token_hint(&self, token: &str, issuer: &str) -> Option { + let key = self.core.oauth.oidc_verifying_key.as_ref()?; + let mut validation = Validation::new(self.core.oauth.oidc_signature_algorithm); + validation.validate_exp = false; + validation.validate_aud = false; + validation.set_issuer(&[issuer]); + validation.set_required_spec_claims(&["iss", "sub", "aud"]); + jsonwebtoken::decode::(token, key, &validation) + .ok() + .map(|data| data.claims) + } + pub fn issue_id_token( &self, subject: impl Into, diff --git a/crates/common/src/auth/oauth/registration.rs b/crates/common/src/auth/oauth/registration.rs index 9cd11d6c..684e58e2 100644 --- a/crates/common/src/auth/oauth/registration.rs +++ b/crates/common/src/auth/oauth/registration.rs @@ -4,8 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::CLIENT_ID_MAX_LEN; +use registry::schema::{ + enums::{OAuthGrantType, OAuthScope}, + prelude::Property, + structs::OAuthClient, +}; +use reqwest::Url; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; #[derive(Serialize, Deserialize, Debug, Default)] #[serde(rename_all = "snake_case")] @@ -131,10 +137,6 @@ pub struct ClientRegistrationRequest { #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub request_uris: Vec, - - #[serde(flatten)] - #[serde(skip_serializing_if = "HashMap::is_empty")] - pub additional_fields: HashMap, } #[derive(Serialize, Deserialize, Debug, Default)] @@ -205,35 +207,48 @@ impl ClientRegistrationError { error_description: Some(description), } } + + pub fn for_post_logout_redirect_uri(self) -> Self { + ClientRegistrationError { + error: "invalid_client_metadata", + error_description: self.error_description, + } + } } +pub const GRANT_TYPE_DEVICE_CODE: &str = "urn:ietf:params:oauth:grant-type:device_code"; +pub const CLIENT_NAME_MAX_LEN: usize = 256; +const HTTPS_PREFIX: &str = "https://"; + pub fn loopback_redirect_parts(uri: &str) -> Option<(&str, &str)> { let uri = uri.strip_prefix("http://")?; for host in ["127.0.0.1", "[::1]"] { if let Some(rest) = uri.strip_prefix(host) { - let rest = match rest.strip_prefix(':') { - Some(after_colon) => { - let rest = after_colon.trim_start_matches(|c: char| c.is_ascii_digit()); - if rest.len() == after_colon.len() { - return None; - } - rest - } - None => rest, - }; - return if let Some(path) = rest.strip_prefix('/') { - Some((host, path)) - } else if rest.is_empty() || rest.starts_with('?') { - Some((host, rest)) - } else { - None - }; + if let Some(path) = rest.strip_prefix('/') { + return Some((host, path)); + } else if let Some(after_colon) = rest.strip_prefix(':') + && let Some((port, path)) = after_colon.split_once('/') + && port.bytes().all(|b| b.is_ascii_digit()) + && port.parse::().is_ok() + { + return Some((host, path)); + } } } None } +pub fn post_logout_redirect_uri_matches<'x>( + registered: impl IntoIterator, + presented: &str, +) -> bool { + is_safe_redirect_target(presented) + && registered + .into_iter() + .any(|registered| registered == presented) +} + pub fn redirect_uri_matches(registered: &str, presented: &str) -> bool { registered == presented || matches!( @@ -245,33 +260,274 @@ pub fn redirect_uri_matches(registered: &str, presented: &str) -> bool { ) } -pub fn validate_redirect_uri(uri: &str) -> Result<(), ClientRegistrationError> { +fn uri_scheme(uri: &str) -> Option<&str> { + uri.split_once(':') + .map(|(scheme, _)| scheme) + .filter(|scheme| { + scheme + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphabetic) + && scheme + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+')) + }) +} + +fn is_private_use_redirect(uri: &str) -> bool { + uri_scheme(uri).is_some_and(|scheme| scheme.contains('.')) +} + +fn is_dangerous_scheme(scheme: &str) -> bool { + hashify::set_ignore_case!( + scheme.as_bytes(), + "javascript", + "data", + "vbscript", + "file", + "blob", + "about", + ) +} + +pub fn is_safe_redirect_target(uri: &str) -> bool { + !uri.bytes() + .any(|b| b.is_ascii_whitespace() || b.is_ascii_control() || b == b'\\') + && uri_scheme(uri).is_some_and(|scheme| !is_dangerous_scheme(scheme)) +} + +pub fn check_client_name(name: &str) -> Result<(), &'static str> { + if name.len() > CLIENT_NAME_MAX_LEN { + Err("client_name must not exceed 256 bytes.") + } else if name.chars().any(|ch| { + ch.is_control() || matches!(ch, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') + }) { + Err("client_name must not contain control or bidirectional formatting characters.") + } else { + Ok(()) + } +} + +fn check_redirect_uri_syntax(uri: &str) -> Result<(), ClientRegistrationError> { if uri.contains('#') { - return Err(ClientRegistrationError::invalid_redirect_uri( + Err(ClientRegistrationError::invalid_redirect_uri( "Redirect URI must not contain a fragment.", - )); + )) } else if uri.contains("..") { - return Err(ClientRegistrationError::invalid_redirect_uri( + Err(ClientRegistrationError::invalid_redirect_uri( "Redirect URI must not contain consecutive dots.", - )); - } else if uri.starts_with("https://") || loopback_redirect_parts(uri).is_some() { - return Ok(()); - } else if let Some((scheme, _)) = uri.split_once(':') - && scheme.contains('.') - && scheme - .as_bytes() - .first() - .is_some_and(u8::is_ascii_alphabetic) - && scheme - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+')) + )) + } else if uri.contains('\\') { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI must not contain a backslash.", + )) + } else if uri_scheme(uri).is_some_and(is_dangerous_scheme) { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI uses a forbidden scheme.", + )) + } else { + Ok(()) + } +} + +pub fn validate_public_redirect_uri(uri: &str) -> Result<(), ClientRegistrationError> { + check_redirect_uri_syntax(uri)?; + if loopback_redirect_parts(uri).is_some() || is_private_use_redirect(uri) { + Ok(()) + } else { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI must be a loopback (http://127.0.0.1/, http://[::1]/) or a private-use scheme containing a dot.", + )) + } +} + +pub fn validate_redirect_uri(uri: &str) -> Result<(), ClientRegistrationError> { + check_redirect_uri_syntax(uri)?; + if uri.starts_with("https://") + || loopback_redirect_parts(uri).is_some() + || is_private_use_redirect(uri) { - return Ok(()); + Ok(()) + } else { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI must be an https URL, a loopback (http://127.0.0.1/, http://[::1]/) or a private-use scheme URI.", + )) + } +} + +pub fn validate_registered_redirect_uri(uri: &str) -> Result<(), ClientRegistrationError> { + if uri.len() > CLIENT_ID_MAX_LEN + || uri + .bytes() + .any(|b| b.is_ascii_whitespace() || b.is_ascii_control() || b == b'\\') + { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI is too long or contains invalid characters.", + )) + } else if uri.contains('#') { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI must not contain a fragment.", + )) + } else { + match uri_scheme(uri) { + Some(scheme) if is_dangerous_scheme(scheme) => { + Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI uses a forbidden scheme.", + )) + } + Some(_) => Ok(()), + None => Err(ClientRegistrationError::invalid_redirect_uri( + "Redirect URI must be an absolute URI.", + )), + } + } +} + +pub fn validate_public_client_metadata( + request: &ClientRegistrationRequest, +) -> Result<(), ClientRegistrationError> { + for uri in &request.redirect_uris { + validate_public_redirect_uri(uri)?; + } + if !matches!( + request.token_endpoint_auth_method, + None | Some(TokenEndpointAuthMethod::None) + ) { + return Err(ClientRegistrationError::invalid_client_metadata( + "token_endpoint_auth_method must be \"none\".", + )); + } + for uri in [ + &request.client_uri, + &request.logo_uri, + &request.tos_uri, + &request.policy_uri, + ] + .into_iter() + .flatten() + { + if !is_https_url(uri) { + return Err(ClientRegistrationError::invalid_client_metadata( + "client_uri, logo_uri, tos_uri and policy_uri must use https.", + )); + } + } + if request + .grant_types + .iter() + .any(|grant| grant == GRANT_TYPE_DEVICE_CODE) + { + return Err(ClientRegistrationError::invalid_client_metadata( + "The device_code grant is not available to dynamically registered clients.", + )); } - Err(ClientRegistrationError::invalid_redirect_uri( - "Redirect URI must be an https URL, a loopback (http://127.0.0.1/, http://[::1]/) or a private-use scheme URI.", - )) + Ok(()) +} + +pub fn redirect_display_host(redirect_uri: &str) -> Option { + let url = Url::parse(redirect_uri).ok()?; + match url.scheme() { + "http" | "https" => { + let host = url.host_str()?; + Some(match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }) + } + scheme => Some(scheme.to_string()), + } +} + +pub fn is_https_url(uri: &str) -> bool { + Url::parse(uri).is_ok_and(|url| { + url.scheme() == "https" + && url.host_str().is_some_and(|host| !host.is_empty()) + && url.username().is_empty() + && url.password().is_none() + }) +} + +pub fn parse_grant_type(grant_type: &str) -> Option { + hashify::map!( + grant_type.as_bytes(), + OAuthGrantType, + "authorization_code" => OAuthGrantType::AuthorizationCode, + "refresh_token" => OAuthGrantType::RefreshToken, + "urn:ietf:params:oauth:grant-type:device_code" => OAuthGrantType::DeviceCode, + ) + .copied() +} + +pub fn parse_oauth_scope(scope: &str) -> Option { + hashify::map!( + scope.as_bytes(), + OAuthScope, + "urn:ietf:params:oauth:scope:mail" => OAuthScope::Mail, + "urn:ietf:params:oauth:scope:contacts" => OAuthScope::Contacts, + "urn:ietf:params:oauth:scope:calendars" => OAuthScope::Calendars, + "urn:ietf:params:oauth:scope:files" => OAuthScope::Files, + ) + .copied() +} + +pub fn validate_client_name( + request: &ClientRegistrationRequest, +) -> Result<(), ClientRegistrationError> { + match request.client_name.as_deref().map(check_client_name) { + Some(Err(description)) => Err(ClientRegistrationError::invalid_client_metadata( + description, + )), + _ => Ok(()), + } +} + +pub fn validate_oauth_client( + client: &OAuthClient, + is_superuser: bool, +) -> Result<(), (Property, ClientRegistrationError)> { + for uri in client.redirect_uris.iter() { + validate_registered_redirect_uri(uri).map_err(|err| (Property::RedirectUris, err))?; + } + for uri in client.post_logout_redirect_uris.iter() { + validate_registered_redirect_uri(uri).map_err(|err| { + ( + Property::PostLogoutRedirectUris, + err.for_post_logout_redirect_uri(), + ) + })?; + } + if !is_superuser + && client + .client_id + .get(..HTTPS_PREFIX.len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(HTTPS_PREFIX)) + { + return Err(( + Property::ClientId, + ClientRegistrationError::invalid_client_metadata( + "Only a superuser can register a client metadata document URL as a client id.", + ), + )); + } + if client.first_party { + if !is_superuser { + return Err(( + Property::FirstParty, + ClientRegistrationError::invalid_client_metadata( + "Only a superuser can register a first-party client.", + ), + )); + } else if client.redirect_uris.is_empty() { + return Err(( + Property::RedirectUris, + ClientRegistrationError::invalid_redirect_uri( + "A first-party client requires at least one redirect URI.", + ), + )); + } + } + Ok(()) } pub fn validate_grant_metadata( @@ -302,54 +558,3 @@ pub fn validate_grant_metadata( Ok(()) } -#[cfg(test)] -mod tests { - use super::{loopback_redirect_parts, redirect_uri_matches}; - - #[test] - fn loopback_matches_any_port() { - // Loopback redirects match regardless of port and of an empty versus "/" path (RFC 8252 §7.3) - for (registered, presented) in [ - ("http://127.0.0.1", "http://127.0.0.1:45678"), - ("http://127.0.0.1/", "http://127.0.0.1:45678"), - ("http://127.0.0.1", "http://127.0.0.1:45678/"), - ("http://127.0.0.1/cb", "http://127.0.0.1:45678/cb"), - ("http://127.0.0.1/?a=1", "http://127.0.0.1:45678?a=1"), - ("http://[::1]", "http://[::1]:8080"), - ("http://[::1]/", "http://[::1]:8080/"), - ] { - assert!( - redirect_uri_matches(registered, presented), - "{registered} should match {presented}" - ); - } - - // Different paths, hosts or schemes never match - for (registered, presented) in [ - ("http://127.0.0.1/cb", "http://127.0.0.1:45678"), - ("http://127.0.0.1", "http://127.0.0.1:45678/cb"), - ("http://127.0.0.1", "http://[::1]:45678"), - ("http://127.0.0.1", "https://127.0.0.1:45678"), - ("http://127.0.0.1", "http://127.0.0.1.evil.com"), - ("http://127.0.0.1", "http://127.0.0.1:80@evil.com"), - ("http://127.0.0.1", "http://127.0.0.1:45678#frag"), - ] { - assert!( - !redirect_uri_matches(registered, presented), - "{registered} should not match {presented}" - ); - } - - // Malformed loopback URIs are not recognised - for uri in [ - "http://127.0.0.1:", - "http://127.0.0.1:/cb", - "http://127.0.0.1:abc/", - "http://127.0.0.1.evil.com/", - "http://127.0.0.1@evil.com/", - "http://localhost:8080/", - ] { - assert_eq!(loopback_redirect_parts(uri), None, "{uri}"); - } - } -} diff --git a/crates/common/src/auth/oauth/scope.rs b/crates/common/src/auth/oauth/scope.rs new file mode 100644 index 00000000..37bbd22f --- /dev/null +++ b/crates/common/src/auth/oauth/scope.rs @@ -0,0 +1,544 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + SCOPE_CALENDARS, SCOPE_CONTACTS, SCOPE_FILES, SCOPE_MAIL, SCOPE_OFFLINE_ACCESS, SCOPE_OPENID, +}; +use crate::auth::{GrantClient, Permissions, TokenScope}; +use directory::backend::oidc::ExternalScopes; +use registry::{ + schema::enums::{OAuthScope, Permission}, + types::map::Map, +}; +use std::ops::{BitAnd, BitOr, BitOrAssign}; +use types::type_state::DataType; +use utils::map::bitmap::Bitmap; + +#[derive( + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + Hash, +)] +#[repr(transparent)] +pub struct OAuthScopes(u8); + +const SCOPE_NAMES: [(OAuthScopes, &str); 6] = [ + (OAuthScopes::OPENID, SCOPE_OPENID), + (OAuthScopes::OFFLINE_ACCESS, SCOPE_OFFLINE_ACCESS), + (OAuthScopes::MAIL, SCOPE_MAIL), + (OAuthScopes::CONTACTS, SCOPE_CONTACTS), + (OAuthScopes::CALENDARS, SCOPE_CALENDARS), + (OAuthScopes::FILES, SCOPE_FILES), +]; + +const INTEROPERABLE_SHIFT: u32 = 2; +const INTEROPERABLE_COMBINATIONS: usize = 1 << 4; + +impl OAuthScopes { + pub const NONE: Self = Self(0); + pub const OPENID: Self = Self(1 << 0); + pub const OFFLINE_ACCESS: Self = Self(1 << 1); + pub const MAIL: Self = Self(1 << 2); + pub const CONTACTS: Self = Self(1 << 3); + pub const CALENDARS: Self = Self(1 << 4); + pub const FILES: Self = Self(1 << 5); + pub const ALWAYS_ALLOWED: Self = Self(Self::OPENID.0 | Self::OFFLINE_ACCESS.0); + pub const INTEROPERABLE: Self = + Self(Self::MAIL.0 | Self::CONTACTS.0 | Self::CALENDARS.0 | Self::FILES.0); + pub const ALL: Self = Self(Self::ALWAYS_ALLOWED.0 | Self::INTEROPERABLE.0); + pub const DAV: Self = Self(Self::CONTACTS.0 | Self::CALENDARS.0 | Self::FILES.0); + + pub const fn from_bits(bits: u8) -> Self { + Self(bits & Self::ALL.0) + } + + pub const fn bits(self) -> u8 { + self.0 + } + + pub const fn from_external(scopes: ExternalScopes) -> Self { + Self((scopes.bits() << INTEROPERABLE_SHIFT) & Self::INTEROPERABLE.0) + } + + pub fn from_name(name: &str) -> Self { + hashify::map!( + name.as_bytes(), + OAuthScopes, + "openid" => OAuthScopes::OPENID, + "offline_access" => OAuthScopes::OFFLINE_ACCESS, + "urn:ietf:params:oauth:scope:mail" => OAuthScopes::MAIL, + "urn:ietf:params:oauth:scope:contacts" => OAuthScopes::CONTACTS, + "urn:ietf:params:oauth:scope:calendars" => OAuthScopes::CALENDARS, + "urn:ietf:params:oauth:scope:files" => OAuthScopes::FILES, + ) + .copied() + .unwrap_or_default() + } + + pub fn parse(scope: &str) -> Self { + scope + .split_ascii_whitespace() + .fold(Self::NONE, |scopes, name| scopes | Self::from_name(name)) + } + + pub fn from_registry<'x>(scopes: impl IntoIterator) -> Self { + scopes + .into_iter() + .fold(Self::NONE, |acc, scope| acc | Self::from(*scope)) + } + + pub fn to_registry(self) -> Map { + Map::new( + [ + (Self::MAIL, OAuthScope::Mail), + (Self::CONTACTS, OAuthScope::Contacts), + (Self::CALENDARS, OAuthScope::Calendars), + (Self::FILES, OAuthScope::Files), + ] + .into_iter() + .filter(|(bit, _)| self.contains(*bit)) + .map(|(_, scope)| scope) + .collect(), + ) + } + + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + pub const fn intersects(self, other: Self) -> bool { + self.0 & other.0 != 0 + } + + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + pub const fn interoperable(self) -> Self { + Self(self.0 & Self::INTEROPERABLE.0) + } + + pub fn names(self) -> impl Iterator { + SCOPE_NAMES + .into_iter() + .filter(move |(bit, _)| self.contains(*bit)) + .map(|(_, name)| name) + } + + pub fn to_wire(self) -> String { + let mut out = String::with_capacity(128); + for name in self.names() { + if !out.is_empty() { + out.push(' '); + } + out.push_str(name); + } + out + } + + const fn interoperable_index(self) -> usize { + ((self.0 & Self::INTEROPERABLE.0) >> INTEROPERABLE_SHIFT) as usize + } +} + +impl From for OAuthScopes { + fn from(scope: OAuthScope) -> Self { + match scope { + OAuthScope::Mail => Self::MAIL, + OAuthScope::Contacts => Self::CONTACTS, + OAuthScope::Calendars => Self::CALENDARS, + OAuthScope::Files => Self::FILES, + } + } +} + +impl BitOr for OAuthScopes { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0) + } +} + +impl BitOrAssign for OAuthScopes { + fn bitor_assign(&mut self, rhs: Self) { + self.0 |= rhs.0; + } +} + +impl BitAnd for OAuthScopes { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0) + } +} + +macro_rules! permission_scopes { + ($($scopes:expr => [$($permission:ident),+ $(,)?]),+ $(,)?) => { + pub const fn permission_scopes(permission: Permission) -> OAuthScopes { + match permission { + $($(Permission::$permission)|+ => $scopes,)+ + _ => OAuthScopes::NONE, + } + } + + static SCOPE_PERMISSIONS: [Permissions; INTEROPERABLE_COMBINATIONS] = { + let mut table = [const { Permissions::new() }; INTEROPERABLE_COMBINATIONS]; + $($(insert_scoped(&mut table, Permission::$permission as usize, $scopes);)+)+ + table + }; + }; +} + +permission_scopes!( + OAuthScopes::ALL => [ + Authenticate, + AuthenticateWithAlias, + ], + OAuthScopes::MAIL => [ + EmailSend, + EmailReceive, + JmapMailboxGet, + JmapMailboxChanges, + JmapMailboxQuery, + JmapMailboxQueryChanges, + JmapMailboxCreate, + JmapMailboxUpdate, + JmapMailboxDestroy, + JmapThreadGet, + JmapThreadChanges, + JmapEmailGet, + JmapEmailChanges, + JmapEmailQuery, + JmapEmailQueryChanges, + JmapEmailCreate, + JmapEmailUpdate, + JmapEmailDestroy, + JmapEmailCopy, + JmapEmailImport, + JmapEmailParse, + JmapSearchSnippetGet, + JmapIdentityGet, + JmapIdentityChanges, + JmapIdentityCreate, + JmapIdentityUpdate, + JmapIdentityDestroy, + JmapEmailSubmissionGet, + JmapEmailSubmissionChanges, + JmapEmailSubmissionQuery, + JmapEmailSubmissionQueryChanges, + JmapEmailSubmissionCreate, + JmapEmailSubmissionUpdate, + JmapEmailSubmissionDestroy, + JmapVacationResponseGet, + JmapVacationResponseCreate, + JmapVacationResponseUpdate, + JmapVacationResponseDestroy, + JmapSieveScriptGet, + JmapSieveScriptQuery, + JmapSieveScriptValidate, + JmapSieveScriptCreate, + JmapSieveScriptUpdate, + JmapSieveScriptDestroy, + ImapAuthenticate, + ImapAclGet, + ImapAclSet, + ImapMyRights, + ImapListRights, + ImapAppend, + ImapCapability, + ImapId, + ImapCopy, + ImapMove, + ImapCreate, + ImapDelete, + ImapEnable, + ImapExpunge, + ImapFetch, + ImapIdle, + ImapList, + ImapLsub, + ImapNamespace, + ImapRename, + ImapSearch, + ImapSort, + ImapSelect, + ImapExamine, + ImapStatus, + ImapStore, + ImapSubscribe, + ImapThread, + ImapMetadataGet, + ImapMetadataSet, + ImapMetadataPrivate, + Pop3Authenticate, + Pop3List, + Pop3Uidl, + Pop3Stat, + Pop3Retr, + Pop3Dele, + SieveAuthenticate, + SieveListScripts, + SieveSetActive, + SieveGetScript, + SievePutScript, + SieveDeleteScript, + SieveRenameScript, + SieveCheckScript, + SieveHaveSpace, + ], + OAuthScopes::CALENDARS => [ + CalendarAlarmsSend, + CalendarSchedulingSend, + CalendarSchedulingReceive, + JmapPrincipalGetAvailability, + JmapCalendarGet, + JmapCalendarChanges, + JmapCalendarCreate, + JmapCalendarUpdate, + JmapCalendarDestroy, + JmapCalendarEventGet, + JmapCalendarEventChanges, + JmapCalendarEventQuery, + JmapCalendarEventQueryChanges, + JmapCalendarEventCreate, + JmapCalendarEventUpdate, + JmapCalendarEventDestroy, + JmapCalendarEventCopy, + JmapCalendarEventParse, + JmapCalendarEventNotificationGet, + JmapCalendarEventNotificationChanges, + JmapCalendarEventNotificationQuery, + JmapCalendarEventNotificationQueryChanges, + JmapCalendarEventNotificationCreate, + JmapCalendarEventNotificationUpdate, + JmapCalendarEventNotificationDestroy, + JmapParticipantIdentityGet, + JmapParticipantIdentityChanges, + JmapParticipantIdentityCreate, + JmapParticipantIdentityUpdate, + JmapParticipantIdentityDestroy, + DavCalPropFind, + DavCalPropPatch, + DavCalGet, + DavCalMkCol, + DavCalDelete, + DavCalPut, + DavCalCopy, + DavCalMove, + DavCalLock, + DavCalAcl, + DavCalQuery, + DavCalMultiGet, + DavCalFreeBusyQuery, + ], + OAuthScopes::INTEROPERABLE => [ + JmapPushSubscriptionGet, + JmapPushSubscriptionCreate, + JmapPushSubscriptionUpdate, + JmapPushSubscriptionDestroy, + JmapPrincipalGet, + JmapPrincipalQuery, + JmapPrincipalChanges, + JmapPrincipalQueryChanges, + JmapQuotaGet, + JmapQuotaChanges, + JmapQuotaQuery, + JmapQuotaQueryChanges, + JmapBlobGet, + JmapBlobCopy, + JmapBlobLookup, + JmapBlobUpload, + JmapShareNotificationGet, + JmapShareNotificationChanges, + JmapShareNotificationQuery, + JmapShareNotificationQueryChanges, + JmapShareNotificationCreate, + JmapShareNotificationUpdate, + JmapShareNotificationDestroy, + JmapCoreEcho, + JmapMetadataGet, + JmapMetadataSet, + JmapMetadataPrivate, + ], + OAuthScopes::CONTACTS => [ + JmapAddressBookGet, + JmapAddressBookChanges, + JmapAddressBookCreate, + JmapAddressBookUpdate, + JmapAddressBookDestroy, + JmapContactCardGet, + JmapContactCardChanges, + JmapContactCardQuery, + JmapContactCardQueryChanges, + JmapContactCardCreate, + JmapContactCardUpdate, + JmapContactCardDestroy, + JmapContactCardCopy, + JmapContactCardParse, + DavCardPropFind, + DavCardPropPatch, + DavCardGet, + DavCardMkCol, + DavCardDelete, + DavCardPut, + DavCardCopy, + DavCardMove, + DavCardLock, + DavCardAcl, + DavCardQuery, + DavCardMultiGet, + ], + OAuthScopes::FILES => [ + JmapFileNodeGet, + JmapFileNodeChanges, + JmapFileNodeQuery, + JmapFileNodeQueryChanges, + JmapFileNodeCreate, + JmapFileNodeUpdate, + JmapFileNodeDestroy, + JmapFileNodeCopy, + DavFilePropFind, + DavFilePropPatch, + DavFileGet, + DavFileMkCol, + DavFileDelete, + DavFilePut, + DavFileCopy, + DavFileMove, + DavFileLock, + DavFileAcl, + ], + OAuthScopes::DAV => [ + DavSyncCollection, + DavExpandProperty, + DavPrincipalAcl, + DavPrincipalList, + DavPrincipalMatch, + DavPrincipalSearch, + DavPrincipalSearchPropSet, + ], +); + +const fn insert_scoped( + table: &mut [Permissions; INTEROPERABLE_COMBINATIONS], + permission: usize, + scopes: OAuthScopes, +) { + let always = scopes.intersects(OAuthScopes::OPENID); + let mut idx = 0; + while idx < INTEROPERABLE_COMBINATIONS { + if always || OAuthScopes((idx as u8) << INTEROPERABLE_SHIFT).intersects(scopes) { + table[idx].insert(permission); + } + idx += 1; + } +} + +pub fn scope_permissions(scopes: OAuthScopes) -> &'static Permissions { + &SCOPE_PERMISSIONS[scopes.interoperable_index()] +} + +impl TokenScope { + #[inline] + pub const fn grant_id(self) -> Option { + match self { + TokenScope::Grant { grant_id, .. } => Some(grant_id), + TokenScope::External { .. } => None, + } + } + + #[inline] + pub const fn restricted_scopes(self) -> Option { + match self { + TokenScope::Grant { + client: GrantClient::FirstParty, + .. + } => None, + TokenScope::Grant { scopes, .. } | TokenScope::External { scopes } => Some(scopes), + } + } + + #[inline] + pub fn permissions(self) -> Option<&'static Permissions> { + self.restricted_scopes().map(scope_permissions) + } +} + +macro_rules! data_type_scopes { + ($($scopes:expr => [$($data_type:ident),+ $(,)?]),+ $(,)?) => { + pub const fn data_type_scopes(data_type: DataType) -> OAuthScopes { + match data_type { + $($(DataType::$data_type)|+ => $scopes,)+ + DataType::None => OAuthScopes::NONE, + } + } + + static SCOPE_DATA_TYPES: [u64; INTEROPERABLE_COMBINATIONS] = { + let mut table = [0u64; INTEROPERABLE_COMBINATIONS]; + $($(insert_data_type(&mut table, DataType::$data_type as u64, $scopes);)+)+ + table + }; + }; +} + +data_type_scopes!( + OAuthScopes::MAIL => [ + Email, + EmailDelivery, + EmailSubmission, + Mailbox, + Thread, + Identity, + SearchSnippet, + VacationResponse, + Mdn, + SieveScript, + ], + OAuthScopes::CALENDARS => [ + Calendar, + CalendarEvent, + CalendarEventNotification, + ParticipantIdentity, + CalendarAlert, + ], + OAuthScopes::CONTACTS => [AddressBook, ContactCard], + OAuthScopes::FILES => [FileNode], + OAuthScopes::INTEROPERABLE => [ + Core, + PushSubscription, + Quota, + Principal, + ShareNotification, + ], +); + +const fn insert_data_type( + table: &mut [u64; INTEROPERABLE_COMBINATIONS], + data_type: u64, + scopes: OAuthScopes, +) { + let mut idx = 0; + while idx < INTEROPERABLE_COMBINATIONS { + if OAuthScopes((idx as u8) << INTEROPERABLE_SHIFT).intersects(scopes) { + table[idx] |= 1 << data_type; + } + idx += 1; + } +} + +pub fn scope_data_types(scopes: OAuthScopes) -> Bitmap { + Bitmap::from_bits(SCOPE_DATA_TYPES[scopes.interoperable_index()]) +} + diff --git a/crates/common/src/auth/oauth/token.rs b/crates/common/src/auth/oauth/token.rs index 044c4a7f..0e9d7b68 100644 --- a/crates/common/src/auth/oauth/token.rs +++ b/crates/common/src/auth/oauth/token.rs @@ -6,10 +6,14 @@ use super::{GrantType, crypto::SymmetricEncrypt}; use crate::Server; -use base64::{Engine, engine::general_purpose}; +use compact_str::CompactString; +use encodify::base64::URL_SAFE_NO_PAD; use std::time::SystemTime; -use store::rand::{RngExt, rng}; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; +use store::{ + blake3, + rand::{RngExt, rng}, +}; +use utils::codec::leb128::{Leb128Reader, Leb128Writer}; pub const FAILED_TO_DECODE_TOKEN: &str = concat!( "Failed to decode token. If you are using an ", @@ -18,7 +22,7 @@ pub const FAILED_TO_DECODE_TOKEN: &str = concat!( ); pub(crate) const TOKEN_HEADER: &str = "sw1."; -const TOKEN_KEY_CONTEXT: &str = "stalwart-oauth-token-sw1"; +pub(super) const TOKEN_KEY_CONTEXT: &str = "stalwart-oauth-token-sw1"; const OAUTH_EPOCH: u64 = 946684800; // Jan 1, 2000 pub struct TokenInfo { @@ -28,6 +32,15 @@ pub struct TokenInfo { pub expiry: u64, pub issued_at: u64, pub expires_in: u64, + pub credential_version: u64, + pub credential_expired: bool, + pub grant: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TokenGrant { + pub id: u64, + pub generation: u64, } struct RawToken { @@ -37,6 +50,16 @@ struct RawToken { issued_at: u64, expiry: u64, credential_version: u64, + grant: Option, +} + +pub struct RefreshTokenParams<'x> { + pub account_id: u32, + pub account_name: &'x str, + pub grant: TokenGrant, + pub rotated_at: u64, + pub credential_version: u64, + pub scope: Option<&'x str>, } impl Server { @@ -50,21 +73,83 @@ impl Server { credential_version: Option, ) -> trc::Result { let issued_at = seconds_since_oauth_epoch(); - let raw = RawToken { - grant_type, - account_id, - claims: claims.map(|claims| claims.to_string()), - issued_at, - expiry: issued_at + expiry_in, - credential_version: credential_version - .filter(|_| !matches!(grant_type, GrantType::Rsvp)) - .unwrap_or_default(), - }; + self.seal_token( + &RawToken { + grant_type, + account_id, + claims: claims.map(|claims| claims.to_string()), + issued_at, + expiry: issued_at + expiry_in, + credential_version: credential_version + .filter(|_| !matches!(grant_type, GrantType::Rsvp)) + .unwrap_or_default(), + grant: None, + }, + account_name, + &rng().random::<[u8; SymmetricEncrypt::NONCE_LEN]>(), + ) + } + pub fn encode_grant_access_token( + &self, + account_id: u32, + account_name: &str, + grant_id: u64, + credential_version: u64, + ) -> trc::Result { + let issued_at = seconds_since_oauth_epoch(); + self.seal_token( + &RawToken { + grant_type: GrantType::AccessToken, + account_id, + claims: None, + issued_at, + expiry: issued_at + self.core.oauth.oauth_expiry_token, + credential_version, + grant: Some(TokenGrant { + id: grant_id, + generation: 0, + }), + }, + account_name, + &rng().random::<[u8; SymmetricEncrypt::NONCE_LEN]>(), + ) + } + + pub fn encode_refresh_token(&self, params: RefreshTokenParams<'_>) -> trc::Result { + let issued_at = params.rotated_at.saturating_sub(OAUTH_EPOCH); + let mut hasher = blake3::Hasher::new_keyed(&self.core.oauth.refresh_nonce_key); + hasher.update(¶ms.grant.id.to_le_bytes()); + hasher.update(¶ms.grant.generation.to_le_bytes()); + let mut nonce = [0u8; SymmetricEncrypt::NONCE_LEN]; + hasher.finalize_xof().fill(&mut nonce); + + self.seal_token( + &RawToken { + grant_type: GrantType::RefreshToken, + account_id: params.account_id, + claims: params.scope.map(|scope| scope.to_string()), + issued_at, + expiry: issued_at + self.core.oauth.oauth_expiry_refresh_token, + credential_version: params.credential_version, + grant: Some(params.grant), + }, + params.account_name, + &nonce, + ) + } + + fn seal_token( + &self, + raw: &RawToken, + account_name: &str, + nonce: &[u8; SymmetricEncrypt::NONCE_LEN], + ) -> trc::Result { seal_token( - self.core.oauth.oauth_key.as_bytes(), - &raw, + &self.core.oauth.token_cipher, + raw, account_name.as_bytes(), + nonce, ) .map_err(|err| { trc::AuthEvent::Error @@ -75,46 +160,30 @@ impl Server { }) } - pub async fn validate_access_token( + pub fn decode_access_token( &self, expected_grant_type: Option, token_: &str, ) -> trc::Result { - let token = open_token(self.core.oauth.oauth_key.as_bytes(), token_).map_err(|_| { + let token = open_token(&self.core.oauth.token_cipher, token_).map_err(|_| { trc::AuthEvent::Error .into_err() .ctx(trc::Key::Reason, FAILED_TO_DECODE_TOKEN) .caused_by(trc::location!()) - .details(token_.to_string()) + .details(CompactString::from(token_)) })?; - // Validate expiration let now = seconds_since_oauth_epoch(); if token.expiry <= now || token.issued_at > now { return Err(trc::AuthEvent::TokenExpired.into_err()); } - // Validate grant type if expected_grant_type.is_some_and(|g| g != token.grant_type) { return Err(trc::AuthEvent::Error .into_err() .details("Invalid grant type")); } - // Enforce credential revocation for long lived tokens - if token.credential_version != 0 { - let current = self - .access_token(token.account_id) - .await - .map_err(|err| trc::AuthEvent::Error.into_err().ctx(trc::Key::Details, err))? - .credential_version(); - if current != token.credential_version { - return Err(trc::AuthEvent::TokenExpired - .into_err() - .details("Token revoked")); - } - } - Ok(TokenInfo { grant_type: token.grant_type, account_id: token.account_id, @@ -122,70 +191,116 @@ impl Server { expiry: token.expiry + OAUTH_EPOCH, issued_at: token.issued_at + OAUTH_EPOCH, expires_in: token.expiry - now, + credential_version: token.credential_version, + credential_expired: false, + grant: token.grant, }) } + + pub async fn validate_access_token( + &self, + expected_grant_type: Option, + token: &str, + ) -> trc::Result { + let mut token = self.decode_access_token(expected_grant_type, token)?; + + if token.credential_version != 0 { + let inner = self + .access_token(token.account_id) + .await + .map_err(|err| trc::AuthEvent::Error.into_err().ctx(trc::Key::Details, err))?; + assert_credential_version(token.credential_version, inner.credential_version())?; + token.credential_expired = inner.is_main_credential_expired(); + } + + Ok(token) + } } -fn seal_token(key: &[u8], token: &RawToken, footer: &[u8]) -> Result { +pub fn assert_credential_version(token_version: u64, current_version: u64) -> trc::Result<()> { + if token_version == 0 || token_version == current_version { + Ok(()) + } else { + Err(trc::AuthEvent::TokenExpired + .into_err() + .details("Token revoked")) + } +} + +fn seal_token( + cipher: &SymmetricEncrypt, + token: &RawToken, + footer: &[u8], + nonce: &[u8; SymmetricEncrypt::NONCE_LEN], +) -> Result { let mut payload = Vec::with_capacity(32); payload.push_leb128(token.account_id); payload.push(token.grant_type.id()); payload.push_leb128(token.issued_at); payload.push_leb128(token.expiry); payload.push_leb128(token.credential_version); + match token.grant { + Some(grant) => { + payload.push_leb128(grant.id.saturating_add(1)); + payload.push_leb128(grant.generation); + } + None => payload.push(0), + } if let Some(claims) = token.claims.as_deref().filter(|claims| !claims.is_empty()) { payload.extend_from_slice(claims.as_bytes()); } - let nonce = rng().random::<[u8; SymmetricEncrypt::NONCE_LEN]>(); - let ciphertext = - SymmetricEncrypt::new(key, TOKEN_KEY_CONTEXT).encrypt_with_aad(&payload, &nonce, footer)?; + let ciphertext = cipher.encrypt_with_aad(&payload, nonce, footer)?; let mut body = Vec::with_capacity(nonce.len() + ciphertext.len()); - body.extend_from_slice(&nonce); + body.extend_from_slice(nonce); body.extend_from_slice(&ciphertext); let mut out = String::with_capacity(TOKEN_HEADER.len() + (body.len() + footer.len()) * 2); out.push_str(TOKEN_HEADER); - general_purpose::URL_SAFE_NO_PAD.encode_string(&body, &mut out); + URL_SAFE_NO_PAD.encode_append(&body, &mut out); if !footer.is_empty() { out.push('.'); - general_purpose::URL_SAFE_NO_PAD.encode_string(footer, &mut out); + URL_SAFE_NO_PAD.encode_append(footer, &mut out); } Ok(out) } -fn open_token(key: &[u8], token: &str) -> Result { +fn open_token(cipher: &SymmetricEncrypt, token: &str) -> Result { let rest = token.strip_prefix(TOKEN_HEADER).ok_or(())?; let (body, footer) = match rest.split_once('.') { Some((body, footer)) => ( body, - general_purpose::URL_SAFE_NO_PAD - .decode(footer.as_bytes()) - .map_err(|_| ())?, + URL_SAFE_NO_PAD.decode(footer.as_bytes()).map_err(|_| ())?, ), None => (rest, Vec::new()), }; - let body = general_purpose::URL_SAFE_NO_PAD - .decode(body.as_bytes()) - .map_err(|_| ())?; + let body = URL_SAFE_NO_PAD.decode(body.as_bytes()).map_err(|_| ())?; if body.len() < SymmetricEncrypt::NONCE_LEN + SymmetricEncrypt::ENCRYPT_TAG_LEN { return Err(()); } let (nonce, ciphertext) = body.split_at(SymmetricEncrypt::NONCE_LEN); - let payload = SymmetricEncrypt::new(key, TOKEN_KEY_CONTEXT) + let payload = cipher .decrypt_with_aad(ciphertext, nonce, &footer) .map_err(|_| ())?; - let mut bytes = payload.iter(); - let account_id: u32 = bytes.next_leb128().ok_or(())?; - let grant_type = GrantType::from_id(bytes.next().copied().ok_or(())?).ok_or(())?; - let issued_at: u64 = bytes.next_leb128().ok_or(())?; - let expiry: u64 = bytes.next_leb128().ok_or(())?; - let credential_version: u64 = bytes.next_leb128().ok_or(())?; - let bytes = bytes.as_slice(); + let mut bytes = payload.as_slice(); + let account_id: u32 = bytes.read_leb128().ok_or(())?; + let (&grant_type, rest) = bytes.split_first().ok_or(())?; + let grant_type = GrantType::from_id(grant_type).ok_or(())?; + bytes = rest; + let issued_at: u64 = bytes.read_leb128().ok_or(())?; + let expiry: u64 = bytes.read_leb128().ok_or(())?; + let credential_version: u64 = bytes.read_leb128().ok_or(())?; + let grant = match bytes.read_leb128::().ok_or(())? { + 0 => None, + id => Some(TokenGrant { + id: id - 1, + generation: bytes.read_leb128().ok_or(())?, + }), + }; let claims = if bytes.is_empty() { None } else { @@ -199,6 +314,7 @@ fn open_token(key: &[u8], token: &str) -> Result { issued_at, expiry, credential_version, + grant, }) } @@ -210,209 +326,3 @@ fn seconds_since_oauth_epoch() -> u64 { .saturating_sub(OAUTH_EPOCH) } -#[cfg(test)] -mod tests { - use super::*; - - const KEY: &[u8] = b"a-test-encryption-key-of-some-length"; - const NAME: &[u8] = b"user@example.org"; - - fn sample(grant_type: GrantType, claims: Option<&str>, cv: u64) -> RawToken { - RawToken { - grant_type, - account_id: 42, - claims: claims.map(|c| c.to_string()), - issued_at: 1_000, - expiry: 2_000, - credential_version: cv, - } - } - - fn assert_eq_fields(a: &RawToken, b: &RawToken) { - assert_eq!(a.account_id, b.account_id); - assert_eq!(a.grant_type, b.grant_type); - assert_eq!(a.claims, b.claims); - assert_eq!(a.issued_at, b.issued_at); - assert_eq!(a.expiry, b.expiry); - assert_eq!(a.credential_version, b.credential_version); - } - - #[test] - fn round_trip_preserves_all_fields() { - for (raw, footer) in [ - (sample(GrantType::AccessToken, None, 0), NAME), - ( - sample(GrantType::RefreshToken, None, 0xdead_beef_cafe), - NAME, - ), - ( - sample(GrantType::Rsvp, Some("attendee@x.org;7"), 0), - b"owner@example.org", - ), - (sample(GrantType::AccessToken, None, 0), b""), - ( - RawToken { - account_id: u32::MAX, - credential_version: u64::MAX, - ..sample(GrantType::AccessToken, Some("名前;1"), 1) - }, - "名字@example.org".as_bytes(), - ), - ] { - let token = seal_token(KEY, &raw, footer).unwrap(); - assert!(token.starts_with(TOKEN_HEADER)); - let opened = open_token(KEY, &token).unwrap(); - assert_eq_fields(&raw, &opened); - - // The footer (account name) round-trips in clear text for proxies - if footer.is_empty() { - assert!(!token[TOKEN_HEADER.len()..].contains('.')); - } else { - let segment = token.rsplit_once('.').unwrap().1; - assert_eq!( - general_purpose::URL_SAFE_NO_PAD.decode(segment).unwrap(), - footer - ); - } - } - } - - #[test] - fn account_name_is_readable_in_clear_text_footer() { - let token = seal_token( - KEY, - &sample(GrantType::AccessToken, None, 0), - b"route-me@example.org", - ) - .unwrap(); - let footer = token.rsplit_once('.').unwrap().1; - let decoded = general_purpose::URL_SAFE_NO_PAD.decode(footer).unwrap(); - assert_eq!(decoded, b"route-me@example.org"); - } - - #[test] - fn wrong_key_is_rejected() { - let token = seal_token(KEY, &sample(GrantType::AccessToken, None, 0), NAME).unwrap(); - assert!(open_token(b"a-different-encryption-key-entirely!", &token).is_err()); - } - - #[test] - fn tampering_with_ciphertext_is_rejected() { - let raw = sample(GrantType::AccessToken, None, 0); - let token = seal_token(KEY, &raw, NAME).unwrap(); - let (header, rest) = token.split_at(TOKEN_HEADER.len()); - let (body_b64, footer) = match rest.split_once('.') { - Some((b, f)) => (b.to_string(), Some(f.to_string())), - None => (rest.to_string(), None), - }; - let mut body = general_purpose::URL_SAFE_NO_PAD.decode(&body_b64).unwrap(); - - for idx in 0..body.len() { - let mut tampered = body.clone(); - tampered[idx] ^= 0x01; - let mut rebuilt = String::from(header); - rebuilt.push_str(&general_purpose::URL_SAFE_NO_PAD.encode(&tampered)); - if let Some(footer) = &footer { - rebuilt.push('.'); - rebuilt.push_str(footer); - } - assert!( - open_token(KEY, &rebuilt).is_err(), - "flipping byte {idx} of the body must invalidate the token" - ); - } - - // Sanity: the untampered token still opens - body[0] ^= 0x00; - assert!(open_token(KEY, &token).is_ok()); - } - - #[test] - fn tampering_with_clear_text_footer_is_rejected() { - let raw = sample(GrantType::AccessToken, None, 0); - let token = seal_token(KEY, &raw, b"victim@example.org").unwrap(); - let (body, _) = token.rsplit_once('.').unwrap(); - - // An attacker rewrites the clear-text account name to impersonate another account - let forged_footer = general_purpose::URL_SAFE_NO_PAD.encode(b"attacker@example.org"); - let forged = format!("{body}.{forged_footer}"); - assert!( - open_token(KEY, &forged).is_err(), - "the footer is bound through the associated data and must be authenticated" - ); - } - - #[test] - fn swapping_footers_between_tokens_is_rejected() { - let a = seal_token( - KEY, - &sample(GrantType::AccessToken, None, 0), - b"alice@example.org", - ) - .unwrap(); - let b = seal_token( - KEY, - &sample(GrantType::AccessToken, None, 0), - b"bob@example.org", - ) - .unwrap(); - let a_body = a.rsplit_once('.').unwrap().0; - let b_footer = b.rsplit_once('.').unwrap().1; - let frankentoken = format!("{a_body}.{b_footer}"); - assert!(open_token(KEY, &frankentoken).is_err()); - } - - #[test] - fn malformed_input_never_panics_and_is_rejected() { - let valid = seal_token(KEY, &sample(GrantType::AccessToken, None, 0), NAME).unwrap(); - let cases = [ - String::new(), - "sw1.".to_string(), - "sw1.!!!not-base64!!!".to_string(), - "sw1...".to_string(), - "wrong-prefix.".to_string(), - "sw1.AAAA".to_string(), - "sw1.AAAA.BBBB".to_string(), - valid.replace("sw1.", "sw2."), - valid[..valid.len() / 2].to_string(), - format!("sw1.{}", "A".repeat(10_000)), - "\u{0}\u{0}\u{0}".to_string(), - ]; - for case in cases { - assert!(open_token(KEY, &case).is_err(), "must reject {case:?}"); - } - } - - #[test] - fn truncating_the_body_is_rejected() { - let token = seal_token(KEY, &sample(GrantType::AccessToken, None, 0), NAME).unwrap(); - let (header, rest) = token.split_at(TOKEN_HEADER.len()); - let body_b64 = rest.split_once('.').map(|(b, _)| b).unwrap_or(rest); - let body = general_purpose::URL_SAFE_NO_PAD.decode(body_b64).unwrap(); - for len in 0..body.len() { - let mut rebuilt = String::from(header); - rebuilt.push_str(&general_purpose::URL_SAFE_NO_PAD.encode(&body[..len])); - assert!( - open_token(KEY, &rebuilt).is_err(), - "truncation to {len} must be rejected" - ); - } - } - - #[test] - fn identical_input_produces_distinct_tokens() { - let raw = sample(GrantType::AccessToken, None, 7); - let a = seal_token(KEY, &raw, NAME).unwrap(); - let b = seal_token(KEY, &raw, NAME).unwrap(); - assert_ne!(a, b, "a random nonce must make each token unique"); - assert_eq_fields(&open_token(KEY, &a).unwrap(), &open_token(KEY, &b).unwrap()); - } - - #[test] - fn claims_with_separators_round_trip_exactly() { - let raw = sample(GrantType::Rsvp, Some("a;b;c;d@e.org;999"), 0); - let token = seal_token(KEY, &raw, b"owner@example.org").unwrap(); - let opened = open_token(KEY, &token).unwrap(); - assert_eq!(opened.claims.as_deref(), Some("a;b;c;d@e.org;999")); - } -} diff --git a/crates/common/src/auth/permissions.rs b/crates/common/src/auth/permissions.rs index bf43a37c..30b18506 100644 --- a/crates/common/src/auth/permissions.rs +++ b/crates/common/src/auth/permissions.rs @@ -45,7 +45,6 @@ impl Server { &self, permissions: &structs::Permissions, role_ids: &[Id], - tenant_id: Option, ) -> trc::Result { // Calculate effective permissions let (mut permissions, roles) = match permissions { @@ -73,25 +72,16 @@ impl Server { access_token: &AccessToken, account: &Account, ) -> trc::Result>> { - let (permissions, role_ids, tenant_id) = match account { + let (permissions, role_ids) = match account { Account::User(account) => ( &account.permissions, match &account.roles { UserRoles::User => self.core.network.security.default_role_ids_user.as_slice(), UserRoles::Admin => { - if access_token.tenant_id().is_none() { - self.core.network.security.default_role_ids_admin.as_slice() - } else { - self.core - .network - .security - .default_role_ids_tenant - .as_slice() - } + self.core.network.security.default_role_ids_admin.as_slice() } UserRoles::Custom(custom_roles) => custom_roles.role_ids.as_slice(), }, - account.member_tenant_id.map(|t| t.document_id()), ), Account::Group(account) => ( &account.permissions, @@ -99,13 +89,13 @@ impl Server { .roles .role_ids() .unwrap_or(self.core.network.security.default_role_ids_group.as_slice()), - account.member_tenant_id.map(|t| t.document_id()), ), }; - self.effective_permissions(permissions, role_ids, tenant_id) - .await - .map(|permissions| access_token.can_grant_permissions(permissions.finalize())) + #[cfg(not(feature = "enterprise"))] + let permissions = self.effective_permissions(permissions, role_ids).await?; + + Ok(access_token.can_grant_permissions(permissions.finalize())) } } @@ -114,7 +104,7 @@ impl AccessToken { &self, mut requested_permissions: Permissions, ) -> Result<(), Vec> { - requested_permissions.difference(self.permissions_bits()); + requested_permissions.difference(&self.effective_permissions()); if requested_permissions.is_empty() { Ok(()) } else { @@ -153,7 +143,6 @@ impl PermissionsListBuilder for Permissions { pub struct DefaultPermissions { pub user: Vec, pub group: Vec, - pub tenant: Vec, pub superuser: Vec, } @@ -199,30 +188,22 @@ impl Default for DefaultPermissions { let mut default = Self { user: Default::default(), group: Default::default(), - tenant: Default::default(), superuser: Default::default(), }; - for permission_id in 0..Permission::COUNT { - let permission = Permission::from_id(permission_id as u16).unwrap(); + for &permission in Permission::ALL { match permission { - Permission::Authenticate - | Permission::AuthenticateWithAlias - | Permission::InteractAi => { + Permission::Authenticate | Permission::AuthenticateWithAlias => { default.user.push(permission); default.superuser.push(permission); - default.tenant.push(permission); } Permission::Impersonate | Permission::UnlimitedRequests - | Permission::UnlimitedUploads - | Permission::LiveMetrics - | Permission::LiveTracing => { + | Permission::UnlimitedUploads => { default.superuser.push(permission); } Permission::FetchAnyBlob | Permission::LiveDeliveryTest => { default.superuser.push(permission); - default.tenant.push(permission); } permission => { let name = permission.as_str(); @@ -248,6 +229,7 @@ impl Default for DefaultPermissions { } else if name.starts_with("sysAccountPassword") || name.starts_with("sysApiKey") || name.starts_with("sysAppPassword") + || name.starts_with("sysOAuthGrant") { default.user.push(permission); default.superuser.push(permission); @@ -262,7 +244,6 @@ impl Default for DefaultPermissions { || name.starts_with("sysDnsServer") || name.starts_with("sysQueuedMessage") { - default.tenant.push(permission); default.superuser.push(permission); } else { default.superuser.push(permission); diff --git a/crates/common/src/auth/rate_limit.rs b/crates/common/src/auth/rate_limit.rs index 92020446..e6812eff 100644 --- a/crates/common/src/auth/rate_limit.rs +++ b/crates/common/src/auth/rate_limit.rs @@ -6,10 +6,16 @@ use crate::auth::AccessToken; use crate::network::ip_to_bytes; -use crate::network::limiter::{InFlight, LimiterResult}; -use crate::{KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, Server}; -use registry::schema::enums::Permission; -use std::net::IpAddr; +use crate::network::limiter::{ + AccountLimits, AccountRate, InFlight, LimiterResult, RateCounter, RateHit, +}; +use crate::{ + KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_IMAP, Server, +}; +use compact_str::format_compact; +use registry::schema::{enums::Permission, structs::Rate}; +use std::{net::IpAddr, sync::Arc}; +use store::{dispatch::lookup::KeyValue, write::now}; use trc::AddContext; impl Server { @@ -18,53 +24,116 @@ impl Server { access_token: &AccessToken, addr: IpAddr, ) -> trc::Result> { - let rate_reset = if let Some(rate) = &self.core.network.http.rate_authenticated { - if self.is_ip_allowed(addr) { - None - } else { - self.core - .storage - .memory - .is_rate_allowed( - KV_RATE_LIMIT_HTTP_AUTHENTICATED, - &access_token.account_id().to_be_bytes(), - rate, - false, - ) - .await - .caused_by(trc::location!())? - .map(|reset| (reset, rate.count)) + if let Some(rate) = &self.core.network.http.rate_authenticated + && !access_token.has_permission(Permission::UnlimitedRequests) + && !self.is_ip_allowed(addr) + && let Some(reset) = self + .is_account_rate_allowed( + KV_RATE_LIMIT_HTTP_AUTHENTICATED, + access_token.account_id(), + rate, + access_token.inner.rate_counter(AccountRate::Http), + ) + .await + .caused_by(trc::location!())? + { + return Err(trc::LimitEvent::TooManyRequests + .into_err() + .ctx(trc::Key::Expires, reset) + .ctx(trc::Key::Limit, rate.count)); + } + + match access_token.is_http_request_allowed() { + LimiterResult::Allowed(in_flight) => Ok(Some(in_flight)), + LimiterResult::Forbidden => { + if access_token.has_permission(Permission::UnlimitedRequests) { + Ok(None) + } else { + Err(trc::LimitEvent::ConcurrentRequest + .into_err() + .ctx(trc::Key::Limit, access_token.concurrent_http_requests())) + } } + LimiterResult::Disabled => Ok(None), + } + } + + pub async fn is_imap_request_allowed(&self, access_token: &AccessToken) -> trc::Result<()> { + if let Some(rate) = &self.core.imap.rate_requests + && !access_token.has_permission(Permission::UnlimitedRequests) + && let Some(reset) = self + .is_account_rate_allowed( + KV_RATE_LIMIT_IMAP, + access_token.account_id(), + rate, + access_token.inner.rate_counter(AccountRate::Imap), + ) + .await + .caused_by(trc::location!())? + { + Err(trc::LimitEvent::TooManyRequests + .into_err() + .ctx(trc::Key::Expires, reset) + .ctx(trc::Key::Limit, rate.count) + .details(format_compact!( + "Too many requests, retry in {reset} seconds" + ))) } else { - None + Ok(()) + } + } + + async fn is_account_rate_allowed( + &self, + prefix: u8, + account_id: u32, + rate: &Rate, + counter: Option<&RateCounter>, + ) -> trc::Result> { + let key = account_id.to_be_bytes(); + let Some(counter) = counter else { + return self + .core + .storage + .memory + .is_rate_allowed(prefix, &key, rate) + .await; }; - if let Some((reset, count)) = rate_reset { - if access_token.has_permission(Permission::UnlimitedRequests) { - Ok(None) - } else { - Err(trc::LimitEvent::TooManyRequests - .into_err() - .ctx(trc::Key::Expires, reset) - .ctx(trc::Key::Limit, count)) - } - } else { - match access_token.is_http_request_allowed() { - LimiterResult::Allowed(in_flight) => Ok(Some(in_flight)), - LimiterResult::Forbidden => { - if access_token.has_permission(Permission::UnlimitedRequests) { - Ok(None) - } else { - Err(trc::LimitEvent::ConcurrentRequest - .into_err() - .ctx(trc::Key::Limit, access_token.concurrent_http_requests())) - } - } - LimiterResult::Disabled => Ok(None), + match counter.hit(rate, now()) { + RateHit::Allowed => Ok(None), + RateHit::Exceeded { reset_in } => Ok(Some(reset_in)), + RateHit::Flush(flush) => { + let window = flush.window(); + let total = self + .core + .storage + .memory + .key_incr( + KeyValue::new(window.bucket(prefix, &key), i64::from(flush.delta())) + .expires(window.expires_in), + ) + .await?; + Ok(flush.settle(rate, u64::try_from(total).unwrap_or_default())) } } } + pub(crate) fn account_limits(&self) -> Option> { + let is_remote = self.core.storage.memory.is_remote(); + AccountLimits::new( + [ + self.core.jmap.request_max_concurrent, + self.core.imap.rate_concurrent, + self.core.jmap.upload_max_concurrent, + ], + [ + self.core.network.http.rate_authenticated.is_some() && is_remote, + self.core.imap.rate_requests.is_some() && is_remote, + ], + ) + } + pub async fn is_http_anonymous_request_allowed(&self, addr: IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.network.http.rate_anonymous && !self.is_ip_allowed(addr) @@ -72,12 +141,7 @@ impl Server { .core .storage .memory - .is_rate_allowed( - KV_RATE_LIMIT_HTTP_ANONYMOUS, - &ip_to_bytes(&addr), - rate, - false, - ) + .is_rate_allowed(KV_RATE_LIMIT_HTTP_ANONYMOUS, &ip_to_bytes(&addr), rate) .await .caused_by(trc::location!())? { diff --git a/crates/common/src/auth/sasl.rs b/crates/common/src/auth/sasl.rs new file mode 100644 index 00000000..532646a4 --- /dev/null +++ b/crates/common/src/auth/sasl.rs @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Server, auth::oauth::SCOPE_MAIL}; +use directory::{Credentials, backend::oidc::openid_configuration_url}; +use encodify::base64::STANDARD; +use serde::Serialize; + +const XOAUTH2_SCHEMES: &str = "Bearer"; +pub const SASL_CANCEL: &str = "*"; +pub const SASL_EMPTY: &str = "="; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BearerMechanism { + OAuthBearer, + XOAuth2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BearerFailure { + InvalidToken, + InsufficientScope, +} + +#[derive(Serialize)] +struct OAuthBearerFailure { + status: &'static str, + scope: &'static str, + #[serde( + rename = "openid-configuration", + skip_serializing_if = "Option::is_none" + )] + openid_configuration: Option, +} + +#[derive(Serialize)] +struct XOAuth2Failure { + status: &'static str, + schemes: &'static str, + scope: &'static str, +} + +impl BearerFailure { + pub fn insufficient_scope(cause: trc::Error) -> trc::Error { + trc::AuthEvent::Error + .into_err() + .details("The token scope does not allow access to this service.") + .caused_by(cause) + } + + pub fn from_error(err: &trc::Error) -> Option { + matches!( + err.as_ref(), + trc::EventType::Auth( + trc::AuthEvent::Failed + | trc::AuthEvent::TokenExpired + | trc::AuthEvent::CredentialExpired + ) + ) + .then_some(BearerFailure::InvalidToken) + } + + fn oauthbearer_status(self) -> &'static str { + match self { + BearerFailure::InvalidToken => "invalid_token", + BearerFailure::InsufficientScope => "insufficient_scope", + } + } + + fn xoauth2_status(self) -> &'static str { + match self { + BearerFailure::InvalidToken => "401", + BearerFailure::InsufficientScope => "403", + } + } +} + +impl Server { + #[inline] + pub fn accepts_bearer_tokens(&self, is_tls: bool) -> bool { + is_tls || self.registry().is_bootstrap_mode() || self.registry().is_recovery_mode() + } + + pub async fn bearer_failure_challenge( + &self, + mechanism: BearerMechanism, + failure: BearerFailure, + credentials: Option<&Credentials>, + ) -> String { + let json = match mechanism { + BearerMechanism::OAuthBearer => serde_json::to_vec(&OAuthBearerFailure { + status: failure.oauthbearer_status(), + scope: SCOPE_MAIL, + openid_configuration: self.sasl_openid_configuration(credentials).await, + }), + BearerMechanism::XOAuth2 => serde_json::to_vec(&XOAuth2Failure { + status: failure.xoauth2_status(), + schemes: XOAUTH2_SCHEMES, + scope: SCOPE_MAIL, + }), + } + .unwrap_or_default(); + + STANDARD.encode(json) + } + + pub async fn sasl_openid_configuration( + &self, + credentials: Option<&Credentials>, + ) -> Option { + if let Some(Credentials::Bearer { username, token }) = credentials + && let Ok(Some(discovery)) = self + .bearer_directory(username.as_deref(), token) + .await + .map(|directory| { + directory.and_then(|directory| directory.oidc_discovery_document()) + }) + { + return Some(discovery.configuration_url().to_string()); + } + + let url_https = &self.core.network.http.url_https; + (!url_https.is_empty()).then(|| openid_configuration_url(url_https)) + } +} + diff --git a/crates/common/src/cache/directory.rs b/crates/common/src/cache/directory.rs index 896214b4..0003a035 100644 --- a/crates/common/src/cache/directory.rs +++ b/crates/common/src/cache/directory.rs @@ -5,9 +5,10 @@ */ use crate::{Server, auth::DomainCache, cache::invalidate::CacheInvalidationBuilder}; +use compact_str::CompactString; use registry::{ schema::{ - prelude::{Object, ObjectType}, + prelude::{Object, ObjectInner, ObjectType}, structs::{ Account, Credential, EmailAlias, GroupAccount, PasswordCredential, Roles, UserAccount, UserRoles, @@ -30,6 +31,14 @@ impl Server { &self, account: directory::Account, ) -> trc::Result { + Box::pin(self.sync_account(account)).await + } + + pub async fn synchronize_group(&self, group: directory::Group) -> trc::Result { + Box::pin(self.sync_group(group)).await + } + + async fn sync_account(&self, account: directory::Account) -> trc::Result { let (local, domain) = self.validate_address(&account.email).await?; match self @@ -51,48 +60,45 @@ impl Server { .ctx(trc::Key::AccountId, account_id) })?; - let mut updated_account = Account::from(current_account.clone()) - .into_user() - .ok_or_else(|| { - trc::AuthEvent::Error + let current_user = match ¤t_account.inner { + ObjectInner::Account(current) if let Account::User(user) = current.as_ref() => { + user + } + _ => { + return Err(trc::AuthEvent::Error .into_err() .details( "Account ID from directory does not correspond to a user account", ) .ctx(trc::Key::AccountName, account.email.clone()) - .ctx(trc::Key::AccountId, account_id) - })?; + .ctx(trc::Key::AccountId, account_id)); + } + }; - let mut has_changes = false; - if let Some(secret) = account.secret - && secret != updated_account.password().unwrap_or_default() - { - has_changes = true; - updated_account.set_password(secret); - } - if account.description.is_some() - && account.description != updated_account.description - { - updated_account.description = account.description; - has_changes = true; - } + let secret = account + .secret + .filter(|secret| secret != current_user.password().unwrap_or_default()); + let description = account + .description + .filter(|description| current_user.description.as_ref() != Some(description)); + let mut new_aliases = Vec::new(); for alias in account.email_aliases { if let Some((local, alias_domain)) = self.validate_alias(&alias).await? - && alias_domain.id_tenant == domain.id_tenant + && alias_domain.is_same_tenant(&domain) && self .rcpt_id_from_parts(local, alias_domain.id) .await? .is_none() { - updated_account.aliases.push(EmailAlias { + new_aliases.push(EmailAlias { name: local.to_string(), domain_id: Id::from(alias_domain.id), - enabled: true, + enable: true, description: None, }); - has_changes = true; } } + let mut new_member_group_ids = None; if let Some(groups) = account.groups { let mut member_group_ids = Vec::with_capacity(groups.len()); for email in groups { @@ -106,18 +112,35 @@ impl Server { .into(), ); } - if updated_account.member_group_ids.len() != member_group_ids.len() - || !updated_account + if current_user.member_group_ids.len() != member_group_ids.len() + || !current_user .member_group_ids .iter() .all(|id| member_group_ids.contains(id)) { - updated_account.member_group_ids = member_group_ids.into(); - has_changes = true; + new_member_group_ids = Some(member_group_ids); } } + let has_changes = secret.is_some() + || description.is_some() + || !new_aliases.is_empty() + || new_member_group_ids.is_some(); + if has_changes { + let mut updated_account = current_user.clone(); + if let Some(secret) = secret { + updated_account.set_password(secret); + } + if description.is_some() { + updated_account.description = description; + } + for alias in new_aliases { + updated_account.aliases.push(alias); + } + if let Some(member_group_ids) = new_member_group_ids { + updated_account.member_group_ids = member_group_ids.into(); + } let updated_account = Object::from(Account::User(updated_account)); match self .registry() @@ -150,7 +173,7 @@ impl Server { } else { Ok(AccountWithId { id: account_id, - account: Account::User(updated_account), + account: Account::from(current_account), }) } } @@ -159,7 +182,7 @@ impl Server { let mut aliases = Vec::with_capacity(account.email_aliases.len()); for alias in account.email_aliases { if let Some((local, alias_domain)) = self.validate_alias(&alias).await? - && alias_domain.id_tenant == domain.id_tenant + && alias_domain.is_same_tenant(&domain) && self .rcpt_id_from_parts(local, alias_domain.id) .await? @@ -168,7 +191,7 @@ impl Server { aliases.push(EmailAlias { name: local.to_string(), domain_id: Id::from(alias_domain.id), - enabled: true, + enable: true, description: None, }); } @@ -192,7 +215,6 @@ impl Server { created_at: UTCDateTime::now(), description: account.description, member_group_ids: member_group_ids.into(), - member_tenant_id: domain.id_tenant.map(Id::from), roles: UserRoles::User, credentials: List::from_iter(account.secret.map(|secret| { Credential::Password(PasswordCredential { @@ -233,7 +255,7 @@ impl Server { } } - pub async fn synchronize_group(&self, group: directory::Group) -> trc::Result { + async fn sync_group(&self, group: directory::Group) -> trc::Result { let (local, domain) = self.validate_address(&group.email).await?; match self @@ -241,6 +263,25 @@ impl Server { .await .caused_by(trc::location!())? { + Some(account_id) if group.description.is_none() && group.email_aliases.is_empty() => { + match self + .try_account(account_id) + .await + .caused_by(trc::location!())? + { + Some(account) if !account.is_user_account() => Ok(account_id), + Some(_) => Err(trc::AuthEvent::Error + .into_err() + .details("Account ID from directory does not correspond to a group account") + .ctx(trc::Key::AccountName, group.email) + .ctx(trc::Key::AccountId, account_id)), + None => Err(trc::AuthEvent::Error + .into_err() + .details("Account ID from directory does not exist in registry") + .ctx(trc::Key::AccountName, group.email) + .ctx(trc::Key::AccountId, account_id)), + } + } Some(account_id) => { let current_account = self .registry() @@ -274,7 +315,7 @@ impl Server { } for alias in group.email_aliases { if let Some((local, alias_domain)) = self.validate_alias(&alias).await? - && alias_domain.id_tenant == domain.id_tenant + && alias_domain.is_same_tenant(&domain) && self .rcpt_id_from_parts(local, alias_domain.id) .await? @@ -283,7 +324,7 @@ impl Server { updated_account.aliases.push(EmailAlias { name: local.to_string(), domain_id: Id::from(alias_domain.id), - enabled: true, + enable: true, description: None, }); has_changes = true; @@ -326,7 +367,7 @@ impl Server { let mut aliases = Vec::with_capacity(group.email_aliases.len()); for alias in group.email_aliases { if let Some((local, alias_domain)) = self.validate_alias(&alias).await? - && alias_domain.id_tenant == domain.id_tenant + && alias_domain.is_same_tenant(&domain) && self .rcpt_id_from_parts(local, alias_domain.id) .await? @@ -335,7 +376,7 @@ impl Server { aliases.push(EmailAlias { name: local.to_string(), domain_id: Id::from(alias_domain.id), - enabled: true, + enable: true, description: None, }); } @@ -347,7 +388,6 @@ impl Server { aliases: aliases.into(), created_at: UTCDateTime::now(), description: group.description, - member_tenant_id: domain.id_tenant.map(Id::from), roles: Roles::Default, ..Default::default() })); @@ -397,12 +437,12 @@ impl Server { trc::AuthEvent::Error .into_err() .details("Account domain does not exist or has been disabled") - .ctx(trc::Key::Domain, domain.to_string()) + .ctx(trc::Key::Domain, CompactString::from(domain)) }), None => { trc::event!( Auth(trc::AuthEvent::Warning), - AccountName = email.to_string().clone(), + AccountName = CompactString::from(email), Details = "Directory account is not an email, appended default domain", ); self.domain_by_id(self.core.email.default_domain_id) diff --git a/crates/common/src/cache/email.rs b/crates/common/src/cache/email.rs new file mode 100644 index 00000000..c8e3efd6 --- /dev/null +++ b/crates/common/src/cache/email.rs @@ -0,0 +1,1924 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + CACHE_CHUNK, ColBlock, CustomKeywords, MAX_RECEIVED_AT, MessageCache, MessageUid, MessagesCache, +}; +use compact_str::CompactString; +use std::{ + iter::{Chain, FusedIterator, Peekable}, + slice::Iter, + sync::Arc, +}; +use store::roaring::RoaringBitmap; +use tinyvec::TinyVec; + +const INDEX_CHUNK: usize = 8000; + +const INDEX_BUCKET: u64 = 16; + +const INDEX_TABLE_FACTOR: usize = 2; + +const INDEX_TAIL: usize = 8; + +const MAX_SHIFT: u32 = 31; + +const _: () = assert!(INDEX_CHUNK <= u16::MAX as usize); + +const APPEND_BLOCK: usize = 1024; + +const THREAD_WINDOW: u32 = 1 << 26; + +impl MessageCache { + #[allow(clippy::too_many_arguments)] + pub fn new( + document_id: u32, + mailboxes: TinyVec<[MessageUid; 2]>, + keywords: u32, + thread_id: u32, + change_id: u64, + size: u32, + received_at: u64, + sent_at: i32, + ) -> Self { + Self { + document_id, + mailboxes, + keywords, + thread_id, + change_id, + size, + received_at, + sent_at, + } + } + + #[inline(always)] + pub fn document_id(&self) -> u32 { + self.document_id + } + + #[inline(always)] + pub fn mailboxes(&self) -> &[MessageUid] { + &self.mailboxes + } + + #[inline(always)] + pub fn keywords(&self) -> u32 { + self.keywords + } + + #[inline(always)] + pub fn thread_id(&self) -> u32 { + self.thread_id + } + + #[inline(always)] + pub fn change_id(&self) -> u64 { + self.change_id + } + + #[inline(always)] + pub fn size(&self) -> u32 { + self.size + } + + #[inline(always)] + pub fn received_at(&self) -> u64 { + self.received_at + } + + #[inline(always)] + pub fn received_at_secs(&self) -> u32 { + debug_assert!( + self.received_at <= MAX_RECEIVED_AT, + "received_at {} exceeds the representable range and should have been clamped by the caller", + self.received_at + ); + self.received_at.min(MAX_RECEIVED_AT) as u32 + } + + #[inline(always)] + pub fn sent_at(&self) -> i32 { + self.sent_at + } + + #[inline(always)] + pub fn sort_rank(&self) -> (u64, u32) { + (self.received_at, self.document_id) + } + + #[inline(always)] + pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { + self.mailboxes + .iter() + .any(|uid| uid.mailbox_id == mailbox_id) + } + + #[inline(always)] + pub fn uid_in(&self, mailbox_id: u32) -> Option { + self.mailboxes + .iter() + .find(|uid| uid.mailbox_id == mailbox_id) + .map(|uid| uid.uid) + } +} + +impl ColBlock { + fn from_records(items: &[MessageCache]) -> Self { + let mut mb_offsets = Vec::with_capacity(items.len() + 1); + let mut mb_arena = Vec::with_capacity(items.len()); + let mut offset = 0u32; + for item in items { + mb_offsets.push(offset); + mb_arena.extend_from_slice(&item.mailboxes); + offset += item.mailboxes.len() as u32; + } + mb_offsets.push(offset); + let thread_ids: Arc<[u32]> = items.iter().map(|item| item.thread_id).collect(); + + ColBlock { + document_ids: items.iter().map(|item| item.document_id).collect(), + change_ids: items.iter().map(|item| item.change_id).collect(), + received_at: items.iter().map(|item| item.received_at_secs()).collect(), + sent_at: items.iter().map(|item| item.sent_at).collect(), + sizes: items.iter().map(|item| item.size).collect(), + keywords: items.iter().map(|item| item.keywords).collect(), + thread_range: ThreadRange::of(&thread_ids), + thread_ids, + mb_offsets: mb_offsets.into(), + mb_arena: mb_arena.into(), + } + } + + #[inline(always)] + fn len(&self) -> usize { + self.document_ids.len() + } + + #[inline(always)] + fn mailboxes(&self, slot: u32) -> &[MessageUid] { + let from = self.mb_offsets[slot as usize] as usize; + let to = self.mb_offsets[slot as usize + 1] as usize; + &self.mb_arena[from..to] + } + + #[inline(always)] + fn rank(&self, slot: usize) -> (u64, u32) { + (self.received_at[slot] as u64, self.document_ids[slot]) + } + + fn patch_slots(&mut self, slots: &[(u32, u32, &MessageCache)]) { + Self::patch_column(&mut self.change_ids, slots, |record| record.change_id); + Self::patch_column(&mut self.keywords, slots, |record| record.keywords); + Self::patch_column(&mut self.thread_ids, slots, |record| record.thread_id); + Self::patch_column(&mut self.sizes, slots, |record| record.size); + self.patch_mailboxes(slots); + for (_, _, record) in slots { + self.thread_range.widen(record.thread_id); + } + } + + fn patch_column( + column: &mut Arc<[T]>, + slots: &[(u32, u32, &MessageCache)], + pick: impl Fn(&MessageCache) -> T, + ) { + if slots + .iter() + .all(|(_, slot, record)| column[*slot as usize] == pick(record)) + { + return; + } + let patched = Arc::make_mut(column); + for (_, slot, record) in slots { + if let Some(value) = patched.get_mut(*slot as usize) { + *value = pick(record); + } + } + } + + fn patch_mailboxes(&mut self, slots: &[(u32, u32, &MessageCache)]) { + if slots + .iter() + .all(|(_, slot, record)| self.mailboxes(*slot) == record.mailboxes.as_slice()) + { + return; + } + let mut mb_offsets = Vec::with_capacity(self.len() + 1); + let mut mb_arena = Vec::with_capacity(self.mb_arena.len()); + let mut offset = 0u32; + let mut updated = slots.iter().peekable(); + for slot in 0..self.len() as u32 { + mb_offsets.push(offset); + let mailboxes = match updated.next_if(|(_, at, _)| *at == slot) { + Some((_, _, record)) => record.mailboxes.as_slice(), + None => self.mailboxes(slot), + }; + mb_arena.extend_from_slice(mailboxes); + offset += mailboxes.len() as u32; + } + mb_offsets.push(offset); + self.mb_offsets = mb_offsets.into(); + self.mb_arena = mb_arena.into(); + } + + fn weight(&self) -> u64 { + ((self.document_ids.len() + * (std::mem::size_of::() * 5 + + std::mem::size_of::() + + std::mem::size_of::())) + + (self.mb_offsets.len() * std::mem::size_of::()) + + (self.mb_arena.len() * std::mem::size_of::()) + + std::mem::size_of::()) as u64 + } +} + +struct ColBuilder { + document_ids: Vec, + change_ids: Vec, + received_at: Vec, + sent_at: Vec, + sizes: Vec, + keywords: Vec, + thread_ids: Vec, + mb_offsets: Vec, + mb_arena: Vec, +} + +impl ColBuilder { + fn with_capacity(rows: usize, arena: usize) -> Self { + ColBuilder { + document_ids: Vec::with_capacity(rows), + change_ids: Vec::with_capacity(rows), + received_at: Vec::with_capacity(rows), + sent_at: Vec::with_capacity(rows), + sizes: Vec::with_capacity(rows), + keywords: Vec::with_capacity(rows), + thread_ids: Vec::with_capacity(rows), + mb_offsets: Vec::with_capacity(rows + 1), + mb_arena: Vec::with_capacity(arena), + } + } + + #[inline(always)] + fn len(&self) -> usize { + self.document_ids.len() + } + + #[inline(always)] + fn push_record(&mut self, record: &MessageCache) { + self.document_ids.push(record.document_id); + self.change_ids.push(record.change_id); + self.received_at.push(record.received_at_secs()); + self.sent_at.push(record.sent_at); + self.sizes.push(record.size); + self.keywords.push(record.keywords); + self.thread_ids.push(record.thread_id); + self.mb_offsets.push(self.mb_arena.len() as u32); + self.mb_arena.extend_from_slice(&record.mailboxes); + } + + fn copy_run(&mut self, old: &ColBlock, from: usize, to: usize) { + if from >= to { + return; + } + self.document_ids + .extend_from_slice(&old.document_ids[from..to]); + self.change_ids.extend_from_slice(&old.change_ids[from..to]); + self.received_at + .extend_from_slice(&old.received_at[from..to]); + self.sent_at.extend_from_slice(&old.sent_at[from..to]); + self.sizes.extend_from_slice(&old.sizes[from..to]); + self.keywords.extend_from_slice(&old.keywords[from..to]); + self.thread_ids.extend_from_slice(&old.thread_ids[from..to]); + + let base = old.mb_offsets[from]; + let shift = self.mb_arena.len() as u32; + self.mb_offsets + .extend(old.mb_offsets[from..to].iter().map(|o| o - base + shift)); + self.mb_arena.extend_from_slice( + &old.mb_arena[old.mb_offsets[from] as usize..old.mb_offsets[to] as usize], + ); + } + + fn finish_into(mut self, out: &mut BlockList) { + let rows = self.len(); + if rows == 0 { + return; + } + self.mb_offsets.push(self.mb_arena.len() as u32); + + if rows <= CACHE_CHUNK * 2 { + out.push(ColBlock { + document_ids: self.document_ids.into(), + change_ids: self.change_ids.into(), + received_at: self.received_at.into(), + sent_at: self.sent_at.into(), + sizes: self.sizes.into(), + keywords: self.keywords.into(), + thread_range: ThreadRange::of(&self.thread_ids), + thread_ids: self.thread_ids.into(), + mb_offsets: self.mb_offsets.into(), + mb_arena: self.mb_arena.into(), + }); + return; + } + + let mut from = 0usize; + while from < rows { + let to = (from + CACHE_CHUNK).min(rows); + let base = self.mb_offsets[from]; + let arena_from = base as usize; + let arena_to = self.mb_offsets[to] as usize; + let mut mb_offsets = self.mb_offsets[from..to] + .iter() + .map(|offset| offset - base) + .collect::>(); + mb_offsets.push((arena_to - arena_from) as u32); + + out.push(ColBlock { + document_ids: self.document_ids[from..to].into(), + change_ids: self.change_ids[from..to].into(), + received_at: self.received_at[from..to].into(), + sent_at: self.sent_at[from..to].into(), + sizes: self.sizes[from..to].into(), + keywords: self.keywords[from..to].into(), + thread_range: ThreadRange::of(&self.thread_ids[from..to]), + thread_ids: self.thread_ids[from..to].into(), + mb_offsets: mb_offsets.into(), + mb_arena: self.mb_arena[arena_from..arena_to].into(), + }); + from = to; + } + } +} + +const SHIFT_BUCKET: usize = 1024; + +struct PositionShift { + steps: Vec<(u32, i32)>, + bucket_first: Vec, + bucket_delta: Vec, + first_key: u32, +} + +impl PositionShift { + fn build( + insert_positions: impl Iterator, + delete_positions: impl Iterator, + old_len: usize, + ) -> Self { + let mut steps: Vec<(u32, i32)> = Vec::new(); + let mut inserted = insert_positions.peekable(); + let mut deleted = delete_positions.map(|position| position + 1).peekable(); + let mut total = 0i32; + loop { + let (key, delta) = match (inserted.peek(), deleted.peek()) { + (Some(insert), Some(delete)) if insert > delete => (deleted.next().unwrap(), -1i32), + (Some(_), _) => (inserted.next().unwrap(), 1i32), + (None, Some(_)) => (deleted.next().unwrap(), -1i32), + (None, None) => break, + }; + + total += delta; + if let Some(last) = steps.last_mut() + && last.0 == key + { + last.1 = total; + } else { + steps.push((key, total)); + } + } + + let buckets = old_len / SHIFT_BUCKET + 2; + let mut bucket_first = Vec::with_capacity(buckets); + let mut bucket_delta = Vec::with_capacity(buckets); + let mut cursor = steps.iter().enumerate().peekable(); + let mut carried = 0i32; + for bucket in 0..buckets { + let start = (bucket * SHIFT_BUCKET) as u32; + while let Some((_, (_, delta))) = cursor.next_if(|(_, (key, _))| *key < start) { + carried = *delta; + } + bucket_first.push(cursor.peek().map_or(steps.len(), |(at, _)| *at) as u32); + bucket_delta.push(carried); + } + + PositionShift { + first_key: steps.first().map_or(u32::MAX, |(key, _)| *key), + steps, + bucket_first, + bucket_delta, + } + } + + fn span(&self, position: u32) -> (i32, u32, u32) { + if position < self.first_key { + return (0, 0, self.first_key); + } + let bucket = (position as usize / SHIFT_BUCKET).min(self.bucket_first.len() - 1); + let from = self.bucket_first[bucket] as usize; + let next = from + + self.steps[from..] + .iter() + .take_while(|(key, _)| *key <= position) + .count(); + let (low, delta) = next + .checked_sub(1) + .and_then(|last| self.steps.get(last)) + .copied() + .unwrap_or((0, 0)); + let high = self.steps.get(next).map_or(u32::MAX, |(key, _)| *key); + (delta, low, high) + } + + fn constant_over(&self, from: u32, to: u32) -> Option { + let next = self.steps.partition_point(|(key, _)| *key <= from); + self.steps + .get(next) + .is_none_or(|(key, _)| *key > to) + .then(|| self.at(from)) + } + + #[inline(always)] + fn at(&self, position: u32) -> i32 { + if position < self.first_key { + return 0; + } + let bucket = (position as usize / SHIFT_BUCKET).min(self.bucket_first.len() - 1); + let from = self.bucket_first[bucket] as usize; + + self.steps[from..] + .iter() + .take_while(|(key, _)| *key <= position) + .last() + .map_or(self.bucket_delta[bucket], |(_, delta)| *delta) + } +} + +struct ShiftCursor<'x> { + shift: &'x PositionShift, + delta: i32, + low: u32, + high: u32, +} + +impl<'x> ShiftCursor<'x> { + fn new(shift: &'x PositionShift) -> Self { + ShiftCursor { + shift, + delta: 0, + low: 1, + high: 0, + } + } + + #[inline(always)] + fn apply(&mut self, position: u32) -> u32 { + if position < self.low || position >= self.high { + (self.delta, self.low, self.high) = self.shift.span(position); + } + position.wrapping_add_signed(self.delta) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct IdIndex { + sealed: Arc<[IndexChunk]>, + tail: Arc<[IndexChunk]>, + firsts: Arc<[u32]>, + table: Arc<[u32]>, + base: u32, + shift: u32, + chunks_weight: u64, +} + +#[derive(Debug, Clone)] +struct IndexChunk { + data: Arc<[u32]>, + first: u32, + len: u32, + min_position: u32, + max_position: u32, + shift: u32, +} + +impl IndexChunk { + fn build(pairs: &[[u32; 2]], directory: &mut Vec) -> Option { + let ([first, _], [last, _]) = (pairs.first()?, pairs.last()?); + let (first, last) = (*first, *last); + let len = pairs.len(); + let shift = Self::shift_of(len, last - first); + + let rows = len as u16; + directory.clear(); + directory.resize(((last - first) >> shift) as usize + 2, rows); + for (at, [document_id, _]) in (0..rows).zip(pairs).rev() { + if let Some(start) = directory.get_mut(((document_id - first) >> shift) as usize) { + *start = at; + } + } + let mut next = rows; + for start in directory.iter_mut().rev() { + next = next.min(*start); + *start = next; + } + + let (min_position, max_position) = Self::position_range(pairs); + + let mut data: Arc<[u32]> = + std::iter::repeat_n(0, len * 2 + directory.len().div_ceil(2)).collect(); + let (flat, words) = Arc::make_mut(&mut data).split_at_mut(len * 2); + flat.copy_from_slice(pairs.as_flattened()); + for (word, halves) in words.iter_mut().zip(directory.chunks(2)) { + *word = halves + .iter() + .rev() + .fold(0, |word, half| (word << 16) | u32::from(*half)); + } + + Some(IndexChunk { + data, + first, + len: len as u32, + min_position, + max_position, + shift, + }) + } + + fn decode( + data: &[T], + rows: usize, + native: impl Fn(&T) -> u32 + Copy, + len: usize, + after: Option, + ) -> Option { + if rows == 0 || rows > INDEX_CHUNK { + return None; + } + let (flat, words) = data.split_at_checked(rows * 2)?; + let first = native(flat.first()?); + let range = native(flat.get(rows * 2 - 2)?).checked_sub(first)?; + let shift = Self::shift_of(rows, range); + let buckets = (range >> shift) as usize + 2; + if words.len() != buckets.div_ceil(2) || after.is_some_and(|after| after >= first) { + return None; + } + + let data = data.iter().map(native).collect::>(); + let (flat, words) = data.split_at(rows * 2); + let pairs = flat.as_chunks::<2>().0; + let [_, first_position] = *pairs.first()?; + let (unordered, min_position, max_position) = pairs.iter().zip(pairs.iter().skip(1)).fold( + (0u32, first_position, first_position), + |(unordered, min, max), ([previous, _], [next, position])| { + ( + unordered | u32::from(previous >= next), + min.min(*position), + max.max(*position), + ) + }, + ); + let bucket_of = |at: usize| { + pairs + .get(at) + .map(|[document_id, _]| (document_id.wrapping_sub(first) >> shift) as usize) + }; + let indexed = (0..buckets).all(|bucket| { + directory_entry(words, bucket).is_some_and(|start| { + (start == 0 || bucket_of(start - 1).is_some_and(|before| before < bucket)) + && (start == rows || bucket_of(start).is_some_and(|at| at >= bucket)) + }) + }); + if unordered != 0 || !indexed || max_position as usize >= len { + return None; + } + + Some(IndexChunk { + first, + len: rows as u32, + min_position, + max_position, + shift, + data, + }) + } + + #[inline(always)] + fn shift_of(len: usize, range: u32) -> u32 { + let span = (u64::from(range) + 1) * INDEX_BUCKET; + let mut shift = 0u32; + while shift < MAX_SHIFT && ((len as u64) << shift) < span { + shift += 1; + } + shift + } + + #[inline(always)] + fn position_range(pairs: &[[u32; 2]]) -> (u32, u32) { + pairs + .iter() + .fold((u32::MAX, 0), |(min, max), [_, position]| { + (min.min(*position), max.max(*position)) + }) + } + + fn shifted(&self, shift: &PositionShift) -> Self { + let mut data = Arc::<[u32]>::from(&self.data[..]); + let (flat, _) = Arc::make_mut(&mut data).split_at_mut(self.len as usize * 2); + let pairs = flat.as_chunks_mut::<2>().0; + match shift.constant_over(self.min_position, self.max_position) { + Some(delta) => { + for [_, position] in pairs { + *position = position.wrapping_add_signed(delta); + } + } + None => { + let mut cursor = ShiftCursor::new(shift); + for [_, position] in pairs { + *position = cursor.apply(*position); + } + } + } + IndexChunk { + data, + min_position: self + .min_position + .wrapping_add_signed(shift.at(self.min_position)), + max_position: self + .max_position + .wrapping_add_signed(shift.at(self.max_position)), + ..*self + } + } + + #[inline(always)] + fn pairs(&self) -> &[[u32; 2]] { + self.data + .get(..self.len as usize * 2) + .unwrap_or_default() + .as_chunks::<2>() + .0 + } + + #[inline(always)] + fn position(&self, document_id: u32) -> Option { + let offset = document_id.checked_sub(self.first)?; + let (pairs, directory) = self.data.split_at_checked(self.len as usize * 2)?; + let bucket = (offset >> self.shift) as usize; + let from = directory_entry(directory, bucket)?; + let to = directory_entry(directory, bucket + 1)?; + let within = (offset & ((1u32 << self.shift) - 1)) as usize; + let width = 1usize << self.shift; + let window = pairs + .as_chunks::<2>() + .0 + .get(from.max(to.saturating_sub(width - within))..to.min(from + within + 1))?; + window + .binary_search_by_key(&document_id, |[id, _]| *id) + .ok() + .and_then(|at| window.get(at)) + .map(|[_, position]| *position) + } + + fn weight(&self) -> u64 { + (self.data.len() * std::mem::size_of::() + std::mem::size_of::()) as u64 + } +} + +#[inline(always)] +fn directory_entry(directory: &[u32], at: usize) -> Option { + directory + .get(at / 2) + .map(|word| ((word >> ((at % 2) * 16)) & 0xffff) as usize) +} + +impl IdIndex { + pub(crate) fn from_sorted(entries: &[(u32, u32)]) -> Self { + let mut writer = IndexWriter::with_capacity( + entries.len().div_ceil(INDEX_CHUNK), + entries.len().min(INDEX_CHUNK), + ); + for (document_id, position) in entries { + writer.push(*document_id, *position); + } + writer.finish() + } + + pub(crate) fn decode<'x, T: 'x>( + chunks: impl Iterator, + native: impl Fn(&T) -> u32 + Copy, + len: usize, + ) -> Option { + let mut decoded = Vec::with_capacity(chunks.size_hint().0); + let mut indexed = 0usize; + let mut after = None; + for (data, rows) in chunks { + let chunk = IndexChunk::decode(data, rows, native, len, after)?; + after = chunk.pairs().last().map(|[document_id, _]| *document_id); + indexed += rows; + decoded.push(chunk); + } + (indexed == len).then(|| Self::from_chunks(decoded)) + } + + fn from_chunks(mut chunks: Vec) -> Self { + let tail = chunks.split_off(chunks.len().saturating_sub(1)); + Self::assemble(chunks.into(), tail) + } + + fn assemble(sealed: Arc<[IndexChunk]>, tail: Vec) -> Self { + let chunks_weight = sealed + .iter() + .chain(&tail) + .map(IndexChunk::weight) + .sum::(); + let firsts = sealed + .iter() + .chain(&tail) + .map(|chunk| chunk.first) + .collect::>(); + let (Some(base), Some(last)) = (firsts.first().copied(), firsts.last().copied()) else { + return IdIndex { + sealed, + tail: tail.into(), + firsts, + table: Arc::from([]), + base: 0, + shift: 0, + chunks_weight, + }; + }; + let span = last - base; + let wanted = firsts.len() * INDEX_TABLE_FACTOR; + let mut shift = 0u32; + while shift < MAX_SHIFT && (span >> shift) as usize >= wanted { + shift += 1; + } + + let buckets = (span >> shift) as usize + 1; + let mut table = Vec::with_capacity(buckets + 1); + let mut following = firsts.iter().skip(1).peekable(); + let mut chunk = 0u32; + for bucket in 0..buckets as u64 { + let start = u64::from(base) + (bucket << shift); + while following + .next_if(|first| u64::from(**first) <= start) + .is_some() + { + chunk += 1; + } + table.push(chunk); + } + table.push(firsts.len() as u32 - 1); + + IdIndex { + sealed, + tail: tail.into(), + firsts, + table: table.into(), + base, + shift, + chunks_weight, + } + } + + #[inline(always)] + fn position(&self, document_id: u32) -> Option { + let offset = document_id.checked_sub(self.base)?; + let bucket = ((offset >> self.shift) as usize).min(self.table.len().checked_sub(2)?); + let &[from, to] = self.table.get(bucket..bucket + 2)? else { + return None; + }; + let from = from as usize; + let chunk = from + + self + .firsts + .get(from + 1..=to as usize)? + .partition_point(|first| *first <= document_id); + self.chunk(chunk)?.position(document_id) + } + + #[inline(always)] + fn chunk(&self, at: usize) -> Option<&IndexChunk> { + match at.checked_sub(self.sealed.len()) { + None => self.sealed.get(at), + Some(at) => self.tail.get(at), + } + } + + fn chunks(&self) -> impl DoubleEndedIterator + Clone + '_ { + self.sealed.iter().chain(self.tail.iter()) + } + + pub(crate) fn chunk_parts(&self) -> impl Iterator + '_ { + self.chunks().map(|chunk| (&chunk.data[..], chunk.len)) + } + + #[cfg(all(test, stalwart_tests))] + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.chunks() + .flat_map(IndexChunk::pairs) + .map(|[document_id, position]| (*document_id, *position)) + } + + fn document_ids(&self) -> DocumentIds<'_> { + DocumentIds { + current: [].iter(), + rest: self.sealed.iter().chain(self.tail.iter()), + } + } + + fn len(&self) -> usize { + self.chunks().map(|chunk| chunk.len as usize).sum() + } + + fn weight(&self) -> u64 { + self.chunks_weight + + ((self.table.len() + self.firsts.len()) * std::mem::size_of::()) as u64 + } + + fn last_id(&self) -> Option { + self.tail + .last() + .and_then(|chunk| chunk.pairs().last()) + .map(|[document_id, _]| *document_id) + } + + fn append(&self, entries: &[(u32, u32)]) -> Self { + if entries.is_empty() { + return self.clone(); + } + let Some((last, settled)) = self.tail.split_last() else { + return Self::from_sorted(entries); + }; + + if last.len as usize + entries.len() <= INDEX_CHUNK { + let mut pairs = Vec::with_capacity(last.len as usize + entries.len()); + pairs.extend_from_slice(last.pairs()); + pairs.extend( + entries + .iter() + .map(|(document_id, position)| [*document_id, *position]), + ); + if let Some(chunk) = IndexChunk::build(&pairs, &mut Vec::new()) { + return IdIndex { + sealed: self.sealed.clone(), + chunks_weight: self.chunks_weight - last.weight() + chunk.weight(), + tail: settled.iter().cloned().chain([chunk]).collect(), + firsts: self.firsts.clone(), + table: self.table.clone(), + base: self.base, + shift: self.shift, + }; + } + } + + let mut writer = IndexWriter::with_capacity( + entries.len() / INDEX_CHUNK + 2, + (last.len as usize % INDEX_CHUNK + entries.len()).min(INDEX_CHUNK), + ); + if (last.len as usize) < INDEX_CHUNK { + writer.extend(last.pairs(), None); + } else { + writer.share(last, None); + } + for (document_id, position) in entries { + writer.push(*document_id, *position); + } + let fresh = writer.into_chunks(); + if settled.len() + fresh.len() <= INDEX_TAIL { + Self::assemble( + self.sealed.clone(), + settled.iter().cloned().chain(fresh).collect(), + ) + } else { + Self::from_chunks( + self.sealed + .iter() + .chain(settled) + .cloned() + .chain(fresh) + .collect(), + ) + } + } + + fn rebuild( + &self, + shift: &PositionShift, + inserted: &[(u32, u32)], + deleted: &[u32], + len: usize, + ) -> Self { + let mut writer = IndexWriter::with_capacity( + self.firsts.len() + inserted.len() / INDEX_CHUNK + 1, + len.min(INDEX_CHUNK), + ); + let mut added = inserted.iter().peekable(); + let mut removed = deleted.iter().peekable(); + let uppers = self + .firsts + .iter() + .skip(1) + .map(|first| Some(*first)) + .chain([None]); + + for (chunk, upper) in self.chunks().zip(uppers) { + let below = |document_id: u32| upper.is_none_or(|upper| document_id < upper); + let shifted = (chunk.max_position >= shift.first_key).then_some(shift); + if !added.peek().is_some_and(|(id, _)| below(*id)) + && !removed.peek().is_some_and(|id| below(**id)) + { + writer.share(chunk, shifted); + continue; + } + + let mut pairs = chunk.pairs(); + loop { + let next_added = added.peek().map(|(id, _)| *id).filter(|id| below(*id)); + let next_removed = removed.peek().map(|id| **id).filter(|id| below(*id)); + let Some(event) = next_added.into_iter().chain(next_removed).min() else { + break; + }; + let (run, rest) = pairs.split_at(pairs.partition_point(|[id, _]| *id < event)); + writer.extend(run, shifted); + pairs = rest; + + if let Some((id, at)) = added.next_if(|(id, _)| *id == event) { + writer.push(*id, *at); + } else if removed.next().is_some() + && let Some(([id, _], rest)) = pairs.split_first() + && *id == event + { + pairs = rest; + } + } + writer.extend(pairs, shifted); + } + + for (id, at) in added { + writer.push(*id, *at); + } + writer.finish() + } +} + +struct DocumentIds<'x> { + current: Iter<'x, [u32; 2]>, + rest: Chain, Iter<'x, IndexChunk>>, +} + +impl Iterator for DocumentIds<'_> { + type Item = u32; + + fn next(&mut self) -> Option { + loop { + if let Some([document_id, _]) = self.current.next() { + return Some(*document_id); + } + self.current = self.rest.next()?.pairs().iter(); + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.current.len() + + self + .rest + .clone() + .map(|chunk| chunk.len as usize) + .sum::(); + (len, Some(len)) + } + + fn fold(self, init: B, mut fold: F) -> B + where + F: FnMut(B, u32) -> B, + { + let init = self + .current + .fold(init, |acc, [document_id, _]| fold(acc, *document_id)); + self.rest.fold(init, |acc, chunk| { + chunk + .pairs() + .iter() + .fold(acc, |acc, [document_id, _]| fold(acc, *document_id)) + }) + } +} + +impl ExactSizeIterator for DocumentIds<'_> {} + +impl FusedIterator for DocumentIds<'_> {} + +struct IndexWriter { + chunks: Vec, + pending: Vec<[u32; 2]>, + directory: Vec, +} + +impl IndexWriter { + fn with_capacity(chunks: usize, pending: usize) -> Self { + IndexWriter { + chunks: Vec::with_capacity(chunks), + pending: Vec::with_capacity(pending), + directory: Vec::new(), + } + } + + #[inline(always)] + fn push(&mut self, document_id: u32, position: u32) { + self.pending.push([document_id, position]); + if self.pending.len() == INDEX_CHUNK { + self.flush(); + } + } + + fn extend(&mut self, mut pairs: &[[u32; 2]], shift: Option<&PositionShift>) { + let mut cursor = shift.map(ShiftCursor::new); + while !pairs.is_empty() { + let (head, rest) = pairs.split_at(pairs.len().min(INDEX_CHUNK - self.pending.len())); + match &mut cursor { + Some(cursor) => self.pending.extend( + head.iter() + .map(|[id, position]| [*id, cursor.apply(*position)]), + ), + None => self.pending.extend_from_slice(head), + } + if self.pending.len() == INDEX_CHUNK { + self.flush(); + } + pairs = rest; + } + } + + fn share(&mut self, chunk: &IndexChunk, shift: Option<&PositionShift>) { + if !self.pending.is_empty() && self.pending.len() + chunk.len as usize <= INDEX_CHUNK { + self.extend(chunk.pairs(), shift); + } else { + self.flush(); + self.chunks.push(match shift { + Some(shift) => chunk.shifted(shift), + None => chunk.clone(), + }); + } + } + + fn flush(&mut self) { + if let Some(chunk) = IndexChunk::build(&self.pending, &mut self.directory) { + self.chunks.push(chunk); + self.pending.clear(); + } + } + + fn into_chunks(mut self) -> Vec { + self.flush(); + self.chunks + } + + fn finish(self) -> IdIndex { + IdIndex::from_chunks(self.into_chunks()) + } +} + +struct BlockList { + blocks: Vec>, + starts: Vec, + len: u32, + removed: u64, + added: u64, +} + +impl BlockList { + fn with_capacity(blocks: usize) -> Self { + BlockList { + blocks: Vec::with_capacity(blocks), + starts: Vec::with_capacity(blocks), + len: 0, + removed: 0, + added: 0, + } + } + + fn of(cache: &MessagesCache) -> Self { + BlockList { + blocks: cache.blocks.clone(), + starts: cache.starts.clone(), + len: cache.len as u32, + removed: 0, + added: 0, + } + } + + #[inline(always)] + fn share(&mut self, block: &Arc, rows: u32) { + self.starts.push(self.len); + self.len += rows; + self.blocks.push(block.clone()); + } + + fn push(&mut self, block: ColBlock) { + self.added += block.weight(); + self.starts.push(self.len); + self.len += block.len() as u32; + self.blocks.push(Arc::new(block)); + } + + fn retire(&mut self, block: &ColBlock) { + self.removed += block.weight(); + } + + fn patch(&mut self, slots: &[(u32, u32, &MessageCache)]) { + let Some(block) = slots + .first() + .and_then(|(block, _, _)| self.blocks.get_mut(*block as usize)) + else { + return; + }; + self.removed += block.weight(); + Arc::make_mut(block).patch_slots(slots); + self.added += block.weight(); + } +} + +struct SpliceCursor<'x, 'r> { + deletes: Peekable>, + edits: Peekable>, + inserts: Peekable>, + inserted: Vec<(u32, u32)>, +} + +impl SpliceCursor<'_, '_> { + #[inline(always)] + fn next_slot(&mut self, block: u32) -> Option { + [ + self.deletes + .peek() + .and_then(|(at, slot)| (*at == block).then_some(*slot)), + self.edits + .peek() + .and_then(|(at, slot, _)| (*at == block).then_some(*slot)), + self.inserts + .peek() + .and_then(|(at, slot, _)| (*at == block).then_some(*slot)), + ] + .into_iter() + .flatten() + .min() + } + + fn splice_into(&mut self, builder: &mut ColBuilder, old: &ColBlock, block: u32, start: u32) { + let mut run = 0usize; + while let Some(slot) = self.next_slot(block) { + let at = slot as usize; + builder.copy_run(old, run, at); + run = run.max(at); + + while let Some((_, _, record)) = self + .inserts + .next_if(|(target, inserted_at, _)| *target == block && *inserted_at == slot) + { + self.inserted + .push((record.document_id, start + builder.len() as u32)); + builder.push_record(record); + } + + let deleted = self + .deletes + .next_if(|(target, deleted_at)| *target == block && *deleted_at == slot) + .is_some(); + let edit = self + .edits + .next_if(|(target, edited_at, _)| *target == block && *edited_at == slot); + if run == at { + if deleted { + run = at + 1; + } else if let Some((_, _, record)) = edit { + builder.push_record(record); + run = at + 1; + } + } + } + builder.copy_run(old, run, old.len()); + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ThreadRange { + min: u32, + max: u32, +} + +impl ThreadRange { + pub(crate) fn of(thread_ids: &[u32]) -> Self { + thread_ids.iter().fold( + ThreadRange { + min: u32::MAX, + max: 0, + }, + |range, thread_id| ThreadRange { + min: range.min.min(*thread_id), + max: range.max.max(*thread_id), + }, + ) + } + + #[inline(always)] + fn widen(&mut self, thread_id: u32) { + self.min = self.min.min(thread_id); + self.max = self.max.max(thread_id); + } +} + +struct ThreadSet { + base: u32, + span: u32, + words: Box<[u64]>, + remaining: u64, +} + +impl ThreadSet { + fn new(base: u32, threads: impl DoubleEndedIterator + Clone) -> Self { + let words = threads + .clone() + .next_back() + .map_or(0, |last| (last - base) as usize / 64) + + 1; + let mut set = ThreadSet { + base, + span: (words * 64).try_into().unwrap_or(u32::MAX), + words: vec![0u64; words].into_boxed_slice(), + remaining: 0, + }; + for thread_id in threads { + set.insert(thread_id); + } + set + } + + #[inline(always)] + fn insert(&mut self, thread_id: u32) { + let offset = thread_id.wrapping_sub(self.base); + if offset < self.span { + let word = &mut self.words[(offset / 64) as usize]; + let bit = 1u64 << (offset % 64); + if *word & bit == 0 { + *word |= bit; + self.remaining += 1; + } + } + } + + #[inline(always)] + fn overlaps(&self, range: ThreadRange) -> bool { + range.max >= self.base && u64::from(range.min) < u64::from(self.base) + u64::from(self.span) + } + + #[inline(always)] + fn contains(&self, thread_id: u32) -> bool { + let offset = thread_id.wrapping_sub(self.base); + offset < self.span && self.words[(offset / 64) as usize] & (1u64 << (offset % 64)) != 0 + } + + #[inline(always)] + fn remove(&mut self, thread_id: u32) { + let offset = thread_id.wrapping_sub(self.base); + if offset < self.span { + let word = &mut self.words[(offset / 64) as usize]; + let bit = 1u64 << (offset % 64); + if *word & bit != 0 { + *word &= !bit; + self.remaining -= 1; + } + } + } + + fn iter(&self) -> impl Iterator + '_ { + let base = self.base; + self.words + .iter() + .enumerate() + .flat_map(move |(index, word)| { + let mut word = *word; + let offset = base + (index * 64) as u32; + std::iter::from_fn(move || { + (word != 0).then(|| { + let bit = word.trailing_zeros(); + word &= word - 1; + offset + bit + }) + }) + }) + } +} + +impl MessagesCache { + pub fn new(change_id: u64, items: Vec, keywords: Vec) -> Self { + debug_assert!( + items.is_sorted_by_key(|item| item.sort_rank()), + "items must be ordered by (received_at, document_id)" + ); + debug_assert!( + keywords.is_sorted_by_key(|entry| entry.document_id), + "custom keywords must be ordered by document_id" + ); + + Self::from_records(change_id, &items, keywords.into()) + } + + fn from_records( + change_id: u64, + items: &[MessageCache], + keywords: Arc<[CustomKeywords]>, + ) -> Self { + let blocks = items + .chunks(CACHE_CHUNK) + .map(|chunk| Arc::new(ColBlock::from_records(chunk))) + .collect::>(); + let (starts, len) = Self::starts_of(&blocks); + + Self::assemble(change_id, blocks, starts, len, keywords) + } + + fn starts_of(blocks: &[Arc]) -> (Vec, usize) { + let mut starts = Vec::with_capacity(blocks.len()); + let mut start = 0u32; + for block in blocks { + starts.push(start); + start += block.len() as u32; + } + (starts, start as usize) + } + + fn assemble( + change_id: u64, + blocks: Vec>, + starts: Vec, + len: usize, + keywords: Arc<[CustomKeywords]>, + ) -> Self { + let mut index = Vec::with_capacity(len); + for (block, start) in blocks.iter().zip(starts.iter()) { + index.extend( + block + .document_ids + .iter() + .zip(*start..) + .map(|(document_id, position)| (*document_id, position)), + ); + } + index.sort_unstable_by_key(|(document_id, _)| *document_id); + + Self::from_parts( + change_id, + blocks, + starts, + IdIndex::from_sorted(&index), + keywords, + len, + ) + } + + pub(crate) fn from_parts( + change_id: u64, + blocks: Vec>, + starts: Vec, + index: IdIndex, + keywords: Arc<[CustomKeywords]>, + len: usize, + ) -> Self { + let size = keywords_weight(&keywords) + + blocks.iter().map(|block| block.weight()).sum::() + + index.weight(); + + MessagesCache { + change_id, + blocks, + starts, + index, + keywords, + len, + size, + } + } + + fn successor( + &self, + change_id: u64, + list: BlockList, + index: IdIndex, + keywords: Arc<[CustomKeywords]>, + ) -> Self { + let (retired_keywords, kept_keywords) = if Arc::ptr_eq(&keywords, &self.keywords) { + (0, 0) + } else { + (keywords_weight(&self.keywords), keywords_weight(&keywords)) + }; + let size = self.size - list.removed - self.index.weight() - retired_keywords + + list.added + + index.weight() + + kept_keywords; + + MessagesCache { + change_id, + blocks: list.blocks, + starts: list.starts, + index, + keywords, + len: list.len as usize, + size, + } + } + + fn block_rows(&self) -> impl Iterator + '_ { + self.starts + .iter() + .copied() + .zip(self.starts.iter().copied().skip(1).chain([self.len as u32])) + .map(|(start, end)| end - start) + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + #[inline(always)] + pub fn keywords(&self) -> &[CustomKeywords] { + &self.keywords + } + + #[inline(always)] + pub fn shared_keywords(&self) -> Arc<[CustomKeywords]> { + self.keywords.clone() + } + + #[inline(always)] + fn locate(&self, position: u32) -> Option<(u32, u32)> { + if (position as usize) >= self.len { + return None; + } + let block = match self.starts.binary_search(&position) { + Ok(block) => block, + Err(next) => next - 1, + }; + Some((block as u32, position - self.starts[block])) + } + + #[inline(always)] + pub fn position(&self, document_id: u32) -> Option { + self.index.position(document_id) + } + + #[inline(always)] + pub fn contains(&self, document_id: u32) -> bool { + self.index.position(document_id).is_some() + } + + #[inline(always)] + pub fn document_ids(&self) -> impl Iterator + '_ { + self.index.document_ids() + } + + pub fn iter(&self) -> impl DoubleEndedIterator> + '_ { + self.blocks + .iter() + .zip(self.starts.iter().copied()) + .flat_map(|(block, start)| { + let block: &ColBlock = block; + (0..block.len() as u32).map(move |slot| MessageRef { + block, + position: start + slot, + slot, + }) + }) + } + + #[inline(always)] + pub fn at(&self, position: u32) -> Option> { + let (block, slot) = self.locate(position)?; + Some(MessageRef { + block: self.blocks.get(block as usize)?, + position, + slot, + }) + } + + #[inline(always)] + pub fn by_id(&self, document_id: u32) -> Option> { + self.position(document_id).and_then(|p| self.at(p)) + } + + pub fn emptied_threads( + &self, + thread_ids: &RoaringBitmap, + deleted_ids: &RoaringBitmap, + ) -> Vec { + let mut emptied = Vec::new(); + let mut next = thread_ids.min(); + while let Some(base) = next { + let window = base..=base.saturating_add(THREAD_WINDOW - 1); + let mut pending = ThreadSet::new(base, thread_ids.range(window.clone())); + self.remove_surviving_threads(&mut pending, deleted_ids); + emptied.extend(pending.iter()); + next = window + .end() + .checked_add(1) + .and_then(|from| thread_ids.range(from..).next()); + } + emptied + } + + fn remove_surviving_threads(&self, pending: &mut ThreadSet, deleted_ids: &RoaringBitmap) { + for block in &self.blocks { + if !pending.overlaps(block.thread_range) { + continue; + } + for (thread_id, document_id) in block.thread_ids.iter().zip(block.document_ids.iter()) { + if pending.contains(*thread_id) && !deleted_ids.contains(*document_id) { + pending.remove(*thread_id); + if pending.remaining == 0 { + return; + } + } + } + } + } + + pub fn patch( + &self, + change_id: u64, + updates: &[(u32, MessageCache)], + keywords: Arc<[CustomKeywords]>, + ) -> Self { + let mut touched = updates + .iter() + .filter_map(|(position, record)| { + let (block, slot) = self.locate(*position)?; + debug_assert_eq!( + self.blocks[block as usize].document_ids[slot as usize], record.document_id, + "a patch must not move a message to a different position" + ); + Some((block, slot, record)) + }) + .collect::>(); + touched.sort_unstable_by_key(|(block, slot, _)| (*block, *slot)); + + let mut list = BlockList::of(self); + for slots in touched.chunk_by(|(left, _, _), (right, _, _)| left == right) { + list.patch(slots); + } + + self.successor(change_id, list, self.index.clone(), keywords) + } + + fn locate_insert(&self, rank: (u64, u32)) -> (u32, u32) { + let block = self + .blocks + .partition_point(|block| block.rank(block.len() - 1) < rank); + match self.blocks.get(block) { + Some(col) => { + let (mut low, mut high) = (0usize, col.len()); + while low < high { + let mid = low + (high - low) / 2; + if col.rank(mid) < rank { + low = mid + 1; + } else { + high = mid; + } + } + (block as u32, low as u32) + } + None => { + let last = self.blocks.len() - 1; + (last as u32, self.blocks[last].len() as u32) + } + } + } + + pub fn splice( + &self, + change_id: u64, + deletes: &[u32], + updates: &[(u32, MessageCache)], + inserts: &[MessageCache], + keywords: Arc<[CustomKeywords]>, + ) -> Self { + debug_assert!( + inserts.is_sorted_by_key(|item| item.sort_rank()), + "inserts must be ordered by (received_at, document_id)" + ); + debug_assert!( + deletes.is_sorted(), + "deletes must be ordered by document id" + ); + + if self.blocks.is_empty() { + return Self::from_records(change_id, inserts, keywords); + } + + let mut deletions = deletes + .iter() + .filter_map(|document_id| { + self.position(*document_id) + .and_then(|position| self.locate(position)) + }) + .collect::>(); + deletions.sort_unstable(); + + let mut edits = updates + .iter() + .filter_map(|(position, record)| { + let (block, slot) = self.locate(*position)?; + debug_assert_eq!( + self.blocks[block as usize].rank(slot as usize), + (record.received_at_secs() as u64, record.document_id), + "an update must not move a message to a different position" + ); + Some((block, slot, record)) + }) + .collect::>(); + edits.sort_unstable_by_key(|(block, slot, _)| (*block, *slot)); + + if deletions.is_empty() && self.appends_at_end(inserts) { + return self.append(change_id, &edits, inserts, keywords); + } + + let insertions = inserts + .iter() + .map(|record| { + let (block, slot) = + self.locate_insert((record.received_at_secs() as u64, record.document_id)); + (block, slot, record) + }) + .collect::>(); + + let extra_arena = inserts + .iter() + .map(|record| record.mailboxes.len()) + .sum::(); + let mut list = BlockList::with_capacity(self.blocks.len() + 1); + let mut cursor = SpliceCursor { + deletes: deletions.iter().peekable(), + edits: edits.iter().peekable(), + inserts: insertions.iter().peekable(), + inserted: Vec::with_capacity(inserts.len()), + }; + + for ((block_idx, old), rows) in (0u32..).zip(self.blocks.iter()).zip(self.block_rows()) { + if cursor.next_slot(block_idx).is_none() { + list.share(old, rows); + continue; + } + + list.retire(old); + let mut builder = ColBuilder::with_capacity( + old.len() + inserts.len(), + old.mb_arena.len() + extra_arena, + ); + cursor.splice_into(&mut builder, old, block_idx, list.len); + builder.finish_into(&mut list); + } + + let len = list.len as usize; + + let shift = PositionShift::build( + insertions + .iter() + .map(|(block, slot, _)| self.starts[*block as usize] + slot), + deletions + .iter() + .map(|(block, slot)| self.starts[*block as usize] + slot), + self.len, + ); + let mut inserted = cursor.inserted; + inserted.sort_unstable_by_key(|(document_id, _)| *document_id); + + let index = self.index.rebuild(&shift, &inserted, deletes, len); + debug_assert_eq!(index.len(), len, "the index must cover every message"); + + self.successor(change_id, list, index, keywords) + } + + fn appends_at_end(&self, inserts: &[MessageCache]) -> bool { + let Some(first) = inserts.first() else { + return true; + }; + let (Some(last_id), Some(last)) = (self.index.last_id(), self.blocks.last()) else { + return false; + }; + last.rank(last.len() - 1) < (first.received_at_secs() as u64, first.document_id) + && inserts.iter().all(|record| record.document_id > last_id) + } + + fn append_from(&self, inserts: usize) -> usize { + if inserts == 0 { + return self.blocks.len(); + } + let (tail_blocks, tail_rows) = self + .blocks + .iter() + .rev() + .take_while(|block| block.len() < CACHE_CHUNK) + .fold((0usize, 0usize), |(count, rows), block| { + (count + 1, rows + block.len()) + }); + if tail_blocks > 1 && tail_rows + inserts >= CACHE_CHUNK { + self.blocks.len() - tail_blocks + } else if self + .blocks + .last() + .is_some_and(|block| block.len() < APPEND_BLOCK) + { + self.blocks.len() - 1 + } else { + self.blocks.len() + } + } + + fn append( + &self, + change_id: u64, + edits: &[(u32, u32, &MessageCache)], + inserts: &[MessageCache], + keywords: Arc<[CustomKeywords]>, + ) -> Self { + let merge_from = self.append_from(inserts.len()); + let mut cursor = SpliceCursor { + deletes: [].iter().peekable(), + edits: edits.iter().peekable(), + inserts: [].iter().peekable(), + inserted: Vec::new(), + }; + let mut list = + BlockList::with_capacity(self.blocks.len() + 1 + inserts.len() / CACHE_CHUNK); + + for ((block_idx, old), rows) in (0u32..) + .zip(self.blocks.iter().take(merge_from)) + .zip(self.block_rows()) + { + if cursor.next_slot(block_idx).is_none() { + list.share(old, rows); + continue; + } + list.retire(old); + let mut builder = ColBuilder::with_capacity(old.len(), old.mb_arena.len()); + cursor.splice_into(&mut builder, old, block_idx, 0); + builder.finish_into(&mut list); + } + + if !inserts.is_empty() { + let tail = self.blocks.iter().skip(merge_from); + let mut builder = ColBuilder::with_capacity( + tail.clone().map(|block| block.len()).sum::() + inserts.len(), + tail.clone() + .map(|block| block.mb_arena.len()) + .chain(inserts.iter().map(|record| record.mailboxes.len())) + .sum::(), + ); + for (block_idx, old) in (merge_from as u32..).zip(tail) { + list.retire(old); + cursor.splice_into(&mut builder, old, block_idx, 0); + } + for record in inserts { + builder.push_record(record); + } + builder.finish_into(&mut list); + } + + let mut entries = inserts + .iter() + .zip(self.len as u32..) + .map(|(record, position)| (record.document_id, position)) + .collect::>(); + entries.sort_unstable_by_key(|(document_id, _)| *document_id); + + let index = self.index.append(&entries); + self.successor(change_id, list, index, keywords) + } + + #[inline(always)] + pub fn custom_keywords_of(&self, document_id: u32) -> &[CompactString] { + self.keywords + .binary_search_by_key(&document_id, |entry| entry.document_id) + .map_or(&[][..], |idx| &self.keywords[idx].names) + } +} + +fn keywords_weight(keywords: &[CustomKeywords]) -> u64 { + keywords + .iter() + .map(|entry| { + std::mem::size_of::() + + (entry.names.len() * std::mem::size_of::()) + }) + .sum::() as u64 +} + +#[derive(Clone, Copy)] +pub struct MessageRef<'x> { + block: &'x ColBlock, + position: u32, + slot: u32, +} + +impl<'x> MessageRef<'x> { + #[inline(always)] + fn col(&self) -> &'x ColBlock { + self.block + } + + #[inline(always)] + pub fn position(&self) -> u32 { + self.position + } + + #[inline(always)] + pub fn document_id(&self) -> u32 { + self.col().document_ids[self.slot as usize] + } + + #[inline(always)] + pub fn mailboxes(&self) -> &'x [MessageUid] { + self.col().mailboxes(self.slot) + } + + #[inline(always)] + pub fn keywords(&self) -> u32 { + self.col().keywords[self.slot as usize] + } + + #[inline(always)] + pub fn thread_id(&self) -> u32 { + self.col().thread_ids[self.slot as usize] + } + + #[inline(always)] + pub fn change_id(&self) -> u64 { + self.col().change_ids[self.slot as usize] + } + + #[inline(always)] + pub fn size(&self) -> u32 { + self.col().sizes[self.slot as usize] + } + + #[inline(always)] + pub fn received_at(&self) -> u64 { + self.col().received_at[self.slot as usize] as u64 + } + + #[inline(always)] + pub fn sent_at(&self) -> i32 { + self.col().sent_at[self.slot as usize] + } + + #[inline(always)] + pub fn sent_at_absolute(&self) -> i64 { + self.received_at() as i64 + self.sent_at() as i64 + } + + #[inline(always)] + pub fn sort_rank(&self) -> (u64, u32) { + (self.received_at(), self.document_id()) + } + + #[inline(always)] + pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { + self.mailboxes() + .iter() + .any(|uid| uid.mailbox_id == mailbox_id) + } + + #[inline(always)] + pub fn uid_in(&self, mailbox_id: u32) -> Option { + self.mailboxes() + .iter() + .find(|uid| uid.mailbox_id == mailbox_id) + .map(|uid| uid.uid) + } + + pub fn to_record(&self) -> MessageCache { + MessageCache::new( + self.document_id(), + self.mailboxes().iter().copied().collect(), + self.keywords(), + self.thread_id(), + self.change_id(), + self.size(), + self.received_at(), + self.sent_at(), + ) + } +} + diff --git a/crates/common/src/cache/http_auth.rs b/crates/common/src/cache/http_auth.rs new file mode 100644 index 00000000..35df78bf --- /dev/null +++ b/crates/common/src/cache/http_auth.rs @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{HttpAuthCache, auth::TokenScope}; +use quick_cache::sync::DefaultLifecycle; +use siphasher::sip128::{Hasher128, SipHasher13}; +use std::{ + hash::{BuildHasher, Hash, Hasher}, + sync::atomic::{AtomicU64, Ordering}, +}; +use store::rand; +use utils::cache::{Cache, CacheItemWeight, TtlClock}; + +pub struct HttpAuthEntries { + entries: Cache< + HttpAuthKey, + HttpAuthCache, + DefaultLifecycle, + HttpAuthKeyHasher, + >, + keys: (u64, u64), + clock: TtlClock, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HttpAuthKey([u64; 2]); + +#[derive(Debug, Clone, Copy, Default)] +pub struct HttpAuthKeyHasher(u64); + +#[derive(Debug, Default)] +pub struct GrantRevocations(AtomicU64); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RevocationEpoch(u64); + +impl HttpAuthEntries { + pub fn new(weight_capacity: u64) -> Self { + HttpAuthEntries { + entries: Cache::new_with_hasher( + weight_capacity, + (size_of::() + size_of::()) as u64, + ) + .with_name("httpAuth"), + keys: (rand::random(), rand::random()), + clock: TtlClock::new(), + } + } + + fn key(&self, token: &str) -> HttpAuthKey { + let mut hasher = SipHasher13::new_with_keys(self.keys.0, self.keys.1); + hasher.write(token.as_bytes()); + let hash = hasher.finish128(); + HttpAuthKey([hash.h1, hash.h2]) + } + + #[inline] + pub fn get(&self, token: &str) -> Option { + self.entries.get(&self.key(token)) + } + + #[inline] + pub fn remove(&self, token: &str) { + self.entries.remove(&self.key(token)); + } + + #[inline] + pub fn is_live(&self, entry: &HttpAuthCache) -> bool { + self.clock.is_live(entry.expires) + } + + #[inline] + pub fn expires_in(&self, ttl_secs: u64) -> u32 { + self.clock + .now() + .saturating_add(u32::try_from(ttl_secs).unwrap_or(u32::MAX)) + } + + pub fn retain(&self, f: impl Fn(&HttpAuthCache) -> bool) { + self.entries.inner().retain(|_, entry| f(entry)); + } + + pub fn weight(&self) -> u64 { + self.entries.weight() + } + + fn insert(&self, key: HttpAuthKey, entry: HttpAuthCache) { + self.entries.insert(key, entry); + } + + fn clear(&self) { + self.entries.clear(); + } +} + +impl GrantRevocations { + #[inline] + pub fn epoch(&self) -> RevocationEpoch { + RevocationEpoch(self.0.load(Ordering::Acquire)) + } + + pub fn insert( + &self, + entries: &HttpAuthEntries, + token: &str, + entry: HttpAuthCache, + epoch: RevocationEpoch, + ) { + if self.epoch() == epoch { + let key = entries.key(token); + entries.insert(key, entry); + if self.epoch() != epoch { + entries.entries.remove(&key); + } + } + } + + pub fn evict(&self, entries: &HttpAuthEntries, grant_ids: &[u64]) { + self.advance(); + entries.retain(|entry| { + entry + .token_scope + .and_then(TokenScope::grant_id) + .is_none_or(|grant_id| !grant_ids.contains(&grant_id)) + }); + } + + pub fn clear(&self, entries: &HttpAuthEntries) { + self.advance(); + entries.clear(); + } + + #[inline] + fn advance(&self) { + self.0.fetch_add(1, Ordering::AcqRel); + } +} + +impl Hash for HttpAuthKey { + #[inline(always)] + fn hash(&self, state: &mut H) { + state.write_u64(self.0[0]); + } +} + +impl Hasher for HttpAuthKeyHasher { + #[inline(always)] + fn finish(&self) -> u64 { + self.0 + } + + #[inline(always)] + fn write(&mut self, bytes: &[u8]) { + for &byte in bytes { + self.0 = self.0.rotate_left(8) ^ u64::from(byte); + } + } + + #[inline(always)] + fn write_u64(&mut self, value: u64) { + self.0 = value; + } +} + +impl BuildHasher for HttpAuthKeyHasher { + type Hasher = HttpAuthKeyHasher; + + #[inline(always)] + fn build_hasher(&self) -> Self::Hasher { + HttpAuthKeyHasher(0) + } +} + +impl CacheItemWeight for HttpAuthKey { + fn weight(&self) -> u64 { + size_of::() as u64 + } +} + diff --git a/crates/common/src/cache/invalidate.rs b/crates/common/src/cache/invalidate.rs index f0dcd12e..0b0e7c1e 100644 --- a/crates/common/src/cache/invalidate.rs +++ b/crates/common/src/cache/invalidate.rs @@ -5,45 +5,107 @@ */ use crate::{ - Server, + Inner, Server, auth::{EmailAddressRef, EmailCache}, - ipc::{BroadcastEvent, CacheInvalidation}, + ipc::{BroadcastEvent, CacheInvalidation, PushNotification}, }; use ahash::AHashSet; use registry::{ schema::{ - prelude::{Object, ObjectInner, ObjectType}, - structs::{Account, EmailAlias}, + prelude::{Object, ObjectInner, ObjectType, Property}, + structs::{Account, Credential, EmailAlias}, }, - types::id::ObjectId, + types::{id::ObjectId, list::List}, }; -use store::{registry::RegistryQuery, roaring::RoaringBitmap}; -use types::id::Id; +use store::{registry::RegistryQuery, roaring::RoaringBitmap, write::AssignedIds}; +use types::{ + collection::{Collection, SyncCollection}, + id::Id, + type_state::DataType, +}; +use utils::map::bitmap::Bitmap; + +const _: () = assert!(SyncCollection::MAX_CHANGE_GROUP < u32::BITS as u8); + +impl Inner { + pub fn mark_cache_stale(&self, account_id: u32, collection: SyncCollection) { + let caches = &self.cache; + match collection { + SyncCollection::Email | SyncCollection::Thread => { + if let Some(cache) = caches.peek_messages(account_id) { + cache.update_lock.mark_stale(); + } + } + _ => { + if let Some(cache) = caches.peek_resources(account_id, collection) { + cache.update_lock.mark_stale(); + } + } + } + } + + pub fn mark_caches_stale_from(&self, assigned_ids: &AssignedIds) { + for counter in assigned_ids.change_counters() { + self.mark_cache_stale(counter.account_id, counter.group.to_u8().into()); + } + } + + pub fn mark_caches_stale(&self, account_id: u32, types: Bitmap) { + let mut marked_groups = 0u32; + for data_type in types { + if let Ok(collection) = Collection::try_from(data_type) { + let group = collection.change_group().to_u8(); + let group_bit = 1u32 << group; + if marked_groups & group_bit == 0 { + marked_groups |= group_bit; + self.mark_cache_stale(account_id, group.into()); + } + } + } + } + + pub fn mark_caches_stale_for(&self, notification: &PushNotification) { + match notification { + PushNotification::StateChange(state_change) => { + self.mark_caches_stale(state_change.account_id, state_change.types); + } + PushNotification::EmailPush(email_push) => { + self.mark_cache_stale(email_push.account_id, SyncCollection::Email); + } + PushNotification::ViewerStateChange(_) | PushNotification::CalendarAlert(_) => {} + } + } +} #[derive(Debug, Default)] pub struct CacheInvalidationBuilder { changes: AHashSet, + revoke_oauth_grants: Vec, } impl CacheInvalidationBuilder { pub fn process_update(&mut self, id: Id, current_object: &Object, new_object: &Object) { let id = id.document_id(); match (¤t_object.inner, &new_object.inner) { - ( - ObjectInner::Account(Account::User(current)), - ObjectInner::Account(Account::User(new)), - ) => { + (ObjectInner::Account(current), ObjectInner::Account(new)) + if let (Account::User(current), Account::User(new)) = + (current.as_ref(), new.as_ref()) => + { let was_renamed = (current.name != new.name) || (current.domain_id != new.domain_id); let quota_changed = current.quotas != new.quotas; let permissions_changed = current.permissions != new.permissions; let roles_changed = current.roles != new.roles; - let tenant_changed = current.member_tenant_id != new.member_tenant_id; + #[cfg(not(feature = "enterprise"))] + let tenant_changed = false; let details_changed = current.locale != new.locale || current.description != new.description; let groups_changed = current.member_group_ids != new.member_group_ids; let aliases_changed = current.aliases != new.aliases; let credentials_changed = current.credentials != new.credentials; + if credentials_changed { + self.process_credentials_update(id, ¤t.credentials, &new.credentials); + } let encryption_changed = current.encryption_at_rest != new.encryption_at_rest; if was_renamed @@ -71,20 +133,21 @@ impl CacheInvalidationBuilder { } if was_renamed { - self.invalidate(CacheInvalidation::DavResources(id)); + self.invalidate(CacheInvalidation::GroupwareResources(id)); } } - ( - ObjectInner::Account(Account::Group(current)), - ObjectInner::Account(Account::Group(new)), - ) => { + (ObjectInner::Account(current), ObjectInner::Account(new)) + if let (Account::Group(current), Account::Group(new)) = + (current.as_ref(), new.as_ref()) => + { let was_renamed = (current.name != new.name) || (current.domain_id != new.domain_id); let quota_changed = current.quotas != new.quotas; let permissions_changed = current.permissions != new.permissions; let roles_changed = current.roles != new.roles; - let tenant_changed = current.member_tenant_id != new.member_tenant_id; + #[cfg(not(feature = "enterprise"))] + let tenant_changed = false; let details_changed = current.locale != new.locale || current.description != new.description; let aliases_changed = current.aliases != new.aliases; @@ -107,19 +170,21 @@ impl CacheInvalidationBuilder { } if was_renamed { - self.invalidate(CacheInvalidation::DavResources(id)); + self.invalidate(CacheInvalidation::GroupwareResources(id)); } } (ObjectInner::Domain(current), ObjectInner::Domain(new)) => { + #[cfg(not(feature = "enterprise"))] + let enterprise_changed = false; + if (current.name != new.name) || (current.aliases != new.aliases) - || (current.directory_id != new.directory_id) - || (current.member_tenant_id != new.member_tenant_id) + || enterprise_changed || (current.catch_all_address != new.catch_all_address) || (current.sub_addressing != new.sub_addressing) || (current.allow_relaying != new.allow_relaying) - || (current.is_enabled != new.is_enabled) + || (current.enable != new.enable) { self.invalidate(CacheInvalidation::Domain(id)); } @@ -128,9 +193,6 @@ impl CacheInvalidationBuilder { self.invalidate(CacheInvalidation::DomainNegative); } - if current.logo != new.logo { - self.invalidate(CacheInvalidation::DomainLogo(id)); - } } (ObjectInner::DkimSignature(current), ObjectInner::DkimSignature(new)) => { @@ -142,28 +204,13 @@ impl CacheInvalidationBuilder { } } - (ObjectInner::Tenant(current), ObjectInner::Tenant(new)) => { - if (current.permissions != new.permissions) - || (current.roles != new.roles) - || (current.quotas != new.quotas) - { - self.invalidate(CacheInvalidation::Tenant(id)); - } - - if current.logo != new.logo { - self.invalidate(CacheInvalidation::TenantLogo(id)); - } - } - (ObjectInner::Role(current), ObjectInner::Role(new)) if (current.enabled_permissions != new.enabled_permissions) || (current.disabled_permissions != new.disabled_permissions) - || (current.member_tenant_id != new.member_tenant_id) || (current.role_ids != new.role_ids) => { self.invalidate(CacheInvalidation::Role(id)); } - (ObjectInner::MailingList(current), ObjectInner::MailingList(new)) if (current.aliases != new.aliases) || (current.name != new.name) @@ -183,12 +230,20 @@ impl CacheInvalidationBuilder { } pub fn process_delete(&mut self, id: Id, object: &Object) { + let grant_id = id.id(); let id = id.document_id(); match &object.inner { + ObjectInner::OAuthGrant(grant) => { + self.invalidate(CacheInvalidation::OAuthGrant { + account_id: grant.account_id.document_id(), + grant_id, + }); + } ObjectInner::Account(_) => { self.invalidate(CacheInvalidation::AccessToken(id)); self.invalidate(CacheInvalidation::Account(id)); - self.invalidate(CacheInvalidation::DavResources(id)); + self.invalidate(CacheInvalidation::GroupwareResources(id)); + self.invalidate(CacheInvalidation::MessageCache(id)); } ObjectInner::Domain(_) => { self.invalidate(CacheInvalidation::Domain(id)); @@ -199,10 +254,6 @@ impl CacheInvalidationBuilder { object.domain_id().document_id(), )); } - ObjectInner::Tenant(_) => { - self.invalidate(CacheInvalidation::Tenant(id)); - self.invalidate(CacheInvalidation::TenantLogo(id)); - } ObjectInner::Role(_) => { self.invalidate(CacheInvalidation::Role(id)); } @@ -214,20 +265,26 @@ impl CacheInvalidationBuilder { } pub fn process_create(&mut self, object: &Object) { - if matches!(&object.inner, ObjectInner::Domain(_)) { - self.invalidate(CacheInvalidation::DomainNegative); + match &object.inner { + ObjectInner::Domain(_) => { + self.invalidate(CacheInvalidation::DomainNegative); + } + ObjectInner::DkimSignature(object) => { + self.invalidate(CacheInvalidation::DkimSignature( + object.domain_id().document_id(), + )); + } + _ => {} } self.invalidate_negative_email(&object.inner); } fn invalidate_negative_email(&mut self, object: &ObjectInner) { let (name, domain_id, aliases) = match object { - ObjectInner::Account(Account::User(account)) => { - (&account.name, account.domain_id, &account.aliases) - } - ObjectInner::Account(Account::Group(account)) => { - (&account.name, account.domain_id, &account.aliases) - } + ObjectInner::Account(account) => match account.as_ref() { + Account::User(account) => (&account.name, account.domain_id, &account.aliases), + Account::Group(account) => (&account.name, account.domain_id, &account.aliases), + }, ObjectInner::MailingList(list) => (&list.name, list.domain_id, &list.aliases), _ => return, }; @@ -236,7 +293,7 @@ impl CacheInvalidationBuilder { domain_id: domain_id.document_id(), local_part_hash: hash_local_part(name), }); - for alias in aliases.iter().filter(|alias: &&EmailAlias| alias.enabled) { + for alias in aliases.iter().filter(|alias: &&EmailAlias| alias.enable) { self.invalidate(CacheInvalidation::EmailNegative { domain_id: alias.domain_id.document_id(), local_part_hash: hash_local_part(&alias.name), @@ -248,6 +305,19 @@ impl CacheInvalidationBuilder { self.changes.insert(change); } + pub fn process_credentials_update( + &mut self, + account_id: u32, + current: &List, + new: &List, + ) { + if password_secret(current) != password_secret(new) + && !self.revoke_oauth_grants.contains(&account_id) + { + self.revoke_oauth_grants.push(account_id); + } + } + pub fn with_invalidation(mut self, change: CacheInvalidation) -> Self { self.invalidate(change); self @@ -255,7 +325,51 @@ impl CacheInvalidationBuilder { } impl Server { - pub async fn invalidate_caches(&self, changes: CacheInvalidationBuilder) -> trc::Result<()> { + pub async fn invalidate_grantee( + &self, + changes: &mut CacheInvalidationBuilder, + grantee_id: u32, + ) { + changes.invalidate(CacheInvalidation::AccessToken(grantee_id)); + let result = match self.try_account(grantee_id).await { + Ok(Some(grantee)) if !grantee.is_user_account() => self + .registry() + .query::( + RegistryQuery::new(ObjectType::Account) + .equal(Property::MemberGroupIds, u64::from(grantee_id)), + ) + .await + .map(|members| { + for member_id in members { + changes.invalidate(CacheInvalidation::AccessToken(member_id)); + } + }), + Ok(_) => Ok(()), + Err(err) => Err(err), + }; + if let Err(err) = result { + trc::error!( + err.account_id(grantee_id) + .details("Failed to invalidate group member access tokens") + .caused_by(trc::location!()) + ); + } + } + + pub async fn invalidate_caches( + &self, + mut changes: CacheInvalidationBuilder, + ) -> trc::Result<()> { + for account_id in std::mem::take(&mut changes.revoke_oauth_grants) { + if let Err(err) = self.revoke_oauth_grants(account_id, &mut changes).await { + trc::error!( + err.account_id(account_id) + .details("Failed to revoke OAuth grants after a password change") + .caused_by(trc::location!()) + ); + } + } + let mut changes = changes.changes; if changes.is_empty() { return Ok(()); @@ -299,7 +413,7 @@ impl Server { } let changes = changes.into_iter().collect::>(); - self.invalidate_local_caches(&changes).await; + self.invalidate_local_caches(&changes); self.cluster_broadcast(BroadcastEvent::CacheInvalidate(changes)) .await; Ok(()) @@ -308,24 +422,27 @@ impl Server { pub fn invalidate_all_local_caches(&self) { self.invalidate_all_local_negative_caches(); self.inner.cache.access_tokens.clear(); + self.inner + .cache + .grant_revocations + .clear(&self.inner.cache.http_auth); self.inner.cache.domains.clear(); self.inner.cache.domain_names.clear(); self.inner.cache.emails.clear(); - self.inner.cache.tenants.clear(); - self.inner.cache.files.clear(); - self.inner.cache.contacts.clear(); - self.inner.cache.events.clear(); - self.inner.cache.scheduling.clear(); + self.inner.cache.clear_resources(); + self.inner.cache.metadata_viewers.clear(); self.inner.cache.dkim_signers.clear(); self.inner.cache.accounts.clear(); self.inner.cache.roles.clear(); self.inner.cache.lists.clear(); + self.inner.cache.directory_recipients.clear(); self.inner.data.logos.lock().clear(); } pub fn invalidate_all_local_negative_caches(&self) { self.inner.cache.domain_names_negative.clear(); self.inner.cache.emails_negative.clear(); + self.inner.cache.directory_recipients.clear(); } pub fn invalidate_local_negative_account_cache( @@ -340,71 +457,55 @@ impl Server { .is_some() } - pub async fn invalidate_local_caches(&self, changes: &[CacheInvalidation]) { + pub fn invalidate_local_caches(&self, changes: &[CacheInvalidation]) { let cache = &self.inner.cache; let mut negative_emails: AHashSet<(u32, u32)> = AHashSet::new(); + let mut http_auth_accounts = IdFilter::default(); + let mut revoked_grants: Vec = Vec::new(); + let mut domain_names = IdFilter::default(); + let mut email_accounts = IdFilter::default(); + let mut email_lists = IdFilter::default(); + let mut domain_logos = IdFilter::default(); for change in changes { match change { CacheInvalidation::AccessToken(id) => { cache.access_tokens.remove(id); - cache.http_auth.inner().retain(|_, v| v.account_id != *id); + http_auth_accounts.insert(*id); } - CacheInvalidation::DavResources(id) => { - cache.files.remove(id); - cache.contacts.remove(id); - cache.events.remove(id); - cache.scheduling.remove(id); + CacheInvalidation::GroupwareResources(id) => { + cache.remove_resources(*id); + let inner = self.inner.clone(); + let account_id = *id; + tokio::spawn(async move { + inner.cache.swap.remove_resources(account_id).await; + }); + } + CacheInvalidation::MessageCache(id) => { + cache.remove_messages(*id); + cache.swap.forget(*id); } CacheInvalidation::Domain(id) => { cache.domains.remove(id); cache.dkim_signers.remove(id); - cache.domain_names.inner().retain(|_, v| v != id); + domain_names.insert(*id); } CacheInvalidation::Account(id) => { cache.accounts.remove(id); - cache.emails.inner().retain(|_, v| { - !matches!( - v, - EmailCache::Account(account_id) - | EmailCache::DisabledAccountAddress(account_id) - if account_id == id - ) - }); + email_accounts.insert(*id); } CacheInvalidation::DkimSignature(id) => { cache.dkim_signers.remove(id); } - CacheInvalidation::Tenant(id) => { - cache.tenants.remove(id); - } CacheInvalidation::Role(id) => { cache.roles.remove(id); } CacheInvalidation::List(id) => { cache.lists.remove(id); - cache.emails.inner().retain(|_, v| { - !matches!( - v, - EmailCache::MailingList(list_id) - | EmailCache::DisabledListAddress(list_id) - if list_id == id - ) - }); + email_lists.insert(*id); } CacheInvalidation::DomainLogo(id) => { - self.inner - .data - .logos - .lock() - .retain(|_, v| v.domain_id != *id); - } - CacheInvalidation::TenantLogo(id) => { - self.inner - .data - .logos - .lock() - .retain(|_, v| v.tenant_id != Some(*id)); + domain_logos.insert(*id); } CacheInvalidation::EmailNegative { domain_id, @@ -415,14 +516,121 @@ impl Server { CacheInvalidation::DomainNegative => { cache.domain_names_negative.clear(); } + CacheInvalidation::PrivateMetadata(id) => { + cache.metadata_viewers.invalidate(*id); + } + CacheInvalidation::OAuthGrant { grant_id, .. } => { + revoked_grants.push(*grant_id); + } + CacheInvalidation::ActiveSieveScript(id) => { + if let Some(account) = cache.accounts.peek(id) { + account.active_script.invalidate(); + } + } } } + for filter in [ + &mut http_auth_accounts, + &mut domain_names, + &mut email_accounts, + &mut email_lists, + &mut domain_logos, + ] { + filter.seal(); + } + + match http_auth_accounts.0.as_slice() { + [] => {} + &[account_id] => { + cache.http_auth.retain(move |v| v.account_id != account_id); + } + _ => { + cache + .http_auth + .retain(|v| !http_auth_accounts.contains(v.account_id)); + } + } + + if !revoked_grants.is_empty() { + cache + .grant_revocations + .evict(&cache.http_auth, &revoked_grants); + } + + if !domain_names.is_empty() { + cache + .domain_names + .inner() + .retain(|_, v| !domain_names.contains(*v)); + } + + if !email_accounts.is_empty() || !email_lists.is_empty() { + cache.emails.inner().retain(|_, v| match v { + EmailCache::Account(id) | EmailCache::DisabledAccountAddress(id) => { + !email_accounts.contains(*id) + } + EmailCache::MailingList(id) | EmailCache::DisabledListAddress(id) => { + !email_lists.contains(*id) + } + }); + } + + #[cfg(not(feature = "enterprise"))] + if !domain_logos.is_empty() { + self.inner + .data + .logos + .lock() + .retain(|_, v| !domain_logos.contains(v.domain_id)); + } + if !negative_emails.is_empty() { cache.emails_negative.retain(|key| { !negative_emails.contains(&(key.domain_id, hash_local_part(&key.local_part))) }); } + + trc::event!(Cache(trc::CacheEvent::Invalidate), Total = changes.len()); + } +} + +#[derive(Default)] +struct IdFilter(Vec); + +impl IdFilter { + const LINEAR_SCAN_MAX: usize = 16; + + fn insert(&mut self, id: u32) { + self.0.push(id); + } + + fn seal(&mut self) { + if self.0.len() > Self::LINEAR_SCAN_MAX { + self.0.sort_unstable(); + self.0.dedup(); + } + } + + fn contains(&self, id: u32) -> bool { + match self.0.as_slice() { + [single] => *single == id, + ids if ids.len() <= Self::LINEAR_SCAN_MAX => ids.contains(&id), + ids => ids.binary_search(&id).is_ok(), + } + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl CacheInvalidation { + pub fn is_covered_by_invalidate_all(&self) -> bool { + !matches!( + self, + CacheInvalidation::MessageCache(_) | CacheInvalidation::GroupwareResources(_) + ) } } @@ -438,3 +646,11 @@ impl From for CacheInvalidationBuilder { builder } } + +fn password_secret(credentials: &List) -> Option<&str> { + credentials.iter().find_map(|credential| match credential { + Credential::Password(password) => Some(password.secret.as_str()), + _ => None, + }) +} + diff --git a/crates/common/src/cache/large.rs b/crates/common/src/cache/large.rs new file mode 100644 index 00000000..d1b859bd --- /dev/null +++ b/crates/common/src/cache/large.rs @@ -0,0 +1,714 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Caches, GroupwareResources, MessageStoreCache, + cache::swap::{evict::SwapEviction, writer::SwapChanges, writer::SwapTarget}, +}; +use ahash::AHashMap; +use parking_lot::Mutex; +use std::{ + hash::Hash, + sync::{ + Arc, Weak, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; +use trc::CacheEvent; +use types::collection::SyncCollection; +use utils::cache::{Cache, CacheGuard, CacheItemWeight}; + +pub type MessageCacheGuard<'x> = CacheGuard<'x, u32, Arc, SwapEviction>; +pub type ResourceCache = Cache, SwapEviction>; +pub type ResourceCacheGuard<'x> = CacheGuard<'x, u32, Arc, SwapEviction>; + +const SWEEPS_PER_TTL: u64 = 16; + +pub type LargeMessageCaches = LargeCaches; +pub type LargeResourceCaches = LargeCaches; + +pub struct LargeCaches { + slots: Mutex>, + occupied: AtomicUsize, + budget: u64, + expiry: Option, +} + +struct IdleExpiry { + epoch: Instant, + ttl: u64, + interval: u64, + next_sweep: AtomicU64, +} + +struct IdleSweep { + now: u64, + ttl: u64, +} + +struct LargeSlots { + accounts: AHashMap>, + pinned_bytes: u64, + tick: u64, + swept_tick: u64, +} + +struct LargeSlot { + pinned: Option>, + shared: Weak, + last_use: u64, + seen_at: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Residency { + Pinned, + Shared, +} + +pub struct Publication { + pub residency: Residency, + pub unpinned: Vec<(K, Arc)>, + _replaced: Option>, +} + +impl Default for LargeSlots { + fn default() -> Self { + LargeSlots { + accounts: AHashMap::new(), + pinned_bytes: 0, + tick: 0, + swept_tick: 0, + } + } +} + +impl LargeSlot { + fn is_live(&self) -> bool { + self.pinned.is_some() || self.shared.strong_count() > 0 + } +} + +impl LargeSlots { + fn sweep(&mut self, idle: Option, unpinned: &mut Vec<(K, Arc)>) { + let LargeSlots { + accounts, + pinned_bytes, + tick, + swept_tick, + } = self; + let Some(IdleSweep { now, ttl }) = idle else { + accounts.retain(|_, slot| slot.is_live()); + return; + }; + + accounts.retain(|account_id, slot| { + if slot.last_use > *swept_tick { + slot.seen_at = now; + } else if now.saturating_sub(slot.seen_at) > ttl + && let Some(cache) = slot.pinned.take() + { + *pinned_bytes = pinned_bytes.saturating_sub(cache.weight()); + unpinned.push((*account_id, cache)); + } + slot.is_live() + }); + *swept_tick = *tick; + } +} + +impl IdleExpiry { + fn new(ttl: Duration) -> Option { + (!ttl.is_zero()).then(|| { + let ttl = ttl.as_secs().max(1); + IdleExpiry { + epoch: Instant::now(), + ttl, + interval: (ttl / SWEEPS_PER_TTL).max(1), + next_sweep: AtomicU64::new(0), + } + }) + } + + fn now(&self) -> u64 { + self.epoch.elapsed().as_secs() + } + + fn is_due(&self, now: u64) -> bool { + let due = self.next_sweep.load(Ordering::Relaxed); + now >= due + && self + .next_sweep + .compare_exchange( + due, + now + self.interval, + Ordering::Relaxed, + Ordering::Relaxed, + ) + .is_ok() + } +} + +impl LargeCaches { + pub fn new(budget: u64, idle_ttl: Duration) -> Self { + LargeCaches { + slots: Mutex::new(LargeSlots::default()), + occupied: AtomicUsize::new(0), + budget, + expiry: IdleExpiry::new(idle_ttl), + } + } + + pub fn is_empty(&self) -> bool { + self.occupied.load(Ordering::Relaxed) == 0 + } + + pub fn lookup(&self, account_id: K) -> Option> { + if self.is_empty() { + return None; + } + + let mut slots = self.slots.lock(); + let LargeSlots { accounts, tick, .. } = &mut *slots; + let slot = accounts.get_mut(&account_id)?; + if let Some(pinned) = &slot.pinned { + *tick += 1; + slot.last_use = *tick; + return Some(pinned.clone()); + } + + let shared = slot.shared.upgrade(); + if shared.is_none() { + accounts.remove(&account_id); + self.occupied.store(accounts.len(), Ordering::Relaxed); + } + shared + } + + pub fn peek(&self, account_id: K) -> Option> { + if self.is_empty() { + return None; + } + + let slots = self.slots.lock(); + let slot = slots.accounts.get(&account_id)?; + slot.pinned.clone().or_else(|| slot.shared.upgrade()) + } + + pub fn contains(&self, account_id: K) -> bool { + !self.is_empty() + && self + .slots + .lock() + .accounts + .get(&account_id) + .is_some_and(LargeSlot::is_live) + } + + pub fn publish(&self, account_id: K, cache: &Arc) -> Publication { + self.publish_at( + account_id, + cache, + self.expiry.as_ref().map_or(0, IdleExpiry::now), + ) + } + + pub fn expire_idle(&self) -> Vec<(K, Arc)> { + match &self.expiry { + Some(expiry) if !self.is_empty() => self.expire_idle_at(expiry.now()), + _ => Vec::new(), + } + } + + fn expire_idle_at(&self, now: u64) -> Vec<(K, Arc)> { + let mut unpinned = Vec::new(); + if let Some(expiry) = &self.expiry + && expiry.is_due(now) + { + let mut slots = self.slots.lock(); + slots.sweep( + Some(IdleSweep { + now, + ttl: expiry.ttl, + }), + &mut unpinned, + ); + self.occupied.store(slots.accounts.len(), Ordering::Relaxed); + } + unpinned + } + + fn publish_at(&self, account_id: K, cache: &Arc, now: u64) -> Publication { + let size = cache.weight(); + let pinnable = self.budget > 0 && size <= self.budget; + let mut unpinned = Vec::new(); + + let mut slots = self.slots.lock(); + let LargeSlots { + accounts, + pinned_bytes, + tick, + .. + } = &mut *slots; + *tick += 1; + let last_use = *tick; + + let replaced = match accounts.get_mut(&account_id) { + Some(slot) => { + slot.shared = Arc::downgrade(cache); + slot.last_use = last_use; + slot.pinned.take() + } + None => { + accounts.insert( + account_id, + LargeSlot { + pinned: None, + shared: Arc::downgrade(cache), + last_use, + seen_at: now, + }, + ); + None + } + }; + if let Some(replaced) = &replaced { + *pinned_bytes = pinned_bytes.saturating_sub(replaced.weight()); + } + slots.sweep( + self.expiry.as_ref().map(|expiry| IdleSweep { + now, + ttl: expiry.ttl, + }), + &mut unpinned, + ); + + let LargeSlots { + accounts, + pinned_bytes, + .. + } = &mut *slots; + let residency = if pinnable { + while *pinned_bytes + size > self.budget { + let Some(victim) = accounts + .iter_mut() + .filter(|(id, slot)| **id != account_id && slot.pinned.is_some()) + .min_by_key(|(_, slot)| slot.last_use) + .and_then(|(id, slot)| slot.pinned.take().map(|cache| (*id, cache))) + else { + break; + }; + *pinned_bytes = pinned_bytes.saturating_sub(victim.1.weight()); + unpinned.push(victim); + } + + if let Some(slot) = accounts.get_mut(&account_id) { + slot.pinned = Some(cache.clone()); + *pinned_bytes += size; + } + Residency::Pinned + } else { + Residency::Shared + }; + + self.occupied.store(accounts.len(), Ordering::Relaxed); + drop(slots); + + Publication { + residency, + unpinned, + _replaced: replaced, + } + } + + pub fn remove(&self, account_id: K) -> Option> { + if self.is_empty() { + return None; + } + + let mut slots = self.slots.lock(); + let LargeSlots { + accounts, + pinned_bytes, + .. + } = &mut *slots; + let pinned = accounts.remove(&account_id)?.pinned; + self.occupied.store(accounts.len(), Ordering::Relaxed); + if let Some(pinned) = &pinned { + *pinned_bytes = pinned_bytes.saturating_sub(pinned.weight()); + } + pinned + } + + pub fn resident_bytes(&self) -> u64 { + if self.is_empty() { + return 0; + } + + let slots = self.slots.lock(); + let shared = slots + .accounts + .values() + .filter(|slot| slot.pinned.is_none()) + .filter_map(|slot| slot.shared.upgrade()) + .collect::>(); + let pinned_bytes = slots.pinned_bytes; + drop(slots); + shared.iter().fold(pinned_bytes, |total, cache| { + total.saturating_add(cache.weight()) + }) + } + + pub fn clear(&self) { + let mut slots = self.slots.lock(); + let cleared = std::mem::take(&mut *slots); + self.occupied.store(0, Ordering::Relaxed); + drop(slots); + drop(cleared); + } +} + +impl Caches { + pub fn messages_weight(&self) -> u64 { + self.messages.weight() + } + + pub fn messages_large_weight(&self) -> u64 { + self.messages_large.resident_bytes() + } + + pub fn other_weight(&self) -> u64 { + let total = self.access_tokens.weight() + + self.http_auth.weight() + + self.files.weight() + + self.contacts.weight() + + self.events.weight() + + self.scheduling.weight() + + self.metadata_viewers.weight() + + self.emails.weight() + + self.emails_negative.weight() + + self.domain_names.weight() + + self.domain_names_negative.weight() + + self.domains.weight() + + self.accounts.weight() + + self.roles.weight() + + self.lists.weight() + + self.dkim_signers.weight() + + self.dns_txt.weight() + + self.dns_mx.weight() + + self.dns_ptr.weight() + + self.dns_ipv4.weight() + + self.dns_ipv6.weight() + + self.dns_tlsa.weight() + + self.dns_mta_sts.weight() + + self.spam_lookup.weight() + + self.directory_recipients.weight() + + self.resources_large.resident_bytes(); + + + total + } + + #[inline] + pub fn resident_messages(&self, account_id: u32) -> Option> { + match self.messages.get(&account_id) { + Some(cache) => Some(cache), + None => self.messages_large.lookup(account_id), + } + } + + pub fn peek_messages(&self, account_id: u32) -> Option> { + match self.messages.peek(&account_id) { + Some(cache) => Some(cache), + None => self.messages_large.peek(account_id), + } + } + + pub fn has_messages(&self, account_id: u32) -> bool { + self.messages.inner().contains_key(&account_id) || self.messages_large.contains(account_id) + } + + pub fn admits_messages(&self, account_id: u32, cache: &Arc) -> bool { + if !self.messages.is_oversized(&account_id, cache) { + return true; + } + + trc::event!( + Cache(CacheEvent::EntryTooLarge), + AccountId = account_id, + Collection = SyncCollection::Email.as_str(), + Size = cache.size, + Limit = self.messages.admission_limit(), + ); + false + } + + pub fn publish_messages( + &self, + account_id: u32, + cache: &Arc, + changes: Option, + ) { + if self.admits_messages(account_id, cache) { + self.messages.update(account_id, cache.clone()); + if !self.messages_large.is_empty() { + drop(self.messages_large.remove(account_id)); + self.expire_large_messages(); + } + if let Some(changes) = changes { + self.swap + .notify_changed(account_id, SyncCollection::Email, changes); + } + } else { + self.publish_large_messages(account_id, cache, changes); + } + } + + pub fn publish_loaded_messages( + &self, + guard: MessageCacheGuard<'_>, + account_id: u32, + cache: &Arc, + changes: Option, + ) { + if self.admits_messages(account_id, cache) { + let _ = guard.insert(cache.clone()); + self.expire_large_messages(); + if let Some(changes) = changes { + self.swap + .notify_changed(account_id, SyncCollection::Email, changes); + } + } else { + self.publish_large_messages(account_id, cache, changes); + drop(guard); + } + } + + pub fn remove_messages(&self, account_id: u32) { + let resident = self.messages.remove(&account_id); + let large = self.messages_large.remove(account_id); + drop((resident, large)); + } + + pub fn clear_messages(&self) { + self.messages.clear(); + self.messages_large.clear(); + } + + fn expire_large_messages(&self) { + for (account_id, cache) in self.messages_large.expire_idle() { + self.swap.notify_evicted_messages(account_id, cache); + } + } + + fn publish_large_messages( + &self, + account_id: u32, + cache: &Arc, + changes: Option, + ) { + if self.messages.inner().contains_key(&account_id) { + drop(self.messages.remove(&account_id)); + } + + let publication = self.messages_large.publish(account_id, cache); + for (unpinned_id, unpinned) in publication.unpinned { + self.swap.notify_evicted_messages(unpinned_id, unpinned); + } + + if let Some(changes) = changes { + match publication.residency { + Residency::Pinned => { + self.swap + .notify_changed(account_id, SyncCollection::Email, changes) + } + Residency::Shared => self + .swap + .notify_refresh_messages(account_id, changes, cache), + } + } + } +} + +impl Caches { + pub fn resources(&self, collection: SyncCollection) -> Option<&ResourceCache> { + match collection { + SyncCollection::Calendar => Some(&self.events), + SyncCollection::AddressBook => Some(&self.contacts), + SyncCollection::FileNode => Some(&self.files), + SyncCollection::CalendarEventNotification => Some(&self.scheduling), + _ => None, + } + } + + #[inline] + pub fn resident_resources( + &self, + account_id: u32, + collection: SyncCollection, + ) -> Option> { + match self.resources(collection)?.get(&account_id) { + Some(cache) => Some(cache), + None => self + .resources_large + .lookup(SwapTarget::new(account_id, collection)), + } + } + + pub fn peek_resources( + &self, + account_id: u32, + collection: SyncCollection, + ) -> Option> { + match self.resources(collection)?.peek(&account_id) { + Some(cache) => Some(cache), + None => self + .resources_large + .peek(SwapTarget::new(account_id, collection)), + } + } + + pub fn has_resources(&self, account_id: u32, collection: SyncCollection) -> bool { + self.resources(collection).is_some_and(|resources| { + resources.inner().contains_key(&account_id) + || self + .resources_large + .contains(SwapTarget::new(account_id, collection)) + }) + } + + pub fn publish_resources( + &self, + account_id: u32, + collection: SyncCollection, + cache: &Arc, + changes: Option, + ) { + let Some(resources) = self.resources(collection) else { + return; + }; + if admits_resources(resources, account_id, collection, cache) { + resources.update(account_id, cache.clone()); + if !self.resources_large.is_empty() { + drop( + self.resources_large + .remove(SwapTarget::new(account_id, collection)), + ); + self.expire_large_resources(); + } + if let Some(changes) = changes { + self.swap.notify_changed(account_id, collection, changes); + } + } else { + self.publish_large_resources(resources, account_id, collection, cache, changes); + } + } + + pub fn publish_loaded_resources( + &self, + guard: ResourceCacheGuard<'_>, + account_id: u32, + collection: SyncCollection, + cache: &Arc, + changes: Option, + ) { + let Some(resources) = self.resources(collection) else { + return; + }; + if admits_resources(resources, account_id, collection, cache) { + let _ = guard.insert(cache.clone()); + self.expire_large_resources(); + if let Some(changes) = changes { + self.swap.notify_changed(account_id, collection, changes); + } + } else { + self.publish_large_resources(resources, account_id, collection, cache, changes); + drop(guard); + } + } + + pub fn remove_resources(&self, account_id: u32) { + for collection in [ + SyncCollection::Calendar, + SyncCollection::AddressBook, + SyncCollection::FileNode, + SyncCollection::CalendarEventNotification, + ] { + let resident = self + .resources(collection) + .and_then(|resources| resources.remove(&account_id)); + let large = self + .resources_large + .remove(SwapTarget::new(account_id, collection)); + drop((resident, large)); + } + } + + pub fn clear_resources(&self) { + self.files.clear(); + self.contacts.clear(); + self.events.clear(); + self.scheduling.clear(); + self.resources_large.clear(); + } + + fn expire_large_resources(&self) { + for (target, cache) in self.resources_large.expire_idle() { + self.swap.notify_evicted_resources(target, cache); + } + } + + fn publish_large_resources( + &self, + resources: &ResourceCache, + account_id: u32, + collection: SyncCollection, + cache: &Arc, + changes: Option, + ) { + if resources.inner().contains_key(&account_id) { + drop(resources.remove(&account_id)); + } + + let publication = self + .resources_large + .publish(SwapTarget::new(account_id, collection), cache); + for (target, unpinned) in publication.unpinned { + self.swap.notify_evicted_resources(target, unpinned); + } + + if let Some(changes) = changes { + match publication.residency { + Residency::Pinned => self.swap.notify_changed(account_id, collection, changes), + Residency::Shared => self + .swap + .notify_refresh_resources(account_id, collection, changes, cache), + } + } + } +} + +fn admits_resources( + resources: &ResourceCache, + account_id: u32, + collection: SyncCollection, + cache: &Arc, +) -> bool { + if !resources.is_oversized(&account_id, cache) { + return true; + } + + trc::event!( + Cache(CacheEvent::EntryTooLarge), + AccountId = account_id, + Collection = collection.as_str(), + Size = cache.size, + Limit = resources.admission_limit(), + ); + false +} + diff --git a/crates/common/src/cache/mod.rs b/crates/common/src/cache/mod.rs index db38fdd4..178dabce 100644 --- a/crates/common/src/cache/mod.rs +++ b/crates/common/src/cache/mod.rs @@ -4,15 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{DavResources, HttpAuthCache, MailboxCache, MessageStoreCache, UpdateLock}; -use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::sync::{Semaphore, SemaphorePermit}; +use crate::{ + GroupwareResources, HttpAuthCache, MailboxCache, MessageStoreCache, MessageUpdateLock, + UpdateLock, Verification, +}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; +use std::time::{Duration, Instant}; +use tokio::sync::{Semaphore, SemaphorePermit, futures::Notified}; use utils::cache::CacheItemWeight; pub mod directory; +pub mod email; +pub mod http_auth; pub mod invalidate; +pub mod large; pub mod principals; +pub mod refresh; pub mod reload; +pub mod swap; impl MailboxCache { pub fn parent_id(&self) -> Option { @@ -41,14 +53,106 @@ pub enum LockResult<'x> { Stale(SemaphorePermit<'x>), } +#[must_use] +pub struct UpdatePermit<'x> { + lock: &'x MessageUpdateLock, +} + +pub trait CacheEpoch { + fn epoch(&self) -> u64; +} + +#[derive(Debug)] +pub struct RevalidateInterval { + started_at: Instant, + interval: AtomicU64, +} + +impl RevalidateInterval { + pub const DEFAULT: Duration = Duration::from_secs(5); + + pub fn new(interval: Duration) -> Self { + RevalidateInterval { + started_at: Instant::now(), + interval: AtomicU64::new(interval.as_millis() as u64), + } + } + + pub fn set(&self, interval: Duration) { + self.interval + .store(interval.as_millis() as u64, Ordering::Relaxed); + } + + pub fn get(&self) -> Duration { + Duration::from_millis(self.interval.load(Ordering::Relaxed)) + } + + pub fn now(&self) -> u64 { + (self.started_at.elapsed().as_millis() as u64).max(1) + } + + pub fn is_within(&self, verified_at: u64) -> bool { + verified_at != 0 + && self.now().saturating_sub(verified_at) < self.interval.load(Ordering::Relaxed) + } +} + +impl Default for RevalidateInterval { + fn default() -> Self { + RevalidateInterval::new(RevalidateInterval::DEFAULT) + } +} + +impl Verification { + pub fn capture(revalidate: &RevalidateInterval, update_lock: &impl CacheEpoch) -> Self { + Verification { + verified_at: revalidate.now(), + epoch: update_lock.epoch(), + } + } + + pub fn is_current(&self, update_lock: &impl CacheEpoch) -> bool { + self.epoch == update_lock.epoch() + } + + pub fn is_fresh(&self, update_lock: &impl CacheEpoch, revalidate: &RevalidateInterval) -> bool { + self.is_current(update_lock) && revalidate.is_within(self.verified_at) + } + + pub fn is_verified_since(&self, epoch: u64, revalidate: &RevalidateInterval) -> bool { + self.epoch >= epoch && revalidate.is_within(self.verified_at) + } +} + +impl MessageStoreCache { + pub fn is_fresh(&self, revalidate: &RevalidateInterval) -> bool { + self.verification.is_fresh(&self.update_lock, revalidate) + } +} + +impl GroupwareResources { + pub fn is_fresh(&self, revalidate: &RevalidateInterval) -> bool { + self.verification.is_fresh(&self.update_lock, revalidate) + } +} + impl UpdateLock { pub fn new() -> Self { Self { semaphore: Semaphore::new(1), revision: AtomicU64::new(0), + epoch: AtomicU64::new(0), } } + pub fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } + + pub fn mark_stale(&self) { + self.epoch.fetch_add(1, Ordering::AcqRel); + } + pub async fn acquire(&self, current_revision: u64) -> trc::Result> { let permit = self.semaphore.acquire().await.map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) @@ -75,6 +179,60 @@ impl Default for UpdateLock { } } +impl MessageUpdateLock { + pub fn try_update(&self) -> Option> { + if self.is_updating.load(Ordering::Relaxed) + || self + .is_updating + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + None + } else { + Some(UpdatePermit { lock: self }) + } + } + + pub fn released(&self) -> Notified<'_> { + self.released.notified() + } + + pub fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } + + pub fn mark_stale(&self) { + self.epoch.fetch_add(1, Ordering::AcqRel); + } +} + +impl Drop for UpdatePermit<'_> { + fn drop(&mut self) { + self.lock.is_updating.store(false, Ordering::Release); + self.lock.released.notify_waiters(); + } +} + +const _: () = assert!(size_of::() <= size_of::()); + +impl CacheEpoch for UpdateLock { + fn epoch(&self) -> u64 { + UpdateLock::epoch(self) + } +} + +impl CacheEpoch for MessageUpdateLock { + fn epoch(&self) -> u64 { + MessageUpdateLock::epoch(self) + } +} + +impl CacheEpoch for Arc { + fn epoch(&self) -> u64 { + L::epoch(self) + } +} + impl CacheItemWeight for MessageStoreCache { fn weight(&self) -> u64 { self.size @@ -87,8 +245,9 @@ impl CacheItemWeight for HttpAuthCache { } } -impl CacheItemWeight for DavResources { +impl CacheItemWeight for GroupwareResources { fn weight(&self) -> u64 { self.size } } + diff --git a/crates/common/src/cache/principals.rs b/crates/common/src/cache/principals.rs index 568b6460..67ba2409 100644 --- a/crates/common/src/cache/principals.rs +++ b/crates/common/src/cache/principals.rs @@ -11,39 +11,42 @@ use crate::{ ACCOUNT_FLAG_ENCRYPT_ALGO_AES256_GCM, ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305, ACCOUNT_FLAG_ENCRYPT_APPEND, ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, ACCOUNT_IS_USER, - AccountCache, AccountInfo, AccountTenantIds, DOMAIN_FLAG_RELAY, DOMAIN_FLAG_SUB_ADDRESSING, - DomainCache, EmailAddress, EmailAddressRef, EmailCache, MailingListCache, PermissionsGroup, - RECOVERY_ADMIN_ID, RoleCache, TenantCache, permissions::BuildPermissions, + AccountCache, AccountInfo, AccountStorage, ChangedBy, DOMAIN_FLAG_RELAY, + DOMAIN_FLAG_SUB_ADDRESSING, DomainCache, EmailAddress, EmailAddressRef, EmailCache, + EncryptionKeys, MailingListCache, PermissionsGroup, RECOVERY_ADMIN_ID, RoleCache, + permissions::BuildPermissions, }, config::smtp::auth::DkimSigners, - expr::if_block::BootstrapExprExt, + expr::{Bump, if_block::BootstrapExprExt}, network::mta::AddressResolver, storage::{ - ObjectQuota, TenantQuota, + ObjectQuota, encryption::{EncryptionMethod, parse_public_key}, }, }; +use compact_str::{CompactString, ToCompactString}; use registry::{ schema::{ - enums::{DkimRotationStage, Locale, StorageQuota, TenantStorageQuota}, + enums::{DkimRotationStage, Locale, StorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, DkimSignature, Domain, EncryptionAtRest, MailingList, MaskedEmail, - Permissions, PublicKey, Role, SubAddressing, Tenant, + Account, DkimSignature, Domain, EncryptionAtRest, MailingList, PublicKey, Role, + SubAddressing, }, }, types::id::ObjectId, }; use std::{borrow::Cow, sync::Arc}; use store::{ - U64_LEN, + U32_LEN, registry::{RegistryQuery, bootstrap::Bootstrap}, - write::{key::KeySerializer, now}, + write::key::KeySerializer, }; -use trc::{AddContext, StoreEvent}; +use trc::{AddContext, CacheEvent}; use types::id::Id; use utils::DomainPart; + impl Server { pub async fn domain(&self, domain: &str) -> trc::Result>> { let Some(domain) = domain.to_ascii_domain() else { @@ -54,9 +57,9 @@ impl Server { if let Some(domain_id) = domain_names.get(domain) { trc::event!( - Store(StoreEvent::CacheHit), - Key = domain.to_string(), - Collection = "domainName", + Cache(CacheEvent::Hit), + Key = CompactString::from(domain), + Name = "domainName", ); let result = self.domain_by_id(domain_id).await?; @@ -68,26 +71,18 @@ impl Server { } else { let domain_names_negative = &self.inner.cache.domain_names_negative; if domain_names_negative.get(domain).is_none() { - let mut object = self + let object = self .registry() - .primary_key( + .first_primary_key( ObjectType::Domain.into(), - Property::Name, - domain.as_bytes().to_vec(), + &[ + (Property::Name, domain.as_bytes()), + (Property::Aliases, domain.as_bytes()), + ], ) .await - .caused_by(trc::location!())?; - if object.is_none() { - object = self - .registry() - .primary_key( - ObjectType::Domain.into(), - Property::Aliases, - domain.as_bytes().to_vec(), - ) - .await - .caused_by(trc::location!())?; - } + .caused_by(trc::location!())? + .map(|(_, object)| object); if let Some(domain) = object { // Cache positive result @@ -95,7 +90,7 @@ impl Server { let domain = self.domain_by_id(domain_id).await?; if let Some(domain) = &domain { for name in domain.names.iter() { - domain_names.insert(name.clone(), domain_id); + domain_names.insert(CompactString::from(name.as_ref()), domain_id); } } @@ -109,18 +104,18 @@ impl Server { ); trc::event!( - Store(StoreEvent::CacheMiss), - Key = domain.to_string(), - Collection = "domainName", + Cache(CacheEvent::Miss), + Key = CompactString::from(domain), + Name = "domainName", ); Ok(None) } } else { trc::event!( - Store(StoreEvent::CacheHit), - Key = domain.to_string(), - Collection = "domainNameNegative", + Cache(CacheEvent::Hit), + Key = CompactString::from(domain), + Name = "domainNameNegative", ); Ok(None) @@ -137,20 +132,12 @@ impl Server { .await { Ok(domain) => { - trc::event!( - Store(StoreEvent::CacheHit), - Key = domain_id, - Collection = "domainId", - ); + trc::event!(Cache(CacheEvent::Hit), Key = domain_id, Name = "domainId",); Ok(Some(domain)) } Err(guard) => { - trc::event!( - Store(StoreEvent::CacheMiss), - Key = domain_id, - Collection = "domainId", - ); + trc::event!(Cache(CacheEvent::Miss), Key = domain_id, Name = "domainId",); let Some(domain) = self.registry().object::(domain_id.into()).await? else { return Ok(None); }; @@ -193,8 +180,6 @@ impl Server { ) .collect(), id: domain_id, - id_directory: domain.directory_id.map(|id| id.document_id()), - id_tenant: domain.member_tenant_id.map(|id| id.document_id()), catch_all: domain.catch_all_address.map(|s| s.into_boxed_str()), sub_addressing_custom, flags, @@ -215,10 +200,10 @@ impl Server { if let Some(email) = emails.get(&EmailAddressRef::new(local_part, domain_id)) { trc::event!( - Store(StoreEvent::CacheHit), - Key = local_part.to_string(), + Cache(CacheEvent::Hit), + Key = CompactString::from(local_part), Domain = domain_id, - Collection = "email", + Name = "email", ); Ok(Some(email)) @@ -229,10 +214,10 @@ impl Server { .is_none() { trc::event!( - Store(StoreEvent::CacheMiss), - Key = local_part.to_string(), + Cache(CacheEvent::Miss), + Key = CompactString::from(local_part), Domain = domain_id, - Collection = "email", + Name = "email", ); if let Some(object) = self @@ -240,9 +225,9 @@ impl Server { .primary_key( None, Property::Email, - KeySerializer::new(local_part.len() + U64_LEN) + KeySerializer::new(local_part.len() + U32_LEN) .write(local_part.as_bytes()) - .write(domain_id as u64) + .write(domain_id) .finalize(), ) .await @@ -259,7 +244,7 @@ impl Server { .iter() .any(|address| { address.domain_id == domain_id - && address.local_part.as_ref() == local_part + && address.local_part == local_part }) { EmailCache::Account(item_id) @@ -271,7 +256,7 @@ impl Server { if let Some(list) = self.try_list(item_id).await? && !list.addresses.iter().any(|address| { address.domain_id == domain_id - && address.local_part.as_ref() == local_part + && address.local_part == local_part }) { EmailCache::DisabledListAddress(item_id) @@ -285,7 +270,7 @@ impl Server { .details( "Object with email property is not an account or mailing list.", ) - .ctx(trc::Key::Id, object.to_string()) + .ctx(trc::Key::Id, object.to_compact_string()) .caused_by(trc::location!())); } }; @@ -303,10 +288,10 @@ impl Server { } } else { trc::event!( - Store(StoreEvent::CacheHit), - Key = local_part.to_string(), + Cache(CacheEvent::Hit), + Key = CompactString::from(local_part), Domain = domain_id, - Collection = "emailNegative", + Name = "emailNegative", ); Ok(None) } @@ -339,14 +324,13 @@ impl Server { .into(), id: RECOVERY_ADMIN_ID, addresses: Default::default(), - id_tenant: Default::default(), id_member_of: Default::default(), quota_disk: Default::default(), - quota_objects: Default::default(), description: Some("Recovery admin account".into()), - encryption_key: Default::default(), + storage: Default::default(), locale: Default::default(), flags: Default::default(), + active_script: Default::default(), })) } else { Err(trc::AuthEvent::Error @@ -366,20 +350,12 @@ impl Server { .await { Ok(account) => { - trc::event!( - Store(StoreEvent::CacheHit), - Key = account_id, - Collection = "account", - ); + trc::event!(Cache(CacheEvent::Hit), Key = account_id, Name = "account",); Ok(Some(account)) } Err(guard) => { - trc::event!( - Store(StoreEvent::CacheMiss), - Key = account_id, - Collection = "account", - ); + trc::event!(Cache(CacheEvent::Miss), Key = account_id, Name = "account",); let Some(account) = self.registry().object::(account_id.into()).await? else { @@ -450,9 +426,10 @@ impl Server { .await .caused_by(trc::location!())? { - parse_public_key(&public_key) - .unwrap_or_default() - .map(|params| { + match parse_public_key(&public_key).and_then(|params| { + params.ok_or(Cow::Borrowed("No public key found")) + }) { + Ok(params) => { match params.method { EncryptionMethod::PGP => { flags |= ACCOUNT_FLAG_ENCRYPT_METHOD_PGP @@ -461,8 +438,23 @@ impl Server { flags |= ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME } } - params.certs - }) + Some(params.certs) + } + Err(err) => { + trc::error!( + trc::EventType::Registry( + trc::RegistryEvent::ValidationError + ) + .into_err() + .details("Invalid public key, encryption disabled") + .reason(err) + .account_id(account_id) + .ctx(trc::Key::Id, settings.public_key.id()) + .caused_by(trc::location!()) + ); + None + } + } } else { None } @@ -482,25 +474,24 @@ impl Server { account .aliases .into_iter() - .filter(|alias| alias.enabled) + .filter(|alias| alias.enable) .map(|alias| EmailAddress { local_part: alias.name.into(), domain_id: alias.domain_id.document_id(), }), ) .collect(), - id_tenant: account.member_tenant_id.map(|id| id.document_id()), id_member_of: account .member_group_ids .into_iter() .map(|id| id.document_id()) .collect(), quota_disk, - quota_objects: quota_objects.map(Box::new), description: account.description.map(Into::into), + storage: AccountStorage::boxed(quota_objects, encryption_key), locale: account.locale, - encryption_key, flags, + active_script: Default::default(), } } Account::Group(account) => { @@ -545,21 +536,20 @@ impl Server { account .aliases .into_iter() - .filter(|alias| alias.enabled) + .filter(|alias| alias.enable) .map(|alias| EmailAddress { local_part: alias.name.into(), domain_id: alias.domain_id.document_id(), }), ) .collect(), - id_tenant: account.member_tenant_id.map(|id| id.document_id()), id_member_of: Default::default(), quota_disk, - quota_objects: quota_objects.map(Box::new), description: account.description.map(Into::into), - encryption_key: None, + storage: AccountStorage::boxed(quota_objects, None), locale: account.locale, flags: 0, + active_script: Default::default(), } } }); @@ -599,10 +589,12 @@ impl Server { if domain.flags & DOMAIN_FLAG_SUB_ADDRESSING != 0 { if let Some(sub_addressing) = &domain.sub_addressing_custom { // Custom sub-addressing resolution + let mut arena = Bump::new(); if let Some(result) = self .eval_if::( sub_addressing, &AddressResolver(local_part.as_ref()), + &mut arena, 0, ) .await @@ -645,28 +637,6 @@ impl Server { self.build_account_info(account).await } - pub async fn scheduling_account_info( - &self, - authenticated_account_id: u32, - owner_account_id: u32, - ) -> trc::Result { - let account = self.account(authenticated_account_id).await?; - let mut account_info = self.build_account_info(account).await?; - - if owner_account_id != authenticated_account_id { - let owner_account = self.account(owner_account_id).await?; - let owner_account_info = self.build_account_info(owner_account).await?; - - for address in owner_account_info.addresses { - if !account_info.addresses.contains(&address) { - account_info.addresses.push(address); - } - } - } - - Ok(account_info) - } - pub async fn build_account_info(&self, account: Arc) -> trc::Result { let mut addresses = Vec::with_capacity(account.id_member_of.len() + account.addresses.len()); @@ -674,7 +644,7 @@ impl Server { if let Some(domain) = self.domain_by_id(address.domain_id).await? { for name in domain.names.iter() { let mut addr = String::with_capacity(name.len() + address.local_part.len() + 1); - addr.push_str(address.local_part.as_ref()); + addr.push_str(&address.local_part); addr.push('@'); addr.push_str(name.as_ref()); addresses.push(addr); @@ -689,7 +659,7 @@ impl Server { for name in domain.names.iter() { let mut addr = String::with_capacity(name.len() + address.local_part.len() + 1); - addr.push_str(address.local_part.as_ref()); + addr.push_str(&address.local_part); addr.push('@'); addr.push_str(name.as_ref()); addresses.push(addr); @@ -710,12 +680,12 @@ impl Server { let cache = &self.inner.cache.roles; match cache.get_value_or_guard_async(&id).await { Ok(role) => { - trc::event!(Store(StoreEvent::CacheHit), Key = id, Collection = "role"); + trc::event!(Cache(CacheEvent::Hit), Key = id, Name = "role"); Ok(role) } Err(guard) => { - trc::event!(Store(StoreEvent::CacheMiss), Key = id, Collection = "role"); + trc::event!(Cache(CacheEvent::Miss), Key = id, Name = "role"); let Some(role) = self.registry().object::(id.into()).await? else { return Err(trc::AuthEvent::Error @@ -748,107 +718,39 @@ impl Server { } } - pub async fn tenant(&self, id: u32) -> trc::Result> { - let cache = &self.inner.cache.tenants; - match cache.get_value_or_guard_async(&id).await { - Ok(tenant) => { - trc::event!(Store(StoreEvent::CacheHit), Key = id, Collection = "tenant"); - - Ok(tenant) - } - Err(guard) => { - trc::event!( - Store(StoreEvent::CacheMiss), - Key = id, - Collection = "tenant" - ); - - let Some(tenant) = self.registry().object::(id.into()).await? else { - return Err(trc::AuthEvent::Error - .into_err() - .details("Tenant not found.") - .ctx(trc::Key::Id, id) - .caused_by(trc::location!())); - }; - - let mut quota_objects: Option = None; - let mut quota_disk = 0; - for (resource, limit) in tenant.quotas { - if resource == TenantStorageQuota::MaxDiskQuota { - quota_disk = limit; - } else { - quota_objects - .get_or_insert_default() - .set(resource, limit as u32); - } - } - - // Calculate effective permissions - let permissions = match tenant.permissions { - Permissions::Inherit => None, - Permissions::Merge(permissions) => Some(Box::new( - PermissionsGroup::from(permissions).with_merge(true), - )), - Permissions::Replace(permissions) => Some(Box::new( - PermissionsGroup::from(permissions).with_merge(false), - )), - }; - - let cache = Arc::new(TenantCache { - id_roles: tenant - .roles - .role_ids() - .unwrap_or( - self.core - .network - .security - .default_role_ids_tenant - .as_slice(), - ) - .iter() - .map(|id| id.document_id()) - .collect(), - quota_disk, - quota_objects: quota_objects.map(Box::new), - permissions, - }); - - let _ = guard.insert(cache.clone()); - Ok(cache) - } - } - } pub async fn try_list(&self, id: u32) -> trc::Result>> { let cache = &self.inner.cache.lists; match cache.get_value_or_guard_async(&id).await { Ok(list) => { - trc::event!(Store(StoreEvent::CacheHit), Key = id, Collection = "list"); + trc::event!(Cache(CacheEvent::Hit), Key = id, Name = "list"); Ok(Some(list)) } Err(guard) => { - trc::event!(Store(StoreEvent::CacheMiss), Key = id, Collection = "list"); + trc::event!(Cache(CacheEvent::Miss), Key = id, Name = "list"); let Some(list) = self.registry().object::(id.into()).await? else { return Ok(None); }; - let cache = - Arc::new(MailingListCache { - addresses: [EmailAddress { - local_part: list.name.into(), - domain_id: list.domain_id.document_id(), - }] - .into_iter() - .chain(list.aliases.into_iter().filter(|alias| alias.enabled).map( - |alias| EmailAddress { + let cache = Arc::new(MailingListCache { + addresses: [EmailAddress { + local_part: list.name.into(), + domain_id: list.domain_id.document_id(), + }] + .into_iter() + .chain( + list.aliases + .into_iter() + .filter(|alias| alias.enable) + .map(|alias| EmailAddress { local_part: alias.name.into(), domain_id: alias.domain_id.document_id(), - }, - )) - .collect(), - recipients: list.recipients.into_iter().map(Into::into).collect(), - }); + }), + ) + .collect(), + recipients: list.recipients.into_iter().map(Into::into).collect(), + }); let _ = guard.insert(cache.clone()); Ok(Some(cache)) } @@ -863,18 +765,18 @@ impl Server { match cache.get_value_or_guard_async(&domain.id).await { Ok(signers) => { trc::event!( - Store(StoreEvent::CacheHit), + Cache(CacheEvent::Hit), Key = domain.id, - Collection = "dkimSigners", + Name = "dkimSigners", ); - Ok(Some(signers)) + Ok(has_dkim_signers(&signers).then_some(signers)) } Err(guard) => { trc::event!( - Store(StoreEvent::CacheMiss), + Cache(CacheEvent::Miss), Key = domain.id, - Collection = "dkimSigners", + Name = "dkimSigners", ); let ids = self @@ -887,7 +789,7 @@ impl Server { let domain_name = &domain.names[0]; let mut signers = DkimSigners { dkim1: Vec::with_capacity(ids.len()), - dkim2: None, + ..Default::default() }; for id in ids { if let Some(signature) = self.registry().object::(id).await? @@ -898,18 +800,18 @@ impl Server { } } - if !signers.dkim1.is_empty() || signers.dkim2.is_some() { - let signers = Arc::new(signers); - let _ = guard.insert(signers.clone()); - Ok(Some(signers)) - } else { - Ok(None) - } + let signers = Arc::new(signers); + let _ = guard.insert(signers.clone()); + Ok(has_dkim_signers(&signers).then_some(signers)) } } } } +fn has_dkim_signers(signers: &DkimSigners) -> bool { + !signers.dkim1.is_empty() || signers.dkim2.is_some() +} + impl AccountInfo { #[inline(always)] pub fn account_id(&self) -> u32 { @@ -925,16 +827,11 @@ impl AccountInfo { self.account.description.as_deref() } - #[inline(always)] - pub fn tenant_id(&self) -> Option { - self.account.id_tenant - } #[inline(always)] - pub fn account_tenant_ids(&self) -> AccountTenantIds { - AccountTenantIds { + pub fn changed_by(&self) -> ChangedBy { + ChangedBy { account_id: self.account_id, - tenant_id: self.account.id_tenant, } } @@ -954,7 +851,7 @@ impl AccountInfo { #[inline(always)] pub fn object_quotas(&self) -> Option<&ObjectQuota> { - self.account.quota_objects.as_deref() + self.account.object_quotas() } #[inline(always)] @@ -979,10 +876,6 @@ impl AccountCache { self.description.as_deref() } - #[inline(always)] - pub fn tenant_id(&self) -> Option { - self.id_tenant - } #[inline(always)] pub fn is_user_account(&self) -> bool { @@ -996,14 +889,22 @@ impl AccountCache { #[inline(always)] pub fn object_quotas(&self) -> Option<&ObjectQuota> { - self.quota_objects.as_deref() + self.storage + .as_ref() + .and_then(|storage| storage.quota_objects.as_ref()) } #[inline(always)] - pub fn account_tenant_ids(&self) -> AccountTenantIds { - AccountTenantIds { + pub fn encryption_key(&self) -> Option<&EncryptionKeys> { + self.storage + .as_ref() + .and_then(|storage| storage.encryption_key.as_ref()) + } + + #[inline(always)] + pub fn changed_by(&self) -> ChangedBy { + ChangedBy { account_id: self.id, - tenant_id: self.id_tenant, } } } diff --git a/crates/common/src/cache/refresh.rs b/crates/common/src/cache/refresh.rs new file mode 100644 index 00000000..503a10ca --- /dev/null +++ b/crates/common/src/cache/refresh.rs @@ -0,0 +1,197 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + ItemDelta, MailboxRow, MailboxRowsMemo, MessagesCache, NO_ID, RefreshState, SharedMailboxRows, + UidNextMemo, +}; +use std::{borrow::Cow, ptr, sync::Arc}; +use tokio::sync::OnceCell; + +pub const MAX_ITEM_DELTAS: usize = 8; +pub const MAX_DELTA_IDS: usize = 4096; +pub const MIN_SHARED_ROWS: usize = 16_384; +pub const MAX_SHARED_MAILBOXES: usize = 4; +pub const SHARED_ROWS_WEIGHT_DIVISOR: u64 = 4; + +impl RefreshState { + pub fn advance(&self, delta: ItemDelta) -> RefreshState { + let mut total = delta.document_ids.len(); + let mut from_change_id = delta.from_change_id; + let mut kept = 0; + for older in self.deltas.iter().rev().take(MAX_ITEM_DELTAS - 1) { + total += older.document_ids.len(); + if older.to_change_id != from_change_id || total > MAX_DELTA_IDS { + break; + } + from_change_id = older.from_change_id; + kept += 1; + } + + let carried = self + .deltas + .get(self.deltas.len() - kept..) + .unwrap_or_default(); + let mut deltas = Vec::with_capacity(carried.len() + 1); + deltas.extend_from_slice(carried); + deltas.push(delta); + + RefreshState { + deltas: deltas.into_boxed_slice(), + uid_next: UidNextMemo::default(), + mailbox_rows: Arc::default(), + } + } + + pub fn with_new_uid_next(&self) -> RefreshState { + RefreshState { + deltas: self.deltas.clone(), + uid_next: UidNextMemo::default(), + mailbox_rows: self.mailbox_rows.clone(), + } + } + + pub fn changed_items(&self, from_change_id: u64, to_change_id: u64) -> Option> { + let start = self + .deltas + .iter() + .position(|delta| delta.from_change_id == from_change_id)?; + let chain = self.deltas.get(start..)?; + let mut next_change_id = from_change_id; + for delta in chain { + if delta.from_change_id != next_change_id { + return None; + } + next_change_id = delta.to_change_id; + } + if next_change_id != to_change_id { + return None; + } + + match chain { + [delta] => Some(Cow::Borrowed(&delta.document_ids)), + _ => { + let mut ids = Vec::with_capacity( + chain + .iter() + .map(|delta| delta.document_ids.len()) + .sum::(), + ); + for delta in chain { + ids.extend_from_slice(&delta.document_ids); + } + ids.sort_unstable(); + ids.dedup(); + Some(Cow::Owned(ids)) + } + } + } +} + +impl ItemDelta { + pub fn new(from_change_id: u64, to_change_id: u64, document_ids: &[u32]) -> Option { + (document_ids.len() <= MAX_DELTA_IDS).then(|| ItemDelta { + from_change_id, + to_change_id, + document_ids: Arc::from(document_ids), + }) + } +} + +impl UidNextMemo { + pub async fn get_or_read(&self, mailbox_id: u32, read: F) -> trc::Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let cell = { + let mut values = self.values.lock(); + let index = values.partition_point(|(id, _)| *id < mailbox_id); + match values.get(index) { + Some((id, cell)) if *id == mailbox_id => Arc::clone(cell), + _ => { + let cell = Arc::new(OnceCell::new()); + values.insert(index, (mailbox_id, Arc::clone(&cell))); + cell + } + } + }; + cell.get_or_try_init(read).await.copied() + } +} + +impl MailboxRow { + #[inline(always)] + pub fn is_live(&self) -> bool { + self.id != NO_ID + } + + pub fn bytes_of(rows: usize) -> u64 { + (rows * size_of::()) as u64 + } +} + +impl MailboxRowsMemo { + pub fn get_or_build( + &self, + emails: &Arc, + mailbox_id: u32, + build: impl FnOnce() -> Vec, + ) -> Arc> { + let shared = self + .entries + .lock() + .iter() + .find(|entry| entry.mailbox_id == mailbox_id && entry.is_of(emails)) + .map(|entry| entry.rows.clone()); + if let Some(rows) = shared { + return rows; + } + + let mut rows = build(); + if rows.len() < MIN_SHARED_ROWS { + return Arc::new(rows); + } + + let mut entries = self.entries.lock(); + entries.retain(|entry| entry.is_of(emails)); + if let Some(entry) = entries.iter().find(|entry| entry.mailbox_id == mailbox_id) { + return entry.rows.clone(); + } + let is_full = entries.len() >= MAX_SHARED_MAILBOXES; + let kept = entries + .iter() + .skip(usize::from(is_full)) + .map(SharedMailboxRows::bytes) + .sum::(); + if kept + MailboxRow::bytes_of(rows.len()) > emails.size / SHARED_ROWS_WEIGHT_DIVISOR { + return Arc::new(rows); + } + if is_full { + entries.remove(0); + } + rows.shrink_to_fit(); + let rows = Arc::new(rows); + entries.push(SharedMailboxRows { + mailbox_id, + emails: Arc::downgrade(emails), + rows: rows.clone(), + }); + rows + } +} + +impl SharedMailboxRows { + fn bytes(&self) -> u64 { + MailboxRow::bytes_of(self.rows.len()) + } + + #[inline(always)] + fn is_of(&self, emails: &Arc) -> bool { + ptr::eq(self.emails.as_ptr(), Arc::as_ptr(emails)) + } +} + diff --git a/crates/common/src/cache/reload.rs b/crates/common/src/cache/reload.rs index 74788771..e9064490 100644 --- a/crates/common/src/cache/reload.rs +++ b/crates/common/src/cache/reload.rs @@ -31,6 +31,10 @@ pub struct ReloadResult { impl Server { pub async fn reload_registry(&self, change: RegistryChange) -> trc::Result { + Box::pin(self.reload(change)).await + } + + async fn reload(&self, change: RegistryChange) -> trc::Result { let mut bootstrap = Bootstrap::new(self.registry().clone()).await; let object = match change { RegistryChange::Insert(id) => { @@ -45,8 +49,7 @@ impl Server { if expires_at > now() { let mut ips = self.inner.data.blocked_ips.write(); if let Some(ip) = ip.address.try_to_ip() { - ips.blocked_ip_addresses - .insert(IpWithTtl::new(ip, expires_at)); + ips.block(ip, expires_at); } else { ips.blocked_ip_networks .push(IpWithTtl::new(ip.address, expires_at)); @@ -104,8 +107,6 @@ impl Server { data: storage.data.clone(), blob: storage.blob.clone(), search: storage.search.clone(), - metrics: storage.metrics.clone(), - tracing: storage.tracing.clone(), memory: storage.memory.clone(), coordinator: storage.coordinator.clone(), directory: directory.default_directory, @@ -113,10 +114,15 @@ impl Server { }; // Parse tracers - let tracers = Telemetry::parse(&mut bootstrap, &storage).await; + let tracers = Telemetry::parse( + &mut bootstrap, + &storage, + self.inner.ipc.index_queue_notify(), + ) + .await; if bootstrap.errors.is_empty() { - let core = Box::pin(Core::parse(&mut bootstrap, storage)).await; + let core = Core::parse(&mut bootstrap, storage).await; if bootstrap.errors.is_empty() { let mut servers = Listeners::parse(&mut bootstrap).await; diff --git a/crates/common/src/cache/swap/blob.rs b/crates/common/src/cache/swap/blob.rs new file mode 100644 index 00000000..7b5535b0 --- /dev/null +++ b/crates/common/src/cache/swap/blob.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{SwapKey, pointer::SwapPointer}; +use registry::schema::enums::CompressionAlgo; +use store::{ + BlobStore, Store, rand, + write::{BatchBuilder, BlobLink, BlobOp, blob::BlobLinkValue, now}, +}; +use trc::AddContext; +use types::blob_hash::BlobHash; + +const LOAD_ATTEMPTS: usize = 2; + +pub struct BlobSwapStore { + blob: BlobStore, + data: Store, + retention: u64, + max_size: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlobSlot { + First, + Second, +} + +impl BlobSwapStore { + pub fn new(blob: BlobStore, data: Store, retention: u64, max_size: u64) -> Self { + BlobSwapStore { + blob, + data, + retention: retention.max(1), + max_size: max_size as usize, + } + } + + pub async fn load(&self, key: SwapKey) -> trc::Result>> { + let mut previous = None; + + for _ in 0..LOAD_ATTEMPTS { + let Some((pointer, slot)) = self.read_pointer(key).await? else { + return Ok(None); + }; + if previous == Some(pointer) { + break; + } + + if pointer.length() > self.max_size as u64 { + trc::event!( + Cache(trc::CacheEvent::SwapError), + AccountId = key.account_id, + Collection = key.collection.as_str(), + Size = pointer.length(), + Limit = self.max_size, + Details = + "Cache snapshot in the blob store exceeds the configured maximum size", + ); + return Ok(None); + } + + if let Some(data) = self.read_slot(key, slot).await? + && data.len() as u64 == pointer.length() + { + return Ok(Some(data)); + } + + previous = Some(pointer); + } + + trc::event!( + Cache(trc::CacheEvent::SwapMiss), + AccountId = key.account_id, + Collection = key.collection.as_str(), + Details = + "Cache snapshot slot in the blob store is missing or does not match its pointer", + ); + + Ok(None) + } + + pub async fn store(&self, key: SwapKey, data: &[u8]) -> trc::Result<()> { + let pointer_hash = key.blob_hash(); + let current = self.read_pointer(key).await?.map(|(_, slot)| slot); + let slot = current.map_or(BlobSlot::First, BlobSlot::other); + let slot_hash = key.slot_hash(slot); + + self.blob + .put_blob(slot_hash.as_slice(), data, CompressionAlgo::None) + .await + .caused_by(trc::location!())?; + let pointer = + SwapPointer::new(rand::random(), slot.generation(), data.len() as u64, None).encode(); + self.link(&[ + (slot_hash, data.len()), + (pointer_hash.clone(), pointer.len()), + ]) + .await?; + + self.blob + .put_blob(pointer_hash.as_slice(), &pointer, CompressionAlgo::None) + .await + .caused_by(trc::location!())?; + + if let Some(superseded) = current + && let Err(err) = self.delete(&[key.slot_hash(superseded)]).await + { + trc::error!( + err.details("Failed to remove a superseded cache snapshot slot") + .ctx(trc::Key::AccountId, key.account_id) + .ctx(trc::Key::Collection, key.collection.as_str()) + ); + } + + Ok(()) + } + + pub async fn remove(&self, key: SwapKey) -> trc::Result<()> { + self.delete(&[ + key.blob_hash(), + key.slot_hash(BlobSlot::First), + key.slot_hash(BlobSlot::Second), + ]) + .await + } + + async fn read_pointer(&self, key: SwapKey) -> trc::Result> { + self.blob + .get_blob(key.blob_hash().as_slice(), 0..usize::MAX) + .await + .caused_by(trc::location!()) + .map(|pointer| { + pointer + .as_deref() + .and_then(SwapPointer::parse) + .and_then(|pointer| Some((pointer, BlobSlot::of(&pointer)?))) + }) + } + + async fn read_slot(&self, key: SwapKey, slot: BlobSlot) -> trc::Result>> { + match self + .blob + .get_blob(key.slot_hash(slot).as_slice(), 0..usize::MAX) + .await + { + Ok(data) => Ok(data), + Err(err) if err.matches(trc::EventType::Store(trc::StoreEvent::DecompressError)) => { + Ok(None) + } + Err(err) => Err(err.caused_by(trc::location!())), + } + } + + async fn link(&self, blobs: &[(BlobHash, usize)]) -> trc::Result<()> { + let until = self.expires_at(); + let mut batch = BatchBuilder::new(); + for (hash, size) in blobs { + batch + .set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }, + BlobLinkValue::new(*size), + ) + .set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); + } + + self.data + .write_batch(&mut batch) + .await + .caused_by(trc::location!()) + .map(|_| ()) + } + + async fn delete(&self, hashes: &[BlobHash]) -> trc::Result<()> { + for hash in hashes { + self.blob + .delete_blob(hash.as_slice()) + .await + .caused_by(trc::location!())?; + } + + let window = self.expires_at(); + let mut batch = BatchBuilder::new(); + for hash in hashes { + for until in [window, window.saturating_sub(self.retention)] { + batch.clear(BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }); + } + batch.clear(BlobOp::Commit { hash: hash.clone() }); + } + + self.data + .write_batch(&mut batch) + .await + .caused_by(trc::location!()) + .map(|_| ()) + } + + fn expires_at(&self) -> u64 { + (now() + self.retention).div_ceil(self.retention) * self.retention + } +} + +impl BlobSlot { + fn of(pointer: &SwapPointer) -> Option { + match (pointer.generation(), pointer.chunks()) { + (0, None) => Some(BlobSlot::First), + (1, None) => Some(BlobSlot::Second), + _ => None, + } + } + + fn generation(self) -> u64 { + match self { + BlobSlot::First => 0, + BlobSlot::Second => 1, + } + } + + fn other(self) -> Self { + match self { + BlobSlot::First => BlobSlot::Second, + BlobSlot::Second => BlobSlot::First, + } + } +} + +impl SwapKey { + pub fn blob_hash(&self) -> BlobHash { + BlobHash::generate(format!( + "STALWART_SWAP_{}_{}_{}", + self.account_id, + u8::from(self.collection), + self.part.code() + )) + } + + pub fn slot_hash(&self, slot: BlobSlot) -> BlobHash { + BlobHash::generate(format!( + "STALWART_SWAP_{}_{}_{}_{}", + self.account_id, + u8::from(self.collection), + self.part.code(), + slot.generation() + )) + } +} + diff --git a/crates/common/src/cache/swap/evict.rs b/crates/common/src/cache/swap/evict.rs new file mode 100644 index 00000000..5976ee15 --- /dev/null +++ b/crates/common/src/cache/swap/evict.rs @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + SwapHandoff, + writer::{Snapshot, SwapTarget}, +}; +use crate::{GroupwareResources, MessageStoreCache}; +use quick_cache::Lifecycle; +use std::sync::Arc; +use types::collection::SyncCollection; + +pub trait SwapCached { + fn into_snapshot(cache: Arc) -> Snapshot; +} + +impl SwapCached for MessageStoreCache { + fn into_snapshot(cache: Arc) -> Snapshot { + Snapshot::Messages(cache) + } +} + +impl SwapCached for GroupwareResources { + fn into_snapshot(cache: Arc) -> Snapshot { + Snapshot::Resources(cache) + } +} + +#[derive(Clone)] +pub struct SwapEviction { + handoff: Option>, + collection: SyncCollection, +} + +pub struct EvictionState { + released: [Option>; 2], + evicted: Vec<(u32, Arc)>, + handoff: Option<(Arc, SyncCollection)>, +} + +impl SwapEviction { + pub(super) fn new(handoff: Option>, collection: SyncCollection) -> Self { + SwapEviction { + handoff, + collection, + } + } +} + +impl Default for EvictionState { + fn default() -> Self { + EvictionState { + released: [None, None], + evicted: Vec::new(), + handoff: None, + } + } +} + +impl Drop for EvictionState { + fn drop(&mut self) { + if let Some((handoff, collection)) = self.handoff.take() { + for (account_id, cache) in self.evicted.drain(..) { + handoff.evicted( + SwapTarget::new(account_id, collection), + T::into_snapshot(cache), + ); + } + } + } +} + +impl Lifecycle> for SwapEviction { + type RequestState = EvictionState; + + fn before_evict(&self, state: &mut Self::RequestState, key: &u32, val: &mut Arc) { + if let Some(handoff) = &self.handoff { + if state.handoff.is_none() { + state.handoff = Some((handoff.clone(), self.collection)); + } + state.evicted.push((*key, val.clone())); + } + } + + fn on_evict(&self, state: &mut Self::RequestState, _: u32, val: Arc) { + if let Some(slot) = state.released.iter_mut().find(|slot| slot.is_none()) { + *slot = Some(val); + } + } +} + diff --git a/crates/common/src/cache/swap/file.rs b/crates/common/src/cache/swap/file.rs new file mode 100644 index 00000000..4662951c --- /dev/null +++ b/crates/common/src/cache/swap/file.rs @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::SwapKey; +use registry::schema::structs; +use std::path::{Path, PathBuf}; +use tokio::{ + fs::{self, File}, + io::AsyncWriteExt, +}; + +const SHARDS: u32 = 256; + +pub struct FileSwapStore { + root: PathBuf, + max_size: u64, +} + +impl FileSwapStore { + pub async fn open(config: &structs::LocalFileSwap) -> Result { + let root = PathBuf::from(config.path.as_str()); + if root.as_os_str().is_empty() { + return Err("No path configured for the local file cache swap tier".to_string()); + } + create_dir(&root) + .await + .map_err(|err| format!("Failed to create cache swap directory {root:?}: {err}"))?; + + let probe = root.join(".stalwart-swap-probe"); + create_file(&probe) + .await + .map_err(|err| format!("Cache swap directory {root:?} is not writable: {err}"))?; + let _ = fs::remove_file(&probe).await; + + remove_stale_temporaries(&root).await; + + Ok(FileSwapStore { + root, + max_size: config.max_account_size, + }) + } + + pub async fn load(&self, key: SwapKey) -> trc::Result>> { + let path = self.path(key); + match fs::metadata(&path).await { + Ok(metadata) if metadata.len() > self.max_size => { + trc::event!( + Cache(trc::CacheEvent::SwapError), + AccountId = key.account_id, + Collection = key.collection.as_str(), + Size = metadata.len(), + Limit = self.max_size, + Details = "Cache snapshot on disk exceeds the configured maximum size", + ); + return Ok(None); + } + Ok(_) => (), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(into_error(err).caused_by(trc::location!())), + } + + match fs::read(&path).await { + Ok(data) => Ok(Some(data)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(into_error(err).caused_by(trc::location!())), + } + } + + pub async fn store(&self, key: SwapKey, data: &[u8]) -> trc::Result<()> { + let path = self.path(key); + let directory = path.parent().unwrap_or(&self.root); + if let Err(err) = create_dir(directory).await { + return Err(into_error(err).caused_by(trc::location!())); + } + + let temporary = path.with_extension("tmp"); + let result = async { + let mut file = create_file(&temporary).await?; + file.write_all(data).await?; + file.sync_all().await?; + drop(file); + fs::rename(&temporary, &path).await + } + .await; + + match result { + Ok(()) => Ok(()), + Err(err) => { + let _ = fs::remove_file(&temporary).await; + Err(into_error(err).caused_by(trc::location!())) + } + } + } + + pub async fn remove(&self, key: SwapKey) -> trc::Result<()> { + match fs::remove_file(self.path(key)).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(into_error(err).caused_by(trc::location!())), + } + } + + fn path(&self, key: SwapKey) -> PathBuf { + let mut path = self.root.join(format!("{:02x}", key.account_id % SHARDS)); + path.push(key.file_name()); + path + } + + pub fn root(&self) -> &Path { + &self.root + } +} + +async fn create_dir(path: &Path) -> std::io::Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + builder.mode(0o700); + builder.create(path).await +} + +async fn create_file(path: &Path) -> std::io::Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + options.mode(0o600); + options.open(path).await +} + +async fn remove_stale_temporaries(root: &Path) { + let mut directories = vec![root.to_path_buf()]; + + while let Some(directory) = directories.pop() { + let Ok(mut entries) = fs::read_dir(&directory).await else { + continue; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let path = entry.path(); + match entry.file_type().await { + Ok(file_type) if file_type.is_dir() => directories.push(path), + Ok(_) if path.extension().is_some_and(|extension| extension == "tmp") => { + let _ = fs::remove_file(&path).await; + } + _ => (), + } + } + } +} + +fn into_error(err: std::io::Error) -> trc::Error { + trc::EventType::Cache(trc::CacheEvent::SwapError) + .reason(err) + .details("Cache swap file error") +} diff --git a/crates/common/src/cache/swap/frame.rs b/crates/common/src/cache/swap/frame.rs new file mode 100644 index 00000000..8b4bf142 --- /dev/null +++ b/crates/common/src/cache/swap/frame.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::SwapPart; + +const MAGIC: u32 = 0x5357_4131; +const VERSION: u16 = 1; + +pub const HEADER_LEN: usize = 32; + +pub struct SwapFrame<'x> { + part: SwapPart, + change_id: u64, + count: u32, + payload: &'x [u8], +} + +impl<'x> SwapFrame<'x> { + pub fn reserve_header() -> Vec { + vec![0u8; HEADER_LEN] + } + + pub fn seal(out: &mut [u8], part: SwapPart, change_id: u64, count: u32) -> bool { + let Some(payload_len) = out + .len() + .checked_sub(HEADER_LEN) + .and_then(|len| u32::try_from(len).ok()) + else { + return false; + }; + let checksum = xxhash_rust::xxh3::xxh3_64(&out[HEADER_LEN..]); + let header = &mut out[..HEADER_LEN]; + + header[0..4].copy_from_slice(&MAGIC.to_le_bytes()); + header[4..6].copy_from_slice(&VERSION.to_le_bytes()); + header[6] = part.code(); + header[7] = 0; + header[8..16].copy_from_slice(&change_id.to_le_bytes()); + header[16..20].copy_from_slice(&count.to_le_bytes()); + header[20..24].copy_from_slice(&payload_len.to_le_bytes()); + header[24..32].copy_from_slice(&checksum.to_le_bytes()); + true + } + + pub fn parse(buf: &'x [u8]) -> Option { + if buf.len() < HEADER_LEN + || u32::from_le_bytes(buf[0..4].try_into().ok()?) != MAGIC + || u16::from_le_bytes(buf[4..6].try_into().ok()?) != VERSION + { + return None; + } + + let payload_len = u32::from_le_bytes(buf[20..24].try_into().ok()?) as usize; + let checksum = u64::from_le_bytes(buf[24..32].try_into().ok()?); + let payload = buf.get(HEADER_LEN..HEADER_LEN.checked_add(payload_len)?)?; + + if xxhash_rust::xxh3::xxh3_64(payload) != checksum { + return None; + } + + Some(SwapFrame { + part: SwapPart::from_code(buf[6])?, + change_id: u64::from_le_bytes(buf[8..16].try_into().ok()?), + count: u32::from_le_bytes(buf[16..20].try_into().ok()?), + payload, + }) + } + + pub fn part(&self) -> SwapPart { + self.part + } + + pub fn change_id(&self) -> u64 { + self.change_id + } + + pub fn count(&self) -> u32 { + self.count + } + + pub fn payload(&self) -> &'x [u8] { + self.payload + } +} + diff --git a/crates/common/src/cache/swap/messages.rs b/crates/common/src/cache/swap/messages.rs new file mode 100644 index 00000000..82b9cea2 --- /dev/null +++ b/crates/common/src/cache/swap/messages.rs @@ -0,0 +1,328 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{SwapPart, frame::SwapFrame}; +use crate::{ + CACHE_CHUNK, ColBlock, CustomKeywords, MessageUid, MessagesCache, + cache::email::{IdIndex, ThreadRange}, +}; +use compact_str::CompactString; +use rkyv::{ + boxed::ArchivedBox, + rend::unaligned::{i32_ule, u32_ule, u64_ule}, + with::InlineAsBox, +}; +use std::{mem::size_of, sync::Arc}; + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedBlock<'x> { + #[rkyv(with = InlineAsBox)] + pub document_ids: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub change_ids: &'x [u64], + #[rkyv(with = InlineAsBox)] + pub received_at: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub sent_at: &'x [i32], + #[rkyv(with = InlineAsBox)] + pub sizes: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub keywords: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub thread_ids: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub mb_offsets: &'x [u32], + #[rkyv(with = InlineAsBox)] + pub mb_arena: &'x [u64], +} + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedIndexChunk<'x> { + #[rkyv(with = InlineAsBox)] + pub data: &'x [u32], + pub len: u32, +} + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedMessages<'x> { + pub change_id: u64, + pub len: u32, + #[rkyv(with = InlineAsBox)] + pub starts: &'x [u32], + pub blocks: Vec>, + pub index: Vec>, + pub custom_ids: Vec, + pub custom_offsets: Vec, + #[rkyv(with = rkyv::with::Map)] + pub custom_names: Vec<&'x str>, +} + +impl MessagesCache { + pub fn to_snapshot(&self) -> Option> { + let arena = self.mailbox_arena(); + Self::seal_snapshot( + &self.pack(&arena), + self.change_id, + self.len as u32, + self.snapshot_len(), + ) + } + + fn snapshot_len(&self) -> usize { + let blocks = self + .blocks + .iter() + .map(|block| { + size_of::() + + block.document_ids.len() * (size_of::() * 6 + size_of::()) + + block.mb_offsets.len() * size_of::() + + block.mb_arena.len() * size_of::() + }) + .sum::(); + let custom = self + .keywords + .iter() + .map(|entry| { + size_of::() * 2 + + entry + .names + .iter() + .map(|name| size_of::>() + name.len()) + .sum::() + }) + .sum::(); + + size_of::() + + blocks + + self.starts.len() * size_of::() + + self + .index + .chunk_parts() + .map(|(data, _)| { + size_of::() + data.len() * size_of::() + }) + .sum::() + + size_of::() + + custom + } + + fn mailbox_arena(&self) -> Vec { + let mut arena = + Vec::with_capacity(self.blocks.iter().map(|block| block.mb_arena.len()).sum()); + for block in &self.blocks { + arena.extend(block.mb_arena.iter().map(MessageUid::pack)); + } + arena + } + + fn pack<'x>(&'x self, arena: &'x [u64]) -> ArchivedMessages<'x> { + let mut blocks = Vec::with_capacity(self.blocks.len()); + let mut rest = arena; + for block in &self.blocks { + let (mb_arena, tail) = rest + .split_at_checked(block.mb_arena.len()) + .unwrap_or((rest, &[])); + rest = tail; + blocks.push(ArchivedBlock { + document_ids: &block.document_ids, + change_ids: &block.change_ids, + received_at: &block.received_at, + sent_at: &block.sent_at, + sizes: &block.sizes, + keywords: &block.keywords, + thread_ids: &block.thread_ids, + mb_offsets: &block.mb_offsets, + mb_arena, + }); + } + + let index = self + .index + .chunk_parts() + .map(|(data, len)| ArchivedIndexChunk { data, len }) + .collect(); + + let mut custom_ids = Vec::with_capacity(self.keywords.len()); + let mut custom_offsets = Vec::with_capacity(self.keywords.len() + 1); + let mut custom_names = Vec::new(); + custom_offsets.push(0u32); + for entry in self.keywords.iter() { + custom_ids.push(entry.document_id); + for name in entry.names.iter() { + custom_names.push(name.as_str()); + } + custom_offsets.push(custom_names.len() as u32); + } + + ArchivedMessages { + change_id: self.change_id, + len: self.len as u32, + starts: &self.starts, + blocks, + index, + custom_ids, + custom_offsets, + custom_names, + } + } + + fn seal_snapshot( + archived: &ArchivedMessages<'_>, + change_id: u64, + count: u32, + payload_len: usize, + ) -> Option> { + let mut out = SwapFrame::reserve_header(); + out.reserve_exact(payload_len); + let mut out = rkyv::api::high::to_bytes_in::<_, rkyv::rancor::Error>(archived, out).ok()?; + + SwapFrame::seal(&mut out, SwapPart::Messages, change_id, count).then_some(out) + } + + pub fn from_snapshot(buf: &[u8]) -> Option { + let frame = SwapFrame::parse(buf)?; + if frame.part() != SwapPart::Messages { + return None; + } + + let archived = + rkyv::access::(frame.payload()).ok()?; + let len = archived.len.to_native() as usize; + let starts = archived + .starts + .iter() + .map(|start| start.to_native()) + .collect::>(); + if starts.len() != archived.blocks.len() { + return None; + } + + let mut blocks = Vec::with_capacity(archived.blocks.len()); + let mut total = 0usize; + for (block, start) in archived.blocks.iter().zip(&starts) { + let count = block.document_ids.len(); + if count == 0 || count > CACHE_CHUNK * 2 { + return None; + } + if block.change_ids.len() != count + || block.received_at.len() != count + || block.sent_at.len() != count + || block.sizes.len() != count + || block.keywords.len() != count + || block.thread_ids.len() != count + || block.mb_offsets.len() != count + 1 + { + return None; + } + if block.mb_offsets[0].to_native() != 0 + || block.mb_offsets[count].to_native() as usize != block.mb_arena.len() + || !block + .mb_offsets + .iter() + .map(|offset| offset.to_native()) + .is_sorted() + { + return None; + } + if *start as usize != total { + return None; + } + + total += count; + let thread_ids = copy_u32(&block.thread_ids); + blocks.push(Arc::new(ColBlock { + document_ids: copy_u32(&block.document_ids), + change_ids: copy_u64(&block.change_ids), + received_at: copy_u32(&block.received_at), + sent_at: copy_i32(&block.sent_at), + sizes: copy_u32(&block.sizes), + keywords: copy_u32(&block.keywords), + thread_range: ThreadRange::of(&thread_ids), + thread_ids, + mb_offsets: copy_u32(&block.mb_offsets), + mb_arena: copy_uid(&block.mb_arena), + })); + } + if total != len { + return None; + } + + let index = IdIndex::decode( + archived + .index + .iter() + .map(|chunk| (&*chunk.data, chunk.len.to_native() as usize)), + |value| value.to_native(), + len, + )?; + + let custom_count = archived.custom_ids.len(); + if archived.custom_offsets.len() != custom_count + 1 { + return None; + } + let mut keywords = Vec::with_capacity(custom_count); + for (document_id, range) in archived + .custom_ids + .iter() + .zip(archived.custom_offsets.windows(2)) + { + let document_id = document_id.to_native(); + let from = range[0].to_native() as usize; + let to = range[1].to_native() as usize; + let names = archived.custom_names.get(from..to)?; + keywords.push(CustomKeywords { + names: names + .iter() + .map(|name| CompactString::from(&**name)) + .collect::>() + .into_boxed_slice(), + document_id, + }); + } + + Some(MessagesCache::from_parts( + archived.change_id.to_native(), + blocks, + starts, + index, + keywords.into(), + len, + )) + } +} + +impl MessageUid { + fn pack(&self) -> u64 { + ((self.uid as u64) << 32) | self.mailbox_id as u64 + } + + fn unpack(packed: u64) -> Self { + MessageUid { + mailbox_id: packed as u32, + uid: (packed >> 32) as u32, + } + } +} + +fn copy_u32(src: &[u32_ule]) -> Arc<[u32]> { + src.iter().map(|value| value.to_native()).collect() +} + +fn copy_i32(src: &[i32_ule]) -> Arc<[i32]> { + src.iter().map(|value| value.to_native()).collect() +} + +fn copy_u64(src: &[u64_ule]) -> Arc<[u64]> { + src.iter().map(|value| value.to_native()).collect() +} + +fn copy_uid(src: &[u64_ule]) -> Arc<[MessageUid]> { + src.iter() + .map(|value| MessageUid::unpack(value.to_native())) + .collect() +} + + diff --git a/crates/common/src/cache/swap/mod.rs b/crates/common/src/cache/swap/mod.rs new file mode 100644 index 00000000..64016937 --- /dev/null +++ b/crates/common/src/cache/swap/mod.rs @@ -0,0 +1,527 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{GroupwareResources, Inner, MessageStoreCache}; +use blob::BlobSwapStore; +use evict::SwapEviction; +use file::FileSwapStore; +#[cfg(feature = "redis")] +use redis::RedisSwapStore; +use registry::schema::{prelude::ObjectType, structs}; +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +#[cfg(feature = "redis")] +use store::InMemoryStore; +use store::{BlobStore, registry::bootstrap::Bootstrap}; +use tokio::sync::{mpsc, oneshot}; +use types::collection::SyncCollection; +use writer::{QUEUE_DEPTH, RefreshSlots, Snapshot, SwapChanges, SwapSignal, SwapTarget}; + +pub mod blob; +pub mod evict; +pub mod file; +pub mod frame; +pub mod messages; +pub mod pointer; +#[cfg(feature = "redis")] +pub mod redis; +pub mod resources; +pub mod writer; + +pub const BLOCKING_CODEC_THRESHOLD: usize = 4 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SwapPart { + Messages, + Resources, +} + +impl SwapPart { + pub fn code(&self) -> u8 { + match self { + SwapPart::Messages => 1, + SwapPart::Resources => 3, + } + } + + pub fn from_code(code: u8) -> Option { + match code { + 1 => Some(SwapPart::Messages), + 3 => Some(SwapPart::Resources), + _ => None, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + SwapPart::Messages => "messages", + SwapPart::Resources => "resources", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SwapKey { + pub account_id: u32, + pub collection: SyncCollection, + pub part: SwapPart, +} + +impl SwapKey { + pub fn new(account_id: u32, collection: SyncCollection, part: SwapPart) -> Self { + SwapKey { + account_id, + collection, + part, + } + } + + pub fn messages(account_id: u32) -> Self { + SwapKey::new(account_id, SyncCollection::Email, SwapPart::Messages) + } + + pub fn file_name(&self) -> String { + format!( + "{:08x}-{}-{}.swap", + self.account_id, + u8::from(self.collection), + self.part.code() + ) + } + + pub fn resource_parts(account_id: u32) -> impl Iterator { + [ + SyncCollection::Calendar, + SyncCollection::AddressBook, + SyncCollection::FileNode, + SyncCollection::CalendarEventNotification, + ] + .into_iter() + .map(move |collection| SwapKey::new(account_id, collection, SwapPart::Resources)) + } + + pub fn all_parts(account_id: u32) -> impl Iterator { + std::iter::once(SwapKey::messages(account_id)).chain(SwapKey::resource_parts(account_id)) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SwapCadence { + pub flush_changes: u32, + pub flush_interval: Duration, + pub min_account_size: u64, + pub max_account_size: u64, +} + +impl Default for SwapCadence { + fn default() -> Self { + SwapCadence { + flush_changes: 2000, + flush_interval: Duration::from_secs(600), + min_account_size: 1024 * 1024, + max_account_size: 1024 * 1024 * 1024, + } + } +} + +pub enum SwapBackend { + Disabled, + File(FileSwapStore), + Blob(BlobSwapStore), + #[cfg(feature = "redis")] + Redis(RedisSwapStore), +} + +pub struct SwapTier { + backend: SwapBackend, + cadence: SwapCadence, + handoff: Arc, + writes: AtomicU64, + dirty_targets: AtomicU64, + #[cfg(stalwart_tests)] + reads: AtomicU64, +} + +pub struct SwapHandoff { + writer: mpsc::Sender, + refresh: RefreshSlots, +} + +pub type SwapReceiver = mpsc::Receiver; + +impl SwapHandoff { + fn signal(&self, signal: SwapSignal) { + let _ = self.writer.try_send(signal); + } + + fn refresh(&self, target: SwapTarget, changes: SwapChanges, snapshot: Snapshot) { + self.refresh.store(target, snapshot); + self.signal(SwapSignal::Changed(target, changes)); + } + + fn evicted(&self, target: SwapTarget, snapshot: Snapshot) { + self.refresh.store(target, snapshot); + self.signal(SwapSignal::Evicted(target)); + } +} + +impl SwapTier { + pub async fn build( + bp: &mut Bootstrap, + config: structs::CacheSwap, + storage: &crate::config::storage::Storage, + ) -> (Self, SwapReceiver) { + #[allow(unreachable_patterns)] + let (backend, cadence) = match config { + structs::CacheSwap::Disabled => return SwapTier::disabled(), + structs::CacheSwap::LocalFile(config) => { + let cadence = SwapCadence { + flush_changes: config.flush_changes as u32, + flush_interval: config.flush_interval.into_inner(), + min_account_size: config.min_account_size, + max_account_size: config.max_account_size, + }; + match FileSwapStore::open(&config).await { + Ok(store) => (SwapBackend::File(store), cadence), + Err(err) => { + bp.build_error(ObjectType::Cache.singleton(), err); + return SwapTier::disabled(); + } + } + } + structs::CacheSwap::BlobStore(config) => { + let cadence = SwapCadence { + flush_changes: config.flush_changes as u32, + flush_interval: config.flush_interval.into_inner(), + min_account_size: config.min_account_size, + max_account_size: config.max_account_size, + }; + let retention = config.retention.into_inner().as_secs(); + match BlobStore::build_cache_swap(bp, config.store, &storage.blob).await { + Ok(store) => ( + SwapBackend::Blob(BlobSwapStore::new( + store, + storage.data.clone(), + retention, + cadence.max_account_size, + )), + cadence, + ), + Err(err) => { + bp.build_error(ObjectType::Cache.singleton(), err); + return SwapTier::disabled(); + } + } + } + #[cfg(feature = "redis")] + structs::CacheSwap::Redis(config) => { + let cadence = SwapCadence { + flush_changes: config.flush_changes as u32, + flush_interval: config.flush_interval.into_inner(), + min_account_size: config.min_account_size, + max_account_size: config.max_account_size, + }; + let chunk_size = config.chunk_size; + let retention = config.retention.into_inner().as_secs(); + match InMemoryStore::build_cache_swap(bp, config.store, &storage.memory).await { + Ok(store) => ( + SwapBackend::Redis(RedisSwapStore::new( + store, + chunk_size, + retention, + cadence.max_account_size, + )), + cadence, + ), + Err(err) => { + bp.build_error(ObjectType::Cache.singleton(), err); + return SwapTier::disabled(); + } + } + } + _ => { + bp.build_error( + ObjectType::Cache.singleton(), + "Binary was not compiled with the selected cache swap backend".to_string(), + ); + return SwapTier::disabled(); + } + }; + + SwapTier::new(backend, cadence) + } + + pub fn start(inner: &Arc, receiver: SwapReceiver) { + if inner.cache.swap.is_enabled() { + SwapTier::spawn_writer(inner.clone(), receiver); + } + } + + pub fn eviction(&self, collection: SyncCollection) -> SwapEviction { + SwapEviction::new(self.is_enabled().then(|| self.handoff.clone()), collection) + } + + pub fn notify_changed( + &self, + account_id: u32, + collection: SyncCollection, + changes: SwapChanges, + ) { + if self.is_enabled() { + self.handoff.signal(SwapSignal::Changed( + SwapTarget::new(account_id, collection), + changes, + )); + } + } + + pub fn notify_refresh_messages( + &self, + account_id: u32, + changes: SwapChanges, + cache: &Arc, + ) { + if self.is_enabled() { + self.handoff.refresh( + SwapTarget::messages(account_id), + changes, + Snapshot::Messages(cache.clone()), + ); + } + } + + pub fn notify_refresh_resources( + &self, + account_id: u32, + collection: SyncCollection, + changes: SwapChanges, + cache: &Arc, + ) { + if self.is_enabled() { + self.handoff.refresh( + SwapTarget::new(account_id, collection), + changes, + Snapshot::Resources(cache.clone()), + ); + } + } + + pub(crate) fn notify_evicted_messages(&self, account_id: u32, cache: Arc) { + if self.is_enabled() { + self.handoff + .evicted(SwapTarget::messages(account_id), Snapshot::Messages(cache)); + } + } + + pub(crate) fn notify_evicted_resources( + &self, + target: SwapTarget, + cache: Arc, + ) { + if self.is_enabled() { + self.handoff.evicted(target, Snapshot::Resources(cache)); + } + } + + pub fn pending_refresh_snapshots(&self) -> usize { + self.handoff.refresh.len() + } + + pub(crate) fn refresh_slots(&self) -> &RefreshSlots { + &self.handoff.refresh + } + + pub fn forget(&self, account_id: u32) { + if self.is_enabled() { + self.handoff.signal(SwapSignal::Forget(account_id)); + } + } + + pub(crate) fn record_write(&self) { + self.writes.fetch_add(1, Ordering::Relaxed); + } + + pub fn pending_targets(&self) -> u64 { + self.dirty_targets.load(Ordering::Relaxed) + } + + fn track_dirty(&self, was_dirty: bool, is_dirty: bool) { + match (was_dirty, is_dirty) { + (false, true) => { + self.dirty_targets.fetch_add(1, Ordering::Relaxed); + } + (true, false) => self.untrack_dirty(1), + _ => {} + } + } + + fn untrack_dirty(&self, count: u64) { + if count > 0 { + self.dirty_targets.fetch_sub(count, Ordering::Relaxed); + } + } + + pub fn snapshots_written(&self) -> u64 { + self.writes.load(Ordering::Relaxed) + } + + pub async fn flush(&self) { + if !self.is_enabled() { + return; + } + let (ack, ack_rx) = oneshot::channel(); + if self + .handoff + .writer + .send(SwapSignal::Flush(ack)) + .await + .is_ok() + { + let _ = ack_rx.await; + } + } + + pub async fn stop(&self) { + if !self.is_enabled() { + return; + } + let (ack, ack_rx) = oneshot::channel(); + if self + .handoff + .writer + .send(SwapSignal::Stop(ack)) + .await + .is_err() + { + return; + } + + let timeout = self.shutdown_timeout(); + match tokio::time::timeout(timeout, ack_rx).await { + Ok(Ok(0)) | Ok(Err(_)) => (), + Ok(Ok(unwritten)) => { + trc::event!( + Cache(trc::CacheEvent::SwapError), + Total = unwritten, + Details = "Shut down with pending cache snapshots whose write failed", + ); + } + Err(_) => { + trc::event!( + Cache(trc::CacheEvent::SwapError), + Elapsed = timeout, + Details = "Timed out flushing the pending cache snapshots on shutdown", + ); + } + } + } + + fn shutdown_timeout(&self) -> Duration { + match &self.backend { + SwapBackend::Blob(_) => Duration::from_secs(60), + _ => Duration::from_secs(10), + } + } + + pub fn new(backend: SwapBackend, cadence: SwapCadence) -> (Self, SwapReceiver) { + let (writer, receiver) = mpsc::channel(QUEUE_DEPTH); + ( + SwapTier { + backend, + cadence, + handoff: Arc::new(SwapHandoff { + writer, + refresh: RefreshSlots::default(), + }), + writes: AtomicU64::new(0), + dirty_targets: AtomicU64::new(0), + #[cfg(stalwart_tests)] + reads: AtomicU64::new(0), + }, + receiver, + ) + } + + pub fn disabled() -> (Self, SwapReceiver) { + SwapTier::new(SwapBackend::Disabled, SwapCadence::default()) + } + + pub fn is_enabled(&self) -> bool { + !matches!(self.backend, SwapBackend::Disabled) + } + + pub fn cadence(&self) -> &SwapCadence { + &self.cadence + } + + pub async fn load(&self, key: SwapKey) -> trc::Result>> { + let snapshot = match &self.backend { + SwapBackend::Disabled => Ok(None), + SwapBackend::File(store) => store.load(key).await, + SwapBackend::Blob(store) => store.load(key).await, + #[cfg(feature = "redis")] + SwapBackend::Redis(store) => Box::pin(store.load(key)).await, + }; + self.record_read(&snapshot); + snapshot + } + + #[cfg(not(stalwart_tests))] + fn record_read(&self, _: &trc::Result>>) {} + + pub async fn store(&self, key: SwapKey, data: &[u8]) -> trc::Result<()> { + match &self.backend { + SwapBackend::Disabled => Ok(()), + SwapBackend::File(store) => store.store(key, data).await, + SwapBackend::Blob(store) => store.store(key, data).await, + #[cfg(feature = "redis")] + SwapBackend::Redis(store) => Box::pin(store.store(key, data)).await, + } + } + + pub async fn remove(&self, key: SwapKey) -> trc::Result<()> { + match &self.backend { + SwapBackend::Disabled => Ok(()), + SwapBackend::File(store) => store.remove(key).await, + SwapBackend::Blob(store) => store.remove(key).await, + #[cfg(feature = "redis")] + SwapBackend::Redis(store) => Box::pin(store.remove(key)).await, + } + } + + pub async fn remove_account(&self, account_id: u32) { + self.remove_keys(account_id, SwapKey::all_parts(account_id)) + .await; + } + + pub async fn remove_resources(&self, account_id: u32) { + self.remove_keys(account_id, SwapKey::resource_parts(account_id)) + .await; + } + + async fn remove_keys(&self, account_id: u32, keys: impl Iterator) { + if !self.is_enabled() { + return; + } + + for key in keys { + if let Err(err) = self.remove(key).await { + trc::error!( + err.details("Failed to remove cache snapshot") + .ctx(trc::Key::AccountId, account_id) + .ctx(trc::Key::Type, key.part.as_str()) + ); + } + } + } +} + + diff --git a/crates/common/src/cache/swap/pointer.rs b/crates/common/src/cache/swap/pointer.rs new file mode 100644 index 00000000..6966a5dc --- /dev/null +++ b/crates/common/src/cache/swap/pointer.rs @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::ops::Range; +use xxhash_rust::xxh3::xxh3_64; + +const MAGIC: u32 = 0x5357_5031; +const VERSION: u16 = 1; + +pub const POINTER_LEN: usize = 48; +pub const MAX_CALL_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkLayout { + size: u32, + count: u32, +} + +pub struct ChunkAssembler { + data: Vec, + length: usize, + chunk_size: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SwapPointer { + write_id: u64, + generation: u64, + length: u64, + chunks: Option, +} + +impl ChunkLayout { + pub fn new(length: u64, size: u32) -> Option { + if size == 0 { + return None; + } + Some(ChunkLayout { + size, + count: u32::try_from(length.div_ceil(u64::from(size))).ok()?, + }) + } + + pub fn size(&self) -> u32 { + self.size + } + + pub fn count(&self) -> u32 { + self.count + } + + pub fn chunks_per_batch(&self, max_bytes: usize) -> u32 { + u32::try_from(max_bytes / self.size as usize) + .unwrap_or(u32::MAX) + .max(1) + } + + pub fn batches(&self, max_bytes: usize) -> impl Iterator> + use<> { + index_batches(self.count, self.chunks_per_batch(max_bytes)) + } + + pub fn split<'x>( + &self, + data: &'x [u8], + max_bytes: usize, + ) -> impl Iterator, &'x [u8])> + use<'x> { + let batch_len = self.size as usize * self.chunks_per_batch(max_bytes) as usize; + self.batches(max_bytes).zip(data.chunks(batch_len)) + } +} + +impl ChunkAssembler { + pub fn new(layout: ChunkLayout, length: usize) -> Self { + ChunkAssembler { + data: Vec::with_capacity(length), + length, + chunk_size: layout.size as usize, + } + } + + pub fn push(&mut self, chunk: &[u8]) -> bool { + let expected = self.chunk_size.min(self.length - self.data.len()); + if expected > 0 && chunk.len() == expected { + self.data.extend_from_slice(chunk); + true + } else { + false + } + } + + pub fn finish(self) -> Option> { + (self.data.len() == self.length).then_some(self.data) + } +} + +pub fn index_batches(count: u32, per_batch: u32) -> impl Iterator> { + let per_batch = per_batch.max(1); + (0..count) + .step_by(per_batch as usize) + .map(move |start| start..start.saturating_add(per_batch).min(count)) +} + +impl SwapPointer { + pub fn new(write_id: u64, generation: u64, length: u64, chunks: Option) -> Self { + SwapPointer { + write_id, + generation, + length, + chunks, + } + } + + pub fn generation(&self) -> u64 { + self.generation + } + + pub fn length(&self) -> u64 { + self.length + } + + pub fn chunks(&self) -> Option { + self.chunks + } + + pub fn encode(&self) -> Vec { + let (chunk_size, chunk_count) = self + .chunks + .map_or((0, 0), |chunks| (chunks.size, chunks.count)); + let mut out = Vec::with_capacity(POINTER_LEN); + out.extend_from_slice(&MAGIC.to_le_bytes()); + out.extend_from_slice(&VERSION.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&self.write_id.to_le_bytes()); + out.extend_from_slice(&self.generation.to_le_bytes()); + out.extend_from_slice(&self.length.to_le_bytes()); + out.extend_from_slice(&chunk_size.to_le_bytes()); + out.extend_from_slice(&chunk_count.to_le_bytes()); + let seal = xxh3_64(&out); + out.extend_from_slice(&seal.to_le_bytes()); + out + } + + pub fn parse(buf: &[u8]) -> Option { + if buf.len() != POINTER_LEN { + return None; + } + let (sealed, seal) = buf.split_last_chunk::<8>()?; + if xxh3_64(sealed) != u64::from_le_bytes(*seal) { + return None; + } + + let (magic, rest) = sealed.split_first_chunk::<4>()?; + let (version, rest) = rest.split_first_chunk::<2>()?; + let (reserved, rest) = rest.split_first_chunk::<2>()?; + let (write_id, rest) = rest.split_first_chunk::<8>()?; + let (generation, rest) = rest.split_first_chunk::<8>()?; + let (length, rest) = rest.split_first_chunk::<8>()?; + let (chunk_size, rest) = rest.split_first_chunk::<4>()?; + let (chunk_count, rest) = rest.split_first_chunk::<4>()?; + + if !rest.is_empty() + || u32::from_le_bytes(*magic) != MAGIC + || u16::from_le_bytes(*version) != VERSION + || u16::from_le_bytes(*reserved) != 0 + { + return None; + } + + let length = u64::from_le_bytes(*length); + let chunk_count = u32::from_le_bytes(*chunk_count); + let chunks = match u32::from_le_bytes(*chunk_size) { + 0 if chunk_count == 0 => None, + 0 => return None, + size => Some(ChunkLayout::new(length, size).filter(|c| c.count == chunk_count)?), + }; + + Some(SwapPointer { + write_id: u64::from_le_bytes(*write_id), + generation: u64::from_le_bytes(*generation), + length, + chunks, + }) + } +} + diff --git a/crates/common/src/cache/swap/redis.rs b/crates/common/src/cache/swap/redis.rs new file mode 100644 index 00000000..d6acc3fe --- /dev/null +++ b/crates/common/src/cache/swap/redis.rs @@ -0,0 +1,244 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + SwapKey, + pointer::{ChunkAssembler, ChunkLayout, MAX_CALL_BYTES, SwapPointer, index_batches}, +}; +use store::{InMemoryStore, backend::redis::RedisStore, rand}; +use trc::AddContext; + +const LOAD_ATTEMPTS: usize = 2; +const MIN_CHUNK_SIZE: u64 = 4096; +const MAX_KEYS_PER_CALL: u32 = 1024; + +pub struct RedisSwapStore { + store: InMemoryStore, + chunk_size: u32, + retention: u64, + max_size: usize, +} + +struct RedisKeys { + base: String, +} + +impl RedisSwapStore { + pub fn new(store: InMemoryStore, chunk_size: u64, retention: u64, max_size: u64) -> Self { + RedisSwapStore { + store, + chunk_size: u32::try_from(chunk_size.max(MIN_CHUNK_SIZE)).unwrap_or(u32::MAX), + retention: retention.max(1), + max_size: max_size as usize, + } + } + + pub async fn load(&self, key: SwapKey) -> trc::Result>> { + let keys = RedisKeys::new(key); + let redis = self.redis(keys.route())?; + let mut previous = None; + + for _ in 0..LOAD_ATTEMPTS { + let Some(pointer) = read_pointer(redis, &keys).await? else { + return Ok(None); + }; + if previous == Some(pointer) { + break; + } + + let Some((layout, length)) = pointer.chunks().zip( + usize::try_from(pointer.length()) + .ok() + .filter(|length| *length <= self.max_size), + ) else { + return Ok(None); + }; + + if let Some(data) = read_generation(redis, &keys, &pointer, layout, length).await? { + return Ok(Some(data)); + } + + previous = Some(pointer); + } + + trc::event!( + Cache(trc::CacheEvent::SwapMiss), + AccountId = key.account_id, + Collection = key.collection.as_str(), + Details = "Cache snapshot generation in Redis is missing or does not match its pointer", + ); + + Ok(None) + } + + pub async fn store(&self, key: SwapKey, data: &[u8]) -> trc::Result<()> { + let keys = RedisKeys::new(key); + let redis = self.redis(keys.route())?; + let layout = ChunkLayout::new(data.len() as u64, self.chunk_size).ok_or_else(|| { + trc::EventType::Cache(trc::CacheEvent::SwapError) + .into_err() + .details("Cache snapshot has too many chunks for the Redis swap tier") + .ctx(trc::Key::Size, data.len()) + })?; + let generation = rand::random(); + let pointer = SwapPointer::new(rand::random(), generation, data.len() as u64, Some(layout)); + let expires = Some(self.retention); + + let mut items = Vec::with_capacity(layout.chunks_per_batch(MAX_CALL_BYTES) as usize); + for (range, payload) in layout.split(data, MAX_CALL_BYTES) { + items.clear(); + items.extend( + range + .zip(payload.chunks(layout.size() as usize)) + .map(|(index, chunk)| (keys.chunk(generation, index), chunk)), + ); + redis + .key_set_many(&items, expires) + .await + .caused_by(trc::location!())?; + } + + let previous = read_pointer(redis, &keys).await?; + redis + .key_set_many(&[(keys.pointer(), pointer.encode().as_slice())], expires) + .await + .caused_by(trc::location!())?; + + if let Some(previous) = previous.filter(|previous| previous.generation() != generation) + && let Err(err) = unlink_generation(redis, &keys, &previous).await + { + trc::error!( + err.details("Failed to remove a superseded cache snapshot generation") + .ctx(trc::Key::AccountId, key.account_id) + .ctx(trc::Key::Collection, key.collection.as_str()) + ); + } + + Ok(()) + } + + pub async fn remove(&self, key: SwapKey) -> trc::Result<()> { + let keys = RedisKeys::new(key); + let redis = self.redis(keys.route())?; + let previous = read_pointer(redis, &keys).await?; + + redis + .key_unlink_many(&[keys.pointer()]) + .await + .caused_by(trc::location!())?; + + match previous { + Some(previous) => unlink_generation(redis, &keys, &previous).await, + None => Ok(()), + } + } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + fn redis(&self, route: &[u8]) -> trc::Result<&RedisStore> { + match &self.store { + InMemoryStore::Redis(store) => Ok(store), + _ => Err(unsupported()), + } + } +} + +async fn read_pointer(redis: &RedisStore, keys: &RedisKeys) -> trc::Result> { + redis + .key_get_many(&[keys.pointer()]) + .await + .caused_by(trc::location!()) + .map(|values| { + values + .into_iter() + .next() + .flatten() + .as_deref() + .and_then(SwapPointer::parse) + }) +} + +async fn read_generation( + redis: &RedisStore, + keys: &RedisKeys, + pointer: &SwapPointer, + layout: ChunkLayout, + length: usize, +) -> trc::Result>> { + let mut assembler = ChunkAssembler::new(layout, length); + let mut batch = Vec::with_capacity(layout.chunks_per_batch(MAX_CALL_BYTES) as usize); + + for range in layout.batches(MAX_CALL_BYTES) { + batch.clear(); + batch.extend(range.map(|index| keys.chunk(pointer.generation(), index))); + + for chunk in redis + .key_get_many(&batch) + .await + .caused_by(trc::location!())? + { + match chunk { + Some(chunk) if assembler.push(&chunk) => (), + _ => return Ok(None), + } + } + } + + Ok(assembler.finish()) +} + +async fn unlink_generation( + redis: &RedisStore, + keys: &RedisKeys, + pointer: &SwapPointer, +) -> trc::Result<()> { + let Some(layout) = pointer.chunks() else { + return Ok(()); + }; + + let mut batch = Vec::with_capacity(layout.count().min(MAX_KEYS_PER_CALL) as usize); + for range in index_batches(layout.count(), MAX_KEYS_PER_CALL) { + batch.clear(); + batch.extend(range.map(|index| keys.chunk(pointer.generation(), index))); + redis + .key_unlink_many(&batch) + .await + .caused_by(trc::location!())?; + } + + Ok(()) +} + +impl RedisKeys { + fn new(key: SwapKey) -> Self { + RedisKeys { + base: format!( + "{{swap:{}:{}:{}}}", + key.account_id, + u8::from(key.collection), + key.part.code() + ), + } + } + + fn route(&self) -> &[u8] { + self.base.as_bytes() + } + + fn pointer(&self) -> Vec { + format!("{}:p", self.base).into_bytes() + } + + fn chunk(&self, generation: u64, index: u32) -> Vec { + format!("{}:{generation:x}:{index}", self.base).into_bytes() + } +} + +fn unsupported() -> trc::Error { + trc::EventType::Cache(trc::CacheEvent::SwapError) + .into_err() + .details("The cache swap tier requires a Redis in-memory store") +} + diff --git a/crates/common/src/cache/swap/resources.rs b/crates/common/src/cache/swap/resources.rs new file mode 100644 index 00000000..5a629813 --- /dev/null +++ b/crates/common/src/cache/swap/resources.rs @@ -0,0 +1,788 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{SwapPart, frame::SwapFrame}; +use crate::{ + ArenaRef, CachedName, DavPath, FileFlags, GroupwareResource, GroupwareResourceMetadata, + GroupwareResources, PathChunk, PathIndex, ResourceChunk, ResourceStore, + TinyCalendarPreferences, UpdateLock, storage::dav::CONTAINER_FLAG, +}; +use calcard::common::timezone::Tz; +use rkyv::with::InlineAsBox; +use std::sync::{Arc, OnceLock}; +use types::{acl::AclGrant, metadata::MetadataKinds}; +use utils::map::bitmap::Bitmap; + +const KIND_FILE: u16 = 1; +const KIND_CALENDAR: u16 = 2; +const KIND_CALENDAR_EVENT: u16 = 3; +const KIND_CALENDAR_EVENT_NOTIFICATION: u16 = 4; +const KIND_ADDRESS_BOOK: u16 = 5; +const KIND_CONTACT_CARD: u16 = 6; + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct FlatResource { + pub ref_a_off: u32, + pub ref_b_off: u32, + pub v0: u32, + pub v1: u32, + pub v2: u32, + pub v3: u32, + pub v4: u32, + pub etag: u32, + pub document_id: u32, + pub ref_a_len: u16, + pub ref_b_len: u16, + pub flags: u16, +} + +struct ChunkTimes { + epoch: i64, + wide: Vec, +} + +impl ChunkTimes { + fn for_records(records: &[GroupwareResource]) -> Self { + Self { + epoch: records + .iter() + .flat_map(|record| record.data.absolute_times()) + .min() + .unwrap_or_default(), + wide: Vec::new(), + } + } + + fn encode(&mut self, time: i64) -> u32 { + match time.checked_sub(self.epoch) { + Some(delta) if (0..=i32::MAX as i64).contains(&delta) => delta as u32, + _ => { + self.wide.push(time); + -(self.wide.len() as i32) as u32 + } + } + } + + fn decode(&self, raw: u32) -> Option { + let delta = raw as i32; + if delta >= 0 { + self.epoch.checked_add(delta as i64) + } else { + self.wide.get((-(delta as i64) - 1) as usize).copied() + } + } +} + +impl GroupwareResourceMetadata { + fn absolute_times(&self) -> impl Iterator { + let (start, created_at) = match self { + GroupwareResourceMetadata::File { modified, .. } => (Some(*modified), None), + GroupwareResourceMetadata::CalendarEvent { + start, created_at, .. + } => (Some(*start), Some(*created_at)), + GroupwareResourceMetadata::CalendarEventNotification { created_at, .. } + | GroupwareResourceMetadata::ContactCard { created_at, .. } => { + (Some(*created_at), None) + } + GroupwareResourceMetadata::Calendar { .. } + | GroupwareResourceMetadata::AddressBook { .. } => (None, None), + }; + start.into_iter().chain(created_at) + } + + fn kind(&self) -> u16 { + match self { + GroupwareResourceMetadata::File { .. } => KIND_FILE, + GroupwareResourceMetadata::Calendar { .. } => KIND_CALENDAR, + GroupwareResourceMetadata::CalendarEvent { .. } => KIND_CALENDAR_EVENT, + GroupwareResourceMetadata::CalendarEventNotification { .. } => { + KIND_CALENDAR_EVENT_NOTIFICATION + } + GroupwareResourceMetadata::AddressBook { .. } => KIND_ADDRESS_BOOK, + GroupwareResourceMetadata::ContactCard { .. } => KIND_CONTACT_CARD, + } + } +} + +impl FlatResource { + fn pack(resource: &GroupwareResource, times: &mut ChunkTimes) -> Option { + let mut flat = FlatResource { + ref_a_off: 0, + ref_b_off: 0, + v0: 0, + v1: 0, + v2: 0, + v3: 0, + v4: 0, + etag: resource.etag(), + document_id: resource.document_id, + ref_a_len: 0, + ref_b_len: 0, + flags: 0, + }; + + match &resource.data { + GroupwareResourceMetadata::File { + name, + size, + parent_id, + acls, + modified, + created_delta, + flags, + .. + } => { + flat.set_ref_a(name)?; + flat.set_ref_b(acls)?; + flat.v0 = *size; + flat.v1 = *parent_id; + flat.v2 = times.encode(*modified); + flat.v3 = *created_delta as u32; + flat.v4 = flags.0; + } + GroupwareResourceMetadata::Calendar { + name, + acls, + preferences, + metadata, + .. + } => { + flat.flags = u16::from(metadata.bits()); + flat.set_ref_a(name)?; + flat.set_ref_b(acls)?; + flat.v0 = preferences.off; + flat.v1 = preferences.len; + } + GroupwareResourceMetadata::CalendarEvent { + names, + start, + duration, + created_at, + modified_at, + uid, + flags, + .. + } => { + flat.flags = *flags; + flat.set_ref_a(names)?; + flat.set_ref_b(uid)?; + flat.v0 = times.encode(*start); + flat.v1 = times.encode(*created_at); + flat.v2 = *duration; + flat.v3 = *modified_at as u32; + } + GroupwareResourceMetadata::CalendarEventNotification { + names, + created_at, + event_id, + changed_by, + principals, + calendar_ids_len, + flags, + .. + } => { + flat.flags = *flags; + flat.set_ref_a(names)?; + flat.set_ref_b(principals)?; + flat.v0 = times.encode(*created_at); + flat.v1 = *event_id; + flat.v2 = *changed_by; + flat.v3 = *calendar_ids_len as u32; + } + GroupwareResourceMetadata::AddressBook { + name, + acls, + metadata, + .. + } => { + flat.flags = u16::from(metadata.bits()); + flat.set_ref_a(name)?; + flat.set_ref_b(acls)?; + } + GroupwareResourceMetadata::ContactCard { + names, + created_at, + modified_at, + uid, + flags, + .. + } => { + flat.flags = *flags; + flat.set_ref_a(names)?; + flat.set_ref_b(uid)?; + flat.v0 = times.encode(*created_at); + flat.v1 = *modified_at as u32; + } + } + + Some(flat) + } + + fn set_ref_a(&mut self, arena: &ArenaRef) -> Option<()> { + self.ref_a_off = arena.off; + self.ref_a_len = u16::try_from(arena.len).ok()?; + Some(()) + } + + fn set_ref_b(&mut self, arena: &ArenaRef) -> Option<()> { + self.ref_b_off = arena.off; + self.ref_b_len = u16::try_from(arena.len).ok()?; + Some(()) + } +} + +impl ArchivedFlatResource { + fn ref_a(&self) -> ArenaRef { + ArenaRef { + off: self.ref_a_off.to_native(), + len: self.ref_a_len.to_native() as u32, + } + } + + fn ref_b(&self) -> ArenaRef { + ArenaRef { + off: self.ref_b_off.to_native(), + len: self.ref_b_len.to_native() as u32, + } + } + + fn metadata_kinds(&self) -> Option { + MetadataKinds::from_bits(u8::try_from(self.flags.to_native()).ok()?) + } + + fn unpack(&self, kind: u16, times: &ChunkTimes) -> Option { + let data = match kind { + KIND_FILE => GroupwareResourceMetadata::File { + name: self.ref_a(), + size: self.v0.to_native(), + parent_id: self.v1.to_native(), + acls: self.ref_b(), + etag: self.etag.to_native(), + modified: times.decode(self.v2.to_native())?, + created_delta: self.v3.to_native() as i32, + flags: FileFlags(self.v4.to_native()), + }, + KIND_CALENDAR => GroupwareResourceMetadata::Calendar { + name: self.ref_a(), + acls: self.ref_b(), + preferences: ArenaRef { + off: self.v0.to_native(), + len: self.v1.to_native(), + }, + etag: self.etag.to_native(), + metadata: self.metadata_kinds()?, + }, + KIND_CALENDAR_EVENT => GroupwareResourceMetadata::CalendarEvent { + names: self.ref_a(), + start: times.decode(self.v0.to_native())?, + duration: self.v2.to_native(), + created_at: times.decode(self.v1.to_native())?, + modified_at: self.v3.to_native() as i32, + uid: self.ref_b(), + etag: self.etag.to_native(), + flags: self.flags.to_native(), + }, + KIND_CALENDAR_EVENT_NOTIFICATION => { + GroupwareResourceMetadata::CalendarEventNotification { + names: self.ref_a(), + created_at: times.decode(self.v0.to_native())?, + event_id: self.v1.to_native(), + etag: self.etag.to_native(), + changed_by: self.v2.to_native(), + principals: self.ref_b(), + calendar_ids_len: u16::try_from(self.v3.to_native()).ok()?, + flags: self.flags.to_native(), + } + } + KIND_ADDRESS_BOOK => GroupwareResourceMetadata::AddressBook { + name: self.ref_a(), + acls: self.ref_b(), + etag: self.etag.to_native(), + metadata: self.metadata_kinds()?, + }, + KIND_CONTACT_CARD => GroupwareResourceMetadata::ContactCard { + names: self.ref_a(), + created_at: times.decode(self.v0.to_native())?, + modified_at: self.v1.to_native() as i32, + uid: self.ref_b(), + etag: self.etag.to_native(), + flags: self.flags.to_native(), + }, + _ => return None, + }; + + Some(GroupwareResource { + document_id: self.document_id.to_native(), + data, + }) + } +} + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedResources<'x> { + pub base_path: String, + pub item_change_id: u64, + pub container_change_id: u64, + pub containers_end: u32, + pub total: u32, + pub path_total: u32, + pub unified_id_space: bool, + pub chunks: Vec>, + pub path_chunks: Vec>, +} + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedResourceChunk<'x> { + pub records: Vec, + #[rkyv(with = InlineAsBox)] + pub bytes: &'x [u8], + pub name_offsets: Vec, + pub name_lengths: Vec, + pub name_parents: Vec, + pub acl_accounts: Vec, + pub acl_grants: Vec, + pub pref_accounts: Vec, + pub pref_timezones: Vec, + pub pref_flags: Vec, + #[rkyv(with = InlineAsBox)] + pub principals: &'x [u32], + pub kind: u16, + pub epoch: i64, + pub wide_times: Vec, +} + +#[derive(rkyv::Archive, rkyv::Serialize)] +pub struct ArchivedPathChunk<'x> { + #[rkyv(with = InlineAsBox)] + pub bytes: &'x [u8], + pub path_lengths: Vec, + pub hierarchy_seqs: Vec, + pub parent_ids: Vec, + pub document_ids: Vec, +} + +const WIRE_CONTAINER_FLAG: u16 = 1 << 15; + +fn fits_within(arena: ArenaRef, limit: usize) -> bool { + (arena.off as usize) + .checked_add(arena.len as usize) + .is_some_and(|end| end <= limit) +} + +impl GroupwareResource { + fn fits_within( + &self, + bytes: usize, + names: usize, + acls: usize, + prefs: usize, + principals_len: usize, + ) -> bool { + match &self.data { + GroupwareResourceMetadata::File { + name, + acls: acl_ref, + flags, + .. + } => { + flags.is_valid() + && fits_within( + ArenaRef { + off: name.off, + len: name.len.saturating_add(flags.extra_len() as u32), + }, + bytes, + ) + && fits_within(*acl_ref, acls) + } + GroupwareResourceMetadata::AddressBook { + name, + acls: acl_ref, + .. + } => fits_within(*name, bytes) && fits_within(*acl_ref, acls), + GroupwareResourceMetadata::Calendar { + name, + acls: acl_ref, + preferences, + .. + } => { + fits_within(*name, bytes) + && fits_within(*acl_ref, acls) + && fits_within(*preferences, prefs) + } + GroupwareResourceMetadata::CalendarEvent { + names: name_refs, + uid, + .. + } + | GroupwareResourceMetadata::ContactCard { + names: name_refs, + uid, + .. + } => fits_within(*name_refs, names) && fits_within(*uid, bytes), + GroupwareResourceMetadata::CalendarEventNotification { + names: name_refs, + principals, + calendar_ids_len, + .. + } => { + fits_within(*name_refs, names) + && fits_within(*principals, principals_len) + && *calendar_ids_len as u32 <= principals.len + } + } + } +} + +impl GroupwareResources { + pub fn to_snapshot(&self) -> Option> { + let (chunks, path_chunks) = self.pack()?; + self.seal_snapshot(chunks, path_chunks) + } + + #[allow(clippy::type_complexity)] + fn pack(&self) -> Option<(Vec>, Vec>)> { + let mut chunks = Vec::with_capacity(self.resources.chunks.len()); + for chunk in &self.resources.chunks { + let mut name_offsets = Vec::with_capacity(chunk.names.len()); + let mut name_lengths = Vec::with_capacity(chunk.names.len()); + let mut name_parents = Vec::with_capacity(chunk.names.len()); + for name in chunk.names.iter() { + name_offsets.push(name.name.off); + name_lengths.push(u16::try_from(name.name.len).ok()?); + name_parents.push(name.parent_id); + } + + let mut acl_accounts = Vec::with_capacity(chunk.acls.len()); + let mut acl_grants = Vec::with_capacity(chunk.acls.len()); + for acl in chunk.acls.iter() { + acl_accounts.push(acl.account_id); + acl_grants.push(acl.grants.bitmap); + } + + let mut pref_accounts = Vec::with_capacity(chunk.prefs.len()); + let mut pref_timezones = Vec::with_capacity(chunk.prefs.len()); + let mut pref_flags = Vec::with_capacity(chunk.prefs.len()); + for pref in chunk.prefs.iter() { + pref_accounts.push(pref.account_id); + pref_timezones.push(pref.tz.as_id()); + pref_flags.push(pref.flags); + } + + let kind = chunk.records.first().map_or(0, |record| record.data.kind()); + if chunk + .records + .iter() + .any(|record| record.data.kind() != kind) + { + return None; + } + + let mut times = ChunkTimes::for_records(&chunk.records); + let mut records = Vec::with_capacity(chunk.records.len()); + for record in chunk.records.iter() { + records.push(FlatResource::pack(record, &mut times)?); + } + + chunks.push(ArchivedResourceChunk { + records, + bytes: &chunk.bytes, + name_offsets, + name_lengths, + name_parents, + acl_accounts, + acl_grants, + pref_accounts, + pref_timezones, + pref_flags, + principals: &chunk.principals, + kind, + epoch: times.epoch, + wide_times: times.wide, + }); + } + + let mut path_chunks = Vec::with_capacity(self.paths.chunks.len()); + for chunk in &self.paths.chunks { + let mut path_lengths = Vec::with_capacity(chunk.paths.len()); + let mut parent_ids = Vec::with_capacity(chunk.paths.len()); + let mut hierarchy_seqs = Vec::with_capacity(chunk.paths.len()); + let mut document_ids = Vec::with_capacity(chunk.paths.len()); + let mut next_off = 0u32; + for path in chunk.paths.iter() { + if path.path.off != next_off { + return None; + } + next_off += path.path.len; + path_lengths.push(u16::try_from(path.path.len).ok()?); + parent_ids.push(path.parent_id); + let depth = u16::try_from(path.hierarchy_seq & !CONTAINER_FLAG).ok()?; + if depth >= WIRE_CONTAINER_FLAG { + return None; + } + hierarchy_seqs.push(if path.hierarchy_seq & CONTAINER_FLAG != 0 { + depth | WIRE_CONTAINER_FLAG + } else { + depth + }); + document_ids.push(path.document_id); + } + if next_off as usize != chunk.bytes.len() { + return None; + } + + path_chunks.push(ArchivedPathChunk { + bytes: &chunk.bytes, + path_lengths, + hierarchy_seqs, + parent_ids, + document_ids, + }); + } + + Some((chunks, path_chunks)) + } + + fn seal_snapshot( + &self, + chunks: Vec>, + path_chunks: Vec>, + ) -> Option> { + let mut out = rkyv::api::high::to_bytes_in::<_, rkyv::rancor::Error>( + &ArchivedResources { + base_path: self.base_path.clone(), + item_change_id: self.item_change_id, + container_change_id: self.container_change_id, + containers_end: self.resources.containers_end as u32, + total: self.resources.total as u32, + path_total: self.paths.total as u32, + unified_id_space: self.resources.unified_id_space, + chunks, + path_chunks, + }, + SwapFrame::reserve_header(), + ) + .ok()?; + + SwapFrame::seal( + &mut out, + SwapPart::Resources, + self.highest_change_id, + self.resources.total as u32, + ) + .then_some(out) + } + + pub fn from_snapshot(buf: &[u8]) -> Option { + let frame = SwapFrame::parse(buf)?; + if frame.part() != SwapPart::Resources { + return None; + } + + let archived = + rkyv::access::(frame.payload()).ok()?; + + let containers_end = archived.containers_end.to_native() as usize; + let unified_id_space = archived.unified_id_space; + if containers_end > archived.chunks.len() + || (unified_id_space && containers_end != archived.chunks.len()) + { + return None; + } + + let mut chunks = Vec::with_capacity(archived.chunks.len()); + let mut total = 0usize; + for chunk in archived.chunks.iter() { + let names_len = chunk.name_offsets.len(); + if chunk.name_lengths.len() != names_len || chunk.name_parents.len() != names_len { + return None; + } + let acls_len = chunk.acl_accounts.len(); + if chunk.acl_grants.len() != acls_len { + return None; + } + let prefs_len = chunk.pref_accounts.len(); + if chunk.pref_timezones.len() != prefs_len || chunk.pref_flags.len() != prefs_len { + return None; + } + + let bytes: Box<[u8]> = (&*chunk.bytes).into(); + + let names: Box<[CachedName]> = chunk + .name_offsets + .iter() + .zip(chunk.name_lengths.iter()) + .zip(chunk.name_parents.iter()) + .map(|((off, len), parent_id)| CachedName { + name: ArenaRef { + off: off.to_native(), + len: len.to_native() as u32, + }, + parent_id: parent_id.to_native(), + }) + .collect(); + if names + .iter() + .any(|name| !fits_within(name.name, bytes.len())) + { + return None; + } + + let acls: Box<[AclGrant]> = chunk + .acl_accounts + .iter() + .zip(chunk.acl_grants.iter()) + .map(|(account_id, grants)| AclGrant { + account_id: account_id.to_native(), + grants: Bitmap::from(grants.to_native()), + }) + .collect(); + + let principals: Box<[u32]> = chunk + .principals + .iter() + .map(|principal| principal.to_native()) + .collect(); + + let prefs: Box<[TinyCalendarPreferences]> = chunk + .pref_accounts + .iter() + .zip(chunk.pref_timezones.iter()) + .zip(chunk.pref_flags.iter()) + .map(|((account_id, tz), flags)| TinyCalendarPreferences { + account_id: account_id.to_native(), + tz: Tz::from_id(tz.to_native()).unwrap_or_default(), + flags: flags.to_native(), + }) + .collect(); + + let times = ChunkTimes { + epoch: chunk.epoch.to_native(), + wide: chunk + .wide_times + .iter() + .map(|time| time.to_native()) + .collect(), + }; + let kind = chunk.kind.to_native(); + + let mut records = Vec::with_capacity(chunk.records.len()); + for record in chunk.records.iter() { + let record = record.unpack(kind, ×)?; + if !record.fits_within( + bytes.len(), + names_len, + acls_len, + prefs_len, + principals.len(), + ) { + return None; + } + records.push(record); + } + + let (Some(min_id), Some(max_id)) = ( + records.first().map(|r| r.document_id), + records.last().map(|r| r.document_id), + ) else { + return None; + }; + + total += records.len(); + chunks.push(Arc::new(ResourceChunk { + uid_index: OnceLock::new(), + records: records.into_boxed_slice(), + bytes, + names, + acls, + prefs, + principals, + min_id, + max_id, + })); + } + + if total != archived.total.to_native() as usize { + return None; + } + + let mut path_chunks = Vec::with_capacity(archived.path_chunks.len()); + let mut path_total = 0usize; + for chunk in archived.path_chunks.iter() { + let paths_len = chunk.path_lengths.len(); + if chunk.parent_ids.len() != paths_len + || chunk.hierarchy_seqs.len() != paths_len + || chunk.document_ids.len() != paths_len + { + return None; + } + + let bytes: Box<[u8]> = (&*chunk.bytes).into(); + let mut off = 0u32; + let paths: Box<[DavPath]> = chunk + .path_lengths + .iter() + .zip(chunk.parent_ids.iter()) + .zip(chunk.hierarchy_seqs.iter()) + .zip(chunk.document_ids.iter()) + .map(|(((len, parent_id), hierarchy_seq), document_id)| { + let len = len.to_native() as u32; + let path = ArenaRef { off, len }; + off += len; + let hierarchy_seq = hierarchy_seq.to_native(); + DavPath { + path, + parent_id: parent_id.to_native(), + hierarchy_seq: (hierarchy_seq & !WIRE_CONTAINER_FLAG) as u32 + | if hierarchy_seq & WIRE_CONTAINER_FLAG != 0 { + CONTAINER_FLAG + } else { + 0 + }, + document_id: document_id.to_native(), + } + }) + .collect(); + if paths.is_empty() || off as usize != bytes.len() { + return None; + } + + path_total += paths_len; + path_chunks.push(Arc::new(PathChunk { paths, bytes })); + } + + if path_total != archived.path_total.to_native() as usize { + return None; + } + + let mut resources = GroupwareResources { + base_path: archived.base_path.to_string(), + paths: Arc::new(PathIndex { + chunks: path_chunks, + total: path_total, + }), + resources: ResourceStore { + chunks, + containers_end, + total, + unified_id_space, + }, + item_change_id: archived.item_change_id.to_native(), + container_change_id: archived.container_change_id.to_native(), + highest_change_id: frame.change_id(), + size: 0, + update_lock: Arc::new(UpdateLock::new()), + verification: Default::default(), + }; + resources.recompute_size(); + resources + .update_lock + .set_revision(resources.highest_change_id); + + Some(resources) + } +} + diff --git a/crates/common/src/cache/swap/writer.rs b/crates/common/src/cache/swap/writer.rs new file mode 100644 index 00000000..be884a69 --- /dev/null +++ b/crates/common/src/cache/swap/writer.rs @@ -0,0 +1,611 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BLOCKING_CODEC_THRESHOLD, SwapCadence, SwapKey, SwapPart, SwapReceiver, SwapTier}; +use crate::{GroupwareResources, Inner, MessageStoreCache}; +use ahash::AHashMap; +use parking_lot::Mutex; +use std::{ + collections::hash_map::Entry, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::sync::oneshot; +use types::collection::SyncCollection; + +pub const QUEUE_DEPTH: usize = 1024; +const METRICS_INTERVAL: Duration = Duration::from_secs(300); +const ENTRIES_PER_FLUSH_CHANGE: u32 = 16; + +pub enum SwapSignal { + Changed(SwapTarget, SwapChanges), + Evicted(SwapTarget), + Forget(u32), + Flush(oneshot::Sender<()>), + Stop(oneshot::Sender), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SwapChanges { + pub changes: u32, + pub base_change_id: Option, + pub change_id: u64, + pub entries: u32, +} + +impl SwapChanges { + pub fn rebuilt(change_id: u64, entries: usize) -> Self { + let entries = u32::try_from(entries).unwrap_or(u32::MAX); + SwapChanges { + changes: entries, + base_change_id: None, + change_id, + entries, + } + } + + pub fn applied(changes: usize, base_change_id: u64, change_id: u64, entries: usize) -> Self { + SwapChanges { + changes: u32::try_from(changes).unwrap_or(u32::MAX), + base_change_id: Some(base_change_id), + change_id, + entries: u32::try_from(entries).unwrap_or(u32::MAX), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SwapTarget { + pub account_id: u32, + pub collection: SyncCollection, +} + +impl SwapTarget { + pub fn new(account_id: u32, collection: SyncCollection) -> Self { + SwapTarget { + account_id, + collection, + } + } + + pub fn messages(account_id: u32) -> Self { + SwapTarget::new(account_id, SyncCollection::Email) + } + + fn key(&self) -> SwapKey { + SwapKey::new( + self.account_id, + self.collection, + if matches!(self.collection, SyncCollection::Email) { + SwapPart::Messages + } else { + SwapPart::Resources + }, + ) + } +} + +#[derive(Clone)] +pub enum Snapshot { + Messages(Arc), + Resources(Arc), +} + +impl Snapshot { + pub fn size(&self) -> u64 { + match self { + Snapshot::Messages(cache) => cache.size, + Snapshot::Resources(cache) => cache.size, + } + } + + fn change_id(&self) -> u64 { + match self { + Snapshot::Messages(cache) => cache.emails.change_id, + Snapshot::Resources(cache) => cache.highest_change_id, + } + } + + fn len(&self) -> usize { + match self { + Snapshot::Messages(cache) => cache.emails.len(), + Snapshot::Resources(cache) => cache.resources.len(), + } + } + + fn encode(&self) -> Option> { + match self { + Snapshot::Messages(cache) => cache.emails.to_snapshot(), + Snapshot::Resources(cache) => cache.to_snapshot(), + } + } + + async fn encode_off_runtime(&self) -> Option> { + if self.size() < BLOCKING_CODEC_THRESHOLD as u64 { + return self.encode(); + } + + let snapshot = self.clone(); + tokio::task::spawn_blocking(move || snapshot.encode()) + .await + .ok() + .flatten() + } +} + +struct RefreshSlot { + snapshot: Snapshot, + stored_at: Instant, +} + +#[derive(Default)] +pub struct RefreshSlots(Mutex>); + +impl RefreshSlots { + pub fn store(&self, target: SwapTarget, snapshot: Snapshot) { + let mut slots = self.0.lock(); + let displaced = match slots.entry(target) { + Entry::Occupied(mut entry) => { + if entry.get().snapshot.change_id() > snapshot.change_id() { + snapshot + } else { + std::mem::replace( + entry.get_mut(), + RefreshSlot { + snapshot, + stored_at: Instant::now(), + }, + ) + .snapshot + } + } + Entry::Vacant(entry) => { + entry.insert(RefreshSlot { + snapshot, + stored_at: Instant::now(), + }); + return; + } + }; + drop(slots); + drop(displaced); + } + + fn take(&self, target: SwapTarget) -> Option { + self.0.lock().remove(&target).map(|slot| slot.snapshot) + } + + fn forget(&self, account_id: u32) { + let removed = self + .0 + .lock() + .extract_if(|target, _| target.account_id == account_id) + .collect::>(); + drop(removed); + } + + fn drain(&self) -> Vec<(SwapTarget, Snapshot)> { + let mut slots = self.0.lock(); + let drained = slots + .drain() + .map(|(target, slot)| (target, slot.snapshot)) + .collect(); + slots.shrink_to_fit(); + drained + } + + fn prune(&self, max_age: Duration) { + let mut slots = self.0.lock(); + let removed = slots + .extract_if(|_, slot| slot.stored_at.elapsed() >= max_age) + .collect::>(); + slots.shrink_to_fit(); + drop(slots); + drop(removed); + } + + pub fn len(&self) -> usize { + self.0.lock().len() + } + + pub fn is_empty(&self) -> bool { + self.0.lock().is_empty() + } +} + +#[derive(Default)] +struct PendingAccount { + changes: u32, + entries: u32, + counted_through: Option, + last_write: Option, + persisted_change_id: Option, +} + +impl PendingAccount { + fn record(&mut self, changes: SwapChanges) { + self.entries = changes.entries; + self.changes = match changes.base_change_id { + Some(base) if self.counted_through.is_none_or(|counted| base >= counted) => { + self.changes.saturating_add(changes.changes) + } + _ => self.changes.max(changes.changes), + }; + self.counted_through = Some( + self.counted_through + .map_or(changes.change_id, |counted| counted.max(changes.change_id)), + ); + } + + fn threshold(&self, cadence: &SwapCadence) -> u32 { + cadence + .flush_changes + .max(self.entries / ENTRIES_PER_FLUSH_CHANGE) + .max(1) + } + + fn is_due(&self, cadence: &SwapCadence) -> bool { + self.changes >= self.threshold(cadence) + && self + .last_write + .is_none_or(|last| last.elapsed() >= cadence.flush_interval) + } + + fn is_due_on_eviction(&self, cadence: &SwapCadence) -> bool { + self.changes >= cadence.flush_changes.max(1) + } + + fn age(&self) -> Option { + self.last_write.map(|last| last.elapsed()) + } + + fn mark_clean(&mut self, tier: &SwapTier) { + tier.track_dirty(self.changes > 0, false); + self.changes = 0; + } +} + +struct SwapWriter { + inner: Arc, + pending: AHashMap, + bytes_written: u64, + snapshots_written: u64, + window_started: Instant, +} + +impl SwapWriter { + fn tier(&self) -> &SwapTier { + &self.inner.cache.swap + } + + fn cadence(&self) -> SwapCadence { + *self.tier().cadence() + } + + fn record_changes(&mut self, target: SwapTarget, changes: SwapChanges) { + let entry = self.pending.entry(target).or_default(); + let was_dirty = entry.changes > 0; + entry.record(changes); + self.inner + .cache + .swap + .track_dirty(was_dirty, entry.changes > 0); + } + + fn remove_pending(&mut self, target: SwapTarget) { + if let Some(entry) = self.pending.remove(&target) { + self.inner.cache.swap.track_dirty(entry.changes > 0, false); + } + } + + fn is_due(&self, target: SwapTarget) -> bool { + let cadence = self.cadence(); + self.pending + .get(&target) + .is_some_and(|entry| entry.is_due(&cadence)) + } + + fn forget(&mut self, account_id: u32) { + let mut dirty_removed = 0; + self.pending.retain(|target, entry| { + let keep = target.account_id != account_id; + if !keep && entry.changes > 0 { + dirty_removed += 1; + } + keep + }); + self.tier().untrack_dirty(dirty_removed); + self.tier().refresh_slots().forget(account_id); + } + + fn unwritten(&self) -> usize { + self.pending + .values() + .filter(|entry| entry.changes > 0) + .count() + } + + fn peek(&self, target: SwapTarget) -> Option { + let caches = &self.inner.cache; + match target.collection { + SyncCollection::Email => caches + .peek_messages(target.account_id) + .map(Snapshot::Messages), + collection => caches + .peek_resources(target.account_id, collection) + .map(Snapshot::Resources), + } + } + + fn take_snapshot(&self, target: SwapTarget) -> Option { + let slot = self.tier().refresh_slots().take(target); + self.peek(target).or(slot) + } + + fn mark_absent(&mut self, target: SwapTarget) { + match self.pending.get_mut(&target) { + Some(entry) if entry.last_write.is_some() => entry.mark_clean(&self.inner.cache.swap), + _ => self.remove_pending(target), + } + } + + async fn flush_now(&mut self, target: SwapTarget) { + match self.take_snapshot(target) { + Some(snapshot) => self.write_if_changed(target, &snapshot).await, + None => self.mark_absent(target), + } + } + + async fn flush_if_due(&mut self, target: SwapTarget) { + if self.is_due(target) { + self.flush_now(target).await; + } + } + + async fn flush_evicted(&mut self, target: SwapTarget) { + let cadence = self.cadence(); + let is_due = self + .pending + .get(&target) + .is_some_and(|entry| entry.is_due_on_eviction(&cadence)); + if let Some(snapshot) = self.take_snapshot(target) + && is_due + { + self.write_if_changed(target, &snapshot).await; + } + } + + async fn flush_due(&mut self) { + let cadence = self.cadence(); + let due = self + .pending + .iter() + .filter(|(_, entry)| entry.is_due(&cadence)) + .map(|(target, _)| *target) + .collect::>(); + + for target in due { + self.flush_if_due(target).await; + } + } + + async fn flush_all(&mut self) -> u64 { + let written_before = self.snapshots_written; + let mut targets = self + .pending + .iter() + .filter(|(_, entry)| entry.changes > 0) + .map(|(target, entry)| (*target, entry.changes)) + .collect::>(); + targets.sort_unstable_by(|(_, left), (_, right)| right.cmp(left)); + + for (target, _) in targets { + self.flush_now(target).await; + } + + for (target, snapshot) in self.tier().refresh_slots().drain() { + let snapshot = self.peek(target).unwrap_or(snapshot); + self.write_if_changed(target, &snapshot).await; + } + + self.snapshots_written - written_before + } + + fn report_flush(&self, written: u64) { + trc::event!( + Cache(trc::CacheEvent::SwapFlush), + Total = written as usize, + Details = self.unwritten(), + ); + } + + async fn write_if_changed(&mut self, target: SwapTarget, snapshot: &Snapshot) { + if let Some(entry) = self.pending.get_mut(&target) + && entry.persisted_change_id == Some(snapshot.change_id()) + { + entry.mark_clean(&self.inner.cache.swap); + return; + } + + self.write(target, snapshot).await; + } + + async fn write(&mut self, target: SwapTarget, snapshot: &Snapshot) { + let cadence = self.cadence(); + let size = snapshot.size(); + if size > cadence.max_account_size { + trc::event!( + Cache(trc::CacheEvent::SwapError), + AccountId = target.account_id, + Collection = target.collection.as_str(), + Size = size, + Limit = cadence.max_account_size, + Details = "Cache snapshot exceeds the configured maximum size", + ); + let entry = self.pending.entry(target).or_default(); + entry.mark_clean(&self.inner.cache.swap); + entry.last_write = Some(Instant::now()); + return; + } + if size < cadence.min_account_size { + self.remove_pending(target); + return; + } + + let start_time = Instant::now(); + let Some(encoded) = snapshot.encode_off_runtime().await else { + trc::event!( + Cache(trc::CacheEvent::SwapError), + AccountId = target.account_id, + Collection = target.collection.as_str(), + Details = "Failed to encode the cache snapshot", + ); + self.remove_pending(target); + return; + }; + + let bytes = encoded.len() as u64; + match self.tier().store(target.key(), &encoded).await { + Ok(()) => { + let entry = self.pending.entry(target).or_default(); + entry.mark_clean(&self.inner.cache.swap); + entry.last_write = Some(Instant::now()); + entry.persisted_change_id = Some(snapshot.change_id()); + self.bytes_written += bytes; + self.snapshots_written += 1; + self.tier().record_write(); + + trc::event!( + Cache(trc::CacheEvent::SwapWrite), + AccountId = target.account_id, + Collection = target.collection.as_str(), + ChangeId = snapshot.change_id(), + Total = snapshot.len(), + Size = bytes, + Elapsed = start_time.elapsed(), + ); + } + Err(err) => { + trc::error!( + err.details("Failed to write the cache snapshot") + .ctx(trc::Key::AccountId, target.account_id) + .ctx(trc::Key::Collection, target.collection.as_str()) + ); + } + } + } + + fn prune(&mut self) { + let flush_interval = self.cadence().flush_interval; + self.pending.retain(|_, entry| { + entry.changes > 0 + || entry + .last_write + .is_some_and(|last| last.elapsed() < flush_interval) + }); + self.pending.shrink_to_fit(); + self.tier().refresh_slots().prune(flush_interval); + } + + fn pending_ages(&self) -> Vec { + let mut ages = self + .pending + .values() + .filter(|entry| entry.changes > 0) + .filter_map(PendingAccount::age) + .map(|age| age.as_secs()) + .collect::>(); + ages.sort_unstable(); + + let percentile = |fraction: f64| -> u64 { + if ages.is_empty() { + 0 + } else { + ages[((ages.len() as f64 * fraction) as usize).min(ages.len() - 1)] + } + }; + + vec![ + percentile(0.5).to_string(), + percentile(0.95).to_string(), + ages.last().copied().unwrap_or_default().to_string(), + ] + } + + fn report_metrics(&mut self) { + let elapsed = self.window_started.elapsed(); + if elapsed.is_zero() { + return; + } + + trc::event!( + Cache(trc::CacheEvent::SwapMetrics), + Total = vec![self.snapshots_written as usize, self.pending.len()], + Size = (self.bytes_written as f64 * 3600.0 / elapsed.as_secs_f64()) as u64, + Details = self.pending_ages(), + Elapsed = elapsed, + ); + + self.bytes_written = 0; + self.snapshots_written = 0; + self.window_started = Instant::now(); + } +} + +impl SwapTier { + pub fn spawn_writer(inner: Arc, mut rx: SwapReceiver) { + tokio::spawn(async move { + let mut writer = SwapWriter { + inner, + pending: AHashMap::new(), + bytes_written: 0, + snapshots_written: 0, + window_started: Instant::now(), + }; + let mut timer = tokio::time::interval(METRICS_INTERVAL); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + timer.tick().await; + + loop { + tokio::select! { + signal = rx.recv() => match signal { + Some(SwapSignal::Changed(target, changes)) => { + writer.record_changes(target, changes); + writer.flush_if_due(target).await; + } + Some(SwapSignal::Evicted(target)) => { + writer.flush_evicted(target).await; + } + Some(SwapSignal::Forget(account_id)) => { + writer.forget(account_id); + } + Some(SwapSignal::Flush(ack)) => { + let written = writer.flush_all().await; + writer.report_flush(written); + let _ = ack.send(()); + } + Some(SwapSignal::Stop(ack)) => { + let written = writer.flush_all().await; + writer.report_flush(written); + let _ = ack.send(writer.unwritten()); + break; + } + None => { + writer.flush_all().await; + break; + } + }, + _ = timer.tick() => { + writer.flush_due().await; + writer.report_metrics(); + writer.prune(); + } + } + } + }); + } +} + diff --git a/crates/common/src/config/groupware.rs b/crates/common/src/config/groupware.rs index 47c0356e..9e9a00f6 100644 --- a/crates/common/src/config/groupware.rs +++ b/crates/common/src/config/groupware.rs @@ -4,11 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::storage::dav::canonical_calcard_segment; use calcard::vcard::VCardVersion; use registry::schema::{ enums::VCardVersion as RegistryVCardVersion, structs::{ - AddressBook, Calendar, CalendarAlarm, CalendarScheduling, DataRetention, FileStorage, + AddressBook, Calendar, CalendarAlarm, CalendarScheduling, DataRetention, FileStorage, Rate, Sharing, SystemSettings, WebDav, }, }; @@ -20,7 +21,6 @@ use utils::template::Template; pub struct GroupwareConfig { // DAV settings pub max_request_size: usize, - pub dead_property_size: Option, pub live_property_size: usize, pub max_lock_timeout: u64, pub max_locks_per_user: usize, @@ -31,6 +31,11 @@ pub struct GroupwareConfig { pub max_ical_size: usize, pub max_ical_instances: usize, pub max_ical_attendees_per_instance: usize, + pub max_calendars_per_event: usize, + pub max_expanded_query_duration: u64, + pub max_availability_duration: u64, + pub availability_rate: Option, + pub max_attachments_size: usize, pub default_calendar_name: Option, pub default_calendar_display_name: Option, pub alarms_enabled: bool, @@ -50,7 +55,9 @@ pub struct GroupwareConfig { // Addressbook settings pub max_vcard_size: usize, + pub max_media_size: usize, pub vcard_version: VCardVersion, + pub max_address_books_per_card: usize, pub default_addressbook_name: Option, pub default_addressbook_display_name: Option, @@ -101,21 +108,38 @@ impl GroupwareConfig { let system = bp.setting_infallible::().await; GroupwareConfig { - max_request_size: dav.request_max_size as usize, - dead_property_size: dav.dead_property_max_size.map(|v| v as usize), - live_property_size: dav.live_property_max_size as usize, + max_request_size: dav.max_request_size as usize, + live_property_size: dav.max_live_property_size as usize, assisted_discovery: dav.enable_assisted_discovery, max_lock_timeout: dav.max_lock_timeout.into_inner().as_secs(), max_locks_per_user: dav.max_locks as usize, max_results: dav.max_results as usize, - default_calendar_name: calendar.default_href_name, - default_calendar_display_name: calendar.default_display_name, - default_addressbook_name: book.default_href_name, - default_addressbook_display_name: book.default_display_name, + default_calendar_name: calendar + .default_href_name + .as_deref() + .map(|name| canonical_calcard_segment(name).into_owned()), + default_calendar_display_name: calendar + .default_display_name + .or(calendar.default_href_name), + default_addressbook_name: book + .default_href_name + .as_deref() + .map(|name| canonical_calcard_segment(name).into_owned()), + default_addressbook_display_name: book.default_display_name.or(book.default_href_name), max_ical_size: calendar.max_i_calendar_size as usize, max_ical_instances: calendar.max_recurrence_expansions as usize, max_ical_attendees_per_instance: calendar.max_attendees as usize, + max_calendars_per_event: calendar.max_calendars_per_event as usize, + max_expanded_query_duration: calendar + .max_expanded_query_duration + .into_inner() + .as_secs(), + max_availability_duration: calendar.max_availability_duration.into_inner().as_secs(), + availability_rate: calendar.availability_rate_limit, + max_attachments_size: calendar.max_attachments_size as usize, max_vcard_size: book.max_v_card_size as usize, + max_media_size: book.max_media_size as usize, + max_address_books_per_card: book.max_address_books_per_card as usize, vcard_version: match book.v_card_version { RegistryVCardVersion::V3 => VCardVersion::V3_0, RegistryVCardVersion::V4 => VCardVersion::V4_0, @@ -125,7 +149,7 @@ impl GroupwareConfig { alarms_minimum_interval: alarm.min_trigger_interval.into_inner().as_secs() as i64, alarms_allow_external_recipients: alarm.allow_external_rcpts, alarms_from_name: alarm.from_name, - alarms_from_email: alarm.from_email, + alarms_from_email: alarm.from_address, alarms_template: Template::parse(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../resources/html-templates/calendar-alarm.html.min" @@ -133,7 +157,7 @@ impl GroupwareConfig { .expect("Failed to parse calendar template"), itip_enabled: sched.enable, itip_auto_add: sched.auto_add_invitations, - itip_inbound_max_ical_size: sched.itip_max_size as usize, + itip_inbound_max_ical_size: sched.max_itip_size as usize, itip_outbound_max_recipients: sched.max_recipients as usize, itip_inbox_auto_expunge: dr .expunge_scheduling_inbox_after @@ -195,89 +219,3 @@ impl FromStr for CalendarTemplateVariable { } } -#[cfg(test)] -mod tests { - use super::CalendarTemplateVariable; - use utils::template::Template; - - const TEMPLATES: [(&str, &str, &str); 2] = [ - ( - "calendar-invite.html", - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/calendar-invite.html" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/calendar-invite.html.min" - )), - ), - ( - "calendar-alarm.html", - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/calendar-alarm.html" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../resources/html-templates/calendar-alarm.html.min" - )), - ), - ]; - - // Every `{{...}}` token in a template, in order of appearance - fn tokens(contents: &str) -> Vec<&str> { - let mut tokens = Vec::new(); - let mut rest = contents; - - while let Some((_, after)) = rest.split_once("{{") { - match after.split_once("}}") { - Some((token, tail)) => { - tokens.push(token.trim()); - rest = tail; - } - None => break, - } - } - - tokens - } - - #[test] - fn shipped_calendar_templates_parse() { - for (name, source, minified) in TEMPLATES { - Template::::parse(source) - .unwrap_or_else(|err| panic!("{name} failed to parse: {err}")); - Template::::parse(minified) - .unwrap_or_else(|err| panic!("{name}.min failed to parse: {err}")); - } - } - - #[test] - fn minified_calendar_templates_are_in_sync() { - for (name, source, minified) in TEMPLATES { - let source = tokens(source); - assert!(source.len() > 10, "{name} yielded no tokens to compare"); - assert_eq!( - source, - tokens(minified), - "{name}.min is stale, re-run resources/scripts/minify_html.sh" - ); - } - } - - #[test] - fn calendar_template_tokens_are_single_line() { - // A newline inside `{{...}}` makes the parser reject the block - for (name, source, minified) in TEMPLATES { - for (suffix, contents) in [("", source), (".min", minified)] { - for token in tokens(contents) { - assert!( - !token.contains('\n') && !token.contains('\r'), - "{name}{suffix} has a multi-line token: {token:?}" - ); - } - } - } - } -} diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 2d6efe6a..a4c90510 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -6,11 +6,17 @@ use super::server::tls::build_self_signed_cert; use crate::{ - Caches, Data, DavResource, DavResources, MailboxCache, MessageStoreCache, MessageUidCache, - TlsConnectors, - auth::{AccessTokenInner, AccountCache, DomainCache, MailingListCache, RoleCache, TenantCache}, + Caches, Data, GroupwareResource, GroupwareResources, MailboxCache, MessageStoreCache, + MessageUid, TlsConnectors, + auth::{AccessTokenInner, AccountCache, DomainCache, MailingListCache, RoleCache}, + cache::{ + RevalidateInterval, + http_auth::{GrantRevocations, HttpAuthEntries}, + large::{LargeMessageCaches, LargeResourceCaches}, + swap::SwapReceiver, + }, config::{ - mailstore::spamfilter::SpamClassifier, + mailstore::spamfilter::{SpamClassifier, SpamLookup}, server::tls::parse_certificates, smtp::{ auth::DkimSigners, @@ -19,10 +25,11 @@ use crate::{ }, manager::application::WebApplications, network::security::BlockedIps, + storage::metadata::MetadataViewerCache, }; use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; -use mail_auth::{MX, Parameters, RecordSet, Txt}; +use mail_auth::{DnsCache, Mx, Negative, Parameters, RecordSet, TxtRecord}; use parking_lot::RwLock; use registry::schema::{prelude::ObjectType, structs}; use std::{ @@ -30,9 +37,10 @@ use std::{ sync::Arc, }; use store::{LookupStores, registry::bootstrap::Bootstrap}; +use types::collection::SyncCollection; use utils::{ UnwrapFailure, - cache::{Cache, CacheWithTtl}, + cache::{Cache, CacheWithTtl, SpillCache}, snowflake::{MAX_NODE_ID, SnowflakeIdGenerator}, tls::build_tls_connector, }; @@ -93,49 +101,75 @@ impl Data { applications, logos: Default::default(), smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"), + memory_budget: Default::default(), asn_geo_data: Default::default(), + password_verifier: Default::default(), } } } impl Caches { - pub async fn parse(bp: &mut Bootstrap) -> Self { + pub async fn parse( + bp: &mut Bootstrap, + storage: &crate::config::storage::Storage, + ) -> (Self, SwapReceiver) { let cache = bp.setting_infallible::().await; + let dav_estimate = (std::mem::size_of::() + + (500 * std::mem::size_of::())) as u64; + let (swap, swap_rx) = crate::cache::swap::SwapTier::build(bp, cache.swap, storage).await; - Caches { - access_tokens: Cache::new_single_shard( + let caches = Caches { + revalidate: RevalidateInterval::new(RevalidateInterval::DEFAULT), + access_tokens: SpillCache::new( cache.access_tokens, (std::mem::size_of::() + 255) as u64, - ), - http_auth: Cache::new(cache.http_auth, (50 + std::mem::size_of::()) as u64), - messages: Cache::new_single_shard( + ) + .with_name("accessTokens"), + http_auth: HttpAuthEntries::new(cache.http_auth), + grant_revocations: GrantRevocations::default(), + messages: Cache::new_single_shard_with( cache.messages, (std::mem::size_of::() + std::mem::size_of::>() - + (1024 * std::mem::size_of::()) + + (1024 * std::mem::size_of::()) + (15 * (std::mem::size_of::() + 60))) as u64, + swap.eviction(SyncCollection::Email), + ) + .with_name("messages"), + messages_large: LargeMessageCaches::new( + cache.messages_large, + cache.messages_large_ttl.into_inner(), ), - files: Cache::new_single_shard( + resources_large: LargeResourceCaches::new( + cache.groupware_large, + cache.groupware_large_ttl.into_inner(), + ), + files: Cache::new_single_shard_with( cache.files, - (std::mem::size_of::() + (500 * std::mem::size_of::())) - as u64, - ), - events: Cache::new_single_shard( + dav_estimate, + swap.eviction(SyncCollection::FileNode), + ) + .with_name("files"), + events: Cache::new_single_shard_with( cache.events, - (std::mem::size_of::() + (500 * std::mem::size_of::())) - as u64, - ), - contacts: Cache::new_single_shard( + dav_estimate, + swap.eviction(SyncCollection::Calendar), + ) + .with_name("events"), + contacts: Cache::new_single_shard_with( cache.contacts, - (std::mem::size_of::() + (500 * std::mem::size_of::())) - as u64, - ), - scheduling: Cache::new_single_shard( + dav_estimate, + swap.eviction(SyncCollection::AddressBook), + ) + .with_name("contacts"), + scheduling: Cache::new_single_shard_with( cache.scheduling, - (std::mem::size_of::() + (500 * std::mem::size_of::())) - as u64, - ), - emails: Cache::new(cache.email_addresses, 255u64), + dav_estimate, + swap.eviction(SyncCollection::CalendarEventNotification), + ) + .with_name("scheduling"), + metadata_viewers: MetadataViewerCache::new(cache.access_tokens / 4), + emails: Cache::new(cache.email_addresses, 255u64).with_name("emailAddresses"), emails_negative: CacheWithTtl::new( cache.email_addresses_negative, (std::mem::size_of::() + 255) as u64, @@ -143,7 +177,8 @@ impl Caches { domain_names: Cache::new( cache.domain_names, (std::mem::size_of::() + 255) as u64, - ), + ) + .with_name("domainNames"), domain_names_negative: CacheWithTtl::new( cache.domain_names_negative, (std::mem::size_of::() + 255) as u64, @@ -151,26 +186,30 @@ impl Caches { domains: Cache::new( cache.domains, (std::mem::size_of::() + 255) as u64, - ), + ) + .with_name("domains"), accounts: Cache::new( cache.accounts, (std::mem::size_of::() + 255) as u64, - ), - roles: Cache::new(cache.roles, (std::mem::size_of::() + 255) as u64), - tenants: Cache::new( - cache.tenants, - (std::mem::size_of::() + 255) as u64, - ), + ) + .with_name("accounts"), + roles: Cache::new(cache.roles, (std::mem::size_of::() + 255) as u64) + .with_name("roles"), lists: Cache::new( cache.mailing_lists, (std::mem::size_of::() + 255) as u64, - ), + ) + .with_name("mailingLists"), dkim_signers: Cache::new( cache.dkim_signatures, - (std::mem::size_of::() + 255) as u64, + (std::mem::size_of::() + 4096) as u64, + ) + .with_name("dkimSignatures"), + dns_txt: CacheWithTtl::new( + cache.dns_txt, + (std::mem::size_of::() + 512) as u64, ), - dns_txt: CacheWithTtl::new(cache.dns_txt, (std::mem::size_of::() + 255) as u64), - dns_mx: CacheWithTtl::new(cache.dns_mx, ((std::mem::size_of::() + 255) * 2) as u64), + dns_mx: CacheWithTtl::new(cache.dns_mx, ((std::mem::size_of::() + 255) * 2) as u64), dns_ptr: CacheWithTtl::new(cache.dns_ptr, (std::mem::size_of::() + 255) as u64), dns_ipv4: CacheWithTtl::new( cache.dns_ipv4, @@ -185,36 +224,50 @@ impl Caches { cache.dns_mta_sts, (std::mem::size_of::() + 255) as u64, ), - dns_rbl: CacheWithTtl::new( - cache.dns_rbl, - ((std::mem::size_of::() + 255) * 2) as u64, + spam_lookup: CacheWithTtl::new( + cache.spam_lookup, + (std::mem::size_of::() + 255 * 2) as u64, ), + directory_recipients: CacheWithTtl::new(cache.directory_recipients, 255u64), + directory_recipients_ttl: cache.directory_recipients_ttl.into_inner(), negative_cache_ttl: cache.negative_ttl.into_inner(), - } + swap, + }; + + (caches, swap_rx) } - #[allow(clippy::type_complexity)] #[inline(always)] - pub fn build_auth_parameters( - &self, - params: T, - ) -> Parameters< - '_, - T, - CacheWithTtl, Txt>, - CacheWithTtl, RecordSet>, - CacheWithTtl, RecordSet>, - CacheWithTtl, RecordSet>, - CacheWithTtl>>, - > { - Parameters { - params, - cache_txt: Some(&self.dns_txt), - cache_mx: Some(&self.dns_mx), - cache_ptr: Some(&self.dns_ptr), - cache_ipv4: Some(&self.dns_ipv4), - cache_ipv6: Some(&self.dns_ipv6), - } + pub fn build_auth_parameters(&self, params: T) -> Parameters<'_, T, Caches> { + Parameters::new(params).with_cache(self) + } +} + +impl DnsCache for Caches { + type Txt = CacheWithTtl, TxtRecord>; + type Mx = CacheWithTtl, Result, Negative>>; + type Ipv4 = CacheWithTtl, Result, Negative>>; + type Ipv6 = CacheWithTtl, Result, Negative>>; + type Ptr = CacheWithTtl>, Negative>>; + + fn txt(&self) -> Option<&Self::Txt> { + Some(&self.dns_txt) + } + + fn mx(&self) -> Option<&Self::Mx> { + Some(&self.dns_mx) + } + + fn ipv4(&self) -> Option<&Self::Ipv4> { + Some(&self.dns_ipv4) + } + + fn ipv6(&self) -> Option<&Self::Ipv6> { + Some(&self.dns_ipv6) + } + + fn ptr(&self) -> Option<&Self::Ptr> { + Some(&self.dns_ptr) } } @@ -233,7 +286,9 @@ impl Default for Data { applications: WebApplications::new(), logos: Default::default(), smtp_connectors: TlsConnectors::try_new().unwrap(), + memory_budget: Default::default(), asn_geo_data: Default::default(), + password_verifier: Default::default(), lookup_stores: Default::default(), } } diff --git a/crates/common/src/config/mailstore/capabilities.rs b/crates/common/src/config/mailstore/capabilities.rs index 525614d3..af8c7ddd 100644 --- a/crates/common/src/config/mailstore/capabilities.rs +++ b/crates/common/src/config/mailstore/capabilities.rs @@ -4,29 +4,42 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::config::mailstore::jmap::JmapConfig; +use crate::{ + config::{mailstore::jmap::JmapConfig, metadata::MetadataConfig}, + storage::dav::{ + FORBIDDEN_FILE_NAME_CHARS, FORBIDDEN_FILE_NODE_NAMES, MAX_DAV_FILE_NAME_LEN, + MAX_FILE_NODE_DEPTH, + }, +}; use ahash::AHashSet; use calcard::icalendar::ICalendarDuration; use jmap_proto::{ - object::{email::EmailComparator, file_node::FileNodeComparator}, + object::{ + email::EmailComparator, + file_node::{FileNodeComparator, FileNodeNodeType}, + }, request::capability::{ BlobCapabilities, CalendarCapabilities, Capabilities, Capability, ContactsCapabilities, - CoreCapabilities, EmptyCapabilities, FileNodeCapabilities, MailCapabilities, - PrincipalAvailabilityCapabilities, PrincipalCapabilities, SieveAccountCapabilities, - SieveSessionCapabilities, SubmissionCapabilities, WebPushCapabilities, + CoreCapabilities, DataTypeMetadataInfo, EmptyCapabilities, FileNodeCapabilities, + MailCapabilities, MetadataCapabilities, PrincipalAvailabilityCapabilities, + PrincipalCapabilities, SieveAccountCapabilities, SieveSessionCapabilities, + SubmissionCapabilities, WebPushCapabilities, }, types::date::UTCDate, }; use registry::{ - schema::structs::{Calendar, Email, SieveUserInterpreter}, + schema::structs::{AddressBook, Calendar, Email, SieveUserInterpreter}, types::EnumImpl, }; use store::registry::bootstrap::Bootstrap; -use types::type_state::DataType; +use types::{ + metadata::{NamespaceScope, REGISTERED_NAMESPACES}, + type_state::DataType, +}; use utils::map::vec_map::VecMap; impl JmapConfig { - pub async fn add_capabilities(&mut self, bp: &mut Bootstrap) { + pub async fn add_capabilities(&mut self, bp: &mut Bootstrap, metadata: &MetadataConfig) { // Add core capabilities self.capabilities.session.append( Capability::Core, @@ -55,7 +68,7 @@ impl JmapConfig { self.capabilities.account.insert( Capability::Mail, Capabilities::Mail(MailCapabilities { - max_mailboxes_per_email: None, + max_mailboxes_per_email: email.max_mailboxes_per_email.into(), max_mailbox_depth: email.max_mailbox_depth, max_size_mailbox_name: email.max_mailbox_name_length, max_size_attachments_per_email: email.max_attachment_size, @@ -75,6 +88,7 @@ impl JmapConfig { ); // Add calendar capabilities + let calendar = bp.setting_infallible::().await; self.capabilities.session.append( Capability::Calendars, Capabilities::Empty(EmptyCapabilities::default()), @@ -82,7 +96,7 @@ impl JmapConfig { self.capabilities.account.insert( Capability::Calendars, Capabilities::Calendar(CalendarCapabilities { - max_calendars_per_event: None, + max_calendars_per_event: calendar.max_calendars_per_event.into(), min_date_time: UTCDate { year: 1, month: 1, @@ -105,13 +119,11 @@ impl JmapConfig { tz_hour: 0, tz_minute: 0, }, - max_expanded_query_duration: ICalendarDuration::from_seconds(86400 * 365) - .to_string(), - max_participants_per_event: bp - .setting_infallible::() - .await - .max_attendees - .into(), + max_expanded_query_duration: ICalendarDuration::from_seconds( + calendar.max_expanded_query_duration.into_inner().as_secs() as i64, + ) + .to_string(), + max_participants_per_event: calendar.max_attendees.into(), may_create_calendar: true, }), ); @@ -126,6 +138,7 @@ impl JmapConfig { ); // Add contacts capabilities + let address_book = bp.setting_infallible::().await; self.capabilities.session.append( Capability::Contacts, Capabilities::Empty(EmptyCapabilities::default()), @@ -133,7 +146,7 @@ impl JmapConfig { self.capabilities.account.insert( Capability::Contacts, Capabilities::Contacts(ContactsCapabilities { - max_address_books_per_card: None, + max_address_books_per_card: address_book.max_address_books_per_card.into(), may_create_address_book: true, }), ); @@ -154,23 +167,28 @@ impl JmapConfig { self.capabilities.account.insert( Capability::FileNode, Capabilities::FileNode(FileNodeCapabilities { - max_file_node_depth: None, - max_size_file_node_name: 255, - forbidden_name_chars: Some("/<>:\"\\|?*".to_string()), + max_file_node_depth: Some(MAX_FILE_NODE_DEPTH as u64), + max_size_file_node_name: MAX_DAV_FILE_NAME_LEN as u64, + forbidden_name_chars: Some(FORBIDDEN_FILE_NAME_CHARS.to_string()), forbidden_node_names: Some( - [ - ".", "..", "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", - "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT0", "LPT1", "LPT2", - "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", - ] - .into_iter() - .map(str::to_string) - .collect(), + FORBIDDEN_FILE_NODE_NAMES + .iter() + .map(|name| name.to_string()) + .collect(), ), file_node_query_sort_options: vec![ FileNodeComparator::Name, FileNodeComparator::Size, + FileNodeComparator::Created, + FileNodeComparator::Modified, + FileNodeComparator::MediaType, FileNodeComparator::NodeType, + FileNodeComparator::Tree, + ], + supported_node_types: &[ + FileNodeNodeType::Directory, + FileNodeNodeType::File, + FileNodeNodeType::SymLink, ], may_create_top_level_file_node: true, case_insensitive_names: false, @@ -198,7 +216,10 @@ impl JmapConfig { self.capabilities.account.insert( Capability::PrincipalsAvailability, Capabilities::PrincipalsAvailability(PrincipalAvailabilityCapabilities { - max_availability_duration: ICalendarDuration::from_seconds(86400 * 365).to_string(), + max_availability_duration: ICalendarDuration::from_seconds( + calendar.max_availability_duration.into_inner().as_secs() as i64, + ) + .to_string(), }), ); @@ -241,6 +262,7 @@ impl JmapConfig { .collect::>(); let mut extensions = sieve::compiler::grammar::Capability::all() .iter() + .filter(|c| !matches!(c, sieve::compiler::grammar::Capability::ImapSieve)) .map(|c| c.to_string()) .filter(|c| !disabled_capabilities.contains(c.as_str())) .collect::>(); @@ -281,6 +303,8 @@ impl JmapConfig { DataType::Email, DataType::Thread, DataType::SieveScript, + DataType::CalendarEvent, + DataType::ContactCard, ], supported_digest_algorithms: vec!["sha", "sha-256", "sha-512"], }), @@ -306,6 +330,42 @@ impl JmapConfig { Capabilities::Empty(EmptyCapabilities::default()), ); + let mut data_types = metadata + .data_types + .map(|data_type| { + ( + data_type, + DataTypeMetadataInfo { + namespaces: REGISTERED_NAMESPACES + .iter() + .filter(|namespace| { + namespace.is_advertised() + && namespace.data_types.contains(&data_type) + && (metadata.private_metadata + || namespace.scope != NamespaceScope::PrivateMetadata) + }) + .map(|namespace| namespace.name) + .collect(), + supports_vendor_namespaces: metadata.vendor_namespaces, + supports_private: metadata.private_metadata, + max_depth: metadata.max_depth, + }, + ) + }) + .filter(|(_, info)| info.supports_vendor_namespaces || !info.namespaces.is_empty()) + .collect::>(); + if !data_types.is_empty() { + data_types.sort_unstable_by_key(); + self.capabilities.session.append( + Capability::Metadata, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.insert( + Capability::Metadata, + Capabilities::Metadata(MetadataCapabilities { data_types }), + ); + } + // Add Web Push VAPID capabilities if let Some(application_server_key) = self .vapid diff --git a/crates/common/src/config/mailstore/email.rs b/crates/common/src/config/mailstore/email.rs index 20b38831..74902f38 100644 --- a/crates/common/src/config/mailstore/email.rs +++ b/crates/common/src/config/mailstore/email.rs @@ -4,6 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{ + config::mailstore::limits::{EmailLimits, MAX_HEADER_ENTRIES}, + storage::ObjectQuota, +}; use ahash::{AHashMap, AHashSet}; use nlp::language::Language; use registry::{ @@ -23,14 +27,14 @@ use registry::{ use std::time::Duration; use store::{ registry::bootstrap::Bootstrap, - search::{CalendarSearchField, ContactSearchField, EmailSearchField, SearchField}, + search::{ + CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField, + }, write::SearchIndex, }; use types::special_use::SpecialUse; use utils::cron::SimpleCron; -use crate::storage::ObjectQuota; - #[derive(Clone)] pub struct EmailConfig { pub default_language: Language, @@ -39,6 +43,7 @@ pub struct EmailConfig { pub mailbox_max_depth: usize, pub mailbox_name_max_len: usize, + pub limits: EmailLimits, pub mail_attachments_max_size: usize, pub mail_max_size: usize, @@ -57,7 +62,9 @@ pub struct EmailConfig { pub encrypt_append: bool, pub index_batch_size: usize, + pub index_concurrency: usize, pub index_fields: AHashMap>, + pub extract_limits: ExtractLimits, pub max_objects: ObjectQuota, pub compression: CompressionAlgo, @@ -67,6 +74,18 @@ pub struct EmailConfig { pub blob_purge_frequency: SimpleCron, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExtractLimits { + pub max_document_size: usize, + pub max_text_size: usize, + pub max_decompressed_size: u64, + pub max_part_size: u64, + pub max_parts: usize, + pub max_archive_entries: usize, + pub max_pdf_objects: usize, + pub max_rtf_depth: usize, +} + #[derive(Clone, Debug)] pub struct DefaultFolder { pub name: String, @@ -116,7 +135,6 @@ impl EmailConfig { (StorageQuota::MaxSieveScripts, sieve.max_scripts), (StorageQuota::MaxEmailIdentities, email.max_identities), (StorageQuota::MaxEmailSubmissions, email.max_submissions), - (StorageQuota::MaxMaskedAddresses, email.max_masked_addresses), (StorageQuota::MaxAppPasswords, auth.max_app_passwords), (StorageQuota::MaxApiKeys, auth.max_api_keys), (StorageQuota::MaxPublicKeys, email.max_public_keys), @@ -211,10 +229,6 @@ impl EmailConfig { SearchEmailField::Subject => EmailSearchField::Subject, SearchEmailField::Body => EmailSearchField::Body, SearchEmailField::Attachment => EmailSearchField::Attachment, - SearchEmailField::ReceivedAt => EmailSearchField::ReceivedAt, - SearchEmailField::SentAt => EmailSearchField::SentAt, - SearchEmailField::Size => EmailSearchField::Size, - SearchEmailField::HasAttachment => EmailSearchField::HasAttachment, SearchEmailField::Headers => EmailSearchField::Headers, }) }) @@ -239,12 +253,17 @@ impl EmailConfig { SearchContactField::OnlineService => ContactSearchField::OnlineService, SearchContactField::Address => ContactSearchField::Address, SearchContactField::Note => ContactSearchField::Note, - SearchContactField::Uid => ContactSearchField::Uid, }) }) .collect(), ); } + if search.index_files { + index_fields.insert( + SearchIndex::File, + AHashSet::from_iter([SearchField::File(FileSearchField::Content)]), + ); + } if search.index_calendar { index_fields.insert( SearchIndex::Calendar, @@ -258,19 +277,35 @@ impl EmailConfig { SearchCalendarField::Location => CalendarSearchField::Location, SearchCalendarField::Owner => CalendarSearchField::Owner, SearchCalendarField::Attendee => CalendarSearchField::Attendee, - SearchCalendarField::Start => CalendarSearchField::Start, - SearchCalendarField::Uid => CalendarSearchField::Uid, }) }) .collect(), ); } + let header_count = (email.max_header_count as usize).min(MAX_HEADER_ENTRIES); + if header_count < email.max_header_count as usize { + bp.build_warning( + ObjectType::Email.singleton(), + format!( + "maxHeaderCount {} exceeds the supported maximum, using {header_count}", + email.max_header_count + ), + ); + } + EmailConfig { default_language: Language::from_iso_639(search.default_language.as_str()) .unwrap_or(Language::English), mailbox_max_depth: email.max_mailbox_depth as usize, mailbox_name_max_len: email.max_mailbox_name_length as usize, + limits: EmailLimits { + mailboxes_per_email: email.max_mailboxes_per_email as usize, + keywords_per_email: email.max_flags_per_email as usize, + keyword_length: email.max_flag_length as usize, + header_count, + header_size: email.max_header_size as usize, + }, mail_attachments_max_size: email.max_attachment_size as usize, mail_max_size: email.max_message_size as usize, mail_autoexpunge_after: dr.expunge_trash_after.map(|d| d.into_inner().as_secs()), @@ -282,8 +317,19 @@ impl EmailConfig { sieve_max_script_name: sieve.max_script_name_length as usize, encrypt: email.encrypt_at_rest, encrypt_append: email.encrypt_on_append, - index_batch_size: search.index_batch_size as usize, + index_batch_size: std::cmp::max(search.index_batch_size as usize, 1), + index_concurrency: std::cmp::max(search.max_concurrent_index_tasks as usize, 1), index_fields, + extract_limits: ExtractLimits { + max_document_size: search.max_extract_document_size as usize, + max_text_size: search.max_extract_text_size as usize, + max_decompressed_size: search.max_extract_decompressed_size, + max_part_size: search.max_extract_part_size, + max_parts: search.max_extract_parts as usize, + max_archive_entries: search.max_extract_archive_entries as usize, + max_pdf_objects: search.max_extract_pdf_objects as usize, + max_rtf_depth: search.max_extract_rtf_depth as usize, + }, max_objects, default_folders, shared_folder, diff --git a/crates/common/src/config/mailstore/imap.rs b/crates/common/src/config/mailstore/imap.rs index af58eee3..2dd522ac 100644 --- a/crates/common/src/config/mailstore/imap.rs +++ b/crates/common/src/config/mailstore/imap.rs @@ -34,11 +34,11 @@ impl ImapConfig { ImapConfig { max_request_size: imap.max_request_size as usize, max_auth_failures: imap.max_auth_failures as u32, - timeout_auth: imap.timeout_authenticated.into_inner(), - timeout_unauth: imap.timeout_anonymous.into_inner(), - timeout_idle: imap.timeout_idle.into_inner(), - rate_requests: imap.max_request_rate, - rate_concurrent: imap.max_concurrent, + timeout_auth: imap.authenticated_timeout.into_inner(), + timeout_unauth: imap.anonymous_timeout.into_inner(), + timeout_idle: imap.idle_timeout.into_inner(), + rate_requests: imap.rate_limit, + rate_concurrent: imap.max_concurrent_connections, allow_plain_auth: imap.allow_plain_text_auth, max_messages_per_command: imap.max_messages_per_command.min(u32::MAX as u64) as u32, max_messages_per_save: imap.max_messages_per_save.min(u32::MAX as u64) as u32, diff --git a/crates/common/src/config/mailstore/jmap.rs b/crates/common/src/config/mailstore/jmap.rs index ba3350eb..ef1e8d94 100644 --- a/crates/common/src/config/mailstore/jmap.rs +++ b/crates/common/src/config/mailstore/jmap.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::network::webpush::{Vapid, VapidKey}; +use crate::{ + config::metadata::MetadataConfig, + network::webpush::{Vapid, VapidKey}, +}; use jmap_proto::request::capability::BaseCapabilities; use registry::schema::{prelude::ObjectType, structs::Jmap}; use std::time::Duration; @@ -54,7 +57,7 @@ pub struct JmapConfig { } impl JmapConfig { - pub async fn parse(bp: &mut Bootstrap) -> Self { + pub async fn parse(bp: &mut Bootstrap, metadata: &MetadataConfig) -> Self { let jmap = bp.setting_infallible::().await; let web_push_key = jmap .web_push_key @@ -78,14 +81,14 @@ impl JmapConfig { }); let mut jmap = JmapConfig { - query_max_results: jmap.query_max_results as usize, - changes_max_results: jmap.changes_max_results as usize, - snippet_max_results: jmap.snippet_max_results as usize, + query_max_results: jmap.max_query_results as usize, + changes_max_results: jmap.max_changes_results as usize, + snippet_max_results: jmap.max_snippet_results as usize, request_max_size: jmap.max_request_size as usize, request_max_calls: jmap.max_method_calls as usize, request_max_concurrent: jmap.max_concurrent_requests, - get_max_objects: jmap.get_max_results as usize, - set_max_objects: jmap.set_max_objects as usize, + get_max_objects: jmap.max_get_results as usize, + set_max_objects: jmap.max_set_objects as usize, upload_max_size: jmap.max_upload_size as usize, upload_max_concurrent: jmap.max_concurrent_uploads, upload_tmp_quota_size: jmap.upload_quota as usize, @@ -99,7 +102,7 @@ impl JmapConfig { web_socket_timeout: jmap.websocket_timeout.into_inner(), web_socket_heartbeat: jmap.websocket_heartbeat.into_inner(), push_attempt_interval: jmap.push_attempt_wait.into_inner(), - push_attempts_max: jmap.push_max_attempts as u32, + push_attempts_max: jmap.max_push_attempts as u32, push_retry_interval: jmap.push_retry_wait.into_inner(), push_timeout: jmap.push_request_timeout.into_inner(), push_verify_timeout: jmap.push_verify_timeout.into_inner(), @@ -128,7 +131,7 @@ impl JmapConfig { .map(|key| Vapid::new(key, web_push_contact)); // Add capabilities - jmap.add_capabilities(bp).await; + jmap.add_capabilities(bp, metadata).await; jmap } } diff --git a/crates/common/src/config/mailstore/limits.rs b/crates/common/src/config/mailstore/limits.rs new file mode 100644 index 00000000..322f0387 --- /dev/null +++ b/crates/common/src/config/mailstore/limits.rs @@ -0,0 +1,199 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{ + error::set::{SetError, SetErrorType}, + object::email::EmailProperty, +}; +use mail_parser::Message; +use std::fmt::Display; +use types::keyword::Keyword; + +pub const MAX_HEADER_ENTRIES: usize = 65_534; +const HEADER_TERMINATOR_LEN: usize = b"\r\n".len(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EmailLimits { + pub mailboxes_per_email: usize, + pub keywords_per_email: usize, + pub keyword_length: usize, + pub header_count: usize, + pub header_size: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EmailLimitError { + TooManyMailboxes { max: usize }, + TooManyKeywords { max: usize }, + KeywordTooLong { max: usize }, + TooManyHeaders { max: usize }, + TooManyMessageHeaders { max: usize }, + HeaderTooLarge { max: usize }, +} + +impl EmailLimits { + pub fn validate_mailbox_count(&self, count: usize) -> Result<(), EmailLimitError> { + if count <= self.mailboxes_per_email { + Ok(()) + } else { + Err(EmailLimitError::TooManyMailboxes { + max: self.mailboxes_per_email, + }) + } + } + + pub fn validate_mailbox_change( + &self, + prev_count: usize, + count: usize, + ) -> Result<(), EmailLimitError> { + if count <= prev_count { + Ok(()) + } else { + self.validate_mailbox_count(count) + } + } + + pub fn validate_keyword_count(&self, count: usize) -> Result<(), EmailLimitError> { + if count <= self.keywords_per_email { + Ok(()) + } else { + Err(EmailLimitError::TooManyKeywords { + max: self.keywords_per_email, + }) + } + } + + pub fn validate_keyword_change( + &self, + prev_count: usize, + count: usize, + ) -> Result<(), EmailLimitError> { + if count <= prev_count { + Ok(()) + } else { + self.validate_keyword_count(count) + } + } + + pub fn validate_keyword_names<'x>( + &self, + names: impl IntoIterator, + ) -> Result<(), EmailLimitError> { + if names + .into_iter() + .all(|name| name.len() <= self.keyword_length) + { + Ok(()) + } else { + Err(EmailLimitError::KeywordTooLong { + max: self.keyword_length, + }) + } + } + + pub fn validate_keywords<'x>( + &self, + count: usize, + names: impl IntoIterator, + ) -> Result<(), EmailLimitError> { + self.validate_keyword_count(count)?; + self.validate_keyword_names(names) + } + + pub fn validate_email( + &self, + mailbox_count: usize, + keywords: &[Keyword], + ) -> Result<(), EmailLimitError> { + self.validate_mailbox_count(mailbox_count)?; + self.validate_keywords( + keywords.len(), + keywords.iter().filter_map(|keyword| keyword.id().err()), + ) + } + + pub fn is_keyword_allowed(&self, keyword: &Keyword) -> bool { + keyword + .id() + .map_or_else(|name| name.len() <= self.keyword_length, |_| true) + } + + pub fn header_fields_size(&self) -> usize { + self.header_size.saturating_sub(HEADER_TERMINATOR_LEN) + } + + pub fn validate_header_section(&self, message: &Message<'_>) -> Result<(), EmailLimitError> { + let root = message.root_part(); + if root.headers().len() > self.header_count { + Err(EmailLimitError::TooManyHeaders { + max: self.header_count, + }) + } else if message.header_count() > MAX_HEADER_ENTRIES { + Err(EmailLimitError::TooManyMessageHeaders { + max: MAX_HEADER_ENTRIES, + }) + } else if root.offset_body().saturating_sub(root.offset_header()) as usize + > self.header_size + { + Err(EmailLimitError::HeaderTooLarge { + max: self.header_size, + }) + } else { + Ok(()) + } + } +} + +impl From for SetError { + fn from(err: EmailLimitError) -> Self { + match err { + EmailLimitError::TooManyMailboxes { .. } => { + SetError::new(SetErrorType::TooManyMailboxes) + .with_property(EmailProperty::MailboxIds) + } + EmailLimitError::TooManyKeywords { .. } => { + SetError::new(SetErrorType::TooManyKeywords).with_property(EmailProperty::Keywords) + } + EmailLimitError::KeywordTooLong { .. } => { + SetError::invalid_properties().with_property(EmailProperty::Keywords) + } + EmailLimitError::TooManyHeaders { .. } + | EmailLimitError::TooManyMessageHeaders { .. } + | EmailLimitError::HeaderTooLarge { .. } => SetError::too_large(), + } + .with_description(err.to_string()) + } +} + +impl Display for EmailLimitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EmailLimitError::TooManyMailboxes { max } => { + write!(f, "An email cannot belong to more than {max} mailboxes.") + } + EmailLimitError::TooManyKeywords { max } => { + write!(f, "An email cannot have more than {max} keywords.") + } + EmailLimitError::KeywordTooLong { max } => { + write!(f, "Keywords cannot be longer than {max} bytes.") + } + EmailLimitError::TooManyHeaders { max } => { + write!(f, "An email header cannot have more than {max} fields.") + } + EmailLimitError::TooManyMessageHeaders { max } => { + write!( + f, + "An email cannot have more than {max} header fields across all its parts." + ) + } + EmailLimitError::HeaderTooLarge { max } => { + write!(f, "An email header cannot exceed {max} bytes.") + } + } + } +} + diff --git a/crates/common/src/config/mailstore/mod.rs b/crates/common/src/config/mailstore/mod.rs index cd528066..1240bf7b 100644 --- a/crates/common/src/config/mailstore/mod.rs +++ b/crates/common/src/config/mailstore/mod.rs @@ -8,5 +8,6 @@ pub mod capabilities; pub mod email; pub mod imap; pub mod jmap; +pub mod limits; pub mod scripts; pub mod spamfilter; diff --git a/crates/common/src/config/mailstore/scripts.rs b/crates/common/src/config/mailstore/scripts.rs index 212ea0aa..69bb54e2 100644 --- a/crates/common/src/config/mailstore/scripts.rs +++ b/crates/common/src/config/mailstore/scripts.rs @@ -6,6 +6,7 @@ use crate::{ VERSION_PUBLIC, + config::mailstore::limits::EmailLimits, expr::if_block::{BootstrapExprExt, IfBlock}, scripts::{ functions::{register_functions_trusted, register_functions_untrusted}, @@ -24,8 +25,9 @@ use registry::{ types::EnumImpl, }; use sieve::{Compiler, Runtime, Sieve, compiler::grammar::Capability}; -use std::{collections::hash_map::Entry, sync::Arc}; +use std::{borrow::Cow, collections::hash_map::Entry}; use store::registry::bootstrap::Bootstrap; +use utils::text::lowercase; pub struct Scripting { pub untrusted_compiler: Compiler, @@ -38,13 +40,13 @@ pub struct Scripting { pub return_path: IfBlock, pub sign: IfBlock, pub untrusted_sign: IfBlock, - pub trusted_scripts: AHashMap>, - pub untrusted_scripts: AHashMap>, + pub trusted_scripts: AHashMap>, + pub untrusted_scripts: AHashMap>, pub http_client: reqwest::Client, } impl Scripting { - pub async fn parse(bp: &mut Bootstrap) -> Self { + pub async fn parse(bp: &mut Bootstrap, limits: &EmailLimits) -> Self { // Parse untrusted compiler let untrusted = bp.setting_infallible::().await; let untrusted_sign = bp.compile_expr( @@ -63,6 +65,13 @@ impl Scripting { .with_max_local_variables(untrusted.max_local_vars as usize) .with_max_header_size(untrusted.max_header_size as usize) .with_max_includes(untrusted.max_includes as usize) + .without_capabilities( + untrusted + .disable_capabilities + .iter() + .map(|cap| cap.as_str()), + ) + .without_capability(Capability::ImapSieve) .register_functions(&mut fnc_map_untrusted); // Parse untrusted runtime @@ -75,6 +84,8 @@ impl Scripting { .with_max_received_headers(usize::MAX) // This is set to usize::MAX here, but the actual limit is enforced during ingestion. .with_max_header_size(untrusted.max_header_size as usize) .with_max_out_messages(untrusted.max_out_messages as usize) + .with_max_header_count(limits.header_count) + .with_max_header_block_size(limits.header_fields_size()) .with_default_vacation_expiry(untrusted.default_expiry_vacation.into_inner().as_secs()) .with_default_duplicate_expiry( untrusted.default_expiry_duplicate.into_inner().as_secs(), @@ -86,6 +97,7 @@ impl Scripting { .iter() .map(|cap| cap.as_str()), ) + .without_capability(Capability::ImapSieve) .with_valid_notification_uris(untrusted.allowed_notify_uris) .with_protected_headers(untrusted.protected_headers) .with_vacation_default_subject(untrusted.default_subject) @@ -140,13 +152,12 @@ impl Scripting { .with_max_out_messages(trusted.max_out_messages as usize) .with_cpu_limit(trusted.max_cpu_cycles as usize) .with_max_nested_includes(trusted.max_nested_includes as usize) - .with_max_received_headers(trusted.max_received_headers as usize) - .with_default_duplicate_expiry(trusted.duplicate_expiry.into_inner().as_secs()); + .with_max_received_headers(trusted.max_received_headers as usize); trusted_runtime.set_local_hostname(local_hostname.clone()); untrusted_runtime.set_local_hostname(local_hostname); // Parse trusted scripts - let mut trusted_scripts: AHashMap> = AHashMap::new(); + let mut trusted_scripts: AHashMap> = AHashMap::new(); for script in bp.list_infallible::().await { if !script.object.is_active { continue; @@ -155,7 +166,7 @@ impl Scripting { match trusted_compiler.compile(script.object.contents.as_bytes()) { Ok(compiled) => match trusted_scripts.entry(script.object.name.to_lowercase()) { Entry::Vacant(entry) => { - entry.insert(compiled.into()); + entry.insert(compiled); } Entry::Occupied(_) => { bp.build_error( @@ -177,7 +188,7 @@ impl Scripting { } // Parse untrusted scripts - let mut untrusted_scripts: AHashMap> = AHashMap::new(); + let mut untrusted_scripts: AHashMap> = AHashMap::new(); for script in bp.list_infallible::().await { if !script.object.is_active { continue; @@ -186,7 +197,7 @@ impl Scripting { match untrusted_compiler.compile(script.object.contents.as_bytes()) { Ok(compiled) => match untrusted_scripts.entry(script.object.name.to_lowercase()) { Entry::Vacant(entry) => { - entry.insert(compiled.into()); + entry.insert(compiled); } Entry::Occupied(_) => { bp.build_error( @@ -199,9 +210,9 @@ impl Scripting { } }, Err(err) => { - bp.build_error( + bp.build_warning( script.id, - format!("Failed to compile user global Sieve script: {err}"), + format!("Failed to compile user global Sieve script, skipping it: {err}"), ); } } @@ -214,7 +225,7 @@ impl Scripting { trusted_compiler, untrusted_scripts, trusted_scripts, - http_client: utils::http::http_client_builder(cfg!(feature = "test_mode")) + http_client: utils::http::http_client_builder(cfg!(stalwart_tests)) .pool_max_idle_per_host(0) .redirect(reqwest::redirect::Policy::none()) .build() @@ -240,22 +251,23 @@ impl Scripting { } } - pub fn trusted_script(&self, name: &str) -> Option<&Arc> { + pub fn trusted_script(&self, name: &str) -> Option<&Sieve<'static>> { script_by_name(&self.trusted_scripts, name) } - pub fn untrusted_script(&self, name: &str) -> Option<&Arc> { + pub fn untrusted_script(&self, name: &str) -> Option<&Sieve<'static>> { script_by_name(&self.untrusted_scripts, name) } } fn script_by_name<'x>( - scripts: &'x AHashMap>, + scripts: &'x AHashMap>, name: &str, -) -> Option<&'x Arc> { - scripts - .get(name) - .or_else(|| scripts.get(name.to_lowercase().as_str())) +) -> Option<&'x Sieve<'static>> { + scripts.get(name).or_else(|| match lowercase(name) { + Cow::Owned(name) => scripts.get(name.as_str()), + Cow::Borrowed(_) => None, + }) } impl Clone for Scripting { diff --git a/crates/common/src/config/mailstore/spamfilter.rs b/crates/common/src/config/mailstore/spamfilter.rs index 54a95030..c84b2671 100644 --- a/crates/common/src/config/mailstore/spamfilter.rs +++ b/crates/common/src/config/mailstore/spamfilter.rs @@ -5,12 +5,14 @@ */ use crate::expr::{ - Variable, + Bump, Constant, ExpressionItem, Variable, functions::ResolveVariable, + guard::ScopeRules, if_block::{BootstrapExprExt, IfBlock}, }; use ahash::AHashSet; -use mail_auth::common::resolver::ToReverseName; +use compact_str::{CompactString, ToCompactString}; +use mail_auth::dns::ToReverseName; use nlp::classifier::model::{CcfhClassifier, FhClassifier}; use registry::schema::{ enums::{ExpressionVariable, ModelSize}, @@ -22,12 +24,18 @@ use registry::schema::{ }; use sieve::SpamStatus; use std::{ + mem::size_of, net::{IpAddr, SocketAddr}, + sync::{Arc, OnceLock}, time::Duration, }; use store::registry::{RegistryObject, bootstrap::Bootstrap}; use tokio::net::lookup_host; -use utils::{cache::CacheItemWeight, glob::GlobMap}; +use utils::{ + cache::CacheItemWeight, + glob::GlobMap, + outbound::{OutboundClient, OutboundPolicy, Redirects}, +}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub enum SpamClassifier { @@ -43,6 +51,10 @@ pub enum SpamClassifier { Disabled, } +impl store::write::ArchiveCompression for SpamClassifier { + const COMPRESSION: store::write::Compression = store::write::Compression::Zstd(None); +} + #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { pub enabled: bool, @@ -57,7 +69,22 @@ pub struct SpamFilterConfig { pub classifier: Option, pub scores: SpamFilterScoreConfig, pub spam_rules_url: Option, - pub url_client: reqwest::Client, + pub url_client: OutboundClient, + pub url_redirect: UrlRedirectConfig, + pub text_limits: SpamTextLimits, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpamTextLimits { + pub max_body: usize, + pub max_attachment: usize, + pub max_message: usize, +} + +#[derive(Debug, Clone, Default)] +pub struct UrlRedirectConfig { + pub max_urls: usize, + pub cache_ttl: Duration, } #[derive(Debug, Clone, Default)] @@ -139,7 +166,7 @@ pub struct ClassifierConfig { pub auto_learn_spam_trap: bool, pub auto_learn_spam_rbl_count: u32, pub hold_samples_for: u64, - pub train_frequency: Option, + pub train_interval: Option, pub log_scale: bool, pub l2_normalize: bool, } @@ -162,15 +189,21 @@ pub struct PyzorConfig { pub ratio: f64, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Debug, Default, Clone)] pub struct SpamFilterRules { - pub url: Vec, - pub domain: Vec, - pub email: Vec, - pub ip: Vec, - pub header: Vec, - pub body: Vec, - pub any: Vec, + pub before_ai: SpamFilterRuleSet, + pub after_ai: SpamFilterRuleSet, +} + +#[derive(Debug, Default, Clone)] +pub struct SpamFilterRuleSet { + pub url: ScopeRules, + pub domain: ScopeRules, + pub email: ScopeRules, + pub ip: ScopeRules, + pub header: ScopeRules, + pub body: ScopeRules, + pub any: ScopeRules, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -230,6 +263,7 @@ impl SpamFilterConfig { enabled: spam.enable, card_is_ham: spam.trust_contacts, trusted_reply: spam.trust_replies, + text_limits: SpamTextLimits::from(&spam), dnsbl: DnsBlConfig::parse(bp).await, rules: SpamFilterRules::parse(bp).await, lists: SpamFilterLists::parse(bp).await, @@ -242,16 +276,40 @@ impl SpamFilterConfig { }, grey_list_expiry: spam.greylist_for.map(|d| d.into_inner().as_secs()), spam_rules_url: spam.spam_filter_rules_url, - url_client: utils::http::http_client_builder(true) - .pool_max_idle_per_host(0) - .redirect(reqwest::redirect::Policy::none()) - .user_agent("Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/118.0") - .build() - .unwrap_or_default(), + url_client: OutboundClient::new( + utils::http::http_client_builder(false) + .pool_max_idle_per_host(0) + .timeout(spam.redirect_timeout.into_inner()) + .user_agent( + "Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/118.0", + ), + OutboundPolicy::PublicOnly, + Redirects::None, + ), + url_redirect: UrlRedirectConfig { + max_urls: spam.max_redirect_urls as usize, + cache_ttl: spam.redirect_cache_ttl.into_inner(), + }, } } } +impl From<&SpamSettings> for SpamTextLimits { + fn from(settings: &SpamSettings) -> Self { + SpamTextLimits { + max_body: usize::try_from(settings.max_body_text).unwrap_or(usize::MAX), + max_attachment: usize::try_from(settings.max_attachment_text).unwrap_or(usize::MAX), + max_message: usize::try_from(settings.max_message_text).unwrap_or(usize::MAX), + } + } +} + +impl Default for SpamTextLimits { + fn default() -> Self { + SpamTextLimits::from(&SpamSettings::default()) + } +} + impl SpamFilterRules { pub async fn parse(bp: &mut Bootstrap) -> SpamFilterRules { let mut rules = vec![]; @@ -262,22 +320,114 @@ impl SpamFilterRules { } rules.sort_by_key(|a| a.priority); - let mut result = SpamFilterRules::default(); + let after_ai = rules_after_ai(rules.iter().map(|rule| &rule.rule)); + let (after, before): (Vec<_>, Vec<_>) = rules + .into_iter() + .zip(after_ai) + .partition(|(_, is_after)| *is_after); + + SpamFilterRules { + before_ai: SpamFilterRuleSet::new(before.into_iter().map(|(rule, _)| rule)), + after_ai: SpamFilterRuleSet::new(after.into_iter().map(|(rule, _)| rule)), + } + } +} + +impl SpamFilterRuleSet { + fn new(rules: impl Iterator) -> Self { + let mut url = Vec::new(); + let mut domain = Vec::new(); + let mut email = Vec::new(); + let mut ip = Vec::new(); + let mut header = Vec::new(); + let mut body = Vec::new(); + let mut any = Vec::new(); for rule in rules { match rule.scope { - Element::Url => result.url.push(rule.rule), - Element::Domain => result.domain.push(rule.rule), - Element::Email => result.email.push(rule.rule), - Element::Ip => result.ip.push(rule.rule), - Element::Header => result.header.push(rule.rule), - Element::Body => result.body.push(rule.rule), - Element::Any => result.any.push(rule.rule), + Element::Url => url.push(rule.rule), + Element::Domain => domain.push(rule.rule), + Element::Email => email.push(rule.rule), + Element::Ip => ip.push(rule.rule), + Element::Header => header.push(rule.rule), + Element::Body => body.push(rule.rule), + Element::Any => any.push(rule.rule), } } - result + let location = Some(ExpressionVariable::Location); + SpamFilterRuleSet { + url: ScopeRules::new(url, location), + domain: ScopeRules::new(domain, location), + email: ScopeRules::new(email, location), + ip: ScopeRules::new(ip, location), + header: ScopeRules::new(header, Some(ExpressionVariable::NameLower)), + body: ScopeRules::new(body, location), + any: ScopeRules::new(any, None), + } } + + pub fn is_empty(&self) -> bool { + self.url.is_empty() + && self.domain.is_empty() + && self.email.is_empty() + && self.ip.is_empty() + && self.header.is_empty() + && self.body.is_empty() + && self.any.is_empty() + } +} + +pub const AI_TAG_PREFIX: &str = "AI_"; + +pub fn rules_after_ai<'x>(rules: impl Iterator) -> Vec { + let rules = rules.collect::>(); + let mut after = rules + .iter() + .map(|rule| rule_globals(rule).any(|tag| tag.starts_with(AI_TAG_PREFIX))) + .collect::>(); + let mut tainted: AHashSet<&str> = AHashSet::new(); + let mut emitted = vec![false; rules.len()]; + + loop { + for ((rule, is_after), is_emitted) in rules.iter().zip(&after).zip(emitted.iter_mut()) { + if *is_after && !*is_emitted { + *is_emitted = true; + tainted.extend(rule_constant_tags(rule)); + } + } + + let mut changed = false; + for (rule, is_after) in rules.iter().zip(after.iter_mut()) { + if !*is_after && rule_globals(rule).any(|tag| tainted.contains(tag)) { + *is_after = true; + changed = true; + } + } + if !changed { + break; + } + } + + after +} + +fn rule_globals(rule: &IfBlock) -> impl Iterator { + rule.all_items().filter_map(|item| match item { + ExpressionItem::Global(name) => Some(name.as_str()), + _ => None, + }) +} + +fn rule_constant_tags(rule: &IfBlock) -> impl Iterator { + rule.if_then + .iter() + .map(|if_then| if_then.then.items()) + .chain([rule.default.items()]) + .filter_map(|items| match items { + [ExpressionItem::Constant(Constant::String(tag))] => Some(tag.as_str()), + _ => None, + }) } struct SpamFilterRule { @@ -524,7 +674,7 @@ impl ClassifierConfig { hold_samples_for: classifier.hold_samples_for.into_inner().as_secs(), min_ham_samples: classifier.min_ham_samples, min_spam_samples: classifier.min_spam_samples, - train_frequency: classifier.train_frequency.map(|d| d.into_inner().as_secs()), + train_interval: classifier.train_interval.map(|d| d.into_inner().as_secs()), log_scale, l2_normalize, } @@ -603,55 +753,88 @@ impl Element { } } +const IPV6_TEXT_MAX_LEN: usize = 39; +const IPV6_REVERSE_LEN: usize = 63; + pub struct IpResolver { ip: IpAddr, - ip_string: String, - reverse: String, - octets: Variable<'static>, + text: OnceLock, + reverse: OnceLock, } impl ResolveVariable for IpResolver { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> { + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a> { match variable { - ExpressionVariable::Ip | ExpressionVariable::Value => self.ip_string.as_str().into(), - ExpressionVariable::IpReverse => self.reverse.as_str().into(), - ExpressionVariable::Octets => self.octets.clone(), + ExpressionVariable::Ip | ExpressionVariable::Value => self.text().into(), + ExpressionVariable::IpReverse => self.reverse().into(), + ExpressionVariable::Octets => { + let octets = |octets: &[u8]| { + arena.alloc_slice_fill_iter( + octets + .iter() + .map(|octet| Variable::Integer(i64::from(*octet))), + ) + }; + Variable::Array(match self.ip { + IpAddr::V4(ip) => octets(&ip.octets()), + IpAddr::V6(ip) => octets(&ip.octets()), + }) + } ExpressionVariable::IsV4 => Variable::Integer(self.ip.is_ipv4() as _), ExpressionVariable::IsV6 => Variable::Integer(self.ip.is_ipv6() as _), _ => Variable::Integer(0), } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } impl IpResolver { pub fn new(ip: IpAddr) -> Self { Self { - ip_string: ip.to_string(), - reverse: ip.to_reverse_name(), - octets: Variable::Array(match ip { - IpAddr::V4(ipv4_addr) => ipv4_addr - .octets() - .iter() - .map(|o| Variable::Integer(*o as _)) - .collect(), - IpAddr::V6(ipv6_addr) => ipv6_addr - .octets() - .iter() - .map(|o| Variable::Integer(*o as _)) - .collect(), - }), ip, + text: OnceLock::new(), + reverse: OnceLock::new(), } } + + pub fn text(&self) -> &str { + self.text.get_or_init(|| self.ip.to_compact_string()) + } + + pub fn reverse(&self) -> &str { + self.reverse + .get_or_init(|| self.ip.to_reverse_name().into()) + } } impl CacheItemWeight for IpResolver { fn weight(&self) -> u64 { - (std::mem::size_of::() + self.ip_string.len() + self.reverse.len()) as u64 + let heap = match self.ip { + IpAddr::V4(_) => 0, + IpAddr::V6(_) => IPV6_TEXT_MAX_LEN + IPV6_REVERSE_LEN, + }; + (size_of::() + heap) as u64 + } +} + +#[derive(Clone)] +pub enum SpamLookup { + None, + Dnsbl(Arc<[IpResolver]>), + Redirect(Arc), +} + +impl CacheItemWeight for SpamLookup { + fn weight(&self) -> u64 { + let heap = match self { + SpamLookup::None => 0, + SpamLookup::Dnsbl(codes) => codes.iter().map(IpResolver::weight).sum(), + SpamLookup::Redirect(url) => url.len() as u64, + }; + size_of::() as u64 + heap } } @@ -664,79 +847,3 @@ impl SpamFilterAction { } } -#[cfg(test)] -mod tests { - use super::*; - - fn config(spam: f32, discard: f32, reject: f32) -> SpamFilterScoreConfig { - SpamFilterScoreConfig { - reject_threshold: reject, - discard_threshold: discard, - spam_threshold: spam, - } - } - - #[test] - fn spam_percentage_defaults() { - let config = config(5.0, 0.0, 0.0); - - for (score, expected) in [ - (-10.0, 0), - (0.0, 0), - (0.5, 5), - (2.5, 25), - (4.9, 49), - (4.999, 49), - (5.0, 50), - (7.5, 75), - (9.9, 99), - (10.0, 100), - (50.0, 100), - ] { - assert_eq!(config.spam_percentage(score), expected, "score {score}"); - } - } - - #[test] - fn spam_percentage_matches_is_spam() { - for config in [ - config(5.0, 0.0, 0.0), - config(5.0, 20.0, 15.0), - config(1.0, 0.0, 3.0), - config(12.5, 25.0, 0.0), - config(0.0, 0.0, 0.0), - ] { - for score in (-2000..=4000).map(|score| score as f32 / 100.0) { - assert_eq!( - config.spam_percentage(score) >= 50, - config.is_spam(score), - "score {score} with {config:?}" - ); - } - } - } - - #[test] - fn spam_percentage_ceiling_is_lowest_enabled_threshold() { - let reject_lowest = config(5.0, 20.0, 15.0); - assert_eq!(reject_lowest.spam_percentage(10.0), 75); - assert_eq!(reject_lowest.spam_percentage(15.0), 100); - - let discard_only = config(5.0, 15.0, 0.0); - assert_eq!(discard_only.spam_percentage(10.0), 75); - - let below_spam_threshold = config(5.0, 3.0, 0.0); - assert_eq!(below_spam_threshold.spam_percentage(7.5), 75); - } - - #[test] - fn spam_status_from_percentage() { - assert!(matches!(spam_status(None), SpamStatus::Unknown)); - assert!(matches!(spam_status(Some(0)), SpamStatus::Ham)); - assert!(matches!(spam_status(Some(100)), SpamStatus::Spam)); - assert!(matches!( - spam_status(Some(50)), - SpamStatus::MaybeSpam(fraction) if fraction == 0.5 - )); - } -} diff --git a/crates/common/src/config/metadata.rs b/crates/common/src/config/metadata.rs new file mode 100644 index 00000000..99116075 --- /dev/null +++ b/crates/common/src/config/metadata.rs @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use registry::schema::{enums::MetadataDataType, structs::Metadata}; +use store::registry::bootstrap::Bootstrap; +use types::{metadata::MetadataLimits, type_state::DataType}; +use utils::map::bitmap::Bitmap; + +#[derive(Default, Clone)] +pub struct MetadataConfig { + pub data_types: Bitmap, + pub vendor_namespaces: bool, + pub private_metadata: bool, + pub max_depth: Option, + pub max_entry_size: usize, + pub max_size: usize, + pub max_private_size: usize, + pub max_entries: usize, + pub query_max_scan: usize, + pub imap_server_comment: Option, + pub imap_server_admin: Option, +} + +impl MetadataConfig { + pub async fn parse(bp: &mut Bootstrap) -> Self { + let metadata = bp.setting_infallible::().await; + + MetadataConfig { + data_types: metadata + .data_types + .iter() + .map(|data_type| match data_type { + MetadataDataType::Email => DataType::Email, + MetadataDataType::Mailbox => DataType::Mailbox, + MetadataDataType::SieveScript => DataType::SieveScript, + MetadataDataType::Calendar => DataType::Calendar, + MetadataDataType::CalendarEvent => DataType::CalendarEvent, + MetadataDataType::AddressBook => DataType::AddressBook, + MetadataDataType::ContactCard => DataType::ContactCard, + MetadataDataType::FileNode => DataType::FileNode, + }) + .collect(), + vendor_namespaces: metadata.vendor_namespaces, + private_metadata: metadata.private_metadata, + max_depth: metadata + .max_depth + .map(|depth| depth.min(u32::MAX as u64) as u32), + max_entry_size: metadata.max_entry_size as usize, + max_size: metadata.max_size as usize, + max_private_size: metadata.max_private_size as usize, + max_entries: metadata.max_entries as usize, + query_max_scan: metadata.max_query_scan as usize, + imap_server_comment: metadata.imap_server_comment, + imap_server_admin: metadata.imap_server_admin, + } + } +} + +impl MetadataConfig { + pub fn limits(&self) -> MetadataLimits { + MetadataLimits { + max_depth: self.max_depth, + max_entry_size: self.max_entry_size, + max_size: self.max_size, + max_private_size: self.max_private_size, + max_entries: self.max_entries, + } + } +} diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 20d09f56..76c2c3a1 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -6,7 +6,9 @@ use std::io::Cursor; -use self::{mailstore::jmap::JmapConfig, smtp::SmtpConfig, storage::Storage}; +use self::{ + mailstore::jmap::JmapConfig, metadata::MetadataConfig, smtp::SmtpConfig, storage::Storage, +}; use crate::{ Core, Network, auth::oauth::config::OAuthConfig, @@ -17,7 +19,7 @@ use crate::{ use arc_swap::ArcSwap; use groupware::GroupwareConfig; use hyper::HeaderMap; -use p256::elliptic_curve::sec1::ToEncodedPoint; +use p256::elliptic_curve::sec1::ToSec1Point; use pkcs8::EncodePrivateKey; use rsa::{ RsaPrivateKey, @@ -31,28 +33,38 @@ use telemetry::Metrics; pub mod groupware; pub mod inner; pub mod mailstore; +pub mod metadata; pub mod network; +pub mod resource; pub mod server; pub mod smtp; pub mod storage; pub mod telemetry; impl Core { - pub async fn parse(bp: &mut Bootstrap, mut storage: Storage) -> Self { + pub async fn parse(bp: &mut Bootstrap, storage: Storage) -> Box { + Box::pin(Self::build(bp, storage)).await + } - Self { - sieve: Scripting::parse(bp).await, + async fn build(bp: &mut Bootstrap, storage: Storage) -> Box { + + let metadata = MetadataConfig::parse(bp).await; + let email = EmailConfig::parse(bp).await; + + Box::new(Self { + sieve: Scripting::parse(bp, &email.limits).await, network: Network::parse(bp).await, - smtp: Box::pin(SmtpConfig::parse(bp)).await, - jmap: JmapConfig::parse(bp).await, + smtp: SmtpConfig::parse(bp).await, + jmap: JmapConfig::parse(bp, &metadata).await, imap: ImapConfig::parse(bp).await, oauth: OAuthConfig::parse(bp).await, metrics: Metrics::parse(bp).await, spam: SpamFilterConfig::parse(bp).await, - email: EmailConfig::parse(bp).await, + email, groupware: GroupwareConfig::parse(bp).await, + metadata, storage, - } + }) } pub fn into_shared(self) -> ArcSwap { @@ -172,7 +184,7 @@ impl EcKeyCurve { let point = p256::SecretKey::from_pkcs8_der(pkcs8) .map_err(|err| format!("Failed to parse PKCS8 ECDSA key: {err}"))? .public_key() - .to_encoded_point(false); + .to_sec1_point(false); Ok(( point.x().map(|x| x.to_vec()).unwrap_or_default(), point.y().map(|y| y.to_vec()).unwrap_or_default(), @@ -182,7 +194,7 @@ impl EcKeyCurve { let point = p384::SecretKey::from_pkcs8_der(pkcs8) .map_err(|err| format!("Failed to parse PKCS8 ECDSA key: {err}"))? .public_key() - .to_encoded_point(false); + .to_sec1_point(false); Ok(( point.x().map(|x| x.to_vec()).unwrap_or_default(), point.y().map(|y| y.to_vec()).unwrap_or_default(), @@ -192,151 +204,3 @@ impl EcKeyCurve { } } -#[cfg(test)] -mod tests { - use super::{EcKeyCurve, build_ecdsa_pem, build_rsa_keypair}; - - const P256_SEC1: &str = "-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIJ9a6n/cu7XaQez5ZX8z8jDFkkfsMB1P9Vbqzbaes2zOoAoGCCqGSM49 -AwEHoUQDQgAEPCbID7bo+8Nk1vIsTFhVKwRWvb9GWTzzwS75Dd8iZuFl23Twn6Sp -V2ZO1FC0WyXxcVOMZN2sJFlCjtaQS+p5Zg== ------END EC PRIVATE KEY-----"; - - const P256_PKCS8: &str = "-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgn1rqf9y7tdpB7Pll -fzPyMMWSR+wwHU/1VurNtp6zbM6hRANCAAQ8JsgPtuj7w2TW8ixMWFUrBFa9v0ZZ -PPPBLvkN3yJm4WXbdPCfpKlXZk7UULRbJfFxU4xk3awkWUKO1pBL6nlm ------END PRIVATE KEY-----"; - - const P384_SEC1: &str = "-----BEGIN EC PRIVATE KEY----- -MIGkAgEBBDAeecJf8ju/70Nf5nbI4DeRo/+Z3VWXUvB+GwuUczew7fyMbyc6B3EE -BskOIqvqu6egBwYFK4EEACKhZANiAAQQjDW03Xn2h9ZmmCMRx+uRaLLfg4o2XITE -pwACH9EY4IjTe9LNNp5CTjERd+RlpWxkYopmDS5Trzycz9sDxxSzzXmq90vomJqt -fTnNHPFHuR2SAiwuzUf26rcPwa7DCWk= ------END EC PRIVATE KEY-----"; - - const P384_PKCS8: &str = "-----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAeecJf8ju/70Nf5nbI -4DeRo/+Z3VWXUvB+GwuUczew7fyMbyc6B3EEBskOIqvqu6ehZANiAAQQjDW03Xn2 -h9ZmmCMRx+uRaLLfg4o2XITEpwACH9EY4IjTe9LNNp5CTjERd+RlpWxkYopmDS5T -rzycz9sDxxSzzXmq90vomJqtfTnNHPFHuR2SAiwuzUf26rcPwa7DCWk= ------END PRIVATE KEY-----"; - - #[test] - fn ecdsa_pem_accepts_sec1_and_pkcs8() { - let sec1 = - build_ecdsa_pem(EcKeyCurve::P256, P256_SEC1).expect("P-256 SEC1 key should parse"); - let pkcs8 = - build_ecdsa_pem(EcKeyCurve::P256, P256_PKCS8).expect("P-256 PKCS8 key should parse"); - assert_eq!((&sec1.x, &sec1.y), (&pkcs8.x, &pkcs8.y)); - assert_eq!(sec1.x.len(), 32); - - let sec1 = - build_ecdsa_pem(EcKeyCurve::P384, P384_SEC1).expect("P-384 SEC1 key should parse"); - let pkcs8 = - build_ecdsa_pem(EcKeyCurve::P384, P384_PKCS8).expect("P-384 PKCS8 key should parse"); - assert_eq!((&sec1.x, &sec1.y), (&pkcs8.x, &pkcs8.y)); - assert_eq!(sec1.x.len(), 48); - } - - const RSA_PKCS1: &str = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAt5Kp7o945bxbnvviI9Kzbjcwi6B5cStu4dBbNhe/ld0Js4tQ\n8Uq9qYaFBlocYzWkEd3e2IG0+uIVB37ewUe0xjq25u6G4ZWeK+SZgzXB4jHinXvh\nuhHW+KzbmO+aYO115451Cu6ymJ8DLVaR6zxT9CJsiS4lMsYZ5JHcLY3az1A5z0df\nF+chjR+sLxdc0ggKqnX6fT/sVXHIlVk6riyeFV929k/v1f0pmRQ2nNu0NMSOK7Mk\nqsvHiAb1e/41LIwlbmbzd5ASHitYYXKP+2YR29SRr2D+52S1M29h4/XbUcP6Zo2U\np5mKgQ0kFZ8pHFhbruamzRp87+yhu98IbZ9ksQIDAQABAoIBAAu2+BGxhbNReR5U\n8Co9krZEntw2NjHG5glSkNOLoe4IIEudJyHy1VYpb7lHTFr3bBw4xrUV1+0PuuxS\nyBfZAdwJmKz1iVWBhQnDiZliN5h9+vp2UqIba9bMPypMFhO766OGh4kWUP7k3ODK\njr7Oh4QDo14AvB54nmPj/ANLM2y50/Upy5s7FK0tm0ntzxSscwQFSZAJ9B0ne6Qe\nu1/PXgiXW4JKNOgrCTrRB2BcOi/Ke6OA/kg54sD+Z9PZivO/qHTx9xXzqivmbg9a\nGmoivaWH/pKwAywFogJnWH/iTe+r//fKdlEDeK+s/iCr0ht//c0w+GxPvPF//wz0\n+1u9+n0CgYEA5M7jzpT8rCYWdORvvRP1BC4+A5jb83zXW4FS+rZRSmq775zDPAif\npm653vAlNHIphEvqSdVw64+36nJFtjuBI17BHCQi0j3iNVjrLC7lbfqIobnNDdmR\n9VeqZ6qwPYt2oi4iBY2dAnPdYVTDMomHSC4vW/SER0l9A9bxt3Co1a0CgYEAzWOQ\n490s6K186CyUMFrNrUmIWEJNd7b6JGI+oCioZLtPZzxO4ebc+bHEPbpbSqx7lJRJ\nt5u6zw/RwUc+6YXXImekvMfZpZMH9v1wjp3djnxGQO4ucmvmu6H25qcYup8tRtlo\n2AVLd1jg3yka1yr7O26M3bhVfm5LOUQfoLuCA5UCgYEA3Iw7882SfFE+RjBHMIcD\nHqOALTFzmhDU+SQAGyAP3V5ihwWg/sYFNYT3btgl1JbSQ+51B/RQIw9mJPs/DPfw\nc2qLU5fVZLg3ylpKXU1a4xaiCtmwuM/mLAnzfHd/5+L9WDiFnLqzBEEwu/fbK2R7\nXOz/w3A+7QP+F+xhFAPpCgUCgYEAsnZOIkA/UlnUi6SYir+LsYOQLihGSbw687xN\n8DoDv6sl3mz/mbhQz8GP45b21hazNrH2r8xn8J0tRATU/HIoMaPe942rZvwv0oP6\n9mDjb3g6TxbmUtPA485iy53rldTTsZkdSX6oSSZ4FlAQG2AkdkqjqdAOsVHCmRrB\nZJco7FUCgYBwk7tQt3YS5b0wi8fH3BIfAH31vJ2VGlAin860H8FXjAj8EZ7Ff9Iq\n5dQIyPbp89TOSxIVxPGniI2ruLy4DZQM7xa42oyxyRir4UeHN2P5D2yEAHaVSwSd\nO6yiiOBj62OATapI8BqeFJZGRFltDsj6XbwC/Z9S2tRKCuE/zp+FLg==\n-----END RSA PRIVATE KEY-----"; - - const RSA_PKCS8: &str = "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3kqnuj3jlvFue\n++Ij0rNuNzCLoHlxK27h0Fs2F7+V3Qmzi1DxSr2phoUGWhxjNaQR3d7YgbT64hUH\nft7BR7TGOrbm7obhlZ4r5JmDNcHiMeKde+G6Edb4rNuY75pg7XXnjnUK7rKYnwMt\nVpHrPFP0ImyJLiUyxhnkkdwtjdrPUDnPR18X5yGNH6wvF1zSCAqqdfp9P+xVcciV\nWTquLJ4VX3b2T+/V/SmZFDac27Q0xI4rsySqy8eIBvV7/jUsjCVuZvN3kBIeK1hh\nco/7ZhHb1JGvYP7nZLUzb2Hj9dtRw/pmjZSnmYqBDSQVnykcWFuu5qbNGnzv7KG7\n3whtn2SxAgMBAAECggEAC7b4EbGFs1F5HlTwKj2StkSe3DY2McbmCVKQ04uh7ggg\nS50nIfLVVilvuUdMWvdsHDjGtRXX7Q+67FLIF9kB3AmYrPWJVYGFCcOJmWI3mH36\n+nZSohtr1sw/KkwWE7vro4aHiRZQ/uTc4MqOvs6HhAOjXgC8HnieY+P8A0szbLnT\n9SnLmzsUrS2bSe3PFKxzBAVJkAn0HSd7pB67X89eCJdbgko06CsJOtEHYFw6L8p7\no4D+SDniwP5n09mK87+odPH3FfOqK+ZuD1oaaiK9pYf+krADLAWiAmdYf+JN76v/\n98p2UQN4r6z+IKvSG3/9zTD4bE+88X//DPT7W736fQKBgQDkzuPOlPysJhZ05G+9\nE/UELj4DmNvzfNdbgVL6tlFKarvvnMM8CJ+mbrne8CU0cimES+pJ1XDrj7fqckW2\nO4EjXsEcJCLSPeI1WOssLuVt+oihuc0N2ZH1V6pnqrA9i3aiLiIFjZ0Cc91hVMMy\niYdILi9b9IRHSX0D1vG3cKjVrQKBgQDNY5Dj3SzorXzoLJQwWs2tSYhYQk13tvok\nYj6gKKhku09nPE7h5tz5scQ9ultKrHuUlEm3m7rPD9HBRz7phdciZ6S8x9mlkwf2\n/XCOnd2OfEZA7i5ya+a7ofbmpxi6ny1G2WjYBUt3WODfKRrXKvs7bozduFV+bks5\nRB+gu4IDlQKBgQDcjDvzzZJ8UT5GMEcwhwMeo4AtMXOaENT5JAAbIA/dXmKHBaD+\nxgU1hPdu2CXUltJD7nUH9FAjD2Yk+z8M9/BzaotTl9VkuDfKWkpdTVrjFqIK2bC4\nz+YsCfN8d3/n4v1YOIWcurMEQTC799srZHtc7P/DcD7tA/4X7GEUA+kKBQKBgQCy\ndk4iQD9SWdSLpJiKv4uxg5AuKEZJvDrzvE3wOgO/qyXebP+ZuFDPwY/jlvbWFrM2\nsfavzGfwnS1EBNT8cigxo973jatm/C/Sg/r2YONveDpPFuZS08DjzmLLneuV1NOx\nmR1JfqhJJngWUBAbYCR2SqOp0A6xUcKZGsFklyjsVQKBgHCTu1C3dhLlvTCLx8fc\nEh8AffW8nZUaUCKfzrQfwVeMCPwRnsV/0irl1AjI9unz1M5LEhXE8aeIjau4vLgN\nlAzvFrjajLHJGKvhR4c3Y/kPbIQAdpVLBJ07rKKI4GPrY4BNqkjwGp4UlkZEWW0O\nyPpdvAL9n1La1EoK4T/On4Uu\n-----END PRIVATE KEY-----"; - - #[test] - fn rsa_pem_accepts_pkcs1_and_pkcs8() { - let a = build_rsa_keypair(RSA_PKCS1).expect("PKCS1 RSA key should parse"); - let b = build_rsa_keypair(RSA_PKCS8).expect("PKCS8 RSA key should parse"); - assert_eq!(a.modulus, b.modulus); - assert_eq!(a.exponent, b.exponent); - assert_eq!(a.pkcs1_der, b.pkcs1_der); - assert_eq!(a.modulus.len(), 256); - } - - #[test] - fn signing_keys_are_accepted_by_jsonwebtoken() { - use jsonwebtoken::{Algorithm, EncodingKey, Header}; - - #[derive(serde::Serialize)] - struct Claims { - sub: &'static str, - } - - let claims = Claims { sub: "test" }; - - for (curve, pem, alg) in [ - (EcKeyCurve::P256, P256_SEC1, Algorithm::ES256), - (EcKeyCurve::P256, P256_PKCS8, Algorithm::ES256), - (EcKeyCurve::P384, P384_SEC1, Algorithm::ES384), - (EcKeyCurve::P384, P384_PKCS8, Algorithm::ES384), - ] { - let key = build_ecdsa_pem(curve, pem).expect("key should parse"); - jsonwebtoken::encode( - &Header::new(alg), - &claims, - &EncodingKey::from_ec_der(&key.pkcs8_der), - ) - .unwrap_or_else(|err| panic!("{alg:?} signing failed: {err}")); - } - - for pem in [RSA_PKCS1, RSA_PKCS8] { - let key = build_rsa_keypair(pem).expect("key should parse"); - for alg in [Algorithm::RS256, Algorithm::PS512] { - jsonwebtoken::encode( - &Header::new(alg), - &claims, - &EncodingKey::from_rsa_der(&key.pkcs1_der), - ) - .unwrap_or_else(|err| panic!("{alg:?} signing failed: {err}")); - } - } - } - - #[test] - fn ecdsa_pem_rejects_keyless_pem() { - let err = match build_ecdsa_pem( - EcKeyCurve::P256, - "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", - ) { - Ok(_) => panic!("expected a certificate-only PEM to be rejected"), - Err(err) => err, - }; - assert!(err.contains("No usable ECDSA private key"), "{err}"); - } - - const P256_ENCRYPTED: &str = "-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIH0MF8GCSqGSIb3DQEFDTBSMDEGCSqGSIb3DQEFDDAkBBCoW4qsep9YbFLRW2u4\nk8ljAgIIADAMBggqhkiG9w0CCQUAMB0GCWCGSAFlAwQBKgQQZYoEYHqh+y9uqT70\n6521jwSBkA9dcdq6hT/7Fzqcu0wX3QVr+8g1Kxc6tCV9dLShi8VU2ax8jG3zZt3h\nBp1CLyX8UfT98SujtoH36PEXPDDTralcP6vWViqGx5AagT4DRFjcI8yucTUXkLoD\n9ZIRBVPviTeznEHt3OvCCMuO76rsyu/gxNC7D46TBtq8JX1OFcaXPctpN8l5GKqH\nh4gA6av3og==\n-----END ENCRYPTED PRIVATE KEY-----"; - - #[test] - fn ecdsa_pem_reports_password_protected_key() { - let err = match build_ecdsa_pem(EcKeyCurve::P256, P256_ENCRYPTED) { - Ok(_) => panic!("expected an encrypted key to be rejected"), - Err(err) => err, - }; - assert!(err.contains("password-protected"), "{err}"); - } - - #[test] - fn rsa_pem_rejects_undersized_modulus() { - const RSA_1024: &str = "-----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCz6gcAg0f+2/HFrudtMfRSylyzI8W/lNmPQZhUpz+R6D/7/+4/ -HFKsUcZIi+nzdOnrzW/kw19nVOKk2ylAUNV9d2TR75HqrPBYsu0LCDRidb9XOyhY -bQJII1KuFlaWNjfxG28Tlg//FdVPPkn/oTQwnhvMcWCK3Hatho6cx9uzWwIDAQAB -AoGBAIPvjwr1OwrOyFIrnVMaWw2LkMdd6FpCEflYJRmPPLMHGkT2vgRSBN6RaVMy -J3J9vj1J/lBIZeIlAb/baDjeDnAj5GBzCB319oxnBuZSmpyYntW1DEsdhbK0Yeu+ -7v05oXBXfzdZvGBWYrwlj5ipoHQo0R+WN4NVXqJFwiagaGBBAkEA6qwv5Pww54za -fHNUD1M6MKRBk5m0Y/GJ58sWmnmFJI6I3sHBIfcy5lylm5KecduzSKoVtAUUbNWf -KOKoZcKHMwJBAMRD2WDEd5+8q5ZxzYG0x5sEdz1lhJkt+YSbudNgfE1kPDDrCE0V -8+hgNdp6Mj1hfihwB0hTCcnaPsXLl9AyAzkCQDVU+HWD0uFso2LRGvN4qKrRSY3v -yo1EIWEqSHLG1zldo0FsqyW69jhgKcrXYWbi1TXYYaJN3Tx2t/skt7yYnv0CQDtD -NYdHq8tbAADcei5ZNRB058BtP/206SwGjbTq5H3F73rh7U7BezXGn1xKG5N3Nc3m -DfzjvgfqU5wMHtopz9kCQBw3AAiRCY8Y0UgejUtu8tXIK76qebaNcMCabBnFrAqV -A4Oj1c5BcOHVtww9W6NeiiRMJpUNN71gmyjsnOyT3cY= ------END RSA PRIVATE KEY-----"; - - let err = match build_rsa_keypair(RSA_1024) { - Ok(_) => panic!("expected a 1024-bit RSA key to be rejected"), - Err(err) => err, - }; - assert!(err.contains("1024 bits"), "{err}"); - } -} diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index be4d848e..5afaa80d 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::*; +use super::{resource::ResourceIdentifiers, *}; use crate::{ expr::if_block::{BootstrapExprExt, IfBlock}, network::{ @@ -12,9 +12,11 @@ use crate::{ Authentication, Configuration, HttpServer, Info, Logo, OAuthPublic, Protocols, Provider, TextServer, }, + forwarded::RemoteIpSource, security::Security, }, }; +use directory::backend::oidc::ExternalScopes; use mail_builder::mime::make_boundary; use registry::schema::{ enums::{AcmeChallengeType, ClusterTaskType, ProviderInfo, ServiceProtocol}, @@ -25,7 +27,10 @@ use registry::schema::{ }, }; use std::{str::FromStr, time::Duration}; -use utils::map::vec_map::VecMap; +use utils::{map::vec_map::VecMap, memory::available_memory}; + +const BUFFERED_MEMORY_SHARE: u64 = 4; +const FALLBACK_BUFFERED_MEMORY: u64 = 1024 * 1024 * 1024; #[derive(Clone)] pub struct Network { @@ -39,7 +44,10 @@ pub struct Network { pub task_manager: TaskManager, pub has_acme_tls_challenge: bool, pub has_acme_http_challenge: bool, + pub max_buffered_memory: u64, + pub include_secrets: bool, pub info: NetworkInfo, + pub resources: ResourceIdentifiers, } #[derive(Clone)] @@ -53,6 +61,8 @@ pub struct NetworkInfo { pub struct Pacc { pub prefix: String, pub suffix: String, + pub password_only: String, + pub oauth_scopes: ExternalScopes, } #[derive(Clone)] @@ -62,7 +72,7 @@ pub struct Http { pub url_https: String, pub allowed_endpoint: IfBlock, pub response_headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, - pub use_forwarded: bool, + pub remote_ip_source: RemoteIpSource, pub redirect_root: Option, } @@ -99,6 +109,7 @@ pub enum AsnGeoLookupConfig { timeout: Duration, max_size: usize, headers: HeaderMap, + allow_invalid_certs: bool, asn_resources: Vec, geo_resources: Vec, }, @@ -320,22 +331,58 @@ impl Network { .rsplit_once(SPLIT_HERE) .map(|(prefix, suffix)| (prefix.to_string(), suffix.to_string())) .unwrap(); + pacc.authentication = Some(Authentication { + oauth_public: None, + password: true, + }); + let password_only = serde_json::to_string(&pacc).unwrap_or_default(); + let mut oauth_scopes = ExternalScopes::MAIL; + for service in system.services.keys() { + oauth_scopes.insert(match service { + ServiceProtocol::Caldav => ExternalScopes::CALENDARS, + ServiceProtocol::Carddav => ExternalScopes::CONTACTS, + ServiceProtocol::Webdav => ExternalScopes::FILES, + _ => ExternalScopes::NONE, + }); + } + let server_name = default_hostname.to_string(); + let http = Http::parse(bp, &http_host).await; + let resources = ResourceIdentifiers::build(&http.url_https, &system.services, &server_name); let mut network = Network { node_id: bp.node_id() as u64, - server_name: default_hostname.to_string(), + server_name, security: Security::parse(bp).await, contact_form: ContactForm::parse(bp).await, asn_geo_lookup: AsnGeoLookupConfig::parse(bp).await.unwrap_or_default(), roles: ClusterRoles::default(), - http: Http::parse(bp, &http_host).await, + http, task_manager: bp.setting_infallible::().await, has_acme_tls_challenge, has_acme_http_challenge, + max_buffered_memory: system.max_buffered_memory.unwrap_or_else(|| { + available_memory().map_or_else( + || { + bp.build_warning( + ObjectType::SystemSettings.singleton(), + "Unable to detect the available memory, using 1 GiB for maxBufferedMemory", + ); + FALLBACK_BUFFERED_MEMORY + }, + |memory| memory / BUFFERED_MEMORY_SHARE, + ) + }), + include_secrets: system.include_secrets, info: NetworkInfo { mxs: system.mail_exchangers.into_iter().collect(), services: system.services, - pacc: Pacc { prefix, suffix }, + pacc: Pacc { + prefix, + suffix, + password_only, + oauth_scopes, + }, }, + resources, }; if let Some(role) = &bp.role { @@ -371,10 +418,7 @@ impl Network { } impl Http { - #[cfg_attr( - any(feature = "dev_mode", feature = "test_mode"), - allow(unused_variables) - )] + #[cfg_attr(any(feature = "dev_mode", stalwart_tests), allow(unused_variables))] pub async fn parse(bp: &mut Bootstrap, server_name: &str) -> Self { let http = bp.setting_infallible::().await; @@ -435,7 +479,7 @@ impl Http { )); } - #[cfg(any(feature = "dev_mode", feature = "test_mode"))] + #[cfg(any(feature = "dev_mode", stalwart_tests))] let server_name = "127.0.0.1"; Http { @@ -464,7 +508,7 @@ impl Http { http.rate_limit_anonymous }, response_headers: http_headers, - use_forwarded: http.use_x_forwarded, + remote_ip_source: RemoteIpSource::new(http.client_ip_source, http.forwarded_hops), redirect_root: http.redirect_root, } } @@ -474,7 +518,7 @@ impl AsnGeoLookupConfig { pub async fn parse(bp: &mut Bootstrap) -> Option { match bp.setting_infallible::().await { Asn::Resource(asn) => Some(AsnGeoLookupConfig::Resource { - expires: asn.expires.into_inner(), + expires: asn.refresh_interval.into_inner(), timeout: asn.timeout.into_inner(), max_size: asn.max_size as usize, headers: asn @@ -488,6 +532,7 @@ impl AsnGeoLookupConfig { ) }) .ok()?, + allow_invalid_certs: asn.allow_invalid_certs, asn_resources: asn.asn_urls.into_inner(), geo_resources: asn.geo_urls.into_inner(), }), diff --git a/crates/common/src/config/resource.rs b/crates/common/src/config/resource.rs new file mode 100644 index 00000000..8734b231 --- /dev/null +++ b/crates/common/src/config/resource.rs @@ -0,0 +1,139 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use registry::schema::{enums::ServiceProtocol, structs::Service}; +use reqwest::Url; +use utils::map::vec_map::VecMap; + +#[derive(Clone, Default, Debug, PartialEq, Eq)] +pub struct ResourceIdentifiers { + pub identifiers: Box<[Box]>, + pub trusted_hosts: Box<[Box]>, + pub registrable_domain: Option>, + pub serves_files: bool, +} + +impl ResourceIdentifiers { + pub fn build( + url_https: &str, + services: &VecMap, + default_hostname: &str, + ) -> Self { + let mut trusted_hosts: Vec> = Vec::with_capacity(services.len() + 1); + let mut bare_hosts: Vec> = Vec::with_capacity(services.len()); + let mut registrable_domain = None; + let mut serves_files = false; + + if let Ok(url) = Url::parse(url_https) + && let Some(host) = url.host_str().filter(|host| !host.is_empty()) + { + let authority: Box = match url.port() { + Some(port) => format!("{host}:{port}").into_boxed_str(), + None => host.into(), + }; + registrable_domain = url.domain().and_then(psl::domain_str).map(Box::from); + push_unique(&mut trusted_hosts, authority); + } + + for (service, details) in services { + let Some(host) = + normalize_host(details.hostname.as_deref().unwrap_or(default_hostname)) + else { + continue; + }; + serves_files |= matches!(service, ServiceProtocol::Jmap | ServiceProtocol::Webdav); + match service { + ServiceProtocol::Jmap + | ServiceProtocol::Caldav + | ServiceProtocol::Carddav + | ServiceProtocol::Webdav => push_unique(&mut trusted_hosts, host), + ServiceProtocol::Imap + | ServiceProtocol::Pop3 + | ServiceProtocol::Smtp + | ServiceProtocol::Managesieve => push_unique(&mut bare_hosts, host), + } + } + + let mut identifiers = Vec::with_capacity(trusted_hosts.len() + bare_hosts.len()); + identifiers.extend( + trusted_hosts + .iter() + .map(|host| format!("https://{host}").into_boxed_str()), + ); + identifiers.extend(bare_hosts); + + ResourceIdentifiers { + identifiers: identifiers.into_boxed_slice(), + trusted_hosts: trusted_hosts.into_boxed_slice(), + registrable_domain, + serves_files, + } + } + + pub fn is_known_resource(&self, resource: &str) -> bool { + if resource.contains([':', '/']) { + let Ok(url) = Url::parse(resource) else { + return false; + }; + url.scheme() == "https" + && url.fragment().is_none() + && url.username().is_empty() + && url.password().is_none() + && url.host_str().is_some_and(|host| { + self.trusted_hosts + .iter() + .any(|trusted| authority_matches(trusted, host, url.port())) + }) + } else { + normalize_host(resource).is_some_and(|host| { + self.identifiers + .iter() + .filter(|identifier| !identifier.starts_with("https://")) + .any(|identifier| identifier.as_ref() == host.as_ref()) + }) + } + } + + pub fn is_trusted_host(&self, authority: &str) -> bool { + self.trusted_hosts + .iter() + .any(|host| host.eq_ignore_ascii_case(authority)) + } +} + +fn normalize_host(host: &str) -> Option> { + let host = host.trim(); + let host = host.strip_suffix('.').unwrap_or(host); + if host.is_empty() { + None + } else if host.is_ascii() { + Some(host.to_ascii_lowercase().into_boxed_str()) + } else { + idna::domain_to_ascii(host).ok().map(String::into_boxed_str) + } +} + +fn authority_matches(trusted: &str, host: &str, port: Option) -> bool { + let (Some(trusted_host), Some(trusted_port)) = + (trusted.get(..host.len()), trusted.get(host.len()..)) + else { + return false; + }; + trusted_host.eq_ignore_ascii_case(host) + && match port { + Some(port) => trusted_port + .strip_prefix(':') + .is_some_and(|trusted_port| trusted_port.parse::().ok() == Some(port)), + None => trusted_port.is_empty(), + } +} + +fn push_unique(list: &mut Vec>, value: Box) { + if !list.contains(&value) { + list.push(value); + } +} + diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index 869015d3..2d7e4e7f 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -35,6 +35,15 @@ use tokio_rustls::TlsAcceptor; use types::id::Id; use utils::snowflake::SnowflakeIdGenerator; +const DEFAULT_RECOVERY_MODE_PORT: u16 = 8080; + +pub fn recovery_mode_port() -> u16 { + std::env::var("STALWART_RECOVERY_MODE_PORT") + .ok() + .and_then(|port| port.parse().ok()) + .unwrap_or(DEFAULT_RECOVERY_MODE_PORT) +} + impl Listeners { pub async fn parse(bp: &mut Bootstrap) -> Self { // Parse ACME managers @@ -67,18 +76,12 @@ impl Listeners { id: ObjectId::new(ObjectType::NetworkListener, Id::singleton()), object: NetworkListener { bind: Map::new(vec![ - SocketAddr::from_str(&format!( - "[::]:{}", - std::env::var("STALWART_RECOVERY_MODE_PORT") - .ok() - .and_then(|p| p.parse::().ok()) - .unwrap_or(8080) - )) - .unwrap(), + SocketAddr::from_str(&format!("[::]:{}", recovery_mode_port())) + .unwrap(), ]), name: "http-recovery".to_string(), protocol: NetworkListenerProtocol::Http, - tls_implicit: false, + implicit_tls: false, ..Default::default() }, revision: 0, @@ -321,7 +324,7 @@ impl Listeners { TcpAcceptor::Tls { acceptor: TlsAcceptor::from(default_config.clone()), config: default_config, - implicit: listener.tls_implicit, + implicit: listener.implicit_tls, } } else { TcpAcceptor::Plain diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index df5660f1..28ea966b 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::network::acme::ParsedCert; +use crate::network::{acme::ParsedCert, tls::lookup_certificate}; use ahash::{AHashMap, AHashSet}; use rcgen::generate_simple_self_signed; use registry::{ @@ -12,7 +12,7 @@ use registry::{ prelude::Object, structs::{Certificate, PublicText, SecretText, SystemSettings}, }, - types::{datetime::UTCDateTime, map::Map}, + types::map::Map, }; use rustls::{ SupportedProtocolVersion, @@ -61,10 +61,8 @@ pub(crate) async fn parse_certificates( }; match ParsedCert::parse(&pem) { Ok(parsed) => { - let not_valid_after = - UTCDateTime::from_timestamp(parsed.valid_not_after.timestamp()); - let not_valid_before = - UTCDateTime::from_timestamp(parsed.valid_not_before.timestamp()); + let not_valid_after = parsed.valid_not_after; + let not_valid_before = parsed.valid_not_before; let sans = Map::new(parsed.sans); if cert.not_valid_after != not_valid_after || cert.not_valid_before != not_valid_before @@ -180,12 +178,7 @@ pub(crate) async fn parse_certificates( let key = Arc::new(key); for name in cert.subject_alternative_names.into_inner() { subject_names.insert(name.as_str().into()); - certificates.insert( - name.strip_prefix("*.") - .map(Into::into) - .unwrap_or_else(|| name.into_boxed_str()), - key.clone(), - ); + certificates.insert(name.into_boxed_str(), key.clone()); } } Err(err) => { @@ -194,6 +187,14 @@ pub(crate) async fn parse_certificates( } } + if !certificates.contains_key("*") + && !system.default_hostname.is_empty() + && let Some(key) = + lookup_certificate(certificates, system.default_hostname.as_str()).cloned() + { + certificates.insert("*".into(), key); + } + // Remove expired certificates if !certs_expired.is_empty() { for (id, sans, object) in certs_expired { diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 55805311..98545be7 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -8,21 +8,26 @@ use crate::expr::{ self, if_block::{BootstrapExprExt, IfBlock}, }; +use compact_str::ToCompactString; +use encodify::{Error as CodecError, base64::LENIENT, pem}; use mail_auth::{ - common::crypto::{Ed25519Key, HashAlgorithm, RsaKey, Sha256, SigningKey}, - dkim::{Canonicalization, Done}, - dkim2::{Dkim2Signer, Done as Dkim2Done, Flag}, + crypto::{DkimKey, Ed25519Key, HashAlgorithm, MlDsa44Ed25519Key, RsaKey, Sha256, SigningKey}, + dkim::Canonicalization, + dkim2::{Dkim2Hash, Dkim2Signer, Flag}, + signer::Ready, }; -use mail_parser::decoders::base64::base64_decode; use registry::{ schema::{ - enums::{self, Dkim2Flag, ExpressionConstant}, + enums::{self, Dkim2Flag, Dkim2HashAlgorithm, ExpressionConstant}, prelude::ObjectType, - structs::{Dkim1Signature, DkimSignature, SenderAuth}, + structs::{ + Dkim1Signature, Dkim2Signature, DkimSignature, MtaStageData, SecretText, SenderAuth, + }, }, - types::{ObjectImpl, map::Map}, + types::ObjectImpl, }; use rustls_pki_types::{PrivateKeyDer, PrivatePkcs1KeyDer, PrivatePkcs8KeyDer, pem::PemObject}; +use std::borrow::Cow; use store::registry::bootstrap::Bootstrap; use utils::cache::CacheItemWeight; @@ -45,7 +50,6 @@ pub struct DkimAuthConfig { #[derive(Clone)] pub struct ArcAuthConfig { pub verify: IfBlock, - //pub seal: IfBlock, } #[derive(Clone)] @@ -73,33 +77,38 @@ pub enum VerifyStrategy { } pub enum Dkim1Signer { - RsaSha256(mail_auth::dkim::DkimSigner, Done>), - Ed25519Sha256(mail_auth::dkim::DkimSigner), + RsaSha256(mail_auth::dkim::DkimSigner, Ready>), + Ed25519Sha256(mail_auth::dkim::DkimSigner), +} + +pub struct Dkim2Signers { + pub signer: Dkim2Signer, + pub hashes: Vec, } #[derive(Default)] pub struct DkimSigners { pub dkim1: Vec, - pub dkim2: Option>, + pub dkim2: Option, } impl MailAuthConfig { pub async fn parse(bp: &mut Bootstrap) -> Self { let auth = bp.setting_infallible::().await; + let data = bp.setting_infallible::().await; MailAuthConfig { dkim: DkimAuthConfig { verify: bp .compile_expr(ObjectType::SenderAuth.singleton(), &auth.ctx_dkim_verify()), sign: bp.compile_expr( - ObjectType::SenderAuth.singleton(), - &auth.ctx_dkim_sign_domain(), + ObjectType::MtaStageData.singleton(), + &data.ctx_dkim_sign_domain(), ), strict: auth.dkim_strict, }, arc: ArcAuthConfig { verify: bp.compile_expr(ObjectType::SenderAuth.singleton(), &auth.ctx_arc_verify()), - //seal: bp.compile_expr(ObjectType::SenderAuth.singleton(), &auth.ctx_arc_seal_domain()), }, spf: SpfAuthConfig { verify_ehlo: bp.compile_expr( @@ -134,104 +143,131 @@ impl DkimSigners { .details( errors .into_iter() - .map(|v| trc::Value::from(v.to_string())) + .map(|v| trc::Value::from(v.to_compact_string())) .collect::>(), )); } match signature { DkimSignature::Dkim1Ed25519Sha256(signature) => { - let private_key = signature - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err))?; - let private_key = simple_pem_parse(&private_key).ok_or_else(|| { - trc::DkimEvent::BuildError - .reason("Failed to parse ED25519 private key PEM") - .details("Invalid PEM format") - })?; - let key = - Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key).map_err(|err| { - trc::DkimEvent::BuildError - .reason(err) - .details("Failed to build ED25519 key") - })?; - + let key = ed25519_key_parse(&private_key(&signature.private_key).await?)?; self.dkim1 .push(Dkim1Signer::Ed25519Sha256(build_dkim1_signer( domain, signature, key, ))); } DkimSignature::Dkim1RsaSha256(signature) => { - let private_key = signature - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err))?; - let key = rsa_key_parse(private_key.as_bytes())?; - + let key = rsa_key_parse(private_key(&signature.private_key).await?.as_bytes())?; self.dkim1.push(Dkim1Signer::RsaSha256(build_dkim1_signer( domain, signature, key, ))); } DkimSignature::Dkim2Ed25519Sha256(signature) => { - let private_key = signature - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err))?; - let private_key = simple_pem_parse(&private_key).ok_or_else(|| { - trc::DkimEvent::BuildError - .reason("Failed to parse ED25519 private key PEM") - .details("Invalid PEM format") - })?; - let key = - Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key).map_err(|err| { - trc::DkimEvent::BuildError - .reason(err) - .details("Failed to build ED25519 key") - })?; - - self.dkim2 = Some(match self.dkim2.take() { - None => Dkim2Signer::from_key(key) - .domain(domain) - .selector(signature.selector) - .flags(map_dkim2_flags(signature.flags)), - Some(signer) => signer - .additional_key(key, signature.selector) - .flags(map_dkim2_flags(signature.flags)), - }); + self.insert_dkim2(domain, signature, |pem| { + ed25519_key_parse(pem).map(DkimKey::from) + }) + .await?; } DkimSignature::Dkim2RsaSha256(signature) => { - let private_key = signature - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err))?; - let key = rsa_key_parse(private_key.as_bytes())?; - - self.dkim2 = Some(match self.dkim2.take() { - None => Dkim2Signer::from_key(key) - .domain(domain) - .selector(signature.selector) - .flags(map_dkim2_flags(signature.flags)), - Some(signer) => signer - .additional_key(key, signature.selector) - .flags(map_dkim2_flags(signature.flags)), - }); + self.insert_dkim2(domain, signature, |pem| { + rsa_key_parse(pem.as_bytes()).map(DkimKey::from) + }) + .await?; + } + DkimSignature::Dkim2MlDsa44Ed25519Sha512(signature) => { + self.insert_dkim2(domain, signature, |pem| { + mldsa44_ed25519_key_parse(pem).map(DkimKey::from) + }) + .await?; } } Ok(()) } + + async fn insert_dkim2( + &mut self, + domain: String, + signature: Dkim2Signature, + parse_key: fn(&str) -> trc::Result, + ) -> trc::Result<()> { + let key = parse_key(&private_key(&signature.private_key).await?)?; + let flags = signature + .flags + .into_inner() + .into_iter() + .map(|flag| match flag { + Dkim2Flag::Feedback => Flag::Feedback, + }); + let hashes = signature + .instance_hashes + .into_inner() + .into_iter() + .map(|hash| match hash { + Dkim2HashAlgorithm::Sha256 => Dkim2Hash::Sha256, + Dkim2HashAlgorithm::Sha512 => Dkim2Hash::Sha512, + }); + + let (signer, mut all_hashes) = match self.dkim2.take() { + None => ( + Dkim2Signer::from_key(key) + .domain(domain) + .selector(signature.selector), + Vec::with_capacity(2), + ), + Some(signers) => ( + signers.signer.additional_key(key, signature.selector), + signers.hashes, + ), + }; + for hash in hashes { + if !all_hashes.contains(&hash) { + all_hashes.push(hash); + } + } + if all_hashes.is_empty() { + all_hashes.push(Dkim2Hash::Sha256); + } + + self.dkim2 = Some(Dkim2Signers { + signer: signer.flags(flags).hashes(all_hashes.iter().copied()), + hashes: all_hashes, + }); + + Ok(()) + } } -fn map_dkim2_flags(flags: Map) -> impl Iterator { - flags.into_inner().into_iter().map(|flag| match flag { - Dkim2Flag::Donotmodify => Flag::DoNotModify, - Dkim2Flag::Donotexplode => Flag::DoNotExplode, - Dkim2Flag::Feedback => Flag::Feedback, +async fn private_key(secret: &SecretText) -> trc::Result> { + secret + .secret() + .await + .map_err(|err| trc::DkimEvent::BuildError.reason(err)) +} + +fn ed25519_key_parse(pem: &str) -> trc::Result { + let der = pem_or_base64_decode(pem).map_err(|_| { + trc::DkimEvent::BuildError + .reason("Failed to parse ED25519 private key PEM") + .details("Invalid PEM format") + })?; + Ed25519Key::from_pkcs8_maybe_unchecked_der(&der).map_err(|err| { + trc::DkimEvent::BuildError + .reason(err) + .details("Failed to build ED25519 key") + }) +} + +fn mldsa44_ed25519_key_parse(pem: &str) -> trc::Result { + let der = pem_or_base64_decode(pem).map_err(|_| { + trc::DkimEvent::BuildError + .reason("Failed to parse ML-DSA-44/Ed25519 private key PEM") + .details("Invalid PEM format") + })?; + MlDsa44Ed25519Key::from_pkcs8_der(&der).map_err(|err| { + trc::DkimEvent::BuildError + .reason(err) + .details("Failed to build ML-DSA-44/Ed25519 key") }) } @@ -253,40 +289,19 @@ pub fn rsa_key_parse(private_key: &[u8]) -> trc::Result> { }) } -pub fn simple_pem_parse(contents: &str) -> Option> { - let mut contents = contents.as_bytes().iter().copied(); - let mut base64 = vec![]; - - 'outer: while let Some(ch) = contents.next() { - if !ch.is_ascii_whitespace() { - if ch == b'-' { - for ch in contents.by_ref() { - if ch == b'\n' { - break; - } - } - } else { - base64.push(ch); - } - - for ch in contents.by_ref() { - if ch == b'-' { - break 'outer; - } else if !ch.is_ascii_whitespace() { - base64.push(ch); - } - } - } +pub fn pem_or_base64_decode(contents: &str) -> Result, CodecError> { + match pem::STANDARD.decode(contents.as_bytes()) { + Ok(block) => Ok(block.contents), + Err(CodecError::NotFound) => LENIENT.decode(contents), + Err(err) => Err(err), } - - base64_decode(&base64) } fn build_dkim1_signer( domain: String, signature: Dkim1Signature, key: T, -) -> mail_auth::dkim::DkimSigner { +) -> mail_auth::dkim::DkimSigner { let mut signer = mail_auth::dkim::DkimSigner::from_key(key) .domain(domain) .selector(signature.selector) @@ -317,11 +332,11 @@ fn build_dkim1_signer( } if let Some(expire) = signature.expire { - signer = signer.expiration(expire.into_inner().as_secs()); + signer = signer.expiration(expire.into_inner()); } if let Some(auid) = signature.auid { - signer = signer.agent_user_identifier(auid); + signer = signer.identity(auid); } if let Some(atps) = signature.third_party { @@ -329,7 +344,7 @@ fn build_dkim1_signer( } if let Some(atpsh) = signature.third_party_hash { - signer = signer.atpsh(match atpsh { + signer = signer.atps_hash(match atpsh { enums::DkimHash::Sha256 => HashAlgorithm::Sha256, enums::DkimHash::Sha1 => HashAlgorithm::Sha1, }); @@ -365,16 +380,30 @@ impl VerifyStrategy { } } +impl Dkim1Signer { + fn heap_size(&self) -> usize { + match self { + Dkim1Signer::RsaSha256(signer) => signer.heap_size(), + Dkim1Signer::Ed25519Sha256(signer) => signer.heap_size(), + } + } +} + impl CacheItemWeight for Dkim1Signer { fn weight(&self) -> u64 { - std::mem::size_of::() as u64 + (std::mem::size_of::() + self.heap_size()) as u64 } } impl CacheItemWeight for DkimSigners { fn weight(&self) -> u64 { (std::mem::size_of::() - + self.dkim1.len() * std::mem::size_of::() - + std::mem::size_of::>()) as u64 + + self.dkim1.capacity() * std::mem::size_of::() + + self.dkim1.iter().map(Dkim1Signer::heap_size).sum::() + + self.dkim2.as_ref().map_or(0, |dkim2| { + dkim2.signer.heap_size() + + dkim2.hashes.capacity() * std::mem::size_of::() + })) as u64 } } + diff --git a/crates/common/src/config/smtp/mod.rs b/crates/common/src/config/smtp/mod.rs index 3268d084..db72842b 100644 --- a/crates/common/src/config/smtp/mod.rs +++ b/crates/common/src/config/smtp/mod.rs @@ -20,6 +20,7 @@ use registry::{ types::id::ObjectId, }; use store::registry::bootstrap::Bootstrap; +use utils::outbound::{OutboundClient, OutboundPolicy, Redirects}; #[derive(Clone)] pub struct SmtpConfig { @@ -28,12 +29,12 @@ pub struct SmtpConfig { pub resolvers: Resolvers, pub mail_auth: MailAuthConfig, pub report: ReportConfig, - pub mta_sts_client: reqwest::Client, - pub tls_report_client: reqwest::Client, + pub mta_sts_client: OutboundClient, + pub tls_report_client: OutboundClient, } #[derive(Debug, Clone)] -//#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +//#[cfg_attr(stalwart_tests, derive(PartialEq, Eq))] pub struct QueueRateLimiter { pub id: ObjectId, pub expr: IfBlock, @@ -54,19 +55,29 @@ pub const THROTTLE_HELO_DOMAIN: u16 = 1 << 9; impl SmtpConfig { pub async fn parse(bp: &mut Bootstrap) -> Self { + let queue = QueueConfig::parse(bp).await; + let resolvers = Resolvers::parse(bp).await; + let mail_auth = MailAuthConfig::parse(bp).await; + let report = Box::new(ReportConfig::parse(bp).await); + let session = SessionConfig::parse(bp).await; let config = Self { - session: SessionConfig::parse(bp).await, - queue: QueueConfig::parse(bp).await, - resolvers: Resolvers::parse(bp).await, - mail_auth: MailAuthConfig::parse(bp).await, - report: ReportConfig::parse(bp).await, - mta_sts_client: utils::http::http_client_builder(false) - .pool_max_idle_per_host(0) - .user_agent(crate::USER_AGENT) - .redirect(reqwest::redirect::Policy::none()) - .build() - .unwrap_or_default(), - tls_report_client: utils::http::unpooled_http_client(false), + session, + queue, + resolvers, + mail_auth, + report: *report, + mta_sts_client: OutboundClient::new( + utils::http::http_client_builder(false) + .pool_max_idle_per_host(0) + .user_agent(crate::USER_AGENT), + OutboundPolicy::PublicOnly, + Redirects::None, + ), + tls_report_client: OutboundClient::new( + utils::http::http_client_builder(false).pool_max_idle_per_host(0), + OutboundPolicy::PublicOnly, + Redirects::None, + ), }; if !config.resolvers.dnssec_available diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index 1dd4faa6..2dab2689 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -16,7 +16,7 @@ use ahash::AHashMap; use directory::Credentials; use mail_auth::IpLookupStrategy; use registry::schema::{ - enums::{self, ExpressionConstant, ExpressionVariable, MtaRequiredOrOptional}, + enums::{self, ExpressionVariable, MtaRequiredOrOptional}, prelude::ObjectType, structs::{ DsnReportSettings, MtaConnectionStrategy, MtaDeliveryExpiration, MtaDeliverySchedule, @@ -322,9 +322,9 @@ impl QueueConfig { let mut source_ipv4 = Vec::new(); let mut source_ipv6 = Vec::new(); - for ip_host in obj.object.source_ips { + for ip_host in obj.object.local_ips { let ip_host = IpAndHost { - ip: ip_host.source_ip.into_inner(), + ip: ip_host.local_ip.into_inner(), host: ip_host.ehlo_hostname, }; if ip_host.ip.is_ipv4() { @@ -380,13 +380,13 @@ impl QueueConfig { queue.routing_strategy.insert( route.name, RoutingStrategy::Relay(RelayConfig { - address: if let Ok(ip) = route.address.parse() { + address: if let Ok(ip) = route.host.parse() { HostOrIp::Ip(IpStr { ip, - ip_str: route.address.into(), + ip_str: route.host.into(), }) } else { - HostOrIp::Host(route.address.into()) + HostOrIp::Host(route.host.into()) }, port: route.port as u16, protocol: match route.protocol { @@ -623,46 +623,6 @@ impl QueueQuotas { } } -impl<'x> TryFrom> for RequireOptional { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Constant(ExpressionConstant::Optional) => Ok(RequireOptional::Optional), - Variable::Constant(ExpressionConstant::Require) => Ok(RequireOptional::Require), - Variable::Constant(ExpressionConstant::Disable) => Ok(RequireOptional::Disable), - _ => Err(()), - } - } -} - -impl<'x> TryFrom> for IpLookupStrategy { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Constant(value) => match value { - ExpressionConstant::Ipv4Only => Ok(IpLookupStrategy::Ipv4Only), - ExpressionConstant::Ipv6Only => Ok(IpLookupStrategy::Ipv6Only), - ExpressionConstant::Ipv6ThenIpv4 => Ok(IpLookupStrategy::Ipv6thenIpv4), - ExpressionConstant::Ipv4ThenIpv6 => Ok(IpLookupStrategy::Ipv4thenIpv6), - _ => Err(()), - }, - Variable::String(value) => { - match value.as_str() { - "ipv4_only" => Ok(IpLookupStrategy::Ipv4Only), - "ipv6_only" => Ok(IpLookupStrategy::Ipv6Only), - //"ipv4_and_ipv6" => IpLookupStrategy::Ipv4AndIpv6, - "ipv6_then_ipv4" => Ok(IpLookupStrategy::Ipv6thenIpv4), - "ipv4_then_ipv6" => Ok(IpLookupStrategy::Ipv4thenIpv6), - _ => Err(()), - } - } - _ => Err(()), - } - } -} - impl std::fmt::Debug for RelayConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RelayConfig") @@ -782,6 +742,12 @@ impl Default for QueueName { } } +impl From<[u8; 8]> for QueueName { + fn from(name: [u8; 8]) -> Self { + QueueName(name) + } +} + impl Display for QueueName { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) diff --git a/crates/common/src/config/smtp/report.rs b/crates/common/src/config/smtp/report.rs index eaf77511..1eacf764 100644 --- a/crates/common/src/config/smtp/report.rs +++ b/crates/common/src/config/smtp/report.rs @@ -22,6 +22,7 @@ use std::{str::FromStr, time::Duration}; #[derive(Clone)] pub struct ReportConfig { pub submitter: IfBlock, + pub domain: Option, pub analysis: ReportAnalysis, pub dkim: Report, @@ -89,6 +90,7 @@ impl ReportConfig { ObjectType::ReportSettings.singleton(), &report.ctx_outbound_report_submitter(), ), + domain: report.outbound_report_domain, analysis: ReportAnalysis { addresses: report .inbound_report_addresses @@ -97,7 +99,7 @@ impl ReportConfig { .collect(), forward: report.inbound_report_forwarding, store: dr.hold_mta_reports_for.map(|d| d.into_inner()), - max_size: std::cmp::max(report.inbound_report_max_size, 1024) as usize, + max_size: std::cmp::max(report.max_inbound_report_size, 1024) as usize, }, dkim: Report { name: bp.compile_expr( @@ -192,7 +194,7 @@ impl ReportConfig { ), max_size: bp.compile_expr( ObjectType::DmarcReportSettings.singleton(), - &dmarc.ctx_aggregate_max_report_size(), + &dmarc.ctx_max_aggregate_report_size(), ), }, tls: AggregateReport { diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index cedf5606..3797ac32 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -20,7 +20,7 @@ use mail_auth::{ use registry::schema::{ enums::{DnsResolverProtocol, PolicyEnforcement}, prelude::ObjectType, - structs::{DnsResolver, MtaSts, SystemSettings}, + structs::{Cache, DnsResolver, MtaSts, SystemSettings}, }; use serde::{Deserialize, Serialize}; use std::{ @@ -29,6 +29,7 @@ use std::{ net::IpAddr, str::FromStr, sync::Arc, + time::Instant, }; use store::registry::bootstrap::Bootstrap; use utils::cache::CacheItemWeight; @@ -90,6 +91,88 @@ pub struct Policy { pub max_age: u64, } +#[derive(Debug, Clone)] +pub struct CachedPolicy { + pub policy: Arc, + pub expires: Instant, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MtaStsFailureKind { + TemporaryFetchError, + PermanentFetchError, + InvalidPolicy, +} + +#[derive(Debug)] +pub struct MtaStsFailure { + pub id: Box, + pub kind: MtaStsFailureKind, + pub reason: Box, + pub retry_after: Instant, + pub fallback: Option, +} + +#[derive(Debug, Clone)] +pub enum MtaStsEntry { + Policy(CachedPolicy), + Failed(Arc), +} + +impl MtaStsEntry { + pub fn policy_for( + &self, + id: &str, + now: Instant, + ) -> Option, Arc>> { + match self { + MtaStsEntry::Policy(cached) => { + (cached.policy.id == id).then(|| Ok(cached.policy.clone())) + } + MtaStsEntry::Failed(failure) => { + let fallback = failure.live_fallback(now); + if let Some(cached) = fallback.filter(|cached| cached.policy.id == id) { + Some(Ok(cached.policy.clone())) + } else { + (&*failure.id == id && now < failure.retry_after).then(|| match fallback { + Some(cached) => Ok(cached.policy.clone()), + None => Err(failure.clone()), + }) + } + } + } + } + + pub fn fallback(&self, now: Instant) -> Option<&CachedPolicy> { + match self { + MtaStsEntry::Policy(cached) => Some(cached), + MtaStsEntry::Failed(failure) => failure.live_fallback(now), + } + } +} + +impl MtaStsFailure { + fn live_fallback(&self, now: Instant) -> Option<&CachedPolicy> { + self.fallback.as_ref().filter(|cached| cached.expires > now) + } +} + +impl CacheItemWeight for MtaStsEntry { + fn weight(&self) -> u64 { + match self { + MtaStsEntry::Policy(cached) => cached.policy.weight(), + MtaStsEntry::Failed(failure) => { + (std::mem::size_of::() + failure.id.len() + failure.reason.len()) + as u64 + + failure + .fallback + .as_ref() + .map_or(0, |cached| cached.policy.weight()) + } + } + } +} + impl CacheItemWeight for Tlsa { fn weight(&self) -> u64 { self.entries @@ -124,7 +207,7 @@ impl Resolvers { Ok((config, options)) => { resolver_config = config; opts = options; - opts.num_concurrent_reqs = resolver.concurrency as usize; + opts.num_concurrent_reqs = resolver.max_concurrent_requests as usize; opts.timeout = resolver.timeout.into_inner(); opts.preserve_intermediates = resolver.preserve_intermediates; opts.try_tcp_on_error = resolver.tcp_on_error; @@ -162,7 +245,7 @@ impl Resolvers { resolver_config.add_name_server(NameServerConfig::new(ip, true, connections)); } - opts.num_concurrent_reqs = resolver.concurrency as usize; + opts.num_concurrent_reqs = resolver.max_concurrent_requests as usize; opts.timeout = resolver.timeout.into_inner(); opts.preserve_intermediates = resolver.preserve_intermediates; opts.try_tcp_on_error = resolver.tcp_on_error; @@ -176,7 +259,7 @@ impl Resolvers { ResolverConfig::udp_and_tcp(&CLOUDFLARE) }; - opts.num_concurrent_reqs = resolver.concurrency as usize; + opts.num_concurrent_reqs = resolver.max_concurrent_requests as usize; opts.timeout = resolver.timeout.into_inner(); opts.preserve_intermediates = resolver.preserve_intermediates; opts.try_tcp_on_error = resolver.tcp_on_error; @@ -189,7 +272,7 @@ impl Resolvers { } else { ResolverConfig::udp_and_tcp(&QUAD9) }; - opts.num_concurrent_reqs = resolver.concurrency as usize; + opts.num_concurrent_reqs = resolver.max_concurrent_requests as usize; opts.timeout = resolver.timeout.into_inner(); opts.preserve_intermediates = resolver.preserve_intermediates; opts.try_tcp_on_error = resolver.tcp_on_error; @@ -198,7 +281,7 @@ impl Resolvers { } DnsResolver::Google(resolver) => { resolver_config = ResolverConfig::udp_and_tcp(&GOOGLE); - opts.num_concurrent_reqs = resolver.concurrency as usize; + opts.num_concurrent_reqs = resolver.max_concurrent_requests as usize; opts.timeout = resolver.timeout.into_inner(); opts.preserve_intermediates = resolver.preserve_intermediates; opts.try_tcp_on_error = resolver.tcp_on_error; @@ -225,18 +308,22 @@ impl Resolvers { .expect("Failed to build DNSSEC resolver"), }; + let negative_ttl = bp.setting_infallible::().await.negative_ttl; + Resolvers { - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] dnssec_available: ensure_dnssec(&resolver_config, &dnssec.resolver).await, - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] dnssec_available: true, - dns: MessageAuthenticator::new(resolver_config, opts).unwrap(), + dns: MessageAuthenticator::new(resolver_config, opts) + .unwrap() + .with_max_negative_ttl(negative_ttl.into_inner()), dnssec, } } } -#[cfg(not(feature = "test_mode"))] +#[cfg(not(stalwart_tests))] async fn ensure_dnssec(config: &ResolverConfig, resolver: &TokioResolver) -> bool { config.name_servers().iter().any(|name_server| { name_server @@ -404,3 +491,4 @@ impl Clone for Resolvers { } } } + diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index 2efcb139..157379e1 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -11,6 +11,7 @@ use crate::expr::{ if_block::{BootstrapExprExt, IfBlock}, }; use ahash::AHashSet; +use bytes::Bytes; use hyper::HeaderMap; use registry::schema::{ enums::{self, ExpressionConstant, MtaStage}, @@ -40,12 +41,18 @@ pub struct SessionConfig { pub rcpt: Rcpt, pub data: Data, pub extensions: Extensions, - pub mta_sts_policy: Option, + pub mta_sts_policy: Option, pub milters: Vec, pub hooks: Vec, } +#[derive(Clone)] +pub struct LocalMtaStsPolicy { + pub policy: Policy, + pub text: Bytes, +} + #[derive(Clone)] pub struct Connect { pub hostname: IfBlock, @@ -219,10 +226,16 @@ impl SessionConfig { }); } + let mta_sts_policy = Policy::try_parse(bp).await.map(|policy| LocalMtaStsPolicy { + text: Bytes::from(policy.to_string()), + policy, + }); + let milters = bp.list_infallible::().await; + SessionConfig { timeout: bp.compile_expr( ObjectType::MtaInboundSession.singleton(), - &session.ctx_timeout(), + &session.ctx_idle_timeout(), ), duration: bp.compile_expr( ObjectType::MtaInboundSession.singleton(), @@ -372,10 +385,8 @@ impl SessionConfig { &ext.ctx_mt_priority(), ), }, - mta_sts_policy: Policy::try_parse(bp).await, - milters: bp - .list_infallible::() - .await + mta_sts_policy, + milters: milters .into_iter() .filter_map(|milter| { let id = milter.id; @@ -384,24 +395,24 @@ impl SessionConfig { Some(Milter { enable: bp.compile_expr(id, &milter.ctx_enable()), id, - addrs: format!("{}:{}", milter.hostname, milter.port) + addrs: format!("{}:{}", milter.host, milter.port) .to_socket_addrs() .map_err(|err| { bp.build_error( id, format!( "Unable to resolve milter hostname {}: {}", - milter.hostname, err + milter.host, err ), ) }) .ok()? .collect(), - hostname: milter.hostname, + hostname: milter.host, port: milter.port as u16, - timeout_connect: milter.timeout_connect.into_inner(), - timeout_command: milter.timeout_command.into_inner(), - timeout_data: milter.timeout_data.into_inner(), + timeout_connect: milter.connect_timeout.into_inner(), + timeout_command: milter.command_timeout.into_inner(), + timeout_data: milter.data_timeout.into_inner(), tls: milter.use_tls, tls_allow_invalid_certs: milter.allow_invalid_certs, tempfail_on_error: milter.temp_fail_on_error, @@ -486,7 +497,7 @@ impl<'x> TryFrom> for Mechanism { let mut mechanism = 0; for item in items { - match item { + match *item { Variable::Constant(value) => mechanism |= Mechanism::try_from(value)?.0, _ => return Err(()), } @@ -537,7 +548,6 @@ impl<'x> TryFrom> for MtPriority { _ => Err(()), }, Variable::String(value) => { - let value = value.as_str(); if value.eq_ignore_ascii_case("MIXER") { Ok(MtPriority::Mixer) } else if value.eq_ignore_ascii_case("STANAG4406") { diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 6bc87b4f..237bbfe3 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -21,8 +21,6 @@ pub struct Storage { pub blob: BlobStore, pub search: SearchStore, pub memory: InMemoryStore, - pub metrics: Store, - pub tracing: Store, pub coordinator: Coordinator, pub directory: Option>, pub directories: IdMap, @@ -48,8 +46,6 @@ impl Storage { search, coordinator: Coordinator::build(bp, &memory).await.unwrap_or_default(), memory, - tracing: Store::build_tracing(bp).await.unwrap_or_default(), - metrics: Store::build_metrics(bp).await.unwrap_or_default(), directory: directory.default_directory, directories: directory.directories, } diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index fea17602..d23d4b06 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -5,12 +5,15 @@ */ use crate::config::storage::Storage; +use crate::ipc::IndexQueueNotify; use ahash::{AHashMap, AHashSet}; -use base64::{Engine, engine::general_purpose::STANDARD}; +use aws_lc_rs::constant_time::verify_slices_are_equal; +use encodify::base64::STANDARD; use hyper::HeaderMap; use opentelemetry::{InstrumentationScope, KeyValue}; use opentelemetry_otlp::{ - LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, + LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig, + tonic_types::metadata::MetadataMap, }; use opentelemetry_sdk::{Resource, metrics::Temporality}; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; @@ -19,9 +22,17 @@ use registry::schema::{ prelude::ObjectType, structs::{self, EventTracingLevel, MetricsPrometheus, Tracer, WebHook}, }; -use std::{collections::HashMap, str::FromStr, sync::Arc, time::Duration}; +use serde::Serialize; +use std::{ + collections::HashMap, + hash::{DefaultHasher, Hash, Hasher}, + str::FromStr, + sync::Arc, + time::Duration, +}; use store::registry::bootstrap::Bootstrap; use trc::{EventType, Level, MetricType, TelemetryEvent, ipc::subscriber::Interests}; +use utils::http::http_client_builder; #[derive(Debug)] pub struct TelemetrySubscriber { @@ -115,15 +126,48 @@ pub struct Metrics { pub log_path: Option, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct PrometheusMetrics { - pub auth: Option, + pub auth: PrometheusAuth, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PrometheusAuth { + None, + Basic(Box), +} + +impl PrometheusAuth { + fn new(username: Option<&str>, secret: Option<&str>) -> Result { + match ( + username.filter(|username| !username.is_empty()), + secret.filter(|secret| !secret.is_empty()), + ) { + (None, None) => Ok(PrometheusAuth::None), + (Some(username), Some(secret)) => Ok(PrometheusAuth::Basic( + STANDARD + .encode(format!("{username}:{secret}")) + .into_boxed_str(), + )), + (Some(_), None) => Err("Prometheus authUsername is set but authSecret is not"), + (None, Some(_)) => Err("Prometheus authSecret is set but authUsername is not"), + } + } + + pub fn is_authorized(&self, basic_credentials: Option<&str>) -> bool { + match self { + PrometheusAuth::None => true, + PrometheusAuth::Basic(expected) => basic_credentials.is_some_and(|credentials| { + verify_slices_are_equal(expected.as_bytes(), credentials.as_bytes()).is_ok() + }), + } + } } impl Telemetry { - pub async fn parse(bp: &mut Bootstrap, storage: &Storage) -> Self { + pub async fn parse(bp: &mut Bootstrap, storage: &Storage, index_tx: IndexQueueNotify) -> Self { let mut telemetry = Telemetry { - tracers: Tracers::parse(bp, storage).await, + tracers: Tracers::parse(bp, storage, index_tx).await, metrics: Interests::default(), }; @@ -140,8 +184,42 @@ impl Telemetry { } } +macro_rules! strip_interests { + ($tracer:expr) => {{ + $tracer.enable = true; + $tracer.lossy = false; + $tracer.events = Default::default(); + $tracer.events_policy = Default::default(); + }}; + ($tracer:expr, level) => {{ + strip_interests!($tracer); + $tracer.level = Default::default(); + }}; +} + +fn sink_fingerprint(value: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + serde_json::to_vec(value) + .unwrap_or_default() + .hash(&mut hasher); + hasher.finish() +} + +fn tracer_sink_fingerprint(tracer: &Tracer) -> u64 { + let mut tracer = tracer.clone(); + match &mut tracer { + Tracer::Log(t) => strip_interests!(t, level), + Tracer::Stdout(t) => strip_interests!(t, level), + Tracer::Journal(t) => strip_interests!(t, level), + Tracer::OtelHttp(t) => strip_interests!(t, level), + Tracer::OtelGrpc(t) => strip_interests!(t, level), + } + sink_fingerprint(&tracer) +} + impl Tracers { - pub async fn parse(bp: &mut Bootstrap, storage: &Storage) -> Self { + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + pub async fn parse(bp: &mut Bootstrap, storage: &Storage, index_tx: IndexQueueNotify) -> Self { let mut custom_levels = AHashMap::new(); let mut tracers: Vec = Vec::new(); let mut global_interests = Interests::default(); @@ -156,6 +234,7 @@ impl Tracers { for tracer in bp.list_infallible::().await { let id = tracer.id; let tracer = tracer.object; + let sink = tracer_sink_fingerprint(&tracer); let level; let lossy; let events; @@ -272,16 +351,34 @@ impl Tracers { let mut span_exporter = SpanExporter::builder() .with_http() - .with_endpoint(tracer.endpoint.clone()) + .with_endpoint(tracer.url.clone()) .with_timeout(tracer.timeout.into_inner()); let mut log_exporter = LogExporter::builder() .with_http() - .with_endpoint(tracer.endpoint) + .with_endpoint(tracer.url) .with_timeout(tracer.timeout.into_inner()); if !headers.is_empty() { span_exporter = span_exporter.with_headers(headers.clone()); log_exporter = log_exporter.with_headers(headers); } + if tracer.allow_invalid_certs { + match http_client_builder(true) + .timeout(tracer.timeout.into_inner()) + .build() + { + Ok(client) => { + span_exporter = span_exporter.with_http_client(client.clone()); + log_exporter = log_exporter.with_http_client(client); + } + Err(err) => { + bp.build_error( + id, + format!("Failed to build OpenTelemetry HTTP client: {err}"), + ); + continue; + } + } + } match (span_exporter.build(), log_exporter.build()) { (Ok(span_exporter), Ok(log_exporter)) => { @@ -316,15 +413,32 @@ impl Tracers { events_policy = tracer.events_policy; enable = tracer.enable; + let metadata = match tracer + .http_auth + .build_headers(tracer.http_headers, None) + .await + { + Ok(headers) => MetadataMap::from_headers(headers), + Err(err) => { + bp.build_error( + id, + format!("Failed to build OpenTelemetry gRPC metadata: {err}"), + ); + continue; + } + }; + let mut span_exporter = SpanExporter::builder() .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(tracer.timeout.into_inner()); + .with_timeout(tracer.timeout.into_inner()) + .with_metadata(metadata.clone()); let mut log_exporter = LogExporter::builder() .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(tracer.timeout.into_inner()); - if let Some(endpoint) = tracer.endpoint { + .with_timeout(tracer.timeout.into_inner()) + .with_metadata(metadata); + if let Some(endpoint) = tracer.url { span_exporter = span_exporter.with_endpoint(endpoint.clone()); log_exporter = log_exporter.with_endpoint(endpoint); } @@ -364,7 +478,7 @@ impl Tracers { // Create tracer let mut tracer = TelemetrySubscriber { - id: format!("t_{}", id.id()), + id: format!("t_{}_{sink:x}", id.id()), interests: Default::default(), lossy, typ, @@ -414,6 +528,11 @@ impl Tracers { for hook in bp.list_infallible::().await { let id = hook.id; let hook = hook.object; + let sink = { + let mut hook = hook.clone(); + strip_interests!(hook, level); + sink_fingerprint(&hook) + }; if !hook.enable { continue; @@ -433,7 +552,7 @@ impl Tracers { // Build tracer let mut tracer = TelemetrySubscriber { - id: format!("w_{}", id.id()), + id: format!("w_{}_{sink:x}", id.id()), interests: Default::default(), lossy: hook.lossy, typ: TelemetrySubscriberType::Webhook(WebhookTracer { @@ -463,8 +582,16 @@ impl Tracers { }; // Parse webhook events + let level = Level::from(hook.level); apply_events(hook.events, hook.events_policy, |event_type| { - if event_type != EventType::Telemetry(TelemetryEvent::WebhookError) { + if event_type != EventType::Telemetry(TelemetryEvent::WebhookError) + && level.is_contained( + custom_levels + .get(&event_type) + .copied() + .unwrap_or(event_type.level()), + ) + { tracer.interests.set(event_type); global_interests.set(event_type); } @@ -473,7 +600,7 @@ impl Tracers { if !tracer.interests.is_empty() { tracers.push(tracer); } else { - bp.build_error(id, "No events enabled for webhook"); + bp.build_warning(id, "No events enabled for webhook"); } } @@ -551,27 +678,30 @@ impl Metrics { Metrics { prometheus: match metrics.prometheus { MetricsPrometheus::Enabled(prom) => { - let secret = prom - .auth_secret - .secret() - .await - .map_err(|err| { + let auth = match prom.auth_secret.secret().await { + Ok(secret) => { + PrometheusAuth::new(prom.auth_username.as_deref(), secret.as_deref()) + .map_err(|err| err.to_string()) + } + Err(err) => { + Err(format!("Unable to retrieve Prometheus auth secret: {err}")) + } + }; + match auth { + Ok(auth) => Some(PrometheusMetrics { auth }), + Err(err) => { bp.build_error( ObjectType::Metrics.singleton(), - format!("Unable to retrieve Prometheus auth secret: {err}"), + format!("{err}, Prometheus endpoint disabled"), ); - }) - .unwrap_or_default(); - Some(PrometheusMetrics { - auth: prom.auth_username.and_then(|user| { - secret.map(|secret| STANDARD.encode(format!("{user}:{secret}"))) - }), - }) + None + } + } } MetricsPrometheus::Disabled => None, }, otel: match metrics.open_telemetry { - structs::MetricsOtel::Http(otel) => { + structs::MetricsOtel::Http(otel) => 'otel: { let headers = match otel.http_auth.build_headers(otel.http_headers, None).await { Ok(headers) => headers @@ -592,11 +722,28 @@ impl Metrics { let mut exporter = MetricExporter::builder() .with_temporality(Temporality::Delta) .with_http() - .with_endpoint(otel.endpoint) + .with_endpoint(otel.url) .with_timeout(otel.timeout.into_inner()); if !headers.is_empty() { exporter = exporter.with_headers(headers); } + if otel.allow_invalid_certs { + match http_client_builder(true) + .timeout(otel.timeout.into_inner()) + .build() + { + Ok(client) => { + exporter = exporter.with_http_client(client); + } + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!("Failed to build OpenTelemetry HTTP client: {err}"), + ); + break 'otel None; + } + } + } match exporter.build() { Ok(exporter) => Some(Arc::new(OtelMetrics { @@ -620,7 +767,7 @@ impl Metrics { .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) .with_timeout(otel.timeout.into_inner()); - if let Some(endpoint) = otel.endpoint { + if let Some(endpoint) = otel.url { exporter = exporter.with_endpoint(endpoint); } @@ -714,3 +861,4 @@ impl std::fmt::Debug for OtelMetrics { .finish() } } + diff --git a/crates/common/src/expr/capture.rs b/crates/common/src/expr/capture.rs new file mode 100644 index 00000000..c5f5645a --- /dev/null +++ b/crates/common/src/expr/capture.rs @@ -0,0 +1,121 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use bumpalo::Bump; +use regex_automata::{ + Input, MatchKind, meta, + util::{primitives::NonMaxUsize, syntax}, +}; +use std::{fmt, sync::Arc}; + +const NFA_SIZE_LIMIT: usize = 10 * (1 << 20); +const HYBRID_CACHE_CAPACITY: usize = 2 * (1 << 20); + +pub type Slot = Option; + +#[derive(Clone)] +pub struct CaptureRegex { + inner: Arc, +} + +struct CompiledRegex { + regex: meta::Regex, + pattern: Box, +} + +#[derive(Default)] +pub(crate) struct Captures<'s> { + haystack: &'s str, + matched: bool, + slots: Option<(u32, &'s mut [Slot])>, +} + +impl CaptureRegex { + pub fn new(pattern: &str) -> Result { + meta::Builder::new() + .configure( + meta::Config::new() + .nfa_size_limit(Some(NFA_SIZE_LIMIT)) + .hybrid_cache_capacity(HYBRID_CACHE_CAPACITY) + .match_kind(MatchKind::LeftmostFirst) + .utf8_empty(true), + ) + .syntax(syntax::Config::new().utf8(true)) + .build(pattern) + .map(|regex| CaptureRegex { + inner: Arc::new(CompiledRegex { + regex, + pattern: pattern.into(), + }), + }) + .map_err(|err| { + let reason = match (err.size_limit(), err.syntax_error()) { + (Some(limit), _) => { + format!("Compiled regex exceeds size limit of {limit} bytes.") + } + (None, Some(syntax)) => syntax.to_string(), + (None, None) => err.to_string(), + }; + format!("Invalid regular expression {pattern:?}: {reason}") + }) + } + + pub fn is_match(&self, haystack: &str) -> bool { + self.inner.regex.is_match(haystack) + } + + pub fn slot_len(&self) -> usize { + self.inner.regex.group_info().slot_len() + } + + pub fn search(&self, haystack: &str, slots: &mut [Slot]) -> bool { + self.inner + .regex + .search_slots(&Input::new(haystack), slots) + .is_some() + } +} + +impl fmt::Debug for CaptureRegex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Regex").field(&self.inner.pattern).finish() + } +} + +impl<'s> Captures<'s> { + pub(crate) fn read( + &mut self, + index: u32, + regex: &CaptureRegex, + haystack: &'s str, + arena: &'s Bump, + ) -> bool { + let slots = match &mut self.slots { + Some((current, slots)) if *current == index => slots, + slots => { + &mut slots + .insert((index, arena.alloc_slice_fill_copy(regex.slot_len(), None))) + .1 + } + }; + self.matched = regex.search(haystack, slots); + self.haystack = haystack; + self.matched + } + + pub(crate) fn get(&self, group: u32) -> &'s str { + let Some((_, slots)) = self.slots.as_ref().filter(|_| self.matched) else { + return ""; + }; + let start = group as usize * 2; + match slots.get(start..start + 2) { + Some([Some(start), Some(end)]) => self.haystack.get(start.get()..end.get()), + _ => None, + } + .unwrap_or_default() + } +} + diff --git a/crates/common/src/expr/eval.rs b/crates/common/src/expr/eval.rs index 3fc27a8f..0fe14385 100644 --- a/crates/common/src/expr/eval.rs +++ b/crates/common/src/expr/eval.rs @@ -5,783 +5,502 @@ */ use super::{ - BinaryOperator, Constant, Expression, ExpressionItem, StringCow, SystemVariable, UnaryOperator, - Variable, - functions::{FUNCTIONS, ResolveVariable}, + BinaryOperator, SystemVariable, UnaryOperator, Variable, + capture::Captures, + functions::{FnCtx, MAX_ASYNC_ARGS, MAX_SYNC_ARGS, ResolveVariable, SyncFn, asynch::AsyncCall}, if_block::IfBlock, + program::{ConstValue, Op, Program, Src}, }; use crate::Server; -use compact_str::{CompactString, ToCompactString, format_compact}; -use hyper::StatusCode; -use registry::{ - schema::prelude::Property, - types::{EnumImpl, id::ObjectId}, -}; -use std::{cmp::Ordering, fmt::Display}; +use bumpalo::Bump; +use compact_str::{ToCompactString, format_compact}; +use registry::{schema::enums::ExpressionVariable, types::EnumImpl}; use trc::{Collector, EvalEvent}; +pub const ARENA_RETAIN_CAP: usize = 4 * 1024; +const INLINE_STACK: usize = 8; +const ASYNC_INLINE_STACK: usize = 4; + impl Server { - pub async fn eval_if<'x, R: TryFrom>, V: ResolveVariable>( + pub fn eval_if<'x, R, V>( &'x self, if_block: &'x IfBlock, resolver: &'x V, + arena: &'x mut Bump, session_id: u64, - ) -> Option { - if if_block.is_empty() { + ) -> impl Future> + Send + 'x + where + R: for<'b> TryFrom> + 'x, + V: ResolveVariable, + { + self.eval_program(if_block, resolver, arena, session_id, convert::) + } + + pub fn eval_if_with<'x, T, V, F>( + &'x self, + if_block: &'x IfBlock, + resolver: &'x V, + arena: &'x mut Bump, + session_id: u64, + with: F, + ) -> impl Future> + Send + 'x + where + T: 'x, + V: ResolveVariable, + F: for<'v> FnOnce(Variable<'v>) -> T + Send + 'x, + { + self.eval_program(if_block, resolver, arena, session_id, |value| { + Some(with(value)) + }) + } + + #[allow(clippy::manual_async_fn)] + fn eval_program<'x, T, V, F>( + &'x self, + if_block: &'x IfBlock, + resolver: &'x V, + arena: &'x mut Bump, + session_id: u64, + finish: F, + ) -> impl Future> + Send + 'x + where + T: 'x, + V: ResolveVariable, + F: for<'v> FnOnce(Variable<'v>) -> Option + Send + 'x, + { + async move { + let program = &if_block.program; + if program.is_empty() { + trc::event!( + Eval(EvalEvent::Result), + SpanId = session_id, + Name = if_block.id.to_compact_string(), + Property = if_block.property.as_str(), + Result = "" + ); + return None; + } + arena.reset(); + if let Some(value) = self.run_inline(program, resolver, &*arena) { + let result = if_block.finish_eval(value, session_id, finish); + recycle_arena(arena); + return result; + } + self.eval_suspending(if_block, resolver, arena, session_id, finish) + .await + } + } + + async fn eval_suspending( + &self, + if_block: &IfBlock, + resolver: &V, + arena: &mut Bump, + session_id: u64, + finish: F, + ) -> Option + where + V: ResolveVariable, + F: for<'v> FnOnce(Variable<'v>) -> Option, + { + let program = &if_block.program; + arena.reset(); + let result = { + let mut inline = [Variable::default(); ASYNC_INLINE_STACK]; + let mut machine = Machine::new(stack(program.max_depth(), &mut inline, &*arena)); + let mut pc = 0; + loop { + let step = machine.run( + &Env { + server: self, + resolver, + arena: &*arena, + program, + }, + pc, + ); + match step { + Step::Done(value) => break if_block.finish_eval(value, session_id, finish), + Step::Suspend { call, resume } => match self.eval_fnc(call, session_id).await { + Ok(output) => { + machine.push(output.into_variable(&*arena)); + pc = resume; + } + Err(err) => { + trc::event!( + Eval(EvalEvent::Error), + SpanId = session_id, + Name = if_block.id.to_compact_string(), + Property = if_block.property.as_str(), + CausedBy = err, + ); + break None; + } + }, + } + } + }; + recycle_arena(arena); + result + } + + fn run_inline<'s, V: ResolveVariable>( + &'s self, + program: &'s Program, + resolver: &'s V, + arena: &'s Bump, + ) -> Option> { + let mut inline = [Variable::default(); INLINE_STACK]; + let mut machine = Machine::new(stack(program.max_depth(), &mut inline, arena)); + let env = Env { + server: self, + resolver, + arena, + program, + }; + match machine.run(&env, 0) { + Step::Done(value) => Some(value), + Step::Suspend { .. } => None, + } + } +} + +impl IfBlock { + fn finish_eval(&self, value: Variable<'_>, session_id: u64, finish: F) -> Option + where + F: for<'v> FnOnce(Variable<'v>) -> Option, + { + trc::event!( + Eval(EvalEvent::Result), + SpanId = session_id, + Name = self.id.to_compact_string(), + Property = self.property.as_str(), + Result = format_compact!("{value:?}"), + ); + let result = finish(value); + if result.is_none() { trc::event!( Eval(EvalEvent::Result), SpanId = session_id, - Id = if_block.id.to_string(), - Key = if_block.property.as_str(), - Result = "" + Name = self.id.to_compact_string(), + Property = self.property.as_str(), + Result = "", ); - - return None; } + result + } +} - match (EvalContext { - resolver, - core: self, - expr: if_block, - captures: Vec::new(), - session_id, - }) - .eval() - .await - { - Ok(result) => { - trc::event!( - Eval(EvalEvent::Result), - SpanId = session_id, - Id = if_block.id.to_string(), - Key = if_block.property.as_str(), - Result = format!("{result:?}"), - ); +fn recycle_arena(arena: &mut Bump) { + arena.reset(); + if arena.allocated_bytes() > ARENA_RETAIN_CAP { + *arena = Bump::new(); + } +} - match result.try_into() { - Ok(value) => Some(value), - Err(_) => { - trc::event!( - Eval(EvalEvent::Result), - SpanId = session_id, - Id = if_block.id.to_string(), - Key = if_block.property.as_str(), - Result = "", - ); +fn convert TryFrom>>(value: Variable<'_>) -> Option { + R::try_from(value).ok() +} - None - } - } - } - Err(err) => { - trc::event!( - Eval(EvalEvent::Error), - SpanId = session_id, - Id = if_block.id.to_string(), - Key = if_block.property.as_str(), - CausedBy = err, - ); - - None +fn stack<'b, 's: 'b, const N: usize>( + depth: usize, + inline: &'b mut [Variable<'s>; N], + arena: &'s Bump, +) -> &'b mut [Variable<'s>] { + if depth <= N { + inline + } else { + arena.alloc_slice_fill_copy(depth, Variable::default()) + } +} + +struct Env<'s, V> { + server: &'s Server, + resolver: &'s V, + arena: &'s Bump, + program: &'s Program, +} + +enum Step<'s> { + Done(Variable<'s>), + Suspend { call: AsyncCall<'s>, resume: usize }, +} + +struct Machine<'s, 'b> { + stack: &'b mut [Variable<'s>], + len: usize, + captures: Captures<'s>, +} + +impl<'s, V: ResolveVariable> Env<'s, V> { + fn load(&self, variable: ExpressionVariable) -> Variable<'s> { + self.resolver.resolve_variable(variable, self.arena) + } + + fn system(&self, variable: SystemVariable) -> Variable<'s> { + let server = self.server; + match variable { + SystemVariable::Hostname => Variable::String(&server.core.network.server_name), + SystemVariable::Domain => Variable::String(&server.core.email.default_domain_name), + SystemVariable::ReportDomain => Variable::String( + server + .core + .smtp + .report + .domain + .as_deref() + .unwrap_or(&server.core.email.default_domain_name), + ), + SystemVariable::NodeId => server.core.network.node_id.into(), + SystemVariable::NodeHostname => Variable::String(server.registry().local_hostname()), + SystemVariable::NodeRole => { + Variable::String(server.registry().cluster_role().unwrap_or_default()) } + SystemVariable::Metric(metric) => Variable::Float(Collector::read_metric(metric)), } } - pub async fn eval_expr<'x, R: TryFrom>, V: ResolveVariable>( - &'x self, - expr: &'x Expression, - resolver: &'x V, - obj_id: ObjectId, - property: Property, - session_id: u64, - ) -> Option { - if expr.is_empty() { - return None; - } + fn constant(&self, index: u32) -> Variable<'s> { + self.program + .constant(index) + .map(|value| value.to_variable(self.arena)) + .unwrap_or_default() + } - match (EvalContext { - resolver, - core: self, - expr, - captures: &mut Vec::new(), - session_id, - }) - .eval() - .await - { - Ok(result) => { - trc::event!( - Eval(EvalEvent::Result), - SpanId = session_id, - Id = obj_id.to_string(), - Key = property.as_str(), - Result = format!("{result:?}"), - ); + fn compare_constant(&self, left: Variable<'s>, op: BinaryOperator, index: u32) -> bool { + self.program + .constant(index) + .and_then(ConstValue::scalar) + .is_some_and(|right| left.compare(op, &right)) + } - match result.try_into() { - Ok(value) => Some(value), - Err(_) => { - trc::event!( - Eval(EvalEvent::Error), - SpanId = session_id, - Id = obj_id.to_string(), - Key = property.as_str(), - Details = "Failed to convert result", - ); - - None - } - } - } - Err(err) => { - trc::event!( - Eval(EvalEvent::Error), - SpanId = session_id, - Id = obj_id.to_string(), - Key = property.as_str(), - CausedBy = err, - ); - - None - } + fn in_array(&self, value: Variable<'s>, index: u32) -> bool { + match self.program.constant(index) { + Some(ConstValue::Array { items, .. }) => items + .iter() + .any(|item| item.scalar().is_some_and(|item| item == value)), + _ => false, } } } -struct EvalContext<'x, V: ResolveVariable, T, C> { - resolver: &'x V, - core: &'x Server, - expr: &'x T, - captures: C, - session_id: u64, -} +impl<'s, 'b> Machine<'s, 'b> { + fn new(stack: &'b mut [Variable<'s>]) -> Self { + Machine { + stack, + len: 0, + captures: Captures::default(), + } + } -impl<'x, V: ResolveVariable> EvalContext<'x, V, IfBlock, Vec> { - async fn eval(&mut self) -> trc::Result> { - for if_then in &self.expr.if_then { - if (EvalContext { - resolver: self.resolver, - core: self.core, - expr: &if_then.expr, - captures: &mut self.captures, - session_id: self.session_id, - }) - .eval() - .await? - .to_bool() - { - return (EvalContext { - resolver: self.resolver, - core: self.core, - expr: &if_then.then, - captures: &mut self.captures, - session_id: self.session_id, - }) - .eval() - .await; + fn push(&mut self, value: Variable<'s>) { + match self.stack.get_mut(self.len) { + Some(slot) => { + *slot = value; + self.len += 1; + } + None => debug_assert!(false, "expression stack overflow"), + } + } + + fn pop(&mut self) -> Variable<'s> { + match self.len.checked_sub(1) { + Some(top) => { + self.len = top; + self.stack.get(top).copied().unwrap_or_default() + } + None => Variable::default(), + } + } + + fn peek_bool(&self) -> bool { + self.len + .checked_sub(1) + .and_then(|top| self.stack.get(top)) + .is_some_and(|value| value.to_bool()) + } + + fn load(&mut self, src: Src, env: &Env<'s, V>) -> Variable<'s> { + match src { + Src::Pop => self.pop(), + Src::Var(variable) => env.load(variable), + } + } + + fn call(&mut self, function: SyncFn, argc: usize, ctx: &FnCtx<'s>) { + let result = match self.len.checked_sub(argc) { + Some(base) => { + let result = function(ctx, self.stack.get(base..self.len).unwrap_or_default()); + self.len = base; + result + } + None => { + let mut padded = [Variable::default(); MAX_SYNC_ARGS]; + let missing = argc - self.len; + for (slot, value) in padded + .iter_mut() + .skip(missing) + .zip(self.stack.iter().take(self.len)) + { + *slot = *value; + } + self.len = 0; + function(ctx, padded.get(..argc).unwrap_or(&padded)) + } + }; + self.push(result); + } + + fn build(&mut self, count: usize, arena: &'s Bump) { + let items: &'s [Variable<'s>] = match self.len.checked_sub(count) { + Some(base) => { + let items = + arena.alloc_slice_copy(self.stack.get(base..self.len).unwrap_or_default()); + self.len = base; + items + } + None => { + let missing = count - self.len; + let present = self.stack.get(..self.len).unwrap_or_default(); + let items = arena.alloc_slice_fill_with(count, |position| { + position + .checked_sub(missing) + .and_then(|position| present.get(position)) + .copied() + .unwrap_or_default() + }); + self.len = 0; + items + } + }; + self.push(Variable::Array(items)); + } + + fn take_async_args(&mut self, argc: usize) -> [Variable<'s>; MAX_ASYNC_ARGS] { + let mut args = [Variable::default(); MAX_ASYNC_ARGS]; + for position in (0..argc).rev() { + let value = self.pop(); + if let Some(slot) = args.get_mut(position) { + *slot = value; } } - - (EvalContext { - resolver: self.resolver, - core: self.core, - expr: &self.expr.default, - captures: &mut self.captures, - session_id: self.session_id, - }) - .eval() - .await + args } -} -impl<'x, V: ResolveVariable> EvalContext<'x, V, Expression, &mut Vec> { - async fn eval(&mut self) -> trc::Result> { - let mut stack = Vec::new(); - let mut exprs = self.expr.items.iter(); - - while let Some(expr) = exprs.next() { - match expr { - ExpressionItem::Variable(v) => { - stack.push(self.resolver.resolve_variable(*v)); + fn run(&mut self, env: &Env<'s, V>, start: usize) -> Step<'s> { + let ops = env.program.ops(); + let mut pc = start; + while let Some(op) = ops.get(pc) { + pc += 1; + match *op { + Op::Var(variable) => { + let value = env.load(variable); + self.push(value); } - ExpressionItem::Global(v) => { - stack.push(self.resolver.resolve_global(v)); + Op::Global(name) => { + let value = env + .resolver + .resolve_global(env.program.string(name), env.arena); + self.push(value); } - ExpressionItem::Constant(val) => { - stack.push(Variable::from(val)); - } - ExpressionItem::Capture(v) => { - stack.push(Variable::String(StringCow::Owned( - self.captures - .get(*v as usize) - .map(|v| v.as_str()) - .unwrap_or_default() - .to_compact_string(), - ))); - } - ExpressionItem::System(setting) => match setting { - SystemVariable::Hostname => { - stack.push(self.core.core.network.server_name.as_str().into()) - } - SystemVariable::Domain => { - stack.push(self.core.core.email.default_domain_name.as_str().into()) - } - SystemVariable::NodeId => stack.push(self.core.core.network.node_id.into()), - SystemVariable::NodeHostname => { - stack.push(self.core.registry().local_hostname().into()) - } - SystemVariable::NodeRole => stack.push( - self.core - .registry() - .cluster_role() - .unwrap_or_default() - .into(), - ), - SystemVariable::Metric(variable) => { - stack.push(Variable::Float(Collector::read_metric(*variable))); - } - }, - ExpressionItem::UnaryOperator(op) => { - let value = stack.pop().unwrap_or_default(); - stack.push(match op { + Op::System(variable) => self.push(env.system(variable)), + Op::Capture(group) => self.push(Variable::String(self.captures.get(group))), + Op::Int(value) => self.push(Variable::Integer(value)), + Op::Float(value) => self.push(Variable::Float(value)), + Op::Static(value) => self.push(Variable::Constant(value)), + Op::Str(index) => self.push(Variable::String(env.program.string(index))), + Op::Array(index) => self.push(env.constant(index)), + Op::Unary(op) => { + let value = self.pop(); + self.push(match op { UnaryOperator::Not => value.op_not(), UnaryOperator::Minus => value.op_minus(), }); } - ExpressionItem::BinaryOperator(op) => { - let right = stack.pop().unwrap_or_default(); - let left = stack.pop().unwrap_or_default(); - stack.push(match op { - BinaryOperator::Add => left.op_add(right), - BinaryOperator::Subtract => left.op_subtract(right), - BinaryOperator::Multiply => left.op_multiply(right), - BinaryOperator::Divide => left.op_divide(right), - BinaryOperator::And => left.op_and(right), - BinaryOperator::Or => left.op_or(right), - BinaryOperator::Xor => left.op_xor(right), - BinaryOperator::Eq => left.op_eq(right), - BinaryOperator::Ne => left.op_ne(right), - BinaryOperator::Lt => left.op_lt(right), - BinaryOperator::Le => left.op_le(right), - BinaryOperator::Gt => left.op_gt(right), - BinaryOperator::Ge => left.op_ge(right), - }); + Op::Binary(op) => { + let right = self.pop(); + let left = self.pop(); + self.push(left.op_binary(op, right, env.arena)); } - ExpressionItem::Function { id, num_args } => { - let num_args = *num_args as usize; - - let mut arguments = Variable::array(num_args); - for arg_num in 0..num_args { - arguments[num_args - arg_num - 1] = stack.pop().unwrap_or_default(); + Op::JmpIf { val, target } => { + if self.peek_bool() == val { + pc = target as usize; } - - let result = if let Some((_, fnc, _)) = FUNCTIONS.get(*id as usize) { - (fnc)(arguments) - } else { - Box::pin(self.core.eval_fnc( - *id - FUNCTIONS.len() as u32, - arguments, - self.session_id, - )) - .await? + } + Op::Call { id, argc } => match Program::function(id) { + Some(function) => self.call(function, argc as usize, &FnCtx::new(env.arena)), + None => { + self.len = self.len.saturating_sub(argc as usize); + self.push(Variable::default()); + } + }, + Op::CallAsync { id, argc } => { + let args = self.take_async_args(argc as usize); + return Step::Suspend { + call: AsyncCall::new(id, args, env.arena), + resume: pc, }; - - stack.push(result); } - ExpressionItem::JmpIf { val, pos } => { - if stack.last().is_some_and(|v| v.to_bool()) == *val { - for _ in 0..*pos { - exprs.next(); + Op::Index => { + let index = self.pop().to_usize().unwrap_or_default(); + let value = self.pop(); + self.push(value.index(index)); + } + Op::IndexConst { src, index } => { + let value = self.load(src, env); + self.push(value.index(index as usize)); + } + Op::Build(count) => self.build(count as usize, env.arena), + Op::Regex(index) => { + let haystack = self.pop().to_str(env.arena); + let matched = env.program.regex(index).is_some_and(|regex| { + if env.program.uses_captures() { + self.captures.read(index, regex, haystack, env.arena) + } else { + regex.is_match(haystack) } + }); + self.push(Variable::Integer(i64::from(matched))); + } + Op::CmpConst { src, op, konst } => { + let left = self.load(src, env); + self.push(env.compare_constant(left, op, konst).into()); + } + Op::Text { src, op, needle } => { + let haystack = self.load(src, env); + let matched = env + .program + .needle(needle) + .is_some_and(|needle| needle.matches(op, haystack, env.arena)); + self.push(matched.into()); + } + Op::ContainsIgnoreCase { src, needle } => { + let haystack = self.load(src, env); + let matched = env + .program + .case_needle(needle) + .is_some_and(|needle| needle.matches(haystack, env.arena)); + self.push(matched.into()); + } + Op::InArray { src, array } => { + let value = self.load(src, env); + self.push(env.in_array(value, array).into()); + } + Op::InSet { src, set } => { + let value = self.load(src, env); + let matched = env.program.set(set).is_some_and(|set| set.contains(value)); + self.push(matched.into()); + } + Op::Test { target } => { + let value = self.pop(); + self.len = 0; + if !value.to_bool() { + pc = target as usize; } } - ExpressionItem::ArrayAccess => { - let index = stack - .pop() - .unwrap_or_default() - .to_usize() - .unwrap_or_default(); - let array = stack.pop().unwrap_or_default().into_array(); - stack.push(array.into_iter().nth(index).unwrap_or_default()); - } - ExpressionItem::ArrayBuild(num_items) => { - let num_items = *num_items as usize; - let mut items = Variable::array(num_items); - for arg_num in 0..num_items { - items[num_items - arg_num - 1] = stack.pop().unwrap_or_default(); - } - stack.push(Variable::Array(items)); - } - ExpressionItem::Regex(regex) => { - self.captures.clear(); - let value = stack.pop().unwrap_or_default().into_string(); - - if let Some(captures_) = regex.captures(value.as_ref()) { - for capture in captures_.iter() { - self.captures - .push(capture.map_or("", |m| m.as_str()).to_compact_string()); - } - } - - stack.push(Variable::Integer(!self.captures.is_empty() as i64)); - } + Op::Return => return Step::Done(self.pop()), } } - - Ok(stack.pop().unwrap_or_default()) - } -} - -impl Expression { - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } - - pub fn items(&self) -> &[ExpressionItem] { - &self.items - } -} - -impl<'x> Variable<'x> { - pub fn op_add(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_add(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a + b), - (Variable::Integer(i), Variable::Float(f)) - | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 + f), - (Variable::Array(a), Variable::Array(b)) => { - Variable::Array(a.into_iter().chain(b).collect::>()) - } - (Variable::Array(a), b) => { - Variable::Array(a.into_iter().chain([b]).collect::>()) - } - (a, Variable::Array(b)) => { - Variable::Array([a].into_iter().chain(b).collect::>()) - } - (Variable::String(a), b) => { - if !a.is_empty() { - Variable::String(StringCow::Owned(format_compact!("{}{}", a, b))) - } else { - b - } - } - (a, Variable::String(b)) => { - if !b.is_empty() { - Variable::String(StringCow::Owned(format_compact!("{}{}", a, b))) - } else { - a - } - } - (a, Variable::Constant(_)) => a, - (Variable::Constant(_), b) => b, - } - } - - pub fn op_subtract(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_sub(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a - b), - (Variable::Integer(a), Variable::Float(b)) => Variable::Float(a as f64 - b), - (Variable::Float(a), Variable::Integer(b)) => Variable::Float(a - b as f64), - (Variable::Array(a), b) | (b, Variable::Array(a)) => { - Variable::Array(a.into_iter().filter(|v| v != &b).collect::>()) - } - (a, b) => a.parse_number().op_subtract(b.parse_number()), - } - } - - pub fn op_multiply(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_mul(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a * b), - (Variable::Integer(i), Variable::Float(f)) - | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 * f), - (a, b) => a.parse_number().op_multiply(b.parse_number()), - } - } - - pub fn op_divide(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => { - Variable::Float(if b != 0 { a as f64 / b as f64 } else { 0.0 }) - } - (Variable::Float(a), Variable::Float(b)) => { - Variable::Float(if b != 0.0 { a / b } else { 0.0 }) - } - (Variable::Integer(a), Variable::Float(b)) => { - Variable::Float(if b != 0.0 { a as f64 / b } else { 0.0 }) - } - (Variable::Float(a), Variable::Integer(b)) => { - Variable::Float(if b != 0 { a / b as f64 } else { 0.0 }) - } - (a, b) => a.parse_number().op_divide(b.parse_number()), - } - } - - pub fn op_and(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() & other.to_bool())) - } - - pub fn op_or(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() | other.to_bool())) - } - - pub fn op_xor(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() ^ other.to_bool())) - } - - pub fn op_eq(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self == other)) - } - - pub fn op_ne(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self != other)) - } - - pub fn op_lt(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self < other)) - } - - pub fn op_le(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self <= other)) - } - - pub fn op_gt(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self > other)) - } - - pub fn op_ge(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self >= other)) - } - - pub fn op_not(self) -> Variable<'static> { - Variable::Integer(i64::from(!self.to_bool())) - } - - pub fn op_minus(self) -> Variable<'static> { - match self { - Variable::Integer(n) => Variable::Integer(-n), - Variable::Float(n) => Variable::Float(-n), - _ => self.parse_number().op_minus(), - } - } - - pub fn parse_number(&self) -> Variable<'static> { - match self { - Variable::String(s) if !s.is_empty() => { - if let Ok(n) = s.as_str().parse::() { - Variable::Integer(n) - } else if let Ok(n) = s.as_str().parse::() { - Variable::Float(n) - } else { - Variable::Integer(0) - } - } - Variable::Integer(n) => Variable::Integer(*n), - Variable::Float(n) => Variable::Float(*n), - Variable::Array(l) => Variable::Integer(l.is_empty() as i64), - _ => Variable::Integer(0), - } - } - - #[inline(always)] - fn array(num_items: usize) -> Vec> { - let mut items = Vec::with_capacity(num_items); - for _ in 0..num_items { - items.push(Variable::Integer(0)); - } - items - } - - pub fn to_ref<'y: 'x>(&'y self) -> Variable<'x> { - match self { - Variable::String(s) => Variable::String(StringCow::Borrowed(s.as_str())), - Variable::Integer(n) => Variable::Integer(*n), - Variable::Float(n) => Variable::Float(*n), - Variable::Constant(c) => Variable::Constant(*c), - Variable::Array(l) => Variable::Array(l.iter().map(|v| v.to_ref()).collect::>()), - } - } - - pub fn to_bool(&self) -> bool { - match self { - Variable::Float(f) => *f != 0.0, - Variable::Integer(n) => *n != 0, - Variable::String(s) => !s.is_empty(), - Variable::Array(a) => !a.is_empty(), - Variable::Constant(_) => true, - } - } - - pub fn to_string(&'_ self) -> StringCow<'_> { - match self { - Variable::String(s) => StringCow::Borrowed(s.as_str()), - Variable::Integer(n) => StringCow::Owned(n.to_compact_string()), - Variable::Float(n) => StringCow::Owned(n.to_compact_string()), - Variable::Array(l) => { - let mut result = CompactString::with_capacity(self.len() * 10); - for item in l { - if !result.is_empty() { - result.push_str("\r\n"); - } - match item { - Variable::String(v) => result.push_str(v.as_str()), - Variable::Integer(v) => result.push_str(&v.to_compact_string()), - Variable::Float(v) => result.push_str(&v.to_compact_string()), - Variable::Array(_) => {} - Variable::Constant(c) => result.push_str(c.as_str()), - } - } - StringCow::Owned(result) - } - Variable::Constant(c) => StringCow::Borrowed(c.as_str()), - } - } - - pub fn into_string(self) -> StringCow<'x> { - match self { - Variable::String(s) => s, - Variable::Integer(n) => StringCow::Owned(n.to_compact_string()), - Variable::Float(n) => StringCow::Owned(n.to_compact_string()), - Variable::Array(l) => { - let mut result = CompactString::with_capacity(l.len() * 10); - for item in l { - if !result.is_empty() { - result.push_str("\r\n"); - } - match item { - Variable::String(v) => result.push_str(v.as_ref()), - Variable::Integer(v) => result.push_str(&v.to_compact_string()), - Variable::Float(v) => result.push_str(&v.to_compact_string()), - Variable::Array(_) => {} - Variable::Constant(c) => result.push_str(c.as_str()), - } - } - StringCow::Owned(result) - } - Variable::Constant(c) => StringCow::Borrowed(c.as_str()), - } - } - - pub fn to_integer(&self) -> Option { - match self { - Variable::Integer(n) => Some(*n), - Variable::Float(n) => Some(*n as i64), - Variable::String(s) if !s.is_empty() => s.as_str().parse::().ok(), - _ => None, - } - } - - pub fn to_usize(&self) -> Option { - match self { - Variable::Integer(n) => Some(*n as usize), - Variable::Float(n) => Some(*n as usize), - Variable::String(s) if !s.is_empty() => s.as_str().parse::().ok(), - _ => None, - } - } - - pub fn len(&self) -> usize { - match self { - Variable::String(s) => s.len(), - Variable::Integer(_) | Variable::Float(_) => 2, - Variable::Array(l) => l.iter().map(|v| v.len() + 2).sum(), - Variable::Constant(c) => c.as_str().len(), - } - } - - pub fn is_empty(&self) -> bool { - match self { - Variable::String(s) => s.is_empty(), - _ => false, - } - } - - pub fn as_array(&'_ self) -> Option<&'_ [Variable<'_>]> { - match self { - Variable::Array(l) => Some(l), - _ => None, - } - } - - pub fn into_array(self) -> Vec> { - match self { - Variable::Array(l) => l, - v if !v.is_empty() => vec![v], - _ => vec![], - } - } - - pub fn to_array(&self) -> Vec> { - match self { - Variable::Array(l) => l.iter().map(|v| v.to_ref()).collect::>(), - v if !v.is_empty() => vec![v.to_ref()], - _ => vec![], - } - } - - pub fn into_owned(self) -> Variable<'static> { - match self { - Variable::String(s) => Variable::String(StringCow::Owned(s.into_owned())), - Variable::Integer(n) => Variable::Integer(n), - Variable::Float(n) => Variable::Float(n), - Variable::Constant(c) => Variable::Constant(c), - Variable::Array(l) => Variable::Array(l.into_iter().map(|v| v.into_owned()).collect()), - } - } -} - -impl PartialEq for Variable<'_> { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Integer(a), Self::Integer(b)) => a == b, - (Self::Float(a), Self::Float(b)) => a == b, - (Self::Integer(a), Self::Float(b)) | (Self::Float(b), Self::Integer(a)) => { - *a as f64 == *b - } - (Self::String(a), Self::String(b)) => a.as_str() == b.as_str(), - (Self::String(_), Self::Integer(_) | Self::Float(_)) => &self.parse_number() == other, - (Self::Integer(_) | Self::Float(_), Self::String(_)) => self == &other.parse_number(), - (Self::Array(a), Self::Array(b)) => a == b, - _ => false, - } - } -} - -impl Eq for Variable<'_> {} - -#[allow(clippy::non_canonical_partial_ord_impl)] -impl PartialOrd for Variable<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - match (self, other) { - (Self::Integer(a), Self::Integer(b)) => a.partial_cmp(b), - (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), - (Self::Integer(a), Self::Float(b)) => (*a as f64).partial_cmp(b), - (Self::Float(a), Self::Integer(b)) => a.partial_cmp(&(*b as f64)), - (Self::String(a), Self::String(b)) => a.as_str().partial_cmp(b.as_str()), - (Self::String(_), Self::Integer(_) | Self::Float(_)) => { - self.parse_number().partial_cmp(other) - } - (Self::Integer(_) | Self::Float(_), Self::String(_)) => { - self.partial_cmp(&other.parse_number()) - } - (Self::Array(a), Self::Array(b)) => a.partial_cmp(b), - (Self::Array(_) | Self::String(_), _) => Ordering::Greater.into(), - (Self::Constant(a), Self::Constant(b)) => a.to_id().partial_cmp(&b.to_id()), - (_, Self::Array(_) | Self::Constant(_)) | (Self::Constant(_), _) => { - Ordering::Less.into() - } - } - } -} - -impl Ord for Variable<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other).unwrap_or(Ordering::Greater) - } -} - -impl Display for Variable<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Variable::String(v) => v.fmt(f), - Variable::Integer(v) => v.fmt(f), - Variable::Float(v) => v.fmt(f), - Variable::Array(v) => { - for (i, v) in v.iter().enumerate() { - if i > 0 { - f.write_str("\n")?; - } - v.fmt(f)?; - } - Ok(()) - } - Variable::Constant(c) => c.as_str().fmt(f), - } - } -} - -impl<'x> From<&'x Constant> for Variable<'x> { - fn from(value: &'x Constant) -> Self { - match value { - Constant::Integer(i) => Variable::Integer(*i), - Constant::Float(f) => Variable::Float(*f), - Constant::String(s) => Variable::String(StringCow::Borrowed(s.as_str())), - Constant::Static(c) => Variable::Constant(*c), - } - } -} - -impl<'x> TryFrom> for CompactString { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - if let Variable::String(s) = value { - Ok(match s { - StringCow::Borrowed(v) => v.into(), - StringCow::Owned(v) => v, - }) - } else { - Err(()) - } - } -} - -impl<'x> TryFrom> for String { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - if let Variable::String(s) = value { - Ok(match s { - StringCow::Borrowed(v) => v.to_string(), - StringCow::Owned(v) => v.into_string(), - }) - } else { - Err(()) - } - } -} - -impl<'x> From> for bool { - fn from(val: Variable<'x>) -> Self { - val.to_bool() - } -} - -impl<'x> TryFrom> for i64 { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_integer().ok_or(()) - } -} - -impl<'x> TryFrom> for u64 { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_integer().map(|v| v as u64).ok_or(()) - } -} - -impl<'x> TryFrom> for usize { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_usize().ok_or(()) - } -} - -impl<'x> TryFrom> for StatusCode { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value.to_integer() { - Some(v) => match StatusCode::from_u16(v as u16) { - Ok(status) => Ok(status), - Err(_) => Err(()), - }, - None => Err(()), - } + Step::Done(self.pop()) } } diff --git a/crates/common/src/expr/functions/array.rs b/crates/common/src/expr/functions/array.rs index 69f10e28..bdeb7fae 100644 --- a/crates/common/src/expr/functions/array.rs +++ b/crates/common/src/expr/functions/array.rs @@ -4,62 +4,123 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{FnCtx, args}; use crate::expr::Variable; +use ahash::RandomState; +use bumpalo::{Bump, collections::Vec as BumpVec}; +use std::cmp::Ordering; -pub(crate) fn fn_count(v: Vec) -> Variable { - match &v[0] { +const DEDUP_LINEAR_MAX_ITEMS: usize = 32; +const EMPTY_SLOT: u32 = u32::MAX; + +pub(crate) fn fn_count<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + match value { Variable::Array(a) => a.len(), - v => { - if !v.is_empty() { - 1 - } else { - 0 - } - } + v => usize::from(!v.is_empty()), } .into() } -pub(crate) fn fn_sort(mut v: Vec) -> Variable { - let is_asc = v[1].to_bool(); - let mut arr = v.remove(0).into_array(); - if is_asc { - arr.sort_unstable(); +pub(crate) fn fn_sort<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, ascending] = args(v); + let arena = ctx.arena(); + let items = arena.alloc_slice_copy(value.into_array(arena)); + let ascending = ascending.to_bool(); + if items.iter().all(|item| matches!(item, Variable::String(_))) { + if ascending { + items.sort_unstable_by(|a, b| compare_strings(a, b)); + } else { + items.sort_unstable_by(|a, b| compare_strings(b, a)); + } + } else if ascending { + items.sort_unstable(); } else { - arr.sort_unstable_by(|a, b| b.cmp(a)); + items.sort_unstable_by(|a, b| b.cmp(a)); } - arr.into() + Variable::Array(items) } -pub(crate) fn fn_dedup(mut v: Vec) -> Variable { - let arr = v.remove(0).into_array(); - let mut result = Vec::with_capacity(arr.len()); - - for item in arr { - if !result.contains(&item) { - result.push(item); +pub(crate) fn fn_dedup<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + let arena = ctx.arena(); + let items = value.into_array(arena); + if (DEDUP_LINEAR_MAX_ITEMS + 1..EMPTY_SLOT as usize).contains(&items.len()) + && items.iter().all(|item| matches!(item, Variable::String(_))) + { + return Variable::Array(dedup_strings(items, arena)); + } + let mut result = BumpVec::with_capacity_in(items.len(), arena); + for item in items { + if !result.contains(item) { + result.push(*item); } } - - result.into() + Variable::Array(result.into_bump_slice()) } -pub(crate) fn fn_is_intersect(v: Vec) -> Variable { - match (&v[0], &v[1]) { +pub(crate) fn fn_is_intersect<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [a, b] = args(v); + match (a, b) { (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), - (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), + (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(&item), _ => false, } .into() } -pub(crate) fn fn_winnow(mut v: Vec) -> Variable { - match v.remove(0) { - Variable::Array(a) => a - .into_iter() - .filter(|i| !i.is_empty()) - .collect::>() - .into(), +pub(crate) fn fn_winnow<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + match value { + Variable::Array(a) => { + let mut result = BumpVec::with_capacity_in(a.len(), ctx.arena()); + result.extend(a.iter().filter(|i| !i.is_empty()).copied()); + Variable::Array(result.into_bump_slice()) + } v => v, } } + +fn string_of<'a>(value: &Variable<'a>) -> &'a str { + match value { + Variable::String(s) => s, + _ => "", + } +} + +fn compare_strings(a: &Variable<'_>, b: &Variable<'_>) -> Ordering { + string_of(a).cmp(string_of(b)) +} + +fn dedup_strings<'a>(items: &[Variable<'a>], arena: &'a Bump) -> &'a [Variable<'a>] { + let hasher = RandomState::new(); + let slots = arena.alloc_slice_fill_copy( + items.len().saturating_mul(2).next_power_of_two(), + EMPTY_SLOT, + ); + let mask = slots.len() - 1; + let mut result = BumpVec::with_capacity_in(items.len(), arena); + for item in items { + let text = string_of(item); + let mut slot = (hasher.hash_one(text) as usize) & mask; + loop { + match slots.get(slot).copied() { + Some(EMPTY_SLOT) => { + if let Some(entry) = slots.get_mut(slot) { + *entry = result.len() as u32; + } + result.push(*item); + break; + } + Some(index) if result.get(index as usize).map(string_of) == Some(text) => break, + Some(_) => slot = (slot + 1) & mask, + None => { + result.push(*item); + break; + } + } + } + } + result.into_bump_slice() +} + diff --git a/crates/common/src/expr/functions/asynch.rs b/crates/common/src/expr/functions/asynch.rs index b66b99e0..e0a9cc54 100644 --- a/crates/common/src/expr/functions/asynch.rs +++ b/crates/common/src/expr/functions/asynch.rs @@ -5,128 +5,189 @@ */ use super::*; -use crate::{Server, expr::StringCow}; -use compact_str::{CompactString, ToCompactString}; -use mail_auth::IpLookupStrategy; -use std::{cmp::Ordering, net::IpAddr, vec::IntoIter}; +use crate::Server; +use bumpalo::collections::{String as BumpString, Vec as BumpVec}; +use mail_auth::{ + IpLookupStrategy, + dns::{Mx, RecordSet}, +}; +use std::{ + borrow::Cow, + cmp::Ordering, + fmt::{Display, Write}, + net::{IpAddr, Ipv4Addr, Ipv6Addr}, +}; use store::{Deserialize, Rows, Value, dispatch::lookup::KeyValue}; use trc::AddContext; +const MAX_DNS_IP_RESULTS: usize = 10; + +const STRING_ARGS: [&[usize]; 9] = [ + &[0], + &[0], + &[0, 1], + &[0, 1], + &[0, 1, 2], + &[0, 1], + &[0, 1], + &[0, 1], + &[0, 1], +]; + +pub(crate) struct AsyncCall<'a> { + pub id: u16, + pub args: [Variable<'a>; MAX_ASYNC_ARGS], +} + +pub(crate) enum AsyncResult { + Value(Variable<'static>), + Text(String), + Ips(Vec), + Mx(RecordSet), + Txt(Vec), + Ptr(RecordSet>), + Ipv4(RecordSet), + Ipv6(RecordSet), + Rows(Rows), +} + +#[derive(Debug)] +enum StoredValue { + Integer(i64), + Float(f64), + Text(String), +} + +impl<'a> AsyncCall<'a> { + pub fn new(id: u16, mut args: [Variable<'a>; MAX_ASYNC_ARGS], arena: &'a Bump) -> Self { + let string_args = STRING_ARGS.get(id as usize).copied().unwrap_or_default(); + for (position, arg) in args.iter_mut().enumerate() { + if string_args.contains(&position) { + *arg = Variable::String(arg.to_str(arena)); + } + } + if u32::from(id) == F_SQL_QUERY + && let Some(arg) = args.get_mut(2) + { + *arg = sql_argument(*arg, arena); + } + AsyncCall { id, args } + } + + fn str(&self, position: usize) -> &'a str { + match self.args.get(position) { + Some(Variable::String(s)) => s, + _ => "", + } + } +} + +fn sql_argument<'a>(value: Variable<'a>, arena: &'a Bump) -> Variable<'a> { + let normalize = |value: Variable<'a>| match value { + Variable::String(_) | Variable::Integer(_) | Variable::Float(_) => value, + value => Variable::String(value.to_str(arena)), + }; + match value { + Variable::Array(items) => { + Variable::Array(arena.alloc_slice_fill_iter(items.iter().map(|item| normalize(*item)))) + } + value => normalize(value), + } +} + impl Server { - pub(crate) async fn eval_fnc<'x>( + pub(crate) async fn eval_fnc( &self, - fnc_id: u32, - params: Vec>, + call: AsyncCall<'_>, session_id: u64, - ) -> trc::Result> { - let mut params = FncParams::new(params); - - match fnc_id { - F_IS_LOCAL_DOMAIN => { - let domain = params.next_as_string(); - - self.domain(domain.as_str()) - .await - .caused_by(trc::location!()) - .map(|v| v.is_some().into()) - } - F_IS_LOCAL_ADDRESS => { - let address = params.next_as_string(); - - self.rcpt_id_from_email(address.as_ref()) - .await - .caused_by(trc::location!()) - .map(|v| v.is_some().into()) - } + ) -> trc::Result { + match u32::from(call.id) { + F_IS_LOCAL_DOMAIN => self + .domain(call.str(0)) + .await + .caused_by(trc::location!()) + .map(|v| AsyncResult::Value(v.is_some().into())), + F_IS_LOCAL_ADDRESS => self + .rcpt_id_from_email(call.str(0)) + .await + .caused_by(trc::location!()) + .map(|v| AsyncResult::Value(v.is_some().into())), F_KEY_GET => { - let Some(store) = self.get_lookup_store(params.next_as_string().as_str()) else { - return Ok(Variable::default()); + let Some(store) = self.get_lookup_store(call.str(0)) else { + return Ok(AsyncResult::Value(Variable::default())); }; - let key = params.next_as_string(); - store - .key_get::(key.as_str()) + .key_get::(call.str(1)) .await - .map(|value| value.map(|v| v.into_inner()).unwrap_or_default()) + .map(|value| match value { + Some(StoredValue::Integer(value)) => { + AsyncResult::Value(Variable::Integer(value)) + } + Some(StoredValue::Float(value)) => { + AsyncResult::Value(Variable::Float(value)) + } + Some(StoredValue::Text(text)) => AsyncResult::Text(text), + None => AsyncResult::Value(Variable::default()), + }) .caused_by(trc::location!()) } F_KEY_EXISTS => { - let Some(store) = self.get_lookup_store(params.next_as_string().as_str()) else { - return Ok(Variable::default()); + let Some(store) = self.get_lookup_store(call.str(0)) else { + return Ok(AsyncResult::Value(Variable::default())); }; - let key = params.next_as_string(); - store - .key_exists(key.as_str()) + .key_exists(call.str(1)) .await .caused_by(trc::location!()) - .map(|v| v.into()) + .map(|v| AsyncResult::Value(v.into())) } F_KEY_SET => { - let Some(store) = self.get_lookup_store(params.next_as_string().as_str()) else { - return Ok(Variable::default()); + let Some(store) = self.get_lookup_store(call.str(0)) else { + return Ok(AsyncResult::Value(Variable::default())); }; - let key = params.next_as_string(); - let value = params.next_as_string(); - store .key_set(KeyValue::new( - key.as_bytes().to_vec(), - value.as_bytes().to_vec(), + call.str(1).as_bytes().to_vec(), + call.str(2).as_bytes().to_vec(), )) .await - .map(|_| true) + .map(|_| AsyncResult::Value(true.into())) .caused_by(trc::location!()) - .map(|v| v.into()) } - F_COUNTER_INCR => { - let Some(store) = self.get_lookup_store(params.next_as_string().as_str()) else { - return Ok(Variable::default()); + F_KEY_INCR => { + let Some(store) = self.get_lookup_store(call.str(0)) else { + return Ok(AsyncResult::Value(Variable::default())); }; - let key = params.next_as_string(); - let value = params.next_as_integer(); - + let value = call + .args + .get(2) + .and_then(|v| v.to_integer()) + .unwrap_or_default(); store - .counter_incr(KeyValue::new(key.into_owned(), value), true) + .key_incr(KeyValue::new(call.str(1).as_bytes().to_vec(), value)) .await - .map(Variable::Integer) + .map(|v| AsyncResult::Value(Variable::Integer(v))) .caused_by(trc::location!()) } - F_COUNTER_GET => { - let Some(store) = self.get_lookup_store(params.next_as_string().as_str()) else { - return Ok(Variable::default()); - }; - let key = params.next_as_string(); - - store - .counter_get(key.as_bytes().to_vec()) - .await - .map(Variable::Integer) - .caused_by(trc::location!()) - } - F_DNS_QUERY => self.dns_query(params).await, - F_SQL_QUERY => self.sql_query(params, session_id).await, - _ => Ok(Variable::default()), + F_DNS_QUERY => self.dns_query(&call).await, + F_SQL_QUERY => self.sql_query(&call, session_id).await, + _ => Ok(AsyncResult::Value(Variable::default())), } } - async fn sql_query<'x>( - &self, - mut arguments: FncParams<'x>, - session_id: u64, - ) -> trc::Result> { - let store_name = arguments.next_as_string(); + async fn sql_query(&self, call: &AsyncCall<'_>, session_id: u64) -> trc::Result { + let store_name = call.str(0); let Some(store) = self - .get_lookup_store(store_name.as_ref()) + .get_lookup_store(store_name) .and_then(|v| v.into_store()) else { return Err(trc::EventType::Eval(trc::EvalEvent::Error) .into_err() - .id(store_name.into_owned()) + .id(store_name.to_string()) .span_id(session_id) .details("Store not found or is not a SQL store")); }; - let query = arguments.next_as_string(); + let query = call.str(1); if query.is_empty() { return Err(trc::EventType::Eval(trc::EvalEvent::Error) @@ -135,251 +196,209 @@ impl Server { .span_id(session_id)); } - // Obtain arguments - let arguments = match arguments.next() { - Variable::Array(l) => l.into_iter().map(to_store_value).collect(), + let arguments = match call.args.get(2).copied().unwrap_or_default() { + Variable::Array(l) => l.iter().map(|v| to_store_value(*v)).collect(), v => vec![to_store_value(v)], }; - // Run query if query .as_bytes() .get(..6) .is_some_and(|q| q.eq_ignore_ascii_case(b"SELECT")) { - let mut rows = store - .sql_query::(query.as_str(), arguments) + store + .sql_query::(query, arguments) .await - .caused_by(trc::location!())?; - Ok(match rows.rows.len().cmp(&1) { - Ordering::Equal => { - let mut row = rows.rows.pop().unwrap().values; - match row.len().cmp(&1) { - Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { - row.pop().map(into_variable).unwrap() - } - Ordering::Less => Variable::default(), - _ => { - Variable::Array(row.into_iter().map(into_variable).collect::>()) - } - } - } - Ordering::Less => Variable::default(), - Ordering::Greater => rows - .rows - .into_iter() - .map(|r| { - Variable::Array(r.values.into_iter().map(into_variable).collect::>()) - }) - .collect::>() - .into(), - }) + .map(AsyncResult::Rows) + .caused_by(trc::location!()) } else { store - .sql_query::(query.as_str(), arguments) + .sql_query::(query, arguments) .await + .map(|v| AsyncResult::Value(v.into())) .caused_by(trc::location!()) - .map(|v| v.into()) } } - async fn dns_query<'x>(&self, mut arguments: FncParams<'x>) -> trc::Result> { - let entry = arguments.next_as_string(); - let record_type = arguments.next_as_string(); + async fn dns_query(&self, call: &AsyncCall<'_>) -> trc::Result { + let entry = call.str(0); + let record_type = call.str(1); + let resolver = &self.core.smtp.resolvers.dns; + let cache = &self.inner.cache; - if record_type.as_str().eq_ignore_ascii_case("ip") { - self.core - .smtp - .resolvers - .dns - .ip_lookup( - entry.as_ref(), - IpLookupStrategy::Ipv4thenIpv6, - 10, - Some(&self.inner.cache.dns_ipv4), - Some(&self.inner.cache.dns_ipv6), - ) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| { - result + hashify::fnc_map_ignore_case!(record_type.as_bytes(), + "ip" => { + resolver + .ip_lookup( + entry, + IpLookupStrategy::Ipv4thenIpv6, + MAX_DNS_IP_RESULTS, + Some(&cache.dns_ipv4), + Some(&cache.dns_ipv6), + ) + .await + .map(AsyncResult::Ips) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + "mx" => { + resolver + .mx_lookup(entry, Some(&cache.dns_mx)) + .await + .map(AsyncResult::Mx) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + "txt" => { + resolver + .txt_raw_lookup(entry) + .await + .map(AsyncResult::Txt) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + "ptr" => { + resolver + .ptr_lookup( + entry.parse::().map_err(|err| { + trc::EventType::Eval(trc::EvalEvent::Error) + .into_err() + .details("Failed to parse IP address") + .reason(err) + })?, + Some(&cache.dns_ptr), + ) + .await + .map(AsyncResult::Ptr) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + "ipv4" => { + resolver + .ipv4_lookup(entry, Some(&cache.dns_ipv4)) + .await + .map(AsyncResult::Ipv4) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + "ipv6" => { + resolver + .ipv6_lookup(entry, Some(&cache.dns_ipv6)) + .await + .map(AsyncResult::Ipv6) + .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) + }, + _ => Ok(AsyncResult::Value(Variable::default())), + ) + } +} + +impl AsyncResult { + pub fn into_variable(self, arena: &Bump) -> Variable<'_> { + match self { + AsyncResult::Value(value) => value, + AsyncResult::Text(text) => Variable::String(arena.alloc_str(&text)), + AsyncResult::Ips(ips) => display_all(arena, ips.iter()), + AsyncResult::Mx(mx) => { + let mut items = BumpVec::new_in(arena); + items.extend(mx.records.iter().flat_map(|mx| { + mx.exchanges.iter().map(|host| { + Variable::String(arena.alloc_str(host.strip_suffix('.').unwrap_or(host))) + }) + })); + Variable::Array(items.into_bump_slice()) + } + AsyncResult::Txt(txt) => std::str::from_utf8(&txt) + .map(|text| Variable::String(arena.alloc_str(text))) + .unwrap_or_default(), + AsyncResult::Ptr(ptr) => Variable::Array( + arena.alloc_slice_fill_iter( + ptr.records .iter() - .map(|ip| Variable::from(ip.to_compact_string())) - .collect::>() - .into() - }) - } else if record_type.as_str().eq_ignore_ascii_case("mx") { - self.core - .smtp - .resolvers - .dns - .mx_lookup(entry.as_str(), Some(&self.inner.cache.dns_mx)) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| { - result - .rrset - .iter() - .flat_map(|mx| { - mx.exchanges.iter().map(|host| { - Variable::String(StringCow::Owned( - host.strip_suffix('.').unwrap_or(host).to_compact_string(), - )) - }) - }) - .collect::>() - .into() - }) - } else if record_type.as_str().eq_ignore_ascii_case("txt") { - self.core - .smtp - .resolvers - .dns - .txt_raw_lookup(entry.as_str()) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| Variable::from(CompactString::from_utf8(result).unwrap_or_default())) - } else if record_type.as_str().eq_ignore_ascii_case("ptr") { - self.core - .smtp - .resolvers - .dns - .ptr_lookup( - entry.as_str().parse::().map_err(|err| { - trc::EventType::Eval(trc::EvalEvent::Error) - .into_err() - .details("Failed to parse IP address") - .reason(err) - })?, - Some(&self.inner.cache.dns_ptr), - ) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| { - result - .rrset - .iter() - .map(|host| Variable::from(host.to_compact_string())) - .collect::>() - .into() - }) - } else if record_type.as_str().eq_ignore_ascii_case("ipv4") { - self.core - .smtp - .resolvers - .dns - .ipv4_lookup(entry.as_str(), Some(&self.inner.cache.dns_ipv4)) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| { - result - .rrset - .iter() - .map(|ip| Variable::from(ip.to_compact_string())) - .collect::>() - .into() - }) - } else if record_type.as_str().eq_ignore_ascii_case("ipv6") { - self.core - .smtp - .resolvers - .dns - .ipv6_lookup(entry.as_str(), Some(&self.inner.cache.dns_ipv6)) - .await - .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) - .map(|result| { - result - .rrset - .iter() - .map(|ip| Variable::from(ip.to_compact_string())) - .collect::>() - .into() - }) - } else { - Ok(Variable::default()) + .map(|host| Variable::String(arena.alloc_str(host))), + ), + ), + AsyncResult::Ipv4(ips) => display_all(arena, ips.records.iter()), + AsyncResult::Ipv6(ips) => display_all(arena, ips.records.iter()), + AsyncResult::Rows(rows) => rows_to_variable(rows, arena), } } } -struct FncParams<'x> { - params: IntoIter>, +fn display_all<'a, 'i, T: Display + 'i>( + arena: &'a Bump, + items: impl ExactSizeIterator, +) -> Variable<'a> { + Variable::Array(arena.alloc_slice_fill_iter(items.map(|item| { + let mut out = BumpString::new_in(arena); + let _ = write!(out, "{item}"); + Variable::String(out.into_bump_str()) + }))) } -impl<'x> FncParams<'x> { - pub fn new(params: Vec>) -> Self { - Self { - params: params.into_iter(), +fn rows_to_variable(mut rows: Rows, arena: &Bump) -> Variable<'_> { + match rows.rows.len().cmp(&1) { + Ordering::Equal => { + let mut row = rows.rows.pop().map(|row| row.values).unwrap_or_default(); + match row.len().cmp(&1) { + Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => row + .pop() + .map(|value| into_variable(value, arena)) + .unwrap_or_default(), + Ordering::Less => Variable::default(), + _ => Variable::Array( + arena.alloc_slice_fill_iter(row.into_iter().map(|v| into_variable(v, arena))), + ), + } + } + Ordering::Less => Variable::default(), + Ordering::Greater => { + Variable::Array( + arena.alloc_slice_fill_iter(rows.rows.into_iter().map(|row| { + Variable::Array(arena.alloc_slice_fill_iter( + row.values.into_iter().map(|v| into_variable(v, arena)), + )) + })), + ) } } - - pub fn next_as_string(&mut self) -> StringCow<'x> { - self.params.next().unwrap().into_string() - } - - pub fn next_as_integer(&mut self) -> i64 { - self.params.next().unwrap().to_integer().unwrap_or_default() - } - - pub fn next(&mut self) -> Variable<'x> { - self.params.next().unwrap() - } } -#[derive(Debug)] -struct VariableWrapper(Variable<'static>); - -impl From for VariableWrapper { - fn from(value: i64) -> Self { - VariableWrapper(Variable::Integer(value)) - } -} - -impl Deserialize for VariableWrapper { +impl Deserialize for StoredValue { fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(VariableWrapper(Variable::String(StringCow::Owned( - CompactString::from_utf8_lossy(bytes), - )))) + Ok(StoredValue::Text( + String::from_utf8_lossy(bytes).into_owned(), + )) } } -impl From> for VariableWrapper { - fn from(value: store::Value<'static>) -> Self { - VariableWrapper(match value { - Value::Integer(v) => Variable::Integer(v), - Value::Bool(v) => Variable::Integer(v as i64), - Value::Float(v) => Variable::Float(v), - Value::Text(v) => Variable::String(StringCow::Owned(v.into())), - Value::Blob(v) => Variable::String(StringCow::Owned(match v { - std::borrow::Cow::Borrowed(v) => CompactString::from_utf8_lossy(v), - std::borrow::Cow::Owned(v) => CompactString::from_utf8_lossy(&v), - })), - Value::Null => Variable::String(StringCow::Borrowed("")), - }) +impl From> for StoredValue { + fn from(value: Value<'static>) -> Self { + match value { + Value::Integer(v) => StoredValue::Integer(v), + Value::Bool(v) => StoredValue::Integer(i64::from(v)), + Value::Float(v) => StoredValue::Float(v), + Value::Text(v) => StoredValue::Text(v.into_owned()), + Value::Blob(v) => StoredValue::Text(String::from_utf8_lossy(&v).into_owned()), + Value::Null => StoredValue::Text(String::new()), + } } } -impl VariableWrapper { - pub fn into_inner(self) -> Variable<'static> { - self.0 - } -} - -fn to_store_value(value: Variable) -> Value { +fn to_store_value(value: Variable<'_>) -> Value<'_> { match value { - Variable::String(v) => Value::Text(v.to_string().into()), + Variable::String(v) => Value::Text(Cow::Borrowed(v)), Variable::Integer(v) => Value::Integer(v), Variable::Float(v) => Value::Float(v), - v => Value::Text(v.to_string().into_owned().into()), + _ => Value::Null, } } -fn into_variable(value: Value) -> Variable { +fn into_variable<'a>(value: Value<'_>, arena: &'a Bump) -> Variable<'a> { match value { Value::Integer(v) => Variable::Integer(v), Value::Bool(v) => Variable::Integer(i64::from(v)), Value::Float(v) => Variable::Float(v), - Value::Text(v) => Variable::String(v.into()), - Value::Blob(v) => Variable::String(StringCow::Owned(CompactString::from_utf8_lossy(&v))), + Value::Text(v) => Variable::String(arena.alloc_str(&v)), + Value::Blob(v) => match String::from_utf8_lossy(&v) { + Cow::Borrowed(text) => Variable::String(arena.alloc_str(text)), + Cow::Owned(text) => Variable::String(arena.alloc_str(&text)), + }, Value::Null => Variable::default(), } } diff --git a/crates/common/src/expr/functions/email.rs b/crates/common/src/expr/functions/email.rs index 786219d4..9e276667 100644 --- a/crates/common/src/expr/functions/email.rs +++ b/crates/common/src/expr/functions/email.rs @@ -4,19 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::CompactString; +use super::{FnCtx, args}; +use crate::expr::Variable; -use crate::expr::{StringCow, Variable}; - -pub(crate) fn fn_is_email(v: Vec) -> Variable { +pub(crate) fn fn_is_email<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); let mut last_ch = 0; let mut in_quote = false; let mut at_count = 0; let mut dot_count = 0; let mut lp_len = 0; - let mut value = 0; + let mut value_len = 0; - for &ch in v[0].to_string().as_bytes() { + for &ch in value.to_str(ctx.arena()).as_bytes() { match ch { b'0'..=b'9' | b'a'..=b'z' @@ -41,11 +41,11 @@ pub(crate) fn fn_is_email(v: Vec) -> Variable { | b'}' | b'~' | 0x7f..=u8::MAX => { - value += 1; + value_len += 1; } b'.' if !in_quote => { - if last_ch != b'.' && last_ch != b'@' && value != 0 { - value += 1; + if last_ch != b'.' && last_ch != b'@' && value_len != 0 { + value_len += 1; if at_count == 1 { dot_count += 1; } @@ -55,11 +55,11 @@ pub(crate) fn fn_is_email(v: Vec) -> Variable { } b'@' if !in_quote => { at_count += 1; - lp_len = value; - value = 0; + lp_len = value_len; + value_len = 0; } b'>' | b':' | b',' | b' ' if in_quote => { - value += 1; + value_len += 1; } b'\"' if !in_quote || last_ch != b'\\' => { in_quote = !in_quote; @@ -75,30 +75,27 @@ pub(crate) fn fn_is_email(v: Vec) -> Variable { last_ch = ch; } - (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() + (at_count == 1 && dot_count > 0 && lp_len > 0 && value_len > 0).into() } -pub(crate) fn fn_email_part(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let part = v.next().unwrap().into_string(); +pub(crate) fn fn_email_part<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, part] = args(v); + let arena = ctx.arena(); + let part = hashify::map!(part.to_str(arena).as_bytes(), EmailPart, + "local" => EmailPart::Local, + "domain" => EmailPart::Domain, + ) + .copied(); - value.transform(|s| match s { - StringCow::Borrowed(s) => s - .rsplit_once('@') - .map(|(u, d)| match part.as_str() { - "local" => Variable::from(u.trim()), - "domain" => Variable::from(d.trim()), - _ => Variable::default(), - }) - .unwrap_or_default(), - StringCow::Owned(s) => s - .rsplit_once('@') - .map(|(u, d)| match part.as_str() { - "local" => Variable::from(CompactString::new(u.trim())), - "domain" => Variable::from(CompactString::new(d.trim())), - _ => Variable::default(), - }) - .unwrap_or_default(), + value.transform(arena, |s| match (part, s.rsplit_once('@')) { + (Some(EmailPart::Local), Some((local, _))) => Variable::String(local.trim()), + (Some(EmailPart::Domain), Some((_, domain))) => Variable::String(domain.trim()), + _ => Variable::default(), }) } + +#[derive(Clone, Copy)] +enum EmailPart { + Local, + Domain, +} diff --git a/crates/common/src/expr/functions/misc.rs b/crates/common/src/expr/functions/misc.rs index a40407ef..b10a4f60 100644 --- a/crates/common/src/expr/functions/misc.rs +++ b/crates/common/src/expr/functions/misc.rs @@ -4,14 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{FnCtx, args}; use crate::expr::Variable; -use compact_str::CompactString; -use mail_auth::common::resolver::ToReverseName; +use mail_auth::dns::ToReverseName; use registry::types::ipmask::IpAddrOrMask; use std::{net::IpAddr, str::FromStr}; -pub(crate) fn fn_is_empty(v: Vec) -> Variable { - match &v[0] { +pub(crate) fn fn_is_empty<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + match value { Variable::String(s) => s.is_empty(), Variable::Integer(_) | Variable::Float(_) | Variable::Constant(_) => false, Variable::Array(a) => a.is_empty(), @@ -19,67 +20,62 @@ pub(crate) fn fn_is_empty(v: Vec) -> Variable { .into() } -pub(crate) fn fn_is_number(v: Vec) -> Variable { - matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() +pub(crate) fn fn_is_number<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + matches!(value, Variable::Integer(_) | Variable::Float(_)).into() } -pub(crate) fn fn_bit_and(v: Vec) -> Variable { - match (v[0].to_integer(), v[1].to_integer()) { +pub(crate) fn fn_bit_and<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [a, b] = args(v); + match (a.to_integer(), b.to_integer()) { (Some(lhs), Some(rhs)) => Variable::Integer(lhs & rhs), _ => Variable::Integer(0), } } -pub(crate) fn fn_is_ip_addr(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .parse::() - .is_ok() +fn parse_ip<'a>(ctx: &FnCtx<'a>, value: Variable<'a>) -> Option { + value.to_str(ctx.arena()).parse::().ok() +} + +pub(crate) fn fn_is_ip_addr<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + parse_ip(ctx, value).is_some().into() +} + +pub(crate) fn fn_is_ipv4_addr<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + parse_ip(ctx, value) + .is_some_and(|ip| matches!(ip, IpAddr::V4(_))) .into() } -pub(crate) fn fn_is_ipv4_addr(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .parse::() - .is_ok_and(|ip| matches!(ip, IpAddr::V4(_))) +pub(crate) fn fn_is_ipv6_addr<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + parse_ip(ctx, value) + .is_some_and(|ip| matches!(ip, IpAddr::V6(_))) .into() } -pub(crate) fn fn_is_ipv6_addr(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .parse::() - .is_ok_and(|ip| matches!(ip, IpAddr::V6(_))) - .into() -} - -pub(crate) fn fn_is_ip_in_cidr(v: Vec) -> Variable { - let Ok(ip) = v[0].to_string().as_str().parse::() else { +pub(crate) fn fn_is_ip_in_cidr<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, mask] = args(v); + let Some(ip) = parse_ip(ctx, value) else { return false.into(); }; - IpAddrOrMask::from_str(v[1].to_string().as_str()) + IpAddrOrMask::from_str(mask.to_str(ctx.arena())) .map(|mask| mask.matches(&ip)) .unwrap_or(false) .into() } -pub(crate) fn fn_ip_reverse_name(v: Vec) -> Variable { - CompactString::new( - v[0].to_string() - .as_str() - .parse::() - .map(|ip| ip.to_reverse_name()) - .unwrap_or_default(), - ) - .into() +pub(crate) fn fn_ip_reverse_name<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + match parse_ip(ctx, value) { + Some(ip) => Variable::String(ctx.arena().alloc_str(&ip.to_reverse_name())), + None => Variable::default(), + } } -pub(crate) fn fn_if_then(v: Vec) -> Variable { - let mut v = v.into_iter(); - let condition = v.next().unwrap(); - let iff = v.next().unwrap(); - let then = v.next().unwrap(); - - if condition.to_bool() { iff } else { then } +pub(crate) fn fn_if_then<'a>(_: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [condition, then, otherwise] = args(v); + if condition.to_bool() { then } else { otherwise } } diff --git a/crates/common/src/expr/functions/mod.rs b/crates/common/src/expr/functions/mod.rs index 0c4a13cf..42f03566 100644 --- a/crates/common/src/expr/functions/mod.rs +++ b/crates/common/src/expr/functions/mod.rs @@ -4,7 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{StringCow, Variable}; +use super::Variable; +use bumpalo::Bump; use registry::schema::enums::ExpressionVariable; pub mod array; @@ -14,106 +15,177 @@ pub mod misc; pub mod text; pub trait ResolveVariable: Sync + Send { - fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_>; - fn resolve_global(&self, variable: &str) -> Variable<'_>; + fn resolve_variable<'a>( + &'a self, + variable: ExpressionVariable, + arena: &'a Bump, + ) -> Variable<'a>; + + fn resolve_global<'a>(&'a self, _: &str, _: &'a Bump) -> Variable<'a> { + Variable::Integer(0) + } } -impl<'x> Variable<'x> { - fn transform(self, f: impl Fn(StringCow<'x>) -> Variable<'x>) -> Variable<'x> { +pub(crate) struct FnCtx<'a> { + arena: &'a Bump, +} + +impl<'a> FnCtx<'a> { + pub fn new(arena: &'a Bump) -> Self { + FnCtx { arena } + } + + pub fn arena(&self) -> &'a Bump { + self.arena + } +} + +pub(crate) type SyncFn = for<'a> fn(&FnCtx<'a>, &[Variable<'a>]) -> Variable<'a>; + +pub(crate) fn args<'a, const N: usize>(v: &[Variable<'a>]) -> [Variable<'a>; N] { + let mut out = [Variable::default(); N]; + for (slot, value) in out.iter_mut().zip(v) { + *slot = *value; + } + out +} + +impl<'a> Variable<'a> { + pub(crate) fn transform( + self, + arena: &'a Bump, + f: impl Fn(&'a str) -> Variable<'a>, + ) -> Variable<'a> { match self { Variable::String(s) => f(s), Variable::Array(list) => Variable::Array( - list.into_iter() - .map(|v| match v { - Variable::String(s) => f(s), - v => f(v.into_string()), - }) - .collect::>(), + arena.alloc_slice_fill_iter(list.iter().map(|v| f(v.to_str(arena)))), ), - v => f(v.into_string()), + v => f(v.to_str(arena)), } } } -#[allow(clippy::type_complexity)] -pub(crate) const FUNCTIONS: &[(&str, fn(Vec) -> Variable, u32)] = &[ - ("count", array::fn_count, 1), - ("sort", array::fn_sort, 2), - ("dedup", array::fn_dedup, 1), - ("winnow", array::fn_winnow, 1), - ("is_intersect", array::fn_is_intersect, 2), - ("is_email", email::fn_is_email, 1), - ("email_part", email::fn_email_part, 2), - ("is_empty", misc::fn_is_empty, 1), - ("is_number", misc::fn_is_number, 1), - ("bit_and", misc::fn_bit_and, 2), - ("is_ip_addr", misc::fn_is_ip_addr, 1), - ("is_ipv4_addr", misc::fn_is_ipv4_addr, 1), - ("is_ipv6_addr", misc::fn_is_ipv6_addr, 1), - ("is_ip_in_cidr", misc::fn_is_ip_in_cidr, 2), - ("ip_reverse_name", misc::fn_ip_reverse_name, 1), - ("trim", text::fn_trim, 1), - ("trim_end", text::fn_trim_end, 1), - ("trim_start", text::fn_trim_start, 1), - ("len", text::fn_len, 1), - ("to_lowercase", text::fn_to_lowercase, 1), - ("to_uppercase", text::fn_to_uppercase, 1), - ("is_uppercase", text::fn_is_uppercase, 1), - ("is_lowercase", text::fn_is_lowercase, 1), - ("has_digits", text::fn_has_digits, 1), - ("count_spaces", text::fn_count_spaces, 1), - ("count_uppercase", text::fn_count_uppercase, 1), - ("count_lowercase", text::fn_count_lowercase, 1), - ("count_chars", text::fn_count_chars, 1), - ("contains", text::fn_contains, 2), - ("contains_ignore_case", text::fn_contains_ignore_case, 2), - ("eq_ignore_case", text::fn_eq_ignore_case, 2), - ("starts_with", text::fn_starts_with, 2), - ("ends_with", text::fn_ends_with, 2), - ("lines", text::fn_lines, 1), - ("substring", text::fn_substring, 3), - ("strip_prefix", text::fn_strip_prefix, 2), - ("strip_suffix", text::fn_strip_suffix, 2), - ("split", text::fn_split, 2), - ("rsplit", text::fn_rsplit, 2), - ("split_once", text::fn_split_once, 2), - ("rsplit_once", text::fn_rsplit_once, 2), - ("split_n", text::fn_split_n, 3), - ("split_words", text::fn_split_words, 1), - ("hash", text::fn_hash, 2), - ("if_then", misc::fn_if_then, 3), -]; +pub(crate) const MAX_SYNC_ARGS: usize = 3; pub const F_IS_LOCAL_DOMAIN: u32 = 0; pub const F_IS_LOCAL_ADDRESS: u32 = 1; pub const F_KEY_GET: u32 = 2; pub const F_KEY_EXISTS: u32 = 3; pub const F_KEY_SET: u32 = 4; -pub const F_COUNTER_INCR: u32 = 5; -pub const F_COUNTER_GET: u32 = 6; -pub const F_SQL_QUERY: u32 = 7; -pub const F_DNS_QUERY: u32 = 8; +pub const F_KEY_INCR: u32 = 5; +pub const F_SQL_QUERY: u32 = 6; +pub const F_DNS_QUERY: u32 = 7; -pub const ASYNC_FUNCTIONS: &[(&str, u32, u32)] = &[ - ("is_local_domain", F_IS_LOCAL_DOMAIN, 1), - ("is_local_address", F_IS_LOCAL_ADDRESS, 1), - ("key_get", F_KEY_GET, 2), - ("key_exists", F_KEY_EXISTS, 2), - ("key_set", F_KEY_SET, 3), - ("counter_incr", F_COUNTER_INCR, 3), - ("counter_get", F_COUNTER_GET, 2), - ("dns_query", F_DNS_QUERY, 2), - ("sql_query", F_SQL_QUERY, 3), -]; +pub(crate) struct FunctionEntry { + pub name: &'static str, + pub id: u32, + pub num_args: u32, +} + +macro_rules! functions { + ( + sync { $($sync:ident $sync_name:tt $sync_args:tt => $sync_fn:path,)* } + async { $($async_id:ident $async_name:tt $async_args:tt,)* } + ) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum SyncFunction { + $($sync,)* + } + + impl SyncFunction { + const ALL: &[SyncFunction] = &[$(SyncFunction::$sync,)*]; + + pub(crate) fn from_id(id: u16) -> Option { + Self::ALL.get(usize::from(id)).copied() + } + } + + pub(crate) const FUNCTIONS: &[SyncFn] = &[$($sync_fn,)*]; + + pub(crate) fn lookup_function(name: &str) -> Option<&'static FunctionEntry> { + hashify::map!(name.as_bytes(), FunctionEntry, + $($sync_name => FunctionEntry { + name: $sync_name, + id: SyncFunction::$sync as u32, + num_args: $sync_args, + },)* + $($async_name => FunctionEntry { + name: $async_name, + id: FUNCTIONS.len() as u32 + $async_id, + num_args: $async_args, + },)* + ) + } + }; +} + +functions! { + sync { + Count "count" 1 => array::fn_count, + Sort "sort" 2 => array::fn_sort, + Dedup "dedup" 1 => array::fn_dedup, + Winnow "winnow" 1 => array::fn_winnow, + IsIntersect "is_intersect" 2 => array::fn_is_intersect, + IsEmail "is_email" 1 => email::fn_is_email, + EmailPart "email_part" 2 => email::fn_email_part, + IsEmpty "is_empty" 1 => misc::fn_is_empty, + IsNumber "is_number" 1 => misc::fn_is_number, + BitAnd "bit_and" 2 => misc::fn_bit_and, + IsIpAddr "is_ip_addr" 1 => misc::fn_is_ip_addr, + IsIpv4Addr "is_ipv4_addr" 1 => misc::fn_is_ipv4_addr, + IsIpv6Addr "is_ipv6_addr" 1 => misc::fn_is_ipv6_addr, + IsIpInCidr "is_ip_in_cidr" 2 => misc::fn_is_ip_in_cidr, + IpReverseName "ip_reverse_name" 1 => misc::fn_ip_reverse_name, + Trim "trim" 1 => text::fn_trim, + TrimEnd "trim_end" 1 => text::fn_trim_end, + TrimStart "trim_start" 1 => text::fn_trim_start, + Len "len" 1 => text::fn_len, + ToLowercase "to_lowercase" 1 => text::fn_to_lowercase, + ToUppercase "to_uppercase" 1 => text::fn_to_uppercase, + IsUppercase "is_uppercase" 1 => text::fn_is_uppercase, + IsLowercase "is_lowercase" 1 => text::fn_is_lowercase, + HasDigits "has_digits" 1 => text::fn_has_digits, + CountSpaces "count_spaces" 1 => text::fn_count_spaces, + CountUppercase "count_uppercase" 1 => text::fn_count_uppercase, + CountLowercase "count_lowercase" 1 => text::fn_count_lowercase, + CountChars "count_chars" 1 => text::fn_count_chars, + Contains "contains" 2 => text::fn_contains, + ContainsIgnoreCase "contains_ignore_case" 2 => text::fn_contains_ignore_case, + EqIgnoreCase "eq_ignore_case" 2 => text::fn_eq_ignore_case, + StartsWith "starts_with" 2 => text::fn_starts_with, + EndsWith "ends_with" 2 => text::fn_ends_with, + Lines "lines" 1 => text::fn_lines, + Substring "substring" 3 => text::fn_substring, + StripPrefix "strip_prefix" 2 => text::fn_strip_prefix, + StripSuffix "strip_suffix" 2 => text::fn_strip_suffix, + Split "split" 2 => text::fn_split, + Rsplit "rsplit" 2 => text::fn_rsplit, + SplitOnce "split_once" 2 => text::fn_split_once, + RsplitOnce "rsplit_once" 2 => text::fn_rsplit_once, + SplitN "split_n" 3 => text::fn_split_n, + SplitWords "split_words" 1 => text::fn_split_words, + Hash "hash" 2 => text::fn_hash, + IfThen "if_then" 3 => misc::fn_if_then, + } + async { + F_IS_LOCAL_DOMAIN "is_local_domain" 1, + F_IS_LOCAL_ADDRESS "is_local_address" 1, + F_KEY_GET "key_get" 2, + F_KEY_EXISTS "key_exists" 2, + F_KEY_SET "key_set" 3, + F_KEY_INCR "key_incr" 3, + F_DNS_QUERY "dns_query" 2, + F_SQL_QUERY "sql_query" 3, + } +} + +pub(crate) const MAX_ASYNC_ARGS: usize = 3; pub struct EmptyResolver; impl ResolveVariable for EmptyResolver { - fn resolve_variable(&self, _: ExpressionVariable) -> Variable<'_> { - Variable::Integer(0) - } - - fn resolve_global(&self, _: &str) -> Variable<'_> { + fn resolve_variable<'a>(&'a self, _: ExpressionVariable, _: &'a Bump) -> Variable<'a> { Variable::Integer(0) } } diff --git a/crates/common/src/expr/functions/text.rs b/crates/common/src/expr/functions/text.rs index 3b0eb96b..1bf30426 100644 --- a/crates/common/src/expr/functions/text.rs +++ b/crates/common/src/expr/functions/text.rs @@ -4,356 +4,295 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use compact_str::{CompactString, ToCompactString, format_compact}; -use sha1::Sha1; +use super::{FnCtx, args}; +use crate::expr::{Variable, kernels}; +use bumpalo::{Bump, collections::Vec as BumpVec}; +use memchr::{memchr_iter, memmem}; +use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; -use utils::HexEncode; -use crate::expr::{StringCow, Variable}; +const EMPTY_SEPARATOR: &str = ""; -pub(crate) fn fn_trim(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - StringCow::Borrowed(s) => Variable::from(s.trim()), - StringCow::Owned(s) => Variable::from(s.trim().to_compact_string()), - }) +fn contains<'a>(haystack: Variable<'a>, needle: Variable<'a>, arena: &'a Bump) -> bool { + match haystack { + Variable::String(s) => s.contains(needle.to_str(arena)), + Variable::Array(items) => items.contains(&needle), + value => value.to_str(arena).contains(needle.to_str(arena)), + } } -pub(crate) fn fn_trim_end(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - StringCow::Borrowed(s) => Variable::from(s.trim_end()), - StringCow::Owned(s) => Variable::from(s.trim_end().to_compact_string()), - }) +pub(crate) fn contains_ignore_case<'a>( + haystack: Variable<'a>, + needle: &str, + arena: &'a Bump, + in_string: impl FnOnce(&'a str) -> bool, +) -> bool { + match haystack { + Variable::String(s) => in_string(s), + Variable::Array(items) => items + .iter() + .any(|item| matches!(item, Variable::String(s) if s.eq_ignore_ascii_case(needle))), + value => value.to_str(arena).contains(needle), + } } -pub(crate) fn fn_trim_start(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - StringCow::Borrowed(s) => Variable::from(s.trim_start()), - StringCow::Owned(s) => Variable::from(s.trim_start().to_compact_string()), - }) +fn collect_strs<'a>(arena: &'a Bump, items: impl Iterator) -> Variable<'a> { + let mut out = BumpVec::new_in(arena); + out.extend(items.map(Variable::String)); + Variable::Array(out.into_bump_slice()) } -pub(crate) fn fn_len(v: Vec) -> Variable { - match &v[0] { +fn split_capacity(value: &str, separator: &str) -> usize { + match separator.as_bytes() { + [] => value.len() + 2, + [byte] => memchr_iter(*byte, value.as_bytes()).count() + 1, + needle => memmem::find_iter(value.as_bytes(), needle).count() + 1, + } +} + +fn single_byte(separator: &str) -> Option { + match separator.as_bytes() { + [byte] => Some(char::from(*byte)), + _ => None, + } +} + +fn pair<'a>(arena: &'a Bump, (a, b): (&'a str, &'a str)) -> Variable<'a> { + Variable::Array(arena.alloc_slice_copy(&[Variable::String(a), Variable::String(b)])) +} + +pub(crate) fn fn_trim<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| Variable::String(s.trim())) +} + +pub(crate) fn fn_trim_end<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| Variable::String(s.trim_end())) +} + +pub(crate) fn fn_trim_start<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| Variable::String(s.trim_start())) +} + +pub(crate) fn fn_len<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + match value { Variable::String(s) => s.len(), Variable::Array(a) => a.len(), - v => v.to_string().len(), + value => value.to_str(ctx.arena()).len(), } .into() } -pub(crate) fn fn_to_lowercase(mut v: Vec) -> Variable { - v.remove(0) - .transform(|s| Variable::from(CompactString::from_str_to_lowercase(s.as_str()))) +pub(crate) fn fn_to_lowercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + let arena = ctx.arena(); + value.transform(arena, |s| Variable::String(kernels::to_lowercase(s, arena))) } -pub(crate) fn fn_to_uppercase(mut v: Vec) -> Variable { - v.remove(0) - .transform(|s| Variable::from(CompactString::from_str_to_uppercase(s.as_str()))) +pub(crate) fn fn_to_uppercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + let arena = ctx.arena(); + value.transform(arena, |s| Variable::String(kernels::to_uppercase(s, arena))) } -pub(crate) fn fn_is_uppercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| { - s.as_str() - .chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_uppercase()) - .into() +pub(crate) fn fn_is_uppercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| kernels::is_uppercase(s).into()) +} + +pub(crate) fn fn_is_lowercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| kernels::is_lowercase(s).into()) +} + +pub(crate) fn fn_has_digits<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + value.transform(ctx.arena(), |s| kernels::has_digits(s).into()) +} + +pub(crate) fn fn_split_words<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + let arena = ctx.arena(); + collect_strs(arena, kernels::split_words(value.to_str(arena))) +} + +pub(crate) fn fn_count_spaces<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + kernels::count_whitespace(value.to_str(ctx.arena())).into() +} + +pub(crate) fn fn_count_uppercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + kernels::count_uppercase(value.to_str(ctx.arena())).into() +} + +pub(crate) fn fn_count_lowercase<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + kernels::count_lowercase(value.to_str(ctx.arena())).into() +} + +pub(crate) fn fn_count_chars<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); + kernels::count_chars(value.to_str(ctx.arena())).into() +} + +pub(crate) fn fn_eq_ignore_case<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [a, b] = args(v); + let arena = ctx.arena(); + a.to_str(arena).eq_ignore_ascii_case(b.to_str(arena)).into() +} + +pub(crate) fn fn_contains<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [haystack, needle] = args(v); + contains(haystack, needle, ctx.arena()).into() +} + +pub(crate) fn fn_contains_ignore_case<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [haystack, needle] = args(v); + let arena = ctx.arena(); + let needle = needle.to_str(arena); + contains_ignore_case(haystack, needle, arena, |s| { + kernels::contains_ignore_case(s, needle) }) -} - -pub(crate) fn fn_is_lowercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| { - s.as_str() - .chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_lowercase()) - .into() - }) -} - -pub(crate) fn fn_has_digits(mut v: Vec) -> Variable { - v.remove(0) - .transform(|s| s.as_str().chars().any(|c| c.is_ascii_digit()).into()) -} - -pub(crate) fn fn_split_words(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(CompactString::new(word))) - .collect::>() - .into() -} - -pub(crate) fn fn_count_spaces(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .chars() - .filter(|c| c.is_whitespace()) - .count() - .into() -} - -pub(crate) fn fn_count_uppercase(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .chars() - .filter(|c| c.is_alphabetic() && c.is_uppercase()) - .count() - .into() -} - -pub(crate) fn fn_count_lowercase(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .chars() - .filter(|c| c.is_alphabetic() && c.is_lowercase()) - .count() - .into() -} - -pub(crate) fn fn_count_chars(v: Vec) -> Variable { - v[0].to_string().as_str().chars().count().into() -} - -pub(crate) fn fn_eq_ignore_case(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .eq_ignore_ascii_case(v[1].to_string().as_str()) - .into() -} - -pub(crate) fn fn_contains(v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.as_str().contains(v[1].to_string().as_str()), - Variable::Array(arr) => arr.contains(&v[1]), - val => val.to_string().as_str().contains(v[1].to_string().as_str()), - } .into() } -pub(crate) fn fn_contains_ignore_case(v: Vec) -> Variable { - let needle = v[1].to_string(); - match &v[0] { - Variable::String(s) => s - .as_str() - .to_lowercase() - .contains(&needle.as_str().to_lowercase()), - Variable::Array(arr) => arr.iter().any(|v| match v { - Variable::String(s) => s.as_str().eq_ignore_ascii_case(needle.as_str()), - _ => false, - }), - val => val.to_string().as_str().contains(needle.as_str()), - } - .into() +pub(crate) fn fn_starts_with<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [a, b] = args(v); + let arena = ctx.arena(); + a.to_str(arena).starts_with(b.to_str(arena)).into() } -pub(crate) fn fn_starts_with(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .starts_with(v[1].to_string().as_str()) - .into() +pub(crate) fn fn_ends_with<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [a, b] = args(v); + let arena = ctx.arena(); + a.to_str(arena).ends_with(b.to_str(arena)).into() } -pub(crate) fn fn_ends_with(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .ends_with(v[1].to_string().as_str()) - .into() -} - -pub(crate) fn fn_lines(mut v: Vec) -> Variable { - match v.remove(0) { - Variable::String(s) => s - .as_str() - .lines() - .map(|s| Variable::from(CompactString::new(s))) - .collect::>() - .into(), - val => val, - } -} - -pub(crate) fn fn_substring(v: Vec) -> Variable { - v[0].to_string() - .as_str() - .chars() - .skip(v[1].to_usize().unwrap_or_default()) - .take(v[2].to_usize().unwrap_or_default()) - .collect::() - .into() -} - -pub(crate) fn fn_strip_prefix(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let prefix = v.next().unwrap().into_string(); - - value.transform(|s| match s { - StringCow::Borrowed(s) => s - .strip_prefix(prefix.as_str()) - .map(Variable::from) - .unwrap_or_default(), - StringCow::Owned(s) => s - .strip_prefix(prefix.as_str()) - .map(|s| Variable::from(CompactString::new(s))) - .unwrap_or_default(), - }) -} - -pub(crate) fn fn_strip_suffix(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let suffix = v.next().unwrap().into_string(); - - value.transform(|s| match s { - StringCow::Borrowed(s) => s - .strip_suffix(suffix.as_str()) - .map(Variable::from) - .unwrap_or_default(), - StringCow::Owned(s) => s - .strip_suffix(suffix.as_str()) - .map(|s| Variable::from(CompactString::new(s))) - .unwrap_or_default(), - }) -} - -pub(crate) fn fn_split(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - +pub(crate) fn fn_lines<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value] = args(v); match value { - StringCow::Borrowed(s) => s - .split(arg.as_str()) - .map(Variable::from) - .collect::>() - .into(), - StringCow::Owned(s) => s - .split(arg.as_str()) - .map(|s| Variable::from(CompactString::new(s))) - .collect::>() - .into(), + Variable::String(s) => collect_strs(ctx.arena(), s.lines()), + value => value, } } -pub(crate) fn fn_rsplit(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - StringCow::Borrowed(s) => s - .rsplit(arg.as_str()) - .map(Variable::from) - .collect::>() - .into(), - StringCow::Owned(s) => s - .rsplit(arg.as_str()) - .map(|s| Variable::from(CompactString::new(s))) - .collect::>() - .into(), - } +pub(crate) fn fn_substring<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, start, count] = args(v); + Variable::String(kernels::substring( + value.to_str(ctx.arena()), + start.to_usize().unwrap_or_default(), + count.to_usize().unwrap_or_default(), + )) } -pub(crate) fn fn_split_n(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - let num = v.next().unwrap().to_integer().unwrap_or_default() as usize; +pub(crate) fn fn_strip_prefix<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, prefix] = args(v); + let arena = ctx.arena(); + let prefix = prefix.to_str(arena); + value.transform(arena, |s| { + s.strip_prefix(prefix) + .map(Variable::String) + .unwrap_or_default() + }) +} - fn split_n<'x, 'y>(s: &'x str, arg: &'y str, num: usize, mut f: impl FnMut(&'x str)) { - let mut s = s; - for _ in 0..num { - if let Some((a, b)) = s.split_once(arg) { - f(a); - s = b; - } else { - break; +pub(crate) fn fn_strip_suffix<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, suffix] = args(v); + let arena = ctx.arena(); + let suffix = suffix.to_str(arena); + value.transform(arena, |s| { + s.strip_suffix(suffix) + .map(Variable::String) + .unwrap_or_default() + }) +} + +pub(crate) fn fn_split<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, separator] = args(v); + let arena = ctx.arena(); + let value = value.to_str(arena); + let separator = separator.to_str(arena); + let mut out = BumpVec::with_capacity_in(split_capacity(value, separator), arena); + match single_byte(separator) { + Some(separator) => out.extend(value.split(separator).map(Variable::String)), + None => out.extend(value.split(separator).map(Variable::String)), + } + Variable::Array(out.into_bump_slice()) +} + +pub(crate) fn fn_rsplit<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, separator] = args(v); + let arena = ctx.arena(); + let value = value.to_str(arena); + let separator = separator.to_str(arena); + let mut out = BumpVec::with_capacity_in(split_capacity(value, separator), arena); + match single_byte(separator) { + Some(separator) => out.extend(value.rsplit(separator).map(Variable::String)), + None => out.extend(value.rsplit(separator).map(Variable::String)), + } + Variable::Array(out.into_bump_slice()) +} + +pub(crate) fn fn_split_n<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, separator, count] = args(v); + let arena = ctx.arena(); + let mut rest = value.to_str(arena); + let separator = separator.to_str(arena); + let count = count.to_integer().unwrap_or_default() as usize; + if separator.is_empty() { + return collect_strs(arena, rest.splitn(count.saturating_add(1), EMPTY_SEPARATOR)); + } + + let mut out = BumpVec::new_in(arena); + for _ in 0..count { + match rest.split_once(separator) { + Some((head, tail)) => { + out.push(Variable::String(head)); + rest = tail; } + None => break, } - f(s); } - - let mut result = Vec::new(); - match value { - StringCow::Borrowed(s) => split_n(s, arg.as_str(), num, |s| result.push(Variable::from(s))), - StringCow::Owned(s) => split_n(&s, arg.as_str(), num, |s| { - result.push(Variable::from(CompactString::new(s))) - }), - } - - result.into() + out.push(Variable::String(rest)); + Variable::Array(out.into_bump_slice()) } -pub(crate) fn fn_split_once(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - StringCow::Borrowed(s) => s - .split_once(arg.as_str()) - .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) - .unwrap_or_default(), - StringCow::Owned(s) => s - .split_once(arg.as_str()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::from(CompactString::new(a)), - Variable::from(CompactString::new(b)), - ]) - }) - .unwrap_or_default(), - } +pub(crate) fn fn_split_once<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, separator] = args(v); + let arena = ctx.arena(); + value + .to_str(arena) + .split_once(separator.to_str(arena)) + .map(|parts| pair(arena, parts)) + .unwrap_or_default() } -pub(crate) fn fn_rsplit_once(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - StringCow::Borrowed(s) => s - .rsplit_once(arg.as_str()) - .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) - .unwrap_or_default(), - StringCow::Owned(s) => s - .rsplit_once(arg.as_str()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::from(CompactString::new(a)), - Variable::from(CompactString::new(b)), - ]) - }) - .unwrap_or_default(), - } +pub(crate) fn fn_rsplit_once<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, separator] = args(v); + let arena = ctx.arena(); + value + .to_str(arena) + .rsplit_once(separator.to_str(arena)) + .map(|parts| pair(arena, parts)) + .unwrap_or_default() } -pub(crate) fn fn_hash(v: Vec) -> Variable { - use sha1::Digest; - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let algo = v.next().unwrap().into_string(); +pub(crate) fn fn_hash<'a>(ctx: &FnCtx<'a>, v: &[Variable<'a>]) -> Variable<'a> { + let [value, algorithm] = args(v); + let arena = ctx.arena(); + let value = value.to_str(arena).as_bytes(); - match algo.as_str() { - "md5" => format_compact!("{:x}", md5::compute(value.as_bytes())).into(), - "sha1" => { - let mut hasher = Sha1::new(); - hasher.update(value.as_bytes()); - hasher.finalize().hex_encode().to_compact_string().into() - } - "sha256" => { - let mut hasher = Sha256::new(); - hasher.update(value.as_bytes()); - hasher.finalize().hex_encode().to_compact_string().into() - } - "sha512" => { - let mut hasher = Sha512::new(); - hasher.update(value.as_bytes()); - hasher.finalize().hex_encode().to_compact_string().into() - } + let hex = |digest: &[u8]| Variable::String(kernels::hex_encode(digest, arena)); + hashify::fnc_map!(algorithm.to_str(arena).as_bytes(), + "md5" => hex(&md5::compute(value).0), + "sha1" => hex(&Sha1::digest(value)), + "sha256" => hex(&Sha256::digest(value)), + "sha512" => hex(&Sha512::digest(value)), _ => Variable::default(), - } + ) } + diff --git a/crates/common/src/expr/guard.rs b/crates/common/src/expr/guard.rs new file mode 100644 index 00000000..3b8a17ba --- /dev/null +++ b/crates/common/src/expr/guard.rs @@ -0,0 +1,350 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + BinaryOperator, Constant, ExpressionItem, UnaryOperator, + functions::{F_KEY_INCR, F_KEY_SET, F_SQL_QUERY, FUNCTIONS, SyncFunction}, + if_block::IfBlock, +}; +use ahash::AHashMap; +use registry::schema::enums::ExpressionVariable; + +const SIDE_EFFECTS: [u32; 3] = [F_KEY_SET, F_KEY_INCR, F_SQL_QUERY]; +const MAX_GUARD_ITEMS: usize = 256; + +#[derive(Debug, Clone, Default)] +pub struct ScopeRules { + rules: Box<[IfBlock]>, + all: Box<[u32]>, + unguarded: Box<[u32]>, + by_value: AHashMap, Box<[u32]>>, +} + +#[derive(Clone, Copy)] +pub struct Selection<'r> { + rules: &'r [IfBlock], + order: &'r [u32], +} + +enum Node<'e> { + Var(ExpressionVariable), + Str(&'e str), + Array(Vec>), + Contains(Box>, Box>), + Eq(Box>, Box>), + Ne(Box>, Box>), + Not(Box>), + And(Box>, Box>), + Or(Box>, Box>), + Other, +} + +struct Span<'e> { + node: Node<'e>, + start: usize, + end: usize, +} + +impl ScopeRules { + pub fn new(rules: Vec, guard: Option) -> Self { + let guards = rules + .iter() + .map(|rule| guard.and_then(|variable| rule_guard(rule, variable))) + .collect::>(); + + let select = |accepts: &dyn Fn(Option<&[&str]>) -> bool| { + guards + .iter() + .enumerate() + .filter(|(_, guard)| accepts(guard.as_deref())) + .map(|(index, _)| index as u32) + .collect::>() + }; + + let by_value = distinct(guards.iter().flatten().flatten().copied().collect()) + .iter() + .map(|value| { + ( + Box::::from(*value), + select(&|guard| guard.is_none_or(|values| values.contains(value))), + ) + }) + .collect(); + + ScopeRules { + unguarded: select(&|guard| guard.is_none()), + all: (0..rules.len() as u32).collect(), + by_value, + rules: rules.into_boxed_slice(), + } + } + + pub fn select(&self, value: &str) -> Selection<'_> { + Selection { + rules: &self.rules, + order: self.by_value.get(value).unwrap_or(&self.unguarded), + } + } + + pub fn all(&self) -> Selection<'_> { + Selection { + rules: &self.rules, + order: &self.all, + } + } + + pub fn is_empty(&self) -> bool { + self.rules.is_empty() + } +} + +#[cfg(all(test, stalwart_tests))] +impl ScopeRules { + fn len(&self) -> usize { + self.rules.len() + } + + fn guarded(&self) -> usize { + self.rules.len() - self.unguarded.len() + } + + fn blocks(&self) -> &[IfBlock] { + &self.rules + } +} + +impl<'r> Selection<'r> { + pub fn is_empty(&self) -> bool { + self.order.is_empty() + } + + pub fn iter(&self) -> impl Iterator + use<'r> { + let (rules, order) = (self.rules, self.order); + order + .iter() + .filter_map(move |index| rules.get(*index as usize)) + } +} + +fn rule_guard(block: &IfBlock, variable: ExpressionVariable) -> Option> { + let mut values = Vec::new(); + for if_then in &block.if_then { + let items = if_then.expr.items(); + if items.len() > MAX_GUARD_ITEMS || items.iter().any(has_side_effects) { + return None; + } + let tree = build_tree(items)?; + if let Some(guard) = conjunct_guard(&tree, variable) { + values.extend(guard); + } else if yields_nothing(if_then.then.items()) + && let Some(exit) = exit_guard(&tree, variable) + { + values.extend(exit); + return Some(distinct(values)); + } else { + return None; + } + } + yields_nothing(block.default.items()).then(|| distinct(values)) +} + +fn distinct(mut values: Vec<&str>) -> Vec<&str> { + values.sort_unstable(); + values.dedup(); + values +} + +fn yields_nothing(items: &[ExpressionItem]) -> bool { + matches!( + items, + [ExpressionItem::Constant( + Constant::Integer(_) | Constant::Float(_) | Constant::Static(_) + )] + ) +} + +fn has_side_effects(item: &ExpressionItem) -> bool { + match item { + ExpressionItem::Function { id, .. } => id + .checked_sub(FUNCTIONS.len() as u32) + .is_some_and(|id| SIDE_EFFECTS.contains(&id)), + _ => false, + } +} + +fn build_tree(items: &[ExpressionItem]) -> Option> { + let mut stack: Vec> = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + let (node, start) = match item { + ExpressionItem::JmpIf { .. } => continue, + ExpressionItem::Variable(variable) => (Node::Var(*variable), index), + ExpressionItem::Constant(Constant::String(value)) => (Node::Str(value.as_str()), index), + ExpressionItem::Constant(_) + | ExpressionItem::Global(_) + | ExpressionItem::System(_) + | ExpressionItem::Capture(_) => (Node::Other, index), + ExpressionItem::UnaryOperator(op) => { + let (mut operands, start) = pop_operands(&mut stack, 1, index)?; + match (op, operands.pop()) { + (UnaryOperator::Not, Some(operand)) => (Node::Not(Box::new(operand)), start), + _ => (Node::Other, start), + } + } + ExpressionItem::Regex(_) => (Node::Other, pop_operands(&mut stack, 1, index)?.1), + ExpressionItem::ArrayAccess => (Node::Other, pop_operands(&mut stack, 2, index)?.1), + ExpressionItem::ArrayBuild(count) => { + let (items, start) = pop_operands(&mut stack, *count as usize, index)?; + (Node::Array(items), start) + } + ExpressionItem::Function { id, num_args } => { + let (mut args, start) = pop_operands(&mut stack, *num_args as usize, index)?; + let node = match (*id == SyncFunction::Contains as u32, args.pop(), args.pop()) { + (true, Some(needle), Some(haystack)) if args.is_empty() => { + Node::Contains(Box::new(haystack), Box::new(needle)) + } + _ => Node::Other, + }; + (node, start) + } + ExpressionItem::BinaryOperator(op) => { + let right = stack.pop()?; + let left = stack.pop()?; + let short_circuit = match op { + BinaryOperator::And => Some(false), + BinaryOperator::Or => Some(true), + _ => None, + }; + let well_formed = right.end == index + && match short_circuit { + Some(val) => { + right.start == left.end + 1 + && matches!(items.get(left.end), Some(ExpressionItem::JmpIf { val: v, pos }) + if *v == val && left.end + *pos as usize == index) + } + None => right.start == left.end, + }; + if !well_formed { + return None; + } + let (left_node, right_node) = (Box::new(left.node), Box::new(right.node)); + let node = match op { + BinaryOperator::And => Node::And(left_node, right_node), + BinaryOperator::Or => Node::Or(left_node, right_node), + BinaryOperator::Eq => Node::Eq(left_node, right_node), + BinaryOperator::Ne => Node::Ne(left_node, right_node), + _ => Node::Other, + }; + (node, left.start) + } + }; + stack.push(Span { + node, + start, + end: index + 1, + }); + } + match (stack.pop(), stack.is_empty()) { + ( + Some(Span { + node, + start: 0, + end, + }), + true, + ) if end == items.len() => Some(node), + _ => None, + } +} + +fn pop_operands<'e>( + stack: &mut Vec>, + count: usize, + index: usize, +) -> Option<(Vec>, usize)> { + let base = stack.len().checked_sub(count)?; + let operands = stack.get(base..)?; + let contiguous = operands.windows(2).all(|pair| match pair { + [left, right] => left.end == right.start, + _ => false, + }) && operands.last().is_none_or(|last| last.end == index); + if !contiguous { + return None; + } + let start = operands.first().map_or(index, |first| first.start); + Some((stack.drain(base..).map(|span| span.node).collect(), start)) +} + +fn compared_value<'e>( + left: &Node<'e>, + right: &Node<'e>, + variable: ExpressionVariable, +) -> Option> { + match (left, right) { + (Node::Var(v), Node::Str(value)) | (Node::Str(value), Node::Var(v)) if *v == variable => { + Some(vec![*value]) + } + _ => None, + } +} + +fn guard_values<'e>(node: &Node<'e>, variable: ExpressionVariable) -> Option> { + match node { + Node::Eq(left, right) => compared_value(left, right, variable), + Node::Contains(haystack, needle) => match (haystack.as_ref(), needle.as_ref()) { + (Node::Array(items), Node::Var(v)) if *v == variable => items + .iter() + .map(|item| match item { + Node::Str(value) => Some(*value), + _ => None, + }) + .collect(), + _ => None, + }, + Node::Or(left, right) => { + let mut values = guard_values(left, variable)?; + values.extend(guard_values(right, variable)?); + Some(values) + } + _ => None, + } +} + +fn exit_guard<'e>(node: &Node<'e>, variable: ExpressionVariable) -> Option> { + match node { + Node::Not(inner) => guard_values(inner, variable), + Node::Ne(left, right) => compared_value(left, right, variable), + Node::Or(left, right) => match (exit_guard(left, variable), exit_guard(right, variable)) { + (Some(left), Some(right)) => { + Some(left.into_iter().filter(|v| right.contains(v)).collect()) + } + (Some(values), None) | (None, Some(values)) => Some(values), + (None, None) => None, + }, + _ => None, + } +} + +fn conjunct_guard<'e>(node: &Node<'e>, variable: ExpressionVariable) -> Option> { + if let Some(values) = guard_values(node, variable) { + return Some(values); + } + match node { + Node::And(left, right) => { + match ( + conjunct_guard(left, variable), + conjunct_guard(right, variable), + ) { + (Some(left), Some(right)) => { + Some(left.into_iter().filter(|v| right.contains(v)).collect()) + } + (Some(values), None) | (None, Some(values)) => Some(values), + (None, None) => None, + } + } + _ => None, + } +} + diff --git a/crates/common/src/expr/if_block.rs b/crates/common/src/expr/if_block.rs index e44974c7..f6e6f95a 100644 --- a/crates/common/src/expr/if_block.rs +++ b/crates/common/src/expr/if_block.rs @@ -7,10 +7,10 @@ use super::{ ExpressionItem, parser::ExpressionParser, + program::{Program, ProgramBuilder}, tokenizer::{TokenMap, Tokenizer}, }; -use crate::expr::{Constant, Expression}; -use compact_str::CompactString; +use crate::expr::Expression; use registry::{ schema::{ prelude::{ExpressionContext, Property}, @@ -26,37 +26,27 @@ pub struct IfThen { pub then: Expression, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct IfBlock { pub id: ObjectId, pub property: Property, pub if_then: Box<[IfThen]>, pub default: Expression, + pub(crate) program: Program, } -impl IfBlock { - pub fn new_default(id: ObjectId, expr_ctx: ExpressionContext<'_>) -> Self { - let token_map = TokenMap::default(); - - if let Some(default) = expr_ctx.default { - Self { - id, - property: expr_ctx.property, - if_then: default - .match_ - .into_iter() - .map(|match_| IfThen { - expr: Expression::parse(&token_map, &match_.if_), - then: Expression::parse(&token_map, &match_.then), - }) - .collect(), - default: Expression::parse(&token_map, &default.else_), - } - } else { - Self::empty(id, expr_ctx.property) - } +impl PartialEq for IfBlock { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.property == other.property + && self.if_then == other.if_then + && self.default == other.default } +} +impl Eq for IfBlock {} + +impl IfBlock { pub fn empty(id: ObjectId, property: Property) -> Self { Self { id, @@ -65,19 +55,74 @@ impl IfBlock { default: Expression { items: Default::default(), }, + program: Program::default(), } } + pub fn new( + id: ObjectId, + property: Property, + if_then: Box<[IfThen]>, + default: Expression, + ) -> Self { + let mut block = Self { + id, + property, + if_then, + default, + program: Program::default(), + }; + if !block.is_empty() { + block.program = block.lower(); + } + block + } + + fn lower(&self) -> Program { + let (ops, widest) = self + .if_then + .iter() + .flat_map(|if_then| [&if_then.expr, &if_then.then]) + .chain([&self.default]) + .fold((2 * self.if_then.len() + 1, 0), |(ops, widest), expr| { + (ops + expr.items.len(), widest.max(expr.items.len())) + }); + let mut builder = ProgramBuilder::with_capacity(ops, widest); + for if_then in &self.if_then { + builder.push_expression(&if_then.expr.items); + let test = builder.push_test(); + builder.push_expression(&if_then.then.items); + builder.push_return(); + builder.patch_test(test); + } + builder.push_expression(&self.default.items); + builder.push_return(); + builder.build() + } + pub fn is_empty(&self) -> bool { self.default.is_empty() && self.if_then.is_empty() } + + pub fn all_items(&self) -> impl Iterator { + self.if_then + .iter() + .flat_map(|if_then| if_then.expr.items().iter().chain(if_then.then.items())) + .chain(self.default.items()) + } } impl Expression { - pub fn parse(token_map: &TokenMap, expr: &str) -> Self { - ExpressionParser::new(Tokenizer::new(expr, token_map)) - .parse() - .unwrap() + pub fn parse(token_map: &TokenMap, expr: &str) -> Result { + ExpressionParser::default().parse(Tokenizer::new(expr, token_map)) + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + pub fn items(&self) -> &[ExpressionItem] { + &self.items } } @@ -151,7 +196,8 @@ impl BootstrapExprExt for Bootstrap { .with_variables(expr_ctx.allowed_variables) .with_constants(expr_ctx.allowed_constants); - let default = match ExpressionParser::new(Tokenizer::new(&expr.else_, &token_map)).parse() { + let mut parser = ExpressionParser::default(); + let default = match parser.parse(Tokenizer::new(&expr.else_, &token_map)) { Ok(expr) => expr, Err(err) => { self.invalid_property( @@ -164,29 +210,27 @@ impl BootstrapExprExt for Bootstrap { }; for (num, match_) in expr.match_.iter().enumerate() { - match ExpressionParser::new(Tokenizer::new(&match_.if_, &token_map)).parse() { - Ok(if_expr) => { - match ExpressionParser::new(Tokenizer::new(&match_.then, &token_map)).parse() { - Ok(then_expr) => { - if_then.push(IfThen { - expr: if_expr, - then: then_expr, - }); - } - Err(err) => { - self.invalid_property( - id, - expr_ctx.property, - format!( - "Error parsing 'then' expression in condition #{}: {}", - num + 1, - err - ), - ); - return None; - } + match parser.parse(Tokenizer::new(&match_.if_, &token_map)) { + Ok(if_expr) => match parser.parse(Tokenizer::new(&match_.then, &token_map)) { + Ok(then_expr) => { + if_then.push(IfThen { + expr: if_expr, + then: then_expr, + }); } - } + Err(err) => { + self.invalid_property( + id, + expr_ctx.property, + format!( + "Error parsing 'then' expression in condition #{}: {}", + num + 1, + err + ), + ); + return None; + } + }, Err(err) => { self.invalid_property( id, @@ -202,49 +246,33 @@ impl BootstrapExprExt for Bootstrap { } } - Some(IfBlock { + Some(IfBlock::new( id, - property: expr_ctx.property, - if_then: if_then.into_boxed_slice(), + expr_ctx.property, + if_then.into_boxed_slice(), default, - }) + )) } } +#[cfg(all(test, stalwart_tests))] impl IfBlock { - pub fn into_default(self, id: ObjectId, property: Property) -> IfBlock { - IfBlock { - id, - property, - if_then: Default::default(), - default: self.default, - } - } + pub(crate) fn compile_test(if_then: &[(&str, &str)], default: &str) -> Self { + use registry::schema::prelude::ObjectType; - pub fn all_items(&self) -> impl Iterator { - self.if_then - .iter() - .flat_map(|if_then| if_then.expr.items().iter().chain(if_then.then.items())) - .chain(self.default.items()) - } - - pub fn default_string(&self) -> Option<&str> { - for expr_item in &self.default.items { - if let ExpressionItem::Constant(Constant::String(value)) = expr_item { - return Some(value.as_str()); - } - } - - None - } - - pub fn into_default_string(self) -> Option { - for expr_item in self.default.items { - if let ExpressionItem::Constant(Constant::String(value)) = expr_item { - return Some(value); - } - } - - None + let token_map = TokenMap::default(); + let parse = |expr: &str| Expression::parse(&token_map, expr).expect("expression parses"); + IfBlock::new( + ObjectType::SpamRule.singleton(), + Property::Condition, + if_then + .iter() + .map(|(expr, then)| IfThen { + expr: parse(expr), + then: parse(then), + }) + .collect(), + parse(default), + ) } } diff --git a/crates/common/src/expr/kernels/case.rs b/crates/common/src/expr/kernels/case.rs new file mode 100644 index 00000000..0bfcd8ca --- /dev/null +++ b/crates/common/src/expr/kernels/case.rs @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use bumpalo::Bump; +use utils::text::CaseChange; + +pub fn to_lowercase<'a>(text: &'a str, arena: &'a Bump) -> &'a str { + convert( + text, + CaseChange::lowercase(text), + arena, + str::make_ascii_lowercase, + str::to_lowercase, + ) +} + +pub fn to_uppercase<'a>(text: &'a str, arena: &'a Bump) -> &'a str { + convert( + text, + CaseChange::uppercase(text), + arena, + str::make_ascii_uppercase, + str::to_uppercase, + ) +} + +fn convert<'a>( + text: &'a str, + change: CaseChange, + arena: &'a Bump, + ascii: fn(&mut str), + unicode: fn(&str) -> String, +) -> &'a str { + match change { + CaseChange::Unchanged => text, + CaseChange::AsciiFrom(position) => { + let out = arena.alloc_str(text); + if let Some(changed) = out.get_mut(position..) { + ascii(changed); + } + out + } + CaseChange::Unicode => arena.alloc_str(&unicode(text)), + } +} diff --git a/crates/common/src/expr/kernels/encode.rs b/crates/common/src/expr/kernels/encode.rs new file mode 100644 index 00000000..b53d21e3 --- /dev/null +++ b/crates/common/src/expr/kernels/encode.rs @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use bumpalo::{Bump, collections::String as BumpString}; +use utils::text::{hex_encode_into, validate_utf8}; + +pub fn hex_encode<'a>(bytes: &[u8], arena: &'a Bump) -> &'a str { + hex_encode_into(bytes, arena.alloc_slice_fill_copy(bytes.len() * 2, b'0')) +} + +pub fn utf8_lossy<'a>(bytes: &'a [u8], arena: &'a Bump) -> &'a str { + validate_utf8(bytes).unwrap_or_else(|| replace_invalid(bytes, arena)) +} + +fn replace_invalid<'a>(bytes: &[u8], arena: &'a Bump) -> &'a str { + let mut out = BumpString::with_capacity_in(bytes.len(), arena); + for chunk in bytes.utf8_chunks() { + out.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + } + } + out.into_bump_str() +} diff --git a/crates/common/src/expr/kernels/mod.rs b/crates/common/src/expr/kernels/mod.rs new file mode 100644 index 00000000..c1dab588 --- /dev/null +++ b/crates/common/src/expr/kernels/mod.rs @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod case; +mod encode; + + +pub use case::{to_lowercase, to_uppercase}; +pub use encode::{hex_encode, utf8_lossy}; +pub use utils::text::{ + ConstNeedle, ConstSet, IgnoreCaseNeedle, SplitWords, contains_ignore_case, count_chars, + count_lowercase, count_uppercase, count_whitespace, has_digits, is_lowercase, is_uppercase, + split_words, substring, +}; diff --git a/crates/common/src/expr/mod.rs b/crates/common/src/expr/mod.rs index d07c1490..1f0886c8 100644 --- a/crates/common/src/expr/mod.rs +++ b/crates/common/src/expr/mod.rs @@ -4,15 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use capture::CaptureRegex; use compact_str::CompactString; -use regex::Regex; use registry::schema::{ enums::{ExpressionConstant, ExpressionVariable}, structs::Rate, }; use std::{ - borrow::Cow, - fmt::{Display, Formatter}, net::{IpAddr, Ipv4Addr, Ipv6Addr}, str::FromStr, time::Duration, @@ -22,11 +20,18 @@ use utils::cache::CacheItemWeight; use crate::expr::if_block::IfBlock; +pub use bumpalo::{self, Bump}; + +pub mod capture; pub mod eval; pub mod functions; +pub mod guard; pub mod if_block; +pub mod kernels; pub mod parser; +pub(crate) mod program; pub mod tokenizer; +pub mod variable; #[derive(Debug, PartialEq, Eq, Clone, Default)] #[repr(transparent)] @@ -43,31 +48,28 @@ pub enum ExpressionItem { Constant(Constant), BinaryOperator(BinaryOperator), UnaryOperator(UnaryOperator), - Regex(Regex), + Regex(CaptureRegex), JmpIf { val: bool, pos: u32 }, Function { id: u32, num_args: u32 }, ArrayAccess, ArrayBuild(u32), } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum Variable<'x> { - String(StringCow<'x>), + String(&'x str), Integer(i64), Float(f64), - Array(Vec>), + Array(&'x [Variable<'x>]), Constant(ExpressionConstant), } -#[derive(Debug, Clone)] -pub enum StringCow<'x> { - Owned(CompactString), - Borrowed(&'x str), -} +#[cfg(target_pointer_width = "64")] +const _: () = assert!(std::mem::size_of::>() == 24); impl Default for Variable<'_> { fn default() -> Self { - Variable::String(StringCow::Borrowed("")) + Variable::String("") } } @@ -154,13 +156,13 @@ pub enum Token { Global(CompactString), Capture(u32), Function { - name: Cow<'static, str>, + name: &'static str, id: u32, num_args: u32, }, Constant(Constant), System(SystemVariable), - Regex(Regex), + Regex(CaptureRegex), BinaryOperator(BinaryOperator), UnaryOperator(UnaryOperator), OpenParen, @@ -170,16 +172,31 @@ pub enum Token { Comma, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum SystemVariable { Hostname, Domain, + ReportDomain, NodeId, NodeHostname, NodeRole, Metric(MetricType), } +impl SystemVariable { + pub fn parse(name: &str) -> Option { + hashify::map!(name.as_bytes(), SystemVariable, + "domain" => SystemVariable::Domain, + "report_domain" => SystemVariable::ReportDomain, + "hostname" => SystemVariable::Hostname, + "node_id" => SystemVariable::NodeId, + "node_hostname" => SystemVariable::NodeHostname, + "node_role" => SystemVariable::NodeRole, + ) + .copied() + } +} + impl From for Variable<'_> { fn from(value: usize) -> Self { Variable::Integer(value as i64) @@ -230,18 +247,12 @@ impl From for Variable<'_> { impl<'x> From<&'x str> for Variable<'x> { fn from(value: &'x str) -> Self { - Variable::String(StringCow::Borrowed(value)) + Variable::String(value) } } -impl From for Variable<'_> { - fn from(value: CompactString) -> Self { - Variable::String(StringCow::Owned(value)) - } -} - -impl<'x> From>> for Variable<'x> { - fn from(value: Vec>) -> Self { +impl<'x> From<&'x [Variable<'x>]> for Variable<'x> { + fn from(value: &'x [Variable<'x>]) -> Self { Variable::Array(value) } } @@ -345,7 +356,7 @@ impl<'x> TryFrom> for Duration { Variable::Integer(value) if value > 0 => Ok(Duration::from_millis(value as u64)), Variable::Float(value) if value > 0.0 => Ok(Duration::from_millis(value as u64)), Variable::String(value) if !value.is_empty() => { - registry::types::duration::Duration::from_str(value.as_str()) + registry::types::duration::Duration::from_str(value) .map(|v| v.into_inner()) .map_err(|_| ()) } @@ -354,79 +365,6 @@ impl<'x> TryFrom> for Duration { } } -impl StringCow<'_> { - pub fn as_str(&self) -> &str { - match self { - StringCow::Owned(s) => s.as_str(), - StringCow::Borrowed(s) => s, - } - } - - pub fn as_bytes(&self) -> &[u8] { - match self { - StringCow::Owned(s) => s.as_bytes(), - StringCow::Borrowed(s) => s.as_bytes(), - } - } - - pub fn is_empty(&self) -> bool { - match self { - StringCow::Owned(s) => s.is_empty(), - StringCow::Borrowed(s) => s.is_empty(), - } - } - - pub fn len(&self) -> usize { - match self { - StringCow::Owned(s) => s.len(), - StringCow::Borrowed(s) => s.len(), - } - } - - pub fn into_owned(self) -> CompactString { - match self { - StringCow::Owned(s) => s, - StringCow::Borrowed(s) => s.into(), - } - } -} - -impl<'x> From> for StringCow<'x> { - fn from(value: Cow<'x, str>) -> Self { - match value { - Cow::Borrowed(s) => StringCow::Borrowed(s), - Cow::Owned(s) => StringCow::Owned(s.into()), - } - } -} - -impl From for StringCow<'_> { - fn from(value: CompactString) -> Self { - StringCow::Owned(value) - } -} - -impl AsRef for StringCow<'_> { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl AsRef<[u8]> for StringCow<'_> { - fn as_ref(&self) -> &[u8] { - self.as_str().as_bytes() - } -} - -impl Display for StringCow<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - StringCow::Owned(s) => write!(f, "{}", s), - StringCow::Borrowed(s) => write!(f, "{}", s), - } - } -} - impl From for Constant { fn from(value: Duration) -> Self { Constant::Integer(value.as_millis() as i64) @@ -438,9 +376,9 @@ impl<'x> TryFrom> for Rate { fn try_from(value: Variable<'x>) -> Result { match value { - Variable::Array(items) if items.len() == 2 => { - let requests = items[0].to_integer().ok_or(())?; - let period = items[1].to_integer().ok_or(())?; + Variable::Array([requests, period]) => { + let requests = requests.to_integer().ok_or(())?; + let period = period.to_integer().ok_or(())?; if requests > 0 && period > 0 { Ok(Rate { @@ -461,7 +399,7 @@ impl<'x> TryFrom> for Ipv4Addr { fn try_from(value: Variable<'x>) -> Result { match value { - Variable::String(value) => value.as_str().parse().map_err(|_| ()), + Variable::String(value) => value.parse().map_err(|_| ()), _ => Err(()), } } @@ -472,7 +410,7 @@ impl<'x> TryFrom> for Ipv6Addr { fn try_from(value: Variable<'x>) -> Result { match value { - Variable::String(value) => value.as_str().parse().map_err(|_| ()), + Variable::String(value) => value.parse().map_err(|_| ()), _ => Err(()), } } @@ -483,24 +421,24 @@ impl<'x> TryFrom> for IpAddr { fn try_from(value: Variable<'x>) -> Result { match value { - Variable::String(value) => value.as_str().parse().map_err(|_| ()), + Variable::String(value) => value.parse().map_err(|_| ()), _ => Err(()), } } } -impl<'x, T: TryFrom>> TryFrom> for Vec -where - Result, ()>: FromIterator>>::Error>>, -{ +impl<'x, T: TryFrom>> TryFrom> for Vec { type Error = (); fn try_from(value: Variable<'x>) -> Result { - value - .into_array() - .into_iter() - .map(|v| T::try_from(v)) - .collect() + match value { + Variable::Array(items) => items + .iter() + .map(|item| T::try_from(*item).map_err(|_| ())) + .collect(), + value if !value.is_empty() => T::try_from(value).map(|item| vec![item]).map_err(|_| ()), + _ => Ok(Vec::new()), + } } } @@ -519,5 +457,6 @@ impl CacheItemWeight for IfBlock { .map(|if_then| if_then.expr.weight() + if_then.then.weight()) .sum::() + self.default.weight() + + self.program.weight() } } diff --git a/crates/common/src/expr/parser.rs b/crates/common/src/expr/parser.rs index e74eb77c..2a054888 100644 --- a/crates/common/src/expr/parser.rs +++ b/crates/common/src/expr/parser.rs @@ -6,9 +6,9 @@ use super::{BinaryOperator, Expression, ExpressionItem, Token, tokenizer::Tokenizer}; -pub struct ExpressionParser<'x> { - pub(crate) tokenizer: Tokenizer<'x>, - pub(crate) output: Vec, +#[derive(Default)] +pub struct ExpressionParser { + output: Vec, operator_stack: Vec<(Token, Option)>, arg_count: Vec, } @@ -16,20 +16,14 @@ pub struct ExpressionParser<'x> { pub(crate) const ID_ARRAY_ACCESS: u32 = u32::MAX; pub(crate) const ID_ARRAY_BUILD: u32 = u32::MAX - 1; -impl<'x> ExpressionParser<'x> { - pub fn new(tokenizer: Tokenizer<'x>) -> Self { - Self { - tokenizer, - output: Vec::new(), - operator_stack: Vec::new(), - arg_count: Vec::new(), - } - } - - pub fn parse(mut self) -> Result { +impl ExpressionParser { + pub fn parse(&mut self, mut tokenizer: Tokenizer<'_>) -> Result { + self.output.clear(); + self.operator_stack.clear(); + self.arg_count.clear(); let mut last_is_var_or_fnc = false; - while let Some(token) = self.tokenizer.next()? { + while let Some(token) = tokenizer.next()? { let mut is_var_or_fnc = false; match token { Token::Variable(v) => { @@ -77,7 +71,7 @@ impl<'x> ExpressionParser<'x> { match self.operator_stack.last() { Some((Token::Function { id, num_args, name }, _)) => { - let got_args = self.arg_count.pop().unwrap(); + let got_args = self.arg_count.pop().unwrap_or_default(); if got_args != *num_args as i32 { return Err(if *id != u32::MAX { format!( @@ -101,19 +95,20 @@ impl<'x> ExpressionParser<'x> { self.operator_stack.pop(); self.output.push(expr); } - Some((Token::Regex(regex), _)) => { - if self.arg_count.pop().unwrap() != 1 { + Some((Token::Regex(_), _)) => { + if self.arg_count.pop().unwrap_or_default() != 1 { return Err("Expression function \"matches\" expected 2 arguments" .to_string()); } - self.output.push(ExpressionItem::Regex(regex.clone())); - self.operator_stack.pop(); + if let Some((Token::Regex(regex), _)) = self.operator_stack.pop() { + self.output.push(ExpressionItem::Regex(regex)); + } } Some((Token::System(setting), _)) => { - if self.arg_count.pop().unwrap() != 0 { + if self.arg_count.pop().unwrap_or_default() != 0 { return Err("Expression function expected 1 argument".to_string()); } - self.output.push(ExpressionItem::System(setting.clone())); + self.output.push(ExpressionItem::System(*setting)); self.operator_stack.pop(); } _ => {} @@ -180,7 +175,7 @@ impl<'x> ExpressionParser<'x> { self.operator_stack.push(( Token::Function { id, - name: "array".into(), + name: "array", num_args, }, None, @@ -223,7 +218,7 @@ impl<'x> ExpressionParser<'x> { if self.operator_stack.is_empty() { Ok(Expression { - items: self.output.into_boxed_slice(), + items: self.output.drain(..).collect(), }) } else { Err("Invalid expression".to_string()) @@ -255,7 +250,7 @@ impl<'x> ExpressionParser<'x> { if let ExpressionItem::JmpIf { pos, .. } = &mut self.output[jmp_pos] { *pos = (cur_pos - jmp_pos) as u32; } else { - #[cfg(test)] + #[cfg(all(test, stalwart_tests))] panic!("Invalid jump position"); } } diff --git a/crates/common/src/expr/program.rs b/crates/common/src/expr/program.rs new file mode 100644 index 00000000..23cb7973 --- /dev/null +++ b/crates/common/src/expr/program.rs @@ -0,0 +1,940 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + BinaryOperator, Constant, ExpressionItem, SystemVariable, UnaryOperator, Variable, + capture::CaptureRegex, + functions::{FUNCTIONS, SyncFn, SyncFunction, text}, + kernels::{self, ConstNeedle, ConstSet, IgnoreCaseNeedle}, +}; +use ahash::AHashMap; +use bumpalo::Bump; +use registry::schema::enums::{ExpressionConstant, ExpressionVariable}; +use std::{fmt, mem::size_of}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Src { + Pop, + Var(ExpressionVariable), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextOp { + Contains, + StartsWith, + EndsWith, +} + +#[derive(Debug, Clone, Copy)] +pub enum Op { + Var(ExpressionVariable), + Global(u32), + System(SystemVariable), + Capture(u32), + Int(i64), + Float(f64), + Static(ExpressionConstant), + Str(u32), + Array(u32), + Unary(UnaryOperator), + Binary(BinaryOperator), + JmpIf { + val: bool, + target: u32, + }, + Call { + id: u16, + argc: u16, + }, + CallAsync { + id: u16, + argc: u16, + }, + Index, + IndexConst { + src: Src, + index: u32, + }, + Build(u32), + Regex(u32), + CmpConst { + src: Src, + op: BinaryOperator, + konst: u32, + }, + Text { + src: Src, + op: TextOp, + needle: u32, + }, + ContainsIgnoreCase { + src: Src, + needle: u32, + }, + InArray { + src: Src, + array: u32, + }, + InSet { + src: Src, + set: u32, + }, + Test { + target: u32, + }, + Return, +} + +const _: () = assert!(size_of::() == 16); + +const JUMP_TARGET: u32 = u32::MAX; + +const LINEAR_STRINGS: usize = 16; + +#[derive(Debug, Clone)] +pub enum ConstValue { + Integer(i64), + Float(f64), + Static(ExpressionConstant), + String(Box), + Array { + items: Box<[ConstValue]>, + view: Option]>>, + }, +} + +pub struct Needle { + needle: ConstNeedle, +} + +pub struct CaseNeedle { + text: Box, + needle: IgnoreCaseNeedle, +} + +pub struct StringSet { + items: Box<[Box]>, + hashed: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct Program { + ops: Box<[Op]>, + strings: Box<[Box]>, + needles: Box<[Needle]>, + case_needles: Box<[CaseNeedle]>, + sets: Box<[StringSet]>, + consts: Box<[ConstValue]>, + regexes: Box<[CaptureRegex]>, + max_depth: usize, + uses_captures: bool, +} + +#[derive(Default)] +pub struct ProgramBuilder { + ops: Vec, + pools: Pools, + scratch: Scratch, +} + +#[derive(Default)] +struct Pools { + strings: Vec>, + string_ids: Option>, + needles: Vec, + case_needles: Vec, + sets: Vec, + consts: Vec, + regexes: Vec, +} + +#[derive(Default)] +struct Scratch { + origin: Vec, + jumps: Vec, +} + +struct Lowering<'b> { + pools: &'b mut Pools, + ops: &'b mut Vec, + base: usize, + origin: &'b mut Vec, + jumps: &'b [u32], + items: &'b [ExpressionItem], +} + +impl ConstValue { + pub fn to_variable<'x>(&'x self, arena: &'x Bump) -> Variable<'x> { + match self { + ConstValue::Integer(n) => Variable::Integer(*n), + ConstValue::Float(n) => Variable::Float(*n), + ConstValue::Static(c) => Variable::Constant(*c), + ConstValue::String(s) => Variable::String(s), + ConstValue::Array { + view: Some(view), .. + } => Variable::Array(view), + ConstValue::Array { items, view: None } => Variable::Array( + arena.alloc_slice_fill_iter(items.iter().map(|item| item.to_variable(arena))), + ), + } + } + + pub fn scalar(&self) -> Option> { + match self { + ConstValue::Integer(n) => Some(Variable::Integer(*n)), + ConstValue::Float(n) => Some(Variable::Float(*n)), + ConstValue::Static(c) => Some(Variable::Constant(*c)), + ConstValue::String(s) => Some(Variable::String(s)), + ConstValue::Array { .. } => None, + } + } + + fn array(items: Box<[ConstValue]>) -> Self { + let view = items + .iter() + .map(|item| match item { + ConstValue::Integer(n) => Some(Variable::Integer(*n)), + ConstValue::Float(n) => Some(Variable::Float(*n)), + ConstValue::Static(c) => Some(Variable::Constant(*c)), + ConstValue::String(_) | ConstValue::Array { .. } => None, + }) + .collect::>>(); + ConstValue::Array { items, view } + } + + fn is_flat_array(&self) -> bool { + matches!(self, ConstValue::Array { items, .. } if items.iter().all(|item| item.scalar().is_some())) + } + + fn string_items(&self) -> Option]>> { + match self { + ConstValue::Array { items, .. } => items + .iter() + .map(|item| match item { + ConstValue::String(s) => Some(s.clone()), + _ => None, + }) + .collect(), + _ => None, + } + } +} + +impl Needle { + pub fn new(text: &str) -> Self { + Needle { + needle: ConstNeedle::new(text), + } + } + + pub fn as_str(&self) -> &str { + self.needle.as_str() + } + + pub fn is_in(&self, haystack: &str) -> bool { + self.needle.contains(haystack) + } + + pub fn is_prefix_of(&self, haystack: &str) -> bool { + self.needle.starts_with(haystack) + } + + pub fn is_suffix_of(&self, haystack: &str) -> bool { + self.needle.ends_with(haystack) + } + + pub fn matches<'a>(&self, op: TextOp, haystack: Variable<'a>, arena: &'a Bump) -> bool { + match op { + TextOp::Contains => match haystack { + Variable::String(s) => self.is_in(s), + Variable::Array(items) => { + let needle = Variable::String(self.as_str()); + items.contains(&needle) + } + value => self.is_in(value.to_str(arena)), + }, + TextOp::StartsWith => self.is_prefix_of(haystack.to_str(arena)), + TextOp::EndsWith => self.is_suffix_of(haystack.to_str(arena)), + } + } +} + +impl Clone for Needle { + fn clone(&self) -> Self { + Needle::new(self.as_str()) + } +} + +impl fmt::Debug for Needle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Needle").field(&self.as_str()).finish() + } +} + +impl CaseNeedle { + pub fn new(text: &str) -> Self { + CaseNeedle { + text: text.into(), + needle: IgnoreCaseNeedle::new(text), + } + } + + pub fn as_str(&self) -> &str { + &self.text + } + + pub fn matches<'a>(&self, haystack: Variable<'a>, arena: &'a Bump) -> bool { + text::contains_ignore_case(haystack, &self.text, arena, |s| { + self.needle + .contains_with(s, |s| kernels::to_lowercase(s, arena)) + }) + } +} + +impl Clone for CaseNeedle { + fn clone(&self) -> Self { + CaseNeedle::new(self.as_str()) + } +} + +impl fmt::Debug for CaseNeedle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("CaseNeedle").field(&self.as_str()).finish() + } +} + +impl StringSet { + pub fn new(items: Box<[Box]>) -> Self { + StringSet { + hashed: (items.len() > ConstSet::LINEAR_MAX_ITEMS).then(|| ConstSet::new(&items)), + items, + } + } + + pub fn items(&self) -> &[Box] { + &self.items + } + + pub fn contains(&self, value: Variable<'_>) -> bool { + match (value, &self.hashed) { + (Variable::String(s), None) => self.items.iter().any(|item| **item == *s), + (Variable::String(s), Some(set)) => set.contains(s), + (value, _) => self + .items + .iter() + .any(|item| Variable::String(item) == value), + } + } +} + +impl Clone for StringSet { + fn clone(&self) -> Self { + StringSet::new(self.items.clone()) + } +} + +impl fmt::Debug for StringSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("StringSet").field(&self.items).finish() + } +} + +impl Program { + pub fn ops(&self) -> &[Op] { + &self.ops + } + + pub fn is_empty(&self) -> bool { + self.ops.is_empty() + } + + pub fn max_depth(&self) -> usize { + self.max_depth + } + + pub fn uses_captures(&self) -> bool { + self.uses_captures + } + + pub fn string(&self, index: u32) -> &str { + self.strings.get(index as usize).map_or("", |s| s) + } + + pub fn needle(&self, index: u32) -> Option<&Needle> { + self.needles.get(index as usize) + } + + pub fn case_needle(&self, index: u32) -> Option<&CaseNeedle> { + self.case_needles.get(index as usize) + } + + pub fn set(&self, index: u32) -> Option<&StringSet> { + self.sets.get(index as usize) + } + + pub fn constant(&self, index: u32) -> Option<&ConstValue> { + self.consts.get(index as usize) + } + + pub fn regex(&self, index: u32) -> Option<&CaptureRegex> { + self.regexes.get(index as usize) + } + + pub fn function(id: u16) -> Option { + FUNCTIONS.get(id as usize).copied() + } + + pub fn weight(&self) -> u64 { + let text = |text: &str| text.len() + size_of::>(); + let bytes = self.ops.len() * size_of::() + + self.strings.iter().map(|s| text(s)).sum::() + + self + .needles + .iter() + .map(|needle| size_of::() + text(needle.as_str())) + .sum::() + + self + .case_needles + .iter() + .map(|needle| size_of::() + text(needle.as_str())) + .sum::() + + self + .sets + .iter() + .map(|set| { + size_of::() + + set.items().iter().map(|item| text(item)).sum::() + }) + .sum::() + + self.consts.len() * size_of::() + + self.regexes.len() * size_of::(); + bytes as u64 + } +} + +impl ProgramBuilder { + pub fn with_capacity(ops: usize, items: usize) -> Self { + ProgramBuilder { + ops: Vec::with_capacity(ops), + pools: Pools::default(), + scratch: Scratch { + origin: Vec::with_capacity(items), + jumps: Vec::with_capacity(ops.max(items) + 1), + }, + } + } + + pub fn push_expression(&mut self, items: &[ExpressionItem]) { + let base = self.ops.len(); + let Scratch { origin, jumps } = &mut self.scratch; + origin.clear(); + mark_jump_targets(items, jumps); + + let mut lowering = Lowering { + pools: &mut self.pools, + ops: &mut self.ops, + base, + origin, + jumps, + items, + }; + for (index, item) in items.iter().enumerate() { + let op = lowering.lower(item, index); + lowering.push(op, index); + } + lowering.intern_strings(); + + let end = self.ops.len() as u32; + jumps.clear(); + jumps.resize(items.len() + 1, end); + for (new, old) in origin.iter().enumerate() { + if let Some(slot) = jumps.get_mut(*old) { + *slot = (base + new) as u32; + } + } + if let Some(slot) = jumps.last_mut() { + *slot = end; + } + + for op in self.ops.iter_mut().skip(base) { + if let Op::JmpIf { target, .. } = op { + *target = jumps.get(*target as usize).copied().unwrap_or(end); + } + } + } + + pub fn push_test(&mut self) -> usize { + self.ops.push(Op::Test { target: u32::MAX }); + self.ops.len() - 1 + } + + pub fn patch_test(&mut self, test: usize) { + let target = self.ops.len() as u32; + if let Some(Op::Test { target: slot }) = self.ops.get_mut(test) { + *slot = target; + } + } + + pub fn push_return(&mut self) { + self.ops.push(Op::Return); + } + + pub fn build(mut self) -> Program { + let pools = self.pools; + Program { + max_depth: max_depth(&self.ops, &mut self.scratch.jumps), + uses_captures: self.ops.iter().any(|op| matches!(op, Op::Capture(_))), + ops: self.ops.into_boxed_slice(), + strings: pools.strings.into_boxed_slice(), + needles: pools.needles.into_boxed_slice(), + case_needles: pools.case_needles.into_boxed_slice(), + sets: pools.sets.into_boxed_slice(), + consts: pools.consts.into_boxed_slice(), + regexes: pools.regexes.into_boxed_slice(), + } + } +} + +impl Pools { + fn add_string(&mut self, value: &str) -> u32 { + if self.strings.len() < LINEAR_STRINGS { + self.find_or_push_string(value) + } else { + self.find_or_push_hashed(value) + } + } + + #[cold] + fn find_or_push_hashed(&mut self, value: &str) -> u32 { + let strings = &self.strings; + let string_ids = self.string_ids.get_or_insert_with(|| { + let mut string_ids = AHashMap::with_capacity(strings.len() * 2); + for (index, text) in strings.iter().enumerate() { + let hash = string_ids.hasher().hash_one(&**text); + string_ids.entry(hash).or_insert(index as u32); + } + string_ids + }); + let hash = string_ids.hasher().hash_one(value); + match string_ids.get(&hash).copied() { + Some(index) if strings.get(index as usize).is_some_and(|s| **s == *value) => index, + Some(_) => self.find_or_push_string(value), + None => { + string_ids.insert(hash, strings.len() as u32); + self.push_string(value) + } + } + } + + fn find_or_push_string(&mut self, value: &str) -> u32 { + match self.strings.iter().position(|s| **s == *value) { + Some(position) => position as u32, + None => self.push_string(value), + } + } + + fn push_string(&mut self, value: &str) -> u32 { + self.strings.push(value.into()); + (self.strings.len() - 1) as u32 + } + + fn add_needle(&mut self, text: &str) -> u32 { + self.needles.push(Needle::new(text)); + (self.needles.len() - 1) as u32 + } + + fn add_case_needle(&mut self, text: &str) -> u32 { + self.case_needles.push(CaseNeedle::new(text)); + (self.case_needles.len() - 1) as u32 + } + + fn add_set(&mut self, items: Box<[Box]>) -> u32 { + self.sets.push(StringSet::new(items)); + (self.sets.len() - 1) as u32 + } + + fn add_const(&mut self, value: ConstValue) -> u32 { + self.consts.push(value); + (self.consts.len() - 1) as u32 + } + + fn take_string_items(&mut self, array: u32) -> Option]>> { + if array as usize + 1 == self.consts.len() + && let Some(ConstValue::Array { items, .. }) = self.consts.last() + && items + .iter() + .all(|item| matches!(item, ConstValue::String(_))) + && let Some(ConstValue::Array { items, .. }) = self.consts.pop() + { + Some( + items + .into_iter() + .filter_map(|item| match item { + ConstValue::String(s) => Some(s), + _ => None, + }) + .collect(), + ) + } else { + self.consts.get(array as usize)?.string_items() + } + } +} + +impl<'b> Lowering<'b> { + fn lower(&mut self, item: &ExpressionItem, index: usize) -> Op { + match item { + ExpressionItem::Variable(v) => Op::Var(*v), + ExpressionItem::Global(name) => Op::Global(self.pools.add_string(name)), + ExpressionItem::System(system) => Op::System(*system), + ExpressionItem::Capture(n) => Op::Capture(*n), + ExpressionItem::Constant(Constant::Integer(n)) => Op::Int(*n), + ExpressionItem::Constant(Constant::Float(n)) => Op::Float(*n), + ExpressionItem::Constant(Constant::Static(c)) => Op::Static(*c), + ExpressionItem::Constant(Constant::String(_)) => Op::Str(u32::MAX), + ExpressionItem::BinaryOperator(op) => Op::Binary(*op), + ExpressionItem::UnaryOperator(op) => Op::Unary(*op), + ExpressionItem::Regex(regex) => { + self.pools.regexes.push(regex.clone()); + Op::Regex((self.pools.regexes.len() - 1) as u32) + } + ExpressionItem::JmpIf { val, pos } => Op::JmpIf { + val: *val, + target: jump_target(index, *pos), + }, + ExpressionItem::Function { id, num_args } => { + let sync_len = FUNCTIONS.len() as u32; + let argc = (*num_args).min(u16::MAX as u32) as u16; + if *id < sync_len { + Op::Call { + id: *id as u16, + argc, + } + } else { + Op::CallAsync { + id: (*id - sync_len).min(u16::MAX as u32) as u16, + argc, + } + } + } + ExpressionItem::ArrayAccess => Op::Index, + ExpressionItem::ArrayBuild(n) => Op::Build(*n), + } + } + + fn push(&mut self, op: Op, index: usize) { + match self.fuse(op, index) { + Some((op, origin)) => { + self.ops.push(op); + self.origin.push(origin); + } + None => { + self.ops.push(op); + self.origin.push(index); + } + } + } + + fn intern_strings(&mut self) { + let Lowering { + pools, + ops, + base, + origin, + items, + .. + } = self; + for (op, origin) in ops.iter_mut().skip(*base).zip(origin.iter()) { + if let Op::Str(index) = op + && let Some(ExpressionItem::Constant(Constant::String(text))) = items.get(*origin) + { + *index = pools.add_string(text); + } + } + } + + fn text(&self, origin: usize) -> Option<&'b str> { + match self.items.get(origin)? { + ExpressionItem::Constant(Constant::String(text)) => Some(text.as_str()), + _ => None, + } + } + + fn const_of(&self, op: &Op, origin: usize) -> Option { + match op { + Op::Int(n) => Some(ConstValue::Integer(*n)), + Op::Float(n) => Some(ConstValue::Float(*n)), + Op::Static(c) => Some(ConstValue::Static(*c)), + Op::Str(_) => self + .text(origin) + .map(|text| ConstValue::String(text.into())), + Op::Array(index) => self.pools.consts.get(*index as usize).cloned(), + _ => None, + } + } + + fn fuse(&mut self, op: Op, index: usize) -> Option<(Op, usize)> { + match op { + Op::Build(n) => self.fold_array(n as usize, index), + Op::Binary( + cmp @ (BinaryOperator::Eq + | BinaryOperator::Ne + | BinaryOperator::Lt + | BinaryOperator::Le + | BinaryOperator::Gt + | BinaryOperator::Ge), + ) => self.fuse_compare(cmp, index), + Op::Call { id, argc: 2 } => self.fuse_call(id, index), + Op::Index => self.fuse_index(index), + _ => None, + } + } + + fn is_clear(&self, from: usize, to: usize) -> bool { + self.jumps + .get(from + 1..=to) + .is_some_and(|window| !window.contains(&JUMP_TARGET)) + } + + fn tail(&self, back: usize) -> Option<(Op, usize)> { + let position = self.origin.len().checked_sub(back)?; + Some(( + *self.ops.get(self.base + position)?, + *self.origin.get(position)?, + )) + } + + fn pop(&mut self, count: usize) { + let len = self.origin.len().saturating_sub(count); + self.ops.truncate(self.base + len); + self.origin.truncate(len); + } + + fn fuse_source(&mut self, consumed: usize, index: usize, start: usize) -> (Src, usize) { + match self.tail(consumed + 1) { + Some((Op::Var(v), origin)) if self.is_clear(origin, index) => { + self.pop(consumed + 1); + (Src::Var(v), origin) + } + _ => { + self.pop(consumed); + (Src::Pop, start) + } + } + } + + fn fold_array(&mut self, count: usize, index: usize) -> Option<(Op, usize)> { + let start = if count == 0 { + index + } else { + let (_, origin) = self.tail(count)?; + origin + }; + if !self.is_clear(start, index) { + return None; + } + let first = self.origin.len().checked_sub(count)?; + let items = self + .ops + .get(self.base + first..)? + .iter() + .zip(self.origin.get(first..)?) + .map(|(op, origin)| self.const_of(op, *origin)) + .collect::>>()?; + self.pop(count); + let konst = self.pools.add_const(ConstValue::array(items)); + Some((Op::Array(konst), start)) + } + + fn fuse_compare(&mut self, op: BinaryOperator, index: usize) -> Option<(Op, usize)> { + let (konst_op, start) = self.tail(1)?; + let konst = match konst_op { + Op::Int(_) | Op::Float(_) | Op::Static(_) | Op::Str(_) => { + self.const_of(&konst_op, start)? + } + _ => return None, + }; + if !self.is_clear(start, index) { + return None; + } + let konst = self.pools.add_const(konst); + let (src, origin) = self.fuse_source(1, index, start); + Some((Op::CmpConst { src, op, konst }, origin)) + } + + fn fuse_call(&mut self, id: u16, index: usize) -> Option<(Op, usize)> { + match SyncFunction::from_id(id)? { + SyncFunction::Contains => self + .fuse_text(TextOp::Contains, index) + .or_else(|| self.fuse_membership(index)), + SyncFunction::StartsWith => self.fuse_text(TextOp::StartsWith, index), + SyncFunction::EndsWith => self.fuse_text(TextOp::EndsWith, index), + SyncFunction::ContainsIgnoreCase => { + let (text, src, origin) = self.needle_operand(index)?; + let needle = self.pools.add_case_needle(text); + Some((Op::ContainsIgnoreCase { src, needle }, origin)) + } + _ => None, + } + } + + fn needle_operand(&mut self, index: usize) -> Option<(&'b str, Src, usize)> { + let (Op::Str(_), start) = self.tail(1)? else { + return None; + }; + let text = self.text(start)?; + if !self.is_clear(start, index) { + return None; + } + let (src, origin) = self.fuse_source(1, index, start); + Some((text, src, origin)) + } + + fn fuse_text(&mut self, op: TextOp, index: usize) -> Option<(Op, usize)> { + let (text, src, origin) = self.needle_operand(index)?; + let needle = self.pools.add_needle(text); + Some((Op::Text { src, op, needle }, origin)) + } + + fn fuse_membership(&mut self, index: usize) -> Option<(Op, usize)> { + let (Some((Op::Array(array), start)), Some((Op::Var(variable), _))) = + (self.tail(2), self.tail(1)) + else { + return None; + }; + if !self.is_clear(start, index) { + return None; + } + let src = Src::Var(variable); + let op = match self.pools.take_string_items(array) { + Some(items) => Op::InSet { + src, + set: self.pools.add_set(items), + }, + None if self + .pools + .consts + .get(array as usize) + .is_some_and(ConstValue::is_flat_array) => + { + Op::InArray { src, array } + } + None => return None, + }; + self.pop(2); + Some((op, start)) + } + + fn fuse_index(&mut self, index: usize) -> Option<(Op, usize)> { + let (Op::Int(position), start) = self.tail(1)? else { + return None; + }; + let position = u32::try_from(position).ok()?; + if !self.is_clear(start, index) { + return None; + } + let (src, origin) = self.fuse_source(1, index, start); + Some(( + Op::IndexConst { + src, + index: position, + }, + origin, + )) + } +} + +fn jump_target(index: usize, pos: u32) -> u32 { + (index + pos as usize + 1) as u32 +} + +fn mark_jump_targets(items: &[ExpressionItem], jumps: &mut Vec) { + jumps.clear(); + jumps.resize(items.len() + 1, 0); + for (index, item) in items.iter().enumerate() { + if let ExpressionItem::JmpIf { pos, .. } = item { + let target = (jump_target(index, *pos) as usize).min(items.len()); + if let Some(slot) = jumps.get_mut(target) { + *slot = JUMP_TARGET; + } + } + } +} + +fn stack_effect(op: &Op) -> (usize, usize) { + match op { + Op::Var(_) + | Op::Global(_) + | Op::System(_) + | Op::Capture(_) + | Op::Int(_) + | Op::Float(_) + | Op::Static(_) + | Op::Str(_) + | Op::Array(_) => (0, 1), + Op::Unary(_) | Op::Regex(_) => (1, 1), + Op::Binary(_) | Op::Index => (2, 1), + Op::JmpIf { .. } => (0, 0), + Op::Call { argc, .. } | Op::CallAsync { argc, .. } => (*argc as usize, 1), + Op::Build(n) => (*n as usize, 1), + Op::IndexConst { src, .. } + | Op::CmpConst { src, .. } + | Op::Text { src, .. } + | Op::ContainsIgnoreCase { src, .. } + | Op::InArray { src, .. } + | Op::InSet { src, .. } => match src { + Src::Pop => (1, 1), + Src::Var(_) => (0, 1), + }, + Op::Test { .. } | Op::Return => (1, 0), + } +} + +fn max_depth(ops: &[Op], incoming: &mut Vec) -> usize { + const UNREACHED: u32 = u32::MAX; + incoming.clear(); + incoming.resize(ops.len() + 1, UNREACHED); + if let Some(first) = incoming.first_mut() { + *first = 0; + } + let mut max = 0; + for (index, op) in ops.iter().enumerate() { + let Some(depth) = incoming + .get(index) + .filter(|depth| **depth != UNREACHED) + .map(|depth| *depth as usize) + else { + continue; + }; + let (pops, pushes) = stack_effect(op); + let after = depth.saturating_sub(pops) + pushes; + max = max.max(depth).max(after); + let mut merge = |target: usize, depth: usize| { + if let Some(slot) = incoming.get_mut(target) { + let depth = depth as u32; + *slot = if *slot == UNREACHED { + depth + } else { + (*slot).max(depth) + }; + } + }; + match op { + Op::JmpIf { target, .. } => { + merge(*target as usize, depth); + merge(index + 1, after); + } + Op::Test { target } => { + merge(*target as usize, 0); + merge(index + 1, 0); + } + Op::Return => {} + _ => merge(index + 1, after), + } + } + max +} + diff --git a/crates/common/src/expr/tokenizer.rs b/crates/common/src/expr/tokenizer.rs index 01a8c863..8fa844d6 100644 --- a/crates/common/src/expr/tokenizer.rs +++ b/crates/common/src/expr/tokenizer.rs @@ -4,22 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - functions::{ASYNC_FUNCTIONS, FUNCTIONS}, - *, -}; -use ahash::AHashSet; -use regex::Regex; +use super::{functions::lookup_function, *}; use registry::{schema::enums::ExpressionConstant, types::EnumImpl}; -use std::{borrow::Cow, iter::Peekable, slice::Iter}; use trc::MetricType; pub struct Tokenizer<'x> { - pub(crate) iter: Peekable>, + input: &'x str, + pos: usize, + word_start: usize, + word_end: usize, token_map: &'x TokenMap, - buf: Vec, depth: u32, - next_token: Vec, + next_token: Option, has_number: bool, has_dot: bool, has_alpha: bool, @@ -29,18 +25,23 @@ pub struct Tokenizer<'x> { #[derive(Debug, Default, Clone)] pub struct TokenMap { - pub variables: AHashSet, - pub constants: AHashSet, + variables: IdSet<{ ExpressionVariable::COUNT.div_ceil(64) }>, + constants: IdSet<{ ExpressionConstant::COUNT.div_ceil(64) }>, } +#[derive(Debug, Clone, Copy)] +struct IdSet([u64; WORDS]); + impl<'x> Tokenizer<'x> { #[allow(clippy::should_implement_trait)] pub fn new(expr: &'x str, token_map: &'x TokenMap) -> Self { Self { - iter: expr.as_bytes().iter().peekable(), - buf: Vec::new(), + input: expr, + pos: 0, + word_start: 0, + word_end: 0, depth: 0, - next_token: Vec::with_capacity(2), + next_token: None, has_number: false, has_dot: false, has_alpha: false, @@ -52,113 +53,93 @@ impl<'x> Tokenizer<'x> { #[allow(clippy::should_implement_trait)] pub fn next(&mut self) -> Result, String> { - if let Some(token) = self.next_token.pop() { + if let Some(token) = self.next_token.take() { return Ok(Some(token)); } else if self.is_eof { return Ok(None); } - while let Some(&ch) = self.iter.next() { + while let Some(ch) = self.next_byte() { match ch { - b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' => { - self.buf.push(ch); - self.has_alpha = true; - } - b'0'..=b'9' => { - self.buf.push(ch); - self.has_number = true; - } - b'.' => { - self.buf.push(ch); - self.has_dot = true; + b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' | b'0'..=b'9' | b'.' => { + self.push_word(); + self.scan_word(ch); } b'}' => { self.is_eof = true; break; } - b'-' if self.buf.last().is_some_and(|c| *c == b'[') => { - self.buf.push(ch); + b'-' if self.word().last().is_some_and(|c| *c == b'[') => { + self.push_word(); } - b':' if self.buf.contains(&b'.') => { - self.buf.push(ch); + b':' if self.has_dot => { + self.push_word(); } - b']' if self.buf.contains(&b'[') => { - self.buf.push(b']'); + b']' if self.word().contains(&b'[') => { + self.push_word(); } - b'*' if self.buf.last().is_some_and(|&c| c == b'[' || c == b'.') => { - self.buf.push(ch); + b'*' if self.word().last().is_some_and(|&c| c == b'[' || c == b'.') => { + self.push_word(); } _ => { - let (prev_token, ch) = if ch == b'(' && !self.buf.is_empty() { - match self.buf.as_slice() { - b"matches" => { - // Parse regular expressions + let (prev_token, ch) = if ch == b'(' && !self.word().is_empty() { + hashify::fnc_map!(self.word(), + "matches" => { let stop_ch = self.find_char(b"\"'")?; let regex_str = self.parse_string(stop_ch)?; - let regex = Regex::new(®ex_str).map_err(|e| { - format!("Invalid regular expression {:?}: {}", regex_str, e) - })?; + let regex = CaptureRegex::new(®ex_str)?; self.has_alpha = false; - self.buf.clear(); + self.clear_word(); self.find_char(b",")?; (Token::Regex(regex).into(), b'(') - } - b"metric" => { + }, + "metric" => { let stop_ch = self.find_char(b"\"'")?; let metric_str = self.parse_string(stop_ch)?; let metric = MetricType::parse(&metric_str).ok_or_else(|| { format!("Invalid metric name {:?}", metric_str) })?; self.has_alpha = false; - self.buf.clear(); + self.clear_word(); (Token::System(SystemVariable::Metric(metric)).into(), b'(') - } - b"system" => { + }, + "system" => { let stop_ch = self.find_char(b"\"'")?; - let var = match self.parse_string(stop_ch)?.as_str() { - "domain" => SystemVariable::Domain, - "hostname" => SystemVariable::Hostname, - "node_id" => SystemVariable::NodeId, - "node_hostname" => SystemVariable::NodeHostname, - "node_role" => SystemVariable::NodeRole, - other => { - return Err(format!( - "Invalid system variable name {:?}", - other - )); - } - }; + let name = self.parse_string(stop_ch)?; + let var = SystemVariable::parse(&name).ok_or_else(|| { + format!("Invalid system variable name {:?}", name) + })?; self.has_alpha = false; - self.buf.clear(); + self.clear_word(); (Token::System(var).into(), b'(') - } + }, _ => { self.is_start = false; - (self.parse_buf()?.into(), ch) + (self.parse_word()?.into(), ch) } - } - } else if !self.buf.is_empty() { + ) + } else if !self.word().is_empty() { self.is_start = false; - (self.parse_buf()?.into(), ch) + (self.parse_word()?.into(), ch) } else { (None, ch) }; let token = match ch { b'&' => { - if matches!(self.iter.peek(), Some(b'&')) { - self.iter.next(); + if self.peek_byte() == Some(b'&') { + self.pos += 1; } Token::BinaryOperator(BinaryOperator::And) } b'|' => { - if matches!(self.iter.peek(), Some(b'|')) { - self.iter.next(); + if self.peek_byte() == Some(b'|') { + self.pos += 1; } Token::BinaryOperator(BinaryOperator::Or) } b'!' => { - if matches!(self.iter.peek(), Some(b'=')) { - self.iter.next(); + if self.peek_byte() == Some(b'=') { + self.pos += 1; Token::BinaryOperator(BinaryOperator::Ne) } else { Token::UnaryOperator(UnaryOperator::Not) @@ -186,22 +167,31 @@ impl<'x> Tokenizer<'x> { Token::BinaryOperator(BinaryOperator::Subtract) } } - b'=' => match self.iter.next() { - Some(b'=') => Token::BinaryOperator(BinaryOperator::Eq), - Some(b'>') => Token::BinaryOperator(BinaryOperator::Ge), - Some(b'<') => Token::BinaryOperator(BinaryOperator::Le), + b'=' => match self.peek_byte() { + Some(b'=') => { + self.pos += 1; + Token::BinaryOperator(BinaryOperator::Eq) + } + Some(b'>') => { + self.pos += 1; + Token::BinaryOperator(BinaryOperator::Ge) + } + Some(b'<') => { + self.pos += 1; + Token::BinaryOperator(BinaryOperator::Le) + } _ => Token::BinaryOperator(BinaryOperator::Eq), }, - b'>' => match self.iter.peek() { + b'>' => match self.peek_byte() { Some(b'=') => { - self.iter.next(); + self.pos += 1; Token::BinaryOperator(BinaryOperator::Ge) } _ => Token::BinaryOperator(BinaryOperator::Gt), }, - b'<' => match self.iter.peek() { + b'<' => match self.peek_byte() { Some(b'=') => { - self.iter.next(); + self.pos += 1; Token::BinaryOperator(BinaryOperator::Le) } _ => Token::BinaryOperator(BinaryOperator::Lt), @@ -227,7 +217,7 @@ impl<'x> Tokenizer<'x> { ); return if prev_token.is_some() { - self.next_token.push(token); + self.next_token = Some(token); Ok(prev_token) } else { Ok(Some(token)) @@ -238,22 +228,71 @@ impl<'x> Tokenizer<'x> { if self.depth > 0 { Err("Unmatched open parenthesis".to_string()) - } else if !self.buf.is_empty() { - self.parse_buf().map(Some) + } else if !self.word().is_empty() { + self.parse_word().map(Some) } else { Ok(None) } } + fn next_byte(&mut self) -> Option { + let ch = *self.input.as_bytes().get(self.pos)?; + self.pos += 1; + Some(ch) + } + + fn peek_byte(&self) -> Option { + self.input.as_bytes().get(self.pos).copied() + } + + fn word(&self) -> &'x [u8] { + self.input + .as_bytes() + .get(self.word_start..self.word_end) + .unwrap_or_default() + } + + fn push_word(&mut self) { + if self.word_start == self.word_end { + self.word_start = self.pos - 1; + } + self.word_end = self.pos; + } + + fn scan_word(&mut self, first: u8) { + self.mark_word_class(first); + while let Some(&ch) = self.input.as_bytes().get(self.pos) { + if !self.mark_word_class(ch) { + break; + } + self.pos += 1; + } + self.word_end = self.pos; + } + + fn mark_word_class(&mut self, ch: u8) -> bool { + match ch { + b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' => self.has_alpha = true, + b'0'..=b'9' => self.has_number = true, + b'.' => self.has_dot = true, + _ => return false, + } + true + } + + fn clear_word(&mut self) { + self.word_end = self.word_start; + } + fn find_char(&mut self, chars: &[u8]) -> Result { - for &ch in self.iter.by_ref() { + while let Some(ch) = self.next_byte() { if !ch.is_ascii_whitespace() { return if chars.contains(&ch) { Ok(ch) } else { Err(format!( "Expected {:?}, found invalid character {:?}", - char::from(chars[0]), + char::from(chars.first().copied().unwrap_or_default()), char::from(ch), )) }; @@ -264,138 +303,150 @@ impl<'x> Tokenizer<'x> { } fn parse_string(&mut self, stop_ch: u8) -> Result { - let mut buf = Vec::with_capacity(16); + let start = self.pos; let mut last_ch = 0; - let mut found_end = false; + let mut translate = false; - for &ch in self.iter.by_ref() { + while let Some(ch) = self.next_byte() { if last_ch != b'\\' { - if ch != stop_ch { - buf.push(ch); - } else { - found_end = true; - break; - } - } else { - match ch { - b'n' => { - buf.push(b'\n'); - } - b'r' => { - buf.push(b'\r'); - } - b't' => { - buf.push(b'\t'); - } - _ => { - buf.push(ch); - } + if ch == stop_ch { + let raw = self.input.get(start..self.pos - 1); + return match raw { + Some(raw) if !translate => Ok(CompactString::from(raw)), + Some(raw) => CompactString::from_utf8(unescape(raw.as_bytes())) + .map_err(|_| "Invalid UTF-8".into()), + None => Err("Invalid UTF-8".into()), + }; } + } else if matches!(ch, b'n' | b'r' | b't') { + translate = true; } - last_ch = ch; } - if found_end { - CompactString::from_utf8(buf).map_err(|_| "Invalid UTF-8".into()) - } else { - Err("Unterminated string".to_string()) - } + Err("Unterminated string".to_string()) } - fn parse_buf(&mut self) -> Result { - let buf = String::from_utf8(std::mem::take(&mut self.buf)).unwrap_or_default(); - if self.has_number && !self.has_alpha { - self.has_number = false; - if self.has_dot { - self.has_dot = false; + fn parse_word(&mut self) -> Result { + let word = self + .input + .get(self.word_start..self.word_end) + .unwrap_or_default(); + let has_number = self.has_number; + let has_alpha = self.has_alpha; + let has_dot = self.has_dot; + self.clear_word(); + self.has_alpha = false; + self.has_number = false; + self.has_dot = false; - buf.parse::() + if has_number && !has_alpha { + if has_dot { + word.parse::() .map(|f| Token::Constant(Constant::Float(f))) - .map_err(|_| format!("Invalid float value {}", buf,)) + .map_err(|_| format!("Invalid float value {}", word,)) } else { - buf.parse::() + word.parse::() .map(|i| Token::Constant(Constant::Integer(i))) - .map_err(|_| format!("Invalid integer value {}", buf,)) + .map_err(|_| format!("Invalid integer value {}", word,)) } - } else { - let has_dot = self.has_dot; - let has_number = self.has_number; - - self.has_alpha = false; - self.has_number = false; - self.has_dot = false; - - if !has_number && !has_dot && [4, 5].contains(&buf.len()) { - if buf == "true" { - return Ok(Token::Constant(Constant::Integer(1))); - } else if buf == "false" { - return Ok(Token::Constant(Constant::Integer(0))); - } - } - - if let Some(variable) = buf.strip_prefix('$').filter(|s| !s.is_empty()) { - if variable.chars().all(|c| c.is_ascii_digit()) { - Ok(variable - .parse::() - .map(Token::Capture) - .unwrap_or_else(|_| Token::Global(variable.into()))) - } else { - Ok(Token::Global(variable.into())) - } - } else if let Some((idx, (name, _, num_args))) = FUNCTIONS - .iter() - .enumerate() - .find(|(_, (name, _, _))| name == &buf) - { - Ok(Token::Function { - name: Cow::Borrowed(*name), - id: idx as u32, - num_args: *num_args, - }) - } else if let Some((name, idx, num_args)) = - ASYNC_FUNCTIONS.iter().find(|(name, _, _)| name == &buf) - { - Ok(Token::Function { - name: Cow::Borrowed(*name), - id: *idx + FUNCTIONS.len() as u32, - num_args: *num_args, - }) - } else if let Some(variable) = ExpressionVariable::parse(buf.as_str()) { - if self.token_map.variables.is_empty() - || self.token_map.variables.contains(&variable) - { - Ok(Token::Variable(variable)) - } else { - Err(format!("Variable {:?} not allowed in this context", buf)) - } - } else if let Some(constant) = ExpressionConstant::parse(buf.as_str()) { - if self.token_map.constants.is_empty() - || self.token_map.constants.contains(&constant) - { - Ok(Token::Constant(Constant::Static(constant))) - } else { - Err(format!("Constant {:?} not allowed in this context", buf)) - } - } else if let Ok(duration) = registry::types::duration::Duration::from_str(&buf) { - Ok(Token::Constant(Constant::Integer( - duration.as_millis() as i64 - ))) + } else if let Some(value) = hashify::map!(word.as_bytes(), i64, "true" => 1, "false" => 0) { + Ok(Token::Constant(Constant::Integer(*value))) + } else if let Some(variable) = word.strip_prefix('$').filter(|s| !s.is_empty()) { + if variable.chars().all(|c| c.is_ascii_digit()) { + Ok(variable + .parse::() + .map(Token::Capture) + .unwrap_or_else(|_| Token::Global(variable.into()))) } else { - Err(format!("Invalid variable or constant {buf:?}")) + Ok(Token::Global(variable.into())) } + } else if let Some(function) = lookup_function(word) { + Ok(Token::Function { + name: function.name, + id: function.id, + num_args: function.num_args, + }) + } else if let Some(variable) = ExpressionVariable::parse(word) { + if self.token_map.variables.allows(variable.to_id()) { + Ok(Token::Variable(variable)) + } else { + Err(format!("Variable {:?} not allowed in this context", word)) + } + } else if let Some(constant) = ExpressionConstant::parse(word) { + if self.token_map.constants.allows(constant.to_id()) { + Ok(Token::Constant(Constant::Static(constant))) + } else { + Err(format!("Constant {:?} not allowed in this context", word)) + } + } else if let Ok(duration) = registry::types::duration::Duration::from_str(word) { + Ok(Token::Constant(Constant::Integer( + duration.as_millis() as i64 + ))) + } else { + Err(format!("Invalid variable or constant {word:?}")) } } } +fn unescape(raw: &[u8]) -> Vec { + let mut buf = Vec::with_capacity(raw.len()); + let mut last_ch = 0; + for &ch in raw { + if last_ch != b'\\' { + buf.push(ch); + } else { + buf.push(match ch { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + _ => ch, + }); + } + last_ch = ch; + } + buf +} + impl TokenMap { pub fn with_variables(mut self, variables: &[ExpressionVariable]) -> Self { - self.variables.extend(variables.iter().copied()); + for variable in variables { + self.variables.insert(variable.to_id()); + } self } pub fn with_constants(mut self, constants: &[ExpressionConstant]) -> Self { - self.constants.extend(constants.iter().copied()); + for constant in constants { + self.constants.insert(constant.to_id()); + } self } } + +impl IdSet { + const BITS: usize = u64::BITS as usize; + + fn insert(&mut self, id: u16) { + let id = id as usize; + if let Some(word) = self.0.get_mut(id / Self::BITS) { + *word |= 1 << (id % Self::BITS); + } + } + + fn allows(&self, id: u16) -> bool { + let id = id as usize; + self.0.iter().all(|word| *word == 0) + || self + .0 + .get(id / Self::BITS) + .is_some_and(|word| word & (1 << (id % Self::BITS)) != 0) + } +} + +impl Default for IdSet { + fn default() -> Self { + IdSet([0; WORDS]) + } +} + diff --git a/crates/common/src/expr/variable.rs b/crates/common/src/expr/variable.rs new file mode 100644 index 00000000..55f9490a --- /dev/null +++ b/crates/common/src/expr/variable.rs @@ -0,0 +1,426 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{BinaryOperator, Variable}; +use bumpalo::{ + Bump, + collections::{String as BumpString, Vec as BumpVec}, +}; +use compact_str::CompactString; +use hyper::StatusCode; +use registry::types::EnumImpl; +use std::{ + cmp::Ordering, + fmt::{self, Display, Write}, +}; + +const CONCAT_NUMBER_RESERVE: usize = 24; + +impl<'x> Variable<'x> { + pub fn op_add(self, other: Variable<'x>, arena: &'x Bump) -> Variable<'x> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_add(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a + b), + (Variable::Integer(i), Variable::Float(f)) + | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 + f), + (Variable::Array(a), Variable::Array(b)) => Variable::Array(concat(arena, a, b)), + (Variable::Array(a), b) => Variable::Array(concat(arena, a, &[b])), + (a, Variable::Array(b)) => Variable::Array(concat(arena, &[a], b)), + (Variable::String(a), b) => { + if !a.is_empty() { + Variable::String(concat_display(arena, self, b)) + } else { + b + } + } + (a, Variable::String(b)) => { + if !b.is_empty() { + Variable::String(concat_display(arena, a, other)) + } else { + a + } + } + (a, Variable::Constant(_)) => a, + (Variable::Constant(_), b) => b, + } + } + + pub fn op_subtract(self, other: Variable<'x>, arena: &'x Bump) -> Variable<'x> { + match (self, other) { + (Variable::Array(a), b) | (b, Variable::Array(a)) => { + let mut items = BumpVec::with_capacity_in(a.len(), arena); + items.extend(a.iter().filter(|v| *v != &b).copied()); + Variable::Array(items.into_bump_slice()) + } + (a, b) => a.op_subtract_number(b), + } + } + + fn op_subtract_number(self, other: Variable<'_>) -> Variable<'static> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_sub(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a - b), + (Variable::Integer(a), Variable::Float(b)) => Variable::Float(a as f64 - b), + (Variable::Float(a), Variable::Integer(b)) => Variable::Float(a - b as f64), + (a, b) => a.parse_number().op_subtract_number(b.parse_number()), + } + } + + pub fn op_multiply(self, other: Variable<'_>) -> Variable<'static> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_mul(b)), + (Variable::Float(a), Variable::Float(b)) => Variable::Float(a * b), + (Variable::Integer(i), Variable::Float(f)) + | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 * f), + (a, b) => a.parse_number().op_multiply(b.parse_number()), + } + } + + pub fn op_divide(self, other: Variable<'_>) -> Variable<'static> { + match (self, other) { + (Variable::Integer(a), Variable::Integer(b)) => { + Variable::Float(if b != 0 { a as f64 / b as f64 } else { 0.0 }) + } + (Variable::Float(a), Variable::Float(b)) => { + Variable::Float(if b != 0.0 { a / b } else { 0.0 }) + } + (Variable::Integer(a), Variable::Float(b)) => { + Variable::Float(if b != 0.0 { a as f64 / b } else { 0.0 }) + } + (Variable::Float(a), Variable::Integer(b)) => { + Variable::Float(if b != 0 { a / b as f64 } else { 0.0 }) + } + (a, b) => a.parse_number().op_divide(b.parse_number()), + } + } + + pub fn op_binary( + self, + op: BinaryOperator, + other: Variable<'x>, + arena: &'x Bump, + ) -> Variable<'x> { + match op { + BinaryOperator::Add => self.op_add(other, arena), + BinaryOperator::Subtract => self.op_subtract(other, arena), + BinaryOperator::Multiply => self.op_multiply(other), + BinaryOperator::Divide => self.op_divide(other), + BinaryOperator::And => Variable::Integer(i64::from(self.to_bool() & other.to_bool())), + BinaryOperator::Or => Variable::Integer(i64::from(self.to_bool() | other.to_bool())), + BinaryOperator::Xor => Variable::Integer(i64::from(self.to_bool() ^ other.to_bool())), + BinaryOperator::Eq + | BinaryOperator::Ne + | BinaryOperator::Lt + | BinaryOperator::Le + | BinaryOperator::Gt + | BinaryOperator::Ge => Variable::Integer(i64::from(self.compare(op, &other))), + } + } + + pub fn compare(&self, op: BinaryOperator, other: &Variable<'_>) -> bool { + match op { + BinaryOperator::Eq => self == other, + BinaryOperator::Ne => self != other, + BinaryOperator::Lt => self < other, + BinaryOperator::Le => self <= other, + BinaryOperator::Gt => self > other, + BinaryOperator::Ge => self >= other, + _ => false, + } + } + + pub fn op_not(self) -> Variable<'static> { + Variable::Integer(i64::from(!self.to_bool())) + } + + pub fn op_minus(self) -> Variable<'static> { + match self { + Variable::Integer(n) => Variable::Integer(n.wrapping_neg()), + Variable::Float(n) => Variable::Float(-n), + _ => self.parse_number().op_minus(), + } + } + + pub fn parse_number(&self) -> Variable<'static> { + match self { + Variable::String(s) if !s.is_empty() => { + if let Ok(n) = s.parse::() { + Variable::Integer(n) + } else if let Ok(n) = s.parse::() { + Variable::Float(n) + } else { + Variable::Integer(0) + } + } + Variable::Integer(n) => Variable::Integer(*n), + Variable::Float(n) => Variable::Float(*n), + Variable::Array(l) => Variable::Integer(l.is_empty() as i64), + _ => Variable::Integer(0), + } + } + + pub fn to_bool(&self) -> bool { + match self { + Variable::Float(f) => *f != 0.0, + Variable::Integer(n) => *n != 0, + Variable::String(s) => !s.is_empty(), + Variable::Array(a) => !a.is_empty(), + Variable::Constant(_) => true, + } + } + + pub fn to_str(self, arena: &'x Bump) -> &'x str { + match self { + Variable::String(s) => s, + Variable::Integer(n) => arena.alloc_str(itoa::Buffer::new().format(n)), + Variable::Float(n) => arena.alloc_str(zmij::Buffer::new().format(n)), + Variable::Array(items) => join_lines(items, arena), + Variable::Constant(c) => c.as_str(), + } + } + + pub fn to_integer(&self) -> Option { + match self { + Variable::Integer(n) => Some(*n), + Variable::Float(n) => Some(*n as i64), + Variable::String(s) if !s.is_empty() => s.parse::().ok(), + _ => None, + } + } + + pub fn to_usize(&self) -> Option { + match self { + Variable::Integer(n) => Some(*n as usize), + Variable::Float(n) => Some(*n as usize), + Variable::String(s) if !s.is_empty() => s.parse::().ok(), + _ => None, + } + } + + pub fn is_empty(&self) -> bool { + match self { + Variable::String(s) => s.is_empty(), + _ => false, + } + } + + pub fn as_array(&self) -> Option<&'x [Variable<'x>]> { + match self { + Variable::Array(l) => Some(l), + _ => None, + } + } + + pub fn into_array(self, arena: &'x Bump) -> &'x [Variable<'x>] { + match self { + Variable::Array(l) => l, + v if !v.is_empty() => arena.alloc_slice_copy(&[v]), + _ => &[], + } + } + + pub fn index(self, index: usize) -> Variable<'x> { + match self { + Variable::Array(l) => l.get(index).copied().unwrap_or_default(), + v if !v.is_empty() && index == 0 => v, + _ => Variable::default(), + } + } +} + +fn concat<'x>(arena: &'x Bump, a: &[Variable<'x>], b: &[Variable<'x>]) -> &'x [Variable<'x>] { + let mut items = BumpVec::with_capacity_in(a.len() + b.len(), arena); + items.extend_from_slice(a); + items.extend_from_slice(b); + items.into_bump_slice() +} + +fn concat_display<'x>(arena: &'x Bump, left: Variable<'_>, right: Variable<'_>) -> &'x str { + let capacity = left.display_hint() + right.display_hint(); + let mut out = BumpString::with_capacity_in(capacity, arena); + left.write_display(&mut out); + right.write_display(&mut out); + out.into_bump_str() +} + +fn join_lines<'x>(items: &[Variable<'_>], arena: &'x Bump) -> &'x str { + let capacity = items + .iter() + .map(|item| match item { + Variable::String(s) => s.len() + 2, + _ => CONCAT_NUMBER_RESERVE, + }) + .sum(); + let mut out = BumpString::with_capacity_in(capacity, arena); + for item in items { + if !out.is_empty() { + out.push_str("\r\n"); + } + match item { + Variable::String(v) => out.push_str(v), + Variable::Integer(v) => out.push_str(itoa::Buffer::new().format(*v)), + Variable::Float(v) => out.push_str(zmij::Buffer::new().format(*v)), + Variable::Array(_) => {} + Variable::Constant(c) => out.push_str(c.as_str()), + } + } + out.into_bump_str() +} + +impl Variable<'_> { + fn display_hint(&self) -> usize { + match self { + Variable::String(s) => s.len(), + Variable::Constant(c) => c.as_str().len(), + Variable::Array(items) => items.iter().map(|item| item.display_hint() + 1).sum(), + Variable::Integer(_) | Variable::Float(_) => CONCAT_NUMBER_RESERVE, + } + } + + fn write_display(&self, out: &mut BumpString<'_>) { + match self { + Variable::String(s) => out.push_str(s), + Variable::Constant(c) => out.push_str(c.as_str()), + Variable::Integer(n) => out.push_str(itoa::Buffer::new().format(*n)), + _ => { + let _ = write!(out, "{self}"); + } + } + } +} + +impl PartialEq for Variable<'_> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Integer(a), Self::Integer(b)) => a == b, + (Self::Float(a), Self::Float(b)) => a == b, + (Self::Integer(a), Self::Float(b)) | (Self::Float(b), Self::Integer(a)) => { + *a as f64 == *b + } + (Self::String(a), Self::String(b)) => a == b, + (Self::String(_), Self::Integer(_) | Self::Float(_)) => &self.parse_number() == other, + (Self::Integer(_) | Self::Float(_), Self::String(_)) => self == &other.parse_number(), + (Self::Array(a), Self::Array(b)) => a == b, + _ => false, + } + } +} + +impl Eq for Variable<'_> {} + +#[allow(clippy::non_canonical_partial_ord_impl)] +impl PartialOrd for Variable<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Integer(a), Self::Integer(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), + (Self::Integer(a), Self::Float(b)) => (*a as f64).partial_cmp(b), + (Self::Float(a), Self::Integer(b)) => a.partial_cmp(&(*b as f64)), + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::String(_), Self::Integer(_) | Self::Float(_)) => { + self.parse_number().partial_cmp(other) + } + (Self::Integer(_) | Self::Float(_), Self::String(_)) => { + self.partial_cmp(&other.parse_number()) + } + (Self::Array(a), Self::Array(b)) => a.partial_cmp(b), + (Self::Array(_) | Self::String(_), _) => Ordering::Greater.into(), + (Self::Constant(a), Self::Constant(b)) => a.to_id().partial_cmp(&b.to_id()), + (_, Self::Array(_) | Self::Constant(_)) | (Self::Constant(_), _) => { + Ordering::Less.into() + } + } + } +} + +impl Ord for Variable<'_> { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap_or(Ordering::Greater) + } +} + +impl Display for Variable<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Variable::String(v) => write!(f, "{}", v), + Variable::Integer(v) => v.fmt(f), + Variable::Float(v) => v.fmt(f), + Variable::Array(v) => { + for (i, v) in v.iter().enumerate() { + if i > 0 { + f.write_str("\n")?; + } + v.fmt(f)?; + } + Ok(()) + } + Variable::Constant(c) => c.as_str().fmt(f), + } + } +} + +impl<'x> TryFrom> for CompactString { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::String(s) => Ok(s.into()), + _ => Err(()), + } + } +} + +impl<'x> TryFrom> for String { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::String(s) => Ok(s.to_string()), + _ => Err(()), + } + } +} + +impl<'x> From> for bool { + fn from(val: Variable<'x>) -> Self { + val.to_bool() + } +} + +impl<'x> TryFrom> for i64 { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + value.to_integer().ok_or(()) + } +} + +impl<'x> TryFrom> for u64 { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + value.to_integer().map(|v| v as u64).ok_or(()) + } +} + +impl<'x> TryFrom> for usize { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + value.to_usize().ok_or(()) + } +} + +impl<'x> TryFrom> for StatusCode { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + value + .to_integer() + .and_then(|code| StatusCode::from_u16(code as u16).ok()) + .ok_or(()) + } +} diff --git a/crates/common/src/i18n.rs b/crates/common/src/i18n.rs index 4b10993a..72c2f321 100644 --- a/crates/common/src/i18n.rs +++ b/crates/common/src/i18n.rs @@ -29,71 +29,3 @@ pub fn locale_or_default(name: &str) -> &'static Locale { locale_by_language(language).unwrap_or(&EN_US_LOCALES) } -#[cfg(test)] -mod tests { - use super::{ALL_LOCALES, locale, locale_or_default}; - - const LOCALES: [&str; 37] = [ - "en-US", "es-ES", "fr-FR", "de-DE", "it-IT", "pt-PT", "pt-BR", "nl-NL", "da-DK", "ca-ES", - "el-GR", "sv-SE", "pl-PL", "ru-RU", "uk-UA", "bg-BG", "cs-CZ", "sk-SK", "sl-SI", "hr-HR", - "lt-LT", "hu-HU", "ro-RO", "fi-FI", "nb-NO", "tr-TR", "zh-CN", "zh-TW", "ja-JP", "ko-KR", - "th-TH", "vi-VN", "id-ID", "hi-IN", "ar-SA", "he-IL", "fa-IR", - ]; - - #[test] - fn locales_are_named_after_themselves() { - for lang in LOCALES { - assert_eq!(locale(lang).expect("locale must exist").name, lang); - } - assert_eq!(ALL_LOCALES.len(), LOCALES.len()); - } - - #[test] - fn bare_and_hyphenated_language_tags_resolve() { - for (input, expected) in [ - ("es-ES", "es-ES"), - ("es", "es-ES"), - ("es-MX", "es-ES"), - ("pt-BR", "pt-BR"), - ("pt-PT", "pt-PT"), - ("pt", "pt-BR"), - ("zh-Hans", "zh-CN"), - ("zh-Hant", "zh-TW"), - ("zh-HK", "zh-TW"), - ("zh-Hant-HK", "zh-TW"), - ("zh", "zh-CN"), - ("zz", "en-US"), - ("", "en-US"), - // BCP 47 tags are case-insensitive - ("ES", "es-ES"), - ("es-es", "es-ES"), - ("PT-br", "pt-BR"), - ("EL-GR", "el-GR"), - ("ZH-HANT", "zh-TW"), - ] { - assert_eq!( - locale_or_default(input).name, - expected, - "failed for {input}" - ); - } - } - - #[test] - fn right_to_left_locales_are_flagged() { - for lang in ["ar-SA", "he-IL", "fa-IR"] { - assert_eq!( - locale_or_default(lang).direction, - "rtl", - "failed for {lang}" - ); - } - for lang in ["en-US", "de-DE", "ja-JP", "ru-RU"] { - assert_eq!( - locale_or_default(lang).direction, - "ltr", - "failed for {lang}" - ); - } - } -} diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index ff8f4053..fd8f89d0 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -10,30 +10,25 @@ use crate::config::smtp::{ resolver::{Policy, Tlsa}, }; use ahash::RandomState; -use mail_auth::{ - dmarc::Dmarc, - mta_sts::TlsRpt, - report::{Record, tlsrpt::FailureDetails}, -}; +use mail_auth::{dmarc::DmarcRecord, mta_sts::TlsRptRecord}; use registry::{schema::prelude::ObjectType, types::id::ObjectId}; +use smtp_report::{dmarc::Record, tlsrpt::FailureDetails}; use std::sync::{ Arc, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU32, Ordering}, }; -use tokio::sync::{Semaphore, SemaphorePermit, mpsc}; +use store::write::TaskTypes; +use tinyvec::TinyVec; +use tokio::sync::{Notify, Semaphore, SemaphorePermit, futures::Notified, mpsc}; use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; +pub mod subscribers; + #[derive(Debug)] pub enum PushEvent { - Subscribe { - account_ids: Vec, - types: Bitmap, - tx: mpsc::Sender, - }, Publish { notification: PushNotification, - broadcast: bool, }, PushServerRegister { activate: Vec, @@ -49,10 +44,19 @@ pub enum PushEvent { #[derive(Debug, Clone)] pub enum PushNotification { StateChange(StateChange), + ViewerStateChange(ViewerStateChange), CalendarAlert(CalendarAlert), EmailPush(EmailPush), } +const _: () = assert!(size_of::() <= 80); + +#[derive(Debug, Clone, Copy)] +pub struct ViewerStateChange { + pub viewer_id: u32, + pub change: StateChange, +} + #[derive(Debug, Clone)] pub struct EmailPush { pub account_id: u32, @@ -63,6 +67,7 @@ pub struct EmailPush { #[derive(Debug, Clone)] pub struct CalendarAlert { pub account_id: u32, + pub event_account_id: u32, pub event_id: u32, pub recurrence_id: Option, pub uid: String, @@ -78,10 +83,56 @@ pub enum BroadcastEvent { CacheInvalidateAll, CacheInvalidateNegative, MtaQueueStatus { is_running: bool }, - QueueRefresh, + QueueRefresh(QueueNames), + TaskQueueRefresh(TaskTypes), + IndexQueueRefresh, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] +pub struct IndexQueueNotify { + notify: Arc, + broadcast: Option>, +} + +impl IndexQueueNotify { + pub fn new(notify: Arc, broadcast: Option>) -> Self { + Self { notify, broadcast } + } + + pub fn notify(&self) { + self.notify.notify_one(); + + if let Some(broadcast) = &self.broadcast { + let _ = broadcast.try_send(BroadcastEvent::IndexQueueRefresh); + } + } +} + +#[derive(Debug, Default)] +pub struct TaskQueueNotify { + notify: Notify, + task_types: AtomicU32, +} + +impl TaskQueueNotify { + pub fn notify(&self, task_types: TaskTypes) { + if !task_types.is_empty() { + self.task_types + .fetch_or(task_types.bits(), Ordering::Release); + } + self.notify.notify_one(); + } + + pub fn take(&self) -> TaskTypes { + TaskTypes::from_bits(self.task_types.swap(0, Ordering::Acquire)) + } + + pub fn notified(&self) -> Notified<'_> { + self.notify.notified() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum RegistryChange { Insert(ObjectId), Delete(ObjectId), @@ -91,25 +142,30 @@ pub enum RegistryChange { #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub enum CacheInvalidation { AccessToken(u32), - DavResources(u32), + GroupwareResources(u32), + MessageCache(u32), Domain(u32), Account(u32), DkimSignature(u32), - Tenant(u32), Role(u32), List(u32), DomainLogo(u32), - TenantLogo(u32), EmailNegative { domain_id: u32, local_part_hash: u32, }, DomainNegative, + PrivateMetadata(u32), + OAuthGrant { + account_id: u32, + grant_id: u64, + }, + ActiveSieveScript(u32), } #[derive(Debug)] pub enum QueueEvent { - Refresh, + Refresh(QueueRefresh), WorkerDone { queue_id: u64, queue_name: QueueName, @@ -120,6 +176,61 @@ pub enum QueueEvent { Stop, } +#[derive(Debug)] +pub enum QueueRefresh { + Queued(QueuedEvents), + Queues(QueueNames), + All, +} + +pub type QueuedEvents = TinyVec<[QueuedEvent; 2]>; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct QueueNames(TinyVec<[QueueName; 2]>); + +impl QueueNames { + pub fn insert(&mut self, name: QueueName) { + if !self.0.contains(&name) { + self.0.push(name); + } + } + + pub fn merge(&mut self, other: &QueueNames) { + for name in other.iter() { + self.insert(name); + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter().copied() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl FromIterator for QueueNames { + fn from_iter>(iter: I) -> Self { + let mut names = QueueNames::default(); + for name in iter { + names.insert(name); + } + names + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct QueuedEvent { + pub queue_name: QueueName, + pub due: u64, + pub queue_id: u64, +} + #[derive(Debug)] pub enum QueueEventStatus { Completed, @@ -138,7 +249,7 @@ pub enum ReportingEvent { pub struct DmarcEvent { pub domain: String, pub report_record: Record, - pub dmarc_record: Arc, + pub dmarc_record: Arc, pub interval: AggregateFrequency, pub span_id: u64, } @@ -148,7 +259,7 @@ pub struct TlsEvent { pub domain: String, pub policy: PolicyType, pub failure: Option, - pub tls_record: Arc, + pub tls_record: Arc, pub interval: AggregateFrequency, pub span_id: u64, } @@ -206,7 +317,7 @@ pub trait ToHash { fn to_hash(&self) -> u64; } -impl ToHash for Dmarc { +impl ToHash for DmarcRecord { fn to_hash(&self) -> u64 { RandomState::with_seeds(1, 9, 7, 9).hash_one(self) } @@ -265,9 +376,10 @@ impl From<(&Option>, &Option>)> for PolicyType { } impl PushNotification { - pub fn account_id(&self) -> u32 { + pub fn recipient(&self) -> u32 { match self { PushNotification::StateChange(state_change) => state_change.account_id, + PushNotification::ViewerStateChange(viewer_change) => viewer_change.viewer_id, PushNotification::CalendarAlert(calendar_alert) => calendar_alert.account_id, PushNotification::EmailPush(email_push) => email_push.account_id, } @@ -276,17 +388,15 @@ impl PushNotification { pub fn filter_types(&self, types: &Bitmap) -> Option { match self { PushNotification::StateChange(state_change) => { - let mut filtered_types = state_change.types; - filtered_types.intersection(types); - if !filtered_types.is_empty() { - Some(PushNotification::StateChange(StateChange { - account_id: state_change.account_id, - change_id: state_change.change_id, - types: filtered_types, - })) - } else { - None - } + filter_state_change(state_change, types).map(PushNotification::StateChange) + } + PushNotification::ViewerStateChange(viewer_change) => { + filter_state_change(&viewer_change.change, types).map(|change| { + PushNotification::ViewerStateChange(ViewerStateChange { + viewer_id: viewer_change.viewer_id, + change, + }) + }) } PushNotification::CalendarAlert(_) => { if types.contains(DataType::CalendarAlert) { @@ -314,6 +424,18 @@ impl PushNotification { } } +fn filter_state_change( + state_change: &StateChange, + types: &Bitmap, +) -> Option { + let mut filtered_types = state_change.types; + filtered_types.intersection(types); + (!filtered_types.is_empty()).then_some(StateChange { + types: filtered_types, + ..*state_change + }) +} + impl EmailPush { pub fn to_state_change(&self) -> StateChange { StateChange { diff --git a/crates/common/src/ipc/subscribers.rs b/crates/common/src/ipc/subscribers.rs new file mode 100644 index 00000000..6f3b2312 --- /dev/null +++ b/crates/common/src/ipc/subscribers.rs @@ -0,0 +1,643 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CalendarAlert, PushNotification, ViewerStateChange}; +use ahash::AHashMap; +use futures::task::AtomicWaker; +use parking_lot::{Mutex, RwLock}; +use std::{ + collections::VecDeque, + future::poll_fn, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + task::Poll, +}; +use tokio::runtime::Handle; +use trc::ServerEvent; +use types::type_state::{DataType, StateChange}; +use utils::map::bitmap::Bitmap; + +const SHARDS: usize = 256; +const _: () = assert!(SHARDS.is_power_of_two()); +const MAX_PENDING_ALERTS: usize = 64; +const MAX_RELAYED_ALERTS: usize = 1024; +const MAX_TRACKED_CHANGES: usize = 64; +const RECLAIM_SLACK: u32 = 2; +const OFFLOAD_THRESHOLD: u32 = 256; +const RELAY_CHUNK: usize = 256; + +pub struct PushSubscribers { + shards: Box<[Shard]>, + is_closed: AtomicBool, + offload_threshold: u32, +} + +#[derive(Default)] +#[repr(align(128))] +struct Shard(RwLock>); + +type Slots = Arc<[OnceLock]>; + +struct Slot { + mailbox: Arc, + types: u64, +} + +#[derive(Default)] +struct AccountSubscribers { + slots: Slots, + relay: Option>, + len: u32, + live: u32, + is_push: bool, +} + +#[derive(Default)] +struct Relay { + queue: Mutex, +} + +#[derive(Default)] +struct RelayQueue { + filtered: Changes, + unfiltered: Changes, + alerts: VecDeque, + is_running: bool, +} + +pub struct PushReceiver { + mailbox: Arc, + subscribers: Arc, + account_ids: Box<[u32]>, +} + +struct Mailbox { + types: AtomicU64, + is_retired: AtomicBool, + pending: Mutex, + waker: AtomicWaker, +} + +#[derive(Default)] +struct Pending { + changes: Changes, + alerts: VecDeque, + is_closed: bool, +} + +#[derive(Default)] +struct Changes { + entries: Vec, + has_pending: bool, +} + +#[derive(Debug, Clone, Copy)] +struct Change { + viewer_id: Option, + account_id: u32, + types: u64, + pending: u64, + change_id: u64, +} + +enum Taken { + Some, + Empty, + Closed, +} + +enum Route { + Inline(Slots, usize), + Relay(Arc, Handle), +} + +impl PushSubscribers { + pub fn with_offload_threshold(offload_threshold: u32) -> Self { + PushSubscribers { + shards: (0..SHARDS).map(|_| Shard::default()).collect(), + is_closed: AtomicBool::new(false), + offload_threshold, + } + } + + fn shard(&self, account_id: u32) -> &RwLock> { + &self.shards[account_id as usize & (SHARDS - 1)].0 + } + + pub fn subscribe( + self: &Arc, + account_ids: impl IntoIterator, + types: Bitmap, + ) -> PushReceiver { + let mailbox = Arc::new(Mailbox { + types: AtomicU64::new(*types), + is_retired: AtomicBool::new(false), + pending: Mutex::default(), + waker: AtomicWaker::new(), + }); + if self.is_closed.load(Ordering::SeqCst) { + mailbox.retire(); + return PushReceiver { + mailbox, + subscribers: self.clone(), + account_ids: Box::default(), + }; + } + let account_ids = account_ids.into_iter().collect::>(); + for &account_id in &account_ids { + let mut accounts = self.shard(account_id).write(); + let retired = accounts.entry(account_id).or_default().add( + &mailbox, + *types, + self.offload_threshold, + ); + drop(accounts); + drop(retired); + } + if self.is_closed.load(Ordering::SeqCst) { + mailbox.close(); + } + PushReceiver { + mailbox, + subscribers: self.clone(), + account_ids, + } + } + + fn unsubscribe(&self, account_ids: &[u32]) { + for &account_id in account_ids { + let mut accounts = self.shard(account_id).write(); + let Some(account) = accounts.get_mut(&account_id) else { + continue; + }; + account.live -= 1; + let retired = if account.live == 0 && !account.is_push { + accounts.remove(&account_id).map(|account| account.slots) + } else { + account.reclaim() + }; + drop(accounts); + drop(retired); + } + } + + pub fn dispatch(self: &Arc, notification: &PushNotification) -> bool { + let account_id = notification.recipient(); + let (route, is_push) = { + let accounts = self.shard(account_id).read(); + let Some(account) = accounts.get(&account_id) else { + return false; + }; + let relay = account.relay.as_ref().filter(|relay| { + account.live > self.offload_threshold || relay.queue.lock().is_running + }); + let route = match relay.and_then(|relay| Some((relay, Handle::try_current().ok()?))) { + Some((relay, handle)) => Route::Relay(relay.clone(), handle), + None => Route::Inline(account.slots.clone(), account.len as usize), + }; + (route, account.is_push) + }; + match route { + Route::Inline(slots, len) => deliver(&slots[..len], notification), + Route::Relay(relay, handle) => { + if relay.queue.lock().push(notification) { + handle.spawn(self.clone().relay(account_id, relay)); + } + } + } + is_push + } + + async fn relay(self: Arc, account_id: u32, relay: Arc) { + let mut batch = RelayQueue::default(); + loop { + { + let mut queue = relay.queue.lock(); + if queue.is_empty() { + queue.is_running = false; + return; + } + queue.swap(&mut batch); + } + let snapshot = { + let accounts = self.shard(account_id).read(); + accounts + .get(&account_id) + .filter(|account| { + account + .relay + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &relay)) + }) + .map(|account| (account.slots.clone(), account.len as usize)) + }; + if let Some((slots, len)) = snapshot { + for chunk in slots[..len].chunks(RELAY_CHUNK) { + batch.deliver(chunk); + tokio::task::yield_now().await; + } + } + batch.clear(); + } + } + + pub fn register_push(&self, activate: &[u32], expired: &[u32]) { + for &account_id in activate { + self.shard(account_id) + .write() + .entry(account_id) + .or_default() + .is_push = true; + } + for &account_id in expired { + let mut accounts = self.shard(account_id).write(); + let retired = match accounts.get_mut(&account_id) { + Some(account) if account.live == 0 => accounts.remove(&account_id), + Some(account) => { + account.is_push = false; + None + } + None => None, + }; + drop(accounts); + drop(retired); + } + } + + pub fn close_all(&self) { + self.is_closed.store(true, Ordering::SeqCst); + for shard in self.shards.iter() { + for account in shard.0.read().values() { + for slot in account.slots() { + slot.mailbox.close(); + } + } + } + } +} + +fn deliver(slots: &[OnceLock], notification: &PushNotification) { + match notification { + PushNotification::StateChange(change) => deliver_change(slots, None, change, true), + PushNotification::ViewerStateChange(ViewerStateChange { viewer_id, change }) => { + deliver_change(slots, Some(*viewer_id), change, true) + } + PushNotification::EmailPush(push) => { + deliver_change(slots, None, &push.to_state_change(), false) + } + PushNotification::CalendarAlert(alert) => deliver_alert(slots, alert), + } +} + +fn deliver_change( + slots: &[OnceLock], + viewer_id: Option, + change: &StateChange, + is_filtered: bool, +) { + let types = *change.types; + for slot in slots.iter().filter_map(OnceLock::get) { + if slot.types & types != 0 { + slot.mailbox.deliver_change(viewer_id, change, is_filtered); + } + } +} + +fn deliver_alert(slots: &[OnceLock], alert: &CalendarAlert) { + let filter = *Bitmap::from_iter([DataType::CalendarAlert]); + for slot in slots.iter().filter_map(OnceLock::get) { + if slot.types & filter != 0 { + slot.mailbox.deliver_alert(alert); + } + } +} + +impl RelayQueue { + fn is_empty(&self) -> bool { + !self.filtered.has_pending && !self.unfiltered.has_pending && self.alerts.is_empty() + } + + fn push(&mut self, notification: &PushNotification) -> bool { + match notification { + PushNotification::StateChange(change) => { + self.filtered.merge(None, change); + } + PushNotification::ViewerStateChange(ViewerStateChange { viewer_id, change }) => { + self.filtered.merge(Some(*viewer_id), change); + } + PushNotification::EmailPush(push) => { + self.unfiltered.merge(None, &push.to_state_change()); + } + PushNotification::CalendarAlert(alert) => { + if self.alerts.len() == MAX_RELAYED_ALERTS { + self.alerts.pop_front(); + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Dropped the oldest calendar alert of a busy account.", + CausedBy = trc::location!() + ); + } + self.alerts.push_back(alert.clone()); + } + } + !std::mem::replace(&mut self.is_running, true) + } + + fn swap(&mut self, batch: &mut RelayQueue) { + std::mem::swap(&mut self.filtered, &mut batch.filtered); + std::mem::swap(&mut self.unfiltered, &mut batch.unfiltered); + std::mem::swap(&mut self.alerts, &mut batch.alerts); + } + + fn deliver(&self, slots: &[OnceLock]) { + for (changes, is_filtered) in [(&self.filtered, true), (&self.unfiltered, false)] { + for (viewer_id, change) in changes.pending() { + deliver_change(slots, viewer_id, &change, is_filtered); + } + } + for alert in &self.alerts { + deliver_alert(slots, alert); + } + } + + fn clear(&mut self) { + self.filtered.clear(); + self.unfiltered.clear(); + self.alerts.clear(); + } +} + +impl AccountSubscribers { + fn slots(&self) -> impl Iterator { + self.slots[..self.len as usize] + .iter() + .filter_map(OnceLock::get) + } + + fn add(&mut self, mailbox: &Arc, types: u64, offload_threshold: u32) -> Option { + let retired = (self.len as usize == self.slots.len()).then(|| self.compact(1)); + let _ = self.slots[self.len as usize].set(Slot { + mailbox: mailbox.clone(), + types, + }); + self.len += 1; + self.live += 1; + if self.live > offload_threshold && self.relay.is_none() { + self.relay = Some(Arc::default()); + } + retired + } + + fn reclaim(&mut self) -> Option { + (self.len > 2 * self.live + RECLAIM_SLACK).then(|| self.compact(0)) + } + + fn compact(&mut self, reserve: usize) -> Slots { + let mut slots = Vec::with_capacity(2 * self.live as usize + reserve); + let mut len = 0u32; + slots.extend( + self.slots() + .filter(|slot| !slot.mailbox.is_retired.load(Ordering::Relaxed)) + .map(|slot| { + len += 1; + OnceLock::from(Slot { + mailbox: slot.mailbox.clone(), + types: slot.types, + }) + }), + ); + self.len = len; + slots.resize_with(2 * slots.len() + reserve, OnceLock::new); + std::mem::replace(&mut self.slots, slots.into()) + } +} + +impl PushReceiver { + pub async fn recv(&mut self, notifications: &mut Vec) -> bool { + poll_fn(|cx| match self.mailbox.take(notifications) { + Taken::Some => Poll::Ready(true), + Taken::Closed => Poll::Ready(false), + Taken::Empty => { + self.mailbox.waker.register(cx.waker()); + match self.mailbox.take(notifications) { + Taken::Some => Poll::Ready(true), + Taken::Closed => Poll::Ready(false), + Taken::Empty => Poll::Pending, + } + } + }) + .await + } +} + +impl Drop for PushReceiver { + fn drop(&mut self) { + self.mailbox.retire(); + self.subscribers.unsubscribe(&self.account_ids); + } +} + +impl Mailbox { + fn deliver_change(&self, viewer_id: Option, change: &StateChange, is_filtered: bool) { + let mut pending = self.pending.lock(); + let types = self.types.load(Ordering::Relaxed); + if types == 0 { + return; + } + let was_empty = pending.is_empty(); + let types = if is_filtered { + types & *change.types + } else { + *change.types + }; + pending + .changes + .merge_types(viewer_id, change.account_id, types, change.change_id); + let wake = was_empty && !pending.is_empty(); + drop(pending); + if wake { + self.waker.wake(); + } + } + + fn deliver_alert(&self, alert: &CalendarAlert) { + let alert = alert.clone(); + let mut pending = self.pending.lock(); + if self.types.load(Ordering::Relaxed) == 0 { + return; + } + let was_empty = pending.is_empty(); + if pending.alerts.len() == MAX_PENDING_ALERTS { + pending.alerts.pop_front(); + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Dropped the oldest calendar alert of a slow subscriber.", + CausedBy = trc::location!() + ); + } + pending.alerts.push_back(alert); + drop(pending); + if was_empty { + self.waker.wake(); + } + } + + fn take(&self, notifications: &mut Vec) -> Taken { + let mut pending = self.pending.lock(); + if pending.is_empty() { + return if pending.is_closed { + Taken::Closed + } else { + Taken::Empty + }; + } + notifications.extend(pending.changes.pending().map( + |(viewer_id, change)| match viewer_id { + Some(viewer_id) => { + PushNotification::ViewerStateChange(ViewerStateChange { viewer_id, change }) + } + None => PushNotification::StateChange(change), + }, + )); + pending.changes.mark_taken(); + notifications.extend( + pending + .alerts + .drain(..) + .map(PushNotification::CalendarAlert), + ); + Taken::Some + } + + fn retire(&self) { + self.types.store(0, Ordering::Relaxed); + self.is_retired.store(true, Ordering::Relaxed); + self.pending.lock().is_closed = true; + } + + fn close(&self) { + self.retire(); + self.waker.wake(); + } +} + +impl Pending { + fn is_empty(&self) -> bool { + !self.changes.has_pending && self.alerts.is_empty() + } +} + +impl Changes { + fn merge(&mut self, viewer_id: Option, change: &StateChange) -> bool { + self.merge_types( + viewer_id, + change.account_id, + *change.types, + change.change_id, + ) + } + + fn merge_types( + &mut self, + viewer_id: Option, + account_id: u32, + mut types: u64, + change_id: u64, + ) -> bool { + if types == 0 { + return false; + } + let same_key = + |entry: &Change| entry.viewer_id == viewer_id && entry.account_id == account_id; + for entry in self.entries.iter_mut().filter(|entry| same_key(entry)) { + let overlap = entry.types & types; + if overlap != 0 { + if entry.change_id >= change_id { + types &= !overlap; + } else { + entry.types &= !overlap; + entry.pending &= !overlap; + } + } + } + self.entries.retain(|entry| entry.types != 0); + if types == 0 { + return false; + } + match self + .entries + .iter_mut() + .find(|entry| same_key(entry) && entry.change_id == change_id) + { + Some(entry) => { + entry.types |= types; + entry.pending |= types; + } + None => self.entries.push(Change { + viewer_id, + account_id, + types, + pending: types, + change_id, + }), + } + self.has_pending = true; + if self.entries.len() > MAX_TRACKED_CHANGES { + self.forget_taken(); + } + true + } + + fn pending(&self) -> impl Iterator, StateChange)> + '_ { + self.entries + .iter() + .filter(|entry| entry.pending != 0) + .map(|entry| { + ( + entry.viewer_id, + StateChange { + account_id: entry.account_id, + change_id: entry.change_id, + types: Bitmap::from(entry.pending), + }, + ) + }) + } + + fn mark_taken(&mut self) { + for entry in &mut self.entries { + entry.pending = 0; + } + self.has_pending = false; + } + + #[cold] + fn forget_taken(&mut self) { + let mut excess = self.entries.len() - MAX_TRACKED_CHANGES / 2; + self.entries.retain(|entry| { + if excess > 0 && entry.pending == 0 { + excess -= 1; + false + } else { + true + } + }); + } + + fn clear(&mut self) { + self.entries.clear(); + self.has_pending = false; + } +} + +impl Default for PushSubscribers { + fn default() -> Self { + Self::with_offload_threshold(OFFLOAD_THRESHOLD) + } +} + diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 63fe0245..5a573aae 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -6,59 +6,69 @@ #![warn(clippy::large_futures)] -use crate::auth::{AccessTokenInner, EmailAddress}; +use crate::auth::{AccessTokenInner, EmailAddress, TokenScope}; use crate::manager::application::WebApplications; use crate::network::asn::AsnGeoLookupData; +use crate::network::memory::MemoryBudget; use crate::{ - auth::{AccountCache, DomainCache, EmailCache, MailingListCache, RoleCache, TenantCache}, + auth::{AccountCache, DomainCache, EmailCache, MailingListCache, RoleCache}, + cache::{ + email::{IdIndex, ThreadRange}, + http_auth::{GrantRevocations, HttpAuthEntries}, + large::{LargeMessageCaches, LargeResourceCaches}, + swap::evict::SwapEviction, + }, config::{ mailstore::{ email::EmailConfig, imap::ImapConfig, scripts::Scripting, - spamfilter::{IpResolver, SpamClassifier, SpamFilterConfig}, + spamfilter::{SpamClassifier, SpamFilterConfig, SpamLookup}, }, smtp::auth::DkimSigners, }, ipc::TrainTaskController, network::security::BlockedIps, + storage::metadata::MetadataViewerCache, }; -use ahash::{AHashMap, AHashSet}; +use ahash::AHashMap; use arc_swap::ArcSwap; use auth::oauth::config::OAuthConfig; use calcard::common::timezone::Tz; +use compact_str::CompactString; use config::{ groupware::GroupwareConfig, mailstore::jmap::JmapConfig, + metadata::MetadataConfig, network::Network, smtp::{ SmtpConfig, - resolver::{Policy, Tlsa}, + resolver::{MtaStsEntry, Tlsa}, }, storage::Storage, telemetry::Metrics, }; -use ipc::{BroadcastEvent, PushEvent, QueueEvent, ReportingEvent}; -use mail_auth::{MX, RecordSet, Txt}; -use manager::application::Resource; +use ipc::{BroadcastEvent, PushEvent, QueueEvent, ReportingEvent, subscribers::PushSubscribers}; +use mail_auth::{Mx, Negative, RecordSet, TxtRecord}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; use std::sync::atomic::AtomicU64; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, - sync::{Arc, atomic::AtomicBool}, - time::{Duration, Instant}, + sync::{Arc, OnceLock, Weak, atomic::AtomicBool}, + time::Duration, }; use store::InMemoryStore; use tinyvec::TinyVec; -use tokio::sync::{Notify, Semaphore, mpsc}; +use tokio::sync::{Notify, OnceCell, Semaphore, mpsc}; use tokio_rustls::TlsConnector; -use types::{acl::AclGrant, special_use::SpecialUse}; +use types::{acl::AclGrant, metadata::MetadataKinds, special_use::SpecialUse}; use utils::{ - cache::{Cache, CacheWithTtl}, + cache::{Cache, CacheWithTtl, SpillCache}, snowflake::SnowflakeIdGenerator, }; + pub mod auth; pub mod cache; pub mod config; @@ -92,10 +102,17 @@ Schema history: 4 - v0.14.0 5 - v0.15.0 6 - v0.16.0 +7 - v1.0.0 */ -pub const DATABASE_SCHEMA_VERSION: u32 = 6; +pub const DATABASE_SCHEMA_VERSION_BASE: u32 = 7; +pub const DATABASE_SCHEMA_OSS_FLAG: u32 = 1 << 31; + +#[cfg(feature = "enterprise")] +pub const DATABASE_SCHEMA_VERSION: u32 = DATABASE_SCHEMA_VERSION_BASE; +#[cfg(not(feature = "enterprise"))] +pub const DATABASE_SCHEMA_VERSION: u32 = DATABASE_SCHEMA_VERSION_BASE | DATABASE_SCHEMA_OSS_FLAG; pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); pub const LONG_1Y_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24 * 365); @@ -114,11 +131,14 @@ pub const KV_RATE_LIMIT_HTTP_AUTHENTICATED: u8 = 8; pub const KV_RATE_LIMIT_HTTP_ANONYMOUS: u8 = 9; pub const KV_RATE_LIMIT_IMAP: u8 = 10; pub const KV_QUOTA_BLOB: u8 = 11; -pub const KV_GREYLIST: u8 = 16; -pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21; -pub const KV_LOCK_TASK: u8 = 23; -pub const KV_LOCK_DAV: u8 = 25; -pub const KV_SIEVE_ID: u8 = 26; +pub const KV_RATE_LIMIT_AVAILABILITY: u8 = 12; +pub const KV_GREYLIST: u8 = 13; +pub const KV_LOCK_QUEUE_MESSAGE: u8 = 14; +pub const KV_LOCK_TASK: u8 = 15; +pub const KV_LOCK_DAV: u8 = 16; +pub const KV_LOCK_SEARCH_INDEX: u8 = 17; +pub const KV_OAUTH_CIMD: u8 = 18; +pub const KV_RATE_LIMIT_CIMD: u8 = 19; #[derive(Clone)] pub struct Server { @@ -144,6 +164,7 @@ pub struct Data { pub lookup_stores: ArcSwap, InMemoryStore>>, pub asn_geo_data: AsnGeoLookupData, + pub password_verifier: directory::core::verifier::PasswordVerifier, pub jmap_id_gen: SnowflakeIdGenerator, pub queue_id_gen: SnowflakeIdGenerator, @@ -155,46 +176,54 @@ pub struct Data { pub logos: Mutex, LogoCache>>, pub smtp_connectors: TlsConnectors, + pub memory_budget: MemoryBudget, } #[derive(Clone)] pub struct LogoCache { domain_id: u32, - tenant_id: Option, - data: Option>>, } pub struct Caches { - pub access_tokens: Cache>, - pub http_auth: Cache, HttpAuthCache>, + pub swap: crate::cache::swap::SwapTier, + pub revalidate: crate::cache::RevalidateInterval, - pub messages: Cache>, - pub files: Cache>, - pub contacts: Cache>, - pub events: Cache>, - pub scheduling: Cache>, + pub access_tokens: SpillCache>, + pub http_auth: HttpAuthEntries, + pub grant_revocations: GrantRevocations, + + pub messages: Cache, SwapEviction>, + pub messages_large: LargeMessageCaches, + pub resources_large: LargeResourceCaches, + pub files: Cache, SwapEviction>, + pub contacts: Cache, SwapEviction>, + pub events: Cache, SwapEviction>, + pub scheduling: Cache, SwapEviction>, + pub metadata_viewers: MetadataViewerCache, pub emails: Cache, pub emails_negative: CacheWithTtl, - pub domain_names: Cache, u32>, - pub domain_names_negative: CacheWithTtl, ()>, + pub domain_names: Cache, + pub domain_names_negative: CacheWithTtl, pub domains: Cache>, pub accounts: Cache>, pub roles: Cache>, - pub tenants: Cache>, pub lists: Cache>, pub dkim_signers: Cache>, - pub dns_txt: CacheWithTtl, Txt>, - pub dns_mx: CacheWithTtl, RecordSet>, - pub dns_ptr: CacheWithTtl>>, - pub dns_ipv4: CacheWithTtl, RecordSet>, - pub dns_ipv6: CacheWithTtl, RecordSet>, - pub dns_tlsa: CacheWithTtl, Arc>, - pub dns_mta_sts: CacheWithTtl, Arc>, - pub dns_rbl: CacheWithTtl, Option>>, + pub dns_txt: CacheWithTtl, TxtRecord>, + pub dns_mx: CacheWithTtl, Result, Negative>>, + pub dns_ptr: CacheWithTtl>, Negative>>, + pub dns_ipv4: CacheWithTtl, Result, Negative>>, + pub dns_ipv6: CacheWithTtl, Result, Negative>>, + pub dns_tlsa: CacheWithTtl, Option>>, + pub dns_mta_sts: CacheWithTtl, MtaStsEntry>, + pub spam_lookup: CacheWithTtl, SpamLookup>, + + pub directory_recipients: CacheWithTtl, ()>, + pub directory_recipients_ttl: Duration, pub negative_cache_ttl: Duration, } @@ -203,9 +232,49 @@ pub struct Caches { pub struct MessageStoreCache { pub emails: Arc, pub mailboxes: Arc, - pub update_lock: Arc, + pub update_lock: Arc, + pub refresh: Arc, pub last_change_id: u64, pub size: u64, + pub verification: Verification, +} + +#[derive(Debug, Default)] +pub struct RefreshState { + pub deltas: Box<[ItemDelta]>, + pub uid_next: UidNextMemo, + pub mailbox_rows: Arc, +} + +#[derive(Debug, Clone)] +pub struct ItemDelta { + pub from_change_id: u64, + pub to_change_id: u64, + pub document_ids: Arc<[u32]>, +} + +#[derive(Debug, Default)] +pub struct UidNextMemo { + values: Mutex>)>>, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct MailboxRow { + pub uid: u32, + pub id: u32, +} + +#[derive(Debug, Default)] +pub struct MailboxRowsMemo { + entries: Mutex>, +} + +#[derive(Debug)] +struct SharedMailboxRows { + mailbox_id: u32, + emails: Weak, + rows: Arc>, } #[derive(Debug, Clone)] @@ -216,27 +285,56 @@ pub struct MailboxesCache { pub size: u64, } +pub const CACHE_CHUNK: usize = 16384; + +pub const MAX_RECEIVED_AT: u64 = u32::MAX as u64; + +#[derive(Debug, Clone)] +pub struct ColBlock { + pub(crate) document_ids: Arc<[u32]>, + pub(crate) change_ids: Arc<[u64]>, + pub(crate) received_at: Arc<[u32]>, + pub(crate) sent_at: Arc<[i32]>, + pub(crate) sizes: Arc<[u32]>, + pub(crate) keywords: Arc<[u32]>, + pub(crate) thread_ids: Arc<[u32]>, + pub(crate) thread_range: ThreadRange, + pub(crate) mb_offsets: Arc<[u32]>, + pub(crate) mb_arena: Arc<[MessageUid]>, +} + #[derive(Debug, Clone)] pub struct MessagesCache { pub change_id: u64, - pub items: Box<[MessageCache]>, - pub index: AHashMap, - pub keywords: Box<[Box]>, + pub(crate) blocks: Vec>, + pub(crate) starts: Vec, + pub(crate) index: IdIndex, + pub(crate) keywords: Arc<[CustomKeywords]>, + pub(crate) len: usize, pub size: u64, } #[derive(Debug, Clone)] -pub struct MessageCache { +pub struct CustomKeywords { + pub names: Box<[CompactString]>, pub document_id: u32, - pub mailboxes: TinyVec<[MessageUidCache; 2]>, - pub keywords: u128, - pub thread_id: u32, - pub change_id: u64, - pub size: u32, } -#[derive(Debug, Default, Clone, Copy)] -pub struct MessageUidCache { +#[derive(Debug, Clone)] +pub struct MessageCache { + pub(crate) document_id: u32, + pub(crate) mailboxes: TinyVec<[MessageUid; 2]>, + pub(crate) keywords: u32, + pub(crate) thread_id: u32, + pub(crate) change_id: u64, + pub(crate) size: u32, + pub(crate) received_at: u64, + pub(crate) sent_at: i32, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct MessageUid { pub mailbox_id: u32, pub uid: u32, } @@ -244,33 +342,47 @@ pub struct MessageUidCache { #[derive(Debug, Clone)] pub struct MailboxCache { pub document_id: u32, - pub name: String, - pub path: String, + pub name: Arc, + pub path: Arc, pub role: SpecialUse, pub parent_id: u32, pub sort_order: u32, pub subscribers: TinyVec<[u32; 4]>, pub uid_validity: u32, pub acls: TinyVec<[AclGrant; 2]>, + pub metadata_kinds: MetadataKinds, } +const _: () = assert!(size_of::() <= 136); + #[derive(Debug, Clone)] pub struct HttpAuthCache { pub account_id: u32, - pub revision: u64, + pub expires: u32, pub credential_id: Option, - pub expires: Instant, + pub revision: u64, + pub token_scope: Option, } +const _: () = assert!(size_of::() <= 40); + pub struct Ipc { pub push_tx: mpsc::Sender, - pub task_tx: Arc, + pub push_subscribers: Arc, + pub task_tx: ipc::TaskQueueNotify, + pub index_tx: Arc, pub queue_tx: mpsc::Sender, pub report_tx: mpsc::Sender, pub broadcast_tx: Option>, pub train_task_controller: Arc, } +impl Ipc { + pub fn index_queue_notify(&self) -> ipc::IndexQueueNotify { + ipc::IndexQueueNotify::new(self.index_tx.clone(), self.broadcast_tx.clone()) + } +} + pub struct TlsConnectors { pub pki_verify: TlsConnector, pub dummy_verify: TlsConnector, @@ -278,75 +390,185 @@ pub struct TlsConnectors { pub struct NameWrapper(pub String); +#[derive(Debug)] +pub struct UpdateLock { + pub semaphore: Semaphore, + pub revision: AtomicU64, + pub epoch: AtomicU64, +} + +#[derive(Debug, Default)] +pub struct MessageUpdateLock { + is_updating: AtomicBool, + released: Notify, + epoch: AtomicU64, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct Verification { + verified_at: u64, + epoch: u64, +} + +pub const DAV_CHUNK: usize = 4096; +pub const NO_ID: u32 = u32::MAX; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ArenaRef { + pub off: u32, + pub len: u32, +} + +impl ArenaRef { + #[inline(always)] + pub fn range(&self) -> std::ops::Range { + self.off as usize..(self.off as usize + self.len as usize) + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + #[derive(Debug, Clone)] -pub struct DavResources { +pub struct GroupwareResources { pub base_path: String, - pub paths: AHashSet, - pub resources: Vec, + pub paths: Arc, + pub resources: ResourceStore, pub item_change_id: u64, pub container_change_id: u64, pub highest_change_id: u64, pub size: u64, pub update_lock: Arc, + pub verification: Verification, } -#[derive(Debug)] -pub struct UpdateLock { - pub semaphore: Semaphore, - pub revision: AtomicU64, +#[derive(Debug, Clone, Default)] +pub struct ResourceStore { + pub chunks: Vec>, + pub containers_end: usize, + pub total: usize, + pub unified_id_space: bool, } -#[derive(Debug, Clone)] +#[derive(Debug, Default)] +pub struct ResourceChunk { + pub records: Box<[GroupwareResource]>, + pub bytes: Box<[u8]>, + pub names: Box<[CachedName]>, + pub acls: Box<[AclGrant]>, + pub prefs: Box<[TinyCalendarPreferences]>, + pub principals: Box<[u32]>, + pub uid_index: OnceLock>, + pub min_id: u32, + pub max_id: u32, +} + +#[derive(Debug, Clone, Default)] +pub struct PathIndex { + pub chunks: Vec>, + pub total: usize, +} + +#[derive(Debug, Default)] +pub struct PathChunk { + pub paths: Box<[DavPath]>, + pub bytes: Box<[u8]>, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CachedName { + pub name: ArenaRef, + pub parent_id: u32, +} + +#[derive(Debug, Clone, Copy, Default)] pub struct DavPath { - pub path: String, - pub parent_id: Option, + pub path: ArenaRef, + pub parent_id: u32, pub hierarchy_seq: u32, - pub resource_idx: usize, + pub document_id: u32, } -#[derive(Debug, Clone)] -pub struct DavResource { +#[derive(Debug, Clone, Copy)] +pub struct GroupwareResource { pub document_id: u32, - pub data: DavResourceMetadata, + pub data: GroupwareResourceMetadata, +} + +#[derive(Debug, Clone, Copy)] +pub struct GroupwareResourceRef<'x> { + pub chunk: &'x ResourceChunk, + pub resource: &'x GroupwareResource, } #[derive(Debug, Clone, Copy)] pub struct DavResourcePath<'x> { pub path: &'x DavPath, - pub resource: &'x DavResource, + pub path_chunk: &'x PathChunk, + pub resource: GroupwareResourceRef<'x>, } -#[derive(Debug, Clone)] -pub enum DavResourceMetadata { +#[derive(Debug, Clone, Copy)] +pub enum GroupwareResourceMetadata { File { - name: String, - size: Option, - parent_id: Option, - acls: TinyVec<[AclGrant; 2]>, + name: ArenaRef, + size: u32, + parent_id: u32, + acls: ArenaRef, + etag: u32, + modified: i64, + created_delta: i32, + flags: FileFlags, }, Calendar { - name: String, - acls: TinyVec<[AclGrant; 2]>, - preferences: TinyVec<[TinyCalendarPreferences; 2]>, + name: ArenaRef, + acls: ArenaRef, + preferences: ArenaRef, + etag: u32, + metadata: MetadataKinds, }, CalendarEvent { - names: TinyVec<[DavName; 2]>, + names: ArenaRef, start: i64, duration: u32, + created_at: i64, + modified_at: i32, + uid: ArenaRef, + etag: u32, + flags: u16, }, CalendarEventNotification { - names: TinyVec<[DavName; 2]>, + names: ArenaRef, + created_at: i64, + event_id: u32, + etag: u32, + changed_by: u32, + principals: ArenaRef, + calendar_ids_len: u16, + flags: u16, }, AddressBook { - name: String, - acls: TinyVec<[AclGrant; 2]>, + name: ArenaRef, + acls: ArenaRef, + etag: u32, + metadata: MetadataKinds, }, ContactCard { - names: TinyVec<[DavName; 2]>, + names: ArenaRef, + created_at: i64, + modified_at: i32, + uid: ArenaRef, + etag: u32, + flags: u16, }, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FileFlags(pub u32); + +#[derive(Debug, Clone, Copy, Default)] pub struct TinyCalendarPreferences { pub account_id: u32, pub tz: Tz, @@ -374,6 +596,7 @@ pub struct Core { pub smtp: SmtpConfig, pub spam: SpamFilterConfig, pub groupware: GroupwareConfig, + pub metadata: MetadataConfig, pub metrics: Metrics, } @@ -407,13 +630,15 @@ pub struct ThrottleKey { pub hash: [u8; 32], } -#[derive(Default)] -pub struct ThrottleKeyHasher { - hash: u64, -} +impl MessageUid { + pub fn new(mailbox_id: u32, uid: u32) -> Self { + MessageUid { mailbox_id, uid } + } -#[derive(Clone, Default)] -pub struct ThrottleKeyHasherBuilder {} + pub fn new_unassigned(mailbox_id: u32) -> Self { + MessageUid { mailbox_id, uid: 0 } + } +} pub const DEFAULT_LOGO_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAMgAAAAnCAMAAAB9lPf7AAABOFBMVEUAAAAAADoAAEkPDkIPDkIQ\r\n\ DkIQDkLcLVTYMVTbLVTbLVQPDkLWM2YQDkIPD0IODEHaJlPbLVQWDT8MC0IQDkIQDkIPDkIODkEQ\r\n\ diff --git a/crates/common/src/manager/app_client.rs b/crates/common/src/manager/app_client.rs new file mode 100644 index 00000000..7d067549 --- /dev/null +++ b/crates/common/src/manager/app_client.rs @@ -0,0 +1,411 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::auth::oauth::{ + cimd::{CIMD_MAX_DOCUMENT_SIZE, is_dot_segment}, + client::ApplicationClient, + registration::{check_client_name, is_https_url, is_safe_redirect_target}, +}; +use serde::{Deserialize, Serialize, de::IgnoredAny}; + +pub const APP_CLIENT_DOCUMENT: &str = "oauth-client.json"; +const MAX_AUTHORITY_LEN: usize = 261; +const MAX_FIELD_LEN: usize = 512; +const MAX_REDIRECT_URIS: usize = 16; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppClientTemplate { + client_name: Option>, + client_uri: Option, + logo_uri: Option, + scope: Option>, + software_id: Option>, + software_version: Option>, + redirect_uris: Box<[Box]>, + post_logout_redirect_uris: Box<[Box]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum AppUri { + Relative(Box), + Absolute(Box), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AppClientId<'x> { + pub is_tls: bool, + pub authority: &'x str, + pub prefix: &'x str, +} + +#[derive(Deserialize)] +struct RawTemplate { + #[serde(default)] + client_name: Option, + #[serde(default)] + client_uri: Option, + #[serde(default)] + logo_uri: Option, + #[serde(default)] + scope: Option, + #[serde(default)] + software_id: Option, + #[serde(default)] + software_version: Option, + #[serde(default)] + redirect_uris: Vec, + #[serde(default)] + post_logout_redirect_uris: Vec, + #[serde(default)] + client_id: Option, + #[serde(default)] + client_secret: Option, + #[serde(default)] + jwks: Option, + #[serde(default)] + jwks_uri: Option, +} + +#[derive(Serialize)] +struct ClientDocument<'x> { + client_id: &'x str, + #[serde(skip_serializing_if = "Option::is_none")] + client_name: Option<&'x str>, + #[serde(skip_serializing_if = "Option::is_none")] + client_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + logo_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + scope: Option<&'x str>, + #[serde(skip_serializing_if = "Option::is_none")] + software_id: Option<&'x str>, + #[serde(skip_serializing_if = "Option::is_none")] + software_version: Option<&'x str>, + redirect_uris: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + post_logout_redirect_uris: Vec, + token_endpoint_auth_method: &'static str, + grant_types: [&'static str; 2], + response_types: [&'static str; 1], +} + +impl AppClientTemplate { + pub fn parse( + bytes: &[u8], + max_prefix_len: usize, + has_file: impl Fn(&str) -> bool, + ) -> Result { + let raw = serde_json::from_slice::(bytes) + .map_err(|_| "oauth-client.json is not a valid JSON object")?; + + if raw.client_id.is_some() { + return Err("oauth-client.json must not contain client_id"); + } else if raw.client_secret.is_some() { + return Err("oauth-client.json must not contain client_secret"); + } else if raw.jwks.is_some() || raw.jwks_uri.is_some() { + return Err("oauth-client.json must not contain jwks or jwks_uri"); + } else if raw.redirect_uris.is_empty() || raw.redirect_uris.len() > MAX_REDIRECT_URIS { + return Err("oauth-client.json must list between 1 and 16 redirect_uris"); + } else if raw.post_logout_redirect_uris.len() > MAX_REDIRECT_URIS { + return Err("oauth-client.json lists too many post_logout_redirect_uris"); + } + + let relative_list = |uris: Vec| { + uris.into_iter() + .map(|uri| { + if is_relative_path(&uri) { + Ok(uri.into_boxed_str()) + } else { + Err("oauth-client.json redirect URIs must be relative paths") + } + }) + .collect::, _>>() + }; + let uri = |uri: Option| -> Result, &'static str> { + match uri { + Some(uri) if is_relative_path(&uri) => { + Ok(Some(AppUri::Relative(uri.into_boxed_str()))) + } + Some(uri) if is_https_url(&uri) && uri.len() <= MAX_FIELD_LEN => { + Ok(Some(AppUri::Absolute(uri.into_boxed_str()))) + } + Some(_) => Err("oauth-client.json URIs must be relative paths or https URLs"), + None => Ok(None), + } + }; + + if let Some(name) = &raw.client_name { + check_client_name(name).map_err(|_| { + "oauth-client.json client_name is too long or contains control or bidirectional formatting characters" + })?; + } + + let template = AppClientTemplate { + client_name: text(raw.client_name)?, + client_uri: uri(raw.client_uri)?, + logo_uri: uri(raw.logo_uri)?, + scope: text(raw.scope)?, + software_id: text(raw.software_id)?, + software_version: text(raw.software_version)?, + redirect_uris: relative_list(raw.redirect_uris)?, + post_logout_redirect_uris: relative_list(raw.post_logout_redirect_uris)?, + }; + + if let Some(AppUri::Relative(logo)) = &template.logo_uri + && !has_file(path_of(logo)) + { + return Err("oauth-client.json logo_uri does not exist in the bundle"); + } + + let base = format!( + "https://{}/{}/", + "a".repeat(MAX_AUTHORITY_LEN), + "a".repeat(max_prefix_len) + ); + if template.document(&base).len() > CIMD_MAX_DOCUMENT_SIZE { + return Err("oauth-client.json exceeds the 5 KB limit"); + } + + Ok(template) + } + + pub fn client(&self, base: &str) -> ApplicationClient { + ApplicationClient { + client_name: self.client_name.as_deref().map(str::to_string), + logo_uri: match &self.logo_uri { + Some(AppUri::Relative(logo)) => Some(resolve(base, logo)), + _ => None, + }, + redirect_uris: self + .redirect_uris + .iter() + .map(|uri| resolve(base, uri)) + .collect(), + } + } + + pub fn post_logout_redirect_uri_matches(&self, client_id: &AppClientId<'_>, uri: &str) -> bool { + is_safe_redirect_target(uri) + && client_id.strip_base(uri).is_some_and(|path| { + self.post_logout_redirect_uris + .iter() + .any(|registered| registered.as_ref() == path) + }) + } + + pub fn document(&self, base: &str) -> Vec { + let client_id = resolve(base, APP_CLIENT_DOCUMENT); + serde_json::to_vec(&ClientDocument { + client_id: &client_id, + client_name: self.client_name.as_deref(), + client_uri: self.client_uri.as_ref().map(|uri| uri.resolve(base)), + logo_uri: self.logo_uri.as_ref().map(|uri| uri.resolve(base)), + scope: self.scope.as_deref(), + software_id: self.software_id.as_deref(), + software_version: self.software_version.as_deref(), + redirect_uris: self + .redirect_uris + .iter() + .map(|uri| resolve(base, uri)) + .collect(), + post_logout_redirect_uris: self + .post_logout_redirect_uris + .iter() + .map(|uri| resolve(base, uri)) + .collect(), + token_endpoint_auth_method: "none", + grant_types: ["authorization_code", "refresh_token"], + response_types: ["code"], + }) + .unwrap_or_default() + } +} + +impl AppUri { + fn resolve(&self, base: &str) -> String { + match self { + AppUri::Relative(path) => resolve(base, path), + AppUri::Absolute(uri) => uri.to_string(), + } + } +} + +impl<'x> AppClientId<'x> { + pub fn parse(client_id: &'x str) -> Option { + let (is_tls, rest) = if let Some(rest) = client_id.strip_prefix("https://") { + (true, rest) + } else { + (false, client_id.strip_prefix("http://")?) + }; + let (authority, path) = rest.split_once('/')?; + let prefix = match path.strip_suffix(APP_CLIENT_DOCUMENT)? { + #[cfg(feature = "dev_mode")] + "" => "", + prefix => { + let prefix = prefix.strip_suffix('/')?; + if !is_url_prefix(prefix) { + return None; + } + prefix + } + }; + + if !authority.is_empty() && authority.bytes().all(is_authority_byte) { + Some(AppClientId { + is_tls, + authority, + prefix, + }) + } else { + None + } + } + + pub fn base(&self) -> String { + app_base(self.is_tls, self.authority, self.prefix) + } + + fn strip_base<'u>(&self, uri: &'u str) -> Option<&'u str> { + let scheme = if self.is_tls { "https://" } else { "http://" }; + let rest = uri + .strip_prefix(scheme)? + .strip_prefix(self.authority)? + .strip_prefix('/')?; + if self.prefix.is_empty() { + Some(rest) + } else { + rest.strip_prefix(self.prefix)?.strip_prefix('/') + } + } +} + +pub fn app_base(is_tls: bool, authority: &str, prefix: &str) -> String { + let scheme = if is_tls { "https" } else { "http" }; + let mut base = String::with_capacity(scheme.len() + authority.len() + prefix.len() + 5); + base.push_str(scheme); + base.push_str("://"); + base.push_str(authority); + base.push('/'); + if !prefix.is_empty() { + base.push_str(prefix); + base.push('/'); + } + base +} + +pub fn normalize_url_prefix(prefix: &str) -> &str { + prefix.trim_matches('/') +} + +pub fn validate_url_prefix(prefix: &str) -> Result<(), &'static str> { + let prefix = normalize_url_prefix(prefix); + if !is_url_prefix(prefix) { + Err(concat!( + "URL prefixes must be a single non-empty path segment such as \"/admin\", ", + "made of letters, digits, '-', '.', '_' or '~'" + )) + } else if is_reserved_url_prefix(prefix) { + Err("This URL prefix is reserved for a built-in server endpoint") + } else { + Ok(()) + } +} + +fn is_reserved_url_prefix(prefix: &str) -> bool { + hashify::set_ignore_case!( + prefix.as_bytes(), + "jmap", + "dav", + ".well-known", + "auth", + "scim", + "api", + "mail", + "calendar", + "autodiscover", + "robots.txt", + "healthz", + "metrics", + "logo", + "form", + "login", + "device", + ) +} + +fn is_url_prefix(prefix: &str) -> bool { + !prefix.is_empty() && prefix.bytes().all(is_prefix_byte) && prefix != "." && prefix != ".." +} + +fn resolve(base: &str, path: &str) -> String { + let mut uri = String::with_capacity(base.len() + path.len()); + uri.push_str(base); + uri.push_str(path); + uri +} + +fn text(value: Option) -> Result>, &'static str> { + match value { + Some(value) if value.len() > MAX_FIELD_LEN => { + Err("oauth-client.json contains a value that is too long") + } + Some(value) if !value.trim().is_empty() => Ok(Some(value.into_boxed_str())), + _ => Ok(None), + } +} + +fn path_of(uri: &str) -> &str { + uri.split_once('?').map_or(uri, |(path, _)| path) +} + +fn is_relative_path(uri: &str) -> bool { + !uri.is_empty() + && uri.len() <= MAX_FIELD_LEN + && !uri.starts_with('/') + && !uri.contains("..") + && uri.bytes().all(is_relative_byte) + && !path_of(uri).split('/').any(is_dot_segment) +} + +pub fn is_safe_authority(authority: &str) -> bool { + !authority.is_empty() + && authority.len() <= MAX_AUTHORITY_LEN + && authority.bytes().all(is_authority_byte) +} + +fn is_relative_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() + || matches!( + b, + b'-' | b'.' + | b'_' + | b'~' + | b'/' + | b'?' + | b'=' + | b'&' + | b'%' + | b'+' + | b'!' + | b'$' + | b'\'' + | b'(' + | b')' + | b'*' + | b',' + | b';' + | b'@' + ) +} + +fn is_authority_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b':' | b'[' | b']' | b'_') +} + +fn is_prefix_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~') +} + diff --git a/crates/common/src/manager/application.rs b/crates/common/src/manager/application.rs index 8b825ab9..1322bd19 100644 --- a/crates/common/src/manager/application.rs +++ b/crates/common/src/manager/application.rs @@ -6,14 +6,20 @@ use crate::{ Server, + auth::oauth::client::{AppOriginPolicy, ApplicationClient}, manager::{ - defaults::{WEBUI_LATEST_URL, WEBUI_MAX_MAJOR_VERSION, WEBUI_PINNED_URL}, + app_client::{ + APP_CLIENT_DOCUMENT, AppClientId, AppClientTemplate, app_base, normalize_url_prefix, + }, fetch_resource, }, }; use ahash::AHashMap; use arc_swap::ArcSwap; +use bytes::Bytes; +use compact_str::{CompactString, format_compact}; use registry::schema::{enums::CompressionAlgo, structs::Application}; +use serde::Deserialize; use std::{ borrow::Cow, io::{self, Cursor, Read, Seek}, @@ -26,21 +32,40 @@ use std::{ }; use store::{ registry::{RegistryObject, bootstrap::Bootstrap}, - write::{BatchBuilder, BlobLink, BlobOp, now}, + write::{BatchBuilder, BlobLink, BlobOp, blob::BlobLinkValue, now}, }; use trc::{AddContext, Key}; -use types::{blob_hash::BlobHash, id::Id}; +use types::{blob_hash::BlobHash, id::Id, semver::Semver}; +use zip::{ZipArchive, result::ZipError}; const APP_BLOB_PREFIX: &str = "STALWART_APP_"; const MAX_APP_SIZE: usize = 100 * 1024 * 1024; +const INDEX_PAGE: &str = "index.html"; const BASE_HREF: &str = " { + id: &'x str, + version: &'x str, + #[serde(rename = "minServerVersion")] + min_server_version: Option<&'x str>, +} enum IndexEdit<'x> { BaseHref(&'x str), OAuthClientId(&'x str), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppClientDocument { + NotShipped, + Untrusted, + Document(Vec), +} + #[allow(clippy::type_complexity)] pub struct WebApplications { applications: ArcSwap>, @@ -50,7 +75,8 @@ pub struct WebApplications { pub struct AppRoutes { resources: AHashMap>, - oauth_client_id_meta: Option, + index_pages: AHashMap, + oauth_client: Option, _bundle_dir: TempDir, } @@ -81,7 +107,7 @@ impl Resource { } pub struct AppResource { - pub resource: Resource>, + pub resource: Resource, pub no_cache: bool, } @@ -94,26 +120,136 @@ impl WebApplications { } } + pub fn has_prefix(&self, prefix: &str) -> bool { + self.routes.load().contains_key(prefix) + } + + pub fn is_first_party_client( + &self, + client_id: &str, + policy: Option>, + ) -> bool { + if self + .applications + .load() + .iter() + .any(|app| app.oauth_client_id.as_deref() == Some(client_id)) + { + return true; + } + let (Some(policy), Some(client_id)) = (policy, AppClientId::parse(client_id)) else { + return false; + }; + if !policy.accepts(client_id.is_tls, client_id.authority) { + return false; + } + let routes = self.routes.load(); + + #[cfg(feature = "dev_mode")] + if client_id.prefix.is_empty() { + return routes.values().any(|app| app.oauth_client.is_some()); + } + + routes + .get(client_id.prefix) + .is_some_and(|app| app.oauth_client.is_some()) + } + + pub fn recognize_client( + &self, + client_id: &str, + policy: AppOriginPolicy<'_>, + ) -> Option { + self.with_client_template(client_id, policy, |template, client_id| { + template.client(&client_id.base()) + }) + } + + pub fn post_logout_redirect_uri_matches( + &self, + client_id: &str, + policy: AppOriginPolicy<'_>, + uri: &str, + ) -> Option { + self.with_client_template(client_id, policy, |template, client_id| { + template.post_logout_redirect_uri_matches(client_id, uri) + }) + } + + fn with_client_template( + &self, + client_id: &str, + policy: AppOriginPolicy<'_>, + f: impl FnOnce(&AppClientTemplate, &AppClientId<'_>) -> T, + ) -> Option { + let client_id = AppClientId::parse(client_id)?; + if !policy.accepts(client_id.is_tls, client_id.authority) { + return None; + } + let routes = self.routes.load(); + + #[cfg(feature = "dev_mode")] + if client_id.prefix.is_empty() { + return routes + .iter() + .filter_map(|(prefix, app)| Some((prefix, app.oauth_client.as_ref()?))) + .min_by_key(|(prefix, _)| prefix.as_str()) + .map(|(_, template)| f(template, &client_id)); + } + + routes + .get(client_id.prefix)? + .oauth_client + .as_ref() + .map(|template| f(template, &client_id)) + } + + pub fn client_document( + &self, + prefix: &str, + is_tls: bool, + authority: &str, + policy: AppOriginPolicy<'_>, + ) -> AppClientDocument { + match self + .routes + .load() + .get(prefix) + .and_then(|routes| routes.oauth_client.as_ref()) + { + Some(template) if policy.accepts(is_tls, authority) => { + AppClientDocument::Document(template.document(&app_base(is_tls, authority, prefix))) + } + Some(_) => AppClientDocument::Untrusted, + None => AppClientDocument::NotShipped, + } + } + pub async fn serve(&self, prefix: &str, path: &str) -> trc::Result> { if let Some(routes) = self.routes.load().get(prefix) && let Some((is_index, resource)) = routes .resources .get(path) - .map(|res| (path == "index.html", res)) - .or_else(|| routes.resources.get("index.html").map(|res| (true, res))) + .map(|res| (path == INDEX_PAGE, res)) + .or_else(|| routes.resources.get(INDEX_PAGE).map(|res| (true, res))) { + if is_index && let Some(page) = routes.index_pages.get(prefix) { + return Ok(Some(AppResource { + resource: Resource { + content_type: resource.content_type.clone(), + contents: page.clone(), + }, + no_cache: true, + })); + } + tokio::fs::read(&resource.contents) .await - .map(|mut contents| { - if is_index && let Ok(html) = std::str::from_utf8(&contents) { - contents = - rewrite_index(html, prefix, routes.oauth_client_id_meta.as_deref()); - } - + .map(|contents| { Some(AppResource { resource: Resource { content_type: resource.content_type.clone(), - contents, + contents: contents.into(), }, no_cache: is_index, }) @@ -121,7 +257,7 @@ impl WebApplications { .map_err(|err| { trc::ResourceEvent::Error .reason(err) - .ctx(trc::Key::Path, path.to_string()) + .ctx(trc::Key::Path, CompactString::from(path)) .caused_by(trc::location!()) }) } else { @@ -132,7 +268,7 @@ impl WebApplications { pub async fn reload(&self, bp: &mut Bootstrap) { let mut apps = Vec::new(); for app in bp.list_infallible::().await { - if app.object.enabled { + if app.object.enable { apps.push(WebApplicationManager::new(app)); } } @@ -169,7 +305,7 @@ impl WebApplications { Resource(trc::ResourceEvent::Error), Reason = err, Url = app.url.clone(), - Details = format!( + Details = format_compact!( "Failed to unpack application for prefixes: {}, {}", app.prefixes.join(", "), if is_retained { @@ -203,24 +339,15 @@ impl WebApplicationManager { Self { base_path, blob_key: BlobHash::generate(format!("{}{}", APP_BLOB_PREFIX, app.id.id()).as_bytes()), - url: if app.object.resource_url == WEBUI_LATEST_URL { - WEBUI_PINNED_URL.to_string() - } else { - app.object.resource_url - }, + url: app.object.resource_url, description: app.object.description, - expiry: app.object.auto_update_frequency.as_secs(), + expiry: app.object.auto_update_interval.as_secs(), oauth_client_id: app.object.oauth_client_id, prefixes: app .object .url_prefix .iter() - .map(|prefix| { - prefix - .trim_end_matches('/') - .trim_start_matches('/') - .to_string() - }) + .map(|prefix| normalize_url_prefix(prefix).to_string()) .collect(), } } @@ -263,74 +390,112 @@ impl WebApplicationManager { let url = self.url.clone(); let bundle_path = staging.path.clone(); - let (resources, bundle) = tokio::task::spawn_blocking(move || -> trc::Result<_> { - let mut archive = zip::ZipArchive::new(Cursor::new(bundle)).map_err(|err| { - trc::ResourceEvent::Error - .caused_by(trc::location!()) - .reason(err) - .ctx(Key::Url, url.clone()) - .details("Failed to decompress application bundle") - })?; - if let Some(version) = unsupported_bundle_version(&mut archive) { - return Err(trc::ResourceEvent::Error - .caused_by(trc::location!()) - .ctx(Key::Url, url.clone()) - .details(format!( - "Application bundle version {version} requires Stalwart v1 or later, set the resource URL to {WEBUI_PINNED_URL}" - ))); - } - - let mut resources = AHashMap::with_capacity(archive.len()); - for i in 0..archive.len() { - let mut file = archive.by_index(i).map_err(|err| { + let max_prefix_len = self + .prefixes + .iter() + .map(String::len) + .max() + .unwrap_or_default(); + let (resources, index, oauth_client, bundle) = + tokio::task::spawn_blocking(move || -> trc::Result<_> { + let mut archive = ZipArchive::new(Cursor::new(bundle)).map_err(|err| { trc::ResourceEvent::Error .caused_by(trc::location!()) .reason(err) - .details("Failed to read file from application bundle") + .ctx(Key::Url, url.clone()) + .details("Failed to decompress application bundle") })?; - if file.is_dir() { - continue; - } - - let mut contents = Vec::new(); - file.read_to_end(&mut contents).map_err(unpack_error)?; - let file_name = file.name().to_string(); - drop(file); - - let path = bundle_path.join(format!("{i:02}")); - std::fs::write(&path, contents).map_err(unpack_error)?; - - let resource = Resource { - content_type: match file_name - .rsplit_once('.') - .map(|(_, ext)| ext) - .unwrap_or_default() - { - "html" => "text/html", - "css" => "text/css", - "wasm" => "application/wasm", - "js" => "application/javascript", - "json" => "application/json", - "png" => "image/png", - "svg" => "image/svg+xml", - "ico" => "image/x-icon", - _ => "application/octet-stream", + verify_manifest(&mut archive, &url)?; + let mut resources = AHashMap::with_capacity(archive.len()); + let mut index = None; + let mut client_document = None; + for i in 0..archive.len() { + let mut file = archive.by_index(i).map_err(|err| { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .reason(err) + .details("Failed to read file from application bundle") + })?; + if file.is_dir() { + continue; } - .into(), - contents: path, - }; - resources.insert(file_name, resource); - } - Ok((resources, archive.into_inner().into_inner())) - }) - .await - .map_err(|err| { - trc::ResourceEvent::Error - .caused_by(trc::location!()) - .reason(err) - .details("Bundle unpack task panicked") - })??; + let mut contents = Vec::new(); + file.read_to_end(&mut contents).map_err(unpack_error)?; + let file_name = file + .name() + .map_err(|err| { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .reason(err) + .details("Failed to read file name from application bundle") + })? + .into_owned(); + drop(file); + + if file_name == APP_CLIENT_DOCUMENT { + client_document = Some(contents); + continue; + } + if file_name == APP_MANIFEST { + continue; + } + + let path = bundle_path.join(format!("{i:02}")); + std::fs::write(&path, &contents).map_err(unpack_error)?; + if file_name == INDEX_PAGE { + index = Some(contents); + } + + let resource = Resource { + content_type: match file_name + .rsplit_once('.') + .map(|(_, ext)| ext) + .unwrap_or_default() + { + "html" => "text/html", + "css" => "text/css", + "wasm" => "application/wasm", + "js" => "application/javascript", + "json" => "application/json", + "png" => "image/png", + "svg" => "image/svg+xml", + "ico" => "image/x-icon", + _ => "application/octet-stream", + } + .into(), + contents: path, + }; + + resources.insert(file_name, resource); + } + let oauth_client = client_document + .map(|document| { + AppClientTemplate::parse(&document, max_prefix_len, |path| { + resources.contains_key(path) + }) + .map_err(|err| { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .ctx(Key::Url, url.clone()) + .details(err) + }) + }) + .transpose()?; + Ok(( + resources, + index, + oauth_client, + archive.into_inner().into_inner(), + )) + }) + .await + .map_err(|err| { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .reason(err) + .details("Bundle unpack task panicked") + })??; if !is_cached && let Err(err) = self.cache(server, &bundle).await { trc::event!( @@ -348,26 +513,35 @@ impl WebApplicationManager { trc::event!( Resource(trc::ResourceEvent::ApplicationUnpacked), Url = self.url.clone(), - Path = staging.path.to_string_lossy().into_owned(), + Path = CompactString::from(staging.path.to_string_lossy()), ); Ok(AppRoutes { resources, - oauth_client_id_meta: self.oauth_client_id.as_deref().map(oauth_client_id_meta), + index_pages: index + .map(|index| index_pages(&index, &self.prefixes, self.oauth_client_id.as_deref())) + .unwrap_or_default(), + oauth_client, _bundle_dir: staging, }) } async fn fetch(&self) -> trc::Result> { - fetch_resource(&self.url, None, Duration::from_secs(60), MAX_APP_SIZE) - .await - .map_err(|err| { - trc::ResourceEvent::Error - .caused_by(trc::location!()) - .ctx(Key::Url, self.url.clone()) - .reason(err) - .details("Failed to fetch application bundle") - }) + fetch_resource( + &self.url, + None, + Duration::from_secs(60), + MAX_APP_SIZE, + false, + ) + .await + .map_err(|err| { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .ctx(Key::Url, self.url.clone()) + .reason(err) + .details("Failed to fetch application bundle") + }) } async fn cache(&self, server: &Server, bundle: &[u8]) -> trc::Result<()> { @@ -386,7 +560,7 @@ impl WebApplicationManager { until: now() + self.expiry, }, }, - vec![], + BlobLinkValue::new(bundle.len()), ) .set( BlobOp::Commit { @@ -396,7 +570,7 @@ impl WebApplicationManager { ); server .store() - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; @@ -445,7 +619,14 @@ impl TempDir { impl Drop for TempDir { fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.path); + let path = std::mem::take(&mut self.path); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn_blocking(move || { + let _ = std::fs::remove_dir_all(path); + }); + } else { + let _ = std::fs::remove_dir_all(path); + } } } @@ -468,6 +649,81 @@ async fn remove_siblings(base_path: &Path, keep: &Path) { } } +fn verify_manifest(archive: &mut ZipArchive, url: &str) -> trc::Result<()> { + let manifest_error = || { + trc::ResourceEvent::Error + .caused_by(trc::location!()) + .ctx(Key::Url, url.to_string()) + }; + + let file = archive.by_name(APP_MANIFEST).map_err(|err| match err { + ZipError::FileNotFound => { + manifest_error().details("Application bundle does not contain a manifest") + } + err => manifest_error() + .reason(err) + .details("Failed to read application manifest"), + })?; + if file.size() > MAX_APP_MANIFEST_SIZE { + return Err(manifest_error().details("Application manifest is too large")); + } + let mut contents = Vec::new(); + file.take(MAX_APP_MANIFEST_SIZE + 1) + .read_to_end(&mut contents) + .map_err(|err| { + manifest_error() + .reason(err) + .details("Failed to read application manifest") + })?; + if contents.len() as u64 > MAX_APP_MANIFEST_SIZE { + return Err(manifest_error().details("Application manifest is too large")); + } + + let manifest = serde_json::from_slice::>(&contents).map_err(|err| { + manifest_error() + .reason(err) + .details("Invalid application manifest") + })?; + let manifest_error = || { + manifest_error() + .ctx(Key::Id, manifest.id.to_string()) + .ctx(Key::Version, manifest.version.to_string()) + }; + let semver = parse_version(manifest.version) + .ok_or_else(|| manifest_error().details("Invalid application manifest version"))?; + + if let Some(min_server_version) = manifest.min_server_version { + let min_server_semver = parse_version(min_server_version).ok_or_else(|| { + manifest_error().details("Invalid application manifest minimum server version") + })?; + let server_semver = Semver::current(); + if min_server_semver > server_semver { + return Err(manifest_error().details(format_compact!( + "Incompatible application bundle: {} version {} requires Stalwart {min_server_semver} or later, running {server_semver}", + manifest.id, + manifest.version + ))); + } + } + + match hashify::map!(manifest.id.as_bytes(), u16, "webui" => 2) { + Some(&major) if semver.major() != major => Err(manifest_error().details(format_compact!( + "Incompatible application bundle: {} version {} found, major version {major} is required", + manifest.id, + manifest.version + ))), + _ => Ok(()), + } +} + +fn parse_version(version: &str) -> Option { + let version = version.trim(); + let version = version + .split_once(['-', '+']) + .map_or(version, |(version, _)| version); + Semver::try_from(version).ok() +} + fn unpack_error(err: std::io::Error) -> trc::Error { trc::ResourceEvent::Error .reason(err) @@ -480,6 +736,25 @@ impl Default for WebApplications { } } +fn index_pages( + index: &[u8], + prefixes: &[String], + oauth_client_id: Option<&str>, +) -> AHashMap { + let client_id_meta = oauth_client_id.map(oauth_client_id_meta); + let html = std::str::from_utf8(index).ok(); + prefixes + .iter() + .map(|prefix| { + let page = match html { + Some(html) => rewrite_index(html, prefix, client_id_meta.as_deref()), + None => index.to_vec(), + }; + (prefix.clone(), Bytes::from(page)) + }) + .collect() +} + fn rewrite_index(html: &str, prefix: &str, oauth_client_id_meta: Option<&str>) -> Vec { let mut edits = [ html.find(BASE_HREF) @@ -515,19 +790,6 @@ fn rewrite_index(html: &str, prefix: &str, oauth_client_id_meta: Option<&str>) - out.into_bytes() } -fn unsupported_bundle_version(archive: &mut zip::ZipArchive) -> Option { - let file = archive.by_name("VERSION").ok()?; - let mut version = String::new(); - let is_supported = file.take(32).read_to_string(&mut version).is_ok() - && version - .trim() - .split('.') - .next() - .and_then(|major| major.parse::().ok()) - .is_some_and(|major| major <= WEBUI_MAX_MAJOR_VERSION); - (!is_supported).then(|| version.trim().to_string()) -} - fn oauth_client_id_meta(client_id: &str) -> String { let mut meta = String::with_capacity(OAUTH_CLIENT_ID.len() + client_id.len()); meta.push_str(" String { meta } -#[cfg(test)] -mod tests { - use super::*; - - const INDEX: &str = concat!( - "\n\n\n\n \n", - " \n \n", - " Portal\n\n\n\n\n\n" - ); - - #[test] - fn index_is_rewritten_with_the_prefix_and_client_id() { - let meta = oauth_client_id_meta("inbuxa-webui"); - let html = String::from_utf8(rewrite_index(INDEX, "admin", Some(&meta))).unwrap(); - - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - assert!(html.contains("Portal"), "{html}"); - assert!(html.starts_with(""), "{html}"); - assert!(html.ends_with("\n"), "{html}"); - } - - #[test] - fn index_keeps_the_empty_placeholder_when_no_client_id_is_configured() { - let html = String::from_utf8(rewrite_index(INDEX, "account", None)).unwrap(); - - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - } - - #[test] - fn index_without_a_placeholder_is_left_alone() { - let bundle = "\n \n"; - let meta = oauth_client_id_meta("inbuxa-webui"); - let html = String::from_utf8(rewrite_index(bundle, "admin", Some(&meta))).unwrap(); - - assert_eq!(html, "\n \n"); - } - - #[test] - fn edits_are_applied_in_document_order() { - let bundle = concat!( - "", - "" - ); - let meta = oauth_client_id_meta("app"); - let html = String::from_utf8(rewrite_index(bundle, "admin", Some(&meta))).unwrap(); - - assert_eq!( - html, - concat!( - "", - "" - ) - ); - } - - #[test] - fn client_ids_are_escaped_for_the_attribute() { - let meta = oauth_client_id_meta("a\"b&c"); - - assert_eq!( - meta, - ") -> WebApplications { - let dir = TempDir::new(std::env::temp_dir().join(format!("stalwart-app-{name}"))); - dir.create().await.unwrap(); - tokio::fs::write(dir.path.join("index.html"), INDEX) - .await - .unwrap(); - tokio::fs::write(dir.path.join("app.js"), "export const x = 1;\n") - .await - .unwrap(); - - let mut resources = AHashMap::new(); - resources.insert( - "index.html".to_string(), - Resource::new("text/html", dir.path.join("index.html")), - ); - resources.insert( - "app.js".to_string(), - Resource::new("text/javascript", dir.path.join("app.js")), - ); - - let routes = Arc::new(AppRoutes { - resources, - oauth_client_id_meta: client_id.map(oauth_client_id_meta), - _bundle_dir: dir, - }); - - let mut map = AHashMap::new(); - map.insert("admin".to_string(), routes.clone()); - map.insert("account".to_string(), routes); - - let apps = WebApplications::new(); - apps.routes.store(Arc::new(map)); - - apps - } - - async fn serve_html(apps: &WebApplications, prefix: &str, path: &str) -> String { - let served = apps.serve(prefix, path).await.unwrap().unwrap(); - assert!(served.no_cache, "index responses must not be cached"); - assert_eq!(served.resource.content_type.as_ref(), "text/html"); - String::from_utf8(served.resource.contents).unwrap() - } - - #[tokio::test] - async fn serving_index_injects_the_prefix_and_client_id() { - let apps = fixture("serve-configured", Some("pocket-id-client")).await; - - let html = serve_html(&apps, "admin", "index.html").await; - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - - let html = serve_html(&apps, "account", "index.html").await; - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - } - - #[tokio::test] - async fn unknown_paths_fall_back_to_a_rewritten_index() { - let apps = fixture("serve-fallback", Some("pocket-id-client")).await; - - let html = serve_html(&apps, "admin", "settings/directory").await; - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - } - - #[tokio::test] - async fn assets_and_unknown_prefixes_are_untouched() { - let apps = fixture("serve-assets", Some("pocket-id-client")).await; - - let served = apps.serve("admin", "app.js").await.unwrap().unwrap(); - assert_eq!(served.resource.contents, b"export const x = 1;\n"); - assert_eq!(served.resource.content_type.as_ref(), "text/javascript"); - assert!(!served.no_cache); - - assert!(apps.serve("unknown", "index.html").await.unwrap().is_none()); - } - - #[tokio::test] - async fn serving_index_without_a_client_id_keeps_the_placeholder() { - let apps = fixture("serve-unconfigured", None).await; - - let html = serve_html(&apps, "admin", "index.html").await; - assert!(html.contains(""), "{html}"); - assert!( - html.contains(""), - "{html}" - ); - } - - #[test] - fn an_unmodified_document_is_returned_verbatim() { - let bundle = "x"; - - assert_eq!(rewrite_index(bundle, "admin", None), bundle.as_bytes()); - } - #[tokio::test] - async fn missing_parent_directories_are_created() { - let base = std::env::temp_dir().join("stalwart-app-nested"); - let _ = tokio::fs::remove_dir_all(&base).await; - - let dir = TempDir::new(base.join("webui").join("0")); - dir.create().await.unwrap(); - - assert!(tokio::fs::metadata(&dir.path).await.is_ok()); - - drop(dir); - let _ = tokio::fs::remove_dir_all(&base).await; - } - - #[tokio::test] - async fn dropping_the_routes_removes_the_bundle_directory() { - let apps = fixture("drop-guard", None).await; - let path = apps - .routes - .load() - .get("admin") - .unwrap() - ._bundle_dir - .path - .clone(); - - assert!(tokio::fs::metadata(&path).await.is_ok()); - - apps.routes.store(Arc::new(AHashMap::new())); - - assert!(tokio::fs::metadata(&path).await.is_err()); - } - - #[tokio::test] - async fn sweeping_orphans_spares_the_current_generation() { - let base = std::env::temp_dir().join("stalwart-app-sweep"); - let _ = tokio::fs::remove_dir_all(&base).await; - - let current = TempDir::new(base.join("1")); - current.create().await.unwrap(); - let orphan = base.join("0"); - tokio::fs::create_dir_all(&orphan).await.unwrap(); - let stray = base.join("webui.zip"); - tokio::fs::write(&stray, b"not a bundle").await.unwrap(); - - remove_siblings(&base, ¤t.path).await; - - assert!(tokio::fs::metadata(¤t.path).await.is_ok()); - assert!(tokio::fs::metadata(&orphan).await.is_err()); - assert!(tokio::fs::metadata(&stray).await.is_err()); - - drop(current); - let _ = tokio::fs::remove_dir_all(&base).await; - } - - #[test] - fn generations_never_repeat() { - let apps = WebApplications::new(); - - assert_ne!(apps.next_generation(), apps.next_generation()); - } - - fn bundle(version: Option<&str>) -> zip::ZipArchive>> { - let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); - let options = zip::write::SimpleFileOptions::default(); - writer.start_file("index.html", options).unwrap(); - io::Write::write_all(&mut writer, INDEX.as_bytes()).unwrap(); - if let Some(version) = version { - writer.start_file("VERSION", options).unwrap(); - io::Write::write_all(&mut writer, version.as_bytes()).unwrap(); - } - zip::ZipArchive::new(writer.finish().unwrap()).unwrap() - } - - #[test] - fn bundles_newer_than_the_supported_major_version_are_rejected() { - // Bundles without VERSION or with a 1.x VERSION are accepted, anything else is rejected - assert_eq!(unsupported_bundle_version(&mut bundle(None)), None); - assert_eq!( - unsupported_bundle_version(&mut bundle(Some("1.0.12\n"))), - None - ); - assert_eq!( - unsupported_bundle_version(&mut bundle(Some("2.0.0\n"))), - Some("2.0.0".to_string()) - ); - assert_eq!( - unsupported_bundle_version(&mut bundle(Some("10.1.0"))), - Some("10.1.0".to_string()) - ); - assert_eq!( - unsupported_bundle_version(&mut bundle(Some("garbage"))), - Some("garbage".to_string()) - ); - } -} diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index 6db3c935..e6b7990e 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -13,11 +13,11 @@ use std::{ sync::mpsc::{self, SyncSender}, }; use store::{ - write::{AnyClass, AnyKey, ValueClass}, + write::{AnyClass, AnyKey, ValueClass, key::SystemKind}, *, }; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; -use utils::{UnwrapFailure, codec::leb128::Leb128_}; +use utils::{UnwrapFailure, codec::leb128::Leb128, exit_failed}; pub(super) const MAGIC_MARKER: u8 = 123; @@ -47,7 +47,7 @@ impl Core { std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory"); } else if !params.dest.is_dir() { eprintln!("Backup destination {:?} is not a directory.", params.dest); - std::process::exit(1); + exit_failed(); } let mut sync_handles = Vec::new(); @@ -55,8 +55,8 @@ impl Core { .storage .data .get_value::(AnyKey { - subspace: SUBSPACE_PROPERTY, - key: vec![0u8], + subspace: Subspace::System, + key: vec![SystemKind::SchemaVersion as u8], }) .await .failed("Could not retrieve database schema version.") @@ -77,13 +77,8 @@ impl Core { .collect(); } - for subspace in params - .families - .into_iter() - .flat_map(|f| f.subspaces()) - .copied() - { - let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS { + for subspace in params.families.into_iter().flat_map(|f| f.subspaces()) { + let (async_handle, sync_handle) = if subspace == Subspace::Blobs { self.backup_blobs(¶ms.dest, subspace, schema_version) } else { self.backup_subspace(¶ms.dest, subspace, schema_version) @@ -97,11 +92,11 @@ impl Core { } } - fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_blobs(&self, dest: &Path, subspace: Subspace, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", char::from(subspace))), + dest.join(format!("subspace_{}", subspace.name())), subspace, schema_version, ); @@ -113,11 +108,11 @@ impl Core { .iterate( IterateParams::new( AnyKey { - subspace: SUBSPACE_BLOB_LINK, + subspace: Subspace::BlobLink, key: vec![0u8], }, AnyKey { - subspace: SUBSPACE_BLOB_LINK, + subspace: Subspace::BlobLink, key: vec![u8::MAX; 32], }, ) @@ -157,16 +152,16 @@ impl Core { ) } - fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_subspace(&self, dest: &Path, subspace: Subspace, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", char::from(subspace))), + dest.join(format!("subspace_{}", subspace.name())), subspace, schema_version, ); ( tokio::spawn(async move { - if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) { + if !store.is_sql() || !matches!(subspace.shape(), Shape::Counter) { store .iterate( IterateParams::new( @@ -179,9 +174,7 @@ impl Core { key: vec![u8::MAX; 32], }, ) - .set_values( - ![SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX].contains(&subspace), - ), + .set_values(!matches!(subspace.shape(), Shape::Presence)), |key, value| { writer .send((key.to_vec(), value.to_vec())) @@ -238,7 +231,7 @@ impl Core { #[allow(clippy::type_complexity)] fn spawn_writer( path: PathBuf, - subspace: u8, + subspace: Subspace, version: u32, ) -> (std::thread::JoinHandle<()>, SyncSender<(Vec, Vec)>) { let (tx, rx) = mpsc::sync_channel::<(Vec, Vec)>(10); @@ -249,19 +242,16 @@ fn spawn_writer( let mut file = FrameEncoder::new(BufWriter::new( std::fs::File::create(path).failed("Failed to create backup file"), )); - file.write_all(&[MAGIC_MARKER, subspace]) + file.write_all(&[MAGIC_MARKER, subspace.byte()]) .failed("Failed to write version"); file.write_all(&version.to_le_bytes()) .failed("Failed to write version"); while let Ok((key, value)) = rx.recv() { - key.len() - .to_leb128_writer(&mut file) + file.write_all(&key.len().to_leb128()) .failed("Failed to write key value"); file.write_all(&key).failed("Failed to write key"); - value - .len() - .to_leb128_writer(&mut file) + file.write_all(&value.len().to_leb128()) .failed("Failed to write key value"); if !value.is_empty() { file.write_all(&value).failed("Failed to write key value"); @@ -305,30 +295,45 @@ impl BackupParams { } } -impl Family { - pub fn subspaces(&self) -> &'static [u8] { +pub(super) trait BackupFamily { + fn family(self) -> Option; +} + +impl BackupFamily for Subspace { + fn family(self) -> Option { match self { - Family::Data => &[ - SUBSPACE_ACL, - SUBSPACE_INDEXES, - SUBSPACE_QUOTA, - SUBSPACE_COUNTER, - SUBSPACE_PROPERTY, - ], - Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_LINK], - Family::Registry => &[ - SUBSPACE_REGISTRY, - SUBSPACE_REGISTRY_IDX, - SUBSPACE_REGISTRY_PK, - SUBSPACE_DIRECTORY, - ], - Family::Changelog => &[SUBSPACE_LOGS], - Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT], - Family::Report => &[SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN], - Family::Telemetry => &[SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC], - Family::Tasks => &[SUBSPACE_TASK_QUEUE], + Subspace::Acl + | Subspace::Indexes + | Subspace::Counter + | Subspace::Property + | Subspace::Immutable + | Subspace::IndexProperty + | Subspace::System => Some(Family::Data), + Subspace::Blobs | Subspace::BlobLink => Some(Family::Blob), + Subspace::Registry + | Subspace::RegistryIndex + | Subspace::RegistryPrimaryKey + | Subspace::Directory + | Subspace::DeletedItems + | Subspace::SpamSamples => Some(Family::Registry), + Subspace::Logs => Some(Family::Changelog), + Subspace::QueueMessage | Subspace::QueueEvent => Some(Family::Queue), + Subspace::ReportOut | Subspace::ReportIn => Some(Family::Report), + Subspace::TelemetrySpan | Subspace::TelemetryMetric => Some(Family::Telemetry), + Subspace::TaskQueue => Some(Family::Tasks), + Subspace::SearchTerm | Subspace::SearchDocument | Subspace::SearchQueue => None, } } +} + +impl Family { + pub fn subspaces(&self) -> Vec { + Subspace::ALL + .iter() + .copied() + .filter(|subspace| subspace.family() == Some(*self)) + .collect() + } pub fn parse(family: &str) -> Result { match family { diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 5f7f5015..88a313ad 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -8,12 +8,16 @@ use super::{backup::BackupParams, console::store_console}; use crate::{ BuildServer, Caches, Core, Data, IPC_CHANNEL_BUFFER, Inner, Ipc, config::{ - network::AsnGeoLookupConfig, server::Listeners, storage::Storage, telemetry::Telemetry, + network::AsnGeoLookupConfig, + server::{Listeners, listener::recovery_mode_port}, + storage::Storage, + telemetry::Telemetry, }, ipc::{BroadcastEvent, PushEvent, QueueEvent, ReportingEvent, TrainTaskController}, manager::defaults::BootstrapDefaults, }; use arc_swap::ArcSwap; +use compact_str::CompactString; use std::{ net::{IpAddr, Ipv4Addr}, path::PathBuf, @@ -21,7 +25,7 @@ use std::{ }; use store::{RegistryStore, registry::bootstrap::Bootstrap}; use tokio::sync::{Notify, mpsc}; -use utils::{UnwrapFailure, failed}; +use utils::{UnwrapFailure, failed, runtime}; pub struct BootManager { pub bootstrap: Bootstrap, @@ -141,17 +145,17 @@ impl BootManager { // Parse storage let storage = Storage::parse(&mut bootstrap).await; + let (ipc, ipc_rxs) = build_ipc(!storage.coordinator.is_none()); // Parse telemetry - let telemetry = Telemetry::parse(&mut bootstrap, &storage).await; + let telemetry = Telemetry::parse(&mut bootstrap, &storage, ipc.index_queue_notify()).await; match import_export { StoreOp::None => { // Parse components - let core: Box = - Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await); + let core = Core::parse(&mut bootstrap, storage).await; let data = Data::parse(&mut bootstrap).await; - let cache = Caches::parse(&mut bootstrap).await; + let (cache, swap_rx) = Caches::parse(&mut bootstrap, &core.storage).await; // Enable telemetry @@ -162,22 +166,27 @@ impl BootManager { if bootstrap.registry.is_bootstrap_mode() { trc::event!( Server(trc::ServerEvent::BootstrapMode), - Hostname = bootstrap.registry.local_hostname().to_string(), - Details = - "No configuration file was found. Port 8080 is open for initial setup.", + Hostname = CompactString::from(bootstrap.registry.local_hostname()), + Details = format!( + "No configuration file was found. Port {} is open for initial setup.", + recovery_mode_port() + ), Version = env!("CARGO_PKG_VERSION"), ); } else if bootstrap.registry.is_recovery_mode() { trc::event!( Server(trc::ServerEvent::RecoveryMode), - Details = "Port 8080 is open for troubleshooting and recovery.", - Hostname = bootstrap.registry.local_hostname().to_string(), + Details = format!( + "Port {} is open for troubleshooting and recovery.", + recovery_mode_port() + ), + Hostname = CompactString::from(bootstrap.registry.local_hostname()), Version = env!("CARGO_PKG_VERSION"), ); } else { trc::event!( Server(trc::ServerEvent::Startup), - Hostname = bootstrap.registry.local_hostname().to_string(), + Hostname = CompactString::from(bootstrap.registry.local_hostname()), Version = env!("CARGO_PKG_VERSION"), ); } @@ -185,12 +194,10 @@ impl BootManager { if core.storage.coordinator.is_enabled() { trc::event!( Cluster(trc::ClusterEvent::Startup), - Id = bootstrap.registry.node_id(), - Type = bootstrap - .registry - .cluster_role() - .unwrap_or("[default]") - .to_string(), + NodeId = bootstrap.registry.node_id(), + Type = CompactString::from( + bootstrap.registry.cluster_role().unwrap_or("[default]") + ), Details = bootstrap.registry.cluster_push_shard() ); } @@ -200,7 +207,6 @@ impl BootManager { core.network.asn_geo_lookup, AsnGeoLookupConfig::Resource { .. } ); - let (ipc, ipc_rxs) = build_ipc(!core.storage.coordinator.is_none()); let inner = Arc::new(Inner { shared_core: ArcSwap::new(Arc::from(core)), data, @@ -209,6 +215,9 @@ impl BootManager { }); if !bootstrap.registry.is_recovery_mode() { + // Start the cache swap tier writer + crate::cache::swap::SwapTier::start(&inner, swap_rx); + // Load spam model if let Err(err) = inner.build_server().spam_model_reload().await { trc::error!( @@ -243,33 +252,30 @@ impl BootManager { telemetry.enable(false); // Parse settings and backup - Box::pin(Core::parse(&mut bootstrap, storage)) + Core::parse(&mut bootstrap, storage) .await .backup(path) .await; - std::process::exit(0); + drop(bootstrap); + runtime::exit(0); } StoreOp::Import(path) => { // Enable telemetry telemetry.enable(false); // Parse settings and restore - Box::pin(Core::parse(&mut bootstrap, storage)) + Core::parse(&mut bootstrap, storage) .await .restore(path) .await; - std::process::exit(0); + drop(bootstrap); + runtime::exit(0); } StoreOp::Console => { // Store console - store_console( - Box::pin(Core::parse(&mut bootstrap, storage)) - .await - .storage - .data, - ) - .await; - std::process::exit(0); + store_console(Core::parse(&mut bootstrap, storage).await.storage.data).await; + drop(bootstrap); + runtime::exit(0); } } } @@ -284,10 +290,12 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { ( Ipc { push_tx, + push_subscribers: Default::default(), queue_tx, report_tx, broadcast_tx: has_pubsub.then_some(broadcast_tx), - task_tx: Arc::new(Notify::new()), + task_tx: Default::default(), + index_tx: Arc::new(Notify::new()), train_task_controller: Arc::new(TrainTaskController::default()), }, IpcReceivers { diff --git a/crates/common/src/manager/console.rs b/crates/common/src/manager/console.rs index 470f6974..49af5648 100644 --- a/crates/common/src/manager/console.rs +++ b/crates/common/src/manager/console.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::Engine; -use base64::engine::general_purpose; +use encodify::base64::STANDARD; use std::env; use std::io::{self, Write}; +use store::write::serialize::RawValue; use store::write::{AnyClass, AnyKey, BatchBuilder, ValueClass}; -use store::{Deserialize, IterateParams, SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX, Store}; +use store::{IterateParams, Shape, Store, Subspace}; const HELP: &str = concat!( "Stalwart Server v", @@ -52,12 +52,17 @@ pub async fn store_console(store: Store) { println!("Scanning from {:?} to {:?}", from_key, to_key); let mut from_key = from_key.into_iter(); let mut to_key = to_key.into_iter(); - let from_subspace = from_key.next().unwrap(); - let to_subspace = to_key.next().unwrap(); + let (Some(from_subspace), Some(to_subspace)) = ( + from_key.next().and_then(Subspace::try_from_byte), + to_key.next().and_then(Subspace::try_from_byte), + ) else { + println!("Unknown subspace."); + continue; + }; if from_subspace != to_subspace { println!("Keys must be in the same subspace."); - return; + continue; } store @@ -72,11 +77,9 @@ pub async fn store_console(store: Store) { key: to_key.collect::>(), }, ) - .set_values( - ![SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX].contains(&from_subspace), - ), + .set_values(!matches!(from_subspace.shape(), Shape::Presence)), |key, value| { - print!("{}", char::from(from_subspace)); + print!("{}", from_subspace.name()); print_escaped(key); print!(" : "); print_escaped(value); @@ -95,18 +98,25 @@ pub async fn store_console(store: Store) { let mut from_key = from_key.into_iter(); let mut to_key = to_key.into_iter(); + let (Some(from_subspace), Some(to_subspace)) = ( + from_key.next().and_then(Subspace::try_from_byte), + to_key.next().and_then(Subspace::try_from_byte), + ) else { + println!("Unknown subspace."); + continue; + }; let from_key = AnyKey { - subspace: from_key.next().unwrap(), + subspace: from_subspace, key: from_key.collect::>(), }; let to_key = AnyKey { - subspace: to_key.next().unwrap(), + subspace: to_subspace, key: to_key.collect::>(), }; if from_key.subspace != to_key.subspace { println!("Keys must be in the same subspace."); - return; + continue; } let mut total = 0; @@ -128,7 +138,7 @@ pub async fn store_console(store: Store) { io::stdin().read_line(&mut response).unwrap(); if !response.trim().eq_ignore_ascii_case("y") { println!("Aborted."); - return; + continue; } store @@ -145,12 +155,11 @@ pub async fn store_console(store: Store) { if let Some(key) = parse_key(key) { println!("Deleting key: {:?}", key); let mut key = key.into_iter(); - let mut batch = BatchBuilder::new(); - batch.clear(ValueClass::Any(AnyClass { - subspace: key.next().unwrap(), - key: key.collect(), - })); - if let Err(err) = store.write(batch.build_all()).await { + let Some(subspace) = key.next().and_then(Subspace::try_from_byte) else { + println!("Unknown subspace."); + continue; + }; + if let Err(err) = delete_raw_key(&store, subspace, key.collect()).await { println!("Failed to delete key: {}", err); } } @@ -164,9 +173,13 @@ pub async fn store_console(store: Store) { println!("Usage: get "); } else if let Some(key) = parse_key(parts[1]) { let mut key = key.into_iter(); + let Some(subspace) = key.next().and_then(Subspace::try_from_byte) else { + println!("Unknown subspace."); + continue; + }; match store .get_value::(AnyKey { - subspace: key.next().unwrap(), + subspace, key: key.collect::>(), }) .await @@ -188,19 +201,21 @@ pub async fn store_console(store: Store) { if parts.len() < 2 { println!("Usage: put []"); } else if let Some(key) = parse_key(parts[1]) { - let value = parts.get(2).map(|v| parse_value(v)).unwrap_or_default(); + let value = match parts.get(2) { + Some(value) => match parse_value(value) { + Some(value) => value, + None => continue, + }, + None => Vec::new(), + }; println!("Putting key: {key:?}"); let mut key = key.into_iter(); - let mut batch = BatchBuilder::new(); - batch.set( - ValueClass::Any(AnyClass { - subspace: key.next().unwrap(), - key: key.collect(), - }), - value, - ); - if let Err(err) = store.write(batch.build_all()).await { + let Some(subspace) = key.next().and_then(Subspace::try_from_byte) else { + println!("Unknown subspace."); + continue; + }; + if let Err(err) = put_raw_key(&store, subspace, key.collect(), value).await { println!("Failed to insert key: {}", err); } } @@ -220,12 +235,11 @@ pub async fn store_console(store: Store) { } fn parse_key(input: &str) -> Option> { - let result = if let Some(key) = input.strip_prefix("base64:") { - base64_decode(key) - } else { - parse_binary(input) - }; - if matches!(result.first(), Some(ch) if ch.is_ascii_alphabetic() && ch.is_ascii_lowercase()) { + let result = parse_value(input)?; + if result + .first() + .is_some_and(|ch| Subspace::try_from_byte(*ch).is_some()) + { Some(result) } else { println!("Invalid key: {result:?}"); @@ -233,18 +247,22 @@ fn parse_key(input: &str) -> Option> { } } -fn parse_value(input: &str) -> Vec { +fn parse_value(input: &str) -> Option> { if let Some(key) = input.strip_prefix("base64:") { base64_decode(key) } else { - parse_binary(input) + Some(parse_binary(input)) } } -fn base64_decode(input: &str) -> Vec { - general_purpose::STANDARD - .decode(input) - .expect("Failed to decode base64") +fn base64_decode(input: &str) -> Option> { + match STANDARD.decode(input) { + Ok(value) => Some(value), + Err(err) => { + println!("Invalid base64 value: {err}"); + None + } + } } fn parse_binary(input: &str) -> Vec { @@ -295,6 +313,64 @@ fn print_escaped(bytes: &[u8]) { } } +pub async fn delete_raw_key(store: &Store, subspace: Subspace, key: Vec) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + clear_raw_chunks(store, subspace, &key, &mut batch).await?; + batch.clear(ValueClass::Any(AnyClass { subspace, key })); + store.write_batch(&mut batch).await.map(|_| ()) +} + +pub async fn put_raw_key( + store: &Store, + subspace: Subspace, + key: Vec, + value: Vec, +) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + clear_raw_chunks(store, subspace, &key, &mut batch).await?; + batch.set(ValueClass::Any(AnyClass { subspace, key }), value); + store.write_batch(&mut batch).await.map(|_| ()) +} + +async fn clear_raw_chunks( + store: &Store, + subspace: Subspace, + key: &[u8], + batch: &mut BatchBuilder, +) -> trc::Result<()> { + let Some(chunk_size) = store.chunk_size() else { + return Ok(()); + }; + let is_chunked = store + .get_value::(AnyKey { subspace, key }) + .await? + .is_some_and(|value| value.0.len() >= chunk_size); + if !is_chunked { + return Ok(()); + } + + let mut chunk_key = Vec::with_capacity(key.len() + 1); + chunk_key.extend_from_slice(key); + chunk_key.push(0); + while store + .key_exists(AnyKey { + subspace, + key: chunk_key.as_slice(), + }) + .await? + { + batch.clear(ValueClass::Any(AnyClass { + subspace, + key: chunk_key.clone(), + })); + match chunk_key.last_mut() { + Some(last) if *last < u8::MAX => *last += 1, + _ => break, + } + } + Ok(()) +} + fn print_help() { println!("Available commands:"); println!(" scan "); @@ -306,11 +382,3 @@ fn print_help() { println!("Note: Keys and values can be prefixed with 'base64:' for base64 encoding"); println!(" or use escaped hex values (e.g., \\x41 for 'A')"); } - -struct RawValue(Vec); - -impl Deserialize for RawValue { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(RawValue(bytes.to_vec())) - } -} diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index 03eda403..c45df396 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -26,12 +26,6 @@ use store::{ }, }; -pub const WEBUI_LATEST_URL: &str = - "https://github.com/stalwartlabs/webui/releases/latest/download/webui.zip"; -pub const WEBUI_PINNED_URL: &str = - "https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip"; -pub const WEBUI_MAX_MAJOR_VERSION: u32 = 1; - pub const ASN_IPV4: &str = "https://github.com/sapics/ip-location-db/releases/download/latest/origin-asn-ipv4.csv"; pub const ASN_IPV6: &str = @@ -60,21 +54,21 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { let is_recovery_mode = bp.registry.is_recovery_mode(); let is_bootstrap_mode = bp.registry.is_bootstrap_mode(); - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] if bp.registry.count_object(ObjectType::Application).await? == 0 { bp.registry .write(RegistryWrite::insert( &Application { - auto_update_frequency: Duration::from_millis(30 * 24 * 60 * 60 * 1000), description: "Stalwart Web Interface".to_string(), - enabled: true, + enable: true, #[cfg(not(feature = "dev_mode"))] - resource_url: WEBUI_PINNED_URL.into(), + resource_url: + "https://github.com/stalwartlabs/webui/releases/download/v2/webui.zip" + .into(), #[cfg(feature = "dev_mode")] - resource_url: "file:///Users/me/code/webui/.ignore/webui.zip".into(), - unpack_directory: None, - oauth_client_id: None, + resource_url: "file://../webui/.ignore/webui.zip".into(), url_prefix: Map::new(vec!["/admin".into(), "/account".into()]), + ..Default::default() } .into(), )) @@ -82,7 +76,7 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { } if is_bootstrap_mode { - #[cfg(not(any(feature = "dev_mode", feature = "test_mode")))] + #[cfg(not(any(feature = "dev_mode", stalwart_tests)))] if bp.registry.count_object(ObjectType::SystemSettings).await? == 0 { bp.registry .write(RegistryWrite::insert( @@ -95,11 +89,11 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { .await?; } - return Ok(()); + return insert_oidc_provider(bp).await; } if is_recovery_mode { - return Ok(()); + return insert_oidc_provider(bp).await; } if bp.registry.count_object(ObjectType::MtaQueueQuota).await? == 0 { @@ -181,12 +175,12 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { .to_string() .into(), name: "dsn".into(), - threads_per_node: 5, + threads_per_node: 16, }, MtaVirtualQueue { description: "DMARC and TLS report delivery queue".to_string().into(), name: "report".into(), - threads_per_node: 5, + threads_per_node: 8, }, ] .into_iter() @@ -346,6 +340,183 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { .await?; } + insert_oidc_provider(bp).await?; + + // Generate a Web Push VAPID signing key (RFC 9749) + if bp.registry.count_object(ObjectType::Jmap).await? == 0 { + match crate::network::webpush::generate_pkcs8_pem() { + Ok(web_push_pem) => { + bp.registry + .write(RegistryWrite::insert( + &Jmap { + web_push_key: SecretTextOptional::Text(SecretTextValue { + secret: web_push_pem, + }), + ..Default::default() + } + .into(), + )) + .await?; + } + Err(err) => { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = "Failed to generate Web Push VAPID key", + Reason = err + ); + } + } + } + + if bp.registry.count_object(ObjectType::Role).await? == 0 { + let permissions = DefaultPermissions::default(); + let mut roles = Vec::with_capacity(4); + roles.push(Role { + description: "User".into(), + enabled_permissions: Map::new(permissions.user), + ..Default::default() + }); + roles.push(Role { + description: "Group".into(), + enabled_permissions: Map::new(permissions.group), + ..Default::default() + }); + roles.push(Role { + description: "System Administrator".into(), + enabled_permissions: Map::new(permissions.superuser), + ..Default::default() + }); + let num_roles = roles.len(); + let mut role_ids = Vec::with_capacity(num_roles); + + for role in roles { + match bp + .registry + .write(RegistryWrite::insert(&role.into())) + .await? + { + RegistryWriteResult::Success(id) => role_ids.push(id), + err => { + bp.build_error( + ObjectType::Role.singleton(), + format!("Failed to insert default role: {err}"), + ); + } + } + } + + if bp.registry.count_object(ObjectType::Authentication).await? == 0 + && role_ids.len() == num_roles + { + bp.registry + .write(RegistryWrite::insert( + &Authentication { + default_user_role_ids: Map::new(vec![role_ids[0]]), + default_group_role_ids: Map::new(vec![role_ids[1]]), + default_admin_role_ids: Map::new(vec![ + role_ids[num_roles - 1], + role_ids[0], + ]), + ..Default::default() + } + .into(), + )) + .await?; + } + } + + if bp + .registry + .count_object(ObjectType::NetworkListener) + .await? + == 0 + { + for (protocol, name, port, implicit_tls) in [ + (NetworkListenerProtocol::Smtp, "smtp", 25, false), + (NetworkListenerProtocol::Smtp, "submissions", 465, true), + (NetworkListenerProtocol::Imap, "imaps", 993, true), + (NetworkListenerProtocol::Pop3, "pop3s", 995, true), + (NetworkListenerProtocol::ManageSieve, "sieve", 4190, false), + (NetworkListenerProtocol::Http, "https", 443, true), + (NetworkListenerProtocol::Http, "http", 8080, false), + ] { + bp.registry + .write(RegistryWrite::insert( + &NetworkListener { + bind: Map::new(vec![ + SocketAddr::from_str(&format!("[::]:{port}")).unwrap(), + ]), + name: name.to_string(), + protocol, + use_tls: true, + implicit_tls, + ..Default::default() + } + .into(), + )) + .await?; + } + } + + #[cfg(not(any(feature = "dev_mode", stalwart_tests)))] + if bp.registry.count_object(ObjectType::Asn).await? == 0 { + bp.registry + .write(RegistryWrite::insert( + &Asn::Resource(AsnResource { + asn_urls: Map::new(vec![ASN_IPV4.into(), ASN_IPV6.into()]), + geo_urls: Map::new(vec![GEO_IPV4.into(), GEO_IPV6.into()]), + max_size: 104857600, + refresh_interval: Duration::from_millis(24 * 60 * 60 * 1000), + timeout: Duration::from_millis(5 * 60 * 1000), + ..Default::default() + }) + .into(), + )) + .await?; + } + + + if bp.registry.count_object(ObjectType::Tracer).await? == 0 { + bp.registry + .write(RegistryWrite::insert( + &Tracer::Log(TracerLog { + enable: true, + ansi: false, + prefix: "stalwart.log".into(), + rotate: LogRotateFrequency::Daily, + path: "/var/log/inbuxa".into(), + ..Default::default() + }) + .into(), + )) + .await?; + } + + #[cfg(not(stalwart_tests))] + { + use store::write::BatchBuilder; + use types::id::Id; + + if bp.registry.count_object(ObjectType::SpamRule).await? == 0 + && bp + .registry + .object::(Id::singleton()) + .await? + .is_none_or(|spam| spam.spam_filter_rules_url.is_some()) + { + let mut batch = BatchBuilder::new(); + batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance { + maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules, + status: TaskStatus::now(), + })); + bp.data_store.write_batch(&mut batch).await?; + } + } + + Ok(()) +} + +pub async fn insert_oidc_provider(bp: &mut Bootstrap) -> trc::Result<()> { if bp.registry.count_object(ObjectType::OidcProvider).await? == 0 { let pkcs8_doc = EcdsaKeyPair::generate_pkcs8(&ECDSA_P256_SHA256_FIXED_SIGNING, &SystemRandom::new()) @@ -376,190 +547,6 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { .into(), )) .await?; - - // Generate a Web Push VAPID signing key (RFC 9749) - if bp.registry.count_object(ObjectType::Jmap).await? == 0 { - match crate::network::webpush::generate_pkcs8_pem() { - Ok(web_push_pem) => { - bp.registry - .write(RegistryWrite::insert( - &Jmap { - web_push_key: SecretTextOptional::Text(SecretTextValue { - secret: web_push_pem, - }), - ..Default::default() - } - .into(), - )) - .await?; - } - Err(err) => { - trc::event!( - Server(trc::ServerEvent::Startup), - Details = "Failed to generate Web Push VAPID key", - Reason = err - ); - } - } - } - } - - if bp.registry.count_object(ObjectType::Role).await? == 0 { - let permissions = DefaultPermissions::default(); - let mut role_ids = Vec::with_capacity(4); - - for role in [ - Role { - description: "User".into(), - enabled_permissions: Map::new(permissions.user), - ..Default::default() - }, - Role { - description: "Group".into(), - enabled_permissions: Map::new(permissions.group), - ..Default::default() - }, - Role { - description: "Tenant Administrator".into(), - enabled_permissions: Map::new(permissions.tenant), - ..Default::default() - }, - Role { - description: "System Administrator".into(), - enabled_permissions: Map::new(permissions.superuser), - ..Default::default() - }, - ] { - match bp - .registry - .write(RegistryWrite::insert(&role.into())) - .await? - { - RegistryWriteResult::Success(id) => role_ids.push(id), - err => { - bp.build_error( - ObjectType::Role.singleton(), - format!("Failed to insert default role: {err}"), - ); - } - } - } - - if bp.registry.count_object(ObjectType::Authentication).await? == 0 && role_ids.len() == 4 { - bp.registry - .write(RegistryWrite::insert( - &Authentication { - default_user_role_ids: Map::new(vec![role_ids[0]]), - default_group_role_ids: Map::new(vec![role_ids[1]]), - default_tenant_role_ids: Map::new(vec![role_ids[2], role_ids[0]]), - default_admin_role_ids: Map::new(vec![role_ids[3], role_ids[0]]), - ..Default::default() - } - .into(), - )) - .await?; - } - } - - if bp - .registry - .count_object(ObjectType::NetworkListener) - .await? - == 0 - { - for (protocol, name, port, tls_implicit) in [ - (NetworkListenerProtocol::Smtp, "smtp", 25, false), - (NetworkListenerProtocol::Smtp, "submissions", 465, true), - (NetworkListenerProtocol::Imap, "imaps", 993, true), - (NetworkListenerProtocol::Pop3, "pop3s", 995, true), - (NetworkListenerProtocol::ManageSieve, "sieve", 4190, false), - (NetworkListenerProtocol::Http, "https", 443, true), - (NetworkListenerProtocol::Http, "http", 8080, false), - ] { - bp.registry - .write(RegistryWrite::insert( - &NetworkListener { - bind: Map::new(vec![ - SocketAddr::from_str(&format!("[::]:{port}")).unwrap(), - ]), - name: name.to_string(), - protocol, - use_tls: true, - tls_implicit, - ..Default::default() - } - .into(), - )) - .await?; - } - } - - #[cfg(not(any(feature = "dev_mode", feature = "test_mode")))] - if bp.registry.count_object(ObjectType::Asn).await? == 0 { - bp.registry - .write(RegistryWrite::insert( - &Asn::Resource(AsnResource { - asn_urls: Map::new(vec![ASN_IPV4.into(), ASN_IPV6.into()]), - geo_urls: Map::new(vec![GEO_IPV4.into(), GEO_IPV6.into()]), - max_size: 104857600, - expires: Duration::from_millis(24 * 60 * 60 * 1000), - timeout: Duration::from_millis(5 * 60 * 1000), - ..Default::default() - }) - .into(), - )) - .await?; - } - - #[cfg(not(feature = "test_mode"))] - if bp.registry.count_object(ObjectType::TracingStore).await? == 0 { - bp.registry - .write(RegistryWrite::insert(&TracingStore::Default.into())) - .await?; - } - - #[cfg(not(feature = "test_mode"))] - if bp.registry.count_object(ObjectType::MetricsStore).await? == 0 { - bp.registry - .write(RegistryWrite::insert(&MetricsStore::Default.into())) - .await?; - } - - if bp.registry.count_object(ObjectType::Tracer).await? == 0 { - bp.registry - .write(RegistryWrite::insert( - &Tracer::Log(TracerLog { - enable: true, - ansi: false, - prefix: "stalwart.log".into(), - rotate: LogRotateFrequency::Daily, - path: "/var/log/inbuxa".into(), - ..Default::default() - }) - .into(), - )) - .await?; - } - - #[cfg(not(feature = "test_mode"))] - { - use store::write::BatchBuilder; - use types::id::Id; - - if bp.registry.count_object(ObjectType::SpamRule).await? == 0 - && bp - .registry - .object::(Id::singleton()) - .await? - .is_none_or(|spam| spam.spam_filter_rules_url.is_some()) - { - let mut batch = BatchBuilder::new(); - batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance { - maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules, - status: TaskStatus::now(), - })); - bp.data_store.write(batch.build_all()).await?; - } } Ok(()) diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 6a990d5d..88e69749 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -6,19 +6,21 @@ use crate::USER_AGENT; use hyper::HeaderMap; -use mail_auth::flate2; +use smtp_report::flate2; use std::{ io::{BufReader, Read}, time::Duration, }; use utils::HttpLimitResponse; +pub mod app_client; pub mod application; pub mod backup; pub mod boot; pub mod console; pub mod defaults; pub mod restore; +pub mod schema; #[cfg(target_os = "linux")] pub mod systemd; @@ -30,22 +32,24 @@ pub async fn fetch_resource( headers: Option, timeout: Duration, max_size: usize, + allow_invalid_certs: bool, ) -> Result, String> { if let Some(path) = url.strip_prefix("file://") { tokio::fs::read(path) .await .map_err(|err| format!("Failed to read {path}: {err}")) } else { - let response = utils::http::http_client_builder(is_localhost_url(url)) - .timeout(timeout) - .user_agent(USER_AGENT) - .build() - .unwrap_or_default() - .get(url) - .headers(headers.unwrap_or_default()) - .send() - .await - .map_err(|err| format!("Failed to fetch {url}: {err}"))?; + let response = + utils::http::http_client_builder(allow_invalid_certs || is_localhost_url(url)) + .timeout(timeout) + .user_agent(USER_AGENT) + .build() + .unwrap_or_default() + .get(url) + .headers(headers.unwrap_or_default()) + .send() + .await + .map_err(|err| format!("Failed to fetch {url}: {err}"))?; if response.status().is_success() { response diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index cfdb0849..99eb01b5 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -14,15 +14,14 @@ use std::{ path::{Path, PathBuf}, }; use store::{ - BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_QUOTA, - SUBSPACE_REGISTRY_PK, Store, U32_LEN, + BlobStore, IterateParams, Shape, Store, Subspace, U32_LEN, write::{ AnyClass, AnyKey, BatchBuilder, ValueClass, key::{DeserializeBigEndian, is_node_id_key}, }, }; use types::{collection::Collection, field::Field}; -use utils::{UnwrapFailure, failed}; +use utils::{UnwrapFailure, exit_failed, failed}; impl Core { pub async fn restore(&self, src: PathBuf) { @@ -59,7 +58,7 @@ impl Core { for path in conflicts { eprintln!(" {}", path.display()); } - std::process::exit(1); + exit_failed(); } let mut tasks = Vec::new(); @@ -77,7 +76,7 @@ impl Core { } } -async fn subspace_has_data(store: &Store, subspace: u8) -> bool { +async fn subspace_has_data(store: &Store, subspace: Subspace) -> bool { let mut has_data = false; store .iterate( @@ -91,9 +90,10 @@ async fn subspace_has_data(store: &Store, subspace: u8) -> bool { key: vec![u8::MAX; 32], }, ) - .no_values(), + .no_values() + .expect_rows(2), |key, _| { - if subspace == SUBSPACE_REGISTRY_PK && is_node_id_key(key) { + if subspace == Subspace::System && is_node_id_key(key) { Ok(true) } else { has_data = true; @@ -113,7 +113,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { let mut batch = BatchBuilder::new(); match reader.subspace { - SUBSPACE_BLOBS => { + Subspace::Blobs => { while let Some((key, value)) = reader.next() { blob_store .put_blob(&key, &value, CompressionAlgo::Lz4) @@ -121,7 +121,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .failed("Failed to write blob"); } } - SUBSPACE_COUNTER | SUBSPACE_QUOTA => { + subspace if matches!(subspace.shape(), Shape::Counter) => { while let Some((key, value)) = reader.next() { batch.add( ValueClass::Any(AnyClass { @@ -136,14 +136,14 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { ); if batch.is_large_batch() { store - .write(batch.build_all()) + .write_batch(&mut batch) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); } } } - SUBSPACE_INDEXES => { + Subspace::Indexes => { while let Some((key, _)) = reader.next() { let account_id = key .as_slice() @@ -168,7 +168,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { if batch.is_large_batch() { store - .write(batch.build_all()) + .write_batch(&mut batch) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); @@ -186,7 +186,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { ); if batch.is_large_batch() { store - .write(batch.build_all()) + .write_batch(&mut batch) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); @@ -197,14 +197,14 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { if !batch.is_empty() { store - .write(batch.build_all()) + .write_batch(&mut batch) .await .failed("Failed to write batch"); } } struct KeyValueReader { - subspace: u8, + subspace: Subspace, file: FrameDecoder>, } @@ -223,7 +223,8 @@ impl KeyValueReader { file.read_exact(&mut buf) .failed(&format!("Failed to read subspace from {path:?}")); - let subspace = buf[0]; + let subspace = Subspace::try_from_byte(buf[0]) + .unwrap_or_else(|| failed(&format!("Unknown subspace in {path:?}"))); let mut buf = [0u8; 4]; file.read_exact(&mut buf) diff --git a/crates/common/src/manager/schema.rs b/crates/common/src/manager/schema.rs new file mode 100644 index 00000000..0a8ba08e --- /dev/null +++ b/crates/common/src/manager/schema.rs @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + DATABASE_SCHEMA_OSS_FLAG, DATABASE_SCHEMA_VERSION, DATABASE_SCHEMA_VERSION_BASE, Server, +}; +use store::{ + IterateParams, SerializeInfallible, Subspace, + write::{AnyClass, AnyKey, BatchBuilder, ValueClass, key::SystemKind}, +}; +use trc::AddContext; + +#[cfg(feature = "enterprise")] +const OTHER_EDITION: &str = "AGPL3-only"; +#[cfg(not(feature = "enterprise"))] +const OTHER_EDITION: &str = "AGPL3/SEL"; + +impl Server { + pub async fn verify_schema_version(&self) -> trc::Result<()> { + let version = self + .store() + .get_value::(AnyKey { + subspace: Subspace::System, + key: vec![SystemKind::SchemaVersion as u8], + }) + .await + .caused_by(trc::location!())?; + + match version { + Some(DATABASE_SCHEMA_VERSION) => Ok(()), + Some(version) => { + let base = version & !DATABASE_SCHEMA_OSS_FLAG; + if base == DATABASE_SCHEMA_VERSION_BASE { + Err(schema_error(format!( + "The data store was created by the Stalwart {OTHER_EDITION} edition and cannot be opened by this edition." + ))) + } else if base < DATABASE_SCHEMA_VERSION_BASE { + Err(schema_error(format!( + "The data store was created by a Stalwart version prior to 1.0 (schema version {base}) and cannot be opened by this version." + ))) + } else { + Err(schema_error(format!( + "The data store was created by a newer Stalwart version (schema version {base}, expected {DATABASE_SCHEMA_VERSION_BASE})." + ))) + } + } + None if self.is_new_install().await? => self.write_schema_version().await, + None => Err(schema_error( + "The data store was created by a Stalwart version prior to 1.0 and cannot be opened by this version." + .to_string(), + )), + } + } + + async fn write_schema_version(&self) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: Subspace::System, + key: vec![SystemKind::SchemaVersion as u8], + }), + DATABASE_SCHEMA_VERSION.serialize(), + ); + + self.store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!()) + .map(|_| ()) + } + + async fn is_new_install(&self) -> trc::Result { + for subspace in [ + Subspace::QueueMessage, + Subspace::ReportIn, + Subspace::ReportOut, + Subspace::Property, + ] { + let mut has_data = false; + + self.store() + .iterate( + IterateParams::new( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 16], + }, + ) + .no_values() + .only_first(), + |_, _| { + has_data = true; + + Ok(false) + }, + ) + .await + .caused_by(trc::location!())?; + + if has_data { + return Ok(false); + } + } + + Ok(true) + } +} + +fn schema_error(message: String) -> trc::Error { + trc::EventType::Server(trc::ServerEvent::StartupError) + .into_err() + .details(message) +} diff --git a/crates/common/src/network/acme/account.rs b/crates/common/src/network/acme/account.rs index 318ae9ba..c0d5c8ab 100644 --- a/crates/common/src/network/acme/account.rs +++ b/crates/common/src/network/acme/account.rs @@ -5,13 +5,13 @@ */ use super::jose::{Body, eab_sign, sign}; -use crate::network::acme::http::{get_header, https}; +use crate::network::acme::http::{acme_client, get_header, https}; use crate::network::acme::{AcmeError, AcmeResult, Directory}; use aws_lc_rs::rand::SystemRandom; use aws_lc_rs::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, EcdsaSigningAlgorithm}; -use base64::Engine; -use base64::engine::general_purpose::{self, URL_SAFE_NO_PAD}; -use registry::schema::structs::AcmeProvider; +use encodify::base64::URL_SAFE_NO_PAD; +use registry::schema::{enums::AcmeChallengeType, structs::AcmeProvider}; +use registry::types::map::Map; use reqwest::Method; use utils::sanitize_email; @@ -49,7 +49,11 @@ pub async fn acme_create_account( *contact = format!("mailto:{}", email); } - let directory = Directory::discover(&provider.directory, provider.max_retries as u32).await?; + let client = acme_client(provider); + let directory = Directory::discover(&client, &provider.directory).await?; + if provider.challenge_type == AcmeChallengeType::DnsPersist01 { + directory.meta.require_persist_support()?; + } let account_key = EcdsaKeyPair::generate_pkcs8(ALG, &SystemRandom::new()).unwrap(); let key_pair = EcdsaKeyPair::from_pkcs8(ALG, account_key.as_ref()) .map_err(|err| AcmeError::Crypto(format!("Failed to create ECDSA key pair: {}", err)))?; @@ -68,29 +72,26 @@ pub async fn acme_create_account( let body = sign( &key_pair, None, - directory.nonce(provider.max_retries as u32).await?, + directory.nonce(&client).await?, &directory.new_account, &payload, )?; provider.account_uri = get_header( - &https( - &directory.new_account, - Method::POST, - Some(body), - provider.max_retries as u32, - ) - .await?, + &https(&client, &directory.new_account, Method::POST, Some(body)).await?, "Location", )?; provider.account_key = URL_SAFE_NO_PAD.encode(account_key.as_ref()); + provider.account_hash_prefix = directory.meta.account_hash_prefix; + provider.issuer_domain_names = Map::new(directory.meta.issuer_domain_names); + provider.caa_identities = Map::new(directory.meta.caa_identities); Ok(()) } impl EabSettings { pub fn new(kid: impl Into, hmac_key: impl AsRef<[u8]>) -> AcmeResult { - let key = general_purpose::URL_SAFE_NO_PAD + let key = URL_SAFE_NO_PAD .decode(hmac_key.as_ref()) .map_err(|err| AcmeError::Invalid(format!("Failed to decode EAB HMAC key: {}", err)))?; Ok(Self { diff --git a/crates/common/src/network/acme/directory.rs b/crates/common/src/network/acme/directory.rs index 5f5d5bef..6b0f2c04 100644 --- a/crates/common/src/network/acme/directory.rs +++ b/crates/common/src/network/acme/directory.rs @@ -7,16 +7,18 @@ // Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. use super::jose::{ - key_authorization, key_authorization_sha256, key_authorization_sha256_base64, sign, + decode_account_key, key_authorization, key_authorization_sha256, + key_authorization_sha256_base64, sign, +}; +use crate::network::acme::http::{ + acme_client, get_header, https, parse_alternate_links, parse_retry_after, read_body, }; -use crate::network::acme::http::{get_header, https, parse_alternate_links, parse_retry_after}; use crate::network::acme::{ AcmeError, AcmeResult, Auth, AuthStatus, Challenge, ChallengeType, Directory, Identifier, Order, SerializedCert, }; -use aws_lc_rs::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, EcdsaSigningAlgorithm}; -use base64::Engine; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use aws_lc_rs::signature::EcdsaKeyPair; +use encodify::base64::URL_SAFE_NO_PAD; use rcgen::{CustomExtension, KeyPair, PKCS_ECDSA_P256_SHA256}; use registry::schema::structs::AcmeProvider; use reqwest::Method; @@ -25,6 +27,7 @@ use serde_json::json; use std::time::Duration; use store::Serialize; use store::write::Archiver; +use utils::outbound::OutboundClient; pub const ACME_TLS_ALPN_NAME: &[u8] = b"acme-tls/1"; @@ -34,8 +37,8 @@ pub struct AcmeRequestBuilder { pub directory: Directory, pub kid: String, pub challenge: ChallengeType, - pub max_retries: u32, pub preferred_chain: Option, + client: OutboundClient, } pub struct AcmeResponse { @@ -45,29 +48,19 @@ pub struct AcmeResponse { pub alternates: Vec, } -static ALG: &EcdsaSigningAlgorithm = &ECDSA_P256_SHA256_FIXED_SIGNING; - impl AcmeRequestBuilder { pub async fn new(provider: AcmeProvider) -> AcmeResult { - let directory = - Directory::discover(&provider.directory, provider.max_retries as u32).await?; - let key_pair = EcdsaKeyPair::from_pkcs8( - ALG, - &URL_SAFE_NO_PAD - .decode(&provider.account_key) - .map_err(|err| { - AcmeError::Crypto(format!("Failed to decode account key: {}", err)) - })?, - ) - .map_err(|err| AcmeError::Crypto(format!("Failed to create ECDSA key pair: {}", err)))?; + let client = acme_client(&provider); + let directory = Directory::discover(&client, &provider.directory).await?; + let key_pair = decode_account_key(&provider.account_key)?; Ok(Self { key_pair, directory, kid: provider.account_uri, challenge: provider.challenge_type.into(), - max_retries: provider.max_retries as u32, preferred_chain: provider.preferred_chain, + client, }) } @@ -79,17 +72,17 @@ impl AcmeRequestBuilder { let body = sign( &self.key_pair, Some(&self.kid), - self.directory.nonce(self.max_retries).await?, + self.directory.nonce(&self.client).await?, url.as_ref(), payload, )?; - let response = https(url.as_ref(), Method::POST, Some(body), self.max_retries).await?; + let response = https(&self.client, url.as_ref(), Method::POST, Some(body)).await?; Ok(AcmeResponse { location: get_header(&response, "Location").ok(), retry_after: parse_retry_after(&response), alternates: parse_alternate_links(&response), - body: response.text().await?, + body: read_body(response).await?, }) } @@ -134,7 +127,9 @@ impl AcmeRequestBuilder { url: impl AsRef, csr: Vec, ) -> AcmeResult, Order>> { - let payload = format!("{{\"csr\":\"{}\"}}", URL_SAFE_NO_PAD.encode(csr)); + let mut payload = String::from("{\"csr\":\""); + URL_SAFE_NO_PAD.encode_append(csr, &mut payload); + payload.push_str("\"}"); AcmeResponse::parse(self.request(&url, &payload).await?) } @@ -188,19 +183,17 @@ impl AcmeRequestBuilder { } impl Directory { - pub async fn discover(url: impl AsRef, max_retries: u32) -> AcmeResult { - serde_json::from_str( - &https(url, Method::GET, None, max_retries) - .await? - .text() - .await?, - ) - .map_err(Into::into) + pub async fn discover(client: &OutboundClient, url: impl AsRef) -> AcmeResult { + let mut directory: Directory = serde_json::from_str( + &read_body(https(client, url.as_ref(), Method::GET, None).await?).await?, + )?; + directory.meta.normalize(); + Ok(directory) } - pub async fn nonce(&self, max_retries: u32) -> AcmeResult { + pub async fn nonce(&self, client: &OutboundClient) -> AcmeResult { get_header( - &https(&self.new_nonce.as_str(), Method::HEAD, None, max_retries).await?, + &https(client, &self.new_nonce, Method::HEAD, None).await?, "replay-nonce", ) } @@ -225,7 +218,7 @@ impl AcmeResponse { } impl AcmeResponse { - pub fn assert_reasonable_retry_after(self, max_retries: u32) -> AcmeResult { + pub fn assert_reasonable_retry_after(self) -> AcmeResult { if let Some(retry_after) = self.retry_after && retry_after > Duration::from_secs(10 * 60) { @@ -236,7 +229,6 @@ impl AcmeResponse { ); return Err(AcmeError::Backoff { - max_retries, wait: retry_after.into(), }); } diff --git a/crates/common/src/network/acme/http.rs b/crates/common/src/network/acme/http.rs index 9ed63db8..a008686c 100644 --- a/crates/common/src/network/acme/http.rs +++ b/crates/common/src/network/acme/http.rs @@ -5,37 +5,51 @@ */ use crate::network::acme::{AcmeError, AcmeResult}; -use chrono::{DateTime, Utc}; +use compact_str::CompactString; use hyper::{ Method, StatusCode, header::{CONTENT_TYPE, USER_AGENT}, }; +use jiff::{Timestamp, fmt::rfc2822::DateTimeParser}; +use registry::schema::structs::AcmeProvider; use reqwest::Response; use std::time::Duration; +use utils::{ + HttpLimitResponse, + outbound::{OutboundClient, OutboundPolicy, Redirects}, +}; + +const MAX_ERROR_BODY_LEN: usize = 512; +const MAX_RESPONSE_LEN: usize = 1024 * 1024; +const MAX_REDIRECTS: usize = 10; + +#[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] +pub(crate) fn acme_client(provider: &AcmeProvider) -> OutboundClient { + #[cfg(any(feature = "dev_mode", stalwart_tests))] + let allow_invalid_certs = provider.directory.starts_with("https://localhost") + || provider.directory.starts_with("https://127.0.0.1"); + #[cfg(not(any(feature = "dev_mode", stalwart_tests)))] + let allow_invalid_certs = false; + + #[cfg(not(feature = "enterprise"))] + let policy = OutboundPolicy::Trusted; + + OutboundClient::new( + utils::http::http1_client_builder(allow_invalid_certs).timeout(Duration::from_secs(30)), + policy, + Redirects::Limited(MAX_REDIRECTS), + ) +} -#[allow(unused_mut)] pub(crate) async fn https( - url: impl AsRef, + client: &OutboundClient, + url: &str, method: Method, body: Option, - max_retries: u32, ) -> AcmeResult { - let url = url.as_ref(); - - #[allow(unused_mut)] - #[allow(unused_assignments)] - let mut allow_invalid_certs = false; - - #[cfg(any(feature = "dev_mode", feature = "test_mode"))] - { - allow_invalid_certs = - url.starts_with("https://localhost") || url.starts_with("https://127.0.0.1"); - } - - let mut request = utils::http::http1_client_builder(allow_invalid_certs) - .timeout(Duration::from_secs(30)) - .build()? + let mut request = client .request(method, url) + .map_err(|err| AcmeError::Invalid(format!("Rejected ACME URL {url}: {err}")))? .header(USER_AGENT, crate::USER_AGENT); if let Some(body) = body { @@ -55,15 +69,15 @@ pub(crate) async fn https( trc::event!( Acme(trc::AcmeEvent::RenewBackoff), - Url = url.to_string(), + Url = CompactString::from(url), Code = response.status().as_u16(), Elapsed = wait.unwrap_or_default(), ); - Err(AcmeError::Backoff { wait, max_retries }) + Err(AcmeError::Backoff { wait }) } else { let status = response.status(); - let text = response.text().await.unwrap_or_default(); + let text = read_body_prefix(response).await; Err(AcmeError::HttpStatus(format!( "Unexpected status {}: {}", status, text @@ -71,6 +85,26 @@ pub(crate) async fn https( } } +pub(crate) async fn read_body(response: Response) -> AcmeResult { + let body = response + .bytes_with_limit(MAX_RESPONSE_LEN) + .await? + .ok_or_else(|| AcmeError::Invalid("ACME response exceeds maximum size".to_string()))?; + String::from_utf8(body) + .map_err(|_| AcmeError::Invalid("ACME response is not valid UTF-8".to_string())) +} + +async fn read_body_prefix(mut response: Response) -> String { + let mut prefix = Vec::with_capacity(MAX_ERROR_BODY_LEN); + while prefix.len() < MAX_ERROR_BODY_LEN + && let Ok(Some(chunk)) = response.chunk().await + { + let remaining = MAX_ERROR_BODY_LEN - prefix.len(); + prefix.extend_from_slice(chunk.get(..remaining).unwrap_or(&chunk)); + } + String::from_utf8_lossy(&prefix).into_owned() +} + pub(crate) fn get_header(response: &Response, header: &'static str) -> AcmeResult { match response.headers().get_all(header).iter().next_back() { Some(value) => Ok(value @@ -118,83 +152,20 @@ fn alternate_links<'a>(values: impl Iterator) -> Vec { } pub(crate) fn parse_retry_after(response: &Response) -> Option { - let value = response.headers().get("Retry-After")?.to_str().ok()?; + retry_after( + response.headers().get("Retry-After")?.to_str().ok()?, + Timestamp::now(), + ) +} + +fn retry_after(value: &str, now: Timestamp) -> Option { if let Ok(secs) = value.parse::() { Some(Duration::from_secs(secs + 1)) - } else if let Ok(dt) = DateTime::parse_from_rfc2822(value) { - Utc::now() - .signed_duration_since(dt.with_timezone(&Utc)) - .to_std() - .map(|dur| dur + Duration::from_secs(1)) - .ok() } else { - None + let retry_at = DateTimeParser::new().parse_timestamp(value).ok()?; + Duration::try_from(retry_at.duration_since(now)) + .ok() + .map(|delay| delay + Duration::from_secs(1)) } } -#[cfg(test)] -mod tests { - use super::alternate_links; - - #[test] - fn parses_single_alternate_link() { - let links = - alternate_links([r#";rel="alternate""#].into_iter()); - assert_eq!(links, vec!["https://acme.example/cert/1/1".to_string()]); - } - - #[test] - fn parses_multiple_alternates_in_one_header() { - let links = alternate_links( - [r#";rel="alternate", ;rel="alternate""#] - .into_iter(), - ); - assert_eq!( - links, - vec![ - "https://acme.example/cert/1/1".to_string(), - "https://acme.example/cert/1/2".to_string(), - ] - ); - } - - #[test] - fn parses_alternates_across_multiple_headers() { - let links = alternate_links( - [ - r#";rel="alternate""#, - r#";rel="alternate""#, - ] - .into_iter(), - ); - assert_eq!( - links, - vec![ - "https://acme.example/cert/1/1".to_string(), - "https://acme.example/cert/1/2".to_string(), - ] - ); - } - - #[test] - fn ignores_non_alternate_relations() { - let links = alternate_links( - [r#";rel="index", ;rel="alternate""#] - .into_iter(), - ); - assert_eq!(links, vec!["https://acme.example/cert/1/1".to_string()]); - } - - #[test] - fn tolerates_unquoted_rel_and_extra_whitespace() { - let links = - alternate_links([r#" ; rel=alternate "#].into_iter()); - assert_eq!(links, vec!["https://acme.example/cert/1/1".to_string()]); - } - - #[test] - fn returns_empty_when_no_alternates() { - let links = alternate_links([r#";rel="index""#].into_iter()); - assert!(links.is_empty()); - } -} diff --git a/crates/common/src/network/acme/jose.rs b/crates/common/src/network/acme/jose.rs index b233c350..235137f7 100644 --- a/crates/common/src/network/acme/jose.rs +++ b/crates/common/src/network/acme/jose.rs @@ -10,9 +10,8 @@ use crate::network::acme::{AcmeError, AcmeResult}; use aws_lc_rs::digest::{Digest, SHA256, digest}; use aws_lc_rs::hmac; use aws_lc_rs::rand::SystemRandom; -use aws_lc_rs::signature::{EcdsaKeyPair, KeyPair}; -use base64::Engine; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use aws_lc_rs::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, KeyPair}; +use encodify::base64::URL_SAFE_NO_PAD; use serde::Serialize; pub(crate) fn sign( @@ -62,12 +61,22 @@ pub(crate) fn eab_sign( }) } +pub(crate) fn decode_account_key(encoded: &str) -> AcmeResult { + EcdsaKeyPair::from_pkcs8( + &ECDSA_P256_SHA256_FIXED_SIGNING, + &URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|err| AcmeError::Crypto(format!("Failed to decode account key: {}", err)))?, + ) + .map_err(|err| AcmeError::Crypto(format!("Failed to create ECDSA key pair: {}", err))) +} + +pub(crate) fn account_thumbprint(key: &EcdsaKeyPair) -> AcmeResult { + Jwk::new(key).thumb_sha256_base64() +} + pub(crate) fn key_authorization(key: &EcdsaKeyPair, token: &str) -> AcmeResult { - Ok(format!( - "{}.{}", - token, - Jwk::new(key).thumb_sha256_base64()? - )) + Ok(format!("{}.{}", token, account_thumbprint(key)?)) } pub(crate) fn key_authorization_sha256(key: &EcdsaKeyPair, token: &str) -> AcmeResult { diff --git a/crates/common/src/network/acme/mod.rs b/crates/common/src/network/acme/mod.rs index 455b1d5a..184932dc 100644 --- a/crates/common/src/network/acme/mod.rs +++ b/crates/common/src/network/acme/mod.rs @@ -9,12 +9,12 @@ pub mod directory; pub mod http; pub mod jose; pub mod order; +pub mod persist; pub mod renew; pub mod resolver; use crate::network::dns::update::DnsUpdater; -use chrono::{DateTime, Utc}; -use registry::schema::enums::AcmeChallengeType; +use registry::{schema::enums::AcmeChallengeType, types::datetime::UTCDateTime}; use rustls::sign::CertifiedKey; use serde::Deserialize; use std::{ @@ -32,7 +32,10 @@ pub enum AcmeError { Json(serde_json::Error), Crypto(String), Invalid(String), - NotDue(String), + NotDue { + domain: String, + renew_at: u64, + }, Dns(String), AuthInvalid(String), OrderInvalid(String), @@ -42,14 +45,9 @@ pub enum AcmeError { }, Internal(trc::Error), Registry(RegistryWriteResult), - OrderTimeout { - max_retries: u32, - }, - AuthTimeout { - max_retries: u32, - }, + OrderTimeout, + AuthTimeout, Backoff { - max_retries: u32, wait: Option, }, } @@ -62,6 +60,10 @@ pub struct SerializedCert { pub private_key: Vec, } +impl store::write::ArchiveCompression for SerializedCert { + const COMPRESSION: store::write::Compression = store::write::Compression::None; +} + pub struct PemCert { pub certificate: String, pub private_key: String, @@ -70,8 +72,8 @@ pub struct PemCert { pub struct ParsedCert { pub sans: Vec, pub issuer: String, - pub valid_not_before: DateTime, - pub valid_not_after: DateTime, + pub valid_not_before: UTCDateTime, + pub valid_not_after: UTCDateTime, } #[derive(Debug, Clone, Deserialize)] @@ -80,6 +82,19 @@ pub struct Directory { pub new_nonce: String, pub new_account: String, pub new_order: String, + #[serde(default)] + pub meta: DirectoryMeta, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DirectoryMeta { + #[serde(default)] + pub caa_identities: Vec, + #[serde(default)] + pub issuer_domain_names: Vec, + #[serde(default)] + pub account_hash_prefix: Option, } #[derive(Debug, Deserialize, Eq, PartialEq, Clone, Copy)] @@ -154,6 +169,8 @@ pub struct Challenge { pub url: String, pub token: Option, pub error: Option, + #[serde(default, rename = "issuerDomainNames")] + pub issuer_domain_names: Vec, } #[derive(Clone, Debug, serde::Serialize, Deserialize)] @@ -266,11 +283,15 @@ impl Display for AcmeError { AcmeError::Dns(err) => write!(f, "DNS error: {}", err), AcmeError::Crypto(err) => write!(f, "Cryptographic error: {}", err), AcmeError::Invalid(err) => write!(f, "Invalid request: {}", err), - AcmeError::NotDue(err) => write!(f, "{}", err), + AcmeError::NotDue { domain, renew_at } => write!( + f, + "Certificate for domain {domain} is still valid; renewal is not due until {}", + UTCDateTime::from_timestamp(*renew_at as i64) + ), AcmeError::AuthInvalid(status) => write!(f, "Authentication failed: {:?}", status), - AcmeError::OrderTimeout { .. } => write!(f, "Order processing timed out"), + AcmeError::OrderTimeout => write!(f, "Order processing timed out"), AcmeError::OrderInvalid(reason) => write!(f, "Order is invalid: {}", reason), - AcmeError::AuthTimeout { .. } => write!(f, "Authentication timed out"), + AcmeError::AuthTimeout => write!(f, "Authentication timed out"), AcmeError::ChallengeNotSupported { requested, supported, @@ -283,7 +304,7 @@ impl Display for AcmeError { } AcmeError::Internal(err) => write!(f, "Internal error: {}", err), AcmeError::Registry(err) => write!(f, "Registry error: {:?}", err), - AcmeError::Backoff { wait, .. } => { + AcmeError::Backoff { wait } => { if let Some(time) = wait { write!(f, "Rate limited. Retry after {} seconds", time.as_secs()) } else { diff --git a/crates/common/src/network/acme/order.rs b/crates/common/src/network/acme/order.rs index 9073c21b..02824c76 100644 --- a/crates/common/src/network/acme/order.rs +++ b/crates/common/src/network/acme/order.rs @@ -12,10 +12,11 @@ use crate::network::acme::{ ParsedCert, PemCert, }; use crate::{KV_ACME, Server}; -use chrono::{TimeZone, Utc}; +use compact_str::{CompactString, ToCompactString, format_compact}; use dns_update::DnsRecord; use futures::future::try_join_all; use rcgen::{CertificateParams, DistinguishedName, KeyPair, PKCS_ECDSA_P256_SHA256}; +use registry::types::datetime::UTCDateTime; use std::collections::BTreeSet; use std::net::{Ipv4Addr, Ipv6Addr}; use std::time::Duration; @@ -25,6 +26,8 @@ use x509_parser::parse_x509_certificate; use x509_parser::prelude::{GeneralName, ParsedExtension}; const HOSTNAMES: &[&str] = &["mta-sts", "ua-auto-config", "autoconfig", "autodiscover"]; +const ORDER_PROCESSING_ATTEMPTS: u64 = 10; +const AUTH_ATTEMPTS: u64 = 5; impl AcmeRequestBuilder { pub fn build_domains( @@ -78,13 +81,7 @@ impl AcmeRequestBuilder { } else { hostnames .iter() - .map(|h| { - if h.contains('.') { - h.clone() - } else { - format!("{h}.{domain}") - } - }) + .map(|h| qualify_hostname(h, domain)) .collect() } } @@ -146,8 +143,8 @@ impl AcmeRequestBuilder { trc::event!( Acme(AcmeEvent::OrderStart), - Url = self.directory.new_order.to_string(), - Details = order_url.to_string(), + Url = CompactString::from(&self.directory.new_order), + Details = CompactString::from(&order_url), Hostname = domains, Type = self.challenge.as_str(), ); @@ -169,7 +166,7 @@ impl AcmeRequestBuilder { } trc::event!( Acme(AcmeEvent::AuthCompleted), - Url = self.directory.new_order.to_string(), + Url = CompactString::from(&self.directory.new_order), Hostname = domains, ); let response = self.order(&order_url).await?; @@ -177,12 +174,13 @@ impl AcmeRequestBuilder { retry_after = response.retry_after; } OrderStatus::Processing => { - for i in 0u64..10 { + for i in 0u64..ORDER_PROCESSING_ATTEMPTS { trc::event!( Acme(AcmeEvent::OrderProcessing), - Url = self.directory.new_order.to_string(), + Url = CompactString::from(&self.directory.new_order), Hostname = domains, - Total = i, + Attempt = i + 1, + Limit = ORDER_PROCESSING_ATTEMPTS, ); tokio::time::sleep( @@ -192,7 +190,7 @@ impl AcmeRequestBuilder { let response = self .order(&order_url) .await? - .assert_reasonable_retry_after(self.max_retries)?; + .assert_reasonable_retry_after()?; order = response.body; retry_after = response.retry_after; if order.status != OrderStatus::Processing { @@ -200,15 +198,13 @@ impl AcmeRequestBuilder { } } if order.status == OrderStatus::Processing { - return Err(AcmeError::OrderTimeout { - max_retries: self.max_retries, - }); + return Err(AcmeError::OrderTimeout); } } OrderStatus::Ready => { trc::event!( Acme(AcmeEvent::OrderReady), - Url = self.directory.new_order.to_string(), + Url = CompactString::from(&self.directory.new_order), Hostname = domains, ); @@ -221,7 +217,7 @@ impl AcmeRequestBuilder { OrderStatus::Valid { certificate } => { trc::event!( Acme(AcmeEvent::OrderValid), - Url = self.directory.new_order.to_string(), + Url = CompactString::from(&self.directory.new_order), Hostname = domains, ); @@ -241,8 +237,8 @@ impl AcmeRequestBuilder { trc::event!( Acme(AcmeEvent::OrderInvalid), - Url = self.directory.new_order.to_string(), - Details = order_url.to_string(), + Url = CompactString::from(&self.directory.new_order), + Details = order_url, Hostname = domains, Reason = reason.clone(), ); @@ -260,10 +256,7 @@ impl AcmeRequestBuilder { dns_parameters: Option<&AcmeDnsParameters>, published: Option<&mut BTreeSet<(String, String)>>, ) -> AcmeResult<()> { - let response = self - .auth(url) - .await? - .assert_reasonable_retry_after(self.max_retries)?; + let response = self.auth(url).await?.assert_reasonable_retry_after()?; let mut retry_after = response.retry_after; let auth = response.body; @@ -273,9 +266,9 @@ impl AcmeRequestBuilder { trc::event!( Acme(AcmeEvent::AuthStart), - Hostname = domain.to_string(), + Hostname = CompactString::from(&domain), Type = self.challenge.as_str(), - Url = self.directory.new_order.to_string(), + Url = CompactString::from(&self.directory.new_order), ); let challenge = auth @@ -345,7 +338,9 @@ impl AcmeRequestBuilder { vec![DnsRecord::TXT(proof.clone())], ) .await - .map_err(AcmeError::Dns)?; + .map_err(|err| { + AcmeError::Dns(format!("Failed to set DNS RRSet: {err}")) + })?; if let Some(published) = published { published.insert((zone.to_string(), challenge_name.clone())); @@ -356,7 +351,9 @@ impl AcmeRequestBuilder { .wait_for_txt_propagation(&challenge_name, zone, &proof) .await; } - ChallengeType::DnsPersist01 => {} + ChallengeType::DnsPersist01 => { + self.directory.meta.verify_persist_challenge(challenge)?; + } ChallengeType::Unknown => unreachable!(), } @@ -367,10 +364,10 @@ impl AcmeRequestBuilder { _ => { trc::event!( Acme(AcmeEvent::AuthError), - Hostname = auth.identifier.hostname().to_string(), + Hostname = CompactString::from(auth.identifier.hostname()), Type = self.challenge.as_str(), - Url = self.directory.new_order.to_string(), - Details = url.to_string(), + Url = CompactString::from(&self.directory.new_order), + Details = CompactString::from(url), Reason = auth.to_error(), ); @@ -378,21 +375,19 @@ impl AcmeRequestBuilder { } }; - for i in 0u64..5 { + for i in 0u64..AUTH_ATTEMPTS { tokio::time::sleep(retry_after.unwrap_or_else(|| Duration::from_secs(1u64 << i))).await; - let response = self - .auth(url) - .await? - .assert_reasonable_retry_after(self.max_retries)?; + let response = self.auth(url).await?.assert_reasonable_retry_after()?; retry_after = response.retry_after; match response.body.status { AuthStatus::Pending => { trc::event!( Acme(AcmeEvent::AuthPending), - Hostname = domain.to_string(), - Url = self.directory.new_order.to_string(), - Total = i, + Hostname = CompactString::from(&domain), + Url = CompactString::from(&self.directory.new_order), + Attempt = i + 1, + Limit = AUTH_ATTEMPTS, ); self.challenge(&challenge_url).await? @@ -400,8 +395,8 @@ impl AcmeRequestBuilder { AuthStatus::Valid => { trc::event!( Acme(AcmeEvent::AuthValid), - Hostname = domain.to_string(), - Url = self.directory.new_order.to_string(), + Hostname = domain, + Url = CompactString::from(&self.directory.new_order), ); return Ok(()); @@ -409,10 +404,10 @@ impl AcmeRequestBuilder { _ => { trc::event!( Acme(AcmeEvent::AuthError), - Hostname = domain.to_string(), + Hostname = CompactString::from(domain), Type = self.challenge.as_str(), - Url = self.directory.new_order.to_string(), - Details = url.to_string(), + Url = CompactString::from(&self.directory.new_order), + Details = CompactString::from(url), Reason = response.body.to_error(), ); @@ -423,16 +418,15 @@ impl AcmeRequestBuilder { trc::event!( Acme(AcmeEvent::AuthTooManyAttempts), - Hostname = domain.to_string(), + Hostname = CompactString::from(&domain), Type = self.challenge.as_str(), - Url = self.directory.new_order.to_string(), - Details = url.to_string(), - Total = 5u64, + Url = CompactString::from(&self.directory.new_order), + Details = CompactString::from(url), + Attempt = AUTH_ATTEMPTS, + Limit = AUTH_ATTEMPTS, ); - Err(AcmeError::AuthTimeout { - max_retries: self.max_retries, - }) + Err(AcmeError::AuthTimeout) } async fn select_certificate(&self, domains: &[String], url: String) -> AcmeResult { @@ -454,9 +448,9 @@ impl AcmeRequestBuilder { Err(err) => { trc::event!( Acme(AcmeEvent::ProcessCert), - Url = alternate.to_string(), + Url = CompactString::from(alternate), Hostname = domains, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -465,7 +459,7 @@ impl AcmeRequestBuilder { trc::event!( Acme(AcmeEvent::ProcessCert), Hostname = domains, - Reason = format!( + Reason = format_compact!( "Preferred certificate chain '{preferred}' not offered by the CA; using the default chain", ), ); @@ -474,6 +468,14 @@ impl AcmeRequestBuilder { } } +pub fn qualify_hostname(hostname: &str, domain: &str) -> String { + if hostname.contains('.') { + hostname.to_string() + } else { + format!("{hostname}.{domain}") + } +} + fn chain_matches(pem_chain: &str, preferred: &str) -> bool { let Ok(blocks) = pem::parse_many(pem_chain) else { return false; @@ -503,7 +505,7 @@ impl ParsedCert { pub fn parse_der(der: &[u8]) -> AcmeResult { parse_x509_certificate(der) .map_err(|err| AcmeError::Crypto(format!("Failed to parse X.509 certificate: {}", err))) - .and_then(|(_, cert)| { + .map(|(_, cert)| { // Add CNs and SANs to the list of names let mut names: BTreeSet = BTreeSet::new(); for name in cert.subject().iter_common_name() { @@ -532,79 +534,17 @@ impl ParsedCert { } } - Ok(ParsedCert { + ParsedCert { sans: names.into_iter().collect(), issuer: cert.tbs_certificate.issuer().to_string(), - valid_not_before: Utc - .timestamp_opt(cert.tbs_certificate.validity().not_before.timestamp(), 0) - .single() - .ok_or_else(|| { - AcmeError::Crypto( - "Certificate not_before time is out of range".to_string(), - ) - })?, - valid_not_after: Utc - .timestamp_opt(cert.tbs_certificate.validity().not_after.timestamp(), 0) - .single() - .ok_or_else(|| { - AcmeError::Crypto( - "Certificate not_after time is out of range".to_string(), - ) - })?, - }) + valid_not_before: UTCDateTime::from_timestamp( + cert.tbs_certificate.validity().not_before.timestamp(), + ), + valid_not_after: UTCDateTime::from_timestamp( + cert.tbs_certificate.validity().not_after.timestamp(), + ), + } }) } } -#[cfg(test)] -mod tests { - use super::chain_matches; - use rcgen::{CertificateParams, DistinguishedName, DnType, KeyPair, PKCS_ECDSA_P256_SHA256}; - - fn self_signed_pem(common_name: &str) -> String { - let mut params = CertificateParams::new(vec!["host.example".to_string()]).unwrap(); - let mut dn = DistinguishedName::new(); - dn.push(DnType::CommonName, common_name); - params.distinguished_name = dn; - let key_pair = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); - params.self_signed(&key_pair).unwrap().pem() - } - - #[test] - fn matches_top_certificate_issuer() { - let chain = self_signed_pem("ISRG Root X1"); - assert!(chain_matches(&chain, "ISRG Root X1")); - } - - #[test] - fn match_is_case_sensitive() { - let chain = self_signed_pem("ISRG Root X1"); - assert!(!chain_matches(&chain, "isrg root x1")); - } - - #[test] - fn match_is_exact_not_substring() { - let chain = self_signed_pem("ISRG Root X10"); - assert!(!chain_matches(&chain, "ISRG Root X1")); - } - - #[test] - fn does_not_match_unrelated_issuer() { - let chain = self_signed_pem("ISRG Root X2"); - assert!(!chain_matches(&chain, "ISRG Root X1")); - } - - #[test] - fn uses_topmost_certificate_not_leaf() { - let leaf = self_signed_pem("Leaf Issuer"); - let top = self_signed_pem("ISRG Root X1"); - let chain = format!("{leaf}{top}"); - assert!(chain_matches(&chain, "ISRG Root X1")); - assert!(!chain_matches(&chain, "Leaf Issuer")); - } - - #[test] - fn rejects_unparseable_chain() { - assert!(!chain_matches("not a pem", "ISRG Root X1")); - } -} diff --git a/crates/common/src/network/acme/persist.rs b/crates/common/src/network/acme/persist.rs new file mode 100644 index 00000000..3da5f5e1 --- /dev/null +++ b/crates/common/src/network/acme/persist.rs @@ -0,0 +1,195 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::jose::{account_thumbprint, decode_account_key}; +use super::order::qualify_hostname; +use crate::network::acme::{AcmeError, AcmeResult, Challenge, DirectoryMeta}; +use aws_lc_rs::digest::{Context, SHA256}; +use encodify::base64::URL_SAFE_NO_PAD; +use registry::schema::structs::AcmeProvider; +use reqwest::Url; +use std::collections::BTreeMap; + +pub const MAX_ISSUER_DOMAIN_NAMES: usize = 10; +const MAX_CAA_IDENTITIES: usize = 32; +const MAX_DOMAIN_NAME_LEN: usize = 253; +const HASH_ALGORITHM: &str = "sha-256"; +const WILDCARD_POLICY: &str = "; policy=wildcard"; + +pub struct PersistRecordBuilder<'x> { + issuer: &'x str, + prefix: &'x str, + thumbprint: String, + account_url: &'x str, +} + +pub fn normalize_domain_name(name: &str) -> Option { + let name = name.trim(); + let name = name.strip_suffix('.').unwrap_or(name); + idna::domain_to_ascii(name) + .ok() + .filter(|name| !name.is_empty() && name.len() <= MAX_DOMAIN_NAME_LEN) +} + +fn normalize_domain_names(names: &mut Vec, max_names: usize) { + let is_valid = names.len() <= max_names + && names + .iter_mut() + .all(|name| match normalize_domain_name(name) { + Some(normalized) => { + *name = normalized; + true + } + None => false, + }); + if !is_valid { + names.clear(); + } +} + +impl DirectoryMeta { + pub(crate) fn normalize(&mut self) { + normalize_domain_names(&mut self.issuer_domain_names, MAX_ISSUER_DOMAIN_NAMES); + normalize_domain_names(&mut self.caa_identities, MAX_CAA_IDENTITIES); + self.account_hash_prefix = self + .account_hash_prefix + .take() + .filter(|prefix| !prefix.trim().is_empty() && Url::parse(prefix.trim()).is_ok()) + .map(|prefix| prefix.trim().to_string()); + } + + pub fn require_persist_support(&self) -> AcmeResult<()> { + if self.account_hash_prefix.is_none() || self.issuer_domain_names.is_empty() { + Err(AcmeError::Invalid( + "The ACME server does not support DNS-PERSIST-01: its directory does not advertise both accountHashPrefix and issuerDomainNames".to_string(), + )) + } else { + Ok(()) + } + } + + pub fn verify_persist_challenge(&self, challenge: &Challenge) -> AcmeResult<()> { + self.require_persist_support()?; + let offered = &challenge.issuer_domain_names; + if offered.is_empty() || offered.len() > MAX_ISSUER_DOMAIN_NAMES { + return Err(AcmeError::Invalid(format!( + "Malformed dns-persist-01 challenge: expected between 1 and {MAX_ISSUER_DOMAIN_NAMES} issuer domain names, found {}", + offered.len() + ))); + } + let mut normalized = Vec::with_capacity(offered.len()); + for name in offered { + normalized.push(normalize_domain_name(name).ok_or_else(|| { + AcmeError::Invalid(format!( + "Malformed dns-persist-01 challenge: invalid issuer domain name {name:?}" + )) + })?); + } + if let Some(missing) = self + .issuer_domain_names + .iter() + .find(|name| !normalized.contains(name)) + { + return Err(AcmeError::Invalid(format!( + "Malformed dns-persist-01 challenge: directory issuer domain name {missing:?} is not offered by the challenge" + ))); + } + if !self.caa_identities.is_empty() + && let Some(undisclosed) = normalized + .iter() + .find(|name| !self.caa_identities.contains(name)) + { + return Err(AcmeError::Invalid(format!( + "Malformed dns-persist-01 challenge: issuer domain name {undisclosed:?} is not listed in the directory CAA identities" + ))); + } + Ok(()) + } +} + +impl<'x> PersistRecordBuilder<'x> { + pub fn new(provider: &'x AcmeProvider) -> AcmeResult { + let (Some(issuer), Some(prefix)) = ( + provider.issuer_domain_names.as_slice().first(), + provider.account_hash_prefix.as_deref(), + ) else { + return Err(AcmeError::Invalid(format!( + "ACME provider {} does not advertise both accountHashPrefix and issuerDomainNames, DNS-PERSIST-01 records cannot be built", + provider.directory + ))); + }; + Ok(Self { + issuer, + prefix, + thumbprint: account_thumbprint(&decode_account_key(&provider.account_key)?)?, + account_url: &provider.account_uri, + }) + } + + pub fn build(&self, fqdn: &str, wildcard: bool) -> Option { + let mut record = String::with_capacity(160); + record.push_str(self.issuer); + record.push_str("; accounturi="); + record.push_str(self.prefix); + record.push_str(HASH_ALGORITHM); + record.push('/'); + URL_SAFE_NO_PAD.encode_append( + account_hash( + &normalize_domain_name(fqdn)?, + &self.thumbprint, + self.account_url, + )?, + &mut record, + ); + if wildcard { + record.push_str(WILDCARD_POLICY); + } + Some(record) + } +} + +fn account_hash(domain: &str, thumbprint: &str, account_url: &str) -> Option> { + let domain_len = u8::try_from(domain.len()).ok()?; + let mut context = Context::new(&SHA256); + context.update(&[domain_len]); + context.update(domain.as_bytes()); + context.update(thumbprint.as_bytes()); + context.update(account_url.as_bytes()); + Some(context.finish()) +} + +pub fn persist_validation_names( + domain: &str, + subject_alternative_names: &[String], +) -> BTreeMap { + let mut names = BTreeMap::new(); + let Some(domain) = normalize_domain_name(domain) else { + return names; + }; + if subject_alternative_names.is_empty() { + names.insert(domain, true); + return names; + } + for name in subject_alternative_names { + let name = qualify_hostname(name, &domain); + let (base, wildcard) = match name.strip_prefix("*.") { + Some(base) => (base, true), + None => (name.as_str(), false), + }; + let Some(base) = normalize_domain_name(base) else { + continue; + }; + if base == domain + || base + .strip_suffix(domain.as_str()) + .is_some_and(|prefix| prefix.ends_with('.')) + { + *names.entry(base).or_insert(false) |= wildcard; + } + } + names +} + diff --git a/crates/common/src/network/acme/renew.rs b/crates/common/src/network/acme/renew.rs index a829456e..79fce0e4 100644 --- a/crates/common/src/network/acme/renew.rs +++ b/crates/common/src/network/acme/renew.rs @@ -11,17 +11,18 @@ use crate::{ AcmeDnsParameters, AcmeError, AcmeResult, ParsedCert, directory::AcmeRequestBuilder, }, }; +use compact_str::ToCompactString; use registry::{ schema::{ enums::{AcmeChallengeType, AcmeRenewBefore, DnsRecordType}, - prelude::{ObjectType, Property}, + prelude::{ObjectInner, ObjectType, Property}, structs::{ AcmeProvider, Certificate, CertificateManagement, DnsManagement, Domain, PublicText, PublicTextValue, SecretText, SecretTextValue, SystemSettings, Task, TaskDnsManagement, TaskDomainManagement, TaskStatus, }, }, - types::{datetime::UTCDateTime, id::ObjectId, map::Map}, + types::{id::ObjectId, map::Map}, }; use store::{ registry::{ @@ -48,9 +49,12 @@ impl Server { } CertificateManagement::Automatic(props) => props, }; - let Some(acme_provider) = self + let Some(provider_object) = self .registry() - .object::(cert.acme_provider_id) + .get(ObjectId::new( + ObjectType::AcmeProvider, + cert.acme_provider_id, + )) .await? else { return Err(AcmeError::Invalid(format!( @@ -58,10 +62,49 @@ impl Server { cert.acme_provider_id ))); }; + let acme_provider = AcmeProvider::from(provider_object.clone()); let challenge_type = acme_provider.challenge_type; let renew_before = acme_provider.renew_before; let reuse_key = acme_provider.reuse_key; let request = AcmeRequestBuilder::new(acme_provider).await?; + if let ObjectInner::AcmeProvider(stored) = &provider_object.inner { + let meta = &request.directory.meta; + if stored.account_hash_prefix != meta.account_hash_prefix + || stored.issuer_domain_names.as_slice() != meta.issuer_domain_names.as_slice() + || stored.caa_identities.as_slice() != meta.caa_identities.as_slice() + { + let mut updated = stored.as_ref().clone(); + updated.account_hash_prefix = meta.account_hash_prefix.clone(); + updated.issuer_domain_names = Map::new(meta.issuer_domain_names.clone()); + updated.caa_identities = Map::new(meta.caa_identities.clone()); + match self + .registry() + .write(RegistryWrite::update( + cert.acme_provider_id, + &updated.into(), + &provider_object, + )) + .await + { + Ok(RegistryWriteResult::Success(_)) => {} + Ok(result) => { + trc::error!( + trc::AcmeEvent::Error + .into_err() + .details("Failed to update ACME provider directory metadata.") + .reason(format!("{result:?}")) + .caused_by(trc::location!()) + ); + } + Err(err) => { + trc::error!( + err.details("Failed to update ACME provider directory metadata.") + .caused_by(trc::location!()) + ); + } + } + } + } let domains = request.build_domains( self, &domain.name, @@ -72,11 +115,10 @@ impl Server { .acme_certificate_renewal_due(&domains, renew_before, now()) .await? { - return Err(AcmeError::NotDue(format!( - "Certificate for domain {} is still valid; renewal is not due until {}", - domain.name, - UTCDateTime::from_timestamp(renew_at as i64) - ))); + return Err(AcmeError::NotDue { + domain: domain.name, + renew_at, + }); } let dns_parameters = match &domain.dns_management { @@ -132,8 +174,8 @@ impl Server { value: pem_cert.certificate, }), issuer: parsed_cert.issuer, - not_valid_after: UTCDateTime::from_timestamp(parsed_cert.valid_not_after.timestamp()), - not_valid_before: UTCDateTime::from_timestamp(parsed_cert.valid_not_before.timestamp()), + not_valid_after: parsed_cert.valid_not_after, + not_valid_before: parsed_cert.valid_not_before, subject_alternative_names: Map::new(parsed_cert.sans), }; let now = now(); @@ -190,7 +232,7 @@ impl Server { // Reload registry let change = RegistryChange::Insert(ObjectId::new(ObjectType::Certificate, id)); - Box::pin(self.reload_registry(change)).await?; + self.reload_registry(change).await?; self.cluster_broadcast(BroadcastEvent::RegistryChange(change)) .await; @@ -205,7 +247,7 @@ impl Server { Acme(trc::AcmeEvent::OrderCompleted), Domain = domain.name.clone(), Hostname = new_sans.as_slice(), - Id = id.to_string(), + Id = id.to_compact_string(), ValidFrom = trc::Value::Timestamp(parsed_cert.valid_not_before.timestamp() as u64), ValidTo = trc::Value::Timestamp(parsed_cert.valid_not_after.timestamp() as u64), diff --git a/crates/common/src/network/acme/resolver.rs b/crates/common/src/network/acme/resolver.rs index 4ff9bbb7..3e096c4e 100644 --- a/crates/common/src/network/acme/resolver.rs +++ b/crates/common/src/network/acme/resolver.rs @@ -8,6 +8,7 @@ use crate::{ KV_ACME, Server, network::acme::{SerializedCert, StaticResolver, directory::ACME_TLS_ALPN_NAME}, }; +use compact_str::{CompactString, ToCompactString}; use rustls::{ ServerConfig, crypto::aws_lc_rs::sign::any_ecdsa_type, @@ -18,7 +19,7 @@ use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use std::sync::Arc; use store::{ dispatch::lookup::KeyValue, - write::{AlignedBytes, Archive}, + write::{Archive, ArchiveBytes}, }; use trc::AcmeEvent; @@ -26,7 +27,7 @@ impl Server { pub(crate) async fn build_acme_certificate(&self, domain: &str) -> Option> { match self .in_memory_store() - .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) + .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) .await { Ok(Some(cert_)) => match cert_.unarchive::() { @@ -41,8 +42,8 @@ impl Server { Err(err) => { trc::event!( Acme(AcmeEvent::Error), - Domain = domain.to_string(), - Reason = err.to_string(), + Domain = CompactString::from(domain), + Reason = err.to_compact_string(), Details = "Failed to parse private key" ); None @@ -53,7 +54,7 @@ impl Server { Err(err) => { trc::event!( Acme(AcmeEvent::Error), - Domain = domain.to_string(), + Domain = CompactString::from(domain), CausedBy = err, Details = "Failed to unarchive certificate" ); @@ -63,13 +64,16 @@ impl Server { Err(err) => { trc::event!( Acme(AcmeEvent::Error), - Domain = domain.to_string(), + Domain = CompactString::from(domain), CausedBy = err ); None } Ok(None) => { - trc::event!(Acme(AcmeEvent::TokenNotFound), Domain = domain.to_string()); + trc::event!( + Acme(AcmeEvent::TokenNotFound), + Domain = CompactString::from(domain) + ); None } } diff --git a/crates/common/src/network/asn.rs b/crates/common/src/network/asn.rs index abf91412..d8625123 100644 --- a/crates/common/src/network/asn.rs +++ b/crates/common/src/network/asn.rs @@ -4,48 +4,37 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::{CompactString, ToCompactString, format_compact}; use std::{ net::IpAddr, sync::{Arc, atomic::AtomicU64}, time::{Duration, Instant}, }; -use ahash::AHashMap; use arc_swap::ArcSwap; -use mail_auth::common::resolver::ToReverseName; +use mail_auth::dns::ToReverseName; +use memchr::memchr3_iter; use store::write::now; use tokio::sync::Semaphore; use crate::{Server, config::network::AsnGeoLookupConfig, manager::fetch_resource}; +pub mod table; + +use table::{AsnBuilder, CountryBuilder}; +pub use table::{AsnData, AsnTable, CountryTable}; + pub struct AsnGeoLookupData { pub lock: Semaphore, expires: AtomicU64, - asn: ArcSwap>>, - country: ArcSwap>>, -} - -#[derive(Clone, Default, Debug)] -pub struct AsnData { - pub id: u32, - pub name: Option, + asn: ArcSwap, + country: ArcSwap, } #[derive(Clone, Default, Debug)] pub struct AsnGeoLookupResult { - pub asn: Option>, - pub country: Option>, -} - -struct Data { - ip4_ranges: Vec>, - ip6_ranges: Vec>, -} - -pub struct IpRange { - pub start: I, - pub end: I, - pub data: T, + pub asn: Option, + pub country: Option>, } impl Server { @@ -62,8 +51,8 @@ impl Server { self.refresh_asn_geo_tables(); } - result.asn = asn_geo.asn.load().lookup(ip).cloned(); - result.country = asn_geo.country.load().lookup(ip).cloned(); + result.asn = asn_geo.asn.load().lookup(ip); + result.country = asn_geo.country.load().lookup(ip); } AsnGeoLookupConfig::Dns { zone_ipv4, @@ -94,37 +83,37 @@ impl Server { if idx == *index_asn { asn = part.parse::().ok(); } else if index_asn_name.is_some_and(|i| i == idx) { - asn_name = Some(part.to_string()); + asn_name = Some(Arc::from(part)); } else if index_country.is_some_and(|i| i == idx) { - country = Some(part.to_string()); + country = Some(part); } } } if let Some(asn) = asn { - result.asn = Some(Arc::new(AsnData { + result.asn = Some(AsnData { id: asn, name: asn_name, - })); + }); } if let Some(country) = country { - result.country = Some(Arc::new(country)); + result.country = Some(country.into()); } } Ok(Err(_)) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to UTF-8 decode ASN/Geo data", - Hostname = format!("{}.{}.", ip.to_reverse_name(), zone), + Hostname = format_compact!("{}.{}.", ip.to_reverse_name(), zone), ); } Err(err) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to lookup ASN/Geo data", - Hostname = format!("{}.{}.", ip.to_reverse_name(), zone), - CausedBy = err.to_string() + Hostname = format_compact!("{}.{}.", ip.to_reverse_name(), zone), + CausedBy = err.to_compact_string() ); } } @@ -152,10 +141,11 @@ impl Server { asn_resources, geo_resources, headers, + allow_invalid_certs, } = &server.core.network.asn_geo_lookup { - let mut asn_data = Data::new(); - let mut country_data = Data::new(); + let mut asn_data = AsnBuilder::new(); + let mut country_data = CountryBuilder::new(); for (is_asn, url) in asn_resources .iter() @@ -163,14 +153,18 @@ impl Server { .chain(geo_resources.iter().map(|url| (false, url))) { let time = Instant::now(); - match fetch_resource(url, headers.clone().into(), *timeout, *max_size) - .await - .map(String::from_utf8) + match fetch_resource( + url, + headers.clone().into(), + *timeout, + *max_size, + *allow_invalid_certs, + ) + .await + .map(String::from_utf8) { Ok(Ok(data)) => { let mut has_errors = false; - let mut asn_mappings = AHashMap::new(); - let mut geo_mappings = AHashMap::new(); let mut from_ip = None; let mut to_ip = None; @@ -182,10 +176,10 @@ impl Server { let mut col_start = 0; let mut line_start = 0; - for (idx, ch) in data.char_indices() { - match ch { - '"' => in_quote = !in_quote, - ',' | '\n' if !in_quote => { + for idx in memchr3_iter(b'"', b',', b'\n', data.as_bytes()) { + match data.as_bytes().get(idx) { + Some(b'"') => in_quote = !in_quote, + Some(&ch) if !in_quote => { let column = data.get(col_start..idx).unwrap_or_default().trim(); match col_num { @@ -201,10 +195,10 @@ impl Server { details = Some(column); } } - _ => break, + _ => {} } - if ch == '\n' { + if ch == b'\n' { let is_success = match (from_ip, to_ip, asn, details) { ( Some(from_ip), @@ -212,26 +206,12 @@ impl Server { Some(asn), asn_name, ) if is_asn => { - let data = asn_mappings - .entry(asn) - .or_insert_with(|| { - Arc::new(AsnData { - id: asn, - name: asn_name.map(String::from), - }) - }) - .clone(); - asn_data.insert(from_ip, to_ip, data) + asn_data.insert(from_ip, to_ip, asn, asn_name) } (Some(from_ip), Some(to_ip), _, Some(code)) if !is_asn && [2, 3].contains(&code.len()) => { - let code = code.to_uppercase(); - let data = geo_mappings - .entry(code.clone()) - .or_insert_with(|| Arc::new(code)) - .clone(); - country_data.insert(from_ip, to_ip, data) + country_data.insert(from_ip, to_ip, code) } (None, None, _, _) => true, // Ignore empty rows _ => false, @@ -242,10 +222,10 @@ impl Server { Resource(trc::ResourceEvent::Error), Details = "Invalid ASN/Geo data", Url = url.clone(), - Details = data - .get(line_start..idx) - .unwrap_or_default() - .to_string(), + Contents = CompactString::from( + data.get(line_start..idx) + .unwrap_or_default() + ), ); has_errors = true; } @@ -297,10 +277,10 @@ impl Server { }; if !asn_data.is_empty() { - asn_geo.asn.store(Arc::new(asn_data.sorted())); + asn_geo.asn.store(Arc::new(asn_data.build())); } if !country_data.is_empty() { - asn_geo.country.store(Arc::new(country_data.sorted())); + asn_geo.country.store(Arc::new(country_data.build())); } asn_geo.expires.store( @@ -312,89 +292,13 @@ impl Server { } } -impl Data { - fn new() -> Self { - Self { - ip4_ranges: Vec::new(), - ip6_ranges: Vec::new(), - } - } - - pub fn lookup(&self, ip: IpAddr) -> Option<&T> { - match ip { - IpAddr::V4(ip) => { - let ip = u32::from(ip); - match self.ip4_ranges.binary_search_by(|range| { - if ip < range.start { - std::cmp::Ordering::Greater - } else if ip > range.end { - std::cmp::Ordering::Less - } else { - std::cmp::Ordering::Equal - } - }) { - Ok(idx) => Some(&self.ip4_ranges[idx].data), - Err(_) => None, - } - } - IpAddr::V6(ip) => { - let ip = u128::from(ip); - match self.ip6_ranges.binary_search_by(|range| { - if ip < range.start { - std::cmp::Ordering::Greater - } else if ip > range.end { - std::cmp::Ordering::Less - } else { - std::cmp::Ordering::Equal - } - }) { - Ok(idx) => Some(&self.ip6_ranges[idx].data), - Err(_) => None, - } - } - } - } - - pub fn insert(&mut self, from_ip: IpAddr, to_ip: IpAddr, data: T) -> bool { - match (from_ip, to_ip) { - (IpAddr::V4(from), IpAddr::V4(to)) => { - self.ip4_ranges.push(IpRange { - start: u32::from(from), - end: u32::from(to), - data, - }); - true - } - (IpAddr::V6(from), IpAddr::V6(to)) => { - self.ip6_ranges.push(IpRange { - start: u128::from(from), - end: u128::from(to), - data, - }); - true - } - _ => false, - } - } - - pub fn sorted(mut self) -> Self { - self.ip4_ranges.sort_unstable_by_key(|range| range.start); - self.ip6_ranges.sort_unstable_by_key(|range| range.start); - self - } - - pub fn is_empty(&self) -> bool { - self.ip4_ranges.is_empty() && self.ip6_ranges.is_empty() - } -} - impl Default for AsnGeoLookupData { fn default() -> Self { Self { lock: Semaphore::new(1), expires: AtomicU64::new(0), - asn: ArcSwap::new(Arc::new(Data::new())), - country: ArcSwap::new(Arc::new(Data::new())), + asn: ArcSwap::new(Arc::new(AsnTable::default())), + country: ArcSwap::new(Arc::new(CountryTable::default())), } } } diff --git a/crates/common/src/network/asn/table.rs b/crates/common/src/network/asn/table.rs new file mode 100644 index 00000000..53cd77a0 --- /dev/null +++ b/crates/common/src/network/asn/table.rs @@ -0,0 +1,389 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashMap; +use std::{collections::hash_map::Entry, iter, net::IpAddr, sync::Arc}; + +const LOW_64: u128 = u64::MAX as u128; +const BLOCK_LEN: usize = 32; + +pub(crate) trait RowIndex: Copy + Eq { + const NONE: Self; + + fn index(self) -> usize; +} + +impl RowIndex for u8 { + const NONE: Self = u8::MAX; + + #[inline(always)] + fn index(self) -> usize { + usize::from(self) + } +} + +impl RowIndex for u16 { + const NONE: Self = u16::MAX; + + #[inline(always)] + fn index(self) -> usize { + usize::from(self) + } +} + +impl RowIndex for u32 { + const NONE: Self = u32::MAX; + + #[inline(always)] + fn index(self) -> usize { + self as usize + } +} + +trait RangeBound: Copy + Ord { + fn successor(self) -> Option; +} + +impl RangeBound for u32 { + fn successor(self) -> Option { + self.checked_add(1) + } +} + +impl RangeBound for u128 { + fn successor(self) -> Option { + self.checked_add(1) + } +} + +struct Row { + start: B, + end: B, + data: I, +} + +struct Ranges { + block_starts: Box<[S]>, + starts: Box<[S]>, + data: Box<[I]>, +} + +enum V6Ranges { + Prefix(Ranges), + Full(Ranges), +} + +struct RangeTable { + v4: Ranges, + v6: V6Ranges, +} + +struct RangeRows { + v4: Vec>, + v6: Vec>, +} + +pub struct AsnTable { + ranges: RangeTable, + asns: Box<[AsnData]>, +} + +pub struct CountryTable { + ranges: CountryRanges, + codes: Box<[Arc]>, +} + +enum CountryRanges { + Narrow(RangeTable), + Wide(RangeTable), +} + +#[derive(Clone, Default, Debug)] +pub struct AsnData { + pub id: u32, + pub name: Option>, +} + +pub(crate) struct AsnBuilder { + rows: RangeRows, + index: AHashMap, + asns: Vec, +} + +pub(crate) struct CountryBuilder { + rows: RangeRows, + index: AHashMap<[u8; 3], u16>, + codes: Vec>, +} + +impl Ranges { + fn empty() -> Self { + Ranges { + block_starts: Box::default(), + starts: Box::default(), + data: Box::default(), + } + } + + fn build( + rows: &[Row], + key: impl Fn(B) -> S, + map: impl Fn(J) -> I, + ) -> Self { + let capacity = rows.len() + with_gaps(rows).filter(|(_, gap)| gap.is_some()).count(); + let mut starts = Vec::with_capacity(capacity); + let mut data = Vec::with_capacity(capacity); + for (row, gap) in with_gaps(rows) { + starts.push(key(row.start)); + data.push(map(row.data)); + if let Some(gap) = gap { + starts.push(key(gap)); + data.push(I::NONE); + } + } + Ranges { + block_starts: starts.iter().step_by(BLOCK_LEN).copied().collect(), + starts: starts.into_boxed_slice(), + data: data.into_boxed_slice(), + } + } + + #[inline(always)] + fn lookup(&self, ip: S) -> Option { + let block = self + .block_starts + .partition_point(|start| *start <= ip) + .checked_sub(1)?; + let pos = self + .starts + .chunks(BLOCK_LEN) + .nth(block)? + .partition_point(|start| *start <= ip); + let data = *self.data.get(block * BLOCK_LEN + pos.checked_sub(1)?)?; + (data != I::NONE).then(|| data.index()) + } +} + +impl RangeTable { + fn empty() -> Self { + RangeTable { + v4: Ranges::empty(), + v6: V6Ranges::Prefix(Ranges::empty()), + } + } + + fn build(rows: &RangeRows, data: impl Fn(J) -> I) -> Self { + let v4 = Ranges::build(&rows.v4, |bound| bound, &data); + let v6 = if rows + .v6 + .iter() + .all(|row| row.start & LOW_64 == 0 && row.end & LOW_64 == LOW_64) + { + V6Ranges::Prefix(Ranges::build(&rows.v6, |bound| (bound >> 64) as u64, &data)) + } else { + V6Ranges::Full(Ranges::build(&rows.v6, |bound| bound, &data)) + }; + RangeTable { v4, v6 } + } + + #[inline(always)] + fn lookup(&self, ip: IpAddr) -> Option { + match ip { + IpAddr::V4(ip) => self.v4.lookup(u32::from(ip)), + IpAddr::V6(ip) => match &self.v6 { + V6Ranges::Prefix(ranges) => ranges.lookup((u128::from(ip) >> 64) as u64), + V6Ranges::Full(ranges) => ranges.lookup(u128::from(ip)), + }, + } + } +} + +fn with_gaps( + rows: &[Row], +) -> impl Iterator, Option)> { + rows.iter() + .zip(rows.iter().skip(1).map(Some).chain(iter::once(None))) + .map(|(row, next)| { + let gap = row + .end + .successor() + .filter(|gap| next.is_none_or(|next| next.start > *gap)); + (row, gap) + }) +} + +impl RangeRows { + fn new() -> Self { + RangeRows { + v4: Vec::new(), + v6: Vec::new(), + } + } + + fn insert(&mut self, from_ip: IpAddr, to_ip: IpAddr, data: I) -> bool { + match (from_ip, to_ip) { + (IpAddr::V4(from), IpAddr::V4(to)) if from <= to => { + self.v4.push(Row { + start: u32::from(from), + end: u32::from(to), + data, + }); + true + } + (IpAddr::V6(from), IpAddr::V6(to)) if from <= to => { + self.v6.push(Row { + start: u128::from(from), + end: u128::from(to), + data, + }); + true + } + _ => false, + } + } + + fn sort(&mut self) { + self.v4.sort_unstable_by_key(|row| row.start); + self.v6.sort_unstable_by_key(|row| row.start); + } + + fn is_empty(&self) -> bool { + self.v4.is_empty() && self.v6.is_empty() + } +} + +impl AsnBuilder { + pub(crate) fn new() -> Self { + AsnBuilder { + rows: RangeRows::new(), + index: AHashMap::new(), + asns: Vec::new(), + } + } + + pub(crate) fn insert( + &mut self, + from_ip: IpAddr, + to_ip: IpAddr, + asn: u32, + name: Option<&str>, + ) -> bool { + let data = match self.index.entry(asn) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let Ok(data) = u32::try_from(self.asns.len()) else { + return false; + }; + if data == u32::NONE { + return false; + } + self.asns.push(AsnData { + id: asn, + name: name.map(Arc::from), + }); + *entry.insert(data) + } + }; + self.rows.insert(from_ip, to_ip, data) + } + + pub(crate) fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub(crate) fn build(mut self) -> AsnTable { + self.rows.sort(); + AsnTable { + ranges: RangeTable::build(&self.rows, |data| data), + asns: self.asns.into_boxed_slice(), + } + } +} + +impl CountryBuilder { + pub(crate) fn new() -> Self { + CountryBuilder { + rows: RangeRows::new(), + index: AHashMap::new(), + codes: Vec::new(), + } + } + + pub(crate) fn insert(&mut self, from_ip: IpAddr, to_ip: IpAddr, code: &str) -> bool { + let mut key = [0u8; 3]; + for (key, byte) in key.iter_mut().zip(code.bytes()) { + *key = byte.to_ascii_uppercase(); + } + let data = match self.index.entry(key) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let Ok(data) = u16::try_from(self.codes.len()) else { + return false; + }; + if data == u16::NONE { + return false; + } + self.codes.push(code.to_uppercase().into()); + *entry.insert(data) + } + }; + self.rows.insert(from_ip, to_ip, data) + } + + pub(crate) fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub(crate) fn build(mut self) -> CountryTable { + self.rows.sort(); + let ranges = if self.codes.len() < usize::from(u8::NONE) { + CountryRanges::Narrow(RangeTable::build(&self.rows, |data| { + u8::try_from(data).unwrap_or(u8::NONE) + })) + } else { + CountryRanges::Wide(RangeTable::build(&self.rows, |data| data)) + }; + CountryTable { + ranges, + codes: self.codes.into_boxed_slice(), + } + } +} + +impl AsnTable { + pub fn lookup(&self, ip: IpAddr) -> Option { + self.asns.get(self.ranges.lookup(ip)?).cloned() + } +} + +impl Default for AsnTable { + fn default() -> Self { + AsnTable { + ranges: RangeTable::empty(), + asns: Box::default(), + } + } +} + +impl CountryTable { + pub fn lookup(&self, ip: IpAddr) -> Option> { + let idx = match &self.ranges { + CountryRanges::Narrow(ranges) => ranges.lookup(ip), + CountryRanges::Wide(ranges) => ranges.lookup(ip), + }?; + self.codes.get(idx).cloned() + } +} + +impl Default for CountryTable { + fn default() -> Self { + CountryTable { + ranges: CountryRanges::Narrow(RangeTable::empty()), + codes: Box::default(), + } + } +} + diff --git a/crates/common/src/network/autoconfig/autodiscover.rs b/crates/common/src/network/autoconfig/autodiscover.rs index 741eae30..8df26540 100644 --- a/crates/common/src/network/autoconfig/autodiscover.rs +++ b/crates/common/src/network/autoconfig/autodiscover.rs @@ -7,10 +7,11 @@ use crate::{Server, manager::application::Resource}; use quick_xml::Reader; use quick_xml::XmlVersion; +use quick_xml::escape::escape; use quick_xml::events::Event; use registry::schema::{enums::ServiceProtocol, structs::Service}; -use std::{borrow::Cow, fmt::Write}; -use utils::map::vec_map::VecMap; +use std::fmt::Write; +use utils::{map::vec_map::VecMap, sanitize_email}; impl Server { pub async fn handle_autodiscover_request( @@ -89,6 +90,8 @@ fn build_autodiscover_response( default_host: &str, services: &VecMap, ) -> String { + let emailaddress = escape(emailaddress); + // Build XML response let mut config = String::with_capacity(1024); let _ = writeln!(&mut config, ""); @@ -173,16 +176,13 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { let found_tag_name = e.name(); - if tag_name - .as_bytes() - .eq_ignore_ascii_case(found_tag_name.as_ref()) - { + if tag_name.eq_ignore_ascii_case(found_tag_name.into_inner()) { continue 'outer; } else { return Err(format!( "Expected tag {}, found unexpected tag {} at position {}.", tag_name, - String::from_utf8_lossy(found_tag_name.as_ref()), + found_tag_name.into_inner(), reader.buffer_position() )); } @@ -213,27 +213,47 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { let local_name = e.local_name(); - let field = hashify::tiny_map_ignore_case!(local_name.as_ref(), - b"EMailAddress" => RequestField::EmailAddress, - b"AcceptableResponseSchema" => RequestField::ResponseSchema, - ); + let field = + hashify::map_ignore_case!(local_name.into_inner().as_bytes(), RequestField, + b"EMailAddress" => RequestField::EmailAddress, + b"AcceptableResponseSchema" => RequestField::ResponseSchema, + ) + .copied(); let value = match reader.read_event_into(&mut value_buf) { Ok(Event::End(_)) => None, Ok(event) => { let value = match event { - Event::Text(text) => text - .xml_content(XmlVersion::Implicit1_0) - .ok() - .map(Cow::into_owned), + Event::Text(text) => { + Some(text.xml_content(XmlVersion::Implicit1_0).into_owned()) + } _ => None, }; - reader - .read_to_end_into(e.name(), &mut value_buf) - .map_err(|err| { - format!("Error at position {}: {:?}", reader.buffer_position(), err) - })?; - value + let is_closed = value.is_some() + && matches!( + reader.read_event_into(&mut value_buf).map_err(|err| { + format!( + "Error at position {}: {:?}", + reader.buffer_position(), + err + ) + })?, + Event::End(_) + ); + if is_closed { + value + } else { + reader + .read_to_end_into(e.name(), &mut value_buf) + .map_err(|err| { + format!( + "Error at position {}: {:?}", + reader.buffer_position(), + err + ) + })?; + None + } } Err(err) => { return Err(format!( @@ -266,183 +286,15 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result Ok(AutodiscoverRequest { - email: email.trim().to_lowercase(), + match email.as_deref().and_then(sanitize_email) { + Some(email) => Ok(AutodiscoverRequest { + email, response_schema, }), - _ => Err(format!( + None => Err(format!( "Expected email address, found unexpected value at position {}.", reader.buffer_position() )), } } -#[cfg(test)] -mod tests { - use super::{AutodiscoverRequest, ResponseSchema, parse_autodiscover_request}; - - #[test] - fn parse_autodiscover() { - const OUTLOOK: &str = - "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; - const MOBILESYNC: &str = - "http://schemas.microsoft.com/exchange/autodiscover/mobilesync/responseschema/2006"; - - for (request, expected) in [ - ( - format!( - r#" - - - Email@Example.com - {OUTLOOK} - - "# - ), - ResponseSchema::Outlook, - ), - ( - format!( - r#" - - {OUTLOOK} - email@example.com - - "# - ), - ResponseSchema::Outlook, - ), - ( - r#" - - email@example.com - - "# - .to_string(), - ResponseSchema::Outlook, - ), - ( - format!( - r#" - - - email@example.com - {MOBILESYNC} - - "# - ), - ResponseSchema::Unsupported, - ), - ( - format!( - r#" - - /o=Example/ou=Users/cn=email - value - {MOBILESYNC} - email@example.com - - "# - ), - ResponseSchema::Unsupported, - ), - ] { - assert_eq!( - parse_autodiscover_request(request.as_bytes()).expect("valid request"), - AutodiscoverRequest { - email: "email@example.com".to_string(), - response_schema: expected, - }, - "{request}" - ); - } - - for request in [ - "", - "", - "no-domain", - "email@example.com", - "email@example.com", - ] { - assert!( - parse_autodiscover_request(request.as_bytes()).is_err(), - "{request}" - ); - } - } - - #[test] - fn autodiscover_encryption() { - use registry::schema::{enums::ServiceProtocol, structs::Service}; - use utils::map::vec_map::VecMap; - - fn tag<'x>(block: &'x str, name: &str) -> &'x str { - block - .split_once(&format!("<{name}>")) - .and_then(|(_, rest)| rest.split_once(&format!(""))) - .map(|(value, _)| value) - .unwrap() - } - - for (cleartext, expected) in [ - ( - false, - vec![ - ("IMAP", "993", "on", "SSL"), - ("POP3", "995", "on", "SSL"), - ("SMTP", "465", "on", "SSL"), - ], - ), - ( - true, - vec![ - ("IMAP", "993", "on", "SSL"), - ("IMAP", "143", "off", "TLS"), - ("POP3", "995", "on", "SSL"), - ("POP3", "110", "off", "TLS"), - ("SMTP", "465", "on", "SSL"), - ("SMTP", "587", "off", "TLS"), - ], - ), - ] { - let services: VecMap = [ - ServiceProtocol::Imap, - ServiceProtocol::Pop3, - ServiceProtocol::Smtp, - ServiceProtocol::Jmap, - ] - .into_iter() - .map(|protocol| { - ( - protocol, - Service { - hostname: None, - cleartext, - }, - ) - }) - .collect(); - let response = super::build_autodiscover_response( - "user@example.com", - "mail.example.com", - &services, - ); - - assert_eq!( - response - .split("") - .skip(1) - .map(|block| ( - tag(block, "Type"), - tag(block, "Port"), - tag(block, "SSL"), - tag(block, "Encryption"), - )) - .collect::>(), - expected, - "cleartext: {cleartext}" - ); - } - } -} diff --git a/crates/common/src/network/autoconfig/legacy_autoconfig.rs b/crates/common/src/network/autoconfig/legacy_autoconfig.rs index 2f2e2cbb..f00a9976 100644 --- a/crates/common/src/network/autoconfig/legacy_autoconfig.rs +++ b/crates/common/src/network/autoconfig/legacy_autoconfig.rs @@ -5,101 +5,107 @@ */ use crate::{Server, manager::application::Resource}; -use registry::schema::enums::ServiceProtocol; +use registry::schema::{enums::ServiceProtocol, structs::Service}; use std::fmt::Write; -use utils::url_params::UrlParams; +use utils::{map::vec_map::VecMap, sanitize_email, url_params::UrlParams}; impl Server { pub async fn handle_autoconfig_request( &self, uri: Option<&str>, ) -> trc::Result>> { - // Obtain parameters - let params = UrlParams::new(uri); - let emailaddress_param = params - .get("emailaddress") - .unwrap_or_default() - .to_lowercase(); - let default_host = &self.core.network.server_name; - let (emailaddress, domain) = if let Some((_, domain)) = emailaddress_param.rsplit_once('@') - { - (emailaddress_param.as_str(), domain) - } else { - ("%EMAILADDRESS%", default_host.as_str()) - }; - - // Build XML response - let mut config = String::with_capacity(1024); - config.push_str("\n"); - config.push_str("\n"); - let _ = writeln!(&mut config, "\t"); - let _ = writeln!(&mut config, "\t\t{domain}"); - let _ = writeln!(&mut config, "\t\t{emailaddress}"); - let _ = writeln!( - &mut config, - "\t\t{domain}" - ); - for (protocol, service) in &self.core.network.info.services { - let (protocol, tag, ports) = match protocol { - ServiceProtocol::Smtp => ("smtp", "outgoingServer", [587, 465]), - ServiceProtocol::Imap => ("imap", "incomingServer", [143, 993]), - ServiceProtocol::Pop3 => ("pop3", "incomingServer", [110, 995]), - _ => continue, - }; - for (is_tls, port) in ports.into_iter().enumerate() { - if is_tls == 1 || service.cleartext { - let server_name = service.hostname.as_deref().unwrap_or(default_host); - let _ = writeln!(&mut config, "\t\t<{tag} type=\"{protocol}\">"); - let _ = writeln!(&mut config, "\t\t\t{server_name}"); - let _ = writeln!(&mut config, "\t\t\t{port}"); - let _ = writeln!( - &mut config, - "\t\t\t{}", - if is_tls == 1 { "SSL" } else { "STARTTLS" } - ); - let _ = writeln!(&mut config, "\t\t\t{emailaddress}"); - let _ = writeln!( - &mut config, - "\t\t\tpassword-cleartext" - ); - let _ = writeln!(&mut config, "\t\t"); - } - } - } - - config.push_str("\t\n"); - - for (protocol, service) in &self.core.network.info.services { - let (tag, protocol, url) = match protocol { - ServiceProtocol::Carddav => ("addressBook", "carddav", "card"), - ServiceProtocol::Caldav => ("calendar", "caldav", "cal"), - ServiceProtocol::Webdav => ("fileShare", "webdav", "file"), - _ => continue, - }; - let server_name = service.hostname.as_deref().unwrap_or(default_host); - - let _ = writeln!(&mut config, "\t<{tag} type=\"{protocol}\">"); - let _ = writeln!(&mut config, "\t\t{emailaddress}"); - let _ = writeln!( - &mut config, - "\t\thttp-basic" - ); - let _ = writeln!( - &mut config, - "\t\thttps://{server_name}/dav/{url}" - ); - let _ = writeln!(&mut config, "\t"); - } - - let _ = writeln!( - &mut config, - "\t" - ); - config.push_str("\n"); - Ok(Resource::new( "application/xml; charset=utf-8", - config.into_bytes(), + build_autoconfig_response( + UrlParams::new(uri).get("emailaddress"), + &self.core.network.server_name, + &self.core.network.info.services, + ) + .into_bytes(), )) } } + +fn build_autoconfig_response( + emailaddress: Option<&str>, + default_host: &str, + services: &VecMap, +) -> String { + let email = emailaddress.and_then(sanitize_email); + let domain = email + .as_deref() + .and_then(|email| email.rsplit_once('@')) + .map_or(default_host, |(_, domain)| domain); + + // Build XML response + let mut config = String::with_capacity(1024); + config.push_str("\n"); + config.push_str("\n"); + let _ = writeln!(&mut config, "\t"); + let _ = writeln!(&mut config, "\t\t{domain}"); + let _ = writeln!(&mut config, "\t\t{domain}"); + let _ = writeln!( + &mut config, + "\t\t{domain}" + ); + for (protocol, service) in services { + let (protocol, tag, ports) = match protocol { + ServiceProtocol::Smtp => ("smtp", "outgoingServer", [587, 465]), + ServiceProtocol::Imap => ("imap", "incomingServer", [143, 993]), + ServiceProtocol::Pop3 => ("pop3", "incomingServer", [110, 995]), + _ => continue, + }; + for (is_tls, port) in ports.into_iter().enumerate() { + if is_tls == 1 || service.cleartext { + let server_name = service.hostname.as_deref().unwrap_or(default_host); + let _ = writeln!(&mut config, "\t\t<{tag} type=\"{protocol}\">"); + let _ = writeln!(&mut config, "\t\t\t{server_name}"); + let _ = writeln!(&mut config, "\t\t\t{port}"); + let _ = writeln!( + &mut config, + "\t\t\t{}", + if is_tls == 1 { "SSL" } else { "STARTTLS" } + ); + let _ = writeln!(&mut config, "\t\t\t%EMAILADDRESS%"); + let _ = writeln!( + &mut config, + "\t\t\tpassword-cleartext" + ); + let _ = writeln!(&mut config, "\t\t"); + } + } + } + + config.push_str("\t\n"); + + for (protocol, service) in services { + let (tag, protocol, url) = match protocol { + ServiceProtocol::Carddav => ("addressBook", "carddav", "card"), + ServiceProtocol::Caldav => ("calendar", "caldav", "cal"), + ServiceProtocol::Webdav => ("fileShare", "webdav", "file"), + _ => continue, + }; + let server_name = service.hostname.as_deref().unwrap_or(default_host); + + let _ = writeln!(&mut config, "\t<{tag} type=\"{protocol}\">"); + let _ = writeln!(&mut config, "\t\t%EMAILADDRESS%"); + let _ = writeln!( + &mut config, + "\t\thttp-basic" + ); + let _ = writeln!( + &mut config, + "\t\thttps://{server_name}/dav/{url}" + ); + let _ = writeln!(&mut config, "\t"); + } + + let _ = writeln!( + &mut config, + "\t" + ); + config.push_str("\n"); + + config +} + diff --git a/crates/common/src/network/autoconfig/pacc.rs b/crates/common/src/network/autoconfig/pacc.rs index 74b38997..9a9ea07b 100644 --- a/crates/common/src/network/autoconfig/pacc.rs +++ b/crates/common/src/network/autoconfig/pacc.rs @@ -150,209 +150,3 @@ pub struct Help { pub contact: Option>, } -#[cfg(test)] -mod tests { - use super::*; - - /// The full example from Section 4.1 of the draft. - const EXAMPLE_JSON: &str = r#"{ - "protocols": { - "jmap": { - "url": "https://jmap.example.com/session" - }, - "imap": { - "host": "imap.example.com" - }, - "pop3": { - "host": "pop3.example.com" - }, - "smtp": { - "host": "smtp.example.com" - }, - "caldav": { - "url": "https://sync.example.com/calendar/" - }, - "carddav": { - "url": "https://sync.example.com/contacts/" - } - }, - "authentication": { - "oauth-public": { - "issuer": "https://auth.example.com/" - }, - "password": true - }, - "info": { - "provider": { - "name": "Example Provider Name", - "shortName": "Example", - "logo": [ - { - "url": "https://www.example.net/logo.svg", - "content-type": "image/svg+xml" - } - ] - }, - "help": { - "documentation": "https://help.example.net/howto/set-up-your-mail-app.html", - "developer": "https://developer.example.net/client-apps/", - "contact": ["mailto:it@team.example.net"] - } - } -}"#; - - #[test] - fn deserialize_full_example() { - let config: Configuration = - serde_json::from_str(EXAMPLE_JSON).expect("failed to deserialize"); - - // Protocols - assert_eq!( - config.protocols.jmap.as_ref().unwrap().url, - "https://jmap.example.com/session" - ); - assert_eq!( - config.protocols.imap.as_ref().unwrap().host, - "imap.example.com" - ); - assert_eq!( - config.protocols.smtp.as_ref().unwrap().host, - "smtp.example.com" - ); - assert_eq!( - config.protocols.pop3.as_ref().unwrap().host, - "pop3.example.com" - ); - assert_eq!( - config.protocols.caldav.as_ref().unwrap().url, - "https://sync.example.com/calendar/" - ); - assert_eq!( - config.protocols.carddav.as_ref().unwrap().url, - "https://sync.example.com/contacts/" - ); - assert!(config.protocols.webdav.is_none()); - assert!(config.protocols.managesieve.is_none()); - - // Authentication - let auth = config.authentication.as_ref().unwrap(); - assert!(auth.password); - assert_eq!( - auth.oauth_public.as_ref().unwrap().issuer, - "https://auth.example.com/" - ); - - // Info - assert_eq!(config.info.provider.name, "Example Provider Name"); - assert_eq!(config.info.provider.short_name.as_deref(), Some("Example")); - - let logos = config.info.provider.logo.as_ref().unwrap(); - assert_eq!(logos.len(), 1); - assert_eq!(logos[0].content_type, "image/svg+xml"); - assert!(logos[0].width.is_none()); - - let help = config.info.help.as_ref().unwrap(); - assert_eq!( - help.documentation.as_deref(), - Some("https://help.example.net/howto/set-up-your-mail-app.html") - ); - assert_eq!( - help.contact.as_ref().unwrap(), - &["mailto:it@team.example.net"] - ); - } - - #[test] - fn roundtrip() { - let config: Configuration = - serde_json::from_str(EXAMPLE_JSON).expect("failed to deserialize"); - let serialized = serde_json::to_string_pretty(&config).expect("failed to serialize"); - let roundtripped: Configuration = - serde_json::from_str(&serialized).expect("failed to re-deserialize"); - assert_eq!(config, roundtripped); - } - - #[test] - fn minimal_config() { - let json = r#"{ - "protocols": {}, - "info": { - "provider": { - "name": "Minimal" - } - } - }"#; - let config: Configuration = serde_json::from_str(json).expect("failed to deserialize"); - assert_eq!(config.info.provider.name, "Minimal"); - assert!(config.authentication.is_none()); - assert!(config.protocols.jmap.is_none()); - } - - #[test] - fn ignores_unknown_properties() { - let json = r#"{ - "protocols": { - "imap": { "host": "imap.example.com" }, - "future-protocol": { "endpoint": "wss://example.com" } - }, - "info": { - "provider": { "name": "Test" } - }, - "futureField": 42 - }"#; - let config: Configuration = serde_json::from_str(json).expect("should ignore unknowns"); - assert_eq!( - config.protocols.imap.as_ref().unwrap().host, - "imap.example.com" - ); - } - - #[test] - fn logo_with_dimensions() { - let json = r#"{ - "protocols": {}, - "info": { - "provider": { - "name": "Test", - "logo": [ - { - "url": "https://example.com/logo.svg", - "content-type": "image/svg+xml" - }, - { - "url": "https://example.com/logo-128.png", - "content-type": "image/png", - "width": 128, - "height": 128 - }, - { - "url": "https://example.com/logo-512.png", - "content-type": "image/png", - "width": 512, - "height": 512 - } - ] - } - } - }"#; - let config: Configuration = serde_json::from_str(json).unwrap(); - let logos = config.info.provider.logo.as_ref().unwrap(); - assert_eq!(logos.len(), 3); - assert!(logos[0].width.is_none()); - assert_eq!(logos[1].width, Some(128)); - assert_eq!(logos[2].height, Some(512)); - } - - #[test] - fn password_only_auth() { - let json = r#"{ - "protocols": { "imap": { "host": "mail.example.com" } }, - "authentication": { "password": true }, - "info": { "provider": { "name": "PW Only" } } - }"#; - let config: Configuration = serde_json::from_str(json).unwrap(); - let auth = config.authentication.unwrap(); - assert!(auth.password); - assert!(auth.oauth_public.is_none()); - } -} diff --git a/crates/common/src/network/dkim.rs b/crates/common/src/network/dkim.rs index eaa36793..db3b8699 100644 --- a/crates/common/src/network/dkim.rs +++ b/crates/common/src/network/dkim.rs @@ -4,16 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::config::smtp::auth::{rsa_key_parse, simple_pem_parse}; -use chrono::Utc; +use crate::config::smtp::auth::{pem_or_base64_decode, rsa_key_parse}; use dns_update::{DnsRecord, NamedDnsRecord}; -use mail_auth::common::crypto::Ed25519Key; +use encodify::{base64::STANDARD, pem}; +use jiff::{Timestamp, fmt::strtime, tz::TimeZone}; +use mail_auth::crypto::{Ed25519Key, MlDsa44Ed25519Key}; use mail_auth::dkim::generate::DkimKeyPair; -use mail_builder::encoders::Base64Encoder; -use pkcs8::Document; use registry::schema::enums::DkimSignatureType; use registry::schema::structs::DkimSignature; -use rsa::pkcs1::DecodeRsaPublicKey; +use rsa::pkcs1::{DecodeRsaPublicKey, der::Document}; use store::rand::distr::Alphanumeric; use store::rand::{self, RngExt}; @@ -27,6 +26,9 @@ pub async fn generate_dkim_private_key( DkimSignatureType::Dkim1Ed25519Sha256 | DkimSignatureType::Dkim2Ed25519Sha256 => { DkimKeyPair::generate_ed25519().map(|key| (key, "PRIVATE KEY")) } + DkimSignatureType::Dkim2MlDsa44Ed25519Sha512 => { + DkimKeyPair::generate_mldsa44_ed25519().map(|key| (key, "PRIVATE KEY")) + } }) .await .map_err(|err| { @@ -36,83 +38,59 @@ pub async fn generate_dkim_private_key( })?; Ok(private_key - .map(|(private_key, pk_type)| { - let mut pem = format!("-----BEGIN {pk_type}-----\n").into_bytes(); - let mut lf_count = 65; - for ch in Base64Encoder::new() - .encode(private_key.private_key()) - .unwrap_or_default() - { - pem.push(ch); - lf_count -= 1; - if lf_count == 0 { - pem.push(b'\n'); - lf_count = 65; - } - } - if lf_count != 65 { - pem.push(b'\n'); - } - pem.extend_from_slice(format!("-----END {pk_type}-----\n").as_bytes()); - - String::from_utf8(pem).unwrap_or_default() - }) + .map(|(private_key, pk_type)| pem::STANDARD.encode(pk_type, private_key.private_key())) .map_err(|err| err.to_string())) } pub async fn generate_dkim_public_key(key: &DkimSignature) -> trc::Result { - let is_rsa = matches!( - key, - DkimSignature::Dkim1RsaSha256(_) | DkimSignature::Dkim2RsaSha256(_) - ); let pem = key .private_key() .secret() .await .map_err(|err| trc::DkimEvent::BuildError.reason(err))?; - if is_rsa { - rsa_key_parse(pem.as_bytes()) - .and_then(|pk| { - Document::from_pkcs1_der(&pk.public_key()).map_err(|err| { + match key { + DkimSignature::Dkim1RsaSha256(_) | DkimSignature::Dkim2RsaSha256(_) => { + rsa_key_parse(pem.as_bytes()) + .and_then(|pk| { + Document::from_pkcs1_der(&pk.public_key()).map_err(|err| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .reason(err) + }) + }) + .map(|pk| STANDARD.encode(pk.as_bytes())) + } + DkimSignature::Dkim1Ed25519Sha256(_) | DkimSignature::Dkim2Ed25519Sha256(_) => { + Ed25519Key::from_pkcs8_maybe_unchecked_der(&decode_private_key(&pem)?) + .map(|pk| STANDARD.encode(pk.public_key())) + .map_err(|err| { trc::EventType::Dkim(trc::DkimEvent::BuildError) .into_err() .reason(err) }) - }) - .map(|pk| { - String::from_utf8( - Base64Encoder::new() - .encode(pk.as_bytes()) - .unwrap_or_default(), - ) - .unwrap_or_default() - }) - } else { - simple_pem_parse(&pem) - .ok_or_else(|| { - trc::EventType::Dkim(trc::DkimEvent::BuildError) - .into_err() - .details("Failed to parse private key PEM") - }) - .and_then(|der| { - Ed25519Key::from_pkcs8_maybe_unchecked_der(&der).map_err(|err| { + } + DkimSignature::Dkim2MlDsa44Ed25519Sha512(_) => { + MlDsa44Ed25519Key::from_pkcs8_der(&decode_private_key(&pem)?) + .and_then(|pk| pk.public_key()) + .map(|pk| STANDARD.encode(pk)) + .map_err(|err| { trc::EventType::Dkim(trc::DkimEvent::BuildError) .into_err() .reason(err) }) - }) - .map(|pk| { - String::from_utf8( - Base64Encoder::new() - .encode(&pk.public_key()) - .unwrap_or_default(), - ) - .unwrap_or_default() - }) + } } } +fn decode_private_key(pem: &str) -> trc::Result> { + pem_or_base64_decode(pem).map_err(|_| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .details("Failed to parse private key PEM") + }) +} + pub async fn generate_dkim_dns_record( key: &DkimSignature, domain: &str, @@ -136,6 +114,10 @@ pub async fn generate_dkim_dns_record( &sign.selector, format!("v=DKIM1; k=rsa; h=sha256; p={public_key}"), ), + DkimSignature::Dkim2MlDsa44Ed25519Sha512(sign) => ( + &sign.selector, + format!("v=DKIM1; k=mldsa44-ed25519-sha512; p={public_key}"), + ), }; Ok(NamedDnsRecord { @@ -151,10 +133,10 @@ pub fn generate_dkim_dns_record_name(key: &DkimSignature, domain: &str) -> Strin /// Generate a DKIM selector from a template string. /// /// Supported variables: -/// - `{algorithm}`: signing algorithm in lowercase (`rsa`, `ed25519`) -/// - `{hash}`: hash algorithm (`sha256`) -/// - `{version}`: DKIM version number (`1`) -/// - `{date-}`: current UTC date formatted with chrono strftime (e.g. `{date-%Y%m%d}`) +/// - `{algorithm}`: signing algorithm in lowercase (`rsa`, `ed25519`, `mldsa44-ed25519`) +/// - `{hash}`: hash algorithm (`sha256`, `sha512`) +/// - `{version}`: DKIM version number (`1`, `2`) +/// - `{date-}`: current UTC date formatted with strftime specifiers (e.g. `{date-%Y%m%d}`) /// - `{epoch}`: current UTC unix timestamp /// - `{random}`: random 8-character alphanumeric string /// @@ -162,7 +144,7 @@ pub fn generate_dkim_selector( template: &str, sig_type: DkimSignatureType, ) -> Result { - let now = Utc::now(); + let now = Timestamp::now().to_zoned(TimeZone::UTC); let mut result = Vec::with_capacity(template.len()); let mut chars = template.as_bytes(); @@ -202,7 +184,7 @@ pub fn generate_dkim_selector( "hash" => result.extend_from_slice(sig_type.hash().as_bytes()), "version" => result.extend_from_slice(sig_type.version().as_bytes()), "epoch" => { - result.extend_from_slice(now.timestamp().to_string().as_bytes()); + result.extend_from_slice(now.timestamp().as_second().to_string().as_bytes()); } "random" => { let rand_str: String = rand::rng() @@ -217,7 +199,8 @@ pub fn generate_dkim_selector( if fmt.is_empty() { return Err("empty strftime format in {date-}".into()); } - let formatted = now.format(fmt).to_string(); + let formatted = strtime::format(fmt, &now) + .map_err(|err| format!("invalid date format '{fmt}': {err}"))?; if formatted.is_empty() { return Err(format!("date format '{fmt}' produced empty output")); } @@ -245,73 +228,3 @@ fn memchr(needle: u8, haystack: &[u8]) -> Option { haystack.iter().position(|&b| b == needle) } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic_algorithm_date() { - let sel = generate_dkim_selector( - "{algorithm}-{date-%Y%m%d}", - DkimSignatureType::Dkim1RsaSha256, - ) - .unwrap(); - let today = Utc::now().format("%Y%m%d").to_string(); - assert_eq!(sel, format!("rsa-{today}")); - } - - #[test] - fn all_variables() { - let sel = generate_dkim_selector( - "v{version}-{algorithm}-{hash}-{epoch}", - DkimSignatureType::Dkim1Ed25519Sha256, - ) - .unwrap(); - assert!(sel.starts_with("v1-ed25519-sha256-")); - } - - #[test] - fn literal_only() { - let sel = - generate_dkim_selector("my-static-selector", DkimSignatureType::default()).unwrap(); - assert_eq!(sel, "my-static-selector"); - } - - #[test] - fn invalid_chars_stripped() { - let sel = generate_dkim_selector("{algorithm} {hash}", DkimSignatureType::Dkim1RsaSha256) - .unwrap(); - assert_eq!(sel, "rsasha256"); - } - - #[test] - fn unrecognized_variable_errors() { - let err = generate_dkim_selector("{bogus}", DkimSignatureType::default()).unwrap_err(); - assert!(err.contains("unrecognized variable")); - } - - #[test] - fn unclosed_brace_errors() { - let err = generate_dkim_selector("{algorithm", DkimSignatureType::default()).unwrap_err(); - assert!(err.contains("unclosed")); - } - - #[test] - fn empty_after_sanitization_errors() { - let err = generate_dkim_selector(" ", DkimSignatureType::default()).unwrap_err(); - assert!(err.contains("empty")); - } - - #[test] - fn empty_date_format_errors() { - let err = generate_dkim_selector("{date-}", DkimSignatureType::default()).unwrap_err(); - assert!(err.contains("empty strftime")); - } - - #[test] - fn date_month_only() { - let sel = generate_dkim_selector("{date-%Y%m}", DkimSignatureType::Dkim1RsaSha256).unwrap(); - let expected = Utc::now().format("%Y%m").to_string(); - assert_eq!(sel, expected); - } -} diff --git a/crates/common/src/network/dns/records.rs b/crates/common/src/network/dns/records.rs index 221ae152..bfb960e6 100644 --- a/crates/common/src/network/dns/records.rs +++ b/crates/common/src/network/dns/records.rs @@ -4,15 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Server, config::network::Pacc, network::dkim::generate_dkim_dns_record}; +use crate::{ + Server, + config::network::Pacc, + network::{ + acme::persist::{PersistRecordBuilder, persist_validation_names}, + dkim::generate_dkim_dns_record, + }, +}; use ahash::{AHashMap, AHashSet}; -use base64::{Engine, engine::general_purpose}; use dns_update::{ CAARecord, DnsRecord, KeyValue, MXRecord, NamedDnsRecord, SRVRecord, TLSARecord, TlsaCertUsage, TlsaMatching, TlsaSelector, bind::BindSerializer, }; +use encodify::base64::STANDARD; use registry::schema::{ - enums::{DnsRecordType, ServiceProtocol}, + enums::{AcmeChallengeType, DnsRecordType, ServiceProtocol}, prelude::{ObjectType, Property}, structs::{AcmeProvider, CertificateManagement, DkimSignature, DnsManagement, Domain}, }; @@ -130,13 +137,14 @@ impl Server { records.push(NamedDnsRecord { name: format!("_mta-sts.{domain_name}."), - record: DnsRecord::TXT(format!("v=STSv1; id={}", policy.id)), + record: DnsRecord::TXT(format!("v=STSv1; id={}", policy.policy.id)), }); } } DnsRecordType::AutoConfig => { let pacc_digest = Sha256::digest(&self.get_pacc_for_domain(domain_name).await?); - let pacc_digest_encoded = general_purpose::STANDARD.encode(pacc_digest); + let mut pacc_record = String::from("v=UAAC1; a=sha256; d="); + STANDARD.encode_append(pacc_digest, &mut pacc_record); records.push(NamedDnsRecord { name: format!("ua-auto-config.{domain_name}."), @@ -144,9 +152,7 @@ impl Server { }); records.push(NamedDnsRecord { name: format!("_ua-auto-config.{domain_name}."), - record: DnsRecord::TXT(format!( - "v=UAAC1; a=sha256; d={pacc_digest_encoded}" - )), + record: DnsRecord::TXT(pacc_record), }); } DnsRecordType::AutoConfigLegacy => { @@ -214,20 +220,33 @@ impl Server { .registry() .object::(props.acme_provider_id) .await? - && let Ok(provider_url) = Url::parse(&provider.directory) - && let Some(provider_name) = provider_domain(&provider_url) { - records.push(NamedDnsRecord { - name: format!("{domain_name}."), - record: DnsRecord::CAA(CAARecord::Issue { - issuer_critical: false, - name: provider_name.to_string().into(), - options: vec![KeyValue { - key: "accounturi".to_string(), - value: provider.account_uri.clone(), - }], - }), - }); + let directory_url = Url::parse(&provider.directory).ok(); + let fallback_issuer = directory_url.as_ref().and_then(provider_domain); + let caa_issuers = if provider.caa_identities.is_empty() { + fallback_issuer.as_slice() + } else { + &[] + }; + + for issuer in provider + .caa_identities + .iter() + .map(String::as_str) + .chain(caa_issuers.iter().copied()) + { + records.push(NamedDnsRecord { + name: format!("{domain_name}."), + record: DnsRecord::CAA(CAARecord::Issue { + issuer_critical: false, + name: issuer.to_string().into(), + options: vec![KeyValue { + key: "accounturi".to_string(), + value: provider.account_uri.clone(), + }], + }), + }); + } if let Some(uri) = &domain.report_address_uri && uri.starts_with("mailto:") @@ -246,19 +265,34 @@ impl Server { }); } - // ACME DNS-PERSIST-01 validation record - records.push(NamedDnsRecord { - name: format!("_validation-persist.{domain_name}."), - record: DnsRecord::TXT(format!( - "{provider_name}; accounturi={}{}", - provider.account_uri, - if props.subject_alternative_names.is_empty() { - "; policy=wildcard" - } else { - "" + if provider.challenge_type == AcmeChallengeType::DnsPersist01 { + match PersistRecordBuilder::new(&provider) { + Ok(builder) => { + for (name, wildcard) in persist_validation_names( + domain_name, + props.subject_alternative_names.as_slice(), + ) { + if let Some(record) = builder.build(&name, wildcard) { + records.push(NamedDnsRecord { + name: format!("_validation-persist.{name}."), + record: DnsRecord::TXT(record), + }); + } + } } - )), - }); + Err(err) => { + trc::error!( + trc::AcmeEvent::Error + .into_err() + .reason(err) + .details(format!( + "Skipping DNS-PERSIST-01 records for {domain_name}" + )) + .caused_by(trc::location!()) + ); + } + } + } } } DnsRecordType::Tlsa => { @@ -382,24 +416,26 @@ impl Server { } pub async fn get_pacc_for_domain(&self, domain_name: &str) -> trc::Result { - self.get_directory_for_domain(domain_name) - .await - .caused_by(trc::location!()) - .map(|directory| { - directory - .and_then(|directory| { - directory - .oidc_discovery_document() - .map(|doc| self.core.network.info.pacc.build(&doc.url)) - }) - .unwrap_or_else(|| { - self.core - .network - .info - .pacc - .build(&self.core.network.http.url_https) - }) - }) + let pacc = &self.core.network.info.pacc; + Ok( + match self + .get_directory_for_domain(domain_name) + .await + .caused_by(trc::location!())? + { + Some(directory) if directory.has_bearer_token_support() => { + match directory.oidc_discovery_document() { + Some(discovery) + if discovery.conformance.serves_oauth_public(pacc.oauth_scopes) => + { + pacc.build(&discovery.document.issuer) + } + _ => pacc.password_only.clone(), + } + } + _ => pacc.build(&self.core.network.http.url_https), + }, + ) } } @@ -414,16 +450,6 @@ impl Pacc { } } -#[inline(always)] -#[allow(unused)] fn provider_domain(url: &Url) -> Option<&str> { - #[cfg(feature = "test_mode")] - { - Some("pebble.letsencrypt.org") - } - - #[cfg(not(feature = "test_mode"))] - { - url.host_str().and_then(psl::domain_str) - } + url.host_str().and_then(psl::domain_str) } diff --git a/crates/common/src/network/dns/resolve.rs b/crates/common/src/network/dns/resolve.rs index 4364937f..06361e74 100644 --- a/crates/common/src/network/dns/resolve.rs +++ b/crates/common/src/network/dns/resolve.rs @@ -18,7 +18,7 @@ impl Server { .mx_lookup(entry, Some(&self.inner.cache.dns_mx)) .await { - Ok(result) => Ok(result.rrset.iter().any(|mx| !mx.exchanges.is_empty())), + Ok(result) => Ok(result.records.iter().any(|mx| !mx.exchanges.is_empty())), Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => Ok(false), Err(err) => Err(err.into()), } @@ -55,7 +55,7 @@ impl Server { .ptr_lookup(addr, Some(&self.inner.cache.dns_ptr)) .await { - Ok(result) => Ok(!result.rrset.is_empty()), + Ok(result) => Ok(!result.records.is_empty()), Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => Ok(false), Err(err) => Err(err.into()), } @@ -73,7 +73,7 @@ impl Server { .ipv4_lookup(entry, Some(&self.inner.cache.dns_ipv4)) .await { - Ok(result) => Ok(!result.rrset.is_empty()), + Ok(result) => Ok(!result.records.is_empty()), Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => Ok(false), Err(err) => Err(err.into()), } @@ -88,7 +88,7 @@ impl Server { .ipv6_lookup(entry, Some(&self.inner.cache.dns_ipv6)) .await { - Ok(result) => Ok(!result.rrset.is_empty()), + Ok(result) => Ok(!result.records.is_empty()), Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => Ok(false), Err(err) => Err(err.into()), } diff --git a/crates/common/src/network/dns/update.rs b/crates/common/src/network/dns/update.rs index 76faa029..73fe0cb8 100644 --- a/crates/common/src/network/dns/update.rs +++ b/crates/common/src/network/dns/update.rs @@ -5,11 +5,12 @@ */ use crate::{Core, Server}; -use base64::{Engine, engine::general_purpose}; +use compact_str::{CompactString, ToCompactString, format_compact}; use dns_update::{ DnsRecord, DnsRecordType, TsigAlgorithm, providers::{ovh::OvhEndpoint, rfc2136::DnsAddress}, }; +use encodify::base64::STANDARD; use registry::schema::{ enums, structs::{DnsManagement, DnsServer, Domain}, @@ -31,7 +32,7 @@ pub struct DnsUpdater { core: Arc, } -#[cfg(feature = "test_mode")] +#[cfg(stalwart_tests)] pub static DNS_RECORDS: std::sync::LazyLock< Arc>>, > = std::sync::LazyLock::new(|| Arc::new(std::sync::Mutex::new(Vec::new()))); @@ -40,7 +41,7 @@ impl DnsUpdater { pub async fn build(server: DnsServer, core: Arc) -> Result { match server { DnsServer::Tsig(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -57,7 +58,7 @@ impl DnsUpdater { )), }, server.key_name, - general_purpose::STANDARD + STANDARD .decode(server.key.secret().await?.as_bytes()) .map_err(|err| format!("Failed to base64 decode TSIG key: {err}"))?, match server.tsig_algorithm { @@ -77,7 +78,7 @@ impl DnsUpdater { }), DnsServer::Cloudflare(server) => { let updater = { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] match server.secret.secret().await.unwrap().as_ref() { "test@pebble.org" => dns_update::DnsUpdater::new_pebble( "http://localhost:8055", @@ -93,7 +94,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] dns_update::DnsUpdater::new_cloudflare( server.secret.secret().await?, server.timeout.into_inner().into(), @@ -102,7 +103,7 @@ impl DnsUpdater { }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -111,7 +112,7 @@ impl DnsUpdater { }) } DnsServer::DigitalOcean(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -123,7 +124,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::DeSEC(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -135,7 +136,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Ovh(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -157,7 +158,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Bunny(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -169,46 +170,46 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Porkbun(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_porkbun( - server.api_key.as_str(), - server.secret_api_key.secret().await?, + server.api_key_id.as_str(), + server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Dnsimple(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_dnsimple( - server.auth_token.secret().await?, + server.token.secret().await?, server.account_identifier.as_str(), server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Spaceship(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_spaceship( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Route53(server) => { - let secret_access_key = server.secret_access_key.secret().await?.into_owned(); + let secret_access_key = server.secret_key.secret().await?.into_owned(); let session_token = server.session_token.secret().await?.map(|c| c.into_owned()); let config = dns_update::providers::route53::Route53Config { access_key_id: server.access_key_id, @@ -220,7 +221,7 @@ impl DnsUpdater { endpoint: None, }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -240,7 +241,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -251,22 +252,18 @@ impl DnsUpdater { } DnsServer::Alidns(server) => { let secret_key = server.secret_key.secret().await?.into_owned(); - let security_token = server - .security_token - .secret() - .await? - .map(|c| c.into_owned()); + let session_token = server.session_token.secret().await?.map(|c| c.into_owned()); Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_alidns( - server.access_key.as_str(), + server.access_key_id.as_str(), secret_key.as_str(), server.region.as_deref(), - security_token.as_deref(), + session_token.as_deref(), server.line.as_deref(), server.timeout.into_inner().into(), ) @@ -274,7 +271,7 @@ impl DnsUpdater { }) } DnsServer::ArvanCloud(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -286,7 +283,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Autodns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -321,7 +318,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -331,13 +328,13 @@ impl DnsUpdater { }) } DnsServer::BaiduCloud(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_baiducloud( - server.access_key.as_str(), + server.access_key_id.as_str(), server.secret_key.secret().await?, server.timeout.into_inner().into(), ) @@ -346,7 +343,7 @@ impl DnsUpdater { DnsServer::BluecatV2(server) => { let password = server.password.secret().await?.into_owned(); let config = dns_update::providers::bluecatv2::BluecatV2Config { - server_url: server.base_url, + server_url: server.url, username: server.username, password, config_name: server.config_name, @@ -355,7 +352,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -365,7 +362,7 @@ impl DnsUpdater { }) } DnsServer::ClouDns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -379,26 +376,26 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Constellix(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_constellix( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret_key.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Cpanel(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_cpanel( - server.base_url.as_str(), + server.url.as_str(), server.username.as_str(), server.token.secret().await?, server.timeout.into_inner().into(), @@ -406,7 +403,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Ddnss(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -418,33 +415,33 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::DnsMadeEasy(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_dnsmadeeasy( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Domeneshop(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_domeneshop( - server.auth_token.as_str(), + server.token.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Dreamhost(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -456,7 +453,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::DuckDns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -468,7 +465,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Dynu(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -480,7 +477,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::EasyDns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -504,7 +501,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -514,20 +511,20 @@ impl DnsUpdater { }) } DnsServer::Exoscale(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_exoscale( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::FreeMyIp(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -539,7 +536,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::GandiV5(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -551,7 +548,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Gcore(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -563,33 +560,33 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Glesys(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_glesys( - server.api_user.as_str(), + server.username.as_str(), server.api_key.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Godaddy(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_godaddy( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Hetzner(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -601,7 +598,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::HostingDe(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -613,7 +610,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Hostinger(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -625,13 +622,13 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::HuaweiCloud(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_huaweicloud( - server.access_key.as_str(), + server.access_key_id.as_str(), server.secret_key.secret().await?, server.region.as_str(), server.timeout.into_inner().into(), @@ -645,7 +642,7 @@ impl DnsUpdater { credentials.insert(cred.zone, secret); } Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -658,7 +655,7 @@ impl DnsUpdater { }) } DnsServer::IbmCloud(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -674,7 +671,7 @@ impl DnsUpdater { let password = server.password.secret().await?.into_owned(); let config = dns_update::providers::infoblox::InfobloxConfig { host: server.host, - port: server.port, + port: server.port.map(|port| port.to_string()), username: server.username, password, wapi_version: server.wapi_version, @@ -682,7 +679,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -692,7 +689,7 @@ impl DnsUpdater { }) } DnsServer::Infomaniak(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -706,7 +703,7 @@ impl DnsUpdater { DnsServer::Inwx(server) => { let password = server.password.secret().await?.into_owned(); Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -721,7 +718,7 @@ impl DnsUpdater { }) } DnsServer::Ionos(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -733,7 +730,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Ipv64(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -759,7 +756,7 @@ impl DnsUpdater { } }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -772,7 +769,7 @@ impl DnsUpdater { }) } DnsServer::Lightsail(server) => { - let secret_access_key = server.secret_access_key.secret().await?.into_owned(); + let secret_access_key = server.secret_key.secret().await?.into_owned(); let session_token = server.session_token.secret().await?.map(|c| c.into_owned()); let config = dns_update::providers::lightsail::LightsailConfig { access_key_id: server.access_key_id, @@ -783,7 +780,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -793,7 +790,7 @@ impl DnsUpdater { }) } DnsServer::Linode(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -805,20 +802,20 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::LuaDns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_luadns( server.username.as_str(), - server.auth_token.secret().await?, + server.token.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::MythicBeasts(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -831,7 +828,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Namecheap(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -846,20 +843,20 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::NameDotCom(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_namedotcom( server.username.as_str(), - server.auth_token.secret().await?, + server.token.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::NameSilo(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -871,21 +868,21 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Netcup(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_netcup( server.customer_number.as_str(), - server.api_key.as_str(), + server.api_key_id.as_str(), server.password.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Netlify(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -897,20 +894,20 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Nifcloud(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_nifcloud( - server.access_key.as_str(), + server.access_key_id.as_str(), server.secret_key.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Ns1(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -939,7 +936,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -949,34 +946,34 @@ impl DnsUpdater { }) } DnsServer::Plesk(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_plesk( - server.base_url.as_str(), + server.url.as_str(), server.api_key.secret().await?, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::PowerDns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_pdns( server.api_key.secret().await?, - server.endpoint, + server.url, server.server_id, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Safedns(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -988,7 +985,7 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::Scaleway(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1003,13 +1000,13 @@ impl DnsUpdater { let secret_key = server.secret_key.secret().await?.into_owned(); let session_token = server.session_token.secret().await?.map(|c| c.into_owned()); Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_tencentcloud( - server.secret_id.as_str(), + server.access_key_id.as_str(), secret_key.as_str(), server.region.as_deref(), session_token.as_deref(), @@ -1019,7 +1016,7 @@ impl DnsUpdater { }) } DnsServer::Transip(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1035,7 +1032,7 @@ impl DnsUpdater { DnsServer::UltraDns(server) => { let password = server.password.secret().await?.into_owned(); Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1043,20 +1040,20 @@ impl DnsUpdater { updater: dns_update::DnsUpdater::new_ultradns( server.username, password, - server.endpoint, + server.url, server.timeout.into_inner().into(), ) .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }) } DnsServer::Vercel(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_vercel( - server.auth_token.secret().await?, + server.token.secret().await?, server.team_id.as_deref(), server.timeout.into_inner().into(), ) @@ -1065,7 +1062,7 @@ impl DnsUpdater { DnsServer::Volcengine(server) => { let secret_key = server.secret_key.secret().await?.into_owned(); let config = dns_update::providers::volcengine::VolcengineConfig { - access_key: server.access_key, + access_key: server.access_key_id, secret_key, region: server.region, host: server.host, @@ -1073,7 +1070,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1083,7 +1080,7 @@ impl DnsUpdater { }) } DnsServer::Vultr(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1095,13 +1092,13 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }), DnsServer::WebSupport(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), core, updater: dns_update::DnsUpdater::new_websupport( - server.api_key.as_str(), + server.api_key_id.as_str(), server.secret.secret().await?, server.timeout.into_inner().into(), ) @@ -1115,7 +1112,7 @@ impl DnsUpdater { request_timeout: Some(server.timeout.into_inner()), }; Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), + polling_interval: server.poll_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), propagation_delay: server.propagation_delay.map(|d| d.into_inner()), ttl: server.ttl.into_inner(), @@ -1124,7 +1121,6 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }) } - DnsServer::Deprecated1 => Err("DNS server type no longer supported".to_string()), } } @@ -1134,10 +1130,10 @@ impl DnsUpdater { name: &str, record_type: DnsRecordType, records: Vec, - ) -> Result<(), String> { + ) -> dns_update::Result<()> { let record_values = records .iter() - .map(|r| trc::Value::String(r.to_string().into())) + .map(|r| trc::Value::String(r.to_compact_string())) .collect::>(); if let Err(err) = self @@ -1151,13 +1147,21 @@ impl DnsUpdater { ) .await { - return Err(format!("Failed to set DNS RRSet: {}", err)); + trc::event!( + Dns(DnsEvent::RecordCreationFailed), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), + Type = record_type.as_str(), + Value = record_values, + Reason = format_compact!("Failed to set DNS RRSet: {err}"), + ); + return Err(err); } trc::event!( Dns(DnsEvent::RecordCreated), - Hostname = name.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), Type = record_type.as_str(), Value = record_values, ); @@ -1183,10 +1187,10 @@ impl DnsUpdater { { trc::event!( Dns(DnsEvent::RecordDeletionFailed), - Hostname = name.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), Type = record_type.as_str(), - Reason = err.to_string(), + Reason = err.to_compact_string(), ); return Err(format!("Failed to delete DNS RRSet: {}", err)); } @@ -1203,7 +1207,7 @@ impl DnsUpdater { ) -> Result<(), String> { let record_values = records .iter() - .map(|r| trc::Value::String(r.to_string().into())) + .map(|r| trc::Value::String(r.to_compact_string())) .collect::>(); if let Err(err) = self @@ -1222,8 +1226,8 @@ impl DnsUpdater { trc::event!( Dns(DnsEvent::RecordCreated), - Hostname = name.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), Type = record_type.as_str(), Value = record_values, ); @@ -1248,7 +1252,7 @@ impl DnsUpdater { ) -> Result<(), String> { let record_values = records .iter() - .map(|r| trc::Value::String(r.to_string().into())) + .map(|r| trc::Value::String(r.to_compact_string())) .collect::>(); match self @@ -1260,9 +1264,9 @@ impl DnsUpdater { Err(err) => { trc::event!( Dns(DnsEvent::RecordDeletionFailed), - Hostname = name.to_string(), - Reason = err.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Reason = err.to_compact_string(), + Domain = CompactString::from(origin), Type = record_type.as_str(), Value = record_values, ); @@ -1271,8 +1275,60 @@ impl DnsUpdater { } } + pub async fn check_txt_propagation(&self, name: &str, origin: &str, expected: &str) -> bool { + #[cfg(stalwart_tests)] + if matches!( + self.updater, + dns_update::DnsUpdater::Pebble(_) | dns_update::DnsUpdater::InMemory(_) + ) { + return true; + } + + let did_propagate = self.lookup_txt(name, origin, expected).await; + if did_propagate { + trc::event!( + Dns(DnsEvent::RecordPropagated), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), + Type = DnsRecordType::TXT.as_str(), + Value = CompactString::from(expected), + ); + } + did_propagate + } + + async fn lookup_txt(&self, name: &str, origin: &str, expected: &str) -> bool { + match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { + Ok(result) => { + let result = std::str::from_utf8(&result).unwrap_or_default(); + if result.contains(expected) { + return true; + } + trc::event!( + Dns(DnsEvent::RecordNotPropagated), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), + Result = CompactString::from(result), + Type = DnsRecordType::TXT.as_str(), + Value = CompactString::from(expected), + ); + } + Err(err) => { + trc::event!( + Dns(DnsEvent::RecordLookupFailed), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), + Reason = err.to_compact_string(), + Type = DnsRecordType::TXT.as_str(), + Value = CompactString::from(expected), + ); + } + } + false + } + pub async fn wait_for_txt_propagation(&self, name: &str, origin: &str, expected: &str) -> bool { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if matches!( self.updater, dns_update::DnsUpdater::Pebble(_) | dns_update::DnsUpdater::InMemory(_) @@ -1286,33 +1342,9 @@ impl DnsUpdater { let wait_until = Instant::now() + self.propagation_timeout; let mut did_propagate = false; while Instant::now() < wait_until { - match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { - Ok(result) => { - let result = std::str::from_utf8(&result).unwrap_or_default(); - if result.contains(expected) { - did_propagate = true; - break; - } else { - trc::event!( - Dns(DnsEvent::RecordNotPropagated), - Hostname = name.to_string(), - Details = origin.to_string(), - Result = result.to_string(), - Type = DnsRecordType::TXT.as_str(), - Value = expected.to_string(), - ); - } - } - Err(err) => { - trc::event!( - Dns(DnsEvent::RecordLookupFailed), - Hostname = name.to_string(), - Details = origin.to_string(), - Reason = err.to_string(), - Type = DnsRecordType::TXT.as_str(), - Value = expected.to_string(), - ); - } + if self.lookup_txt(name, origin, expected).await { + did_propagate = true; + break; } tokio::time::sleep(self.polling_interval).await; @@ -1321,18 +1353,18 @@ impl DnsUpdater { if did_propagate { trc::event!( Dns(DnsEvent::RecordPropagated), - Hostname = name.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), Type = DnsRecordType::TXT.as_str(), - Value = expected.to_string(), + Value = CompactString::from(expected), ); } else { trc::event!( Dns(DnsEvent::RecordPropagationTimeout), - Hostname = name.to_string(), - Details = origin.to_string(), + Hostname = CompactString::from(name), + Domain = CompactString::from(origin), Type = DnsRecordType::TXT.as_str(), - Value = expected.to_string(), + Value = CompactString::from(expected), ); } did_propagate diff --git a/crates/common/src/network/forwarded.rs b/crates/common/src/network/forwarded.rs new file mode 100644 index 00000000..0aeeea8e --- /dev/null +++ b/crates/common/src/network/forwarded.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use hyper::header::{FORWARDED, HeaderMap, HeaderName}; +use registry::schema::enums::ClientIpSource; +use std::{ + net::{IpAddr, Ipv6Addr, SocketAddr}, + num::NonZeroU8, + str::{FromStr, from_utf8}, +}; + +pub const MAX_FORWARDED_HOPS: u8 = 16; + +const X_FORWARDED_FOR: HeaderName = HeaderName::from_static("x-forwarded-for"); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RemoteIpSource { + Peer, + XForwardedFor { hops: NonZeroU8 }, + Forwarded { hops: NonZeroU8 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientIp { + Peer, + Forwarded(IpAddr), + Missing, + Invalid, +} + +impl RemoteIpSource { + pub fn new(source: ClientIpSource, hops: u64) -> Self { + let hops = u8::try_from(hops.clamp(1, u64::from(MAX_FORWARDED_HOPS))) + .ok() + .and_then(NonZeroU8::new) + .unwrap_or(NonZeroU8::MIN); + match source { + ClientIpSource::Peer => RemoteIpSource::Peer, + ClientIpSource::XForwardedFor => RemoteIpSource::XForwardedFor { hops }, + ClientIpSource::Forwarded => RemoteIpSource::Forwarded { hops }, + } + } + + pub fn is_peer(&self) -> bool { + matches!(self, RemoteIpSource::Peer) + } + + pub fn resolve(&self, headers: &HeaderMap) -> ClientIp { + let node = match self { + RemoteIpSource::Peer => return ClientIp::Peer, + RemoteIpSource::XForwardedFor { hops } if headers.contains_key(X_FORWARDED_FOR) => { + headers + .get_all(X_FORWARDED_FOR) + .iter() + .rev() + .flat_map(|value| value.as_bytes().rsplit(|&byte| byte == b',')) + .map(<[u8]>::trim_ascii) + .filter(|entry| !entry.is_empty()) + .nth(usize::from(hops.get() - 1)) + } + RemoteIpSource::Forwarded { hops } if headers.contains_key(FORWARDED) => headers + .get_all(FORWARDED) + .iter() + .rev() + .flat_map(|value| RSplitUnquoted::new(value.as_bytes(), b',')) + .map(<[u8]>::trim_ascii) + .filter(|element| !element.is_empty()) + .nth(usize::from(hops.get() - 1)) + .and_then(forwarded_for), + RemoteIpSource::XForwardedFor { .. } | RemoteIpSource::Forwarded { .. } => { + return ClientIp::Missing; + } + }; + node.and_then(parse_node) + .map_or(ClientIp::Invalid, ClientIp::Forwarded) + } +} + +fn forwarded_for(element: &[u8]) -> Option<&[u8]> { + let mut node = None; + for pair in RSplitUnquoted::new(element, b';') { + let pair = pair.trim_ascii(); + if pair.is_empty() { + continue; + } + let mut parts = pair.splitn(2, |&byte| byte == b'='); + let (Some(name), Some(value)) = (parts.next(), parts.next()) else { + return None; + }; + if name.trim_ascii().eq_ignore_ascii_case(b"for") + && node.replace(value.trim_ascii()).is_some() + { + return None; + } + } + node.map(|node: &[u8]| { + node.strip_prefix(b"\"") + .and_then(|node| node.strip_suffix(b"\"")) + .unwrap_or(node) + }) +} + +fn parse_node(node: &[u8]) -> Option { + let node = from_utf8(node).ok()?; + let ip = IpAddr::from_str(node) + .ok() + .or_else(|| SocketAddr::from_str(node).ok().map(|addr| addr.ip())) + .or_else(|| { + node.strip_prefix('[')? + .strip_suffix(']')? + .parse::() + .ok() + .map(IpAddr::V6) + })? + .to_canonical(); + (!is_loopback_claim(ip)).then_some(ip) +} + +fn is_loopback_claim(ip: IpAddr) -> bool { + ip.is_loopback() + || matches!(ip, IpAddr::V6(ip) if matches!(ip.octets(), [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, ..])) +} + +struct RSplitUnquoted<'x> { + bytes: Option<&'x [u8]>, + delimiter: u8, +} + +impl<'x> RSplitUnquoted<'x> { + fn new(bytes: &'x [u8], delimiter: u8) -> Self { + RSplitUnquoted { + bytes: Some(bytes), + delimiter, + } + } +} + +impl<'x> Iterator for RSplitUnquoted<'x> { + type Item = &'x [u8]; + + fn next(&mut self) -> Option<&'x [u8]> { + let bytes = self.bytes?; + let mut in_quotes = false; + for (pos, &byte) in bytes.iter().enumerate().rev() { + if byte == b'"' { + let is_escaped = in_quotes + && bytes + .get(..pos) + .unwrap_or_default() + .iter() + .rev() + .take_while(|&&byte| byte == b'\\') + .count() + % 2 + == 1; + if !is_escaped { + in_quotes = !in_quotes; + } + } else if byte == self.delimiter && !in_quotes { + self.bytes = bytes.get(..pos); + return bytes.get(pos + 1..); + } + } + self.bytes = None; + Some(bytes) + } +} + diff --git a/crates/common/src/network/limiter.rs b/crates/common/src/network/limiter.rs index 8c5025ab..e388c974 100644 --- a/crates/common/src/network/limiter.rs +++ b/crates/common/src/network/limiter.rs @@ -4,14 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ThrottleKey, ThrottleKeyHasher, ThrottleKeyHasherBuilder}; +use crate::ThrottleKey; +use registry::schema::structs::Rate; use std::{ - hash::{BuildHasher, Hasher}, + hash::Hasher, sync::{ Arc, - atomic::{AtomicU64, Ordering}, + atomic::{AtomicU32, AtomicU64, Ordering}, }, }; +use store::dispatch::lookup::RateWindow; #[derive(Debug, Clone)] #[repr(transparent)] @@ -23,11 +25,72 @@ pub struct ConcurrencyLimiterInner { concurrent: AtomicU64, } -pub struct InFlight(Arc); +#[derive(Debug, Default)] +pub struct AccountLimits { + concurrency: [AccountConcurrency; AccountLimit::COUNT], + rates: [RateCounter; AccountRate::COUNT], + enabled: u8, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccountLimit { + HttpRequests = 0, + ImapRequests = 1, + Uploads = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccountRate { + Http = 0, + Imap = 1, +} + +#[derive(Debug, Default)] +struct AccountConcurrency { + max_concurrent: u32, + concurrent: AtomicU32, +} + +pub struct InFlight(InFlightSlot); + +enum InFlightSlot { + Limiter(Arc), + Account(Arc, AccountLimit), + Untracked, +} impl Drop for InFlight { fn drop(&mut self) { - self.0.concurrent.fetch_sub(1, Ordering::Relaxed); + match &self.0 { + InFlightSlot::Limiter(limiter) => { + limiter.concurrent.fetch_sub(1, Ordering::Relaxed); + } + InFlightSlot::Account(limits, limit) => { + limits.concurrency[*limit as usize] + .concurrent + .fetch_sub(1, Ordering::Relaxed); + } + InFlightSlot::Untracked => {} + } + } +} + +impl AccountConcurrency { + fn new(max_concurrent: u64) -> Self { + AccountConcurrency { + max_concurrent: u32::try_from(max_concurrent).unwrap_or(u32::MAX), + concurrent: AtomicU32::new(0), + } + } + + #[inline] + fn try_acquire(&self) -> bool { + if self.concurrent.load(Ordering::Relaxed) < self.max_concurrent { + self.concurrent.fetch_add(1, Ordering::Relaxed); + true + } else { + false + } } } @@ -43,7 +106,7 @@ impl ConcurrencyLimiter { if self.0.concurrent.load(Ordering::Relaxed) < self.0.max_concurrent { // Return in-flight request self.0.concurrent.fetch_add(1, Ordering::Relaxed); - LimiterResult::Allowed(InFlight(self.0.clone())) + LimiterResult::Allowed(InFlight(InFlightSlot::Limiter(self.0.clone()))) } else { LimiterResult::Forbidden } @@ -62,9 +125,79 @@ impl ConcurrencyLimiter { } } +impl AccountLimit { + const COUNT: usize = 3; +} + +impl AccountRate { + const COUNT: usize = 2; +} + +impl AccountLimits { + pub fn new( + concurrency: [Option; AccountLimit::COUNT], + rates: [bool; AccountRate::COUNT], + ) -> Option> { + let mut limits = AccountLimits::default(); + for (bit, (max_concurrent, limiter)) in concurrency + .into_iter() + .zip(&mut limits.concurrency) + .enumerate() + { + if let Some(max_concurrent) = max_concurrent { + *limiter = AccountConcurrency::new(max_concurrent); + limits.enabled |= 1 << bit; + } + } + for (bit, enabled) in rates.into_iter().enumerate() { + if enabled { + limits.enabled |= 1 << (AccountLimit::COUNT + bit); + } + } + (limits.enabled != 0).then(|| Arc::new(limits)) + } + + #[inline] + fn has_limit(&self, limit: AccountLimit) -> bool { + self.enabled & (1 << limit as u8) != 0 + } + + pub fn is_allowed(self: &Arc, limit: AccountLimit) -> LimiterResult { + if !self.has_limit(limit) { + LimiterResult::Disabled + } else if self.concurrency[limit as usize].try_acquire() { + LimiterResult::Allowed(InFlight(InFlightSlot::Account(self.clone(), limit))) + } else { + LimiterResult::Forbidden + } + } + + pub fn max_concurrent(&self, limit: AccountLimit) -> u64 { + if self.has_limit(limit) { + u64::from(self.concurrency[limit as usize].max_concurrent) + } else { + 0 + } + } + + #[inline] + pub fn rate_counter(&self, rate: AccountRate) -> Option<&RateCounter> { + (self.enabled & (1 << (AccountLimit::COUNT + rate as usize)) != 0) + .then(|| &self.rates[rate as usize]) + } +} + impl InFlight { pub fn num_concurrent(&self) -> u64 { - self.0.concurrent.load(Ordering::Relaxed) + match &self.0 { + InFlightSlot::Limiter(limiter) => limiter.concurrent.load(Ordering::Relaxed), + InFlightSlot::Account(limits, limit) => u64::from( + limits.concurrency[*limit as usize] + .concurrent + .load(Ordering::Relaxed), + ), + InFlightSlot::Untracked => 0, + } } } @@ -79,11 +212,208 @@ impl From for Option { match result { LimiterResult::Allowed(in_flight) => Some(in_flight), LimiterResult::Forbidden => None, - LimiterResult::Disabled => Some(InFlight(Arc::new(ConcurrencyLimiterInner::default()))), + LimiterResult::Disabled => Some(InFlight(InFlightSlot::Untracked)), } } } +#[derive(Debug, Default)] +#[repr(transparent)] +pub struct RateCounter(AtomicU64); + +#[derive(Debug)] +pub enum RateHit<'x> { + Allowed, + Exceeded { reset_in: u64 }, + Flush(RateFlush<'x>), +} + +#[derive(Debug)] +pub struct RateFlush<'x> { + counter: &'x RateCounter, + window: RateWindow, + delta: u32, + owner: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RateState { + window: u32, + total: u32, + learned: bool, + in_flight: bool, + pending: u32, +} + +const RATE_PENDING_BITS: u32 = 5; +const RATE_PENDING_MASK: u64 = (1 << RATE_PENDING_BITS) - 1; +const RATE_IN_FLIGHT: u64 = 1 << RATE_PENDING_BITS; +const RATE_LEARNED: u64 = 1 << (RATE_PENDING_BITS + 1); +const RATE_TOTAL_SHIFT: u32 = RATE_PENDING_BITS + 2; +const RATE_TOTAL_MAX: u32 = (1 << (u32::BITS - RATE_TOTAL_SHIFT)) - 1; +const RATE_BATCH_DIVISOR: u64 = 64; +const RATE_MAX_BATCH: u64 = 32; + +const _: () = assert!(RATE_MAX_BATCH - 1 <= RATE_PENDING_MASK); + +impl RateCounter { + fn batch_size(rate: &Rate) -> u32 { + if rate.count <= u64::from(RATE_TOTAL_MAX) { + (rate.count / RATE_BATCH_DIVISOR).clamp(1, RATE_MAX_BATCH) as u32 + } else { + 1 + } + } + + pub fn hit(&self, rate: &Rate, now: u64) -> RateHit<'_> { + let window = RateWindow::new(rate, now); + let batch = Self::batch_size(rate); + if batch == 1 { + return self.flush(window, 1, false); + } + + let index = window.index as u32; + let mut current = self.0.load(Ordering::Relaxed); + loop { + let state = RateState::from(current); + let (next, flush) = if state.window == index { + if u64::from(state.total) + u64::from(state.pending) >= rate.count { + return RateHit::Exceeded { + reset_in: window.expires_in, + }; + } + let pending = state.pending + 1; + if !state.learned || (pending == batch && state.in_flight) { + ( + RateState { + total: state.total.saturating_add(1).min(RATE_TOTAL_MAX), + ..state + }, + Some((1, false)), + ) + } else if pending < batch { + (RateState { pending, ..state }, None) + } else { + ( + RateState { + total: state.total.saturating_add(pending).min(RATE_TOTAL_MAX), + in_flight: true, + pending: 0, + ..state + }, + Some((pending, true)), + ) + } + } else if state.window == index.wrapping_add(1) { + return self.flush(window, 1, false); + } else { + ( + RateState { + window: index, + total: 1, + learned: false, + in_flight: true, + pending: 0, + }, + Some((1, true)), + ) + }; + + match self.0.compare_exchange_weak( + current, + next.into(), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + return match flush { + Some((delta, owner)) => self.flush(window, delta, owner), + None => RateHit::Allowed, + }; + } + Err(actual) => current = actual, + } + } + } + + fn flush(&self, window: RateWindow, delta: u32, owner: bool) -> RateHit<'_> { + RateHit::Flush(RateFlush { + counter: self, + window, + delta, + owner, + }) + } + + fn update(&self, window: RateWindow, known: u32, release: bool) { + let index = window.index as u32; + let _ = self + .0 + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + let state = RateState::from(current); + let next = RateState { + total: state.total.max(known), + learned: state.learned || known > 0, + in_flight: state.in_flight && !release, + ..state + }; + (state.window == index && next != state).then(|| next.into()) + }); + } +} + +impl RateFlush<'_> { + pub fn window(&self) -> RateWindow { + self.window + } + + pub fn delta(&self) -> u32 { + self.delta + } + + pub fn settle(mut self, rate: &Rate, total: u64) -> Option { + let known = u32::try_from(total).unwrap_or(u32::MAX).min(RATE_TOTAL_MAX); + self.counter + .update(self.window, known, std::mem::take(&mut self.owner)); + (total > rate.count).then_some(self.window.expires_in) + } +} + +impl Drop for RateFlush<'_> { + fn drop(&mut self) { + if self.owner { + self.counter.update(self.window, 0, true); + } + } +} + +impl From for RateState { + fn from(value: u64) -> Self { + RateState { + window: (value >> u32::BITS) as u32, + total: ((value >> RATE_TOTAL_SHIFT) as u32) & RATE_TOTAL_MAX, + learned: value & RATE_LEARNED != 0, + in_flight: value & RATE_IN_FLIGHT != 0, + pending: (value & RATE_PENDING_MASK) as u32, + } + } +} + +impl From for u64 { + fn from(state: RateState) -> Self { + let mut value = (u64::from(state.window) << u32::BITS) + | (u64::from(state.total) << RATE_TOTAL_SHIFT) + | u64::from(state.pending); + if state.learned { + value |= RATE_LEARNED; + } + if state.in_flight { + value |= RATE_IN_FLIGHT; + } + value + } +} + impl PartialEq for ThrottleKey { fn eq(&self, other: &Self) -> bool { self.hash == other.hash @@ -102,26 +432,9 @@ impl AsRef<[u8]> for ThrottleKey { } } -impl Hasher for ThrottleKeyHasher { - fn finish(&self) -> u64 { - self.hash - } - - fn write(&mut self, bytes: &[u8]) { - debug_assert!( - bytes.len() >= std::mem::size_of::(), - "ThrottleKeyHasher: input too short {bytes:?}" - ); - self.hash = bytes - .get(0..std::mem::size_of::()) - .map_or(0, |b| u64::from_ne_bytes(b.try_into().unwrap())); +impl ThrottleKey { + pub fn quota_key(&self) -> u128 { + u128::from_be_bytes(self.hash[..std::mem::size_of::()].try_into().unwrap()) } } -impl BuildHasher for ThrottleKeyHasherBuilder { - type Hasher = ThrottleKeyHasher; - - fn build_hasher(&self) -> Self::Hasher { - ThrottleKeyHasher::default() - } -} diff --git a/crates/common/src/network/listen.rs b/crates/common/src/network/listen.rs index c790838e..ec6a7d2f 100644 --- a/crates/common/src/network/listen.rs +++ b/crates/common/src/network/listen.rs @@ -12,8 +12,11 @@ use crate::{ BuildServer, Inner, Server, config::server::{Listener, Listeners, ServerProtocol, TcpListener}, }; +use compact_str::{ToCompactString, format_compact}; use proxy_header::io::ProxiedStream; +use registry::types::id::ObjectId; use rustls::crypto::aws_lc_rs::cipher_suite::TLS13_AES_128_GCM_SHA256; +use socket2::{SockRef, TcpKeepalive}; use std::{ io, net::{IpAddr, SocketAddr}, @@ -137,8 +140,8 @@ impl Listener { opts.apply(&stream); tokio::spawn(async move { - match ProxiedStream::create_from_tokio(stream, Default::default()).await { - Ok(stream) =>{ + match timeout(instance.tls_timeout, ProxiedStream::create_from_tokio(stream, Default::default())).await { + Ok(Ok(stream)) =>{ let (remote_addr, local_addr) = stream.proxy_header() .proxied_address() .map(|addr| { @@ -158,14 +161,27 @@ impl Listener { manager.spawn(session, is_tls, enable_acme, span_start, span_end); } } - Err(err) => { + Err(_) => { + trc::event!( + Network(trc::NetworkEvent::Timeout), + ListenerId = instance.id.clone(), + LocalIp = local_addr.ip(), + LocalPort = local_addr.port(), + RemoteIp = remote_addr.ip(), + Tls = is_tls, + Reason = "PROXY protocol header timeout", + ); + } + Ok(Err(err)) => { trc::event!( Network(trc::NetworkEvent::ProxyError), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), LocalPort = local_addr.port(), + RemoteIp = remote_addr.ip(), + RemotePort = remote_addr.port(), Tls = is_tls, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -194,7 +210,7 @@ impl Listener { LocalIp = local_addr.ip(), LocalPort = local_addr.port(), Tls = is_tls, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); tokio::select! { @@ -301,22 +317,56 @@ pub struct SocketOpts { pub ttl: Option, } +const KEEPALIVE_IDLE: Duration = Duration::from_secs(300); +#[cfg(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd", + windows +))] +const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +const KEEPALIVE_RETRIES: u32 = 4; +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +const KEEPALIVE: TcpKeepalive = TcpKeepalive::new() + .with_time(KEEPALIVE_IDLE) + .with_interval(KEEPALIVE_INTERVAL) + .with_retries(KEEPALIVE_RETRIES); +#[cfg(windows)] +const KEEPALIVE: TcpKeepalive = TcpKeepalive::new() + .with_time(KEEPALIVE_IDLE) + .with_interval(KEEPALIVE_INTERVAL); +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd", + windows +)))] +const KEEPALIVE: TcpKeepalive = TcpKeepalive::new().with_time(KEEPALIVE_IDLE); + impl SocketOpts { pub fn apply(&self, stream: &TcpStream) { // Set TCP options if let Err(err) = stream.set_nodelay(self.nodelay) { trc::event!( Network(trc::NetworkEvent::SetOptError), - Reason = err.to_string(), + Reason = err.to_compact_string(), Details = "Failed to set TCP_NODELAY", ); } + if let Err(err) = SockRef::from(stream).set_tcp_keepalive(&KEEPALIVE) { + trc::event!( + Network(trc::NetworkEvent::SetOptError), + Reason = err.to_compact_string(), + Details = "Failed to set TCP keepalive", + ); + } if let Some(ttl) = self.ttl && let Err(err) = stream.set_ttl(ttl) { trc::event!( Network(trc::NetworkEvent::SetOptError), - Reason = err.to_string(), + Reason = err.to_compact_string(), Details = "Failed to set TTL", ); } @@ -324,18 +374,11 @@ impl SocketOpts { } impl Listeners { - pub fn bind_and_drop_priv(&self, bp: &mut Bootstrap) { + pub fn bind_and_drop_priv(&mut self, bp: &mut Bootstrap) { // Bind as root - for server in &self.servers { - for listener in &server.listeners { - if let Err(err) = listener.socket.bind(listener.addr) { - bp.build_error( - server.registry_id, - format!("Failed to bind to {}: {}", listener.addr, err), - ); - } - } - } + self.bind(|registry_id, addr, err| { + bp.build_error(registry_id, format!("Failed to bind to {addr}: {err}")); + }); // Drop privileges #[cfg(not(target_env = "msvc"))] @@ -354,6 +397,30 @@ impl Listeners { } } + fn bind(&mut self, mut on_error: impl FnMut(ObjectId, SocketAddr, io::Error)) { + for server in &mut self.servers { + let registry_id = server.registry_id; + let server_id = &server.id; + server + .listeners + .retain(|listener| match listener.socket.bind(listener.addr) { + Ok(()) => true, + Err(err) => { + trc::event!( + Network(trc::NetworkEvent::ListenError), + ListenerId = server_id.clone(), + LocalIp = listener.addr.ip(), + LocalPort = listener.addr.port(), + Reason = err.to_compact_string(), + ); + on_error(registry_id, listener.addr, err); + false + } + }); + } + self.servers.retain(|server| !server.listeners.is_empty()); + } + pub fn spawn( mut self, spawn: impl Fn(Listener, TcpAcceptor, watch::Receiver), @@ -374,9 +441,26 @@ impl Listeners { impl TcpListener { pub fn listen(self) -> Result { - self.socket - .listen(self.backlog.unwrap_or(1024)) - .map_err(|err| format!("Failed to listen on {}: {}", self.addr, err)) + match self.socket.local_addr() { + Ok(local_addr) if self.is_bound_to(local_addr) => self + .socket + .listen(self.backlog.unwrap_or(1024)) + .map_err(|err| format!("Failed to listen on {}: {}", self.addr, err)), + Ok(local_addr) => Err(format!( + "Refusing to listen on {}: socket is bound to {local_addr}", + self.addr + )), + Err(err) => Err(format!( + "Refusing to listen on {}: socket is not bound: {err}", + self.addr + )), + } + } + + fn is_bound_to(&self, local_addr: SocketAddr) -> bool { + local_addr.ip() == self.addr.ip() + && local_addr.port() != 0 + && (self.addr.port() == 0 || local_addr.port() == self.addr.port()) } } @@ -397,7 +481,7 @@ impl ServerInstance { Tls(trc::TlsEvent::Handshake), ListenerId = self.id.clone(), SpanId = session_id, - Version = format!( + Version = format_compact!( "{:?}", stream .get_ref() @@ -405,7 +489,7 @@ impl ServerInstance { .protocol_version() .unwrap_or(rustls::ProtocolVersion::TLSv1_3) ), - Details = format!( + Cipher = format_compact!( "{:?}", stream .get_ref() @@ -421,7 +505,7 @@ impl ServerInstance { Tls(trc::TlsEvent::HandshakeError), ListenerId = self.id.clone(), SpanId = session_id, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); Err(()) } @@ -438,3 +522,4 @@ impl ServerInstance { } } } + diff --git a/crates/common/src/network/memory.rs b/crates/common/src/network/memory.rs new file mode 100644 index 00000000..939dbb29 --- /dev/null +++ b/crates/common/src/network/memory.rs @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +pub const MEMORY_STEP: u64 = 1024 * 1024; +pub const MEMORY_RETRY_AFTER: u64 = 5; +const RESERVATION_GRANULE: u64 = 64 * 1024; +const CLASS_FLOOR_SHARE: u64 = 4; +const SOFT_LIMIT_PERCENT: u64 = 75; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MemoryClass { + SmtpInbound, + Read, +} + +#[derive(Debug, Default, Clone)] +#[repr(transparent)] +pub struct MemoryBudget(Arc<[ClassCounter; 2]>); + +#[derive(Debug, Default)] +#[repr(align(128))] +struct ClassCounter(AtomicU64); + +#[derive(Debug)] +pub struct MemoryReservation { + budget: MemoryBudget, + class: MemoryClass, + limit: u64, + reserved: u64, +} + +#[derive(Debug)] +pub struct ReservationScope<'x> { + reservation: &'x mut MemoryReservation, + start: u64, + committed: bool, +} + +impl MemoryClass { + fn index(self) -> usize { + match self { + MemoryClass::SmtpInbound => 0, + MemoryClass::Read => 1, + } + } + + fn other(self) -> Self { + match self { + MemoryClass::SmtpInbound => MemoryClass::Read, + MemoryClass::Read => MemoryClass::SmtpInbound, + } + } +} + +impl MemoryBudget { + pub fn reservation(&self, class: MemoryClass, limit: u64) -> MemoryReservation { + MemoryReservation { + budget: self.clone(), + class, + limit, + reserved: 0, + } + } + + pub fn is_accepting(&self, class: MemoryClass, limit: u64) -> bool { + self.used(class) + < class_capacity(limit, self.used(class.other())) / 100 * SOFT_LIMIT_PERCENT + } + + pub fn used(&self, class: MemoryClass) -> u64 { + self.counter(class).load(Ordering::SeqCst) + } + + fn counter(&self, class: MemoryClass) -> &AtomicU64 { + &self.0[class.index()].0 + } +} + +impl MemoryReservation { + pub fn reserve_to(&mut self, bytes: u64) -> bool { + if bytes <= self.reserved { + return true; + } + + let delta = (bytes - self.reserved).next_multiple_of(RESERVATION_GRANULE); + let counter = self.budget.counter(self.class); + let previous = counter.fetch_add(delta, Ordering::SeqCst); + let other = self.budget.used(self.class.other()); + if previous.saturating_add(delta) <= class_capacity(self.limit, other) + || previous == self.reserved + { + self.reserved += delta; + true + } else { + counter.fetch_sub(delta, Ordering::SeqCst); + false + } + } + + pub fn try_reserve_to(&mut self, bytes: u64) -> trc::Result<()> { + if self.reserve_to(bytes) { + Ok(()) + } else { + Err(self.budget_error(bytes)) + } + } + + pub fn try_reserve_within_limit(&mut self, bytes: u64) -> trc::Result<()> { + if bytes <= self.limit && self.reserve_to(bytes) { + Ok(()) + } else { + Err(self.budget_error(bytes)) + } + } + + #[cold] + fn budget_error(&self, bytes: u64) -> trc::Error { + trc::LimitEvent::MemoryBudget + .into_err() + .ctx(trc::Key::Size, bytes) + .ctx(trc::Key::Limit, self.limit) + } + + pub fn scope(&mut self) -> ReservationScope<'_> { + ReservationScope { + start: self.reserved, + reservation: self, + committed: false, + } + } + + pub fn shrink_to(&mut self, bytes: u64) { + let keep = bytes.next_multiple_of(RESERVATION_GRANULE); + if keep < self.reserved { + self.budget + .counter(self.class) + .fetch_sub(self.reserved - keep, Ordering::SeqCst); + self.reserved = keep; + } + } + + pub fn reserved(&self) -> u64 { + self.reserved + } + + pub fn limit(&self) -> u64 { + self.limit + } +} + +impl ReservationScope<'_> { + pub fn try_reserve_extra(&mut self, extra: u64) -> trc::Result<()> { + self.reservation + .try_reserve_within_limit(self.start.saturating_add(extra)) + } + + pub fn commit(mut self) { + self.committed = true; + } +} + +impl Drop for ReservationScope<'_> { + fn drop(&mut self) { + if !self.committed { + self.reservation.shrink_to(self.start); + } + } +} + +impl Drop for MemoryReservation { + fn drop(&mut self) { + if self.reserved > 0 { + self.budget + .counter(self.class) + .fetch_sub(self.reserved, Ordering::SeqCst); + } + } +} + +fn class_capacity(limit: u64, other_used: u64) -> u64 { + let floor = limit / CLASS_FLOOR_SHARE; + limit.saturating_sub(other_used.max(floor)).max(floor) +} + diff --git a/crates/common/src/network/mod.rs b/crates/common/src/network/mod.rs index 6e7727c2..6a728608 100644 --- a/crates/common/src/network/mod.rs +++ b/crates/common/src/network/mod.rs @@ -5,22 +5,19 @@ */ use self::limiter::{ConcurrencyLimiter, InFlight}; -use crate::{ - Server, - config::server::ServerProtocol, - expr::{functions::ResolveVariable, *}, -}; -use compact_str::ToCompactString; -use registry::{schema::enums::ExpressionVariable, types::ipmask::IpAddrOrMask}; +use crate::{Server, config::server::ServerProtocol}; +use compact_str::{CompactString, ToCompactString}; +use registry::types::ipmask::IpAddrOrMask; use rustls::ServerConfig; use std::fmt::Debug; use std::{ borrow::Cow, io, - net::{IpAddr, Ipv4Addr, Ipv6Addr}, + net::IpAddr, sync::Arc, time::{Duration, Instant}, }; +use tinyvec::ArrayVec; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::watch, @@ -35,8 +32,10 @@ pub mod asn; pub mod autoconfig; pub mod dkim; pub mod dns; +pub mod forwarded; pub mod limiter; pub mod listen; +pub mod memory; pub mod mta; pub mod security; pub mod stream; @@ -150,7 +149,11 @@ pub trait SessionManager: Sync + Send + 'static + Clone { Event::with_keys( span_start, vec![ - (Key::ListenerId, session.instance.id.clone().into()), + ( + Key::ListenerId, + CompactString::from(session.instance.id.as_str()) + .into(), + ), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), @@ -180,7 +183,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { LocalPort = local_port, RemoteIp = session.remote_ip, RemotePort = session.remote_port, - Reason = err.to_string(), + Reason = err.to_compact_string(), ); return; @@ -196,7 +199,10 @@ pub trait SessionManager: Sync + Send + 'static + Clone { Event::with_keys( span_start, vec![ - (Key::ListenerId, session.instance.id.clone().into()), + ( + Key::ListenerId, + CompactString::from(session.instance.id.as_str()).into(), + ), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), @@ -216,7 +222,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { LocalPort = local_port, RemoteIp = session.remote_ip, RemotePort = session.remote_port, - Reason = io::Error::from(io::ErrorKind::TimedOut).to_string(), + Reason = io::Error::from(io::ErrorKind::TimedOut).to_compact_string(), ); return; @@ -231,7 +237,10 @@ pub trait SessionManager: Sync + Send + 'static + Clone { Event::with_keys( span_start, vec![ - (Key::ListenerId, session.instance.id.clone().into()), + ( + Key::ListenerId, + CompactString::from(session.instance.id.as_str()).into(), + ), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), @@ -263,25 +272,6 @@ pub trait SessionManager: Sync + Send + 'static + Clone { fn shutdown(&self) -> impl std::future::Future + Send; } -impl ResolveVariable for SessionData { - fn resolve_variable(&self, variable: ExpressionVariable) -> crate::expr::Variable<'_> { - match variable { - ExpressionVariable::RemoteIp => self.remote_ip.to_compact_string().into(), - ExpressionVariable::RemotePort => self.remote_port.into(), - ExpressionVariable::LocalIp => self.local_ip.to_compact_string().into(), - ExpressionVariable::LocalPort => self.local_port.into(), - ExpressionVariable::Listener => self.instance.id.as_str().into(), - ExpressionVariable::Protocol => self.protocol.as_str().into(), - ExpressionVariable::IsTls => self.stream.is_tls().into(), - _ => crate::expr::Variable::default(), - } - } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } -} - impl Debug for TcpAcceptor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -297,87 +287,13 @@ impl Debug for TcpAcceptor { } } -pub fn is_global_ip(ip: &IpAddr) -> bool { +pub fn ip_to_bytes(ip: &IpAddr) -> ArrayVec<[u8; 16]> { + let mut bytes = ArrayVec::new(); match ip { - IpAddr::V4(ip) => is_global_ipv4(ip), - IpAddr::V6(ip) => is_global_ipv6(ip), - } -} - -fn is_global_ipv4(ip: &Ipv4Addr) -> bool { - let [a, b, c, _] = ip.octets(); - let is_this_network = a == 0; - let is_shared = a == 100 && (64..128).contains(&b); - let is_protocol_assignment = a == 192 && b == 0 && c == 0; - let is_benchmarking = a == 198 && (b & 0xfe) == 18; - let is_relay_6to4 = a == 192 && b == 88 && c == 99; - let is_reserved = a >= 240; - - !(ip.is_unspecified() - || ip.is_loopback() - || ip.is_private() - || ip.is_link_local() - || ip.is_multicast() - || ip.is_broadcast() - || ip.is_documentation() - || is_this_network - || is_shared - || is_protocol_assignment - || is_benchmarking - || is_relay_6to4 - || is_reserved) -} - -fn is_global_ipv6(ip: &Ipv6Addr) -> bool { - if ip.is_unspecified() || ip.is_loopback() || ip.is_multicast() { - return false; - } - - if let Some(ip) = ip.to_ipv4() { - return is_global_ipv4(&ip); - } - - let segments = ip.segments(); - - if segments[0] == 0x2002 { - return is_global_ipv4(&Ipv4Addr::from( - ((segments[1] as u32) << 16) | segments[2] as u32, - )); - } - - if segments[0] == 0x0064 && segments[1] == 0xff9b { - let is_well_known_prefix = - segments[2] == 0 && segments[3] == 0 && segments[4] == 0 && segments[5] == 0; - - return is_well_known_prefix - && is_global_ipv4(&Ipv4Addr::from( - ((segments[6] as u32) << 16) | segments[7] as u32, - )); - } - - let is_unique_local = (segments[0] & 0xfe00) == 0xfc00; - let is_link_local = (segments[0] & 0xffc0) == 0xfe80; - let is_site_local = (segments[0] & 0xffc0) == 0xfec0; - let is_discard_only = - segments[0] == 0x0100 && segments[1] == 0 && segments[2] == 0 && segments[3] == 0; - let is_documentation = segments[0] == 0x2001 && segments[1] == 0x0db8; - let is_teredo = segments[0] == 0x2001 && segments[1] == 0; - let is_orchid = segments[0] == 0x2001 && (segments[1] & 0xfff0) == 0x0020; - - !(is_unique_local - || is_link_local - || is_site_local - || is_discard_only - || is_documentation - || is_teredo - || is_orchid) -} - -pub fn ip_to_bytes(ip: &IpAddr) -> Vec { - match ip { - IpAddr::V4(ip) => ip.octets().to_vec(), - IpAddr::V6(ip) => ip.octets().to_vec(), + IpAddr::V4(ip) => bytes.extend_from_slice(&ip.octets()), + IpAddr::V6(ip) => bytes.extend_from_slice(&ip.octets()), } + bytes } pub fn ip_to_bytes_prefix(prefix: u8, ip: &IpAddr) -> Vec { @@ -396,81 +312,3 @@ pub fn ip_to_bytes_prefix(prefix: u8, ip: &IpAddr) -> Vec { } } } - -#[cfg(test)] -mod tests { - use super::is_global_ip; - use std::net::IpAddr; - - #[test] - fn global_ip_classification() { - for ip in [ - "8.8.8.8", - "1.1.1.1", - "93.184.216.34", - "172.32.0.1", - "100.63.255.255", - "100.128.0.1", - "198.20.0.1", - "192.0.3.1", - "2606:4700::1111", - "2a00:1450:4001::200e", - "::ffff:8.8.8.8", - "64:ff9b::808:808", - "2002:0808:0808::", - "2001:db9::1", - ] { - assert!( - is_global_ip(&ip.parse::().unwrap()), - "expected {ip} to be global" - ); - } - - for ip in [ - "127.0.0.1", - "127.1.2.3", - "10.0.0.1", - "172.16.0.1", - "172.31.255.255", - "192.168.1.1", - "169.254.169.254", - "100.64.0.1", - "100.127.255.255", - "198.18.0.1", - "198.19.255.255", - "192.0.0.1", - "192.0.2.5", - "192.88.99.1", - "240.0.0.1", - "255.255.255.255", - "224.0.0.1", - "0.0.0.0", - "0.1.2.3", - "::1", - "::", - "fc00::1", - "fd12:3456::1", - "fe80::1", - "febf::1", - "fec0::1", - "2001:db8::1", - "ff02::1", - "::ffff:127.0.0.1", - "::ffff:10.0.0.1", - "::127.0.0.1", - "64:ff9b::7f00:1", - "64:ff9b::a00:1", - "64:ff9b:1::7f00:1", - "2002:7f00:1::", - "2002:c0a8:101::", - "100::1", - "2001::1", - "2001:20::1", - ] { - assert!( - !is_global_ip(&ip.parse::().unwrap()), - "expected {ip} to be rejected" - ); - } - } -} diff --git a/crates/common/src/network/mta.rs b/crates/common/src/network/mta.rs index 5fcc998e..90296496 100644 --- a/crates/common/src/network/mta.rs +++ b/crates/common/src/network/mta.rs @@ -17,14 +17,15 @@ use crate::{ }, }, }, - expr::{Variable, functions::ResolveVariable}, + expr::{Bump, Variable, functions::ResolveVariable}, manager::SPAM_CLASSIFIER_KEY, network::RcptResolution, }; use ahash::AHashSet; +use compact_str::{CompactString, ToCompactString}; use directory::Recipient; use mail_auth::IpLookupStrategy; -use registry::schema::{enums::ExpressionVariable, structs::MaskedEmail}; +use registry::schema::enums::ExpressionVariable; use sieve::Sieve; use std::{ borrow::Cow, @@ -33,12 +34,12 @@ use std::{ }; use store::{ Deserialize, IterateParams, ValueKey, - write::{AlignedBytes, Archive, QueueClass, ValueClass, now}, + write::{Archive, ArchiveBytes, QueueClass, ValueClass}, }; use trc::{AddContext, SpamEvent}; -use types::id::Id; use utils::DomainPart; + impl Server { pub async fn rcpt_resolve( &self, @@ -60,10 +61,12 @@ impl Server { if domain.flags & DOMAIN_FLAG_SUB_ADDRESSING != 0 { if let Some(sub_addressing) = &domain.sub_addressing_custom { // Custom sub-addressing resolution + let mut arena = Bump::new(); if let Some(result) = self .eval_if::( sub_addressing, &AddressResolver(local_part.as_ref()), + &mut arena, session_id, ) .await @@ -87,22 +90,34 @@ impl Server { } else { Cow::Borrowed(rcpt) }; + let accept = |address: Cow<'_, str>| { + if is_subaddressed { + RcptResolution::Rewrite(address.into_owned()) + } else { + RcptResolution::Accept + } + }; + + if self + .inner + .cache + .directory_recipients + .get(address.as_ref()) + .is_some() + { + return Ok(accept(address)); + } + match directory.recipient(address.as_ref()).await? { Recipient::Account(account) => { - Box::pin(self.synchronize_account(account)).await?; - return Ok(if is_subaddressed { - RcptResolution::Rewrite(address.into_owned()) - } else { - RcptResolution::Accept - }); + self.synchronize_account(account).await?; + self.cache_directory_recipient(address.as_ref()); + return Ok(accept(address)); } Recipient::Group(group) => { - Box::pin(self.synchronize_group(group)).await?; - return Ok(if is_subaddressed { - RcptResolution::Rewrite(address.into_owned()) - } else { - RcptResolution::Accept - }); + self.synchronize_group(group).await?; + self.cache_directory_recipient(address.as_ref()); + return Ok(accept(address)); } Recipient::Invalid => {} } @@ -172,28 +187,42 @@ impl Server { list_id: u32, recipients: Arc<[Box]>, ) -> trc::Result]>> { - let mut has_nested = false; - for member in recipients.iter() { - if let Some(EmailCache::MailingList(_)) = self.rcpt_id_from_email(member).await? { - has_nested = true; + let mut first_nested = None; + for (index, member) in recipients.iter().enumerate() { + if let Some(EmailCache::MailingList(nested_id)) = + self.rcpt_id_from_email(member).await? + { + first_nested = Some((index, nested_id)); break; } } - if !has_nested { + let Some((mut start, nested_id)) = first_nested else { return Ok(recipients); - } + }; let mut expanded = Vec::with_capacity(recipients.len()); let mut seen: AHashSet> = AHashSet::with_capacity(recipients.len()); let mut visited = AHashSet::from_iter([list_id]); let mut pending: Vec]>> = Vec::new(); - let mut members = recipients; + let mut resolved = Some(nested_id); + for member in recipients.get(..start).unwrap_or_default() { + if seen.insert(member.to_canonical_address().into()) { + expanded.push(member.clone()); + } + } + + let mut members = recipients; loop { - for member in members.iter() { - if let Some(EmailCache::MailingList(nested_id)) = - self.rcpt_id_from_email(member).await? - { + for member in members.get(start..).unwrap_or_default() { + let nested_id = match resolved.take() { + Some(nested_id) => Some(nested_id), + None => match self.rcpt_id_from_email(member).await? { + Some(EmailCache::MailingList(nested_id)) => Some(nested_id), + _ => None, + }, + }; + if let Some(nested_id) = nested_id { if !visited.insert(nested_id) { continue; } @@ -212,6 +241,7 @@ impl Server { break; }; members = next; + start = 0; } Ok(expanded.into()) @@ -227,7 +257,7 @@ impl Server { } else { trc::event!( Dkim(trc::DkimEvent::SignerNotFound), - Id = domain.to_string(), + Id = CompactString::from(domain), SpanId = session_id, ); @@ -235,11 +265,11 @@ impl Server { } } - pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc> { + pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Sieve<'static>> { self.core.sieve.trusted_script(name).or_else(|| { trc::event!( Sieve(trc::SieveEvent::ScriptNotFound), - Id = name.to_string(), + Script = CompactString::from(name), SpanId = session_id, ); @@ -247,11 +277,15 @@ impl Server { }) } - pub fn get_untrusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc> { + pub fn get_untrusted_sieve_script( + &self, + name: &str, + session_id: u64, + ) -> Option<&Sieve<'static>> { self.core.sieve.untrusted_script(name).or_else(|| { trc::event!( Sieve(trc::SieveEvent::ScriptNotFound), - Id = name.to_string(), + Script = CompactString::from(name), SpanId = session_id, ); @@ -277,7 +311,7 @@ impl Server { _ => { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), - Id = name.to_string(), + Name = CompactString::from(name), Details = "Gateway not found", SpanId = session_id, ); @@ -297,7 +331,7 @@ impl Server { if name != &DEFAULT_QUEUE_NAME { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), - Id = name.to_string(), + Name = name.to_compact_string(), Details = "Virtual queue not found", ); } @@ -333,7 +367,7 @@ impl Server { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), - Id = name.to_string(), + Name = CompactString::from(name), Details = "Queue strategy not found", SpanId = session_id, ); @@ -361,7 +395,7 @@ impl Server { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), - Id = name.to_string(), + Name = CompactString::from(name), Details = "TLS strategy not found", SpanId = session_id, ); @@ -393,7 +427,7 @@ impl Server { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), - Id = name.to_string(), + Name = CompactString::from(name), Details = "Connection strategy not found", SpanId = session_id, ); @@ -410,7 +444,7 @@ impl Server { .get_blob(SPAM_CLASSIFIER_KEY, 0..usize::MAX) .await .and_then(|archive| match archive { - Some(archive) => as Deserialize>::deserialize(&archive) + Some(archive) => as Deserialize>::deserialize(&archive) .and_then(|archive| archive.deserialize_untrusted::()) .map(Some), None => Ok(None), @@ -459,16 +493,20 @@ impl Server { .caused_by(trc::location!()) .map(|_| total) } + + fn cache_directory_recipient(&self, address: &str) { + self.inner.cache.directory_recipients.insert( + address.into(), + (), + self.inner.cache.directory_recipients_ttl, + ); + } } pub struct AddressResolver<'x>(pub &'x str); impl ResolveVariable for AddressResolver<'_> { - fn resolve_variable(&'_ self, _: ExpressionVariable) -> crate::expr::Variable<'_> { + fn resolve_variable<'a>(&'a self, _: ExpressionVariable, _: &'a Bump) -> Variable<'a> { Variable::from(self.0) } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } diff --git a/crates/common/src/network/security.rs b/crates/common/src/network/security.rs index 7ca03064..040f95b7 100644 --- a/crates/common/src/network/security.rs +++ b/crates/common/src/network/security.rs @@ -13,13 +13,14 @@ use ahash::AHashSet; use registry::{ schema::{ enums::{BlockReason, PasswordHashAlgorithm, PasswordStrength}, - prelude::{Object, ObjectType}, + prelude::{Object, ObjectInner, ObjectType}, structs::{self, AllowedIp, BlockedIp, Rate, SystemSettings}, }, types::{datetime::UTCDateTime, ipmask::IpAddrOrMask}, }; use std::{fmt::Debug, hash::Hash, net::IpAddr}; use store::{ + blake3, registry::{ bootstrap::Bootstrap, write::{RegistryWrite, RegistryWriteResult}, @@ -50,7 +51,6 @@ pub struct Security { pub default_role_ids_user: Vec, pub default_role_ids_group: Vec, - pub default_role_ids_tenant: Vec, pub default_role_ids_admin: Vec, pub password_hash_algorithm: PasswordHashAlgorithm, @@ -141,14 +141,14 @@ impl Security { trc::event!( Security(trc::SecurityEvent::IpAllowExpired), - Details = expired_allows + Ip = expired_allows .into_iter() .map(|(_, ip, _)| trc::Value::from(ip.into_inner().0)) .collect::>() ); } - #[cfg(not(feature = "test_mode"))] + #[cfg(not(stalwart_tests))] { // Add loopback addresses allowed_ip_addresses.insert(IpWithTtl::new( @@ -187,12 +187,11 @@ impl Security { scanner_fail_rate: security.scan_ban_rate.filter(|_| !is_recovery_mode), default_role_ids_user: auth.default_user_role_ids.into_inner(), default_role_ids_group: auth.default_group_role_ids.into_inner(), - default_role_ids_tenant: auth.default_tenant_role_ids.into_inner(), default_role_ids_admin: auth.default_admin_role_ids.into_inner(), password_hash_algorithm: auth.password_hash_algorithm, - password_max_length: auth.password_max_length as u32, - password_min_length: auth.password_min_length as u32, - password_min_strength: match auth.password_min_strength { + password_max_length: auth.max_password_length as u32, + password_min_length: auth.min_password_length as u32, + password_min_strength: match auth.min_password_strength { PasswordStrength::Zero => Score::Zero, PasswordStrength::One => Score::One, PasswordStrength::Two => Score::Two, @@ -216,24 +215,31 @@ impl Security { impl Server { pub async fn is_rcpt_fail2banned(&self, ip: IpAddr, rcpt: &str) -> trc::Result { - if let Some(rate) = &self.core.network.security.rcpt_fail_rate { - let is_allowed = self.is_ip_allowed(ip) - || (self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_RCPT, &ip_to_bytes(&ip), rate, false) + if let Some(rate) = &self.core.network.security.rcpt_fail_rate + && !self.is_ip_allowed(ip) + { + let store = self.in_memory_store(); + let ip_key = ip_to_bytes(&ip); + let rcpt_key = blake3::hash(rcpt.as_bytes()); + let is_allowed = store + .is_rate_allowed(KV_RATE_LIMIT_RCPT, &ip_key, rate) + .await? + .is_none() + && store + .is_rate_allowed(KV_RATE_LIMIT_RCPT, rcpt_key.as_bytes(), rate) .await? - .is_none() - && self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_RCPT, rcpt.as_bytes(), rate, false) - .await? - .is_none()); + .is_none(); if !is_allowed { return self - .block_ip(ip, BlockReason::RcptToFailure) - .await - .map(|_| true); + .ban_and_reset( + ip, + BlockReason::RcptToFailure, + KV_RATE_LIMIT_RCPT, + rate, + &[ip_key.as_slice(), rcpt_key.as_bytes()], + ) + .await; } } @@ -242,22 +248,11 @@ impl Server { pub async fn is_scanner_fail2banned(&self, ip: IpAddr) -> trc::Result { if let Some(rate) = &self.core.network.security.scanner_fail_rate { - let is_allowed = self.is_ip_allowed(ip) - || self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_SCAN, &ip_to_bytes(&ip), rate, false) - .await? - .is_none(); - - if !is_allowed { - return self - .block_ip(ip, BlockReason::PortScanning) - .await - .map(|_| true); - } + self.is_ip_fail2banned(ip, KV_RATE_LIMIT_SCAN, rate, BlockReason::PortScanning) + .await + } else { + Ok(false) } - - Ok(false) } pub async fn is_http_banned_path(&self, path: &str, ip: IpAddr) -> trc::Result { @@ -274,44 +269,87 @@ impl Server { pub async fn is_loiter_fail2banned(&self, ip: IpAddr) -> trc::Result { if let Some(rate) = &self.core.network.security.loiter_fail_rate { - let is_allowed = self.is_ip_allowed(ip) - || self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_LOITER, &ip_to_bytes(&ip), rate, false) - .await? - .is_none(); + self.is_ip_fail2banned(ip, KV_RATE_LIMIT_LOITER, rate, BlockReason::Loitering) + .await + } else { + Ok(false) + } + } - if !is_allowed { - return self - .block_ip(ip, BlockReason::Loitering) - .await - .map(|_| true); - } + async fn is_ip_fail2banned( + &self, + ip: IpAddr, + prefix: u8, + rate: &Rate, + reason: BlockReason, + ) -> trc::Result { + if self.is_ip_allowed(ip) { + return Ok(false); } - Ok(false) + let ip_key = ip_to_bytes(&ip); + if self + .in_memory_store() + .is_rate_allowed(prefix, &ip_key, rate) + .await? + .is_none() + { + Ok(false) + } else { + self.ban_and_reset(ip, reason, prefix, rate, &[ip_key.as_slice()]) + .await + } + } + + async fn ban_and_reset( + &self, + ip: IpAddr, + reason: BlockReason, + prefix: u8, + rate: &Rate, + keys: &[&[u8]], + ) -> trc::Result { + self.block_ip(ip, reason).await?; + for key in keys { + if let Err(err) = self.in_memory_store().rate_reset(prefix, key, rate).await { + trc::error!( + err.details("Failed to reset fail2ban counter") + .caused_by(trc::location!()) + ); + } + } + Ok(true) } pub async fn is_auth_fail2banned(&self, ip: IpAddr, login: Option<&str>) -> trc::Result { - if let Some(rate) = &self.core.network.security.auth_fail_rate { - let login = login.unwrap_or_default(); - let is_allowed = self.is_ip_allowed(ip) - || (self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_AUTH, &ip_to_bytes(&ip), rate, false) - .await? - .is_none() - && (login.is_empty() - || self - .in_memory_store() - .is_rate_allowed(KV_RATE_LIMIT_AUTH, login.as_bytes(), rate, false) - .await? - .is_none())); + if let Some(rate) = &self.core.network.security.auth_fail_rate + && !self.is_ip_allowed(ip) + { + let store = self.in_memory_store(); + let ip_key = ip_to_bytes(&ip); + let login_key = login + .filter(|login| !login.is_empty()) + .map(|login| blake3::hash(login.as_bytes())); + let is_allowed = store + .is_rate_allowed(KV_RATE_LIMIT_AUTH, &ip_key, rate) + .await? + .is_none() + && match &login_key { + Some(login_key) => store + .is_rate_allowed(KV_RATE_LIMIT_AUTH, login_key.as_bytes(), rate) + .await? + .is_none(), + None => true, + }; + if !is_allowed { + let keys: &[&[u8]] = match &login_key { + Some(login_key) => &[ip_key.as_slice(), login_key.as_bytes()], + None => &[ip_key.as_slice()], + }; return self - .block_ip(ip, BlockReason::AuthFailure) - .await - .map(|_| true); + .ban_and_reset(ip, BlockReason::AuthFailure, KV_RATE_LIMIT_AUTH, rate, keys) + .await; } } @@ -327,29 +365,52 @@ impl Server { .security .ban_period(reason) .map(|v| now + v); - self.inner - .data - .blocked_ips - .write() - .blocked_ip_addresses - .insert(IpWithTtl::new(ip, expires_at.unwrap_or(u64::MAX))); + let expiry = expires_at.unwrap_or(u64::MAX); + self.inner.data.blocked_ips.write().block(ip, expiry); // Write blocked IP to config - let RegistryWriteResult::Success(id) = self - .registry() - .write(RegistryWrite::insert( - &BlockedIp { - address: IpAddrOrMask::from_ip(ip), - created_at: UTCDateTime::from_timestamp(now as i64), - expires_at: expires_at.map(|ts| UTCDateTime::from_timestamp(ts as i64)), - reason, - } - .into(), - )) + let object = Object::from(BlockedIp { + address: IpAddrOrMask::from_ip(ip), + created_at: UTCDateTime::from_timestamp(now as i64), + expires_at: expires_at.map(|ts| UTCDateTime::from_timestamp(ts as i64)), + reason, + }); + let registry = self.registry(); + let id = match registry + .write(RegistryWrite::insert(&object)) .await .caused_by(trc::location!())? - else { - return Ok(()); + { + RegistryWriteResult::Success(id) => id, + RegistryWriteResult::PrimaryKeyConflict { existing_id, .. } => { + let Some(current) = registry + .get(existing_id) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + let ObjectInner::BlockedIp(current_ip) = ¤t.inner else { + return Ok(()); + }; + if current_ip + .expires_at + .as_ref() + .map_or(u64::MAX, |dt| dt.timestamp() as u64) + >= expiry + { + return Ok(()); + } + let RegistryWriteResult::Success(id) = registry + .write(RegistryWrite::update(existing_id.id(), &object, ¤t)) + .await + .caused_by(trc::location!())? + else { + return Ok(()); + }; + id + } + _ => return Ok(()), }; // Increment version @@ -423,6 +484,17 @@ impl Server { } impl BlockedIps { + pub fn block(&mut self, ip: IpAddr, expires_at: u64) { + let entry = IpWithTtl::new(ip, expires_at); + if self + .blocked_ip_addresses + .get(&entry) + .is_none_or(|current| current.expires_at < expires_at) + { + self.blocked_ip_addresses.replace(entry); + } + } + pub async fn parse(bp: &mut Bootstrap) -> Self { let mut ips = Self::default(); @@ -445,8 +517,7 @@ impl BlockedIps { if ip.expires_at.as_ref().is_none_or(|ip| ip.timestamp() > now) { if let Some(ip) = ip.address.try_to_ip() { - ips.blocked_ip_addresses - .insert(IpWithTtl::new(ip, expires_at)); + ips.block(ip, expires_at); } else { ips.blocked_ip_networks .push(IpWithTtl::new(ip.address, expires_at)); @@ -478,7 +549,7 @@ impl BlockedIps { } trc::event!( Security(trc::SecurityEvent::IpBlockExpired), - Details = expired_blocks + Ip = expired_blocks .into_iter() .map(|(_, ip, _)| trc::Value::from(ip.into_inner().0)) .collect::>() @@ -513,3 +584,4 @@ impl IpWithTtl { self.expires_at <= now() } } + diff --git a/crates/common/src/network/tls.rs b/crates/common/src/network/tls.rs index 3e55a334..01667617 100644 --- a/crates/common/src/network/tls.rs +++ b/crates/common/src/network/tls.rs @@ -9,6 +9,8 @@ use super::{ acme::resolver::{IsTlsAlpnChallenge, build_acme_static_resolver}, }; use crate::{Inner, Server}; +use ahash::AHashMap; +use compact_str::{CompactString, ToCompactString}; use rustls::{ SupportedProtocolVersion, server::{ClientHello, ResolvesServerCert}, @@ -26,6 +28,29 @@ use tokio_rustls::{Accept, LazyConfigAcceptor}; pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; +const MAX_HOSTNAME_LEN: usize = 253; +const WILDCARD_PREFIX: &[u8] = b"*."; + +pub(crate) fn lookup_certificate<'x>( + certs: &'x AHashMap, Arc>, + name: &str, +) -> Option<&'x Arc> { + certs.get(name).or_else(|| lookup_wildcard(certs, name)) +} + +fn lookup_wildcard<'x>( + certs: &'x AHashMap, Arc>, + name: &str, +) -> Option<&'x Arc> { + let (_, parent) = name.split_once('.')?; + let mut buf = [0u8; MAX_HOSTNAME_LEN]; + let key = buf.get_mut(..WILDCARD_PREFIX.len() + parent.len())?; + let (prefix, rest) = key.split_at_mut(WILDCARD_PREFIX.len()); + prefix.copy_from_slice(WILDCARD_PREFIX); + rest.copy_from_slice(parent.as_bytes()); + certs.get(str::from_utf8(key).ok()?) +} + #[derive(Clone)] pub struct CertificateResolver { pub inner: Arc, @@ -45,16 +70,7 @@ impl ResolvesServerCert for CertificateResolver { impl Server { pub fn resolve_certificate(&self, name: &str) -> Option> { - let certs = self.inner.data.tls_certificates.load(); - - certs - .get(name) - .or_else(|| { - // Try with a wildcard certificate - name.split_once('.') - .and_then(|(_, domain)| certs.get(domain)) - }) - .cloned() + lookup_certificate(&self.inner.data.tls_certificates.load(), name).cloned() } } @@ -65,20 +81,13 @@ impl CertificateResolver { name.map_or_else( || certs.get("*"), |name| { - certs - .get(name) - .or_else(|| { - // Try with a wildcard certificate - name.split_once('.') - .and_then(|(_, domain)| certs.get(domain)) - }) - .or_else(|| { - trc::event!( - Tls(trc::TlsEvent::CertificateNotFound), - Hostname = name.to_string(), - ); - certs.get("*") - }) + lookup_certificate(&certs, name).or_else(|| { + trc::event!( + Tls(trc::TlsEvent::CertificateNotFound), + Hostname = CompactString::from(name), + ); + certs.get("*") + }) }, ) .or_else(|| match certs.len().cmp(&1) { @@ -132,7 +141,7 @@ impl TcpAcceptor { trc::event!( Acme(trc::AcmeEvent::ClientSuppliedSni), ListenerId = instance.id.clone(), - Domain = domain.to_string(), + Domain = CompactString::from(domain), Result = key.is_some(), ); @@ -164,7 +173,7 @@ impl TcpAcceptor { trc::event!( Acme(trc::AcmeEvent::TlsAlpnError), ListenerId = instance.id.clone(), - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -178,7 +187,7 @@ impl TcpAcceptor { trc::event!( Tls(trc::TlsEvent::HandshakeError), ListenerId = instance.id.clone(), - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -212,3 +221,4 @@ impl std::fmt::Debug for CertificateResolver { f.debug_struct("CertificateResolver").finish() } } + diff --git a/crates/common/src/network/webpush.rs b/crates/common/src/network/webpush.rs index 60df2045..1411ca52 100644 --- a/crates/common/src/network/webpush.rs +++ b/crates/common/src/network/webpush.rs @@ -5,17 +5,19 @@ */ use ahash::AHashMap; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use encodify::base64::URL_SAFE_NO_PAD; use p256::{ SecretKey, ecdsa::{Signature, SigningKey, signature::Signer}, - pkcs8::{DecodePrivateKey, PrivateKeyInfo, der::SecretDocument}, + pkcs8::{DecodePrivateKey, PrivateKeyInfoRef, der::SecretDocument}, }; use parking_lot::Mutex; use reqwest::{Url, header::HeaderValue}; use std::sync::Arc; const VAPID_TOKEN_TTL: u64 = 12 * 60 * 60; +const VAPID_PREFIX: &str = "vapid t="; +const VAPID_JWT_HEADER: &[u8] = br#"{"typ":"JWT","alg":"ES256"}"#; const VAPID_TOKEN_REFRESH: u64 = VAPID_TOKEN_TTL / 2; #[derive(Clone)] @@ -44,18 +46,18 @@ impl Vapid { } pub fn authorization(&self, endpoint: &str, now: u64) -> Option { - let prefix = endpoint_prefix(endpoint)?; + let origin = endpoint_origin(endpoint)?; if let Some(token) = self .tokens .lock() - .get(prefix) + .get(&origin) .filter(|token| token.is_fresh(now)) { return Some(token.authorization.clone()); } - let authorization = HeaderValue::try_from(self.key.authorization( - endpoint, + let authorization = HeaderValue::try_from(self.key.authorization_for_origin( + &origin, self.contact.as_deref(), now, )?) @@ -63,7 +65,7 @@ impl Vapid { let mut tokens = self.tokens.lock(); tokens.retain(|_, token| token.is_fresh(now)); tokens.insert( - prefix.to_string(), + origin, VapidToken { authorization: authorization.clone(), issued_at: now, @@ -112,12 +114,8 @@ impl VapidKey { } fn from_signing_key(signing_key: SigningKey) -> Self { - let public_key = URL_SAFE_NO_PAD.encode( - signing_key - .verifying_key() - .to_encoded_point(false) - .as_bytes(), - ); + let public_key = + URL_SAFE_NO_PAD.encode(signing_key.verifying_key().to_sec1_point(false).as_bytes()); Self { signing_key, public_key, @@ -129,32 +127,38 @@ impl VapidKey { } pub fn authorization(&self, endpoint: &str, contact: Option<&str>, now: u64) -> Option { + self.authorization_for_origin(&endpoint_origin(endpoint)?, contact, now) + } + + fn authorization_for_origin( + &self, + origin: &str, + contact: Option<&str>, + now: u64, + ) -> Option { let mut claims = serde_json::Map::new(); - claims.insert("aud".into(), endpoint_origin(endpoint)?.into()); + claims.insert("aud".into(), origin.into()); claims.insert("exp".into(), (now + VAPID_TOKEN_TTL).into()); if let Some(sub) = contact { claims.insert("sub".into(), sub.into()); } - let header = URL_SAFE_NO_PAD.encode(br#"{"typ":"JWT","alg":"ES256"}"#); - let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).ok()?); - let signing_input = format!("{header}.{payload}"); - let signature: Signature = self.signing_key.sign(signing_input.as_bytes()); + let mut authorization = String::from(VAPID_PREFIX); + URL_SAFE_NO_PAD.encode_append(VAPID_JWT_HEADER, &mut authorization); + authorization.push('.'); + URL_SAFE_NO_PAD.encode_append(serde_json::to_vec(&claims).ok()?, &mut authorization); + let signature: Signature = self + .signing_key + .sign(authorization.get(VAPID_PREFIX.len()..)?.as_bytes()); + authorization.push('.'); + URL_SAFE_NO_PAD.encode_append(signature.to_bytes(), &mut authorization); + authorization.push_str(", k="); + authorization.push_str(&self.public_key); - Some(format!( - "vapid t={signing_input}.{}, k={}", - URL_SAFE_NO_PAD.encode(signature.to_bytes()), - self.public_key - )) + Some(authorization) } } -fn endpoint_prefix(url: &str) -> Option<&str> { - let (scheme, rest) = url.split_once("://")?; - let authority = rest.split(['/', '?', '#']).next()?; - url.get(..scheme.len() + "://".len() + authority.len()) -} - fn endpoint_origin(url: &str) -> Option { let origin = Url::parse(url).ok()?.origin(); origin.is_tuple().then(|| origin.ascii_serialization()) @@ -173,10 +177,11 @@ pub fn normalize_contact(contact: &str) -> Option { } pub fn generate_pkcs8_pem() -> Result { - use p256::elliptic_curve::rand_core::OsRng; + use p256::elliptic_curve::Generate; use p256::pkcs8::{EncodePrivateKey, LineEnding}; - SigningKey::random(&mut OsRng) + SigningKey::try_generate() + .map_err(|err| err.to_string())? .to_pkcs8_pem(LineEnding::LF) .map(|pem| pem.to_string()) .map_err(|err| err.to_string()) @@ -184,232 +189,7 @@ pub fn generate_pkcs8_pem() -> Result { fn secret_key_from_explicit_params(pem: &str) -> Option { let (_, document) = SecretDocument::from_pem(pem).ok()?; - let private_key_info = PrivateKeyInfo::try_from(document.as_bytes()).ok()?; - SecretKey::from_sec1_der(private_key_info.private_key).ok() + let private_key_info = PrivateKeyInfoRef::try_from(document.as_bytes()).ok()?; + SecretKey::from_sec1_der(private_key_info.private_key.as_bytes()).ok() } -#[cfg(test)] -mod tests { - use super::*; - use p256::ecdsa::{Signature, VerifyingKey, signature::Verifier}; - - fn test_key() -> VapidKey { - VapidKey::from_pkcs8_pem(&generate_pkcs8_pem().unwrap()).unwrap() - } - - #[test] - fn generated_key_round_trips_through_pkcs8_pem() { - let pem = generate_pkcs8_pem().unwrap(); - assert_eq!( - VapidKey::from_pkcs8_pem(&pem).unwrap().public_key(), - VapidKey::from_pkcs8_pem(&pem).unwrap().public_key() - ); - } - - #[test] - fn endpoint_origin_normalizes() { - assert_eq!( - endpoint_origin("HTTPS://Push.Example.COM:443/push?x=1").unwrap(), - "https://push.example.com" - ); - assert_eq!( - endpoint_origin("https://127.0.0.1:19000/push").unwrap(), - "https://127.0.0.1:19000" - ); - assert_eq!( - endpoint_origin("https://user:pass@fcm.googleapis.com/fcm/send/x").unwrap(), - "https://fcm.googleapis.com" - ); - assert_eq!( - endpoint_origin("http://[2001:DB8::1]:80/p").unwrap(), - "http://[2001:db8::1]" - ); - assert_eq!( - endpoint_origin("https://attacker.example\\@fcm.googleapis.com/fcm/send/x").unwrap(), - "https://attacker.example" - ); - assert!(endpoint_origin("not-a-url").is_none()); - assert!(endpoint_origin("mailto:admin@example.org").is_none()); - } - - #[test] - fn authorization_signs_a_verifiable_es256_token() { - let key = test_key(); - let now = 1_700_000_000; - let header = key - .authorization( - "https://push.example.com/push/abc?token=1", - Some("mailto:admin@example.org"), - now, - ) - .unwrap(); - - let (token, advertised_key) = header - .strip_prefix("vapid ") - .and_then(|rest| rest.split_once(", ")) - .unwrap(); - let jwt = token.strip_prefix("t=").unwrap(); - assert_eq!(advertised_key.strip_prefix("k=").unwrap(), key.public_key()); - - let parts = jwt.split('.').collect::>(); - assert_eq!(parts.len(), 3); - - let verifying_key = - VerifyingKey::from_sec1_bytes(&URL_SAFE_NO_PAD.decode(key.public_key()).unwrap()) - .unwrap(); - let signature = Signature::from_slice(&URL_SAFE_NO_PAD.decode(parts[2]).unwrap()).unwrap(); - verifying_key - .verify(format!("{}.{}", parts[0], parts[1]).as_bytes(), &signature) - .unwrap(); - - assert_eq!( - URL_SAFE_NO_PAD.decode(parts[0]).unwrap(), - br#"{"typ":"JWT","alg":"ES256"}"# - ); - let claims: serde_json::Value = - serde_json::from_slice(&URL_SAFE_NO_PAD.decode(parts[1]).unwrap()).unwrap(); - assert_eq!(claims["aud"], "https://push.example.com"); - assert_eq!(claims["sub"], "mailto:admin@example.org"); - assert_eq!(claims["exp"], now + VAPID_TOKEN_TTL); - } - - const SEC1_PEM: &str = "-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIP4Zv7be5hDH0x4ur6ditW+whzyZBXK1Vyjn6aIDo0jhoAoGCCqGSM49 -AwEHoUQDQgAEVc4PXr+z61s9/dIas44+S0Nza3gm1UW/avddp99dUsEi3JV0H4Yk -1yfqVJ/O9KPvQ69uMAY0t3A5lx/GvOOZfg== ------END EC PRIVATE KEY-----"; - - const PKCS8_NAMED_PEM: &str = "-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg/hm/tt7mEMfTHi6v -p2K1b7CHPJkFcrVXKOfpogOjSOGhRANCAARVzg9ev7PrWz390hqzjj5LQ3NreCbV -Rb9q912n311SwSLclXQfhiTXJ+pUn870o+9Dr24wBjS3cDmXH8a845l+ ------END PRIVATE KEY-----"; - - const PKCS8_EXPLICIT_PEM: &str = "-----BEGIN PRIVATE KEY----- -MIIBeQIBADCCAQMGByqGSM49AgEwgfcCAQEwLAYHKoZIzj0BAQIhAP////8AAAAB -AAAAAAAAAAAAAAAA////////////////MFsEIP////8AAAABAAAAAAAAAAAAAAAA -///////////////8BCBaxjXYqjqT57PrvVV2mIa8ZR0GsMxTsPY7zjw+J9JgSwMV -AMSdNgiG5wSTamZ44ROdJreBn36QBEEEaxfR8uEsQkf4vOblY6RA8ncDfYEt6zOg -9KE5RdiYwpZP40Li/hp/m47n60p8D54WK84zV2sxXs7LtkBoN79R9QIhAP////8A -AAAA//////////+85vqtpxeehPO5ysL8YyVRAgEBBG0wawIBAQQg/hm/tt7mEMfT -Hi6vp2K1b7CHPJkFcrVXKOfpogOjSOGhRANCAARVzg9ev7PrWz390hqzjj5LQ3Nr -eCbVRb9q912n311SwSLclXQfhiTXJ+pUn870o+9Dr24wBjS3cDmXH8a845l+ ------END PRIVATE KEY-----"; - - const PKCS8_P384_PEM: &str = "-----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDCUx+yT22yGHP9q+Y1y -UedDkevSvPaUuSPH8Q4FJBdYKKLqX4a5VdBIOonKPC4Yj7yhZANiAAQPRBsMOJy/ -B4yDfR2rGOd2H6Kv3fQNHPj9Nu5Tks8QYMLzrX8ONCNoFnNUQl9S0r0QS6phVqD0 -1kt0wbEvKr7mPM/R8XS8dX0xYC58CXHqBsTM0piQN2R7kqWDJ5i4OjE= ------END PRIVATE KEY-----"; - - #[test] - fn accepts_equivalent_p256_encodings() { - let named = VapidKey::from_pkcs8_pem(PKCS8_NAMED_PEM).unwrap(); - let sec1 = VapidKey::from_pkcs8_pem(SEC1_PEM).unwrap(); - let explicit = VapidKey::from_pkcs8_pem(PKCS8_EXPLICIT_PEM).unwrap(); - - assert_eq!(named.public_key(), sec1.public_key()); - assert_eq!(named.public_key(), explicit.public_key()); - } - - #[test] - fn accepts_pem_with_leading_bom_and_whitespace() { - let dirty = format!("\u{feff} \n{PKCS8_NAMED_PEM}\n "); - assert_eq!( - VapidKey::from_pkcs8_pem(&dirty).unwrap().public_key(), - VapidKey::from_pkcs8_pem(PKCS8_NAMED_PEM) - .unwrap() - .public_key() - ); - } - - #[test] - fn rejects_wrong_curve_key() { - assert!(VapidKey::from_pkcs8_pem(PKCS8_P384_PEM).is_err()); - } - - #[test] - fn rejects_garbage_with_actionable_error() { - let err = VapidKey::from_pkcs8_pem("not a key").err().unwrap(); - assert!(err.contains("openssl pkey"), "{err}"); - } - - #[test] - fn contact_is_normalized_to_a_uri() { - for (input, expected) in [ - ("hello@stalw.art", Some("mailto:hello@stalw.art")), - (" hello@stalw.art ", Some("mailto:hello@stalw.art")), - ("mailto:hello@stalw.art", Some("mailto:hello@stalw.art")), - ("MAILTO:hello@stalw.art", Some("MAILTO:hello@stalw.art")), - ( - "https://stalw.art/contact", - Some("https://stalw.art/contact"), - ), - ("stalw.art", None), - ("http://stalw.art", None), - ("tel:+123456789", None), - ("", None), - ] { - assert_eq!( - normalize_contact(input).as_deref(), - expected, - "unexpected normalization of {input:?}" - ); - } - } - - #[test] - fn authorization_is_reused_per_endpoint_prefix() { - let vapid = Vapid::new(test_key(), None); - let now = 1_700_000_000; - let token = vapid - .authorization("https://push.example.com/push/a", now) - .unwrap(); - - assert_eq!( - vapid - .authorization("https://push.example.com/push/b?x=1", now + 60) - .unwrap(), - token - ); - assert_ne!( - vapid - .authorization("https://other.example.com/push/a", now) - .unwrap(), - token - ); - assert_ne!( - vapid - .authorization("https://push.example.com/push/a", now - 1) - .unwrap(), - token - ); - let refreshed = vapid - .authorization("https://push.example.com/push/a", now + VAPID_TOKEN_REFRESH) - .unwrap(); - assert_ne!(refreshed, token); - assert_eq!( - vapid - .authorization( - "https://push.example.com/push/c", - now + VAPID_TOKEN_REFRESH + 1 - ) - .unwrap(), - refreshed - ); - } - - #[test] - fn authorization_omits_subject_when_no_contact() { - let key = test_key(); - let header = key - .authorization("https://fcm.googleapis.com/fcm/send/xyz", None, 0) - .unwrap(); - let payload = header.split('.').nth(1).unwrap(); - let claims: serde_json::Value = - serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload).unwrap()).unwrap(); - assert_eq!(claims["aud"], "https://fcm.googleapis.com"); - assert!(claims.get("sub").is_none()); - } -} diff --git a/crates/common/src/scripts/functions/array.rs b/crates/common/src/scripts/functions/array.rs index 5d854f0d..e184b67b 100644 --- a/crates/common/src/scripts/functions/array.rs +++ b/crates/common/src/scripts/functions/array.rs @@ -4,11 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::collections::{HashMap, HashSet}; - +use ahash::{AHashMap, AHashSet}; use sieve::{Context, runtime::Variable}; -pub fn fn_count<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +const MAX_PRESIZE: usize = 2048; + +pub fn fn_count<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::Array(a) => a.len(), v => { @@ -22,18 +23,18 @@ pub fn fn_count<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_sort<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_sort<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let is_asc = v[1].to_bool(); - let mut arr = (*v[0].to_array()).clone(); + let mut arr = v[0].to_array().into_owned(); if is_asc { arr.sort_unstable(); } else { arr.sort_unstable_by(|a, b| b.cmp(a)); } - arr.into() + Variable::Array(arr.into()) } -pub fn fn_dedup<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_dedup<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let arr = v[0].to_array(); let mut result = Vec::with_capacity(arr.len()); @@ -46,10 +47,19 @@ pub fn fn_dedup<'x>(_: &'x Context<'x>, v: Vec) -> Variable { result.into() } -pub fn fn_cosine_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - let mut word_freq: HashMap = HashMap::new(); +pub fn fn_cosine_similarity<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let mut word_freq: AHashMap = AHashMap::with_capacity( + v.iter() + .map(|var| match var { + Variable::Array(l) => l.len(), + _ => 0, + }) + .max() + .unwrap_or_default() + .min(MAX_PRESIZE), + ); - for (idx, var) in v.into_iter().enumerate() { + for (idx, var) in v.iter().enumerate() { match var { Variable::Array(l) => { for item in l.iter() { @@ -83,7 +93,8 @@ pub fn fn_cosine_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variabl } pub fn cosine_similarity(a: &[&str], b: &[&str]) -> f64 { - let mut word_freq: HashMap<&str, [u32; 2]> = HashMap::new(); + let mut word_freq: AHashMap<&str, [u32; 2]> = + AHashMap::with_capacity(a.len().max(b.len()).min(MAX_PRESIZE)); for (idx, items) in [a, b].into_iter().enumerate() { for item in items { @@ -108,10 +119,10 @@ pub fn cosine_similarity(a: &[&str], b: &[&str]) -> f64 { } } -pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - let mut word_freq = [HashSet::new(), HashSet::new()]; +pub fn fn_jaccard_similarity<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let mut word_freq = [AHashSet::new(), AHashSet::new()]; - for (idx, var) in v.into_iter().enumerate() { + for (idx, var) in v.iter().enumerate() { match var { Variable::Array(l) => { for item in l.iter() { @@ -137,7 +148,7 @@ pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variab .into() } -pub fn fn_is_intersect<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_intersect<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match (&v[0], &v[1]) { (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), @@ -146,14 +157,14 @@ pub fn fn_is_intersect<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_winnow<'x>(_: &'x Context<'x>, mut v: Vec) -> Variable { - match v.remove(0) { +pub fn fn_winnow<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + match &v[0] { Variable::Array(a) => a .iter() .filter(|i| !i.is_empty()) .cloned() .collect::>() .into(), - v => v, + v => v.clone(), } } diff --git a/crates/common/src/scripts/functions/email.rs b/crates/common/src/scripts/functions/email.rs index 6feac109..10c4ebf2 100644 --- a/crates/common/src/scripts/functions/email.rs +++ b/crates/common/src/scripts/functions/email.rs @@ -5,10 +5,11 @@ */ use sieve::{Context, runtime::Variable}; +use std::borrow::Cow; use super::ApplyString; -pub fn fn_is_email<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_email<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let mut last_ch = 0; let mut in_quote = false; let mut at_count = 0; @@ -78,13 +79,14 @@ pub fn fn_is_email<'x>(_: &'x Context<'x>, v: Vec) -> Variable { (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() } -pub fn fn_email_part<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|s| { +pub fn fn_email_part<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let part = v[1].to_string(); + v[0].transform_str(|s| { s.rsplit_once('@') - .map(|(u, d)| match v[1].to_string().as_ref() { - "local" => Variable::from(u.trim()), - "domain" => Variable::from(d.trim()), - _ => Variable::default(), + .map(|(u, d)| match part.as_ref() { + "local" => Cow::Borrowed(u.trim()), + "domain" => Cow::Borrowed(d.trim()), + _ => Cow::Borrowed(""), }) .unwrap_or_default() }) diff --git a/crates/common/src/scripts/functions/header.rs b/crates/common/src/scripts/functions/header.rs index 00235ad8..0a58cda3 100644 --- a/crates/common/src/scripts/functions/header.rs +++ b/crates/common/src/scripts/functions/header.rs @@ -4,91 +4,86 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use mail_parser::{HeaderName, HeaderValue, MimeHeaders, parsers::fields::thread::thread_name}; +use mail_parser::{HeaderName, thread_name}; use sieve::{Context, compiler::ReceivedPart, runtime::Variable}; use super::ApplyString; -pub fn fn_received_part<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { - if let (Ok(part), Some(HeaderValue::Received(rcvd))) = ( +pub fn fn_received_part<'x>(ctx: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + if let (Ok(part), Some(rcvd)) = ( ReceivedPart::try_from(v[1].to_string().as_ref()), ctx.message() .part(ctx.part()) .and_then(|p| { - p.headers - .iter() - .filter(|h| h.name == HeaderName::Received) + p.headers() + .all(HeaderName::Received) .nth((v[0].to_integer() as usize).saturating_sub(1)) }) - .map(|h| &h.value), + .and_then(|h| h.value().as_received()), ) { - part.eval(rcvd).unwrap_or_default() + ctx.received_part(&part, rcvd).unwrap_or_default() } else { Variable::default() } } -pub fn fn_is_encoding_problem<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { +pub fn fn_is_encoding_problem<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { ctx.message() .part(ctx.part()) - .map(|p| p.is_encoding_problem) + .map(|p| p.has_problems() || !p.decoded_checked().1.is_empty()) .unwrap_or_default() .into() } -pub fn fn_is_attachment<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { - ctx.message().attachments.contains(&ctx.part()).into() -} - -pub fn fn_is_body<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { - (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part())) +pub fn fn_is_attachment<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { + let part_id = ctx.part(); + ctx.message() + .attachments() + .any(|p| p.id() == part_id) .into() } -pub fn fn_attachment_name<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { +pub fn fn_is_body<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { + let part_id = ctx.part(); + let message = ctx.message(); + message + .text_body() + .chain(message.html_body()) + .any(|p| p.id() == part_id) + .into() +} + +pub fn fn_attachment_name<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { ctx.message() .part(ctx.part()) .and_then(|p| p.attachment_name()) + .map(Variable::from) .unwrap_or_default() - .into() } -pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { +pub fn fn_mime_part_len<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { ctx.message() .part(ctx.part()) - .map(|p| p.len()) + .map(|p| p.decoded_len()) .unwrap_or_default() .into() } -pub fn fn_thread_name<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|s| thread_name(s).into()) +pub fn fn_thread_name<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + v[0].transform_str(|s| thread_name(s).into()) } -pub fn fn_is_header_utf8_valid<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_header_utf8_valid<'x>(ctx: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { ctx.message() .part(ctx.part()) .map(|p| { - let raw = ctx.message().raw_message(); - let mut is_valid = true; - if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { - for header in &p.headers { - if header.name == header_name - && raw - .get(header.offset_start() as usize..header.offset_end() as usize) - .and_then(|raw| std::str::from_utf8(raw).ok()) - .is_none() - { - is_valid = false; - break; - } - } + let is_valid = if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { + p.headers() + .all(header_name) + .all(|header| std::str::from_utf8(header.raw_value()).is_ok()) } else { - is_valid = raw - .get(p.raw_header_offset() as usize..p.raw_body_offset() as usize) - .and_then(|raw| std::str::from_utf8(raw).ok()) - .is_some(); - } + std::str::from_utf8(p.raw_headers()).is_ok() + }; Variable::from(is_valid) }) diff --git a/crates/common/src/scripts/functions/image.rs b/crates/common/src/scripts/functions/image.rs index 49af80d0..2f406871 100644 --- a/crates/common/src/scripts/functions/image.rs +++ b/crates/common/src/scripts/functions/image.rs @@ -4,55 +4,73 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::sniff::{SniffPrefix, Sniffed}; +use mail_parser::MessagePart; use sieve::{Context, runtime::Variable}; -pub fn fn_img_metadata<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { +pub trait ImageMetadata { + fn image_metadata(&self, property: &str) -> Option>; +} + +pub fn fn_img_metadata<'x>(ctx: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let property = v[1].to_string(); ctx.message() .part(ctx.part()) - .map(|p| p.contents()) - .and_then(|bytes| { - let arg = v[1].to_string(); - match arg.as_ref() { - "type" => imagesize::image_type(bytes).ok().map(|t| { - Variable::from(match t { - imagesize::ImageType::Aseprite => "aseprite", - imagesize::ImageType::Bmp => "bmp", - imagesize::ImageType::Dds(_) => "dds", - imagesize::ImageType::Exr => "exr", - imagesize::ImageType::Farbfeld => "farbfeld", - imagesize::ImageType::Gif => "gif", - imagesize::ImageType::Hdr => "hdr", - imagesize::ImageType::Heif(_) => "heif", - imagesize::ImageType::Ico => "ico", - imagesize::ImageType::Jpeg => "jpeg", - imagesize::ImageType::Jxl => "jxl", - imagesize::ImageType::Ktx2 => "ktx2", - imagesize::ImageType::Png => "png", - imagesize::ImageType::Pnm => "pnm", - imagesize::ImageType::Psd => "psd", - imagesize::ImageType::Qoi => "qoi", - imagesize::ImageType::Tga => "tga", - imagesize::ImageType::Tiff => "tiff", - imagesize::ImageType::Vtf => "vtf", - imagesize::ImageType::Webp => "webp", - imagesize::ImageType::Ilbm => "ilbm", - _ => "unknown", - }) - }), - "width" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width as i64)), - "height" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.height as i64)), - "area" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)), - "dimension" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)), - _ => None, - } - }) + .and_then(|part| part.image_metadata(&property)) .unwrap_or_default() } + +impl ImageMetadata for MessagePart<'_> { + fn image_metadata(&self, property: &str) -> Option> { + match self.sniff() { + Sniffed::Whole(bytes) => image_property(&bytes, property), + Sniffed::Prefix(prefix) => image_property(&prefix, property) + .or_else(|| image_property(&self.decoded(), property)), + } + } +} + +fn image_property(bytes: &[u8], property: &str) -> Option> { + match property { + "type" => imagesize::image_type(bytes).ok().map(|t| { + Variable::from(match t { + imagesize::ImageType::Aseprite => "aseprite", + imagesize::ImageType::Bmp => "bmp", + imagesize::ImageType::Dds(_) => "dds", + imagesize::ImageType::Exr => "exr", + imagesize::ImageType::Farbfeld => "farbfeld", + imagesize::ImageType::Gif => "gif", + imagesize::ImageType::Hdr => "hdr", + imagesize::ImageType::Heif(_) => "heif", + imagesize::ImageType::Ico => "ico", + imagesize::ImageType::Jpeg => "jpeg", + imagesize::ImageType::Jxl => "jxl", + imagesize::ImageType::Ktx2 => "ktx2", + imagesize::ImageType::Png => "png", + imagesize::ImageType::Pnm => "pnm", + imagesize::ImageType::Psd => "psd", + imagesize::ImageType::Qoi => "qoi", + imagesize::ImageType::Tga => "tga", + imagesize::ImageType::Tiff => "tiff", + imagesize::ImageType::Vtf => "vtf", + imagesize::ImageType::Webp => "webp", + imagesize::ImageType::Ilbm => "ilbm", + _ => "unknown", + }) + }), + "width" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width as i64)), + "height" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.height as i64)), + "area" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)), + "dimension" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)), + _ => None, + } +} + diff --git a/crates/common/src/scripts/functions/misc.rs b/crates/common/src/scripts/functions/misc.rs index 723d337e..de1f4a02 100644 --- a/crates/common/src/scripts/functions/misc.rs +++ b/crates/common/src/scripts/functions/misc.rs @@ -6,16 +6,16 @@ use std::{net::IpAddr, str::FromStr}; -use mail_auth::common::resolver::ToReverseName; +use mail_auth::dns::ToReverseName; use registry::types::ipmask::IpAddrOrMask; use sha1::Sha1; use sha2::{Sha256, Sha512}; use sieve::{Context, runtime::Variable}; use utils::HexEncode; -use super::ApplyString; +use super::{ApplyString, sniff::SniffPrefix}; -pub fn fn_is_empty<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_empty<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.is_empty(), Variable::Integer(_) | Variable::Float(_) => false, @@ -24,29 +24,29 @@ pub fn fn_is_empty<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_is_number<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_number<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() } -pub fn fn_is_ip_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_ip_addr<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string().parse::().is_ok().into() } -pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_ipv4_addr<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V4(_))) .into() } -pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_ipv6_addr<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V6(_))) .into() } -pub fn fn_is_ip_in_cidr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_ip_in_cidr<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let Ok(ip) = v[0].to_string().parse::() else { return false.into(); }; @@ -56,7 +56,7 @@ pub fn fn_is_ip_in_cidr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_ip_reverse_name<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .parse::() .map(|ip| ip.to_reverse_name()) @@ -64,24 +64,22 @@ pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x>, v: Vec) -> Variable .into() } -pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_detect_file_type<'x>(ctx: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let as_extension = v[0].to_string() == "ext"; ctx.message() .part(ctx.part()) - .and_then(|p| infer::get(p.contents())) - .map(|t| { - Variable::from( - if v[0].to_string() != "ext" { - t.mime_type() - } else { - t.extension() - } - .to_string(), - ) + .and_then(|p| infer::get(&p.sniff_prefix())) + .map(|file_type| { + Variable::borrowed(if as_extension { + file_type.extension() + } else { + file_type.mime_type() + }) }) .unwrap_or_default() } -pub fn fn_hash<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_hash<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { use sha1::Digest; let hash = v[1].to_string(); @@ -106,7 +104,7 @@ pub fn fn_hash<'x>(_: &'x Context<'x>, v: Vec) -> Variable { }) } -pub fn fn_get_var_names<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { +pub fn fn_get_var_names<'x>(ctx: &Context<'x>, _: &[Variable<'x>]) -> Variable<'x> { Variable::Array( ctx.global_variable_names() .map(|v| Variable::from(v.to_uppercase())) diff --git a/crates/common/src/scripts/functions/mod.rs b/crates/common/src/scripts/functions/mod.rs index 2312e564..9fcde900 100644 --- a/crates/common/src/scripts/functions/mod.rs +++ b/crates/common/src/scripts/functions/mod.rs @@ -9,11 +9,13 @@ mod email; mod header; pub mod image; pub mod misc; +pub mod sniff; pub mod text; pub mod unicode; pub mod url; use sieve::{FunctionMap, runtime::Variable}; +use std::borrow::Cow; use self::{array::*, email::*, header::*, image::*, misc::*, text::*, unicode::*, url::*}; @@ -135,11 +137,26 @@ pub fn register_functions_untrusted() -> FunctionMap { } pub trait ApplyString<'x> { - fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable; + fn transform(&self, f: impl Fn(&str) -> Variable<'x>) -> Variable<'x>; + fn transform_str(&self, f: F) -> Variable<'x>; + fn split_str(&self, split: F) -> Variable<'x>; + fn split_str_once(&self, split: F) -> Variable<'x>; } -impl ApplyString<'_> for Variable { - fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable { +pub trait StrTransform: for<'s> Fn(&'s str) -> Cow<'s, str> {} + +impl Fn(&'s str) -> Cow<'s, str>> StrTransform for F {} + +pub trait StrSplit: for<'s> Fn(&'s str, &mut dyn FnMut(&'s str)) {} + +impl Fn(&'s str, &mut dyn FnMut(&'s str))> StrSplit for F {} + +pub trait StrSplitOnce: for<'s> Fn(&'s str) -> Option<(&'s str, &'s str)> {} + +impl Fn(&'s str) -> Option<(&'s str, &'s str)>> StrSplitOnce for F {} + +impl<'x> ApplyString<'x> for Variable<'x> { + fn transform(&self, f: impl Fn(&str) -> Variable<'x>) -> Variable<'x> { match self { Variable::String(s) => f(s), Variable::Array(list) => list @@ -153,4 +170,63 @@ impl ApplyString<'_> for Variable { v => f(v.to_string().as_ref()), } } + + fn transform_str(&self, f: F) -> Variable<'x> { + match self { + Variable::Array(list) => list + .iter() + .map(|v| transform_str_item(v, &f)) + .collect::>() + .into(), + v => transform_str_item(v, &f), + } + } + + fn split_str(&self, split: F) -> Variable<'x> { + let mut parts = Vec::new(); + match self { + Variable::String(Cow::Borrowed(text)) => { + split(text, &mut |part| parts.push(Variable::borrowed(part))); + } + value => { + let text = value.to_string(); + split(text.as_ref(), &mut |part| { + parts.push(Variable::from(part.to_string())) + }); + } + } + parts.into() + } + + fn split_str_once(&self, split: F) -> Variable<'x> { + match self { + Variable::String(Cow::Borrowed(text)) => split(text) + .map(|(head, tail)| { + Variable::from(vec![Variable::borrowed(head), Variable::borrowed(tail)]) + }) + .unwrap_or_default(), + value => { + let text = value.to_string(); + split(text.as_ref()) + .map(|(head, tail)| { + Variable::from(vec![ + Variable::from(head.to_string()), + Variable::from(tail.to_string()), + ]) + }) + .unwrap_or_default() + } + } + } +} + +fn transform_str_item<'x, F: StrTransform>(value: &Variable<'x>, f: &F) -> Variable<'x> { + match value { + Variable::String(Cow::Borrowed(s)) => match f(s) { + Cow::Borrowed(s) => Variable::borrowed(s), + Cow::Owned(s) => Variable::from(s), + }, + Variable::String(s) => Variable::from(f(s).into_owned()), + v => Variable::from(f(v.to_string().as_ref()).into_owned()), + } } diff --git a/crates/common/src/scripts/functions/sniff.rs b/crates/common/src/scripts/functions/sniff.rs new file mode 100644 index 00000000..ed0f399d --- /dev/null +++ b/crates/common/src/scripts/functions/sniff.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use encodify::{base64, qp}; +use mail_parser::{Encoding, MessagePart}; +use std::borrow::Cow; + +pub const SNIFF_WINDOW: usize = 64 * 1024; +const SNIFF_MIN: usize = 8 * 1024; +const OLE2_MAGIC: &[u8] = &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; +const ZIP_MAGIC: &[u8; 4] = b"PK\x03\x04"; +const OOXML_SCAN: usize = 49 + 3 * (6000 + 4 + 26) + 26 + 8; +const ZSTD_SKIPPABLE: u32 = 0x184D_2A50; +const ZSTD_SKIPPABLE_MASK: u32 = 0xFFFF_FFF0; +const HTML_SIGNATURE_LEN: usize = b" { + Prefix(Cow<'m, [u8]>), + Whole(Cow<'m, [u8]>), +} + +pub trait SniffPrefix<'m> { + fn sniff(&self) -> Sniffed<'m>; + + fn sniff_prefix(&self) -> Cow<'m, [u8]> { + match self.sniff() { + Sniffed::Prefix(bytes) | Sniffed::Whole(bytes) => bytes, + } + } +} + +impl<'m> SniffPrefix<'m> for MessagePart<'m> { + fn sniff(&self) -> Sniffed<'m> { + let Some((window, _)) = self + .raw_body() + .split_at_checked(SNIFF_WINDOW) + .filter(|(_, rest)| !rest.is_empty()) + else { + return Sniffed::Whole(self.decoded()); + }; + let prefix = match self.encoding() { + Encoding::None => Cow::Borrowed(window), + Encoding::Base64 => { + let mut decoded = Vec::with_capacity(window.len() / 4 * 3 + 3); + base64::MIME.decode_prefix(window, &mut decoded); + Cow::Owned(decoded) + } + Encoding::QuotedPrintable => { + let lines = window + .iter() + .rposition(|&byte| byte == b'\n') + .and_then(|end| window.get(..=end)) + .unwrap_or_default(); + qp::BODY.decode(lines).unwrap_or(Cow::Borrowed(lines)) + } + }; + if is_enough_to_sniff(&prefix) { + Sniffed::Prefix(prefix) + } else { + Sniffed::Whole(self.decoded()) + } + } +} + +fn is_enough_to_sniff(prefix: &[u8]) -> bool { + let Some(&[b0, b1, b2, b3, b4, b5, b6, b7]) = prefix.first_chunk::<8>() else { + return false; + }; + let magic = [b0, b1, b2, b3]; + if prefix.len() < SNIFF_MIN + || prefix.starts_with(OLE2_MAGIC) + || u32::from_le_bytes(magic) & ZSTD_SKIPPABLE_MASK == ZSTD_SKIPPABLE + || ([b4, b5, b6, b7] == *b"ftyp" && u32::from_be_bytes(magic) as usize > prefix.len()) + { + false + } else if magic == *ZIP_MAGIC { + prefix + .get(18..22) + .and_then(|size| size.try_into().ok()) + .is_some_and(|size: [u8; 4]| { + (u32::from_le_bytes(size) as usize).saturating_add(OOXML_SCAN) < prefix.len() + }) + } else { + let text = trim_blanks(prefix); + text.len() >= HTML_SIGNATURE_LEN && trim_byte_order_marks(text).len() >= XML_SIGNATURE_LEN + } +} + +fn trim_blanks(bytes: &[u8]) -> &[u8] { + let blanks = bytes + .iter() + .take_while(|byte| matches!(byte, 0x09 | 0x0A | 0x0C | 0x0D | 0x20)) + .count(); + bytes.get(blanks..).unwrap_or_default() +} + +fn trim_byte_order_marks(mut bytes: &[u8]) -> &[u8] { + loop { + bytes = match bytes { + [0xEF, 0xBB, 0xBF, rest @ ..] => rest, + [0xFE, 0xFF, _, ..] | [0xFF, 0xFE, _, ..] => bytes.get(2..).unwrap_or_default(), + _ => return bytes, + }; + } +} + diff --git a/crates/common/src/scripts/functions/text.rs b/crates/common/src/scripts/functions/text.rs index fb411feb..d95c8d08 100644 --- a/crates/common/src/scripts/functions/text.rs +++ b/crates/common/src/scripts/functions/text.rs @@ -6,22 +6,24 @@ use mail_parser::decoders::html::html_to_text; use sieve::{Context, runtime::Variable}; +use std::borrow::Cow; +use utils::text::contains_ignore_case; use super::ApplyString; -pub fn fn_trim<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim())) +pub fn fn_trim<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + v[0].transform_str(|s| s.trim().into()) } -pub fn fn_trim_end<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim_end())) +pub fn fn_trim_end<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + v[0].transform_str(|s| s.trim_end().into()) } -pub fn fn_trim_start<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim_start())) +pub fn fn_trim_start<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + v[0].transform_str(|s| s.trim_start().into()) } -pub fn fn_len<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_len<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.len(), Variable::Array(a) => a.len(), @@ -30,15 +32,15 @@ pub fn fn_len<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_to_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_to_lowercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].transform(|s| Variable::from(s.to_lowercase())) } -pub fn fn_to_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_to_uppercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].transform(|s| Variable::from(s.to_uppercase())) } -pub fn fn_is_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_uppercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) @@ -47,7 +49,7 @@ pub fn fn_is_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { }) } -pub fn fn_is_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_lowercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) @@ -56,20 +58,19 @@ pub fn fn_is_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { }) } -pub fn fn_has_digits<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_has_digits<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) } -pub fn tokenize_words(v: &Variable) -> Variable { - v.to_string() - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(word.to_string())) - .collect::>() - .into() +pub fn tokenize_words<'x>(v: &Variable<'x>) -> Variable<'x> { + v.split_str(|text, word| { + text.split_whitespace() + .filter(|word| word.chars().all(|c| c.is_alphanumeric())) + .for_each(word) + }) } -pub fn fn_count_spaces<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_count_spaces<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .as_ref() .chars() @@ -78,7 +79,7 @@ pub fn fn_count_spaces<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_count_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_count_uppercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .as_ref() .chars() @@ -87,7 +88,7 @@ pub fn fn_count_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable .into() } -pub fn fn_count_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_count_lowercase<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .as_ref() .chars() @@ -96,17 +97,17 @@ pub fn fn_count_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable .into() } -pub fn fn_count_chars<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_count_chars<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string().as_ref().chars().count().into() } -pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_eq_ignore_case<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .eq_ignore_ascii_case(v[1].to_string().as_ref()) .into() } -pub fn fn_contains<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_contains<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.contains(v[1].to_string().as_ref()), Variable::Array(arr) => arr.contains(&v[1]), @@ -115,10 +116,10 @@ pub fn fn_contains<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_contains_ignore_case<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let needle = v[1].to_string(); match &v[0] { - Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), + Variable::String(s) => contains_ignore_case(s, &needle), Variable::Array(arr) => arr.iter().any(|v| match v { Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), _ => false, @@ -128,109 +129,100 @@ pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x>, v: Vec) -> Vari .into() } -pub fn fn_starts_with<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_starts_with<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string() .starts_with(v[1].to_string().as_ref()) .into() } -pub fn fn_ends_with<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_ends_with<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { v[0].to_string().ends_with(v[1].to_string().as_ref()).into() } -pub fn fn_lines<'x>(_: &'x Context<'x>, mut v: Vec) -> Variable { - match v.remove(0) { - Variable::String(s) => s +pub fn fn_lines<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + match &v[0] { + Variable::String(Cow::Borrowed(text)) => text .lines() - .map(|s| Variable::from(s.to_string())) + .map(Variable::borrowed) .collect::>() .into(), - val => val, + Variable::String(text) => text + .lines() + .map(|line| Variable::from(line.to_string())) + .collect::>() + .into(), + value => value.clone(), } } -pub fn fn_substring<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].to_string() - .chars() - .skip(v[1].to_usize()) - .take(v[2].to_usize()) - .collect::() - .into() +pub fn fn_substring<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let start = v[1].to_usize(); + let len = v[2].to_usize(); + match &v[0] { + Variable::String(Cow::Borrowed(s)) => Variable::borrowed(char_range(s, start, len)), + value => value + .to_string() + .chars() + .skip(start) + .take(len) + .collect::() + .into(), + } } -pub fn fn_strip_prefix<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +fn char_range(s: &str, start: usize, len: usize) -> &str { + let from = s.char_indices().nth(start).map_or(s.len(), |(at, _)| at); + let to = s[from..] + .char_indices() + .nth(len) + .map_or(s.len(), |(at, _)| from + at); + &s[from..to] +} + +pub fn fn_strip_prefix<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let prefix = v[1].to_string(); - v[0].transform(|s| { - s.strip_prefix(prefix.as_ref()) - .map(Variable::from) - .unwrap_or_default() - }) + v[0].transform_str(|s| s.strip_prefix(prefix.as_ref()).unwrap_or_default().into()) } -pub fn fn_strip_suffix<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_strip_suffix<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let suffix = v[1].to_string(); - v[0].transform(|s| { - s.strip_suffix(suffix.as_ref()) - .map(Variable::from) - .unwrap_or_default() + v[0].transform_str(|s| s.strip_suffix(suffix.as_ref()).unwrap_or_default().into()) +} + +pub fn fn_split<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let separator = v[1].to_string(); + v[0].split_str(|text, part| text.split(separator.as_ref()).for_each(part)) +} + +pub fn fn_rsplit<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let separator = v[1].to_string(); + v[0].split_str(|text, part| text.rsplit(separator.as_ref()).for_each(part)) +} + +pub fn fn_split_n<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let separator = v[1].to_string(); + let limit = v[2].to_integer() as usize; + v[0].split_str(|text, part| { + let mut rest = text; + for _ in 0..limit { + let Some((head, tail)) = rest.split_once(separator.as_ref()) else { + break; + }; + part(head); + rest = tail; + } + part(rest); }) } -pub fn fn_split<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].to_string() - .split(v[1].to_string().as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into() +pub fn fn_split_once<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let separator = v[1].to_string(); + v[0].split_str_once(|text| text.split_once(separator.as_ref())) } -pub fn fn_rsplit<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].to_string() - .rsplit(v[1].to_string().as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into() -} - -pub fn fn_split_n<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - let value = v[0].to_string(); - let arg = v[1].to_string(); - let num = v[2].to_integer() as usize; - let mut result = Vec::new(); - - let mut s = value.as_ref(); - for _ in 0..num { - if let Some((a, b)) = s.split_once(arg.as_ref()) { - result.push(Variable::from(a.to_string())); - s = b; - } else { - break; - } - } - result.push(Variable::from(s.to_string())); - result.into() -} - -pub fn fn_split_once<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].to_string() - .split_once(v[1].to_string().as_ref()) - .map(|(a, b)| { - Variable::Array( - vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), - ) - }) - .unwrap_or_default() -} - -pub fn fn_rsplit_once<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].to_string() - .rsplit_once(v[1].to_string().as_ref()) - .map(|(a, b)| { - Variable::Array( - vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), - ) - }) - .unwrap_or_default() +pub fn fn_rsplit_once<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + let separator = v[1].to_string(); + v[0].split_str_once(|text| text.rsplit_once(separator.as_ref())) } /** @@ -240,7 +232,7 @@ pub fn fn_rsplit_once<'x>(_: &'x Context<'x>, v: Vec) -> Variable { * * Copyright (c) 2016 Titus Wormer */ -pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_levenshtein_distance<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let a = v[0].to_string(); let b = v[1].to_string(); @@ -305,13 +297,13 @@ pub fn levenshtein_distance(a: &str, b: &str) -> usize { result } -pub fn fn_detect_language<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_detect_language<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { whatlang::detect_lang(v[0].to_string().as_ref()) .map(|l| l.code()) .unwrap_or("unknown") .into() } -pub fn fn_html_to_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_html_to_text<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { html_to_text(v[0].to_string().as_ref()).into() } diff --git a/crates/common/src/scripts/functions/unicode.rs b/crates/common/src/scripts/functions/unicode.rs index 90e9d115..e649c8a6 100644 --- a/crates/common/src/scripts/functions/unicode.rs +++ b/crates/common/src/scripts/functions/unicode.rs @@ -8,7 +8,7 @@ use sieve::{Context, runtime::Variable}; use crate::scripts::IsMixedCharset; -pub fn fn_is_ascii<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_ascii<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.is_ascii(), Variable::Integer(_) | Variable::Float(_) => true, @@ -20,7 +20,7 @@ pub fn fn_is_ascii<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_has_zwsp<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_has_zwsp<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_zwsp()), Variable::Array(a) => a.iter().any(|v| match v { @@ -32,7 +32,7 @@ pub fn fn_has_zwsp<'x>(_: &'x Context<'x>, v: Vec) -> Variable { .into() } -pub fn fn_has_obscured<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_has_obscured<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_obscured()), Variable::Array(a) => a.iter().any(|v| match v { @@ -68,20 +68,20 @@ impl CharUtils for char { } } -pub fn fn_cure_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_cure_text<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { decancer::cure(v[0].to_string().as_ref(), decancer::Options::default()) .map(String::from) .unwrap_or_default() .into() } -pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_unicode_skeleton<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { unicode_security::skeleton(v[0].to_string().as_ref()) .collect::() .into() } -pub fn fn_is_mixed_charset<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_is_mixed_charset<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let text = v[0].to_string(); if !text.is_empty() { text.as_ref().is_mixed_charset() diff --git a/crates/common/src/scripts/functions/url.rs b/crates/common/src/scripts/functions/url.rs index cdaeac25..a188e216 100644 --- a/crates/common/src/scripts/functions/url.rs +++ b/crates/common/src/scripts/functions/url.rs @@ -9,7 +9,7 @@ use sieve::{Context, runtime::Variable}; use super::ApplyString; -pub fn fn_uri_part<'x>(_: &'x Context<'x>, v: Vec) -> Variable { +pub fn fn_uri_part<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { let part = v[1].to_string(); v[0].transform(|uri| { uri.parse::() @@ -32,8 +32,8 @@ pub fn fn_uri_part<'x>(_: &'x Context<'x>, v: Vec) -> Variable { }) } -pub fn fn_puny_decode<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].transform(|domain| { +pub fn fn_puny_decode<'x>(_: &Context<'x>, v: &[Variable<'x>]) -> Variable<'x> { + v[0].transform_str(|domain| { if domain.contains("xn--") { let mut decoded = String::with_capacity(domain.len()); for part in domain.split('.') { diff --git a/crates/common/src/scripts/mod.rs b/crates/common/src/scripts/mod.rs index 75c92d35..18241285 100644 --- a/crates/common/src/scripts/mod.rs +++ b/crates/common/src/scripts/mod.rs @@ -4,9 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - use sieve::{Envelope, runtime::Variable}; +use std::borrow::Cow; use store::Value; use unicode_security::mixed_script::AugmentedScriptSet; @@ -19,39 +18,33 @@ pub mod plugins; #[serde(tag = "action")] #[serde(rename_all = "camelCase")] pub enum ScriptModification { - SetEnvelope { - name: Envelope, - value: String, - }, - AddHeader { - name: Arc, - value: Arc, - }, + SetEnvelope { name: Envelope, value: String }, + AddHeader { name: String, value: String }, } -pub fn into_sieve_value(value: Value) -> Variable { +pub fn into_sieve_value(value: Value<'_>) -> Variable<'_> { match value { Value::Integer(v) => Variable::Integer(v), Value::Bool(v) => Variable::Integer(i64::from(v)), Value::Float(v) => Variable::Float(v), - Value::Text(v) => Variable::String(v.into_owned().into()), + Value::Text(v) => Variable::String(v), Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), Value::Null => Variable::default(), } } -pub fn into_store_value(value: Variable) -> Value<'static> { +pub fn into_store_value(value: Variable<'_>) -> Value<'static> { match value { - Variable::String(v) => Value::Text(v.to_string().into()), + Variable::String(v) => Value::Text(v.into_owned().into()), Variable::Integer(v) => Value::Integer(v), Variable::Float(v) => Value::Float(v), v => Value::Text(v.to_string().into_owned().into()), } } -pub fn to_store_value(value: &Variable) -> Value<'static> { +pub fn to_store_value<'x>(value: &'x Variable<'_>) -> Value<'x> { match value { - Variable::String(v) => Value::Text(v.to_string().into()), + Variable::String(v) => Value::Text(Cow::Borrowed(v.as_ref())), Variable::Integer(v) => Value::Integer(*v), Variable::Float(v) => Value::Float(*v), v => Value::Text(v.to_string().into_owned().into()), diff --git a/crates/common/src/scripts/plugins/dns.rs b/crates/common/src/scripts/plugins/dns.rs index ba48877d..58f3c300 100644 --- a/crates/common/src/scripts/plugins/dns.rs +++ b/crates/common/src/scripts/plugins/dns.rs @@ -19,7 +19,7 @@ pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("dns_exists", plugin_id, 2); } -pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec(ctx: PluginContext<'_>) -> trc::Result> { let entry = ctx.arguments[0].to_string(); let record_type = ctx.arguments[1].to_string(); @@ -57,7 +57,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { .await { Ok(result) => result - .rrset + .records .iter() .flat_map(|mx| { mx.exchanges @@ -69,7 +69,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("txt") { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { if entry.contains("origin") { return Ok(Variable::from("23028|US|arin|2002-01-04".to_string())); @@ -100,7 +100,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { .await { Ok(result) => result - .rrset + .records .iter() .map(|host| Variable::from(host.to_string())) .collect::>() @@ -111,7 +111,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { Variable::default() } } else if record_type.eq_ignore_ascii_case("ipv4") { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { if entry.contains(".168.192.") { let parts = entry.split('.').collect::>(); @@ -129,7 +129,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { .await { Ok(result) => result - .rrset + .records .iter() .map(|ip| Variable::from(ip.to_string())) .collect::>() @@ -147,7 +147,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { .await { Ok(result) => result - .rrset + .records .iter() .map(|ip| Variable::from(ip.to_string())) .collect::>() @@ -159,7 +159,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { }) } -pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result> { let entry = ctx.arguments[0].to_string(); let record_type = ctx.arguments[1].to_string(); @@ -170,7 +170,7 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result { } else if record_type.eq_ignore_ascii_case("ptr") { ctx.server.dns_exists_ptr(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ipv4") { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { if entry.starts_with("2.0.168.192.") { return Ok(1.into()); diff --git a/crates/common/src/scripts/plugins/exec.rs b/crates/common/src/scripts/plugins/exec.rs index 36d94037..74cec9e7 100644 --- a/crates/common/src/scripts/plugins/exec.rs +++ b/crates/common/src/scripts/plugins/exec.rs @@ -14,8 +14,13 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("exec", plugin_id, 2); } -pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { - let mut arguments = ctx.arguments.into_iter(); +pub async fn exec(ctx: PluginContext<'_>) -> trc::Result> { + let mut arguments = ctx + .arguments + .into_iter() + .map(Variable::into_owned) + .collect::>() + .into_iter(); tokio::task::spawn_blocking(move || { let command = arguments diff --git a/crates/common/src/scripts/plugins/headers.rs b/crates/common/src/scripts/plugins/headers.rs index 7b0bddee..d7a70bd8 100644 --- a/crates/common/src/scripts/plugins/headers.rs +++ b/crates/common/src/scripts/plugins/headers.rs @@ -14,17 +14,20 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("add_header", plugin_id, 2); } -pub fn exec(ctx: PluginContext<'_>) -> trc::Result { - Ok(if let (Variable::String(name), Variable::String(value)) = - (&ctx.arguments[0], &ctx.arguments[1]) - { - ctx.modifications.push(ScriptModification::AddHeader { - name: name.clone(), - value: value.clone(), - }); - true - } else { - false - } - .into()) +pub fn exec(ctx: PluginContext<'_>) -> trc::Result> { + let mut arguments = ctx.arguments.into_iter(); + Ok( + if let (Some(Variable::String(name)), Some(Variable::String(value))) = + (arguments.next(), arguments.next()) + { + ctx.modifications.push(ScriptModification::AddHeader { + name: name.into_owned(), + value: value.into_owned(), + }); + true + } else { + false + } + .into(), + ) } diff --git a/crates/common/src/scripts/plugins/http.rs b/crates/common/src/scripts/plugins/http.rs index 7241f741..f49d80ff 100644 --- a/crates/common/src/scripts/plugins/http.rs +++ b/crates/common/src/scripts/plugins/http.rs @@ -15,13 +15,13 @@ pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("http_header", plugin_id, 4); } -pub async fn exec_header(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec_header(ctx: PluginContext<'_>) -> trc::Result> { let url = ctx.arguments[0].to_string(); let header = ctx.arguments[1].to_string(); let agent = ctx.arguments[2].to_string(); let timeout = ctx.arguments[3].to_string().parse::().unwrap_or(5000); - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] if url.contains("redirect.") { return Ok(Variable::from(url.split_once("/?").unwrap().1.to_string())); } diff --git a/crates/common/src/scripts/plugins/llm_prompt.rs b/crates/common/src/scripts/plugins/llm_prompt.rs index 2be23a29..0bf2353d 100644 --- a/crates/common/src/scripts/plugins/llm_prompt.rs +++ b/crates/common/src/scripts/plugins/llm_prompt.rs @@ -4,17 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use sieve::{FunctionMap, compiler::Number, runtime::Variable}; -use std::time::Instant; -use trc::{AiEvent, SecurityEvent}; +use sieve::{FunctionMap, runtime::Variable}; use super::PluginContext; + pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("llm_prompt", plugin_id, 3); } -pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { +#[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] +pub async fn exec(ctx: PluginContext<'_>) -> trc::Result> { Ok(false.into()) } diff --git a/crates/common/src/scripts/plugins/lookup.rs b/crates/common/src/scripts/plugins/lookup.rs index 06458c0a..420c23db 100644 --- a/crates/common/src/scripts/plugins/lookup.rs +++ b/crates/common/src/scripts/plugins/lookup.rs @@ -6,6 +6,7 @@ use super::PluginContext; use crate::scripts::into_sieve_value; +use compact_str::CompactString; use sieve::{FunctionMap, runtime::Variable}; use store::{Deserialize, Value, dispatch::lookup::KeyValue}; @@ -25,14 +26,17 @@ pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("is_local_domain", plugin_id, 1); } -pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec(ctx: PluginContext<'_>) -> trc::Result> { let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_str()), + Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_ref()), _ => Some(ctx.server.core.storage.memory.clone()), } .ok_or_else(|| { trc::SieveEvent::RuntimeError - .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) + .ctx( + trc::Key::Name, + CompactString::from(ctx.arguments[0].to_string()), + ) .details("Unknown store") })?; @@ -51,14 +55,17 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { .into()) } -pub async fn exec_get(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec_get(ctx: PluginContext<'_>) -> trc::Result> { match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_str()), + Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_ref()), _ => Some(ctx.server.core.storage.memory.clone()), } .ok_or_else(|| { trc::SieveEvent::RuntimeError - .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) + .ctx( + trc::Key::Name, + CompactString::from(ctx.arguments[0].to_string()), + ) .details("Unknown store") })? .key_get::(ctx.arguments[1].to_string()) @@ -66,7 +73,7 @@ pub async fn exec_get(ctx: PluginContext<'_>) -> trc::Result { .map(|v| v.map(|v| v.into_inner()).unwrap_or_default()) } -pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result> { let expires = match &ctx.arguments[3] { Variable::Integer(v) => Some(*v as u64), Variable::Float(v) => Some(*v as u64), @@ -74,12 +81,15 @@ pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result { }; match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_str()), + Variable::String(v) if !v.is_empty() => ctx.server.get_lookup_store(v.as_ref()), _ => Some(ctx.server.core.storage.memory.clone()), } .ok_or_else(|| { trc::SieveEvent::RuntimeError - .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) + .ctx( + trc::Key::Name, + CompactString::from(ctx.arguments[0].to_string()), + ) .details("Unknown store") })? .key_set( @@ -98,7 +108,7 @@ pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result { .map(|_| true.into()) } -pub async fn exec_local_domain(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec_local_domain(ctx: PluginContext<'_>) -> trc::Result> { let domain = ctx.arguments[0].to_string(); if !domain.is_empty() { @@ -112,16 +122,19 @@ pub async fn exec_local_domain(ctx: PluginContext<'_>) -> trc::Result } #[derive(Debug, PartialEq, Eq)] -pub struct VariableWrapper(Variable); +pub struct VariableWrapper(Variable<'static>); impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(VariableWrapper( - bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) - .map(|v| v.0) - .unwrap_or_else(|_| { - Variable::String(String::from_utf8_lossy(bytes).into_owned().into()) - }), + bincode::serde::decode_from_slice::, _>( + bytes, + bincode::config::standard(), + ) + .map(|v| v.0) + .unwrap_or_else(|_| { + Variable::String(String::from_utf8_lossy(bytes).into_owned().into()) + }), )) } } @@ -133,7 +146,7 @@ impl From for VariableWrapper { } impl VariableWrapper { - pub fn into_inner(self) -> Variable { + pub fn into_inner(self) -> Variable<'static> { self.0 } } diff --git a/crates/common/src/scripts/plugins/mod.rs b/crates/common/src/scripts/plugins/mod.rs index 5ea29744..3d68bc18 100644 --- a/crates/common/src/scripts/plugins/mod.rs +++ b/crates/common/src/scripts/plugins/mod.rs @@ -28,7 +28,7 @@ pub struct PluginContext<'x> { pub server: &'x Server, pub message: &'x Message<'x>, pub modifications: &'x mut Vec, - pub arguments: Vec, + pub arguments: Vec>, } const PLUGINS_REGISTER: [RegisterPluginFnc; 13] = [ @@ -54,7 +54,7 @@ pub trait RegisterSievePlugins { impl RegisterSievePlugins for FunctionMap { fn register_plugins_trusted(mut self) -> Self { - #[cfg(feature = "test_mode")] + #[cfg(stalwart_tests)] { self.set_external_function("print", PLUGINS_REGISTER.len() as u32, 1) } @@ -72,10 +72,10 @@ impl RegisterSievePlugins for FunctionMap { } impl Core { - pub async fn run_plugin(&self, id: u32, ctx: PluginContext<'_>) -> Input { - #[cfg(feature = "test_mode")] + pub async fn run_plugin(&self, id: u32, ctx: PluginContext<'_>) -> Input<'static> { + #[cfg(stalwart_tests)] if id == PLUGINS_REGISTER.len() as u32 { - return test_print(ctx); + return test_support::test_print(ctx); } let session_id = ctx.session_id; @@ -100,14 +100,9 @@ impl Core { Ok(result) => result.into(), Err(err) => { trc::error!(err.span_id(session_id).details("Sieve runtime error")); - Input::FncResult(Variable::default()) + Input::Value(Variable::default()) } } } } -#[cfg(feature = "test_mode")] -pub fn test_print(ctx: PluginContext<'_>) -> Input { - println!("{}", ctx.arguments[0].to_string()); - Input::True -} diff --git a/crates/common/src/scripts/plugins/query.rs b/crates/common/src/scripts/plugins/query.rs index a3665e8d..a646bc08 100644 --- a/crates/common/src/scripts/plugins/query.rs +++ b/crates/common/src/scripts/plugins/query.rs @@ -6,6 +6,7 @@ use super::PluginContext; use crate::scripts::{into_sieve_value, to_store_value}; +use compact_str::CompactString; use sieve::{FunctionMap, runtime::Variable}; use std::cmp::Ordering; use store::{Rows, Value}; @@ -14,18 +15,21 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("query", plugin_id, 3); } -pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { +pub async fn exec(ctx: PluginContext<'_>) -> trc::Result> { // Obtain store name let store = match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => ctx .server - .get_lookup_store(v.as_str()) + .get_lookup_store(v.as_ref()) .and_then(|v| v.into_store()), _ => Some(ctx.server.core.storage.data.clone()), } .ok_or_else(|| { trc::SieveEvent::RuntimeError - .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) + .ctx( + trc::Key::Name, + CompactString::from(ctx.arguments[0].to_string()), + ) .details("Unknown store") })?; @@ -34,7 +38,10 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { if query.is_empty() { trc::bail!( trc::SieveEvent::RuntimeError - .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) + .ctx( + trc::Key::Name, + CompactString::from(ctx.arguments[0].to_string()) + ) .details("Empty query string") ); } diff --git a/crates/common/src/scripts/plugins/text.rs b/crates/common/src/scripts/plugins/text.rs index d4c4dbd7..97b2a940 100644 --- a/crates/common/src/scripts/plugins/text.rs +++ b/crates/common/src/scripts/plugins/text.rs @@ -6,6 +6,7 @@ use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use sieve::{FunctionMap, runtime::Variable}; +use std::borrow::Cow; use crate::scripts::functions::{ApplyString, text::tokenize_words}; @@ -19,10 +20,10 @@ pub fn register_domain_part(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("domain_part", plugin_id, 2); } -pub fn exec_tokenize(ctx: PluginContext<'_>) -> trc::Result { +pub fn exec_tokenize(ctx: PluginContext<'_>) -> trc::Result> { let mut v = ctx.arguments; let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { - "words" => return Ok(tokenize_words(&v[0])), + "words" => return Ok(tokenize_words(&v[0]).into_owned()), "uri" | "url" => (true, true, true), "uri_strict" | "url_strict" => (true, false, false), "email" => (false, false, true), @@ -47,7 +48,7 @@ pub fn exec_tokenize(ctx: PluginContext<'_>) -> trc::Result { .collect::>() .into() } - v => v, + v => v.into_owned(), }) } @@ -57,7 +58,7 @@ enum DomainPart { Host, } -pub fn exec_domain_part(ctx: PluginContext<'_>) -> trc::Result { +pub fn exec_domain_part(ctx: PluginContext<'_>) -> trc::Result> { let v = ctx.arguments; let part = match v[1].to_string().as_ref() { "sld" => DomainPart::Sld, @@ -66,13 +67,15 @@ pub fn exec_domain_part(ctx: PluginContext<'_>) -> trc::Result { _ => return Ok(Variable::default()), }; - Ok(v[0].transform(|domain| { - match part { - DomainPart::Sld => psl::domain_str(domain), - DomainPart::Tld => domain.rsplit_once('.').map(|(_, tld)| tld), - DomainPart::Host => domain.split_once('.').map(|(host, _)| host), - } - .map(Variable::from) - .unwrap_or_default() - })) + Ok(v[0] + .transform_str(|domain| { + match part { + DomainPart::Sld => psl::domain_str(domain), + DomainPart::Tld => domain.rsplit_once('.').map(|(_, tld)| tld), + DomainPart::Host => domain.split_once('.').map(|(host, _)| host), + } + .map(Cow::Borrowed) + .unwrap_or_default() + }) + .into_owned()) } diff --git a/crates/common/src/sharing/acl.rs b/crates/common/src/sharing/acl.rs deleted file mode 100644 index f07c4f18..00000000 --- a/crates/common/src/sharing/acl.rs +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{Server, cache::invalidate::CacheInvalidationBuilder, ipc::CacheInvalidation}; -use types::acl::{AclGrant, ArchivedAclGrant}; - -impl Server { - pub async fn refresh_acls( - &self, - acl_changes: &[AclGrant], - current: Option<&[AclGrant]>, - ) -> trc::Result<()> { - let mut changed_principals = CacheInvalidationBuilder::default(); - if let Some(acl_current) = current { - for current_item in acl_current { - let mut invalidate = true; - for change_item in acl_changes { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals - .invalidate(CacheInvalidation::AccessToken(current_item.account_id)); - } - } - - for change_item in acl_changes { - let mut invalidate = true; - for current_item in acl_current { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals - .invalidate(CacheInvalidation::AccessToken(change_item.account_id)); - } - } - } else { - for value in acl_changes { - changed_principals.invalidate(CacheInvalidation::AccessToken(value.account_id)); - } - } - - self.invalidate_caches(changed_principals).await - } - - pub async fn refresh_archived_acls( - &self, - acl_changes: &[AclGrant], - acl_current: &[ArchivedAclGrant], - ) -> trc::Result<()> { - let mut changed_principals = CacheInvalidationBuilder::default(); - - for current_item in acl_current.iter() { - let mut invalidate = true; - for change_item in acl_changes { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals.invalidate(CacheInvalidation::AccessToken( - current_item.account_id.to_native(), - )); - } - } - - for change_item in acl_changes { - let mut invalidate = true; - for current_item in acl_current.iter() { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals - .invalidate(CacheInvalidation::AccessToken(change_item.account_id)); - } - } - - self.invalidate_caches(changed_principals).await - } -} diff --git a/crates/common/src/sharing/file.rs b/crates/common/src/sharing/file.rs new file mode 100644 index 00000000..9044ba92 --- /dev/null +++ b/crates/common/src/sharing/file.rs @@ -0,0 +1,214 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{GroupwareResources, auth::AccessToken}; +use ahash::AHashMap; +use store::roaring::RoaringBitmap; +use types::acl::Acl; +use utils::map::bitmap::Bitmap; + +const MAX_ANCESTOR_WALK: usize = 256; + +#[derive(Debug, Default)] +pub struct FileNodeAccess { + pub readable: RoaringBitmap, + pub discoverable: RoaringBitmap, + grants: AHashMap>, +} + +impl FileNodeAccess { + #[inline(always)] + pub fn acl(&self, document_id: u32) -> Bitmap { + self.grants.get(&document_id).copied().unwrap_or_default() + } + + #[inline(always)] + pub fn has_acl(&self, document_id: u32, acl: Acl) -> bool { + self.grants + .get(&document_id) + .is_some_and(|grants| grants.contains(acl)) + } + + pub fn node_rights(&self, document_id: u32, parent_id: Option) -> Bitmap { + let mut acl = self.acl(document_id); + if self.may_unbind_from(parent_id) { + acl.insert(Acl::RemoveItems); + } + acl + } + + pub fn may_delete(&self, document_id: u32, parent_id: Option) -> bool { + self.discoverable.contains(document_id) + && (self.has_acl(document_id, Acl::RemoveItems) || self.may_unbind_from(parent_id)) + } + + pub fn may_destroy(&self, mut subtree: impl Iterator)>) -> bool { + subtree.all(|(document_id, parent_id)| self.may_delete(document_id, parent_id)) + } + + fn may_unbind_from(&self, parent_id: Option) -> bool { + parent_id.is_some_and(|parent_id| self.has_acl(parent_id, Acl::RemoveItems)) + } + + pub fn with_acl(&self, acl: Acl) -> RoaringBitmap { + self.grants + .iter() + .filter(|(_, grants)| grants.contains(acl)) + .map(|(document_id, _)| *document_id) + .collect() + } +} + +impl GroupwareResources { + pub fn file_acl(&self, access_token: &AccessToken, document_id: u32) -> Bitmap { + let mut grants = Bitmap::new(); + let mut current = self.resources.find_any(document_id); + for _ in 0..MAX_ANCESTOR_WALK { + let Some(resource) = current else { + break; + }; + if resource.is_container() { + for acl in resource.acls() { + if access_token.is_member(acl.account_id) { + grants.union(&acl.grants); + } + } + } + current = resource + .parent_id() + .and_then(|parent_id| self.resources.find_any(parent_id)); + } + grants + } + + pub fn file_discoverable(&self, access_token: &AccessToken, document_id: u32) -> bool { + is_readable(&self.file_acl(access_token, document_id)) + || self + .file_access(access_token) + .discoverable + .contains(document_id) + } + + pub fn file_access(&self, access_token: &AccessToken) -> FileNodeAccess { + let mut direct = DirectGrants::new(); + for resource in self.resources_with_acls() { + if !resource.is_container() { + continue; + } + let mut found = Bitmap::new(); + for acl in resource.acls() { + if access_token.is_member(acl.account_id) { + found.union(&acl.grants); + } + } + if !found.is_empty() { + direct.insert(resource.document_id(), found); + } + } + if direct.is_empty() { + return FileNodeAccess::default(); + } + + let grants = if self.paths.len() == self.resources.len() { + self.inherit_from_paths(&direct) + } else { + self.inherit_from_parents(&direct) + }; + + let readable = grants + .iter() + .filter(|(_, found)| is_readable(found)) + .map(|(document_id, _)| *document_id) + .collect::(); + let mut discoverable = readable.clone(); + for &document_id in direct.keys() { + discoverable.insert(document_id); + let mut current = self + .resources + .find_any(document_id) + .and_then(|node| node.parent_id()); + for _ in 0..MAX_ANCESTOR_WALK { + let Some(parent_id) = current else { + break; + }; + if !discoverable.insert(parent_id) { + break; + } + current = self + .resources + .find_any(parent_id) + .and_then(|node| node.parent_id()); + } + } + FileNodeAccess { + readable, + discoverable, + grants, + } + } + + fn inherit_from_paths(&self, direct: &DirectGrants) -> DirectGrants { + let mut grants = DirectGrants::with_capacity(direct.len()); + let mut members = RoaringBitmap::new(); + for (root, acl) in direct.iter().filter_map(|(document_id, acl)| { + self.any_resource_path_by_id(*document_id) + .map(|root| (root, acl)) + }) { + members.clear(); + members.insert(root.document_id()); + grants.entry(root.document_id()).or_default().union(acl); + for (_, path) in self.paths.range(format!("{}/", root.path())) { + if members.contains(path.parent_id) { + members.insert(path.document_id); + grants.entry(path.document_id).or_default().union(acl); + } + } + } + grants + } + + fn inherit_from_parents(&self, direct: &DirectGrants) -> DirectGrants { + let mut grants = DirectGrants::new(); + let mut unshared = RoaringBitmap::new(); + let mut chain = Vec::with_capacity(16); + for resource in self.resources.iter() { + let mut current = Some(resource.document_id()); + let mut found = Bitmap::new(); + chain.clear(); + while let Some(document_id) = current { + if let Some(inherited) = grants.get(&document_id) { + found = *inherited; + break; + } else if unshared.contains(document_id) || chain.len() >= MAX_ANCESTOR_WALK { + break; + } + chain.push(document_id); + current = self + .resources + .find_any(document_id) + .and_then(|node| node.parent_id()); + } + for document_id in chain.drain(..).rev() { + if let Some(acl) = direct.get(&document_id) { + found.union(acl); + } + if found.is_empty() { + unshared.insert(document_id); + } else { + grants.insert(document_id, found); + } + } + } + grants + } +} + +type DirectGrants = AHashMap>; + +fn is_readable(acl: &Bitmap) -> bool { + acl.contains(Acl::Read) || acl.contains(Acl::ReadItems) +} + diff --git a/crates/common/src/sharing/grants.rs b/crates/common/src/sharing/grants.rs new file mode 100644 index 00000000..70ea1263 --- /dev/null +++ b/crates/common/src/sharing/grants.rs @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::object::{JmapRight, calendar::CalendarRight}; +use rkyv::vec::ArchivedVec; +use types::{ + acl::{Acl, AclGrant, ArchivedAclGrant}, + collection::Collection, +}; +use utils::map::bitmap::Bitmap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShareViolation { + TooManyShares(usize), + OwnerShared, + RightNotHeld, +} + +pub trait AclGrantLookup { + fn grants_of(&self, account_id: u32) -> Bitmap; +} + +impl AclGrantLookup for [AclGrant] { + fn grants_of(&self, account_id: u32) -> Bitmap { + self.iter() + .find(|grant| grant.account_id == account_id) + .map_or_else(Bitmap::new, |grant| grant.grants) + } +} + +impl AclGrantLookup for ArchivedVec { + fn grants_of(&self, account_id: u32) -> Bitmap { + self.iter() + .find(|grant| u32::from(grant.account_id) == account_id) + .map_or_else(Bitmap::new, |grant| Bitmap::from(&grant.grants)) + } +} + +pub struct ShareUpdate<'x, G: AclGrantLookup + ?Sized> { + pub collection: Collection, + pub owner_id: u32, + pub actor: Option>, + pub current: &'x G, + pub max_shares: usize, +} + +impl ShareUpdate<'_, G> { + pub fn validate(&self, grants: &[AclGrant]) -> Result<(), ShareViolation> { + if grants.len() > self.max_shares { + return Err(ShareViolation::TooManyShares(self.max_shares)); + } + if grants.iter().any(|grant| grant.account_id == self.owner_id) { + return Err(ShareViolation::OwnerShared); + } + if let Some(actor) = self.actor { + let actor = actor.with_implied_rights(self.collection).into_inner(); + if grants.iter().any(|grant| { + let current = self + .current + .grants_of(grant.account_id) + .with_implied_rights(self.collection) + .into_inner(); + grant.grants.into_inner() & !current & !actor != 0 + }) { + return Err(ShareViolation::RightNotHeld); + } + } + Ok(()) + } +} + +pub trait ImpliedRights { + fn with_implied_rights(self, collection: Collection) -> Self; +} + +impl ImpliedRights for Bitmap { + fn with_implied_rights(mut self, collection: Collection) -> Self { + if collection == Collection::Calendar { + for right in CalendarRight::all_rights() { + let implied_by = right.implied_by_acl(); + if !implied_by.is_empty() && implied_by.iter().all(|acl| self.contains(*acl)) { + self.insert_many(right.to_acl().iter().copied()); + } + } + } + self + } +} + diff --git a/crates/common/src/sharing/mod.rs b/crates/common/src/sharing/mod.rs index 5f14e145..ec6d9ac1 100644 --- a/crates/common/src/sharing/mod.rs +++ b/crates/common/src/sharing/mod.rs @@ -9,7 +9,8 @@ use rkyv::vec::ArchivedVec; use types::acl::{Acl, AclGrant, ArchivedAclGrant}; use utils::map::bitmap::Bitmap; -pub mod acl; +pub mod file; +pub mod grants; pub mod notification; pub mod resources; diff --git a/crates/common/src/sharing/notification.rs b/crates/common/src/sharing/notification.rs index 78e0a806..e820e60c 100644 --- a/crates/common/src/sharing/notification.rs +++ b/crates/common/src/sharing/notification.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use store::{Deserialize, SerializeInfallible, U32_LEN, U64_LEN, write::key::KeySerializer}; +use store::{ + Deserialize, SerializeInfallible, U32_LEN, U64_LEN, + write::{Patch, PatchSource, PendingId, SizedSetValue, key::KeySerializer}, +}; use types::{acl::Acl, collection::Collection}; use utils::map::bitmap::Bitmap; @@ -41,6 +44,25 @@ impl Deserialize for ShareNotification { } impl ShareNotification { + const OBJECT_ID_OFFSET: usize = U32_LEN; + + pub fn into_value(mut self, object_id: PendingId) -> SizedSetValue { + self.object_id = object_id.assigned().unwrap_or_default(); + let payload = self.serialize(); + + match object_id { + PendingId::Assigned(_) => payload.into(), + PendingId::Slot(slot) => ( + payload, + vec![Patch { + offset: Self::OBJECT_ID_OFFSET as u32, + source: PatchSource::SlotBeU32(slot), + }], + ) + .into(), + } + } + fn deserialize_from_slice(bytes: &[u8]) -> Option { Some(Self { object_account_id: bytes @@ -48,7 +70,7 @@ impl ShareNotification { .and_then(|b| b.try_into().ok()) .map(u32::from_be_bytes)?, object_id: bytes - .get(U32_LEN..U32_LEN * 2) + .get(Self::OBJECT_ID_OFFSET..Self::OBJECT_ID_OFFSET + U32_LEN) .and_then(|b| b.try_into().ok()) .map(u32::from_be_bytes)?, object_type: bytes.get(U32_LEN * 2).copied().map(Collection::from)?, diff --git a/crates/common/src/sharing/resources.rs b/crates/common/src/sharing/resources.rs index c12fc56c..2ca08107 100644 --- a/crates/common/src/sharing/resources.rs +++ b/crates/common/src/sharing/resources.rs @@ -4,12 +4,29 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{DavResources, auth::AccessToken}; +use crate::{ + DavResourcePath, GroupwareResourceRef, GroupwareResources, auth::AccessToken, + storage::dav::CachedUid, +}; use store::roaring::RoaringBitmap; -use types::acl::Acl; +use types::acl::{Acl, AclGrant}; use utils::map::bitmap::Bitmap; -impl DavResources { +fn grant_matches( + access_token: &AccessToken, + acl: &AclGrant, + check_acls: &Bitmap, + match_any: bool, +) -> bool { + if !access_token.is_member(acl.account_id) { + return false; + } + let mut grants = acl.grants; + grants.intersection(check_acls); + grants == *check_acls || (match_any && !grants.is_empty()) +} + +impl GroupwareResources { pub fn shared_containers( &self, access_token: &AccessToken, @@ -19,73 +36,48 @@ impl DavResources { let check_acls = Bitmap::::from_iter(check_acls); let mut document_ids = RoaringBitmap::new(); - for resource in &self.resources { - if let Some(acls) = resource.acls() { - for acl in acls { - if access_token.is_member(acl.account_id) { - let mut grants = acl.grants; - grants.intersection(&check_acls); - if grants == check_acls || (match_any && !grants.is_empty()) { - document_ids.insert(resource.document_id); - } - } - } + for resource in self.resources_with_acls() { + if resource + .acls() + .iter() + .any(|acl| grant_matches(access_token, acl, &check_acls, match_any)) + { + document_ids.insert(resource.document_id()); } } document_ids } + pub fn is_shared_item( + &self, + item: &GroupwareResourceRef<'_>, + access_token: &AccessToken, + check_acls: &Bitmap, + match_any: bool, + ) -> bool { + item.child_names().iter().any(|name| { + self.resources + .find(name.parent_id, true) + .is_some_and(|container| { + container + .acls() + .iter() + .any(|acl| grant_matches(access_token, acl, check_acls, match_any)) + }) + }) + } + pub fn shared_items( &self, access_token: &AccessToken, check_acls: impl IntoIterator, match_any: bool, ) -> RoaringBitmap { - let shared_containers = self.shared_containers(access_token, check_acls, match_any); - - if !shared_containers.is_empty() { - let mut document_ids = RoaringBitmap::new(); - - for resource in &self.resources { - if !resource.is_container() && shared_containers.contains(resource.document_id) { - document_ids.insert(resource.document_id); - } - } - - for path in &self.paths { - if let Some(parent_id) = path.parent_id - && shared_containers.contains(parent_id) - { - document_ids.insert(self.resources[path.resource_idx].document_id); - } - } - - document_ids - } else { - shared_containers + let mut document_ids = RoaringBitmap::new(); + for container_id in &self.shared_containers(access_token, check_acls, match_any) { + document_ids.extend(self.children_ids(container_id)); } - } - - pub fn shared_documents( - &self, - access_token: &AccessToken, - check_acls: impl IntoIterator, - match_any: bool, - ) -> RoaringBitmap { - let shared_containers = self.shared_containers(access_token, check_acls, match_any); - let mut document_ids = shared_containers.clone(); - - if !shared_containers.is_empty() { - for path in &self.paths { - if let Some(parent_id) = path.parent_id - && shared_containers.contains(parent_id) - { - document_ids.insert(self.resources[path.resource_idx].document_id); - } - } - } - document_ids } @@ -95,64 +87,86 @@ impl DavResources { document_id: u32, check_acls: impl Into>, ) -> bool { - let check_acls = check_acls.into(); - - for resource in &self.resources { - if resource.document_id == document_id - && let Some(acls) = resource.acls() - { - for acl in acls { - if access_token.is_member(acl.account_id) { - let mut grants = acl.grants; - grants.intersection(&check_acls); - return !grants.is_empty(); - } - } - break; - } - } - - false + let mut grants = self.container_acl(access_token, document_id); + grants.intersection(&check_acls.into()); + !grants.is_empty() } pub fn container_acl(&self, access_token: &AccessToken, document_id: u32) -> Bitmap { let mut account_acls = Bitmap::::new(); - for resource in &self.resources { - if resource.document_id == document_id - && let Some(acls) = resource.acls() - { - for acl in acls { - if access_token.is_member(acl.account_id) { - account_acls.union(&acl.grants); - } + if let Some(resource) = self.resources.find_any(document_id) { + for acl in resource.acls() { + if access_token.is_member(acl.account_id) { + account_acls.union(&acl.grants); } - break; } } account_acls } - pub fn document_ids(&self, is_container: bool) -> impl Iterator { - self.resources.iter().filter_map(move |resource| { - if resource.is_container() == is_container { - Some(resource.document_id) - } else { - None + pub fn uid_matches(&self, uid: &str) -> RoaringBitmap { + self.resources + .find_by_uid(uid.cached_uid()) + .map(|resource| resource.document_id()) + .collect() + } + + pub fn children_with_uid<'x>( + &'x self, + container_id: u32, + uid: &'x str, + ) -> impl Iterator> + 'x { + self.resources + .find_by_uid(uid.cached_uid()) + .filter(move |resource| { + resource + .child_names() + .iter() + .any(|name| name.parent_id == container_id) + }) + .filter_map(move |resource| self.child_path(resource.document_id(), container_id)) + } + + pub fn event_ids_with_flags(&self, mask: u16) -> RoaringBitmap { + let [document_ids] = self.event_ids_with_flag_masks([mask]); + document_ids + } + + pub fn event_ids_with_flag_masks(&self, masks: [u16; N]) -> [RoaringBitmap; N] { + let mut document_ids = std::array::from_fn(|_| RoaringBitmap::new()); + for resource in self.resources.iter_run(false) { + if let Some(flags) = resource.event_flags() { + let document_id = resource.document_id(); + for (mask, document_ids) in masks.iter().zip(document_ids.iter_mut()) { + if flags & *mask != 0 { + document_ids.insert(document_id); + } + } } - }) + } + document_ids + } + + pub fn document_ids(&self, is_container: bool) -> impl Iterator { + self.resources + .iter_run(is_container) + .filter_map(move |resource| { + if resource.is_container() == is_container { + Some(resource.document_id()) + } else { + None + } + }) } pub fn has_container_id(&self, id: &u32) -> bool { - self.resources - .iter() - .any(|r| r.document_id == *id && r.is_container()) + self.resources.find(*id, true).is_some() } pub fn has_item_id(&self, id: &u32) -> bool { - self.resources - .iter() - .any(|r| r.document_id == *id && !r.is_container()) + self.resources.find(*id, false).is_some() } } + diff --git a/crates/common/src/storage/archive.rs b/crates/common/src/storage/archive.rs index 00b38673..c89703dc 100644 --- a/crates/common/src/storage/archive.rs +++ b/crates/common/src/storage/archive.rs @@ -6,9 +6,9 @@ use crate::Server; use store::{ - Deserialize, IterateParams, U32_LEN, ValueKey, - dispatch::DocumentSet, - write::{AlignedBytes, Archive, ValueClass, key::DeserializeBigEndian}, + Deserialize, IterateParams, Key, U32_LEN, ValueKey, + dispatch::{DocumentSet, POINT_SCAN_GAP, ReadShape, ScanShape}, + write::{AnyKey, Archive, ArchiveBytes, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{collection::Collection, field::Field}; @@ -18,43 +18,148 @@ impl Server { &self, account_id: u32, collection: Collection, + field: Field, documents: &I, mut cb: CB, ) -> trc::Result<()> where I: DocumentSet + Send + Sync, - CB: FnMut(u32, Archive) -> trc::Result + Send + Sync, + CB: FnMut(u32, Archive) -> trc::Result + Send + Sync, { let collection: u8 = collection.into(); + let field: u8 = field.into(); + let property = PropertyRange { + account_id, + collection, + field, + }; + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + as Deserialize>::deserialize(value) + .and_then(|archive| cb(document_id, archive)) + } else { + Ok(true) + } + }; + let store = &self.core.storage.data; - self.core - .storage - .data - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection, - document_id: documents.min(), - class: ValueClass::Property(Field::ARCHIVE.into()), - }, - ValueKey { - account_id, - collection, - document_id: documents.max(), - class: ValueClass::Property(Field::ARCHIVE.into()), - }, - ), - |key, value| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if documents.contains(document_id) { - as Deserialize>::deserialize(value) - .and_then(|archive| cb(document_id, archive)) - } else { - Ok(true) + match documents.scan_shape(store.scan_gap()) { + ScanShape::Range(from_document_id, to_document_id) => { + let Some(to_document_id) = last_member(documents, from_document_id, to_document_id) + else { + return Ok(()); + }; + store + .iterate( + property.range(from_document_id, to_document_id), + &mut collect, + ) + .await + } + ScanShape::Ranges(ranges) => { + store + .iterate_many( + ranges + .into_iter() + .map(|(from_document_id, to_document_id)| { + property.read(from_document_id, to_document_id) + }) + .collect(), + &mut collect, + ) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } + + pub async fn archives_fields( + &self, + account_id: u32, + collection: Collection, + shape: ReadShape, + fields: &[(Field, &S)], + mut cb: CB, + ) -> trc::Result<()> + where + S: DocumentSet, + CB: FnMut(Field, u32, Archive) -> trc::Result + Send + Sync, + { + debug_assert!( + fields.windows(2).all(|pair| match pair { + [(previous, _), (next, _)] => u8::from(*previous) < u8::from(*next), + _ => true, + }), + "archives_fields expects fields in ascending order" + ); + let collection: u8 = collection.into(); + let store = &self.core.storage.data; + let mut ranges = Vec::new(); + for (field, documents) in fields { + let property = PropertyRange { + account_id, + collection, + field: u8::from(*field), + }; + let gap = match shape { + ReadShape::Ranges(gap) => gap, + ReadShape::Keys if documents.len() == 0 => POINT_SCAN_GAP, + ReadShape::Keys => { + ranges.extend( + documents + .iterate() + .map(|document_id| property.read(document_id, document_id)), + ); + continue; + } + }; + match documents.scan_shape(gap) { + ScanShape::Range(from_document_id, to_document_id) => { + ranges.extend( + last_member(*documents, from_document_id, to_document_id) + .map(|to_document_id| property.range(from_document_id, to_document_id)), + ); + } + ScanShape::Ranges(shape) => { + ranges.extend(shape.into_iter().map(|(from_document_id, to_document_id)| { + property.read(from_document_id, to_document_id) + })); + } + } + } + if ranges.is_empty() { + return Ok(()); + } + + let mut pending = fields.iter(); + let mut current = pending.next(); + store + .iterate_many(ranges, |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + let field = key + .get(U32_LEN + 1) + .copied() + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; + while let Some((pending_field, _)) = current + && u8::from(*pending_field) < field + { + current = pending.next(); + } + match current { + Some((pending_field, documents)) + if u8::from(*pending_field) == field && documents.contains(document_id) => + { + as Deserialize>::deserialize(value) + .and_then(|archive| cb(*pending_field, document_id, archive)) } - }, - ) + _ => Ok(true), + } + }) .await .add_context(|err| { err.caused_by(trc::location!()) @@ -63,15 +168,43 @@ impl Server { }) } - pub async fn all_archives( + pub async fn archive_and_field( &self, account_id: u32, collection: Collection, - field: u8, + document_id: u32, + field: Field, + ) -> trc::Result<(Option>, Option>)> { + let mut archive = None; + let mut value = None; + self.archives_fields( + account_id, + collection, + ReadShape::Keys, + &[(field, &document_id), (Field::ARCHIVE, &document_id)], + |read_field, _, read| { + if read_field == Field::ARCHIVE { + archive = Some(read); + } else { + value = Some(read); + } + Ok(true) + }, + ) + .await + .map(|_| (archive, value)) + } + + pub async fn all_values( + &self, + account_id: u32, + collection: Collection, + class: ValueClass, mut cb: CB, ) -> trc::Result<()> where - CB: FnMut(u32, Archive) -> trc::Result<()> + Send + Sync, + T: Deserialize, + CB: FnMut(u32, T) -> trc::Result<()> + Send + Sync, { let collection: u8 = collection.into(); @@ -84,19 +217,18 @@ impl Server { account_id, collection, document_id: 0, - class: ValueClass::Property(field), + class: class.clone(), }, ValueKey { account_id, collection, document_id: u32::MAX, - class: ValueClass::Property(field), + class, }, ), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - let archive = as Deserialize>::deserialize(value)?; - cb(document_id, archive)?; + cb(document_id, T::deserialize(value)?)?; Ok(true) }, @@ -108,4 +240,118 @@ impl Server { .collection(collection) }) } + + pub async fn values_window( + &self, + account_id: u32, + collection: Collection, + class: &ValueClass, + from_document_id: u32, + limit: Option, + mut cb: CB, + ) -> trc::Result> + where + T: Deserialize, + CB: FnMut(u32, T) -> trc::Result<()> + Send + Sync, + { + let collection: u8 = collection.into(); + let mut rows = 0; + let mut last_document_id = from_document_id; + let params = IterateParams::new( + ValueKey { + account_id, + collection, + document_id: from_document_id, + class: class.clone(), + }, + ValueKey { + account_id, + collection, + document_id: u32::MAX, + class: class.clone(), + }, + ); + let params = match limit { + Some(limit) => params.limit(limit), + None => params, + }; + + self.core + .storage + .data + .iterate(params, |key, value| { + last_document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + cb(last_document_id, T::deserialize(value)?)?; + rows += 1; + Ok(limit.is_none_or(|limit| rows < limit)) + }) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + })?; + + Ok(limit + .filter(|limit| rows == *limit) + .and_then(|_| last_document_id.checked_add(1))) + } +} + +struct PropertyRange { + account_id: u32, + collection: u8, + field: u8, +} + +impl PropertyRange { + fn key(&self, document_id: u32) -> Vec { + ValueKey { + account_id: self.account_id, + collection: self.collection, + document_id, + class: ValueClass::Property(self.field), + } + .serialize(0) + } + + fn range(&self, from_document_id: u32, to_document_id: u32) -> IterateParams>> { + let mut end = self.key(to_document_id); + end.push(u8::MAX); + self.params(self.key(from_document_id), end) + } + + fn read(&self, from_document_id: u32, to_document_id: u32) -> IterateParams>> { + if from_document_id == to_document_id { + let key = self.key(from_document_id); + self.params(key.clone(), key) + } else { + self.range(from_document_id, to_document_id) + } + } + + fn params(&self, begin: Vec, end: Vec) -> IterateParams>> { + let subspace = ValueClass::Property(self.field).subspace(self.collection); + IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { subspace, key: end }, + ) + } +} + +pub(super) fn last_member( + documents: &impl DocumentSet, + from_document_id: u32, + to_document_id: u32, +) -> Option { + if documents.contains(to_document_id) { + Some(to_document_id) + } else if to_document_id > from_document_id { + Some(to_document_id - 1) + } else { + None + } } diff --git a/crates/common/src/storage/batch.rs b/crates/common/src/storage/batch.rs new file mode 100644 index 00000000..714dd424 --- /dev/null +++ b/crates/common/src/storage/batch.rs @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Server; +use store::{ + ValueKey, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{collection::Collection, field::Field}; + +pub const MAX_BATCH_DOCUMENTS: usize = 256; +pub const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024; + +pub struct DocumentBatch { + values: Vec<(u32, T)>, + pending: Vec, + bytes: usize, + max_bytes: usize, + round: u32, + state: BatchState, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BatchState { + Requesting, + Partial, + Settled, +} + +#[derive(Debug)] +pub enum Lookup<'x, T> { + Found(&'x T), + Missing, + Skip, +} + +pub struct BatchSink<'x, T> { + values: &'x mut Vec<(u32, T)>, + bytes: &'x mut usize, + max_bytes: usize, +} + +impl DocumentBatch { + pub fn new(max_bytes: usize) -> Self { + DocumentBatch { + values: Vec::new(), + pending: Vec::new(), + bytes: 0, + max_bytes, + round: 0, + state: BatchState::Requesting, + } + } + + pub fn reset(&mut self) { + self.values.clear(); + self.pending.clear(); + self.bytes = 0; + self.round = 0; + self.state = BatchState::Requesting; + } + + pub fn request(&mut self, document_id: u32) { + debug_assert_eq!(self.state, BatchState::Requesting); + self.pending.push(document_id); + } + + pub fn is_first_round(&self) -> bool { + self.round == 0 + } + + pub fn begin_read(&mut self) -> Option<(&Vec, BatchSink<'_, T>)> { + if self.state == BatchState::Requesting { + self.pending.sort_unstable(); + self.pending.dedup(); + } + self.values.clear(); + self.bytes = 0; + if self.pending.is_empty() { + self.state = BatchState::Settled; + return None; + } + self.state = BatchState::Partial; + Some(( + &self.pending, + BatchSink { + values: &mut self.values, + bytes: &mut self.bytes, + max_bytes: self.max_bytes, + }, + )) + } + + pub fn end_read(&mut self) { + if !self + .values + .is_sorted_by_key(|(document_id, _)| *document_id) + { + self.values + .sort_unstable_by_key(|(document_id, _)| *document_id); + } + if self.bytes < self.max_bytes || self.values.len() >= self.pending.len() { + self.state = BatchState::Settled; + } + } + + pub fn lookup(&self, document_id: u32) -> Lookup<'_, T> { + if self.pending.binary_search(&document_id).is_err() { + Lookup::Skip + } else if let Some(value) = self.get(document_id) { + Lookup::Found(value) + } else if self.state == BatchState::Settled { + Lookup::Missing + } else { + Lookup::Skip + } + } + + pub fn get(&self, document_id: u32) -> Option<&T> { + self.values + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| self.values.get(idx)) + .map(|(_, value)| value) + } + + pub fn next_round(&mut self) -> bool { + match self.state { + BatchState::Requesting => true, + BatchState::Settled => false, + BatchState::Partial => { + let values = &self.values; + self.pending.retain(|document_id| { + values + .binary_search_by_key(document_id, |(id, _)| *id) + .is_err() + }); + self.values.clear(); + self.bytes = 0; + self.round += 1; + true + } + } + } +} + +impl BatchSink<'_, T> { + pub fn push(&mut self, document_id: u32, value: T, bytes: usize) -> bool { + self.values.push((document_id, value)); + *self.bytes = self.bytes.saturating_add(bytes); + *self.bytes < self.max_bytes + } +} + +impl Server { + pub async fn read_archive_batch( + &self, + account_id: u32, + collection: Collection, + field: Field, + batch: &mut DocumentBatch>, + ) -> trc::Result<()> { + let Some((pending, mut sink)) = batch.begin_read() else { + return Ok(()); + }; + if let &[document_id] = pending.as_slice() { + if let Some(archive) = self + .store() + .get_value::>(ValueKey::property( + account_id, + collection, + document_id, + field, + )) + .await + .caused_by(trc::location!())? + { + let bytes = archive.as_bytes().len(); + sink.push(document_id, archive, bytes); + } + } else { + self.archives( + account_id, + collection, + field, + pending, + |document_id, archive| { + let bytes = archive.as_bytes().len(); + Ok(sink.push(document_id, archive, bytes)) + }, + ) + .await + .caused_by(trc::location!())?; + } + batch.end_read(); + Ok(()) + } +} + diff --git a/crates/common/src/storage/blob.rs b/crates/common/src/storage/blob.rs index ca957ffe..6a1b2782 100644 --- a/crates/common/src/storage/blob.rs +++ b/crates/common/src/storage/blob.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{KV_QUOTA_BLOB, Server}; -use mail_parser::{ - Encoding, - decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, +use crate::{ + KV_QUOTA_BLOB, Server, + network::memory::{MEMORY_STEP, MemoryClass, MemoryReservation}, }; +use bytes::Bytes; +use mail_parser::Encoding; +use registry::schema::enums::CompressionAlgo; +use std::{borrow::Cow, ops::Deref, ops::Range}; use store::{ U32_LEN, U64_LEN, dispatch::lookup::KeyValue, - write::{BatchBuilder, BlobLink, BlobOp, now}, + write::{BatchBuilder, BlobLink, BlobOp, blob::BlobLinkValue, now}, }; use trc::AddContext; use types::{ @@ -24,11 +27,141 @@ const COUNT_BYTES: u32 = 20; const COUNT_SHIFT: u32 = 64 - COUNT_BYTES; const SIZE_MASK: u64 = (1u64 << COUNT_SHIFT) - 1; +const READ_COPY_FACTOR: u64 = 2; + +pub struct FetchedBlob { + bytes: Vec, + reservation: MemoryReservation, +} + +impl FetchedBlob { + fn fit(mut bytes: Vec, mut reservation: MemoryReservation) -> trc::Result { + if bytes.capacity() - bytes.len() > MEMORY_STEP as usize { + bytes.shrink_to_fit(); + } + let capacity = bytes.capacity() as u64; + reservation + .try_reserve_to(capacity) + .caused_by(trc::location!())?; + reservation.shrink_to(capacity); + Ok(FetchedBlob { bytes, reservation }) + } + + pub fn try_reserve_extra(&mut self, extra: usize) -> trc::Result<()> { + self.reservation + .try_reserve_to(self.bytes.capacity().saturating_add(extra) as u64) + .caused_by(trc::location!()) + } + + pub fn replace_prefix(&mut self, prefix_len: usize, replacement: &[u8]) -> trc::Result<()> { + let len = replacement.len() + self.bytes.len().saturating_sub(prefix_len); + self.try_reserve_extra(len)?; + let mut bytes = Vec::with_capacity(len); + bytes.extend_from_slice(replacement); + bytes.extend_from_slice(self.bytes.get(prefix_len..).unwrap_or_default()); + self.bytes = bytes; + self.reservation.shrink_to(self.bytes.capacity() as u64); + Ok(()) + } + + pub fn into_parts(self) -> (Vec, MemoryReservation) { + (self.bytes, self.reservation) + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } +} + +impl Deref for FetchedBlob { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.bytes + } +} + +impl From for Bytes { + fn from(blob: FetchedBlob) -> Self { + Bytes::from_owner(blob) + } +} + +impl AsRef<[u8]> for FetchedBlob { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + pub struct BlobQuotaStatus { pub allowed: bool, pub expires_in: u64, } +pub struct BlobHold { + hash: BlobHash, + account_id: u32, + until: u64, + uploaded: bool, +} + +impl BlobHold { + pub fn release(self, batch: &mut BatchBuilder) -> &mut BatchBuilder { + batch.with_account_id(self.account_id).clear(BlobOp::Link { + hash: self.hash.clone(), + to: BlobLink::Temporary { until: self.until }, + }); + self.commit(batch) + } + + pub fn commit(self, batch: &mut BatchBuilder) -> &mut BatchBuilder { + if self.uploaded { + batch.set(BlobOp::Commit { hash: self.hash }, Vec::new()) + } else { + batch + } + } +} + +pub trait SectionDecode { + fn fetch_range(&self) -> Range; + + fn decode_fetched(&self, fetched: Vec) -> Option>; +} + +impl SectionDecode for BlobSection { + fn fetch_range(&self) -> Range { + self.outermost().range() + } + + fn decode_fetched(&self, fetched: Vec) -> Option> { + let Some((outermost, containers)) = self.containers().split_first() else { + return Some( + match Encoding::from(self.part().encoding).decode(&fetched) { + Cow::Owned(decoded) => decoded, + Cow::Borrowed(decoded) if decoded.len() != fetched.len() => decoded.to_vec(), + Cow::Borrowed(_) => fetched, + }, + ); + }; + let mut buffer = Vec::new(); + Encoding::from(outermost.encoding).decode_append(&fetched, &mut buffer); + drop(fetched); + for container in containers { + let mut decoded = Vec::new(); + Encoding::from(container.encoding) + .decode_append(buffer.get(container.range())?, &mut decoded); + buffer = decoded; + } + let part = self.part(); + Some( + Encoding::from(part.encoding) + .decode(buffer.get(part.range())?) + .into_owned(), + ) + } +} + impl Server { pub async fn blob_has_quota( &self, @@ -48,9 +181,8 @@ impl Server { bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); self.in_memory_store() - .counter_incr( + .key_incr( KeyValue::new(bucket, 1i64 << COUNT_SHIFT | bytes as i64).expires(expires_in), - true, ) .await .caused_by(trc::location!()) @@ -89,13 +221,13 @@ impl Server { hash: hash.clone(), to: BlobLink::Temporary { until }, }, - vec![], + BlobLinkValue::new(data.len()), ); self.core .storage .data - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; @@ -121,7 +253,7 @@ impl Server { self.core .storage .data - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; } @@ -142,8 +274,35 @@ impl Server { data: &[u8], hold_for: u64, ) -> trc::Result<(BlobHash, BlobOp)> { + let (hash, hold) = self + .hold_blob(account_id, BlobHash::generate(data), data, hold_for) + .await?; + let link = BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until: hold.until }, + }; + let mut batch = BatchBuilder::new(); + hold.commit(&mut batch); + if !batch.is_empty() { + self.core + .storage + .data + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + } + + Ok((hash, link)) + } + + pub async fn hold_blob( + &self, + account_id: u32, + hash: BlobHash, + data: &[u8], + hold_for: u64, + ) -> trc::Result<(BlobHash, BlobHold)> { // First reserve the hash - let hash = BlobHash::generate(data); let mut batch = BatchBuilder::new(); let until = now() + hold_for; @@ -152,68 +311,115 @@ impl Server { hash: hash.clone(), to: BlobLink::Temporary { until }, }, - vec![], + BlobLinkValue::new(data.len()), ); self.core .storage .data - .write(batch.build_all()) + .write_batch(&mut batch) .await .caused_by(trc::location!())?; - if !self + let uploaded = !self .core .storage .data .blob_exists(&hash) .await - .caused_by(trc::location!())? - { - // Upload blob to store + .caused_by(trc::location!())?; + if uploaded { self.core .storage .blob .put_blob(hash.as_ref(), data, self.core.email.compression) .await .caused_by(trc::location!())?; - - // Commit blob - let mut batch = BatchBuilder::new(); - batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); - self.core - .storage - .data - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; } Ok(( hash.clone(), - BlobOp::Link { + BlobHold { hash, - to: BlobLink::Temporary { until }, + account_id, + until, + uploaded, }, )) } - pub async fn get_blob_section( + pub fn memory_reservation(&self, class: MemoryClass) -> MemoryReservation { + self.inner + .data + .memory_budget + .reservation(class, self.core.network.max_buffered_memory) + } + + pub fn is_memory_available(&self, class: MemoryClass) -> bool { + self.inner + .data + .memory_budget + .is_accepting(class, self.core.network.max_buffered_memory) + } + + pub async fn fetch_blob( + &self, + hash: &BlobHash, + range: Range, + size_hint: Option, + ) -> trc::Result> { + let factor = if range.start == 0 && self.core.email.compression == CompressionAlgo::None { + 1 + } else { + READ_COPY_FACTOR + }; + let reservation = self.read_reservation(size_hint, factor)?; + match self + .blob_store() + .get_blob(hash.as_slice(), range) + .await + .caused_by(trc::location!())? + { + Some(bytes) => FetchedBlob::fit(bytes, reservation).map(Some), + None => Ok(None), + } + } + + pub async fn fetch_blob_section( &self, hash: &BlobHash, section: &BlobSection, - ) -> trc::Result>> { - Ok(self + size_hint: Option, + ) -> trc::Result> { + let reservation = self.read_reservation(size_hint, READ_COPY_FACTOR)?; + match self .blob_store() - .get_blob( - hash.as_slice(), - (section.offset_start)..(section.offset_start.saturating_add(section.size)), - ) - .await? - .and_then(|bytes| match Encoding::from(section.encoding) { - Encoding::None => Some(bytes), - Encoding::Base64 => base64_decode(&bytes), - Encoding::QuotedPrintable => quoted_printable_decode(&bytes), - })) + .get_blob(hash.as_slice(), section.fetch_range()) + .await + .caused_by(trc::location!())? + .and_then(|bytes| section.decode_fetched(bytes)) + { + Some(bytes) => FetchedBlob::fit(bytes, reservation).map(Some), + None => Ok(None), + } + } + + pub fn account_blob(&self, bytes: Vec) -> trc::Result { + FetchedBlob::fit(bytes, self.memory_reservation(MemoryClass::Read)) + } + + fn read_reservation( + &self, + size_hint: Option, + factor: u64, + ) -> trc::Result { + let mut reservation = self.memory_reservation(MemoryClass::Read); + if let Some(size) = size_hint { + reservation + .try_reserve_to(size.saturating_mul(factor)) + .caused_by(trc::location!())?; + } + Ok(reservation) } } + diff --git a/crates/common/src/storage/chunk.rs b/crates/common/src/storage/chunk.rs new file mode 100644 index 00000000..802d2164 --- /dev/null +++ b/crates/common/src/storage/chunk.rs @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Server; +use store::{ + ValueKey, + dispatch::DocumentSet, + write::{Archive, ArchiveBytes}, +}; +use trc::AddContext; +use types::{collection::Collection, field::Field}; + +#[derive(Default)] +pub struct ArchiveChunk { + archives: Vec<(u32, Option>)>, + values: Vec<(u32, Option>)>, +} + +struct SortedIds<'x>(&'x [u32]); + +impl Server { + pub async fn archive_chunk( + &self, + account_id: u32, + collection: Collection, + document_ids: &[u32], + value: Option<(Field, &[u32])>, + ) -> trc::Result { + debug_assert!(document_ids.is_sorted_by(|a, b| a < b)); + debug_assert!(value.is_none_or(|(_, ids)| ids.is_sorted_by(|a, b| a < b))); + + let value = value.filter(|(_, ids)| !ids.is_empty()); + let mut chunk = ArchiveChunk { + archives: Vec::with_capacity(document_ids.len()), + values: Vec::with_capacity(value.map_or(0, |(_, ids)| ids.len())), + }; + match (document_ids, value) { + ([], None) => {} + ([document_id], None) => { + let archive = self + .store() + .get_value::>(ValueKey::archive( + account_id, + collection, + *document_id, + )) + .await + .caused_by(trc::location!())?; + chunk.archives.push((*document_id, archive)); + } + ([], Some((field, [document_id]))) => { + let value = self + .store() + .get_value::>(ValueKey::property( + account_id, + collection, + *document_id, + field, + )) + .await + .caused_by(trc::location!())?; + chunk.values.push((*document_id, value)); + } + (document_ids, value) => { + let shape = self.core.storage.data.read_shape(); + let archive_ids = SortedIds(document_ids); + let ArchiveChunk { archives, values } = &mut chunk; + let collect = |field: Field, document_id: u32, archive| { + if field == Field::ARCHIVE { + archives.push((document_id, Some(archive))); + } else { + values.push((document_id, Some(archive))); + } + Ok(true) + }; + match value { + Some((field, value_ids)) => { + self.archives_fields( + account_id, + collection, + shape, + &[ + (field, &SortedIds(value_ids)), + (Field::ARCHIVE, &archive_ids), + ], + collect, + ) + .await + } + None => { + self.archives_fields( + account_id, + collection, + shape, + &[(Field::ARCHIVE, &archive_ids)], + collect, + ) + .await + } + } + .caused_by(trc::location!())?; + } + } + Ok(chunk) + } +} + +impl ArchiveChunk { + pub fn take_archive(&mut self, document_id: u32) -> Option> { + take(&mut self.archives, document_id) + } + + pub fn take_value(&mut self, document_id: u32) -> Option> { + take(&mut self.values, document_id) + } + + pub fn value(&self, document_id: u32) -> Option<&Archive> { + self.values + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| self.values.get(idx)) + .and_then(|(_, archive)| archive.as_ref()) + } +} + +fn take( + entries: &mut [(u32, Option>)], + document_id: u32, +) -> Option> { + entries + .binary_search_by_key(&document_id, |(id, _)| *id) + .ok() + .and_then(|idx| entries.get_mut(idx)) + .and_then(|(_, archive)| archive.take()) +} + +impl DocumentSet for SortedIds<'_> { + fn min(&self) -> u32 { + self.0.first().copied().unwrap_or(0) + } + + fn max(&self) -> u32 { + self.0.last().map_or(0, |id| id.saturating_add(1)) + } + + fn contains(&self, id: u32) -> bool { + self.0.binary_search(&id).is_ok() + } + + fn len(&self) -> usize { + self.0.len() + } + + fn iterate(&self) -> impl Iterator { + self.0.iter().copied() + } +} diff --git a/crates/common/src/storage/dav.rs b/crates/common/src/storage/dav.rs deleted file mode 100644 index c36eeed5..00000000 --- a/crates/common/src/storage/dav.rs +++ /dev/null @@ -1,403 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{ - DavName, DavPath, DavResource, DavResourceMetadata, DavResourcePath, DavResources, - TinyCalendarPreferences, -}; -use std::hash::{Hash, Hasher}; -use store::rand::{RngExt, distr::Alphanumeric}; -use types::acl::AclGrant; - -const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1; - -impl DavResourcePath<'_> { - #[inline(always)] - pub fn document_id(&self) -> u32 { - self.resource.document_id - } - - #[inline(always)] - pub fn parent_id(&self) -> Option { - self.path.parent_id - } - - #[inline(always)] - pub fn path(&self) -> &str { - self.path.path.as_str() - } - - #[inline(always)] - pub fn is_container(&self) -> bool { - self.resource.is_container() - } - - #[inline(always)] - pub fn hierarchy_seq(&self) -> u32 { - self.path.hierarchy_seq - } - - #[inline(always)] - pub fn size(&self) -> u32 { - self.resource.size().unwrap_or_default() - } -} - -impl DavResources { - pub fn by_path(&self, name: &str) -> Option> { - self.paths.get(name).map(|path| DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - } - - pub fn container_resource_by_id(&self, id: u32) -> Option<&DavResource> { - self.resources - .iter() - .find(|res| res.document_id == id && res.is_container()) - } - - pub fn container_resource_path_by_id(&self, id: u32) -> Option> { - self.resources - .iter() - .enumerate() - .find(|(_, resource)| resource.document_id == id && resource.is_container()) - .and_then(|(idx, resource)| { - self.paths - .iter() - .find(|path| path.resource_idx == idx) - .map(|path| DavResourcePath { path, resource }) - }) - } - - pub fn any_resource_path_by_id(&self, id: u32) -> Option> { - self.resources - .iter() - .enumerate() - .find(|(_, resource)| resource.document_id == id) - .and_then(|(idx, resource)| { - self.paths - .iter() - .find(|path| path.resource_idx == idx) - .map(|path| DavResourcePath { path, resource }) - }) - } - - pub fn subtree(&self, search_path: &str) -> impl Iterator> { - let prefix = format!("{search_path}/"); - self.paths.iter().filter_map(move |path| { - if path.path.starts_with(&prefix) || path.path == search_path { - Some(DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - } else { - None - } - }) - } - - pub fn subtree_with_depth( - &self, - search_path: &str, - depth: usize, - ) -> impl Iterator> { - let prefix = format!("{search_path}/"); - self.paths.iter().filter_map(move |path| { - if path - .path - .strip_prefix(&prefix) - .is_some_and(|name| name.as_bytes().iter().filter(|&&c| c == b'/').count() < depth) - || path.path.as_str() == search_path - { - Some(DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - } else { - None - } - }) - } - - pub fn tree_with_depth(&self, depth: usize) -> impl Iterator> { - self.paths.iter().filter_map(move |path| { - if path.path.as_bytes().iter().filter(|&&c| c == b'/').count() <= depth { - Some(DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - } else { - None - } - }) - } - - pub fn children(&self, parent_id: u32) -> impl Iterator> { - self.paths - .iter() - .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id)) - .map(|path| DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - } - - pub fn children_ids(&self, parent_id: u32) -> impl Iterator { - self.paths - .iter() - .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id)) - .map(|path| self.resources[path.resource_idx].document_id) - } - - pub fn format_resource(&self, resource: DavResourcePath<'_>) -> String { - if resource.resource.is_container() { - format!("{}{}/", self.base_path, resource.path.path) - } else { - format!("{}{}", self.base_path, resource.path.path) - } - } - - pub fn format_resource_paths_by_id( - &self, - document_id: u32, - ) -> impl Iterator + '_ { - self.paths - .iter() - .filter(move |path| self.resources[path.resource_idx].document_id == document_id) - .map(move |path| { - self.format_resource(DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - }) - } - - pub fn format_resource_path_by_parent( - &self, - document_id: u32, - parent_id: u32, - ) -> Option { - self.paths - .iter() - .find(|path| { - self.resources[path.resource_idx].document_id == document_id - && path.parent_id == Some(parent_id) - }) - .map(|path| { - self.format_resource(DavResourcePath { - path, - resource: &self.resources[path.resource_idx], - }) - }) - } - - pub fn format_collection(&self, name: &str) -> String { - format!("{}{name}/", self.base_path) - } - - pub fn format_item(&self, name: &str) -> String { - format!("{}{}", self.base_path, name) - } -} - -impl DavResource { - pub fn is_child_of(&self, parent_id: u32) -> bool { - match &self.data { - DavResourceMetadata::File { parent_id: id, .. } => id.is_some_and(|id| id == parent_id), - DavResourceMetadata::CalendarEvent { names, .. } => { - names.iter().any(|name| name.parent_id == parent_id) - } - DavResourceMetadata::ContactCard { names } => { - names.iter().any(|name| name.parent_id == parent_id) - } - DavResourceMetadata::CalendarEventNotification { names } => { - names.is_empty() && parent_id == SCHEDULE_INBOX_ID - } - _ => false, - } - } - - pub fn parent_id(&self) -> Option { - match &self.data { - DavResourceMetadata::File { parent_id, .. } => *parent_id, - DavResourceMetadata::CalendarEvent { names, .. } => { - names.first().map(|name| name.parent_id) - } - DavResourceMetadata::ContactCard { names } => names.first().map(|name| name.parent_id), - DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => { - Some(SCHEDULE_INBOX_ID) - } - _ => None, - } - } - - pub fn child_names(&self) -> Option<&[DavName]> { - match &self.data { - DavResourceMetadata::CalendarEvent { names, .. } => Some(names.as_slice()), - DavResourceMetadata::ContactCard { names } => Some(names.as_slice()), - DavResourceMetadata::CalendarEventNotification { names } if !names.is_empty() => { - Some(names.as_slice()) - } - _ => None, - } - } - - pub fn container_name(&self) -> Option<&str> { - match &self.data { - DavResourceMetadata::File { name, .. } => Some(name.as_str()), - DavResourceMetadata::Calendar { name, .. } => Some(name.as_str()), - DavResourceMetadata::AddressBook { name, .. } => Some(name.as_str()), - DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => { - Some(if self.document_id == SCHEDULE_INBOX_ID { - "inbox" - } else { - "outbox" - }) - } - _ => None, - } - } - - pub fn has_hierarchy_changes(&self, other: &DavResource) -> bool { - match (&self.data, &other.data) { - ( - DavResourceMetadata::File { - name: a, - parent_id: c, - .. - }, - DavResourceMetadata::File { - name: b, - parent_id: d, - .. - }, - ) => a != b || c != d, - ( - DavResourceMetadata::Calendar { name: a, .. }, - DavResourceMetadata::Calendar { name: b, .. }, - ) => a != b, - ( - DavResourceMetadata::AddressBook { name: a, .. }, - DavResourceMetadata::AddressBook { name: b, .. }, - ) => a != b, - ( - DavResourceMetadata::CalendarEvent { names: a, .. }, - DavResourceMetadata::CalendarEvent { names: b, .. }, - ) => a != b, - ( - DavResourceMetadata::ContactCard { names: a, .. }, - DavResourceMetadata::ContactCard { names: b, .. }, - ) => a != b, - ( - DavResourceMetadata::CalendarEventNotification { names: a, .. }, - DavResourceMetadata::CalendarEventNotification { names: b, .. }, - ) => a != b, - _ => unreachable!(), - } - } - - pub fn event_time_range(&self) -> Option<(i64, i64)> { - match &self.data { - DavResourceMetadata::CalendarEvent { - start, duration, .. - } => Some((*start, *start + *duration as i64)), - _ => None, - } - } - - pub fn calendar_preferences(&self, account_id: u32) -> Option<&TinyCalendarPreferences> { - match &self.data { - DavResourceMetadata::Calendar { preferences, .. } => preferences - .iter() - .find(|pref| pref.account_id == account_id) - .or_else(|| preferences.first()), - _ => None, - } - } - - pub fn is_container(&self) -> bool { - match &self.data { - DavResourceMetadata::File { size, .. } => size.is_none(), - DavResourceMetadata::Calendar { .. } | DavResourceMetadata::AddressBook { .. } => true, - DavResourceMetadata::CalendarEventNotification { names } => names.is_empty(), - _ => false, - } - } - - pub fn size(&self) -> Option { - match &self.data { - DavResourceMetadata::File { size, .. } => *size, - _ => None, - } - } - - pub fn acls(&self) -> Option<&[AclGrant]> { - match &self.data { - DavResourceMetadata::File { acls, .. } => Some(acls.as_slice()), - DavResourceMetadata::Calendar { acls, .. } => Some(acls.as_slice()), - DavResourceMetadata::AddressBook { acls, .. } => Some(acls.as_slice()), - _ => None, - } - } -} - -impl Hash for DavPath { - fn hash(&self, state: &mut H) { - self.path.hash(state); - } -} - -impl PartialEq for DavPath { - fn eq(&self, other: &Self) -> bool { - self.path == other.path - } -} - -impl Eq for DavPath {} - -impl std::borrow::Borrow for DavPath { - fn borrow(&self) -> &str { - &self.path - } -} - -impl std::hash::Hash for DavResource { - fn hash(&self, state: &mut H) { - self.document_id.hash(state); - } -} - -impl PartialEq for DavResource { - fn eq(&self, other: &Self) -> bool { - self.document_id == other.document_id - } -} - -impl Eq for DavResource {} - -impl std::borrow::Borrow for DavResource { - fn borrow(&self) -> &u32 { - &self.document_id - } -} - -impl DavName { - pub fn new(name: String, parent_id: u32) -> Self { - Self { name, parent_id } - } - - pub fn new_with_rand_name(parent_id: u32) -> Self { - Self { - name: store::rand::rng() - .sample_iter(Alphanumeric) - .take(10) - .map(char::from) - .collect::(), - parent_id, - } - } -} diff --git a/crates/common/src/storage/dav/file.rs b/crates/common/src/storage/dav/file.rs new file mode 100644 index 00000000..7784706f --- /dev/null +++ b/crates/common/src/storage/dav/file.rs @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{FilePresence, ResourceChunkBuilder}; +use crate::{ArenaRef, FileFlags, GroupwareResourceMetadata, GroupwareResourceRef, ResourceChunk}; +use types::{media_type::MediaTypeId, metadata::MetadataKinds}; + +const KIND_MASK: u32 = 0b11; +pub const FILE_KIND_DIRECTORY: u8 = 0; +pub const FILE_KIND_FILE: u8 = 1; +pub const FILE_KIND_SYMLINK: u8 = 2; +const EXECUTABLE: u32 = 1 << 2; +const ROLE_SHIFT: u32 = 3; +const ROLE_MASK: u32 = 0xF; +const MEDIA_SHIFT: u32 = 7; +const MEDIA_MASK: u32 = (1 << MediaTypeId::BITS) - 1; +const EXTRA_SHIFT: u32 = 19; +const EXTRA_MASK: u32 = 0xFF; +const PRESENCE_SHIFT: u32 = EXTRA_SHIFT + 8; +const PRESENCE_MASK: u32 = FilePresence::MASK as u32; +const MEDIA_TYPE_EXACT_SHIFT: u32 = PRESENCE_SHIFT + PRESENCE_MASK.count_ones(); +const MEDIA_TYPE_EXACT: u32 = 1 << MEDIA_TYPE_EXACT_SHIFT; +const RESERVED_SHIFT: u32 = MEDIA_TYPE_EXACT_SHIFT + 1; + +pub const MAX_FILE_EXTRA_LEN: usize = EXTRA_MASK as usize; +pub const FILE_DEFAULT_MEDIA_TYPE: &str = "application/octet-stream"; + +impl FileFlags { + pub fn new( + kind: u8, + executable: bool, + role: u8, + media_type: MediaTypeId, + extra_len: u16, + ) -> Self { + FileFlags( + (kind as u32 & KIND_MASK) + | if executable { EXECUTABLE } else { 0 } + | (role as u32 & ROLE_MASK) << ROLE_SHIFT + | (media_type.raw() as u32 & MEDIA_MASK) << MEDIA_SHIFT + | (extra_len as u32 & EXTRA_MASK) << EXTRA_SHIFT, + ) + } + + #[inline(always)] + pub fn kind(self) -> u8 { + (self.0 & KIND_MASK) as u8 + } + + #[inline(always)] + pub fn is_directory(self) -> bool { + self.kind() == FILE_KIND_DIRECTORY + } + + #[inline(always)] + pub fn is_executable(self) -> bool { + self.0 & EXECUTABLE != 0 + } + + #[inline(always)] + pub fn role(self) -> u8 { + ((self.0 >> ROLE_SHIFT) & ROLE_MASK) as u8 + } + + #[inline(always)] + pub fn media_type_id(self) -> MediaTypeId { + MediaTypeId::from_raw(((self.0 >> MEDIA_SHIFT) & MEDIA_MASK) as u16) + } + + #[inline(always)] + pub fn extra_len(self) -> usize { + ((self.0 >> EXTRA_SHIFT) & EXTRA_MASK) as usize + } + + pub fn with_presence(self, presence: FilePresence) -> Self { + FileFlags( + (self.0 & !(PRESENCE_MASK << PRESENCE_SHIFT)) + | (presence.bits() as u32 & PRESENCE_MASK) << PRESENCE_SHIFT, + ) + } + + #[inline(always)] + pub fn presence(self) -> FilePresence { + FilePresence::from_bits(((self.0 >> PRESENCE_SHIFT) & PRESENCE_MASK) as u16) + } + + #[inline(always)] + pub fn metadata_kinds(self) -> MetadataKinds { + self.presence().kinds() + } + + #[inline(always)] + pub fn has_dav_display_name(self) -> bool { + self.presence().has_dav_display_name() + } + + pub fn with_exact_media_type(self) -> Self { + FileFlags(self.0 | MEDIA_TYPE_EXACT) + } + + #[inline(always)] + pub fn has_exact_media_type(self) -> bool { + self.0 & MEDIA_TYPE_EXACT != 0 + } + + pub fn is_valid(self) -> bool { + self.kind() <= FILE_KIND_SYMLINK + && self.0 >> RESERVED_SHIFT == 0 + && (self.extra_len() == 0 || self.media_type_id().is_uncatalogued()) + } +} + +impl ResourceChunkBuilder { + pub fn push_file_name( + &mut self, + name: &str, + media_type: Option<&str>, + ) -> (ArenaRef, MediaTypeId, u16) { + let media_id = media_type.map_or(MediaTypeId::NONE, MediaTypeId::lookup); + let name_ref = self.push_str(name); + if !media_id.is_uncatalogued() { + return (name_ref, media_id, 0); + } + match media_type.filter(|media_type| media_type.len() <= MAX_FILE_EXTRA_LEN) { + Some(media_type) => { + self.bytes.extend_from_slice(media_type.as_bytes()); + (name_ref, media_id, media_type.len() as u16) + } + None => (name_ref, MediaTypeId::NONE, 0), + } + } + + pub(super) fn push_file_bytes( + &mut self, + chunk: &ResourceChunk, + name: ArenaRef, + flags: FileFlags, + ) -> ArenaRef { + let off = self.bytes.len() as u32; + if let Some(bytes) = chunk + .bytes + .get(name.off as usize..(name.off as usize + name.len as usize + flags.extra_len())) + { + self.bytes.extend_from_slice(bytes); + } + ArenaRef { off, len: name.len } + } +} + +impl<'x> GroupwareResourceRef<'x> { + #[inline(always)] + pub fn file_flags(&self) -> Option { + match &self.resource.data { + GroupwareResourceMetadata::File { flags, .. } => Some(*flags), + _ => None, + } + } + + #[inline(always)] + pub fn file_kind(&self) -> Option { + self.file_flags().map(FileFlags::kind) + } + + #[inline(always)] + pub fn is_executable(&self) -> bool { + self.file_flags().is_some_and(FileFlags::is_executable) + } + + #[inline(always)] + pub fn file_role(&self) -> u8 { + self.file_flags().map_or(0, FileFlags::role) + } + + #[inline(always)] + pub fn has_dav_display_name(&self) -> bool { + self.file_flags() + .is_some_and(FileFlags::has_dav_display_name) + } + + pub fn media_type(&self) -> Option<&'x str> { + let flags = self.file_flags()?; + let id = flags.media_type_id(); + if id.is_uncatalogued() { + self.file_extra_bytes(flags) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + } else { + id.as_str() + } + .or((flags.kind() == FILE_KIND_FILE).then_some(FILE_DEFAULT_MEDIA_TYPE)) + } + + fn file_extra_bytes(&self, flags: FileFlags) -> Option<&'x [u8]> { + match &self.resource.data { + GroupwareResourceMetadata::File { name, .. } if flags.extra_len() > 0 => { + let start = name.off as usize + name.len as usize; + self.chunk.bytes.get(start..start + flags.extra_len()) + } + _ => None, + } + } +} + diff --git a/crates/common/src/storage/dav/hierarchy.rs b/crates/common/src/storage/dav/hierarchy.rs new file mode 100644 index 00000000..698434d6 --- /dev/null +++ b/crates/common/src/storage/dav/hierarchy.rs @@ -0,0 +1,256 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{CONTAINER_FLAG, FILE_SEGMENT, canonical_calcard_segment, canonical_path_segment}; +use crate::{ + ArenaRef, DavPath, GroupwareResourceMetadata, GroupwareResourceRef, GroupwareResources, NO_ID, + PathChunk, PathIndex, ResourceChunk, ResourceStore, +}; +use ahash::{AHashMap, AHashSet}; +use percent_encoding::utf8_percent_encode; +use std::borrow::Cow; + +const MAX_HIERARCHY_DEPTH: usize = 128; +const NESTED_SEGMENT_CAPACITY: usize = 16; + +pub enum PathUpdate { + Shared, + Patched(PathIndex), + Rebuild, +} + +impl GroupwareResources { + pub fn patch_paths( + &self, + previous: &ResourceStore, + staging: &ResourceChunk, + changes: &AHashMap<(bool, u32), Option>, + ) -> PathUpdate { + let mut removes: AHashSet = AHashSet::with_capacity(changes.len()); + let mut adds: Vec<(String, DavPath)> = Vec::with_capacity(changes.len()); + + for ((is_container, document_id), slot) in changes { + let old = if previous.unified_id_space { + previous.find_any(*document_id) + } else { + previous.find(*document_id, *is_container) + }; + let new = slot.map(|slot| GroupwareResourceRef { + chunk: staging, + resource: &staging.records[slot as usize], + }); + + match (old, new) { + (Some(old), Some(new)) => { + if old.is_container() != new.is_container() { + return PathUpdate::Rebuild; + } + if !new.has_hierarchy_changes(&old) { + continue; + } + if old.is_container() { + return PathUpdate::Rebuild; + } + let (Some(before), Some(after)) = + (self.materialized_paths(&old), self.materialized_paths(&new)) + else { + return PathUpdate::Rebuild; + }; + self.collect_owned_removes(&old, before, &mut removes); + adds.extend(after); + } + (Some(old), None) => { + if old.is_container() && self.has_child_paths(&old) { + return PathUpdate::Rebuild; + } + let Some(before) = self.materialized_paths(&old) else { + return PathUpdate::Rebuild; + }; + self.collect_owned_removes(&old, before, &mut removes); + } + (None, Some(new)) => { + let Some(after) = self.materialized_paths(&new) else { + return PathUpdate::Rebuild; + }; + adds.extend(after); + } + (None, None) => {} + } + } + + if removes.is_empty() && adds.is_empty() { + PathUpdate::Shared + } else { + PathUpdate::Patched(self.paths.patch(&removes, adds)) + } + } + + fn collect_owned_removes( + &self, + owner: &GroupwareResourceRef<'_>, + before: Vec<(String, DavPath)>, + removes: &mut AHashSet, + ) { + let document_id = owner.document_id(); + removes.extend(before.into_iter().map(|(path, _)| path).filter(|path| { + !self + .paths + .get(path) + .is_some_and(|(_, entry)| entry.document_id != document_id) + })); + } + + fn has_child_paths(&self, container: &GroupwareResourceRef<'_>) -> bool { + match self.container_path_of(container) { + Some(path) => self.paths.range(format!("{path}/")).next().is_some(), + None => true, + } + } + + pub(crate) fn container_path_entry(&self, container_id: u32) -> Option<(&PathChunk, &DavPath)> { + let container = self.resources.find(container_id, true)?; + self.container_path_entry_of(&container) + } + + pub(super) fn container_path_entry_of( + &self, + container: &GroupwareResourceRef<'_>, + ) -> Option<(&PathChunk, &DavPath)> { + let candidate = self.container_path_of(container)?; + let entry = self.paths.get(&candidate)?; + (entry.1.document_id == container.document_id() + && entry.1.hierarchy_seq & CONTAINER_FLAG != 0) + .then_some(entry) + } + + pub(crate) fn materialized_paths( + &self, + resource: &GroupwareResourceRef<'_>, + ) -> Option> { + match &resource.resource.data { + GroupwareResourceMetadata::File { .. } => { + let (path, parent_id, parent_seq) = match Self::nesting_parent(resource) { + Some(parent_id) => match self.container_path_entry(parent_id) { + Some((parent_chunk, parent)) => ( + format!( + "{}/{}", + parent_chunk.path_str(parent), + canonical_path_segment(resource.container_name()?) + ), + parent_id, + Some(parent.hierarchy_seq & !CONTAINER_FLAG), + ), + None if self.resources.find(parent_id, true).is_some() => return None, + None => (self.nested_path_of(resource)?, parent_id, None), + }, + None => (self.nested_path_of(resource)?, NO_ID, None), + }; + Some(vec![( + path, + DavPath { + path: ArenaRef::default(), + parent_id, + hierarchy_seq: parent_seq.map_or(0, |seq| seq + 1) + | if resource.is_container() { + CONTAINER_FLAG + } else { + 0 + }, + document_id: resource.document_id(), + }, + )]) + } + _ if resource.is_container() => Some(vec![( + self.container_path_of(resource)?.into_owned(), + DavPath { + path: ArenaRef::default(), + parent_id: NO_ID, + hierarchy_seq: 1 | CONTAINER_FLAG, + document_id: resource.document_id(), + }, + )]), + _ => { + let mut entries = Vec::with_capacity(resource.child_names().len()); + for name in resource.child_names() { + let Some(container) = self.resources.find(name.parent_id, true) else { + continue; + }; + let parent = self.container_path_of(&container)?; + if self + .paths + .get(&parent) + .is_none_or(|(_, entry)| entry.document_id != name.parent_id) + { + return None; + } + entries.push(( + format!( + "{parent}/{}", + canonical_calcard_segment(resource.child_name_at(name)) + ), + DavPath { + path: ArenaRef::default(), + parent_id: name.parent_id, + hierarchy_seq: 0, + document_id: resource.document_id(), + }, + )); + } + Some(entries) + } + } + } + + fn container_path_of<'x>( + &'x self, + container: &'x GroupwareResourceRef<'x>, + ) -> Option> { + match &container.resource.data { + GroupwareResourceMetadata::File { .. } => { + self.nested_path_of(container).map(Cow::Owned) + } + _ => container.container_name().map(canonical_calcard_segment), + } + } + + fn nested_path_of(&self, resource: &GroupwareResourceRef<'_>) -> Option { + let mut chain = vec![GroupwareResourceRef { + chunk: resource.chunk, + resource: resource.resource, + }]; + let mut parent = Self::nesting_parent(resource); + + while let Some(parent_id) = parent { + if chain.len() > MAX_HIERARCHY_DEPTH { + return None; + } + let Some(ancestor) = self.resources.find_any(parent_id) else { + break; + }; + parent = Self::nesting_parent(&ancestor); + chain.push(ancestor); + } + + let mut path = String::with_capacity(chain.len() * NESTED_SEGMENT_CAPACITY); + for (idx, node) in chain.iter().rev().enumerate() { + if idx > 0 { + path.push('/'); + } + path.extend(utf8_percent_encode(node.container_name()?, FILE_SEGMENT)); + } + Some(path) + } + + fn nesting_parent(resource: &GroupwareResourceRef<'_>) -> Option { + match &resource.resource.data { + GroupwareResourceMetadata::File { parent_id, .. } if *parent_id != NO_ID => { + Some(*parent_id) + } + _ => None, + } + } +} + diff --git a/crates/common/src/storage/dav/mod.rs b/crates/common/src/storage/dav/mod.rs new file mode 100644 index 00000000..d16d0691 --- /dev/null +++ b/crates/common/src/storage/dav/mod.rs @@ -0,0 +1,283 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod file; +pub mod hierarchy; +pub mod paths; +pub mod presence; +pub mod resource; +pub mod store; + +pub use file::{ + FILE_DEFAULT_MEDIA_TYPE, FILE_KIND_DIRECTORY, FILE_KIND_FILE, FILE_KIND_SYMLINK, + MAX_FILE_EXTRA_LEN, +}; +pub use presence::{DISPLAY_NAME_PROPERTY, FilePresence, PresenceBits}; +pub use store::ResourceChunkBuilder; + +use ::store::blake3; +use percent_encoding::{ + AsciiSet, CONTROLS, percent_decode_str, percent_encode, utf8_percent_encode, +}; +use std::{borrow::Cow, convert::Infallible}; +use utils::text::hex_encode_into; + +pub(crate) const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1; +pub const CONTAINER_FLAG: u32 = 1 << 31; +pub const MAX_CACHED_UID_LEN: usize = 255; +const HASHED_UID_MARKER: u8 = 0; +const HASHED_UID_DIGEST_LEN: usize = 16; +const HASHED_UID_SUFFIX_LEN: usize = 1 + 2 * HASHED_UID_DIGEST_LEN; +const HASHED_UID_PREFIX_LEN: usize = MAX_CACHED_UID_LEN - HASHED_UID_SUFFIX_LEN; + +pub trait CachedUid { + fn cached_uid(&self) -> Cow<'_, str>; + fn is_hashed_uid(&self) -> bool; +} + +impl CachedUid for str { + fn cached_uid(&self) -> Cow<'_, str> { + if self.len() <= MAX_CACHED_UID_LEN && !self.as_bytes().contains(&HASHED_UID_MARKER) { + Cow::Borrowed(self) + } else { + Cow::Owned(hashed_uid(self)) + } + } + + #[inline(always)] + fn is_hashed_uid(&self) -> bool { + matches!( + self.as_bytes().split_last_chunk::(), + Some((_, [HASHED_UID_MARKER, ..])) + ) + } +} + +#[cold] +#[inline(never)] +fn hashed_uid(uid: &str) -> String { + let prefix = uid + .get(..uid.floor_char_boundary(HASHED_UID_PREFIX_LEN)) + .unwrap_or_default(); + let digest = blake3::hash(uid.as_bytes()); + let mut hex = [0u8; 2 * HASHED_UID_DIGEST_LEN]; + let mut cached = String::with_capacity(prefix.len() + HASHED_UID_SUFFIX_LEN); + cached.push_str(prefix); + cached.push(char::from(HASHED_UID_MARKER)); + cached.push_str(hex_encode_into( + &digest.as_bytes()[..HASHED_UID_DIGEST_LEN], + &mut hex, + )); + cached +} +pub const MAX_FILE_NODE_DEPTH: usize = 64; + +pub const RFC_3986: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'!') + .add(b'"') + .add(b'#') + .add(b'$') + .add(b'%') + .add(b'&') + .add(b'\'') + .add(b'(') + .add(b')') + .add(b'*') + .add(b'+') + .add(b',') + .add(b'/') + .add(b':') + .add(b';') + .add(b'<') + .add(b'=') + .add(b'>') + .add(b'?') + .add(b'@') + .add(b'[') + .add(b'\\') + .add(b']') + .add(b'^') + .add(b'`') + .add(b'{') + .add(b'|') + .add(b'}'); + +pub const FILE_SEGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'%') + .add(b'/') + .add(b'<') + .add(b'>') + .add(b'?') + .add(b'[') + .add(b'\\') + .add(b']') + .add(b'^') + .add(b'`') + .add(b'{') + .add(b'|') + .add(b'}'); + +pub const MAX_DAV_FILE_NAME_LEN: usize = 255; +pub const FORBIDDEN_FILE_NAME_CHARS: &str = "/<>:\"\\|?*"; +pub const FORBIDDEN_FILE_NODE_NAMES: &[&str] = &[ + ".", "..", "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", + "COM7", "COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", + "LPT9", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DavFileNameError { + Empty, + TooLong, + InvalidCharacter, + ReservedName, +} + +pub fn canonical_path_segment(name: &str) -> Cow<'_, str> { + utf8_percent_encode(name, FILE_SEGMENT).into() +} + +pub fn canonical_uri_path(path: &str) -> Result, DavFileNameError> { + canonicalize_path(path, push_file_segment) +} + +pub fn canonical_dav_resource_uri(uri: &str) -> Result, DavFileNameError> { + canonicalize_resource_uri(uri, push_file_segment) +} + +pub fn canonical_calcard_segment(segment: &str) -> Cow<'_, str> { + if segment.bytes().all(is_pchar) { + return Cow::Borrowed(segment); + } + let mut canonical = String::with_capacity(segment.len() + 8); + let Ok(()) = push_calcard_segment(&mut canonical, segment); + Cow::Owned(canonical) +} + +pub fn canonical_calcard_uri(uri: &str) -> Cow<'_, str> { + let Ok(uri) = canonicalize_resource_uri(uri, push_calcard_segment); + uri +} + +fn canonicalize_path( + path: &str, + push_segment: impl Fn(&mut String, &str) -> Result<(), E>, +) -> Result, E> { + if path.bytes().all(|byte| is_pchar(byte) || byte == b'/') { + return Ok(Cow::Borrowed(path)); + } + let mut canonical = String::with_capacity(path.len() + 8); + for (idx, segment) in path.split('/').enumerate() { + if idx > 0 { + canonical.push('/'); + } + push_segment(&mut canonical, segment)?; + } + Ok(Cow::Owned(canonical)) +} + +fn canonicalize_resource_uri( + uri: &str, + push_segment: impl Fn(&mut String, &str) -> Result<(), E>, +) -> Result, E> { + let Some((base, (collection, (account, resource)))) = + uri.split_once("/dav/").and_then(|(base, path)| { + path.split_once('/').and_then(|(collection, rest)| { + rest.split_once('/') + .map(|account| (base, (collection, account))) + }) + }) + else { + return Ok(Cow::Borrowed(uri)); + }; + Ok(match canonicalize_path(resource, push_segment)? { + Cow::Borrowed(_) => Cow::Borrowed(uri), + Cow::Owned(resource) => Cow::Owned(format!("{base}/dav/{collection}/{account}/{resource}")), + }) +} + +fn push_file_segment(canonical: &mut String, segment: &str) -> Result<(), DavFileNameError> { + canonical.extend(utf8_percent_encode(&decode_segment(segment)?, FILE_SEGMENT)); + Ok(()) +} + +fn push_calcard_segment(canonical: &mut String, segment: &str) -> Result<(), Infallible> { + let decoded: Cow<'_, [u8]> = percent_decode_str(segment).into(); + canonical.extend(percent_encode(&decoded, FILE_SEGMENT)); + Ok(()) +} + +pub fn dav_file_name(segment: &str) -> Result { + let name = decode_segment(segment)?; + if name.is_empty() { + Err(DavFileNameError::Empty) + } else if name.len() > MAX_DAV_FILE_NAME_LEN { + Err(DavFileNameError::TooLong) + } else if matches!(name.as_ref(), "." | "..") { + Err(DavFileNameError::ReservedName) + } else if name.contains(char::is_control) + || percent_decode_str(segment).any(|byte| byte.is_ascii_control()) + { + Err(DavFileNameError::InvalidCharacter) + } else { + Ok(name.into_owned()) + } +} + +fn decode_segment(segment: &str) -> Result, DavFileNameError> { + let decoded: Cow<'_, [u8]> = percent_decode_str(segment).into(); + if decoded.iter().any(|byte| matches!(byte, b'/' | b'\0')) { + return Err(DavFileNameError::InvalidCharacter); + } + Ok(match decoded { + Cow::Borrowed(_) => Cow::Borrowed(segment), + Cow::Owned(bytes) => match String::from_utf8(bytes) { + Ok(decoded) => Cow::Owned(decoded), + Err(err) => Cow::Owned(escape_invalid_utf8(err.as_bytes())), + }, + }) +} + +fn escape_invalid_utf8(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut name = String::with_capacity(bytes.len() * 3); + for chunk in bytes.utf8_chunks() { + name.push_str(chunk.valid()); + for byte in chunk.invalid() { + let _ = write!(name, "%{byte:02X}"); + } + } + name +} + +fn is_pchar(byte: u8) -> bool { + matches!(byte, + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' + | b'.' + | b'_' + | b'~' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b':' + | b'@') +} + diff --git a/crates/common/src/storage/dav/paths.rs b/crates/common/src/storage/dav/paths.rs new file mode 100644 index 00000000..6aaf0b53 --- /dev/null +++ b/crates/common/src/storage/dav/paths.rs @@ -0,0 +1,379 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ArenaRef, DAV_CHUNK, DavPath, PathChunk, PathIndex}; +use ahash::AHashSet; +use std::{cmp::Ordering, sync::Arc}; + +struct ArenaRun { + old_start: usize, + old_end: usize, + new_base: usize, +} + +struct ChunkWriter<'a> { + source: &'a PathChunk, + paths: Vec, + bytes: Vec, + run: Option, +} + +impl<'a> ChunkWriter<'a> { + fn new(source: &'a PathChunk, adds: &[(String, DavPath)]) -> Self { + Self { + source, + paths: Vec::with_capacity(source.paths.len() + adds.len()), + bytes: Vec::with_capacity( + source.bytes.len() + adds.iter().map(|(name, _)| name.len()).sum::(), + ), + run: None, + } + } + + fn keep(&mut self, path: &DavPath) { + let range = path.path.range(); + let (old_start, new_base) = match &mut self.run { + Some(run) if run.old_end == range.start => { + run.old_end = range.end; + (run.old_start, run.new_base) + } + _ => { + self.flush(); + let new_base = self.bytes.len(); + self.run = Some(ArenaRun { + old_start: range.start, + old_end: range.end, + new_base, + }); + (range.start, new_base) + } + }; + self.paths.push(DavPath { + path: ArenaRef { + off: (new_base + (range.start - old_start)) as u32, + len: path.path.len, + }, + ..*path + }); + } + + fn insert(&mut self, name: &str, path: &DavPath) { + self.flush(); + let off = self.bytes.len() as u32; + self.bytes.extend_from_slice(name.as_bytes()); + self.paths.push(DavPath { + path: ArenaRef { + off, + len: name.len() as u32, + }, + ..*path + }); + } + + fn flush(&mut self) { + if let Some(run) = self.run.take() { + self.bytes + .extend_from_slice(&self.source.bytes[run.old_start..run.old_end]); + } + } + + fn finish(mut self) -> Vec> { + self.flush(); + PathChunk::from_contiguous(self.paths, self.bytes) + } +} + +impl PathChunk { + fn from_contiguous(paths: Vec, bytes: Vec) -> Vec> { + if paths.is_empty() { + Vec::new() + } else if paths.len() <= 2 * DAV_CHUNK { + vec![Arc::new(PathChunk { + paths: paths.into_boxed_slice(), + bytes: bytes.into_boxed_slice(), + })] + } else { + paths + .chunks(DAV_CHUNK) + .map(|group| { + let start = group[0].path.off as usize; + let end = group[group.len() - 1].path.range().end; + let paths = group + .iter() + .map(|path| DavPath { + path: ArenaRef { + off: path.path.off - start as u32, + len: path.path.len, + }, + ..*path + }) + .collect::>(); + Arc::new(PathChunk { + paths: paths.into_boxed_slice(), + bytes: bytes[start..end].to_vec().into_boxed_slice(), + }) + }) + .collect() + } + } + + fn splice(&self, removed_slots: &[usize], adds: &[(String, DavPath)]) -> Vec> { + let mut writer = ChunkWriter::new(self, adds); + let mut next_removed = 0usize; + let mut next_add = 0usize; + + 'paths: for (slot, path) in self.paths.iter().enumerate() { + if removed_slots.get(next_removed) == Some(&slot) { + next_removed += 1; + continue; + } + let name = &self.bytes[path.path.range()]; + while let Some((add_name, add_path)) = adds.get(next_add) { + match add_name.as_bytes().cmp(name) { + Ordering::Less => { + writer.insert(add_name, add_path); + next_add += 1; + } + Ordering::Equal => { + next_add += 1; + if add_path.document_id < path.document_id { + writer.insert(add_name, add_path); + continue 'paths; + } + break; + } + Ordering::Greater => break, + } + } + writer.keep(path); + } + + for (add_name, add_path) in &adds[next_add..] { + writer.insert(add_name, add_path); + } + + writer.finish() + } + + #[inline(always)] + pub fn path_at(&self, idx: usize) -> &str { + self.path_str(&self.paths[idx]) + } + + #[inline(always)] + pub fn path_str(&self, path: &DavPath) -> &str { + std::str::from_utf8(self.path_bytes(path)).unwrap_or_default() + } + + #[inline(always)] + pub fn path_bytes(&self, path: &DavPath) -> &[u8] { + &self.bytes[path.path.range()] + } + + #[inline(always)] + fn first_path_bytes(&self) -> &[u8] { + self.path_bytes(&self.paths[0]) + } + + #[inline(always)] + pub fn first_path(&self) -> &str { + self.path_at(0) + } + + #[inline(always)] + pub fn last_path(&self) -> &str { + self.path_at(self.paths.len() - 1) + } + + pub fn heap_size(&self) -> u64 { + (self.paths.len() * std::mem::size_of::() + + self.bytes.len() + + std::mem::size_of::()) as u64 + } +} + +impl PathIndex { + fn entry_order(a: &(String, DavPath), b: &(String, DavPath)) -> Ordering { + a.0.as_bytes() + .cmp(b.0.as_bytes()) + .then_with(|| a.1.document_id.cmp(&b.1.document_id)) + } + + fn sort_entries(entries: &mut Vec<(String, DavPath)>) { + entries.sort_unstable_by(Self::entry_order); + entries.dedup_by(|a, b| a.0 == b.0); + } + + pub fn pack(mut entries: Vec<(String, DavPath)>) -> Self { + Self::sort_entries(&mut entries); + + let total = entries.len(); + let mut chunks = Vec::with_capacity(total.div_ceil(DAV_CHUNK).max(1)); + for group in entries.chunks(DAV_CHUNK) { + let mut bytes: Vec = Vec::with_capacity(group.len() * 48); + let mut paths = Vec::with_capacity(group.len()); + for (name, path) in group { + let off = bytes.len() as u32; + bytes.extend_from_slice(name.as_bytes()); + paths.push(DavPath { + path: ArenaRef { + off, + len: name.len() as u32, + }, + ..*path + }); + } + chunks.push(Arc::new(PathChunk { + paths: paths.into_boxed_slice(), + bytes: bytes.into_boxed_slice(), + })); + } + + Self { chunks, total } + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.total + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.total == 0 + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.chunks + .iter() + .flat_map(|chunk| chunk.paths.iter().map(move |path| (chunk.as_ref(), path))) + } + + fn chunk_for(&self, key: &[u8]) -> usize { + let mut lo = 0usize; + let mut hi = self.chunks.len(); + while lo + 1 < hi { + let mid = (lo + hi) / 2; + if self.chunks[mid].first_path_bytes() <= key { + lo = mid; + } else { + hi = mid; + } + } + lo + } + + fn locate(&self, name: &str) -> Option<(usize, usize)> { + if self.chunks.is_empty() { + return None; + } + let key = name.as_bytes(); + let idx = self.chunk_for(key); + let chunk = &self.chunks[idx]; + chunk + .paths + .binary_search_by(|probe| chunk.path_bytes(probe).cmp(key)) + .ok() + .map(|slot| (idx, slot)) + } + + pub fn get(&self, name: &str) -> Option<(&PathChunk, &DavPath)> { + self.locate(name).map(|(idx, slot)| { + let chunk = &self.chunks[idx]; + (chunk.as_ref(), &chunk.paths[slot]) + }) + } + + fn lower_bound(&self, key: &[u8]) -> (usize, usize) { + if self.chunks.is_empty() { + return (0, 0); + } + let idx = self.chunk_for(key); + let chunk = &self.chunks[idx]; + let slot = chunk + .paths + .partition_point(|probe| chunk.path_bytes(probe) < key); + if slot < chunk.paths.len() { + (idx, slot) + } else { + (idx + 1, 0) + } + } + + pub fn range(&self, prefix: String) -> impl Iterator + '_ { + let (start_chunk, start_slot) = self.lower_bound(prefix.as_bytes()); + self.chunks + .iter() + .skip(start_chunk) + .enumerate() + .flat_map(move |(offset, chunk)| { + chunk + .paths + .iter() + .skip(if offset == 0 { start_slot } else { 0 }) + .map(move |path| (chunk.as_ref(), path)) + }) + .take_while(move |(chunk, path)| chunk.path_bytes(path).starts_with(prefix.as_bytes())) + } + + pub fn heap_size(&self) -> u64 { + self.chunks.iter().map(|chunk| chunk.heap_size()).sum() + } + + pub fn patch(&self, removes: &AHashSet, mut adds: Vec<(String, DavPath)>) -> Self { + if removes.is_empty() && adds.is_empty() { + return self.clone(); + } + + if self.chunks.is_empty() { + return PathIndex::pack(adds); + } + + Self::sort_entries(&mut adds); + + let mut removed: Vec<(usize, usize)> = + removes.iter().filter_map(|key| self.locate(key)).collect(); + removed.sort_unstable(); + + let mut chunks = Vec::with_capacity(self.chunks.len() + 2); + let mut total = 0usize; + let mut add_start = 0usize; + let mut removed_start = 0usize; + + for (idx, chunk) in self.chunks.iter().enumerate() { + let add_end = match self.chunks.get(idx + 1) { + Some(next) => { + let bound = next.first_path_bytes(); + add_start + + adds[add_start..].partition_point(|(name, _)| name.as_bytes() < bound) + } + None => adds.len(), + }; + let removed_end = removed_start + + removed[removed_start..].partition_point(|(target, _)| *target <= idx); + + if add_start == add_end && removed_start == removed_end { + total += chunk.paths.len(); + chunks.push(Arc::clone(chunk)); + continue; + } + + let removed_slots = removed[removed_start..removed_end] + .iter() + .map(|(_, slot)| *slot) + .collect::>(); + for piece in chunk.splice(&removed_slots, &adds[add_start..add_end]) { + total += piece.paths.len(); + chunks.push(piece); + } + + add_start = add_end; + removed_start = removed_end; + } + + Self { chunks, total } + } +} + diff --git a/crates/common/src/storage/dav/presence.rs b/crates/common/src/storage/dav/presence.rs new file mode 100644 index 00000000..541d11a1 --- /dev/null +++ b/crates/common/src/storage/dav/presence.rs @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use types::metadata::{MetadataKinds, MetadataView, XmlName}; + +pub const DISPLAY_NAME_PROPERTY: XmlName<'static> = XmlName::borrowed(Some("DAV:"), "displayname"); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PresenceBits { + jmap: u16, + dav: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[repr(transparent)] +pub struct FilePresence(u16); + +impl PresenceBits { + pub const CALENDAR_EVENT: PresenceBits = PresenceBits::new(1 << 10, 1 << 4); + pub const CONTACT_CARD: PresenceBits = PresenceBits::new(1 << 1, 1); + pub const FILE_NODE: PresenceBits = PresenceBits::new(1, 1 << 1); + + const fn new(jmap: u16, dav: u16) -> Self { + PresenceBits { jmap, dav } + } + + pub const fn jmap(self) -> u16 { + self.jmap + } + + pub const fn dav(self) -> u16 { + self.dav + } + + pub const fn mask(self) -> u16 { + self.jmap | self.dav + } + + pub const fn kinds(self, flags: u16) -> MetadataKinds { + let jmap = if flags & self.jmap != 0 { + MetadataKinds::JMAP + } else { + MetadataKinds::NONE + }; + let dav = if flags & self.dav != 0 { + MetadataKinds::DAV + } else { + MetadataKinds::NONE + }; + jmap.union(dav) + } + + pub const fn apply(self, flags: u16, kinds: MetadataKinds) -> u16 { + let mut flags = flags & !self.mask(); + if kinds.intersects(MetadataKinds::JMAP) { + flags |= self.jmap; + } + if kinds.intersects(MetadataKinds::DAV) { + flags |= self.dav; + } + flags + } +} + +impl FilePresence { + pub const NONE: FilePresence = FilePresence(0); + const DAV_DISPLAY_NAME: u16 = 1 << 2; + pub(crate) const MASK: u16 = PresenceBits::FILE_NODE.mask() | Self::DAV_DISPLAY_NAME; + + pub const fn from_bits(bits: u16) -> Self { + FilePresence(bits & Self::MASK) + } + + pub const fn from_kinds(kinds: MetadataKinds) -> Self { + FilePresence(PresenceBits::FILE_NODE.apply(0, kinds)) + } + + pub fn from_view(view: &MetadataView<'_>) -> Self { + let kinds = view.kinds(); + let mut presence = if kinds.intersects(MetadataKinds::JMAP) { + FilePresence::from_kinds(MetadataKinds::JMAP) + } else { + FilePresence::NONE + }; + if kinds.intersects(MetadataKinds::DAV) { + for (name, _) in view.dav() { + if name == DISPLAY_NAME_PROPERTY { + presence = presence.with_dav_display_name(); + } else { + presence.0 |= PresenceBits::FILE_NODE.dav(); + } + } + } + presence + } + + pub const fn with_dav_display_name(self) -> Self { + FilePresence(self.0 | Self::DAV_DISPLAY_NAME) + } + + pub const fn bits(self) -> u16 { + self.0 + } + + pub const fn apply(self, flags: u16) -> u16 { + (flags & !Self::MASK) | self.0 + } + + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + pub const fn has_dav_display_name(self) -> bool { + self.0 & Self::DAV_DISPLAY_NAME != 0 + } + + pub const fn has_dead_properties(self) -> bool { + self.0 & PresenceBits::FILE_NODE.dav() != 0 + } + + pub const fn kinds(self) -> MetadataKinds { + let kinds = PresenceBits::FILE_NODE.kinds(self.0); + if self.has_dav_display_name() { + kinds.union(MetadataKinds::DAV) + } else { + kinds + } + } +} + diff --git a/crates/common/src/storage/dav/resource.rs b/crates/common/src/storage/dav/resource.rs new file mode 100644 index 00000000..79361f22 --- /dev/null +++ b/crates/common/src/storage/dav/resource.rs @@ -0,0 +1,680 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + ArchivedDavName, ArenaRef, CachedName, DavName, DavPath, DavResourcePath, GroupwareResource, + GroupwareResourceMetadata, GroupwareResourceRef, GroupwareResources, NO_ID, PathChunk, + TinyCalendarPreferences, +}; +use store::rand::{RngExt, distr::Alphanumeric}; +use types::{acl::AclGrant, metadata::MetadataKinds}; + +use super::{CONTAINER_FLAG, FILE_KIND_FILE, PresenceBits, SCHEDULE_INBOX_ID}; + +impl<'x> GroupwareResourceRef<'x> { + #[inline(always)] + pub fn document_id(&self) -> u32 { + self.resource.document_id + } + + #[inline(always)] + pub fn is_container(&self) -> bool { + self.resource.is_container() + } + + #[inline(always)] + pub fn size(&self) -> Option { + self.resource.size() + } + + #[inline(always)] + pub fn is_child_of(&self, parent_id: u32) -> bool { + self.resource.is_child_of(parent_id) + } + + #[inline(always)] + pub fn parent_id(&self) -> Option { + self.resource.parent_id() + } + + #[inline(always)] + pub fn event_time_range(&self) -> Option<(i64, i64)> { + self.resource.event_time_range() + } + + #[inline(always)] + pub fn event_flags(&self) -> Option { + self.resource.event_flags() + } + + #[inline(always)] + pub fn created_at(&self) -> Option { + self.resource.created_at() + } + + #[inline(always)] + pub fn modified_at(&self) -> Option { + self.resource.modified_at() + } + + #[inline(always)] + pub fn event_id(&self) -> Option { + self.resource.event_id() + } + + #[inline(always)] + pub fn etag(&self) -> u32 { + self.resource.etag() + } + + #[inline(always)] + pub fn metadata_kinds(&self) -> MetadataKinds { + self.resource.metadata_kinds() + } + + #[inline(always)] + pub fn uid(&self) -> Option<&'x str> { + match &self.resource.data { + GroupwareResourceMetadata::CalendarEvent { uid, .. } + | GroupwareResourceMetadata::ContactCard { uid, .. } => Some(self.chunk.str_at(*uid)), + _ => None, + } + } + + #[inline(always)] + pub fn has_cached_uid(&self, cached_uid: &[u8]) -> bool { + match &self.resource.data { + GroupwareResourceMetadata::CalendarEvent { uid, .. } + | GroupwareResourceMetadata::ContactCard { uid, .. } => { + self.chunk.bytes_at(*uid) == cached_uid + } + _ => false, + } + } + + #[inline(always)] + pub fn container_name(&self) -> Option<&'x str> { + match &self.resource.data { + GroupwareResourceMetadata::File { name, .. } + | GroupwareResourceMetadata::Calendar { name, .. } + | GroupwareResourceMetadata::AddressBook { name, .. } => Some(self.chunk.str_at(*name)), + GroupwareResourceMetadata::CalendarEventNotification { names, .. } + if names.is_empty() => + { + Some(if self.resource.document_id == SCHEDULE_INBOX_ID { + "inbox" + } else { + "outbox" + }) + } + _ => None, + } + } + + #[inline(always)] + pub fn child_names(&self) -> &[CachedName] { + match self.resource.names_ref() { + Some(names) => self.chunk.names_at(names), + None => &[], + } + } + + #[inline(always)] + pub fn child_name_at(&self, name: &CachedName) -> &str { + self.chunk.str_at(name.name) + } + + #[inline(always)] + pub fn acls(&self) -> &'x [AclGrant] { + match self.resource.acls_ref() { + Some(acls) => self.chunk.acls_at(acls), + None => &[], + } + } + + #[inline(always)] + pub fn has_acls(&self) -> bool { + self.resource + .acls_ref() + .is_some_and(|acls| !acls.is_empty()) + } + + pub fn notification(&self) -> Option> { + match &self.resource.data { + GroupwareResourceMetadata::CalendarEventNotification { + changed_by, + principals, + calendar_ids_len, + flags, + .. + } => { + let ids = self.chunk.principals_at(*principals); + let (calendar_ids, dismissed_by) = + ids.split_at((*calendar_ids_len as usize).min(ids.len())); + Some(NotificationRef { + changed_by: *changed_by, + calendar_ids, + dismissed_by, + flags: *flags, + }) + } + _ => None, + } + } + + pub fn personal_calendar_preferences( + &self, + account_id: u32, + ) -> Option<&TinyCalendarPreferences> { + match &self.resource.data { + GroupwareResourceMetadata::Calendar { preferences, .. } => self + .chunk + .prefs_at(*preferences) + .iter() + .find(|pref| pref.account_id == account_id), + _ => None, + } + } + + pub fn all_calendar_preferences(&self) -> &'x [TinyCalendarPreferences] { + match &self.resource.data { + GroupwareResourceMetadata::Calendar { preferences, .. } => { + self.chunk.prefs_at(*preferences) + } + _ => &[], + } + } + + pub fn calendar_preferences(&self, account_id: u32) -> Option<&TinyCalendarPreferences> { + match &self.resource.data { + GroupwareResourceMetadata::Calendar { preferences, .. } => { + let prefs = self.chunk.prefs_at(*preferences); + prefs + .iter() + .find(|pref| pref.account_id == account_id) + .or_else(|| prefs.first()) + } + _ => None, + } + } + + pub fn has_hierarchy_changes(&self, other: &GroupwareResourceRef<'_>) -> bool { + match (&self.resource.data, &other.resource.data) { + ( + GroupwareResourceMetadata::File { + name: a, + parent_id: c, + .. + }, + GroupwareResourceMetadata::File { + name: b, + parent_id: d, + .. + }, + ) => c != d || self.chunk.str_at(*a) != other.chunk.str_at(*b), + ( + GroupwareResourceMetadata::Calendar { name: a, .. }, + GroupwareResourceMetadata::Calendar { name: b, .. }, + ) + | ( + GroupwareResourceMetadata::AddressBook { name: a, .. }, + GroupwareResourceMetadata::AddressBook { name: b, .. }, + ) => self.chunk.str_at(*a) != other.chunk.str_at(*b), + _ => { + let a = self.child_names(); + let b = other.child_names(); + a.len() != b.len() + || a.iter().zip(b.iter()).any(|(x, y)| { + x.parent_id != y.parent_id + || self.chunk.str_at(x.name) != other.chunk.str_at(y.name) + }) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NotificationRef<'x> { + pub changed_by: u32, + pub calendar_ids: &'x [u32], + pub dismissed_by: &'x [u32], + pub flags: u16, +} + +impl GroupwareResource { + #[inline(always)] + pub fn uid_ref(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { uid, .. } + | GroupwareResourceMetadata::ContactCard { uid, .. } => Some(*uid), + _ => None, + } + } + + #[inline(always)] + pub fn names_ref(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { names, .. } + | GroupwareResourceMetadata::ContactCard { names, .. } + | GroupwareResourceMetadata::CalendarEventNotification { names, .. } => Some(*names), + _ => None, + } + } + + #[inline(always)] + pub fn acls_ref(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::File { acls, .. } + | GroupwareResourceMetadata::Calendar { acls, .. } + | GroupwareResourceMetadata::AddressBook { acls, .. } => Some(*acls), + _ => None, + } + } + + #[inline(always)] + pub fn is_container(&self) -> bool { + match &self.data { + GroupwareResourceMetadata::File { flags, .. } => flags.is_directory(), + GroupwareResourceMetadata::Calendar { .. } + | GroupwareResourceMetadata::AddressBook { .. } => true, + GroupwareResourceMetadata::CalendarEventNotification { names, .. } => names.is_empty(), + _ => false, + } + } + + #[inline(always)] + pub fn size(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::File { size, flags, .. } + if flags.kind() == FILE_KIND_FILE => + { + Some(*size) + } + _ => None, + } + } + + #[inline(always)] + pub fn parent_id(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::File { parent_id, .. } if *parent_id != NO_ID => { + Some(*parent_id) + } + GroupwareResourceMetadata::CalendarEventNotification { names, .. } + if names.is_empty() => + { + Some(SCHEDULE_INBOX_ID) + } + _ => None, + } + } + + #[inline(always)] + pub fn event_time_range(&self) -> Option<(i64, i64)> { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { + start, duration, .. + } => Some((*start, start.saturating_add(*duration as i64))), + _ => None, + } + } + + #[inline(always)] + pub fn event_flags(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { flags, .. } => Some(*flags), + _ => None, + } + } + + #[inline(always)] + pub fn created_at(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { created_at, .. } + | GroupwareResourceMetadata::ContactCard { created_at, .. } + | GroupwareResourceMetadata::CalendarEventNotification { created_at, .. } => { + Some(*created_at) + } + GroupwareResourceMetadata::File { + modified, + created_delta, + .. + } => modified.checked_add(*created_delta as i64), + _ => None, + } + } + + #[inline(always)] + pub fn modified_at(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEvent { + created_at, + modified_at, + .. + } + | GroupwareResourceMetadata::ContactCard { + created_at, + modified_at, + .. + } => Some(*created_at + *modified_at as i64), + GroupwareResourceMetadata::File { modified, .. } => Some(*modified), + _ => None, + } + } + + #[inline(always)] + pub fn event_id(&self) -> Option { + match &self.data { + GroupwareResourceMetadata::CalendarEventNotification { event_id, .. } => { + Some(*event_id) + } + _ => None, + } + } + + #[inline(always)] + pub fn etag(&self) -> u32 { + match &self.data { + GroupwareResourceMetadata::File { etag, .. } + | GroupwareResourceMetadata::Calendar { etag, .. } + | GroupwareResourceMetadata::CalendarEvent { etag, .. } + | GroupwareResourceMetadata::CalendarEventNotification { etag, .. } + | GroupwareResourceMetadata::AddressBook { etag, .. } + | GroupwareResourceMetadata::ContactCard { etag, .. } => *etag, + } + } + + #[inline(always)] + pub fn metadata_kinds(&self) -> MetadataKinds { + match &self.data { + GroupwareResourceMetadata::File { flags, .. } => flags.metadata_kinds(), + GroupwareResourceMetadata::Calendar { metadata, .. } + | GroupwareResourceMetadata::AddressBook { metadata, .. } => *metadata, + GroupwareResourceMetadata::CalendarEvent { flags, .. } => { + PresenceBits::CALENDAR_EVENT.kinds(*flags) + } + GroupwareResourceMetadata::ContactCard { flags, .. } => { + PresenceBits::CONTACT_CARD.kinds(*flags) + } + GroupwareResourceMetadata::CalendarEventNotification { .. } => MetadataKinds::NONE, + } + } + + pub fn is_child_of(&self, parent_id: u32) -> bool { + match &self.data { + GroupwareResourceMetadata::File { parent_id: id, .. } => *id == parent_id, + GroupwareResourceMetadata::CalendarEventNotification { names, .. } => { + names.is_empty() && parent_id == SCHEDULE_INBOX_ID + } + _ => false, + } + } +} + +impl DavResourcePath<'_> { + #[inline(always)] + pub fn document_id(&self) -> u32 { + self.path.document_id + } + + #[inline(always)] + pub fn parent_id(&self) -> Option { + (self.path.parent_id != NO_ID).then_some(self.path.parent_id) + } + + #[inline(always)] + pub fn path(&self) -> &str { + std::str::from_utf8(&self.path_chunk.bytes[self.path.path.range()]).unwrap_or_default() + } + + #[inline(always)] + pub fn is_container(&self) -> bool { + self.resource.is_container() + } + + #[inline(always)] + pub fn hierarchy_seq(&self) -> u32 { + self.path.hierarchy_seq & !CONTAINER_FLAG + } + + #[inline(always)] + pub fn size(&self) -> u32 { + self.resource.size().unwrap_or_default() + } +} + +impl DavName { + pub fn new(name: String, parent_id: u32) -> Self { + Self { name, parent_id } + } + + pub fn new_with_rand_name(parent_id: u32) -> Self { + Self { + name: store::rand::rng() + .sample_iter(Alphanumeric) + .take(10) + .map(char::from) + .collect::(), + parent_id, + } + } + + #[inline(always)] + pub fn parent_id(&self) -> u32 { + self.parent_id + } +} + +impl ArchivedDavName { + #[inline(always)] + pub fn parent_id(&self) -> u32 { + self.parent_id.to_native() + } +} + +impl GroupwareResources { + fn pair<'x>(&'x self, entry: (&'x PathChunk, &'x DavPath)) -> Option> { + let (path_chunk, path) = entry; + let want_container = path.hierarchy_seq & CONTAINER_FLAG != 0; + self.resources + .find(path.document_id, want_container) + .map(|resource| DavResourcePath { + path, + path_chunk, + resource, + }) + } + + pub fn by_path(&self, name: &str) -> Option> { + self.paths.get(name).and_then(|entry| self.pair(entry)) + } + + pub fn container_resource_by_id(&self, id: u32) -> Option> { + self.resources.find(id, true) + } + + pub fn item_by_id(&self, id: u32) -> Option> { + self.resources.find(id, false) + } + + fn derived_path_entries(&self, document_id: u32) -> Option> { + let mut entries = Vec::with_capacity(2); + + let found = if self.resources.unified_id_space { + [self.resources.find_any(document_id), None] + } else { + [ + self.resources.find(document_id, true), + self.resources.find(document_id, false), + ] + }; + + for resource in found.into_iter().flatten() { + for (path, _) in self.materialized_paths(&resource)? { + let entry = self.paths.get(&path)?; + if entry.1.document_id != document_id { + return None; + } + entries.push(entry); + } + } + + Some(entries) + } + + fn path_entries_by_id(&self, document_id: u32) -> Vec<(&PathChunk, &DavPath)> { + self.derived_path_entries(document_id).unwrap_or_else(|| { + self.paths + .iter() + .filter(|(_, path)| path.document_id == document_id) + .collect() + }) + } + + pub(crate) fn child_path( + &self, + document_id: u32, + parent_id: u32, + ) -> Option> { + self.path_entries_by_id(document_id) + .into_iter() + .find(|(_, path)| path.parent_id == parent_id) + .and_then(|entry| self.pair(entry)) + } + + pub fn container_resource_path_by_id(&self, id: u32) -> Option> { + let container = self.resources.find(id, true)?; + self.container_path_entry_of(&container) + .or_else(|| { + self.paths.iter().find(|(_, path)| { + path.document_id == id && path.hierarchy_seq & CONTAINER_FLAG != 0 + }) + }) + .and_then(|entry| self.pair(entry)) + } + + pub fn any_resource_path_by_id(&self, id: u32) -> Option> { + self.path_entries_by_id(id) + .into_iter() + .next() + .and_then(|entry| self.pair(entry)) + } + + pub fn subtree(&self, search_path: &str) -> impl Iterator> { + self.by_path(search_path).into_iter().chain( + self.paths + .range(format!("{search_path}/")) + .filter_map(move |entry| self.pair(entry)), + ) + } + + pub fn subtree_with_depth( + &self, + search_path: &str, + depth: usize, + ) -> impl Iterator> { + let prefix = format!("{search_path}/"); + let cut = prefix.len(); + self.by_path(search_path) + .into_iter() + .chain(self.paths.range(prefix).filter_map(move |entry| { + (count_separators(entry.0.path_bytes(entry.1).get(cut..).unwrap_or_default()) + < depth) + .then(|| self.pair(entry)) + .flatten() + })) + } + + pub fn tree_with_depth(&self, depth: usize) -> impl Iterator> { + self.paths.iter().filter_map(move |entry| { + (count_separators(entry.0.path_bytes(entry.1)) <= depth) + .then(|| self.pair(entry)) + .flatten() + }) + } + + fn child_entries(&self, parent_id: u32) -> impl Iterator { + let prefix = self.resources.find(parent_id, true).map(|container| { + match self.container_path_entry_of(&container) { + Some((chunk, path)) => format!("{}/", chunk.path_str(path)), + None => String::new(), + } + }); + prefix + .into_iter() + .flat_map(move |prefix| self.paths.range(prefix)) + .filter(move |(_, path)| path.parent_id == parent_id) + } + + pub fn children(&self, parent_id: u32) -> impl Iterator> { + self.child_entries(parent_id) + .filter_map(move |entry| self.pair(entry)) + } + + pub fn children_ids(&self, parent_id: u32) -> impl Iterator { + self.child_entries(parent_id) + .map(|(_, path)| path.document_id) + } + + pub fn format_resource(&self, resource: DavResourcePath<'_>) -> String { + if resource.is_container() { + format!("{}{}/", self.base_path, resource.path()) + } else { + format!("{}{}", self.base_path, resource.path()) + } + } + + pub fn format_resource_paths_by_id(&self, document_id: u32) -> std::vec::IntoIter { + self.path_entries_by_id(document_id) + .into_iter() + .filter_map(|entry| self.pair(entry)) + .map(|resource| self.format_resource(resource)) + .collect::>() + .into_iter() + } + + pub fn format_resource_path_by_parent( + &self, + document_id: u32, + parent_id: u32, + ) -> Option { + self.path_entries_by_id(document_id) + .into_iter() + .find(|(_, path)| path.parent_id == parent_id) + .and_then(|entry| self.pair(entry)) + .map(|resource| self.format_resource(resource)) + } + + pub fn format_collection(&self, name: &str) -> String { + format!("{}{name}/", self.base_path) + } + + pub fn format_item(&self, name: &str) -> String { + format!("{}{}", self.base_path, name) + } + + #[inline(always)] + pub fn containers(&self) -> impl Iterator> + '_ { + self.resources.iter_run(true).filter(|r| r.is_container()) + } + + #[inline(always)] + pub fn resources_with_acls(&self) -> impl Iterator> + '_ { + self.resources.iter_with_acls() + } + + pub fn recompute_size(&mut self) { + self.size = std::mem::size_of::() as u64 + + self.base_path.len() as u64 + + self.resources.heap_size() + + self.paths.heap_size(); + } +} + +#[inline(always)] +fn count_separators(path: &[u8]) -> usize { + path.iter().filter(|&&byte| byte == b'/').count() +} diff --git a/crates/common/src/storage/dav/store.rs b/crates/common/src/storage/dav/store.rs new file mode 100644 index 00000000..25f298b3 --- /dev/null +++ b/crates/common/src/storage/dav/store.rs @@ -0,0 +1,673 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + ArenaRef, CachedName, DAV_CHUNK, DavName, GroupwareResource, GroupwareResourceMetadata, + GroupwareResourceRef, NO_ID, ResourceChunk, ResourceStore, TinyCalendarPreferences, +}; +use ahash::AHashMap; +use std::{ + borrow::Cow, + ops::Range, + sync::{Arc, OnceLock}, +}; +use types::acl::AclGrant; +use xxhash_rust::xxh3::xxh3_64; + +const UID_SLOT_MASK: u64 = u32::MAX as u64; + +#[inline(always)] +fn uid_key(cached_uid: &[u8]) -> u64 { + xxh3_64(cached_uid) & !UID_SLOT_MASK +} + +impl ResourceChunk { + #[inline(always)] + pub fn str_at(&self, r: ArenaRef) -> &str { + std::str::from_utf8(self.bytes_at(r)).unwrap_or_default() + } + + #[inline(always)] + pub fn bytes_at(&self, r: ArenaRef) -> &[u8] { + &self.bytes[r.range()] + } + + #[inline(always)] + pub fn names_at(&self, r: ArenaRef) -> &[CachedName] { + &self.names[r.range()] + } + + #[inline(always)] + pub fn acls_at(&self, r: ArenaRef) -> &[AclGrant] { + &self.acls[r.range()] + } + + #[inline(always)] + pub fn prefs_at(&self, r: ArenaRef) -> &[TinyCalendarPreferences] { + &self.prefs[r.range()] + } + + #[inline(always)] + pub fn principals_at(&self, r: ArenaRef) -> &[u32] { + &self.principals[r.range()] + } + + fn build_uid_index(records: &[GroupwareResource], bytes: &[u8]) -> Box<[u64]> { + let mut index = Vec::new(); + for (slot, record) in records.iter().enumerate() { + if let Some(uid) = record.uid_ref().and_then(|uid| bytes.get(uid.range())) { + if index.is_empty() { + index.reserve_exact(records.len() - slot); + } + index.push(uid_key(uid) | slot as u64); + } + } + index.sort_unstable(); + index.into_boxed_slice() + } + + fn with_uid_key(&self, key: u64) -> impl Iterator> { + let uid_index = self + .uid_index + .get_or_init(|| Self::build_uid_index(&self.records, &self.bytes)); + let start = uid_index.partition_point(|entry| *entry < key); + uid_index + .get(start..) + .unwrap_or_default() + .iter() + .take_while(move |entry| *entry & !UID_SLOT_MASK == key) + .filter_map(move |entry| { + self.records + .get((*entry & UID_SLOT_MASK) as usize) + .map(|resource| GroupwareResourceRef { + chunk: self, + resource, + }) + }) + } + + fn uid_index_size(&self) -> u64 { + if self + .records + .first() + .is_some_and(|record| record.uid_ref().is_some()) + { + (self.records.len() * std::mem::size_of::()) as u64 + } else { + 0 + } + } + + pub fn heap_size(&self) -> u64 { + (self.records.len() * std::mem::size_of::() + + self.bytes.len() + + self.names.len() * std::mem::size_of::() + + self.acls.len() * std::mem::size_of::() + + self.prefs.len() * std::mem::size_of::() + + self.principals.len() * std::mem::size_of::() + + std::mem::size_of::()) as u64 + } +} + +#[derive(Default)] +pub struct ResourceChunkBuilder { + pub records: Vec, + pub bytes: Vec, + pub names: Vec, + pub acls: Vec, + pub prefs: Vec, + pub principals: Vec, + principal_runs: AHashMap, ArenaRef>, +} + +impl ResourceChunkBuilder { + pub fn with_capacity(n: usize) -> Self { + Self { + records: Vec::with_capacity(n), + bytes: Vec::with_capacity(n * 48), + names: Vec::with_capacity(n), + acls: Vec::with_capacity(8), + prefs: Vec::with_capacity(8), + principals: Vec::new(), + principal_runs: AHashMap::new(), + } + } + + pub fn push_str(&mut self, s: &str) -> ArenaRef { + self.push_arena_bytes(s.as_bytes()) + } + + fn push_arena_bytes(&mut self, bytes: &[u8]) -> ArenaRef { + let off = self.bytes.len() as u32; + self.bytes.extend_from_slice(bytes); + ArenaRef { + off, + len: bytes.len() as u32, + } + } + + pub fn push_uid(&mut self, uid: &str, names: ArenaRef) -> ArenaRef { + self.push_uid_bytes(uid.as_bytes(), names) + } + + fn push_uid_bytes(&mut self, uid: &[u8], names: ArenaRef) -> ArenaRef { + for name in self.names.get(names.range()).unwrap_or_default() { + if self + .bytes + .get(name.name.range()) + .is_some_and(|bytes| bytes.starts_with(uid)) + { + return ArenaRef { + off: name.name.off, + len: uid.len() as u32, + }; + } + } + self.push_arena_bytes(uid) + } + + pub fn push_acls(&mut self, acls: &[AclGrant]) -> ArenaRef { + let off = self.acls.len() as u32; + self.acls.extend_from_slice(acls); + ArenaRef { + off, + len: acls.len() as u32, + } + } + + pub fn push_prefs(&mut self, prefs: &[TinyCalendarPreferences]) -> ArenaRef { + let off = self.prefs.len() as u32; + self.prefs.extend_from_slice(prefs); + ArenaRef { + off, + len: prefs.len() as u32, + } + } + + pub fn push_principals(&mut self, principals: impl IntoIterator) -> ArenaRef { + let off = self.principals.len(); + self.principals.extend(principals); + if self.principals.len() == off { + return ArenaRef::default(); + } + + if let Some(interned) = self.principal_runs.get(&self.principals[off..]).copied() { + self.principals.truncate(off); + return interned; + } + + let arena = ArenaRef { + off: off as u32, + len: (self.principals.len() - off) as u32, + }; + self.principal_runs + .insert(Box::from(&self.principals[off..]), arena); + arena + } + + pub fn push_names(&mut self, names: &[DavName]) -> ArenaRef { + let off = self.names.len() as u32; + for name in names { + let name_ref = self.push_str(&name.name); + self.names.push(CachedName { + name: name_ref, + parent_id: name.parent_id, + }); + } + ArenaRef { + off, + len: names.len() as u32, + } + } + + pub fn push_from(&mut self, src: &GroupwareResourceRef<'_>) { + let data = match &src.resource.data { + GroupwareResourceMetadata::File { + name, + size, + parent_id, + acls, + etag, + modified, + created_delta, + flags, + } => GroupwareResourceMetadata::File { + name: self.push_file_bytes(src.chunk, *name, *flags), + size: *size, + parent_id: *parent_id, + acls: self.push_acls(src.chunk.acls_at(*acls)), + etag: *etag, + modified: *modified, + created_delta: *created_delta, + flags: *flags, + }, + GroupwareResourceMetadata::Calendar { + name, + acls, + preferences, + etag, + metadata, + } => GroupwareResourceMetadata::Calendar { + name: self.push_arena_bytes(src.chunk.bytes_at(*name)), + acls: self.push_acls(src.chunk.acls_at(*acls)), + preferences: self.push_prefs(src.chunk.prefs_at(*preferences)), + etag: *etag, + metadata: *metadata, + }, + GroupwareResourceMetadata::AddressBook { + name, + acls, + etag, + metadata, + } => GroupwareResourceMetadata::AddressBook { + name: self.push_arena_bytes(src.chunk.bytes_at(*name)), + acls: self.push_acls(src.chunk.acls_at(*acls)), + etag: *etag, + metadata: *metadata, + }, + GroupwareResourceMetadata::CalendarEvent { + names, + start, + duration, + created_at, + modified_at, + uid, + etag, + flags, + } => { + let names = self.push_cached_names(src.chunk, *names); + GroupwareResourceMetadata::CalendarEvent { + names, + start: *start, + duration: *duration, + created_at: *created_at, + modified_at: *modified_at, + uid: self.push_uid_bytes(src.chunk.bytes_at(*uid), names), + etag: *etag, + flags: *flags, + } + } + GroupwareResourceMetadata::ContactCard { + names, + created_at, + modified_at, + uid, + etag, + flags, + } => { + let names = self.push_cached_names(src.chunk, *names); + GroupwareResourceMetadata::ContactCard { + names, + created_at: *created_at, + modified_at: *modified_at, + uid: self.push_uid_bytes(src.chunk.bytes_at(*uid), names), + etag: *etag, + flags: *flags, + } + } + GroupwareResourceMetadata::CalendarEventNotification { + names, + created_at, + event_id, + etag, + changed_by, + principals, + calendar_ids_len, + flags, + } => GroupwareResourceMetadata::CalendarEventNotification { + names: self.push_cached_names(src.chunk, *names), + created_at: *created_at, + event_id: *event_id, + etag: *etag, + changed_by: *changed_by, + principals: self + .push_principals(src.chunk.principals_at(*principals).iter().copied()), + calendar_ids_len: *calendar_ids_len, + flags: *flags, + }, + }; + self.records.push(GroupwareResource { + document_id: src.resource.document_id, + data, + }); + } + + fn push_cached_names(&mut self, chunk: &ResourceChunk, names: ArenaRef) -> ArenaRef { + let off = self.names.len() as u32; + for name in chunk.names_at(names) { + let name_ref = self.push_arena_bytes(chunk.bytes_at(name.name)); + self.names.push(CachedName { + name: name_ref, + parent_id: name.parent_id, + }); + } + ArenaRef { + off, + len: names.len, + } + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn finish(self) -> ResourceChunk { + let min_id = self.records.first().map(|r| r.document_id).unwrap_or(NO_ID); + let max_id = self.records.last().map(|r| r.document_id).unwrap_or(0); + ResourceChunk { + uid_index: OnceLock::new(), + records: self.records.into_boxed_slice(), + bytes: self.bytes.into_boxed_slice(), + names: self.names.into_boxed_slice(), + acls: self.acls.into_boxed_slice(), + prefs: self.prefs.into_boxed_slice(), + principals: self.principals.into_boxed_slice(), + min_id, + max_id, + } + } +} + +struct SplitChunks<'x> { + target: &'x mut Vec>, + current: ResourceChunkBuilder, +} + +impl<'x> SplitChunks<'x> { + fn new(target: &'x mut Vec>, capacity: usize) -> Self { + Self { + target, + current: ResourceChunkBuilder::with_capacity(capacity.min(DAV_CHUNK)), + } + } + + fn push(&mut self, resource: &GroupwareResourceRef<'_>) { + if self.current.len() == DAV_CHUNK { + let full = + std::mem::replace(&mut self.current, ResourceChunkBuilder::with_capacity(64)); + self.target.push(Arc::new(full.finish())); + } + self.current.push_from(resource); + } + + fn finish(self) { + if !self.current.is_empty() { + self.target.push(Arc::new(self.current.finish())); + } + } +} + +impl ResourceStore { + pub fn from_sorted( + containers: Vec, + items: Vec, + unified_id_space: bool, + ) -> Self { + let mut chunks = Vec::with_capacity(containers.len() + items.len()); + let mut total = 0; + for builder in containers.into_iter().filter(|builder| !builder.is_empty()) { + debug_assert!( + builder.records.is_sorted_by_key(|r| r.document_id), + "chunk records must be ordered by document_id for binary search" + ); + total += builder.len(); + chunks.push(Arc::new(builder.finish())); + } + let containers_end = chunks.len(); + for builder in items.into_iter().filter(|builder| !builder.is_empty()) { + debug_assert!( + builder.records.is_sorted_by_key(|r| r.document_id), + "chunk records must be ordered by document_id for binary search" + ); + total += builder.len(); + chunks.push(Arc::new(builder.finish())); + } + Self { + chunks, + containers_end, + total, + unified_id_space, + } + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.total + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.total == 0 + } + + pub fn count(&self, is_container: bool) -> usize { + if self.unified_id_space { + self.iter() + .filter(|resource| resource.is_container() == is_container) + .count() + } else { + self.chunks[self.run(is_container)] + .iter() + .map(|chunk| chunk.records.len()) + .sum() + } + } + + pub fn iter(&self) -> impl Iterator> + '_ { + Self::iter_chunks(&self.chunks) + } + + pub fn iter_run(&self, is_container: bool) -> impl Iterator> { + Self::iter_chunks(&self.chunks[self.run(is_container)]) + } + + fn iter_chunks( + chunks: &[Arc], + ) -> impl Iterator> { + chunks.iter().flat_map(|chunk| { + chunk + .records + .iter() + .map(move |resource| GroupwareResourceRef { + chunk: chunk.as_ref(), + resource, + }) + }) + } + + pub fn iter_with_acls(&self) -> impl Iterator> + '_ { + self.chunks + .iter() + .filter(|chunk| !chunk.acls.is_empty()) + .flat_map(|chunk| { + chunk + .records + .iter() + .map(move |resource| GroupwareResourceRef { + chunk: chunk.as_ref(), + resource, + }) + }) + .filter(|resource| resource.has_acls()) + } + + pub fn heap_size(&self) -> u64 { + let indexed = self.chunks[self.run(false)] + .split_last() + .map_or(0, |(_, indexed)| { + indexed.iter().map(|chunk| chunk.uid_index_size()).sum() + }); + self.chunks + .iter() + .map(|chunk| chunk.heap_size()) + .sum::() + + indexed + } + + fn run(&self, is_container: bool) -> Range { + if self.unified_id_space { + 0..self.chunks.len() + } else if is_container { + 0..self.containers_end + } else { + self.containers_end..self.chunks.len() + } + } + + fn chunk_for(&self, run: &Range, document_id: u32) -> Option { + let chunks = &self.chunks[run.clone()]; + let slot = chunks + .partition_point(|chunk| chunk.min_id <= document_id) + .checked_sub(1)?; + Some(run.start + slot) + } + + fn locate(&self, run: Range, document_id: u32) -> Option> { + let chunk = &self.chunks[self.chunk_for(&run, document_id)?]; + if document_id > chunk.max_id { + return None; + } + chunk + .records + .binary_search_by_key(&document_id, |r| r.document_id) + .ok() + .map(|slot| GroupwareResourceRef { + chunk: chunk.as_ref(), + resource: &chunk.records[slot], + }) + } + + pub fn find_by_uid<'x>( + &'x self, + cached_uid: Cow<'x, str>, + ) -> impl Iterator> + 'x { + let key = uid_key(cached_uid.as_bytes()); + let (tail, indexed) = self.chunks[self.run(false)] + .split_last() + .map_or((&[][..], &[][..]), |(tail, indexed)| { + (std::slice::from_ref(tail), indexed) + }); + indexed + .iter() + .flat_map(move |chunk| chunk.with_uid_key(key)) + .chain(Self::iter_chunks(tail)) + .filter(move |resource| resource.has_cached_uid(cached_uid.as_bytes())) + } + + pub fn find(&self, document_id: u32, want_container: bool) -> Option> { + self.locate(self.run(want_container), document_id) + .filter(|resource| resource.is_container() == want_container) + } + + #[inline(always)] + pub fn find_any(&self, document_id: u32) -> Option> { + if self.unified_id_space { + self.locate(self.run(true), document_id) + } else { + self.locate(self.run(true), document_id) + .or_else(|| self.locate(self.run(false), document_id)) + } + } + + pub fn rebuild( + &self, + staging: &ResourceChunk, + changes: &ahash::AHashMap<(bool, u32), Option>, + ) -> Self { + let unified = self.unified_id_space; + let mut container_chunks: Vec> = Vec::new(); + let mut item_chunks: Vec> = Vec::new(); + + for is_container in [true, false] { + if unified && !is_container { + continue; + } + let key_flag = if unified { true } else { is_container }; + + let mut pending: Vec<(u32, Option)> = changes + .iter() + .filter(|((flag, _), _)| *flag == key_flag) + .map(|((_, document_id), slot)| (*document_id, *slot)) + .collect(); + pending.sort_unstable_by_key(|(document_id, _)| *document_id); + let mut pending = pending.into_iter().peekable(); + + let staged = |slot: u32| GroupwareResourceRef { + chunk: staging, + resource: &staging.records[slot as usize], + }; + + let target = if is_container { + &mut container_chunks + } else { + &mut item_chunks + }; + + let run = &self.chunks[self.run(is_container)]; + for (position, chunk) in run.iter().enumerate() { + let is_last = position + 1 == run.len(); + let owned = |(document_id, slot): &(u32, Option)| { + (is_last && slot.is_some()) || *document_id <= chunk.max_id + }; + + if !pending.peek().is_some_and(owned) { + target.push(Arc::clone(chunk)); + continue; + } + + let mut builder = SplitChunks::new(target, chunk.records.len() + 4); + for record in chunk.records.iter() { + while let Some((_, slot)) = + pending.next_if(|(document_id, _)| *document_id < record.document_id) + { + if let Some(slot) = slot { + builder.push(&staged(slot)); + } + } + match pending.next_if(|(document_id, _)| *document_id == record.document_id) { + Some((_, Some(slot))) => builder.push(&staged(slot)), + Some((_, None)) => {} + None => builder.push(&GroupwareResourceRef { + chunk, + resource: record, + }), + } + } + while let Some((_, slot)) = pending.next_if(owned) { + if let Some(slot) = slot { + builder.push(&staged(slot)); + } + } + builder.finish(); + } + + let mut builder = SplitChunks::new(target, 4); + for (_, slot) in pending { + if let Some(slot) = slot { + builder.push(&staged(slot)); + } + } + builder.finish(); + } + + let containers_end = container_chunks.len(); + let mut chunks = container_chunks; + chunks.extend(item_chunks); + let total = chunks.iter().map(|chunk| chunk.records.len()).sum(); + + Self { + chunks, + containers_end, + total, + unified_id_space: unified, + } + } +} + diff --git a/crates/common/src/storage/encryption.rs b/crates/common/src/storage/encryption.rs index 9b4247ad..59de82fe 100644 --- a/crates/common/src/storage/encryption.rs +++ b/crates/common/src/storage/encryption.rs @@ -5,13 +5,17 @@ */ use crate::auth::EncryptionKeys; -use mail_parser::decoders::base64::base64_decode; +use encodify::base64::LENIENT; use registry::schema::structs::PublicKey; use sequoia_openpgp::{Cert, parse::Parse, policy::StandardPolicy, types::KeyFlags}; -use std::borrow::Cow; +use std::{borrow::Cow, str::SplitInclusive}; const P: StandardPolicy<'static> = StandardPolicy::new(); +const ARMOR_BEGIN: &str = "-----BEGIN "; +const ARMOR_END: &str = "-----END"; +const ARMOR_DASHES: &str = "-----"; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionMethod { PGP, @@ -23,135 +27,168 @@ pub struct EncryptionParams { pub method: EncryptionMethod, } -#[allow(clippy::type_complexity)] pub fn parse_public_key(pk: &PublicKey) -> Result, Cow<'static, str>> { - let bytes_ = pk.key.as_bytes(); - let mut bytes = bytes_.iter().enumerate(); - let mut buf = vec![]; let mut method = None; - let mut certs: Vec> = vec![]; + let mut certs: Vec> = Vec::new(); + let mut has_usable_pgp_key = false; - loop { - // Find start of PEM block - let mut start_pos = 0; - for (pos, &ch) in bytes.by_ref() { - if ch.is_ascii_whitespace() { - continue; - } else if ch == b'-' { - start_pos = pos; - break; - } else { - return Ok(None); - } + for block in ArmoredBlocks::new(&pk.key) { + let block = block?; + if contains_ignore_ascii_case(block.label, "PRIVATE") { + return Err("Private keys cannot be used as public keys".into()); } - - // Find block type - for (_, &ch) in bytes.by_ref() { - match ch { - b'-' => (), - b'\n' => break, - _ => { - if ch.is_ascii() { - buf.push(ch.to_ascii_uppercase()); - } else { - return Ok(None); - } - } - } - } - if buf.is_empty() { - break; - } - - // Find type - let tag = std::str::from_utf8(&buf).unwrap(); - if tag.contains("CERTIFICATE") { - if method.is_some_and(|m| m == EncryptionMethod::PGP) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::SMIME); - } - } else if tag.contains("PGP") { - if method.is_some_and(|m| m == EncryptionMethod::SMIME) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::PGP); - } + let block_method = if contains_ignore_ascii_case(block.label, "CERTIFICATE") { + EncryptionMethod::SMIME + } else if contains_ignore_ascii_case(block.label, "PGP") { + EncryptionMethod::PGP } else { - // Ignore block - let mut found_end = false; - for (_, &ch) in bytes.by_ref() { - if ch == b'-' { - found_end = true; - } else if ch == b'\n' && found_end { - break; - } - } - buf.clear(); continue; - } + }; - // Collect base64 - buf.clear(); - let mut found_end = false; - let mut end_pos = 0; - for (pos, &ch) in bytes.by_ref() { - match ch { - b'-' => { - found_end = true; - } - b'\n' => { - if found_end { - end_pos = pos; - break; - } - } - _ => { - if !ch.is_ascii_whitespace() { - buf.push(ch); - } - } + match method { + Some(method) if method != block_method => { + return Err("Cannot mix OpenPGP and S/MIME certificates".into()); } + _ => method = Some(block_method), } - // Decode base64 - let cert = base64_decode(&buf) - .ok_or_else(|| Cow::from("Failed to decode base64 certificate."))? - .into_boxed_slice(); - match method.unwrap() { - EncryptionMethod::PGP => match Cert::from_bytes(bytes_) { - Ok(cert) => { - if !has_pgp_keys(cert) { - return Err("Could not find any suitable keys in OpenPGP public key".into()); - } - certs.push( - bytes_ - .get(start_pos..end_pos + 1) - .unwrap_or_default() - .into(), - ); + match block_method { + EncryptionMethod::PGP => { + let cert = Cert::from_bytes(block.armored.as_bytes()) + .map_err(|err| format!("Failed to decode OpenPGP public key: {err}"))?; + if cert.is_tsk() { + return Err("Private keys cannot be used as public keys".into()); } - Err(err) => { - return Err(format!("Failed to decode OpenPGP public key: {err}").into()); - } - }, + has_usable_pgp_key |= has_pgp_keys(&cert); + certs.push(block.armored.as_bytes().into()); + } EncryptionMethod::SMIME => { - if let Err(err) = rasn::der::decode::(&cert) { - return Err(format!("Failed to decode X509 certificate: {err}").into()); + let cert = LENIENT + .decode(block.body) + .map_err(|_| Cow::from("Failed to decode base64 certificate."))?; + let is_complete = rasn::der::decode_with_remainder::(&cert) + .map_err(|err| format!("Failed to decode X509 certificate: {err}"))? + .1 + .is_empty(); + if !is_complete { + return Err("Unexpected data after X509 certificate".into()); } - certs.push(cert); + certs.push(cert.into_boxed_slice()); } } - buf.clear(); } - Ok(method.map(|method| EncryptionParams { - method, - certs: certs.into_boxed_slice(), - })) + match method { + Some(EncryptionMethod::PGP) if !has_usable_pgp_key => { + Err("Could not find any suitable keys in OpenPGP public key".into()) + } + Some(method) => Ok(Some(EncryptionParams { + method, + certs: certs.into_boxed_slice(), + })), + None if pk.key.trim().is_empty() => Ok(None), + None => Err("No OpenPGP public key or X.509 certificate found".into()), + } } -fn has_pgp_keys(cert: Cert) -> bool { +struct ArmoredBlock<'x> { + label: &'x str, + armored: &'x str, + body: &'x str, +} + +struct ArmoredBlocks<'x> { + text: &'x str, + lines: SplitInclusive<'x, char>, + offset: usize, +} + +impl<'x> ArmoredBlocks<'x> { + fn new(text: &'x str) -> Self { + ArmoredBlocks { + text, + lines: text.split_inclusive('\n'), + offset: 0, + } + } + + fn next_line(&mut self) -> Option<(usize, &'x str)> { + let line = self.lines.next()?; + let start = self.offset; + self.offset += line.len(); + Some((start, line)) + } + + fn fail( + &mut self, + reason: &'static str, + ) -> Option, Cow<'static, str>>> { + self.lines = "".split_inclusive('\n'); + Some(Err(reason.into())) + } +} + +impl<'x> Iterator for ArmoredBlocks<'x> { + type Item = Result, Cow<'static, str>>; + + fn next(&mut self) -> Option { + let (block_start, label) = loop { + let (line_start, line) = self.next_line()?; + let begin = line.trim(); + if begin.is_empty() { + continue; + } + let Some(label) = begin + .strip_prefix(ARMOR_BEGIN) + .and_then(|label| label.strip_suffix(ARMOR_DASHES)) + .filter(|label| !label.is_empty()) + else { + return self.fail("Unexpected text outside of a PEM or OpenPGP armor block"); + }; + break (line_start + line.len() - line.trim_start().len(), label); + }; + let body_start = self.offset; + + loop { + let Some((line_start, line)) = self.next_line() else { + return self.fail("PEM or OpenPGP armor block is missing its END line"); + }; + let line = line.trim(); + if let Some(end_label) = line.strip_prefix(ARMOR_END) { + if end_label + .strip_prefix(' ') + .and_then(|end_label| end_label.strip_suffix(ARMOR_DASHES)) + != Some(label) + { + return self + .fail("PEM or OpenPGP armor END line does not match its BEGIN line"); + } + return match ( + self.text.get(block_start..self.offset), + self.text.get(body_start..line_start), + ) { + (Some(armored), Some(body)) => Some(Ok(ArmoredBlock { + label, + armored, + body, + })), + _ => self.fail("Invalid PEM or OpenPGP armor block"), + }; + } else if line.starts_with(ARMOR_BEGIN) { + return self.fail("PEM or OpenPGP armor block is missing its END line"); + } + } + } +} + +fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool { + haystack + .as_bytes() + .windows(needle.len()) + .any(|window| window.eq_ignore_ascii_case(needle.as_bytes())) +} + +fn has_pgp_keys(cert: &Cert) -> bool { cert.keys() .with_policy(&P, None) .supported() @@ -161,3 +198,4 @@ fn has_pgp_keys(cert: Cert) -> bool { .next() .is_some() } + diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs deleted file mode 100644 index 9b305caa..00000000 --- a/crates/common/src/storage/index.rs +++ /dev/null @@ -1,784 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{auth::AccountTenantIds, sharing::notification::ShareNotification}; -use registry::schema::{ - enums::IndexDocumentType, - structs::{Task, TaskIndexDocument, TaskStatus}, -}; -use rkyv::{ - option::ArchivedOption, - primitive::{ArchivedU32, ArchivedU64}, - string::ArchivedString, -}; -use std::{borrow::Cow, fmt::Debug}; -use store::{ - Serialize, SerializeInfallible, - write::{ - Archive, Archiver, BatchBuilder, BlobLink, BlobOp, IntoOperations, Params, SearchIndex, - ValueClass, - }, -}; -use types::{ - acl::AclGrant, - blob_hash::BlobHash, - collection::{Collection, SyncCollection}, - field::Field, -}; -use utils::{cheeky_hash::CheekyHash, map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum IndexValue<'x> { - Index { - field: Field, - value: IndexItem<'x>, - }, - Property { - field: ValueClass, - value: IndexItem<'x>, - }, - SearchIndex { - index: SearchIndex, - hash: u64, - }, - Blob { - value: BlobHash, - }, - Quota { - used: u32, - }, - LogContainer { - sync_collection: SyncCollection, - }, - LogContainerProperty { - sync_collection: SyncCollection, - ids: Vec, - }, - LogItem { - sync_collection: SyncCollection, - prefix: Option, - }, - Acl { - value: Cow<'x, [AclGrant]>, - }, -} - -#[derive(Debug, Clone)] -pub enum IndexItem<'x> { - Vec(Vec), - Slice(&'x [u8]), - ShortInt([u8; std::mem::size_of::()]), - LongInt([u8; std::mem::size_of::()]), - Hash(CheekyHash), - None, -} - -impl IndexItem<'_> { - pub fn as_slice(&self) -> &[u8] { - match self { - IndexItem::Vec(v) => v, - IndexItem::Slice(s) => s, - IndexItem::ShortInt(s) => s, - IndexItem::LongInt(s) => s, - IndexItem::Hash(h) => h.as_bytes(), - IndexItem::None => &[], - } - } - - pub fn into_owned(self) -> Vec { - match self { - IndexItem::Vec(v) => v, - IndexItem::Slice(s) => s.to_vec(), - IndexItem::ShortInt(s) => s.to_vec(), - IndexItem::LongInt(s) => s.to_vec(), - IndexItem::Hash(h) => h.as_bytes().to_vec(), - IndexItem::None => vec![], - } - } - - pub fn is_empty(&self) -> bool { - match self { - IndexItem::Vec(v) => v.is_empty(), - IndexItem::Slice(s) => s.is_empty(), - IndexItem::None => true, - _ => false, - } - } - - pub fn is_none(&self) -> bool { - matches!(self, IndexItem::None) - } - - pub fn is_some(&self) -> bool { - !self.is_none() - } -} - -impl PartialEq for IndexItem<'_> { - fn eq(&self, other: &Self) -> bool { - self.as_slice() == other.as_slice() - } -} - -impl Eq for IndexItem<'_> {} - -impl std::hash::Hash for IndexItem<'_> { - fn hash(&self, state: &mut H) { - match self { - IndexItem::Vec(v) => v.as_slice().hash(state), - IndexItem::Slice(s) => s.hash(state), - IndexItem::ShortInt(s) => s.as_slice().hash(state), - IndexItem::LongInt(s) => s.as_slice().hash(state), - IndexItem::Hash(h) => h.hash(state), - IndexItem::None => 0.hash(state), - } - } -} - -impl From for IndexItem<'_> { - fn from(value: u32) -> Self { - IndexItem::ShortInt(value.to_be_bytes()) - } -} - -impl From<&u32> for IndexItem<'_> { - fn from(value: &u32) -> Self { - IndexItem::ShortInt(value.to_be_bytes()) - } -} - -impl From for IndexItem<'_> { - fn from(value: u64) -> Self { - IndexItem::LongInt(value.to_be_bytes()) - } -} - -impl From for IndexItem<'_> { - fn from(value: i64) -> Self { - IndexItem::LongInt(value.to_be_bytes()) - } -} - -impl<'x> From<&'x [u8]> for IndexItem<'x> { - fn from(value: &'x [u8]) -> Self { - IndexItem::Slice(value) - } -} - -impl From> for IndexItem<'_> { - fn from(value: Vec) -> Self { - IndexItem::Vec(value) - } -} - -impl<'x> From<&'x str> for IndexItem<'x> { - fn from(value: &'x str) -> Self { - IndexItem::Slice(value.as_bytes()) - } -} - -impl<'x> From<&'x String> for IndexItem<'x> { - fn from(value: &'x String) -> Self { - IndexItem::Slice(value.as_bytes()) - } -} - -impl From for IndexItem<'_> { - fn from(value: String) -> Self { - IndexItem::Vec(value.into_bytes()) - } -} - -impl<'x> From<&'x ArchivedString> for IndexItem<'x> { - fn from(value: &'x ArchivedString) -> Self { - IndexItem::Slice(value.as_bytes()) - } -} - -impl From for IndexItem<'_> { - fn from(value: ArchivedU32) -> Self { - IndexItem::ShortInt(value.to_native().to_be_bytes()) - } -} - -impl From<&ArchivedU32> for IndexItem<'_> { - fn from(value: &ArchivedU32) -> Self { - IndexItem::ShortInt(value.to_native().to_be_bytes()) - } -} - -impl From for IndexItem<'_> { - fn from(value: ArchivedU64) -> Self { - IndexItem::LongInt(value.to_native().to_be_bytes()) - } -} - -impl<'x, T: Into>> From> for IndexItem<'x> { - fn from(value: Option) -> Self { - match value { - Some(v) => v.into(), - None => IndexItem::None, - } - } -} - -impl<'x, T: Into>> From> for IndexItem<'x> { - fn from(value: ArchivedOption) -> Self { - match value { - ArchivedOption::Some(v) => v.into(), - ArchivedOption::None => IndexItem::None, - } - } -} - -pub trait IndexableObject: Sync + Send { - fn index_values(&self) -> impl Iterator>; -} - -pub trait IndexableAndSerializableObject: - IndexableObject - + rkyv::Archive - + for<'a> rkyv::Serialize< - rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, - rkyv::ser::allocator::ArenaHandle<'a>, - rkyv::rancor::Error, - >, - > -{ - fn is_versioned() -> bool; -} - -#[derive(Debug)] -pub struct ObjectIndexBuilder { - changed_by: u32, - tenant_id: Option, - current: Option>, - changes: Option, -} - -impl Default for ObjectIndexBuilder { - fn default() -> Self { - Self::new() - } -} - -impl ObjectIndexBuilder { - pub fn new() -> Self { - Self { - current: None, - changes: None, - tenant_id: None, - changed_by: u32::MAX, - } - } - - pub fn with_current(mut self, current: Archive) -> Self { - self.current = Some(current); - self - } - - pub fn with_changes(mut self, changes: N) -> Self { - self.changes = Some(changes); - self - } - - pub fn with_current_opt(mut self, current: Option>) -> Self { - self.current = current; - self - } - - pub fn changes(&self) -> Option<&N> { - self.changes.as_ref() - } - - pub fn changes_mut(&mut self) -> Option<&mut N> { - self.changes.as_mut() - } - - pub fn current(&self) -> Option<&Archive> { - self.current.as_ref() - } - - pub fn with_changed_by(mut self, ids: AccountTenantIds) -> Self { - self.tenant_id = ids.tenant_id; - self.changed_by = ids.account_id; - self - } - - pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { - self.tenant_id = tenant_id; - self - } -} - -impl IntoOperations - for ObjectIndexBuilder -{ - fn build(self, batch: &mut BatchBuilder) -> trc::Result<()> { - match (self.current, self.changes) { - (None, Some(changes)) => { - // Insertion - for item in changes.index_values() { - build_index(batch, item, self.changed_by, self.tenant_id, true); - } - if N::is_versioned() { - let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; - batch.set_fnc( - Field::ARCHIVE, - Params::with_capacity(2).with_bytes(bytes).with_u64(offset), - |params, ids| { - let change_id = ids.current_change_id()?; - let archive = params.bytes(0); - let offset = params.u64(1); - - let mut bytes = Vec::with_capacity(archive.len()); - bytes.extend_from_slice(&archive[..offset as usize]); - bytes.extend_from_slice(&change_id.to_be_bytes()[..]); - bytes.push(archive.last().copied().unwrap()); // Marker - Ok(bytes) - }, - ); - } else { - batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?); - } - } - (Some(current), Some(changes)) => { - // Update - batch.assert_value(Field::ARCHIVE, ¤t); - for (current, change) in current.inner.index_values().zip(changes.index_values()) { - if current != change { - merge_index(batch, current, change, self.changed_by, self.tenant_id)?; - } else { - match current { - IndexValue::LogContainer { sync_collection } => { - batch.log_container_update(sync_collection); - } - IndexValue::LogItem { - sync_collection, - prefix, - } => { - batch.log_item_update(sync_collection, prefix); - } - _ => (), - } - } - } - if N::is_versioned() { - let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; - batch.set_fnc( - Field::ARCHIVE, - Params::with_capacity(2).with_bytes(bytes).with_u64(offset), - |params, ids| { - let change_id = ids.current_change_id()?; - let archive = params.bytes(0); - let offset = params.u64(1); - - let mut bytes = Vec::with_capacity(archive.len()); - bytes.extend_from_slice(&archive[..offset as usize]); - bytes.extend_from_slice(&change_id.to_be_bytes()[..]); - bytes.push(archive.last().copied().unwrap()); // Marker - Ok(bytes) - }, - ); - } else { - batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?); - } - } - (Some(current), None) => { - // Deletion - batch.assert_value(Field::ARCHIVE, ¤t); - for item in current.inner.index_values() { - build_index(batch, item, self.changed_by, self.tenant_id, false); - } - - batch.clear(Field::ARCHIVE); - } - (None, None) => unreachable!(), - } - - Ok(()) - } -} - -fn build_index( - batch: &mut BatchBuilder, - item: IndexValue<'_>, - changed_by: u32, - tenant_id: Option, - set: bool, -) { - match item { - IndexValue::Index { field, value } => { - if !value.is_empty() { - if set { - batch.index(field, value.into_owned()); - } else { - batch.unindex(field, value.into_owned()); - } - } - } - IndexValue::SearchIndex { index, .. } => { - let task = TaskIndexDocument { - account_id: batch.last_account_id().unwrap().into(), - document_id: batch.last_document_id().unwrap().into(), - document_type: match index { - SearchIndex::Email => IndexDocumentType::Email, - SearchIndex::Calendar => IndexDocumentType::Calendar, - SearchIndex::Contacts => IndexDocumentType::Contacts, - SearchIndex::File => IndexDocumentType::File, - SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), - }, - status: TaskStatus::now(), - }; - batch.schedule_task(if set { - Task::IndexDocument(task) - } else { - Task::UnindexDocument(task) - }); - } - IndexValue::Property { field, value } => { - if !value.is_none() { - if set { - batch.set(field, value.into_owned()); - } else { - batch.clear(field); - } - } - } - IndexValue::Blob { value } => { - if set { - batch.set( - BlobOp::Link { - hash: value, - to: BlobLink::Document, - }, - vec![], - ); - } else { - batch.clear(BlobOp::Link { - hash: value, - to: BlobLink::Document, - }); - } - } - IndexValue::Acl { value } => { - let object_account_id = batch.last_account_id().unwrap_or_default(); - let object_type = batch.last_collection().unwrap_or(Collection::None); - let object_id = batch.last_document_id().unwrap_or_default(); - let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); - - for item in value.as_ref() { - if set { - batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.log_share_notification( - notification_id, - item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights: Default::default(), - new_rights: item.grants, - name: Default::default(), - }, - ); - } else { - batch.acl_revoke(item.account_id); - batch.log_share_notification( - notification_id, - item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights: item.grants, - new_rights: Default::default(), - name: Default::default(), - }, - ); - } - } - } - IndexValue::Quota { used } => { - let value = if set { used as i64 } else { -(used as i64) }; - - batch.add(ValueClass::Quota, value); - - if let Some(tenant_id) = tenant_id { - batch.add(ValueClass::TenantQuota(tenant_id), value); - } - } - IndexValue::LogItem { - sync_collection, - prefix, - } => { - if set { - batch.log_item_insert(sync_collection, prefix); - } else { - batch.log_item_delete(sync_collection, prefix); - } - } - IndexValue::LogContainer { sync_collection } => { - if set { - batch.log_container_insert(sync_collection); - } else { - batch.log_container_delete(sync_collection); - } - } - IndexValue::LogContainerProperty { - sync_collection, - ids, - } => { - for parent_id in ids { - batch.log_container_property_change(sync_collection, parent_id); - } - } - } -} - -fn merge_index( - batch: &mut BatchBuilder, - current: IndexValue<'_>, - change: IndexValue<'_>, - changed_by: u32, - tenant_id: Option, -) -> trc::Result<()> { - match (current, change) { - ( - IndexValue::Index { - field, - value: old_value, - }, - IndexValue::Index { - value: new_value, .. - }, - ) => { - if !old_value.is_empty() { - batch.unindex(field, old_value.into_owned()); - } - - if !new_value.is_empty() { - batch.index(field, new_value.into_owned()); - } - } - (IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => { - batch.schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: batch.last_account_id().unwrap().into(), - document_id: batch.last_document_id().unwrap().into(), - document_type: match index { - SearchIndex::Email => IndexDocumentType::Email, - SearchIndex::Calendar => IndexDocumentType::Calendar, - SearchIndex::Contacts => IndexDocumentType::Contacts, - SearchIndex::File => IndexDocumentType::File, - SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), - }, - status: TaskStatus::now(), - })); - } - ( - IndexValue::Property { - field: old_field, - value: old_value, - }, - IndexValue::Property { - field: new_field, - value: new_value, - .. - }, - ) => { - if old_field != new_field { - batch.clear(old_field); - batch.set(new_field, new_value.into_owned()); - } else if new_value != old_value { - if new_value.is_some() { - batch.set(old_field, new_value.into_owned()); - } else { - batch.clear(old_field); - } - } - } - (IndexValue::Blob { value: old_hash }, IndexValue::Blob { value: new_hash }) => { - batch.clear(BlobOp::Link { - hash: old_hash, - to: BlobLink::Document, - }); - batch.set( - BlobOp::Link { - hash: new_hash, - to: BlobLink::Document, - }, - vec![], - ); - } - (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => { - let has_old_acl = !old_acl.is_empty(); - let has_new_acl = !new_acl.is_empty(); - - if !has_old_acl && !has_new_acl { - return Ok(()); - } - - let object_account_id = batch.last_account_id().unwrap_or_default(); - let object_type = batch.last_collection().unwrap_or(Collection::None); - let object_id = batch.last_document_id().unwrap_or_default(); - let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); - - match (has_old_acl, has_new_acl) { - (true, true) => { - // Remove deleted ACLs - for current_item in old_acl.as_ref() { - if !new_acl - .iter() - .any(|item| item.account_id == current_item.account_id) - { - batch.acl_revoke(current_item.account_id); - batch.log_share_notification( - notification_id, - current_item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights: current_item.grants, - new_rights: Default::default(), - name: Default::default(), - }, - ); - } - } - - // Update ACLs - for item in new_acl.as_ref() { - let mut add_item = true; - let mut old_rights = Bitmap::default(); - for current_item in old_acl.as_ref() { - if item.account_id == current_item.account_id { - if item.grants == current_item.grants { - add_item = false; - } else { - old_rights = current_item.grants; - } - break; - } - } - if add_item { - batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.log_share_notification( - notification_id, - item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights, - new_rights: item.grants, - name: Default::default(), - }, - ); - } - } - } - (false, true) => { - // Add all ACLs - for item in new_acl.as_ref() { - batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.log_share_notification( - notification_id, - item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights: Default::default(), - new_rights: item.grants, - name: Default::default(), - }, - ); - } - } - (true, false) => { - // Remove all ACLs - for item in old_acl.as_ref() { - batch.acl_revoke(item.account_id); - batch.log_share_notification( - notification_id, - item.account_id, - ShareNotification { - object_account_id, - object_id, - object_type, - changed_by, - old_rights: item.grants, - new_rights: Default::default(), - name: Default::default(), - }, - ); - } - } - _ => {} - } - } - (IndexValue::Quota { used: old_used }, IndexValue::Quota { used: new_used }) => { - let value = new_used as i64 - old_used as i64; - batch.add(ValueClass::Quota, value); - - if let Some(tenant_id) = tenant_id { - batch.add(ValueClass::TenantQuota(tenant_id), value); - } - } - ( - IndexValue::LogItem { - sync_collection, - prefix: old_prefix, - }, - IndexValue::LogItem { - prefix: new_prefix, .. - }, - ) => { - batch.log_item_delete(sync_collection, old_prefix); - batch.log_item_insert(sync_collection, new_prefix); - } - ( - IndexValue::LogContainerProperty { - sync_collection, - ids: old_ids, - }, - IndexValue::LogContainerProperty { ids: new_ids, .. }, - ) => { - for parent_id in &old_ids { - if !new_ids.contains(parent_id) { - batch.log_container_property_change(sync_collection, *parent_id); - } - } - for parent_id in new_ids { - if !old_ids.contains(&parent_id) { - batch.log_container_property_change(sync_collection, parent_id); - } - } - } - _ => unreachable!(), - } - - Ok(()) -} - -impl IndexableObject for () { - fn index_values(&self) -> impl Iterator> { - std::iter::empty() - } -} - -impl IndexableAndSerializableObject for () { - fn is_versioned() -> bool { - false - } -} diff --git a/crates/common/src/storage/index/build.rs b/crates/common/src/storage/index/build.rs new file mode 100644 index 00000000..dfb42c79 --- /dev/null +++ b/crates/common/src/storage/index/build.rs @@ -0,0 +1,464 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{IndexValue, RevokeCleanup}; +use crate::sharing::notification::ShareNotification; +use registry::schema::{ + enums::SharedObjectType, + structs::{Task, TaskShareRevokeCleanup, TaskStatus}, +}; +use store::{ + SerializeInfallible, U32_LEN, + write::{ + BatchBuilder, BlobLink, BlobOp, Patch, PatchSource, PendingId, QueueDocumentId, SetValue, + SizedSetValue, Slot, TaskId, ValueClass, blob::BlobLinkValue, now, + }, +}; +use types::collection::Collection; +use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; + +pub(crate) fn build_index( + batch: &mut BatchBuilder, + item: IndexValue<'_>, + changed_by: u32, + set: bool, + pending_id: Option, +) { + match item { + IndexValue::Index { field, value } => { + if !value.is_empty() { + if set { + batch.index(field, value.into_owned()); + } else { + batch.unindex(field, value.into_owned()); + } + } + } + IndexValue::ParentProperty { field, value } => { + if set { + batch.set(field, parent_property(value, pending_id)); + } else { + batch.clear(field); + } + } + IndexValue::SearchIndex { index, .. } => { + let account_id = batch.last_account_id().unwrap(); + + if set { + batch.queue_document_index(index, account_id, QueueDocumentId::Current); + } else { + batch.queue_document_unindex(index, account_id, QueueDocumentId::Current); + } + } + IndexValue::Property { field, value } => { + if !value.is_none() { + if set { + batch.set(field, value.into_owned()); + } else { + batch.clear(field); + } + } + } + IndexValue::Blob { value, size } => { + if set { + batch.set( + BlobOp::Link { + hash: value, + to: BlobLink::Document, + }, + BlobLinkValue::new(size), + ); + } else { + batch.clear(BlobOp::Link { + hash: value, + to: BlobLink::Document, + }); + } + } + IndexValue::Acl { value } => { + let object_account_id = batch.last_account_id().unwrap_or_default(); + let object_type = batch.last_collection().unwrap_or(Collection::None); + let object_id = batch.last_document_id().unwrap_or(PendingId::Assigned(0)); + let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); + + for item in value.as_ref() { + if set { + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); + batch.log_share_notification( + notification_id, + item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights: Default::default(), + new_rights: item.grants, + ..Default::default() + } + .into_value(object_id), + ); + } else { + batch.acl_revoke(item.account_id); + batch.log_share_notification( + notification_id, + item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights: item.grants, + new_rights: Default::default(), + ..Default::default() + } + .into_value(object_id), + ); + } + } + } + IndexValue::Quota { used } => { + let value = if set { used as i64 } else { -(used as i64) }; + + batch.add(ValueClass::Quota, value); + + } + IndexValue::ObjectCount(kind) => { + batch.add(ValueClass::ObjectCount(kind), if set { 1 } else { -1 }); + } + IndexValue::LogItem { + sync_collection, + prefix, + } => { + if set { + batch.log_item_insert(sync_collection, prefix); + } else { + batch.log_item_delete(sync_collection, prefix); + } + } + IndexValue::LogContainer { sync_collection } => { + if set { + batch.log_container_insert(sync_collection); + } else { + batch.log_container_delete(sync_collection); + } + } + IndexValue::LogContainerProperty { + sync_collection, + ids, + } => { + for parent_id in ids { + batch.log_container_property_change(sync_collection, parent_id); + } + } + } +} + +pub(crate) fn merge_index( + batch: &mut BatchBuilder, + current: IndexValue<'_>, + change: IndexValue<'_>, + changed_by: u32, + revoke_cleanup: RevokeCleanup, + pending_id: Option, +) -> trc::Result<()> { + match (current, change) { + ( + IndexValue::Index { + field, + value: old_value, + }, + IndexValue::Index { + value: new_value, .. + }, + ) => { + if !old_value.is_empty() { + batch.unindex(field, old_value.into_owned()); + } + + if !new_value.is_empty() { + batch.index(field, new_value.into_owned()); + } + } + ( + IndexValue::ParentProperty { + field: old_field, .. + }, + IndexValue::ParentProperty { field, value }, + ) => { + if old_field != field { + batch.clear(old_field); + } + batch.set(field, parent_property(value, pending_id)); + } + (IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => { + batch.queue_document_index( + index, + batch.last_account_id().unwrap(), + QueueDocumentId::Current, + ); + } + ( + IndexValue::Property { + field: old_field, + value: old_value, + }, + IndexValue::Property { + field: new_field, + value: new_value, + .. + }, + ) => { + if old_field != new_field { + batch.clear(old_field); + batch.set(new_field, new_value.into_owned()); + } else if new_value != old_value { + if new_value.is_some() { + batch.set(old_field, new_value.into_owned()); + } else { + batch.clear(old_field); + } + } + } + ( + IndexValue::Blob { + value: old_hash, .. + }, + IndexValue::Blob { + value: new_hash, + size, + }, + ) => { + batch.clear(BlobOp::Link { + hash: old_hash, + to: BlobLink::Document, + }); + batch.set( + BlobOp::Link { + hash: new_hash, + to: BlobLink::Document, + }, + BlobLinkValue::new(size), + ); + } + (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => { + let has_old_acl = !old_acl.is_empty(); + let has_new_acl = !new_acl.is_empty(); + + if !has_old_acl && !has_new_acl { + return Ok(()); + } + + let object_account_id = batch.last_account_id().unwrap_or_default(); + let object_type = batch.last_collection().unwrap_or(Collection::None); + let object_id = batch.last_document_id().unwrap_or(PendingId::Assigned(0)); + let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); + let mut cleanup_type = match (revoke_cleanup, object_id) { + (RevokeCleanup::Scheduled, PendingId::Assigned(document_id)) => { + SharedObjectType::from_collection(object_type) + .map(|shared_type| (shared_type, document_id)) + } + _ => None, + }; + + match (has_old_acl, has_new_acl) { + (true, true) => { + // Remove deleted ACLs + for current_item in old_acl.as_ref() { + if !new_acl + .iter() + .any(|item| item.account_id == current_item.account_id) + { + batch.acl_revoke(current_item.account_id); + if let Some((shared_type, document_id)) = cleanup_type.take() { + schedule_revoke_cleanup( + batch, + object_account_id, + shared_type, + document_id, + ); + } + batch.log_share_notification( + notification_id, + current_item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights: current_item.grants, + new_rights: Default::default(), + ..Default::default() + } + .into_value(object_id), + ); + } + } + + // Update ACLs + for item in new_acl.as_ref() { + let mut add_item = true; + let mut old_rights = Bitmap::default(); + for current_item in old_acl.as_ref() { + if item.account_id == current_item.account_id { + if item.grants == current_item.grants { + add_item = false; + } else { + old_rights = current_item.grants; + } + break; + } + } + if add_item { + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); + batch.log_share_notification( + notification_id, + item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights, + new_rights: item.grants, + ..Default::default() + } + .into_value(object_id), + ); + } + } + } + (false, true) => { + // Add all ACLs + for item in new_acl.as_ref() { + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); + batch.log_share_notification( + notification_id, + item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights: Default::default(), + new_rights: item.grants, + ..Default::default() + } + .into_value(object_id), + ); + } + } + (true, false) => { + // Remove all ACLs + for item in old_acl.as_ref() { + batch.acl_revoke(item.account_id); + if let Some((shared_type, document_id)) = cleanup_type.take() { + schedule_revoke_cleanup( + batch, + object_account_id, + shared_type, + document_id, + ); + } + batch.log_share_notification( + notification_id, + item.account_id, + ShareNotification { + object_account_id, + object_type, + changed_by, + old_rights: item.grants, + new_rights: Default::default(), + ..Default::default() + } + .into_value(object_id), + ); + } + } + _ => {} + } + } + (IndexValue::Quota { used: old_used }, IndexValue::Quota { used: new_used }) => { + let value = new_used as i64 - old_used as i64; + batch.add(ValueClass::Quota, value); + + } + (IndexValue::ObjectCount(old_kind), IndexValue::ObjectCount(new_kind)) => { + batch + .add(ValueClass::ObjectCount(old_kind), -1) + .add(ValueClass::ObjectCount(new_kind), 1); + } + ( + IndexValue::LogItem { + sync_collection, + prefix: old_prefix, + }, + IndexValue::LogItem { + prefix: new_prefix, .. + }, + ) => { + batch.log_item_delete(sync_collection, old_prefix); + batch.log_item_insert(sync_collection, new_prefix); + } + ( + IndexValue::LogContainerProperty { + sync_collection, + ids: old_ids, + }, + IndexValue::LogContainerProperty { ids: new_ids, .. }, + ) => { + for parent_id in &old_ids { + if !new_ids.contains(parent_id) { + batch.log_container_property_change(sync_collection, *parent_id); + } + } + for parent_id in new_ids { + if !old_ids.contains(&parent_id) { + batch.log_container_property_change(sync_collection, parent_id); + } + } + } + _ => unreachable!(), + } + + Ok(()) +} + +fn parent_property(value: Vec, pending_id: Option) -> SizedSetValue { + match pending_id { + Some(slot) if value.len() >= U32_LEN => SetValue::patched( + value, + vec![Patch { + offset: 0, + source: PatchSource::SlotBeU32(slot), + }], + ), + _ => SetValue::fixed(value), + } +} + +#[cfg(not(stalwart_tests))] +pub(crate) const REVOKE_CLEANUP_WINDOW: u64 = 300; +#[cfg(stalwart_tests)] +pub(crate) const REVOKE_CLEANUP_WINDOW: u64 = 1; + +pub(crate) fn revoke_cleanup_schedule(now: u64) -> (u64, u64) { + let window = now / REVOKE_CLEANUP_WINDOW; + (window, (window + 2) * REVOKE_CLEANUP_WINDOW) +} + +fn schedule_revoke_cleanup( + batch: &mut BatchBuilder, + account_id: u32, + object_type: SharedObjectType, + document_id: u32, +) { + let (window, due) = revoke_cleanup_schedule(now()); + batch.schedule_task_with_id( + TaskId::shared_object(account_id, object_type.collection(), document_id, window), + Task::ShareRevokeCleanup(TaskShareRevokeCleanup { + account_id: account_id.into(), + document_id: document_id.into(), + object_type, + status: TaskStatus::at(due as i64), + }), + ); +} diff --git a/crates/common/src/storage/index/mod.rs b/crates/common/src/storage/index/mod.rs new file mode 100644 index 00000000..ae507f73 --- /dev/null +++ b/crates/common/src/storage/index/mod.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod build; +pub mod object; +pub mod presence; +pub mod split; +pub mod value; + +pub use object::*; +pub use presence::*; +pub use split::*; +pub use value::*; + +use crate::auth::ChangedBy; +use build::{build_index, merge_index}; +use store::write::{BatchBuilder, IntoOperations, Slot}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum RevokeCleanup { + #[default] + Scheduled, + Inline, +} + +#[derive(Debug)] +pub struct ObjectIndexBuilder { + changed_by: u32, + current: Option, + changes: Option, + pending_id: Option, + revoke_cleanup: RevokeCleanup, +} + +impl Default for ObjectIndexBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ObjectIndexBuilder { + pub fn new() -> Self { + Self { + current: None, + changes: None, + changed_by: u32::MAX, + pending_id: None, + revoke_cleanup: RevokeCleanup::Scheduled, + } + } + + pub fn with_current(mut self, current: C) -> Self { + self.current = Some(current); + self + } + + pub fn with_changes(mut self, changes: N) -> Self { + self.changes = Some(changes); + self + } + + pub fn with_pending_id(mut self, slot: Slot) -> Self { + self.pending_id = Some(slot); + self + } + + pub fn with_pending_id_opt(mut self, slot: Option) -> Self { + self.pending_id = slot; + self + } + + pub fn with_current_opt(mut self, current: Option) -> Self { + self.current = current; + self + } + + pub fn changes(&self) -> Option<&N> { + self.changes.as_ref() + } + + pub fn changes_mut(&mut self) -> Option<&mut N> { + self.changes.as_mut() + } + + pub fn current(&self) -> Option<&C> { + self.current.as_ref() + } + + pub fn with_changed_by(mut self, ids: ChangedBy) -> Self { + self.changed_by = ids.account_id; + self + } + + + pub fn with_revoke_cleanup(mut self, revoke_cleanup: RevokeCleanup) -> Self { + self.revoke_cleanup = revoke_cleanup; + self + } +} + +impl IntoOperations for ObjectIndexBuilder { + fn build(self, batch: &mut BatchBuilder) -> trc::Result<()> { + match (self.current, self.changes) { + (None, Some(changes)) => { + // Insertion + for item in changes.index_values() { + #[cfg(not(feature = "enterprise"))] + build_index(batch, item, self.changed_by, true, self.pending_id); + } + changes.serialize_into(batch, self.pending_id)?; + } + (Some(current), Some(changes)) => { + // Update + current.assert(batch); + for (current, change) in current.index_values().zip(changes.index_values()) { + if current != change + || (self.pending_id.is_some() + && matches!(change, IndexValue::ParentProperty { .. })) + { + #[cfg(not(feature = "enterprise"))] + merge_index( + batch, + current, + change, + self.changed_by, + self.revoke_cleanup, + self.pending_id, + )?; + } else { + match current { + IndexValue::LogContainer { sync_collection } => { + batch.log_container_update(sync_collection); + } + IndexValue::LogItem { + sync_collection, + prefix, + } => { + batch.log_item_update(sync_collection, prefix); + } + _ => (), + } + } + } + changes.serialize_into(batch, self.pending_id)?; + } + (Some(current), None) => { + // Deletion + current.assert(batch); + for item in current.index_values() { + #[cfg(not(feature = "enterprise"))] + build_index(batch, item, self.changed_by, false, None); + } + current.clear(batch); + } + (None, None) => unreachable!(), + } + + Ok(()) + } +} + diff --git a/crates/common/src/storage/index/object.rs b/crates/common/src/storage/index/object.rs new file mode 100644 index 00000000..df69ece6 --- /dev/null +++ b/crates/common/src/storage/index/object.rs @@ -0,0 +1,246 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::IndexValue; +use store::{ + Serialize, U32_LEN, U64_LEN, + write::{ + Archive, ArchiveBytes, ArchiveCompression, Archiver, AssignedIds, BatchBuilder, Patch, + PatchSource, SerializeWithIds, SetValue, SizedSetValue, Slot, metadata::MetadataClass, + }, +}; +use types::{field::Field, metadata::MetadataKinds}; + +pub trait IndexableObject: Sync + Send { + fn index_values(&self) -> impl Iterator>; + + fn metadata_kinds(&self) -> MetadataKinds { + MetadataKinds::NONE + } +} + +pub trait IndexableAndSerializableObject: + IndexableObject + + ArchiveCompression + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + Vec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + > +{ + const PENDING_ID_FIELD: Option = None; + + fn is_versioned() -> bool; + + fn set_pending_id(&mut self, _document_id: u32) -> trc::Result<()> { + Err(trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Object carries no pending document id.")) + } + + fn size_hint(&self) -> usize { + unreachable!("object carries no pending document id") + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ArchivedIdField { + offset: usize, + base: u32, +} + +impl ArchivedIdField { + pub const fn new(offset: usize) -> Self { + Self { offset, base: 0 } + } + + pub const fn one_based(self) -> Self { + Self { + offset: self.offset, + base: 1, + } + } + + pub fn patch(&self, root_offset: usize, slot: Slot) -> Patch { + Patch { + offset: (root_offset + self.offset) as u32, + source: PatchSource::SlotArchivedU32 { + slot, + base: self.base, + }, + } + } +} + +pub trait CurrentObject: Send + Sync { + fn assert(&self, batch: &mut BatchBuilder); + fn index_values(&self) -> impl Iterator>; + fn clear(&self, batch: &mut BatchBuilder); +} + +impl CurrentObject for Archive { + #[inline(always)] + fn assert(&self, batch: &mut BatchBuilder) { + batch.assert_value(Field::ARCHIVE, self); + } + + #[inline(always)] + fn index_values(&self) -> impl Iterator> { + self.inner.index_values() + } + + #[inline(always)] + fn clear(&self, batch: &mut BatchBuilder) { + batch.clear(Field::ARCHIVE); + if !self.inner.metadata_kinds().is_empty() { + batch.clear(MetadataClass::Shared); + } + } +} + +impl CurrentObject for () { + fn assert(&self, _: &mut BatchBuilder) {} + fn index_values(&self) -> impl Iterator> { + std::iter::empty() + } + fn clear(&self, _: &mut BatchBuilder) {} +} + +pub trait SerializableObject: IndexableObject { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()>; +} + +struct PendingObject +where + T: IndexableAndSerializableObject, +{ + archiver: Archiver, + slot: Slot, + size_hint: usize, +} + +impl SerializeWithIds for PendingObject { + fn serialize_with_ids(&mut self, ids: &AssignedIds) -> trc::Result<(Vec, Option)> { + self.archiver.inner.set_pending_id(ids.slot(self.slot))?; + + let mut archive = self.archiver.serialize()?; + if T::is_versioned() { + let offset = archive.len() - U64_LEN - 1; + archive[offset..offset + U64_LEN] + .copy_from_slice(&ids.current_change_id().to_be_bytes()); + } + let hash = Archive::::extract_hash(&archive); + + Ok((archive, hash)) + } + + fn size_hint(&self) -> usize { + self.size_hint + } +} + +impl From> for SizedSetValue { + fn from(object: PendingObject) -> Self { + SetValue::serializable(object) + } +} + +pub fn serialize_object( + object: T, + batch: &mut BatchBuilder, + pending_id: Option, +) -> trc::Result<()> { + if let Some(slot) = pending_id { + if let Some(field) = T::PENDING_ID_FIELD { + const { + assert!(std::mem::align_of::() == 1); + }; + debug_assert!( + field.offset + U32_LEN <= std::mem::size_of::(), + "pending id field is outside the archived root" + ); + + let archiver = Archiver::new(object); + let (payload_len, archive) = if T::is_versioned() { + archiver.with_version().serialize_patchable()? + } else { + archiver.serialize_patchable()? + }; + let root_offset = payload_len as usize - std::mem::size_of::(); + let mut patches = vec![field.patch(root_offset, slot)]; + + if T::is_versioned() { + patches.push(Patch { + offset: (archive.len() - U64_LEN - 1) as u32, + source: PatchSource::ChangeIdBe, + }); + } + + batch + .set_archive_hash(None) + .set(Field::ARCHIVE, (archive, patches)); + } else { + let size_hint = object.size_hint(); + let archiver = Archiver::new(object); + let archiver = if T::is_versioned() { + archiver.with_version() + } else { + archiver + }; + + batch.set_archive_hash(None).set( + Field::ARCHIVE, + PendingObject { + archiver, + slot, + size_hint, + }, + ); + } + } else if T::is_versioned() { + let (offset, archive) = Archiver::new(object).serialize_versioned()?; + batch + .set_archive_hash(Archive::::extract_hash(&archive)) + .set( + Field::ARCHIVE, + ( + archive, + vec![Patch { + offset: offset as u32, + source: PatchSource::ChangeIdBe, + }], + ), + ); + } else { + let archive = Archiver::new(object).serialize()?; + batch + .set_archive_hash(Archive::::extract_hash(&archive)) + .set(Field::ARCHIVE, archive); + } + + Ok(()) +} + +impl IndexableObject for () { + fn index_values(&self) -> impl Iterator> { + std::iter::empty() + } +} + +impl IndexableAndSerializableObject for () { + fn is_versioned() -> bool { + false + } +} + +impl SerializableObject for () { + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + serialize_object(self, batch, pending_id) + } +} diff --git a/crates/common/src/storage/index/presence.rs b/crates/common/src/storage/index/presence.rs new file mode 100644 index 00000000..115c9feb --- /dev/null +++ b/crates/common/src/storage/index/presence.rs @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{IndexableAndSerializableObject, IndexableObject, SerializableObject}; +use rkyv::{ + api::high::{HighDeserializer, HighValidator}, + bytecheck::CheckBytes, + rancor, +}; +use store::write::{Archive, BatchBuilder}; +use trc::AddContext; +use types::{collection::Collection, field::Field, metadata::MetadataKinds}; + +pub trait PresenceFlags: SerializableObject + IndexableAndSerializableObject { + const COLLECTION: Collection; + const TRACKED: MetadataKinds; + + fn set_metadata_kinds(&mut self, kinds: MetadataKinds); +} + +pub trait RewritePresence: rkyv::Archive { + fn rewrite_presence( + current: &Archive<&Self::Archived>, + kinds: MetadataKinds, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result<()>; +} + +impl RewritePresence for T +where + T: PresenceFlags, + T::Archived: for<'a> CheckBytes> + + rkyv::Deserialize> + + Sync + + Send, + for<'a> &'a T::Archived: IndexableObject, +{ + fn rewrite_presence( + current: &Archive<&T::Archived>, + kinds: MetadataKinds, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + ) -> trc::Result<()> { + if current.inner.metadata_kinds().bits() == kinds.bits() & T::TRACKED.bits() { + return Ok(()); + } + let mut object = current.deserialize::().caused_by(trc::location!())?; + object.set_metadata_kinds(kinds); + rewrite_archive( + current, + object, + T::COLLECTION, + account_id, + document_id, + batch, + ) + } +} + +pub fn rewrite_archive( + current: &Archive, + changes: impl SerializableObject, + collection: Collection, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, +) -> trc::Result<()> { + batch + .with_account_id(account_id) + .with_collection(collection) + .with_document(document_id) + .assert_value(Field::ARCHIVE, current); + changes.serialize_into(batch, None) +} diff --git a/crates/common/src/storage/index/split.rs b/crates/common/src/storage/index/split.rs new file mode 100644 index 00000000..bf0b4eda --- /dev/null +++ b/crates/common/src/storage/index/split.rs @@ -0,0 +1,261 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + CurrentObject, IndexValue, IndexableAndSerializableObject, IndexableObject, ObjectIndexBuilder, + SerializableObject, serialize_object, +}; +use crate::auth::ChangedBy; +use store::{ + Serialize, + write::{Archive, ArchiveCompression, Archiver, BatchBuilder, Slot, metadata::MetadataClass}, +}; +use types::{field::Field, metadata::MetadataKinds}; + +pub trait SerializableContent: + ArchiveCompression + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + Vec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + > + Sync + + Send +{ +} + +impl SerializableContent for T where + T: ArchiveCompression + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + Vec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + > + Sync + + Send +{ +} + +pub trait SplitObject: IndexableAndSerializableObject +where + Self::Archived: ArchivedSplitObject, +{ + type Content: SerializableContent; + type Context: Copy; + + const CONTENT_FIELD: Field; + + fn meta_hash(&self) -> u32; + + fn set_etag(&mut self, etag: u32); + + fn etag_value(&self) -> u32; + + fn refresh_from_content(&mut self, content: &Self::Content, ctx: Self::Context); + + fn full_index_values<'x>(&'x self, content: &'x Self::Content) -> Vec>; +} + +pub trait ArchivedSplitObject: Sync + Send { + type ArchivedContent: Sync + Send; + + const CONTENT_FIELD: Field; + + fn meta_hash(&self) -> u32; + + fn etag(&self) -> u32; + + fn meta_index_values(&self) -> Vec>; + + fn full_index_values<'x>(&'x self, content: &'x Self::ArchivedContent) -> Vec>; + + fn metadata_kinds(&self) -> MetadataKinds { + MetadataKinds::NONE + } +} + +pub enum GroupwareWrite +where + M::Archived: ArchivedSplitObject, +{ + Full { + meta: M, + content: M::Content, + content_archive: Vec, + }, + MetaOnly(M), +} + +impl GroupwareWrite +where + M::Archived: ArchivedSplitObject, +{ + pub fn full(mut meta: M, content: M::Content, ctx: M::Context) -> trc::Result { + let archiver = Archiver::new(content); + let content_archive = archiver.serialize()?; + let content = archiver.inner; + + meta.refresh_from_content(&content, ctx); + meta.set_etag(content_hash(&content_archive) ^ meta.meta_hash()); + + Ok(GroupwareWrite::Full { + meta, + content, + content_archive, + }) + } + + pub fn meta_only(mut meta: M, current: &M::Archived) -> Self { + let payload = current.etag() ^ current.meta_hash(); + meta.set_etag(payload ^ meta.meta_hash()); + + GroupwareWrite::MetaOnly(meta) + } + + pub fn meta(&self) -> &M { + match self { + GroupwareWrite::Full { meta, .. } | GroupwareWrite::MetaOnly(meta) => meta, + } + } +} + +fn content_hash(archive: &[u8]) -> u32 { + store::xxhash_rust::xxh3::xxh3_64(archive) as u32 +} + +pub struct SplitUpdate<'x, M: SplitObject> +where + M::Archived: ArchivedSplitObject, +{ + current: SplitCurrent<'x, M::Archived>, + changes: GroupwareWrite, +} + +impl<'x, M: SplitObject> SplitUpdate<'x, M> +where + M::Archived: ArchivedSplitObject, +{ + pub fn full( + current_meta: Archive<&'x M::Archived>, + current_content: &'x ::ArchivedContent, + meta: M, + content: M::Content, + ctx: M::Context, + ) -> trc::Result { + Ok(SplitUpdate { + current: SplitCurrent::Full { + meta: current_meta, + content: current_content, + }, + changes: GroupwareWrite::full(meta, content, ctx)?, + }) + } + + pub fn meta_only(current_meta: Archive<&'x M::Archived>, meta: M) -> Self { + let changes = GroupwareWrite::meta_only(meta, current_meta.inner); + + SplitUpdate { + current: SplitCurrent::MetaOnly(current_meta), + changes, + } + } + + pub fn etag(&self) -> String { + format!("\"{}\"", self.changes.meta().etag_value()) + } + + pub fn into_builder( + self, + changed_by: ChangedBy, + pending_id: Option, + ) -> ObjectIndexBuilder, GroupwareWrite> { + ObjectIndexBuilder::new() + .with_current(self.current) + .with_changes(self.changes) + .with_changed_by(changed_by) + .with_pending_id_opt(pending_id) + } +} + +impl IndexableObject for GroupwareWrite +where + M::Archived: ArchivedSplitObject, +{ + fn index_values(&self) -> impl Iterator> { + match self { + GroupwareWrite::Full { meta, content, .. } => { + meta.full_index_values(content).into_iter() + } + GroupwareWrite::MetaOnly(meta) => meta.index_values().collect::>().into_iter(), + } + } +} + +impl SerializableObject for GroupwareWrite +where + M::Archived: ArchivedSplitObject, +{ + fn serialize_into(self, batch: &mut BatchBuilder, pending_id: Option) -> trc::Result<()> { + match self { + GroupwareWrite::Full { + meta, + content_archive, + .. + } => { + serialize_object(meta, batch, pending_id)?; + batch.set(M::CONTENT_FIELD, content_archive); + } + GroupwareWrite::MetaOnly(meta) => serialize_object(meta, batch, pending_id)?, + } + + batch.set_archive_hash(None); + + Ok(()) + } +} + +pub enum SplitCurrent<'x, M: ArchivedSplitObject> { + Full { + meta: Archive<&'x M>, + content: &'x M::ArchivedContent, + }, + MetaOnly(Archive<&'x M>), +} + +impl<'x, M: ArchivedSplitObject> SplitCurrent<'x, M> { + pub fn meta(&self) -> &Archive<&'x M> { + match self { + SplitCurrent::Full { meta, .. } | SplitCurrent::MetaOnly(meta) => meta, + } + } +} + +impl CurrentObject for SplitCurrent<'_, M> { + fn assert(&self, batch: &mut BatchBuilder) { + batch.assert_value(Field::ARCHIVE, self.meta()); + } + + fn index_values(&self) -> impl Iterator> { + match self { + SplitCurrent::Full { meta, content } => { + meta.inner.full_index_values(content).into_iter() + } + SplitCurrent::MetaOnly(meta) => meta.inner.meta_index_values().into_iter(), + } + } + + fn clear(&self, batch: &mut BatchBuilder) { + batch.clear(Field::ARCHIVE); + batch.clear(M::CONTENT_FIELD); + if !self.meta().inner.metadata_kinds().is_empty() { + batch.clear(MetadataClass::Shared); + } + } +} diff --git a/crates/common/src/storage/index/value.rs b/crates/common/src/storage/index/value.rs new file mode 100644 index 00000000..4f7ffa06 --- /dev/null +++ b/crates/common/src/storage/index/value.rs @@ -0,0 +1,223 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use registry::schema::enums::StorageQuota; +use rkyv::{ + option::ArchivedOption, + primitive::{ArchivedU32, ArchivedU64}, + string::ArchivedString, +}; +use std::borrow::Cow; +use store::write::{PendingId, SearchIndex, ValueClass}; +use types::{acl::AclGrant, blob_hash::BlobHash, collection::SyncCollection, field::Field}; + +pub const SEARCH_HASH_UNCHANGED: u64 = 0; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IndexValue<'x> { + Index { + field: Field, + value: IndexItem<'x>, + }, + Property { + field: ValueClass, + value: IndexItem<'x>, + }, + ParentProperty { + field: ValueClass, + value: Vec, + }, + SearchIndex { + index: SearchIndex, + hash: u64, + }, + Blob { + value: BlobHash, + size: u64, + }, + Quota { + used: u32, + }, + ObjectCount(StorageQuota), + LogContainer { + sync_collection: SyncCollection, + }, + LogContainerProperty { + sync_collection: SyncCollection, + ids: Vec, + }, + LogItem { + sync_collection: SyncCollection, + prefix: Option, + }, + Acl { + value: Cow<'x, [AclGrant]>, + }, +} + +#[derive(Debug, Clone)] +pub enum IndexItem<'x> { + Vec(Vec), + Slice(&'x [u8]), + ShortInt([u8; std::mem::size_of::()]), + LongInt([u8; std::mem::size_of::()]), + None, +} + +impl IndexItem<'_> { + pub fn as_slice(&self) -> &[u8] { + match self { + IndexItem::Vec(v) => v, + IndexItem::Slice(s) => s, + IndexItem::ShortInt(s) => s, + IndexItem::LongInt(s) => s, + IndexItem::None => &[], + } + } + + pub fn into_owned(self) -> Vec { + match self { + IndexItem::Vec(v) => v, + IndexItem::Slice(s) => s.to_vec(), + IndexItem::ShortInt(s) => s.to_vec(), + IndexItem::LongInt(s) => s.to_vec(), + IndexItem::None => vec![], + } + } + + pub fn is_empty(&self) -> bool { + match self { + IndexItem::Vec(v) => v.is_empty(), + IndexItem::Slice(s) => s.is_empty(), + IndexItem::None => true, + _ => false, + } + } + + pub fn is_none(&self) -> bool { + matches!(self, IndexItem::None) + } + + pub fn is_some(&self) -> bool { + !self.is_none() + } +} + +impl PartialEq for IndexItem<'_> { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl Eq for IndexItem<'_> {} + +impl std::hash::Hash for IndexItem<'_> { + fn hash(&self, state: &mut H) { + match self { + IndexItem::Vec(v) => v.as_slice().hash(state), + IndexItem::Slice(s) => s.hash(state), + IndexItem::ShortInt(s) => s.as_slice().hash(state), + IndexItem::LongInt(s) => s.as_slice().hash(state), + IndexItem::None => 0.hash(state), + } + } +} + +impl From for IndexItem<'_> { + fn from(value: u32) -> Self { + IndexItem::ShortInt(value.to_be_bytes()) + } +} + +impl From<&u32> for IndexItem<'_> { + fn from(value: &u32) -> Self { + IndexItem::ShortInt(value.to_be_bytes()) + } +} + +impl From for IndexItem<'_> { + fn from(value: u64) -> Self { + IndexItem::LongInt(value.to_be_bytes()) + } +} + +impl From for IndexItem<'_> { + fn from(value: i64) -> Self { + IndexItem::LongInt(value.to_be_bytes()) + } +} + +impl<'x> From<&'x [u8]> for IndexItem<'x> { + fn from(value: &'x [u8]) -> Self { + IndexItem::Slice(value) + } +} + +impl From> for IndexItem<'_> { + fn from(value: Vec) -> Self { + IndexItem::Vec(value) + } +} + +impl<'x> From<&'x str> for IndexItem<'x> { + fn from(value: &'x str) -> Self { + IndexItem::Slice(value.as_bytes()) + } +} + +impl<'x> From<&'x String> for IndexItem<'x> { + fn from(value: &'x String) -> Self { + IndexItem::Slice(value.as_bytes()) + } +} + +impl From for IndexItem<'_> { + fn from(value: String) -> Self { + IndexItem::Vec(value.into_bytes()) + } +} + +impl<'x> From<&'x ArchivedString> for IndexItem<'x> { + fn from(value: &'x ArchivedString) -> Self { + IndexItem::Slice(value.as_bytes()) + } +} + +impl From for IndexItem<'_> { + fn from(value: ArchivedU32) -> Self { + IndexItem::ShortInt(value.to_native().to_be_bytes()) + } +} + +impl From<&ArchivedU32> for IndexItem<'_> { + fn from(value: &ArchivedU32) -> Self { + IndexItem::ShortInt(value.to_native().to_be_bytes()) + } +} + +impl From for IndexItem<'_> { + fn from(value: ArchivedU64) -> Self { + IndexItem::LongInt(value.to_native().to_be_bytes()) + } +} + +impl<'x, T: Into>> From> for IndexItem<'x> { + fn from(value: Option) -> Self { + match value { + Some(v) => v.into(), + None => IndexItem::None, + } + } +} + +impl<'x, T: Into>> From> for IndexItem<'x> { + fn from(value: ArchivedOption) -> Self { + match value { + ArchivedOption::Some(v) => v.into(), + ArchivedOption::None => IndexItem::None, + } + } +} diff --git a/crates/common/src/storage/metadata/change.rs b/crates/common/src/storage/metadata/change.rs new file mode 100644 index 00000000..03d37b7a --- /dev/null +++ b/crates/common/src/storage/metadata/change.rs @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{MetadataLog, MetadataWrite, PrivateMetadataWrite, StoredContainer}; +use crate::auth::AccountCache; +use store::write::metadata::MetadataBuf; +use types::{ + collection::Collection, + metadata::{EncodedMetadata, MetadataEdit}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContainerChange { + previous: Option, + next: Option, + edit: MetadataEdit, +} + +impl ContainerChange { + pub fn new( + previous: Option<&MetadataBuf>, + next: Option, + edit: MetadataEdit, + ) -> Option { + match (previous, &next) { + (Some(previous), Some(next)) if previous.view().as_bytes() == next.as_bytes() => None, + (None, None) => None, + _ => Some(ContainerChange { + previous: previous.map(StoredContainer::from), + next, + edit, + }), + } + } + + pub fn copied(next: EncodedMetadata) -> Self { + ContainerChange { + previous: None, + next: Some(next), + edit: MetadataEdit::Write, + } + } + + pub fn previous(&self) -> Option<&StoredContainer> { + self.previous.as_ref() + } + + pub fn next(&self) -> Option<&EncodedMetadata> { + self.next.as_ref() + } + + pub fn edit(&self) -> MetadataEdit { + self.edit + } + + pub fn into_next(self) -> Option { + self.next + } + + pub fn into_write( + self, + owner: &AccountCache, + collection: Collection, + log: MetadataLog, + ) -> MetadataWrite { + MetadataWrite { + account_id: owner.id, + collection, + previous: self.previous, + next: self.next, + log, + } + } + + pub fn into_private_write( + self, + owner_id: u32, + viewer: &AccountCache, + collection: Collection, + log: MetadataLog, + ) -> PrivateMetadataWrite { + PrivateMetadataWrite { + owner_id, + viewer_id: viewer.id, + collection, + previous: self.previous, + next: self.next, + log, + } + } +} diff --git a/crates/common/src/storage/metadata/cleanup.rs b/crates/common/src/storage/metadata/cleanup.rs new file mode 100644 index 00000000..8f28f8ea --- /dev/null +++ b/crates/common/src/storage/metadata/cleanup.rs @@ -0,0 +1,339 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + BatchCursor, MetadataViewerEntry, StoredEntries, StoredEntry, add_quota, metadata_key, + purge::PrivateKey, +}; +use crate::{ + Server, auth::ChangedBy, cache::invalidate::CacheInvalidationBuilder, ipc::CacheInvalidation, +}; +use store::{ + IterateParams, + roaring::RoaringBitmap, + write::{BatchBuilder, LogCollection, assert::AssertValue, metadata::MetadataClass}, +}; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + metadata::MetadataKinds, +}; + +#[derive(Debug)] +pub struct ContainerCleanup { + collection: Collection, + entries: StoredEntries, +} + +struct QuotaRefund { + bytes: i64, +} + +impl ContainerCleanup { + pub fn empty(collection: Collection) -> Self { + ContainerCleanup { + collection, + entries: StoredEntries::default(), + } + } + + pub fn release_or_assert_absent( + &self, + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + ) { + if !self.release(batch, account_id, document_id) { + batch + .with_account_id(account_id) + .with_collection(self.collection) + .with_document(document_id) + .assert_value(MetadataClass::Shared, AssertValue::None); + } + } + + fn release(&self, batch: &mut BatchBuilder, account_id: u32, document_id: u32) -> bool { + let Some(entry) = self.entries.get(document_id) else { + return false; + }; + batch + .with_account_id(account_id) + .with_collection(self.collection); + #[cfg(not(feature = "enterprise"))] + entry.release(batch); + true + } +} + +impl Server { + pub async fn preload_container_cleanup( + &self, + changed_by: ChangedBy, + account_id: u32, + collection: Collection, + flagged: &RoaringBitmap, + ) -> trc::Result { + if flagged.is_empty() { + return Ok(ContainerCleanup::empty(collection)); + } + let entries = self + .stored_metadata_entries(account_id, collection, flagged) + .await + .caused_by(trc::location!())?; + self.container_cleanup(changed_by, account_id, collection, entries) + .await + } + + pub async fn container_cleanup_from( + &self, + changed_by: ChangedBy, + account_id: u32, + collection: Collection, + entries: impl IntoIterator, + ) -> trc::Result { + self.container_cleanup( + changed_by, + account_id, + collection, + entries.into_iter().collect(), + ) + .await + } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + async fn container_cleanup( + &self, + changed_by: ChangedBy, + account_id: u32, + collection: Collection, + entries: StoredEntries, + ) -> trc::Result { + Ok(ContainerCleanup { + collection, + entries, + }) + } + + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] + pub async fn release_or_read( + &self, + cleanup: &ContainerCleanup, + batch: &mut BatchBuilder, + changed_by: ChangedBy, + account_id: u32, + document_id: u32, + kinds: MetadataKinds, + ) -> trc::Result<()> { + if cleanup.release(batch, account_id, document_id) || kinds.is_empty() { + return Ok(()); + } + if let Some(entry) = self + .stored_entry( + account_id, + cleanup.collection, + MetadataClass::Shared, + document_id, + ) + .await + .caused_by(trc::location!())? + { + let cursor = BatchCursor::save(batch); + batch + .with_account_id(account_id) + .with_collection(cleanup.collection); + #[cfg(not(feature = "enterprise"))] + entry.release(batch); + cursor.restore(batch); + } + Ok(()) + } + + + pub async fn destroy_viewer_metadata(&self, viewer_id: u32) -> trc::Result<()> { + let owners = self.metadata_owners(viewer_id).await?; + if owners.is_empty() { + return Ok(()); + } + + let store = &self.core.storage.data; + let class = MetadataClass::Private { viewer: viewer_id }; + let mut batch = BatchBuilder::new(); + for &(owner_id, collection) in &owners { + let collection_id = u8::from(collection); + store + .delete_range( + metadata_key(owner_id, collection_id, 0, class), + metadata_key(owner_id, collection_id, u32::MAX, class), + ) + .await + .caused_by(trc::location!())?; + batch + .with_account_id(owner_id) + .with_collection(collection) + .clear(MetadataClass::Viewer { viewer: viewer_id }); + } + + let mut logs = owners + .iter() + .map(|(owner_id, collection)| (*owner_id, u8::from(SyncCollection::from(*collection)))) + .collect::>(); + logs.sort_unstable(); + logs.dedup(); + for (owner_id, sync_collection) in logs { + let log = LogCollection::Private { + collection: SyncCollection::from(sync_collection), + viewer: viewer_id, + }; + store + .delete_range(log.log_key(owner_id, 0), log.log_key(owner_id, u64::MAX)) + .await + .caused_by(trc::location!())?; + } + + store + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + self.invalidate_caches(CacheInvalidation::PrivateMetadata(viewer_id).into()) + .await + } + + pub async fn destroy_owner_metadata(&self, owner_id: u32) -> trc::Result<()> { + let mut viewers = self.all_metadata_viewers(owner_id).await?; + if viewers.is_empty() { + return Ok(()); + } + viewers.sort_unstable_by_key(|link| link.viewer_id); + + let mut usage: Vec<(u32, i64)> = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(owner_id, 0, 0, MetadataClass::Private { viewer: 0 }), + metadata_key( + owner_id, + u8::MAX, + u32::MAX, + MetadataClass::Private { viewer: u32::MAX }, + ), + ) + .ascending(), + |key, value| { + let viewer_id = PrivateKey::parse(key)?.viewer_id; + let size = value.len() as i64; + match usage.last_mut() { + Some((last, total)) if *last == viewer_id => *total += size, + _ => usage.push((viewer_id, size)), + } + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(owner_id))?; + + let mut batch = BatchBuilder::new(); + let mut invalidations = CacheInvalidationBuilder::default(); + for links in viewers.chunk_by(|a, b| a.viewer_id == b.viewer_id) { + let Some(viewer_id) = links.first().map(|link| link.viewer_id) else { + continue; + }; + let bytes = usage + .binary_search_by_key(&viewer_id, |(viewer_id, _)| *viewer_id) + .ok() + .and_then(|position| usage.get(position)) + .map_or(0, |(_, bytes)| *bytes); + #[cfg(not(feature = "enterprise"))] + let refund = if bytes != 0 { + self.try_account(viewer_id) + .await? + .map(|_| QuotaRefund { bytes }) + } else { + None + }; + release_viewer(&mut batch, owner_id, links, refund); + invalidations.invalidate(CacheInvalidation::PrivateMetadata(viewer_id)); + } + + let result = self + .core + .storage + .data + .write_batch(&mut batch) + .await + .caused_by(trc::location!()); + self.invalidate_caches(invalidations).await?; + result.map(|_| ()) + } + + pub async fn metadata_used_quota(&self, account_id: u32) -> trc::Result { + let store = &self.core.storage.data; + let mut total = 0i64; + store + .iterate( + IterateParams::new( + metadata_key(account_id, 0, 0, MetadataClass::Shared), + metadata_key(account_id, u8::MAX, u32::MAX, MetadataClass::Shared), + ), + |_, value| { + total += value.len() as i64; + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))?; + + for (owner_id, collection) in self.metadata_owners(account_id).await? { + let collection = u8::from(collection); + let class = MetadataClass::Private { viewer: account_id }; + store + .iterate( + IterateParams::new( + metadata_key(owner_id, collection, 0, class), + metadata_key(owner_id, collection, u32::MAX, class), + ), + |_, value| { + total += value.len() as i64; + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(owner_id))?; + } + + Ok(total) + } +} + +fn release_viewer( + batch: &mut BatchBuilder, + owner_id: u32, + links: &[MetadataViewerEntry], + refund: Option, +) { + let Some(viewer_id) = links.first().map(|link| link.viewer_id) else { + return; + }; + batch.with_account_id(viewer_id); + if let Some(refund) = refund { + #[cfg(not(feature = "enterprise"))] + add_quota(batch, -refund.bytes); + } + for link in links { + batch + .with_collection(link.collection) + .clear(MetadataClass::Owner { owner: owner_id }); + } + batch.with_account_id(owner_id); + for link in links { + batch + .with_collection(link.collection) + .clear(MetadataClass::Viewer { viewer: viewer_id }); + } + batch.commit_point(); +} + diff --git a/crates/common/src/storage/metadata/containers.rs b/crates/common/src/storage/metadata/containers.rs new file mode 100644 index 00000000..b6e9f437 --- /dev/null +++ b/crates/common/src/storage/metadata/containers.rs @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::StoredContainer; +use crate::Server; +use store::{roaring::RoaringBitmap, write::metadata::MetadataBuf}; +use types::{collection::Collection, metadata::MetadataView}; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct MetadataContainers { + containers: Vec<(u32, MetadataBuf)>, +} + +impl MetadataContainers { + fn new(mut containers: Vec<(u32, MetadataBuf)>) -> Self { + if !containers.is_sorted_by_key(|(document_id, _)| *document_id) { + containers.sort_unstable_by_key(|(document_id, _)| *document_id); + } + MetadataContainers { containers } + } + + pub fn get(&self, document_id: u32) -> Option<&MetadataBuf> { + self.containers + .binary_search_by_key(&document_id, |(document_id, _)| *document_id) + .ok() + .and_then(|position| self.containers.get(position)) + .map(|(_, container)| container) + } + + pub fn iter(&self) -> impl Iterator { + self.containers + .iter() + .map(|(document_id, container)| (*document_id, container)) + } + + pub fn is_empty(&self) -> bool { + self.containers.is_empty() + } +} + +impl FromIterator<(u32, MetadataBuf)> for MetadataContainers { + fn from_iter>(iter: T) -> Self { + MetadataContainers::new(iter.into_iter().collect()) + } +} + +impl Server { + pub async fn load_metadata_containers( + &self, + account_id: u32, + collection: Collection, + documents: &RoaringBitmap, + ) -> trc::Result { + let mut containers = Vec::new(); + if !documents.is_empty() { + self.metadata_containers(account_id, collection, documents, collect(&mut containers)) + .await?; + } + Ok(MetadataContainers::new(containers)) + } + + pub async fn load_private_metadata_containers( + &self, + owner_id: u32, + viewer_id: u32, + collection: Collection, + documents: &RoaringBitmap, + ) -> trc::Result { + let mut containers = Vec::new(); + if !documents.is_empty() { + self.private_metadata_containers( + owner_id, + viewer_id, + collection, + documents, + collect(&mut containers), + ) + .await?; + } + Ok(MetadataContainers::new(containers)) + } +} + +fn collect( + containers: &mut Vec<(u32, MetadataBuf)>, +) -> impl for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + Send + Sync + '_ +{ + |document_id, view, stored| { + containers.push(( + document_id, + MetadataBuf::from_view(&view, stored.size, stored.hash), + )); + Ok(true) + } +} diff --git a/crates/common/src/storage/metadata/mod.rs b/crates/common/src/storage/metadata/mod.rs new file mode 100644 index 00000000..6b4210df --- /dev/null +++ b/crates/common/src/storage/metadata/mod.rs @@ -0,0 +1,445 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +mod change; +mod cleanup; +mod containers; +mod purge; +mod quota; +mod read; +mod stored; +mod viewer; + +pub use change::ContainerChange; +pub use cleanup::ContainerCleanup; +pub use containers::MetadataContainers; +pub use read::MetadataViewerEntry; +pub use stored::{StoredEntries, StoredEntry}; +pub use viewer::{MetadataViewer, MetadataViewerCache, ViewerStates}; + +use crate::{Server, storage::archive::last_member}; +use store::{ + Deserialize, IterateParams, Key, SerializeInfallible, ValueKey, + dispatch::{DocumentSet, ScanShape}, + write::{ + AnyKey, ArchiveVersion, AssignedIds, BatchBuilder, MergeResult, PendingId, ValueClass, + assert::AssertValue, + metadata::{MetadataBuf, MetadataClass, StoredMetadata, ViewerState}, + }, +}; +use trc::AddContext; +use types::{ + collection::{ChangeGroup, Collection, SyncCollection}, + metadata::{EncodedMetadata, MetadataKinds, MetadataView}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct StoredContainer { + pub size: u32, + pub kinds: MetadataKinds, + pub hash: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetadataLog { + Item { prefix: Option }, + Container, + None, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct MetadataPresence { + pub before: MetadataKinds, + pub after: MetadataKinds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataWrite { + pub account_id: u32, + pub collection: Collection, + pub previous: Option, + pub next: Option, + pub log: MetadataLog, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrivateMetadataWrite { + pub owner_id: u32, + pub viewer_id: u32, + pub collection: Collection, + pub previous: Option, + pub next: Option, + pub log: MetadataLog, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct PrivateMetadataCommit { + changes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PrivateMetadataChange { + owner_id: u32, + viewer_id: u32, + collection: Collection, + is_logged: bool, + containers: i32, +} + +struct BatchCursor { + account_id: Option, + collection: Option, + document_id: Option, +} + +impl StoredContainer { + pub fn new(view: &MetadataView<'_>, stored: &[u8]) -> trc::Result { + Ok(StoredContainer { + size: u32::try_from(stored.len()).unwrap_or(u32::MAX), + kinds: view.kinds(), + hash: StoredMetadata::trailer_hash(stored)?, + }) + } + + pub fn assertion(previous: Option<&StoredContainer>) -> AssertValue { + previous.map_or(AssertValue::None, |previous| { + AssertValue::Archive(ArchiveVersion::Hashed { + hash: previous.hash, + }) + }) + } +} + +impl From<&MetadataBuf> for StoredContainer { + fn from(container: &MetadataBuf) -> Self { + StoredContainer { + size: container.stored_len(), + kinds: container.kinds(), + hash: container.hash(), + } + } +} + +impl MetadataPresence { + fn new(previous: Option<&StoredContainer>, next: Option<&EncodedMetadata>) -> Self { + MetadataPresence { + before: previous.map_or(MetadataKinds::NONE, |previous| previous.kinds), + after: next.map_or(MetadataKinds::NONE, |next| next.kinds()), + } + } + + pub fn has_changed(&self) -> bool { + self.before != self.after + } + + pub fn has_changed_for(&self, kinds: MetadataKinds) -> bool { + self.before.bits() & kinds.bits() != self.after.bits() & kinds.bits() + } +} + +impl BatchCursor { + fn save(batch: &BatchBuilder) -> Self { + BatchCursor { + account_id: batch.last_account_id(), + collection: batch.last_collection(), + document_id: batch.last_document_id(), + } + } + + fn restore(self, batch: &mut BatchBuilder) { + if let Some(account_id) = self.account_id { + batch.with_account_id(account_id); + } + if let Some(collection) = self.collection { + batch.with_collection(collection); + } + if let Some(document_id) = self.document_id + && batch.last_document_id() != Some(document_id) + { + batch.with_pending_document(document_id); + } + } +} + +fn add_quota( + batch: &mut BatchBuilder, + delta: i64, +) { + if delta != 0 { + batch.add(ValueClass::Quota, delta); + } +} + +fn stored_size(stored: Option<&StoredMetadata>) -> i64 { + stored.map_or(0, |stored| stored.len() as i64) +} + +fn previous_size(previous: Option<&StoredContainer>) -> i64 { + previous.map_or(0, |previous| i64::from(previous.size)) +} + +fn merge_viewer_state( + batch: &mut BatchBuilder, + class: MetadataClass, + owner_id: u32, + group: ChangeGroup, + is_logged: bool, + containers: i32, +) { + batch.merge_fnc(class, move |ids: &AssignedIds, current| { + let change_id = is_logged.then(|| ids.change_id(owner_id, group)).flatten(); + next_viewer_state(current, change_id, containers) + .map(|state| MergeResult::Update(state.serialize())) + }); +} + +fn next_viewer_state( + current: Option<&[u8]>, + change_id: Option, + containers: i32, +) -> trc::Result { + let mut state = current + .map(ViewerState::deserialize) + .transpose()? + .unwrap_or_default(); + if let Some(change_id) = change_id { + state.change_id = state.change_id.max(change_id); + } + state.containers = state.containers.saturating_add_signed(containers); + Ok(state) +} + +impl MetadataWrite { + pub fn build( + self, + document_id: PendingId, + batch: &mut BatchBuilder, + ) -> trc::Result { + let presence = MetadataPresence::new(self.previous.as_ref(), self.next.as_ref()); + if self.previous.is_none() && self.next.is_none() { + return Ok(presence); + } + + let stored = self.next.map(StoredMetadata::new).transpose()?; + let delta = stored_size(stored.as_ref()) - previous_size(self.previous.as_ref()); + let cursor = BatchCursor::save(batch); + + batch + .with_account_id(self.account_id) + .with_collection(self.collection) + .with_pending_document(document_id) + .assert_value( + MetadataClass::Shared, + StoredContainer::assertion(self.previous.as_ref()), + ); + match stored { + Some(stored) => batch.set(MetadataClass::Shared, stored.into_bytes()), + None => batch.clear(MetadataClass::Shared), + }; + #[cfg(not(feature = "enterprise"))] + add_quota(batch, delta); + + let sync_collection = SyncCollection::from(self.collection); + match self.log { + MetadataLog::Item { prefix } => batch.log_item_metadata(sync_collection, prefix), + MetadataLog::Container => batch.log_container_metadata(sync_collection), + MetadataLog::None => batch, + }; + + cursor.restore(batch); + Ok(presence) + } +} + +impl PrivateMetadataWrite { + pub fn build( + self, + document_id: PendingId, + batch: &mut BatchBuilder, + commit: &mut PrivateMetadataCommit, + ) -> trc::Result<()> { + if self.previous.is_none() && self.next.is_none() { + return Ok(()); + } + + let stored = self.next.map(StoredMetadata::new).transpose()?; + let delta = stored_size(stored.as_ref()) - previous_size(self.previous.as_ref()); + let containers: i32 = match (self.previous.is_some(), stored.is_some()) { + (false, true) => 1, + (true, false) => -1, + _ => 0, + }; + let viewer = self.viewer_id; + let owner = self.owner_id; + let sync_collection = SyncCollection::from(self.collection); + let cursor = BatchCursor::save(batch); + + batch + .with_account_id(owner) + .with_collection(self.collection) + .with_pending_document(document_id) + .assert_value( + MetadataClass::Private { viewer }, + StoredContainer::assertion(self.previous.as_ref()), + ); + match stored { + Some(stored) => batch.set(MetadataClass::Private { viewer }, stored.into_bytes()), + None => batch.clear(MetadataClass::Private { viewer }), + }; + + let is_logged = match self.log { + MetadataLog::Item { prefix } => { + batch.log_private_item_metadata(sync_collection, viewer, prefix); + true + } + MetadataLog::Container => { + batch.log_private_container_metadata(sync_collection, viewer); + true + } + MetadataLog::None => false, + }; + + if is_logged || containers != 0 { + let group = sync_collection.change_group(); + merge_viewer_state( + batch, + MetadataClass::Viewer { viewer }, + owner, + group, + is_logged, + containers, + ); + batch.with_account_id(viewer); + merge_viewer_state( + batch, + MetadataClass::Owner { owner }, + owner, + group, + is_logged, + containers, + ); + commit.record(PrivateMetadataChange { + owner_id: owner, + viewer_id: viewer, + collection: self.collection, + is_logged, + containers, + }); + } + + if delta != 0 { + batch.with_account_id(viewer); + #[cfg(not(feature = "enterprise"))] + add_quota(batch, delta); + } + + cursor.restore(batch); + Ok(()) + } +} + +impl PrivateMetadataCommit { + fn record(&mut self, change: PrivateMetadataChange) { + if let Some(existing) = self.changes.iter_mut().find(|existing| { + existing.owner_id == change.owner_id + && existing.viewer_id == change.viewer_id + && existing.collection == change.collection + }) { + existing.is_logged |= change.is_logged; + existing.containers = existing.containers.saturating_add(change.containers); + } else { + self.changes.push(change); + } + } + + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } +} + +fn metadata_key( + account_id: u32, + collection: u8, + document_id: u32, + class: MetadataClass, +) -> ValueKey { + ValueKey { + account_id, + collection, + document_id, + class: ValueClass::Metadata(class), + } +} + +impl Server { + async fn iterate_documents( + &self, + account_id: u32, + collection: u8, + class: MetadataClass, + documents: &I, + cb: &mut CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Send + Sync, + { + let subspace = ValueClass::Metadata(class).subspace(collection); + let key = + |document_id| metadata_key(account_id, collection, document_id, class).serialize(0); + let params = |begin, end| { + IterateParams::new( + AnyKey { + subspace, + key: begin, + }, + AnyKey { subspace, key: end }, + ) + }; + let range = |from_document_id, to_document_id| { + let mut end = key(to_document_id); + end.push(u8::MAX); + params(key(from_document_id), end) + }; + let store = &self.core.storage.data; + + match documents.scan_shape(store.scan_gap()) { + ScanShape::Range(from_document_id, to_document_id) => { + let Some(to_document_id) = last_member(documents, from_document_id, to_document_id) + else { + return Ok(()); + }; + store + .iterate(range(from_document_id, to_document_id), cb) + .await + } + ScanShape::Ranges(ranges) => { + store + .iterate_many( + ranges + .into_iter() + .map(|(from_document_id, to_document_id)| { + if from_document_id == to_document_id { + let point = key(from_document_id); + params(point.clone(), point) + } else { + range(from_document_id, to_document_id) + } + }) + .collect(), + cb, + ) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } +} + diff --git a/crates/common/src/storage/metadata/purge.rs b/crates/common/src/storage/metadata/purge.rs new file mode 100644 index 00000000..9abc2604 --- /dev/null +++ b/crates/common/src/storage/metadata/purge.rs @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{add_quota, metadata_key}; +use crate::{ + Server, cache::invalidate::CacheInvalidationBuilder, ipc::CacheInvalidation, + storage::archive::last_member, +}; +use store::{ + Deserialize, IterateParams, SerializeInfallible, U32_LEN, ValueKey, + dispatch::{DocumentSet, ScanShape}, + roaring::RoaringBitmap, + write::{ + BatchBuilder, LogCollection, MergeResult, ValueClass, + key::DeserializeBigEndian, + metadata::{MetadataClass, ViewerState}, + }, +}; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + field::Field, +}; + +const PRIVATE_KEY_SUFFIX: usize = U32_LEN + 1 + U32_LEN; +const ORPHAN_CHUNK: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct PrivateKey { + pub(super) collection: u8, + pub(super) viewer_id: u32, + pub(super) document_id: u32, +} + +impl Server { + pub async fn purge_private_metadata( + &self, + owner_id: u32, + max_history: Option, + ) -> trc::Result<()> { + let viewers = self.all_metadata_viewers(owner_id).await?; + if viewers.is_empty() { + return Ok(()); + } + + if viewers.iter().any(|entry| entry.state.containers > 0) { + self.purge_orphaned_private_metadata(owner_id).await?; + } + + if let Some(max_history) = max_history { + let mut logs = viewers + .iter() + .map(|entry| { + ( + entry.viewer_id, + u8::from(SyncCollection::from(entry.collection)), + ) + }) + .collect::>(); + logs.sort_unstable(); + logs.dedup(); + for (viewer, collection) in logs { + self.truncate_change_log( + owner_id, + LogCollection::Private { + collection: SyncCollection::from(collection), + viewer, + }, + max_history, + ) + .await?; + } + } + + Ok(()) + } + + async fn purge_orphaned_private_metadata(&self, owner_id: u32) -> trc::Result<()> { + let mut containers = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(owner_id, 0, 0, MetadataClass::Private { viewer: 0 }), + metadata_key( + owner_id, + u8::MAX, + u32::MAX, + MetadataClass::Private { viewer: u32::MAX }, + ), + ) + .no_values(), + |key, _| { + containers.push(PrivateKey::parse(key)?); + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(owner_id))?; + containers.sort_unstable(); + + let mut orphaned_viewers = Vec::new(); + for by_collection in containers.chunk_by(|a, b| a.collection == b.collection) { + let Some(first) = by_collection.first() else { + continue; + }; + let collection = Collection::from(first.collection); + if !is_object_collection(collection) { + continue; + } + let candidates = by_collection + .iter() + .map(|key| key.document_id) + .collect::(); + let existing = self + .existing_documents(owner_id, collection, &candidates) + .await?; + if existing.len() == candidates.len() { + continue; + } + + for by_viewer in by_collection.chunk_by(|a, b| a.viewer_id == b.viewer_id) { + let orphans = by_viewer + .iter() + .map(|key| key.document_id) + .filter(|document_id| !existing.contains(*document_id)) + .collect::(); + let Some(viewer_id) = by_viewer.first().map(|key| key.viewer_id) else { + continue; + }; + if !orphans.is_empty() { + self.remove_orphaned_containers(owner_id, viewer_id, collection, &orphans) + .await?; + orphaned_viewers.push(viewer_id); + } + } + } + + if orphaned_viewers.is_empty() { + return Ok(()); + } + orphaned_viewers.sort_unstable(); + orphaned_viewers.dedup(); + let mut invalidations = CacheInvalidationBuilder::default(); + for viewer_id in orphaned_viewers { + invalidations.invalidate(CacheInvalidation::PrivateMetadata(viewer_id)); + } + self.invalidate_caches(invalidations).await + } + + async fn remove_orphaned_containers( + &self, + owner_id: u32, + viewer_id: u32, + collection: Collection, + orphans: &RoaringBitmap, + ) -> trc::Result<()> { + let class = MetadataClass::Private { viewer: viewer_id }; + let entries = self + .stored_entries(owner_id, collection, class, orphans) + .await?; + if entries.is_empty() { + return Ok(()); + } + let account = self.try_account(viewer_id).await?; + + let mut batch = BatchBuilder::new(); + for chunk in entries.chunks(ORPHAN_CHUNK) { + batch.with_account_id(owner_id).with_collection(collection); + let mut total = 0i64; + for entry in chunk { + entry.clear(&mut batch, class); + total += i64::from(entry.size); + } + let count = u32::try_from(chunk.len()).unwrap_or(u32::MAX); + decrement_containers( + &mut batch, + MetadataClass::Viewer { viewer: viewer_id }, + count, + ); + batch.with_account_id(viewer_id); + decrement_containers(&mut batch, MetadataClass::Owner { owner: owner_id }, count); + #[cfg(not(feature = "enterprise"))] + if account.is_some() { + add_quota(&mut batch, -total); + } + + if batch.is_large_batch() { + self.write_orphan_batch(&mut batch).await?; + batch = BatchBuilder::new(); + } + } + + if !batch.is_empty() { + self.write_orphan_batch(&mut batch).await?; + } + Ok(()) + } + + async fn write_orphan_batch(&self, batch: &mut BatchBuilder) -> trc::Result<()> { + match self.core.storage.data.write_batch(batch).await { + Ok(_) => Ok(()), + Err(err) if err.is_assertion_failure() => { + trc::event!( + Store(trc::StoreEvent::AssertValueFailed), + Details = "Private metadata changed during the orphan sweep", + CausedBy = trc::location!() + ); + Ok(()) + } + Err(err) => Err(err.caused_by(trc::location!())), + } + } + + async fn existing_documents( + &self, + account_id: u32, + collection: Collection, + documents: &RoaringBitmap, + ) -> trc::Result { + let collection = u8::from(collection); + let key = |document_id| ValueKey { + account_id, + collection, + document_id, + class: ValueClass::Property(Field::ARCHIVE.into()), + }; + let mut existing = RoaringBitmap::new(); + let mut collect = |key: &[u8], _: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + existing.insert(document_id); + } + Ok(true) + }; + let store = &self.core.storage.data; + + match documents.scan_shape(store.scan_gap()) { + ScanShape::Range(from_document_id, to_document_id) => { + let Some(to_document_id) = last_member(documents, from_document_id, to_document_id) + else { + return Ok(existing); + }; + store + .iterate( + IterateParams::new(key(from_document_id), key(to_document_id)).no_values(), + &mut collect, + ) + .await + } + ScanShape::Ranges(ranges) => { + store + .iterate_many( + ranges + .into_iter() + .map(|(from_document_id, to_document_id)| { + IterateParams::new(key(from_document_id), key(to_document_id)) + .no_values() + }) + .collect(), + &mut collect, + ) + .await + } + } + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + })?; + + Ok(existing) + } +} + +fn is_object_collection(collection: Collection) -> bool { + matches!( + collection, + Collection::Email + | Collection::Mailbox + | Collection::SieveScript + | Collection::Calendar + | Collection::CalendarEvent + | Collection::AddressBook + | Collection::ContactCard + | Collection::FileNode + ) +} + +impl PrivateKey { + pub(super) fn parse(key: &[u8]) -> trc::Result { + key.len() + .checked_sub(PRIVATE_KEY_SUFFIX) + .and_then(|offset| key.get(offset..)) + .and_then(|suffix| suffix.split_first_chunk::()) + .and_then(|(viewer, rest)| { + let (collection, document) = rest.split_first()?; + Some(PrivateKey { + collection: *collection, + viewer_id: u32::from_be_bytes(*viewer), + document_id: u32::from_be_bytes(document.try_into().ok()?), + }) + }) + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!())) + } +} + +fn decrement_containers(batch: &mut BatchBuilder, class: MetadataClass, count: u32) { + batch.merge_fnc(class, move |_, current| { + let Some(current) = current else { + return Ok(MergeResult::Skip); + }; + let mut state = ViewerState::deserialize(current)?; + state.containers = state.containers.saturating_sub(count); + Ok(MergeResult::Update(state.serialize())) + }); +} diff --git a/crates/common/src/storage/metadata/quota.rs b/crates/common/src/storage/metadata/quota.rs new file mode 100644 index 00000000..70c57b74 --- /dev/null +++ b/crates/common/src/storage/metadata/quota.rs @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ContainerChange, StoredContainer}; +use crate::{Server, auth::AccountCache}; +use store::write::{compress::compress, metadata::METADATA_COMPRESS_WATERMARK}; +use trc::{EventType, LimitEvent}; +use types::metadata::{EncodedMetadata, MetadataEdit, STORAGE_TRAILER_CAPACITY}; + +const COMPRESSED_TRAILER_LEN: usize = 1; + +impl Server { + pub async fn has_metadata_quota( + &self, + account: &AccountCache, + change: &ContainerChange, + ) -> trc::Result { + match change.next() { + Some(next) => { + has_room_for(change.edit(), change.previous(), next, move |growth| { + self.has_quota_for_growth(account, growth) + }) + .await + } + None => Ok(true), + } + } + + async fn has_quota_for_growth(&self, account: &AccountCache, growth: u64) -> trc::Result { + quota_outcome(self.has_available_quota(account, growth).await) + } +} + +async fn has_room_for( + edit: MetadataEdit, + previous: Option<&StoredContainer>, + next: &EncodedMetadata, + mut has_quota: F, +) -> trc::Result +where + F: FnMut(u64) -> Fut, + Fut: Future>, +{ + if edit == MetadataEdit::RemovalOnly && previous.is_some() { + return Ok(true); + } + let previous = previous.map_or(0, |previous| u64::from(previous.size)); + let bound = (next.len() + STORAGE_TRAILER_CAPACITY) as u64; + if bound <= previous || has_quota(bound - previous).await? { + Ok(true) + } else if next.len() >= METADATA_COMPRESS_WATERMARK { + let stored = stored_len(next)?; + Ok(stored <= previous || has_quota(stored - previous).await?) + } else { + Ok(false) + } +} + +fn stored_len(next: &EncodedMetadata) -> trc::Result { + let contents = next.as_bytes(); + if contents.len() >= METADATA_COMPRESS_WATERMARK { + let compressed = compress(None, contents, 0).map_err(|err| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(err) + })?; + if compressed.len() < contents.len() { + return Ok((compressed.len() + COMPRESSED_TRAILER_LEN) as u64); + } + } + Ok((contents.len() + STORAGE_TRAILER_CAPACITY) as u64) +} + +fn quota_outcome(result: trc::Result<()>) -> trc::Result { + match result { + Ok(()) => Ok(true), + Err(err) if err.matches(EventType::Limit(LimitEvent::Quota)) => Ok(false), + Err(err) => Err(err), + } +} + diff --git a/crates/common/src/storage/metadata/read.rs b/crates/common/src/storage/metadata/read.rs new file mode 100644 index 00000000..25b17e15 --- /dev/null +++ b/crates/common/src/storage/metadata/read.rs @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{StoredContainer, metadata_key, viewer::parse_owner_key}; +use crate::Server; +use store::{ + Deserialize, IterateParams, U32_LEN, + dispatch::DocumentSet, + write::{ + key::DeserializeBigEndian, + metadata::{MetadataBuf, MetadataClass, StoredMetadata, ViewerState}, + }, +}; +use trc::AddContext; +use types::{collection::Collection, metadata::MetadataView}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetadataViewerEntry { + pub collection: Collection, + pub viewer_id: u32, + pub state: ViewerState, +} + +impl Server { + pub async fn metadata_containers( + &self, + account_id: u32, + collection: Collection, + documents: &I, + cb: CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + self.containers_of(account_id, collection, MetadataClass::Shared, documents, cb) + .await + } + + pub async fn private_metadata_containers( + &self, + owner_id: u32, + viewer_id: u32, + collection: Collection, + documents: &I, + cb: CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + self.containers_of( + owner_id, + collection, + MetadataClass::Private { viewer: viewer_id }, + documents, + cb, + ) + .await + } + + pub async fn all_metadata_containers( + &self, + account_id: u32, + collection: Collection, + cb: CB, + ) -> trc::Result<()> + where + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + self.all_containers_of(account_id, collection, MetadataClass::Shared, cb) + .await + } + + pub async fn all_private_metadata_containers( + &self, + owner_id: u32, + viewer_id: u32, + collection: Collection, + cb: CB, + ) -> trc::Result<()> + where + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + self.all_containers_of( + owner_id, + collection, + MetadataClass::Private { viewer: viewer_id }, + cb, + ) + .await + } + + pub async fn metadata_container( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + ) -> trc::Result> { + self.core + .storage + .data + .get_value(metadata_key( + account_id, + collection.into(), + document_id, + MetadataClass::Shared, + )) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .document_id(document_id) + }) + } + + pub async fn private_metadata_container( + &self, + owner_id: u32, + viewer_id: u32, + collection: Collection, + document_id: u32, + ) -> trc::Result> { + self.core + .storage + .data + .get_value(metadata_key( + owner_id, + collection.into(), + document_id, + MetadataClass::Private { viewer: viewer_id }, + )) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(owner_id) + .document_id(document_id) + }) + } + + pub async fn all_metadata_viewers( + &self, + owner_id: u32, + ) -> trc::Result> { + let mut viewers = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(owner_id, 0, 0, MetadataClass::Viewer { viewer: 0 }), + metadata_key( + owner_id, + u8::MAX, + 0, + MetadataClass::Viewer { viewer: u32::MAX }, + ), + ), + |key, value| { + let (collection, viewer_id) = key + .len() + .checked_sub(U32_LEN + 1) + .and_then(|offset| key.get(offset..)) + .and_then(|suffix| suffix.split_first()) + .and_then(|(collection, viewer)| { + Some((*collection, u32::from_be_bytes(viewer.try_into().ok()?))) + }) + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; + viewers.push(MetadataViewerEntry { + collection: Collection::from(collection), + viewer_id, + state: ViewerState::deserialize(value)?, + }); + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(owner_id))?; + Ok(viewers) + } + + pub async fn metadata_owners(&self, viewer_id: u32) -> trc::Result> { + let mut owners = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(viewer_id, 0, 0, MetadataClass::Owner { owner: 0 }), + metadata_key( + viewer_id, + u8::MAX, + 0, + MetadataClass::Owner { owner: u32::MAX }, + ), + ) + .no_values(), + |key, _| { + let (owner, collection) = parse_owner_key(key)?; + owners.push((owner, Collection::from(collection))); + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(viewer_id))?; + Ok(owners) + } + + async fn containers_of( + &self, + account_id: u32, + collection: Collection, + class: MetadataClass, + documents: &I, + mut cb: CB, + ) -> trc::Result<()> + where + I: DocumentSet + Send + Sync, + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + let collection = u8::from(collection); + let mut scratch = Vec::new(); + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + let view = StoredMetadata::view(value, &mut scratch)?; + let stored = StoredContainer::new(&view, value)?; + cb(document_id, view, stored) + } else { + Ok(true) + } + }; + self.iterate_documents(account_id, collection, class, documents, &mut collect) + .await + } + + async fn all_containers_of( + &self, + account_id: u32, + collection: Collection, + class: MetadataClass, + mut cb: CB, + ) -> trc::Result<()> + where + CB: for<'x> FnMut(u32, MetadataView<'x>, StoredContainer) -> trc::Result + + Send + + Sync, + { + let collection = u8::from(collection); + let mut scratch = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(account_id, collection, 0, class), + metadata_key(account_id, collection, u32::MAX, class), + ), + |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + let view = StoredMetadata::view(value, &mut scratch)?; + let stored = StoredContainer::new(&view, value)?; + cb(document_id, view, stored) + }, + ) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } +} diff --git a/crates/common/src/storage/metadata/stored.rs b/crates/common/src/storage/metadata/stored.rs new file mode 100644 index 00000000..4f15e4e3 --- /dev/null +++ b/crates/common/src/storage/metadata/stored.rs @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{add_quota, metadata_key}; +use crate::Server; +use std::slice; +use store::{ + Deserialize, U32_LEN, + dispatch::DocumentSet, + write::{ + ArchiveVersion, BatchBuilder, + assert::AssertValue, + key::DeserializeBigEndian, + metadata::{MetadataBuf, MetadataClass, StoredMetadata}, + }, +}; +use trc::AddContext; +use types::collection::Collection; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StoredEntry { + pub document_id: u32, + pub size: u32, + pub hash: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StoredEntries { + entries: Vec, +} + +struct StoredValue(StoredEntry); + +impl Deserialize for StoredValue { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(StoredValue(StoredEntry::new(0, bytes))) + } +} + +impl StoredEntry { + pub fn new(document_id: u32, stored: &[u8]) -> Self { + StoredEntry { + document_id, + size: u32::try_from(stored.len()).unwrap_or(u32::MAX), + hash: StoredMetadata::trailer_hash(stored).ok(), + } + } + + pub fn from_container(document_id: u32, container: &MetadataBuf) -> Self { + StoredEntry { + document_id, + size: container.stored_len(), + hash: Some(container.hash()), + } + } + + pub fn clear(&self, batch: &mut BatchBuilder, class: MetadataClass) { + batch.with_document(self.document_id); + if let Some(hash) = self.hash { + batch.assert_value(class, AssertValue::Archive(ArchiveVersion::Hashed { hash })); + } + batch.clear(class); + } + + pub fn release( + &self, + batch: &mut BatchBuilder, + ) { + self.clear(batch, MetadataClass::Shared); + #[cfg(not(feature = "enterprise"))] + add_quota(batch, -i64::from(self.size)); + } +} + +impl StoredEntries { + fn new(mut entries: Vec) -> Self { + if !entries.is_sorted_by_key(|entry| entry.document_id) { + entries.sort_unstable_by_key(|entry| entry.document_id); + } + StoredEntries { entries } + } + + pub fn get(&self, document_id: u32) -> Option<&StoredEntry> { + self.entries + .binary_search_by_key(&document_id, |entry| entry.document_id) + .ok() + .and_then(|position| self.entries.get(position)) + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn iter(&self) -> slice::Iter<'_, StoredEntry> { + self.entries.iter() + } +} + +impl FromIterator for StoredEntries { + fn from_iter>(iter: T) -> Self { + StoredEntries::new(iter.into_iter().collect()) + } +} + +impl Server { + pub async fn stored_metadata_entries( + &self, + account_id: u32, + collection: Collection, + documents: &I, + ) -> trc::Result + where + I: DocumentSet + Send + Sync, + { + self.stored_entries(account_id, collection, MetadataClass::Shared, documents) + .await + .map(StoredEntries::new) + } + + pub(super) async fn stored_entries( + &self, + account_id: u32, + collection: Collection, + class: MetadataClass, + documents: &I, + ) -> trc::Result> + where + I: DocumentSet + Send + Sync, + { + let mut entries = Vec::new(); + match documents.len() { + 0 => return Ok(entries), + 1 => { + return self + .stored_entry(account_id, collection, class, documents.min()) + .await + .map(|entry| entry.into_iter().collect()); + } + _ => {} + } + let mut collect = |key: &[u8], value: &[u8]| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if documents.contains(document_id) { + entries.push(StoredEntry::new(document_id, value)); + } + Ok(true) + }; + self.iterate_documents( + account_id, + u8::from(collection), + class, + documents, + &mut collect, + ) + .await?; + Ok(entries) + } + + pub(super) async fn stored_entry( + &self, + account_id: u32, + collection: Collection, + class: MetadataClass, + document_id: u32, + ) -> trc::Result> { + self.core + .storage + .data + .get_value::(metadata_key( + account_id, + u8::from(collection), + document_id, + class, + )) + .await + .map(|value| { + value.map(|StoredValue(entry)| StoredEntry { + document_id, + ..entry + }) + }) + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .document_id(document_id) + }) + } +} + diff --git a/crates/common/src/storage/metadata/viewer.rs b/crates/common/src/storage/metadata/viewer.rs new file mode 100644 index 00000000..4f725150 --- /dev/null +++ b/crates/common/src/storage/metadata/viewer.rs @@ -0,0 +1,402 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{PrivateMetadataChange, PrivateMetadataCommit, metadata_key}; +use crate::{ + Server, + auth::AccessToken, + cache::invalidate::CacheInvalidationBuilder, + ipc::{BroadcastEvent, CacheInvalidation, PushNotification, ViewerStateChange}, +}; +use registry::schema::enums::Permission; +use std::sync::{ + Arc, + atomic::{AtomicU32, AtomicU64, Ordering}, +}; +use store::{ + Deserialize, IterateParams, U32_LEN, + write::{ + AssignedIds, BatchBuilder, + metadata::{MetadataClass, ViewerState}, + }, +}; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + type_state::{DataType, StateChange}, +}; +use utils::{ + cache::{Cache, CacheItemWeight, EntryCharge}, + map::bitmap::Bitmap, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetadataViewer(u32); + +pub struct MetadataViewerCache { + states: Cache>, + epochs: [AtomicU64; EPOCH_SHARDS], +} + +#[derive(Debug, Default)] +pub struct ViewerStates { + entries: Box<[ViewerStateEntry]>, +} + +#[derive(Debug)] +struct ViewerStateEntry { + owner_id: u32, + collection: u8, + change_id: AtomicU64, + containers: AtomicU32, +} + +const OWNER_KEY_SUFFIX: usize = U32_LEN + 1; +const EPOCH_SHARDS: usize = 64; +pub(super) const CACHE_SLOT_SIZE: usize = 32; +pub(super) const CACHE_INDEX_SIZE: usize = 8; +const ARC_COUNTERS_SIZE: usize = 2 * size_of::(); +const VALUE_SLOT_SIZE: usize = CACHE_SLOT_SIZE - size_of::(); +const ESTIMATED_ENTRIES: usize = 2; + +impl MetadataViewer { + pub fn account_id(self) -> u32 { + self.0 + } +} + +impl MetadataViewerCache { + pub fn new(weight_capacity: u64) -> Self { + MetadataViewerCache { + states: Cache::new_with_charge( + weight_capacity, + size_of::() as u64 + ViewerStates::footprint(ESTIMATED_ENTRIES), + EntryCharge::Payload, + ) + .with_name("metadataViewers"), + epochs: [const { AtomicU64::new(0) }; EPOCH_SHARDS], + } + } + + pub fn weight(&self) -> u64 { + self.states.weight() + } + + pub fn invalidate(&self, viewer_id: u32) { + self.shard(viewer_id).fetch_add(1, Ordering::AcqRel); + self.states.remove(&viewer_id); + } + + pub fn clear(&self) { + for epoch in &self.epochs { + epoch.fetch_add(1, Ordering::AcqRel); + } + self.states.clear(); + } + + pub(super) fn get(&self, viewer_id: u32) -> Option> { + self.states.get(&viewer_id) + } + + fn shard(&self, viewer_id: u32) -> &AtomicU64 { + &self.epochs[viewer_id as usize % EPOCH_SHARDS] + } + + pub(super) fn epoch(&self, viewer_id: u32) -> u64 { + self.shard(viewer_id).load(Ordering::Acquire) + } + + pub(super) fn insert(&self, viewer_id: u32, states: Arc, epoch: u64) { + if self.epoch(viewer_id) == epoch { + self.states.insert(viewer_id, states); + self.discard_if_stale(viewer_id, epoch); + } + } + + fn discard_if_stale(&self, viewer_id: u32, epoch: u64) { + if self.epoch(viewer_id) != epoch { + self.states.remove(&viewer_id); + } + } + + pub(super) fn apply(&self, change: &PrivateMetadataChange, change_id: Option) { + self.shard(change.viewer_id).fetch_add(1, Ordering::AcqRel); + let Ok(added) = u32::try_from(change.containers) else { + self.states.remove(&change.viewer_id); + return; + }; + let Some(states) = self.states.peek(&change.viewer_id) else { + return; + }; + match states.entry(change.owner_id, change.collection) { + Some(entry) => { + if let Some(change_id) = change_id { + entry.change_id.fetch_max(change_id, Ordering::AcqRel); + } + if added > 0 { + let _ = entry.containers.try_update( + Ordering::AcqRel, + Ordering::Acquire, + |containers| Some(containers.saturating_add(added)), + ); + } + } + None => { + self.states.remove(&change.viewer_id); + } + } + } +} + +impl ViewerStates { + pub fn get(&self, owner_id: u32, collection: Collection) -> ViewerState { + self.entry(owner_id, collection) + .map(ViewerStateEntry::load) + .unwrap_or_default() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn entry(&self, owner_id: u32, collection: Collection) -> Option<&ViewerStateEntry> { + let key = (owner_id, u8::from(collection)); + self.entries + .binary_search_by_key(&key, |entry| (entry.owner_id, entry.collection)) + .ok() + .and_then(|index| self.entries.get(index)) + } + + fn footprint(entries: usize) -> u64 { + (VALUE_SLOT_SIZE + + CACHE_INDEX_SIZE + + ARC_COUNTERS_SIZE + + size_of::() + + (entries * size_of::())) as u64 + } + + pub(super) fn new(entries: impl IntoIterator) -> Self { + ViewerStates { + entries: entries + .into_iter() + .map(|(owner_id, collection, state)| { + ViewerStateEntry::new(owner_id, collection, state) + }) + .collect(), + } + } + + fn logged_owners(&self) -> impl Iterator + '_ { + self.entries + .iter() + .filter(|entry| entry.change_id.load(Ordering::Acquire) > 0) + .map(|entry| { + ( + entry.owner_id, + SyncCollection::from(Collection::from(entry.collection)), + ) + }) + } +} + +impl ViewerStateEntry { + fn new(owner_id: u32, collection: u8, state: ViewerState) -> Self { + ViewerStateEntry { + owner_id, + collection, + change_id: AtomicU64::new(state.change_id), + containers: AtomicU32::new(state.containers), + } + } + + fn load(&self) -> ViewerState { + ViewerState { + change_id: self.change_id.load(Ordering::Acquire), + containers: self.containers.load(Ordering::Acquire), + } + } +} + +impl CacheItemWeight for ViewerStates { + fn weight(&self) -> u64 { + ViewerStates::footprint(self.entries.len()) + } +} + +impl Server { + pub fn metadata_viewer( + &self, + access_token: &AccessToken, + permission: Permission, + ) -> Option { + (self.core.metadata.private_metadata && access_token.has_permission(permission)) + .then(|| MetadataViewer(access_token.account_id())) + } + + pub async fn metadata_viewer_state( + &self, + viewer: MetadataViewer, + owner_id: u32, + collection: Collection, + ) -> trc::Result { + self.metadata_viewer_states(viewer) + .await + .map(|states| states.get(owner_id, collection)) + } + + pub async fn metadata_viewer_states( + &self, + viewer: MetadataViewer, + ) -> trc::Result> { + let cache = &self.inner.cache.metadata_viewers; + if let Some(states) = cache.get(viewer.0) { + return Ok(states); + } + + let epoch = cache.epoch(viewer.0); + let states = Arc::new(self.load_viewer_states(viewer.0).await?); + cache.insert(viewer.0, states.clone(), epoch); + Ok(states) + } + + pub async fn commit_metadata_batch( + &self, + batch: BatchBuilder, + commit: PrivateMetadataCommit, + ) -> trc::Result { + let is_split = batch.has_commit_points(); + match self.commit_batch(batch).await { + Ok(assigned_ids) => { + self.private_metadata_committed(commit, &assigned_ids).await; + Ok(assigned_ids) + } + Err(err) => { + if is_split { + self.private_metadata_uncertain(&commit).await; + } + Err(err) + } + } + } + + async fn private_metadata_uncertain(&self, commit: &PrivateMetadataCommit) { + let mut invalidations = CacheInvalidationBuilder::default(); + for change in &commit.changes { + invalidations.invalidate(CacheInvalidation::PrivateMetadata(change.viewer_id)); + } + if let Err(err) = self.invalidate_caches(invalidations).await { + trc::error!(err.caused_by(trc::location!())); + } + } + + pub async fn private_metadata_committed( + &self, + commit: PrivateMetadataCommit, + assigned_ids: &AssignedIds, + ) { + if commit.is_empty() { + return; + } + + let cache = &self.inner.cache.metadata_viewers; + let mut viewers = Vec::with_capacity(1); + let mut pushes: Vec<(u32, u32, SyncCollection, Bitmap)> = Vec::new(); + for change in &commit.changes { + let sync_collection = SyncCollection::from(change.collection); + let change_id = change + .is_logged + .then(|| assigned_ids.change_id(change.owner_id, sync_collection)) + .flatten(); + cache.apply(change, change_id); + + if !viewers.contains(&change.viewer_id) { + viewers.push(change.viewer_id); + } + + if change_id.is_some() + && let Ok(data_type) = DataType::try_from(change.collection) + { + match pushes.iter_mut().find(|(owner_id, viewer_id, pushed, _)| { + *owner_id == change.owner_id + && *viewer_id == change.viewer_id + && *pushed == sync_collection + }) { + Some((_, _, _, types)) => types.insert(data_type), + None => pushes.push(( + change.owner_id, + change.viewer_id, + sync_collection, + Bitmap::from_iter([data_type]), + )), + } + } + } + + self.cluster_broadcast(BroadcastEvent::CacheInvalidate( + viewers + .into_iter() + .map(CacheInvalidation::PrivateMetadata) + .collect(), + )) + .await; + + for (owner_id, viewer_id, sync_collection, types) in pushes { + if let Some(change_id) = assigned_ids.change_id(owner_id, sync_collection) { + self.broadcast_push_notification(PushNotification::ViewerStateChange( + ViewerStateChange { + viewer_id, + change: StateChange { + account_id: owner_id, + change_id, + types, + }, + }, + )) + .await; + } + } + } + + async fn load_viewer_states(&self, viewer_id: u32) -> trc::Result { + let mut entries = Vec::new(); + self.core + .storage + .data + .iterate( + IterateParams::new( + metadata_key(viewer_id, 0, 0, MetadataClass::Owner { owner: 0 }), + metadata_key( + viewer_id, + u8::MAX, + 0, + MetadataClass::Owner { owner: u32::MAX }, + ), + ), + |key, value| { + let (owner_id, collection) = parse_owner_key(key)?; + entries.push((owner_id, collection, ViewerState::deserialize(value)?)); + Ok(true) + }, + ) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(viewer_id))?; + let states = ViewerStates::new(entries); + for (owner_id, collection) in states.logged_owners() { + self.inner.mark_cache_stale(owner_id, collection); + } + Ok(states) + } +} + +pub(super) fn parse_owner_key(key: &[u8]) -> trc::Result<(u32, u8)> { + key.len() + .checked_sub(OWNER_KEY_SUFFIX) + .and_then(|offset| key.get(offset..)) + .and_then(|suffix| suffix.split_first_chunk::()) + .and_then(|(owner, collection)| Some((u32::from_be_bytes(*owner), *collection.first()?))) + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!())) +} + diff --git a/crates/common/src/storage/mod.rs b/crates/common/src/storage/mod.rs index f65943b8..fa0535ed 100644 --- a/crates/common/src/storage/mod.rs +++ b/crates/common/src/storage/mod.rs @@ -7,30 +7,34 @@ use crate::Server; use directory::Directory; use registry::{ - schema::{ - enums::{StorageQuota, TenantStorageQuota}, - prelude::ObjectType, - }, + schema::{enums::StorageQuota, prelude::ObjectType}, types::EnumImpl, }; +use registry::schema::enums::TaskType; use std::sync::Arc; -use store::{BlobStore, InMemoryStore, RegistryStore, SearchStore, Store}; +use store::{ + BlobStore, InMemoryStore, IterateParams, RegistryStore, SearchStore, Store, U16_LEN, ValueKey, + write::{TaskId, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, +}; +use trc::AddContext; pub mod archive; +pub mod batch; pub mod blob; +pub mod chunk; pub mod dav; pub mod document; pub mod encryption; pub mod index; +pub mod metadata; pub mod quota; +pub mod search; pub mod state; pub mod transaction; #[derive(Debug, Clone)] pub struct ObjectQuota([u32; StorageQuota::COUNT - 1]); -#[derive(Debug, Clone)] -pub struct TenantQuota([u32; TenantStorageQuota::COUNT - 1]); impl Server { #[inline(always)] @@ -58,15 +62,6 @@ impl Server { &self.core.storage.memory } - #[inline(always)] - pub fn tracing_store(&self) -> &Store { - &self.core.storage.tracing - } - - #[inline(always)] - pub fn metrics_store(&self) -> &Store { - &self.core.storage.metrics - } #[inline(always)] pub fn get_directory(&self, id: &u32) -> Option<&Arc> { @@ -95,11 +90,52 @@ impl Server { self.registry().count_object(ObjectType::Domain).await } + pub async fn total_tasks_pending(&self) -> trc::Result<(u64, u64)> { + let (Some(first), Some(last)) = (TaskType::ALL.first(), TaskType::ALL.last()) else { + return Ok((0, 0)); + }; + const DUE_OFFSET: usize = size_of::() + U16_LEN; + let now = now(); + let mut pending = 0u64; + let mut due = 0u64; + let mut scanned = 0u64; + self.store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(0), + typ: *first, + due: 0, + })), + ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: TaskId::Assigned(u64::MAX), + typ: *last, + due: u64::MAX, + })), + ) + .no_values(), + |key, _| { + scanned += 1; + let task_due = key.deserialize_be_u64(DUE_OFFSET)?; + if task_due != u64::MAX { + pending += 1; + if task_due <= now { + due += 1; + } + } + Ok(scanned < search::MAX_PENDING_SCAN) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| (pending, due)) + } + #[cfg(not(feature = "enterprise"))] pub async fn logo_resource( &self, _: &str, - ) -> trc::Result>>> { + ) -> trc::Result>> { Ok(None) } } diff --git a/crates/common/src/storage/quota.rs b/crates/common/src/storage/quota.rs index b106f235..98287a30 100644 --- a/crates/common/src/storage/quota.rs +++ b/crates/common/src/storage/quota.rs @@ -4,16 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - Server, - auth::AccountCache, - storage::{ObjectQuota, TenantQuota}, +use crate::{Server, auth::AccountCache, storage::ObjectQuota}; +use registry::{schema::enums::StorageQuota, types::EnumImpl}; +use store::{ + ValueKey, + write::{AccountCounter, BatchBuilder, ValueClass}, }; -use registry::{ - schema::enums::{StorageQuota, TenantStorageQuota}, - types::EnumImpl, -}; -use store::{ValueKey, write::ValueClass}; use trc::AddContext; impl Server { @@ -31,11 +27,6 @@ impl Server { .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) } - #[cfg(not(feature = "enterprise"))] - pub async fn get_used_quota_tenant(&self, _tenant_id: u32) -> trc::Result { - Ok(0) - } - pub async fn has_available_quota( &self, account: &AccountCache, @@ -48,7 +39,7 @@ impl Server { return Err(trc::LimitEvent::Quota .into_err() .ctx(trc::Key::Limit, account.quota_disk) - .ctx(trc::Key::Size, used_quota)); + .ctx(trc::Key::Used, used_quota)); } } @@ -56,10 +47,187 @@ impl Server { Ok(()) } + + pub fn limit_disk_quota(&self, batch: &mut BatchBuilder, account: &AccountCache) { + if account.quota_disk != 0 { + batch.limit_counter(AccountCounter::Quota, account.quota_disk); + } + } + + pub fn limit_object_quota( + &self, + batch: &mut BatchBuilder, + account: &AccountCache, + object: StorageQuota, + ) { + if let Some(limit) = self.object_quota_limit(account, object) { + batch.limit_counter(AccountCounter::ObjectCount(object), limit as u64); + } + } + + pub async fn object_count(&self, account_id: u32, object: StorageQuota) -> trc::Result { + self.core + .storage + .data + .get_counter(ValueKey { + account_id, + collection: 0, + document_id: 0, + class: ValueClass::ObjectCount(object), + }) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) + .map(|count| usize::try_from(count).unwrap_or_default()) + } + + pub async fn disk_quota_budget(&self, account: &AccountCache) -> trc::Result { + let mut budget = DiskQuotaBudget::unlimited(); + if account.quota_disk != 0 { + let used_quota = self.get_used_quota_account(account.id).await?.max(0) as u64; + budget.limit_to( + account.quota_disk.saturating_sub(used_quota), + trc::LimitEvent::Quota, + ); + } + + Ok(budget) + } + + #[inline(always)] pub fn object_quota(&self, user_quotas: Option<&ObjectQuota>, object: StorageQuota) -> u32 { user_quotas.unwrap_or(&self.core.email.max_objects).0[object as usize] } + + #[inline(always)] + pub fn object_quota_limit( + &self, + account: &AccountCache, + object: StorageQuota, + ) -> Option { + let limit = self.object_quota(account.object_quotas(), object); + (limit != u32::MAX).then_some(limit as usize) + } + + pub fn object_quota_usage( + &self, + account: &AccountCache, + object: StorageQuota, + used: impl FnOnce() -> usize, + ) -> ObjectQuotaUsage { + self.object_quota_limit(account, object) + .map_or_else(ObjectQuotaUsage::unlimited, |limit| ObjectQuotaUsage { + used: used(), + limit, + }) + } + + pub fn assert_object_quota( + &self, + account: &AccountCache, + object: StorageQuota, + requested: usize, + used: impl FnOnce() -> usize, + ) -> trc::Result<()> { + let Some(limit) = self.object_quota_limit(account, object) else { + return Ok(()); + }; + let used = used(); + if used.saturating_add(requested) <= limit { + Ok(()) + } else { + Err(trc::LimitEvent::Quota + .into_err() + .account_id(account.id) + .ctx(trc::Key::ObjectType, object.as_str()) + .ctx(trc::Key::Limit, limit) + .ctx(trc::Key::Used, used)) + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ObjectQuotaUsage { + pub used: usize, + pub limit: usize, +} + +impl ObjectQuotaUsage { + pub fn unlimited() -> Self { + Self { + used: 0, + limit: usize::MAX, + } + } + + #[inline(always)] + pub fn has_room(&self, pending: usize) -> bool { + self.used.saturating_add(pending) < self.limit + } +} + +#[derive(Debug, Clone, Copy)] +pub struct DiskQuotaBudget { + available: Option, + binding: trc::LimitEvent, +} + +impl DiskQuotaBudget { + pub const fn unlimited() -> Self { + Self { + available: None, + binding: trc::LimitEvent::Quota, + } + } + + pub fn limited(available: u64, binding: trc::LimitEvent) -> Self { + Self { + available: Some(saturating_i64(available)), + binding, + } + } + + fn limit_to(&mut self, available: u64, binding: trc::LimitEvent) { + let available = saturating_i64(available); + if self.available.is_none_or(|current| available < current) { + self.available = Some(available); + self.binding = binding; + } + } + + #[inline(always)] + pub fn check(&self, bytes: u64) -> Result<(), trc::LimitEvent> { + match self.available { + Some(available) if bytes > 0 && saturating_i64(bytes) > available => Err(self.binding), + _ => Ok(()), + } + } + + #[inline(always)] + pub fn try_reserve(&mut self, bytes: u64) -> Result<(), trc::LimitEvent> { + self.check(bytes)?; + self.reserve(bytes); + Ok(()) + } + + #[inline(always)] + pub fn reserve(&mut self, bytes: u64) { + if let Some(available) = &mut self.available { + *available = available.saturating_sub(saturating_i64(bytes)); + } + } + + #[inline(always)] + pub fn release(&mut self, bytes: u64) { + if let Some(available) = &mut self.available { + *available = available.saturating_add(saturating_i64(bytes)); + } + } +} + +#[inline(always)] +fn saturating_i64(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) } impl ObjectQuota { @@ -74,17 +242,6 @@ impl ObjectQuota { } } -impl TenantQuota { - #[inline(always)] - pub fn set(&mut self, item: TenantStorageQuota, max: u32) { - self.0[item as usize] = max; - } - - #[inline(always)] - pub fn get(&self, item: TenantStorageQuota) -> u32 { - self.0[item as usize] - } -} impl Default for ObjectQuota { fn default() -> Self { @@ -92,8 +249,4 @@ impl Default for ObjectQuota { } } -impl Default for TenantQuota { - fn default() -> Self { - Self([u32::MAX; TenantStorageQuota::COUNT - 1]) - } -} + diff --git a/crates/common/src/storage/search.rs b/crates/common/src/storage/search.rs new file mode 100644 index 00000000..0a4537c8 --- /dev/null +++ b/crates/common/src/storage/search.rs @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{KV_LOCK_SEARCH_INDEX, Server}; +use store::{ + IterateParams, U32_LEN, U64_LEN, ValueKey, + write::{ + QueueDocumentId, SearchIndex, SearchIndexClass, ValueClass, + key::{DeserializeBigEndian, KeySerializer}, + serialize::RawValue, + }, +}; +use trc::AddContext; + +pub(crate) const MAX_PENDING_SCAN: u64 = 250_000; + +#[derive(Debug, Clone, Default)] +pub struct SearchIndexStatus { + pub next_retry: u64, + pub attempts: u32, + pub reason: String, +} + +impl SearchIndexStatus { + pub fn deserialize(value: &[u8]) -> trc::Result { + Ok(SearchIndexStatus { + next_retry: value.deserialize_be_u64(0)?, + attempts: value.deserialize_be_u32(U64_LEN)?, + reason: value + .get(U64_LEN + U32_LEN..) + .map(String::from_utf8_lossy) + .unwrap_or_default() + .into_owned(), + }) + } + + pub fn serialize(&self) -> Vec { + KeySerializer::new(U64_LEN + U32_LEN + self.reason.len()) + .write(self.next_retry) + .write(self.attempts) + .write(self.reason.as_bytes()) + .finalize() + } +} + +impl Server { + pub async fn total_index_pending(&self) -> trc::Result<[u64; 5]> { + let mut pending = [0u64; 5]; + let mut scanned = 0u64; + self.store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::Queue { + index: SearchIndex::Email, + id_prefix: 0, + id_suffix: QueueDocumentId::Assigned(0), + created_at: 0, + })), + ValueKey::from(ValueClass::SearchIndex(SearchIndexClass::Queue { + index: SearchIndex::Tracing, + id_prefix: u32::MAX, + id_suffix: QueueDocumentId::Assigned(u32::MAX), + created_at: u64::MAX, + })), + ), + |key, value| { + if value.first().is_some_and(|flag| *flag != 0) + && let Some(count) = key + .first() + .and_then(|index| pending.get_mut(*index as usize)) + { + *count += 1; + } + scanned += 1; + Ok(scanned < MAX_PENDING_SCAN) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| pending) + } + + pub async fn search_index_status( + &self, + index: SearchIndex, + partition: u32, + ) -> trc::Result> { + match self + .store() + .get_value::(ValueKey::from(ValueClass::SearchIndex( + SearchIndexClass::QueueStatus { index, partition }, + ))) + .await + .caused_by(trc::location!())? + { + Some(value) => SearchIndexStatus::deserialize(value.0.as_slice()).map(Some), + None => Ok(None), + } + } + + pub async fn try_lock_search_index( + &self, + index: SearchIndex, + partition: u32, + expiry: u64, + ) -> bool { + match self + .in_memory_store() + .try_lock( + KV_LOCK_SEARCH_INDEX, + &search_index_lock_key(index, partition), + expiry, + ) + .await + { + Ok(result) => result, + Err(err) => { + trc::error!( + err.details("Failed to lock search index partition") + .ctx(trc::Key::Index, index.name()) + .ctx(trc::Key::Partition, partition) + .caused_by(trc::location!()) + ); + + false + } + } + } + + pub async fn renew_search_index_lock( + &self, + index: SearchIndex, + partition: u32, + expiry: u64, + ) -> bool { + match self + .in_memory_store() + .renew_lock( + KV_LOCK_SEARCH_INDEX, + &search_index_lock_key(index, partition), + expiry, + ) + .await + { + Ok(_) => true, + Err(err) => { + trc::error!( + err.details("Failed to renew search index partition lock") + .ctx(trc::Key::Index, index.name()) + .ctx(trc::Key::Partition, partition) + .caused_by(trc::location!()) + ); + + false + } + } + } + + pub async fn unlock_search_index(&self, index: SearchIndex, partition: u32) { + if let Err(err) = self + .in_memory_store() + .remove_lock( + KV_LOCK_SEARCH_INDEX, + &search_index_lock_key(index, partition), + ) + .await + { + trc::error!( + err.details("Failed to unlock search index partition") + .ctx(trc::Key::Index, index.name()) + .ctx(trc::Key::Partition, partition) + .caused_by(trc::location!()) + ); + } + } +} + +fn search_index_lock_key(index: SearchIndex, partition: u32) -> [u8; U32_LEN + 1] { + let mut key = [index.to_u8(); U32_LEN + 1]; + key[1..].copy_from_slice(&partition.to_be_bytes()); + key +} diff --git a/crates/common/src/storage/state.rs b/crates/common/src/storage/state.rs index 979fb4a9..7ad90944 100644 --- a/crates/common/src/storage/state.rs +++ b/crates/common/src/storage/state.rs @@ -5,71 +5,79 @@ */ use crate::{ - IPC_CHANNEL_BUFFER, Server, + Inner, Server, auth::AccessToken, - ipc::{BroadcastEvent, PushEvent, PushNotification}, + ipc::{BroadcastEvent, PushEvent, PushNotification, subscribers::PushReceiver}, }; -use tokio::sync::mpsc; +use store::write::{QueueNotify, TaskTypes}; use types::type_state::DataType; use utils::map::bitmap::Bitmap; impl Server { - pub async fn subscribe_push_manager( + pub fn subscribe_push_manager( &self, access_token: &AccessToken, types: Bitmap, - ) -> trc::Result> { - let (tx, rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); - let push_tx = self.inner.ipc.push_tx.clone(); - - push_tx - .send(PushEvent::Subscribe { - account_ids: access_token.member_ids().collect(), - types, - tx, - }) - .await - .map_err(|err| { - trc::EventType::Server(trc::ServerEvent::ThreadError) - .reason(err) - .caused_by(trc::location!()) - })?; - - Ok(rx) + ) -> PushReceiver { + self.inner + .ipc + .push_subscribers + .subscribe(access_token.member_ids(), types) } #[inline(always)] pub fn notify_task_queue(&self) { - self.inner.ipc.task_tx.notify_one(); + self.inner.ipc.task_tx.notify(TaskTypes::ALL); } - pub async fn broadcast_push_notification(&self, notification: PushNotification) -> bool { - match self - .inner - .ipc - .push_tx - .clone() - .send(PushEvent::Publish { - notification, - broadcast: true, - }) - .await - { - Ok(_) => true, - Err(_) => { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending state change.", - CausedBy = trc::location!() - ); + #[inline(always)] + pub fn notify_task_types(&self, task_types: TaskTypes) { + self.inner.ipc.task_tx.notify(task_types); + } - false - } + #[inline(always)] + pub fn notify_index_queue(&self) { + self.inner.ipc.index_tx.notify_one(); + } + + pub async fn notify_queues(&self, queues: QueueNotify) { + if !queues.task_types.is_empty() { + self.notify_task_types(queues.task_types); + self.cluster_broadcast(BroadcastEvent::TaskQueueRefresh(queues.task_types)) + .await; + } + + if queues.search_index { + self.notify_index_queue(); + self.cluster_broadcast(BroadcastEvent::IndexQueueRefresh) + .await; } } + pub async fn broadcast_push_notification(&self, notification: PushNotification) -> bool { + let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx else { + return self.inner.publish_push_notification(notification).await; + }; + let is_published = self + .inner + .publish_push_notification(notification.clone()) + .await; + if broadcast_tx + .send(BroadcastEvent::PushNotification(notification)) + .await + .is_err() + { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending broadcast event.", + CausedBy = trc::location!() + ); + } + is_published + } + pub async fn cluster_broadcast(&self, event: BroadcastEvent) { - if let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx.clone() + if let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx && broadcast_tx.send(event).await.is_err() { trc::event!( @@ -80,3 +88,26 @@ impl Server { } } } + +impl Inner { + pub async fn publish_push_notification(&self, notification: PushNotification) -> bool { + if !self.ipc.push_subscribers.dispatch(¬ification) + || self + .ipc + .push_tx + .send(PushEvent::Publish { notification }) + .await + .is_ok() + { + true + } else { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + + false + } + } +} diff --git a/crates/common/src/storage/transaction.rs b/crates/common/src/storage/transaction.rs index f8e6fc01..6a98c88a 100644 --- a/crates/common/src/storage/transaction.rs +++ b/crates/common/src/storage/transaction.rs @@ -4,50 +4,88 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Server, ipc::PushNotification}; +use crate::{Server, cache::invalidate::CacheInvalidationBuilder, ipc::PushNotification}; use std::time::Duration; use store::{ - IterateParams, Key, LogKey, SUBSPACE_LOGS, U64_LEN, - write::{AnyClass, AssignedIds, BatchBuilder, ValueClass, key::DeserializeBigEndian}, + IterateParams, Key, LogKey, Subspace, U64_LEN, + write::{ + AnyClass, AssignedIds, BatchBuilder, ChangedCollection, LogCollection, QueueNotify, + ValueClass, key::DeserializeBigEndian, + }, }; use trc::AddContext; use types::{ - collection::SyncCollection, + collection::{ChangeGroup, SyncCollection}, type_state::{DataType, StateChange}, }; -use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; +use utils::{ + map::{bitmap::Bitmap, vec_map::VecMap}, + snowflake::SnowflakeIdGenerator, +}; impl Server { pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result { - let mut assigned_ids = AssignedIds::default(); - let mut commit_points = builder.commit_points(); + let mut assigned_ids = match self.store().write_batch(&mut builder).await { + Ok(assigned_ids) => assigned_ids, + Err(err) => { + if let Some(changes) = builder.changes() { + self.invalidate_share_grantees(&changes).await; + } + return Err(err); + } + }; + self.inner.mark_caches_stale_from(&assigned_ids); - for commit_point in commit_points.iter() { - let batch = builder.build_one(commit_point); - assigned_ids - .ids - .extend(self.store().write(batch).await?.ids); + if let Some(hash) = builder.last_archive_hash() { + assigned_ids.push_archive_hash(hash); + } + + let queues = builder.queue_notify(); + if queues != QueueNotify::default() { + self.notify_queues(queues).await; } if let Some(changes) = builder.changes() { + self.invalidate_share_grantees(&changes).await; + let mut group_types = + [Bitmap::::default(); SyncCollection::MAX_CHANGE_GROUP as usize + 1]; + for (account_id, changed_collections) in changes { - let mut state_change = StateChange::new(account_id); - for changed_collection in changed_collections.changed_containers { - if let Some(data_type) = DataType::try_from_sync(changed_collection, true) { - state_change.set_change(data_type); + group_types.fill(Bitmap::default()); + + for (changed_collection, is_container) in changed_collections + .changed_containers + .into_iter() + .map(|collection| (collection, true)) + .chain( + changed_collections + .changed_items + .into_iter() + .map(|collection| (collection, false)), + ) + { + if let Some(data_type) = + DataType::try_from_sync(changed_collection, is_container) + { + group_types[changed_collection.change_group().to_u8() as usize] + .insert(data_type); } } - for changed_collection in changed_collections.changed_items { - if let Some(data_type) = DataType::try_from_sync(changed_collection, false) { - state_change.set_change(data_type); + + for (group, types) in group_types.iter().enumerate() { + if !types.is_empty() { + self.broadcast_push_notification(PushNotification::StateChange( + StateChange { + account_id, + change_id: assigned_ids + .last_change_id(account_id, ChangeGroup::from_u8(group as u8)), + types: *types, + }, + )) + .await; } } - if state_change.has_changes() { - self.broadcast_push_notification(PushNotification::StateChange( - state_change.with_change_id(assigned_ids.last_change_id(account_id)?), - )) - .await; - } + if let Some(change_id) = changed_collections.share_notification_id { self.broadcast_push_notification(PushNotification::StateChange(StateChange { account_id, @@ -62,6 +100,19 @@ impl Server { Ok(assigned_ids) } + async fn invalidate_share_grantees(&self, changes: &VecMap) { + let mut acl_changes = CacheInvalidationBuilder::default(); + for (account_id, _) in changes + .iter() + .filter(|(_, changed)| changed.share_notification_id.is_some()) + { + self.invalidate_grantee(&mut acl_changes, *account_id).await; + } + if let Err(err) = self.invalidate_caches(acl_changes).await { + trc::error!(err.caused_by(trc::location!())); + } + } + pub async fn delete_changes( &self, account_id: u32, @@ -80,92 +131,31 @@ impl Server { SyncCollection::Calendar, SyncCollection::CalendarEventNotification, ] { - let collection = sync_collection.into(); - let from_key = LogKey { - account_id, - collection, - change_id: 0, - }; - let to_key = LogKey { - account_id, - collection, - change_id: u64::MAX, + let Some(first_change_id) = self + .truncate_change_log(account_id, sync_collection.into(), max_entries) + .await? + else { + continue; }; - let mut first_change_id = 0; - let mut num_changes = 0; - - self.store() - .iterate( - IterateParams::new(from_key, to_key) - .descending() - .no_values(), - |key, _| { - first_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; - num_changes += 1; - - Ok(num_changes <= max_entries) - }, - ) - .await - .caused_by(trc::location!())?; - - if num_changes > max_entries { + if let Some(vanished_collection) = + sync_collection.vanished_collection().map(u8::from) + { self.store() .delete_range( LogKey { account_id, - collection, + collection: vanished_collection, change_id: 0, }, LogKey { account_id, - collection, + collection: vanished_collection, change_id: first_change_id, }, ) .await .caused_by(trc::location!())?; - - // Delete vanished items - if let Some(vanished_collection) = - sync_collection.vanished_collection().map(u8::from) - { - self.store() - .delete_range( - LogKey { - account_id, - collection: vanished_collection, - change_id: 0, - }, - LogKey { - account_id, - collection: vanished_collection, - change_id: first_change_id, - }, - ) - .await - .caused_by(trc::location!())?; - } - - // Write truncation entry for cache - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id).set( - ValueClass::Any(AnyClass { - subspace: SUBSPACE_LOGS, - key: LogKey { - account_id, - collection, - change_id: first_change_id, - } - .serialize(0), - }), - Vec::new(), - ); - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; } } } @@ -191,6 +181,60 @@ impl Server { Ok(()) } + pub async fn truncate_change_log( + &self, + account_id: u32, + collection: LogCollection, + max_entries: usize, + ) -> trc::Result> { + let mut first_change_id = 0; + let mut num_changes = 0; + + self.store() + .iterate( + IterateParams::new( + collection.log_key(account_id, 0), + collection.log_key(account_id, u64::MAX), + ) + .descending() + .no_values() + .limit(max_entries.saturating_add(1)), + |key, _| { + first_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + num_changes += 1; + + Ok(num_changes <= max_entries) + }, + ) + .await + .caused_by(trc::location!())?; + + if num_changes <= max_entries { + return Ok(None); + } + + let first_key = collection.log_key(account_id, first_change_id); + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id).set( + ValueClass::Any(AnyClass { + subspace: Subspace::Logs, + key: first_key.serialize(0), + }), + Vec::new(), + ); + self.store() + .write_batch(&mut batch) + .await + .caused_by(trc::location!())?; + + self.store() + .delete_range(collection.log_key(account_id, 0), first_key) + .await + .caused_by(trc::location!())?; + + Ok(Some(first_change_id)) + } + #[inline(always)] pub fn generate_snowflake_id(&self) -> u64 { self.inner.data.jmap_id_gen.generate() diff --git a/crates/common/src/telemetry/metrics/mod.rs b/crates/common/src/telemetry/metrics/mod.rs index 831aa425..424b5946 100644 --- a/crates/common/src/telemetry/metrics/mod.rs +++ b/crates/common/src/telemetry/metrics/mod.rs @@ -4,7 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{Inner, network::memory::MemoryClass}; +use trc::{Collector, MetricType}; + pub mod otel; pub mod prometheus; + +impl Inner { + pub fn refresh_live_gauges(&self) { + let budget = &self.data.memory_budget; + Collector::update_gauge( + MetricType::ServerBufferedMemorySmtp, + budget.used(MemoryClass::SmtpInbound), + ); + Collector::update_gauge( + MetricType::ServerBufferedMemoryRead, + budget.used(MemoryClass::Read), + ); + Collector::update_gauge( + MetricType::CacheSwapPending, + self.cache.swap.pending_targets(), + ); + } +} diff --git a/crates/common/src/telemetry/metrics/otel.rs b/crates/common/src/telemetry/metrics/otel.rs index e7dd9efd..2f4291ac 100644 --- a/crates/common/src/telemetry/metrics/otel.rs +++ b/crates/common/src/telemetry/metrics/otel.rs @@ -5,6 +5,7 @@ */ use crate::config::telemetry::OtelMetrics; +use compact_str::ToCompactString; use opentelemetry_sdk::metrics::{ Temporality, data::{ @@ -83,7 +84,7 @@ impl OtelMetrics { if let Err(err) = self.exporter.export(&rm).await { trc::event!( Telemetry(TelemetryEvent::OtelMetricsExporterError), - Reason = err.to_string(), + Reason = err.to_compact_string(), ); } } @@ -93,7 +94,7 @@ impl OtelMetrics { /*let _ = set_error_handler(|error| { trc::event!( Telemetry(TelemetryEvent::OtelMetricsExporterError), - Reason = error.to_string(), + Reason = error.to_compact_string(), ); });*/ } diff --git a/crates/common/src/telemetry/metrics/prometheus.rs b/crates/common/src/telemetry/metrics/prometheus.rs index cea5bfd1..cbcb71bd 100644 --- a/crates/common/src/telemetry/metrics/prometheus.rs +++ b/crates/common/src/telemetry/metrics/prometheus.rs @@ -20,6 +20,10 @@ impl Server { #[cfg(not(feature = "enterprise"))] let is_enterprise = false; + if is_enterprise { + self.inner.refresh_live_gauges(); + } + // Add counters for counter in Collector::collect_counters(is_enterprise) { let mut metric = MetricFamily::default(); diff --git a/crates/common/src/telemetry/mod.rs b/crates/common/src/telemetry/mod.rs index 363a761e..ecb44b36 100644 --- a/crates/common/src/telemetry/mod.rs +++ b/crates/common/src/telemetry/mod.rs @@ -69,33 +69,10 @@ impl Telemetry { Collector::set_metrics(self.metrics); Collector::reload(); } - - #[cfg(feature = "test_mode")] - pub fn test_tracer(level: trc::Level) { - let mut interests = trc::ipc::subscriber::Interests::default(); - for event in trc::EventType::variants() { - if level.is_contained(event.level()) { - interests.set(*event); - } - } - - spawn_console_tracer( - SubscriberBuilder::new("stderr".to_string()) - .with_interests(interests.clone()) - .with_lossy(false), - crate::config::telemetry::ConsoleTracer { - ansi: true, - multiline: false, - buffered: false, - }, - ); - - Collector::union_interests(interests); - Collector::reload(); - } } impl TelemetrySubscriberType { + #[cfg_attr(not(feature = "enterprise"), allow(unused_variables))] pub fn spawn(self, builder: SubscriberBuilder, is_enterprise: bool) { match self { TelemetrySubscriberType::ConsoleTracer(settings) => { @@ -111,3 +88,4 @@ impl TelemetrySubscriberType { } } } + diff --git a/crates/common/src/telemetry/tracers/journald.rs b/crates/common/src/telemetry/tracers/journald.rs index a2376510..5751f77c 100644 --- a/crates/common/src/telemetry/tracers/journald.rs +++ b/crates/common/src/telemetry/tracers/journald.rs @@ -4,27 +4,66 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashSet; +use compact_str::ToCompactString; +#[cfg(target_os = "linux")] use std::io::Write; +use std::thread::Builder; use trc::ipc::subscriber::SubscriberBuilder; -use trc::{Event, EventDetails, Level, TelemetryEvent}; +use trc::{Event, EventDetails, Level, TelemetryEvent, event::KeySet}; pub(crate) fn spawn_journald_tracer(builder: SubscriberBuilder, subscriber: Subscriber) { - let (_, mut rx) = builder.register(); - tokio::spawn(async move { - while let Some(events) = rx.recv().await { - for event in events { - subscriber.send_event(&event); + let (_, mut rx, _) = builder.register(); + if let Err(err) = Builder::new() + .name("stalwart-journald".to_string()) + .spawn(move || { + let mut buf = Vec::with_capacity(1024); + let mut lost_events = 0u64; + let mut is_failing = false; + + while let Some(events) = rx.blocking_recv() { + for event in events { + buf.clear(); + subscriber.encode_event(&event, &mut buf); + + match subscriber.send_payload(&buf) { + Ok(_) => { + if is_failing { + is_failing = false; + trc::event!( + Telemetry(TelemetryEvent::JournalError), + Details = "Resumed sending events to journald", + Total = std::mem::take(&mut lost_events), + ); + } + } + Err(err) => { + lost_events += 1; + if !is_failing { + is_failing = true; + trc::event!( + Telemetry(TelemetryEvent::JournalError), + Details = "Failed to send event to journald", + Reason = err.to_compact_string(), + ); + } + } + } + } } - } - }); + }) + { + trc::event!( + Telemetry(TelemetryEvent::JournalError), + Details = "Failed to spawn journald writer thread", + Reason = err.to_compact_string(), + ); + } } impl Subscriber { - fn send_event(&self, event: &Event) { - let mut buf = Vec::with_capacity(256); + fn encode_event(&self, event: &Event, buf: &mut Vec) { put_field_wellformed( - &mut buf, + buf, "PRIORITY", &[match event.inner.level { Level::Error => self.priority_mappings.error as u8, @@ -34,35 +73,25 @@ impl Subscriber { Level::Trace | Level::Disable => self.priority_mappings.trace as u8, }], ); - put_field_length_encoded(&mut buf, "SYSLOG_IDENTIFIER", |buf| { - write!(buf, "{}", self.syslog_identifier).unwrap() + put_field_length_encoded(buf, "SYSLOG_IDENTIFIER", |buf| { + buf.extend_from_slice(self.syslog_identifier.as_bytes()) }); - put_field_length_encoded(&mut buf, "MESSAGE", |buf| { - write!(buf, "{}", event.inner.typ.description()).unwrap() + put_field_length_encoded(buf, "MESSAGE", |buf| { + buf.extend_from_slice(event.inner.typ.description().as_bytes()) }); - let mut seen_keys = AHashSet::new(); + let mut seen_keys = KeySet::default(); for (key, value) in event.keys.iter().chain( event .inner .span .as_ref() - .map_or(([]).iter(), |span| span.keys.iter()), + .map_or(&[][..], |span| span.keys.as_slice()), ) { if seen_keys.insert(*key) { - put_field_length_encoded(&mut buf, key.as_str(), |buf| { - write!(buf, "{value}").unwrap() - }); + put_field_length_encoded(buf, key.as_str(), |buf| value.write_display(buf)); } } - - if let Err(err) = self.send_payload(&buf) { - trc::event!( - Telemetry(TelemetryEvent::JournalError), - Details = "Failed to send event to journald", - Reason = err.to_string() - ); - } } } @@ -325,9 +354,7 @@ impl std::fmt::Debug for Subscriber { /// not delete from `buf`, but may append arbitrary data. This function then determines the length /// of the data written and adds it in the appropriate place in `buf`. fn put_field_length_encoded(buf: &mut Vec, name: &str, write_value: impl FnOnce(&mut Vec)) { - for ch in name.as_bytes() { - buf.push(ch.to_ascii_uppercase()); - } + buf.extend(name.bytes().map(|ch| ch.to_ascii_uppercase())); buf.push(b'\n'); buf.extend_from_slice(&[0; 8]); // Length tag, to be populated let start = buf.len(); @@ -404,8 +431,8 @@ pub fn send_one_fd_to>(socket: &UnixDatagram, fd: RawFd, path: P) msg.msg_control = unsafe { cmsg_buffer.buffer.as_mut_ptr() as _ }; msg.msg_controllen = unsafe { CMSG_SPACE(size_of::() as _) as _ }; - let cmsg: &mut cmsghdr = - unsafe { CMSG_FIRSTHDR(&msg).as_mut() }.expect("Control message buffer exhausted"); + let cmsg: &mut cmsghdr = unsafe { CMSG_FIRSTHDR(&msg).as_mut() } + .ok_or_else(|| Error::other("Control message buffer exhausted"))?; cmsg.cmsg_level = SOL_SOCKET; cmsg.cmsg_type = SCM_RIGHTS; @@ -441,13 +468,7 @@ fn create(flags: c_uint) -> Result { /// See: https://github.com/tokio-rs/tracing/issues/1879 #[cfg(target_os = "linux")] fn memfd_create_syscall(flags: c_uint) -> c_int { - unsafe { - syscall( - SYS_memfd_create, - "tracing-journald\0".as_ptr() as *const c_char, - flags, - ) as c_int - } + unsafe { syscall(SYS_memfd_create, c"tracing-journald".as_ptr(), flags) as c_int } } #[cfg(target_os = "linux")] diff --git a/crates/common/src/telemetry/tracers/log.rs b/crates/common/src/telemetry/tracers/log.rs index 455db0f8..088bac03 100644 --- a/crates/common/src/telemetry/tracers/log.rs +++ b/crates/common/src/telemetry/tracers/log.rs @@ -4,75 +4,171 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{path::PathBuf, time::SystemTime}; +use std::{ + fs::{File, OpenOptions}, + io::{ErrorKind, Write}, + path::PathBuf, + thread::Builder, + time::SystemTime, +}; use crate::config::telemetry::{LogTracer, RotationStrategy}; +use compact_str::{CompactString, ToCompactString}; use mail_parser::DateTime; -use tokio::{ - fs::{File, OpenOptions}, - io::BufWriter, +use tokio::sync::mpsc::Receiver; +use trc::{ + TelemetryEvent, + ipc::subscriber::{EventBatch, SubscriberBuilder}, + serializers::text::FmtWriter, }; -use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; + +const FLUSH_THRESHOLD: usize = 1 << 16; pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) { - let (_, mut rx) = builder.register(); - tokio::spawn(async move { - if let Some(writer) = settings.build_writer().await { - let mut buf = FmtWriter::new(writer) + let (_, rx, _) = builder.register(); + if let Err(err) = Builder::new() + .name("stalwart-log".to_string()) + .spawn(move || LogWriter::new(settings).run(rx)) + { + trc::event!( + Telemetry(TelemetryEvent::LogError), + Details = "Failed to spawn log writer thread", + Reason = err.to_compact_string(), + ); + } +} + +struct LogWriter { + formatter: FmtWriter, + settings: LogTracer, + file: Option, + file_timestamp: u64, + rotation_timestamp: u64, + buf: Vec, + lost_events: u64, + pending_events: u64, + is_failing: bool, + failure_kind: Option, + is_torn: bool, +} + +impl LogWriter { + fn new(settings: LogTracer) -> Self { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + + Self { + formatter: FmtWriter::new() .with_ansi(settings.ansi) - .with_multiline(settings.multiline); - let mut roatation_timestamp = settings.next_rotation(); + .with_multiline(settings.multiline), + file: None, + file_timestamp: now, + rotation_timestamp: settings.next_rotation(now), + settings, + buf: Vec::with_capacity(FLUSH_THRESHOLD), + lost_events: 0, + pending_events: 0, + is_failing: false, + failure_kind: None, + is_torn: false, + } + } - while let Some(events) = rx.recv().await { - for event in events { - // Check if we need to rotate the log file - if roatation_timestamp != 0 && event.inner.timestamp > roatation_timestamp { - if let Err(err) = buf.flush().await { - trc::event!( - Telemetry(TelemetryEvent::LogError), - Reason = err.to_string(), - Details = "Failed to flush log buffer" - ); - } + fn run(mut self, mut rx: Receiver) { + self.flush(); - if let Some(writer) = settings.build_writer().await { - buf.update_writer(writer); - roatation_timestamp = settings.next_rotation(); - } else { - return; - }; - } - - if let Err(err) = buf.write(&event).await { - trc::event!( - Telemetry(TelemetryEvent::LogError), - Reason = err.to_string(), - Details = "Failed to write event to log" - ); - return; - } + while let Some(events) = rx.blocking_recv() { + for event in events { + let timestamp = event.inner.timestamp; + if self.rotation_timestamp != 0 && timestamp > self.rotation_timestamp { + self.flush(); + self.file = None; + self.file_timestamp = timestamp; + self.rotation_timestamp = self.settings.next_rotation(timestamp); } - if let Err(err) = buf.flush().await { + self.formatter.write(&event, &mut self.buf); + self.pending_events += 1; + if self.buf.len() >= FLUSH_THRESHOLD { + self.flush(); + } + } + + self.flush(); + } + } + + fn flush(&mut self) { + if self.is_torn && !self.buf.is_empty() { + self.buf.insert(0, b'\n'); + } + + let result = match &mut self.file { + Some(file) => file.write_all(&self.buf), + None => match self.settings.open(self.file_timestamp) { + Ok(file) => self.file.insert(file).write_all(&self.buf), + Err((path, err)) => { + self.failed(err, "Failed to create log file", Some(path)); + return; + } + }, + }; + + match result { + Ok(()) => { + if !self.buf.is_empty() { + self.is_torn = false; + } + self.buf.clear(); + self.pending_events = 0; + if self.is_failing { + self.is_failing = false; + self.failure_kind = None; trc::event!( Telemetry(TelemetryEvent::LogError), - Reason = err.to_string(), - Details = "Failed to flush log buffer" + Details = "Resumed writing to log file", + Total = std::mem::take(&mut self.lost_events), ); } } + Err(err) => { + self.file = None; + self.is_torn = true; + self.failed(err, "Failed to write to log file", None); + } } - }); + } + + fn failed(&mut self, err: std::io::Error, details: &'static str, path: Option) { + self.buf.clear(); + self.lost_events += std::mem::take(&mut self.pending_events); + if self.is_failing && self.failure_kind == Some(err.kind()) { + return; + } + self.is_failing = true; + self.failure_kind = Some(err.kind()); + + match path { + Some(path) => trc::event!( + Telemetry(TelemetryEvent::LogError), + Details = details, + Path = path, + Reason = err.to_compact_string(), + ), + None => trc::event!( + Telemetry(TelemetryEvent::LogError), + Details = details, + Reason = err.to_compact_string(), + ), + } + } } impl LogTracer { - pub async fn build_writer(&self) -> Option> { - let now = DateTime::from_timestamp( - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) as i64, - ); + fn open(&self, timestamp: u64) -> Result { + let now = DateTime::from_timestamp(timestamp as i64); let file_name = match self.rotate { RotationStrategy::Daily => { format!( @@ -96,31 +192,15 @@ impl LogTracer { }; let path = PathBuf::from(&self.path).join(file_name); - match OpenOptions::new() + OpenOptions::new() .create(true) .append(true) .open(&path) - .await - { - Ok(writer) => Some(BufWriter::new(writer)), - Err(err) => { - trc::event!( - Telemetry(TelemetryEvent::LogError), - Details = "Failed to create log file", - Path = path.to_string_lossy().into_owned(), - Reason = err.to_string(), - ); - None - } - } + .map_err(|err| (CompactString::from(path.to_string_lossy()), err)) } - pub fn next_rotation(&self) -> u64 { - let mut now = DateTime::from_timestamp( - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) as i64, - ); + fn next_rotation(&self, timestamp: u64) -> u64 { + let mut now = DateTime::from_timestamp(timestamp as i64); now.second = 0; diff --git a/crates/common/src/telemetry/tracers/mod.rs b/crates/common/src/telemetry/tracers/mod.rs index 3461756b..28aa822e 100644 --- a/crates/common/src/telemetry/tracers/mod.rs +++ b/crates/common/src/telemetry/tracers/mod.rs @@ -8,105 +8,6 @@ pub mod journald; pub mod log; pub mod otel; +pub mod spans; pub mod stdout; - -use registry::{ - schema::structs::{ - Trace, TraceEvent, TraceKeyValue, TraceValue, TraceValueBoolean, TraceValueDuration, - TraceValueEvent, TraceValueFloat, TraceValueInteger, TraceValueIpAddr, TraceValueList, - TraceValueString, TraceValueUTCDateTime, TraceValueUnsignedInt, - }, - types::{datetime::UTCDateTime, ipaddr::IpAddr, list::List}, -}; -use trc::{Event, EventDetails, Value}; - -pub trait TraceEvents { - fn build_trace_events<'x>( - span_events: impl IntoIterator>, - num_events: usize, - ) -> Vec; - - fn from_events<'x>( - span_events: impl IntoIterator>, - num_events: usize, - ) -> Self; -} - -impl TraceEvents for Trace { - fn build_trace_events<'x>( - span_events: impl IntoIterator>, - num_events: usize, - ) -> Vec { - let mut events = Vec::with_capacity(num_events); - - for event in span_events { - let mut key_values = Vec::with_capacity(event.keys.len()); - for (key, value) in &event.keys { - key_values.push(TraceKeyValue { - key: *key, - value: map_value(value), - }); - } - - events.push(TraceEvent { - event: event.inner.typ, - timestamp: UTCDateTime::from_timestamp(event.inner.timestamp as i64), - key_values: key_values.into(), - }); - } - - events - } - - fn from_events<'x>( - span_events: impl IntoIterator>, - num_events: usize, - ) -> Self { - Trace { - events: Self::build_trace_events(span_events, num_events).into(), - } - } -} - -fn map_value(value: &Value) -> TraceValue { - match value { - Value::String(value) => TraceValue::String(TraceValueString { - value: value.to_string(), - }), - Value::UInt(value) => TraceValue::UnsignedInt(TraceValueUnsignedInt { value: *value }), - Value::Int(value) => TraceValue::Integer(TraceValueInteger { value: *value }), - Value::Float(value) => TraceValue::Float(TraceValueFloat { - value: (*value).into(), - }), - Value::Timestamp(value) => TraceValue::UTCDateTime(TraceValueUTCDateTime { - value: UTCDateTime::from_timestamp(*value as i64), - }), - Value::Duration(value) => TraceValue::Duration(TraceValueDuration { value: *value }), - Value::Bytes(items) => TraceValue::String(TraceValueString { - value: String::from_utf8_lossy(items).to_string(), - }), - Value::Bool(value) => TraceValue::Boolean(TraceValueBoolean { value: *value }), - Value::Ipv4(ipv4_addr) => TraceValue::IpAddr(TraceValueIpAddr { - value: IpAddr((*ipv4_addr).into()), - }), - Value::Ipv6(ipv6_addr) => TraceValue::IpAddr(TraceValueIpAddr { - value: IpAddr((*ipv6_addr).into()), - }), - Value::Event(event) => TraceValue::Event(TraceValueEvent { - value: event - .keys() - .iter() - .map(|(k, v)| TraceKeyValue { - key: *k, - value: map_value(v), - }) - .collect(), - event: event.event_type(), - }), - Value::Array(values) => TraceValue::List(TraceValueList { - value: List::from_iter(values.iter().map(map_value)), - }), - Value::None => TraceValue::Null, - } -} diff --git a/crates/common/src/telemetry/tracers/otel.rs b/crates/common/src/telemetry/tracers/otel.rs index 79903c8f..a62389cb 100644 --- a/crates/common/src/telemetry/tracers/otel.rs +++ b/crates/common/src/telemetry/tracers/otel.rs @@ -5,7 +5,7 @@ */ use crate::{LONG_1Y_SLUMBER, config::telemetry::OtelTracer}; -use ahash::{AHashMap, AHashSet}; +use compact_str::ToCompactString; use mail_parser::DateTime; use opentelemetry::{ InstrumentationScope, Key, KeyValue, Value, @@ -19,12 +19,17 @@ use opentelemetry_sdk::{ }; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use trc::{Event, EventDetails, Level, TelemetryEvent, ipc::subscriber::SubscriberBuilder}; +use trc::{ + Event, EventDetails, Level, TelemetryEvent, event::KeySet, ipc::subscriber::SubscriberBuilder, +}; -const MAX_EVENTS: usize = 2048; +use super::spans::{SpanTracker, report_discarded_spans, tracks_span}; + +const MAX_PENDING_LOGS: usize = 1 << 16; +const MAX_PENDING_SPANS: usize = 1 << 14; pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer) { - let (_, mut rx) = builder.register(); + let (_, mut rx, interests) = builder.register(); tokio::spawn(async move { let resource = Resource::builder() .with_service_name("stalwart") @@ -44,7 +49,9 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer let mut pending_logs = Vec::new(); let mut pending_spans = Vec::new(); - let mut active_spans = AHashMap::new(); + let mut active_spans = SpanTracker::default(); + let mut discarded_logs = 0usize; + let mut discarded_spans = 0usize; loop { // Wait for the next event or timeout @@ -52,9 +59,20 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer match event_or_timeout { Ok(Some(events)) => { + let observed = SystemTime::now(); + if otel.span_exporter_enable + && let Some(event) = events.last() + { + active_spans.sweep(event.inner.timestamp); + } + for event in events { if otel.log_exporter_enable { - pending_logs.push(otel.build_log_record(&event)); + if pending_logs.len() < MAX_PENDING_LOGS { + pending_logs.push(otel.build_log_record(&event, observed)); + } else { + discarded_logs += 1; + } } if otel.span_exporter_enable @@ -62,20 +80,28 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer { let span_id = span.span_id().unwrap(); if !event.inner.typ.is_span_end() { - let events = active_spans.entry(span_id).or_insert_with(Vec::new); - if events.len() < MAX_EVENTS { - events.push(event); + if !tracks_span(&interests, span) { + continue; + } + active_spans.track(span_id, event); + } else if let Some(events) = active_spans.finish(span_id) { + if pending_spans.len() < MAX_PENDING_SPANS { + pending_spans.push(build_span_data( + span, + &event, + events.iter().chain(std::iter::once(&event)), + &instrumentation, + )); + } else { + discarded_spans += 1; } - } else if let Some(events) = active_spans.remove(&span_id) { - pending_spans.push(build_span_data( - span, - &event, - events.iter().chain(std::iter::once(&event)), - &instrumentation, - )); } } } + + if let Some((events, spans)) = active_spans.take_discarded() { + report_discarded_spans(events, spans); + } } Ok(None) => { break; @@ -90,18 +116,35 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer if !pending_spans.is_empty() || !pending_logs.is_empty() { next_delivery = now + otel.throttle; - if !pending_spans.is_empty() - && let Err(err) = otel - .span_exporter - .export(std::mem::take(&mut pending_spans)) - .await - { + if discarded_logs > 0 { trc::event!( Telemetry(TelemetryEvent::OtelExporterError), - Details = "Failed to export spans", - Reason = err.to_string() + Details = "Log export queue full, discarded logs", + Total = std::mem::take(&mut discarded_logs), ); } + if discarded_spans > 0 { + trc::event!( + Telemetry(TelemetryEvent::OtelExporterError), + Details = "Span export queue full, discarded spans", + Total = std::mem::take(&mut discarded_spans), + ); + } + + if !pending_spans.is_empty() { + match otel.span_exporter.export(pending_spans.clone()).await { + Ok(_) => { + pending_spans.clear(); + } + Err(err) => { + trc::event!( + Telemetry(TelemetryEvent::OtelExporterError), + Details = "Failed to export spans, will retry", + Reason = err.to_compact_string() + ); + } + } + } if !pending_logs.is_empty() { let logs = pending_logs @@ -109,14 +152,25 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer .map(|log| (log, &instrumentation)) .collect::>(); - if let Err(err) = otel.log_exporter.export(LogBatch::new(&logs)).await { - trc::event!( - Telemetry(TelemetryEvent::OtelExporterError), - Details = "Failed to export logs", - Reason = err.to_string() - ); + let result = otel.log_exporter.export(LogBatch::new(&logs)).await; + drop(logs); + + match result { + Ok(_) => { + pending_logs.clear(); + } + Err(err) => { + trc::event!( + Telemetry(TelemetryEvent::OtelExporterError), + Details = "Failed to export logs, will retry", + Reason = err.to_compact_string() + ); + } } - pending_logs.clear(); + } + + if !pending_spans.is_empty() || !pending_logs.is_empty() { + next_retry = Some(next_delivery.saturating_duration_since(Instant::now())); } } } else if !pending_logs.is_empty() || !pending_spans.is_empty() { @@ -185,7 +239,7 @@ where } impl OtelTracer { - fn build_log_record(&self, event: &Event) -> SdkLogRecord { + fn build_log_record(&self, event: &Event, observed: SystemTime) -> SdkLogRecord { use opentelemetry::logs::LogRecord; let mut record = SdkLogRecord::new(); @@ -201,13 +255,13 @@ impl OtelTracer { record.set_severity_text(event.inner.level.as_str()); record.set_body(AnyValue::String(event.inner.typ.description().into())); record.set_timestamp(UNIX_EPOCH + Duration::from_secs(event.inner.timestamp)); - record.set_observed_timestamp(SystemTime::now()); + record.set_observed_timestamp(observed); if let Some(span_id) = event.span_id().filter(|span_id| *span_id != 0) { record.set_trace_context((span_id as u128).into(), span_id.into(), None); } - let mut seen_keys = AHashSet::new(); + let mut seen_keys = KeySet::default(); for (k, v) in event.keys.iter().chain( event .inner @@ -229,7 +283,7 @@ fn build_key_value(key_value: &(trc::Key, trc::Value)) -> Option { KeyValue::new( build_key(&key_value.0), match &key_value.1 { - trc::Value::String(v) => Value::String(v.to_string().into()), + trc::Value::String(v) => Value::String(String::from(v.as_str()).into()), trc::Value::UInt(v) => Value::I64(*v as i64), trc::Value::Int(v) => Value::I64(*v), trc::Value::Float(v) => Value::F64(*v), @@ -255,7 +309,7 @@ fn build_key(key: &trc::Key) -> Key { fn build_any_value(value: &trc::Value) -> AnyValue { match value { - trc::Value::String(v) => AnyValue::String(v.to_string().into()), + trc::Value::String(v) => AnyValue::String(String::from(v.as_str()).into()), trc::Value::UInt(v) => AnyValue::Int(*v as i64), trc::Value::Int(v) => AnyValue::Int(*v), trc::Value::Float(v) => AnyValue::Double(*v), @@ -263,7 +317,7 @@ fn build_any_value(value: &trc::Value) -> AnyValue { AnyValue::String(DateTime::from_timestamp(*v as i64).to_rfc3339().into()) } trc::Value::Duration(v) => AnyValue::Int(*v as i64), - trc::Value::Bytes(v) => AnyValue::Bytes(Box::new(v.clone())), + trc::Value::Bytes(_) => AnyValue::String("[binary data]".into()), trc::Value::Bool(v) => AnyValue::Boolean(*v), trc::Value::Ipv4(v) => AnyValue::String(v.to_string().into()), trc::Value::Ipv6(v) => AnyValue::String(v.to_string().into()), diff --git a/crates/common/src/telemetry/tracers/spans.rs b/crates/common/src/telemetry/tracers/spans.rs new file mode 100644 index 00000000..fbf13ae7 --- /dev/null +++ b/crates/common/src/telemetry/tracers/spans.rs @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use ahash::AHashMap; +use std::sync::Arc; +use trc::{ + Event, EventDetails, TelemetryEvent, + ipc::{collector::SPAN_MAX_HOLD, subscriber::SharedInterests}, +}; + +const MAX_SPAN_EVENTS: usize = 2048; +const MAX_TRACKED_EVENTS: usize = 1 << 17; +const SWEEP_INTERVAL: u64 = 60 * 60; +const PRESSURE_SWEEP_INTERVAL: u64 = 60; +const PRESSURE_MAX_HOLD: u64 = 60 * 60; + +type SpanEvents = Vec>>; + +#[derive(Default)] +pub(crate) struct SpanTracker { + spans: AHashMap, + tracked_events: usize, + discarded_events: usize, + discarded_spans: usize, + next_sweep: u64, + next_pressure_sweep: u64, +} + +impl SpanTracker { + pub fn track(&mut self, span_id: u64, event: Arc>) { + if self.tracked_events >= MAX_TRACKED_EVENTS { + let timestamp = event.inner.timestamp; + if timestamp < self.next_pressure_sweep { + self.discarded_events += 1; + return; + } + self.next_pressure_sweep = timestamp + PRESSURE_SWEEP_INTERVAL; + self.retain_younger_than(PRESSURE_MAX_HOLD, timestamp); + if self.tracked_events >= MAX_TRACKED_EVENTS { + self.discarded_events += 1; + return; + } + } + + let events = self.spans.entry(span_id).or_default(); + if events.len() < MAX_SPAN_EVENTS { + events.push(event); + self.tracked_events += 1; + } else { + self.discarded_events += 1; + } + } + + pub fn take_discarded(&mut self) -> Option<(usize, usize)> { + if self.discarded_events > 0 || self.discarded_spans > 0 { + Some(( + std::mem::take(&mut self.discarded_events), + std::mem::take(&mut self.discarded_spans), + )) + } else { + None + } + } + + pub fn finish(&mut self, span_id: u64) -> Option { + let events = self.spans.remove(&span_id)?; + self.tracked_events -= events.len(); + Some(events) + } + + pub fn sweep(&mut self, timestamp: u64) { + if timestamp >= self.next_sweep { + self.next_sweep = timestamp + SWEEP_INTERVAL; + self.retain_younger_than(SPAN_MAX_HOLD, timestamp); + } + } + + fn retain_younger_than(&mut self, max_age: u64, timestamp: u64) { + let mut tracked_events = 0; + let mut discarded_spans = 0; + self.spans.retain(|_, events| { + let keep = events + .first() + .and_then(|event| event.inner.span.as_ref()) + .is_some_and(|span| timestamp.saturating_sub(span.inner.timestamp) < max_age); + if keep { + tracked_events += events.len(); + } else { + discarded_spans += 1; + } + keep + }); + self.tracked_events = tracked_events; + self.discarded_spans += discarded_spans; + } +} + +pub(crate) fn tracks_span(interests: &SharedInterests, span: &Event) -> bool { + span.inner + .typ + .span_end() + .is_some_and(|span_end| interests.get(span_end.to_id() as usize)) +} + +pub(crate) fn report_discarded_spans(events: usize, spans: usize) { + if events > 0 { + trc::event!( + Telemetry(TelemetryEvent::EventsDropped), + Details = "Span tracker full, discarded span events", + Total = events, + ); + } + if spans > 0 { + trc::event!( + Telemetry(TelemetryEvent::EventsDropped), + Details = "Span tracker evicted incomplete spans", + Total = spans, + ); + } +} + diff --git a/crates/common/src/telemetry/tracers/stdout.rs b/crates/common/src/telemetry/tracers/stdout.rs index be591f1c..edaa66d0 100644 --- a/crates/common/src/telemetry/tracers/stdout.rs +++ b/crates/common/src/telemetry/tracers/stdout.rs @@ -5,90 +5,51 @@ */ use std::{ - io::{Error, stderr}, - pin::Pin, - task::{Context, Poll}, + io::{Write, stderr}, + thread::Builder, }; use crate::config::telemetry::ConsoleTracer; -use std::io::Write; -use tokio::io::AsyncWrite; -use trc::{ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; +use compact_str::ToCompactString; +use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; + +const FLUSH_THRESHOLD: usize = 1 << 16; pub(crate) fn spawn_console_tracer(builder: SubscriberBuilder, settings: ConsoleTracer) { - let (_, mut rx) = builder.register(); - tokio::spawn(async move { - let mut buf = FmtWriter::new(StdErrWriter::default()) - .with_ansi(settings.ansi) - .with_multiline(settings.multiline); + let (_, mut rx, _) = builder.register(); + if let Err(err) = Builder::new() + .name("stalwart-console".to_string()) + .spawn(move || { + let mut formatter = FmtWriter::new() + .with_ansi(settings.ansi) + .with_multiline(settings.multiline); + let mut buf = Vec::with_capacity(if settings.buffered { + FLUSH_THRESHOLD + } else { + 1024 + }); - while let Some(events) = rx.recv().await { - for event in events { - let _ = buf.write(&event).await; + while let Some(events) = rx.blocking_recv() { + let mut stderr = stderr().lock(); + for event in events { + formatter.write(&event, &mut buf); + if !settings.buffered || buf.len() >= FLUSH_THRESHOLD { + let _ = stderr.write_all(&buf); + buf.clear(); + } + } - if !settings.buffered { - let _ = buf.flush().await; + if !buf.is_empty() { + let _ = stderr.write_all(&buf); + buf.clear(); } } - - if settings.buffered { - let _ = buf.flush().await; - } - } - }); -} - -const BUFFER_CAPACITY: usize = 4096; - -pub struct StdErrWriter { - buffer: Vec, -} - -impl AsyncWrite for StdErrWriter { - fn poll_write( - mut self: Pin<&mut Self>, - _: &mut Context<'_>, - bytes: &[u8], - ) -> Poll> { - let bytes_len = bytes.len(); - let buffer_len = self.buffer.len(); - - if buffer_len + bytes_len < BUFFER_CAPACITY { - self.buffer.extend_from_slice(bytes); - Poll::Ready(Ok(bytes_len)) - } else if bytes_len > BUFFER_CAPACITY { - let result = stderr() - .write_all(&self.buffer) - .and_then(|_| stderr().write_all(bytes)); - self.buffer.clear(); - Poll::Ready(result.map(|_| bytes_len)) - } else { - let result = stderr().write_all(&self.buffer); - self.buffer.clear(); - self.buffer.extend_from_slice(bytes); - Poll::Ready(result.map(|_| bytes_len)) - } - } - - fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(if !self.buffer.is_empty() { - let result = stderr().write_all(&self.buffer); - self.buffer.clear(); - result - } else { - Ok(()) }) - } - - fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} - -impl Default for StdErrWriter { - fn default() -> Self { - Self { - buffer: Vec::with_capacity(BUFFER_CAPACITY), - } + { + trc::event!( + Telemetry(TelemetryEvent::LogError), + Details = "Failed to spawn console writer thread", + Reason = err.to_compact_string(), + ); } } diff --git a/crates/common/src/telemetry/webhooks/mod.rs b/crates/common/src/telemetry/webhooks/mod.rs index 33f966ae..c2659ed0 100644 --- a/crates/common/src/telemetry/webhooks/mod.rs +++ b/crates/common/src/telemetry/webhooks/mod.rs @@ -6,165 +6,270 @@ use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer}; use aws_lc_rs::hmac; -use base64::{Engine, engine::general_purpose::STANDARD}; -use serde::Serialize; -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Instant, -}; +use bytes::Bytes; +use compact_str::{CompactString, format_compact}; +use encodify::base64::STANDARD; +use reqwest::header::HeaderValue; +use std::{sync::Arc, time::Instant}; use store::write::now; -use tokio::sync::mpsc; +use tokio::task::JoinHandle; use trc::{ - Event, EventDetails, ServerEvent, TelemetryEvent, + TelemetryEvent, ipc::subscriber::{EventBatch, SubscriberBuilder}, serializers::json::JsonEventSerializer, }; +struct Payload { + body: Bytes, + signature: Option, + events: EventBatch, + oldest_event: u64, +} + +enum Delivery { + Events(EventBatch), + Retry(Payload), +} + +const MAX_PENDING_EVENTS: usize = 1 << 16; + +type DeliveryResult = Result<(), (Option>, CompactString)>; + pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) { - let (tx, mut rx) = builder.register(); + let (_, mut rx, _) = builder.register(); tokio::spawn(async move { let settings = Arc::new(settings); - let mut wakeup_time = LONG_1Y_SLUMBER; let discard_after = settings.discard_after.as_secs(); let mut pending_events = Vec::new(); + let mut failed_payload = None; let mut next_delivery = Instant::now(); - let in_flight = Arc::new(AtomicBool::new(false)); + let mut in_flight: Option> = None; + let mut in_flight_events = 0; + let mut is_failing = false; + let mut last_error = CompactString::const_new(""); + let mut overflowed_events = 0; loop { - // Wait for the next event or timeout - let event_or_timeout = tokio::time::timeout(wakeup_time, rx.recv()).await; - let now = now(); + let has_work = !pending_events.is_empty() || failed_payload.is_some(); + let wakeup_time = if has_work && in_flight.is_none() { + next_delivery.saturating_duration_since(Instant::now()) + } else { + LONG_1Y_SLUMBER + }; - match event_or_timeout { - Ok(Some(events)) => { - let mut discard_count = 0; - for event in events { - if now.saturating_sub(event.inner.timestamp) < discard_after { - pending_events.push(event) - } else { - discard_count += 1; + tokio::select! { + events = rx.recv() => match events { + Some(events) => { + let now = now(); + let mut stale = 0; + for event in events { + if now.saturating_sub(event.inner.timestamp) >= discard_after { + stale += 1; + } else if pending_events.len() < MAX_PENDING_EVENTS { + pending_events.push(event); + } else { + overflowed_events += 1; + } + } + report_discarded(stale); + } + None => break, + }, + result = wait_for_delivery(&mut in_flight, in_flight_events) => { + match result { + Ok(()) => { + if is_failing { + is_failing = false; + last_error = CompactString::const_new(""); + trc::event!( + Telemetry(TelemetryEvent::WebhookError), + Details = "Resumed webhook deliveries", + ); + } + } + Err((payload, error)) => { + failed_payload = payload.map(|payload| *payload); + if !is_failing || last_error != error { + is_failing = true; + last_error = error.clone(); + trc::event!( + Telemetry(TelemetryEvent::WebhookError), + Details = error, + ); + } } } - - if discard_count > 0 { - trc::event!( - Telemetry(TelemetryEvent::WebhookError), - Details = "Discarded stale events", - Total = discard_count - ); - } } - Ok(None) => { - break; - } - Err(_) => (), + _ = tokio::time::sleep(wakeup_time) => {} } - // Process events - let mut next_retry = None; - let now = Instant::now(); - if next_delivery <= now { - if !pending_events.is_empty() { - next_delivery = now + settings.throttle; - if !in_flight.load(Ordering::Relaxed) { - spawn_webhook_handler( - settings.clone(), - in_flight.clone(), - std::mem::take(&mut pending_events), - tx.clone(), - ); - } + if in_flight.is_some() || Instant::now() < next_delivery { + continue; + } + + let delivery = if let Some(mut payload) = failed_payload.take() { + let now = now(); + if now.saturating_sub(payload.oldest_event) < discard_after { + Some(Delivery::Retry(payload)) + } else { + let expected = payload.events.len(); + payload + .events + .retain(|event| now.saturating_sub(event.inner.timestamp) < discard_after); + report_discarded(expected - payload.events.len()); + (!payload.events.is_empty()).then(|| Delivery::Events(payload.events)) } } else if !pending_events.is_empty() { - // Retry later - let this_retry = next_delivery - now; - match next_retry { - Some(next_retry) if this_retry >= next_retry => {} - _ => { - next_retry = Some(this_retry); - } - } + let now = now(); + let expected = pending_events.len(); + pending_events + .retain(|event| now.saturating_sub(event.inner.timestamp) < discard_after); + report_discarded(expected - pending_events.len()); + (!pending_events.is_empty()) + .then(|| Delivery::Events(std::mem::take(&mut pending_events))) + } else { + None + }; + + if overflowed_events > 0 { + trc::event!( + Telemetry(TelemetryEvent::WebhookError), + Details = "Webhook queue full, discarded events", + Total = std::mem::take(&mut overflowed_events), + ); + } + + if let Some(delivery) = delivery { + in_flight_events = delivery.num_events(); + next_delivery = Instant::now() + settings.throttle; + in_flight = Some(spawn_webhook_handler(settings.clone(), delivery)); } - wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER); } }); } -#[derive(Serialize)] -struct EventWrapper { - events: JsonEventSerializer>>>, +fn report_discarded(discarded: usize) { + if discarded > 0 { + trc::event!( + Telemetry(TelemetryEvent::WebhookError), + Details = "Discarded stale events", + Total = discarded + ); + } +} + +async fn wait_for_delivery( + in_flight: &mut Option>, + num_events: usize, +) -> DeliveryResult { + match in_flight { + Some(handle) => { + let result = handle.await; + *in_flight = None; + result.unwrap_or_else(|err| { + Err(( + None, + format_compact!( + "Webhook delivery task failed, discarded {num_events} events: {err}" + ), + )) + }) + } + None => std::future::pending().await, + } +} + +impl Delivery { + fn num_events(&self) -> usize { + match self { + Delivery::Events(events) => events.len(), + Delivery::Retry(payload) => payload.events.len(), + } + } } fn spawn_webhook_handler( settings: Arc, - in_flight: Arc, - events: EventBatch, - webhook_tx: mpsc::Sender, -) { + delivery: Delivery, +) -> JoinHandle { tokio::spawn(async move { - in_flight.store(true, Ordering::Relaxed); + match delivery { + Delivery::Events(events) => match Payload::build(&settings, events) { + Ok(payload) => post_webhook_events(&settings, payload).await, + Err((err, discarded)) => { + report_discarded(discarded); + Err((None, err)) + } + }, + Delivery::Retry(payload) => post_webhook_events(&settings, payload).await, + } + }) +} + +impl Payload { + fn build(settings: &WebhookTracer, events: EventBatch) -> Result { + let oldest_event = events + .iter() + .map(|event| event.inner.timestamp) + .min() + .unwrap_or_default(); + let num_events = events.len(); let wrapper = EventWrapper { events: JsonEventSerializer::new(events).with_id().with_spans(), }; + let body = serde_json::to_vec(&wrapper).map_err(|err| { + ( + format_compact!("Failed to serialize events: {err}"), + num_events, + ) + })?; - if let Err(err) = post_webhook_events(&settings, &wrapper).await { - trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err); + let signature = if !settings.key.is_empty() { + let key = hmac::Key::new(hmac::HMAC_SHA256, settings.key.as_bytes()); + let tag = hmac::sign(&key, &body); + HeaderValue::try_from(STANDARD.encode(tag.as_ref())).ok() + } else { + None + }; - if webhook_tx.send(wrapper.events.into_inner()).await.is_err() { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Failed to send failed webhook events back to main thread", - CausedBy = trc::location!() - ); - } - } - - in_flight.store(false, Ordering::Relaxed); - }); + Ok(Payload { + body: Bytes::from(body), + signature, + events: wrapper.events.into_inner(), + oldest_event, + }) + } } -async fn post_webhook_events( - settings: &WebhookTracer, - events: &EventWrapper, -) -> Result<(), String> { - // Serialize body - let body = serde_json::to_string(events) - .map_err(|err| format!("Failed to serialize events: {}", err))?; +#[derive(serde::Serialize)] +struct EventWrapper { + events: JsonEventSerializer, +} - // Add HMAC-SHA256 signature +async fn post_webhook_events(settings: &WebhookTracer, payload: Payload) -> DeliveryResult { let mut headers = settings.headers.clone(); - if !settings.key.is_empty() { - let key = hmac::Key::new(hmac::HMAC_SHA256, settings.key.as_bytes()); - let tag = hmac::sign(&key, body.as_bytes()); - - headers.insert( - "X-Signature", - STANDARD.encode(tag.as_ref()).parse().unwrap(), - ); + if let Some(signature) = &payload.signature { + headers.insert("X-Signature", signature.clone()); } - // Send request - let response = settings + let error = match settings .client .post(&settings.url) .timeout(settings.timeout) .headers(headers) - .body(body) + .body(payload.body.clone()) .send() .await - .map_err(|err| format!("Webhook request to {} failed: {err}", settings.url))?; - - if response.status().is_success() { - Ok(()) - } else { - Err(format!( + { + Ok(response) if response.status().is_success() => return Ok(()), + Ok(response) => format_compact!( "Webhook request to {} failed with code {}: {}", settings.url, response.status().as_u16(), response.status().canonical_reason().unwrap_or("Unknown") - )) - } + ), + Err(err) => format_compact!("Webhook request to {} failed: {err}", settings.url), + }; + + Err((Some(Box::new(payload)), error)) } diff --git a/crates/coordinator/Cargo.toml b/crates/coordinator/Cargo.toml index fe5cd3d8..ec45fa8e 100644 --- a/crates/coordinator/Cargo.toml +++ b/crates/coordinator/Cargo.toml @@ -1,26 +1,27 @@ [package] name = "coordinator" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] store = { path = "../store" } registry = { path = "../registry" } +utils = { path = "../utils", optional = true } trc = { path = "../trc" } futures = { version = "0.3", optional = true } -tokio = { version = "1.53", features = ["sync", "fs", "io-util"] } +tokio = { version = "1.54", features = ["sync", "fs", "io-util"] } async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true } -zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true } -rdkafka = { version = "0.39", features = ["cmake-build"], optional = true } -redis = { version = "1.6", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls", "cluster-async", "sentinel"], optional = true } +zenoh = { version = "1.10.1", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true } +rdkafka = { version = "0.39", features = ["cmake-build", "ssl"], optional = true } +redis = { version = "1.8", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls", "cluster-async", "sentinel"], optional = true } +deadpool = { version = "0.13", default-features = false, features = ["managed"], optional = true } [features] -nats = ["async-nats", "futures"] +nats = ["async-nats", "futures", "dep:utils"] zenoh = ["dep:zenoh"] -kafka = ["rdkafka"] -redis = ["dep:redis", "futures"] -enterprise = [] -test_mode = [] +kafka = ["rdkafka", "futures"] +redis = ["dep:redis", "dep:deadpool", "futures", "store/redis"] +enterprise = ["store/enterprise", "registry/enterprise", "utils?/enterprise", "trc/enterprise"] [lints] workspace = true diff --git a/crates/coordinator/src/backend/kafka/mod.rs b/crates/coordinator/src/backend/kafka/mod.rs index bc36e793..f97f73d4 100644 --- a/crates/coordinator/src/backend/kafka/mod.rs +++ b/crates/coordinator/src/backend/kafka/mod.rs @@ -25,30 +25,38 @@ pub struct KafkaPubSub { } impl KafkaPubSub { - pub async fn open(config: KafkaCoordinator) -> Result { - if config.brokers.is_empty() { - return Err("No Kafka brokers specified".to_string()); + pub async fn open(config: KafkaCoordinator, node_id: u16) -> Result { + if config.hosts.is_empty() { + return Err("No Kafka hosts specified".to_string()); } - let brokers = config.brokers.into_inner().join(","); - let mut consumer_builder = ClientConfig::new(); + let mut client_config = ClientConfig::new(); + client_config.set("bootstrap.servers", config.hosts.into_inner().join(",")); + if config.use_tls { + client_config.set("security.protocol", "ssl"); + if config.allow_invalid_certs { + client_config.set("enable.ssl.certificate.verification", "false"); + } + } + let mut consumer_builder = client_config.clone(); consumer_builder - .set("group.id", config.group_id) - .set("bootstrap.servers", &brokers) + .set("group.id", format!("{}-{node_id}", config.group_id)) + .set("auto.offset.reset", "latest") .set("enable.partition.eof", "false") .set( "session.timeout.ms", - config.timeout_session.as_millis().to_string(), + config.session_timeout.as_millis().to_string(), ) - .set("enable.auto.commit", "true"); + .set("enable.auto.commit", "true") + .set("auto.commit.interval.ms", "100"); - let producer = ClientConfig::new() - .set("bootstrap.servers", brokers) + let producer = client_config .set( "message.timeout.ms", - config.timeout_message.as_millis().to_string(), + config.message_timeout.as_millis().to_string(), ) + .set("linger.ms", "0") .create() .map_err(|err| format!("Failed to create Kafka producer: {}", err))?; @@ -65,7 +73,7 @@ impl std::fmt::Debug for KafkaPubSub { } } -pub(super) struct CustomContext; +pub struct CustomContext; impl ClientContext for CustomContext {} @@ -76,3 +84,4 @@ impl ConsumerContext for CustomContext { fn commit_callback(&self, _: KafkaResult<()>, _: &TopicPartitionList) {} } + diff --git a/crates/coordinator/src/backend/kafka/pubsub.rs b/crates/coordinator/src/backend/kafka/pubsub.rs index 00b77e71..d0c98390 100644 --- a/crates/coordinator/src/backend/kafka/pubsub.rs +++ b/crates/coordinator/src/backend/kafka/pubsub.rs @@ -5,25 +5,55 @@ */ use super::{CustomContext, KafkaPubSub, LoggingConsumer}; -use crate::{Msg, PubSubStream}; +use crate::{Msg, Subscription}; +use futures::StreamExt; use rdkafka::{ - Message, - consumer::{CommitMode, Consumer, StreamConsumer}, - producer::FutureRecord, + consumer::{Consumer, MessageStream}, + producer::{FutureProducer, FutureRecord}, }; use std::time::Duration; use trc::{ClusterEvent, Error, EventType}; -pub struct KafkaPubSubStream { - subs: LoggingConsumer, +pub struct KafkaSubscription { + consumer: LoggingConsumer, +} + +pub type KafkaStream<'x> = MessageStream<'x, CustomContext>; + +pub struct KafkaPublisher { + producer: FutureProducer, + topic: &'static str, } impl KafkaPubSub { - pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { + pub(crate) fn publisher(&self, topic: &'static str) -> KafkaPublisher { + KafkaPublisher { + producer: self.producer.clone(), + topic, + } + } + + pub async fn subscribe(&self, topic: &'static str) -> trc::Result { + let consumer: LoggingConsumer = self + .consumer_builder + .create_with_context(CustomContext) + .map_err(|err| { + Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) + })?; + consumer.subscribe(&[topic]).map_err(|err| { + Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) + })?; + + Ok(Subscription::Kafka(KafkaSubscription { consumer })) + } +} + +impl KafkaPublisher { + pub async fn publish(&mut self, payload: &[u8]) -> trc::Result<()> { self.producer .send( - FutureRecord::<(), [u8]>::to(topic).payload(message.as_slice()), - Duration::from_secs(0), + FutureRecord::<(), [u8]>::to(self.topic).payload(payload), + Duration::ZERO, ) .await .map(|_| ()) @@ -31,26 +61,14 @@ impl KafkaPubSub { Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err) }) } +} - pub async fn subscribe(&self, topic: &'static str) -> trc::Result { - let subs: StreamConsumer = self - .consumer_builder - .create_with_context(CustomContext) - .map_err(|err| { - Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) - })?; - subs.subscribe(&[topic]).map_err(|err| { - Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) - })?; - - Ok(PubSubStream::Kafka(KafkaPubSubStream { subs })) +impl KafkaSubscription { + pub(crate) fn stream(&self) -> KafkaStream<'_> { + self.consumer.stream() } } -impl KafkaPubSubStream { - pub async fn next(&mut self) -> Option { - let msg = self.subs.recv().await.ok()?; - let _ = self.subs.commit_message(&msg, CommitMode::Async); - Msg::Kafka(msg.payload().unwrap_or_default().to_vec()).into() - } +pub(crate) async fn kafka_next<'x>(stream: &mut KafkaStream<'x>) -> Option> { + stream.next().await?.ok().map(Msg::Kafka) } diff --git a/crates/coordinator/src/backend/nats/mod.rs b/crates/coordinator/src/backend/nats/mod.rs index ee01fd53..dc577433 100644 --- a/crates/coordinator/src/backend/nats/mod.rs +++ b/crates/coordinator/src/backend/nats/mod.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use crate::Coordinator; use async_nats::Client; use registry::schema::structs::NatsCoordinator; +use utils::tls::rustls_client_config; pub mod pubsub; @@ -19,20 +20,24 @@ pub struct NatsPubSub { impl NatsPubSub { pub async fn open(config: NatsCoordinator) -> Result { - if config.addresses.is_empty() { - return Err("No Nats addresses specified".to_string()); + if config.hosts.is_empty() { + return Err("No Nats hosts specified".to_string()); } let mut opts = async_nats::ConnectOptions::new() .max_reconnects(config.max_reconnects.map(|v| v as usize)) - .connection_timeout(config.timeout_connection.into_inner()) - .request_timeout(config.timeout_request.into_inner().into()) + .connection_timeout(config.connect_timeout.into_inner()) + .request_timeout(config.request_timeout.into_inner().into()) .ping_interval(config.ping_interval.into_inner()) - .client_capacity(config.capacity_client as usize) - .subscription_capacity(config.capacity_subscription as usize) - .read_buffer_capacity(config.capacity_read_buffer as u16) + .client_capacity(config.client_capacity as usize) + .subscription_capacity(config.subscription_capacity as usize) + .read_buffer_capacity(config.read_buffer_capacity as u16) .require_tls(config.use_tls); + if config.allow_invalid_certs { + opts = opts.tls_client_config(rustls_client_config(true)?); + } + if config.no_echo { opts = opts.no_echo(); } @@ -47,7 +52,7 @@ impl NatsPubSub { opts = opts.token(credentials); } - async_nats::connect_with_options(config.addresses.into_inner(), opts) + async_nats::connect_with_options(config.hosts.into_inner(), opts) .await .map(|client| Coordinator::Nats(Arc::new(NatsPubSub { client }))) .map_err(|err| format!("Failed to connect to Nats: {}", err)) diff --git a/crates/coordinator/src/backend/nats/pubsub.rs b/crates/coordinator/src/backend/nats/pubsub.rs index 3b1d8766..61e1a68d 100644 --- a/crates/coordinator/src/backend/nats/pubsub.rs +++ b/crates/coordinator/src/backend/nats/pubsub.rs @@ -5,35 +5,48 @@ */ use super::NatsPubSub; -use crate::{Msg, PubSubStream}; +use crate::{Msg, Subscription}; +use async_nats::{Client, Subscriber, connection::State}; use futures::StreamExt; use trc::{ClusterEvent, Error, EventType}; -pub struct NatsPubSubStream { - subs: async_nats::Subscriber, +pub struct NatsPublisher { + client: Client, + topic: &'static str, } impl NatsPubSub { - pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { - self.client - .publish(topic, message.into()) - .await - .map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)) + pub(crate) fn publisher(&self, topic: &'static str) -> NatsPublisher { + NatsPublisher { + client: self.client.clone(), + topic, + } } - pub async fn subscribe(&self, topic: &'static str) -> trc::Result { + pub async fn subscribe(&self, topic: &'static str) -> trc::Result { self.client .subscribe(topic) .await - .map(|subs| PubSubStream::Nats(NatsPubSubStream { subs })) + .map(Subscription::Nats) .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) }) } } -impl NatsPubSubStream { - pub async fn next(&mut self) -> Option { - self.subs.next().await.map(Msg::Nats) +impl NatsPublisher { + pub async fn publish(&mut self, payload: &mut Vec) -> trc::Result<()> { + if self.client.connection_state() != State::Connected { + return Err(Error::new(EventType::Cluster(ClusterEvent::PublisherError)) + .details("Not connected to NATS")); + } + self.client + .publish(self.topic, std::mem::take(payload).into()) + .await + .map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)) } } + +pub(crate) async fn nats_next<'x>(subscriber: &mut Subscriber) -> Option> { + subscriber.next().await.map(Msg::Nats) +} diff --git a/crates/coordinator/src/backend/redis/pubsub.rs b/crates/coordinator/src/backend/redis/pubsub.rs index 547f1ecd..5e1eb937 100644 --- a/crates/coordinator/src/backend/redis/pubsub.rs +++ b/crates/coordinator/src/backend/redis/pubsub.rs @@ -4,59 +4,79 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Msg, PubSubStream}; +use crate::{Msg, Subscription}; +use deadpool::managed::Manager; use futures::StreamExt; -use redis::{AsyncCommands, PushInfo, cluster::ClusterConfig, cluster_async::ClusterConnection}; -use std::fmt::Display; +use redis::{ + AsyncCommands, PushInfo, aio::MultiplexedConnection, cluster::ClusterConfig, + cluster_async::ClusterConnection, +}; +use std::{fmt::Display, sync::Arc}; use store::backend::redis::{RedisPool, RedisStore}; use tokio::sync::mpsc::UnboundedReceiver; -pub struct RedisPubSubStream { - stream: redis::aio::PubSubStream, +pub enum RedisSubscription { + Single(redis::aio::PubSubStream), + Cluster { + _conn: ClusterConnection, + rx: UnboundedReceiver, + }, } -pub struct RedisClusterPubSubStream { - _conn: ClusterConnection, - rx: UnboundedReceiver, -} - -pub(crate) async fn redis_publish( - redis: &RedisStore, +pub struct RedisPublisher { + store: Arc, topic: &'static str, - message: Vec, -) -> trc::Result<()> { - match &redis.pool { - RedisPool::Single(pool) => pool - .get() - .await - .map_err(into_error)? - .as_mut() - .publish(topic, message) - .await - .map_err(into_error), - RedisPool::Cluster(pool) => pool - .get() - .await - .map_err(into_error)? - .as_mut() - .publish(topic, message) - .await - .map_err(into_error), - RedisPool::Sentinel(pool) => pool - .get() - .await - .map_err(into_error)? - .as_mut() - .publish(topic, message) - .await - .map_err(into_error), + connection: Option, +} + +enum PublishConnection { + Multiplexed(MultiplexedConnection), + Cluster(ClusterConnection), +} + +impl RedisPublisher { + pub(crate) fn new(store: Arc, topic: &'static str) -> Self { + RedisPublisher { + store, + topic, + connection: None, + } + } + + pub async fn publish(&mut self, payload: &[u8]) -> trc::Result<()> { + let connection = match &mut self.connection { + Some(connection) => connection, + None => self + .connection + .insert(PublishConnection::open(&self.store).await?), + }; + let result = match connection { + PublishConnection::Multiplexed(connection) => { + connection.publish(self.topic, payload).await + } + PublishConnection::Cluster(connection) => connection.publish(self.topic, payload).await, + }; + if result.is_err() { + self.connection = None; + } + result.map_err(into_error) + } +} + +impl PublishConnection { + async fn open(store: &RedisStore) -> trc::Result { + match &store.pool { + RedisPool::Single(pool) => pool.manager().create().await.map(Self::Multiplexed), + RedisPool::Cluster(pool) => pool.manager().create().await.map(Self::Cluster), + RedisPool::Sentinel(pool) => pool.manager().create().await.map(Self::Multiplexed), + } } } pub(crate) async fn redis_subscribe( redis: &RedisStore, topic: &'static str, -) -> trc::Result { +) -> trc::Result { match &redis.pool { RedisPool::Single(pool) => { let mut pubsub = pool @@ -67,9 +87,9 @@ pub(crate) async fn redis_subscribe( .map_err(into_error)?; pubsub.subscribe(topic).await.map_err(into_error)?; - Ok(PubSubStream::Redis(RedisPubSubStream { - stream: pubsub.into_on_message(), - })) + Ok(Subscription::Redis(RedisSubscription::Single( + pubsub.into_on_message(), + ))) } RedisPool::Cluster(pool) => { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); @@ -83,7 +103,7 @@ pub(crate) async fn redis_subscribe( _conn.subscribe(topic).await.map_err(into_error)?; - Ok(PubSubStream::RedisCluster(RedisClusterPubSubStream { + Ok(Subscription::Redis(RedisSubscription::Cluster { _conn, rx, })) @@ -101,25 +121,22 @@ pub(crate) async fn redis_subscribe( let mut pubsub = client.get_async_pubsub().await.map_err(into_error)?; pubsub.subscribe(topic).await.map_err(into_error)?; - Ok(PubSubStream::Redis(RedisPubSubStream { - stream: pubsub.into_on_message(), - })) + Ok(Subscription::Redis(RedisSubscription::Single( + pubsub.into_on_message(), + ))) } } } -impl RedisPubSubStream { - pub async fn next(&mut self) -> Option { - self.stream.next().await.map(Msg::Redis) - } -} - -impl RedisClusterPubSubStream { - pub async fn next(&mut self) -> Option { - loop { - if let Some(msg) = redis::Msg::from_push_info(self.rx.recv().await?) { - return Some(Msg::Redis(msg)); - } +impl RedisSubscription { + pub async fn next<'x>(&mut self) -> Option> { + match self { + RedisSubscription::Single(stream) => stream.next().await.map(Msg::Redis), + RedisSubscription::Cluster { rx, .. } => loop { + if let Some(msg) = redis::Msg::from_push_info(rx.recv().await?) { + return Some(Msg::Redis(msg)); + } + }, } } } diff --git a/crates/coordinator/src/backend/zenoh/mod.rs b/crates/coordinator/src/backend/zenoh/mod.rs index 17b5a960..31cf85b0 100644 --- a/crates/coordinator/src/backend/zenoh/mod.rs +++ b/crates/coordinator/src/backend/zenoh/mod.rs @@ -5,6 +5,7 @@ */ use registry::schema::structs::ZenohCoordinator; +use std::sync::Arc; use crate::Coordinator; pub mod pubsub; @@ -22,6 +23,6 @@ impl ZenohPubSub { .await .map_err(|err| format!("Failed to create Zenoh session: {}", err)) .map(|session| ZenohPubSub { session }) - .map(|store| Coordinator::Zenoh(std::sync::Arc::new(store))) + .map(|store| Coordinator::Zenoh(Arc::new(store))) } } diff --git a/crates/coordinator/src/backend/zenoh/pubsub.rs b/crates/coordinator/src/backend/zenoh/pubsub.rs index 6650edda..403b6579 100644 --- a/crates/coordinator/src/backend/zenoh/pubsub.rs +++ b/crates/coordinator/src/backend/zenoh/pubsub.rs @@ -5,43 +5,84 @@ */ use super::ZenohPubSub; -use crate::{Msg, PubSubStream}; +use crate::{Msg, Subscription}; +use std::sync::Arc; use trc::{ClusterEvent, Error, EventType}; +use zenoh::{handlers::FifoChannelHandler, pubsub::Publisher, sample::Sample}; -pub struct ZenohPubSubStream { - subs: zenoh::pubsub::Subscriber>, +pub type ZenohSubscription = zenoh::pubsub::Subscriber>; + +pub enum ZenohMsg { + Contiguous(Sample), + Fragmented(Vec), +} + +pub struct ZenohPublisher { + pubsub: Arc, + topic: &'static str, + publisher: Option>, } impl ZenohPubSub { - pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { - self.session - .declare_publisher(topic) - .await - .map_err(|err| { - Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err) - })? - .put(message) - .await - .map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)) - } - - pub async fn subscribe(&self, topic: &'static str) -> trc::Result { + pub async fn subscribe(&self, topic: &'static str) -> trc::Result { self.session .declare_subscriber(topic) .await - .map(|subs| PubSubStream::Zenoh(ZenohPubSubStream { subs })) + .map(Subscription::Zenoh) .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) }) } } -impl ZenohPubSubStream { - pub async fn next(&mut self) -> Option { - self.subs - .recv_async() - .await - .map(|sample| Msg::Zenoh(sample.payload().to_bytes().into_owned())) - .ok() +impl ZenohPublisher { + pub(crate) fn new(pubsub: Arc, topic: &'static str) -> Self { + ZenohPublisher { + pubsub, + topic, + publisher: None, + } + } + + pub async fn publish(&mut self, payload: &mut Vec) -> trc::Result<()> { + let publisher = match &mut self.publisher { + Some(publisher) => publisher, + None => self.publisher.insert( + self.pubsub + .session + .declare_publisher(self.topic) + .await + .map_err(into_error)?, + ), + }; + let result = publisher.put(std::mem::take(payload)).await; + if result.is_err() { + self.publisher = None; + } + result.map_err(into_error) } } + +pub(crate) async fn zenoh_next<'x>(subscription: &ZenohSubscription) -> Option> { + let sample = subscription.recv_async().await.ok()?; + let mut slices = sample.payload().slices(); + let msg = if slices.nth(1).is_none() { + ZenohMsg::Contiguous(sample) + } else { + ZenohMsg::Fragmented(sample.payload().to_bytes().into_owned()) + }; + Some(Msg::Zenoh(msg)) +} + +impl ZenohMsg { + pub fn payload(&self) -> &[u8] { + match self { + ZenohMsg::Contiguous(sample) => sample.payload().slices().next().unwrap_or_default(), + ZenohMsg::Fragmented(payload) => payload, + } + } +} + +fn into_error(err: impl std::fmt::Display) -> trc::Error { + Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err) +} diff --git a/crates/coordinator/src/bootstrap.rs b/crates/coordinator/src/bootstrap.rs index 8cd35d67..8fdd08df 100644 --- a/crates/coordinator/src/bootstrap.rs +++ b/crates/coordinator/src/bootstrap.rs @@ -14,19 +14,13 @@ impl Coordinator { let result = match bp.setting_infallible::().await { structs::Coordinator::Disabled => Ok(Coordinator::None), #[cfg(feature = "redis")] - structs::Coordinator::Default => { - if let InMemoryStore::Redis(redis) = &_in_memory { - Ok(Coordinator::Redis(redis.clone())) - } else { - Err( - "Default coordinator requires Redis or Redis Cluster in-memory backend" - .to_string(), - ) - } - } + structs::Coordinator::Default => match _in_memory { + InMemoryStore::Redis(redis) => Ok(Coordinator::Redis(redis.clone())), + _ => Err(default_coordinator_error()), + }, #[cfg(feature = "kafka")] structs::Coordinator::Kafka(kafka_coordinator) => { - crate::backend::kafka::KafkaPubSub::open(kafka_coordinator).await + crate::backend::kafka::KafkaPubSub::open(kafka_coordinator, bp.node_id()).await } #[cfg(feature = "nats")] structs::Coordinator::Nats(nats_coordinator) => { @@ -71,6 +65,11 @@ impl Coordinator { } } +#[cfg(feature = "redis")] +fn default_coordinator_error() -> String { + "Default coordinator requires a Redis, Redis Cluster, Redis Sentinel or sharded Redis in-memory store".to_string() +} + #[cfg(feature = "redis")] fn unwrap_redis(store: InMemoryStore) -> Coordinator { if let InMemoryStore::Redis(redis) = store { diff --git a/crates/coordinator/src/dispatch.rs b/crates/coordinator/src/dispatch.rs index bd84b67c..51010f15 100644 --- a/crates/coordinator/src/dispatch.rs +++ b/crates/coordinator/src/dispatch.rs @@ -4,27 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Coordinator, Msg, PubSubStream}; +use crate::{Coordinator, Msg, PubSubStream, Publisher, Subscription}; +use std::marker::PhantomData; #[allow(unused_variables)] impl Coordinator { - pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { + pub fn publisher(&self, topic: &'static str) -> Option { match self { #[cfg(feature = "redis")] - Coordinator::Redis(store) => { - crate::backend::redis::pubsub::redis_publish(store, topic, message).await - } + Coordinator::Redis(store) => Some(Publisher::Redis( + crate::backend::redis::pubsub::RedisPublisher::new(store.clone(), topic), + )), #[cfg(feature = "nats")] - Coordinator::Nats(store) => store.publish(topic, message).await, + Coordinator::Nats(pubsub) => Some(Publisher::Nats(pubsub.publisher(topic))), #[cfg(feature = "zenoh")] - Coordinator::Zenoh(store) => store.publish(topic, message).await, + Coordinator::Zenoh(pubsub) => Some(Publisher::Zenoh( + crate::backend::zenoh::pubsub::ZenohPublisher::new(pubsub.clone(), topic), + )), #[cfg(feature = "kafka")] - Coordinator::Kafka(store) => store.publish(topic, message).await, - Coordinator::None => Err(trc::StoreEvent::NotSupported.into_err()), + Coordinator::Kafka(pubsub) => Some(Publisher::Kafka(pubsub.publisher(topic))), + Coordinator::None => None, } } - pub async fn subscribe(&self, topic: &'static str) -> trc::Result { + pub async fn subscribe(&self, topic: &'static str) -> trc::Result { match self { #[cfg(feature = "redis")] Coordinator::Redis(store) => { @@ -45,25 +48,61 @@ impl Coordinator { } } -impl PubSubStream { - pub async fn next(&mut self) -> Option { +#[allow(unused_variables)] +impl Publisher { + #[allow(clippy::ptr_arg)] + pub async fn publish(&mut self, payload: &mut Vec) -> trc::Result<()> { match self { #[cfg(feature = "redis")] - PubSubStream::Redis(stream) => stream.next().await, - #[cfg(feature = "redis")] - PubSubStream::RedisCluster(stream) => stream.next().await, + Publisher::Redis(publisher) => publisher.publish(payload).await, #[cfg(feature = "nats")] - PubSubStream::Nats(stream) => stream.next().await, + Publisher::Nats(publisher) => publisher.publish(payload).await, #[cfg(feature = "zenoh")] - PubSubStream::Zenoh(stream) => stream.next().await, + Publisher::Zenoh(publisher) => publisher.publish(payload).await, #[cfg(feature = "kafka")] - PubSubStream::Kafka(stream) => stream.next().await, - PubSubStream::Unimplemented => None, + Publisher::Kafka(publisher) => publisher.publish(payload).await, + Publisher::Unimplemented => Err(trc::StoreEvent::NotSupported.into_err()), } } } -impl Msg { +impl Subscription { + pub fn stream(&mut self) -> PubSubStream<'_> { + match self { + #[cfg(feature = "redis")] + Subscription::Redis(subscription) => PubSubStream::Redis(subscription), + #[cfg(feature = "nats")] + Subscription::Nats(subscriber) => PubSubStream::Nats(subscriber), + #[cfg(feature = "zenoh")] + Subscription::Zenoh(subscriber) => PubSubStream::Zenoh(subscriber), + #[cfg(feature = "kafka")] + Subscription::Kafka(subscription) => PubSubStream::Kafka(subscription.stream()), + Subscription::Unimplemented => PubSubStream::Unimplemented(PhantomData), + } + } +} + +impl<'x> PubSubStream<'x> { + pub async fn next(&mut self) -> Option> { + match self { + #[cfg(feature = "redis")] + PubSubStream::Redis(subscription) => subscription.next().await, + #[cfg(feature = "nats")] + PubSubStream::Nats(subscriber) => { + crate::backend::nats::pubsub::nats_next(subscriber).await + } + #[cfg(feature = "zenoh")] + PubSubStream::Zenoh(subscriber) => { + crate::backend::zenoh::pubsub::zenoh_next(subscriber).await + } + #[cfg(feature = "kafka")] + PubSubStream::Kafka(stream) => crate::backend::kafka::pubsub::kafka_next(stream).await, + PubSubStream::Unimplemented(_) => None, + } + } +} + +impl Msg<'_> { pub fn payload(&self) -> &[u8] { match self { #[cfg(feature = "redis")] @@ -71,24 +110,10 @@ impl Msg { #[cfg(feature = "nats")] Msg::Nats(msg) => msg.payload.as_ref(), #[cfg(feature = "zenoh")] - Msg::Zenoh(msg) => msg.as_slice(), + Msg::Zenoh(msg) => msg.payload(), #[cfg(feature = "kafka")] - Msg::Kafka(msg) => msg.as_slice(), - Msg::Unimplemented => &[], - } - } - - pub fn topic(&self) -> &str { - match self { - #[cfg(feature = "redis")] - Msg::Redis(msg) => msg.get_channel_name(), - #[cfg(feature = "nats")] - Msg::Nats(msg) => msg.subject.as_str(), - #[cfg(feature = "zenoh")] - Msg::Zenoh(_) => "", - #[cfg(feature = "kafka")] - Msg::Kafka(_) => "", - Msg::Unimplemented => "", + Msg::Kafka(msg) => rdkafka::Message::payload(msg).unwrap_or_default(), + Msg::Unimplemented(_) => &[], } } } diff --git a/crates/coordinator/src/lib.rs b/crates/coordinator/src/lib.rs index 3abbde90..adb9ecab 100644 --- a/crates/coordinator/src/lib.rs +++ b/crates/coordinator/src/lib.rs @@ -6,6 +6,7 @@ #![warn(clippy::large_futures)] +use std::marker::PhantomData; #[allow(unused_imports)] use std::sync::Arc; @@ -27,28 +28,50 @@ pub enum Coordinator { None, } -pub enum PubSubStream { +pub enum Publisher { #[cfg(feature = "redis")] - Redis(crate::backend::redis::pubsub::RedisPubSubStream), - #[cfg(feature = "redis")] - RedisCluster(crate::backend::redis::pubsub::RedisClusterPubSubStream), + Redis(backend::redis::pubsub::RedisPublisher), #[cfg(feature = "nats")] - Nats(crate::backend::nats::pubsub::NatsPubSubStream), + Nats(backend::nats::pubsub::NatsPublisher), #[cfg(feature = "zenoh")] - Zenoh(crate::backend::zenoh::pubsub::ZenohPubSubStream), + Zenoh(backend::zenoh::pubsub::ZenohPublisher), #[cfg(feature = "kafka")] - Kafka(crate::backend::kafka::pubsub::KafkaPubSubStream), + Kafka(backend::kafka::pubsub::KafkaPublisher), Unimplemented, } -pub enum Msg { +pub enum Subscription { + #[cfg(feature = "redis")] + Redis(backend::redis::pubsub::RedisSubscription), + #[cfg(feature = "nats")] + Nats(async_nats::Subscriber), + #[cfg(feature = "zenoh")] + Zenoh(backend::zenoh::pubsub::ZenohSubscription), + #[cfg(feature = "kafka")] + Kafka(backend::kafka::pubsub::KafkaSubscription), + Unimplemented, +} + +pub enum PubSubStream<'x> { + #[cfg(feature = "redis")] + Redis(&'x mut backend::redis::pubsub::RedisSubscription), + #[cfg(feature = "nats")] + Nats(&'x mut async_nats::Subscriber), + #[cfg(feature = "zenoh")] + Zenoh(&'x backend::zenoh::pubsub::ZenohSubscription), + #[cfg(feature = "kafka")] + Kafka(backend::kafka::pubsub::KafkaStream<'x>), + Unimplemented(PhantomData<&'x ()>), +} + +pub enum Msg<'x> { #[cfg(feature = "redis")] Redis(redis::Msg), #[cfg(feature = "nats")] Nats(async_nats::Message), #[cfg(feature = "zenoh")] - Zenoh(Vec), + Zenoh(backend::zenoh::pubsub::ZenohMsg), #[cfg(feature = "kafka")] - Kafka(Vec), - Unimplemented, + Kafka(rdkafka::message::BorrowedMessage<'x>), + Unimplemented(PhantomData<&'x ()>), } diff --git a/crates/dav-proto/Cargo.toml b/crates/dav-proto/Cargo.toml index d976289d..a9bd65cb 100644 --- a/crates/dav-proto/Cargo.toml +++ b/crates/dav-proto/Cargo.toml @@ -1,29 +1,27 @@ [package] name = "dav-proto" -version = "0.16.27" +version = "1.0.0" edition = "2024" [dependencies] trc = { path = "../trc" } types = { path = "../types" } -hashify = "0.2.9" -quick-xml = "0.41" -calcard = { version = "0.3", features = ["rkyv"] } -mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } -hyper = "1.11.1" -chrono = { version = "0.4.45", features = ["serde"], optional = true } +hashify = "0.3" +quick-xml = "0.42" +calcard = { path = "../calcard", features = ["rkyv"] } +mail-parser = { version = "1.0", features = ["full_encoding"] } +hyper = "1.12.0" compact_str = "0.10.0" +memchr = "2.8.3" [dev-dependencies] -calcard = { version = "0.3", features = ["serde", "rkyv"] } -types = { path = "../types", features = ["test_mode"] } +calcard = { path = "../calcard", features = ["serde", "rkyv"] } +types = { path = "../types" } serde = { version = "1.0.229", features = ["derive"] } -serde_json = "1.0.151" -chrono = { version = "0.4.45", features = ["serde"] } +serde_json = "1.0.154" [features] -test_mode = ["chrono"] -enterprise = [] +enterprise = ["trc/enterprise"] [lints] workspace = true diff --git a/crates/dav-proto/resources/requests/acl-001.json b/crates/dav-proto/resources/requests/acl-001.json deleted file mode 100644 index 56d5f5e0..00000000 --- a/crates/dav-proto/resources/requests/acl-001.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "aces": [ - { - "principal": { - "Href": "http://www.example.com/users/friends" - }, - "invert": false, - "grant_deny": { - "Grant": [ - "Read" - ] - }, - "protected": false, - "inherited": null - }, - { - "principal": { - "Href": "http://www.example.com/users/ygoland-so" - }, - "invert": false, - "grant_deny": { - "Deny": [ - "Read" - ] - }, - "protected": false, - "inherited": null - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-001.xml b/crates/dav-proto/resources/requests/acl-001.xml deleted file mode 100644 index 1853ad1c..00000000 --- a/crates/dav-proto/resources/requests/acl-001.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - http://www.example.com/users/friends - - - - - - http://www.example.com/users/ygoland-so - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-002.json b/crates/dav-proto/resources/requests/acl-002.json deleted file mode 100644 index ccf9b500..00000000 --- a/crates/dav-proto/resources/requests/acl-002.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "aces": [ - { - "principal": { - "Href": "http://www.example.com/users/ejw" - }, - "invert": false, - "grant_deny": { - "Grant": [ - "Write" - ] - }, - "protected": false, - "inherited": null - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-002.xml b/crates/dav-proto/resources/requests/acl-002.xml deleted file mode 100644 index f97bc128..00000000 --- a/crates/dav-proto/resources/requests/acl-002.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - http://www.example.com/users/ejw - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-003.json b/crates/dav-proto/resources/requests/acl-003.json deleted file mode 100644 index 9b8648c4..00000000 --- a/crates/dav-proto/resources/requests/acl-003.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "aces": [ - { - "principal": { - "Href": "http://www.example.com/users/esedlar" - }, - "invert": true, - "grant_deny": { - "Deny": [ - "Write" - ] - }, - "protected": true, - "inherited": "http://www.example.com/container/" - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-003.xml b/crates/dav-proto/resources/requests/acl-003.xml deleted file mode 100644 index d62274bb..00000000 --- a/crates/dav-proto/resources/requests/acl-003.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - http://www.example.com/users/esedlar - - - - - - - http://www.example.com/container/ - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-004.json b/crates/dav-proto/resources/requests/acl-004.json deleted file mode 100644 index b56597e4..00000000 --- a/crates/dav-proto/resources/requests/acl-004.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "aces": [ - { - "principal": { - "Href": "http://www.example.com/users/esedlar" - }, - "invert": false, - "grant_deny": { - "Grant": [ - "Read", - "Write" - ] - }, - "protected": false, - "inherited": null - }, - { - "principal": { - "Property": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "Owner" - } - }, - "value": "Null" - } - ] - }, - "invert": false, - "grant_deny": { - "Grant": [ - "ReadAcl", - "WriteAcl" - ] - }, - "protected": false, - "inherited": null - }, - { - "principal": "All", - "invert": false, - "grant_deny": { - "Grant": [ - "Read" - ] - }, - "protected": false, - "inherited": null - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/acl-004.xml b/crates/dav-proto/resources/requests/acl-004.xml deleted file mode 100644 index 9414370f..00000000 --- a/crates/dav-proto/resources/requests/acl-004.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - http://www.example.com/users/esedlar - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/lockinfo-001.json b/crates/dav-proto/resources/requests/lockinfo-001.json deleted file mode 100644 index 34cb0cdd..00000000 --- a/crates/dav-proto/resources/requests/lockinfo-001.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lock_scope": "Shared", - "lock_type": "Write", - "owner": [ - { - "type": "ElementStart", - "data": { - "name": "href", - "attrs": "xmlns=\"DAV:\"" - } - }, - { - "type": "Text", - "data": "http://example.org/~ejw/contact.html" - }, - { - "type": "ElementEnd" - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/lockinfo-001.xml b/crates/dav-proto/resources/requests/lockinfo-001.xml deleted file mode 100644 index 5a9b42e3..00000000 --- a/crates/dav-proto/resources/requests/lockinfo-001.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - http://example.org/~ejw/contact.html - - diff --git a/crates/dav-proto/resources/requests/lockinfo-002.json b/crates/dav-proto/resources/requests/lockinfo-002.json deleted file mode 100644 index e6138681..00000000 --- a/crates/dav-proto/resources/requests/lockinfo-002.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lock_scope": "Exclusive", - "lock_type": "Write", - "owner": [ - { - "type": "ElementStart", - "data": { - "name": "href", - "attrs": "xmlns=\"DAV:\"" - } - }, - { - "type": "Text", - "data": "http://example.org/~ejw/contact.html" - }, - { - "type": "ElementEnd" - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/lockinfo-002.xml b/crates/dav-proto/resources/requests/lockinfo-002.xml deleted file mode 100644 index 05fc33af..00000000 --- a/crates/dav-proto/resources/requests/lockinfo-002.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - http://example.org/~ejw/contact.html - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-001.json b/crates/dav-proto/resources/requests/mkcol-001.json deleted file mode 100644 index 9265cdfd..00000000 --- a/crates/dav-proto/resources/requests/mkcol-001.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "is_mkcalendar": false, - "props": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - "value": { - "ResourceTypes": [ - "Collection", - "AddressBook" - ] - } - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "value": { - "String": "Lisa's Contacts" - } - }, - { - "property": { - "type": "CardDav", - "data": { - "type": "AddressbookDescription" - } - }, - "value": { - "String": "My primary address book." - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-001.xml b/crates/dav-proto/resources/requests/mkcol-001.xml deleted file mode 100644 index c7dff1bb..00000000 --- a/crates/dav-proto/resources/requests/mkcol-001.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - Lisa's Contacts - My primary address book. - - - diff --git a/crates/dav-proto/resources/requests/mkcol-002.json b/crates/dav-proto/resources/requests/mkcol-002.json deleted file mode 100644 index e98a7dc6..00000000 --- a/crates/dav-proto/resources/requests/mkcol-002.json +++ /dev/null @@ -1,369 +0,0 @@ -{ - "is_mkcalendar": true, - "props": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "value": { - "String": "Lisa's Events" - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "CalendarDescription" - } - }, - "value": { - "String": "Calendar restricted to events." - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "SupportedCalendarComponentSet" - } - }, - "value": { - "Components": [ - "VEvent" - ] - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "CalendarTimezone" - } - }, - "value": { - "ICalendar": { - "components": [ - { - "component_type": "VCalendar", - "entries": [ - { - "name": { - "type": "Prodid" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "-//Example Corp.//CalDAV Client//EN" - } - ] - }, - { - "name": { - "type": "Version" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "2.0" - } - ] - } - ], - "component_ids": [ - 1 - ] - }, - { - "component_type": "VTimezone", - "entries": [ - { - "name": { - "type": "Tzid" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "US-Eastern" - } - ] - }, - { - "name": { - "type": "LastModified" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1987, - "month": 1, - "day": 1, - "hour": 0, - "minute": 0, - "second": 0, - "tz_hour": 0, - "tz_minute": 0, - "tz_minus": false - } - } - ] - } - ], - "component_ids": [ - 2, - 3 - ] - }, - { - "component_type": "Standard", - "entries": [ - { - "name": { - "type": "Dtstart" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1967, - "month": 10, - "day": 29, - "hour": 2, - "minute": 0, - "second": 0, - "tz_hour": null, - "tz_minute": null, - "tz_minus": false - } - } - ] - }, - { - "name": { - "type": "Rrule" - }, - "params": [], - "values": [ - { - "type": "RecurrenceRule", - "data": { - "freq": "Yearly", - "until": null, - "count": null, - "interval": null, - "bysecond": [], - "byminute": [], - "byhour": [], - "byday": [ - { - "ordwk": -1, - "weekday": "Sunday" - } - ], - "bymonthday": [], - "byyearday": [], - "byweekno": [], - "bymonth": [ - 10 - ], - "bysetpos": [], - "wkst": null, - "rscale": null, - "skip": null - } - } - ] - }, - { - "name": { - "type": "Tzoffsetfrom" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 4, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzoffsetto" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 5, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzname" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "Eastern Standard Time (US & Canada)" - } - ] - } - ], - "component_ids": [] - }, - { - "component_type": "Daylight", - "entries": [ - { - "name": { - "type": "Dtstart" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1987, - "month": 4, - "day": 5, - "hour": 2, - "minute": 0, - "second": 0, - "tz_hour": null, - "tz_minute": null, - "tz_minus": false - } - } - ] - }, - { - "name": { - "type": "Rrule" - }, - "params": [], - "values": [ - { - "type": "RecurrenceRule", - "data": { - "freq": "Yearly", - "until": null, - "count": null, - "interval": null, - "bysecond": [], - "byminute": [], - "byhour": [], - "byday": [ - { - "ordwk": 1, - "weekday": "Sunday" - } - ], - "bymonthday": [], - "byyearday": [], - "byweekno": [], - "bymonth": [ - 4 - ], - "bysetpos": [], - "wkst": null, - "rscale": null, - "skip": null - } - } - ] - }, - { - "name": { - "type": "Tzoffsetfrom" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 5, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzoffsetto" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 4, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzname" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "Eastern Daylight Time (US & Canada)" - } - ] - } - ], - "component_ids": [] - } - ] - } - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-002.xml b/crates/dav-proto/resources/requests/mkcol-002.xml deleted file mode 100644 index d31a33db..00000000 --- a/crates/dav-proto/resources/requests/mkcol-002.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - Lisa's Events - Calendar restricted to events. - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-003.json b/crates/dav-proto/resources/requests/mkcol-003.json deleted file mode 100644 index c56383b0..00000000 --- a/crates/dav-proto/resources/requests/mkcol-003.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "is_mkcalendar": false, - "props": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - "value": { - "ResourceTypes": [ - "Collection" - ] - } - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "value": { - "String": "Special Resource" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-003.xml b/crates/dav-proto/resources/requests/mkcol-003.xml deleted file mode 100644 index 0d7e13a7..00000000 --- a/crates/dav-proto/resources/requests/mkcol-003.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - Special Resource - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-004.json b/crates/dav-proto/resources/requests/mkcol-004.json deleted file mode 100644 index 5692c8e7..00000000 --- a/crates/dav-proto/resources/requests/mkcol-004.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "is_mkcalendar": false, - "props": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - "value": { - "ResourceTypes": [ - "Collection", - "Calendar" - ] - } - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "value": { - "String": "Lisa's Events" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/mkcol-004.xml b/crates/dav-proto/resources/requests/mkcol-004.xml deleted file mode 100644 index a7d09fd1..00000000 --- a/crates/dav-proto/resources/requests/mkcol-004.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - Lisa's Events - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propertyupdate-001.json b/crates/dav-proto/resources/requests/propertyupdate-001.json deleted file mode 100644 index d16de765..00000000 --- a/crates/dav-proto/resources/requests/propertyupdate-001.json +++ /dev/null @@ -1,398 +0,0 @@ -{ - "set": [ - { - "property": { - "type": "CardDav", - "data": { - "type": "AddressbookDescription" - } - }, - "value": { - "String": "Adresses de Oliver Daboo" - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "CalendarDescription" - } - }, - "value": { - "String": "Calendrier de Mathilde Desruisseaux" - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "SupportedCalendarComponentSet" - } - }, - "value": { - "Components": [ - "VEvent", - "VTodo" - ] - } - }, - { - "property": { - "type": "CalDav", - "data": { - "type": "CalendarTimezone" - } - }, - "value": { - "ICalendar": { - "components": [ - { - "component_type": "VCalendar", - "entries": [ - { - "name": { - "type": "Prodid" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "-//Example Corp.//CalDAV Client//EN" - } - ] - }, - { - "name": { - "type": "Version" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "2.0" - } - ] - } - ], - "component_ids": [ - 1 - ] - }, - { - "component_type": "VTimezone", - "entries": [ - { - "name": { - "type": "Tzid" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "US-Eastern" - } - ] - }, - { - "name": { - "type": "LastModified" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1987, - "month": 1, - "day": 1, - "hour": 0, - "minute": 0, - "second": 0, - "tz_hour": 0, - "tz_minute": 0, - "tz_minus": false - } - } - ] - } - ], - "component_ids": [ - 2, - 3 - ] - }, - { - "component_type": "Standard", - "entries": [ - { - "name": { - "type": "Dtstart" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1967, - "month": 10, - "day": 29, - "hour": 2, - "minute": 0, - "second": 0, - "tz_hour": null, - "tz_minute": null, - "tz_minus": false - } - } - ] - }, - { - "name": { - "type": "Rrule" - }, - "params": [], - "values": [ - { - "type": "RecurrenceRule", - "data": { - "freq": "Yearly", - "until": null, - "count": null, - "interval": null, - "bysecond": [], - "byminute": [], - "byhour": [], - "byday": [ - { - "ordwk": -1, - "weekday": "Sunday" - } - ], - "bymonthday": [], - "byyearday": [], - "byweekno": [], - "bymonth": [ - 10 - ], - "bysetpos": [], - "wkst": null, - "rscale": null, - "skip": null - } - } - ] - }, - { - "name": { - "type": "Tzoffsetfrom" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 4, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzoffsetto" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 5, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzname" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "Eastern Standard Time (US & Canada)" - } - ] - } - ], - "component_ids": [] - }, - { - "component_type": "Daylight", - "entries": [ - { - "name": { - "type": "Dtstart" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": 1987, - "month": 4, - "day": 5, - "hour": 2, - "minute": 0, - "second": 0, - "tz_hour": null, - "tz_minute": null, - "tz_minus": false - } - } - ] - }, - { - "name": { - "type": "Rrule" - }, - "params": [], - "values": [ - { - "type": "RecurrenceRule", - "data": { - "freq": "Yearly", - "until": null, - "count": null, - "interval": null, - "bysecond": [], - "byminute": [], - "byhour": [], - "byday": [ - { - "ordwk": 1, - "weekday": "Sunday" - } - ], - "bymonthday": [], - "byyearday": [], - "byweekno": [], - "bymonth": [ - 4 - ], - "bysetpos": [], - "wkst": null, - "rscale": null, - "skip": null - } - } - ] - }, - { - "name": { - "type": "Tzoffsetfrom" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 5, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzoffsetto" - }, - "params": [], - "values": [ - { - "type": "PartialDateTime", - "data": { - "year": null, - "month": null, - "day": null, - "hour": null, - "minute": null, - "second": null, - "tz_hour": 4, - "tz_minute": 0, - "tz_minus": true - } - } - ] - }, - { - "name": { - "type": "Tzname" - }, - "params": [], - "values": [ - { - "type": "Text", - "data": "Eastern Daylight Time (US & Canada)" - } - ] - } - ], - "component_ids": [] - } - ] - } - } - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - "value": { - "ResourceTypes": [ - "Collection", - "AddressBook" - ] - } - } - ], - "remove": [ - { - "type": "CalDav", - "data": { - "type": "CalendarTimezone" - } - }, - { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - } - ], - "set_first": true -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propertyupdate-001.xml b/crates/dav-proto/resources/requests/propertyupdate-001.xml deleted file mode 100644 index 3b92ee96..00000000 --- a/crates/dav-proto/resources/requests/propertyupdate-001.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - Adresses de Oliver Daboo - - Calendrier de Mathilde Desruisseaux - - - - - BEGIN:VCALENDAR -PRODID:-//Example Corp.//CalDAV Client//EN -VERSION:2.0 -BEGIN:VTIMEZONE -TZID:US-Eastern -LAST-MODIFIED:19870101T000000Z -BEGIN:STANDARD -DTSTART:19671029T020000 -RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 -TZOFFSETFROM:-0400 -TZOFFSETTO:-0500 -TZNAME:Eastern Standard Time (US & Canada) -END:STANDARD -BEGIN:DAYLIGHT -DTSTART:19870405T020000 -RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 -TZOFFSETFROM:-0500 -TZOFFSETTO:-0400 -TZNAME:Eastern Daylight Time (US & Canada) -END:DAYLIGHT -END:VTIMEZONE -END:VCALENDAR - - - - - - - - - - - - - - diff --git a/crates/dav-proto/resources/requests/propertyupdate-002.json b/crates/dav-proto/resources/requests/propertyupdate-002.json deleted file mode 100644 index f9e28327..00000000 --- a/crates/dav-proto/resources/requests/propertyupdate-002.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "set": [ - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop0", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value0" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop1", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value1" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop2", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value2" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop3", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value3" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop4", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value4" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop5", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value5" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop6", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value6" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop7", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value7" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop8", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value8" - } - ] - } - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "prop9", - "attrs": "xmlns=\"http://example.com/neon/litmus/\"" - } - }, - "value": { - "DeadProperty": [ - { - "type": "Text", - "data": "value9" - } - ] - } - } - ], - "remove": [], - "set_first": true -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propertyupdate-002.xml b/crates/dav-proto/resources/requests/propertyupdate-002.xml deleted file mode 100644 index 1c386888..00000000 --- a/crates/dav-proto/resources/requests/propertyupdate-002.xml +++ /dev/null @@ -1,11 +0,0 @@ -value0 -value1 -value2 -value3 -value4 -value5 -value6 -value7 -value8 -value9 - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-001.json b/crates/dav-proto/resources/requests/propfind-001.json deleted file mode 100644 index 769ce837..00000000 --- a/crates/dav-proto/resources/requests/propfind-001.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "DeadProperty", - "data": { - "name": "bigbox", - "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "author", - "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "DingALing", - "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "Random", - "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-001.xml b/crates/dav-proto/resources/requests/propfind-001.xml deleted file mode 100644 index 4f719419..00000000 --- a/crates/dav-proto/resources/requests/propfind-001.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/crates/dav-proto/resources/requests/propfind-002.json b/crates/dav-proto/resources/requests/propfind-002.json deleted file mode 100644 index 7d15ff6b..00000000 --- a/crates/dav-proto/resources/requests/propfind-002.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "PropName" -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-002.xml b/crates/dav-proto/resources/requests/propfind-002.xml deleted file mode 100644 index 8d045183..00000000 --- a/crates/dav-proto/resources/requests/propfind-002.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/crates/dav-proto/resources/requests/propfind-003.json b/crates/dav-proto/resources/requests/propfind-003.json deleted file mode 100644 index 0d980075..00000000 --- a/crates/dav-proto/resources/requests/propfind-003.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "type": "AllProp", - "data": [] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-003.xml b/crates/dav-proto/resources/requests/propfind-003.xml deleted file mode 100644 index b509470e..00000000 --- a/crates/dav-proto/resources/requests/propfind-003.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/crates/dav-proto/resources/requests/propfind-004.json b/crates/dav-proto/resources/requests/propfind-004.json deleted file mode 100644 index 70bf3159..00000000 --- a/crates/dav-proto/resources/requests/propfind-004.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "AllProp", - "data": [ - { - "type": "WebDav", - "data": { - "type": "SupportedReportSet" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-004.xml b/crates/dav-proto/resources/requests/propfind-004.xml deleted file mode 100644 index ed6f0ded..00000000 --- a/crates/dav-proto/resources/requests/propfind-004.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/crates/dav-proto/resources/requests/propfind-005.json b/crates/dav-proto/resources/requests/propfind-005.json deleted file mode 100644 index ac4ba7a0..00000000 --- a/crates/dav-proto/resources/requests/propfind-005.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "CurrentUserPrincipal" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-005.xml b/crates/dav-proto/resources/requests/propfind-005.xml deleted file mode 100644 index b9b9185b..00000000 --- a/crates/dav-proto/resources/requests/propfind-005.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/crates/dav-proto/resources/requests/propfind-006.json b/crates/dav-proto/resources/requests/propfind-006.json deleted file mode 100644 index c1b1d1b4..00000000 --- a/crates/dav-proto/resources/requests/propfind-006.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "Principal", - "data": { - "type": "CalendarHomeSet" - } - }, - { - "type": "Principal", - "data": { - "type": "GroupMembership" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-006.xml b/crates/dav-proto/resources/requests/propfind-006.xml deleted file mode 100644 index b5f399f0..00000000 --- a/crates/dav-proto/resources/requests/propfind-006.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/crates/dav-proto/resources/requests/propfind-007.json b/crates/dav-proto/resources/requests/propfind-007.json deleted file mode 100644 index 3963b0e4..00000000 --- a/crates/dav-proto/resources/requests/propfind-007.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "CurrentUserPrivilegeSet" - } - }, - { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "calendar-color", - "attrs": "xmlns=\"http://apple.com/ns/ical/\"" - } - }, - { - "type": "CalDav", - "data": { - "type": "SupportedCalendarComponentSet" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-007.xml b/crates/dav-proto/resources/requests/propfind-007.xml deleted file mode 100644 index 0a39e9a1..00000000 --- a/crates/dav-proto/resources/requests/propfind-007.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/crates/dav-proto/resources/requests/propfind-008.json b/crates/dav-proto/resources/requests/propfind-008.json deleted file mode 100644 index 425a66af..00000000 --- a/crates/dav-proto/resources/requests/propfind-008.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - { - "type": "WebDav", - "data": { - "type": "GetCTag" - } - }, - { - "type": "CalDav", - "data": { - "type": "SupportedCalendarComponentSet" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-008.xml b/crates/dav-proto/resources/requests/propfind-008.xml deleted file mode 100644 index 50545fed..00000000 --- a/crates/dav-proto/resources/requests/propfind-008.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-009.json b/crates/dav-proto/resources/requests/propfind-009.json deleted file mode 100644 index e69de29b..00000000 diff --git a/crates/dav-proto/resources/requests/propfind-009.xml b/crates/dav-proto/resources/requests/propfind-009.xml deleted file mode 100644 index cd262d14..00000000 --- a/crates/dav-proto/resources/requests/propfind-009.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-010.json b/crates/dav-proto/resources/requests/propfind-010.json deleted file mode 100644 index 42e94363..00000000 --- a/crates/dav-proto/resources/requests/propfind-010.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "ResourceType" - } - }, - { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - { - "type": "WebDav", - "data": { - "type": "SyncToken" - } - }, - { - "type": "WebDav", - "data": { - "type": "GetCTag" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "me-card", - "attrs": "xmlns=\"http://calendarserver.org/ns/\" hello=\"world & test\"" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/propfind-010.xml b/crates/dav-proto/resources/requests/propfind-010.xml deleted file mode 100644 index b9fb7388..00000000 --- a/crates/dav-proto/resources/requests/propfind-010.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-001.json b/crates/dav-proto/resources/requests/report-001.json deleted file mode 100644 index 891205de..00000000 --- a/crates/dav-proto/resources/requests/report-001.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [ - { - "component": "VCalendar", - "name": { - "type": "Version" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Summary" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Uid" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Dtstart" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Dtend" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Duration" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Rrule" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Rdate" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Exrule" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "Exdate" - }, - "no_value": false - }, - { - "component": "VEvent", - "name": { - "type": "RecurrenceId" - }, - "no_value": false - }, - { - "component": "VTimezone", - "name": null, - "no_value": false - } - ], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VEvent" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1136332800, - "end": 1136419200 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-001.xml b/crates/dav-proto/resources/requests/report-001.xml deleted file mode 100644 index 29eea631..00000000 --- a/crates/dav-proto/resources/requests/report-001.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-002.json b/crates/dav-proto/resources/requests/report-002.json deleted file mode 100644 index bc9b0a46..00000000 --- a/crates/dav-proto/resources/requests/report-002.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": { - "start": 1136246400, - "end": 1136419200 - }, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VEvent" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1136246400, - "end": 1136419200 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-002.xml b/crates/dav-proto/resources/requests/report-002.xml deleted file mode 100644 index 796017d2..00000000 --- a/crates/dav-proto/resources/requests/report-002.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - diff --git a/crates/dav-proto/resources/requests/report-003.json b/crates/dav-proto/resources/requests/report-003.json deleted file mode 100644 index 21dc8a00..00000000 --- a/crates/dav-proto/resources/requests/report-003.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": { - "start": 1136246400, - "end": 1136419200 - }, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VEvent" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1136246400, - "end": 1136419200 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-003.xml b/crates/dav-proto/resources/requests/report-003.xml deleted file mode 100644 index 1600fb46..00000000 --- a/crates/dav-proto/resources/requests/report-003.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-004.json b/crates/dav-proto/resources/requests/report-004.json deleted file mode 100644 index bd7ed8d3..00000000 --- a/crates/dav-proto/resources/requests/report-004.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": { - "start": 1136160000, - "end": 1136246400 - } - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VFreebusy" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1136160000, - "end": 1136246400 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-004.xml b/crates/dav-proto/resources/requests/report-004.xml deleted file mode 100644 index cdd92b10..00000000 --- a/crates/dav-proto/resources/requests/report-004.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-005.json b/crates/dav-proto/resources/requests/report-005.json deleted file mode 100644 index c4552aca..00000000 --- a/crates/dav-proto/resources/requests/report-005.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VTodo", - "VAlarm" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1136541600, - "end": 1136628000 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-005.xml b/crates/dav-proto/resources/requests/report-005.xml deleted file mode 100644 index d748d2a9..00000000 --- a/crates/dav-proto/resources/requests/report-005.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-006.json b/crates/dav-proto/resources/requests/report-006.json deleted file mode 100644 index a3da5973..00000000 --- a/crates/dav-proto/resources/requests/report-006.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Property", - "comp": [ - "VCalendar", - "VEvent" - ], - "prop": { - "type": "Uid" - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "DC6C50A017428C5216A2F1CD@example.com", - "collation": "Octet", - "negate": false - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-006.xml b/crates/dav-proto/resources/requests/report-006.xml deleted file mode 100644 index 7818a482..00000000 --- a/crates/dav-proto/resources/requests/report-006.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - DC6C50A017428C5216A2F1CD@example.com - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-007.json b/crates/dav-proto/resources/requests/report-007.json deleted file mode 100644 index 68b17c2a..00000000 --- a/crates/dav-proto/resources/requests/report-007.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Property", - "comp": [ - "VCalendar", - "VEvent" - ], - "prop": { - "type": "Attendee" - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "mailto:lisa@example.com", - "collation": "AsciiCasemap", - "negate": false - } - } - }, - { - "type": "Parameter", - "comp": [ - "VCalendar", - "VEvent" - ], - "prop": { - "type": "Attendee" - }, - "param": "Partstat", - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "NEEDS-ACTION", - "collation": "AsciiCasemap", - "negate": false - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-007.xml b/crates/dav-proto/resources/requests/report-007.xml deleted file mode 100644 index 88983b5f..00000000 --- a/crates/dav-proto/resources/requests/report-007.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - mailto:lisa@example.com - - NEEDS-ACTION - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-008.json b/crates/dav-proto/resources/requests/report-008.json deleted file mode 100644 index 1c6f4900..00000000 --- a/crates/dav-proto/resources/requests/report-008.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VEvent" - ], - "op": { - "type": "Exists" - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-008.xml b/crates/dav-proto/resources/requests/report-008.xml deleted file mode 100644 index 2ea17131..00000000 --- a/crates/dav-proto/resources/requests/report-008.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-009.json b/crates/dav-proto/resources/requests/report-009.json deleted file mode 100644 index a6ee4296..00000000 --- a/crates/dav-proto/resources/requests/report-009.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Property", - "comp": [ - "VCalendar", - "VTodo" - ], - "prop": { - "type": "Completed" - }, - "op": { - "type": "Undefined" - } - }, - { - "type": "Property", - "comp": [ - "VCalendar", - "VTodo" - ], - "prop": { - "type": "Status" - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "CANCELLED", - "collation": "AsciiCasemap", - "negate": true - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-009.xml b/crates/dav-proto/resources/requests/report-009.xml deleted file mode 100644 index 76c2e2ad..00000000 --- a/crates/dav-proto/resources/requests/report-009.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - CANCELLED - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-010.json b/crates/dav-proto/resources/requests/report-010.json deleted file mode 100644 index d7cabdfb..00000000 --- a/crates/dav-proto/resources/requests/report-010.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [ - { - "type": "Property", - "comp": [ - "VCalendar", - "VEvent" - ], - "prop": { - "type": "Other", - "data": "X-ABC-GUID" - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "ABC", - "collation": "AsciiCasemap", - "negate": false - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-010.xml b/crates/dav-proto/resources/requests/report-010.xml deleted file mode 100644 index bbeb447f..00000000 --- a/crates/dav-proto/resources/requests/report-010.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - ABC - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-011.json b/crates/dav-proto/resources/requests/report-011.json deleted file mode 100644 index 6f5c8d09..00000000 --- a/crates/dav-proto/resources/requests/report-011.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - } - ] - }, - "filters": [ - { - "type": "Component", - "comp": [ - "VCalendar", - "VEvent" - ], - "op": { - "type": "TimeRange", - "data": { - "start": 1094083200, - "end": 1094169600 - } - } - } - ], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-011.xml b/crates/dav-proto/resources/requests/report-011.xml deleted file mode 100644 index 8afce337..00000000 --- a/crates/dav-proto/resources/requests/report-011.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-012.json b/crates/dav-proto/resources/requests/report-012.json deleted file mode 100644 index 72f4b644..00000000 --- a/crates/dav-proto/resources/requests/report-012.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "CalendarQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "filters": [], - "timezone": { - "type": "None" - } -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-012.xml b/crates/dav-proto/resources/requests/report-012.xml deleted file mode 100644 index 65b3f959..00000000 --- a/crates/dav-proto/resources/requests/report-012.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-013.json b/crates/dav-proto/resources/requests/report-013.json deleted file mode 100644 index 6ecabfd7..00000000 --- a/crates/dav-proto/resources/requests/report-013.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "CalendarMultiGet", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CalDav", - "data": { - "type": "CalendarData", - "data": { - "properties": [], - "expand": null, - "limit_recurrence": null, - "limit_freebusy": null - } - } - } - ] - }, - "hrefs": [ - "/bernard/work/abcd1.ics", - "/bernard/work/mtg1.ics" - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-013.xml b/crates/dav-proto/resources/requests/report-013.xml deleted file mode 100644 index a87e867d..00000000 --- a/crates/dav-proto/resources/requests/report-013.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - /bernard/work/abcd1.ics - /bernard/work/mtg1.ics - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-014.json b/crates/dav-proto/resources/requests/report-014.json deleted file mode 100644 index 967e5dc8..00000000 --- a/crates/dav-proto/resources/requests/report-014.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "type": "AddressbookQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CardDav", - "data": { - "type": "AddressData", - "data": { - "properties": [ - { - "group": null, - "name": { - "type": "Version" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Uid" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Nickname" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Email" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Fn" - }, - "no_value": false - } - ] - } - } - } - ] - }, - "filters": [ - { - "type": "Property", - "comp": null, - "prop": { - "name": { - "type": "Nickname" - }, - "group": null - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Equals", - "value": "me", - "collation": "UnicodeCasemap", - "negate": false - } - } - } - ], - "limit": null -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-014.xml b/crates/dav-proto/resources/requests/report-014.xml deleted file mode 100644 index c367c61b..00000000 --- a/crates/dav-proto/resources/requests/report-014.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - me - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-015.json b/crates/dav-proto/resources/requests/report-015.json deleted file mode 100644 index 2fadae00..00000000 --- a/crates/dav-proto/resources/requests/report-015.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "AddressbookQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CardDav", - "data": { - "type": "AddressData", - "data": { - "properties": [ - { - "group": null, - "name": { - "type": "Version" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Uid" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Nickname" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Email" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Fn" - }, - "no_value": false - } - ] - } - } - } - ] - }, - "filters": [ - { - "type": "AnyOf" - }, - { - "type": "Property", - "comp": null, - "prop": { - "name": { - "type": "Fn" - }, - "group": null - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "daboo", - "collation": "UnicodeCasemap", - "negate": false - } - } - }, - { - "type": "Property", - "comp": null, - "prop": { - "name": { - "type": "Email" - }, - "group": null - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "daboo", - "collation": "UnicodeCasemap", - "negate": false - } - } - } - ], - "limit": null -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-015.xml b/crates/dav-proto/resources/requests/report-015.xml deleted file mode 100644 index 4374910c..00000000 --- a/crates/dav-proto/resources/requests/report-015.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - daboo - - - daboo - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-016.json b/crates/dav-proto/resources/requests/report-016.json deleted file mode 100644 index 6d1bc453..00000000 --- a/crates/dav-proto/resources/requests/report-016.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "type": "AddressbookQuery", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - } - ] - }, - "filters": [ - { - "type": "AnyOf" - }, - { - "type": "Property", - "comp": null, - "prop": { - "name": { - "type": "Fn" - }, - "group": null - }, - "op": { - "type": "TextMatch", - "data": { - "type": "TextMatch", - "match_type": "Contains", - "value": "daboo", - "collation": "UnicodeCasemap", - "negate": false - } - } - } - ], - "limit": 2 -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-016.xml b/crates/dav-proto/resources/requests/report-016.xml deleted file mode 100644 index 2cd16ff4..00000000 --- a/crates/dav-proto/resources/requests/report-016.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - daboo - - - - 2 - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-017.json b/crates/dav-proto/resources/requests/report-017.json deleted file mode 100644 index fa10e19e..00000000 --- a/crates/dav-proto/resources/requests/report-017.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "type": "AddressbookMultiGet", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CardDav", - "data": { - "type": "AddressData", - "data": { - "properties": [ - { - "group": null, - "name": { - "type": "Version" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Uid" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Nickname" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Email" - }, - "no_value": false - }, - { - "group": null, - "name": { - "type": "Fn" - }, - "no_value": false - } - ] - } - } - } - ] - }, - "hrefs": [ - "/home/bernard/addressbook/vcf102.vcf", - "/home/bernard/addressbook/vcf1.vcf" - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-017.xml b/crates/dav-proto/resources/requests/report-017.xml deleted file mode 100644 index 414e774c..00000000 --- a/crates/dav-proto/resources/requests/report-017.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - /home/bernard/addressbook/vcf102.vcf - /home/bernard/addressbook/vcf1.vcf - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-018.json b/crates/dav-proto/resources/requests/report-018.json deleted file mode 100644 index c468ee0c..00000000 --- a/crates/dav-proto/resources/requests/report-018.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "type": "AddressbookMultiGet", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - }, - { - "type": "CardDav", - "data": { - "type": "AddressData", - "data": { - "properties": [] - } - } - } - ] - }, - "hrefs": [ - "/home/bernard/addressbook/vcf3.vcf" - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-018.xml b/crates/dav-proto/resources/requests/report-018.xml deleted file mode 100644 index 8accfe59..00000000 --- a/crates/dav-proto/resources/requests/report-018.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - /home/bernard/addressbook/vcf3.vcf - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-019.json b/crates/dav-proto/resources/requests/report-019.json deleted file mode 100644 index 10d2bbd6..00000000 --- a/crates/dav-proto/resources/requests/report-019.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "SyncCollection", - "sync_token": "abc", - "properties": { - "type": "Prop", - "data": [ - { - "type": "WebDav", - "data": { - "type": "GetETag" - } - } - ] - }, - "depth": "Infinity", - "limit": 9 -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-019.xml b/crates/dav-proto/resources/requests/report-019.xml deleted file mode 100644 index 5c561f83..00000000 --- a/crates/dav-proto/resources/requests/report-019.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - abc - infinite - 9 - - - - diff --git a/crates/dav-proto/resources/requests/report-020.json b/crates/dav-proto/resources/requests/report-020.json deleted file mode 100644 index 52a49643..00000000 --- a/crates/dav-proto/resources/requests/report-020.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "AclPrincipalPropSet", - "properties": [ - { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-020.xml b/crates/dav-proto/resources/requests/report-020.xml deleted file mode 100644 index 5b9b9b5f..00000000 --- a/crates/dav-proto/resources/requests/report-020.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/crates/dav-proto/resources/requests/report-021.json b/crates/dav-proto/resources/requests/report-021.json deleted file mode 100644 index d12ead20..00000000 --- a/crates/dav-proto/resources/requests/report-021.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "PrincipalMatch", - "principal_properties": { - "Properties": [ - { - "type": "WebDav", - "data": { - "type": "Owner" - } - } - ] - }, - "properties": [] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-021.xml b/crates/dav-proto/resources/requests/report-021.xml deleted file mode 100644 index 114793a8..00000000 --- a/crates/dav-proto/resources/requests/report-021.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-022.json b/crates/dav-proto/resources/requests/report-022.json deleted file mode 100644 index 76eb5a5d..00000000 --- a/crates/dav-proto/resources/requests/report-022.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "type": "PrincipalPropertySearch", - "property_search": [ - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "match_": "doE" - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "title", - "attrs": "xmlns=\"http://www.example.com/ns/\"" - } - }, - "match_": "Sales" - } - ], - "properties": [ - { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "department", - "attrs": "xmlns=\"http://www.example.com/ns/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "phone", - "attrs": "xmlns=\"http://www.example.com/ns/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "office", - "attrs": "xmlns=\"http://www.example.com/ns/\"" - } - }, - { - "type": "DeadProperty", - "data": { - "name": "salary", - "attrs": "xmlns=\"http://www.example.com/ns/\"" - } - } - ], - "apply_to_principal_collection_set": false -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-022.xml b/crates/dav-proto/resources/requests/report-022.xml deleted file mode 100644 index 767edbc1..00000000 --- a/crates/dav-proto/resources/requests/report-022.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - doE - - - - - - Sales - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-023.json b/crates/dav-proto/resources/requests/report-023.json deleted file mode 100644 index b160fb60..00000000 --- a/crates/dav-proto/resources/requests/report-023.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "PrincipalSearchPropertySet" -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-023.xml b/crates/dav-proto/resources/requests/report-023.xml deleted file mode 100644 index f8b7d559..00000000 --- a/crates/dav-proto/resources/requests/report-023.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-024.json b/crates/dav-proto/resources/requests/report-024.json deleted file mode 100644 index 5c82d5af..00000000 --- a/crates/dav-proto/resources/requests/report-024.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "type": "ExpandProperty", - "properties": [ - { - "property": { - "type": "DeadProperty", - "data": { - "name": "version-history", - "attrs": "name=\"version-history\"" - } - }, - "depth": 0 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "version-set", - "attrs": "name=\"version-set\"" - } - }, - "depth": 1 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "creator-displayname", - "attrs": "name=\"creator-displayname\"" - } - }, - "depth": 2 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "activity-set", - "attrs": "name=\"activity-set\"" - } - }, - "depth": 2 - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-024.xml b/crates/dav-proto/resources/requests/report-024.xml deleted file mode 100644 index 72b15729..00000000 --- a/crates/dav-proto/resources/requests/report-024.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-025.json b/crates/dav-proto/resources/requests/report-025.json deleted file mode 100644 index c2519035..00000000 --- a/crates/dav-proto/resources/requests/report-025.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "type": "ExpandProperty", - "properties": [ - { - "property": { - "type": "DeadProperty", - "data": { - "name": "calendar-proxy-read-for", - "attrs": "name=\"calendar-proxy-read-for\" namespace=\"\nhttp://calendarserver.org/ns/\"" - } - }, - "depth": 0 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "email-address-set", - "attrs": "name=\"email-address-set\"\nnamespace=\"http://calendarserver.org/ns/\"" - } - }, - "depth": 1 - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "depth": 1 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "calendar-user-address-set", - "attrs": "name=\"calendar-user-address-set\"\nnamespace=\"urn:ietf:params:xml:ns:caldav\"" - } - }, - "depth": 1 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "calendar-proxy-write-for", - "attrs": "name=\"calendar-proxy-write-for\"\nnamespace=\"http://calendarserver.org/ns/\"" - } - }, - "depth": 0 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "email-address-set", - "attrs": "name=\"email-address-set\" namespace=\"http://calendarserver.org/ns/\"" - } - }, - "depth": 1 - }, - { - "property": { - "type": "WebDav", - "data": { - "type": "DisplayName" - } - }, - "depth": 1 - }, - { - "property": { - "type": "DeadProperty", - "data": { - "name": "calendar-user-address-set", - "attrs": "name=\"calendar-user-address-set\"\nnamespace=\"urn:ietf:params:xml:ns:caldav\"" - } - }, - "depth": 1 - } - ] -} \ No newline at end of file diff --git a/crates/dav-proto/resources/requests/report-025.xml b/crates/dav-proto/resources/requests/report-025.xml deleted file mode 100644 index fa75338a..00000000 --- a/crates/dav-proto/resources/requests/report-025.xml +++ /dev/null @@ -1,12 +0,0 @@ - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/001.xml b/crates/dav-proto/resources/responses/001.xml deleted file mode 100644 index 07731548..00000000 --- a/crates/dav-proto/resources/responses/001.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - /locked/ - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/002.xml b/crates/dav-proto/resources/responses/002.xml deleted file mode 100644 index d840870d..00000000 --- a/crates/dav-proto/resources/responses/002.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - http://www.example.com/file - - - Box type A - - HTTP/1.1 200 OK - - - Box type B - HTTP/1.1 403 Forbidden - The user does not have access to the DingALing property. - - - There has been an access violation error. - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/003.xml b/crates/dav-proto/resources/responses/003.xml deleted file mode 100644 index fe755bfe..00000000 --- a/crates/dav-proto/resources/responses/003.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - /container/ - - - 1997-12-02T01:42:21Z - Example collection - - - - - - - - - - - - - HTTP/1.1 200 OK - - - - /container/front.html - - - 1997-12-02T02:27:21Z - Example HTML resource - 4525 - text/html - "zzyzx" - Mon, 12 Jan 1998 09:25:56 GMT - - - - - - - - - - - - - HTTP/1.1 200 OK - - - diff --git a/crates/dav-proto/resources/responses/004.xml b/crates/dav-proto/resources/responses/004.xml deleted file mode 100644 index 5a74c55c..00000000 --- a/crates/dav-proto/resources/responses/004.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - http://www.example.com/container/resource3 - HTTP/1.1 423 Locked - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/005.xml b/crates/dav-proto/resources/responses/005.xml deleted file mode 100644 index 03c199ea..00000000 --- a/crates/dav-proto/resources/responses/005.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - infinity - - http://example.org/~ejw/contact.html - - Second-604800 - - urn:uuid:e71d4fae-5dec-22d6-fea5-00a0c91e6be4 - - - http://example.com/workspace/webdav/proposal.doc - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/006.xml b/crates/dav-proto/resources/responses/006.xml deleted file mode 100644 index ce54a793..00000000 --- a/crates/dav-proto/resources/responses/006.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - http://www.example.com/container/ - - - - - - - 0 - Jane Smith - Infinite - - urn:uuid:f81de2ad-7f3d-a1b2-4f3c-00a0c91a9d76 - - - http://www.example.com/container/ - - - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/007.xml b/crates/dav-proto/resources/responses/007.xml deleted file mode 100644 index d6f0c114..00000000 --- a/crates/dav-proto/resources/responses/007.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - /workspace/webdav/ - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/008.xml b/crates/dav-proto/resources/responses/008.xml deleted file mode 100644 index fa1ac79a..00000000 --- a/crates/dav-proto/resources/responses/008.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - http://cal.example.com/bernard/work/abcd2.ics - - - "fffff-abcd2" - - - HTTP/1.1 200 OK - - - - http://cal.example.com/bernard/work/abcd3.ics - - - "fffff-abcd3" - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/009.xml b/crates/dav-proto/resources/responses/009.xml deleted file mode 100644 index b9b8621c..00000000 --- a/crates/dav-proto/resources/responses/009.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - HTTP/1.1 200 OK - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/010.xml b/crates/dav-proto/resources/responses/010.xml deleted file mode 100644 index 52f36ba3..00000000 --- a/crates/dav-proto/resources/responses/010.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - /home/bernard/addressbook/v102.vcf - - - "23ba4d-ff11fb" - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/011.xml b/crates/dav-proto/resources/responses/011.xml deleted file mode 100644 index 1d2b4fbe..00000000 --- a/crates/dav-proto/resources/responses/011.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - /home/bernard/addressbook/ - HTTP/1.1 507 Insufficient Storage - - Only two matching records were returned - - - /home/bernard/addressbook/v102.vcf - - - "23ba4d-ff11fb" - - HTTP/1.1 200 OK - - - - /home/bernard/addressbook/v104.vcf - - - "23ba4d-ff11fc" - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/012.xml b/crates/dav-proto/resources/responses/012.xml deleted file mode 100644 index df7c55e1..00000000 --- a/crates/dav-proto/resources/responses/012.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - /a - - - - /c - - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/013.xml b/crates/dav-proto/resources/responses/013.xml deleted file mode 100644 index ac089fde..00000000 --- a/crates/dav-proto/resources/responses/013.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - Full name - - - - - - Job title - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/014.xml b/crates/dav-proto/resources/responses/014.xml deleted file mode 100644 index a8b5d897..00000000 --- a/crates/dav-proto/resources/responses/014.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - http://www.example.com/papers/ - - - - - - - Any operation - - - Read any object - - - - Read ACL - - - - - - - Read current user privilege set property - - - - - Write any object - - - - Write ACL - - - - Write properties - - - - Write resource content - - - - - Unlock resource - - - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/015.xml b/crates/dav-proto/resources/responses/015.xml deleted file mode 100644 index 0b3d38c8..00000000 --- a/crates/dav-proto/resources/responses/015.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - http://www.example.com/papers/ - - - - - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/016.xml b/crates/dav-proto/resources/responses/016.xml deleted file mode 100644 index 683b4aa9..00000000 --- a/crates/dav-proto/resources/responses/016.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - http://www.example.com/papers/ - - - - - - http://www.example.com/acl/groups/maintainers - - - - - - - - - - - - - - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/017.xml b/crates/dav-proto/resources/responses/017.xml deleted file mode 100644 index 2e62b8f6..00000000 --- a/crates/dav-proto/resources/responses/017.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - http://www.example.com/papers/ - - - - - - - - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/018.xml b/crates/dav-proto/resources/responses/018.xml deleted file mode 100644 index 9577ac7b..00000000 --- a/crates/dav-proto/resources/responses/018.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - http://www.example.com/papers/ - - - - http://www.example.com/acl/users/ - http://www.example.com/acl/groups/ - - - HTTP/1.1 200 OK - - - \ No newline at end of file diff --git a/crates/dav-proto/resources/responses/019.xml b/crates/dav-proto/resources/responses/019.xml deleted file mode 100644 index dcaf8cfc..00000000 --- a/crates/dav-proto/resources/responses/019.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - http://www.example.com/top/container/ - - - - http://www.example.com/users/gclemm - - - - - - Any operation - - - Read any object - - - - - Write any object - - - - Read the ACL - - - - Write the ACL - - - - - - - - - - - http://www.example.com/users/esedlar - - - - - - - - - - http://www.example.com/groups/mrktng - - - - - - - - - - - - - - - - - - - - - http://www.example.com/top - - - - - HTTP/1.1 200 OK - - - diff --git a/crates/dav-proto/resources/responses/020.xml b/crates/dav-proto/resources/responses/020.xml deleted file mode 100644 index 89eb7c2b..00000000 --- a/crates/dav-proto/resources/responses/020.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - -mailto:wilfredo@example.com - -2.0;Success - - - - -mailto:bernard@example.net - -2.0;Success - - - - -mailto:mike@example.org - -3.7;Invalid calendar user - - diff --git a/crates/dav-proto/src/lib.rs b/crates/dav-proto/src/lib.rs index c3681d60..bd21855e 100644 --- a/crates/dav-proto/src/lib.rs +++ b/crates/dav-proto/src/lib.rs @@ -36,6 +36,7 @@ pub fn xml_pretty_print(xml_string: &str) -> String { #[derive(Debug, Default, PartialEq, Eq)] pub struct RequestHeaders<'x> { pub uri: &'x str, + pub raw_uri: &'x str, pub depth: Depth, pub timeout: Timeout, pub content_type: Option<&'x str>, @@ -87,8 +88,11 @@ pub enum Condition<'x> { } #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum Timeout { Infinite, Second(u64), @@ -97,7 +101,10 @@ pub enum Timeout { } #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum Depth { Zero, One, diff --git a/crates/dav-proto/src/parser/header.rs b/crates/dav-proto/src/parser/header.rs index 7908aa45..6d3a93df 100644 --- a/crates/dav-proto/src/parser/header.rs +++ b/crates/dav-proto/src/parser/header.rs @@ -12,6 +12,7 @@ impl<'x> RequestHeaders<'x> { pub fn new(uri: &'x str) -> Self { RequestHeaders { uri, + raw_uri: uri, ..Default::default() } } @@ -25,6 +26,7 @@ impl<'x> RequestHeaders<'x> { } }, "Destination" => { + let value = strip_query_and_fragment(value); self.destination = Some(value); return true; }, @@ -400,551 +402,23 @@ pub fn dav_base_uri(uri: &str) -> Option<&str> { impl Depth { pub fn parse(value: &[u8]) -> Option { - hashify::tiny_map!(value, + hashify::map!(value, Depth, "0" => Depth::Zero, "1" => Depth::One, "infinity" => Depth::Infinity, "infinite" => Depth::Infinity, ) + .copied() } } +pub fn strip_query_and_fragment(uri: &str) -> &str { + uri.split_once(['?', '#']).map_or(uri, |(path, _)| path) +} + fn try_unwrap_coded_url(url: &str) -> &str { url.strip_prefix("<") .and_then(|url| url.strip_suffix(">")) .unwrap_or(url) } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_accept_vcard_version() { - for (header, expected) in [ - ("text/vcard", None), - ("application/json", None), - ("text/vcard; version=3.0", Some(VCardVersion::V3_0)), - ("text/vcard;version=\"3.0\"", Some(VCardVersion::V3_0)), - ("text/vcard; VERSION=3.0", Some(VCardVersion::V3_0)), - ("text/vcard; version=3.0; q=1.0", Some(VCardVersion::V3_0)), - ( - "text/vcard; version=3.0; charset=utf-8", - Some(VCardVersion::V3_0), - ), - ( - "text/vcard; charset=utf-8; version=3.0", - Some(VCardVersion::V3_0), - ), - ( - "text/vcard; version=4.0, text/vcard; version=3.0", - Some(VCardVersion::V4_0), - ), - ( - "text/vcard; version=4.0; q=0.5, text/vcard; version=3.0", - Some(VCardVersion::V3_0), - ), - ( - "text/vcard; q=0.5; version=4.0, text/vcard; q=1.0; version=3.0", - Some(VCardVersion::V3_0), - ), - ( - "text/vcard; version=4.0; q=0, text/vcard; version=3.0; q=0.1", - Some(VCardVersion::V3_0), - ), - ("*/*, text/vcard; version=3.0", Some(VCardVersion::V3_0)), - ("text/vcard-custom; version=3.0", None), - ] { - let mut headers = RequestHeaders::new("/dav/card/test/default/"); - assert!(headers.parse("Accept", header)); - assert_eq!(headers.vcard_version, expected, "failed for {header:?}"); - } - } - - #[test] - fn parse_range() { - for (header, expected) in [ - ( - "bytes=0-499", - Some(ByteRange::Offset { - start: 0, - end: Some(499), - }), - ), - ( - "bytes=500-", - Some(ByteRange::Offset { - start: 500, - end: None, - }), - ), - ( - "bytes = 500 - 999 ", - Some(ByteRange::Offset { - start: 500, - end: Some(999), - }), - ), - ( - "BYTES=0-0", - Some(ByteRange::Offset { - start: 0, - end: Some(0), - }), - ), - ("bytes=-500", Some(ByteRange::Suffix(500))), - ("bytes=-0", Some(ByteRange::Suffix(0))), - ("bytes=0-499,600-999", None), - ("bytes=499-100", None), - ("bytes=abc-def", None), - ("bytes=-", None), - ("bytes=0", None), - ("items=0-499", None), - ("0-499", None), - ] { - let mut headers = RequestHeaders::new("/dav/file/test/file.txt"); - assert_eq!(headers.parse("Range", header), expected.is_some()); - assert_eq!(headers.range, expected, "failed for {header:?}"); - } - - for (range, size, expected) in [ - ( - ByteRange::Offset { - start: 0, - end: Some(499), - }, - 1000, - Some(0..500), - ), - ( - ByteRange::Offset { - start: 0, - end: Some(499), - }, - 100, - Some(0..100), - ), - ( - ByteRange::Offset { - start: 500, - end: None, - }, - 1000, - Some(500..1000), - ), - ( - ByteRange::Offset { - start: 999, - end: Some(u64::MAX), - }, - 1000, - Some(999..1000), - ), - ( - ByteRange::Offset { - start: 1000, - end: None, - }, - 1000, - None, - ), - ( - ByteRange::Offset { - start: 0, - end: None, - }, - 0, - None, - ), - (ByteRange::Suffix(500), 1000, Some(500..1000)), - (ByteRange::Suffix(5000), 1000, Some(0..1000)), - (ByteRange::Suffix(0), 1000, None), - (ByteRange::Suffix(500), 0, None), - ] { - assert_eq!( - range.resolve(size), - expected, - "failed for {range:?} of {size}" - ); - } - } - - #[test] - fn eval_if_range_header() { - const LAST_MODIFIED: &str = "Mon, 10 Aug 2026 12:00:00 GMT"; - - let mut headers = RequestHeaders::new("/dav/file/test/file.txt"); - assert!(headers.eval_if_range("\"etag\"", Some(LAST_MODIFIED))); - - for (validator, expected) in [ - ("\"etag\"", true), - (LAST_MODIFIED, true), - ("W/\"etag\"", false), - ("\"other\"", false), - ("Sun, 09 Aug 2026 12:00:00 GMT", false), - ] { - assert!(headers.parse("If-Range", validator)); - assert_eq!( - headers.eval_if_range("\"etag\"", Some(LAST_MODIFIED)), - expected, - "failed for {validator:?}" - ); - } - - for (validator, expected) in [("\"etag\"", true), (LAST_MODIFIED, false)] { - assert!(headers.parse("If-Range", validator)); - assert_eq!( - headers.eval_if_range("\"etag\"", None), - expected, - "failed for {validator:?}" - ); - } - } - - #[test] - fn base_uri() { - for (uri, expected_base) in [ - ( - "http://host/dav/collection/account/test/", - Some("http://host/dav/collection/account"), - ), - ( - "http://host/dav/collection/account/test", - Some("http://host/dav/collection/account"), - ), - ( - "http://host/dav/collection/account/", - Some("http://host/dav/collection/account"), - ), - ( - "http://host/dav/collection/account", - Some("http://host/dav/collection/account"), - ), - ( - "http://host/dev/dav/collection/account/test/", - Some("http://host/dev/dav/collection/account"), - ), - ( - "http://host/dev/dav/collection/account/test", - Some("http://host/dev/dav/collection/account"), - ), - ( - "http://host/dev/dav/collection/account/", - Some("http://host/dev/dav/collection/account"), - ), - ( - "http://host/dev/dav/collection/account", - Some("http://host/dev/dav/collection/account"), - ), - ( - "/dav/collection/account/test/", - Some("/dav/collection/account"), - ), - ( - "/dav/collection/account/test", - Some("/dav/collection/account"), - ), - ("/dav/collection/account/", Some("/dav/collection/account")), - ("/dav/collection/account", Some("/dav/collection/account")), - ] { - assert_eq!(RequestHeaders::new(uri).base_uri(), expected_base); - } - } - - #[test] - fn eval_if_header() { - let mut headers = RequestHeaders::default(); - assert!(headers.parse( - "If", - r#"( - ["I am an ETag"]) - (["I am another ETag"])"#, - )); - - assert!(headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - etag: "\"I am an ETag\"" - }])); - assert!(headers.eval_if(&[ResourceState { - resource: None, - state_token: "", - etag: "\"I am another ETag\"" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "", - etag: "\"Unknown ETag\"" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - etag: "" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - etag: "\"Other ETag\"" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "", - etag: "\"I am an ETag\"" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:blah", - etag: "\"I am an ETag\"" - }])); - - assert!(headers.parse( - "If", - r#"(Not - )"#, - )); - assert!(headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:58f202ac-22cf-11d1-b12d-002035b29092", - etag: "" - }])); - assert!(!headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - etag: "" - }])); - - assert!(headers.parse( - "If", - r#"() - (Not )"# - )); - assert!(headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - etag: "" - }])); - assert!(headers.eval_if(&[ResourceState { - resource: None, - state_token: "urn:other-token", - etag: "" - }])); - } - - #[test] - fn parse_headers() { - let mut headers = RequestHeaders::default(); - assert!(headers.parse("Depth", "0")); - assert_eq!(headers.depth, Depth::Zero); - - assert!(headers.parse("Destination", "/path/to/destination")); - assert_eq!(headers.destination, Some("/path/to/destination")); - - assert!(headers.parse("Lock-Token", "")); - assert_eq!(headers.lock_token, Some("urn:uuid:1234")); - - for (input, expected) in [ - ( - "()", - vec![If { - resource: "urn:uuid:1234".into(), - list: vec![Condition::StateToken { - is_not: false, - token: "urn:uuid:1234", - }], - }], - ), - ( - "<>(<>)", - vec![If { - resource: "".into(), - list: vec![Condition::StateToken { - is_not: false, - token: "", - }], - }], - ), - ( - r#"( - ["I am an ETag"]) - (["I am another ETag"])"#, - vec![ - If { - resource: None, - list: vec![ - Condition::StateToken { - is_not: false, - token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - }, - Condition::ETag { - is_not: false, - tag: "\"I am an ETag\"", - }, - ], - }, - If { - resource: None, - list: vec![Condition::ETag { - is_not: false, - tag: "\"I am another ETag\"", - }], - }, - ], - ), - ( - r#"(Not - )"#, - vec![If { - resource: None, - list: vec![ - Condition::StateToken { - is_not: true, - token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - }, - Condition::StateToken { - is_not: false, - token: "urn:uuid:58f202ac-22cf-11d1-b12d-002035b29092", - }, - ], - }], - ), - ( - r#"() - (Not )"#, - vec![ - If { - resource: None, - list: vec![Condition::StateToken { - is_not: false, - token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - }], - }, - If { - resource: None, - list: vec![Condition::StateToken { - is_not: true, - token: "DAV:no-lock", - }], - }, - ], - ), - ( - r#" - ( - [W/"A weak ETag"]) (["strong ETag"])"#, - vec![ - If { - resource: "/resource1".into(), - list: vec![ - Condition::StateToken { - is_not: false, - token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - }, - Condition::ETag { - is_not: false, - tag: "W/\"A weak ETag\"", - }, - ], - }, - If { - resource: None, - list: vec![Condition::ETag { - is_not: false, - tag: "\"strong ETag\"", - }], - }, - ], - ), - ( - r#" - ()"#, - vec![If { - resource: "http://www.example.com/specs/".into(), - list: vec![Condition::StateToken { - is_not: false, - token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", - }], - }], - ), - ( - r#" (["4217"])"#, - vec![If { - resource: "/specs/rfc2518.doc".into(), - list: vec![Condition::ETag { - is_not: false, - tag: "\"4217\"", - }], - }], - ), - ( - r#" (Not ["4217"])"#, - vec![If { - resource: "/specs/rfc2518.doc".into(), - list: vec![Condition::ETag { - is_not: true, - tag: "\"4217\"", - }], - }], - ), - ( - r#" (["1234"]) (Not ["4217"])"#, - vec![ - If { - resource: "/test/file.txt".into(), - list: vec![Condition::ETag { - is_not: false, - tag: "\"1234\"", - }], - }, - If { - resource: "/specs/rfc2518.doc".into(), - list: vec![Condition::ETag { - is_not: true, - tag: "\"4217\"", - }], - }, - ], - ), - ] { - assert!(headers.parse("If", input)); - assert_eq!(headers.if_, expected, "Failed for input: {}", input); - headers.if_.clear(); - } - - assert!(headers.parse("If-Match", "*")); - assert_eq!( - headers.if_, - vec![If { - resource: None, - list: vec![Condition::Exists { is_not: false }], - }] - ); - headers.if_.clear(); - - assert!(headers.parse("If-None-Match", "etag1, etag2")); - assert_eq!( - headers.if_, - vec![If { - resource: None, - list: vec![ - Condition::ETag { - is_not: true, - tag: "etag1", - }, - Condition::ETag { - is_not: true, - tag: "etag2", - } - ], - },] - ); - - assert!(headers.parse("Timeout", "Second-10")); - assert_eq!(headers.timeout, Timeout::Second(10)); - - assert!(headers.parse("Timeout", "Infinite, Second-4100000000")); - assert_eq!(headers.timeout, Timeout::Infinite); - - assert!(headers.parse("Overwrite", "F")); - assert!(headers.overwrite_fail); - } -} diff --git a/crates/dav-proto/src/parser/mod.rs b/crates/dav-proto/src/parser/mod.rs index 150d4131..00159282 100644 --- a/crates/dav-proto/src/parser/mod.rs +++ b/crates/dav-proto/src/parser/mod.rs @@ -9,19 +9,30 @@ use std::{ fmt::{Display, Formatter}, }; -use quick_xml::events::BytesStart; +use quick_xml::events::{BytesStart, Event}; use tokenizer::Tokenizer; use crate::schema::{Element, NamedElement, Namespace}; +use types::metadata::XmlError; pub mod header; pub mod property; pub mod tokenizer; +pub mod value; #[derive(Debug, Clone)] pub enum Error { Xml(Box), UnexpectedToken(Box), + Filter(FilterError), + Value(XmlError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FilterError { + Invalid(Namespace), + UnsupportedCollation(Namespace, String), + TooComplex(Namespace), } #[derive(Debug, Clone)] @@ -42,13 +53,14 @@ pub enum Token<'x> { Bytes(Cow<'x, [u8]>), Text(Cow<'x, str>), UnknownElement(RawElement<'x>), + Content(Event<'x>), Eof, } #[derive(Debug, Clone)] pub struct RawElement<'x> { pub element: BytesStart<'x>, - pub namespace: Option>, + pub namespace: Option>, } pub trait DavParser: Sized { @@ -101,6 +113,7 @@ impl Token<'_> { Token::Bytes(bytes) => Token::Bytes(bytes.into_owned().into()), Token::Text(text) => Token::Text(text.into_owned().into()), Token::UnknownElement(raw) => Token::UnknownElement(raw.into_owned()), + Token::Content(event) => Token::Content(event.into_owned()), Token::Eof => Token::Eof, } } @@ -124,11 +137,11 @@ impl<'x> RawElement<'x> { pub fn with_namespace(self, namespace: quick_xml::name::Namespace<'_>) -> Self { RawElement { element: self.element, - namespace: Some(Cow::Owned(namespace.into_inner().to_vec())), + namespace: Some(Cow::Owned(namespace.into_inner().to_string())), } } - pub fn with_namespace_static(self, namespace: &'static [u8]) -> Self { + pub fn with_namespace_static(self, namespace: &'static str) -> Self { RawElement { element: self.element, namespace: Some(Cow::Borrowed(namespace)), @@ -143,7 +156,7 @@ impl<'x> RawElement<'x> { } } -#[cfg(test)] +#[cfg(all(test, stalwart_tests))] impl PartialEq for Token<'_> { fn eq(&self, other: &Self) -> bool { match (self, other) { @@ -167,8 +180,8 @@ impl PartialEq for Token<'_> { (Self::Bytes(l0), Self::Bytes(r0)) => l0 == r0, (Self::Text(l0), Self::Text(r0)) => l0 == r0, (Self::UnknownElement(l0), Self::UnknownElement(r0)) => { - let l0: &[u8] = l0.element.as_ref(); - let r0: &[u8] = r0.element.as_ref(); + let l0: &str = &l0.element; + let r0: &str = &r0.element; l0.eq_ignore_ascii_case(r0) } _ => core::mem::discriminant(self) == core::mem::discriminant(other), @@ -176,6 +189,18 @@ impl PartialEq for Token<'_> { } } +impl From for Error { + fn from(err: FilterError) -> Self { + Error::Filter(err) + } +} + +impl Error { + pub fn invalid_filter(namespace: Namespace) -> Self { + Error::Filter(FilterError::Invalid(namespace)) + } +} + impl NamedElement { pub fn into_unexpected(self) -> Error { Error::UnexpectedToken(Box::new(UnexpectedToken { @@ -205,6 +230,12 @@ impl Display for Error { } Ok(()) } + Error::Filter(FilterError::Invalid(_)) => write!(f, "Invalid filter"), + Error::Filter(FilterError::UnsupportedCollation(_, collation)) => { + write!(f, "Unsupported collation: {collation}") + } + Error::Filter(FilterError::TooComplex(_)) => write!(f, "Filter is too complex"), + Error::Value(err) => write!(f, "Invalid property value: {err}"), } } } diff --git a/crates/dav-proto/src/parser/property.rs b/crates/dav-proto/src/parser/property.rs index 59946467..5a07897d 100644 --- a/crates/dav-proto/src/parser/property.rs +++ b/crates/dav-proto/src/parser/property.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{DavParser, RawElement, Token, XmlValueParser, tokenizer::Tokenizer}; +use super::{RawElement, Token, XmlValueParser, tokenizer::Tokenizer, value::InheritedLang}; use crate::schema::{ Attribute, AttributeValue, Element, NamedElement, Namespace, property::{ @@ -21,7 +21,7 @@ use calcard::{ vcard::{VCardParameterName, VCardProperty, VCardVersion}, }; use mail_parser::DateTime; -use types::{TimeRange, dead_property::DeadProperty}; +use types::TimeRange; impl Tokenizer<'_> { pub(crate) fn collect_properties( @@ -61,17 +61,18 @@ impl Tokenizer<'_> { version, })); } - Token::ElementStart { name, .. } => { - if let Some(property) = DavProperty::from_element(name) { - elements.push(property); - } + Token::ElementStart { name, raw } => { + elements.push(match DavProperty::from_element(name) { + Some(property) => property, + None => DavProperty::Dead(raw.dead_name()?), + }); self.expect_element_end()?; } Token::ElementEnd => { break; } - Token::UnknownElement(name) => { - elements.push(DavProperty::DeadProperty((&name).into())); + Token::UnknownElement(raw) => { + elements.push(DavProperty::Dead(raw.dead_name()?)); self.expect_element_end()?; } token => return Err(token.into_unexpected()), @@ -290,14 +291,15 @@ impl Tokenizer<'_> { } } -impl Tokenizer<'_> { +impl<'x> Tokenizer<'x> { pub(crate) fn collect_property_values( &mut self, - elements: &mut Vec, + mut push: impl FnMut(DavPropertyValue), + lang: &mut InheritedLang<'_, 'x>, ) -> crate::parser::Result<()> { loop { match self.token()? { - Token::ElementStart { name, .. } => { + Token::ElementStart { name, raw } => { if let Some(property) = DavProperty::from_element(name) { let value = match property { DavProperty::WebDav(WebDavProperty::ResourceType) => { @@ -366,20 +368,16 @@ impl Tokenizer<'_> { .unwrap_or(DavValue::Null), }; - elements.push(DavPropertyValue { property, value }); + push(DavPropertyValue { property, value }); } else { - // Ignore unknown elements - self.seek_element_end()?; + push(self.collect_dead_property(&raw, lang.get()?)?); } } Token::ElementEnd | Token::Eof => { break; } Token::UnknownElement(raw) => { - elements.push(DavPropertyValue { - property: DavProperty::DeadProperty((&raw).into()), - value: DavValue::DeadProperty(DeadProperty::parse(self)?), - }); + push(self.collect_dead_property(&raw, lang.get()?)?); } token => return Err(token.into_unexpected()), } @@ -387,10 +385,22 @@ impl Tokenizer<'_> { Ok(()) } + + fn collect_dead_property( + &mut self, + raw: &RawElement<'x>, + lang: Option<&str>, + ) -> crate::parser::Result { + Ok(DavPropertyValue { + property: DavProperty::Dead(raw.dead_name()?), + value: DavValue::Dead(Box::new(self.collect_xml_value(raw, lang)?)), + }) + } } pub(crate) trait TimeRangeFromRaw { fn from_raw(raw: &RawElement<'_>) -> super::Result>; + fn from_raw_strict(raw: &RawElement<'_>) -> super::Result>; } impl TimeRangeFromRaw for TimeRange { @@ -422,6 +432,33 @@ impl TimeRangeFromRaw for TimeRange { Ok(None) } } + + fn from_raw_strict(raw: &RawElement<'_>) -> super::Result> { + let mut range = TimeRange { + start: i64::MIN, + end: i64::MAX, + }; + let mut has_bounds = false; + + for attribute in raw.attributes::() { + match attribute? { + Attribute::Start(start) => { + range.start = start.0; + has_bounds = true; + } + Attribute::End(end) => { + range.end = end.0; + has_bounds = true; + } + Attribute::Unknown { param, .. } if param == "start" || param == "end" => { + return Ok(None); + } + _ => {} + } + } + + Ok((has_bounds && range.start < range.end).then_some(range)) + } } impl DavProperty { diff --git a/crates/dav-proto/src/parser/tokenizer.rs b/crates/dav-proto/src/parser/tokenizer.rs index 72870873..99124a2e 100644 --- a/crates/dav-proto/src/parser/tokenizer.rs +++ b/crates/dav-proto/src/parser/tokenizer.rs @@ -4,17 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Error, RawElement, Token, UnexpectedToken, XmlValueParser}; -use crate::schema::{Attribute, AttributeValue, Element, NamedElement, Namespace}; +use super::{ + Error, RawElement, Token, UnexpectedToken, XmlValueParser, + value::{extension_namespace, known_namespace}, +}; +use crate::schema::{Attribute, AttributeValue, Element, NamedElement}; use quick_xml::{ NsReader, XmlVersion, - events::{Event, attributes::AttrError}, + events::{BytesRef, Event, attributes::AttrError}, name::ResolveResult, }; +use std::{borrow::Cow, mem}; pub struct Tokenizer<'x> { - xml: NsReader<&'x [u8]>, - last_is_end: bool, + pub(super) xml: NsReader<&'x [u8]>, + pub(super) last_is_end: bool, + content: bool, } impl<'x> Tokenizer<'x> { @@ -24,10 +29,11 @@ impl<'x> Tokenizer<'x> { Self { xml, last_is_end: false, + content: false, } } - pub fn token(&'_ mut self) -> super::Result> { + pub fn token(&mut self) -> super::Result> { loop { if self.last_is_end { self.last_is_end = false; @@ -44,33 +50,23 @@ impl<'x> Tokenizer<'x> { Event::End(_) => { return Ok(Token::ElementEnd); } - Event::Text(text) if text.iter().any(|ch| !ch.is_ascii_whitespace()) => { - return text - .xml_content(XmlVersion::Implicit1_0) - .map(Token::Text) - .map_err(|err| Error::Xml(Box::new(err.into()))); + event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_)) + if self.content => + { + return Ok(Token::Content(event)); + } + Event::Text(text) if text.bytes().any(|ch| !ch.is_ascii_whitespace()) => { + return Ok(Token::Text(text.xml_content(XmlVersion::Implicit1_0))); } Event::GeneralRef(entity) => { - let entity_ref: &[u8] = entity.as_ref(); - hashify::fnc_map!(entity_ref, - b"lt" => { return Ok(Token::Text("<".into())); }, - b"gt" => { return Ok(Token::Text(">".into())); }, - b"amp" => { return Ok(Token::Text("&".into())); }, - b"apos" => { return Ok(Token::Text("'".into())); }, - b"quot" => { return Ok(Token::Text("\"".into())); }, - _ => { - if let Ok(Some(gr)) = entity.resolve_char_ref() { - return Ok(Token::Text(gr.to_string().into())); - } - } - ); - - return entity - .xml_content(XmlVersion::Implicit1_0) - .map(Token::Text) - .map_err(|err| Error::Xml(Box::new(err.into()))); + return Ok(Token::Text(general_ref_text(entity))); + } + Event::CData(bytes) => { + return Ok(Token::Bytes(match bytes.into_inner() { + Cow::Borrowed(text) => Cow::Borrowed(text.as_bytes()), + Cow::Owned(text) => Cow::Owned(text.into_bytes()), + })); } - Event::CData(bytes) => return Ok(Token::Bytes(bytes.into_inner())), Event::Eof => return Ok(Token::Eof), _ => { continue; @@ -80,21 +76,23 @@ impl<'x> Tokenizer<'x> { // Parse element let name = tag.name(); match resolve_result { - ResolveResult::Bound(raw_ns) if !raw_ns.as_ref().is_empty() => { - if let (Some(ns), Some(element)) = ( - Namespace::try_parse(raw_ns.as_ref()), - Element::try_parse(name.local_name().as_ref()).copied(), - ) { - return Ok(Token::ElementStart { - name: NamedElement { ns, element }, - raw: RawElement::new(tag) - .with_namespace_static(ns.namespace().as_bytes()), - }); - } else { - return Ok(Token::UnknownElement( - RawElement::new(tag).with_namespace(raw_ns), - )); - } + ResolveResult::Bound(raw_ns) if !raw_ns.0.is_empty() => { + let element = + Element::try_parse(name.local_name().into_inner().as_bytes()).copied(); + let raw = RawElement::new(tag); + return Ok(match (known_namespace(raw_ns.0), element) { + (Some((ns, uri)), Some(element)) => Token::ElementStart { + name: NamedElement { ns: *ns, element }, + raw: raw.with_namespace_static(uri), + }, + (Some((_, uri)), None) => { + Token::UnknownElement(raw.with_namespace_static(uri)) + } + (None, _) => Token::UnknownElement(match extension_namespace(raw_ns.0) { + Some(uri) => raw.with_namespace_static(uri), + None => raw.with_namespace(raw_ns), + }), + }); } ResolveResult::Unknown(p) => { return Err(Error::Xml(Box::new(quick_xml::Error::Namespace( @@ -119,8 +117,15 @@ impl<'x> Tokenizer<'x> { } pub fn expect_named_element(&mut self, expected: NamedElement) -> super::Result<()> { + self.expect_named_element_raw(expected).map(|_| ()) + } + + pub fn expect_named_element_raw( + &mut self, + expected: NamedElement, + ) -> super::Result> { match self.token()? { - Token::ElementStart { name, .. } if name == expected => Ok(()), + Token::ElementStart { name, raw } if name == expected => Ok(raw), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: Token::ElementStart { name: expected, @@ -132,10 +137,13 @@ impl<'x> Tokenizer<'x> { } } - pub fn expect_named_element_or_eof(&mut self, expected: NamedElement) -> super::Result { + pub fn expect_named_element_or_eof( + &mut self, + expected: NamedElement, + ) -> super::Result>> { match self.token()? { - Token::ElementStart { name, .. } if name == expected => Ok(true), - Token::Eof => Ok(false), + Token::ElementStart { name, raw } if name == expected => Ok(Some(raw)), + Token::Eof => Ok(None), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: Token::ElementStart { name: expected, @@ -174,9 +182,27 @@ impl<'x> Tokenizer<'x> { } } + pub(super) fn with_content( + &mut self, + read: impl FnOnce(&mut Self) -> super::Result, + ) -> super::Result { + self.content = true; + let result = read(self); + self.content = false; + result + } + pub fn collect_string_value(&mut self) -> super::Result> { - let mut depth = 1; - let mut value: Option = None; + if mem::take(&mut self.last_is_end) { + return Ok(None); + } + self.with_content(Tokenizer::collect_string_content) + } + + fn collect_string_content(&mut self) -> super::Result> { + let mut depth = 1u32; + let mut value = String::new(); + let mut has_content = false; loop { match self.token()? { @@ -187,25 +213,24 @@ impl<'x> Tokenizer<'x> { break; } } - Token::Text(text) => { - if let Some(ref mut v) = value { - v.push_str(&text); - } else { - value = Some(text.into_owned()); - } + Token::Content(Event::Text(text)) => { + has_content |= text.bytes().any(|ch| !ch.is_ascii_whitespace()); + append_text(&mut value, text.xml_content(XmlVersion::Implicit1_0)); } - Token::Bytes(bytes) => { - if let Some(ref mut v) = value { - v.push_str(&String::from_utf8_lossy(&bytes)); - } else { - value = Some(String::from_utf8_lossy(&bytes).into_owned()); - } + Token::Content(Event::CData(bytes)) => { + has_content = true; + append_text(&mut value, bytes.into_inner()); + } + Token::Content(Event::GeneralRef(entity)) => { + has_content = true; + append_text(&mut value, general_ref_text(entity)); } Token::Eof => return Err(Token::Eof.into_unexpected()), + Token::Content(_) | Token::Text(_) | Token::Bytes(_) => {} } } - Ok(value) + Ok(has_content.then_some(value)) } pub fn parse_value(&mut self) -> super::Result>> { @@ -235,6 +260,7 @@ impl<'x> Tokenizer<'x> { result = Some(Err(String::from_utf8_lossy(&bytes).into_owned())); } } + Token::Content(_) => {} Token::Eof => return Err(Token::Eof.into_unexpected()), } } @@ -277,13 +303,37 @@ impl<'x> Tokenizer<'x> { } } +pub(super) fn append_text(value: &mut String, piece: Cow<'_, str>) { + if value.is_empty() { + *value = piece.into_owned(); + } else { + value.push_str(&piece); + } +} + +fn general_ref_text(entity: BytesRef<'_>) -> Cow<'_, str> { + if let Some(text) = hashify::map!(entity.as_bytes(), &'static str, + "lt" => "<", + "gt" => ">", + "amp" => "&", + "apos" => "'", + "quot" => "\"" + ) { + return Cow::Borrowed(text); + } + if let Ok(Some(ch)) = entity.resolve_char_ref() { + return Cow::Owned(ch.to_string()); + } + entity.xml_content(XmlVersion::Implicit1_0) +} + impl RawElement<'_> { pub fn attributes( &self, ) -> impl Iterator>> + '_ { self.element.attributes().filter_map(|attr| match attr { Ok(attr) => match attr.normalized_value(XmlVersion::Implicit1_0) { - Ok(value) => Attribute::from_param(attr.key.as_ref(), value).map(Ok), + Ok(value) => Attribute::from_param(attr.key.into_inner().as_bytes(), value).map(Ok), Err(err) => Some(Err(err.into())), }, Err(err) => Some(Err(err.into())), @@ -303,201 +353,3 @@ impl From for Error { } } -#[cfg(test)] -mod tests { - - use std::borrow::Cow; - - use crate::schema::{Collation, MatchType}; - - use super::*; - - #[derive(Debug, PartialEq, Eq)] - pub enum TestToken<'x> { - ElementStart(NamedElement), - ElementEnd, - Attribute(Attribute), - Bytes(Cow<'x, [u8]>), - Text(Cow<'x, str>), - } - - #[test] - fn test_tokenizer() { - for (input, expected) in [ - ( - r#" - - - - - - - - - "#, - vec![ - TestToken::ElementStart(NamedElement { - ns: Namespace::CalDav, - element: Element::CalendarQuery, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::Dav, - element: Element::Prop, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::Dav, - element: Element::Getetag, - }), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CalDav, - element: Element::CalendarData, - }), - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CalDav, - element: Element::Filter, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::CalDav, - element: Element::CompFilter, - }), - TestToken::Attribute(Attribute::Name("VCALENDAR".to_string())), - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementEnd, - ], - ), - ( - r#" - - - - - - - - - - - - - - me - - - "#, - vec![ - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::AddressbookQuery, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::Dav, - element: Element::Prop, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::Dav, - element: Element::Getetag, - }), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::AddressData, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Prop, - }), - TestToken::Attribute(Attribute::Name("VERSION".to_string())), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Prop, - }), - TestToken::Attribute(Attribute::Name("UID".to_string())), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Prop, - }), - TestToken::Attribute(Attribute::Name("NICKNAME".to_string())), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Prop, - }), - TestToken::Attribute(Attribute::Name("EMAIL".to_string())), - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Prop, - }), - TestToken::Attribute(Attribute::Name("FN".to_string())), - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::Filter, - }), - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::PropFilter, - }), - TestToken::Attribute(Attribute::Name("NICKNAME".to_string())), - TestToken::ElementStart(NamedElement { - ns: Namespace::CardDav, - element: Element::TextMatch, - }), - TestToken::Attribute(Attribute::Collation(Collation::UnicodeCasemap)), - TestToken::Attribute(Attribute::MatchType(MatchType::Equals)), - TestToken::Text("me".into()), - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementEnd, - TestToken::ElementEnd, - ], - ), - ] { - let mut tokenizer = Tokenizer::new(input.as_bytes()); - let mut result = vec![]; - - loop { - match tokenizer.token() { - Ok(token) => match token { - Token::ElementStart { name, raw } => { - result.push(TestToken::ElementStart(name)); - for attr in raw.attributes::() { - result.push(TestToken::Attribute(attr.unwrap())); - } - } - Token::ElementEnd => { - result.push(TestToken::ElementEnd); - } - Token::Bytes(cow) => { - result.push(TestToken::Bytes(cow.into_owned().into())); - } - Token::Text(cow) => { - result.push(TestToken::Text(cow.into_owned().into())); - } - Token::UnknownElement(_) => { - //result.push(TestToken::UnknownElement(unknown_element)); - } - Token::Eof => break, - }, - Err(err) => { - panic!("Error: {:?}", err); - } - } - } - - assert_eq!(result, expected); - } - } -} diff --git a/crates/dav-proto/src/parser/value.rs b/crates/dav-proto/src/parser/value.rs new file mode 100644 index 00000000..c824b78e --- /dev/null +++ b/crates/dav-proto/src/parser/value.rs @@ -0,0 +1,314 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{ + Error, RawElement, Result, Token, + tokenizer::{Tokenizer, append_text}, +}; +use crate::schema::Namespace; +use quick_xml::{ + XmlVersion, + escape::EscapeError, + events::{ + BytesRef, BytesStart, Event, + attributes::{Attribute, Attributes}, + }, + name::{NamespaceError, QName, ResolveResult}, +}; +use std::{borrow::Cow, mem}; +use types::metadata::{XmlAttribute, XmlElement, XmlError, XmlName, XmlNode, XmlValue}; + + +const XML_LANG: &str = "xml:lang"; +const XMLNS: &str = "xmlns"; + +enum LangScope<'s> { + Property(&'s mut Option), + Element, +} + +impl<'x> Tokenizer<'x> { + pub fn collect_xml_value( + &mut self, + property: &RawElement<'x>, + lang: Option<&str>, + ) -> Result> { + let mut own_lang = None; + let attributes = + self.value_attributes(&property.element, LangScope::Property(&mut own_lang))?; + let lang = match own_lang { + Some(lang) => Some(lang), + None => lang.map(str::to_string), + }; + let mut value = XmlValue { + lang: lang.filter(|lang| !lang.is_empty()).map(Cow::Owned), + attributes, + children: Vec::new(), + }; + + if !mem::take(&mut self.last_is_end) { + self.collect_xml_children(&mut value)?; + } + + value.validate().map_err(Error::Value)?; + Ok(value) + } + + fn collect_xml_children(&mut self, value: &mut XmlValue<'static>) -> Result<()> { + self.with_content(|tokenizer| tokenizer.collect_xml_content(value)) + } + + fn collect_xml_content(&mut self, value: &mut XmlValue<'static>) -> Result<()> { + let mut open: Vec> = Vec::new(); + let mut text = String::new(); + + loop { + let raw = match self.token()? { + Token::ElementStart { raw, .. } | Token::UnknownElement(raw) => raw, + Token::ElementEnd => { + flush_text(&mut text, children_of(&mut open, value)); + match open.pop() { + Some(element) => { + children_of(&mut open, value).push(XmlNode::Element(element)) + } + None => return Ok(()), + } + continue; + } + Token::Content(Event::Text(content)) => { + append_text(&mut text, content.xml_content(XmlVersion::Implicit1_0)); + continue; + } + Token::Content(Event::CData(content)) => { + append_text(&mut text, content.xml_content(XmlVersion::Implicit1_0)); + continue; + } + Token::Content(Event::GeneralRef(reference)) => { + push_reference(&mut text, &reference)?; + continue; + } + Token::Eof => return Err(Token::Eof.into_unexpected()), + Token::Content(_) | Token::Text(_) | Token::Bytes(_) => continue, + }; + + if open.len() >= XmlValue::MAX_DEPTH { + return Err(Error::Value(XmlError::NestingTooDeep)); + } + let namespace = raw.namespace.as_deref().map(namespace_uri).transpose()?; + let (local_name, prefix) = raw.element.name().decompose(); + let element = XmlElement { + name: XmlName { + namespace, + name: Cow::Owned(local_name.into_inner().to_string()), + }, + prefix: prefix.map(|prefix| Cow::Owned(prefix.into_inner().to_string())), + attributes: self.value_attributes(&raw.element, LangScope::Element)?, + children: Vec::new(), + }; + + flush_text(&mut text, children_of(&mut open, value)); + open.push(element); + } + } + + fn value_attributes( + &self, + tag: &BytesStart<'_>, + mut scope: LangScope<'_>, + ) -> Result>> { + let mut attributes = Vec::new(); + for attribute in unchecked_attributes(tag) { + let attribute = attribute?; + if attribute.key.as_namespace_binding().is_some() { + continue; + } + let value = attribute + .normalized_value(XmlVersion::Implicit1_0)? + .into_owned(); + if let LangScope::Property(lang) = &mut scope + && attribute.key.into_inner() == XML_LANG + { + if lang.replace(value).is_some() { + return Err(Error::Value(XmlError::DuplicateAttribute)); + } + continue; + } + let (resolved, local_name) = self.xml.resolver().resolve_attribute(attribute.key); + attributes.push(XmlAttribute { + name: XmlName { + namespace: resolved_namespace(resolved)?, + name: Cow::Owned(local_name.into_inner().to_string()), + }, + value: Cow::Owned(value), + }); + } + Ok(attributes) + } +} + +pub(crate) struct InheritedLang<'a, 'x> { + scopes: [&'a RawElement<'x>; 3], + resolved: Option>, +} + +impl<'a, 'x> InheritedLang<'a, 'x> { + pub fn new( + prop: &'a RawElement<'x>, + set: &'a RawElement<'x>, + request: &'a RawElement<'x>, + ) -> Self { + InheritedLang { + scopes: [prop, set, request], + resolved: None, + } + } + + pub fn get(&mut self) -> Result> { + if self.resolved.is_none() { + let mut lang = None; + for scope in self.scopes { + let scope_lang = scope.xml_lang()?; + if lang.is_none() { + lang = scope_lang; + } + } + self.resolved = Some(lang); + } + Ok(self.resolved.as_ref().and_then(Option::as_deref)) + } +} + +impl RawElement<'_> { + pub fn dead_name(&self) -> Result> { + let name = XmlName { + namespace: self.namespace.as_deref().map(namespace_uri).transpose()?, + name: Cow::Owned(self.element.local_name().into_inner().to_string()), + }; + name.validate_property().map_err(Error::Value)?; + Ok(name) + } + + pub fn xml_lang(&self) -> Result> { + if self.element.attributes_raw().trim_ascii_start().is_empty() { + return Ok(None); + } + for attribute in unchecked_attributes(&self.element) { + let attribute = attribute?; + if attribute.key.into_inner() == XML_LANG { + return Ok(Some( + attribute + .normalized_value(XmlVersion::Implicit1_0)? + .into_owned(), + )); + } + } + Ok(None) + } +} + +fn unchecked_attributes<'t>(tag: &'t BytesStart<'_>) -> Attributes<'t> { + let mut attributes = tag.attributes(); + attributes.with_checks(false); + attributes +} + +fn children_of<'v>( + open: &'v mut [XmlElement<'static>], + value: &'v mut XmlValue<'static>, +) -> &'v mut Vec> { + match open.last_mut() { + Some(element) => &mut element.children, + None => &mut value.children, + } +} + +fn flush_text(text: &mut String, children: &mut Vec>) { + if !text.is_empty() { + children.push(XmlNode::Text(Cow::Owned(mem::take(text)))); + } +} + +fn push_reference(text: &mut String, reference: &BytesRef<'_>) -> Result<()> { + if let Some(ch) = reference.resolve_char_ref()? { + text.push(ch); + return Ok(()); + } + let name: &str = reference; + match hashify::map!(name.as_bytes(), &'static str, + "lt" => "<", + "gt" => ">", + "amp" => "&", + "apos" => "'", + "quot" => "\"" + ) { + Some(replacement) => { + text.push_str(replacement); + Ok(()) + } + None => Err(xml(EscapeError::UnrecognizedEntity( + 0..name.len(), + name.to_string(), + ))), + } +} + +fn resolved_namespace(resolved: ResolveResult<'_>) -> Result>> { + match resolved { + ResolveResult::Bound(namespace) => namespace_uri(namespace.into_inner()).map(Some), + ResolveResult::Unbound => Ok(None), + ResolveResult::Unknown(prefix) => Err(xml(NamespaceError::UnknownPrefix(prefix))), + } +} + +pub(super) fn known_namespace(raw: &str) -> Option<&'static (Namespace, &'static str)> { + Namespace::try_parse_uri(raw.as_bytes()).or_else(|| { + raw.contains('&') + .then(|| normalized_uri(raw).ok()) + .flatten() + .and_then(|uri| Namespace::try_parse_uri(uri.as_bytes())) + }) +} + +pub(super) fn extension_namespace(raw: &str) -> Option<&'static str> { + hashify::map!(raw.as_bytes(), &'static str, + "http://apple.com/ns/ical/" => "http://apple.com/ns/ical/", + "http://me.com/_namespace/" => "http://me.com/_namespace/", + "urn:mobileme:davservices" => "urn:mobileme:davservices", + "http://owncloud.org/ns" => "http://owncloud.org/ns", + "http://nextcloud.org/ns" => "http://nextcloud.org/ns", + "http://sabredav.org/ns" => "http://sabredav.org/ns" + ) + .copied() +} + +fn namespace_uri(raw: &str) -> Result> { + if let Some((_, uri)) = Namespace::try_parse_uri(raw.as_bytes()) { + Ok(Cow::Borrowed(uri)) + } else if let Some(uri) = extension_namespace(raw) { + Ok(Cow::Borrowed(uri)) + } else if raw + .bytes() + .any(|byte| matches!(byte, b'&' | b'\t' | b'\n' | b'\r')) + { + normalized_uri(raw) + .map(|uri| Cow::Owned(uri.into_owned())) + .map_err(xml) + } else { + Ok(Cow::Owned(raw.to_string())) + } +} + +fn normalized_uri(raw: &str) -> quick_xml::Result> { + Attribute { + key: QName(XMLNS), + value: Cow::Borrowed(raw), + } + .normalized_value(XmlVersion::Implicit1_0) +} + +fn xml(err: impl Into) -> Error { + Error::Xml(Box::new(err.into())) +} diff --git a/crates/dav-proto/src/requests/acl.rs b/crates/dav-proto/src/requests/acl.rs index c47d384d..b74361e5 100644 --- a/crates/dav-proto/src/requests/acl.rs +++ b/crates/dav-proto/src/requests/acl.rs @@ -10,8 +10,8 @@ use crate::{ Element, NamedElement, Namespace, property::{DavValue, Privilege}, request::{ - Acl, AclPrincipalPropSet, DavPropertyValue, PrincipalMatch, PrincipalMatchProperties, - PrincipalPropertySearch, PropertySearch, + Acl, AclPrincipalPropSet, DavPropertyValue, FilterTest, PrincipalMatch, + PrincipalMatchProperties, PrincipalPropertySearch, PropertySearch, }, response::{Ace, GrantDeny, Href, List, Principal}, }, @@ -369,6 +369,7 @@ impl DavParser for PrincipalPropertySearch { property_search: vec![], properties: vec![], apply_to_principal_collection_set: false, + test: FilterTest::AllOf, }; loop { @@ -424,7 +425,7 @@ impl DavParser for PrincipalPropertySearch { impl PropertySearch { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result> { - let mut property = None; + let mut properties = Vec::new(); let mut match_ = None; loop { @@ -437,7 +438,7 @@ impl PropertySearch { }, .. } => { - property = stream.collect_properties(Vec::new())?.into_iter().next(); + properties = stream.collect_properties(properties)?; } Token::ElementStart { name: @@ -461,8 +462,8 @@ impl PropertySearch { } } - Ok(property.map(|property| PropertySearch { - property, + Ok((!properties.is_empty()).then(|| PropertySearch { + properties, match_: match_.unwrap_or_default(), })) } diff --git a/crates/dav-proto/src/requests/filter.rs b/crates/dav-proto/src/requests/filter.rs new file mode 100644 index 00000000..56d7d6aa --- /dev/null +++ b/crates/dav-proto/src/requests/filter.rs @@ -0,0 +1,489 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + parser::{ + Error, FilterError, RawElement, Token, property::TimeRangeFromRaw, tokenizer::Tokenizer, + }, + schema::{ + Attribute, AttributeValue, Collation, Element, MatchType, Namespace, + request::{ + CalendarPropFilter, CalendarPropMatch, CardFilter, CardPropFilter, CardPropMatch, + CompFilter, CompFilterMatch, FilterTest, ParamFilter, Presence, PropValueMatch, + TextMatch, VCardPropertyWithGroup, + }, + }, +}; +use calcard::{ + icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, + vcard::VCardParameterName, +}; +use types::TimeRange; + +pub const MAX_FILTER_ELEMENTS: usize = 256; +pub const MAX_FILTER_DEPTH: usize = 16; + +struct Frame { + name: N, + is_not_defined: bool, + filter: T, +} + +enum CalendarFrame { + Comp(Frame), + Prop(Frame), + Param(Frame>), +} + +enum CardFrame { + Prop(Frame), + Param(Frame>), +} + +trait FilterConditions { + fn has_conditions(&self) -> bool; +} + +trait ComponentNesting { + fn can_contain(&self, child: &Self) -> bool; +} + +trait DateProperty { + fn has_date_value(&self) -> bool; +} + +struct TextMatchOptions { + match_type: MatchType, + collation: Collation, + negate: bool, +} + +struct ElementBudget { + elements: usize, + namespace: Namespace, +} + +impl CompFilter { + pub(crate) fn parse_filter(stream: &mut Tokenizer<'_>) -> crate::parser::Result> { + let mut stack: Vec = Vec::with_capacity(4); + let mut roots: Vec = Vec::with_capacity(1); + let mut budget = ElementBudget::new(Namespace::CalDav); + + loop { + match stream.token()? { + Token::ElementStart { name, raw } => { + budget.consume(stack.len())?; + if name.ns != Namespace::CalDav { + return Err(Error::invalid_filter(Namespace::CalDav)); + } + match (name.element, stack.last_mut()) { + (Element::CompFilter, None | Some(CalendarFrame::Comp(_))) => { + let name = raw.component_name()?; + stack.push(CalendarFrame::Comp(Frame::new(name))); + } + (Element::PropFilter, Some(CalendarFrame::Comp(_))) => { + let name = raw.filter_name::(Namespace::CalDav)?; + stack.push(CalendarFrame::Prop(Frame::new(name))); + } + (Element::ParamFilter, Some(CalendarFrame::Prop(_))) => { + let name = + raw.filter_name::(Namespace::CalDav)?; + stack.push(CalendarFrame::Param(Frame::new(name))); + } + (Element::IsNotDefined, Some(frame)) => { + frame.set_not_defined(); + stream.expect_element_end()?; + } + (Element::TimeRange, Some(CalendarFrame::Comp(frame))) + if frame.filter.time_range.is_none() => + { + frame.filter.time_range = Some(raw.filter_time_range()?); + stream.expect_element_end()?; + } + (Element::TimeRange, Some(CalendarFrame::Prop(frame))) + if frame.filter.value.is_none() => + { + frame.filter.value = + Some(PropValueMatch::TimeRange(raw.filter_time_range()?)); + stream.expect_element_end()?; + } + (Element::TextMatch, Some(CalendarFrame::Prop(frame))) + if frame.filter.value.is_none() => + { + let options = TextMatchOptions::parse(&raw, Namespace::CalDav)?; + frame.filter.value = Some(PropValueMatch::Text( + options.into_text_match(stream.collect_string_value()?), + )); + } + (Element::TextMatch, Some(CalendarFrame::Param(frame))) + if frame.filter.is_none() => + { + let options = TextMatchOptions::parse(&raw, Namespace::CalDav)?; + frame.filter = + Some(options.into_text_match(stream.collect_string_value()?)); + } + _ => return Err(Error::invalid_filter(Namespace::CalDav)), + } + } + Token::ElementEnd => match stack.pop() { + Some(CalendarFrame::Comp(frame)) => { + let (name, test) = frame.into_parts(Namespace::CalDav)?; + match stack.last_mut() { + Some(CalendarFrame::Comp(parent)) if parent.name.can_contain(&name) => { + parent.filter.comp_filters.push(CompFilter { name, test }); + } + None => roots.push(CompFilter { name, test }), + Some(_) => return Err(Error::invalid_filter(Namespace::CalDav)), + } + } + Some(CalendarFrame::Prop(frame)) => { + if matches!(frame.filter.value, Some(PropValueMatch::TimeRange(_))) + && !frame.name.has_date_value() + { + return Err(Error::invalid_filter(Namespace::CalDav)); + } + let (name, test) = frame.into_parts(Namespace::CalDav)?; + let Some(CalendarFrame::Comp(parent)) = stack.last_mut() else { + return Err(Error::invalid_filter(Namespace::CalDav)); + }; + parent + .filter + .prop_filters + .push(CalendarPropFilter { name, test }); + } + Some(CalendarFrame::Param(frame)) => { + let (name, test) = frame.into_parts(Namespace::CalDav)?; + let Some(CalendarFrame::Prop(parent)) = stack.last_mut() else { + return Err(Error::invalid_filter(Namespace::CalDav)); + }; + parent.filter.param_filters.push(ParamFilter { name, test }); + } + None => break, + }, + Token::UnknownElement(_) => { + stream.seek_element_end()?; + } + token => return Err(token.into_unexpected()), + } + } + + CompFilter::from_roots(roots) + } + + fn from_roots(roots: Vec) -> crate::parser::Result> { + let calendar = ICalendarComponentType::VCalendar; + let mut roots = roots.into_iter(); + match (roots.next(), roots.next()) { + (None, _) => Ok(None), + (Some(root), None) if root.name == calendar => Ok(Some(root)), + (Some(root), None) if calendar.can_contain(&root.name) => Ok(Some(CompFilter { + name: calendar, + test: Presence::IsDefined(CompFilterMatch { + comp_filters: vec![root], + ..Default::default() + }), + })), + _ => Err(Error::invalid_filter(Namespace::CalDav)), + } + } +} + +impl CardFilter { + pub(crate) fn parse_filter( + test: FilterTest, + stream: &mut Tokenizer<'_>, + ) -> crate::parser::Result { + let mut filter = CardFilter { + test, + prop_filters: Vec::new(), + }; + let mut stack: Vec = Vec::with_capacity(2); + let mut budget = ElementBudget::new(Namespace::CardDav); + + loop { + match stream.token()? { + Token::ElementStart { name, raw } => { + budget.consume(stack.len())?; + if name.ns != Namespace::CardDav { + return Err(Error::invalid_filter(Namespace::CardDav)); + } + match (name.element, stack.last_mut()) { + (Element::PropFilter, None) => { + let mut frame = Frame::<_, CardPropMatch>::new( + raw.filter_name::(Namespace::CardDav)?, + ); + frame.filter.test = raw.filter_test()?; + stack.push(CardFrame::Prop(frame)); + } + (Element::ParamFilter, Some(CardFrame::Prop(_))) => { + let name = raw.filter_name::(Namespace::CardDav)?; + stack.push(CardFrame::Param(Frame::new(name))); + } + (Element::IsNotDefined, Some(frame)) => { + frame.set_not_defined(); + stream.expect_element_end()?; + } + (Element::TextMatch, Some(CardFrame::Prop(frame))) => { + let options = TextMatchOptions::parse(&raw, Namespace::CardDav)?; + frame + .filter + .text_matches + .push(options.into_text_match(stream.collect_string_value()?)); + } + (Element::TextMatch, Some(CardFrame::Param(frame))) + if frame.filter.is_none() => + { + let options = TextMatchOptions::parse(&raw, Namespace::CardDav)?; + frame.filter = + Some(options.into_text_match(stream.collect_string_value()?)); + } + _ => return Err(Error::invalid_filter(Namespace::CardDav)), + } + } + Token::ElementEnd => match stack.pop() { + Some(CardFrame::Prop(frame)) => { + let (name, test) = frame.into_parts(Namespace::CardDav)?; + filter.prop_filters.push(CardPropFilter { name, test }); + } + Some(CardFrame::Param(frame)) => { + let (name, test) = frame.into_parts(Namespace::CardDav)?; + let Some(CardFrame::Prop(parent)) = stack.last_mut() else { + return Err(Error::invalid_filter(Namespace::CardDav)); + }; + parent.filter.param_filters.push(ParamFilter { name, test }); + } + None => break, + }, + Token::UnknownElement(_) => { + stream.seek_element_end()?; + } + token => return Err(token.into_unexpected()), + } + } + + Ok(filter) + } +} + +impl Frame { + fn new(name: N) -> Self { + Frame { + name, + is_not_defined: false, + filter: T::default(), + } + } + + fn into_parts(self, namespace: Namespace) -> crate::parser::Result<(N, Presence)> { + match (self.is_not_defined, self.filter.has_conditions()) { + (false, _) => Ok((self.name, Presence::IsDefined(self.filter))), + (true, false) => Ok((self.name, Presence::IsNotDefined)), + (true, true) => Err(Error::invalid_filter(namespace)), + } + } +} + +impl CalendarFrame { + fn set_not_defined(&mut self) { + match self { + CalendarFrame::Comp(frame) => frame.is_not_defined = true, + CalendarFrame::Prop(frame) => frame.is_not_defined = true, + CalendarFrame::Param(frame) => frame.is_not_defined = true, + } + } +} + +impl CardFrame { + fn set_not_defined(&mut self) { + match self { + CardFrame::Prop(frame) => frame.is_not_defined = true, + CardFrame::Param(frame) => frame.is_not_defined = true, + } + } +} + +impl FilterConditions for CompFilterMatch { + fn has_conditions(&self) -> bool { + self.time_range.is_some() || !self.prop_filters.is_empty() || !self.comp_filters.is_empty() + } +} + +impl FilterConditions for CalendarPropMatch { + fn has_conditions(&self) -> bool { + self.value.is_some() || !self.param_filters.is_empty() + } +} + +impl FilterConditions for CardPropMatch { + fn has_conditions(&self) -> bool { + !self.text_matches.is_empty() || !self.param_filters.is_empty() + } +} + +impl FilterConditions for Option { + fn has_conditions(&self) -> bool { + self.is_some() + } +} + +impl ComponentNesting for ICalendarComponentType { + fn can_contain(&self, child: &Self) -> bool { + use ICalendarComponentType::*; + match (self, child) { + (_, VCalendar) => false, + ( + VCalendar, + VEvent | VTodo | VJournal | VFreebusy | VTimezone | VAvailability | Other(_), + ) => true, + (VCalendar, _) => false, + (_, VEvent | VTodo | VJournal | VFreebusy | VTimezone | VAvailability) => false, + (VEvent | VTodo, VAlarm) + | (VTimezone, Standard | Daylight) + | (VAvailability, Available) => true, + (_, VAlarm | Standard | Daylight | Available) => false, + _ => true, + } + } +} + +impl DateProperty for ICalendarProperty { + fn has_date_value(&self) -> bool { + matches!( + self, + ICalendarProperty::Dtstart + | ICalendarProperty::Dtend + | ICalendarProperty::Due + | ICalendarProperty::Completed + | ICalendarProperty::Created + | ICalendarProperty::Dtstamp + | ICalendarProperty::LastModified + | ICalendarProperty::RecurrenceId + | ICalendarProperty::Exdate + | ICalendarProperty::Rdate + | ICalendarProperty::Trigger + | ICalendarProperty::Acknowledged + | ICalendarProperty::Other(_) + ) + } +} + +impl ElementBudget { + fn new(namespace: Namespace) -> Self { + ElementBudget { + elements: 0, + namespace, + } + } + + fn consume(&mut self, depth: usize) -> crate::parser::Result<()> { + self.elements += 1; + if self.elements <= MAX_FILTER_ELEMENTS && depth < MAX_FILTER_DEPTH { + Ok(()) + } else { + Err(FilterError::TooComplex(self.namespace).into()) + } + } +} + +impl TextMatchOptions { + fn parse(raw: &RawElement<'_>, namespace: Namespace) -> crate::parser::Result { + let mut options = TextMatchOptions { + match_type: MatchType::Contains, + collation: Collation::default_for(namespace), + negate: false, + }; + + for attribute in raw.attributes::() { + match attribute? { + Attribute::MatchType(match_type) => { + options.match_type = match_type; + } + Attribute::NegateCondition(negate) => { + options.negate = negate; + } + Attribute::Collation(collation) => { + options.collation = collation; + } + Attribute::Unknown { param, value } if param == "collation" => { + if value == "default" { + options.collation = Collation::default_for(namespace); + } else { + return Err(FilterError::UnsupportedCollation(namespace, value).into()); + } + } + Attribute::Unknown { param, .. } + if param == "match-type" || param == "negate-condition" => + { + return Err(Error::invalid_filter(namespace)); + } + _ => {} + } + } + + Ok(options) + } + + fn into_text_match(self, value: Option) -> TextMatch { + TextMatch::new( + value.unwrap_or_default(), + self.match_type, + self.collation, + self.negate, + ) + } +} + +impl RawElement<'_> { + pub(crate) fn filter_test(&self) -> crate::parser::Result { + self.test_attribute() + .map(|test| test.unwrap_or(FilterTest::AnyOf)) + } + + pub(crate) fn principal_search_test(&self) -> crate::parser::Result { + self.test_attribute() + .map(|test| test.unwrap_or(FilterTest::AllOf)) + } + + fn test_attribute(&self) -> crate::parser::Result> { + for attribute in self.attributes::() { + if let Attribute::TestAllOf(all_of) = attribute? { + return Ok(Some(if all_of { + FilterTest::AllOf + } else { + FilterTest::AnyOf + })); + } + } + Ok(None) + } + + fn filter_time_range(&self) -> crate::parser::Result { + TimeRange::from_raw_strict(self)?.ok_or_else(|| Error::invalid_filter(Namespace::CalDav)) + } + + fn filter_name(&self, namespace: Namespace) -> crate::parser::Result { + for attribute in self.attributes::() { + if let Attribute::Name(name) = attribute? { + return Ok(name); + } + } + Err(Error::invalid_filter(namespace)) + } + + fn component_name(&self) -> crate::parser::Result { + for attribute in self.attributes::() { + match attribute? { + Attribute::Name(name) => return Ok(name), + Attribute::Unknown { param, value } if param == "name" => { + return Ok(ICalendarComponentType::Other(value)); + } + _ => {} + } + } + Err(Error::invalid_filter(Namespace::CalDav)) + } +} + diff --git a/crates/dav-proto/src/requests/lockinfo.rs b/crates/dav-proto/src/requests/lockinfo.rs index 47f6837d..5dc075e4 100644 --- a/crates/dav-proto/src/requests/lockinfo.rs +++ b/crates/dav-proto/src/requests/lockinfo.rs @@ -12,7 +12,6 @@ use crate::{ request::LockInfo, }, }; -use types::dead_property::DeadProperty; impl DavParser for LockInfo { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { @@ -22,7 +21,10 @@ impl DavParser for LockInfo { owner: None, }; - if stream.expect_named_element_or_eof(NamedElement::dav(Element::Lockinfo))? { + if let Some(raw) = + stream.expect_named_element_or_eof(NamedElement::dav(Element::Lockinfo))? + { + let lang = raw.xml_lang()?; loop { match stream.token()? { Token::ElementStart { @@ -51,9 +53,9 @@ impl DavParser for LockInfo { ns: Namespace::Dav, element: Element::Owner, }, - .. + raw, } => { - lockinfo.owner = Some(DeadProperty::parse(stream)?); + lockinfo.owner = Some(stream.collect_xml_value(&raw, lang.as_deref())?); } Token::ElementEnd | Token::Eof => { break; diff --git a/crates/dav-proto/src/requests/mkcol.rs b/crates/dav-proto/src/requests/mkcol.rs index da919e87..bdd14628 100644 --- a/crates/dav-proto/src/requests/mkcol.rs +++ b/crates/dav-proto/src/requests/mkcol.rs @@ -5,7 +5,7 @@ */ use crate::{ - parser::{DavParser, Token, tokenizer::Tokenizer}, + parser::{DavParser, Token, tokenizer::Tokenizer, value::InheritedLang}, schema::{Element, NamedElement, Namespace, request::MkCol}, }; @@ -15,24 +15,25 @@ impl DavParser for MkCol { is_mkcalendar: false, props: Vec::new(), }; - match stream.token()? { + let request = match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Mkcol, }, - .. - } => {} + raw, + } => raw, Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Mkcalendar, }, - .. + raw, } => { mkcol.is_mkcalendar = true; + raw } Token::Eof => { return Ok(mkcol); @@ -48,10 +49,13 @@ impl DavParser for MkCol { ns: Namespace::Dav, element: Element::Set, }, - .. + raw, } => { - stream.expect_named_element(NamedElement::dav(Element::Prop))?; - stream.collect_property_values(&mut mkcol.props)?; + let prop = stream.expect_named_element_raw(NamedElement::dav(Element::Prop))?; + stream.collect_property_values( + |value| mkcol.props.push(value), + &mut InheritedLang::new(&prop, &raw, &request), + )?; stream.expect_element_end()?; } Token::ElementEnd | Token::Eof => { diff --git a/crates/dav-proto/src/requests/mod.rs b/crates/dav-proto/src/requests/mod.rs index e00526c5..9c0f790d 100644 --- a/crates/dav-proto/src/requests/mod.rs +++ b/crates/dav-proto/src/requests/mod.rs @@ -4,198 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - parser::{DavParser, RawElement, Token, tokenizer::Tokenizer}, - schema::Namespace, -}; -use types::dead_property::{DeadElementTag, DeadProperty, DeadPropertyTag}; - pub mod acl; +pub mod filter; pub mod lockinfo; pub mod mkcol; pub mod propertyupdate; pub mod propfind; pub mod report; -impl DavParser for DeadProperty { - fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { - let mut depth = 1; - let mut items = DeadProperty::default(); - - loop { - match stream.token()? { - Token::ElementStart { raw, .. } | Token::UnknownElement(raw) => { - items.0.push(DeadPropertyTag::ElementStart((&raw).into())); - depth += 1; - } - Token::ElementEnd => { - depth -= 1; - if depth == 0 { - break; - } - items.0.push(DeadPropertyTag::ElementEnd); - } - Token::Text(text) => { - items.0.push(DeadPropertyTag::Text(text.into_owned())); - } - Token::Bytes(bytes) => { - items.0.push(DeadPropertyTag::Text( - String::from_utf8_lossy(&bytes).into_owned(), - )); - } - Token::Eof => { - break; - } - } - } - - Ok(items) - } -} - -pub trait NsDeadProperty { - fn single_with_ns(namespace: Namespace, name: &str) -> Self; -} - -impl NsDeadProperty for DeadProperty { - fn single_with_ns(namespace: Namespace, name: &str) -> Self { - DeadProperty(vec![ - DeadPropertyTag::ElementStart(DeadElementTag { - name: format!("{}:{name}", namespace.prefix()), - attrs: None, - }), - DeadPropertyTag::ElementEnd, - ]) - } -} - -impl From<&RawElement<'_>> for DeadElementTag { - fn from(raw: &RawElement<'_>) -> Self { - let name = std::str::from_utf8(raw.element.local_name().as_ref()) - .unwrap_or("invalid-utf8") - .trim_ascii() - .to_string(); - let mut attrs = String::with_capacity(raw.element.attributes_raw().len()); - if let Some(namespace) = &raw.namespace { - attrs.push_str("xmlns=\""); - attrs.push_str(std::str::from_utf8(namespace).unwrap_or("invalid-utf8")); - attrs.push('"'); - } - - for attr in raw.element.attributes().flatten() { - if attr.key.as_ref() == b"xmlns" || attr.key.as_ref().starts_with(b"xmlns:") { - // Skip namespace attributes - continue; - } - if let (Ok(key), Ok(value)) = ( - std::str::from_utf8(attr.key.as_ref()), - std::str::from_utf8(attr.value.as_ref()), - ) { - if !attrs.is_empty() { - attrs.push(' '); - } - attrs.push_str(key); - attrs.push('='); - attrs.push('"'); - attrs.push_str(value); - attrs.push('"'); - } - } - - DeadElementTag { - name, - attrs: (!attrs.is_empty()).then_some(attrs), - } - } -} - -#[cfg(test)] -mod tests { - use calcard::vcard::VCardVersion; - - use crate::{ - parser::{DavParser, tokenizer::Tokenizer}, - schema::{ - property::{CardDavProperty, DavProperty}, - request::{Acl, LockInfo, MkCol, PropFind, PropertyUpdate, Report}, - }, - }; - - #[test] - fn parse_address_data_version() { - let xml = r#" - - - - - - - "#; - - let mut tokenizer = Tokenizer::new(xml.as_bytes()); - let report = Report::parse(&mut tokenizer).unwrap(); - let Report::AddressbookQuery(query) = report else { - panic!("expected addressbook-query, got {report:?}"); - }; - let PropFind::Prop(properties) = query.properties else { - panic!("expected prop, got {:?}", query.properties); - }; - - let version = properties.iter().find_map(|property| match property { - DavProperty::CardDav(CardDavProperty::AddressData { version, .. }) => Some(*version), - _ => None, - }); - - assert_eq!(version, Some(Some(VCardVersion::V3_0))); - } - - #[test] - fn parse_requests() { - for entry in std::fs::read_dir("resources/requests").unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - - if path.extension().map(|ext| ext == "xml").unwrap_or(false) { - println!("Parsing: {:?}", path); - let filename = path.file_name().unwrap().to_str().unwrap(); - let xml = std::fs::read_to_string(&path).unwrap(); - let mut tokenizer = Tokenizer::new(xml.as_bytes()); - - let json_path = path.with_extension("json"); - let json_output = match filename.split_once('-').unwrap().0 { - "propfind" => match PropFind::parse(&mut tokenizer) { - Ok(propfind) => serde_json::to_string_pretty(&propfind).unwrap(), - Err(_) => String::new(), - }, - "propertyupdate" => serde_json::to_string_pretty( - &PropertyUpdate::parse(&mut tokenizer).unwrap(), - ) - .unwrap(), - "mkcol" => serde_json::to_string_pretty(&MkCol::parse(&mut tokenizer).unwrap()) - .unwrap(), - "lockinfo" => { - serde_json::to_string_pretty(&LockInfo::parse(&mut tokenizer).unwrap()) - .unwrap() - } - "report" => { - serde_json::to_string_pretty(&Report::parse(&mut tokenizer).unwrap()) - .unwrap() - } - "acl" => { - serde_json::to_string_pretty(&Acl::parse(&mut tokenizer).unwrap()).unwrap() - } - _ => { - panic!("Unknown method: {}", filename); - } - }; - - /*if json_path.exists() { - let expected = std::fs::read_to_string(json_path).unwrap(); - assert_eq!(json_output, expected); - } else {*/ - std::fs::write(json_path, json_output).unwrap(); - //} - } - } - } -} diff --git a/crates/dav-proto/src/requests/propertyupdate.rs b/crates/dav-proto/src/requests/propertyupdate.rs index 324b6fe3..83825ee8 100644 --- a/crates/dav-proto/src/requests/propertyupdate.rs +++ b/crates/dav-proto/src/requests/propertyupdate.rs @@ -5,17 +5,19 @@ */ use crate::{ - parser::{DavParser, Token, tokenizer::Tokenizer}, - schema::{Element, NamedElement, Namespace, request::PropertyUpdate}, + parser::{DavParser, Token, tokenizer::Tokenizer, value::InheritedLang}, + schema::{ + Element, NamedElement, Namespace, + request::{PropertyUpdate, PropertyUpdateOp}, + }, }; impl DavParser for PropertyUpdate { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { - stream.expect_named_element(NamedElement::dav(Element::Propertyupdate))?; + let request = + stream.expect_named_element_raw(NamedElement::dav(Element::Propertyupdate))?; let mut update = PropertyUpdate { - set: Vec::with_capacity(4), - remove: Vec::with_capacity(4), - set_first: true, + ops: Vec::with_capacity(4), }; loop { @@ -26,12 +28,14 @@ impl DavParser for PropertyUpdate { ns: Namespace::Dav, element: Element::Set, }, - .. + raw, } => { - stream.expect_named_element(NamedElement::dav(Element::Prop))?; - stream.collect_property_values(&mut update.set)?; + let prop = stream.expect_named_element_raw(NamedElement::dav(Element::Prop))?; + stream.collect_property_values( + |value| update.ops.push(PropertyUpdateOp::Set(value)), + &mut InheritedLang::new(&prop, &raw, &request), + )?; stream.expect_element_end()?; - update.set_first = update.remove.is_empty(); } Token::ElementStart { name: @@ -42,7 +46,12 @@ impl DavParser for PropertyUpdate { .. } => { stream.expect_named_element(NamedElement::dav(Element::Prop))?; - update.remove = stream.collect_properties(update.remove)?; + update.ops.extend( + stream + .collect_properties(Vec::new())? + .into_iter() + .map(PropertyUpdateOp::Remove), + ); stream.expect_element_end()?; } Token::ElementEnd | Token::Eof => { diff --git a/crates/dav-proto/src/requests/propfind.rs b/crates/dav-proto/src/requests/propfind.rs index 7629830d..c7904fe5 100644 --- a/crates/dav-proto/src/requests/propfind.rs +++ b/crates/dav-proto/src/requests/propfind.rs @@ -11,40 +11,69 @@ use crate::{ impl DavParser for PropFind { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { - if stream.expect_named_element_or_eof(NamedElement::dav(Element::Propfind))? { - match stream.unwrap_named_element()? { - NamedElement { - ns: Namespace::Dav, - element: Element::Propname, - } => Ok(PropFind::PropName), - NamedElement { - ns: Namespace::Dav, - element: Element::Allprop, + if stream + .expect_named_element_or_eof(NamedElement::dav(Element::Propfind))? + .is_none() + { + return Ok(PropFind::AllProp(vec![])); + } + + loop { + match stream.token()? { + Token::ElementStart { + name: + NamedElement { + ns: Namespace::Dav, + element: Element::Propname, + }, + .. + } => return Ok(PropFind::PropName), + Token::ElementStart { + name: + NamedElement { + ns: Namespace::Dav, + element: Element::Allprop, + }, + .. } => { stream.expect_element_end()?; - if matches!( - stream.token()?, - Token::ElementStart { - name: NamedElement { - ns: Namespace::Dav, - element: Element::Include - }, - .. - } - ) { - stream.collect_properties(Vec::new()).map(PropFind::AllProp) - } else { - Ok(PropFind::AllProp(vec![])) - } + return PropFind::parse_allprop_include(stream); } - NamedElement { - ns: Namespace::Dav, - element: Element::Prop, - } => stream.collect_properties(Vec::new()).map(PropFind::Prop), - element => Err(element.into_unexpected()), + Token::ElementStart { + name: + NamedElement { + ns: Namespace::Dav, + element: Element::Prop, + }, + .. + } => return stream.collect_properties(Vec::new()).map(PropFind::Prop), + Token::ElementStart { .. } | Token::UnknownElement(_) => { + stream.seek_element_end()?; + } + Token::ElementEnd | Token::Eof => return Ok(PropFind::AllProp(vec![])), + token => return Err(token.into_unexpected()), + } + } + } +} + +impl PropFind { + fn parse_allprop_include(stream: &mut Tokenizer<'_>) -> crate::parser::Result { + loop { + match stream.token()? { + Token::ElementStart { + name: + NamedElement { + ns: Namespace::Dav, + element: Element::Include, + }, + .. + } => return stream.collect_properties(Vec::new()).map(PropFind::AllProp), + Token::ElementStart { .. } | Token::UnknownElement(_) => { + stream.seek_element_end()?; + } + _ => return Ok(PropFind::AllProp(vec![])), } - } else { - Ok(PropFind::AllProp(vec![])) } } } diff --git a/crates/dav-proto/src/requests/report.rs b/crates/dav-proto/src/requests/report.rs index fce3c2aa..2aeae2a1 100644 --- a/crates/dav-proto/src/requests/report.rs +++ b/crates/dav-proto/src/requests/report.rs @@ -7,29 +7,30 @@ use crate::{ Depth, parser::{ - DavParser, RawElement, Token, XmlValueParser, property::TimeRangeFromRaw, + DavParser, Error, RawElement, Token, XmlValueParser, property::TimeRangeFromRaw, tokenizer::Tokenizer, }, schema::{ - Attribute, Collation, Element, MatchType, NamedElement, Namespace, + Element, NamedElement, Namespace, property::DavProperty, request::{ - AclPrincipalPropSet, AddressbookQuery, CalendarQuery, ExpandProperty, - ExpandPropertyItem, Filter, FilterOp, FreeBusyQuery, MultiGet, PrincipalMatch, - PrincipalPropertySearch, PropFind, Report, SyncCollection, TextMatch, Timezone, - VCardPropertyWithGroup, + AclPrincipalPropSet, AddressbookQuery, CalendarQuery, CardFilter, CompFilter, + ExpandProperty, ExpandPropertyItem, FreeBusyQuery, MultiGet, PrincipalMatch, + PrincipalPropertySearch, PropFind, Report, SyncCollection, Timezone, }, }, }; -use calcard::{ - icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, - vcard::VCardParameterName, -}; -use types::{TimeRange, dead_property::DeadElementTag}; +use quick_xml::XmlVersion; +use std::borrow::Cow; +use types::{TimeRange, metadata::XmlName}; impl DavParser for Report { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { - match stream.unwrap_named_element()? { + let (name, principal_search_test) = match stream.token()? { + Token::ElementStart { name, raw } => (name, raw.principal_search_test()?), + token => return Err(token.into_unexpected()), + }; + match name { NamedElement { ns: Namespace::CalDav, element: Element::CalendarQuery, @@ -65,7 +66,10 @@ impl DavParser for Report { NamedElement { ns: Namespace::Dav, element: Element::PrincipalPropertySearch, - } => PrincipalPropertySearch::parse(stream).map(Report::PrincipalPropertySearch), + } => PrincipalPropertySearch::parse(stream).map(|mut search| { + search.test = principal_search_test; + Report::PrincipalPropertySearch(search) + }), NamedElement { ns: Namespace::Dav, element: Element::PrincipalSearchPropertySet, @@ -85,158 +89,56 @@ impl DavParser for CalendarQuery { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut cq = CalendarQuery { properties: PropFind::AllProp(vec![]), - filters: vec![], + filter: None, timezone: Timezone::None, }; - let mut depth = 1; - let mut components = Vec::with_capacity(3); - let mut property = None; - let mut parameter = None; loop { match stream.token()? { - Token::ElementStart { name, raw } => match name { + Token::ElementStart { name, .. } => match name { NamedElement { ns: Namespace::Dav, element: Element::Propname, - } if depth == 1 => { + } => { cq.properties = PropFind::PropName; stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Allprop, - } if depth == 1 => { + } => { stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Prop, - } if depth == 1 => { + } => { cq.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::CalDav, element: Element::Filter, - } if depth == 1 => { - depth += 1; + } if cq.filter.is_none() => { + cq.filter = CompFilter::parse_filter(stream)?; } NamedElement { ns: Namespace::CalDav, element: Element::Timezone, - } if depth == 1 => { + } => { cq.timezone = Timezone::Name(stream.collect_string_value()?.unwrap_or_default()); } NamedElement { ns: Namespace::CalDav, element: Element::TimezoneId, - } if depth == 1 => { + } => { cq.timezone = Timezone::Id(stream.collect_string_value()?.unwrap_or_default()); } - NamedElement { - ns: Namespace::CalDav, - element: Element::CompFilter, - } if depth >= 2 => { - for attribute in raw.attributes::() { - if let Attribute::Name(name) = attribute? { - components.push((name, depth)); - } - } - depth += 1; - } - - NamedElement { - ns: Namespace::CalDav, - element: Element::PropFilter, - } if depth >= 3 => { - for attribute in raw.attributes::() { - if let Attribute::Name(name) = attribute? { - property = Some(name); - } - } - depth += 1; - } - NamedElement { - ns: Namespace::CalDav, - element: Element::ParamFilter, - } if depth >= 4 => { - for attribute in raw.attributes::() { - if let Attribute::Name(name) = attribute? { - parameter = Some(name); - } - } - depth += 1; - } - NamedElement { - ns: Namespace::CalDav, - element: Element::IsNotDefined, - } => { - stream.expect_element_end()?; - if let Some(filter) = Filter::from_parts( - components.iter().map(|(c, _)| c.clone()).collect(), - property.clone(), - parameter.clone(), - FilterOp::Undefined, - ) { - cq.filters.push(filter); - } - } - NamedElement { - ns: Namespace::CalDav, - element: Element::TextMatch, - } => { - let mut tm = TextMatch::parse(raw)?; - tm.value = stream.collect_string_value()?.unwrap_or_default(); - if let Some(filter) = Filter::from_parts( - components.iter().map(|(c, _)| c.clone()).collect(), - property.clone(), - parameter.clone(), - FilterOp::TextMatch(tm), - ) { - cq.filters.push(filter); - } - } - NamedElement { - ns: Namespace::CalDav, - element: Element::TimeRange, - } => { - let range = TimeRange::from_raw(&raw)?; - stream.expect_element_end()?; - if let Some(filter) = range.and_then(|range| { - Filter::from_parts( - components.iter().map(|(c, _)| c.clone()).collect(), - property.clone(), - parameter.clone(), - FilterOp::TimeRange(range), - ) - }) { - cq.filters.push(filter); - } - } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { - depth -= 1; - if depth == 0 { - break; - } - if matches!(components.last(), Some((_, d)) if *d == depth) { - if components.len() > 1 - && cq - .filters - .last() - .and_then(|c| c.components()) - .is_none_or(|c| c.len() < components.len()) - { - cq.filters.push(Filter::Component { - comp: components.iter().map(|(c, _)| c.clone()).collect(), - op: FilterOp::Exists, - }); - } - components.pop(); - } + break; } Token::UnknownElement(_) => { stream.seek_element_end()?; @@ -253,12 +155,10 @@ impl DavParser for AddressbookQuery { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut aq = AddressbookQuery { properties: PropFind::AllProp(vec![]), - filters: vec![], + filter: CardFilter::default(), limit: None, }; - let mut depth = 1; - let mut property = None; - let mut parameter = None; + let mut has_filter = false; loop { match stream.token()? { @@ -266,110 +166,44 @@ impl DavParser for AddressbookQuery { NamedElement { ns: Namespace::Dav, element: Element::Propname, - } if depth == 1 => { + } => { aq.properties = PropFind::PropName; stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Allprop, - } if depth == 1 => { + } => { stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Prop, - } if depth == 1 => { + } => { aq.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::CardDav, element: Element::Filter, - } if depth == 1 => { - if let Some(filter) = Filter::parse(raw)? { - aq.filters.push(filter); - } - depth += 1; + } if !has_filter => { + let test = raw.filter_test()?; + aq.filter = CardFilter::parse_filter(test, stream)?; + has_filter = true; } NamedElement { ns: Namespace::CardDav, element: Element::Limit, - } if depth == 1 => { + } => { stream.expect_named_element(NamedElement::carddav(Element::Nresults))?; if let Some(Ok(limit)) = stream.parse_value::()? { aq.limit = limit.into(); } stream.expect_element_end()?; } - NamedElement { - ns: Namespace::CardDav, - element: Element::PropFilter, - } if depth == 2 => { - let mut filter = None; - for attribute in raw.attributes::() { - match attribute? { - Attribute::Name(name) => { - property = Some(name); - } - Attribute::TestAllOf(all_of) => { - filter = - (if all_of { Filter::AllOf } else { Filter::AnyOf }).into(); - } - _ => {} - } - } - if let Some(filter) = filter { - aq.filters.push(filter); - } - depth += 1; - } - NamedElement { - ns: Namespace::CardDav, - element: Element::ParamFilter, - } if depth == 3 => { - for attribute in raw.attributes::() { - if let Attribute::Name(name) = attribute? { - parameter = Some(name); - } - } - depth += 1; - } - NamedElement { - ns: Namespace::CardDav, - element: Element::IsNotDefined, - } => { - stream.expect_element_end()?; - if let Some(filter) = Filter::from_parts( - (), - property.clone(), - parameter.clone(), - FilterOp::Undefined, - ) { - aq.filters.push(filter); - } - } - NamedElement { - ns: Namespace::CardDav, - element: Element::TextMatch, - } => { - let mut tm = TextMatch::parse(raw)?; - tm.value = stream.collect_string_value()?.unwrap_or_default(); - if let Some(filter) = Filter::from_parts( - (), - property.clone(), - parameter.clone(), - FilterOp::TextMatch(tm), - ) { - aq.filters.push(filter); - } - } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { - depth -= 1; - if depth == 0 { - break; - } + break; } Token::UnknownElement(_) => { stream.seek_element_end()?; @@ -515,35 +349,14 @@ impl DavParser for ExpandProperty { match stream.token()? { Token::ElementStart { name, raw } => match name { NamedElement { - ns, element: Element::Property, + .. } => { - for attribute in raw.attributes::() { - if let Attribute::Name(name) = attribute? { - if let Some(property) = Element::try_parse(name.as_bytes()) - .copied() - .and_then(|element| { - DavProperty::from_element(NamedElement { ns, element }) - }) - { - ep.properties.push(ExpandPropertyItem { - property, - depth: depth - 1, - }); - } else { - let attrs = raw.element.attributes_raw().trim_ascii(); - ep.properties.push(ExpandPropertyItem { - property: DavProperty::DeadProperty(DeadElementTag { - name, - attrs: (!attrs.is_empty()).then(|| { - String::from_utf8_lossy(attrs).into_owned() - }), - }), - depth: depth - 1, - }); - } - break; - } + if let Some(property) = raw.expand_property()? { + ep.properties.push(ExpandPropertyItem { + property, + depth: depth - 1, + }); } depth += 1; } @@ -567,66 +380,41 @@ impl DavParser for ExpandProperty { } } -impl TextMatch { - fn parse(raw: RawElement<'_>) -> crate::parser::Result { - let mut tm = TextMatch { - match_type: MatchType::Contains, - value: String::new(), - collation: Collation::AsciiCasemap, - negate: false, - }; - - for attribute in raw.attributes::() { - match attribute? { - Attribute::MatchType(match_type) => { - tm.match_type = match_type; - } - Attribute::NegateCondition(negate) => { - tm.negate = negate; - } - Attribute::Collation(collation) => { - tm.collation = collation; - } +impl RawElement<'_> { + fn expand_property(&self) -> crate::parser::Result> { + let mut name = None; + let mut namespace = None; + for attribute in self.element.attributes() { + let attribute = attribute?; + hashify::fnc_map!(attribute.key.into_inner().as_bytes(), + b"name" => { + name = Some(attribute.normalized_value(XmlVersion::Implicit1_0)?); + }, + b"namespace" => { + namespace = Some(attribute.normalized_value(XmlVersion::Implicit1_0)?); + }, _ => {} - } + ); } - - Ok(tm) - } -} - -impl Filter { - fn from_parts(comp: A, prop: Option, param: Option, op: FilterOp) -> Option { - match (prop, param) { - (Some(prop), Some(param)) => Some(Filter::Parameter { - comp, - prop, - param, - op, - }), - (Some(prop), None) => Some(Filter::Property { comp, prop, op }), - (None, None) => Some(Filter::Component { comp, op }), - _ => None, + let Some(name) = name else { + return Ok(None); + }; + let name = name.trim_ascii(); + let namespace = namespace + .as_deref() + .map_or(Namespace::Dav.namespace(), str::trim_ascii); + if let Some(property) = Namespace::try_parse(namespace.as_bytes()) + .zip(Element::try_parse(name.as_bytes()).copied()) + .and_then(|(ns, element)| DavProperty::from_element(NamedElement { ns, element })) + { + return Ok(Some(property)); } - } - - fn components(&self) -> Option<&A> { - match self { - Filter::Component { comp, .. } => Some(comp), - Filter::Property { comp, .. } => Some(comp), - Filter::Parameter { comp, .. } => Some(comp), - _ => None, - } - } - - fn parse(raw: RawElement<'_>) -> crate::parser::Result> { - for attribute in raw.attributes::() { - if let Attribute::TestAllOf(all_of) = attribute? { - return Ok(Some(if all_of { Filter::AllOf } else { Filter::AnyOf })); - } - } - - Ok(None) + let name = XmlName { + namespace: Some(Cow::Owned(namespace.to_string())), + name: Cow::Owned(name.to_string()), + }; + name.validate_property().map_err(Error::Value)?; + Ok(Some(DavProperty::Dead(name))) } } diff --git a/crates/dav-proto/src/responses/error.rs b/crates/dav-proto/src/responses/error.rs index 14e5b0bd..38bf83f8 100644 --- a/crates/dav-proto/src/responses/error.rs +++ b/crates/dav-proto/src/responses/error.rs @@ -4,9 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::schema::{ - Namespace, Namespaces, - response::{BaseCondition, CalCondition, CardCondition, Condition, ErrorResponse}, +use crate::{ + responses::XmlEscape, + schema::{ + Namespace, Namespaces, + response::{BaseCondition, CalCondition, CardCondition, Condition, ErrorResponse}, + }, }; use std::fmt::Display; @@ -106,9 +109,11 @@ impl Display for CalCondition { write!(f, "") } CalCondition::SupportedCalendarData => write!(f, ""), - CalCondition::SupportedFilter(_) => write!(f, ""), + CalCondition::SupportedFilter => write!(f, ""), CalCondition::SupportedCollation(c) => { - write!(f, "{c}") + write!(f, "")?; + c.write_escaped_to(f)?; + write!(f, "") } CalCondition::MinDateTime => write!(f, ""), CalCondition::MaxDateTime => write!(f, ""), @@ -150,9 +155,11 @@ impl Display for CardCondition { CardCondition::SupportedAddressDataConversion => { write!(f, "") } - CardCondition::SupportedFilter(_) => write!(f, ""), + CardCondition::SupportedFilter => write!(f, ""), CardCondition::SupportedCollation(c) => { - write!(f, "{c}") + write!(f, "")?; + c.write_escaped_to(f)?; + write!(f, "") } CardCondition::ValidAddressData => write!(f, ""), CardCondition::NoUidConflict(uid) => { diff --git a/crates/dav-proto/src/responses/lock.rs b/crates/dav-proto/src/responses/lock.rs index a7d1aaac..b2914df3 100644 --- a/crates/dav-proto/src/responses/lock.rs +++ b/crates/dav-proto/src/responses/lock.rs @@ -6,15 +6,21 @@ use crate::{ Depth, Timeout, - responses::DeadPropertyFormat, schema::{ + Namespace, property::{ActiveLock, LockDiscovery, LockEntry, LockScope, LockType, SupportedLock}, request::LockInfo, response::{Href, List}, }, }; -use std::fmt::Display; -use types::dead_property::DeadProperty; +use std::fmt::{self, Display}; +use types::metadata::{DavValueView, EncodedDavValue, XmlName}; + +const OWNER: XmlName<'static> = XmlName::borrowed(Some("DAV:"), "owner"); + +fn write_owner(owner: DavValueView<'_>, f: &mut fmt::Formatter<'_>) -> fmt::Result { + owner.write_property_prefixed(&OWNER, Namespace::Dav.prefix(), f) +} impl Display for SupportedLock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -37,9 +43,7 @@ impl Display for ActiveLock { )?; if let Some(owner) = &self.owner { - f.write_str("")?; - owner.fmt(f)?; - f.write_str("")?; + write_owner(owner.view(), f)?; } write!(f, "{}", self.timeout)?; @@ -82,9 +86,7 @@ impl Display for LockInfo { write!(f, "{}{}", self.lock_scope, self.lock_type)?; if let Some(owner) = &self.owner { - f.write_str("")?; - owner.fmt(f)?; - f.write_str("")?; + write_owner(owner.to_encoded().map_err(|_| fmt::Error)?.view(), f)?; } write!(f, "",) @@ -142,12 +144,12 @@ impl ActiveLock { self } - pub fn with_owner_opt(mut self, owner: Option) -> Self { + pub fn with_owner_opt(mut self, owner: Option) -> Self { self.owner = owner; self } - pub fn with_owner(mut self, owner: DeadProperty) -> Self { + pub fn with_owner(mut self, owner: EncodedDavValue) -> Self { self.owner = Some(owner); self } diff --git a/crates/dav-proto/src/responses/mod.rs b/crates/dav-proto/src/responses/mod.rs index 1ce07a87..727bf56d 100644 --- a/crates/dav-proto/src/responses/mod.rs +++ b/crates/dav-proto/src/responses/mod.rs @@ -19,7 +19,6 @@ use crate::schema::{ response::{Href, List, Location, ResponseDescription, Status, SyncToken}, }; use std::fmt::{Display, Write}; -use types::dead_property::{DeadProperty, DeadPropertyTag}; trait XmlEscape { fn write_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result; @@ -29,45 +28,45 @@ trait XmlCdataEscape { fn write_cdata_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result; } +const CDATA_END: &[u8] = b"]]>"; + impl> XmlEscape for T { fn write_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result { - let str = self.as_ref(); + let text = self.as_ref(); + let mut start = 0; - for c in str.chars() { - match c { - '<' => out.write_str("<")?, - '>' => out.write_str(">")?, - '&' => out.write_str("&")?, - '"' => out.write_str(""")?, - '\'' => out.write_str("'")?, - _ => out.write_char(c)?, - } + for (pos, byte) in text.bytes().enumerate() { + let entity = match byte { + b'<' => "<", + b'>' => ">", + b'&' => "&", + b'"' => """, + b'\'' => "'", + _ => continue, + }; + out.write_str(text.get(start..pos).unwrap_or_default())?; + out.write_str(entity)?; + start = pos + 1; } - Ok(()) + out.write_str(text.get(start..).unwrap_or_default()) } } impl> XmlCdataEscape for T { fn write_cdata_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result { - let str = self.as_ref(); - let mut last_ch = '\0'; - let mut last_ch2 = '\0'; + let text = self.as_ref(); + let mut start = 0; out.write_str("' if last_ch == ']' && last_ch2 == ']' => { - out.write_str("]]>")?; - } - _ => out.write_char(ch)?, - } - - last_ch2 = last_ch; - last_ch = ch; + for pos in memchr::memmem::find_iter(text.as_bytes(), CDATA_END) { + out.write_str(text.get(start..pos).unwrap_or_default())?; + out.write_str("]]]]>")?; + start = pos + CDATA_END.len(); } + out.write_str(text.get(start..).unwrap_or_default())?; out.write_str("]]>") } } @@ -168,583 +167,3 @@ impl Display for SupportedCollation { } } -pub trait DeadPropertyFormat { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result; -} - -impl DeadPropertyFormat for DeadProperty { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut last_tag = ""; - - for item in &self.0 { - match item { - DeadPropertyTag::ElementStart(tag) => { - let name = &tag.name; - if let Some(attrs) = &tag.attrs { - write!(f, "<{name} {attrs}>")?; - } else { - write!(f, "<{name}>")?; - } - last_tag = name; - } - DeadPropertyTag::ElementEnd => { - write!(f, "", last_tag)?; - } - DeadPropertyTag::Text(text) => { - text.write_escaped_to(f)?; - } - } - } - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::fmt::Display; - - use calcard::{icalendar::ICalendar, vcard::VCard}; - use hyper::StatusCode; - use mail_parser::DateTime; - use types::dead_property::{DeadElementTag, DeadProperty, DeadPropertyTag}; - - use crate::{ - Depth, - parser::{Token, tokenizer::Tokenizer}, - responses::XmlCdataEscape, - schema::{ - Namespace, - property::{ - ActiveLock, CalDavProperty, CardDavProperty, DavValue, LockScope, Privilege, - ResourceType, Rfc1123DateTime, SupportedLock, WebDavProperty, - }, - request::DavPropertyValue, - response::{ - Ace, AclRestrictions, BaseCondition, ErrorResponse, GrantDeny, Href, List, - MkColResponse, MultiStatus, Principal, PrincipalSearchProperty, - PrincipalSearchPropertySet, PropResponse, PropStat, RequiredPrincipal, Resource, - Response, ScheduleResponse, ScheduleResponseItem, SupportedPrivilege, - }, - }, - }; - - impl List { - pub fn new(vec: impl IntoIterator) -> Self { - List(vec.into_iter().collect()) - } - } - - impl From for DavValue { - fn from(v: ICalendar) -> Self { - DavValue::ICalendar(v) - } - } - - impl From for DavValue { - fn from(v: VCard) -> Self { - DavValue::VCard(v) - } - } - - #[test] - fn parse_responses() { - for (num, test) in [ - // 001.xml - ErrorResponse::new(BaseCondition::LockTokenSubmitted(List::new([Href( - "/locked/".to_string(), - )]))) - .to_string(), - // 002.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/file", - vec![ - PropStat::new(DavPropertyValue::new( - WebDavProperty::DisplayName, - "Box type A", - )), - PropStat::new(DavPropertyValue::new( - WebDavProperty::DisplayName, - "Box type B", - )) - .with_status(StatusCode::FORBIDDEN) - .with_response_description( - "The user does not have access to the DingALing property.", - ), - ], - )]) - .with_response_description("There has been an access violation error.") - .to_string(), - // 003.xml - MultiStatus::new(vec![ - Response::new_propstat( - "/container/", - vec![PropStat::new_list(vec![ - DavPropertyValue::new( - WebDavProperty::CreationDate, - DateTime::parse_rfc3339("1997-12-01T17:42:21-08:00Z").unwrap(), - ), - DavPropertyValue::new(WebDavProperty::DisplayName, "Example collection"), - DavPropertyValue::new( - WebDavProperty::ResourceType, - vec![ResourceType::Collection], - ), - DavPropertyValue::new( - WebDavProperty::SupportedLock, - SupportedLock::default(), - ), - ])], - ), - Response::new_propstat( - "/container/front.html", - vec![PropStat::new_list(vec![ - DavPropertyValue::new( - WebDavProperty::CreationDate, - DateTime::parse_rfc3339("1997-12-01T18:27:21-08:00").unwrap(), - ), - DavPropertyValue::new(WebDavProperty::DisplayName, "Example HTML resource"), - DavPropertyValue::new(WebDavProperty::GetContentLength, 4525u64), - DavPropertyValue::new(WebDavProperty::GetContentType, "text/html"), - DavPropertyValue::new(WebDavProperty::GetETag, "\"zzyzx\""), - DavPropertyValue::new( - WebDavProperty::GetLastModified, - DavValue::Rfc1123Date(Rfc1123DateTime::new( - DateTime::parse_rfc822("Mon, 12 Jan 1998 09:25:56 GMT") - .unwrap() - .to_timestamp(), - )), - ), - DavPropertyValue::new(WebDavProperty::ResourceType, DavValue::Null), - DavPropertyValue::new( - WebDavProperty::SupportedLock, - SupportedLock::default(), - ), - ])], - ), - ]) - .to_string(), - // 004.xml - MultiStatus::new(vec![ - Response::new_status( - ["http://www.example.com/container/resource3"], - StatusCode::LOCKED, - ) - .with_error(BaseCondition::LockTokenSubmitted(List(vec![]))), - ]) - .to_string(), - // 005.xml - PropResponse::new(vec![DavPropertyValue::new( - WebDavProperty::LockDiscovery, - vec![ - ActiveLock::new( - "http://example.com/workspace/webdav/proposal.doc", - LockScope::Exclusive, - ) - .with_owner(DeadProperty(vec![ - DeadPropertyTag::ElementStart(DeadElementTag { - name: "D:href".to_string(), - attrs: None, - }), - DeadPropertyTag::Text("http://example.org/~ejw/contact.html".to_string()), - DeadPropertyTag::ElementEnd, - ])) - .with_timeout(604800) - .with_lock_token("urn:uuid:e71d4fae-5dec-22d6-fea5-00a0c91e6be4"), - ], - )]) - .to_string(), - // 006.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/container/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::LockDiscovery, - vec![ - ActiveLock::new("http://www.example.com/container/", LockScope::Shared) - .with_owner(DeadProperty(vec![DeadPropertyTag::Text( - "Jane Smith".to_string(), - )])) - .with_depth(Depth::Zero) - .with_lock_token("urn:uuid:f81de2ad-7f3d-a1b2-4f3c-00a0c91a9d76"), - ], - )])], - )]) - .to_string(), - // 007.xml - ErrorResponse::new(BaseCondition::LockTokenSubmitted(List(vec![Href( - "/workspace/webdav/".to_string(), - )]))) - .to_string(), - // 008.xml - MultiStatus::new(vec![ - Response::new_propstat( - "http://cal.example.com/bernard/work/abcd2.ics", - vec![PropStat::new_list(vec![ - DavPropertyValue::new(WebDavProperty::GetETag, "\"fffff-abcd2\""), - DavPropertyValue::new( - CalDavProperty::CalendarData(Default::default()), - DavValue::CData( - r#"BEGIN:VCALENDAR -VERSION:2.0 -BEGIN:VEVENT -DTSTART;TZID=US/Eastern:20060106T140000 -DURATION:PT1H -RECURRENCE-ID;TZID=US/Eastern:20060106T120000 -SUMMARY:Event #2 bis bis -UID:00959BC664CA650E933C892C@example.com -END:VEVENT -END:VCALENDAR -"# - .to_string(), - ), - ), - ])], - ), - Response::new_propstat( - "http://cal.example.com/bernard/work/abcd3.ics", - vec![PropStat::new_list(vec![ - DavPropertyValue::new(WebDavProperty::GetETag, "\"fffff-abcd3\""), - DavPropertyValue::new( - CalDavProperty::CalendarData(Default::default()), - DavValue::CData( - r#"BEGIN:VCALENDAR -VERSION:2.0 -PRODID:-//Example Corp.//CalDAV Client//EN -BEGIN:VEVENT -DTSTART;TZID=US/Eastern:20060104T100000 -DURATION:PT1H -SUMMARY:Event #3 -UID:DC6C50A017428C5216A2F1CD@example.com -END:VEVENT -END:VCALENDAR -"# - .to_string(), - ), - ), - ])], - ), - ]) - .with_namespace(Namespace::CalDav) - .to_string(), - // 009.xml - MkColResponse::new(vec![PropStat::new_list(vec![ - DavPropertyValue::new(WebDavProperty::ResourceType, DavValue::Null), - DavPropertyValue::new(WebDavProperty::DisplayName, DavValue::Null), - DavPropertyValue::new(CardDavProperty::AddressbookDescription, DavValue::Null), - ])]) - .with_namespace(Namespace::CardDav) - .to_string(), - // 010.xml - MultiStatus::new(vec![Response::new_propstat( - "/home/bernard/addressbook/v102.vcf", - vec![PropStat::new_list(vec![ - DavPropertyValue::new(WebDavProperty::GetETag, "\"23ba4d-ff11fb\""), - DavPropertyValue::new( - CardDavProperty::AddressData { - properties: Default::default(), - version: None, - }, - DavValue::CData( - r#"BEGIN:VCARD -VERSION:3.0 -NICKNAME:me -UID:34222-232@example.com -FN:Cyrus Daboo -EMAIL:daboo@example.com -END:VCARD -"# - .to_string(), - ), - ), - ])], - )]) - .with_namespace(Namespace::CardDav) - .to_string(), - // 011.xml - MultiStatus::new(vec![ - Response::new_status( - ["/home/bernard/addressbook/"], - StatusCode::INSUFFICIENT_STORAGE, - ) - .with_error(BaseCondition::NumberOfMatchesWithinLimit) - .with_response_description("Only two matching records were returned"), - Response::new_propstat( - "/home/bernard/addressbook/v102.vcf", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::GetETag, - "\"23ba4d-ff11fb\"", - )])], - ), - Response::new_propstat( - "/home/bernard/addressbook/v104.vcf", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::GetETag, - "\"23ba4d-ff11fc\"", - )])], - ), - ]) - .with_namespace(Namespace::CardDav) - .to_string(), - // 012.xml - ErrorResponse::new(BaseCondition::NeedPrivileges(List(vec![ - Resource::new("/a", Privilege::Unbind), - Resource::new("/c", Privilege::Bind), - ]))) - .to_string(), - // 013.xml - PrincipalSearchPropertySet::new(vec![ - PrincipalSearchProperty::new(WebDavProperty::DisplayName, "Full name"), - PrincipalSearchProperty::new(WebDavProperty::DisplayName, "Job title"), - ]) - .to_string(), - // 014.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/papers/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::SupportedPrivilegeSet, - vec![ - SupportedPrivilege::new(Privilege::All, "Any operation") - .with_abstract() - .with_supported_privilege( - SupportedPrivilege::new(Privilege::Read, "Read any object") - .with_supported_privilege( - SupportedPrivilege::new(Privilege::ReadAcl, "Read ACL") - .with_abstract(), - ) - .with_supported_privilege( - SupportedPrivilege::new( - Privilege::ReadCurrentUserPrivilegeSet, - "Read current user privilege set property", - ) - .with_abstract(), - ), - ) - .with_supported_privilege( - SupportedPrivilege::new(Privilege::Write, "Write any object") - .with_supported_privilege( - SupportedPrivilege::new(Privilege::WriteAcl, "Write ACL") - .with_abstract(), - ) - .with_supported_privilege(SupportedPrivilege::new( - Privilege::WriteProperties, - "Write properties", - )) - .with_supported_privilege(SupportedPrivilege::new( - Privilege::WriteContent, - "Write resource content", - )), - ) - .with_supported_privilege(SupportedPrivilege::new( - Privilege::Unlock, - "Unlock resource", - )), - ], - )])], - )]) - .to_string(), - // 015.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/papers/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::CurrentUserPrivilegeSet, - vec![Privilege::Read], - )])], - )]) - .to_string(), - // 016.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/papers/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::Acl, - vec![ - Ace::new( - Principal::Href(Href( - "http://www.example.com/acl/groups/maintainers".to_string(), - )), - GrantDeny::grant(vec![Privilege::Write]), - ), - Ace::new(Principal::All, GrantDeny::grant(vec![Privilege::Read])), - ], - )])], - )]) - .to_string(), - // 017.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/papers/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::AclRestrictions, - AclRestrictions::new() - .with_grant_only() - .with_required_principal(RequiredPrincipal::All), - )])], - )]) - .to_string(), - // 018.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/papers/", - vec![PropStat::new_list(vec![DavPropertyValue::new( - WebDavProperty::PrincipalCollectionSet, - vec![ - Href("http://www.example.com/acl/users/".to_string()), - Href("http://www.example.com/acl/groups/".to_string()), - ], - )])], - )]) - .to_string(), - // 019.xml - MultiStatus::new(vec![Response::new_propstat( - "http://www.example.com/top/container/", - vec![PropStat::new_list(vec![ - DavPropertyValue::new( - WebDavProperty::Owner, - vec![Href("http://www.example.com/users/gclemm".to_string())], - ), - DavPropertyValue::new( - WebDavProperty::SupportedPrivilegeSet, - vec![ - SupportedPrivilege::new(Privilege::All, "Any operation") - .with_abstract() - .with_supported_privilege(SupportedPrivilege::new( - Privilege::Read, - "Read any object", - )) - .with_supported_privilege( - SupportedPrivilege::new(Privilege::Write, "Write any object") - .with_abstract(), - ) - .with_supported_privilege(SupportedPrivilege::new( - Privilege::ReadAcl, - "Read the ACL", - )) - .with_supported_privilege(SupportedPrivilege::new( - Privilege::WriteAcl, - "Write the ACL", - )), - ], - ), - DavPropertyValue::new( - WebDavProperty::CurrentUserPrivilegeSet, - vec![Privilege::Read, Privilege::ReadAcl], - ), - DavPropertyValue::new( - WebDavProperty::Acl, - vec![ - Ace::new( - Principal::Href(Href( - "http://www.example.com/users/esedlar".to_string(), - )), - GrantDeny::grant(vec![ - Privilege::Read, - Privilege::Write, - Privilege::ReadAcl, - ]), - ), - Ace::new( - Principal::Href(Href( - "http://www.example.com/groups/mrktng".to_string(), - )), - GrantDeny::deny(vec![Privilege::Read]), - ), - Ace::new( - Principal::Property(List(vec![DavPropertyValue::new( - WebDavProperty::Owner, - DavValue::Null, - )])), - GrantDeny::grant(vec![Privilege::ReadAcl, Privilege::WriteAcl]), - ), - Ace::new(Principal::All, GrantDeny::grant(vec![Privilege::Read])) - .with_inherited("http://www.example.com/top"), - ], - ), - ])], - )]) - .to_string(), - // 020.xml - ScheduleResponse { - items: List(vec![ - ScheduleResponseItem { - recipient: Href("mailto:wilfredo@example.com".to_string()), - request_status: "2.0;Success".into(), - calendar_data: Some("BEGIN:VCALENDAR".to_string()), - }, - ScheduleResponseItem { - recipient: Href("mailto:bernard@example.net".to_string()), - request_status: "2.0;Success".into(), - calendar_data: Some("END:VCALENDAR".to_string()), - }, - ScheduleResponseItem { - recipient: Href("mailto:mike@example.org".to_string()), - request_status: "3.7;Invalid calendar user".into(), - calendar_data: None, - }, - ]), - } - .to_string(), - ] - .into_iter() - .enumerate() - { - let xml = - std::fs::read_to_string(format!("resources/responses/{:03}.xml", num + 1)).unwrap(); - let mut output_token = Tokenizer::new(test.as_bytes()); - let mut expected_token = Tokenizer::new(xml.as_bytes()); - let mut output_tokens = Vec::new(); - let mut expected_tokens = Vec::new(); - - for (tokens, tokenizer) in [ - (&mut output_tokens, &mut output_token), - (&mut expected_tokens, &mut expected_token), - ] { - while let Ok(token) = tokenizer.token() { - if token == Token::Eof { - break; - } - match (tokens.last_mut(), token) { - (Some(Token::Text(text)), Token::Text(new_text)) => { - *text = format!("{}{}", text, new_text).into(); - } - (_, element) => { - tokens.push(element.into_owned()); - } - } - } - } - - assert!(!output_tokens.is_empty()); - assert!(!expected_tokens.is_empty()); - assert_eq!(output_tokens.len(), expected_tokens.len()); - - for (output, expected) in output_tokens.iter().zip(expected_tokens.iter()) { - if output != expected { - eprintln!("{test}"); - } - assert_eq!(output, expected, "failed for {:03}.xml", num + 1); - } - } - } - - #[test] - fn escape_cdata() { - for (test, expected) in [ - ("", ""), - ("hello", ""), - ("hello world", ""), - ("", "]]>"), - ("&hello;", ""), - ("'hello'", ""), - ("\"hello\"", ""), - ("<>&'\"", "&'\"]]>"), - (">", "]]>"), - ("]]>]", "]]]>"), - ("]]>", "]]>"), - ("hello]]>world", "world]]>"), - ( - "hello]]>pure-evil", - "pure-evil]]>", - ), - ] { - let mut output = String::new(); - test.write_cdata_escaped_to(&mut output).unwrap(); - assert_eq!(output, expected, "failed for input: {test:?}"); - } - } -} diff --git a/crates/dav-proto/src/responses/property.rs b/crates/dav-proto/src/responses/property.rs index 44dffde6..ce78127e 100644 --- a/crates/dav-proto/src/responses/property.rs +++ b/crates/dav-proto/src/responses/property.rs @@ -5,26 +5,20 @@ */ use super::{XmlCdataEscape, XmlEscape}; -use crate::{ - responses::DeadPropertyFormat, - schema::{ - Namespace, Namespaces, - property::{ - ActiveLock, CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, - LockDiscovery, LockEntry, PrincipalProperty, Privilege, ReportSet, ResourceType, - Rfc1123DateTime, SupportedCollation, SupportedLock, WebDavProperty, - }, - request::DavPropertyValue, - response::{Ace, AclRestrictions, Href, List, PropResponse, SupportedPrivilege}, +use crate::schema::{ + Namespace, Namespaces, + property::{ + ActiveLock, CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, LockDiscovery, + LockEntry, PrincipalProperty, Privilege, ReportSet, ResourceType, Rfc1123DateTime, + SupportedCollation, SupportedLock, WebDavProperty, }, + request::DavPropertyValue, + response::{Ace, AclRestrictions, Href, List, PropResponse, SupportedPrivilege}, }; use calcard::icalendar::ICalendarComponentType; -use mail_parser::{ - DateTime, - parsers::fields::date::{DOW, MONTH}, -}; -use std::fmt::Display; -use types::dead_property::DeadProperty; +use mail_parser::{DOW, DateTime, MONTH}; +use std::fmt::{self, Display}; +use types::metadata::{EncodedDavValue, XmlValue}; impl Display for PropResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -38,14 +32,21 @@ impl Display for PropResponse { impl Display for DavPropertyValue { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let (name, attrs) = self.property.tag_name(); - - write!(f, "<{}", name)?; - - if let Some(attrs) = attrs { - write!(f, " {attrs}")?; + if let DavProperty::Dead(name) = &self.property { + return match &self.value { + DavValue::DeadEncoded(value) => value.view().write_property(name, f), + DavValue::Dead(value) => value + .to_encoded() + .map_err(|_| fmt::Error)? + .view() + .write_property(name, f), + _ => name.write_empty(f), + }; } + let name = self.property.tag_name(); + write!(f, "<{}", name)?; + if !matches!(self.value, DavValue::Null) { write!(f, ">{}", self.value, name) } else { @@ -100,7 +101,8 @@ impl Display for DavValue { DavValue::Privileges(v) => v.fmt(f), DavValue::Acl(v) => v.fmt(f), DavValue::AclRestrictions(v) => v.fmt(f), - DavValue::DeadProperty(v) => v.fmt(f), + DavValue::Opaque => f.write_str(""), + DavValue::Transparent => f.write_str(""), DavValue::SupportedAddressData => { write!( f, @@ -121,7 +123,11 @@ impl Display for DavValue { ) } DavValue::Response(v) => v.fmt(f), - DavValue::VCard(_) | DavValue::ICalendar(_) | DavValue::Null => Ok(()), + DavValue::VCard(_) + | DavValue::ICalendar(_) + | DavValue::Dead(_) + | DavValue::DeadEncoded(_) + | DavValue::Null => Ok(()), } } } @@ -147,81 +153,76 @@ impl DavValue { } impl DavProperty { - fn tag_name(&self) -> (&str, Option<&str>) { - ( - match self { - DavProperty::WebDav(prop) => match prop { - WebDavProperty::CreationDate => "D:creationdate", - WebDavProperty::DisplayName => "D:displayname", - WebDavProperty::GetContentLanguage => "D:getcontentlanguage", - WebDavProperty::GetContentLength => "D:getcontentlength", - WebDavProperty::GetContentType => "D:getcontenttype", - WebDavProperty::GetETag => "D:getetag", - WebDavProperty::GetLastModified => "D:getlastmodified", - WebDavProperty::ResourceType => "D:resourcetype", - WebDavProperty::LockDiscovery => "D:lockdiscovery", - WebDavProperty::SupportedLock => "D:supportedlock", - WebDavProperty::CurrentUserPrincipal => "D:current-user-principal", - WebDavProperty::QuotaAvailableBytes => "D:quota-available-bytes", - WebDavProperty::QuotaUsedBytes => "D:quota-used-bytes", - WebDavProperty::SupportedReportSet => "D:supported-report-set", - WebDavProperty::SyncToken => "D:sync-token", - WebDavProperty::Owner => "D:owner", - WebDavProperty::Group => "D:group", - WebDavProperty::SupportedPrivilegeSet => "D:supported-privilege-set", - WebDavProperty::CurrentUserPrivilegeSet => "D:current-user-privilege-set", - WebDavProperty::Acl => "D:acl", - WebDavProperty::AclRestrictions => "D:acl-restrictions", - WebDavProperty::InheritedAclSet => "D:inherited-acl-set", - WebDavProperty::PrincipalCollectionSet => "D:principal-collection-set", - WebDavProperty::GetCTag => "C:getctag", - }, - DavProperty::CardDav(prop) => match prop { - CardDavProperty::AddressbookDescription => "B:addressbook-description", - CardDavProperty::SupportedAddressData => "B:supported-address-data", - CardDavProperty::SupportedCollationSet => "B:supported-collation-set", - CardDavProperty::MaxResourceSize => "B:max-resource-size", - CardDavProperty::AddressData { .. } => "B:address-data", - }, - DavProperty::CalDav(prop) => match prop { - CalDavProperty::CalendarDescription => "A:calendar-description", - CalDavProperty::CalendarTimezone => "A:calendar-timezone", - CalDavProperty::SupportedCalendarComponentSet => { - "A:supported-calendar-component-set" - } - CalDavProperty::SupportedCalendarData => "A:supported-calendar-data", - CalDavProperty::SupportedCollationSet => "A:supported-collation-set", - CalDavProperty::MaxResourceSize => "A:max-resource-size", - CalDavProperty::MinDateTime => "A:min-date-time", - CalDavProperty::MaxDateTime => "A:max-date-time", - CalDavProperty::MaxInstances => "A:max-instances", - CalDavProperty::MaxAttendeesPerInstance => "A:max-attendees-per-instance", - CalDavProperty::CalendarData(_) => "A:calendar-data", - CalDavProperty::TimezoneServiceSet => "A:timezone-service-set", - CalDavProperty::TimezoneId => "A:calendar-timezone-id", - CalDavProperty::ScheduleDefaultCalendarURL => "A:schedule-default-calendar-URL", - CalDavProperty::ScheduleTag => "A:schedule-tag", - CalDavProperty::ScheduleCalendarTransp => "A:schedule-calendar-transp", - }, - DavProperty::Principal(prop) => match prop { - PrincipalProperty::AlternateURISet => "D:alternate-URI-set", - PrincipalProperty::PrincipalURL => "D:principal-URL", - PrincipalProperty::GroupMemberSet => "D:group-member-set", - PrincipalProperty::GroupMembership => "D:group-membership", - PrincipalProperty::CalendarHomeSet => "A:calendar-home-set", - PrincipalProperty::AddressbookHomeSet => "B:addressbook-home-set", - PrincipalProperty::PrincipalAddress => "B:principal-address", - PrincipalProperty::CalendarUserAddressSet => "A:calendar-user-address-set", - PrincipalProperty::CalendarUserType => "A:calendar-user-type", - PrincipalProperty::ScheduleInboxURL => "A:schedule-inbox-URL", - PrincipalProperty::ScheduleOutboxURL => "A:schedule-outbox-URL", - }, - DavProperty::DeadProperty(dead) => { - return (dead.name.as_str(), dead.attrs.as_deref()); - } + fn tag_name(&self) -> &str { + match self { + DavProperty::WebDav(prop) => match prop { + WebDavProperty::CreationDate => "D:creationdate", + WebDavProperty::DisplayName => "D:displayname", + WebDavProperty::GetContentLanguage => "D:getcontentlanguage", + WebDavProperty::GetContentLength => "D:getcontentlength", + WebDavProperty::GetContentType => "D:getcontenttype", + WebDavProperty::GetETag => "D:getetag", + WebDavProperty::GetLastModified => "D:getlastmodified", + WebDavProperty::ResourceType => "D:resourcetype", + WebDavProperty::LockDiscovery => "D:lockdiscovery", + WebDavProperty::SupportedLock => "D:supportedlock", + WebDavProperty::CurrentUserPrincipal => "D:current-user-principal", + WebDavProperty::QuotaAvailableBytes => "D:quota-available-bytes", + WebDavProperty::QuotaUsedBytes => "D:quota-used-bytes", + WebDavProperty::SupportedReportSet => "D:supported-report-set", + WebDavProperty::SyncToken => "D:sync-token", + WebDavProperty::Owner => "D:owner", + WebDavProperty::Group => "D:group", + WebDavProperty::SupportedPrivilegeSet => "D:supported-privilege-set", + WebDavProperty::CurrentUserPrivilegeSet => "D:current-user-privilege-set", + WebDavProperty::Acl => "D:acl", + WebDavProperty::AclRestrictions => "D:acl-restrictions", + WebDavProperty::InheritedAclSet => "D:inherited-acl-set", + WebDavProperty::PrincipalCollectionSet => "D:principal-collection-set", + WebDavProperty::GetCTag => "C:getctag", }, - None, - ) + DavProperty::CardDav(prop) => match prop { + CardDavProperty::AddressbookDescription => "B:addressbook-description", + CardDavProperty::SupportedAddressData => "B:supported-address-data", + CardDavProperty::SupportedCollationSet => "B:supported-collation-set", + CardDavProperty::MaxResourceSize => "B:max-resource-size", + CardDavProperty::AddressData { .. } => "B:address-data", + }, + DavProperty::CalDav(prop) => match prop { + CalDavProperty::CalendarDescription => "A:calendar-description", + CalDavProperty::CalendarTimezone => "A:calendar-timezone", + CalDavProperty::SupportedCalendarComponentSet => { + "A:supported-calendar-component-set" + } + CalDavProperty::SupportedCalendarData => "A:supported-calendar-data", + CalDavProperty::SupportedCollationSet => "A:supported-collation-set", + CalDavProperty::MaxResourceSize => "A:max-resource-size", + CalDavProperty::MinDateTime => "A:min-date-time", + CalDavProperty::MaxDateTime => "A:max-date-time", + CalDavProperty::MaxInstances => "A:max-instances", + CalDavProperty::MaxAttendeesPerInstance => "A:max-attendees-per-instance", + CalDavProperty::CalendarData(_) => "A:calendar-data", + CalDavProperty::TimezoneServiceSet => "A:timezone-service-set", + CalDavProperty::TimezoneId => "A:calendar-timezone-id", + CalDavProperty::ScheduleDefaultCalendarURL => "A:schedule-default-calendar-URL", + CalDavProperty::ScheduleTag => "A:schedule-tag", + CalDavProperty::ScheduleCalendarTransp => "A:schedule-calendar-transp", + }, + DavProperty::Principal(prop) => match prop { + PrincipalProperty::AlternateURISet => "D:alternate-URI-set", + PrincipalProperty::PrincipalURL => "D:principal-URL", + PrincipalProperty::GroupMemberSet => "D:group-member-set", + PrincipalProperty::GroupMembership => "D:group-membership", + PrincipalProperty::CalendarHomeSet => "A:calendar-home-set", + PrincipalProperty::AddressbookHomeSet => "B:addressbook-home-set", + PrincipalProperty::PrincipalAddress => "B:principal-address", + PrincipalProperty::CalendarUserAddressSet => "A:calendar-user-address-set", + PrincipalProperty::CalendarUserType => "A:calendar-user-type", + PrincipalProperty::ScheduleInboxURL => "A:schedule-inbox-URL", + PrincipalProperty::ScheduleOutboxURL => "A:schedule-outbox-URL", + }, + DavProperty::Dead(name) => name.name(), + } } pub fn namespace(&self) -> Namespace { @@ -246,7 +247,7 @@ impl DavProperty { impl AsRef for DavProperty { fn as_ref(&self) -> &str { - self.tag_name().0 + self.tag_name() } } @@ -274,11 +275,9 @@ impl Display for ReportSet { impl Display for DavProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let (name, attrs) = self.tag_name(); - if let Some(attrs) = attrs { - write!(f, "<{name} {attrs}/>") - } else { - write!(f, "<{name}/>") + match self { + DavProperty::Dead(name) => name.write_empty(f), + property => write!(f, "<{}/>", property.tag_name()), } } } @@ -417,9 +416,15 @@ impl From for DavValue { } } -impl From for DavValue { - fn from(v: DeadProperty) -> Self { - DavValue::DeadProperty(v) +impl From for DavValue { + fn from(v: EncodedDavValue) -> Self { + DavValue::DeadEncoded(v) + } +} + +impl From> for DavValue { + fn from(v: XmlValue<'static>) -> Self { + DavValue::Dead(Box::new(v)) } } diff --git a/crates/dav-proto/src/responses/propstat.rs b/crates/dav-proto/src/responses/propstat.rs index 4d1089dd..3d92b778 100644 --- a/crates/dav-proto/src/responses/propstat.rs +++ b/crates/dav-proto/src/responses/propstat.rs @@ -33,7 +33,7 @@ impl Display for Prop { } impl PropStat { - #[cfg(test)] + #[cfg(all(test, stalwart_tests))] pub(crate) fn new(prop: impl Into) -> Self { PropStat { prop: Prop(List(vec![prop.into()])), diff --git a/crates/dav-proto/src/schema/collation.rs b/crates/dav-proto/src/schema/collation.rs new file mode 100644 index 00000000..ca013c9c --- /dev/null +++ b/crates/dav-proto/src/schema/collation.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::{Namespace, request::TextMatch}; +use std::{borrow::Cow, convert::identity}; +use types::collation::unicode_casemap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub enum Collation { + AsciiCasemap, + Octet, + UnicodeCasemap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub enum MatchType { + Equals, + Contains, + StartsWith, + EndsWith, +} + +impl Collation { + pub fn default_for(namespace: Namespace) -> Self { + if namespace == Namespace::CardDav { + Collation::UnicodeCasemap + } else { + Collation::AsciiCasemap + } + } + + pub fn try_parse(s: &str) -> Option { + hashify::fnc_map!(s.as_bytes(), + "i;ascii-casemap" => Some(Collation::AsciiCasemap), + "i;octet" => Some(Collation::Octet), + "i;unicode-casemap" => Some(Collation::UnicodeCasemap), + _ => None, + ) + } + + pub fn as_str(&self) -> &'static str { + match self { + Collation::AsciiCasemap => "i;ascii-casemap", + Collation::Octet => "i;octet", + Collation::UnicodeCasemap => "i;unicode-casemap", + } + } + + fn prepare<'x>(&self, text: &'x str) -> Cow<'x, str> { + match self { + Collation::Octet => Cow::Borrowed(text), + Collation::AsciiCasemap => { + if text.bytes().any(|ch| ch.is_ascii_lowercase()) { + Cow::Owned(text.to_ascii_uppercase()) + } else { + Cow::Borrowed(text) + } + } + Collation::UnicodeCasemap => { + if text.is_ascii() { + Collation::AsciiCasemap.prepare(text) + } else { + let mut folded = String::with_capacity(text.len()); + unicode_casemap(text, &mut folded); + Cow::Owned(folded) + } + } + } + } +} + +impl MatchType { + pub fn try_parse(s: &str) -> Option { + hashify::fnc_map!(s.as_bytes(), + "equals" => Some(MatchType::Equals), + "contains" => Some(MatchType::Contains), + "starts-with" => Some(MatchType::StartsWith), + "ends-with" => Some(MatchType::EndsWith), + _ => None, + ) + } + + fn matches(&self, haystack: &[u8], needle: &[u8], fold: Fold) -> bool { + match fold { + Fold::Identity => self.matches_with(haystack, needle, identity, |haystack, needle| { + memchr::memmem::find(haystack, needle).is_some() + }), + Fold::AsciiUppercase => self.matches_with( + haystack, + needle, + |byte| byte.to_ascii_uppercase(), + ascii_uppercase_contains, + ), + } + } + + fn matches_with( + &self, + haystack: &[u8], + needle: &[u8], + fold: impl Fn(u8) -> u8, + prefiltered_contains: impl Fn(&[u8], &[u8]) -> bool, + ) -> bool { + let is_equal = |(h, n): (&u8, &u8)| fold(*h) == *n; + match self { + MatchType::Equals => { + haystack.len() == needle.len() && haystack.iter().zip(needle).all(is_equal) + } + MatchType::StartsWith => { + haystack.len() >= needle.len() && haystack.iter().zip(needle).all(is_equal) + } + MatchType::EndsWith => { + haystack.len() >= needle.len() + && haystack.iter().rev().zip(needle.iter().rev()).all(is_equal) + } + MatchType::Contains if needle.is_empty() => true, + MatchType::Contains if haystack.len() < PREFILTER_MIN_LEN => haystack + .windows(needle.len()) + .any(|window| window.iter().zip(needle).all(is_equal)), + MatchType::Contains => prefiltered_contains(haystack, needle), + } + } +} + +const PREFILTER_MIN_LEN: usize = 64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fold { + Identity, + AsciiUppercase, +} + +fn ascii_uppercase_contains(haystack: &[u8], needle: &[u8]) -> bool { + let Some(first) = needle.first() else { + return true; + }; + memchr::memchr2_iter( + first.to_ascii_uppercase(), + first.to_ascii_lowercase(), + haystack, + ) + .any(|start| { + haystack + .get(start..start + needle.len()) + .is_some_and(|window| { + window + .iter() + .zip(needle) + .all(|(h, n)| h.to_ascii_uppercase() == *n) + }) + }) +} + +impl TextMatch { + pub fn new(value: String, match_type: MatchType, collation: Collation, negate: bool) -> Self { + TextMatch { + needle: collation.prepare(&value).into_owned(), + match_type, + value, + collation, + negate, + } + } + + pub fn is_match(&self, text: &str) -> bool { + self.is_match_with(text, &mut String::new()) + } + + fn is_match_with(&self, text: &str, folded: &mut String) -> bool { + let needle = self.needle.as_bytes(); + match self.collation { + Collation::Octet => self + .match_type + .matches(text.as_bytes(), needle, Fold::Identity), + Collation::AsciiCasemap => { + self.match_type + .matches(text.as_bytes(), needle, Fold::AsciiUppercase) + } + Collation::UnicodeCasemap if text.is_ascii() => { + self.needle.is_ascii() + && self + .match_type + .matches(text.as_bytes(), needle, Fold::AsciiUppercase) + } + Collation::UnicodeCasemap => { + folded.clear(); + unicode_casemap(text, folded); + self.match_type + .matches(folded.as_bytes(), needle, Fold::Identity) + } + } + } + + pub fn is_match_any>(&self, mut values: impl Iterator) -> bool { + let mut folded = String::new(); + values.any(|value| self.is_match_with(value.as_ref(), &mut folded)) != self.negate + } +} + diff --git a/crates/dav-proto/src/schema/mod.rs b/crates/dav-proto/src/schema/mod.rs index 06f996e7..7704a758 100644 --- a/crates/dav-proto/src/schema/mod.rs +++ b/crates/dav-proto/src/schema/mod.rs @@ -6,20 +6,28 @@ use std::borrow::Cow; -use request::TextMatch; +pub mod collation; pub mod property; pub mod request; pub mod response; +pub use collation::{Collation, MatchType}; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct NamedElement { pub ns: Namespace, pub element: Element, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(u8)] pub enum Namespace { Dav, @@ -29,7 +37,10 @@ pub enum Namespace { } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Namespaces { pub(crate) cal: bool, pub(crate) card: bool, @@ -49,12 +60,16 @@ impl Namespaces { impl Namespace { pub fn try_parse(value: &[u8]) -> Option { - hashify::tiny_map!(value, - "DAV:" => Namespace::Dav, - "urn:ietf:params:xml:ns:caldav" => Namespace::CalDav, - "urn:ietf:params:xml:ns:carddav" => Namespace::CardDav, - "http://calendarserver.org/ns/" => Namespace::CalendarServer, - "http://calendarserver.org/ns" => Namespace::CalendarServer + Namespace::try_parse_uri(value).map(|(namespace, _)| *namespace) + } + + pub fn try_parse_uri(value: &[u8]) -> Option<&'static (Namespace, &'static str)> { + hashify::map!(value, (Namespace, &'static str), + "DAV:" => (Namespace::Dav, "DAV:"), + "urn:ietf:params:xml:ns:caldav" => (Namespace::CalDav, "urn:ietf:params:xml:ns:caldav"), + "urn:ietf:params:xml:ns:carddav" => (Namespace::CardDav, "urn:ietf:params:xml:ns:carddav"), + "http://calendarserver.org/ns/" => (Namespace::CalendarServer, "http://calendarserver.org/ns/"), + "http://calendarserver.org/ns" => (Namespace::CalendarServer, "http://calendarserver.org/ns") ) } @@ -84,7 +99,10 @@ impl AsRef for Namespace { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum Element { Abstract, Ace, @@ -1346,55 +1364,6 @@ impl AttributeValue for String { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -pub enum Collation { - AsciiNumeric, - AsciiCasemap, - Octet, - UnicodeCasemap, -} - -impl Collation { - pub fn try_parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), - "i;ascii-numeric" => Collation::AsciiNumeric, - "i;ascii-casemap" => Collation::AsciiCasemap, - "i;octet" => Collation::Octet, - "i;unicode-casemap" => Collation::UnicodeCasemap, - ) - } - - pub fn as_str(&self) -> &'static str { - match self { - Collation::AsciiNumeric => "i;ascii-numeric", - Collation::AsciiCasemap => "i;ascii-casemap", - Collation::Octet => "i;octet", - Collation::UnicodeCasemap => "i;unicode-casemap", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -pub enum MatchType { - Equals, - Contains, - StartsWith, - EndsWith, -} - -impl MatchType { - pub fn try_parse(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), - "equals" => MatchType::Equals, - "contains" => MatchType::Contains, - "starts-with" => MatchType::StartsWith, - "ends-with" => MatchType::EndsWith, - ) - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum XsiType { String, @@ -1421,26 +1390,27 @@ pub enum XsiType { impl XsiType { fn from_str(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), - "xs:string" => XsiType::String, - "xs:boolean" => XsiType::Boolean, - "xs:decimal" => XsiType::Decimal, - "xs:float" => XsiType::Float, - "xs:double" => XsiType::Double, - "xs:duration" => XsiType::Duration, - "xs:dateTime" => XsiType::DateTime, - "xs:time" => XsiType::Time, - "xs:date" => XsiType::Date, - "xs:gYearMonth" => XsiType::GYearMonth, - "xs:gYear" => XsiType::GYear, - "xs:gMonthDay" => XsiType::GMonthDay, - "xs:gDay" => XsiType::GDay, - "xs:gMonth" => XsiType::GMonth, - "xs:hexBinary" => XsiType::HexBinary, - "xs:base64Binary" => XsiType::Base64Binary, - "xs:anyURI" => XsiType::AnyUri, - "xs:QName" => XsiType::QName, - "xs:NOTATION" => XsiType::Notation, + hashify::fnc_map!(s.as_bytes(), + "xs:string" => Some(XsiType::String), + "xs:boolean" => Some(XsiType::Boolean), + "xs:decimal" => Some(XsiType::Decimal), + "xs:float" => Some(XsiType::Float), + "xs:double" => Some(XsiType::Double), + "xs:duration" => Some(XsiType::Duration), + "xs:dateTime" => Some(XsiType::DateTime), + "xs:time" => Some(XsiType::Time), + "xs:date" => Some(XsiType::Date), + "xs:gYearMonth" => Some(XsiType::GYearMonth), + "xs:gYear" => Some(XsiType::GYear), + "xs:gMonthDay" => Some(XsiType::GMonthDay), + "xs:gDay" => Some(XsiType::GDay), + "xs:gMonth" => Some(XsiType::GMonth), + "xs:hexBinary" => Some(XsiType::HexBinary), + "xs:base64Binary" => Some(XsiType::Base64Binary), + "xs:anyURI" => Some(XsiType::AnyUri), + "xs:QName" => Some(XsiType::QName), + "xs:NOTATION" => Some(XsiType::Notation), + _ => None, ) } } @@ -1449,36 +1419,10 @@ struct YesNo; impl YesNo { fn from_str(s: &str) -> Option { - hashify::tiny_map!(s.as_bytes(), + hashify::map!(s.as_bytes(), bool, "yes" => true, "no" => false, ) - } -} - -impl TextMatch { - pub fn matches(&self, text: &str) -> bool { - match self.collation { - Collation::Octet => { - (match self.match_type { - MatchType::Equals => text == self.value, - MatchType::Contains => text.contains(&self.value), - MatchType::StartsWith => text.starts_with(&self.value), - MatchType::EndsWith => text.ends_with(&self.value), - }) ^ self.negate - } - _ => { - (match self.match_type { - MatchType::Equals => text.to_lowercase() == self.value.to_lowercase(), - MatchType::Contains => text.to_lowercase().contains(&self.value.to_lowercase()), - MatchType::StartsWith => { - text.to_lowercase().starts_with(&self.value.to_lowercase()) - } - MatchType::EndsWith => { - text.to_lowercase().ends_with(&self.value.to_lowercase()) - } - }) ^ self.negate - } - } + .copied() } } diff --git a/crates/dav-proto/src/schema/property.rs b/crates/dav-proto/src/schema/property.rs index 28b20bf2..7853d258 100644 --- a/crates/dav-proto/src/schema/property.rs +++ b/crates/dav-proto/src/schema/property.rs @@ -16,23 +16,29 @@ use calcard::{ }; use types::{ TimeRange, - dead_property::{DeadElementTag, DeadProperty}, + metadata::{EncodedDavValue, XmlName, XmlValue}, }; #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum DavProperty { WebDav(WebDavProperty), CardDav(CardDavProperty), CalDav(CalDavProperty), Principal(PrincipalProperty), - DeadProperty(DeadElementTag), + Dead(XmlName<'static>), } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum WebDavProperty { CreationDate, DisplayName, @@ -65,8 +71,11 @@ pub enum WebDavProperty { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum CardDavProperty { AddressbookDescription, SupportedAddressData, @@ -74,13 +83,16 @@ pub enum CardDavProperty { MaxResourceSize, AddressData { properties: Vec, - #[cfg_attr(test, serde(skip))] + #[cfg_attr(all(test, stalwart_tests), serde(skip))] version: Option, }, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct CardDavPropertyName { pub group: Option, pub name: VCardProperty, @@ -88,8 +100,11 @@ pub struct CardDavPropertyName { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum CalDavProperty { CalendarDescription, CalendarTimezone, @@ -110,8 +125,11 @@ pub enum CalDavProperty { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum PrincipalProperty { AlternateURISet, PrincipalURL, @@ -127,7 +145,10 @@ pub enum PrincipalProperty { } #[derive(Debug, Default, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct CalendarData { pub properties: Vec, pub expand: Option, @@ -136,7 +157,10 @@ pub struct CalendarData { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct CalDavPropertyName { pub component: Option, pub name: Option, @@ -144,12 +168,18 @@ pub struct CalDavPropertyName { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] #[repr(transparent)] pub struct Rfc1123DateTime(pub(crate) i64); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum DavValue { Timestamp(i64), Rfc1123Date(Rfc1123DateTime), @@ -170,14 +200,21 @@ pub enum DavValue { Acl(List), AclRestrictions(AclRestrictions), Response(Box), - DeadProperty(DeadProperty), + Dead(Box>), + #[cfg_attr(all(test, stalwart_tests), serde(skip))] + DeadEncoded(EncodedDavValue), SupportedAddressData, SupportedCalendarData, + Opaque, + Transparent, Null, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum ReportSet { SyncCollection, ExpandProperty, @@ -193,18 +230,27 @@ pub enum ReportSet { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct Comp(pub ICalendarComponentType); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct SupportedCollation { pub collation: Collation, pub namespace: Namespace, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum ResourceType { Collection, Principal, @@ -215,48 +261,70 @@ pub enum ResourceType { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct LockDiscovery(pub List); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct ActiveLock { pub lock_scope: LockScope, pub lock_type: LockType, pub depth: Depth, - pub owner: Option, + #[cfg_attr(all(test, stalwart_tests), serde(skip))] + pub owner: Option, pub timeout: Timeout, pub lock_token: Option, pub lock_root: Href, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct SupportedLock(pub List); #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct LockEntry { pub lock_scope: LockScope, pub lock_type: LockType, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum LockType { Write, Other, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum LockScope { Exclusive, Shared, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub enum Privilege { Read, Write, @@ -392,7 +460,7 @@ impl DavProperty { | DavProperty::WebDav(WebDavProperty::GetContentLanguage) | DavProperty::WebDav(WebDavProperty::GetContentLength) | DavProperty::WebDav(WebDavProperty::GetContentType) - | DavProperty::DeadProperty(_) + | DavProperty::Dead(_) ) } } diff --git a/crates/dav-proto/src/schema/request.rs b/crates/dav-proto/src/schema/request.rs index 878c03e6..ab759fbf 100644 --- a/crates/dav-proto/src/schema/request.rs +++ b/crates/dav-proto/src/schema/request.rs @@ -14,14 +14,14 @@ use calcard::{ icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::{VCardParameterName, VCardProperty}, }; -use types::{ - TimeRange, - dead_property::{ArchivedDeadProperty, ArchivedDeadPropertyTag, DeadElementTag, DeadProperty}, -}; +use types::{TimeRange, metadata::XmlValue}; #[derive(Debug, Clone, PartialEq, Eq, Default)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum PropFind { #[default] PropName, @@ -30,38 +30,61 @@ pub enum PropFind { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct PropertyUpdate { - pub set: Vec, - pub remove: Vec, - pub set_first: bool, + pub ops: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub enum PropertyUpdateOp { + Set(DavPropertyValue), + Remove(DavProperty), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct DavPropertyValue { pub property: DavProperty, pub value: DavValue, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct MkCol { pub is_mkcalendar: bool, pub props: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct LockInfo { pub lock_scope: LockScope, pub lock_type: LockType, - pub owner: Option, + pub owner: Option>, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type"))] pub enum Report { AddressbookQuery(AddressbookQuery), AddressbookMultiGet(MultiGet), @@ -77,45 +100,62 @@ pub enum Report { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct ExpandProperty { pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct ExpandPropertyItem { pub property: DavProperty, pub depth: u32, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct AddressbookQuery { pub properties: PropFind, - pub filters: Vec>, + pub filter: CardFilter, pub limit: Option, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct VCardPropertyWithGroup { pub name: VCardProperty, pub group: Option, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct CalendarQuery { pub properties: PropFind, - pub filters: - Vec, ICalendarProperty, ICalendarParameterName>>, + pub filter: Option, pub timezone: Timezone, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type"))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] pub enum Timezone { Name(String), Id(String), @@ -123,20 +163,29 @@ pub enum Timezone { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct FreeBusyQuery { pub range: Option, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct MultiGet { pub properties: PropFind, pub hrefs: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] pub struct SyncCollection { pub sync_token: Option, pub properties: PropFind, @@ -144,93 +193,207 @@ pub struct SyncCollection { pub limit: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type"))] -pub enum Filter { +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub enum FilterTest { + #[default] AnyOf, AllOf, - Component { - comp: A, - op: FilterOp, - }, - Property { - comp: A, - prop: B, - op: FilterOp, - }, - Parameter { - comp: A, - prop: B, - param: C, - op: FilterOp, - }, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type", content = "data"))] -pub enum FilterOp { - Exists, - Undefined, +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] +pub enum Presence { + IsNotDefined, + IsDefined(T), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CompFilter { + pub name: ICalendarComponentType, + pub test: Presence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CompFilterMatch { + pub time_range: Option, + pub prop_filters: Vec, + pub comp_filters: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CalendarPropFilter { + pub name: ICalendarProperty, + pub test: Presence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct CalendarPropMatch { + pub value: Option, + pub param_filters: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(all(test, stalwart_tests), serde(tag = "type", content = "data"))] +pub enum PropValueMatch { TimeRange(TimeRange), - TextMatch(TextMatch), + Text(TextMatch), } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, serde(tag = "type"))] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + all(test, stalwart_tests), + derive(serde::Serialize, serde::Deserialize) +)] +pub struct ParamFilter